[
  {
    "path": ".devops/cublas.Dockerfile",
    "content": "ARG UBUNTU_VERSION=22.04\n\n# This needs to generally match the container host's environment.\nARG CUDA_VERSION=11.7.1\n\n# Target the CUDA build image\nARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}\n\nFROM ${BASE_CUDA_DEV_CONTAINER} as build\n\n# Unless otherwise specified, we make a fat build.\nARG CUDA_DOCKER_ARCH=all\n\nRUN apt-get update && \\\n    apt-get install -y build-essential git cmake libsdl2-dev wget git\n\nWORKDIR /app\n\nCOPY . .\n\n# Set nvcc architecture\nENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}\n# Enable cuBLAS\nENV GGML_CUDA=1\n\nRUN make base.en\n\nENTRYPOINT [\"/app/main\"]\n"
  },
  {
    "path": ".devops/main-cuda.Dockerfile",
    "content": "ARG UBUNTU_VERSION=22.04\n# This needs to generally match the container host's environment.\nARG CUDA_VERSION=13.0.0\n# Target the CUDA build image\nARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}\n# Target the CUDA runtime image\nARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}\n\nFROM ${BASE_CUDA_DEV_CONTAINER} AS build\nWORKDIR /app\n\n# Unless otherwise specified, we make a fat build.\nARG CUDA_DOCKER_ARCH=all\n# Set nvcc architecture\nENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}\n\nRUN apt-get update && \\\n    apt-get install -y build-essential libsdl2-dev wget cmake git \\\n    && apt-get clean \\\n    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\n# Ref: https://stackoverflow.com/a/53464012\nENV CUDA_MAIN_VERSION=13.0\nENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH\n\nCOPY .. .\n# Enable cuBLAS\nRUN make base.en CMAKE_ARGS=\"-DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES='75;80;86;90'\"\n\nRUN find /app/build -name \"*.o\" -delete && \\\n    find /app/build -name \"*.a\" -delete && \\\n    rm -rf /app/build/CMakeFiles && \\\n    rm -rf /app/build/cmake_install.cmake && \\\n    rm -rf /app/build/_deps\n\nFROM ${BASE_CUDA_RUN_CONTAINER} AS runtime\nENV CUDA_MAIN_VERSION=13.0\nENV LD_LIBRARY_PATH /usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y curl ffmpeg wget cmake git \\\n  && apt-get clean \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY --from=build /app /app\nRUN du -sh /app/*\nRUN find /app -type f -size +100M\nENV PATH=/app/build/bin:$PATH\nENTRYPOINT [ \"bash\", \"-c\" ]\n"
  },
  {
    "path": ".devops/main-intel.Dockerfile",
    "content": "ARG ONEAPI_VERSION=2025.1.1-0-devel-ubuntu24.04\n\nFROM intel/oneapi-basekit:$ONEAPI_VERSION AS build\nWORKDIR /app\n\nRUN apt-get update && \\\n    apt-get install -y build-essential libsdl2-dev wget cmake git \\\n    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY .. .\n# Enable SYCL\nARG GGML_SYCL_F16=OFF\nRUN if [ \"${GGML_SYCL_F16}\" = \"ON\" ]; then \\\n        echo \"GGML_SYCL_F16 is set\" \\\n        && export OPT_SYCL_F16=\"-DGGML_SYCL_F16=ON\"; \\\n    fi && \\\n    make base.en CMAKE_ARGS=\"-DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ${OPT_SYCL_F16}\"\n\nFROM intel/oneapi-basekit:$ONEAPI_VERSION AS runtime\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y curl ffmpeg libsdl2-dev wget cmake git \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY --from=build /app /app\nENV PATH=/app/build/bin:$PATH\nENTRYPOINT [ \"bash\", \"-c\" ]\n"
  },
  {
    "path": ".devops/main-musa.Dockerfile",
    "content": "ARG UBUNTU_VERSION=22.04\n# This needs to generally match the container host's environment.\nARG MUSA_VERSION=rc4.2.0\n# Target the MUSA build image\nARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64\n# Target the MUSA runtime image\nARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64\n\nFROM ${BASE_MUSA_DEV_CONTAINER} AS build\nWORKDIR /app\n\nRUN apt-get update && \\\n    apt-get install -y build-essential libsdl2-dev wget cmake git && \\\n    apt-get clean && \\\n    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/*\n\nCOPY .. .\n# Enable muBLAS\nRUN make base.en CMAKE_ARGS=\"-DGGML_MUSA=1\"\n\nRUN find /app/build -name \"*.o\" -delete && \\\n    find /app/build -name \"*.a\" -delete && \\\n    rm -rf /app/build/CMakeFiles && \\\n    rm -rf /app/build/cmake_install.cmake && \\\n    rm -rf /app/build/_deps\n\nFROM ${BASE_MUSA_RUN_CONTAINER} AS runtime\nWORKDIR /app\n\nRUN apt-get update && \\\n    apt-get install -y curl ffmpeg wget cmake git && \\\n    apt-get clean && \\\n    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/*\n\nCOPY --from=build /app/build/bin /app/build/bin\nCOPY --from=build /app/samples /app/samples\nCOPY --from=build /app/models /app/models\n\nENV PATH=/app/build/bin:$PATH\nENTRYPOINT [ \"bash\", \"-c\" ]\n"
  },
  {
    "path": ".devops/main-vulkan.Dockerfile",
    "content": "FROM ubuntu:24.04 AS build\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y build-essential wget cmake git libvulkan-dev glslc \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY .. .\nRUN make base.en CMAKE_ARGS=\"-DGGML_VULKAN=1\"\n\nFROM ubuntu:24.04 AS runtime\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y curl ffmpeg libsdl2-dev wget cmake git libvulkan1 mesa-vulkan-drivers \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY --from=build /app /app\nENV PATH=/app/build/bin:$PATH\nENTRYPOINT [ \"bash\", \"-c\" ]\n"
  },
  {
    "path": ".devops/main.Dockerfile",
    "content": "FROM ubuntu:22.04 AS build\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y build-essential wget cmake git \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY .. .\nRUN make base.en\n\nFROM ubuntu:22.04 AS runtime\nWORKDIR /app\n\nRUN apt-get update && \\\n  apt-get install -y curl ffmpeg libsdl2-dev wget cmake git \\\n  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*\n\nCOPY --from=build /app /app\nENV PATH=/app/build/bin:$PATH\nENTRYPOINT [ \"bash\", \"-c\" ]\n"
  },
  {
    "path": ".dockerignore",
    "content": "build*/\n.github/\n.devops/"
  },
  {
    "path": ".github/workflows/bindings-go.yml",
    "content": "name: Bindings Tests (Go)\non:\n  push:\n    paths:\n      - bindings/go/**\n      - whisper.h\n  pull_request:\n    paths:\n      - bindings/go/**\n      - whisper.h\n\njobs:\n  ubuntu-22:\n    runs-on: ubuntu-22.04\n    steps:\n      - uses: actions/setup-go@v6\n        with:\n          go-version: '^1.23'\n      - uses: actions/checkout@v6\n      - run: |\n          cd bindings/go\n          make test\n"
  },
  {
    "path": ".github/workflows/bindings-ruby.yml",
    "content": "name: Bindings Tests (Ruby)\n\non:\n  push:\n    branches:\n      - master\n  pull_request:\n    types: [opened, synchronize, reopened]\n\njobs:\n  ubuntu-22:\n    runs-on: ubuntu-22.04\n    defaults:\n      run:\n        working-directory: bindings/ruby\n    steps:\n      - uses: ruby/setup-ruby@v1\n        with:\n          ruby-version: '3.2'\n      - uses: actions/checkout@v6\n      - run: rake test\n"
  },
  {
    "path": ".github/workflows/build.yml",
    "content": "name: CI\n\non:\n  push:\n    branches:\n      - master\n    tags:\n      - 'v*'\n    paths: ['.github/workflows/build.yml',\n            '**/CMakeLists.txt',\n            '**/Makefile',\n            '**/*.mk',\n            '**/*.cmake',\n            '**/*.in',\n            '**/*.h',\n            '**/*.hpp',\n            '**/*.c',\n            '**/*.cpp',\n            '**/*.cu',\n            '**/*.cuh',\n            '**/*.cl',\n            '**/*.swift',\n            '**/*.m',\n            '**/*.mm',\n            '**/*.metal',\n            '**/*.comp',\n            '**/*.java']\n\n  pull_request:\n    types: [opened, synchronize, reopened]\n  workflow_dispatch:\n    inputs:\n      create_release:\n        description: 'Create new release'\n        required: true\n        type: boolean\n      pre_release_tag:\n        description: 'Pre-release tag name'\n        required: false\n        type: string\n      run_type:\n        description: 'Workflow type to run'\n        required: true\n        type: choice\n        options:\n          - full-ci\n          - release-only\n\nconcurrency:\n  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}\n  cancel-in-progress: true\n\npermissions:\n  contents: write  # for creating release\n\nenv:\n  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}\n  ubuntu_image: \"ubuntu:22.04\"\n  VCPKG_BINARY_SOURCES: \"clear;x-gha,readwrite\"\n\njobs:\n  determine-tag:\n    runs-on: ubuntu-latest\n    outputs:\n      tag_name: ${{ steps.tag.outputs.name }}\n      should_release: ${{ steps.tag.outputs.should_release }}\n\n    steps:\n      - name: Checkout with full history\n        uses: actions/checkout@v6\n        with:\n          fetch-depth: 0\n\n      - name: Determine tag name\n        id: tag\n        shell: bash\n        run: |\n          BUILD_NUMBER=$(git rev-list --count HEAD)\n          SHORT_HASH=$(git rev-parse --short=7 HEAD)\n          CUSTOM_TAG=\"${{ github.event.inputs.pre_release_tag }}\"\n          SHOULD_RELEASE=\"false\"\n\n          echo \"Raw values:\"\n          echo \"BUILD_NUMBER: $BUILD_NUMBER\"\n          echo \"SHORT_HASH: $SHORT_HASH\"\n          echo \"BRANCH_NAME: ${{ env.BRANCH_NAME }}\"\n          echo \"CUSTOM_TAG: $CUSTOM_TAG\"\n\n          if [[ \"${{ github.ref_type }}\" == \"tag\" ]]; then\n            echo \"Using pushed tag name\"\n            TAG_NAME=\"${{ github.ref_name }}\"\n            SHOULD_RELEASE=\"true\"\n          elif [[ -n \"$CUSTOM_TAG\" ]]; then\n            echo \"Using custom tag\"\n            TAG_NAME=\"${CUSTOM_TAG}\"\n            SHOULD_RELEASE=\"true\"\n          elif [[ \"${{ github.event.inputs.create_release }}\" == \"true\" ]]; then\n            echo \"Manual release requested\"\n            SHOULD_RELEASE=\"true\"\n            TAG_NAME=\"b${BUILD_NUMBER}\"\n          elif [[ \"${{ env.BRANCH_NAME }}\" == \"master\" ]]; then\n            echo \"Using master branch format\"\n            TAG_NAME=\"b${BUILD_NUMBER}\"\n            SHOULD_RELEASE=\"false\"\n          else\n            echo \"Using non-master branch format\"\n            SAFE_NAME=$(echo \"${{ env.BRANCH_NAME }}\" | tr '/' '-')\n            TAG_NAME=\"${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}\"\n            SHOULD_RELEASE=\"false\"\n          fi\n\n          echo \"Final tag name: $TAG_NAME\"\n          echo \"Should release: $SHOULD_RELEASE\"\n          echo \"name=$TAG_NAME\" >> $GITHUB_OUTPUT\n          echo \"should_release=$SHOULD_RELEASE\" >> $GITHUB_OUTPUT\n\n\n  ubuntu-22:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        arch: [linux/amd64, linux/ppc64le]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential libsdl2-dev cmake git\n            cmake -B build\n            cmake --build build --config Release -j $(nproc)'\n\n  ubuntu-22-arm64:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        arch: [linux/arm64]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential libsdl2-dev cmake git\n            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a\n            cmake --build build --config Release -j $(nproc)'\n\n  ubuntu-22-arm-v7:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        arch: [linux/arm/v7]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential libsdl2-dev cmake git\n            cmake -B build -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp\n            cmake --build build --config Release -j $(nproc)'\n\n  macOS-latest:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: macOS-latest\n\n    strategy:\n      matrix:\n        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: hendrikmuhs/ccache-action@v1.2.16\n        with:\n          key: macOS-latest-swift\n          evict-old-files: 1d\n\n      - name: Dependencies\n        run: |\n          brew update\n          cmake --version\n          brew install sdl2\n\n      - name: Build\n        run: |\n          sysctl -a\n          cmake -B build -G Xcode \\\n            -DGGML_METAL_USE_BF16=ON \\\n            -DGGML_METAL_EMBED_LIBRARY=ON \\\n            -DWHISPER_BUILD_EXAMPLES=OFF \\\n            -DWHISPER_BUILD_TESTS=OFF \\\n            -DWHISPER_BUILD_SERVER=OFF \\\n            -DCMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\"\n          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)\n\n\n#  freeBSD-latest:\n#    runs-on: macos-13\n#\n#    steps:\n#      - name: Clone\n#        uses: actions/checkout@v6\n#\n#      - name: Build\n#        uses: cross-platform-actions/action@v0.27.0\n#        with:\n#          operating_system: freebsd\n#          version: '14.2'\n#          run: |\n#            sudo pkg update\n#            sudo pkg install -y gmake sdl2 cmake git\n#            cmake -B build\n#            cmake --build build --config Release\n\n  ubuntu-22-gcc:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        build: [Debug, Release]\n        arch: [linux/amd64, linux/ppc64le]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential cmake libsdl2-dev git\n            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }}\n            make\n            ctest -L gh --output-on-failure'\n\n  ubuntu-22-gcc-arm64:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        build: [Debug, Release]\n        arch: [linux/arm64]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential cmake libsdl2-dev git\n            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8-a\n            make\n            ctest -L gh --output-on-failure'\n\n  ubuntu-22-gcc-arm-v7:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        build: [Debug, Release]\n        arch: [linux/arm/v7]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential cmake libsdl2-dev git\n            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv7-a+fp\n            make\n            ctest -L gh --output-on-failure'\n\n  ubuntu-22-clang:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        build: [Debug, Release]\n        #arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]\n        # TODO: arm/v7 disabled due to clang bug\n        #       https://github.com/ggerganov/whisper.cpp/actions/runs/9657764109/job/26637633042?pr=2256#step:4:1990\n        arch: [linux/amd64, linux/arm64, linux/ppc64le]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y clang build-essential cmake libsdl2-dev git\n            cmake . -DWHISPER_SDL2=ON -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang\n            make\n            ctest -L gh --output-on-failure'\n\n  ubuntu-22-gcc-sanitized:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        sanitizer: [ADDRESS, THREAD, UNDEFINED]\n        arch: [linux/amd64]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n\n      - name: Build ${{ matrix.arch }}\n        run: |\n          docker run --platform ${{ matrix.arch }} --rm \\\n            -v ${{ github.workspace }}:/workspace \\\n            -w /workspace ${{ env.ubuntu_image }} /bin/sh -c '\n            set -e\n            export DEBIAN_FRONTEND=noninteractive\n            sed -i \"s|archive.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n            sed -i \"s|security.ubuntu.com|mirrors.kernel.org|g\" /etc/apt/sources.list\n\n            apt update\n            apt install -y build-essential cmake git\n            cmake . -DCMAKE_BUILD_TYPE=Debug \\\n              -DWHISPER_SANITIZE_${{ matrix.sanitizer }}=ON \\\n              -DGGML_OPENMP=OFF\n            make\n            ctest -L gh --output-on-failure'\n\n  ubuntu-22-cmake-sycl:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        dwhisper_sycl: [ON]\n        dcmake_c_compiler: [icx]\n        dcmake_cxx_compiler: [icpx]\n        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]\n\n    continue-on-error: true\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: add oneAPI to apt\n        shell: bash\n        run: |\n          cd /tmp\n          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          sudo add-apt-repository \"deb https://apt.repos.intel.com/oneapi all main\"\n\n      - name: install oneAPI dpcpp compiler\n        shell: bash\n        run: |\n          sudo apt update\n          sudo apt install intel-oneapi-compiler-dpcpp-cpp git\n\n      - name: install oneAPI MKL library\n        shell: bash\n        run: |\n          sudo apt install intel-oneapi-mkl-devel git\n\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Build\n        id: cmake_build\n        run: |\n          source /opt/intel/oneapi/setvars.sh\n          mkdir build\n          cd build\n          cmake -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ..\n          cmake --build . --config Release -j $(nproc)\n\n  ubuntu-22-cmake-sycl-fp16:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      fail-fast: false\n      matrix:\n        dwhisper_sycl: [ON]\n        dcmake_c_compiler: [icx]\n        dcmake_cxx_compiler: [icpx]\n        arch: [linux/amd64, linux/arm64, linux/arm/v7, linux/ppc64le]\n\n    continue-on-error: true\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: add oneAPI to apt\n        shell: bash\n        run: |\n          cd /tmp\n          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\n          sudo add-apt-repository \"deb https://apt.repos.intel.com/oneapi all main\"\n\n      - name: install oneAPI dpcpp compiler\n        shell: bash\n        run: |\n          sudo apt update\n          sudo apt install intel-oneapi-compiler-dpcpp-cpp git\n\n      - name: install oneAPI MKL library\n        shell: bash\n        run: |\n          sudo apt install intel-oneapi-mkl-devel\n\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Build\n        id: cmake_build\n        run: |\n          source /opt/intel/oneapi/setvars.sh\n          mkdir build\n          cd build\n          cmake -DGGML_SYCL_F16=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ..\n          cmake --build . --config Release -j $(nproc)\n\n  windows-msys2:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: windows-latest\n\n    strategy:\n      fail-fast: false\n      matrix:\n        include:\n          - { sys: UCRT64,  env: ucrt-x86_64,  build: Release }\n          - { sys: CLANG64, env: clang-x86_64, build: Release }\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Setup ${{ matrix.sys }}\n        uses: msys2/setup-msys2@v2\n        with:\n          update: true\n          msystem: ${{matrix.sys}}\n          install: >-\n            base-devel\n            git\n            mingw-w64-${{matrix.env}}-toolchain\n            mingw-w64-${{matrix.env}}-cmake\n            mingw-w64-${{matrix.env}}-SDL2\n            mingw-w64-${{matrix.env}}-openblas\n\n      - name: Build using CMake\n        shell: msys2 {0}\n        run: |\n            cmake -B build -DWHISPER_SDL2=ON\n            cmake --build build --config ${{ matrix.build }} -j $(nproc)\n\n      - name: Clean after building using CMake\n        shell: msys2 {0}\n        run: |\n            rm -rf build\n\n      - name: Build using CMake w/ OpenBLAS\n        shell: msys2 {0}\n        run: |\n            cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS\n            cmake --build build --config ${{ matrix.build }} -j $(nproc)\n\n  windows:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: windows-latest\n    needs: determine-tag\n\n    strategy:\n      matrix:\n        build: [Release]\n        arch: [Win32, x64]\n        sdl2: [ON]\n        include:\n          - arch: Win32\n            s2arc: x86\n            jnaPath: win32-x86\n          - arch: x64\n            s2arc: x64\n            jnaPath: win32-x86-64\n          - sdl2: ON\n            s2ver: 2.28.5\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Add msbuild to PATH\n        uses: microsoft/setup-msbuild@v2\n\n      - name: Fetch SDL2 and set SDL2_DIR\n        if: matrix.sdl2 == 'ON'\n        run: |\n          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip\n          7z x sdl2.zip\n          echo \"SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake\" >> $env:GITHUB_ENV\n\n      - name: Configure\n        run: >\n          cmake -S . -B ./build -A ${{ matrix.arch }}\n          -DCMAKE_BUILD_TYPE=${{ matrix.build }}\n          -DBUILD_SHARED_LIBS=ON\n          -DWHISPER_SDL2=${{ matrix.sdl2 }}\n\n      - name: Build\n        run: |\n          cd ./build\n          msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }}\n\n      - name: Copy SDL2.dll\n        if: matrix.sdl2 == 'ON'\n        run: copy \"$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll\" build/bin/${{ matrix.build }}\n\n      - name: Upload SDL2.dll\n        if: matrix.sdl2 == 'ON'\n        uses: actions/upload-artifact@v6\n        with:\n          name: ${{ matrix.s2arc }}_SDL2.dll\n          path: build/bin/${{ matrix.build }}/SDL2.dll\n\n      - name: Upload whisper dll\n        uses: actions/upload-artifact@v6\n        with:\n          name: whisper_${{ matrix.arch }}.dll\n          path: build/bin/${{ matrix.build }}/whisper.dll\n\n      - name: Upload ggml dll\n        uses: actions/upload-artifact@v6\n        with:\n          name: ggml_${{ matrix.arch }}.dll\n          path: build/bin/${{ matrix.build }}/ggml.dll\n\n      - name: Upload ggml base dll\n        uses: actions/upload-artifact@v6\n        with:\n          name: ggml_base_${{ matrix.arch }}.dll\n          path: build/bin/${{ matrix.build }}/ggml-base.dll\n\n      - name: Upload ggml cpu dll\n        uses: actions/upload-artifact@v6\n        with:\n          name: ggml_cpu_${{ matrix.arch }}.dll\n          path: build/bin/${{ matrix.build }}/ggml-cpu.dll\n\n      - name: Pack bin artifacts\n        shell: pwsh\n        run: |\n              Compress-Archive -Path \"build/bin/${{ matrix.build }}\" -DestinationPath \"whisper-bin-${{ matrix.arch }}.zip\"\n\n      - name: Upload binaries\n        if: matrix.sdl2 == 'ON' && ${{ needs.determine-tag.outputs.should_release }}\n        uses: actions/upload-artifact@v6\n        with:\n          name: whisper-bin-${{ matrix.arch }}.zip\n          path: whisper-bin-${{ matrix.arch }}.zip\n\n  windows-blas:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: windows-latest\n\n    strategy:\n      matrix:\n        build: [Release]\n        arch: [Win32, x64]\n        blas: [ON]\n        sdl2: [ON]\n        blasver: [0.3.29]\n        include:\n          - arch: Win32\n            s2arc: x86\n            blasfile: x86\n          - arch: x64\n            s2arc: x64\n            blasfile: x64_64\n          - sdl2: ON\n            s2ver: 2.28.5\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Export GitHub Actions cache environment variables\n        uses: actions/github-script@v8\n        with:\n          script: |\n            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');\n            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');\n\n      - name: Add msbuild to PATH\n        uses: microsoft/setup-msbuild@v2\n\n      - name: Install OpenBLAS and pkgconfiglite\n        if: matrix.blas == 'ON'\n        run: |\n          Invoke-WebRequest \"https://github.com/OpenMathLib/OpenBLAS/releases/download/v${{matrix.blasver}}/OpenBLAS-${{matrix.blasver}}_${{matrix.blasfile}}.zip\" -OutFile \"OpenBLAS-${{matrix.blasver}}.zip\"\n          Expand-Archive \"OpenBLAS-${{matrix.blasver}}.zip\" -DestinationPath \"OpenBLAS-${{matrix.blasver}}\"\n          choco install pkgconfiglite\n\n      - name: Fetch SDL2 and set SDL2_DIR\n        if: matrix.sdl2 == 'ON'\n        run: |\n          C:/msys64/usr/bin/wget.exe -qO sdl2.zip https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.s2ver }}/SDL2-devel-${{ matrix.s2ver }}-VC.zip\n          7z x sdl2.zip\n          echo \"SDL2_DIR=$env:GITHUB_WORKSPACE/SDL2-${{ matrix.s2ver }}/cmake\" >> $env:GITHUB_ENV\n\n      - name: Configure\n        run: >\n          cmake -S . -B ./build -A ${{ matrix.arch }}\n          -DCMAKE_TOOLCHAIN_FILE=\"$env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake\"\n          -DCMAKE_BUILD_TYPE=${{ matrix.build }}\n          -DGGML_BLAS=${{ matrix.blas }}\n          -DGGML_BLAS_VENDOR=OpenBLAS\n          -DBLAS_LIBRARIES=\"$env:GITHUB_WORKSPACE/OpenBLAS-${{matrix.blasver}}/lib/libopenblas.lib\"\n          -DBLAS_INCLUDE_DIRS=\"$env:GITHUB_WORKSPACE/OpenBLAS-${{matrix.blasver}}/include\"\n          -DWHISPER_SDL2=${{ matrix.sdl2 }}\n\n      - name: Build\n        run: |\n          cd ./build\n          msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }}\n\n      - name: Copy openblas.dll\n        if: matrix.blas == 'ON'\n        run: copy \"$env:GITHUB_WORKSPACE/OpenBLAS-${{matrix.blasver}}/bin/libopenblas.dll\" build/bin/${{ matrix.build }}\n\n      - name: Copy SDL2.dll\n        if: matrix.sdl2 == 'ON'\n        run: copy \"$env:SDL2_DIR/../lib/${{ matrix.s2arc }}/SDL2.dll\" build/bin/${{ matrix.build }}\n\n      - name: Pack bin artifacts\n        shell: pwsh\n        run: |\n              Compress-Archive -Path \"build/bin/${{ matrix.build }}\" -DestinationPath \"whisper-blas-bin-${{ matrix.arch }}.zip\"\n\n      - name: Upload binaries\n        if: matrix.blas == 'ON' && matrix.sdl2 == 'ON' && ${{ needs.determine-tag.outputs.should_release }}\n        uses: actions/upload-artifact@v6\n        with:\n          name: whisper-blas-bin-${{ matrix.arch }}.zip\n          path: whisper-blas-bin-${{ matrix.arch }}.zip\n\n  windows-cublas:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: windows-2022\n    needs: determine-tag\n    strategy:\n      fail-fast: false\n      matrix:\n        build: [Release]\n        arch: [x64]\n        cublas: [ON]\n        sdl2: [ON]\n        cuda-toolkit: [12.4.0, 11.8.0]\n        include:\n          - arch: x64\n            sdl2: ON\n            sdl2_ver: 2.28.5\n    steps:\n      - name: Clone repository\n        uses: actions/checkout@v6\n\n      - name: Install Ninja\n        id: install_ninja\n        run: |\n          choco install ninja\n\n      - name: Install ccache\n        uses: hendrikmuhs/ccache-action@v1.2.16\n        with:\n          key: ${{ github.job }}-${{ matrix.cuda-toolkit }}-${{ matrix.build }}\n          variant: sccache\n          evict-old-files: 5d\n\n      - name: Install Cuda Toolkit 11.8.0\n        if: ${{ matrix.cuda-toolkit == '11.8.0' }}\n        run: |\n          $CUDA_VERSION = ${{ matrix.cuda-toolkit }}\n          $CUDA_TOOLKIT_DIR = \"C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v$CUDA_VERSION\"\n          $CUDA_DOWNLOAD = \"https://developer.download.nvidia.com/compute/cuda/redist\"\n\n          # Components versions\n          $CUDART_VER = \"11.8.89\"\n          $NVCC_VER   = \"11.8.89\"\n          $NVRTC_VER  = \"11.8.89\"\n          $CUBLAS_VER = \"11.8.1.74\"\n          $NVTX_VER   = \"11.8.86\"\n          $VS_VER     = \"11.8.86\"\n          $NVPROF_VER = \"11.8.87\"\n          $CCCL_VER   = \"11.8.89\"\n\n          # Create the directory where the CUDA Toolkit will be installed\n          mkdir -p $CUDA_TOOLKIT_DIR\n\n          # Install unzip to extract the downloaded files\n          choco install unzip -y\n\n          # Download all the required components\n          curl -O \"$CUDA_DOWNLOAD/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-${CUDART_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-${NVCC_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/libcublas/windows-x86_64/libcublas-windows-x86_64-${CUBLAS_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-${NVTX_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-${VS_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip\"\n\n          # Extract all the downloaded files to the CUDA Toolkit directory\n          unzip '*.zip' -d $CUDA_TOOLKIT_DIR\n\n          # Copy all the extracted files to the main CUDA Toolkit directory\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvcc-windows-x86_64-${NVCC_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\libcublas-windows-x86_64-${CUBLAS_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvtx-windows-x86_64-${NVTX_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_cccl-windows-x86_64-${CCCL_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\visual_studio_integration-windows-x86_64-${VS_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n\n          # Visual Studio integration\n          xcopy \"$CUDA_TOOLKIT_DIR\\visual_studio_integration-windows-x86_64-${VS_VER}-archive\\visual_studio_integration\\MSBuildExtensions\\*\" \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\MSBuild\\Microsoft\\VC\\v170\\BuildCustomizations\" /E /I /H /Y\n\n          # Set environment variables\n          echo \"$CUDA_TOOLKIT_DIR\\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n          echo \"$CUDA_TOOLKIT_DIR\\libnvvp\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n          echo \"CUDA_PATH=$CUDA_TOOLKIT_DIR\" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8\n          echo \"CUDA_PATH_V11_8=$CUDA_TOOLKIT_DIR\" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8\n\n      - name: Install Cuda Toolkit 12.4.0\n        if: ${{ matrix.cuda-toolkit == '12.4.0' }}\n        run: |\n          $CUDA_VERSION = ${{ matrix.cuda-toolkit }}\n          $CUDA_TOOLKIT_DIR = \"C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v$CUDA_VERSION\"\n          $CUDA_DOWNLOAD = \"https://developer.download.nvidia.com/compute/cuda/redist\"\n\n          # Components versions\n          $CUDART_VER   = \"12.4.127\"\n          $NVCC_VER     = \"12.4.131\"\n          $NVRTC_VER    = \"12.4.127\"\n          $CUBLAS_VER   = \"12.4.5.8\"\n          $NVTX_VER     = \"12.4.127\"\n          $PROFILER_VER = \"12.4.127\"\n          $VS_VER       = \"12.4.127\"\n          $NVPROF_VER   = \"12.4.128\"\n          $CCCL_VER     = \"12.4.127\"\n\n          # Create the directory where the CUDA Toolkit will be installed\n          mkdir -p $CUDA_TOOLKIT_DIR\n\n          # Install unzip to extract the downloaded files\n          choco install unzip -y\n\n          # Download all the required components\n          curl -O \"$CUDA_DOWNLOAD/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-${CUDART_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-${NVCC_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/libcublas/windows-x86_64/libcublas-windows-x86_64-${CUBLAS_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-${NVTX_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-${PROFILER_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-${VS_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive.zip\"\n          curl -O \"$CUDA_DOWNLOAD/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-${CCCL_VER}-archive.zip\"\n\n          # Extract all the downloaded files to the CUDA Toolkit directory\n          unzip -q '*.zip' -d $CUDA_TOOLKIT_DIR\n\n          # Copy all the extracted files to the main CUDA Toolkit directory\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_cudart-windows-x86_64-${CUDART_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvcc-windows-x86_64-${NVCC_VER}-archive\\*\"     \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvrtc-windows-x86_64-${NVRTC_VER}-archive\\*\"   \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\libcublas-windows-x86_64-${CUBLAS_VER}-archive\\*\"   \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvtx-windows-x86_64-${NVTX_VER}-archive\\*\"     \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_nvprof-windows-x86_64-${NVPROF_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_cccl-windows-x86_64-${CCCL_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\cuda_profiler_api-windows-x86_64-${PROFILER_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n          xcopy \"$CUDA_TOOLKIT_DIR\\visual_studio_integration-windows-x86_64-${VS_VER}-archive\\*\" \"$CUDA_TOOLKIT_DIR\" /E /I /H /Y\n\n          # Visual Studio integration\n          xcopy \"$CUDA_TOOLKIT_DIR\\visual_studio_integration-windows-x86_64-${VS_VER}-archive\\visual_studio_integration\\MSBuildExtensions\\*\" \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\MSBuild\\Microsoft\\VC\\v170\\BuildCustomizations\" /E /I /H /Y\n\n          # Set environment variables\n          echo \"$CUDA_TOOLKIT_DIR\\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n          echo \"$CUDA_TOOLKIT_DIR\\libnvvp\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append\n          echo \"CUDA_PATH=$CUDA_TOOLKIT_DIR\" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8\n          echo \"CUDA_PATH_V12_2=$CUDA_TOOLKIT_DIR\" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8\n\n      - name: Add msbuild to PATH\n        uses: microsoft/setup-msbuild@v2\n\n      - name: Install 7-Zip\n        run: choco install 7zip -y\n\n      - name: Fetch SDL2 and set SDL2_DIR\n        if: matrix.sdl2 == 'ON'\n        run: |\n          Invoke-WebRequest -Uri https://github.com/libsdl-org/SDL/releases/download/release-${{ matrix.sdl2_ver }}/SDL2-devel-${{ matrix.sdl2_ver }}-VC.zip -OutFile sdl2.zip\n          7z x sdl2.zip\n          echo \"SDL2_DIR=${{ github.workspace }}\\SDL2-${{ matrix.sdl2_ver }}\\cmake\" | Out-File -FilePath $env:GITHUB_ENV -Append\n          echo \"${{ github.workspace }}\\SDL2-${{ matrix.sdl2_ver }}\\cmake\" > SDL2_PATH.txt\n\n      - name: Install cmake\n        run: choco install cmake\n\n      - name: Build Project\n        shell: cmd\n        run: |\n          call \"C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat\"\n          cmake --version\n          where cmake\n          if \"${{ matrix.cuda-toolkit }}\" == \"11.8.0\" (\n            set CUDA_FLAGS=-allow-unsupported-compiler -D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH -D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR\n          ) else (\n            set CUDA_FLAGS=\n          )\n          cmake -S . -B build -G \"Ninja Multi-Config\" ^\n            -DCMAKE_BUILD_TYPE=${{ matrix.build }} ^\n            -DGGML_CUDA=${{ matrix.cublas }} ^\n            -DWHISPER_SDL2=${{ matrix.sdl2 }} ^\n            -DSDL2_DIR=\"%SDL2_DIR%\" ^\n            -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ^\n            -DCMAKE_CUDA_FLAGS=\"%CUDA_FLAGS%\"\n          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1\n          cmake --build build --config ${{ matrix.build }} -j %NUMBER_OF_PROCESSORS%\n\n      - name: Check sccache status after build\n        run: |\n          sccache --show-stats\n\n      - name: Copy CUDA DLLs\n        run: |\n          Get-ChildItem \"$env:CUDA_PATH\\bin\\\" -Filter \"*.dll\" |\n          Copy-Item -Destination \"build/bin/${{ matrix.build }}\"\n\n      - name: Copy SDL2.dll\n        if: matrix.sdl2 == 'ON'\n        run: copy \"$env:SDL2_DIR/../lib/${{ matrix.arch }}/SDL2.dll\" build/bin/${{ matrix.build }}\n\n      - name: Pack bin artifacts\n        shell: pwsh\n        run: |\n              Compress-Archive -Path \"build/bin/${{ matrix.build }}\" -DestinationPath \"whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}.zip\"\n\n      - name: Upload binaries\n        if: ${{ needs.determine-tag.outputs.should_release }}\n        uses: actions/upload-artifact@v6\n        with:\n          name: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}.zip\n          path: whisper-cublas-${{ matrix.cuda-toolkit }}-bin-${{ matrix.arch }}.zip\n\n  emscripten:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    strategy:\n      matrix:\n        build: [Release]\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Setup emsdk\n        uses: mymindstorm/setup-emsdk@v14\n\n      - name: Verify\n        run: emcc -v\n\n      - name: Build\n        run: |\n          emcmake cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }}\n          make\n\n  ios-xcode-build:\n    runs-on: macos-latest\n    needs: determine-tag\n\n    strategy:\n      matrix:\n        build: [Release]\n\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v6\n\n      - name: Configure\n        run: |\n          cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin\n          mkdir models/ggml-base.en-encoder.mlmodelc\n\n      - name: Build\n        id: cmake_build\n        run: |\n          sysctl -a\n          mkdir build\n          cd build\n          cmake -G Xcode .. \\\n            -DGGML_METAL_USE_BF16=ON \\\n            -DGGML_METAL_EMBED_LIBRARY=ON \\\n            -DWHISPER_BUILD_EXAMPLES=OFF \\\n            -DWHISPER_BUILD_TESTS=OFF \\\n            -DWHISPER_BUILD_SERVER=OFF \\\n            -DCMAKE_SYSTEM_NAME=iOS \\\n            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \\\n            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml\n          cmake --build . --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO\n\n      - name: xcodebuild for swift package\n        id: xcodebuild\n        run: |\n          ./build-xcframework.sh\n\n      - name: Build objc example\n        run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGN_IDENTITY=\"\" CODE_SIGNING_REQUIRED=NO FRAMEWORK_FOLDER_PATH=./build-ios build\n\n      - name: Build swiftui example\n        run: xcodebuild -project examples/whisper.swiftui/whisper.swiftui.xcodeproj -scheme WhisperCppDemo -configuration ${{ matrix.build }} -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build\n\n      - name: Pack artifacts\n        id: pack_artifacts\n        run: |\n          zip --symlinks -r whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework.zip build-apple/whisper.xcframework\n\n      - name: Upload artifacts\n        if: ${{ needs.determine-tag.outputs.should_release }}\n        uses: actions/upload-artifact@v6\n        with:\n          path: whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework.zip\n          name: whisper-${{ needs.determine-tag.outputs.tag_name }}-xcframework.zip\n\n  android:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n        with:\n          path: whisper\n\n      - name: Install Java\n        uses: actions/setup-java@v5\n        with:\n          distribution: zulu\n          java-version: 21\n\n      - name: Setup Android SDK\n        uses: android-actions/setup-android@v3\n\n      - name: Build\n        run: |\n          cd whisper/examples/whisper.android\n          ./gradlew assembleRelease --no-daemon\n\n      - name: Build with external ggml\n        run: |\n          export PATH_TO_GGML=$PWD/ggml\n          cd whisper/examples/whisper.android\n          ./gradlew assembleRelease --no-daemon\n\n  android_java:\n    runs-on: ubuntu-22.04\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: set up JDK 11\n        uses: actions/setup-java@v5\n        with:\n          java-version: '11'\n          distribution: 'temurin'\n          cache: gradle\n\n      - name: Setup Android SDK\n        uses: android-actions/setup-android@v3\n        with:\n          cmdline-tools-version: 9.0\n\n      - name: Build\n        run: |\n          cd examples/whisper.android.java\n          chmod +x ./gradlew\n          ./gradlew assembleRelease\n\n  bindings-java:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    needs: ['windows']\n    runs-on: windows-latest\n    steps:\n      - uses: actions/checkout@v6\n\n      - name: Install Java\n        uses: actions/setup-java@v5\n        with:\n          distribution: zulu\n          java-version: 20\n\n      - name: Download Whisper Windows lib\n        uses: actions/download-artifact@v7\n        with:\n          name: whisper_x64.dll\n\n      - name: Download GGML Windows lib\n        uses: actions/download-artifact@v7\n        with:\n          name: ggml_x64.dll\n\n      - name: Download GGML Base Windows lib\n        uses: actions/download-artifact@v7\n        with:\n          name: ggml_base_x64.dll\n\n      - name: Download GGML CPU Windows lib\n        uses: actions/download-artifact@v7\n        with:\n          name: ggml_cpu_x64.dll\n\n      - name: Download SDL2.dll\n        uses: actions/download-artifact@v7\n        with:\n          name: x64_SDL2.dll\n\n      - name: List downloaded files\n        shell: pwsh\n        run: |\n          Get-ChildItem -Path \".\" -Recurse -Filter \"*.dll\"\n\n      - name: Move DLL to correct location\n        shell: pwsh\n        run: |\n          New-Item -Path \"build\\bin\\Release\" -ItemType Directory -Force\n\n          Copy-Item -Path \"whisper.dll\" -Destination \"build\\bin\\Release\\whisper.dll\" -Force\n          Write-Host \"Copied whisper.dll to build\\bin\\Release\\whisper.dll directory\"\n\n          Copy-Item -Path \"ggml.dll\" -Destination \"build\\bin\\Release\\ggml.dll\" -Force\n          Write-Host \"Copied ggml.dll to build\\bin\\Release\\ggml.dll directory\"\n\n          Copy-Item -Path \"ggml-base.dll\" -Destination \"build\\bin\\Release\\ggml-base.dll\" -Force\n          Write-Host \"Copied ggml-base.dll to build\\bin\\Release\\ggml-base.dll directory\"\n\n          Copy-Item -Path \"ggml-cpu.dll\" -Destination \"build\\bin\\Release\\ggml-cpu.dll\" -Force\n          Write-Host \"Copied ggml-cpu.dll to build\\bin\\Release\\ggml-cpu.dll directory\"\n\n          Copy-Item -Path \"SDL2.dll\" -Destination \"build\\bin\\Release\\SDL2.dll\" -Force\n          Write-Host \"Copied SDL2.dll to build\\bin\\Release\\SDL2.dll directory\"\n\n      - name: List build release files\n        shell: pwsh\n        run: |\n          Get-ChildItem -Path \"build\\Release\" -Recurse -Filter \"*.dll\"\n\n      - name: Build\n        run: |\n          models\\download-ggml-model.cmd tiny.en models/\n          cd bindings/java\n          chmod +x ./gradlew\n          ./gradlew build --info\n\n      - name: Pack jar artifacts\n        shell: pwsh\n        run: |\n              Compress-Archive -Path \"bindings/java/build/libs/whispercpp-*.jar\" -DestinationPath \"whispercpp.jar.zip\"\n\n      - name: Upload jar\n        uses: actions/upload-artifact@v6\n        with:\n          name: whispercpp.jar.zip\n          path: whispercpp.jar.zip\n\n#      - name: Publish package\n#        if: ${{ github.ref == 'refs/heads/master' }}\n#        uses: gradle/gradle-build-action@v2.4.2\n#        with:\n#          arguments: publish\n#          build-root-directory: bindings/java\n#        env:\n#          MAVEN_USERNAME: ${{ secrets.JIRA_USER }}\n#          MAVEN_PASSWORD: ${{ secrets.JIRA_PASS }}\n#          PGP_SECRET: ${{ secrets.GPG_PRIVATE_KEY }}\n#          PGP_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}\n\n  quantize:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-22.04\n\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Test quantize\n        run: |\n          ./models/download-ggml-model.sh tiny.en\n          cmake -B build\n          cmake --build build --config Release\n          ./build/bin/whisper-quantize models/ggml-tiny.en.bin models/ggml-tiny.en-q4_0.bin q4_0\n\n  release:\n    if: ${{ github.event.inputs.create_release == 'true' || github.event.inputs.pre_release_tag != '' || startsWith(github.ref, 'refs/tags/v') }}\n\n    runs-on: ubuntu-latest\n\n    needs:\n      - determine-tag\n      - ios-xcode-build\n      - windows\n      - windows-blas\n      - windows-cublas\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n        with:\n          fetch-depth: 0\n\n      - name: ccache\n        uses: hendrikmuhs/ccache-action@v1.2.16\n        with:\n          key: release\n          evict-old-files: 1d\n\n      # Downloads all the artifacts from the previous jobs\n      - name: Download artifacts\n        id: download-artifact\n        uses: actions/download-artifact@v7\n        with:\n          path: ./artifact\n\n      - name: Move artifacts\n        id: move_artifacts\n        run: mkdir -p ./artifact/release && mv ./artifact/*/*.zip ./artifact/release\n\n      - name: Create release\n        id: create_release\n        uses: ggml-org/action-create-release@v1\n        env:\n          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n        with:\n          tag_name: ${{ needs.determine-tag.outputs.tag_name }}\n          prerelease: ${{ github.event.inputs.pre_release_tag != '' }}\n          draft: true\n\n      - name: Upload release\n        id: upload_release\n        uses: actions/github-script@v3\n        with:\n          github-token: ${{secrets.GITHUB_TOKEN}}\n          script: |\n            const path = require('path');\n            const fs = require('fs');\n            const release_id = '${{ steps.create_release.outputs.id }}';\n            for (let file of await fs.readdirSync('./artifact/release')) {\n              if (path.extname(file) === '.zip') {\n                console.log('uploadReleaseAsset', file);\n                await github.repos.uploadReleaseAsset({\n                  owner: context.repo.owner,\n                  repo: context.repo.repo,\n                  release_id: release_id,\n                  name: file,\n                  data: await fs.readFileSync(`./artifact/release/${file}`)\n                });\n              }\n            }\n\n  coreml-base-en:\n    if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') ||\n            github.event.inputs.create_release == 'true' ||\n            github.event.inputs.pre_release_tag != '' ||\n            startsWith(github.ref, 'refs/tags/v') }}\n    runs-on: macos-latest\n    needs: determine-tag\n\n    steps:\n      - name: Checkout code\n        uses: actions/checkout@v6\n\n      - name: Set environment variables\n        id: set_vars\n        run: |\n          echo \"MODEL_NAME=base.en\" >> $GITHUB_ENV\n          echo \"GEN_MODEL_NAME=whisper-${{ needs.determine-tag.outputs.tag_name }}-ggml-base.en-encoder.mlmodelc\" >> $GITHUB_ENV\n\n      - name: Download model\n        run: |\n          ./models/download-ggml-model.sh ${{ env.MODEL_NAME }}\n\n      - name: Generate CoreML model\n        run: |\n          python3.11 -m venv venv\n          source venv/bin/activate\n          pip install ane_transformers openai-whisper coremltools\n          ./models/generate-coreml-model.sh ${{ env.MODEL_NAME }}\n\n  vad:\n    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' ||\n            github.event.inputs.run_type == 'full-ci' }}\n    runs-on: ubuntu-latest\n\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v6\n\n      - name: Build\n        shell: bash\n        run: |\n          cmake -B build\n          cmake --build build --config Release\n\n      - name: Test\n        shell: bash\n        run: |\n          ctest -R ^test-vad$ --test-dir build --output-on-failure -VV\n\n# TODO: simplify the following workflows using a matrix\n  ggml-ci-x64-cpu-low-perf:\n    runs-on: ubuntu-22.04\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: ggml-org/ccache-action@v1.2.16\n        with:\n          key: ggml-ci-x64-cpu-low-perf\n          evict-old-files: 1d\n\n      - name: Dependencies\n        id: depends\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential libcurl4-openssl-dev\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n  ggml-ci-arm64-cpu-low-perf:\n    runs-on: ubuntu-22.04-arm\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: ggml-org/ccache-action@v1.2.16\n        with:\n          key: ggml-ci-arm64-cpu-low-perf\n          evict-old-files: 1d\n\n      - name: Dependencies\n        id: depends\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential libcurl4-openssl-dev\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n  ggml-ci-x64-cpu-high-perf:\n    runs-on: ubuntu-22.04\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: ggml-org/ccache-action@v1.2.16\n        with:\n          key: ggml-ci-x64-cpu-high-perf\n          evict-old-files: 1d\n\n      - name: Dependencies\n        id: depends\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential libcurl4-openssl-dev\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n  ggml-ci-arm64-cpu-high-perf:\n    runs-on: ubuntu-22.04-arm\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: ggml-org/ccache-action@v1.2.16\n        with:\n          key: ggml-ci-arm64-cpu-high-perf\n          evict-old-files: 1d\n\n      - name: Dependencies\n        id: depends\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential libcurl4-openssl-dev\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n  ggml-ci-arm64-cpu-high-perf-sve:\n    runs-on: ubuntu-22.04-arm\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: ccache\n        uses: ggml-org/ccache-action@v1.2.16\n        with:\n          key: ggml-ci-arm64-cpu-high-perf-sve\n          evict-old-files: 1d\n\n      - name: Dependencies\n        id: depends\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential libcurl4-openssl-dev\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n  ggml-ci-x64-nvidia-cuda:\n    runs-on: [self-hosted, Linux, mnt-root, NVIDIA]\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          nvidia-smi\n          GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp\n\n  ggml-ci-x64-nvidia-vulkan-cm:\n    runs-on: [self-hosted, Linux, mnt-root, NVIDIA]\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          vulkaninfo --summary\n          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp\n\n  ggml-ci-x64-nvidia-vulkan-cm2:\n    runs-on: [self-hosted, Linux, mnt-root, NVIDIA, COOPMAT2]\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          vulkaninfo --summary\n          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp\n\n  #ggml-ci-x64-cpu-amx:\n  #  runs-on: [self-hosted, Linux, X64, CPU, AMX]\n\n  #  steps:\n  #    - name: Clone\n  #      id: checkout\n  #      uses: actions/checkout@v6\n\n  #    - name: Test\n  #      id: ggml-ci\n  #      run: |\n  #        bash ./ci/run.sh ~/results/whisper.cpp /mnt/whisper.cpp\n\n  ggml-ci-mac-metal:\n    runs-on: [self-hosted, macOS, ARM64]\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp\n\n  ggml-ci-mac-vulkan:\n    runs-on: [self-hosted, macOS, ARM64]\n\n    steps:\n      - name: Clone\n        id: checkout\n        uses: actions/checkout@v6\n\n      - name: Test\n        id: ggml-ci\n        run: |\n          vulkaninfo --summary\n          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp\n"
  },
  {
    "path": ".github/workflows/docker.yml",
    "content": "name: Publish Docker image\n\non:\n  pull_request:\n  push:\n    branches:\n      - master\n\njobs:\n  push_to_registry:\n    name: Push Docker image to Docker Hub\n    if: github.event.pull_request.draft == false\n\n    runs-on: ubuntu-22.04\n    env:\n      COMMIT_SHA: ${{ github.sha }}\n    strategy:\n      fail-fast: false\n      matrix:\n        config:\n          - { tag: \"main\", dockerfile: \".devops/main.Dockerfile\", platform: \"linux/amd64\" }\n          - { tag: \"main-musa\", dockerfile: \".devops/main-musa.Dockerfile\", platform: \"linux/amd64\" }\n          - { tag: \"main-intel\", dockerfile: \".devops/main-intel.Dockerfile\", platform: \"linux/amd64\" }\n          - { tag: \"main-cuda\", dockerfile: \".devops/main-cuda.Dockerfile\", platform: \"linux/amd64\" }\n          - { tag: \"main-vulkan\", dockerfile: \".devops/main-vulkan.Dockerfile\", platform: \"linux/amd64\" }\n\n    steps:\n      - name: Check out the repo\n        uses: actions/checkout@v6\n\n      - name: Set up QEMU\n        uses: docker/setup-qemu-action@v3\n        with:\n          image: tonistiigi/binfmt:qemu-v7.0.0-28\n\n      - name: Set up Docker Buildx\n        uses: docker/setup-buildx-action@v3\n\n      - name: Log in to Docker Hub\n        uses: docker/login-action@v3\n        with:\n          registry: ghcr.io\n          username: ${{ github.repository_owner }}\n          password: ${{ secrets.GITHUB_TOKEN }}\n\n      - name: Free up disk space\n        run: |\n          sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' '^mysql-.*' '^postgresql-.*'\n          sudo apt-get autoremove -y\n          sudo apt-get autoclean\n\n          sudo rm -rf /usr/share/dotnet\n          sudo rm -rf /usr/local/lib/android\n          sudo rm -rf /opt/ghc\n          sudo rm -rf /opt/hostedtoolcache/CodeQL\n\n          docker system prune -af\n\n          df -h\n\n      - name: Generate tags\n        id: tags\n        run: |\n          TAGS=\"ghcr.io/${{ github.repository }}:${{ matrix.config.tag }}\"\n          if [ \"${{ github.event_name }}\" == \"push\" ]; then\n            TAGS=\"$TAGS,ghcr.io/${{ github.repository }}:${{ matrix.config.tag }}-${{ env.COMMIT_SHA }}\"\n          fi\n          echo \"tags=$TAGS\" >> $GITHUB_OUTPUT\n\n      - name: Build and push Docker image (tagged)\n        uses: docker/build-push-action@v6\n        with:\n          context: .\n          push: ${{ github.event_name == 'push' }}\n          platforms: ${{ matrix.config.platform }}\n          tags: ${{ steps.tags.outputs.tags }}\n          file: ${{ matrix.config.dockerfile }}\n"
  },
  {
    "path": ".github/workflows/examples-wasm.yml",
    "content": "name: Examples WASM\non:\n  push:\n    branches: [\"master\"]\n\n  workflow_dispatch:\n\npermissions:\n  contents: read\n  pages: write\n  id-token: write\n\nconcurrency:\n  group: \"pages\"\n  cancel-in-progress: false\n\njobs:\n  deploy-wasm-github-pages:\n    environment:\n      name: github-pages\n      url: ${{ steps.deployment.outputs.page_url }}\n    runs-on: ubuntu-latest\n    steps:\n      - name: Checkout\n        uses: actions/checkout@v6\n\n      - name: Setup Pages\n        uses: actions/configure-pages@v5\n\n      - name: Setup emsdk\n        uses: mymindstorm/setup-emsdk@v14\n\n      - name: Build WASM Examples\n        # Enable for real build later in whisper.cpp\n        run: |\n          mkdir -p build-em && cd build-em\n          emcmake cmake .. -DCMAKE_BUILD_TYPE=Release\n          make -j\n\n      - name: Create staging directory\n        run: mkdir -p staging\n\n      - name: Create .nojekyll file in staging directory\n        run: touch staging/.nojekyll\n\n      - name: Copy application files\n        run: |\n          build_dir=build-em/bin\n\n          ls ${build_dir}\n\n          # command.wasm\n          target_dir=staging/command.wasm\n          mkdir -p ${target_dir}\n          cp ${build_dir}/command.wasm/{index.html,command.js,helpers.js} ${target_dir}\n          cp ${build_dir}/libcommand.js ${target_dir}\n\n          # bench.wasm\n          target_dir=staging/bench.wasm\n          mkdir -p ${target_dir}\n          cp ${build_dir}/bench.wasm/{index.html,bench.js,helpers.js} ${target_dir}\n          cp ${build_dir}/libbench.js ${target_dir}\n\n          # stream.wasm\n          target_dir=staging/stream.wasm\n          mkdir -p ${target_dir}\n          cp ${build_dir}/stream.wasm/{index.html,stream.js,helpers.js} ${target_dir}\n          cp ${build_dir}/libstream.js ${target_dir}\n\n          # wchess.wasm\n          target_dir=staging/wchess.wasm\n          mkdir -p ${target_dir}\n          cp -r ${build_dir}/wchess.wasm/{index.html,css,img,js} ${target_dir}\n          cp ${build_dir}/wchess.wasm.js ${target_dir}\n\n          # whisper.wasm (this will be the main example page)\n          target_dir=staging\n          mkdir -p ${target_dir}\n          cp ${build_dir}/whisper.wasm/{index.html,main.js,helpers.js} ${target_dir}\n          cp ${build_dir}/libmain.js ${target_dir}\n\n          # Copy Cross-Origin Isolation service worker\n          cp -v examples/coi-serviceworker.js staging/\n\n      - name: List files in staging directory (for debugging)\n        run: |\n          echo \"Files in staging directory:\"\n          find staging -type f | sort\n\n      - name: Upload artifact\n        uses: actions/upload-pages-artifact@v4\n        with:\n          path: ./staging\n\n      - name: Deploy to GitHub Pages\n        id: deployment\n        uses: actions/deploy-pages@v4\n"
  },
  {
    "path": ".github/workflows/examples.yml",
    "content": "name: Examples Tests\non:\n  push:\n    paths:\n      - examples/addon.node/**\n      - whisper.h\n  pull_request:\n    paths:\n      - examples/addon.node/**\n      - whisper.h\n\njobs:\n  addon_node-ubuntu-22:\n    runs-on: ubuntu-22.04\n    strategy:\n      matrix:\n        node-version: [ 16.x, 18.x ]\n    steps:\n      - name: Clone\n        uses: actions/checkout@v6\n\n      - name: Dependencies\n        run: |\n          sudo apt-get update\n          sudo apt-get install build-essential git\n          sudo apt-get install cmake\n          sudo apt-get install libsdl2-dev\n\n      - name: Use Node.js ${{ matrix.node-version }}\n        uses: actions/setup-node@v6\n        with:\n          node-version: ${{ matrix.node-version }}\n          cache: 'npm'\n\n      - name: Install package.json dependencies\n        working-directory: ./examples/addon.node\n        run: npm install\n\n      - name: Compile addon.node\n        run: npx cmake-js compile -T addon.node -B Release\n\n      - name: Download test model\n        run: |\n          bash ./models/download-ggml-model.sh base.en\n      - name: Test\n        run: |\n          cd examples/addon.node\n          npm run test\n"
  },
  {
    "path": ".gitignore",
    "content": "*.o\n*.a\n*.d\n.cache/\n.coreml/\n.test/\n.venv/\n.vs/\n.vscode/\n.DS_Store\n.vimspector.json\n/CMakeSettings.json\n/talk-llama.dSYM/\n\nbuild/\nbuild-*/\nbuild_*/\ntmp/\n\n# SPM\n.build/\n.swiftpm\n*.metallib\n\nggml-metal-embed.metal\nggml-metal-embed.metal.tmp\n\n/main\n/stream\n/command\n/talk\n/talk-llama\n/bench\n/quantize\n/server\n/lsp\n\narm_neon.h\nsync.sh\nlibwhisper.a\nlibwhisper.so\ncompile_commands.json\n\nexamples/arm_neon.h\nexamples/whisper.objc/whisper.objc.xcodeproj/xcshareddata\nexamples/whisper.objc/whisper.objc.xcodeproj/xcuserdata/\nexamples/whisper.objc/whisper.objc.xcodeproj/project.xcworkspace/xcuserdata\n\nextra/bench-gg.txt\n\nmodels/*.mlmodel\nmodels/*.mlmodelc\nmodels/*.mlpackage\nmodels/*-encoder-openvino.xml\nmodels/*-encoder-openvino-cache/\nbindings/java/.gradle/\nbindings/java/.idea/\n.idea/\n\nbenchmark_results.csv\ncmake-build-debug/\n.cxx/\n.gradle/\nlocal.properties\n.log\n.exe\n"
  },
  {
    "path": "AUTHORS",
    "content": "# date: Tue Feb  4 13:03:35 EET 2025\n# this file is auto-generated by scripts/gen-authors.sh\n\n0/0 <zero@imaskeleton.me>\n0cc4m <picard12@live.de>\n0xsourcecode <134374803+0xsourcecode@users.noreply.github.com>\n65a <10104049+65a@users.noreply.github.com>\nAIWintermuteAI <32562299+AIWintermuteAI@users.noreply.github.com>\nAT <manyoso@users.noreply.github.com>\nAarni Koskela <akx@iki.fi>\nAaron Pham <29749331+aarnphm@users.noreply.github.com>\nAaron Taylor <aaron@exphat.com>\nAbhilash Majumder <30946547+abhilash1910@users.noreply.github.com>\nAbitofevrything <54505189+abitofevrything@users.noreply.github.com>\nAdam Jones <domdomegg+git@gmail.com>\nAdrien Gallouët <adrien@gallouet.fr>\nAdrien Gallouët <angt@huggingface.co>\nAfryMask <AfryMask@163.com>\nAhmad Bilal <ahmad.bilal@empglabs.com>\nAhmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>\nAidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>\nAidanBeltonS <aidan.belton@codeplay.com>\nAkarshan Biswas <akarshan.biswas@gmail.com>\nAkarshan Biswas <akarshanbiswas@fedoraproject.org>\nAkash Mahajan <akash7190@gmail.com>\nAkash Mahajan <akashmjn@stanford.edu>\nAl Hoang <3811822-hoanga@users.noreply.gitlab.com>\nAlan <unknown>\nAlbert Jin <albert.jin@gmail.com>\nAlberto Cabrera Pérez <alberto.cabrera@codeplay.com>\nAlberto Cabrera Pérez <alberto.cabrera@intel.com>\nAleksander Andrzejewski <18704749+aleksanderandrzejewski@users.noreply.github.com>\nAlex Azarov <alex@azarov.by>\nAlex Bacart <13940752+alex-bacart@users.noreply.github.com>\nAlex Evgrashin <aevgrashin@yandex.ru>\nAlex O'Connell <35843486+acon96@users.noreply.github.com>\nAlexandr Graschenkov <alexandr.graschenkov91@gmail.com>\nAlexandru Mariuti <alex@mariuti.com>\nAlexey Kharlamov <alexey@kharlamov.biz>\nAlfredo Montesinos <alfredo.montesinos@g.austincc.edu>\nAli Alameh <ali.alameh@isae.edu.lb>\nAlter <0x7c48@gmail.com>\nAnanta Bastola <anantarajbastola@gmail.com>\nAndreas Kieslinger <47689530+aendk@users.noreply.github.com>\nAndreas Lubbe <git@lubbe.org>\nAndreu Huguet <andreuhuguet@gmail.com>\nAndrew Huynh <a5thuynh@gmail.com>\nAndrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>\nAndrew S <andrews54757@gmail.com>\nAndy Maloney <asmaloney@gmail.com>\nAnton Kostin <masguit42@users.noreply.github.com>\nArtyom Mezin <psycho.fading@gmail.com>\nAsad Memon <asad.lionpk@gmail.com>\nAshraful Islam <ashraful.meche@gmail.com>\nAsukaMinato <asukaminato@nyan.eu.org>\nAustinMroz <austinmroz@utexas.edu>\nAvik Sengupta <avik@sengupta.net>\nBader-eddine Ouaich <49657842+baderouaich@users.noreply.github.com>\nBaffin Lee <baffinlee@gmail.com>\nBen Ashbaugh <ben.ashbaugh@intel.com>\nBen Nortier <bjnortier@gmail.com>\nBenjamin Heiniger <benjamin.heiniger@bluewin.ch>\nBernhard M. Wiedemann <githubbmwprimary@lsmod.de>\nBinozo <70137898+Binozo@users.noreply.github.com>\nBo-Yi Wu <appleboy.tw@gmail.com>\nBoris Bliznioukov <blib@mail.com>\nBorislav Stanimirov <b.stanimirov@abv.bg>\nBrad Murray <59848399+bradmurray-dt@users.noreply.github.com>\nBrian Murray <brian@bmurray.ca>\nCRD716 <crd716@gmail.com>\nCanis Lupus <Canis-UK@users.noreply.github.com>\nCarlos Zoido <mrgalleta@gmail.com>\nCarolinabanana <140120812+Carolinabanana@users.noreply.github.com>\nCarterLi999 <664681047@qq.com>\nChangSeok Oh <shivamidow@users.noreply.github.com>\nChangyeon Kim <cyzero.kim@samsung.com>\nChaoqun <27287694+OpenWaygate@users.noreply.github.com>\nCharles Xu <63788048+chaxu01@users.noreply.github.com>\nCharles Xu <charles.xu@arm.com>\nChen Xi <xi2.chen@intel.com>\nChen Xi <xixichen08@foxmail.com>\nChenguang Li <87689256+noemotiovon@users.noreply.github.com>\nChia-Hsiang Cheng <88014292+garychia@users.noreply.github.com>\nChidi Williams <williamschidi1@gmail.com>\nChris Elrod <elrodc@gmail.com>\nChristian <12550267+iceychris@users.noreply.github.com>\nChristian Kastner <ckk@kvr.at>\nClifford Heath <clifford.heath@gmail.com>\nClint Herron <hanclinto@gmail.com>\nColin <github@whoisc.cc>\nConrad Kramer <conrad@conradkramer.com>\nCorey Earwood <iamcgn+github@gmail.com>\nCrispStrobe <154636388+CrispStrobe@users.noreply.github.com>\nDAN™ <dranger003@gmail.com>\nDGdev91 <DGdev91@users.noreply.github.com>\nDamian Czaja <trojan295@protonmail.com>\nDan Johansson <164997844+eddnjjn@users.noreply.github.com>\nDan Johansson <dan.johansson@arm.com>\nDaniel Bevenius <daniel.bevenius@gmail.com>\nDaniel Valdivia <18384552+dvaldivia@users.noreply.github.com>\nDaniel Ziegenberg <daniel@ziegenberg.at>\nDaniele <57776841+daniandtheweb@users.noreply.github.com>\nDave <dave-fl@users.noreply.github.com>\nDave Airlie <airlied@gmail.com>\nDave Airlie <airlied@redhat.com>\nDaven Sanassy <daven@vochlea.co.uk>\nDavid <dnhkng@gmail.com>\nDavid Thorpe <djt@mutablelogic.com>\nDavidKorczynski <david@adalogics.com>\nDavidson Francis <davidsondfgl@gmail.com>\nDener Stassun <denerstassun@gmail.com>\nDibakar Gope <dibakar.gope@arm.com>\nDidzis Gosko <didzis@users.noreply.github.com>\nDiego Devesa <slarengh@gmail.com>\nDigipom <admin@digipom.com>\nDimo <dimo@ieee.org>\nDjip007 <3705339+Djip007@users.noreply.github.com>\nDjip007 <djip.perois@free.fr>\nDody Suria Wijaya <dodysw@gmail.com>\nDou Xinpeng <15529241576@163.com>\nDou Xinpeng <81913537+Dou-Git@users.noreply.github.com>\nDr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>\nDuncan McConnell <ddmcconnell4@gmail.com>\nEgor Egorov <me@egorfine.com>\nElkana Bardugo <ttv200@gmail.com>\nEmmanuel Schmidbauer <eschmidbauer@gmail.com>\nEngininja2 <139037756+Engininja2@users.noreply.github.com>\nEric Curtin <ericcurtin17@gmail.com>\nEric Swanson <eswanson@alloscomp.com>\nEric Tendian <erictendian@gmail.com>\nEric Zhang <34133756+EZForever@users.noreply.github.com>\nErik Scholz <Green-Sky@users.noreply.github.com>\nEvan Jones <evan.q.jones@gmail.com>\nEvan Martin <evan.martin@gmail.com>\nEve <139727413+netrunnereve@users.noreply.github.com>\nEvgeny Kuznetsov <evgeny@kuznetsov.md>\nF1L1P <78918286+F1L1Pv2@users.noreply.github.com>\nFaisal Zaghloul <quic_fzaghlou@quicinc.com>\nFangjun Kuang <csukuangfj@gmail.com>\nFelix <stenbackfelix@gmail.com>\nFinn Voorhees <finnvoorhees@gmail.com>\nFirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>\nFlippFuzz <41221030+FlippFuzz@users.noreply.github.com>\nFrankie Robertson <frankier@users.noreply.github.com>\nGang Chen <goncha@gmail.com>\nGavin Cai <gavin1818@hotmail.com>\nGeorge Hindle <george@georgehindle.com>\nGeorgi Gerganov <ggerganov@gmail.com>\nGilad S <7817232+giladgd@users.noreply.github.com>\nGilad S <giladgd@users.noreply.github.com>\nGilad S. <7817232+giladgd@users.noreply.github.com>\nGitAritron <103900385+GitAritron@users.noreply.github.com>\nGiviMAD <GiviMAD@users.noreply.github.com>\nGleicon Moraes <gleicon@gmail.com>\nGregor Jasny <gjasny@googlemail.com>\nGuillaume Wenzek <gwenzek@users.noreply.github.com>\nHY. Kelvin Lee <34256578+hykelvinlee42@users.noreply.github.com>\nHalalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>\nHang <bebound@gmail.com>\nHaus1 <haus.xda@gmail.com>\nHerman Semenov <GermanAizek@yandex.ru>\nHimariO <dsfhe49854@gmail.com>\nHong Bo PENG <penghb@cn.ibm.com>\nHrishikesh Barman <geekodour@users.noreply.github.com>\nHugo <hugo@whynothugo.nl>\nIan Bicking <ian@ianbicking.org>\nIan Bull <irbull@eclipsesource.com>\nIhar Hrachyshka <ihrachys@redhat.com>\nIkko Ashimine <eltociear@gmail.com>\nIkko Eltociear Ashimine <eltociear@gmail.com>\nInconsolableCellist <23345188+InconsolableCellist@users.noreply.github.com>\nIsmatulla Mansurov <47342870+sapoepsilon@users.noreply.github.com>\nIvan <nekotekina@gmail.com>\nIvan Filipov <159561759+vanaka11@users.noreply.github.com>\nIvan Gorin <ivangorin21@gmail.com>\nIvo von Putzer Reibegg <ivo.putzer@gmail.com>\nJJ <103335846+computerscienceiscool@users.noreply.github.com>\nJack Mousseau <jmousseau@users.noreply.github.com>\nJacobLinCool <jacoblincool@gmail.com>\nJakub Ráček <blizzcz@gmail.com>\nJared Van Bortel <jared@nomic.ai>\nJay Binks <jaybinks@gmail.com>\nJayant <jayantyadav202@gmail.com>\nJeff Bolz <jbolz@nvidia.com>\nJeroen Mostert <jeroen.mostert@cm.com>\nJhen-Jie Hong <developer@jhen.me>\nJhen-Jie Hong <iainst0409@gmail.com>\nJidongZhang-THU <1119708529@qq.com>\nJo Liss <joliss42@gmail.com>\nJoe Todd <joe.todd@codeplay.com>\nJohan <jr.raffin@gmail.com>\nJohannes Gäßler <johannesg@5d6.de>\nJohn Balis <phobossystems@gmail.com>\nJohnnyB <jboero@users.noreply.github.com>\nJonathan Soo <jcsoo@agora.com>\nJonno <1160532+razodactyl@users.noreply.github.com>\nJoonas Pihlajamaa <joonas.pihlajamaa@iki.fi>\nJose <34888496+Jerry-Master@users.noreply.github.com>\nJosh Bleecher Snyder <josharian@gmail.com>\nJosscii <jossciiweiyi@gmail.com>\nJudd <foldl@users.noreply.github.com>\nJumper775 <78500318+jumpers775@users.noreply.github.com>\nJun Hee Yoo <contact.jhyoo@gmail.com>\nJunil Kim <logyourself@gmail.com>\nJustina Cho <justcho5@gmail.com>\nJustine Tunney <jtunney@gmail.com>\nJustine Tunney <jtunney@mozilla.com>\nKITAITI Makoto <KitaitiMakoto@gmail.com>\nKP Kaiser <kirk@zothcorp.com>\nKamilake <exjang0@gmail.com>\nKarol Kontny <82021046+kkontny@users.noreply.github.com>\nKarthick <j.karthic2004@gmail.com>\nKartik Saranathan <278928+Kartiku@users.noreply.github.com>\nKasumi <90275229+kasumi-1@users.noreply.github.com>\nKawrakow <48489457+ikawrakow@users.noreply.github.com>\nKendrick Taylor <kendrick@circuitsix.com>\nKevin Brothaler <admin@digipom.com>\nKevin Gibbons <bakkot@gmail.com>\nKonosuke Sakai <konosuke@konosuke.work>\nKonstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>\nKreijstal <rainb@tfwno.gf>\nKylin <56434533+KyL0N@users.noreply.github.com>\nLBlue <153975653+lbluep@users.noreply.github.com>\nLarry Battle <larry.battle.tech@gmail.com>\nLaytan Laats <laytanlaats@hotmail.com>\nLeo Moll <leo.moll@yeasoft.com>\nLexevolution <31176843+Lexevolution@users.noreply.github.com>\nLittleLoli <26589867+WhichWho@users.noreply.github.com>\nLucas Zanek <57494138+LucasZNK@users.noreply.github.com>\nLuis Herrera <herrera-luis@users.noreply.github.com>\nLukas Rist <glaslos@gmail.com>\nM. A. Ali <73258591+MightyStud@users.noreply.github.com>\nM. Eren Akbiyik <erenakbiyik@gmail.com>\nMa Mingfei <mingfei.ma@intel.com>\nMaciek <maciek.mab122@gmail.com>\nMahesh Madhav <67384846+heshpdx@users.noreply.github.com>\nMarcin Mielniczuk <marmistrz.dev@zoho.eu>\nMark Karpelès <MagicalTux@users.noreply.github.com>\nMark Zhuang <zhuangqiubin@gmail.com>\nMarkus Tavenrath <mtavenrath@users.noreply.github.com>\nMartin Delille <martin@delille.org>\nMartin Warnaar <martinwarnaar@gmail.com>\nMasaya, Kato <62578291+msy-kato@users.noreply.github.com>\nMatheus de Sousa <23645013+keyehzy@users.noreply.github.com>\nMathieu Baudier <mbaudier@argeo.org>\nMathijs de Bruin <mathijs@mathijsfietst.nl>\nMatija Pevec <mightymatth@users.noreply.github.com>\nMatt Stephenson <mstephenson6@users.noreply.github.com>\nMax Krasnyansky <max.krasnyansky@gmail.com>\nMax Krasnyansky <quic_maxk@quicinc.com>\nMaximiliano Levi <8160966+maxilevi@users.noreply.github.com>\nMeng, Hengyu <hengyu.meng@intel.com>\nMengqing Cao <cmq0113@163.com>\nMichael Podvitskiy <podvitskiymichael@gmail.com>\nMichael Rienstra <mrienstra@gmail.com>\nMikhail Grigorev <sleuthhound@gmail.com>\nMohammadreza Hendiani <hendiani.mohammadreza@gmail.com>\nMohit Agarwal <mohit@sdf.org>\nMolly Sophia <mollysophia379@gmail.com>\nMurilo Santana <mvrilo@gmail.com>\nNETZkultur GmbH <mulholland@netzkultur.de>\nNatsu <chino@hotococoa.moe>\nNeil Chudleigh <nchudleigh@users.noreply.github.com>\nNeo Zhang <14088817+arthw@users.noreply.github.com>\nNeo Zhang Jianyu <jianyu.zhang@intel.com>\nNeuman Vong <neuman.vong@gmail.com>\nNicholai Tukanov <nicholaitukanov@gmail.com>\nNicholas Albion <nalbion@yahoo.com>\nNico Bosshard <nico@bosshome.ch>\nNicolò Scipione <nicolo.scipione@codeplay.com>\nNiels Mayer <Niels.Mayer@gmail.com>\nNikita Sarychev <42014488+sARY77@users.noreply.github.com>\nNikolaj Olsson <nikse.dk@gmail.com>\nOkabintaro <103938900+Okabintaro@users.noreply.github.com>\nOleg Sidorov <me@whitebox.io>\nOleg Sidorov <oleg@sidorov.nl>\nOlivier Chafik <ochafik@users.noreply.github.com>\nOndrej Kokes <ondrej.kokes@gmail.com>\nOuadie EL FAROUKI <ouadie.elfarouki@codeplay.com>\nPAB <pierreantoine.bannier@gmail.com>\nPaul Tsochantaris <ptsochantaris@icloud.com>\nPedro Probst <pprobst@insiberia.net>\nPeng <hzp1024@qq.com>\nPeter <peter277@users.noreply.github.com>\nPhilipp Zabel <philipp.zabel@gmail.com>\nPhilippe Normand <phil@base-art.net>\nPhilippe Normand <philn@igalia.com>\nPlamen Minev <pacominev@gmail.com>\nPrashant Vithule <119530321+Vithulep@users.noreply.github.com>\nPrzemysław Pawełczyk <przemoc@gmail.com>\nQianhe Chen <54462604+chenqianhe@users.noreply.github.com>\nR0CKSTAR <xiaodong.ye@mthreads.com>\nR0CKSTAR <yeahdongcn@gmail.com>\nRadoslav Gerganov <rgerganov@gmail.com>\nRadosław Gryta <radek.gryta@gmail.com>\nRahul Vadhyar <107788610+RahulVadhyar@users.noreply.github.com>\nRaiya Araki <83504221+rai62@users.noreply.github.com>\nReinforce-II <fate@eastal.com>\nReinis Muiznieks <muiznieks.reinis@gmail.com>\nRelatedTitle <r3latedtitle@gmail.com>\nRémy Oudompheng <oudomphe@phare.normalesup.org>\nRhinoDevel <RhinoDevel@users.noreply.github.com>\nRich Jones <miserlou@gmail.com>\nRobert Ormandi <52251610+ormandi@users.noreply.github.com>\nRobin <robin.xw@hotmail.com>\nRoddur Dasgupta <roddurd@gmail.com>\nRoland Rabien <figbug@gmail.com>\nRomain Biessy <romain.biessy@codeplay.com>\nRonsor <ronsor@ronsor.pw>\nRotem Dan <rotemdan@gmail.com>\nRyan Hitchman <hitchmanr@gmail.com>\nRyan Metcalfe <107415876+RyanMetcalfeInt8@users.noreply.github.com>\nRyanChang <ftes90015@gmail.com>\nSRHMorris <69468379+SRHMorris@users.noreply.github.com>\nSXX <sxx1136965276@gmail.com>\nSacha Arbonel <sacha.arbonel@hotmail.fr>\nSalman Faroz <stsfaroz@gmail.com>\nSalvatore Mesoraca <s.mesoraca16@gmail.com>\nSam <49637763+Onlyartist9@users.noreply.github.com>\nSam Pullara <spullara@gmail.com>\nSamuel Durante <44513615+samueldurantes@users.noreply.github.com>\nSanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>\nSandro Hanea <40202887+sandrohanea@users.noreply.github.com>\nSergio López <slp@redhat.com>\nSergio López <slp@sinrega.org>\nShanshan Shen <467638484@qq.com>\nShijie <821898965@qq.com>\nShupei Fan <dymarkfan@outlook.com>\nSiddharth Ramakrishnan <srr2141@columbia.edu>\nSigbjørn Skjæret <sigbjorn.skjaeret@scala.com>\nSimon Moisselin <simon.moisstoll@gmail.com>\nSindre Sorhus <sindresorhus@gmail.com>\nSlava Primenko <primenko.s@gmail.com>\nSrihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>\nStavros Panakakis <53979866+Stavrospanakakis@users.noreply.github.com>\nStefan Sydow <s.sydow@heinlein-video.de>\nStefan Sydow <stefan@sydow.email>\nSyahmi Azhar <prsyahmi@gmail.com>\nSyed Jafri <syedjafri97@gmail.com>\nSơn Phan Trung <phantrungson17@gmail.com>\nTaisei Mima <bhbstar.me@gmail.com>\nTakeshi Inoue <inoue.takeshi@gmail.com>\nTamotsu Takahashi <ttakah+github@gmail.com>\nTaras Glek <taras@thegp.com>\nTauseef Mohiuddin <35351464+tauseefmohammed2@users.noreply.github.com>\nThamster <Thamster@users.noreply.github.com>\nThijs Raymakers <thijs@raymakers.nl>\nThomas Fitzsimmons <fitzsim@fitzsim.org>\nTiago Fassoni <tiagofassoni@users.noreply.github.com>\nTienshiao Ma <tienshiao@tienshiao.org>\nTim Miller <drasticactions@users.noreply.github.com>\nTimothy Cronin <40186632+4imothy@users.noreply.github.com>\nTobrun <tobrun.van.nuland@gmail.com>\nTodd <taf2@users.noreply.github.com>\nToliver <teejae@gmail.com>\nTong Li <31761981+litongjava@users.noreply.github.com>\nTony Wasserka <4840017+neobrain@users.noreply.github.com>\nTopping1 <78745143+Topping1@users.noreply.github.com>\nTravis Cline <travis.cline@gmail.com>\nUEXTM.com <84163508+uextm@users.noreply.github.com>\nUsernamesLame <156965854+UsernamesLame@users.noreply.github.com>\nVadim Peretokin <vperetokin@hey.com>\nValentin Gosu <1454649+valenting@users.noreply.github.com>\nVin Misra <vinith@alum.mit.edu>\nVulcan <93451215+trholding@users.noreply.github.com>\nWhiteOlivierus <36532695+WhiteOlivierus@users.noreply.github.com>\nWilliam Tambellini <william.tambellini@gmail.com>\nWilliam Tambellini <wtambellini@sdl.com>\nWilson Silva <wilson.dsigns@gmail.com>\nXiang (Kevin) Li <kevinli020508@gmail.com>\nXiao-Yong Jin <jinxiaoyong@gmail.com>\nXiaotaoChen <chenxiaotao1234@gmail.com>\nXingchen Song(宋星辰) <xingchensong1996@163.com>\nXinpeng Dou <81913537+Dou-Git@users.noreply.github.com>\nXuan Son Nguyen <thichthat@gmail.com>\nYajing Tang <phillis@google.com>\nYang Shen <aplshenyang@gmail.com>\nYunès <jean.baptiste.yunes@free.fr>\nYuri Khrustalev <ykhrustalev@users.noreply.github.com>\nYusuf Redžić <48274562+redzic@users.noreply.github.com>\nZaBlazzingZephyrus <119159668+blazingzephyr@users.noreply.github.com>\nZhenwei Jin <109658203+kylo5aby@users.noreply.github.com>\nZhiyuan Li <lizhiyuan@uniartisan.com>\nZhiyuan Li <uniartisan2017@gmail.com>\nZigfrid Zvezdin <ziggerZZ@gmail.com>\nZollner <24618122+Zolliner@users.noreply.github.com>\na3sh <38979186+A3shTnT@users.noreply.github.com>\nag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>\nagray3 <agray3@users.noreply.github.com>\nai-at-home <149282006+ai-at-home@users.noreply.github.com>\naldorof <aldorof@users.noreply.github.com>\nalonfaraj <alonfaraj@gmail.com>\namd-dwang <dong.wang@amd.com>\namritahs-ibm <amritahs@linux.vnet.ibm.com>\nandypayne <apayne@gmail.com>\nardfork <134447697+ardfork@users.noreply.github.com>\narizhih <40765267+arizhih@users.noreply.github.com>\nautomaticcat <daogiatuank54@gmail.com>\nbandoti <141645996+bandoti@users.noreply.github.com>\nbe-next <jerome.ramette@gmail.com>\nbert hubert <bert@hubertnet.nl>\nbillyct <billy_allen@126.com>\nbmwl <brian.marshall@tolko.com>\nbobqianic <129547291+bobqianic@users.noreply.github.com>\nbocytko <bocytko+github@gmail.com>\nboolemancer <48014766+boolemancer@users.noreply.github.com>\nboolemancer <boolemancer@gmail.com>\nbradmit <151883577+bradmit@users.noreply.github.com>\nbrunofaustino <b.fa.amorim@gmail.com>\nbssrdf <merlintiger@hotmail.com>\nbyte-6174 <88070277+byte-6174@users.noreply.github.com>\ncdosoftei <ciprian.dosoftei@gmail.com>\nclach04 <Chris.Clark@actian.com>\ncompilade <113953597+compilade@users.noreply.github.com>\ncompilade <git@compilade.net>\nconradg <conradjgodfrey@gmail.com>\ncrummyh <elijah@crums.us>\nddpasa <112642920+ddpasa@users.noreply.github.com>\ndenersc <denerstassun@gmail.com>\ndscripka <dscripka@users.noreply.github.com>\nduthils <duthils@duthils.net>\necneladis <ecneladis@users.noreply.github.com>\nfaker <nspyia2002@gmail.com>\nfitzsim <fitzsim@fitzsim.org>\nfj-y-saito <85871716+fj-y-saito@users.noreply.github.com>\nfraxy-v <65565042+fraxy-v@users.noreply.github.com>\ngenevera (she/her) <genevera@users.noreply.github.com>\ngeniusnut <geniusnut@gmail.com>\ngilbertgong <gilbert.gong@gmail.com>\ngn64 <yukikaze.jp@gmail.com>\ngoldwaving <77494627+goldwaving@users.noreply.github.com>\ngreeshmay <greeshmay@gmail.com>\nhaopeng <657407891@qq.com>\nhipudding <huafengchun@gmail.com>\nhsinhoyeh <yhh92u@gmail.com>\nhydai <z54981220@gmail.com>\niamthad <thadeus.j.fleming@gmail.com>\nissixx <46835150+issixx@users.noreply.github.com>\njames wolf <contractorwolf@hotmail.com>\njdomke <28772296+jdomke@users.noreply.github.com>\njettoblack <jettoblack@gmail.com>\njiez <373447296@qq.com>\njoecryptotoo <80373433+joecryptotoo@users.noreply.github.com>\njorismertz <35079666+jorismertz@users.noreply.github.com>\njunchao-loongson <68935141+junchao-loongson@users.noreply.github.com>\njunkfood <69683722+JunkFood02@users.noreply.github.com>\njwijffels <jwijffels@bnosac.be>\nk.h.lai <adrian.k.h.lai@outlook.com>\nkamranjon <kamranjon@gmail.com>\nkatsu560 <katsu560oo-@docomo.ne.jp>\nkennethge <57784063+kenneth-ge@users.noreply.github.com>\nkeyehzy <msamuel@aluno.puc-rio.br>\nkunnis <kunnis@users.noreply.github.com>\nl3utterfly <gc.pthzfoldr@gmail.com>\nleejet <leejet714@gmail.com>\nleo-pony <nengjunma@outlook.com>\nlhez <quic_lih@quicinc.com>\nlitong <31761981+litongjava@users.noreply.github.com>\nliuwei-git <14815172+liuwei-git@users.noreply.github.com>\nlnyan <lkwq007@gmail.com>\nluoyu-intel <yu.luo@intel.com>\nm.bell <m.bell@techsmith.com>\nmahorozte <41834471+mahorozte@users.noreply.github.com>\nmashizora <30516315+mashizora@users.noreply.github.com>\nmatt23654 <matthew.webber@protonmail.com>\nmatteo <matteogeniaccio@yahoo.it>\nmgrachten <maarten@grachten.eu>\nmkiol <mkiol@users.noreply.github.com>\nmky_coder <47767389+mkycoder@users.noreply.github.com>\nnovag <7754358+novag@users.noreply.github.com>\npajowu <pajowu@pajowu.de>\npengxin99 <pengxin.yuan@intel.com>\npetterreinholdtsen <pere-github@hungry.com>\npolarmoon <90010972+polarmoon@users.noreply.github.com>\nrlapray <lapray.romain@gmail.com>\nsandrohanea <40202887+sandrohanea@users.noreply.github.com>\nsemiformal-net <84111142+semiformal-net@users.noreply.github.com>\nshibukazu <61775791+shibukazu@users.noreply.github.com>\nshikokuchuo <53399081+shikokuchuo@users.noreply.github.com>\nslaren <slarengh@gmail.com>\nslashlib <slashlib@users.noreply.github.com>\nsnadampal <87143774+snadampal@users.noreply.github.com>\nsomeone13574 <81528246+someone13574@users.noreply.github.com>\nst-gr <38470677+st-gr@users.noreply.github.com>\nstduhpf <stephduh@live.fr>\nstormofice <58337328+stormofice@users.noreply.github.com>\ntexmex76 <40733439+texmex76@users.noreply.github.com>\nthefinaldegree <thefinaldegree@gmail.com>\nthewh1teagle <61390950+thewh1teagle@users.noreply.github.com>\ntoboil-features <160222185+toboil-features@users.noreply.github.com>\ntrixirt <trix@redhat.com>\nulatekh <ulatekh@yahoo.com>\nundef <undefdev@gmail.com>\nuvos <devnull@uvos.xyz>\nuvos <philipp@uvos.xyz>\nvalVk <valVk@users.noreply.github.com>\nvenkr <venkateshrameshkumar+1@gmail.com>\nvicalloy <zbirder@gmail.com>\nwangshuai09 <391746016@qq.com>\nwoachk <24752637+woachk@users.noreply.github.com>\nxctan <axunlei@gmail.com>\nxdrudis <xavierdrudis@yahoo.es>\nyuri@FreeBSD <yuri@FreeBSD>\nzhangjixiong <code.zjx@gmail.com>\nzhentaoyu <zhentao.yu@intel.com>\nzhouwg <6889919+zhouwg@users.noreply.github.com>\nzhouwg <zhouwg2000@gmail.com>\n谢乃闻 <sienaiwun@users.noreply.github.com>\n布客飞龙 <562826179@qq.com>\nАртём Земляк <azemlyak@smart-consulting.ru>\n"
  },
  {
    "path": "CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.5) # for add_link_options and implicit target directories.\nproject(\"whisper.cpp\" C CXX)\nproject(\"whisper.cpp\" VERSION 1.8.4)\ninclude(CheckIncludeFileCXX)\n\nset(SOVERSION 1)\n\n#set(CMAKE_WARN_DEPRECATED YES)\nset(CMAKE_WARN_UNUSED_CLI YES)\n\nset(CMAKE_EXPORT_COMPILE_COMMANDS ON)\n\nif (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)\n    set(CMAKE_BUILD_TYPE Release CACHE STRING \"Build type\" FORCE)\n    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS \"Debug\" \"Release\" \"MinSizeRel\" \"RelWithDebInfo\")\nendif()\n\n# Add path to modules\nlist(APPEND CMAKE_MODULE_PATH \"${CMAKE_CURRENT_SOURCE_DIR}/cmake/\")\n\nset(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n\nif (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)\n    set(WHISPER_STANDALONE ON)\n\n    include(git-vars)\n\n    # configure project version\n    configure_file(${CMAKE_SOURCE_DIR}/bindings/javascript/package-tmpl.json ${CMAKE_SOURCE_DIR}/bindings/javascript/package.json @ONLY)\nelse()\n    set(WHISPER_STANDALONE OFF)\nendif()\n\nif (EMSCRIPTEN)\n    set(BUILD_SHARED_LIBS_DEFAULT OFF)\n\n    set(CMAKE_CXX_STANDARD 17)\n    set(CMAKE_CXX_STANDARD_REQUIRED ON)\n\n    option(WHISPER_WASM_SINGLE_FILE \"whisper: embed WASM inside the generated whisper.js\" ON)\n\n    # TODO: without these, we get the following error:\n    #       wasm-ld: error: --shared-memory is disallowed by whisper.cpp.o because it was not compiled with 'atomics' or 'bulk-memory' features.\n    set(CMAKE_C_FLAGS   \"${CMAKE_C_FLAGS}   -pthread\")\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -pthread\")\n\n    set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} -s TOTAL_STACK=5242880\")\n    set(CMAKE_SHARED_LINKER_FLAGS \"${CMAKE_SHARED_LINKER_FLAGS} -s TOTAL_STACK=5242880\")\n\n    set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wno-deprecated\")\nelse()\n    if (MINGW)\n        set(BUILD_SHARED_LIBS_DEFAULT OFF)\n    else()\n        set(BUILD_SHARED_LIBS_DEFAULT ON)\n    endif()\nendif()\n\noption(BUILD_SHARED_LIBS \"build shared libraries\" ${BUILD_SHARED_LIBS_DEFAULT})\n\n#\n# option list\n#\n\n# debug\noption(WHISPER_ALL_WARNINGS           \"whisper: enable all compiler warnings\"                   ON)\noption(WHISPER_ALL_WARNINGS_3RD_PARTY \"whisper: enable all compiler warnings in 3rd party libs\" OFF)\n\n# build\noption(WHISPER_FATAL_WARNINGS  \"whisper: enable -Werror flag\"               OFF)\noption(WHISPER_USE_SYSTEM_GGML \"whisper: use system-installed GGML library\" OFF)\n\n# sanitizers\noption(WHISPER_SANITIZE_THREAD    \"whisper: enable thread sanitizer\"    OFF)\noption(WHISPER_SANITIZE_ADDRESS   \"whisper: enable address sanitizer\"   OFF)\noption(WHISPER_SANITIZE_UNDEFINED \"whisper: enable undefined sanitizer\" OFF)\n\n# extra artifacts\noption(WHISPER_BUILD_TESTS    \"whisper: build tests\"          ${WHISPER_STANDALONE})\noption(WHISPER_BUILD_EXAMPLES \"whisper: build examples\"       ${WHISPER_STANDALONE})\noption(WHISPER_BUILD_SERVER   \"whisper: build server example\" ${WHISPER_STANDALONE})\n\n# 3rd party libs\noption(WHISPER_CURL \"whisper: use libcurl to download model from an URL\" OFF)\noption(WHISPER_SDL2 \"whisper: support for libSDL2\" OFF)\n\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n    option(WHISPER_FFMPEG \"whisper: support building and linking with ffmpeg libs (avcodec, swresample, ...)\" OFF)\nendif()\n\noption(WHISPER_COREML                \"whisper: enable Core ML framework\"  OFF)\noption(WHISPER_COREML_ALLOW_FALLBACK \"whisper: allow non-CoreML fallback\" OFF)\noption(WHISPER_OPENVINO              \"whisper: support for OpenVINO\"      OFF)\n\n# Required for relocatable CMake package\ninclude(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)\n\n# override ggml options\nset(GGML_SANITIZE_THREAD    ${WHISPER_SANITIZE_THREAD})\nset(GGML_SANITIZE_ADDRESS   ${WHISPER_SANITIZE_ADDRESS})\nset(GGML_SANITIZE_UNDEFINED ${WHISPER_SANITIZE_UNDEFINED})\nset(GGML_ALL_WARNINGS       ${WHISPER_ALL_WARNINGS})\nset(GGML_FATAL_WARNINGS     ${WHISPER_FATAL_WARNINGS})\n\n# transition helpers\nfunction (whisper_option_depr TYPE OLD NEW)\n    if (${OLD})\n        message(${TYPE} \"${OLD} is deprecated and will be removed in the future.\\nUse ${NEW} instead\\n\")\n        set(${NEW} ON)\n    endif()\nendfunction()\n\nwhisper_option_depr(FATAL_ERROR WHISPER_CUBLAS              GGML_CUDA)\nwhisper_option_depr(WARNING     WHISPER_CUDA                GGML_CUDA)\nwhisper_option_depr(WARNING     WHISPER_KOMPUTE             GGML_KOMPUTE)\nwhisper_option_depr(WARNING     WHISPER_METAL               GGML_METAL)\nwhisper_option_depr(WARNING     WHISPER_METAL_EMBED_LIBRARY GGML_METAL_EMBED_LIBRARY)\nwhisper_option_depr(WARNING     WHISPER_NATIVE              GGML_NATIVE)\nwhisper_option_depr(WARNING     WHISPER_OPENMP              GGML_OPENMP)\nwhisper_option_depr(WARNING     WHISPER_RPC                 GGML_RPC)\nwhisper_option_depr(WARNING     WHISPER_SYCL                GGML_SYCL)\nwhisper_option_depr(WARNING     WHISPER_SYCL_F16            GGML_SYCL_F16)\nwhisper_option_depr(WARNING     WHISPER_CCACHE              GGML_CCACHE)\n\nif (GGML_CUDA AND NOT MSVC)\n    #GGML_CUDA enabled, add the necessary compile options -Wno-deprecated-gpu-targets\n    add_compile_options(-Wno-deprecated-gpu-targets)\nendif()\n\n#\n# build the library\n#\n\nif (NOT TARGET ggml)\n    if (WHISPER_USE_SYSTEM_GGML)\n        find_package(ggml REQUIRED)\n        if (NOT ggml_FOUND)\n            message(FATAL_ERROR \"System-installed GGML library not found.\")\n        endif()\n        add_library(ggml ALIAS ggml::ggml)\n    else()\n        add_subdirectory(ggml)\n        if(WIN32)\n            # The following adds a _DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR macro and is a workaround for\n            # the Windows C++ standard library which does not support constexpr mutexes.\n            # From the release notes://github.com/microsoft/STL/wiki/Changelog\n            #  Disable constexpr mutex constructor on Windows\n            #  Fixed mutex's constructor to be constexpr. #3824 #4000 #4339\n            #  Note: Programs that aren't following the documented restrictions on binary compatibility may encounter\n            #  null dereferences in mutex machinery. You must follow this rule:\n            #  When you mix binaries built by different supported versions of the toolset, the Redistributable version\n            #  must be at least as new as the latest toolset used by any app component.\n            #  You can define _DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR as an escape hatch.\n            #\n            # Specifically to whisper.cpp this would cause a crash when using the Java bindings.\n            # resulting in a Invalid memory access error.\n            target_compile_definitions(ggml-base PRIVATE _DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR)\n        endif()\n    endif()\n    # ... otherwise assume ggml is added by a parent CMakeLists.txt\nendif()\nadd_subdirectory(src)\n\n#\n# install\n#\n\ninclude(GNUInstallDirs)\ninclude(CMakePackageConfigHelpers)\n\nset(WHISPER_BUILD_NUMBER        ${BUILD_NUMBER})\nset(WHISPER_BUILD_COMMIT        ${BUILD_COMMIT})\nset(WHISPER_INSTALL_VERSION     ${CMAKE_PROJECT_VERSION})\n\nset(WHISPER_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH \"Location of header  files\")\nset(WHISPER_LIB_INSTALL_DIR     ${CMAKE_INSTALL_LIBDIR}     CACHE PATH \"Location of library files\")\nset(WHISPER_BIN_INSTALL_DIR     ${CMAKE_INSTALL_BINDIR}     CACHE PATH \"Location of binary  files\")\n\nget_directory_property(WHISPER_TRANSIENT_DEFINES COMPILE_DEFINITIONS)\n\nset_target_properties(whisper PROPERTIES PUBLIC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/whisper.h)\ninstall(TARGETS whisper LIBRARY PUBLIC_HEADER)\n\ntarget_compile_definitions(whisper PRIVATE\n    WHISPER_VERSION=\"${PROJECT_VERSION}\"\n)\n\nconfigure_package_config_file(\n        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/whisper-config.cmake.in\n        ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake\n    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper\n    PATH_VARS\n    WHISPER_INCLUDE_INSTALL_DIR\n    WHISPER_LIB_INSTALL_DIR\n    WHISPER_BIN_INSTALL_DIR )\n\nwrite_basic_package_version_file(\n    ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake\n    VERSION ${WHISPER_INSTALL_VERSION}\n    COMPATIBILITY SameMajorVersion)\n\ninstall(FILES ${CMAKE_CURRENT_BINARY_DIR}/whisper-config.cmake\n              ${CMAKE_CURRENT_BINARY_DIR}/whisper-version.cmake\n        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/whisper)\n\nconfigure_file(cmake/whisper.pc.in\n        \"${CMAKE_CURRENT_BINARY_DIR}/whisper.pc\"\n        @ONLY)\n\ninstall(FILES \"${CMAKE_CURRENT_BINARY_DIR}/whisper.pc\"\n        DESTINATION lib/pkgconfig)\n\n#\n# programs, examples and tests\n#\n\nif (WHISPER_BUILD_TESTS AND NOT CMAKE_JS_VERSION)\n    include(CTest)\n    add_subdirectory(tests)\nendif ()\n\nif (WHISPER_BUILD_EXAMPLES)\n    add_subdirectory(examples)\nendif()\n\nif (MSVC)\n    set(MSVC_WARNING_FLAGS\n        /wd4101  # Unreferenced local variable\n        /wd4005  # Macro redefinition\n        /wd4065  # switch statement contains 'default' but no 'case' labels\n        /wd4267  # Conversion from 'size_t' to a smaller type, possible loss of data\n        /wd4244  # Conversion from one type to another type, possible loss of ata\n        /wd4805  # Unsafe mix of type\n        /wd4305  # Truncation from 'type1' to 'type2' (often double to float)\n        /wd4996  # Function or variable may be unsafe/deprecated\n    )\n    function(disable_msvc_warnings target_name)\n        if(TARGET ${target_name})\n            target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})\n        endif()\n    endfunction()\n\n    if (WHISPER_BUILD_EXAMPLES)\n        disable_msvc_warnings(whisper)\n        disable_msvc_warnings(common)\n        disable_msvc_warnings(common-sdl)\n        disable_msvc_warnings(lsp)\n        disable_msvc_warnings(wchess-core)\n        disable_msvc_warnings(whisper-command)\n        disable_msvc_warnings(whisper-cli)\n        disable_msvc_warnings(whisper-server)\n        disable_msvc_warnings(whisper-stream)\n        disable_msvc_warnings(whisper-talk-llama)\n        disable_msvc_warnings(whisper-bench)\n        disable_msvc_warnings(quantize)\n        disable_msvc_warnings(vad-speech-segments)\n    endif()\nendif()\n"
  },
  {
    "path": "LICENSE",
    "content": "MIT License\n\nCopyright (c) 2023-2026 The ggml authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "Makefile",
    "content": "#\n# Audio samples\n#\n\n.PHONY: build\nbuild:\n\tcmake -B build $(CMAKE_ARGS)\n\tcmake --build build --config Release\n\n# download a few audio samples into folder \"./samples\":\n.PHONY: samples\nsamples:\n\t@echo \"Downloading samples...\"\n\t@mkdir -p samples\n\t@wget --quiet --show-progress -O samples/gb0.ogg https://upload.wikimedia.org/wikipedia/commons/2/22/George_W._Bush%27s_weekly_radio_address_%28November_1%2C_2008%29.oga\n\t@wget --quiet --show-progress -O samples/gb1.ogg https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg\n\t@wget --quiet --show-progress -O samples/hp0.ogg https://upload.wikimedia.org/wikipedia/en/d/d4/En.henryfphillips.ogg\n\t@wget --quiet --show-progress -O samples/mm1.wav https://cdn.openai.com/whisper/draft-20220913a/micro-machines.wav\n\t@wget --quiet --show-progress -O samples/a13.mp3 https://upload.wikimedia.org/wikipedia/commons/transcoded/6/6f/Apollo13-wehaveaproblem.ogg/Apollo13-wehaveaproblem.ogg.mp3\n\t@wget --quiet --show-progress -O samples/diffusion2023-07-03.flac https://archive.org/download/diffusion2023-07-03/diffusion2023-07-03.flac\n\n#\n# Models\n#\n\n# if not already downloaded, the following targets download the specified model and\n# runs it on all samples in the folder \"./samples\":\n\n.PHONY: tiny.en\n.PHONY: tiny\n.PHONY: base.en\n.PHONY: base\n.PHONY: small.en\n.PHONY: small\n.PHONY: medium.en\n.PHONY: medium\n.PHONY: large-v1\n.PHONY: large-v2\n.PHONY: large-v3\n.PHONY: large-v3-turbo\n\ntiny.en tiny base.en base small.en small medium.en medium large-v1 large-v2 large-v3 large-v3-turbo:\n\tbash ./models/download-ggml-model.sh $@\n\tcmake -B build $(CMAKE_ARGS)\n\tcmake --build build --config Release\n\t@echo \"\"\n\t@echo \"===============================================\"\n\t@echo \"Running $@ on all samples in ./samples ...\"\n\t@echo \"===============================================\"\n\t@echo \"\"\n\t@for f in samples/*.{flac,mp3,ogg,wav}; do \\\n\t\techo \"----------------------------------------------\" ; \\\n\t\techo \"[+] Running $@ on $$f ... (run 'ffplay $$f' to listen)\" ; \\\n\t\techo \"----------------------------------------------\" ; \\\n\t\techo \"\" ; \\\n\t\t./build/bin/whisper-cli -m models/ggml-$@.bin -f $$f ; \\\n\t\techo \"\" ; \\\n\tdone\n"
  },
  {
    "path": "README.md",
    "content": "# whisper.cpp\n\n![whisper.cpp](https://user-images.githubusercontent.com/1991296/235238348-05d0f6a4-da44-4900-a1de-d0707e75b763.jpeg)\n\n[![Actions Status](https://github.com/ggml-org/whisper.cpp/workflows/CI/badge.svg)](https://github.com/ggml-org/whisper.cpp/actions)\n[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)\n[![Conan Center](https://shields.io/conan/v/whisper-cpp)](https://conan.io/center/whisper-cpp)\n[![npm](https://img.shields.io/npm/v/whisper.cpp.svg)](https://www.npmjs.com/package/whisper.cpp/)\n\nStable: [v1.8.1](https://github.com/ggml-org/whisper.cpp/releases/tag/v1.8.1) / [Roadmap](https://github.com/orgs/ggml-org/projects/4/)\n\nHigh-performance inference of [OpenAI's Whisper](https://github.com/openai/whisper) automatic speech recognition (ASR) model:\n\n- Plain C/C++ implementation without dependencies\n- Apple Silicon first-class citizen - optimized via ARM NEON, Accelerate framework, Metal and [Core ML](#core-ml-support)\n- AVX intrinsics support for x86 architectures\n- [VSX intrinsics support for POWER architectures](#power-vsx-intrinsics)\n- Mixed F16 / F32 precision\n- [Integer quantization support](#quantization)\n- Zero memory allocations at runtime\n- [Vulkan support](#vulkan-gpu-support)\n- Support for CPU-only inference\n- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)\n- [OpenVINO Support](#openvino-support)\n- [Ascend NPU Support](#ascend-npu-support)\n- [Moore Threads GPU Support](#moore-threads-gpu-support)\n- [C-style API](https://github.com/ggml-org/whisper.cpp/blob/master/include/whisper.h)\n- [Voice Activity Detection (VAD)](#voice-activity-detection-vad)\n\nSupported platforms:\n\n- [x] Mac OS (Intel and Arm)\n- [x] [iOS](examples/whisper.objc)\n- [x] [Android](examples/whisper.android)\n- [x] [Java](bindings/java/README.md)\n- [x] Linux / [FreeBSD](https://github.com/ggml-org/whisper.cpp/issues/56#issuecomment-1350920264)\n- [x] [WebAssembly](examples/whisper.wasm)\n- [x] Windows ([MSVC](https://github.com/ggml-org/whisper.cpp/blob/master/.github/workflows/build.yml#L117-L144) and [MinGW](https://github.com/ggml-org/whisper.cpp/issues/168))\n- [x] [Raspberry Pi](https://github.com/ggml-org/whisper.cpp/discussions/166)\n- [x] [Docker](https://github.com/ggml-org/whisper.cpp/pkgs/container/whisper.cpp)\n\nThe entire high-level implementation of the model is contained in [whisper.h](include/whisper.h) and [whisper.cpp](src/whisper.cpp).\nThe rest of the code is part of the [`ggml`](https://github.com/ggml-org/ggml) machine learning library.\n\nHaving such a lightweight implementation of the model allows to easily integrate it in different platforms and applications.\nAs an example, here is a video of running the model on an iPhone 13 device - fully offline, on-device: [whisper.objc](examples/whisper.objc)\n\nhttps://user-images.githubusercontent.com/1991296/197385372-962a6dea-bca1-4d50-bf96-1d8c27b98c81.mp4\n\nYou can also easily make your own offline voice assistant application: [command](examples/command)\n\nhttps://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a76d-5735c25c49da.mp4\n\nOn Apple Silicon, the inference runs fully on the GPU via Metal:\n\nhttps://github.com/ggml-org/whisper.cpp/assets/1991296/c82e8f86-60dc-49f2-b048-d2fdbd6b5225\n\n## Quick start\n\nFirst clone the repository:\n\n```bash\ngit clone https://github.com/ggml-org/whisper.cpp.git\n```\n\nNavigate into the directory:\n\n```\ncd whisper.cpp\n```\n\nThen, download one of the Whisper [models](models/README.md) converted in [`ggml` format](#ggml-format). For example:\n\n```bash\nsh ./models/download-ggml-model.sh base.en\n```\n\nNow build the [whisper-cli](examples/cli) example and transcribe an audio file like this:\n\n```bash\n# build the project\ncmake -B build\ncmake --build build -j --config Release\n\n# transcribe an audio file\n./build/bin/whisper-cli -f samples/jfk.wav\n```\n\n---\n\nFor a quick demo, simply run `make base.en`.\n\nThe command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`.\n\nFor detailed usage instructions, run: `./build/bin/whisper-cli -h`\n\nNote that the [whisper-cli](examples/cli) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.\nFor example, you can use `ffmpeg` like this:\n\n```bash\nffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav\n```\n\n## More audio samples\n\nIf you want some extra audio samples to play with, simply run:\n\n```\nmake -j samples\n```\n\nThis will download a few more audio files from Wikipedia and convert them to 16-bit WAV format via `ffmpeg`.\n\nYou can download and run the other models as follows:\n\n```\nmake -j tiny.en\nmake -j tiny\nmake -j base.en\nmake -j base\nmake -j small.en\nmake -j small\nmake -j medium.en\nmake -j medium\nmake -j large-v1\nmake -j large-v2\nmake -j large-v3\nmake -j large-v3-turbo\n```\n\n## Memory usage\n\n| Model  | Disk    | Mem     |\n| ------ | ------- | ------- |\n| tiny   | 75 MiB  | ~273 MB |\n| base   | 142 MiB | ~388 MB |\n| small  | 466 MiB | ~852 MB |\n| medium | 1.5 GiB | ~2.1 GB |\n| large  | 2.9 GiB | ~3.9 GB |\n\n## POWER VSX Intrinsics\n\n`whisper.cpp` supports POWER architectures and includes code which\nsignificantly speeds operation on Linux running on POWER9/10, making it\ncapable of faster-than-realtime transcription on underclocked Raptor\nTalos II. Ensure you have a BLAS package installed, and replace the\nstandard cmake setup with:\n\n```bash\n# build with GGML_BLAS defined\ncmake -B build -DGGML_BLAS=1\ncmake --build build -j --config Release\n./build/bin/whisper-cli [ .. etc .. ]\n```\n\n## Quantization\n\n`whisper.cpp` supports integer quantization of the Whisper `ggml` models.\nQuantized models require less memory and disk space and depending on the hardware can be processed more efficiently.\n\nHere are the steps for creating and using a quantized model:\n\n```bash\n# quantize a model with Q5_0 method\ncmake -B build\ncmake --build build -j --config Release\n./build/bin/quantize models/ggml-base.en.bin models/ggml-base.en-q5_0.bin q5_0\n\n# run the examples as usual, specifying the quantized model file\n./build/bin/whisper-cli -m models/ggml-base.en-q5_0.bin ./samples/gb0.wav\n```\n\n## Core ML support\n\nOn Apple Silicon devices, the Encoder inference can be executed on the Apple Neural Engine (ANE) via Core ML. This can result in significant\nspeed-up - more than x3 faster compared with CPU-only execution. Here are the instructions for generating a Core ML model and using it with `whisper.cpp`:\n\n- Install Python dependencies needed for the creation of the Core ML model:\n\n  ```bash\n  pip install ane_transformers\n  pip install openai-whisper\n  pip install coremltools\n  ```\n\n  - To ensure `coremltools` operates correctly, please confirm that [Xcode](https://developer.apple.com/xcode/) is installed and execute `xcode-select --install` to install the command-line tools.\n  - Python 3.11 is recommended.\n  - MacOS Sonoma (version 14) or newer is recommended, as older versions of MacOS might experience issues with transcription hallucination.\n  - [OPTIONAL] It is recommended to utilize a Python version management system, such as [Miniconda](https://docs.conda.io/en/latest/miniconda.html) for this step:\n    - To create an environment, use: `conda create -n py311-whisper python=3.11 -y`\n    - To activate the environment, use: `conda activate py311-whisper`\n\n- Generate a Core ML model. For example, to generate a `base.en` model, use:\n\n  ```bash\n  ./models/generate-coreml-model.sh base.en\n  ```\n\n  This will generate the folder `models/ggml-base.en-encoder.mlmodelc`\n\n- Build `whisper.cpp` with Core ML support:\n\n  ```bash\n  # using CMake\n  cmake -B build -DWHISPER_COREML=1\n  cmake --build build -j --config Release\n  ```\n\n- Run the examples as usual. For example:\n\n  ```text\n  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav\n\n  ...\n\n  whisper_init_state: loading Core ML model from 'models/ggml-base.en-encoder.mlmodelc'\n  whisper_init_state: first run on a device may take a while ...\n  whisper_init_state: Core ML model loaded\n\n  system_info: n_threads = 4 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | COREML = 1 |\n\n  ...\n  ```\n\n  The first run on a device is slow, since the ANE service compiles the Core ML model to some device-specific format.\n  Next runs are faster.\n\nFor more information about the Core ML implementation please refer to PR [#566](https://github.com/ggml-org/whisper.cpp/pull/566).\n\n## OpenVINO support\n\nOn platforms that support [OpenVINO](https://github.com/openvinotoolkit/openvino), the Encoder inference can be executed\non OpenVINO-supported devices including x86 CPUs and Intel GPUs (integrated & discrete).\n\nThis can result in significant speedup in encoder performance. Here are the instructions for generating the OpenVINO model and using it with `whisper.cpp`:\n\n- First, setup python virtual env. and install python dependencies. Python 3.10 is recommended.\n\n  Windows:\n\n  ```powershell\n  cd models\n  python -m venv openvino_conv_env\n  openvino_conv_env\\Scripts\\activate\n  python -m pip install --upgrade pip\n  pip install -r requirements-openvino.txt\n  ```\n\n  Linux and macOS:\n\n  ```bash\n  cd models\n  python3 -m venv openvino_conv_env\n  source openvino_conv_env/bin/activate\n  python -m pip install --upgrade pip\n  pip install -r requirements-openvino.txt\n  ```\n\n- Generate an OpenVINO encoder model. For example, to generate a `base.en` model, use:\n\n  ```\n  python convert-whisper-to-openvino.py --model base.en\n  ```\n\n  This will produce ggml-base.en-encoder-openvino.xml/.bin IR model files. It's recommended to relocate these to the same folder as `ggml` models, as that\n  is the default location that the OpenVINO extension will search at runtime.\n\n- Build `whisper.cpp` with OpenVINO support:\n\n  Download OpenVINO package from [release page](https://github.com/openvinotoolkit/openvino/releases). The recommended version to use is [2024.6.0](https://github.com/openvinotoolkit/openvino/releases/tag/2024.6.0). Ready to use Binaries of the required libraries can be found in the [OpenVino Archives](https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.6/)\n\n  After downloading & extracting package onto your development system, set up required environment by sourcing setupvars script. For example:\n\n  Linux:\n\n  ```bash\n  source /path/to/l_openvino_toolkit_ubuntu22_2023.0.0.10926.b4452d56304_x86_64/setupvars.sh\n  ```\n\n  Windows (cmd):\n\n  ```powershell\n  C:\\Path\\To\\w_openvino_toolkit_windows_2023.0.0.10926.b4452d56304_x86_64\\setupvars.bat\n  ```\n\n  And then build the project using cmake:\n\n  ```bash\n  cmake -B build -DWHISPER_OPENVINO=1\n  cmake --build build -j --config Release\n  ```\n\n- Run the examples as usual. For example:\n\n  ```text\n  $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav\n\n  ...\n\n  whisper_ctx_init_openvino_encoder: loading OpenVINO model from 'models/ggml-base.en-encoder-openvino.xml'\n  whisper_ctx_init_openvino_encoder: first run on a device may take a while ...\n  whisper_openvino_init: path_model = models/ggml-base.en-encoder-openvino.xml, device = GPU, cache_dir = models/ggml-base.en-encoder-openvino-cache\n  whisper_ctx_init_openvino_encoder: OpenVINO model loaded\n\n  system_info: n_threads = 4 / 8 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | VSX = 0 | COREML = 0 | OPENVINO = 1 |\n\n  ...\n  ```\n\n  The first time run on an OpenVINO device is slow, since the OpenVINO framework will compile the IR (Intermediate Representation) model to a device-specific 'blob'. This device-specific blob will get\n  cached for the next run.\n\nFor more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).\n\n## NVIDIA GPU support\n\nWith NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels.\nFirst, make sure you have installed `cuda`: https://developer.nvidia.com/cuda-downloads\n\nNow build `whisper.cpp` with CUDA support:\n\n```\ncmake -B build -DGGML_CUDA=1\ncmake --build build -j --config Release\n```\n\nor for newer NVIDIA GPU's (RTX 5000 series):\n```\ncmake -B build -DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES=\"86\"\ncmake --build build -j --config Release\n```\n\n## Vulkan GPU support\nCross-vendor solution which allows you to accelerate workload on your GPU.\nFirst, make sure your graphics card driver provides support for Vulkan API.\n\nNow build `whisper.cpp` with Vulkan support:\n```\ncmake -B build -DGGML_VULKAN=1\ncmake --build build -j --config Release\n```\n\n## BLAS CPU support via OpenBLAS\n\nEncoder processing can be accelerated on the CPU via OpenBLAS.\nFirst, make sure you have installed `openblas`: https://www.openblas.net/\n\nNow build `whisper.cpp` with OpenBLAS support:\n\n```\ncmake -B build -DGGML_BLAS=1\ncmake --build build -j --config Release\n```\n\n## Ascend NPU support\n\nAscend NPU provides inference acceleration via [`CANN`](https://www.hiascend.com/en/software/cann) and AI cores.\n\nFirst, check if your Ascend NPU device is supported:\n\n**Verified devices**\n| Ascend NPU                    | Status  |\n|:-----------------------------:|:-------:|\n| Atlas 300T A2                 | Support |\n| Atlas 300I Duo                | Support |\n\nThen, make sure you have installed [`CANN toolkit`](https://www.hiascend.com/en/software/cann/community) . The lasted version of CANN is recommanded.\n\nNow build `whisper.cpp` with CANN support:\n\n```\ncmake -B build -DGGML_CANN=1\ncmake --build build -j --config Release\n```\n\nRun the inference examples as usual, for example:\n\n```\n./build/bin/whisper-cli -f samples/jfk.wav -m models/ggml-base.en.bin -t 8\n```\n\n*Notes:*\n\n- If you have trouble with Ascend NPU device, please create a issue with **[CANN]** prefix/tag.\n- If you run successfully with your Ascend NPU device, please help update the table `Verified devices`.\n\n## Moore Threads GPU support\n\nWith Moore Threads cards the processing of the models is done efficiently on the GPU via muBLAS and custom MUSA kernels.\nFirst, make sure you have installed `MUSA SDK rc4.2.0`: https://developer.mthreads.com/sdk/download/musa?equipment=&os=&driverVersion=&version=4.2.0\n\nNow build `whisper.cpp` with MUSA support:\n\n```\ncmake -B build -DGGML_MUSA=1\ncmake --build build -j --config Release\n```\n\nor specify the architecture for your Moore Threads GPU. For example, if you have a MTT S80 GPU, you can specify the architecture as follows:\n\n```\ncmake -B build -DGGML_MUSA=1 -DMUSA_ARCHITECTURES=\"21\"\ncmake --build build -j --config Release\n```\n\n## FFmpeg support (Linux only)\n\nIf you want to support more audio formats (such as Opus and AAC), you can turn on the `WHISPER_FFMPEG` build flag to enable FFmpeg integration.\n\nFirst, you need to install required libraries:\n\n```bash\n# Debian/Ubuntu\nsudo apt install libavcodec-dev libavformat-dev libavutil-dev\n\n# RHEL/Fedora\nsudo dnf install libavcodec-free-devel libavformat-free-devel libavutil-free-devel\n```\n\nThen you can build the project as follows:\n\n```bash\ncmake -B build -D WHISPER_FFMPEG=yes\ncmake --build build\n```\n\nRun the following example to confirm it's working:\n\n```bash\n# Convert an audio file to Opus format\nffmpeg -i samples/jfk.wav jfk.opus\n\n# Transcribe the audio file\n./build/bin/whisper-cli --model models/ggml-base.en.bin --file jfk.opus\n```\n\n## Docker\n\n### Prerequisites\n\n- Docker must be installed and running on your system.\n- Create a folder to store big models & intermediate files (ex. /whisper/models)\n\n### Images\n\nWe have multiple Docker images available for this project:\n\n1. `ghcr.io/ggml-org/whisper.cpp:main`: This image includes the main executable file as well as `curl` and `ffmpeg`. (platforms: `linux/amd64`, `linux/arm64`)\n2. `ghcr.io/ggml-org/whisper.cpp:main-cuda`: Same as `main` but compiled with CUDA support. (platforms: `linux/amd64`)\n3. `ghcr.io/ggml-org/whisper.cpp:main-musa`: Same as `main` but compiled with MUSA support. (platforms: `linux/amd64`)\n4. `ghcr.io/ggml-org/whisper.cpp:main-vulkan`: Same as `main` but compiled with Vulkan support. (platforms: `linux/amd64`)\n\n### Usage\n\n```shell\n# download model and persist it in a local folder\ndocker run -it --rm \\\n  -v path/to/models:/models \\\n  whisper.cpp:main \"./models/download-ggml-model.sh base /models\"\n\n# transcribe an audio file\ndocker run -it --rm \\\n  -v path/to/models:/models \\\n  -v path/to/audios:/audios \\\n  whisper.cpp:main \"whisper-cli -m /models/ggml-base.bin -f /audios/jfk.wav\"\n\n# transcribe an audio file in samples folder\ndocker run -it --rm \\\n  -v path/to/models:/models \\\n  whisper.cpp:main \"whisper-cli -m /models/ggml-base.bin -f ./samples/jfk.wav\"\n\n# run the web server\ndocker run -it --rm -p \"8080:8080\" \\\n  -v path/to/models:/models \\\n  whisper.cpp:main \"whisper-server --host 127.0.0.1 -m /models/ggml-base.bin\"\n  \n# run the bench too on the small.en model using 4 threads\ndocker run -it --rm \\\n  -v path/to/models:/models \\\n  whisper.cpp:main \"whisper-bench -m /models/ggml-small.en.bin -t 4\"\n```\n\n## Installing with Conan\n\nYou can install pre-built binaries for whisper.cpp or build it from source using [Conan](https://conan.io/). Use the following command:\n\n```\nconan install --requires=\"whisper-cpp/[*]\" --build=missing\n```\n\nFor detailed instructions on how to use Conan, please refer to the [Conan documentation](https://docs.conan.io/2/).\n\n## Limitations\n\n- Inference only\n\n## Real-time audio input example\n\nThis is a naive example of performing real-time inference on audio from your microphone.\nThe [stream](examples/stream) tool samples the audio every half a second and runs the transcription continuously.\nMore info is available in [issue #10](https://github.com/ggml-org/whisper.cpp/issues/10).\nYou will need to have [sdl2](https://wiki.libsdl.org/SDL2/Installation) installed for it to work properly.\n\n```bash\ncmake -B build -DWHISPER_SDL2=ON\ncmake --build build -j --config Release\n./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000\n```\n\nhttps://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4\n\n## Confidence color-coding\n\nAdding the `--print-colors` argument will print the transcribed text using an experimental color coding strategy\nto highlight words with high or low confidence:\n\n```bash\n./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/gb0.wav --print-colors\n```\n\n<img width=\"965\" alt=\"image\" src=\"https://user-images.githubusercontent.com/1991296/197356445-311c8643-9397-4e5e-b46e-0b4b4daa2530.png\">\n\n## Controlling the length of the generated text segments (experimental)\n\nFor example, to limit the line length to a maximum of 16 characters, simply add `-ml 16`:\n\n```text\n$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 16\n\nwhisper_model_load: loading model from './models/ggml-base.en.bin'\n...\nsystem_info: n_threads = 4 / 10 | AVX2 = 0 | AVX512 = 0 | NEON = 1 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 |\n\nmain: processing './samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...\n\n[00:00:00.000 --> 00:00:00.850]   And so my\n[00:00:00.850 --> 00:00:01.590]   fellow\n[00:00:01.590 --> 00:00:04.140]   Americans, ask\n[00:00:04.140 --> 00:00:05.660]   not what your\n[00:00:05.660 --> 00:00:06.840]   country can do\n[00:00:06.840 --> 00:00:08.430]   for you, ask\n[00:00:08.430 --> 00:00:09.440]   what you can do\n[00:00:09.440 --> 00:00:10.020]   for your\n[00:00:10.020 --> 00:00:11.000]   country.\n```\n\n## Word-level timestamp (experimental)\n\nThe `--max-len` argument can be used to obtain word-level timestamps. Simply use `-ml 1`:\n\n```text\n$ ./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -ml 1\n\nwhisper_model_load: loading model from './models/ggml-base.en.bin'\n...\nsystem_info: n_threads = 4 / 10 | AVX2 = 0 | AVX512 = 0 | NEON = 1 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 |\n\nmain: processing './samples/jfk.wav' (176000 samples, 11.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, timestamps = 1 ...\n\n[00:00:00.000 --> 00:00:00.320]\n[00:00:00.320 --> 00:00:00.370]   And\n[00:00:00.370 --> 00:00:00.690]   so\n[00:00:00.690 --> 00:00:00.850]   my\n[00:00:00.850 --> 00:00:01.590]   fellow\n[00:00:01.590 --> 00:00:02.850]   Americans\n[00:00:02.850 --> 00:00:03.300]  ,\n[00:00:03.300 --> 00:00:04.140]   ask\n[00:00:04.140 --> 00:00:04.990]   not\n[00:00:04.990 --> 00:00:05.410]   what\n[00:00:05.410 --> 00:00:05.660]   your\n[00:00:05.660 --> 00:00:06.260]   country\n[00:00:06.260 --> 00:00:06.600]   can\n[00:00:06.600 --> 00:00:06.840]   do\n[00:00:06.840 --> 00:00:07.010]   for\n[00:00:07.010 --> 00:00:08.170]   you\n[00:00:08.170 --> 00:00:08.190]  ,\n[00:00:08.190 --> 00:00:08.430]   ask\n[00:00:08.430 --> 00:00:08.910]   what\n[00:00:08.910 --> 00:00:09.040]   you\n[00:00:09.040 --> 00:00:09.320]   can\n[00:00:09.320 --> 00:00:09.440]   do\n[00:00:09.440 --> 00:00:09.760]   for\n[00:00:09.760 --> 00:00:10.020]   your\n[00:00:10.020 --> 00:00:10.510]   country\n[00:00:10.510 --> 00:00:11.000]  .\n```\n\n## Speaker segmentation via tinydiarize (experimental)\n\nMore information about this approach is available here: https://github.com/ggml-org/whisper.cpp/pull/1058\n\nSample usage:\n\n```py\n# download a tinydiarize compatible model\n./models/download-ggml-model.sh small.en-tdrz\n\n# run as usual, adding the \"-tdrz\" command-line argument\n./build/bin/whisper-cli -f ./samples/a13.wav -m ./models/ggml-small.en-tdrz.bin -tdrz\n...\nmain: processing './samples/a13.wav' (480000 samples, 30.0 sec), 4 threads, 1 processors, lang = en, task = transcribe, tdrz = 1, timestamps = 1 ...\n...\n[00:00:00.000 --> 00:00:03.800]   Okay Houston, we've had a problem here. [SPEAKER_TURN]\n[00:00:03.800 --> 00:00:06.200]   This is Houston. Say again please. [SPEAKER_TURN]\n[00:00:06.200 --> 00:00:08.260]   Uh Houston we've had a problem.\n[00:00:08.260 --> 00:00:11.320]   We've had a main beam up on a volt. [SPEAKER_TURN]\n[00:00:11.320 --> 00:00:13.820]   Roger main beam interval. [SPEAKER_TURN]\n[00:00:13.820 --> 00:00:15.100]   Uh uh [SPEAKER_TURN]\n[00:00:15.100 --> 00:00:18.020]   So okay stand, by thirteen we're looking at it. [SPEAKER_TURN]\n[00:00:18.020 --> 00:00:25.740]   Okay uh right now uh Houston the uh voltage is uh is looking good um.\n[00:00:27.620 --> 00:00:29.940]   And we had a a pretty large bank or so.\n```\n\n## Karaoke-style movie generation (experimental)\n\nThe [whisper-cli](examples/cli) example provides support for output of karaoke-style movies, where the\ncurrently pronounced word is highlighted. Use the `-owts` argument and run the generated bash script.\nThis requires to have `ffmpeg` installed.\n\nHere are a few _\"typical\"_ examples:\n\n```bash\n./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/jfk.wav -owts\nsource ./samples/jfk.wav.wts\nffplay ./samples/jfk.wav.mp4\n```\n\nhttps://user-images.githubusercontent.com/1991296/199337465-dbee4b5e-9aeb-48a3-b1c6-323ac4db5b2c.mp4\n\n---\n\n```bash\n./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/mm0.wav -owts\nsource ./samples/mm0.wav.wts\nffplay ./samples/mm0.wav.mp4\n```\n\nhttps://user-images.githubusercontent.com/1991296/199337504-cc8fd233-0cb7-4920-95f9-4227de3570aa.mp4\n\n---\n\n```bash\n./build/bin/whisper-cli -m ./models/ggml-base.en.bin -f ./samples/gb0.wav -owts\nsource ./samples/gb0.wav.wts\nffplay ./samples/gb0.wav.mp4\n```\n\nhttps://user-images.githubusercontent.com/1991296/199337538-b7b0c7a3-2753-4a88-a0cd-f28a317987ba.mp4\n\n---\n\n## Video comparison of different models\n\nUse the [scripts/bench-wts.sh](https://github.com/ggml-org/whisper.cpp/blob/master/scripts/bench-wts.sh) script to generate a video in the following format:\n\n```bash\n./scripts/bench-wts.sh samples/jfk.wav\nffplay ./samples/jfk.wav.all.mp4\n```\n\nhttps://user-images.githubusercontent.com/1991296/223206245-2d36d903-cf8e-4f09-8c3b-eb9f9c39d6fc.mp4\n\n---\n\n## Benchmarks\n\nIn order to have an objective comparison of the performance of the inference across different system configurations,\nuse the [whisper-bench](examples/bench) tool. The tool simply runs the Encoder part of the model and prints how much time it\ntook to execute it. The results are summarized in the following Github issue:\n\n[Benchmark results](https://github.com/ggml-org/whisper.cpp/issues/89)\n\nAdditionally a script to run whisper.cpp with different models and audio files is provided [bench.py](scripts/bench.py).\n\nYou can run it with the following command, by default it will run against any standard model in the models folder.\n\n```bash\npython3 scripts/bench.py -f samples/jfk.wav -t 2,4,8 -p 1,2\n```\n\nIt is written in python with the intention of being easy to modify and extend for your benchmarking use case.\n\nIt outputs a csv file with the results of the benchmarking.\n\n## `ggml` format\n\nThe original models are converted to a custom binary format. This allows to pack everything needed into a single file:\n\n- model parameters\n- mel filters\n- vocabulary\n- weights\n\nYou can download the converted models using the [models/download-ggml-model.sh](models/download-ggml-model.sh) script\nor manually from here:\n\n- https://huggingface.co/ggerganov/whisper.cpp\n\nFor more details, see the conversion script [models/convert-pt-to-ggml.py](models/convert-pt-to-ggml.py) or [models/README.md](models/README.md).\n\n## [Bindings](https://github.com/ggml-org/whisper.cpp/discussions/categories/bindings)\n\n- [x] Rust: [tazz4843/whisper-rs](https://github.com/tazz4843/whisper-rs) | [#310](https://github.com/ggml-org/whisper.cpp/discussions/310)\n- [x] JavaScript: [bindings/javascript](bindings/javascript) | [#309](https://github.com/ggml-org/whisper.cpp/discussions/309)\n  - React Native (iOS / Android): [whisper.rn](https://github.com/mybigday/whisper.rn)\n- [x] Go: [bindings/go](bindings/go) | [#312](https://github.com/ggml-org/whisper.cpp/discussions/312)\n- [x] Java:\n  - [GiviMAD/whisper-jni](https://github.com/GiviMAD/whisper-jni)\n- [x] Ruby: [bindings/ruby](bindings/ruby) | [#507](https://github.com/ggml-org/whisper.cpp/discussions/507)\n- [x] Objective-C / Swift: [ggml-org/whisper.spm](https://github.com/ggml-org/whisper.spm) | [#313](https://github.com/ggml-org/whisper.cpp/discussions/313)\n  - [exPHAT/SwiftWhisper](https://github.com/exPHAT/SwiftWhisper)\n- [x] .NET: | [#422](https://github.com/ggml-org/whisper.cpp/discussions/422)\n  - [sandrohanea/whisper.net](https://github.com/sandrohanea/whisper.net)\n  - [NickDarvey/whisper](https://github.com/NickDarvey/whisper)\n- [x] Python: | [#9](https://github.com/ggml-org/whisper.cpp/issues/9)\n  - [stlukey/whispercpp.py](https://github.com/stlukey/whispercpp.py) (Cython)\n  - [AIWintermuteAI/whispercpp](https://github.com/AIWintermuteAI/whispercpp) (Updated fork of aarnphm/whispercpp)\n  - [aarnphm/whispercpp](https://github.com/aarnphm/whispercpp) (Pybind11)\n  - [abdeladim-s/pywhispercpp](https://github.com/abdeladim-s/pywhispercpp) (Pybind11)\n- [x] R: [bnosac/audio.whisper](https://github.com/bnosac/audio.whisper)\n- [x] Unity: [macoron/whisper.unity](https://github.com/Macoron/whisper.unity)\n\n## XCFramework\nThe XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,\nand macOS. It can be used in Swift projects without the need to compile the\nlibrary from source. For example, the v1.7.5 version of the XCFramework can be\nused as follows:\n\n```swift\n// swift-tools-version: 5.10\n// The swift-tools-version declares the minimum version of Swift required to build this package.\n\nimport PackageDescription\n\nlet package = Package(\n    name: \"Whisper\",\n    targets: [\n        .executableTarget(\n            name: \"Whisper\",\n            dependencies: [\n                \"WhisperFramework\"\n            ]),\n        .binaryTarget(\n            name: \"WhisperFramework\",\n            url: \"https://github.com/ggml-org/whisper.cpp/releases/download/v1.7.5/whisper-v1.7.5-xcframework.zip\",\n            checksum: \"c7faeb328620d6012e130f3d705c51a6ea6c995605f2df50f6e1ad68c59c6c4a\"\n        )\n    ]\n)\n```\n\n## Voice Activity Detection (VAD)\nSupport for Voice Activity Detection (VAD) can be enabled using the `--vad`\nargument to `whisper-cli`. In addition to this option a VAD model is also\nrequired.\n\nThe way this works is that first the audio samples are passed through\nthe VAD model which will detect speech segments. Using this information,\nonly the speech segments that are detected are extracted from the original audio\ninput and passed to whisper for processing. This reduces the amount of audio\ndata that needs to be processed by whisper and can significantly speed up the\ntranscription process.\n\nThe following VAD models are currently supported:\n\n### Silero-VAD\n[Silero-vad](https://github.com/snakers4/silero-vad) is a lightweight VAD model\nwritten in Python that is fast and accurate.\n\nModels can be downloaded by running the following command on Linux or MacOS:\n```console\n$ ./models/download-vad-model.sh silero-v6.2.0\nDownloading ggml model silero-v6.2.0 from 'https://huggingface.co/ggml-org/whisper-vad' ...\nggml-silero-v6.2.0.bin        100%[==============================================>] 864.35K  --.-KB/s    in 0.04s\nDone! Model 'silero-v6.2.0' saved in '/path/models/ggml-silero-v6.2.0.bin'\nYou can now use it like this:\n\n  $ ./build/bin/whisper-cli -vm /path/models/ggml-silero-v6.2.0.bin --vad -f samples/jfk.wav -m models/ggml-base.en.bin\n\n```\nAnd the following command on Windows:\n```console\n> .\\models\\download-vad-model.cmd silero-v6.2.0\nDownloading vad model silero-v6.2.0...\nDone! Model silero-v6.2.0 saved in C:\\Users\\danie\\work\\ai\\whisper.cpp\\ggml-silero-v6.2.0.bin\nYou can now use it like this:\n\nC:\\path\\build\\bin\\Release\\whisper-cli.exe -vm C:\\path\\ggml-silero-v6.2.0.bin --vad -m models/ggml-base.en.bin -f samples\\jfk.wav\n\n```\n\nTo see a list of all available models, run the above commands without any\narguments.\n\nThis model can be also be converted manually to ggml using the following command:\n```console\n$ python3 -m venv venv && source venv/bin/activate\n$ (venv) pip install silero-vad\n$ (venv) $ python models/convert-silero-vad-to-ggml.py --output models/silero.bin\nSaving GGML Silero-VAD model to models/silero-v6.2.0-ggml.bin\n```\nAnd it can then be used with whisper as follows:\n```console\n$ ./build/bin/whisper-cli \\\n   --file ./samples/jfk.wav \\\n   --model ./models/ggml-base.en.bin \\\n   --vad \\\n   --vad-model ./models/silero-v6.2.0-ggml.bin\n```\n\n### VAD Options\n\n* --vad-threshold: Threshold probability for speech detection. A probability\nfor a speech segment/frame above this threshold will be considered as speech.\n\n* --vad-min-speech-duration-ms: Minimum speech duration in milliseconds. Speech\nsegments shorter than this value will be discarded to filter out brief noise or\nfalse positives.\n\n* --vad-min-silence-duration-ms: Minimum silence duration in milliseconds. Silence\nperiods must be at least this long to end a speech segment. Shorter silence\nperiods will be ignored and included as part of the speech.\n\n* --vad-max-speech-duration-s: Maximum speech duration in seconds. Speech segments\nlonger than this will be automatically split into multiple segments at silence\npoints exceeding 98ms to prevent excessively long segments.\n\n* --vad-speech-pad-ms: Speech padding in milliseconds. Adds this amount of padding\nbefore and after each detected speech segment to avoid cutting off speech edges.\n\n* --vad-samples-overlap: Amount of audio to extend from each speech segment into\nthe next one, in seconds (e.g., 0.10 = 100ms overlap). This ensures speech isn't\ncut off abruptly between segments when they're concatenated together.\n\n## Examples\n\nThere are various examples of using the library for different projects in the [examples](examples) folder.\nSome of the examples are even ported to run in the browser using WebAssembly. Check them out!\n\n| Example                                             | Web                                   | Description                                                                                                                     |\n| --------------------------------------------------- | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |\n| [whisper-cli](examples/cli)                         | [whisper.wasm](examples/whisper.wasm) | Tool for translating and transcribing audio using Whisper                                                                       |\n| [whisper-bench](examples/bench)                     | [bench.wasm](examples/bench.wasm)     | Benchmark the performance of Whisper on your machine                                                                            |\n| [whisper-stream](examples/stream)                   | [stream.wasm](examples/stream.wasm)   | Real-time transcription of raw microphone capture                                                                               |\n| [whisper-command](examples/command)                 | [command.wasm](examples/command.wasm) | Basic voice assistant example for receiving voice commands from the mic                                                         |\n| [whisper-server](examples/server)                   |                                       | HTTP transcription server with OAI-like API                                                                                     |\n| [whisper-talk-llama](examples/talk-llama)           |                                       | Talk with a LLaMA bot                                                                                                           |\n| [whisper.objc](examples/whisper.objc)               |                                       | iOS mobile application using whisper.cpp                                                                                        |\n| [whisper.swiftui](examples/whisper.swiftui)         |                                       | SwiftUI iOS / macOS application using whisper.cpp                                                                               |\n| [whisper.android](examples/whisper.android)         |                                       | Android mobile application using whisper.cpp                                                                                    |\n| [whisper.nvim](examples/whisper.nvim)               |                                       | Speech-to-text plugin for Neovim                                                                                                |\n| [generate-karaoke.sh](examples/generate-karaoke.sh) |                                       | Helper script to easily [generate a karaoke video](https://youtu.be/uj7hVta4blM) of raw audio capture                           |\n| [livestream.sh](examples/livestream.sh)             |                                       | [Livestream audio transcription](https://github.com/ggml-org/whisper.cpp/issues/185)                                            |\n| [yt-wsp.sh](examples/yt-wsp.sh)                     |                                       | Download + transcribe and/or translate any VOD [(original)](https://gist.github.com/DaniruKun/96f763ec1a037cc92fe1a059b643b818) |\n| [wchess](examples/wchess)                           | [wchess.wasm](examples/wchess)        | Voice-controlled chess                                                                                                          |\n\n## [Discussions](https://github.com/ggml-org/whisper.cpp/discussions)\n\nIf you have any kind of feedback about this project feel free to use the Discussions section and open a new topic.\nYou can use the [Show and tell](https://github.com/ggml-org/whisper.cpp/discussions/categories/show-and-tell) category\nto share your own projects that use `whisper.cpp`. If you have a question, make sure to check the\n[Frequently asked questions (#126)](https://github.com/ggml-org/whisper.cpp/discussions/126) discussion.\n"
  },
  {
    "path": "README_sycl.md",
    "content": "# whisper.cpp for SYCL\n\n[Background](#background)\n\n[OS](#os)\n\n[Intel GPU](#intel-gpu)\n\n[Linux](#linux)\n\n[Environment Variable](#environment-variable)\n\n[Known Issue](#known-issue)\n\n[Todo](#todo)\n\n## Background\n\nSYCL is a higher-level programming model to improve programming productivity on various hardware accelerators—such as CPUs, GPUs, and FPGAs. It is a single-source embedded domain-specific language based on pure C++17.\n\noneAPI is a specification that is open and standards-based, supporting multiple architecture types including but not limited to GPU, CPU, and FPGA. The spec has both direct programming and API-based programming paradigms.\n\nIntel uses the SYCL as direct programming language to support CPU, GPUs and FPGAs.\n\nTo avoid  re-inventing the wheel, this code refers other code paths in llama.cpp (like OpenBLAS, cuBLAS, CLBlast). We use a open-source tool [SYCLomatic](https://github.com/oneapi-src/SYCLomatic) (Commercial release [Intel® DPC++ Compatibility Tool](https://www.intel.com/content/www/us/en/developer/tools/oneapi/dpc-compatibility-tool.html)) migrate to SYCL.\n\nThe whisper.cpp for SYCL is used to support Intel GPUs.\n\nFor Intel CPU, recommend to use whisper.cpp for X86 (Intel MKL build).\n\n## OS\n\n|OS|Status|Verified|\n|-|-|-|\n|Linux|Support|Ubuntu 22.04|\n|Windows|Ongoing| |\n\n\n## Intel GPU\n\n|Intel GPU| Status | Verified Model|\n|-|-|-|\n|Intel Data Center Max Series| Support| Max 1550|\n|Intel Data Center Flex Series| Support| Flex 170|\n|Intel Arc Series| Support| Arc 770|\n|Intel built-in Arc GPU| Support| built-in Arc GPU in Meteor Lake|\n|Intel iGPU| Support| iGPU in i5-1250P, i7-1165G7|\n\n\n## Linux\n\n### Setup Environment\n\n1. Install Intel GPU driver.\n\na. Please install Intel GPU driver by official guide: [Install GPU Drivers](https://dgpu-docs.intel.com/driver/installation.html).\n\nNote: for iGPU, please install the client GPU driver.\n\nb. Add user to group: video, render.\n\n```\nsudo usermod -aG render username\nsudo usermod -aG video username\n```\n\nNote: re-login to enable it.\n\nc. Check\n\n```\nsudo apt install clinfo\nsudo clinfo -l\n```\n\nOutput (example):\n\n```\nPlatform #0: Intel(R) OpenCL Graphics\n `-- Device #0: Intel(R) Arc(TM) A770 Graphics\n\n\nPlatform #0: Intel(R) OpenCL HD Graphics\n `-- Device #0: Intel(R) Iris(R) Xe Graphics [0x9a49]\n```\n\n2. Install Intel® oneAPI Base toolkit.\n\n\na. Please follow the procedure in [Get the Intel® oneAPI Base Toolkit ](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html).\n\nRecommend to install to default folder: **/opt/intel/oneapi**.\n\nFollowing guide use the default folder as example. If you use other folder, please modify the following guide info with your folder.\n\nb. Check\n\n```\nsource /opt/intel/oneapi/setvars.sh\n\nsycl-ls\n```\n\nThere should be one or more level-zero devices. Like **[ext_oneapi_level_zero:gpu:0]**.\n\nOutput (example):\n```\n[opencl:acc:0] Intel(R) FPGA Emulation Platform for OpenCL(TM), Intel(R) FPGA Emulation Device OpenCL 1.2  [2023.16.10.0.17_160000]\n[opencl:cpu:1] Intel(R) OpenCL, 13th Gen Intel(R) Core(TM) i7-13700K OpenCL 3.0 (Build 0) [2023.16.10.0.17_160000]\n[opencl:gpu:2] Intel(R) OpenCL Graphics, Intel(R) Arc(TM) A770 Graphics OpenCL 3.0 NEO  [23.30.26918.50]\n[ext_oneapi_level_zero:gpu:0] Intel(R) Level-Zero, Intel(R) Arc(TM) A770 Graphics 1.3 [1.3.26918]\n\n```\n\n2. Build locally:\n\n```\nmkdir -p build\ncd build\nsource /opt/intel/oneapi/setvars.sh\n\n#for FP16\n#cmake .. -DWHISPER_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DWHISPER_SYCL_F16=ON \n\n#for FP32\ncmake .. -DWHISPER_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx\n\n#build example/main only\n#cmake --build . --config Release --target main\n\n#build all binary\ncmake --build . --config Release -v\n\n```\n\nor\n\n```\n./examples/sycl/build.sh\n```\n\nNote:\n\n- By default, it will build for all binary files. It will take more time. To reduce the time, we recommend to build for **example/main** only.\n\n### Run\n\n1. Put model file to folder **models**\n\n2. Enable oneAPI running environment\n\n```\nsource /opt/intel/oneapi/setvars.sh\n```\n\n3. List device ID\n\nRun without parameter:\n\n```\n./build/bin/ls-sycl-device\n\nor\n\n./build/bin/main\n```\n\nCheck the ID in startup log, like:\n\n```\nfound 4 SYCL devices:\n  Device 0: Intel(R) Arc(TM) A770 Graphics,\tcompute capability 1.3,\n    max compute_units 512,\tmax work group size 1024,\tmax sub group size 32,\tglobal mem size 16225243136\n  Device 1: Intel(R) FPGA Emulation Device,\tcompute capability 1.2,\n    max compute_units 24,\tmax work group size 67108864,\tmax sub group size 64,\tglobal mem size 67065057280\n  Device 2: 13th Gen Intel(R) Core(TM) i7-13700K,\tcompute capability 3.0,\n    max compute_units 24,\tmax work group size 8192,\tmax sub group size 64,\tglobal mem size 67065057280\n  Device 3: Intel(R) Arc(TM) A770 Graphics,\tcompute capability 3.0,\n    max compute_units 512,\tmax work group size 1024,\tmax sub group size 32,\tglobal mem size 16225243136\n\n```\n\n|Attribute|Note|\n|-|-|\n|compute capability 1.3|Level-zero running time, recommended |\n|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|\n\n4. Set device ID and execute whisper.cpp\n\nSet device ID = 0 by **GGML_SYCL_DEVICE=0**\n\n```\nGGML_SYCL_DEVICE=0 ./build/bin/main -m models/ggml-base.en.bin -f samples/jfk.wav\n```\nor run by script:\n\n```\n./examples/sycl/run_whisper.sh\n```\n\n\n\n5. Check the device ID in output\n\nLike:\n```\nUsing device **0** (Intel(R) Arc(TM) A770 Graphics) as main device\n```\n\n\n## Environment Variable\n\n#### Build\n\n|Name|Value|Function|\n|-|-|-|\n|WHISPER_SYCL|ON (mandatory)|Enable build with SYCL code path. <br>For FP32/FP16, WHISPER_SYCL=ON is mandatory.|\n|WHISPER_SYCL_F16|ON (optional)|Enable FP16 build with SYCL code path.For FP32, do not set it.|\n|CMAKE_C_COMPILER|icx|Use icx compiler for SYCL code path|\n|CMAKE_CXX_COMPILER|icpx|use icpx for SYCL code path|\n\n#### Running\n\n\n|Name|Value|Function|\n|-|-|-|\n|GGML_SYCL_DEVICE|0 (default) or 1|Set the device id used. Check the device ids by default running output|\n|GGML_SYCL_DEBUG|0 (default) or 1|Enable log function by macro: GGML_SYCL_DEBUG|\n\n## Known Issue\n\n- Error:  `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`.\n\n  Miss to enable oneAPI running environment.\n\n  Install oneAPI base toolkit and enable it by: `source /opt/intel/oneapi/setvars.sh`.\n\n\n- Hang during startup\n\n  llama.cpp use mmap as default way to read model file and copy to GPU. In some system, memcpy will be abnormal and block.\n\n  Solution: add **--no-mmap**.\n\n## Todo\n\n- Support to build in Windows.\n\n- Support multiple cards.\n"
  },
  {
    "path": "bindings/CMakeLists.txt",
    "content": "if (EMSCRIPTEN)\n    add_subdirectory(javascript)\n\n    add_custom_command(\n        OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/javascript/publish.log\n        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/javascript/whisper.js\n        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/javascript/libwhisper.worker.js\n        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/javascript/package.json\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/javascript\n        COMMAND npm publish\n        COMMAND touch publish.log\n        COMMENT \"Publishing npm module v${PROJECT_VERSION}\"\n        VERBATIM\n        )\n\n    add_custom_target(publish-npm\n        DEPENDS javascript/publish.log\n        )\nendif()\n"
  },
  {
    "path": "bindings/go/.gitignore",
    "content": "build\nmodels\n"
  },
  {
    "path": "bindings/go/LICENSE",
    "content": "MIT License\n\nCopyright (c) 2022 David Thorpe\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "bindings/go/Makefile",
    "content": "ifndef UNAME_S\nUNAME_S := $(shell uname -s)\nendif\n\nifndef UNAME_P\nUNAME_P := $(shell uname -p)\nendif\n\nifndef UNAME_M\nUNAME_M := $(shell uname -m)\nendif\n\nGGML_METAL_PATH_RESOURCES := $(abspath ../..)\nBUILD_DIR := build_go\nMODELS_DIR := models\nEXAMPLES_DIR := $(wildcard examples/*)\nINCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)\nLIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src):$(abspath ../../${BUILD_DIR}/ggml/src)\n\nifeq ($(GGML_CUDA),1)\n\tLIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/\n\tBUILD_FLAGS := -ldflags \"-extldflags '-lcudart -lcuda -lcublas'\"\nendif\n\nifeq ($(UNAME_S),Darwin)\n\tLIBRARY_PATH := $(LIBRARY_PATH):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-blas):$(abspath ../../${BUILD_DIR}/ggml/src/ggml-metal)\n\tEXT_LDFLAGS := -framework Foundation -framework Metal -framework MetalKit -lggml-metal -lggml-blas\nendif\n\nall: clean whisper examples\n\nwhisper: mkdir\n\tcmake -S ../.. -B ../../${BUILD_DIR} \\\n\t\t-DCMAKE_BUILD_TYPE=Release \\\n\t\t-DBUILD_SHARED_LIBS=OFF\n\tcmake --build ../../${BUILD_DIR} --target whisper\n\ntest: model-small whisper modtidy\nifeq ($(UNAME_S),Darwin)\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} GGML_METAL_PATH_RESOURCES=${GGML_METAL_PATH_RESOURCES} go test -ldflags \"-extldflags '$(EXT_LDFLAGS)'\" -v .\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} GGML_METAL_PATH_RESOURCES=${GGML_METAL_PATH_RESOURCES} go test -ldflags \"-extldflags '$(EXT_LDFLAGS)'\" -v ./pkg/whisper/...\nelse\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} go test -v .\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} go test -v ./pkg/whisper/...\nendif\n\nexamples: $(EXAMPLES_DIR)\n\nmodel-small: mkdir examples/go-model-download\n\t@${BUILD_DIR}/go-model-download -out models ggml-small.en.bin\n\n$(EXAMPLES_DIR): mkdir whisper modtidy\n\t@echo Build example $(notdir $@)\nifeq ($(UNAME_S),Darwin)\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} GGML_METAL_PATH_RESOURCES=${GGML_METAL_PATH_RESOURCES} go build ${BUILD_FLAGS} -ldflags \"-extldflags '$(EXT_LDFLAGS)'\" -o ${BUILD_DIR}/$(notdir $@) ./$@\nelse\n\t@C_INCLUDE_PATH=${INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} go build ${BUILD_FLAGS} -o ${BUILD_DIR}/$(notdir $@) ./$@\nendif\n\nmkdir:\n\t@echo Mkdir ${BUILD_DIR}\n\t@install -d ${BUILD_DIR}\n\t@echo Mkdir ${MODELS_DIR}\n\t@install -d ${MODELS_DIR}\n\nmodtidy:\n\t@go mod tidy\n\nclean:\n\t@echo Clean\n\t@rm -fr $(BUILD_DIR)\n\t@go clean\n"
  },
  {
    "path": "bindings/go/README.md",
    "content": "# Go bindings for Whisper\n\nThis package provides Go bindings for whisper.cpp. They have been tested on:\n\n  * Darwin (OS X) 12.6 on x64_64\n  * Debian Linux on arm64\n  * Fedora Linux on x86_64\n\nThe \"low level\" bindings are in the `bindings/go` directory and there is a more\nGo-style package in the `bindings/go/pkg/whisper` directory. The most simple usage\nis as follows:\n\n```go\nimport (\n\t\"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n)\n\nfunc main() {\n\tvar modelpath string // Path to the model\n\tvar samples []float32 // Samples to process\n\n\t// Load the model\n\tmodel, err := whisper.New(modelpath)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer model.Close()\n\n\t// Process samples\n\tcontext, err := model.NewContext()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tif err := context.Process(samples, nil, nil, nil); err != nil {\n\t\treturn err\n\t}\n\n\t// Print out the results\n\tfor {\n\t\tsegment, err := context.NextSegment()\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tfmt.Printf(\"[%6s->%6s] %s\\n\", segment.Start, segment.End, segment.Text)\n\t}\n}\n```\n\n## Building & Testing\n\nIn order to build, you need to have the Go compiler installed. You can get it from [here](https://golang.org/dl/). Run the tests with:\n\n```bash\ngit clone https://github.com/ggml-org/whisper.cpp.git\ncd whisper.cpp/bindings/go\nmake test\n```\n\nThis will compile a static `libwhisper.a` in a `build` folder, download a model file, then run the tests. To build the examples:\n\n```bash\nmake examples\n```\n\nTo build using cuda support add `GGML_CUDA=1`:\n\n```bash\nGGML_CUDA=1 make examples\n```\n\nThe examples are placed in the `build` directory. Once built, you can download all the models with the following command:\n\n```bash\n./build/go-model-download -out models\n```\n\nAnd you can then test a model against samples with the following command:\n\n```bash\n./build/go-whisper -model models/ggml-tiny.en.bin samples/jfk.wav\n```\n\n## Using the bindings\n\nTo use the bindings in your own software,\n\n  1. Import `github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper` (or `github.com/ggerganov/whisper.cpp/bindings/go` into your package;\n  2. Compile `libwhisper.a` (you can use `make whisper` in the `bindings/go` directory);\n  3. Link your go binary against whisper by setting the environment variables `C_INCLUDE_PATH` and `LIBRARY_PATH`\n     to point to the `whisper.h` file directory and `libwhisper.a` file directory respectively.\n\nLook at the `Makefile` in the `bindings/go` directory for an example.\n\nThe API Documentation:\n\n  * https://pkg.go.dev/github.com/ggerganov/whisper.cpp/bindings/go\n  * https://pkg.go.dev/github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\n\nGetting help:\n\n  * Follow the discussion for the go bindings [here](https://github.com/ggml-org/whisper.cpp/discussions/312)\n\n## License\n\nThe license for the Go bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.\n\n"
  },
  {
    "path": "bindings/go/doc.go",
    "content": "/*\ngithub.com/ggml-org/whisper.cpp/bindings/go\nprovides a speech-to-text service bindings for the Go programming language.\n*/\npackage whisper\n"
  },
  {
    "path": "bindings/go/examples/go-model-download/context.go",
    "content": "package main\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"os/signal\"\n)\n\n// ContextForSignal returns a context object which is cancelled when a signal\n// is received. It returns nil if no signal parameter is provided\nfunc ContextForSignal(signals ...os.Signal) context.Context {\n    if len(signals) == 0 {\n        return nil\n    }\n\n    ch := make(chan os.Signal, 1) // Buffered channel with space for 1 signal\n    ctx, cancel := context.WithCancel(context.Background())\n\n    // Send message on channel when signal received\n    signal.Notify(ch, signals...)\n\n    // When any signal is received, call cancel\n    go func() {\n        <-ch\n        cancel()\n    }()\n\n    // Return success\n    return ctx\n}\n\n"
  },
  {
    "path": "bindings/go/examples/go-model-download/main.go",
    "content": "package main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// CONSTANTS\n\nconst (\n\tsrcUrl  = \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/\" // The location of the models\n\tsrcExt  = \".bin\"                                                       // Filename extension\n\tbufSize = 1024 * 64                                                    // Size of the buffer used for downloading the model\n)\n\nvar (\n\t// The models which will be downloaded, if no model is specified as an argument\n\tmodelNames = []string{\n\t\t\"tiny\", \"tiny-q5_1\", \"tiny-q8_0\",\n\t\t\"tiny.en\", \"tiny.en-q5_1\", \"tiny.en-q8_0\",\n\t\t\"base\", \"base-q5_1\", \"base-q8_0\",\n\t\t\"base.en\", \"base.en-q5_1\", \"base.en-q8_0\",\n\t\t\"small\", \"small-q5_1\", \"small-q8_0\",\n\t\t\"small.en\", \"small.en-q5_1\", \"small.en-q8_0\",\n\t\t\"medium\", \"medium-q5_0\", \"medium-q8_0\",\n\t\t\"medium.en\", \"medium.en-q5_0\", \"medium.en-q8_0\",\n\t\t\"large-v1\",\n\t\t\"large-v2\", \"large-v2-q5_0\", \"large-v2-q8_0\",\n\t\t\"large-v3\", \"large-v3-q5_0\",\n\t\t\"large-v3-turbo\", \"large-v3-turbo-q5_0\", \"large-v3-turbo-q8_0\",\n\t}\n)\n\nvar (\n\t// The output folder. When not set, use current working directory.\n\tflagOut = flag.String(\"out\", \"\", \"Output folder\")\n\n\t// HTTP timeout parameter - will timeout if takes longer than this to download a model\n\tflagTimeout = flag.Duration(\"timeout\", 30*time.Minute, \"HTTP timeout\")\n\n\t// Quiet parameter - will not print progress if set\n\tflagQuiet = flag.Bool(\"quiet\", false, \"Quiet mode\")\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// MAIN\n\nfunc main() {\n\tflag.Usage = func() {\n\t\tname := filepath.Base(flag.CommandLine.Name())\n\t\tfmt.Fprintf(flag.CommandLine.Output(), `\n\t\t\tUsage: %s [options] [<model>...]\n\n\t\t\tOptions:\n  \t\t\t-out string     Specify the output folder where models will be saved.\n                  \t\t\tDefault: Current working directory.\n  \t\t\t-timeout duration Set the maximum duration for downloading a model.\n            \t\t\t      Example: 10m, 1h (default: 30m0s).\n  \t\t\t-quiet           Suppress all output except errors.\n\n\t\t\tExamples:\n  \t\t\t1. Download a specific model:\n     \t\t\t%s -out ./models tiny-q8_0\n\n\t\t\t  2. Download all models:\n     \t\t\t%s -out ./models\n\n\t\t\t`, name, name, name)\n\n\t\tflag.PrintDefaults()\n\t}\n\tflag.Parse()\n\n\t// Get output path\n\tout, err := GetOut()\n\tif err != nil {\n\t\tfmt.Fprintln(os.Stderr, \"Error:\", err)\n\t\tos.Exit(-1)\n\t}\n\n\t// Create context which quits on SIGINT or SIGQUIT\n\tctx := ContextForSignal(os.Interrupt, syscall.SIGQUIT)\n\n\t// Progress filehandle\n\tprogress := os.Stdout\n\tif *flagQuiet {\n\t\tprogress, err = os.Open(os.DevNull)\n\t\tif err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, \"Error:\", err)\n\t\t\tos.Exit(-1)\n\t\t}\n\t\tdefer progress.Close()\n\t}\n\n\t// Download models - exit on error or interrupt\n\tfor _, model := range GetModels() {\n\t\turl, err := URLForModel(model)\n\t\tif err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, \"Error:\", err)\n\t\t\tcontinue\n\t\t} else if path, err := Download(ctx, progress, url, out); err == nil || err == io.EOF {\n\t\t\tcontinue\n\t\t} else if err == context.Canceled {\n\t\t\tos.Remove(path)\n\t\t\tfmt.Fprintln(progress, \"\\nInterrupted\")\n\t\t\tbreak\n\t\t} else if err == context.DeadlineExceeded {\n\t\t\tos.Remove(path)\n\t\t\tfmt.Fprintln(progress, \"Timeout downloading model\")\n\t\t\tcontinue\n\t\t} else {\n\t\t\tos.Remove(path)\n\t\t\tfmt.Fprintln(os.Stderr, \"Error:\", err)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\n// GetOut returns the path to the output directory\nfunc GetOut() (string, error) {\n\tif *flagOut == \"\" {\n\t\treturn os.Getwd()\n\t}\n\tif info, err := os.Stat(*flagOut); err != nil {\n\t\treturn \"\", err\n\t} else if !info.IsDir() {\n\t\treturn \"\", fmt.Errorf(\"not a directory: %s\", info.Name())\n\t} else {\n\t\treturn *flagOut, nil\n\t}\n}\n\n// GetModels returns the list of models to download\nfunc GetModels() []string {\n\tif flag.NArg() == 0 {\n\t\tfmt.Println(\"No model specified.\")\n\t\tfmt.Println(\"Preparing to download all models...\")\n\n\t\t// Calculate total download size\n\t\tfmt.Println(\"Calculating total download size...\")\n\t\ttotalSize, err := CalculateTotalDownloadSize(modelNames)\n\t\tif err != nil {\n\t\t\tfmt.Println(\"Error calculating download sizes:\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\tfmt.Println(\"View available models: https://huggingface.co/ggerganov/whisper.cpp/tree/main\")\n\t\tfmt.Printf(\"Total download size: %.2f GB\\n\", float64(totalSize)/(1024*1024*1024))\n\t\tfmt.Println(\"Would you like to download all models? (y/N)\")\n\n\t\t// Prompt for user input\n\t\tvar response string\n\t\tfmt.Scanln(&response)\n\t\tif response != \"y\" && response != \"Y\" {\n\t\t\tfmt.Println(\"Aborting. Specify a model to download.\")\n\t\t\tos.Exit(0)\n\t\t}\n\n\t\treturn modelNames // Return all models if confirmed\n\t}\n\treturn flag.Args() // Return specific models if arguments are provided\n}\n\nfunc CalculateTotalDownloadSize(models []string) (int64, error) {\n\tvar totalSize int64\n\tclient := http.Client{}\n\n\tfor _, model := range models {\n\t\tmodelURL, err := URLForModel(model)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\t// Issue a HEAD request to get the file size\n\t\treq, err := http.NewRequest(\"HEAD\", modelURL, nil)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tresp, err := client.Do(req)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tresp.Body.Close()\n\n\t\tif resp.StatusCode != http.StatusOK {\n\t\t\tfmt.Printf(\"Warning: Unable to fetch size for %s (HTTP %d)\\n\", model, resp.StatusCode)\n\t\t\tcontinue\n\t\t}\n\n\t\tsize := resp.ContentLength\n\t\ttotalSize += size\n\t}\n\treturn totalSize, nil\n}\n\n// URLForModel returns the URL for the given model on huggingface.co\nfunc URLForModel(model string) (string, error) {\n\t// Ensure \"ggml-\" prefix is added only once\n\tif !strings.HasPrefix(model, \"ggml-\") {\n\t\tmodel = \"ggml-\" + model\n\t}\n\n\t// Ensure \".bin\" extension is added only once\n\tif filepath.Ext(model) != srcExt {\n\t\tmodel += srcExt\n\t}\n\n\t// Parse the base URL\n\turl, err := url.Parse(srcUrl)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t// Ensure no trailing slash in the base URL\n\turl.Path = fmt.Sprintf(\"%s/%s\", strings.TrimSuffix(url.Path, \"/\"), model)\n\treturn url.String(), nil\n}\n\n// Download downloads the model from the given URL to the given output directory\nfunc Download(ctx context.Context, p io.Writer, model, out string) (string, error) {\n\t// Create HTTP client\n\tclient := http.Client{\n\t\tTimeout: *flagTimeout,\n\t}\n\n\t// Initiate the download\n\treq, err := http.NewRequest(\"GET\", model, nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tresp, err := client.Do(req)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn \"\", fmt.Errorf(\"%s: %s\", model, resp.Status)\n\t}\n\n\t// If output file exists and is the same size as the model, skip\n\tpath := filepath.Join(out, filepath.Base(model))\n\tif info, err := os.Stat(path); err == nil && info.Size() == resp.ContentLength {\n\t\tfmt.Fprintln(p, \"Skipping\", model, \"as it already exists\")\n\t\treturn \"\", nil\n\t}\n\n\t// Create file\n\tw, err := os.Create(path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer w.Close()\n\n\t// Report\n\tfmt.Fprintln(p, \"Downloading\", model, \"to\", out)\n\n\t// Progressively download the model\n\tdata := make([]byte, bufSize)\n\tcount, pct := int64(0), int64(0)\n\tticker := time.NewTicker(5 * time.Second)\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\t// Cancelled, return error\n\t\t\treturn path, ctx.Err()\n\t\tcase <-ticker.C:\n\t\t\tpct = DownloadReport(p, pct, count, resp.ContentLength)\n\t\tdefault:\n\t\t\t// Read body\n\t\t\tn, err := resp.Body.Read(data)\n\t\t\tif n > 0 {\n\t\t\t\tif m, err := w.Write(data[:n]); err != nil {\n\t\t\t\t\treturn path, err\n\t\t\t\t} else {\n\t\t\t\t\tcount += int64(m)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tif err == io.EOF {\n\t\t\t\t\tDownloadReport(p, pct, count, resp.ContentLength)\n\t\t\t\t\treturn path, nil\n\t\t\t\t}\n\t\t\t\treturn path, err\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Report periodically reports the download progress when percentage changes\nfunc DownloadReport(w io.Writer, pct, count, total int64) int64 {\n\tpct_ := count * 100 / total\n\tif pct_ > pct {\n\t\tfmt.Fprintf(w, \"  ...%d MB written (%d%%)\\n\", count/1e6, pct_)\n\t}\n\treturn pct_\n}\n"
  },
  {
    "path": "bindings/go/examples/go-whisper/color.go",
    "content": "package main\n\nimport \"fmt\"\n\n///////////////////////////////////////////////////////////////////////////////\n// CONSTANTS\n\nconst (\n\tReset     = \"\\033[0m\"\n\tRGBPrefix = \"\\033[38;5;\" // followed by RGB values in decimal format separated by colons\n\tRGBSuffix = \"m\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\n// Colorize text with RGB values, from 0 to 23\nfunc Colorize(text string, v int) string {\n\t// https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit\n\t// Grayscale colors are in the range 232-255\n\treturn RGBPrefix + fmt.Sprint(v%24+232) + RGBSuffix + text + Reset\n}\n"
  },
  {
    "path": "bindings/go/examples/go-whisper/flags.go",
    "content": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t// Packages\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// TYPES\n\ntype Flags struct {\n\t*flag.FlagSet\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// LIFECYCLE\n\nfunc NewFlags(name string, args []string) (*Flags, error) {\n\tflags := &Flags{\n\t\tFlagSet: flag.NewFlagSet(name, flag.ContinueOnError),\n\t}\n\n\t// Register the command line arguments\n\tregisterFlags(flags)\n\n\t// Parse command line\n\tif err := flags.Parse(args); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Return success\n\treturn flags, nil\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\nfunc (flags *Flags) GetModel() string {\n\treturn flags.Lookup(\"model\").Value.String()\n}\n\nfunc (flags *Flags) GetLanguage() string {\n\treturn flags.Lookup(\"language\").Value.String()\n}\n\nfunc (flags *Flags) IsTranslate() bool {\n\treturn flags.Lookup(\"translate\").Value.(flag.Getter).Get().(bool)\n}\n\nfunc (flags *Flags) GetOffset() time.Duration {\n\treturn flags.Lookup(\"offset\").Value.(flag.Getter).Get().(time.Duration)\n}\n\nfunc (flags *Flags) GetDuration() time.Duration {\n\treturn flags.Lookup(\"duration\").Value.(flag.Getter).Get().(time.Duration)\n}\n\nfunc (flags *Flags) GetThreads() uint {\n\treturn flags.Lookup(\"threads\").Value.(flag.Getter).Get().(uint)\n}\n\nfunc (flags *Flags) GetOut() string {\n\treturn strings.ToLower(flags.Lookup(\"out\").Value.String())\n}\n\nfunc (flags *Flags) IsTokens() bool {\n\treturn flags.Lookup(\"tokens\").Value.String() == \"true\"\n}\n\nfunc (flags *Flags) IsColorize() bool {\n\treturn flags.Lookup(\"colorize\").Value.String() == \"true\"\n}\n\nfunc (flags *Flags) GetMaxLen() uint {\n\treturn flags.Lookup(\"max-len\").Value.(flag.Getter).Get().(uint)\n}\n\nfunc (flags *Flags) GetMaxTokens() uint {\n\treturn flags.Lookup(\"max-tokens\").Value.(flag.Getter).Get().(uint)\n}\n\nfunc (flags *Flags) GetWordThreshold() float32 {\n\treturn float32(flags.Lookup(\"word-thold\").Value.(flag.Getter).Get().(float64))\n}\n\nfunc (flags *Flags) SetParams(context whisper.Context) error {\n\tif lang := flags.GetLanguage(); lang != \"\" && lang != \"auto\" {\n\t\tfmt.Fprintf(flags.Output(), \"Setting language to %q\\n\", lang)\n\t\tif err := context.SetLanguage(lang); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif flags.IsTranslate() && context.IsMultilingual() {\n\t\tfmt.Fprintf(flags.Output(), \"Setting translate to true\\n\")\n\t\tcontext.SetTranslate(true)\n\t}\n\tif offset := flags.GetOffset(); offset != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting offset to %v\\n\", offset)\n\t\tcontext.SetOffset(offset)\n\t}\n\tif duration := flags.GetDuration(); duration != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting duration to %v\\n\", duration)\n\t\tcontext.SetDuration(duration)\n\t}\n\tif threads := flags.GetThreads(); threads != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting threads to %d\\n\", threads)\n\t\tcontext.SetThreads(threads)\n\t}\n\tif max_len := flags.GetMaxLen(); max_len != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting max_segment_length to %d\\n\", max_len)\n\t\tcontext.SetMaxSegmentLength(max_len)\n\t}\n\tif max_tokens := flags.GetMaxTokens(); max_tokens != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting max_tokens to %d\\n\", max_tokens)\n\t\tcontext.SetMaxTokensPerSegment(max_tokens)\n\t}\n\tif word_threshold := flags.GetWordThreshold(); word_threshold != 0 {\n\t\tfmt.Fprintf(flags.Output(), \"Setting word_threshold to %f\\n\", word_threshold)\n\t\tcontext.SetTokenThreshold(word_threshold)\n\t}\n\n\t// Return success\n\treturn nil\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PRIVATE METHODS\n\nfunc registerFlags(flag *Flags) {\n\tflag.String(\"model\", \"\", \"Path to the model file\")\n\tflag.String(\"language\", \"\", \"Spoken language\")\n\tflag.Bool(\"translate\", false, \"Translate from source language to english\")\n\tflag.Duration(\"offset\", 0, \"Time offset\")\n\tflag.Duration(\"duration\", 0, \"Duration of audio to process\")\n\tflag.Uint(\"threads\", 0, \"Number of threads to use\")\n\tflag.Uint(\"max-len\", 0, \"Maximum segment length in characters\")\n\tflag.Uint(\"max-tokens\", 0, \"Maximum tokens per segment\")\n\tflag.Float64(\"word-thold\", 0, \"Maximum segment score\")\n\tflag.Bool(\"tokens\", false, \"Display tokens\")\n\tflag.Bool(\"colorize\", false, \"Colorize tokens\")\n\tflag.String(\"out\", \"\", \"Output format (srt, none or leave as empty string)\")\n}\n"
  },
  {
    "path": "bindings/go/examples/go-whisper/main.go",
    "content": "package main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t// Packages\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n)\n\nfunc main() {\n\tflags, err := NewFlags(filepath.Base(os.Args[0]), os.Args[1:])\n\tif err == flag.ErrHelp {\n\t\tos.Exit(0)\n\t} else if err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t} else if flags.GetModel() == \"\" {\n\t\tfmt.Fprintln(os.Stderr, \"Use -model flag to specify which model file to use\")\n\t\tos.Exit(1)\n\t} else if flags.NArg() == 0 {\n\t\tfmt.Fprintln(os.Stderr, \"No input files specified\")\n\t\tos.Exit(1)\n\t}\n\n\t// Load model\n\tmodel, err := whisper.New(flags.GetModel())\n\tif err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n\tdefer model.Close()\n\n\t// Process files\n\tfor _, filename := range flags.Args() {\n\t\tif err := Process(model, filename, flags); err != nil {\n\t\t\tfmt.Fprintln(os.Stderr, err)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "bindings/go/examples/go-whisper/process.go",
    "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"time\"\n\n\t// Package imports\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n\twav \"github.com/go-audio/wav\"\n)\n\nfunc Process(model whisper.Model, path string, flags *Flags) error {\n\tvar data []float32\n\n\t// Create processing context\n\tcontext, err := model.NewContext()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Set the parameters\n\tif err := flags.SetParams(context); err != nil {\n\t\treturn err\n\t}\n\n\tfmt.Printf(\"\\n%s\\n\", context.SystemInfo())\n\n\t// Open the file\n\tfmt.Fprintf(flags.Output(), \"Loading %q\\n\", path)\n\tfh, err := os.Open(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer fh.Close()\n\n\t// Decode the WAV file - load the full buffer\n\tdec := wav.NewDecoder(fh)\n\tif buf, err := dec.FullPCMBuffer(); err != nil {\n\t\treturn err\n\t} else if dec.SampleRate != whisper.SampleRate {\n\t\treturn fmt.Errorf(\"unsupported sample rate: %d\", dec.SampleRate)\n\t} else if dec.NumChans != 1 {\n\t\treturn fmt.Errorf(\"unsupported number of channels: %d\", dec.NumChans)\n\t} else {\n\t\tdata = buf.AsFloat32Buffer().Data\n\t}\n\n\t// Segment callback when -tokens is specified\n\tvar cb whisper.SegmentCallback\n\tif flags.IsTokens() {\n\t\tcb = func(segment whisper.Segment) {\n\t\t\tfmt.Fprintf(flags.Output(), \"%02d [%6s->%6s] \", segment.Num, segment.Start.Truncate(time.Millisecond), segment.End.Truncate(time.Millisecond))\n\t\t\tfor _, token := range segment.Tokens {\n\t\t\t\tif flags.IsColorize() && context.IsText(token) {\n\t\t\t\t\tfmt.Fprint(flags.Output(), Colorize(token.Text, int(token.P*24.0)), \" \")\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Fprint(flags.Output(), token.Text, \" \")\n\t\t\t\t}\n\t\t\t}\n\t\t\tfmt.Fprintln(flags.Output(), \"\")\n\t\t\tfmt.Fprintln(flags.Output(), \"\")\n\t\t}\n\t}\n\n\t// Process the data\n\tfmt.Fprintf(flags.Output(), \"  ...processing %q\\n\", path)\n\tcontext.ResetTimings()\n\tif err := context.Process(data, nil, cb, nil); err != nil {\n\t\treturn err\n\t}\n\n\tcontext.PrintTimings()\n\n\t// Print out the results\n\tswitch {\n\tcase flags.GetOut() == \"srt\":\n\t\treturn OutputSRT(os.Stdout, context)\n\tcase flags.GetOut() == \"none\":\n\t\treturn nil\n\tdefault:\n\t\treturn Output(os.Stdout, context, flags.IsColorize())\n\t}\n}\n\n// Output text as SRT file\nfunc OutputSRT(w io.Writer, context whisper.Context) error {\n\tn := 1\n\tfor {\n\t\tsegment, err := context.NextSegment()\n\t\tif err == io.EOF {\n\t\t\treturn nil\n\t\t} else if err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfmt.Fprintln(w, n)\n\t\tfmt.Fprintln(w, srtTimestamp(segment.Start), \" --> \", srtTimestamp(segment.End))\n\t\tfmt.Fprintln(w, segment.Text)\n\t\tfmt.Fprintln(w, \"\")\n\t\tn++\n\t}\n}\n\n// Output text to terminal\nfunc Output(w io.Writer, context whisper.Context, colorize bool) error {\n\tfor {\n\t\tsegment, err := context.NextSegment()\n\t\tif err == io.EOF {\n\t\t\treturn nil\n\t\t} else if err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfmt.Fprintf(w, \"[%6s->%6s]\", segment.Start.Truncate(time.Millisecond), segment.End.Truncate(time.Millisecond))\n\t\tif colorize {\n\t\t\tfor _, token := range segment.Tokens {\n\t\t\t\tif !context.IsText(token) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfmt.Fprint(w, \" \", Colorize(token.Text, int(token.P*24.0)))\n\t\t\t}\n\t\t\tfmt.Fprint(w, \"\\n\")\n\t\t} else {\n\t\t\tfmt.Fprintln(w, \" \", segment.Text)\n\t\t}\n\t}\n}\n\n// Return srtTimestamp\nfunc srtTimestamp(t time.Duration) string {\n\treturn fmt.Sprintf(\"%02d:%02d:%02d,%03d\", t/time.Hour, (t%time.Hour)/time.Minute, (t%time.Minute)/time.Second, (t%time.Second)/time.Millisecond)\n}\n"
  },
  {
    "path": "bindings/go/go.mod",
    "content": "module github.com/ggerganov/whisper.cpp/bindings/go\n\ngo 1.23\n\nrequire (\n\tgithub.com/go-audio/wav v1.1.0\n\tgithub.com/stretchr/testify v1.9.0\n)\n\nrequire (\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/go-audio/audio v1.0.0 // indirect\n\tgithub.com/go-audio/riff v1.0.0 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n)\n"
  },
  {
    "path": "bindings/go/go.sum",
    "content": "github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=\ngithub.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=\ngithub.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA=\ngithub.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498=\ngithub.com/go-audio/wav v1.1.0 h1:jQgLtbqBzY7G+BM8fXF7AHUk1uHUviWS4X39d5rsL2g=\ngithub.com/go-audio/wav v1.1.0/go.mod h1:mpe9qfwbScEbkd8uybLuIpTgHyrISw/OTuvjUW2iGtE=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=\ngithub.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n"
  },
  {
    "path": "bindings/go/params.go",
    "content": "package whisper\n\nimport (\n\t\"fmt\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// CGO\n\n/*\n#include <whisper.h>\n*/\nimport \"C\"\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\nfunc (p *Params) SetTranslate(v bool) {\n\tp.translate = toBool(v)\n}\n\nfunc (p *Params) SetSplitOnWord(v bool) {\n\tp.split_on_word = toBool(v)\n}\n\nfunc (p *Params) SetNoContext(v bool) {\n\tp.no_context = toBool(v)\n}\n\nfunc (p *Params) SetSingleSegment(v bool) {\n\tp.single_segment = toBool(v)\n}\n\nfunc (p *Params) SetPrintSpecial(v bool) {\n\tp.print_special = toBool(v)\n}\n\nfunc (p *Params) SetPrintProgress(v bool) {\n\tp.print_progress = toBool(v)\n}\n\nfunc (p *Params) SetPrintRealtime(v bool) {\n\tp.print_realtime = toBool(v)\n}\n\nfunc (p *Params) SetPrintTimestamps(v bool) {\n\tp.print_timestamps = toBool(v)\n}\n\n// Voice Activity Detection (VAD)\nfunc (p *Params) SetVAD(v bool) {\n\tp.vad = toBool(v)\n}\n\nfunc (p *Params) SetVADModelPath(path string) {\n\tp.vad_model_path = C.CString(path)\n}\n\nfunc (p *Params) SetVADThreshold(t float32) {\n\tp.vad_params.threshold = C.float(t)\n}\n\nfunc (p *Params) SetVADMinSpeechMs(ms int) {\n\tp.vad_params.min_speech_duration_ms = C.int(ms)\n}\n\nfunc (p *Params) SetVADMinSilenceMs(ms int) {\n\tp.vad_params.min_silence_duration_ms = C.int(ms)\n}\n\nfunc (p *Params) SetVADMaxSpeechSec(s float32) {\n\tp.vad_params.max_speech_duration_s = C.float(s)\n}\n\nfunc (p *Params) SetVADSpeechPadMs(ms int) {\n\tp.vad_params.speech_pad_ms = C.int(ms)\n}\n\nfunc (p *Params) SetVADSamplesOverlap(sec float32) {\n\tp.vad_params.samples_overlap = C.float(sec)\n}\n\n// Set language id\nfunc (p *Params) SetLanguage(lang int) error {\n\tif lang == -1 {\n\t\tp.language = nil\n\t\treturn nil\n\t}\n\tstr := C.whisper_lang_str(C.int(lang))\n\tif str == nil {\n\t\treturn ErrInvalidLanguage\n\t} else {\n\t\tp.language = str\n\t}\n\treturn nil\n}\n\n// Get language id\nfunc (p *Params) Language() int {\n\tif p.language == nil {\n\t\treturn -1\n\t}\n\treturn int(C.whisper_lang_id(p.language))\n}\n\n// Threads available\nfunc (p *Params) Threads() int {\n\treturn int(p.n_threads)\n}\n\n// Set number of threads to use\nfunc (p *Params) SetThreads(threads int) {\n\tp.n_threads = C.int(threads)\n}\n\n// Set start offset in ms\nfunc (p *Params) SetOffset(offset_ms int) {\n\tp.offset_ms = C.int(offset_ms)\n}\n\n// Set audio duration to process in ms\nfunc (p *Params) SetDuration(duration_ms int) {\n\tp.duration_ms = C.int(duration_ms)\n}\n\n// Set timestamp token probability threshold (~0.01)\nfunc (p *Params) SetTokenThreshold(t float32) {\n\tp.thold_pt = C.float(t)\n}\n\n// Set timestamp token sum probability threshold (~0.01)\nfunc (p *Params) SetTokenSumThreshold(t float32) {\n\tp.thold_ptsum = C.float(t)\n}\n\n// Set max segment length in characters\nfunc (p *Params) SetMaxSegmentLength(n int) {\n\tp.max_len = C.int(n)\n}\n\nfunc (p *Params) SetTokenTimestamps(b bool) {\n\tp.token_timestamps = toBool(b)\n}\n\n// Set max tokens per segment (0 = no limit)\nfunc (p *Params) SetMaxTokensPerSegment(n int) {\n\tp.max_tokens = C.int(n)\n}\n\n// Set audio encoder context\nfunc (p *Params) SetAudioCtx(n int) {\n\tp.audio_ctx = C.int(n)\n}\n\nfunc (p *Params) SetMaxContext(n int) {\n\tp.n_max_text_ctx = C.int(n)\n}\n\nfunc (p *Params) SetBeamSize(n int) {\n\tp.beam_search.beam_size = C.int(n)\n}\n\nfunc (p *Params) SetEntropyThold(t float32) {\n\tp.entropy_thold = C.float(t)\n}\n\nfunc (p *Params) SetTemperature(t float32) {\n\tp.temperature = C.float(t)\n}\n\n// Sets the fallback temperature incrementation\n// Pass -1.0 to disable this feature\nfunc (p *Params) SetTemperatureFallback(t float32) {\n\tp.temperature_inc = C.float(t)\n}\n\n// Set initial prompt\nfunc (p *Params) SetInitialPrompt(prompt string) {\n\tp.initial_prompt = C.CString(prompt)\n}\n\nfunc (p *Params) SetCarryInitialPrompt(v bool) {\n\tp.carry_initial_prompt = toBool(v)\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PRIVATE METHODS\n\nfunc toBool(v bool) C.bool {\n\tif v {\n\t\treturn C.bool(true)\n\t}\n\treturn C.bool(false)\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// STRINGIFY\n\nfunc (p *Params) String() string {\n\tstr := \"<whisper.params\"\n\tstr += fmt.Sprintf(\" strategy=%v\", p.strategy)\n\tstr += fmt.Sprintf(\" n_threads=%d\", p.n_threads)\n\tif p.language != nil {\n\t\tstr += fmt.Sprintf(\" language=%s\", C.GoString(p.language))\n\t}\n\tstr += fmt.Sprintf(\" n_max_text_ctx=%d\", p.n_max_text_ctx)\n\tstr += fmt.Sprintf(\" offset_ms=%d\", p.offset_ms)\n\tstr += fmt.Sprintf(\" duration_ms=%d\", p.duration_ms)\n\tstr += fmt.Sprintf(\" audio_ctx=%d\", p.audio_ctx)\n\tstr += fmt.Sprintf(\" initial_prompt=%s\", C.GoString(p.initial_prompt))\n\tstr += fmt.Sprintf(\" entropy_thold=%f\", p.entropy_thold)\n\tstr += fmt.Sprintf(\" temperature=%f\", p.temperature)\n\tstr += fmt.Sprintf(\" temperature_inc=%f\", p.temperature_inc)\n\tstr += fmt.Sprintf(\" beam_size=%d\", p.beam_search.beam_size)\n\tif p.translate {\n\t\tstr += \" translate\"\n\t}\n\tif p.no_context {\n\t\tstr += \" no_context\"\n\t}\n\tif p.single_segment {\n\t\tstr += \" single_segment\"\n\t}\n\tif p.print_special {\n\t\tstr += \" print_special\"\n\t}\n\tif p.print_progress {\n\t\tstr += \" print_progress\"\n\t}\n\tif p.print_realtime {\n\t\tstr += \" print_realtime\"\n\t}\n\tif p.print_timestamps {\n\t\tstr += \" print_timestamps\"\n\t}\n\tif p.token_timestamps {\n\t\tstr += \" token_timestamps\"\n\t}\n\tif p.carry_initial_prompt {\n\t\tstr += \" carry_initial_prompt\"\n\t}\n\n\treturn str + \">\"\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/consts.go",
    "content": "package whisper\n\nimport (\n\t\"errors\"\n\n\t// Bindings\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// ERRORS\n\nvar (\n\tErrUnableToLoadModel    = errors.New(\"unable to load model\")\n\tErrInternalAppError     = errors.New(\"internal application error\")\n\tErrProcessingFailed     = errors.New(\"processing failed\")\n\tErrUnsupportedLanguage  = errors.New(\"unsupported language\")\n\tErrModelNotMultilingual = errors.New(\"model is not multilingual\")\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// CONSTANTS\n\n// SampleRate is the sample rate of the audio data.\nconst SampleRate = whisper.SampleRate\n\n// SampleBits is the number of bytes per sample.\nconst SampleBits = whisper.SampleBits\n"
  },
  {
    "path": "bindings/go/pkg/whisper/context.go",
    "content": "package whisper\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"runtime\"\n\t\"strings\"\n\t\"time\"\n\n\t// Bindings\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// TYPES\n\ntype context struct {\n\tn      int\n\tmodel  *model\n\tparams whisper.Params\n}\n\n// Make sure context adheres to the interface\nvar _ Context = (*context)(nil)\n\n///////////////////////////////////////////////////////////////////////////////\n// LIFECYCLE\n\nfunc newContext(model *model, params whisper.Params) (Context, error) {\n\tcontext := new(context)\n\tcontext.model = model\n\tcontext.params = params\n\n\t// Return success\n\treturn context, nil\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\n// Set the language to use for speech recognition.\nfunc (context *context) SetLanguage(lang string) error {\n\tif context.model.ctx == nil {\n\t\treturn ErrInternalAppError\n\t}\n\tif !context.model.IsMultilingual() {\n\t\treturn ErrModelNotMultilingual\n\t}\n\n\tif lang == \"auto\" {\n\t\tcontext.params.SetLanguage(-1)\n\t} else if id := context.model.ctx.Whisper_lang_id(lang); id < 0 {\n\t\treturn ErrUnsupportedLanguage\n\t} else if err := context.params.SetLanguage(id); err != nil {\n\t\treturn err\n\t}\n\t// Return success\n\treturn nil\n}\n\nfunc (context *context) IsMultilingual() bool {\n\treturn context.model.IsMultilingual()\n}\n\n// Get language\nfunc (context *context) Language() string {\n\tid := context.params.Language()\n\tif id == -1 {\n\t\treturn \"auto\"\n\t}\n\treturn whisper.Whisper_lang_str(context.params.Language())\n}\n\nfunc (context *context) DetectedLanguage() string {\n\treturn whisper.Whisper_lang_str(context.model.ctx.Whisper_full_lang_id())\n}\n\n// Set translate flag\nfunc (context *context) SetTranslate(v bool) {\n\tcontext.params.SetTranslate(v)\n}\n\n// Voice Activity Detection (VAD)\nfunc (context *context) SetVAD(v bool) {\n\tcontext.params.SetVAD(v)\n}\n\nfunc (context *context) SetVADModelPath(path string) {\n\tcontext.params.SetVADModelPath(path)\n}\n\nfunc (context *context) SetVADThreshold(t float32) {\n\tcontext.params.SetVADThreshold(t)\n}\n\nfunc (context *context) SetVADMinSpeechMs(ms int) {\n\tcontext.params.SetVADMinSpeechMs(ms)\n}\n\nfunc (context *context) SetVADMinSilenceMs(ms int) {\n\tcontext.params.SetVADMinSilenceMs(ms)\n}\n\nfunc (context *context) SetVADMaxSpeechSec(s float32) {\n\tcontext.params.SetVADMaxSpeechSec(s)\n}\n\nfunc (context *context) SetVADSpeechPadMs(ms int) {\n\tcontext.params.SetVADSpeechPadMs(ms)\n}\n\nfunc (context *context) SetVADSamplesOverlap(sec float32) {\n\tcontext.params.SetVADSamplesOverlap(sec)\n}\n\nfunc (context *context) SetSplitOnWord(v bool) {\n\tcontext.params.SetSplitOnWord(v)\n}\n\n// Set number of threads to use\nfunc (context *context) SetThreads(v uint) {\n\tcontext.params.SetThreads(int(v))\n}\n\n// Set time offset\nfunc (context *context) SetOffset(v time.Duration) {\n\tcontext.params.SetOffset(int(v.Milliseconds()))\n}\n\n// Set duration of audio to process\nfunc (context *context) SetDuration(v time.Duration) {\n\tcontext.params.SetDuration(int(v.Milliseconds()))\n}\n\n// Set timestamp token probability threshold (~0.01)\nfunc (context *context) SetTokenThreshold(t float32) {\n\tcontext.params.SetTokenThreshold(t)\n}\n\n// Set timestamp token sum probability threshold (~0.01)\nfunc (context *context) SetTokenSumThreshold(t float32) {\n\tcontext.params.SetTokenSumThreshold(t)\n}\n\n// Set max segment length in characters\nfunc (context *context) SetMaxSegmentLength(n uint) {\n\tcontext.params.SetMaxSegmentLength(int(n))\n}\n\n// Set token timestamps flag\nfunc (context *context) SetTokenTimestamps(b bool) {\n\tcontext.params.SetTokenTimestamps(b)\n}\n\n// Set max tokens per segment (0 = no limit)\nfunc (context *context) SetMaxTokensPerSegment(n uint) {\n\tcontext.params.SetMaxTokensPerSegment(int(n))\n}\n\n// Set audio encoder context\nfunc (context *context) SetAudioCtx(n uint) {\n\tcontext.params.SetAudioCtx(int(n))\n}\n\n// Set maximum number of text context tokens to store\nfunc (context *context) SetMaxContext(n int) {\n\tcontext.params.SetMaxContext(n)\n}\n\n// Set Beam Size\nfunc (context *context) SetBeamSize(n int) {\n\tcontext.params.SetBeamSize(n)\n}\n\n// Set Entropy threshold\nfunc (context *context) SetEntropyThold(t float32) {\n\tcontext.params.SetEntropyThold(t)\n}\n\n// Set Temperature\nfunc (context *context) SetTemperature(t float32) {\n\tcontext.params.SetTemperature(t)\n}\n\n// Set the fallback temperature incrementation\n// Pass -1.0 to disable this feature\nfunc (context *context) SetTemperatureFallback(t float32) {\n\tcontext.params.SetTemperatureFallback(t)\n}\n\n// Set initial prompt\nfunc (context *context) SetInitialPrompt(prompt string) {\n\tcontext.params.SetInitialPrompt(prompt)\n}\n\n// ResetTimings resets the mode timings. Should be called before processing\nfunc (context *context) ResetTimings() {\n\tcontext.model.ctx.Whisper_reset_timings()\n}\n\n// PrintTimings prints the model timings to stdout.\nfunc (context *context) PrintTimings() {\n\tcontext.model.ctx.Whisper_print_timings()\n}\n\n// SystemInfo returns the system information\nfunc (context *context) SystemInfo() string {\n\treturn fmt.Sprintf(\"system_info: n_threads = %d / %d | %s\\n\",\n\t\tcontext.params.Threads(),\n\t\truntime.NumCPU(),\n\t\twhisper.Whisper_print_system_info(),\n\t)\n}\n\n// Use mel data at offset_ms to try and auto-detect the spoken language\n// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\n// Returns the probabilities of all languages.\nfunc (context *context) WhisperLangAutoDetect(offset_ms int, n_threads int) ([]float32, error) {\n\tlangProbs, err := context.model.ctx.Whisper_lang_auto_detect(offset_ms, n_threads)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn langProbs, nil\n}\n\n// Process new sample data and return any errors\nfunc (context *context) Process(\n\tdata []float32,\n\tcallEncoderBegin EncoderBeginCallback,\n\tcallNewSegment SegmentCallback,\n\tcallProgress ProgressCallback,\n) error {\n\tif context.model.ctx == nil {\n\t\treturn ErrInternalAppError\n\t}\n\t// If the callback is defined then we force on single_segment mode\n\tif callNewSegment != nil {\n\t\tcontext.params.SetSingleSegment(true)\n\t}\n\n\t// We don't do parallel processing at the moment\n\tprocessors := 0\n\tif processors > 1 {\n\t\tif err := context.model.ctx.Whisper_full_parallel(context.params, data, processors, callEncoderBegin,\n\t\t\tfunc(new int) {\n\t\t\t\tif callNewSegment != nil {\n\t\t\t\t\tnum_segments := context.model.ctx.Whisper_full_n_segments()\n\t\t\t\t\ts0 := num_segments - new\n\t\t\t\t\tfor i := s0; i < num_segments; i++ {\n\t\t\t\t\t\tcallNewSegment(toSegment(context.model.ctx, i))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if err := context.model.ctx.Whisper_full(context.params, data, callEncoderBegin,\n\t\tfunc(new int) {\n\t\t\tif callNewSegment != nil {\n\t\t\t\tnum_segments := context.model.ctx.Whisper_full_n_segments()\n\t\t\t\ts0 := num_segments - new\n\t\t\t\tfor i := s0; i < num_segments; i++ {\n\t\t\t\t\tcallNewSegment(toSegment(context.model.ctx, i))\n\t\t\t\t}\n\t\t\t}\n\t\t}, func(progress int) {\n\t\t\tif callProgress != nil {\n\t\t\t\tcallProgress(progress)\n\t\t\t}\n\t\t}); err != nil {\n\t\treturn err\n\t}\n\n\t// Reset n so that more Segments can be available within NextSegment call\n\tcontext.n = 0\n\n\t// Return success\n\treturn nil\n}\n\n// Return the next segment of tokens\nfunc (context *context) NextSegment() (Segment, error) {\n\tif context.model.ctx == nil {\n\t\treturn Segment{}, ErrInternalAppError\n\t}\n\tif context.n >= context.model.ctx.Whisper_full_n_segments() {\n\t\treturn Segment{}, io.EOF\n\t}\n\n\t// Populate result\n\tresult := toSegment(context.model.ctx, context.n)\n\n\t// Increment the cursor\n\tcontext.n++\n\n\t// Return success\n\treturn result, nil\n}\n\n// Test for text tokens\nfunc (context *context) IsText(t Token) bool {\n\tswitch {\n\tcase context.IsBEG(t):\n\t\treturn false\n\tcase context.IsSOT(t):\n\t\treturn false\n\tcase whisper.Token(t.Id) >= context.model.ctx.Whisper_token_eot():\n\t\treturn false\n\tcase context.IsPREV(t):\n\t\treturn false\n\tcase context.IsSOLM(t):\n\t\treturn false\n\tcase context.IsNOT(t):\n\t\treturn false\n\tdefault:\n\t\treturn true\n\t}\n}\n\n// Test for \"begin\" token\nfunc (context *context) IsBEG(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_beg()\n}\n\n// Test for \"start of transcription\" token\nfunc (context *context) IsSOT(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_sot()\n}\n\n// Test for \"end of transcription\" token\nfunc (context *context) IsEOT(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_eot()\n}\n\n// Test for \"start of prev\" token\nfunc (context *context) IsPREV(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_prev()\n}\n\n// Test for \"start of lm\" token\nfunc (context *context) IsSOLM(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_solm()\n}\n\n// Test for \"No timestamps\" token\nfunc (context *context) IsNOT(t Token) bool {\n\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_not()\n}\n\n// Test for token associated with a specific language\nfunc (context *context) IsLANG(t Token, lang string) bool {\n\tif id := context.model.ctx.Whisper_lang_id(lang); id >= 0 {\n\t\treturn whisper.Token(t.Id) == context.model.ctx.Whisper_token_lang(id)\n\t} else {\n\t\treturn false\n\t}\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PRIVATE METHODS\n\nfunc toSegment(ctx *whisper.Context, n int) Segment {\n\treturn Segment{\n\t\tNum:    n,\n\t\tText:   strings.TrimSpace(ctx.Whisper_full_get_segment_text(n)),\n\t\tStart:  time.Duration(ctx.Whisper_full_get_segment_t0(n)) * time.Millisecond * 10,\n\t\tEnd:    time.Duration(ctx.Whisper_full_get_segment_t1(n)) * time.Millisecond * 10,\n\t\tTokens: toTokens(ctx, n),\n\t}\n}\n\nfunc toTokens(ctx *whisper.Context, n int) []Token {\n\tresult := make([]Token, ctx.Whisper_full_n_tokens(n))\n\tfor i := 0; i < len(result); i++ {\n\t\tdata := ctx.Whisper_full_get_token_data(n, i)\n\n\t\tresult[i] = Token{\n\t\t\tId:    int(ctx.Whisper_full_get_token_id(n, i)),\n\t\t\tText:  ctx.Whisper_full_get_token_text(n, i),\n\t\t\tP:     ctx.Whisper_full_get_token_p(n, i),\n\t\t\tStart: time.Duration(data.T0()) * time.Millisecond * 10,\n\t\t\tEnd:   time.Duration(data.T1()) * time.Millisecond * 10,\n\t\t}\n\t}\n\treturn result\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/context_test.go",
    "content": "package whisper_test\n\nimport (\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n\t\"github.com/go-audio/wav\"\n\tassert \"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSetLanguage(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\n\t// This returns an error since\n\t// the model 'models/ggml-small.en.bin'\n\t// that is loaded is not multilingual\n\terr = context.SetLanguage(\"en\")\n\tassert.Error(err)\n}\n\nfunc TestContextModelIsMultilingual(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\n\tisMultilingual := context.IsMultilingual()\n\n\t// This returns false since\n\t// the model 'models/ggml-small.en.bin'\n\t// that is loaded is not multilingual\n\tassert.False(isMultilingual)\n}\n\nfunc TestLanguage(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\n\t// This always returns en since\n\t// the model 'models/ggml-small.en.bin'\n\t// that is loaded is not multilingual\n\texpectedLanguage := \"en\"\n\tactualLanguage := context.Language()\n\tassert.Equal(expectedLanguage, actualLanguage)\n}\n\nfunc TestProcess(t *testing.T) {\n\tassert := assert.New(t)\n\n\tfh, err := os.Open(SamplePath)\n\tassert.NoError(err)\n\tdefer fh.Close()\n\n\t// Decode the WAV file - load the full buffer\n\tdec := wav.NewDecoder(fh)\n\tbuf, err := dec.FullPCMBuffer()\n\tassert.NoError(err)\n\tassert.Equal(uint16(1), dec.NumChans)\n\n\tdata := buf.AsFloat32Buffer().Data\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\n\terr = context.Process(data, nil, nil, nil)\n\tassert.NoError(err)\n}\n\nfunc TestDetectedLanguage(t *testing.T) {\n\tassert := assert.New(t)\n\n\tfh, err := os.Open(SamplePath)\n\tassert.NoError(err)\n\tdefer fh.Close()\n\n\t// Decode the WAV file - load the full buffer\n\tdec := wav.NewDecoder(fh)\n\tbuf, err := dec.FullPCMBuffer()\n\tassert.NoError(err)\n\tassert.Equal(uint16(1), dec.NumChans)\n\n\tdata := buf.AsFloat32Buffer().Data\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\n\terr = context.Process(data, nil, nil, nil)\n\tassert.NoError(err)\n\n\texpectedLanguage := \"en\"\n\tactualLanguage := context.DetectedLanguage()\n\tassert.Equal(expectedLanguage, actualLanguage)\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/doc.go",
    "content": "/*\nThis is the higher-level speech-to-text whisper.cpp API for go\n*/\npackage whisper\n"
  },
  {
    "path": "bindings/go/pkg/whisper/interface.go",
    "content": "package whisper\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// TYPES\n\n// SegmentCallback is the callback function for processing segments in real\n// time. It is called during the Process function\ntype SegmentCallback func(Segment)\n\n// ProgressCallback is the callback function for reporting progress during\n// processing. It is called during the Process function\ntype ProgressCallback func(int)\n\n// EncoderBeginCallback is the callback function for checking if we want to\n// continue processing. It is called during the Process function\ntype EncoderBeginCallback func() bool\n\n// Model is the interface to a whisper model. Create a new model with the\n// function whisper.New(string)\ntype Model interface {\n\tio.Closer\n\n\t// Return a new speech-to-text context.\n\tNewContext() (Context, error)\n\n\t// Return true if the model is multilingual.\n\tIsMultilingual() bool\n\n\t// Return all languages supported.\n\tLanguages() []string\n}\n\n// Context is the speech recognition context.\ntype Context interface {\n\tSetLanguage(string) error // Set the language to use for speech recognition, use \"auto\" for auto detect language.\n\tSetTranslate(bool)        // Set translate flag\n\tIsMultilingual() bool     // Return true if the model is multilingual.\n\tLanguage() string         // Get language\n\tDetectedLanguage() string // Get detected language\n\n\tSetOffset(time.Duration)          // Set offset\n\tSetDuration(time.Duration)        // Set duration\n\tSetThreads(uint)                  // Set number of threads to use\n\tSetSplitOnWord(bool)              // Set split on word flag\n\tSetTokenThreshold(float32)        // Set timestamp token probability threshold\n\tSetTokenSumThreshold(float32)     // Set timestamp token sum probability threshold\n\tSetMaxSegmentLength(uint)         // Set max segment length in characters\n\tSetTokenTimestamps(bool)          // Set token timestamps flag\n\tSetMaxTokensPerSegment(uint)      // Set max tokens per segment (0 = no limit)\n\tSetAudioCtx(uint)                 // Set audio encoder context\n\tSetMaxContext(n int)              // Set maximum number of text context tokens to store\n\tSetBeamSize(n int)                // Set Beam Size\n\tSetEntropyThold(t float32)        // Set Entropy threshold\n\tSetInitialPrompt(prompt string)   // Set initial prompt\n\tSetTemperature(t float32)         // Set temperature\n\tSetTemperatureFallback(t float32) // Set temperature incrementation\n\n\tSetVAD(v bool)\n\tSetVADModelPath(path string)\n\tSetVADThreshold(t float32)\n\tSetVADMinSpeechMs(ms int)\n\tSetVADMinSilenceMs(ms int)\n\tSetVADMaxSpeechSec(s float32)\n\tSetVADSpeechPadMs(ms int)\n\tSetVADSamplesOverlap(sec float32)\n\n\t// Process mono audio data and return any errors.\n\t// If defined, newly generated segments are passed to the\n\t// callback function during processing.\n\tProcess([]float32, EncoderBeginCallback, SegmentCallback, ProgressCallback) error\n\n\t// After process is called, return segments until the end of the stream\n\t// is reached, when io.EOF is returned.\n\tNextSegment() (Segment, error)\n\n\tIsBEG(Token) bool          // Test for \"begin\" token\n\tIsSOT(Token) bool          // Test for \"start of transcription\" token\n\tIsEOT(Token) bool          // Test for \"end of transcription\" token\n\tIsPREV(Token) bool         // Test for \"start of prev\" token\n\tIsSOLM(Token) bool         // Test for \"start of lm\" token\n\tIsNOT(Token) bool          // Test for \"No timestamps\" token\n\tIsLANG(Token, string) bool // Test for token associated with a specific language\n\tIsText(Token) bool         // Test for text token\n\n\t// Timings\n\tPrintTimings()\n\tResetTimings()\n\n\tSystemInfo() string\n}\n\n// Segment is the text result of a speech recognition.\ntype Segment struct {\n\t// Segment Number\n\tNum int\n\n\t// Time beginning and end timestamps for the segment.\n\tStart, End time.Duration\n\n\t// The text of the segment.\n\tText string\n\n\t// The tokens of the segment.\n\tTokens []Token\n}\n\n// Token is a text or special token\ntype Token struct {\n\tId         int\n\tText       string\n\tP          float32\n\tStart, End time.Duration\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/model.go",
    "content": "package whisper\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t// Bindings\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// TYPES\n\ntype model struct {\n\tpath string\n\tctx  *whisper.Context\n}\n\n// Make sure model adheres to the interface\nvar _ Model = (*model)(nil)\n\n///////////////////////////////////////////////////////////////////////////////\n// LIFECYCLE\n\nfunc New(path string) (Model, error) {\n\tmodel := new(model)\n\tif _, err := os.Stat(path); err != nil {\n\t\treturn nil, err\n\t} else if ctx := whisper.Whisper_init(path); ctx == nil {\n\t\treturn nil, ErrUnableToLoadModel\n\t} else {\n\t\tmodel.ctx = ctx\n\t\tmodel.path = path\n\t}\n\n\t// Return success\n\treturn model, nil\n}\n\nfunc (model *model) Close() error {\n\tif model.ctx != nil {\n\t\tmodel.ctx.Whisper_free()\n\t}\n\n\t// Release resources\n\tmodel.ctx = nil\n\n\t// Return success\n\treturn nil\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// STRINGIFY\n\nfunc (model *model) String() string {\n\tstr := \"<whisper.model\"\n\tif model.ctx != nil {\n\t\tstr += fmt.Sprintf(\" model=%q\", model.path)\n\t}\n\treturn str + \">\"\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\n// Return true if model is multilingual (language and translation options are supported)\nfunc (model *model) IsMultilingual() bool {\n\treturn model.ctx.Whisper_is_multilingual() != 0\n}\n\n// Return all recognized languages. Initially it is set to auto-detect\nfunc (model *model) Languages() []string {\n\tresult := make([]string, 0, whisper.Whisper_lang_max_id())\n\tfor i := 0; i < whisper.Whisper_lang_max_id(); i++ {\n\t\tstr := whisper.Whisper_lang_str(i)\n\t\tif model.ctx.Whisper_lang_id(str) >= 0 {\n\t\t\tresult = append(result, str)\n\t\t}\n\t}\n\treturn result\n}\n\nfunc (model *model) NewContext() (Context, error) {\n\tif model.ctx == nil {\n\t\treturn nil, ErrInternalAppError\n\t}\n\n\t// Create new context\n\tparams := model.ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY)\n\tparams.SetTranslate(false)\n\tparams.SetPrintSpecial(false)\n\tparams.SetPrintProgress(false)\n\tparams.SetPrintRealtime(false)\n\tparams.SetPrintTimestamps(false)\n\tparams.SetThreads(runtime.NumCPU())\n\tparams.SetNoContext(true)\n\n\t// Return new context\n\treturn newContext(model, params)\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/model_test.go",
    "content": "package whisper_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper\"\n\tassert \"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNew(t *testing.T) {\n\tassert := assert.New(t)\n\tt.Run(\"valid model path\", func(t *testing.T) {\n\t\tmodel, err := whisper.New(ModelPath)\n\t\tassert.NoError(err)\n\t\tassert.NotNil(model)\n\t\tdefer model.Close()\n\n\t})\n\n\tt.Run(\"invalid model path\", func(t *testing.T) {\n\t\tinvalidModelPath := \"invalid-model-path.bin\"\n\t\tmodel, err := whisper.New(invalidModelPath)\n\t\tassert.Error(err)\n\t\tassert.Nil(model)\n\t})\n}\n\nfunc TestClose(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\n\terr = model.Close()\n\tassert.NoError(err)\n}\n\nfunc TestNewContext(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tcontext, err := model.NewContext()\n\tassert.NoError(err)\n\tassert.NotNil(context)\n}\n\nfunc TestIsMultilingual(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\tisMultilingual := model.IsMultilingual()\n\n\t// This returns false since\n\t// the model 'models/ggml-small.en.bin'\n\t// that is loaded is not multilingual\n\tassert.False(isMultilingual)\n}\n\nfunc TestLanguages(t *testing.T) {\n\tassert := assert.New(t)\n\n\tmodel, err := whisper.New(ModelPath)\n\tassert.NoError(err)\n\tassert.NotNil(model)\n\tdefer model.Close()\n\n\texpectedLanguages := []string{\n\t\t\"en\", \"zh\", \"de\", \"es\", \"ru\", \"ko\", \"fr\", \"ja\", \"pt\", \"tr\", \"pl\",\n\t\t\"ca\", \"nl\", \"ar\", \"sv\", \"it\", \"id\", \"hi\", \"fi\", \"vi\", \"he\", \"uk\",\n\t\t\"el\", \"ms\", \"cs\", \"ro\", \"da\", \"hu\", \"ta\", \"no\", \"th\", \"ur\", \"hr\",\n\t\t\"bg\", \"lt\", \"la\", \"mi\", \"ml\", \"cy\", \"sk\", \"te\", \"fa\", \"lv\", \"bn\",\n\t\t\"sr\", \"az\", \"sl\", \"kn\", \"et\", \"mk\", \"br\", \"eu\", \"is\", \"hy\", \"ne\",\n\t\t\"mn\", \"bs\", \"kk\", \"sq\", \"sw\", \"gl\", \"mr\", \"pa\", \"si\", \"km\", \"sn\",\n\t\t\"yo\", \"so\", \"af\", \"oc\", \"ka\", \"be\", \"tg\", \"sd\", \"gu\", \"am\", \"yi\",\n\t\t\"lo\", \"uz\", \"fo\", \"ht\", \"ps\", \"tk\", \"nn\", \"mt\", \"sa\", \"lb\", \"my\",\n\t\t\"bo\", \"tl\", \"mg\", \"as\", \"tt\", \"haw\", \"ln\", \"ha\", \"ba\", \"jw\", \"su\",\n\t}\n\n\tactualLanguages := model.Languages()\n\n\tassert.Equal(expectedLanguages, actualLanguages)\n}\n"
  },
  {
    "path": "bindings/go/pkg/whisper/util_test.go",
    "content": "package whisper_test\n\nconst (\n\tModelPath  = \"../../models/ggml-small.en.bin\"\n\tSamplePath = \"../../samples/jfk.wav\"\n)\n"
  },
  {
    "path": "bindings/go/whisper.go",
    "content": "package whisper\n\nimport (\n\t\"errors\"\n\t\"unsafe\"\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// CGO\n\n/*\n#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu -lm -lstdc++\n#cgo linux LDFLAGS: -fopenmp\n#cgo darwin LDFLAGS: -lggml-metal -lggml-blas\n#cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics\n#include <whisper.h>\n#include <stdlib.h>\n\nextern void callNewSegment(void* user_data, int new);\nextern void callProgress(void* user_data, int progress);\nextern bool callEncoderBegin(void* user_data);\n\n// Text segment callback\n// Called on every newly generated text segment\n// Use the whisper_full_...() functions to obtain the text segments\nstatic void whisper_new_segment_cb(struct whisper_context* ctx, struct whisper_state* state, int n_new, void* user_data) {\n    if(user_data != NULL && ctx != NULL) {\n        callNewSegment(user_data, n_new);\n    }\n}\n\n// Progress callback\n// Called on every newly generated text segment\n// Use the whisper_full_...() functions to obtain the text segments\nstatic void whisper_progress_cb(struct whisper_context* ctx, struct whisper_state* state, int progress, void* user_data) {\n    if(user_data != NULL && ctx != NULL) {\n        callProgress(user_data, progress);\n    }\n}\n\n// Encoder begin callback\n// If not NULL, called before the encoder starts\n// If it returns false, the computation is aborted\nstatic bool whisper_encoder_begin_cb(struct whisper_context* ctx, struct whisper_state* state, void* user_data) {\n    if(user_data != NULL && ctx != NULL) {\n        return callEncoderBegin(user_data);\n    }\n    return false;\n}\n\n// Get default parameters and set callbacks\nstatic struct whisper_full_params whisper_full_default_params_cb(struct whisper_context* ctx, enum whisper_sampling_strategy strategy) {\n\tstruct whisper_full_params params = whisper_full_default_params(strategy);\n\tparams.new_segment_callback = whisper_new_segment_cb;\n\tparams.new_segment_callback_user_data = (void*)(ctx);\n\tparams.encoder_begin_callback = whisper_encoder_begin_cb;\n\tparams.encoder_begin_callback_user_data = (void*)(ctx);\n\tparams.progress_callback = whisper_progress_cb;\n\tparams.progress_callback_user_data = (void*)(ctx);\n\treturn params;\n}\n*/\nimport \"C\"\n\n///////////////////////////////////////////////////////////////////////////////\n// TYPES\n\ntype (\n\tContext          C.struct_whisper_context\n\tToken            C.whisper_token\n\tTokenData        C.struct_whisper_token_data\n\tSamplingStrategy C.enum_whisper_sampling_strategy\n\tParams           C.struct_whisper_full_params\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// GLOBALS\n\nconst (\n\tSAMPLING_GREEDY      SamplingStrategy = C.WHISPER_SAMPLING_GREEDY\n\tSAMPLING_BEAM_SEARCH SamplingStrategy = C.WHISPER_SAMPLING_BEAM_SEARCH\n)\n\nconst (\n\tSampleRate = C.WHISPER_SAMPLE_RATE                 // Expected sample rate, samples per second\n\tSampleBits = uint16(unsafe.Sizeof(C.float(0))) * 8 // Sample size in bits\n\tNumFFT     = C.WHISPER_N_FFT\n\tHopLength  = C.WHISPER_HOP_LENGTH\n\tChunkSize  = C.WHISPER_CHUNK_SIZE\n)\n\nvar (\n\tErrTokenizerFailed  = errors.New(\"whisper_tokenize failed\")\n\tErrAutoDetectFailed = errors.New(\"whisper_lang_auto_detect failed\")\n\tErrConversionFailed = errors.New(\"whisper_convert failed\")\n\tErrInvalidLanguage  = errors.New(\"invalid language\")\n)\n\n///////////////////////////////////////////////////////////////////////////////\n// PUBLIC METHODS\n\n// Allocates all memory needed for the model and loads the model from the given file.\n// Returns NULL on failure.\nfunc Whisper_init(path string) *Context {\n\tcPath := C.CString(path)\n\tdefer C.free(unsafe.Pointer(cPath))\n\tif ctx := C.whisper_init_from_file_with_params(cPath, C.whisper_context_default_params()); ctx != nil {\n\t\treturn (*Context)(ctx)\n\t} else {\n\t\treturn nil\n\t}\n}\n\n// Frees all memory allocated by the model.\nfunc (ctx *Context) Whisper_free() {\n\tC.whisper_free((*C.struct_whisper_context)(ctx))\n}\n\n// Convert RAW PCM audio to log mel spectrogram.\n// The resulting spectrogram is stored inside the provided whisper context.\nfunc (ctx *Context) Whisper_pcm_to_mel(data []float32, threads int) error {\n\tif C.whisper_pcm_to_mel((*C.struct_whisper_context)(ctx), (*C.float)(&data[0]), C.int(len(data)), C.int(threads)) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// This can be used to set a custom log mel spectrogram inside the provided whisper context.\n// Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.\n// n_mel must be 80\nfunc (ctx *Context) Whisper_set_mel(data []float32, n_mel int) error {\n\tif C.whisper_set_mel((*C.struct_whisper_context)(ctx), (*C.float)(&data[0]), C.int(len(data)), C.int(n_mel)) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// Run the Whisper encoder on the log mel spectrogram stored inside the provided whisper context.\n// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\n// offset can be used to specify the offset of the first frame in the spectrogram.\nfunc (ctx *Context) Whisper_encode(offset, threads int) error {\n\tif C.whisper_encode((*C.struct_whisper_context)(ctx), C.int(offset), C.int(threads)) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// Run the Whisper decoder to obtain the logits and probabilities for the next token.\n// Make sure to call whisper_encode() first.\n// tokens + n_tokens is the provided context for the decoder.\n// n_past is the number of tokens to use from previous decoder calls.\nfunc (ctx *Context) Whisper_decode(tokens []Token, past, threads int) error {\n\tif C.whisper_decode((*C.struct_whisper_context)(ctx), (*C.whisper_token)(&tokens[0]), C.int(len(tokens)), C.int(past), C.int(threads)) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// Convert the provided text into tokens. The tokens pointer must be large enough to hold the resulting tokens.\n// Returns the number of tokens on success\nfunc (ctx *Context) Whisper_tokenize(text string, tokens []Token) (int, error) {\n\tcText := C.CString(text)\n\tdefer C.free(unsafe.Pointer(cText))\n\tif n := C.whisper_tokenize((*C.struct_whisper_context)(ctx), cText, (*C.whisper_token)(&tokens[0]), C.int(len(tokens))); n >= 0 {\n\t\treturn int(n), nil\n\t} else {\n\t\treturn 0, ErrTokenizerFailed\n\t}\n}\n\n// Return the id of the specified language, returns -1 if not found\n// Examples:\n//\n//\t\"de\" -> 2\n//\t\"german\" -> 2\nfunc (ctx *Context) Whisper_lang_id(lang string) int {\n\treturn int(C.whisper_lang_id(C.CString(lang)))\n}\n\n// Largest language id (i.e. number of available languages - 1)\nfunc Whisper_lang_max_id() int {\n\treturn int(C.whisper_lang_max_id())\n}\n\n// Return the short string of the specified language id (e.g. 2 -> \"de\"),\n// returns empty string if not found\nfunc Whisper_lang_str(id int) string {\n\treturn C.GoString(C.whisper_lang_str(C.int(id)))\n}\n\n// Use mel data at offset_ms to try and auto-detect the spoken language\n// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\n// Returns the probabilities of all languages.\n// ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69\nfunc (ctx *Context) Whisper_lang_auto_detect(offset_ms, n_threads int) ([]float32, error) {\n\tprobs := make([]float32, Whisper_lang_max_id()+1)\n\tif n := int(C.whisper_lang_auto_detect((*C.struct_whisper_context)(ctx), C.int(offset_ms), C.int(n_threads), (*C.float)(&probs[0]))); n < 0 {\n\t\treturn nil, ErrAutoDetectFailed\n\t} else {\n\t\treturn probs, nil\n\t}\n}\n\nfunc (ctx *Context) Whisper_n_len() int {\n\treturn int(C.whisper_n_len((*C.struct_whisper_context)(ctx)))\n}\n\nfunc (ctx *Context) Whisper_n_vocab() int {\n\treturn int(C.whisper_n_vocab((*C.struct_whisper_context)(ctx)))\n}\n\nfunc (ctx *Context) Whisper_n_text_ctx() int {\n\treturn int(C.whisper_n_text_ctx((*C.struct_whisper_context)(ctx)))\n}\n\nfunc (ctx *Context) Whisper_n_audio_ctx() int {\n\treturn int(C.whisper_n_audio_ctx((*C.struct_whisper_context)(ctx)))\n}\n\nfunc (ctx *Context) Whisper_is_multilingual() int {\n\treturn int(C.whisper_is_multilingual((*C.struct_whisper_context)(ctx)))\n}\n\n// The probabilities for the next token\n//func (ctx *Whisper_context) Whisper_get_probs() []float32 {\n//\treturn (*[1 << 30]float32)(unsafe.Pointer(C.whisper_get_probs((*C.struct_whisper_context)(ctx))))[:ctx.Whisper_n_vocab()]\n//}\n\n// Token Id -> String. Uses the vocabulary in the provided context\nfunc (ctx *Context) Whisper_token_to_str(token Token) string {\n\treturn C.GoString(C.whisper_token_to_str((*C.struct_whisper_context)(ctx), C.whisper_token(token)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_eot() Token {\n\treturn Token(C.whisper_token_eot((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_sot() Token {\n\treturn Token(C.whisper_token_sot((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_prev() Token {\n\treturn Token(C.whisper_token_prev((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_solm() Token {\n\treturn Token(C.whisper_token_solm((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_not() Token {\n\treturn Token(C.whisper_token_not((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_beg() Token {\n\treturn Token(C.whisper_token_beg((*C.struct_whisper_context)(ctx)))\n}\n\n// Special tokens\nfunc (ctx *Context) Whisper_token_lang(lang_id int) Token {\n\treturn Token(C.whisper_token_lang((*C.struct_whisper_context)(ctx), C.int(lang_id)))\n}\n\n// Task tokens\nfunc (ctx *Context) Whisper_token_translate() Token {\n\treturn Token(C.whisper_token_translate((*C.struct_whisper_context)(ctx)))\n}\n\n// Task tokens\nfunc (ctx *Context) Whisper_token_transcribe() Token {\n\treturn Token(C.whisper_token_transcribe((*C.struct_whisper_context)(ctx)))\n}\n\n// Performance information\nfunc (ctx *Context) Whisper_print_timings() {\n\tC.whisper_print_timings((*C.struct_whisper_context)(ctx))\n}\n\n// Performance information\nfunc (ctx *Context) Whisper_reset_timings() {\n\tC.whisper_reset_timings((*C.struct_whisper_context)(ctx))\n}\n\n// Print system information\nfunc Whisper_print_system_info() string {\n\treturn C.GoString(C.whisper_print_system_info())\n}\n\n// Return default parameters for a strategy\nfunc (ctx *Context) Whisper_full_default_params(strategy SamplingStrategy) Params {\n\t// Get default parameters\n\treturn Params(C.whisper_full_default_params_cb((*C.struct_whisper_context)(ctx), C.enum_whisper_sampling_strategy(strategy)))\n}\n\n// Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n// Uses the specified decoding strategy to obtain the text.\nfunc (ctx *Context) Whisper_full(\n\tparams Params,\n\tsamples []float32,\n\tencoderBeginCallback func() bool,\n\tnewSegmentCallback func(int),\n\tprogressCallback func(int),\n) error {\n\tregisterEncoderBeginCallback(ctx, encoderBeginCallback)\n\tregisterNewSegmentCallback(ctx, newSegmentCallback)\n\tregisterProgressCallback(ctx, progressCallback)\n\tdefer registerEncoderBeginCallback(ctx, nil)\n\tdefer registerNewSegmentCallback(ctx, nil)\n\tdefer registerProgressCallback(ctx, nil)\n\tif C.whisper_full((*C.struct_whisper_context)(ctx), (C.struct_whisper_full_params)(params), (*C.float)(&samples[0]), C.int(len(samples))) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// Split the input audio in chunks and process each chunk separately using whisper_full()\n// It seems this approach can offer some speedup in some cases.\n// However, the transcription accuracy can be worse at the beginning and end of each chunk.\nfunc (ctx *Context) Whisper_full_parallel(params Params, samples []float32, processors int, encoderBeginCallback func() bool, newSegmentCallback func(int)) error {\n\tregisterEncoderBeginCallback(ctx, encoderBeginCallback)\n\tregisterNewSegmentCallback(ctx, newSegmentCallback)\n\tdefer registerEncoderBeginCallback(ctx, nil)\n\tdefer registerNewSegmentCallback(ctx, nil)\n\n\tif C.whisper_full_parallel((*C.struct_whisper_context)(ctx), (C.struct_whisper_full_params)(params), (*C.float)(&samples[0]), C.int(len(samples)), C.int(processors)) == 0 {\n\t\treturn nil\n\t} else {\n\t\treturn ErrConversionFailed\n\t}\n}\n\n// Return the id of the autodetected language, returns -1 if not found\n// Added to whisper.cpp in\n// https://github.com/ggerganov/whisper.cpp/commit/a1c1583cc7cd8b75222857afc936f0638c5683d6\n//\n// Examples:\n//\n//\t\"de\" -> 2\n//\t\"german\" -> 2\nfunc (ctx *Context) Whisper_full_lang_id() int {\n\treturn int(C.whisper_full_lang_id((*C.struct_whisper_context)(ctx)))\n}\n\n// Number of generated text segments.\n// A segment can be a few words, a sentence, or even a paragraph.\nfunc (ctx *Context) Whisper_full_n_segments() int {\n\treturn int(C.whisper_full_n_segments((*C.struct_whisper_context)(ctx)))\n}\n\n// Get the start and end time of the specified segment.\nfunc (ctx *Context) Whisper_full_get_segment_t0(segment int) int64 {\n\treturn int64(C.whisper_full_get_segment_t0((*C.struct_whisper_context)(ctx), C.int(segment)))\n}\n\n// Get the start and end time of the specified segment.\nfunc (ctx *Context) Whisper_full_get_segment_t1(segment int) int64 {\n\treturn int64(C.whisper_full_get_segment_t1((*C.struct_whisper_context)(ctx), C.int(segment)))\n}\n\n// Get the text of the specified segment.\nfunc (ctx *Context) Whisper_full_get_segment_text(segment int) string {\n\treturn C.GoString(C.whisper_full_get_segment_text((*C.struct_whisper_context)(ctx), C.int(segment)))\n}\n\n// Get number of tokens in the specified segment.\nfunc (ctx *Context) Whisper_full_n_tokens(segment int) int {\n\treturn int(C.whisper_full_n_tokens((*C.struct_whisper_context)(ctx), C.int(segment)))\n}\n\n// Get the token text of the specified token index in the specified segment.\nfunc (ctx *Context) Whisper_full_get_token_text(segment int, token int) string {\n\treturn C.GoString(C.whisper_full_get_token_text((*C.struct_whisper_context)(ctx), C.int(segment), C.int(token)))\n}\n\n// Get the token of the specified token index in the specified segment.\nfunc (ctx *Context) Whisper_full_get_token_id(segment int, token int) Token {\n\treturn Token(C.whisper_full_get_token_id((*C.struct_whisper_context)(ctx), C.int(segment), C.int(token)))\n}\n\n// Get token data for the specified token in the specified segment.\n// This contains probabilities, timestamps, etc.\nfunc (ctx *Context) Whisper_full_get_token_data(segment int, token int) TokenData {\n\treturn TokenData(C.whisper_full_get_token_data((*C.struct_whisper_context)(ctx), C.int(segment), C.int(token)))\n}\n\n// Get the probability of the specified token in the specified segment.\nfunc (ctx *Context) Whisper_full_get_token_p(segment int, token int) float32 {\n\treturn float32(C.whisper_full_get_token_p((*C.struct_whisper_context)(ctx), C.int(segment), C.int(token)))\n}\n\n///////////////////////////////////////////////////////////////////////////////\n// CALLBACKS\n\nvar (\n\tcbNewSegment   = make(map[unsafe.Pointer]func(int))\n\tcbProgress     = make(map[unsafe.Pointer]func(int))\n\tcbEncoderBegin = make(map[unsafe.Pointer]func() bool)\n)\n\nfunc registerNewSegmentCallback(ctx *Context, fn func(int)) {\n\tif fn == nil {\n\t\tdelete(cbNewSegment, unsafe.Pointer(ctx))\n\t} else {\n\t\tcbNewSegment[unsafe.Pointer(ctx)] = fn\n\t}\n}\n\nfunc registerProgressCallback(ctx *Context, fn func(int)) {\n\tif fn == nil {\n\t\tdelete(cbProgress, unsafe.Pointer(ctx))\n\t} else {\n\t\tcbProgress[unsafe.Pointer(ctx)] = fn\n\t}\n}\n\nfunc registerEncoderBeginCallback(ctx *Context, fn func() bool) {\n\tif fn == nil {\n\t\tdelete(cbEncoderBegin, unsafe.Pointer(ctx))\n\t} else {\n\t\tcbEncoderBegin[unsafe.Pointer(ctx)] = fn\n\t}\n}\n\n//export callNewSegment\nfunc callNewSegment(user_data unsafe.Pointer, new C.int) {\n\tif fn, ok := cbNewSegment[user_data]; ok {\n\t\tfn(int(new))\n\t}\n}\n\n//export callProgress\nfunc callProgress(user_data unsafe.Pointer, progress C.int) {\n\tif fn, ok := cbProgress[user_data]; ok {\n\t\tfn(int(progress))\n\t}\n}\n\n//export callEncoderBegin\nfunc callEncoderBegin(user_data unsafe.Pointer) C.bool {\n\tif fn, ok := cbEncoderBegin[user_data]; ok {\n\t\tif fn() {\n\t\t\treturn C.bool(true)\n\t\t} else {\n\t\t\treturn C.bool(false)\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (t TokenData) T0() int64 {\n\treturn int64(t.t0)\n}\n\nfunc (t TokenData) T1() int64 {\n\treturn int64(t.t1)\n}\n\nfunc (t TokenData) Id() Token {\n\treturn Token(t.id)\n}\n"
  },
  {
    "path": "bindings/go/whisper_test.go",
    "content": "package whisper_test\n\nimport (\n\t\"os\"\n\t\"runtime\"\n\t\"testing\"\n\t\"time\"\n\n\t// Packages\n\twhisper \"github.com/ggerganov/whisper.cpp/bindings/go\"\n\twav \"github.com/go-audio/wav\"\n\tassert \"github.com/stretchr/testify/assert\"\n)\n\nconst (\n\tModelPath  = \"models/ggml-small.en.bin\"\n\tSamplePath = \"samples/jfk.wav\"\n)\n\nfunc Test_Whisper_000(t *testing.T) {\n\tassert := assert.New(t)\n\tif _, err := os.Stat(ModelPath); os.IsNotExist(err) {\n\t\tt.Skip(\"Skipping test, model not found:\", ModelPath)\n\t}\n\tctx := whisper.Whisper_init(ModelPath)\n\tassert.NotNil(ctx)\n\tctx.Whisper_free()\n}\n\nfunc Test_Whisper_001(t *testing.T) {\n\tassert := assert.New(t)\n\tif _, err := os.Stat(ModelPath); os.IsNotExist(err) {\n\t\tt.Skip(\"Skipping test, model not found:\", ModelPath)\n\t}\n\tif _, err := os.Stat(SamplePath); os.IsNotExist(err) {\n\t\tt.Skip(\"Skipping test, sample not found:\", SamplePath)\n\t}\n\n\t// Open samples\n\tfh, err := os.Open(SamplePath)\n\tassert.NoError(err)\n\tdefer fh.Close()\n\n\t// Read samples\n\td := wav.NewDecoder(fh)\n\tbuf, err := d.FullPCMBuffer()\n\tassert.NoError(err)\n\n\t// Run whisper\n\tctx := whisper.Whisper_init(ModelPath)\n\tassert.NotNil(ctx)\n\tdefer ctx.Whisper_free()\n\tparams := ctx.Whisper_full_default_params(whisper.SAMPLING_GREEDY)\n\tdata := buf.AsFloat32Buffer().Data\n\terr = ctx.Whisper_full(params, data, nil, nil, nil)\n\tassert.NoError(err)\n\n\t// Print out tokens\n\tnum_segments := ctx.Whisper_full_n_segments()\n\tassert.GreaterOrEqual(num_segments, 1)\n\tfor i := 0; i < num_segments; i++ {\n\t\tstr := ctx.Whisper_full_get_segment_text(i)\n\t\tassert.NotEmpty(str)\n\t\tt0 := time.Duration(ctx.Whisper_full_get_segment_t0(i)) * time.Millisecond\n\t\tt1 := time.Duration(ctx.Whisper_full_get_segment_t1(i)) * time.Millisecond\n\t\tt.Logf(\"[%6s->%-6s] %q\", t0, t1, str)\n\t}\n}\n\nfunc Test_Whisper_002(t *testing.T) {\n\tassert := assert.New(t)\n\tfor i := 0; i < whisper.Whisper_lang_max_id(); i++ {\n\t\tstr := whisper.Whisper_lang_str(i)\n\t\tassert.NotEmpty(str)\n\t\tt.Log(str)\n\t}\n}\n\nfunc Test_Whisper_003(t *testing.T) {\n\tthreads := runtime.NumCPU()\n\tassert := assert.New(t)\n\tif _, err := os.Stat(ModelPath); os.IsNotExist(err) {\n\t\tt.Skip(\"Skipping test, model not found:\", ModelPath)\n\t}\n\tif _, err := os.Stat(SamplePath); os.IsNotExist(err) {\n\t\tt.Skip(\"Skipping test, sample not found:\", SamplePath)\n\t}\n\n\t// Open samples\n\tfh, err := os.Open(SamplePath)\n\tassert.NoError(err)\n\tdefer fh.Close()\n\n\t// Read samples\n\td := wav.NewDecoder(fh)\n\tbuf, err := d.FullPCMBuffer()\n\tassert.NoError(err)\n\n\t// Make the model\n\tctx := whisper.Whisper_init(ModelPath)\n\tassert.NotNil(ctx)\n\tdefer ctx.Whisper_free()\n\n\t// Get MEL\n\tassert.NoError(ctx.Whisper_pcm_to_mel(buf.AsFloat32Buffer().Data, threads))\n\n\t// Get Languages\n\tlanguages, err := ctx.Whisper_lang_auto_detect(0, threads)\n\tassert.NoError(err)\n\tfor i, p := range languages {\n\t\tt.Logf(\"%s: %f\", whisper.Whisper_lang_str(i), p)\n\t}\n}\n"
  },
  {
    "path": "bindings/java/README.md",
    "content": "# Java JNI bindings for Whisper\n\nThis package provides Java JNI bindings for whisper.cpp. They have been tested on:\n\n  * <strike>Darwin (OS X) 12.6 on x64_64</strike>\n  * Ubuntu on x86_64\n  * Windows on x86_64\n\nThe \"low level\" bindings are in `WhisperCppJnaLibrary`. The most simple usage is as follows:\n\nJNA will attempt to load the `whispercpp` shared library from:\n\n- jna.library.path\n- jna.platform.library\n- ~/Library/Frameworks\n- /Library/Frameworks\n- /System/Library/Frameworks\n- classpath\n\n```java\nimport io.github.ggerganov.whispercpp.WhisperCpp;\n\npublic class Example {\n\n    public static void main(String[] args) {\n        \n        WhisperCpp whisper = new WhisperCpp();\n        try {\n            // By default, models are loaded from ~/.cache/whisper/ and are usually named \"ggml-${name}.bin\"\n            // or you can provide the absolute path to the model file.\n            whisper.initContext(\"../ggml-base.en.bin\"); \n            WhisperFullParams.ByValue whisperParams = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH); \n            \n            // custom configuration if required      \n            //whisperParams.n_threads = 8;\n            whisperParams.temperature = 0.0f;\n            whisperParams.temperature_inc = 0.2f;\n            //whisperParams.language = \"en\";\n                            \n            float[] samples = readAudio(); // divide each value by 32767.0f\n            List<WhisperSegment> whisperSegmentList = whisper.fullTranscribeWithTime(whisperParams, samples);\n            \n            for (WhisperSegment whisperSegment : whisperSegmentList) {\n\n                long start = whisperSegment.getStart();\n                long end = whisperSegment.getEnd();\n\n                String text = whisperSegment.getSentence();\n                    \n                System.out.println(\"start: \"+start);\n                System.out.println(\"end: \"+end);\n                System.out.println(\"text: \"+text);\n                \n            }\n    \n        } catch (IOException e) {\n            e.printStackTrace();\n        } finally {\n            whisper.close();\n        }\n        \n     }\n}\n```\n\n## Building & Testing\n\nIn order to build, you need to have the JDK 8 or higher installed. Run the tests with:\n\n```bash\ngit clone https://github.com/ggml-org/whisper.cpp.git\ncd whisper.cpp/bindings/java\n\n./gradlew build\n```\n\nYou need to have the `whisper` library in your [JNA library path](https://java-native-access.github.io/jna/4.2.1/com/sun/jna/NativeLibrary.html). On Windows the dll is included in the jar and you can update it:\n\n```bash\ncopy /y ..\\..\\build\\bin\\Release\\whisper.dll build\\generated\\resources\\main\\win32-x86-64\\whisper.dll\n```\n\n\n## License\n\nThe license for the Java bindings is the same as the license for the rest of the whisper.cpp project, which is the MIT License. See the `LICENSE` file for more details.\n\n"
  },
  {
    "path": "bindings/java/build.gradle",
    "content": "plugins {\r\n    id 'java'\r\n    id 'java-library'\r\n    id 'maven-publish'\r\n    id 'signing'\r\n}\r\n\r\narchivesBaseName = 'whispercpp'\r\ngroup = 'io.github.ggerganov'\r\nversion = '1.4.0'\r\n\r\n\r\nsourceCompatibility = 1.8\r\ntargetCompatibility = 1.8\r\n\r\nsourceSets {\r\n    main {\r\n        resources {\r\n            srcDirs = ['src/main/resources', 'build/generated/resources/main']\r\n        }\r\n    }\r\n    test {\r\n        runtimeClasspath += files('build/generated/resources/main')\r\n    }\r\n}\r\n\r\ntasks.register('copyLibwhisperDynlib', Copy) {\r\n    from '../../build/src'\r\n    include 'libwhisper.dylib'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyLibwhisperSo', Copy) {\r\n    from '../../build/src'\r\n    include 'libwhisper.so'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyWhisperDLL', Copy) {\r\n    from '../../build/bin/Release'\r\n    include 'whisper.dll'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyGGML_BASE_DLL', Copy) {\r\n    from '../../build/bin/Release'\r\n    include 'ggml-base.dll'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyGGML_DLL', Copy) {\r\n    from '../../build/bin/Release'\r\n    include 'ggml.dll'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyGGML_CPU_DLL', Copy) {\r\n    from '../../build/bin/Release'\r\n    include 'ggml-cpu.dll'\r\n    into 'build/generated/resources/main'\r\n}\r\n\r\ntasks.register('copyLibs') {\r\n    dependsOn copyLibwhisperDynlib, copyLibwhisperSo, copyWhisperDLL, copyGGML_BASE_DLL, copyGGML_DLL, copyGGML_CPU_DLL\r\n}\r\n\r\ntest {\r\n    systemProperty 'jna.library.path', project.file('build/generated/resources/main').absolutePath\r\n}\r\n\r\njava {\r\n    withSourcesJar()\r\n    withJavadocJar()\r\n}\r\n\r\nsourcesJar() {\r\n    dependsOn copyLibs\r\n}\r\n\r\njar {\r\n    dependsOn copyLibs\r\n    exclude '**/whisper_java.exp', '**/whisper_java.lib'\r\n}\r\n\r\njavadoc {\r\n    options.addStringOption('Xdoclint:none', '-quiet')\r\n}\r\n\r\ntasks.withType(Test) {\r\n    useJUnitPlatform()\r\n}\r\n\r\ntest.dependsOn copyLibs\r\nprocessResources.dependsOn copyLibs\r\n\r\ndependencies {\r\n    implementation \"net.java.dev.jna:jna:5.13.0\"\r\n    testImplementation \"org.junit.jupiter:junit-jupiter:5.9.2\"\r\n    testImplementation \"org.assertj:assertj-core:3.24.2\"\r\n}\r\n\r\nrepositories {\r\n    mavenCentral()\r\n}\r\n\r\npublishing {\r\n    publications {\r\n        mavenJava(MavenPublication) {\r\n            artifactId = 'whispercpp'\r\n            from components.java\r\n            pom {\r\n                name = 'whispercpp'\r\n                description = \"Java JNA bindings for OpenAI's Whisper model, implemented in C/C++\"\r\n                url = 'https://github.com/ggerganov/whisper.cpp'\r\n                licenses {\r\n                    license {\r\n                        name = 'MIT licence'\r\n                        url = 'https://raw.githubusercontent.com/ggerganov/whisper.cpp/master/LICENSE'\r\n                    }\r\n                }\r\n                developers {\r\n                    developer {\r\n                        id = 'ggerganov'\r\n                        name = 'Georgi Gerganov'\r\n                        email = 'ggerganov@gmail.com'\r\n                    }\r\n                    developer {\r\n                        id = 'nalbion'\r\n                        name = 'Nicholas Albion'\r\n                        email = 'nalbion@yahoo.com'\r\n                    }\r\n                }\r\n                scm {\r\n                    connection = 'scm:git:git://github.com/ggerganov/whisper.cpp.git'\r\n                    url = 'https://github.com/ggerganov/whisper.cpp'\r\n                }\r\n            }\r\n        }\r\n    }\r\n\r\n    repositories {\r\n        maven {\r\n            def releasesRepoUrl = 'https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/'\r\n            def snapshotsRepoUrl = 'https://s01.oss.sonatype.org/content/repositories/snapshots/'\r\n            url = version.endsWith('-SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl\r\n            credentials {\r\n                username = System.getenv(\"MAVEN_USERNAME\")\r\n                password = System.getenv(\"MAVEN_PASSWORD\")\r\n            }\r\n        }\r\n    }\r\n}\r\n\r\nsigning {\r\n    def signingKey = System.getenv(\"PGP_SECRET\")\r\n    def signingPassword = System.getenv(\"PGP_PASSPHRASE\")\r\n    useInMemoryPgpKeys(signingKey, signingPassword)\r\n    sign publishing.publications.mavenJava\r\n}\r\n"
  },
  {
    "path": "bindings/java/gradle/wrapper/gradle-wrapper.properties",
    "content": "distributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-8.1-bin.zip\nnetworkTimeout=10000\nzipStoreBase=GRADLE_USER_HOME\nzipStorePath=wrapper/dists\n"
  },
  {
    "path": "bindings/java/gradle.properties",
    "content": "org.gradle.jvmargs=-Xms256m -Xmx1024m\r\nsystem.include.dir=/usr/include\r\n#system.local.include.dir=../../include\r\nsystem.local.include.dir=./build/generated/sources/headers/java/main\r\njni.include.dir=/usr/lib/jvm/java-8-openjdk-amd64/include/\r\njni.lib.dir=/usr/lib/jvm/java-8-openjdk-amd64/lib/\r\n"
  },
  {
    "path": "bindings/java/gradlew",
    "content": "#!/bin/sh\n\n#\n# Copyright © 2015-2021 the original authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n##############################################################################\n#\n#   Gradle start up script for POSIX generated by Gradle.\n#\n#   Important for running:\n#\n#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is\n#       noncompliant, but you have some other compliant shell such as ksh or\n#       bash, then to run this script, type that shell name before the whole\n#       command line, like:\n#\n#           ksh Gradle\n#\n#       Busybox and similar reduced shells will NOT work, because this script\n#       requires all of these POSIX shell features:\n#         * functions;\n#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,\n#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;\n#         * compound commands having a testable exit status, especially «case»;\n#         * various built-in commands including «command», «set», and «ulimit».\n#\n#   Important for patching:\n#\n#   (2) This script targets any POSIX shell, so it avoids extensions provided\n#       by Bash, Ksh, etc; in particular arrays are avoided.\n#\n#       The \"traditional\" practice of packing multiple parameters into a\n#       space-separated string is a well documented source of bugs and security\n#       problems, so this is (mostly) avoided, by progressively accumulating\n#       options in \"$@\", and eventually passing that to Java.\n#\n#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,\n#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;\n#       see the in-line comments for details.\n#\n#       There are tweaks for specific operating systems such as AIX, CygWin,\n#       Darwin, MinGW, and NonStop.\n#\n#   (3) This script is generated from the Groovy template\n#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt\n#       within the Gradle project.\n#\n#       You can find Gradle at https://github.com/gradle/gradle/.\n#\n##############################################################################\n\n# Attempt to set APP_HOME\n\n# Resolve links: $0 may be a link\napp_path=$0\n\n# Need this for daisy-chained symlinks.\nwhile\n    APP_HOME=${app_path%\"${app_path##*/}\"}  # leaves a trailing /; empty if no leading path\n    [ -h \"$app_path\" ]\ndo\n    ls=$( ls -ld \"$app_path\" )\n    link=${ls#*' -> '}\n    case $link in             #(\n      /*)   app_path=$link ;; #(\n      *)    app_path=$APP_HOME$link ;;\n    esac\ndone\n\n# This is normally unused\n# shellcheck disable=SC2034\nAPP_BASE_NAME=${0##*/}\nAPP_HOME=$( cd \"${APP_HOME:-./}\" && pwd -P ) || exit\n\n# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nDEFAULT_JVM_OPTS='\"-Xmx64m\" \"-Xms64m\"'\n\n# Use the maximum available, or set MAX_FD != -1 to use that value.\nMAX_FD=maximum\n\nwarn () {\n    echo \"$*\"\n} >&2\n\ndie () {\n    echo\n    echo \"$*\"\n    echo\n    exit 1\n} >&2\n\n# OS specific support (must be 'true' or 'false').\ncygwin=false\nmsys=false\ndarwin=false\nnonstop=false\ncase \"$( uname )\" in                #(\n  CYGWIN* )         cygwin=true  ;; #(\n  Darwin* )         darwin=true  ;; #(\n  MSYS* | MINGW* )  msys=true    ;; #(\n  NONSTOP* )        nonstop=true ;;\nesac\n\nCLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar\n\n\n# Determine the Java command to use to start the JVM.\nif [ -n \"$JAVA_HOME\" ] ; then\n    if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n        # IBM's JDK on AIX uses strange locations for the executables\n        JAVACMD=$JAVA_HOME/jre/sh/java\n    else\n        JAVACMD=$JAVA_HOME/bin/java\n    fi\n    if [ ! -x \"$JAVACMD\" ] ; then\n        die \"ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\n    fi\nelse\n    JAVACMD=java\n    which java >/dev/null 2>&1 || die \"ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\nfi\n\n# Increase the maximum file descriptors if we can.\nif ! \"$cygwin\" && ! \"$darwin\" && ! \"$nonstop\" ; then\n    case $MAX_FD in #(\n      max*)\n        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.\n        # shellcheck disable=SC3045\n        MAX_FD=$( ulimit -H -n ) ||\n            warn \"Could not query maximum file descriptor limit\"\n    esac\n    case $MAX_FD in  #(\n      '' | soft) :;; #(\n      *)\n        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.\n        # shellcheck disable=SC3045\n        ulimit -n \"$MAX_FD\" ||\n            warn \"Could not set maximum file descriptor limit to $MAX_FD\"\n    esac\nfi\n\n# Collect all arguments for the java command, stacking in reverse order:\n#   * args from the command line\n#   * the main class name\n#   * -classpath\n#   * -D...appname settings\n#   * --module-path (only if needed)\n#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.\n\n# For Cygwin or MSYS, switch paths to Windows format before running java\nif \"$cygwin\" || \"$msys\" ; then\n    APP_HOME=$( cygpath --path --mixed \"$APP_HOME\" )\n    CLASSPATH=$( cygpath --path --mixed \"$CLASSPATH\" )\n\n    JAVACMD=$( cygpath --unix \"$JAVACMD\" )\n\n    # Now convert the arguments - kludge to limit ourselves to /bin/sh\n    for arg do\n        if\n            case $arg in                                #(\n              -*)   false ;;                            # don't mess with options #(\n              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath\n                    [ -e \"$t\" ] ;;                      #(\n              *)    false ;;\n            esac\n        then\n            arg=$( cygpath --path --ignore --mixed \"$arg\" )\n        fi\n        # Roll the args list around exactly as many times as the number of\n        # args, so each arg winds up back in the position where it started, but\n        # possibly modified.\n        #\n        # NB: a `for` loop captures its iteration list before it begins, so\n        # changing the positional parameters here affects neither the number of\n        # iterations, nor the values presented in `arg`.\n        shift                   # remove old arg\n        set -- \"$@\" \"$arg\"      # push replacement arg\n    done\nfi\n\n# Collect all arguments for the java command;\n#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of\n#     shell script including quotes and variable substitutions, so put them in\n#     double quotes to make sure that they get re-expanded; and\n#   * put everything else in single quotes, so that it's not re-expanded.\n\nset -- \\\n        \"-Dorg.gradle.appname=$APP_BASE_NAME\" \\\n        -classpath \"$CLASSPATH\" \\\n        org.gradle.wrapper.GradleWrapperMain \\\n        \"$@\"\n\n# Stop when \"xargs\" is not available.\nif ! command -v xargs >/dev/null 2>&1\nthen\n    die \"xargs is not available\"\nfi\n\n# Use \"xargs\" to parse quoted args.\n#\n# With -n1 it outputs one arg per line, with the quotes and backslashes removed.\n#\n# In Bash we could simply go:\n#\n#   readarray ARGS < <( xargs -n1 <<<\"$var\" ) &&\n#   set -- \"${ARGS[@]}\" \"$@\"\n#\n# but POSIX shell has neither arrays nor command substitution, so instead we\n# post-process each arg (as a line of input to sed) to backslash-escape any\n# character that might be a shell metacharacter, then use eval to reverse\n# that process (while maintaining the separation between arguments), and wrap\n# the whole thing up as a single \"set\" statement.\n#\n# This will of course break if any of these variables contains a newline or\n# an unmatched quote.\n#\n\neval \"set -- $(\n        printf '%s\\n' \"$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS\" |\n        xargs -n1 |\n        sed ' s~[^-[:alnum:]+,./:=@_]~\\\\&~g; ' |\n        tr '\\n' ' '\n    )\" '\"$@\"'\n\nexec \"$JAVACMD\" \"$@\"\n"
  },
  {
    "path": "bindings/java/gradlew.bat",
    "content": "@rem\r\n@rem Copyright 2015 the original author or authors.\r\n@rem\r\n@rem Licensed under the Apache License, Version 2.0 (the \"License\");\r\n@rem you may not use this file except in compliance with the License.\r\n@rem You may obtain a copy of the License at\r\n@rem\r\n@rem      https://www.apache.org/licenses/LICENSE-2.0\r\n@rem\r\n@rem Unless required by applicable law or agreed to in writing, software\r\n@rem distributed under the License is distributed on an \"AS IS\" BASIS,\r\n@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n@rem See the License for the specific language governing permissions and\r\n@rem limitations under the License.\r\n@rem\r\n\r\n@if \"%DEBUG%\"==\"\" @echo off\r\n@rem ##########################################################################\r\n@rem\r\n@rem  Gradle startup script for Windows\r\n@rem\r\n@rem ##########################################################################\r\n\r\n@rem Set local scope for the variables with windows NT shell\r\nif \"%OS%\"==\"Windows_NT\" setlocal\r\n\r\nset DIRNAME=%~dp0\r\nif \"%DIRNAME%\"==\"\" set DIRNAME=.\r\n@rem This is normally unused\r\nset APP_BASE_NAME=%~n0\r\nset APP_HOME=%DIRNAME%\r\n\r\n@rem Resolve any \".\" and \"..\" in APP_HOME to make it shorter.\r\nfor %%i in (\"%APP_HOME%\") do set APP_HOME=%%~fi\r\n\r\n@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\r\nset DEFAULT_JVM_OPTS=\"-Xmx64m\" \"-Xms64m\"\r\n\r\n@rem Find java.exe\r\nif defined JAVA_HOME goto findJavaFromJavaHome\r\n\r\nset JAVA_EXE=java.exe\r\n%JAVA_EXE% -version >NUL 2>&1\r\nif %ERRORLEVEL% equ 0 goto execute\r\n\r\necho.\r\necho ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\r\necho.\r\necho Please set the JAVA_HOME variable in your environment to match the\r\necho location of your Java installation.\r\n\r\ngoto fail\r\n\r\n:findJavaFromJavaHome\r\nset JAVA_HOME=%JAVA_HOME:\"=%\r\nset JAVA_EXE=%JAVA_HOME%/bin/java.exe\r\n\r\nif exist \"%JAVA_EXE%\" goto execute\r\n\r\necho.\r\necho ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\r\necho.\r\necho Please set the JAVA_HOME variable in your environment to match the\r\necho location of your Java installation.\r\n\r\ngoto fail\r\n\r\n:execute\r\n@rem Setup the command line\r\n\r\nset CLASSPATH=%APP_HOME%\\gradle\\wrapper\\gradle-wrapper.jar\r\n\r\n\r\n@rem Execute Gradle\r\n\"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% \"-Dorg.gradle.appname=%APP_BASE_NAME%\" -classpath \"%CLASSPATH%\" org.gradle.wrapper.GradleWrapperMain %*\r\n\r\n:end\r\n@rem End local scope for the variables with windows NT shell\r\nif %ERRORLEVEL% equ 0 goto mainEnd\r\n\r\n:fail\r\nrem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\r\nrem the _cmd.exe /c_ return code!\r\nset EXIT_CODE=%ERRORLEVEL%\r\nif %EXIT_CODE% equ 0 set EXIT_CODE=1\r\nif not \"\"==\"%GRADLE_EXIT_CONSOLE%\" exit %EXIT_CODE%\r\nexit /b %EXIT_CODE%\r\n\r\n:mainEnd\r\nif \"%OS%\"==\"Windows_NT\" endlocal\r\n\r\n:omega\r\n"
  },
  {
    "path": "bindings/java/settings.gradle",
    "content": "rootProject.name = \"whispercpp\"\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperConstants.java",
    "content": "package io.github.ggerganov.whispercpp;\n\n/**\n * Presets for alignment heads in DTW token timestamps\n */\npublic class WhisperConstants {\n    // Alignment heads presets\n    public static final int WHISPER_AHEADS_NONE = 0;\n    public static final int WHISPER_AHEADS_TINY_EN = 1;\n    public static final int WHISPER_AHEADS_TINY = 2;\n    public static final int WHISPER_AHEADS_BASE_EN = 3;\n    public static final int WHISPER_AHEADS_BASE = 4;\n    public static final int WHISPER_AHEADS_SMALL_EN = 5;\n    public static final int WHISPER_AHEADS_SMALL = 6;\n    public static final int WHISPER_AHEADS_MEDIUM_EN = 7;\n    public static final int WHISPER_AHEADS_MEDIUM = 8;\n    public static final int WHISPER_AHEADS_LARGE_V1 = 9;\n    public static final int WHISPER_AHEADS_LARGE_V2 = 10;\n    public static final int WHISPER_AHEADS_LARGE_V3 = 11;\n    public static final int WHISPER_AHEADS_LARGE_V3_TURBO = 12;\n    public static final int WHISPER_AHEADS_CUSTOM = 13;\n    public static final int WHISPER_AHEADS_N_TOP_MOST = 14;\n    public static final int WHISPER_AHEADS_COUNT = 15;\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperContext.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport com.sun.jna.NativeLong;\r\nimport com.sun.jna.Structure;\r\nimport com.sun.jna.ptr.PointerByReference;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.ggml.GgmlType;\r\nimport io.github.ggerganov.whispercpp.WhisperModel;\r\nimport io.github.ggerganov.whispercpp.params.WhisperContextParams;\r\n\r\nimport java.util.List;\r\n\r\npublic class WhisperContext extends Structure {\r\n    public NativeLong t_load_us;\r\n    public NativeLong t_start_us;\r\n\r\n    /** weight type (FP32 / FP16 / QX) */\r\n    public GgmlType wtype = GgmlType.GGML_TYPE_F16;\r\n    /** intermediate type (FP32 or FP16) */\r\n    public GgmlType itype = GgmlType.GGML_TYPE_F16;\r\n\r\n    public WhisperContextParams.ByValue params;\r\n\r\n    public Pointer model;\r\n    public Pointer vocab;\r\n    public Pointer state;\r\n\r\n    /** populated by whisper_init_from_file_with_params() */\r\n    public Pointer path_model;\r\n\r\n    @Override\r\n    protected List<String> getFieldOrder() {\r\n        return List.of(\"t_load_us\", \"t_start_us\", \"wtype\", \"itype\",\r\n                \"params\", \"model\", \"vocab\", \"state\", \"path_model\");\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCpp.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport com.sun.jna.Native;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.bean.WhisperSegment;\r\nimport io.github.ggerganov.whispercpp.params.WhisperContextParams;\r\nimport io.github.ggerganov.whispercpp.params.WhisperFullParams;\r\nimport io.github.ggerganov.whispercpp.params.WhisperSamplingStrategy;\r\n\r\nimport java.io.File;\r\nimport java.io.FileNotFoundException;\r\nimport java.io.IOException;\r\nimport java.util.ArrayList;\r\nimport java.util.List;\r\n\r\n/**\r\n * Before calling most methods, you must call `initContext(modelPath)` to initialise the `ctx` Pointer.\r\n */\r\npublic class WhisperCpp implements AutoCloseable {\r\n    private WhisperCppJnaLibrary lib = WhisperCppJnaLibrary.instance;\r\n    private Pointer ctx = null;\r\n    private Pointer paramsPointer = null;\r\n    private Pointer greedyParamsPointer = null;\r\n    private Pointer beamParamsPointer = null;\r\n\r\n    public File modelDir() {\r\n        String modelDirPath = System.getenv(\"XDG_CACHE_HOME\");\r\n        if (modelDirPath == null) {\r\n            modelDirPath = System.getProperty(\"user.home\") + \"/.cache\";\r\n        }\r\n\r\n        return new File(modelDirPath, \"whisper\");\r\n    }\r\n\r\n    /**\r\n     * @param modelPath - absolute path, or just the name (eg: \"base\", \"base-en\" or \"base.en\")\r\n     */\r\n    public void initContext(String modelPath) throws FileNotFoundException {\r\n        initContextImpl(modelPath, getContextDefaultParams());\r\n    }\r\n\r\n    /**\r\n     * @param modelPath - absolute path, or just the name (eg: \"base\", \"base-en\" or \"base.en\")\r\n     * @param params - params to use when initialising the context\r\n     */\r\n    public void initContext(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {\r\n        initContextImpl(modelPath, params);\r\n    }\r\n\r\n    private void initContextImpl(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {\r\n        if (ctx != null) {\r\n            lib.whisper_free(ctx);\r\n        }\r\n\r\n        if (!modelPath.contains(\"/\") && !modelPath.contains(\"\\\\\")) {\r\n            if (!modelPath.endsWith(\".bin\")) {\r\n                modelPath = \"ggml-\" + modelPath.replace(\"-\", \".\") + \".bin\";\r\n            }\r\n\r\n            modelPath = new File(modelDir(), modelPath).getAbsolutePath();\r\n        }\r\n\r\n        ctx = lib.whisper_init_from_file_with_params(modelPath, params);\r\n\r\n        if (ctx == null) {\r\n            throw new FileNotFoundException(modelPath);\r\n        }\r\n    }\r\n\r\n    /**\r\n     * Provides default params which can be used with `whisper_init_from_file_with_params()` etc.\r\n     * Returns a ByValue instance to ensure proper parameter passing to native code.\r\n     */\r\n    public WhisperContextParams.ByValue getContextDefaultParams() {\r\n        WhisperContextParams.ByValue valueParams = new WhisperContextParams.ByValue(\r\n            lib.whisper_context_default_params_by_ref());\r\n        valueParams.read();\r\n        return valueParams;\r\n    }\r\n    \r\n    /**\r\n     * Provides default params which can be used with `whisper_full()` etc.\r\n     * Because this function allocates memory for the params, the caller must call either:\r\n     * - call `whisper_free_params()`\r\n     * - `Native.free(Pointer.nativeValue(pointer));`\r\n     *\r\n     * @param strategy - GREEDY\r\n     */\r\n    public WhisperFullParams.ByValue getFullDefaultParams(WhisperSamplingStrategy strategy) {\r\n        Pointer pointer;\r\n\r\n        // whisper_full_default_params_by_ref allocates memory which we need to delete, so only create max 1 pointer for each strategy.\r\n        if (strategy == WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY) {\r\n            if (greedyParamsPointer == null) {\r\n                greedyParamsPointer = lib.whisper_full_default_params_by_ref(strategy.ordinal());\r\n            }\r\n            pointer = greedyParamsPointer;\r\n        } else {\r\n            if (beamParamsPointer == null) {\r\n                beamParamsPointer = lib.whisper_full_default_params_by_ref(strategy.ordinal());\r\n            }\r\n            pointer = beamParamsPointer;\r\n        }\r\n\r\n        WhisperFullParams.ByValue params = new WhisperFullParams.ByValue(pointer);\r\n        params.read();\r\n        return params;\r\n    }\r\n\r\n    @Override\r\n    public void close() {\r\n        freeContext();\r\n        freeParams();\r\n        System.out.println(\"Whisper closed\");\r\n    }\r\n\r\n    private void freeContext() {\r\n        if (ctx != null) {\r\n            lib.whisper_free(ctx);\r\n        }\r\n    }\r\n\r\n    private void freeParams() {\r\n        if (paramsPointer != null) {\r\n            Native.free(Pointer.nativeValue(paramsPointer));\r\n            paramsPointer = null;\r\n        }\r\n        if (greedyParamsPointer != null) {\r\n            Native.free(Pointer.nativeValue(greedyParamsPointer));\r\n            greedyParamsPointer = null;\r\n        }\r\n        if (beamParamsPointer != null) {\r\n            Native.free(Pointer.nativeValue(beamParamsPointer));\r\n            beamParamsPointer = null;\r\n        }\r\n    }\r\n\r\n    /**\r\n     * Run the entire model: PCM -&gt; log mel spectrogram -&gt; encoder -&gt; decoder -&gt; text.\r\n     * Not thread safe for same context\r\n     * Uses the specified decoding strategy to obtain the text.\r\n     */\r\n    public String fullTranscribe(WhisperFullParams.ByValue whisperParams, float[] audioData) throws IOException {\r\n        if (ctx == null) {\r\n            throw new IllegalStateException(\"Model not initialised\");\r\n        }\r\n\r\n        /*\r\n        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(\r\n            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));\r\n        valueParams.read();\r\n        */\r\n\r\n        if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {\r\n            throw new IOException(\"Failed to process audio\");\r\n        }\r\n\r\n        int nSegments = lib.whisper_full_n_segments(ctx);\r\n\r\n        StringBuilder str = new StringBuilder();\r\n\r\n        for (int i = 0; i < nSegments; i++) {\r\n            String text = lib.whisper_full_get_segment_text(ctx, i);\r\n            System.out.println(\"Segment:\" + text);\r\n            str.append(text);\r\n        }\r\n\r\n        return str.toString().trim();\r\n    }\r\n\r\n    /**\r\n     * Full transcribe with time list.\r\n     *\r\n     * @param whisperParams the whisper params\r\n     * @param audioData     the audio data\r\n     * @return the list\r\n     * @throws IOException the io exception\r\n     */\r\n    public List<WhisperSegment> fullTranscribeWithTime(WhisperFullParams.ByValue whisperParams, float[] audioData) throws IOException {\r\n        if (ctx == null) {\r\n            throw new IllegalStateException(\"Model not initialised\");\r\n        }\r\n\r\n        if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {\r\n            throw new IOException(\"Failed to process audio\");\r\n        }\r\n\r\n        int nSegments = lib.whisper_full_n_segments(ctx);\r\n        List<WhisperSegment> segments= new ArrayList<>(nSegments);\r\n\r\n        for (int i = 0; i < nSegments; i++) {\r\n            long t0 = lib.whisper_full_get_segment_t0(ctx, i);\r\n            String text = lib.whisper_full_get_segment_text(ctx, i);\r\n            long t1 = lib.whisper_full_get_segment_t1(ctx, i);\r\n            segments.add(new WhisperSegment(t0,t1,text));\r\n        }\r\n\r\n        return segments;\r\n    }\r\n\r\n//    public int getTextSegmentCount(Pointer ctx) {\r\n//        return lib.whisper_full_n_segments(ctx);\r\n//    }\r\n//    public String getTextSegment(Pointer ctx, int index) {\r\n//        return lib.whisper_full_get_segment_text(ctx, index);\r\n//    }\r\n\r\n    public String getSystemInfo() {\r\n        return lib.whisper_print_system_info();\r\n    }\r\n\r\n    public int benchMemcpy(int nthread) {\r\n        return lib.whisper_bench_memcpy(nthread);\r\n    }\r\n\r\n    public int benchGgmlMulMat(int nthread) {\r\n        return lib.whisper_bench_ggml_mul_mat(nthread);\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCppJnaLibrary.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport com.sun.jna.Library;\r\nimport com.sun.jna.Native;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.model.WhisperModelLoader;\r\nimport io.github.ggerganov.whispercpp.model.WhisperTokenData;\r\nimport io.github.ggerganov.whispercpp.params.WhisperContextParams;\r\nimport io.github.ggerganov.whispercpp.params.WhisperFullParams;\r\n\r\npublic interface WhisperCppJnaLibrary extends Library {\r\n\r\n    WhisperCppJnaLibrary instance = Native.load(\"whisper\", WhisperCppJnaLibrary.class);\r\n\r\n    String whisper_print_system_info();\r\n\r\n    /**\r\n     * DEPRECATED. Allocate (almost) all memory needed for the model by loading from a file.\r\n     *\r\n     * @param path_model Path to the model file\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_from_file(String path_model);\r\n\r\n    /**\r\n     * Provides default params which can be used with `whisper_init_from_file_with_params()` etc.\r\n     * Because this function allocates memory for the params, the caller must call either:\r\n     * - call `whisper_free_context_params()`\r\n     * - `Native.free(Pointer.nativeValue(pointer));`\r\n     */\r\n    Pointer whisper_context_default_params_by_ref();\r\n\r\n    void whisper_free_context_params(Pointer params);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model by loading from a file.\r\n     *\r\n     * @param path_model Path to the model file\r\n     * @param params     Pointer to whisper_context_params\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams.ByValue params);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model by loading from a buffer.\r\n     *\r\n     * @param buffer       Model buffer\r\n     * @param buffer_size  Size of the model buffer\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_from_buffer(Pointer buffer, int buffer_size);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model using a model loader.\r\n     *\r\n     * @param loader Model loader\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init(WhisperModelLoader loader);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model by loading from a file without allocating the state.\r\n     *\r\n     * @param path_model Path to the model file\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_from_file_no_state(String path_model);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model by loading from a buffer without allocating the state.\r\n     *\r\n     * @param buffer       Model buffer\r\n     * @param buffer_size  Size of the model buffer\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_from_buffer_no_state(Pointer buffer, int buffer_size);\r\n\r\n//    Pointer whisper_init_from_buffer_no_state(Pointer buffer, long buffer_size);\r\n\r\n    /**\r\n     * Allocate (almost) all memory needed for the model using a model loader without allocating the state.\r\n     *\r\n     * @param loader Model loader\r\n     * @return Whisper context on success, null on failure\r\n     */\r\n    Pointer whisper_init_no_state(WhisperModelLoader loader);\r\n\r\n    /**\r\n     * Allocate memory for the Whisper state.\r\n     *\r\n     * @param ctx Whisper context\r\n     * @return Whisper state on success, null on failure\r\n     */\r\n    Pointer whisper_init_state(Pointer ctx);\r\n\r\n    /**\r\n     * Free all allocated memory associated with the Whisper context.\r\n     *\r\n     * @param ctx Whisper context\r\n     */\r\n    void whisper_free(Pointer ctx);\r\n\r\n    /**\r\n     * Free all allocated memory associated with the Whisper state.\r\n     *\r\n     * @param state Whisper state\r\n     */\r\n    void whisper_free_state(Pointer state);\r\n\r\n\r\n    /**\r\n     * Convert RAW PCM audio to log mel spectrogram.\r\n     * The resulting spectrogram is stored inside the default state of the provided whisper context.\r\n     *\r\n     * @param ctx - Pointer to a WhisperContext\r\n     * @return 0 on success\r\n     */\r\n    int whisper_pcm_to_mel(Pointer ctx, final float[] samples, int n_samples, int n_threads);\r\n\r\n    /**\r\n     * @param ctx Pointer to a WhisperContext\r\n     * @param state Pointer to WhisperState\r\n     * @param n_samples\r\n     * @param n_threads\r\n     * @return 0 on success\r\n     */\r\n    int whisper_pcm_to_mel_with_state(Pointer ctx, Pointer state, final float[] samples, int n_samples, int n_threads);\r\n\r\n    /**\r\n     * This can be used to set a custom log mel spectrogram inside the default state of the provided whisper context.\r\n     * Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.\r\n     * n_mel must be 80\r\n     * @return 0 on success\r\n     */\r\n    int whisper_set_mel(Pointer ctx, final float[] data, int n_len, int n_mel);\r\n    int whisper_set_mel_with_state(Pointer ctx, Pointer state, final float[] data, int n_len, int n_mel);\r\n\r\n    /**\r\n     * Run the Whisper encoder on the log mel spectrogram stored inside the default state in the provided whisper context.\r\n     * Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\r\n     * Offset can be used to specify the offset of the first frame in the spectrogram.\r\n     * @return 0 on success\r\n     */\r\n    int whisper_encode(Pointer ctx, int offset, int n_threads);\r\n\r\n    int whisper_encode_with_state(Pointer ctx, Pointer state, int offset, int n_threads);\r\n\r\n    /**\r\n     * Run the Whisper decoder to obtain the logits and probabilities for the next token.\r\n     * Make sure to call whisper_encode() first.\r\n     * tokens + n_tokens is the provided context for the decoder.\r\n     * n_past is the number of tokens to use from previous decoder calls.\r\n     * Returns 0 on success\r\n     * TODO: add support for multiple decoders\r\n     */\r\n    int whisper_decode(Pointer ctx, Pointer tokens, int n_tokens, int n_past, int n_threads);\r\n\r\n    /**\r\n     * @param ctx\r\n     * @param state\r\n     * @param tokens Pointer to int tokens\r\n     * @param n_tokens\r\n     * @param n_past\r\n     * @param n_threads\r\n     * @return\r\n     */\r\n    int whisper_decode_with_state(Pointer ctx, Pointer state, Pointer tokens, int n_tokens, int n_past, int n_threads);\r\n\r\n    /**\r\n     * Convert the provided text into tokens.\r\n     * The tokens pointer must be large enough to hold the resulting tokens.\r\n     * Returns the number of tokens on success, no more than n_max_tokens\r\n     * Returns -1 on failure\r\n     * TODO: not sure if correct\r\n     */\r\n    int whisper_tokenize(Pointer ctx, String text, Pointer tokens, int n_max_tokens);\r\n\r\n    /** Largest language id (i.e. number of available languages - 1) */\r\n    int whisper_lang_max_id();\r\n\r\n    /**\r\n     * @return the id of the specified language, returns -1 if not found.\r\n     * Examples:\r\n     *   \"de\" -&gt; 2\r\n     *   \"german\" -&gt; 2\r\n     */\r\n    int whisper_lang_id(String lang);\r\n\r\n    /** @return the short string of the specified language id (e.g. 2 -&gt; \"de\"), returns nullptr if not found */\r\n    String whisper_lang_str(int id);\r\n\r\n    /**\r\n     * Use mel data at offset_ms to try and auto-detect the spoken language.\r\n     * Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first\r\n     * Returns the top language id or negative on failure\r\n     * If not null, fills the lang_probs array with the probabilities of all languages\r\n     * The array must be whisper_lang_max_id() + 1 in size\r\n     *\r\n     * ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69\r\n     */\r\n    int whisper_lang_auto_detect(Pointer ctx, int offset_ms, int n_threads, float[] lang_probs);\r\n\r\n    int whisper_lang_auto_detect_with_state(Pointer ctx, Pointer state, int offset_ms, int n_threads, float[] lang_probs);\r\n\r\n    int whisper_n_len           (Pointer ctx); // mel length\r\n    int whisper_n_len_from_state(Pointer state); // mel length\r\n    int whisper_n_vocab         (Pointer ctx);\r\n    int whisper_n_text_ctx      (Pointer ctx);\r\n    int whisper_n_audio_ctx     (Pointer ctx);\r\n    int whisper_is_multilingual (Pointer ctx);\r\n\r\n    int whisper_model_n_vocab      (Pointer ctx);\r\n    int whisper_model_n_audio_ctx  (Pointer ctx);\r\n    int whisper_model_n_audio_state(Pointer ctx);\r\n    int whisper_model_n_audio_head (Pointer ctx);\r\n    int whisper_model_n_audio_layer(Pointer ctx);\r\n    int whisper_model_n_text_ctx   (Pointer ctx);\r\n    int whisper_model_n_text_state (Pointer ctx);\r\n    int whisper_model_n_text_head  (Pointer ctx);\r\n    int whisper_model_n_text_layer (Pointer ctx);\r\n    int whisper_model_n_mels       (Pointer ctx);\r\n    int whisper_model_ftype        (Pointer ctx);\r\n    int whisper_model_type         (Pointer ctx);\r\n\r\n    /**\r\n     * Token logits obtained from the last call to whisper_decode().\r\n     * The logits for the last token are stored in the last row\r\n     * Rows: n_tokens\r\n     * Cols: n_vocab\r\n     */\r\n    float[] whisper_get_logits           (Pointer ctx);\r\n    float[] whisper_get_logits_from_state(Pointer state);\r\n\r\n    // Token Id -> String. Uses the vocabulary in the provided context\r\n    String whisper_token_to_str(Pointer ctx, int token);\r\n    String whisper_model_type_readable(Pointer ctx);\r\n\r\n    // Special tokens\r\n    int whisper_token_eot (Pointer ctx);\r\n    int whisper_token_sot (Pointer ctx);\r\n    int whisper_token_prev(Pointer ctx);\r\n    int whisper_token_solm(Pointer ctx);\r\n    int whisper_token_not (Pointer ctx);\r\n    int whisper_token_beg (Pointer ctx);\r\n    int whisper_token_lang(Pointer ctx, int lang_id);\r\n\r\n    // Task tokens\r\n    int whisper_token_translate (Pointer ctx);\r\n    int whisper_token_transcribe(Pointer ctx);\r\n\r\n    // Performance information from the default state.\r\n    void whisper_print_timings(Pointer ctx);\r\n    void whisper_reset_timings(Pointer ctx);\r\n\r\n    // Note: Even if `whisper_full_params is stripped back to just 4 ints, JNA throws \"Invalid memory access\"\r\n    //       when `whisper_full_default_params()` tries to return a struct.\r\n    // WhisperFullParams whisper_full_default_params(int strategy);\r\n\r\n    /**\r\n     * Provides default params which can be used with `whisper_full()` etc.\r\n     * Because this function allocates memory for the params, the caller must call either:\r\n     * - call `whisper_free_params()`\r\n     * - `Native.free(Pointer.nativeValue(pointer));`\r\n     *\r\n     * @param strategy - WhisperSamplingStrategy.value\r\n     */\r\n    Pointer whisper_full_default_params_by_ref(int strategy);\r\n\r\n    void whisper_free_params(Pointer params);\r\n\r\n    /**\r\n     * Run the entire model: PCM -&gt; log mel spectrogram -&gt; encoder -&gt; decoder -&gt; text\r\n     * Not thread safe for same context\r\n     * Uses the specified decoding strategy to obtain the text.\r\n     */\r\n    int whisper_full(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples);\r\n\r\n    public int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams.ByValue params, float[] samples, int n_samples);\r\n    //int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);\r\n\r\n    // Split the input audio in chunks and process each chunk separately using whisper_full_with_state()\r\n    // Result is stored in the default state of the context\r\n    // Not thread safe if executed in parallel on the same context.\r\n    // It seems this approach can offer some speedup in some cases.\r\n    // However, the transcription accuracy can be worse at the beginning and end of each chunk.\r\n    int whisper_full_parallel(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples, int n_processors);\r\n\r\n    /**\r\n     * Number of generated text segments.\r\n     * A segment can be a few words, a sentence, or even a paragraph.\r\n     * @param ctx Pointer to WhisperContext\r\n     */\r\n    int whisper_full_n_segments (Pointer ctx);\r\n\r\n    /**\r\n     * @param state Pointer to WhisperState\r\n     */\r\n    int whisper_full_n_segments_from_state(Pointer state);\r\n\r\n    /**\r\n     * Language id associated with the context's default state.\r\n     * @param ctx Pointer to WhisperContext\r\n     */\r\n    int whisper_full_lang_id(Pointer ctx);\r\n\r\n    /** Language id associated with the provided state */\r\n    int whisper_full_lang_id_from_state(Pointer state);\r\n\r\n\r\n    /** Get the start time of the specified segment. */\r\n    long whisper_full_get_segment_t0(Pointer ctx, int i_segment);\r\n\r\n    /** Get the start time of the specified segment from the state. */\r\n    long whisper_full_get_segment_t0_from_state(Pointer state, int i_segment);\r\n\r\n    /** Get the end time of the specified segment. */\r\n    long whisper_full_get_segment_t1(Pointer ctx, int i_segment);\r\n\r\n    /** Get the end time of the specified segment from the state. */\r\n    long whisper_full_get_segment_t1_from_state(Pointer state, int i_segment);\r\n\r\n    /** Get the text of the specified segment. */\r\n    String whisper_full_get_segment_text(Pointer ctx, int i_segment);\r\n\r\n    /** Get the text of the specified segment from the state. */\r\n    String whisper_full_get_segment_text_from_state(Pointer state, int i_segment);\r\n\r\n    /** Get the number of tokens in the specified segment. */\r\n    int whisper_full_n_tokens(Pointer ctx, int i_segment);\r\n\r\n    /** Get the number of tokens in the specified segment from the state. */\r\n    int whisper_full_n_tokens_from_state(Pointer state, int i_segment);\r\n\r\n    /** Get the token text of the specified token in the specified segment. */\r\n    String whisper_full_get_token_text(Pointer ctx, int i_segment, int i_token);\r\n\r\n\r\n    /** Get the token text of the specified token in the specified segment from the state. */\r\n    String whisper_full_get_token_text_from_state(Pointer ctx, Pointer state, int i_segment, int i_token);\r\n\r\n    /** Get the token ID of the specified token in the specified segment. */\r\n    int whisper_full_get_token_id(Pointer ctx, int i_segment, int i_token);\r\n\r\n    /** Get the token ID of the specified token in the specified segment from the state. */\r\n    int whisper_full_get_token_id_from_state(Pointer state, int i_segment, int i_token);\r\n\r\n    /** Get token data for the specified token in the specified segment. */\r\n    WhisperTokenData whisper_full_get_token_data(Pointer ctx, int i_segment, int i_token);\r\n\r\n    /** Get token data for the specified token in the specified segment from the state. */\r\n    WhisperTokenData whisper_full_get_token_data_from_state(Pointer state, int i_segment, int i_token);\r\n\r\n    /** Get the probability of the specified token in the specified segment. */\r\n    float whisper_full_get_token_p(Pointer ctx, int i_segment, int i_token);\r\n\r\n    /** Get the probability of the specified token in the specified segment from the state. */\r\n    float whisper_full_get_token_p_from_state(Pointer state, int i_segment, int i_token);\r\n\r\n    /**\r\n     * Benchmark function for memcpy.\r\n     *\r\n     * @param nThreads Number of threads to use for the benchmark.\r\n     * @return The result of the benchmark.\r\n     */\r\n    int whisper_bench_memcpy(int nThreads);\r\n\r\n    /**\r\n     * Benchmark function for memcpy as a string.\r\n     *\r\n     * @param nThreads Number of threads to use for the benchmark.\r\n     * @return The result of the benchmark as a string.\r\n     */\r\n    String whisper_bench_memcpy_str(int nThreads);\r\n\r\n    /**\r\n     * Benchmark function for ggml_mul_mat.\r\n     *\r\n     * @param nThreads Number of threads to use for the benchmark.\r\n     * @return The result of the benchmark.\r\n     */\r\n    int whisper_bench_ggml_mul_mat(int nThreads);\r\n\r\n    /**\r\n     * Benchmark function for ggml_mul_mat as a string.\r\n     *\r\n     * @param nThreads Number of threads to use for the benchmark.\r\n     * @return The result of the benchmark as a string.\r\n     */\r\n    String whisper_bench_ggml_mul_mat_str(int nThreads);\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/bean/WhisperSegment.java",
    "content": "package io.github.ggerganov.whispercpp.bean;\n\n/**\n * Created by litonglinux@qq.com on 10/21/2023_7:48 AM\n */\npublic class WhisperSegment {\n  private long start, end;\n  private String sentence;\n\n  public WhisperSegment() {\n  }\n\n  public WhisperSegment(long start, long end, String sentence) {\n    this.start = start;\n    this.end = end;\n    this.sentence = sentence;\n  }\n\n  public long getStart() {\n    return start;\n  }\n\n  public long getEnd() {\n    return end;\n  }\n\n  public String getSentence() {\n    return sentence;\n  }\n\n  public void setStart(long start) {\n    this.start = start;\n  }\n\n  public void setEnd(long end) {\n    this.end = end;\n  }\n\n  public void setSentence(String sentence) {\n    this.sentence = sentence;\n  }\n\n  @Override\n  public String toString() {\n    return \"[\" + start + \" --> \" + end + \"]:\" + sentence;\n  }\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/GgmlAbortCallback.java",
    "content": "package io.github.ggerganov.whispercpp.callbacks;\n\nimport com.sun.jna.Callback;\n\n/**\n * Callback for aborting GGML computation\n * Maps to the C typedef: bool (*ggml_abort_callback)(void * data)\n */\npublic interface GgmlAbortCallback extends Callback {\n    /**\n     * Return true to abort the computation, false to continue\n     *\n     * @param data User data passed to the callback\n     * @return true to abort, false to continue\n     */\n    boolean invoke(com.sun.jna.Pointer data);\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/WhisperEncoderBeginCallback.java",
    "content": "package io.github.ggerganov.whispercpp.callbacks;\r\n\r\nimport com.sun.jna.Callback;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.WhisperContext;\r\nimport io.github.ggerganov.whispercpp.model.WhisperState;\r\n\r\n/**\r\n * Callback before the encoder starts.\r\n * If not null, called before the encoder starts.\r\n * If it returns false, the computation is aborted.\r\n */\r\npublic interface WhisperEncoderBeginCallback extends Callback {\r\n\r\n    /**\r\n     * Callback method before the encoder starts.\r\n     *\r\n     * @param ctx        The whisper context.\r\n     * @param state      The whisper state.\r\n     * @param user_data  User data.\r\n     * @return True if the computation should proceed, false otherwise.\r\n     */\r\n    boolean callback(Pointer ctx, Pointer state, Pointer user_data);\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/WhisperLogitsFilterCallback.java",
    "content": "package io.github.ggerganov.whispercpp.callbacks;\r\n\r\nimport com.sun.jna.Callback;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.model.WhisperTokenData;\r\n\r\n/**\r\n * Callback to filter logits.\r\n * Can be used to modify the logits before sampling.\r\n * If not null, called after applying temperature to logits.\r\n */\r\npublic interface WhisperLogitsFilterCallback extends Callback {\r\n\r\n    /**\r\n     * Callback method to filter logits.\r\n     *\r\n     * @param ctx        The whisper context.\r\n     * @param state      The whisper state.\r\n     * @param tokens     The array of whisper_token_data.\r\n     * @param n_tokens   The number of tokens.\r\n     * @param logits     The array of logits.\r\n     * @param user_data  User data.\r\n     */\r\n    void callback(Pointer ctx, Pointer state, WhisperTokenData[] tokens, int n_tokens, float[] logits, Pointer user_data);\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/WhisperNewSegmentCallback.java",
    "content": "package io.github.ggerganov.whispercpp.callbacks;\r\n\r\nimport com.sun.jna.Callback;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.WhisperContext;\r\nimport io.github.ggerganov.whispercpp.model.WhisperState;\r\n\r\n/**\r\n * Callback for the text segment.\r\n * Called on every newly generated text segment.\r\n * Use the whisper_full_...() functions to obtain the text segments.\r\n */\r\npublic interface WhisperNewSegmentCallback extends Callback {\r\n\r\n    /**\r\n     * Callback method for the text segment.\r\n     *\r\n     * @param ctx        The whisper context.\r\n     * @param state      The whisper state.\r\n     * @param n_new      The number of newly generated text segments.\r\n     * @param user_data  User data.\r\n     */\r\n    void callback(Pointer ctx, Pointer state, int n_new, Pointer user_data);\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/WhisperProgressCallback.java",
    "content": "package io.github.ggerganov.whispercpp.callbacks;\r\n\r\nimport com.sun.jna.Callback;\r\nimport com.sun.jna.Pointer;\r\nimport io.github.ggerganov.whispercpp.WhisperContext;\r\nimport io.github.ggerganov.whispercpp.model.WhisperState;\r\n\r\n/**\r\n * Callback for progress updates.\r\n */\r\npublic interface WhisperProgressCallback extends Callback {\r\n\r\n    /**\r\n     * Callback method for progress updates.\r\n     *\r\n     * @param ctx        The whisper context.\r\n     * @param state      The whisper state.\r\n     * @param progress   The progress value.\r\n     * @param user_data  User data.\r\n     */\r\n    void callback(Pointer ctx, Pointer state, int progress, Pointer user_data);\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/ggml/GgmlTensor.java",
    "content": "package io.github.ggerganov.whispercpp.ggml;\r\n\r\npublic class GgmlTensor {\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/ggml/GgmlType.java",
    "content": "package io.github.ggerganov.whispercpp.ggml;\r\n\r\npublic enum GgmlType {\r\n    GGML_TYPE_F32,\r\n    GGML_TYPE_F16,\r\n    GGML_TYPE_Q4_0,\r\n    GGML_TYPE_Q4_1,\r\n    REMOVED_GGML_TYPE_Q4_2,  // support has been removed\r\n    REMOVED_GGML_TYPE_Q4_3, // support has been removed\r\n    GGML_TYPE_Q5_0,\r\n    GGML_TYPE_Q5_1,\r\n    GGML_TYPE_Q8_0,\r\n    GGML_TYPE_Q8_1,\r\n    GGML_TYPE_I8,\r\n    GGML_TYPE_I16,\r\n    GGML_TYPE_I32,\r\n    GGML_TYPE_COUNT,\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/model/EModel.java",
    "content": "package io.github.ggerganov.whispercpp.model;\r\n\r\npublic enum EModel {\r\n    MODEL_UNKNOWN,\r\n    MODEL_TINY,\r\n    MODEL_BASE,\r\n    MODEL_SMALL,\r\n    MODEL_MEDIUM,\r\n    MODEL_LARGE,\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/model/WhisperModel.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport io.github.ggerganov.whispercpp.ggml.GgmlTensor;\r\nimport io.github.ggerganov.whispercpp.model.EModel;\r\n\r\npublic class WhisperModel {\r\n//    EModel type = EModel.MODEL_UNKNOWN;\r\n//\r\n//    WhisperHParams hparams;\r\n//    WhisperFilters filters;\r\n//\r\n//    // encoder.positional_embedding\r\n//    GgmlTensor e_pe;\r\n//\r\n//    // encoder.conv1\r\n//    GgmlTensor e_conv_1_w;\r\n//    GgmlTensor e_conv_1_b;\r\n//\r\n//    // encoder.conv2\r\n//    GgmlTensor e_conv_2_w;\r\n//    GgmlTensor e_conv_2_b;\r\n//\r\n//    // encoder.ln_post\r\n//    GgmlTensor e_ln_w;\r\n//    GgmlTensor e_ln_b;\r\n//\r\n//    // decoder.positional_embedding\r\n//    GgmlTensor d_pe;\r\n//\r\n//    // decoder.token_embedding\r\n//    GgmlTensor d_te;\r\n//\r\n//    // decoder.ln\r\n//    GgmlTensor d_ln_w;\r\n//    GgmlTensor d_ln_b;\r\n//\r\n//    std::vector<whisper_layer_encoder> layers_encoder;\r\n//    std::vector<whisper_layer_decoder> layers_decoder;\r\n//\r\n//    // context\r\n//    struct ggml_context * ctx;\r\n//\r\n//    // the model memory buffer is read-only and can be shared between processors\r\n//    std::vector<uint8_t> * buf;\r\n//\r\n//    // tensors\r\n//    int n_loaded;\r\n//    Map<String, GgmlTensor> tensors;\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/model/WhisperModelLoader.java",
    "content": "package io.github.ggerganov.whispercpp.model;\r\n\r\nimport com.sun.jna.Callback;\r\nimport com.sun.jna.Pointer;\r\nimport com.sun.jna.Structure;\r\n\r\n\r\npublic class WhisperModelLoader extends Structure {\r\n    public Pointer context;\r\n    public ReadFunction read;\r\n    public EOFFunction eof;\r\n    public CloseFunction close;\r\n\r\n    public static class ReadFunction implements Callback {\r\n        public Pointer invoke(Pointer ctx, Pointer output, int readSize) {\r\n            // TODO\r\n            return ctx;\r\n        }\r\n    }\r\n\r\n    public static class EOFFunction implements Callback {\r\n        public boolean invoke(Pointer ctx) {\r\n            // TODO\r\n            return false;\r\n        }\r\n    }\r\n\r\n    public static class CloseFunction implements Callback {\r\n        public void invoke(Pointer ctx) {\r\n            // TODO\r\n        }\r\n    }\r\n\r\n//    public WhisperModelLoader(Pointer p) {\r\n//        super(p);\r\n//        read = new ReadFunction();\r\n//        eof = new EOFFunction();\r\n//        close = new CloseFunction();\r\n//        read.setCallback(this);\r\n//        eof.setCallback(this);\r\n//        close.setCallback(this);\r\n//        read.write();\r\n//        eof.write();\r\n//        close.write();\r\n//    }\r\n\r\n    public WhisperModelLoader() {\r\n        super();\r\n    }\r\n\r\n    public interface ReadCallback extends Callback {\r\n        Pointer invoke(Pointer ctx, Pointer output, int readSize);\r\n    }\r\n\r\n    public interface EOFCallback extends Callback {\r\n        boolean invoke(Pointer ctx);\r\n    }\r\n\r\n    public interface CloseCallback extends Callback {\r\n        void invoke(Pointer ctx);\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/model/WhisperState.java",
    "content": "package io.github.ggerganov.whispercpp.model;\r\n\r\npublic class WhisperState {\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/model/WhisperTokenData.java",
    "content": "package io.github.ggerganov.whispercpp.model;\r\n\r\nimport com.sun.jna.Structure;\r\n\r\nimport java.util.Arrays;\r\nimport java.util.List;\r\n\r\n/**\r\n * Structure representing token data.\r\n */\r\npublic class WhisperTokenData extends Structure {\r\n\r\n    /** Token ID. */\r\n    public int id;\r\n\r\n    /** Forced timestamp token ID. */\r\n    public int tid;\r\n\r\n    /** Probability of the token. */\r\n    public float p;\r\n\r\n    /** Log probability of the token. */\r\n    public float plog;\r\n\r\n    /** Probability of the timestamp token. */\r\n    public float pt;\r\n\r\n    /** Sum of probabilities of all timestamp tokens. */\r\n    public float ptsum;\r\n\r\n    /**\r\n     * Start time of the token (token-level timestamp data).\r\n     * Do not use if you haven't computed token-level timestamps.\r\n     */\r\n    public long t0;\r\n\r\n    /**\r\n     * End time of the token (token-level timestamp data).\r\n     * Do not use if you haven't computed token-level timestamps.\r\n     */\r\n    public long t1;\r\n\r\n    /** Voice length of the token. */\r\n    public float vlen;\r\n\r\n    @Override\r\n    protected List<String> getFieldOrder() {\r\n        return Arrays.asList(\"id\", \"tid\", \"p\", \"plog\", \"pt\", \"ptsum\", \"t0\", \"t1\", \"vlen\");\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/BeamSearchParams.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\nimport com.sun.jna.Structure;\r\n\r\nimport java.util.Arrays;\r\nimport java.util.List;\r\n\r\npublic class BeamSearchParams extends Structure {\r\n    /** ref: <a href=\"https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L265\">...</a> */\r\n    public int beam_size;\r\n\r\n    /** ref: <a href=\"https://arxiv.org/pdf/2204.05424.pdf\">...</a> */\r\n    public float patience;\r\n\r\n    @Override\r\n    protected List<String> getFieldOrder() {\r\n        return Arrays.asList(\"beam_size\", \"patience\");\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/CBool.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\nimport com.sun.jna.IntegerType;\r\n\r\nimport java.util.function.BooleanSupplier;\r\n\r\npublic class CBool extends IntegerType implements BooleanSupplier {\r\n    public static final int SIZE = 1;\r\n    public static final CBool FALSE = new CBool(0);\r\n    public static final CBool TRUE = new CBool(1);\r\n\r\n\r\n    public CBool() {\r\n        this(0);\r\n    }\r\n\r\n    public CBool(long value) {\r\n        super(SIZE, value, true);\r\n    }\r\n\r\n    @Override\r\n    public boolean getAsBoolean() {\r\n        return intValue() == 1;\r\n    }\r\n\r\n    @Override\r\n    public String toString() {\r\n        return intValue() == 1 ? \"true\" : \"false\";\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/GreedyParams.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\nimport com.sun.jna.Structure;\r\n\r\nimport java.util.Collections;\r\nimport java.util.List;\r\n\r\npublic class GreedyParams extends Structure {\r\n    /** <a href=\"https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/transcribe.py#L264\">...</a> */\r\n    public int best_of;\r\n\r\n    @Override\r\n    protected List<String> getFieldOrder() {\r\n        return Collections.singletonList(\"best_of\");\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperAhead.java",
    "content": "package io.github.ggerganov.whispercpp.params;\nimport com.sun.jna.*;\nimport java.util.Arrays;\nimport java.util.List;\n\npublic class WhisperAhead extends Structure {\n\n    public int n_text_layer;\n\n    public int n_head;\n\n    public WhisperAhead() {\n        super();\n    }\n\n    public WhisperAhead(int textLayer, int head) {\n        super();\n        this.n_text_layer = textLayer;\n        this.n_head = head;\n    }\n\n    @Override\n    protected List<String> getFieldOrder() {\n        return Arrays.asList(\"n_text_layer\", \"n_head\");\n    }\n\n    public static class ByReference extends WhisperAhead implements Structure.ByReference {}\n\n    public static class ByValue extends WhisperAhead implements Structure.ByValue {}\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperAheads.java",
    "content": "package io.github.ggerganov.whispercpp.params;\nimport com.sun.jna.*;\nimport java.util.Arrays;\nimport java.util.List;\n\npublic class WhisperAheads extends Structure {\n    public NativeLong n_heads;\n\n    public Pointer heads;\n\n    public WhisperAheads() {\n        super();\n    }\n\n    /**\n     * Create alignment heads from an array of WhisperAhead objects\n     */\n    public void setHeads(WhisperAhead[] aheadsArray) {\n        this.n_heads = new NativeLong(aheadsArray.length);\n\n        int structSize = aheadsArray[0].size();\n        Memory mem = new Memory(structSize * aheadsArray.length);\n\n        for (int i = 0; i < aheadsArray.length; i++) {\n            aheadsArray[i].write();\n            byte[] buffer = aheadsArray[i].getPointer().getByteArray(0, structSize);\n            mem.write(i * structSize, buffer, 0, buffer.length);\n        }\n\n        this.heads = mem;\n    }\n\n    @Override\n    protected List<String> getFieldOrder() {\n        return Arrays.asList(\"n_heads\", \"heads\");\n    }\n\n    public static class ByReference extends WhisperAheads implements Structure.ByReference {}\n\n    public static class ByValue extends WhisperAheads implements Structure.ByValue {}\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperContextParams.java",
    "content": "package io.github.ggerganov.whispercpp.params;\nimport com.sun.jna.*;\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * Parameters for the whisper_init_from_file_with_params() function.\n * If you change the order or add new parameters, make sure to update the default values in whisper.cpp:\n * whisper_context_default_params()\n */\npublic class WhisperContextParams extends Structure {\n    public WhisperContextParams(Pointer p) {\n        super(p);\n    }\n\n    public WhisperContextParams() {\n        super();\n    }\n\n    /** Use GPU for inference (default = true) */\n    public CBool use_gpu;\n\n    /** Use flash attention (default = true) */\n    public CBool flash_attn;\n\n    /** CUDA device to use (default = 0) */\n    public int gpu_device;\n\n    /** [EXPERIMENTAL] Enable token-level timestamps with DTW (default = false) */\n    public CBool dtw_token_timestamps;\n\n    /** [EXPERIMENTAL] Alignment heads preset for DTW */\n    public int dtw_aheads_preset;\n\n    /** Number of top layers to use for DTW when using WHISPER_AHEADS_N_TOP_MOST preset */\n    public int dtw_n_top;\n\n    public WhisperAheads.ByValue dtw_aheads;\n\n    /** DTW memory size (internal use) */\n    public NativeLong dtw_mem_size;\n\n    /** Use GPU for inference */\n    public void useGpu(boolean enable) {\n        use_gpu = enable ? CBool.TRUE : CBool.FALSE;\n    }\n\n    /** Use flash attention */\n    public void useFlashAttn(boolean enable) {\n        flash_attn = enable ? CBool.TRUE : CBool.FALSE;\n    }\n\n    /** Enable DTW token-level timestamps */\n    public void enableDtwTokenTimestamps(boolean enable) {\n        dtw_token_timestamps = enable ? CBool.TRUE : CBool.FALSE;\n    }\n\n    /** Set DTW alignment heads preset */\n    public void setDtwAheadsPreset(int preset) {\n        dtw_aheads_preset = preset;\n    }\n\n    @Override\n    protected List<String> getFieldOrder() {\n        return Arrays.asList(\n            \"use_gpu\",\n            \"flash_attn\",\n            \"gpu_device\",\n            \"dtw_token_timestamps\",\n            \"dtw_aheads_preset\",\n            \"dtw_n_top\",\n            \"dtw_aheads\",\n            \"dtw_mem_size\"\n        );\n    }\n\n    public static class ByValue extends WhisperContextParams implements Structure.ByValue {\n        public ByValue() { super(); }\n        public ByValue(Pointer p) { super(p); }\n    }\n}\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperFilters.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\nimport java.util.List;\r\n\r\npublic class WhisperFilters {\r\n    int n_mel;\r\n    int n_fft;\r\n\r\n    List<Float> data;\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperFullParams.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\nimport com.sun.jna.*;\r\nimport io.github.ggerganov.whispercpp.callbacks.WhisperEncoderBeginCallback;\r\nimport io.github.ggerganov.whispercpp.callbacks.WhisperLogitsFilterCallback;\r\nimport io.github.ggerganov.whispercpp.callbacks.WhisperNewSegmentCallback;\r\nimport io.github.ggerganov.whispercpp.callbacks.WhisperProgressCallback;\r\nimport io.github.ggerganov.whispercpp.callbacks.GgmlAbortCallback;\r\n\r\nimport java.util.Arrays;\r\nimport java.util.List;\r\n\r\n/**\r\n * Parameters for the whisper_full() function.\r\n * If you change the order or add new parameters, make sure to update the default values in whisper.cpp:\r\n * whisper_full_default_params()\r\n */\r\npublic class WhisperFullParams extends Structure {\r\n\r\n    public WhisperFullParams() {\r\n        super();\r\n    }\r\n\r\n    public WhisperFullParams(Pointer p) {\r\n        super(p);\r\n    }\r\n\r\n    /** Sampling strategy for whisper_full() function. */\r\n    public int strategy;\r\n\r\n    /** Number of threads. (default = 4) */\r\n    public int n_threads;\r\n\r\n    /** Maximum tokens to use from past text as a prompt for the decoder. (default = 16384) */\r\n    public int n_max_text_ctx;\r\n\r\n    /** Start offset in milliseconds. (default = 0) */\r\n    public int offset_ms;\r\n\r\n    /** Audio duration to process in milliseconds. (default = 0) */\r\n    public int duration_ms;\r\n\r\n    /** Translate flag. (default = false) */\r\n    public CBool translate;\r\n\r\n    /** The compliment of translateMode() */\r\n    public void transcribeMode() {\r\n        translate = CBool.FALSE;\r\n    }\r\n\r\n    /** The compliment of transcribeMode() */\r\n    public void translateMode() {\r\n        translate = CBool.TRUE;\r\n    }\r\n\r\n    /** Flag to indicate whether to use past transcription (if any) as an initial prompt for the decoder. (default = true) */\r\n    public CBool no_context;\r\n\r\n    /** Flag to indicate whether to use past transcription (if any) as an initial prompt for the decoder. (default = true) */\r\n    public void enableContext(boolean enable) {\r\n        no_context = enable ? CBool.FALSE : CBool.TRUE;\r\n    }\r\n\r\n    /** Generate timestamps or not? */\r\n    public CBool no_timestamps;\r\n\r\n    /** Flag to force single segment output (useful for streaming). (default = false) */\r\n    public CBool single_segment;\r\n\r\n    /** Flag to force single segment output (useful for streaming). (default = false) */\r\n    public void singleSegment(boolean single) {\r\n        single_segment = single ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Flag to print special tokens (e.g., &lt;SOT&gt;, &lt;EOT&gt;, &lt;BEG&gt;, etc.). (default = false) */\r\n    public CBool print_special;\r\n\r\n    /** Flag to print special tokens (e.g., &lt;SOT&gt;, &lt;EOT&gt;, &lt;BEG&gt;, etc.). (default = false) */\r\n    public void printSpecial(boolean enable) {\r\n        print_special = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Flag to print progress information. (default = true) */\r\n    public CBool print_progress;\r\n\r\n    /** Flag to print progress information. (default = true) */\r\n    public void printProgress(boolean enable) {\r\n        print_progress = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Flag to print results from within whisper.cpp (avoid it, use callback instead). (default = true) */\r\n    public CBool print_realtime;\r\n\r\n    /** Flag to print results from within whisper.cpp (avoid it, use callback instead). (default = true) */\r\n    public void printRealtime(boolean enable) {\r\n        print_realtime = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Flag to print timestamps for each text segment when printing realtime. (default = true) */\r\n    public CBool print_timestamps;\r\n\r\n    /** Flag to print timestamps for each text segment when printing realtime. (default = true) */\r\n    public void printTimestamps(boolean enable) {\r\n        print_timestamps = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** [EXPERIMENTAL] Flag to enable token-level timestamps. (default = false) */\r\n    public CBool token_timestamps;\r\n\r\n    /** [EXPERIMENTAL] Flag to enable token-level timestamps. (default = false) */\r\n    public void tokenTimestamps(boolean enable) {\r\n        token_timestamps = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** [EXPERIMENTAL] Timestamp token probability threshold (~0.01). (default = 0.01) */\r\n    public float thold_pt;\r\n\r\n    /** [EXPERIMENTAL] Timestamp token sum probability threshold (~0.01). */\r\n    public float thold_ptsum;\r\n\r\n    /** Maximum segment length in characters. (default = 0) */\r\n    public int max_len;\r\n\r\n    /** Flag to split on word rather than on token (when used with max_len). (default = false) */\r\n    public CBool split_on_word;\r\n\r\n    /** Flag to split on word rather than on token (when used with max_len). (default = false) */\r\n    public void splitOnWord(boolean enable) {\r\n        split_on_word = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Maximum tokens per segment (0, default = no limit) */\r\n    public int max_tokens;\r\n\r\n    /** [EXPERIMENTAL] Enable debug mode for extra info */\r\n    public CBool debug_mode;\r\n\r\n    /** Enable debug mode */\r\n    public void enableDebugMode(boolean enable) {\r\n        debug_mode = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Overwrite the audio context size (0 = use default). */\r\n    public int audio_ctx;\r\n\r\n    /** Enable tinydiarize (default = false) */\r\n    public CBool tdrz_enable;\r\n\r\n    /** Enable tinydiarize (default = false) */\r\n    public void tdrzEnable(boolean enable) {\r\n        tdrz_enable = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Regular expression matching tokens to suppress. */\r\n    public String suppress_regex;\r\n\r\n    /** Tokens to provide to the whisper decoder as an initial prompt.\r\n     * These are prepended to any existing text context from a previous call. */\r\n    public String initial_prompt;\r\n    /** Always prepend initial_prompt for every decode chunk. */\r\n    public CBool carry_initial_prompt;\r\n\r\n    /** Prompt tokens. (int*) */\r\n    public Pointer prompt_tokens;\r\n\r\n    public void setPromptTokens(int[] tokens) {\r\n        Memory mem = new Memory(tokens.length * 4L);\r\n        mem.write(0, tokens, 0, tokens.length);\r\n        prompt_tokens = mem;\r\n    }\r\n\r\n    /** Number of prompt tokens. */\r\n    public int prompt_n_tokens;\r\n\r\n    /** Language for auto-detection.\r\n     * For auto-detection, set to `null`, `\"\"`, or \"auto\". */\r\n    public String language;\r\n\r\n    /** Flag to indicate whether to detect language automatically. */\r\n    public CBool detect_language;\r\n\r\n    /** Flag to indicate whether to detect language automatically. */\r\n    public void detectLanguage(boolean enable) {\r\n        detect_language = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    // Common decoding parameters.\r\n\r\n    /** Flag to suppress blank tokens. */\r\n    public CBool suppress_blank;\r\n\r\n    public void suppressBlanks(boolean enable) {\r\n        suppress_blank = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Flag to suppress non-speech tokens. */\r\n    public CBool suppress_nst;\r\n\r\n    /** Flag to suppress non-speech tokens. */\r\n    public void suppressNonSpeechTokens(boolean enable) {\r\n        suppress_nst = enable ? CBool.TRUE : CBool.FALSE;\r\n    }\r\n\r\n    /** Initial decoding temperature. */\r\n    public float temperature;\r\n\r\n    /** Maximum initial timestamp. */\r\n    public float max_initial_ts;\r\n\r\n    /** Length penalty. */\r\n    public float length_penalty;\r\n\r\n    // Fallback parameters.\r\n\r\n    /** Temperature increment. */\r\n    public float temperature_inc;\r\n\r\n    /** Entropy threshold (similar to OpenAI's \"compression_ratio_threshold\"). */\r\n    public float entropy_thold;\r\n\r\n    /** Log probability threshold. */\r\n    public float logprob_thold;\r\n\r\n    /** No speech threshold. */\r\n    public float no_speech_thold;\r\n\r\n    /** Greedy decoding parameters. */\r\n    public GreedyParams greedy;\r\n\r\n    /**\r\n     * Beam search decoding parameters.\r\n     */\r\n    public BeamSearchParams beam_search;\r\n\r\n    public void setBestOf(int bestOf) {\r\n        if (greedy == null) {\r\n            greedy = new GreedyParams();\r\n        }\r\n        greedy.best_of = bestOf;\r\n    }\r\n\r\n    public void setBeamSize(int beamSize) {\r\n        if (beam_search == null) {\r\n            beam_search = new BeamSearchParams();\r\n        }\r\n        beam_search.beam_size = beamSize;\r\n    }\r\n\r\n    public void setBeamSizeAndPatience(int beamSize, float patience) {\r\n        if (beam_search == null) {\r\n            beam_search = new BeamSearchParams();\r\n        }\r\n        beam_search.beam_size = beamSize;\r\n        beam_search.patience = patience;\r\n    }\r\n\r\n    /**\r\n     * Callback for every newly generated text segment.\r\n     * WhisperNewSegmentCallback\r\n     */\r\n    public Pointer new_segment_callback;\r\n\r\n    /**\r\n     * User data for the new_segment_callback.\r\n     */\r\n    public Pointer new_segment_callback_user_data;\r\n\r\n    /**\r\n     * Callback on each progress update.\r\n     * WhisperProgressCallback\r\n     */\r\n    public Pointer progress_callback;\r\n\r\n    /**\r\n     * User data for the progress_callback.\r\n     */\r\n    public Pointer progress_callback_user_data;\r\n\r\n    /**\r\n     * Callback each time before the encoder starts.\r\n     * WhisperEncoderBeginCallback\r\n     */\r\n    public Pointer encoder_begin_callback;\r\n\r\n    /**\r\n     * User data for the encoder_begin_callback.\r\n     */\r\n    public Pointer encoder_begin_callback_user_data;\r\n\r\n    /** Callback used to abort GGML computation */\r\n    public Pointer abort_callback;\r\n\r\n    /** User data for the abort_callback */\r\n    public Pointer abort_callback_user_data;\r\n\r\n    public void setAbortCallback(GgmlAbortCallback callback) {\r\n        abort_callback = CallbackReference.getFunctionPointer(callback);\r\n    }\r\n\r\n    /**\r\n     * Callback by each decoder to filter obtained logits.\r\n     * WhisperLogitsFilterCallback\r\n     */\r\n    public Pointer logits_filter_callback;\r\n\r\n    /**\r\n     * User data for the logits_filter_callback.\r\n     */\r\n    public Pointer logits_filter_callback_user_data;\r\n\r\n\r\n    public void setNewSegmentCallback(WhisperNewSegmentCallback callback) {\r\n        new_segment_callback = CallbackReference.getFunctionPointer(callback);\r\n    }\r\n\r\n    public void setProgressCallback(WhisperProgressCallback callback) {\r\n        progress_callback = CallbackReference.getFunctionPointer(callback);\r\n    }\r\n\r\n    public void setEncoderBeginCallbackeginCallbackCallback(WhisperEncoderBeginCallback callback) {\r\n        encoder_begin_callback = CallbackReference.getFunctionPointer(callback);\r\n    }\r\n\r\n    public void setLogitsFilterCallback(WhisperLogitsFilterCallback callback) {\r\n        logits_filter_callback = CallbackReference.getFunctionPointer(callback);\r\n    }\r\n\r\n    /** Grammar stuff */\r\n    public Pointer grammar_rules;\r\n    public long n_grammar_rules;\r\n    public long i_start_rule;\r\n    public float grammar_penalty;\r\n\r\n    @Override\r\n    protected List<String> getFieldOrder() {\r\n        return Arrays.asList(\"strategy\", \"n_threads\", \"n_max_text_ctx\",\r\n                \"offset_ms\", \"duration_ms\", \"translate\", \"no_context\",\r\n                \"no_timestamps\", \"single_segment\", \"print_special\",\r\n                \"print_progress\", \"print_realtime\", \"print_timestamps\",\r\n                \"token_timestamps\", \"thold_pt\", \"thold_ptsum\", \"max_len\",\r\n                \"split_on_word\", \"max_tokens\", \"debug_mode\", \"audio_ctx\",\r\n                \"tdrz_enable\", \"suppress_regex\", \"initial_prompt\", \"carry_initial_prompt\",\r\n                \"prompt_tokens\", \"prompt_n_tokens\", \"language\", \"detect_language\",\r\n                \"suppress_blank\", \"suppress_nst\", \"temperature\",\r\n                \"max_initial_ts\", \"length_penalty\", \"temperature_inc\",\r\n                \"entropy_thold\", \"logprob_thold\", \"no_speech_thold\", \"greedy\",\r\n                \"beam_search\", \"new_segment_callback\", \"new_segment_callback_user_data\",\r\n                \"progress_callback\", \"progress_callback_user_data\",\r\n                \"encoder_begin_callback\", \"encoder_begin_callback_user_data\",\r\n                \"abort_callback\", \"abort_callback_user_data\",\r\n                \"logits_filter_callback\", \"logits_filter_callback_user_data\",\r\n                \"grammar_rules\", \"n_grammar_rules\", \"i_start_rule\", \"grammar_penalty\");\r\n    }\r\n\r\n    public static class ByValue extends WhisperFullParams implements Structure.ByValue {\r\n        public ByValue() { super(); }\r\n        public ByValue(Pointer p) { super(p); }\r\n    }\r\n\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperHParams.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\npublic class WhisperHParams {\r\n    int n_vocab       = 51864;\r\n    int n_audio_ctx   = 1500;\r\n    int n_audio_state = 384;\r\n    int n_audio_head  = 6;\r\n    int n_audio_layer = 4;\r\n    int n_text_ctx    = 448;\r\n    int n_text_state  = 384;\r\n    int n_text_head   = 6;\r\n    int n_text_layer  = 4;\r\n    int n_mels        = 80;\r\n    int ftype         = 1;\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperSamplingStrategy.java",
    "content": "package io.github.ggerganov.whispercpp.params;\r\n\r\n/** Available sampling strategies */\r\npublic enum WhisperSamplingStrategy {\r\n    /** similar to OpenAI's GreedyDecoder */\r\n    WHISPER_SAMPLING_GREEDY,\r\n\r\n    /** similar to OpenAI's BeamSearchDecoder */\r\n    WHISPER_SAMPLING_BEAM_SEARCH\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/test/java/io/github/ggerganov/whispercpp/WhisperCppTest.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport static org.junit.jupiter.api.Assertions.*;\r\n\r\nimport io.github.ggerganov.whispercpp.bean.WhisperSegment;\r\nimport io.github.ggerganov.whispercpp.params.CBool;\r\nimport io.github.ggerganov.whispercpp.params.WhisperContextParams;\r\nimport io.github.ggerganov.whispercpp.params.WhisperFullParams;\r\nimport io.github.ggerganov.whispercpp.params.WhisperSamplingStrategy;\r\nimport org.junit.jupiter.api.BeforeAll;\r\nimport org.junit.jupiter.api.Test;\r\nimport javax.sound.sampled.AudioInputStream;\r\nimport javax.sound.sampled.AudioSystem;\r\nimport java.io.File;\r\nimport java.io.FileNotFoundException;\r\nimport java.util.List;\r\n\r\nclass WhisperCppTest {\r\n    private static WhisperCpp whisper = new WhisperCpp();\r\n    private static boolean modelInitialised = false;\r\n\r\n    @BeforeAll\r\n    static void init() throws FileNotFoundException {\r\n        // By default, models are loaded from ~/.cache/whisper/ and are usually named \"ggml-${name}.bin\"\r\n        // or you can provide the absolute path to the model file.\r\n        //String modelName = \"../../models/ggml-tiny.bin\";\r\n        String modelName = \"../../models/ggml-tiny.en.bin\";\r\n        try {\r\n            WhisperContextParams.ByValue contextParams = whisper.getContextDefaultParams();\r\n            contextParams.useFlashAttn(false);  // Disable flash attention\r\n            whisper.initContext(modelName, contextParams);\r\n            //whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);\r\n            //whisper.getJavaDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);\r\n            modelInitialised = true;\r\n        } catch (FileNotFoundException ex) {\r\n            System.out.println(\"Model \" + modelName + \" not found\");\r\n        }\r\n    }\r\n\r\n    @Test\r\n    void testGetDefaultFullParams_BeamSearch() {\r\n        // When\r\n        WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);\r\n\r\n        // Then\r\n        assertEquals(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal(), params.strategy);\r\n        assertNotEquals(0, params.n_threads);\r\n        assertEquals(16384, params.n_max_text_ctx);\r\n        assertFalse(params.translate);\r\n        assertEquals(0.01f, params.thold_pt);\r\n        assertEquals(5, params.beam_search.beam_size);\r\n        assertEquals(-1.0f, params.beam_search.patience);\r\n    }\r\n\r\n    @Test\r\n    void testGetDefaultFullParams_Greedy() {\r\n        // When\r\n        WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);\r\n\r\n        // Then\r\n        assertEquals(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY.ordinal(), params.strategy);\r\n        assertNotEquals(0, params.n_threads);\r\n        assertEquals(16384, params.n_max_text_ctx);\r\n        assertEquals(5, params.greedy.best_of);\r\n    }\r\n\r\n    @Test\r\n    void testFullTranscribe() throws Exception {\r\n        if (!modelInitialised) {\r\n            System.out.println(\"Model not initialised, skipping test\");\r\n            return;\r\n        }\r\n\r\n        // Given\r\n        File file = new File(System.getProperty(\"user.dir\"), \"../../samples/jfk.wav\");\r\n        AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(file);\r\n\r\n        byte[] b = new byte[audioInputStream.available()];\r\n        float[] floats = new float[b.length / 2];\r\n\r\n        //WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);\r\n        WhisperFullParams.ByValue params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);\r\n        params.setProgressCallback((ctx, state, progress, user_data) -> System.out.println(\"progress: \" + progress));\r\n        params.print_progress = CBool.FALSE;\r\n        //params.initial_prompt = \"and so my fellow Americans um, like\";\r\n\r\n\r\n        try {\r\n            audioInputStream.read(b);\r\n\r\n            for (int i = 0, j = 0; i < b.length; i += 2, j++) {\r\n                int intSample = (int) (b[i + 1]) << 8 | (int) (b[i]) & 0xFF;\r\n                floats[j] = intSample / 32767.0f;\r\n            }\r\n\r\n            // When\r\n            String result = whisper.fullTranscribe(params, floats);\r\n\r\n            // Then\r\n            System.err.println(result);\r\n            assertEquals(\"And so my fellow Americans ask not what your country can do for you \" +\r\n                    \"ask what you can do for your country.\",\r\n                    result.replace(\",\", \"\"));\r\n        } finally {\r\n            audioInputStream.close();\r\n        }\r\n    }\r\n\r\n    @Test\r\n    void testFullTranscribeWithTime() throws Exception {\r\n        if (!modelInitialised) {\r\n            System.out.println(\"Model not initialised, skipping test\");\r\n            return;\r\n        }\r\n\r\n        // Given\r\n        File file = new File(System.getProperty(\"user.dir\"), \"../../samples/jfk.wav\");\r\n        AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(file);\r\n\r\n        byte[] b = new byte[audioInputStream.available()];\r\n        float[] floats = new float[b.length / 2];\r\n\r\n        //WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);\r\n        WhisperFullParams.ByValue params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);\r\n        params.setProgressCallback((ctx, state, progress, user_data) -> System.out.println(\"progress: \" + progress));\r\n        params.print_progress = CBool.FALSE;\r\n        //params.initial_prompt = \"and so my fellow Americans um, like\";\r\n\r\n        try {\r\n            audioInputStream.read(b);\r\n\r\n            for (int i = 0, j = 0; i < b.length; i += 2, j++) {\r\n                int intSample = (int) (b[i + 1]) << 8 | (int) (b[i]) & 0xFF;\r\n                floats[j] = intSample / 32767.0f;\r\n            }\r\n\r\n            List<WhisperSegment> segments = whisper.fullTranscribeWithTime(params, floats);\r\n            assertTrue(segments.size() > 0, \"The size of segments should be greater than 0\");\r\n            for (WhisperSegment segment : segments) {\r\n                System.out.println(segment);\r\n            }\r\n        } finally {\r\n            audioInputStream.close();\r\n        }\r\n    }\r\n\r\n}\r\n"
  },
  {
    "path": "bindings/java/src/test/java/io/github/ggerganov/whispercpp/WhisperJnaLibraryTest.java",
    "content": "package io.github.ggerganov.whispercpp;\r\n\r\nimport static org.junit.jupiter.api.Assertions.*;\r\n\r\nimport org.junit.jupiter.api.Test;\r\n\r\nclass WhisperJnaLibraryTest {\r\n\r\n    @Test\r\n    void testWhisperPrint_system_info() {\r\n        String systemInfo = WhisperCppJnaLibrary.instance.whisper_print_system_info();\r\n        // eg: \"AVX = 1 | AVX2 = 1 | AVX512 = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0\r\n        //    | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | VSX = 0 | COREML = 0 | \"\r\n        System.out.println(\"System info: \" + systemInfo);\r\n        assertTrue(systemInfo.length() > 10);\r\n    }\r\n}\r\n"
  },
  {
    "path": "bindings/javascript/.gitignore",
    "content": "publish.log\n"
  },
  {
    "path": "bindings/javascript/CMakeLists.txt",
    "content": "set(TARGET libwhisper)\n\nadd_executable(${TARGET}\n    emscripten.cpp\n    )\n\ntarget_link_libraries(${TARGET} PRIVATE\n    whisper\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside whisper.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libwhisper.js\n        ${CMAKE_CURRENT_SOURCE_DIR}/whisper.js\n        )\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libwhisper.worker.js\n        ${CMAKE_CURRENT_SOURCE_DIR}/libwhisper.worker.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s MODULARIZE=1 \\\n    -s EXPORT_NAME=\\\"'whisper_factory'\\\" \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE=8 \\\n    -s ALLOW_MEMORY_GROWTH=1 \\\n    ${EXTRA_FLAGS} \\\n    \")\n"
  },
  {
    "path": "bindings/javascript/README.md",
    "content": "# whisper.cpp\n\nNode.js package for Whisper speech recognition\n\nPackage: https://www.npmjs.com/package/whisper.cpp\n\n## Details\n\nThe performance is comparable to when running `whisper.cpp` in the browser via WASM.\n\nThe API is currently very rudimentary: [bindings/javascript/emscripten.cpp](/bindings/javascript/emscripten.cpp)\n\nFor sample usage check [tests/test-whisper.js](/tests/test-whisper.js)\n\n## Package building + test\n\n```bash\n# load emscripten\nsource /path/to/emsdk/emsdk_env.sh\n\n# clone repo\ngit clone https://github.com/ggerganov/whisper.cpp\ncd whisper.cpp\n\n# grab base.en model\n./models/download-ggml-model.sh base.en\n\n# prepare PCM sample for testing\nffmpeg -i samples/jfk.wav -f f32le -acodec pcm_f32le samples/jfk.pcmf32\n\n# build\nmkdir build-em && cd build-em\nemcmake cmake .. && make -j\n\n# run test\nnode ../tests/test-whisper.js\n\n# For Node.js versions prior to v16.4.0, experimental features need to be enabled:\nnode --experimental-wasm-threads --experimental-wasm-simd ../tests/test-whisper.js\n\n# publish npm package\nmake publish-npm\n```\n\n## Sample run\n\n```text\n$ node --experimental-wasm-threads --experimental-wasm-simd ../tests/test-whisper.js\n\nwhisper_model_load: loading model from 'whisper.bin'\nwhisper_model_load: n_vocab       = 51864\nwhisper_model_load: n_audio_ctx   = 1500\nwhisper_model_load: n_audio_state = 512\nwhisper_model_load: n_audio_head  = 8\nwhisper_model_load: n_audio_layer = 6\nwhisper_model_load: n_text_ctx    = 448\nwhisper_model_load: n_text_state  = 512\nwhisper_model_load: n_text_head   = 8\nwhisper_model_load: n_text_layer  = 6\nwhisper_model_load: n_mels        = 80\nwhisper_model_load: f16           = 1\nwhisper_model_load: type          = 2\nwhisper_model_load: adding 1607 extra tokens\nwhisper_model_load: mem_required  =  506.00 MB\nwhisper_model_load: ggml ctx size =  140.60 MB\nwhisper_model_load: memory size   =   22.83 MB\nwhisper_model_load: model size    =  140.54 MB\n\nsystem_info: n_threads = 8 / 10 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | NEON = 0 | F16C = 0 | FP16_VA = 0 | WASM_SIMD = 1 | BLAS = 0 |\n\noperator(): processing 176000 samples, 11.0 sec, 8 threads, 1 processors, lang = en, task = transcribe ...\n\n[00:00:00.000 --> 00:00:11.000]   And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.\n\nwhisper_print_timings:     load time =   162.37 ms\nwhisper_print_timings:      mel time =   183.70 ms\nwhisper_print_timings:   sample time =     4.27 ms\nwhisper_print_timings:   encode time =  8582.63 ms / 1430.44 ms per layer\nwhisper_print_timings:   decode time =   436.16 ms / 72.69 ms per layer\nwhisper_print_timings:    total time =  9370.90 ms\n```\n"
  },
  {
    "path": "bindings/javascript/emscripten.cpp",
    "content": "//\n// This is the Javascript API of whisper.cpp\n//\n// Very crude at the moment.\n// Feel free to contribute and make this better!\n//\n// See the tests/test-whisper.js for sample usage\n//\n\n#include \"whisper.h\"\n\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <thread>\n#include <vector>\n\nstruct whisper_context * g_context;\n\nEMSCRIPTEN_BINDINGS(whisper) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model) {\n        if (g_context == nullptr) {\n            g_context = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n            if (g_context != nullptr) {\n                return true;\n            } else {\n                return false;\n            }\n        }\n\n        return false;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([]() {\n        if (g_context) {\n            whisper_free(g_context);\n            g_context = nullptr;\n        }\n    }));\n\n    emscripten::function(\"full_default\", emscripten::optional_override([](const emscripten::val & audio, const std::string & lang, bool translate) {\n        if (g_context == nullptr) {\n            return -1;\n        }\n\n        struct whisper_full_params params = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n\n        params.print_realtime   = true;\n        params.print_progress   = false;\n        params.print_timestamps = true;\n        params.print_special    = false;\n        params.translate        = translate;\n        params.language         = whisper_is_multilingual(g_context) ? lang.c_str() : \"en\";\n        params.n_threads        = std::min(8, (int) std::thread::hardware_concurrency());\n        params.offset_ms        = 0;\n\n        std::vector<float> pcmf32;\n        const int n = audio[\"length\"].as<int>();\n\n        emscripten::val heap = emscripten::val::module_property(\"HEAPU8\");\n        emscripten::val memory = heap[\"buffer\"];\n\n        pcmf32.resize(n);\n\n        emscripten::val memoryView = audio[\"constructor\"].new_(memory, reinterpret_cast<uintptr_t>(pcmf32.data()), n);\n        memoryView.call<void>(\"set\", audio);\n\n        // print system information\n        {\n            printf(\"\\n\");\n            printf(\"system_info: n_threads = %d / %d | %s\\n\",\n                    params.n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());\n\n            printf(\"\\n\");\n            printf(\"%s: processing %d samples, %.1f sec, %d threads, %d processors, lang = %s, task = %s ...\\n\",\n                    __func__, int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,\n                    params.n_threads, 1,\n                    params.language,\n                    params.translate ? \"translate\" : \"transcribe\");\n\n            printf(\"\\n\");\n        }\n\n        // run whisper\n        {\n            whisper_reset_timings(g_context);\n            whisper_full(g_context, params, pcmf32.data(), pcmf32.size());\n            whisper_print_timings(g_context);\n        }\n\n        return 0;\n    }));\n}\n"
  },
  {
    "path": "bindings/javascript/libwhisper.worker.js",
    "content": "\"use strict\";var Module={};var ENVIRONMENT_IS_NODE=typeof process==\"object\"&&typeof process.versions==\"object\"&&typeof process.versions.node==\"string\";if(ENVIRONMENT_IS_NODE){var nodeWorkerThreads=require(\"worker_threads\");var parentPort=nodeWorkerThreads.parentPort;parentPort.on(\"message\",data=>onmessage({data:data}));var fs=require(\"fs\");Object.assign(global,{self:global,require:require,Module:Module,location:{href:__filename},Worker:nodeWorkerThreads.Worker,importScripts:f=>(0,eval)(fs.readFileSync(f,\"utf8\")+\"//# sourceURL=\"+f),postMessage:msg=>parentPort.postMessage(msg),performance:global.performance||{now:Date.now}})}var initializedJS=false;function threadPrintErr(){var text=Array.prototype.slice.call(arguments).join(\" \");if(ENVIRONMENT_IS_NODE){fs.writeSync(2,text+\"\\n\");return}console.error(text)}function threadAlert(){var text=Array.prototype.slice.call(arguments).join(\" \");postMessage({cmd:\"alert\",text:text,threadId:Module[\"_pthread_self\"]()})}var err=threadPrintErr;self.alert=threadAlert;Module[\"instantiateWasm\"]=(info,receiveInstance)=>{var module=Module[\"wasmModule\"];Module[\"wasmModule\"]=null;var instance=new WebAssembly.Instance(module,info);return receiveInstance(instance)};self.onunhandledrejection=e=>{throw e.reason||e};function handleMessage(e){try{if(e.data.cmd===\"load\"){let messageQueue=[];self.onmessage=e=>messageQueue.push(e);self.startWorker=instance=>{Module=instance;postMessage({\"cmd\":\"loaded\"});for(let msg of messageQueue){handleMessage(msg)}self.onmessage=handleMessage};Module[\"wasmModule\"]=e.data.wasmModule;for(const handler of e.data.handlers){Module[handler]=(...args)=>{postMessage({cmd:\"callHandler\",handler:handler,args:args})}}Module[\"wasmMemory\"]=e.data.wasmMemory;Module[\"buffer\"]=Module[\"wasmMemory\"].buffer;Module[\"ENVIRONMENT_IS_PTHREAD\"]=true;if(typeof e.data.urlOrBlob==\"string\"){importScripts(e.data.urlOrBlob)}else{var objectUrl=URL.createObjectURL(e.data.urlOrBlob);importScripts(objectUrl);URL.revokeObjectURL(objectUrl)}whisper_factory(Module)}else if(e.data.cmd===\"run\"){Module[\"__emscripten_thread_init\"](e.data.pthread_ptr,0,0,1);Module[\"__emscripten_thread_mailbox_await\"](e.data.pthread_ptr);Module[\"establishStackSpace\"]();Module[\"PThread\"].receiveObjectTransfer(e.data);Module[\"PThread\"].threadInitTLS();if(!initializedJS){Module[\"__embind_initialize_bindings\"]();initializedJS=true}try{Module[\"invokeEntryPoint\"](e.data.start_routine,e.data.arg)}catch(ex){if(ex!=\"unwind\"){throw ex}}}else if(e.data.cmd===\"cancel\"){if(Module[\"_pthread_self\"]()){Module[\"__emscripten_thread_exit\"](-1)}}else if(e.data.target===\"setimmediate\"){}else if(e.data.cmd===\"checkMailbox\"){if(initializedJS){Module[\"checkMailbox\"]()}}else if(e.data.cmd){err(`worker.js received unknown command ${e.data.cmd}`);err(e.data)}}catch(ex){if(Module[\"__emscripten_thread_crashed\"]){Module[\"__emscripten_thread_crashed\"]()}throw ex}}self.onmessage=handleMessage;\n"
  },
  {
    "path": "bindings/javascript/package-tmpl.json",
    "content": "{\n  \"name\": \"whisper.cpp\",\n  \"version\": \"@PROJECT_VERSION@\",\n  \"description\": \"Whisper speech recognition\",\n  \"main\": \"whisper.js\",\n  \"scripts\": {\n    \"test\": \"echo \\\"todo: add tests\\\" && exit 0\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/ggerganov/whisper.cpp\"\n  },\n  \"keywords\": [\n    \"openai\",\n    \"whisper\",\n    \"speech-to-text\",\n    \"speech-recognition\",\n    \"transformer\"\n  ],\n  \"author\": \"Georgi Gerganov\",\n  \"license\": \"MIT\",\n  \"bugs\": {\n    \"url\": \"https://github.com/ggerganov/whisper.cpp/issues\"\n  },\n  \"homepage\": \"https://github.com/ggerganov/whisper.cpp#readme\"\n}\n"
  },
  {
    "path": "bindings/javascript/package.json",
    "content": "{\n  \"name\": \"whisper.cpp\",\n  \"version\": \"1.8.4\",\n  \"description\": \"Whisper speech recognition\",\n  \"main\": \"whisper.js\",\n  \"scripts\": {\n    \"test\": \"echo \\\"todo: add tests\\\" && exit 0\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/ggerganov/whisper.cpp\"\n  },\n  \"keywords\": [\n    \"openai\",\n    \"whisper\",\n    \"speech-to-text\",\n    \"speech-recognition\",\n    \"transformer\"\n  ],\n  \"author\": \"Georgi Gerganov\",\n  \"license\": \"MIT\",\n  \"bugs\": {\n    \"url\": \"https://github.com/ggerganov/whisper.cpp/issues\"\n  },\n  \"homepage\": \"https://github.com/ggerganov/whisper.cpp#readme\"\n}\n"
  },
  {
    "path": "bindings/javascript/whisper.js",
    "content": "\nvar whisper_factory = (() => {\n  var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined;\n  if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename;\n  return (\nfunction(moduleArg = {}) {\n\nfunction GROWABLE_HEAP_I8(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAP8}function GROWABLE_HEAP_U8(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAPU8}function GROWABLE_HEAP_I16(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAP16}function GROWABLE_HEAP_U16(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAPU16}function GROWABLE_HEAP_I32(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAP32}function GROWABLE_HEAP_U32(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAPU32}function GROWABLE_HEAP_F32(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAPF32}function GROWABLE_HEAP_F64(){if(wasmMemory.buffer!=HEAP8.buffer){updateMemoryViews()}return HEAPF64}var Module=moduleArg;var readyPromiseResolve,readyPromiseReject;Module[\"ready\"]=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject});var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram=\"./this.program\";var quit_=(status,toThrow)=>{throw toThrow};var ENVIRONMENT_IS_WEB=typeof window==\"object\";var ENVIRONMENT_IS_WORKER=typeof importScripts==\"function\";var ENVIRONMENT_IS_NODE=typeof process==\"object\"&&typeof process.versions==\"object\"&&typeof process.versions.node==\"string\";var ENVIRONMENT_IS_PTHREAD=Module[\"ENVIRONMENT_IS_PTHREAD\"]||false;var scriptDirectory=\"\";function locateFile(path){if(Module[\"locateFile\"]){return Module[\"locateFile\"](path,scriptDirectory)}return scriptDirectory+path}var read_,readAsync,readBinary,setWindowTitle;if(ENVIRONMENT_IS_NODE){var fs=require(\"fs\");var nodePath=require(\"path\");if(ENVIRONMENT_IS_WORKER){scriptDirectory=nodePath.dirname(scriptDirectory)+\"/\"}else{scriptDirectory=__dirname+\"/\"}read_=(filename,binary)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);return fs.readFileSync(filename,binary?undefined:\"utf8\")};readBinary=filename=>{var ret=read_(filename,true);if(!ret.buffer){ret=new Uint8Array(ret)}return ret};readAsync=(filename,onload,onerror,binary=true)=>{filename=isFileURI(filename)?new URL(filename):nodePath.normalize(filename);fs.readFile(filename,binary?undefined:\"utf8\",(err,data)=>{if(err)onerror(err);else onload(binary?data.buffer:data)})};if(!Module[\"thisProgram\"]&&process.argv.length>1){thisProgram=process.argv[1].replace(/\\\\/g,\"/\")}arguments_=process.argv.slice(2);quit_=(status,toThrow)=>{process.exitCode=status;throw toThrow};Module[\"inspect\"]=()=>\"[Emscripten Module object]\";let nodeWorkerThreads;try{nodeWorkerThreads=require(\"worker_threads\")}catch(e){console.error('The \"worker_threads\" module is not supported in this node.js build - perhaps a newer version is needed?');throw e}global.Worker=nodeWorkerThreads.Worker}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!=\"undefined\"&&document.currentScript){scriptDirectory=document.currentScript.src}if(_scriptDir){scriptDirectory=_scriptDir}if(scriptDirectory.indexOf(\"blob:\")!==0){scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,\"\").lastIndexOf(\"/\")+1)}else{scriptDirectory=\"\"}if(!ENVIRONMENT_IS_NODE){read_=url=>{var xhr=new XMLHttpRequest;xhr.open(\"GET\",url,false);xhr.send(null);return xhr.responseText};if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open(\"GET\",url,false);xhr.responseType=\"arraybuffer\";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=(url,onload,onerror)=>{var xhr=new XMLHttpRequest;xhr.open(\"GET\",url,true);xhr.responseType=\"arraybuffer\";xhr.onload=()=>{if(xhr.status==200||xhr.status==0&&xhr.response){onload(xhr.response);return}onerror()};xhr.onerror=onerror;xhr.send(null)}}setWindowTitle=title=>document.title=title}else{}if(ENVIRONMENT_IS_NODE){if(typeof performance==\"undefined\"){global.performance=require(\"perf_hooks\").performance}}var defaultPrint=console.log.bind(console);var defaultPrintErr=console.error.bind(console);if(ENVIRONMENT_IS_NODE){defaultPrint=(...args)=>fs.writeSync(1,args.join(\" \")+\"\\n\");defaultPrintErr=(...args)=>fs.writeSync(2,args.join(\" \")+\"\\n\")}var out=Module[\"print\"]||defaultPrint;var err=Module[\"printErr\"]||defaultPrintErr;Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module[\"arguments\"])arguments_=Module[\"arguments\"];if(Module[\"thisProgram\"])thisProgram=Module[\"thisProgram\"];if(Module[\"quit\"])quit_=Module[\"quit\"];var wasmBinary;if(Module[\"wasmBinary\"])wasmBinary=Module[\"wasmBinary\"];var noExitRuntime=Module[\"noExitRuntime\"]||true;if(typeof WebAssembly!=\"object\"){abort(\"no native wasm support detected\")}var wasmMemory;var wasmModule;var ABORT=false;var EXITSTATUS;function assert(condition,text){if(!condition){abort(text)}}var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module[\"HEAP8\"]=HEAP8=new Int8Array(b);Module[\"HEAP16\"]=HEAP16=new Int16Array(b);Module[\"HEAPU8\"]=HEAPU8=new Uint8Array(b);Module[\"HEAPU16\"]=HEAPU16=new Uint16Array(b);Module[\"HEAP32\"]=HEAP32=new Int32Array(b);Module[\"HEAPU32\"]=HEAPU32=new Uint32Array(b);Module[\"HEAPF32\"]=HEAPF32=new Float32Array(b);Module[\"HEAPF64\"]=HEAPF64=new Float64Array(b)}var INITIAL_MEMORY=Module[\"INITIAL_MEMORY\"]||16777216;assert(INITIAL_MEMORY>=65536,\"INITIAL_MEMORY should be larger than STACK_SIZE, was \"+INITIAL_MEMORY+\"! (STACK_SIZE=\"+65536+\")\");if(ENVIRONMENT_IS_PTHREAD){wasmMemory=Module[\"wasmMemory\"]}else{if(Module[\"wasmMemory\"]){wasmMemory=Module[\"wasmMemory\"]}else{wasmMemory=new WebAssembly.Memory({\"initial\":INITIAL_MEMORY/65536,\"maximum\":2147483648/65536,\"shared\":true});if(!(wasmMemory.buffer instanceof SharedArrayBuffer)){err(\"requested a shared WebAssembly.Memory but the returned buffer is not a SharedArrayBuffer, indicating that while the browser has SharedArrayBuffer it does not have WebAssembly threads support - you may need to set a flag\");if(ENVIRONMENT_IS_NODE){err(\"(on node you may need: --experimental-wasm-threads --experimental-wasm-bulk-memory and/or recent version)\")}throw Error(\"bad memory\")}}}updateMemoryViews();INITIAL_MEMORY=wasmMemory.buffer.byteLength;var wasmTable;var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;var runtimeKeepaliveCounter=0;function keepRuntimeAlive(){return noExitRuntime||runtimeKeepaliveCounter>0}function preRun(){if(Module[\"preRun\"]){if(typeof Module[\"preRun\"]==\"function\")Module[\"preRun\"]=[Module[\"preRun\"]];while(Module[\"preRun\"].length){addOnPreRun(Module[\"preRun\"].shift())}}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;if(ENVIRONMENT_IS_PTHREAD)return;if(!Module[\"noFSInit\"]&&!FS.init.initialized)FS.init();FS.ignorePermissions=false;TTY.init();callRuntimeCallbacks(__ATINIT__)}function postRun(){if(ENVIRONMENT_IS_PTHREAD)return;if(Module[\"postRun\"]){if(typeof Module[\"postRun\"]==\"function\")Module[\"postRun\"]=[Module[\"postRun\"]];while(Module[\"postRun\"].length){addOnPostRun(Module[\"postRun\"].shift())}}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function getUniqueRunDependency(id){return id}function addRunDependency(id){runDependencies++;if(Module[\"monitorRunDependencies\"]){Module[\"monitorRunDependencies\"](runDependencies)}}function removeRunDependency(id){runDependencies--;if(Module[\"monitorRunDependencies\"]){Module[\"monitorRunDependencies\"](runDependencies)}if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){if(Module[\"onAbort\"]){Module[\"onAbort\"](what)}what=\"Aborted(\"+what+\")\";err(what);ABORT=true;EXITSTATUS=1;what+=\". Build with -sASSERTIONS for more info.\";var e=new WebAssembly.RuntimeError(what);readyPromiseReject(e);throw e}var dataURIPrefix=\"data:application/octet-stream;base64,\";function isDataURI(filename){return filename.startsWith(dataURIPrefix)}function isFileURI(filename){return filename.startsWith(\"file://\")}var wasmBinaryFile;wasmBinaryFile=\"data:application/octet-stream;base64,AGFzbQEAAAAByARKYAF/AX9gAX8AYAJ/fwBgA39/fwF/YAJ/fwF/YAN/f38AYAR/f39/AGAGf39/f39/AX9gBH9/f38Bf2AAAGAFf39/f38Bf2AFf39/f38AYAZ/f39/f38AYAh/f39/f39/fwF/YAABf2AHf39/f39/fwF/YAd/f39/f39/AGABfQF9YAV/fn5+fgBgA39+fwF+YAV/f39/fgF/YAR/f39/AX5gBH9+fn8AYAp/f39/f39/f39/AX9gB39/f39/fn4Bf2AGf39/f35+AX9gBX9/fn9/AGADf39/AXxgBH9/f38BfGAEf39+fgF/YAJ/fgBgAXwBfWACfH8BfGAFf39+fn4Bf2AIf39/f39/f38AYAx/f39/f39/f39/f38Bf2ADf35/AX9gD39/f39/f39/f39/f39/fwBgCn9/f39/f39/f38AYAt/f39/f39/f39/fwF/YAZ/fH9/f38Bf2AJf39/f39/f39/AGAFf39/f3wBf2AAAXxgA39/fgF/YAABfmACf3wAYAN/f30Bf2AEfn5+fgF/YAJ9fQF9YAJ+fwF/YAN/f3wBf2ABfABgCH9/fn5+f39/AX9gBX9/f399AGAGf39+fn9/AX9gAn5+AXxgAnx8AXxgBn9/f39+fgBgBH9/fn4AYAR/f35/AX9gA39/fwF9YAR/f39+AX5gA39/fgBgAn5+AX1gAn9/AX5gA35+fgF/YAF+AX9gAXwBfGABfwF+YAJ9fwF/YAV/f399fQBgBH9/f30AYAl/f39/f39/f38BfwKmAjABYQFiAAkBYQFjAAUBYQFkAAUBYQFlACsBYQFmAAEBYQFnAAsBYQFoABABYQFpAAQBYQFqAAABYQFrAAUBYQFsAAEBYQFtAAEBYQFuAAkBYQFvAAABYQFwAAgBYQFxAAMBYQFyAAIBYQFzAAUBYQF0AAgBYQF1AAABYQF2AAEBYQF3ABsBYQF4AAoBYQF5ABABYQF6AA4BYQFBAA4BYQFCAAoBYQFDAAQBYQFEAAQBYQFFAAABYQFGAAUBYQFHAAgBYQFIABwBYQFJAAEBYQFKAAEBYQFLAAgBYQFMAAMBYQFNAAgBYQFOAAkBYQFPAAEBYQFQAA4BYQFRAAIBYQFSAAYBYQFTAAIBYQFUAAEBYQFVAAYBYQFWAAQBYQFhAgOAAoCAAgOhB58HAQAFAAAFEQkEBAICAAYBAgACAgQEDhEJCSwHAxIACQUDAAMAAQAdABEDAgIAAAAAAxIFBAoGAhYLAC0RBQAAAgMEEgADAwACBQUEEQMCAgEEHgEAHx8AAgMCCgoIAC4BAQIBDQ0HBwIGCAICAAYABQQFBQECAgEEAQIELwMCATAWCCACAjECBwEABQIFIRcAABcAAgUEADIEBQACBSIKAwUEBQEBAzM0AQgCAgQLAAUFBTUhAB0ABAIAAAIQAxAPDwgFAAICAAMBAQMBAw4RCQkJAgUJAgMDAh4BNgUEAgICBQIBAAABDAYFCQAODgEEBAQCAgQFCwAEFQA3CgMABAQDAyMLAyMLBgASAAILBQAAATgEChEICQkOCQkDATkDJAAJAAgKAQUIBgYCCQkEAgcLDAMHAwMCAgIFBgUJAwcFBAIAAgIFOgIFOwABAAAGBAkBBAQACAsiAgQCBAMBAgEBFQEAAAUCBQAABQIFBAE8AAAKDQ0KDQ0ACg0ABAEEAAECAAMDJSYDJSYAAAInAgAABQInAgQCDAsMDAsMDAADEAMQBwMGGz0VCAcIFQgFBT4ACAEEBAM/QEEGEhIWEkIAAAQEAgMBAAAFAgACAAEBBAMOAwkEBQYADyAEAREEBQkBBAACBAEFCQEBAQQEAQIBAAIAQwEFREUkAwQAAgVGCQQEBQYCAQcHAwIJCggACAUJCQYAAgkHAwMDAQUDAQYIRwIFBQUCAgUAAwQEAwcpAQUGBwUOBgQAAgsLC0gBCQYGF0kPEAoMAAEOAAAAAAwMDAsLCwYGBgMDAQECAAEBAAEAAQABAAEAAQABAAEAAQABAAEAAwEAAQABAAICAgICAgAAAQEACgAKDQ0BCgoDCAMEAwQBCgMIAwQDBAgICAMBAQEJDAwHGAcYDw8IDw8PDw0HBwcHBw0HBwcHBwoZKhQKFAoKChkqFAoUCgoHBwcHBwcHBwcHCQcHBwcHBwUHBwMGCgMFBgoGAwYGAQYBAAYCAAYaAwQEAAEGAAAGAQYBAAMAAAYDBhoDAQAABgIFAwIoBQkFEw4BBQEBBQEBABwCAQEBBQEBBQkFAAMDBRMJDgUJAA4FAAQAAgAABAACAAAEAAIAAAQABQQCAAIFAAUCAAIFAgUCAgUCAQICAQACAQAFAgIBAAIBAAUFAgEAAgEAAgUCBQQBBQIFAAkFBQAAAQEFAwABBQEBBAICAgQAAQUFCwsABAABAAQJBAcBcAGdBJ0EBj8MfwFBwJk5C38BQQALfwFBAAt/AUEAC38BQQALfwFBAAt/AUEAC38BQQALfwFBAAt/AUEAC38BQQQLfwFBAAsHjgEZAVcA7wQBWAEAAVkALwFaADsBXwDmBgEkAMgGAmFhAOUGAmJhAOQGAmNhAOIGAmRhAPcEAmVhAOEGAmZhANEGAmdhAMwGAmhhAMsGAmlhAMUGAmphAL8GAmthAPoEAmxhAPkEAm1hAPgEAm5hAP4EAm9hAPYEAnBhAPUEAnFhAPQEAnJhAPMEAnNhAPIECALNBwn0BwEAQQELnAT7BtAGrwaIBt0FpQX8AoUBrgeTAvEElgaPBvAElAe5B+QEvgaKB7UH4QS2BocHsAfVBLAG/wasB8wErgb6BqUHpwerBpwHnQfnBuMGswShBt8G2wauBJ4G2QbWBqYEnAbNBsoGlwSbBsYGxAaPBJkGwAbLB8oHxwfMB8kHyAfCB8EHwAe/B74HvQe8B8YHxQfEB8MHugeFAbgHtwe2B98ErQeNBbQHsgexB7sHswevB4UBPasHqgeFAT2pB6gHPYAHjwHFBI8BhAKPAaYHjwGkB6MHogehB6AHnweeB48BjAWbB5oHmQeYB5cHlgePAZUHkweSB5EHkAePB44HjwGNB74EjAeLB48BiQePAYgHgwfVAoYHhQfVAj2EB9UCggeBBz3FBI8B/gaFAT39BvwGogE9+Qb4BvcGPfYG9QaiAT30BvMG8gY98QbwBqIBPe8G7gbtBj3sBusGogE96gbpBugGsATgBt4G3QbcBtgG1wbSAdUGmQTUBpgE0wbOBtIGzwaiAckGbccGwwbCBsEGPb0GrAS8BrsGhAK6BrkGuAZtbbcGtQa0BoAEswaABL8C/wOyBrEG8gGqBqIGpgalBqQGowapBqgGpwa+AvYDoAafBroCnQaaBi+iAdIFrQOqBagFpgWjBaEFnwWdBZsFmQWXBZUFkwWRBY8FrwPTBdEFqwPFBcQFwwXCBcEFrAPABb8FvgWyA7wFuwW6BbkFuAVttwW2BaEDtQWzBbIFsQWvBa0FoAO0Ba0GrAawBa4FrAWFAT090AXPBc4FzQXMBcsFygXJBawDyAXHBcYFPaoDqgPNAa0CrQK9Ba0CPacDpgPNAW1tpQPjAT2nA6YDzQFtbaUD4wE9pAOjA80BbW2iA+MBPaQDowPNAW1togPjAYUBPZgGlwaVBoUBPZQGkwaSBj2RBpAGjgaNBtoD2gOMBosGigaJBocGPYYGhQaEBoMG0wPTA4IGgQaABv8F/gU9/QX8BfsF+gX5BfgF9wX2BT31BfQF8wXyBfEF8AXvBe4FhQE9zgPtBewF6wXqBekF6AWrBacFogWWBZIFngWaBYUBPc4D5wXmBeUF5AXjBeIFqQWkBaAFlAWQBZwFmAWpAp4D4QWpAp4D4AU96QHpAXx8fMYDbZoBmgE96QHpAXx8fMYDbZoBmgE96AHoAXx8fMQDbZoBmgE96AHoAXx8fMQDbZoBmgE93wXeBT3cBdsFPdoF2QU92AXXBT2zA9YFhAI9swPVBYQCjgWAA40DlAOLBYADhQE9ogGiAYkFPYgF/wSCBYcFPYAFgwWGBT2BBYQFhQU9/AQ9+wQ9/QSVApQDlAKNA5UClQIMARsKuaojnwemDAEHfwJAIABFDQBBvIc1LQAAQQJxBEBBwIc1EFYNAQsgAEEIayICIABBBGsoAgAiAUF4cSIAaiEFAkACQCABQQFxDQAgAUEDcUUNASACIAIoAgAiAWsiAkGQhDUoAgBJDQEgACABaiEAAkACQEGUhDUoAgAgAkcEQCABQf8BTQRAIAFBA3YhBCACKAIMIgEgAigCCCIDRgRAQYCENUGAhDUoAgBBfiAEd3E2AgAMBQsgAyABNgIMIAEgAzYCCAwECyACKAIYIQYgAiACKAIMIgFHBEAgAigCCCIDIAE2AgwgASADNgIIDAMLIAJBFGoiBCgCACIDRQRAIAIoAhAiA0UNAiACQRBqIQQLA0AgBCEHIAMiAUEUaiIEKAIAIgMNACABQRBqIQQgASgCECIDDQALIAdBADYCAAwCCyAFKAIEIgFBA3FBA0cNAkGIhDUgADYCACAFIAFBfnE2AgQgAiAAQQFyNgIEIAUgADYCAAwDC0EAIQELIAZFDQACQCACKAIcIgNBAnRBsIY1aiIEKAIAIAJGBEAgBCABNgIAIAENAUGEhDVBhIQ1KAIAQX4gA3dxNgIADAILIAZBEEEUIAYoAhAgAkYbaiABNgIAIAFFDQELIAEgBjYCGCACKAIQIgMEQCABIAM2AhAgAyABNgIYCyACKAIUIgNFDQAgASADNgIUIAMgATYCGAsgAiAFTw0AIAUoAgQiAUEBcUUNAAJAAkACQAJAIAFBAnFFBEBBmIQ1KAIAIAVGBEBBmIQ1IAI2AgBBjIQ1QYyENSgCACAAaiIANgIAIAIgAEEBcjYCBCACQZSENSgCAEcNBkGIhDVBADYCAEGUhDVBADYCAAwGC0GUhDUoAgAgBUYEQEGUhDUgAjYCAEGIhDVBiIQ1KAIAIABqIgA2AgAgAiAAQQFyNgIEIAAgAmogADYCAAwGCyABQXhxIABqIQAgAUH/AU0EQCABQQN2IQQgBSgCDCIBIAUoAggiA0YEQEGAhDVBgIQ1KAIAQX4gBHdxNgIADAULIAMgATYCDCABIAM2AggMBAsgBSgCGCEGIAUgBSgCDCIBRwRAQZCENSgCABogBSgCCCIDIAE2AgwgASADNgIIDAMLIAVBFGoiBCgCACIDRQRAIAUoAhAiA0UNAiAFQRBqIQQLA0AgBCEHIAMiAUEUaiIEKAIAIgMNACABQRBqIQQgASgCECIDDQALIAdBADYCAAwCCyAFIAFBfnE2AgQgAiAAQQFyNgIEIAAgAmogADYCAAwDC0EAIQELIAZFDQACQCAFKAIcIgNBAnRBsIY1aiIEKAIAIAVGBEAgBCABNgIAIAENAUGEhDVBhIQ1KAIAQX4gA3dxNgIADAILIAZBEEEUIAYoAhAgBUYbaiABNgIAIAFFDQELIAEgBjYCGCAFKAIQIgMEQCABIAM2AhAgAyABNgIYCyAFKAIUIgNFDQAgASADNgIUIAMgATYCGAsgAiAAQQFyNgIEIAAgAmogADYCACACQZSENSgCAEcNAEGIhDUgADYCAAwBCyAAQf8BTQRAIABBeHFBqIQ1aiEBAn9BgIQ1KAIAIgNBASAAQQN2dCIAcUUEQEGAhDUgACADcjYCACABDAELIAEoAggLIQAgASACNgIIIAAgAjYCDCACIAE2AgwgAiAANgIIDAELQR8hAyAAQf///wdNBEAgAEEmIABBCHZnIgFrdkEBcSABQQF0a0E+aiEDCyACIAM2AhwgAkIANwIQIANBAnRBsIY1aiEBAkACQAJAQYSENSgCACIEQQEgA3QiB3FFBEBBhIQ1IAQgB3I2AgAgASACNgIAIAIgATYCGAwBCyAAQRkgA0EBdmtBACADQR9HG3QhAyABKAIAIQEDQCABIgQoAgRBeHEgAEYNAiADQR12IQEgA0EBdCEDIAQgAUEEcWoiB0EQaigCACIBDQALIAcgAjYCECACIAQ2AhgLIAIgAjYCDCACIAI2AggMAQsgBCgCCCIAIAI2AgwgBCACNgIIIAJBADYCGCACIAQ2AgwgAiAANgIIC0GghDVBoIQ1KAIAQQFrIgBBfyAAGzYCAAtBvIc1LQAAQQJxRQ0AQcCHNRBSGgsLmwIBA38gAEUEQEHAuwMoAgAiAARAIAAQMCEBC0GougMoAgAiAARAIAAQMCABciEBCxD8ASgCACIABEADQEEAIQIgACgCTEEATgRAIAAQggEhAgsgACgCFCAAKAIcRwRAIAAQMCABciEBCyACBEAgABCBAQsgACgCOCIADQALC0HU8jQQ0wEgAQ8LIAAoAkxBAE4EQCAAEIIBIQILAkACQAJAIAAoAhQgACgCHEYNACAAQQBBACAAKAIkEQMAGiAAKAIUDQBBfyEBIAINAQwCCyAAKAIEIgEgACgCCCIDRwRAIAAgASADa6xBASAAKAIoERMAGgtBACEBIABBADYCHCAAQgA3AxAgAEIANwIEIAJFDQELIAAQgQELIAELJAEBfyMAQRBrIgMkACADIAI2AgwgACABIAIQhgQgA0EQaiQACzcBAX9BASAAIABBAU0bIQACQANAIAAQOyIBDQFBrJk1/hACACIBBEAgAREJAAwBCwsQAAALIAELJQAgAC0AC0EHdgRAIAAgACgCACAAKAIIQf////8HcRDxAQsgAAuJAQEDfyMAQZAIayIDJAAgAyACNgKMCAJAIANBgAggASACELABIgRB/wdMBEAgACADQezXNCgCAEHMuAMoAgARBQAMAQsgBEEBaiIFEDIiAiAFIAEgAygCjAgQsAEaIAIgBGpBADoAACAAIAJB7Nc0KAIAQcy4AygCABEFACACEC8LIANBkAhqJAALjQECAX0CfyAAvCICQRd2Qf8BcSIDQZUBTQR9IANB/QBNBEAgAEMAAAAAlA8LAn0gACAAjCACQQBOGyIAQwAAAEuSQwAAAMuSIACTIgFDAAAAP14EQCAAIAGSQwAAgL+SDAELIAAgAZIiACABQwAAAL9fRQ0AGiAAQwAAgD+SCyIAIACMIAJBAE4bBSAACwsJAEHiExDuBAALDAAgACABIAEQaBBfC4ICAQR/An8gARBoIQIjAEEQayIFJAACfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQsiBEEATwRAAkAgAiAALQALQQd2BH8gACgCCEH/////B3FBAWsFQQoLIgMgBGtNBEAgAkUNAQJ/IAAtAAtBB3YEQCAAKAIADAELIAALIgMgBAR/IAIgA2ogAyAE/AoAACABIAJBACADIARqIAFLG0EAIAEgA08bagUgAQsgAvwKAAAgACACIARqIgEQmQEgBUEAOgAPIAEgA2ogBS0ADzoAAAwBCyAAIAMgAiAEaiADayAEQQBBACACIAEQzAELIAVBEGokACAADAELEOwCAAsL5gMBCH8jAEEgayIEJAAgBEEMaiEFAkAgBEEVaiIHIgIgBEEgaiIGRg0AIAFBAE4NACACQS06AAAgAkEBaiECQQAgAWshAQsgBQJ/IAYiAyACayIIQQlMBEBBPSAIQSAgAUEBcmdrQdEJbEEMdSIJIAlBAnRBoKoDaigCACABTWpIDQEaCwJ/IAFBv4Q9TQRAIAFBj84ATQRAIAFB4wBNBEAgAUEJTQRAIAIgAUEwajoAACACQQFqDAQLIAIgARCpAQwDCyABQecHTQRAIAIgAUHkAG4iA0EwajoAACACQQFqIAEgA0HkAGxrEKkBDAMLIAIgARCgAgwCCyABQZ+NBk0EQCACIAFBkM4AbiIDQTBqOgAAIAJBAWogASADQZDOAGxrEKACDAILIAIgARCfAgwBCyABQf/B1y9NBEAgAUH/rOIETQRAIAIgAUHAhD1uIgNBMGo6AAAgAkEBaiABIANBwIQ9bGsQnwIMAgsgAiABEJ4CDAELIAFB/5Pr3ANNBEAgAiABQYDC1y9uIgNBMGo6AAAgAkEBaiABIANBgMLXL2xrEJ4CDAELIAIgAUGAwtcvbiIDEKkBIAEgA0GAwtcvbGsQngILIQNBAAs2AgQgBSADNgIAIAAgByAEKAIMEN8DIAYkAAuDAgEDfwJAAn8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELIgIgAUkEQCMAQRBrIgQkACABIAJrIgIEQCACIAAtAAtBB3YEfyAAKAIIQf////8HcUEBawVBCgsiAwJ/IAAtAAtBB3YEQCAAKAIEDAELIAAtAAtB/wBxCyIBa0sEQCAAIAMgAiADayABaiABIAEQpQILIAECfyAALQALQQd2BEAgACgCAAwBCyAACyIDaiACQQAQpAIgACABIAJqIgAQmQEgBEEAOgAPIAAgA2ogBC0ADzoAAAsgBEEQaiQADAELIAACfyAALQALQQd2BEAgACgCAAwBCyAACyABEJwDCwuMKQELf0HogzUoAgBFBEAQhAQLAkBBvIc1LQAAQQJxBEBBwIc1EFYNAQsCQAJAIABB9AFNBEBBgIQ1KAIAIgFBECAAQQtqQXhxIABBC0kbIgVBA3YiA3YiAkEDcQRAAkAgAkF/c0EBcSADaiIDQQN0IgBBqIQ1aiICIABBsIQ1aigCACIGKAIIIgBGBEBBgIQ1IAFBfiADd3E2AgAMAQsgACACNgIMIAIgADYCCAsgBkEIaiEEIAYgA0EDdCIAQQNyNgIEIAAgBmoiACAAKAIEQQFyNgIEDAMLIAVBiIQ1KAIAIgRNDQEgAgRAAkBBAiADdCIAQQAgAGtyIAIgA3RxaCIDQQN0IgBBqIQ1aiICIABBsIQ1aigCACIKKAIIIgBGBEBBgIQ1IAFBfiADd3EiATYCAAwBCyAAIAI2AgwgAiAANgIICyAKIAVBA3I2AgQgBSAKaiICIANBA3QiACAFayIDQQFyNgIEIAAgCmogAzYCACAEBEAgBEF4cUGohDVqIQVBlIQ1KAIAIQYCfyABQQEgBEEDdnQiAHFFBEBBgIQ1IAAgAXI2AgAgBQwBCyAFKAIICyEAIAUgBjYCCCAAIAY2AgwgBiAFNgIMIAYgADYCCAsgCkEIaiEEQZSENSACNgIAQYiENSADNgIADAMLQYSENSgCACIGRQ0BAn8gBmhBAnRBsIY1aigCACICKAIEQXhxIAVrIQcgAiEAA0ACQCAAKAIQIgFFBEAgACgCFCIBRQ0BCyABKAIEQXhxIAVrIgAgByAAIAdJIgAbIQcgASACIAAbIQIgASEADAELC0EAIAVBAEwNABogAigCGCEKAkAgAiACKAIMIgNHBEBBkIQ1KAIAGiACKAIIIgAgAzYCDCADIAA2AggMAQsCQCACQRRqIgAoAgAiAUUEQCACKAIQIgFFDQEgAkEQaiEACwNAIAAhBCABIgNBFGoiACgCACIBDQAgA0EQaiEAIAMoAhAiAQ0ACyAEQQA2AgAMAQtBACEDCwJAIApFDQACQCACKAIcIgFBAnRBsIY1aiIAKAIAIAJGBEAgACADNgIAIAMNAUGEhDUgBkF+IAF3cTYCAAwCCyAKQRBBFCAKKAIQIAJGG2ogAzYCACADRQ0BCyADIAo2AhggAigCECIABEAgAyAANgIQIAAgAzYCGAsgAigCFCIARQ0AIAMgADYCFCAAIAM2AhgLAkAgB0EPTQRAIAIgBSAHaiIAQQNyNgIEIAAgAmoiACAAKAIEQQFyNgIEDAELIAIgBUEDcjYCBCACIAVqIgMgB0EBcjYCBCADIAdqIAc2AgBBiIQ1KAIAIgAEQCAAQXhxQaiENWohBkGUhDUoAgAhBAJ/QYCENSgCACIBQQEgAEEDdnQiAHFFBEBBgIQ1IAAgAXI2AgAgBgwBCyAGKAIICyEAIAYgBDYCCCAAIAQ2AgwgBCAGNgIMIAQgADYCCAtBlIQ1IAM2AgBBiIQ1IAc2AgALIAJBCGoLIgQNAgwBC0F/IQUgAEG/f0sNACAAQQtqIgBBeHEhBUGEhDUoAgAiCUUNAEEAIAVrIQQCQAJAAkACf0EAIAVBgAJJDQAaQR8gBUH///8HSw0AGiAFQSYgAEEIdmciAGt2QQFxIABBAXRrQT5qCyIKQQJ0QbCGNWooAgAiA0UEQEEAIQAMAQtBACEAIAVBGSAKQQF2a0EAIApBH0cbdCEBA0ACQCADKAIEQXhxIAVrIgYgBE8NACADIQIgBiIEDQBBACEEIAIhAAwDCyAAIAMoAhQiBiAGIAMgAUEddkEEcWooAhAiA0YbIAAgBhshACABQQF0IQEgAw0ACwsgACACckUEQEEAIQJBAiAKdCIAQQAgAGtyIAlxIgBFDQMgAGhBAnRBsIY1aigCACEACyAARQ0BCwNAIAAoAgRBeHEgBWsiASAESSEDIAEgBCADGyEEIAAgAiADGyECIAAoAhAiAQR/IAEFIAAoAhQLIgANAAsLIAJFDQAgBEGIhDUoAgAgBWtPDQAgAigCGCEKAkAgAiACKAIMIgFHBEBBkIQ1KAIAGiACKAIIIgAgATYCDCABIAA2AggMAQsCQCACQRRqIgMoAgAiAEUEQCACKAIQIgBFDQEgAkEQaiEDCwNAIAMhBiAAIgFBFGoiAygCACIADQAgAUEQaiEDIAEoAhAiAA0ACyAGQQA2AgAMAQtBACEBCwJAIApFDQACQCACKAIcIgNBAnRBsIY1aiIAKAIAIAJGBEAgACABNgIAIAENAUGEhDUgCUF+IAN3cSIJNgIADAILIApBEEEUIAooAhAgAkYbaiABNgIAIAFFDQELIAEgCjYCGCACKAIQIgAEQCABIAA2AhAgACABNgIYCyACKAIUIgBFDQAgASAANgIUIAAgATYCGAsCQCAEQQ9NBEAgAiAEIAVqIgBBA3I2AgQgACACaiIAIAAoAgRBAXI2AgQMAQsgAiAFQQNyNgIEIAIgBWoiBiAEQQFyNgIEIAQgBmogBDYCACAEQf8BTQRAIARBeHFBqIQ1aiEDAn9BgIQ1KAIAIgFBASAEQQN2dCIAcUUEQEGAhDUgACABcjYCACADDAELIAMoAggLIQAgAyAGNgIIIAAgBjYCDCAGIAM2AgwgBiAANgIIDAELQR8hACAEQf///wdNBEAgBEEmIARBCHZnIgBrdkEBcSAAQQF0a0E+aiEACyAGIAA2AhwgBkIANwIQIABBAnRBsIY1aiEBAkACQCAJQQEgAHQiA3FFBEBBhIQ1IAMgCXI2AgAgASAGNgIADAELIARBGSAAQQF2a0EAIABBH0cbdCEAIAEoAgAhBQNAIAUiASgCBEF4cSAERg0CIABBHXYhAyAAQQF0IQAgASADQQRxaiIDKAIQIgUNAAsgAyAGNgIQCyAGIAE2AhggBiAGNgIMIAYgBjYCCAwBCyABKAIIIgAgBjYCDCABIAY2AgggBkEANgIYIAYgATYCDCAGIAA2AggLIAJBCGohBAwBCyAFQYiENSgCACIATQRAQZSENSgCACECAkAgACAFayIDQRBPBEAgAiAFaiIBIANBAXI2AgQgACACaiADNgIAIAIgBUEDcjYCBAwBCyACIABBA3I2AgQgACACaiIAIAAoAgRBAXI2AgRBACEBQQAhAwtBiIQ1IAM2AgBBlIQ1IAE2AgAgAkEIaiEEDAELIAVBjIQ1KAIAIgBJBEBBjIQ1IAAgBWsiATYCAEGYhDVBmIQ1KAIAIgIgBWoiADYCACAAIAFBAXI2AgQgAiAFQQNyNgIEIAJBCGohBAwBC0EAIQRB6IM1KAIARQRAEIQEC0HwgzUoAgAiACAFQS9qIgpqQQAgAGtxIgMgBU0NAEG4hzUoAgAiAgRAQbCHNSgCACIBIANqIgAgAU0NASAAIAJLDQELAkACQAJAAkACQAJAAkACQEG8hzUtAABBBHFFBEACQAJAAkACQEGYhDUoAgAiAgRAQdiHNSEAA0AgAiAAKAIAIgFPBEAgASAAKAIEaiACSw0DCyAAKAIIIgANAAsLQfCHNRBWGkEAEMUBIgFBf0YNAyADIQJB7IM1KAIAIgRBAWsiACABcQRAIAIgAWsgACABakEAIARrcWohAgsgAiAFTQ0DQbiHNSgCACIGBEBBsIc1KAIAIgQgAmoiACAETQ0EIAAgBksNBAsgAhDFASIAIAFHDQEMBQtB8Ic1EFYaQfCDNSgCACIBIApBjIQ1KAIAa2pBACABa3EiAhDFASIBIAAoAgAgACgCBGpGDQEgASEACyAAQX9GDQEgBUEwaiACSwRAQfCDNSgCACIBIAogAmtqQQAgAWtxIgEQxQFBf0YNAiABIAJqIQILIAAhAQwDCyABQX9HDQILQbyHNUG8hzUoAgBBBHI2AgBB8Ic1EFIaC0HwhzUQVhogAxDFASEBQQAQxQEhAEHwhzUQUhogAUF/Rg0CIABBf0YNAiAAIAFNDQIgACABayICIAVBKGpNDQIMAQtB8Ic1EFIaC0GwhzVBsIc1KAIAIAJqIgA2AgBBtIc1KAIAIABJBEBBtIc1IAA2AgALAkACQAJAQZiENSgCACIIBEBB2Ic1IQADQCABIAAoAgAiBCAAKAIEIgNqRg0CIAAoAggiAA0ACwwCC0GQhDUoAgAiAEEAIAAgAU0bRQRAQZCENSABNgIAC0EAIQBB3Ic1IAI2AgBB2Ic1IAE2AgBBoIQ1QX82AgBBpIQ1QeiDNSgCADYCAEHkhzVBADYCAANAIABBA3QiBEGwhDVqIARBqIQ1aiIDNgIAIARBtIQ1aiADNgIAIABBAWoiAEEgRw0AC0GMhDUgAkEoayIDQXggAWtBB3EiAGsiAjYCAEGYhDUgACABaiIANgIAIAAgAkEBcjYCBCABIANqQSg2AgRBnIQ1QfiDNSgCADYCAAwCCyABIAhNDQAgBCAISw0AIAAoAgxBCHENACAAIAIgA2o2AgRBmIQ1IAhBeCAIa0EHcSIAaiIDNgIAQYyENUGMhDUoAgAgAmoiASAAayIANgIAIAMgAEEBcjYCBCABIAhqQSg2AgRBnIQ1QfiDNSgCADYCAAwBC0GQhDUoAgAgAUsEQEGQhDUgATYCAAsgASACaiEDQdiHNSEAAkACQAJAA0AgAyAAKAIARwRAIAAoAggiAA0BDAILCyAALQAMQQhxRQ0BC0HYhzUhAANAIAggACgCACIDTwRAIAMgACgCBGoiBiAISw0DCyAAKAIIIQAMAAsACyAAIAE2AgAgACAAKAIEIAJqNgIEIAFBeCABa0EHcWoiCiAFQQNyNgIEIANBeCADa0EHcWoiCSAFIApqIgdrIQAgCCAJRgRAQZiENSAHNgIAQYyENUGMhDUoAgAgAGoiADYCACAHIABBAXI2AgQMCAtBlIQ1KAIAIAlGBEBBlIQ1IAc2AgBBiIQ1QYiENSgCACAAaiIANgIAIAcgAEEBcjYCBCAAIAdqIAA2AgAMCAsgCSgCBCIEQQNxQQFHDQYgBEF4cSEGIARB/wFNBEAgCSgCDCICIAkoAggiAUYEQEGAhDVBgIQ1KAIAQX4gBEEDdndxNgIADAcLIAEgAjYCDCACIAE2AggMBgsgCSgCGCEFIAkgCSgCDCIBRwRAIAkoAggiAiABNgIMIAEgAjYCCAwFCyAJQRRqIgMoAgAiBEUEQCAJKAIQIgRFDQQgCUEQaiEDCwNAIAMhAiAEIgFBFGoiAygCACIEDQAgAUEQaiEDIAEoAhAiBA0ACyACQQA2AgAMBAtBjIQ1IAJBKGsiBEF4IAFrQQdxIgBrIgM2AgBBmIQ1IAAgAWoiADYCACAAIANBAXI2AgQgASAEakEoNgIEQZyENUH4gzUoAgA2AgAgCCAGQScgBmtBB3FqQS9rIgAgACAIQRBqSRsiA0EbNgIEIANB4Ic1KQIANwIQIANB2Ic1KQIANwIIQeCHNSADQQhqNgIAQdyHNSACNgIAQdiHNSABNgIAQeSHNUEANgIAIANBGGohAANAIABBBzYCBCAAQQhqIQsgAEEEaiEAIAsgBkkNAAsgAyAIRg0AIAMgAygCBEF+cTYCBCAIIAMgCGsiBEEBcjYCBCADIAQ2AgAgBEH/AU0EQCAEQXhxQaiENWohAgJ/QYCENSgCACIBQQEgBEEDdnQiAHFFBEBBgIQ1IAAgAXI2AgAgAgwBCyACKAIICyEAIAIgCDYCCCAAIAg2AgwgCCACNgIMIAggADYCCAwBC0EfIQAgBEH///8HTQRAIARBJiAEQQh2ZyIAa3ZBAXEgAEEBdGtBPmohAAsgCCAANgIcIAhCADcCECAAQQJ0QbCGNWohAQJAAkBBhIQ1KAIAIgNBASAAdCICcUUEQEGEhDUgAiADcjYCACABIAg2AgAMAQsgBEEZIABBAXZrQQAgAEEfRxt0IQAgASgCACECA0AgAiIBKAIEQXhxIARGDQIgAEEddiECIABBAXQhACABIAJBBHFqIgMoAhAiAg0ACyADIAg2AhALIAggATYCGCAIIAg2AgwgCCAINgIIDAELIAEoAggiACAINgIMIAEgCDYCCCAIQQA2AhggCCABNgIMIAggADYCCAtBjIQ1KAIAIgAgBU0NAEGMhDUgACAFayIBNgIAQZiENUGYhDUoAgAiAiAFaiIANgIAIAAgAUEBcjYCBCACIAVBA3I2AgQgAkEIaiEEDAYLIwNBHGpBMDYCAEEAIQQMBQtBACEBCyAFRQ0AAkAgCSgCHCIDQQJ0QbCGNWoiAigCACAJRgRAIAIgATYCACABDQFBhIQ1QYSENSgCAEF+IAN3cTYCAAwCCyAFQRBBFCAFKAIQIAlGG2ogATYCACABRQ0BCyABIAU2AhggCSgCECICBEAgASACNgIQIAIgATYCGAsgCSgCFCICRQ0AIAEgAjYCFCACIAE2AhgLIAAgBmohACAGIAlqIgkoAgQhBAsgCSAEQX5xNgIEIAcgAEEBcjYCBCAAIAdqIAA2AgAgAEH/AU0EQCAAQXhxQaiENWohAgJ/QYCENSgCACIBQQEgAEEDdnQiAHFFBEBBgIQ1IAAgAXI2AgAgAgwBCyACKAIICyEAIAIgBzYCCCAAIAc2AgwgByACNgIMIAcgADYCCAwBC0EfIQQgAEH///8HTQRAIABBJiAAQQh2ZyIBa3ZBAXEgAUEBdGtBPmohBAsgByAENgIcIAdCADcCECAEQQJ0QbCGNWohAwJAAkBBhIQ1KAIAIgJBASAEdCIBcUUEQEGEhDUgASACcjYCACADIAc2AgAgByADNgIYDAELIABBGSAEQQF2a0EAIARBH0cbdCEEIAMoAgAhAQNAIAEiAigCBEF4cSAARg0CIARBHXYhASAEQQF0IQQgAiABQQRxaiIDKAIQIgENAAsgAyAHNgIQIAcgAjYCGAsgByAHNgIMIAcgBzYCCAwBCyACKAIIIgAgBzYCDCACIAc2AgggB0EANgIYIAcgAjYCDCAHIAA2AggLIApBCGohBAtBvIc1LQAAQQJxRQ0AQcCHNRBSGgsgBAv/AgEHfyAAAn8CQAJAIAEoAgQiBEUEQCABQQRqIgUhAgwBCyACKAIAIAIgAi0ACyIFwEEASCIGGyEIIAIoAgQgBSAGGyEGA0ACQAJAAkACQAJAIAQiAigCFCACLQAbIgQgBMBBAEgiBxsiBCAGIAQgBkkiCRsiBQRAIAggAigCECACQRBqIAcbIgcgBRBKIgpFBEAgBCAGSw0CDAMLIApBAE4NAgwBCyAEIAZNDQILIAIhBSACKAIAIgQNBAwFCyAHIAggBRBKIgQNAQsgCQ0BDAQLIARBAE4NAwsgAigCBCIEDQALIAJBBGohBQtBIBAyIgQgAygCACIDKQIANwIQIAQgAygCCDYCGCADQgA3AgAgA0EANgIIIAQgAjYCCCAEQgA3AgAgBEEANgIcIAUgBDYCACAEIQIgASgCACgCACIDBEAgASADNgIAIAUoAgAhAgsgASgCBCACEKQBIAEgASgCCEEBajYCCEEBDAELIAIhBEEACzoABCAAIAQ2AgALBgAgABAvC4cFAgp/AX4jAEFAaiICJAAgAiAALQAIOgA4IAIgACkDADcDMCAAKAIUIQMgACgCECEEIABCADcDECAAKAIYIQUgAEEANgIYIAIgACgCRDYCKCACIAApAjw3AyAgAiAA/QACLP0LAxAgAiAA/QACHP0LAwAgACgCXCEGIAAoAlghByAAQgA3A1ggACgCTCEIIAAoAlAhCSAAKAJUIQogACgCSCELIAD9DAAAAAAAAAAAAAAAAAAAAAD9CwNIIAAgASkDADcDACAAIAEtAAg6AAggACABKAIQNgIQIAAgASgCFDYCFCAAIAEoAhg2AhggACkDYCEMIAFBADYCGCABQgA3AxAgACABKAJENgJEIAAgASkCPDcCPCAAIAH9AAIs/QsCLCAAIAH9AAIc/QsCHCAAQcgAaiABQcgAahCHAiABIAItADg6AAggASACKQMwNwMAIAEoAhAiAARAIAEgADYCFCAAEC8LIAEgBTYCGCABIAM2AhQgASAENgIQIAEgAigCKDYCRCABIAIpAyA3AjwgASAC/QADEP0LAiwgASAC/QADAP0LAhwgASgCSCIEBEAgASgCTCIDIAQiAEcEQANAIANBDGsiACgCACIFBEAgA0EIayAFNgIAIAUQLwsgACIDIARHDQALIAEoAkghAAsgASAENgJMIAAQLwsgASALNgJIIAEgCTYCUCABIAg2AkwgASgCVCIEBEAgASgCWCIDIAQiAEcEQANAIANBDGsiACgCACIFBEAgA0EIayAFNgIAIAUQLwsgACIDIARHDQALIAEoAlQhAAsgASAENgJYIAAQLwsgASAKNgJUIAEgDDcDYCABIAY2AlwgASAHNgJYIAJBQGskAAuMAgIDfwJ+AkAgACkDcCIEQgBSIAQgACkDeCAAKAIEIgEgACgCLCICa6x8IgVXcUUEQCMAQRBrIgIkAEF/IQECQCAAENACDQAgACACQQ9qQQEgACgCIBEDAEEBRw0AIAItAA8hAQsgAkEQaiQAIAEiA0EATg0BIAAoAgQhASAAKAIsIQILIABCfzcDcCAAIAE2AmggACAFIAIgAWusfDcDeEF/DwsgBUIBfCEFIAAoAgQhASAAKAIIIQICQCAAKQNwIgRQDQAgBCAFfSIEIAIgAWusWQ0AIAEgBKdqIQILIAAgAjYCaCAAIAUgACgCLCIAIAFrrHw3A3ggACABTwRAIAFBAWsgAzoAAAsgAwu4AQEDfwJAIAEQkwMiAiAALQALQQd2BH8gACgCCEH/////B3FBAWsFQQELIgNNBEACfyAALQALQQd2BEAgACgCAAwBCyAACyIDIAEgAkECdCIE/AoAACMAQRBrIgEkACAAIAIQmQEgAUEANgIMIAMgBGogASgCDDYCACABQRBqJAAMAQsgACADIAIgA2sCfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQsiAEEAIAAgAiABEIcDCwsJACAAIAEQiAMLEAAgABDwAyABEPADc0EBcwsQACAAEPEDIAEQ8QNzQQFzC+ECAQR/AkBB9Ik1/hIAAEEBcQ0AQfSJNRBURQ0AQZCINRClBCMAQSBrIgEkAAJAAkADQCABQQhqIgIgAEECdGogAEHlM0Hh+QBBASAAdEH/////B3EbEOUDIgM2AgAgA0F/Rg0BIABBAWoiAEEGRw0AC0G40QIhACACQbjRAkEYEEpFDQFB0NECIQAgAkHQ0QJBGBBKRQ0BQQAhAEHIiDUtAABFBEADQCAAQQJ0QZiINWogAEHh+QAQ5QM2AgAgAEEBaiIAQQZHDQALQciINUEBOgAAQbCINUGYiDUoAgA2AgALQZiINSEAIAFBCGoiAkGYiDVBGBBKRQ0BQbCINSEAIAJBsIg1QRgQSkUNAUEYEDsiAEUNACAAIAEpAgg3AgAgACABKQIYNwIQIAAgASkCEDcCCAwBC0EAIQALIAFBIGokAEGQiDUQ0wFB8Ik1IAA2AgBB9Ik1EFMLQfCJNSgCAAv2AQMCfAJ/AX4CfQJAIAC8IgNBFHZB/w9xIgRBqwhJDQBDAAAAACADQYCAgHxGDQEaIARB+A9PBEAgACAAkg8LIABDF3KxQl4EQCMAQRBrIgNDAAAAcDgCDCADKgIMQwAAAHCUDwsgAEO08c/CXUUNACMAQRBrIgNDAAAAEDgCDCADKgIMQwAAABCUDwtBgNsBKwMAQfjaASsDACAAu6IiASABQfDaASsDACIBoCICIAGhoSIBokGI2wErAwCgIAEgAaKiQZDbASsDACABokQAAAAAAADwP6CgIAK9IgVCL4YgBadBH3FBA3RB0NgBaikDAHy/orYLCwUAEAAACy4BAX9BBBBeIgBB6LUDNgIAIABBwLUDNgIAIABB1LUDNgIAIABBxLYDQQgQAgALMAECfyMAQRBrIgMkACADIAI3AwggACABQQEgA0EIakEAQQAQSSEEIANBEGokACAEC5IJAwl/AXsDfiMAQSBrIgkkAAJAIARFBEAMAQsgBCgCyAEiCEUEQCAEIQgMAQsgBCgCzAEgBWohBQsgAUEkbCIEQZibAWooAgAiDCADKQMAIARBlJsBajQCACISf6dsIQcCQCACQQJIDQACQCACQQJGBEBBASEEDAEL/QwAAAAAAQAAAAAAAAAAAAAAIAf9HAAhDyACQQFrIgZBAXIhBCAGQX5xIQpBACEHA0AgDyADIAdBA3RBCHJq/QADACAP/Q0AAQIDCAkKCwABAgMAAQID/bUBIQ8gB0ECaiIHIApHDQALIA8gDyAP/Q0EBQYHAAECAwABAgMAAQID/bUB/RsAIQcgBiAKRg0BCwNAIAcgAyAEQQN0aigCAGwhByAEQQFqIgQgAkcNAAsLAkACQAJ/IAgEQCAFIAdqIQYgCCkDICEQIAgpAxghEQJ/IAgoAgAiBEEQa0FxTQRAIARBJGxBmJsBaigCACAIKAIwIAgoAhBBAWtsaiAIKAI0IBGnQQFrbGogCCgCOCAQp0EBa2xqIAgoAjwgCCgCKEEBa2xqDAELIAg1AjQgEUIBfX4gCCkDECAINQIwfiAEQSRsQZSbAWo1AgB/fCAINQI4IBBCAX1+fCAINQI8IAgpAyhCAX1+fKcLIAZJDQMgCCgC0AEiBCAFakEAIAQbIQtBAAwBC0EAIAAtAAkNABogACgCICIGRQRAIAcMAQsgByAAKAIYIgdqIgQgACgCHCIKSwRAIAkgCjYCCCAJIAQ2AgQgCUHzGTYCAEG+8wAgCRCzAUEAIQYMAgsgACAENgIYIAYgB2ohC0EACyEKQQAhBCAAQQAgCkGoAmoQsAIhByAAKAIEIg0gBygCACIHaiIGIAI2AgwgBkIANwIEIAYgATYCACAGQeCiAf0ABAD9CwMQIAZBIGpB8KIB/QAEAP0LAwAgBkEwakEAQZgB/AsAIAYgBkGoAmogCyAKGzYC0AEgBiAFNgLMASAGIAg2AsgBIAZB1AFqQQBB1AD8CwACQCACQQBMDQACQCACQQhJDQAgByANaiADa0EQakEQSQ0AIAJBfnEhBEEAIQcDQCAGIAdBA3QiAWogASADav0AAwD9CwMQIAdBAmoiByAERw0ACyACIARGDQELIARBf3MgAmohDiACIARrQQNxIgUEQEEAIQcDQCAGIARBA3QiCGogAyAIaikDADcDECAEQQFqIQQgB0EBaiIHIAVHDQALCyAOQQJNDQADQCAGQRBqIgUgBEEDdCIBaiABIANqKQMANwMAIAUgAUEIaiIHaiADIAdqKQMANwMAIAUgAUEQaiIHaiADIAdqKQMANwMAIAUgAUEYaiIBaiABIANqKQMANwMAIARBBGoiBCACRw0ACwsgBiAMNgIwIAYgDCAGKQMQIBJ/p2wiATYCNCAGIAEgBigCGGwiATYCOCAGIAEgBigCIGw2AjwgACAAKAIMQQFqNgIMCyAJQSBqJAAgBg8LQbzDAigCABAwGiAJQebVADYCGCAJQbETNgIUIAlB5CY2AhBBuMMCKAIAQcvkACAJQRBqEDEQAAALgQEBAn8CQAJAIAJBBE8EQCAAIAFyQQNxDQEDQCAAKAIAIAEoAgBHDQIgAUEEaiEBIABBBGohACACQQRrIgJBA0sNAAsLIAJFDQELA0AgAC0AACIDIAEtAAAiBEYEQCABQQFqIQEgAEEBaiEAIAJBAWsiAg0BDAILCyADIARrDwtBAAvFCgIFfxF+IwBB4ABrIgUkACAEQv///////z+DIQwgAiAEhUKAgICAgICAgIB/gyEKIAJC////////P4MiDUIgiCEOIARCMIinQf//AXEhBwJAAkAgAkIwiKdB//8BcSIJQf//AWtBgoB+TwRAIAdB//8Ba0GBgH5LDQELIAFQIAJC////////////AIMiC0KAgICAgIDA//8AVCALQoCAgICAgMD//wBRG0UEQCACQoCAgICAgCCEIQoMAgsgA1AgBEL///////////8AgyICQoCAgICAgMD//wBUIAJCgICAgICAwP//AFEbRQRAIARCgICAgICAIIQhCiADIQEMAgsgASALQoCAgICAgMD//wCFhFAEQCACIAOEUARAQoCAgICAgOD//wAhCkIAIQEMAwsgCkKAgICAgIDA//8AhCEKQgAhAQwCCyADIAJCgICAgICAwP//AIWEUARAIAEgC4QhGUIAIQEgGVAEQEKAgICAgIDg//8AIQoMAwsgCkKAgICAgIDA//8AhCEKDAILIAEgC4RQBEBCACEBDAILIAIgA4RQBEBCACEBDAILIAtC////////P1gEQCAFQdAAaiABIA0gASANIA1QIgYbeSAGQQZ0rXynIgZBD2sQZkEQIAZrIQYgBSkDWCINQiCIIQ4gBSkDUCEBCyACQv///////z9WDQAgBUFAayADIAwgAyAMIAxQIggbeSAIQQZ0rXynIghBD2sQZiAGIAhrQRBqIQYgBSkDSCEMIAUpA0AhAwsgA0IPhiILQoCA/v8PgyICIAFCIIgiBH4iECALQiCIIhMgAUL/////D4MiAX58Ig9CIIYiESABIAJ+fCILIBFUrSACIA1C/////w+DIg1+IhUgBCATfnwiESAMQg+GIhIgA0IxiIRC/////w+DIgMgAX58IhQgDyAQVK1CIIYgD0IgiIR8Ig8gAiAOQoCABIQiDH4iFiANIBN+fCIOIBJCIIhCgICAgAiEIgIgAX58IhAgAyAEfnwiEkIghnwiF3whASAHIAlqIAZqQf//AGshBgJAIAIgBH4iGCAMIBN+fCIEIBhUrSAEIAQgAyANfnwiBFatfCACIAx+fCAEIAQgESAVVK0gESAUVq18fCIEVq18IAMgDH4iAyACIA1+fCICIANUrUIghiACQiCIhHwgBCACQiCGfCICIARUrXwgAiACIBAgElatIA4gFlStIA4gEFatfHxCIIYgEkIgiIR8IgJWrXwgAiACIA8gFFStIA8gF1atfHwiAlatfCIEQoCAgICAgMAAg0IAUgRAIAZBAWohBgwBCyALQj+IIRogBEIBhiACQj+IhCEEIAJCAYYgAUI/iIQhAiALQgGGIQsgGiABQgGGhCEBCyAGQf//AU4EQCAKQoCAgICAgMD//wCEIQpCACEBDAELAn4gBkEATARAQQEgBmsiB0H/AE0EQCAFQTBqIAsgASAGQf8AaiIGEGYgBUEgaiACIAQgBhBmIAVBEGogCyABIAcQrwEgBSACIAQgBxCvASAFKQMwIAUpAziEQgBSrSAFKQMgIAUpAxCEhCELIAUpAyggBSkDGIQhASAFKQMAIQIgBSkDCAwCC0IAIQEMAgsgBEL///////8/gyAGrUIwhoQLIAqEIQogC1AgAUIAWSABQoCAgICAgICAgH9RG0UEQCAKIAJCAXwiAVCtfCEKDAELIAsgAUKAgICAgICAgIB/hYRCAFIEQCACIQEMAQsgCiACIAJCAYN8IgEgAlStfCEKCyAAIAE3AwAgACAKNwMIIAVB4ABqJAAL7AEBBn8jAEEgayIBJAAgAUEANgIQIAFB+AE2AgwgASABKQIMNwMAIAFBFGoiAyABKQIANwIEIAMgADYCACMAQRBrIgIkACAA/hACAEF/RwRAAkAgAkEMaiIFIAM2AgAgAkEIaiIEIAU2AgBBkJg1EFYaA0AgACgCAEEBRgRAQaiYNUGQmDUQjgMMAQsLIAAoAgBFBEAgAEEB/hcCAEGQmDUQUhogBBCtA0GQmDUQVhogAEF//hcCAEGQmDUQUhpBqJg1EKYCGgwBC0GQmDUQUhoLCyACQRBqJAAgACgCBCEGIAFBIGokACAGQQFrCwkAQcEdEO4EAAujCgEJfyMAQRBrIgkkACABQQRqQQH+HgIAGiMAQRBrIgMkACADIAE2AgwgCSADKAIMNgIMIANBEGokACACIABBCGoiACgCBCAAKAIAIgNrQQJ1TwRAAkAgAkEBaiIBIAAoAgQiBCADa0ECdSIDSwRAIwBBIGsiCyQAAkAgASADayIGIAAoAgggBGtBAnVNBEAgACAGELEDDAELIABBEGohByALQQxqIQECfyAGIAAoAgQgACgCAGtBAnVqIQUjAEEQayIEJAAgBCAFNgIMIAUgABCYAyIDTQRAIAAoAgggACgCAGtBAnUiBSADQQF2SQRAIAQgBUEBdDYCCCMAQRBrIgMkACAEQQhqIgUoAgAgBEEMaiIIKAIASSEKIANBEGokACAIIAUgChsoAgAhAwsgBEEQaiQAIAMMAQsQNgALIQUgACgCBCAAKAIAa0ECdSEIQQAhAyMAQRBrIgQkACAEQQA2AgwgAUEANgIMIAEgBzYCECAFBH8gBEEEaiABKAIQIAUQlwMgBCgCBCEDIAQoAggFQQALIQUgASADNgIAIAEgAyAIQQJ0aiIHNgIIIAEgBzYCBCABIAMgBUECdGo2AgwgBEEQaiQAIwBBEGsiBCQAIAQgASgCCDYCBCABKAIIIQMgBCABQQhqNgIMIAQgAyAGQQJ0ajYCCCAEKAIEIQMDQCAEKAIIIANHBEAgASgCEBogBCgCBEEANgIAIAQgBCgCBEEEaiIDNgIEDAELCyAEKAIMIAQoAgQ2AgAgBEEQaiQAIwBBEGsiBiQAIAAoAggaIAAoAgAaIAYgACgCBDYCCCAGIAAoAgA2AgQgBiABKAIENgIAIAYoAgghByAGKAIEIQggBigCACEKIwBBEGsiBSQAIwBBIGsiAyQAIwBBEGsiBCQAIAQgBzYCDCAEIAg2AgggAyAEKAIMNgIYIAMgBCgCCDYCHCAEQRBqJAAgAygCGCEHIAMoAhwhCCMAQRBrIgQkACAEIAg2AgggBCAHNgIMIAQgCjYCBANAIAQoAgwiByAEKAIIRwRAIAQoAgRBBGsgB0EEaygCADYCACAEIAQoAgxBBGs2AgwgBCAEKAIEQQRrNgIEDAELCyADIAQoAgw2AhAgAyAEKAIENgIUIARBEGokACADIAMoAhA2AgwgAyADKAIUNgIIIAUgAygCDDYCCCAFIAMoAgg2AgwgA0EgaiQAIAUoAgwhAyAFQRBqJAAgBiADNgIMIAEgBigCDDYCBCAAKAIAIQMgACABKAIENgIAIAEgAzYCBCAAKAIEIQMgACABKAIINgIEIAEgAzYCCCAAKAIIIQMgACABKAIMNgIIIAEgAzYCDCABIAEoAgQ2AgAgACgCBBogACgCABogACgCCBogACgCABogBkEQaiQAIAEoAgQhAwNAIAMgASgCCCIERwRAIAEoAhAaIAEgBEEEazYCCAwBCwsgASgCACIDBEAgASgCECADIAEoAgwgA2tBAnUQlQMLCyALQSBqJAAMAQsgASADSQRAIAAoAgQaIAAoAgAhAyAAIAFBAnQgA2oQlgMgACgCCBogACgCBBogACgCABoLCwsgACgCACACQQJ0aigCACIBBEAgAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALCyAJKAIMIQEgCUEANgIMIAAoAgAgAkECdGogATYCACAJKAIMIQAgCUEANgIMIAAEQCAAQQRqQX/+HgIARQRAIAAgACgCACgCCBEBAAsLIAlBEGokAAv0AgIDfwh+IwBBMGsiAyQAAn8CQAJAIAEpAxAiBiACKQMQIgeBQgBSDQAgASkDGCIIIAIpAxgiCYFCAFINACABKQMgIgogAikDICILgUIAUg0AIAEpAygiDCACKQMoIg2BQgBSDQAgAUEQaiEEAkAgASgCiAENACACKAKIAQ0ADAILAkAgBiAHUg0AIAggCVINACAKIAtSDQBBASEFIAwgDVENAgtBvMMCKAIAEDAaIANBq9cANgIYIANB8Rg2AhQgA0HkJjYCEEG4wwIoAgBBy+QAIANBEGoQMRAAAAtBvMMCKAIAEDAaIANBpdgANgIoIANB6xg2AiQgA0HkJjYCIEG4wwIoAgBBy+QAIANBIGoQMRAAAAsgACABKAIAIAEoAgwgBEEAQQAQSSIEQQI2AkBBACAFRQ0AGiAAIAQoAgAgBCgCDCAEQRBqQQBBABBJCyEAIAQgATYCjAEgBCAANgKIASAEIAI2ApABIANBMGokACAECyUAIAAtAAtBB3YEQCAAIAAoAgAgACgCCEH/////B3EQ4AELIAALNAEBfyMAQRBrIgMkACADIAE2AgwgACADKAIMNgIAIABBBGogAigCADYCACADQRBqJAAgAAumAgEHfyAAKAIIIQUCQAJAIAAoAgAiA0EPcUUEQCAAQQRqIgFBAP5BAgAhAAwBC0E/IQIjAyIEKAIYIAAoAgQiBkH/////A3FHDQECQCADQQNxQQFHDQAgACgCFCIBRQ0AIAAgAUEBazYCFEEADwsgA0GAAXEiAgRAIAQgAEEQajYCVEEAQQH+HgLQ+zQaCyAAQQRqIQEgACgCDCIHIAAoAhAiADYCACAEQcwAaiAARwRAIABBBGsgBzYCAAsgASAGQQF0IANBHXRxQR91Qf////8Hcf5BAgAhACACRQ0AIARBADYCVAJAQQBBf/4eAtD7NEEBRw0AQdT7NCgCAEUNAEHQ+zRB/////wcQtQELC0EAIQIgBUUgAEEATnENACABEI4BCyACC34BBX8jAEEgayICJAAgAkEIaiAAEIMDIgEoAgBBAf4ZAAAjAEEQayIAJAAgAEEMakHjIRCCAyEFIAEoAgQiAS0AACEEIAFBAToAACAFEIEDAkAgBEEEcUUNAEH8mDUQpgJFDQAgAEHjITYCABBGAAsgAEEQaiQAIAJBIGokAAvfAQEGfyMAQSBrIgIkACACQQhqIAAQgwMiACgCAP4SAAAEf0EABQJ/IwBBEGsiAyQAIANBDGpBgiIQggMhBQJAAkAgAC0ADEUNACAAKAIELQAAQQJxRQ0AIAAoAggoAgAgACgCEEYNAQsDQCAAKAIEIgQtAAAiAUECcQRAIAQgAUEEcjoAAEH8mDVB5Jg1EI4DDAELCyABQQFGIgFFBEAgAC0ADARAIAAoAgggACgCEDYCAAsgBEECOgAACyAFEIEDIANBEGokACABDAELEEYAC0EBcwshBiACQSBqJAAgBgs0AQJ/IwBBEGsiBCQAIAQgAzcDCCAEIAI3AwAgACABQQIgBEEAQQAQSSEFIARBEGokACAFC7kCAQd/AkAgAC0AAEEPcQ0AIABBBGpBAEEK/kgCAA0AQQAPCwJ/AkAgACgCACICQQ9xRQRAIABBBGpBAEEK/kgCAEUNASAAKAIAIQILIAAQ9QEiAUEKRw0AIABBCGohBCAAQQRqIQNB5AAhAQNAAkAgAUUNACADKAIARQ0AIAFBAWshASAEKAIARQ0BCwsgABD1ASIBQQpHDQAgAkF/c0GAAXEhBSACQQRxRSEGIAJBA3FBAkchAgNAAkAgAygCACIBQf////8DcSIHIAFBAEcgBnFyRQ0AAkAgAg0AIAcjAygCGEcNAEEQDAQLIARBAf4eAgAaIAMgASABQYCAgIB4ciIB/kgCABogAyABIAUQywIhASAEQQH+JQIAGiABQRtGDQAgAQ0CCyAAEPUBIgFBCkYNAAsLIAELC4wCAgJ/AnwgALwiAUGAgID8A0YEQEMAAAAADwsCQCABQYCAgPwHa0H///+HeE0EQCABQQF0IgJFBEAjAEEQayIBQwAAgL84AgwgASoCDEMAAAAAlQ8LIAFBgICA/AdGDQEgAkGAgIB4SSABQQBOcUUEQCAAIACTIgAgAJUPCyAAQwAAAEuUvEGAgIDcAGshAQtBsKACKwMAIAEgAUGAgMz5A2siAUGAgIB8cWu+uyABQQ92QfABcSICQaieAmorAwCiRAAAAAAAAPC/oCIDIAOiIgSiQbigAisDACADokHAoAIrAwCgoCAEoiABQRd1t0GooAIrAwCiIAJBsJ4CaisDAKAgA6CgtiEACyAAC+4CAgZ/A34jAEFAaiIDJAACQAJAIAEpAxAgAikDEFINACACKQMgIgkgASkDIIFCAFINACACKQMoIgogASkDKIFCAFINACABKAIwIAEoAjRLDQECfwJAIAEoAogBDQAgAigCiAENAEEADAELQQELIQggAyABKQMYNwMgIAIpAxghCyADIAo3AzggAyAJNwMwIAMgCzcDKCAAQQAgASgCDCIEIAIoAgwiByAEIAdKGyADQSBqQQBBABBJIgRBFzYCQCAIBEAgACAEKAIAIAQoAgwgBEEQakEAQQAQSSEFCyAEIAE2AowBIAQgBTYCiAEgBCACNgKQASADQUBrJAAgBA8LQbzDAigCABAwGiADQZTXADYCGCADQecfNgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEQAAALQbzDAigCABAwGiADQdvXADYCCCADQegfNgIEIANB5CY2AgBBuMMCKAIAQcvkACADEDEQAAALoQICA38BfgJAIAAoAgwiAkEBcQRAQRgQMiECIAAoAiQoAgQhAyACQfysATYCACACIAM2AgQgAkEIaiAAKAIAIgM2AgAgA0EEakEB/h4CABogAiAAKQIEIgU3AgwgAiAFpyIDIAEgAygCACgCFBEEADoAFCAAKAIkIAI2AgQMAQsgAkEIcQRAQRgQMiECIAAoAiQoAgQhAyACQditATYCACACIAM2AgQgAkEIaiAAKAIAIgM2AgAgA0EEakEB/h4CABogACkCBCEFIAIgAToAFCACIAU3AgwgACgCJCACNgIEDAELQQwQMiECIAAoAiQiAygCBCEEIAIgAToACCACQbSuATYCACACIAQ2AgQgAyACNgIECyAAIAAoAiQoAgQ2AiQLdQEBfyMAQRBrIgIkACACIAE6AA8CQCAALQALQQd2BEAgACgCACEBIABBATYCBAwBCyAAIgEgAS0AC0GAAXFBAXI6AAsgASABLQALQf8AcToACwsgASACLQAPOgAAIAJBADoADiABIAItAA46AAEgAkEQaiQAC6sBAgJ+AX8gACkDICEBIAApAxghAiAAKAIAIgNBEGtBcU0EQCADQSRsQZibAWooAgAgACgCMCAAKAIQQQFrbGogACgCNCACp0EBa2xqIAAoAjggAadBAWtsaiAAKAI8IAAoAihBAWtsag8LIAA1AjQgAkIBfX4gACkDECAANQIwfiADQSRsQZSbAWo1AgB/fCAANQI4IAFCAX1+fCAANQI8IAApAyhCAX1+fKcLDQAgACgCABD7AxogAAsNACAAKAIAEP0DGiAACw4AIABB0ABqEDtB0ABqC8EBAQN/IwBBEGsiBSQAAkAgAiAALQALQQd2BH8gACgCCEH/////B3FBAWsFQQoLIgQCfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQsiA2tNBEAgAkUNAQJ/IAAtAAtBB3YEQCAAKAIADAELIAALIgQgA2ogASACEHcgACACIANqIgEQmQEgBUEAOgAPIAEgBGogBS0ADzoAAAwBCyAAIAQgAiAEayADaiADIANBACACIAEQzAELIAVBEGokACAAC3UBAX4gACABIAR+IAIgA358IANCIIgiAiABQiCIIgR+fCADQv////8PgyIDIAFC/////w+DIgF+IgVCIIggAyAEfnwiA0IgiHwgASACfiADQv////8Pg3wiAUIgiHw3AwggACAFQv////8PgyABQiCGhDcDAAsYACAALQAAQSBxRQRAIAEgAiAAEM4CGgsLqwEBBn8jAEEQayICJAAgACgCBCEDIAEgACgCACIEcCIFIQACQAJAA0ACQCADIABBAnRqIgYoAgAiB0UNACABIAdGDQAgAEEBaiIAQQAgACAERxsiACAFRw0BDAILCyAAQX9HDQELQbzDAigCABAwGiACQbUwNgIIIAJBlvEANgIEIAJB5CY2AgBBuMMCKAIAQcvkACACEDEQAAALIAYgATYCACACQRBqJAAgAAtGAQJ/IwBBEGsiBSQAIAUgAjYCDCAFIAQ2AgggBUEEaiAFQQxqEH8hBiAAIAEgAyAFKAIIELABIQAgBhB+IAVBEGokACAAC+0BAQJ/An8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELIQQCQCACIAFrQQVIDQAgBEUNACABIAIQ6gEgAkEEayEEAn8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELAn8gAC0AC0EHdgRAIAAoAgAMAQsgAAsiAmohBQJAA0ACQCACLAAAIQAgASAETw0AAkAgAEEATA0AIABB/wBODQAgACABKAIARw0DCyABQQRqIQEgAiAFIAJrQQFKaiECDAELCyAAQQBMDQEgAEH/AE4NASACLAAAIAQoAgBBAWtLDQELIANBBDYCAAsLdgEBfyMAQRBrIgIkACAALQALQQd2BEAgACAAKAIAIAAoAghB/////wdxEPEBCyAAIAEoAgg2AgggACABKQIANwIAIAEgAS0AC0GAAXE6AAsgASABLQALQf8AcToACyACQQA6AA8gASACLQAPOgAAIAJBEGokAAtQAQF+AkAgA0HAAHEEQCABIANBQGqthiECQgAhAQwBCyADRQ0AIAIgA60iBIYgAUHAACADa62IhCECIAEgBIYhAQsgACABNwMAIAAgAjcDCAtvAQF/IwBBgAJrIgUkAAJAIAIgA0wNACAEQYDABHENACAFIAFB/wFxIAIgA2siA0GAAiADQYACSSIBGxCgASABRQRAA0AgACAFQYACEGEgA0GAAmsiA0H/AUsNAAsLIAAgBSADEGELIAVBgAJqJAALegEDfwJAAkAgACIBQQNxRQ0AIAEtAABFBEBBAA8LA0AgAUEBaiIBQQNxRQ0BIAEtAAANAAsMAQsDQCABIgJBBGohASACKAIAIgNBf3MgA0GBgoQIa3FBgIGChHhxRQ0ACwNAIAIiAUEBaiECIAEtAAANAAsLIAEgAGsLOAIDfwF+IwBBEGsiACQAIAAQ1wEgACkDACEDIAAoAgghAiAAQRBqJAAgAkHoB22sIANCwIQ9fnwL9gIDA38BfAF9IwBBEGsiASQAAn0gALwiA0H/////B3EiAkHan6T6A00EQEMAAIA/IAJBgICAzANJDQEaIAC7EIQBDAELIAJB0aftgwRNBEAgAkHkl9uABE8EQEQYLURU+yEJQEQYLURU+yEJwCADQQBIGyAAu6AQhAGMDAILIAC7IQQgA0EASARAIAREGC1EVPsh+T+gEIMBDAILRBgtRFT7Ifk/IAShEIMBDAELIAJB1eOIhwRNBEAgAkHg27+FBE8EQEQYLURU+yEZQEQYLURU+yEZwCADQQBIGyAAu6AQhAEMAgsgA0EASARARNIhM3982RLAIAC7oRCDAQwCCyAAu0TSITN/fNkSwKAQgwEMAQsgACAAkyACQYCAgPwHTw0AGgJAAkACQAJAIAAgAUEIahCvBEEDcQ4DAAECAwsgASsDCBCEAQwDCyABKwMImhCDAQwCCyABKwMIEIQBjAwBCyABKwMIEIMBCyEFIAFBEGokACAFC9YBAQJ/IwBBEGsiBCQAAkACQCACQQtJBEAgACIDIAAtAAtBgAFxIAJyOgALIAAgAC0AC0H/AHE6AAsMAQsgAkHv////B0sNASAEQQhqIAAgAkELTwR/IAJBEGpBcHEiAyADQQFrIgMgA0ELRhsFQQoLQQFqEMMBIAQoAgwaIAAgBCgCCCIDNgIAIAAgACgCCEGAgICAeHEgBCgCDEH/////B3FyNgIIIAAgACgCCEGAgICAeHI2AgggACACNgIECyADIAEgAkEBahB3IARBEGokAA8LEE0ACwoAIABBlIo1EHkLBABBAAuJAgEHfyABIAAoAggiAyAAKAIEIgJrQQJ1TQRAIAAgAQR/IAJBACABQQJ0IgD8CwAgACACagUgAgs2AgQPCwJAIAIgACgCACIFa0ECdSIHIAFqIgRBgICAgARJBEBB/////wMgAyAFayIDQQF1IgggBCAEIAhJGyADQfz///8HTxsiAwRAIANBgICAgARPDQIgA0ECdBAyIQYLIAdBAnQgBmoiBEEAIAFBAnQiAfwLACABIARqIQEgAiAFRwRAA0AgBEEEayIEIAJBBGsiAioCADgCACACIAVHDQALCyAAIAYgA0ECdGo2AgggACABNgIEIAAgBDYCACAFBEAgBRAvCw8LEDYACxBHAAstACACRQRAIAAoAgQgASgCBEYPCyAAIAFGBEBBAQ8LIAAoAgQgASgCBBCfAUULUgEBfyABQQFrIgJBEE0EfyACQQJ0QdypA2ooAgAFQcYiCyECIABB6LUDNgIAIABB7LYDNgIAIABBBGogAhCKAyAAIAE2AgggAEGooQM2AgAgAAvHCQIEfwZ+IwBB8ABrIgYkACAEQv///////////wCDIQkCQAJAIAFQIgUgAkL///////////8AgyIKQoCAgICAgMD//wB9QoCAgICAgMCAgH9UIApQG0UEQCADQgBSIAlCgICAgICAwP//AH0iC0KAgICAgIDAgIB/ViALQoCAgICAgMCAgH9RGw0BCyAFIApCgICAgICAwP//AFQgCkKAgICAgIDA//8AURtFBEAgAkKAgICAgIAghCEEIAEhAwwCCyADUCAJQoCAgICAgMD//wBUIAlCgICAgICAwP//AFEbRQRAIARCgICAgICAIIQhBAwCCyABIApCgICAgICAwP//AIWEUARAQoCAgICAgOD//wAgAiABIAOFIAIgBIVCgICAgICAgICAf4WEUCIFGyEEQgAgASAFGyEDDAILIAMgCUKAgICAgIDA//8AhYRQDQEgASAKhFAEQCADIAmEQgBSDQIgASADgyEDIAIgBIMhBAwCCyADIAmEQgBSDQAgASEDIAIhBAwBCyADIAEgASADVCAJIApWIAkgClEbIggbIQogBCACIAgbIgtC////////P4MhCSACIAQgCBsiAkIwiKdB//8BcSEHIAtCMIinQf//AXEiBUUEQCAGQeAAaiAKIAkgCiAJIAlQIgUbeSAFQQZ0rXynIgVBD2sQZiAGKQNoIQkgBikDYCEKQRAgBWshBQsgASADIAgbIQMgAkL///////8/gyEEIAdFBEAgBkHQAGogAyAEIAMgBCAEUCIHG3kgB0EGdK18pyIHQQ9rEGZBECAHayEHIAYpA1ghBCAGKQNQIQMLIARCA4YgA0I9iIRCgICAgICAgASEIQEgCUIDhiAKQj2IhCEOIAIgC4UhDQJ+IANCA4YiAiAFIAdGDQAaIAUgB2siB0H/AEsEQEIAIQFCAQwBCyAGQUBrIAIgAUGAASAHaxBmIAZBMGogAiABIAcQrwEgBikDOCEBIAYpAzAgBikDQCAGKQNIhEIAUq2ECyEJIA5CgICAgICAgASEIQwgCkIDhiEKAkAgDUIAUwRAQgAhA0IAIQQgCSAKhSABIAyFhFANAiAKIAl9IQIgDCABfSAJIApWrX0iBEL/////////A1YNASAGQSBqIAIgBCACIAQgBFAiBxt5IAdBBnStfKdBDGsiBxBmIAUgB2shBSAGKQMoIQQgBikDICECDAELIAkgCnwiAiAJVK0gASAMfHwiBEKAgICAgICACINQDQAgCUIBgyAEQj+GIAJCAYiEhCECIAVBAWohBSAEQgGIIQQLIAtCgICAgICAgICAf4MhASAFQf//AU4EQCABQoCAgICAgMD//wCEIQRCACEDDAELQQAhBwJAIAVBAEoEQCAFIQcMAQsgBkEQaiACIAQgBUH/AGoQZiAGIAIgBEEBIAVrEK8BIAYpAwAgBikDECAGKQMYhEIAUq2EIQIgBikDCCEECyACp0EHcSIFQQRLrSAEQj2GIAJCA4iEIgJ8IgMgAlStIARCA4hC////////P4MgB61CMIaEIAGEfCEEAkAgBUEERgRAIAQgA0IBgyIBIAN8IgMgAVStfCEEDAELIAVFDQELCyAAIAM3AwAgACAENwMIIAZB8ABqJAALCgAgAEGcijUQeQsuAQF/IwBBEGsiAyQAIAMgAjYCDCAAQdQBakHAACABIAIQsAEaIANBEGokACAAC2QAIAIoAgRBsAFxIgJBIEYEQCABDwsCQCACQRBHDQACQAJAIAAtAAAiAkEraw4DAAEAAQsgAEEBag8LIAEgAGtBAkgNACACQTBHDQAgAC0AAUEgckH4AEcNACAAQQJqIQALIAALPQECfwJ/IAAtAAtBB3YEQCAAKAIADAELIAALIQEjAEEQayIAJAAgACABNgIMIAAoAgwhAiAAQRBqJAAgAgt+AgJ/AX4jAEEQayIDJAAgAAJ+IAFFBEBCAAwBCyADIAEgAUEfdSICcyACayICrUIAIAJnIgJB0QBqEGYgAykDCEKAgICAgIDAAIVBnoABIAJrrUIwhnwgAUGAgICAeHGtQiCGhCEEIAMpAwALNwMAIAAgBDcDCCADQRBqJAALDAAgASACIAAQgQQaC9wBAQR/IwBBIGsiAyQAIAAoAgAiBEUEQCAAKAIUIQYgAyAAKQIENwMYIAYgA0EYaiABEGJBAnRqKAIAIQQLAkAgASgCyAEiAEUNACAAKALQASIFRQ0AIAIEQCABIAAoAgQ2AgQLIAEgACgCCDYCCCABIAUgASgCzAFqNgLQASAELQCYEEUEQCAEKAIAIgAoAggiAgRAIAAgASACEQIACwsgA0EgaiQADwtBvMMCKAIAEDAaIANB/i82AgggA0HAAzYCBCADQdonNgIAQbjDAigCAEHL5AAgAxAxEAAACycAIAAoAgAiACABEEwiARCuA0UEQBBGAAsgACgCCCABQQJ0aigCAAuKAwIDfwF8IwBBEGsiASQAAkAgALwiA0H/////B3EiAkHan6T6A00EQCACQYCAgMwDSQ0BIAC7EIMBIQAMAQsgAkHRp+2DBE0EQCAAuyEEIAJB45fbgARNBEAgA0EASARAIAREGC1EVPsh+T+gEIQBjCEADAMLIAREGC1EVPsh+b+gEIQBIQAMAgtEGC1EVPshCcBEGC1EVPshCUAgA0EAThsgBKCaEIMBIQAMAQsgAkHV44iHBE0EQCACQd/bv4UETQRAIAC7IQQgA0EASARAIARE0iEzf3zZEkCgEIQBIQAMAwsgBETSITN/fNkSwKAQhAGMIQAMAgtEGC1EVPshGUBEGC1EVPshGcAgA0EASBsgALugEIMBIQAMAQsgAkGAgID8B08EQCAAIACTIQAMAQsCQAJAAkACQCAAIAFBCGoQrwRBA3EOAwABAgMLIAErAwgQgwEhAAwDCyABKwMIEIQBIQAMAgsgASsDCJoQgwEhAAwBCyABKwMIEIQBjCEACyABQRBqJAAgAAuPBAEDfyACQYAETwRAIAAgASAC/AoAACAADwsgACACaiEEAkAgACABc0EDcUUEQAJ/IAAgAEEDcUUNABogACACRQ0AGiAAIAEgAEF/cyICIAQgAEEBaiIDIAMgBEkbaiIDIAJBA3EiAiACIANLG0EBaiIC/AoAACABIAJqIQEgACACagshAgJAIARBfHEiA0HAAEkNACACIANBQGoiBUsNAANAIAIgASgCADYCACACIAEoAgQ2AgQgAiABKAIINgIIIAIgASgCDDYCDCACIAEoAhA2AhAgAiABKAIUNgIUIAIgASgCGDYCGCACIAEoAhw2AhwgAiABKAIgNgIgIAIgASgCJDYCJCACIAEoAig2AiggAiABKAIsNgIsIAIgASgCMDYCMCACIAEoAjQ2AjQgAiABKAI4NgI4IAIgASgCPDYCPCABQUBrIQEgAkFAayICIAVNDQALCyACIANPDQEgAiABIAJBf3MgAyACQQRqIgUgAyAFSxtqQXxxQQRqIgP8CgAAIAEgA2ohASACIANqIQIMAQsgBEEESQRAIAAhAgwBCyAAIARBBGsiA0sEQCAAIQIMAQsgACECA0AgAiABLQAAOgAAIAIgAS0AAToAASACIAEtAAI6AAIgAiABLQADOgADIAFBBGohASACQQRqIgIgA00NAAsLIAIgBEkEQCACIAEgBCACa/wKAAALIAALIAAjAEEQayIBJAAgAEIANwIAIABBADYCCCABQRBqJAALugcBCH8jAEHQAGsiBCQAIAQgACkCGDcDSAJAAkACQAJ/IwBBIGsiAiQAIAQoAkwhBiABIAQoAkgiB3AiCCEDAkADQAJAIAYgA0ECdGoiCSgCACIFRQ0AIAEgBUYNACADQQFqIgNBACADIAdHGyIDIAhHDQEMAgsLIANBf0YNAAJAAkAgASAFRgRAQX4hAwwBCyAFDQEgCSABNgIACyACQSBqJAAgAwwCC0G8wwIoAgAQMBogAkG+LzYCGCACQY7xADYCFCACQeQmNgIQQbjDAigCAEHL5AAgAkEQahAxEAAAC0G8wwIoAgAQMBogAkG1MDYCCCACQYfxADYCBCACQeQmNgIAQbjDAigCAEHL5AAgAhAxEAAAC0F+Rg0AIAFBJEEAIAAoAiAiAkEBRhtqKAKMASIDBEAgACADEH0gACgCICECCyABQSBBBCACQQFGG2ooAowBIgMEQCAAIAMQfSAAKAIgIQILIAFBHEEIIAJBAUYbaigCjAEiAwRAIAAgAxB9IAAoAiAhAgsgAUEYQQwgAkEBRhtqKAKMASIDBEAgACADEH0gACgCICECCyABQRRBECACQQFGG2ooAowBIgMEQCAAIAMQfSAAKAIgIQILIAFBEEEUIAJBAUYbaigCjAEiAwRAIAAgAxB9IAAoAiAhAgsgAUEMQRggAkEBRhtqKAKMASIDBEAgACADEH0gACgCICECCyABQQhBHCACQQFGG2ooAowBIgMEQCAAIAMQfSAAKAIgIQILIAFBJEEAIAFBBEEgIAJBAUYbaigCjAEiAwR/IAAgAxB9IAAoAiAFIAILQQFHG2ooAowBIgIEQCAAIAIQfQsCQCABKAJADQAgASgCiAENACAAKAIIIgIgACgCAE4NAiABLQDUAUUEQCAEIAI2AgAgAUHwJSAEEHMaIAAoAgghAgsgACgCFCACQQJ0aiABNgIAIAAgAkEBajYCCAwBCyAAKAIEIgIgACgCAE4NAiABLQDUAUUEQCAEIAI2AiAgAUH4JSAEQSBqEHMaIAAoAgQhAgsgAkECdCIDIAAoAgxqIAE2AgAgACgCECIFBEAgAyAFaiABKAKIATYCAAsgACACQQFqNgIECyAEQdAAaiQADwtBvMMCKAIAEDAaIARBwB42AhggBEGh+QA2AhQgBEHkJjYCEEG4wwIoAgBBy+QAIARBEGoQMRAAAAtBvMMCKAIAEDAaIARB3x42AjggBEGq+QA2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMRAAAAsSACAAKAIAIgAEQCAAEOEDGgsLEQAgACABKAIAEOEDNgIAIAALRwECfyAAIAE3A3AgACAAKAIsIAAoAgQiA2usNwN4IAAoAgghAgJAIAFQDQAgAiADa6wgAVcNACADIAGnaiECCyAAIAI2AmgLHwAgAEHMAGoiAEEA/kECAEGAgICABHEEQCAAEI4BCwuQAQEEfwJAIwMoAhgiASAAKAJMQf////97cUYNAEEBIQMgAEHMAGoiAkEAIAH+SAIARQ0AIAJBACABQYCAgIAEciIE/kgCACIARQ0AA0AgAEGAgICABHIhAQJAIABBgICAgARxRQRAIAAgAiAAIAH+SAIARw0BCyACIAEQrQQLIAJBACAE/kgCACIADQALCyADC0sBAnwgACAAoiIBIACiIgIgASABoqIgAUSnRjuMh83GPqJEdOfK4vkAKr+goiACIAFEsvtuiRARgT+iRHesy1RVVcW/oKIgAKCgtgtPAQF8IAAgAKIiACAAIACiIgGiIABEaVDu4EKT+T6iRCceD+iHwFa/oKIgAURCOgXhU1WlP6IgAESBXgz9///fv6JEAAAAAAAA8D+goKC2CwQAIAALkgQBB38jAEEgayIDJAAgACgCACIGRQRAIAAoAhQhCCADIAApAgQ3AxggCCADQRhqIAEQYkECdGooAgAhBgsCQCABKALQAQ0AIAEoAsgBBEAgACABQQEQeAwBCwJAAkAgASgCQEECaw48AAABAAAAAAAAAQEBAQEBAQEBAAEBAQEBAAEBAQEBAQEBAQEAAAABAAEBAQEBAQEBAQEBAQEBAQEBAQEAAQsDQCABIAdBAnRqKAKMASICRQ0BAkAgAigCCCAGKAIARw0AIAMgACkCBDcDECADQRBqIAIQYiEFIAIoAtABRQ0AIAAoAgwgBUEDdGoiBCgCAEEBRw0AIAQoAgQNACABKAIAIAIoAgBHDQAgASkDECACKQMQUg0AIAEoAjAgAigCMEcNACABKQMYIAIpAxhSDQAgASgCNCACKAI0Rw0AIAEpAyAgAikDIFINACABKAI4IAIoAjhHDQAgASkDKCACKQMoUg0AIAEoAjwgAigCPEcNACACKALIASIFBEAgAyAAKQIENwMIIANBCGogBRBiIQQgACgCDCAEQQN0aiIEKAIEQQFHDQEgBCgCAA0BIAUoAtABIAIoAtABRw0BIAEgBTYCyAEgBEECNgIEIAAgAUEAEHgMBAsgASACNgLIASAEQQE2AgQgACABQQAQeAwDCyAHQQFqIgdBCkcNAAsLIAYgARDpBAsgA0EgaiQAC4YDAQd/An8jAEFAaiIEJAAgASkDKCABKQMgIAEpAxggASkDEH5+fiACKQMoIAIpAyAgAikDGCACKQMQfn5+UQRAIAJBEGohAwJ/IAEoAogBRQRAQQAgAigCiAFFDQEaC0EBCyEJIAAgAigCACACKAIMIAMgAkEAEEkhCCAEIAJB1AFqIgc2AiAgCEHJywAgBEEgahBzIgMgAigCMDYCMCADIAIoAjQ2AjQgAyACKAI4NgI4IAMgAigCPDYCPCABQdQBaiEFAkAgAi0A1AEEQCAEIAU2AhQgBCAHNgIQIANB0NMAIARBEGoQcxoMAQsgBCAFNgIAIANBv8sAIAQQcxoLIANBHDYCQEEAIQUgCQRAIAAgAygCACADKAIMIANBEGpBAEEAEEkhBQsgAyABNgKMASADIAU2AogBIAMgAjYCkAEgBEFAayQAIAMMAQtBvMMCKAIAEDAaIARB29YANgI4IARByyE2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMRAAAAsLdgEBfyMAQRBrIgIkACAALQALQQd2BEAgACAAKAIAIAAoAghB/////wdxEOABCyAAIAEoAgg2AgggACABKQIANwIAIAEgAS0AC0GAAXE6AAsgASABLQALQf8AcToACyACQQA2AgwgASACKAIMNgIAIAJBEGokAAuyAgEEfyMAQRBrIgckACAHIAE2AgxBACEBQQYhBQJAAkAgACAHQQxqEEINAEEEIQUgA0HAAAJ/IAAoAgAiBigCDCIIIAYoAhBGBEAgBiAGKAIAKAIkEQAADAELIAgoAgALIgYgAygCACgCDBEDAEUNACADIAZBACADKAIAKAI0EQMAIQEDQAJAIAAQXBogAUEwayEBIAAgB0EMahBCDQAgBEECSA0AIANBwAACfyAAKAIAIgUoAgwiBiAFKAIQRgRAIAUgBSgCACgCJBEAAAwBCyAGKAIACyIFIAMoAgAoAgwRAwBFDQMgBEEBayEEIAMgBUEAIAMoAgAoAjQRAwAgAUEKbGohAQwBCwtBAiEFIAAgB0EMahBCRQ0BCyACIAIoAgAgBXI2AgALIAdBEGokACABC9gCAQR/IwBBEGsiByQAIAcgATYCDEEAIQFBBiEFAkACQCAAIAdBDGoQQw0AQQQhBQJ/IAAoAgAiBigCDCIIIAYoAhBGBEAgBiAGKAIAKAIkEQAADAELIAgtAAALwCIGQQBOBH8gAygCCCAGQf8BcUECdGooAgBBwABxQQBHBUEAC0UNACADIAZBACADKAIAKAIkEQMAIQEDQAJAIAAQXRogAUEwayEBIAAgB0EMahBDDQAgBEECSA0AAn8gACgCACIFKAIMIgYgBSgCEEYEQCAFIAUoAgAoAiQRAAAMAQsgBi0AAAvAIgVBAE4EfyADKAIIIAVB/wFxQQJ0aigCAEHAAHFBAEcFQQALRQ0DIARBAWshBCADIAVBACADKAIAKAIkEQMAIAFBCmxqIQEMAQsLQQIhBSAAIAdBDGoQQ0UNAQsgAiACKAIAIAVyNgIACyAHQRBqJAAgAQuZAQEEfyMAQRBrIgQkACAEIAE2AgwgBCADNgIIIARBBGogBEEMahB/IQcgBCgCCCEDIwBBEGsiASQAIAEgAzYCDCABIAM2AghBfyEFAkBBAEEAIAIgAxCwASIDQQBIDQAgACADQQFqIgMQOyIANgIAIABFDQAgACADIAIgASgCDBCwASEFCyABQRBqJAAgBxB+IARBEGokACAFCy4AAkAgACgCBEHKAHEiAARAIABBwABGBEBBCA8LIABBCEcNAUEQDwtBAA8LQQoL+QECA34CfyMAQRBrIgUkAAJ+IAG9IgNC////////////AIMiAkKAgICAgICACH1C/////////+//AFgEQCACQjyGIQQgAkIEiEKAgICAgICAgDx8DAELIAJCgICAgICAgPj/AFoEQCADQjyGIQQgA0IEiEKAgICAgIDA//8AhAwBCyACUARAQgAMAQsgBSACQgAgA6dnQSBqIAJCIIinZyACQoCAgIAQVBsiBkExahBmIAUpAwAhBCAFKQMIQoCAgICAgMAAhUGM+AAgBmutQjCGhAshAiAAIAQ3AwAgACACIANCgICAgICAgICAf4OENwMIIAVBEGokAAsJACAAQQEQtQELKAEBfyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAAQLwsMACAAKAIAIAEQ6QQLbAECfyAAKAIEIgEEQCABKAIIIgIEQCACEC8LIAEoAgwiAgRAIAIQLwsgASgCFCICBEAgAhAvCyABKAIYIgIEQCACEC8LIAEQLwsgACgCACIBBEAgAS0ABARAIAEoAgAQpwELIAEQLwsgABAvC40FAQR/IwBBEGsiCSQAIAkgAjYCCCAJIAE2AgwgCUEEaiIBIAMoAhwiAjYCACACQQRqQQH+HgIAGiABEGwhCCABKAIAIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAEQQA2AgBBACEBAkADQCAGIAdGDQEgAQ0BAkAgCUEMaiAJQQhqEEINAAJAIAggBigCAEEAIAgoAgAoAjQRAwBBJUYEQCAGQQRqIAdGDQJBACECAn8CQCAIIAYoAgRBACAIKAIAKAI0EQMAIgFBxQBGDQBBASEKIAFB/wFxQTBGDQAgAQwBCyAGQQhqIAdGDQNBAiEKIAEhAiAIIAYoAghBACAIKAIAKAI0EQMACyEBIAkgACAJKAIMIAkoAgggAyAEIAUgASACIAAoAgAoAiQRDQA2AgwgBiAKQQJ0akEEaiEGDAELIAhBASAGKAIAIAgoAgAoAgwRAwAEQANAAkAgByAGQQRqIgZGBEAgByEGDAELIAhBASAGKAIAIAgoAgAoAgwRAwANAQsLA0AgCUEMaiICIAlBCGoQQg0CIAhBAQJ/IAIoAgAiASgCDCIKIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAooAgALIAgoAgAoAgwRAwBFDQIgAhBcGgwACwALIAgCfyAJQQxqIgIoAgAiASgCDCIKIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAooAgALIAgoAgAoAhwRBAAgCCAGKAIAIAgoAgAoAhwRBABGBEAgBkEEaiEGIAIQXBoMAQsgBEEENgIACyAEKAIAIQEMAQsLIARBBDYCAAsgCUEMaiAJQQhqEEIEQCAEIAQoAgBBAnI2AgALIAkoAgwhCyAJQRBqJAAgCwu/BQEEfyMAQRBrIggkACAIIAI2AgggCCABNgIMIAhBBGoiASADKAIcIgI2AgAgAkEEakEB/h4CABogARByIQkgASgCACIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgBEEANgIAQQAhAQJAA0AgBiAHRg0BIAENAQJAIAhBDGogCEEIahBDDQACQCAJIAYsAABBACAJKAIAKAIkEQMAQSVGBEAgBkEBaiAHRg0CQQAhAgJ/AkAgCSAGLAABQQAgCSgCACgCJBEDACIBQcUARg0AQQEhCiABQf8BcUEwRg0AIAEMAQsgBkECaiAHRg0DQQIhCiABIQIgCSAGLAACQQAgCSgCACgCJBEDAAshASAIIAAgCCgCDCAIKAIIIAMgBCAFIAEgAiAAKAIAKAIkEQ0ANgIMIAYgCmpBAWohBgwBCyAGLAAAIgFBAE4EfyAJKAIIIAFB/wFxQQJ0aigCAEEBcQVBAAsEQANAAkAgByAGQQFqIgZGBEAgByEGDAELIAYsAAAiAUEATgR/IAkoAgggAUH/AXFBAnRqKAIAQQFxBUEACw0BCwsDQCAIQQxqIgIgCEEIahBDDQICfyACKAIAIgEoAgwiCiABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAKLQAAC8AiAUEATgR/IAkoAgggAUH/AXFBAnRqKAIAQQFxBUEAC0UNAiACEF0aDAALAAsgCQJ/IAhBDGoiAigCACIBKAIMIgogASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgCi0AAAvAIAkoAgAoAgwRBAAgCSAGLAAAIAkoAgAoAgwRBABGBEAgBkEBaiEGIAIQXRoMAQsgBEEENgIACyAEKAIAIQEMAQsLIARBBDYCAAsgCEEMaiAIQQhqEEMEQCAEIAQoAgBBAnI2AgALIAgoAgwhCyAIQRBqJAAgCwvjAQEEfyMAQRBrIggkAAJAIABFDQAgBCgCDCEGIAIgAWsiB0EASgRAIAAgASAHQQJ2IgcgACgCACgCMBEDACAHRw0BCyAGIAMgAWtBAnUiAWtBACABIAZIGyIBQQBKBEAgAAJ/IAhBBGogASAFEM8DIgUtAAtBB3YEQCAFKAIADAELIAULIAEgACgCACgCMBEDACEGIAUQUBogASAGRw0BCyADIAJrIgFBAEoEQCAAIAIgAUECdiIBIAAoAgAoAjARAwAgAUcNAQsgBCgCDBogBEEANgIMIAAhCQsgCEEQaiQAIAkL1gEBBH8jAEEQayIHJAACQCAARQ0AIAQoAgwhBiACIAFrIghBAEoEQCAAIAEgCCAAKAIAKAIwEQMAIAhHDQELIAYgAyABayIBa0EAIAEgBkgbIgFBAEoEQCAAAn8gB0EEaiABIAUQ0QMiBS0AC0EHdgRAIAUoAgAMAQsgBQsgASAAKAIAKAIwEQMAIQYgBRAzGiABIAZHDQELIAMgAmsiAUEASgRAIAAgAiABIAAoAgAoAjARAwAgAUcNAQsgBCgCDBogBEEANgIMIAAhCQsgB0EQaiQAIAkLHwAgAQRAIAAgASgCABCWASAAIAEoAgQQlgEgARAvCwv0AQECfyMAQTBrIgQkAAJAIAAoAtABRQRAQbzDAigCABAwGiAEQZzfADYCCCAEQZEBNgIEIARBmyc2AgBBuMMCKAIAQcvkACAEEDEMAQsgACgCCEUEQEG8wwIoAgAQMBogBEHU3AA2AhggBEGSATYCFCAEQZsnNgIQQbjDAigCAEHL5AAgBEEQahAxDAELIAAQWyACIANqSQRAQbzDAigCABAwGiAEQcXdADYCKCAEQZMBNgIkIARBmyc2AiBBuMMCKAIAQcvkACAEQSBqEDEMAQsgACgCCCIFIAAgASACIAMgBSgCDBELACAEQTBqJAAPCxAAAAveBgEKfyMAQeABayIEJAACQAJAAkACQAJAAkACQCACQQRJBEAgA0EETw0BIAJBAkYNAiACIANGDQMgAkEDRg0EIANBAkYNBSADQQNGDQYgASgCiAEhDSAAIAEoAgAgASgCDCABQRBqIAFBABBJIQwgBCABQdQBaiIGNgJwIAxBycsAIARB8ABqEHMiByABKAIwNgIwIAcgASgCNDYCNCAHIAEoAjg2AjggByABKAI8NgI8IAQgBjYCYCAHQfTUACAEQeAAahBzIQUgAkECdCIKIARB0AFqIgZqIAEpAxA+AgAgBkEIaiABKQMYPgIAIAYgA0ECdCILaiABKQMgPgIAIAZBDGogASkDKD4CACAKIARBwAFqIgZqIAEoAjA2AgAgBiABKAI0NgIIIAYgC2ogASgCODYCACAGIAEoAjw2AgwgBSAENALQATcDECAFIAQ0AtQBNwMYIAUgBDQC2AE3AyAgBSAENALcATcDKCAHIAT9AATAAf0LAzAgBUEgNgJAIA0EQCAAIAUoAgAgBSgCDCAFQRBqQQBBABBJIQgLIAUgATYCjAEgBSAINgKIASAFIAI2AkQgBUEDNgJQIAUgAzYCTCAFQQI2AkggBEHgAWokACAFDwtBvMMCKAIAEDAaIARB8y02ArgBIARBuiQ2ArQBIARB5CY2ArABQbjDAigCAEHL5AAgBEGwAWoQMQwGC0G8wwIoAgAQMBogBEGrLTYCmAEgBEG8JDYClAEgBEHkJjYCkAFBuMMCKAIAQcvkACAEQZABahAxDAULQbzDAigCABAwGiAEQZY9NgIIIARBvyQ2AgQgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMQwEC0G8wwIoAgAQMBogBEH8OjYCGCAEQcAkNgIUIARB5CY2AhBBuMMCKAIAQcvkACAEQRBqEDEMAwtBvMMCKAIAEDAaIARBljo2AiggBEHBJDYCJCAEQeQmNgIgQbjDAigCAEHL5AAgBEEgahAxDAILQbzDAigCABAwGiAEQe06NgI4IARBwiQ2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMQwBC0G8wwIoAgAQMBogBEH4OTYCWCAEQcQkNgJUIARB5CY2AlBBuMMCKAIAQcvkACAEQdAAahAxCxAAAAs0ACAALQALQQd2BEAgACABNgIEDwsgACAALQALQYABcSABcjoACyAAIAAtAAtB/wBxOgALCwwAIABBgoaAIDYAAAtfAQJ/An8gAC0AC0EHdgRAIAAoAgAMAQsgAAsCfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQtBAnRqIQEjAEEQayIAJAAgACABNgIMIAAoAgwhAiAAQRBqJAAgAgusAQEBfwJAIANBgBBxRQ0AIANBygBxIgRBCEYNACAEQcAARg0AIAJFDQAgAEErOgAAIABBAWohAAsgA0GABHEEQCAAQSM6AAAgAEEBaiEACwNAIAEtAAAiBARAIAAgBDoAACAAQQFqIQAgAUEBaiEBDAELCyAAAn9B7wAgA0HKAHEiAUHAAEYNABpB2ABB+AAgA0GAgAFxGyABQQhGDQAaQeQAQfUAIAIbCzoAAAtcAQJ/An8gAC0AC0EHdgRAIAAoAgAMAQsgAAsCfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQtqIQEjAEEQayIAJAAgACABNgIMIAAoAgwhAiAAQRBqJAAgAgsSACABIAEgAkECdGogABD1AxoLTQECfyABLQAAIQICQCAALQAAIgNFDQAgAiADRw0AA0AgAS0AASECIAAtAAEiA0UNASABQQFqIQEgAEEBaiEAIAIgA0YNAAsLIAMgAmsLhAMCAn8BfiACQYAETwRAIAAgAcAgAvwLAA8LAkAgAkUNACAAIAE6AAAgACACaiIDQQFrIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0EDayABOgAAIANBAmsgAToAACACQQdJDQAgACABOgADIANBBGsgAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiADYCACADIAIgBGtBfHEiAmoiAUEEayAANgIAIAJBCUkNACADIAA2AgggAyAANgIEIAFBCGsgADYCACABQQxrIAA2AgAgAkEZSQ0AIAMgADYCGCADIAA2AhQgAyAANgIQIAMgADYCDCABQRBrIAA2AgAgAUEUayAANgIAIAFBGGsgADYCACABQRxrIAA2AgAgAiADQQRxQRhyIgJrIgFBIEkNACAArUKBgICAEH4hBSACIANqIQADQCAAIAU3AxggACAFNwMQIAAgBTcDCCAAIAU3AwAgAEEgaiEAIAFBIGsiAUEfSw0ACwsLzAECBH8BfEHkACEDAkACfwJAA0AgAwRAIAEEQCABKAIADQMLIANBAWshAyAAKAIAIAJGDQEMBAsLIAENAEEBDAELIAFBAf4eAgAaQQALIQYjBSEDAkAgACgCACACRw0AQQFB5AAgAxu3IQcjAyEEA0ACQAJAIANFBEAgBC0AKUEBRw0BCwNAIAQoAiQNBCAAIAIgBxDVAUG3f0YNAAsMAQsgACACRAAAAAAAAPB/ENUBGgsgACgCACACRg0ACwsgBg0AIAFBAf4lAgAaCwsCAAuwCgENfwJAAkACQAJAIAAtAFkEQCAAKAIMIgMgASADKAIAKAIUEQQAIQEgACgCGCICIAAoAhwiBUkNBCACIAAoAhQiBmsiB0EBaiIDQQBIDQIgB0H/////ByAFIAZrIgVBAXQiBCADIAMgBEkbIAVB/////wNPGyIEBH8gBBAyBUEACyIFaiIDIAE6AAAgBCAFaiEIIANBAWohCQJAIAIgBkYEQCADIQUMAQsCQAJAIAdBEEkNACAGIAIgBWprIAJqQRBJDQAgA0EQayEKIAJBEGshCyACIAdBcHEiBGshAiADIARrIQNBACEBA0AgCiABayALIAFr/QAAAP0LAAAgAUEQaiIBIARHDQALIAQgB0YNAQsgBkF/cyACaiEMIAIgBmtBA3EiBARAQQAhAQNAIANBAWsiAyACQQFrIgItAAA6AAAgAUEBaiIBIARHDQALCyAMQQNJDQADQCADQQFrIAJBAWstAAA6AAAgA0ECayACQQJrLQAAOgAAIANBA2sgAkEDay0AADoAACADQQRrIgMgAkEEayICLQAAOgAAIAIgBkcNAAsLIAAoAhQhAgsgACAINgIcIAAgCTYCGCAAIAU2AhQgAkUNASACEC8PCyAAKAIcIQMgACgCGCECAkAgAC0AWgRAIAIgA0kNBSACIAAoAhQiBmsiB0EBaiIFQQBIDQQgB0H/////ByADIAZrIgNBAXQiBCAFIAQgBUsbIANB/////wNPGyIEBH8gBBAyBUEACyIFaiIDIAE6AAAgBCAFaiEIIANBAWohCQJAIAIgBkYEQCADIQUMAQsCQAJAIAdBEEkNACAGIAIgBWprIAJqQRBJDQAgA0EQayEKIAJBEGshCyACIAdBcHEiBGshAiADIARrIQNBACEBA0AgCiABayALIAFr/QAAAP0LAAAgAUEQaiIBIARHDQALIAQgB0YNAQsgBkF/cyACaiENIAIgBmtBA3EiBARAQQAhAQNAIANBAWsiAyACQQFrIgItAAA6AAAgAUEBaiIBIARHDQALCyANQQNJDQADQCADQQFrIAJBAWstAAA6AAAgA0ECayACQQJrLQAAOgAAIANBA2sgAkEDay0AADoAACADQQRrIgMgAkEEayICLQAAOgAAIAIgBkcNAAsLIAAoAhQhAgsgACAINgIcIAAgCTYCGCAAIAU2AhQgAg0BDAILIAIgA0cNBCACIAAoAhQiB2siBkEBaiIDQQBIDQMgBkH/////ByAGQQF0IgUgAyADIAVJGyAGQf////8DTxsiBAR/IAQQMgVBAAsiBWoiAyABOgAAIAQgBWohCCADQQFqIQkCQCACIAdGBEAgAyEFDAELAkACQCAGQRBJDQAgByACIAVqayACakEQSQ0AIANBEGshCiACQRBrIQsgAiAGQXBxIgRrIQIgAyAEayEDQQAhAQNAIAogAWsgCyABa/0AAAD9CwAAIAFBEGoiASAERw0ACyAEIAZGDQELIAdBf3MgAmohDiACIAdrQQNxIgQEQEEAIQEDQCADQQFrIgMgAkEBayICLQAAOgAAIAFBAWoiASAERw0ACwsgDkEDSQ0AA0AgA0EBayACQQFrLQAAOgAAIANBAmsgAkECay0AADoAACADQQNrIAJBA2stAAA6AAAgA0EEayIDIAJBBGsiAi0AADoAACACIAdHDQALCyAAKAIUIQILIAAgCDYCHCAAIAk2AhggACAFNgIUIAJFDQELIAIQLwsPCxA2AAsQNgALIAIgAToAACAAIAJBAWo2AhgLlAQBBH8gASAAIAFGIgI6AAwCQCACDQADQCABKAIIIgItAAwNAQJAIAIgAigCCCIDKAIAIgRGBEACQCADKAIEIgRFDQAgBC0ADA0ADAILAkAgASACKAIARgRAIAIhAQwBCyACIAIoAgQiASgCACIANgIEIAEgAAR/IAAgAjYCCCACKAIIBSADCzYCCCACKAIIIgAgACgCACACR0ECdGogATYCACABIAI2AgAgAiABNgIIIAEoAggiAygCACECCyABQQE6AAwgA0EAOgAMIAMgAigCBCIANgIAIAAEQCAAIAM2AggLIAIgAygCCDYCCCADKAIIIgAgACgCACADR0ECdGogAjYCACACIAM2AgQgAyACNgIIDwsCQCAERQ0AIAQtAAwNAAwBCwJAIAEgAigCAEcEQCACIQEMAQsgAiABKAIEIgA2AgAgASAABH8gACACNgIIIAIoAggFIAMLNgIIIAIoAggiACAAKAIAIAJHQQJ0aiABNgIAIAEgAjYCBCACIAE2AgggASgCCCEDCyABQQE6AAwgA0EAOgAMIAMgAygCBCIAKAIAIgE2AgQgAQRAIAEgAzYCCAsgACADKAIINgIIIAMoAggiASABKAIAIANHQQJ0aiAANgIAIAAgAzYCACADIAA2AggMAgsgBEEMaiEFIAJBAToADCADIAAgA0Y6AAwgBUEBOgAAIAMiASAARw0ACwsLnAIBBH8gACgCiAEiAQRAIAAgATYCjAEgARAvCyAAKAJ8IgEEQCAAIAE2AoABIAEQLwsgACgCcCIBBEAgACABNgJ0IAEQLwsgACgCZCIBBEAgACABNgJoIAEQLwsgACgCRCIDBEAgAyECIAMgACgCSCIBRwRAA0AgAUEMayICKAIAIgQEQCABQQhrIAQ2AgAgBBAvCyACIgEgA0cNAAsgACgCRCECCyAAIAM2AkggAhAvCyAAKAI4IgMEQCADIQIgAyAAKAI8IgFHBEADQCABQQxrIgIoAgAiBARAIAFBCGsgBDYCACAEEC8LIAIiASADRw0ACyAAKAI4IQILIAAgAzYCPCACEC8LIAAoAgAiAQRAIAAgATYCBCABEC8LC3ABA38gARBoIgJB8P///wdJBEACQCACQQpNBEAgACACOgALIAAhAwwBCyACQQ9yQQFqIgQQMiEDIAAgBEGAgICAeHI2AgggACADNgIAIAAgAjYCBAsgAyABIAL8CgAAIAIgA2pBADoAACAADwsQTQALHgEBfyAABEAgACgCACIBBEAgACABEQEACyAAEC8LC3sBAn8jAEEQayICJAAgACgCBCEDIAAgARB9AkAgAyAAKAIEIgNODQAgACgCDCADQQJ0akEEaygCACABRg0AQbzDAigCABAwGiACQekTNgIIIAJByPkANgIEIAJB5CY2AgBBuMMCKAIAQcvkACACEDEQAAALIAJBEGokAAsTACABQQF0QdCqA2pBAiAAEIEEC84BAQF/An8jAEEwayIDJAAgASgCiAFFBEAgACABKAIAIAEoAgwgAUEQakEAQQAQSSIARQRAQbzDAigCABAwGiADQdcvNgIYIANBxhQ2AhQgA0HkJjYCEEG4wwIoAgBBy+QAIANBEGoQMRAAAAsgACABNgKMASAAQQA2AogBIABBEzYCQCAAIAI4AkQgA0EwaiQAIAAMAQtBvMMCKAIAEDAaIANBmyE2AiggA0HcHjYCJCADQeQmNgIgQbjDAigCAEHL5AAgA0EgahAxEAAACwv0AgIDfwh+IwBBMGsiAyQAAn8CQAJAIAEpAxAiBiACKQMQIgeBQgBSDQAgASkDGCIIIAIpAxgiCYFCAFINACABKQMgIgogAikDICILgUIAUg0AIAEpAygiDCACKQMoIg2BQgBSDQAgAUEQaiEEAkAgASgCiAENACACKAKIAQ0ADAILAkAgBiAHUg0AIAggCVINACAKIAtSDQBBASEFIAwgDVENAgtBvMMCKAIAEDAaIANBq9cANgIYIANByBo2AhQgA0HkJjYCEEG4wwIoAgBBy+QAIANBEGoQMRAAAAtBvMMCKAIAEDAaIANBpdgANgIoIANBwho2AiQgA0HkJjYCIEG4wwIoAgBBy+QAIANBIGoQMRAAAAsgACABKAIAIAEoAgwgBEEAQQAQSSIEQQY2AkBBACAFRQ0AGiAAIAQoAgAgBCgCDCAEQRBqQQBBABBJCyEAIAQgATYCjAEgBCAANgKIASAEIAI2ApABIANBMGokACAEC3YBAX8jAEEQayICJAAgAiAANgIMAkAgACABRg0AA0AgAiABQQFrIgE2AgggACABTw0BIAIoAgwiAC0AACEBIAAgAigCCCIALQAAOgAAIAAgAToAACACIAIoAgxBAWoiADYCDCACKAIIIQEMAAsACyACQRBqJAAL2wgBAX9BAEEB/h4CkNY0QQBKBEADQEEAQQH+JQKQ1jQaEAMQ1gFBAEEB/h4CkNY0QQBKDQALCwJAAkAgAEHguwNGDQAgAEGUvANGBEBBASEBDAELIABByLwDRgRAQQIhAQwBCyAAQfy8A0YEQEEDIQEMAQsgAEGwvQNGBEBBBCEBDAELIABB5L0DRgRAQQUhAQwBCyAAQZi+A0YEQEEGIQEMAQsgAEHMvgNGBEBBByEBDAELIABBgL8DRgRAQQghAQwBCyAAQbS/A0YEQEEJIQEMAQsgAEHovwNGBEBBCiEBDAELIABBnMADRgRAQQshAQwBCyAAQdDAA0YEQEEMIQEMAQsgAEGEwQNGBEBBDSEBDAELIABBuMEDRgRAQQ4hAQwBCyAAQezBA0YEQEEPIQEMAQsgAEGgwgNGBEBBECEBDAELIABB1MIDRgRAQREhAQwBCyAAQYjDA0YEQEESIQEMAQsgAEG8wwNGBEBBEyEBDAELIABB8MMDRgRAQRQhAQwBCyAAQaTEA0YEQEEVIQEMAQsgAEHYxANGBEBBFiEBDAELIABBjMUDRgRAQRchAQwBCyAAQcDFA0YEQEEYIQEMAQsgAEH0xQNGBEBBGSEBDAELIABBqMYDRgRAQRohAQwBCyAAQdzGA0YEQEEbIQEMAQsgAEGQxwNGBEBBHCEBDAELIABBxMcDRgRAQR0hAQwBCyAAQfjHA0YEQEEeIQEMAQsgAEGsyANGBEBBHyEBDAELIABB4MgDRgRAQSAhAQwBCyAAQZTJA0YEQEEhIQEMAQsgAEHIyQNGBEBBIiEBDAELIABB/MkDRgRAQSMhAQwBCyAAQbDKA0YEQEEkIQEMAQsgAEHkygNGBEBBJSEBDAELIABBmMsDRgRAQSYhAQwBCyAAQczLA0YEQEEnIQEMAQsgAEGAzANGBEBBKCEBDAELIABBtMwDRgRAQSkhAQwBCyAAQejMA0YEQEEqIQEMAQsgAEGczQNGBEBBKyEBDAELIABB0M0DRgRAQSwhAQwBCyAAQYTOA0YEQEEtIQEMAQsgAEG4zgNGBEBBLiEBDAELIABB7M4DRgRAQS8hAQwBCyAAQaDPA0YEQEEwIQEMAQsgAEHUzwNGBEBBMSEBDAELIABBiNADRgRAQTIhAQwBCyAAQbzQA0YEQEEzIQEMAQsgAEHw0ANGBEBBNCEBDAELIABBpNEDRgRAQTUhAQwBCyAAQdjRA0YEQEE2IQEMAQsgAEGM0gNGBEBBNyEBDAELIABBwNIDRgRAQTghAQwBCyAAQfTSA0YEQEE5IQEMAQsgAEGo0wNGBEBBOiEBDAELIABB3NMDRgRAQTshAQwBCyAAQZDUA0YEQEE8IQEMAQsgAEHE1ANGBEBBPSEBDAELIABB+NQDRgRAQT4hAQwBCyAAQazVA0cNAUE/IQELIAFBNGxB3LsDakEAOgAAIAAtAAhFDQAgACgCBBAvC0EAQQH+JQKQ1jQaC9sBAgF/An5BASEEAkAgAEIAUiABQv///////////wCDIgVCgICAgICAwP//AFYgBUKAgICAgIDA//8AURsNACACQgBSIANC////////////AIMiBkKAgICAgIDA//8AViAGQoCAgICAgMD//wBRGw0AIAAgAoQgBSAGhIRQBEBBAA8LIAEgA4NCAFkEQEF/IQQgACACVCABIANTIAEgA1EbDQEgACAChSABIAOFhEIAUg8LQX8hBCAAIAJWIAEgA1UgASADURsNACAAIAKFIAEgA4WEQgBSIQQLIAQLUAEBfgJAIANBwABxBEAgAiADQUBqrYghAUIAIQIMAQsgA0UNACACQcAAIANrrYYgASADrSIEiIQhASACIASIIQILIAAgATcDACAAIAI3AwgLqQEBA38jAEGgAWsiBCQAIAQgACAEQZ4BaiABGyIFNgKUAUF/IQAgBCABQQFrIgZBACABIAZPGzYCmAEgBEEAQZAB/AsAIARBfzYCTCAEQc8BNgIkIARBfzYCUCAEIARBnwFqNgIsIAQgBEGUAWo2AlQCQCABQQBIBEAjA0EcakE9NgIADAELIAVBADoAACAEIAIgA0HNAUHOARDDAiEACyAEQaABaiQAIAALqAEAAkAgAUGACE4EQCAARAAAAAAAAOB/oiEAIAFB/w9JBEAgAUH/B2shAQwCCyAARAAAAAAAAOB/oiEAQf0XIAEgAUH9F04bQf4PayEBDAELIAFBgXhKDQAgAEQAAAAAAABgA6IhACABQbhwSwRAIAFByQdqIQEMAQsgAEQAAAAAAABgA6IhAEHwaCABIAFB8GhMG0GSD2ohAQsgACABQf8Haq1CNIa/ogswAQF/IAEjAyICKAJIIABBAnRqIgAoAgBHBEAgACABNgIAIAIgAi0AKkEBcjoAKgsLJgEBfyMAQRBrIgIkACACIAE2AgxBsLoDIAAgARCGBCACQRBqJAALyQUEBH8CfAF9AX4gAbwiBEEBdEGAgIAIakGBgIAISSECAkACQAJAAkAgALwiA0GAgID8B2tBgICAiHhPBEAgAg0BDAMLIAJFDQELQwAAgD8hCCADQYCAgPwDRg0CIARBAXQiAkUNAiACQYGAgHhJIANBAXQiAkGAgIB4TXFFBEAgACABkg8LIAJBgICA+AdGDQJDAAAAACABIAGUIAJB////9wdLIARBAE5zGw8LIANBAXRBgICACGpBgYCACEkEQCAAIACUIQggA0EASARAIAiMIAggBBCjBEEBRhshCAsgBEEATg0CIwBBEGsiAkMAAIA/IAiVOAIMIAIqAgwPCyADQQBIBEAgBBCjBCICRQRAIAAgAJMiACAAlQ8LIANB/////wdxIQMgAkEBRkEQdCEFCyADQf///wNLDQAgAEMAAABLlLxB/////wdxQYCAgNwAayEDCwJAQZDDAisDACADIANBgIDM+QNrIgRBgICAfHFrvrsgBEEPdkHwAXEiAkGQwQJqKwMAokQAAAAAAADwv6AiBqJBmMMCKwMAoCAGIAaiIgcgB6KiQaDDAisDACAGokGowwIrAwCgIAeiQbDDAisDACAGoiACQZjBAmorAwAgBEEXdbegoKCgIAG7oiIHvUKAgICAgIDg//8Ag0KBgICAgIDAr8AAVA0AIAdEcdXR////X0BkBEAjAEEQayICQwAAAPBDAAAAcCAFGzgCDCACKgIMQwAAAHCUDwsgB0QAAAAAAMBiwGVFDQAjAEEQayICQwAAAJBDAAAAECAFGzgCDCACKgIMQwAAABCUDwtB2NoBKwMAIAdB0NoBKwMAIgYgB6AiByAGoaEiBqJB4NoBKwMAoCAGIAaiokHo2gErAwAgBqJEAAAAAAAA8D+goCAHvSIJIAWtfEIvhiAJp0EfcUEDdEHQ2AFqKQMAfL+itiEICyAIC2UBAX8CQCAARQ0AIAFBAEgNACAAQQNxDQAgAUUEQA8LIABBACAAQQD+SAKQ8TQiAiAAIAJGGyECAkAgAUH/////B0YNACAAIAJHDQAgAUECSQ0BIAFBAWshAQsgACAB/gACABoLC4cCAQJ/IwBBEGsiBiQAIAAgAjYCBCAAQeSwATYCACAAQQhqIgIgASgCACIHNgIAIAdBBGpBAf4eAgAaIAAgASkCBDcCDCAAQRRqQQBBxAD8CwAgACAFOgBaIAAgBDoAWSAAIAM6AFggBiACKAIAIgE2AgAgAUEEakEB/h4CABogBkEEaiIDIAYiAigCAEGYAWoQ5wFBASEBIAYoAgggBi0ADyIEIATAIgRBAEgiBRtBAUYEQCAGKAIEIAMgBRstAABBwwBHIQELIARBAEgEQCAGKAIEEC8LIAIoAgAiAkEEakF//h4CAEUEQCACIAIoAgAoAggRAQALIAAgAToAWyAGQRBqJAAgAAsfAEEIEF4gABCJAyIAQdC3AzYCACAAQfC3A0EHEAIACxAAIAAoAgAEQBCZAgALIAALGQAgASACEJ0DIQEgACACNgIEIAAgATYCAAuGAgEEfyMAQRBrIgUkACABEJMDIQIjAEEQayIDJAACQCACQe////8DTQRAAkAgAkECSQRAIAAgAC0AC0GAAXEgAnI6AAsgACAALQALQf8AcToACyAAIQQMAQsgA0EIaiAAIAJBAk8EfyACQQRqQXxxIgQgBEEBayIEIARBAkYbBUEBC0EBahC5ASADKAIMGiAAIAMoAggiBDYCACAAIAAoAghBgICAgHhxIAMoAgxB/////wdxcjYCCCAAIAAoAghBgICAgHhyNgIIIAAgAjYCBAsgBCABIAIQngEgA0EANgIEIAQgAkECdGogAygCBDYCACADQRBqJAAMAQsQTQALIAVBEGokAAvkAQEGfyMAQRBrIgUkACAAKAIEIQMCfyACKAIAIAAoAgBrIgRB/////wdJBEAgBEEBdAwBC0F/CyIEQQQgBBshBCABKAIAIQcgACgCACEIIANB9wFGBH9BAAUgACgCAAsgBBDEASIGBEAgA0H3AUcEQCAAKAIAGiAAQQA2AgALIAVB9gE2AgQgACAFQQhqIAYgBUEEahBRIgMQvgMgAygCACEGIANBADYCACAGBEAgBiADKAIEEQEACyABIAAoAgAgByAIa2o2AgAgAiAAKAIAIARBfHFqNgIAIAVBEGokAA8LEEYACzsBAn8jAEEgayIFJAAgBSAENwMQIAUgAzcDCCAFIAI3AwAgACABQQMgBUEAQQAQSSEGIAVBIGokACAGC5ADAQJ/IwBBEGsiCiQAIAogADYCDAJAAkACQCADKAIAIAJHDQBBKyELIAAgCSgCYEcEQEEtIQsgCSgCZCAARw0BCyADIAJBAWo2AgAgAiALOgAADAELAkACfyAGLQALQQd2BEAgBigCBAwBCyAGLQALQf8AcQtFDQAgACAFRw0AQQAhACAIKAIAIgEgB2tBnwFKDQIgBCgCACEAIAggAUEEajYCACABIAA2AgAMAQtBfyEAIAkgCUHoAGogCkEMahCxAiAJayIGQdwASg0BIAZBAnUhBQJAAkACQCABQQhrDgMAAgABCyABIAVKDQEMAwsgAUEQRw0AIAZB2ABIDQAgAygCACIBIAJGDQIgASACa0ECSg0CIAFBAWstAABBMEcNAkEAIQAgBEEANgIAIAMgAUEBajYCACABIAVB0OoCai0AADoAAAwCCyADIAMoAgAiAEEBajYCACAAIAVB0OoCai0AADoAACAEIAQoAgBBAWo2AgBBACEADAELQQAhACAEQQA2AgALIApBEGokACAAC5UMAgZ/An0jAEFAaiIDJABBAEEB/h4CkNY0QQBKBEADQEEAQQH+JQKQ1jQaEAMQ1gFBAEEB/h4CkNY0QQBKDQALC0GE1gQtAABFBEAgA0EwahDXAQNAIAJBAnRBkNYEaiACQf//AXFBgICA+ANyvkMAAAC/kiACQRF0IgFBBHZBgICAgAdyvkMAAIAHlCABQYCAgMAASRu8IAJBEHRBgICAgHhxciIENgIAIAJBAXQiAUGQ1hRqQYD8ASAEviIHQwAAAD+UIAdDKkJMP5QgB0MTJzc9lCAHlEMAAIA/kpQQjQRDAACAP5KUIgiLQwAAgHeUQwAAgAiUQYCAgIgHIAi8IgRBAXQiBUGAgIB4cSIGIAZBgICAiAdNG0EBdkGAgIA8ar6SvCIGQQ12QYD4AXEgBkH/H3FqIAVBgICAeEsbIARBEHZBgIACcXI7AQAgAUGQ1hxqQYD8AUMAAIA/IAdDI9vZv5QQRUMAAIA/kpUgB5QiCItDAACAd5RDAACACJRBgICAiAcgCLwiBEEBdCIFQYCAgHhxIgYgBkGAgICIB00bQQF2QYCAgDxqvpK8IgZBDXZBgPgBcSAGQf8fcWogBUGAgIB4SxsgBEEQdkGAgAJxcjsBACABQZDWJGpBgPwBIAcgB4wQRUMAAIA/kpUiCItDAACAd5RDAACACJRBgICAiAcgCLwiBEEBdCIFQYCAgHhxIgYgBkGAgICIB00bQQF2QYCAgDxqvpK8IgZBDXZBgPgBcSAGQf8fcWogBUGAgIB4SxsgBEEQdkGAgAJxcjsBACABQZDWLGpBgPwBIAcQRSIHQwAAgHeUQwAAgAiUQYCAgIgHIAe8IgFBAXQiBEGAgIB4cSIFIAVBgICAiAdNG0EBdkGAgIA8ar6SvCIFQQ12QYD4AXEgBUH/H3FqIARBgICAeEsbIAFBEHZBgIACcXI7AQAgAkEBaiICQYCABEcNAAsgA0EwaiIBENcBIAEQ1wFB3LsDQQBBqJoB/AsAIAEQ1wFB5tY0QQE6AABB2NY0QQE6AABBz9Y0QQE6AABBzdY0QQE6AABBxdY0QYECOwAAQcPWNEEBOgAAQbvWNEEBOgAAQbnWNEEBOgAAQbfWNEGBAjsAAEGk1jRBAToAAEHc1jRBAToAAEG21zRBAToAAEGE1gRBAToAAAtBACECAkACQAJAAkADQCACQTRsQdy7A2oiAS0AAEUEQCACIQQMAgsgAkEBciIEQTRsQdy7A2oiAS0AAEUNASACQQJyIgRBNGxB3LsDaiIBLQAARQ0BIAJBA3IiBEE0bEHcuwNqIgEtAABFDQEgAkEEaiICQcAARw0AC0EAIQIMAQsgAUEBOgAAIARBNGxB4LsDaiECIAAoAgAiAUUEQCAAQQQ2AgBBBCEBCyABIAFBA2pBfHEiBSAAKAIEIgQbIQYCQCAEIgENACAFRQRAQQAhAUH8ugMoAgBBAE4EQEGwugMQggEhAQsCQEGe4ABBsLoDEKsEQQBIDQACQEGAuwMoAgBBCkYNAEHEugMoAgAiBUHAugMoAgBGDQBBxLoDIAVBAWo2AgAgBUEKOgAADAELENECCyABBEBBsLoDEIEBC0EAIQEMAQsgA0EANgIwIANBMGpBBCAFEIMEIgEEQCADAn8gAUEwRwRAQZYUIAFBHEcNARpBpB8MAQtBzQgLNgIkIANBgCY2AiAgAyAFuEQAAAAAAACwPqI5AyhB6PgAIANBIGoQogRBACEBDAELIAMoAjAhAQsgAiAALQAIIgA6AAogAiAAOgAJIAIgBEU6AAggAiABNgIEIAIgBjYCACAC/QwAAAAAAAAAAAAAAAAAAAAA/QsACyAC/QwAAAAAAAAAAAAAAAAAAAAA/QsAGyACQgA3ACggAUUNASABQQNxDQILQQBBAf4lApDWNBogA0FAayQAIAIPC0G8wwIoAgAQMBogA0HmLzYCCCADQf8RNgIEIANB5CY2AgBBuMMCKAIAQcvkACADEDEQAAALQbzDAigCABAwGiADQdXBADYCGCADQYESNgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEQAAALCgAgAEHcijUQeQuMAwEDfyMAQRBrIgokACAKIAA6AA8CQAJAAkAgAygCACACRw0AQSshCyAAQf8BcSIMIAktABhHBEBBLSELIAktABkgDEcNAQsgAyACQQFqNgIAIAIgCzoAAAwBCwJAAn8gBi0AC0EHdgRAIAYoAgQMAQsgBi0AC0H/AHELRQ0AIAAgBUcNAEEAIQAgCCgCACIBIAdrQZ8BSg0CIAQoAgAhACAIIAFBBGo2AgAgASAANgIADAELQX8hACAJIAlBGmogCkEPahC0AiAJayIFQRdKDQECQAJAAkAgAUEIaw4DAAIAAQsgASAFSg0BDAMLIAFBEEcNACAFQRZIDQAgAygCACIBIAJGDQIgASACa0ECSg0CIAFBAWstAABBMEcNAkEAIQAgBEEANgIAIAMgAUEBajYCACABIAVB0OoCai0AADoAAAwCCyADIAMoAgAiAEEBajYCACAAIAVB0OoCai0AADoAACAEIAQoAgBBAWo2AgBBACEADAELQQAhACAEQQA2AgALIApBEGokACAACwoAIABB1Io1EHkLYwIBfwF+IwBBEGsiAiQAIAACfiABRQRAQgAMAQsgAiABrUIAIAFnIgFB0QBqEGYgAikDCEKAgICAgIDAAIVBnoABIAFrrUIwhnwhAyACKQMACzcDACAAIAM3AwggAkEQaiQACxkAIAEgAhDzAyEBIAAgAjYCBCAAIAE2AgALuQgBC38gAEUEQCABEDsPCyABQUBPBEAjA0EcakEwNgIAQQAPCwJAQbyHNS0AAEECcQRAQcCHNRBWDQELAn9BECABQQtqQXhxIAFBC0kbIQUgAEEIayIEKAIEIghBeHEhAgJAIAhBA3FFBEBBACAFQYACSQ0CGiAFQQRqIAJNBEAgBCEDIAIgBWtB8IM1KAIAQQF0TQ0CC0EADAILIAIgBGohBgJAIAIgBU8EQCACIAVrIgNBEEkNASAEIAhBAXEgBXJBAnI2AgQgBCAFaiICIANBA3I2AgQgBiAGKAIEQQFyNgIEIAIgAxD0AQwBC0GYhDUoAgAgBkYEQEGMhDUoAgAgAmoiAiAFTQ0CIAQgCEEBcSAFckECcjYCBCAEIAVqIgMgAiAFayICQQFyNgIEQYyENSACNgIAQZiENSADNgIADAELQZSENSgCACAGRgRAQYiENSgCACACaiICIAVJDQICQCACIAVrIgNBEE8EQCAEIAhBAXEgBXJBAnI2AgQgBCAFaiIHIANBAXI2AgQgAiAEaiICIAM2AgAgAiACKAIEQX5xNgIEDAELIAQgCEEBcSACckECcjYCBCACIARqIgMgAygCBEEBcjYCBEEAIQMLQZSENSAHNgIAQYiENSADNgIADAELIAYoAgQiB0ECcQ0BIAdBeHEgAmoiCSAFSQ0BIAkgBWshCwJAIAdB/wFNBEAgBigCDCIDIAYoAggiAkYEQEGAhDVBgIQ1KAIAQX4gB0EDdndxNgIADAILIAIgAzYCDCADIAI2AggMAQsgBigCGCEKAkAgBiAGKAIMIgJHBEBBkIQ1KAIAGiAGKAIIIgMgAjYCDCACIAM2AggMAQsCQCAGQRRqIgMoAgAiB0UEQCAGKAIQIgdFDQEgBkEQaiEDCwNAIAMhDCAHIgJBFGoiAygCACIHDQAgAkEQaiEDIAIoAhAiBw0ACyAMQQA2AgAMAQtBACECCyAKRQ0AAkAgBigCHCIDQQJ0QbCGNWoiBygCACAGRgRAIAcgAjYCACACDQFBhIQ1QYSENSgCAEF+IAN3cTYCAAwCCyAKQRBBFCAKKAIQIAZGG2ogAjYCACACRQ0BCyACIAo2AhggBigCECIDBEAgAiADNgIQIAMgAjYCGAsgBigCFCIDRQ0AIAIgAzYCFCADIAI2AhgLIAtBD00EQCAEIAhBAXEgCXJBAnI2AgQgBCAJaiIDIAMoAgRBAXI2AgQMAQsgBCAIQQFxIAVyQQJyNgIEIAQgBWoiAyALQQNyNgIEIAQgCWoiAiACKAIEQQFyNgIEIAMgCxD0AQsgBCEDCyADCyEDQbyHNS0AAEECcQRAQcCHNRBSGgsgAwRAIANBCGoPCyABEDsiA0UEQEEADwsgAyAAQXxBeCAAQQRrKAIAIgRBA3EbIARBeHFqIgQgASABIARLGxB7GiAAEC8LIAMLXwECfyAAQQdqQXhxIQICQANAIAJBAEHEuwP+EAIAIgAgAmoiASAATRsNASABPwBBEHRLBEAgARAdRQ0CC0EAIAAgAf5IAsS7AyAARw0ACyAADwsjA0EcakEwNgIAQX8LgwECBX8BfgJAIABCgICAgBBUBEAgACEHDAELA0AgAUEBayIBIAAgAEIKgCIHQgp+fadBMHI6AAAgAEL/////nwFWIQUgByEAIAUNAAsLIAenIgIEQANAIAFBAWsiASACIAJBCm4iA0EKbGtBMHI6AAAgAkEJSyEGIAMhAiAGDQALCyABCxoAIAAgARCOBCIAQQAgAC0AACABQf8BcUYbC88FAQh/IwBBEGsiBSQAIAACfyAFQQhqIQMgASEGAkACQEG81zQiBEG81zRHBEAgASgCACIHQczXNCgCACIBTg0BC0G81zQoAgAhBEG81zQhAwJAQbjXNCgCAEG81zRHBEACQCAERQRAQbzXNCEBA0AgASgCCCIDKAIAIAFGIQkgAyEBIAkNAAsMAQsgBCEBA0AgASIDKAIEIgENAAsLIAYoAgAiBiADKAIQTA0BCyAERQ0CIAUgAzYCDCADQQRqDAMLQbzXNCgCACIBRQ0BQbzXNCEEA0ACQCABIgMoAhAiASAGSgRAIAMiBCgCACIBDQIMAQsgASAGTg0AIANBBGohBCADKAIEIgENAQsLIAUgAzYCDCAEDAILIAEgB0gEQAJAQcDXNCgCACIGRQRAQbzXNCEBA0AgASgCCCIDKAIAIAFHIQogAyEBIAoNAAsMAQsgBiEBA0AgASIDKAIAIgENAAsLAkAgA0G81zRHBEAgByADKAIQTg0BCyAGRQRAIAVBvNc0NgIMQcDXNAwECyAFIAM2AgwgAwwDC0G81zQoAgAiAUUNAQNAAkAgASIDKAIQIgEgB0oEQCADIgQoAgAiAQ0CDAELIAEgB04NACADQQRqIQQgAygCBCIBDQELCyAFIAM2AgwgBAwCCyAFQbzXNDYCDCADQbzXNDYCACADDAELIAVBvNc0NgIMQbzXNAsiBCgCACIBBH9BAAVBIBAyIgEgAigCADYCECABQRRqIQMCQCACLAAPQQBOBEAgAyACKQIENwIAIAMgAigCDDYCCAwBCyADIAIoAgQgAigCCBBrCyABIAUoAgw2AgggAUIANwIAIAQgATYCACABIQJBuNc0KAIAKAIAIgMEQEG41zQgAzYCACAEKAIAIQILQbzXNCgCACACEKQBQcDXNEHA1zQoAgBBAWo2AgBBAQs6AAQgACABNgIAIAVBEGokAAuIAQEDf0GcEBA7IQEgABDoBCEDIAEgACgCHCICIAIoAgQRAAAiAjYCDCABIAM2AgggAUEANgIEIAEgADYCACABQRxqQQBBgBD8CwAgAUEBNgIQIAEgAyACIAMgAnBrIAJwIgJqNgIUIAEgACgCJCACazYCGEEIEDsiABCCBDYCBCAAIAE2AgAgAAuTwwcF1gF/On4gewZ9BHwjAEFAaiIuJAACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQCAABEACQCABKAJAIgJFDQACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAIAJBAWsOSEMAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobRERERBwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ2NzU4OTo7PD0+P0BBQkQLIAEoAowBIQUgASgCkAEhAyMAQYADayIEJAACQAJAIAUoAgAiAkEOSw0AAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkBBASACdEHM+wFxRQRAAkACQAJAIAIEQCACQQFHDRggAygCAA4CAgEDCyAFKQMQItkBINkBIAMpAxAi2gF/It0BINoBflINBSAFKQMYItsBIAMpAxgi3gGBQgBSDQUgBSkDICLYASADKQMgIt8BgUIAUg0FIAUpAygi4AEgAykDKCLhAYFCAFINBSDZASABKQMQUg0FINsBIAEpAxhSDQUg2AEgASkDIFINBSDgASABKQMoUg0FAkAgACgCAA4DBQAFAAsgASgCMEEERw0GIAUoAjBBBEcNByABKAI8IQkgASgCOCEKIAEoAjQhCCADKAI8IQsgAygCOCEMIAMoAjQhDSAFKAI8IQ8gBSgCOCEOIAUoAjQhECDgASDYASDbAX4i3AF+pyICIAAoAggiB2pBAWsgB20iByAAKAIEbCIAIAdqIgcgAiACIAdKGyEHIAMoAjAiAkEERwRAIAAgB04NBSDZAUIAVw0FIAUoAtABIQUgASgC0AEhBiDZAUJ+gyHiASDZAUIBgyHjASAHrCHkASAArCHdASADKALQASEHA0AgBSAPIN0BINwBfyLYAaciAWxqIA4g3QEg2AEg3AF+fSLlASDbAX8i4AGnIgNsaiAQIOUBINsBIOABfn0i5QGnIhFsaiEAIAYgASAJbGogAyAKbGogCCARbGohASAHIAsg2AEg4QGBp2xqIAwg4AEg3wGBp2xqIA0g5QEg3gGBp2xqIQNCACHYAUIAIeABINkBQgFSBEADQCABINgBp0ECdCIRaiAAIBFqKgIAIAMgAiDYASDaAYGnbGoqAgCSOAIAIAEg2AFCAYQi5QGnQQJ0IhFqIAAgEWoqAgAgAyACIOUBINoBgadsaioCAJI4AgAg2AFCAnwh2AEg4AFCAnwi4AEg4gFSDQALCyDjAacEQCABINgBp0ECdCIRaiAAIBFqKgIAIAMgAiDYASDaAYGnbGoqAgCSOAIACyDdAUIBfCLdASDkAVINAAsMBQsgACAHTg0EIN0BQgBXDQQgAygC0AEhEyAFKALQASERIAEoAtABIRJBACDaAaciAmshFiACQQFxIRUgAkF8cSEBINoBQgKGQvz///8PgyHiASAHrCHjASAArCHZASACQQhJIRgDQCDZASDcAX8i2AEg4QGBIeQBINkBINgBINwBfn0i5QEg2wF/IuABIN8BgSHmASDlASDbASDgAX59IuUBIN4BgSHnASACQQBKBEAgECDlAaciAGwiBSAOIOABpyIDbCIHIBEgDyDYAaciBmwiGmpqaiEZIAAgCGwiACADIApsIhQgEiAGIAlsIgZqamohFyATIAsg5AGnbGogDCDmAadsaiANIOcBp2xqIQMgBSARaiAaaiAHaiEaIAAgEmogBmogFGohFEIAIdgBA0AgGSDYASDaAX6nQQJ0IgBqIQUgACAXaiEHQQAhAAJAAkAgGA0AIBQg2AEg4gF+pyIGaiIbIAYgGmprQRBJDQAgGyADa0EQSQ0AA0AgByAAQQJ0IgZqIAUgBmr9AAIAIAMgBmr9AAIA/eQB/QsCACAAQQRqIgAgAUcNAAsgASIAIAJGDQELIABBf3MhGyAVBEAgByAAQQJ0IgZqIAUgBmoqAgAgAyAGaioCAJI4AgAgAEEBciEACyAWIBtGDQADQCAHIABBAnQiBmogBSAGaioCACADIAZqKgIAkjgCACAHIAZBBGoiBmogBSAGaioCACADIAZqKgIAkjgCACAAQQJqIgAgAkcNAAsLINgBQgF8ItgBIN0BUg0ACwsg2QFCAXwi2QEg4wFSDQALDAQLIAUpAxAi2wEgAykDEFINByAFKQMYItwBIAMpAxhSDQcgBSkDICLYASADKQMgUg0HIAUpAygi2gEgAykDKFINByDbASABKQMQUg0HINwBIAEpAxhSDQcg2AEgASkDIFINByDaASABKQMoUg0HAkAgACgCAA4DBAAEAAsgASgCAEEBRw0IIAEoAjBBAkcNCSAFKAIwQQJHDQog2gEg2AEg3AF+It0BfqciAiAAKAIIIgdqQQFrIAdtIQcgAygCMEECRgRAIAcgACgCBGwiACAAIAdqIgcgAiACIAdKGyIHTg0EINsBQgBXDQQgASgCPCEKIAEoAjghCCABKAI0IQsgAygCPCEMIAMoAjghDSADKAI0IQ8gBSgCPCEOIAUoAjghECAFKAI0IREgAygC0AEhAiAFKALQASEDIAEoAtABIQEg2wFCfIMh2gEgB6wh3gEgAKwh2QEDQCACIAwg2QEg3QF/ItgBpyIAbCISaiANINkBINgBxCDdAX59ItgBINwBfyLfAaciBWwiE2ogDyDYASDcASDfAX59pyIHbCIWaiEGIAcgEWwiFSAFIBBsIhggAyAAIA5sIhpqamohCSAHIAtsIgcgBSAIbCIFIAEgACAKbCIZampqIQBCACHYAQJAAkAg2wFCBFQNACABIAdqIBlqIAVqIgUgAyAVaiAaaiAYamtBCEkNACAFIAIgFmogEmogE2prQQhJDQADQCAAINgBp0EBdCIFav0MAH4AAAB+AAAAfgAAAH4AACAFIAlq/QQBACKSAv0bAEECdEGQ1gRq/QkCACCSAv0bAUECdEGQ1gRqKgIA/SABIJIC/RsCQQJ0QZDWBGoqAgD9IAIgkgL9GwNBAnRBkNYEaioCAP0gAyAFIAZq/QQBACKSAv0bAEECdEGQ1gRq/QkCACCSAv0bAUECdEGQ1gRqKgIA/SABIJIC/RsCQQJ0QZDWBGoqAgD9IAIgkgL9GwNBAnRBkNYEaioCAP0gA/3kASKSAv3gAf0MAACAdwAAgHcAAIB3AACAd/3mAf0MAACACAAAgAgAAIAIAACACP3mASCSAkEB/asBIpMC/QwAAAD/AAAA/wAAAP8AAAD//U79DAAAAHEAAABxAAAAcQAAAHH9uQFBAf2tAf0MAACABwAAgAcAAIAHAACAB/2uAf3kASKUAkEN/a0B/QwAfAAAAHwAAAB8AAAAfAAA/U4glAL9DP8PAAD/DwAA/w8AAP8PAAD9Tv2uASCTAv0MAAAA/wAAAP8AAAD/AAAA//08/VIgkgJBEP2tAf0MAIAAAACAAAAAgAAAAIAAAP1O/VAgkgL9DQABBAUICQwNAAEAAQABAAH9WwEAACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2wFRDQELA0AgACDYAadBAXQiBWpBgPwBIAUgCWovAQBBAnRBkNYEaioCACAFIAZqLwEAQQJ0QZDWBGoqAgCSIrICi0MAAIB3lEMAAIAIlEGAgICIByCyArwiBUEBdCIHQYCAgHhxIhIgEkGAgICIB00bQQF2QYCAgDxqvpK8IhJBDXZBgPgBcSASQf8fcWogB0GAgIB4SxsgBUEQdkGAgAJxcjsBACDYAUIBfCLYASDbAVINAAsLINkBQgF8ItkBIN4BUg0ACwwEC0G8wwIoAgAQMBogBEGbITYCWCAEQYE4NgJUIARB5CY2AlBBuMMCKAIAQcvkACAEQdAAahAxDG8LIAUpAxAi2wEgAykDEFINCiAFKQMYItwBIAMpAxhSDQogBSkDICLYASADKQMgUg0KIAUpAygi2gEgAykDKFINCiDbASABKQMQUg0KINwBIAEpAxhSDQog2AEgASkDIFINCiDaASABKQMoUg0KAkAgACgCAA4DAwADAAsgASgCPCEHIAEoAjghBiABKAI0IQkgAygCPCEKIAMoAjghCCADKAI0IQsgBSgCPCEMIAUoAjghDSAFKAI0IQ8gASgCMCECIAMoAjAhowEgBSgCMCGiASAAKAIIIQ4gACgCBCEAAkACQAJAAkAgASgCACISDgIAAgELIAJBBEYNAkG8wwIoAgAQMBogBEHd0AA2AtgBIARBmDc2AtQBIARB5CY2AtABQbjDAigCAEHL5AAgBEHQAWoQMQxxC0G8wwIoAgAQMBogBEG9OTYCqAEgBEGbNzYCpAEgBEHkJjYCoAFBuMMCKAIAQcvkACAEQaABahAxDHALIAJBAkcNDAsgogFBAkcNDCDaASDYASDcAX4i3QF+pyICIA5qQQFrIA5tIQ4gowFBBEYEQCAOIAAgDmwiAGoiDiACIAIgDkobIQIgEkEBRwRAIAAgAk4NBCDbAUIAVw0EIAMoAtABIQ4gBSgC0AEhECABKALQASERINsBQn6DId4BINsBQgGDId8BIAKsIeABIACsIdoBA0AgDiAKINoBIN0BfyLYAaciAmxqIAgg2gEg2AHEIN0Bfn0i2AEg3AF/ItkBpyIDbGogCyDYASDZASDcAX59pyIFbGohACAQIAIgDGxqIAMgDWxqIAUgD2xqIQEgESACIAdsaiADIAZsaiAFIAlsaiECQgAh2AFCACHZASDbAUIBUgRAA0AgAiDYAaciA0ECdCIFaiABIANBAXRqLwEAQQJ0QZDWBGoqAgAgACAFaioCAJI4AgAgAiADQQFyIgNBAnQiBWogASADQQF0ai8BAEECdEGQ1gRqKgIAIAAgBWoqAgCSOAIAINgBQgJ8IdgBINkBQgJ8ItkBIN4BUg0ACwsg3wGnBEAgAiDYAaciA0ECdCIFaiABIANBAXRqLwEAQQJ0QZDWBGoqAgAgACAFaioCAJI4AgALINoBQgF8ItoBIOABUg0ACwwECyAAIAJODQMg2wFCAFcNAyADKALQASERIAUoAtABIQMgASgC0AEhASDbAUJ8gyHaASACrCHeASAArCHZAQNAIBEgCiDZASDdAX8i2AGnIgBsaiAIINkBINgBxCDdAX59ItgBINwBfyLfAaciAmxqIAsg2AEg3AEg3wF+faciBWxqIQ4gBSAPbCISIAIgDWwiEyADIAAgDGwiFmpqaiEQIAUgCWwiBSACIAZsIgIgASAAIAdsIhVqamohAEIAIdgBAkACQCDbAUIEVA0AIAEgBWogFWogAmogAyASaiAWaiATamtBCEkNAANAIAAg2AGnIgJBAXQiBWr9DAB+AAAAfgAAAH4AAAB+AAAgBSAQav0EAQAikgL9GwBBAnRBkNYEav0JAgAgkgL9GwFBAnRBkNYEaioCAP0gASCSAv0bAkECdEGQ1gRqKgIA/SACIJIC/RsDQQJ0QZDWBGoqAgD9IAMgDiACQQJ0av0AAgD95AEikgL94AH9DAAAgHcAAIB3AACAdwAAgHf95gH9DAAAgAgAAIAIAACACAAAgAj95gEgkgJBAf2rASKTAv0MAAAA/wAAAP8AAAD/AAAA//1O/QwAAABxAAAAcQAAAHEAAABx/bkBQQH9rQH9DAAAgAcAAIAHAACABwAAgAf9rgH95AEilAJBDf2tAf0MAHwAAAB8AAAAfAAAAHwAAP1OIJQC/Qz/DwAA/w8AAP8PAAD/DwAA/U79rgEgkwL9DAAAAP8AAAD/AAAA/wAAAP/9PP1SIJICQRD9rQH9DACAAAAAgAAAAIAAAACAAAD9Tv1QIJIC/Q0AAQQFCAkMDQABAAEAAQAB/VsBAAAg2AFCBHwi2AEg2gFSDQALINoBItgBINsBUQ0BCwNAIAAg2AGnIgJBAXQiBWpBgPwBIAUgEGovAQBBAnRBkNYEaioCACAOIAJBAnRqKgIAkiKyAotDAACAd5RDAACACJRBgICAiAcgsgK8IgJBAXQiBUGAgIB4cSISIBJBgICAiAdNG0EBdkGAgIA8ar6SvCISQQ12QYD4AXEgEkH/H3FqIAVBgICAeEsbIAJBEHZBgIACcXI7AQAg2AFCAXwi2AEg2wFSDQALCyDZAUIBfCLZASDeAVINAAsMAwtBvMMCKAIAEDAaIARBmyE2ArgBIARByzc2ArQBIARB5CY2ArABQbjDAigCAEHL5AAgBEGwAWoQMQxuC0G8wwIoAgAQMBogBEGbITYCSCAEQeY4NgJEIARB5CY2AkBBuMMCKAIAQcvkACAEQUBrEDEMbQsgBSkDECLYASADKQMQUg0LIAUpAxgi2gEgAykDGFINCyAFKQMgItkBIAMpAyBSDQsgBSkDKCLbASADKQMoUg0LINgBIAEpAxBSDQsg2gEgASkDGFINCyDZASABKQMgUg0LINsBIAEpAyhSDQsCQCAAKAIADgMBAAEACyAFKAIwIAJBJGwiB0GYmwFqKAIARw0MIAMoAjBBBEcNDSABKAIwIgogASgCNCIISw0OIAggASgCOCILSw0PIAsgASgCPCIPSw0QQcz/AyACdkEBcUUNESADKAIADRIg2wEg2QEg2gF+ItkBfqciAiAAKAIIIgZqQQFrIAZtIgYgACgCBCIMbCIJIAYgCWoiBiACIAIgBkobIg5ODQAgAygCPCEQIAMoAjghESADKAI0IRIgBSgCPCETIAUoAjghFiAFKAI0IRUgB0GgmwFqKAIAIRggASgCACINQSRsQaSbAWooAgAhGiAAKAIQIgAgDCDYAaciB0EQamwiGUECdGohBiAHQQNxIQwgB0F8cSECIAdBAnQhFCAHIApsIRdBsYAcIA12QQFxIRsgACAHIBlqQQJ0aiEZIAmsIdgBIAdBCEkhHANAIAMoAtABIQogASgC0AEhpgEgBSgC0AEgFiDYASDYASDZAX8i2wHEINkBfn0i3AEg2gF/It0BpyIAbCATINsBpyIJbGogFSDcASDaASDdAX59pyINbGpqIAYgByAYEQUAIKYBIdcBIAAgC2wgCSAPbGogCCANbGohpQECQCAHQQBMDQAgCiANIBJsIh4gACARbCIjIAkgEGwiJWpqaiEJQQAhDUEAIQACQCAcDQAgCSAZSSAKIBRqIB4gJWogI2pqIAZLcQ0AA0AgBiAAQQJ0IgpqIh4gCSAKav0AAgAgHv0AAgD95AH9CwIAIABBBGoiACACRw0ACyACIgAgB0YNAQsgAEF/cyAHaiGkASAMBEADQCAGIABBAnQiHmoiIyAJIB5qKgIAICMqAgCSOAIAIABBAWohACANQQFqIg0gDEcNAAsLIKQBQQNJDQADQCAGIABBAnQiCmoiDSAJIApqKgIAIA0qAgCSOAIAIAYgCkEEaiINaiIeIAkgDWoqAgAgHioCAJI4AgAgBiAKQQhqIg1qIh4gCSANaioCACAeKgIAkjgCACAGIApBDGoiCmoiDSAJIApqKgIAIA0qAgCSOAIAIABBBGoiACAHRw0ACwsg1wEgpQFqIQACQCAbRQRAIAYgACAHIBoRBQAMAQsgACAGIBf8CgAACyAOINgBQgF8ItgBp0cNAAsLIARBgANqJAAMEwtBvMMCKAIAEDAaIARBms8ANgI4IARBujY2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMQxqC0G8wwIoAgAQMBogBEHd0AA2AiggBEHHNjYCJCAEQeQmNgIgQbjDAigCAEHL5AAgBEEgahAxDGkLQbzDAigCABAwGiAEQbbRADYCGCAEQcg2NgIUIARB5CY2AhBBuMMCKAIAQcvkACAEQRBqEDEMaAtBvMMCKAIAEDAaIARB2M4ANgKYASAEQdQ3NgKUASAEQeQmNgKQAUG4wwIoAgBBy+QAIARBkAFqEDEMZwtBvMMCKAIAEDAaIARBvTk2AogBIARB4zc2AoQBIARB5CY2AoABQbjDAigCAEHL5AAgBEGAAWoQMQxmC0G8wwIoAgAQMBogBEHD0gA2AnggBEHlNzYCdCAEQeQmNgJwQbjDAigCAEHL5AAgBEHwAGoQMQxlC0G8wwIoAgAQMBogBEGX0wA2AmggBEHmNzYCZCAEQeQmNgJgQbjDAigCAEHL5AAgBEHgAGoQMQxkC0G8wwIoAgAQMBogBEHYzgA2AvgBIARBhzc2AvQBIARB5CY2AvABQbjDAigCAEHL5AAgBEHwAWoQMQxjC0G8wwIoAgAQMBogBEHD0gA2AugBIARBnDc2AuQBIARB5CY2AuABQbjDAigCAEHL5AAgBEHgAWoQMQxiC0G8wwIoAgAQMBogBEGX0wA2AsgBIARBnzc2AsQBIARB5CY2AsABQbjDAigCAEHL5AAgBEHAAWoQMQxhC0G8wwIoAgAQMBogBEHYzgA2AvgCIARBijg2AvQCIARB5CY2AvACQbjDAigCAEHL5AAgBEHwAmoQMQxgC0G8wwIoAgAQMBogBEHX1AA2AugCIARBnTg2AuQCIARB5CY2AuACQbjDAigCAEHL5AAgBEHgAmoQMQxfC0G8wwIoAgAQMBogBEGJ0QA2AtgCIARBnjg2AtQCIARB5CY2AtACQbjDAigCAEHL5AAgBEHQAmoQMQxeC0G8wwIoAgAQMBogBEHOPTYCiAIgBEGhODYChAIgBEHkJjYCgAJBuMMCKAIAQcvkACAEQYACahAxDF0LQbzDAigCABAwGiAEQcY7NgKYAiAEQaI4NgKUAiAEQeQmNgKQAkG4wwIoAgBBy+QAIARBkAJqEDEMXAtBvMMCKAIAEDAaIARBsTo2AqgCIARBozg2AqQCIARB5CY2AqACQbjDAigCAEHL5AAgBEGgAmoQMQxbC0G8wwIoAgAQMBogBEG51AA2ArgCIARBpTg2ArQCIARB5CY2ArACQbjDAigCAEHL5AAgBEGwAmoQMQxaC0G8wwIoAgAQMBogBEGlPDYCyAIgBEGmODYCxAIgBEHkJjYCwAJBuMMCKAIAQcvkACAEQcACahAxDFkLQbzDAigCABAwGiAEQZshNgIIIARB+Dg2AgQgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMQxYCwxDCyABKAKMASEDIAEoApABIQUjAEGQA2siBCQAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAIAMoAgAiAkEGa0EJSQ0AAkACQCACDgQAAQICGwsgAykDECLYASABKQMQUg0DIAMpAxgi2gEgASkDGFINAyADKQMgItkBIAEpAyBSDQMgAykDKCLbASABKQMoUg0DIAUpAxBCAVINBCAFKQMYQgFSDQQgBSkDIEIBUg0EIAUpAyhCAVINBAJAIAAoAgAOAwMAAwALIAEoAjBBBEcNBSADKAIwQQRHDQYg2wEg2QEg2gF+ItkBfqciAiAAKAIIIgdqQQFrIAdtIgcgACgCBGwiACAAIAdqIgcgAiACIAdKGyIHTg0CINgBpyICQQBMDQIgASgCPCEIIAEoAjghCyABKAI0IQwgAygCPCENIAMoAjghDyADKAI0IQ4gBSgC0AEhECADKALQASEGIAEoAtABIQkgAkEDcSEKIAJBfHEhASAHrCHbASAArCHYASACQQxJIREDQCAGIA0g2AEg2QF/ItwBpyIAbCISaiAPINgBINwBxCDZAX59ItwBINoBfyLdAaciBWwiE2ogDiDcASDaASDdAX59pyIHbCIWaiEDIAcgDGwiFSAFIAtsIhggCSAAIAhsIhpqamohBSAQKgIAIbICQQAhB0EAIQACQAJAIBENACAJIBVqIBpqIBhqIAYgFmogEmogE2prQRBJDQAgsgL9EyGSAgNAIAUgAEECdCISaiCSAiADIBJq/QACAP3kAf0LAgAgAEEEaiIAIAFHDQALIAEiACACRg0BCyAAQX9zIAJqIacBIAoEQANAIAUgAEECdCITaiCyAiADIBNqKgIAkjgCACAAQQFqIQAgB0EBaiIHIApHDQALCyCnAUEDSQ0AA0AgBSAAQQJ0IgdqILICIAMgB2oqAgCSOAIAIAUgB0EEaiISaiCyAiADIBJqKgIAkjgCACAFIAdBCGoiEmogsgIgAyASaioCAJI4AgAgBSAHQQxqIgdqILICIAMgB2oqAgCSOAIAIABBBGoiACACRw0ACwsg2AFCAXwi2AEg2wFSDQALDAILAkACQAJAIAUoAgAOAgEAAgsgAykDECLbASABKQMQUg0IIAMpAxgi3AEgASkDGFINCCADKQMgItgBIAEpAyBSDQggAykDKCLaASABKQMoUg0IIAUpAxBCAVINCSAFKQMYQgFSDQkgBSkDIEIBUg0JIAUpAyhCAVINCQJAIAAoAgAOAwQABAALIAEoAgBBAUcNCiABKAIwQQJHDQsgAygCMEECRw0MINoBINgBINwBfiLdAX6nIgIgACgCCCIHakEBayAHbSIHIAAoAgRsIgAgACAHaiIHIAIgAiAHShsiB04NAyDbAUIAVw0DIAEoAjwhBiABKAI4IQkgASgCNCEKIAMoAjwhCCADKAI4IQsgAygCNCEMIAMoAtABIQIgASgC0AEhASDbAUJ8gyHaASAFKALQAS8BAEECdEGQ1gRqKgIAIrIC/RMhkwIgB6wh3gEgAKwh2QEDQCACIAgg2QEg3QF/ItgBpyIDbCIFaiALINkBINgBxCDdAX59ItgBINwBfyLfAaciB2wiDWogDCDYASDcASDfAX59pyIPbCIOaiEAIAogD2wiDyAHIAlsIgcgASADIAZsIhBqamohA0IAIdgBAkACQCDbAUIEVA0AIAEgD2ogEGogB2ogAiAOaiAFaiANamtBCEkNAANAIAMg2AGnQQF0IgVq/QwAfgAAAH4AAAB+AAAAfgAAIJMCIAAgBWr9BAEAIpIC/RsAQQJ0QZDWBGr9CQIAIJIC/RsBQQJ0QZDWBGoqAgD9IAEgkgL9GwJBAnRBkNYEaioCAP0gAiCSAv0bA0ECdEGQ1gRqKgIA/SAD/eQBIpIC/eAB/QwAAIB3AACAdwAAgHcAAIB3/eYB/QwAAIAIAACACAAAgAgAAIAI/eYBIJICQQH9qwEilAL9DAAAAP8AAAD/AAAA/wAAAP/9Tv0MAAAAcQAAAHEAAABxAAAAcf25AUEB/a0B/QwAAIAHAACABwAAgAcAAIAH/a4B/eQBIpUCQQ39rQH9DAB8AAAAfAAAAHwAAAB8AAD9TiCVAv0M/w8AAP8PAAD/DwAA/w8AAP1O/a4BIJQC/QwAAAD/AAAA/wAAAP8AAAD//Tz9UiCSAkEQ/a0B/QwAgAAAAIAAAACAAAAAgAAA/U79UCCSAv0NAAEEBQgJDA0AAQABAAEAAf1bAQAAINgBQgR8ItgBINoBUg0ACyDaASLYASDbAVENAQsDQCADINgBp0EBdCIFakGA/AEgsgIgACAFai8BAEECdEGQ1gRqKgIAkiKzAotDAACAd5RDAACACJRBgICAiAcgswK8IgVBAXQiB0GAgIB4cSINIA1BgICAiAdNG0EBdkGAgIA8ar6SvCINQQ12QYD4AXEgDUH/H3FqIAdBgICAeEsbIAVBEHZBgIACcXI7AQAg2AFCAXwi2AEg2wFSDQALCyDZAUIBfCLZASDeAVINAAsMAwsgAykDECLbASABKQMQUg0MIAMpAxgi3AEgASkDGFINDCADKQMgItgBIAEpAyBSDQwgAykDKCLaASABKQMoUg0MIAUpAxBCAVINDSAFKQMYQgFSDQ0gBSkDIEIBUg0NIAUpAyhCAVINDQJAIAAoAgAOAwMAAwALIAEoAgBBAUcNDiABKAIwQQJHDQ8gAygCMEECRw0QINoBINgBINwBfiLdAX6nIgIgACgCCCIHakEBayAHbSIHIAAoAgRsIgAgACAHaiIHIAIgAiAHShsiB04NAiDbAUIAVw0CIAEoAjwhBiABKAI4IQkgASgCNCEKIAMoAjwhCCADKAI4IQsgAygCNCEMIAMoAtABIQIgASgC0AEhASDbAUJ8gyHaASAFKALQASoCACKyAv0TIZMCIAesId4BIACsIdkBA0AgAiAIINkBIN0BfyLYAaciA2wiBWogCyDZASDYAcQg3QF+fSLYASDcAX8i3wGnIgdsIg1qIAwg2AEg3AEg3wF+faciD2wiDmohACAKIA9sIg8gByAJbCIHIAEgAyAGbCIQampqIQNCACHYAQJAAkAg2wFCBFQNACABIA9qIBBqIAdqIAIgDmogBWogDWprQQhJDQADQCADINgBp0EBdCIFav0MAH4AAAB+AAAAfgAAAH4AACCTAiAAIAVq/QQBACKSAv0bAEECdEGQ1gRq/QkCACCSAv0bAUECdEGQ1gRqKgIA/SABIJIC/RsCQQJ0QZDWBGoqAgD9IAIgkgL9GwNBAnRBkNYEaioCAP0gA/3kASKSAv3gAf0MAACAdwAAgHcAAIB3AACAd/3mAf0MAACACAAAgAgAAIAIAACACP3mASCSAkEB/asBIpQC/QwAAAD/AAAA/wAAAP8AAAD//U79DAAAAHEAAABxAAAAcQAAAHH9uQFBAf2tAf0MAACABwAAgAcAAIAHAACAB/2uAf3kASKVAkEN/a0B/QwAfAAAAHwAAAB8AAAAfAAA/U4glQL9DP8PAAD/DwAA/w8AAP8PAAD9Tv2uASCUAv0MAAAA/wAAAP8AAAD/AAAA//08/VIgkgJBEP2tAf0MAIAAAACAAAAAgAAAAIAAAP1O/VAgkgL9DQABBAUICQwNAAEAAQABAAH9WwEAACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2wFRDQELA0AgAyDYAadBAXQiBWpBgPwBILICIAAgBWovAQBBAnRBkNYEaioCAJIiswKLQwAAgHeUQwAAgAiUQYCAgIgHILMCvCIFQQF0IgdBgICAeHEiDSANQYCAgIgHTRtBAXZBgICAPGq+krwiDUENdkGA+AFxIA1B/x9xaiAHQYCAgHhLGyAFQRB2QYCAAnFyOwEAINgBQgF8ItgBINsBUg0ACwsg2QFCAXwi2QEg3gFSDQALDAILQbzDAigCABAwGiAEQZshNgJYIARB7To2AlQgBEHkJjYCUEG4wwIoAgBBy+QAIARB0ABqEDEMcQsgAykDECLYASABKQMQUg0PIAMpAxgi2gEgASkDGFINDyADKQMgItkBIAEpAyBSDQ8gAykDKCLbASABKQMoUg0PIAUpAxBCAVINECAFKQMYQgFSDRAgBSkDIEIBUg0QIAUpAyhCAVINEAJAIAAoAgAOAwEAAQALIAMoAjAgAkEkbCIHQZibAWooAgBHDREgASgCMCIIIAEoAjQiCUsNEiAJIAEoAjgiCksNEyAKIAEoAjxLDRRBzP8DIAJ2QQFxRQ0VIAEoAgAgAkcNFiAFKAIADRcg2wEg2QEg2gF+ItkBfqciAiAAKAIIIgZqQQFrIAZtIgYgACgCBCIRbCILIAYgC2oiBiACIAIgBkobIgJODQAgAygCPCEMIAMoAjghDSADKAI0IQ8gB0GkmwFqKAIAIQ4gB0GgmwFqKAIAIRAgACgCECARINgBpyIHQRBqbEECdGohBiACrCHbASALrCHYASAHQQBMBEADQCABKALQASEAIAMoAtABIA0g2AEg2AEg2QF/ItwBxCDZAX59It0BINoBfyLeAaciAmwgDCDcAaciBWxqIA8g3QEg2gEg3gF+faciC2xqaiAGIAcgEBEFACAGIAAgAiAKbCAFIAhsaiAJIAtsamogByAOEQUAINgBQgF8ItgBINsBUg0ADAILAAsgB0F8cSECIAUoAtABKgIAIrIC/RMhkgIgB0EESSEFA0AgASgC0AEhCyADKALQASANINgBINgBINkBfyLcAcQg2QF+fSLdASDaAX8i3gGnIgBsIAwg3AGnIhFsaiAPIN0BINoBIN4Bfn2nIhJsamogBiAHIBARBQAgACAKbCAIIBFsaiERQQAhAAJAIAVFBEADQCAGIABBAnRqIhMgkgIgE/0AAgD95AH9CwIAIABBBGoiACACRw0ACyACIgAgB0YNAQsDQCAGIABBAnRqIhMgsgIgEyoCAJI4AgAgAEEBaiIAIAdHDQALCyAGIAsgESAJIBJsamogByAOEQUAINgBQgF8ItgBINsBUg0ACwsgBEGQA2okAAwYC0G8wwIoAgAQMBogBEG5zwA2AkggBEGEOTYCRCAEQeQmNgJAQbjDAigCAEHL5AAgBEFAaxAxDG4LQbzDAigCABAwGiAEQdTYADYCOCAEQYU5NgI0IARB5CY2AjBBuMMCKAIAQcvkACAEQTBqEDEMbQtBvMMCKAIAEDAaIARB3dAANgIoIARBkjk2AiQgBEHkJjYCIEG4wwIoAgBBy+QAIARBIGoQMQxsC0G8wwIoAgAQMBogBEG20QA2AhggBEGTOTYCFCAEQeQmNgIQQbjDAigCAEHL5AAgBEEQahAxDGsLQbzDAigCABAwGiAEQbnPADYCqAEgBEHqOTYCpAEgBEHkJjYCoAFBuMMCKAIAQcvkACAEQaABahAxDGoLQbzDAigCABAwGiAEQdTYADYCmAEgBEHrOTYClAEgBEHkJjYCkAFBuMMCKAIAQcvkACAEQZABahAxDGkLQbzDAigCABAwGiAEQb05NgKIASAEQf05NgKEASAEQeQmNgKAAUG4wwIoAgBBy+QAIARBgAFqEDEMaAtBvMMCKAIAEDAaIARBw9IANgJ4IARB/zk2AnQgBEHkJjYCcEG4wwIoAgBBy+QAIARB8ABqEDEMZwtBvMMCKAIAEDAaIARBl9MANgJoIARBgDo2AmQgBEHkJjYCYEG4wwIoAgBBy+QAIARB4ABqEDEMZgtBvMMCKAIAEDAaIARBuc8ANgL4ASAEQbg5NgL0ASAEQeQmNgLwAUG4wwIoAgBBy+QAIARB8AFqEDEMZQtBvMMCKAIAEDAaIARB1NgANgLoASAEQbk5NgLkASAEQeQmNgLgAUG4wwIoAgBBy+QAIARB4AFqEDEMZAtBvMMCKAIAEDAaIARBvTk2AtgBIARByzk2AtQBIARB5CY2AtABQbjDAigCAEHL5AAgBEHQAWoQMQxjC0G8wwIoAgAQMBogBEHD0gA2AsgBIARBzTk2AsQBIARB5CY2AsABQbjDAigCAEHL5AAgBEHAAWoQMQxiC0G8wwIoAgAQMBogBEGX0wA2ArgBIARBzjk2ArQBIARB5CY2ArABQbjDAigCAEHL5AAgBEGwAWoQMQxhC0G8wwIoAgAQMBogBEG5zwA2AogDIARBnDo2AoQDIARB5CY2AoADQbjDAigCAEHL5AAgBEGAA2oQMQxgC0G8wwIoAgAQMBogBEHU2AA2AvgCIARBnTo2AvQCIARB5CY2AvACQbjDAigCAEHL5AAgBEHwAmoQMQxfC0G8wwIoAgAQMBogBEHX1AA2AugCIARBsjo2AuQCIARB5CY2AuACQbjDAigCAEHL5AAgBEHgAmoQMQxeC0G8wwIoAgAQMBogBEHOPTYCiAIgBEG1OjYChAIgBEHkJjYCgAJBuMMCKAIAQcvkACAEQYACahAxDF0LQbzDAigCABAwGiAEQcY7NgKYAiAEQbY6NgKUAiAEQeQmNgKQAkG4wwIoAgBBy+QAIARBkAJqEDEMXAtBvMMCKAIAEDAaIARBsTo2AqgCIARBtzo2AqQCIARB5CY2AqACQbjDAigCAEHL5AAgBEGgAmoQMQxbC0G8wwIoAgAQMBogBEG51AA2ArgCIARBuTo2ArQCIARB5CY2ArACQbjDAigCAEHL5AAgBEGwAmoQMQxaC0G8wwIoAgAQMBogBEGuIjYC2AIgBEG6OjYC1AIgBEHkJjYC0AJBuMMCKAIAQcvkACAEQdACahAxDFkLQbzDAigCABAwGiAEQaU8NgLIAiAEQbs6NgLEAiAEQeQmNgLAAkG4wwIoAgBBy+QAIARBwAJqEDEMWAtBvMMCKAIAEDAaIARBmyE2AgggBEGAOzYCBCAEQeQmNgIAQbjDAigCAEHL5AAgBBAxDFcLDEILIAEoAowBIQQgASgCkAEhAiMAQeAAayIDJAACQAJAAkACQAJAAkAgBCgCAEUEQCAEKQMQItkBIAEpAxBSDQEgBCkDGCLYASABKQMYUg0BIAQpAyAi2gEgASkDIFINASAEKQMoItsBIAEpAyhSDQEgASgCMCIFIAEoAgAiCUEkbCIHQZibAWooAgBHDQIgASgCNCIKrSLcASDZASAFrX4i3gEgB0GUmwFqKAIAIgisf1INAiABKAI4Ig+tIt0BINgBINwBflINAiABKAI8Ig6tIt8BINoBIN0BflINAiAEKAIwQQRHDQIgBDUCNCLgASDZAUIChlINAiAENQI4IuEBINgBIOABflINAiAENQI8INoBIOEBflINAiABKAJQIQcgASgCTCELIAEoAkghDCABKAJEIQ0CQCABKAJURQRAIAAoAgAiBg0BIAEoAtABIAQoAtABIAlBEGtBcU0EfyAKINgBp0EBa2wgBSDZAadsaiAPINoBp0EBa2xqIA4g2wGnQQFrbGoFINoBQgF9IN0BfiDYAUIBfSDcAX58INsBQgF9IN8Bfnwg3gEgCK1/fKcL/AoAAAsgACgCACEGCwJAAkAgBg4DAQABAAsgB60gDa0gAikDGCLYAUIBfSLaAUIAINgBINoBWht+fCAMrSACKQMgItoBQgF9ItkBQgAg2QEg2gFYG358IAutIAIpAygi2QFCAX0i2wFCACDZASDbAVobfnwgBCgCACIFQSRsQZibAWooAgAiCa0gAikDECLbAUIBfSLcAUIAINsBINwBWht+fCHcASACKAI8IQ4gAigCOCEQIAIoAjQhESABKQMoId0BIAEpAxAh3gEgASkDICHfASABKQMYIeABIAIoAjAhqAEgACgCCCEGIAAoAgQhCCDcAQJ/IAEoAgAiAEEQa0FxTQRAIABBJGxBmJsBaigCACABKAIwIN4Bp0EBa2xqIAEoAjQg4AGnQQFrbGogASgCOCDfAadBAWtsaiABKAI8IN0Bp0EBa2xqDAELIAE1AjQg4AFCAX1+IN4BIAE1AjB+IABBJGxBlJsBajUCAH98IAE1Ajgg3wFCAX1+fCABNQI8IN0BQgF9fnynC61ZDQQgBCkDICHdASAEKQMYId4BINwBAn8gBUEQa0FxTQRAIAQoAjAgBCgCEEEBa2wgCWogBCgCNCDeAadBAWtsaiAEKAI4IN0Bp0EBa2xqIAQoAjwgBCgCKEEBa2xqDAELIAQ1AjQg3gFCAX1+IAQpAxAgBDUCMH4gBUEkbEGUmwFqNQIAf3wgBDUCOCDdAUIBfX58IAQ1AjwgBCkDKEIBfX58pwutWQ0FIKgBQQRHDQYgBiDZASDYASDaAX4i3AF+pyIAakEBayAGbSIFIAhsIgYgBSAGaiIFIAAgACAFShsiAE4NACDbAaciBUEATA0AIAIoAtABIQ9BACAFayESIAVBAXEhEyAFQXxxIQIgByAEKALQASIWaiEVIAcgASgC0AEiGGohGiAArCHZASAGrCHaASAFQRBJIRkDQCAPIA4g2gEg3AF/ItsBpyIAbCIUaiAQINoBINsBxCDcAX59ItsBINgBfyLdAaciBGwiF2ogESDbASDYASDdAX59pyIGbCIbaiEBIAYgDWwiCSAEIAxsIgogFiAAIAtsIghqamogB2ohBCAIIBhqIApqIAlqIAdqIQZBACEAAkACQCAZDQAgCSAaaiAIaiAKaiIcIAkgFWogCGogCmprQRBJDQAgHCAPIBtqIBRqIBdqa0EQSQ0AA0AgBiAAQQJ0IglqIAQgCWr9AAIAIAEgCWr9AAIA/eQB/QsCACAAQQRqIgAgAkcNAAsgAiIAIAVGDQELIABBf3MhqQEgEwRAIAYgAEECdCIJaiAEIAlqKgIAIAEgCWoqAgCSOAIAIABBAXIhAAsgqQEgEkYNAANAIAYgAEECdCIJaiAEIAlqKgIAIAEgCWoqAgCSOAIAIAYgCUEEaiIJaiAEIAlqKgIAIAEgCWoqAgCSOAIAIABBAmoiACAFRw0ACwsg2gFCAXwi2gEg2QFSDQALCyADQeAAaiQADAYLQbzDAigCABAwGiADQZshNgJYIANB7zs2AlQgA0HkJjYCUEG4wwIoAgBBy+QAIANB0ABqEDEMWwtBvMMCKAIAEDAaIANBuc8ANgJIIANBjDs2AkQgA0HkJjYCQEG4wwIoAgBBy+QAIANBQGsQMQxaC0G8wwIoAgAQMBogA0HE2QA2AjggA0GNOzYCNCADQeQmNgIwQbjDAigCAEHL5AAgA0EwahAxDFkLQbzDAigCABAwGiADQc/MADYCKCADQbU7NgIkIANB5CY2AiBBuMMCKAIAQcvkACADQSBqEDEMWAtBvMMCKAIAEDAaIANBpNoANgIYIANBtjs2AhQgA0HkJjYCEEG4wwIoAgBBy+QAIANBEGoQMQxXC0G8wwIoAgAQMBogA0GJ0QA2AgggA0G4OzYCBCADQeQmNgIAQbjDAigCAEHL5AAgAxAxDFYLDEELIAEoAowBIQQgASgCkAEhAyMAQTBrIgIkAAJAAkACQCAEKAIARQRAAkACQCAAKAIADgMBAAEACyABKAIwQQRHDQIgBCgCMEEERw0DIAQpAyggBCkDICAEKQMYfn4i2AGnIQAgASgCPCEJIAEoAjghCiABKAI0IQggASkDICHaASABKQMYIdwBIAMoAjwhCyADKAI4IQwgAygCNCENIAQoAjwhDyAEKAI4IQ4gBCgCNCEQIAEpAxAh2wEgAygCMCIFQQRHBEAgAEEATA0BINsBQgBXDQEg2gEg3AF+Id4BIAVBAUYg2wFCE1ZxIQYg2wFCAYMh3wEg2wFCfIMh2gEg2AFC/////w+DIeABINsBpyIAIAMoAtABIhFqQQNqIRIgBCgC0AEiEyAAQQJ0IgBqIRYgASgC0AEiFSAAaiEYA0AgESALIN0BIN4BfyLYAaciAWwiGmogDCDdASDYAcQg3gF+fSLYASDcAX8i2QGnIgNsIhlqIA0g2AEg2QEg3AF+faciB2wiFGohACAHIBBsIhcgAyAObCIbIBMgASAPbCIcampqIQQgByAIbCIHIAMgCmwiAyAVIAEgCWwiHWpqaiEBQgAh2AECQAJAIAZFDQAgASAWIBcgG2ogHGpqSSAEIBggAyAHaiAdamoiA0lxDQAgACADSSABIBIgFCAZaiAaampJcQ0A/QwAAAAAAQAAAAIAAAADAAAAIZICA0AgASDYAadBAnQiA2ogAyAEav0AAgAgACCSAv0bAGr9CQIAIAAgkgL9GwFqKgIA/SABIAAgkgL9GwJqKgIA/SACIAAgkgL9GwNqKgIA/SAD/eUB/QsCACCSAv0MBAAAAAQAAAAEAAAABAAAAP2uASGSAiDYAUIEfCLYASDaAVINAAsg2gEi2AEg2wFRDQELINgBQgGEIdkBIN8BpwRAIAEg2AGnIgNBAnQiB2ogBCAHaioCACAAIAMgBWxqKgIAkzgCACDZASHYAQsg2QEg2wFRDQADQCABINgBpyIDQQJ0IgdqIAQgB2oqAgAgACADIAVsaioCAJM4AgAgASADQQFqIgNBAnQiB2ogBCAHaioCACAAIAMgBWxqKgIAkzgCACDYAUICfCLYASDbAVINAAsLIN0BQgF8It0BIOABUg0ACwwBCyAAQQBMDQAg2wGnIgVBAEwNACDaASDcAX4h2gEgAygC0AEhESAEKALQASESIAEoAtABIRNBACAFayEWIAVBAXEhFSAFQXxxIQEg2AFC/////w+DIdkBQgAh2AEgBUEQSSEYA0AgESALINgBINoBfyLbAaciAGwiGmogDCDYASDbAcQg2gF+fSLbASDcAX8i3QGnIgdsIhlqIA0g2wEg3AEg3QF+faciBmwiFGohBCAGIBBsIhcgByAObCIbIBIgACAPbCIcampqIQMgBiAIbCIGIAcgCmwiHSATIAAgCWwiH2pqaiEHQQAhAAJAAkAgGA0AIAYgE2ogH2ogHWoiBiASIBdqIBxqIBtqa0EQSQ0AIAYgESAUaiAaaiAZamtBEEkNAANAIAcgAEECdCIGaiADIAZq/QACACAEIAZq/QACAP3lAf0LAgAgAEEEaiIAIAFHDQALIAEiACAFRg0BCyAAQX9zIRogFQRAIAcgAEECdCIGaiADIAZqKgIAIAQgBmoqAgCTOAIAIABBAXIhAAsgFiAaRg0AA0AgByAAQQJ0IgZqIAMgBmoqAgAgBCAGaioCAJM4AgAgByAGQQRqIgZqIAMgBmoqAgAgBCAGaioCAJM4AgAgAEECaiIAIAVHDQALCyDYAUIBfCLYASDZAVINAAsLIAJBMGokAAwDC0G8wwIoAgAQMBogAkGbITYCKCACQb48NgIkDEcLQbzDAigCABAwGiACQd3QADYCGCACQYY8NgIUDEcLQbzDAigCABAwGiACQbbRADYCCCACQYc8NgIEDEcLDEALIAEoAowBIQQgASgCkAEhAyMAQdAAayICJAACQAJAAkACQCADKAIARQRAIAQoAgBFBEAgBCkDECLbASDbASADKQMQItkBfyLfASDZAX5SDQIgBCkDGCLcASADKQMYIuABgUIAUg0CIAQpAyAi2AEgAykDICLhAYFCAFINAiAEKQMoItoBIAMpAygi4gGBQgBSDQIg2wEgASkDEFINAiDcASABKQMYUg0CINgBIAEpAyBSDQIg2gEgASkDKFINAgJAAkAgACgCAA4DAQABAAsgASgCMEEERw0EIAQoAjBBBEcNBSDaASDYASDcAX4i3QF+Id4BIAEoAjwhCSABKAI4IQogASgCNCEIIAMoAjwhCyADKAI4IQwgAygCNCENIAQoAjwhDyAEKAI4IQ4gBCgCNCEQIAAoAgghByAANAIEIdoBIAMoAjAiAEEERgRAINoBIN4BWQ0BIN8BQgBXDQEg2QGnIgVBAEwNASAHrCHjASADKALQASERIAQoAtABIRIgASgC0AEhE0EAIAVrIRYgBUEBcSEVIAVBfHEhASDZAUIChkL8////D4Mh5AEgBUEISSEYA0AgEiAPINoBIN0BfyLYAaciAGxqIgMgDiDaASDYASDdAX59IuUBINwBfyLbAaciBGwiB2ogECDlASDbASDcAX59IuUBpyIGbCIaaiEZIAYgCGwiBiATIAAgCWxqIgAgBCAKbCIUamohFyARIAsg2AEg4gGBp2xqIAwg2wEg4QGBp2xqIA0g5QEg4AGBp2xqIQQgAyAaaiAHaiEaIAAgBmogFGohFEIAIdgBA0AgGSDYASDZAX6nQQJ0IgBqIQMgACAXaiEHQQAhAAJAAkAgGA0AIBQg2AEg5AF+pyIGaiIbIAYgGmprQRBJDQAgGyAEa0EQSQ0AA0AgByAAQQJ0IgZqIAMgBmr9AAIAIAQgBmr9AAIA/eYB/QsCACAAQQRqIgAgAUcNAAsgASIAIAVGDQELIABBf3MhGyAVBEAgByAAQQJ0IgZqIAMgBmoqAgAgBCAGaioCAJQ4AgAgAEEBciEACyAWIBtGDQADQCAHIABBAnQiBmogAyAGaioCACAEIAZqKgIAlDgCACAHIAZBBGoiBmogAyAGaioCACAEIAZqKgIAlDgCACAAQQJqIgAgBUcNAAsLINgBQgF8ItgBIN8BUg0ACyDaASDjAXwi2gEg3gFTDQALDAELINoBIN4BWQ0AINsBQgBXDQAgB6wh4wEgBCgC0AEhBSABKALQASEHINsBQn6DIeQBINsBQgGDIeUBIAMoAtABIQYDQCAFIA8g2gEg3QF/ItgBpyIEbGogDiDaASDYASDdAX59IuYBINwBfyLfAaciA2xqIBAg5gEg3AEg3wF+fSLmAaciEWxqIQEgByAEIAlsaiADIApsaiAIIBFsaiEEIAYgCyDYASDiAYGnbGogDCDfASDhAYGnbGogDSDmASDgAYGnbGohA0IAIdgBQgAh3wEg2wFCAVIEQANAIAQg2AGnQQJ0IhFqIAEgEWoqAgAgAyAAINgBINkBgadsaioCAJQ4AgAgBCDYAUIBhCLmAadBAnQiEWogASARaioCACADIAAg5gEg2QGBp2xqKgIAlDgCACDYAUICfCHYASDfAUICfCLfASDkAVINAAsLIOUBpwRAIAQg2AGnQQJ0IhFqIAEgEWoqAgAgAyAAINgBINkBgadsaioCAJQ4AgALINoBIOMBfCLaASDeAVMNAAsLIAJB0ABqJAAMBQtBvMMCKAIAEDAaIAJBmyE2AjggAkGmPTYCNAxLC0G8wwIoAgAQMBogAkGT3AA2AkggAkGdPTYCRAxLC0G8wwIoAgAQMBogAkGazwA2AiggAkHKPDYCJAxGC0G8wwIoAgAQMBogAkHd0AA2AhggAkHhPDYCFAxGC0G8wwIoAgAQMBogAkG20QA2AgggAkHiPDYCBAxGCww/CyABKAKMASEEIAEoApABIQMjAEFAaiICJAACQAJAAkACQCAEKAIARQRAIAQpAxAi3AEg3AEgAykDECLbAX8i3wEg2wF+Ug0BIAQpAxgi2QEgAykDGCLgAYFCAFINASAEKQMgItgBIAMpAyAi4QGBQgBSDQEgBCkDKCLaASADKQMoIuIBgUIAUg0BINwBIAEpAxBSDQEg2QEgASkDGFINASDYASABKQMgUg0BINoBIAEpAyhSDQECQAJAIAAoAgAOAwEAAQALIAEoAjBBBEcNAyAEKAIwQQRHDQQg2gEg2AEg2QF+It0BfiHeASABKAI8IQkgASgCOCEKIAEoAjQhCCADKAI8IQsgAygCOCEMIAMoAjQhDSAEKAI8IQ8gBCgCOCEOIAQoAjQhECAAKAIIIQcgADQCBCHYASADKAIwIgBBBEYEQCDYASDeAVkNASDfAUIAVw0BINsBpyIFQQBMDQEgB6wh4wEgAygC0AEhESAEKALQASESIAEoAtABIRNBACAFayEWIAVBAXEhFSAFQXxxIQEg2wFCAoZC/P///w+DIeQBIAVBCEkhGANAIBIgDyDYASDdAX8i2gGnIgBsaiIDIA4g2AEg2gEg3QF+fSLlASDZAX8i3AGnIgRsIgdqIBAg5QEg2QEg3AF+fSLlAaciBmwiGmohGSAGIAhsIgYgEyAAIAlsaiIAIAQgCmwiFGpqIRcgESALINoBIOIBgadsaiAMINwBIOEBgadsaiANIOUBIOABgadsaiEEIAMgGmogB2ohGiAAIAZqIBRqIRRCACHaAQNAIBkg2gEg2wF+p0ECdCIAaiEDIAAgF2ohB0EAIQACQAJAIBgNACAUINoBIOQBfqciBmoiGyAGIBpqa0EQSQ0AIBsgBGtBEEkNAANAIAcgAEECdCIGaiADIAZq/QACACAEIAZq/QACAP3nAf0LAgAgAEEEaiIAIAFHDQALIAEiACAFRg0BCyAAQX9zIRsgFQRAIAcgAEECdCIGaiADIAZqKgIAIAQgBmoqAgCVOAIAIABBAXIhAAsgFiAbRg0AA0AgByAAQQJ0IgZqIAMgBmoqAgAgBCAGaioCAJU4AgAgByAGQQRqIgZqIAMgBmoqAgAgBCAGaioCAJU4AgAgAEECaiIAIAVHDQALCyDaAUIBfCLaASDfAVINAAsg2AEg4wF8ItgBIN4BUw0ACwwBCyDYASDeAVkNACDcAUIAVw0AIAesIeMBIAQoAtABIQQgASgC0AEhASADKALQASEDA0AgBCAPINgBIN0BfyLaAaciBWxqIA4g2AEg2gEg3QF+fSLkASDZAX8i3wGnIgdsaiAQIOQBINkBIN8Bfn0i5AGnIgZsaiERIAEgBSAJbGogByAKbGogBiAIbGohBSADIAsg2gEg4gGBp2xqIAwg3wEg4QGBp2xqIA0g5AEg4AGBp2xqIQdCACHaAQNAIAUg2gGnQQJ0IgZqIAYgEWoqAgAgByAAINoBINsBgadsaioCAJU4AgAg2gFCAXwi2gEg3AFSDQALINgBIOMBfCLYASDeAVMNAAsLIAJBQGskAAwEC0G8wwIoAgAQMBogAkGbITYCOCACQYI+NgI0DEkLQbzDAigCABAwGiACQZrPADYCKCACQbI9NgIkDEULQbzDAigCABAwGiACQd3QADYCGCACQb89NgIUDEULQbzDAigCABAwGiACQbbRADYCCCACQcA9NgIEDEULDD4LIAEoAowBIQIjAEEQayIFJAACQCACKAIARQRAAkACQCAAKAIADgMBAAEACyACKQMoIAIpAyAgAikDGH5+pyIKQQBMDQAgAigCECIDQQBMDQAgAigCNCEIIAEoAjQhCyACKALQASEMIAEoAtABIQ0gA0EDcSEJIANBfHEhASADQQRJIQ8DQCAMIAcgCGxqIQIgDSAHIAtsaiEEQQAhAAJAAkAgDw0AIAQgAmtBEEkNAANAIAQgAEECdCIGaiACIAZq/QACACKSAiCSAv3mAf0LAgAgAEEEaiIAIAFHDQALIAEiACADRg0BCyAAQX9zIANqIaoBQQAhBiAJBEADQCAEIABBAnQiEGogAiAQaioCACKyAiCyApQ4AgAgAEEBaiEAIAZBAWoiBiAJRw0ACwsgqgFBA0kNAANAIAQgAEECdCIGaiACIAZqKgIAIrICILIClDgCACAEIAZBBGoiDmogAiAOaioCACKyAiCyApQ4AgAgBCAGQQhqIg5qIAIgDmoqAgAisgIgsgKUOAIAIAQgBkEMaiIGaiACIAZqKgIAIrICILIClDgCACAAQQRqIgAgA0cNAAsLIAdBAWoiByAKRw0ACwsgBUEQaiQADAELQbzDAigCABAwGiAFQZshNgIIIAVBrD42AgQMRwsMPQsgASgCjAEhAiMAQRBrIgUkAAJAIAIoAgBFBEACQAJAIAAoAgAOAwEAAQALIAIpAyggAikDICACKQMYfn6nIgpBAEwNACACKAIQIgNBAEwNACACKAI0IQggASgCNCELIAIoAtABIQwgASgC0AEhDSADQQNxIQkgA0F8cSEBIANBBEkhDwNAIAwgByAIbGohAiANIAcgC2xqIQRBACEAAkACQCAPDQAgBCACa0EQSQ0AA0AgBCAAQQJ0IgZqIAIgBmr9AAIA/eMB/QsCACAAQQRqIgAgAUcNAAsgASIAIANGDQELIABBf3MgA2ohqwFBACEGIAkEQANAIAQgAEECdCIQaiACIBBqKgIAkTgCACAAQQFqIQAgBkEBaiIGIAlHDQALCyCrAUEDSQ0AA0AgBCAAQQJ0IgZqIAIgBmoqAgCROAIAIAQgBkEEaiIOaiACIA5qKgIAkTgCACAEIAZBCGoiDmogAiAOaioCAJE4AgAgBCAGQQxqIgZqIAIgBmoqAgCROAIAIABBBGoiACADRw0ACwsgB0EBaiIHIApHDQALCyAFQRBqJAAMAQtBvMMCKAIAEDAaIAVBmyE2AgggBUHWPjYCBAxGCww8CyABKAKMASEEIwBB0ABrIgIkAAJAAkACQAJAAkAgBCgCAEUEQCAAKAIEDQEgBCkDECLYASABKQMQUg0CIAQpAxgi2gEgASkDGFINAiAEKQMgItkBIAEpAyBSDQIgBCkDKCLbASABKQMoUg0CAkACQCAAKAIADgMBAAEACyABKAIwQQRHDQQgBCgCMEEERw0FINkBINoBfiDbAX6nIgpBAEwNACDYAaciBUEATA0AIAQoAjQhCCABKAI0IQsgBCgC0AEhDCABKALQASENIAVBA3EhCSAFQXxxIQEgBUEESSEPA0AgDCAHIAhsaiEEIA0gByALbGohA0EAIQACQAJAIA8NACADIARrQRBJDQADQCADIABBAnQiBmogBCAGav0AAgAikgL9HwAQV/0TIJIC/R8BEFf9IAEgkgL9HwIQV/0gAiCSAv0fAxBX/SAD/QsCACAAQQRqIgAgAUcNAAsgASIAIAVGDQELIABBf3MgBWohrAFBACEGIAkEQANAIAMgAEECdCIQaiAEIBBqKgIAEFc4AgAgAEEBaiEAIAZBAWoiBiAJRw0ACwsgrAFBA0kNAANAIAMgAEECdCIGaiAEIAZqKgIAEFc4AgAgAyAGQQRqIg5qIAQgDmoqAgAQVzgCACADIAZBCGoiDmogBCAOaioCABBXOAIAIAMgBkEMaiIGaiAEIAZqKgIAEFc4AgAgAEEEaiIAIAVHDQALCyAHQQFqIgcgCkcNAAsLIAJB0ABqJAAMBQtBvMMCKAIAEDAaIAJBmyE2AkggAkGAPzYCRAxIC0G8wwIoAgAQMBogAkH9wAA2AjggAkHhPjYCNAxGC0G8wwIoAgAQMBogAkG5zwA2AiggAkHiPjYCJAxCC0G8wwIoAgAQMBogAkHizwA2AhggAkHrPjYCFAxCC0G8wwIoAgAQMBogAkH+zwA2AgggAkHsPjYCBAxCCww7CyABKAKMASECIwBBEGsiBCQAAkACQAJAAkACQCACKAIADgIAAQMLAkAgACgCAA4DAgACAAsCQCACKQMoItsBQgBXDQAgAikDICLcAUIAVw0AIAIpAxgi3QFCAFcNACACKAIQIgBBAEwNACACKAI8IQkgAigCOCEKIAIoAjQhCCACKALQASELIABBfHEhDCAAQQNxIQYgAEEESSENA0AgCyAJINkBp2xqIQ9CACHYAQNAIA8gCiDYAadsaiEOQgAh2gEDQCAOIAgg2gGnbGohAkQAAAAAAAAAACG4AkEAIQNBACEAQQAhBSANRQRAA0AguAIgAiAAQQJ0IgdqKgIAu6AgAiAHQQRyaioCALugIAIgB0EIcmoqAgC7oCACIAdBDHJqKgIAu6AhuAIgAEEEaiEAIAVBBGoiBSAMRw0ACwsgBgRAA0AguAIgAiAAQQJ0aioCALugIbgCIABBAWohACADQQFqIgMgBkcNAAsLILkCILgCoCG5AiDaAUIBfCLaASDdAVINAAsg2AFCAXwi2AEg3AFSDQALINkBQgF8ItkBINsBUg0ACyC5ArYhsgILIAEoAtABILICOAIADAELAkAgACgCAA4DAQABAAsCQCACKQMoItsBQgBXDQAgAikDICLcAUIAVw0AIAIpAxgi3QFCAFcNACACKAIQIgBBAEwNACACKAI8IQkgAigCOCEKIAIoAjQhCCACKALQASELIABBfHEhDCAAQQNxIQYgAEEESSENA0AgCyAJINkBp2xqIQ9CACHYAQNAIA8gCiDYAadsaiEOQgAh2gEDQCAOIAgg2gGnbGohAkMAAAAAIbICQQAhA0EAIQBBACEFIA1FBEADQCCyAiACIABBAXQiB2ovAQBBAnRBkNYEaioCAJIgAiAHQQJyai8BAEECdEGQ1gRqKgIAkiACIAdBBHJqLwEAQQJ0QZDWBGoqAgCSIAIgB0EGcmovAQBBAnRBkNYEaioCAJIhsgIgAEEEaiEAIAVBBGoiBSAMRw0ACwsgBgRAA0AgsgIgAiAAQQF0ai8BAEECdEGQ1gRqKgIAkiGyAiAAQQFqIQAgA0EBaiIDIAZHDQALCyCzAiCyApIhswIg2gFCAXwi2gEg3QFSDQALINgBQgF8ItgBINwBUg0ACyDZAUIBfCLZASDbAVINAAsLIAEoAtABQYD8ASCzAotDAACAd5RDAACACJRBgICAiAcgswK8IgBBAXQiAUGAgIB4cSICIAJBgICAiAdNG0EBdkGAgIA8ar6SvCICQQ12QYD4AXEgAkH/H3FqIAFBgICAeEsbIABBEHZBgIACcXI7AQALIARBEGokAAwBC0G8wwIoAgAQMBogBEGbITYCCCAEQdc/NgIEDEULDDoLIAEoAowBIQQjAEGAAWsiAiQAAkACQAJAAkACQAJAAkACQCAEKAIARQRAIAAoAgQNAQJAAkAgACgCAA4DAQABAAsgBCgCMEEERw0DIAEoAjBBBEcNBCABKQMQQgFSDQUgBCkDGCLZASABKQMYUg0GIAQpAyAi3QEgASkDIFINByAEKQMoIt4BIAEpAyhSDQgg3gFCAFcNACDdAUIAVw0AINkBQgBXDQAgASgCPCEFIAEoAjghByABKAI0IQMgASgC0AEhBiAEKQMQpyIAQQBMBEAgA0EBRiDZAUIDVnEhBCDZAUIDgyHgASDZAUJ8gyHaAQNAIAYgBSDcAadsaiEJQgAh3wEDQCAJIAcg3wGnbGohAEIAIdsBQgAh2AECQCAEBED9DAAAAAABAAAAAgAAAAMAAAAhkgIDQCAAIJIC/RsAakEANgIAIAAgkgL9GwFqQQA2AgAgACCSAv0bAmpBADYCACAAIJIC/RsDakEANgIAIJIC/QwEAAAABAAAAAQAAAAEAAAA/a4BIZICINgBQgR8ItgBINoBUg0ACyDaASLYASDZAVENAQsg2QEg2AFCf4V8IYUCIOABQgBSBEADQCAAIAMg2AGnbGpBADYCACDYAUIBfCHYASDbAUIBfCLbASDgAVINAAsLIIUCQgNUDQADQCAAIAMg2AGnIgFsakEANgIAIAAgAyABQQFqbGpBADYCACAAIAMgAUECamxqQQA2AgAgACADIAFBA2psakEANgIAINgBQgR8ItgBINkBUg0ACwsg3wFCAXwi3wEg3QFSDQALINwBQgF8ItwBIN4BUg0ACwwBCyAEKAI8IQsgBCgCOCEMIAQoAjQhDSAEKALQASEPIABBfHEhDiAAQQNxIQkgAEEESSEQA0AgBiAFINoBpyIAbGohESAPIAAgC2xqIRJCACHbAQNAIBEgByDbAaciAGxqIRMgEiAAIAxsaiEWQgAh2AEDQCAWIA0g2AGnIhVsaiEBRAAAAAAAAAAAIbgCQQAhCkEAIQBBACEIIBBFBEADQCC4AiABIABBAnQiBGoqAgC7oCABIARBBHJqKgIAu6AgASAEQQhyaioCALugIAEgBEEMcmoqAgC7oCG4AiAAQQRqIQAgCEEEaiIIIA5HDQALCyAJBEADQCC4AiABIABBAnRqKgIAu6AhuAIgAEEBaiEAIApBAWoiCiAJRw0ACwsgEyADIBVsaiC4ArY4AgAg2AFCAXwi2AEg2QFSDQALINsBQgF8ItsBIN0BUg0ACyDaAUIBfCLaASDeAVINAAsLIAJBgAFqJAAMCAtBvMMCKAIAEDAaIAJBmyE2AnggAkGKwAA2AnQgAkHkJjYCcEG4wwIoAgBBy+QAIAJB8ABqEDEMVQtBvMMCKAIAEDAaIAJB/cAANgJoIAJB4j82AmQgAkHkJjYCYEG4wwIoAgBBy+QAIAJB4ABqEDEMVAtBvMMCKAIAEDAaIAJB/s8ANgJYIAJB6D82AlQgAkHkJjYCUEG4wwIoAgBBy+QAIAJB0ABqEDEMUwtBvMMCKAIAEDAaIAJB4s8ANgJIIAJB6T82AkQgAkHkJjYCQEG4wwIoAgBBy+QAIAJBQGsQMQxSC0G8wwIoAgAQMBogAkHVPzYCOCACQe0/NgI0IAJB5CY2AjBBuMMCKAIAQcvkACACQTBqEDEMUQtBvMMCKAIAEDAaIAJBkT42AiggAkHuPzYCJCACQeQmNgIgQbjDAigCAEHL5AAgAkEgahAxDFALQbzDAigCABAwGiACQYo9NgIYIAJB7z82AhQgAkHkJjYCEEG4wwIoAgBBy+QAIAJBEGoQMQxPC0G8wwIoAgAQMBogAkHhOjYCCCACQfA/NgIEIAJB5CY2AgBBuMMCKAIAQcvkACACEDEMTgsMOQsgASgCjAEhAiMAQRBrIgMkAAJAIAIoAgBFBEACQAJAIAAoAgAOAwEAAQALIAIpAygi3gFCAFcNACACKQMgIt8BQgBXDQAgAikDGCLZAUIAVw0AIAEoAjwhBSABKAI4IQcgASgCNCEEIAIpAxAi2AG0IbICIAEoAtABIQYg2AGnIgBBAEwEQEMAAAAAILIClSGyAiAEQQFGINkBQgNWcSECINkBQgODIeABINkBQnyDIdoBA0AgBiAFINwBp2xqIQlCACHdAQNAIAkgByDdAadsaiEAQgAh2wFCACHYAQJAIAIEQP0MAAAAAAEAAAACAAAAAwAAACGSAgNAIAAgkgL9GwBqILICOAIAIAAgkgL9GwFqILICOAIAIAAgkgL9GwJqILICOAIAIAAgkgL9GwNqILICOAIAIJIC/QwEAAAABAAAAAQAAAAEAAAA/a4BIZICINgBQgR8ItgBINoBUg0ACyDaASLYASDZAVENAQsg2QEg2AFCf4V8IYYCIOABQgBSBEADQCAAIAQg2AGnbGogsgI4AgAg2AFCAXwh2AEg2wFCAXwi2wEg4AFSDQALCyCGAkIDVA0AA0AgACAEINgBpyIBbGogsgI4AgAgACAEIAFBAWpsaiCyAjgCACAAIAQgAUECamxqILICOAIAIAAgBCABQQNqbGogsgI4AgAg2AFCBHwi2AEg2QFSDQALCyDdAUIBfCLdASDfAVINAAsg3AFCAXwi3AEg3gFSDQALDAELIAIoAjwhCyACKAI4IQwgAigCNCENIAIoAtABIQ8gAEF8cSEOIABBA3EhCSAAQQRJIRADQCAPIAsg2gGnIgBsaiERIAYgACAFbGohEkIAIdsBA0AgESAMINsBpyIAbGohEyASIAAgB2xqIRZCACHYAQNAIBMgDSDYAaciFWxqIQFEAAAAAAAAAAAhuAJBACEKQQAhAEEAIQggEEUEQANAILgCIAEgAEECdCICaioCALugIAEgAkEEcmoqAgC7oCABIAJBCHJqKgIAu6AgASACQQxyaioCALugIbgCIABBBGohACAIQQRqIgggDkcNAAsLIAkEQANAILgCIAEgAEECdGoqAgC7oCG4AiAAQQFqIQAgCkEBaiIKIAlHDQALCyAWIAQgFWxqILgCtiCyApU4AgAg2AFCAXwi2AEg2QFSDQALINsBQgF8ItsBIN8BUg0ACyDaAUIBfCLaASDeAVINAAsLIANBEGokAAwBC0G8wwIoAgAQMBogA0GbITYCCCADQcHAADYCBAxECww4CyABKAKMASEEIwBBEGsiAyQAAkAgBCgCAEUEQAJAAkAgACgCAA4DAQABAAsgBCkDGCLaAUIAVw0AIAEoAjAhAiABKALQASEBIAQoAhAiAEEASgRAIAQoAjQhCiAEKALQASEIIABBfHEhCyAAQQNxIQcgAEEESSEMA0AgCCAKINgBpyINbGohBEMAAID/IbICQQAhBkEAIQBBACEFQQAhCSAMRQRAA0AgAEEDciIPIABBAnIiDiAAQQFyIhAgACAFILICIAQgAEECdGoqAgAiswIgsgIgswJeGyKyAiCzAlsbILICIAQgEEECdGoqAgAiswIgsgIgswJeGyKyAiCzAlsbILICIAQgDkECdGoqAgAiswIgsgIgswJeGyKyAiCzAlsbILICIAQgD0ECdGoqAgAiswIgsgIgswJeGyKyAiCzAlsbIQUgAEEEaiEAIAlBBGoiCSALRw0ACwsgBwRAA0AgACAFILICIAQgAEECdGoqAgAiswIgsgIgswJeGyKyAiCzAlsbIQUgAEEBaiEAIAZBAWoiBiAHRw0ACwsgASACIA1saiAFNgIAINgBQgF8ItgBINoBUg0ACwwBCwJAINoBQgRUDQAgAkEBRw0AINoBQnyDIdgB/QwAAAAAAQAAAAIAAAADAAAAIZICA0AgASCSAv0bAGpBADYCACABIJIC/RsBakEANgIAIAEgkgL9GwJqQQA2AgAgASCSAv0bA2pBADYCACCSAv0MBAAAAAQAAAAEAAAABAAAAP2uASGSAiDZAUIEfCLZASDYAVINAAsg2AEg2gFRDQELINoBINgBQn+FfCGHAiDaAUIDgyLcAUIAUgRAA0AgASACINgBp2xqQQA2AgAg2AFCAXwh2AEg2wFCAXwi2wEg3AFSDQALCyCHAkIDVA0AA0AgASACINgBpyIAbGpBADYCACABIAIgAEEBamxqQQA2AgAgASACIABBAmpsakEANgIAIAEgAiAAQQNqbGpBADYCACDYAUIEfCLYASDaAVINAAsLIANBEGokAAwBC0G8wwIoAgAQMBogA0GbITYCCCADQe/AADYCBAxDCww3CyABKAKMASEDIwBBkAFrIgQkAAJAAkACQAJAAkACQAJAAkACQAJAAkACQCADKAIADgIBAAQLIAAoAgQNBCABKQMQItgBINgBIAMpAxAi2QF/ItgBINkBflINBSABKQMYItoBINoBIAMpAxgi3gF/ItoBIN4BflINBSABKQMgItsBINsBIAMpAyAi3wF/ItsBIN8BflINBSABKQMoItwBINwBIAMpAygi4AF/ItwBIOABflINBQJAIAAoAgAOAwIAAgALIAEoAjBBAkcNBiADKAIwQQJGBEAg3AGnIgtBAEwNAiDYAaciDEEATA0CIN4BQgBXDQIg2gGnIg1BAEwNAiDfAUIAVw0CIOABQgBXDQIg2wGnIg9BAEwNAiDZAUIAVw0CINkBpyIAQQF0IQ4g2QFCA4Mh5AEg2QFCeIMh2gEgAEEBdCEQIAEoAjQiCSDeAaciEWytIeYBIAEoAjgiCiDfAaciEmytIecBIAEoAjwiCCDgAaciE2ytIegBIAMoAjQiFq0h6QEgAygCOCIVrSHqASADKAI8IhitIesBIAMoAtABIhqtIewBIAmtIe0BIAqtIe4BIAitIe8BIAEoAtABIhmtIfABA0AgBiATbCEUIOEBIOgBfiDwAXwh8QFCACHbAQNAINsBIOsBfiDsAXwh8gEg8QEg2wEg7wF+fCHzASAaIBgg2wGnIgBsaiEXIBkgACAUaiAIbGohG0EAIQFCACHiAQNAIAEgEmwhHCDzASDiASDnAX58IfQBQgAh3AEDQCDyASDcASDqAX58IfUBIPQBINwBIO4Bfnwh9gEgFyAVINwBpyIAbGohHSAbIAAgHGogCmxqIR9BACEDQgAh4wEDQCADIBFsIR4g9gEg4wEg5gF+fCH3AUIAId0BA0AgHSAWIN0BpyICbGohACAfIAIgHmogCWxqISMg9QEg3QEg6QF+fKchJSD3ASDdASDtAX58pyEgQQAhBQNAICMgBSAObGohAkIAIeUBQgAh2AECQAJAINkBQghUDQAgBSAQbCAgaiAla0EQSQ0AA0AgAiDYAadBAXQiB2ogACAHav0AAQD9CwEAINgBQgh8ItgBINoBUg0ACyDaASLYASDZAVENAQsg2QEg2AFCf4V8IYgCIOQBQgBSBEADQCACINgBp0EBdCIHaiAAIAdqLwEAOwEAINgBQgF8IdgBIOUBQgF8IuUBIOQBUg0ACwsgiAJCA1QNAANAIAIg2AGnQQF0IgdqIAAgB2ovAQA7AQAgAiAHQQJqIihqIAAgKGovAQA7AQAgAiAHQQRqIihqIAAgKGovAQA7AQAgAiAHQQZqIgdqIAAgB2ovAQA7AQAg2AFCBHwi2AEg2QFSDQALCyAFQQFqIgUgDEcNAAsg3QFCAXwi3QEg3gFSDQALIOMBQgF8IeMBIANBAWoiAyANRw0ACyDcAUIBfCLcASDfAVINAAsg4gFCAXwh4gEgAUEBaiIBIA9HDQALINsBQgF8ItsBIOABUg0ACyDhAUIBfCHhASAGQQFqIgYgC0cNAAsMAgtBvMMCKAIAEDAaIARBl9MANgIYIARBtsEANgIUIARB5CY2AhBBuMMCKAIAQcvkACAEQRBqEDEMVQsgACgCBA0GIAEpAxAi2AEg2AEgAykDECLYAX8i2gEg2AF+Ug0HIAEpAxgi2QEg2QEgAykDGCLeAX8i2QEg3gF+Ug0HIAEpAyAi2wEg2wEgAykDICLfAX8i2wEg3wF+Ug0HIAEpAygi3AEg3AEgAykDKCLgAX8i3AEg4AF+Ug0HAkAgACgCAA4DAQABAAsgASgCMEEERw0IIAMoAjBBBEcNASDcAaciDkEATA0AINoBpyIQQQBMDQAg3gFCAFcNACDZAaciEUEATA0AIN8BQgBXDQAg4AFCAFcNACDbAaciEkEATA0AINgBpyIFQQBMDQAgBUECdCETIAVBA3EhCyAFQXxxIQIgASgCNCIMIN4BpyIWbK0h4QEgASgCOCININ8BpyIVbK0h4gEgASgCPCIPIOABpyIYbK0h4wEgAygCNCIarSHkASADKAI4IhmtIeUBIAMoAjwiFK0h5gEgAygC0AEiF60h5wEgDK0h6AEgDa0h6QEgD60h6gEgASgC0AEiG60h6wFCACHbASAFQQRJIRwDQCAJIBhsIR0g2wEg4wF+IOsBfCHsAUIAIdoBA0Ag2gEg5gF+IOcBfCHtASDsASDaASDqAX58Ie4BIBcgFCDaAaciAGxqIR8gGyAAIB1qIA9saiEeQQAhCkIAIdwBA0AgCiAVbCEjIO4BINwBIOIBfnwh7wFCACHZAQNAIO0BINkBIOUBfnwh8AEg7wEg2QEg6QF+fCHxASAfIBkg2QGnIgBsaiElIB4gACAjaiANbGohIEEAIQhCACHdAQNAIAggFmwhKCDxASDdASDhAX58IfIBQgAh2AEDQCAlIBog2AGnIgBsaiEHICAgACAoaiAMbGohKyDwASDYASDkAX58pyEsIPIBINgBIOgBfnynIS1BACEBA0AgKyABIBNsIgNqIQZBACEAAkACQCAcDQAgAyAtaiAsa0EQSQ0AA0AgBiAAQQJ0IgNqIAMgB2r9AAIA/QsCACAAQQRqIgAgAkcNAAsgAiIAIAVGDQELIABBf3MgBWohrQFBACEDIAsEQANAIAYgAEECdCIvaiAHIC9qKgIAOAIAIABBAWohACADQQFqIgMgC0cNAAsLIK0BQQNJDQADQCAGIABBAnQiA2ogAyAHaioCADgCACAGIANBBGoiIWogByAhaioCADgCACAGIANBCGoiIWogByAhaioCADgCACAGIANBDGoiA2ogAyAHaioCADgCACAAQQRqIgAgBUcNAAsLIAFBAWoiASAQRw0ACyDYAUIBfCLYASDeAVINAAsg3QFCAXwh3QEgCEEBaiIIIBFHDQALINkBQgF8ItkBIN8BUg0ACyDcAUIBfCHcASAKQQFqIgogEkcNAAsg2gFCAXwi2gEg4AFSDQALINsBQgF8IdsBIAlBAWoiCSAORw0ACwsgBEGQAWokAAwIC0G8wwIoAgAQMBogBEG20QA2AlggBEGLwQA2AlQgBEHkJjYCUEG4wwIoAgBBy+QAIARB0ABqEDEMUgtBvMMCKAIAEDAaIARBmyE2AgggBEHewQA2AgQgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMQxRC0G8wwIoAgAQMBogBEH9wAA2AkggBEGlwQA2AkQgBEHkJjYCQEG4wwIoAgBBy+QAIARBQGsQMQxQC0G8wwIoAgAQMBogBEG9zgA2AjggBEGmwQA2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMQxPC0G8wwIoAgAQMBogBEHD0gA2AiggBEG1wQA2AiQgBEHkJjYCIEG4wwIoAgBBy+QAIARBIGoQMQxOC0G8wwIoAgAQMBogBEH9wAA2AogBIARB+sAANgKEASAEQeQmNgKAAUG4wwIoAgBBy+QAIARBgAFqEDEMTQtBvMMCKAIAEDAaIARBvc4ANgJ4IARB+8AANgJ0IARB5CY2AnBBuMMCKAIAQcvkACAEQfAAahAxDEwLQbzDAigCABAwGiAEQd3QADYCaCAEQYrBADYCZCAEQeQmNgJgQbjDAigCAEHL5AAgBEHgAGoQMQxLCww2CyABKAKMASEEIwBB0ABrIgIkAAJAAkACQAJAAkAgBCgCAEUEQCAAKAIEDQEgBCkDECLYASDYASABKQMQIt4BfyLiASDeAX5SDQIgBCkDGCLYASDYASABKQMYItoBfyLjASDaAX5SDQIgBCkDICLYASDYASABKQMgItsBfyLkASDbAX5SDQIgBCkDKCLYASDYASABKQMoItwBfyLYASDcAX5SDQICQAJAIAAoAgAOAwEAAQALIAEoAjBBBEcNBCAEKAIwQQRHDQUgASgCPCEGIAEoAjghCSABKAI0IQMgBCgCPCEMIAQoAjghDSAEKAI0IQ8g2AGnIRMCQAJAAkACQCABKAIAIgAOEwABAQEBAQEBAQEBAQEBAQEBAQABCyADrSLYASDeAUIChiAAQSRsQZSbAWo0AgB/Ug0AIAmtItkBINgBINoBflINACAGrSDZASDbAX5RDQELINwBQgBXDQEg2gFCAFcNASDbAUIAVw0BIN4BpyIFQQBMDQEgASgC0AEhACDaAUJ8gyHlASDaAUIDgyHgASAFQQJ0IQVCACHZAQNAIAYg2QGnbCEIQgAh3QEDQCAJIN0Bp2wgCGohB0IAId8BQgAh2AFCACHhASDaAUIEWgRAA0AgACAHIAMg2AGnIgpsampBACAF/AsAIAAgByADIApBAXJsampBACAF/AsAIAAgByADIApBAnJsampBACAF/AsAIAAgByADIApBA3JsampBACAF/AsAINgBQgR8IdgBIOEBQgR8IuEBIOUBUg0ACwsg4AFCAFIEQANAIAAgByADINgBp2xqakEAIAX8CwAg2AFCAXwh2AEg3wFCAXwi3wEg4AFSDQALCyDdAUIBfCLdASDbAVINAAsg2QFCAXwi2QEg3AFSDQALDAELINoBIN4BfiDbAX4g3AF+pyIAQQBMDQAgASgC0AFBACAAQQJ0/AsACyATQQBMDQAg4gGnIhhBAEwNACDaAUIAVw0AIOMBpyIaQQBMDQAg2wFCAFcNACDcAUIAVw0AIOQBpyIZQQBMDQAg3gGnIgVBAEwNACAFQQJ0IRYgBCgC0AEhDiABKALQASEQIAVBA3EhFSAFQXxxIQEg3gFCAoYi4QFC/P///w+DIeIBIA8g2gGnIhRsrSHjASANINsBpyIXbK0h5AEgDCDcAaciG2ytIeUBIA+tIeYBIA2tIecBIAytIegBIAOtIekBIAmtIeoBIAatIesBQgAh3QEgBUEESSEcA0AgCyAbbCEdIOIBIN0BIOUBfiLsAXwh7QFCACHeAQNAIO0BIN4BIOgBfiLYAXwh7gEg2AEg7AF8Ie8BIOEBIN4BIOsBfiLwAXwh8QEgECAGIN4BpyIAbGohHyAOIAAgHWogDGxqIR5BACERQgAh4AEDQCARIBdsISMg7gEg4AEg5AF+ItgBfCHyASDYASDvAXwh8wFCACHZAQNAIPIBINkBIOcBfiLYAXwh9AEg2AEg8wF8IfUBIPEBINkBIOoBfiLYAXwh9gEg2AEg8AF8IfcBIB8gCSDZAaciAGxqISUgHiAAICNqIA1saiEgQQAhEkIAId8BA0AgEiAUbCEoIPQBIN8BIOMBfiLYAXwh+AEg2AEg9QF8IfoBQgAh2AEDQCAlIAMg2AGnIgBsaiEEICAgACAoaiAPbGohKyAQIPYBINgBIOkBfiL5AXynaiEsIBAg9wEg+QF8p2ohLSD4ASDYASDmAX4i+QF8pyEhIPkBIPoBfKchL0EAIQoDQCArIAogFmwiCGohB0EAIQACQAJAIBwNACAOIAggL2pqICxJIA4gCCAhamogLUtxDQADQCAEIABBAnQiCGoiIiAHIAhq/QACACAi/QACAP3kAf0LAgAgAEEEaiIAIAFHDQALIAEiACAFRg0BCyAAQX9zIAVqIa4BQQAhCCAVBEADQCAEIABBAnQiJGoiJyAHICRqKgIAICcqAgCSOAIAIABBAWohACAIQQFqIgggFUcNAAsLIK4BQQNJDQADQCAEIABBAnQiCGoiIiAHIAhqKgIAICIqAgCSOAIAIAQgCEEEaiIiaiIkIAcgImoqAgAgJCoCAJI4AgAgBCAIQQhqIiJqIiQgByAiaioCACAkKgIAkjgCACAEIAhBDGoiCGoiIiAHIAhqKgIAICIqAgCSOAIAIABBBGoiACAFRw0ACwsgCkEBaiIKIBhHDQALINgBQgF8ItgBINoBUg0ACyDfAUIBfCHfASASQQFqIhIgGkcNAAsg2QFCAXwi2QEg2wFSDQALIOABQgF8IeABIBFBAWoiESAZRw0ACyDeAUIBfCLeASDcAVINAAsg3QFCAXwh3QEgC0EBaiILIBNHDQALCyACQdAAaiQADAULQbzDAigCABAwGiACQZshNgJIIAJBqcIANgJEDEILQbzDAigCABAwGiACQf3AADYCOCACQenBADYCNAxAC0G8wwIoAgAQMBogAkHl2wA2AiggAkHqwQA2AiQMPAtBvMMCKAIAEDAaIAJB3dAANgIYIAJB+cEANgIUDDwLQbzDAigCABAwGiACQbbRADYCCCACQfrBADYCBAw8Cww1CyABKAKMASEEIAEoApABIQcjAEFAaiICJAACQAJAAkAgBCgCAEUEQAJAAkACQCAAKAIADgMBAAEACyAEKAIwQQRHDQMgASgCMEEERw0EIAcoAjBBBEcNASABKQMoIuIBQgBXDQAgASkDECLcAUIAVw0AIAEpAyAi4wEgACgCBCIDrCLbAVcNACABKQMYIuEBQgBXDQAgASgCPCEOIAEoAjQhCiAHKAI8IRAgBygCNCERIAQoAjwhEiAEKAI0IRMg3AFCA4Mh3wEg3AFCfIMh2gEgBygCOCIIIAAoAggiAGwhFiABKAI4IgkgAGwhCyADIAlsIRUgBCgCOCIMIABsIRggAyAMbCEaIAggAyAEKQMgIuQBpyIZa2whFCAArCHlAQNAIBIg4AGnIgBsIhcgGmohGyAAIA5sIg0gFWohDyAAIBBsIhwgFGohHUEAIQAg2wEh2QEDQCDZAachAwJAINkBIOQBUwRAIAQoAtABIgUgGyAAIBhsamohHyABKALQASIGIA8gACALbGpqIR4gBiADIAlsaiANaiEjIAUgAyAMbGogF2ohJUIAId0BA0AgIyAKIN0BpyIFbCIGaiEDICUgBSATbCIgaiEFQgAh2AECQAJAINwBQgRUDQAgBiAeaiAfICBqa0EQSQ0AA0AgAyDYAadBAnQiBmogBSAGav0AAgD9CwIAINgBQgR8ItgBINoBUg0ACyDaASLYASDcAVENAQsg3AEg2AFCf4V8IYkCQgAh3gEg3wFCAFIEQANAIAMg2AGnQQJ0IgZqIAUgBmoqAgA4AgAg2AFCAXwh2AEg3gFCAXwi3gEg3wFSDQALCyCJAkIDVA0AA0AgAyDYAadBAnQiBmogBSAGaioCADgCACADIAZBBGoiIGogBSAgaioCADgCACADIAZBCGoiIGogBSAgaioCADgCACADIAZBDGoiBmogBSAGaioCADgCACDYAUIEfCLYASDcAVINAAsLIN0BQgF8It0BIOEBUg0ACwwBCyAHKALQASIFIB0gACAWbGpqIR8gASgC0AEiBiAPIAAgC2xqaiEeIAYgAyAJbGogDWohIyAFIAMgGWsgCGxqIBxqISVCACHdAQNAICMgCiDdAaciBWwiBmohAyAlIAUgEWwiIGohBUIAIdgBAkACQCDcAUIEVA0AIAYgHmogHyAgamtBEEkNAANAIAMg2AGnQQJ0IgZqIAUgBmr9AAIA/QsCACDYAUIEfCLYASDaAVINAAsg2gEi2AEg3AFRDQELINwBINgBQn+FfCGKAkIAId4BIN8BQgBSBEADQCADINgBp0ECdCIGaiAFIAZqKgIAOAIAINgBQgF8IdgBIN4BQgF8It4BIN8BUg0ACwsgigJCA1QNAANAIAMg2AGnQQJ0IgZqIAUgBmoqAgA4AgAgAyAGQQRqIiBqIAUgIGoqAgA4AgAgAyAGQQhqIiBqIAUgIGoqAgA4AgAgAyAGQQxqIgZqIAUgBmoqAgA4AgAg2AFCBHwi2AEg3AFSDQALCyDdAUIBfCLdASDhAVINAAsLIABBAWohACDjASDZASDlAXwi2QFVDQALIOABQgF8IuABIOIBUg0ACwsgAkFAayQADAQLQbzDAigCABAwGiACQYnRADYCCCACQcTCADYCBAw+C0G8wwIoAgAQMBogAkGbITYCOCACQezCADYCNAw+C0G8wwIoAgAQMBogAkH+zwA2AiggAkG6wgA2AiQMOgtBvMMCKAIAEDAaIAJB3dAANgIYIAJBwsIANgIUDDoLDDQLIAEoAowBIQQgASgCkAEhAyMAQeAAayICJAACQAJAAkACQAJAAkAgBCgCAEUEQCADKAIwIAMoAgBBJGxBmJsBaigCAEcNASADNQI4ItkBIAMpAxgi2AEgAygCNCIHrX5SDQEgAzUCPCDZASADKQMgItoBflINASAEKAIwQQRHDQIgBDUCOCLcASAEKQMYItkBIAQoAjQiBq1+Ug0CIAQ1Ajwg3AEgBCkDICLbAX5SDQIgASgCMCABKAIAQSRsQZibAWooAgBHDQMgATUCOCLcASABKQMYIt0BIAEoAjQiCa1+Ug0DIAE1Ajwg3AEgASkDICLeAX5SDQMg2wEg3gFSDQQg2QEg3QFSDQQgBCkDECLcASABKQMQUg0EIAQpAygi3QEgASkDKFINBCDaASDbAVINBSDYASDZAVINBSDcASADKQMQUg0FIN0BIAMpAyhSDQUCQAJAIAAoAgAOAwEAAQALINgBINoBfiDdAX6nIgUgACgCCCIKakEBayAKbSIKIAAoAgRsIgAgACAKaiIKIAUgBSAKShsiBU4NACDcAaciCkEATA0AIAMoAtABIQMgBCgC0AEhCCABKALQASELA0AgAyAAIAdsaiEMIAggACAGbGohDSALIAAgCWxqIQ9BACEBA0BBgPwBIA0gAUECdCIEaioCACKyAotDAACAd5RDAACACJRBgICAiAcgsgK8Ig5BAXQiEEGAgIB4cSIRIBFBgICAiAdNG0EBdkGAgIA8ar6SvCIRQQ12QYD4AXEgEUH/H3FqIBBBgICAeEsbIA5BEHZBgIACcXJBAnRBkNYEaioCACKyAowQRSGzAiAEIA9qIAQgDGoqAgBDAACAPyCzAkMAAIA/kpUiswKUILICQwAAgD8gswKTlEMAAIA/kpQ4AgAgAUEBaiIBIApHDQALIABBAWoiACAFRw0ACwsgAkHgAGokAAwGC0G8wwIoAgAQMBogAkGbITYCWCACQa/HADYCVCACQeQmNgJQQbjDAigCAEHL5AAgAkHQAGoQMQxNC0G8wwIoAgAQMBogAkGg1QA2AkggAkH7xgA2AkQgAkHkJjYCQEG4wwIoAgBBy+QAIAJBQGsQMQxMC0G8wwIoAgAQMBogAkG/2wA2AjggAkH8xgA2AjQgAkHkJjYCMEG4wwIoAgBBy+QAIAJBMGoQMQxLC0G8wwIoAgAQMBogAkH5zQA2AiggAkH9xgA2AiQgAkHkJjYCIEG4wwIoAgBBy+QAIAJBIGoQMQxKC0G8wwIoAgAQMBogAkG5zwA2AhggAkH+xgA2AhQgAkHkJjYCEEG4wwIoAgBBy+QAIAJBEGoQMQxJC0G8wwIoAgAQMBogAkHG1QA2AgggAkH/xgA2AgQgAkHkJjYCAEG4wwIoAgBBy+QAIAIQMQxICwwzCyABKAKMASEFIwBBMGsiAyQAAkAgBSgCAEUEQAJAIAUpAxAi2wEgASkDEFINACAFKQMYIt8BIAEpAxhSDQAgBSkDICLgASABKQMgUg0AIAUpAygi4QEgASkDKFINAAJAAkACQCAAKAIADgMBAAEACyAFKAIwQQRHDQEg4QFCAFcNACDgAUIAVw0AIN8BIAA0AgQi2gFXDQAgASgCPCEIIAEoAjghCyABKAI0IQwgBSgCPCENIAUoAjghDyAFKAI0IQ4gASoCRCG1AiAANAIIIeQBINsBuSG6AiDbAaciBkFwcSICIAZBD3EiECAGQQNxIhFrIhJqIQQg2wFCfoMh5QEg2wFCAYMh5gEg2wFCfIMh5wEg2wFCA4Mh4gEgAkEBayITQQR2QQFqIgBB/v///wFxIRYgAEEBcSEVA0AgCCDdAaciAGwhGCAAIA1sIRpCACHeAQNAIAsg3gGnIgBsIRkgACAPbCEUINoBIdgBA0AgBSgC0AEgDiDYAaciCWxqIBRqIBpqIQBEAAAAAAAAAAAhuAJEAAAAAAAAAAAhuQICQCDbAUIAVyIKDQBCACHcAUIAIdkBQgAh4wEg2wFCA1YEQANAILkCIAAg2QGnQQJ0IgdqKgIAu6AgACAHQQRyaioCALugIAAgB0EIcmoqAgC7oCAAIAdBDHJqKgIAu6AhuQIg2QFCBHwh2QEg4wFCBHwi4wEg5wFSDQALCyDiAVANAANAILkCIAAg2QGnQQJ0aioCALugIbkCINkBQgF8IdkBINwBQgF8ItwBIOIBUg0ACwsgASgC0AEgCSAMbGogGWogGGohBwJAIAoNACC5AiC6AqO2IbICQgAh2QFCACHcASDbAUIBUgRAA0AgByDZAadBAnQiCWogACAJaioCACCyApMiswI4AgAgByAJQQRyIglqIAAgCWoqAgAgsgKTIrQCOAIAILgCILMCILMClLugILQCILQClLugIbgCINkBQgJ8IdkBINwBQgJ8ItwBIOUBUg0ACwsg5gFQDQAgByDZAadBAnQiCWogACAJaioCACCyApMisgI4AgAguAIgsgIgsgKUu6AhuAILQwAAgD8gtQIguAIgugKjtpKRlSGyAgJAIAJBAEwNACCyAv0TIZICQQAhCUEAIQogE0EPRwRAA0AgByAJQQJ0IhdqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAcgF0HAAHJqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAlBIGohCSAKQQJqIgogFkcNAAsLIBVFDQAgByAJQQJ0aiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMAsCQCACIAZODQAgAiEAIBBBBE8EQCCyAv0TIZICQQAhAANAIAcgACACakECdGoiCSCSAiAJ/QACAP3mAf0LAgAgAEEEaiIAIBJHDQALIAQhACARRQ0BCwNAIAcgAEECdGoiCSCyAiAJKgIAlDgCACAAQQFqIgAgBkcNAAsLINgBIOQBfCLYASDfAVMNAAsg3gFCAXwi3gEg4AFSDQALIN0BQgF8It0BIOEBUg0ACwsgA0EwaiQADAMLQbzDAigCABAwGiADQf7PADYCCCADQcDHADYCBAxAC0G8wwIoAgAQMBogA0G5zwA2AhggA0G6xwA2AhQMQAtBvMMCKAIAEDAaIANBmyE2AiggA0H0xwA2AiQMQAsMMgsgASgCjAEhBSMAQTBrIgMkAAJAIAUoAgBFBEACQCAFKQMQItkBIAEpAxBSDQAgBSkDGCLeASABKQMYUg0AIAUpAyAi3wEgASkDIFINACAFKQMoIuABIAEpAyhSDQACQAJAAkAgACgCAA4DAQABAAsgBSgCMEEERw0BIOABQgBXDQAg3wFCAFcNACDeASAANAIEItoBVw0AIAEoAjwhCCABKAI4IQsgASgCNCEMIAUoAjwhDSAFKAI4IQ8gBSgCNCEOIAEqAkQhswIg2QGnIgdBAnQhECAANAIIIeQBINkBuSG5AiAHQXBxIgIgB0EPcSIRIAdBA3EiEmsiE2ohBCDZAUJ8gyHlASDZAUIDgyHhASACQQFrIhZBBHZBAWoiAEH+////AXEhFSAAQQFxIRgDQCAIINwBpyIAbCEaIAAgDWwhGUIAId0BA0AgCyDdAaciAGwhFCAAIA9sIRcg2gEh2AEDQCAFKALQASAOINgBpyIJbGogF2ogGWohAAJAINkBQgBXBEBEAAAAAAAAAAAhuAIMAQtEAAAAAAAAAAAhuAJCACHiAUIAIdsBQgAh4wEg2QFCA1YEQANAILgCIAAg2wGnQQJ0IgZqKgIAIrICILIClLugIAAgBkEEcmoqAgAisgIgsgKUu6AgACAGQQhyaioCACKyAiCyApS7oCAAIAZBDHJqKgIAIrICILIClLugIbgCINsBQgR8IdsBIOMBQgR8IuMBIOUBUg0ACwsg4QFQDQADQCC4AiAAINsBp0ECdGoqAgAisgIgsgKUu6AhuAIg2wFCAXwh2wEg4gFCAXwi4gEg4QFSDQALCyABKALQASAJIAxsaiAUaiAaaiIGIAAgEPwKAABDAACAPyCzAiC4AiC5AqO2kpGVIbICAkAgAkEATA0AILIC/RMhkgJBACEJQQAhCiAWQQ9HBEADQCAGIAlBAnQiG2oiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgBiAbQcAAcmoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCUEgaiEJIApBAmoiCiAVRw0ACwsgGEUNACAGIAlBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCwJAIAIgB04NACACIQAgEUEETwRAILIC/RMhkgJBACEAA0AgBiAAIAJqQQJ0aiIJIJICIAn9AAIA/eYB/QsCACAAQQRqIgAgE0cNAAsgBCEAIBJFDQELA0AgBiAAQQJ0aiIJILICIAkqAgCUOAIAIABBAWoiACAHRw0ACwsg2AEg5AF8ItgBIN4BUw0ACyDdAUIBfCLdASDfAVINAAsg3AFCAXwi3AEg4AFSDQALCyADQTBqJAAMAwtBvMMCKAIAEDAaIANB/s8ANgIIIANBhcgANgIEDD8LQbzDAigCABAwGiADQbnPADYCGCADQf/HADYCFAw/C0G8wwIoAgAQMBogA0GbITYCKCADQbbIADYCJAw/CwwxCyABKAKMASEJIAEoApABIQsgASEDIwBBMGsiBiQAAkAgCSgCAEUEQAJAIAkpAxAi2QEgASkDEFINACAJKQMYIt4BIAEpAxhSDQAgCSkDICLfASABKQMgUg0AIAkpAygi4AEgASkDKFINACDZASALKQMQUg0AIN4BIAspAxhSDQAg3wEgCykDIFINACDgASALKQMoUg0AAkACQAJAIAAoAgAOAwEAAQALIAkoAjBBBEcNASDgAUIAVw0AIN8BQgBXDQAg3gEgACgCBCIFrCLaAVcNACADKgJEIbMCINkBtCG0AiAAKAIIIgCsIeIBINkBpyIHQXBxIgEgB0EPcSIKIAdBA3EiDGsiDmohBCAHQQJ0IgIgCygCNCIQIAVsaq0h4wEgAygCNCIRIAVsIgggAmqtIeQBIAdBfHEhAiDZAUJ+gyHlASDZAUIBgyHmASAJKAI0IhIgAGwhFyAAIBFsIRsgACAQbCEcIAFBAWsiE0EEdkEBaiIAQf7///8BcSEWIABBAXEhFSAFIBJsrSHnASAIrSHoASAJKAI4Ih2tIekBIAkoAjwiH60h6gEgAygCOCIerSHrASADKAI8IiOtIewBIAsoAjgiJa0h7QEgCygCPCIgrSHuASAHQQhJIRggCkEESSEaA0AgIyDbAaciAGwhKCAAICBsISsgACAfbCEsINsBIO4BfiDjAXwh7wEg2wEg7AF+ItgBIOQBfCHwASDbASDqAX4g5wF8IfEBINgBIOgBfCHyAUIAIdwBA0AgHiDcAaciAGwhLSAAICVsISEgACAdbCEvIO8BINwBIO0BfnynISIg8AEg3AEg6wF+ItgBfKchJCDxASDcASDpAX58pyEnINgBIPIBfKchMEEAIQ0g2gEh2AEDQCALKALQASIpIBAg2AGnIgBsaiAhaiAraiEKIAkoAtABIiogACASbGogL2ogLGohCAJAINkBQgBXBEBEAAAAAAAAAAAhuAJEAAAAAAAAAAAhuQIMAQtEAAAAAAAAAAAhuQJCACHdAUQAAAAAAAAAACG4AkIAIeEBINkBQgFSBEADQCC5AiAIIN0Bp0ECdCIFaioCACKyAiAFIApqKgIAlLugIAggBUEEciIFaioCACK1AiAFIApqKgIAlLugIbkCILgCILICILIClLugILUCILUClLugIbgCIN0BQgJ8Id0BIOEBQgJ8IuEBIOUBUg0ACwsg5gFQDQAguQIgCCDdAadBAnQiBWoqAgAisgIgBSAKaioCAJS7oCG5AiC4AiCyAiCyApS7oCG4AgsgDSAbbCEZILMCILQClCC4ArYisgKSIbUCIAMoAtABIhQgACARbGogLWogKGohBQJAIAdBAEwiMw0AQQAhD0EAIQACQCAYDQAgGSAwaiAUaiANIBdsICdqICpqa0EQSQ0AA0AgBSAAQQJ0IipqIAggKmr9AAIA/QsCACAAQQRqIgAgAkcNAAsgAiIAIAdGDQELIABBf3MgB2ohrwEgDARAA0AgBSAAQQJ0IjFqIAggMWoqAgA4AgAgAEEBaiEAIA9BAWoiDyAMRw0ACwsgrwFBA0kNAANAIAUgAEECdCIPaiAIIA9qKgIAOAIAIAUgD0EEaiIqaiAIICpqKgIAOAIAIAUgD0EIaiIqaiAIICpqKgIAOAIAIAUgD0EMaiIPaiAIIA9qKgIAOAIAIABBBGoiACAHRw0ACwsgsgIgtAKVIbYCILkCtowgtQKVIbICAkAgAUEATCIqDQAgsgL9EyGSAkEAIQhBACEPIBNBD0cEQANAIAUgCEECdCIxaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAFIDFBwAByaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAIQSBqIQggD0ECaiIPIBZHDQALCyAVRQ0AIAUgCEECdGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADALILMCILYCkiG3AgJAIAEgB04iDw0AIAEhACAaRQRAILIC/RMhkgJBACEAA0AgBSAAIAFqQQJ0aiIIIJICIAj9AAIA/eYB/QsCACAAQQRqIgAgDkcNAAsgBCEAIAxFDQELA0AgBSAAQQJ0aiIIILICIAgqAgCUOAIAIABBAWoiACAHRw0ACwsgtwKRIbICAkAgMw0AQQAhCEEAIQACQCAYDQAgCiAUIBkgJGpqSSApIA0gHGwgImpqIAVLcQ0AA0AgBSAAQQJ0IhlqIhQgCiAZav0AAgAgFP0AAgD95AH9CwIAIABBBGoiACACRw0ACyACIgAgB0YNAQsgAEF/cyAHaiGwASAMBEADQCAFIABBAnQiFGoiKSAKIBRqKgIAICkqAgCSOAIAIABBAWohACAIQQFqIgggDEcNAAsLILABQQNJDQADQCAFIABBAnQiCGoiGSAIIApqKgIAIBkqAgCSOAIAIAUgCEEEaiIZaiIUIAogGWoqAgAgFCoCAJI4AgAgBSAIQQhqIhlqIhQgCiAZaioCACAUKgIAkjgCACAFIAhBDGoiCGoiGSAIIApqKgIAIBkqAgCSOAIAIABBBGoiACAHRw0ACwtDAACAPyCyApUhsgICQCAqDQAgsgL9EyGSAkEAIQpBACEIIBNBD0cEQANAIAUgCkECdCIZaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAFIBlBwAByaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAKQSBqIQogCEECaiIIIBZHDQALCyAVRQ0AIAUgCkECdGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADALAkAgDw0AIAEhACAaRQRAILIC/RMhkgJBACEAA0AgBSAAIAFqQQJ0aiIKIJICIAr9AAIA/eYB/QsCACAAQQRqIgAgDkcNAAsgBCEAIAxFDQELA0AgBSAAQQJ0aiIKILICIAoqAgCUOAIAIABBAWoiACAHRw0ACwsgDUEBaiENINgBIOIBfCLYASDeAVMNAAsg3AFCAXwi3AEg3wFSDQALINsBQgF8ItsBIOABUg0ACwsgBkEwaiQADAMLQbzDAigCABAwGiAGQf7PADYCCCAGQcbIADYCBCAGQeQmNgIAQbjDAigCAEHL5AAgBhAxDEcLQbzDAigCABAwGiAGQenYADYCGCAGQcDIADYCFCAGQeQmNgIQQbjDAigCAEHL5AAgBkEQahAxDEYLQbzDAigCABAwGiAGQZshNgIoIAZB5MkANgIkIAZB5CY2AiBBuMMCKAIAQcvkACAGQSBqEDEMRQsMMAsgASgCjAEhByABIQQjAEEwayIFJAACQCAHKAIARQRAAkAgBykDECLcASABKQMQUg0AIAcpAxgi3QEgASkDGFINACAHKQMgItgBIAEpAyBSDQAgBykDKCLiASABKQMoUg0AAkACQAJAIAAoAgAOAwEAAQALIAcoAjBBBEcNASDYAaciDSAEKAJEIgFqQQFrIAFtIQ8gACgCBCIDIAFODQAg4gFCAFcNACDdAUIAVw0AIAQoAjwhDiAEKAI4IQggBCgCNCELIAcoAjwhGSAHKAI4IRAgBygCNCERIAA0Aggh5AEg3AEg3QF+IeUBIAGsIeYBIA+sIecBINwBpyIGQXBxIgEgBkEPcSISIAZBA3EiE2siFmohAiDcAUJ+gyHoASDcAUIBgyHpASDcAUJ8gyHqASDcAUIDgyHjASABQQFrIhVBBHZBAWoiAEH+////AXEhGCAAQQFxIRogA6wh4AEDQCAPIOABIOcBfiLYAadqIgAgDSAAIA1IG6wi3wEg2AFVBEBDAACAP0QAAAAAAAAAACDlASDfASDYAX1+uSK5AqO2Q703hjWSkZUiswL9EyGTAkIAIeEBA0Ag4QGnIQMCeyDcAUIAVQRAIAcoAtABIAMgGWxqIQpEAAAAAAAAAAAhuAIg2AEh2QEDQCAKIBAg2QGnbGohDEIAId4BA0AgDCARIN4Bp2xqIQBCACHbAUIAIdoBINwBQgRaBEADQCC4AiAAINsBp0ECdCIJaioCALugIAAgCUEEcmoqAgC7oCAAIAlBCHJqKgIAu6AgACAJQQxyaioCALugIbgCINsBQgR8IdsBINoBQgR8ItoBIOoBUg0ACwtCACHaASDjAUIAUgRAA0AguAIgACDbAadBAnRqKgIAu6AhuAIg2wFCAXwh2wEg2gFCAXwi2gEg4wFSDQALCyDeAUIBfCLeASDdAVINAAsg2QFCAXwi2QEg3wFTDQALIAMgDmwiDCAEKALQAWohFCC4AiC5AqO2IbICRAAAAAAAAAAAIbgCINgBIdkBA0AgFCAIINkBpyIAbGohFyAKIAAgEGxqIRtCACHeAQNAIBcgCyDeAaciA2xqIQAgGyADIBFsaiEDQgAh2wFCACHaASDcAUIBUgRAA0AgACDbAadBAnQiCWogAyAJaioCACCyApMitAI4AgAgACAJQQRyIglqIAMgCWoqAgAgsgKTIrUCOAIAILgCILQCILQClLugILUCILUClLugIbgCINsBQgJ8IdsBINoBQgJ8ItoBIOgBUg0ACwsg6QGnBEAgACDbAadBAnQiCWogAyAJaioCACCyApMitAI4AgAguAIgtAIgtAKUu6AhuAILIN4BQgF8It4BIN0BUg0ACyDZAUIBfCLZASDfAVMNAAtDAACAPyC4AiC5AqO2Q703hjWSkZUisgL9EwwBCyADIA5sIQwgswIhsgIgkwILIZICAkACQAJAIAFBAEoEQCDYASHaASABIAZODQEgsgL9EyGUAgNAIAgg2gGnbCEUQgAh2wEDQCAEKALQASALINsBp2xqIBRqIAxqIQlBACEDQQAhCiAVQQ9HBEADQCAJIANBAnQiF2oiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCSAXQcAAcmoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgA0EgaiEDIApBAmoiCiAYRw0ACwsgGgRAIAkgA0ECdGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADALQQAhAyABIQACQCASQQRPBEADQCAJIAEgA2pBAnRqIgAglAIgAP0AAgD95gH9CwIAIANBBGoiAyAWRw0ACyACIQAgE0UNAQsDQCAJIABBAnRqIgMgsgIgAyoCAJQ4AgAgAEEBaiIAIAZHDQALCyDbAUIBfCLbASDdAVINAAsg2gFCAXwi2gEg3wFTDQALDAMLIAEgBkgNAQwCCwNAIAgg2gGnbCEUQgAh2wEDQCAEKALQASALINsBp2xqIBRqIAxqIQlBACEDQQAhCiAVQQ9HBEADQCAJIANBAnQiF2oiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCSAXQcAAcmoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgA0EgaiEDIApBAmoiCiAYRw0ACwsgGgRAIAkgA0ECdGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADALINsBQgF8ItsBIN0BUg0ACyDfASDaAUIBfCLaAVUNAAsMAQsgBCgC0AEgDGohCiCyAv0TIZICINgBIdoBA0AgCiAIINoBp2xqIQxCACHbAQNAIAwgCyDbAadsaiEJQQAhAyABIQACQCASQQRPBEADQCAJIAEgA2pBAnRqIgAgkgIgAP0AAgD95gH9CwIAIANBBGoiAyAWRw0ACyACIQAgE0UNAQsDQCAJIABBAnRqIgMgsgIgAyoCAJQ4AgAgAEEBaiIAIAZHDQALCyDbAUIBfCLbASDdAVINAAsg2gFCAXwi2gEg3wFTDQALCyDhAUIBfCLhASDiAVINAAsLIOABIOQBfCLgASDmAVMNAAsLIAVBMGokAAwDC0G8wwIoAgAQMBogBUH+zwA2AgggBUH1yQA2AgQMOwtBvMMCKAIAEDAaIAVBuc8ANgIYIAVB78kANgIUDD8LQbzDAigCABAwGiAFQZshNgIoIAVBvsoANgIkDD8LDC8LIAAgASgCjAEgASgCkAEgAUIAIAEpAxgQ9gIMLgsgASgCjAEhAiABKAKQASEDIwBBEGsiBCQAAkACQAJAAkACQCAAKAIADgMAAQABCyAAIAEoApQBIAMgAUIAIAEpAxgQ9gIMAQsgAikDGEIAVw0AIAEoAkghByABKAJEIQYDQCACKALQASACKAI0INgBp2xqIAIoAjAgBmxqKAIAIgVBAEgNAiAFIAdODQIgACAFQQJ0IAFqKAKUASADIAEg2AFCARD2AiDYAUIBfCLYASACKQMYUw0ACwsgBEEQaiQADAELQbzDAigCABAwGiAEQZwTNgIIIARB1cwANgIEDDgLDC0LIAEoAowBIQkgASgCkAEhCiABIQQjAEGgCGsiAyQAAkACQCAJKAIAIgFBDksNAAJAAkACQAJAAkACQAJAAkACQAJAAkACQEEBIAF0Qcz7AXFFBEAgAUUNASABQQFHDQ1BvMMCKAIAEDAaIANBmyE2ApgBIANBns8ANgKUASADQeQmNgKQAUG4wwIoAgBBy+QAIANBkAFqEDEMTwsgCSkDICLYASAKKQMgUg0IIAopAygi2gEgCSkDKFINByAEKQMgINgBUg0GIAQpAygg2gFSDQUgCSgCMCABQSRsQZibAWooAgBHDQQgBCgCMEEERw0DIAkpAxAi4gEgBCkDEFINAiAKKQMQItsBIAQpAxhSDQECQCAAKAIADgMKAAwACyDaASDYASDbAX4i3AF+ItgBIAA0Aggi2gF8QgF9INoBfyLaASAAKAIEIgKsfiLZASDZASDaAXwi2gEg2AEg2AEg2gFVGyLfAVkNCyAJKQMYIt0BQgBXDQsgBDUCPCHjASAENQI4IeQBIAQ1AjQh5QEgCjUCPCHmASAKNQI4IecBIAo1AjQh6AEgCTUCPCHgASAJNQI4IeEBIAk1AjQh3gEgAUEkbEGgmwFqKAIAIQggCjUCMCHpASAAKAIQIAIg4gGnIgVBEGpsQQJ0aiEHIAVBcHEiAUEASgRAIAEgBUEPcSIAIAVBA3EiDWsiD2ohAiAAQQRJIQ4DQCDZASDZASDcAX8i2AEg3AF+fSLiASDbAX8i2gEg5wF+INgBIOYBfnwg4gEg2gEg2wF+fSLiASDpAX58IeoBINoBIOQBfiDYASDjAX58IOIBIOUBfnynIRAg2gEg4QF+INgBIOABfnwh2gFCACHYAQNAIAooAtABIbIBIAQoAtABIbEBIAkoAtABINoBINgBIN4BfnynaiAHIAUgCBEFACCxASAQaiELILIBIOoBINgBIOgBfnynaioCACKyAv0TIZICQQAhAANAIAsgAEECdCIMaiIGIJICIAcgDGoiDP0AAAD95gEgBv0AAAD95AH9CwAAIAYgkgIgDP0AABD95gEgBv0AABD95AH9CwAQIAYgkgIgDP0AACD95gEgBv0AACD95AH9CwAgIAYgkgIgDP0AADD95gEgBv0AADD95AH9CwAwIABBEGoiACABSA0ACwJAIAEgBU4NACABIQAgDkUEQEEAIQADQCALIAAgAWpBAnQiBmoiDCAGIAdq/QACACCSAv3mASAM/QACAP3kAf0LAgAgAEEEaiIAIA9HDQALIAIhACANRQ0BCwNAIAsgAEECdCIGaiIMIAYgB2oqAgAgsgKUIAwqAgCSOAIAIABBAWoiACAFRw0ACwsg2AFCAXwi2AEg3QFSDQALINkBQgF8ItkBIN8BUg0ACwwMCyABIAVOBEAg3QFCfoMh4wEg3QFCAYMh5AEDQCDZASDZASDcAX8i2AEg3AF+fSDbAX8g4QF+INgBIOABfnwh4gFCACHYAUIAIdoBIN0BQgFSBEADQCAJKALQASDYASDeAX4g4gF8p2ogByAFIAgRBQAgCSgC0AEg2AFCAYQg3gF+IOIBfKdqIAcgBSAIEQUAINgBQgJ8IdgBINoBQgJ8ItoBIOMBUg0ACwsg5AGnBEAgCSgC0AEg2AEg3gF+IOIBfKdqIAcgBSAIEQUACyDZAUIBfCLZASDfAVINAAsMDAsgASAFQQ9xIgAgBUEDcSILayIMaiECIABBBEkhDQNAINkBINkBINwBfyLYASDcAX59IuIBINsBfyLaASDnAX4g2AEg5gF+fCDiASDaASDbAX59IuIBIOkBfnwh6gEg2gEg5AF+INgBIOMBfnwg4gEg5QF+fKchDyDaASDhAX4g2AEg4AF+fCHaAUIAIdgBA0AgCigC0AEhtAEgBCgC0AEhswEgCSgC0AEg2AEg3gF+INoBfKdqIAcgBSAIEQUAILMBIA9qIQYgtAEg6gEg2AEg6AF+fKdqKgIAIbICIAEhAAJAIA1FBEAgsgL9EyGSAkEAIQADQCAGIAAgAWpBAnQiDmoiECAHIA5q/QACACCSAv3mASAQ/QACAP3kAf0LAgAgAEEEaiIAIAxHDQALIAIhACALRQ0BCwNAIAYgAEECdCIOaiIQIAcgDmoqAgAgsgKUIBAqAgCSOAIAIABBAWoiACAFRw0ACwsg2AFCAXwi2AEg3QFSDQALINkBQgF8ItkBIN8BUg0ACwwLCyAJKQMQItoBIAQpAxBRBEAgBCkDGCLhASAKKQMQUQRAIAkpAyAi2AEgBCkDIFEEQCAKKQMgINgBUQRAIAopAygi2wEgBCkDKFEEQCDbASAJKQMoItkBUQRAIAkoAjBBBEYEQCAEKAIwQQRGBEACQAJAIAAoAgAOAwABFAELINgBIOEBfiDaAX4g2QF+pyIAQQBKDRIMEwsg2AEg4QF+IuMBINkBfiLYASAANAIIItkBfEIBfSDZAX8i2wEgADQCBH4i2QEg2QEg2wF8ItsBINgBINgBINsBVRsi5AFZDRIgCSkDGCLlAUIAVw0SIAQ1Ajwh5gEgBDUCOCHnASAENQI0IegBIAo1Ajwh6QEgCjUCOCHqASAJNQI8IesBIAk1Ajgh7AEgCjUCMCHtASAKKAI0IgdBH2whRiAJKAI0IgZBH2whRyAHQR5sIUggBkEebCE6IAdBHWwhOyAGQR1sIT4gB0EcbCFJIAZBHGwhSiAHQRtsIUsgBkEbbCFMIAdBGmwhTSAGQRpsIU4gB0EZbCFPIAZBGWwhUCAHQRhsIVEgBkEYbCFSIAdBF2whUyAGQRdsIVQgB0EWbCFVIAZBFmwhViAHQRVsIVcgBkEVbCFYIAdBFGwhWSAGQRRsIVogB0ETbCFbIAZBE2whXCAHQRJsIV0gBkESbCFeIAdBEWwhXyAGQRFsIWAgB0EEdCFhIAZBBHQhYiAHQQ9sIWMgBkEPbCFkIAdBDmwhZSAGQQ5sIWYgB0ENbCFnIAZBDWwhaCAHQQxsIWkgBkEMbCFqIAdBC2whayAGQQtsIWwgB0EKbCFtIAZBCmwhbiAHQQlsIW8gBkEJbCFwIAdBA3QhcSAGQQN0IXIgB0EHbCFzIAZBB2whdCAHQQZsIXUgBkEGbCF2IAdBBWwhdyAGQQVsIXggB0ECdCF5IAZBAnQheiAHQQNsIXsgBkEDbCF8IAdBAXQhfSAGQQF0IX4gB60h7gEgBq0h7wEg2gGnIgtBcHEiASALQQ9xIhwgC0EDcSIdayIfaiECA0BCACHaASDZAUIQfCLdASDkASDdASDkAVMifxsi8QEg2QFVBEADQCDaAUIgfCLeASDlASDeASDlAVMigAEbIvABIPABQiCBIvIBfSHcASDZASHbAQNAINsBINsBIOMBfyLfASDjAX59ItgBINgBIOEBfyLgASDhAX59IeIBINoBINwBUwRAIOABIOwBfiDfASDrAX58IfMBIOABIOoBfiDfASDpAX58IOIBIO0Bfnwh9AEg4AEg5wF+IN8BIOYBfnwg4gEg6AF+fKchgQEg2gEh2AEDQCAEKALQASG1ASAKKALQASEAIAMgCSgC0AEg8wEg2AEg7wF+fKdqIgU2AqAHIAMgBSAGaiIeNgKkByADIAUgfmoiIzYCqAcgAyAFIHxqIiU2AqwHIAMgBSB6aiIgNgKwByADIAUgeGoiKDYCtAcgAyAFIHZqIis2ArgHIAMgBSB0aiIsNgK8ByADIAUgcmoiLTYCwAcgAyAAIPQBINgBIO4BfnynaiIANgKgBiADIAAgB2o2AqQGIAMgACB9ajYCqAYgAyAAIHtqNgKsBiADIAAgeWo2ArAGIAMgACB3ajYCtAYgAyAAIHVqNgK4BiADIAAgc2o2ArwGIAMgACBxajYCwAYgAyAAIG9qNgLEBiADIAUgcGoiITYCxAcgAyAFIG5qIi82AsgHIAMgACBtajYCyAYgAyAFIGxqIiI2AswHIAMgACBrajYCzAYgAyAFIGpqIiQ2AtAHIAMgACBpajYC0AYgAyAFIGhqIic2AtQHIAMgACBnajYC1AYgAyAFIGZqIjA2AtgHIAMgACBlajYC2AYgAyAFIGRqIik2AtwHIAMgACBjajYC3AYgAyAFIGJqIio2AuAHIAMgACBhajYC4AYgAyAFIGBqIjM2AuQHIAMgACBfajYC5AYgAyAFIF5qIoIBNgLoByADIAAgXWo2AugGIAMgBSBcaiKDATYC7AcgAyAAIFtqNgLsBiADIAUgWmoihAE2AvAHIAMgACBZajYC8AYgAyAFIFhqIoUBNgL0ByADIAAgV2o2AvQGIAMgBSBWaiKGATYC+AcgAyAAIFVqNgL4BiADIAUgVGoihwE2AvwHIAMgACBTajYC/AYgAyAFIFJqIogBNgKACCADIAAgUWo2AoAHIAMgBSBQaiKJATYChAggAyAAIE9qNgKEByADIAUgTmoiigE2AogIIAMgACBNajYCiAcgAyAFIExqIosBNgKMCCADIAAgS2o2AowHIAMgBSBKaiKMATYCkAggAyAAIElqNgKQByADIAUgPmoijQE2ApQIIAMgACA7ajYClAcgAyAFIDpqIo4BNgKYCCADIAAgSGo2ApgHIAMgBSBHaiKPATYCnAggAyAAIEZqNgKcB0EAIQADQCADQaACaiIMIABBBHRqIANBoAZqIg0gAEECdGooAgD9CQIA/QsEACAMIABBAXIiDEEEdGogDEECdCANaigCAP0JAgD9CwQAIABBAmoiAEEgRw0ACyC1ASCBAWohDCABQQBKBEAgM0EQaiGQASAqQRBqIZEBIClBEGohkgEgMEEQaiGTASAnQRBqIZQBICRBEGohlQEgIkEQaiGWASAvQRBqIZcBICFBEGohmAEgLUEQaiGZASAsQRBqIZoBICtBEGohmwEgKEEQaiGcASAgQRBqIZ0BICVBEGohngEgI0EQaiGfASAeQRBqIaABIAVBEGohoQFBACENIAP9AASQBiGSAiAD/QAEgAYhkwIgA/0ABPAFIZQCIAP9AATgBSGVAiAD/QAE0AUhlgIgA/0ABMAFIZcCIAP9AASwBSGYAiAD/QAEoAUhmgIgA/0ABJAFIZkCIAP9AASABSGbAiAD/QAE8AQhnAIgA/0ABOAEIZ0CIAP9AATQBCGeAiAD/QAEwAQhnwIgA/0ABLAEIaACIAP9AASgBCGhAiAD/QAEkAQhogIgA/0ABIAEIaMCIAP9AATwAyGkAiAD/QAE4AMhpQIgA/0ABNADIaYCIAP9AATAAyGnAiAD/QAEsAMhqAIgA/0ABKADIakCIAP9AASQAyGqAiAD/QAEgAMhqwIgA/0ABPACIawCIAP9AATgAiGtAiAD/QAE0AIhrgIgA/0ABMACIa8CIAP9AASwAiGwAiAD/QAEoAIhsQIDQCAMIA1BAnQiAGoiCCCSAiAAII8BaiIP/QAAAP3mASCTAiAAII4BaiIO/QAAAP3mASCUAiAAII0BaiIQ/QAAAP3mASCVAiAAIIwBaiIR/QAAAP3mASCWAiAAIIsBaiIS/QAAAP3mASCXAiAAIIoBaiIT/QAAAP3mASCYAiAAIIkBaiIW/QAAAP3mASCaAiAAIIgBaiIV/QAAAP3mASCZAiAAIIcBaiIY/QAAAP3mASCbAiAAIIYBaiIa/QAAAP3mASCcAiAAIIUBaiIZ/QAAAP3mASCdAiAAIIQBaiIU/QAAAP3mASCeAiAAIIMBaiIX/QAAAP3mASCfAiAAIIIBaiIb/QAAAP3mASCgAiAAIDNqIjH9AAAA/eYBIKECIAAgKmoiNP0AAAD95gEgogIgACApaiIy/QAAAP3mASCjAiAAIDBqIjX9AAAA/eYBIKQCIAAgJ2oiNv0AAAD95gEgpQIgACAkaiI3/QAAAP3mASCmAiAAICJqIib9AAAA/eYBIKcCIAAgL2oiOP0AAAD95gEgqAIgACAhaiI5/QAAAP3mASCpAiAAIC1qIjz9AAAA/eYBIKoCIAAgLGoiPf0AAAD95gEgqwIgACAraiI//QAAAP3mASCsAiAAIChqIkD9AAAA/eYBIK0CIAAgIGoiQf0AAAD95gEgrgIgACAlaiJC/QAAAP3mASCvAiAAICNqIkP9AAAA/eYBILACIAAgHmoiRP0AAAD95gEgCP0AAAAgsQIgACAFaiJF/QAAAP3mAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf0LAAAgCCCSAiAP/QAAEP3mASCTAiAO/QAAEP3mASCUAiAQ/QAAEP3mASCVAiAR/QAAEP3mASCWAiAS/QAAEP3mASCXAiAT/QAAEP3mASCYAiAW/QAAEP3mASCaAiAV/QAAEP3mASCZAiAY/QAAEP3mASCbAiAa/QAAEP3mASCcAiAZ/QAAEP3mASCdAiAU/QAAEP3mASCeAiAX/QAAEP3mASCfAiAb/QAAEP3mASCgAiAAIJABav0AAAD95gEgoQIgACCRAWr9AAAA/eYBIKICIAAgkgFq/QAAAP3mASCjAiAAIJMBav0AAAD95gEgpAIgACCUAWr9AAAA/eYBIKUCIAAglQFq/QAAAP3mASCmAiAAIJYBav0AAAD95gEgpwIgACCXAWr9AAAA/eYBIKgCIAAgmAFq/QAAAP3mASCpAiAAIJkBav0AAAD95gEgqgIgACCaAWr9AAAA/eYBIKsCIAAgmwFq/QAAAP3mASCsAiAAIJwBav0AAAD95gEgrQIgACCdAWr9AAAA/eYBIK4CIAAgngFq/QAAAP3mASCvAiAAIJ8Bav0AAAD95gEgsAIgACCgAWr9AAAA/eYBIAj9AAAQILECIAAgoQFq/QAAAP3mAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf0LABAgCCCSAiAP/QAAIP3mASCTAiAO/QAAIP3mASCUAiAQ/QAAIP3mASCVAiAR/QAAIP3mASCWAiAS/QAAIP3mASCXAiAT/QAAIP3mASCYAiAW/QAAIP3mASCaAiAV/QAAIP3mASCZAiAY/QAAIP3mASCbAiAa/QAAIP3mASCcAiAZ/QAAIP3mASCdAiAU/QAAIP3mASCeAiAX/QAAIP3mASCfAiAb/QAAIP3mASCgAiAx/QAAIP3mASChAiA0/QAAIP3mASCiAiAy/QAAIP3mASCjAiA1/QAAIP3mASCkAiA2/QAAIP3mASClAiA3/QAAIP3mASCmAiAm/QAAIP3mASCnAiA4/QAAIP3mASCoAiA5/QAAIP3mASCpAiA8/QAAIP3mASCqAiA9/QAAIP3mASCrAiA//QAAIP3mASCsAiBA/QAAIP3mASCtAiBB/QAAIP3mASCuAiBC/QAAIP3mASCvAiBD/QAAIP3mASCwAiBE/QAAIP3mASAI/QAAICCxAiBF/QAAIP3mAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf0LACAgCCCSAiAP/QAAMP3mASCTAiAO/QAAMP3mASCUAiAQ/QAAMP3mASCVAiAR/QAAMP3mASCWAiAS/QAAMP3mASCXAiAT/QAAMP3mASCYAiAW/QAAMP3mASCaAiAV/QAAMP3mASCZAiAY/QAAMP3mASCbAiAa/QAAMP3mASCcAiAZ/QAAMP3mASCdAiAU/QAAMP3mASCeAiAX/QAAMP3mASCfAiAb/QAAMP3mASCgAiAx/QAAMP3mASChAiA0/QAAMP3mASCiAiAy/QAAMP3mASCjAiA1/QAAMP3mASCkAiA2/QAAMP3mASClAiA3/QAAMP3mASCmAiAm/QAAMP3mASCnAiA4/QAAMP3mASCoAiA5/QAAMP3mASCpAiA8/QAAMP3mASCqAiA9/QAAMP3mASCrAiA//QAAMP3mASCsAiBA/QAAMP3mASCtAiBB/QAAMP3mASCuAiBC/QAAMP3mASCvAiBD/QAAMP3mASCwAiBE/QAAMP3mASAI/QAAMCCxAiBF/QAAMP3mAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf3kAf0LADAgDUEQaiINIAFIDQALC0EAIQUgASALSARAA0AgBUECdCIAIANBoAdqaigCACEIIANBoAZqIABqKAIAKgIAIbICIAEhAAJAIBxBBE8EQCCyAv0TIZICQQAhAANAIAwgACABakECdCINaiIPIAggDWr9AAIAIJIC/eYBIA/9AAIA/eQB/QsCACAAQQRqIgAgH0cNAAsgAiEAIB1FDQELA0AgDCAAQQJ0Ig1qIg8gCCANaioCACCyApQgDyoCAJI4AgAgAEEBaiIAIAtHDQALCyAFQQFqIgVBIEcNAAsLINgBQiB8ItgBINwBUw0ACwsg8gFCAFUEQCDgASDsAX4g3wEg6wF+fCHzASDgASDqAX4g3wEg6QF+fCDiASDtAX58IfQBIOABIOcBfiDfASDmAX58IOIBIOgBfnynIQ8g3AEh2AEDQCAEKALQASAPaiEIIAkoAtABIPMBINgBIO8BfnynaiEMIAooAtABIPQBINgBIO4BfnynaioCACGyAiABQQBKBEAgsgL9EyGSAkEAIQADQCAIIABBAnQiDWoiBSCSAiAMIA1qIg39AAAA/eYBIAX9AAAA/eQB/QsAACAFIJICIA39AAAQ/eYBIAX9AAAQ/eQB/QsAECAFIJICIA39AAAg/eYBIAX9AAAg/eQB/QsAICAFIJICIA39AAAw/eYBIAX9AAAw/eQB/QsAMCAAQRBqIgAgAUgNAAsLAkAgASALTg0AIAEhACAcQQRPBEAgsgL9EyGSAkEAIQADQCAIIAAgAWpBAnQiBWoiDSAFIAxq/QACACCSAv3mASAN/QACAP3kAf0LAgAgAEEEaiIAIB9HDQALIAIhACAdRQ0BCwNAIAggAEECdCIFaiINIAUgDGoqAgAgsgKUIA0qAgCSOAIAIABBAWoiACALRw0ACwsg2AFCAXwi2AEg8AFTDQALCyDbAUIBfCLbASDxAVMNAAsg3gEh2gEggAENAAsLIN0BIdkBIH8NAAsMEgtBvMMCKAIAEDAaIANB3dAANgKoASADQfbMADYCpAEgA0HkJjYCoAFBuMMCKAIAQcvkACADQaABahAxDFQLQbzDAigCABAwGiADQbbRADYCuAEgA0HzzAA2ArQBIANB5CY2ArABQbjDAigCAEHL5AAgA0GwAWoQMQxTC0G8wwIoAgAQMBogA0HUOjYCyAEgA0HwzAA2AsQBIANB5CY2AsABQbjDAigCAEHL5AAgA0HAAWoQMQxSC0G8wwIoAgAQMBogA0HIOjYC2AEgA0HvzAA2AtQBIANB5CY2AtABQbjDAigCAEHL5AAgA0HQAWoQMQxRC0G8wwIoAgAQMBogA0H9PDYC6AEgA0HuzAA2AuQBIANB5CY2AuABQbjDAigCAEHL5AAgA0HgAWoQMQxQC0G8wwIoAgAQMBogA0GKPTYC+AEgA0HtzAA2AvQBIANB5CY2AvABQbjDAigCAEHL5AAgA0HwAWoQMQxPC0G8wwIoAgAQMBogA0G9wAA2AogCIANB7MwANgKEAiADQeQmNgKAAkG4wwIoAgBBy+QAIANBgAJqEDEMTgtBvMMCKAIAEDAaIANBycAANgKYAiADQevMADYClAIgA0HkJjYCkAJBuMMCKAIAQcvkACADQZACahAxDE0LQbzDAigCABAwGiADQb3AADYCGCADQbrOADYCFCADQeQmNgIQQbjDAigCAEHL5AAgA0EQahAxDEwLQbzDAigCABAwGiADQcnAADYCKCADQbnOADYCJCADQeQmNgIgQbjDAigCAEHL5AAgA0EgahAxDEsLQbzDAigCABAwGiADQd3QADYCOCADQbTOADYCNCADQeQmNgIwQbjDAigCAEHL5AAgA0EwahAxDEoLQbzDAigCABAwGiADQdfUADYCSCADQbHOADYCRCADQeQmNgJAQbjDAigCAEHL5AAgA0FAaxAxDEkLQbzDAigCABAwGiADQcg6NgJYIANBrs4ANgJUIANB5CY2AlBBuMMCKAIAQcvkACADQdAAahAxDEgLQbzDAigCABAwGiADQfE8NgJoIANBrc4ANgJkIANB5CY2AmBBuMMCKAIAQcvkACADQeAAahAxDEcLQbzDAigCABAwGiADQdQ6NgJ4IANBrM4ANgJ0IANB5CY2AnBBuMMCKAIAQcvkACADQfAAahAxDEYLQbzDAigCABAwGiADQf08NgKIASADQavOADYChAEgA0HkJjYCgAFBuMMCKAIAQcvkACADQYABahAxDEULINgBINsBfiDiAX4g2gF+pyIAQQBMDQELIAQoAtABQQAgAEECdPwLAAsgA0GgCGokAAwBC0G8wwIoAgAQMBogA0GbITYCCCADQafPADYCBCADQeQmNgIAQbjDAigCAEHL5AAgAxAxDEELDCwLIAEoAowBIQcgASgCkAEhBCABIQIjAEHQAGsiBSQAAkACQCAHKAIARQRAAkAgBygCMEEERw0AIAcoAjQiDK0i2QEgBykDECLYAUIChlINACAHNQI4ItsBINkBIAcpAxgi2gF+Ug0AIAc1Ajwg2wEgBykDICLZAX5SDQACQCABKAIwIgEgAigCAEEkbCIDQZibAWooAgBHDQAgAigCNCIIrSLbASACKQMQItwBIAGtfiADQZSbAWo0AgB/Ug0AIAI1Ajgi3QEg2wEgAikDGCLeAX5SDQAgAjUCPCDdASACKQMgItsBflINAAJAINgBINwBUg0AINoBIN4BUg0AINkBINsBUg0AIAcpAygi2wEgAikDKFINAAJAIAQpAxBCAVINACAEKQMYQgFSDQAgBCkDIEIBUg0AIAQpAyhCAVINAAJAIAAoAgAOAwcABwALINkBINoBfiDbAX6nIgEgACgCCCIDakEBayADbSIGIAAoAgRsIgMgBmoiACABIAAgAUgbIgsgA0wNBiDYAaciBkECdCENIAQoAtABKgIAIrIC/RMhkgIgBkFwcSIBQQBMDQUgASAGQQ9xIg4gBkEDcSIQayIRaiEEIAFBAWsiEkEEdkEBaiIAQf7///8BcSETIABBAXEhFgNAIAMgCGwhACACKALQASIJIAcoAtABIgpHBH8gACAJaiAKIAMgDGxqIA38CgAAIAIoAtABBSAJCyAAaiEKQQAhCUEAIQ8gEkEPRwRAA0AgCiAJQQJ0IhVqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAogFUHAAHJqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAlBIGohCSAPQQJqIg8gE0cNAAsLIBYEQCAKIAlBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCwJAIAEgBk4NAEEAIQkgASEAIA5BBE8EQANAIAogASAJakECdGoiACCSAiAA/QACAP3mAf0LAgAgCUEEaiIJIBFHDQALIAQhACAQRQ0BCwNAIAogAEECdGoiCSCyAiAJKgIAlDgCACAAQQFqIgAgBkcNAAsLIANBAWoiAyALRw0ACwwGC0G8wwIoAgAQMBogBUHU2AA2AgggBUG2zwA2AgQMOgtBvMMCKAIAEDAaIAVBuc8ANgIYIAVBtc8ANgIUDD4LQbzDAigCABAwGiAFQfTLADYCKCAFQbTPADYCJAw+C0G8wwIoAgAQMBogBUHf2QA2AjggBUGzzwA2AjQgBUHkJjYCMEG4wwIoAgBBy+QAIAVBMGoQMQxCC0G8wwIoAgAQMBogBUGbITYCSCAFQeXPADYCRCAFQeQmNgJAQbjDAigCAEHL5AAgBUFAaxAxDEELIAcoAtABIQQgAigC0AEhCSABIAZIBEAgBkEPcSEAIAQgCUcEQCABIAAgBkEDcSIOayIQaiEEIABBBEkhEQNAIAMgCGwhACAHKALQASIKIAlHBEAgACAJaiAKIAMgDGxqIA38CgAAIAIoAtABIQkLIAAgCWohD0EAIQogASEAAkAgEUUEQANAIA8gASAKakECdGoiACCSAiAA/QACAP3mAf0LAgAgCkEEaiIKIBBHDQALIAQhACAORQ0BCwNAIA8gAEECdGoiCiCyAiAKKgIAlDgCACAAQQFqIgAgBkcNAAsLIANBAWoiAyALRw0ACwwCCyABIAAgBkEDcSIHayIMaiECIABBBEkhDQNAIAkgAyAIbGohBEEAIQogASEAAkAgDUUEQANAIAQgASAKakECdGoiACCSAiAA/QACAP3mAf0LAgAgCkEEaiIKIAxHDQALIAIhACAHRQ0BCwNAIAQgAEECdGoiCiCyAiAKKgIAlDgCACAAQQFqIgAgBkcNAAsLIANBAWoiAyALRw0ACwwBCyAEIAlGDQAgA0EBaiG2ASALIANrQQFxBEAgAigC0AEiASAHKALQASIERwRAIAEgAyAIbGogBCADIAxsaiAN/AoAAAsgA0EBaiEDCyC2ASALRg0AA0AgAigC0AEiACAHKALQASIBRwRAIAAgAyAIbGogASADIAxsaiAN/AoAAAsgAigC0AEiACAHKALQASIBRwRAIAAgA0EBaiIEIAhsaiABIAQgDGxqIA38CgAACyADQQJqIgMgC0cNAAsLIAVB0ABqJAAMKwsgASgCjAEhAiABKAKQASEDIwBB0ABrIgQkAAJAAkACQAJAAkAgAigCAEUEQCACKQMQItkBIAEpAxBSDQEgAikDGCLYASABKQMYUg0BIAIpAyAi2gEgASkDIFINASACKQMoItsBIAEpAyhSDQEgASgCMCIFIAEoAgAiC0EkbCIHQZibAWooAgBHDQIgASgCNCIMrSLcASDZASAFrX4i3gEgB0GUmwFqKAIAIg2sf1INAiABKAI4Ig+tIt0BINgBINwBflINAiABKAI8Ig6tIt8BINoBIN0BflINAiACKAIwQQRHDQIgAjUCNCLgASDZAUIChlINAiACNQI4IuEBINgBIOABflINAiACNQI8INoBIOEBflINAiABKAJQIQYgASgCTCEJIAEoAkghCiABKAJEIQgCQCABKAJURQRAIAAoAgAiBw0BIAEoAtABIAIoAtABIAtBEGtBcU0EfyAMINgBp0EBa2wgBSDZAadsaiAPINoBp0EBa2xqIA4g2wGnQQFrbGoFINoBQgF9IN0BfiDYAUIBfSDcAX58INsBQgF9IN8Bfnwg3gEgDa1/fKcL/AoAAAsgACgCACEHCwJAAkAgBw4DAQABAAsgCCADKQMYItgBQgF9ItoBQgAg2AEg2gFaG6dsIAZqIAogAykDICLaAUIBfSLZAUIAINkBINoBWBunbGogCSADKQMoItkBQgF9ItsBQgAg2QEg2wFaG6dsaiACKAIAQSRsQZibAWooAgAgAykDECLbAUIBfSLcAUIAINsBINwBWhunbGohBSADKAI8IQ0gAygCOCEPIAMoAjQhDiABKQMoIdwBIAEpAxAh3QEgASkDICHeASABKQMYId8BIAMoAjAhtwEgACgCCCECIAAoAgQhCwJ/IAEoAgAiAEEQa0FxTQRAIABBJGxBmJsBaigCACABKAIwIN0Bp0EBa2xqIAEoAjQg3wGnQQFrbGogASgCOCDeAadBAWtsaiABKAI8INwBp0EBa2xqDAELIAE1AjQg3wFCAX1+IN0BIAE1AjB+IABBJGxBlJsBajUCAH98IAE1Ajgg3gFCAX1+fCABNQI8INwBQgF9fnynCyAFSQ0EILcBQQRHDQUgAiDZASDYASDaAX4i3AF+pyIAakEBayACbSICIAtsIgcgAiAHaiICIAAgACACShsiAE4NACDbAaciBUEATA0AIAMoAtABIQsgBUEDcSEMIAVBfHEhAiAGIAEoAtABIhBqIREgAKwh2QEgB6wh2gEgBUEQSSESA0AgCyANINoBINwBfyLbAaciAGwiE2ogDyDaASDbAcQg3AF+fSLbASDYAX8i3QGnIgNsIhZqIA4g2wEg2AEg3QF+faciB2wiFWohASAHIAhsIhggAyAKbCIaIBAgACAJbCIZampqIAZqIQNBACEHQQAhAAJAAkAgEg0AIBEgGGogGWogGmogCyAVaiATaiAWamtBEEkNAANAIAMgAEECdCITaiABIBNq/QACAP0LAgAgAEEEaiIAIAJHDQALIAIiACAFRg0BCyAAQX9zIAVqIbgBIAwEQANAIAMgAEECdCIWaiABIBZqKgIAOAIAIABBAWohACAHQQFqIgcgDEcNAAsLILgBQQNJDQADQCADIABBAnQiB2ogASAHaioCADgCACADIAdBBGoiE2ogASATaioCADgCACADIAdBCGoiE2ogASATaioCADgCACADIAdBDGoiB2ogASAHaioCADgCACAAQQRqIgAgBUcNAAsLINoBQgF8ItoBINkBUg0ACwsgBEHQAGokAAwFC0G8wwIoAgAQMBogBEGbITYCSCAEQcvQADYCRAw/C0G8wwIoAgAQMBogBEG5zwA2AjggBEHxzwA2AjQMPwtBvMMCKAIAEDAaIARBxNkANgIoIARB8s8ANgIkDD8LQbzDAigCABAwGiAEQYzMADYCCCAEQZrQADYCBAw2C0G8wwIoAgAQMBogBEGJ0QA2AhggBEGc0AA2AhQMPgsMKgsgACABKAKMASABEPUCDCkLIAAgASgCjAEgARD1AgwoCyABKAKMASEEIAEoApABIQIjAEEQayIFJAACQAJAAkACQAJAAkAgBCgCACIDQQZrQQlJDQAgAw4EAgEAAAQLAkAgACgCAA4DAwADAAsgAikDICLbAUIAVw0CIAIpAxgi3AFCAFcNAiACKQMQIt0BQgBXDQIgASgCPCEHIAEoAjghBiABKAI0IQkgAigCOCEKIAIoAjQhCCAEKAI8IQsgBCgCOCEMIAQoAjQhDSADQSRsQaCbAWooAgAhAyACKAIwIQ8gBCgCECEOA0AgByDaAaciAGwhECAAIAtsIREgACAKbCESQgAh2AEDQCAGINgBpyIAbCETIAAgDGwhFiAAIAhsIQBCACHZAQNAIAQoAtABIAIoAtABIA8g2QGnIhVsaiAAaiASaigCACANbGogFmogEWogASgC0AEgCSAVbGogE2ogEGogDiADEQUAINkBQgF8ItkBIN0BUg0ACyDYAUIBfCLYASDcAVINAAsg2gFCAXwi2gEg2wFSDQALDAILAkAgACgCAA4DAgACAAsgAikDICLbAUIAVw0BIAIpAxgi3AFCAFcNASACKQMQIt0BQgBXDQEgBCgCECIAQQBMDQEgASgCPCEGIAEoAjghCSABKAI0IQogAigCOCEIIAIoAjQhCyAEKAI8IQwgBCgCOCENIAQoAjQhDyACKAIwIQ4gASgC0AEhECAEKALQASERIAIoAtABIRIgAEF8cSETIABBA3EhBCAAQQRJIRYDQCAQIAYg2gGnIgBsaiEVIBEgACAMbGohGCASIAAgCGxqIRpCACHYAQNAIBUgCSDYAaciAGxqIRkgGCAAIA1saiEUIBogACALbGohF0IAIdkBA0AgGSAKINkBpyIAbGohASAUIBcgACAObGooAgAgD2xqIQJBACEHQQAhAEEAIQMgFkUEQANAIAEgAEECdGogAiAAQQF0ai8BAEECdEGQ1gRqKgIAOAIAIAEgAEEBciIbQQJ0aiACIBtBAXRqLwEAQQJ0QZDWBGoqAgA4AgAgASAAQQJyIhtBAnRqIAIgG0EBdGovAQBBAnRBkNYEaioCADgCACABIABBA3IiG0ECdGogAiAbQQF0ai8BAEECdEGQ1gRqKgIAOAIAIABBBGohACADQQRqIgMgE0cNAAsLIAQEQANAIAEgAEECdGogAiAAQQF0ai8BAEECdEGQ1gRqKgIAOAIAIABBAWohACAHQQFqIgcgBEcNAAsLINkBQgF8ItkBIN0BUg0ACyDYAUIBfCLYASDcAVINAAsg2gFCAXwi2gEg2wFSDQALDAELAkAgACgCAA4DAQABAAsgAikDICLbAUIAVw0AIAIpAxgi3AFCAFcNACACKQMQIt0BQgBXDQAgBCgCECIDQQBMDQAgAigCOCEJIAIoAjQhCiAEKAI0IQggAigCMCELIAIoAtABIQwgA0EDcSEGIANBfHEhAiAEKAI4Ig2tId4BIAQoAjwiD60h3wEgBCgC0AEiDq0h4AEgASgCNCIQrSHhASABKAI4IhGtIeIBIAEoAjwiEq0h4wEgASgC0AEiE60h5AEgA0EISSEWA0Ag2AEg3wF+IOABfCHlASDYASDjAX4g5AF8IeYBIA4gDyDYAaciAGxqIRUgEyAAIBJsaiEYIAwgACAJbGohGkIAIdoBA0Ag5gEg2gEg4gF+fCHnASAVIA0g2gGnIgBsaiEZIBggACARbGohFCAaIAAgCmxqIRcg5QEg2gEg3gF+fKchG0IAIdkBA0AgFCAQINkBpyIAbGohASAZIBcgACALbGooAgAgCGwiHGohBEEAIQdBACEAAkACQCAWDQAg5wEg2QEg4QF+fKcgGyAcamtBEEkNAANAIAEgAEECdCIcaiAEIBxq/QACAP0LAgAgAEEEaiIAIAJHDQALIAIiACADRg0BCyAAQX9zIANqIbkBIAYEQANAIAEgAEECdCIdaiAEIB1qKgIAOAIAIABBAWohACAHQQFqIgcgBkcNAAsLILkBQQNJDQADQCABIABBAnQiB2ogBCAHaioCADgCACABIAdBBGoiHGogBCAcaioCADgCACABIAdBCGoiHGogBCAcaioCADgCACABIAdBDGoiB2ogBCAHaioCADgCACAAQQRqIgAgA0cNAAsLINkBQgF8ItkBIN0BUg0ACyDaAUIBfCLaASDcAVINAAsg2AFCAXwi2AEg2wFSDQALCyAFQRBqJAAMAQtBvMMCKAIAEDAaIAVBmyE2AgggBUGY0gA2AgQMMQsMJwsgASgCjAEhAyABKAKQASECIwBBkAFrIgQkAAJAAkACQAJAAkACQAJAAkACQAJAAkACQCADKAIADgIBAAQLIAAoAgQNBCABKAIwIgUgASgCACIHQSRsIgZBmJsBaigCAEcNBSABKAI0IgmtItgBIAEpAxAi3AEgBa1+It0BIAZBlJsBaigCACIGrH9SDQUgASgCOCIKrSLaASABKQMYItkBINgBflINBSABKAI8IgitIt4BIAEpAyAi2wEg2gF+Ug0FAkAgACgCACILBH8gCwUgASgC0AFBAAJ/IAdBEGtBcU0EQCDZAadBAWsgCWwgBSDcAadsaiDbAadBAWsgCmxqIAEoAihBAWsgCGxqDAELINsBQgF9INoBfiDZAUIBfSDYAX58IN0BIAatf3wgASkDKEIBfSDeAX58pwv8CwAgACgCAAsOAwIAAgALIAEpAxAgAykDECLYAcRSDQYgAygCMEECRgRAIAIpAyggAikDICACKQMYIAIpAxB+fn6nIgZBAEwNAiDYAaciBUEATA0CIAIoAtABIQkgBUF+cSEKIAVBAXEhCCADKAI0IQsgASgCNCEMIAEoAtABIQ0gAygC0AEhD0EAIQEDQCAPIAEgC2xqIQIgDSAJIAFBAnRqKAIAIAxsaiEDQQAhAEEAIQcgBUEBRwRAA0AgAyAAQQJ0aiIOIAIgAEEBdGovAQBBAnRBkNYEaioCACAOKgIAkjgCACADIABBAXIiDkECdGoiECACIA5BAXRqLwEAQQJ0QZDWBGoqAgAgECoCAJI4AgAgAEECaiEAIAdBAmoiByAKRw0ACwsgCARAIAMgAEECdGoiAyACIABBAXRqLwEAQQJ0QZDWBGoqAgAgAyoCAJI4AgALIAFBAWoiASAGRw0ACwwCC0G8wwIoAgAQMBogBEHM0QA2AhggBEHH0gA2AhQgBEHkJjYCEEG4wwIoAgBBy+QAIARBEGoQMQxFCyAAKAIEDQYgASgCMCIFIAEoAgAiB0EkbCIGQZibAWooAgBHDQcgASgCNCIJrSLYASABKQMQItwBIAWtfiLdASAGQZSbAWooAgAiBqx/Ug0HIAEoAjgiCq0i2gEgASkDGCLZASDYAX5SDQcgASgCPCIIrSLeASABKQMgItsBINoBflINBwJAIAAoAgAiCwR/IAsFIAEoAtABQQACfyAHQRBrQXFNBEAg2QGnQQFrIAlsIAUg3AGnbGog2wGnQQFrIApsaiABKAIoQQFrIAhsagwBCyDbAUIBfSDaAX4g2QFCAX0g2AF+fCDdASAGrX98IAEpAyhCAX0g3gF+fKcL/AsAIAAoAgALDgMBAAEACyABKQMQIAMpAxAi2AHEUg0IIAMoAjBBBEcNASACKQMoIAIpAyAgAikDGCACKQMQfn5+pyIIQQBMDQAg2AGnIgVBAEwNACADKAI0IQsgAygC0AEhCSABKAI0IQwgAigC0AEhDSAFQQNxIQogBUF8cSECIAEoAtABIg8gBUECdCIOaiEQQQAhBiAFQQhJIREDQCAJIAYgC2wiB2ohASAPIA0gBkECdGooAgAgDGwiEmohA0EAIQACQAJAIBENACABIBAgEmpJIAkgByAOamogA0txDQADQCADIABBAnQiB2oiEiAS/QACACABIAdq/QACAP3kAf0LAgAgAEEEaiIAIAJHDQALIAIiACAFRg0BCyAAQX9zIAVqIboBQQAhByAKBEADQCADIABBAnQiE2oiFiAWKgIAIAEgE2oqAgCSOAIAIABBAWohACAHQQFqIgcgCkcNAAsLILoBQQNJDQADQCADIABBAnQiB2oiEiASKgIAIAEgB2oqAgCSOAIAIAMgB0EEaiISaiITIBMqAgAgASASaioCAJI4AgAgAyAHQQhqIhJqIhMgEyoCACABIBJqKgIAkjgCACADIAdBDGoiB2oiEiASKgIAIAEgB2oqAgCSOAIAIABBBGoiACAFRw0ACwsgBkEBaiIGIAhHDQALCyAEQZABaiQADAgLQbzDAigCABAwGiAEQf7PADYCWCAEQenSADYCVCAEQeQmNgJQQbjDAigCAEHL5AAgBEHQAGoQMQxCC0G8wwIoAgAQMBogBEGbITYCCCAEQYXTADYCBCAEQeQmNgIAQbjDAigCAEHL5AAgBBAxDEELQbzDAigCABAwGiAEQf3AADYCSCAEQbbSADYCRCAEQeQmNgJAQbjDAigCAEHL5AAgBEFAaxAxDEALQbzDAigCABAwGiAEQfTLADYCOCAEQbfSADYCNCAEQeQmNgIwQbjDAigCAEHL5AAgBEEwahAxDD8LQbzDAigCABAwGiAEQbYmNgIoIARBxtIANgIkIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEMPgtBvMMCKAIAEDAaIARB/cAANgKIASAEQdjSADYChAEgBEHkJjYCgAFBuMMCKAIAQcvkACAEQYABahAxDD0LQbzDAigCABAwGiAEQfTLADYCeCAEQdnSADYCdCAEQeQmNgJwQbjDAigCAEHL5AAgBEHwAGoQMQw8C0G8wwIoAgAQMBogBEG2JjYCaCAEQejSADYCZCAEQeQmNgJgQbjDAigCAEHL5AAgBEHgAGoQMQw7CwwmCyABKAKMASEEIwBBkAFrIgIkAAJAAkACQAJAAkACQAJAAkACQCAEKAIARQRAIAAoAgQNAQJAAkAgACgCAA4DAQABAAsgBCkDECLZASABKQMQUg0DINkBIAEpAxhSDQQgBCkDGEIBUg0FIAQpAyAi2wEgASkDIFINBiAEKQMoItwBIAEpAyhSDQcgBCgCMEEERw0IIAEoAjBBBEcNCSDcAUIAVw0AINkBQgBXDQAg2wFCAFcNACABKAI8IQYgASgCOCEAIAQoAjwhCSAEKAI4IQMgBCgC0AEhCiABKALQASEEINkBQgFWBEAgASgCNCILQQRqIQwg2QGnQQJ0QQRrIQUDQCAKIAkg2gGnIgFsaiENIAQgASAGbCIHaiEPIAdBBGohDkIAId0BA0AgDyAAIN0BpyIIbCIBaiIQIA0gAyAIbGoiESoCADgCACAEIAEgDmoiEmpBACAF/AsAIAEgB2ohE0IBIdgBA0AgBCATIAsg2AGnIghsIhZqakEAIAhBAnQiAfwLACAQIBZqIAFqIAEgEWoqAgA4AgAg2AFCAXwi2AEg2QFTBEAgBCAIIAxsIBJqakEAIAUgAWv8CwALINgBINkBUg0ACyDdAUIBfCLdASDbAVINAAsg2gFCAXwi2gEg3AFSDQALDAELINsBQnyDId4BINsBQgODId0BA0AgCiAJINoBpyIFbGohASAEIAUgBmxqIQVCACHYAUIAIdkBINsBQgRaBEADQCAFIAAg2AGnIgdsaiABIAMgB2xqKgIAOAIAIAUgACAHQQFyIghsaiABIAMgCGxqKgIAOAIAIAUgACAHQQJyIghsaiABIAMgCGxqKgIAOAIAIAUgACAHQQNyIgdsaiABIAMgB2xqKgIAOAIAINgBQgR8IdgBINkBQgR8ItkBIN4BUg0ACwtCACHZASDdAUIAUgRAA0AgBSAAINgBpyIHbGogASADIAdsaioCADgCACDYAUIBfCHYASDZAUIBfCLZASDdAVINAAsLINoBQgF8ItoBINwBUg0ACwsgAkGQAWokAAwJC0G8wwIoAgAQMBogAkGbITYCiAEgAkHR0wA2AoQBIAJB5CY2AoABQbjDAigCAEHL5AAgAkGAAWoQMQxCC0G8wwIoAgAQMBogAkH9wAA2AnggAkGi0wA2AnQgAkHkJjYCcEG4wwIoAgBBy+QAIAJB8ABqEDEMQQtBvMMCKAIAEDAaIAJBiMAANgJoIAJBrNMANgJkIAJB5CY2AmBBuMMCKAIAQcvkACACQeAAahAxDEALQbzDAigCABAwGiACQcI9NgJYIAJBrdMANgJUIAJB5CY2AlBBuMMCKAIAQcvkACACQdAAahAxDD8LQbzDAigCABAwGiACQcs/NgJIIAJBrtMANgJEIAJB5CY2AkBBuMMCKAIAQcvkACACQUBrEDEMPgtBvMMCKAIAEDAaIAJBujs2AjggAkGv0wA2AjQgAkHkJjYCMEG4wwIoAgBBy+QAIAJBMGoQMQw9C0G8wwIoAgAQMBogAkGlOjYCKCACQbDTADYCJCACQeQmNgIgQbjDAigCAEHL5AAgAkEgahAxDDwLQbzDAigCABAwGiACQbbRADYCGCACQbLTADYCFCACQeQmNgIQQbjDAigCAEHL5AAgAkEQahAxDDsLQbzDAigCABAwGiACQd3QADYCCCACQbPTADYCBCACQeQmNgIAQbjDAigCAEHL5AAgAhAxDDoLDCULIAEoAowBIQQjAEEQayICJAACQCAEKAIARQRAIAAgBCABQwAAgP8Q7QQgAkEQaiQADAELQbzDAigCABAwGiACQZshNgIIIAJBldQANgIEDCsLDCQLIAEoAowBIQQjAEEQayICJAACQCAEKAIARQRAIAAgBCABQwAAAAAQ7QQgAkEQaiQADAELQbzDAigCABAwGiACQZshNgIIIAJBpdQANgIEDCoLDCMLIAEoAowBIQsgASgCkAEhDCABIQMjAEEQayINJAACQCALKAIARQRAAkACQCAAKAIADgMBAAEACyAAKAIIIQEgACgCBCECIAwEfiAMKQMYBUIBCyHaASABIAspAyggCykDICALKQMYfn6nIgRqQQFrIAFtIgEgAmwiBiABIAZqIgEgBCABIARIGyIJTg0AIAsoAhAiB0FwcSIBIAdBD3EiDyAHQQNxIgprIg5qIQQgACgCECAHQRBqIAJsQQJ0aiIFIAdBAnQiGmohGSAHQXxxIQIgB0EBayEQIAFBAWsiEUEEdkEBaiIAQf7///8BcSESIABBAXEhEyADKgJEIrQC/RMhkgIgCawh2QEgBqwh2AEgB0EISSEWA0AgAygCNCG8ASADKALQASEXIAsoAjQhACALKALQASEGIAwEfyAMKALQASAMKAI0INgBINoBgadsagVBAAshCSDYAachFQJAIAdBAEwiGA0AIAYgACAVbGohCEEAIQZBACEAAkAgFg0AIAUgCGtBEEkNAANAIAUgAEECdCIbaiAIIBtq/QACAP0LAgAgAEEEaiIAIAJHDQALIAcgAiIARg0BCyAQIABrIbsBIAoEQANAIAUgAEECdCIcaiAIIBxqKgIAOAIAIABBAWohACAGQQFqIgYgCkcNAAsLILsBQQNJDQADQCAFIABBAnQiBmogBiAIaioCADgCACAFIAZBBGoiG2ogCCAbaioCADgCACAFIAZBCGoiG2ogCCAbaioCADgCACAFIAZBDGoiBmogBiAIaioCADgCACAAQQRqIgAgB0cNAAsLAkAgAUEATCIbDQBBACEIQQAhBiARQQ9HBEADQCAFIAhBAnQiHGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgBSAcQcAAcmoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCEEgaiEIIAZBAmoiBiASRw0ACwsgE0UNACAFIAhBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCyC8ASAVbCG9AQJAIAEgB04iFA0AQQAhCCABIQAgD0EETwRAA0AgBSABIAhqQQJ0aiIAIJICIAD9AAIA/eYB/QsCACAIQQRqIgggDkcNAAsgBCEAIApFDQELA0AgBSAAQQJ0aiIIILQCIAgqAgCUOAIAIABBAWoiACAHRw0ACwsgvQEgF2ohCAJAAkACQCAJRQ0AIBgNAUEAIQZBACEAAkAgFg0AIAkgGUkgCSAaaiAFS3ENAANAIAUgAEECdCIVaiIXIAkgFWr9AAIAIBf9AAIA/eQB/QsCACAAQQRqIgAgAkcNAAsgByACIgBGDQELIABBf3MgB2ohvgEgCgRAA0AgBSAAQQJ0IhdqIhwgCSAXaioCACAcKgIAkjgCACAAQQFqIQAgBkEBaiIGIApHDQALCyC+AUEDSQ0AA0AgBSAAQQJ0IgZqIhUgBiAJaioCACAVKgIAkjgCACAFIAZBBGoiFWoiFyAJIBVqKgIAIBcqAgCSOAIAIAUgBkEIaiIVaiIXIAkgFWoqAgAgFyoCAJI4AgAgBSAGQQxqIgZqIhUgBiAJaioCACAVKgIAkjgCACAAQQRqIgAgB0cNAAsLIBgNAEMAAID/IbICQQAhCUEAIQBBACEVIBBBA08EQANAILICIAUgAEECdCIGaioCACKzAiCyAiCzAl4bIrICIAUgBkEEcmoqAgAiswIgsgIgswJeGyKyAiAFIAZBCHJqKgIAIrMCILICILMCXhsisgIgBSAGQQxyaioCACKzAiCyAiCzAl4bIbICIABBBGohACAVQQRqIhUgAkcNAAsLIAoEQANAILICIAUgAEECdGoqAgAiswIgsgIgswJeGyGyAiAAQQFqIQAgCUEBaiIJIApHDQALC0QAAAAAAAAAACG4AkEAIQADQEMAAAAAIbMCIAUgAEECdCIGaioCACK1AkMAAID/XARAILgCQYD8ASC1AiCyApMiswKLQwAAgHeUQwAAgAiUQYCAgIgHILMCvCIJQQF0IhVBgICAeHEiGCAYQYCAgIgHTRtBAXZBgICAPGq+krwiGEENdkGA+AFxIBhB/x9xaiAVQYCAgHhLGyAJQRB2QYCAAnFyQQF0QZDWLGovAQBBAnRBkNYEaioCACKzArugIbgCCyAGIAhqILMCOAIAIAcgAEEBaiIARw0ACwwBC0QAAAAAAAAAACG4AgtEAAAAAAAA8D8guAKjtiGyAgJAIBsNACCyAv0TIZMCQQAhBkEAIQkgEUEPRwRAA0AgCCAGQQJ0IhVqIgAgkwIgAP0AAAD95gH9CwAAIAAgkwIgAP0AABD95gH9CwAQIAAgkwIgAP0AACD95gH9CwAgIAAgkwIgAP0AADD95gH9CwAwIAggFUHAAHJqIgAgkwIgAP0AAAD95gH9CwAAIAAgkwIgAP0AABD95gH9CwAQIAAgkwIgAP0AACD95gH9CwAgIAAgkwIgAP0AADD95gH9CwAwIAZBIGohBiAJQQJqIgkgEkcNAAsLIBNFDQAgCCAGQQJ0aiIAIJMCIAD9AAAA/eYB/QsAACAAIJMCIAD9AAAQ/eYB/QsAECAAIJMCIAD9AAAg/eYB/QsAICAAIJMCIAD9AAAw/eYB/QsAMAsCQCAUDQAgASEAIA9BBE8EQCCyAv0TIZMCQQAhAANAIAggACABakECdGoiBiAG/QACACCTAv3mAf0LAgAgAEEEaiIAIA5HDQALIAQhACAKRQ0BCwNAIAggAEECdGoiBiAGKgIAILIClDgCACAAQQFqIgAgB0cNAAsLINgBQgF8ItgBINkBUg0ACwsgDUEQaiQADAELQbzDAigCABAwGiANQZshNgIIIA1Bj9UANgIEIA1B5CY2AgBBuMMCKAIAQcvkACANEDEMNwsMIgsgASgCjAEhAiABKAKQASEEIwBB4ABrIgMkAAJAAkACQAJAAkACQCACKAIARQRAIAIoAjBBBEcNASACKAI0IgytItkBIAIpAxAi2AFCAoZSDQEgAjUCOCLbASDZASACKQMYItoBflINASACNQI8INsBIAIpAyAi2QF+Ug0BIAQoAjAiBSAEKAIAQSRsIgdBmJsBaigCAEcNAiAEKAI0Ig2tItsBIAQpAxAi3AEgBa1+IAdBlJsBajQCAH9SDQIgBDUCOCLdASAEKQMYIt4BINsBflINAiAENQI8IN0BIAQpAyAi3wF+Ug0CIAEoAjAiBSABKAIAQSRsIgdBmJsBaigCAEcNAyABKAI0Ig+tItsBIAEpAxAi3QEgBa1+IAdBlJsBajQCAH9SDQMgATUCOCLgASDbASABKQMYIuEBflINAyABNQI8IOABIAEpAyAi2wF+Ug0DINgBIN0BUg0EINoBIOEBUg0EINkBINsBUg0EIAIpAygi2wEgASkDKFINBCDYASDcAVINBSDaASDeAVINBSDZASDfAVINBSAEKQMoINsBUg0FAkACQCAAKAIADgMBAAEACyDZASDaAX4g2wF+pyIFIAAoAggiB2pBAWsgB20iByAAKAIEIgZsIgogByAKaiIAIAUgACAFSBsiFk4NACAGIA1sIAdsIhUg2AGnIgVBAnQiAGohGCAGIA9sIAdsIhIgAGohGiAFQQNxIQggBUF8cSEAIAIoAtABIhkgBiAMbCAHbGohFCASIAEoAtABIg5qIRcgBCgC0AEhECAFQXBxIgRBAEwhGyAEQX9zIAVqQQNJIRwDQCAQIAogDWxqIQYgGSAKIAxsaiEJ/QwAAAAAAAAAAAAAAAAAAAAAIpICIZMC/QwAAAAAAAAAAAAAAAAAAAAAIZQC/QwAAAAAAAAAAAAAAAAAAAAAIZUCQQAhAiAbRQRAA0AgkgIgBiACQQJ0IgdqIgH9AAAwIAcgCWoiB/0AADD95gH95AEhkgIgkwIgAf0AACAgB/0AACD95gH95AEhkwIglAIgAf0AABAgB/0AABD95gH95AEhlAIglQIgAf0AAAAgB/0AAAD95gH95AEhlQIgAkEQaiICIARIDQALIJUCIJMC/eQBIJQCIJIC/eQB/eQBIZICCyCSAv0fAyCSAv0fAiCSAv0fACCSAv0fAZKSkiGyAgJAIAQgBU4NAEEAIQIgBCEBIAgEQANAIAYgAUECdCIHaioCACAHIAlqKgIAlCCyApIhsgIgAUEBaiEBIAJBAWoiAiAIRw0ACwsgHA0AA0AgBiABQQJ0IgJBDGoiB2oqAgAgByAJaioCAJQgBiACQQhqIgdqKgIAIAcgCWoqAgCUIAYgAkEEaiIHaioCACAHIAlqKgIAlCACIAZqKgIAIAIgCWoqAgCUILICkpKSkiGyAiABQQRqIgEgBUcNAAsLAkAgBUEATA0AIAsgD2whESAOIAogD2xqIQdBACEBQQAhAgJAAkAgBUEESSITDQAgESAXaiAUIAsgDGxqa0EQSQ0AA0AgByACQQJ0Ih1qIAkgHWr9AAIA/QsCACACQQRqIgIgAEcNAAsgBSAAIgJGDQELIAJBf3MgBWohvwEgCARAA0AgByACQQJ0Ih9qIAkgH2oqAgA4AgAgAkEBaiECIAFBAWoiASAIRw0ACwsgvwFBA0kNAANAIAcgAkECdCIBaiABIAlqKgIAOAIAIAcgAUEEaiIdaiAJIB1qKgIAOAIAIAcgAUEIaiIdaiAJIB1qKgIAOAIAIAcgAUEMaiIBaiABIAlqKgIAOAIAIAJBBGoiAiAFRw0ACwtBACEBAkAgE0UEQCCyAv0TIZICA0AgByABQQJ0aiICIAL9AAIAIJIC/eUB/QsCACABQQRqIgEgAEcNAAsgBSAAIgFGDQELA0AgByABQQJ0aiICIAIqAgAgsgKTOAIAIAFBAWoiASAFRw0ACwtBACECQQAhAQJAIBMNACAOIBEgEmpqIBAgGCALIA1sIglqakkgECAJIBVqaiAOIBEgGmpqSXENAANAIAcgAUECdCIJaiIRIBH9AAIAIAYgCWr9AAIA/eYB/QsCACABQQRqIgEgAEcNAAsgBSAAIgFGDQELIAFBf3MgBWohwAEgCARAA0AgByABQQJ0IhFqIhMgEyoCACAGIBFqKgIAlDgCACABQQFqIQEgAkEBaiICIAhHDQALCyDAAUEDSQ0AA0AgByABQQJ0IgJqIgkgCSoCACACIAZqKgIAlDgCACAHIAJBBGoiCWoiESARKgIAIAYgCWoqAgCUOAIAIAcgAkEIaiIJaiIRIBEqAgAgBiAJaioCAJQ4AgAgByACQQxqIgJqIgkgCSoCACACIAZqKgIAlDgCACABQQRqIgEgBUcNAAsLIAtBAWohCyAKQQFqIgogFkcNAAsLIANB4ABqJAAMBgtBvMMCKAIAEDAaIANBmyE2AlggA0Hv1QA2AlQgA0HkJjYCUEG4wwIoAgBBy+QAIANB0ABqEDEMOwtBvMMCKAIAEDAaIANB39kANgJIIANBm9UANgJEIANB5CY2AkBBuMMCKAIAQcvkACADQUBrEDEMOgtBvMMCKAIAEDAaIANBu9gANgI4IANBnNUANgI0IANB5CY2AjBBuMMCKAIAQcvkACADQTBqEDEMOQtBvMMCKAIAEDAaIANB9MsANgIoIANBndUANgIkIANB5CY2AiBBuMMCKAIAQcvkACADQSBqEDEMOAtBvMMCKAIAEDAaIANBuc8ANgIYIANBntUANgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEMNwtBvMMCKAIAEDAaIANBns4ANgIIIANBn9UANgIEIANB5CY2AgBBuMMCKAIAQcvkACADEDEMNgsMIQsgASgCjAEhBCABKAKQASEDIwBBEGsiAiQAAkACQAJAAkAgBCgCAA4CAAIBCyAAIAQgAyABQQEQ7AQMAgtBvMMCKAIAEDAaIAJBmyE2AgggAkHR2gA2AgQMKAsgACAEIAMgAUEBEOsECyACQRBqJAAMIAsgASgCjAEhBCABKAKQASEDIwBBEGsiAiQAAkACQAJAAkAgBCgCAA4CAAIBCyAAIAQgAyABQQAQ7AQMAgtBvMMCKAIAEDAaIAJBmyE2AgggAkHo2gA2AgQMJwsgACAEIAMgAUEAEOsECyACQRBqJAAMHwsgASgCjAEhBCMAQdAAayICJAACQAJAAkACQAJAAkAgBCgCACIDQRNLDQAgAwRAIANBAUcEQEEBIAN0Qcz/P3FFDQJBvMMCKAIAEDAaIAJBmyE2AkggAkGM1wA2AkQMLgsCQCAAKAIADgMCAAIACyAEKQMoIAQpAyAi2AEgBCkDGCLaAX5+pyDaAaciA20hBSAEKAIwQQJHDQIgASgCSCIAINgBp0cNAyAEKAI4IQkgBCgCNCEKIAQoAhAhByABKgJMIrICQwAAAL+UQQECfyAAtxCnBJwiuAKZRAAAAAAAAOBBYwRAILgCqgwBC0GAgICAeAt0IgayIrMClRD9ASG0AiCyAowgswKVEP0BIbMCIAdBAEwNASADQQBMDQEgBUEATA0BIAEoAtABIQggBCgC0AEhC0EAIQEDQCAIIAFBAXQiAGohDCAAIAtqIQ0gAbIhtQJBACEEA0AgDCAEIApsIgBqIQ8gACANaiEOQQAhAANAIA8gACAJbCIQaiHBASAOIBBqIRACQCAAIAZOBEAgtAIgACAGa0EBdEEBcrIQtAEhsgIgAEEBaiEADAELILMCIABBAWoiALIQtAEhsgILIMEBILUCILIClCAQLwEAQQJ0QZDWBGoqAgCSOAIAIAAgBUcNAAsgBEEBaiIEIANHDQALIAFBAWoiASAHRw0ACwwBCwJAIAAoAgAOAwEAAQALIAQpAyggBCkDICLYASAEKQMYItkBfn4g2QF/IdwBIAQoAjBBBEcNAyDYASABKAJIIgCsUg0EIAQoAjghAyAEKAI0IQUgBCkDECHdASABKgJMIrICQwAAAL+UQQECfyAAtxCnBJwiuAKZRAAAAAAAAOBBYwRAILgCqgwBC0GAgICAeAt0IgCyIrMClRD9ASG0AiCyAowgswKVEP0BIbMCIN0BQgBXDQAg2QFCAFcNACDcAUIAVw0AIACsId4BIAEoAtABIQAgBCgC0AEhAQNAIAAg2gGnQQJ0IgRqIQcgASAEaiEEINoBtCG1AkIAIdsBA0AgByAFINsBp2wiBmohCSAEIAZqIQZCACHYAQNAIAkgAyDYAadsIgpqIcIBIAYgCmohCgJAINgBIN4BWQRAILQCINgBIN4BfUIBhkIBhLQQtAEhsgIg2AFCAXwh2AEMAQsgswIg2AFCAXwi2AG0ELQBIbICCyDCASC1AiCyApQgCioCAJI4AgAg2AEg3AFSDQALINsBQgF8ItsBINkBUg0ACyDaAUIBfCLaASDdAVINAAsLIAJB0ABqJAAMBAtBvMMCKAIAEDAaIAJBw9IANgIYIAJBzdYANgIUDCcLQbzDAigCABAwGiACQaw7NgIIIAJBz9YANgIEDCcLQbzDAigCABAwGiACQd3QADYCOCACQZLWADYCNAwnC0G8wwIoAgAQMBogAkGsOzYCKCACQZPWADYCJAwjCwweCyABKAKMASEDIwBBMGsiBCQAAkACQAJAAkAgAygCACICQRNLDQAgAgRAQQEgAnRBzv8/cUUNAUG8wwIoAgAQMBogBEGbITYCKCAEQdbXADYCJAw0CwJAIAAoAgAOAwEAAQALIAEoAjBBBEcNAiADKAIwQQRHDQEgACgCBCIFIAMpAyggAykDICADKQMYfn6nIgxODQAgAygCECIHQQBMDQBBACAHayENIAdBAXEhDyAHQXxxIQIgAygCNCIJIAAoAggiCmwhDiABKAI0IgggCmwhECADKALQASIRIAUgCWxqIRIgASgC0AEiEyAFIAhsaiEWIAEqAkQisgL9EyGSAiABKgJIIrMC/RMhkwIgB0EESSEVA0AgESAFIAlsaiEBIBMgBSAIbGohA0EAIQACQAJAIBUNACAWIAYgEGxqIBIgBiAObGprQRBJDQADQCADIABBAnQiC2ogkgIgkwIgASALav0AAgD96gH96wH9CwIAIABBBGoiACACRw0ACyACIgAgB0YNAQsgAEF/cyHDASAPBEAgAyAAQQJ0IhhqIAEgGGoqAgAitAIgswIgswIgtAJeGyK0AiCyAiCyAiC0Al0bOAIAIABBAXIhAAsgwwEgDUYNAANAIAMgAEECdCILaiABIAtqKgIAIrQCILMCILMCILQCXhsitAIgsgIgsgIgtAJdGzgCACADIAtBBGoiC2ogASALaioCACK0AiCzAiCzAiC0Al4bIrQCILICILICILQCXRs4AgAgAEECaiIAIAdHDQALCyAGQQFqIQYgBSAKaiIFIAxIDQALCyAEQTBqJAAMAgtBvMMCKAIAEDAaIARBttEANgIIIARBr9cANgIEDCkLQbzDAigCABAwGiAEQd3QADYCGCAEQa7XADYCFAwxCwwdCyABKAKMASECIAEoApABIQMjAEGQAWsiBCQAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgAigCAA4CAQALCyADKAIADQIgASgCAA0DIAIoAjBBAkcNBCADKAIwQQRHDQUgAikDICLfASACKQMQItwBfiLiASACKQMYIuMBfqchByADKQMYIeEBIAMpAxAh4AECQAJAIAAoAgAOAwABAwELIAMoAjQhCiACKAI4IQYgAigCNCEJIAAoAhBBACAAKAIM/AsAIAAoAhAhBQJAIN8BQgBXDQAg4wFCAFcNACDcAUIAVw0AIN8BQgFSINwBQghUciEIINwBQgODIeQBINwBQniDIdoBINwBQgGGIeUBIAmtIeYBIAatIecBIAIoAtABIgutIegBIAWtIekBA0Ag2QEg5wF+IOgBfCHqASDZAUIBhiDpAXwh6wEgCyAGINkBp2xqIQxCACHdAQNAIAwgCSDdAadsaiEAIAUg3QEg4gF+p0EBdGohAkIAId4BQgAh2wFCACHYAQJAIAgg6wEg3QEg5QF+fKcg6gEg3QEg5gF+fKdrQRBJckUEQANAIAIg2QEg2wF8p0EBdGogACDbAadBAXRq/QABAP0LAQAg2wFCCHwi2wEg2gFSDQALINoBItgBINwBUQ0BCyDcASDYAUJ/hXwhiwIg5AFCAFIEQANAIAIg2AEg3wF+INkBfKdBAXRqIAAg2AGnQQF0ai8BADsBACDYAUIBfCHYASDeAUIBfCLeASDkAVINAAsLIIsCQgNUDQADQCACINgBIN8BfiDZAXynQQF0aiAAINgBp0EBdGovAQA7AQAgAiDYAUIBfCLbASDfAX4g2QF8p0EBdGogACDbAadBAXRqLwEAOwEAIAIg2AFCAnwi2wEg3wF+INkBfKdBAXRqIAAg2wGnQQF0ai8BADsBACACINgBQgN8ItsBIN8BfiDZAXynQQF0aiAAINsBp0EBdGovAQA7AQAg2AFCBHwi2AEg3AFSDQALCyDdAUIBfCLdASDjAVINAAsg2QFCAXwi2QEg3wFSDQALCwJAIOEBQgBXDQAg4AFCAFcNACAFIAdBAXRqIQAgAygC0AEhAyDhAUIBUSDgAUIDVnEhBSDgAUJ8gyHaAUIAIdsBA0AgAyAKINsBp2xqIQJCACHZAUIAIdgBAkAgBQRAA0AgACDZASDbAXynQQF0av0MAH4AAAB+AAAAfgAAAH4AACACINkBp0ECdGr9AAIAIpIC/eAB/QwAAIB3AACAdwAAgHcAAIB3/eYB/QwAAIAIAACACAAAgAgAAIAI/eYBIJICQQH9qwEikwL9DAAAAP8AAAD/AAAA/wAAAP/9Tv0MAAAAcQAAAHEAAABxAAAAcf25AUEB/a0B/QwAAIAHAACABwAAgAcAAIAH/a4B/eQBIpQCQQ39rQH9DAB8AAAAfAAAAHwAAAB8AAD9TiCUAv0M/w8AAP8PAAD/DwAA/w8AAP1O/a4BIJMC/QwAAAD/AAAA/wAAAP8AAAD//Tz9UiCSAkEQ/a0B/QwAgAAAAIAAAACAAAAAgAAA/U79UCCSAv0NAAEEBQgJDA0AAQABAAEAAf1bAQAAINkBQgR8ItkBINoBUg0ACyDaASLYASDgAVENAQsDQCAAINgBIOEBfiDbAXynQQF0akGA/AEgAiDYAadBAnRqKgIAIrICi0MAAIB3lEMAAIAIlEGAgICIByCyArwiB0EBdCIGQYCAgHhxIgkgCUGAgICIB00bQQF2QYCAgDxqvpK8IglBDXZBgPgBcSAJQf8fcWogBkGAgIB4SxsgB0EQdkGAgAJxcjsBACDYAUIBfCLYASDgAVINAAsLINsBQgF8ItsBIOEBUg0ACwsgASkDICHYASABKQMYIdoBIAEoAtABQQACfyABKAIAIgBBEGtBcU0EQCAAQSRsQZibAWooAgAgASgCMCABKAIQQQFrbGogASgCNCDaAadBAWtsaiABKAI4INgBp0EBa2xqIAEoAjwgASgCKEEBa2xqDAELIAE1AjQg2gFCAX1+IAEpAxAgATUCMH4gAEEkbEGUmwFqNQIAf3wgATUCOCDYAUIBfX58IAE1AjwgASkDKEIBfX58pwv8CwAMAgsgASkDGKciAiAAKAIIIgNqQQFrIANtIgUgACgCBGwiAyADIAVqIgUgAiACIAVKGyIJTg0BIOABQgBXDQEg3AFCAFcNASABKAI0IQogACgCECINIAdBAXRqIQ8gASgCRCEIIOEBpyEOIAEoAtABIQsg3wGnIgVBcHEiAUEASgRAIAFBAXIhECAFQQFxIREg4gGnIRJBACAFayABQX9zRiETA0AgCyADIApsaiEWIA0gAyASbEEBdGohFUIAIdkBA0AgDyDZAaciACAObEEBdGoiDCABQQF0IhhqIRogACAIbCEZQgAh2AEDQCAVINgBpyIUIAVsQQF0aiEGQQAhB/0MAAAAAAAAAAAAAAAAAAAAACKSAiGTAv0MAAAAAAAAAAAAAAAAAAAAACGUAv0MAAAAAAAAAAAAAAAAAAAAACGVAgNAIJICIAwgB0EBdCICaiIALwEeQQJ0QZDWBGogAC8BHEECdEGQ1gRqIAAvARpBAnRBkNYEaiAALwEYQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAIgBmoiAi8BHkECdEGQ1gRqIAIvARxBAnRBkNYEaiACLwEaQQJ0QZDWBGogAi8BGEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGSAiCTAiAALwEWQQJ0QZDWBGogAC8BFEECdEGQ1gRqIAAvARJBAnRBkNYEaiAALwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAIvARZBAnRBkNYEaiACLwEUQQJ0QZDWBGogAi8BEkECdEGQ1gRqIAIvARBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhkwIglAIgAC8BDkECdEGQ1gRqIAAvAQxBAnRBkNYEaiAALwEKQQJ0QZDWBGogAC8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyACLwEOQQJ0QZDWBGogAi8BDEECdEGQ1gRqIAIvAQpBAnRBkNYEaiACLwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZQCIJUCIAAvAQZBAnRBkNYEaiAALwEEQQJ0QZDWBGogAC8BAkECdEGQ1gRqIAAvAQBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgAi8BBkECdEGQ1gRqIAIvAQRBAnRBkNYEaiACLwECQQJ0QZDWBGogAi8BAEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGVAiAHQRBqIgcgAUgNAAsglQIgkwL95AEglAIgkgL95AH95AEikgL9HwMgkgL9HwIgkgL9HwAgkgL9HwGSkpIhsgIgASAFSARAILICuyG4AiARBH8guAIgGi8BAEECdEGQ1gRqKgIAIAYgGGovAQBBAnRBkNYEaioCAJS7oCG4AiAQBSABCyEAIBNFBEADQCC4AiAMIABBAXQiAmovAQBBAnRBkNYEaioCACACIAZqLwEAQQJ0QZDWBGoqAgCUu6AgDCACQQJqIgJqLwEAQQJ0QZDWBGoqAgAgAiAGai8BAEECdEGQ1gRqKgIAlLugIbgCIABBAmoiACAFRw0ACwsguAK2IbICCyAWIBQgGWpBAnRqIgAgsgIgACoCAJI4AgAg2AFCAXwi2AEg3AFSDQALINkBQgF8ItkBIOABUg0ACyADQQFqIgMgCUcNAAsMAgsgASAFTgRAINwBQnyDIdoBIAitId0BA0AgCyADIApsaiEAQgAh3gEDQCDdASDeAX4h2wFCACHZAUIAIdgBAkAg3AFCBFoEQANAIAAg2QEg2wF8p0ECdGoiASAB/QACAP0MAAAAAAAAAAAAAAAAAAAAAP3kAf0LAgAg2QFCBHwi2QEg2gFSDQALINoBItgBINwBUQ0BCwNAIAAg2AEg2wF8p0ECdGoiASABKgIAQwAAAACSOAIAINgBQgF8ItgBINwBUg0ACwsg3gFCAXwi3gEg4AFSDQALIANBAWoiAyAJRw0ACwwCCyABQQFyIQIgBUEBcSEQIOIBpyERQQAgBWsgAUF/c0YhEgNAIAsgAyAKbGohEyANIAMgEWxBAXRqIRZCACHZAQNAIA8g2QGnIgAgDmxBAXRqIgYgAUEBdCIVaiEYIAAgCGwhGkIAIdgBA0AgFiDYAaciGSAFbEEBdGohB0QAAAAAAAAAACG4AiABIQAgEARAIBgvAQBBAnRBkNYEaioCACAHIBVqLwEAQQJ0QZDWBGoqAgCUu0QAAAAAAAAAAKAhuAIgAiEACyASRQRAA0AguAIgBiAAQQF0IgxqLwEAQQJ0QZDWBGoqAgAgByAMai8BAEECdEGQ1gRqKgIAlLugIAYgDEECaiIMai8BAEECdEGQ1gRqKgIAIAcgDGovAQBBAnRBkNYEaioCAJS7oCG4AiAAQQJqIgAgBUcNAAsLIBMgGSAaakECdGoiACAAKgIAILgCtpI4AgAg2AFCAXwi2AEg3AFSDQALINkBQgF8ItkBIOABUg0ACyADQQFqIgMgCUcNAAsMAQsgAygCAA0FIAEoAgANBiACKAIwQQRHDQcgAygCMEEERw0IIAIpAyAi4AEgAikDECLcAX4i4gEgAikDGCLjAX6nIQcgAykDGCHhASADKQMQId8BAkACQCAAKAIADgMAAQIBCyADKAI0IQogAigCOCEGIAIoAjQhCSAAKAIQQQAgACgCDPwLACAAKAIQIQUCQCDgAUIAVw0AIOMBQgBXDQAg3AFCAFcNACDgAUIBUiDcAUIIVHIhCCDcAUIDgyHkASDcAUJ8gyHaASDcAUIChiHlASAJrSHmASAGrSHnASACKALQASILrSHoASAFrSHpAQNAINkBIOcBfiDoAXwh6gEg2QFCAoYg6QF8IesBIAsgBiDZAadsaiEMQgAh3QEDQCAMIAkg3QGnbGohACAFIN0BIOIBfqdBAnRqIQJCACHeAUIAIdsBQgAh2AECQCAIIOsBIN0BIOUBfnynIOoBIN0BIOYBfnyna0EQSXJFBEADQCACINkBINsBfKdBAnRqIAAg2wGnQQJ0av0AAgD9CwIAINsBQgR8ItsBINoBUg0ACyDaASLYASDcAVENAQsg3AEg2AFCf4V8IYwCIOQBQgBSBEADQCACINgBIOABfiDZAXynQQJ0aiAAINgBp0ECdGoqAgA4AgAg2AFCAXwh2AEg3gFCAXwi3gEg5AFSDQALCyCMAkIDVA0AA0AgAiDYASDgAX4g2QF8p0ECdGogACDYAadBAnRqKgIAOAIAIAIg2AFCAXwi2wEg4AF+INkBfKdBAnRqIAAg2wGnQQJ0aioCADgCACACINgBQgJ8ItsBIOABfiDZAXynQQJ0aiAAINsBp0ECdGoqAgA4AgAgAiDYAUIDfCLbASDgAX4g2QF8p0ECdGogACDbAadBAnRqKgIAOAIAINgBQgR8ItgBINwBUg0ACwsg3QFCAXwi3QEg4wFSDQALINkBQgF8ItkBIOABUg0ACwsCQCDhAUIAVw0AIN8BQgBXDQAgBSAHQQJ0aiEAIAMoAtABIQMg4QFCAVIg3wFCCFRyIQYg3wFCA4Mh3AEg3wFCfIMh2gFCACHZAQNAQgAh2wFCACHYAQJAIAYgByDZAaciAmpBAnQgBWogAiAKbCADaiICa0EQSXJFBEADQCAAINkBINsBfKdBAnRqIAIg2wGnQQJ0av0AAgD9CwIAINsBQgR8ItsBINoBUg0ACyDaASLYASDfAVENAQsg3wEg2AFCf4V8IY0CQgAh2wEg3AFCAFIEQANAIAAg2AEg4QF+INkBfKdBAnRqIAIg2AGnQQJ0aioCADgCACDYAUIBfCHYASDbAUIBfCLbASDcAVINAAsLII0CQgNUDQADQCAAINgBIOEBfiDZAXynQQJ0aiACINgBp0ECdGoqAgA4AgAgACDYAUIBfCLbASDhAX4g2QF8p0ECdGogAiDbAadBAnRqKgIAOAIAIAAg2AFCAnwi2wEg4QF+INkBfKdBAnRqIAIg2wGnQQJ0aioCADgCACAAINgBQgN8ItsBIOEBfiDZAXynQQJ0aiACINsBp0ECdGoqAgA4AgAg2AFCBHwi2AEg3wFSDQALCyDZAUIBfCLZASDhAVINAAsLIAEpAyAh2AEgASkDGCHaASABKALQAUEAAn8gASgCACIAQRBrQXFNBEAgAEEkbEGYmwFqKAIAIAEoAjAgASgCEEEBa2xqIAEoAjQg2gGnQQFrbGogASgCOCDYAadBAWtsaiABKAI8IAEoAihBAWtsagwBCyABNQI0INoBQgF9fiABKQMQIAE1AjB+IABBJGxBlJsBajUCAH98IAE1Ajgg2AFCAX1+fCABNQI8IAEpAyhCAX1+fKcL/AsADAELIAEpAxinIgIgACgCCCIDakEBayADbSIFIAAoAgRsIgMgAyAFaiIFIAIgAiAFShsiCU4NACDfAUIAVw0AINwBQgBXDQAgASgCNCEKIAAoAhAiDCAHQQJ0aiENIAEoAkQhCCDhAachDyABKALQASELIOABpyICQXBxIgFBAEoEQCDiAachECABIAJOBEADQCALIAMgCmxqIQYgDCADIBBsQQJ0aiEOQgAh2QEDQCAIINkBpyIAbCERIA0gACAPbEECdGohEkIAIdgBA0AgDiDYAaciEyACbEECdGohFkEAIQf9DAAAAAAAAAAAAAAAAAAAAAAikgIhkwL9DAAAAAAAAAAAAAAAAAAAAAAhlAL9DAAAAAAAAAAAAAAAAAAAAAAhlQIDQCCSAiASIAdBAnQiBWoiAP0AADAgBSAWaiIF/QAAMP3mAf3kASGSAiCTAiAA/QAAICAF/QAAIP3mAf3kASGTAiCUAiAA/QAAECAF/QAAEP3mAf3kASGUAiCVAiAA/QAAACAF/QAAAP3mAf3kASGVAiAHQRBqIgcgAUgNAAsgBiARIBNqQQJ0aiIAIJUCIJMC/eQBIJQCIJIC/eQB/eQBIpIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIAAqAgCSOAIAINgBQgF8ItgBINwBUg0ACyDZAUIBfCLZASDfAVINAAsgA0EBaiIDIAlHDQAMAwsACyACQQNxIREgAUF/cyACakEDSSESA0AgCyADIApsaiETIAwgAyAQbEECdGohFkIAIdkBA0AgCCDZAaciAGwhFSANIAAgD2xBAnRqIQVCACHYAQNAIBYg2AGnIhggAmxBAnRqIQdBACEG/QwAAAAAAAAAAAAAAAAAAAAAIpICIZMC/QwAAAAAAAAAAAAAAAAAAAAAIZQC/QwAAAAAAAAAAAAAAAAAAAAAIZUCA0AgkgIgBSAGQQJ0Ig5qIgD9AAAwIAcgDmoiDv0AADD95gH95AEhkgIgkwIgAP0AACAgDv0AACD95gH95AEhkwIglAIgAP0AABAgDv0AABD95gH95AEhlAIglQIgAP0AAAAgDv0AAAD95gH95AEhlQIgBkEQaiIGIAFIDQALIJUCIJMC/eQBIJQCIJIC/eQB/eQBIpIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIbICQQAhBiABIQAgEQRAA0AgBSAAQQJ0Ig5qKgIAIAcgDmoqAgCUILICkiGyAiAAQQFqIQAgBkEBaiIGIBFHDQALCyASRQRAA0AgBSAAQQJ0IgZBDGoiDmoqAgAgByAOaioCAJQgBSAGQQhqIg5qKgIAIAcgDmoqAgCUIAUgBkEEaiIOaioCACAHIA5qKgIAlCAFIAZqKgIAIAYgB2oqAgCUILICkpKSkiGyAiAAQQRqIgAgAkcNAAsLIBMgFSAYakECdGoiACCyAiAAKgIAkjgCACDYAUIBfCLYASDcAVINAAsg2QFCAXwi2QEg3wFSDQALIANBAWoiAyAJRw0ACwwBCyABIAJOBEAg3AFCfIMh2gEgCK0h3QEDQCALIAMgCmxqIQBCACHeAQNAIN0BIN4BfiHbAUIAIdkBQgAh2AECQCDcAUIEWgRAA0AgACDZASDbAXynQQJ0aiIBIAH9AAIA/QwAAAAAAAAAAAAAAAAAAAAA/eQB/QsCACDZAUIEfCLZASDaAVINAAsg2gEi2AEg3AFRDQELA0AgACDYASDbAXynQQJ0aiIBIAEqAgBDAAAAAJI4AgAg2AFCAXwi2AEg3AFSDQALCyDeAUIBfCLeASDfAVINAAsgA0EBaiIDIAlHDQALDAELIAJBA3EhDiDiAachECABQX9zIAJqQQNJIREDQCALIAMgCmxqIRIgDCADIBBsQQJ0aiETQgAh2QEDQCAIINkBpyIAbCEWIA0gACAPbEECdGohBUIAIdgBA0AgEyDYAaciFSACbEECdGohBkEAIQdDAAAAACGyAiABIQAgDgRAA0AgBSAAQQJ0IhhqKgIAIAYgGGoqAgCUILICkiGyAiAAQQFqIQAgB0EBaiIHIA5HDQALCyARRQRAA0AgBSAAQQJ0IgdBDGoiGGoqAgAgBiAYaioCAJQgBSAHQQhqIhhqKgIAIAYgGGoqAgCUIAUgB0EEaiIYaioCACAGIBhqKgIAlCAFIAdqKgIAIAYgB2oqAgCUILICkpKSkiGyAiAAQQRqIgAgAkcNAAsLIBIgFSAWakECdGoiACCyAiAAKgIAkjgCACDYAUIBfCLYASDcAVINAAsg2QFCAXwi2QEg3wFSDQALIANBAWoiAyAJRw0ACwsgBEGQAWokAAwJC0G8wwIoAgAQMBogBEGlPDYCSCAEQfXaADYCRCAEQeQmNgJAQbjDAigCAEHL5AAgBEFAaxAxDDkLQbzDAigCABAwGiAEQYo8NgI4IARB9toANgI0IARB5CY2AjBBuMMCKAIAQcvkACAEQTBqEDEMOAtBvMMCKAIAEDAaIARBl9MANgIoIARBgtsANgIkIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEMNwtBvMMCKAIAEDAaIARBidEANgIYIARBg9sANgIUIARB5CY2AhBBuMMCKAIAQcvkACAEQRBqEDEMNgtBvMMCKAIAEDAaIARBpTw2AogBIARB09sANgKEASAEQeQmNgKAAUG4wwIoAgBBy+QAIARBgAFqEDEMNQtBvMMCKAIAEDAaIARBijw2AnggBEHU2wA2AnQgBEHkJjYCcEG4wwIoAgBBy+QAIARB8ABqEDEMNAtBvMMCKAIAEDAaIARBttEANgJoIARB4NsANgJkIARB5CY2AmBBuMMCKAIAQcvkACAEQeAAahAxDDMLQbzDAigCABAwGiAEQYnRADYCWCAEQeHbADYCVCAEQeQmNgJQQbjDAigCAEHL5AAgBEHQAGoQMQwyC0G8wwIoAgAQMBogBEGbITYCCCAEQbvcADYCBCAEQeQmNgIAQbjDAigCAEHL5AAgBBAxDDELDBwLIAEoAowBIQUgASgCkAEhBCMAQeAAayICJAACQAJAAkACQAJAAkAgBSgCACIDQQFHBEAgAw0BQbzDAigCABAwGiACQZshNgJYIAJBod0ANgJUIAJB5CY2AlBBuMMCKAIAQcvkACACQdAAahAxDDYLIAQoAgANASABKAIAQQFHDQIgBSgCMEECRw0DIAQoAjBBBEcNBAJAAkAgACgCAA4DAQABAAsgBCkDKCAEKQMgItgBIAEoAlxBAUYiAxsi5wFCAFcNACABKQMgQgEgAxsi4QFCAFcNACABKQMYIuIBQgBXDQAg2AEgBCkDGCLYASADGyLjASAANAIEItoBVw0AIAUpAxhCASADGyLfAUIAVw0AIAUpAxAi2QFCAFcNACAEKAI4IgUgBCgCNCADGyEHIAQoAjwgBSADGyEFINgBQgEgAxsh6AEgATQCWCHpASABNAJUIeoBIAE0AlAh6wEgATQCTCHsASABNAJIIe0BIAE0AkQh7gEgBCkDECHlASABKALQASEBINkBIN8BfiLYASDjAX4h7wEgBCgC0AEhBCDYASAANAIIIvABfiHxASDZAadBAXQhAyDYASDaAX5CAYZC/v///w+DIfIBINkBIN8BIOMBfn4i2AFCAYZC/v///w+DIfMBINgBIOIBfkIBhiLYAUL+////D4Mh9AEg2AEg4QF+Qv7///8PgyH1AQNAINsBIOEBfiH2ASDbASD1AX4g8gF8IfcBIAQgBSDbAadsaiEGQgAh3AEDQCD3ASDcASD0AX58IfgBINwBIO0BfiDrAX0h+gEg3AEg9gF8IOIBfiH5AUIAId0BA0Ag+AEg3QEg8wF+fCH7ASDdASDuAX4g7AF9IfwBIAEg7wEg3QEg+QF8fqdBAXRqIQlCACHkASDaASHYAQNAINgBIN8BfiH9ASDkASDxAX4h/gEgBiAHINgBp2xqIQpCACHeAQNAAkACQCDeASDpAX4g+gF8IuABQgBTDQAg4AEg6AFZDQAg3gEg/QF8INkBfiH/ASDgASDlAX4hgAJCACHgAQNAQQAhAAJAIOABIOoBfiD8AXwi5gFCAFMNACDlASDmAVcNAEGA/AEgCiDmASCAAnynQQJ0aioCACKyAotDAACAd5RDAACACJRBgICAiAcgsgK8IgBBAXQiCEGAgIB4cSILIAtBgICAiAdNG0EBdkGAgIA8ar6SvCILQQ12QYD4AXEgC0H/H3FqIAhBgICAeEsbIABBEHZBgIACcXIhAAsgCSDgASD/AXynQQF0aiAAOwEAIOABQgF8IuABINkBUg0ACwwBCyABIPsBINkBIN4BfiD+AXxCAYZ8p2pBACAD/AsACyDeAUIBfCLeASDfAVINAAsg5AFCAXwh5AEg2AEg8AF8ItgBIOMBUw0ACyDdAUIBfCLdASDiAVINAAsg3AFCAXwi3AEg4QFSDQALINsBQgF8ItsBIOcBUg0ACwsgAkHgAGokAAwFC0G8wwIoAgAQMBogAkGbITYCCCACQaXdADYCBCACQeQmNgIAQbjDAigCAEHL5AAgAhAxDDQLQbzDAigCABAwGiACQaU8NgJIIAJBydwANgJEIAJB5CY2AkBBuMMCKAIAQcvkACACQUBrEDEMMwtBvMMCKAIAEDAaIAJBvTk2AjggAkHK3AA2AjQgAkHkJjYCMEG4wwIoAgBBy+QAIAJBMGoQMQwyC0G8wwIoAgAQMBogAkGX0wA2AiggAkHq3AA2AiQgAkHkJjYCIEG4wwIoAgBBy+QAIAJBIGoQMQwxC0G8wwIoAgAQMBogAkGJ0QA2AhggAkHr3AA2AhQgAkHkJjYCEEG4wwIoAgBBy+QAIAJBEGoQMQwwCwwbCyAAIQIgASgCjAEhAyABKAKQASEFIAEhACMAQdAAayIEJAACQCADKAIAQQFGBEAgBSgCAEUEQCAAKAIARQRAIAMoAjBBAkYEQCAFKAIwQQRGBEAgAykDKCLeASAD/QADECKTAv0dACLcAX4i4wEgkwL9HQEi4AF+IuQBIAMpAyAi5QF+pyEGIAUpAyAh3wEgBf0AAxAhkgICQAJAIAIoAgAOAwABBwELIAUoAjghCCAFKAI0IQsgAygCPCEJIAMoAjghCiACKAIQQQAgAigCDPwLACACKAIQIQcCQCDeAUIAVw0AIOUBQgBXDQAgkwL9DAEAAAAAAAAAAQAAAAAAAAD92AEikwL9HQEgkwL9HQCEp0EBcQ0AIN4BQgFSINwBQghUciEMINwBQgGDIeMBINwBQniDIdkBINwBQgGGQv7///8PgyHmASDcASDgAX5CAYYh5wEgCq0h6AEgCa0h6QEgAygC0AEiA60h6gEgB60h6wEDQCDbASDpAX4g6gF8IewBINsBQgGGIOsBfCHtASADIAkg2wGnbGohDUIAId0BA0Ag7AEg3QEg6AF+fCHuASDtASDdASDnAX58Ie8BIA0gCiDdAadsaiEBIAcg3QEg5AF+p0EBdGohAkIAIeEBA0Ag3AEg4QF+IeIBQgAh2AFCACHaAQJAIAwg7wEg4QEg5gF+IvABfKcg7gEg8AF8p2tBEElyRQRAA0AgAiDYASDiAXwi2gEg2wF8p0EBdGogASDaAadBAXRq/QABAP0LAQAg2AFCCHwi2AEg2QFSDQALINkBItoBINwBUQ0BCyDaAUIBhCHYASDjAacEQCACINoBIOIBfCLaASDeAX4g2wF8p0EBdGogASDaAadBAXRqLwEAOwEAINgBIdoBCyDYASDcAVENAANAIAIg2gEg4gF8ItgBIN4BfiDbAXynQQF0aiABINgBp0EBdGovAQA7AQAgAiDYAUIBfCLYASDeAX4g2wF8p0EBdGogASDYAadBAXRqLwEAOwEAINoBQgJ8ItoBINwBUg0ACwsg4QFCAXwi4QEg4AFSDQALIN0BQgF8It0BIOUBUg0ACyDbAUIBfCLbASDeAVINAAsLAkAg3wFCAFcNACCSAv0dASLdAUIAVw0AIJIC/R0AItsBQgBXDQAgByAGQQF0aiEHINsBIN8BfiHeASDfAachASAFKALQASEGIN8BQgFRINsBQgNWcSEJINsBQnyDIdkBQgAh3AEDQCAGIAgg3AGnIgJsaiEKQgAh2gEDQCAKIAsg2gGnbGohAyAHINoBIN4BfqdBAXRqIQVCACHYAQJAIAkEQANAIAUg2AGnIgwgAWwgAmpBAXRq/QwAfgAAAH4AAAB+AAAAfgAAIAMgDEECdGr9AAIAIpIC/eAB/QwAAIB3AACAdwAAgHcAAIB3/eYB/QwAAIAIAACACAAAgAgAAIAI/eYBIJICQQH9qwEikwL9DAAAAP8AAAD/AAAA/wAAAP/9Tv0MAAAAcQAAAHEAAABxAAAAcf25AUEB/a0B/QwAAIAHAACABwAAgAcAAIAH/a4B/eQBIpQCQQ39rQH9DAB8AAAAfAAAAHwAAAB8AAD9TiCUAv0M/w8AAP8PAAD/DwAA/w8AAP1O/a4BIJMC/QwAAAD/AAAA/wAAAP8AAAD//Tz9UiCSAkEQ/a0B/QwAgAAAAIAAAACAAAAAgAAA/U79UCCSAv0NAAEEBQgJDA0AAQABAAEAAf1bAQAAINgBQgR8ItgBINkBUg0ACyDZASLYASDbAVENAQsDQCAFINgBpyIMIAFsIAJqQQF0akGA/AEgAyAMQQJ0aioCACKyAotDAACAd5RDAACACJRBgICAiAcgsgK8IgxBAXQiDUGAgIB4cSIPIA9BgICAiAdNG0EBdkGAgIA8ar6SvCIPQQ12QYD4AXEgD0H/H3FqIA1BgICAeEsbIAxBEHZBgIACcXI7AQAg2AFCAXwi2AEg2wFSDQALCyDaAUIBfCLaASDdAVINAAsg3AFCAXwi3AEg3wFSDQALCyAAKQMoIdgBIAApAxgh2gEgACkDECHZASAAKQMgIdsBIAAoAtABQQACfyAAKAIAIgFBEGtBcU0EQCABQSRsQZibAWooAgAgACgCMCDZAadBAWtsaiAAKAI0INoBp0EBa2xqIAAoAjgg2wGnQQFrbGogACgCPCDYAadBAWtsagwBCyAANQI0INoBQgF9fiDZASAANQIwfiABQSRsQZSbAWo1AgB/fCAANQI4INsBQgF9fnwgADUCPCDYAUIBfX58pwv8CwAMBgsgACkDIKciASACKAIIIgNqQQFrIANtIgMgAigCBGwiByADIAdqIgMgASABIANKGyIRTg0FIJIC/QwBAAAAAAAAAAEAAAAAAAAA/dgBIJMC/QwBAAAAAAAAAAEAAAAAAAAA/dgB/Q0AAQIDCAkKCxAREhMYGRob/VMNBSAAKAI4IRIgAigCECITIAZBAXRqIQ0gACkDEKchCCAAKALQASEWIN4BpyIFQXBxIgFBAXIhAiAFQQFxIQ8g3AFCfIMh2gEgAUF/cyAFaiEOIJIC/R0BIeIBIJIC/R0AIeEBIOQBpyEVIAAoAkQiBq0h5AEgAUEASiEYA0AgFiAHIBJsaiEJIBMgByAVbEEBdGohEEIAId4BAkACQCAYRQRAQgAh3QEgASAFSA0BA0Ag3QEg5AF+IeUBQgAh3gEDQCAGIN4Bp2whA0IAIdkBA0Ag2QEg5QF8pyAIbCADaiEAQgAh2wFCACHYAQJAINwBQgRaBEADQCAJIAAg2wGnakECdGoiCiAK/QACAP0MAAAAAAAAAAAAAAAAAAAAAP3kAf0LAgAg2wFCBHwi2wEg2gFSDQALINoBItgBINwBUQ0BCwNAIAkgACDYAadqQQJ0aiIKIAoqAgBDAAAAAJI4AgAg2AFCAXwi2AEg3AFSDQALCyDZAUIBfCLZASDgAVINAAsg3gFCAXwi3gEg4QFSDQALIN0BQgF8It0BIOIBUg0ACwwCCwNAIN4BIOEBfiHdASAGIN4Bp2whGkIAIdkBA0AgDSDZASDdAXwg3wF+p0EBdGoiCyABQQF0IhlqIRQgBiDZAadsIRdCACHbAQNAIBog2wGnaiAIbCAXaiEbIBAg2wEg4wF+p0EBdGohHEIAIdgBA0AgHCDYAaciHSAFbEEBdGohCkEAIQz9DAAAAAAAAAAAAAAAAAAAAAAikgIhkwL9DAAAAAAAAAAAAAAAAAAAAAAhlAL9DAAAAAAAAAAAAAAAAAAAAAAhlQIDQCCSAiALIAxBAXQiA2oiAC8BHkECdEGQ1gRqIAAvARxBAnRBkNYEaiAALwEaQQJ0QZDWBGogAC8BGEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyADIApqIgMvAR5BAnRBkNYEaiADLwEcQQJ0QZDWBGogAy8BGkECdEGQ1gRqIAMvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhkgIgkwIgAC8BFkECdEGQ1gRqIAAvARRBAnRBkNYEaiAALwESQQJ0QZDWBGogAC8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyADLwEWQQJ0QZDWBGogAy8BFEECdEGQ1gRqIAMvARJBAnRBkNYEaiADLwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZMCIJQCIAAvAQ5BAnRBkNYEaiAALwEMQQJ0QZDWBGogAC8BCkECdEGQ1gRqIAAvAQhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgAy8BDkECdEGQ1gRqIAMvAQxBAnRBkNYEaiADLwEKQQJ0QZDWBGogAy8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGUAiCVAiAALwEGQQJ0QZDWBGogAC8BBEECdEGQ1gRqIAAvAQJBAnRBkNYEaiAALwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAMvAQZBAnRBkNYEaiADLwEEQQJ0QZDWBGogAy8BAkECdEGQ1gRqIAMvAQBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhlQIgDEEQaiIMIAFIDQALIJUCIJMC/eQBIJQCIJIC/eQB/eQBIpIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIbICIAEgBUgEQCCyArshuAIgDwR/ILgCIBQvAQBBAnRBkNYEaioCACAKIBlqLwEAQQJ0QZDWBGoqAgCUu6AhuAIgAgUgAQshACAOBEADQCC4AiALIABBAXQiA2ovAQBBAnRBkNYEaioCACADIApqLwEAQQJ0QZDWBGoqAgCUu6AgCyADQQJqIgNqLwEAQQJ0QZDWBGoqAgAgAyAKai8BAEECdEGQ1gRqKgIAlLugIbgCIABBAmoiACAFRw0ACwsguAK2IbICCyAJIBsgHWpBAnRqIgAgsgIgACoCAJI4AgAg2AFCAXwi2AEg3AFSDQALINsBQgF8ItsBIOABUg0ACyDZAUIBfCLZASDhAVINAAsg4gEg3gFCAXwi3gFSDQALDAELA0Ag3QEg4QF+Id4BIAYg3QGnbCEMQgAh2QEDQCANINkBIN4BfCDfAX6nQQF0aiIKIAFBAXQiGmohGSAGINkBp2whFEIAIdsBA0AgDCDbAadqIAhsIBRqIRcgECDbASDjAX6nQQF0aiEbQgAh2AEDQCAbINgBpyIcIAVsQQF0aiEDRAAAAAAAAAAAIbgCIAEhACAPBEAgGS8BAEECdEGQ1gRqKgIAIAMgGmovAQBBAnRBkNYEaioCAJS7RAAAAAAAAAAAoCG4AiACIQALIA4EQANAILgCIAogAEEBdCILai8BAEECdEGQ1gRqKgIAIAMgC2ovAQBBAnRBkNYEaioCAJS7oCAKIAtBAmoiC2ovAQBBAnRBkNYEaioCACADIAtqLwEAQQJ0QZDWBGoqAgCUu6AhuAIgAEECaiIAIAVHDQALCyAJIBcgHGpBAnRqIgAgACoCACC4AraSOAIAINgBQgF8ItgBINwBUg0ACyDbAUIBfCLbASDgAVINAAsg2QFCAXwi2QEg4QFSDQALIN0BQgF8It0BIOIBUg0ACwsgESAHQQFqIgdHDQALDAULQbzDAigCABAwGiAEQYnRADYCCCAEQcDdADYCBAwpC0G8wwIoAgAQMBogBEGX0wA2AhggBEG/3QA2AhQMMQtBvMMCKAIAEDAaIARBijw2AiggBEGz3QA2AiQMLwtBvMMCKAIAEDAaIARBpTw2AjggBEGy3QA2AjQMLQtBvMMCKAIAEDAaIARB2Dk2AkggBEGx3QA2AkQMKwsgBEHQAGokAAwaCyABKAKMASECIwBBMGsiBCQAAkACQAJAIAEoAlBFBEAgASgCSCIFIAEoAkxHDQECQAJAIAAoAgAOAwEAAQALIAEoAkQhACACKQMgIdgBIAIpAxgh2gEgAigC0AEhAwJ/IAIoAgAiB0EQa0FxTQRAIAdBJGxBmJsBaigCACACKAIwIAIoAhBBAWtsaiACKAI0IgYg2gGnQQFrbGogAigCOCDYAadBAWtsaiACKAI8IAIoAihBAWtsagwBCyACKAI0IgatINoBQgF9fiACKQMQIAI1AjB+IAdBJGxBlJsBajUCAH98IAI1Ajgg2AFCAX1+fCACNQI8IAIpAyhCAX1+fKcLIgJBAEwNACABKQMQItkBQgBXDQAgAEECRg0DIAIgA2ohCSABKALQASECINkBpyEKIAWyIbMCIAVBAEoEQCAAQQFGBEAgBUEDcSELIAVBBEkhDANAQgAh2AFBACEBA0AgAiDYAadBAnRqIgdBADYCAEMAAAAAIbICIAEhAEEAIQggCwRAA0AgByCyAiADIABBAnRqKgIAkiKyAjgCACAAQQFqIQAgCEEBaiIIIAtHDQALCyABIAVqIQEgDEUEQANAIAcgsgIgAyAAQQJ0aiIIKgIAkiKyAjgCACAHILICIAgqAgSSIrICOAIAIAcgsgIgCCoCCJIisgI4AgAgByCyAiAIKgIMkiKyAjgCACAAQQRqIgAgAUcNAAsLIAcgsgIgswKVOAIAINgBQgF8ItgBINkBUg0ACyACIApBAnRqIQIgAyAGaiIDIAlJDQALDAILIAANASAFQQNxIQsgBUEESSEMA0BBACEBQgAh2AEDQCACINgBp0ECdGoiB0H///97NgIAQ///f/8hsgIgASEAQQAhCCALBEADQCCyAiADIABBAnRqKgIAIrMCXQRAIAcgswI4AgAgswIhsgILIABBAWohACAIQQFqIgggC0cNAAsLIAEgBWohASAMRQRAA0AgsgIgAyAAQQJ0aiIIKgIAIrMCXQRAIAcgswI4AgAgswIhsgILILICIAgqAgQiswJdBEAgByCzAjgCACCzAiGyAgsgsgIgCCoCCCKzAl0EQCAHILMCOAIAILMCIbICCyCyAiAIKgIMIrMCXQRAIAcgswI4AgAgswIhsgILIABBBGoiACABRw0ACwsg2AFCAXwi2AEg2QFSDQALIAIgCkECdGohAiADIAZqIgMgCUkNAAsMAQsgAEEBRgRAINkBQnyDIdoBQwAAAAAgswKVIrIC/RMhkgIg2QFCBFQhAANAQgAh2AECQCAARQRAA0AgAiDYAadBAnRqIJIC/QsCACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2QFRDQELA0AgAiDYAadBAnRqILICOAIAINgBQgF8ItgBINkBUg0ACwsgAiAKQQJ0aiECIAMgBmoiAyAJSQ0ACwwBCyAADQAg2QFCfIMh2gEg2QFCBFQhAANAQgAh2AECQCAARQRAA0AgAiDYAadBAnRq/Qz//3////9/////f////3///QsCACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2QFRDQELA0AgAiDYAadBAnRqQf///3s2AgAg2AFCAXwi2AEg2QFSDQALCyACIApBAnRqIQIgAyAGaiIDIAlJDQALCyAEQTBqJAAMAwtBvMMCKAIAEDAaIARBmcIANgIoIARB0N4ANgIkDC4LQbzDAigCABAwGiAEQf8/NgIYIARB0d4ANgIUDC4LQbzDAigCABAwGiAEQZshNgIIIARBrt4ANgIEDCQLDBkLIAAoAgAhAiABKAKMASEAIwBBIGsiBSQAAkACQCACDgMBAAEACyAAKQMgIdgBIAApAxgh3gEgASgCXCEPIAEoAlghDCABKAJUIQ4gASgCUCENIAEoAkwhCCABKAJIIQsgACgC0AEhBCABKAJEIQMCfyAAKAIAIgdBEGtBcU0EQCAHQSRsQZibAWooAgAgACgCMCAAKQMQItkBp0EBa2xqIAAoAjQiAiDeAadBAWtsaiAAKAI4Igcg2AGnQQFrbGogACgCPCAAKAIoQQFrbGoMAQsgACgCNCICrSDeAUIBfX4gACkDECLZASAANQIwfiAHQSRsQZSbAWo1AgB/fCAAKAI4IgetINgBQgF9fnwgADUCPCAAKQMoQgF9fnynCyIAQQBMDQAgASkDGCLdAUIAVw0AIAEpAxAi2wFCAFcNAAJAAkAgA0ECRwRAIAAgBGohCSABKALQASEAINsBIN0BfqchCiAIIAtssiGyAiDbAachBiAIQQBMDQEgC0EATA0CIAutItgBQv7///8PgyHiASDYAUIBgyHjASDYAUIBfSHkASAIrSHlASADQQFrIQsDQEIAId8BA0AgACDfAaciASAGbEECdGohECABIA5sIA9rrCHmAUIAIeABA0AgECDgAaciCEECdGohAQJAAkACQCADDgIBAAILIAFBADYCACAIIA1sIAxrrCHhAUIAIdoBA0ACQCDaASDmAXwi2AFCAFMNACDeASDYAUL/////D4NXDQAgBCACINgBp2xqIQhCACHYAUIAIdwBIOQBQgBSBEADQAJAINgBIOEBfCLnAUIAUw0AINkBIOcBQv////8Pg1cNACABIAgg5wGnQQJ0aioCACABKgIAkjgCAAsCQCDYAUIBhCDhAXwi5wFCAFMNACDZASDnAUL/////D4NXDQAgASAIIOcBp0ECdGoqAgAgASoCAJI4AgALINgBQgJ8IdgBINwBQgJ8ItwBIOIBUg0ACwsg4wFQDQAg2AEg4QF8ItgBQgBTDQAg2QEg2AFC/////w+DVw0AIAEgCCDYAadBAnRqKgIAIAEqAgCSOAIACyDaAUIBfCLaASDlAVINAAsMAQsgAUH///97NgIAIAggDWwgDGusIeEBQgAh2gEDQAJAINoBIOYBfCLYAUIAUw0AIN4BINgBQv////8Pg1cNACAEIAIg2AGnbGohCEIAIdgBQgAh3AEg5AFCAFIEQANAAkAg2AEg4QF8IucBQgBTDQAg2QEg5wFC/////w+DVw0AIAgg5wGnQQJ0aioCACKzAiABKgIAXkUNACABILMCOAIACwJAINgBQgGEIOEBfCLnAUIAUw0AINkBIOcBQv////8Pg1cNACAIIOcBp0ECdGoqAgAiswIgASoCAF5FDQAgASCzAjgCAAsg2AFCAnwh2AEg3AFCAnwi3AEg4gFSDQALCyDjAVANACDYASDhAXwi2AFCAFMNACDZASDYAUL/////D4NXDQAgCCDYAadBAnRqKgIAIrMCIAEqAgBeRQ0AIAEgswI4AgALINoBQgF8ItoBIOUBUg0ACwsCQAJAAkAgCw4CAQACC0G8wwIoAgAQMBogBUGbITYCGCAFQZbfADYCFAwvCyABIAEqAgAgsgKVOAIACyDgAUIBfCLgASDbAVINAAsg3wFCAXwi3wEg3QFSDQALIAAgCkECdGohACAEIAdqIgQgCUkNAAsMAwtBvMMCKAIAEDAaIAVBmyE2AgggBUGA3wA2AgQMJAsgA0EBRgRAINsBQnyDIdoBQwAAAAAgsgKVIrIC/RMhkgIDQEIAIdwBA0AgACDcAacgBmxBAnRqIQFCACHZAUIAIdgBAkAg2wFCBFoEQANAIAEg2QGnQQJ0aiCSAv0LAgAg2QFCBHwi2QEg2gFSDQALINoBItgBINsBUQ0BCwNAIAEg2AGnQQJ0aiCyAjgCACDYAUIBfCLYASDbAVINAAsLINwBQgF8ItwBIN0BUg0ACyAAIApBAnRqIQAgBCAHaiIEIAlJDQALDAILINsBQnyDIdoBIAZBAnQhCCAKQQJ0IQpBACEBIAAhAgNAIAEgCmwhDEIAIdwBA0Ag3AGnIQsCQAJAAkAgAw4CAAECCyACIAYgC2xBAnRqIQtCACHYASDbAUIEWgRAA0AgCyDYAadBAnRq/Qz//3////9/////f////3///QsCACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2wFRDQILA0AgCyDYAadBAnRqQf///3s2AgAg2AFCAXwi2AEg2wFSDQALDAELIAAgDCAIIAtsampBACAI/AsACyDcAUIBfCLcASDdAVINAAsgAUEBaiEBIAIgCmohAiAEIAdqIgQgCUkNAAsMAQsgA0EBRwRAINsBQnyDId4BINsBQgODIdwBA0BCACHaAQNAIAAg2gGnIAZsQQJ0aiEBQgAh2AFCACHZASDbAUIEWgRAA0Ag2AGnQQJ0IQJD//9//yGyAgJAAkACQCADDgIBAAILQwAAAAAhsgILIAEgAmogsgI4AgALQ///f/8hsgICQAJAAkAgAw4CAQACC0MAAAAAIbICCyABIAJBBHJqILICOAIAC0P//3//IbICAkACQAJAIAMOAgEAAgtDAAAAACGyAgsgASACQQhyaiCyAjgCAAtD//9//yGyAgJAAkACQCADDgIBAAILQwAAAAAhsgILIAEgAkEMcmogsgI4AgALINgBQgR8IdgBINkBQgR8ItkBIN4BUg0ACwtCACHZASDcAUIAUgRAA0BD//9//yGyAgJAAkACQCADDgIBAAILQwAAAAAhsgILIAEg2AGnQQJ0aiCyAjgCAAsg2AFCAXwh2AEg2QFCAXwi2QEg3AFSDQALCyDaAUIBfCLaASDdAVINAAsgACAKQQJ0aiEAIAQgB2oiBCAJSQ0ACwwBCyDbAUJ8gyHaAUMAAAAAILIClSKyAv0TIZICA0BCACHcAQNAIAAg3AGnIAZsQQJ0aiEBQgAh2QFCACHYAQJAINsBQgRaBEADQCABINkBp0ECdGogkgL9CwIAINkBQgR8ItkBINoBUg0ACyDaASLYASDbAVENAQsDQCABINgBp0ECdGogsgI4AgAg2AFCAXwi2AEg2wFSDQALCyDcAUIBfCLcASDdAVINAAsgACAKQQJ0aiEAIAQgB2oiBCAJSQ0ACwsgBUEgaiQADBgLIAEoAowBIQQjAEEgayICJAACQAJAIAQoAgBFBEACQAJAIAAoAgAOAwEAAQALIAQoAjBBBEcNAiABKQMoIuABQgBXDQAgASkDICLhASAANAIEItoBVw0AIAEpAxgi4gFCAFcNACABKQMQIt4BQgBXDQAgASgCPCEFIAEoAjghByABKAI0IQYgBCgCPCEJIAQoAjghCiAEKAI0IQggASgCMCEDIAA0Aggh4wEgATQCRCHbASDeAUJ+gyHkASDeAUIBgyHlASABKALQASELIAQoAtABIQQDQCALIAUg3QGnIgBsaiEMIAQgACAJbGohDSDaASHYAQNAIAwgByDYAaciAGxqIQ8gDSAAIApsaiEOQgAh3AEDQCAPIAYg3AGnbGohACAOIAgg3AEg2wF/p2xqIQFCACHZAUIAId8BIN4BQgFSBEADQCAAIAMg2QGnbGogASDZASDbAX+nQQJ0aioCADgCACAAIAMg2QFCAYQi5gGnbGogASDmASDbAX+nQQJ0aioCADgCACDZAUICfCHZASDfAUICfCLfASDkAVINAAsLIOUBpwRAIAAgAyDZAadsaiABINkBINsBf6dBAnRqKgIAOAIACyDcAUIBfCLcASDiAVINAAsg2AEg4wF8ItgBIOEBUw0ACyDdAUIBfCLdASDgAVINAAsLIAJBIGokAAwCC0G8wwIoAgAQMBogAkGbITYCGCACQdTfADYCFAweC0G8wwIoAgAQMBogAkH+zwA2AgggAkGr3wA2AgQMHgsMFwsgASgCjAEhBCMAQTBrIgIkAAJAIAQoAgBFBEACQAJAAkACQCAAKAIADgMBAAEACyAEKAIwQQRHDQIgASgCMEEERw0BIAEpAyAi3wFCAFcNACABKQMYIuABIAA0AgQi2wFXDQAgASkDECLhAUIAVw0AIAEpAygi3gFCAFcNACAEKAI8IQMgBCgCOCEFIAQoAjQhByAEKQMoIeIBIAQpAyAh4wEgBCkDGCHkASABKALQASEBIAQpAxAh5QEgADQCCCHmASDeAUJ8gyHZASDhAf0SIZQCIOAB/RIhlQIg3wH9EiGWAgNAIAUg3AGnbCEAINwB/RIhlwIg2wEh2gEDQCAHINoBp2whBiDaAf0SIZgCQgAh3QEDQAJAIN0BIOUBUyDaASDkAVNxINwBIOMBU3EEQCDdAadBAnQhCUIAIdgBA0AgASDYASDfAX4g3AF8IOABfiDaAXwg4QF+IN0BfKdBAnRqINgBIOIBUwR9IAQoAtABIAMg2AGnbGogAGogBmogCWoqAgAFQwAAAAALOAIAINgBQgF8ItgBIN4BUg0ACwwBC0IAIdgBIN4BQgRaBEAg3QH9EiGaAv0MAgAAAAAAAAADAAAAAAAAACGSAv0MAAAAAAAAAAABAAAAAAAAACGTAgNAIAEgkwIglgL91QEglwL9zgEglQL91QEgmAL9zgEglAL91QEgmgL9zgEimQL9GwBBAnRqQQA2AgAgASCZAv0bAkECdGpBADYCACABIJICIJYC/dUBIJcC/c4BIJUC/dUBIJgC/c4BIJQC/dUBIJoC/c4BIpkC/RsAQQJ0akEANgIAIAEgmQL9GwJBAnRqQQA2AgAgkwL9DAQAAAAAAAAABAAAAAAAAAD9zgEhkwIgkgL9DAQAAAAAAAAABAAAAAAAAAD9zgEhkgIg2AFCBHwi2AEg2QFSDQALINkBItgBIN4BUQ0BCwNAIAEg2AEg3wF+INwBfCDgAX4g2gF8IOEBfiDdAXynQQJ0akEANgIAINgBQgF8ItgBIN4BUg0ACwsg3QFCAXwi3QEg4QFSDQALINoBIOYBfCLaASDgAVMNAAsg3AFCAXwi3AEg3wFSDQALCyACQTBqJAAMAwtBvMMCKAIAEDAaIAJB4s8ANgIIIAJB5d8ANgIEDB8LQbzDAigCABAwGiACQf7PADYCGCACQeTfADYCFAwdC0G8wwIoAgAQMBogAkGbITYCKCACQY7gADYCJAwbCwwWCyABKAKMASEEIwBBIGsiAiQAAkACQAJAIAQoAgBFBEACQCAAKAIADgMEAAQACyABKAIwQQRGBEAgBCkDKCAEKQMgIAQpAxh+fiLeASAANAIEIt0BVw0EIAEpAxAi3AFCAFcNBCABKAI0IQMgBCgCNCEGIAA0Aggh3wEgBCgC0AEhCSABKALQASEFAkAgASgCRA4CAAQDCyDcAUJ8gyHbASDcAUICfSHgASDcAUIEVCEKA0AgBiDdAaciAGwhxAEgBSAAIANsaiEEQgAh2AECQCAKRQRA/QwAAAAAAQAAAAIAAAADAAAAIZICA0AgBCDYAadBAnRqIJIC/QsCACCSAv0MBAAAAAQAAAAEAAAABAAAAP2uASGSAiDYAUIEfCLYASDbAVINAAsg2wEi2AEg3AFRDQELA0AgBCDYAaciAEECdGogADYCACDYAUIBfCLYASDcAVINAAsLIMQBIAlqIQBCACHaAQNAAkAg2gEi2QFCAXwi2gEg3AFZDQAgBCDZAadBAnRqIQEg2gEh2AEg3AEg2QFCf4V8QgGDpwRAIAAgASgCACIHQQJ0aioCACAAIAQg2AGnQQJ0aiIIKAIAIgtBAnRqKgIAXgRAIAEgCzYCACAIIAc2AgALINkBQgJ8IdgBCyDZASDgAVENAANAIAAgASgCACIIQQJ0aioCACAAIAQg2AGnQQJ0aiIHKAIAIgtBAnRqKgIAXgRAIAEgCzYCACAHIAg2AgALIAAgASgCACIIQQJ0aioCACAAIAcoAgQiC0ECdGoqAgBeBEAgASALNgIAIAcgCDYCBAsg2AFCAnwi2AEg3AFSDQALCyDaASDcAVINAAsg3QEg3wF8It0BIN4BUw0ACwwEC0G8wwIoAgAQMBogAkHd0AA2AgggAkGg4AA2AgQMHwtBvMMCKAIAEDAaIAJBmyE2AhggAkHL4AA2AhQMHQsg3AFCfIMh2gEg3AFCBFQhAQNAIAUgAyDdAadsaiEAQgAh2AECQCABRQRA/QwAAAAAAQAAAAIAAAADAAAAIZICA0AgACDYAadBAnRqIJIC/QsCACCSAv0MBAAAAAQAAAAEAAAABAAAAP2uASGSAiDYAUIEfCLYASDaAVINAAsg2gEi2AEg3AFRDQELA0AgACDYAaciBEECdGogBDYCACDYAUIBfCLYASDcAVINAAsLIN0BIN8BfCLdASDeAVMNAAsMAQsg3AFCfIMh2wEg3AFCAn0h4AEg3AFCBFQhCgNAIAYg3QGnIgBsIcUBIAUgACADbGohBEIAIdgBAkAgCkUEQP0MAAAAAAEAAAACAAAAAwAAACGSAgNAIAQg2AGnQQJ0aiCSAv0LAgAgkgL9DAQAAAAEAAAABAAAAAQAAAD9rgEhkgIg2AFCBHwi2AEg2wFSDQALINsBItgBINwBUQ0BCwNAIAQg2AGnIgBBAnRqIAA2AgAg2AFCAXwi2AEg3AFSDQALCyDFASAJaiEAQgAh2gEDQAJAINoBItkBQgF8ItoBINwBWQ0AIAQg2QGnQQJ0aiEBINoBIdgBINwBINkBQn+FfEIBg6cEQCAAIAEoAgAiB0ECdGoqAgAgACAEINgBp0ECdGoiCCgCACILQQJ0aioCAF0EQCABIAs2AgAgCCAHNgIACyDZAUICfCHYAQsg2QEg4AFRDQADQCAAIAEoAgAiCEECdGoqAgAgACAEINgBp0ECdGoiBygCACILQQJ0aioCAF0EQCABIAs2AgAgByAINgIACyAAIAEoAgAiCEECdGoqAgAgACAHKAIEIgtBAnRqKgIAXQRAIAEgCzYCACAHIAg2AgQLINgBQgJ8ItgBINwBUg0ACwsg2gEg3AFSDQALIN0BIN8BfCLdASDeAVMNAAsLIAJBIGokAAwVCyABKAKMASECIwBBEGsiAyQAAkAgAigCAEUEQAJAAkAgACgCAA4DAQABAAsgAikDKCACKQMgIAIpAxh+fqciCUEATA0AIAIoAhAiBEEATA0AIAIoAjQhCiABKAI0IQggAigC0AEhCyABKALQASEMQQAgBGshDSAEQQFxIQ8gBEF8cSECIAEqAkQisgL9EyGSAiAEQQRJIQ4DQCALIAcgCmxqIQEgDCAHIAhsaiEFQQAhAAJAAkAgDg0AIAUgAWtBEEkNAANAIAUgAEECdCIGaiCSAv0MAAAAAAAAAAAAAAAAAAAAACABIAZq/QACACKTAv3qAf3mAf0MAAAAAAAAAAAAAAAAAAAAACCTAv3rAf3kAf0LAgAgAEEEaiIAIAJHDQALIAIiACAERg0BCyAAQX9zIcYBIA8EQCAFIABBAnQiEGogsgIgASAQaioCACKzAkMAAAAAILMCQwAAAABdG5QgswJDAAAAACCzAkMAAAAAXhuSOAIAIABBAXIhAAsgxgEgDUYNAANAIAUgAEECdCIGaiCyAiABIAZqKgIAIrMCQwAAAAAgswJDAAAAAF0blCCzAkMAAAAAILMCQwAAAABeG5I4AgAgBSAGQQRqIgZqILICIAEgBmoqAgAiswJDAAAAACCzAkMAAAAAXRuUILMCQwAAAAAgswJDAAAAAF4bkjgCACAAQQJqIgAgBEcNAAsLIAdBAWoiByAJRw0ACwsgA0EQaiQADAELQbzDAigCABAwGiADQZshNgIIIANB78YANgIEDCALDBQLIAEoAkQiAkECTw0VIAAhBSABKAKMASEMIAEoApABIQ0gASgClAEhDyACQQBHIRggASEEIwBBkANrIgokAAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgDCgCAA4CDgABCyAMKQMQIt0BIAQpAxBSDQwgDCkDGCLeASAEKQMYUg0LIA0pAxgi3AEg3gF9ItoBQgBTDQogDCgCMEECRw0JIA0oAjBBAkcNCCAPKAIwQQJHDQcgDSkDECDdAVINBiAPKQMYIN0BUg0FIAQoAjBBBEcNBCAEKAI0IhlBA00NAyAZIAQoAjgiFEsNAiAUIAQoAjwiHUsNAQJAIAUoAgAOAw8ADwALIAwpAyAg3gF+IuEBIAwpAyh+pyIAIAUoAggiAWpBAWsgAW0iASAFKAIEIgtsIgggASAIaiIBIAAgACABShsiKU4NDiAPKAI8IR8gDygCOCEeIA8oAjQhFyAPKQMgIeIBIA0oAjwhIyANKAI4ISUgDSgCNCEbIA0pAyAh4wEgDCgCPCEgIAwoAjghKCAMKAI0ISsg3AGnIgNBcHEiASADQQ9xIiwgA0EDcSIVayItaiECINoBIN4BINwBINoBQgF8ItgBINgBINwBUxt8INwBfSLkAUJ8gyLlAXwh2wEg3AFCfIMh2QEgA0F8cSEqIN0BQgGDIecBINwBQgGDIegBINoB/RIikgL9DAIAAAAAAAAAAwAAAAAAAAD9zgEhlAIgkgL9DAAAAAAAAAAAAQAAAAAAAAD9zgEhlQIgDykDEKciE0EBcSEzIN0BpyIWQQFxITEgGCDeAUIAVXEhNCATQXBxIgdBAXIhMiAWQXBxIgZBAXIhNSADQQNqQXxxIhIgA2siIUF8cSIvIANqIQkgAUEBayIiQQR2QQFqIgBB/v///wFxISQgAEEBcSEnQwAAgD8g3QG0kZUitQL9EyGYAiAIrCHfASALIBJBAXRBEGpsQQJ0ITBBACATayAHQX9zRyE2QQAgFmsgBkF/c0chNwNAIN8BIN8BIOEBfyLgAcQg4QF+fSLYASDYASDeAX8i2AEg3gF+fSHmASAFKAIQIDBqIQsCQCADIBJODQBBACEIIAMhACAhQQRPBEADQCALIAMgCGpBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACAIQQRqIgggL0cNAAsgCSEAICEgL0YNAQsDQCALIABBAnRqQYCAgHw2AgAgAEEBaiIAIBJHDQALCyDmAachGCDgAachESDYAcQh4AEg2AGnIRoCQCDoAacEQCDcAUIAVw0BICUg4AEg4wGBp2wgESAjbGohJiAMKALQASAaIChsIBEgIGxqIBggK2xqaiIcIAZBAXQiOGohOSANKALQASE8QgAh2AEDQCA8ICYgGyDYAaciPWxqaiEQQQAhDv0MAAAAAAAAAAAAAAAAAAAAACKSAiGTAv0MAAAAAAAAAAAAAAAAAAAAACGWAv0MAAAAAAAAAAAAAAAAAAAAACGXAiAGQQBKBEADQCCSAiAQIA5BAXQiCGoiAC8BHkECdEGQ1gRqIAAvARxBAnRBkNYEaiAALwEaQQJ0QZDWBGogAC8BGEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyAIIBxqIggvAR5BAnRBkNYEaiAILwEcQQJ0QZDWBGogCC8BGkECdEGQ1gRqIAgvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhkgIgkwIgAC8BFkECdEGQ1gRqIAAvARRBAnRBkNYEaiAALwESQQJ0QZDWBGogAC8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyAILwEWQQJ0QZDWBGogCC8BFEECdEGQ1gRqIAgvARJBAnRBkNYEaiAILwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZMCIJYCIAAvAQ5BAnRBkNYEaiAALwEMQQJ0QZDWBGogAC8BCkECdEGQ1gRqIAAvAQhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgCC8BDkECdEGQ1gRqIAgvAQxBAnRBkNYEaiAILwEKQQJ0QZDWBGogCC8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGWAiCXAiAALwEGQQJ0QZDWBGogAC8BBEECdEGQ1gRqIAAvAQJBAnRBkNYEaiAALwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAgvAQZBAnRBkNYEaiAILwEEQQJ0QZDWBGogCC8BAkECdEGQ1gRqIAgvAQBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhlwIgDkEQaiIOIAZIDQALIJcCIJMC/eQBIJYCIJIC/eQB/eQBIZICCyCSAv0fAyCSAv0fAiCSAv0fACCSAv0fAZKSkiGyAiA9QQJ0IAtqIAYgFkgEfSCyArshuAIgMQR/ILgCIBAgOGovAQBBAnRBkNYEaioCACA5LwEAQQJ0QZDWBGoqAgCUu6AhuAIgNQUgBgshACA3BEADQCC4AiAQIABBAXQiCGovAQBBAnRBkNYEaioCACAIIBxqLwEAQQJ0QZDWBGoqAgCUu6AgECAIQQJqIghqLwEAQQJ0QZDWBGoqAgAgCCAcai8BAEECdEGQ1gRqKgIAlLugIbgCIABBAmoiACAWRw0ACwsguAK2BSCyAgs4AgAg2AFCAXwi2AEg3AFSDQALDAELINwBQgBXDQAgGiAobCARICBsaiAYICtsaiEAICUg4AEg4wGBp2wgESAjbGohCEIAIdgBA0AgFiAbIAsg2AGnIg5BAnRqIA0oAtABIAggDiAbbGpqIAwoAtABIABqEOoEINgBQgJ8ItgBINwBUw0ACwsCQCABQQBMDQBBACEIQQAhDiAiQQ9HBEADQCALIAhBAnQiEGoiACCYAiAA/QAAAP3mAf0LAAAgACCYAiAA/QAAEP3mAf0LABAgACCYAiAA/QAAIP3mAf0LACAgACCYAiAA/QAAMP3mAf0LADAgCyAQQcAAcmoiACCYAiAA/QAAAP3mAf0LAAAgACCYAiAA/QAAEP3mAf0LABAgACCYAiAA/QAAIP3mAf0LACAgACCYAiAA/QAAMP3mAf0LADAgCEEgaiEIIA5BAmoiDiAkRw0ACwsgJ0UNACALIAhBAnRqIgAgmAIgAP0AAAD95gH9CwAAIAAgmAIgAP0AABD95gH9CwAQIAAgmAIgAP0AACD95gH9CwAgIAAgmAIgAP0AADD95gH9CwAwCwJAIAEgA04iHA0AQQAhCCABIQAgLEEETwRAA0AgCyABIAhqQQJ0aiIAIJgCIAD9AAIA/eYB/QsCACAIQQRqIgggLUcNAAsgAiEAIBVFDQELA0AgCyAAQQJ0aiIIILUCIAgqAgCUOAIAIABBAWoiACADRw0ACwsCQCA0RQ0AIOYBxCDaAXwh5gEg2gEh2AEg5AFCBFoEQCDmAf0SIZYCQgAh2AEglQIhkgIglAIhkwIDQCDYASDaAXynIQAgkgIglgL92QEilwL9GwBBAXEEQCALIABBAnRqQYCAgHw2AgALIJcC/RsCQQFxBEAgAEECdCALakGAgIB8NgIECyCTAiCWAv3ZASKXAv0bAEEBcQRAIABBAnQgC2pBgICAfDYCCAsglwL9GwJBAXEEQCAAQQJ0IAtqQYCAgHw2AgwLIJIC/QwEAAAAAAAAAAQAAAAAAAAA/c4BIZICIJMC/QwEAAAAAAAAAAQAAAAAAAAA/c4BIZMCINgBQgR8ItgBIOUBUg0ACyDbASHYASDkASDlAVENAQsDQCDYASDmAVUEQCALINgBp0ECdGpBgICAfDYCAAsg2AFCAXwi2AEg3AFTDQALCwJAIANBAEwEQEMAAID/IbICDAELQwAAgP8hsgJBACEOQQAhAEEAIRAgA0EETwRAA0AgsgIgCyAAQQJ0IghqKgIAIrMCILICILMCXhsisgIgCyAIQQRyaioCACKzAiCyAiCzAl4bIrICIAsgCEEIcmoqAgAiswIgsgIgswJeGyKyAiALIAhBDHJqKgIAIrMCILICILMCXhshsgIgAEEEaiEAIBBBBGoiECAqRw0ACwsgFUUNAANAILICIAsgAEECdGoqAgAiswIgsgIgswJeGyGyAiAAQQFqIQAgDkEBaiIOIBVHDQALC0EAIQhEAAAAAAAAAAAhuAJEAAAAAAAAAAAhuQJEAAAAAAAAAAAhugJEAAAAAAAAAAAhuwIgEkEASgRAA0BDAAAAACGzAgJAIAsgCEECdGoiACoCACK0AkMAAID/WwRAQwAAAAAhtAIMAQsguAJBgPwBILQCILICkyK0AotDAACAd5RDAACACJRBgICAiAcgtAK8Ig5BAXQiEEGAgIB4cSImICZBgICAiAdNG0EBdkGAgIA8ar6SvCImQQ12QYD4AXEgJkH/H3FqIBBBgICAeEsbIA5BEHZBgIACcXJBAXRBkNYsai8BAEECdEGQ1gRqKgIAIrQCu6AhuAILIAAgtAI4AgAgACoCBCK0AkMAAID/XARAILkCQYD8ASC0AiCyApMiswKLQwAAgHeUQwAAgAiUQYCAgIgHILMCvCIOQQF0IhBBgICAeHEiJiAmQYCAgIgHTRtBAXZBgICAPGq+krwiJkENdkGA+AFxICZB/x9xaiAQQYCAgHhLGyAOQRB2QYCAAnFyQQF0QZDWLGovAQBBAnRBkNYEaioCACKzArugIbkCCyAAILMCOAIEQwAAAAAhswICQCAAKgIIIrQCQwAAgP9bBEBDAAAAACG0AgwBCyC6AkGA/AEgtAIgsgKTIrQCi0MAAIB3lEMAAIAIlEGAgICIByC0ArwiDkEBdCIQQYCAgHhxIiYgJkGAgICIB00bQQF2QYCAgDxqvpK8IiZBDXZBgPgBcSAmQf8fcWogEEGAgIB4SxsgDkEQdkGAgAJxckEBdEGQ1ixqLwEAQQJ0QZDWBGoqAgAitAK7oCG6AgsgACC0AjgCCCAAKgIMIrQCQwAAgP9cBEAguwJBgPwBILQCILICkyKzAotDAACAd5RDAACACJRBgICAiAcgswK8Ig5BAXQiEEGAgIB4cSImICZBgICAiAdNG0EBdkGAgIA8ar6SvCImQQ12QYD4AXEgJkH/H3FqIBBBgICAeEsbIA5BEHZBgIACcXJBAXRBkNYsai8BAEECdEGQ1gRqKgIAIrMCu6AhuwILIAAgswI4AgwgCEEEaiIIIBJIDQALC0QAAAAAAADwPyC4AkQAAAAAAAAAAKAguQKgILoCoCC7AqCjtiGyAgJAIAFBAEwNACCyAv0TIZICQQAhCEEAIQ4gIkEPRwRAA0AgCyAIQQJ0IhBqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAsgEEHAAHJqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwIAhBIGohCCAOQQJqIg4gJEcNAAsLICdFDQAgCyAIQQJ0aiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMAsCQCAcDQAgASEAICxBBE8EQCCyAv0TIZICQQAhAANAIAsgACABakECdGoiCCAI/QACACCSAv3mAf0LAgAgAEEEaiIAIC1HDQALIAIhACAVRQ0BCwNAIAsgAEECdGoiCCAIKgIAILIClDgCACAAQQFqIgAgA0cNAAsLIAUoAhAgMGogEkECdGohEAJAINwBQgBXDQBCACHYASDcAUIEWgRAA0AgECDYAaciAEEBdGr9DAB+AAAAfgAAAH4AAAB+AAAgCyAAQQJ0av0AAgAikgL94AH9DAAAgHcAAIB3AACAdwAAgHf95gH9DAAAgAgAAIAIAACACAAAgAj95gEgkgJBAf2rASKTAv0MAAAA/wAAAP8AAAD/AAAA//1O/QwAAABxAAAAcQAAAHEAAABx/bkBQQH9rQH9DAAAgAcAAIAHAACABwAAgAf9rgH95AEilgJBDf2tAf0MAHwAAAB8AAAAfAAAAHwAAP1OIJYC/Qz/DwAA/w8AAP8PAAD/DwAA/U79rgEgkwL9DAAAAP8AAAD/AAAA/wAAAP/9PP1SIJICQRD9rQH9DACAAAAAgAAAAIAAAACAAAD9Tv1QIJIC/Q0AAQQFCAkMDQABAAEAAQAB/VsBAAAg2AFCBHwi2AEg2QFSDQALINkBItgBINwBUQ0BCwNAIBAg2AGnIgBBAXRqQYD8ASALIABBAnRqKgIAIrICi0MAAIB3lEMAAIAIlEGAgICIByCyArwiAEEBdCIIQYCAgHhxIg4gDkGAgICIB00bQQF2QYCAgDxqvpK8Ig5BDXZBgPgBcSAOQf8fcWogCEGAgIB4SxsgAEEQdkGAgAJxcjsBACDYAUIBfCLYASDcAVINAAsLAkAg5wGnBEAg3QFCAFcNASAUIBpsIBEgHWxqIBggGWxqIRggHiDgASDiAYGnbCARIB9saiERIBAgB0EBdCIaaiEcIA8oAtABISYgBCgC0AEhOEIAIdgBA0AgJiARIBcg2AGnIjlsamohC0EAIQ79DAAAAAAAAAAAAAAAAAAAAAAikgIhkwL9DAAAAAAAAAAAAAAAAAAAAAAhlgL9DAAAAAAAAAAAAAAAAAAAAAAhlwIgB0EASgRAA0AgkgIgCyAOQQF0IghqIgAvAR5BAnRBkNYEaiAALwEcQQJ0QZDWBGogAC8BGkECdEGQ1gRqIAAvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgCCAQaiIILwEeQQJ0QZDWBGogCC8BHEECdEGQ1gRqIAgvARpBAnRBkNYEaiAILwEYQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZICIJMCIAAvARZBAnRBkNYEaiAALwEUQQJ0QZDWBGogAC8BEkECdEGQ1gRqIAAvARBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgCC8BFkECdEGQ1gRqIAgvARRBAnRBkNYEaiAILwESQQJ0QZDWBGogCC8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGTAiCWAiAALwEOQQJ0QZDWBGogAC8BDEECdEGQ1gRqIAAvAQpBAnRBkNYEaiAALwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAgvAQ5BAnRBkNYEaiAILwEMQQJ0QZDWBGogCC8BCkECdEGQ1gRqIAgvAQhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhlgIglwIgAC8BBkECdEGQ1gRqIAAvAQRBAnRBkNYEaiAALwECQQJ0QZDWBGogAC8BAEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyAILwEGQQJ0QZDWBGogCC8BBEECdEGQ1gRqIAgvAQJBAnRBkNYEaiAILwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZcCIA5BEGoiDiAHSA0ACyCXAiCTAv3kASCWAiCSAv3kAf3kASGSAgsgkgL9HwMgkgL9HwIgkgL9HwAgkgL9HwGSkpIhsgIgOUECdCAYaiA4aiAHIBNIBH0gsgK7IbgCIDMEfyC4AiALIBpqLwEAQQJ0QZDWBGoqAgAgHC8BAEECdEGQ1gRqKgIAlLugIbgCIDIFIAcLIQAgNgRAA0AguAIgCyAAQQF0IghqLwEAQQJ0QZDWBGoqAgAgCCAQai8BAEECdEGQ1gRqKgIAlLugIAsgCEECaiIIai8BAEECdEGQ1gRqKgIAIAggEGovAQBBAnRBkNYEaioCAJS7oCG4AiAAQQJqIgAgE0cNAAsLILgCtgUgsgILOAIAINgBQgF8ItgBIN0BUg0ACwwBCyDdAUIAVw0AIBQgGmwgESAdbGogGCAZbGohACAeIOABIOIBgadsIBEgH2xqIQhCACHYAQNAIBMgFyAEKALQASAAINgBpyILQQJ0amogDygC0AEgCCALIBdsamogEBDqBCDYAUICfCLYASDdAVMNAAsLICkg3wFCAXwi3wGnRw0ACwwOC0G8wwIoAgAQMBogCkGbITYCCCAKQYTkADYCBCAKQeQmNgIAQbjDAigCAEHL5AAgChAxDEILQbzDAigCABAwGiAKQbE6NgIoIApBweIANgIkIApB5CY2AiBBuMMCKAIAQcvkACAKQSBqEDEMQQtBvMMCKAIAEDAaIApBxjs2AhggCkHA4gA2AhQgCkHkJjYCEEG4wwIoAgBBy+QAIApBEGoQMQxAC0G8wwIoAgAQMBogCkHOPTYCOCAKQb/iADYCNCAKQeQmNgIwQbjDAigCAEHL5AAgCkEwahAxDD8LQbzDAigCABAwGiAKQd3QADYCSCAKQb7iADYCRCAKQeQmNgJAQbjDAigCAEHL5AAgCkFAaxAxDD4LQbzDAigCABAwGiAKQZszNgJYIApBt+IANgJUIApB5CY2AlBBuMMCKAIAQcvkACAKQdAAahAxDD0LQbzDAigCABAwGiAKQbAzNgJoIApBtuIANgJkIApB5CY2AmBBuMMCKAIAQcvkACAKQeAAahAxDDwLQbzDAigCABAwGiAKQe/RADYCeCAKQbPiADYCdCAKQeQmNgJwQbjDAigCAEHL5AAgCkHwAGoQMQw7C0G8wwIoAgAQMBogCkGn0gA2AogBIApBsuIANgKEASAKQeQmNgKAAUG4wwIoAgBBy+QAIApBgAFqEDEMOgtBvMMCKAIAEDAaIApBi9IANgKYASAKQbHiADYClAEgCkHkJjYCkAFBuMMCKAIAQcvkACAKQZABahAxDDkLQbzDAigCABAwGiAKQfbAADYCqAEgCkGv4gA2AqQBIApB5CY2AqABQbjDAigCAEHL5AAgCkGgAWoQMQw4C0G8wwIoAgAQMBogCkHTLjYCuAEgCkGu4gA2ArQBIApB5CY2ArABQbjDAigCAEHL5AAgCkGwAWoQMQw3C0G8wwIoAgAQMBogCkG6MzYCyAEgCkGt4gA2AsQBIApB5CY2AsABQbjDAigCAEHL5AAgCkHAAWoQMQw2CyAMKQMQItkBIAQpAxBSDQwgDCkDGCLcASAEKQMYUg0LIA0pAxgi2wEg3AF9ItgBQgBTDQogDCgCMEEERw0JIA0oAjBBBEcNCCAPKAIwQQRHDQcgDSkDECDZAVINBiAPKQMYINkBUg0FIAQoAjBBBEcNBCAEKAI0IhNBA00NAyATIAQoAjgiFksNAiAWIAQoAjwiHEsNAQJAIAUoAgAOAwEAAQALIAwpAyAg3AF+It8BIAwpAyh+pyIAIAUoAggiAWpBAWsgAW0iAiAFKAIEIgNsIgEgASACaiICIAAgACACShsiAE4NACAPKAI8IR0gDygCOCEfIA8oAjQhHiAPKQMgIeQBIA0oAjwhIyANKAI4ISUgDSgCNCEgIA0pAyAh5QEgDCgCPCEoIAwoAjghKyAMKAI0ISwg2AFCAXwh5gEg2QGnIhFBA3EhFSDbAaciAkEDakF8cSIOIAJrIhpBfHEiGSACaiEHQwAAgD8g2QG0kZUitAL9EyGUAiAArCHnASABrCHdASAOrCHgASADIA5BEGpsQQJ0IS0gGkEESSEhIBFBcHEiA0F/cyARakEDSSEvA0Ag3QEg3QEg3wF/It4BxCDfAX59ItgBINgBINwBfyLYASDcAX59IdoBIAUoAhAgLWohBgJAIAIgDk4NAEEAIQggAiEAICFFBEADQCAGIAIgCGpBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACAIQQRqIgggGUcNAAsgByEAIBkgGkYNAQsDQCAGIABBAnRqQYCAgHw2AgAgAEEBaiIAIA5HDQALCyDaAachFCDeAachCyDYAcQh4QEg2AGnIRcg5gEg2gHEfCDbASAYGyLaAUIAVyIiRQRAICUg4QEg5QGBp2wgCyAjbGohEiAMKALQASAXICtsIAsgKGxqIBQgLGxqaiEBIA0oAtABIRtCACHYAQNAIBsgEiAgINgBpyIkbGpqIQlBACEA/QwAAAAAAAAAAAAAAAAAAAAAIpICIZMC/QwAAAAAAAAAAAAAAAAAAAAAIZYC/QwAAAAAAAAAAAAAAAAAAAAAIZcCIANBAEoEQANAIJICIAkgAEECdCIQaiII/QAAMCABIBBqIhD9AAAw/eYB/eQBIZICIJMCIAj9AAAgIBD9AAAg/eYB/eQBIZMCIJYCIAj9AAAQIBD9AAAQ/eYB/eQBIZYCIJcCIAj9AAAAIBD9AAAA/eYB/eQBIZcCIABBEGoiACADSA0ACyCXAiCTAv3kASCWAiCSAv3kAf3kASGSAgsgJEECdCEQIJIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIbICAkAgAyARTg0AQQAhCCADIQAgFQRAA0AgCSAAQQJ0IiRqKgIAIAEgJGoqAgCUILICkiGyAiAAQQFqIQAgCEEBaiIIIBVHDQALCyAvDQADQCAJIABBAnQiCEEMaiIkaioCACABICRqKgIAlCAJIAhBCGoiJGoqAgAgASAkaioCAJQgCSAIQQRqIiRqKgIAIAEgJGoqAgCUIAggCWoqAgAgASAIaioCAJQgsgKSkpKSIbICIABBBGoiACARRw0ACwsgBiAQaiCyAjgCACDYAUIBfCLYASDaAVINAAsLAkAg2gGnIglBcHEiAUEATCIkDQAgAUEBayIAQQR2QQFqIhBBAXEhxwFBACEIIABBD0cEQCAQQf7///8BcSEbQQAhEANAIAYgCEECdCInaiIAIJQCIAD9AAAA/eYB/QsAACAAIJQCIAD9AAAQ/eYB/QsAECAAIJQCIAD9AAAg/eYB/QsAICAAIJQCIAD9AAAw/eYB/QsAMCAGICdBwAByaiIAIJQCIAD9AAAA/eYB/QsAACAAIJQCIAD9AAAQ/eYB/QsAECAAIJQCIAD9AAAg/eYB/QsAICAAIJQCIAD9AAAw/eYB/QsAMCAIQSBqIQggEEECaiIQIBtHDQALCyDHAUUNACAGIAhBAnRqIgAglAIgAP0AAAD95gH9CwAAIAAglAIgAP0AABD95gH9CwAQIAAglAIgAP0AACD95gH9CwAgIAAglAIgAP0AADD95gH9CwAwCwJAIAEgCU4iGw0AIAEhACAJQQ9xIghBBE8EQCAAIAggCUEDcSIQayISaiEAQQAhCANAIAYgASAIakECdGoiJyCUAiAn/QACAP3mAf0LAgAgCEEEaiIIIBJHDQALIBBFDQELA0AgBiAAQQJ0aiIIILQCIAgqAgCUOAIAIABBAWoiACAJRw0ACwsCQCDaASDbAVkNACDbASDaASLYAX0i4gFCBFoEQCDYASDiAUJ8gyLjAXwh2AFCACHeAQNAIAYg2gEg3gF8p0ECdGr9DAAAgP8AAID/AACA/wAAgP/9CwIAIN4BQgR8It4BIOMBUg0ACyDiASDjAVENAQsDQCAGINgBp0ECdGpBgICAfDYCACDYAUIBfCLYASDbAVINAAsLAkAgCUEATARAQwAAgP8hsgIMAQtDAACA/yGyAkEAIRBBACEAIAlBBE8EQCAJQXxxISdBACEIA0AgsgIgBiAAQQJ0IhJqKgIAIrMCILICILMCXhsisgIgBiASQQRyaioCACKzAiCyAiCzAl4bIrICIAYgEkEIcmoqAgAiswIgsgIgswJeGyKyAiAGIBJBDHJqKgIAIrMCILICILMCXhshsgIgAEEEaiEAIAhBBGoiCCAnRw0ACwsgCUEDcSIIRQ0AA0AgsgIgBiAAQQJ0aioCACKzAiCyAiCzAl4bIbICIABBAWohACAQQQFqIhAgCEcNAAsLRAAAAAAAAAAAIbgCAkAgDkEATARARAAAAAAAAAAAIbkCRAAAAAAAAAAAIboCRAAAAAAAAAAAIbsCDAELRAAAAAAAAAAAIbkCRAAAAAAAAAAAIboCRAAAAAAAAAAAIbsCICINACDgASDaASDaASDgAVUbId4BQgAh2AEDQEMAAAAAIbMCIAYg2AGnQQJ0aiIAKgIAIrUCQwAAgP9cBEAguAIgtQIgsgKTEEUiswK7oCG4AgsgACCzAjgCAAJAINoBINgBQgGEVw0AQwAAAAAhswIgACoCBCK1AkMAAID/XARAILkCILUCILICkxBFIrMCu6AhuQILIAAgswI4AgQg2gEg2AFCAoRXDQBDAAAAACGzAiAAKgIIIrUCQwAAgP9cBEAgugIgtQIgsgKTEEUiswK7oCG6AgsgACCzAjgCCCDaASDYAUIDhFcNAEMAAAAAIbMCIAAqAgwitQJDAACA/1wEQCC7AiC1AiCyApMQRSKzArugIbsCCyAAILMCOAIMCyDYAUIEfCLYASDeAVMNAAsLRAAAAAAAAPA/ILgCRAAAAAAAAAAAoCC5AqAgugKgILsCoKO2IbICAkAgAUEATA0AIAFBAWsiAEEEdkEBaiIQQQFxIcgBILIC/RMhkgJBACEIIABBD0cEQCAQQf7///8BcSEiQQAhEANAIAYgCEECdCInaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAGICdBwAByaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAIQSBqIQggEEECaiIQICJHDQALCyDIAUUNACAGIAhBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCwJAIBsNACABIQAgCUEPcSIIQQRPBEAgACAIIAlBA3EiEGsiEmohACCyAv0TIZICQQAhCANAIAYgASAIakECdGoiIiAi/QACACCSAv3mAf0LAgAgCEEEaiIIIBJHDQALIBBFDQELA0AgBiAAQQJ0aiIIIAgqAgAgsgKUOAIAIABBAWoiACAJRw0ACwsg2QFCAFUEQCAWIBdsIAsgHGxqIBMgFGxqIRQgHyDhASDkAYGnbCALIB1saiEXIAlBA3EhEiABQX9zIAlqISIgDygC0AEhJyAEKALQASEwQgAh2AEDQCAnIBcgHiDYAaciKWxqaiELQQAhCP0MAAAAAAAAAAAAAAAAAAAAACKSAiGTAv0MAAAAAAAAAAAAAAAAAAAAACGWAv0MAAAAAAAAAAAAAAAAAAAAACGXAiAkRQRAA0AgkgIgCyAIQQJ0IhBqIgD9AAAwIAYgEGoiEP0AADD95gH95AEhkgIgkwIgAP0AACAgEP0AACD95gH95AEhkwIglgIgAP0AABAgEP0AABD95gH95AEhlgIglwIgAP0AAAAgEP0AAAD95gH95AEhlwIgCEEQaiIIIAFIDQALIJcCIJMC/eQBIJYCIJIC/eQB/eQBIZICCyApQQJ0IBRqIckBIJIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIbICAkAgGw0AQQAhCCABIQAgEgRAA0AgCyAAQQJ0IilqKgIAIAYgKWoqAgCUILICkiGyAiAAQQFqIQAgCEEBaiIIIBJHDQALCyAiQQNJDQADQCALIABBAnQiCEEMaiIpaioCACAGIClqKgIAlCALIAhBCGoiKWoqAgAgBiApaioCAJQgCyAIQQRqIilqKgIAIAYgKWoqAgCUIAggC2oqAgAgBiAIaioCAJQgsgKSkpKSIbICIABBBGoiACAJRw0ACwsgyQEgMGogsgI4AgAg2AFCAXwi2AEg2QFSDQALCyDdAUIBfCLdASDnAVINAAsLIApBkANqJAAMDAtBvMMCKAIAEDAaIApBsTo2AugBIApBg+EANgLkASAKQeQmNgLgAUG4wwIoAgBBy+QAIApB4AFqEDEMMwtBvMMCKAIAEDAaIApBxjs2AtgBIApBguEANgLUASAKQeQmNgLQAUG4wwIoAgBBy+QAIApB0AFqEDEMMgtBvMMCKAIAEDAaIApBzj02AvgBIApBgeEANgL0ASAKQeQmNgLwAUG4wwIoAgBBy+QAIApB8AFqEDEMMQtBvMMCKAIAEDAaIApB3dAANgKIAiAKQYDhADYChAIgCkHkJjYCgAJBuMMCKAIAQcvkACAKQYACahAxDDALQbzDAigCABAwGiAKQZszNgKYAiAKQfngADYClAIgCkHkJjYCkAJBuMMCKAIAQcvkACAKQZACahAxDC8LQbzDAigCABAwGiAKQbAzNgKoAiAKQfjgADYCpAIgCkHkJjYCoAJBuMMCKAIAQcvkACAKQaACahAxDC4LQbzDAigCABAwGiAKQZvQADYCuAIgCkH14AA2ArQCIApB5CY2ArACQbjDAigCAEHL5AAgCkGwAmoQMQwtC0G8wwIoAgAQMBogCkHH0AA2AsgCIApB9OAANgLEAiAKQeQmNgLAAkG4wwIoAgBBy+QAIApBwAJqEDEMLAtBvMMCKAIAEDAaIApBsdAANgLYAiAKQfPgADYC1AIgCkHkJjYC0AJBuMMCKAIAQcvkACAKQdACahAxDCsLQbzDAigCABAwGiAKQfbAADYC6AIgCkHx4AA2AuQCIApB5CY2AuACQbjDAigCAEHL5AAgCkHgAmoQMQwqC0G8wwIoAgAQMBogCkHTLjYC+AIgCkHw4AA2AvQCIApB5CY2AvACQbjDAigCAEHL5AAgCkHwAmoQMQwpC0G8wwIoAgAQMBogCkG6MzYCiAMgCkHv4AA2AoQDIApB5CY2AoADQbjDAigCAEHL5AAgCkGAA2oQMQwoCwwTCyABKAKMASEMIAEoApABIQ0gASgClAEhCiABKAKYASEOIAEoApwBIRAjAEHQAmsiBiQAAkACQCANKAIAIgJBAUcEQCACDQFBvMMCKAIAEDAaIAZBmyE2AsgCIAZBn+UANgLEAiAGQeQmNgLAAkG4wwIoAgBBy+QAIAZBwAJqEDEMKQsgDCkDECLbASABKQMQUQRAIAwpAxgi3QEgASkDGFEEQCAMKQMgItgBIAEpAyBRBEAgDCgCMEECRgRAIA0oAjBBAkYEQCAKKAIwQQRGBEAgDigCMEECRgRAIBAoAjBBBEYEQCDbASANKQMQUQRAIA0pAxgi2QEgCikDEFEEQCAKKQMYQgFRBEAg2QEgDikDEFEEQCDbASAOKQMYUQRAINsBIBApAxBRBEAgECkDGEIBUQRAIAEoAjBBBEYEQCABKAI0IhhBA0sEQCABKAI4IhogGE8EQCABKAI8Ih0gGk8EQAJAAkAgACgCAA4DAQABAAsg2AEg3QF+It4BIAwpAyh+pyICIAAoAggiBGpBAWsgBG0iBCAAKAIEIgNsIhEgBCARaiIEIAIgAiAEShsiEk4NACAOKAI8IR8gDigCOCEeIA4oAjQhIyANKAI8ISUgDSgCOCEgIA0oAjQhKCAMKAI8ISsgDCgCOCEsIAwoAjQhLSAAKAIQIAMg2QGnIglBAXRBEGpsQQJ0aiILIAlBAnQiAGoiDyAJQXBxIgJBAXQiIWohLyDZAUJ8gyHaASDbAaciCEEDcSEZIAhBfHEhAyAJQQFxISIgCUF4cSEFIAlBA3EhFCAJQXxxIQcgCEEBcSEkIAlBAWshFyACQQFyIScgCEFwcSIEQQFyITAgECgC0AEiECAIQQJ0IhNqISkgASgC0AEiGyATaiEqIBKsId8BIBGsIdwBQQAgCGsgBEF/c0chMyALIAooAtABIhEgAGpJIA8gEUtxIAlBBElyITEDQCDcASDcASDeAX8i2AHEIN4Bfn0i4AEg4AEg3QF/IuABIN0Bfn2nIRwg2AGnIRIg4AGnIRMg2QFCAFciNEUEQCATICBsIBIgJWxqITIgDCgC0AEgEyAsbCASICtsaiAcIC1samoiFSAEQQF0IjVqITYgDSgC0AEhN0IAIdgBA0AgNyAyICgg2AGnIiZsamohFkEAIQr9DAAAAAAAAAAAAAAAAAAAAAAikgIhkwL9DAAAAAAAAAAAAAAAAAAAAAAhlAL9DAAAAAAAAAAAAAAAAAAAAAAhlQIgBEEASgRAA0AgkgIgFiAKQQF0IgFqIgAvAR5BAnRBkNYEaiAALwEcQQJ0QZDWBGogAC8BGkECdEGQ1gRqIAAvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgASAVaiIBLwEeQQJ0QZDWBGogAS8BHEECdEGQ1gRqIAEvARpBAnRBkNYEaiABLwEYQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZICIJMCIAAvARZBAnRBkNYEaiAALwEUQQJ0QZDWBGogAC8BEkECdEGQ1gRqIAAvARBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgAS8BFkECdEGQ1gRqIAEvARRBAnRBkNYEaiABLwESQQJ0QZDWBGogAS8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGTAiCUAiAALwEOQQJ0QZDWBGogAC8BDEECdEGQ1gRqIAAvAQpBAnRBkNYEaiAALwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAEvAQ5BAnRBkNYEaiABLwEMQQJ0QZDWBGogAS8BCkECdEGQ1gRqIAEvAQhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhlAIglQIgAC8BBkECdEGQ1gRqIAAvAQRBAnRBkNYEaiAALwECQQJ0QZDWBGogAC8BAEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyABLwEGQQJ0QZDWBGogAS8BBEECdEGQ1gRqIAEvAQJBAnRBkNYEaiABLwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZUCIApBEGoiCiAESA0ACyCVAiCTAv3kASCUAiCSAv3kAf3kASGSAgsgkgL9HwMgkgL9HwIgkgL9HwAgkgL9HwGSkpIhsgIgJkECdCALaiAEIAhIBH0gsgK7IbgCICQEfyC4AiAWIDVqLwEAQQJ0QZDWBGoqAgAgNi8BAEECdEGQ1gRqKgIAlLugIbgCIDAFIAQLIQAgMwRAA0AguAIgFiAAQQF0IgFqLwEAQQJ0QZDWBGoqAgAgASAVai8BAEECdEGQ1gRqKgIAlLugIBYgAUECaiIBai8BAEECdEGQ1gRqKgIAIAEgFWovAQBBAnRBkNYEaioCAJS7oCG4AiAAQQJqIgAgCEcNAAsLILgCtgUgsgILOAIAINgBQgF8ItgBINkBUg0ACwsCQCAJQQBMIhYNAEEAIQpBACEAIDFFBEADQCALIABBAnQiAWoiFSAV/QACACABIBFq/QACAP3kAf0LAgAgAEEEaiIAIAdHDQALIAciACAJRg0BCyAXIABrIcoBIBQEQANAIAsgAEECdCIVaiIyIDIqAgAgESAVaioCAJI4AgAgAEEBaiEAIApBAWoiCiAURw0ACwsgygFBA0kNAANAIAsgAEECdCIBaiIKIAoqAgAgASARaioCAJI4AgAgCyABQQRqIgpqIhUgFSoCACAKIBFqKgIAkjgCACALIAFBCGoiCmoiFSAVKgIAIAogEWoqAgCSOAIAIAsgAUEMaiIBaiIKIAoqAgAgASARaioCAJI4AgAgAEEEaiIAIAlHDQALCwJAIDQNAEIAIdgBINkBQgRaBEADQCAPINgBpyIAQQF0av0MAH4AAAB+AAAAfgAAAH4AACALIABBAnRq/QACACKSAv3gAf0MAACAdwAAgHcAAIB3AACAd/3mAf0MAACACAAAgAgAAIAIAACACP3mASCSAkEB/asBIpMC/QwAAAD/AAAA/wAAAP8AAAD//U79DAAAAHEAAABxAAAAcQAAAHH9uQFBAf2tAf0MAACABwAAgAcAAIAHAACAB/2uAf3kASKUAkEN/a0B/QwAfAAAAHwAAAB8AAAAfAAA/U4glAL9DP8PAAD/DwAA/w8AAP8PAAD9Tv2uASCTAv0MAAAA/wAAAP8AAAD/AAAA//08/VIgkgJBEP2tAf0MAIAAAACAAAAAgAAAAIAAAP1O/VAgkgL9DQABBAUICQwNAAEAAQABAAH9WwEAACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2QFRDQELA0AgDyDYAaciAEEBdGpBgPwBIAsgAEECdGoqAgAisgKLQwAAgHeUQwAAgAiUQYCAgIgHILICvCIAQQF0IgFBgICAeHEiCiAKQYCAgIgHTRtBAXZBgICAPGq+krwiCkENdkGA+AFxIApB/x9xaiABQYCAgHhLGyAAQRB2QYCAAnFyOwEAINgBQgF8ItgBINkBUg0ACwsCQCAWDQBBACEAIAlBCE8EQANAIA8gAEEBdGoiASAB/QABACKTAiCSAv0NCAkKCwwNDg8AAQABAAEAAf2pASKSAv0bA0EBdEGQ1hRqIJIC/RsCQQF0QZDWFGogkgL9GwFBAXRBkNYUaiCSAv0bAEEBdEGQ1hRqIJMC/akBIpIC/RsDQQF0QZDWFGogkgL9GwJBAXRBkNYUaiCSAv0bAUEBdEGQ1hRqIJIC/RsAQQF0QZDWFGr9CAEA/VUBAAH9VQEAAv1VAQAD/VUBAAT9VQEABf1VAQAG/VUBAAf9CwEAIABBCGoiACAFRw0ACyAFIgAgCUYNAQsDQCAPIABBAXRqIgEgAS8BAEEBdEGQ1hRqLwEAOwEAIABBAWoiACAJRw0ACwsgGCAcbCIVIBMgGmwiHCASIB1sIjRqaiEWINsBQgBVBEAgEyAebCASIB9saiETIA4oAtABITJCACHYAQNAIDIgEyAjINgBpyI1bGpqIRJBACEK/QwAAAAAAAAAAAAAAAAAAAAAIpICIZMC/QwAAAAAAAAAAAAAAAAAAAAAIZQC/QwAAAAAAAAAAAAAAAAAAAAAIZUCIAJBAEoEQANAIJICIBIgCkEBdCIBaiIALwEeQQJ0QZDWBGogAC8BHEECdEGQ1gRqIAAvARpBAnRBkNYEaiAALwEYQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAEgD2oiAS8BHkECdEGQ1gRqIAEvARxBAnRBkNYEaiABLwEaQQJ0QZDWBGogAS8BGEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGSAiCTAiAALwEWQQJ0QZDWBGogAC8BFEECdEGQ1gRqIAAvARJBAnRBkNYEaiAALwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAEvARZBAnRBkNYEaiABLwEUQQJ0QZDWBGogAS8BEkECdEGQ1gRqIAEvARBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhkwIglAIgAC8BDkECdEGQ1gRqIAAvAQxBAnRBkNYEaiAALwEKQQJ0QZDWBGogAC8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyABLwEOQQJ0QZDWBGogAS8BDEECdEGQ1gRqIAEvAQpBAnRBkNYEaiABLwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIZQCIJUCIAAvAQZBAnRBkNYEaiAALwEEQQJ0QZDWBGogAC8BAkECdEGQ1gRqIAAvAQBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgAS8BBkECdEGQ1gRqIAEvAQRBAnRBkNYEaiABLwECQQJ0QZDWBGogAS8BAEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASGVAiAKQRBqIgogAkgNAAsglQIgkwL95AEglAIgkgL95AH95AEhkgILIJIC/R8DIJIC/R8CIJIC/R8AIJIC/R8BkpKSIbICIDVBAnQgFmogG2ogAiAJSAR9ILICuyG4AiAiBH8guAIgEiAhai8BAEECdEGQ1gRqKgIAIC8vAQBBAnRBkNYEaioCAJS7oCG4AiAnBSACCyEAIAIgF0cEQANAILgCIBIgAEEBdCIBai8BAEECdEGQ1gRqKgIAIAEgD2ovAQBBAnRBkNYEaioCAJS7oCASIAFBAmoiAWovAQBBAnRBkNYEaioCACABIA9qLwEAQQJ0QZDWBGoqAgCUu6AhuAIgAEECaiIAIAlHDQALCyC4ArYFILICCzgCACDYAUIBfCLYASDbAVINAAsLAkAgCEEATA0AIBYgG2ohAUEAIQpBACEAAkAgCEEISQ0AIAEgKUkgECAqIBUgNGogHGpqSXENAANAIAEgAEECdCISaiITIBP9AAIAIBAgEmr9AAIA/eQB/QsCACAAQQRqIgAgA0cNAAsgAyIAIAhGDQELIABBf3MgCGohywEgGQRAA0AgASAAQQJ0IhNqIhYgFioCACAQIBNqKgIAkjgCACAAQQFqIQAgCkEBaiIKIBlHDQALCyDLAUEDSQ0AA0AgASAAQQJ0IgpqIhIgEioCACAKIBBqKgIAkjgCACABIApBBGoiEmoiEyATKgIAIBAgEmoqAgCSOAIAIAEgCkEIaiISaiITIBMqAgAgECASaioCAJI4AgAgASAKQQxqIgpqIhIgEioCACAKIBBqKgIAkjgCACAAQQRqIgAgCEcNAAsLINwBQgF8ItwBIN8BUg0ACwsgBkHQAmokAAwUC0G8wwIoAgAQMBogBkGxOjYCKCAGQcLkADYCJCAGQeQmNgIgQbjDAigCAEHL5AAgBkEgahAxDDoLQbzDAigCABAwGiAGQcY7NgIYIAZBweQANgIUIAZB5CY2AhBBuMMCKAIAQcvkACAGQRBqEDEMOQtBvMMCKAIAEDAaIAZBzj02AjggBkHA5AA2AjQgBkHkJjYCMEG4wwIoAgBBy+QAIAZBMGoQMQw4C0G8wwIoAgAQMBogBkHd0AA2AkggBkG/5AA2AkQgBkHkJjYCQEG4wwIoAgBBy+QAIAZBQGsQMQw3C0G8wwIoAgAQMBogBkG1PzYCWCAGQbzkADYCVCAGQeQmNgJQQbjDAigCAEHL5AAgBkHQAGoQMQw2C0G8wwIoAgAQMBogBkHNMzYCaCAGQbvkADYCZCAGQeQmNgJgQbjDAigCAEHL5AAgBkHgAGoQMQw1C0G8wwIoAgAQMBogBkGlMzYCeCAGQbrkADYCdCAGQeQmNgJwQbjDAigCAEHL5AAgBkHwAGoQMQw0C0G8wwIoAgAQMBogBkGPLzYCiAEgBkG55AA2AoQBIAZB5CY2AoABQbjDAigCAEHL5AAgBkGAAWoQMQwzC0G8wwIoAgAQMBogBkHAPzYCmAEgBkG35AA2ApQBIAZB5CY2ApABQbjDAigCAEHL5AAgBkGQAWoQMQwyC0G8wwIoAgAQMBogBkGELzYCqAEgBkG25AA2AqQBIAZB5CY2AqABQbjDAigCAEHL5AAgBkGgAWoQMQwxC0G8wwIoAgAQMBogBkHYMzYCuAEgBkG05AA2ArQBIAZB5CY2ArABQbjDAigCAEHL5AAgBkGwAWoQMQwwC0G8wwIoAgAQMBogBkHy0AA2AsgBIAZBsuQANgLEASAGQeQmNgLAAUG4wwIoAgBBy+QAIAZBwAFqEDEMLwtBvMMCKAIAEDAaIAZB+tIANgLYASAGQbHkADYC1AEgBkHkJjYC0AFBuMMCKAIAQcvkACAGQdABahAxDC4LQbzDAigCABAwGiAGQZ/RADYC6AEgBkGw5AA2AuQBIAZB5CY2AuABQbjDAigCAEHL5AAgBkHgAWoQMQwtC0G8wwIoAgAQMBogBkGz0wA2AvgBIAZBr+QANgL0ASAGQeQmNgLwAUG4wwIoAgBBy+QAIAZB8AFqEDEMLAtBvMMCKAIAEDAaIAZB3tIANgKIAiAGQa7kADYChAIgBkHkJjYCgAJBuMMCKAIAQcvkACAGQYACahAxDCsLQbzDAigCABAwGiAGQdE7NgKYAiAGQazkADYClAIgBkHkJjYCkAJBuMMCKAIAQcvkACAGQZACahAxDCoLQbzDAigCABAwGiAGQdk9NgKoAiAGQavkADYCpAIgBkHkJjYCoAJBuMMCKAIAQcvkACAGQaACahAxDCkLQbzDAigCABAwGiAGQaLAADYCuAIgBkGq5AA2ArQCIAZB5CY2ArACQbjDAigCAEHL5AAgBkGwAmoQMQwoC0G8wwIoAgAQMBogBkGbITYCCCAGQaPlADYCBCAGQeQmNgIAQbjDAigCAEHL5AAgBhAxDCcLDBILIAEoAkQiAkECTw0UIAAhBSABKAKMASEPIAEoApABIQ4gASgClAEhEiABKAKYASEQIAJBAEchLyMAQeABayIKJAACQAJAIA8oAgBFBEAgDikDGCLbASAPKQMYIt0BfSLaAUIAWQRAIA8oAjBBBEYEQCAOKAIwQQRGBEAgEigCMEEERgRAIA8pAxAi2QEgDikDEFEEQCDZASASKQMYUQRAINkBIBApAxBRBEAg3QEgECkDGFEEQCABKAIwQQRGBEAgASgCNCIAQQNLBEAgACABKAI4IgJNBEAgASgCPCACTwRAIAUoAgQhAAJAAkAgBSgCAA4DAAEPAQsgAA0OIAEoAtABQQAgASkDKCABKQMgIAEpAxAgASkDGH5+fqdBAnT8CwAMDgsgASgCACIEQRBrQXJJBEAgDykDICLYASAOKQMgIt4BfyGOAiAOKQMoIN4BfiLfAaciAiAFKAIIIgNqQQFrIANtIgMgAGwiCSADIAlqIgMgAiACIANKGyIITg0OII4CpyIiQQBMDQ4g3QFCAFcNDiAQKAI8ISQgECgCOCEnIBAoAjQhGyASKAI8ITAgEigCOCEpIBIoAjQhKiAOKAI8ITMgDigCOCExIA4oAjQhHCAPKAI8ITQgDygCOCEyIA8oAjQhHSAQKAIwIR8gASgC0AEiHiAEQSRsQZibAWooAgAiASAPKQMoINkBIN0BfiDYAX5+p2xBA2pBfHEiAiABINkBINsBfiDfAX6nbEEDakF8cWpqITUgAiAeaiE2IBIpAxBCAoYi5QEg2QF+ItwBINgBfiHmASDbASDZAUIChiLgAX4i3wEg2AF+IecBIOABpyE3INwBpyEmIN8BpyE4IN0BIOABfiLcAachOSDYASDcAX6nITwg2gFCAXwh6AFDAACAPyDZAbSRlSK0Av0TIZICIN4BpyE9INkBpyIRQXBxIgIgEUEPcSIjIBFBA3EiFmsiJWohBCDbAaciA0F8cSEHIAAg2QEgA0EDakF8cSIVrCLhASDZASDhAVUbpyIAQRBqIj9BAXRsIgFBAnQhQCAVIANrIiBBfHEiKCADaiEGIAAgAWpBAnRBQGshQSAIrCHpASAJrCHfASAgQQRJIUIgAkF/cyARakEDSSFDA0Ag3wGnIN8BIN4BfyLYASDeAX6nayIYIClsIDAg2AGnIgBsaiFEIBggMWwgACAzbGohKyAYICZsrSDYASDmAX5C/P///w+DfCHqASAYIDhsrSDYASDnAX5C/P///w+DfCHrASAAIDxsIUUgACAkbCFGIAAgNGwhR0EAIRkDQCAZID1sIBhqIgAgOWwgRWohSCAAICdsIEZqISwgACAybCBHaiEtQgAh3AEDQCAFKAIQIjogQGohCAJAIAMgFU4NAEEAIQkgAyEAIEJFBEADQCAIIAMgCWpBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACAJQQRqIgkgKEcNAAsgBiEAICAgKEYNAQsDQCAIIABBAnRqQYCAgHw2AgAgAEEBaiIAIBVHDQALCyDcASDoAXwg2wEgLxsi2gFCAFUiOwRAIA8oAtABIB0g3AGnbCAtamohASAOKALQASENQgAh2AEDQCANIBwg2AGnIhNsICtqaiELQQAhCf0MAAAAAAAAAAAAAAAAAAAAACKTAiGUAv0MAAAAAAAAAAAAAAAAAAAAACGVAv0MAAAAAAAAAAAAAAAAAAAAACGWAiACQQBKBEADQCCTAiALIAlBAnQiDGoiAP0AADAgASAMaiIM/QAAMP3mAf3kASGTAiCUAiAA/QAAICAM/QAAIP3mAf3kASGUAiCVAiAA/QAAECAM/QAAEP3mAf3kASGVAiCWAiAA/QAAACAM/QAAAP3mAf3kASGWAiAJQRBqIgkgAkgNAAsglgIglAL95AEglQIgkwL95AH95AEhkwILIBNBAnQhDCCTAv0fAyCTAv0fAiCTAv0fACCTAv0fAZKSkiGyAgJAIAIgEU4NAEEAIQkgAiEAIBYEQANAIAsgAEECdCITaioCACABIBNqKgIAlCCyApIhsgIgAEEBaiEAIAlBAWoiCSAWRw0ACwsgQw0AA0AgCyAAQQJ0IglBDGoiE2oqAgAgASATaioCAJQgCyAJQQhqIhNqKgIAIAEgE2oqAgCUIAsgCUEEaiITaioCACABIBNqKgIAlCAJIAtqKgIAIAEgCWoqAgCUILICkpKSkiGyAiAAQQRqIgAgEUcNAAsLIAggDGogsgI4AgAg2AFCAXwi2AEg2gFSDQALCwJAINoBpyIMQXBxIgFBAEwiGg0AIAFBAWsiAEEEdkEBaiILQQFxIcwBQQAhCSAAQQ9HBEAgC0H+////AXEhE0EAIQsDQCAIIAlBAnQiFGoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCCAUQcAAcmoiACCSAiAA/QAAAP3mAf0LAAAgACCSAiAA/QAAEP3mAf0LABAgACCSAiAA/QAAIP3mAf0LACAgACCSAiAA/QAAMP3mAf0LADAgCUEgaiEJIAtBAmoiCyATRw0ACwsgzAFFDQAgCCAJQQJ0aiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMAsCQCABIAxOIhMNACABIQAgDEEPcSIJQQRPBEAgACAJIAxBA3EiC2siDWohAEEAIQkDQCAIIAEgCWpBAnRqIhQgkgIgFP0AAgD95gH9CwIAIAlBBGoiCSANRw0ACyALRQ0BCwNAIAggAEECdGoiCSC0AiAJKgIAlDgCACAAQQFqIgAgDEcNAAsLAkAg2gEg2wFZDQAg2wEg2gEi2AF9IuMBQgRaBEAg2AEg4wFCfIMi5AF8IdgBQgAh4gEDQCAIINoBIOIBfKdBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACDiAUIEfCLiASDkAVINAAsg4wEg5AFRDQELA0AgCCDYAadBAnRqQYCAgHw2AgAg2AFCAXwi2AEg2wFSDQALCwJAIAxBAEwiIQRAQwAAgP8hsgIMAQtDAACA/yGyAkEAIQtBACEAIAxBAWtBA08EQCAMQXxxIRRBACEJA0AgsgIgCCAAQQJ0Ig1qKgIAIrMCILICILMCXhsisgIgCCANQQRyaioCACKzAiCyAiCzAl4bIrICIAggDUEIcmoqAgAiswIgsgIgswJeGyKyAiAIIA1BDHJqKgIAIrMCILICILMCXhshsgIgAEEEaiEAIAlBBGoiCSAURw0ACwsgDEEDcSIJRQ0AA0AgsgIgCCAAQQJ0aioCACKzAiCyAiCzAl4bIbICIABBAWohACALQQFqIgsgCUcNAAsLIAggP0ECdGohDUQAAAAAAAAAACG7AgJAIBVBAEwEQEQAAAAAAAAAACG4AkQAAAAAAAAAACG5AkQAAAAAAAAAACG6AgwBC0QAAAAAAAAAACG4AkQAAAAAAAAAACG5AkQAAAAAAAAAACG6AiDaAUIAVw0AIOEBINoBINoBIOEBVRsh4gFCACHYAQNAQwAAAAAhswIgCCDYAadBAnQiCWoiACoCACK1AkMAAID/XARAILsCILUCILICkxBFIrMCu6AhuwILIAkgDWoiCSCzAjgCAAJAINoBINgBQgGEVw0AQwAAAAAhswIgACoCBCK1AkMAAID/XARAILgCILUCILICkxBFIrMCu6AhuAILIAkgswI4AgQg2gEg2AFCAoRXDQBDAAAAACGzAiAAKgIIIrUCQwAAgP9cBEAguQIgtQIgsgKTEEUiswK7oCG5AgsgCSCzAjgCCCDaASDYAUIDhFcNAEMAAAAAIbMCIAAqAgwitQJDAACA/1wEQCC6AiC1AiCyApMQRSKzArugIboCCyAJILMCOAIMCyDYAUIEfCLYASDiAVMNAAsLRAAAAAAAAPA/ILsCRAAAAAAAAAAAoCC4AqAguQKgILoCoKO2IbICAkAgAUEATA0AIAFBAWsiAEEEdkEBaiILQQFxIc0BILIC/RMhkwJBACEJIABBD0cEQCALQf7///8BcSEXQQAhCwNAIA0gCUECdCI+aiIAIJMCIAD9AAAA/eYB/QsAACAAIJMCIAD9AAAQ/eYB/QsAECAAIJMCIAD9AAAg/eYB/QsAICAAIJMCIAD9AAAw/eYB/QsAMCANID5BwAByaiIAIJMCIAD9AAAA/eYB/QsAACAAIJMCIAD9AAAQ/eYB/QsAECAAIJMCIAD9AAAg/eYB/QsAICAAIJMCIAD9AAAw/eYB/QsAMCAJQSBqIQkgC0ECaiILIBdHDQALCyDNAUUNACANIAlBAnRqIgAgkwIgAP0AAAD95gH9CwAAIAAgkwIgAP0AABD95gH9CwAQIAAgkwIgAP0AACD95gH9CwAgIAAgkwIgAP0AADD95gH9CwAwCwJAIBMNACABIQAgDEEPcSIJQQRPBEAgACAJIAxBA3EiC2siFGohACCyAv0TIZMCQQAhCQNAIA0gASAJakECdGoiFyAX/QACACCTAv3mAf0LAgAgCUEEaiIJIBRHDQALIAtFDQELA0AgDSAAQQJ0aiIJIAkqAgAgsgKUOAIAIABBAWoiACAMRw0ACwsgIUUEQCAIQQAgDEECdPwLAAsg2QFCAFUiPgRAIAEgDEEPcSJJIAxBA3EiSmsiS2ohCSAbINwBp2wgLGohTEIAIdgBA0AgEigC0AEgRCAqINgBpyIAbGpqIRQgECgC0AEgTCAAIB9samoqAgAhsgIgGkUEQCCyAv0TIZMCQQAhCwNAIAggC0ECdCIXaiIAIJMCIBQgF2oiF/0AAAD95gEgAP0AAAD95AH9CwAAIAAgkwIgF/0AABD95gEgAP0AABD95AH9CwAQIAAgkwIgF/0AACD95gEgAP0AACD95AH9CwAgIAAgkwIgF/0AADD95gEgAP0AADD95AH9CwAwIAtBEGoiCyABSA0ACwsCQCATDQAgASEAIElBBE8EQCCyAv0TIZMCQQAhAANAIAggACABakECdCILaiIXIAsgFGr9AAIAIJMC/eYBIBf9AAIA/eQB/QsCACAAQQRqIgAgS0cNAAsgCSEAIEpFDQELA0AgCCAAQQJ0IgtqIhcgCyAUaioCACCyApQgFyoCAJI4AgAgAEEBaiIAIAxHDQALCyDYAUIBfCLYASDZAVINAAsLQQAhC/0MAAAAAAAAAAAAAAAAAAAAACKTAiGUAv0MAAAAAAAAAAAAAAAAAAAAACGVAv0MAAAAAAAAAAAAAAAAAAAAACGWAiAaRQRAA0AgkwIgDSALQQJ0IglqIgD9AAAwIAggCWoiCf0AADD95gH95AEhkwIglAIgAP0AACAgCf0AACD95gH95AEhlAIglQIgAP0AABAgCf0AABD95gH95AEhlQIglgIgAP0AAAAgCf0AAAD95gH95AEhlgIgC0EQaiILIAFIDQALIJYCIJQC/eQBIJUCIJMC/eQB/eQBIZMCCyCTAv0fAyCTAv0fAiCTAv0fACCTAv0fAZKSkiGyAgJAIBMNAEEAIQkgASEAIAxBA3EiCwRAA0AgDSAAQQJ0IhRqKgIAIAggFGoqAgCUILICkiGyAiAAQQFqIQAgCUEBaiIJIAtHDQALCyABQX9zIAxqQQNJDQADQCANIABBAnQiCUEMaiILaioCACAIIAtqKgIAlCANIAlBCGoiC2oqAgAgCCALaioCAJQgDSAJQQRqIgtqKgIAIAggC2oqAgCUIAkgDWoqAgAgCCAJaioCAJQgsgKSkpKSIbICIABBBGoiACAMRw0ACwsCQCADQQBMDQBBACEAIANBBE8EQCCyAv0TIZMCA0AgCCAAQQJ0aiIJIAn9AAIAIJMC/eUB/QsCACAAQQRqIgAgB0cNAAsgByIAIANGDQELA0AgCCAAQQJ0aiIJIAkqAgAgsgKTOAIAIABBAWoiACADRw0ACwsCQCAhDQBBACEJQQAhAAJAIAxBDEkNACAMQQJ0IgsgOiBBamogCEsgDSAIIAtqSXENACAMQXxxIQBBACELA0AgCCALQQJ0IhRqIhcgF/0AAgAgDSAUav0AAgD95gH9CwIAIAtBBGoiCyAARw0ACyAAIAxGDQELIABBf3MgDGohzgEgDEEDcSIUBEADQCAIIABBAnQiF2oiISAhKgIAIA0gF2oqAgCUOAIAIABBAWohACAJQQFqIgkgFEcNAAsLIM4BQQNJDQADQCAIIABBAnQiCWoiCyALKgIAIAkgDWoqAgCUOAIAIAggCUEEaiILaiIUIBQqAgAgCyANaioCAJQ4AgAgCCAJQQhqIgtqIhQgFCoCACALIA1qKgIAlDgCACAIIAlBDGoiCWoiCyALKgIAIAkgDWoqAgCUOAIAIABBBGoiACAMRw0ACwsCQCAaDQAgAUEBayIAQQR2QQFqIgtBAXEhzwFBACEJIABBD0cEQCALQf7///8BcSEXQQAhCwNAIAggCUECdCIhaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAIICFBwAByaiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMCAJQSBqIQkgC0ECaiILIBdHDQALCyDPAUUNACAIIAlBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCwJAIBMNACABIQAgDEEPcSIJQQRPBEAgACAJIAxBA3EiC2siFGohAEEAIQkDQCAIIAEgCWpBAnRqIhcgkgIgF/0AAgD95gH9CwIAIAlBBGoiCSAURw0ACyALRQ0BCwNAIAggAEECdGoiCSC0AiAJKgIAlDgCACAAQQFqIgAgDEcNAAsLIDsEQCAeIEgg3AGnIiEgN2xqaiEJQgAh2AEDQCAOKALQASAcINgBpyIAbCAramohFCAIIABBAnRqKgIAIbICIAJBAEwiOkUEQCCyAv0TIZMCQQAhCwNAIAkgC0ECdCIXaiIAIJMCIBQgF2oiF/0AAAD95gEgAP0AAAD95AH9CwAAIAAgkwIgF/0AABD95gEgAP0AABD95AH9CwAQIAAgkwIgF/0AACD95gEgAP0AACD95AH9CwAgIAAgkwIgF/0AADD95gEgAP0AADD95AH9CwAwIAtBEGoiCyACSA0ACwsCQCACIBFOIjsNACACIQAgI0EETwRAILIC/RMhkwJBACEAA0AgCSAAIAJqQQJ0IgtqIhcgCyAUav0AAgAgkwL95gEgF/0AAgD95AH9CwIAIABBBGoiACAlRw0ACyAEIQAgFkUNAQsDQCAJIABBAnQiC2oiFyALIBRqKgIAILIClCAXKgIAkjgCACAAQQFqIgAgEUcNAAsLINgBQgF8ItgBINoBUg0ACyAdICFsIC1qISFCACHYAQNAIA8oAtABICFqIQkgNiDrASDYASDgAX58p2ohFCAIINgBp0ECdGoqAgAhsgIgOkUEQCCyAv0TIZMCQQAhCwNAIBQgC0ECdCIXaiIAIJMCIAkgF2oiF/0AAAD95gEgAP0AAAD95AH9CwAAIAAgkwIgF/0AABD95gEgAP0AABD95AH9CwAQIAAgkwIgF/0AACD95gEgAP0AACD95AH9CwAgIAAgkwIgF/0AADD95gEgAP0AADD95AH9CwAwIAtBEGoiCyACSA0ACwsCQCA7DQAgAiEAICNBBE8EQCCyAv0TIZMCQQAhAANAIBQgACACakECdCILaiIXIAkgC2r9AAIAIJMC/eYBIBf9AAIA/eQB/QsCACAAQQRqIgAgJUcNAAsgBCEAIBZFDQELA0AgFCAAQQJ0IgtqIhcgCSALaioCACCyApQgFyoCAJI4AgAgAEEBaiIAIBFHDQALCyDYAUIBfCLYASDaAVINAAsLID4EQCABIAxBD3EiFyAMQQNxIiFrIjpqIQkgGyDcAadsICxqITtCACHYAQNAIDUg6gEg2AEg5QF+fKdqIQggECgC0AEgOyAfINgBp2xqaioCACGyAiAaRQRAILIC/RMhkwJBACELA0AgCCALQQJ0IhRqIgAgkwIgDSAUaiIU/QAAAP3mASAA/QAAAP3kAf0LAAAgACCTAiAU/QAAEP3mASAA/QAAEP3kAf0LABAgACCTAiAU/QAAIP3mASAA/QAAIP3kAf0LACAgACCTAiAU/QAAMP3mASAA/QAAMP3kAf0LADAgC0EQaiILIAFIDQALCwJAIBMNACABIQAgF0EETwRAILIC/RMhkwJBACEAA0AgCCAAIAFqQQJ0IgtqIhQgCyANav0AAgAgkwL95gEgFP0AAgD95AH9CwIAIABBBGoiACA6Rw0ACyAJIQAgIUUNAQsDQCAIIABBAnQiC2oiFCALIA1qKgIAILIClCAUKgIAkjgCACAAQQFqIgAgDEcNAAsLINgBQgF8ItgBINkBUg0ACwsg3AFCAXwi3AEg3QFSDQALIBlBAWoiGSAiRw0ACyDpASDfAUIBfCLfAVINAAsMDgtBvMMCKAIAEDAaIApB3j82AiggCkHy5QA2AiQgCkHkJjYCIEG4wwIoAgBBy+QAIApBIGoQMQw0C0G8wwIoAgAQMBogCkGxOjYCGCAKQeHlADYCFCAKQeQmNgIQQbjDAigCAEHL5AAgCkEQahAxDDMLQbzDAigCABAwGiAKQcY7NgIIIApB4OUANgIEIApB5CY2AgBBuMMCKAIAQcvkACAKEDEMMgtBvMMCKAIAEDAaIApBzj02AjggCkHf5QA2AjQgCkHkJjYCMEG4wwIoAgBBy+QAIApBMGoQMQwxC0G8wwIoAgAQMBogCkHd0AA2AkggCkHe5QA2AkQgCkHkJjYCQEG4wwIoAgBBy+QAIApBQGsQMQwwC0G8wwIoAgAQMBogCkHcLjYCWCAKQdvlADYCVCAKQeQmNgJQQbjDAigCAEHL5AAgCkHQAGoQMQwvC0G8wwIoAgAQMBogCkHDMzYCaCAKQdblADYCZCAKQeQmNgJgQbjDAigCAEHL5AAgCkHgAGoQMQwuC0G8wwIoAgAQMBogCkGbMzYCeCAKQdXlADYCdCAKQeQmNgJwQbjDAigCAEHL5AAgCkHwAGoQMQwtC0G8wwIoAgAQMBogCkGwMzYCiAEgCkHU5QA2AoQBIApB5CY2AoABQbjDAigCAEHL5AAgCkGAAWoQMQwsC0G8wwIoAgAQMBogCkGb0AA2ApgBIApB0eUANgKUASAKQeQmNgKQAUG4wwIoAgBBy+QAIApBkAFqEDEMKwtBvMMCKAIAEDAaIApBx9AANgKoASAKQdDlADYCpAEgCkHkJjYCoAFBuMMCKAIAQcvkACAKQaABahAxDCoLQbzDAigCABAwGiAKQbHQADYCuAEgCkHP5QA2ArQBIApB5CY2ArABQbjDAigCAEHL5AAgCkGwAWoQMQwpC0G8wwIoAgAQMBogCkH2wAA2AsgBIApBzeUANgLEASAKQeQmNgLAAUG4wwIoAgBBy+QAIApBwAFqEDEMKAtBvMMCKAIAEDAaIApBmyE2AtgBIApBmugANgLUASAKQeQmNgLQAUG4wwIoAgBBy+QAIApB0AFqEDEMJwsgCkHgAWokAAwACwwRCyABKAKMASEEIwBBEGsiAyQAAkAgBCgCAEUEQAJAAkAgACgCAA4DAQABAAsgASgCSCIAQQBMDQAgASgCRCIFQQBMDQAgASkDICLhAUIAVw0AIAEpAxgi4gFCAFcNACABKQMQItkBQgBXDQAgBCkDICHqASDZAUIDgyHkASDZAUJ8gyHaASAEKQMQIuMBQgKGIesBINkBQgKGIewBIAQpAxgi5QEg4wF+ItgBQgKGIe0BIOEBIOIBfiDZAX4h7gEg2QGnIgJBAnQhByACIOIBp2xBAnQhBiDZASDiAX4i3AFCAoYi7wEg4QF+IfABINgBIAEoAkwiAqwi5gF+QgKGQvz///8PgyHxASDcASDhAX4i2AFCAoZC/P///w+DIfIBINgBIAWtIucBfkIChkL8////D4Mh8wEgAiDjAadsQQJ0rSH0ASAArSH1AQNAINsBIOYBfiH2ASDbASDnAX4h9wEg2wEg8QF+IfgBINsBIPMBfiH6ASAFINsBp2whCUIAIdwBA0Ag3AEg5gF+IfkBINwBIPcBfCLYASDhAX4h+wEg+AEg3AEg9AF+fCH8ASD6ASDcASDyAX58If0BINgBIO4BfkIChiH+ASDwASAJINwBp2qtfkL8////D4Mh/wFCACHdAQNAIN0BIO8BfiHYAQJAIOoBIN0BIPYBfCLeAVUEQCDeASDlAX4hgAIg3QEg+wF8IOIBfiGBAiDYASD/AXwhggIg/AEg3QEg7QF+fCGDAiDYASD9AXwhhAJCACHeAQNAIN4BIOwBfiHoAQJAIOUBIN4BIPkBfCLYAVcEQCABKALQASDoASCCAnynakEAIAf8CwAMAQsg3gEggQJ8INkBfiHfASDYASCAAnwg4wF+IeABIAEoAtABIQAgBCgC0AEhAkIAIekBQgAh2AECQCDZAUIIVA0AIAAg6AEghAJ8p2ogAiCDAiDeASDrAX58p2prQRBJDQADQCAAINgBIN8BfKdBAnRqIAIg2AEg4AF8p0ECdGr9AAIA/QsCACDYAUIEfCLYASDaAVINAAsg2gEi2AEg2QFRDQELINkBINgBQn+FfCGPAiDkAUIAUgRAA0AgACDYASDfAXynQQJ0aiACINgBIOABfKdBAnRqKgIAOAIAINgBQgF8IdgBIOkBQgF8IukBIOQBUg0ACwsgjwJCA1QNAANAIAAg2AEg3wF8p0ECdGogAiDYASDgAXynQQJ0aioCADgCACAAINgBQgF8IugBIN8BfKdBAnRqIAIg4AEg6AF8p0ECdGoqAgA4AgAgACDYAUICfCLoASDfAXynQQJ0aiACIOABIOgBfKdBAnRqKgIAOAIAIAAg2AFCA3wi6AEg3wF8p0ECdGogAiDgASDoAXynQQJ0aioCADgCACDYAUIEfCLYASDZAVINAAsLIN4BQgF8It4BIOIBUg0ACwwBCyABKALQASDYASD+AXynakEAIAb8CwALIN0BQgF8It0BIOEBUg0ACyDcAUIBfCLcASDnAVINAAsg2wFCAXwi2wEg9QFSDQALCyADQRBqJAAMAQtBvMMCKAIAEDAaIANBmyE2AgggA0HY6AA2AgQMHAsMEAsgASgCjAEhAiMAQRBrIgQkAAJAIAIoAgBFBEACQAJAIAAoAgAOAwEAAQALIAE0AkQi2wEgASkDGCLgASDbAYF9INsBgSDgAXwg2wF/IeMBIAEpAyAi5AFCAFcNACDgAUIAVw0AIAEpAxAi2QFCAFcNACDZAUIDgyHhASDZAUJ8gyHaASDZAUIChiHlASACKQMQIuYBp0ECdCEDINkBIOABfkIChiHnASACKQMgIugBpyEFIAIpAxgi6QGnIQcgAigC0AEhACABKALQASIBrSHqAQNAINwBIOABfiHrASDcASDnAX4g6gF8IewBINwBINsBfyLYASDjAX4h7QEg3AEg2AEg2wF+fSLuAachAkIAId0BA0Ag3QEg2wF/ItgBIO0BfCLvAcQg6AF+IO4BfCDpAX4g3QEg2AEg2wF+fSLwAXwg5gF+Id4BIN0BIOsBfCDZAX4h3wFCACHiAUIAIdgBAkACQCDZAUIMVA0AIOwBIN0BIOUBfnynIAMg8AGnIAUg7wGnbCACaiAHbGpsIABqa0EQSQ0AA0AgASDYASDfAXynQQJ0aiAAINgBIN4BfKdBAnRq/QACAP0LAgAg2AFCBHwi2AEg2gFSDQALINoBItgBINkBUQ0BCyDZASDYAUJ/hXwhkAIg4QFCAFIEQANAIAEg2AEg3wF8p0ECdGogACDYASDeAXynQQJ0aioCADgCACDYAUIBfCHYASDiAUIBfCLiASDhAVINAAsLIJACQgNUDQADQCABINgBIN8BfKdBAnRqIAAg2AEg3gF8p0ECdGoqAgA4AgAgASDYAUIBfCLiASDfAXynQQJ0aiAAIN4BIOIBfKdBAnRqKgIAOAIAIAEg2AFCAnwi4gEg3wF8p0ECdGogACDeASDiAXynQQJ0aioCADgCACABINgBQgN8IuIBIN8BfKdBAnRqIAAg3gEg4gF8p0ECdGoqAgA4AgAg2AFCBHwi2AEg2QFSDQALCyDdAUIBfCLdASDgAVINAAsg3AFCAXwi3AEg5AFSDQALCyAEQRBqJAAMAQtBvMMCKAIAEDAaIARBmyE2AgggBEGU6QA2AgQMGgsMDwsgASgCjAEhBCMAQdACayICJAACQAJAAkACQAJAAkACQAJAAkACQCABKAJAQT1GBEACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQCABKAJEDgoAAQIDBAUGBwgJDAsgBCgCAEUEQAJAIAAoAgAOAwsACwALIAQpAyggBCkDICAEKQMYfn6nIgpBAEwNCiAEKAIQIgZBAEwNCiAEKAI0IQggASgCNCELIAQoAtABIQwgASgC0AEhDSAGQQNxIQkgBkF8cSEBQQAhBCAGQQRJIQ8DQCAMIAQgCGxqIQUgDSAEIAtsaiEHQQAhAAJAAkAgDw0AIAcgBWtBEEkNAANAIAcgAEECdCIDaiADIAVq/QACAP3gAf0LAgAgAEEEaiIAIAFHDQALIAEiACAGRg0BCyAAQX9zIAZqIdABQQAhAyAJBEADQCAHIABBAnQiEGogBSAQaioCAIs4AgAgAEEBaiEAIANBAWoiAyAJRw0ACwsg0AFBA0kNAANAIAcgAEECdCIDaiADIAVqKgIAizgCACAHIANBBGoiDmogBSAOaioCAIs4AgAgByADQQhqIg5qIAUgDmoqAgCLOAIAIAcgA0EMaiIDaiADIAVqKgIAizgCACAAQQRqIgAgBkcNAAsLIARBAWoiBCAKRw0ACwwKC0G8wwIoAgAQMBogAkGbITYCGCACQZbDADYCFCACQeQmNgIQQbjDAigCAEHL5AAgAkEQahAxDDkLIAQoAgBFBEACQCAAKAIADgMKAAoACyAEKQMoIAQpAyAgBCkDGH5+pyIJQQBMDQkgBCgCECIDQQBMDQkgBCgCNCEKIAEoAjQhCCAEKALQASELIAEoAtABIQxBACEEQQAgA2shDSADQQFxIQ8gA0F8cSEBIANBBEkhDgNAIAsgBCAKbGohBSAMIAQgCGxqIQdBACEAAkACQCAODQAgByAFa0EQSQ0AA0AgByAAQQJ0IgZq/QwAAIA/AACAPwAAgD8AAIA//QwAAIC/AACAvwAAgL8AAIC//QwAAAAAAAAAAAAAAAAAAAAAIAUgBmr9AAIAIpIC/QwAAAAAAAAAAAAAAAAAAAAA/UP9UiCSAv0MAAAAAAAAAAAAAAAAAAAAAP1E/VL9CwIAIABBBGoiACABRw0ACyABIgAgA0YNAQsgAEF/cyHRASAPBEAgByAAQQJ0IhBqQwAAgD9DAACAv0MAAAAAIAUgEGoqAgAisgJDAAAAAF0bILICQwAAAABeGzgCACAAQQFyIQALINEBIA1GDQADQCAHIABBAnQiBmpDAACAP0MAAIC/QwAAAAAgBSAGaioCACKyAkMAAAAAXRsgsgJDAAAAAF4bOAIAIAcgBkEEaiIGakMAAIA/QwAAgL9DAAAAACAFIAZqKgIAIrICQwAAAABdGyCyAkMAAAAAXhs4AgAgAEECaiIAIANHDQALCyAEQQFqIgQgCUcNAAsMCQtBvMMCKAIAEDAaIAJBmyE2AiggAkHAwwA2AiQgAkHkJjYCIEG4wwIoAgBBy+QAIAJBIGoQMQw4CyAEKAIARQRAAkAgACgCAA4DCQAJAAsgBCkDKCAEKQMgIAQpAxh+fqciCkEATA0IIAQoAhAiBkEATA0IIAQoAjQhCCABKAI0IQsgBCgC0AEhDCABKALQASENIAZBA3EhCSAGQXxxIQFBACEEIAZBBEkhDwNAIAwgBCAIbGohBSANIAQgC2xqIQdBACEAAkACQCAPDQAgByAFa0EQSQ0AA0AgByAAQQJ0IgNqIAMgBWr9AAIA/eEB/QsCACAAQQRqIgAgAUcNAAsgASIAIAZGDQELIABBf3MgBmoh0gFBACEDIAkEQANAIAcgAEECdCIQaiAFIBBqKgIAjDgCACAAQQFqIQAgA0EBaiIDIAlHDQALCyDSAUEDSQ0AA0AgByAAQQJ0IgNqIAMgBWoqAgCMOAIAIAcgA0EEaiIOaiAFIA5qKgIAjDgCACAHIANBCGoiDmogBSAOaioCAIw4AgAgByADQQxqIgNqIAMgBWoqAgCMOAIAIABBBGoiACAGRw0ACwsgBEEBaiIEIApHDQALDAgLQbzDAigCABAwGiACQZshNgI4IAJB6sMANgI0IAJB5CY2AjBBuMMCKAIAQcvkACACQTBqEDEMNwsgBCgCAEUEQAJAIAAoAgAOAwgACAALIAQpAyggBCkDICAEKQMYfn6nIgpBAEwNByAEKAIQIgZBAEwNByAEKAI0IQggASgCNCELIAQoAtABIQwgASgC0AEhDSAGQQNxIQkgBkF8cSEBQQAhBCAGQQRJIQ8DQCAMIAQgCGxqIQUgDSAEIAtsaiEHQQAhAAJAAkAgDw0AIAcgBWtBEEkNAANAIAcgAEECdCIDav0MAACAPwAAgD8AAIA/AACAP/0MAAAAAAAAAAAAAAAAAAAAACADIAVq/QACAP0MAAAAAAAAAAAAAAAAAAAAAP1E/VL9CwIAIABBBGoiACABRw0ACyABIgAgBkYNAQsgAEF/cyAGaiHTAUEAIQMgCQRAA0AgByAAQQJ0IhBqQwAAgD9DAAAAACAFIBBqKgIAQwAAAABeGzgCACAAQQFqIQAgA0EBaiIDIAlHDQALCyDTAUEDSQ0AA0AgByAAQQJ0IgNqQwAAgD9DAAAAACADIAVqKgIAQwAAAABeGzgCACAHIANBBGoiDmpDAACAP0MAAAAAIAUgDmoqAgBDAAAAAF4bOAIAIAcgA0EIaiIOakMAAIA/QwAAAAAgBSAOaioCAEMAAAAAXhs4AgAgByADQQxqIgNqQwAAgD9DAAAAACADIAVqKgIAQwAAAABeGzgCACAAQQRqIgAgBkcNAAsLIARBAWoiBCAKRw0ACwwHC0G8wwIoAgAQMBogAkGbITYCSCACQZTEADYCRCACQeQmNgJAQbjDAigCAEHL5AAgAkFAaxAxDDYLIAQoAgBFBEACQCAAKAIADgMHAAcACyAEKQMoIAQpAyAgBCkDGH5+pyIFQQBMDQYgBCgCECIHQQBMDQYgBCgCNCEGIAEoAjQhCSAEKALQASEEIAEoAtABIQEDQCAEIAMgBmxqIQogASADIAlsaiEIQQAhAANAIAggAEECdCILaiAKIAtqKgIAEI0EOAIAIABBAWoiACAHRw0ACyADQQFqIgMgBUcNAAsMBgtBvMMCKAIAEDAaIAJBmyE2AlggAkG+xAA2AlQgAkHkJjYCUEG4wwIoAgBBy+QAIAJB0ABqEDEMNQsgBCgCAEUEQAJAIAAoAgAOAwYABgALIAQpAyggBCkDICAEKQMYfn6nIgZBAEwNBSAEKAIQIgNBAEwNBSAEKAI0IQkgASgCNCEKIAQoAtABIQggASgC0AEhC0EAIQRBACADayEMIANBAXEhDSADQXxxIQEgA0EESSEPA0AgCCAEIAlsaiEFIAsgBCAKbGohB0EAIQACQAJAIA8NACAHIAVrQRBJDQADQCAFIABBAnQiDmr9AAIAIZICIAcgDmogkgIgkgL9HwAQRf0TIJIC/R8BEEX9IAEgkgL9HwIQRf0gAiCSAv0fAxBF/SAD/QwAAIC/AACAvwAAgL8AAIC//eQBIJIC/QwAAAAAAAAAAAAAAAAAAAAA/UT9Uv0LAgAgAEEEaiIAIAFHDQALIAEiACADRg0BCyAAQX9zIQ4gDQRAIAUgAEECdCIQaioCACKyAkMAAAAAXkUEQCCyAhBFQwAAgL+SIbICCyAHIBBqILICOAIAIABBAXIhAAsgDCAORg0AA0AgBSAAQQJ0Ig5qKgIAIrICQwAAAABeRQRAILICEEVDAACAv5IhsgILIAcgDmogsgI4AgAgBSAAQQFqQQJ0Ig5qKgIAIrICQwAAAABeRQRAILICEEVDAACAv5IhsgILIAcgDmogsgI4AgAgAEECaiIAIANHDQALCyAEQQFqIgQgBkcNAAsMBQtBvMMCKAIAEDAaIAJBmyE2AmggAkHoxAA2AmQgAkHkJjYCYEG4wwIoAgBBy+QAIAJB4ABqEDEMNAsgBCgCAEUEQAJAIAAoAgAOAwUABQALIAQpAyggBCkDICAEKQMYfn6nIgpBAEwNBCAEKAIQIgZBAEwNBCAEKAI0IQggASgCNCELIAQoAtABIQwgASgC0AEhDSAGQQNxIQkgBkF8cSEBQQAhBCAGQQRJIQ8DQCAMIAQgCGxqIQUgDSAEIAtsaiEHQQAhAAJAAkAgDw0AIAcgBWtBEEkNAANAIAcgAEECdCIDav0MAAAAAAAAAAAAAAAAAAAAACADIAVq/QACAP3rAf0LAgAgAEEEaiIAIAFHDQALIAEiACAGRg0BCyAAQX9zIAZqIdQBQQAhAyAJBEADQCAHIABBAnQiEGogBSAQaioCACKyAkMAAAAAILICQwAAAABeGzgCACAAQQFqIQAgA0EBaiIDIAlHDQALCyDUAUEDSQ0AA0AgByAAQQJ0IgNqIAMgBWoqAgAisgJDAAAAACCyAkMAAAAAXhs4AgAgByADQQRqIg5qIAUgDmoqAgAisgJDAAAAACCyAkMAAAAAXhs4AgAgByADQQhqIg5qIAUgDmoqAgAisgJDAAAAACCyAkMAAAAAXhs4AgAgByADQQxqIgNqIAMgBWoqAgAisgJDAAAAACCyAkMAAAAAXhs4AgAgAEEEaiIAIAZHDQALCyAEQQFqIgQgCkcNAAsMBAtBvMMCKAIAEDAaIAJBmyE2AnggAkGSxQA2AnQgAkHkJjYCcEG4wwIoAgBBy+QAIAJB8ABqEDEMMwsgBCgCAEUEQCAEKAIwQQRHDQcgBDUCOCLZASAEKQMYItgBIAQoAjQiB61+Ug0HIAQ1Ajwg2QEgBCkDICLaAX5SDQcgASgCMCABKAIAQSRsQZibAWooAgBHDQggATUCOCLZASABKQMYItsBIAEoAjQiBq1+Ug0IIAE1Ajwg2QEgASkDICLcAX5SDQgg2gEg3AFSDQkg2AEg2wFSDQkgBCkDECLZASABKQMQUg0JIAQpAygi2wEgASkDKFINCQJAIAAoAgAOAwQABAALINgBINoBfiDbAX6nIgUgACgCCCIDakEBayADbSIJIAAoAgRsIgMgAyAJaiIAIAUgACAFSBsiBU4NAyDZAaciCUEATA0DIAQoAtABIQQgASgC0AEhAQNAIAQgAyAHbGohCiABIAMgBmxqIQhBACEAA0AgCCAAQQJ0IgtqQYD8ASAKIAtqKgIAIrICi0MAAIB3lEMAAIAIlEGAgICIByCyArwiC0EBdCIMQYCAgHhxIg0gDUGAgICIB00bQQF2QYCAgDxqvpK8Ig1BDXZBgPgBcSANQf8fcWogDEGAgIB4SxsgC0EQdkGAgAJxckEBdEGQ1hRqLwEAQQJ0QZDWBGoqAgA4AgAgAEEBaiIAIAlHDQALIANBAWoiAyAFRw0ACwwDC0G8wwIoAgAQMBogAkGbITYCuAEgAkHNxQA2ArQBIAJB5CY2ArABQbjDAigCAEHL5AAgAkGwAWoQMQwyCyAEKAIARQRAIAQoAjBBBEcNCSAENQI4ItkBIAQpAxgi2AEgBCgCNCIHrX5SDQkgBDUCPCDZASAEKQMgItoBflINCSABKAIwIAEoAgBBJGxBmJsBaigCAEcNCiABNQI4ItkBIAEpAxgi2wEgASgCNCIGrX5SDQogATUCPCDZASABKQMgItwBflINCiDaASDcAVINCyDYASDbAVINCyAEKQMQItkBIAEpAxBSDQsgBCkDKCLbASABKQMoUg0LAkAgACgCAA4DAwADAAsg2AEg2gF+INsBfqciBSAAKAIIIgNqQQFrIANtIgkgACgCBGwiAyADIAlqIgAgBSAAIAVIGyIFTg0CINkBpyIJQQBMDQIgBCgC0AEhBCABKALQASEBA0AgBCADIAdsaiEKIAEgAyAGbGohCEEAIQADQCAIIABBAnQiC2pBgPwBIAogC2oqAgAisgKLQwAAgHeUQwAAgAiUQYCAgIgHILICvCILQQF0IgxBgICAeHEiDSANQYCAgIgHTRtBAXZBgICAPGq+krwiDUENdkGA+AFxIA1B/x9xaiAMQYCAgHhLGyALQRB2QYCAAnFyQQF0QZDWHGovAQBBAnRBkNYEaioCADgCACAAQQFqIgAgCUcNAAsgA0EBaiIDIAVHDQALDAILQbzDAigCABAwGiACQZshNgL4ASACQYjGADYC9AEgAkHkJjYC8AFBuMMCKAIAQcvkACACQfABahAxDDELIAQoAgANASAEKAIwQQRHDQogBDUCOCLZASAEKQMYItgBIAQoAjQiB61+Ug0KIAQ1Ajwg2QEgBCkDICLaAX5SDQogASgCMCABKAIAQSRsQZibAWooAgBHDQsgATUCOCLZASABKQMYItsBIAEoAjQiBq1+Ug0LIAE1Ajwg2QEgASkDICLcAX5SDQsg2gEg3AFSDQwg2AEg2wFSDQwgBCkDECLZASABKQMQUg0MIAQpAygi2wEgASkDKFINDAJAIAAoAgAOAwEAAQALINgBINoBfiDbAX6nIgUgACgCCCIDakEBayADbSIJIAAoAgRsIgMgAyAJaiIAIAUgACAFSBsiBU4NACDZAaciCUEATA0AIAQoAtABIQQgASgC0AEhAQNAIAQgAyAHbGohCiABIAMgBmxqIQhBACEAA0AgCCAAQQJ0IgtqQYD8ASAKIAtqKgIAIrICi0MAAIB3lEMAAIAIlEGAgICIByCyArwiC0EBdCIMQYCAgHhxIg0gDUGAgICIB00bQQF2QYCAgDxqvpK8Ig1BDXZBgPgBcSANQf8fcWogDEGAgIB4SxsgC0EQdkGAgAJxckEBdEGQ1iRqLwEAQQJ0QZDWBGoqAgA4AgAgAEEBaiIAIAlHDQALIANBAWoiAyAFRw0ACwsgAkHQAmokAAwMC0G8wwIoAgAQMBogAkGbITYCuAIgAkHDxgA2ArQCIAJB5CY2ArACQbjDAigCAEHL5AAgAkGwAmoQMQwuC0G8wwIoAgAQMBogAkGbITYCCCACQczpADYCBCACQeQmNgIAQbjDAigCAEHL5AAgAhAxDC0LQbzDAigCABAwGiACQb8qNgLIAiACQeoXNgLEAiACQeQmNgLAAkG4wwIoAgBBy+QAIAJBwAJqEDEMLAtBvMMCKAIAEDAaIAJBv9sANgKoASACQZ3FADYCpAEgAkHkJjYCoAFBuMMCKAIAQcvkACACQaABahAxDCsLQbzDAigCABAwGiACQfnNADYCmAEgAkGexQA2ApQBIAJB5CY2ApABQbjDAigCAEHL5AAgAkGQAWoQMQwqC0G8wwIoAgAQMBogAkG5zwA2AogBIAJBn8UANgKEASACQeQmNgKAAUG4wwIoAgBBy+QAIAJBgAFqEDEMKQtBvMMCKAIAEDAaIAJBv9sANgLoASACQdjFADYC5AEgAkHkJjYC4AFBuMMCKAIAQcvkACACQeABahAxDCgLQbzDAigCABAwGiACQfnNADYC2AEgAkHZxQA2AtQBIAJB5CY2AtABQbjDAigCAEHL5AAgAkHQAWoQMQwnC0G8wwIoAgAQMBogAkG5zwA2AsgBIAJB2sUANgLEASACQeQmNgLAAUG4wwIoAgBBy+QAIAJBwAFqEDEMJgtBvMMCKAIAEDAaIAJBv9sANgKoAiACQZPGADYCpAIgAkHkJjYCoAJBuMMCKAIAQcvkACACQaACahAxDCULQbzDAigCABAwGiACQfnNADYCmAIgAkGUxgA2ApQCIAJB5CY2ApACQbjDAigCAEHL5AAgAkGQAmoQMQwkC0G8wwIoAgAQMBogAkG5zwA2AogCIAJBlcYANgKEAiACQeQmNgKAAkG4wwIoAgBBy+QAIAJBgAJqEDEMIwsMDgsgASgCjAEhBCMAQRBrIgIkAAJAIAQoAgBBAUYEQAJAAkAgACgCAA4DAQABAAsgASkDICLjAUIAVw0AIAEpAxgi3wFCAFcNACABKQMQItkBQgBXDQAg2QFCA4Mh4QEg2QFCeIMh2gEgBCkDECLgAUL+////D34h5AEg4AFCAYYh5QEg2QFCAYYh5gEg2QEg3wF+QgGGIecBIAQoAtABIgAg4AGnIN8Bp0EBa2xBAXRqrSHoASABKALQASIBrSHpAQNAINsBIN8BfiHqASDbASDfAXwh6wEg2wEg5QF+IOgBfCHsASDbASDnAX4g6QF8Ie0BQgAh3AEDQCDcASDqAXwg2QF+Id0BIOsBINwBQn+FfCDgAX4h3gFCACHiAUIAIdgBAkACQCDZAUIIVA0AIO0BINwBIOYBfnynIOwBINwBIOQBfnyna0EQSQ0AA0AgASDYASDdAXynQQF0aiAAINgBIN4BfKdBAXRq/QABAP0LAQAg2AFCCHwi2AEg2gFSDQALINoBItgBINkBUQ0BCyDZASDYAUJ/hXwhkQIg4QFCAFIEQANAIAEg2AEg3QF8p0EBdGogACDYASDeAXynQQF0ai8BADsBACDYAUIBfCHYASDiAUIBfCLiASDhAVINAAsLIJECQgNUDQADQCABINgBIN0BfKdBAXRqIAAg2AEg3gF8p0EBdGovAQA7AQAgASDYAUIBfCLiASDdAXynQQF0aiAAIN4BIOIBfKdBAXRqLwEAOwEAIAEg2AFCAnwi4gEg3QF8p0EBdGogACDeASDiAXynQQF0ai8BADsBACABINgBQgN8IuIBIN0BfKdBAXRqIAAg3gEg4gF8p0EBdGovAQA7AQAg2AFCBHwi2AEg2QFSDQALCyDcAUIBfCLcASDfAVINAAsg2wFCAXwi2wEg4wFSDQALCyACQRBqJAAMAQtBvMMCKAIAEDAaIAJBmyE2AgggAkH56QA2AgQMFAsMDQsgASgCjAEhAyABKAKQASECIAEoApQBIQcjAEEQayIEJAACQCADKAIARQRAAkAgACgCACIFIAEoAkRyRQRAIAEpAyAh2AEgASkDGCHaASADKALQASEAIAEoAtABIQIgASgCACIDQRBrQXFNBEAgAiAAIANBJGxBmJsBaigCACABKAIwIAEoAhBBAWtsaiABKAI0INoBp0EBa2xqIAEoAjgg2AGnQQFrbGogASgCPCABKAIoQQFrbGr8CgAADAILIAIgACABNQI0INoBQgF9fiABKQMQIAE1AjB+IANBJGxBlJsBajUCAH98IAE1Ajgg2AFCAX1+fCABNQI8IAEpAyhCAX1+fKf8CgAADAELAkAgBQ4DAQABAAsgAigCKCIDIAAoAggiBWpBAWsgBW0iBiAAKAIEbCIFIAUgBmoiACADIAAgA0gbIgBODQAgAikDICLgAUIAVw0AIAIpAxgi4QFCAFcNACACKQMQItgBQgBXDQAgAKwh4wEgASgC0AEhACAHKALQASEBIAIoAtABIQJCASDYAX0h5AEg2AFCfoMh5QEg2AFCAYMh5gEgBawh2wEg2AFCAVEhAwNAINsBIOABfiHnAUIAIdwBA0Ag3AEg5wF8IOEBfiHoAUIAId0BA0Ag3QEg6AF8INgBfiHpAUIAIdkBA0Ag2QEg6QF8ItoBINgBfiLeASDZASDkAX58Id8BIAEg2gGnQQJ0IgVqKgIAIbICIAIgBWoqAgAhswJCACHaAUIAIeIBIANFBEADQCAAINoBIN4BfKdBAnRqIgUgsgIgBSoCAJI4AgAgACDfASDYASDaAX58p0ECdGoiBSCzAiAFKgIAkjgCACAAINoBQgGEIuoBIN4BfKdBAnRqIgUgsgIgBSoCAJI4AgAgACDfASDYASDqAX58p0ECdGoiBSCzAiAFKgIAkjgCACDaAUICfCHaASDiAUICfCLiASDlAVINAAsLIOYBpwRAIAAg2gEg3gF8p0ECdGoiBSCyAiAFKgIAkjgCACAAIN8BINgBINoBfnynQQJ0aiIFILMCIAUqAgCSOAIACyDZAUIBfCLZASDYAVINAAsg3QFCAXwi3QEg4QFSDQALINwBQgF8ItwBIOABUg0ACyDbAUIBfCLbASDjAVINAAsLIARBEGokAAwBC0G8wwIoAgAQMBogBEGbITYCCCAEQc7qADYCBAwXCwwMCyABKAKMASECIAEoAkQhAyMAQSBrIgQkAAJAAkAgAigCAEUEQCACKQMQItgBIAEpAxBSDQEgAikDGCLaASABKQMYUg0BIAIpAyAi2QEgASkDIFINASACKQMoItsBIAEpAyhSDQECQAJAIAAoAgAOAwEAAQALINkBINoBfiDbAX6nIgVBAEwNACDYAachB0EAIQAgBUEBRwRAIAVBfnEhCQNAIAcgASgC0AEgASgCNCAAbGogAigC0AEgAigCNCAAbGogAxEFACAHIAEoAtABIABBAXIiCiABKAI0bGogAigC0AEgAigCNCAKbGogAxEFACAAQQJqIQAgBkECaiIGIAlHDQALCyAFQQFxRQ0AIAcgASgC0AEgASgCNCAAbGogAigC0AEgAigCNCAAbGogAxEFAAsgBEEgaiQADAILQbzDAigCABAwGiAEQZshNgIYIARB+eoANgIUDCALQbzDAigCABAwGiAEQbnPADYCCCAEQdrqADYCBAwWCwwLCyABKAKMASECIAEoApABIQMgASgCRCEFIwBBEGsiBCQAAkAgAigCAEUEQAJAAkAgACgCAA4DAQABAAsgAikDKCACKQMgIAIpAxh+fqciB0EATA0AIAIoAhAhBkEAIQADQCAGIAEoAtABIAEoAjQgAGxqIAIoAtABIAIoAjQgAGxqIAMoAtABIAMoAjQgAGxqIAURBgAgAEEBaiIAIAdHDQALCyAEQRBqJAAMAQtBvMMCKAIAEDAaIARBmyE2AgggBEGp6wA2AgQMFQsMCgsCQCAAKAIADgMKAAoACyABIAEoAowBIAEoAkQRAgAMCQsCQCAAKAIADgMJAAkACyABIAEoAowBIAEoApABIAEoAkQRBQAMCAsCQCAAKAIADgMIAAgACyABIAEoAowBIAEoApABIAEoApQBIAEoAkQRBgAMBwsgASgCjAEhAgJAAkAgACgCAA4DAQABAAsgASACIAAoAgQgACgCCCABKAJMIAEoAkQRCwALDAYLIAEoAowBIQIgASgCkAEhBAJAAkAgACgCAA4DAQABAAsgASACIAQgACgCBCAAKAIIIAEoAkwgASgCRBEMAAsMBQsgASgCjAEhAiABKAKQASEEIAEoApQBIQMCQAJAIAAoAgAOAwEAAQALIAEgAiAEIAMgACgCBCAAKAIIIAEoAkwgASgCRBEQAAsMBAsgACEEIAEoAowBIQkgASgCkAEhCiMAQeAAayIFJAACQAJAAn0CQCAJKAIARQRAAkAgCSgCMEEERw0AIAk1AjQi2QEgCSkDECLYAUIChlINACAJNQI4ItsBINkBIAkpAxgi2gF+Ug0AIAk1Ajwg2wEgCSkDICLZAX5SDQACQCAKKAIwIgAgCigCAEEkbCICQZibAWooAgBHDQAgCjUCNCLbASAKKQMQItwBIACtfiACQZSbAWo0AgB/Ug0AIAo1Ajgi3QEg2wEgCikDGCLeAX5SDQAgCjUCPCDdASAKKQMgItsBflINAAJAIAEpAxBCAVINACABKQMYQgFSDQAgASkDIEIBUg0AIAEpAyhCAVINAAJAINgBINwBUg0AINoBIN4BUg0AINkBINsBUg0AIAkpAygi2wEgCikDKFINACAEKAIIIgAg2AGnIgZBAnQiE0EEamwiAyAEKAIMTQRAINkBINoBfiDbAX6nIQIgBCgCECEHIAQoAgQhCAJAAkACQCAEKAIADgMAAgECCyAIDQogB0EAIAP8CwAMCgsgCA0JIAEoAtABIQlDAAAAACAAQQBMDQgaIABBA3EhBEEAIQMgAEEESQRAQQAhAAwICyAAQXxxIQpBACEAQQAhBgNAILgCIAcgAEECdCIBaioCALugIAcgAUEEcmoqAgC7oCAHIAFBCHJqKgIAu6AgByABQQxyaioCALugIbgCIABBBGohACAGQQRqIgYgCkcNAAsMBwsgACACakEBayAAbSIBIAhsIg0gASANaiIBIAIgASACSBsiFk4NCCAGQXBxIgIgBkEPcSIVIAZBA3EiDGsiGGohAyAAQQJ0IhogCEECdCIAQQRqIAZsaiEZIAZBfHEhASAGQX5xIRQgBkEBcSEXIAZBAWshDiAAIAdqIRAgAkEBayIbQQR2QQFqIgBB/v///wFxIRwgAEEBcSEdIAYgCGxBAnQhHwNAIAQoAhAiHiAaaiAfaiEHIAooAjQhIyAKKALQASERAkAgBkEATCIlBEBEAAAAAAAAAAAhuAIMAQsgCSgC0AEgCSgCNCANbGohC0MAAID/IbICQQAhD0EAIQBBACESIA5BA08EQANAILICIAsgAEECdCIIaioCACKzAiCyAiCzAl4bIrICIAsgCEEEcmoqAgAiswIgsgIgswJeGyKyAiALIAhBCHJqKgIAIrMCILICILMCXhsisgIgCyAIQQxyaioCACKzAiCyAiCzAl4bIbICIABBBGohACASQQRqIhIgAUcNAAsLIAwEQANAILICIAsgAEECdGoqAgAiswIgsgIgswJeGyGyAiAAQQFqIQAgD0EBaiIPIAxHDQALC0QAAAAAAAAAACG4AkEAIQBBACEIIA4EQANAQwAAAAAhswICQCALIABBAnQiD2oqAgAitAJDAACA/1sEQEMAAAAAIbQCDAELILgCILQCILICkxBFIrQCu6AhuAILIAcgD2ogtAI4AgAgCyAAQQFyQQJ0Ig9qKgIAIrQCQwAAgP9cBEAguAIgtAIgsgKTEEUiswK7oCG4AgsgByAPaiCzAjgCACAAQQJqIQAgCEECaiIIIBRHDQALCyAXRQ0AQwAAAAAhswIgCyAAQQJ0IgBqKgIAIrQCQwAAgP9cBEAguAIgtAIgsgKTEEUiswK7oCG4AgsgACAHaiCzAjgCAAtEoY92////7z8guAKjtiGyAgJAIAJBAEwNACCyAv0TIZICQQAhAEEAIQggG0EPRwRAA0AgByAAQQJ0Ig9qIgsgkgIgC/0AAAD95gH9CwAAIAsgkgIgC/0AABD95gH9CwAQIAsgkgIgC/0AACD95gH9CwAgIAsgkgIgC/0AADD95gH9CwAwIAcgD0HAAHJqIgsgkgIgC/0AAAD95gH9CwAAIAsgkgIgC/0AABD95gH9CwAQIAsgkgIgC/0AACD95gH9CwAgIAsgkgIgC/0AADD95gH9CwAwIABBIGohACAIQQJqIgggHEcNAAsLIB1FDQAgByAAQQJ0aiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMAsCQCACIAZODQAgAiEAIBVBBE8EQCCyAv0TIZICQQAhAANAIAcgACACakECdGoiCCAI/QACACCSAv3mAf0LAgAgAEEEaiIAIBhHDQALIAMhACAMRQ0BCwNAIAcgAEECdGoiCCAIKgIAILIClDgCACAAQQFqIgAgBkcNAAsLIBAgJQR9QwAAAAAFQQAhAAJAIAZBBEkiCEUEQANAIAcgAEECdGoiCyAL/QACAP0MX3CJMF9wiTBfcIkwX3CJMP3kAf0LAgAgAEEEaiIAIAFHDQALIAEiACAGRg0BCwNAIAcgAEECdGoiCyALKgIAQ19wiTCSOAIAIABBAWoiACAGRw0ACwsgDSAjbCEPQQAhAAJAIAhFBEADQCAHIABBAnRqIgggCP0AAgAikgL9HwAQV/0TIJIC/R8BEFf9IAEgkgL9HwIQV/0gAiCSAv0fAxBX/SAD/QsCACAAQQRqIgAgAUcNAAsgASIAIAZGDQELA0AgByAAQQJ0aiIIIAgqAgAQVzgCACAAQQFqIgAgBkcNAAsLIA8gEWohC0EAIQhBACEAAkACQCAGQQhJDQAgCyAZIB5qSSARIBNqIA9qIAdLcQ0AA0AgByAAQQJ0Ig9qIhEgEf0AAgAgCyAPav0AAgD95gH9CwIAIABBBGoiACABRw0ACyABIgAgBkYNAQsgAEF/cyAGaiHVASAMBEADQCAHIABBAnQiEWoiEiASKgIAIAsgEWoqAgCUOAIAIABBAWohACAIQQFqIgggDEcNAAsLINUBQQJNDQADQCAHIABBAnQiCGoiDyAPKgIAIAggC2oqAgCUOAIAIAcgCEEEaiIPaiIRIBEqAgAgCyAPaioCAJQ4AgAgByAIQQhqIg9qIhEgESoCACALIA9qKgIAlDgCACAHIAhBDGoiCGoiDyAPKgIAIAggC2oqAgCUOAIAIABBBGoiACAGRw0ACwtEAAAAAAAAAAAhuAJBACEIQQAhAEEAIQ8gDkEDTwRAA0AguAIgByAAQQJ0IgtqKgIAu6AgByALQQRyaioCALugIAcgC0EIcmoqAgC7oCAHIAtBDHJqKgIAu6AhuAIgAEEEaiEAIA9BBGoiDyABRw0ACwsgDARAA0AguAIgByAAQQJ0aioCALugIbgCIABBAWohACAIQQFqIgggDEcNAAsLILgCtgsgECoCAJI4AgAgFiANQQFqIg1HDQALDAgLQbzDAigCABAwGiAFQanWADYCCCAFQabsADYCBCAFQeQmNgIAQbjDAigCAEHL5AAgBRAxDCALQbzDAigCABAwGiAFQYvZADYCGCAFQZvsADYCFCAFQeQmNgIQQbjDAigCAEHL5AAgBUEQahAxDB8LQbzDAigCABAwGiAFQeXNADYCKCAFQZrsADYCJCAFQeQmNgIgQbjDAigCAEHL5AAgBUEgahAxDB4LQbzDAigCABAwGiAFQbvYADYCOCAFQZnsADYCNCAFQeQmNgIwQbjDAigCAEHL5AAgBUEwahAxDB0LQbzDAigCABAwGiAFQd/ZADYCSCAFQZjsADYCRCAFQeQmNgJAQbjDAigCAEHL5AAgBUFAaxAxDBwLQbzDAigCABAwGiAFQZshNgJYIAVBie0ANgJUIAVB5CY2AlBBuMMCKAIAQcvkACAFQdAAahAxDBsLIAQEQANAILgCIAcgAEECdGoqAgC7oCG4AiAAQQFqIQAgA0EBaiIDIARHDQALCyC4ArYLIbICIAlDAACAvyACspUgsgKUOAIACyAFQeAAaiQADAALDAMLIAEoAowBIQkgASgCkAEhCiABKAKUASECIAEhAyMAQeAAayIFJAACQCAJKAIARQRAAkAgASgCMCIBIAMoAgBBJGwiBEGYmwFqKAIARw0AIAM1AjQi2QEgAykDECLYASABrX4gBEGUmwFqNAIAf1INACADNQI4ItsBINkBIAMpAxgi2gF+Ug0AIAM1Ajwg2wEgAykDICLZAX5SDQACQCAJKAIwQQRHDQAgCTUCNCLdASAJKQMQItsBQgKGUg0AIAk1Ajgi3gEg3QEgCSkDGCLcAX5SDQAgCTUCPCDeASAJKQMgIt0BflINAAJAIAooAjAiASAKKAIAQSRsIgRBmJsBaigCAEcNACAKNQI0It4BIAopAxAi3wEgAa1+IARBlJsBajQCAH9SDQAgCjUCOCLgASDeASAKKQMYIuEBflINACAKNQI8IOABIAopAyAi3gF+Ug0AAkAgAigCMCIBIAIoAgBBJGwiBEGYmwFqKAIARw0AIAI1AjQi4AEgAikDECABrX4gBEGUmwFqNAIAf1INACACNQI4IuIBIAIpAxgg4AF+Ug0AIAI1AjwgAikDICDiAX5SDQACQCDbASDfAVINACDcASDhAVINACDdASDeAVINACDZASDdAVINACDaASDcAVINACDYASDbAVINACAJKQMoItsBIAopAyhSDQAg2wEgAykDKFINAAJAAkAgACgCAA4DAQABAAsg2QEg2gF+INsBfiLZASAANAIIItoBfEIBfSDaAX8i2wEgADQCBH4i2gEg2gEg2wF8ItsBINkBINkBINsBVRsi3AFZDQAgAigC0AEhGCDYAaciBkFwcSIBIAZBD3EiDSAGQQNxIgxrIg9qIQQg2AFCfoMh3QEg2AFCAYMh3gEgBkF8cSECIAZBAnQhDiABQQFrIhBBBHZBAWoiAEH+////AXEhESAAQQFxIRIg2QG0IbUCIAZBAWtBA0khGgNAIAkoAtABINoBpyITIAkoAjRsaiEAIAMoAjQgE2whFkMAAID/IbICAkAgBkEATCIZDQBBACEIQQAhB0EAIRUgGkUEQANAILICIAAgB0ECdCILaioCACKzAiCyAiCzAl4bIrICIAAgC0EEcmoqAgAiswIgsgIgswJeGyKyAiAAIAtBCHJqKgIAIrMCILICILMCXhsisgIgACALQQxyaioCACKzAiCyAiCzAl4bIbICIAdBBGohByAVQQRqIhUgAkcNAAsLIAxFDQADQCCyAiAAIAdBAnRqKgIAIrMCILICILMCXhshsgIgB0EBaiEHIAhBAWoiCCAMRw0ACwsgCigCNCEUIAooAtABIRUgAygC0AEiFyAWaiEHAkAg2AFCAFcEQEQAAAAAAAAAACG4AgwBC0QAAAAAAAAAACG4AkIAIdkBQgAh2wEg2AFCAVIEQANAQwAAAAAhswICQCAAINkBpyIIQQJ0IgtqKgIAIrQCQwAAgP9bBEBDAAAAACG0AgwBCyC4AiC0AiCyApMQRSK0ArugIbgCCyAHIAtqILQCOAIAIAAgCEEBckECdCIIaioCACK0AkMAAID/XARAILgCILQCILICkxBFIrMCu6AhuAILIAcgCGogswI4AgAg2QFCAnwh2QEg2wFCAnwi2wEg3QFSDQALCyDeAVANAEMAAAAAIbMCIAAg2QGnQQJ0IghqKgIAIrQCQwAAgP9cBEAguAIgtAIgsgKTEEUiswK7oCG4AgsgByAIaiCzAjgCAAtEoY92////7z8guAKjtiGyAgJAIAFBAEwiGw0AILIC/RMhkgJBACEAQQAhCCAQQQ9HBEADQCAHIABBAnQiHGoiCyCSAiAL/QAAAP3mAf0LAAAgCyCSAiAL/QAAEP3mAf0LABAgCyCSAiAL/QAAIP3mAf0LACAgCyCSAiAL/QAAMP3mAf0LADAgByAcQcAAcmoiCyCSAiAL/QAAAP3mAf0LAAAgCyCSAiAL/QAAEP3mAf0LABAgCyCSAiAL/QAAIP3mAf0LACAgCyCSAiAL/QAAMP3mAf0LADAgAEEgaiEAIAhBAmoiCCARRw0ACwsgEkUNACAHIABBAnRqIgAgkgIgAP0AAAD95gH9CwAAIAAgkgIgAP0AABD95gH9CwAQIAAgkgIgAP0AACD95gH9CwAgIAAgkgIgAP0AADD95gH9CwAwCwJAIAEgBk4iHA0AIAEhACANQQRPBEAgsgL9EyGSAkEAIQADQCAHIAAgAWpBAnRqIgggCP0AAgAgkgL95gH9CwIAIABBBGoiACAPRw0ACyAEIQAgDEUNAQsDQCAHIABBAnRqIgggCCoCACCyApQ4AgAgAEEBaiIAIAZHDQALCwJAIBkNACATIBRsIRNBACEAAkAgBkEETwRAA0AgByAAQQJ0aiIIIAj9AAIA/QxfcIkwX3CJMF9wiTBfcIkw/eQB/QsCACAAQQRqIgAgAkcNAAsgBiACIgBGDQELA0AgByAAQQJ0aiIIIAgqAgBDX3CJMJI4AgAgAEEBaiIAIAZHDQALCyATIBVqIQtBACEIQQAhAAJAIAZBCEkNACALIA4gF2ogFmpJIA4gFWogE2ogB0txDQADQCAHIABBAnQiE2oiFiAW/QACACALIBNq/QACAP3lAf0LAgAgAEEEaiIAIAJHDQALIAYgAiIARg0BCyAAQX9zIAZqIdYBIAwEQANAIAcgAEECdCIWaiIVIBUqAgAgCyAWaioCAJM4AgAgAEEBaiEAIAhBAWoiCCAMRw0ACwsg1gFBA0kNAANAIAcgAEECdCIIaiITIBMqAgAgCCALaioCAJM4AgAgByAIQQRqIhNqIhYgFioCACALIBNqKgIAkzgCACAHIAhBCGoiE2oiFiAWKgIAIAsgE2oqAgCTOAIAIAcgCEEMaiIIaiITIBMqAgAgCCALaioCAJM4AgAgAEEEaiIAIAZHDQALCyAYKgIAILUClSGyAgJAIBsNACCyAv0TIZICQQAhAEEAIQggEEEPRwRAA0AgByAAQQJ0IhNqIgsgkgIgC/0AAAD95gH9CwAAIAsgkgIgC/0AABD95gH9CwAQIAsgkgIgC/0AACD95gH9CwAgIAsgkgIgC/0AADD95gH9CwAwIAcgE0HAAHJqIgsgkgIgC/0AAAD95gH9CwAAIAsgkgIgC/0AABD95gH9CwAQIAsgkgIgC/0AACD95gH9CwAgIAsgkgIgC/0AADD95gH9CwAwIABBIGohACAIQQJqIgggEUcNAAsLIBJFDQAgByAAQQJ0aiIAIJICIAD9AAAA/eYB/QsAACAAIJICIAD9AAAQ/eYB/QsAECAAIJICIAD9AAAg/eYB/QsAICAAIJICIAD9AAAw/eYB/QsAMAsCQCAcDQAgASEAIA1BBE8EQCCyAv0TIZICQQAhAANAIAcgACABakECdGoiCCCSAiAI/QACAP3mAf0LAgAgAEEEaiIAIA9HDQALIAQhACAMRQ0BCwNAIAcgAEECdGoiCCCyAiAIKgIAlDgCACAAQQFqIgAgBkcNAAsLINoBQgF8ItoBINwBUg0ACwsgBUHgAGokAAwGC0G8wwIoAgAQMBogBUHYzgA2AgggBUGa7QA2AgQgBUHkJjYCAEG4wwIoAgBBy+QAIAUQMQwcC0G8wwIoAgAQMBogBUGr2QA2AhggBUGZ7QA2AhQgBUHkJjYCEEG4wwIoAgBBy+QAIAVBEGoQMQwbC0G8wwIoAgAQMBogBUG72AA2AiggBUGY7QA2AiQgBUHkJjYCIEG4wwIoAgBBy+QAIAVBIGoQMQwaC0G8wwIoAgAQMBogBUHf2QA2AjggBUGX7QA2AjQgBUHkJjYCMEG4wwIoAgBBy+QAIAVBMGoQMQwZC0G8wwIoAgAQMBogBUH0ywA2AkggBUGW7QA2AkQgBUHkJjYCQEG4wwIoAgBBy+QAIAVBQGsQMQwYC0G8wwIoAgAQMBogBUGbITYCWCAFQfftADYCVCAFQeQmNgJQQbjDAigCAEHL5AAgBUHQAGoQMQwXCwwCC0G8wwIoAgAQMBogLkGbITYCGCAuQcvwADYCFCAuQeQmNgIQQbjDAigCAEHL5AAgLkEQahAxDBULIAAgASgCjAEgARD1AgsgLkFAayQADwtBvMMCKAIAEDAaIC5BxRA2AgggLkH/7QA2AgQgLkHkJjYCAEG4wwIoAgBBy+QAIC4QMQwSC0G8wwIoAgAQMBogLkGpPjYCKCAuQebvADYCJCAuQeQmNgIgQbjDAigCAEHL5AAgLkEgahAxDBELQbzDAigCABAwGiAuQak+NgI4IC5B8e8ANgI0IC5B5CY2AjBBuMMCKAIAQcvkACAuQTBqEDEMEAsgAkHkJjYCIEG4wwIoAgBBy+QAIAJBIGoQMQwPCyACQeQmNgIQQbjDAigCAEHL5AAgAkEQahAxDA4LIAJB5CY2AgBBuMMCKAIAQcvkACACEDEMDQsgAkHkJjYCMEG4wwIoAgBBy+QAIAJBMGoQMQwMCyACQeQmNgJAQbjDAigCAEHL5AAgAkFAaxAxDAsLIAVB5CY2AgBBuMMCKAIAQcvkACAFEDEMCgsgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMQwJCyADQeQmNgIAQbjDAigCAEHL5AAgAxAxDAgLIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEMBwsgA0HkJjYCIEG4wwIoAgBBy+QAIANBIGoQMQwGCyAFQeQmNgIQQbjDAigCAEHL5AAgBUEQahAxDAULIAVB5CY2AiBBuMMCKAIAQcvkACAFQSBqEDEMBAsgBEHkJjYCQEG4wwIoAgBBy+QAIARBQGsQMQwDCyAEQeQmNgIwQbjDAigCAEHL5AAgBEEwahAxDAILIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEMAQsgBEHkJjYCEEG4wwIoAgBBy+QAIARBEGoQMQsQAAALiAEBAX8gAC0AC0EHdgR/IAAoAghB/////wdxQQFrBUEKCyIDIAJPBEACfyAALQALQQd2BEAgACgCAAwBCyAACyIDIAEgAvwKAAAgACADIAIQnAMPCyAAIAMgAiADawJ/IAAtAAtBB3YEQCAAKAIEDAELIAAtAAtB/wBxCyIAQQAgACACIAEQzAELiAMBBX8jAEEQayIIJAAgAiABQX9zQe////8Hak0EQAJ/IAAtAAtBB3YEQCAAKAIADAELIAALIQogCEEEaiIJIAAgAUHn////A0kEfyAIIAFBAXQ2AgwgCCABIAJqNgIEIwBBEGsiAiQAIAkoAgAgCEEMaiILKAIASSEMIAJBEGokACALIAkgDBsoAgAiAkELTwR/IAJBEGpBcHEiAiACQQFrIgIgAkELRhsFQQoLQQFqBUHv////BwsQwwEgCCgCBCECIAgoAggaIAQEQCACIAogBBB3CyAGBEAgAiAEaiAHIAYQdwsgAyAEIAVqIglrIQcgAyAJRwRAIAIgBGogBmogBCAKaiAFaiAHEHcLIAFBAWoiAUELRwRAIAAgCiABEPEBCyAAIAI2AgAgACAAKAIIQYCAgIB4cSAIKAIIQf////8HcXI2AgggACAAKAIIQYCAgIB4cjYCCCAAIAQgBmogB2oiADYCBCAIQQA6AAwgACACaiAILQAMOgAAIAhBEGokAA8LEE0ACwsAIAQgAjYCAEEDC+kCAgV/AX4CfyMAQTBrIgMkAAJAAkAgAikDEEIBUg0AIAIpAxhCAVINACACKQMgQgFSDQAgAikDKEIBUg0AIAEoAjAgASgCACIEQSRsQZibAWooAgBHDQEgATUCOCIIIAEpAxggATUCNH5SDQEgATUCPCABKQMgIAh+Ug0BAn8CQCABKAKIAQ0AIAIoAogBDQBBAAwBC0EBCyEHIAAgBCABKAIMIAFBEGpBAEEAEEkiBEEaNgJAIAcEQCAAIAQoAgAgBCgCDCAEQRBqQQBBABBJIQULIAQgATYCjAEgBCAFNgKIASAEIAI2ApABIANBMGokACAEDAILQbzDAigCABAwGiADQYLXADYCKCADQcogNgIkIANB5CY2AiBBuMMCKAIAQcvkACADQSBqEDEQAAALQbzDAigCABAwGiADQfLXADYCGCADQcsgNgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEQAAALC3gBAn8jAEEQayIDJAAgA0EMaiIEIAEoAhwiATYCACABQQRqQQH+HgIAGiACIAQQvwEiASABKAIAKAIQEQAANgIAIAAgASABKAIAKAIUEQIAIAQoAgAiAEEEakF//h4CAEUEQCAAIAAoAgAoAggRAQALIANBEGokAAtwAQJ/IwBBEGsiAiQAIAJBDGoiAyAAKAIcIgA2AgAgAEEEakEB/h4CABogAxBsIgBB0OoCQerqAiABIAAoAgAoAjARCAAaIAMoAgAiAEEEakF//h4CAEUEQCAAIAAoAgAoAggRAQALIAJBEGokACABC3gBAn8jAEEQayIDJAAgA0EMaiIEIAEoAhwiATYCACABQQRqQQH+HgIAGiACIAQQwQEiASABKAIAKAIQEQAAOgAAIAAgASABKAIAKAIUEQIAIAQoAgAiAEEEakF//h4CAEUEQCAAIAAoAgAoAggRAQALIANBEGokAAsLACAAEJoEIAAQLwspAAJAIAAoAgBBAE4NACAAQf////8H/h4CAEGBgICAeEYNACAAEI4BCwtIAQJ/AkAgAigCTEEASARAIAAgASACEM4CIQAMAQsgAhCCASEEIAAgASACEM4CIQAgBEUNACACEIEBCyAAIAFGBEAgAQ8LIAALjwICAX8CfCAAQQNxBH9BZAVEAAAAAAAAAAAQ1gEjBkUEQAJ/EAMhBUEAQQAgAP5IApDxNBoCQBADIgQgBSACoCICZA0AAn8DQEEAIABBACAAQQD+SAKQ8TQiAyAAIANGG0UNARogBBDWASABIAD+EAIARgRAQQBBACAA/kgCkPE0GhADIgQgAmQNAwwBCwtBegsMAQtBACAAQQD+SAKQ8TQaQbd/Cw8LIAJEAAAAAAAA8H9iIQNBekG3f0EAIAAgAQJ+IAJEAAAAAABAj0CiRAAAAAAAQI9AoiICmUQAAAAAAADgQ2MEQCACsAwBC0KAgICAgICAgIB/C0J/IAMb/gECACIAQQJGGyAAQQFGGwsLJQACQCMFBEBBjPE0/hACAA0BENoGCw8LQYzxNP4QAgAQJxAmAAuyAQMCfAF/AX4jASIDLQAIRQRAIAMQKDoACSADQQE6AAgLIAACfgJ8IwEtAAkEQBADDAELIwNBHGpBHDYCAA8LIgFEAAAAAABAj0CjIgKZRAAAAAAAAOBDYwRAIAKwDAELQoCAgICAgICAgH8LIgQ3AwAgAAJ/IAEgBELoB365oUQAAAAAAECPQKJEAAAAAABAj0CiIgGZRAAAAAAAAOBBYwRAIAGqDAELQYCAgIB4CzYCCAvKBAICfwJ8IwBBEGshBCACKwMAIQYCfwJAIAErAwAiByAAKwMAZEUEQEEAIAYgB2RFDQIaIAQgAf0AAwD9CwMAIAEgAigCCDYCCCABIAIpAwA3AwAgAiAEKAIINgIIIAIgBCkDADcDAEEBIAErAwAgACsDAGRFDQIaIAQgAP0AAwD9CwMAIAAgASgCCDYCCCAAIAEpAwA3AwAgASAEKAIINgIIIAEgBCkDADcDAAwBCyAGIAdkBEAgBCAA/QADAP0LAwAgACACKAIINgIIIAAgAikDADcDACACIAQoAgg2AgggAiAEKQMANwMAQQEMAgsgBCAA/QADAP0LAwAgACABKAIINgIIIAAgASkDADcDACABIAQoAgg2AgggASAEKQMANwMAQQEgAisDACABKwMAZEUNARogBCAB/QADAP0LAwAgASACKAIINgIIIAEgAikDADcDACACIAQoAgg2AgggAiAEKQMANwMAC0ECCyEFIAMrAwAgAisDAGQEfyAEIAL9AAMA/QsDACACIAMoAgg2AgggAiADKQMANwMAIAMgBCgCCDYCCCADIAQpAwA3AwAgAisDACABKwMAZEUEQCAFQQFqDwsgBCAB/QADAP0LAwAgASACKAIINgIIIAEgAikDADcDACACIAQoAgg2AgggAiAEKQMANwMAIAErAwAgACsDAGRFBEAgBUECag8LIAQgAP0AAwD9CwMAIAAgASgCCDYCCCAAIAEpAwA3AwAgASAEKAIINgIIIAEgBCkDADcDACAFQQNqBSAFCwunAwELfwJAAkAgACgCBCIDIAAoAgBHBEAgAyECDAELIAAoAggiBCAAKAIMIgJJBEAgBCACIARrQQJ1QQFqQQJtQQJ0IgZqIAQgA2siBWsiAiADIAX8CgAAIAAgAjYCBCAAIAAoAgggBmo2AggMAQtBASACIANrQQF1IAIgA0YbIgVBgICAgARPDQEgBUECdCICEDIiByACaiEMIAcgBUEDaiIIQXxxaiICIQkCQCADIARGDQAgAiAEIANrIgRBfHFqIQkgAiEGIAMhBQJAIARBBGsiBEEcSQ0AIAhBfHEgB2ogA2tBEEkNACADIARBAnZBAWoiCEH8////B3EiCkECdCIGaiEFIAIgBmohBgNAIAIgC0ECdCIEaiADIARq/QACAP0LAgAgC0EEaiILIApHDQALIAggCkYNAQsDQCAGIAUoAgA2AgAgBUEEaiEFIAZBBGoiBiAJRw0ACwsgACAMNgIMIAAgCTYCCCAAIAI2AgQgACAHNgIAIANFDQAgAxAvIAAoAgQhAgsgAkEEayABKAIANgIAIAAgACgCBEEEazYCBA8LEEcAC78VARh/IAAoAgAhEyAAKAIEIgMoAgghAAJAIAEoAhgiCyADKAIETQRAIAtBA3QhBCALQQJ0IQIMAQsgAARAIAAQLwsgAygCDCIABEAgABAvCyALQQJ0IgIQOyEAIAMgCzYCBCADIAA2AgggAyALQQN0IgQQOzYCDAsgAEEAIAL8CwAgAygCDEEAIAT8CwAgAyATNgIAQQAhACMAQfAAayIFJAAgAygCHCENIAMoAhghEiABIgsoAgQiCEEASgRAA0ACQCALKAIMIABBAnRqKAIAIgQoAsgBIgFFDQAgBSADKQIENwNoIAVB6ABqIAEQYiEBIAMoAgwgAUEDdGoiASABKAIEQQFqNgIEIAQoAggNACAEKALQAUUNACADIARBARB4CwJAIAQoAowBIgJFDQAgBSADKQIENwNgIAVB4ABqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAAkAgAigCyAFFDQAgAigCCA0AIAIoAtABRQ0AIAMgAkEBEHgLIAQoApABIgJFDQAgBSADKQIENwNYIAVB2ABqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAAkAgAigCyAFFDQAgAigCCA0AIAIoAtABRQ0AIAMgAkEBEHgLIAQoApQBIgJFDQAgBSADKQIENwNQIAVB0ABqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAAkAgAigCyAFFDQAgAigCCA0AIAIoAtABRQ0AIAMgAkEBEHgLIAQoApgBIgJFDQAgBSADKQIENwNIIAVByABqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAAkAgAigCyAFFDQAgAigCCA0AIAIoAtABRQ0AIAMgAkEBEHgLIAQoApwBIgJFDQAgBSADKQIENwNAIAVBQGsgAhBiIQEgAygCDCABQQN0aiIBIAEoAgBBAWo2AgACQCACKALIAUUNACACKAIIDQAgAigC0AFFDQAgAyACQQEQeAsgBCgCoAEiAkUNACAFIAMpAgQ3AzggBUE4aiACEGIhASADKAIMIAFBA3RqIgEgASgCAEEBajYCAAJAIAIoAsgBRQ0AIAIoAggNACACKALQAUUNACADIAJBARB4CyAEKAKkASICRQ0AIAUgAykCBDcDMCAFQTBqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAAkAgAigCyAFFDQAgAigCCA0AIAIoAtABRQ0AIAMgAkEBEHgLIAQoAqgBIgJFDQAgBSADKQIENwMoIAVBKGogAhBiIQEgAygCDCABQQN0aiIBIAEoAgBBAWo2AgACQCACKALIAUUNACACKAIIDQAgAigC0AFFDQAgAyACQQEQeAsgBCgCrAEiAkUNACAFIAMpAgQ3AyAgBUEgaiACEGIhASADKAIMIAFBA3RqIgEgASgCAEEBajYCAAJAIAIoAsgBRQ0AIAIoAggNACACKALQAUUNACADIAJBARB4CyAEKAKwASICRQ0AIAUgAykCBDcDGCAFQRhqIAIQYiEBIAMoAgwgAUEDdGoiASABKAIAQQFqNgIAIAIoAsgBRQ0AIAIoAggNACACKALQAUUNACADIAJBARB4CyAAQQFqIgAgCygCBCIISA0ACwsgDSAIIA0bIhVBAEoEQEEAIQgDQCAIIQAgFQJ/AkAgDQRAIBIgAEECdGooAgAiAEF/Rg0BCwJAIAsoAgwgAEECdGooAgAiASgCjAEiAEUNACADIAAQhgEgASgCkAEiAEUNACADIAAQhgEgASgClAEiAEUNACADIAAQhgEgASgCmAEiAEUNACADIAAQhgEgASgCnAEiAEUNACADIAAQhgEgASgCoAEiAEUNACADIAAQhgEgASgCpAEiAEUNACADIAAQhgEgASgCqAEiAEUNACADIAAQhgEgASgCrAEiAEUNACADIAAQhgEgASgCsAEiAEUNACADIAAQhgELIAMgARCGASANRQ0AIBIgCEECdGooAgBBf0YNACAIQQFqDAELIBEgCCANGyIAIAggCEEBaiIUIA0bIhZIBEADQCAAIQIgCygCDCANBH8gEiAAQQJ0aigCAAUgAgtBAnRqKAIAIRdBACEIA0AgFyAIQQJ0aigCjAEiAARAIAUgAykCBDcDECAFQRBqIAAQYiEBIAMoAgwgAUEDdGoiBCAEKAIAQQFrIgE2AgACQCABDQAgBCgCBA0AIAAoAsgBIgEEQCAFIAMpAgQ3AwggBUEIaiABEGIhACADKAIMIABBA3RqIgQgBCgCBEEBayIANgIEIAANASABIQAgBCgCAA0BCyMAQSBrIg8kACADKAIAIgZFBEAgAygCFCEYIA8gAykCBDcDGCAYIA9BGGogABBiQQJ0aigCACEGCwJAAkACQCAGKAIAIgEgACgCCEcNACAAKALQASEQIAEgABDnBCIAIAYoAgwiASAAIAFwayABcGohDgJAIAYoAhAiAUEATARAQQAhAAwBCyAOIBBqIQRBACEAA0AgECAGIABBA3RqIgooAhQiDCAKKAIYIgdqRgRAIAogByAOaiIJNgIYIAAgAUEBayIOTg0DIAYgAEEBaiIEQQN0aiIHKAIUIAkgDGpHDQMgCiAHKAIYIAlqNgIYIAYgDjYCECAEIA5ODQMCQCABIABrQQJrIgxBAkkEQCAEIQAMAQsgBCAMQX5xIglqIQBBACEBA0AgBiABIARqQQN0aiIHIAf9AAIc/QsCFCABQQJqIgEgCUcNAAsgCSAMRg0ECyAGQRRqIQEDQCABIABBA3RqIAEgAEEBaiIAQQN0aikCADcCACAAIA5HDQALDAMLIAQgDEYEQCAKIBA2AhQgCiAHIA5qIgc2AhggAEUNAyAQIABBA3QgBmoiCSgCDCAJKAIQIgRqRw0DIAkgBCAHajYCECAGIAFBAWsiDDYCECAAIAxODQMCQCAMIABrIglBAkkEQCAAIQEMAQsgACAJQX5xIgdqIQFBACEKA0AgBiAAIApqQQN0aiIEIAT9AAIc/QsCFCAKQQJqIgogB0cNAAsgByAJRg0ECyAGQRRqIQADQCAAIAFBA3RqIAAgAUEBaiIBQQN0aikCADcCACABIAxHDQALDAMLIABBAWoiACABRw0ACyABQYACTg0CQQAhAAJAA0AgBiAAQQN0aigCFCAQTw0BIABBAWoiACABRw0ACyABIQAMAQsgACABTg0AIAEgAEF/c2ohGSABIABrQQNxIgQEQEEAIQogBkEUaiEJA0AgCSABQQN0aiAJIAFBAWsiAUEDdGopAgA3AgAgCkEBaiIKIARHDQALCyAZQQJLBEAgBkEUaiEHA0AgByABQQN0aiIEIARBCGspAgA3AgAgBEEQayAEQRhrIgT9AAIA/QsCACAEIAcgAUEEayIBQQN0aikCADcCACAAIAFIDQALCyAGKAIQIQELIAYgAEEDdGoiACAONgIYIAAgEDYCFCAGIAFBAWo2AhALIA9BIGokAAwBC0G8wwIoAgAQMBogD0GG3QA2AgggD0HPATYCBCAPQdonNgIAQbjDAigCAEHL5AAgDxAxEAAACwsgCEEBaiIIQQpHDQELCyACQQFqIgAgFkcNAAsLIBQgESANGyERIBQLIghHDQALCyAFQfAAaiQAIANBADYCACATKAKUEBoLggUCA38BfiMAQeAAayICJAAgASEDAkACQAJAAkACQAJAAkACQAJAIAAoAkAOSQcICAgIBwgIBwcHBwcHBwcHCAgICAgICAgIBwcIBwcHBwcHBwcICAEICAgHBwgICAcHCAgIBwgICAcHBwgABwcHBwcCAwQICAUGC0EBIQMjAEEQayIEJAAgACgCQEE9RwRAQbzDAigCABAwGiAEQb8qNgIIIARB6hc2AgQgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMRAAAAsgACgCRCEAIARBEGokACAAQQdJDQcgASEDIABBB2tBAk0NB0G8wwIoAgAQMBogAkGbITYCSCACQcT8ADYCRCACQeQmNgJAQbjDAigCAEHL5AAgAkFAaxAxEAAAC0EEIAEgAUEEThsiASAAKAKMASIAKQMgIAApAxh+IAApAyh+IgWnIAUgAaxVGyEDDAYLIAEgACgCSCIAIAEgACABSBsgAEF/RhshAwwFCyABIAAoAkgiACABIAAgAUgbIABBf0YbIQMMBAsgASAAKAJIIgAgASAAIAFIGyAAQX9GGyEDDAMLQbzDAigCABAwGiACQZshNgJYIAJB9/0ANgJUIAJB5CY2AlBBuMMCKAIAQcvkACACQdAAahAxEAAACyACQcwQNgIwQbjDAigCACIBQYLiACACQTBqEDECQCAAKAJAIgBBxwBNBEAgAiAAQQJ0QcCgAWooAgA2AhAgAUHw5AAgAkEQahAxDAELIAIgADYCICABQaDuACACQSBqEDELQbzDAigCABAwGiACQZshNgIIIAJBgf4ANgIEIAJB5CY2AgAgAUHL5AAgAhAxEAAAC0EBIQMLIAJB4ABqJAAgAwtJAQJ/IAAoAgQiBUEIdSEGIAAoAgAiACABIAVBAXEEfyAGIAIoAgBqKAIABSAGCyACaiADQQIgBUECcRsgBCAAKAIAKAIYEQsACxkBAX8gACgCACIBBEAgARCKBQsgARAvIAALewEBfyMAQRBrIgMkAAJAIAJBCk0EQCAAIAAtAAtBgAFxIAJyOgALIAAgAC0AC0H/AHE6AAsgACABIAIQdyADQQA6AA8gACACaiADLQAPOgAADAELIABBCiACQQprIAAtAAtB/wBxIgBBACAAIAIgARDMAQsgA0EQaiQAC3gBAn8jAEEQayIEJAACQCACIAAoAghB/////wdxIgNJBEAgACgCACEDIAAgAjYCBCADIAEgAhB3IARBADoADyACIANqIAQtAA86AAAMAQsgACADQQFrIAIgA2tBAWogACgCBCIAQQAgACACIAEQzAELIARBEGokAAsJACAAIAEQmwMLUQEBfyMAQSBrIggkACAIIAM3AwggCCACNwMAIAggBDcDECAAIAFBAyAIIAcQqwIiACAEpyAGbDYCPCAAIAY2AjggACAFNgI0IAhBIGokACAAC4sDAgd/A34jAEHQAGsiBSQAAkACQCABKAIwIgYgASgCACIIQSRsIgdBmJsBaigCAEcNACABNQI0IgwgASkDECIOIAatfiAHQZSbAWo0AgB/Ug0AIAE1AjgiDSAMIAEpAxgiDH5SDQAgATUCPCANIAEpAyAiDX5SDQAgASkDKCAMIA5+IA1+fiACIAN+IAR+Ug0BIAEoAogBIQsgBSAENwNAIAUgAzcDOCAFIAI3AzBBACEHIAAgCEEDIAVBMGogAUEAEEkhCiAFIAFB1AFqNgIAIApBktUAIAUQcyIGQR42AkAgCwRAIAAgBigCACAGKAIMIAZBEGpBAEEAEEkhBwsgBiABNgKMASAGIAc2AogBIAVB0ABqJAAgBg8LQbzDAigCABAwGiAFQcXXADYCKCAFQZcjNgIkIAVB5CY2AiBBuMMCKAIAQcvkACAFQSBqEDEQAAALQbzDAigCABAwGiAFQYs7NgIYIAVBmCM2AhQgBUHkJjYCEEG4wwIoAgBBy+QAIAVBEGoQMRAAAAsEAEEEC/8CAgd/A34jAEFAaiIEJAACQAJAIAEoAjAiBSABKAIAIgdBJGwiBkGYmwFqKAIARw0AIAE1AjQiCyABKQMQIg0gBa1+IAZBlJsBajQCAH9SDQAgATUCOCIMIAsgASkDGCILflINACABNQI8IAwgASkDICIMflINACABKQMoIAsgDX4gDH5+IAIgA35SDQEgASgCiAEhCiAEIAM3AzggBCACNwMwQQAhBiAAIAdBAiAEQTBqIAFBABBJIQkgBCABQdQBajYCACAJQZLVACAEEHMiBUEeNgJAIAoEQCAAIAUoAgAgBSgCDCAFQRBqQQBBABBJIQYLIAUgATYCjAEgBSAGNgKIASAEQUBrJAAgBQ8LQbzDAigCABAwGiAEQcXXADYCKCAEQf0iNgIkIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEQAAALQbzDAigCABAwGiAEQaU9NgIYIARB/iI2AhQgBEHkJjYCEEG4wwIoAgBBy+QAIARBEGoQMRAAAAukAQEBfwJAQYyKNf4SAABBAXENAEGMijUQVEUNAAJAQYCKNf4SAABBAXENAEGAijUQVEUNABDUBUH4iTVB6JY1NgIAQfyJNUH4iTU2AgBBgIo1EFMLQYSKNUH8iTUoAgAoAgAiATYCACABQQRqQQH+HgIAGkGIijVBhIo1NgIAQYyKNRBTCyAAQYiKNSgCACgCACIBNgIAIAFBBGpBAf4eAgAaIAALXwEEfyABKAKIASEFIAAgASgCACABKAIMIAFBEGpBAEEAEEkiAkK9gICA8AA3A0AgBQRAIAAgAigCACACKAIMIAJBEGpBAEEAEEkhAwsgAiABNgKMASACIAM2AogBIAILRwEBfyMAQRBrIgIkAAJAIAEtAAtBB3ZFBEAgACABKAIINgIIIAAgASkCADcCAAwBCyAAIAEoAgAgASgCBBBrCyACQRBqJAALCABB/////wcLBQBB/wALdgEBfyMAQRBrIgIkACACIAA2AgwCQCAAIAFGDQADQCACIAFBBGsiATYCCCAAIAFPDQEgAigCDCIAKAIAIQEgACACKAIIIgAoAgA2AgAgACABNgIAIAIgAigCDEEEaiIANgIMIAIoAgghAQwACwALIAJBEGokAAv2BAEIfyMAQRBrIgskACAGEGwhCSALQQRqIgcgBhC/ASIIIAgoAgAoAhQRAgACQAJ/IActAAtBB3YEQCAHKAIEDAELIActAAtB/wBxC0UEQCAJIAAgAiADIAkoAgAoAjARCAAaIAUgAyACIABrQQJ0aiIGNgIADAELIAUgAzYCAAJAAkAgACIKLQAAIgZBK2sOAwABAAELIAkgBsAgCSgCACgCLBEEACEHIAUgBSgCACIGQQRqNgIAIAYgBzYCACAAQQFqIQoLAkAgAiAKa0ECSA0AIAotAABBMEcNACAKLQABQSByQfgARw0AIAlBMCAJKAIAKAIsEQQAIQcgBSAFKAIAIgZBBGo2AgAgBiAHNgIAIAkgCiwAASAJKAIAKAIsEQQAIQcgBSAFKAIAIgZBBGo2AgAgBiAHNgIAIApBAmohCgsgCiACEKwBIAggCCgCACgCEBEAACEOQQAhByAKIQYDfyACIAZNBH8gAyAKIABrQQJ0aiAFKAIAEOoBIAUoAgAFAkACfyALQQRqIggtAAtBB3YEQCAIKAIADAELIAgLIAdqLQAARQ0AIAwCfyAILQALQQd2BEAgCCgCAAwBCyAICyAHaiwAAEcNACAFIAUoAgAiDUEEajYCACANIA42AgAgByAHAn8gCC0AC0EHdgRAIAgoAgQMAQsgCC0AC0H/AHELQQFrSWohB0EAIQwLIAkgBiwAACAJKAIAKAIsEQQAIQ0gBSAFKAIAIghBBGo2AgAgCCANNgIAIAZBAWohBiAMQQFqIQwMAQsLIQYLIAQgBiADIAEgAGtBAnRqIAEgAkYbNgIAIAtBBGoQMxogC0EQaiQAC9ABAQJ/IAJBgBBxBEAgAEErOgAAIABBAWohAAsgAkGACHEEQCAAQSM6AAAgAEEBaiEACyACQYQCcSIDQYQCRwRAIABBrtQAOwAAIABBAmohAAsgAkGAgAFxIQIDQCABLQAAIgQEQCAAIAQ6AAAgAEEBaiEAIAFBAWohAQwBCwsgAAJ/AkAgA0GAAkcEQCADQQRHDQFBxgBB5gAgAhsMAgtBxQBB5QAgAhsMAQtBwQBB4QAgAhsgA0GEAkYNABpBxwBB5wAgAhsLOgAAIANBhAJHC+0EAQh/IwBBEGsiCyQAIAYQciEJIAtBBGoiByAGEMEBIgggCCgCACgCFBECAAJAAn8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELRQRAIAkgACACIAMgCSgCACgCIBEIABogBSADIAIgAGtqIgY2AgAMAQsgBSADNgIAAkACQCAAIgotAAAiBkEraw4DAAEAAQsgCSAGwCAJKAIAKAIcEQQAIQcgBSAFKAIAIgZBAWo2AgAgBiAHOgAAIABBAWohCgsCQCACIAprQQJIDQAgCi0AAEEwRw0AIAotAAFBIHJB+ABHDQAgCUEwIAkoAgAoAhwRBAAhByAFIAUoAgAiBkEBajYCACAGIAc6AAAgCSAKLAABIAkoAgAoAhwRBAAhByAFIAUoAgAiBkEBajYCACAGIAc6AAAgCkECaiEKCyAKIAIQrAEgCCAIKAIAKAIQEQAAIQ5BACEHIAohBgN/IAIgBk0EfyADIAogAGtqIAUoAgAQrAEgBSgCAAUCQAJ/IAtBBGoiCC0AC0EHdgRAIAgoAgAMAQsgCAsgB2otAABFDQAgDAJ/IAgtAAtBB3YEQCAIKAIADAELIAgLIAdqLAAARw0AIAUgBSgCACINQQFqNgIAIA0gDjoAACAHIAcCfyAILQALQQd2BEAgCCgCBAwBCyAILQALQf8AcQtBAWtJaiEHQQAhDAsgCSAGLAAAIAkoAgAoAhwRBAAhDSAFIAUoAgAiCEEBajYCACAIIA06AAAgBkEBaiEGIAxBAWohDAwBCwshBgsgBCAGIAMgASAAa2ogASACRhs2AgAgC0EEahAzGiALQRBqJAAL7QUBC38jAEGAAWsiCSQAIAkgATYCfCAJQfYBNgIQIAlBCGpBACAJQRBqIggQUSELAkACQCADIAJrQQxtIgpB5QBPBEAgChA7IghFDQEgCygCACEBIAsgCDYCACABBEAgASALKAIEEQEACwsgCCEHIAIhAQNAIAEgA0YEQANAIAAgCUH8AGoQQkEBIAobBEAgACAJQfwAahBCBEAgBSAFKAIAQQJyNgIACwwFCwJ/IAAoAgAiBygCDCIBIAcoAhBGBEAgByAHKAIAKAIkEQAADAELIAEoAgALIQ4gBkUEQCAEIA4gBCgCACgCHBEEACEOCyAPQQFqIQ1BACEQIAghByACIQEDQCABIANGBEAgDSEPIBBFDQIgABBcGiAIIQcgAiEBIAogDGpBAkkNAgNAIAEgA0YEQAwEBQJAIActAABBAkcNAAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyAPRg0AIAdBADoAACAMQQFrIQwLIAdBAWohByABQQxqIQEMAQsACwAFAkAgBy0AAEEBRw0AAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsgD0ECdGooAgAhEQJAIAYEfyARBSAEIBEgBCgCACgCHBEEAAsgDkYEQEEBIRACfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsgDUcNAiAHQQI6AAAgDEEBaiEMDAELIAdBADoAAAsgCkEBayEKCyAHQQFqIQcgAUEMaiEBDAELAAsACwAFIAdBAkEBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELRSINGzoAACAHQQFqIQcgAUEMaiEBIAwgDWohDCAKIA1rIQoMAQsACwALEEYACwJAAkADQCACIANGDQEgCC0AAEECRwRAIAhBAWohCCACQQxqIQIMAQsLIAIhAwwBCyAFIAUoAgBBBHI2AgALIAsoAgAhACALQQA2AgAgAARAIAAgCygCBBEBAAsgCUGAAWokACADC/QFAQt/IwBBgAFrIgkkACAJIAE2AnwgCUH2ATYCECAJQQhqQQAgCUEQaiIIEFEhCwJAAkAgAyACa0EMbSIKQeUATwRAIAoQOyIIRQ0BIAsoAgAhASALIAg2AgAgAQRAIAEgCygCBBEBAAsLIAghByACIQEDQCABIANGBEADQCAAIAlB/ABqEENBASAKGwRAIAAgCUH8AGoQQwRAIAUgBSgCAEECcjYCAAsMBQsCfyAAKAIAIgcoAgwiASAHKAIQRgRAIAcgBygCACgCJBEAAAwBCyABLQAAC8AhDiAGRQRAIAQgDiAEKAIAKAIMEQQAIQ4LIA9BAWohDUEAIRAgCCEHIAIhAQNAIAEgA0YEQCANIQ8gEEUNAiAAEF0aIAghByACIQEgCiAMakECSQ0CA0AgASADRgRADAQFAkAgBy0AAEECRw0AAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIA9GDQAgB0EAOgAAIAxBAWshDAsgB0EBaiEHIAFBDGohAQwBCwALAAUCQCAHLQAAQQFHDQACfyABLQALQQd2BEAgASgCAAwBCyABCyAPai0AACERAkAgDkH/AXEgBgR/IBEFIAQgEcAgBCgCACgCDBEEAAtB/wFxRgRAQQEhEAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyANRw0CIAdBAjoAACAMQQFqIQwMAQsgB0EAOgAACyAKQQFrIQoLIAdBAWohByABQQxqIQEMAQsACwALAAUgB0ECQQECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtFIg0bOgAAIAdBAWohByABQQxqIQEgDCANaiEMIAogDWshCgwBCwALAAsQRgALAkACQANAIAIgA0YNASAILQAAQQJHBEAgCEEBaiEIIAJBDGohAgwBCwsgAiEDDAELIAUgBSgCAEEEcjYCAAsgCygCACEAIAtBADYCACAABEAgACALKAIEEQEACyAJQYABaiQAIAMLvgIBBH8gA0GIiDUgAxsiBSgCACEDAkACfwJAIAFFBEAgAw0BQQAPC0F+IAJFDQEaAkAgAwRAIAIhBAwBCyABLQAAIgPAIgRBAE4EQCAABEAgACADNgIACyAEQQBHDwsjAygCYCgCAEUEQEEBIABFDQMaIAAgBEH/vwNxNgIAQQEPCyADQcIBayIDQTJLDQEgA0ECdEGwzwJqKAIAIQMgAkEBayIERQ0DIAFBAWohAQsgAS0AACIGQQN2IgdBEGsgA0EadSAHanJBB0sNAANAIARBAWshBCAGQYABayADQQZ0ciIDQQBOBEAgBUEANgIAIAAEQCAAIAM2AgALIAIgBGsPCyAERQ0DIAFBAWoiAS0AACIGQcABcUGAAUYNAAsLIAVBADYCACMDQRxqQRk2AgBBfwsPCyAFIAM2AgBBfgsJACABQQEQ9AMLcQEBfyAAQdTIAjYCACAAEPcDGgJAIAAtAGBFDQAgACgCICIBRQ0AIAEQLwsCQCAALQBhRQ0AIAAoAjgiAUUNACABEC8LIABB5McCNgIAIAAoAgQiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAALDwAgACAAKAIQIAFyELsCC7YLAQZ/IAAgAWohBQJAAkAgACgCBCICQQFxDQAgAkEDcUUNASAAKAIAIgIgAWohAQJAAkACQCAAIAJrIgBBlIQ1KAIARwRAIAJB/wFNBEAgAkEDdiEEIAAoAggiAyAAKAIMIgJHDQJBgIQ1QYCENSgCAEF+IAR3cTYCAAwFCyAAKAIYIQYgACAAKAIMIgJHBEBBkIQ1KAIAGiAAKAIIIgMgAjYCDCACIAM2AggMBAsgAEEUaiIEKAIAIgNFBEAgACgCECIDRQ0DIABBEGohBAsDQCAEIQcgAyICQRRqIgQoAgAiAw0AIAJBEGohBCACKAIQIgMNAAsgB0EANgIADAMLIAUoAgQiAkEDcUEDRw0DQYiENSABNgIAIAUgAkF+cTYCBCAAIAFBAXI2AgQgBSABNgIADwsgAyACNgIMIAIgAzYCCAwCC0EAIQILIAZFDQACQCAAKAIcIgNBAnRBsIY1aiIEKAIAIABGBEAgBCACNgIAIAINAUGEhDVBhIQ1KAIAQX4gA3dxNgIADAILIAZBEEEUIAYoAhAgAEYbaiACNgIAIAJFDQELIAIgBjYCGCAAKAIQIgMEQCACIAM2AhAgAyACNgIYCyAAKAIUIgNFDQAgAiADNgIUIAMgAjYCGAsCQAJAAkACQCAFKAIEIgJBAnFFBEBBmIQ1KAIAIAVGBEBBmIQ1IAA2AgBBjIQ1QYyENSgCACABaiIBNgIAIAAgAUEBcjYCBCAAQZSENSgCAEcNBkGIhDVBADYCAEGUhDVBADYCAA8LQZSENSgCACAFRgRAQZSENSAANgIAQYiENUGIhDUoAgAgAWoiATYCACAAIAFBAXI2AgQgACABaiABNgIADwsgAkF4cSABaiEBIAJB/wFNBEAgAkEDdiEEIAUoAgwiAiAFKAIIIgNGBEBBgIQ1QYCENSgCAEF+IAR3cTYCAAwFCyADIAI2AgwgAiADNgIIDAQLIAUoAhghBiAFIAUoAgwiAkcEQEGQhDUoAgAaIAUoAggiAyACNgIMIAIgAzYCCAwDCyAFQRRqIgQoAgAiA0UEQCAFKAIQIgNFDQIgBUEQaiEECwNAIAQhByADIgJBFGoiBCgCACIDDQAgAkEQaiEEIAIoAhAiAw0ACyAHQQA2AgAMAgsgBSACQX5xNgIEIAAgAUEBcjYCBCAAIAFqIAE2AgAMAwtBACECCyAGRQ0AAkAgBSgCHCIDQQJ0QbCGNWoiBCgCACAFRgRAIAQgAjYCACACDQFBhIQ1QYSENSgCAEF+IAN3cTYCAAwCCyAGQRBBFCAGKAIQIAVGG2ogAjYCACACRQ0BCyACIAY2AhggBSgCECIDBEAgAiADNgIQIAMgAjYCGAsgBSgCFCIDRQ0AIAIgAzYCFCADIAI2AhgLIAAgAUEBcjYCBCAAIAFqIAE2AgAgAEGUhDUoAgBHDQBBiIQ1IAE2AgAPCyABQf8BTQRAIAFBeHFBqIQ1aiECAn9BgIQ1KAIAIgNBASABQQN2dCIBcUUEQEGAhDUgASADcjYCACACDAELIAIoAggLIQEgAiAANgIIIAEgADYCDCAAIAI2AgwgACABNgIIDwtBHyEDIAFB////B00EQCABQSYgAUEIdmciAmt2QQFxIAJBAXRrQT5qIQMLIAAgAzYCHCAAQgA3AhAgA0ECdEGwhjVqIQICQAJAQYSENSgCACIEQQEgA3QiB3FFBEBBhIQ1IAQgB3I2AgAgAiAANgIAIAAgAjYCGAwBCyABQRkgA0EBdmtBACADQR9HG3QhAyACKAIAIQIDQCACIgQoAgRBeHEgAUYNAiADQR12IQIgA0EBdCEDIAQgAkEEcWoiB0EQaigCACICDQALIAcgADYCECAAIAQ2AhgLIAAgADYCDCAAIAA2AggPCyAEKAIIIgEgADYCDCAEIAA2AgggAEEANgIYIAAgBDYCDCAAIAE2AggLC5ADAQZ/IAAtAABBD3FFBEAgAEEEakEAQQr+SAIAQQpxDwsCfyAAKAIAIQICQAJAAkAjAyIBKAIYIgQgACgCBCIDQf////8DcSIGRw0AAkAgAkEIcUUNACAAKAIUQQBODQAgAEEANgIUIANBgICAgARxIQMMAgsgAkEDcUEBRw0AQQYhBSAAKAIUIgFB/v///wdLDQIgACABQQFqNgIUQQAMAwtBOCEFIAZB/////wNGDQECQCAGDQBBACADIAJBBHEbDQAgAyAAQQRqIAMgAkGAAXEEfyABKAJQRQRAIAFBdDYCUAsgACgCCCEGIAEgAEEQajYCVCAEQYCAgIB4ciAEIAYbBSAECyADQYCAgIAEcXL+SAIARg0BIAFBADYCVCACQQxxQQxHDQAgACgCCA0CC0EKDAILIAEoAkwhAiAAIAFBzABqIgU2AgwgACACNgIQIABBEGohBCACIAVHBEAgAkEEayAENgIACyABIAQ2AkxBACEFIAFBADYCVCADRQ0AIABBADYCFEE+DAELIAULC5sFAQV/IwBBMGsiBSQAAkAgAEUEQEEcIQQMAQtBmPM0KAIARQRAQZjzNEErNgIAC0GZ8TQtAABFBEAQ/AEoAgAiAwRAA0ACQCADRQ0AIAMoAkxBAE4NACADQQA2AkwLIAMoAjgiAw0ACwtB1PI0ENMBAkBBlPE0KAIAIgNFDQAgAygCTEEATg0AIANBADYCTAsCQEHAuwMoAgAiA0UNACADKAJMQQBODQAgA0EANgJMCwJAQai6AygCACIDRQ0AIAMoAkxBAE4NACADQQA2AkwLQZnxNEEBOgAACyAFQQhqQQBBKPwLACAFQYy5AygCACIDNgIEQZC5AygCAEGaAWoiBEEAIANBD2ogBSgCDBtqIgYQOyIDQQAgBBCgASADIAY2AjAgAyADNgIsIAMgAzYCAEGY8zRBmPM0KAIAIgRBAWo2AgAgAyADQcwAajYCTCADIAQ2AhggA0G48TQ2AmAgA0EDQQIgBSgCEBs2AiAgAyAFKAIEIgY2AjggAyADQYcBakF8cSIENgJ0IARBEGohBEGQuQMoAgAEQCADIARBA2pBfHEiBDYCSEGQuQMoAgAgBGohBAsgAyAFKAIMIgcgBCAGakEPakFwcSAHGzYCNCADEIwEIwMhBBDKAiAEKAIMIQYgAyAENgIIIAMgBjYCDCAGIAM2AgggAygCCCADNgIMEMkCQZzxNEGc8TQoAgAiBEEBajYCACAERQRAQZvxNEEBOgAACyADIAVBBGogASACEB8iBARAQZzxNEGc8TQoAgBBAWsiADYCACAARQRAQZvxNEEAOgAACxDKAiADKAIMIgAgAygCCDYCCCADKAIIIAA2AgwgAyADNgIMIAMgAzYCCBDJAgwBCyAAIAM2AgALIAVBMGokACAECxUAIABBAP5BAgBBAkYEQCAAEI4BCwsyACAAQQBBAf5IAgAEQCAAQQFBAv5IAgAaA0AgAEEAQQIQoQEgAEEAQQL+SAIADQALCwvnAQICfwN8IwBBEGsiAiQAAkACfwJAAkAjBSIDDQAjAyIELQAoQQFHDQAgBC0AKUUNAQtBAUHkACADG7chBRADRAAAAAAAAPB/oCEHIwMhAwNAAkAgAygCJEUEQCAHEAOhIgZEAAAAAAAAAABlRQ0BQckADAQLQQshAAwECyAAIAEgBSAGIAUgBmMbENUBIgRBt39GDQALQQAgBGsMAQtBACAAIAFEAAAAAAAA8H8Q1QFrCyIAQQAgAEFvcUELRhsgACAAQckARxsiAEEbRw0AQRtBAEGI8zQoAgAbIQALIAJBEGokACAACw0AIABB/////wcQtQELrgQBBX8jAEEQayIEJAAgABBWGgJ/AkAgACABEJwEIgNFBEAgACgCHCIDIAAoAiBGBEAgACgCGCADQQF0QQEgAxsiA0ECdBDEASIFRQ0CIAAgAzYCICAAIAU2AhgLIAEQoQQiA0UNASAAIAAoAhwiAUEBajYCHCAAKAIYIAFBAnRqIAM2AgALIAMMAQtBAAshASAAEFIaIAEEfyAEIAIoAgg2AgggBCACKQIANwMAIwBBMGsiAiQAAkACfyABIgAoAhwiA/4QAnwhAQNAQQAgAUUNARogASADIAEgAUEBav5IAnwiAUcNAAtBAQtFBEBBACEDDAELIABBBGoiARBWGiACIAQoAgg2AiAgAiAEKQIANwMYIAAgAkEYahCdBCEDIAEQUhoCfyADBH8gAEEC/kECACEFQQEhAyAAKAIcIgEgBUECRg0BGiACIAA2AiwgAiAANgIQIAJBvgE2AiggAkG/ATYCJCACIAIpAiQ3AwgjAEEQayIDJAAgASgCeEEEahBWGiABKAJ4IQYgAyACKAIQNgIIIAMgAikCCDcDACAGIAMQnQQaIAEoAnhBBGoQUhoCQCABKAJ4QQL+QQIAQQJGDQAgAf4QAoABBEAgAUF//gACABoMAQsgASMDQdDxNBAeCyADQRBqJABBAQVBAAshAyAAKAIcCyIAQQH+JQJ8QQFGBEAgAEH8AGpB/////wcQtQELCyACQTBqJAAgAwVBAAshByAEQRBqJAAgBwsNAEHU8jQQpQRB2PI0C+4BAwJ8An8BfgJ9AkAgALwiA0EUdkH/D3EiBEGwCEkNAEMAAAAAIANBgICAfEYNARogBEH4D08EQCAAIACSDwsgAEMAAAAAXgRAIwBBEGsiA0MAAABwOAIMIAMqAgxDAAAAcJQPCyAAQwAAFsNfRQ0AIwBBEGsiA0MAAAAQOAIMIAMqAgxDAAAAEJQPC0HY2gErAwAgALsiASABQdDaASsDACIBoCICIAGhoSIBokHg2gErAwCgIAEgAaKiQejaASsDACABokQAAAAAAADwP6CgIAK9IgVCL4YgBadBH3FBA3RB0NgBaikDAHy/orYLCxQAQQwQXkEFEHBB0KkDQdIAEAIACxQAQQwQXkEEEHBB0KkDQdIAEAIACxQAQQwQXkEDEHBB0KkDQdIAEAIAC6UEAQR/AkACQAJAIAAoAgQgACgCACICa0E0bSIFQQFqIgNBxZ2xJ0kEQEHEnbEnIAAoAgggAmtBNG0iAkEBdCIEIAMgAyAESRsgAkHiztgTTxsiAwR/IANBxZ2xJ08NAiADQTRsEDIFQQALIgQgBUE0bGoiAiAB/QACAP0LAgAgAiAB/QACEP0LAhAgAiABKAIgNgIgIAIgASgCJDYCJCABQgA3AiAgAf0MAAAAAAAAAAAAAAAAAAAAAP0LAhAgAiABLQAwOgAwIAIgASkCKDcCKCAEIANBNGxqIQMgAkE0aiEFIAAoAgQiASAAKAIAIgRGDQIDQCACQTRrIgIgAUE0ayIB/QACAP0LAgAgAkEANgIYIAIgASgCEDYCECACIAEoAhQ2AhQgAiABKAIYNgIYIAFBADYCGCABQgA3AhAgAkEANgIkIAIgASgCHDYCHCACIAEoAiA2AiAgAiABKAIkNgIkIAFBADYCJCABQgA3AhwgAiABLQAwOgAwIAIgASkCKDcCKCABIARHDQALIAAgAzYCCCAAKAIEIQMgACAFNgIEIAAoAgAhASAAIAI2AgAgASADRg0DA0AgA0E0ayIAKAIcIgIEQCADQRRrIAI2AgAgAhAvCyADQSRrKAIAIgIEQCADQSBrIAI2AgAgAhAvCyAAIgMgAUcNAAsMAwsQNgALEEcACyAAIAM2AgggACAFNgIEIAAgAjYCAAsgAQRAIAEQLwsLqQUBCH8gASAAKAIIIgMgACgCACIEa0EMbU0EQAJAIAAoAgQiBSAEa0EMbSIIIAEgASAISxsiB0UNACAEIQMgByEGIAdBA3EiCQRAA0AgAyACKAIANgIAIAMgAigCBDYCBCADIAItAAg6AAggBkEBayEGIANBDGohAyAKQQFqIgogCUcNAAsLIAdBBEkNAANAIAMgAigCADYCACADIAIoAgQ2AgQgAyACLQAIOgAIIAMgAigCADYCDCADIAIoAgQ2AhAgAyACLQAIOgAUIAMgAigCADYCGCADIAIoAgQ2AhwgAyACLQAIOgAgIAMgAigCADYCJCADIAIoAgQ2AiggAyACLQAIOgAsIANBMGohAyAGQQRrIgYNAAsLIAEgCEsEQCAFIAEgCGtBDGxqIQEDQCAFIAIpAgA3AgAgBSACKAIINgIIIAVBDGoiBSABRw0ACyAAIAE2AgQPCyAAIAQgAUEMbGo2AgQPCyAEBEAgACAENgIEIAQQLyAAQQA2AgggAEIANwIAQQAhAwsCQCABQdaq1aoBTw0AQdWq1aoBIANBDG0iA0EBdCIEIAEgASAESRsgA0Gq1arVAE8bIgNB1qrVqgFPDQAgACADQQxsIgMQMiIENgIEIAAgBDYCACAAIAMgBGo2AgggBCEDIAFBDGwiAUEMayIGQQxuQQFqQQNxIgcEQANAIAMgAikCADcCACADIAIoAgg2AgggA0EMaiEDIAVBAWoiBSAHRw0ACwsgASAEaiEBIAZBJE8EQANAIAMgAikCADcCACADIAIoAgg2AgggAyACKAIINgIUIAMgAikCADcCDCADIAIoAgg2AiAgAyACKQIANwIYIAMgAikCADcCJCADIAIoAgg2AiwgA0EwaiIDIAFHDQALCyAAIAE2AgQPCxA2AAsUAEEMEF5BBhBwQdCpA0HSABACAAsCAAv9BwEHfyAAKAIQIQcgACgCJCEIAkAgAiABIgNGDQACQAJAIAMsAAAiBEEkaw5ZAgEBAQIAAgIBAQIBAQEBAQEBAQEBAQEBAQEBAgEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQICAQIBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAgIBCyAAKAIYDQELIAAgBBBZIANBAWohAwsCQAJAAkACQAJAIAEgA0cNAAJ/AkAgAiABIgNGDQAgA0EBaiIFIAJGDQAgAy0AAEHcAEcNAAJAAkAgBSwAACIEQSRrDloAAQEBAAAAAAEBAAEBAQEBAQEBAQEBAQEBAQEAAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAAABAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEAAAABCyAAIAQQWSADQQJqDAILIAAoAgxB8AdxQcAARgRAIAAgBSACQQAQ1wIMAgsgA0ECQQACf0EAIQUCQAJAIARBeHFBMEcgBEH+AXFBOEdxDQAgBEH/AXEiBEExa0EISw0AIARBMGsiBCAAKAIQSw0BIAAgBBDaAkEBIQULIAUMAQsQ/wEACxtqIQMLIAMLIgMgAUcNAAJAIAEgAkYNACABLQAAQS5HDQBBCBAyIQMgACgCJCIEKAIEIQUgA0GktQE2AgAgAyAFNgIEIAQgAzYCBAwCCyAAIAEgAhDiAiEDCwJAAkAgASADRw0AIAIgA0YNAAJAAkAgASwAACIDQSRrDgUDBgYGAQALIANB3gBHDQVBDBAyIQMgACgCDCEEIAAoAiQiBSgCBCEGIANB9KkBNgIAIAMgBjYCBCADIARB8A9xQYAMRjoACCAFIAM2AgQMAwsCQCAALQAMQQJxBEAgACgCECEEDAELQQwQMiEDIAAgACgCEEEBaiIENgIQIAAoAiQiBSgCBCEGIAMgBDYCCCADQcCxATYCACADIAY2AgQgBSADNgIEIAAgACgCJCgCBDYCJAsgACAAKAIYQQFqNgIYIAAgAUEBaiACEIYCIgUgAkYNBSAFLQAAQSlHDQUgAC0ADEECcUUEQEEMEDIhAyAAKAIkIgYoAgQhCSADIAQ2AgggA0GQsgE2AgAgAyAJNgIEIAYgAzYCBCAAIAAoAiQoAgQ2AiQLIAAgACgCGEEBazYCGCAFQQFqIQMLIAEgA0cNAgwDC0EMEDIhAyAAKAIMIQQgACgCJCIFKAIEIQYgA0G8qgE2AgAgAyAGNgIEIAMgBEHwD3FBgAxGOgAIIAUgAzYCBAsgACAAKAIkKAIENgIkIAFBAWohAwsgACADIAIgCCAHQQFqIAAoAhBBAWoQ4wIhAQsgAQ8LEIMCAAuxAgEEfyAAKAIkIQYCQCAAIAEgAhCFAiIDIAFGDQADQCAAIAMiBCACEIUCIgMgBEcNAAsgASAERg0AIAIgBEcEQANAIAQtAABB/ABHBEAgBA8LIAAoAiQhBSAAIARBAWoiASACEIUCIgMgAUYNAgNAIAAgAyIEIAIQhQIiAyAERw0ACyABIARGDQJBDBAyIQMgBigCBCEBIAMgBSgCBDYCCCADIAE2AgQgA0GctAE2AgAgBiADNgIEIAVBADYCBEEIEDIhAyAAKAIkKAIEIQEgA0HIqAE2AgAgAyABNgIEIAUgAzYCBCAAKAIkQQA2AgRBCBAyIQMgBSgCBCEBIANB3LQBNgIAIAMgATYCBCAAKAIkIAM2AgQgACAFKAIENgIkIAIgBEcNAAsLIAIPCxDGBAALrgIBBH8gACgCACIDBEAgAyECIAMgACgCBCIERwRAA0AgBEEMayICKAIAIgUEQCAEQQhrIAU2AgAgBRAvCyACIgQgA0cNAAsgACgCACECCyAAIAM2AgQgAhAvIABBADYCCCAAQgA3AgALIAAgASgCADYCACAAIAEoAgQ2AgQgACABKAIINgIIIAFBADYCCCABQgA3AgAgACgCDCIDBEAgAyECIAMgACgCECIERwRAA0AgBEEMayICKAIAIgUEQCAEQQhrIAU2AgAgBRAvCyACIgQgA0cNAAsgACgCDCECCyAAIAM2AhAgAhAvIABBADYCFCAAQgA3AgwLIAAgASgCDDYCDCAAIAEoAhA2AhAgACABKAIUNgIUIAFBADYCFCABQgA3AgwgACABKQIYNwIYC/ABAgR/AX4jAEFAaiICJAAgAiABQsD8FX8iBj4CACACIAZCgKOkfn4gAUIKfnwiAULg1AN/IgY+AgQgAiAGQqCrfH4gAXwiAULoB38iBj4CCCACIAZCmPj//w9+IAF8PgIQIAJB0coANgIMIAJBIGoiBEEgQdslIAIQxQIaIAQQaCIDQfD///8HSQRAAkAgA0EKTQRAIAAgAzoACwwBCyADQQ9yQQFqIgUQMiEEIAAgBUGAgICAeHI2AgggACAENgIAIAAgAzYCBCAEIQALIAAgAkEgaiAD/AoAACAAIANqQQA6AAAgAkFAayQADwsQTQAL60EEGX8FfQN8AX4jAEGAAWsiBiQAAkACQCAAKAIAQQH+HgIAIhQgACgCBCgCAEgEQCAGQdwAaiERIAZB0ABqIRIDQAJAIAAoAggoAgAgFEHYFGxqIggtALECDQAgCC0AsAINAAJAAkACQAJAAkACQCAAKAIMIgEoAgAOAgABBgsgACgCFCgCACILKAL8ASECIAsoAsABIQogACgCECoCAEO9N4Y1XUUNAUQAAAAAAAAAACEfQQAhBEEAIQNEAAAAAAAAAAAhICACIApODQMgCiACayIBQQFxIQsgCCgCtAIhByACQQFqIApGBEAgAiEBDAMLIAFBfnEhDCACIQFBACEFA0ACfwJAIAcgAUECdGoqAgAiGkMAAID/Ww0AICAgGrsiIaAhICAfICFjRQ0AIAEMAQsgHyEhIAMLIQkCQCAHIAFBAWoiA0ECdGoqAgAiGkMAAID/XARAICAgGrsiH6AhICAfICFkDQELIAkhAyAhIR8LIAFBAmohASAMIAVBAmoiBUcNAAsMAgsgASgCdCENAkAgACgCFCgCACITKALAASIMIAgoAtwCIgIgCCgC2AIiBGtBBHUiAUsEQEEAIQpBACEHAkAgDCABayIBIAgoAuACIgUgAmtBBHVNBEACQCABRQ0AIAIhAyABQQdxIgQEQEEAIQUDQCADQQA2AgggA0IANwMAIANBEGohAyAFQQFqIgUgBEcNAAsLIAFBBHQgAmohAiABQQFrQf////8AcUEHSQ0AA0AgA0IANwNwIANCADcDYCADQgA3A1AgA0IANwNAIANCADcDMCADQgA3AyAgA0IANwMQIANBADYCCCADQgA3AwAgA0EANgJ4IANBADYCaCADQQA2AlggA0EANgJIIANBADYCOCADQQA2AiggA0EANgIYIANBgAFqIgMgAkcNAAsLIAggAjYC3AIMAQsCQCACIAgoAtgCIgRrQQR1IgsgAWoiA0GAgICAAUkEQEH/////ACAFIARrIgVBA3UiCSADIAMgCUkbIAVB8P///wdPGyIJBEAgCUGAgICAAU8NAiAJQQR0EDIhBwsgByALQQR0aiIFIQMgAUEHcSILBEADQCADQQA2AgggA0IANwMAIANBEGohAyAKQQFqIgogC0cNAAsLIAFBBHQgBWohCiABQQFrQf////8AcUEHTwRAA0AgA0IANwNwIANCADcDYCADQgA3A1AgA0IANwNAIANCADcDMCADQgA3AyAgA0IANwMQIANBADYCCCADQgA3AwAgA0EANgJ4IANBADYCaCADQQA2AlggA0EANgJIIANBADYCOCADQQA2AiggA0EANgIYIANBgAFqIgMgCkcNAAsLIAIgBEcEQANAIAVBEGsiBSACQRBrIgIpAwA3AwAgBSACKAIINgIIIAIgBEcNAAsLIAggByAJQQR0ajYC4AIgCCAKNgLcAiAIKALYAiEBIAggBTYC2AIgAQRAIAEQLwsMAgsMDAsQRwALIAgoAtgCIQQMAQsgASAMTQ0AIAggBCAMQQR0ajYC3AILAkAgDEEATA0AIAgoAsACIQJBACEBIAxBAUcEQCAMQX5xIQVBACELA0AgAiABQQJ0aioCACEaIAQgAUEEdGoiAyABNgIIIAMgGrs5AwAgAiABQQFyIgNBAnRqKgIAIRogBCADQQR0aiIJIAM2AgggCSAauzkDACABQQJqIQEgC0ECaiILIAVHDQALCyAMQQFxRQ0AIAIgAUECdGoqAgAhGiAEIAFBBHRqIgIgATYCCCACIBq7OQMACyAGAn8gDUUEQEEAIQlBAAwBCyAEIA1BBHQiDmohByAIKALcAiEQIA5BBHUhDwJAIA5BEUgiFQ0AIA9BAmtBAXYiCiEBIA5BIEkNAANAAkAgCiABIglIDQAgBCABQQR0aiEDIAQgAUEBdCICQQFyIgVBBHRqIQECQCAPIAJBAmoiAkwEQCAFIQIMAQsgASsDACABKwMQZEUEQCAFIQIMAQsgAUEQaiEBCyADKwMAIh8gASsDAGMNACADKAIIIQsDQAJAIAMiBSABIgMpAwA3AwAgBSABKAIINgIIIAIgCkoNACAEIAJBAXQiAkEBciIFQQR0aiEBAkAgDyACQQJqIgJMBEAgBSECDAELIAErAwAgASsDEGRFBEAgBSECDAELIAFBEGohAQsgASsDACAfZEUNAQsLIAMgCzYCCCADIB85AwALIAlBAWshASAJQQBKDQALCwJAIAcgEEYNACAHIQEgDkEfSgRAIARBIGohCyAEQRBqIQkgD0ECa0EBdiEWIAEhBQNAAkAgBSsDACAEKwMAZEUNACAGIAX9AAMA/QsDCCAFIAQoAgg2AgggBSAEKQMANwMAIAQgBigCEDYCCCAEIAYpAwg3AwBBASECAkAgDkEgRgRAIAkhAQwBCyAJIgErAwAgCysDAGRFDQBBAiECIAshAQsgBCsDACIfIAErAwBjDQAgBCgCCCEXIAQhAwNAAkAgAyIKIAEiAykDADcDACAKIAEoAgg2AgggAiAWSg0AIAQgAkEBdCICQQFyIgpBBHRqIQECQCAPIAJBAmoiAkwEQCAKIQIMAQsgASsDACABKwMQZEUEQCAKIQIMAQsgAUEQaiEBCyABKwMAIB9kRQ0BCwsgAyAXNgIIIAMgHzkDAAsgBUEQaiIFIBBHDQALDAELA0AgASsDACAEKwMAZARAIAYgAf0AAwD9CwMIIAEgBCgCCDYCCCABIAQpAwA3AwAgBCAGKAIQNgIIIAQgBikDCDcDAAsgAUEQaiIBIBBHDQALCyAVRQRAIA1B/////wBxIQEDQCAGIAT9AAMA/QsDCCABIglBAmtBAXYhCkEAIQIgBCEBA0AgAkEBdCILQQFyIQMgASIFIAJBBHRqQRBqIQECQCAJIAtBAmoiAkwEQCADIQIMAQsgASsDACABKwMQZEUEQCADIQIMAQsgAUEQaiEBCyAFIAEpAwA3AwAgBSABKAIINgIIIAIgCkwNAAsCQCAHQRBrIgcgAUYEQCABIAYpAwg3AwAgASAGKAIQNgIIDAELIAEgBykDADcDACABIAcoAgg2AgggByAGKQMINwMAIAcgBigCEDYCCCABIARrQRBqIgJBEUgNACABKwMAIh8gBCACQQR2QQJrQQF2IgtBBHRqIgUrAwBjRQ0AIAEoAgghAwNAAkAgASAFIgIpAwA3AwAgASACKAIINgIIIAtFDQAgAiEBIAQgC0EBa0EBdiILQQR0aiIFKwMAIB9kDQELCyACIAM2AgggAiAfOQMACyAJQQFrIQEgCUECSg0ACwsgBkEANgJ8IAZCADcCdCANQdaq1SpPDQkgDUEwbCIBEDIiCSABagsiAzYCfCAGIAk2AnggBiAJNgJ0IAgoArQCIQREAAAAAAAAAAAhIAJAIBMoAvwBIgUgDE4EQEQAAAAAAAAAACEfDAELIAVBAWohAkQAAAAAAAAAACEfIAwgBSIBa0EBcQRAAkAgBCABQQJ0aioCACIaQwAAgP9bDQAgGrsiH0QAAAAAAAAAAKAhICAaQwAAAABeDQBEAAAAAAAAAAAhHwsgAiEBCyACIAxGDQADQAJ/AkAgBCABQQJ0aioCACIaQwAAgP9bDQAgICAauyIhoCEgIB8gIWNFDQAgAQwBCyAfISEgBQshAgJAIAQgAUEBaiIFQQJ0aioCACIaQwAAgP9cBEAgICAauyIfoCEgIB8gIWQNAQsgISEfIAIhBQsgDCABQQJqIgFHDQALCyAIKAK4AiEHIAZBADYCECAGQgA3AgggBCAHRwRAIAcgBGsiAUH9////B08NCSAGIAFBAXQQMiIKNgIIIAYgCiABQQJ1QQN0ajYCEAJAAkAgAUEEayIBQQRJBEAgBCECIAohAQwBCyAKIAFBAnZBAWoiDkH+////B3EiDEEDdGohASAEIAxBAnRqIQJBACELA0AgCiALQQN0aiAEIAtBAnRq/V0CAP1f/QsDACALQQJqIgsgDEcNAAsgDCAORg0BCwNAIAEgAioCALs5AwAgAUEIaiEBIAJBBGoiAiAHRw0ACwsgBiABNgIMCyAGQQhqEM4EAkAgDUEATARAIAkhCgwBCyAfICBEu73X2d982z2go7YhGiAgtiEbIAhB5AJqIQxBACELIAYoAnwhBCAGKAJ4IQogCSEBAkACQAJAA0AgBkEIaiICIAwgAhDNBCICQQJ0IgcgCCgCzAJqKgIAIRwgCCgCtAIgB2oqAgAhHQJAIAEgA0kEQCABQn83AxggASAbOAIUIAEgGjgCECABIBw4AgwgASAdOAIIIAEgBTYCBCABIAI2AgAgAUEANgIoIAFCfzcDICABQTBqIQoMAQsgASAJa0EwbSIOQQFqIgdB1qrVKk8NAkHVqtUqIAMgCWtBMG0iA0EBdCIPIAcgByAPSRsgA0Gq1aoVTxsiBwR/IAdB1qrVKk8NBCAHQTBsEDIFQQALIgQgDkEwbGoiA0J/NwMYIAMgGzgCFCADIBo4AhAgAyAcOAIMIAMgHTgCCCADIAU2AgQgAyACNgIAIANBADYCKCADQn83AyAgAyECIAEgCUcEQANAIAJBMGsiAiABQTBrIgH9AAMA/QsDACACIAH9AAMg/QsDICACIAH9AAMQ/QsDECABIAlHDQALCyAHQTBsIARqIQQgA0EwaiEKIAkEQCAJEC8LIAQhAyACIQkLIAkgC0EwbGoiASgCACICIBMoAvwBTgRAIAEgAjYCBCABIAEqAgg4AhALIAohASANIAtBAWoiC0cNAAsMAgsgBiAENgJ8IAYgCjYCeCAGIAk2AnQMCwsgBiAENgJ8IAYgCjYCeBBHAAsgBiAENgJ8IAYgCjYCeCAGIAk2AnQLIAYoAggiAQRAIAYgATYCDCABEC8LIAkgCkcEQCAAKAIYKAIAIQsCQAJAA0ACQCAGIBQ2AgggBiAIKAKsAjYCDCAILQCyAiEBIAZBADYCICAGQgA3AxggBiABOgAQIAgoAtQBIgEgCCgC0AEiA2siBEEwbSECIAEgA0cEQCACQdaq1SpPDQEgBiAEEDIiATYCHCAGIAE2AhggBiABIAJBMGxqNgIgIAgoAtABIgIgCCgC1AEiA0cEQANAIAEgAv0AAwD9CwMAIAEgAv0AAyD9CwMgIAEgAv0AAxD9CwMQIAFBMGohASACQTBqIgIgA0cNAAsLIAYgATYCHAsgBiAI/QAC3AH9CwIkIAYgCCgChAI2AkwgBiAIKQL8ATcCRCAGIAj9AALsAf0LAjQCfyASQQA2AgggEkIANwIAIAgoAowCIgIgCCgCiAIiA2siBEEMbSEBAkAgAiADRwRAIAFB1qrVqgFPDQEgEiAEEDIiBzYCBCASIAc2AgAgEiAHIAFBDGxqNgIIIAgoAogCIgUgCCgCjAIiD0cEQANAIAdBADYCCCAHQgA3AgAgBSgCBCIBIAUoAgAiAkcEQAJAAkACQCABIAJrIgFBAE4EQCAHIAEQMiICNgIEIAcgAjYCACAHIAIgAUF4cWo2AgggBSgCACIEIAUoAgQiDUYEQCACIQEMBAsgDSAEa0EIayIBQRhJDQEgAiAEa0EQSQ0BIAIgAUEDdkEBaiIQQf7///8DcSIOQQN0IgNqIQEgAyAEaiEDQQAhDANAIAIgDEEDdCITaiAEIBNq/QACAP0LAgAgDEECaiIMIA5HDQALIA4gEEYNAwwCCwwWCyAEIQMgAiEBCwNAIAEgAykCADcCACABQQhqIQEgA0EIaiIDIA1HDQALCyAHIAE2AgQLIAdBDGohByAFQQxqIgUgD0cNAAsLIBIgBzYCBAsgEgwBCwwOCyENAn8gEUEANgIIIBFCADcCACAIKAKYAiICIAgoApQCIgNrIgRBDG0hAQJAIAIgA0cEQCABQdaq1aoBTw0BIBEgBBAyIgc2AgQgESAHNgIAIBEgByABQQxsajYCCCAIKAKUAiIFIAgoApgCIhBHBEADQCAHQQA2AgggB0IANwIAIAUoAgQiASAFKAIAIgJHBEACQAJAAkAgASACayIBQQBOBEAgByABEDIiAjYCBCAHIAI2AgAgByACIAFBfHFqNgIIIAUoAgAiBCAFKAIEIg5GBEAgAiEBDAQLIA4gBGtBBGsiAUEMSQ0BIAIgBGtBEEkNASACIAFBAnZBAWoiE0H8////B3EiD0ECdCIDaiEBIAMgBGohA0EAIQwDQCACIAxBAnQiFWogBCAVav0AAgD9CwIAIAxBBGoiDCAPRw0ACyAPIBNGDQMMAgsMFgsgBCEDIAIhAQsDQCABIAMoAgA2AgAgAUEEaiEBIANBBGoiAyAORw0ACwsgByABNgIECyAHQQxqIQcgBUEMaiIFIBBHDQALCyARIAc2AgQLIBEMAQsMDgshDCAGIAgpAqACIiI3A2gCQCALIBRBDGwiDmoiAigCBCIBIAIoAggiA0kEQCABIAYpAwg3AwAgASAGLQAQOgAIIAFBADYCGCABQgA3AxAgASAGKAIYNgIQIAEgBigCHDYCFCABIAYoAiA2AhggBkEANgIgIAZCADcDGCABIAYoAkw2AkQgASAGKQJENwI8IAEgBv0AAjT9CwIsIAEgBv0AAiT9CwIcIAH9DAAAAAAAAAAAAAAAAAAAAAD9CwNIIAEgDSgCADYCSCABIAYoAlQ2AkwgBigCWCEDIAFCADcDWCABIAM2AlAgASAMKAIANgJUIAEgBigCYDYCWCABIAYoAmQ2AlwgBkIANwNgIA39DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAEgBikDaDcDYCACIAFB6ABqNgIEDAELIAEgAigCACIEa0HoAG0iBUEBaiIBQePO2BNPDQNB4s7YEyADIARrQegAbSIDQQF0IgQgASABIARJGyADQbGn7AlPGyIBQePO2BNPDQ0gAUHoAGwiAxAyIgQgBUHoAGxqIgEgBikDCDcDACABIAYtABA6AAggASAGKAIYNgIQIAEgBigCHDYCFCABIAYoAiA2AhggBkEANgIgIAZCADcDGCABIAYoAkw2AkQgASAGKQJENwI8IAEgBv0AAjT9CwIsIAEgBv0AAiT9CwIcIAEgDf0AAwD9CwNIIAEgBigCYDYCWCABIAYoAmQ2AlwgBkIANwNgIA39DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAEgIjcDYCADIARqIQQgAUHoAGohBwJAIAIoAgQiAyACKAIAIgVGBEAgAiABNgIAIAIgBzYCBCACIAQ2AggMAQsDQCABQegAayIBIANB6ABrIgMpAwA3AwAgASADLQAIOgAIIAEgAygCEDYCECABIAMoAhQ2AhQgASADKAIYNgIYIANBADYCGCADQgA3AxAgASADKAJENgJEIAEgAykCPDcCPCABIAP9AAIs/QsCLCABIAP9AAIc/QsCHCABIAMoAkg2AkggASADKAJMNgJMIAEgAygCUDYCUCADQQA2AlAgA0IANwNIIAEgAygCVDYCVCABIAMoAlg2AlggASADKAJcNgJcIANBADYCXCADQgA3AlQgASADKQNgNwNgIAMgBUcNAAsgAigCBCEFIAIoAgAhAyACIAE2AgAgAiAHNgIEIAIgBDYCCCADIAVGDQADQCAFQegAayIHKAJUIgQEQCAFQRBrIg8oAgAiAiAEIgFHBEADQCACQQxrIgEoAgAiCwRAIAJBCGsgCzYCACALEC8LIAEiAiAERw0ACyAHKAJUIQELIA8gBDYCACABEC8LIAVBIGsiDygCACIEBEAgBUEcayIQKAIAIgIgBCIBRwRAA0AgAkEMayIBKAIAIgsEQCACQQhrIAs2AgAgCxAvCyABIgIgBEcNAAsgDygCACEBCyAQIAQ2AgAgARAvCyAFQdgAaygCACIBBEAgBUHUAGsgATYCACABEC8LIAciBSADRw0ACwsgA0UNACADEC8LIAwoAgAiAwRAIAYoAmAiAiADIgFHBEADQCACQQxrIgEoAgAiBARAIAJBCGsgBDYCACAEEC8LIAEiAiADRw0ACyAMKAIAIQELIAYgAzYCYCABEC8LIA0oAgAiAwRAIAYoAlQiAiADIgFHBEADQCACQQxrIgEoAgAiBARAIAJBCGsgBDYCACAEEC8LIAEiAiADRw0ACyANKAIAIQELIAYgAzYCVCABEC8LIAYoAhgiAQRAIAYgATYCHCABEC8LAkAgACgCGCgCACAOaigCBCICQegAayIDKAIUIgEgAygCGEcEQCABIAn9AAMA/QsDACABIAn9AAMg/QsDICABIAn9AAMQ/QsDECADIAFBMGo2AhQMAQsgASACQdgAayIFKAIAIgdrQTBtIgJBAWoiBEHWqtUqTw0OQdWq1SogAkEBdCILIAQgBCALSRsgAkGq1aoVTxsiBAR/IARB1qrVKk8NDiAEQTBsEDIFQQALIgsgAkEwbGoiAiAJ/QADAP0LAwAgAiAJ/QADIP0LAyAgAiAJ/QADEP0LAxAgAkEwaiEMIAEgB0cEQANAIAJBMGsiAiABQTBrIgH9AAMA/QsDACACIAH9AAMg/QsDICACIAH9AAMQ/QsDECABIAdHDQALIAUoAgAhAQsgBSACNgIAIAMgDDYCFCADIAsgBEEwbGo2AhggAUUNACABEC8LIAAoAhgoAgAiCyAOaigCBEHIAGsiASABKwMAIAkqAgy7oDkDACAKIAlBMGoiCUcNAQwDCwsMCwsQNgALIAYoAnQhCgsgCkUNBCAGIAo2AnggChAvDAQLIAgoArQCIQQCQCACIApOBEBEAAAAAAAAAAAhH0EAIQVEAAAAAAAAAAAhIAwBCyAKIAJrIgFBAXEhGAJAIAJBAWogCkYEQEQAAAAAAAAAACEgQQAhBUQAAAAAAAAAACEfDAELIAFBfnEhCkQAAAAAAAAAACEgQQAhBUQAAAAAAAAAACEfQQAhAQNAAn8CQCAEIAJBAnRqKgIAIhpDAACA/1sNACAgIBq7IiGgISAgHyAhY0UNACACDAELIB8hISAFCyEDAkAgBCACQQFqIgVBAnRqKgIAIhpDAACA/1wEQCAgIBq7Ih+gISAgHyAhZA0BCyADIQUgISEfCyACQQJqIQIgAUECaiIBIApHDQALCyAYRQ0AIAQgAkECdGoqAgAiGkMAAID/Ww0AICAgGrsiIaAhICAfICFjRQ0AIAIhBSAhIR8LIAgoArgCIQogBkEANgIQIAZCADcCCAJAIAQgCkcEQCAKIARrIgFB/f///wdPDQEgBiABQQF0EDIiCTYCCCAGIAkgAUECdUEDdGo2AhACQAJAIAFBBGsiAUEESQRAIAQhAiAJIQEMAQsgCSABQQJ2QQFqIgxB/v///wdxIgdBA3RqIQEgBCAHQQJ0aiECQQAhAwNAIAkgA0EDdGogBCADQQJ0av1dAgD9X/0LAwAgA0ECaiIDIAdHDQALIAcgDEYNAQsDQCABIAIqAgC7OQMAIAFBCGohASACQQRqIgIgCkcNAAsLIAYgATYCDAsgBkEIaiIBEM4EIAEgCEHkAmogARDNBCIDQQJ0IgEgCCgCzAJqKgIAIRsgCCgCtAIgAWoqAgAhGiAGKAIIIgEEQCAGIAE2AgwgARAvCyAgtiEcIB8gIES7vdfZ33zbPaCjtiAaIAMgCygC/AFIIgEbIR0gBSADIAEbIQkgCCgC1AEiASAIKALYASIFSQRAIAFCfzcDGCABIBw4AhQgASAdOAIQIAEgGzgCDCABIBo4AgggASAJNgIEIAEgAzYCACABQgA3AyggAUJ/NwMgIAggAUEwajYC1AEMBAsgASAIKALQASIEa0EwbSIKQQFqIgJB1qrVKk8NCEHVqtUqIAUgBGtBMG0iBUEBdCIHIAIgAiAHSRsgBUGq1aoVTxsiBQR/IAVB1qrVKk8NCCAFQTBsEDIFQQALIgcgCkEwbGoiAkJ/NwMYIAIgHDgCFCACIB04AhAgAiAbOAIMIAIgGjgCCCACIAk2AgQgAiADNgIAIAJCADcDKCACQn83AyAgAkEwaiEDIAEgBEcEQANAIAJBMGsiAiABQTBrIgH9AAMA/QsDACACIAH9AAMg/QsDICACIAH9AAMQ/QsDECABIARHDQALIAgoAtABIQELIAggByAFQTBsajYC2AEgCCADNgLUASAIIAI2AtABIAFFDQMgARAvDAMLDAcLIAtFDQAgByABQQJ0aioCACIaQwAAgP9bDQAgICAauyIhoCEgIB8gIWNFDQAgASEDICEhHwsgHyAgRLu919nffNs9oKO2IR4CQCAKQQBMBEBDAAAAACEcQwAAAAAhGgwBCyAKQQFxIRkgCCgCzAIhByAIKAK0AiELAkAgCkEBRgRAQwAAAAAhHEMAAAAAIRpBACEBDAELIApBfnEhCkMAAAAAIRxDAAAAACEaQQAhAUEAIQUDQCALIAFBAnQiCWoqAgAiGyAaXgRAIAcgCWoqAgAhHCAbIRogASEECyALIAFBAXIiCUECdCINaioCACIbIBpeBEAgByANaioCACEcIBshGiAJIQQLIAFBAmohASAFQQJqIgUgCkcNAAsLIBlFDQAgGiALIAFBAnQiBWoqAgAiG11FDQAgBSAHaioCACEcIAEhBCAbIRoLICC2IRsgAyAEIAIgBEoiARshCSAeIBogARshHSAIKALUASIBIAgoAtgBIgVJBEAgAUJ/NwMYIAEgGzgCFCABIB04AhAgASAcOAIMIAEgGjgCCCABIAk2AgQgASAENgIAIAFCADcDKCABQn83AyAgCCABQTBqNgLUAQwBCyABIAgoAtABIgNrQTBtIgpBAWoiAkHWqtUqTw0FQdWq1SogBSADa0EwbSIFQQF0IgcgAiACIAdJGyAFQarVqhVPGyIFBH8gBUHWqtUqTw0FIAVBMGwQMgVBAAsiByAKQTBsaiICQn83AxggAiAbOAIUIAIgHTgCECACIBw4AgwgAiAaOAIIIAIgCTYCBCACIAQ2AgAgAkIANwMoIAJCfzcDICACQTBqIQQgASADRwRAA0AgAkEwayICIAFBMGsiAf0AAwD9CwMAIAIgAf0AAyD9CwMgIAIgAf0AAxD9CwMQIAEgA0cNAAsgCCgC0AEhAQsgCCAHIAVBMGxqNgLYASAIIAQ2AtQBIAggAjYC0AEgAUUNACABEC8LIAggCCsD4AEgCCgC1AFBJGsqAgC7oDkD4AELIAAoAgBBAf4eAgAiFCAAKAIEKAIASA0ACwsgBkGAAWokAA8LEEcACxA2AAvvOAQXfwN9AnsBfiMAQeAAayIIJAACQAJAIAAoAtgBIgogACgCwAFGBEAgAigCACEMIAIoAgQhCSACQfAAaiEQAkAgAigCdCACKAJwIgVrQQJ1IgYgCkkEQCAQIAogBmsQbiAQKAIAIQUMAQsgBiAKTQ0AIAIgBSAKQQJ0ajYCdAsgBSABKAKEqAEgAigCWCAKbEECdGogCkECdPwKAAACQCAEQwAAAABeRQ0AIApBAEwNACAQKAIAIQZBACEFIApBBE8EQCAKQXxxIQUgBP0TIR8DQCAGIAdBAnRqIgsgC/0AAgAgH/3nAf0LAgAgB0EEaiIHIAVHDQALIAUgCkYNAQsDQCAGIAVBAnRqIgcgByoCACAElTgCACAFQQFqIgUgCkcNAAsLIAJB5ABqIRUCQCACKAJoIAIoAmQiBmtBAnUiBSAKSQRAIBUgCiAFaxBuDAELIAUgCk0NACACIAYgCkECdGo2AmgLIAJB/ABqIRQCQCACKAKAASACKAJ8IgZrQQJ1IgUgCkkEQCAUIAogBWsQbgwBCyAFIApNDQAgAiAGIApBAnRqNgKAAQsgECgCACEHAkAgCSAMRyIWDQAgAy0AUUUNACAHIAAoAtwBQQJ0akGAgIB8NgIAIAhBIDsBICAIQQE6ACsgAEHEAWogCEEgahCMAiEFIBAoAgAiByAFKAIAQQJ0akGAgIB8NgIAIAgsACtBAE4NACAIKAIgEC8gECgCACEHCyAHIAAoAvgBQQJ0akGAgIB8NgIAAkAgAy0AFkUNACAKIAAoAvwBIgxMDQACQCAKIAxrIglBBEkEQCAMIQUMAQsgDCAJQXxxIgtqIQVBACEGA0AgByAGIAxqQQJ0av0MAACA/wAAgP8AAID/AACA//0LAgAgBkEEaiIGIAtHDQALIAkgC0YNAQsDQCAHIAVBAnRqQYCAgHw2AgAgBUEBaiIFIApHDQALCyAHIAAoAuABIglBAnRqQYCAgHw2AgAgByAAKAL0AUECdGpBgICAfDYCACADLQA8RQRAIAcgACgC7AFBAnRqQYCAgHw2AgALIAcgACgC5AFBAnRqQYCAgHw2AgAgByAAKALoAUECdGpBgICAfDYCACAHIAAoAvABQQJ0aiEYQQAhBQJAQczXNCgCACIMRQ0AIAxBBE8EQCAMQXxxIQVBACEGA0AgByAGQQFyIAlqQQJ0av0MAACA/wAAgP8AAID/AACA//0LAgAgBkEEaiIGIAVHDQALIAUgDEYNAQsDQCAHIAVBAWoiBSAJakECdGpBgICAfDYCACAFIAxHDQALCyAYQYCAgHw2AgAgAygCnAEiBQRAIAAgASACKAIAIgEgAigCBCABa0EwbSAHIAMoAqABIAURDAALIAMtAFJFDQJB4Nc0KAIAIgFB5Nc0KAIAIhdGDQEgAEHIAWohByAIQSBqQQxyIQkDQAJAIAEsAAtBAE4EQCAIIAEpAgA3AyAgCCABKAIINgIoDAELIAhBIGogASgCACABKAIEEGsLIwBBEGsiDSQAQZniABBoIQYCfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshCwJ/An8jAEEQayIMJAAgBiALaiIFQe////8HTQRAAkAgBUELSQRAIAlCADcCACAJQQA2AgggCSAJLQALQYABcSAFcjoACyAJIAktAAtB/wBxOgALDAELIAkgBUELTwR/IAVBEGpBcHEiDiAOQQFrIg4gDkELRhsFQQoLQQFqIg4Q8wMhEiAJIAkoAghBgICAgHhxIA5B/////wdxcjYCCCAJIAkoAghBgICAgHhyNgIIIAkgEjYCACAJIAU2AgQLIAxBEGokACAJDAELEE0ACyIFLQALQQd2BEAgBSgCAAwBCyAFCyIFQZniACAGEHcgBSAGaiIFAn8gASIMLQALQQd2BEAgASgCAAwBCyAMCyALEHcgBSALakEBQQAQpAIgDUEQaiQAAkAgBygCACIGRQ0AIBAoAgAhEiAIKAIkIAgtACsiASABwEEASCIBGyENIAgoAiAgCEEgaiABGyEOIAchASAGIQUDQAJAIA0gBSgCFCAFLQAbIgsgC8BBAEgiCxsiDyANIA9JIhEbIhMEQCAFKAIQIAVBEGogCxsgDiATEEoiCw0BC0F/IBEgDSAPSxshCwsgASAFIAtBAEgiCxshASAFQQRqIAUgCxsoAgAiBQ0ACwJAAkAgASAHRg0AAkACQCABKAIUIAEtABsiBSAFwEEASCILGyIFIA0gBSANSRsiDwRAIA4gASgCECABQRBqIAsbIA8QSiIBDQELIAUgDU0NAQwCCyABQQBIDQELIAYhBQNAAkACQAJAAkACQAJAIAUoAhQgBS0AGyIBIAHAQQBIIg8bIgEgDSABIA1JIhEbIgsEQCAOIAUoAhAgBUEQaiAPGyIPIAsQSiITRQRAIAEgDUsNAgwDCyATQQBODQIMAQsgASANTQ0CCyAFKAIAIgUNBQwHCyAPIA4gCxBKIgENAQsgEQ0BDAILIAFBAE4NAQsgBSgCBCIFDQEMAwsLIBIgBSgCHEECdGpBgICAfDYCAAsgCCgCLCAJIAgtADciAcBBAEgiBRshDiAIKAIwIAEgBRshDSAHIQEgBiEFA0ACQCANIAUoAhQgBS0AGyILIAvAQQBIIgsbIg8gDSAPSSIRGyITBEAgBSgCECAFQRBqIAsbIA4gExBKIgsNAQtBfyARIA0gD0sbIQsLIAEgBSALQQBIIgsbIQEgBUEEaiAFIAsbKAIAIgUNAAsgASAHRg0BAkACQCABKAIUIAEtABsiBSAFwEEASCILGyIFIA0gBSANSRsiD0UNACAOIAEoAhAgAUEQaiALGyAPEEoiAUUNACABQQBODQEMAwsgBSANSw0CCwNAAkACQAJAAkAgBigCFCAGLQAbIgEgAcBBAEgiCxsiASANIAEgDUkiDxsiBQRAAkAgDiAGKAIQIAZBEGogCxsiCyAFEEoiEQRAIBFBAE4NAQwGCyABIA1LDQULIAsgDiAFEEoiAUUNASABQQBODQIMAwsgASANSw0DCyAPDQELIBIgBigCHEECdGpBgICAfDYCAAwECyAGKAIEIgZFDQIMAQsgBigCACIGDQALC0HxIxC3AQALIAgsADdBAEgEQCAIKAIsEC8LIAgsACtBAEgEQCAIKAIgEC8LIAxBDGoiASAXRw0ACwwBCyAIQa4oNgIYIAhBpSM2AhQgCEGmFTYCEEECQbDkACAIQRBqEDQQAAALIAhBADoAIiAIQaDaADsBICAIQQI6ACsgAEHEAWoiASAIQSBqEPACIQUgCCwAK0EASARAIAgoAiAQLwsCQCAFIABByAFqIgZGDQAgCEEAOgAiIAhBoNoAOwEgIAhBAjoAKyABIAhBIGoQjAIhBSAQKAIAIAUoAgBBAnRqQYCAgHw2AgAgCCwAK0EATg0AIAgoAiAQLwsgCEEAOgAiIAhBoM4AOwEgIAhBAjoAKyABIAhBIGoQ8AIhGSAILAArQQBIBEAgCCgCIBAvCyAZIAZGDQAgCEEAOgAiIAhBoM4AOwEgIAhBAjoAKyABIAhBIGoQjAIhASAQKAIAIAEoAgBBAnRqQYCAgHw2AgAgCCwAK0EATg0AIAgoAiAQLwsgAigCBCIBIAIoAgAiBWtBMG0hBgJAIAEgBUYNACABQTBrKAIAIQcgACgC/AEhAQJAAkAgBkECTwRAIAEgB0oNAyAGQTBsIAVqQeAAaygCACABTg0BIAAoAtwBIgFBAEwNAyAQKAIAIQZBACEFIAFBBE8EQCABQXxxIQVBACEHA0AgBiAHQQJ0av0MAACA/wAAgP8AAID/AACA//0LAgAgB0EEaiIHIAVHDQALIAEgBUYNBAsDQCAGIAVBAnRqQYCAgHw2AgAgBUEBaiIFIAFHDQALDAMLIAEgB0oNAiABIApIDQEMAgsgASAKTg0BCyAQKAIAIQYCQCAKIAFrIgxBBEkEQCABIQUMAQsgASAMQXxxIglqIQVBACEHA0AgBiABIAdqQQJ0av0MAACA/wAAgP8AAID/AACA//0LAgAgB0EEaiIHIAlHDQALIAkgDEYNAQsDQCAGIAVBAnRqQYCAgHw2AgAgBUEBaiIFIApHDQALCwJAIBYNACADKgJYIgRDAAAAAF5FDQACfyAEQwAA8EEgACgCJLKVlRA1IgSLQwAAAE9dBEAgBKgMAQtBgICAgHgLIgUgACgC/AEiB2pBAWoiASAKTg0AIBAoAgAhBgJAIAogBUF/c2ogB2siDEEESQRAIAEhBQwBCyABIAxBfHEiCWohBUEAIQcDQCAGIAEgB2pBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACAHQQRqIgcgCUcNAAsgCSAMRg0BCwNAIAYgBUECdGpBgICAfDYCACAFQQFqIgUgCkgNAAsLAkAgAi0AYkUEQCAQKAIAIQcMAQsgAigCXCIBQQJtIQUgAigCcCEHIAFBAkgNAAJAIAUgACgC/AEiAWoiDCABQQFqIgUgBSAMSBsgAWsiCUEESQRAIAEhBQwBCyABIAlBfHEiC2ohBUEAIQYDQCAHIAEgBmpBAnRq/QwAAID/AACA/wAAgP8AAID//QsCACAGQQRqIgYgC0cNAAsgCSALRg0BCwNAIAcgBUECdGpBgICAfDYCACAFQQFqIgUgDEgNAAsLIAchBgJAIAcgAigCdCIBRg0AIAdBBGoiBSABRg0AIAcqAgAhBANAIAUgBiAEIAUqAgAiHF0iDBshBiAcIAQgDBshBCAFQQRqIgUgAUcNAAsLAkAgCkEATA0AIAYqAgAhHCAKQQFxIQECQCAKQQFGBEBDAAAAACEEQQAhBQwBCyAKQX5xIQxDAAAAACEEQQAhBUEAIQYDQCAHIAVBAnQiCWoqAgAiHUMAAID/XgRAIAQgHSAckxBFkiEECyAHIAlBBHJqKgIAIh1DAACA/14EQCAEIB0gHJMQRZIhBAsgBUECaiEFIAZBAmoiBiAMRw0ACwsCQCABRQ0AIAcgBUECdGoqAgAiHUMAAID/XkUNACAEIB0gHJMQRZIhBAsgBBBXIQQgCkEATA0AIBwgBJIhBCAUKAIAIQxBACEFAkAgCkEESQ0AIAwgB2tBEEkNACAKQXxxIQUgBP0TIR9BACEGA0AgDCAGQQJ0IgFqIAEgB2r9AAIAIiAgH/3lAf0MAACA/wAAgP8AAID/AACA/yAg/QwAAID/AACA/wAAgP8AAID//UT9Uv0LAgAgBkEEaiIGIAVHDQALIAUgCkYNAQsgBUEBciEBIApBAXEEQCAMIAVBAnQiBWogBSAHaioCACIcIASTQwAAgP8gHEMAAID/Xhs4AgAgASEFCyABIApGDQADQCAMIAVBAnQiAWogASAHaioCACIcIASTQwAAgP8gHEMAAID/Xhs4AgAgDCABQQRqIgFqIAEgB2oqAgAiHCAEk0MAAID/IBxDAACA/14bOAIAIAVBAmoiBSAKRw0ACwsgAigCfCIJIAAoAvwBIgxBAnRqIgEhBgJAIAEgAigCgAEiC0YNACABQQRqIgUgC0YNACABKgIAIQQDQCAFIAYgBCAFKgIAIhxdIg0bIQYgHCAEIA0bIQQgBUEEaiIFIAtHDQALC0MAAID/IR0CQCAKIAxMDQAgBioCACEcIAxBAWohGgJ9IAogDGtBAXFFBEAgDCEFQwAAAAAMAQsgDEEBaiEFIAkgDEECdGoqAgAiBEMAAID/XgR9IAQgHJMQRQVDAAAAAAsLIQQgGiAKRwRAA0AgCSAFQQJ0aiIGKgIAIh5DAACA/14EQCAEIB4gHJMQRZIhBAsgBioCBCIeQwAAgP9eBEAgBCAeIByTEEWSIQQLIAVBAmoiBSAKRw0ACwsgBEMAAAAAXkUNACAcIAQQV5IhHQsgCSEGAkAgDEECSQ0AIAZBBGohBSAGKgIAIQQgDEEBa0EDcSINBEBBACELA0AgBSAGIAQgBSoCACIcXSIOGyEGIBwgBCAOGyEEIAVBBGohBSALQQFqIgsgDUcNAAsLIAxB/v///wNqQf////8DcUEDSQ0AA0AgBUEMaiAFQQhqIAVBBGogBSAGIAQgBSoCACIcXSIGGyAcIAQgBhsiBCAFKgIEIhxdIgYbIBwgBCAGGyIEIAUqAggiHF0iBhsgHCAEIAYbIgQgBSoCDCIcXSILGyEGIBwgBCALGyEEIAVBEGoiBSABRw0ACwsCQAJAIAYqAgAgHV0EQCAMQQBMDQFBACEBQQAhBQJAIAxBBEkNACAJIAdrQRBJDQAgDEF8cSEFQQAhBgNAIAcgBkECdCIAav0MAACA/wAAgP8AAID/AACA//0LAgAgACAJav0MAACA/wAAgP8AAID/AACA//0LAgAgBkEEaiIGIAVHDQALIAUgDEYNAgsgDCAFQX9zaiEbIAxBA3EiAgRAA0AgByAFQQJ0IgNqQYCAgHw2AgAgAyAJakGAgIB8NgIAIAVBAWohBSABQQFqIgEgAkcNAAsLIBtBA0kNAQNAIAcgBUECdCIAakGAgIB8NgIAIAAgCWpBgICAfDYCACAHIABBBGoiAWpBgICAfDYCACABIAlqQYCAgHw2AgAgByAAQQhqIgFqQYCAgHw2AgAgASAJakGAgIB8NgIAIAcgAEEMaiIAakGAgIB8NgIAIAAgCWpBgICAfDYCACAFQQRqIgUgDEcNAAsMAQsgAygCqAFFDQACQCACKAI4IAIoAjxGDQAgAkHEAGoiDigCACACKAJIRg0AIAAoAtwBIQsgCEEANgJcIAhCADcCVCAIQQA2AlAgCEIANwJIAkAgC0EATA0AIABB1AFqIQxBACEGAkADQAJAAkAgDCIBIgUoAgAiB0UNAANAIAciBSgCECIBIAZKBEAgBSEBIAUoAgAiBw0BDAILIAEgBk4NAiAFKAIEIgcNAAsgBUEEaiEBC0EgEDIiByAGNgIQIAcgBTYCCCAHQgA3AgAgB0IANwIUIAdBADYCHCABIAc2AgAgByEFIAAoAtABKAIAIgkEQCAAIAk2AtABIAEoAgAhBQsgACgC1AEgBRCkASAAIAAoAtgBQQFqNgLYAQsCQAJAIAcoAhggBy0AHyIBIAHAQQBIIgEbRQ0AIAdBFGoiBSgCACEHIAggAikDUCIhNwMIIAggITcDQCAIQSBqIAcgBSABGyAIQQhqENgEAkAgCCgCWCIBIAgoAlxJBEAgAUEANgIIIAFCADcCACABIAgoAiA2AgAgASAIKAIkNgIEIAEgCCgCKDYCCCAIQQA2AiggCEIANwIgIAEgCCkCLDcCDCAIIAFBFGo2AlgMAQsCQAJAAkAgCCgCWCAIKAJUIgVrQRRtIgdBAWoiAUHNmbPmAEkEQEHMmbPmACAIKAJcIAVrQRRtIgVBAXQiCSABIAEgCUkbIAVB5syZM08bIgFBzZmz5gBPDQEgAUEUbCIBEDIiCSAHQRRsaiIFIAgoAiA2AgAgBSAIKAIkNgIEIAUgCCgCKDYCCCAIQQA2AiggCEIANwIgIAUgCCkCLDcCDCABIAlqIQEgBUEUaiEJIAgoAlgiByAIKAJUIg1GDQIDQCAFQRRrIgUgB0EUayIHKAIANgIAIAUgBygCBDYCBCAFIAcoAgg2AgggB0EANgIIIAdCADcCACAFIAcpAgw3AgwgByANRw0ACyAIIAE2AlwgCCgCWCEBIAggCTYCWCAIKAJUIQcgCCAFNgJUIAEgB0YNAwNAIAFBFGsiBSgCACIJBEAgAUEQayAJNgIAIAkQLwsgBSIBIAdHDQALDAMLEDYACxBHAAsgCCABNgJcIAggCTYCWCAIIAU2AlQLIAcEQCAHEC8LIAgoAiAiAUUNACAIIAE2AiQgARAvCyAIKAJYQRRrIgEoAgAhDSABKQIMISEgCCgCTCIFIAgoAlAiCUkEQCAFICE3AgggBSANNgIEIAUgBjYCACAIIAVBEGo2AkwMAQsgBSAIKAJIIgFrQQR1IhJBAWoiB0GAgICAAU8NAUH/////ACAJIAFrIglBA3UiDyAHIAcgD0kbIAlB8P///wdPGyIJBH8gCUGAgICAAU8NBCAJQQR0EDIFQQALIg8gEkEEdGoiByAhNwIIIAcgDTYCBCAHIAY2AgAgB0EQaiENIAEgBUcEQANAIAdBEGsiByAFQRBrIgX9AAIA/QsCACABIAVHDQALIAEhBQsgCCAPIAlBBHRqNgJQIAggDTYCTCAIIAc2AkggBUUNACAFEC8LIAZBAWoiBiALRw0BDAMLCxA2AAsQRwALIAhBIGogAkE4aiAOIAhByABqEM8EIAgoAiAiACAIKAIkIgFHBEAgAyoCsAEhBCAQKAIAIQMgACEFA0AgAyAFKAIAQQJ0aiIGIAYqAgAgBJM4AgAgBUEQaiIFIAFHDQALCyAABEAgCCAANgIkIAAQLwsgCCgCSCIABEAgABAvCyAIKAJUIgBFDQAgCCgCWCIHIAAiBUcEQANAIAdBFGsiASgCACIDBEAgB0EQayADNgIAIAMQLwsgASIHIABHDQALIAgoAlQhBQsgCCAANgJYIAUQLwsgAigCcCIBIQYCQCABIAIoAnQiAEYNACABQQRqIgUgAEYNACAGKgIAIQQDQCAFIAYgBCAFKgIAIhxdIgIbIQYgHCAEIAIbIQQgBUEEaiIFIABHDQALCyAKQQBMDQEgBioCACEcIApBAXEhAAJAIApBAUYEQEMAAAAAIQRBACEFDAELIApBfnEhAkMAAAAAIQRBACEFQQAhBgNAIAEgBUECdCIDaioCACIdQwAAgP9eBEAgBCAdIByTEEWSIQQLIAEgA0EEcmoqAgAiHUMAAID/XgRAIAQgHSAckxBFkiEECyAFQQJqIQUgBkECaiIGIAJHDQALCwJAIABFDQAgASAFQQJ0aioCACIdQwAAgP9eRQ0AIAQgHSAckxBFkiEECyAEEFchBCAKQQBMDQEgHCAEkiEEIBQoAgAhAkEAIQUCQCAKQQRJDQAgAiABa0EQSQ0AIApBfHEhBSAE/RMhH0EAIQYDQCACIAZBAnQiAGogACABav0AAgAiICAf/eUB/QwAAID/AACA/wAAgP8AAID/ICD9DAAAgP8AAID/AACA/wAAgP/9RP1S/QsCACAGQQRqIgYgBUcNAAsgBSAKRg0BCyAFQQFyIQAgCkEBcQRAIAIgBUECdCIDaiABIANqKgIAIhwgBJNDAACA/yAcQwAAgP9eGzgCACAAIQULIAAgCkYNAANAIAIgBUECdCIAaiAAIAFqKgIAIhwgBJNDAACA/yAcQwAAgP9eGzgCACACIABBBGoiAGogACABaioCACIcIASTQwAAgP8gHEMAAID/Xhs4AgAgBUECaiIFIApHDQALCyAKQQBMDQAgFSgCACEAIBQoAgAhASAQKAIAIQJBACEFIApBAUcEQCAKQX5xIQZBACELA0BDAAAAACEEQwAAAAAhHCACIAVBAnQiA2oqAgBDAACA/1wEQCABIANqKgIAEEUhHAsgACADaiAcOAIAIAIgBUEBckECdCIDaioCAEMAAID/XARAIAEgA2oqAgAQRSEECyAAIANqIAQ4AgAgBUECaiEFIAtBAmoiCyAGRw0ACwsgCkEBcUUNAEMAAAAAIQQgAiAFQQJ0IgNqKgIAQwAAgP9cBEAgASADaioCABBFIQQLIAAgA2ogBDgCAAsgCEHgAGokAAvUEQEQfyMAQSBrIggkAAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgASgCBCIGIAEoAgBGBEAgAigCBCIHIAIoAghGDQEgB0EANgIIIAdCADcCACABKAIEIgAgASgCACIGRg0MIAAgBmsiBkEASA0CIAcgBhAyIgA2AgQgByAANgIAIAcgACAGQXxxajYCCCABKAIAIgQgASgCBCIBRgRAIAAhBgwMCyABIARrQQRrIgZBDEkNCSAAIARrQRBJDQkgACAGQQJ2QQFqIglB/P///wdxIgpBAnQiA2ohBiADIARqIQMDQCAAIAVBAnQiC2ogBCALav0AAgD9CwIAIAVBBGoiBSAKRw0ACyAJIApGDQsMCgsCQCAGQQRrKAIAIgooAgAiBkEDa0ECTwRAIAZBAkcNASAKQQhqIQsgACgCACAKKAIEQQxsaigCACEGA0AgASgCACEDIAEoAgQhBCAIQQA2AhwgCEIANwIUAkACQCAEQQRrIgQgA0cEQCAEIANrIgVBAEgNEyAIIAUQMiIENgIYIAggBDYCFCAIIAQgBUF8cSIFaiIHNgIcIAQgAyAF/AoAACAIIAQgBWoiAzYCGCAKKAIIQQJJDQIgAyAHTw0BIAMgCzYCACAIIANBBGo2AhgMAgtBACEEQQAhB0EAIQMgCigCCEECSQ0BCyADIARrIgxBAnUiDUEBaiIFQYCAgIAETw0RQf////8DIAcgBGsiB0EBdSIJIAUgBSAJSRsgB0H8////B08bIgkEfyAJQYCAgIAETw0HIAlBAnQQMgVBAAsiDiANQQJ0aiIFIAs2AgAgBUEEaiENIAMgBEcEQAJAIAxBBGsiB0EMTwRAIAVBEGshDyADQRBrIRAgAyAHQQJ2QQFqIhFB/P///wdxIgxBAnQiB2shAyAFIAdrIQVBACEHA0AgDyAHQQJ0IhJrIBAgEmv9AAIA/QsCACAHQQRqIgcgDEcNAAsgDCARRg0BCwNAIAVBBGsiBSADQQRrIgMoAgA2AgAgAyAERw0ACwsgCCgCFCEECyAIIA4gCUECdGo2AhwgCCANNgIYIAggBTYCFCAERQ0AIAQQLwsCQCAGKAIAQQJJDQAgCCgCGCIDIAgoAhxHBEAgAyAGNgIAIAggA0EEajYCGAwBCyADIAgoAhQiDGsiBEECdSIHQQFqIgVBgICAgARPDRFB/////wMgBEEBdSIJIAUgBSAJSRsgBEH8////B08bIgkEfyAJQYCAgIAETw0HIAlBAnQQMgVBAAsiDSAHQQJ0aiIFIAY2AgAgBUEEaiEOIAMgDEcEQAJAAkAgBEEEayIHQSxJDQAgAyAEIA1qa0EQSQ0AIAVBEGshDyADQRBrIRAgAyAHQQJ2QQFqIhFB/P///wdxIgRBAnQiB2shAyAFIAdrIQVBACEHA0AgDyAHQQJ0IhJrIBAgEmv9AAIA/QsCACAHQQRqIgcgBEcNAAsgBCARRg0BCwNAIAVBBGsiBSADQQRrIgMoAgA2AgAgAyAMRw0ACwsgCCgCFCEDCyAIIA0gCUECdGo2AhwgCCAONgIYIAggBTYCFCADRQ0AIAMQLwsgACAIQRRqIAIQiwIDQCAGIgNBCGohBiADKAIAIgRBAUsNAAsgBEEBRiIEQQN0IQUgCCgCFCIGBEAgCCAGNgIYIAYQLwsgAyAFaiEGIAQNAAsMDgsgAigCBCIHIAIoAghGDQEgB0EANgIIIAdCADcCACABKAIEIgAgASgCACIGRg0IIAAgBmsiBkEASA0EIAcgBhAyIgA2AgQgByAANgIAIAcgACAGQXxxajYCCCABKAIAIgQgASgCBCIBRgRAIAAhBgwICyABIARrQQRrIgZBDEkNBSAAIARrQRBJDQUgACAGQQJ2QQFqIglB/P///wdxIgpBAnQiA2ohBiADIARqIQMDQCAAIAVBAnQiC2ogBCALav0AAgD9CwIAIAVBBGoiBSAKRw0ACyAJIApGDQcMBgsgCEGbITYCCCAIQfUfNgIEIAhBphU2AgBBAkGw5AAgCBA0EAAAC0EAIQdBACEKAkACQAJAAkACQAJAIAIoAgQiBCACKAIAIglrQQxtIgZBAWoiAEHWqtWqAUkEQEHVqtWqASACKAIIIAlrQQxtIgNBAXQiBSAAIAAgBUkbIANBqtWq1QBPGyILBEAgC0HWqtWqAU8NAiALQQxsEDIhBwsgByAGQQxsaiIFQQA2AgggBUIANwIAIAEoAgQiDCABKAIAIgFGDQYgDCABayIAQQBIDQIgBSAAEDIiBjYCACAFIAYgAEF8cWo2AgggAEEEayIAQQxJDQMgBiABa0EQSQ0DIAYgAEECdkEBaiIOQfz///8HcSINQQJ0IgNqIQAgASADaiEDA0AgBiAKQQJ0Ig9qIAEgD2r9AAIA/QsCACAKQQRqIgogDUcNAAsgDSAORg0FDAQLEDYACxBHAAsQNgALIAEhAyAGIQALA0AgACADKAIANgIAIABBBGohACADQQRqIgMgDEcNAAsLIAUgADYCBAsgByALQQxsaiEAIAVBDGohAQJAIAQgCUcEQANAIAVBDGsiBUEANgIIIAUgBEEMayIEKAIANgIAIAUgBCgCBDYCBCAFIAQoAgg2AgggBEEANgIIIARCADcCACAEIAlHDQALIAIgADYCCCACKAIEIQAgAiABNgIEIAIoAgAhBCACIAU2AgAgACAERg0BA0AgAEEMayIBKAIAIgIEQCAAQQhrIAI2AgAgAhAvCyABIgAgBEcNAAsMAQsgAiAANgIIIAIgATYCBCACIAU2AgALIAQEQCAEEC8LDAsLEDYACxBHAAsQNgALIAQhAyAAIQYLA0AgBiADKAIANgIAIAZBBGohBiADQQRqIgMgAUcNAAsLIAcgBjYCBAsgAiAHQQxqNgIEDAQLIAQhAyAAIQYLA0AgBiADKAIANgIAIAZBBGohBiADQQRqIgMgAUcNAAsLIAcgBjYCBAsgAiAHQQxqNgIECyAIQSBqJAAPCxA2AAvcAQEGfwJAAkAgACgCBCIARQ0AIAEoAgAgASABLQALIgLAQQBIIgMbIQUgASgCBCACIAMbIQEDQAJAAkACQAJAAkAgACgCFCAALQAbIgIgAsBBAEgiBBsiAiABIAEgAksiBhsiAwRAIAUgACgCECAAQRBqIAQbIgQgAxBKIgdFBEAgASACSQ0CDAMLIAdBAE4NAgwBCyABIAJPDQILIAAoAgAiAA0EDAULIAQgBSADEEoiAg0BCyAGDQEMBAsgAkEATg0DCyAAKAIEIgANAAsLQfEjELcBAAsgAEEcagsxACABBEAgACABKAIAEI0CIAAgASgCBBCNAiABLAAbQQBIBEAgASgCEBAvCyABEC8LCzEAIAEEQCAAIAEoAgAQjgIgACABKAIEEI4CIAEsABtBAEgEQCABKAIQEC8LIAEQLwsLMQAgAQRAIAAgASgCABCPAiAAIAEoAgQQjwIgASwAH0EASARAIAEoAhQQLwsgARAvCwv6AgEDfyABIAEoAggRAAAiAUEBIAEoAgARBAAhA0GcEBA7IQEgAxDoBCEEIAEgAygCHCIFIAUoAgQRAAAiBTYCDCABIAQ2AgggAUEBNgIEIAEgAzYCACABQRxqQQBBgBD8CwAgAygCJBogAUH/////BzYCGCABQQE2AhAgAUEBOgCYECABIAQgBSAEIAVwayAFcGo2AhRBCBA7IgMQggQ2AgQgAyABNgIAIAAgAzYCAEEAIQFBICEDA0AgASADaiIEQQF2IgVBAWogASAEQRhJIgQbIgEgAyAFIAQbIgNJDQALAkAgAUEfTQR/IAFBAnRBgKMBaigCAEECdEHMgAFqBUHQgAILQYCAzwBqIgEgACgCCCAAKAIEIgRrIgNLBEAgAEEEaiABIANrEPECDAELIAEgA08NACAAIAEgBGo2AggLIAIoAhAiAUUEQEEEEF4iAEGoxwI2AgAgAEHQxwJBzAAQAgALIAAoAgAgASABKAIAKAIYEQAAENoBC14BAX8jAEEQayICJAAgACgCAEE8RwRAQbzDAigCABAwGiACQdPLADYCCCACQdIENgIEIAJBmyc2AgBBuMMCKAIAQcvkACACEDEQAAALIAAoAjQgATYCACACQRBqJAALTwECfyAAKAIAIgBBATYCECAAIAAoAggiAiAAKAIMIgEgAiABcGsgAXAiAWo2AhQgACAALQCYEAR/Qf////8HBSAAKAIAKAIkIAFrCzYCGAuoBQEJfyMAQSBrIgMkACAAKAIIIgEoAhghByABKAIEIQUgASgCACEGQX8hAgNAAkACQCAFKAIMIgFFBEAgACgCCCEBDAELIAUoAhAgAREAACEJIAAoAgghASAJRQ0AQQEhAiABQQH+HgIgGgwBCwJAIAFBAf4lAhxBAUYEQCADQQA2AhQgA0ICNwIMIAMgBSgCADYCGCADIAUoAgQ2AhwgAkF/RwRAIAYoAgwgAkECdGooAgAiASgCQEHw1jRqLQAABEAgAyABIAcQ2wE2AhQgA0EMaiABEMoBCyAAKAIIIQQgASABKAK0AUEBajYCtAEgASAB/QADuAEgBP0AAwj90QH9CwO4AQsCQCACQQFqIgEgBigCBE4NAANAIAYoAgwgAUECdGooAgAiAiAHENsBIQQgACgCCP0MAAAAAAAAAAAAAAAAAAAAAP0LAwggAyAENgIUIAIoAkBBoNY0ai0AAARAIANBADYCDCADQQxqIAIQygELIARBAUcNASADQQE2AgwgA0EMaiIIIAIQygEgAigCQEHw1jRqLQAABEAgA0ECNgIMIAggAhDKAQsgACgCCCEEIAIgAigCtAFBAWo2ArQBIAIgAv0AA7gBIAT9AAMI/dEB/QsDuAEgBSgCDCICBEAgBSgCECACEQAADQILIAFBAWoiASAGKAIESA0ACwsgACgCCCAH/hcCHCAAKAIIIAH+FwIgDAELA0AgACgCCP4QAiAiASACRg0ACwsgBigCBCABTARAQQAhAgwBCyAGKAIMIAFBAnRqKAIAIgggBxDbASECIANBATYCDCAAKAIEIQQgAyACNgIUIAMgBDYCECADIAUoAgA2AhggAyAFKAIENgIcIAIgBEoEQCADQQxqIAgQygELIAEhAgwBCwsgA0EgaiQAIAILFQAgAEHstgM2AgAgAEEEahD7AiAACwwAIAAQ/AIaIAAQLwtLAQJ/IAAoAgQiBkEIdSEHIAAoAgAiACABIAIgBkEBcQR/IAcgAygCAGooAgAFIAcLIANqIARBAiAGQQJxGyAFIAAoAgAoAhQRDAALmgEAIABBAToANQJAIAAoAgQgAkcNACAAQQE6ADQCQCAAKAIQIgJFBEAgAEEBNgIkIAAgAzYCGCAAIAE2AhAgA0EBRw0CIAAoAjBBAUYNAQwCCyABIAJGBEAgACgCGCICQQJGBEAgACADNgIYIAMhAgsgACgCMEEBRw0CIAJBAUYNAQwCCyAAIAAoAiRBAWo2AiQLIABBAToANgsLXQEBfyAAKAIQIgNFBEAgAEEBNgIkIAAgAjYCGCAAIAE2AhAPCwJAIAEgA0YEQCAAKAIYQQJHDQEgACACNgIYDwsgAEEBOgA2IABBAjYCGCAAIAAoAiRBAWo2AiQLCxAAQci7A/4QAgARCQAQRgALHQEBf0EYEDIQhAMiAUEMahCEAxogACABNgIAIAALQwACQEHgmDX+EgAAQQFxDQBB4Jg1EFRFDQBB2Jg1Qf0DEJIEBEAQRgALQdyYNUHYmDU2AgBB4Jg1EFMLQdyYNSgCAAuaAQECfwJ/QcitAy4BACIBRQRAIwNBHGpBHDYCAEF/DAELAkACQCABQX5KDQBB6aAMIQACQAJAAkACQAJAAkACQCABQf8BcUEBaw4LCAABAgMEBAUFBgMHC0GAgAgMCAtBgIACDAcLQYCABAwGC0H/////BwwFCxAZDAQLEBhBEHYMAwtBAAwCCyABIQALIAALIgBBACAAQQBKGwsjAQF/AkAgACgCACIBBEAgARCTBEUNAQsQRgALIABBADYCAAsdACAAIAFBwIQ9biIAEKkBIAEgAEHAhD1saxCfAgsdACAAIAFBkM4AbiIAEKkBIAEgAEGQzgBsaxCgAgsbACAAIAFB5ABuIgAQqQEgASAAQeQAbGsQqQEL+wEBA38jAEEQayICJAAgAiABNgIMAkACQAJ/IAAtAAsiA0EHdiIERQRAQQEhASADQf8AcQwBCyAAKAIIQf////8HcUEBayEBIAAoAgQLIgMgAUYEQCAAIAFBASABIAEQhgMCfyAALQALQQd2BEAgACgCAAwBC0EACxoMAQsCfyAALQALQQd2BEAgACgCAAwBC0EACxogBA0AIAAiASADQQFqIAAtAAtBgAFxcjoACyAAIAAtAAtB/wBxOgALDAELIAAoAgAhASAAIANBAWo2AgQLIAEgA0ECdGoiACACKAIMNgIAIAJBADYCCCAAIAIoAgg2AgQgAkEQaiQAC/gBAQN/IwBBEGsiAiQAIAIgAToADwJAAkACfyAALQALIgNBB3YiBEUEQEEKIQEgA0H/AHEMAQsgACgCCEH/////B3FBAWshASAAKAIECyIDIAFGBEAgACABQQEgASABEKUCAn8gAC0AC0EHdgRAIAAoAgAMAQtBAAsaDAELAn8gAC0AC0EHdgRAIAAoAgAMAQtBAAsaIAQNACAAIgEgA0EBaiAALQALQYABcXI6AAsgACAALQALQf8AcToACwwBCyAAKAIAIQEgACADQQFqNgIECyABIANqIgAgAi0ADzoAACACQQA6AA4gACACLQAOOgABIAJBEGokAAvgAgIFfwJ+An8jAEHwAGsiAyQAAkACQAJAIAEoAjAiAiABKAIAIgVBJGwiBEGYmwFqKAIARw0AIAE1AjQiByABKQMQIAKtfiAEQZSbAWo0AgB/Ug0AIAE1AjgiCCAHIAEpAxh+Ug0AIAE1AjwgASkDICAIflINACABKAKIASEGIAAgBSABKAIMIAFBEGpBAEEAEEkiAkUNASACQSc2AkAgAkMAAIA/OAJEIAYEfyAAIAIoAgAgAigCDCACQRBqQQBBABBJBUEACyEAIAIgATYCjAEgAiAANgKIASACQQA2ApABIANB8ABqJAAgAgwDC0G8wwIoAgAQMBogA0HF1wA2AmggA0GvJjYCZCADQeQmNgJgQbjDAigCAEHL5AAgA0HgAGoQMQwBC0G8wwIoAgAQMBogA0HXLzYCGCADQcYUNgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDELEAAACws9AQF/IwBBEGsiAyQAIAMgAjoADwNAIAEEQCAAIAMtAA86AAAgAUEBayEBIABBAWohAAwBCwsgA0EQaiQAC8MCAQV/IwBBEGsiBSQAIAJB7////wcgAWtNBEACfyAALQALQQd2BEAgACgCAAwBCyAACyEGIAVBBGoiByAAIAFB5////wNJBH8gBSABQQF0NgIMIAUgASACajYCBCMAQRBrIgIkACAHKAIAIAVBDGoiCCgCAEkhCSACQRBqJAAgCCAHIAkbKAIAIgJBC08EfyACQRBqQXBxIgIgAkEBayICIAJBC0YbBUEKC0EBagVB7////wcLEMMBIAUoAgQhAiAFKAIIGiAEBEAgAiAGIAQQdwsgAyAERwRAIAIgBGogBCAGaiADIARrEHcLIAFBAWoiAUELRwRAIAAgBiABEPEBCyAAIAI2AgAgACAAKAIIQYCAgIB4cSAFKAIIQf////8HcXI2AgggACAAKAIIQYCAgIB4cjYCCCAFQRBqJAAPCxBNAAswACAAKAIARQRAIABBfxCVBA8LIAAoAgwEQCAAQQhqIgBBAf4eAgAaIAAQ+gELQQAL5wEBB38jAEEgayIEJAAgASgCiAEhCCAAIAEoAgAgASgCDCABQRBqIAFBABBJIQcgBCABQdQBaiICNgIQIAdBycsAIARBEGoQcyIDIAEoAjA2AjAgAyABKAI0NgI0IAMgASgCODYCOCADIAEoAjw2AjwgBCACNgIAIANBgtUAIAQQcyICIAEpAxg3AxAgAiABKQMQNwMYIAMgASgCNDYCMCADIAEoAjA2AjQgAkEhNgJAIAgEQCAAIAIoAgAgAigCDCACQRBqQQBBABBJIQULIAIgATYCjAEgAiAFNgKIASAEQSBqJAAgAgsNACAAIAEgAkJ/EOADCxcAIAAoAggQREcEQCAAKAIIEOMDCyAAC0wBAX8jAEEQayIGJAAgBiACNwMAIAYgAzcDCCAAIAFBAiAGIAUQqwIiACADpyAEbCIBNgI8IAAgATYCOCAAIAQ2AjQgBkEQaiQAIAALwgEBA38jAEEgayIFJAAgASgCiAEhBiAAIAEoAgAgAiADIAEgBBBJIQcgBSABQdQBajYCECAHQcnLACAFQRBqEHMiAgRAIAJBHzYCQCACIAQ2AkQgBgR/IAAgAigCACACKAIMIAJBEGpBAEEAEEkFQQALIQQgAiABNgKMASACIAQ2AogBIAVBIGokACACDwtBvMMCKAIAEDAaIAVB1y82AgggBUHGFDYCBCAFQeQmNgIAQbjDAigCAEHL5AAgBRAxEAAACzgBAn8jAEEQayIDJAAgAyACNgIMIANBCGogA0EMahB/IQQgACABEMICIQAgBBB+IANBEGokACAACwQAQQELNwECfyMAQRBrIgIkACACIAAoAgA2AgwgAiACKAIMIAFBAnRqNgIMIAIoAgwhAyACQRBqJAAgAws0AQJ/IwBBEGsiAiQAIAIgACgCADYCDCACIAIoAgwgAWo2AgwgAigCDCEDIAJBEGokACADC4QCAQV/IwBBIGsiAyQAAkACQCAAKAIAIgUgAkEDakF8cSIHIAAoAhQiBAR/IAQoAgQgBCgCAGoFQQALIgZqIgJBFGpJBEAgAyAFNgIIIAMgAjYCBCADQb4PNgIAQe/yACADELMBQQAhAgwBCyAAKAIEIgUgBmoiAkEANgIQIAIgATYCDCACQQA2AgggAiAHNgIEIAIgBkEUaiIBNgIAIAEgBWpBA3ENASAEQQhqIABBEGogBBsgAjYCACAAIAI2AhQLIANBIGokACACDwtBvMMCKAIAEDAaIANBlsEANgIYIANBjBM2AhQgA0HkJjYCEEG4wwIoAgBBy+QAIANBEGoQMRAAAAsxACACKAIAIQIDQAJAIAAgAUcEfyAAKAIAIAJHDQEgAAUgAQsPCyAAQQRqIQAMAAsAC/oEAQF/IwBBEGsiDCQAIAwgADYCDAJAAkAgACAFRgRAIAEtAABFDQFBACEAIAFBADoAACAEIAQoAgAiAUEBajYCACABQS46AAACfyAHLQALQQd2BEAgBygCBAwBCyAHLQALQf8AcQtFDQIgCSgCACIBIAhrQZ8BSg0CIAooAgAhAiAJIAFBBGo2AgAgASACNgIADAILAkAgACAGRw0AAn8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELRQ0AIAEtAABFDQFBACEAIAkoAgAiASAIa0GfAUoNAiAKKAIAIQAgCSABQQRqNgIAIAEgADYCAEEAIQAgCkEANgIADAILQX8hACALIAtBgAFqIAxBDGoQsQIgC2siBkH8AEoNASAGQQJ1QdDqAmosAAAhBQJAAkAgBkF7cSIAQdgARwRAIABB4ABHDQEgAyAEKAIAIgFHBEBBfyEAIAFBAWssAAAiA0HfAHEgAyADQeEAa0EaSRsgAiwAACICQd8AcSACIAJB4QBrQRpJG0cNBQsgBCABQQFqNgIAIAEgBToAAEEAIQAMBAsgAkHQADoAAAwBCyAFQd8AcSAFIAVB4QBrQRpJGyIAIAIsAABHDQAgAiAAQSByIAAgAEHBAGtBGkkbOgAAIAEtAABFDQAgAUEAOgAAAn8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELRQ0AIAkoAgAiACAIa0GfAUoNACAKKAIAIQEgCSAAQQRqNgIAIAAgATYCAAsgBCAEKAIAIgBBAWo2AgAgACAFOgAAQQAhACAGQdQASg0BIAogCigCAEEBajYCAAwBC0F/IQALIAxBEGokACAAC6YBAQJ/IwBBEGsiBiQAIAZBDGoiBSABKAIcIgE2AgAgAUEEakEB/h4CABogBRBsIgFB0OoCQfDqAiACIAEoAgAoAjARCAAaIAMgBRC/ASIBIAEoAgAoAgwRAAA2AgAgBCABIAEoAgAoAhARAAA2AgAgACABIAEoAgAoAhQRAgAgBSgCACIAQQRqQX/+HgIARQRAIAAgACgCACgCCBEBAAsgBkEQaiQACzEAIAItAAAhAgNAAkAgACABRwR/IAAtAAAgAkcNASAABSABCw8LIABBAWohAAwACwAL7gQBAX8jAEEQayIMJAAgDCAAOgAPAkACQCAAIAVGBEAgAS0AAEUNAUEAIQAgAUEAOgAAIAQgBCgCACIBQQFqNgIAIAFBLjoAAAJ/IActAAtBB3YEQCAHKAIEDAELIActAAtB/wBxC0UNAiAJKAIAIgEgCGtBnwFKDQIgCigCACECIAkgAUEEajYCACABIAI2AgAMAgsCQCAAIAZHDQACfyAHLQALQQd2BEAgBygCBAwBCyAHLQALQf8AcQtFDQAgAS0AAEUNAUEAIQAgCSgCACIBIAhrQZ8BSg0CIAooAgAhACAJIAFBBGo2AgAgASAANgIAQQAhACAKQQA2AgAMAgtBfyEAIAsgC0EgaiAMQQ9qELQCIAtrIgZBH0oNASAGQdDqAmosAAAhBQJAAkACQAJAIAZBfnFBFmsOAwECAAILIAMgBCgCACIBRwRAIAFBAWssAAAiA0HfAHEgAyADQeEAa0EaSRsgAiwAACICQd8AcSACIAJB4QBrQRpJG0cNBQsgBCABQQFqNgIAIAEgBToAAEEAIQAMBAsgAkHQADoAAAwBCyAFQd8AcSAFIAVB4QBrQRpJGyIAIAIsAABHDQAgAiAAQSByIAAgAEHBAGtBGkkbOgAAIAEtAABFDQAgAUEAOgAAAn8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELRQ0AIAkoAgAiACAIa0GfAUoNACAKKAIAIQEgCSAAQQRqNgIAIAAgATYCAAsgBCAEKAIAIgBBAWo2AgAgACAFOgAAQQAhACAGQRVKDQEgCiAKKAIAQQFqNgIADAELQX8hAAsgDEEQaiQAIAALpgEBAn8jAEEQayIGJAAgBkEMaiIFIAEoAhwiATYCACABQQRqQQH+HgIAGiAFEHIiAUHQ6gJB8OoCIAIgASgCACgCIBEIABogAyAFEMEBIgEgASgCACgCDBEAADoAACAEIAEgASgCACgCEBEAADoAACAAIAEgASgCACgCFBECACAFKAIAIgBBBGpBf/4eAgBFBEAgACAAKAIAKAIIEQEACyAGQRBqJAALfwICfwJ+IwBBoAFrIgQkACAEIAE2AjwgBCABNgIUIARBfzYCGCAEQRBqIgVCABCAASAEIAUgA0EBEOoDIAQpAwghBiAEKQMAIQcgAgRAIAIgASAEKAIUIAQoAogBaiAEKAI8a2o2AgALIAAgBjcDCCAAIAc3AwAgBEGgAWokAAvbAQEIfyAAIABBPRCOBCIBRgRAQQAPCwJAIAAgASAAayIFai0AAA0AQYyINSgCACICRQ0AIAIoAgAiAUUNAANAAkACfyAAIQRBACAFIgZFDQAaIAAtAAAiAwR/AkADQCADIAEtAAAiB0cNASAHRQ0BIAZBAWsiBkUNASABQQFqIQEgBC0AASEDIARBAWohBCADDQALQQAhAwsgAwVBAAsgAS0AAGsLRQRAIAIoAgAgBWoiAS0AAEE9Rg0BCyACKAIEIQEgAkEEaiECIAENAQwCCwsgAUEBaiEICyAIC0QBAX8jAEEQayIFJAAgBSABIAIgAyAEQoCAgICAgICAgH+FEHEgBSkDACEBIAAgBSkDCDcDCCAAIAE3AwAgBUEQaiQAC4QBAQJ/IABBqMwCNgIAIAAoAighAQNAIAEEQEEAIAAgAUEBayIBQQJ0IgIgACgCJGooAgAgACgCICACaigCABEFAAwBCwsgACgCHCIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgACgCIBAvIAAoAiQQLyAAKAIwEC8gACgCPBAvIAALIAAgACAAKAIYRSABciIBNgIQIAAoAhQgAXEEQBBGAAsLRgAjAEEQayIBJAAgASADNgIMIAQgAiADIAJrIgL8CgAAIAEgAiAEajYCCCAAIAEoAgw2AgAgACABKAIINgIEIAFBEGokAAs2AQF/IwBBEGsiAyQAIAMgATYCDCADIAI2AgggACADKAIMNgIAIAAgAygCCDYCBCADQRBqJAALOwEBfyAAQfDKAigCACIBNgIAIAAgAUEMaygCAGpB/MoCKAIANgIAIABBCGoQ8gEaIABB7ABqEMACIAALDAAgAEEIahDAAiAACwgAIAAQugIaC9IDAgJ+An8jAEEgayIEJAACQCABQv///////////wCDIgNCgICAgICAwIA8fSADQoCAgICAgMD/wwB9VARAIAFCBIYgAEI8iIQhAyAAQv//////////D4MiAEKBgICAgICAgAhaBEAgA0KBgICAgICAgMAAfCECDAILIANCgICAgICAgIBAfSECIABCgICAgICAgIAIUg0BIAIgA0IBg3whAgwBCyAAUCADQoCAgICAgMD//wBUIANCgICAgICAwP//AFEbRQRAIAFCBIYgAEI8iIRC/////////wODQoCAgICAgID8/wCEIQIMAQtCgICAgICAgPj/ACECIANC////////v//DAFYNAEIAIQIgA0IwiKciBUGR9wBJDQAgBEEQaiAAIAFC////////P4NCgICAgICAwACEIgIgBUGB9wBrEGYgBCAAIAJBgfgAIAVrEK8BIAQpAwhCBIYgBCkDACIAQjyIhCECIAQpAxAgBCkDGIRCAFKtIABC//////////8Pg4QiAEKBgICAgICAgAhaBEAgAkIBfCECDAELIABCgICAgICAgIAIUg0AIAJCAYMgAnwhAgsgBEEgaiQAIAIgAUKAgICAgICAgIB/g4S/C4gCAAJAIAAEfyABQf8ATQ0BAkAjAygCYCgCAEUEQCABQYB/cUGAvwNGDQMMAQsgAUH/D00EQCAAIAFBP3FBgAFyOgABIAAgAUEGdkHAAXI6AABBAg8LIAFBgEBxQYDAA0cgAUGAsANPcUUEQCAAIAFBP3FBgAFyOgACIAAgAUEMdkHgAXI6AAAgACABQQZ2QT9xQYABcjoAAUEDDwsgAUGAgARrQf//P00EQCAAIAFBP3FBgAFyOgADIAAgAUESdkHwAXI6AAAgACABQQZ2QT9xQYABcjoAAiAAIAFBDHZBP3FBgAFyOgABQQQPCwsjA0EcakEZNgIAQX8FQQELDwsgACABOgAAQQEL1AIBBH8jAEHQAWsiBSQAIAUgAjYCzAEgBUGgAWoiAkEAQSj8CwAgBSAFKALMATYCyAECQEEAIAEgBUHIAWogBUHQAGogAiADIAQQiQRBAEgEQEF/IQQMAQsgACgCTEEATgRAIAAQggEhBgsgACAAKAIAIghBX3E2AgACfwJAAkAgACgCMEUEQCAAQdAANgIwIABBADYCHCAAQgA3AxAgACgCLCEHIAAgBTYCLAwBCyAAKAIQDQELQX8gABDSAg0BGgsgACABIAVByAFqIAVB0ABqIAVBoAFqIAMgBBCJBAshAiAHBEAgAEEAQQAgACgCJBEDABogAEEANgIwIAAgBzYCLCAAQQA2AhwgACgCFCEBIABCADcDECACQX8gARshAgsgACAAKAIAIgEgCEEgcXI2AgBBfyACIAFBIHEbIQQgBkUNACAAEIEBCyAFQdABaiQAIAQLugQCBH0CfwJAAkACQAJ9AkAgALwiBkH/////B3EiBUHE8NaMBE8EQCAFQYCAgPwHSw0FIAZBAEgEQEMAAIC/DwsgBUGY5MWVBEkNASAAQwAAAH+UDwsgBUGZ5MX1A0kNAiAFQZGrlPwDSw0AIAZBAE4EQEEBIQVD0fcXNyEBIABDgHExv5IMAgtBfyEFQ9H3F7chASAAQ4BxMT+SDAELAn8gAEM7qrg/lEMAAAA/IACYkiIBi0MAAABPXQRAIAGoDAELQYCAgIB4CyIFsiICQ9H3FzeUIQEgACACQ4BxMb+UkgsiACAAIAGTIgCTIAGTIQEMAQsgBUGAgICYA0kNAUEAIQULIAAgAEMAAAA/lCIDlCICIAIgAkMQMM86lENoiAi9kpRDAACAP5IiBEMAAEBAIAQgA5STIgOTQwAAwEAgACADlJOVlCEDIAVFBEAgACAAIAOUIAKTkw8LIAAgAyABk5QgAZMgApMhAQJAAkACQCAFQQFqDgMAAgECCyAAIAGTQwAAAD+UQwAAAL+SDwsgAEMAAIC+XQRAIAEgAEMAAAA/kpNDAAAAwJQPCyAAIAGTIgAgAJJDAACAP5IPCyAFQRd0IgZBgICA/ANqviECIAVBOU8EQCAAIAGTQwAAgD+SIgAgAJJDAAAAf5QgACAClCAFQYABRhtDAACAv5IPC0GAgID8AyAGa74hAyAFQRZNBH1DAACAPyADkyAAIAGTkgUgACABIAOSk0MAAIA/kgsgApQhAAsgAAsqAQJ/IwBBEGsiBCQAIAQgAzYCDCAAIAEgAiADELABIQUgBEEQaiQAIAUL2wEBAX8CQEH8ugMoAgAiAEEATgRAIABFDQEjAygCGCAAQf////97cUcNAQsCQEGAuwMoAgBBCkYNAEHEugMoAgAiAEHAugMoAgBGDQBBxLoDIABBAWo2AgAgAEEKOgAADwsQ0QIPC0EAQQBB/////wP+SAL8ugMEQEGwugMQggEaCwJAAkBBgLsDKAIAQQpGDQBBxLoDKAIAIgBBwLoDKAIARg0AQcS6AyAAQQFqNgIAIABBCjoAAAwBCxDRAgtBAEEA/kEC/LoDQYCAgIAEcQRAQfy6A0EBELUBCwt+AQR/Qaz3NCgCACMDKAIYRgRAQaz3NEEANgIACwNAQaT3NCgCACECQaD3NEGg9zQoAgAiACAAQQFrQQAgAEH/////B3EiAUEBRxtBACABQf////8HRxsiA/5IAgAgAEcNAAsCQCADDQAgAkUgAEEATnENAEGg9zQgARC1AQsLUgEDfwJAAkADQEEGIQFBCiECAkBBoPc0KAIAIgBB/////wdxQf7///8Haw4CAwIACyAAQaD3NCAAIABBAWr+SAIARw0AC0EAIQILIAIhAQsgAQs4AQF/QZDzNCgCACIABEBBkPM0IABBAWs2AgAPC0GM8zRBAP4XAgBBlPM0KAIABEBBjPM0EI4BCwtZAQJ/IwMoAhgiAEGM8zQoAgBHBEBBjPM0QQAgAP5IAgAiAQRAA0BBjPM0QZTzNCABEKEBQYzzNEEAIAD+SAIAIgENAAsLDwtBkPM0QZDzNCgCAEEBajYCAAtaAQN/IwBBEGsiAiQAIwMhAyACQQxqIgQEQCAEIAMtACg2AgALIANBAToAKCAAIAEgAhD5ASEFIAIoAgwiAUECTQR/IwMgAToAKEEABUEcCxogAkEQaiQAIAULcAECfyMAQRBrIgEkACAAQQE2AiAgAEEEaiICEFYaIAAoAiwgACgCMEcEQANAIAFBBGogABCfBCACEFIaIAEoAgwgASgCBBEBACACEFYaIAAoAiwgACgCMEcNAAsLIAIQUhogAEEANgIgIAFBEGokAAueDAMGfAN+CH8jAEEQayIOJAACQAJAIAG9IglCNIinIg1B/w9xIg9BvghrIhBB/35LIAC9IghCNIinIgtB/w9rQYJwT3ENACAJQgGGIgpCgICAgICAgBB8QoGAgICAgIAQVARARAAAAAAAAPA/IQIgCEKAgICAgICA+D9RDQIgClANAiAKQoGAgICAgIBwVCAIQgGGIghCgICAgICAgHBYcUUEQCAAIAGgIQIMAwsgCEKAgICAgICA8P8AUQ0CRAAAAAAAAAAAIAEgAaIgCEL/////////7/8AViAJQgBZcxshAgwCCyAIQgGGQoCAgICAgIAQfEKBgICAgICAEFQEQCAAIACiIQIgCEIAUwRAIAKaIAIgCRCkBEEBRhshAgsgCUIAWQ0CIwBBEGsiC0QAAAAAAADwPyACozkDCCALKwMIIQIMAgsgCEIAUwRAIAkQpAQiDEUEQCAAIAChIgAgAKMhAgwDCyALQf8PcSELIAxBAUZBEnQhDCAIQv///////////wCDIQgLIBBB/35NBEBEAAAAAAAA8D8hAiAIQoCAgICAgID4P1ENAiAPQb0HTQRAIAEgAZogCEKAgICAgICA+D9WG0QAAAAAAADwP6AhAgwDCyANQYAQSSAIQoGAgICAgID4P1RHBEAjAEEQayILRAAAAAAAAABwOQMIIAsrAwhEAAAAAAAAAHCiIQIMAwsjAEEQayILRAAAAAAAAAAQOQMIIAsrAwhEAAAAAAAAABCiIQIMAgsgCw0AIABEAAAAAAAAMEOivUL///////////8Ag0KAgICAgICAoAN9IQgLAnwgCUKAgIBAg78iBSEHIA4gCEKAgICA0Kql8z99IglCNIentyIDQdCgAisDAKIgCUItiKdB/wBxQQV0IgtBqKECaisDAKAgCCAJQoCAgICAgIB4g30iCEKAgICACHxCgICAgHCDvyIAIAtBkKECaisDACIEokQAAAAAAADwv6AiAiAIvyAAoSAEoiIEoCIAIANByKACKwMAoiALQaChAmorAwCgIgMgACADoCIDoaCgIAQgAEHYoAIrAwAiBKIiBiACIASiIgSgoqAgAiAEoiICIAMgAyACoCICoaCgIAAgACAGoiIDoiADIAMgAEGIoQIrAwCiQYChAisDAKCiIABB+KACKwMAokHwoAIrAwCgoKIgAEHooAIrAwCiQeCgAisDAKCgoqAiACACIAIgAKAiAqGgOQMIIAcgAr1CgICAQIO/IgOiIQAgASAFoSADoiAOKwMIIAIgA6GgIAGioCEBAkAgAL1CNIinQf8PcSILQckHa0E/SQ0AIAtByQdJBEAgAEQAAAAAAADwP6AiAJogACAMGwwCCyALQYkISSERQQAhCyARDQAgAL1CAFMEQCMAQRBrIgtEAAAAAAAAAJBEAAAAAAAAABAgDBs5AwggCysDCEQAAAAAAAAAEKIMAgsjAEEQayILRAAAAAAAAADwRAAAAAAAAABwIAwbOQMIIAsrAwhEAAAAAAAAAHCiDAELQZjbASsDACAAokGg2wErAwAiAqAiAyACoSICQbDbASsDAKIgAkGo2wErAwCiIACgoCABoCIAIACiIgEgAaIgAEHQ2wErAwCiQcjbASsDAKCiIAEgAEHA2wErAwCiQbjbASsDAKCiIAO9IgmnQQR0QfAPcSINQYjcAWorAwAgAKCgoCEAIA1BkNwBaikDACAJIAytfEIthnwhCCALRQRAAnwgCUKAgICACINQBEAgCEKAgICAgICAiD99vyIBIACiIAGgRAAAAAAAAAB/ogwBCyAIQoCAgICAgIDwP3wiCL8iASAAoiIDIAGgIgCZRAAAAAAAAPA/YwR8IwBBEGsiCyESIAtEAAAAAAAAEAA5AwggEiALKwMIRAAAAAAAABAAojkDCCAIQoCAgICAgICAgH+DvyAARAAAAAAAAPC/RAAAAAAAAPA/IABEAAAAAAAAAABjGyICoCIFIAMgASAAoaAgACACIAWhoKCgIAKhIgAgAEQAAAAAAAAAAGEbBSAAC0QAAAAAAAAQAKILDAELIAi/IgEgAKIgAaALIQILIA5BEGokACACC8EBAQN/AkAgASACKAIQIgMEfyADBSACENICDQEgAigCEAsgAigCFCIFa0sEQCACIAAgASACKAIkEQMADwsCQCACKAJQQQBIBEBBACEDDAELIAEhBANAIAQiA0UEQEEAIQMMAgsgACADQQFrIgRqLQAAQQpHDQALIAIgACADIAIoAiQRAwAiBCADSQ0BIAAgA2ohACABIANrIQEgAigCFCEFCyAFIAAgARB7GiACIAIoAhQgAWo2AhQgASADaiEECyAECzcBAn8gACgCTEEASARAIAAgASACEKkEDwsgABCCASEEIAAgASACEKkEIQIgBARAIAAQgQELIAILfAECfyAAIAAoAkgiAUEBayABcjYCSCAAKAIUIAAoAhxHBEAgAEEAQQAgACgCJBEDABoLIABBADYCHCAAQgA3AxAgACgCACIBQQRxBEAgACABQSByNgIAQX8PCyAAIAAoAiwgACgCMGoiAjYCCCAAIAI2AgQgAUEbdEEfdQuNAQECfyMAQRBrIgAkACAAQQo6AA8CQAJAQcC6AygCACIBBH8gAQVBsLoDENICDQJBwLoDKAIAC0HEugMoAgAiAUYNAEGAuwMoAgBBCkYNAEHEugMgAUEBajYCACABQQo6AAAMAQtBsLoDIABBD2pBAUHUugMoAgARAwBBAUcNACAALQAPGgsgAEEQaiQAC1kBAX8gACAAKAJIIgFBAWsgAXI2AkggACgCACIBQQhxBEAgACABQSByNgIAQX8PCyAAQgA3AgQgACAAKAIsIgE2AhwgACABNgIUIAAgASAAKAIwajYCEEEAC9MBAgF/AnwgAisDICEFAn8CQCABKwMgIgYgACsDIGRFBEBBACAFIAZkRQ0CGiABIAIQPkEBIAErAyAgACsDIGRFDQIaIAAgARA+DAELIAUgBmQEQCAAIAIQPkEBDAILIAAgARA+QQEgAisDICABKwMgZEUNARogASACED4LQQILIQQgAysDICACKwMgZAR/IAIgAxA+IAIrAyAgASsDIGRFBEAgBEEBag8LIAEgAhA+IAErAyAgACsDIGRFBEAgBEECag8LIAAgARA+IARBA2oFIAQLC34BAX8gACABIAIgAxDTAiEFIAQrAyAgAysDIGQEfyADIAQQPiADKwMgIAIrAyBkRQRAIAVBAWoPCyACIAMQPiACKwMgIAErAyBkRQRAIAVBAmoPCyABIAIQPiABKwMgIAArAyBkRQRAIAVBA2oPCyAAIAEQPiAFQQRqBSAFCwtIAQF/IABBwLMBNgIAIAAoAggiAQRAIAEgASgCACgCBBEBAAsgAEHcqQE2AgAgACgCBCIBBEAgASABKAIAKAIEEQEACyAAEC8LiwkBCn8CQAJAAkACQAJAAkAgAC0AWQRAIAAoAgwiAyABIAMoAgAoAhQRBAAhBSAAKAIMIgEgAiABKAIAKAIUEQQAIQggACgCPCIDIABBQGsoAgAiBEkEQCADIAVB/wFxIAhBCHRyOwAADAcLIAMgACgCOCIBayICQX1MDQJB/////wcgBCABayIEIAJBAXUiCUEBaiIHIAQgB0sbIARB/v///wdPGyIHBH8gB0EASA0EIAdBAXQQMgVBAAshBiAGIAlBAXRqIgQgBUH/AXEgCEEIdHI7AAAgBEECaiEIIAEgA0cEQAJAAkAgAkECayIFQR5JDQAgAyACQX5xIAZqa0EQSQ0AIARBEGshCSADQRBrIQogBCAFQQF2QQFqIgtBeHEiBUEBdCICayEEIAMgAmshA0EAIQIDQCAJIAJBAXQiDGsgCiAMa/0AAAD9CwAAIAJBCGoiAiAFRw0ACyAFIAtGDQELA0AgBEECayIEIANBAmsiAy8AADsAACABIANHDQALCyAAKAI4IQMLIAAgBiAHQQF0ajYCQCAAIAg2AjwgACAENgI4IANFDQEgAxAvDwsgAEFAaygCACEEIAAoAjwhAwJAIAAtAFoEQCADIARJDQYgAyAAKAI4IgdrIgZBfUwNBUH/////ByAEIAdrIgQgBkEBdSIJQQFqIgUgBCAFSxsgBEH+////B08bIgUEfyAFQQBIDQUgBUEBdBAyBUEACyEIIAggCUEBdGoiBCABQf8BcSACQQh0cjsAACAEQQJqIQkgAyAHRwRAAkACQCAGQQJrIgFBHkkNACADIAZBfnEgCGprQRBJDQAgBEEQayEGIANBEGshCiAEIAFBAXZBAWoiC0F4cSIBQQF0IgJrIQQgAyACayEDQQAhAgNAIAYgAkEBdCIMayAKIAxr/QAAAP0LAAAgAkEIaiICIAFHDQALIAEgC0YNAQsDQCAEQQJrIgQgA0ECayIDLwAAOwAAIAMgB0cNAAsLIAAoAjghAwsgACAIIAVBAXRqNgJAIAAgCTYCPCAAIAQ2AjggAw0BDAILIAMgBEkNBSADIAAoAjgiB2siBkF9TA0EQf////8HIAQgB2siBCAGQQF1IglBAWoiBSAEIAVLGyAEQf7///8HTxsiBQR/IAVBAEgNBCAFQQF0EDIFQQALIQggCCAJQQF0aiIEIAFB/wFxIAJBCHRyOwAAIARBAmohCSADIAdHBEACQAJAIAZBAmsiAUEeSQ0AIAMgBkF+cSAIamtBEEkNACAEQRBrIQYgA0EQayEKIAQgAUEBdkEBaiILQXhxIgFBAXQiAmshBCADIAJrIQNBACECA0AgBiACQQF0IgxrIAogDGv9AAAA/QsAACACQQhqIgIgAUcNAAsgASALRg0BCwNAIARBAmsiBCADQQJrIgMvAAA7AAAgAyAHRw0ACwsgACgCOCEDCyAAIAggBUEBdGo2AkAgACAJNgI8IAAgBDYCOCADRQ0BCyADEC8LDwsQNgALEEcACxA2AAsgAyABQf8BcSACQQh0cjsAAAsgACADQQJqNgI8C+sDAQN/AkACQCABIAJGDQACQAJAAkACQAJAAkACQAJAAkAgASwAACIEQSJrDlUACAgICAgICAgICAgIAAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAAgICAgBAggICAMICAgICAgIBAgICAUIBggHCAsCQCADBEAgAyAEEFoMAQsgACAEEFkLDAkLAkAgAwRAIANBBxBaDAELIABBBxBZCwwICwJAIAMEQCADQQgQWgwBCyAAQQgQWQsMBwsCQCADBEAgA0EMEFoMAQsgAEEMEFkLDAYLAkAgAwRAIANBChBaDAELIABBChBZCwwFCwJAIAMEQCADQQ0QWgwBCyAAQQ0QWQsMBAsCQCADBEAgA0EJEFoMAQsgAEEJEFkLDAMLAkAgAwRAIANBCxBaDAELIABBCxBZCwwCCyAEQXhxQTBHDQAgBEEwayEFAkAgAUEBaiIEIAJGDQAgBC0AACIGQfgBcUEwRwRAIAQhAgwBCyAFQQN0IAZqQTBrIQUgAUECaiIEIAJGDQAgBC0AACIGQfgBcUEwRwRAIAQhAgwBCyABQQNqIQIgBUEDdCAGakEwayEFCyADBEAgAyAFwBBaIAIPCyAAIAXAEFkgAg8LEIACAAsgAUEBagvLAgEEfyMAQRBrIgQkACADIAJrIgVB8P///wdJBEACQCAFQQpNBEAgBCAFOgAPIARBBGohBgwBCyAFQQ9yQQFqIgcQMiEGIAQgB0GAgICAeHI2AgwgBCAGNgIEIAQgBTYCCAsgAiADRwR/IAYgAiAF/AoAACAFIAZqBSAGC0EAOgAAIAAgASgCCCIBIAQoAgQgBEEEaiAELQAPIgLAQQBIIgMbIgUgBSAEKAIIIAIgAxtqIAEoAgAoAhARBgACQAJAAkAgACgCBCAALQALIgEgAcAiAUEASCICG0EBaw4MAgEBAQEBAQEBAQEAAQsgACgCACAAIAIbIgAgAC0AAzoACwwBCyABQQBIBEAgACgCAEEAOgAAIABBADYCBAwBCyAAQQA6AAsgAEEAOgAACyAELAAPQQBIBEAgBCgCBBAvCyAEQRBqJAAPCxBNAAv4BQEIfyMAQSBrIgQkACADIAJrIgVB8P///wdJBEACQCAFQQpNBEAgBCAFOgAfIARBFGohBgwBCyAFQQ9yQQFqIgcQMiEGIAQgB0GAgICAeHI2AhwgBCAGNgIUIAQgBTYCGAsgAiADRwR/IAYgAiAF/AoAACAFIAZqBSAGC0EAOgAAIABBADYCCCAAQgA3AgACQAJAIAQoAhggBC0AHyICIALAQQBIIgIbRQ0AIAQoAhQgBEEUaiIKIAIbIQcjAEEQayIGJAAgBiAHNgIMIAZBDGohCyMAQRBrIgMkAEHAoQMhAkHvACEFA0AgBQRAIAMgAjYCDCADIAMoAgwgBUEBdiIJQQN0ajYCDCADKAIMIghBCGogAiAIIAsQiwMiCBshAiAFIAlBf3NqIAkgCBshBQwBCwsgA0EQaiQAIwBBEGsiBSQAIARBCGoiA0IANwIAIANBADYCCCAFQRBqJAACQCACQbioA0YNACAHIAIoAgAQnwENACADIAIsAAQQWgsgBkEQaiQAIAAgBCgCEDYCCCAAIAQpAgg3AgAgACgCBCAALQALIgIgAsAiAkEASBsNACAEKAIYIAQtAB8iBSAFwEEASCIFGyIGQQJLDQAgAyABKAIIIgEgBCgCFCAKIAUbIgMgAyAGaiABKAIAKAIQEQYAIAJBAEgEQCAAKAIAEC8LIAAgBCkCCDcCACAAIAQoAhA2AggCQAJAIAAoAgQgAC0ACyIBIAHAIgFBAEgbQQFrDgwAAQEBAQEBAQEBAQABCyAEQRRqIABGDQEgBC0AHyIDwCECIAFBAE4EQCACQQBOBEAgACAEKQIUNwIAIAAgBCgCHDYCCAwECyAAIAQoAhQgBCgCGBDeAQwCCyAAIAQoAhQgBEEUaiACQQBIIgAbIAQoAhggAyAAGxDfAQwBCyABQQBIBEAgACgCAEEAOgAAIABBADYCBAwBCyAAQQA6AAsgAEEAOgAACyAELAAfQQBODQAgBCgCFBAvCyAEQSBqJAAPCxBNAAuTAgIDfwF+AkAgACgCDCICQQFxBEBBGBAyIQIgACgCJCgCBCEDIAJB9K4BNgIAIAIgAzYCBCACQQhqIAAoAgAiAzYCACADQQRqQQH+HgIAGiAAKQIEIQUgAiABNgIUIAIgBTcCDCAAKAIkIAI2AgQMAQsgAkEIcQRAQRgQMiECIAAoAiQoAgQhAyACQcyvATYCACACIAM2AgQgAkEIaiAAKAIAIgM2AgAgA0EEakEB/h4CABogACkCBCEFIAIgATYCFCACIAU3AgwgACgCJCACNgIEDAELQQwQMiECIAAoAiQiAygCBCEEIAIgATYCCCACQaiwATYCACACIAQ2AgQgAyACNgIECyAAIAAoAiQoAgQ2AiQLFABBDBBeQRAQcEHQqQNB0gAQAgALFABBDBBeQQwQcEHQqQNB0gAQAgAL2gMCCX8BeyAB/QACACELIABBADYCGCAAQgA3AhAgACAL/QsCACABKAIUIgUgASgCECICayIEQQxtIQMCQAJAAkACQAJAAkAgAiAFRwRAIANB1qrVqgFPDQEgACAEEDIiAjYCFCAAIAI2AhAgACACIANBDGxqNgIYIAEoAhAiAyABKAIUIgRHBEADQCACIAMpAgA3AgAgAiADKAIINgIIIAJBDGohAiADQQxqIgMgBEcNAAsLIAAgAjYCFAsgAEIANwIcIABBADYCJCABKAIgIgIgASgCHCIERg0FIAIgBGsiAkEASA0BIAAgAhAyIgQ2AiAgACAENgIcIAAgBCACQXhxajYCJCABKAIcIgUgASgCICIHRgRAIAQhAgwFCyAHIAVrQQhrIgJBGEkNAiAEIAVrQRBJDQIgBCACQQN2QQFqIglB/v///wNxIghBA3QiA2ohAiADIAVqIQMDQCAEIAZBA3QiCmogBSAKav0AAgD9CwIAIAZBAmoiBiAIRw0ACyAIIAlGDQQMAwsQNgALEDYACyAFIQMgBCECCwNAIAIgAykCADcCACACQQhqIQIgA0EIaiIDIAdHDQALCyAAIAI2AiALIAAgASkCKDcCKCAAIAEtADA6ADAgAAuQAgEHfyABIAAoAggiAyAAKAIEIgJrQQN1TQRAIAAgAQR/IAJBACABQQN0IgD8CwAgACACagUgAgs2AgQPCwJAIAIgACgCACIFa0EDdSIHIAFqIgRBgICAgAJJBEBB/////wEgAyAFayIDQQJ1IgggBCAEIAhJGyADQfj///8HTxsiAwRAIANBgICAgAJPDQIgA0EDdBAyIQYLIAdBA3QgBmoiBEEAIAFBA3QiAfwLACABIARqIQEgAiAFRwRAA0AgBEEIayIEIAJBCGsiAikCADcCACACIAVHDQALIAAoAgAhAgsgACAGIANBA3RqNgIIIAAgATYCBCAAIAQ2AgAgAgRAIAIQLwsPCxA2AAsQRwAL6AsBCX8jAEHQAGsiBiQAIAZBADYCTCAGQgA3AkQCQAJAAkAgACgCHCILRQ0AIAZBADoAQCAGIAI2AjwgBiACNgI4IAb9DAAAAAAAAAAAAAAAAAAAAAD9CwMgIAb9DAAAAAAAAAAAAAAAAAAAAAD9CwMQIAb9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAZBxABqIAYQgQIgBigCHCIHBEAgBiAHNgIgIAcQLwsgBigCECIHBEAgBiAHNgIUIAcQLwsgBigCSCIHQTRrIghBADYCACAIIAI2AgwgCCABNgIIIAggATYCBAJAIAAoAhAiCSAIKAIUIAhBEGoiDCgCACINa0EMbSIKSwRAIAwgCSAKayAGQThqEMAEIAYoAkghBwwBCyAJIApPDQAgCCANIAlBDGxqNgIUCwJAIAAoAhQiACAHQTRrIggiCSgCICAIQRxqIgooAgAiDGtBA3UiCEsEQCAKIAAgCGsQ3gIgBigCSCEHDAELIAAgCE8NACAJIAwgAEEDdGo2AiALIAtBAEchCCAHQTRrIgAgBToAMCAAIAQ2AiwgACALNgIoIARBgCBxIQkgBEEgcSELIAIgAWshCkEAIQACQANAIABBAWoiAEH/H3FFIABBDHYgCk5xDQMgB0E0ayIEKAIoIgUEQCAFIAQgBSgCACgCCBECAAsCQAJAAkACQCAEKAIAQegHag4KAAgICAgDAwIBAwgLIAdBLGsoAgAhBAJAIAtFDQAgASAERw0AIAYoAkgiBEEYaygCACIFBEAgBEEUayAFNgIAIAUQLwsgBEEkaygCACIFBEAgBEEgayAFNgIAIAUQLwsgBiAEQTRrNgJIDAMLAkAgCUUNACACIARGDQAgBigCSCIEQRhrKAIAIgUEQCAEQRRrIAU2AgAgBRAvCyAEQSRrKAIAIgUEQCAEQSBrIAU2AgAgBRAvCyAGIARBNGs2AkgMAwsgAygCACIAQQE6AAggACAENgIEIAAgATYCAAJAIAdBNGsiASgCFCIDIAEoAhAiAkYNAEEBIAMgAmtBDG0iAyADQQFNGyIEQQFxIQ5BACEBIANBAk8EQCAEQX5xIQlBACEFA0AgACABQQFyQQxsIgtqIgMgAiABQQxsIgpqIgQoAgA2AgAgAyAEKAIENgIEIAMgBC0ACDoACCAAIApqIgMgAiALaiIEKAIANgIYIAMgBCgCBDYCHCADIAQtAAg6ACAgAUECaiEBIAVBAmoiBSAJRw0ACwsgDkUNACABQQxsIgEgAGoiACABIAJqIgEoAgA2AgwgACABKAIENgIQIAAgAS0ACDoAFAsgBigCRCEEDAQLIAYgBBDdAiEFIAQoAigiB0EBIAQgBygCACgCDBEFACAGKAIoIgRBACAFIAQoAgAoAgwRBQACQCAGKAJIIgQgBigCTEkEQCAEIAX9AAIA/QsCACAE/QwAAAAAAAAAAAAAAAAAAAAA/QsCECAEIAYoAhA2AhAgBCAGKAIUNgIUIAQgBigCGDYCGCAGQQA2AhAgBEIANwIgIAQgBigCHDYCHCAEIAYoAiA2AiAgBCAGKAIkNgIkIAb9DAAAAAAAAAAAAAAAAAAAAAD9CwIYIAQgBi0AMDoAMCAEIAYpAig3AiggBiAEQTRqNgJIDAELIAZBxABqIAUQgQIgBigCHCIERQ0AIAYgBDYCICAEEC8LIAYoAhAiBEUNASAGIAQ2AhQgBBAvDAELIAYoAkgiBEEYaygCACIFBEAgBEEUayAFNgIAIAUQLwsgBEEkaygCACIFBEAgBEEgayAFNgIAIAUQLwsgBiAEQTRrNgJICyAGKAJEIgQgBigCSCIHRw0AC0EAIQgLIARFDQAgBCEAIAQgBigCSCIBRwRAA0AgAUE0ayIAKAIcIgIEQCABQRRrIAI2AgAgAhAvCyABQSRrKAIAIgIEQCABQSBrIAI2AgAgAhAvCyAAIgEgBEcNAAsgBigCRCEACyAGIAQ2AkggABAvCyAGQdAAaiQAIAgPCxDcAgALENsCAAvCAQEEf0EIEDIhByAAKAIkIgUoAgQhBiAHQcioATYCACAHIAY2AgQgBUEANgIEQSQQMiEFIAAoAhQhBiACKAIEIQggBSAHNgIIIAUgCDYCBCAFQQA6ACAgBSAENgIcIAUgAzYCGCAFIAY2AhQgBUF/NgIQIAUgATYCDCAFQdyyATYCACACQQA2AgRBCBAyIgFB2LMBNgIAIAEgBTYCBCAAKAIkIAE2AgQgACAFKAIINgIkIAIgBTYCBCAAIAZBAWo2AhQLwgEBBH9BCBAyIQggACgCJCIGKAIEIQcgCEHIqAE2AgAgCCAHNgIEIAZBADYCBEEkEDIhBiAAKAIUIQcgAygCBCEJIAYgCDYCCCAGIAk2AgQgBkEAOgAgIAYgBTYCHCAGIAQ2AhggBiAHNgIUIAYgAjYCECAGIAE2AgwgBkHcsgE2AgAgA0EANgIEQQgQMiIBQdizATYCACABIAY2AgQgACgCJCABNgIEIAAgBigCCDYCJCADIAY2AgQgACAHQQFqNgIUC6sfAhB/AX4CQAJAIAEgAkYNACABLQAAQdsARw0AIAFBAWoiCiACRg0BIAEtAAEhA0HcABAyIAAgACgCJCgCBCADQd4ARiIDIAAoAgwiBUEBcSAFQQhxQQN2ELYBIQkgACgCJCAJNgIEIAAgCTYCJCABQQJqIAogAxsiAyACRg0BAkAgACgCDEHwB3FBgARGDQAgAy0AAEHdAEcNACAJQd0AEKMBIANBAWohAwsgAiADRg0BA0BBACEOIwBBQGoiByQAAkACQCADIgEgAiIKRg0AIAMtAAAiBUHdAEYNACAHQQA2AjggB0IANwMwAkACQAJAAkACQCADQQFqIApGDQAgBUHbAEcNAAJAAkAgAy0AAUEuaw4QAwICAgICAgICAgICAQICAAILAn8jAEEgayIIJAACQAJAIAoiBiADQQJqIgRrIgpBAkgNACAEIApqQQFrIQsgBCEKA0AgCkEBaiEFIAotAABBPUYEQCAFLQAAQd0ARwRAIAUiCiALRw0CDAMLIAYgCkYNAiAIQRRqIgUgACAEIAoQ2QIgCCgCGCAILQAfIgQgBMBBAEgiBBsiBkUNAyAIQQhqIAAgCCgCFCAFIAQbIgUgBSAGahDYAgJAAkAgCCgCDCIEIAgtABMiBSAFwCIGQQBIGwRAIAkoAkgiBSAJKAJMRg0BAkAgBkEATgRAIAUgCCkCCDcCACAFIAgoAhA2AggMAQsgBSAIKAIIIAQQawsgCSAFQQxqNgJIDAILAkACQCAIKAIYIAgtAB8iBSAFwEEASCIFG0EBaw4CAAEHCyAJIAgoAhQgCEEUaiAFGywAABCjAQwCCyAJIAgoAhQgCEEUaiAFGyIFLAAAIAUsAAEQ1gIMAQtBACELAkACQAJAIAkoAkgiBSAJKAJEIgZrQQxtIgxBAWoiBEHWqtWqAUkEQEHVqtWqASAJKAJMIAZrQQxtIg1BAXQiDyAEIAQgD0kbIA1BqtWq1QBPGyIEBEAgBEHWqtWqAU8NAiAEQQxsEDIhCwsgBEEMbCENIAsgDEEMbGohBAJAIAgsABNBAE4EQCAEIAgpAgg3AgAgBCAIKAIQNgIIDAELIAQgCCgCCCAIKAIMEGsgCSgCRCEGIAkoAkghBQsgCyANaiELIARBDGohDCAFIAZGDQIDQCAEQQxrIgQgBUEMayIFKQIANwIAIAQgBSgCCDYCCCAFQgA3AgAgBUEANgIIIAUgBkcNAAsgCSALNgJMIAkoAkghBSAJIAw2AkggCSgCRCEGIAkgBDYCRCAFIAZGDQMDQCAFQQxrIQQgBUEBaywAAEEASARAIAQoAgAQLwsgBCIFIAZHDQALDAMLEDYACxBHAAsgCSALNgJMIAkgDDYCSCAJIAQ2AkQLIAYEQCAGEC8LCyAILAATQQBIBEAgCCgCCBAvCyAILAAfQQBIBEAgCCgCFBAvCyAIQSBqJAAgCkECagwECyAFIgogC0cNAAsLEP4BAAsQuwQACyEKDAULAn8CQAJAIAoiBiADQQJqIgRrIgpBAkgNACAEIApqQQFrIQggBCEKA0AgCkEBaiEFIAotAABBOkYEQCAFLQAAQd0ARwRAIAUiCiAIRw0CDAMLIAYgCkYNAgJ/IAAoAgxBAXEhESMAQRBrIgUkACAKIARrIgZB8P///wdJBEACQCAGQQpNBEAgBSAGOgAPIAVBBGohCAwBCyAGQQ9yQQFqIgsQMiEIIAUgC0GAgICAeHI2AgwgBSAINgIEIAUgBjYCCAsgBCAKRwR/IAggBCAG/AoAACAGIAhqBSAIC0EAOgAAIAAoAgQiBCAFKAIEIAVBBGoiBiAFLQAPIgjAQQBIIgsbIgwgDCAFKAIIIAggCxtqIAQoAgAoAhgRAwAaIAUoAgQgBiAFLAAPQQBIGyENQQAhCCMAQRBrIgwkACAMIA02AgwgDEEMaiESIwBBEGsiBiQAQcCoAyEEQQ8hCwNAIAsEQCAGIAQ2AgwgBiAGKAIMIAtBAXYiD0EDdGo2AgwgBigCDCIQQQhqIAQgECASEIsDIhAbIQQgCyAPQX9zaiAPIBAbIQsMAQsLIAZBEGokAAJAIARBuKkDRg0AIA0gBCgCABCfAQ0AQfgIIAQoAgQiBCAEQYAIRiIGGyEIIAYNACARRQ0AIARBIHIgBCAEQRhxGyEICyAMQRBqJAAgBSwAD0EASARAIAUoAgQQLwsgBUEQaiQAIAgMAQsQTQALIgVFDQMgCSAJKAJQIAVyNgJQIApBAmoMBAsgBSIKIAhHDQALCxD+AQALQQwQXkECEHBB0KkDQdIAEAIACyEKDAQLIAAoAgxB8AdxIQQMAQsgACADQQJqIAogB0EwahC9BCEDIAAoAgxB8AdxIQQgBygCNCAHLQA7IgUgBcBBAEgbDQELAkACQCAEQcAARg0AIARBgARGDQAgAy0AACEODAELIAMtAAAiDkHcAEcNACADQQFqIQMgBEGABEYEQCAAIAMgCiAHQTBqIAkQvAQhA0GABCEEDAILIAAgAyAKIAdBMGoQ1wIhAwwBCyAHQTBqIA7AEFogA0EBaiEDCwJAAkACQAJAIAMgCkYNACADLQAAIgVB3QBGDQBBASEOAkAgA0EBaiAKRg0AIAVBLUcNACADLQABQd0ARg0AIAdBADYCKCAHQgA3AyACQCADQQJqIgUgCkYNACADLQABQdsARw0AIAUtAABBLkcNACAAIANBA2ogCiAHQSBqEL0EIQUMBAsCQCAEQcAARg0AIARBgARGDQAgAy0AASEDDAMLIAMtAAEiA0HcAEcNAiAEQYAERgRAIAAgBSAKIAdBIGogCRC8BCEFDAQLIAAgBSAKIAdBIGoQ1wIhBQwDCwJAAkAgBygCNCAHLQA7IgUgBcBBAEgiBRsOAgYAAQsgCSAHKAIwIAdBMGogBRssAAAQowEMBAsgCSAHKAIwIAdBMGogBRsiCiwAACAKLAABENYCDAMLQQEhDgJAAkAgBygCNCAHLQA7IgUgBcBBAEgiBRsOAgUAAQsgCSAHKAIwIAdBMGogBRssAAAQowEMAwsgCSAHKAIwIAdBMGogBRsiCiwAACAKLAABENYCDAILIAdBIGogA8AQWgsgByAHKAI4NgIYIAdBADYCOCAHKAIoIQogB0EANgIoIAcgCjYCCCAHIAcpAzA3AxAgB0IANwMwIAcpAyAhEyAHQgA3AyAgByATNwMAIAdBEGohBCMAQdAAayIDJAACQAJAAkACQAJAIAktAFoEQAJAIAktAFlFDQAgBCgCBCAELQALIgYgBsBBAEgiCBsEQEEAIQYDQCAJKAIMIgsgBCgCACAEIAhBAXEbIAZqLAAAIAsoAgAoAhQRBAAhCCAEKAIAIAQgBCwAC0EASBsgBmogCDoAACAGQQFqIgYgBCgCBCAELQALIgggCMBBAEgiCBtJDQALCyAHKAIEIActAAsiBiAGwEEASCIIG0UNAEEAIQYDQCAJKAIMIgsgBygCACAHIAhBAXEbIAZqLAAAIAsoAgAoAhQRBAAhCCAHKAIAIAcgBywAC0EASBsgBmogCDoAACAGQQFqIgYgBygCBCAHLQALIgggCMBBAEgiCBtJDQALCyAEKAIEIAQtAAsiBiAGwEEASCILGyIGQfD///8HTw0CIAQoAgAhDAJ/AkAgBkELTwRAIAZBD3JBAWoiDRAyIQggAyANQYCAgIB4cjYCMCADIAg2AiggAyAGNgIsDAELIAMgBjoAMyADQShqIgggBkUNARoLIAggDCAEIAsbIAb8CgAAIAYgCGoLQQA6AAAgA0EYaiAJKAIQIgQgAygCKCADQShqIAMtADMiBsBBAEgiCBsiCyALIAMoAiwgBiAIG2ogBCgCACgCEBEGACADLAAzQQBIBEAgAygCKBAvCyAHKAIEIActAAsiBCAEwEEASCIIGyIEQfD///8HTw0DIAcoAgAhCwJ/AkAgBEELTwRAIARBD3JBAWoiDBAyIQYgAyAMQYCAgIB4cjYCTCADIAY2AkQgAyAENgJIDAELIAMgBDoATyADQcQAaiIGIARFDQEaCyAGIAsgByAIGyAE/AoAACAEIAZqC0EAOgAAIANBCGogCSgCECIEIAMoAkQgA0HEAGogAy0ATyIGwEEASCIIGyILIAsgAygCSCAGIAgbaiAEKAIAKAIQEQYAIAMsAE9BAEgEQCADKAJEEC8LIAMoAiAhBCADQQA2AiAgAyAENgIwIAMoAhAhBCADQQA2AhAgAyAENgI8IAMpAxghEyADQgA3AxggAyATNwMoIAMpAwghEyADQgA3AwggAyATNwI0AkAgCSgCMCIEIAkoAjRJBEAgBCADKQMoNwIAIAQgAygCMDYCCCADQQA2AjAgA0IANwMoIAQgAygCPDYCFCAEIAMpAjQ3AgwgA0IANwI0IANBADYCPCAJIARBGGo2AjAMAQsgCUEsaiADQShqELoEIAMsAD9BAE4NACADKAI0EC8LIAMsADNBAEgEQCADKAIoEC8LIAMsABNBAEgEQCADKAIIEC8LIAMsACNBAE4NASADKAIYEC8MAQsgBCgCBCAELQALIgYgBsAiBkEASBtBAUcNAyAHKAIEIActAAsiCCAIwEEASBtBAUcNAyAJLQBZBEAgCSgCDCIIIAQoAgAgBCAGQQBIGywAACAIKAIAKAIUEQQAIQYgBCgCACAEIAQsAAtBAEgbIAY6AAAgCSgCDCIGIAcoAgAgByAHLAALQQBIGywAACAGKAIAKAIUEQQAIQYgBygCACAHIAcsAAtBAEgbIAY6AAALIAMgBCgCCDYCMCADIAQpAgA3AyggBEIANwIAIARBADYCCCADIAcoAgg2AjwgAyAHKQIANwI0IAdCADcCACAHQQA2AggCQCAJKAIwIgQgCSgCNEkEQCAEIAMpAyg3AgAgBCADKAIwNgIIIANBADYCMCADQgA3AyggBCADKAI8NgIUIAQgAykCNDcCDCADQgA3AjQgA0EANgI8IAkgBEEYajYCMAwBCyAJQSxqIANBKGoQugQgAywAP0EATg0AIAMoAjQQLwsgAywAM0EATg0AIAMoAigQLwsgA0HQAGokAAwDCxBNAAsQTQALQQwQXkEJEHBB0KkDQdIAEAIACyAHLAALQQBIBEAgBygCABAvCyAHLAAbQQBIBEAgBygCEBAvCyAHLAArQQBIBEAgBygCIBAvCyAFIQMLCyAHLAA7QQBIBEAgBygCMBAvCyAORQ0BCyADIQoLIAdBQGskACABIAoiA0cNAAsgASACRg0BIAEtAABBLUYEQCAJQS0QowEgAUEBaiEBCyABIAJGDQEgAS0AAEHdAEcNASABQQFqIQELIAEPCxD+AQAL3A0BBX8CQAJAAkACQAJAIAEgAkYNACAAKAIMQfAHcSEGAkACQAJAAkAgASwAACIHQSprDhYDAgQEBAQEBAQEBAQEBAQEBAQEBAQBAAsgB0H7AEcNAyABQQFqIgggAkYNBCAILQAAIgdB+AFxQTBHIAdB/gFxQThHcQ0EIAdBMGshBwJAIAIgAUECaiIBRwRAA0AgAS0AACIJQfgBcUEwRyAJQf4BcUE4R3ENAiAHQcyZs+YATg0HIAdBCmwgCWpBMGshByABQQFqIgEgAkcNAAsLIAIhAQsgASAIRg0EAkAgASACRg0AIAEsAAAiCEEsRwRAIAhB/QBHDQYgAUEBaiEIAkAgBkGABEcNACACIAhGDQAgCC0AAEE/Rw0AIAAgByAHIAMgBCAFEOECDAgLQQgQMiECIAAoAiQiASgCBCEGIAJByKgBNgIAIAIgBjYCBCABQQA2AgRBJBAyIQEgACgCFCEGIAMoAgQhCSABIAI2AgggASAJNgIEIAFBAToAICABIAU2AhwgASAENgIYIAEgBjYCFCABIAc2AhAMCAsgAUEBaiIJIAJGDQUgCS0AACIIQf0ARgRAIAFBAmohCAJAIAZBgARHDQAgAiAIRg0AIAgtAABBP0cNACAAIAcgAyAEIAUQ4AIgAUEDag8LQQgQMiECIAAoAiQiASgCBCEGIAJByKgBNgIAIAIgBjYCBCABQQA2AgRBJBAyIQEgACgCFCEGIAMoAgQhCSABIAI2AgggASAJNgIEIAFBAToAICABIAU2AhwgASAENgIYIAEgBjYCFCABQX82AhAMCAsgCEH4AXFBMEcgCEH+AXFBOEdxDQAgCEEwayEIAkAgAiABQQJqIgFHBEADQCABLQAAIgpB+AFxQTBHIApB/gFxQThHcQ0CIAhBzJmz5gBODQggCEEKbCAKakEwayEIIAFBAWoiASACRw0ACwsgAiEBCyABIAlGDQAgASACRg0AIAEtAABB/QBHDQAgByAISg0FIAFBAWohCQJAIAZBgARHDQAgAiAJRg0AIAktAABBP0cNACAAIAcgCCADIAQgBRDhAgwHC0EIEDIhAiAAKAIkIgEoAgQhBiACQcioATYCACACIAY2AgQgAUEANgIEQSQQMiEBIAAoAhQhBiADKAIEIQogASACNgIIIAEgCjYCBCABQQE6ACAgASAFNgIcIAEgBDYCGCABIAY2AhQgASAINgIQIAEgBzYCDCABQdyyATYCACADQQA2AgRBCBAyIgJB2LMBNgIAIAIgATYCBCAAKAIkIAI2AgQgACABKAIINgIkIAMgATYCBCAAIAZBAWo2AhQgCQ8LEMIEAAsgAUEBaiEHAkAgBkGABEcNACACIAdGDQAgBy0AAEE/Rw0AIABBAEEBIAMgBCAFEOECDAULQQgQMiECIAAoAiQiASgCBCEGIAJByKgBNgIAIAIgBjYCBCABQQA2AgRBJBAyIQEgACgCFCEGIAMoAgQhCCABIAI2AgggASAINgIEIAFBAToAICABIAU2AhwgASAENgIYIAEgBjYCFCABQoCAgIAQNwIMDAYLIAFBAWohBwJAIAZBgARHDQAgAiAHRg0AIActAABBP0cNACAAQQEgAyAEIAUQ4AIMBAtBCBAyIQIgACgCJCIBKAIEIQYgAkHIqAE2AgAgAiAGNgIEIAFBADYCBEEkEDIhASAAKAIUIQYgAygCBCEIIAEgAjYCCCABIAg2AgQgAUEBOgAgIAEgBTYCHCABIAQ2AhggASAGNgIUIAFCgYCAgHA3AgwMBQsgAUEBaiEHAkAgBkGABEcNACACIAdGDQAgBy0AAEE/Rw0AIABBACADIAQgBRDgAgwDC0EIEDIhAiAAKAIkIgEoAgQhBiACQcioATYCACACIAY2AgQgAUEANgIEQSQQMiEBIAAoAhQhBiADKAIEIQggASACNgIIIAEgCDYCBCABQQE6ACAgASAFNgIcIAEgBDYCGCABIAY2AhQgAUKAgICAcDcCDCABQdyyATYCACADQQA2AgRBCBAyIgJB2LMBNgIAIAIgATYCBCAAKAIkIAI2AgQgACABKAIINgIkIAMgATYCBCAAIAZBAWo2AhQgByEBCyABDwsQwQQACyABQQJqDwsgASAHNgIMIAFB3LIBNgIAIANBADYCBEEIEDIiAkHYswE2AgAgAiABNgIEIAAoAiQgAjYCBCAAIAEoAgg2AiQgAyABNgIEIAAgBkEBajYCFCAIDwsgAUHcsgE2AgAgA0EANgIEQQgQMiICQdizATYCACACIAE2AgQgACgCJCACNgIEIAAgASgCCDYCJCADIAE2AgQgACAGQQFqNgIUIAcLswIBBH8gASACRgRAIAEPCyABLQAAQd4ARgRAQQwQMiEDIAAoAgwhBCAAKAIkIgUoAgQhBiADQfSpATYCACADIAY2AgQgAyAEQfAPcUGADEY6AAggBSADNgIEIAAgACgCJCgCBDYCJCABQQFqIQELAkAgASACRg0AA0AgASIDIAJGIgQNASAAKAIQIQEgACgCJCEFIAMgACADIAIQyAQiBkcEQCAAIAYgAiAFIAFBAWogACgCEEEBahDHBCIBIANHDQELCwJAIANBAWogAkcNACADLQAAQSRHDQBBDBAyIQEgACgCDCEDIAAoAiQiBCgCBCEFIAFBvKoBNgIAIAEgBTYCBCABIANB8A9xQYAMRjoACCAEIAE2AgQgACAAKAIkKAIENgIkDAELIAQNABDGBAALIAILhQQBBn8gACgCJCEHIAEhAwNAAkAgAyIEIAAgAyACEMoEIgNGBEAgACgCECEDIAAoAiQhBSAAIAQgAhDJBCIGIARGDQEgACAGIAIgBSADQQFqIAAoAhBBAWoQ4wIhAwsgAyAERw0BCwsgASAERgRAQQgQMiEBIAAoAiQiAygCBCEFIAFByKgBNgIAIAEgBTYCBCADIAE2AgQgACAAKAIkKAIENgIkCyACIARHBEADQCAELQAAQfwARwRAIAQPCyAAKAIkIQEgBEEBaiIFIQMDQAJAIAMiBCAAIAMgAhDKBCIDRgRAIAAoAhAhAyAAKAIkIQYgACAEIAIQyQQiCCAERg0BIAAgCCACIAYgA0EBaiAAKAIQQQFqEOMCIQMLIAMgBEcNAQsLIAQgBUYEQEEIEDIhAyAAKAIkIgUoAgQhBiADQcioATYCACADIAY2AgQgBSADNgIEIAAgACgCJCgCBDYCJAtBDBAyIQMgBygCBCEFIAMgASgCBDYCCCADIAU2AgQgA0GctAE2AgAgByADNgIEIAFBADYCBEEIEDIhAyAAKAIkKAIEIQUgA0HIqAE2AgAgAyAFNgIEIAEgAzYCBCAAKAIkQQA2AgRBCBAyIQMgASgCBCEFIANB3LQBNgIAIAMgBTYCBCAAKAIkIAM2AgQgACABKAIENgIkIAIgBEcNAAsLIAILnRkDF38JewZ9IwBBMGsiCCQAAkAgACgCBCAAKAIAIgJrQQF1IgYgASgCBCABKAIAIgdrQQJ1IgNLBEAgASAGIANrEG4gACgCACECDAELIAMgBk0NACABIAcgBkECdGo2AgQLAkAgACgCBCACayIJQQRGBEAgAioCACEiIAEoAgAiAEEANgIEIAAgIjgCAAwBCyAJQQJ1Ig1BgYCAgHhxQQFGBEACQCAJQQF1IgIgASgCBCABKAIAIgNrQQJ1IgZLBEAgASACIAZrEG4MAQsgAiAGTw0AIAEgAyACQQJ0ajYCBAtBkAMgDW4hBiAJQQBMDQEgASgCACEBIAAoAgAhAEEAIQMDQCADIAZsIQdDAAAAACEiQQAhAkMAAAAAISMDQCAiIAAgAkECdGoqAgAiJCACIAdsQZADb0ECdCIEQYDYNGoqAgCUkyEiICQgBEHA5DRqKgIAlCAjkiEjIAJBAWoiAiANRw0ACyABIANBA3RqIgIgIzgCACACICI4AgQgA0EBaiIDIA1HDQALDAELQQAhAiAIQQA2AiwgCEIANwIkIAhBADYCHCANQQJtIRQCQAJAAn8gDUEBakEDSQRAQQAhBkEADAELIBRBgICAgARPDQEgCCAUQQJ0IgMQMiIGNgIoIAMgBmohDEEAIA1BAWpBA0kNABogCCADEDIiAjYCHCACIANqCyELIAlBAEwEQCAGIQcgAiEDDAILQQEgDSANQQFMGyEVIAIhAyAGIQcDQCAAKAIAIA5BAnRqIQoCQAJAAkACQAJAIA5BAXFFBEAgBiAMRwRAIAYgCioCADgCACAIIAZBBGoiBjYCKAwGCyAGIAdrIgVBAnUiEEEBaiIEQYCAgIAETw0BQf////8DIAVBAXUiDCAEIAQgDEkbIAVB/P///wdPGyIMBH8gDEGAgICABE8NAyAMQQJ0EDIFQQALIhEgEEECdGoiBCAKKgIAOAIAIAxBAnQhFyAEQQRqIQoCQCAGIAdGDQACQCAFQQRrIgxB/AFJDQAgBSARaiIPQQRrIhIgBkEEayIFIAdrQXxxIhNrIBJLDQAgBSATayAFSw0AIAYgD2tBEEkNACAEQRBrIQ8gBkEQayESIAYgDEECdkEBaiITQfz///8HcSIMQQJ0IgVrIQYgBCAFayEEQQAhBQNAIA8gBUECdCIWayASIBZr/QACAP0LAgAgBUEEaiIFIAxHDQALIAwgE0YNAQsDQCAEQQRrIgQgBkEEayIGKgIAOAIAIAYgB0cNAAsLIBcgEWohDCAIIAo2AiggB0UNBCAHEC8MBAsgAiALRwRAIAIgCioCADgCACAIIAJBBGoiAjYCHAwFCyACIANrIgVBAnUiEEEBaiIEQYCAgIAETw0CQf////8DIAVBAXUiCyAEIAQgC0kbIAVB/P///wdPGyILBH8gC0GAgICABE8NAiALQQJ0EDIFQQALIhEgEEECdGoiBCAKKgIAOAIAIAtBAnQhGCAEQQRqIQoCQCACIANGDQACQCAFQQRrIgtB/AFJDQAgBSARaiIPQQRrIhIgAkEEayIFIANrQXxxIhNrIBJLDQAgBSATayAFSw0AIAIgD2tBEEkNACAEQRBrIQ8gAkEQayESIAIgC0ECdkEBaiITQfz///8HcSILQQJ0IgVrIQIgBCAFayEEQQAhBQNAIA8gBUECdCIWayASIBZr/QACAP0LAgAgBUEEaiIFIAtHDQALIAsgE0YNAQsDQCAEQQRrIgQgAkEEayICKgIAOAIAIAIgA0cNAAsLIBggEWohCyAIIAo2AhwgAwRAIAMQLwsgCiECIAQhAwwECyAIIAc2AiQgCCAGNgIsEDYACxBHAAsgCCADNgIYIAggAjYCICAIIAw2AiwgCCAHNgIkEDYACyAKIQYgBCEHCyAVIA5BAWoiDkcNAAsMAQsQNgALIAggAzYCGCAIIAs2AiAgCCAMNgIsIAggBzYCJEEAIQYgCEEANgIUIAhCADcCDCAIQQA2AgggCEIANwIAQZADIA1tIRYgCEEkaiAIQQxqEOYCIAhBGGogCBDmAiAIKAIAIQICQAJAIAlBBU4EQCABKAIAIQAgCCgCDCEBAkBBASAUIBRBAUwbIgxBIEkNACAAIBRBA3QiB2oiAyAMQQFrIgRBA3QiCmogA0kNACAWQQFHDQAgA0EEaiIDIApqIANJDQAgBEH/////AUsNACAAIAAgDEEDdCIJaiIESSAAQQRqIgogACAJQQRrIg9qIg1JcQ0AIAAgACAHIAlqaiIDSSAAIAdqIgdBBGoiBSANSXENACAAIAxBAnQiDkHA5DRqIgtJIA1BwOQ0S3ENACAAIA5BgNg0aiIOSSANQYDYNEtxDQAgAiANSSAAIAIgD2oiEUlxDQAgACACIAlqIhVJIAJBBGoiECANSXENACABIA1JIAAgASAPaiIPSXENACAAIAEgCWoiEkkgAUEEaiITIA1JcQ0AIAAgA0EEayIJSSAHIA1JcQ0AIAMgCksgBCAFS3ENACAKIAtJIARBwOQ0S3ENACAKIA5JIARBgNg0S3ENACACIARJIAogEUlxDQAgCiAVSSAEIBBLcQ0AIAogD0kgASAESXENACAKIBJJIAQgE0txDQAgCSAKSyAEIAdLcQ0AIAUgC0kgA0HA5DRLcQ0AIAUgDkkgA0GA2DRLcQ0AIAIgA0kgBSARSXENACAFIBVJIAMgEEtxDQAgBSAPSSABIANJcQ0AIAUgEkkgAyATS3ENACAFIAlJIAMgB0txDQAgByALSSAJQcDkNEtxDQAgByAOSSAJQYDYNEtxDQAgAiAJSSAHIBFJcQ0AIAcgFUkgCSAQS3ENACABIAlJIAcgD0lxDQAgByASSSAJIBNLcQ0AIAxB/P///wdxIQYgFP0RIR79DAAAAAABAAAAAgAAAAMAAAAhG0EAIQMDQCAAIBtBAf2rASIZ/RsAQQJ0IgdqIANBAnQiCUGA2DRq/QAEACIcIAIgGf0MAQAAAAEAAAABAAAAAQAAAP1QIhr9GwBBAnQiBGr9CQIAIAIgGv0bAUECdCIKaioCAP0gASACIBr9GwJBAnQiDWoqAgD9IAIgAiAa/RsDQQJ0IgVqKgIA/SADIhr95gEiHyAJQcDkNGr9AAQAIh0gAiAHav0JAgAgAiAZ/RsBQQJ0IglqKgIA/SABIAIgGf0bAkECdCILaioCAP0gAiACIBn9GwNBAnQiDmoqAgD9IAMiIP3mASIhIAEgB2oiB/0JAgAgASAJaiIRKgIA/SABIAEgC2oiFSoCAP0gAiABIA5qIhAqAgD9IAP95AH95AEiGf0fADgCACAAIAlqIBn9HwE4AgAgACALaiAZ/R8COAIAIAAgDmogGf0fAzgCACAAIARqIB0gGv3mASIdIAEgBGoiBP0JAgAgASAKaiIJKgIA/SABIAEgDWoiCyoCAP0gAiABIAVqIg4qAgD9IAP95AEgHCAg/eYBIhz95QEiGf0fADgCACAAIApqIBn9HwE4AgAgACANaiAZ/R8COAIAIAAgBWogGf0fAzgCACAAIBsgHv2uAUEB/asBIhn9GwBBAnRqIAf9CQIAIBEqAgD9IAEgFSoCAP0gAiAQKgIA/SADICH95QEgH/3lASIa/R8AOAIAIAAgGf0bAUECdGogGv0fATgCACAAIBn9GwJBAnRqIBr9HwI4AgAgACAZ/RsDQQJ0aiAa/R8DOAIAIAAgGf0MAQAAAAEAAAABAAAAAQAAAP1QIhn9GwBBAnRqIBwgBP0JAgAgCSoCAP0gASALKgIA/SACIA4qAgD9IAMgHf3lAf3kASIa/R8AOAIAIAAgGf0bAUECdGogGv0fATgCACAAIBn9GwJBAnRqIBr9HwI4AgAgACAZ/RsDQQJ0aiAa/R8DOAIAIBv9DAQAAAAEAAAABAAAAAQAAAD9rgEhGyADQQRqIgMgBkcNAAsgBiAMRg0CCwNAIAAgBkEDdCIDaiAGIBZsQQJ0IgRBgNg0aioCACIiIAIgA0EEciIHaioCACIjlCIkIARBwOQ0aioCACIlIAIgA2oqAgAiJpQiJyABIANqIgMqAgCSkjgCACAAIAdqICUgI5QiIyABIAdqIgcqAgCSICIgJpQiIpM4AgAgACAGIBRqQQN0aiIEIAMqAgAgJ5MgJJM4AgAgBCAiIAcqAgAgI5OSOAIEIAZBAWoiBiAMRw0ACwwBCyACRQ0BCyAIIAI2AgQgAhAvCyAIKAIMIgAEQCAIIAA2AhAgABAvCyAIKAIYIgAEQCAAEC8LIAgoAiQiAEUNACAIIAA2AiggABAvCyAIQTBqJAAL/AMBBH8CQAJAAkAgACgCBCAAKAIAIgJrQTBtIgVBAWoiA0HWqtUqSQRAQdWq1SogACgCCCACa0EwbSICQQF0IgQgAyADIARJGyACQarVqhVPGyIDQdaq1SpPDQEgA0EwbCIDEDIiBCAFQTBsaiICIAH9AAMA/QsDACACIAEoAhg2AhggAiABKQMQNwMQIAFBADYCGCABQgA3AxAgAiABKAIcNgIcIAIgASgCIDYCICACIAEoAiQ2AiQgAUEANgIkIAFCADcCHCACIAEtACg6ACggAyAEaiEDIAJBMGohBSAAKAIEIgEgACgCACIERg0CA0AgAkEwayICIAFBMGsiAf0AAwD9CwMAIAIgASgCGDYCGCACIAEpAxA3AxAgAUEANgIYIAFCADcDECACIAEoAhw2AhwgAiABKAIgNgIgIAIgASgCJDYCJCABQQA2AiQgAUIANwIcIAIgAS0AKDoAKCABIARHDQALIAAgAzYCCCAAKAIEIQMgACAFNgIEIAAoAgAhASAAIAI2AgAgASADRg0DA0AgA0EwayIAKAIcIgIEQCADQRBrIAI2AgAgAhAvCyADQRVrLAAAQQBIBEAgA0EgaygCABAvCyAAIgMgAUcNAAsMAwsQNgALEEcACyAAIAM2AgggACAFNgIEIAAgAjYCAAsgAQRAIAEQLwsLHwAgAQRAIAAgASgCABDoAiAAIAEoAgQQ6AIgARAvCwv/KwMVfwJ8AX4jAEHwAGsiCCQAA0AgAUHIAGshCiABQegAayEJA0AgACEHA0ACQAJAAkACQAJAAkACQAJAIAEgB2siDUHoAG0iEA4GBwcAAQQCAwsgAUHIAGsrAwAgBysDIGRFDQYgByABQegAaxA+DAYLIAdB6ABqIQIgAUHoAGsiACsDICEZIAcrA4gBIhggBysDIGRFBEAgGCAZY0UNBiACIAAQPiAHKwOIASAHKwMgZEUNBiAHIAIQPgwGCyAYIBljBEAgByAAED4MBgsgByACED4gACsDICAHKwOIAWRFDQUgAiAAED4MBQsgByAHQegAaiAHQdABaiAHQbgCaiABQegAaxDUAhoMBAsgDUHXBUwEQCAHQegAaiECIAdB0AFqIQAgBysD8AEhGQJAIAcrA4gBIhggBysDIGRFBEAgGCAZY0UNASACIAAQPiAHKwOIASAHKwMgZEUNASAHIAIQPgwBCyAYIBljBEAgByAAED4MAQsgByACED4gBysD8AEgBysDiAFkRQ0AIAIgABA+CyAHQbgCaiIDIAFGDQQDQCADKwMgIhggACsDIGQEQCAIIAMtAAg6AEggCCADKQMANwNAIAMoAhQhESADKAIQIRIgA0IANwMQIAMoAhghEyADQQA2AhggAygCHCEPIAggA/0AAjj9CwMQIAggA/0AAij9CwMAIAMoAlwhFCADKAJYIRUgA0IANwNYIAMoAkwhDiADKAJQIQogAygCVCEJIAMoAkghCyAD/QwAAAAAAAAAAAAAAAAAAAAA/QsDSCADKQNgIRogAyEFA0ACQCAFIAAiAikDADcDACAFIAItAAg6AAggBSgCECIABEAgBSAANgIUIAAQLyAFQQA2AhgLIAUgAigCEDYCECAFIAIoAhQ2AhQgBSACKAIYNgIYIAJBADYCGCACQgA3AxAgBSACKAJENgJEIAUgAikCPDcCPCAFIAL9AAIs/QsCLCAFIAL9AAIc/QsCHCAFKAJIIgQEQCAFKAJMIgYgBCIARwRAA0AgBkEMayIAKAIAIgwEQCAGQQhrIAw2AgAgDBAvCyAAIgYgBEcNAAsgBSgCSCEACyAFIAQ2AkwgABAvIAVBADYCUAsgBSACKAJINgJIIAUgAigCTDYCTCAFIAIoAlA2AlAgAkEANgJQIAJCADcCSCAFKAJUIgQEQCAFKAJYIgYgBCIARwRAA0AgBkEMayIAKAIAIgwEQCAGQQhrIAw2AgAgDBAvCyAAIgYgBEcNAAsgBSgCVCEACyAFIAQ2AlggABAvIAVBADYCXAsgBSACKAJUNgJUIAUgAigCWDYCWCAFIAIoAlw2AlwgAkEANgJcIAJCADcCVCAFIAIpAmA3AmAgAiAHRgRAIAchBQwBCyAYIAIiBUHoAGsiACsDIGQNAQsLIAUgCCkDQDcDACAFIAgtAEg6AAggBSgCECIABEAgBSAANgIUIAAQLwsgBSATNgIYIAUgETYCFCAFIBI2AhAgAiAPNgIcIAUgGDkCICAFIAj9AAMQ/QsCOCAFIAj9AAMA/QsCKCACKAJIIgQEQCAFKAJMIgYgBCIARwRAA0AgBkEMayIAKAIAIg8EQCAGQQhrIA82AgAgDxAvCyAAIgYgBEcNAAsgAigCSCEACyAFIAQ2AkwgABAvCyACIAs2AkggBSAKNgJQIAUgDjYCTCAFKAJUIgIEQCAFKAJYIgYgAiIARwRAA0AgBkEMayIAKAIAIgQEQCAGQQhrIAQ2AgAgBBAvCyAAIgYgAkcNAAsgBSgCVCEACyAFIAI2AlggABAvCyAFIBo3AmAgBSAUNgJcIAUgFTYCWCAFIAk2AlQLIAMiAEHoAGoiAiEDIAEgAkcNAAsMBAsgAkUEQCABIAdGDQQgEEECa0EBdiIMIQADQAJAIAwgACIESA0AIAcgAEHoAGxqIQAgByAEQQF0IgZBAXIiAkHoAGxqIQMCQCAQIAZBAmoiBUwEQCACIQUMAQsgAysDICADKwOIAWRFBEAgAiEFDAELIANB6ABqIQMLIAArAyAiGCADKwMgYw0AIAggAC0ACDoASCAIIAApAwA3A0AgACgCGCERIABBADYCGCAAKAIQIRIgACgCFCETIABCADcDECAAKAIcIQ8gCCAA/QACOP0LAxAgCCAA/QACKP0LAwAgACgCVCEUIAAoAlAhFSAAKAJMIQ4gACgCSCEKIAD9DAAAAAAAAAAAAAAAAAAAAAD9CwNIIAAoAlwhCSAAKAJYIQsgAEIANwNYIAApA2AhGgNAIAAiAiADIgApAwA3AwAgAiAALQAIOgAIIAIoAhAiAwRAIAIgAzYCFCADEC8gAkEANgIYIAJCADcDEAsgAiAAKAIQNgIQIAIgACgCFDYCFCACIAAoAhg2AhggAEEANgIYIABCADcDECACIAAoAkQ2AkQgAiAAKQI8NwI8IAIgAP0AAiz9CwIsIAIgAP0AAhz9CwIcIAJByABqIABByABqEIcCIAUgDEwEQCAHIAVBAXQiBkEBciICQegAbGohAwJAIBAgBkECaiIFTARAIAIhBQwBCyADKwMgIAMrA4gBZEUEQCACIQUMAQsgA0HoAGohAwsgAysDICAYZEUNAQsLIAAgCCkDQDcDACAAIAgtAEg6AAggACgCECICBEAgACACNgIUIAIQLwsgACAPNgIcIAAgETYCGCAAIBM2AhQgACASNgIQIAAgGDkCICAAIAj9AAMA/QsCKCAAIAj9AAMQ/QsCOCAAKAJIIgUEQCAAKAJMIgMgBSIGRwRAA0AgA0EMayICKAIAIgYEQCADQQhrIAY2AgAgBhAvCyACIgMgBUcNAAsgACgCSCEGCyAAIAU2AkwgBhAvCyAAIAo2AkggACAVNgJQIAAgDjYCTCAAKAJUIgUEQCAAKAJYIgMgBSIGRwRAA0AgA0EMayICKAIAIgYEQCADQQhrIAY2AgAgBhAvCyACIgMgBUcNAAsgACgCVCEGCyAAIAU2AlggBhAvCyAAIBQ2AlQgACAaNwJgIAAgCTYCXCAAIAs2AlgLIARBAWshACAEDQALIA1B6ABuIQADQCABIQkgCCAHLQAIOgA4IAggBykDADcDMCAHKAIQIRAgBygCFCENIAdCADcDECAHKAIYIQwgB0EANgIYIAggBygCRDYCKCAIIAcpAjw3AyAgCCAH/QACLP0LAxAgCCAH/QACHP0LAwAgBygCSCERIAcoAkwhEiAHKAJQIRMgBygCVCEPIAf9DAAAAAAAAAAAAAAAAAAAAAD9CwNIIAcoAlghFCAHKAJcIRUgB0IANwNYIAAiC0ECa0EBdiEKIAcpA2AhGiAHIQNBACEBA0AgAUEBdCICQQFyIQAgAUHoAGwgA2pB6ABqIQQCQCALIAJBAmoiAUwEQCAAIQEMAQsgBCsDICAEKwOIAWRFBEAgACEBDAELIARB6ABqIQQLIAMgBCkDADcDACADIAQtAAg6AAggAygCECIABEAgAyAANgIUIAAQLyADQQA2AhggA0IANwMQCyADIAQoAhA2AhAgAyAEKAIUNgIUIAMgBCgCGDYCGCAEQQA2AhggBEIANwMQIAMgBCgCRDYCRCADIAQpAjw3AjwgAyAE/QACLP0LAiwgAyAE/QACHP0LAhwgBEHIAGohFiADKAJIIgIEQCADKAJMIgYgAiIARwRAA0AgBkEMayIAKAIAIgUEQCAGQQhrIAU2AgAgBRAvCyAAIgYgAkcNAAsgAygCSCEACyADIAI2AkwgABAvIANBADYCUCADQgA3AkgLIAMgFigCADYCSCADIAQiAigCTDYCTCADIAIoAlA2AlAgAkEANgJQIBZCADcCACADKAJUIgUEQCADKAJYIgYgBSIARwRAA0AgBkEMayIAKAIAIg4EQCAGQQhrIA42AgAgDhAvCyAAIgYgBUcNAAsgAygCVCEACyADIAU2AlggABAvIANBADYCXCADQgA3AlQLIAMgAigCVDYCVCADIAIoAlg2AlggAyACKAJcNgJcIAJBADYCXCACQgA3AlQgAyACKQJgNwJgIAIhAyABIApMDQALAkAgCUHoAGsiASAERgRAIAQgCCkDMDcDACAEIAgtADg6AAggBCgCECIABEAgBCAANgIUIAAQLwsgBCAMNgIYIAQgDTYCFCAEIBA2AhAgBCAIKAIoNgJEIAQgCCkDIDcCPCAEIAj9AAMQ/QsCLCAEIAj9AAMA/QsCHCAWKAIAIgYEQCACKAJMIgQgBiIDRwRAA0AgBEEMayIAKAIAIgMEQCAEQQhrIAM2AgAgAxAvCyAAIgQgBkcNAAsgFigCACEDCyACIAY2AkwgAxAvCyACIBM2AlAgAiASNgJMIBYgETYCACACKAJUIgYEQCACKAJYIgQgBiIDRwRAA0AgBEEMayIAKAIAIgMEQCAEQQhrIAM2AgAgAxAvCyAAIgQgBkcNAAsgAigCVCEDCyACIAY2AlggAxAvCyACIBo3AmAgAiAVNgJcIAIgFDYCWCACIA82AlQMAQsgBCABKQMANwMAIAQgAS0ACDoACCAEKAIQIgAEQCAEIAA2AhQgABAvIARBADYCGCAEQgA3AxALIAQgASgCEDYCECAEIAEoAhQ2AhQgBCABKAIYNgIYIAFBADYCGCABQgA3AxAgBCABKAJENgJEIAQgASkCPDcCPCAEIAH9AAIs/QsCLCAEIAH9AAIc/QsCHCAWIAlBIGsiDhCHAiABIAgtADg6AAggASAIKQMwNwMAIAEoAhAiAARAIAEgADYCFCAAEC8LIAEgEDYCECABIA02AhQgASAMNgIYIAEgCCgCKDYCRCABIAgpAyA3AjwgASAI/QADEP0LAiwgASAI/QADAP0LAhwgDigCACIGBEAgCUEcayIKKAIAIgUgBiIDRwRAA0AgBUEMayIAKAIAIgMEQCAFQQhrIAM2AgAgAxAvCyAAIgUgBkcNAAsgDigCACEDCyAKIAY2AgAgAxAvCyAOIBE2AgAgCUHoAGsiCiATNgJQIAogEjYCTCAKKAJUIgYEQCAJQRBrIgkoAgAiBSAGIgNHBEADQCAFQQxrIgAoAgAiAwRAIAVBCGsgAzYCACADEC8LIAAiBSAGRw0ACyAKKAJUIQMLIAkgBjYCACADEC8LIAogDzYCVCAKIBo3AmAgCiAVNgJcIAogFDYCWCAEQegAaiAHayIAQekASA0AIAQrAyAiGCAHIABB6ABuQQJrQQF2IgVB6ABsaiIDKwMgY0UNACAIIAQtAAg6AGggCCAEKQMANwNgIAQoAhAhDCAEKAIUIREgBEIANwMQIAQoAhghEiAEQQA2AhggBCgCHCETIAggBP0AAjj9CwNQIAggBP0AAij9CwNAIAIoAlwhDyACKAJYIRQgAkIANwNYIBYoAgAhFSACKAJMIQ4gAigCUCEKIAIoAlQhCSAW/QwAAAAAAAAAAAAAAAAAAAAA/QsDACACKQNgIRoDQCAEIAMiAikDADcDACAEIAItAAg6AAggBCgCECIABEAgBCAANgIUIAAQLyAEQQA2AhggBEIANwMQCyAEIAIoAhA2AhAgBCACKAIUNgIUIAQgAigCGDYCGCACQQA2AhggAkIANwMQIAQgAigCRDYCRCAEIAIpAjw3AjwgBCAC/QACLP0LAiwgBCAC/QACHP0LAhwgBCgCSCIDBEAgBCgCTCIGIAMiAEcEQANAIAZBDGsiACgCACINBEAgBkEIayANNgIAIA0QLwsgACIGIANHDQALIAQoAkghAAsgBCADNgJMIAAQLyAEQQA2AlAgBEIANwJICyAEIAIoAkg2AkggBCACKAJMNgJMIAQgAigCUDYCUCACQQA2AlAgAkIANwJIIAQoAlQiAwRAIAQoAlgiBiADIgBHBEADQCAGQQxrIgAoAgAiDQRAIAZBCGsgDTYCACANEC8LIAAiBiADRw0ACyAEKAJUIQALIAQgAzYCWCAAEC8gBEEANgJcIARCADcCVAsgBCACKAJUNgJUIAQgAigCWDYCWCAEIAIoAlw2AlwgAkEANgJcIAJCADcCVCAEIAIpAmA3AmAgBQRAIAIhBCAHIAVBAWtBAXYiBUHoAGxqIgMrAyAgGGQNAQsLIAIgCCkDYDcDACACIAgtAGg6AAggAigCECIABEAgAiAANgIUIAAQLwsgAiATNgIcIAIgEjYCGCACIBE2AhQgAiAMNgIQIAIgGDkCICACIAj9AANA/QsCKCACIAj9AANQ/QsCOCACKAJIIgMEQCACKAJMIgYgAyIARwRAA0AgBkEMayIAKAIAIgUEQCAGQQhrIAU2AgAgBRAvCyAAIgYgA0cNAAsgAigCSCEACyACIAM2AkwgABAvCyACIAo2AlAgAiAONgJMIAIgFTYCSCACKAJUIgMEQCACKAJYIgYgAyIARwRAA0AgBkEMayIAKAIAIgUEQCAGQQhrIAU2AgAgBRAvCyAAIgYgA0cNAAsgAigCVCEACyACIAM2AlggABAvCyACIBo3AmAgAiAPNgJcIAIgFDYCWCACIAk2AlQLIAtBAWshACALQQJKDQALDAQLIAcgEEEBdkHoAGxqIQUCfyANQdmrBk8EQCAHIAcgEEECdkHoAGwiAGogBSAAIAVqIAkQ1AIMAQsgCisDACEZAkAgBSsDICIYIAcrAyBkRQRAQQAgGCAZY0UNAhogBSAJED5BASAFKwMgIAcrAyBkRQ0CGiAHIAUQPgwBCyAYIBljBEAgByAJED5BAQwCCyAHIAUQPkEBIAorAwAgBSsDIGRFDQEaIAUgCRA+C0ECCyEEIAJBAWshAiAJIQYCQCAHIgsrAyAiGSAFKwMgIhhkBEAgBiEADAELA0AgBkHoAGsiACALRgRAIAtB6ABqIQYgGSAKKwMAZA0FIAYgCUYNBgNAIAYrAyAgGWMEQCAGIAkQPiAGQegAaiEGDAcLIAZB6ABqIgYgCUcNAAsMBgsgBkHIAGshFyAAIQYgFysDACAYZEUNAAsgCyAAED4gBEEBaiEECyALQegAaiIDIABPDQEDQCAFKwMgIRgDQCADIgZB6ABqIQMgBisDICAYZA0ACwNAIABB6ABrIgArAyAgGGRFDQALIAAgBkkEQCAGIQMMAwUgBiAAED4gACAFIAUgBkYbIQUgBEEBaiEEDAELAAsACyAHIAdB6ABqIAdB0AFqIAFB6ABrENMCGgwCCwJAIAMgBUYNACAFKwMgIAMrAyBkRQ0AIAMgBRA+IARBAWohBAsgBEUEQCALIAMQsQQhBiADQegAaiIAIAEQsQQEQCALIQAgAyEBIAZFDQYMAwsgBg0ECyADIAtrQegAbSABIANrQegAbUgEQCALIAMgAhDpAiADQegAaiEADAQLIANB6ABqIAEgAhDpAiALIQAgAyEBDAQLIAkiACAGRg0AA0AgCysDICEYA0AgBiIHQegAaiEGIBggBysDIGRFDQALA0AgGCAAQegAayIAKwMgZA0ACyAAIAdNDQIgByAAED4MAAsACwsLCyAIQfAAaiQAC6YHAQx/IAAoAgAiBiABIAZrIgxBfHFqIQ0CQAJAAkACQAJAAkAgAyACayIIQQBMDQAgCEECdSIKIAAoAggiByAAKAIEIgtrQQJ1TARAAkAgCyANayIIQQJ1IgEgCk4EQCALIQQgAyEHDAELIAshBAJAIAIgAUECdGoiByADRg0AIAchAQJAIAMgAiAIaiIFa0EEayIJQRxJDQAgBCAFa0EQSQ0AIAQgCUECdkEBaiIOQfz///8HcSIJQQJ0IgVqIQQgASAFaiEBQQAhBQNAIAsgBUECdCIPaiAHIA9q/QACAP0LAgAgBUEEaiIFIAlHDQALIAkgDkYNAQsDQCAEIAEoAgA2AgAgBEEEaiEEIAFBBGoiASADRw0ACwsgACAENgIEIAhBAEwNAgsgBCANIApBAnQiA2prIghBAnUhCSAEIQEgBCADayIKIAtPDQYgBkF/cyALIAxBfHEiASAIaiIDIAZqQQRqIgUgBSALSRtqIANrIgNBLEkNBCAEIAEgBmogCGprQRBJDQQgBCADQQJ2QQFqIgxB/P///wdxIgZBAnQiA2ohASADIApqIQVBACEDA0AgBCADQQJ0Ig5qIAogDmr9AAIA/QsCACADQQRqIgMgBkcNAAsgBiAMRg0GDAULIAsgBmtBAnUgCmoiBEGAgICABE8NAUH/////AyAHIAZrIgdBAXUiBSAEIAQgBUkbIAdB/P///wdPGyIFBH8gBUGAgICABE8NAyAFQQJ0EDIFQQALIgkgDEF8cWoiBCEHIAIgA0cEQCAEIAIgCEF8cfwKAAAgBCAKQQJ0aiEHCyAFQQJ0IAlqIQUCQCABIAZGDQAgDSEBAkAgDEEEayICQTxJDQAgDEF8cSIDIAZqIAMgCWprQRBJDQAgBEEQayEKIAFBEGshDCABIAJBAnZBAWoiCEH8////B3EiAkECdCIDayEBIAQgA2shBEEAIQMDQCAKIANBAnQiCWsgDCAJa/0AAgD9CwIAIANBBGoiAyACRw0ACyACIAhGDQELA0AgBEEEayIEIAFBBGsiASgCADYCACABIAZHDQALCyAHIA0gCyANayIB/AoAACAAIAU2AgggACABIAdqNgIEIAAgBDYCACAGBEAgBhAvCwsPCxA2AAsQRwALIAohBSAEIQELA0AgASAFKAIANgIAIAFBBGohASAFQQRqIgUgC0kNAAsLIAAgATYCBCAEIAlBAnRrIA0gCPwKAAAgDSACIAcgAmv8CgAAC+kaAgl/A3wjAEEQayIDJAADQCABQRBrIQcDQCAAIQQDQAJAAkACQAJAAkACQAJAAkAgASAEayIIQQR1IgkOBgcHAAEEAgMLIAFBEGsiACsDACAEKwMAZEUNBiADIAT9AAMA/QsDACAEIAAoAgg2AgggBCAAKQMANwMAIAAgAygCCDYCCCAAIAMpAwA3AwAMBgsgBEEQaiEAIAFBEGsiASsDACEMIAQrAxAiDSAEKwMAZEUEQCAMIA1kRQ0GIAMgAP0AAwD9CwMAIAAgASgCCDYCCCAAIAEpAwA3AwAgASADKAIINgIIIAEgAykDADcDACAEKwMQIAQrAwBkRQ0GIAMgBP0AAwD9CwMAIAQgACgCCDYCCCAEIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDAAwGCyAMIA1kBEAgAyAE/QADAP0LAwAgBCABKAIINgIIIAQgASkDADcDACABIAMoAgg2AgggASADKQMANwMADAYLIAMgBP0AAwD9CwMAIAQgACgCCDYCCCAEIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDACABKwMAIAQrAxBkRQ0FIAMgAP0AAwD9CwMAIAAgASgCCDYCCCAAIAEpAwA3AwAgASADKAIINgIIIAEgAykDADcDAAwFCyAEIARBEGoiAiAEQSBqIgAgBEEwaiIFENgBGiABQRBrIgErAwAgBCsDMGRFDQQgAyAF/QADAP0LAwAgBSABKAIINgIIIAUgASkDADcDACABIAMoAgg2AgggASADKQMANwMAIAUrAwAgACsDAGRFDQQgAyAA/QADAP0LAwAgACAFKAIINgIIIAAgBSkDADcDACAFIAMoAgg2AgggBSADKQMANwMAIAArAwAgAisDAGRFDQQgAyAC/QADAP0LAwAgAiAAKAIINgIIIAIgACkDADcDACAAIAMoAgg2AgggACADKQMANwMAIAQrAxAgBCsDAGRFDQQgAyAE/QADAP0LAwAgBCACKAIINgIIIAQgAikDADcDACACIAMoAgg2AgggAiADKQMANwMADAQLIAhB7wNMBEAgBEEQaiEAIARBIGohBSAEKwMgIQwCQCAEKwMQIg0gBCsDACIOZEUEQCAMIA1kRQ0BIAMgAP0AAwD9CwMAIAAgBUEIaigCADYCCCAAIAUpAwA3AwAgBSADKAIINgIIIAUgAykDADcDACAAKwMAIA5kRQ0BIAMgBP0AAwD9CwMAIAQgACgCCDYCCCAEIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDAAwBCyAMIA1kBEAgAyAE/QADAP0LAwAgBCAFQQhqKAIANgIIIAQgBSkDADcDACAFIAMoAgg2AgggBSADKQMANwMADAELIAMgBP0AAwD9CwMAIAQgACgCCDYCCCAEIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDACAMIAQrAxBkRQ0AIAMgAP0AAwD9CwMAIAAgBUEIaigCADYCCCAAIAUpAwA3AwAgBSADKAIINgIIIAUgAykDADcDAAsgBEEwaiIGIAFGDQQDQCAGKwMAIgwgBSsDAGQEQCAGKAIIIQcgBiECA0ACQCACIAUiACkDADcDACACIAAoAgg2AgggACAERgRAIAQhAAwBCyAAIQIgDCAAQRBrIgUrAwBkDQELCyAAIAc2AgggACAMOQMACyAGIgVBEGoiACEGIAAgAUcNAAsMBAsgAkUEQCABIARGDQQgCUECa0EBdiIKIQADQAJAIAogACIHSA0AIAQgAEEEdGohBSAEIABBAXQiAkEBciIGQQR0aiEAAkAgCSACQQJqIgJMBEAgBiECDAELIAArAwAgACsDEGRFBEAgBiECDAELIABBEGohAAsgBSsDACIMIAArAwBjDQAgBSgCCCELA0ACQCAFIgYgACIFKQMANwMAIAYgACgCCDYCCCACIApKDQAgBCACQQF0IgJBAXIiBkEEdGohAAJAIAkgAkECaiICTARAIAYhAgwBCyAAKwMAIAArAxBkRQRAIAYhAgwBCyAAQRBqIQALIAArAwAgDGRFDQELCyAFIAs2AgggBSAMOQMACyAHQQFrIQAgB0EASg0ACyAIQQR2IQADQCADIAT9AAMA/QsDACAAIgdBAmtBAXYhCUEAIQIgBCEAA0AgAkEBdCIIQQFyIQUgACIGIAJBBHRqQRBqIQACQCAHIAhBAmoiAkwEQCAFIQIMAQsgACsDACAAKwMQZEUEQCAFIQIMAQsgAEEQaiEACyAGIAApAwA3AwAgBiAAQQhqKAIANgIIIAIgCUwNAAsCQCABQRBrIgEgAEYEQCAAIAMpAwA3AwAgACADKAIINgIIDAELIAAgASkDADcDACAAIAFBCGooAgA2AgggASADKQMANwMAIAEgAygCCDYCCCAAIARrQRBqIgJBEUgNACAAKwMAIgwgBCACQQR2QQJrQQF2IgZBBHRqIggrAwBjRQ0AIAAoAgghBQNAAkAgACAIIgIpAwA3AwAgACACKAIINgIIIAZFDQAgAiEAIAQgBkEBa0EBdiIGQQR0aiIIKwMAIAxkDQELCyACIAU2AgggAiAMOQMACyAHQQFrIQAgB0ECSg0ACwwECyAEIAlBA3RBcHFqIQYCQCAIQfH8AE8EQCAEIAQgCUECdEFwcSIFaiIAIAYgBSAGaiIFENgBIQggBysDACAFKwMAZEUNASADIAX9AAMA/QsDACAFIAcoAgg2AgggBSAHKQMANwMAIAcgAygCCDYCCCAHIAMpAwA3AwAgBSsDACAGKwMAZEUEQCAIQQFqIQgMAgsgAyAG/QADAP0LAwAgBkEIaiAFKAIINgIAIAYgBSkDADcDACAFIAMoAgg2AgggBSADKQMANwMAIAYrAwAgACsDAGRFBEAgCEECaiEIDAILIAMgAP0AAwD9CwMAIAAgBigCCDYCCCAAIAYpAwA3AwAgBiADKAIINgIIIAYgAykDADcDACAAKwMAIAQrAwBkRQRAIAhBA2ohCAwCCyADIAT9AAMA/QsDACAEIAAoAgg2AgggBCAAKQMANwMAIAAgAygCCDYCCCAAIAMpAwA3AwAgCEEEaiEIDAELIAcrAwAhDAJAIAYrAwAiDSAEKwMAZEUEQEEAIQggDCANZEUNAiADIAb9AAMA/QsDACAGQQhqIAcoAgg2AgAgBiAHKQMANwMAIAcgAygCCDYCCCAHIAMpAwA3AwBBASEIIAYrAwAgBCsDAGRFDQIgAyAE/QADAP0LAwAgBCAGKAIINgIIIAQgBikDADcDACAGIAMoAgg2AgggBiADKQMANwMADAELIAwgDWQEQCADIAT9AAMA/QsDACAEIAcoAgg2AgggBCAHKQMANwMAIAcgAygCCDYCCCAHIAMpAwA3AwBBASEIDAILIAMgBP0AAwD9CwMAIAQgBkEIaigCADYCCCAEIAYpAwA3AwAgBiADKAIINgIIIAYgAykDADcDAEEBIQggBysDACAGKwMAZEUNASADIAb9AAMA/QsDACAGIAcoAgg2AgggBiAHKQMANwMAIAcgAygCCDYCCCAHIAMpAwA3AwALQQIhCAsgAkEBayECIAchACAEKwMAIgwgBisDACINZEUEQANAIABBEGsiACAERgRAIARBEGohBSAMIAcrAwBkDQUgBSAHRg0GA0AgBSsDACAMYwRAIAMgBf0AAwD9CwMAIAUgBygCCDYCCCAFIAcpAwA3AwAgByADKAIINgIIIAcgAykDADcDACAFQRBqIQUMBwsgBUEQaiIFIAdHDQALDAYLIAArAwAgDWRFDQALIAMgBP0AAwD9CwMAIAQgAEEIaigCADYCCCAEIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDACAIQQFqIQgLIARBEGoiBSAATw0BA0AgBisDACEMA0AgBSIJQRBqIQUgCSsDACAMZA0ACwNAIABBEGsiACsDACAMZEUNAAsgACAJSQRAIAkhBQwDBSADIAn9AAMA/QsDACAJIABBCGooAgA2AgggCSAAKQMANwMAIAAgAygCCDYCCCAAIAMpAwA3AwAgACAGIAYgCUYbIQYgCEEBaiEIDAELAAsACyAEIARBEGogBEEgaiABQRBrENgBGgwCCwJAIAUgBkYNACAGKwMAIAUrAwBkRQ0AIAMgBf0AAwD9CwMAIAUgBkEIaigCADYCCCAFIAYpAwA3AwAgBiADKAIINgIIIAYgAykDADcDACAIQQFqIQgLIAhFBEAgBCAFELIEIQYgBUEQaiIAIAEQsgQEQCAEIQAgBSEBIAZFDQYMAwsgBg0ECyAFIARrIAEgBWtIBEAgBCAFIAIQ6wIgBUEQaiEADAQLIAVBEGogASACEOsCIAQhACAFIQEMBAsgByIAIAVGDQADQCAEKwMAIQwDQCAFIgZBEGohBSAMIAYrAwBkRQ0ACwNAIAwgAEEQayIAKwMAZA0ACyAAIAZNBEAgBiEEDAMFIAMgBv0AAwD9CwMAIAYgAEEIaigCADYCCCAGIAApAwA3AwAgACADKAIINgIIIAAgAykDADcDAAwBCwALAAsLCwsgA0EQaiQACwkAQcEdELcBAAuSCQEEf0EEEDIiBEHspQE2AgBBCBAyIgNByKgBNgIAIAMgBDYCBEEQEDIiBCADNgIMIARBxKYBNgIAIARCADcCBCAAIAM2AhwgACgCICEDIAAgBDYCIAJAIANFDQAgA0F//h4CBA0AIAMgAygCACgCCBEBAAJAIANBCGoiBP4QAgAEQCAEQX/+HgIADQELIAMgAygCACgCEBEBAAsLIAAgACgCHDYCJAJAAkACQAJAIAAoAgxB8AdxIgNB/wBNBEAgA0EQaw4RAwICAgICAgICAgICAgICAgQBCwJAIANBgAFHBEAgA0GAAkYNASADQYAERw0DIAAgASACEOUCDwsgACIDKAIkIQYCQAJAIAEgAkYNACABIQACQANAIAAtAABBCkYNASAAQQFqIgAgAkcNAAsgAiEACyAAIAFGDQAgAyABIAAQ5AIaIAAhAQwBC0EIEDIhACAGKAIEIQQgAEHIqAE2AgAgACAENgIEIAYgADYCBCADIAMoAiQoAgQ2AiQLIAIgASABIAJHaiIBRwRAA0AgASEAAkADQCAALQAAQQpGDQEgAEEBaiIAIAJHDQALIAIhAAsgAygCJCEEAkAgACABRwRAIAMgASAAEOQCGgwBC0EIEDIhASAEKAIEIQUgAUHIqAE2AgAgASAFNgIEIAQgATYCBCADIAMoAiQoAgQ2AiQLQQwQMiEBIAYoAgQhBSABIAQoAgQ2AgggASAFNgIEIAFBnLQBNgIAIAYgATYCBCAEQQA2AgRBCBAyIQEgAygCJCgCBCEFIAFByKgBNgIAIAEgBTYCBCAEIAE2AgQgAygCJEEANgIEQQgQMiEBIAQoAgQhBSABQdy0ATYCACABIAU2AgQgAygCJCABNgIEIAMgBCgCBDYCJCAAIAAgAkdqIgEgAkcNAAsLIAIPCyAAIgMoAiQhBgJAAkAgASACRg0AIAEhAAJAA0AgAC0AAEEKRg0BIABBAWoiACACRw0ACyACIQALIAAgAUYNACADIAEgABCGAhogACEBDAELQQgQMiEAIAYoAgQhBCAAQcioATYCACAAIAQ2AgQgBiAANgIEIAMgAygCJCgCBDYCJAsgAiABIAEgAkdqIgFHBEADQCABIQACQANAIAAtAABBCkYNASAAQQFqIgAgAkcNAAsgAiEACyADKAIkIQQCQCAAIAFHBEAgAyABIAAQhgIaDAELQQgQMiEBIAQoAgQhBSABQcioATYCACABIAU2AgQgBCABNgIEIAMgAygCJCgCBDYCJAtBDBAyIQEgBigCBCEFIAEgBCgCBDYCCCABIAU2AgQgAUGctAE2AgAgBiABNgIEIARBADYCBEEIEDIhASADKAIkKAIEIQUgAUHIqAE2AgAgASAFNgIEIAQgATYCBCADKAIkQQA2AgRBCBAyIQEgBCgCBCEFIAFB3LQBNgIAIAEgBTYCBCADKAIkIAE2AgQgAyAEKAIENgIkIAAgACACR2oiASACRw0ACwsgAg8LIANBwABGDQILQQwQXkEOEHBB0KkDQdIAEAIACyAAIAEgAhDkAg8LIAAgASACEIYCC/gLAhF/AX4jAEEQayINJAAQaSEXAn8CQAJAIAIoAgAiCiABKAJQIgtNBEAgACgCICEQIAEoAlghCSABKAJMIQwgCkUNAQNAQQAhBiALIAogDGpJBEAgAUEANgJMIAsgDGsgCGohCEEAIQwMAQsCQANAIAZBAWohByAJIAYgDGpBBHRqKAIAQQBODQEgByIGIApHDQALIAIoAgwhCANAIAwgEWpBBHQiFSABKAJYaiARQQJ0IhIgAigCCGooAgA2AgBBACETIAggEmooAgBBAEoEQANAIAIoAhAgEmooAgAgE0ECdGooAgAhFCABKAJYIBVqIg9BCGoiDiEJAkACQCAOIgYoAgAiB0UNAANAIAciBigCECIHIBRKBEAgBiEJIAYoAgAiBw0BDAILIAcgFE4NAiAGKAIEIgcNAAsgBkEEaiEJC0EUEDIiCCAGNgIIIAhCADcCACAIIBQ2AhAgCSAINgIAIA8oAgQoAgAiBgRAIA8gBjYCBCAJKAIAIQgLIAggCCAOKAIAIg5GIgY6AAwCQCAGDQADQCAIKAIIIgctAAwNAQJAIAcgBygCCCIGKAIAIglGBEACQCAGKAIEIglFDQAgCS0ADA0ADAILAkAgCCAHKAIARgRAIAchCQwBCyAHIAcoAgQiCSgCACIINgIEIAkgCAR/IAggBzYCCCAHKAIIBSAGCzYCCCAHKAIIIgYgBigCACAHR0ECdGogCTYCACAJIAc2AgAgByAJNgIIIAkoAggiBigCACEHCyAJQQE6AAwgBkEAOgAMIAYgBygCBCIINgIAIAgEQCAIIAY2AggLIAcgBigCCDYCCCAGKAIIIgggCCgCACAGR0ECdGogBzYCACAHIAY2AgQgBiAHNgIIDAMLAkAgCUUNACAJLQAMDQAMAQsCQCAIIAcoAgBHBEAgByEIDAELIAcgCCgCBCIJNgIAIAggCQR/IAkgBzYCCCAHKAIIBSAGCzYCCCAHKAIIIgYgBigCACAHR0ECdGogCDYCACAIIAc2AgQgByAINgIIIAgoAgghBgsgCEEBOgAMIAZBADoADCAGIAYoAgQiBygCACIINgIEIAgEQCAIIAY2AggLIAcgBigCCDYCCCAGKAIIIgggCCgCACAGR0ECdGogBzYCACAHIAY2AgAgBiAHNgIIDAILIAdBAToADCAGIAYgDkY6AAwgCUEBOgAMIA4gBiIIRw0ACwsgDyAPKAIMQQFqNgIMIAIoAgwhCAsgE0EBaiITIAggEmooAgBIDQALCyARQQFqIhEgCkcNAAsgASgCWCEJDAQLIAEgByAMaiIMNgJMIAcgCGoiCCALSQ0AC0EADAMLIA0gCzYCCCANIAo2AgQgDUGKCzYCAEECQbvqACANEDRBAAwCCyALIAxPDQAgAUEANgJMCwJAIAtBAWsiBgRAA0AgCSAGQQR0aiIHKAIAQQBOBEAgBygCDA0DCyAGIgtBAWsiBg0ACwtBASELCyABIAs2AlQgASgC0KcBEJICIAAgASACENoEIQYgASgC0KcBIAYQ2gEgBigCDCAGKAIEQQJ0akEEaygCACEHIAEoApCnASIAKAIAQTxGBEAgACADEJECCyABQYSoAWohAyAAIAYgACgCLBECACAAKAIcIgYEQCAAIAYRAQALAkAgCiAQbCIAIAFBiKgBaigCACABKAKEqAEiCGtBAnUiBksEQCADIAAgBmsQbgwBCyAAIAZPDQAgASAIIABBAnRqNgKIqAELIApBAEoEQCAQQQJ0IQBBACEGA0AgAigCFCAGai0AAARAIAcgBiAQbEECdCIIIAMoAgBqIAggABDmBAsgBkEBaiIGIApHDQALCwJAIAIoAgAiAEEBRgRAIAEQaSAXfSABKQMQfDcDECABIAEoAjhBAWo2AjgMAQsQaSAXfSEXIABBD0wEQCABIBcgASkDGHw3AxggASABKAI8IApqNgI8DAELIAEgFyABKQMgfDcDICABIAEoAkAgCmo2AkALQQEgBEUNABogBSAEEQAAQQFzCyEWIA1BEGokACAWC7oNAQx/AkAgACgCBCIDRQ0AIAJBACACQQBKGyEMIAAoAgwhBwJAIAFBAEgEQCADIQgDQAJAIAcgCUEEdCICaiIBKAIAIgUgDEgNACAFQf////8HRg0AIAFBBGogAUEIaiIDKAIAEJYBIAEgAzYCBCABQgA3AgggACgCBCEDIAAoAgwiByACaiIBKAIMDQAgAUF/NgIAIAkgCCADIAhGGyEICyAJQQFqIgkgA0kNAAsMAQsgAyEIA0ACQCAHIAlBBHQiDWoiBigCACICIAxIDQAgAkH/////B0YNACAGQQhqIgohBCAKKAIAIgUhAiAFRQ0AA0AgBCACIAIoAhAgAUgiCxshBCACQQRqIAIgCxsoAgAiAg0ACyAEIApGDQAgBCgCECABSg0AIAZBCGoiCiEEIAUhAgNAIAQgAiACKAIQIAFIIgsbIQQgAkEEaiACIAsbKAIAIgINAAsCQCAEIApGDQAgBCgCECABSg0AAkAgBCIDKAIEIgcEQANAIAciAigCACIHDQAMAgsACwNAIAMoAggiAigCACADRyEOIAIhAyAODQALCyAEIAYoAgRGBEAgBiACNgIECyAGIAYoAgxBAWs2AgwCfwJAIAQiByIDKAIAIgQEQCADKAIEIgJFDQEDQCACIgMoAgAiAg0ACwsgAygCBCIEDQBBACEEQQEMAQsgBCADKAIINgIIQQALIQoCQCADIAMoAggiBigCACICRgRAIAYgBDYCACADIAVGBEBBACECIAQhBQwCCyAGKAIEIQIMAQsgBiAENgIECyADLQAMIQsgAyAHRwRAIAMgBygCCCIGNgIIIAYgBygCCCgCACAHR0ECdGogAzYCACADIAcoAgAiBjYCACAGIAM2AgggAyAHKAIEIgY2AgQgBgRAIAYgAzYCCAsgAyAHLQAMOgAMIAMgBSAFIAdGGyEFCwJAIAtFDQAgBUUNACAKBEADQCACLQAMIQQCQCACIAIoAggiAygCAEcEQCAERQRAIAJBAToADCADQQA6AAwgAyADKAIEIgQoAgAiBjYCBCAGBEAgBiADNgIICyAEIAMoAgg2AgggAygCCCIGIAYoAgAgA0dBAnRqIAQ2AgAgBCADNgIAIAMgBDYCCCACIAUgBSACKAIAIgJGGyEFIAIoAgQhAgsCQAJAAkACQCACKAIAIgMEQCADLQAMRQ0BCyACKAIEIgQEQCAELQAMRQ0CCyACQQA6AAwCQCAFIAIoAggiAkYEQCAFIQIMAQsgAi0ADA0GCyACQQE6AAwMCAsgAigCBCIERQ0BCyAELQAMDQAgAiEDDAELIANBAToADCACQQA6AAwgAiADKAIEIgU2AgAgBQRAIAUgAjYCCAsgAyACKAIINgIIIAIoAggiBSAFKAIAIAJHQQJ0aiADNgIAIAMgAjYCBCACIAM2AgggAiEECyADIAMoAggiAi0ADDoADCACQQE6AAwgBEEBOgAMIAIgAigCBCIDKAIAIgU2AgQgBQRAIAUgAjYCCAsgAyACKAIINgIIIAIoAggiBSAFKAIAIAJHQQJ0aiADNgIAIAMgAjYCACACIAM2AggMBAsgBEUEQCACQQE6AAwgA0EAOgAMIAMgAigCBCIENgIAIAQEQCAEIAM2AggLIAIgAygCCDYCCCADKAIIIgQgBCgCACADR0ECdGogAjYCACACIAM2AgQgAyACNgIIIAIgBSADIAVGGyEFIAMoAgAhAgsCQAJAIAIoAgAiBEUNACAELQAMDQAgAiEDDAELAkAgAigCBCIDBEAgAy0ADEUNAQsgAkEAOgAMIAIoAggiAi0ADEEAIAIgBUcbDQIgAkEBOgAMDAULIAQEQCAELQAMRQRAIAIhAwwCCyACKAIEIQMLIANBAToADCACQQA6AAwgAiADKAIAIgU2AgQgBQRAIAUgAjYCCAsgAyACKAIINgIIIAIoAggiBSAFKAIAIAJHQQJ0aiADNgIAIAMgAjYCACACIAM2AgggAiEECyADIAMoAggiAi0ADDoADCACQQE6AAwgBEEBOgAMIAIgAigCACIDKAIEIgU2AgAgBQRAIAUgAjYCCAsgAyACKAIINgIIIAIoAggiBSAFKAIAIAJHQQJ0aiADNgIAIAMgAjYCBCACIAM2AggMAwsgAigCCCIDIAMoAgAgAkZBAnRqKAIAIQIMAAsACyAEQQE6AAwLIAcQLyAAKAIEIQMgACgCDCEHCyAHIA1qIgIoAgwNACACQX82AgAgCSAIIAMgCEYbIQgLIAlBAWoiCSADSQ0ACwsgAyAIRg0AIAAgCDYCAAsLiwIBB38gAEEEaiEGAkACQCAAKAIEIgBFDQAgASgCACABIAEtAAsiA8BBAEgiAhshBSABKAIEIAMgAhshAyAGIQEDQAJAIAMgACgCFCAALQAbIgIgAsBBAEgiBBsiAiACIANLIgcbIggEQCAAKAIQIABBEGogBBsgBSAIEEoiBA0BC0F/IAcgAiADSRshBAsgASAAIARBAEgiAhshASAAQQRqIAAgAhsoAgAiAA0ACyABIAZGDQACQCABKAIUIAEtABsiACAAwEEASCICGyIAIAMgACADSRsiBARAIAUgASgCECABQRBqIAIbIAQQSiIFDQELIAAgA0sNAQwCCyAFQQBODQELIAYhAQsgAQvOAwELfyABIAAoAggiBCAAKAIEIgJrTQRAIAAgAQR/IAJBACAB/AsAIAEgAmoFIAILNgIEDwsgAiAAKAIAIgVrIgYgAWoiA0EATgRAQf////8HIAQgBWsiBEEBdCIIIAMgAyAISRsgBEH/////A08bIgQEQCAEEDIhBwsgBiAHaiIDQQAgAfwLACAEIAdqIQggASADaiEJAkAgAiAFRgRAIAMhBwwBCwJAAkAgBkEQSQ0AIAUgAiAHamsgAmpBEEkNACADQRBrIQogAkEQayELIAIgBkFwcSIEayECIAMgBGshA0EAIQEDQCAKIAFrIAsgAWv9AAAA/QsAACABQRBqIgEgBEcNAAsgBCAGRg0BCyAFQX9zIAJqIQwgAiAFa0EDcSIEBEBBACEBA0AgA0EBayIDIAJBAWsiAi0AADoAACABQQFqIgEgBEcNAAsLIAxBA0kNAANAIANBAWsgAkEBay0AADoAACADQQJrIAJBAmstAAA6AAAgA0EDayACQQNrLQAAOgAAIANBBGsiAyACQQRrIgItAAA6AAAgAiAFRw0ACwsgACgCACECCyAAIAg2AgggACAJNgIEIAAgBzYCACACBEAgAhAvCw8LEDYAC5AFAQR/IAAoAsyoASIBBEAgAEHQqAFqIAE2AgAgARAvCyAALAC3qAFBAEgEQCAAKAKsqAEQLwsgACgCnKgBIgEEQCAAQaCoAWogATYCACABEC8LIAAoApCoASIDBEAgAEGUqAFqKAIAIgIgAyIBRwRAA0AgAkEwayIBKAIcIgQEQCACQRBrIAQ2AgAgBBAvCyACQRVrLAAAQQBIBEAgAkEgaygCABAvCyABIgIgA0cNAAsgACgCkKgBIQELIAAgAzYClKgBIAEQLwsgACgChKgBIgEEQCAAQYioAWogATYCACABEC8LIAAoAvinASIBBEAgAEH8pwFqIAE2AgAgARAvCyAAKALspwEiAQRAIABB8KcBaiABNgIAIAEQLwsgAEHUpwFqKAIAIgEEQCAAQdinAWogATYCACABEC8LIABBwKcBaigCACIBBEAgAEHEpwFqIAE2AgAgARAvCyAAQaynAWooAgAiAQRAIABBsKcBaiABNgIAIAEQLwsgAEGYpwFqKAIAIgEEQCAAQZynAWogATYCACABEC8LIABBuJIBahClASAAQeD9AGoQpQEgAEGI6QBqEKUBIABBsNQAahClASAAQdg/ahClASAAQYArahClASAAQagWahClASAAQdABahClASAAKAKoASIBBEAgACABNgKsASABEC8LIAAoAoABIgMEQCAAKAKEASICIAMiAUcEQANAIAJBDGsgAkEQayICKAIIEJYBIAIgA0cNAAsgACgCgAEhAQsgACADNgKEASABEC8LIAAoAlgiAwRAIAAoAlwiAiADIgFHBEADQCACQQxrIAJBEGsiAigCCBCWASACIANHDQALIAAoAlghAQsgACADNgJcIAEQLwsgAAtDACABBEAgACABKAIAEPMCIAAgASgCBBDzAiABLAArQQBIBEAgASgCIBAvCyABLAAbQQBIBEAgASgCEBAvCyABEC8LCzEAIAEEQCAAIAEoAgAQ9AIgACABKAIEEPQCIAEsAB9BAEgEQCABKAIUEC8LIAEQLwsLmUgEF34XfwR7AX0jAEGQAWsiIyQAAkACQAJAAkACQAJAAkACQAJAAkACQAJAIAEoAjAiHSABKAIAIhpBJGwiG0GYmwFqKAIARyImDQAgATUCNCIDIAEpAxAgHa1+IBtBlJsBajQCAH9SDQAgATUCOCIFIAEpAxggA35SDQAgATUCPCABKQMgIAV+Ug0AIAIoAjAiGyACKAIAIhxBJGwiIkGYmwFqKAIARw0AIAI1AjQiAyACKQMQIButfiAiQZSbAWo0AgB/Ug0AIAI1AjgiBSACKQMYIAN+Ug0AIBogHEcNACACNQI8IAIpAyAgBX5RDQELAkACQAJAIBoOAgIAAQsgAikDKCIRIAIpAyAiDyACKQMYIhAgAikDECIMfn5+IAEpAygiDSABKQMgIgcgASkDGCIOIAEpAxAiCX5+flEEQAJAIAAoAgAOAw0ADQALIAIoAjwhISACKAI4IR8gAigCNCElIAEoAjwhKCABKAI4ISQgASgCNCEiIAAoAgghGyAAKAIEIScgAigCMCEaAkAgJg0AICKtIgMgCSAdrX5SDQAgJK0iBSADIA5+Ug0AICitIAUgB35SDQAgGiACKAIAIhxBJGwiHkGYmwFqKAIARw0AICWtIgMgDCAarX4gHkGUmwFqNAIAf1INACAfrSIFIAMgEH5SDQAgIa0gBSAPflINACAcQQFGDQQLIA6nIh4gG2pBAWsgG20iKSAnbCIbIClqIhwgHiAcIB5IGyEcIAIoAgAhIAJAICYNACAJIAxSDQAgIEEBRw0AIBogHUcNACANQgBXDQ0gB0IAVw0NIBsgHE4NDSAdIAynbCEAIBysIQsgG6whBQNAICggBKciGmwhGyAaICFsIRpCACEGA0AgJCAGpyIdbCEcIB0gH2whHSAFIQMDQCACKALQASAlIAOnIh5saiAdaiAaaiABKALQASAeICJsaiAcaiAbaiAA/AoAACADQgF8IgMgC1INAAsgBkIBfCIGIAdSDQALIARCAXwiBCANUg0ACwwNCyAaICBBJGwiJkGYmwFqKAIARw0IICWtIgMgDCAarX4gJkGUmwFqNAIAIgV/Ug0IIB+tIgsgAyAQflINCCAhrSALIA9+Ug0IIB1BAkcNCQJAICAOAgwACwsgDUIAVw0MIAdCAFcNDCACKALQASEdIBsgCaciAkEBdCIAbCEhIB4gHGsgAGwhHyAHpyAbIB5qIBxrbCACbEEBdCElIBsgImwhJiAcIBtrQQFxISdBACECIBwgG0EBaiIpRiEqA0ACQCAbIBxIBEAgKCAIp2whHkIAIQMDQCACICFqIRogJCADp2whICAnBH8gGiAdaiABKALQASAmaiAgaiAeaiAA/AoAACAAIBpqIRogKQUgGwshAiAqRQRAA0AgGiAdaiABKALQASACICJsaiAgaiAeaiAA/AoAACAdIAAgGmoiGmogASgC0AEgAkEBaiAibGogIGogHmogAPwKAAAgACAaaiEaIAJBAmoiAiAcRw0ACwsgGiAfaiECIANCAXwiAyAHUg0ACwwBCyACICVqIQILIAhCAXwiCCANUg0ACwwMC0G8wwIoAgAQMBogI0H42QA2AkggI0GXMjYCRCAjQeQmNgJAQbjDAigCAEHL5AAgI0FAaxAxDAwLQbzDAigCABAwGiAjQZshNgIIICNBrjY2AgQgI0HkJjYCAEG4wwIoAgBBy+QAICMQMQwLCyACKQMoIhEgAikDICIPIAIpAxgiECACKQMQIgx+fn4gASkDKCIOIAEpAyAiDSABKQMYIgMgASkDECIHfn5+Ug0CAkAgACgCAA4DCgAKAAsgAigCPCEkIAIoAjghICACKAI0ISEgASgCPCEeIAEoAjghKCABKAI0ISIgACgCCCEbIAAoAgQhJSACKAIwIRogJg0BICKtIgUgByAdrX5SDQEgKK0iCyADIAV+Ug0BIB6tIAsgDX5SDQEgGiACKAIAIhxBJGwiH0GYmwFqKAIARw0BICGtIgUgDCAarX4gH0GUmwFqNAIAf1INASAgrSILIAUgEH5SDQEgJK0gCyAPflINASAcDQELIwBBMGsiGiQAAkACQAJAIAIpAyggAikDICIDIAIpAxgiBSACKQMQIgt+fn4iCSABKQMoIAEpAyAiBCABKQMYIgYgASkDECIIfn5+UQRAIAIoAjAiHSACKAIAIhxBJGwiG0GYmwFqKAIARw0BIAI1AjQiCiALIB2tfiAbQZSbAWo0AgB/Ug0BIAI1AjgiCyAFIAp+Ug0BIAI1AjwgAyALflINASABKAIwIhsgASgCACIiQSRsIh5BmJsBaigCAEcNASABNQI0IgMgCCAbrX4gHkGUmwFqNAIAf1INASABNQI4IgUgAyAGflINASABNQI8IAQgBX5SDQEgHCAiRw0CAkACQCAAKAIADgMBAAEACyAJpyIcIAAoAggiImpBAWsgIm0iIiAAKAIEbCIAICJqIiIgHCAcICJKGyIcIABMDQAgAigC0AEgACAdbGogASgC0AEgACAbbGogHCAAayAbbPwKAAALIBpBMGokAAwDC0G8wwIoAgAQMBogGkH42QA2AiggGkH3MTYCJCAaQeQmNgIgQbjDAigCAEHL5AAgGkEgahAxEAAAC0G8wwIoAgAQMBogGkHE2QA2AhggGkH4MTYCFCAaQeQmNgIQQbjDAigCAEHL5AAgGkEQahAxEAAAC0G8wwIoAgAQMBogGkGWIjYCCCAaQfkxNgIEIBpB5CY2AgBBuMMCKAIAQcvkACAaEDEQAAALDAgLIAOnIh8gG2pBAWsgG20iJyAlbCIbICdqIgAgHyAAIB9IGyEcIAIoAgAhAAJAICYNACAHIAxSDQAgAA0AIBogHUcNACAOQgBXDQggDUIAVw0IIBsgHE4NCCAdIAynbCEAIBysIQsgG6whBQNAIB4gBKciGmwhGyAaICRsIRpCACEGA0AgKCAGpyIdbCEcIB0gIGwhHSAFIQMDQCACKALQASAhIAOnIh9saiAdaiAaaiABKALQASAfICJsaiAcaiAbaiAA/AoAACADQgF8IgMgC1INAAsgBkIBfCIGIA1SDQALIA4gBEIBfCIEUg0ACwwICwJAIBogAEEkbCImQZibAWooAgBHDQAgIa0iBSAMIBqtfiAmQZSbAWo0AgAiC39SDQAgIK0iCSAFIBB+Ug0AICStIAkgD35SDQAgHUEERgRAIABFBEAgDkIAVw0KIA1CAFcNCiAbIBxODQogAigC0AEhHSAbIAenQQJ0IgBsISEgHyAcayAAbCEfIBsgImwhJSAcIBtrQQFxISZBACECIBwgG0EBaiInRiEpA0AgHiAIp2whJEIAIQMDQCACICFqIRogKCADp2whICAmBH8gGiAdaiABKALQASAlaiAgaiAkaiAA/AoAACAAIBpqIRogJwUgGwshAiApRQRAA0AgGiAdaiABKALQASACICJsaiAgaiAkaiAA/AoAACAdIAAgGmoiGmogASgC0AEgAkEBaiAibGogIGogJGogAPwKAAAgACAaaiEaIAJBAmoiAiAcRw0ACwsgGiAfaiECIANCAXwiAyANUg0ACyAOIAhCAXwiCFINAAsMCgtBsYAcIAB2QQFxRQRAIAcgC38hAyAOQgBXDQogDUIAVw0KIBsgHE4NCiAAQSRsQaSbAWooAgAhHSACKALQASEkIBogA6dsIhogG2whJSAaIB8gHGtsISYgB6chICAbQQFqIScgGyAibCEpIBwgG2siKkEBcSErQQAhAANAIB4gCKdsISFCACEDA0AgACAlaiECICggA6dsIR8gKwR/IAEoAtABIClqIB9qICFqIAIgJGogICAdEQUAIAIgGmohAiAnBSAbCyEAICpBAUcEQANAIAEoAtABIAAgImxqIB9qICFqIAIgJGogICAdEQUAIAEoAtABIABBAWogImxqIB9qICFqICQgAiAaaiICaiAgIB0RBQAgAiAaaiECIABBAmoiACAcRw0ACwsgAiAmaiEAIANCAXwiAyANUg0ACyAOIAhCAXwiCFINAAsMCgtBvMMCKAIAEDAaICNBmyE2AlggI0H2NDYCVCAjQeQmNgJQQbjDAigCAEHL5AAgI0HQAGoQMQwKCwJAAkACQCAADgIAAQILIA5CAFcNCiANQgBXDQogGyAcTg0KIAdCAFcNCiAbIAenIgBsISYgAyAcrX0gB36nISkgHUEBRiAHQgtWcSEqIAdCA4MhCSACKALQASIgIABBAnRqISsgB0J8gyILpyEtICIgJWwgJ2wiAq0hBiAAIAJqQQNqrSEKICitIQwgHq0hEiABKALQASEhQQAhAUIAIQUDQCAFIBJ+IgMgCnwhDyADIAZ8IRAgISAeIAWnbGohJUIAIQQDQCABICZqIQIgJSAoIASnbGohJyAPIAQgDH4iA3ynIS4gAyAQfKchLEEAIQAgGyEaA0AgJyAaICJsaiEkQgAhCAJAAkAgKkUEQEIAIQMMAQsCQCAgIAJBAnQiAWogISAAICJsIh8gLmpqTw0AICEgHyAsamogASArak8NAEIAIQMMAQsgAiAtaiEv/QwAAAAAAQAAAAIAAAADAAAAITFCACEDA0AgICACIAOnakECdGogJCAx/RsAav0JAgAgJCAx/RsBaioCAP0gASAkIDH9GwJqKgIA/SACICQgMf0bA2oqAgD9IAP9CwIAIDH9DAQAAAAEAAAABAAAAAQAAAD9rgEhMSADQgR8IgMgC1INAAsgLyECIAsiAyAHUQ0BCyAHIANCf4V8IRcgCUIAUgRAA0AgICACQQJ0aiAkIB0gA6dsaioCADgCACADQgF8IQMgAkEBaiECIAhCAXwiCCAJUg0ACwsgF0IDVA0AA0AgICACQQJ0aiIBICQgHSADpyIfbGoqAgA4AgAgASAkIB0gH0EBamxqKgIAOAIEIAEgJCAdIB9BAmpsaioCADgCCCABICQgHSAfQQNqbGoqAgA4AgwgAkEEaiECIANCBHwiAyAHUg0ACwsgAEEBaiEAIBpBAWoiGiAcRw0ACyACIClqIQEgBEIBfCIEIA1SDQALIA4gBUIBfCIFUg0ACwwKCyAOQgBXDQkgB0IAVw0JIA1CAFcNCSAbIBxODQkgAigC0AEhJCAbIAenbCEgIAMgHK19IAd+pyEhIB1BAUYgB0IDVnEhHyAHQnyDIgWnISUgASgC0AEhJkEAIQEDQCAmIB4gCKdsaiEnQgAhBgNAIAEgIGohASAnICggBqdsaiEpIBshAANAICkgACAibGohAkIAIQMCQAJAIB9FBEAgASEaDAELIAEgJWohGv0MAAAAAAEAAAACAAAAAwAAACExA0AgJCABIAOnakEBdGr9DAB+AAAAfgAAAH4AAAB+AAAgAiAx/RsAav0JAgAgAiAx/RsBaioCAP0gASACIDH9GwJqKgIA/SACIAIgMf0bA2oqAgD9IAMiMv3gAf0MAACAdwAAgHcAAIB3AACAd/3mAf0MAACACAAAgAgAAIAIAACACP3mASAyQQH9qwEiM/0MAAAA/wAAAP8AAAD/AAAA//1O/QwAAABxAAAAcQAAAHEAAABx/bkBQQH9rQH9DAAAgAcAAIAHAACABwAAgAf9rgH95AEiNEEN/a0B/QwAfAAAAHwAAAB8AAAAfAAA/U4gNP0M/w8AAP8PAAD/DwAA/w8AAP1O/a4BIDP9DAAAAP8AAAD/AAAA/wAAAP/9PP1SIDJBEP2tAf0MAIAAAACAAAAAgAAAAIAAAP1O/VAgMf0NAAEEBQgJDA0AAQABAAEAAf1bAQAAIDH9DAQAAAAEAAAABAAAAAQAAAD9rgEhMSADQgR8IgMgBVINAAsgGiEBIAUiAyAHUQ0BCyAaIQEDQCAkIAFBAXRqQYD8ASACIB0gA6dsaioCACI1i0MAAIB3lEMAAIAIlEGAgICIByA1vCIaQQF0IipBgICAeHEiKyArQYCAgIgHTRtBAXZBgICAPGq+krwiK0ENdkGA+AFxICtB/x9xaiAqQYCAgHhLGyAaQRB2QYCAAnFyOwEAIAFBAWohASADQgF8IgMgB1INAAsLIABBAWoiACAcRw0ACyABICFqIQEgBkIBfCIGIA1SDQALIA4gCEIBfCIIUg0ACwwJC0G8wwIoAgAQMBogI0GbITYCaCAjQaA1NgJkICNB5CY2AmBBuMMCKAIAQcvkACAjQeAAahAxDAkLAkAgAA4CAwACCyAOQgBXDQcgB0IAVw0HIA1CAFcNByAbIBxODQcgByAbrCIJfiETIAMgHKwiFH0gB34hFSACKALQASEAIAEoAtABIQFCACEDA0AgASAeIBKnbGohAkIAIQsDQCAMIAMgE3wiA1cEQANAAkAgBkIBfCIGIBBSDQBCACEGIApCAXwiCiAPUg0AIARCAXwiBUIAIAUgEVIbIQRCACEKCyADIAx9IgMgDFkNAAsLIAIgKCALp2xqIRsgCSEFA0AgGyAiIAWnbGohHEIAIQgDQCAAIBogA6dsaiAhIAanbGogICAKp2xqICQgBKdsakGA/AEgHCAdIAinbGoqAgAiNYtDAACAd5RDAACACJRBgICAiAcgNbwiH0EBdCIlQYCAgHhxIiYgJkGAgICIB00bQQF2QYCAgDxqvpK8IiZBDXZBgPgBcSAmQf8fcWogJUGAgIB4SxsgH0EQdkGAgAJxcjsBAAJAIANCAXwiAyAMUg0AQgAhAyAGQgF8IgYgEFINACAPIApCAXwiClIEQEIAIQYMAQsgBEIBfCIEQgAgBCARUhshBEIAIQZCACEKCyAIQgF8IgggB1INAAsgBUIBfCIFIBRSDQALIAwgAyAVfCIDVwRAA0ACQCAGQgF8IgYgEFINAEIAIQYgCkIBfCIKIA9SDQAgBEIBfCIFQgAgBSARUhshBEIAIQoLIAMgDH0iAyAMWQ0ACwsgC0IBfCILIA1SDQALIA4gEkIBfCISUg0ACwwHC0G8wwIoAgAQMBogI0H42QA2AogBICNBpjQ2AoQBICNB5CY2AoABQbjDAigCAEHL5AAgI0GAAWoQMQwHC0G8wwIoAgAQMBogI0GbITYCeCAjQZc2NgJ0ICNB5CY2AnBBuMMCKAIAQcvkACAjQfAAahAxDAYLIA5CAFcNBCAHQgBXDQQgDUIAVw0EIBsgHE4NBCAHIBusIgl+IRMgAyAcrCIUfSAHfiEVQgAhAwNAIB4gEqdsIQBCACELA0AgDCADIBN8IgNXBEADQAJAIAZCAXwiBiAQUg0AQgAhBiAKQgF8IgogD1INACAEQgF8IgVCACAFIBFSGyEEQgAhCgsgAyAMfSIDIAxZDQALCyAoIAunbCEbIAkhBQNAICIgBadsIRxCACEIA0AgAigC0AEgGiADp2xqICEgBqdsaiAgIAqnbGogJCAEp2xqIAEoAtABIB0gCKdsaiAcaiAbaiAAaigAADYAAAJAIANCAXwiAyAMUg0AQgAhAyAGQgF8IgYgEFINACAPIApCAXwiClIEQEIAIQYMAQsgBEIBfCIEQgAgBCARUhshBEIAIQZCACEKCyAIQgF8IgggB1INAAsgBUIBfCIFIBRSDQALIAwgAyAVfCIDVwRAA0ACQCAGQgF8IgYgEFINAEIAIQYgCkIBfCIKIA9SDQAgBEIBfCIFQgAgBSARUhshBEIAIQoLIAMgDH0iAyAMWQ0ACwsgC0IBfCILIA1SDQALIA4gEkIBfCISUg0ACwwECwJAAkACQCAgDgIAAgELIA1CAFcNBSAJQgBXDQUgB0IAVw0FIBsgHE4NBSAJIBusIgt+IRMgDiAcrCIUfSAJfiEVIAIoAtABIQAgASgC0AEhAUIAIQMDQCABICggEqdsaiECQgAhDgNAIAwgAyATfCIDVwRAA0ACQCAIQgF8IgggEFINAEIAIQggBEIBfCIEIA9SDQAgCkIBfCIFQgAgBSARUhshCkIAIQQLIAMgDH0iAyAMWQ0ACwsgAiAkIA6nbGohGyALIQUDQCAbICIgBadsaiEcQgAhBgNAIAAgGiADp2xqICUgCKdsaiAfIASnbGogISAKp2xqIBwgHSAGp2xqLwEAQQJ0QZDWBGoqAgA4AgACQCADQgF8IgMgDFINAEIAIQMgCEIBfCIIIBBSDQAgDyAEQgF8IgRSBEBCACEIDAELIApCAXwiBEIAIAQgEVIbIQpCACEIQgAhBAsgBkIBfCIGIAlSDQALIAVCAXwiBSAUUg0ACyAMIAMgFXwiA1cEQANAAkAgCEIBfCIIIBBSDQBCACEIIARCAXwiBCAPUg0AIApCAXwiBUIAIAUgEVIbIQpCACEECyADIAx9IgMgDFkNAAsLIA5CAXwiDiAHUg0ACyASQgF8IhIgDVINAAsMBQtBvMMCKAIAEDAaICNBmyE2AjggI0GeNDYCNCAjQeQmNgIwQbjDAigCAEHL5AAgI0EwahAxDAULIA1CAFcNAyAJQgBXDQMgB0IAVw0DIBsgHE4NAyAJIBusIgV+IRQgDiAcrCIVfSAJfiEWQgAhAwNAICggEqdsIQBCACETA0AgDCADIBR8IgNXBEADQAJAIAhCAXwiCCAQUg0AQgAhCCAEQgF8IgQgD1INACAKQgF8IgtCACALIBFSGyEKQgAhBAsgAyAMfSIDIAxZDQALCyAkIBOnbCEbIAUhCwNAICIgC6dsIRxCACEGA0AgAigC0AEgGiADp2xqICUgCKdsaiAfIASnbGogISAKp2xqIAEoAtABIB0gBqdsaiAcaiAbaiAAai8AADsAAAJAIANCAXwiAyAJUg0AQgAhAyAIQgF8IgggDlINACAHIARCAXwiBFIEQEIAIQgMAQsgCkIBfCIEQgAgBCANUhshCkIAIQhCACEECyAGQgF8IgYgCVINAAsgC0IBfCILIBVSDQALIAwgAyAWfCIDVwRAA0ACQCAIQgF8IgggEFINAEIAIQggBEIBfCIEIA9SDQAgCkIBfCILQgAgCyARUhshCkIAIQQLIAMgDH0iAyAMWQ0ACwsgE0IBfCITIAdSDQALIBJCAXwiEiANUg0ACwwDCwJAAkACQCAgDgIAAgELIA1CAFcNBCAHQgBXDQQgAigC0AEhICAbIAmnIgBsISEgDiAcrX0gCX6nIR8gCUJ+gyEFIAlCAYMhCyAHpyAbIB5qIBxrbCAAbCEeQQAhAgNAAkACQCAbIBxODQAgCUIAVw0AIAEoAtABICggBKdsaiElQgAhCgNAIAIgIWohAiAlICQgCqdsaiEmIBshAANAICYgACAibGohGkIAIQNCACEGIAlCAVIEQANAICAgAkECdGoiJyAaIB0gA6ciKWxqLwEAQQJ0QZDWBGoqAgA4AgAgJyAaIB0gKUEBcmxqLwEAQQJ0QZDWBGoqAgA4AgQgA0ICfCEDIAJBAmohAiAGQgJ8IgYgBVINAAsLIAunBEAgICACQQJ0aiAaIB0gA6dsai8BAEECdEGQ1gRqKgIAOAIAIAJBAWohAgsgAEEBaiIAIBxHDQALIAIgH2ohAiAHIApCAXwiClINAAsMAQsgAiAeaiECCyAEQgF8IgQgDVINAAsMBAtBvMMCKAIAEDAaICNBmyE2AiggI0GpMzYCJCAjQeQmNgIgQbjDAigCAEHL5AAgI0EgahAxDAQLIA1CAFcNAiAHQgBXDQIgGyAcTg0CIAlCAFcNAiAbIAmnIgBsISUgDiAcrX0gCX6nISYgHUEBRiAJQg9WcSEqIAlCA4MhCCACKALQASIgIABBAXRqISsgCUJ4gyIFpyEtICIgJ2wgKWwiAq0hCiAAIAJqQQFqrSEMICStIQ4gKK0hEiABKALQASEhQQAhAUIAIQsDQCALIBJ+IgMgDHwhDyADIAp8IRAgISAoIAunbGohJ0IAIQQDQCABICVqIQIgJyAkIASnbGohKSAPIAQgDn4iA3ynIS4gAyAQfKchLEEAIQAgGyEaA0AgKSAaICJsaiEeQgAhBgJAAkAgKkUEQEIAIQMMAQsCQCAgIAJBAXQiAWogISAAICJsIh8gLmpqTw0AICEgHyAsamogASArak8NAEIAIQMMAQsgAiAtaiEw/QwEAAAABQAAAAYAAAAHAAAAITH9DAAAAAABAAAAAgAAAAMAAAAhMkIAIQMDQCAgIAIgA6dqQQF0aiAeIDH9GwNqIB4gMf0bAmogHiAx/RsBaiAeIDH9GwBqIB4gMv0bA2ogHiAy/RsCaiAeIDL9GwFqIB4gMv0bAGr9CAEA/VUBAAH9VQEAAv1VAQAD/VUBAAT9VQEABf1VAQAG/VUBAAf9CwEAIDL9DAgAAAAIAAAACAAAAAgAAAD9rgEhMiAx/QwIAAAACAAAAAgAAAAIAAAA/a4BITEgA0IIfCIDIAVSDQALIDAhAiAFIgMgCVENAQsgCSADQn+FfCEYIAhCAFIEQANAICAgAkEBdGogHiAdIAOnbGovAQA7AQAgA0IBfCEDIAJBAWohAiAGQgF8IgYgCFINAAsLIBhCA1QNAANAICAgAkEBdGoiASAeIB0gA6ciH2xqLwEAOwEAIAEgHiAdIB9BAWpsai8BADsBAiABIB4gHSAfQQJqbGovAQA7AQQgASAeIB0gH0EDamxqLwEAOwEGIAJBBGohAiADQgR8IgMgCVINAAsLIABBAWohACAaQQFqIhogHEcNAAsgAiAmaiEBIARCAXwiBCAHUg0ACyALQgF8IgsgDVINAAsMAgtBsYAcICB2QQFxRQRAIAkgBX8hGSANQgBXDQIgB0IAVw0CICBBJGxBpJsBaigCACEgIAAoAhAgJyAJpyIhQRBqbEECdGohACACKALQASEfIBmnIgIgGyAeaiAcayAaIAenbGxsISogAiAabCIdIBtsISYgHSAeIBxrbCEnIAlCfIMhCiAJQgODIQsgHCAbayIaQQNxISlBACECQgAhBSAaQQFrQQNJISsDQAJAIBsgHE4EQCACICpqIQIMAQtCACEEIAlCAFUEQCAoIAWnbCEtA0AgAiAmaiECICQgBKdsIS4gGyEaA0AgASgC0AEgGiAibGogLmogLWohHkIAIQZCACEDQgAhCCAJQgRaBEADQCAAIAOnIiVBAnRqIB4gJUEBdGovAQBBAnRBkNYEaioCADgCACAAICVBAXIiLEECdGogHiAsQQF0ai8BAEECdEGQ1gRqKgIAOAIAIAAgJUECciIsQQJ0aiAeICxBAXRqLwEAQQJ0QZDWBGoqAgA4AgAgACAlQQNyIiVBAnRqIB4gJUEBdGovAQBBAnRBkNYEaioCADgCACADQgR8IQMgCEIEfCIIIApSDQALCyALQgBSBEADQCAAIAOnIiVBAnRqIB4gJUEBdGovAQBBAnRBkNYEaioCADgCACADQgF8IQMgBkIBfCIGIAtSDQALCyAAIAIgH2ogISAgEQUAIAIgHWohAiAaQQFqIhogHEcNAAsgAiAnaiECIARCAXwiBCAHUg0ACwwBCwNAIAIgJmohAkEAIR4gGyEaICkEQANAIAAgAiAfaiAhICARBQAgGkEBaiEaIAIgHWohAiAeQQFqIh4gKUcNAAsLICtFBEADQCAAIAIgH2ogISAgEQUAIAAgHyACIB1qIgJqICEgIBEFACAAIB8gAiAdaiICaiAhICARBQAgACAfIAIgHWoiAmogISAgEQUAIAIgHWohAiAaQQRqIhogHEcNAAsLIAIgJ2ohAiAEQgF8IgQgB1INAAsLIAVCAXwiBSANUg0ACwwCC0G8wwIoAgAQMBogI0GbITYCGCAjQf8yNgIUICNB5CY2AhBBuMMCKAIAQcvkACAjQRBqEDEMAgsgDUIAVw0AIAdCAFcNACAbIBxODQAgAigC0AEhGiAbIAmnIgBsIR0gDiAcrX0gCX6nISAgB6cgGyAeaiAca2wgAGwhHiAJQn6DIQUgCUIBgyELIAEoAtABISFBACEAA0ACQCAJQgBVBEAgISAoIASnbGohH0IAIQoDQCAAIB1qIQAgHyAkIAqnbGohJSAbIQIDQCAlIAIgImxqIQFCACEDQgAhBiAJQgFSBEADQCAaIABBAnRqIiYgASADp0EBdCInai8BAEECdEGQ1gRqKgIAOAIAICYgASAnQQJyai8BAEECdEGQ1gRqKgIAOAIEIANCAnwhAyAAQQJqIQAgBkICfCIGIAVSDQALCyALpwRAIBogAEECdGogASADp0EBdGovAQBBAnRBkNYEaioCADgCACAAQQFqIQALIAJBAWoiAiAcRw0ACyAAICBqIQAgCkIBfCIKIAdSDQALDAELIAAgHmohAAsgBEIBfCIEIA1SDQALCyAjQZABaiQADwsQAAAL4Q4CEX8ffiMAQeABayIGJAAgAygCPCEQIAMoAjghCSADKAI0IQogAykDKCEaIAMpAyAhGyADKQMYIRwgAigCPCENIAIoAjghDiACKAI0IQsgAikDKCEYIAIpAyAhHSABNQI8ISggATUCOCEpIAEoAjQhESABKQMoISEgASkDICEiIAEpAxghGSACKQMYIR4gASgCACEIIAA0AgghHyAANAIEISAgAygCMCEPIAMpAxAhIyACKQMQIRcgASgCMCESIAEpAxAhJAJAIAIoAjAiByACKAIAIgxBJGwiE0GYmwFqKAIAIhRHDQAgC60iJSAXIAetfiATQZSbAWo0AgB/Ug0AIA6tIiYgHiAlflINACANrSAdICZ+USEVCwJAAkAgGSAjUQRAIBwgHlEEQCAbIB1RBEAgGCAaUQRAIAhBJGwiCEGYmwFqKAIAIBJGBEAgByAURgRAIA9BBEYEQCAKQQNLBEAgCSAKTwRAIAkgEE0EQCAIQbCbAWooAgAhByAYICF/ISUgHSAifyEmAkACQCAAKAIADgMAAQwBCyAHIAxGDQsgFyAHQSRsIgFBmJsBajUCAH4gAUGUmwFqNAIAfyE1IBhCAFcNCyAdQgBXDQsgHkIAVw0LIAFBpJsBaigCACEBIDWnIQMgF6chCCAeQn6DIQQgHkIBgyEfIAAoAhAhAEIAIQUDQCANIAWnbCEHQgAhGgNAIA4gGqdsIQlCACEXQgAhGSAeQgFSBEADQCACKALQASAHaiAJaiALIBenIgpsaiAAIAggAREFACACKALQASAHaiAJaiALIApBAXJsaiAAIANqIgAgCCABEQUAIBdCAnwhFyAAIANqIQAgGUICfCIZIARSDQALCyAfpwRAIAIoAtABIAdqIAlqIAsgF6dsaiAAIAggAREFACAAIANqIQALIBpCAXwiGiAdUg0ACyAFQgF8IgUgGFINAAsMCwsgBSAdfiIqIBh+IhpCASAfIBkgGlUiDxsiGHxCAX0gGH8hGwJAIBkgH0IBIA8bIhh8QgF9IBh/IhwgICAYICAgGH8iIH59fiIfIBwgH3wiGCAZIBggGVMbIiJTBEAgGyAgfiIYIBggG3wiGSAaIBkgGlMbIiNTDQELEAMQ1gEMCwsgCEGsmwFqKAIAIQggAkHQAWogAEEQaiAHIAxGGygCACEAIBcgB0EkbCIMQZibAWo1AgB+IAxBlJsBajQCAH9C/////w+DISsgJKchDCAQrSEsIAmtIS0gCq0hLiANrSEvIA6tITAgC60hMQNAIB8hGSAYQhB8IiAgIyAgICNTIgsbIjIgGFUEQANAIBlCEHwiGyAiIBsgIlMiCRsiJCAZfadBAnQhCiAZpyENIBghGgNAIBogGiAqfyIcIB1+IicgBX59IhcgFyAFfyIXIAV+fSAEfCEhIBwgJX8hMyAXICZ/ITQCfgJAIBUNACACKAIAIAdHDQAgFyAwfiAcIC9+fCAhIDF+fAwBCyAhIBcgJ3wgHn58ICt+CyEnIAMoAtABIBcgLX4gHCAsfnwgISAufnynaiEWIBkgJFMEQCABKALQASApIDR+ICggM358p2ohECAAICenaiEPIBkhFwNAIAwgBkGgAWogFyAZfadBAnRqIBAgESAXp2xqIA8gCBEGACAXQgF8IhcgJFMNAAsLIBYgDUECdGogBkGgAWogCvwKAAAgGkIBfCIaIDJTDQALIBshGSAJDQALCyAgIRggCw0ACwwKC0G8wwIoAgAQMBogBkGxOjYCGCAGQY3LADYCFCAGQeQmNgIQQbjDAigCAEHL5AAgBkEQahAxDAoLQbzDAigCABAwGiAGQcY7NgIIIAZBjMsANgIEIAZB5CY2AgBBuMMCKAIAQcvkACAGEDEMCQtBvMMCKAIAEDAaIAZBzj02AiggBkGLywA2AiQgBkHkJjYCIEG4wwIoAgBBy+QAIAZBIGoQMQwIC0G8wwIoAgAQMBogBkHd0AA2AjggBkGKywA2AjQgBkHkJjYCMEG4wwIoAgBBy+QAIAZBMGoQMQwHC0G8wwIoAgAQMBogBkGW1AA2AkggBkGHywA2AkQgBkHkJjYCQEG4wwIoAgBBy+QAIAZBQGsQMQwGC0G8wwIoAgAQMBogBkHX1AA2AlggBkGGywA2AlQgBkHkJjYCUEG4wwIoAgBBy+QAIAZB0ABqEDEMBQtBvMMCKAIAEDAaIAZByDo2AmggBkGDywA2AmQgBkHkJjYCYEG4wwIoAgBBy+QAIAZB4ABqEDEMBAtBvMMCKAIAEDAaIAZB8Tw2AnggBkGCywA2AnQgBkHkJjYCcEG4wwIoAgBBy+QAIAZB8ABqEDEMAwtBvMMCKAIAEDAaIAZBhT42AogBIAZBgcsANgKEASAGQeQmNgKAAUG4wwIoAgBBy+QAIAZBgAFqEDEMAgtBvMMCKAIAEDAaIAZBnT42ApgBIAZBgMsANgKUASAGQeQmNgKQAUG4wwIoAgBBy+QAIAZBkAFqEDEMAQsgBkHgAWokAA8LEAAAC+QEAQR/IwBBgAFrIgIkAAJAAkAgAQRAIAEoAggiBEEATA0BIAEoAgAEQCABKAIERQ0DCyACIAE2AlQgAiAANgJQIAL9DAAAAAAAAAAAAAAAAAAAAAD9CwNYIAJBADYCeCACQv////8PNwNwIAIgBDYCbCACIAQ2AmhBASEBIAIgBEEMbEEPakFwcWsiAyQAAkAgBEEBSwRAAkADQAJAIAMgAUEMbGoiBSABNgIEIAVBADYCACAFIAJB0ABqNgIIIAVBCiAFEPYBDQAgBCABQQFqIgFHDQEMAgsLQbzDAigCABAwGiACQY7BADYCKCACQeCAATYCJCACQeQmNgIgQbjDAigCAEHL5AAgAkEgahAxEAAACyADQQA2AgQgAyACQdAAajYCCCADEJMCGkEBIQEDQCADIAFBDGxqKAIAEJMERQRAIAFBAWoiASAERw0BDAMLC0G8wwIoAgAQMBogAkGOwQA2AhggAkH1gAE2AhQgAkHkJjYCEEG4wwIoAgBBy+QAIAJBEGoQMRAAAAsgA0EANgIEIAMgAkHQAGo2AgggAxCTAhoLIAAgACgCJEEBajYCJCACQYABaiQADwtBvMMCKAIAEDAaIAJBoRg2AgggAkG/gAE2AgQgAkHkJjYCAEG4wwIoAgBBy+QAIAIQMRAAAAtBvMMCKAIAEDAaIAJB1cAANgJIIAJBwIABNgJEIAJB5CY2AkBBuMMCKAIAQcvkACACQUBrEDEQAAALQbzDAigCABAwGiACQdwoNgI4IAJBw4ABNgI0IAJB5CY2AjBBuMMCKAIAQcvkACACQTBqEDEQAAAL2QoCB38JfiMAQdAAayIEJAAgAEIANwIMQQQgAiACQQBMGyEHAn8CQCABKAIEQQBMDQACQAJAAkACQANAAkAgASgCDCAIQQJ0aigCACIDIAcQ2wEhBUEAIQICQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgAygCQEEBaw5IAAEBAg4ODg4ODg4ODg4ODg4ODg4ODgMEBQ4OAA4ODg4ODg4ODg4NDg4ODg4GDgcODg4ODg4ICQoODg4ODg4ODg4ODg4OCw4MDgtBzP8DIAMoAgB2QQFxRQ0NIAUgAygCEGxBAnQhAgwNC0HM/wMgAygCjAEiAygCAHZBAXFFDQwgBSADKAIQbEECdCECDAwLQcz/AyADKAKMASgCAHZBAXFFDQsgBSADKAKQASgCEGxBAnQhAgwLCyADKAKMASgCAEEkbEGwmwFqKAIAIgUgAygCkAEiAygCAEYNCiADKQMoIAMpAyAgAykDGCADKQMQIAVBJGwiAkGYmwFqNQIAfn5+fiACQZSbAWo0AgB/pyECDAoLIAMoApQBKAIAQSRsQbCbAWooAgAiBSADKAKQASIDKAIARg0JIAMpAyggAykDICADKQMYIAMpAxAgBUEkbCICQZibAWo1AgB+fn5+IAJBlJsBajQCAH+nIQIMCQtBzP8DIAMoAowBIgMoAgB2QQFxRQ0IIAUgAygCEGxBAnQhAgwICyADKAKMASIFKQMoQgFSDQggAygCkAEiAikDIEIBUg0KIAIpAyhCAVINCyACKQMYIREgBSkDICEMIAUpAxghDSARIRIgAikDECEQIAUpAxAhDwJAAkACQCAFKAIADgIAAQ8LQgIhCiACKAIARQ0BDA4LQgEhCiACKAIADQ0LIBIgEH4gDCANfiAPfnynIAqndCECDAcLIAMoApABIgIpAxAgAikDGH4gAikDIH5CAYYgAygCjAEiAikDECACKQMYfiACKQMgfiACKQMofkIBhnynIQIMBgsgAygCkAEiCSgCGEEDakF8cSEDAkACQCAJKAIADgIAAQcLIAMgBWxBA3QhAgwGCyADIAVsQQN0IQIMBQsCQAJAIAMoApABIgMoAgAOAgABBgsgBSADKAIYbEEDdCECDAULIAUgAygCGGxBA3QhAgwECyADKAKMASkDECIKIAMoApABIgMpAxhCIIZCgICAgDB8QiCHQnyDIgsgCiALVRshCgJAAkAgAygCAA4CAAEFCyAFIAqnbEEEdCECDAQLIAUgCqdsQQR0IQIMAwsgAygCjAEoAhBBAWogAygCAEEkbEGYmwFqKAIAIAVsbCECDAILQbzDAigCABAwGiAEQZshNgJIIARBqYABNgJEIARB5CY2AkBBuMMCKAIAQcvkACAEQUBrEDEQAAALIAUgAygCEGxBAnQhAgsgBiACIAIgBkkbIQYgCEEBaiIIIAEoAgRIDQEMBQsLQbzDAigCABAwGiAEQfI+NgI4IARB2v8ANgI0IARB5CY2AjBBuMMCKAIAQcvkACAEQTBqEDEQAAALQbzDAigCABAwGiAEQZw/NgIoIARB2/8ANgIkIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEQAAALQbzDAigCABAwGiAEQdk+NgIYIARB3P8ANgIUIARB5CY2AhBBuMMCKAIAQcvkACAEQRBqEDEQAAALQbzDAigCABAwGiAEQZshNgIIIARB7v8ANgIEIARB5CY2AgBBuMMCKAIAQcvkACAEEDEQAAALIAZFDQAgB0EGdCAGakFAagwBC0EACyECIABBADYCBCAAIAI2AgAgACAHNgIIIARB0ABqJAALHAAgACABQQggAqcgAkIgiKcgA6cgA0IgiKcQFwuuAgEGf0EgIQEDQCABIAJqIgNBAXYiBEEBaiACIANBGEkiAxsiAiABIAQgAxsiAUkNAAtBvIACIQEgAEEBIAJBH00EfyACQQJ0QYCjAWooAgBBAnRBuIABagVBvIACCxCwAigCACEDIAAoAgQhBkEgIQFBACECA0AgASACaiIEQQF2IgVBAWogAiAEQRhJIgQbIgIgASAFIAQbIgFJDQALQYEgIQEgAkEfTQRAIAJBAnRBgKMBaigCACEBCyAGIANqIgBBuIABaiICQQAgAUECdPwLACAAQgA3AyAgACACNgIcIAAgATYCGCAAIABBuMAAajYCFCAAQQA2AhAgACAAQThqNgIMIABBADYCCCAAQoAQNwMAIAD9DAAAAAAAAAAAAAAAAAAAAAD9CwMoIAALJAACQCAAKAIAQQxrIgBBCGpBf/4eAgBBAWtBAE4NACAAEC8LCxUAIABB2LYDNgIAIABBBGoQ+wIgAAvMAgEHf0EgIQIDQCABIAJqQQF2IgNBAWogASADQQJ0QYCjAWooAgBBgMAASSIEGyIBIAIgAyAEGyICSQ0ACwJ/IAFBH00EQCABQQJ0QYCjAWooAgAMAQtBgcAACyECQQAhASAAQQEgAkECdEG4gAJqELACKAIAIQcgACgCBCEGQSAhAgNAIAEgAmpBAXYiAEEBaiABIABBAnRBgKMBaigCAEGAwABJIgUbIgEgAiAAIAUbIgJJDQALIAcgBmoiAEE4aiICQYCAAWoiA0GAgAFqIgRBAAJ/IAFBH00EQCABQQJ0QYCjAWooAgAMAQtBgcAACyIBQQJ0/AsAIABCADcDICAAIAQ2AhwgACABNgIYIAAgAzYCFCAAIAI2AgwgAEIANwIEIABBgCA2AgAgAP0MAAAAAAAAAAAAAAAAAAAAAP0LAyggAEEANgIQIAALUgEBfyAAKAIEIQQgACgCACIAIAECf0EAIAJFDQAaIARBCHUiASAEQQFxRQ0AGiABIAIoAgBqKAIACyACaiADQQIgBEECcRsgACgCACgCHBEGAAu6AgEDfyMAQUBqIgIkACAAKAIAIgNBBGsoAgAhBCADQQhrKAIAIQMgAkIANwIgIAJCADcCKCACQgA3AjAgAkIANwA3IAJCADcCGCACQQA2AhQgAkG4sAM2AhAgAiAANgIMIAIgATYCCCAAIANqIQBBACEDAkAgBCABQQAQbwRAIAJBATYCOCAEIAJBCGogACAAQQFBACAEKAIAKAIUEQwAIABBACACKAIgQQFGGyEDDAELIAQgAkEIaiAAQQFBACAEKAIAKAIYEQsAAkACQCACKAIsDgIAAQILIAIoAhxBACACKAIoQQFGG0EAIAIoAiRBAUYbQQAgAigCMEEBRhshAwwBCyACKAIgQQFHBEAgAigCMA0BIAIoAiRBAUcNASACKAIoQQFHDQELIAIoAhghAwsgAkFAayQAIAMLBQAQRgALKgEBfyMAQRBrIgEkAEHkmDUQUgRAIAEgACgCADYCABBGAAsgAUEQaiQACzMBAX8jAEEQayICJAAgACABNgIAQeSYNRBWBEAgAiAAKAIANgIAEEYACyACQRBqJAAgAAstACAAIAE2AgAgAEEEakEAOgAIIABBADYCCCAAIAFBAWo2AgQgAEEAOgAUIAALLgEBfyMAQRBrIgEkACAAQgA3AgAgAUEANgIMIABBCGpBADYCACABQRBqJAAgAAvkAwIDfwR+IAAgACAAAn8gASkDECIHQgJ/pyEGIwBB4ABrIgQkAAJAAkAgASkDGCIIIAIpAxhRBEAgASgCiAENASACKAKIAQ0BIAIpAxAgBkEBdKx8IAdCAX1Cf4V8IAOsf0IBfCEJIAIpAyAhCiAEQgE3A1ggBCAKNwNQIAQgCTcDSCAEIAcgCH43A0AgAEEBQQQgBEFAa0EAQQAQSSIFRQ0CIAUgAzYCRCAFIAE2AowBIAVBADYCiAEgBUEuNgJAIAVBADYCXCAFQQA2AlggBUEBNgJUIAVBADYCUCAFIAY2AkwgBUEANgJIIAUgAjYCkAEgBEHgAGokACAFDAMLQbzDAigCABAwGiAEQagqNgI4IARB0Sk2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMRAAAAtBvMMCKAIAEDAaIARBmyE2AhggBEHWKTYCFCAEQeQmNgIQQbjDAigCAEHL5AAgBEEQahAxEAAAC0G8wwIoAgAQMBogBEHXLzYCCCAEQcYUNgIEIARB5CY2AgBBuMMCKAIAQcvkACAEEDEQAAALIgIgAikDECACKQMYIAIpAyB+EOQBIAAgASABKQMYIAEpAxB+IAEpAyAQ5AEQWCACKQMYIAEpAyAgAikDIBDiAQvKAgEFfyMAQRBrIgUkACACQe////8DIAFrTQRAAn8gAC0AC0EHdgRAIAAoAgAMAQsgAAshByAFQQRqIgYgACABQef///8BSQR/IAUgAUEBdDYCDCAFIAEgAmo2AgQjAEEQayICJAAgBigCACAFQQxqIggoAgBJIQkgAkEQaiQAIAggBiAJGygCACICQQJPBH8gAkEEakF8cSICIAJBAWsiAiACQQJGGwVBAQtBAWoFQe////8DCxC5ASAFKAIEIQIgBSgCCBogBARAIAIgByAEEJ4BCyADIARHBEAgBEECdCIGIAJqIAYgB2ogAyAEaxCeAQsgAUEBaiIBQQJHBEAgACAHIAEQ4AELIAAgAjYCACAAIAAoAghBgICAgHhxIAUoAghB/////wdxcjYCCCAAIAAoAghBgICAgHhyNgIIIAVBEGokAA8LEE0AC5wDAQV/IwBBEGsiCCQAIAIgAUF/c0Hv////A2pNBEACfyAALQALQQd2BEAgACgCAAwBCyAACyEKIAhBBGoiCSAAIAFB5////wFJBH8gCCABQQF0NgIMIAggASACajYCBCMAQRBrIgIkACAJKAIAIAhBDGoiCygCAEkhDCACQRBqJAAgCyAJIAwbKAIAIgJBAk8EfyACQQRqQXxxIgIgAkEBayICIAJBAkYbBUEBC0EBagVB7////wMLELkBIAgoAgQhAiAIKAIIGiAEBEAgAiAKIAQQngELIAYEQCAEQQJ0IAJqIAcgBhCeAQsgAyAEIAVqIglrIQcgAyAJRwRAIARBAnQiAyACaiAGQQJ0aiADIApqIAVBAnRqIAcQngELIAFBAWoiAUECRwRAIAAgCiABEOABCyAAIAI2AgAgACAAKAIIQYCAgIB4cSAIKAIIQf////8HcXI2AgggACAAKAIIQYCAgIB4cjYCCCAAIAQgBmogB2oiADYCBCAIQQA2AgwgAiAAQQJ0aiAIKAIMNgIAIAhBEGokAA8LEE0ACw0AIAAgASABEGgQywELIAAgAEHotQM2AgAgAEHYtgM2AgAgAEEEaiABEIoDIAALPQECfyABEGgiA0ENahAyIgJBADYCCCACIAM2AgQgAiADNgIAIAJBDGoiAiABIANBAWr8CgAAIAAgAjYCAAsSACAAKAIAIAEoAgAQnwFBH3YLgAMCBX8DfiMAQdAAayIDJAACQAJAIAEpAyAiCCACKQMYUQRAIAIpAyhCAVINASACKAIAQRJHDQICfwJAIAEoAogBDQAgAigCiAENAEEADAELQQELIQcgASkDECEJIAIpAxAhCiADIAIpAyA3A0ggAyAINwNAIAMgCjcDOCADIAk3AzAgAEEAQQQgA0EwakEAQQAQSSIEQSI2AkAgBwRAIAAgBCgCACAEKAIMIARBEGpBAEEAEEkhBQsgBCABNgKMASAEIAU2AogBIAQgAjYCkAEgA0HQAGokACAEDwtBvMMCKAIAEDAaIANBkyo2AiggA0GRJTYCJCADQeQmNgIgQbjDAigCAEHL5AAgA0EgahAxEAAAC0G8wwIoAgAQMBogA0HLPjYCGCADQZIlNgIUIANB5CY2AhBBuMMCKAIAQcvkACADQRBqEDEQAAALQbzDAigCABAwGiADQfE7NgIIIANBkyU2AgQgA0HkJjYCAEG4wwIoAgBBy+QAIAMQMRAAAAsMACAAEJQCGiAAEC8LCQAgACABEJQECwcAIAAQUhoLCAAgABCmAhoLFgAgACABIAJCgICAgICAgICAfxDgAwsJACAAEEQ2AgALIwECfyAAIQEDQCABIgJBBGohASACKAIADQALIAIgAGtBAnULBwAgACgCBAswACMAQRBrIgIkAAJAIAAgAUYEQCABQQA6AHgMAQsgAkEPaiABEJsDCyACQRBqJAALJgEBfyAAKAIEIQIDQCABIAJHBEAgAkEEayECDAELCyAAIAE2AgQLSwEBfyMAQRBrIgMkAAJAAkAgAkEeSw0AIAEtAHgNACABQQE6AHgMAQsgA0EPaiACEJ0DIQELIANBEGokACAAIAI2AgQgACABNgIAC18BBX8jAEEQayIAJAAgAEH/////AzYCDCAAQf////8HNgIIIwBBEGsiASQAIABBCGoiAigCACAAQQxqIgMoAgBJIQQgAUEQaiQAIAIgAyAEGygCACEFIABBEGokACAFC0IBA38jAEEQayIBJAAgASAANgIMIAEoAgwhAiMAQRBrIgAkACAAIAI2AgwgACgCDCEDIABBEGokACABQRBqJAAgAws8AQF/IwBBEGsiAyQAIAMgARCZAzYCDCADIAIQmQM2AgggACADKAIMNgIAIAAgAygCCDYCBCADQRBqJAALCQAgAUEEEPQDCy8BAX8jAEEQayIDJAAgACACEJkBIANBADoADyABIAJqIAMtAA86AAAgA0EQaiQACxsAIAFB/////wNLBEAQRwALIAFBAnRBBBDyAwsJACAAEKkCEC8LLwECfyMAQRBrIgQkACAEIAI3AwggACABQQEgBEEIaiADEKsCIQUgBEEQaiQAIAULFQAgAEHw9QI2AgAgAEEQahAzGiAACxUAIABByPUCNgIAIABBDGoQMxogAAusAwEGfwJAIAMgAiIAa0EDSA0ACwNAAkAgACADTw0AIAQgB00NACAALAAAIgFB/wFxIQYCQCABQQBOBEBBASEBDAELIAFBQkkNASABQV9NBEAgAyAAa0ECSA0CIAAtAAFBwAFxQYABRw0CQQIhAQwBCyABQW9NBEAgAyAAa0EDSA0CIAAtAAIhCiAALQABIQECQAJAIAZB7QFHBEAgBkHgAUcNASABQeABcUGgAUYNAgwFCyABQeABcUGAAUcNBAwBCyABQcABcUGAAUcNAwsgCkHAAXFBgAFHDQJBAyEBDAELIAFBdEsNASADIABrQQRIDQEgAC0AAyEIIAAtAAIhCSAALQABIQUCQAJAAkACQCAGQfABaw4FAAICAgECCyAFQfAAakH/AXFBME8NBAwCCyAFQfABcUGAAUcNAwwBCyAFQcABcUGAAUcNAgsgCUHAAXFBgAFHDQEgCEHAAXFBgAFHDQFBBCEBIAhBP3EgCUEGdEHAH3EgBkESdEGAgPAAcSAFQT9xQQx0cnJyQf//wwBLDQELIAdBAWohByAAIAFqIQAMAQsLIAAgAmsLzwQBBn8jAEEQayIAJAAgACACNgIMIAAgBTYCCAJ/IAAgAjYCDCAAIAU2AggCQAJAA0ACQCAAKAIMIgEgA08NACAAKAIIIgwgBk8NACABLAAAIgVB/wFxIQICQCAFQQBOBEAgAkH//8MATQRAQQEhBQwCC0ECDAYLQQIhCiAFQUJJDQMgBUFfTQRAIAMgAWtBAkgNBSABLQABIghBwAFxQYABRw0EQQIhBSAIQT9xIAJBBnRBwA9xciECDAELIAVBb00EQCADIAFrQQNIDQUgAS0AAiEJIAEtAAEhCAJAAkAgAkHtAUcEQCACQeABRw0BIAhB4AFxQaABRg0CDAcLIAhB4AFxQYABRg0BDAYLIAhBwAFxQYABRw0FCyAJQcABcUGAAUcNBEEDIQUgCUE/cSACQQx0QYDgA3EgCEE/cUEGdHJyIQIMAQsgBUF0Sw0DIAMgAWtBBEgNBCABLQADIQkgAS0AAiELIAEtAAEhCAJAAkACQAJAIAJB8AFrDgUAAgICAQILIAhB8ABqQf8BcUEwSQ0CDAYLIAhB8AFxQYABRg0BDAULIAhBwAFxQYABRw0ECyALQcABcUGAAUcNAyAJQcABcUGAAUcNA0EEIQUgCUE/cSALQQZ0QcAfcSACQRJ0QYCA8ABxIAhBP3FBDHRycnIiAkH//8MASw0DCyAMIAI2AgAgACABIAVqNgIMIAAgACgCCEEEajYCCAwBCwsgASADSSEKCyAKDAELQQELIQ0gBCAAKAIMNgIAIAcgACgCCDYCACAAQRBqJAAgDQuRBAEBfyMAQRBrIgAkACAAIAI2AgwgACAFNgIIAn8gACACNgIMIAAgBTYCCCAAKAIMIQECQANAIAEgA08EQEEAIQIMAgtBAiECIAEoAgAiAUH//8MASw0BIAFBgHBxQYCwA0YNAQJAAkAgAUH/AE0EQEEBIQIgBiAAKAIIIgVrQQBMDQQgACAFQQFqNgIIIAUgAToAAAwBCyABQf8PTQRAIAYgACgCCCICa0ECSA0CIAAgAkEBajYCCCACIAFBBnZBwAFyOgAAIAAgACgCCCICQQFqNgIIIAIgAUE/cUGAAXI6AAAMAQsgBiAAKAIIIgJrIQUgAUH//wNNBEAgBUEDSA0CIAAgAkEBajYCCCACIAFBDHZB4AFyOgAAIAAgACgCCCICQQFqNgIIIAIgAUEGdkE/cUGAAXI6AAAgACAAKAIIIgJBAWo2AgggAiABQT9xQYABcjoAAAwBCyAFQQRIDQEgACACQQFqNgIIIAIgAUESdkHwAXI6AAAgACAAKAIIIgJBAWo2AgggAiABQQx2QT9xQYABcjoAACAAIAAoAggiAkEBajYCCCACIAFBBnZBP3FBgAFyOgAAIAAgACgCCCICQQFqNgIIIAIgAUE/cUGAAXI6AAALIAAgACgCDEEEaiIBNgIMDAELC0EBDAELIAILIQggBCAAKAIMNgIAIAcgACgCCDYCACAAQRBqJAAgCAu3AwEFfwJAIAMgAiIAa0EDSA0ACwNAAkAgACADTw0AIAQgBk0NAAJ/IABBAWogAC0AACIBwEEATg0AGiABQcIBSQ0BIAFB3wFNBEAgAyAAa0ECSA0CIAAtAAFBwAFxQYABRw0CIABBAmoMAQsgAUHvAU0EQCADIABrQQNIDQIgAC0AAiEJIAAtAAEhBQJAAkAgAUHtAUcEQCABQeABRw0BIAVB4AFxQaABRg0CDAULIAVB4AFxQYABRw0EDAELIAVBwAFxQYABRw0DCyAJQcABcUGAAUcNAiAAQQNqDAELIAFB9AFLDQEgAyAAa0EESA0BIAQgBmtBAkkNASAALQADIQcgAC0AAiEIIAAtAAEhBQJAAkACQAJAIAFB8AFrDgUAAgICAQILIAVB8ABqQf8BcUEwTw0EDAILIAVB8AFxQYABRw0DDAELIAVBwAFxQYABRw0CCyAIQcABcUGAAUcNASAHQcABcUGAAUcNASAHQT9xIAhBBnRBwB9xIAFBEnRBgIDwAHEgBUE/cUEMdHJyckH//8MASw0BIAZBAWohBiAAQQRqCyEAIAZBAWohBgwBCwsgACACawuoBQEFfyMAQRBrIgAkACAAIAI2AgwgACAFNgIIAn8gACACNgIMIAAgBTYCCAJAAkACQANAAkAgACgCDCIBIANPDQAgACgCCCIFIAZPDQBBAiEKIAACfyABLQAAIgLAQQBOBEAgBSACOwEAIAFBAWoMAQsgAkHCAUkNBSACQd8BTQRAIAMgAWtBAkgNBSABLQABIghBwAFxQYABRw0EIAUgCEE/cSACQQZ0QcAPcXI7AQAgAUECagwBCyACQe8BTQRAIAMgAWtBA0gNBSABLQACIQkgAS0AASEIAkACQCACQe0BRwRAIAJB4AFHDQEgCEHgAXFBoAFGDQIMBwsgCEHgAXFBgAFGDQEMBgsgCEHAAXFBgAFHDQULIAlBwAFxQYABRw0EIAUgCUE/cSAIQT9xQQZ0IAJBDHRycjsBACABQQNqDAELIAJB9AFLDQVBASEKIAMgAWtBBEgNAyABLQADIQkgAS0AAiEIIAEtAAEhAQJAAkACQAJAIAJB8AFrDgUAAgICAQILIAFB8ABqQf8BcUEwTw0IDAILIAFB8AFxQYABRw0HDAELIAFBwAFxQYABRw0GCyAIQcABcUGAAUcNBSAJQcABcUGAAUcNBSAGIAVrQQRIDQNBAiEKIAlBP3EiCSAIQQZ0IgtBwB9xIAFBDHRBgOAPcSACQQdxIgJBEnRycnJB///DAEsNAyAFIAhBBHZBA3EgAUECdCIBQcABcSACQQh0ciABQTxxcnJBwP8AakGAsANyOwEAIAAgBUECajYCCCAFIAtBwAdxIAlyQYC4A3I7AQIgACgCDEEEags2AgwgACAAKAIIQQJqNgIIDAELCyABIANJIQoLIAoMAgtBAQwBC0ECCyEMIAQgACgCDDYCACAHIAAoAgg2AgAgAEEQaiQAIAwL6gUBAn8jAEEQayIAJAAgACACNgIMIAAgBTYCCAJ/IAAgAjYCDCAAIAU2AgggACgCDCECAkACQANAIAIgA08EQEEAIQUMAwtBAiEFAkACQCACLwEAIgFB/wBNBEBBASEFIAYgACgCCCICa0EATA0FIAAgAkEBajYCCCACIAE6AAAMAQsgAUH/D00EQCAGIAAoAggiAmtBAkgNBCAAIAJBAWo2AgggAiABQQZ2QcABcjoAACAAIAAoAggiAkEBajYCCCACIAFBP3FBgAFyOgAADAELIAFB/68DTQRAIAYgACgCCCICa0EDSA0EIAAgAkEBajYCCCACIAFBDHZB4AFyOgAAIAAgACgCCCICQQFqNgIIIAIgAUEGdkE/cUGAAXI6AAAgACAAKAIIIgJBAWo2AgggAiABQT9xQYABcjoAAAwBCyABQf+3A00EQEEBIQUgAyACa0EESA0FIAIvAQIiCEGA+ANxQYC4A0cNAiAGIAAoAghrQQRIDQUgCEH/B3EgAUEKdEGA+ANxIAFBwAdxIgVBCnRyckH//z9LDQIgACACQQJqNgIMIAAgACgCCCICQQFqNgIIIAIgBUEGdkEBaiICQQJ2QfABcjoAACAAIAAoAggiBUEBajYCCCAFIAJBBHRBMHEgAUECdkEPcXJBgAFyOgAAIAAgACgCCCICQQFqNgIIIAIgCEEGdkEPcSABQQR0QTBxckGAAXI6AAAgACAAKAIIIgFBAWo2AgggASAIQT9xQYABcjoAAAwBCyABQYDAA0kNBCAGIAAoAggiAmtBA0gNAyAAIAJBAWo2AgggAiABQQx2QeABcjoAACAAIAAoAggiAkEBajYCCCACIAFBBnZBP3FBgAFyOgAAIAAgACgCCCICQQFqNgIIIAIgAUE/cUGAAXI6AAALIAAgACgCDEECaiICNgIMDAELC0ECDAILQQEMAQsgBQshCSAEIAAoAgw2AgAgByAAKAIINgIAIABBEGokACAJCz4BA38jAEEQayIBJAAgASAANgIMIAFBCGogAUEMahB/IQNBBEEBIwMoAmAoAgAbIQIgAxB+IAFBEGokACACCzwBAn8jAEEQayIFJAAgBSAENgIMIAVBCGogBUEMahB/IQYgACABIAIgAxDwASEAIAYQfiAFQRBqJAAgAAsSACAEIAI2AgAgByAFNgIAQQMLKAEBfyAAQdzsAjYCAAJAIAAoAggiAUUNACAALQAMRQ0AIAEQLwsgAAsEACABC0ABAn8gACgCACgCACIAKAIAIAAoAggiAkEBdWohASAAKAIEIQAgASACQQFxBH8gASgCACAAaigCAAUgAAsRAQALLgAgASAAQQhqIgAoAgQgACgCACIAa0ECdUkEfyABQQJ0IABqKAIAQQBHBUEACwvLAQEDfyAAQcjsAjYCACAAQQhqIQMDQCACIAAoAgwgACgCCCIBa0ECdUkEQCACQQJ0IAFqKAIAIgEEQCABQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsLIAJBAWohAgwBCwsgAEGYAWoQMxojAEEQayICJAAgAiADNgIMIAIoAgwiASgCBBogASgCCBogASgCABogASgCAARAIAEQsAMgAigCDEEQaiACKAIMIgEoAgAgASgCCCABKAIAa0ECdRCVAwsgAkEQaiQAIAALDAAgACAAKAIAEJYDC3ABAX8jAEEQayICJAAgAiAANgIEIAIgACgCBCIANgIIIAIgACABQQJ0ajYCDCACKAIIIQEgAigCDCEAA0AgACABRgRAIAIoAgQgAigCCDYCBCACQRBqJAAFIAFBADYCACACIAFBBGoiATYCCAwBCwsLIAAgAEGY9QI2AgAgACgCCBBERwRAIAAoAggQ4wMLIAALBABBfwvgAQEIfyMAQRBrIgUkACMAQSBrIgMkACADQRhqIAAgARCaAyADQRBqIANBDGogAygCGCADKAIcIAIQvAIgAygCECEEIwBBEGsiASQAIAEgADYCDCABQQxqIgAhCiAEIQcgACgCACEEIwBBEGsiACQAIAAgBDYCDCAKIAchCSAAKAIMIQYgAEEQaiQAIAkgBmtBAnUQrgIhACABQRBqJAAgAyAANgIMIAMgAiADKAIUIAJrajYCCCAFIAMoAgw2AgggBSADKAIINgIMIANBIGokACAFKAIMIQggBUEQaiQAIAgL8gcBDH8jAEEQayITJAAgAiAANgIAIANBgARxIRUgB0ECdCEWA0AgFEEERgRAAn8gDS0AC0EHdgRAIA0oAgQMAQsgDS0AC0H/AHELQQFLBEAgEyANEHU2AgwgAiATQQxqQQEQrgIgDRCbASACKAIAELQDNgIACyADQbABcSIDQRBHBEAgASADQSBGBH8gAigCAAUgAAs2AgALIBNBEGokAAUCQAJAAkACQAJAAkAgCCAUaiwAAA4FAAEDAgQFCyABIAIoAgA2AgAMBAsgASACKAIANgIAIAZBICAGKAIAKAIsEQQAIQcgAiACKAIAIg9BBGo2AgAgDyAHNgIADAMLAn8gDS0AC0EHdgRAIA0oAgQMAQsgDS0AC0H/AHELRQ0CAn8gDS0AC0EHdgRAIA0oAgAMAQsgDQsoAgAhByACIAIoAgAiD0EEajYCACAPIAc2AgAMAgsCfyAMLQALQQd2BEAgDCgCBAwBCyAMLQALQf8AcQtFIRkgFUUNASAZDQEgAiAMEHUgDBCbASACKAIAELQDNgIADAELIAIoAgAhGiAEIBZqIgQhBwNAAkAgBSAHTQ0AIAZBwAAgBygCACAGKAIAKAIMEQMARQ0AIAdBBGohBwwBCwsgDkEASgRAIAIoAgAhDyAOIRADQAJAIAQgB08NACAQRQ0AIBBBAWshECAHQQRrIgcoAgAhESACIA9BBGoiEjYCACAPIBE2AgAgEiEPDAELCwJAIBBFBEBBACERDAELIAZBMCAGKAIAKAIsEQQAIREgAigCACEPCwNAIA9BBGohEiAQQQBKBEAgDyARNgIAIBBBAWshECASIQ8MAQsLIAIgEjYCACAPIAk2AgALAkAgBCAHRgRAIAZBMCAGKAIAKAIsEQQAIQ8gAiACKAIAIhBBBGoiBzYCACAQIA82AgAMAQsCfyALLQALQQd2BEAgCygCBAwBCyALLQALQf8AcQsEfwJ/IAstAAtBB3YEQCALKAIADAELIAsLLAAABUF/CyERQQAhD0EAIRADQCAEIAdHBEACQCAPIBFHBEAgDyESDAELIAIgAigCACISQQRqNgIAIBIgCjYCAEEAIRICfyALLQALQQd2BEAgCygCBAwBCyALLQALQf8AcQsgEEEBaiIQTQRAIA8hEQwBCwJ/IAstAAtBB3YEQCALKAIADAELIAsLIBBqLQAAQf8ARgRAQX8hEQwBCwJ/IAstAAtBB3YEQCALKAIADAELIAsLIBBqLAAAIRELIAdBBGsiBygCACEPIAIgAigCACIYQQRqNgIAIBggDzYCACASQQFqIQ8MAQsLIAIoAgAhBwsgGiAHEOoBCyAUQQFqIRQMAQsLC+cDAQF/IwBBEGsiCiQAIAkCfyAABEAgAhC7AyEAAkAgAQRAIApBBGoiASAAIAAoAgAoAiwRAgAgAyAKKAIENgAAIAEgACAAKAIAKAIgEQIADAELIApBBGoiASAAIAAoAgAoAigRAgAgAyAKKAIENgAAIAEgACAAKAIAKAIcEQIACyAIIAEQiAEgARBQGiAEIAAgACgCACgCDBEAADYCACAFIAAgACgCACgCEBEAADYCACAKQQRqIgEgACAAKAIAKAIUEQIAIAYgARBlIAEQMxogASAAIAAoAgAoAhgRAgAgByABEIgBIAEQUBogACAAKAIAKAIkEQAADAELIAIQugMhAAJAIAEEQCAKQQRqIgEgACAAKAIAKAIsEQIAIAMgCigCBDYAACABIAAgACgCACgCIBECAAwBCyAKQQRqIgEgACAAKAIAKAIoEQIAIAMgCigCBDYAACABIAAgACgCACgCHBECAAsgCCABEIgBIAEQUBogBCAAIAAoAgAoAgwRAAA2AgAgBSAAIAAoAgAoAhARAAA2AgAgCkEEaiIBIAAgACgCACgCFBECACAGIAEQZSABEDMaIAEgACAAKAIAKAIYEQIAIAcgARCIASABEFAaIAAgACgCACgCJBEAAAs2AgAgCkEQaiQAC90BAQh/IwBBEGsiBSQAIwBBIGsiAyQAIANBGGogACABEJoDIANBEGogA0EMaiADKAIYIAMoAhwgAhC8AiADKAIQIQQjAEEQayIBJAAgASAANgIMIAFBDGoiACEKIAQhByAAKAIAIQQjAEEQayIAJAAgACAENgIMIAogByEJIAAoAgwhBiAAQRBqJAAgCSAGaxCvAiEAIAFBEGokACADIAA2AgwgAyACIAMoAhQgAmtqNgIIIAUgAygCDDYCCCAFIAMoAgg2AgwgA0EgaiQAIAUoAgwhCCAFQRBqJAAgCAveBwEMfyMAQRBrIhMkACACIAA2AgAgA0GABHEhFgNAIBRBBEYEQAJ/IA0tAAtBB3YEQCANKAIEDAELIA0tAAtB/wBxC0EBSwRAIBMgDRB1NgIMIAIgE0EMakEBEK8CIA0QnQEgAigCABC3AzYCAAsgA0GwAXEiA0EQRwRAIAEgA0EgRgR/IAIoAgAFIAALNgIACyATQRBqJAAFAkACQAJAAkACQAJAIAggFGosAAAOBQABAwIEBQsgASACKAIANgIADAQLIAEgAigCADYCACAGQSAgBigCACgCHBEEACEPIAIgAigCACIQQQFqNgIAIBAgDzoAAAwDCwJ/IA0tAAtBB3YEQCANKAIEDAELIA0tAAtB/wBxC0UNAgJ/IA0tAAtBB3YEQCANKAIADAELIA0LLQAAIQ8gAiACKAIAIhBBAWo2AgAgECAPOgAADAILAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRSEZIBZFDQEgGQ0BIAIgDBB1IAwQnQEgAigCABC3AzYCAAwBCyACKAIAIRogBCAHaiIEIREDQAJAIAUgEU0NACARLAAAIg9BAE4EfyAGKAIIIA9B/wFxQQJ0aigCAEHAAHFBAEcFQQALRQ0AIBFBAWohEQwBCwsgDiIPQQBKBEADQAJAIAQgEU8NACAPRQ0AIA9BAWshDyARQQFrIhEtAAAhECACIAIoAgAiEkEBajYCACASIBA6AAAMAQsLIA8EfyAGQTAgBigCACgCHBEEAAVBAAshEgNAIAIgAigCACIQQQFqNgIAIA9BAEoEQCAQIBI6AAAgD0EBayEPDAELCyAQIAk6AAALAkAgBCARRgRAIAZBMCAGKAIAKAIcEQQAIQ8gAiACKAIAIhBBAWo2AgAgECAPOgAADAELAn8gCy0AC0EHdgRAIAsoAgQMAQsgCy0AC0H/AHELBH8CfyALLQALQQd2BEAgCygCAAwBCyALCywAAAVBfwshEkEAIQ9BACEQA0AgBCARRg0BAkAgDyASRwRAIA8hFQwBCyACIAIoAgAiEkEBajYCACASIAo6AABBACEVAn8gCy0AC0EHdgRAIAsoAgQMAQsgCy0AC0H/AHELIBBBAWoiEE0EQCAPIRIMAQsCfyALLQALQQd2BEAgCygCAAwBCyALCyAQai0AAEH/AEYEQEF/IRIMAQsCfyALLQALQQd2BEAgCygCAAwBCyALCyAQaiwAACESCyARQQFrIhEtAAAhDyACIAIoAgAiGEEBajYCACAYIA86AAAgFUEBaiEPDAALAAsgGiACKAIAEKwBCyAUQQFqIRQMAQsLC+MDAQF/IwBBEGsiCiQAIAkCfyAABEAgAhDAAyEAAkAgAQRAIApBBGoiASAAIAAoAgAoAiwRAgAgAyAKKAIENgAAIAEgACAAKAIAKAIgEQIADAELIApBBGoiASAAIAAoAgAoAigRAgAgAyAKKAIENgAAIAEgACAAKAIAKAIcEQIACyAIIAEQZSABEDMaIAQgACAAKAIAKAIMEQAAOgAAIAUgACAAKAIAKAIQEQAAOgAAIApBBGoiASAAIAAoAgAoAhQRAgAgBiABEGUgARAzGiABIAAgACgCACgCGBECACAHIAEQZSABEDMaIAAgACgCACgCJBEAAAwBCyACEL8DIQACQCABBEAgCkEEaiIBIAAgACgCACgCLBECACADIAooAgQ2AAAgASAAIAAoAgAoAiARAgAMAQsgCkEEaiIBIAAgACgCACgCKBECACADIAooAgQ2AAAgASAAIAAoAgAoAhwRAgALIAggARBlIAEQMxogBCAAIAAoAgAoAgwRAAA6AAAgBSAAIAAoAgAoAhARAAA6AAAgCkEEaiIBIAAgACgCACgCFBECACAGIAEQZSABEDMaIAEgACAAKAIAKAIYEQIAIAcgARBlIAEQMxogACAAKAIAKAIkEQAACzYCACAKQRBqJAALCgAgAEGwiTUQeQsKACAAQbiJNRB5Cx8BAX8gASgCABD7AyECIAAgASgCADYCBCAAIAI2AgALwBgBCX8jAEGQBGsiCyQAIAsgCjYCiAQgCyABNgKMBAJAIAAgC0GMBGoQQgRAIAUgBSgCAEEEcjYCAEEAIQAMAQsgC0H3ATYCSCALIAtB6ABqIAtB8ABqIAtByABqIg8QUSIRKAIAIgE2AmQgCyABQZADajYCYCMAQRBrIgEkACAPQgA3AgAgD0EANgIIIAFBEGokACMAQRBrIgEkACALQTxqIg5CADcCACAOQQA2AgggAUEQaiQAIwBBEGsiASQAIAtBMGoiDUIANwIAIA1BADYCCCABQRBqJAAjAEEQayIBJAAgC0EkaiIMQgA3AgAgDEEANgIIIAFBEGokACMAQRBrIgEkACALQRhqIhBCADcCACAQQQA2AgggAUEQaiQAIwBBEGsiCiQAIAsCfyACBEAgCkEEaiICIAMQuwMiASABKAIAKAIsEQIAIAsgCigCBDYAXCACIAEgASgCACgCIBECACAMIAIQiAEgAhBQGiACIAEgASgCACgCHBECACANIAIQiAEgAhBQGiALIAEgASgCACgCDBEAADYCWCALIAEgASgCACgCEBEAADYCVCACIAEgASgCACgCFBECACAPIAIQZSACEDMaIAIgASABKAIAKAIYEQIAIA4gAhCIASACEFAaIAEgASgCACgCJBEAAAwBCyAKQQRqIgIgAxC6AyIBIAEoAgAoAiwRAgAgCyAKKAIENgBcIAIgASABKAIAKAIgEQIAIAwgAhCIASACEFAaIAIgASABKAIAKAIcEQIAIA0gAhCIASACEFAaIAsgASABKAIAKAIMEQAANgJYIAsgASABKAIAKAIQEQAANgJUIAIgASABKAIAKAIUEQIAIA8gAhBlIAIQMxogAiABIAEoAgAoAhgRAgAgDiACEIgBIAIQUBogASABKAIAKAIkEQAACzYCFCAKQRBqJAAgCSAIKAIANgIAIARBgARxIRJBACEDQQAhAQNAIAEhAgJAAkACQAJAIANBBEYNACAAIAtBjARqEEINAEEAIQoCQAJAAkACQAJAAkAgC0HcAGogA2osAAAOBQEABAMFCQsgA0EDRg0HIAdBAQJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQoAgALIAcoAgAoAgwRAwAEQCALQQxqIAAQvAMgECALKAIMEKECDAILIAUgBSgCAEEEcjYCAEEAIQAMBgsgA0EDRg0GCwNAIAAgC0GMBGoQQg0GIAdBAQJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQoAgALIAcoAgAoAgwRAwBFDQYgC0EMaiAAELwDIBAgCygCDBChAgwACwALAkACfyANLQALQQd2BEAgDSgCBAwBCyANLQALQf8AcQtFDQACfyAAKAIAIgEoAgwiBCABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAEKAIACwJ/IA0tAAtBB3YEQCANKAIADAELIA0LKAIARw0AIAAQXBogBkEAOgAAIA0gAgJ/IA0tAAtBB3YEQCANKAIEDAELIA0tAAtB/wBxC0EBSxshAQwGCwJAAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRQ0AAn8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBCgCAAsCfyAMLQALQQd2BEAgDCgCAAwBCyAMCygCAEcNACAAEFwaIAZBAToAACAMIAICfyAMLQALQQd2BEAgDCgCBAwBCyAMLQALQf8AcQtBAUsbIQEMBgsCQAJ/IA0tAAtBB3YEQCANKAIEDAELIA0tAAtB/wBxC0UNAAJ/IAwtAAtBB3YEQCAMKAIEDAELIAwtAAtB/wBxC0UNACAFIAUoAgBBBHI2AgBBACEADAQLAn8gDS0AC0EHdgRAIA0oAgQMAQsgDS0AC0H/AHELRQRAAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRQ0FCyAGAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRToAAAwECwJAIANBAkkNACACDQAgEg0AQQAhASADQQJGIAstAF9BAEdxRQ0FCyALIA4QdTYCCCALIAsoAgg2AgwCQCADRQ0AIAMgC2otAFtBAUsNAANAAkAgCyAOEJsBNgIIIAsoAgwiASALKAIIRg0AIAdBASABKAIAIAcoAgAoAgwRAwBFDQAgCyALKAIMQQRqNgIMDAELCyALIA4QdTYCCAJ/IBAtAAtBB3YEQCAQKAIEDAELIBAtAAtB/wBxCyALKAIMIAtBCGoiASgCAGtBAnUiBE8EQCALIBAQmwE2AgggAUEAIARrEK4CIQQgEBCbASEKIA4QdSETIwBBEGsiASQAIAEgCjYCCCABIAQ2AgwgASATNgIEA0ACQCABKAIMIgQgASgCCEciCkUNACAEKAIAIAEoAgQoAgBHDQAgASAEQQRqNgIMIAEgASgCBEEEajYCBAwBCwsgAUEQaiQAIApFDQELIAsgDhB1NgIEIAsgCygCBDYCCCALIAsoAgg2AgwLIAsgCygCDDYCCANAAkAgCyAOEJsBNgIEIAsoAgggCygCBEYNACAAIAtBjARqEEINAAJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQoAgALIAsoAggoAgBHDQAgABBcGiALIAsoAghBBGo2AggMAQsLIBJFDQMgCyAOEJsBNgIEIAsoAgggCygCBEYNAyAFIAUoAgBBBHI2AgBBACEADAILA0ACQCAAIAtBjARqEEINAAJ/IAdBwAACfyAAKAIAIgEoAgwiBCABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAEKAIACyIBIAcoAgAoAgwRAwAEQCAJKAIAIgQgCygCiARGBEAgCCAJIAtBiARqELsBIAkoAgAhBAsgCSAEQQRqNgIAIAQgATYCACAKQQFqDAELAn8gDy0AC0EHdgRAIA8oAgQMAQsgDy0AC0H/AHELRQ0BIApFDQEgASALKAJURw0BIAsoAmQiASALKAJgRgRAIBEgC0HkAGogC0HgAGoQuwEgCygCZCEBCyALIAFBBGo2AmQgASAKNgIAQQALIQogABBcGgwBCwsCQCALKAJkIgEgESgCAEYNACAKRQ0AIAsoAmAgAUYEQCARIAtB5ABqIAtB4ABqELsBIAsoAmQhAQsgCyABQQRqNgJkIAEgCjYCAAsCQCALKAIUQQBMDQACQCAAIAtBjARqEEJFBEACfyAAKAIAIgEoAgwiBCABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAEKAIACyALKAJYRg0BCyAFIAUoAgBBBHI2AgBBACEADAMLA0AgABBcGiALKAIUQQBMDQECQCAAIAtBjARqEEJFBEAgB0HAAAJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQoAgALIAcoAgAoAgwRAwANAQsgBSAFKAIAQQRyNgIAQQAhAAwECyAJKAIAIAsoAogERgRAIAggCSALQYgEahC7AQsCfyAAKAIAIgEoAgwiBCABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAEKAIACyEBIAkgCSgCACIEQQRqNgIAIAQgATYCACALIAsoAhRBAWs2AhQMAAsACyACIQEgCCgCACAJKAIARw0DIAUgBSgCAEEEcjYCAEEAIQAMAQsCQCACRQ0AQQEhCgNAAn8gAi0AC0EHdgRAIAIoAgQMAQsgAi0AC0H/AHELIApNDQECQCAAIAtBjARqEEJFBEACfyAAKAIAIgEoAgwiAyABKAIQRgRAIAEgASgCACgCJBEAAAwBCyADKAIACwJ/IAItAAtBB3YEQCACKAIADAELIAILIApBAnRqKAIARg0BCyAFIAUoAgBBBHI2AgBBACEADAMLIAAQXBogCkEBaiEKDAALAAtBASEAIBEoAgAgCygCZEYNAEEAIQAgC0EANgIMIA8gESgCACALKAJkIAtBDGoQZCALKAIMBEAgBSAFKAIAQQRyNgIADAELQQEhAAsgEBBQGiAMEFAaIA0QUBogDhBQGiAPEDMaIBEoAgAhASARQQA2AgAgAQRAIAEgESgCBBEBAAsMAwsgAiEBCyADQQFqIQMMAAsACyALQZAEaiQAIAALOQECfyABKAIAIQMgAUEANgIAIAAoAgAhAiAAIAM2AgAgAgRAIAIgACgCBBEBAAsgACABKAIENgIECwoAIABBoIk1EHkLCgAgAEGoiTUQeQvkAQEGfyMAQRBrIgUkACAAKAIEIQNBAQJ/IAIoAgAgACgCAGsiBEH/////B0kEQCAEQQF0DAELQX8LIgQgBEEBTRshBCABKAIAIQcgACgCACEIIANB9wFGBH9BAAUgACgCAAsgBBDEASIGBEAgA0H3AUcEQCAAKAIAGiAAQQA2AgALIAVB9gE2AgQgACAFQQhqIAYgBUEEahBRIgMQvgMgAygCACEGIANBADYCACAGBEAgBiADKAIEEQEACyABIAAoAgAgByAIa2o2AgAgAiAEIAAoAgBqNgIAIAVBEGokAA8LEEYACyABAX8gASgCABD9A8AhAiAAIAEoAgA2AgQgACACOgAAC6kZAQl/IwBBkARrIgskACALIAo2AogEIAsgATYCjAQCQCAAIAtBjARqEEMEQCAFIAUoAgBBBHI2AgBBACEADAELIAtB9wE2AkwgCyALQegAaiALQfAAaiALQcwAaiIPEFEiESgCACIBNgJkIAsgAUGQA2o2AmAjAEEQayIBJAAgD0IANwIAIA9BADYCCCABQRBqJAAjAEEQayIBJAAgC0FAayIOQgA3AgAgDkEANgIIIAFBEGokACMAQRBrIgEkACALQTRqIg1CADcCACANQQA2AgggAUEQaiQAIwBBEGsiASQAIAtBKGoiDEIANwIAIAxBADYCCCABQRBqJAAjAEEQayIBJAAgC0EcaiIQQgA3AgAgEEEANgIIIAFBEGokACMAQRBrIgokACALAn8gAgRAIApBBGoiAiADEMADIgEgASgCACgCLBECACALIAooAgQ2AFwgAiABIAEoAgAoAiARAgAgDCACEGUgAhAzGiACIAEgASgCACgCHBECACANIAIQZSACEDMaIAsgASABKAIAKAIMEQAAOgBbIAsgASABKAIAKAIQEQAAOgBaIAIgASABKAIAKAIUEQIAIA8gAhBlIAIQMxogAiABIAEoAgAoAhgRAgAgDiACEGUgAhAzGiABIAEoAgAoAiQRAAAMAQsgCkEEaiICIAMQvwMiASABKAIAKAIsEQIAIAsgCigCBDYAXCACIAEgASgCACgCIBECACAMIAIQZSACEDMaIAIgASABKAIAKAIcEQIAIA0gAhBlIAIQMxogCyABIAEoAgAoAgwRAAA6AFsgCyABIAEoAgAoAhARAAA6AFogAiABIAEoAgAoAhQRAgAgDyACEGUgAhAzGiACIAEgASgCACgCGBECACAOIAIQZSACEDMaIAEgASgCACgCJBEAAAs2AhggCkEQaiQAIAkgCCgCADYCACAEQYAEcSESQQAhA0EAIQEDQCABIQICQAJAAkACQCADQQRGDQAgACALQYwEahBDDQBBACEKAkACQAJAAkACQAJAIAtB3ABqIANqLAAADgUBAAQDBQkLIANBA0YNBwJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQtAAALwCIBQQBOBH8gBygCCCABQf8BcUECdGooAgBBAXEFQQALBEAgC0EQaiAAEMIDIBAgCywAEBCiAgwCCyAFIAUoAgBBBHI2AgBBACEADAYLIANBA0YNBgsDQCAAIAtBjARqEEMNBgJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQtAAALwCIBQQBOBH8gBygCCCABQf8BcUECdGooAgBBAXEFQQALRQ0GIAtBEGogABDCAyAQIAssABAQogIMAAsACwJAAn8gDS0AC0EHdgRAIA0oAgQMAQsgDS0AC0H/AHELRQ0AAn8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBC0AAAvAQf8BcQJ/IA0tAAtBB3YEQCANKAIADAELIA0LLQAARw0AIAAQXRogBkEAOgAAIA0gAgJ/IA0tAAtBB3YEQCANKAIEDAELIA0tAAtB/wBxC0EBSxshAQwGCwJAAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRQ0AAn8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBC0AAAvAQf8BcQJ/IAwtAAtBB3YEQCAMKAIADAELIAwLLQAARw0AIAAQXRogBkEBOgAAIAwgAgJ/IAwtAAtBB3YEQCAMKAIEDAELIAwtAAtB/wBxC0EBSxshAQwGCwJAAn8gDS0AC0EHdgRAIA0oAgQMAQsgDS0AC0H/AHELRQ0AAn8gDC0AC0EHdgRAIAwoAgQMAQsgDC0AC0H/AHELRQ0AIAUgBSgCAEEEcjYCAEEAIQAMBAsCfyANLQALQQd2BEAgDSgCBAwBCyANLQALQf8AcQtFBEACfyAMLQALQQd2BEAgDCgCBAwBCyAMLQALQf8AcQtFDQULIAYCfyAMLQALQQd2BEAgDCgCBAwBCyAMLQALQf8AcQtFOgAADAQLAkAgA0ECSQ0AIAINACASDQBBACEBIANBAkYgCy0AX0EAR3FFDQULIAsgDhB1NgIMIAsgCygCDDYCEAJAIANFDQAgAyALai0AW0EBSw0AA0ACQCALIA4QnQE2AgwgCygCECIBIAsoAgxGDQAgASwAACIBQQBOBH8gBygCCCABQf8BcUECdGooAgBBAXEFQQALRQ0AIAsgCygCEEEBajYCEAwBCwsgCyAOEHU2AgwCfyAQLQALQQd2BEAgECgCBAwBCyAQLQALQf8AcQsgCygCECALQQxqIgEoAgBrIgRPBEAgCyAQEJ0BNgIMIAFBACAEaxCvAiEEIBAQnQEhCiAOEHUhEyMAQRBrIgEkACABIAo2AgggASAENgIMIAEgEzYCBANAAkAgASgCDCIEIAEoAghHIgpFDQAgBC0AACABKAIELQAARw0AIAEgBEEBajYCDCABIAEoAgRBAWo2AgQMAQsLIAFBEGokACAKRQ0BCyALIA4QdTYCCCALIAsoAgg2AgwgCyALKAIMNgIQCyALIAsoAhA2AgwDQAJAIAsgDhCdATYCCCALKAIMIAsoAghGDQAgACALQYwEahBDDQACfyAAKAIAIgEoAgwiBCABKAIQRgRAIAEgASgCACgCJBEAAAwBCyAELQAAC8BB/wFxIAsoAgwtAABHDQAgABBdGiALIAsoAgxBAWo2AgwMAQsLIBJFDQMgCyAOEJ0BNgIIIAsoAgwgCygCCEYNAyAFIAUoAgBBBHI2AgBBACEADAILA0ACQCAAIAtBjARqEEMNAAJ/An8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBC0AAAvAIgFBAE4EfyAHKAIIIAFB/wFxQQJ0aigCAEHAAHEFQQALBEAgCSgCACIEIAsoAogERgRAIAggCSALQYgEahDBAyAJKAIAIQQLIAkgBEEBajYCACAEIAE6AAAgCkEBagwBCwJ/IA8tAAtBB3YEQCAPKAIEDAELIA8tAAtB/wBxC0UNASAKRQ0BIAstAFogAUH/AXFHDQEgCygCZCIBIAsoAmBGBEAgESALQeQAaiALQeAAahC7ASALKAJkIQELIAsgAUEEajYCZCABIAo2AgBBAAshCiAAEF0aDAELCwJAIAsoAmQiASARKAIARg0AIApFDQAgCygCYCABRgRAIBEgC0HkAGogC0HgAGoQuwEgCygCZCEBCyALIAFBBGo2AmQgASAKNgIACwJAIAsoAhhBAEwNAAJAIAAgC0GMBGoQQ0UEQAJ/IAAoAgAiASgCDCIEIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAQtAAALwEH/AXEgCy0AW0YNAQsgBSAFKAIAQQRyNgIAQQAhAAwDCwNAIAAQXRogCygCGEEATA0BAkAgACALQYwEahBDRQRAAn8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBC0AAAvAIgFBAE4EfyAHKAIIIAFB/wFxQQJ0aigCAEHAAHEFQQALDQELIAUgBSgCAEEEcjYCAEEAIQAMBAsgCSgCACALKAKIBEYEQCAIIAkgC0GIBGoQwQMLAn8gACgCACIBKAIMIgQgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgBC0AAAvAIQEgCSAJKAIAIgRBAWo2AgAgBCABOgAAIAsgCygCGEEBazYCGAwACwALIAIhASAIKAIAIAkoAgBHDQMgBSAFKAIAQQRyNgIAQQAhAAwBCwJAIAJFDQBBASEKA0ACfyACLQALQQd2BEAgAigCBAwBCyACLQALQf8AcQsgCk0NAQJAIAAgC0GMBGoQQ0UEQAJ/IAAoAgAiASgCDCIDIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAMtAAALwEH/AXECfyACLQALQQd2BEAgAigCAAwBCyACCyAKai0AAEYNAQsgBSAFKAIAQQRyNgIAQQAhAAwDCyAAEF0aIApBAWohCgwACwALQQEhACARKAIAIAsoAmRGDQBBACEAIAtBADYCECAPIBEoAgAgCygCZCALQRBqEGQgCygCEARAIAUgBSgCAEEEcjYCAAwBC0EBIQALIBAQMxogDBAzGiANEDMaIA4QMxogDxAzGiARKAIAIQEgEUEANgIAIAEEQCABIBEoAgQRAQALDAMLIAIhAQsgA0EBaiEDDAALAAsgC0GQBGokACAACwwAIABBAUEtEM8DGgtsAQJ/IwBBEGsiAiQAIAAgASgCACABKAIMIAFBEGogAUEAEEkhAyACIAFB1AFqNgIAIANBycsAIAIQcyIAIAEoAjA2AjAgACABKAI0NgI0IAAgASgCODYCOCAAIAEoAjw2AjwgAkEQaiQAIAALDAAgAEEBQS0Q0QMaC20BAX8jAEEQayIGJAAgBkEAOgAPIAYgBToADiAGIAQ6AA0gBkElOgAMIAUEQCAGLQANIQQgBiAGLQAOOgANIAYgBDoADgsgAiABIAIoAgAgAWsgBkEMaiADIAAoAgAQGiABajYCACAGQRBqJAALQgAgASACIAMgBEEEEIkBIQEgAy0AAEEEcUUEQCAAIAFB0A9qIAFB7A5qIAEgAUHkAEgbIAFBxQBIG0HsDms2AgALC0AAIAIgAyAAQQhqIAAoAggoAgQRAAAiACAAQaACaiAFIARBABDuASAAayIAQZ8CTARAIAEgAEEMbUEMbzYCAAsLQAAgAiADIABBCGogACgCCCgCABEAACIAIABBqAFqIAUgBEEAEO4BIABrIgBBpwFMBEAgASAAQQxtQQdvNgIACwtCACABIAIgAyAEQQQQigEhASADLQAAQQRxRQRAIAAgAUHQD2ogAUHsDmogASABQeQASBsgAUHFAEgbQewOazYCAAsLQAAgAiADIABBCGogACgCCCgCBBEAACIAIABBoAJqIAUgBEEAEO8BIABrIgBBnwJMBEAgASAAQQxtQQxvNgIACwtAACACIAMgAEEIaiAAKAIIKAIAEQAAIgAgAEGoAWogBSAEQQAQ7wEgAGsiAEGnAUwEQCABIABBDG1BB282AgALCwQAQQILuQIBBX8jAEEQayIHJAAjAEEQayIDJAACQCABQe////8DTQRAAkAgAUECSQRAIAAgAC0AC0GAAXEgAXI6AAsgACAALQALQf8AcToACyAAIQQMAQsgA0EIaiAAIAFBAk8EfyABQQRqQXxxIgQgBEEBayIEIARBAkYbBUEBC0EBahC5ASADKAIMGiAAIAMoAggiBDYCACAAIAAoAghBgICAgHhxIAMoAgxB/////wdxcjYCCCAAIAAoAghBgICAgHhyNgIIIAAgATYCBAsjAEEQayIFJAAgBSACNgIMIAQhAiABIQYDQCAGBEAgAiAFKAIMNgIAIAZBAWshBiACQQRqIQIMAQsLIAVBEGokACADQQA2AgQgBCABQQJ0aiADKAIENgIAIANBEGokAAwBCxBNAAsgB0EQaiQAIAALhQcBC38jAEEQayIMJAAgBhBsIQkgDEEEaiAGEL8BIg0iBiAGKAIAKAIUEQIAIAUgAzYCAAJAAkAgACIILQAAIgZBK2sOAwABAAELIAkgBsAgCSgCACgCLBEEACEGIAUgBSgCACIHQQRqNgIAIAcgBjYCACAAQQFqIQgLAkACQCACIAgiBmtBAUwNACAGLQAAQTBHDQAgBi0AAUEgckH4AEcNACAJQTAgCSgCACgCLBEEACEHIAUgBSgCACIIQQRqNgIAIAggBzYCACAJIAYsAAEgCSgCACgCLBEEACEHIAUgBSgCACIIQQRqNgIAIAggBzYCACAGQQJqIgghBgNAIAIgBk0NAiAGLAAAIQcQRBogB0Ewa0EKSSAHQSByQeEAa0EGSXJFDQIgBkEBaiEGDAALAAsDQCACIAZNDQEgBiwAACEREEQaIBFBMGtBCk8NASAGQQFqIQYMAAsACwJAAn8gDC0AD0EHdgRAIAwoAggMAQsgDC0AD0H/AHELRQRAIAkgCCAGIAUoAgAgCSgCACgCMBEIABogBSAFKAIAIAYgCGtBAnRqNgIADAELIAggBhCsASANIA0oAgAoAhARAAAhDyAIIQcDQCAGIAdNBEAgAyAIIABrQQJ0aiAFKAIAEOoBBQJAAn8gDEEEaiIKLQALQQd2BEAgCigCAAwBCyAKCyAOaiwAAEEATA0AIAsCfyAKLQALQQd2BEAgCigCAAwBCyAKCyAOaiwAAEcNACAFIAUoAgAiC0EEajYCACALIA82AgAgDiAOAn8gCi0AC0EHdgRAIAooAgQMAQsgCi0AC0H/AHELQQFrSWohDkEAIQsLIAkgBywAACAJKAIAKAIsEQQAIQogBSAFKAIAIhBBBGo2AgAgECAKNgIAIAdBAWohByALQQFqIQsMAQsLCwJAAkADQCACIAZNDQEgBkEBaiEHIAYtAAAiBkEuRwRAIAkgBsAgCSgCACgCLBEEACEGIAUgBSgCACIIQQRqNgIAIAggBjYCACAHIQYMAQsLIA0gDSgCACgCDBEAACEGIAUgBSgCACIIQQRqIgs2AgAgCCAGNgIADAELIAUoAgAhCyAGIQcLIAkgByACIAsgCSgCACgCMBEIABogBSAFKAIAIAIgB2tBAnRqIgU2AgAgBCAFIAMgASAAa0ECdGogASACRhs2AgAgDEEEahAzGiAMQRBqJAAL/gEBA38jAEEQayIFJAAjAEEQayIDJAACQCABQe////8HTQRAAkAgAUELSQRAIAAgAC0AC0GAAXEgAXI6AAsgACAALQALQf8AcToACyAAIQQMAQsgA0EIaiAAIAFBC08EfyABQRBqQXBxIgQgBEEBayIEIARBC0YbBUEKC0EBahDDASADKAIMGiAAIAMoAggiBDYCACAAIAAoAghBgICAgHhxIAMoAgxB/////wdxcjYCCCAAIAAoAghBgICAgHhyNgIIIAAgATYCBAsgBCABIAIQpAIgA0EAOgAHIAEgBGogAy0ABzoAACADQRBqJAAMAQsQTQALIAVBEGokACAAC/UGAQt/IwBBEGsiCyQAIAYQciEJIAtBBGogBhDBASINIgYgBigCACgCFBECACAFIAM2AgACQAJAIAAiCC0AACIGQStrDgMAAQABCyAJIAbAIAkoAgAoAhwRBAAhBiAFIAUoAgAiB0EBajYCACAHIAY6AAAgAEEBaiEICwJAAkAgAiAIIgZrQQFMDQAgBi0AAEEwRw0AIAYtAAFBIHJB+ABHDQAgCUEwIAkoAgAoAhwRBAAhByAFIAUoAgAiCEEBajYCACAIIAc6AAAgCSAGLAABIAkoAgAoAhwRBAAhByAFIAUoAgAiCEEBajYCACAIIAc6AAAgBkECaiIIIQYDQCACIAZNDQIgBiwAACEHEEQaIAdBMGtBCkkgB0EgckHhAGtBBklyRQ0CIAZBAWohBgwACwALA0AgAiAGTQ0BIAYsAAAhERBEGiARQTBrQQpPDQEgBkEBaiEGDAALAAsCQAJ/IAstAA9BB3YEQCALKAIIDAELIAstAA9B/wBxC0UEQCAJIAggBiAFKAIAIAkoAgAoAiARCAAaIAUgBSgCACAGIAhrajYCAAwBCyAIIAYQrAEgDSANKAIAKAIQEQAAIQ8gCCEHA0AgBiAHTQRAIAMgCCAAa2ogBSgCABCsAQUCQAJ/IAtBBGoiCi0AC0EHdgRAIAooAgAMAQsgCgsgDmosAABBAEwNACAMAn8gCi0AC0EHdgRAIAooAgAMAQsgCgsgDmosAABHDQAgBSAFKAIAIgxBAWo2AgAgDCAPOgAAIA4gDgJ/IAotAAtBB3YEQCAKKAIEDAELIAotAAtB/wBxC0EBa0lqIQ5BACEMCyAJIAcsAAAgCSgCACgCHBEEACEKIAUgBSgCACIQQQFqNgIAIBAgCjoAACAHQQFqIQcgDEEBaiEMDAELCwsDQAJAAkAgAiAGTQRAIAYhBwwBCyAGQQFqIQcgBi0AACIGQS5HDQEgDSANKAIAKAIMEQAAIQYgBSAFKAIAIghBAWo2AgAgCCAGOgAACyAJIAcgAiAFKAIAIAkoAgAoAiARCAAaIAUgBSgCACACIAdraiIFNgIAIAQgBSADIAEgAGtqIAEgAkYbNgIAIAtBBGoQMxogC0EQaiQADwsgCSAGwCAJKAIAKAIcEQQAIQYgBSAFKAIAIghBAWo2AgAgCCAGOgAAIAchBgwACwALmgUBBX8jAEHQAmsiACQAIAAgAjYCyAIgACABNgLMAiADEIwBIQYgAyAAQdABahDQASEHIABBxAFqIAMgAEHEAmoQzwEjAEEQayICJAAgAEG4AWoiAUIANwIAIAFBADYCCCACQRBqJAAgASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIgI2ArQBIAAgAEEQajYCDCAAQQA2AggDQAJAIABBzAJqIABByAJqEEINACAAKAK0AQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyACakYEQAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyEDIAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtBAXQQOiABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAIAMCfyABLQALQQd2BEAgASgCAAwBCyABCyICajYCtAELAn8gAEHMAmoiCCgCACIDKAIMIgkgAygCEEYEQCADIAMoAgAoAiQRAAAMAQsgCSgCAAsgBiACIABBtAFqIABBCGogACgCxAIgAEHEAWogAEEQaiAAQQxqIAcQvQENACAIEFwaDAELCwJAAn8gAC0AzwFBB3YEQCAAKALIAQwBCyAALQDPAUH/AHELRQ0AIAAoAgwiAyAAQRBqa0GfAUoNACAAIANBBGo2AgwgAyAAKAIINgIACyAFIAIgACgCtAEgBCAGENkDNgIAIABBxAFqIABBEGogACgCDCAEEGQgAEHMAmogAEHIAmoQQgRAIAQgBCgCAEECcjYCAAsgACgCzAIhCiABEDMaIABBxAFqEDMaIABB0AJqJAAgCgtFAQJ/IwBBEGsiAyQAIAMgATYCDCADIAI2AgggA0EEaiADQQxqEH8hBCAAQfkVIAMoAggQ5gMhACAEEH4gA0EQaiQAIAALrgICBH4GfyMAQSBrIggkAAJAAkACQCABIAJHBEAjA0EcaiIMKAIAIQ0gDEEANgIAIwBBEGsiCSQAEEQaIwBBEGsiCiQAIwBBEGsiCyQAIAsgASAIQRxqQQIQtwIgCykDACEEIAogCykDCDcDCCAKIAQ3AwAgC0EQaiQAIAopAwAhBCAJIAopAwg3AwggCSAENwMAIApBEGokACAJKQMAIQQgCCAJKQMINwMQIAggBDcDCCAJQRBqJAAgCCkDECEEIAgpAwghBSAMKAIAIgFFDQEgCCgCHCACRw0CIAUhBiAEIQcgAUHEAEcNAwwCCyADQQQ2AgAMAgsgDCANNgIAIAgoAhwgAkYNAQsgA0EENgIAIAYhBSAHIQQLIAAgBTcDACAAIAQ3AwggCEEgaiQAC7MBAgR/AnwjAEEQayIDJAACQAJAAkAgACABRwRAIwNBHGoiBSgCACEGIAVBADYCABBEGiMAQRBrIgQkACAEIAAgA0EMakEBELcCIAQpAwAgBCkDCBDBAiEHIARBEGokACAFKAIAIgBFDQEgAygCDCABRw0CIAchCCAAQcQARw0DDAILIAJBBDYCAAwCCyAFIAY2AgAgAygCDCABRg0BCyACQQQ2AgAgCCEHCyADQRBqJAAgBwuzAQIEfwJ9IwBBEGsiAyQAAkACQAJAIAAgAUcEQCMDQRxqIgUoAgAhBiAFQQA2AgAQRBojAEEQayIEJAAgBCAAIANBDGpBABC3AiAEKQMAIAQpAwgQ6AMhByAEQRBqJAAgBSgCACIARQ0BIAMoAgwgAUcNAiAHIQggAEHEAEcNAwwCCyACQQQ2AgAMAgsgBSAGNgIAIAMoAgwgAUYNAQsgAkEENgIAIAghBwsgA0EQaiQAIAcLwwECBH8CfiMAQRBrIgQkAAJ+AkACQCAAIAFHBEACQAJAIAAtAAAiBkEtRw0AIABBAWoiACABRw0ADAELIwNBHGoiBSgCACEHIAVBADYCACAAIARBDGogAxBEEKgCIQgCQCAFKAIAIgAEQCAEKAIMIAFHDQEgAEHEAEYNBAwFCyAFIAc2AgAgBCgCDCABRg0ECwsLIAJBBDYCAEIADAILIAJBBDYCAEJ/DAELQgAgCH0gCCAGQS1GGwshCSAEQRBqJAAgCQvUAQIFfwF+IwBBEGsiBCQAAn8CQAJAAkAgACABRwRAAkACQCAALQAAIgZBLUcNACAAQQFqIgAgAUcNAAwBCyMDQRxqIgUoAgAhByAFQQA2AgAgACAEQQxqIAMQRBCoAiEJAkAgBSgCACIABEAgBCgCDCABRw0BIABBxABGDQUMBAsgBSAHNgIAIAQoAgwgAUYNAwsLCyACQQQ2AgBBAAwDCyAJQv////8PWA0BCyACQQQ2AgBBfwwBC0EAIAmnIgBrIAAgBkEtRhsLIQggBEEQaiQAIAgLkAUBBH8jAEGAAmsiACQAIAAgAjYC+AEgACABNgL8ASADEIwBIQYgAEHEAWogAyAAQfcBahDRASMAQRBrIgIkACAAQbgBaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCtAEgACAAQRBqNgIMIABBADYCCANAAkAgAEH8AWogAEH4AWoQQw0AIAAoArQBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgK0AQsCfyAAQfwBaiIHKAIAIgMoAgwiCCADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAILQAAC8AgBiACIABBtAFqIABBCGogACwA9wEgAEHEAWogAEEQaiAAQQxqQdDqAhDAAQ0AIAcQXRoMAQsLAkACfyAALQDPAUEHdgRAIAAoAsgBDAELIAAtAM8BQf8AcQtFDQAgACgCDCIDIABBEGprQZ8BSg0AIAAgA0EEajYCDCADIAAoAgg2AgALIAUgAiAAKAK0ASAEIAYQ2QM2AgAgAEHEAWogAEEQaiAAKAIMIAQQZCAAQfwBaiAAQfgBahBDBEAgBCAEKAIAQQJyNgIACyAAKAL8ASEJIAEQMxogAEHEAWoQMxogAEGAAmokACAJC9kBAgV/AX4jAEEQayIEJAACfwJAAkACQCAAIAFHBEACQAJAIAAtAAAiBkEtRw0AIABBAWoiACABRw0ADAELIwNBHGoiBSgCACEHIAVBADYCACAAIARBDGogAxBEEKgCIQkCQCAFKAIAIgAEQCAEKAIMIAFHDQEgAEHEAEYNBQwECyAFIAc2AgAgBCgCDCABRg0DCwsLIAJBBDYCAEEADAMLIAlC//8DWA0BCyACQQQ2AgBB//8DDAELQQAgCaciAGsgACAGQS1GGwshCCAEQRBqJAAgCEH//wNxC7MBAgF+A38jAEEQayIFJAACQAJAIAAgAUcEQCMDQRxqIgYoAgAhByAGQQA2AgAgACAFQQxqIAMQRBCRAyEEAkAgBigCACIABEAgBSgCDCABRw0BIABBxABGDQMMBAsgBiAHNgIAIAUoAgwgAUYNAwsLIAJBBDYCAEIAIQQMAQsgAkEENgIAIARCAFUEQEL///////////8AIQQMAQtCgICAgICAgICAfyEECyAFQRBqJAAgBAvBAQIEfwF+IwBBEGsiBCQAAn8CQAJAIAAgAUcEQCMDQRxqIgUoAgAhBiAFQQA2AgAgACAEQQxqIAMQRBCRAyEIAkAgBSgCACIABEAgBCgCDCABRw0BIABBxABGDQQMAwsgBSAGNgIAIAQoAgwgAUYNAgsLIAJBBDYCAEEADAILIAhCgICAgHhTDQAgCEL/////B1UNACAIpwwBCyACQQQ2AgBB/////wcgCEIAVQ0AGkGAgICAeAshByAEQRBqJAAgBwuJAgEDfyMAQRBrIgQkACACIAFrQQJ1IgVB7////wNNBEACQCAFQQJJBEAgACAALQALQYABcSAFcjoACyAAIAAtAAtB/wBxOgALIAAhAwwBCyAEQQhqIAAgBUECTwR/IAVBBGpBfHEiAyADQQFrIgMgA0ECRhsFQQELQQFqELkBIAQoAgwaIAAgBCgCCCIDNgIAIAAgACgCCEGAgICAeHEgBCgCDEH/////B3FyNgIIIAAgACgCCEGAgICAeHI2AgggACAFNgIECwNAIAEgAkcEQCADIAEoAgA2AgAgA0EEaiEDIAFBBGohAQwBCwsgBEEANgIEIAMgBCgCBDYCACAEQRBqJAAPCxBNAAsdAQF/IwBBEGsiAyQAIAAgASACEPkDIANBEGokAAuiBAIHfwR+IwBBEGsiCCQAAkACQAJAIAJBJEwEQCAALQAAIgUNASAAIQQMAgsjA0EcakEcNgIAQgAhAwwCCyAAIQQCQANAIAXAIgVBIEYgBUEJa0EFSXJFDQEgBC0AASEFIARBAWohBCAFDQALDAELAkAgBC0AACIFQStrDgMAAQABC0F/QQAgBUEtRhshByAEQQFqIQQLAn8CQCACQRByQRBHDQAgBC0AAEEwRw0AQQEhCSAELQABQd8BcUHYAEYEQCAEQQJqIQRBEAwCCyAEQQFqIQQgAkEIIAIbDAELIAJBCiACGwsiCq0hDEEAIQIDQAJAQVAhBQJAIAQsAAAiBkEwa0H/AXFBCkkNAEGpfyEFIAZB4QBrQf8BcUEaSQ0AQUkhBSAGQcEAa0H/AXFBGUsNAQsgBSAGaiIGIApODQAgCCAMQgAgC0IAEGBBASEFAkAgCCkDCEIAUg0AIAsgDH4iDSAGrSIOQn+FVg0AIA0gDnwhC0EBIQkgAiEFCyAEQQFqIQQgBSECDAELCyABBEAgASAEIAAgCRs2AgALAkACQCACBEAjA0EcakHEADYCACAHQQAgA0IBgyIMUBshByADIQsMAQsgAyALVg0BIANCAYMhDAsCQCAMpw0AIAcNACMDQRxqQcQANgIAIANCAX0hAwwCCyADIAtaDQAjA0EcakHEADYCAAwBCyALIAesIgOFIAN9IQMLIAhBEGokACADCy8BAn8jAyICKAJgIQEgAARAIAJBuPE0IAAgAEF/Rhs2AmALQX8gASABQbjxNEYbC6oIAQR/IAEoAgAhBAJAAkACQAJAAkACQAJAAn8CQAJAAkACQCADRQ0AIAMoAgAiBUUNACAARQRAIAIhAwwDCyADQQA2AgAgAiEDDAELAkAjAygCYCgCAEUEQCAARQ0BIAJFDQwgAiEFA0AgBCwAACIDBEAgACADQf+/A3E2AgAgAEEEaiEAIARBAWohBCAFQQFrIgUNAQwOCwsgAEEANgIAIAFBADYCACACIAVrDwsgAiEDIABFDQMMBQsgBBBoDwtBASEGDAMLQQAMAQtBAQshBgNAIAZFBEAgBC0AAEEDdiIGQRBrIAVBGnUgBmpyQQdLDQMCfyAEQQFqIAVBgICAEHFFDQAaIAQtAAFBwAFxQYABRwRAIARBAWshBAwHCyAEQQJqIAVBgIAgcUUNABogBC0AAkHAAXFBgAFHBEAgBEEBayEEDAcLIARBA2oLIQQgA0EBayEDQQEhBgwBCwNAIAQtAAAhBQJAIARBA3ENACAFQQFrQf4ASw0AIAQoAgAiBUGBgoQIayAFckGAgYKEeHENAANAIANBBGshAyAEKAIEIQUgBEEEaiEEIAUgBUGBgoQIa3JBgIGChHhxRQ0ACwsgBUH/AXEiBkEBa0H+AE0EQCADQQFrIQMgBEEBaiEEDAELCyAGQcIBayIGQTJLDQMgBEEBaiEEIAZBAnRBsM8CaigCACEFQQAhBgwACwALA0AgBkUEQCADRQ0HA0ACQAJAAkAgBC0AACIGQQFrIgdB/gBLBEAgBiEFDAELIANBBUkNASAEQQNxDQECQANAIAQoAgAiBUGBgoQIayAFckGAgYKEeHENASAAIAVB/wFxNgIAIAAgBC0AATYCBCAAIAQtAAI2AgggACAELQADNgIMIABBEGohACAEQQRqIQQgA0EEayIDQQRLDQALIAQtAAAhBQsgBUH/AXEiBkEBayEHCyAHQf4ASw0BCyAAIAY2AgAgAEEEaiEAIARBAWohBCADQQFrIgMNAQwJCwsgBkHCAWsiBkEySw0DIARBAWohBCAGQQJ0QbDPAmooAgAhBUEBIQYMAQsgBC0AACIGQQN2IgdBEGsgByAFQRp1anJBB0sNAQJAAkACfyAEQQFqIAZBgAFrIAVBBnRyIgZBAE4NABogBC0AAUGAAWsiB0E/Sw0BIARBAmogByAGQQZ0ciIGQQBODQAaIAQtAAJBgAFrIgdBP0sNASAHIAZBBnRyIQYgBEEDagshBCAAIAY2AgAgA0EBayEDIABBBGohAAwBCyMDQRxqQRk2AgAgBEEBayEEDAULQQAhBgwACwALIARBAWshBCAFDQEgBC0AACEFCyAFQf8BcQ0AIAAEQCAAQQA2AgAgAUEANgIACyACIANrDwsjA0EcakEZNgIAIABFDQELIAEgBDYCAAtBfw8LIAEgBDYCACACCy4AIABBAEcgAEG40QJHcSAAQdDRAkdxIABBmIg1R3EgAEGwiDVHcQRAIAAQLwsLKQECfyMAQRBrIgIkACACIAE2AgwgAEG8HiABEOYDIQMgAkEQaiQAIAML5gIBA38CQCABLQAADQBBzjAQuAIiAQRAIAEtAAANAQsgAEEMbEHw0QJqELgCIgEEQCABLQAADQELQfMxELgCIgEEQCABLQAADQELQa05IQELAkADQAJAIAEgAmotAAAiBEUNACAEQS9GDQBBFyEEIAJBAWoiAkEXRw0BDAILCyACIQQLQa05IQMCQAJAAkACQAJAIAEtAAAiAkEuRg0AIAEgBGotAAANACABIQMgAkHDAEcNAQsgAy0AAUUNAQsgA0GtORCfAUUNACADQYkrEJ8BDQELIABFBEBBlNECIQIgAy0AAUEuRg0CC0EADwtBlIg1KAIAIgIEQANAIAMgAkEIahCfAUUNAiACKAIgIgINAAsLQSQQOyICBEAgAkGU0QIpAgA3AgAgAkEIaiIBIAMgBBB7GiABIARqQQA6AAAgAkGUiDUoAgA2AiBBlIg1IAI2AgALIAJBlNECIAAgAnIbIQILIAILqx8CEX8FfiMAQZABayIDJAAgA0EAQZAB/AsAIANBfzYCTCADIAA2AiwgA0H1ATYCICADIAA2AlQgASEEIAIhDUEAIQAjAEGwAmsiBiQAIAMoAkxBAE4EQCADEIIBIRELAkACQAJAAkAgAygCBA0AIAMQ0AIaIAMoAgQNAAwBCyAELQAAIgFFDQICQAJAAkACQANAAkACQCABQf8BcSIBQSBGIAFBCWtBBUlyBEADQCAEIgFBAWohBCABLQABIgJBIEYgAkEJa0EFSXINAAsgA0IAEIABA0ACfyADKAIEIgIgAygCaEcEQCADIAJBAWo2AgQgAi0AAAwBCyADED8LIgJBIEYgAkEJa0EFSXINAAsgAygCBCEEIAMpA3BCAFkEQCADIARBAWsiBDYCBAsgBCADKAIsa6wgAykDeCAWfHwhFgwBCwJ/AkACQCAELQAAQSVGBEAgBC0AASIBQSpGDQEgAUElRw0CCyADQgAQgAECQCAELQAAQSVGBEADQAJ/IAMoAgQiASADKAJoRwRAIAMgAUEBajYCBCABLQAADAELIAMQPwsiAUEgRiABQQlrQQVJcg0ACyAEQQFqIQQMAQsgAygCBCIBIAMoAmhHBEAgAyABQQFqNgIEIAEtAAAhAQwBCyADED8hAQsgBC0AACABRwRAIAMpA3BCAFkEQCADIAMoAgRBAWs2AgQLIAFBAE4NDUEAIQcgDg0NDAsLIAMoAgQgAygCLGusIAMpA3ggFnx8IRYgBCEBDAMLQQAhCCAEQQJqDAELAkAgAUEwa0EKTw0AIAQtAAJBJEcNACAELQABQTBrIQEjAEEQayICIA02AgwgAiANIAFBAnRBBGtBACABQQFLG2oiAUEEajYCCCABKAIAIQggBEEDagwBCyANKAIAIQggDUEEaiENIARBAWoLIQFBACEMQQAhBCABLQAAQTBrQQpJBEADQCABLQAAIARBCmxqQTBrIQQgAS0AASETIAFBAWohASATQTBrQQpJDQALCyABLQAAIglB7QBHBH8gAQVBACEKIAhBAEchDCABLQABIQlBACEAIAFBAWoLIgJBAWohAUEDIQUgDCEHAkACQAJAAkACQAJAIAlBwQBrDjoEDAQMBAQEDAwMDAMMDAwMDAwEDAwMDAQMDAQMDAwMDAQMBAQEBAQABAUMAQwEBAQMDAQCBAwMBAwCDAsgAkECaiABIAItAAFB6ABGIgIbIQFBfkF/IAIbIQUMBAsgAkECaiABIAItAAFB7ABGIgIbIQFBA0EBIAIbIQUMAwtBASEFDAILQQIhBQwBC0EAIQUgAiEBC0EBIAUgAS0AACICQS9xQQNGIgUbIQ8CQCACQSByIAIgBRsiC0HbAEYNAAJAIAtB7gBHBEAgC0HjAEcNAUEBIAQgBEEBTBshBAwCCyAIIA8gFhDnAwwCCyADQgAQgAEDQAJ/IAMoAgQiAiADKAJoRwRAIAMgAkEBajYCBCACLQAADAELIAMQPwsiAkEgRiACQQlrQQVJcg0ACyADKAIEIQIgAykDcEIAWQRAIAMgAkEBayICNgIECyACIAMoAixrrCADKQN4IBZ8fCEWCyADIASsIhQQgAECQCADKAIEIgIgAygCaEcEQCADIAJBAWo2AgQMAQsgAxA/QQBIDQYLIAMpA3BCAFkEQCADIAMoAgRBAWs2AgQLQRAhAgJAAkACQAJAAkACQAJAAkACQAJAIAtB2ABrDiEGCQkCCQkJCQkBCQIEAQEBCQUJCQkJCQMGCQkCCQQJCQYACyALQcEAayICQQZLDQhBASACdEHxAHFFDQgLIAZBCGogAyAPQQAQ6gMgAykDeEIAIAMoAgQgAygCLGusfVINBQwMCyALQRByQfMARgRAIAZBIGpBf0GBAhCgASAGQQA6ACAgC0HzAEcNBiAGQQA6AEEgBkEAOgAuIAZBADYBKgwGCyAGQSBqIAEtAAEiAkHeAEYiBUGBAhCgASAGQQA6ACAgAUECaiABQQFqIAUbIQcCfwJAAkAgAUECQQEgBRtqLQAAIgFBLUcEQCABQd0ARg0BIAJB3gBHIQUgBwwDCyAGIAJB3gBHIgU6AE4MAQsgBiACQd4ARyIFOgB+CyAHQQFqCyEBA0ACQCABLQAAIgJBLUcEQCACRQ0PIAJB3QBGDQgMAQtBLSECIAEtAAEiB0UNACAHQd0ARg0AIAFBAWohCQJAIAcgAUEBay0AACIBTQRAIAchAgwBCwNAIAFBAWoiASAGQSBqaiAFOgAAIAEgCS0AACICSQ0ACwsgCSEBCyACIAZqIAU6ACEgAUEBaiEBDAALAAtBCCECDAILQQohAgwBC0EAIQILQgAhFEEAIQVBACEHQQAhCSMAQRBrIhAkAAJAIAJBAUcgAkEkTXFFBEAjA0EcNgIcDAELA0ACfyADKAIEIgQgAygCaEcEQCADIARBAWo2AgQgBC0AAAwBCyADED8LIgRBIEYgBEEJa0EFSXINAAsCQAJAIARBK2sOAwABAAELQX9BACAEQS1GGyEJIAMoAgQiBCADKAJoRwRAIAMgBEEBajYCBCAELQAAIQQMAQsgAxA/IQQLAkACQAJAAkACQCACQQBHIAJBEEdxDQAgBEEwRw0AAn8gAygCBCIEIAMoAmhHBEAgAyAEQQFqNgIEIAQtAAAMAQsgAxA/CyIEQV9xQdgARgRAQRAhAgJ/IAMoAgQiBCADKAJoRwRAIAMgBEEBajYCBCAELQAADAELIAMQPwsiBEGhzQJqLQAAQRBJDQMgAykDcEIAWQRAIAMgAygCBEEBazYCBAsgA0IAEIABDAYLIAINAUEIIQIMAgsgAkEKIAIbIgIgBEGhzQJqLQAASw0AIAMpA3BCAFkEQCADIAMoAgRBAWs2AgQLIANCABCAASMDQRw2AhwMBAsgAkEKRw0AIARBMGsiBUEJTQRAQQAhBANAIARBCmwgBWoiBEGZs+bMAUkCfyADKAIEIgIgAygCaEcEQCADIAJBAWo2AgQgAi0AAAwBCyADED8LQTBrIgVBCU1xDQALIAStIRQLIAVBCUsNAiAUQgp+IRUgBa0hFwNAAn8gAygCBCICIAMoAmhHBEAgAyACQQFqNgIEIAItAAAMAQsgAxA/CyIEQTBrIgVBCU0gFSAXfCIUQpqz5syZs+bMGVRxRQRAQQohAiAFQQlNDQMMBAsgFEIKfiIVIAWtIhdCf4VYDQALQQohAgwBCyACIAJBAWtxBEAgBEGhzQJqLQAAIgcgAkkEQANAIAcgAiAFbGoiBUHH4/E4SQJ/IAMoAgQiBCADKAJoRwRAIAMgBEEBajYCBCAELQAADAELIAMQPwsiBEGhzQJqLQAAIgcgAklxDQALIAWtIRQLIAIgB00NASACrSEVA0AgFCAVfiIXIAetQv8BgyIYQn+FVg0CIBcgGHwhFCACAn8gAygCBCIEIAMoAmhHBEAgAyAEQQFqNgIEIAQtAAAMAQsgAxA/CyIEQaHNAmotAAAiB00NAiAQIBVCACAUQgAQYCAQKQMIUA0ACwwBCyACQRdsQQV2QQdxQaHPAmosAAAhEiAEQaHNAmotAAAiBSACSQRAA0AgBSAHIBJ0ciIHQYCAgMAASQJ/IAMoAgQiBCADKAJoRwRAIAMgBEEBajYCBCAELQAADAELIAMQPwsiBEGhzQJqLQAAIgUgAklxDQALIAetIRQLIAIgBU0NAEJ/IBKtIhWIIhcgFFQNAANAIAWtQv8BgyAUIBWGhCEUIAICfyADKAIEIgQgAygCaEcEQCADIARBAWo2AgQgBC0AAAwBCyADED8LIgRBoc0Cai0AACIFTQ0BIBQgF1gNAAsLIAIgBEGhzQJqLQAATQ0AA0AgAgJ/IAMoAgQiBCADKAJoRwRAIAMgBEEBajYCBCAELQAADAELIAMQPwtBoc0Cai0AAEsNAAsjA0HEADYCHEEAIQlCfyEUCyADKQNwQgBZBEAgAyADKAIEQQFrNgIECwJAIBRCf1INAAsgFCAJrCIVhSAVfSEUCyAQQRBqJAAgFCEVIAMpA3hCACADKAIEIAMoAixrrH1RDQcCQCALQfAARw0AIAhFDQAgCCAUPgIADAMLIAggDyAVEOcDDAILIAhFDQEgBikDECEUIAYpAwghFQJAAkACQCAPDgMAAQIECyAIIBUgFBDoAzgCAAwDCyAIIBUgFBDBAjkDAAwCCyAIIBU3AwAgCCAUNwMIDAELQR8gBEEBaiALQeMARyIJGyEFAkAgD0EBRgRAIAghAiAMBEAgBUECdBA7IgJFDQcLIAZCADcCqAJBACEEA0AgAiEAAkADQAJ/IAMoAgQiAiADKAJoRwRAIAMgAkEBajYCBCACLQAADAELIAMQPwsiAiAGai0AIUUNASAGIAI6ABsgBkEcaiAGQRtqQQEgBkGoAmoQ8AEiAkF+Rg0AQQAhCiACQX9GDQsgAARAIAAgBEECdGogBigCHDYCACAEQQFqIQQLIAxFDQAgBCAFRw0AC0EBIQcgACAFQQF0QQFyIgVBAnQQxAEiAg0BDAsLC0EAIQogACEFIAZBqAJqBH8gBigCqAIFQQALDQgMAQsgDARAQQAhBCAFEDsiAkUNBgNAIAIhAANAAn8gAygCBCICIAMoAmhHBEAgAyACQQFqNgIEIAItAAAMAQsgAxA/CyICIAZqLQAhRQRAQQAhBSAAIQoMBAsgACAEaiACOgAAIARBAWoiBCAFRw0AC0EBIQcgACAFQQF0QQFyIgUQxAEiAg0ACyAAIQpBACEADAkLQQAhBCAIBEADQAJ/IAMoAgQiACADKAJoRwRAIAMgAEEBajYCBCAALQAADAELIAMQPwsiACAGai0AIQRAIAQgCGogADoAACAEQQFqIQQMAQVBACEFIAgiACEKDAMLAAsACwNAAn8gAygCBCIAIAMoAmhHBEAgAyAAQQFqNgIEIAAtAAAMAQsgAxA/CyAGai0AIQ0AC0EAIQBBACEKQQAhBQsgAygCBCECIAMpA3BCAFkEQCADIAJBAWsiAjYCBAsgAykDeCACIAMoAixrrHwiFVANAiAJIBQgFVFyRQ0CIAwEQCAIIAA2AgALAkAgC0HjAEYNACAFBEAgBSAEQQJ0akEANgIACyAKRQRAQQAhCgwBCyAEIApqQQA6AAALIAUhAAsgAygCBCADKAIsa6wgAykDeCAWfHwhFiAOIAhBAEdqIQ4LIAFBAWohBCABLQABIgENAQwICwsgBSEADAELQQEhB0EAIQpBACEADAILIAwhBwwDCyAMIQcLIA4NAQtBfyEOCyAHRQ0AIAoQLyAAEC8LIBEEQCADEIEBCyAGQbACaiQAIANBkAFqJAAgDgtDAAJAIABFDQACQAJAAkACQCABQQJqDgYAAQICBAMECyAAIAI8AAAPCyAAIAI9AQAPCyAAIAI+AgAPCyAAIAI3AwALC7UDAgN/AX4jAEEgayIDJAACQCABQv///////////wCDIgVCgICAgICAwMA/fSAFQoCAgICAgMC/wAB9VARAIAFCGYinIQQgAFAgAUL///8PgyIFQoCAgAhUIAVCgICACFEbRQRAIARBgYCAgARqIQIMAgsgBEGAgICABGohAiAAIAVCgICACIWEQgBSDQEgAiAEQQFxaiECDAELIABQIAVCgICAgICAwP//AFQgBUKAgICAgIDA//8AURtFBEAgAUIZiKdB////AXFBgICA/gdyIQIMAQtBgICA/AchAiAFQv///////7+/wABWDQBBACECIAVCMIinIgRBkf4ASQ0AIANBEGogACABQv///////z+DQoCAgICAgMAAhCIFIARBgf4AaxBmIAMgACAFQYH/ACAEaxCvASADKQMIIgBCGYinIQIgAykDACADKQMQIAMpAxiEQgBSrYQiBVAgAEL///8PgyIAQoCAgAhUIABCgICACFEbRQRAIAJBAWohAgwBCyAFIABCgICACIWEQgBSDQAgAkEBcSACaiECCyADQSBqJAAgAiABQiCIp0GAgICAeHFyvguNBAIFfwF+AkACQAJAAkACQAJ/IAAoAgQiAiAAKAJoRwRAIAAgAkEBajYCBCACLQAADAELIAAQPwsiAkEraw4DAAEAAQsgAkEtRiEFAn8gACgCBCIDIAAoAmhHBEAgACADQQFqNgIEIAMtAAAMAQsgABA/CyIDQTprIQQgAUUNASAEQXVLDQEgACkDcEIAUw0CIAAgACgCBEEBazYCBAwCCyACQTprIQQgAiEDCyAEQXZJDQACQCADQTBrQQpPDQBBACECA0AgAyACQQpsaiEGAn8gACgCBCICIAAoAmhHBEAgACACQQFqNgIEIAItAAAMAQsgABA/CyEDIAZBMGshAiACQcyZs+YASCADQTBrIgFBCU1xDQALIAKsIQcgAUEKTw0AA0AgA60gB0IKfnwhBwJ/IAAoAgQiASAAKAJoRwRAIAAgAUEBajYCBCABLQAADAELIAAQPwsiA0EwayIBQQlNIAdCMH0iB0Kuj4XXx8LrowFTcQ0ACyABQQpPDQADQAJ/IAAoAgQiASAAKAJoRwRAIAAgAUEBajYCBCABLQAADAELIAAQPwtBMGtBCkkNAAsLIAApA3BCAFkEQCAAIAAoAgRBAWs2AgQLQgAgB30gByAFGyEHDAELQoCAgICAgICAgH8hByAAKQNwQgBTDQAgACAAKAIEQQFrNgIEQoCAgICAgICAgH8PCyAHC9syAxF/B34BfCMAQTBrIg4kAAJAIAJBAk0EQCACQQJ0IgJBjM0CaigCACESIAJBgM0CaigCACERA0ACfyABKAIEIgIgASgCaEcEQCABIAJBAWo2AgQgAi0AAAwBCyABED8LIgJBIEYgAkEJa0EFSXINAAtBASEJAkACQCACQStrDgMAAQABC0F/QQEgAkEtRhshCSABKAIEIgIgASgCaEcEQCABIAJBAWo2AgQgAi0AACECDAELIAEQPyECCwJAAkADQCAGQcQIaiwAACACQSByRgRAAkAgBkEGSw0AIAEoAgQiAiABKAJoRwRAIAEgAkEBajYCBCACLQAAIQIMAQsgARA/IQILIAZBAWoiBkEIRw0BDAILCyAGQQNHBEAgBkEIRiIMDQEgA0UNAiAGQQRJDQIgDA0BCyABKQNwIhVCAFkEQCABIAEoAgRBAWs2AgQLIANFDQAgBkEESQ0AIBVCAFMhAgNAIAJFBEAgASABKAIEQQFrNgIECyAGQQFrIgZBA0sNAAsLQgAhFSMAQRBrIgMkAAJ+IAmyQwAAgH+UvCICQf////8HcSIBQYCAgARrQf////cHTQRAIAGtQhmGQoCAgICAgIDAP3wMAQsgAq1CGYZCgICAgICAwP//AIQgAUGAgID8B08NABpCACABRQ0AGiADIAGtQgAgAWciAUHRAGoQZiADKQMAIRUgAykDCEKAgICAgIDAAIVBif8AIAFrrUIwhoQLIRYgDiAVNwMAIA4gFiACQYCAgIB4ca1CIIaENwMIIANBEGokACAOKQMIIRUgDikDACEWDAILAkACQAJAIAYNAEEAIQYDQCAGQZYYaiwAACACQSByRw0BAkAgBkEBSw0AIAEoAgQiAiABKAJoRwRAIAEgAkEBajYCBCACLQAAIQIMAQsgARA/IQILIAZBAWoiBkEDRw0ACwwBCwJAAkAgBg4EAAEBAgELAkAgAkEwRw0AAn8gASgCBCIMIAEoAmhHBEAgASAMQQFqNgIEIAwtAAAMAQsgARA/C0FfcUHYAEYEQCMAQbADayIFJAACfyABKAIEIgIgASgCaEcEQCABIAJBAWo2AgQgAi0AAAwBCyABED8LIQYCQAJ/A0ACQCAGQTBHBEAgBkEuRw0EIAEoAgQiAiABKAJoRg0BIAEgAkEBajYCBCACLQAADAMLIAEoAgQiAiABKAJoRwRAIAEgAkEBajYCBCACLQAAIQYFIAEQPyEGC0EBIRAMAQsLIAEQPwshBkEBIQsgBkEwRw0AA0AgGEIBfSEYAn8gASgCBCICIAEoAmhHBEAgASACQQFqNgIEIAItAAAMAQsgARA/CyIGQTBGDQALQQEhEAtCgICAgICAwP8/IRYDQAJAIAZBIHIhDQJAAkAgBkEwayIMQQpJDQAgBkEuRyICIA1B4QBrQQVLcQ0CIAINACALDQJBASELIBUhGAwBCyANQdcAayAMIAZBOUobIQICQCAVQgdXBEAgAiAHQQR0aiEHDAELIBVCHFgEQCAFQTBqIAIQdiAFQSBqIBogFkIAQoCAgICAgMD9PxBLIAVBEGogBSkDMCAFKQM4IAUpAyAiGiAFKQMoIhYQSyAFIAUpAxAgBSkDGCAXIBkQcSAFKQMIIRkgBSkDACEXDAELIAJFDQAgCA0AIAVB0ABqIBogFkIAQoCAgICAgID/PxBLIAVBQGsgBSkDUCAFKQNYIBcgGRBxIAUpA0ghGUEBIQggBSkDQCEXCyAVQgF8IRVBASEQCyABKAIEIgIgASgCaEcEfyABIAJBAWo2AgQgAi0AAAUgARA/CyEGDAELCwJ+IBBFBEACQAJAIAEpA3BCAFkEQCABIAEoAgQiAkEBazYCBCADRQ0BIAEgAkECazYCBCALRQ0CIAEgAkEDazYCBAwCCyADDQELIAFCABCAAQsgBUHgAGogCbdEAAAAAAAAAACiEI0BIAUpA2AhFyAFKQNoDAELIBVCB1cEQCAVIRYDQCAHQQR0IQcgFkIBfCIWQghSDQALCwJAAkACQCAGQV9xQdAARgRAIAEgAxDpAyIWQoCAgICAgICAgH9SDQMgAwRAIAEpA3BCAFkNAgwDC0IAIRcgAUIAEIABQgAMBAtCACEWIAEpA3BCAFMNAgsgASABKAIEQQFrNgIEC0IAIRYLIAdFBEAgBUHwAGogCbdEAAAAAAAAAACiEI0BIAUpA3AhFyAFKQN4DAELIBggFSALG0IChiAWfEIgfSIVQQAgEmutVQRAIwNBHGpBxAA2AgAgBUGgAWogCRB2IAVBkAFqIAUpA6ABIAUpA6gBQn9C////////v///ABBLIAVBgAFqIAUpA5ABIAUpA5gBQn9C////////v///ABBLIAUpA4ABIRcgBSkDiAEMAQsgEkHiAWusIBVXBEAgB0EATgRAA0AgBUGgA2ogFyAZQgBCgICAgICAwP+/fxBxIBcgGUKAgICAgICA/z8Q7wMhASAFQZADaiAXIBkgBSkDoAMgFyABQQBOIgEbIAUpA6gDIBkgARsQcSAVQgF9IRUgBSkDmAMhGSAFKQOQAyEXIAdBAXQgAXIiB0EATg0ACwsCfiAVIBKsfUIgfCIWpyIBQQAgAUEAShsgESAWIBGtUxsiAUHxAE4EQCAFQYADaiAJEHYgBSkDiAMhGCAFKQOAAyEaQgAMAQsgBUHgAmpEAAAAAAAA8D9BkAEgAWsQsQEQjQEgBUHQAmogCRB2IAVB8AJqIAUpA+ACIAUpA+gCIAUpA9ACIhogBSkD2AIiGBDuAyAFKQP4AiEbIAUpA/ACCyEWIAVBwAJqIAcgB0EBcUUgFyAZQgBCABCuAUEARyABQSBIcXEiAWoQwgEgBUGwAmogGiAYIAUpA8ACIAUpA8gCEEsgBUGQAmogBSkDsAIgBSkDuAIgFiAbEHEgBUGgAmogGiAYQgAgFyABG0IAIBkgARsQSyAFQYACaiAFKQOgAiAFKQOoAiAFKQOQAiAFKQOYAhBxIAVB8AFqIAUpA4ACIAUpA4gCIBYgGxC5AiAFKQPwASIYIAUpA/gBIhZCAEIAEK4BRQRAIwNBHGpBxAA2AgALIAVB4AFqIBggFiAVpxDtAyAFKQPgASEXIAUpA+gBDAELIwNBHGpBxAA2AgAgBUHQAWogCRB2IAVBwAFqIAUpA9ABIAUpA9gBQgBCgICAgICAwAAQSyAFQbABaiAFKQPAASAFKQPIAUIAQoCAgICAgMAAEEsgBSkDsAEhFyAFKQO4AQshFSAOIBc3AxAgDiAVNwMYIAVBsANqJAAgDikDGCEVIA4pAxAhFgwGCyABKQNwQgBTDQAgASABKAIEQQFrNgIECyABIQggAiEHIAkhDCADIQlBACEDIwBBkMYAayIEJABBACASayINIBFrIRQCQAJ/A0ACQCAHQTBHBEAgB0EuRw0EIAgoAgQiASAIKAJoRg0BIAggAUEBajYCBCABLQAADAMLIAgoAgQiASAIKAJoRwRAIAggAUEBajYCBCABLQAAIQcFIAgQPyEHC0EBIQMMAQsLIAgQPwshB0EBIRAgB0EwRw0AA0AgFUIBfSEVAn8gCCgCBCIBIAgoAmhHBEAgCCABQQFqNgIEIAEtAAAMAQsgCBA/CyIHQTBGDQALQQEhAwsgBEEANgKQBiAHQTBrIQICfgJAAkACQAJAAkACQCAHQS5GIgENACACQQlNDQAMAQsDQAJAIAFBAXEEQCAQRQRAIBYhFUEBIRAMAgsgA0UhAQwECyAWQgF8IRYgC0H8D0wEQCAPIBanIAdBMEYbIQ8gBEGQBmogC0ECdGoiASAKBH8gByABKAIAQQpsakEwawUgAgs2AgBBASEDQQAgCkEBaiIBIAFBCUYiARshCiABIAtqIQsMAQsgB0EwRg0AIAQgBCgCgEZBAXI2AoBGQdyPASEPCwJ/IAgoAgQiASAIKAJoRwRAIAggAUEBajYCBCABLQAADAELIAgQPwsiB0EwayECIAdBLkYiAQ0AIAJBCkkNAAsLIBUgFiAQGyEVAkAgA0UNACAHQV9xQcUARw0AAkAgCCAJEOkDIhdCgICAgICAgICAf1INACAJRQ0EQgAhFyAIKQNwQgBTDQAgCCAIKAIEQQFrNgIECyAVIBd8IRUMBAsgA0UhASAHQQBIDQELIAgpA3BCAFMNACAIIAgoAgRBAWs2AgQLIAFFDQEjA0EcakEcNgIACyAIQgAQgAFCACEVQgAMAQsgBCgCkAYiAUUEQCAEIAy3RAAAAAAAAAAAohCNASAEKQMIIRUgBCkDAAwBCwJAIBZCCVUNACAVIBZSDQAgEUEeTEEAIAEgEXYbDQAgBEEwaiAMEHYgBEEgaiABEMIBIARBEGogBCkDMCAEKQM4IAQpAyAgBCkDKBBLIAQpAxghFSAEKQMQDAELIA1BAXatIBVTBEAjA0EcakHEADYCACAEQeAAaiAMEHYgBEHQAGogBCkDYCAEKQNoQn9C////////v///ABBLIARBQGsgBCkDUCAEKQNYQn9C////////v///ABBLIAQpA0ghFSAEKQNADAELIBJB4gFrrCAVVQRAIwNBHGpBxAA2AgAgBEGQAWogDBB2IARBgAFqIAQpA5ABIAQpA5gBQgBCgICAgICAwAAQSyAEQfAAaiAEKQOAASAEKQOIAUIAQoCAgICAgMAAEEsgBCkDeCEVIAQpA3AMAQsgCgRAIApBCEwEQCAEQZAGaiALQQJ0aiIBKAIAIQYDQCAGQQpsIQYgCkEBaiIKQQlHDQALIAEgBjYCAAsgC0EBaiELCyAVpyEKAkAgD0EJTg0AIAogD0gNACAKQRFKDQAgCkEJRgRAIARBwAFqIAwQdiAEQbABaiAEKAKQBhDCASAEQaABaiAEKQPAASAEKQPIASAEKQOwASAEKQO4ARBLIAQpA6gBIRUgBCkDoAEMAgsgCkEITARAIARBkAJqIAwQdiAEQYACaiAEKAKQBhDCASAEQfABaiAEKQOQAiAEKQOYAiAEKQOAAiAEKQOIAhBLIARB4AFqQQAgCmtBAnRBgM0CaigCABB2IARB0AFqIAQpA/ABIAQpA/gBIAQpA+ABIAQpA+gBEOwDIAQpA9gBIRUgBCkD0AEMAgsgESAKQX1sakEbaiICQR5MQQAgBCgCkAYiASACdhsNACAEQeACaiAMEHYgBEHQAmogARDCASAEQcACaiAEKQPgAiAEKQPoAiAEKQPQAiAEKQPYAhBLIARBsAJqIApBAnRBuMwCaigCABB2IARBoAJqIAQpA8ACIAQpA8gCIAQpA7ACIAQpA7gCEEsgBCkDqAIhFSAEKQOgAgwBCwNAIARBkAZqIAsiAUEBayILQQJ0aigCAEUNAAtBACEPAkAgCkEJbyIDRQRAQQAhAgwBC0EAIQIgA0EJaiADIApBAEgbIQUCQCABRQRAQQAhAQwBC0GAlOvcA0EAIAVrQQJ0QYDNAmooAgAiEG0hDUEAIQdBACEGA0AgBEGQBmoiCyAGQQJ0aiIDIAcgAygCACIIIBBuIglqIgM2AgAgAkEBakH/D3EgAiADRSACIAZGcSIDGyECIApBCWsgCiADGyEKIA0gCCAJIBBsa2whByAGQQFqIgYgAUcNAAsgB0UNACABQQJ0IAtqIAc2AgAgAUEBaiEBCyAKIAVrQQlqIQoLA0AgBEGQBmogAkECdGohBwJAA0AgCkEkTgRAIApBJEcNAiAHKAIAQdHp+QRPDQILIAFB/w9qIQtBACEDA0AgASEJIAOtIARBkAZqIAtB/w9xIg1BAnRqIgE1AgBCHYZ8IhVCgZTr3ANUBH9BAAUgFSAVQoCU69wDgCIWQoCU69wDfn0hFSAWpwshAyABIBWnIgE2AgAgCSAJIAkgDSABGyACIA1GGyANIAlBAWtB/w9xIghHGyEBIA1BAWshCyACIA1HDQALIA9BHWshDyAJIQEgA0UNAAsgAkEBa0H/D3EiAiABRgRAIARBkAZqIgkgAUH+D2pB/w9xQQJ0aiIBIAEoAgAgCEECdCAJaigCAHI2AgAgCCEBCyAKQQlqIQogBEGQBmogAkECdGogAzYCAAwBCwsCQANAIAFBAWpB/w9xIQkgBEGQBmogAUEBa0H/D3FBAnRqIQUDQEEJQQEgCkEtShshEwJAA0AgAiEDQQAhBgJAA0ACQCADIAZqQf8PcSICIAFGDQAgBEGQBmogAkECdGooAgAiCCAGQQJ0QdDMAmooAgAiAkkNACACIAhJDQIgBkEBaiIGQQRHDQELCyAKQSRHDQBCACEVQQAhBkIAIRYDQCABIAMgBmpB/w9xIgJGBEAgAUEBakH/D3EiAUECdCAEakEANgKMBgsgBEGABmogBEGQBmogAkECdGooAgAQwgEgBEHwBWogFSAWQgBCgICAgOWat47AABBLIARB4AVqIAQpA/AFIAQpA/gFIAQpA4AGIAQpA4gGEHEgBCkD6AUhFiAEKQPgBSEVIAZBAWoiBkEERw0ACyAEQdAFaiAMEHYgBEHABWogFSAWIAQpA9AFIAQpA9gFEEsgBCkDyAUhFkIAIRUgBCkDwAUhFyAPQfEAaiIIIBJrIgtBACALQQBKGyARIAsgEUgiCRsiB0HwAEwNAgwFCyAPIBNqIQ8gASECIAEgA0YNAAtBgJTr3AMgE3YhEEF/IBN0QX9zIQ1BACEGIAMhAgNAIARBkAZqIgcgA0ECdGoiCCAGIAgoAgAiCyATdmoiCDYCACACQQFqQf8PcSACIAhFIAIgA0ZxIggbIQIgCkEJayAKIAgbIQogCyANcSAQbCEGIANBAWpB/w9xIgMgAUcNAAsgBkUNASACIAlHBEAgAUECdCAHaiAGNgIAIAkhAQwDCyAFIAUoAgBBAXI2AgAMAQsLCyAEQZAFakQAAAAAAADwP0HhASAHaxCxARCNASAEQbAFaiAEKQOQBSAEKQOYBSAXIBYQ7gMgBCkDuAUhGiAEKQOwBSEZIARBgAVqRAAAAAAAAPA/QfEAIAdrELEBEI0BIARBoAVqIBcgFiAEKQOABSAEKQOIBRDrAyAEQfAEaiAXIBYgBCkDoAUiFSAEKQOoBSIYELkCIARB4ARqIBkgGiAEKQPwBCAEKQP4BBBxIAQpA+gEIRYgBCkD4AQhFwsCQCADQQRqQf8PcSICIAFGDQACQCAEQZAGaiACQQJ0aigCACICQf/Jte4BTQRAIAJFIANBBWpB/w9xIAFGcQ0BIARB8ANqIAy3RAAAAAAAANA/ohCNASAEQeADaiAVIBggBCkD8AMgBCkD+AMQcSAEKQPoAyEYIAQpA+ADIRUMAQsgAkGAyrXuAUcEQCAEQdAEaiAMt0QAAAAAAADoP6IQjQEgBEHABGogFSAYIAQpA9AEIAQpA9gEEHEgBCkDyAQhGCAEKQPABCEVDAELIAy3IRwgASADQQVqQf8PcUYEQCAEQZAEaiAcRAAAAAAAAOA/ohCNASAEQYAEaiAVIBggBCkDkAQgBCkDmAQQcSAEKQOIBCEYIAQpA4AEIRUMAQsgBEGwBGogHEQAAAAAAADoP6IQjQEgBEGgBGogFSAYIAQpA7AEIAQpA7gEEHEgBCkDqAQhGCAEKQOgBCEVCyAHQe8ASg0AIARB0ANqIBUgGEIAQoCAgICAgMD/PxDrAyAEKQPQAyAEKQPYA0IAQgAQrgENACAEQcADaiAVIBhCAEKAgICAgIDA/z8QcSAEKQPIAyEYIAQpA8ADIRULIARBsANqIBcgFiAVIBgQcSAEQaADaiAEKQOwAyAEKQO4AyAZIBoQuQIgBCkDqAMhFiAEKQOgAyEXAkAgFEECayAIQf////8HcU4NACAEIBZC////////////AIM3A5gDIAQgFzcDkAMgBEGAA2ogFyAWQgBCgICAgICAgP8/EEsgBCkDkAMgBCkDmANCgICAgICAgLjAABDvAyEDIAQpA4gDIBYgA0EATiICGyEWIAQpA4ADIBcgAhshFyAVIBhCAEIAEK4BIQEgFCACIA9qIg9B7gBqTgRAIAkgByALRyADQQBIcnEgAUEAR3FFDQELIwNBHGpBxAA2AgALIARB8AJqIBcgFiAPEO0DIAQpA/gCIRUgBCkD8AILIRYgDiAVNwMoIA4gFjcDICAEQZDGAGokACAOKQMoIRUgDikDICEWDAQLIAEpA3BCAFkEQCABIAEoAgRBAWs2AgQLDAELAkACfyABKAIEIgIgASgCaEcEQCABIAJBAWo2AgQgAi0AAAwBCyABED8LQShGBEBBASEGDAELQoCAgICAgOD//wAhFSABKQNwQgBTDQMgASABKAIEQQFrNgIEDAMLA0ACfyABKAIEIgIgASgCaEcEQCABIAJBAWo2AgQgAi0AAAwBCyABED8LIglBwQBrIQICQAJAIAlBMGtBCkkNACACQRpJDQAgCUHfAEYNACAJQeEAa0EaTw0BCyAGQQFqIQYMAQsLQoCAgICAgOD//wAhFSAJQSlGDQIgASkDcCIYQgBZBEAgASABKAIEQQFrNgIECwJAIAMEQCAGDQEMBAsMAQsDQCAYQgBZBEAgASABKAIEQQFrNgIECyAGQQFrIgYNAAsMAgsjA0EcakEcNgIAIAFCABCAAQtCACEVCyAAIBY3AwAgACAVNwMIIA5BMGokAAvKBgIFfwR+IwBBgAFrIgUkAAJAAkACQCADIARCAEIAEK4BRQ0AAn8gBEL///////8/gyELAn8gBEIwiKdB//8BcSIGQf//AUcEQEEEIAYNARpBAkEDIAMgC4RQGwwCCyADIAuEUAsLIQkgAkIwiKciCEH//wFxIgdB//8BRg0AIAkNAQsgBUEQaiABIAIgAyAEEEsgBSAFKQMQIgIgBSkDGCIBIAIgARDsAyAFKQMIIQIgBSkDACEEDAELIAEgAkL///////////8AgyILIAMgBEL///////////8AgyIKEK4BQQBMBEAgASALIAMgChCuAQRAIAEhBAwCCyAFQfAAaiABIAJCAEIAEEsgBSkDeCECIAUpA3AhBAwBCyAEQjCIp0H//wFxIQYgBwR+IAEFIAVB4ABqIAEgC0IAQoCAgICAgMC7wAAQSyAFKQNoIgtCMIinQfgAayEHIAUpA2ALIQQgBkUEQCAFQdAAaiADIApCAEKAgICAgIDAu8AAEEsgBSkDWCIKQjCIp0H4AGshBiAFKQNQIQMLIApC////////P4NCgICAgICAwACEIQwgC0L///////8/g0KAgICAgIDAAIQhCyAGIAdIBEADQAJ+IAsgDH0gAyAEVq19IgpCAFkEQCAKIAQgA30iBIRQBEAgBUEgaiABIAJCAEIAEEsgBSkDKCECIAUpAyAhBAwFCyAKQgGGIARCP4iEDAELIAtCAYYgBEI/iIQLIQsgBEIBhiEEIAdBAWsiByAGSg0ACyAGIQcLAkAgCyAMfSADIARWrX0iCkIAUwRAIAshCgwBCyAKIAQgA30iBIRCAFINACAFQTBqIAEgAkIAQgAQSyAFKQM4IQIgBSkDMCEEDAELIApC////////P1gEQANAIARCP4ghDSAHQQFrIQcgBEIBhiEEIA0gCkIBhoQiCkKAgICAgIDAAFQNAAsLIAhBgIACcSEGIAdBAEwEQCAFQUBrIAQgCkL///////8/gyAHQfgAaiAGcq1CMIaEQgBCgICAgICAwMM/EEsgBSkDSCECIAUpA0AhBAwBCyAKQv///////z+DIAYgB3KtQjCGhCECCyAAIAQ3AwAgACACNwMIIAVBgAFqJAALqg8CBX8PfiMAQdACayIFJAAgBEL///////8/gyELIAJC////////P4MhCiACIASFQoCAgICAgICAgH+DIQ0gBEIwiKdB//8BcSEIAkACQCACQjCIp0H//wFxIglB//8Ba0GCgH5PBEAgCEH//wFrQYGAfksNAQsgAVAgAkL///////////8AgyIMQoCAgICAgMD//wBUIAxCgICAgICAwP//AFEbRQRAIAJCgICAgICAIIQhDQwCCyADUCAEQv///////////wCDIgJCgICAgICAwP//AFQgAkKAgICAgIDA//8AURtFBEAgBEKAgICAgIAghCENIAMhAQwCCyABIAxCgICAgICAwP//AIWEUARAIAMgAkKAgICAgIDA//8AhYRQBEBCACEBQoCAgICAgOD//wAhDQwDCyANQoCAgICAgMD//wCEIQ1CACEBDAILIAMgAkKAgICAgIDA//8AhYRQBEBCACEBDAILIAEgDIRQBEBCgICAgICA4P//ACANIAIgA4RQGyENQgAhAQwCCyACIAOEUARAIA1CgICAgICAwP//AIQhDUIAIQEMAgsgDEL///////8/WARAIAVBwAJqIAEgCiABIAogClAiBht5IAZBBnStfKciBkEPaxBmQRAgBmshBiAFKQPIAiEKIAUpA8ACIQELIAJC////////P1YNACAFQbACaiADIAsgAyALIAtQIgcbeSAHQQZ0rXynIgdBD2sQZiAGIAdqQRBrIQYgBSkDuAIhCyAFKQOwAiEDCyAFQaACaiALQoCAgICAgMAAhCISQg+GIANCMYiEIgJCAEKAgICAsOa8gvUAIAJ9IgRCABBgIAVBkAJqQgAgBSkDqAJ9QgAgBEIAEGAgBUGAAmogBSkDmAJCAYYgBSkDkAJCP4iEIgRCACACQgAQYCAFQfABaiAEQgBCACAFKQOIAn1CABBgIAVB4AFqIAUpA/gBQgGGIAUpA/ABQj+IhCIEQgAgAkIAEGAgBUHQAWogBEIAQgAgBSkD6AF9QgAQYCAFQcABaiAFKQPYAUIBhiAFKQPQAUI/iIQiBEIAIAJCABBgIAVBsAFqIARCAEIAIAUpA8gBfUIAEGAgBUGgAWogAkIAIAUpA7gBQgGGIAUpA7ABQj+IhEIBfSICQgAQYCAFQZABaiADQg+GQgAgAkIAEGAgBUHwAGogAkIAQgAgBSkDqAEgBSkDoAEiDCAFKQOYAXwiBCAMVK18IARCAVatfH1CABBgIAVBgAFqQgEgBH1CACACQgAQYCAGIAkgCGtqIQYCfyAFKQNwIhNCAYYiDiAFKQOIASIPQgGGIAUpA4ABQj+IhHwiEELn7AB9IhRCIIgiAiAKQoCAgICAgMAAhCIVQgGGIhZCIIgiBH4iESABQgGGIgxCIIgiCyAQIBRWrSAOIBBWrSAFKQN4QgGGIBNCP4iEIA9CP4h8fHxCAX0iE0IgiCIQfnwiDiARVK0gDiAOIBNC/////w+DIhMgAUI/iCIXIApCAYaEQv////8PgyIKfnwiDlatfCAEIBB+fCAEIBN+IhEgCiAQfnwiDyARVK1CIIYgD0IgiIR8IA4gDiAPQiCGfCIOVq18IA4gDiAUQv////8PgyIUIAp+IhEgAiALfnwiDyARVK0gDyAPIBMgDEL+////D4MiEX58Ig9WrXx8Ig5WrXwgDiAEIBR+IhggECARfnwiBCACIAp+fCIKIAsgE358IhBCIIggCiAQVq0gBCAYVK0gBCAKVq18fEIghoR8IgQgDlStfCAEIA8gAiARfiICIAsgFH58IgtCIIggAiALVq1CIIaEfCICIA9UrSACIBBCIIZ8IAJUrXx8IgIgBFStfCIEQv////////8AWARAIBYgF4QhFSAFQdAAaiACIAQgAyASEGAgAUIxhiAFKQNYfSAFKQNQIgFCAFKtfSEKQgAgAX0hCyAGQf7/AGoMAQsgBUHgAGogBEI/hiACQgGIhCICIARCAYgiBCADIBIQYCABQjCGIAUpA2h9IAUpA2AiDEIAUq19IQpCACAMfSELIAEhDCAGQf//AGoLIgZB//8BTgRAIA1CgICAgICAwP//AIQhDUIAIQEMAQsCfiAGQQBKBEAgCkIBhiALQj+IhCEKIARC////////P4MgBq1CMIaEIQwgC0IBhgwBCyAGQY9/TARAQgAhAQwCCyAFQUBrIAIgBEEBIAZrEK8BIAVBMGogDCAVIAZB8ABqEGYgBUEgaiADIBIgBSkDQCICIAUpA0giDBBgIAUpAzggBSkDKEIBhiAFKQMgIgFCP4iEfSAFKQMwIgQgAUIBhiIBVK19IQogBCABfQshBCAFQRBqIAMgEkIDQgAQYCAFIAMgEkIFQgAQYCAMIAIgAiADIAJCAYMiASAEfCIDVCAKIAEgA1atfCIBIBJWIAEgElEbrXwiAlatfCIEIAIgAiAEQoCAgICAgMD//wBUIAMgBSkDEFYgASAFKQMYIgRWIAEgBFEbca18IgJWrXwiBCACIARCgICAgICAwP//AFQgAyAFKQMAViABIAUpAwgiA1YgASADURtxrXwiASACVK18IA2EIQ0LIAAgATcDACAAIA03AwggBUHQAmokAAu/AgEBfyMAQdAAayIEJAACQCADQYCAAU4EQCAEQSBqIAEgAkIAQoCAgICAgID//wAQSyAEKQMoIQIgBCkDICEBIANB//8BSQRAIANB//8AayEDDAILIARBEGogASACQgBCgICAgICAgP//ABBLQf3/AiADIANB/f8CThtB/v8BayEDIAQpAxghAiAEKQMQIQEMAQsgA0GBgH9KDQAgBEFAayABIAJCAEKAgICAgICAORBLIAQpA0ghAiAEKQNAIQEgA0H0gH5LBEAgA0GN/wBqIQMMAQsgBEEwaiABIAJCAEKAgICAgICAORBLQeiBfSADIANB6IF9TBtBmv4BaiEDIAQpAzghAiAEKQMwIQELIAQgASACQgAgA0H//wBqrUIwhhBLIAAgBCkDCDcDCCAAIAQpAwA3AwAgBEHQAGokAAs1ACAAIAE3AwAgACACQv///////z+DIARCMIinQYCAAnEgAkIwiKdB//8BcXKtQjCGhDcDCAvAAQIBfwJ+QX8hAwJAIABCAFIgAUL///////////8AgyIEQoCAgICAgMD//wBWIARCgICAgICAwP//AFEbDQAgAkL///////////8AgyIFQoCAgICAgMD//wBWIAVCgICAgICAwP//AFJxDQAgACAEIAWEhFAEQEEADwsgASACg0IAWQRAIAEgAlIgASACU3ENASAAIAEgAoWEQgBSDwsgAEIAUiABIAJVIAEgAlEbDQAgACABIAKFhEIAUiEDCyADC0sBAn8gACgCACIBBEACfyABKAIMIgIgASgCEEYEQCABIAEoAgAoAiQRAAAMAQsgAigCAAtBf0cEQCAAKAIARQ8LIABBADYCAAtBAQtLAQJ/IAAoAgAiAQRAAn8gASgCDCICIAEoAhBGBEAgASABKAIAKAIkEQAADAELIAItAAALQX9HBEAgACgCAEUPCyAAQQA2AgALQQELpAEBA38gAUEISwRAQQQgASABQQRNGyEBQQEgACAAQQFNGyEAA0ACQCAAIAFqQQFrQQAgAWtxIgIgACAAIAJJGyEDQQAhBCMAQRBrIgIkAAJAIAFBA3ENACADIAFwDQAgAkEMaiABIAMQgwQhA0EAIAIoAgwgAxshBAsgAkEQaiQAIAQiAg0AQayZNf4QAgAiA0UNACADEQkADAELCyACDwsgABAyCwkAIAFBARDyAwsTACABQQhLBEAgABAvDwsgABAvC4YBAQN/IwBBEGsiBCQAIwBBIGsiAyQAIANBGGogACABEL0CIANBEGogA0EMaiADKAIYIAMoAhwgAhC8AiADIAAgAygCECAAa2o2AgwgAyACIAMoAhQgAmtqNgIIIAQgAygCDDYCCCAEIAMoAgg2AgwgA0EgaiQAIAQoAgwhBSAEQRBqJAAgBQsJACAAEL4CEC8LpQEBB38jAEEQayICJAAgACgCQCIBBH8gAkHSATYCBCACQQhqIAEgAkEEahBRIQEgACAAKAIAKAIYEQAAIQQgASgCACEGIAFBADYCACAGEKwEIQUgAEEANgJAIABBAEEAIAAoAgAoAgwRAwAaIAEoAgAhAyABQQA2AgAgAwRAIAMgAUEEaigCABEAABoLQQAgACAEIAVyGwVBAAshByACQRBqJAAgBwsKACAAQaSKNRB5C4YCAQN/IwBBEGsiBCQAIAIgAWsiBUHv////B00EQAJAIAVBC0kEQCAAIAAtAAtBgAFxIAVyOgALIAAgAC0AC0H/AHE6AAsgACEDDAELIARBCGogACAFQQtPBH8gBUEQakFwcSIDIANBAWsiAyADQQtGGwVBCgtBAWoQwwEgBCgCDBogACAEKAIIIgM2AgAgACAAKAIIQYCAgIB4cSAEKAIMQf////8HcXI2AgggACAAKAIIQYCAgIB4cjYCCCAAIAU2AgQLA0AgASACRwRAIAMgAS0AADoAACADQQFqIQMgAUEBaiEBDAELCyAEQQA6AAcgAyAELQAHOgAAIARBEGokAA8LEE0AC1QBAn8CQCAAKAIAIgJFDQACfyACKAIYIgMgAigCHEYEQCACIAEgAigCACgCNBEEAAwBCyACIANBBGo2AhggAyABNgIAIAELQX9HDQAgAEEANgIACwsxAQF/IAAoAgwiASAAKAIQRgRAIAAgACgCACgCKBEAAA8LIAAgAUEEajYCDCABKAIAC1wBAn8CQCAAKAIAIgJFDQACfyACKAIYIgMgAigCHEYEQCACIAFB/wFxIAIoAgAoAjQRBAAMAQsgAiADQQFqNgIYIAMgAToAACABQf8BcQtBf0cNACAAQQA2AgALCzEBAX8gACgCDCIBIAAoAhBGBEAgACAAKAIAKAIoEQAADwsgACABQQFqNgIMIAEtAAALsQIBAn8jAEEQayIBJAAgACAAKAIAQQxrKAIAaigCGARAIAEgADYCDCABQQA6AAggACAAKAIAQQxrKAIAaigCEEUEQCAAIAAoAgBBDGsoAgBqKAJIIgIEQCACEP4DCyABQQE6AAgLAkAgAS0ACEUNACAAIAAoAgBBDGsoAgBqKAIYIgIgAigCACgCGBEAAEF/Rw0AIAAgACgCAEEMaygCAGpBARDzAQsCQCABKAIMIgAgACgCAEEMaygCAGooAhhFDQAgACgCAEEMaygCACAAaigCEA0AIAAoAgBBDGsoAgAgAGooAgRBgMAAcUUNACAAKAIAQQxrKAIAIABqKAIYIgAgACgCACgCGBEAAEF/Rw0AIAEoAgwiACAAKAIAQQxrKAIAakEBEPMBCwsgAUEQaiQACwkAIAAQvwIQLwsEAEF/Cw4AIAAgACABaiACEPUDCykBAX8CQEEgEDsiAEUNACAAQQRrLQAAQQNxRQ0AIABBAEEgEKABCyAAC7oEAQV/AkACfyABQQhGBEAgAhA7DAELQRwhBCABQQRJDQEgAUEDcQ0BIAFBAnYiAyADQQFrcQ0BQTAhBEFAIAFrIAJJDQECf0EQIQMCQEEQQRAgASABQRBNGyIBIAFBEE0bIgQgBEEBa3FFBEAgBCEBDAELA0AgAyIBQQF0IQMgASAESQ0ACwsgAkFAIAFrTwRAIwNBHGpBMDYCAEEADAELQQBBECACQQtqQXhxIAJBC0kbIgQgAWpBDGoQOyIDRQ0AGkEAIQICQEG8hzUtAABBAnEEQEHAhzUQVg0BCyADQQhrIQICQCABQQFrIANxRQRAIAIhAQwBCyADQQRrIgYoAgAiB0F4cSABIANqQQFrQQAgAWtxQQhrIgMgAUEAIAMgAmtBD00baiIBIAJrIgNrIQUgB0EDcUUEQCACKAIAIQIgASAFNgIEIAEgAiADajYCAAwBCyABIAUgASgCBEEBcXJBAnI2AgQgASAFaiIFIAUoAgRBAXI2AgQgBiADIAYoAgBBAXFyQQJyNgIAIAIgA2oiBSAFKAIEQQFyNgIEIAIgAxD0AQsCQCABKAIEIgJBA3FFDQAgAkF4cSIDIARBEGpNDQAgASAEIAJBAXFyQQJyNgIEIAEgBGoiAiADIARrIgRBA3I2AgQgASADaiIDIAMoAgRBAXI2AgQgAiAEEPQBCyABQQhqIQJBvIc1LQAAQQJxRQ0AQcCHNRBSGgsgAgsLIgFFBEBBMA8LIAAgATYCAEEAIQQLIAQLyAEBA38jAEEQayIBJABB8Ic1EFYaQeiDNSgCAEUEQEH8gzVBAjYCAEH0gzVCfzcCAEHsgzVCgKCAgICABDcCAEG8hzVBAjYCACABQQxqIgJBADYCACMAQSBrIgBCADcDGCAAQgA3AxAgAEIANwMIQcCHNSAAKQMINwIAQdCHNSAAKQMYNwIAQciHNSAAKQMQNwIAIAIEQEHAhzUgAigCADYCAAtB6IM1IAFBCGpBcHFB2KrVqgVzNgIAC0HwhzUQUhogAUEQaiQACxIAIABFBEBBAA8LIAAgARDCAgsQACAAIAEgAkEAQQAQwwIaC7wCAAJAAkACQAJAAkACQAJAAkACQAJAAkAgAUEJaw4SAAgJCggJAQIDBAoJCgoICQUGBwsgAiACKAIAIgFBBGo2AgAgACABKAIANgIADwsgAiACKAIAIgFBBGo2AgAgACABMgEANwMADwsgAiACKAIAIgFBBGo2AgAgACABMwEANwMADwsgAiACKAIAIgFBBGo2AgAgACABMAAANwMADwsgAiACKAIAIgFBBGo2AgAgACABMQAANwMADwsgAiACKAIAQQdqQXhxIgFBCGo2AgAgACABKwMAOQMADwsgACACIAMRAgALDwsgAiACKAIAIgFBBGo2AgAgACABNAIANwMADwsgAiACKAIAIgFBBGo2AgAgACABNQIANwMADwsgAiACKAIAQQdqQXhxIgFBCGo2AgAgACABKQMANwMAC3IBA38gACgCACwAAEEwa0EKTwRAQQAPCwNAIAAoAgAhA0F/IQEgAkHMmbPmAE0EQEF/IAMsAABBMGsiASACQQpsIgJqIAEgAkH/////B3NKGyEBCyAAIANBAWo2AgAgASECIAMsAAFBMGtBCkkNAAsgAguZEwIWfwF+IwBB0ABrIggkACAIIAE2AkwgCEE3aiEYIAhBOGohEgJAAkACQANAQQAhBwNAIAEhDSAHIBFB/////wdzSg0CIAcgEWohEQJAAkACQCABIgctAAAiCQRAA0ACQAJAIAlB/wFxIgFFBEAgByEBDAELIAFBJUcNASAHIQkDQCAJLQABQSVHBEAgCSEBDAILIAdBAWohByAJLQACIRogCUECaiIBIQkgGkElRg0ACwsgByANayIHIBFB/////wdzIhlKDQggAARAIAAgDSAHEGELIAcNBiAIIAE2AkwgAUEBaiEHQX8hDgJAIAEsAAFBMGsiC0EKTw0AIAEtAAJBJEcNACABQQNqIQcgCyEOQQEhEwsgCCAHNgJMQQAhDAJAIAcsAAAiCUEgayIBQR9LBEAgByELDAELIAchC0EBIAF0IgFBidEEcUUNAANAIAggB0EBaiILNgJMIAEgDHIhDCAHLAABIglBIGsiAUEgTw0BIAshB0EBIAF0IgFBidEEcQ0ACwsCQCAJQSpGBEAgC0EBaiEJAn8CQCALLAABQTBrQQpPDQAgCy0AAkEkRw0AIAksAABBMGshASALQQNqIQlBASETAn8gAEUEQCAEIAFBAnRqQQo2AgBBAAwBCyADIAFBA3RqKAIACwwBCyATDQYgAEUEQCAIIAk2AkxBACETQQAhDwwDCyACIAIoAgAiAUEEajYCAEEAIRMgASgCAAshDyAIIAk2AkwgD0EATg0BQQAgD2shDyAMQYDAAHIhDAwBCyAIQcwAahCIBCIPQQBIDQkgCCgCTCEJC0EAIQdBfyEKAn8gCS0AAEEuRwRAIAkhAUEADAELIAktAAFBKkYEQCAJQQJqIQECQAJAIAksAAJBMGtBCk8NACAJLQADQSRHDQAgASwAAEEwayEBAn8gAEUEQCAEIAFBAnRqQQo2AgBBAAwBCyADIAFBA3RqKAIACyEKIAlBBGohAQwBCyATDQYgAEUEQEEAIQoMAQsgAiACKAIAIgtBBGo2AgAgCygCACEKCyAIIAE2AkwgCkF/c0EfdgwBCyAIIAlBAWo2AkwgCEHMAGoQiAQhCiAIKAJMIQFBAQshFANAIAchFUEcIRAgASIWLAAAIgdB+wBrQUZJDQogAUEBaiEBIAcgFUE6bGpB/8ICai0AACIHQQFrQQhJDQALIAggATYCTAJAIAdBG0cEQCAHRQ0LIA5BAE4EQCAARQRAIAQgDkECdGogBzYCAAwLCyAIIAMgDkEDdGopAwA3A0AMAgsgAEUNByAIQUBrIAcgAiAGEIcEDAELIA5BAE4NCkEAIQcgAEUNBwtBfyEQIAAtAABBIHENCiAMQf//e3EiCSAMIAxBgMAAcRshDEEAIQ5BlgohFyASIQsCQAJAAkACfwJAAkACQAJAAn8CQAJAAkACQAJAAkACQCAWLAAAIgdBX3EgByAHQQ9xQQNGGyAHIBUbIgdB2ABrDiEEFBQUFBQUFBQOFA8GDg4OFAYUFBQUAgUDFBQJFAEUFAQACwJAIAdBwQBrDgcOFAsUDg4OAAsgB0HTAEYNCQwTCyAIKQNAIR1BlgoMBQtBACEHAkACQAJAAkACQAJAAkAgFUH/AXEOCAABAgMEGgUGGgsgCCgCQCARNgIADBkLIAgoAkAgETYCAAwYCyAIKAJAIBGsNwMADBcLIAgoAkAgETsBAAwWCyAIKAJAIBE6AAAMFQsgCCgCQCARNgIADBQLIAgoAkAgEaw3AwAMEwtBCCAKIApBCE0bIQogDEEIciEMQfgAIQcLIBIhDSAIKQNAIh1CAFIEQCAHQSBxIRYDQCANQQFrIg0gHadBD3FBkMcCai0AACAWcjoAACAdQg9WIRsgHUIEiCEdIBsNAAsLIAgpA0BQDQMgDEEIcUUNAyAHQQR2QZYKaiEXQQIhDgwDCyASIQcgCCkDQCIdQgBSBEADQCAHQQFrIgcgHadBB3FBMHI6AAAgHUIHViEcIB1CA4ghHSAcDQALCyAHIQ0gDEEIcUUNAiAKIBIgB2siB0EBaiAHIApIGyEKDAILIAgpA0AiHUIAUwRAIAhCACAdfSIdNwNAQQEhDkGWCgwBCyAMQYAQcQRAQQEhDkGXCgwBC0GYCkGWCiAMQQFxIg4bCyEXIB0gEhDGASENCyAUQQAgCkEASBsNDyAMQf//e3EgDCAUGyEMAkAgCCkDQCIdQgBSDQAgCg0AIBIhDUEAIQoMDAsgCiAdUCASIA1raiIHIAcgCkgbIQoMCwsgCCgCQCIHQfbTACAHGyINQf////8HIAogCkH/////B08bIgsQiwQiByANayALIAcbIgcgDWohCyAKQQBOBEAgCSEMIAchCgwLCyAJIQwgByEKIAstAAANDgwKCyAKBEAgCCgCQAwCC0EAIQcgAEEgIA9BACAMEGcMAgsgCEEANgIMIAggCCkDQD4CCCAIIAhBCGoiBzYCQEF/IQogBwshCUEAIQcCQANAIAkoAgAiDUUNAQJAIAhBBGogDRCFBCILQQBIIg0NACALIAogB2tLDQAgCUEEaiEJIAcgC2oiByAKSQ0BDAILCyANDQ4LQT0hECAHQQBIDQwgAEEgIA8gByAMEGcgB0UEQEEAIQcMAQtBACELIAgoAkAhCQNAIAkoAgAiDUUNASAIQQRqIgogDRCFBCINIAtqIgsgB0sNASAAIAogDRBhIAlBBGohCSAHIAtLDQALCyAAQSAgDyAHIAxBgMAAcxBnIA8gByAHIA9IGyEHDAgLIBRBACAKQQBIGw0JQT0hECAAIAgrA0AgDyAKIAwgByAFESgAIgdBAE4NBwwKCyAIIAgpA0A8ADdBASEKIBghDSAJIQwMBAsgBy0AASEJIAdBAWohBwwACwALIBEhECAADQcgE0UNAkEBIQcDQCAEIAdBAnRqKAIAIgAEQCADIAdBA3RqIAAgAiAGEIcEQQEhECAHQQFqIgdBCkcNAQwJCwtBASEQIAdBCk8NBwNAIAQgB0ECdGooAgANASAHQQFqIgdBCkcNAAsMBwtBHCEQDAULIAogCyANayIJIAkgCkgbIgsgDkH/////B3NKDQNBPSEQIA8gCyAOaiIKIAogD0gbIgcgGUoNBCAAQSAgByAKIAwQZyAAIBcgDhBhIABBMCAHIAogDEGAgARzEGcgAEEwIAsgCUEAEGcgACANIAkQYSAAQSAgByAKIAxBgMAAcxBnDAELCwtBACEQDAILQT0hEAsjA0EcaiAQNgIAQX8hEAsgCEHQAGokACAQC38CAX8BfiAAvSIDQjSIp0H/D3EiAkH/D0cEfCACRQRAIAEgAEQAAAAAAAAAAGEEf0EABSAARAAAAAAAAPBDoiABEIoEIQAgASgCAEFAags2AgAgAA8LIAEgAkH+B2s2AgAgA0L/////////h4B/g0KAgICAgICA8D+EvwUgAAsLuAEBAX8gAUEARyECAkACQAJAIABBA3FFDQAgAUUNAANAIAAtAABFDQIgAUEBayIBQQBHIQIgAEEBaiIAQQNxRQ0BIAENAAsLIAJFDQECQCAALQAARQ0AIAFBBEkNAANAIAAoAgAiAkF/cyACQYGChAhrcUGAgYKEeHENAiAAQQRqIQAgAUEEayIBQQNLDQALCyABRQ0BCwNAIAAtAABFBEAgAA8LIABBAWohACABQQFrIgENAAsLQQALHQAgACAAEKEENgJ4IABBAf4XAnwgAEEA/hcCgAELswECAX0CfyAAiyEBAkAgALwiA0H/////B3EiAkHVvrL4A08EQCACQYGAgIkETwRAQwAAAAAgAZVDAACAP5IhAQwCC0MAAIA/QwAAAEAgASABkhDEAkMAAABAkpWTIQEMAQsgAkH5iov0A08EQCABIAGSEMQCIgAgAEMAAABAkpUhAQwBCyACQYCAgARJDQAgAUMAAADAlBDEAiIAjCAAQwAAAECSlSEBCyABjCABIANBAEgbC9oBAQJ/AkAgAUH/AXEiAwRAIABBA3EEQANAIAAtAAAiAkUNAyACIAFB/wFxRg0DIABBAWoiAEEDcQ0ACwsCQCAAKAIAIgJBf3MgAkGBgoQIa3FBgIGChHhxDQAgA0GBgoQIbCEDA0AgAiADcyICQX9zIAJBgYKECGtxQYCBgoR4cQ0BIAAoAgQhAiAAQQRqIQAgAkGBgoQIayACQX9zcUGAgYKEeHFFDQALCwNAIAAiAi0AACIDBEAgAEEBaiEAIAMgAUH/AXFHDQELCyACDwsgABBoIABqDwsgAAveGgMVfwV7In0jAEHAAmsiAyQAIAJBgAJOBEAgAkGAAm0hCgNAQwAAAAAhJkMAAAAAISVBACEEA0AgA0FAayAEQQR0aiEJAkAgACAEQQZ0aiIG/QACMCIY/R8DIh6LIh8gGP0fAiIgiyIhIBj9HwEiIosiIyAY/R8AIieLIiQgBv0AAiAiGf0fAyIziyIoIBn9HwIiNIsiKSAZ/R8BIjWLIiogGf0fACI2iyIrIAb9AAIQIhr9HwMiN4siLCAa/R8CIjiLIi0gGv0fASI5iyIuIBr9HwAiOosiLyAG/QACACIb/R8DIjuLIjAgG/0fAiI8iyIxIBv9HwEiPYsiMiAb/R8AIh2LQwAAAAAgHUMAAAAAXSAdQwAAAABeciICGyI+IDIgPl4iBRsiMiAxIDJeIgcbIjEgMCAxXiILGyIwIC8gMF4iDBsiLyAuIC9eIg0bIi4gLSAuXiIOGyItICwgLV4iDxsiLCArICxeIhAbIisgKiArXiIRGyIqICkgKl4iEhsiKSAoICleIhMbIiggJCAoXiIUGyIkICMgJF4iFRsiIyAhICNeIhYbIiEgHyAhXiIXG0NgQqINXQRAIAn9DAAAAAAAAAAAAAAAAAAAAAD9CwQAQwAAAAAhHwwBC0MAAABCIB4gICAiICcgMyA0IDUgNiA3IDggOSA6IDsgPCA9IB1DAAAAACACGyAFGyAHGyALGyAMGyANGyAOGyAPGyAQGyARGyASGyATGyAUGyAVGyAWGyAXGyInlSEhQwAAAAAhHUEAIQJDAAAAACEeA0AgAiAJakHg//8BQZ+AgAJDAABASyAhIAYgAkECdGoqAgAiH5STvEH///8DcSIFIAVBn4CAAk8bIgUgBUHg//8BTBsiBUEgajoAACAfIB+UIiIgBUGAgIACa7IiIJQgIJQgHZIhHSAfICKUICCUIB6SIR4gAkEBaiICQRBHDQALIB4gHiAdlSIflCEhQXchBQNAAkAgBUUNACAFskPNzMw9lEMAAABCkowgJ5UhIkMAAAAAIR5BACECQwAAAAAhHQNAIAYgAkECdGoqAgAiICAglCIkQeD//wFBn4CAAiAiICCUQwAAQEuSvEH///8DcSIHIAdBn4CAAk8bIgcgB0Hg//8BTBtBgICAAmuyIiOUICOUIB6SIR4gICAklCAjlCAdkiEdIAJBAWoiAkEQRw0ACyAeQwAAAABeRQ0AIB0gHZQgISAelF5FDQAgCSAbICL9EyIc/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DB8AQAAfAEAAHwBAAB8AQAD9tgH9DOD/PwDg/z8A4P8/AOD/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9TiAaIBz95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MHwBAAB8AQAAfAEAAHwBAAP22Af0M4P8/AOD/PwDg/z8A4P8/AP24Af0M/wAAAP8AAAD/AAAA/wAAAP1O/YYBIBkgHP3mAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/Qzg/z8A4P8/AOD/PwDg/z8A/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U4gGCAc/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DB8AQAAfAEAAHwBAAB8AQAD9tgH9DOD/PwDg/z8A4P8/AOD/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GAf1m/QwgICAgICAgICAgICAgICAg/W79CwQAIB0gHSAelSIflCEhCyAFQQFqIgVBCkcNAAsLIAMgBEECdGogHzgCACAfICYgH4siHSAlXiICGyEmIB0gJSACGyElIARBAWoiBEEQRw0ACwJAICVDAAAAAFsEQCABIAhB0gFsakEAQdIB/AsADAELIAEgCEHSAWxqIgRBgPwBQwAAgD9DAAAAwyAmlSIdlSIei0MAAIB3lEMAAIAIlEGAgICIByAevCICQQF0IgZBgICAeHEiBSAFQYCAgIgHTRtBAXZBgICAPGq+krwiBUENdkGA+AFxIAVB/x9xaiAGQYCAgHhLGyACQRB2QYCAAnFyIgI7AdABIAQgHf0TIhggA/0ABAD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MfwBAAH8AQAB/AEAAfwBAAP22Af0M/wAAAP8AAAD/AAAA/wAAAP1OIBggA/0ABBD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MfwBAAH8AQAB/AEAAfwBAAP22Af0M/wAAAP8AAAD/AAAA/wAAAP1O/YYBIBggA/0ABCD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MfwBAAH8AQAB/AEAAfwBAAP22Af0M/wAAAP8AAAD/AAAA/wAAAP1OIBggA/0ABDD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MfwBAAH8AQAB/AEAAfwBAAP22Af0M/wAAAP8AAAD/AAAA/wAAAP1O/YYB/Wb9CwDAASACQQJ0QZDWBGoqAgAhHUEAIQIDQCAdIAIgBGosAMABspQiHkMAAAAAXARAIANBQGsgAkEEdGogACACQQZ0aiIG/QACACAe/RMiGP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/Qzg/z8A4P8/AOD/PwDg/z8A/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U4gBv0AAhAgGP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/Qzg/z8A4P8/AOD/PwDg/z8A/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U79hgEgBv0AAiAgGP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/Qzg/z8A4P8/AOD/PwDg/z8A/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U4gBv0AAjAgGP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/Qzg/z8A4P8/AOD/PwDg/z8A/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U79hgH9Zv0MICAgICAgICAgICAgICAgIP1u/QsEAAsgAkEBaiICQRBHDQALIAP9AASAASEYIAP9AARAIRkgBCAD/QAEoAEiGkEE/WsgA/0ABGAiG/0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9CwAgIAQgGEEE/WsgGf0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9CwAAIAQgG0EC/Wz9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9TiAZQQT9bP1QIBj9DPDw8PDw8PDw8PDw8PDw8PD9Tv1QIBpBAv1r/QzAwMDAwMDAwMDAwMDAwMDA/U79UP0LAIABIAP9AASQASEYIAP9AARQIRkgBCAD/QAEsAEiGkEE/WsgA/0ABHAiG/0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9CwAwIAQgGEEE/WsgGf0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9CwAQIAQgG0EC/Wz9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9TiAZQQT9bP1QIBj9DPDw8PDw8PDw8PDw8PDw8PD9Tv1QIBpBAv1r/QzAwMDAwMDAwMDAwMDAwMDA/U79UP0LAJABIAP9AASAAiEYIAP9AATAASEZIAQgA/0ABKACIhpBBP1rIAP9AATgASIb/QwPDw8PDw8PDw8PDw8PDw8P/U79UP0LAGAgBCAYQQT9ayAZ/QwPDw8PDw8PDw8PDw8PDw8P/U79UP0LAEAgBCAbQQL9bP0M/Pz8/Pz8/Pz8/Pz8/Pz8/P1OIBlBBP1s/VAgGP0M8PDw8PDw8PDw8PDw8PDw8P1O/VAgGkEC/Wv9DMDAwMDAwMDAwMDAwMDAwMD9Tv1Q/QsAoAEgA/0ABJACIRggA/0ABNABIRkgBCAD/QAEsAIiGkEE/WsgA/0ABPABIhv9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/QsAcCAEIBhBBP1rIBn9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/QsAUCAEIBtBAv1s/Qz8/Pz8/Pz8/Pz8/Pz8/Pz8/U4gGUEE/Wz9UCAY/Qzw8PDw8PDw8PDw8PDw8PDw/U79UCAaQQL9a/0MwMDAwMDAwMDAwMDAwMDAwP1O/VD9CwCwAQsgAEGACGohACAIQQFqIgggCkcNAAsLIANBwAJqJAALtwEBAX8CQBDIAkEKRw0AQeQAIQADQAJAIABFDQBBoPc0KAIARQ0AIABBAWshAEGk9zQoAgBFDQELCxDIAkEKRw0AA0ACQEGg9zQoAgAiAEH/////B3FB/////wdHDQBBpPc0QQH+HgIAGkGg9zQgACAAQYCAgIB4ciIA/kgCABpBoPc0IABBqPc0KAIAQYABcxDLAiEAQaT3NEEB/iUCABogAEUNACAAQRtHDQILEMgCQQpGDQALCws1ACAAKAIAQYEBTgRAQdD7NCgCACIABEADQEHQ+zRB1Ps0IAAQoQFB0Ps0KAIAIgANAAsLCwufAwEEfyMDIgIoAkhFBEAgAkGg8zQ2AkgLAkBBrPc0KAIAIwMoAhhGDQBBoPc0QQBB/////wf+SAIABH9BCgVBrPc0IwMoAhg2AgBBAAtBCkcNAEHkACECA0ACQCACRQ0AQaD3NCgCAEUNACACQQFrIQJBpPc0KAIARQ0BCwtBoPc0QQBB/////wf+SAIABH9BCgVBrPc0IwMoAhg2AgBBAAtBCkYEQANAAkBBoPc0KAIAIgJFDQBBpPc0QQH+HgIAGkGg9zQgAiACQYCAgIB4ciIC/kgCABpBoPc0IAJBqPc0KAIAQYABcxDLAiECQaT3NEEB/iUCABogAkUNACACQRtHDQMLQaD3NEEAQf////8H/kgCAAR/QQoFQaz3NCMDKAIYNgIAQQALQQpGDQALC0Gs9zQjAygCGDYCAAsgAUHJASABGyECQcD3NCgCACIDIQECfwNAIAFBAnRB0Pc0aiIEKAIARQRAIAAgATYCAEHA9zQgATYCACAEIAI2AgBBAAwCCyABQQFqQf8AcSIBIANHDQALQQYLIQUQxwIgBQvkAgEEfxAMIwBBEGsiAiQAAkAgACAAKAIARwRAQccAIQEMAQsCQCAAKAIgQQNGDQAgACMDRw0AQRAhAQwBCyMDIQEgAkEMagRAIAIgAS0AKDYCDAsgAUEBOgAoIAIoAgxFBEAjA0EAOgAoCwJAIABBIGoiBCgCACIDBEADQCADQQNOBEAgAigCDCIAQQJNBH8jAyAAOgAoQQAFQRwLGkEcIQEMBAsgBCADQQEQ+QEhAQJAIAQoAgAiA0UNACABQckARg0AIAFBHEcNAQsLIAIoAgwiBEECTQR/IwMgBDoAKEEABUEcCxogAUEcRg0CIAFByQBGDQIMAQtBACEDIAIoAgwiAUECTQR/IwMgAToAKEEABUEcCxoLIwBBEGtBADYCDAJAQYzzNCgCACIBRQ0AQYzzNEGU8zQgARChAUGU8zQoAgBFDQBBjPM0EI4BC0EAIQEgAw0AIAAQCwsgAkEQaiQAIAELiAYBBn8jAEEgayICJAAgAkEANgIYIAJCADcDECACQgA3AwggACgCEBojBARAEAwLAkAgAS0AAEEPcQRAIwMoAhggASgCBEH/////B3FHDQELAn8gACgCACIHBEAgACgCCCEEIABBDGpBAf4eAgAaIABBCGoMAQsgAEEgaiIDEPgBQQIhBCACQQI2AhQgAkEANgIQIAIgACgCBCIFNgIMIAAgAkEIaiIGNgIEIAUgAEEUaiAAKAIUGyAGNgIAIAMQ9wEgAkEUagshBSABEFIaIwMhAyACQQRqBEAgAiADLQAoNgIECyADQQI6ACggAigCBEEBRgRAIwNBAToAKAsgBSAEIAdFIgYQ+QEhAwJAIAUoAgAgBEcNAANAIANBG0dBACADGw0BIAUgBCAGEPkBIQMgBSgCACAERg0ACwsgA0EAIANBG0cbIQMCfwJAIAcEQCADQQtGBEBBC0EAIAAoAgggBEYbIQMLIABBDGoiAEF//h4CAEGBgICAeEcNASAAEI4BDAELIAJBEGpBAEEC/kgCAEUEQCAAQSBqIgQQ+AECQCAAKAIEIAJBCGpGBEAgACACKAIMNgIEDAELIAIoAggiBUUNACAFIAIoAgw2AgQLAkAgACgCFCACQQhqRgRAIAAgAigCCDYCFAwBCyACKAIMIgBFDQAgACACKAIINgIACyAEEPcBIAIoAhgiAEUNASAAQX/+HgIAQQFHDQEgAigCGBCOAQwBCyACQRRqEPgBIAEQVhoCQCACKAIMDQAgAS0AAEEIcQ0AIAFBCGpBAf4eAgAaCwJAIAIoAggiAwRAIAEoAgQiAEEASgRAIAFBBGogACAAQYCAgIB4cv5IAgAaCyADQQxqIgBBAP4XAgAgAEH/////BxC1AQwBCyABLQAAQQhxDQAgAUEIakEB/iUCABoLIAIoAgQMAQsgARBWIQAgAigCBCIBQQJNBH8jAyABOgAoQQAFQRwLGiAAIAMgABtBC0cNAUEBCyIAQQJNBH8jAyAAOgAoQQAFQRwLGgsgAkEgaiQAC/8BAQV/IwBBEGsiAyQAIANBADYCDCAAQSBqIgYQ+AEgACgCFCICQQBHIQQCQCABRQ0AIAJFDQADQAJAIAJBCGpBAEEB/kgCAARAIAMgAygCDEEBajYCDCACIANBDGo2AhAMAQsgBSACIAUbIQUgAUEBayEBCyACKAIAIgJBAEchBCABRQ0BIAINAAsLAkAgBARAIAJBBGohASACKAIEIgRFDQEgBEEANgIADAELIABBBGohAQsgAUEANgIAIAAgAjYCFCAGEPcBIAMoAgwiAgRAA0AgA0EMakEAIAIQoQEgAygCDCICDQALCyAFBEAgBUEMahD3AQsgA0EQaiQAQQALLwAgACgCAEUEQCAAQQEQlQQaDwsgACgCDARAIABBCGoiAEEB/h4CABogABCOAQsLzSMDEH0MfwJ7IwBB4ANrIhYkACACQYACTgRAIAJBgAJtIR0DQEMAAAAAIQtDAAAAACEMQQAhFQNAIBYgACAVQQd0aiITKgJ8IgMgA5QgEyoCeCIDIAOUIBMqAnQiAyADlCATKgJwIgMgA5QgEyoCbCIDIAOUIBMqAmgiAyADlCATKgJkIgMgA5QgEyoCYCIDIAOUIBMqAlwiAyADlCATKgJYIgMgA5QgEyoCVCIDIAOUIBMqAlAiAyADlCATKgJMIgMgA5QgEyoCSCIDIAOUIBMqAkQiAyADlCATQUBrIgIqAgAiAyADlCATKgI8IgMgA5QgEyoCOCIDIAOUIBMqAjQiAyADlCATKgIwIgMgA5QgEyoCLCIDIAOUIBMqAigiAyADlCATKgIkIgMgA5QgEyoCICIDIAOUIBMqAhwiAyADlCATKgIYIgMgA5QgEyoCFCIDIAOUIBMqAhAiAyADlCATKgIMIgMgA5QgEyoCCCIDIAOUIBMqAgQiAyADlCATKgIAIgMgA5RDAAAAAJKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSQwAAAD2Ukf0TIh8gE/0AAgD94AH95AEiIP0LBCAgFiAfIBP9AAIQ/eAB/eQB/QsEMCAWIB8gE/0AAiD94AH95AH9CwRAIBYgHyAT/QACMP3gAf3kAf0LBFAgFiAfIAL9AAIA/eAB/eQB/QsEYCAWIB8gE/0AAlD94AH95AH9CwRwIBYgHyAT/QACYP3gAf3kAf0LBIABIBYgHyAT/QACcP3gAf3kAf0LBJABIBMqAgAiAyAg/R8AIgeUIQYgFkHgAWogFUEFdGohF0EBIQIgAyEEAkADQAJAIBMgAkECdCIUaioCACIFIAQgBCAFXRshDSAFIAMgAyAFXhshAyAWQSBqIBRqKgIAIgQgBZQgBpIhDyAHIASSIRAgAkEBaiIUQSBGBEAgFUECdCIZIBZBwAFqaiEYIA1DAAAAACADIANDAAAAAF4bIgVcDQEgF/0MAAAAAAAAAAAAAAAAAAAAAP0LBAAgF/0MAAAAAAAAAAAAAAAAAAAAAP0LBBBDAAAAACEGDAMFIBMgFEECdCIUaioCACIFIA0gBSANXhshBCAFIAMgAyAFXhshAyAWQSBqIBRqKgIAIgogBZQgD5IhBiACQQJqIQIgECAKkiEHDAILAAsLQwAAgD9DAAD4QSANIAWTlSIDlSEGQwAAAAAhCkEAIQIDQCACIBdqQYCAgAJBn4CAAiADIBMgAkECdCIUaioCACIEIAWTlEMAAEBLkrxB////A3EiGiAaQZ+AgAJPGyIaIBpBgICAAkwbIho6AAAgFkEgaiAUaioCACAGIBpBgICAAmuylCAFkiAEkyIEIASUlCAKkiEKIAJBAWoiAkEgRw0ACyAPjCERQQAhAgNAIAIiFLJDzczMPZRDAAAAv5JDAAD4QZIgDSAFk5UhCEMAAAAAIQdBACECQwAAAAAhBEMAAAAAIQMDQCACIBZqQYCAgAJBn4CAAiAIIBMgAkECdCIaaioCACIOIAWTlEMAAEBLkrxB////A3EiGyAbQZ+AgAJPGyIbIBtBgICAAkwbIhs6AAAgFkEgaiAaaioCACAbQYCAgAJrsiISlCIJIBKUIASSIQQgCSADkiEDIAkgDpQgB5IhByACQQFqIgJBIEcNAAsCQCAQIASUIAMgA5STIg5DAAAAAF5FDQBDAAAAACEJQwAAAAAgBCAPlCAHIAOMlJIgDpUiCCAIQwAAAABeIgIbIQggByAElSAQIAeUIAMgEZSSIA6VIAIbIQNBACECA0AgAkEBciIaQQJ0IhsgFkEgaiIeaioCACADIBYgGmotAACzlCAIkiATIBtqKgIAkyIEIASUlCAeIAJBAnQiGmoqAgAgAyACIBZqLQAAs5QgCJIgEyAaaioCAJMiBCAElJQgCZKSIQkgAkECaiICQSBHDQALIAkgCl1FDQAgFyAW/QAEAP0LBAAgFyAW/QAEEP0LBBAgAyEGIAkhCiAIIQULIBRBAWohAiAUQQ9HDQALCyAYIAWMIgM4AgAgFkGgAWogGWogBjgCACAGIAsgBiALXhshCyADIAwgAyAMXhshDCAVQQFqIhVBCEcNAAsgFioCzAEhBCAWKgLIASEFIBYqAsABIQYgFioCxAEhCiAWKgKsASEJIBYqAqgBIQggFioCpAEhByABIBxBsAFsaiITQwAAfEIgC5VDAAAAACALQwAAAABeGyID/RMgFv0ABLAB/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz/AAAA/wAAAP8AAAD/AAAA/U79DD8AAAA/AAAAPwAAAD8AAAD9twEiH/0WAEECdEHAAXFBPyADIBYqAqABlEMAAEBLkrxB/wFxIgIgAkE/TxtyOgAEIBb9AATQASEgIBMgH/0WBEECdEHAAXFBPyADIAeUQwAAQEuSvEH/AXEiAiACQT9PG3I6AAUgEyAf/RYIQQJ0QcABcUE/IAMgCJRDAABAS5K8Qf8BcSICIAJBP08bcjoABiATIB/9FgxBAnRBwAFxQT8gAyAJlEMAAEBLkrxB/wFxIgIgAkE/TxtyOgAHIBNBgPwBIAxDAAB8QpUiA4tDAACAd5RDAACACJRBgICAiAcgA7wiAkEBdCIUQYCAgHhxIhUgFUGAgICIB00bQQF2QYCAgDxqvpK8IhVBDXZBgPgBcSAVQf8fcWogFEGAgIB4SxsgAkEQdkGAgAJxciICOwECIBNBgPwBIAtDAAB8QpUiA4tDAACAd5RDAACACJRBgICAiAcgA7wiFEEBdCIVQYCAgHhxIhcgF0GAgICIB00bQQF2QYCAgDxqvpK8IhdBDXZBgPgBcSAXQf8fcWogFUGAgIB4SxsgFEEQdkGAgAJxciIUOwEAIBNBDGogIEMAAHxCIAyVQwAAAAAgDEMAAAAAXhsiA/0T/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz/AAAA/wAAAP8AAAD/AAAA/U79DD8AAAA/AAAAPwAAAD8AAAD9twEiIEEE/asBIB/9DA8AAAAPAAAADwAAAA8AAAD9Tv1Q/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoAAAAgEyAg/RYEQQJ0QcABcUE/IAMgCpRDAABAS5K8Qf8BcSIVIBVBP08bcjoACSATICD9FgBBAnRBwAFxQT8gAyAGlEMAAEBLkrxB/wFxIhUgFUE/TxtyOgAIIBMgIP0WCEECdEHAAXFBPyADIAWUQwAAQEuSvEH/AXEiFSAVQT9PG3I6AAogEyAg/RYMQQJ0QcABcUE/IAMgBJRDAABAS5K8Qf8BcSIVIBVBP08bcjoACyACQQJ0QZDWBGohGSAUQQJ0QZDWBGoqAgAhAyATQQRqIRdBACECA0ACQCACQQNNBEAgAiAXaiIULQAAQT9xIRUgFC0ABEE/cSEUDAELIAIgF2oiFS0AAEECdkEwcSAVLQAEIhhBBHZyIRQgFUEEay0AAEECdkEwcSAYQQ9xciEVCyADIBWzlCIEQwAAAABcBEAgAkEFdCIVIBZB4AFqIhhqIBkqAgAgFLOU/RMiHyAAIAJBB3Rq/QACAP3kASAE/RMiIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQQRyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQQhyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQQxyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQRByIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQRRyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQRhyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCAVQRxyIhRqIB8gACAUQQJ0av0AAgD95AEgIP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwfAEAAHwBAAB8AQAAfAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAALIAJBAWoiAkEIRw0ACyAT/QwAAAAAAAAAAAAAAAAAAAAA/QsAECAT/QwAAAAAAAAAAAAAAAAAAAAA/QsAICATQTBqIRcgE0EQaiEVQQAhAgNAIAIgF2ogFkHgAWogAmotAAAiFEEQTwR/IAIgFWoiEyATLQAAQQFyOgAAIBRBEGsFIBQLIAIgFmotAIACIhNBEE8EfyACIBVqIhkgGS0AAEECcjoAACATQRBrBSATC0EEdHI6AAAgAkEBaiICQSBHDQALIBdBIGohGUEAIQIDQCACIBlqIAIgFkHgAWpqIhNBQGstAAAiFEEQTwR/IAIgFWoiGCAYLQAAQQRyOgAAIBRBEGsFIBQLIBMtAGAiE0EQTwR/IAIgFWoiGCAYLQAAQQhyOgAAIBNBEGsFIBMLQQR0cjoAACACQQFqIgJBIEcNAAsgF0FAayEZQQAhAgNAIAIgGWogAiAWQeABamoiEy0AgAEiFEEQTwR/IAIgFWoiGCAYLQAAQRByOgAAIBRBEGsFIBQLIBMtAKABIhNBEE8EfyACIBVqIhggGC0AAEEgcjoAACATQRBrBSATC0EEdHI6AAAgAkEBaiICQSBHDQALIBdB4ABqIRdBACECA0AgAiAXaiACIBZB4AFqaiITLQDAASIUQRBPBH8gAiAVaiIZIBktAABBwAByOgAAIBRBEGsFIBQLIBMtAOABIhNBEE8EfyACIBVqIhkgGS0AAEGAAXI6AAAgE0EQawUgEwtBBHRyOgAAIAJBAWoiAkEgRw0ACyAAQYAIaiEAIBxBAWoiHCAdRw0ACwsgFkHgA2okAAsNAEGA8zRBxgEQkgQaC5EBAQR/IwBBIGsiASQAAkAgACgCCEUEQCAAQRBqIgIQVhogAEECNgIMIAIQUhogAEEoahCWBAwBCyAAKAIYBEAgACgCECECIAAoAgwhBCABIAA2AhwgASAANgIQIAFBwAE2AhggAUHFATYCFCABIAEpAhQ3AwggBCACIAFBCGoQ+wENAQsgABDSAQsgAUEgaiQAC38BAn8gACgCCEUEQCAAQRBqEJEEAkAgAEEoaiIAKAIARQ0AIAAoAgxFDQAgAEEMaiIBQYCAgIB4/jMCABogAEEIaiICQQH+HgIAGiACEPoBIAAoAgwiAEH/////B3FFDQADQCABQQAgABChASABKAIAIgBB/////wdxDQALCwsLaAEBfyAAIAAoAlhGBEAgAEIANwJYQYDzNCgCAEEAELIBDwsgACMDKAJIQYDzNCgCACIBQQJ0aigCAEYEQCABIAAoAlgQsgELIAAoAlwiASAAKAJYNgJYIAAoAlggATYCXCAAQgA3AlgLSgEDfwJAIAAoAhwiAkEATA0AIAAoAhghA0EAIQADQCABIAMgAEECdGooAgAiBCgCHEcEQCACIABBAWoiAEcNAQwCCwsgBA8LQQAL/wEBBX8CQCAAKAIsIAAoAjBBAWogACgCKCICb0cNACACQRhsEDsiAwR/IAJBAXQhBQJAIAAoAjAiBCAAKAIsIgJOBEAgAyAAKAIkIAJBDGxqIAQgAmsiAkEMbBB7GgwBCyADIAAoAiQgAkEMbGogACgCKCACayICQQxsIgYQexogAyAGaiAAKAIkIARBDGwQexogAiAEaiECCyAAKAIkEC8gACACNgIwIABBADYCLCAAIAU2AiggACADNgIkQQEFQQALDQBBAA8LIAAoAiQgACgCMEEMbGoiAyABKQIANwIAIAMgASgCCDYCCCAAIAAoAjBBAWogACgCKG82AjBBAQtnAQN/IwBBEGsiASQAIABBBGoiAhBWGiAAKAIsIAAoAjBHBEADQCABQQRqIAAQnwQgASgCCCIDBEAgASgCDCADEQEACyAAKAIsIAAoAjBHDQALCyACEFIaIABBAP4XAgAgAUEQaiQACzgBAn8gACABKAIkIAEoAiwiAkEMbGoiAykCADcCACAAIAMoAgg2AgggASACQQFqIAEoAihvNgIsCxUAIABBBGoQkQQgACgCJBAvIAAQLwvkAgEGfyMAQUBqIgEkAEHUuAMQ9QFFBEBBiLkDKAIAIgJB0LgDRwRAA0AgAigCOCEGIAL+EAIARQRAIAIoAjQiBCACKAI4NgI4IAIoAjggBDYCNCACEKAECyAGIgJB0LgDRw0ACwtB1LgDEFIaCwJAQTwQOyICRQ0AQYAMEDsiA0UEQCACEC8MAQsgAUIANwMoIAFCADcDMCABQQA2AjwgAUIANwMgIAEgADYCHCABQQA2AhggASADNgIUIAFBgAE2AhAgAUEANgIMIAFBADYCCCABQQA2AgQgAUEANgIAIAIgASgCPDYCACACIAEpAzA3AhQgAiABKQMoNwIMIAIgASkDIDcCBCACIAEoAhw2AhwgAiABKAIYNgIgIAIgASgCFDYCJCACIAEoAhA2AiggAiABKAIMNgIsIAIgASgCCDYCMCACIAEoAgQ2AjQgAiABKAIANgI4IAIhBQsgAUFAayQAIAULLAEBfyMAQRBrIgIkACACIAE2AgxBsLoDIAAgAUHNAUEAEMMCGiACQRBqJAALRgEBfwJ/QQAgAEEXdkH/AXEiAUH/AEkNABpBAiABQZYBSw0AGkEAQQFBlgEgAWt0IgFBAWsgAHENABpBAUECIAAgAXEbCwtOAgF/AX4Cf0EAIABCNIinQf8PcSIBQf8HSQ0AGkECIAFBswhLDQAaQQBCAUGzCCABa62GIgJCAX0gAINCAFINABpBAkEBIAAgAoNQGwsLswEBA38CQEGb8TQsAAAiAkUNACAAQQBBgYCAgHj+SAIAIQEgAkEASARAQZvxNEEAOgAACyABRQ0AA0AgACABQf////8HaiABIAFBAEgbIgIgAkH/////B2v+SAIAIgEgAkYNASADQQFqIgNBCkcNAAsgAEEB/h4CAEEBaiEBA0AgAUEASARAIAAgARCtBCABQf////8HaiEBCyABIAAgASABQYCAgIB4cv5IAgAiAUcNAAsLC+kgAxB9DX8CeyMAQeADayIUJAAgAkGAAk4EQCACQYACbSEdA0BDAAAAACELQwAAAAAhDEEAIRcDQCAUIAAgF0EHdGoiEyoCfCIDIAOUIBMqAngiAyADlCATKgJ0IgMgA5QgEyoCcCIDIAOUIBMqAmwiAyADlCATKgJoIgMgA5QgEyoCZCIDIAOUIBMqAmAiAyADlCATKgJcIgMgA5QgEyoCWCIDIAOUIBMqAlQiAyADlCATKgJQIgMgA5QgEyoCTCIDIAOUIBMqAkgiAyADlCATKgJEIgMgA5QgE0FAayICKgIAIgMgA5QgEyoCPCIDIAOUIBMqAjgiAyADlCATKgI0IgMgA5QgEyoCMCIDIAOUIBMqAiwiAyADlCATKgIoIgMgA5QgEyoCJCIDIAOUIBMqAiAiAyADlCATKgIcIgMgA5QgEyoCGCIDIAOUIBMqAhQiAyADlCATKgIQIgMgA5QgEyoCDCIDIAOUIBMqAggiAyADlCATKgIEIgMgA5QgEyoCACIDIAOUQwAAAACSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkkMAAAA9lJH9EyIgIBP9AAIA/eAB/eQBIiH9CwRAIBQgICAT/QACEP3gAf3kAf0LBFAgFCAgIBP9AAIg/eAB/eQB/QsEYCAUICAgE/0AAjD94AH95AH9CwRwIBQgICAC/QACAP3gAf3kAf0LBIABIBQgICAT/QACUP3gAf3kAf0LBJABIBQgICAT/QACYP3gAf3kAf0LBKABIBQgICAT/QACcP3gAf3kAf0LBLABIBMqAgAiAyAh/R8AIgeUIQYgFEHgAWogF0EFdGohFkEBIQIgAyEEAkADQAJAIBMgAkECdCIVaioCACIFIAQgBCAFXRshDSAFIAMgAyAFXhshAyAUQUBrIBVqKgIAIgQgBZQgBpIhDyAHIASSIRAgAkEBaiIVQSBGBEAgF0ECdCIbIBRBIGpqIRwgDUMAAAAAIAMgA0MAAAAAXhsiBVwNASAW/QwAAAAAAAAAAAAAAAAAAAAA/QsEACAW/QwAAAAAAAAAAAAAAAAAAAAA/QsEEEMAAAAAIQYMAwUgEyAVQQJ0IhVqKgIAIgUgDSAFIA1eGyEEIAUgAyADIAVeGyEDIBRBQGsgFWoqAgAiCiAFlCAPkiEGIAJBAmohAiAQIAqSIQcMAgsACwtDAACAP0MAAHBBIA0gBZOVIgOVIQZDAAAAACEKQQAhAgNAIAIgFmpBgICAAkGPgIACIAMgEyACQQJ0IhVqKgIAIgQgBZOUQwAAQEuSvEH///8DcSIYIBhBj4CAAk8bIhggGEGAgIACTBsiGDoAACAUQUBrIBVqKgIAIAYgGEGAgIACa7KUIAWSIASTIgQgBJSUIAqSIQogAkEBaiICQSBHDQALIA+MIRFBACECA0AgAiIVskPNzMw9lEMAAIC/kkMAAHBBkiANIAWTlSEIQwAAAAAhB0EAIQJDAAAAACEEQwAAAAAhAwNAIBRBwAFqIAJqQYCAgAJBj4CAAiAIIBMgAkECdCIYaioCACIOIAWTlEMAAEBLkrxB////A3EiGSAZQY+AgAJPGyIZIBlBgICAAkwbIhk6AAAgFEFAayAYaioCACAZQYCAgAJrsiISlCIJIBKUIASSIQQgCSADkiEDIAkgDpQgB5IhByACQQFqIgJBIEcNAAsCQCAQIASUIAMgA5STIg5DAAAAAF5FDQBDAAAAACEJQwAAAAAgBCAPlCAHIAOMlJIgDpUiCCAIQwAAAABeIgIbIQggByAElSAQIAeUIAMgEZSSIA6VIAIbIQNBACECA0AgAkEBciIYQQJ0IhkgFEFAayIeaioCACADIBRBwAFqIh8gGGotAACzlCAIkiATIBlqKgIAkyIEIASUlCAeIAJBAnQiGGoqAgAgAyACIB9qLQAAs5QgCJIgEyAYaioCAJMiBCAElJQgCZKSIQkgAkECaiICQSBHDQALIAkgCl1FDQAgFiAU/QAEwAH9CwQAIBYgFP0ABNAB/QsEECADIQYgCSEKIAghBQsgFUEBaiECIBVBFEcNAAsLIBwgBYwiAzgCACAUIBtqIAY4AgAgBiALIAYgC14bIQsgAyAMIAMgDF4bIQwgF0EBaiIXQQhHDQALIBQqAiwhBCAUKgIoIQUgFCoCICEGIBQqAiQhCiAUKgIMIQkgFCoCCCEIIBQqAgQhByABIBpBkAFsaiIVQwAAfEIgC5VDAAAAACALQwAAAABeGyID/RMgFP0ABBD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP8AAAD/AAAA/wAAAP8AAAD9Tv0MPwAAAD8AAAA/AAAAPwAAAP23ASIg/RYAQQJ0QcABcUE/IAMgFCoCAJRDAABAS5K8Qf8BcSICIAJBP08bcjoABCAU/QAEMCEhIBUgIP0WBEECdEHAAXFBPyADIAeUQwAAQEuSvEH/AXEiAiACQT9PG3I6AAUgFSAg/RYIQQJ0QcABcUE/IAMgCJRDAABAS5K8Qf8BcSICIAJBP08bcjoABiAVICD9FgxBAnRBwAFxQT8gAyAJlEMAAEBLkrxB/wFxIgIgAkE/TxtyOgAHIBVBgPwBIAxDAAB8QpUiA4tDAACAd5RDAACACJRBgICAiAcgA7wiAkEBdCITQYCAgHhxIhcgF0GAgICIB00bQQF2QYCAgDxqvpK8IhdBDXZBgPgBcSAXQf8fcWogE0GAgIB4SxsgAkEQdkGAgAJxciICOwECIBVBgPwBIAtDAAB8QpUiA4tDAACAd5RDAACACJRBgICAiAcgA7wiE0EBdCIXQYCAgHhxIhYgFkGAgICIB00bQQF2QYCAgDxqvpK8IhZBDXZBgPgBcSAWQf8fcWogF0GAgIB4SxsgE0EQdkGAgAJxciITOwEAIBVBDGogIUMAAHxCIAyVQwAAAAAgDEMAAAAAXhsiA/0T/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz/AAAA/wAAAP8AAAD/AAAA/U79DD8AAAA/AAAAPwAAAD8AAAD9twEiIUEE/asBICD9DA8AAAAPAAAADwAAAA8AAAD9Tv1Q/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoAAAAgFSAh/RYEQQJ0QcABcUE/IAMgCpRDAABAS5K8Qf8BcSIXIBdBP08bcjoACSAVICH9FgBBAnRBwAFxQT8gAyAGlEMAAEBLkrxB/wFxIhcgF0E/TxtyOgAIIBUgIf0WCEECdEHAAXFBPyADIAWUQwAAQEuSvEH/AXEiFyAXQT9PG3I6AAogFSAh/RYMQQJ0QcABcUE/IAMgBJRDAABAS5K8Qf8BcSIXIBdBP08bcjoACyACQQJ0QZDWBGohGyATQQJ0QZDWBGoqAgAhAyAVQQRqIRdBACECA0ACQCACQQNNBEAgAiAXaiIWLQAAQT9xIRMgFi0ABEE/cSEWDAELIAIgF2oiEy0AAEECdkEwcSATLQAEIhxBBHZyIRYgE0EEay0AAEECdkEwcSAcQQ9xciETCyADIBOzlCIEQwAAAABcBEAgAkEFdCITIBRB4AFqIhhqIBsqAgAgFrOU/RMiICAAIAJBB3Rq/QACAP3kASAE/RMiIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQQRyIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQQhyIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQQxyIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQRByIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQRRyIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQRhyIhZqICAgACAWQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAAgGCATQRxyIhNqICAgACATQQJ0av0AAgD95AEgIf3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwPAEAADwBAAA8AQAAPAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/QwAAEBLAABASwAAQEsAAEBL/Q0ABAgMAAAAAAAAAAAAAAAA/VoCAAALIAJBAWoiAkEIRw0ACyAVIBT9AASAAkEE/WsgFP0ABOAB/VD9CwAQIBUgFP0ABJACQQT9ayAU/QAE8AH9UP0LACAgFSAU/QAEwAJBBP1rIBT9AASgAv1Q/QsAMCAVQUBrIBT9AATQAkEE/WsgFP0ABLAC/VD9CwAAIBUgFP0ABIADQQT9ayAU/QAE4AL9UP0LAFAgFSAU/QAEkANBBP1rIBT9AATwAv1Q/QsAYCAVIBT9AATAA0EE/WsgFP0ABKAD/VD9CwBwIBUgFP0ABNADQQT9ayAU/QAEsAP9UP0LAIABIABBgAhqIQAgGkEBaiIaIB1HDQALCyAUQeADaiQAC/8EAwF/B3wCfiAAvSIJQjCIpyEBIAlCgICAgPCVqfc/fUL/////n5WEAVgEQCAJQoCAgICAgID4P1EEQEQAAAAAAAAAAA8LQYjsASsDACIDIABEAAAAAAAA8L+gIgC9QoCAgIBwg78iBKIiBSAAIACiIgIgAEHQ7AErAwCiQcjsASsDAKCiIgagIgcgAiACoiIIIAggAiAAQZDtASsDAKJBiO0BKwMAoKIgAEGA7QErAwCiQfjsASsDAKCgoiACIABB8OwBKwMAokHo7AErAwCgoiAAQeDsASsDAKJB2OwBKwMAoKCgoiAAIAShIAOiIABBkOwBKwMAoqAgBiAFIAehoKCgoA8LAkAgAUHw/wFrQZ+Afk0EQCAJQv///////////wCDUARAIwBBEGsiAUQAAAAAAADwvzkDCCABKwMIRAAAAAAAAAAAow8LIAlCgICAgICAgPj/AFENASABQfD/AXFB8P8BRyABQf//AU1xRQRAIAAgAKEiACAAow8LIABEAAAAAAAAMEOivUKAgICAgICAoAN9IQkLIAlCgICAgICAgPM/fSIKQi6Ip0E/cUEEdCIBQaDtAWorAwAgCkI0h6e3oCIDQYjsASsDACIEIAFBmO0BaisDACAJIApCgICAgICAgHiDfb8gAUGY9QFqKwMAoSABQaD1AWorAwChoiIAvUKAgICAcIO/IgWiIgagIgcgACAAoiICIAIgAqIgAEHA7AErAwCiQbjsASsDAKCiIAIgAEGw7AErAwCiQajsASsDAKCiIABBoOwBKwMAokGY7AErAwCgoKCiIAAgBaEgBKJBkOwBKwMAIACioCAGIAMgB6GgoKCgIQALIAALcAICfwF+IAAoAighAkEBIQECQCAAQgAgAC0AAEGAAXEEf0EBQQIgACgCFCAAKAIcRhsFQQELIAIREwAiA0IAUw0AIAMgACgCCCIBBH8gAEEEagUgACgCHCIBRQ0BIABBFGoLKAIAIAFrrHwhAwsgAwucAQEBfwJAIAJBA08EQCMDQRxqQRw2AgAMAQsCQCACQQFHDQAgACgCCCIDRQ0AIAEgAyAAKAIEa6x9IQELIAAoAhQgACgCHEcEQCAAQQBBACAAKAIkEQMAGiAAKAIURQ0BCyAAQQA2AhwgAEIANwMQIAAgASACIAAoAigREwBCAFMNACAAQgA3AgQgACAAKAIAQW9xNgIAQQAPC0F/C8ABAQN/IAIoAkxBAE4EQCACEIIBIQULIAIgAigCSCIDQQFrIANyNgJIIAIoAgQiAyACKAIIIgRGBH8gAQUgACADIAQgA2siAyABIAEgA0sbIgMQexogAiACKAIEIANqNgIEIAAgA2ohACABIANrCyIDBEADQAJAIAIQ0AJFBEAgAiAAIAMgAigCIBEDACIEDQELIAUEQCACEIEBCyABIANrDwsgACAEaiEAIAMgBGsiAw0ACwsgBQRAIAIQgQELIAELFwBBf0EAIAAgABBoIgAgARDUASAARxsLlgEBBn8gACgCTEEASAR/QQAFIAAQggELRSEGIAAQMCEEIAAgACgCDBEAACEFIAZFBEAgABCBAQsgAC0AAEEBcUUEQBD8ASEBIAAoAjQiAgRAIAIgACgCODYCOAsgACgCOCIDBEAgAyACNgI0CyAAIAEoAgBGBEAgASADNgIAC0HU8jQQ0wEgACgCYBAvIAAQLwsgBCAFcgsLACAAQQAgARChAQv/GQMifRV/AXsjAEHAAmsiKiQAIAJBgAJOBEAgAkGAAm0hLgNAQwAAAAAhCkMAAAAAIQtBACErA0AgKkFAayArQQR0aiEpICogK0ECdGoCfSAAICtBBnRqIgIqAjwiA4siBiACKgI4IgSLIgcgAioCNCIIiyIJIAIqAjAiGIsiDCACKgIsIhmLIg0gAioCKCIaiyIOIAIqAiQiG4siDyACKgIgIhyLIhAgAioCHCIdiyIRIAIqAhgiHosiEiACKgIUIh+LIhMgAioCECIgiyIUIAIqAgwiIYsiFSACKgIIIiKLIhYgAioCBCIjiyIXIAIqAgAiBYtDAAAAACAFQwAAAABdIAVDAAAAAF5yGyIkIBcgJF4iJRsiFyAWIBdeIiYbIhYgFSAWXiInGyIVIBQgFV4iKBsiFCATIBReIiwbIhMgEiATXiIvGyISIBEgEl4iMBsiESAQIBFeIjEbIhAgDyAQXiIyGyIPIA4gD14iMxsiDiANIA5eIjQbIg0gDCANXiI1GyIMIAkgDF4iNhsiCSAHIAleIjcbIgcgBiAHXiI4G0MAAAAAWwRAICn9DAAAAAAAAAAAAAAAAAAAAAD9CwQAQwAAAAAMAQtDAAAAACEGQwAAgEAgAyAEIAggGCAZIBogGyAcIB0gHiAfICAgISAiICMgBUMAAAAAIAVDAAAAAF0gBUMAAAAAXnIbICUbICYbICcbICgbICwbIC8bIDAbIDEbIDIbIDMbIDQbIDUbIDYbIDcbIDgblSEHQQAhJUEAISZDAAAAACEFA0AgJiApakH8//8BQYOAgAJDAABASyAHIAIgJkECdGoqAgAiA5STvEH///8DcSInICdBg4CAAk8bIicgJ0H8//8BTBsiJzoAACADIAOUIgggJ0GAgIACa7IiBJQgBJQgBpIhBiADIAiUIASUIAWSIQUgJkEBaiImQRBHDQALQQAhJgNAAkAgBSACICVBAnRqKgIAIgMgAyADlCIElCIJICUgKWoiJywAACIosiIHlJMiCEMAAAAAXkUNACAoQfz//wFBg4CAAiADIASMIAeUIAeUIAaSIgOUIAiVQwAAQEuSvEH///8DcSIoIChBg4CAAk8bIiggKEH8//8BTBsiKEGAgIACayIsRg0AIAQgLLIiBJQgBJQgA5IiA0MAAAAAXkUNACAGIAkgBJQgCJIiBCAElJQgBSAFlCADlF5FDQAgJyAoOgAAICZBAWohJiAEIQUgAyEGCyAlQQFqIiVBEEcNAAtBACElQQAhJwJAICZFDQADQAJAIAUgAiAlQQJ0aioCACIDIAMgA5QiBJQiCSAlIClqIiYsAAAiKLIiB5STIghDAAAAAF5FDQAgKEH8//8BQYOAgAIgAyAEjCAHlCAHlCAGkiIDlCAIlUMAAEBLkrxB////A3EiKCAoQYOAgAJPGyIoIChB/P//AUwbIihBgICAAmsiLEYNACAEICyyIgSUIASUIAOSIgNDAAAAAF5FDQAgBiAJIASUIAiSIgQgBJSUIAUgBZQgA5ReRQ0AICYgKDoAACAnQQFqIScgBCEFIAMhBgsgJUEBaiIlQRBHDQALQQAhJUEAISYgJ0UNAANAAkAgBSACICVBAnRqKgIAIgMgAyADlCIElCIJICUgKWoiJywAACIosiIHlJMiCEMAAAAAXkUNACAoQfz//wFBg4CAAiADIASMIAeUIAeUIAaSIgOUIAiVQwAAQEuSvEH///8DcSIoIChBg4CAAk8bIiggKEH8//8BTBsiKEGAgIACayIsRg0AIAQgLLIiBJQgBJQgA5IiA0MAAAAAXkUNACAGIAkgBJQgCJIiBCAElJQgBSAFlCADlF5FDQAgJyAoOgAAICZBAWohJiAEIQUgAyEGCyAlQQFqIiVBEEcNAAtBACElQQAhJyAmRQ0AA0ACQCAFIAIgJUECdGoqAgAiAyADIAOUIgSUIgkgJSApaiImLAAAIiiyIgeUkyIIQwAAAABeRQ0AIChB/P//AUGDgIACIAMgBIwgB5QgB5QgBpIiA5QgCJVDAABAS5K8Qf///wNxIiggKEGDgIACTxsiKCAoQfz//wFMGyIoQYCAgAJrIixGDQAgBCAssiIElCAElCADkiIDQwAAAABeRQ0AIAYgCSAElCAIkiIEIASUlCAFIAWUIAOUXkUNACAmICg6AAAgJ0EBaiEnIAQhBSADIQYLICVBAWoiJUEQRw0AC0EAISUgJ0UNAANAAkAgBSACICVBAnRqKgIAIgMgAyADlCIElCIJICUgKWoiJiwAACInsiIHlJMiCEMAAAAAXkUNACAnQfz//wFBg4CAAiADIASMIAeUIAeUIAaSIgOUIAiVQwAAQEuSvEH///8DcSInICdBg4CAAk8bIicgJ0H8//8BTBsiJ0GAgIACayIoRg0AIAQgKLIiBJQgBJQgA5IiA0MAAAAAXkUNACAGIAkgBJQgCJIiBCAElJQgBSAFlCADlF5FDQAgJiAnOgAAIAQhBSADIQYLICVBAWoiJUEQRw0ACwsgKSAp/QAEAP0MBAQEBAQEBAQEBAQEBAQEBP1u/QsEACAFIAaVCyIFOAIAIAUgCiAFiyIFIAteIgIbIQogBSALIAIbIQsgK0EBaiIrQRBHDQALIAEgLUHuAGxqIilCADcBYEEAIQIgKUEANgFoAn8gCkMAAAAAWwRAQQAhJUEADAELQwAAAMIgCpUhBUEAISUDQEFgQR8gBSAqICVBAnRqKgIAlEMAAEBLkrzAIiYgJkEfThsiJiAmQWBMGyImQSBqIScCQCAlQQdNBEAgJSApaiAmQQ9xOgBgDAELICUgKWoiK0HYAGogKy0AWCAmQQR0cjoAAAsgKSAlQQNxaiImQegAaiAmLQBoICdBBHYgJUEBdkH+////A3F0cjoAACAlQQFqIiVBEEcNAAtBgPwBQwAAgD8gBZUiBYtDAACAd5RDAACACJRBgICAiAcgBbwiJUEBdCImQYCAgHhxIicgJ0GAgICIB00bQQF2QYCAgDxqvpK8IidBDXZBgPgBcSAnQf8fcWogJkGAgIB4SxsgJUEQdkGAgAJxciIlCyE5ICkgJTsBbCA5QQJ0QZDWBGoqAgAhBQNAIAUCfyACQQdNBEAgAiApai0AYEEPcQwBCyACIClqLQBYQQR2CyApIAJBA3FqLQBoIAJBAXZB/v///wNxdkEEdEEwcXJBIGuylCIGQwAAAABcBEAgKkFAayACQQR0aiAAIAJBBnRqIiX9AAIAIAb9EyI6/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DPz/PwD8/z8A/P8/APz/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9TiAl/QACECA6/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DPz/PwD8/z8A/P8/APz/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GASAl/QACICA6/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DPz/PwD8/z8A/P8/APz/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9TiAl/QACMCA6/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DPz/PwD8/z8A/P8/APz/PwD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GAf1m/QwEBAQEBAQEBAQEBAQEBAQE/W79CwQACyACQQFqIgJBEEcNAAsgKf0MAAAAAAAAAAAAAAAAAAAAAP0LAQAgKf0MAAAAAAAAAAAAAAAAAAAAAP0LARBBASElQQAhAkEAISYDQCAqQUBrIAJqIicsAAAiK0EETgRAICYgKWoiKCAoLQAAICVyOgAAICcgK0EEazoAAAtBACAmQQFqIiYgJkEgRiInGyEmICUgJ3QhJSACQQFqIgJBgAJHDQALICkgKv0ABGBBAv1rICr9AARA/VAgKv0ABIABQQT9a/1QICr9AASgAUEG/Wv9UP0LACAgKSAq/QAEcEEC/WsgKv0ABFD9UCAq/QAEkAFBBP1r/VAgKv0ABLABQQb9a/1Q/QsAMCApQUBrICr9AATgAUEC/WsgKv0ABMAB/VAgKv0ABIACQQT9a/1QICr9AASgAkEG/Wv9UP0LAAAgKSAq/QAE8AFBAv1rICr9AATQAf1QICr9AASQAkEE/Wv9UCAq/QAEsAJBBv1r/VD9CwBQIABBgAhqIQAgLUEBaiItIC5HDQALCyAqQcACaiQAC4AQAhh/A3wjAEEQayILJAACQCAAvCIRQf////8HcSIDQdqfpO4ETQRAIAEgALsiGyAbRIPIyW0wX+Q/okQAAAAAAAA4Q6BEAAAAAAAAOMOgIhpEAAAAUPsh+b+ioCAaRGNiGmG0EFG+oqAiHDkDACAcRAAAAGD7Iem/YyEWAn8gGplEAAAAAAAA4EFjBEAgGqoMAQtBgICAgHgLIQMgFgRAIAEgGyAaRAAAAAAAAPC/oCIaRAAAAFD7Ifm/oqAgGkRjYhphtBBRvqKgOQMAIANBAWshAwwCCyAcRAAAAGD7Iek/ZEUNASABIBsgGkQAAAAAAADwP6AiGkQAAABQ+yH5v6KgIBpEY2IaYbQQUb6ioDkDACADQQFqIQMMAQsgA0GAgID8B08EQCABIAAgAJO7OQMAQQAhAwwBCyALIAMgA0EXdkGWAWsiA0EXdGu+uzkDCCALQQhqIQ4jAEGwBGsiBSQAIAMgA0EDa0EYbSICQQAgAkEAShsiDUFobGohBkGwwgEoAgAiB0EATgRAIAdBAWohAyANIQIDQCAFQcACaiAEQQN0aiACQQBIBHxEAAAAAAAAAAAFIAJBAnRBwMIBaigCALcLOQMAIAJBAWohAiAEQQFqIgQgA0cNAAsLIAZBGGshCEEAIQMgB0EAIAdBAEobIQQDQEEAIQJEAAAAAAAAAAAhGgNAIA4gAkEDdGorAwAgBUHAAmogAyACa0EDdGorAwCiIBqgIRogAkEBaiICQQFHDQALIAUgA0EDdGogGjkDACADIARGIRcgA0EBaiEDIBdFDQALQS8gBmshEkEwIAZrIQ8gBkEZayETIAchAwJAA0AgBSADQQN0aisDACEaQQAhAiADIQQgA0EATCIJRQRAA0AgBUHgA2ogAkECdGoCfwJ/IBpEAAAAAAAAcD6iIhuZRAAAAAAAAOBBYwRAIBuqDAELQYCAgIB4C7ciG0QAAAAAAABwwaIgGqAiGplEAAAAAAAA4EFjBEAgGqoMAQtBgICAgHgLNgIAIAUgBEEBayIEQQN0aisDACAboCEaIAJBAWoiAiADRw0ACwsCfyAaIAgQsQEiGiAaRAAAAAAAAMA/opxEAAAAAAAAIMCioCIamUQAAAAAAADgQWMEQCAaqgwBC0GAgICAeAshCiAaIAq3oSEaAkACQAJAAn8gCEEATCIURQRAIANBAnQgBWoiAiACKALcAyICIAIgD3UiAiAPdGsiBDYC3AMgAiAKaiEKIAQgEnUMAQsgCA0BIANBAnQgBWooAtwDQRd1CyIMQQBMDQIMAQtBAiEMIBpEAAAAAAAA4D9mDQBBACEMDAELQQAhAkEAIQQgCUUEQANAIAVB4ANqIAJBAnRqIhUoAgAhCUH///8HIRACfwJAIAQNAEGAgIAIIRAgCQ0AQQAMAQsgFSAQIAlrNgIAQQELIQQgAkEBaiICIANHDQALCwJAIBQNAEH///8DIQICQAJAIBMOAgEAAgtB////ASECCyADQQJ0IAVqIgkgCSgC3AMgAnE2AtwDCyAKQQFqIQogDEECRw0ARAAAAAAAAPA/IBqhIRpBAiEMIARFDQAgGkQAAAAAAADwPyAIELEBoSEaCyAaRAAAAAAAAAAAYQRAQQAhBAJAIAcgAyICTg0AA0AgBUHgA2ogAkEBayICQQJ0aigCACAEciEEIAIgB0oNAAsgBEUNACAIIQYDQCAGQRhrIQYgBUHgA2ogA0EBayIDQQJ0aigCAEUNAAsMAwtBASECA0AgAiIEQQFqIQIgBUHgA2ogByAEa0ECdGooAgBFDQALIAMgBGohBANAIAVBwAJqIANBAWoiA0EDdGogAyANakECdEHAwgFqKAIAtzkDAEEAIQJEAAAAAAAAAAAhGgNAIA4gAkEDdGorAwAgBUHAAmogAyACa0EDdGorAwCiIBqgIRogAkEBaiICQQFHDQALIAUgA0EDdGogGjkDACADIARIDQALIAQhAwwBCwsCQCAaQRggBmsQsQEiGkQAAAAAAABwQWYEQCAFQeADaiADQQJ0agJ/An8gGkQAAAAAAABwPqIiG5lEAAAAAAAA4EFjBEAgG6oMAQtBgICAgHgLIgK3RAAAAAAAAHDBoiAaoCIamUQAAAAAAADgQWMEQCAaqgwBC0GAgICAeAs2AgAgA0EBaiEDDAELAn8gGplEAAAAAAAA4EFjBEAgGqoMAQtBgICAgHgLIQIgCCEGCyAFQeADaiADQQJ0aiACNgIAC0QAAAAAAADwPyAGELEBIRoCQCADQQBIDQAgAyECA0AgBSACIgRBA3RqIBogBUHgA2ogAkECdGooAgC3ojkDACACQQFrIQIgGkQAAAAAAABwPqIhGiAEDQALIANBAEgNACADIQQDQEQAAAAAAAAAACEaQQAhAiAHIAMgBGsiBiAGIAdKGyIIQQBOBEADQCACQQN0QZDYAWorAwAgBSACIARqQQN0aisDAKIgGqAhGiACIAhHIRggAkEBaiECIBgNAAsLIAVBoAFqIAZBA3RqIBo5AwAgBEEASiEZIARBAWshBCAZDQALC0QAAAAAAAAAACEaIANBAE4EQANAIAMiAkEBayEDIBogBUGgAWogAkEDdGorAwCgIRogAg0ACwsgCyAamiAaIAwbOQMAIAVBsARqJAAgCkEHcSEDIAsrAwAhGiARQQBIBEAgASAamjkDAEEAIANrIQMMAQsgASAaOQMACyALQRBqJAAgAwuLBABBqLIDQZMkECtBtLIDQYgaQQFBABAqQcCyA0GLFUEBQYB/Qf8AEAVB2LIDQYQVQQFBgH9B/wAQBUHMsgNBghVBAUEAQf8BEAVB5LIDQf4KQQJBgIB+Qf//ARAFQfCyA0H1CkECQQBB//8DEAVB/LIDQbQLQQRBgICAgHhB/////wcQBUGIswNBqwtBBEEAQX8QBUGUswNBox1BBEGAgICAeEH/////BxAFQaCzA0GaHUEEQQBBfxAFQayzA0HrD0KAgICAgICAgIB/Qv///////////wAQ+QJBuLMDQeoPQgBCfxD5AkHEswNB0g9BBBARQdCzA0GVI0EIEBFBrPoAQc4dEBBBtLwBQeU3EBBB/LwBQQRBtB0QCUHIvQFBAkHaHRAJQZS+AUEEQekdEAlB5PoAQbYaEClBvL4BQQBBoDcQAUHkvgFBAEGGOBABQYy/AUEBQb43EAFBtL8BQQJB7TMQAUHcvwFBA0GMNBABQYTAAUEEQbQ0EAFBrMABQQVB0TQQAUHUwAFBBEGrOBABQfzAAUEFQck4EAFB5L4BQQBBtzUQAUGMvwFBAUGWNRABQbS/AUECQfk1EAFB3L8BQQNB1zUQAUGEwAFBBEH/NhABQazAAUEFQd02EAFBpMEBQQhBvDYQAUHMwQFBCUGaNhABQfTBAUEGQfc0EAFBnMIBQQdB8DgQAQuOCQMRfwJ8AX4jAEEwayIGJABBASEDAkACQAJAAkACQAJAIAEgAGtB6ABtDgYFBQABAgMECyABQcgAaysDACAAKwMgZEUNBCAAIAFB6ABrED4MBAsgAEHoAGohAiABQegAayIBKwMgIRMgACsDiAEiFCAAKwMgZEUEQCATIBRkRQ0EIAIgARA+IAArA4gBIAArAyBkRQ0EIAAgAhA+DAQLIBMgFGQEQCAAIAEQPgwECyAAIAIQPiABKwMgIAArA4gBZEUNAyACIAEQPgwDCyAAIABB6ABqIABB0AFqIAFB6ABrENMCGgwCCyAAIABB6ABqIABB0AFqIABBuAJqIAFB6ABrENQCGgwBCyAAQegAaiECIABB0AFqIQUgACsD8AEhEwJAIAArA4gBIhQgACsDIGRFBEAgEyAUZEUNASACIAUQPiAAKwOIASAAKwMgZEUNASAAIAIQPgwBCyATIBRkBEAgACAFED4MAQsgACACED4gACsD8AEgACsDiAFkRQ0AIAIgBRA+CyAAQbgCaiIEIAFGDQADQAJAIAQrAyAiEyAFKwMgZEUNACAGIAQtAAg6ACggBiAEKQMANwMgIAQoAhQhCCAEKAIQIQkgBEIANwMQIAQoAhghCiAEQQA2AhggBCgCHCELIAYgBP0AAjj9CwMQIAYgBP0AAij9CwMAIAQoAlwhDCAEKAJYIQ0gBEIANwNYIAQoAkwhDiAEKAJQIQ8gBCgCVCEQIAQoAkghESAE/QwAAAAAAAAAAAAAAAAAAAAA/QsDSCAEKQNgIRUgBCECA0AgAiAFIgMpAwA3AwAgAiADLQAIOgAIIAIoAhAiBQRAIAIgBTYCFCAFEC8gAkEANgIYCyACIAMoAhA2AhAgAiADKAIUNgIUIAIgAygCGDYCGCADQQA2AhggA0IANwMQIAIgAygCRDYCRCACIAMpAjw3AjwgAiAD/QACLP0LAiwgAiADIgf9AAIc/QsCHCACQcgAaiADQcgAahCHAgJAIAAgA0YEQCAAIQMMAQsgByECIBMgA0HoAGsiBSsDIGQNAQsLIAMgBikDIDcDACADIAYtACg6AAggAygCECICBEAgAyACNgIUIAIQLwsgAyAKNgIYIAMgCDYCFCADIAk2AhAgByALNgIcIAMgEzkCICADIAb9AAMQ/QsCOCADIAb9AAMA/QsCKCAHKAJIIggEQCADKAJMIgUgCCICRwRAA0AgBUEMayICKAIAIgkEQCAFQQhrIAk2AgAgCRAvCyACIgUgCEcNAAsgBygCSCECCyADIAg2AkwgAhAvCyAHIBE2AkggAyAPNgJQIAMgDjYCTCADKAJUIgcEQCADKAJYIgUgByICRwRAA0AgBUEMayICKAIAIggEQCAFQQhrIAg2AgAgCBAvCyACIgUgB0cNAAsgAygCVCECCyADIAc2AlggAhAvCyADIBA2AlQgAyAVNwJgIAMgDDYCXCADIA02AlggEkEBaiISQQhHDQAgBEHoAGogAUYhAwwCCyAEIgVB6ABqIgIhBCABIAJHDQALQQEhAwsgBkEwaiQAIAMLqQoCB38DfCMAQRBrIgIkAEEBIQYCQAJAAkACQAJAAkAgASAAa0EEdQ4GBQUAAQIDBAsgAUEQayIBKwMAIAArAwBkRQ0EIAIgAP0AAwD9CwMAIAAgASgCCDYCCCAAIAEpAwA3AwAgASACKAIINgIIIAEgAikDADcDAAwECyAAQRBqIQMgAUEQayIBKwMAIQkgACsDECIKIAArAwBkRQRAIAkgCmRFDQQgAiAD/QADAP0LAwAgAyABKAIINgIIIAMgASkDADcDACABIAIoAgg2AgggASACKQMANwMAIAArAxAgACsDAGRFDQQgAiAA/QADAP0LAwAgACADKAIINgIIIAAgAykDADcDACADIAIoAgg2AgggAyACKQMANwMADAQLIAkgCmQEQCACIAD9AAMA/QsDACAAIAEoAgg2AgggACABKQMANwMAIAEgAigCCDYCCCABIAIpAwA3AwAMBAsgAiAA/QADAP0LAwAgACADKAIINgIIIAAgAykDADcDACADIAIoAgg2AgggAyACKQMANwMAIAErAwAgACsDEGRFDQMgAiAD/QADAP0LAwAgAyABKAIINgIIIAMgASkDADcDACABIAIoAgg2AgggASACKQMANwMADAMLIAAgAEEQaiAAQSBqIAFBEGsQ2AEaDAILIAAgAEEQaiIEIABBIGoiAyAAQTBqIgUQ2AEaIAFBEGsiASsDACAAKwMwZEUNASACIAX9AAMA/QsDACAFIAEoAgg2AgggBSABKQMANwMAIAEgAigCCDYCCCABIAIpAwA3AwAgBSsDACADKwMAZEUNASACIAP9AAMA/QsDACADIAUoAgg2AgggAyAFKQMANwMAIAUgAigCCDYCCCAFIAIpAwA3AwAgAysDACAEKwMAZEUNASACIAT9AAMA/QsDACAEIAMoAgg2AgggBCADKQMANwMAIAMgAigCCDYCCCADIAIpAwA3AwAgACsDECAAKwMAZEUNASACIAD9AAMA/QsDACAAIAQoAgg2AgggACAEKQMANwMAIAQgAigCCDYCCCAEIAIpAwA3AwAMAQsgAEEQaiEEIABBIGohAyAAKwMgIQkCQCAAKwMQIgogACsDACILZEUEQCAJIApkRQ0BIAIgBP0AAwD9CwMAIAQgA0EIaigCADYCCCAEIAMpAwA3AwAgAyACKAIINgIIIAMgAikDADcDACAEKwMAIAtkRQ0BIAIgAP0AAwD9CwMAIAAgBCgCCDYCCCAAIAQpAwA3AwAgBCACKAIINgIIIAQgAikDADcDAAwBCyAJIApkBEAgAiAA/QADAP0LAwAgACADQQhqKAIANgIIIAAgAykDADcDACADIAIoAgg2AgggAyACKQMANwMADAELIAIgAP0AAwD9CwMAIAAgBCgCCDYCCCAAIAQpAwA3AwAgBCACKAIINgIIIAQgAikDADcDACAJIAArAxBkRQ0AIAIgBP0AAwD9CwMAIAQgA0EIaigCADYCCCAEIAMpAwA3AwAgAyACKAIINgIIIAMgAikDADcDAAsgAEEwaiIEIAFGDQADQAJAIAQrAwAiCSADKwMAZEUNACAEKAIIIQggBCEFA0ACQCAFIAMiBikDADcDACAFIAMoAgg2AgggACADRgRAIAAhBgwBCyAJIAYiBUEQayIDKwMAZA0BCwsgBiAINgIIIAYgCTkDACAHQQFqIgdBCEcNACAEQRBqIAFGIQYMAgsgBCIDQRBqIgYhBCABIAZHDQALQQEhBgsgAkEQaiQAIAYLzRoDJH0NfwV7IwBB0ANrIickACACQYACTgRAIAJBgAJtIS8DQCABIC1B1ABsaiEqQwAAAAAhEEMAAAAAIRFBACEsA0AgJyAAICxBBnRqIij9AAIAIjT94AEiNv0LBIABICcgKP0AAhD94AEiN/0LBJABICcgKP0AAiD94AEiOP0LBKABICcgKP0AAjD94AEiNf0LBLABICxBAnQiMCAnQUBraiEzICdB0AFqICxBBHRqIS4CQCAoKgI8IgggKCoCOCIJICgqAjQiCiAoKgIwIgsgKCoCLCIMICgqAigiFCAoKgIkIhUgKCoCICIWICgqAhwiFyAoKgIYIg0gKCoCFCIOICgqAhAiEiAoKgIMIgUgKCoCCCIEICgqAgQiBiA0/R8AIgcgBiAHXhsiAyADIARdGyIDIAMgBV0bIgMgAyASXRsiAyADIA5dGyIDIAMgDV0bIgMgAyAXXRsiAyADIBZdGyIDIAMgFV0bIgMgAyAUXRsiAyADIAxdGyIDIAMgC10bIgMgAyAKXRsiAyADIAldGyIDIAMgCF0bIhpDAAAAACAIIAkgCiALIAwgFCAVIBYgFyANIA4gEiAFIAQgBiAHIAYgB10bIgMgAyAEXhsiAyADIAVeGyIDIAMgEl4bIgMgAyAOXhsiAyADIA1eGyIDIAMgF14bIgMgAyAWXhsiAyADIBVeGyIDIAMgFF4bIgMgAyAMXhsiAyADIAteGyIDIAMgCl4bIgMgAyAJXhsiAyADIAheGyIDIANDAAAAAF4bIg9bBEAgLv0MAAAAAAAAAAAAAAAAAAAAAP0LBABDAAAAACEGDAELIDX9HwMiGyAIlCA1/R8CIhwgCZQgNf0fASIdIAqUIDX9HwAiHiALlCA4/R8DIh8gDJQgOP0fAiIgIBSUIDj9HwEiISAVlCA4/R8AIiIgFpQgN/0fAyIjIBeUIDf9HwIiJCANlCA3/R8BIiUgDpQgN/0fACImIBKUIDb9HwMiCCAFlCA2/R8CIgkgBJQgNv0fASIKIAaUIAcgNv0fACILlJKSkpKSkpKSkpKSkpKSkiEMIAsgCpIgCZIgCJIgJpIgJZIgJJIgI5IgIpIgIZIgIJIgH5IgHpIgHZIgHJIgG5IhDUMAAIA/QwAAQEAgGiAPk5UiBJUhBkMAAAAAIQdBACECA0AgAiAuakGAgIACQYOAgAIgBCAoIAJBAnQiK2oqAgAiAyAPk5RDAABAS5K8Qf///wNxIikgKUGDgIACTxsiKSApQYCAgAJMGyIpOgAAICdBgAFqICtqKgIAIAYgKUGAgIACa7KUIA+SIAOTi5QgB5IhByACQQFqIgJBEEcNAAsgDIwhEkEAIQIDQCACIimyQ83MzD2UQwAAAL+SQwAAQECSIBogD5OVIQVDAAAAACEYQQAhAkMAAAAAIRlDAAAAACETA0AgJ0HAAWogAmpBgICAAkGDgIACIAUgKCACQQJ0IjJqKgIAIgQgD5OUQwAAQEuSvEH///8DcSIrICtBg4CAAk8bIisgK0GAgIACTBsiKzoAACAnQYABaiAyaioCACArQYCAgAJrsiIDlCIOIAOUIBmSIRkgDiATkiETIA4gBJQgGJIhGCACQQFqIgJBEEcNAAsCQCANIBmUIBMgE5STIgNDAAAAAF5FDQAgGyAYIBmVIA0gGJQgEyASlJIgA5UgGSAMlCAYIBOMlJIgA5UiA0MAAAAAXiICGyIFICctAM8Bs5RDAAAAACADIAIbIgSSICgqAjyTi5QgHCAFICctAM4Bs5QgBJIgKCoCOJOLlCAdIAUgJy0AzQGzlCAEkiAoKgI0k4uUIB4gBSAnLQDMAbOUIASSICgqAjCTi5QgHyAFICctAMsBs5QgBJIgKCoCLJOLlCAgIAUgJy0AygGzlCAEkiAUk4uUICEgBSAnLQDJAbOUIASSIBWTi5QgIiAFICctAMgBs5QgBJIgFpOLlCAjIAUgJy0AxwGzlCAEkiAXk4uUICQgBSAnLQDGAbOUIASSICgqAhiTi5QgJSAFICctAMUBs5QgBJIgKCoCFJOLlCAmIAUgJy0AxAGzlCAEkiAoKgIQk4uUIAggBSAnLQDDAbOUIASSICgqAgyTi5QgCSAFICctAMIBs5QgBJIgKCoCCJOLlCAKIAUgJy0AwQGzlCAEkiAoKgIEk4uUIAsgBSAnLQDAAbOUIASSICgqAgCTi5RDAAAAAJKSkpKSkpKSkpKSkpKSkpIiAyAHXUUNACAuICf9AATAAf0LBAAgBSEGIAMhByAEIQ8LIClBAWohAiApQQ9HDQALCyAzIA+MIgM4AgAgJyAwaiAGOAIAIAYgECAGIBBeGyEQIAMgESADIBFeGyERICxBAWoiLEEQRw0ACyAqAn8gEEMAAAAAXkUEQCAq/QwAAAAAAAAAAAAAAAAAAAAA/QsAAEEADAELICpDAABwQSAQlf0TIjQgJ/0ABAD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP8AAAD/AAAA/wAAAP8AAAD9TiA0ICf9AAQQ/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz/AAAA/wAAAP8AAAD/AAAA/U79hgEgNCAn/QAEIP3mAf0MAABASwAAQEsAAEBLAABAS/3kAf0M/wAAAP8AAAD/AAAA/wAAAP1OIDQgJ/0ABDD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GAf1m/QsAAEGA/AEgEEMAAHBBlSIDi0MAAIB3lEMAAIAIlEGAgICIByADvCIrQQF0IilBgICAeHEiAiACQYCAgIgHTRtBAXZBgICAPGq+krwiAkENdkGA+AFxIAJB/x9xaiApQYCAgHhLGyArQRB2QYCAAnFyCyICOwFQAkAgEUMAAAAAXgRAICpDAABwQSARlf0TIjQgJ/0ABED95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP8AAAD/AAAA/wAAAP8AAAD9TiA0ICf9AARQ/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz/AAAA/wAAAP8AAAD/AAAA/U79hgEgNCAn/QAEYP3mAf0MAABASwAAQEsAAEBLAABAS/3kAf0M/wAAAP8AAAD/AAAA/wAAAP1OIDQgJ/0ABHD95gH9DAAAQEsAAEBLAABASwAAQEv95AH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GAf1mQQT9ayAq/QAAAP1Q/QsAACAqQYD8ASARQwAAcEGVIgOLQwAAgHeUQwAAgAiUQYCAgIgHIAO8IitBAXQiKUGAgIB4cSICIAJBgICAiAdNG0EBdkGAgIA8ar6SvCICQQ12QYD4AXEgAkH/H3FqIClBgICAeEsbICtBEHZBgIACcXI7AVIgKi8BUCECDAELICpBADsBUgsgAkH//wNxQQJ0QZDWBGoqAgAhB0EAIQIDQCAHIAIgKmotAAAiKUEPcbOUIgNDAAAAAFwEQCAnQdABaiACQQR0aiAqLwFSQQJ0QZDWBGoqAgAgKUEEdrOU/RMiNSAAIAJBBnRqIin9AAIA/eQBIAP9EyI0/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DAAAQAAAAEAAAABAAAAAQAD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9TiA1ICn9AAIQ/eQBIDT95wH9DAAAQEsAAEBLAABASwAAQEv95AH9DP//fwD//38A//9/AP//fwD9Tv0MAwBAAAMAQAADAEAAAwBAAP22Af0MAABAAAAAQAAAAEAAAABAAP24Af0M/wAAAP8AAAD/AAAA/wAAAP1O/YYBIDUgKf0AAiD95AEgNP3nAf0MAABASwAAQEsAAEBLAABAS/3kAf0M//9/AP//fwD//38A//9/AP1O/QwDAEAAAwBAAAMAQAADAEAA/bYB/QwAAEAAAABAAAAAQAAAAEAA/bgB/Qz/AAAA/wAAAP8AAAD/AAAA/U4gNSAp/QACMP3kASA0/ecB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DAMAQAADAEAAAwBAAAMAQAD9tgH9DAAAQAAAAEAAAABAAAAAQAD9uAH9DP8AAAD/AAAA/wAAAP8AAAD9Tv2GAf1m/QsEAAsgAkEBaiICQRBHDQALICogJ/0ABPABQQL9ayAn/QAE0AH9UCAn/QAEkAJBBP1r/VAgJ/0ABLACQQb9a/1Q/QsAECAqICf9AASAAkEC/WsgJ/0ABOAB/VAgJ/0ABKACQQT9a/1QICf9AATAAkEG/Wv9UP0LACAgKiAn/QAE8AJBAv1rICf9AATQAv1QICf9AASQA0EE/Wv9UCAn/QAEsANBBv1r/VD9CwAwICpBQGsgJ/0ABIADQQL9ayAn/QAE4AL9UCAn/QAEoANBBP1r/VAgJ/0ABMADQQb9a/1Q/QsAACAAQYAIaiEAIC1BAWoiLSAvRw0ACwsgJ0HQA2okAAvOFgILfwF+IwBB4ABrIgYkACAAQQA2AgggAEIANwIAAkACQAJAAkACQAJAAkACQCACKAIEIgggAigCAEYEQCADKAIAIgUgAygCBCIJRg0IA0ACQCAFKAIMIgEgBSgCBCIIKAIAckUNACAFKAIIIQcgBSgCACENIAAoAgggBEcEQCAEIAE2AgwgBCAHNgIIIAQgCDYCBCAEIA02AgAgACAEQRBqIgQ2AgQMAQsgBCAAKAIAIgtrIgJBBHUiDEEBaiIDQYCAgIABTw0DQf////8AIAJBA3UiCiADIAMgCkkbIAJB8P///wdPGyICBH8gAkGAgICAAU8NBSACQQR0EDIFQQALIgogDEEEdGoiAyABNgIMIAMgBzYCCCADIAg2AgQgAyANNgIAIANBEGohASAEIAtHBEADQCADQRBrIgMgBEEQayIE/QACAP0LAgAgBCALRw0ACyAAKAIAIQQLIAAgCiACQQR0ajYCCCAAIAE2AgQgACADNgIAIAQEQCAEEC8LIAEhBAsgBUEQaiIFIAlHDQALDAgLIAhBBGsoAgAhCCAGQQA2AlwgBkIANwJUAkAgAygCACINIAMoAgQiDkYNAAJAAkADQAJAIA0pAgghDyANKAIAIQcCQCANKAIEIgsoAgAiCUUEQCAPQiCIpyIDRQ0BIAgoAgAiCUEDa0ECTw0CAkAgD0IAUw0AIANBAUYgD6ciBEECSXENACAEIANBBmwiBXQiBEGAECADQQNGQRB0IANBAkYbIAQbIQwgBEF/IAV0QX9zciEKIAghBAJAA0AgBCgCBCEFAkAgBCgCCCIDQQVGBEAgBSAKTQRAIAwgBCgCDE0NBAsgBCgCECEDIARBEGohBAwBCyAEQQhqIQQgBSAMSQ0AIAUgCk0NAgsgA0EGRg0ACyAJQQNGDQEMAwsgCUEDRg0CCyAAKAIEIgQgACgCCEcEQCAEIA83AgggBCALNgIEIAQgBzYCACAAIARBEGo2AgQMAgsgBCAAKAIAIglrIgNBBHUiDEEBaiIFQYCAgIABTw0HQf////8AIANBA3UiCiAFIAUgCkkbIANB8P///wdPGyIFBH8gBUGAgICAAU8NCSAFQQR0EDIFQQALIgogDEEEdGoiAyAPNwIIIAMgCzYCBCADIAc2AgAgA0EQaiEHIAQgCUcEQANAIANBEGsiAyAEQRBrIgT9AAIA/QsCACAEIAlHDQALIAAoAgAhBAsgACAKIAVBBHRqNgIIIAAgBzYCBCAAIAM2AgAgBEUNASAEEC8MAQtBACEFIAgiBCgCACIKQQNrQQJPDQMDQAJ/IAQoAggiA0EFRgRAAn9BASAFQQFxDQAaQQAgBCgCBCAJSw0AGiAEKAIMIAlPCyEFIAQoAhAhAyAEQRBqDAELIARBCGohDCAFQQFxBEBBASEFIAwMAQsgBCgCBCAJRiEFIAwLIQQgA0EGRg0ACyAKQQNGIAVzRQRAIAtBBGohCSAGKAJYIgQgBigCXCILSQRAIAQgDzcCCCAEIAk2AgQgBCAHNgIAIAYgBEEQajYCWAwCCyAEIAYoAlQiBWtBBHUiDEEBaiIDQYCAgIABTw0FQf////8AIAsgBWsiC0EDdSIKIAMgAyAKSRsgC0Hw////B08bIgsEfyALQYCAgIABTw0JIAtBBHQQMgVBAAsiCiAMQQR0aiIDIA83AgggAyAJNgIEIAMgBzYCACADQRBqIQcgBCAFRwRAA0AgA0EQayIDIARBEGsiBP0AAgD9CwIAIAQgBUcNAAsgBSEECyAGIAogC0EEdGo2AlwgBiAHNgJYIAYgAzYCVCAERQ0BIAQQLwwBCyAAKAIEIgQgACgCCEcEQCAEIA83AgggBCALNgIEIAQgBzYCACAAIARBEGo2AgQMAQsgBCAAKAIAIglrIgNBBHUiDEEBaiIFQYCAgIABTw0GQf////8AIANBA3UiCiAFIAUgCkkbIANB8P///wdPGyIFBH8gBUGAgICAAU8NCCAFQQR0EDIFQQALIgogDEEEdGoiAyAPNwIIIAMgCzYCBCADIAc2AgAgA0EQaiEHIAQgCUcEQANAIANBEGsiAyAEQRBrIgT9AAIA/QsCACAEIAlHDQALIAAoAgAhBAsgACAKIAVBBHRqNgIIIAAgBzYCBCAAIAM2AgAgBEUNACAEEC8LIA1BEGoiDSAORw0BDAQLCyAGQdgrNgIYIAZBmB82AhQgBkGmFTYCEEECQbDkACAGQRBqEDQQAAALIAZB2Cs2AiggBkH/HjYCJCAGQaYVNgIgQQJBsOQAIAZBIGoQNBAAAAsQNgALIAgoAgBBA2tBAk8NAkEAIQMDQAJ/IAgoAggiBEEFRgRAIAgoAhAhBCADQQFxBH9BAAUgCCgCBAtFIQMgCEEQagwBCyAIQQhqIQUgA0EBcQRAQQEhAyAFDAELIAgoAgRFIQMgBQshCCAEQQZGDQALIAIoAgAhByACKAIEIQJBACEFIAZBADYCUCAGQgA3AkgCQAJAIAJBBGsiAiAHRwRAIAIgB2siAkEASA0GIAYgAhAyIgU2AkggBiAFIAJBfHEiAmoiAzYCUCAFIAcgAvwKAAAgBiACIAVqIgI2AkwgCCgCAEECSQ0CIAIgA0YNASACIAg2AgAgBiACQQRqNgJMDAILQQAhAyAEQQJJDQELIAMgBWsiAkECdSINQQFqIgRBgICAgARPDQVB/////wMgAkEBdSIHIAQgBCAHSRsgAkH8////B08bIgcEfyAHQYCAgIAETw0DIAdBAnQQMgVBAAsiCyANQQJ0aiIEIAg2AgAgBEEEaiENIAMgBUcEQAJAIAJBBGsiAkEMTwRAIARBEGshCSADQRBrIQwgAyACQQJ2QQFqIgpB/P///wdxIghBAnQiAmshAyAEIAJrIQRBACECA0AgCSACQQJ0Ig5rIAwgDmv9AAIA/QsCACACQQRqIgIgCEcNAAsgCCAKRg0BCwNAIARBBGsiBCADQQRrIgMoAgA2AgAgAyAFRw0ACwsgBigCSCEFCyAGIAsgB0ECdGo2AlAgBiANNgJMIAYgBDYCSCAFRQ0AIAUQLwsgBkEANgJEIAZCADcCPCABIAZByABqIAZBPGoiAhCLAiAGQTBqIAEgAiAGQdQAahDPBCAGKAIwIgIgBigCNCINRg0GIAAoAgQhBANAIAIoAgRBBGshASACKQIIIQ8gAigCACEHAkAgACgCCCIFIARLBEAgBCAPNwIIIAQgATYCBCAEIAc2AgAgACAEQRBqIgQ2AgQMAQsgBCAAKAIAIghrQQR1IgtBAWoiA0GAgICAAU8NAkH/////ACAFIAhrIgVBA3UiCSADIAMgCUkbIAVB8P///wdPGyIFBH8gBUGAgICAAU8NBCAFQQR0EDIFQQALIgkgC0EEdGoiAyAPNwIIIAMgATYCBCADIAc2AgAgA0EQaiEBIAQgCEcEQANAIANBEGsiAyAEQRBrIgT9AAIA/QsCACAEIAhHDQALIAAoAgAhBAsgACAJIAVBBHRqNgIIIAAgATYCBCAAIAM2AgAgBARAIAQQLwsgASEECyANIAJBEGoiAkcNAAsMBQsQNgALEEcACyAGQdgrNgIIIAZB/x42AgQgBkGmFTYCAEECQbDkACAGEDQQAAALEDYACxA2AAsgBigCMCECCyACBEAgBiACNgI0IAIQLwsgBigCPCIABEAgBigCQCIDIAAiBEcEQANAIANBDGsiASgCACICBEAgA0EIayACNgIAIAIQLwsgASIDIABHDQALIAYoAjwhBAsgBiAANgJAIAQQLwsgBigCSCIABEAgBiAANgJMIAAQLwsgBigCVCIARQ0AIAYgADYCWCAAEC8LIAZB4ABqJAALogMBDH8CQCAAKAIIIgIgACgCDEcEQCACIQMMAQsgACgCBCIEIAAoAgAiB0sEQCAEIAQgB2tBAnVBAWpBfm1BAnQiBWoiAyAEIAIgBGsiBPwKAAAgACADIARqIgM2AgggACAAKAIEIAVqNgIEDAELAkACQAJAQQEgAiAHa0EBdSACIAdGGyIDQYCAgIAESQRAIANBAnQiBRAyIgggBWohCyAIIANBfHEiBmoiBSEDIAIgBEYNAyAFIAIgBGsiAkF8cWohAyACQQRrIgJBHEkNASAGIAhqIARrQRBJDQEgBCACQQJ2QQFqIgxB/P///wdxIglBAnQiBmohAiAFIAZqIQYDQCAFIApBAnQiDWogBCANav0AAgD9CwIAIApBBGoiCiAJRw0ACyAJIAxGDQMMAgsQRwALIAUhBiAEIQILA0AgBiACKAIANgIAIAJBBGohAiAGQQRqIgYgA0cNAAsLIAAgCzYCDCAAIAM2AgggACAFNgIEIAAgCDYCACAHRQ0AIAcQLyAAKAIIIQMLIAMgASgCADYCACAAIAAoAghBBGo2AggL/AoCDn8BeyMAQSBrIgUkAAJAAkACQAJAAkACQAJAAkACQAJ7AkACQCAAKAIQIgFBzgBPBEAgACABQc4AazYCECAAKAIEIgEoAgAhCyAAIAFBBGoiBDYCBCAAKAIIIgEgACgCDEcEQCABIQIMDAsgACgCACIHIARJBEAgBCAEIAdrQQJ1QQFqQX5tQQJ0IgNqIgIgBCABIARrIgH8CgAAIAAgASACaiICNgIIIAAgACgCBCADajYCBAwMC0EBIAEgB2tBAXUgASAHRhsiA0GAgICABE8NASADQQJ0IgIQMiIJIAJqIQwgCSADQXxxIgNqIgghAiABIARGDQogCCABIARrIgFBfHFqIQIgAUEEayIBQRxJDQggAyAJaiAEa0EQSQ0IIAQgAUECdkEBaiINQfz///8HcSIKQQJ0IgFqIQMgASAIaiEBA0AgCCAGQQJ0Ig5qIAQgDmr9AAIA/QsCACAGQQRqIgYgCkcNAAsgCiANRg0KDAkLIAAoAggiASAAKAIEIgNrIghBAnUiAiAAKAIMIgYgACgCACIEayIHQQJ1SQRAIAEgBkcEQCAFQdgfEDI2AgAgACAFELUEDA0LIAVB2B8QMjYCACAAIAUQ2QEgACgCBCIBKAIAIQsgACABQQRqIgQ2AgQgACgCCCIBIAAoAgxHBEAgASECDAgLIAAoAgAiByAESQRAIAQgBCAHa0ECdUEBakF+bUECdCIDaiICIAQgASAEayIB/AoAACAAIAEgAmoiAjYCCCAAIAAoAgQgA2o2AgQMCAtBASABIAdrQQF1IAEgB0YbIgNBgICAgARPDQEgA0ECdCICEDIiCSACaiEMIAkgA0F8cSIDaiIIIQIgASAERg0GIAggASAEayIBQXxxaiECIAFBBGsiAUEcSQ0EIAMgCWogBGtBEEkNBCAEIAFBAnZBAWoiDUH8////B3EiCkECdCIBaiEDIAEgCGohAUEAIQYDQCAIIAZBAnQiDmogBCAOav0AAgD9CwIAIAZBBGoiBiAKRw0ACyAKIA1GDQYMBQsgBSAAQQxqNgIQQQEgB0EBdSAEIAZGGyIHQYCAgIAETw0AIAUgB0ECdCIEEDIiBjYCACAFIAQgBmoiCTYCDCAFIAYgAkECdGoiBDYCBEHYHxAyIQoCQCACIAdHDQAgCEEASgRAIAUgBCACQQFqQX5tQQJ0aiIENgIEDAELQQEgCEEBdSABIANGGyIBQYCAgIAETw0BIAUgAUECdCIDEDIiAjYCACAFIAIgA2oiCTYCDCAFIAIgAUF8cWoiBDYCBCAGEC8gACgCBCEDIAAoAgghASACIQYLIAQgCjYCACAFIARBBGoiAjYCCCABIANGBEAgAyEBIAb9ESAE/RwBIAL9HAIgCf0cAwwDCwNAIAUgAUEEayIBENkBIAEgACgCBEcNAAsMAQsQRwALIAAoAgghAyAF/QAEAAshDyAFIAAoAgAiAjYCACAFIAM2AgggBSABNgIEIAAoAgwhBiAAIA/9CwIAIAUgBjYCDCABIANHBEAgBSADIAEgA2tBA2pBfHFqNgIICyACRQ0IIAIQLwwICyAIIQEgBCEDCwNAIAEgAygCADYCACADQQRqIQMgAUEEaiIBIAJHDQALCyAAIAw2AgwgACACNgIIIAAgCDYCBCAAIAk2AgAgB0UNACAHEC8gACgCCCECCyACIAs2AgAgACAAKAIIQQRqNgIIDAQLIAghASAEIQMLA0AgASADKAIANgIAIANBBGohAyABQQRqIgEgAkcNAAsLIAAgDDYCDCAAIAI2AgggACAINgIEIAAgCTYCACAHRQ0AIAcQLyAAKAIIIQILIAIgCzYCACAAIAAoAghBBGo2AggLIAVBIGokAAvnFQEQfyMAQYABayIGJAAgBkEANgJ8IAZCADcCdCAG/QwAAAAAAAAAAAAAAAAAAAAA/QsDYCAG/QwAAAAAAAAAAAAAAAAAAAAA/QsDUCAG/QwAAAAAAAAAAAAAAAAAAAAA/QsDQAJAAkACf0EAIAAoAhwiEEUNABogBkEAOgA8IAYgAjYCOCAGIAI2AjQgBv0MAAAAAAAAAAAAAAAAAAAAAP0LAyAgBv0MAAAAAAAAAAAAAAAAAAAAAP0LAxAgBv0MAAAAAAAAAAAAAAAAAAAAAP0LAwAgBkH0AGogBhCBAiAGKAIcIgcEQCAGIAc2AiAgBxAvCyAGKAIQIgcEQCAGIAc2AhQgBxAvCyAGKAJ4IgdBNGsiCEEANgIAIAggAjYCDCAIIAE2AgggCCABNgIEAkAgACgCECIJIAgoAhQgCEEQaiIKKAIAIgtrQQxtIg1LBEAgCiAJIA1rIAZBNGoQwAQgBigCeCEHDAELIAkgDU8NACAIIAsgCUEMbGo2AhQLAkAgACgCFCIAIAdBNGsiCCIJKAIgIAhBHGoiDSgCACIKa0EDdSIISwRAIA0gACAIaxDeAiAGKAJ4IQcMAQsgACAITw0AIAkgCiAAQQN0ajYCIAsgAiABayENIAdBNGsiACAFOgAwIAAgBDYCLCAAIBA2AiggBEGAIHEhESAEQSBxIRJBACEIQQAhAEEAIQUDQCAAQQFqIgBB/x9xRSAAQQx2IA1OcQ0CIAdBNGsiCigCKCIEBEAgBCAKIAQoAgAoAggRAgALAkACQAJAAkAgCigCAEHoB2oOCgAHBwcHAwMCAQMHCyAHQSxrKAIAIQQCQCASRQ0AIAEgBEcNACAGKAJ4IgRBGGsoAgAiBwRAIARBFGsgBzYCACAHEC8LIARBJGsoAgAiBwRAIARBIGsgBzYCACAHEC8LIAYgBEE0azYCeAwDCwJAIBFFDQAgAiAERg0AIAYoAngiBEEYaygCACIHBEAgBEEUayAHNgIAIAcQLwsgBEEkaygCACIHBEAgBEEgayAHNgIAIAcQLwsgBiAEQTRrNgJ4DAMLIAggBSAEIAdBMGsoAgBrIhBOcUUEQCAGIAr9AAIA/QsDQCAKIAZBQGtHBEACQCAKKAIUIgwgCigCECIHayIJQQxtIgggBigCWCIEIAYoAlAiBWtBDG1NBEAgByAHIAYoAlQiCSAFIgRrQQxtIg5BDGxqIgsgDCAIIA5LGyIPRwRAA0AgBCAHKAIANgIAIAQgBygCBDYCBCAEIActAAg6AAggBEEMaiEEIAdBDGoiByAPRw0ACwsgCCAOSwRAIAsgDEcEQANAIAkgCykCADcCACAJIAsoAgg2AgggCUEMaiEJIAtBDGoiCyAMRw0ACwsgBiAJNgJUDAILIAYgBSAEIAVrQQxtQQxsajYCVAwBCyAFBEAgBiAFNgJUIAUQLyAGQQA2AlggBkIANwJQQQAhBAsCQCAIQdaq1aoBTw0AQdWq1aoBIARBDG0iBEEBdCIFIAggBSAISxsgBEGq1arVAE8bIgRB1qrVqgFPDQAgBiAEQQxsIgUQMiIENgJUIAYgBDYCUCAGIAQgBWo2AlggBiAHIAxHBH8gBCAHIAlBDGsiBSAFQQxwa0EMaiIF/AoAACAEIAVqBSAECzYCVAwBCxA2AAsCQAJAAkACQCAKKAIgIgsgCigCHCIHayIIQQN1IgwgBigCZCIEIAYoAlwiBWtBA3VNBEAgByAHIAYoAmAiCCAFayIOaiIJIAsgDCAOQQN1Ig9LGyITRwRAIAchBANAIAUgBCgCADYCACAFIAQoAgQ2AgQgBUEIaiEFIARBCGoiBCATRw0ACwsgDCAPSwRAIAkgC0YEQCAGIAg2AmAMBgsgCyAHIA5qIgRrQQhrIgVBOEkNAiAIIARrQRBJDQIgCCAFQQN2QQFqIg5B/v///wNxIgxBA3QiBGohBSAEIAlqIQRBACEHA0AgCCAHQQN0Ig9qIAkgD2r9AAIA/QsCACAHQQJqIgcgDEcNAAsgDCAORg0EDAMLIAYgBTYCYAwECyAFBEAgBiAFNgJgIAUQLyAGQQA2AmQgBkIANwJcQQAhBAsCQCAIQQBIDQBB/////wEgBEECdSIFIAwgBSAMSxsgBEH4////B08bIgRBgICAgAJPDQAgBiAEQQN0IgUQMiIENgJgIAYgBDYCXCAGIAQgBWo2AmQgBiAHIAtHBH8gBCAHIAhBCGtBeHFBCGoiBfwKAAAgBCAFagUgBAs2AmAMBAsQNgALIAkhBCAIIQULA0AgBSAEKQIANwIAIAVBCGohBSAEQQhqIgQgC0cNAAsLIAYgBTYCYAsLIAYgCikCKDcCaCAGIAotADA6AHAgECEFCyAGKAJ4IQQgBSANRgRAIAYoAnQiByAERwRAA0AgBEE0ayIFKAIcIggEQCAEQRRrIAg2AgAgCBAvCyAEQSRrKAIAIggEQCAEQSBrIAg2AgAgCBAvCyAFIgQgB0cNAAsLIAYgBzYCeEEBIQggDSEFDAMLIARBGGsoAgAiBwRAIARBFGsgBzYCACAHEC8LIARBJGsoAgAiBwRAIARBIGsgBzYCACAHEC8LIAYgBEE0azYCeEEBIQgMAgsgBiAKEN0CIQcgCigCKCIEQQEgCiAEKAIAKAIMEQUAIAYoAigiBEEAIAcgBCgCACgCDBEFAAJAIAYoAngiBCAGKAJ8SQRAIAQgB/0AAgD9CwIAIAT9DAAAAAAAAAAAAAAAAAAAAAD9CwIQIAQgBigCEDYCECAEIAYoAhQ2AhQgBCAGKAIYNgIYIAZBADYCECAEQgA3AiAgBCAGKAIcNgIcIAQgBigCIDYCICAEIAYoAiQ2AiQgBv0MAAAAAAAAAAAAAAAAAAAAAP0LAhggBCAGLQAwOgAwIAQgBikCKDcCKCAGIARBNGo2AngMAQsgBkH0AGogBxCBAiAGKAIcIgRFDQAgBiAENgIgIAQQLwsgBigCECIERQ0BIAYgBDYCFCAEEC8MAQsgBigCeCIEQRhrKAIAIgcEQCAEQRRrIAc2AgAgBxAvCyAEQSRrKAIAIgcEQCAEQSBrIAc2AgAgBxAvCyAGIARBNGs2AngLIAYoAngiByAGKAJ0Rw0ACyAIBEAgAygCACIAQQE6AAggACABIAVqNgIEIAAgATYCAAJAIAYoAlQiAiAGKAJQIgFGDQBBASACIAFrQQxtIgIgAkEBTRsiA0EBcSEUQQAhBCACQQJPBEAgA0F+cSEIQQAhAgNAIAAgBEEBckEMbCIJaiIDIAEgBEEMbCINaiIFKAIANgIAIAMgBSgCBDYCBCADIAUtAAg6AAggACANaiIDIAEgCWoiBSgCADYCGCADIAUoAgQ2AhwgAyAFLQAIOgAgIARBAmohBCACQQJqIgIgCEcNAAsLIBRFDQAgBEEMbCICIABqIgAgASACaiIBKAIANgIMIAAgASgCBDYCECAAIAEtAAg6ABQLQQEMAQtBAAshFSAGKAJcIgAEQCAGIAA2AmAgABAvCyAGKAJQIgAEQCAGIAA2AlQgABAvCyAGKAJ0IgEEQCAGKAJ4IgQgASIARwRAA0AgBEE0ayIAKAIcIgIEQCAEQRRrIAI2AgAgAhAvCyAEQSRrKAIAIgIEQCAEQSBrIAI2AgAgAhAvCyAAIgQgAUcNAAsgBigCdCEACyAGIAE2AnggABAvCyAGQYABaiQAIBUPCxDcAgALENsCAAuyHAEWfyMAQdAAayIGJAAgBkIANwNIIAb9DAAAAAAAAAAAAAAAAAAAAAD9CwM4An9BACAAKAIcIhJFDQAaIAZBOGoQtgQgBigCPCAGKAJMIAYoAkhqIghBzgBuIgdBAnRqKAIAIAggB0HOAGxrQTRsaiIH/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAH/QwAAAAAAAAAAAAAAAAAAAAA/QsCICAH/QwAAAAAAAAAAAAAAAAAAAAA/QsCECAGIAYoAkwiCUEBaiIINgJMIAYoAjwiByAJIAYoAkgiD2oiCkHOAG4iCUECdGoiDSgCACAKIAlBzgBsa0E0bCIKaiIJIAE2AgQgCUEANgIAIA0oAgAgCmogATYCCCANKAIAIApqIAI2AgwCQCAAKAIUIhAgDSgCACAKaiIKKAIgIAooAhwiAGtBA3UiCUsEQCAKQRxqIBAgCWsQ3gIgBigCPCEHIAYoAkghDyAGKAJMIQgMAQsgCSAQTQ0AIAogACAQQQN0ajYCIAsgAiABayEQIAcgCCAPakEBayIJQc4AbiIAQQJ0aiIHKAIAIAkgAEHOAGxrQTRsIgBqIBI2AiggBygCACAAaiIAIAU6ADAgACAENgIsIARBgCBxIRUgBEEgcSEWQQAhD0EAIQRBACEFA0ACQAJAAkACQAJAIARBAWoiBEH/H3FFIARBDHYgEE5xRQRAIAYoAjwgCCAGKAJIakEBayIHQc4AbiIAQQJ0aigCACAHIABBzgBsa0E0bGoiDCgCKCIABEAgACAMIAAoAgAoAggRAgALAkACQAJAAkAgDCgCAEHoB2oOCgAIBQUFAQgDAggFCyAMKAIIIQACQCAWRQ0AIAAgAUcNACAGKAI8IAYoAkwgBigCSGpBAWsiB0HOAG4iAEECdGooAgAgByAAQc4AbGtBNGxqIgcoAhwiAARAIAcgADYCICAAEC8LIAcoAhAiAARAIAcgADYCFCAAEC8LIAYgBigCTCIHQQFrNgJMIAYoAkAiCCAGKAI8IgBrQQJ1Qc4AbEEBa0EAIAAgCEcbIAcgBigCSGprQQFqQZwBSQ0IIAhBBGsoAgAQLyAGIAYoAkBBBGs2AkAMCAsCQCAVRQ0AIAAgAkYNACAGKAI8IAYoAkwgBigCSGpBAWsiB0HOAG4iAEECdGooAgAgByAAQc4AbGtBNGxqIgcoAhwiAARAIAcgADYCICAAEC8LIAcoAhAiAARAIAcgADYCFCAAEC8LIAYgBigCTCIHQQFrNgJMIAYoAkAiCCAGKAI8IgBrQQJ1Qc4AbEEBa0EAIAAgCEcbIAcgBigCSGprQQFqQZwBSQ0IIAhBBGsoAgAQLyAGIAYoAkBBBGs2AkAMCAsgECAFIAAgDCgCBGsiACAAIAVMGyAAIA8bIgVGBEAgBigCQCIHIAYoAjwiAEYEQCAAIQcMCAsgACAGKAJIIglBzgBuIgVBAnRqIgooAgAgCSAFQc4AbGtBNGxqIgggACAGKAJMIAlqIglBzgBuIgVBAnRqKAIAIAkgBUHOAGxrQTRsaiIFRg0HA0AgCCgCHCIABEAgCCAANgIgIAAQLwsgCCgCECIABEAgCCAANgIUIAAQLwsgCEE0aiIIIAooAgBrQdgfRgRAIAooAgQhCCAKQQRqIQoLIAUgCEcNAAsMBgsgBigCPCAGKAJMIAYoAkhqQQFrIgdBzgBuIgBBAnRqKAIAIAcgAEHOAGxrQTRsaiIHKAIcIgAEQCAHIAA2AiAgABAvCyAHKAIQIgAEQCAHIAA2AhQgABAvCyAGIAYoAkwiB0EBazYCTEEBIQ8gBigCQCIIIAYoAjwiAGtBAnVBzgBsQQFrQQAgACAIRxsgByAGKAJIamtBAWpBnAFJDQcgCEEEaygCABAvIAYgBigCQEEEazYCQAwHCyAGKAJIIgpFBEAjAEEQayIOJAACQCAGQThqIgsoAggiDSALKAIEIhFrQQJ1IghBzgBsQQFrQQAgDSARRxsgCygCECIAIAsoAhRqa0HOAE8EQCALIABBzgBqNgIQIA4gDUEEayIAKAIANgIMIAsgADYCCCALIA5BDGoQ2QEMAQsgCCALKAIMIgcgCygCACIJayIAQQJ1SQRAAkAgCSARRwRAIA5B2B8QMjYCDCALIA5BDGoQ2QEMAQsgDkHYHxAyNgIMIAsgDkEMaiIHELUEIA4gCygCCEEEayIAKAIANgIMIAsgADYCCCALIAcQ2QELIAtBJyALKAIQQc4AaiALKAIIIAsoAgRrQQRGGzYCEAwBCwJAAkBBASAAQQF1IAcgCUYbIglBgICAgARPDQAgCUECdCIHEDIhAEHYHxAyIQggCQR/IAAgB2oFQQQQMiEaIAAQLyALKAIIIQ0gCygCBCERIBoiAEEEagshCiAAIAg2AgAgAEEEaiEHIA0gEUYEQCAAIQgMAgsgACEIA0ACQCAHIApHBEAgByENDAELIAAgCEsEQCAAIAAgCGtBAnVBAWpBfm1BAnRqIgkgACAHIABrIgD8CgAAIAAgCWohDSAJIQAgByEKDAELQQEgByAIa0EBdSAHIAhGGyIJQYCAgIAETw0CIAlBAnQiFxAyIhIgCUF8cSIKaiIJIQ0CQCAAIAdGDQAgCSAHIABrIgdBfHFqIQ0CQAJAIAdBBGsiB0EcSQ0AIAogEmogAGtBEEkNACAAIAdBAnZBAWoiGEH8////B3EiFEECdCIKaiEHIAkgCmohCkEAIRMDQCAJIBNBAnQiGWogACAZav0AAgD9CwIAIBNBBGoiEyAURw0ACyAUIBhGDQIMAQsgCSEKIAAhBwsDQCAKIAcoAgA2AgAgB0EEaiEHIApBBGoiCiANRw0ACwsgEiAXaiEKIAgEQCAIEC8LIBIhCCAJIQALIA0gESgCADYCACANQQRqIQcgEUEEaiIRIAsoAghHDQALDAELEEcACyALIAo2AgwgCyAHNgIIIAsgADYCBCALKAIAIQkgCyAINgIAIAtBJyALKAIQQc4AaiAHIABrQQRGGzYCECAJRQ0AIAkQLwsgDkEQaiQAIAYoAkghCgsgBigCPCIJIApBzgBuIgBBAnRqIggoAgAiByAKIABBzgBsa0E0bGpBACAGKAJAIAlHGyIAIAdGBH8gCEEEaygCAEHYH2oFIAALQTRrIgAgDP0AAgD9CwIAIABBADYCGCAAQgA3AhAgACAMKAIQNgIQIAAgDCgCFDYCFCAAIAwoAhg2AhggDEEANgIYIAxCADcCECAAQQA2AiQgAEIANwIcIAAgDCgCHDYCHCAAIAwoAiA2AiAgACAMKAIkNgIkIAxBADYCJCAMQgA3AhwgACAMLQAwOgAwIAAgDCkCKDcCKCAGIAYoAkwiB0EBajYCTCAGIAYoAkhBAWsiADYCSCAGKAI8IAAgB2oiB0HOAG4iAEECdGooAgAgByAAQc4AbGtBNGxqIgcoAhwiAARAIAcgADYCICAAEC8LIAcoAhAiAARAIAcgADYCFCAAEC8LIAYgBigCTCIHQQFrNgJMIAYoAkAiCCAGKAI8IgBrQQJ1Qc4AbEEBa0EAIAAgCEcbIAcgBigCSGprQQFqQZwBSQ0GIAhBBGsoAgAQLyAGIAYoAkBBBGs2AkAMBgsgBkEEaiAMEN0CIQkgDCgCKCIAQQEgDCAAKAIAKAIMEQUAIAYoAiwiAEEAIAkgACgCACgCDBEFACAGKAJMIAYoAkhqIAYoAkAiACAGKAI8IghrQQJ1Qc4AbEEBa0EAIAAgCEcbRgR/IAZBOGoQtgQgBigCPCEIIAYoAkAFIAALIAhGBH9BAAUgCCAGKAJMIAYoAkhqIgdBzgBuIgBBAnRqKAIAIAcgAEHOAGxrQTRsagsiByAJ/QACAP0LAgAgB/0MAAAAAAAAAAAAAAAAAAAAAP0LAhAgByAGKAIUNgIQIAcgBigCGDYCFCAGKAIcIQAgB0IANwIgIAcgADYCGCAHIAYoAiA2AhwgByAGKAIkNgIgIAcgBigCKDYCJCAGQgA3AiQgBv0MAAAAAAAAAAAAAAAAAAAAAP0LAhQgByAGLQA0OgAwIAcgBikCLDcCKCAGIAYoAkxBAWoiCDYCTAwGCyAGKAI8IAYoAkwgBigCSGpBAWsiB0HOAG4iAEECdGooAgAgByAAQc4AbGtBNGxqIgcoAhwiAARAIAcgADYCICAAEC8LIAcoAhAiAARAIAcgADYCFCAAEC8LIAYgBigCTCIHQQFrNgJMIAYoAkAiCCAGKAI8IgBrQQJ1Qc4AbEEBa0EAIAAgCEcbIAcgBigCSGprQQFqQZwBSQ0EIAhBBGsoAgAQLyAGIAYoAkBBBGs2AkAMBAsQ3AIACxDbAgALIAYoAjwhACAGKAJAIQcLIAZBADYCTCAHIABrIghBCEsEQANAIAAoAgAQLyAGIAYoAjxBBGoiADYCPCAGKAJAIABrIghBCEsNAAsLQQEhD0EnIQAgECEFAkACQCAIQQJ2QQFrDgIBAAILQc4AIQALIAYgADYCSAsgBigCTCEICyAIDQALQQAgD0UNABogAygCACIAQQE6AAggACABIAVqNgIEIAAgATYCAEEBCyEbIAZBADYCTCAGKAJAIgQgBigCPCIIayIAQQlPBEADQCAIKAIAEC8gBiAGKAI8QQRqIgg2AjwgBigCQCIEIAhrIgBBCEsNAAsLQSchAQJAAkACQCAAQQJ2QQFrDgIBAAILQc4AIQELIAYgATYCSAsCQCAEIAhGDQADQCAIKAIAEC8gCEEEaiIIIARHDQALIAYoAkAiASAGKAI8IgBGDQAgBiABIAAgAWtBA2pBfHFqNgJACyAGKAI4IgAEQCAAEC8LIAZB0ABqJAAgGwt7AAJAIAEgAkYNACABQQFqIAJGDQAgAS0AAEHcAEcNAAJAIAEsAAEiAkEkaw47AAEBAQEBAAEBAQABAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQAAAQABCyAAIAIQWSABQQJqIQELIAELxgMBBX8CQAJAAkAgACgCBCAAKAIAIgRrQRhtIgVBAWoiAkGr1arVAEkEQEGq1arVACAAKAIIIARrQRhtIgRBAXQiBiACIAIgBkkbIARB1arVKk8bIgQEQCAEQavVqtUATw0CIARBGGwQMiEDCyAFQRhsIANqIgIgASkCADcCACACIAEoAgg2AgggAUIANwIAIAFBADYCCCACIAEoAhQ2AhQgAiABKQIMNwIMIAFCADcCDCABQQA2AhQgAyAEQRhsaiEDIAJBGGohBCAAKAIEIgEgACgCACIFRg0CA0AgAkEYayICIAFBGGsiASkCADcCACACIAEoAgg2AgggAUIANwIAIAFBADYCCCACIAEoAhQ2AhQgAiABKQIMNwIMIAFCADcCDCABQQA2AhQgASAFRw0ACyAAIAM2AgggACgCBCEDIAAgBDYCBCAAKAIAIQEgACACNgIAIAEgA0YNAwNAIANBAWssAABBAEgEQCADQQxrKAIAEC8LIANBGGshACADQQ1rLAAAQQBIBEAgACgCABAvCyAAIgMgAUcNAAsMAwsQNgALEEcACyAAIAM2AgggACAENgIEIAAgAjYCAAsgAQRAIAEQLwsLFABBDBBeQQEQcEHQqQNB0gAQAgALsQwBC38CQCABIAJHBEACQAJAAkACQAJAAkACQAJAAkAgASwAACIFQcQAaw40AwgICAgICAgICAgICAgIBQgICAcICAgICAgICAgIAQgCCAgICAgICAgICAgICAgECAgIBgALIAUNByADIAUQWgwJCyADQQgQWgwICyAEIAQoAlBBwAByNgJQDAcLIAQgBCgCVEHAAHI2AlQMBgsgBCAEKAJQQQFyNgJQDAULIAQgBCgCVEEBcjYCVAwECyAEIAQoAlBB4AByNgJQIARB3wAQowEMAwsgBCAEKAJUQeAAcjYCVEHfACEFAkACQAJAAkAgBC0AWQRAIAQoAgwiAEHfACAAKAIAKAIUEQQAIQUgBCgCJCIDIAQoAigiAkkNAyADIAQoAiAiBmsiCEEBaiIAQQBIDQEgCEH/////ByACIAZrIgJBAXQiByAAIAAgB0kbIAJB/////wNPGyIHBH8gBxAyBUEACyICaiIAIAU6AAAgAiAHaiEJIABBAWohCgJAIAMgBkYEQCAAIQIMAQsCQAJAIAhBEEkNACAGIAIgA2prIANqQRBJDQAgAEEQayELIANBEGshDCADIAhBcHEiB2shAyAAIAdrIQBBACEFA0AgCyAFayAMIAVr/QAAAP0LAAAgBUEQaiIFIAdHDQALIAcgCEYNAQsgBkF/cyADaiENIAMgBmtBA3EiBwRAQQAhBQNAIABBAWsiACADQQFrIgMtAAA6AAAgBUEBaiIFIAdHDQALCyANQQNJDQADQCAAQQFrIANBAWstAAA6AAAgAEECayADQQJrLQAAOgAAIABBA2sgA0EDay0AADoAACAAQQRrIgAgA0EEayIDLQAAOgAAIAMgBkcNAAsLIAQoAiAhAwsgBCAJNgIoIAQgCjYCJCAEIAI2AiAgA0UNBCADEC8MBAsgBCgCKCEAIAQoAiQhAwJAIAQtAFoEQCAAIANLDQQgAyAEKAIgIgZrIghBAWoiAkEASA0DIAhB/////wcgACAGayIAQQF0IgUgAiACIAVJGyAAQf////8DTxsiBQR/IAUQMgVBAAsiAmoiAEHfADoAACACIAVqIQkgAEEBaiEKAkAgAyAGRgRAIAAhAgwBCwJAAkAgCEEQSQ0AIAYgAiADamsgA2pBEEkNACAAQRBrIQsgA0EQayEMIAMgCEFwcSIHayEDIAAgB2shAEEAIQUDQCALIAVrIAwgBWv9AAAA/QsAACAFQRBqIgUgB0cNAAsgByAIRg0BCyAGQX9zIANqIQ4gAyAGa0EDcSIHBEBBACEFA0AgAEEBayIAIANBAWsiAy0AADoAACAFQQFqIgUgB0cNAAsLIA5BA0kNAANAIABBAWsgA0EBay0AADoAACAAQQJrIANBAmstAAA6AAAgAEEDayADQQNrLQAAOgAAIABBBGsiACADQQRrIgMtAAA6AAAgAyAGRw0ACwsgBCgCICEDCyAEIAk2AiggBCAKNgIkIAQgAjYCICADDQEMBQsgACADRw0DIAMgBCgCICIIayIGQQFqIgBBAEgNAiAGQf////8HIAZBAXQiAiAAIAAgAkkbIAZB/////wNPGyIFBH8gBRAyBUEACyICaiIAQd8AOgAAIAIgBWohCSAAQQFqIQoCQCADIAhGBEAgACECDAELAkACQCAGQRBJDQAgCCACIANqayADakEQSQ0AIABBEGshCyADQRBrIQwgAyAGQXBxIgdrIQMgACAHayEAQQAhBQNAIAsgBWsgDCAFa/0AAAD9CwAAIAVBEGoiBSAHRw0ACyAGIAdGDQELIAhBf3MgA2ohDyADIAhrQQNxIgcEQEEAIQUDQCAAQQFrIgAgA0EBayIDLQAAOgAAIAVBAWoiBSAHRw0ACwsgD0EDSQ0AA0AgAEEBayADQQFrLQAAOgAAIABBAmsgA0ECay0AADoAACAAQQNrIANBA2stAAA6AAAgAEEEayIAIANBBGsiAy0AADoAACADIAhHDQALCyAEKAIgIQMLIAQgCTYCKCAEIAo2AiQgBCACNgIgIANFDQQLIAMQLwwDCxA2AAsQNgALIAMgBToAACAEIANBAWo2AiQLDAILIAAgASACIAMQvwQPCxCAAgALIAFBAWoLzQEBBH8jAEEQayIFJAACQAJAIAIgAWsiBEECSA0AIAEgBGpBAWshByABIQQDQCAEQQFqIQYgBC0AAEEuRgRAIAYtAABB3QBHBEAgBiIEIAdHDQIMAwsgAiAERg0CIAVBBGogACABIAQQ2QIgAywAC0EASARAIAMoAgAQLwsgAyAFKQIENwIAIAMgBSgCDDYCCCADKAIEIAMtAAsiACAAwEEASBtBAWtBAk8NAyAFQRBqJAAgBEECag8LIAYiBCAHRw0ACwsQ/gEACxC7BAALzQIBA38gAEHksAE2AgAgACgCRCIDBEAgACgCSCICIAMiAUcEQANAIAJBDGshASACQQFrLAAAQQBIBEAgASgCABAvCyABIgIgA0cNAAsgACgCRCEBCyAAIAM2AkggARAvCyAAKAI4IgEEQCAAIAE2AjwgARAvCyAAKAIsIgMEQCAAKAIwIgEgAyICRwRAA0AgAUEBaywAAEEASARAIAFBDGsoAgAQLwsgAUEYayECIAFBDWssAABBAEgEQCACKAIAEC8LIAIiASADRw0ACyAAKAIsIQILIAAgAzYCMCACEC8LIAAoAiAiAQRAIAAgATYCJCABEC8LIAAoAhQiAQRAIAAgATYCGCABEC8LIAAoAggiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgAAueBgEEfwJAAkAgASACRwR/AkACQAJAAkACQAJAAkACQAJAAkAgAS0AACIGwCIEQTBrDkkICQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkFCQkACQkJCQkJCQEJCQkCCQMGBAkHCQsCQCADBEAgA0EMEFoMAQsgAEEMEFkLDAsLAkAgAwRAIANBChBaDAELIABBChBZCwwKCwJAIAMEQCADQQ0QWgwBCyAAQQ0QWQsMCQsCQCADBEAgA0EJEFoMAQsgAEEJEFkLDAgLAkAgAwRAIANBCxBaDAELIABBCxBZCwwHCyABQQFqIAJGDQUgASwAASICQcEASA0FIAJB4QBrQf8BcUEZSyACQdsAT3ENBSACQR9xIQICQCADBEAgAyACEFoMAQsgACACEFkLIAFBAmoPCyABQQFqIAJGDQRB0P8DIQYCQCABLQABIgVB+AFxQTBGDQAgBUH+AXFBOEYNAEGp/wMhBiAFQSByIgVB4QBrQf8BcUEGTw0FCyABQQJqIgEgAkYNBEHQ/wMhBwJAIAEtAAAiBEH4AXFBMEYNACAEQf4BcUE4Rg0AQan/AyEHIARBIHIiBEHhAGtB/wFxQQZPDQULIAQgB2pBBHQgBSAGakEIdGohBQsgAUEBaiACRg0DQdD/AyEGAkAgAS0AASIEQfgBcUEwRg0AIARB/gFxQThGDQBBqf8DIQYgBEEgciIEQeEAa0H/AXFBBk8NBAsgAUECaiACRg0DQdD/AyEHAkAgAS0AAiICQfgBcUEwRg0AIAJB/gFxQThGDQBBqf8DIQcgAkEgciICQeEAa0H/AXFBBk8NBAsgAiAHaiAEIAZqIAVqQQR0aiECAkAgAwRAIAMgAsAQWgwBCyAAIALAEFkLIAFBA2oPCwJAIAMEQCADQQAQWgwBCyAAQQAQWQsMAwsgBEHfAEYNASAEQQBOBEAgACgCBCgCCCAGQQJ0ai0AAEHgAHENAgsCQCADBEAgAyAEEFoMAQsgACAEEFkLIAFBAWoFIAELDwsQgAIACyABQQFqC+gEAQh/IAEgACgCCCIGIAAoAgQiBGtBDG1NBEACQCABRQ0AIAQhAyABQQxsQQxrIgVBDG5BAWpBA3EiBgRAA0AgAyACKQIANwIAIAMgAigCCDYCCCADQQxqIQMgCUEBaiIJIAZHDQALCyABQQxsIARqIQQgBUEkSQ0AA0AgAyACKQIANwIAIAMgAigCCDYCCCADIAIoAgg2AhQgAyACKQIANwIMIAMgAigCCDYCICADIAIpAgA3AhggAyACKQIANwIkIAMgAigCCDYCLCADQTBqIgMgBEcNAAsLIAAgBDYCBA8LAkAgBCAAKAIAIgNrQQxtIgUgAWoiB0HWqtWqAUkEQEHVqtWqASAGIANrQQxtIgZBAXQiAyAHIAMgB0sbIAZBqtWq1QBPGyIHBEAgB0HWqtWqAU8NAiAHQQxsEDIhCgsgCiAFQQxsaiIIIQMgAUEMbCIFQQxrIgZBDG5BAWpBA3EiAQRAA0AgAyACKQIANwIAIAMgAigCCDYCCCADQQxqIQMgCUEBaiIJIAFHDQALCyAFIAhqIQUgBkEkTwRAA0AgAyACKQIANwIAIAMgAigCCDYCCCADIAIoAgg2AhQgAyACKQIANwIMIAMgAigCCDYCICADIAIpAgA3AhggAyACKQIANwIkIAMgAigCCDYCLCADQTBqIgMgBUcNAAsLIAAoAgAiASAERwRAA0AgCEEMayIIIARBDGsiBCkCADcCACAIIAQoAgg2AgggASAERw0ACyAAKAIAIQQLIAAgCiAHQQxsajYCCCAAIAU2AgQgACAINgIAIAQEQCAEEC8LDwsQNgALEEcACxQAQQwQXkEIEHBB0KkDQdIAEAIACxQAQQwQXkEHEHBB0KkDQdIAEAIAC7QBAQJ/QTgQMiEEIAAoAiQoAgQhBSAEQdyrATYCACAEIAU2AgQgBEEIaiABKAIAIgU2AgAgBUEEakEB/h4CABogBCABKQIENwIMIAQgASkCDDcCFCAEIAEpAhQ3AhwgBCABKAIcNgIkIAQgASgCICIFNgIoIAUEQCAFQQH+HgIEGgsgASgCJCEBIAQgAjoANCAEIAM2AjAgBCABNgIsIAAoAiQgBDYCBCAAIAAoAiQoAgQ2AiQLUAEBfyAAEOUBIgAgAEGcijUQeTYCBCAAQdCINRB5IQEgAEIANwIQIABBgAQ2AgwgACABNgIIIAD9DAAAAAAAAAAAAAAAAAAAAAD9CwIYIAALFAAgAUGeeDYCACABIAAoAgQ2AigLFABBDBBeQQ8QcEHQqQNB0gAQAgAL1wkBBX8CQAJAAkAgASACRg0AIAEtAAAiBkEqRgRAQQgQMiEGIAAoAiQiAigCBCEIIAZByKgBNgIAIAYgCDYCBCACQQA2AgRBJBAyIQIgACgCFCEIIAMoAgQhByACIAY2AgggAiAHNgIEIAJBAToAICACIAU2AhwgAiAENgIYIAIgCDYCFCACQoCAgIBwNwIMIAJB3LIBNgIAIANBADYCBEEIEDIiBEHYswE2AgAgBCACNgIEIAAoAiQgBDYCBCAAIAIoAgg2AiQgAyACNgIEIAAgCEEBajYCFCABQQFqDwsgAUEBaiACRg0AIAZB3ABHDQAgAS0AASIGQfsARw0AIAEgBkH7AEZBAXRqIgYgAkYNASAGLQAAIgFB+AFxQTBHIAFB/gFxQThHcQ0BIAFBMGshCAJAIAIgBkEBaiIBRwRAA0AgAS0AACIHQfgBcUEwRyAHQf4BcUE4R3ENAiAIQcyZs+YATg0EIAhBCmwgB2pBMGshCCABQQFqIgEgAkcNAAsLIAIhAQsgASAGRg0BIAEgAkYNAiABQQFqIQYgAS0AACIHQSxHBEAgAiAGRg0DIAdB3ABHDQMgBi0AAEH9AEcNA0EIEDIhBiAAKAIkIgIoAgQhByAGQcioATYCACAGIAc2AgQgAkEANgIEQSQQMiECIAAoAhQhByADKAIEIQkgAiAGNgIIIAIgCTYCBCACQQE6ACAgAiAFNgIcIAIgBDYCGCACIAc2AhQgAiAINgIQIAIgCDYCDCACQdyyATYCACADQQA2AgRBCBAyIgRB2LMBNgIAIAQgAjYCBCAAKAIkIAQ2AgQgACACKAIINgIkIAMgAjYCBCAAIAdBAWo2AhQgAUECag8LIAIgBkYNAgJAIAYtAAAiCUH4AXFBMEcEQEF/IQcgCUH+AXFBOEcNAQsgAUECaiIGIAJGDQMgCUEwayEHA0AgBi0AACIBQfgBcUEwRyABQf4BcUE4R3ENASAHQcyZs+YATg0DIAdBCmwgAWpBMGshByAGQQFqIgYgAkcNAAsMAwsgAiAGRg0CIAZBAWogAkYNAiAGLQAAQdwARw0CIAYtAAFB/QBGIgFFDQIgBiABQQF0aiEBIAdBf0YEQEEIEDIhBiAAKAIkIgIoAgQhByAGQcioATYCACAGIAc2AgQgAkEANgIEQSQQMiECIAAoAhQhByADKAIEIQkgAiAGNgIIIAIgCTYCBCACQQE6ACAgAiAFNgIcIAIgBDYCGCACIAc2AhQgAkF/NgIQIAIgCDYCDCACQdyyATYCACADQQA2AgRBCBAyIgRB2LMBNgIAIAQgAjYCBCAAKAIkIAQ2AgQgACACKAIINgIkIAMgAjYCBCAAIAdBAWo2AhQgAQ8LIAcgCEgNAUEIEDIhBiAAKAIkIgIoAgQhCSAGQcioATYCACAGIAk2AgQgAkEANgIEQSQQMiECIAAoAhQhCSADKAIEIQogAiAGNgIIIAIgCjYCBCACQQE6ACAgAiAFNgIcIAIgBDYCGCACIAk2AhQgAiAHNgIQIAIgCDYCDCACQdyyATYCACADQQA2AgRBCBAyIgRB2LMBNgIAIAQgAjYCBCAAKAIkIAQ2AgQgACACKAIINgIkIAMgAjYCBCAAIAlBAWo2AhQLIAEPCxDBBAALEMIEAAuOBQEFfwJAAkACQAJ/AkACQCABIAJGIgRFBEACQCACIAFBAWoiA0YgAS0AACIFQSRGcQ0AIAVB2wBrQQJJDQAgBUEuRg0AIAAgBcAQWSADDAQLIAAgASACELkEIgMgAUcNAiAEDQEgAS0AAEEuRw0BQQgQMiEFIAAoAiQiBCgCBCEDIAVBpLUBNgIAIAUgAzYCBCAEIAU2AgQgACAAKAIkKAIENgIkIAFBAWoMAwsgACABIAIQuQQiAyABRw0BCyAAIAEgAhDiAiEDCyADCyIDIAFHDQAgAiADRg0AIAIgAUEBakYEQCABDwsgAS0AAEHcAEcEQCABDwsgAS0AASIDQShGBEAgA0EoRkEBdCEFAkAgAC0ADEECcQRAIAAoAhAhBwwBC0EMEDIhBiAAIAAoAhBBAWoiBzYCECAAKAIkIgQoAgQhAyAGIAc2AgggBkHAsQE2AgAgBiADNgIEIAQgBjYCBCAAIAAoAiQoAgQ2AiQLIAEgBWohAwNAIAMiASACRg0DIAAoAhAhBSAAKAIkIQQgASAAIAEgAhDIBCIDRwRAIAAgAyACIAQgBUEBaiAAKAIQQQFqEMcEIgMgAUcNAQsLIAFBAWogAkYNAiABLQAAQdwARw0CIAEtAAFBKUYiAkUNAiABIAJBAXRqIQMgAC0ADEECcQ0BQQwQMiEEIAAoAiQiAigCBCEBIAQgBzYCCCAEQZCyATYCACAEIAE2AgQgAiAENgIEIAAgACgCJCgCBDYCJCADDwsCQCADQfgBcUEwRg0AIANB/gFxQThGDQAgAQ8LIANBMWtBCEsEQCABDwsgA0EwayICIAAoAhBLDQIgACACENoCIAFBAmohAwsgAw8LEIMCAAsQ/wEAC6oLAQV/AkACQCABIAJHBEACQAJAAkACQAJAAkAgASwAACIEQShrDjUDBQgIBQUABQUFBQUFBQUFBQUFBQUFBQgFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUCAQQLQQgQMiECIAAoAiQiBCgCBCEFIAJBsKwBNgIAIAIgBTYCBCAEIAI2AgQgACAAKAIkKAIENgIkIAFBAWoPCwJ/IAEgASACRg0AGiABIAEtAABB3ABHDQAaAkAgAiABQQFqIgVHBEAgBS0AACIGQTBGBEAgAEEAEFkgAUECagwDCwJAIAZBMWtB/wFxQQhNBEAgBkEwayEDAkACQCABQQJqIgQgAkYNACAELQAAIgdBMGtB/wFxQQpPDQEDQCADQZmz5swBTw0GIANBCmwgB2oiBkEwayEDIARBAWoiBCACRg0BIAQtAAAiB0Ewa0H/AXFBCU0NAAsMAQsgAiEECyAAKAIQIAZBMWtNDQMgACADENoCIAQgBUcNAQsCfwJAAkACQAJAIAIgBSIERg0AAkACQAJAAkACQAJAIAQsAABBxABrDjQBBgYGBgYGBgYGBgYGBgYDBgYGBQYGBgYGBgYGBgYGBgAGBgYGBgYGBgYGBgYGBgIGBgYEBgtB3AAQMiAAIAAoAiQoAgRBACAAKAIMIgNBAXEgA0EIcUEDdhC2ASEDDAYLQdwAEDIgACAAKAIkKAIEQQEgACgCDCIDQQFxIANBCHFBA3YQtgEhAwwFC0HcABAyIAAgACgCJCgCBEEAIAAoAgwiA0EBcSADQQhxQQN2ELYBIQMMBQtB3AAQMiAAIAAoAiQoAgRBASAAKAIMIgNBAXEgA0EIcUEDdhC2ASEDDAQLQdwAEDIgACAAKAIkKAIEQQAgACgCDCIDQQFxIANBCHFBA3YQtgEhAyAAKAIkIAM2AgQgACADNgIkIAMgAygCUEHgAHI2AlAgA0HfABCjAQwEC0HcABAyIAAgACgCJCgCBEEBIAAoAgwiA0EBcSADQQhxQQN2ELYBIQMgACgCJCADNgIEIAAgAzYCJCADIAMoAlBB4AByNgJQIANB3wAQowEgBEEBaiEECyAEDAMLIAAoAiQgAzYCBCAAIAM2AiQgAyADKAJQQcAAcjYCUAwBCyAAKAIkIAM2AgQgACADNgIkIAMgAygCUEEBcjYCUAsgBEEBagsiBCAFRw0AIAEgACAFIAJBABC/BCIAIAAgBUYbIQQLIAQMAgsQgAIACxD/AQALDwsgACABIAIQ4gIPCyABQQFqIgQgAkYNAwJAIAFBAmogAkYNACAELQAAQT9HDQAgAS0AAkE6Rw0AIAAgACgCGEEBajYCGCAAIAFBA2ogAhDlAiIBIAJGDQQgAS0AAEEpRw0EIAAgACgCGEEBazYCGCABQQFqDwsCQCAALQAMQQJxBEAgACgCECEDDAELQQwQMiEBIAAgACgCEEEBaiIDNgIQIAAoAiQiBSgCBCEGIAEgAzYCCCABQcCxATYCACABIAY2AgQgBSABNgIEIAAgACgCJCgCBDYCJAsgACAAKAIYQQFqNgIYIAIgACAEIAIQ5QIiBEYNAyAELQAAQSlHDQMgAC0ADEECcUUEQEEMEDIhASAAKAIkIgIoAgQhBSABIAM2AgggAUGQsgE2AgAgASAFNgIEIAIgATYCBCAAIAAoAiQoAgQ2AiQLIAAgACgCGEEBazYCGCAEQQFqDwsgBEH7AEYNAwsCQCABIAJGDQACQCABLAAAIgJBJGsOWgEAAAABAQEBAAABAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAQEBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBAQALIAAgAhBZIAFBAWohAQsLIAEPCxCDAgALQQwQXkELEHBB0KkDQdIAEAIAC/MFAgR/AX4jAEEwayIFJAACQAJAIAEgAkYNAAJAAkACQAJAIAEsAAAiA0Ekaw4FAQQEBAMACwJAIANB3ABrDgMCBAAEC0EMEDIhAiAAKAIMIQMgACgCJCIEKAIEIQYgAkH0qQE2AgAgAiAGNgIEIAIgA0HwD3FBgAxGOgAIIAQgAjYCBCAAIAAoAiQoAgQ2AiQgAUEBaiEBDAMLQQwQMiECIAAoAgwhAyAAKAIkIgQoAgQhBiACQbyqATYCACACIAY2AgQgAiADQfAPcUGADEY6AAggBCACNgIEIAAgACgCJCgCBDYCJCABQQFqIQEMAgsgAUEBaiACRg0BIAEtAAEiAkHCAEcEQCACQeIARw0CQRgQMiECIAAoAiQoAgQhAyACQYSrATYCACACIAM2AgQgAkEIaiAAKAIAIgM2AgAgA0EEakEB/h4CABogACkCBCEHIAJBADoAFCACIAc3AgwgACgCJCACNgIEIAAgACgCJCgCBDYCJCABQQJqIQEMAgtBGBAyIQIgACgCJCgCBCEDIAJBhKsBNgIAIAIgAzYCBCACQQhqIAAoAgAiAzYCACADQQRqQQH+HgIAGiAAKQIEIQcgAkEBOgAUIAIgBzcCDCAAKAIkIAI2AgQgACAAKAIkKAIENgIkIAFBAmohAQwBCyABQQFqIAJGDQAgAS0AAUE/Rw0AIAFBAmogAkYNACABLAACIgNBIUcEQCADQT1HDQEgBUEIahDEBCIDIAAoAgw2AgwgAyABQQNqIAIQ7QIhASADKAIQIQQgACADQQAgACgCEBDDBCAAIAQgACgCEGo2AhAgASACRg0CIAEtAABBKUcNAiADEMsEIAFBAWohAQwBCyAFQQhqEMQEIgMgACgCDDYCDCADIAFBA2ogAhDtAiEBIAMoAhAhBCAAIANBASAAKAIQEMMEIAAgBCAAKAIQajYCECABIAJGDQEgAS0AAEEpRw0BIAMQywQgAUEBaiEBCyAFQTBqJAAgAQ8LEIMCAAtxAQJ/AkAgACgCICIBRQ0AIAFBf/4eAgQNACABIAEoAgAoAggRAQACQCABQQhqIgL+EAIABEAgAkF//h4CAA0BCyABIAEoAgAoAhARAQALCyAAKAIAIgBBBGpBf/4eAgBFBEAgACAAKAIAKAIIEQEACwv4GwMhfRJ7C38gAkEgTgRAIAJBIG0hOwNAIAEgOkEYbGoiOUGA/AEgACA6QQd0aiI2KgJ8IgMgNioCeCIFIDYqAnQiBiA2KgJwIgcgNioCbCIIIDYqAmgiCSA2KgJkIgogNioCYCILIDYqAlwiDCA2KgJYIg0gNioCVCIOIDYqAlAiDyA2KgJMIhAgNioCSCIRIDYqAkQiEiA2QUBrIjwqAgAiEyA2KgI8IhQgNioCOCIVIDYqAjQiFiA2KgIwIhcgNioCLCIYIDYqAigiGSA2KgIkIhogNioCICIbIDYqAhwiHCA2KgIYIh0gNioCFCIeIDYqAhAiHyA2KgIMIiAgNioCCCIhIDYqAgQiIiA2KgIAIiND//9/fyAjQ///f39dGyIEIAQgIl4bIgQgBCAhXhsiBCAEICBeGyIEIAQgH14bIgQgBCAeXhsiBCAEIB1eGyIEIAQgHF4bIgQgBCAbXhsiBCAEIBpeGyIEIAQgGV4bIgQgBCAYXhsiBCAEIBdeGyIEIAQgFl4bIgQgBCAVXhsiBCAEIBReGyIEIAQgE14bIgQgBCASXhsiBCAEIBFeGyIEIAQgEF4bIgQgBCAPXhsiBCAEIA5eGyIEIAQgDV4bIgQgBCAMXhsiBCAEIAteGyIEIAQgCl4bIgQgBCAJXhsiBCAEIAheGyIEIAQgB14bIgQgBCAGXhsiBCAEIAVeGyIEIAMgBF0bIgSLQwAAgHeUQwAAgAiUQYCAgIgHIAS8IgJBAXQiOEGAgIB4cSI3IDdBgICAiAdNG0EBdkGAgIA8ar6SvCI3QQ12QYD4AXEgN0H/H3FqIDhBgICAeEsbIAJBEHZBgIACcXI7AQIgOUGA/AEgAyAFIAYgByAIIAkgCiALIAwgDSAOIA8gECARIBIgEyAUIBUgFiAXIBggGSAaIBsgHCAdIB4gHyAgICEgIiAjQ///f/8gI0P//3//XhsiIyAiICNeGyIiICEgIl4bIiEgICAhXhsiICAfICBeGyIfIB4gH14bIh4gHSAeXhsiHSAcIB1eGyIcIBsgHF4bIhsgGiAbXhsiGiAZIBpeGyIZIBggGV4bIhggFyAYXhsiFyAWIBdeGyIWIBUgFl4bIhUgFCAVXhsiFCATIBReGyITIBIgE14bIhIgESASXhsiESAQIBFeGyIQIA8gEF4bIg8gDiAPXhsiDiANIA5eGyINIAwgDV4bIgwgCyAMXhsiCyAKIAteGyIKIAkgCl4bIgkgCCAJXhsiCCAHIAheGyIHIAYgB14bIgYgBSAGXhsiBSADIAVeGyAEk0MAAPhBlSIDi0MAAIB3lEMAAIAIlEGAgICIByADvCICQQF0IjhBgICAeHEiNyA3QYCAgIgHTRtBAXZBgICAPGq+krwiN0ENdkGA+AFxIDdB/x9xaiA4QYCAgHhLGyACQRB2QYCAAnFyOwEAAn9DAACAPyADlUMAAAAAIANDAAAAAFwb/RMiJCA2/QACACAE/RMiJv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIl/R8BIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyECAn8gJf0fACIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshPQJ/ICX9HwIiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALITggPf0PIS4CfyAl/R8DIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyE3IC4gAv0XASEvAn8gJCA8/QACACAm/eUB/eYB/QwAAAA/AAAAPwAAAD8AAAA//eQBIiX9HwEiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALIQIgLyA4/RcCIDf9FwMiKP0MDw8PDw8PDw8PDw8PDw8PD/1OIScgOQJ/ICX9HwAiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQAL/Q8gAv0XAQJ/ICX9HwIiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQAL/RcCAn8gJf0fAyIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAv9FwMiLEEE/WsgJ/1Q/VoACAACfyAkIDb9AAIQICb95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiJf0fASIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshAgJ/ICX9HwAiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALIT4CfyAl/R8CIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyE3ID79DyEwAn8gJf0fAyIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshOCAwIAL9FwEhMQJ/ICQgNv0AAlAgJv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIl/R8BIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyECIDEgN/0XAiA4/RcDIin9DA8PDw8PDw8PDw8PDw8PDw/9TiErIDlBDGoCfyAl/R8AIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EAC/0PIAL9FwECfyAl/R8CIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EAC/0XAgJ/ICX9HwMiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQAL/RcDIi1BBP1rICv9UP1aAAAAAn8gJCA2/QACICAm/eUB/eYB/QwAAAA/AAAAPwAAAD8AAAA//eQBIiX9HwEiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALIQICfyAl/R8AIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyE/An8gJf0fAiIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshOCA//Q8hMgJ/ICX9HwMiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALITcgMiAC/RcBITMCfyAkIDb9AAJgICb95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiJf0fASIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshAiAzIDj9FwIgN/0XAyIn/QwPDw8PDw8PDw8PDw8PDw8P/U4hKiA5QRBqAn8gJf0fACIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAv9DyAC/RcBAn8gJf0fAiIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAv9FwICfyAl/R8DIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EAC/0XAyIrQQT9ayAq/VD9WgAAAAJ/ICQgNv0AAjAgJv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIl/R8BIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyECAn8gJf0fACIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAshQAJ/ICX9HwIiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALITcgQP0PITQCfyAl/R8DIgNDAACAT10gA0MAAAAAYHEEQCADqQwBC0EACyE4IDQgAv0XASE1An8gJCA2/QACcCAm/eUB/eYB/QwAAAA/AAAAPwAAAD8AAAA//eQBIiT9HwEiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQALIQIgNSA3/RcCIDj9FwMiJv0MDw8PDw8PDw8PDw8PDw8PD/1OISogOUEUagJ/ICT9HwAiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQAL/Q8gAv0XAQJ/ICT9HwIiA0MAAIBPXSADQwAAAABgcQRAIAOpDAELQQAL/RcCAn8gJP0fAyIDQwAAgE9dIANDAAAAAGBxBEAgA6kMAQtBAAv9FwMiJUEE/WsgKv1Q/VoAAAAgOSAmQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiT9GwBBDHT9ESAk/RsBQQ10/RwBICT9GwJBDnT9HAIgJP0bA0EPdP0cAyAnQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiT9GwBBCHT9ESAk/RsBQQl0/RwBICT9GwJBCnT9HAIgJP0bA0ELdP0cAyApQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiT9GwBBBHT9ESAk/RsBQQV0/RwBICT9GwJBBnT9HAIgJP0bA0EHdP0cAyAoQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiQgJP0bAUEBdP0cASAk/RsCQQJ0/RwCICT9GwNBA3T9HAMgLEEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIk/RsAQRB0/REgJP0bAUERdP0cASAk/RsCQRJ0/RwCICT9GwNBE3T9HAP9UP1QIC1BBP1t/QwBAQEBAQEBAQEBAQEBAQEB/U79iQH9qQEiJP0bAEEUdP0RICT9GwFBFXT9HAEgJP0bAkEWdP0cAiAk/RsDQRd0/RwD/VD9UCArQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiT9GwBBGHT9ESAk/RsBQRl0/RwBICT9GwJBGnT9HAIgJP0bA0EbdP0cA/1Q/VAgJUEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIk/RsAQRx0/REgJP0bAUEddP0cASAk/RsCQR50/RwCICT9GwNBH3T9HAP9UCIkICQgJP0NCAkKCwwNDg8AAQIDAAECA/1QIiQgJCAk/Q0EBQYHAAECAwABAgMAAQID/VD9WgEEACA6QQFqIjogO0cNAAsLC7kDAgR/AXwgAigCBCEDIAIoAgAhACABIAEoAsATIgJBAnRqIgQgASACQY0DakHwBHBBAnRqKAIAIAEgAkEBakHwBHAiAkECdGoiBSgCACIGQf7///8HcSAEKAIAQYCAgIB4cXJBAXZzQQAgBkEBcWtB3+GiyHlxcyIENgIAIAUgAUGNA0GdfiACQeMBSRsgAmpBAnRqKAIAIAEgAkEBaiICQQAgAkHwBEcbIgZBAnRqKAIAIgJB/v///wdxIAUoAgBBgICAgHhxckEBdnNBACACQQFxa0Hf4aLIeXFzIgI2AgAgASAGNgLAEyAAIgEgA0cEQCACQQt2IAJzIgJBB3RBgK2x6XlxIAJzIgJBD3RBgICY/n5xIAJzIgJBEnYgAnO4RAAAAAAAAPBBoiAEQQt2IARzIgJBB3RBgK2x6XlxIAJzIgJBD3RBgICY/n5xIAJzIgJBEnYgAnO4oEQAAAAAAADwO6JEAAAAAAAAAACgIQcgAyABa0EDdSECA0AgASABIAJBAXYiA0EDdGoiAUEIaiAHIAErAwBjIgUbIQEgAyACIANBf3NqIAUbIgINAAsLIAEgAGtBA3UL0wMDCX8BfAF7IwBBEGsiCCQAAkACQCAAKAIAIgIgACgCBCIERg0AAkACQCAEIAJrIgFBCU8EQCABQQN1IQcgAiEBA0AgCiABKwMAoCEKIAFBCGoiASAERw0ACyACIARPDQEgAiEBIAJBf3MgBCACQQhqIgMgAyAESRtqIgNBCE8EQCACIANBA3ZBAWoiBUH+////A3EiCUEDdGohASAK/RQhCwNAIAIgBkEDdGoiAyAD/QADACAL/fMB/QsDACAGQQJqIgYgCUcNAAsgBSAJRg0CCwNAIAEgASsDACAKozkDACABQQhqIgEgBEkNAAsMAQsgACACNgIEIAAoAgggAkYNAiAAQQA2AgggAEIANwIADAELQQAhBiAIQQA2AgwgCEIANwIEQQAhBSAHQQFrIgEEQCABQYCAgIACTw0DIAFBA3QiARAyIgVBACAB/AsAIAEgBWohBgsCQCACIARBCGsiB0YNACAFIAIrAwAiCjkDACACQQhqIgEgB0YNACAFIQMDQCADIAogASsDAKAiCjkDCCADQQhqIQMgAUEIaiIBIAdHDQALCyAAIAY2AgggACAGNgIEIAAgBTYCAAsgAkUNACACEC8LIAhBEGokAA8LEDYAC8QBAQN/IwBBEGsiBCQAAkACQCADKAIAIAMoAgRHBEAgAigCACIFIAIoAgRHDQELIABBADYCCCAAQgA3AgAMAQsgACABIAUgAxC0BCACKAIEIAIoAgBrQQxtIgZBAkkNAEEBIQMDQCAEQQRqIAEgAigCACADQQxsaiAAELQEIAAoAgAiBQRAIAAgBTYCBCAFEC8LIAAgBCgCBDYCACAAIAQoAgg2AgQgACAEKAIMNgIIIANBAWoiAyAGRw0ACwsgBEEQaiQAC+sUARF/IwBB8ABrIgUkACAFIAFBlKgBaigCACIKQTBr/QADAP0LA0AgCkEgayEEIAVB0ABqIQ8CQCAKQRVrLAAAQQBOBEAgDyAEKQMANwMAIA8gBCgCCDYCCAwBCyAPIAQoAgAgCkEcaygCABBrCyAFQQA2AmQgBUIANwJcIApBMGsiCygCICIIIAsoAhwiBmsiBEEwbSEJAkACQAJAIAYgCEYEQEEAIQRBACEGDAELIAlB1qrVKk8NASAFIAQQMiIENgJgIAUgBDYCXCAFIAQgCUEwbGo2AmQgBCEGIAsoAhwiCSALKAIgIghHBEADQCAGIAn9AAMA/QsDACAGIAn9AAMg/QsDICAGIAn9AAMQ/QsDECAGQTBqIQYgCUEwaiIJIAhHDQALCyAFIAY2AmALIAUgCkEIay0AADoAaCAFQQA2AjggBUIANwMwQQEhESAGIARrQQBMDQEgAUGQqAFqIRNBACEJA0ACQCAEIAxBMGwiCmoiCygCACIIIAAoAtwBTg0AAkACQCAAKALUASIGRQ0AA0AgCCAGKAIQIgRIBEAgBigCACIGDQEMAgsgBCAITg0CIAYoAgQiBg0ACwtB8SMQtwEACyAGQRRqIgQoAgAgBCAGLAAfQQBIGyIEEGggCWohCQJAIAxBAEwNACACIAlODQAgAwRAIAQtAABBIEcNAQsgASgClKgBIgRBIGshBiAEQRVrLAAAQQBIBEAgBigCABAvCyAGIAUpAzA3AgAgBiAFKAI4NgIIIAVBADoAOyAFQQA6ADAgASgClKgBIgZBMGsiByALKQMYNwMIAkAgBygCICAHQRxqIggoAgAiBGtBMG0iCSAMSQRAIAggDCAJaxDSBCABKAKUqAEhBgwBCyAJIAxNDQAgByAEIApqNgIgCyAGQQhrQQA6AAAgBUEAOgAoIAVCADcDICAF/QwAAAAAAAAAAAAAAAAAAAAA/QsDECAF/QwAAAAAAAAAAAAAAAAAAAAA/QsDAAJAIAEoApioASAGSwRAIAb9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAYgBSgCGDYCGCAGIAUpAxA3AxAgBUIANwMQIAVBADYCGCAGQQA2AiQgBkIANwIcIAYgBSgCHDYCHCAGIAUoAiA2AiAgBiAFKAIkNgIkIAVBADYCJCAFQgA3AhwgBiAFLQAoOgAoIAEgBkEwajYClKgBDAELIBMgBRDnAiAFKAIcIgRFDQAgBSAENgIgIAQQLwsgBSwAG0EASARAIAUoAhAQLwsgASgClKgBIgZBMGsiCCALKQMYNwMAIAggBSkDSDcDCCAFKAJgIgsgBSgCXCAKaiIHayIEQTBtIRAgBkEUayIOKAIAIg0gCCgCICANa0EwbSIIQTBsaiEKAkAgBEEATA0AIA4oAggiBCAOKAIEIglrQTBtIBBOBEACQCAJIAprIgxBMG0iBCAQTgRAIAkhCCALIQYMAQsgCSEIIAsgByAEQTBsaiIGRwRAIAYhBANAIAggBP0AAwD9CwMAIAggBP0AAyD9CwMgIAggBP0AAxD9CwMQIAhBMGohCCAEQTBqIgQgC0cNAAsLIA4gCDYCBCAMQQBMDQILIAkgCiAIIgQgCiAQQTBsamsiC0EwbUEwbGoiDUsEQANAIAQgDf0AAwD9CwMAIAQgDf0AAyD9CwMgIAQgDf0AAxD9CwMQIARBMGohBCANQTBqIg0gCUkNAAsLIA4gBDYCBCAIIAtBUG1BMGxqIAogC/wKAAAgCiAHIAYgB2v8CgAADAELAkAgCSANa0EwbSAQaiIMQdaq1SpJBEBB1arVKiAEIA1rQTBtIgZBAXQiBCAMIAQgDEsbIAZBqtWqFU8bIhIEfyASQdaq1SpPDQIgEkEwbBAyBUEACyIUIAhBMGxqIgYhCCAHIAtHBEACQCAQQTBsIgxBMGsiC0EwbkEBakEDcSIIRQRAIAYhBAwBC0EAIQ0gBiEEA0AgBCAH/QADAP0LAwAgBCAH/QADIP0LAyAgBCAH/QADEP0LAxAgB0EwaiEHIARBMGohBCANQQFqIg0gCEcNAAsLIAYgDGohCCALQZABTwRAA0AgBCAH/QADAP0LAwAgBCAH/QADIP0LAyAgBCAH/QADEP0LAxAgBEFAayAHQUBr/QADAP0LAwAgBCAH/QADUP0LA1AgBCAH/QADMP0LAzAgBCAH/QADYP0LA2AgBCAH/QADcP0LA3AgBCAH/QADgAH9CwOAASAEIAf9AAOQAf0LA5ABIAQgB/0AA6AB/QsDoAEgBCAH/QADsAH9CwOwASAHQcABaiEHIARBwAFqIgQgCEcNAAsLIA4oAgAhDQsgBiEEIAggCiAKIgcgDUcEfwNAIARBMGsiBCAHQTBrIgf9AAMA/QsDACAEIAf9AAMg/QsDICAEIAf9AAMQ/QsDECAHIA1HDQALIA4oAgAhByAOKAIEBSAJCyAKayIG/AoAACAOIBJBMGwgFGo2AgggDiAENgIAIA4gCCAGQTBtQTBsajYCBCAHBEAgBxAvCwwCCxA2AAsQRwALIAEoApSoAUEIayAFLQBoOgAAAn8gBSwAO0EASARAIAVBADYCNCAFKAIwDAELIAVBADoAOyAFQTBqC0EAOgAAIAUgASgClKgBIgdBMGsiCP0AAwD9CwNAIAggBUFAa0cEQCAHQSBrIQkgB0EVay0AACIGwCEEAkAgBSwAW0EATgRAIARBAE4EQCAPIAkpAwA3AwAgDyAJKAIINgIIDAILIA8gCSgCACAHQRxrKAIAEN4BDAELIA8gCSgCACAJIARBAEgiBBsgB0EcaygCACAGIAQbEN8BCwJAIAgoAiAiCSAIKAIcIgprIghBMG0iCyAFKAJkIgQgBSgCXCIMa0EwbU0EQCAFKAJgIAxrIgZBMG0iBCALSQRAIAwgCiAG/AoAACAFKAJgIQYgCSAKIARBMGxqIgRHBEADQCAGIAT9AAMA/QsDACAGIAT9AAMg/QsDICAGIAT9AAMQ/QsDECAGQTBqIQYgBEEwaiIEIAlHDQALCyAFIAY2AmAMAgsgDCAKIAj8CgAAIAUgDCALQTBsajYCYAwBCyAMBEAgBSAMNgJgIAwQLyAFQQA2AmQgBUIANwJcQQAhBAsCQCALQdaq1SpPDQBB1arVKiAEQTBtIgZBAXQiBCALIAQgC0sbIAZBqtWqFU8bIgRB1qrVKk8NACAFIARBMGwiBBAyIgY2AmAgBSAGNgJcIAUgBCAGajYCZCAFIAkgCkcEfyAGIAogCEEwayIEIARBMHBrQTBqIgT8CgAAIAQgBmoFIAYLNgJgDAELEDYACwsgBSAHQQhrLQAAOgBoIBFBAWohEUEAIQlBfyEMDAELIAVBMGogBBA3GgsgDEEBaiIMIAUoAmAgBSgCXCIEa0EwbUgNAAsMAQsQNgALIAEoApSoASIAQSBrIQEgAEEVaywAAEEASARAIAEoAgAQLwsgASAFKQMwNwIAIAEgBSgCODYCCCAFKAJcIgAEQCAFIAA2AmAgABAvCyAFLABbQQBIBEAgBSgCUBAvCyAFQfAAaiQAIBEL2hIFEn8DfgR9AXsDfCMAQRBrIgskAAJAIAFB0KgBaigCACIPIAEoAsyoASIQRgRAIAtBkBA2AgBBAkGe6AAgCxA0DAELIAEoApCoASACQTBsaiIK/QADACEeAkACQCAKKAIgIAooAhwiCGsiDUEwbSIJDgICAAELIAggHv0LAxgMAQsgDUEASgRAQQEgCSAJQQFMGyERIB79HQEhGSAe/R0AIRcCQANAIAooAhwiAiAHQTBsIgxqIggoAgAhBSAAKAL8ASEGAkAgBw0AIAUgBkYEQCACIBc3AyAgAiAXNwMYIAIgFzcDSCABIAY2AsioASABIBc3A8CoASABIBc3A7ioASAIKAIAIQUgACgC/AEhBgwBCyACIAEpA8CoATcDGAsCQAJAIAAoAtQBIgJFDQAgASkDuKgBIAgoAgQgBmtBAXSsfCEYA0AgBSACKAIQIgZIBEAgAigCACICDQEMAgsgBSAGTA0CIAIoAgQiAg0ACwtB8SMQtwEACyACQRRqIgUoAgAgBSACLAAfQQBIGyIGEGgiAkHw////B0kEQAJAIAJBCk0EQCALIAI6AA8gC0EEaiEFDAELIAJBD3JBAWoiDhAyIQUgCyAOQYCAgIB4cjYCDCALIAU2AgQgCyACNgIICyAFIAYgAvwKAAAgAiAFakEAOgAAAkAgCygCCCALLQAPIgIgAsBBAEgiBhsiBUUEQEMAAAAAIRoMAQtDAAAAACEaIAsoAgQiEiALQQRqIAYbIg4hAiAFQQFxBEBDCtcjPCEaAkACQAJAAkAgDi0AACICQSBrDiADAAICAgICAgICAgIBAgACAgICAgICAgICAgICAgICAAILQwAAQEAhGgwCC0MAAABAIRoMAQtDAABAQEMAAIA/IAJBMGtB/wFxQQpJGyEaCyASIAtBBGogBhtBAWohAgsgBUEBRg0AIAUgDmohBQNAQwrXIzwhG0MK1yM8IRwCQAJAAkACQCACLQAAIgZBIGsOIAMBAgICAgICAgICAgACAQICAgICAgICAgICAgICAgIBAgtDAAAAQCEcDAILQwAAQEAhHAwBC0MAAEBAQwAAgD8gBkEwa0H/AXFBCkkbIRwLIBogHJIhHQJAAkACQAJAIAItAAEiBkEgaw4gAwACAgICAgICAgICAQIAAgICAgICAgICAgICAgICAgACC0MAAEBAIRsMAgtDAAAAQCEbDAELQwAAQEBDAACAPyAGQTBrQf8BcUEKSRshGwsgHSAbkiEaIAJBAmoiAiAFRw0ACwsgCigCHCAMaiAaOAIoIAssAA9BAEgEQCALKAIEEC8LAkAgCCoCECADXkUNACAIKgIUIAReRQ0AIAgoAgQiBSABKALIqAFMDQAgGCAZVQ0AIAooAhwhAiAHBEAgAiAMakEQayAYNwMACyACIAxqIBg3AxggASAFNgLIqAELIAdBAWoiByARRg0CDAELCxBNAAsgCigCHCEICyAPIBBrQQJ1IQ4gCUEwbCAIaiICQRhrIB4gHv0NCAkKCwwNDg8ICQoLDA0OD/0LAwAgAkFAaiAe/VsDAAEgASAe/VsDwKgBASAJQQFrIQpBACEFA0AgBSEGAn9BASAFIAlODQAaQQAgCCAFIgJBMGxqKQMgQgBZDQAaAn8DQCAJIAkgAkEBaiICRg0BGiAIIAJBMGxqKQMgQgBTDQALIAILIQYgAiAJTgshAiAFIAYgAmsiAiIGSARAQQAhB0QAAAAAAAAAACEfIAIgBSICayIMQQFqQQNxIg8EQANAIB8gCCACQTBsaioCKLugIR8gAkEBaiECIAdBAWoiByAPRw0ACwsgDEECSwRAA0AgHyAIIAJBMGxqIgcqAii7oCAHKgJYu6AgByoCiAG7oCAHKgK4AbugIR8gAkEDaiEHIAJBBGohAiAGIAdHDQALCyAIIAZBMGxqKQMgIAggBUEwbGopAxgiF325ISADQAJ+ICAgCCAFQTBsaiICKgIou6IgH6MgF7mgIiGZRAAAAAAAAOBDYwRAICGwDAELQoCAgICAgICAgH8LIRcgAiAXNwNIIAIgFzcDICAFQQFqIgUgBkcNAAsLIAZBAWoiBSAJSA0ACwJAIA1BMU4EQCAIKQMgIhdCAFMEQCAIIBc3A0gLIA1BkAFJDQFBASEFQQEgCiAKQQFMGyEGA0AgCCAFQTBsaiICKQMgIhdCAFMEQCACIBc3A0gLIAJBEGspAwAiGCACKQMYVQRAIAIgGDcDGCACIBggFyAXIBhTGzcDIAsgBUEBaiIFIAZHDQALCyANQQBMDQELQQEgCSAJQQFMGyERIA5BAWshCUEAIQ0DQAJAIAggDUEwbGoiBygCACAAKALcAU4NACAOIAcoAiBBoAFsIgIgCSACIAlIGyICQQAgAkEAShsiBUHQD2oiAiACIA5KGyIPQdAPIAcoAhhBoAFsIgIgCSACIAlIGyIMIAxB0A9MGyIGQdAPayICayEQIAxBAEohEiABKALMqAEhCkMAAAAAIRsCQCACIA9ODQAgDyAGayITQc8PaiEVQQAhBiATQQNxIhMEQANAIBsgCiACQQJ0aioCAJIhGyACQQFqIQIgBkEBaiIGIBNHDQALCyAVQQJNDQADQCAbIAogAkECdGoiBioCAJIgBioCBJIgBioCCJIgBioCDJIhGyACQQRqIgIgD0cNAAsLIAxBACASGyEGIBu7RAAAAAAAAOA/oiAQt6O2IQMCQAJAIA1FDQAgCiAGQQJ0aioCACADXkUNACAGIQICQCAMQQBMDQADQCAKIAJBAnRqKgIAIANeRQ0BIAJBAUohFiACQQFrIQIgFg0AC0EAIQILIAcgB0EQaykDACIXIAJBoAFtrCIYIBcgGFUiDBs3AxggBiACIAwbIQIMAQsDQCAFIAYiAksEQCACQQFqIQYgCiACQQJ0aioCACADXQ0BCwsgByACQaABbq03AxgLAkAgAyAKIAVBAnRqKgIAXQRAAkAgBSAJTg0AA0AgCiAFQQJ0aioCACADXkUNASAFQQFqIgUgCUcNAAsgCSEFCyAHIAVBoAFurSIYNwMgIA0gEEEBa04NAiAYIAcpA0giF1UNAQwCCwNAIAIgBSIGSARAIAVBAWshBSAKIAZBAnRqKgIAIANdDQELCyAGQaABbawhFwsgByAXNwMgCyANQQFqIg0gEUcNAAsLIAtBEGokAAvFAgEHfyABIAAoAggiBCAAKAIEIgJrQTBtTQRAIAAgAQR/IAJBACABQTBsQTBrIgAgAEEwcGtBMGoiAPwLACAAIAJqBSACCzYCBA8LAkAgAiAAKAIAIgVrQTBtIgcgAWoiA0HWqtUqSQRAQdWq1SogBCAFa0EwbSIEQQF0IgggAyADIAhJGyAEQarVqhVPGyIEBEAgBEHWqtUqTw0CIARBMGwQMiEGCyAHQTBsIAZqIgNBACABQTBsQTBrIgEgAUEwcGtBMGoiAfwLACABIANqIQEgAiAFRwRAA0AgA0EwayIDIAJBMGsiAv0AAwD9CwMAIAMgAv0AAyD9CwMgIAMgAv0AAxD9CwMQIAIgBUcNAAsgACgCACECCyAAIAYgBEEwbGo2AgggACABNgIEIAAgAzYCACACBEAgAhAvCw8LEDYACxBHAAvQCwEPfwJAIAIgAWtBDG0iCiAAKAIIIgMgACgCACIGa0EMbU0EQAJAIAEgASAAKAIEIAZrQQxtIg9BDGxqIgggAiAKIA9LGyIQRgRAIAYhAwwBCyAGIQMDQCABIANHBEACQEEAIQ0CQAJAAkAgASgCBCIMIAEoAgAiBWsiCUECdSIHIAMoAggiCyADKAIAIgRrQQJ1TQRAIAMoAgQgBGsiC0ECdSIOIAdJBEAgBCAFIAv8CgAAIAMoAgQhBCAMIAUgDkECdGoiCUYEQCADIAQ2AgQMBgsgDCAFIAtqIgVrQQRrIgdBHEkNAiAEIAVrQRBJDQIgBCAHQQJ2QQFqIg5B/P///wdxIgtBAnQiB2ohBSAHIAlqIQcDQCAEIA1BAnQiEWogCSARav0AAgD9CwIAIA1BBGoiDSALRw0ACyALIA5GDQQMAwsgBCAFIAn8CgAAIAMgBCAHQQJ0ajYCBAwECyAEBEAgAyAENgIEIAQQLyADQQA2AgggA0IANwIAQQAhCwsCQCAJQQBIDQBB/////wMgC0EBdSIEIAcgBCAHSxsgC0H8////B08bIgRBgICAgARPDQAgAyAEQQJ0IgcQMiIENgIEIAMgBDYCACADIAQgB2o2AgggAyAFIAxHBH8gBCAFIAlBBGtBfHFBBGoiBfwKAAAgBCAFagUgBAs2AgQMBAsQNgALIAkhByAEIQULA0AgBSAHKAIANgIAIAVBBGohBSAHQQRqIgcgDEcNAAsLIAMgBTYCBAsLIANBDGohAyABQQxqIgEgEEcNAAsLIAMgBmtBDG0hASAAKAIEIQUgCiAPTQ0BIAIgCEcEQANAIAVBADYCCCAFQgA3AgAgCCgCBCIBIAgoAgAiA0cEQAJAAkACQCABIANrIgNBAE4EQCAFIAMQMiIBNgIEIAUgATYCACAFIAEgA0F8cWo2AgggCCgCACIGIAgoAgQiCUYEQCABIQMMBAsgCSAGa0EEayIDQQxJDQEgASAGa0EQSQ0BIAEgA0ECdkEBaiIMQfz///8HcSIKQQJ0IgRqIQMgBCAGaiEEQQAhBwNAIAEgB0ECdCINaiAGIA1q/QACAP0LAgAgB0EEaiIHIApHDQALIAogDEYNAwwCCxA2AAsgBiEEIAEhAwsDQCADIAQoAgA2AgAgA0EEaiEDIARBBGoiBCAJRw0ACwsgBSADNgIECyAFQQxqIQUgCEEMaiIIIAJHDQALCyAAIAU2AgQPCyAGBEAgACgCBCIEIAYiA0cEQANAIARBDGsiAygCACIFBEAgBEEIayAFNgIAIAUQLwsgAyIEIAZHDQALIAAoAgAhAwsgACAGNgIEIAMQLyAAQQA2AgggAEIANwIAQQAhAwsCQCAKQdaq1aoBTw0AQdWq1aoBIANBDG0iA0EBdCIEIAogBCAKSxsgA0Gq1arVAE8bIgNB1qrVqgFPDQAgACADQQxsIgMQMiIINgIEIAAgCDYCACAAIAMgCGo2AgggASACRwRAA0AgCEEANgIIIAhCADcCACABKAIEIgMgASgCACIFRwRAAkACQAJAIAMgBWsiA0EATgRAIAggAxAyIgY2AgQgCCAGNgIAIAggBiADQXxxajYCCCABKAIEIgkgBUYEQCAGIQMMBAsgCSAFa0EEayIDQQxJDQEgBiAFa0EQSQ0BIAYgA0ECdkEBaiIMQfz///8HcSIKQQJ0IgRqIQMgBCAFaiEEQQAhBwNAIAYgB0ECdCINaiAFIA1q/QACAP0LAgAgB0EEaiIHIApHDQALIAogDEYNAwwCCxA2AAsgBSEEIAYhAwsDQCADIAQoAgA2AgAgA0EEaiEDIARBBGoiBCAJRw0ACwsgCCADNgIECyAIQQxqIQggAUEMaiIBIAJHDQALCyAAIAg2AgQPCxA2AAsgBiABQQxsaiICIAVHBEADQCAFQQxrIgEoAgAiAwRAIAVBCGsgAzYCACADEC8LIAEiBSACRw0ACwsgACACNgIEC9ALAQ9/AkAgAiABa0EMbSIKIAAoAggiAyAAKAIAIgZrQQxtTQRAAkAgASABIAAoAgQgBmtBDG0iD0EMbGoiCCACIAogD0sbIhBGBEAgBiEDDAELIAYhAwNAIAEgA0cEQAJAQQAhDQJAAkACQCABKAIEIgwgASgCACIFayIJQQN1IgcgAygCCCILIAMoAgAiBGtBA3VNBEAgAygCBCAEayILQQN1Ig4gB0kEQCAEIAUgC/wKAAAgAygCBCEEIAwgBSAOQQN0aiIJRgRAIAMgBDYCBAwGCyAMIAUgC2oiBWtBCGsiB0E4SQ0CIAQgBWtBEEkNAiAEIAdBA3ZBAWoiDkH+////A3EiC0EDdCIHaiEFIAcgCWohBwNAIAQgDUEDdCIRaiAJIBFq/QACAP0LAgAgDUECaiINIAtHDQALIAsgDkYNBAwDCyAEIAUgCfwKAAAgAyAEIAdBA3RqNgIEDAQLIAQEQCADIAQ2AgQgBBAvIANBADYCCCADQgA3AgBBACELCwJAIAlBAEgNAEH/////ASALQQJ1IgQgByAEIAdLGyALQfj///8HTxsiBEGAgICAAk8NACADIARBA3QiBxAyIgQ2AgQgAyAENgIAIAMgBCAHajYCCCADIAUgDEcEfyAEIAUgCUEIa0F4cUEIaiIF/AoAACAEIAVqBSAECzYCBAwECxA2AAsgCSEHIAQhBQsDQCAFIAcpAgA3AgAgBUEIaiEFIAdBCGoiByAMRw0ACwsgAyAFNgIECwsgA0EMaiEDIAFBDGoiASAQRw0ACwsgAyAGa0EMbSEBIAAoAgQhBSAKIA9NDQEgAiAIRwRAA0AgBUEANgIIIAVCADcCACAIKAIEIgEgCCgCACIDRwRAAkACQAJAIAEgA2siA0EATgRAIAUgAxAyIgE2AgQgBSABNgIAIAUgASADQXhxajYCCCAIKAIAIgYgCCgCBCIJRgRAIAEhAwwECyAJIAZrQQhrIgNBGEkNASABIAZrQRBJDQEgASADQQN2QQFqIgxB/v///wNxIgpBA3QiBGohAyAEIAZqIQRBACEHA0AgASAHQQN0Ig1qIAYgDWr9AAIA/QsCACAHQQJqIgcgCkcNAAsgCiAMRg0DDAILEDYACyAGIQQgASEDCwNAIAMgBCkCADcCACADQQhqIQMgBEEIaiIEIAlHDQALCyAFIAM2AgQLIAVBDGohBSAIQQxqIgggAkcNAAsLIAAgBTYCBA8LIAYEQCAAKAIEIgQgBiIDRwRAA0AgBEEMayIDKAIAIgUEQCAEQQhrIAU2AgAgBRAvCyADIgQgBkcNAAsgACgCACEDCyAAIAY2AgQgAxAvIABBADYCCCAAQgA3AgBBACEDCwJAIApB1qrVqgFPDQBB1arVqgEgA0EMbSIDQQF0IgQgCiAEIApLGyADQarVqtUATxsiA0HWqtWqAU8NACAAIANBDGwiAxAyIgg2AgQgACAINgIAIAAgAyAIajYCCCABIAJHBEADQCAIQQA2AgggCEIANwIAIAEoAgQiAyABKAIAIgVHBEACQAJAAkAgAyAFayIDQQBOBEAgCCADEDIiBjYCBCAIIAY2AgAgCCAGIANBeHFqNgIIIAEoAgQiCSAFRgRAIAYhAwwECyAJIAVrQQhrIgNBGEkNASAGIAVrQRBJDQEgBiADQQN2QQFqIgxB/v///wNxIgpBA3QiBGohAyAEIAVqIQRBACEHA0AgBiAHQQN0Ig1qIAUgDWr9AAIA/QsCACAHQQJqIgcgCkcNAAsgCiAMRg0DDAILEDYACyAFIQQgBiEDCwNAIAMgBCkCADcCACADQQhqIQMgBEEIaiIEIAlHDQALCyAIIAM2AgQLIAhBDGohCCABQQxqIgEgAkcNAAsLIAAgCDYCBA8LEDYACyAGIAFBDGxqIgIgBUcEQANAIAVBDGsiASgCACIDBEAgBUEIayADNgIAIAMQLwsgASIFIAJHDQALCyAAIAI2AgQL9xoDIH0Wewd/IAJBIE4EQCACQSBtIT4DQCABIDxBFmxqIjtBgPwBIAAgPEEHdGoiOSoCfCIiIDkqAngiBCA5KgJ0IgUgOSoCcCIGIDkqAmwiByA5KgJoIgggOSoCZCIJIDkqAmAiCiA5KgJcIgsgOSoCWCIMIDkqAlQiDSA5KgJQIg4gOSoCTCIPIDkqAkgiECA5KgJEIhEgOUFAayI/KgIAIhIgOSoCPCITIDkqAjgiFCA5KgI0IhUgOSoCMCIWIDkqAiwiFyA5KgIoIhggOSoCJCIZIDkqAiAiGiA5KgIcIhsgOSoCGCIcIDkqAhQiHSA5KgIQIh4gOSoCDCIfIDkqAggiICA5KgIEIiEgOSoCACIDQwAAAAAgA0MAAAAAXSADQwAAAABeciICGyADi0MAAAAAIAIbIgMgIYsiIV0iAhsgISADIAIbIgMgIIsiIF0iAhsgICADIAIbIgMgH4siH10iAhsgHyADIAIbIgMgHosiHl0iAhsgHiADIAIbIgMgHYsiHV0iAhsgHSADIAIbIgMgHIsiHF0iAhsgHCADIAIbIgMgG4siG10iAhsgGyADIAIbIgMgGosiGl0iAhsgGiADIAIbIgMgGYsiGV0iAhsgGSADIAIbIgMgGIsiGF0iAhsgGCADIAIbIgMgF4siF10iAhsgFyADIAIbIgMgFosiFl0iAhsgFiADIAIbIgMgFYsiFV0iAhsgFSADIAIbIgMgFIsiFF0iAhsgFCADIAIbIgMgE4siE10iAhsgEyADIAIbIgMgEosiEl0iAhsgEiADIAIbIgMgEYsiEV0iAhsgESADIAIbIgMgEIsiEF0iAhsgECADIAIbIgMgD4siD10iAhsgDyADIAIbIgMgDosiDl0iAhsgDiADIAIbIgMgDYsiDV0iAhsgDSADIAIbIgMgDIsiDF0iAhsgDCADIAIbIgMgC4siC10iAhsgCyADIAIbIgMgCosiCl0iAhsgCiADIAIbIgMgCYsiCV0iAhsgCSADIAIbIgMgCIsiCF0iAhsgCCADIAIbIgMgB4siB10iAhsgByADIAIbIgMgBosiBl0iAhsgBiADIAIbIgMgBYsiBV0iAhsgBSADIAIbIgMgBIsiBF0iAhsgIosgBCADIAIbXhtDAACAvZQiA4tDAACAd5RDAACACJRBgICAiAcgA7wiAkEBdCI6QYCAgHhxIj0gPUGAgICIB00bQQF2QYCAgDxqvpK8Ij1BDXZBgPgBcSA9Qf8fcWogOkGAgIB4SxsgAkEQdkGAgAJxcjsBAAJ/QwAAgD8gA5VDAAAAACADQwAAAABcG/0TIiMgOf0AAgD95gH9DAAAhEEAAIRBAACEQQAAhEH95AEiJP0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECAn8gJP0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIS0CfyAk/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLITogLSAC/RcBIS4CfyAk/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgLiA6/RcCIS8CfyAjID/9AAIA/eYB/QwAAIRBAACEQQAAhEEAAIRB/eQBIiT9HwEiA4tDAAAAT10EQCADqAwBC0GAgICAeAshOiAvIAL9FwP9DB8fHx8fHx8fHx8fHx8fHx/9diIm/QwPDw8PDw8PDw8PDw8PDw8P/U4hJSA7An8gJP0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIDr9FwECfyAk/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gJP0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MHx8fHx8fHx8fHx8fHx8fH/12IipBBP1rICX9UP1aAAYAAn8gIyA5/QACEP3mAf0MAACEQQAAhEEAAIRBAACEQf3kASIk/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAk/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8hMAJ/ICT9HwIiA4tDAAAAT10EQCADqAwBC0GAgICAeAshOiAwIAL9FwEhMQJ/ICT9HwMiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAxIDr9FwIhMgJ/ICMgOf0AAlD95gH9DAAAhEEAAIRBAACEQQAAhEH95AEiJP0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyE6IDIgAv0XA/0MHx8fHx8fHx8fHx8fHx8fH/12Iif9DA8PDw8PDw8PDw8PDw8PDw/9TiEpIDtBCmoCfyAk/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gOv0XAQJ/ICT9HwIiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwICfyAk/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcD/QwfHx8fHx8fHx8fHx8fHx8f/XYiK0EE/WsgKf1Q/VoAAAACfyAjIDn9AAIg/eYB/QwAAIRBAACEQQAAhEEAAIRB/eQBIiT9HwEiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAgJ/ICT9HwAiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyEzAn8gJP0fAiIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyE6IDMgAv0XASE0An8gJP0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDQgOv0XAiE1An8gIyA5/QACYP3mAf0MAACEQQAAhEEAAIRBAACEQf3kASIk/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLITogNSAC/RcD/QwfHx8fHx8fHx8fHx8fHx8f/XYiJf0MDw8PDw8PDw8PDw8PDw8PD/1OISggO0EOagJ/ICT9HwAiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyA6/RcBAn8gJP0fAiIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAgJ/ICT9HwMiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwP9DB8fHx8fHx8fHx8fHx8fHx/9diIpQQT9ayAo/VD9WgAAAAJ/ICMgOf0AAjD95gH9DAAAhEEAAIRBAACEQQAAhEH95AEiJP0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECAn8gJP0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PITYCfyAk/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLITogNiAC/RcBITcCfyAk/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgNyA6/RcCITgCfyAjIDn9AAJw/eYB/QwAAIRBAACEQQAAhEEAAIRB/eQBIiP9HwEiA4tDAAAAT10EQCADqAwBC0GAgICAeAshOSA4IAL9FwP9DB8fHx8fHx8fHx8fHx8fHx/9diIk/QwPDw8PDw8PDw8PDw8PDw8P/U4hLCA7QRJqAn8gI/0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIDn9FwECfyAj/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gI/0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MHx8fHx8fHx8fHx8fHx8fH/12IihBBP1rICz9UP1aAAAAIDsgJEEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIj/RsAQQx0/REgI/0bAUENdP0cASAj/RsCQQ50/RwCICP9GwNBD3T9HAMgJUEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIj/RsAQQh0/REgI/0bAUEJdP0cASAj/RsCQQp0/RwCICP9GwNBC3T9HAMgJ0EE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIj/RsAQQR0/REgI/0bAUEFdP0cASAj/RsCQQZ0/RwCICP9GwNBB3T9HAMgJkEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIjICP9GwFBAXT9HAEgI/0bAkECdP0cAiAj/RsDQQN0/RwDICpBBP1t/QwBAQEBAQEBAQEBAQEBAQEB/U79iQH9qQEiI/0bAEEQdP0RICP9GwFBEXT9HAEgI/0bAkESdP0cAiAj/RsDQRN0/RwD/VD9UCArQQT9bf0MAQEBAQEBAQEBAQEBAQEBAf1O/YkB/akBIiP9GwBBFHT9ESAj/RsBQRV0/RwBICP9GwJBFnT9HAIgI/0bA0EXdP0cA/1Q/VAgKUEE/W39DAEBAQEBAQEBAQEBAQEBAQH9Tv2JAf2pASIj/RsAQRh0/REgI/0bAUEZdP0cASAj/RsCQRp0/RwCICP9GwNBG3T9HAP9UP1QIChBBP1t/QwBAQEBAQEBAQEBAQEBAQEB/U79iQH9qQEiI/0bAEEcdP0RICP9GwFBHXT9HAEgI/0bAkEedP0cAiAj/RsDQR90/RwD/VAiIyAjICP9DQgJCgsMDQ4PAAECAwABAgP9UCIjICMgI/0NBAUGBwABAgMAAQIDAAECA/1Q/VoBAgAgPEEBaiI8ID5HDQALCwv7AgEKfyAAKAIEIgIgACgCCCIDSQRAIAIgASgCADYCACAAIAJBBGo2AgQPCwJAIAIgACgCACIGayIHQQJ1IghBAWoiBEGAgICABEkEQEH/////AyADIAZrIgNBAXUiBSAEIAQgBUkbIANB/P///wdPGyIDBH8gA0GAgICABE8NAiADQQJ0EDIFQQALIgUgCEECdGoiBCABKAIANgIAIAUgA0ECdGohCCAEQQRqIQkCQCACIAZGDQACQCAHQQRrIgFBLEkNACACIAUgB2prQRBJDQAgBEEQayEHIAJBEGshBSACIAFBAnZBAWoiCkH8////B3EiA0ECdCIBayECIAQgAWshBEEAIQEDQCAHIAFBAnQiC2sgBSALa/0AAgD9CwIAIAFBBGoiASADRw0ACyADIApGDQELA0AgBEEEayIEIAJBBGsiAigCADYCACACIAZHDQALCyAAIAg2AgggACAJNgIEIAAgBDYCACAGBEAgBhAvCw8LEDYACxBHAAuQAwEKfyABIAAoAggiAyAAKAIEIgJrQQJ1TQRAIAAgAQR/IAJBACABQQJ0IgD8CwAgACACagUgAgs2AgQPCwJAIAIgACgCACIFayIHQQJ1IgggAWoiBEGAgICABEkEQEH/////AyADIAVrIgNBAXUiCSAEIAQgCUkbIANB/P///wdPGyIDBEAgA0GAgICABE8NAiADQQJ0EDIhBgsgBiAIQQJ0aiIEQQAgAUECdCIB/AsAIAEgBGohCCAGIANBAnRqIQkCQCACIAVGDQACQCAHQQRrIgFBLEkNACACIAYgB2prQRBJDQAgBEEQayEGIAJBEGshByACIAFBAnZBAWoiCkH8////B3EiA0ECdCIBayECIAQgAWshBEEAIQEDQCAGIAFBAnQiC2sgByALa/0AAgD9CwIAIAFBBGoiASADRw0ACyADIApGDQELA0AgBEEEayIEIAJBBGsiAigCADYCACACIAVHDQALCyAAIAk2AgggACAINgIEIAAgBDYCACAFBEAgBRAvCw8LEDYACxBHAAuPCwEPfyMAQRBrIgokACAKQQA2AgwgCkIANwIEIAIoAgQhAyACKAIAIQsCQAJAIAACfgJAAkAgAS0AACICRQ0AIANBAEwNACADIQcDQCACQcABcUGAAUYEQCACQT9xIAtBBnRyIQsgB0EBayEEIAFBAWohBSABLQABIgJFDQMgB0EBSiEQIAUhASAEIQcgEA0BDAMLC0EEEDIiAUEANgIAIAAgAUEEaiICNgIIIAAgAjYCBCAAIAE2AgBCgICAgHAMAgsgAyEEIAEhBQsCQCADQQBMBEBBACEDDAELQQAhAyAEDQBBBBAyIgYgCzYCACAKIAZBBGoiAzYCDCAKIAM2AgggCiAGNgIECwJAAkAgBS0AACIHRQRAIAMhCAwBCyADIQgDQCAHQQJ2QTxxQZClAWooAgAiBEEBayECIAdBBHZBDHEiCUEIRgRAAkAgBiAISQRAIAZBADYCACAGQQRqIQEMAQtB/////wNBASAIIAZrIgFBAXUiBSAFQQFNGyABQfz///8HTxsiAUGAgICABE8NBiABQQJ0IgEQMiIFQQA2AgAgASAFaiEIIAVBBGohASAGBEAgBhAvCyAFIQYLIAAgCDYCCCAAIAE2AgQgACAGNgIAIAKtQiCGDAQLIAVBAWohASAHQX9BCCAEa3RBf3NxQf8BcSELAkAgBS0AASIHRQRAIAIhBCABIQUMAQsgAiEEIAEhBSAJQQxHDQADQCAHQT9xIAtBBnRyIQsgAUEBaiEFIAJBAWshBCABLQABIgdFDQEgAkEBSiERIAUhASAEIQIgEQ0ACwsCQCAEDQAgAyAIRwRAIAMgCzYCACAKIANBBGoiAzYCCEEAIQQMAQsCQCADIAZrIgJBAnUiB0EBaiIBQYCAgIAESQRAQf////8DIAJBAXUiBCABIAEgBEkbIAJB/P///wdPGyIEBH8gBEGAgICABE8NCCAEQQJ0EDIFQQALIgkgB0ECdGoiASALNgIAIAkgBEECdGohCCABQQRqIQcCQCADIAZGDQACQCACQQRrIgRB/AFJDQAgAiAJaiIJQQRrIgwgA0EEayICIAZrQXxxIg1rIAxLDQAgAiANayACSw0AIAMgCWtBEEkNACABQRBrIQkgA0EQayEMIAMgBEECdkEBaiINQfz///8HcSIEQQJ0IgJrIQMgASACayEBQQAhAgNAIAkgAkECdCIOayAMIA5r/QACAP0LAgAgAkEEaiICIARHDQALIAQgDUYNAQsDQCABQQRrIgEgA0EEayIDKAIANgIAIAMgBkcNAAsLIAogCDYCDCAKIAc2AgggCiABNgIEQQAhBCAGRQ0BIAYQLwwBCxA2AAsgByEDIAEhBgsgBS0AACIHDQALIAMgCE8NACADQQA2AgAgA0EEaiECIAooAgwhBSAKKAIEIQEMAQsgAyAGayICQQJ1IgdBAWoiAUGAgICABE8NA0EAIQVB/////wMgCCAGayIIQQF1IgkgASABIAlJGyAIQfz///8HTxsiAQRAIAFBgICAgARPDQMgAUECdBAyIQULIAUgB0ECdGoiB0EANgIAIAFBAnQhCSAHIQECQCADIAZGDQACQCACQQRrIghBLEkNACADIAIgBWprQRBJDQAgAUEQayEMIANBEGshDSADIAhBAnZBAWoiDkH8////B3EiCEECdCICayEDIAEgAmshAUEAIQIDQCAMIAJBAnQiD2sgDSAPa/0AAgD9CwIAIAJBBGoiAiAIRw0ACyAIIA5GDQELA0AgAUEEayIBIANBBGsiAygCADYCACADIAZHDQALCyAFIAlqIQUgB0EEaiECIAZFDQAgBhAvCyAAIAU2AgggACACNgIEIAAgATYCACALrSAErUIghoQLNwIMIApBEGokAA8LEEcACxA2AAuYBwEOfyMAQSBrIgQkAAJAAkACQAJAIAAQaCIDQfD///8HSQRAAkAgA0EKTQRAIAQgAzoAHyAEQRRqIQEMAQsgA0EPckEBaiICEDIhASAEIAJBgICAgHhyNgIcIAQgATYCFCAEIAM2AhgLIAEgACAD/AoAACABIANqQQA6AAAgBC0AHyIBwCEJQQEhBwJAQcjXNCgCACICRQ0AIAQoAhggASAJQQBIIgEbIQUgBCgCFCAEQRRqIAEbIQoDQAJAAkACQAJAAkACQCACKAIUIAItABsiASABwEEASCIBGyIIIAUgBSAISyIGGyILBEAgCiACKAIQIAJBEGogARsiAyALEEoiAQ0BIAUgCE8NAgwGCyAFIAhPDQIMBQsgAUEASA0ECyADIAogCxBKIgENAQsgBg0BQQAhBwwECyABQQBIDQBBACEHDAMLIAJBBGohAgsgAigCACICDQALCyAJQQBIBEAgBCgCFBAvCyAHBEACQEHE1zQoAgAiAkHI1zRGDQACQAJAAkAgABBoIgZBAWoOAgABAgsDQCACLAArQQBIBEAgAigCJEF/Rg0HCwJAIAIoAgQiAwRAA0AgAyIBKAIAIgMNAAwCCwALA0AgAigCCCIBKAIAIAJHIQwgASECIAwNAAsLIAEiAkHI1zRHDQALDAILA0AgAigCJCACLQArIgEgAcBBAEgbRQ0GAkAgAiIDKAIEIgEEQANAIAEiAigCACIBDQAMAgsACwNAIAMoAggiAigCACADRyENIAIhAyANDQALCyACQcjXNEcNAAsMAQsDQCACKAIkIAItACsiASABwEEASCIDGyAGRgRAIAJBIGoiASgCACABIAMbIAAgBhBKRQ0GCwJAIAIoAgQiAwRAA0AgAyIBKAIAIgMNAAwCCwALA0AgAigCCCIBKAIAIAJHIQ4gASECIA4NAAsLIAEiAkHI1zRHDQALCyAEIAA2AgQgBEGYJDYCAEECQcf5ACAEEDRBfyECDAULIAAQaCIDQfD///8HTw0DAkAgA0EKTQRAIAQgAzoAHyAEQRRqIQEMAQsgA0EPckEBaiICEDIhASAEIAJBgICAgHhyNgIcIAQgATYCFCAEIAM2AhgLIAEgACAD/AoAACABIANqQQA6AABBxNc0IARBFGoQjAIoAgAhAiAELAAfQQBODQQgBCgCFBAvDAQLEE0ACxDsAgALIAIoAhwhAgwBCxBNAAsgBEEgaiQAIAILvg8CH38HfiMAQTBrIggkACABKAJsBEAgAEFAaygCACEYIAAoAjwhDiAAKAI4IQogASgCUCESIAAoAiQhFyABKALYqAEhDSACKAIAIQQgASgC0KcBIgYoAgAtAJgQIQsgASgCVCEMIAYoAgAtAJgQIRkgAUHYpwFqKAIAIQUgAUHUpwFqKAIAIQMgASgCTCEaIAhBAToALCAIIAgoAiw2AiAgCCADNgIoIAggBSADazYCJCAIIAgpAiQ3AxggCEEYahC+ASIDEP0CIRUgBiADQRIgBKwiIhBIIhMQkAEgBigCAC0AmBBFBEAgEyACKAIEQQAgBCATKAIAQSRsQZibAWooAgBsEJcBCyAGIANBEiAiEEgiCRCQAQJAIAYoAgAtAJgQDQAgBEEATA0AQQAhBQNAIAggBUECdCIUIAIoAghqKAIANgIkIAkgCEEkaiAUQQQQlwEgBUEBaiIFIARHDQALCyAGIANBAEIBEEgiFBCQASAGKAIALQCYEEUEQCAIIAqyIA6ylbtEAAAAAAAA0L8QzQK2OAIkIBQgCEEkakEAQQQQlwELIAYgA0EAIBIgDCALGyIMrCImICJCARC8ASILEJABIAYoAgAtAJgQRQRAIAFB+KcBaiEHAkAgBCAMbCIFIAFB/KcBaigCACABKAL4pwEiD2tBAnUiBksEQCAHIAUgBmsQbiAHKAIAIQ8MAQsgBSAGTw0AIAEgDyAFQQJ0ajYC/KcBCyAPQQAgCxBb/AsAAkAgBEEATA0AIAxBAEwNACABKAJYIRsgAigCECEcIAIoAgghHUEAIQIDQCACIAxsIR4gHSACQQJ0IgVqKAIAIR8gBSAcaigCACgCACEWQQAhEANAIBsgEEEEdGoiIEEIaiIRIQYCQAJAIBEoAgAiBUUNAANAIAYgBSAFKAIQIBZIIiEbIQYgBUEEaiAFICEbKAIAIgUNAAsgBiARRg0AIBYgBigCEEgNACAgKAIAIB9MDQELIA8gECAeakECdGpBgICAfDYCAAsgEEEBaiIQIAxHDQALIAJBAWoiAiAERw0ACwsgCyAHKAIAQQAgCykDKCALKQMgIAspAxggCykDEH5+fqdBAnQQlwELIAMgAyAAKAKEASATEIwDIAMgACgCgAEgCRCMAxBPIQUgGEEASgRAIBIgBGsgGiAZGyEMIA0gFyANQQBKGyIPIApsIRAgCiASbCETIAQgCmysISggD6whJyAOrCEjIAqsISVBACEGA0AgACgCnAEhAiADIAMgAyAFIAAqAkwQqgEgAiAGQeAAbGoiAigCABCrASACKAIEEE8hBCADIAMgAyACKAIQIAQQWCACKAIUEE8gFBDOASEHIAMgAyACKAIYIAQQWCAUEM4BIQ0gAyADIAMgAyACKAIcIAQQWCACKAIgEE8gJSAiEOQBEKcCIQQgAyABKAJkIgkgKCAJKAIAQSRsQZibAWooAgAgBiASbCIJIAxqIApsbBCfAyEWIAEoAmgiESgCAEEkbEGYmwFqKAIAIRcgAyARICIgJSASIBdsIAwgESgCAEEkbEGYmwFqKAIAIhFsIBEgCSAKbGxqEKoCIQkgFSADIA0gFhCHARCoASAVIAMgBCAJEIcBEKgBIAMgAyAHIAogDm2sIiQgIyAiEOIBQQBBARCYASEEIAMgAyADIAMgASgCZCIHICQgJiAjIAogBygCAEEkbEGYmwFqKAIAbCAKIAEoAmQoAgBBJGxBmJsBaigCACIHbCAObiAHIAYgE2wiB2wQ4QEgBBBYIAsQTxCjAiEEIAMgAyADIAMgASgCaCINICYgJCAjIBIgDSgCAEEkbEGYmwFqKAIAbCATIAEoAmgoAgBBJGxBmJsBaigCAGwgDm4gByABKAJoKAIAQSRsQZibAWooAgBsEOEBIAQQWEEAQQEQmAEgA0EAICUgIhBVEIcBIQQgAyADIAMgAyADIAMgAigCCCAEEFggAigCDBBPIAUQTyIFIAAqAkwQqgEgAigCJBCrASACKAIoEE8hBCADIAMgAyACKAI0IAQQWCACKAI4EE8gFBDOASEEIAMgASgCjAEiByAkICcgIyAKIAcoAgBBJGxBmJsBaigCAGwgCiAHKAIAQSRsQZibAWooAgAiB2wgDm4gByAGIBBsIgdsEOEBIQ0gAyADIAMgAyABKAKQASIJICcgJCAjIA8gCSgCAEEkbEGYmwFqKAIAbCAQIAkoAgBBJGxBmJsBaigCACIJbCAObiAHIAlsEOEBIAMgAyANIAMgAyAEICQgIyAiEOIBQQBBARCYARBYEKMCEFhBAEEBEJgBIANBACAlICIQVRCHASEEIAMgAyADIAMgAyADIAIoAiwgBBBYIAIoAjAQTyAFEE8iBSAAKgJMEKoBIAIoAkgQqwEgAigCTBBPIQQgAyADIAMgAigCUCAEEFggAigCVBBPEOYBIQQgAyADIAMgAigCWCAEEFggAigCXBBPIAUQTyEFIAZBAWoiAiEGIAIgGEcNAAsLIAMgAyADIAUgACoCTBCqASAAKAKIARCrASAAKAKMARBPIQEgFSADIAAoAoQBIAEQWBCoASADEK0BIAhBMGokACAVDwsgCEHDCTYCCCAIQfYQNgIEIAhBphU2AgBBAkGw5AAgCBA0EAAAC+8DAg1/A34jAEEgayIDJAAgACgCLCEFIAAoAighCSAAKAIkIQogAUHEpwFqKAIAIQcgAUHApwFqKAIAIQIgASgC2KgBIQQgA0EBOgAcIAMgAygCHDYCECADIAI2AhggAyAHIAJrNgIUIAMgAykCFDcDCCADQQhqEL4BIgIQ+gIhByABKAK8pwEhBiACIAEoAuinARDFAyELIAYgAkEAQgEQSCIMEJABIAYoAgAtAJgQRQRAIAMgCbIgBbKVu0QAAAAAAADQvxDNArY4AhQgDCADQRRqQQBBBBCXAQsgAEFAaygCAEEASgRAIAkgBCAKIARBAEobIgZsrCERIAasIQ8gCawhEEEAIQQDQCACIAIgACgCnAEgBEHgAGxqIgUoAjwgCxBYIAwQzgEhCiACIAIgAiACIAUoAkAgCxBYIAUoAkQQTyAQIA8Q5AEQpwIhBSACIAEoAowBIgggESAIKAIAQSRsQZibAWooAgAgBCAGbCAJbCIIbBCfAyEOIAIgASgCkAEiDSAPIBAgBiANKAIAQSRsQZibAWooAgBsIAggDSgCAEEkbEGYmwFqKAIAbBCqAiEIIAcgAiAKIA4QhwEQqAEgByACIAUgCBCHARCoASAEQQFqIgQgACgCQEgNAAsLIAIQrQEgA0EgaiQAIAcLogcCEX8EfiMAQSBrIgYkACAAKAIwIQsgACgCLCEJIAAoAighCiAAKAIkIQcgAUGwpwFqKAIAIQQgAUGspwFqKAIAIQIgASgC2KgBIQMgBkEBOgAcIAYgBigCHDYCECAGIAI2AhggBiAEIAJrNgIUIAYgBikCFDcDCCAGQQhqEL4BIgIQ/QIhDCABKAKopwEhBCACIAEoAuSnARDFAyEFIAQgAkEAQgEQSCINEJABIAQoAgAtAJgQRQRAIAZDAACAPyAKsiAJspWRlTgCFCANIAZBFGpBAEEEEJcBCyAAKAJkIgQoAgBBJGxBmJsBaigCABogAiESIAIgBCAEKQMQIAMgByADQQBKG6wiEyAEIgMoAhAgAygCAEEkbEGYmwFqKAIAbEEAEKoCIREgAiAFEKcCIQNBACEFIwBBEGsiByQAIAMoAogBIRAgAiADKAIAIAMoAgwgA0EQakEAQQAQSSEPIAcgA0HUAWo2AgAgD0HYzwAgBxBzIgRBHTYCQCAQBEAgAiAEKAIAIAQoAgwgBEEQakEAQQAQSSEFCyAEIAM2AowBIAQgBTYCiAEgB0EQaiQAIBIgESAEEE8hBCALQQBKBEAgCqwhFiAJrCEUQQAhBwNAIAAoApABIQMgAiACIAIgBCAAKgJMEKoBIAMgB0E8bGoiAygCABCrASADKAIEEE8hBSACIAIgAygCECAFEFggAygCFBBPIQggAiADKAIYIAUQWCEOIAIgAiADKAIcIAUQWCADKAIgEE8hBSACIAIgCCACQQAgCiAJbawiFSAUIBMQvAEQhwFBAEEBEJgBIQggAiACIAIgAiACIA4gAiAAKAIUIBUgFCATELwBEIcBQQBBARCYASAIEFggDRDOARCjAiEIIAIgAiACIAIgAiACIAUgFSAUIBMQ4gFBAUEAEJgBIAIgACgCFCATIBUgFBC8ARCHASAIEFhBAEEBEJgBIAJBACAWIBMQVRCHASEFIAIgAiACIAIgAiACIAMoAgggBRBYIAMoAgwQTyAEEE8iBCAAKgJMEKoBIAMoAiQQqwEgAygCKBBPIQUgAiACIAIgAygCLCAFEFggAygCMBBPEOYBIQUgAiACIAIgAygCNCAFEFggAygCOBBPIAQQTyEEIAdBAWoiAyEHIAMgC0cNAAsLIAwgAiACIAIgBCAAKgJMEKoBIAAoAngQqwEgACgCfBBPIgAQqAEgASAANgLopwEgAhCtASAGQSBqJAAgDAvzCAIafwF+IwBBIGsiCCQAIAA0AkQhHSAAKAIkIQUgAUGcpwFqKAIAIQYgAUGYpwFqKAIAIQMgASgC2KgBIQQgCEEBOgAcIAggCCgCHDYCECAIIAM2AhggCCAGIANrNgIUIAggCCkCFDcDCCAIQQhqEL4BIgkQ+gIhESABKAKUpwEiAyAJQQAgBCAFIARBAEobIgVBAXQiEqwgHRBVIgYQkAEgAygCAC0AmBBFBEAgAUHspwFqIRACQCAGKQMoIAYpAyAgBikDGCAGKQMQfn5+pyIDIAFB8KcBaigCACABKALspwEiB2tBAnUiBEsEQCAQIAMgBGsQbiAQKAIAIQcMAQsgAyAETw0AIAEgByADQQJ0ajYC8KcBCyAHQQAgBhBb/AsAAkAgASgCpAEiFUEATA0AIAEoApwBIg0gAiASaiIDIAMgDUobIg8gDSACIAIgDUobIgRMDQAgDUECdCEWIAVBA3QhFyABKAKoASILIARBAnRqIRggBCAPIARrIhNBfHEiFGohBSATQQRJIRkDQCANIA5sIQwgDiASbCEKIAQhAwJAAkAgGQ0AQQAhAiAOIBdsIAdqIBggDiAWbGprQRBJDQADQCAHIAIgCmpBAnRqIAsgAiAEaiAMakECdGr9AAIA/QsCACACQQRqIgIgFEcNAAsgBSEDIBMgFEYNAQsgCiAEayEKIA8gA0F/c2ohHEEAIQIgDyADa0EDcSIbBEADQCAHIAMgCmpBAnRqIAsgAyAMakECdGoqAgA4AgAgA0EBaiEDIAJBAWoiAiAbRw0ACwsgHEEDSQ0AA0AgByADIApqQQJ0aiALIAMgDGpBAnRqKgIAOAIAIAcgCiADQQFqIgJqQQJ0aiALIAIgDGpBAnRqKgIAOAIAIAcgCiADQQJqIgJqQQJ0aiALIAIgDGpBAnRqKgIAOAIAIAcgCiADQQNqIgJqQQJ0aiALIAIgDGpBAnRqKgIAOAIAIANBBGoiAyAPRw0ACwsgDkEBaiIOIBVHDQALCyAGIBAoAgBBACAGKQMoIAYpAyAgBikDGCAGKQMQfn5+p0ECdBCXAQsgCSAJIAkgACgCaCAGQQEQhQMgACgCbBBPEOYBIQRBwgohA0HAACECAkACQAJAAkAgCSAJIAkgACgCcCAEQQIQhQMgACgCdBBPEOYBIgRB1AFqIgBBwgpzQQNxDQADQAJAIAAgAy0AACIFOgAAIAVFDQUgAEEBaiEAIAJBAWsiAkEARyEFIANBAWoiA0EDcUUNACACDQELCyAFRQ0CIAMtAABFDQMgAkEESQ0AA0AgAygCACIFQX9zIAVBgYKECGtxQYCBgoR4cQ0CIAAgBTYCACAAQQRqIQAgA0EEaiEDIAJBBGsiAkEDSw0ACwsgAkUNAQsDQCAAIAMtAAAiBToAACAFRQ0CIABBAWohACADQQFqIQMgAkEBayICDQALC0EAIQILIABBACACEKABIARBADoAkwIgASAENgLkpwEgESAEEKgBIAkQrQEgCEEgaiQAIBELugICAX8BfhBpIQcgASgClKcBEJICIAAgASACEN0EIQYgASgClKcBIAYQ2gEgASgCkKcBIgIoAgBBPEYEQCACIAMQkQILIAIgBiACKAIsEQIAIAIoAhwiBgRAIAIgBhEBAAsgASgCqKcBEJICIAAgARDcBCEGIAEoAqinASAGENoBIAEoApCnASICKAIAQTxGBEAgAiADEJECCyACIAYgAigCLBECACACKAIcIgYEQCACIAYRAQALIAEoArynARCSAiAAIAEQ2wQhAiABKAK8pwEgAhDaASABKAKQpwEiACgCAEE8RgRAIAAgAxCRAgsgACACIAAoAiwRAgAgACgCHCICBEAgACACEQEACyABEGkgB30gASkDCHw3AwggASABKAI0QQFqNgI0IAQEfyAFIAQRAABBAXMFQQELC+4QBBt/B3wBfQF+IwBBIGsiDCQAIAxBADYCHCAMQgA3AhQCQAJAIAQEQCAEQYCAgIAETw0BIAwgBEECdCIJEDIiDzYCFCAMIAkgD2oiFzYCHCAPQQAgCfwLACAMIBc2AhgLIAxBADYCECAMQgA3AggCQCAFBEAgBUEBdCILQYCAgIAETw0BIAwgBUEDdCIKEDIiCTYCCCAMIAkgC0ECdGo2AhAgCUEAIAr8CwAgDCAJIApqNgIMCyAIKAIAIgogAyAFbUEBaiIVIAogFUgbIABMDQIgBEEBdiISQQFqIRggBEF+cSEaIARBAXEhGyASQQFqIRwgEkECayEdIAUgBmwiHkECdCEfIAAgBWwiCUECdCEgIAMgCWshEyAEQQZJISEDQAJAIAMgACAFbCIRayIUIAQgBCAUSiIWG0EATA0AIAIoAgAhDSABKAIAIRBBACEJAkAgBCATIAQgE0gbIg5BCEkNACAPIBBrQRBJDQAgDyAgIBkgH2xqIA1qa0EQSQ0AIA5BfHEhCUEAIQsDQCAPIAtBAnQiCmogCiAQav0AAgAgDSALIBFqQQJ0av0AAgD95gH9CwIAIAtBBGoiCyAJRw0ACyAJIA5GDQELIAlBAXIhCiAOQQFxBEAgDyAJQQJ0IgtqIAsgEGoqAgAgDSAJIBFqQQJ0aioCAJQ4AgAgCiEJCyAKIA5GDQADQCAPIAlBAnQiCmogCiAQaioCACANIAkgEWpBAnRqKgIAlDgCACAPIAlBAWoiCkECdCILaiALIBBqKgIAIA0gCiARakECdGoqAgCUOAIAIAlBAmoiCSAORw0ACwsCQCAWRQ0AIBcgDyAUQQJ0aiIKayIJQQBMDQAgCkEAIAlBAnYgCUEDS2tBAnRBBGr8CwALIAxBFGogDEEIahDmAgJAIARFDQBBACEJIAwoAgghC0EAIQogBEEBRwRAA0AgCyAJQQJ0aiALIAlBA3RqIg0qAgAiKyArlCANKgIEIisgK5SSOAIAIAsgCUEBciINQQJ0aiALIA1BA3RqIg0qAgAiKyArlCANKgIEIisgK5SSOAIAIAlBAmohCSAKQQJqIgogGkcNAAsLIBtFDQAgCyAJQQJ0aiALIAlBA3RqIgkqAgAiKyArlCAJKgIEIisgK5SSOAIACyAIKAIIIhFBAEoEQCAIKAIMIRQgCCgCACEWIAcoAgghEEEAIQ0gDCgCCCELA0BEAAAAAAAAAAAhJEEAIQkgIUUEQCANIBhsISIDQCAkIAsgCUECdCIKQQxyaioCACAQIAkgImpBAnRqIg4qAgyUIAsgCkEIcmoqAgAgDioCCJQgCiALaioCACAOKgIAlCALIApBBHJqKgIAIA4qAgSUkpKSu6AhJCAJQQRqIgkgHUgNAAsLAkAgCSASSg0AIA0gGGwhDiAcIAlrQQFxBH8gJCALIAlBAnRqKgIAIBAgCSAOakECdGoqAgCUu6AhJCAJQQFqBSAJCyEKIAkgEkYNAANAICQgCyAKQQJ0aioCACAQIAogDmpBAnRqKgIAlLugIAsgCkEBaiIJQQJ0aioCACAQIAkgDmpBAnRqKgIAlLugISQgCkECaiEKIAkgEkcNAAsLIBQgDSAWbCAAakECdGoCfAJAAkACQAJARLu919nffNs9ICQgJES7vdfZ33zbPWMbIiS9IixCAFkEQCAsQiCIpyIJQf//P0sNAQtEAAAAAAAA8L8gJCAkoqMgLEL///////////8Ag1ANBBogLEIAWQ0BICQgJKFEAAAAAAAAAACjDAQLIAlB//+//wdLDQJBgIDA/wMhCkGBeCEOIAlBgIDA/wNHBEAgCSEKDAILICynDQFEAAAAAAAAAAAMAwsgJEQAAAAAAABQQ6K9IixCIIinIQpBy3chDgsgDiAKQeK+JWoiCUEUdmq3IilEAGCfUBNE0z+iIiUgLEL/////D4MgCUH//z9xQZ7Bmv8Daq1CIIaEv0QAAAAAAADwv6AiJCAkICREAAAAAAAA4D+ioiInob1CgICAgHCDvyIoRAAAIBV7y9s/oiImoCIqICYgJSAqoaAgJCAkRAAAAAAAAABAoKMiJSAnICUgJaIiJiAmoiIlICUgJUSfxnjQCZrDP6JEr3iOHcVxzD+gokQE+peZmZnZP6CiICYgJSAlICVERFI+3xLxwj+iRN4Dy5ZkRsc/oKJEWZMilCRJ0j+gokSTVVVVVVXlP6CioKCiICQgKKEgJ6GgIiREAAAgFXvL2z+iIClENivxEfP+WT2iICQgKKBE1a2ayjiUuz2ioKCgoCEkCyAkC7Y4AgAgDUEBaiINIBFHDQALCyAZQQFqIRkgEyAeayETIAAgBmoiACAIKAIAIgogFSAKIBVIG0gNAAsMAgsQNgALEDYACwJAIAAgCk4NACAIKAIIIgJBAEwNACAIKAIMIQMgCkEBRiACQQNLcSEHIAJBA3EhBSACQXxxIQEDQEEAIQtBACEEQQAhCQJAIAcEQANAIAMgACAEakECdGr9DAAAIMEAACDBAAAgwQAAIMH9CwIAIARBBGoiBCABRw0ACyABIgkgAkYNAQsgAiAJQX9zaiEjIAUEQANAIAMgCSAKbCAAakECdGpBgICAiXw2AgAgCUEBaiEJIAtBAWoiCyAFRw0ACwsgI0EDSQ0AA0AgAyAJIApsIABqQQJ0akGAgICJfDYCACADIAlBAWogCmwgAGpBAnRqQYCAgIl8NgIAIAMgCUECaiAKbCAAakECdGpBgICAiXw2AgAgAyAJQQNqIApsIABqQQJ0akGAgICJfDYCACAJQQRqIgkgAkcNAAsLIAAgBmoiACAKSA0ACwsgDCgCCCIABEAgDCAANgIMIAAQLwsgDwRAIA8QLwsgDEEgaiQAC8UEAQZ/IAAEQCAAKAKoASIBBEAgARCtAQsgACgCrAEiAQRAIAEQpwELIAAoAoACIgEEQCABKAJsIgIEQCACEK0BIAEoAnAQpwEgAUEANgJsCyABKAKUASICBEAgAhCtASABKAKYARCnASABQQA2ApQBCyABKALIASEFIAEoAsQBIQIgASgCwAEhAyABKAK8ASEEIAEoArgBIgYEQCAGEC8LIAQEQCAEEC8LIAMEQCADEC8LIAIEQCACKAIAIgMEQEEAIQQDQCADEC8gAiAEQQFqIgRBAnRqKAIAIgMNAAsLIAIQLwsgBQRAIAUQLwsgASgClKcBIgIEQCACEJEBIAFBpKcBaigCABCnASABQQA2ApSnAQsgASgCqKcBIgIEQCACEJEBIAFBuKcBaigCABCnASABQQA2AqinAQsgASgCvKcBIgIEQCACEJEBIAFBzKcBaigCABCnASABQQA2ArynAQsgASgC0KcBIgIEQCACEJEBIAFB4KcBaigCABCnASABQQA2AtCnAQsgASgCkKcBIgIEQCACIAIoAgQRAQALIAEQ8gIQLwsgACgChAIiAQRAIAEgASgCBBEBAAsgACwAkwJBAEgEQCAAKAKIAhAvCyAAQdABaiAAKALUARCPAiAAQcQBaiAAKALIARCOAiAAQbQBaiAAKAK4ARCNAiAAKAKcASIBBEAgACABNgKgASABEC8LIAAoApABIgEEQCAAIAE2ApQBIAEQLwsgACgCWCIBBEAgACABNgJcIAEQLwsgABAvCwufFgMhfQt/DHsgAkEgTgRAIAJBIG0hKQNAIAEgKEEUbGoiJ0GA/AEgACAoQQd0aiIkKgJ8IgMgJCoCeCIFICQqAnQiBiAkKgJwIgcgJCoCbCIIICQqAmgiCSAkKgJkIgogJCoCYCILICQqAlwiDCAkKgJYIg0gJCoCVCIOICQqAlAiDyAkKgJMIhAgJCoCSCIRICQqAkQiEiAkQUBrIioqAgAiEyAkKgI8IhQgJCoCOCIVICQqAjQiFiAkKgIwIhcgJCoCLCIYICQqAigiGSAkKgIkIhogJCoCICIbICQqAhwiHCAkKgIYIh0gJCoCFCIeICQqAhAiHyAkKgIMIiAgJCoCCCIhICQqAgQiIiAkKgIAIiND//9/fyAjQ///f39dGyIEIAQgIl4bIgQgBCAhXhsiBCAEICBeGyIEIAQgH14bIgQgBCAeXhsiBCAEIB1eGyIEIAQgHF4bIgQgBCAbXhsiBCAEIBpeGyIEIAQgGV4bIgQgBCAYXhsiBCAEIBdeGyIEIAQgFl4bIgQgBCAVXhsiBCAEIBReGyIEIAQgE14bIgQgBCASXhsiBCAEIBFeGyIEIAQgEF4bIgQgBCAPXhsiBCAEIA5eGyIEIAQgDV4bIgQgBCAMXhsiBCAEIAteGyIEIAQgCl4bIgQgBCAJXhsiBCAEIAheGyIEIAQgB14bIgQgBCAGXhsiBCAEIAVeGyIEIAMgBF0bIgSLQwAAgHeUQwAAgAiUQYCAgIgHIAS8IgJBAXQiJkGAgIB4cSIlICVBgICAiAdNG0EBdkGAgIA8ar6SvCIlQQ12QYD4AXEgJUH/H3FqICZBgICAeEsbIAJBEHZBgIACcXI7AQIgJ0GA/AEgAyAFIAYgByAIIAkgCiALIAwgDSAOIA8gECARIBIgEyAUIBUgFiAXIBggGSAaIBsgHCAdIB4gHyAgICEgIiAjQ///f/8gI0P//3//XhsiIyAiICNeGyIiICEgIl4bIiEgICAhXhsiICAfICBeGyIfIB4gH14bIh4gHSAeXhsiHSAcIB1eGyIcIBsgHF4bIhsgGiAbXhsiGiAZIBpeGyIZIBggGV4bIhggFyAYXhsiFyAWIBdeGyIWIBUgFl4bIhUgFCAVXhsiFCATIBReGyITIBIgE14bIhIgESASXhsiESAQIBFeGyIQIA8gEF4bIg8gDiAPXhsiDiANIA5eGyINIAwgDV4bIgwgCyAMXhsiCyAKIAteGyIKIAkgCl4bIgkgCCAJXhsiCCAHIAheGyIHIAYgB14bIgYgBSAGXhsiBSADIAVeGyAEk0MAAHBBlSIDi0MAAIB3lEMAAIAIlEGAgICIByADvCICQQF0IiZBgICAeHEiJSAlQYCAgIgHTRtBAXZBgICAPGq+krwiJUENdkGA+AFxICVB/x9xaiAmQYCAgHhLGyACQRB2QYCAAnFyOwEAAn9DAACAPyADlUMAAAAAIANDAAAAAFwb/RMiMSAk/QACACAE/RMiMv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIv/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAv/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISsCfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISYgK/0PITMCfyAv/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISUgMyAC/RcBITQCfyAxICr9AAIAIDL95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiL/0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDQgJv0XAiAl/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XYhMCAnAn8gL/0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gL/0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MDw8PDw8PDw8PDw8PDw8PD/12QQT9ayAw/VD9WgAEAAJ/IDEgJP0AAhAgMv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIv/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAv/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISwCfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISUgLP0PITUCfyAv/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISYgNSAC/RcBITYCfyAxICT9AAJQIDL95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiL/0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDYgJf0XAiAm/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XYhMCAnQQhqAn8gL/0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gL/0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MDw8PDw8PDw8PDw8PDw8PD/12QQT9ayAw/VD9WgAAAAJ/IDEgJP0AAiAgMv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIv/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAv/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIS0CfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISYgLf0PITcCfyAv/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISUgNyAC/RcBITgCfyAxICT9AAJgIDL95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiL/0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDggJv0XAiAl/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XYhMCAnQQxqAn8gL/0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gL/0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MDw8PDw8PDw8PDw8PDw8PD/12QQT9ayAw/VD9WgAAAAJ/IDEgJP0AAjAgMv3lAf3mAf0MAAAAPwAAAD8AAAA/AAAAP/3kASIv/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAv/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIS4CfyAv/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISUgLv0PITkCfyAv/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISYgOSAC/RcBIToCfyAxICT9AAJwIDL95QH95gH9DAAAAD8AAAA/AAAAPwAAAD/95AEiMf0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDogJf0XAiAm/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XYhMiAnQRBqAn8gMf0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAx/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gMf0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MDw8PDw8PDw8PDw8PDw8PD/12QQT9ayAy/VD9WgAAACAoQQFqIiggKUcNAAsLC5YDAQd/IAACfwJAAkAgASgCBCIERQRAIAFBBGoiBiECDAELIAIoAgAgAiACLQALIgbAQQBIIgUbIQggAigCBCAGIAUbIQUDQAJAAkACQAJAAkAgBCICKAIUIAItABsiBCAEwEEASCIHGyIEIAUgBCAFSSIJGyIGBEAgCCACKAIQIAJBEGogBxsiByAGEEoiCkUEQCAEIAVLDQIMAwsgCkEATg0CDAELIAQgBU0NAgsgAiEGIAIoAgAiBA0EDAULIAcgCCAGEEoiBA0BCyAJDQEMBAsgBEEATg0DCyACKAIEIgQNAAsgAkEEaiEGC0EgEDIiBEEQaiEFAkAgAygCACIDLAALQQBOBEAgBSADKQIANwIAIAUgAygCCDYCCAwBCyAFIAMoAgAgAygCBBBrCyAEIAI2AgggBEIANwIAIARBADYCHCAGIAQ2AgAgBCECIAEoAgAoAgAiAwRAIAEgAzYCACAGKAIAIQILIAEoAgQgAhCkASABIAEoAghBAWo2AghBAQwBCyACIQRBAAs6AAQgACAENgIAC7wJAgp/AX4jAEEgayIKJAAgAiAFNgIEIAJBADYCACACKAIQIgcgAigCDCIGRwRAA0AgB0EMayAHQRBrIgcoAggQlgEgBiAHRw0ACyACKAIMIQcLIAIgBjYCEAJAIAYgB2tBBHUiCCAFSQRAIAUgCGsiCyACKAIUIgggAigCECIHa0EEdU0EQAJAIAtFDQAgByEGIAtBA3EiDARAA0AgBkF/NgIAIAZBCGoiCEIANwIAIAYgCDYCBCAGQRBqIQYgCUEBaiIJIAxHDQALCyALQQR0IAdqIQcgC0EBa0H/////AHFBA0kNAANAIAZBfzYCACAGQX82AhAgBkF/NgIgIAZBfzYCMCAGQQhqIgxCADcCACAGQRhqIghCADcCACAGIAw2AgQgBkEoaiIMQgA3AgAgBiAINgIUIAZBOGoiCEIANwIAIAYgDDYCJCAGIAg2AjQgBkFAayIGIAdHDQALCyACIAc2AhAMAgsCQAJAAkAgByACKAIMIgZrQQR1IgwgC2oiDUGAgICAAUkEQEEAIQdB/////wAgCCAGayIIQQN1IgYgDSAGIA1LGyAIQfD///8HTxsiDQRAIA1BgICAgAFPDQIgDUEEdBAyIQ4LIA4gDEEEdGoiCSEGIAtBA3EiDARAA0AgBkF/NgIAIAZBCGoiCEIANwIAIAYgCDYCBCAGQRBqIQYgB0EBaiIHIAxHDQALCyALQQR0IAlqIQ8gC0EBa0H/////AHFBA08EQANAIAZBfzYCACAGQX82AhAgBkF/NgIgIAZBfzYCMCAGQQhqIghCADcCACAGQRhqIgdCADcCACAGIAg2AgQgBkEoaiIIQgA3AgAgBiAHNgIUIAZBOGoiB0IANwIAIAYgCDYCJCAGIAc2AjQgBkFAayIGIA9HDQALCyAOIA1BBHRqIQsgAigCECIGIAIoAgwiDkYNAgNAIAlBEGsiCSAGQRBrIgYoAgA2AgAgCSAGKAIENgIEIAlBCGoiDSAGQQhqIgwoAgAiCDYCACAJIAYoAgwiBzYCDAJAIAdFBEAgCSANNgIEDAELIAggDTYCCCAGIAw2AgQgBkEANgIIIAZBADYCDAsgBiAORw0ACyACIAs2AhQgAigCECEHIAIgDzYCECACKAIMIQYgAiAJNgIMIAYgB0YNAwNAIAdBDGsgB0EQayIHKAIIEJYBIAYgB0cNAAsMAwsQNgALEEcACyACIAs2AhQgAiAPNgIQIAIgCTYCDAsgBgRAIAYQLwsMAQsgBSAITw0AIAcgBUEEdGoiByAGRwRAA0AgBkEMayAGQRBrIgYoAggQlgEgBiAHRw0ACwsgAiAHNgIQCyAKQQE6ABwgCiAKKAIcNgIQIApBADYCGCAKQfgENgIUIAogCikCFDcDCCACIApBCGoQvgEiBzYCIAJAIAdFBEAgCkHkCzYCAEECQbzoACAKEDQMAQsgAiAHIAQgBawgAawgAKx+fiIQEEg2AhggAiACKAIgIAQgEBBINgIcIAIoAhgQWyACKAIcEFtqIQEgAiADIAMoAggRAAAiACABIAAoAgARBAAiADYCJCAAEMkBIgAgAigCGBCQASAAIAIoAhwQkAEgABCRAQsgCkEgaiQAIAdBAEcL/hQDIH0Lfwt7IAJBIE4EQCACQSBtISgDQCABICZBEmxqIidBgPwBIAAgJkEHdGoiIyoCfCIiICMqAngiBCAjKgJ0IgUgIyoCcCIGICMqAmwiByAjKgJoIgggIyoCZCIJICMqAmAiCiAjKgJcIgsgIyoCWCIMICMqAlQiDSAjKgJQIg4gIyoCTCIPICMqAkgiECAjKgJEIhEgI0FAayIpKgIAIhIgIyoCPCITICMqAjgiFCAjKgI0IhUgIyoCMCIWICMqAiwiFyAjKgIoIhggIyoCJCIZICMqAiAiGiAjKgIcIhsgIyoCGCIcICMqAhQiHSAjKgIQIh4gIyoCDCIfICMqAggiICAjKgIEIiEgIyoCACIDQwAAAAAgA0MAAAAAXSADQwAAAABeciICGyADi0MAAAAAIAIbIgMgIYsiIV0iAhsgISADIAIbIgMgIIsiIF0iAhsgICADIAIbIgMgH4siH10iAhsgHyADIAIbIgMgHosiHl0iAhsgHiADIAIbIgMgHYsiHV0iAhsgHSADIAIbIgMgHIsiHF0iAhsgHCADIAIbIgMgG4siG10iAhsgGyADIAIbIgMgGosiGl0iAhsgGiADIAIbIgMgGYsiGV0iAhsgGSADIAIbIgMgGIsiGF0iAhsgGCADIAIbIgMgF4siF10iAhsgFyADIAIbIgMgFosiFl0iAhsgFiADIAIbIgMgFYsiFV0iAhsgFSADIAIbIgMgFIsiFF0iAhsgFCADIAIbIgMgE4siE10iAhsgEyADIAIbIgMgEosiEl0iAhsgEiADIAIbIgMgEYsiEV0iAhsgESADIAIbIgMgEIsiEF0iAhsgECADIAIbIgMgD4siD10iAhsgDyADIAIbIgMgDosiDl0iAhsgDiADIAIbIgMgDYsiDV0iAhsgDSADIAIbIgMgDIsiDF0iAhsgDCADIAIbIgMgC4siC10iAhsgCyADIAIbIgMgCosiCl0iAhsgCiADIAIbIgMgCYsiCV0iAhsgCSADIAIbIgMgCIsiCF0iAhsgCCADIAIbIgMgB4siB10iAhsgByADIAIbIgMgBosiBl0iAhsgBiADIAIbIgMgBYsiBV0iAhsgBSADIAIbIgMgBIsiBF0iAhsgIosgBCADIAIbXhtDAAAAvpQiA4tDAACAd5RDAACACJRBgICAiAcgA7wiAkEBdCIlQYCAgHhxIiQgJEGAgICIB00bQQF2QYCAgDxqvpK8IiRBDXZBgPgBcSAkQf8fcWogJUGAgIB4SxsgAkEQdkGAgAJxcjsBAAJ/QwAAgD8gA5VDAAAAACADQwAAAABcG/0TIjAgI/0AAgD95gH9DAAACEEAAAhBAAAIQQAACEH95AEiLv0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECAn8gLv0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyEqAn8gLv0fAiIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyElICr9DyExAn8gLv0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyEkIDEgAv0XASEyAn8gMCAp/QACAP3mAf0MAAAIQQAACEEAAAhBAAAIQf3kASIu/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgMiAl/RcCICT9FwP9DA8PDw8PDw8PDw8PDw8PDw/9diEvICcCfyAu/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IC79HwIiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwICfyAu/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XZBBP1rIC/9UP1aAAIAAn8gMCAj/QACEP3mAf0MAAAIQQAACEEAAAhBAAAIQf3kASIu/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAu/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISsCfyAu/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISQgK/0PITMCfyAu/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLISUgMyAC/RcBITQCfyAwICP9AAJQ/eYB/QwAAAhBAAAIQQAACEEAAAhB/eQBIi79HwEiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiA0ICT9FwIgJf0XA/0MDw8PDw8PDw8PDw8PDw8PD/12IS8gJ0EGagJ/IC79HwAiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyAC/RcBAn8gLv0fAiIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAgJ/IC79HwMiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwP9DA8PDw8PDw8PDw8PDw8PDw/9dkEE/WsgL/1Q/VoAAAACfyAwICP9AAIg/eYB/QwAAAhBAAAIQQAACEEAAAhB/eQBIi79HwEiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAgJ/IC79HwAiA4tDAAAAT10EQCADqAwBC0GAgICAeAshLAJ/IC79HwIiA4tDAAAAT10EQCADqAwBC0GAgICAeAshJSAs/Q8hNQJ/IC79HwMiA4tDAAAAT10EQCADqAwBC0GAgICAeAshJCA1IAL9FwEhNgJ/IDAgI/0AAmD95gH9DAAACEEAAAhBAAAIQQAACEH95AEiLv0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIDYgJf0XAiAk/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XYhLyAnQQpqAn8gLv0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAu/R8CIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gLv0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XA/0MDw8PDw8PDw8PDw8PDw8PD/12QQT9ayAv/VD9WgAAAAJ/IDAgI/0AAjD95gH9DAAACEEAAAhBAAAIQQAACEH95AEiLv0fASIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECAn8gLv0fACIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyEtAn8gLv0fAiIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyEkIC39DyE3An8gLv0fAyIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyElIDcgAv0XASE4An8gMCAj/QACcP3mAf0MAAAIQQAACEEAAAhBAAAIQf3kASIw/R8BIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgOCAk/RcCICX9FwP9DA8PDw8PDw8PDw8PDw8PDw/9diEuICdBDmoCfyAw/R8AIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IDD9HwIiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwICfyAw/R8DIgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcD/QwPDw8PDw8PDw8PDw8PDw8P/XZBBP1rIC79UP1aAAAAICZBAWoiJiAoRw0ACwsLWQECf0EMEDsiAUEANgIIIAFCBDcCAEE4EDsiACABNgI0IABB8KQBKAIANgIwIABB4KQB/QADAP0LAiAgAEHQpAH9AAMA/QsCECAAQcCkAf0AAwD9CwIAIAAL9AEBAn8jAEEwayIEJAACQCAAKALQAUUEQEG8wwIoAgAQMBogBEGc3wA2AgggBEGZATYCBCAEQZsnNgIAQbjDAigCAEHL5AAgBBAxDAELIAAoAghFBEBBvMMCKAIAEDAaIARB1NwANgIYIARBmgE2AhQgBEGbJzYCEEG4wwIoAgBBy+QAIARBEGoQMQwBCyAAEFsgAiADakkEQEG8wwIoAgAQMBogBEGK3gA2AiggBEGbATYCJCAEQZsnNgIgQbjDAigCAEHL5AAgBEEgahAxDAELIAAoAggiBSAAIAEgAiADIAUoAhARCwAgBEEwaiQADwsQAAALIQEBfyAAKAIcIgAoAggiAgRAIAAgASACEQQADwsgARBbC1sBAX8jAEEQayIBJAAgACAAKAIEEQAAIgBFBEBBvMMCKAIAEDAaIAFBy98ANgIIIAFBzQA2AgQgAUGbJzYCAEG4wwIoAgBBy+QAIAEQMRAAAAsgAUEQaiQAIAAL+gUBD38jAEFAaiIDJAACQCABKALIAQRAQbzDAigCABAwGiADQeDTADYCOCADQdMANgI0IANB2ic2AjBBuMMCKAIAQcvkACADQTBqEDEMAQsgASgC0AEEQEG8wwIoAgAQMBogA0GpLzYCKCADQdQANgIkIANB2ic2AiBBuMMCKAIAQcvkACADQSBqEDEMAQsgACgCACABEOcEIgYgACgCDCICIAYgAnBrIAJwaiEJQX8hAiAAKAIQIgRBAWshBgJAAkACQCAEQQJIDQAgBkEBcSEQAkAgBEECRgRAQX8hCEEAIQQMAQsgBkF+cSENIABBFGohC0F/IQhBACEEA0AgBSALIARBA3RqKAIEIgcgBSAHSxsiBSALIARBAXIiDkEDdGooAgQiCiAFIApLGyEFIAIgBCAHIAlJIAcgCEtyIgIbIA4gCiAIIAcgAhsiCEsgCSAKS3IiBxshAiAIIAogBxshCCAEQQJqIQQgD0ECaiIPIA1HDQALCyAQBEAgBSAAIARBA3RqKAIYIgcgBSAHSxshBSACIAIgBCAHIAhLGyAHIAlJGyECCyACQX9GDQAgACACQQN0aigCGCEEDAELIAAgBkEDdGooAhgiBCAJSQ0BIAYhAgsgACACQQN0aiIFIAQgCWs2AhggBSAFKAIUIgUgCWoiCDYCFAJAIAQgCUcNACAAIAY2AhAgAiAGTg0AIABBFGohBgNAIAYgAkEDdGogBiACQQFqIgJBA3RqKQIANwIAIAIgACgCEEgNAAsLIAEgBTYC0AEgASAAKAIAIgI2AgggAC0AmBBFBEAgAigCCCIGBEAgAiABIAYRAgALCyAAIAAoApQQIgEgCCAAKAIIayIAIAAgAUkbNgKUECADQUBrJAAPCyADIAk2AhQgA0GUJjYCECADIAUgBCAEIAVJGzYCGEG4wwIoAgAiAEGh8gAgA0EQahAxQbzDAigCABAwGiADQfreADYCCCADQfQANgIEIANB2ic2AgAgAEHL5AAgAxAxCxAAAAvaCQQFfwl7An0CfCABIANqIQkCQCAAQXBxIgdBAEwEQAwBCwNAIAsgBCAIQQF0IgVqIgEvAR5BAnRBkNYEaiABLwEcQQJ0QZDWBGogAS8BGkECdEGQ1gRqIAEvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMiDCAFIAlqIgYvAR5BAnRBkNYEaiAGLwEcQQJ0QZDWBGogBi8BGkECdEGQ1gRqIAYvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhCyAPIAwgAyAFaiIFLwEeQQJ0QZDWBGogBS8BHEECdEGQ1gRqIAUvARpBAnRBkNYEaiAFLwEYQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIQ8gCiABLwEWQQJ0QZDWBGogAS8BFEECdEGQ1gRqIAEvARJBAnRBkNYEaiABLwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIgwgBi8BFkECdEGQ1gRqIAYvARRBAnRBkNYEaiAGLwESQQJ0QZDWBGogBi8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASEKIBAgDCAFLwEWQQJ0QZDWBGogBS8BFEECdEGQ1gRqIAUvARJBAnRBkNYEaiAFLwEQQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIRAgDSABLwEOQQJ0QZDWBGogAS8BDEECdEGQ1gRqIAEvAQpBAnRBkNYEaiABLwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIgwgBi8BDkECdEGQ1gRqIAYvAQxBAnRBkNYEaiAGLwEKQQJ0QZDWBGogBi8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASENIBEgDCAFLwEOQQJ0QZDWBGogBS8BDEECdEGQ1gRqIAUvAQpBAnRBkNYEaiAFLwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIREgDiABLwEGQQJ0QZDWBGogAS8BBEECdEGQ1gRqIAEvAQJBAnRBkNYEaiABLwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIgwgBi8BBkECdEGQ1gRqIAYvAQRBAnRBkNYEaiAGLwECQQJ0QZDWBGogBi8BAEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASEOIBIgDCAFLwEGQQJ0QZDWBGogBS8BBEECdEGQ1gRqIAUvAQJBAnRBkNYEaiAFLwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIRIgCEEQaiIIIAdIDQALIA4gCv3kASANIAv95AH95AEhCiASIBD95AEgESAP/eQB/eQBIQsLIAr9HwMgCv0fAiAK/R8AIAr9HwGSkpIhEyAL/R8DIAv9HwIgC/0fACAL/R8BkpKSIRQgAiAAIAdKBH0gFLshFSATuyEWA0AgFiAJIAdBAXQiAWovAQBBAnRBkNYEaioCACABIARqLwEAQQJ0QZDWBGoqAgAiE5S7oCEWIBUgASADai8BAEECdEGQ1gRqKgIAIBOUu6AhFSAHQQFqIgcgAEcNAAsgFbYhFCAWtgUgEws4AgQgAiAUOAIAC+MXAxF9DH4dfyMAQTBrIiMkAAJAAkAgACgCAA4DAQABAAsCQCADKAIwQQJGBEAgAykDECIcIAMoAkgiJ6wiGFkEQCAnQQFxRQRAIAMoAjwhMyADKAI4ITQgAygCNCE1IAEoAjwhNiABKAI4ITcgASgCNCE4IAMqAmwhBiADKgJkIQ8gAyoCYCEKIAMqAlwhCyADKgJYIQUgAygCUCE9IAMoAkwhKCABKAIwITkgACgCBCEsIAMpAygiHiADKQMgIh8gAykDGCIafn6nIiogACgCCCIAakEBayAAbSEAIAMoAlSyIgcgAyoCaCIIIAiSQ9sPSUCUlRBXIQkgBRBXIQggAigC0AEhOiAFQwAAAMAgJ7IiBZUQtAEhEiAeQgBXDQQgByAGIAaSQ9sPSUCUlRBXIQYgJ0ECbSEtQwAAgD8gC5UQVyEHIB9CAFcNBCAaQgBXDQQgACAsbCIsIABqIgAgKiAAICpIGyEqQwAAgD9DAACAvyAEGyETIChBBHFFDQMgHEIEfyEdID1BAmusIRkgLCAqICogLEgbISsgGqchLkEAIQAgLUEGbCEoA0AgMyAbpyICbCEvIAIgNmwhMEIAIRcDQEIAIRYCQCAcQgRTBEAgACEEA0AgBEEBaiEAIAQgK04NAiAAIQQgFkIBfCIWIBpTDQALDAELIDogF6ciAkECdGo0AgAiFiAZIBYgGVMbtCEFIBYgGX0iFkIAIBZCAFUbtCEPIAIgNGwhIiACIDdsISUgACAuaiE+QgAhGANAIABBAWohAiAAICxOBEAgACAqTgRAIAIhAAwDCyADKALQASAvaiAiaiA1IBinIgBsaiEmIAEoAtABIDBqICVqIAAgOGxqITtCACEWIAUhByAPIQYDQCA7IDkgFqciJGxqIgAgLUEBdCIpai8BAEECdEGQ1gRqKgIAIQggBxB6IQ0gAC8BAEECdEGQ1gRqKgIAIQkgBxBqIQogACAnQQF0IjFqLwEAQQJ0QZDWBGoqAgAhCyAAIChqLwEAQQJ0QZDWBGoqAgAhDCAmICRBAXRqIgBBgPwBIAkgCpQgCCATIA2UIg2UkyIOi0MAAIB3lEMAAIAIlEGAgICIByAOvCIkQQF0IjJBgICAeHEiPCA8QYCAgIgHTRtBAXZBgICAPGq+krwiPEENdkGA+AFxIDxB/x9xaiAyQYCAgHhLGyAkQRB2QYCAAnFyOwEAIAAgKWpBgPwBIAkgDZQgCiAIlJIiCItDAACAd5RDAACACJRBgICAiAcgCLwiJEEBdCIpQYCAgHhxIjIgMkGAgICIB00bQQF2QYCAgDxqvpK8IjJBDXZBgPgBcSAyQf8fcWogKUGAgIB4SxsgJEEQdkGAgAJxcjsBACAGEHohCCAAIDFqQYD8ASALIAYQaiIJlCAMIBMgCJQiCJSTIgqLQwAAgHeUQwAAgAiUQYCAgIgHIAq8IiRBAXQiKUGAgIB4cSIxIDFBgICAiAdNG0EBdkGAgIA8ar6SvCIxQQ12QYD4AXEgMUH/H3FqIClBgICAeEsbICRBEHZBgIACcXI7AQAgACAoakGA/AEgCyAIlCAJIAyUkiIIi0MAAIB3lEMAAIAIlEGAgICIByAIvCIAQQF0IiRBgICAeHEiKSApQYCAgIgHTRtBAXZBgICAPGq+krwiKUENdkGA+AFxIClB/x9xaiAkQYCAgHhLGyAAQRB2QYCAAnFyOwEAIBIgBpQhBiASIAeUIQcgFkIBfCIWIB1SDQALCyACIQAgGEIBfCIYIBpSDQALID4hAAsgF0IBfCIXIB9SDQALIBtCAXwiGyAeUg0ACwwEC0G8wwIoAgAQMBogI0GJwgA2AhggI0HN2QA2AhQgI0HkJjYCEEG4wwIoAgBBy+QAICNBEGoQMRAAAAtBvMMCKAIAEDAaICNBlMAANgIIICNBzNkANgIEICNB5CY2AgBBuMMCKAIAQcvkACAjEDEQAAALQbzDAigCABAwGiAjQcPSADYCKCAjQcXZADYCJCAjQeQmNgIgQbjDAigCAEHL5AAgI0EgahAxEAAAC0NvEoM6ICdBAWuyIgwgBiAFlCAIIAiSIgaVjSIIIAggDF4bQwAAAAAgCSAFlCAGlY4iBiAGQwAAAABdGyIMkyIGIAZDbxKDOl0bIQ0gDyAHQ83MzD2UQwAAgD+SlCEIIChBAnFFBEAgHEIAVw0BIBqnISdBACEAA0AgMyAXpyICbCEtIAIgNmwhKEIAIRgDQCAAICdqIQQgNCAYpyICbCErIAIgN2whLiA6IAJBAnRqKAIAsiEJQgAhGQJ/A0AgAEEBaiECIAAgLE4EQCACIAAgKk4NAhogAygC0AEgLWogK2ogNSAZpyIAbGohLyABKALQASAoaiAuaiAAIDhsaiEwQgAhFiAJIQYDQCALIAaUIQcgFqchACAKQwAAAABbBH0gDwUgB0MAAIA/IApDAAAAAEMAAIA/QwAAAAAgAEEBdbIgDJMgDZUiBSAFQwAAAABdGyIFkyAFQwAAgD9eG5QiBZOUIAYgBZSSIQcgCAshBSAHEGohECAwIAAgOWxqIiIvAQJBAnRBkNYEaioCACEOIAcQeiEHIC8gAEEBdGoiAEGA/AEgIi8BAEECdEGQ1gRqKgIAIhEgEyAHIAWUlCIHlCAOIBAgBZQiBZSSIhCLQwAAgHeUQwAAgAiUQYCAgIgHIBC8IiJBAXQiJUGAgIB4cSImICZBgICAiAdNG0EBdkGAgIA8ar6SvCImQQ12QYD4AXEgJkH/H3FqICVBgICAeEsbICJBEHZBgIACcXI7AQIgAEGA/AEgESAFlCAHIA6UkyIFi0MAAIB3lEMAAIAIlEGAgICIByAFvCIAQQF0IiJBgICAeHEiJSAlQYCAgIgHTRtBAXZBgICAPGq+krwiJUENdkGA+AFxICVB/x9xaiAiQYCAgHhLGyAAQRB2QYCAAnFyOwEAIBIgBpQhBiAWQgJ8IhYgHFMNAAsLIAIhACAZQgF8IhkgGlINAAsgBAshACAYQgF8IhggH1INAAsgHiAXQgF8IhdSDQALDAELICdBAEwNAEMAAIC/IAWVIQ4gGqchJ0EAIQADQCAzIBunIgJsISggAiA2bCErQgAhHQNAIAAgJ2ohBCA0IB2nIgJsIS4gAiA3bCEvIAsgOiACQQJ0aigCALKUIQlCACEZAkADQCAAIgJBAWohAAJAIAIgLEgNACACICpODQIgHCAYfyIgQgBXDQAgAygC0AEgKGogLmogNSAZpyICbGohMCABKALQASAraiAvaiACIDhsaiECQgAhFyAJIQYDQCAXIBh+ISEgF7SMIRBCACEWA0AgCyAGlCEHIApDAAAAAFsEfSAPBSAHQwAAgD8gCkMAAAAAQwAAgD9DAAAAAAJ+IA4gFrSUIBCSIgWLQwAAAF9dBEAgBa4MAQtCgICAgICAgICAfwunQQJtsiAMkyANlSIFIAVDAAAAAF0bIgWTIAVDAACAP14blCIFk5QgBiAFlJIhByAICyEFIAcQaiERIDAgFkIBiCAhfKciIkEBdGoiJUGA/AEgAiAiIDlsaiIiLwEAQQJ0QZDWBGoqAgAiFCARIAWUIhGUICIgLUEBdCImai8BAEECdEGQ1gRqKgIAIhUgEyAHEHogBZSUIgWUkyIHi0MAAIB3lEMAAIAIlEGAgICIByAHvCIiQQF0IjtBgICAeHEiJCAkQYCAgIgHTRtBAXZBgICAPGq+krwiJEENdkGA+AFxICRB/x9xaiA7QYCAgHhLGyAiQRB2QYCAAnFyOwEAICUgJmpBgPwBIBQgBZQgESAVlJIiBYtDAACAd5RDAACACJRBgICAiAcgBbwiIkEBdCIlQYCAgHhxIiYgJkGAgICIB00bQQF2QYCAgDxqvpK8IiZBDXZBgPgBcSAmQf8fcWogJUGAgIB4SxsgIkEQdkGAgAJxcjsBACASIAaUIQYgFkICfCIWIBhTDQALIBdCAXwiFyAgUg0ACwsgGUIBfCIZIBpSDQALIAQhAAsgHUIBfCIdIB9SDQALIBtCAXwiGyAeUg0ACwsgI0EwaiQAC5QSAxN9DH4YfyMAQTBrIiUkAAJAAkAgACgCAA4DAQABAAsCQAJAIAEoAjBBBEYEQCADKQMQIhwgAygCSCInrCIaWQRAICdBAXFFBEAgAygCPCEzIAMoAjghNCADKAI0ITUgASgCPCE2IAEoAjghNyABKAI0ITggAy0AdCEoIAMqAnAhCCADKgJsIQYgAyoCZCEOIAMqAmAhCiADKgJcIQkgAyoCWCEHIAMoAlAhKSADKAJMISQgAygCMCEyIAAoAgQhLSADKQMoIh8gAykDICIgIAMpAxgiHX5+pyIrIAAoAggiAGpBAWsgAG0hACADKAJUsiILIAMqAmgiBSAFkkPbD0lAlJUQVyEMIAcQVyEFIAIoAtABITkgB0MAAADAICeyIgeVELQBIREgH0IAVw0FIAsgBiAGkkPbD0lAlJUQVyEGIBxCBH8hHiAnQQJtISxDAACAPyAJlRBXIQsgIEIAVw0FIB1CAFcNBSAAIC1sIi0gAGoiACArIAAgK0gbIStDAACAP0MAAIC/IAQbIRIgJEEEcQ0DQ28SgzogJ0EBa7IiDyAGIAeUIAUgBZIiBpWNIgUgBSAPXhtDAAAAACAMIAeUIAaVjiIGIAZDAAAAAF0bIg+TIgYgBkNvEoM6XRshFCAOIAtDzczMPZRDAACAP5KUIQsgJEECcUUNBEMAAIC/IAeVIQwgHachKEIAIR5BACEAA0AgMyAepyICbCEpIAIgNmwhLkIAISEDQCAAIChqIQQgNCAhpyICbCEvIAIgN2whKiAJIDkgAkECdGooAgCylCEGQgAhIgJAA0AgACICQQFqIQACQCACIC1IDQAgAiArTg0CIBwgGn8iI0IAVw0AICdBAEwNACADKALQASApaiAvaiA1ICKnIiRsaiECIAEoAtABIC5qICpqICQgOGxqISRCACEZIAYiByEFIApDAAAAAFsEQANAIBkgGn4hG0IAIRgDQCAJIAeUIgUQaiEIIAIgMiAYQgGIIBt8pyImbGoiMCAkICZBAnRqIiYqAgAiDSAIIA6UIgiUICYgLEECdCIxaioCACIQIBIgBRB6IA6UlCIFlJM4AgAgMCAxaiANIAWUIAggEJSSOAIAIBEgB5QhByAYQgJ8IhggGlMNAAsgIyAZQgF8IhlSDQAMAgsACwNAIBkgGn4hGyAZtIwhB0IAIRgDQCAJIAWUQwAAgD8gCkMAAAAAQwAAgD9DAAAAAAJ+IAwgGLSUIAeSIgiLQwAAAF9dBEAgCK4MAQtCgICAgICAgICAfwunQQJtsiAPkyAUlSIIIAhDAAAAAF0bIgiTIAhDAACAP14blCIIk5QgBSAIlJIiCBBqIQ0gAiAyIBhCAYggG3ynIiZsaiIwICQgJkECdGoiJioCACIQIA0gC5QiDZQgEiAIEHogC5SUIgggJiAsQQJ0IjFqKgIAIhOUkzgCACAwIDFqIBAgCJQgDSATlJI4AgAgESAFlCEFIBhCAnwiGCAaUw0ACyAZQgF8IhkgI1INAAsLICJCAXwiIiAdUg0ACyAEIQALICFCAXwiISAgUg0ACyAeQgF8Ih4gH1INAAsMBQtBvMMCKAIAEDAaICVBicIANgIYICVBsNgANgIUICVB5CY2AhBBuMMCKAIAQcvkACAlQRBqEDEQAAALQbzDAigCABAwGiAlQZTAADYCCCAlQa/YADYCBCAlQeQmNgIAQbjDAigCAEHL5AAgJRAxEAAAC0G8wwIoAgAQMBogJUG20QA2AiggJUGo2AA2AiQgJUHkJjYCIEG4wwIoAgBBy+QAICVBIGoQMRAAAAsgHEIEUw0BIClBAmusIRwgHachKUEAIQAgLEEMbCEoA0AgMyAbpyICbCEuIAIgNmwhL0IAIRkDQCA5IBmnIgJBAnRqNAIAIhggHCAYIBxTG7QhBkIAIRogGCAcfSIYQgAgGEIAVRu0IQ4gACApaiEEIAIgNGwhKiACIDdsISYCfwNAIABBAWohAiAAIC1OBEAgAiAAICtODQIaIAMoAtABIC5qICpqIDUgGqciAGxqITAgASgC0AEgL2ogJmogACA4bGohMUIAIRggBiEFIA4hBwNAIDEgGKciJEECdGoiACAnQQJ0IjpqKgIAIQsgACAoaioCACEMIAAgLEECdCI7aioCACEJIAUQeiEKIDAgJCAybGoiJCAAKgIAIgggBRBqIg+UIAkgEiAKlCIKlJM4AgAgJCA7aiAIIAqUIA8gCZSSOAIAIAcQeiEJICQgOmogCyAHEGoiCpQgDCASIAmUIgmUkzgCACAkIChqIAsgCZQgCiAMlJI4AgAgESAHlCEHIBEgBZQhBSAYQgF8IhggHlINAAsLIAIhACAaQgF8IhogHVINAAsgBAshACAZQgF8IhkgIFINAAsgG0IBfCIbIB9SDQALDAELIBxCAFcNACAoQQFxIScgHLQiB0MzM7M/lCETIB2nISxBACEAIAdDzczMPpQhFQNAIDMgGaciAmwhJCACIDZsIShCACEaA0AgACAsaiEEIDQgGqciAmwhKSACIDdsIS4gOSACQQJ0aigCALIiDCAIlSEWQgAhGwJ/A0AgAEEBaiECIAAgLU4EQCACIAAgK04NAhogAygC0AEgJGogKWogNSAbpyIAbGohLyABKALQASAoaiAuaiAAIDhsaiEAQgAhGCAMIQcDQCAJIAeUIQUgCkMAAAAAWwR9IA4FIAVDAACAPyAKQwAAAABDAACAP0MAAAAAIBinQQF1siAPkyAUlSIGIAZDAAAAAF0bIgaTIAZDAACAP14blCIGk5QgByAGlJIhBSALCyEGIAUQaiAGlCENIBIgBRB6IAaUlCEQIAhDAAAAAFsEfUMAAIA/BSAVIBi0kiATlSAWELQBCyEFIC8gMiAYpyIqbGoiJiAQIAAgKkECdGoiKioCACIXlEMAAIA/IAWVIAUgJxsiBpQgBiANICoqAgQiBZSUkjgCBCAmIA0gF5QgBpQgBiAQIAWUlJM4AgAgESAHlCEHIBhCAnwiGCAcUw0ACwsgAiEAIBtCAXwiGyAdUg0ACyAECyEAIBpCAXwiGiAgUg0ACyAZQgF8IhkgH1INAAsLICVBMGokAAueCAIOfwx+IwBB0ABrIgQkAAJAAkACQAJAIAIoAkQiBkEATgRAIAAoAgghECAAKAIEIQ8CQCABKALQASIHIAIoAtABIglHBEAgACgCACIFDQEgAikDKCIVIAIpAyAiEiACKQMYIhMgAikDECIWfn5+IAEpAyggASkDICIZIAEpAxgiGiABKQMQIhR+fn5SDQMgAigCMCIFIAIoAgAiCkEkbCIIQZibAWooAgBHDQQgAigCNCILrSIXIBYgBa1+IhsgCEGUmwFqKAIAIgisf1INBCACKAI4IgytIhggEyAXflINBCACKAI8Ig2tIhwgEiAYflINBCABKAIwIg4gASgCAEEkbCIRQZibAWooAgBHDQQgATUCNCIdIBQgDq1+IBFBlJsBajQCAH9SDQQgATUCOCIUIBogHX5SDQQgATUCPCAUIBl+Ug0EIAkgByAKQRBrQXFNBH8gBSAWp2wgCyATp0EBa2xqIAwgEqdBAWtsaiANIBWnQQFrbGoFIBNCAX0gF34gGyAIrX98IBJCAX0gGH58IBVCAX0gHH58pwv8CgAACyAAKAIAIQULAkACQCAFDgMBAAEACyABKQMoIAEpAxgiEiABKQMgfn6nIBKnIgptIQggAigCMEEERw0EIAEoAjBBBEcNBSAIQQBMDQAgCiAPTA0AIAEoAhAiCSAGTA0AQQAhByAGQQFqIQAgBkECdCEMIAkgBmtBAXEhDUEAIAlrIAZBf3NGIQ4DQCAPIQUDQCAGIQECQCANRQ0AIAAhASAFQQBODQAgAigC0AEgAigCOCAHbGogAigCNCAFbGogDGogAzgCAAsgDkUEQCAFIAZqIQsDQCABIAtKBEAgAigC0AEgAigCOCAHbGogAigCNCAFbGogAUECdGogAzgCAAsgASALTgRAIAIoAtABIAIoAjggB2xqIAIoAjQgBWxqIAFBAnRqIAM4AgQLIAFBAmoiASAJRw0ACwsgBSAQaiIFIApIDQALIAdBAWoiByAIRw0ACwsgBEHQAGokAA8LQbzDAigCABAwGiAEQerAADYCSCAEQeTTADYCRCAEQeQmNgJAQbjDAigCAEHL5AAgBEFAaxAxEAAAC0G8wwIoAgAQMBogBEH42QA2AjggBEHp0wA2AjQgBEHkJjYCMEG4wwIoAgBBy+QAIARBMGoQMRAAAAtBvMMCKAIAEDAaIARBxNkANgIoIARB6tMANgIkIARB5CY2AiBBuMMCKAIAQcvkACAEQSBqEDEQAAALQbzDAigCABAwGiAEQeLPADYCGCAEQfzTADYCFCAEQeQmNgIQQbjDAigCAEHL5AAgBEEQahAxEAAAC0G8wwIoAgAQMBogBEH+zwA2AgggBEH90wA2AgQgBEHkJjYCAEG4wwIoAgBBy+QAIAQQMRAAAAsfAEEIEF4gABCJAyIAQZy3AzYCACAAQby3A0EHEAIAC+WqAQITfwF+QcCZOSQIQcCZNSQHQdDxNBAiQdDxNEHQ8TQ2AgBBhPI0IwgiAjYCAEHw8TRBAjYCAEGI8jQgAiMHazYCAEGc8jRBnPI0NgIAQbDyNEG48TQ2AgBB6PE0QSo2AgBBmPI0QaDzNDYCAEHc8TRB0PE0NgIAQdjxNEHQ8TQ2AgBB0PE0EIwEQdDxNBAhIwBBEGsiAiQAAkAgAkEMaiACQQhqEBwNAEGMiDUgAigCDEECdEEEahA7IgQ2AgAgBEUNACACKAIIEDsiBARAQYyINSgCACACKAIMQQJ0akEANgIAQYyINSgCACAEEBtFDQELQYyINUEANgIACyACQRBqJABB1LsDQQk2AgBB2LsDQQA2AgBB7QtBAkHk+QBBtPoAQQFBAkEAEAZBsyNBAUG4+gBBvPoAQQNBBEEAEAZB1wtBBEHA+gBB7PoAQQVBBkEAEAZB2LsDQYDxNCgCADYCAEGA8TRB1LsDNgIAIwBBkAVrIgEkACABQQA6ACAgAUEAOgAwIAFBjRooAAA2AjwgAUFAa0GRGi0AADoAACABQQc6ABcgAUGYFigAADYADyABQQA2AgggAUEEOgAnIAFBADoAEyABQQQ6ADcgAUKBgICAwK6at/kANwIYIAFBBToARyABQQM2AjggAUKCgICAoKzYueUANwIoIAFBlRYoAAA2AgxBvNc0QgA3AgBBuNc0QbzXNDYCACABQdgZKAAANgJMIAFB3BkvAAA7AVAgAUGnIygAADYCXCABQasjLQAAOgBgIAFBBjoAVyABQQQ2AkggAUEAOgBBIAFBBTYCWCABQQU6AGcgAUEAOgBSIAFBADoAYSABIAFBCGoiAiACEMgBIAEgAUEYaiICIAIQyAEgASABQShqIgIgAhDIASABIAFBOGoiAiACEMgBIAEgAUHIAGoiAiACEMgBIAEgAUHYAGoiAiACEMgBIAEsAGdBAEgEQCABKAJcEC8LIAEsAFdBAEgEQCABKAJMEC8LIAEsAEdBAEgEQCABKAI8EC8LIAEsADdBAEgEQCABKAIsEC8LIAEsACdBAEgEQCABKAIcEC8LIAEsABdBAEgEQCABKAIMEC8LQQAhBCMAQcArayIAJAAgAEEAOgC+FSAAQeXcATsBvBUgAEECOgDHFSAAQbMVakGzHCgAADYAACAAQdAVaiAAQcQVaigCADYCACAAQbAcKAAANgKwFSAAQQA6ALcVIABB2BVqIAApArAVNwMAIABBBzoAuxUgAEHgFWogAEG4FWooAgA2AgAgAEEANgLUFSAAQQA2AqwVIAAgACkCvBU3A8gVIABBADoAohUgAEH60AE7AaAVIABBAjoAqxUgAEGXFWpBySEoAAA2AAAgAEEHOgCfFSAAQQE2ApAVIABBADoAmxUgAEHGISgAADYClBUgAEHsFWogAEGoFWooAgA2AgAgACAAKQKgFTcC5BUgAEHwFWpBATYCACAAQfwVaiAAQZwVaigCADYCACAAQfQVaiAAKQKUFTcCACAAQQA6AIYVIABB5MoBOwGEFSAAQQI6AI8VIABB/BRqQZ4YLwAAOwEAIABBBjoAgxUgAEECNgL0FCAAQQA6AP4UIABBmhgoAAA2AvgUIABBiBZqIABBjBVqKAIANgIAIAAgACkChBU3A4AWIABBjBZqQQI2AgAgAEGYFmogAEGAFWooAgA2AgAgAEGQFmogACkC+BQ3AwAgAEEAOgDqFCAAQeXmATsB6BQgAEECOgDzFCAAQd8UakGdHCgAADYAACAAQQc6AOcUIABBAzYC2BQgAEEAOgDjFCAAQZocKAAANgLcFCAAQaQWaiAAQfAUaigCADYCACAAIAApAugUNwKcFiAAQagWakEDNgIAIABBtBZqIABB5BRqKAIANgIAIABBrBZqIAApAtwUNwIAIABBADoAzhQgAEHy6gE7AcwUIABBAjoA1xQgAEHDFGpBxRgoAAA2AAAgAEEHOgDLFCAAQQQ2ArwUIABBADoAxxQgAEHCGCgAADYCwBQgAEHAFmogAEHUFGooAgA2AgAgACAAKQLMFDcDuBYgAEHEFmpBBDYCACAAQdAWaiAAQcgUaigCADYCACAAQcgWaiAAKQLAFDcDACAAQQA6ALIUIABB694BOwGwFCAAQQI6ALsUIABBqBRqQcsZLwAAOwEAIABBBjoArxQgAEEFNgKgFCAAQQA6AKoUIABBxxkoAAA2AqQUIABB3BZqIABBuBRqKAIANgIAIAAgACkCsBQ3AtQWIABB4BZqQQU2AgAgAEHsFmogAEGsFGooAgA2AgAgAEHkFmogACkCpBQ3AgAgAEEAOgCWFCAAQebkATsBlBQgAEECOgCfFCAAQYwUakGFHS8AADsBACAAQQY6AJMUIABBBjYChBQgAEEAOgCOFCAAQYEdKAAANgKIFCAAQfgWaiAAQZwUaigCADYCACAAIAApApQUNwPwFiAAQfwWakEGNgIAIABBiBdqIABBkBRqKAIANgIAIABBgBdqIAApAogUNwMAIABBADoA+hMgAEHqwgE7AfgTIABBAjoAgxQgAEH0E2oiAkEAOgAAIABBCDoA9xMgAEEHNgLoEyAAQurCwYvmrdm55QA3AuwTIABBlBdqIABBgBRqKAIANgIAIAAgACkC+BM3AowXIABBmBdqQQc2AgAgAEGkF2ogAigCADYCACAAQZwXakLqwsGL5q3ZueUANwIAIABBADoA3hMgAEHw6AE7AdwTIABBAjoA5xMgAEHYE2oiAkGpIS8AADsBACAAQYAUOwHaEyAAQQg2AswTIABBoSEpAAAiEzcC0BMgAEGwF2ogAEHkE2ooAgA2AgAgACAAKQLcEzcDqBcgAEG0F2pBCDYCACAAQcAXaiACKAIANgIAIABBuBdqIBM3AwAgAEEAOgDCEyAAQfTkATsBwBMgAEECOgDLEyAAQbcTakG7HCgAADYAACAAQQc6AL8TIABBCTYCsBMgAEEAOgC7EyAAQbgcKAAANgK0EyAAQcwXaiAAQcgTaigCADYCACAAIAApAsATNwLEFyAAQdAXakEJNgIAIABB3BdqIABBvBNqKAIANgIAIABB1BdqIAApArQTNwIAIABBADoAphMgAEHw2AE7AaQTIABBAjoArxMgAEGcE2pBrRwvAAA7AQAgAEEGOgCjEyAAQQo2ApQTIABBADoAnhMgAEGpHCgAADYCmBMgAEHoF2ogAEGsE2ooAgA2AgAgACAAKQKkEzcD4BcgAEHsF2pBCjYCACAAQfgXaiAAQaATaigCADYCACAAQfAXaiAAKQKYEzcDACAAQQA6AIoTIABB48IBOwGIEyAAQQI6AJMTIABB/xJqQaoYKAAANgAAIABBBzoAhxMgAEELNgL4EiAAQQA6AIMTIABBpxgoAAA2AvwSIABBhBhqIABBkBNqKAIANgIAIAAgACkCiBM3AvwXIABBiBhqQQs2AgAgAEGUGGogAEGEE2ooAgA2AgAgAEGMGGogACkC/BI3AgAgAEEAOgDuEiAAQe7YATsB7BIgAEECOgD3EiAAQeQSakH5HC0AADoAACAAQQU6AOsSIABBDDYC3BIgAEEAOgDlEiAAQfUcKAAANgLgEiAAQaAYaiAAQfQSaigCADYCACAAIAApAuwSNwOYGCAAQaQYakEMNgIAIABBsBhqIABB6BJqKAIANgIAIABBqBhqIAApAuASNwMAIABBADoA0hIgAEHh5AE7AdASIABBAjoA2xIgAEHIEmpB3SYvAAA7AQAgAEEGOgDPEiAAQQ02AsASIABBADoAyhIgAEHZJigAADYCxBIgAEG8GGogAEHYEmooAgA2AgAgACAAKQLQEjcCtBggAEHAGGpBDTYCACAAQcwYaiAAQcwSaigCADYCACAAQcQYaiAAKQLEEjcCACAAQQA6ALYSIABB8+wBOwG0EiAAQQI6AL8SIABBqxJqQdEcKAAANgAAIABBBzoAsxIgAEEONgKkEiAAQQA6AK8SIABBzhwoAAA2AqgSIABB2BhqIABBvBJqKAIANgIAIAAgACkCtBI3A9AYIABB3BhqQQ42AgAgAEHoGGogAEGwEmooAgA2AgAgAEHgGGogACkCqBI3AwAgAEEAOgCaEiAAQenoATsBmBIgAEECOgCjEiAAQY8SakGwGSgAADYAACAAQQc6AJcSIABBDzYCiBIgAEEAOgCTEiAAQa0ZKAAANgKMEiAAQfQYaiAAQaASaigCADYCACAAIAApApgSNwLsGCAAQfgYakEPNgIAIABBhBlqIABBlBJqKAIANgIAIABB/BhqIAApAowSNwIAIABBADoA/hEgAEHpyAE7AfwRIABBAjoAhxIgAEH4EWoiAkHaGC8AADsBACAAQYAUOwH6ESAAQRA2AuwRIABB0hgpAAAiEzcC8BEgAEGQGWogAEGEEmooAgA2AgAgACAAKQL8ETcDiBkgAEGUGWpBEDYCACAAQaAZaiACKAIANgIAIABBmBlqIBM3AwAgAEEAOgDiESAAQejSATsB4BEgAEECOgDrESAAQdgRakHtGy0AADoAACAAQQU6AN8RIABBETYC0BEgAEEAOgDZESAAQekbKAAANgLUESAAQawZaiAAQegRaigCADYCACAAIAApAuARNwKkGSAAQbAZakERNgIAIABBvBlqIABB3BFqKAIANgIAIABBtBlqIAApAtQRNwIAIABBADoAxhEgAEHm0gE7AcQRIABBAjoAzxEgAEG7EWpBlRwoAAA2AAAgAEEHOgDDESAAQRI2ArQRIABBADoAvxEgAEGSHCgAADYCuBEgAEHIGWogAEHMEWooAgA2AgAgACAAKQLEETcDwBkgAEHMGWpBEjYCACAAQdgZaiAAQcARaigCADYCACAAQdAZaiAAKQK4ETcDACAAQQA6AKoRIABB9tIBOwGoESAAQQI6ALMRIABBpBFqIgJB4CEvAAA7AQAgAEGAFDsBphEgAEETNgKYESAAQdghKQAAIhM3ApwRIABB5BlqIABBsBFqKAIANgIAIAAgACkCqBE3AtwZIABB6BlqQRM2AgAgAEH0GWogAigCADYCACAAQewZaiATNwIAIABBADoAjhEgAEHoygE7AYwRIABBAjoAlxEgAEGEEWpBtwovAAA7AQAgAEEGOgCLESAAQRQ2AvwQIABBADoAhhEgAEGzCigAADYCgBEgAEGAGmogAEGUEWooAgA2AgAgACAAKQKMETcD+BkgAEGEGmpBFDYCACAAQZAaaiAAQYgRaigCADYCACAAQYgaaiAAKQKAETcDACAAQQA6APIQIABB9dYBOwHwECAAQQI6APsQIABB7BBqIgJBjBktAAA6AAAgAEEJOgDvECAAQRU2AuAQIABBADoA7RAgAEGEGSkAACITNwLkECAAQZwaaiAAQfgQaigCADYCACAAIAApAvAQNwKUGiAAQaAaakEVNgIAIABBrBpqIAIoAgA2AgAgAEGkGmogEzcCACAAQQA6ANYQIABB5dgBOwHUECAAQQI6AN8QIABBzBBqQZUbLQAAOgAAIABBBToA0xAgAEEWNgLEECAAQQA6AM0QIABBkRsoAAA2AsgQIABBuBpqIABB3BBqKAIANgIAIAAgACkC1BA3A7AaIABBvBpqQRY2AgAgAEHIGmogAEHQEGooAgA2AgAgAEHAGmogACkCyBA3AwAgAEEAOgC6ECAAQe3mATsBuBAgAEECOgDDECAAQbAQakH7CC0AADoAACAAQQU6ALcQIABBFzYCqBAgAEEAOgCxECAAQfcIKAAANgKsECAAQdQaaiAAQcAQaigCADYCACAAIAApArgQNwLMGiAAQdgaakEXNgIAIABB5BpqIABBtBBqKAIANgIAIABB3BpqIAApAqwQNwIAIABBADoAnhAgAEHj5gE7AZwQIABBAjoApxAgAEGUEGpBjB0tAAA6AAAgAEEFOgCbECAAQRg2AowQIABBADoAlRAgAEGIHSgAADYCkBAgAEHwGmogAEGkEGooAgA2AgAgACAAKQKcEDcD6BogAEH0GmpBGDYCACAAQYAbaiAAQZgQaigCADYCACAAQfgaaiAAKQKQEDcDACAAQQA6AIIQIABB8t4BOwGAECAAQQI6AIsQIABB/A9qIgJBADoAACAAQQg6AP8PIABBGTYC8A8gAELy3rWL5q3asO4ANwL0DyAAQYwbaiAAQYgQaigCADYCACAAIAApAoAQNwKEGyAAQZAbakEZNgIAIABBnBtqIAIoAgA2AgAgAEGUG2pC8t61i+at2rDuADcCACAAQQA6AOYPIABB5MIBOwHkDyAAQQI6AO8PIABB3A9qQaYcLwAAOwEAIABBBjoA4w8gAEEaNgLUDyAAQQA6AN4PIABBohwoAAA2AtgPIABBqBtqIABB7A9qKAIANgIAIAAgACkC5A83A6AbIABBrBtqQRo2AgAgAEG4G2ogAEHgD2ooAgA2AgAgAEGwG2ogACkC2A83AwAgAEEAOgDKDyAAQejqATsByA8gAEECOgDTDyAAQcQPaiICQeUYLQAAOgAAIABBCToAxw8gAEEbNgK4DyAAQQA6AMUPIABB3RgpAAAiEzcCvA8gAEHEG2ogAEHQD2ooAgA2AgAgACAAKQLIDzcCvBsgAEHIG2pBGzYCACAAQdQbaiACKAIANgIAIABBzBtqIBM3AgAgAEEAOgCuDyAAQfTCATsBrA8gAEECOgC3DyAAQaQPakG0Gi0AADoAACAAQQU6AKsPIABBHDYCnA8gAEEAOgClDyAAQbAaKAAANgKgDyAAQeAbaiAAQbQPaigCADYCACAAIAApAqwPNwPYGyAAQeQbakEcNgIAIABB8BtqIABBqA9qKAIANgIAIABB6BtqIAApAqAPNwMAIABBADoAkg8gAEHu3gE7AZAPIABBAjoAmw8gAEGMD2oiAkG9GS0AADoAACAAQQk6AI8PIABBHTYCgA8gAEEAOgCNDyAAQbUZKQAAIhM3AoQPIABB/BtqIABBmA9qKAIANgIAIAAgACkCkA83AvQbIABBgBxqQR02AgAgAEGMHGogAigCADYCACAAQYQcaiATNwIAIABBADoA9g4gAEH00AE7AfQOIABBAjoA/w4gAEHsDmpBADoAACAAQQQ6APMOIABBHjYC5A4gAEH00IXLBjYC6A4gAEGYHGogAEH8DmooAgA2AgAgACAAKQL0DjcDkBwgAEGcHGpBHjYCACAAQagcaiAAQfAOaigCADYCACAAQaAcaiAAKQLoDjcDACAAQQA6ANoOIABB9eQBOwHYDiAAQQI6AOMOIABB0A5qQQA6AAAgAEEEOgDXDiAAQR82AsgOIABB9eSRqwc2AswOIABBtBxqIABB4A5qKAIANgIAIAAgACkC2A43AqwcIABBuBxqQR82AgAgAEHEHGogAEHUDmooAgA2AgAgAEG8HGogACkCzA43AgAgAEEAOgC+DiAAQejkATsBvA4gAEECOgDHDiAAQbgOaiICQQA6AAAgAEEIOgC7DiAAQSA2AqwOIABC4+S9i8au2rDuADcCsA4gAEHQHGogAEHEDmooAgA2AgAgACAAKQK8DjcDyBwgAEHUHGpBIDYCACAAQeAcaiACKAIANgIAIABB2BxqQuPkvYvGrtqw7gA3AwAgAEEAOgCiDiAAQeLOATsBoA4gAEECOgCrDiAAQZwOaiICQe8YLQAAOgAAIABBCToAnw4gAEEhNgKQDiAAQQA6AJ0OIABB5xgpAAAiEzcClA4gAEHsHGogAEGoDmooAgA2AgAgACAAKQKgDjcC5BwgAEHwHGpBITYCACAAQfwcaiACKAIANgIAIABB9BxqIBM3AgAgAEEAOgCGDiAAQezoATsBhA4gAEECOgCPDiAAQYAOaiICQaAZLwAAOwEAIABBCjoAgw4gAEEiNgL0DSAAQQA6AIIOIABBmBkpAAAiEzcC+A0gAEGIHWogAEGMDmooAgA2AgAgACAAKQKEDjcDgB0gAEGMHWpBIjYCACAAQZgdaiACKAIANgIAIABBkB1qIBM3AwAgAEEAOgDqDSAAQezCATsB6A0gAEECOgDzDSAAQeANakGTFy0AADoAACAAQQU6AOcNIABBIzYC2A0gAEEAOgDhDSAAQY8XKAAANgLcDSAAQaQdaiAAQfANaigCADYCACAAIAApAugNNwKcHSAAQagdakEjNgIAIABBtB1qIABB5A1qKAIANgIAIABBrB1qIAApAtwNNwIAIABBADoAzg0gAEHt0gE7AcwNIABBAjoA1w0gAEHEDWpBqhstAAA6AAAgAEEFOgDLDSAAQSQ2ArwNIABBADoAxQ0gAEGmGygAADYCwA0gAEHAHWogAEHUDWooAgA2AgAgACAAKQLMDTcDuB0gAEHEHWpBJDYCACAAQdAdaiAAQcgNaigCADYCACAAQcgdaiAAKQLADTcDACAAQQA6ALINIABB7dgBOwGwDSAAQQI6ALsNIABBrA1qIgJB5xktAAA6AAAgAEEJOgCvDSAAQSU2AqANIABBADoArQ0gAEHfGSkAACITNwKkDSAAQdwdaiAAQbgNaigCADYCACAAIAApArANNwLUHSAAQeAdakElNgIAIABB7B1qIAIoAgA2AgAgAEHkHWogEzcCACAAQQA6AJYNIABB4/IBOwGUDSAAQQI6AJ8NIABBjA1qQZAcLQAAOgAAIABBBToAkw0gAEEmNgKEDSAAQQA6AI0NIABBjBwoAAA2AogNIABB+B1qIABBnA1qKAIANgIAIAAgACkClA03A/AdIABB/B1qQSY2AgAgAEGIHmogAEGQDWooAgA2AgAgAEGAHmogACkCiA03AwAgAEEAOgD6DCAAQfPWATsB+AwgAEECOgCDDSAAQfAMakGhGy8AADsBACAAQQY6APcMIABBJzYC6AwgAEEAOgDyDCAAQZ0bKAAANgLsDCAAQZQeaiAAQYANaigCADYCACAAIAApAvgMNwKMHiAAQZgeakEnNgIAIABBpB5qIABB9AxqKAIANgIAIABBnB5qIAApAuwMNwIAIABBADoA3gwgAEH0ygE7AdwMIABBAjoA5wwgAEHUDGpB1AovAAA7AQAgAEEGOgDbDCAAQSg2AswMIABBADoA1gwgAEHQCigAADYC0AwgAEGwHmogAEHkDGooAgA2AgAgACAAKQLcDDcDqB4gAEG0HmpBKDYCACAAQcAeaiAAQdgMaigCADYCACAAQbgeaiAAKQLQDDcDACAAQQA6AMIMIABB5sIBOwHADCAAQQI6AMsMIABBtwxqQc0YKAAANgAAIABBBzoAvwwgAEEpNgKwDCAAQQA6ALsMIABByhgoAAA2ArQMIABBzB5qIABByAxqKAIANgIAIAAgACkCwAw3AsQeIABB0B5qQSk2AgAgAEHcHmogAEG8DGooAgA2AgAgAEHUHmogACkCtAw3AgAgAEEAOgCmDCAAQezsATsBpAwgAEECOgCvDCAAQZsMakGyGCgAADYAACAAQQc6AKMMIABBKjYClAwgAEEAOgCfDCAAQa8YKAAANgKYDCAAQegeaiAAQawMaigCADYCACAAIAApAqQMNwPgHiAAQeweakEqNgIAIABB+B5qIABBoAxqKAIANgIAIABB8B5qIAApApgMNwMAIABBADoAigwgAEHi3AE7AYgMIABBAjoAkwwgAEH/C2pB1RsoAAA2AAAgAEEHOgCHDCAAQSs2AvgLIABBADoAgwwgAEHSGygAADYC/AsgAEGEH2ogAEGQDGooAgA2AgAgACAAKQKIDDcC/B4gAEGIH2pBKzYCACAAQZQfaiAAQYQMaigCADYCACAAQYwfaiAAKQL8CzcCACAAQQA6AO4LIABB8+QBOwHsCyAAQQI6APcLIABB4wtqQcIZKAAANgAAIABBBzoA6wsgAEEsNgLcCyAAQQA6AOcLIABBvxkoAAA2AuALIABBoB9qIABB9AtqKAIANgIAIAAgACkC7As3A5gfIABBpB9qQSw2AgAgAEGwH2ogAEHoC2ooAgA2AgAgAEGoH2ogACkC4As3AwAgAEEAOgDSCyAAQeH0ATsB0AsgAEECOgDbC0EQEDIiC0EAOgALIAtBtxsoAAA2AAcgC0GwGykAADcAACAAQcAfakEtNgIAIABBvB9qIABB2AtqKAIANgIAIAAgACkC0As3ArQfIABBxB9qIAtBCxBrIABBADoAxgsgAEHz2AE7AcQLIABBAjoAzwsgAEHAC2oiAkGWGS0AADoAACAAQdwfakEuNgIAIABB4B9qQY4ZKQAAIhM3AwAgAEHYH2ogAEHMC2ooAgA2AgAgAEEJOgDDCyAAQQA6AMELIABB6B9qIAIoAgA2AgAgAEEuNgK0CyAAIBM3ArgLIAAgACkCxAs3A9AfIABBADoAqgsgAEHr3AE7AagLIABBAjoAswsgAEGfC2pBmCkoAAA2AAAgAEEHOgCnCyAAQS82ApgLIABBADoAowsgAEGVKSgAADYCnAsgAEH0H2ogAEGwC2ooAgA2AgAgACAAKQKoCzcC7B8gAEH4H2pBLzYCACAAQYQgaiAAQaQLaigCADYCACAAQfwfaiAAKQKcCzcCACAAQQA6AI4LIABB5egBOwGMCyAAQQI6AJcLIABBiAtqIgJBADoAACAAQQg6AIsLIABBMDYC/AogAELl5tH75q3asO4ANwKACyAAQZAgaiAAQZQLaigCADYCACAAIAApAowLNwOIICAAQZQgakEwNgIAIABBoCBqIAIoAgA2AgAgAEGYIGpC5ebR++at2rDuADcDACAAQQA6APIKIABB7dYBOwHwCiAAQQI6APsKIABB7ApqIgJBgRkvAAA7AQAgAEEKOgDvCiAAQTE2AuAKIABBADoA7gogAEH5GCkAACITNwLkCiAAQawgaiAAQfgKaigCADYCACAAIAApAvAKNwKkICAAQbAgakExNgIAIABBvCBqIAIoAgA2AgAgAEG0IGogEzcCACAAQQA6ANYKIABB4uQBOwHUCiAAQQI6AN8KIABBzApqQbkWLwAAOwEAIABBBjoA0wogAEEyNgLECiAAQQA6AM4KIABBtRYoAAA2AsgKIABByCBqIABB3ApqKAIANgIAIAAgACkC1Ao3A8AgIABBzCBqQTI2AgAgAEHYIGogAEHQCmooAgA2AgAgAEHQIGogACkCyAo3AwAgAEEAOgC6CiAAQeXqATsBuAogAEECOgDDCiAAQbAKakGhHy8AADsBACAAQQY6ALcKIABBMzYCqAogAEEAOgCyCiAAQZ0fKAAANgKsCiAAQeQgaiAAQcAKaigCADYCACAAIAApArgKNwLcICAAQeggakEzNgIAIABB9CBqIABBtApqKAIANgIAIABB7CBqIAApAqwKNwIAIABBADoAngogAEHp5gE7AZwKIABBAjoApwogAEGYCmoiAkHXJi0AADoAACAAQQk6AJsKIABBNDYCjAogAEEAOgCZCiAAQc8mKQAAIhM3ApAKIABBgCFqIABBpApqKAIANgIAIAAgACkCnAo3A/ggIABBhCFqQTQ2AgAgAEGQIWogAigCADYCACAAQYghaiATNwMAIABBADoAggogAEHo8gE7AYAKIABBAjoAiwogAEH8CWoiAkEAOgAAIABBCDoA/wkgAEE1NgLwCSAAQuHktavmrdqw7gA3AvQJIABBnCFqIABBiApqKAIANgIAIAAgACkCgAo3ApQhIABBoCFqQTU2AgAgAEGsIWogAigCADYCACAAQaQhakLh5LWr5q3asO4ANwIAIABBADoA5gkgAEHuygE7AeQJIABBAjoA7wkgAEHcCWpByBsvAAA7AQAgAEEGOgDjCSAAQTY2AtQJIABBADoA3gkgAEHEGygAADYC2AkgAEG4IWogAEHsCWooAgA2AgAgACAAKQLkCTcDsCEgAEG8IWpBNjYCACAAQcghaiAAQeAJaigCADYCACAAQcAhaiAAKQLYCTcDACAAQQA6AMoJIABB7dwBOwHICSAAQQI6ANMJIABBxAlqIgJBqxktAAA6AAAgAEEJOgDHCSAAQTc2ArgJIABBADoAxQkgAEGjGSkAACITNwK8CSAAQdQhaiAAQdAJaigCADYCACAAIAApAsgJNwLMISAAQdghakE3NgIAIABB5CFqIAIoAgA2AgAgAEHcIWogEzcCACAAQQA6AK4JIABB4uYBOwGsCSAAQQI6ALcJIABBowlqQfQYKAAANgAAIABBBzoAqwkgAEE4NgKcCSAAQQA6AKcJIABB8RgoAAA2AqAJIABB8CFqIABBtAlqKAIANgIAIAAgACkCrAk3A+ghIABB9CFqQTg2AgAgAEGAImogAEGoCWooAgA2AgAgAEH4IWogACkCoAk3AwAgAEEAOgCSCSAAQevWATsBkAkgAEECOgCbCSAAQYgJakHyHC8AADsBACAAQQY6AI8JIABBOTYCgAkgAEEAOgCKCSAAQe4cKAAANgKECSAAQYwiaiAAQZgJaigCADYCACAAIAApApAJNwKEIiAAQZAiakE5NgIAIABBnCJqIABBjAlqKAIANgIAIABBlCJqIAApAoQJNwIAIABBADoA9gggAEHz4gE7AfQIIABBAjoA/wggAEHwCGoiAkEAOgAAIABBCDoA8wggAEE6NgLkCCAAQuHYiYvmrdqw7gA3AugIIABBqCJqIABB/AhqKAIANgIAIAAgACkC9Ag3A6AiIABBrCJqQTo2AgAgAEG4ImogAigCADYCACAAQbAiakLh2ImL5q3asO4ANwMAIABBADoA2gggAEHz7gE7AdgIIABBAjoA4wggAEHPCGpBvxsoAAA2AAAgAEEHOgDXCCAAQTs2AsgIIABBADoA0wggAEG8GygAADYCzAggAEHEImogAEHgCGooAgA2AgAgACAAKQLYCDcCvCIgAEHIImpBOzYCACAAQdQiaiAAQdQIaigCADYCACAAQcwiaiAAKQLMCDcCACAAQQA6AL4IIABB59gBOwG8CCAAQQI6AMcIIABBuAhqIgJBADoAACAAQQg6ALsIIABBPDYCrAggAELnwrHLtqzasO4ANwKwCCAAQeAiaiAAQcQIaigCADYCACAAIAApArwINwPYIiAAQeQiakE8NgIAIABB8CJqIAIoAgA2AgAgAEHoImpC58Kxy7as2rDuADcDACAAQQA6AKIIIABB7eQBOwGgCCAAQQI6AKsIIABBlwhqQd0bKAAANgAAIABBBzoAnwggAEE9NgKQCCAAQQA6AJsIIABB2hsoAAA2ApQIIABB/CJqIABBqAhqKAIANgIAIAAgACkCoAg3AvQiIABBgCNqQT02AgAgAEGMI2ogAEGcCGooAgA2AgAgAEGEI2ogACkClAg3AgAgAEEAOgCGCCAAQfDCATsBhAggAEECOgCPCCAAQfIbKAAANgD7ByAAQQc6AIMIIABBPjYC9AcgAEEAOgD/ByAAQe8bKAAANgL4ByAAQZgjaiAAQYwIaigCADYCACAAIAApAoQINwOQIyAAQZwjakE+NgIAIABBqCNqIABBgAhqKAIANgIAIABBoCNqIAApAvgHNwMAIABBADoA6gcgAEHz0gE7AegHIABBAjoA8wcgAEGCKSgAADYA3wcgAEEHOgDnByAAQT82AtgHIABBADoA4wcgAEH/KCgAADYC3AcgAEG0I2ogACgC8Ac2AgAgACAAKQLoBzcCrCMgAEG4I2pBPzYCACAAQcQjaiAAKALkBzYCACAAQbwjaiAAKQLcBzcCACAAQQA6AM4HIABB69oBOwHMByAAQQI6ANcHIABBxxQtAAA6AMQHIABBBToAywcgAEHAADYCvAcgAEEAOgDFByAAQcMUKAAANgLAByAAQdAjaiAAKALUBzYCACAAIAApAswHNwPIIyAAQdQjakHAADYCACAAQeAjaiAAKALIBzYCACAAQdgjaiAAKQLABzcDACAAQQA6ALIHIABB89wBOwGwByAAQQI6ALsHIABB9ygtAAA6AKgHIABBBToArwcgAEHBADYCoAcgAEEAOgCpByAAQfMoKAAANgKkByAAQewjaiAAKAK4BzYCACAAIAApArAHNwLkIyAAQfAjakHBADYCACAAQfwjaiAAKAKsBzYCACAAQfQjaiAAKQKkBzcCACAAQQA6AJYHIABB+d4BOwGUByAAQQI6AJ8HIABBoSkvAAA7AYwHIABBBjoAkwcgAEHCADYChAcgAEEAOgCOByAAQZ0pKAAANgKIByAAQYgkaiAAKAKcBzYCACAAIAApApQHNwOAJCAAQYwkakHCADYCACAAQZgkaiAAKAKQBzYCACAAQZAkaiAAKQKIBzcDACAAQQA6APoGIABB894BOwH4BiAAQQI6AIMHIABBzxsvAAA7AfAGIABBBjoA9wYgAEHDADYC6AYgAEEAOgDyBiAAQcsbKAAANgLsBiAAQaQkaiAAKAKABzYCACAAIAApAvgGNwKcJCAAQagkakHDADYCACAAQbQkaiAAKAL0BjYCACAAQawkaiAAKQLsBjcCACAAQQA6AN4GIABB4cwBOwHcBiAAQQI6AOcGIABBwxAtAAA6ANgGIABBCToA2wYgAEHEADYCzAYgAEEAOgDZBiAAQbsQKQAAIhM3AtAGIABBwCRqIAAoAuQGNgIAIAAgACkC3AY3A7gkIABBxCRqQcQANgIAIABB0CRqIAAoAtgGNgIAIABByCRqIBM3AwAgAEEAOgDCBiAAQe/GATsBwAYgAEECOgDLBiAAQYkYKAAANgC3BiAAQQc6AL8GIABBxQA2ArAGIABBADoAuwYgAEGGGCgAADYCtAYgAEHcJGogACgCyAY2AgAgACAAKQLABjcC1CQgAEHgJGpBxQA2AgAgAEHsJGogACgCvAY2AgAgAEHkJGogACkCtAY3AgAgAEEAOgCmBiAAQevCATsBpAYgAEECOgCvBiAAQQA6AKAGIABBCDoAowYgAEHGADYClAYgAELnyr2T96zasO4ANwKYBiAAQfgkaiAAKAKsBjYCACAAIAApAqQGNwPwJCAAQfwkakHGADYCACAAQYglaiAAKAKgBjYCACAAQYAlakLnyr2T96zasO4ANwMAIABBADoAigYgAEHiygE7AYgGIABBAjoAkwYgAEG/GC8AADsBhAYgAEEKOgCHBiAAQccANgL4BSAAQQA6AIYGIABBtxgpAAAiEzcC/AUgAEGUJWogACgCkAY2AgAgACAAKQKIBjcCjCUgAEGYJWpBxwA2AgAgAEGkJWogACgChAY2AgAgAEGcJWogEzcCACAAQQA6AO4FIABB9M4BOwHsBSAAQQI6APcFIABBjxstAAA6AOQFIABBBToA6wUgAEHIADYC3AUgAEEAOgDlBSAAQYsbKAAANgLgBSAAQbAlaiAAKAL0BTYCACAAIAApAuwFNwOoJSAAQbQlakHIADYCACAAQcAlaiAAKALoBTYCACAAQbglaiAAKQLgBTcDACAAQQA6ANIFIABB88gBOwHQBSAAQQI6ANsFIABB5hsvAAA7AcgFIABBBjoAzwUgAEHJADYCwAUgAEEAOgDKBSAAQeIbKAAANgLEBSAAQcwlaiAAKALYBTYCACAAIAApAtAFNwLEJSAAQdAlakHJADYCACAAQdwlaiAAKALMBTYCACAAQdQlaiAAKQLEBTcCACAAQQA6ALYFIABB5+oBOwG0BSAAQQI6AL8FIABBADoAsAUgAEEIOgCzBSAAQcoANgKkBSAAQufqqYumrpi66QA3AqgFIABB6CVqIAAoArwFNgIAIAAgACkCtAU3A+AlIABB7CVqQcoANgIAIABB+CVqIAAoArAFNgIAIABB8CVqQufqqYumrpi66QA3AwAgAEEAOgCaBSAAQeHaATsBmAUgAEECOgCjBSAAQcomKAAANgCPBSAAQQc6AJcFIABBywA2AogFIABBADoAkwUgAEHHJigAADYCjAUgAEGEJmogACgCoAU2AgAgACAAKQKYBTcC/CUgAEGIJmpBywA2AgAgAEGUJmogACgClAU2AgAgAEGMJmogACkCjAU3AgAgAEEAOgD+BCAAQfnSATsB/AQgAEECOgCHBSAAQdkcKAAANgDzBCAAQQc6APsEIABBzAA2AuwEIABBADoA9wQgAEHWHCgAADYC8AQgAEGgJmogACgChAU2AgAgACAAKQL8BDcDmCYgAEGkJmpBzAA2AgAgAEGwJmogACgC+AQ2AgAgAEGoJmogACkC8AQ3AwAgAEEAOgDiBCAAQezeATsB4AQgAEECOgDrBCAAQZMWLQAAOgDWBCAAQQM6AN8EIABBzQA2AtAEIABBADoA1wQgAEGRFi8AADsB1AQgAEG8JmogACgC6AQ2AgAgACAAKQLgBDcCtCYgAEHAJmpBzQA2AgAgAEHMJmogACgC3AQ2AgAgAEHEJmogACkC1AQ3AgAgAEEAOgDGBCAAQfX0ATsBxAQgAEECOgDPBCAAQZsbLQAAOgC8BCAAQQU6AMMEIABBzgA2ArQEIABBADoAvQQgAEGXGygAADYCuAQgAEHYJmogACgCzAQ2AgAgACAAKQLEBDcD0CYgAEHcJmpBzgA2AgAgAEHoJmogACgCwAQ2AgAgAEHgJmogACkCuAQ3AwAgAEEAOgCqBCAAQebeATsBqAQgAEECOgCzBCAAQbchKAAANgCfBCAAQQc6AKcEIABBzwA2ApgEIABBADoAowQgAEG0ISgAADYCnAQgAEH0JmogACgCsAQ2AgAgACAAKQKoBDcC7CYgAEH4JmpBzwA2AgAgAEGEJ2ogACgCpAQ2AgAgAEH8JmogACkCnAQ3AgAgAEEAOgCOBCAAQejoATsBjAQgAEECOgCXBEEQEDIiDEEAOgAOIAxBjCMpAAA3AAYgDEGGIykAADcAACAAQZQnakHQADYCACAAQZAnaiAAKAKUBDYCACAAIAApAowENwOIJyAAQZgnaiAMQQ4QayAAQQA6AIIEIABB8OYBOwGABCAAQQI6AIsEIABBiRYvAAA7AfgDIABBsCdqQdEANgIAIABBrCdqIAAoAogENgIAIABBhRYoAAA2AvQDIABBADoA+gMgAEG0J2ogACkC9AM3AgAgAEEGOgD/AyAAQbwnaiAAKAL8AzYCACAAQdEANgLwAyAAIAApAoAENwKkJyAAQQA6AOYDIABB9NYBOwHkAyAAQQI6AO8DIABB+hcoAAA2ANsDIABBBzoA4wMgAEHSADYC1AMgAEEAOgDfAyAAQfcXKAAANgLYAyAAQcgnaiAAKALsAzYCACAAIAApAuQDNwPAJyAAQcwnakHSADYCACAAQdgnaiAAKALgAzYCACAAQdAnaiAAKQLYAzcDACAAQQA6AMoDIABB7twBOwHIAyAAQQI6ANMDIABByRooAAA2AL8DIABBBzoAxwMgAEHTADYCuAMgAEEAOgDDAyAAQcYaKAAANgK8AyAAQeQnaiAAKALQAzYCACAAIAApAsgDNwLcJyAAQegnakHTADYCACAAQfQnaiAAKALEAzYCACAAQewnaiAAKQK8AzcCACAAQQA6AK4DIABB7egBOwGsAyAAQQI6ALcDIABBryEoAAA2AKMDIABBBzoAqwMgAEHUADYCnAMgAEEAOgCnAyAAQawhKAAANgKgAyAAQYAoaiAAKAK0AzYCACAAIAApAqwDNwP4JyAAQYQoakHUADYCACAAQZAoaiAAKAKoAzYCACAAQYgoaiAAKQKgAzcDACAAQQA6AJIDIABB88IBOwGQAyAAQQI6AJsDIABBADoAjAMgAEEIOgCPAyAAQdUANgKAAyAAQvPCuZu3zdy09AA3AoQDIABBnChqIAAoApgDNgIAIAAgACkCkAM3ApQoIABBoChqQdUANgIAIABBrChqIAAoAowDNgIAIABBpChqQvPCuZu3zdy09AA3AgAgAEEAOgD2AiAAQezEATsB9AIgAEECOgD/AkEQEDIiDUEAOgANIA1BxRwpAAA3AAUgDUHAHCkAADcAACAAQbwoakHWADYCACAAQbgoaiAAKAL8AjYCACAAIAApAvQCNwOwKCAAQcAoaiANQQ0QayAAQQA6AOoCIABB7fIBOwHoAiAAQQI6APMCIABB/RQoAAA2AN8CIABB2ChqQdcANgIAIABB1ChqIAAoAvACNgIAIABB+hQoAAA2AtwCIABBADoA4wIgAEHcKGogACkC3AI3AgAgAEEHOgDnAiAAQeQoaiAAKALkAjYCACAAQdcANgLYAiAAIAApAugCNwLMKCAAQQA6AM4CIABB4t4BOwHMAiAAQQI6ANcCIABBkRgoAAA2AMMCIABBBzoAywIgAEHYADYCvAIgAEEAOgDHAiAAQY4YKAAANgLAAiAAQfAoaiAAKALUAjYCACAAIAApAswCNwPoKCAAQfQoakHYADYCACAAQYApaiAAKALIAjYCACAAQfgoaiAAKQLAAjcDACAAQQA6ALICIABB9NgBOwGwAiAAQQI6ALsCIABBlR0oAAA2AKcCIABBBzoArwIgAEHZADYCoAIgAEEAOgCrAiAAQZIdKAAANgKkAiAAQYwpaiAAKAK4AjYCACAAIAApArACNwKEKSAAQZApakHZADYCACAAQZwpaiAAKAKsAjYCACAAQZQpaiAAKQKkAjcCACAAQQA6AJYCIABB7c4BOwGUAiAAQQI6AJ8CIABBADoAkAIgAEEIOgCTAiAAQdoANgKEAiAAQu3CsYv2rNi5+QA3AogCIABBqClqIAAoApwCNgIAIAAgACkClAI3A6ApIABBrClqQdoANgIAIABBuClqIAAoApACNgIAIABBsClqQu3CsYv2rNi5+QA3AwAgAEEAOgD6ASAAQeHmATsB+AEgAEECOgCDAiAAQQA6APQBIABBCDoA9wEgAEHbADYC6AEgAELh5s2L1q3ZueUANwLsASAAQcQpaiAAKAKAAjYCACAAIAApAvgBNwK8KSAAQcgpakHbADYCACAAQdQpaiAAKAL0ATYCACAAQcwpakLh5s2L1q3ZueUANwIAIABBADoA3gEgAEH06AE7AdwBIABBAjoA5wEgAEH4FC0AADoA1AEgAEEFOgDbASAAQdwANgLMASAAQQA6ANUBIABB9BQoAAA2AtABIABB4ClqIAAoAuQBNgIAIAAgACkC3AE3A9gpIABB5ClqQdwANgIAIABB8ClqIAAoAtgBNgIAIABB6ClqIAApAtABNwMAIABBAzoAywEgAEEAOgDDASAAQbwKLQAAOgDCASAAQboKLwAAOwHAASAAQQA6ALwBIABBCDoAvwEgAEHdADYCsAEgAELowt2Llq3asO4ANwK0ASAAQfwpaiAAKALIATYCACAAIAApAsABNwL0KSAAQYAqakHdADYCACAAQYwqaiAAKAK8ATYCACAAQYQqakLowt2Llq3asO4ANwIAIABBADoApgEgAEHs3AE7AaQBIABBAjoArwEgAEGKKSgAADYAmwEgAEEHOgCjASAAQd4ANgKUASAAQQA6AJ8BIABBhykoAAA2ApgBIABBmCpqIAAoAqwBNgIAIAAgACkCpAE3A5AqIABBnCpqQd4ANgIAIABBqCpqIAAoAqABNgIAIABBoCpqIAApApgBNwMAIABBADoAigEgAEHowgE7AYgBIABBAjoAkwEgAEHxKC0AADoAgAEgAEEFOgCHASAAQd8ANgJ4IABBADoAgQEgAEHtKCgAADYCfCAAQbQqaiAAKAKQATYCACAAIAApAogBNwKsKiAAQbgqakHfADYCACAAQcQqaiAAKAKEATYCACAAQbwqaiAAKQJ8NwIAIABBADoAbiAAQeLCATsBbCAAQQI6AHcgAEGyFCgAADYAYyAAQQc6AGsgAEHgADYCXCAAQQA6AGcgAEGvFCgAADYCYCAAQdAqaiAAKAJ0NgIAIAAgACkCbDcDyCogAEHUKmpB4AA2AgAgAEHgKmogACgCaDYCACAAQdgqaiAAKQJgNwMAIABBADoAUiAAQeruATsBUCAAQQI6AFsgAEEAOgBMIABBCDoATyAAQeEANgJAIABC6sLZi+at2bnlADcCRCAAQewqaiAAKAJYNgIAIAAgACkCUDcC5CogAEHwKmpB4QA2AgAgAEH8KmogACgCTDYCACAAQfQqakLqwtmL5q3ZueUANwIAIABBADoANiAAQfPqATsBNCAAQQI6AD8gAEHWIS0AADoAMCAAQQk6ADMgAEHiADYCJCAAQQA6ADEgAEHOISkAACITNwIoIABBiCtqIAAoAjw2AgAgACAAKQI0NwOAKyAAQYwrakHiADYCACAAQZgraiAAKAIwNgIAIABBkCtqIBM3AwAgAEEDOgAjIABBADoAGyAAQZYfLQAAOgAaIABBlB8vAAA7ARggAEHEIS0AADoAFCAAQQk6ABcgAEHjADYCCCAAQQA6ABUgAEG8ISkAACITNwIMIABBpCtqIAAoAiA2AgAgACAAKQIYNwKcKyAAQagrakHjADYCACAAQbQraiAAKAIUNgIAIABBrCtqIBM3AgBByNc0QgA3AgBBxNc0QcjXNDYCAANAIwBBEGsiCSQAIABBuCtqIhACfyAAQcgVaiAEQRxsaiEFQcjXNCgCACEGAkACQAJAQcjXNCIDQcTXNCgCAEYNAAJAIAZFBEBByNc0IQIDQCACKAIIIgMoAgAgAkYhEiADIQIgEg0ACwwBCyAGIQIDQCACIgMoAgQiAg0ACwsCQCAFKAIEIAUtAAsiAiACwCIOQQBIIggbIgcgAygCFCADLQAbIgIgAsBBAEgiChsiAiACIAdLGyIPBEAgAygCECADQRBqIAobIAUoAgAgBSAIGyAPEEoiCA0BCyACIAdJDQEMAgsgCEEATg0BCyAGRQ0BIAkgAzYCDCADQQRqDAILQcjXNCgCACICRQ0AIAUoAgAgBSAOQQBIGyEOQcjXNCEGA0ACQAJAAkACQAJAAkAgAiIDKAIUIAItABsiAiACwEEASCIKGyICIAcgAiAHSSIPGyIIBEAgDiADKAIQIANBEGogChsiCiAIEEoiEUUEQCACIAdLDQIMAwsgEUEATg0CDAELIAIgB00NAgsgAyEGIAMoAgAiAg0FDAQLIAogDiAIEEoiAg0BCyAPDQEMAgsgAkEATg0BCyADQQRqIQYgAygCBCICDQELCyAJIAM2AgwgBgwBCyAJQcjXNDYCDEHI1zQLIgYoAgAiAgR/QQAFQSwQMiICQRBqIQMCQCAFLAALQQBOBEAgAyAFKQIANwIAIAMgBSgCCDYCCAwBCyADIAUoAgAgBSgCBBBrCyACIAUoAgw2AhwgAkEgaiEDAkAgBSwAG0EATgRAIAMgBSkCEDcCACADIAUoAhg2AggMAQsgAyAFKAIQIAUoAhQQawsgAiAJKAIMNgIIIAJCADcCACAGIAI2AgAgAiEDQcTXNCgCACgCACIFBEBBxNc0IAU2AgAgBigCACEDC0HI1zQoAgAgAxCkAUHM1zRBzNc0KAIAQQFqNgIAQQELOgAEIBAgAjYCACAJQRBqJAAgBEEBaiIEQeQARw0ACyAQIQQDQCAEQQFrLAAAQQBIBEAgBEEMaygCABAvCyAEQRxrIQIgBEERaywAAEEASARAIAIoAgAQLwsgAiIEIABByBVqRw0ACyAALAAXQQBIBEAgACgCDBAvCyAALAAjQQBIBEAgACgCGBAvCyAALAAzQQBIBEAgACgCKBAvCyAALAA/QQBIBEAgACgCNBAvCyAALABPQQBIBEAgACgCRBAvCyAALABbQQBIBEAgACgCUBAvCyAALABrQQBIBEAgACgCYBAvCyAALAB3QQBIBEAgACgCbBAvCyAALACHAUEASARAIAAoAnwQLwsgACwAkwFBAEgEQCAAKAKIARAvCyAALACjAUEASARAIAAoApgBEC8LIAAsAK8BQQBIBEAgACgCpAEQLwsgACwAvwFBAEgEQCAAKAK0ARAvCyAALADLAUEASARAIAAoAsABEC8LIAAsANsBQQBIBEAgACgC0AEQLwsgACwA5wFBAEgEQCAAKALcARAvCyAALAD3AUEASARAIAAoAuwBEC8LIAAsAIMCQQBIBEAgACgC+AEQLwsgACwAkwJBAEgEQCAAKAKIAhAvCyAALACfAkEASARAIAAoApQCEC8LIAAsAK8CQQBIBEAgACgCpAIQLwsgACwAuwJBAEgEQCAAKAKwAhAvCyAALADLAkEASARAIAAoAsACEC8LIAAsANcCQQBIBEAgACgCzAIQLwsgACwA5wJBAEgEQCAAKALcAhAvCyAALADzAkEASARAIAAoAugCEC8LIA0QLyAALAD/AkEASARAIAAoAvQCEC8LIAAsAI8DQQBIBEAgACgChAMQLwsgACwAmwNBAEgEQCAAKAKQAxAvCyAALACrA0EASARAIAAoAqADEC8LIAAsALcDQQBIBEAgACgCrAMQLwsgACwAxwNBAEgEQCAAKAK8AxAvCyAALADTA0EASARAIAAoAsgDEC8LIAAsAOMDQQBIBEAgACgC2AMQLwsgACwA7wNBAEgEQCAAKALkAxAvCyAALAD/A0EASARAIAAoAvQDEC8LIAAsAIsEQQBIBEAgACgCgAQQLwsgDBAvIAAsAJcEQQBIBEAgACgCjAQQLwsgACwApwRBAEgEQCAAKAKcBBAvCyAALACzBEEASARAIAAoAqgEEC8LIAAsAMMEQQBIBEAgACgCuAQQLwsgACwAzwRBAEgEQCAAKALEBBAvCyAALADfBEEASARAIAAoAtQEEC8LIAAsAOsEQQBIBEAgACgC4AQQLwsgACwA+wRBAEgEQCAAKALwBBAvCyAALACHBUEASARAIAAoAvwEEC8LIAAsAJcFQQBIBEAgACgCjAUQLwsgACwAowVBAEgEQCAAKAKYBRAvCyAALACzBUEASARAIAAoAqgFEC8LIAAsAL8FQQBIBEAgACgCtAUQLwsgACwAzwVBAEgEQCAAKALEBRAvCyAALADbBUEASARAIAAoAtAFEC8LIAAsAOsFQQBIBEAgACgC4AUQLwsgACwA9wVBAEgEQCAAKALsBRAvCyAALACHBkEASARAIAAoAvwFEC8LIAAsAJMGQQBIBEAgACgCiAYQLwsgACwAowZBAEgEQCAAKAKYBhAvCyAALACvBkEASARAIAAoAqQGEC8LIAAsAL8GQQBIBEAgACgCtAYQLwsgACwAywZBAEgEQCAAKALABhAvCyAALADbBkEASARAIAAoAtAGEC8LIAAsAOcGQQBIBEAgACgC3AYQLwsgACwA9wZBAEgEQCAAKALsBhAvCyAALACDB0EASARAIAAoAvgGEC8LIAAsAJMHQQBIBEAgACgCiAcQLwsgACwAnwdBAEgEQCAAKAKUBxAvCyAALACvB0EASARAIAAoAqQHEC8LIAAsALsHQQBIBEAgACgCsAcQLwsgACwAywdBAEgEQCAAKALABxAvCyAALADXB0EASARAIAAoAswHEC8LIAAsAOcHQQBIBEAgACgC3AcQLwsgACwA8wdBAEgEQCAAKALoBxAvCyAALACDCEEASARAIAAoAvgHEC8LIAAsAI8IQQBIBEAgACgChAgQLwsgACwAnwhBAEgEQCAAKAKUCBAvCyAALACrCEEASARAIAAoAqAIEC8LIAAsALsIQQBIBEAgACgCsAgQLwsgACwAxwhBAEgEQCAAKAK8CBAvCyAALADXCEEASARAIAAoAswIEC8LIAAsAOMIQQBIBEAgACgC2AgQLwsgACwA8whBAEgEQCAAKALoCBAvCyAALAD/CEEASARAIAAoAvQIEC8LIAAsAI8JQQBIBEAgACgChAkQLwsgACwAmwlBAEgEQCAAKAKQCRAvCyAALACrCUEASARAIAAoAqAJEC8LIAAsALcJQQBIBEAgACgCrAkQLwsgACwAxwlBAEgEQCAAKAK8CRAvCyAALADTCUEASARAIAAoAsgJEC8LIAAsAOMJQQBIBEAgACgC2AkQLwsgACwA7wlBAEgEQCAAKALkCRAvCyAALAD/CUEASARAIAAoAvQJEC8LIAAsAIsKQQBIBEAgACgCgAoQLwsgACwAmwpBAEgEQCAAKAKQChAvCyAALACnCkEASARAIAAoApwKEC8LIAAsALcKQQBIBEAgACgCrAoQLwsgACwAwwpBAEgEQCAAKAK4ChAvCyAALADTCkEASARAIAAoAsgKEC8LIAAsAN8KQQBIBEAgACgC1AoQLwsgACwA7wpBAEgEQCAAKALkChAvCyAALAD7CkEASARAIAAoAvAKEC8LIAAsAIsLQQBIBEAgACgCgAsQLwsgACwAlwtBAEgEQCAAKAKMCxAvCyAALACnC0EASARAIAAoApwLEC8LIAAsALMLQQBIBEAgACgCqAsQLwsgACwAwwtBAEgEQCAAKAK4CxAvCyAALADPC0EASARAIAAoAsQLEC8LIAsQLyAALADbC0EASARAIAAoAtALEC8LIAAsAOsLQQBIBEAgACgC4AsQLwsgACwA9wtBAEgEQCAAKALsCxAvCyAALACHDEEASARAIAAoAvwLEC8LIAAsAJMMQQBIBEAgACgCiAwQLwsgACwAowxBAEgEQCAAKAKYDBAvCyAALACvDEEASARAIAAoAqQMEC8LIAAsAL8MQQBIBEAgACgCtAwQLwsgACwAywxBAEgEQCAAKALADBAvCyAALADbDEEASARAIAAoAtAMEC8LIAAsAOcMQQBIBEAgACgC3AwQLwsgACwA9wxBAEgEQCAAKALsDBAvCyAALACDDUEASARAIAAoAvgMEC8LIAAsAJMNQQBIBEAgACgCiA0QLwsgACwAnw1BAEgEQCAAKAKUDRAvCyAALACvDUEASARAIAAoAqQNEC8LIAAsALsNQQBIBEAgACgCsA0QLwsgACwAyw1BAEgEQCAAKALADRAvCyAALADXDUEASARAIAAoAswNEC8LIAAsAOcNQQBIBEAgACgC3A0QLwsgACwA8w1BAEgEQCAAKALoDRAvCyAALACDDkEASARAIAAoAvgNEC8LIAAsAI8OQQBIBEAgACgChA4QLwsgACwAnw5BAEgEQCAAKAKUDhAvCyAALACrDkEASARAIAAoAqAOEC8LIAAsALsOQQBIBEAgACgCsA4QLwsgACwAxw5BAEgEQCAAKAK8DhAvCyAALADXDkEASARAIAAoAswOEC8LIAAsAOMOQQBIBEAgACgC2A4QLwsgACwA8w5BAEgEQCAAKALoDhAvCyAALAD/DkEASARAIAAoAvQOEC8LIAAsAI8PQQBIBEAgACgChA8QLwsgACwAmw9BAEgEQCAAKAKQDxAvCyAALACrD0EASARAIAAoAqAPEC8LIAAsALcPQQBIBEAgACgCrA8QLwsgACwAxw9BAEgEQCAAKAK8DxAvCyAALADTD0EASARAIAAoAsgPEC8LIAAsAOMPQQBIBEAgACgC2A8QLwsgACwA7w9BAEgEQCAAKALkDxAvCyAALAD/D0EASARAIAAoAvQPEC8LIAAsAIsQQQBIBEAgACgCgBAQLwsgACwAmxBBAEgEQCAAKAKQEBAvCyAALACnEEEASARAIAAoApwQEC8LIAAsALcQQQBIBEAgACgCrBAQLwsgACwAwxBBAEgEQCAAKAK4EBAvCyAALADTEEEASARAIAAoAsgQEC8LIAAsAN8QQQBIBEAgACgC1BAQLwsgACwA7xBBAEgEQCAAKALkEBAvCyAALAD7EEEASARAIAAoAvAQEC8LIAAsAIsRQQBIBEAgACgCgBEQLwsgACwAlxFBAEgEQCAAKAKMERAvCyAALACnEUEASARAIAAoApwREC8LIAAsALMRQQBIBEAgACgCqBEQLwsgACwAwxFBAEgEQCAAKAK4ERAvCyAALADPEUEASARAIAAoAsQREC8LIAAsAN8RQQBIBEAgACgC1BEQLwsgACwA6xFBAEgEQCAAKALgERAvCyAALAD7EUEASARAIAAoAvAREC8LIAAsAIcSQQBIBEAgACgC/BEQLwsgACwAlxJBAEgEQCAAKAKMEhAvCyAALACjEkEASARAIAAoApgSEC8LIAAsALMSQQBIBEAgACgCqBIQLwsgACwAvxJBAEgEQCAAKAK0EhAvCyAALADPEkEASARAIAAoAsQSEC8LIAAsANsSQQBIBEAgACgC0BIQLwsgACwA6xJBAEgEQCAAKALgEhAvCyAALAD3EkEASARAIAAoAuwSEC8LIAAsAIcTQQBIBEAgACgC/BIQLwsgACwAkxNBAEgEQCAAKAKIExAvCyAALACjE0EASARAIAAoApgTEC8LIAAsAK8TQQBIBEAgACgCpBMQLwsgACwAvxNBAEgEQCAAKAK0ExAvCyAALADLE0EASARAIAAoAsATEC8LIAAsANsTQQBIBEAgACgC0BMQLwsgACwA5xNBAEgEQCAAKALcExAvCyAALAD3E0EASARAIAAoAuwTEC8LIAAsAIMUQQBIBEAgACgC+BMQLwsgACwAkxRBAEgEQCAAKAKIFBAvCyAALACfFEEASARAIAAoApQUEC8LIAAsAK8UQQBIBEAgACgCpBQQLwsgACwAuxRBAEgEQCAAKAKwFBAvCyAALADLFEEASARAIAAoAsAUEC8LIAAsANcUQQBIBEAgACgCzBQQLwsgACwA5xRBAEgEQCAAKALcFBAvCyAALADzFEEASARAIAAoAugUEC8LIAAsAIMVQQBIBEAgACgC+BQQLwsgACwAjxVBAEgEQCAAKAKEFRAvCyAALACfFUEASARAIAAoApQVEC8LIAAsAKsVQQBIBEAgACgCoBUQLwsgACwAuxVBAEgEQCAAKAKwFRAvCyAALADHFUEASARAIAAoArwVEC8LIABBwCtqJAAgAUEBOgAfIAFBIjsBCCABQYHGADsAEyABQQE6ACsgAUEAOgAVIAFBAToANyABQSg7ASAgAUEBOgBDIAFBKTsBLCABQQE6AE8gAUEqOwE4IAFBAToAWyABQSs7AUQgAUEBOgBnIAFBLzsBUCABQTo7AVwgAUEBOgBzIAFBOzsBaCABQQE6AH8gAUEBOgCLASABQTw7AXQgAUE9OwGAASABQQE6AJcBIAFBPjsBjAEgAUEBOgCjASABQcAAOwGYASABQQE6AK8BIAFB2wA7AaQBIAFBAToAuwEgAUHcADsBsAEgAUEBOgDHASABQd0AOwG8ASABQQE6ANMBIAFB3gA7AcgBIAFBAToA3wEgAUHfADsB1AEgAUEBOgDrASABQcAILQAAOgCeAiABQbwILQAAOgCqAiABQQE6APcBIAFB4AA7AeABIAFBAToAgwIgAUH7ADsB7AEgAUH8ADsB+AEgAUEBOgCPAiABQf0AOwGEAiABQQE6AJsCIAFB/gA7AZACIAFBAzoApwIgAUEDOgCzAiABQQA6AJ8CIAFBvggvAAA7AZwCIAFBuggvAAA7AagCIAFBuAgtAAA6ALYCIAFBtAgtAAA6AMICIAFBAzoAvwIgAUEAOgCrAiABQQM6AMsCIAFBADoAtwIgAUECOgDXAiABQQA6AMMCIAFBvPgAOwHMAiABQQA6AM4CIAFBAjoA4wIgAUEDOgDvAiABQQA6ANoCIAFBvvwAOwHYAiABQbYILwAAOwG0AiABQbIILwAAOwHAAiABQZY5LQAAOgDmAiABQZQ5LwAAOwHkAiABQQM6APsCIAFBADoA5wIgAUGSOS0AADoA8gIgAUGQOS8AADsB8AIgAUECOgCHAyABQQA6APMCIAFBAzoAkwMgAUEAOgD+AiABQa3aADsB/AIgAUHVygAtAAA6AIoDIAFB08oALwAAOwGIAyABQQI6AJ8DIAFBADoAiwMgAUECOgCrAyABQQA6AJYDIAFBrdAAOwGUAyABQQI6ALcDIAFBADoAogMgAUGttgE7AaADIAFBAjoAwwMgAUEAOgCuAyABQajOADsBrAMgAUECOgDPAyABQQA6ALoDIAFBqMQAOwG4AyABQQI6ANsDIAFBADoAxgMgAUGo0AA7AcQDIAFBAzoA5wMgAUEAOgDSAyABQanSADsB0AMgAUGR3AAtAAA6AN4DIAFBj9wALwAAOwHcAyABQQM6APMDIAFBADoA3wMgAUGC3AAtAAA6AOoDIAFBgNwALwAAOwHoAyABQQI6AP8DIAFBADoA6wMgAUECOgCLBCABQQA6APYDIAFB27YBOwH0AyABQQI6AJcEIAFBADoAggQgAUHdugE7AYAEIAFBAjoAowQgAUEAOgCOBCABQfv2ATsBjAQgAUEGOgCvBCABQQA6AJoEIAFB/foBOwGYBCABQasILwAAOwGoBCABQacIKAAANgKkBCABQQk6ALsEIAFBADoAqgQgAUGsCC0AADoAuAQgAUGkCCkAADcCsAQgAUEDOgDHBCABQQA6ALkEIAFBsAgtAAA6AL4EIAFBrggvAAA7AbwEIAFBAzoA0wQgAUEAOgC/BCABQawILQAAOgDKBCABQaoILwAAOwHIBCABQQM6AN8EIAFBADoAywQgAUGiCC0AADoA1gQgAUGgCC8AADsB1AQgAUEDOgDrBCABQQA6ANcEIAFBnggtAAA6AOIEIAFBnAgvAAA7AeAEIAFBAzoA9wQgAUEAOgDjBCABQZoILQAAOgDuBCABQZgILwAAOwHsBCABQQM6AIMFIAFBADoA7wQgAUGWCC0AADoA+gQgAUGUCC8AADsB+AQgAUEDOgCPBSABQQA6APsEIAFBkggtAAA6AIYFIAFBkAgvAAA7AYQFIAFBADoAhwVB4Nc0QQA2AgBB5Nc0QQA2AgBB5Nc0QYgFEDIiAjYCAEHg1zQgAjYCAEHo1zQgAkGIBWo2AgBBACEEA0ACQCABQQhqIARBDGxqIgMsAAtBAE4EQCACIAMpAgA3AgAgAiADKAIINgIIDAELIAIgAygCACADKAIEEGsLIAJBDGohAiAEQQFqIgRBNkcNAAtB5Nc0IAI2AgAgAUGQBWohBANAIARBDGshAiAEQQFrLAAAQQBIBEAgAigCABAvCyACIgQgAUEIakcNAAsgAUGQBWokAEGE8TRBuQE2AgBBiPE0QQA2AgAQsARBiPE0QYDxNCgCADYCAEGA8TRBhPE0NgIAC5sHBAR/BHsBfAF9IABBcHEiBkEASgRAA0AgCCACIAdBAXQiBWoiBC8BHkECdEGQ1gRqIAQvARxBAnRBkNYEaiAELwEaQQJ0QZDWBGogBC8BGEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyADIAVqIgUvAR5BAnRBkNYEaiAFLwEcQQJ0QZDWBGogBS8BGkECdEGQ1gRqIAUvARhBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhCCAJIAQvARZBAnRBkNYEaiAELwEUQQJ0QZDWBGogBC8BEkECdEGQ1gRqIAQvARBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAMgBS8BFkECdEGQ1gRqIAUvARRBAnRBkNYEaiAFLwESQQJ0QZDWBGogBS8BEEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAA/3mAf3kASEJIAogBC8BDkECdEGQ1gRqIAQvAQxBAnRBkNYEaiAELwEKQQJ0QZDWBGogBC8BCEECdEGQ1gRq/QkCAP1WAgAB/VYCAAL9VgIAAyAFLwEOQQJ0QZDWBGogBS8BDEECdEGQ1gRqIAUvAQpBAnRBkNYEaiAFLwEIQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgAD/eYB/eQBIQogCyAELwEGQQJ0QZDWBGogBC8BBEECdEGQ1gRqIAQvAQJBAnRBkNYEaiAELwEAQQJ0QZDWBGr9CQIA/VYCAAH9VgIAAv1WAgADIAUvAQZBAnRBkNYEaiAFLwEEQQJ0QZDWBGogBS8BAkECdEGQ1gRqIAUvAQBBAnRBkNYEav0JAgD9VgIAAf1WAgAC/VYCAAP95gH95AEhCyAHQRBqIgcgBkgNAAsgCyAJ/eQBIAogCP3kAf3kASEICyAI/R8DIAj9HwIgCP0fACAI/R8BkpKSIQ0gASAAIAZKBH0gBkF/cyEEIA27IQwgAEEBcQRAIAwgAiAGQQF0IgVqLwEAQQJ0QZDWBGoqAgAgAyAFai8BAEECdEGQ1gRqKgIAlLugIQwgBkEBciEGC0EAIABrIARHBEADQCAMIAIgBkEBdCIEai8BAEECdEGQ1gRqKgIAIAMgBGovAQBBAnRBkNYEaioCAJS7oCACIARBAmoiBGovAQBBAnRBkNYEaioCACADIARqLwEAQQJ0QZDWBGoqAgCUu6AhDCAGQQJqIgYgAEcNAAsLIAy2BSANCzgCAAuHAwMGfwR7AX0gAEFwcSIGQQBKBEADQCACIARBAnQiB2oiBf0AADAgAyAHaiIH/QAAMP3mASAK/eQBIQogBf0AACAgB/0AACD95gEgC/3kASELIAX9AAAQIAf9AAAQ/eYBIAz95AEhDCAF/QAAACAH/QAAAP3mASAN/eQBIQ0gBEEQaiIEIAZIDQALIA0gC/3kASAMIAr95AH95AEhCgsgCv0fAyAK/R8CIAr9HwAgCv0fAZKSkiEOAkAgACAGTA0AIAZBf3MgAGohCSAAQQNxIgcEQEEAIQQDQCACIAZBAnQiCGoqAgAgAyAIaioCAJQgDpIhDiAGQQFqIQYgBEEBaiIEIAdHDQALCyAJQQJNDQADQCACIAZBAnQiBEEMaiIFaioCACADIAVqKgIAlCACIARBCGoiBWoqAgAgAyAFaioCAJQgAiAEQQRqIgVqKgIAIAMgBWoqAgCUIAIgBGoqAgAgAyAEaioCAJQgDpKSkpIhDiAGQQRqIgYgAEcNAAsLIAEgDjgCAAslACABIAIgAyAEIAUgBq0gB61CIIaEIAitIAmtQiCGhCAAERgACyMAIAEgAiADIAQgBa0gBq1CIIaEIAetIAitQiCGhCAAERkACxkAIAEgAiADIAQgBa0gBq1CIIaEIAARFAALGQAgASACIAOtIAStQiCGhCAFIAYgABEaAAsiAQF+IAEgAq0gA61CIIaEIAQgABETACIFQiCIpyQJIAWnCygAIAAkAyABJAQgAiQFIAMkBgJAIAJFDQAgBEUNAEGMuQMgBDYCAAsLEAAjACAAa0FwcSIAJAAgAAsGACAAJAALBAAjAAsFAEH3GwsFAEGnJgsFAEHGFgsXACAARQRAQQAPCyAAQcixAxD/AkEARwsbACAAIAEoAgggBRBvBEAgASACIAMgBBCXAgsLOAAgACABKAIIIAUQbwRAIAEgAiADIAQQlwIPCyAAKAIIIgAgASACIAMgBCAFIAAoAgAoAhQRDAALkAIBB38gACABKAIIIAUQbwRAIAEgAiADIAQQlwIPCyABLQA1IQYgACgCDCEIIAFBADoANSABLQA0IQcgAUEAOgA0IABBEGoiDCABIAIgAyAEIAUQlgIgAS0ANCIKIAdyQQBHIQcgAS0ANSILIAZyQQBHIQYCQCAAQRhqIgkgDCAIQQN0aiIITw0AA0AgAS0ANg0BAkAgCgRAIAEoAhhBAUYNAyAALQAIQQJxDQEMAwsgC0UNACAALQAIQQFxRQ0CCyABQQA7ATQgCSABIAIgAyAEIAUQlgIgBiABLQA1IgtyQQBHIQYgByABLQA0IgpyQQBHIQcgCUEIaiIJIAhJDQALCyABIAY6ADUgASAHOgA0C6cBACAAIAEoAgggBBBvBEACQCABKAIEIAJHDQAgASgCHEEBRg0AIAEgAzYCHAsPCwJAIAAgASgCACAEEG9FDQACQCACIAEoAhBHBEAgASgCFCACRw0BCyADQQFHDQEgAUEBNgIgDwsgASACNgIUIAEgAzYCICABIAEoAihBAWo2AigCQCABKAIkQQFHDQAgASgCGEECRw0AIAFBAToANgsgAUEENgIsCwuIAgAgACABKAIIIAQQbwRAAkAgASgCBCACRw0AIAEoAhxBAUYNACABIAM2AhwLDwsCQCAAIAEoAgAgBBBvBEACQCACIAEoAhBHBEAgASgCFCACRw0BCyADQQFHDQIgAUEBNgIgDwsgASADNgIgAkAgASgCLEEERg0AIAFBADsBNCAAKAIIIgAgASACIAJBASAEIAAoAgAoAhQRDAAgAS0ANQRAIAFBAzYCLCABLQA0RQ0BDAMLIAFBBDYCLAsgASACNgIUIAEgASgCKEEBajYCKCABKAIkQQFHDQEgASgCGEECRw0BIAFBAToANg8LIAAoAggiACABIAIgAyAEIAAoAgAoAhgRCwALC7IEAQN/IAAgASgCCCAEEG8EQAJAIAEoAgQgAkcNACABKAIcQQFGDQAgASADNgIcCw8LAkAgACABKAIAIAQQbwRAAkAgAiABKAIQRwRAIAEoAhQgAkcNAQsgA0EBRw0CIAFBATYCIA8LIAEgAzYCICABKAIsQQRHBEAgAEEQaiIFIAAoAgxBA3RqIQZBACEDIAECfwJAA0ACQCAFIAZPDQAgAUEAOwE0IAUgASACIAJBASAEEJYCIAEtADYNAAJAIAEtADVFDQAgAS0ANARAIAEoAhhBAUcEQEEBIQNBASEHIAAtAAhBAnENAgsgAUEDNgIsDwtBASEDIAAtAAhBAXFFDQMLIAVBCGohBQwBCwtBBCADRQ0BGgtBAws2AiwgBw0CCyABIAI2AhQgASABKAIoQQFqNgIoIAEoAiRBAUcNASABKAIYQQJHDQEgAUEBOgA2DwsgACgCDCEGIABBEGoiByABIAIgAyAEENwBIABBGGoiBSAHIAZBA3RqIgZPDQACQCAAKAIIIgBBAnFFBEAgASgCJEEBRw0BCwNAIAEtADYNAiAFIAEgAiADIAQQ3AEgBUEIaiIFIAZJDQALDAELIABBAXFFBEADQCABLQA2DQIgASgCJEEBRg0CIAUgASACIAMgBBDcASAFQQhqIgUgBkkNAAwCCwALA0AgAS0ANg0BIAEoAiRBAUYEQCABKAIYQQFGDQILIAUgASACIAMgBBDcASAFQQhqIgUgBkkNAAsLC2sBAn8gACABKAIIQQAQbwRAIAEgAiADEJgCDwsgACgCDCEEIABBEGoiBSABIAIgAxD+AgJAIABBGGoiACAFIARBA3RqIgRPDQADQCAAIAEgAiADEP4CIAEtADYNASAAQQhqIgAgBEkNAAsLCzIAIAAgASgCCEEAEG8EQCABIAIgAxCYAg8LIAAoAggiACABIAIgAyAAKAIAKAIcEQYACxkAIAAgASgCCEEAEG8EQCABIAIgAxCYAgsLngEBAn8jAEFAaiIDJAACf0EBIAAgAUEAEG8NABpBACABRQ0AGkEAIAFB6LADEP8CIgFFDQAaIANBDGpBAEE0/AsAIANBATYCOCADQX82AhQgAyAANgIQIAMgATYCCCABIANBCGogAigCAEEBIAEoAgAoAhwRBgAgAygCICIAQQFGBEAgAiADKAIYNgIACyAAQQFGCyEEIANBQGskACAECwoAIAAgAUEAEG8LjAUBB38jAEEQayIDJAAgAEEMaiIEKAIAIQIjAEEQayIBJAAgASACNgIMIAEoAgwhAiABQRBqJAAgAyACNgIMIAAoAhAhAiMAQRBrIgEkACABIAI2AgwgASgCDCECIAFBEGokACADIAI2AggDQCADKAIMIgEgAygCCEcEQCABKAIAEJADIAMoAgwoAgQQjwMgAyADKAIMQQhqNgIMDAEFAkAgACgCACECIwBBEGsiASQAIAEgAjYCDCABKAIMIQIgAUEQaiQAIAMgAjYCDCAAKAIEIQIjAEEQayIBJAAgASACNgIMIAEoAgwhAiABQRBqJAAgAyACNgIIA0AgAygCDCIBIAMoAghGDQEgASgCACECIwBBEGsiASQAIAFBAToADCABIAJBDGoiBTYCCCAFEFYEQBBGAAsgAiACKAJUQQRyNgJUIAJBJGoQkAMgAS0ADARAIAEoAggQjwMLIAFBEGokACADKAIMKAIAIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyADIAMoAgxBBGo2AgwMAAsACwsLIwBBEGsiAiQAIAIgBDYCDCACKAIMIgEoAgAaIAEoAgQhBCABKAIIGiABKAIAIgUEQANAIAQgBUcEQCAEQQhrIQQMAQsLIAEgBTYCBCACKAIMIgEoAgAhBiABKAIIGiABKAIAGiAGEC8LIAJBEGokACMAQRBrIgEkACABIAA2AgwgASgCDCIAKAIEGiAAKAIIGiAAKAIAGiAAKAIABEAgACgCACEEIAAoAgQhAgNAIAIgBEcEQCACQQRrIQIMAQsLIAAgBDYCBCABKAIMIgAoAgAhByAAKAIIGiAAKAIAGiAHEC8LIAFBEGokACADQRBqJAALEQAgAARAIAAQ3QEaCyAAEC8LVAEBfyABAn8gASgCCCICIAEoAgxHBEACQAJAIAIsAABBCmsOBAEAAAEACyABQZ14NgIAIAEgAkEBajYCCCAAKAIEDAILCyABQZ94NgIAQQALNgIoCwcAIAAQlAILAwAACwkAQYSMNRBQGgswAAJAQZCMNf4SAABBAXENAEGQjDUQVEUNAEGEjDVB2PcCELoBQZCMNRBTC0GEjDULCQBB9Is1EDMaCzAAAkBBgIw1/hIAAEEBcQ0AQYCMNRBURQ0AQfSLNUHwFRCmARpBgIw1EFMLQfSLNQsJAEHkizUQUBoLMAACQEHwizX+EgAAQQFxDQBB8Is1EFRFDQBB5Is1QYT3AhC6AUHwizUQUwtB5Is1CwkAQdSLNRAzGgswAAJAQeCLNf4SAABBAXENAEHgizUQVEUNAEHUizVB9CoQpgEaQeCLNRBTC0HUizULCQBBxIs1EFAaCzAAAkBB0Is1/hIAAEEBcQ0AQdCLNRBURQ0AQcSLNUHg9gIQugFB0Is1EFMLQcSLNQsJAEG0izUQMxoLMAACQEHAizX+EgAAQQFxDQBBwIs1EFRFDQBBtIs1QZcuEKYBGkHAizUQUwtBtIs1CwkAQaSLNRBQGgswAAJAQbCLNf4SAABBAXENAEGwizUQVEUNAEGkizVBvPYCELoBQbCLNRBTC0GkizULCQBBlIs1EDMaCzAAAkBBoIs1/hIAAEEBcQ0AQaCLNRBURQ0AQZSLNUG6CRCmARpBoIs1EFMLQZSLNQsbAEGYlDUhAANAIABBDGsQUCIAQYCUNUcNAAsLZgACQEGQizX+EgAAQQFxDQBBkIs1EFRFDQACQEGYlDX+EgAAQQFxDQBBmJQ1EFRFDQBBmJQ1EFMLQYCUNUHQnwMQQEGMlDVB3J8DEEBBjIs1QYCUNTYCAEGQizUQUwtBjIs1KAIACxsAQfiTNSEAA0AgAEEMaxAzIgBB4JM1Rw0ACwtkAAJAQYiLNf4SAABBAXENAEGIizUQVEUNAAJAQfiTNf4SAABBAXENAEH4kzUQVEUNAEH4kzUQUwtB4JM1QYEvEEFB7JM1Qf4uEEFBhIs1QeCTNTYCAEGIizUQUwtBhIs1KAIACxsAQdCTNSEAA0AgAEEMaxBQIgBBsJE1Rw0ACwvCAgACQEGAizX+EgAAQQFxDQBBgIs1EFRFDQACQEHQkzX+EgAAQQFxDQBB0JM1EFRFDQBB0JM1EFMLQbCRNUHImwMQQEG8kTVB6JsDEEBByJE1QYycAxBAQdSRNUGknAMQQEHgkTVBvJwDEEBB7JE1QcycAxBAQfiRNUHgnAMQQEGEkjVB9JwDEEBBkJI1QZCdAxBAQZySNUG4nQMQQEGokjVB2J0DEEBBtJI1QfydAxBAQcCSNUGgngMQQEHMkjVBsJ4DEEBB2JI1QcCeAxBAQeSSNUHQngMQQEHwkjVBvJwDEEBB/JI1QeCeAxBAQYiTNUHwngMQQEGUkzVBgJ8DEEBBoJM1QZCfAxBAQayTNUGgnwMQQEG4kzVBsJ8DEEBBxJM1QcCfAxBAQfyKNUGwkTU2AgBBgIs1EFMLQfyKNSgCAAu+wQIFP38KfAJ+AX0DeyMAQdADayISJAACQAJAQdC7AygCAEUEQEF/IQAMAQsgEkEANgKEAhCcAiEDIBJCADcCkAIgEkGAgAE2AowCIBJBADoAmAIgEkL/////j4CAwL9/NwL4AiASQpqz5vhzNwLwAiASQpqz5oCEgIDAv383AugCIBL9DAAAAAAAAIA/AACAv83MTD79CwLYAiASQQA6ANYCIBJBgAI7AdQCIBJBgxg2AtACIBJBADYCzAIgEkIANwLEAiASQQA6AMACIBJBADYCvAIgEkEAOwG4AiASQQA2ArQCIBJBADoAsAIgEkEANgKsAiASQoquj+Gj4fWRPDcCpAIgEkKBgICAkIDAADcAmQIgEkEEIAMgA0EEThs2AogCIBL9DAAAAAAAAAAAAAAAAAAAAAD9CwKgAyAS/QwAAAAAAAAAAAAAAAAAAAAA/QsCkAMgEv0MAAAAAAAAAAAAAAAAAAAAAP0LAoADIBJCgICAgICAgOTCADcCsAMgEkEFNgL0AiASQYCAhAg2ApwCIBIgAjoAmAIgEkGDGCABKAIAIAEgASwAC0EASBtB0LsDKAIAKALAAUGYlQNMGzYC0AIQnAIhASASQQA2ApACIBJBCCABIAFBCE4bNgKIAiASQQA2AoACIBJCADcC+AEgACgCAEGFHBAIIgIQByEBIAIQBCABQfyyAyASQbgDahAVIUMgEigCuAMQFCABEARBpjkQEyIvQckUEAgiARAHISkgARAEAn8gQ5lEAAAAAAAA4EFjBEAgQ6oMAQtBgICAgHgLIgEEQCASQfgBaiABEG4LIAAoAgBB1hMQCCIDEAchAiADEAQgEigC+AEhAyApEAogEiABNgLIAyASIAM2AsADIBIgKTYCuAMgAkEDQfT6ACASQbgDahASISsgAhAEAkAjCi0AAEEBcQRAIwshAQwBCyMKITFBAkGA+wAQLiECIDFBAToAACMLIgEgAjYCAAsgASgCACEyIAAoAgAQCiASIAAoAgA2ArgDIDIgK0HkDiASQbgDahAtEMYCIBIoAogCIQIQnAIhAyMAQTBrIgAkAAJAQdzXNP4SAABBAXENAEHc1zQQVEUNAEHQ1zRCADcCAEHY1zRBADYCAEHc1zQQUwsCf0Hb1zQsAABBAEgEQEHU1zRBADYCAEHQ1zQoAgAMAQtB29c0QQA6AABB0Nc0C0EAOgAAIABBBGoiAUEAEDkgACABQffgABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUHw4QAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwsgAEEEaiIBQQAQOSAAIAFB+OEAEDgiASgCCDYCGCAAIAEpAgA3AxAgAUIANwIAIAFBADYCCCAAIABBEGpB8+AAEDciASgCCDYCKCAAIAEpAgA3AyAgAUIANwIAIAFBADYCCEHQ1zQgACgCICAAQSBqIAAtACsiAcBBAEgiBBsgACgCJCABIAQbEF8aIAAsACtBAEgEQCAAKAIgEC8LIAAsABtBAEgEQCAAKAIQEC8LIAAsAA9BAEgEQCAAKAIEEC8LIABBBGoiAUEAEDkgACABQdjhABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUGZ4QAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwsgAEEEaiIBQQAQOSAAIAFB1OEAEDgiASgCCDYCGCAAIAEpAgA3AxAgAUIANwIAIAFBADYCCCAAIABBEGpB8+AAEDciASgCCDYCKCAAIAEpAgA3AyAgAUIANwIAIAFBADYCCEHQ1zQgACgCICAAQSBqIAAtACsiAcBBAEgiBBsgACgCJCABIAQbEF8aIAAsACtBAEgEQCAAKAIgEC8LIAAsABtBAEgEQCAAKAIQEC8LIAAsAA9BAEgEQCAAKAIEEC8LIABBBGoiAUEAEDkgACABQavhABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUHB4QAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwsgAEEEaiIBQQAQOSAAIAFByeEAEDgiASgCCDYCGCAAIAEpAgA3AxAgAUIANwIAIAFBADYCCCAAIABBEGpB8+AAEDciASgCCDYCKCAAIAEpAgA3AyAgAUIANwIAIAFBADYCCEHQ1zQgACgCICAAQSBqIAAtACsiAcBBAEgiBBsgACgCJCABIAQbEF8aIAAsACtBAEgEQCAAKAIgEC8LIAAsABtBAEgEQCAAKAIQEC8LIAAsAA9BAEgEQCAAKAIEEC8LIABBBGoiAUEBEDkgACABQbThABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUGF4QAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwsgAEEEaiIBQQAQOSAAIAFB6OEAEDgiASgCCDYCGCAAIAEpAgA3AxAgAUIANwIAIAFBADYCCCAAIABBEGpB8+AAEDciASgCCDYCKCAAIAEpAgA3AyAgAUIANwIAIAFBADYCCEHQ1zQgACgCICAAQSBqIAAtACsiAcBBAEgiBBsgACgCJCABIAQbEF8aIAAsACtBAEgEQCAAKAIgEC8LIAAsABtBAEgEQCAAKAIQEC8LIAAsAA9BAEgEQCAAKAIEEC8LIABBBGoiAUEAEDkgACABQefhABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUH+4AAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwsgAEEEaiIBQQAQOSAAIAFB3+EAEDgiASgCCDYCGCAAIAEpAgA3AxAgAUIANwIAIAFBADYCCCAAIABBEGpB8+AAEDciASgCCDYCKCAAIAEpAgA3AyAgAUIANwIAIAFBADYCCEHQ1zQgACgCICAAQSBqIAAtACsiAcBBAEgiBBsgACgCJCABIAQbEF8aIAAsACtBAEgEQCAAKAIgEC8LIAAsABtBAEgEQCAAKAIQEC8LIAAsAA9BAEgEQCAAKAIEEC8LIABBBGoiAUEAEDkgACABQaHhABA4IgEoAgg2AhggACABKQIANwMQIAFCADcCACABQQA2AgggACAAQRBqQfPgABA3IgEoAgg2AiggACABKQIANwMgIAFCADcCACABQQA2AghB0Nc0IAAoAiAgAEEgaiAALQArIgHAQQBIIgQbIAAoAiQgASAEGxBfGiAALAArQQBIBEAgACgCIBAvCyAALAAbQQBIBEAgACgCEBAvCyAALAAPQQBIBEAgACgCBBAvCyAAQQRqIgFBABA5IAAgAUGN4QAQOCIBKAIINgIYIAAgASkCADcDECABQgA3AgAgAUEANgIIIAAgAEEQakHz4AAQNyIBKAIINgIoIAAgASkCADcDICABQgA3AgAgAUEANgIIQdDXNCAAKAIgIABBIGogAC0AKyIBwEEASCIEGyAAKAIkIAEgBBsQXxogACwAK0EASARAIAAoAiAQLwsgACwAG0EASARAIAAoAhAQLwsgACwAD0EASARAIAAoAgQQLwtB29c0LAAAIQFB0Nc0KAIAIQQgAEEwaiQAIBJB0Nc0IAQgAUEAThs2AugBIBIgAzYC5AEgEiACNgLgAUGJ5AAgEkHgAWoQswEQxgIgEkEBNgLUASASIBIoAogCNgLQASASIBIoAtACNgLYASASQZEhQd8jIBItAJgCGzYC3AEgEkGE3AA2AsABIBIgEigC/AEgEigC+AFrQQJ1IgA2AsQBIBIgALNDAAB6RpW7OQPIAUHH8QAgEkHAAWoQogQQxgJBACEAQdC7AygCACIBEGk3AwggASgCgAIiAQRAIAH9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAH9DAAAAAAAAAAAAAAAAAAAAAD9CwMgIAFCADcDECAB/QwAAAAAAAAAAAAAAAAAAAAA/QsDMCABQUBrQQA2AgALQdC7AygCACEBIBIoAvwBIQYgEigC+AEhEyASQQxqIgIgEkGEAmpBtAH8CgAAIwBBwAFrIiwkACABKAKAAiEbICxBDGoiECACQbQB/AoAACMAQcAaayIFJAAgBSAbNgKcGiAFIAE2AqAaIBtBlKgBaigCACIBIBsoApCoASIDRwRAA0AgAUEwayICKAIcIgQEQCABQRBrIAQ2AgAgBBAvCyABQRVrLAAAQQBIBEAgAUEgaygCABAvCyACIgEgA0cNAAsLIBsgAzYClKgBAkAgBiATa0ECdSIVQQBKBEAgEC0ANARAIAVBxSA2ApAGQQJBteUAIAVBkAZqEDQMAgsgBSgCnBoiBCEPIAUoAqAaQdAAaiIGKAIAIQMgECgCBCEBIwBB0ABrIgkkACAJQaABNgJIIAlBkAM2AkwgCSABNgJEEGkhTCAJQQA2AkAgCUIANwI4IAlBOGpBkAMQbiAJKAI4IQH9DAAAAAABAAAAAgAAAAMAAAAhTwNAIAEgDEECdGr9DAAAAAAAAPA/AAAAAAAA8D8gT/3+Af0MGC1EVPshGUAYLURU+yEZQP3yAf0MAAAAAAAAeUAAAAAAAAB5QP3zASJQ/SEAthBq/RMgUP0hAbYQav0gASBPIE/9DQgJCgsMDQ4PAAECAwABAgP9/gH9DBgtRFT7IRlAGC1EVPshGUD98gH9DAAAAAAAAHlAAAAAAAAAeUD98wEiUP0hALYQav0gAiBQ/SEBthBq/SADIlD9X/3xAf0MAAAAAAAA4D8AAAAAAADgP/3yASJR/SEAtv0TIFH9IQG2/SAB/QwAAAAAAADwPwAAAAAAAPA/IFAgT/0NCAkKCwwNDg8AAQIDAAECA/1f/fEB/QwAAAAAAADgPwAAAAAAAOA//fIBIlD9IQC2/SACIFD9IQG2/SAD/QsCACBP/QwEAAAABAAAAAQAAAAEAAAA/a4BIU8gDEEEaiIMQZADRw0ACyAJQQA2AjQgCUIANwIsIBWsIk1CkAN8p0GAph1qIgEEfyAJQSxqIAEQbiAJKAIsBUEAC0GgBmogEyAVQQJ0IgL8CgAAIAIgCSgCLCIBaiICIAJBoAZqIgprQcCk9QBqIgJBAEoEQCAKQQAgAkECdiACQQNLa0ECdEEEavwLAAsgBEGcAWohDUHJASEMAkACQAJAIAEgE0EEaiICQaAGak8NACACIAFBoAZqTw0AIAEhAgwBCyATQQxrIQIDQCABIA5BAnRqIAJByAEgDmtBAnRq/QACACBP/Q0MDQ4PCAkKCwQFBgcAAQID/QsCACAOQQRqIg5ByAFHDQALDAELA0AgAiAMQQJ0IBNqIgRBBGsqAgA4AgAgAiAEQQhrKgIAOAIEIAIgBEEMayoCADgCCCACIBMgDEEEayIMQQJ0aioCADgCDCACQRBqIQIgDEEBRw0ACwsgDSADNgIIIA0gCSgCMCABa0ECdSAJKAJMIgxrIAkoAkgiB24iATYCACANIE1CyAF8Ik0gDKx9IAesf6dBAWo2AgQgDUEMaiEfAkAgASADbCIBIA0oAhAgDSgCDCIDa0ECdSICSwRAIB8gASACaxBuDAELIAEgAk8NACANIAMgAUECdGo2AhALIAlBADYCKCAJQgA3AiBBACEOQQEhAgJAAkACQCAJKAJEQQFrIgEEQCABQYCAgIAETw0HIAkgAUECdCIBEDIiDjYCICAJIAEgDmoiETYCKCAOQQAgAfwLACAJIBE2AiRBACEMA0AgCSAMQQFqIgo2AhggCSAJQThqNgIUIAkgTTcDCCAJIAY2AgQgCSANNgIAQQQQMhCaAiEBQcAAEDIiB0HQADYCBCAHIAE2AgAgByAJKAIYNgIIIAkoAhQhASAHQQA2AhggB0IANwMQIAcgATYCDCAJKAIwIhggCSgCLCICRwRAAkACQAJAIBggAmsiAUEATgRAIAcgARAyIgM2AhAgByADIAFBfHFqNgIYIAFBBGsiAUEMSQ0BIAMgAmtBEEkNASADIAFBAnZBAWoiIUH8////B3EiFkECdCIBaiEEIAEgAmohAUEAIRQDQCADIBRBAnQiF2ogAiAXav0AAgD9CwIAIBRBBGoiFCAWRw0ACyAWICFGDQMMAgsMDAsgAiEBIAMhBAsDQCAEIAEqAgA4AgAgBEEEaiEEIAFBBGoiASAYRw0ACwsgByAENgIUCyAHIAkpAwg3AyAgByAJKAJMNgIoIAcgCSgCSDYCLCAHIAkoAkQ2AjAgByAJKAIENgI0IAcgCSgCADYCOCAJQRxqIgFB0QAgBxD2AQRAEEYACyAOIAxBAnRqIgIoAgANAiACIAEoAgA2AgAgAUEANgIAIAEQuAEaIAoiDCAJKAJEIgJBAWtIDQALIAkoAkwhDCAJKAJIIQcLQQAgCUE4aiAJQSxqIE2nIAwgByACIAYgDRDfBCAJKAJEQQFKBEADQCAOIAtBAnRqEJ0CIAtBAWoiCyAJKAJEQQFrSA0ACwsgDgRAIA4gEUcEQANAIBFBBGsQuAEiESAORw0ACwsgDhAvCyANKAIAIA0oAghsIgFBAEwNAiABQQNxIQYgHygCACEDQQAhDiABQQRJBEBEQIy1eB2vFcQhQ0EAIQIMAgsgAUF8cSEKRECMtXgdrxXEIUNBACECQQAhCwNAIAMgAkECdCIEQQxyaioCALsiQiADIARBCHJqKgIAuyJEIAMgBEEEcmoqAgC7IkUgAyAEaioCALsiRiBDIEMgRmMbIkMgQyBFYxsiQyBDIERjGyJDIEIgQ2QbIUMgAkEEaiECIAtBBGoiCyAKRw0ACwwBCxCZAgALIAYEQANAIAMgAkECdGoqAgC7IkIgQyBCIENkGyFDIAJBAWohAiAOQQFqIg4gBkcNAAsLIAFBAEwNACAfKAIAIQMgQ0QAAAAAAAAgwKAiQ7a7IUJBACECIAFBBE8EQCABQXxxIQIgQv0UIU8gQ/0UIVBBACEMA0AgAyAMQQJ0aiIEIE8gBP1dAgD9XyJRIFAgUf1K/VL9DAAAAAAAABBAAAAAAAAAEED98AH9DAAAAAAAANA/AAAAAAAA0D/98gEiUf0hALb9EyBR/SEBtv0gASBPIARBCGr9XQIA/V8iUSBQIFH9Sv1S/QwAAAAAAAAQQAAAAAAAABBA/fAB/QwAAAAAAADQPwAAAAAAANA//fIBIlH9IQC2/SACIFH9IQG2/SAD/QsCACAMQQRqIgwgAkcNAAsgASACRg0BCwNAIAMgAkECdGoiBCBCIAQqAgC7IkQgQyBEZBtEAAAAAAAAEECgRAAAAAAAANA/orY4AgAgAkEBaiICIAFHDQALCyAPEGkgTH0gDykDKHw3AyggCSgCLCIBBEAgCSABNgIwIAEQLwsgCSgCOCIBBEAgCSABNgI8IAEQLwsgCUHQAGokAAsCQAJAIBAoAkwiAUUNACABLQAARQ0AIAFBgBYQnwFFDQAgEC0AUEUNAQsgBQJ/QcTXNCgCACIDQcjXNEcEQANAIAggAygCHCIESiEGAkAgAygCBCICBEADQCACIgEoAgAiAg0ADAILAAsDQCADKAIIIgEoAgAgA0chMyABIQMgMw0ACwsgCCAEIAYbIQggASIDQcjXNEcNAAsgBUEANgLQBiAFQgA3AsgGIAhBAWogCEH/////A0kNARoMBQtBAQtBAnQiARAyIgogAWoiAjYC0AYgCkEAIAH8CwAgBSACNgLMBiAFKAKgGiEIIAUoApwaIg4hBCAQKAIEIQdEAAAAAAAAAAAhQyMAQdAAayIGJAACQCAEKAKgASIBQQBMBEAgBkEANgJEIAZBoSA2AkAgBiABQQpsNgJIQQJBi/QAIAZBQGsQNEF+IQEMAQsgCCAEQQAgB0EAQQAQ3gRFBEAgBkHdIDYCMEECQY3mACAGQTBqEDQgBkGhIDYCIEECQffpACAGQSBqEDRBeiEBDAELIAgoAuABIQFBBBAyIgwgATYCAEEAIQEjAEEQayIDJAAgBEEBNgK0ASAEQbQBaiEJAkAgDEUEQANAIAFBAnQiAiAEKAK8AWogATYCACAEKALAASACakEBNgIAIAQoAsQBIAJqKAIAQQA2AgAgBCgCyAEgAWpBADoAACABQQFqIgFBAUcNAAwCCwALA0AgAUECdCICIAQoArgBaiACIAxqKAIANgIAIAQoArwBIAJqIAE2AgAgBCgCwAEgAmpBATYCACAEKALEASACaigCAEEANgIAIAQoAsgBIAFqQQA6AAAgAUEBaiIBQQFHDQALCyAEKALIAUEBOgAAQQAhASAEQcwAakEAQQAQ7wIgCCAEIAkgB0EAQQAQ7gJFBEAgA0H3IDYCAEECQY3mACADEDRBASEBCyADQRBqJAAgAQRAIAZBoSA2AhBBAkGN6gAgBkEQahA0QXkhASAMEC8MAQsgBCAEKALYAiIBNgLcAiABIQNBxNc0KAIAIgJByNc0RwRAAkACQANAAkAgBCgChKgBIAIoAhwiAyAIKALgAWpBAnRqIQkCQCAEKALcAiIBIAQoAuACIg1JBEAgCSoCBCFOIAEgAzYCCCABIE67OQMAIAQgAUEQajYC3AIMAQsgASAEKALYAiIHa0EEdSIRQQFqIgNBgICAgAFPDQFB/////wAgDSAHayINQQN1IgsgAyADIAtJGyANQfD///8HTxsiA0GAgICAAU8NAyADQQR0Ig0QMiILIBFBBHRqIgMgCSoCBLs5AwAgAyACKAIcNgIIIANBEGohCSABIAdHBEADQCADQRBrIgMgAUEQayIBKQMANwMAIAMgASgCCDYCCCABIAdHDQALCyAEIAsgDWo2AuACIAQgCTYC3AIgBCADNgLYAiAHRQ0AIAcQLwsCQCACKAIEIgMEQANAIAMiASgCACIDDQAMAgsACwNAIAIoAggiASgCACACRyE0IAEhAiA0DQALCyABIgJByNc0Rw0BDAMLCwwHCxBHAAsgBCgC2AIhAyAEKALcAiEBCyADIAFBPiABIANrQQR1Z0EBdGtBACABIANHGxDrAgJAIAQoAtgCIgIgBCgC3AIiB0YiCA0AIAIrAwAhRSACIQEDQCABAnwCQCABKwMAIEWhIkK9QjSIp0H/D3EiBEHJB2tBP0kEQCAEIQMMAQsgQkQAAAAAAADwP6AiRCAEQckHSQ0BGkEAIQMgBEGJCEkNAEQAAAAAAAAAACBCvSJNQoCAgICAgIB4UQ0BGiBEIARB/w9PDQEaIE1CAFMEQCMAQRBrIgNEAAAAAAAAABA5AwggAysDCEQAAAAAAAAAEKIMAgsjAEEQayIDRAAAAAAAAABwOQMIIAMrAwhEAAAAAAAAAHCiDAELQZjbASsDACBCokGg2wErAwAiRKAiRiBEoSJEQbDbASsDAKIgREGo2wErAwCiIEKgoCJCIEKiIkQgRKIgQkHQ2wErAwCiQcjbASsDAKCiIEQgQkHA2wErAwCiQbjbASsDAKCiIEa9IkynQQR0QfAPcSIEQYjcAWorAwAgQqCgoCFCIARBkNwBaikDACBMQi2GfCFNIANFBEACfCBMQoCAgIAIg1AEQCBNQoCAgICAgICIP32/IkQgQqIgRKBEAAAAAAAAAH+iDAELIE1CgICAgICAgPA/fL8iRCBCoiJGIESgIkJEAAAAAAAA8D9jBHwjAEEQayIDITUgA0KAgICAgICACDcDCCA1IAMrAwhEAAAAAAAAEACiOQMIRAAAAAAAAAAAIEJEAAAAAAAA8D+gIkcgRiBEIEKhoCBCRAAAAAAAAPA/IEehoKCgRAAAAAAAAPC/oCJCIEJEAAAAAAAAAABhGwUgQgtEAAAAAAAAEACiCwwBCyBNvyJEIEKiIESgCyJCOQMAIEMgQqAhQyABQRBqIgEgB0cNAAsgCA0AIAIhAQNAIAEgASsDACBDozkDACABQRBqIgEgB0cNAAsgCA0AIApFDQAgAiEBA0AgCiABKAIIQQJ0aiABKwMAtjgCACABQRBqIgEgB0cNAAsLIAIoAgghASAMEC8LIAZB0ABqJAAgASIEQQBIBEAgBUHFIDYCAEECQdPpACAFEDQgBSAKNgLMBiAKEC8MAgsgDiAENgKoqAEgEAJ/AkBBxNc0KAIAIgFByNc0RwRAA0AgASgCHCAERg0CAkAgASIDKAIEIgIEQANAIAIiASgCACICDQAMAgsACwNAIAMoAggiASgCACADRyE2IAEhAyA2DQALCyABQcjXNEcNAAsLIAUgBDYChAYgBUHBEzYCgAZBAkGg6wAgBUGABmoQNEEADAELIAEoAhAgAUEQaiABLAAbQQBIGwsiATYCTCAKIAEQ2QRBAnRqKgIAIU4gBSABNgL0BSAFQcUgNgLwBSAFIE67OQP4BUEEQYv3ACAFQfAFahA0IBAtAFAhNyAFIAo2AswGIAoQLyA3DQELAkAgEC0AHEUNACAFKAKcGiIC/QwAAAAAAAAAAAAAAAAAAAAA/QsDuKgBIAJByKgBakEANgIAIBVBAEwNACAFQQA2AtAGIAVCADcCyAYgFUGAgICABE8NAyAVQQJ0IgYQMiIEQQAgBvwLAEEAIQMDQEMAAAAAIU5BYCEBA0ACQCABIANqIgpBAEgNACAKIBVODQAgTiATIApBAnRqKgIAi5IhTgsgAUEBciIKQSFHBEACQCADIApqIgpBAEgNACAKIBVODQAgTiATIApBAnRqKgIAi5IhTgsgAUECaiEBDAELCyAEIANBAnRqIE5DAACCQpU4AgAgA0EBaiIDIBVHDQALIAIoAsyoASIBBEAgAkHQqAFqIAE2AgAgARAvCyACIAQ2AsyoASACQdSoAWogBCAGaiIBNgIAIAJB0KgBaiABNgIACyAQKAIMQQptIh9BMkHkACAQLQA0G2oCfyAQKAIQIgFFBEAgBSgCnBooAqABDAELIAFBCm0gH2oLIiFKDQBBACEBIAVBADYClBoCQAJAAkACQAJAIBAqAmBDAAAAAF4EQAJAAkAgECoCVCJOQwgAgD9dRQRAQQAhA0EAIQQMAQtBACEEQQAhAwNAAkAgASAERwRAIAEgTjgCACAFIAFBBGoiATYClBoMAQsgASADayIEQQJ1IgdBAWoiAkGAgICABE8NA0H/////AyAEQQF1IgYgAiACIAZJGyAEQfz///8HTxsiBgR/IAZBgICAgARPDQcgBkECdBAyBUEACyIKIAdBAnRqIgIgTjgCACAGQQJ0ITggAkEEaiEGAkAgASADRg0AAkAgBEEEayIHQfwBSQ0AIAQgCmoiDEEEayIJIAFBBGsiBCADa0F8cSIOayAJSw0AIAQgDmsgBEsNACABIAxrQRBJDQAgAkEQayEMIAFBEGshCSABIAdBAnZBAWoiDkH8////B3EiB0ECdCIEayEBIAIgBGshAkEAIQQDQCAMIARBAnQiDWsgCSANa/0AAgD9CwIAIARBBGoiBCAHRw0ACyAHIA5GDQELA0AgAkEEayICIAFBBGsiASoCADgCACABIANHDQALCyA4IApqIQQgBSAGNgKUGiADBEAgAxAvCyAGIQEgAiEDCyBOIBAqAmCSIk5DCACAP10NAAsLIAUgAzYCkBogBSAENgKYGgwCCyAFIAM2ApAaIAUgATYCmBoMCAtBBBAyIgEgECoCVDgCACAFIAFBBGoiAjYCmBogBSACNgKUGiAFIAE2ApAaC0EBIRECQAJAAkACQAJAQQECfwJAAkAgECgCAA4CAAEDCyAQKAJwDAELIBAoAnAiASAQKAJ0IgIgASACShsLIgEgAUEBTBsiA0EISw0BIANBAkkNAEEBIQgDQAJAIAhB2BRsIgcgBSgCnBoiCkHQAWpqIgQoAgggBCgCACIGa0EwbSAKKALYASAKKALQAWsiDEEwbSICTw0AIAJB1qrVKk8NDSAEKAIEIQEgDBAyIgwgAkEwbGohCSAMIAEgBmtBMG1BMGxqIgwhAiABIAZHBEADQCACQTBrIgIgAUEwayIB/QADAP0LAwAgAiAB/QADIP0LAyAgAiAB/QADEP0LAxAgASAGRw0ACwsgBCAJNgIIIAQgDDYCBCAEIAI2AgAgBkUNACAGEC8LAkAgBSgCoBoiASgCwAEiAiAHIApqIgQoArgCIARBtAJqIgooAgAiB2tBAnUiBksEQCAKIAIgBmsQbgwBCyACIAZPDQAgBCAHIAJBAnRqNgK4AiAFKAKgGiEBCwJAIAEoAsABIgIgBCgCxAIgBEHAAmoiCigCACIHa0ECdSIGSwRAIAogAiAGaxBuDAELIAIgBk8NACAEIAcgAkECdGo2AsQCIAUoAqAaIQELAkAgASgCwAEiAiAEKALQAiAEQcwCaiIKKAIAIgdrQQJ1IgZLBEAgCiACIAZrEG4MAQsgAiAGTw0AIAQgByACQQJ0ajYC0AIgBSgCoBohAQsCQCABKAIgIgIgBCgC4AIgBCgC2AIiBmtBBHVNDQAgAkGAgICAAU8NDSAEKALcAiEBIAJBBHQiAhAyIgogAmohByAKIAEgBmtqIgohAiABIAZHBEADQCACQRBrIgIgAUEQayIBKQMANwMAIAIgASgCCDYCCCABIAZHDQALCyAEIAI2AtgCIAQgCjYC3AIgBCAHNgLgAiAGRQ0AIAYQLwtBACECIAVBADYCyAZBASEBA0AgBUHIBmoiBiABQQJ0aiACQR52IAJzQeWSnuAGbCABaiICNgIAIAFBAWoiCkECdCAGaiACQR52IAJzQeWSnuAGbCAKaiICNgIAIAFBAmoiCkECdCAGaiACQR52IAJzQeWSnuAGbCAKaiICNgIAIAFBA2oiCkHwBEcEQCAKQQJ0IAZqIAJBHnYgAnNB5ZKe4AZsIApqIgI2AgAgAUEEaiEBDAELCyAFQQA2AogaIARB5AJqIAVByAZqQcQT/AoAACAIQQFqIgggA0cNAAsgAyERCyAFKAKcGiEYIBAtABUEQCAYQaCoAWogGCgCnKgBNgIACyAFQQA2AtAGIAVCADcCyAYCQAJAIBAoAkQiAUUEQCAQKAJADQELIBAoAkghAgwBCyAFQcgGaiIZQYAIENcEAkACfyAFKAKgGiETIBAoAkAhAyAFKALIBiIBIQ8gBSgCzAYgAWtBAnUhIkEAIRcjAEHgAWsiCyQAIAMQaCICQfD///8HSQRAAkAgAkEKTQRAIAsgAjoAHyALQRRqIQEMAQsgAkEPckEBaiIEEDIhASALIARBgICAgHhyNgIcIAsgATYCFCALIAI2AhgLIAEgAyAC/AoAACABIAJqQQA6AAAgC0EANgKkASALQgA3ApwBAkAgCywAH0EATgRAIAsgCygCHDYCmAEgCyALKQIUNwOQAQwBCyALQZABaiALKAIUIAsoAhgQawsgC0HgABAyIgE2AoQBIAtC3oCAgICMgICAfzcCiAEgAUHaygBB3gD8CgAAIAFBADoAXiALQdwAahDlASIHIAdBnIo1EHk2AgQgB0HQiDUQeSEBIAdCADcCECAHQYAENgIMIAcgATYCCCAH/QwAAAAAAAAAAAAAAAAAAAAA/QsCGCAHIAsoAoQBIAtBhAFqIAstAI8BIgHAQQBIIgIbIgMgAyALKAKIASABIAIbaiIBEO0CIAFGBEAgC0FAa0EAOgAAIAtBADoATCALQgA3AC0gC0IANwM4IAtCADcCRCALQQA2AlQgC0EAOgBQIAv9DAAAAAAAAAAAAAAAAAAAAAD9CwMgA0AgC0IANwLAASALQQA6AMgBIAtBADoA1AEgC0IANwLMASALQgA3ALUBIAtBADYC3AEgC0EAOgDYASAL/QwAAAAAAAAAAAAAAAAAAAAA/QsDqAECfyALKAKQASALQZABaiIGIAssAJsBIgFBAEgiAxshAiACIAsoApQBIAFB/wFxIAMbaiEDIAcoAhAhBCALQagBaiIBQQA6ABQgASADNgIQIAEgAzYCDCABIARBAWogAUEMaiIEEIICIAFBADoAICABIAI2AhwgASACNgIYIAEgASkCDDcCJCABIAEtABQ6ACwgASACNgI0IAFBAToAMAJAAkAgBygCDEHwB3FBgARGBEAgByACIAMgAUEAQQEQ3wJFDQEMAgsgBygCEEUEQCAHIAIgAyABQQBBARC4BA0CDAELIAcgAiADIAFBAEEBELcERQ0ADAELAkAgAiADRg0AIAJBAWoiAiADRg0AA0AgASABKAIEIAEoAgBrQQxtIAQQggICQCAHKAIMQfAHcUGABEYEQCAHIAIgAyABQYABQQAQ3wJFDQEMBAsgBygCEEUEQCAHIAIgAyABQYABQQAQuAQNBAwBCyAHIAIgAyABQYABQQAQtwRFDQAMAwsgASABKAIEIAEoAgBrQQxtIAQQggIgAkEBaiICIANHDQALCyABIAEoAgA2AgRBAAwBCyABIAQgASgCACICIAIgASgCBEYbIgIoAgAiAzYCHCABIAMgASgCGEc6ACAgASACKAIEIgI2AiQgASACIAEoAihHOgAsQQELIRUgCygCkAEgBiALLQCbASICwEEASCIEGyEDIAMgCygClAEgAiAEG2ohCSABKAIYIQQCQCABKAIEIAEoAgBrQQxtIgYgCygCJCICIAsoAiAiDmtBDG0iCEsEQEEAIQpBACEUQQAhDQJAIAYgCGsiCCALKAIoIgwgAmtBDG1NBEACQCAIRQ0AIAIhBiAIQQxsQQxrIgxBDG5BAWpBB3EiDgRAA0AgBkIANwIAIAZBADoACCAGQQxqIQYgCkEBaiIKIA5HDQALCyAIQQxsIAJqIQIgDEHUAEkNAANAIAZCADcCACAGQgA3AgwgBkIANwIYIAZCADcCJCAGQgA3AjAgBkIANwI8IAZCADcCSCAGQgA3AlQgBkEAOgAIIAZBADoAFCAGQQA6ACAgBkEAOgAsIAZBADoAOCAGQQA6AEQgBkEAOgBQIAZBADoAXCAGQeAAaiIGIAJHDQALCyALIAI2AiQMAQsCQCACIAsoAiAiCmtBDG0iDiAIaiIGQdaq1aoBSQRAQdWq1aoBIAwgCmtBDG0iCkEBdCIMIAYgBiAMSRsgCkGq1arVAE8bIgwEQCAMQdaq1aoBTw0CIAxBDGwQMiENCyANIA5BDGxqIgohBiAIQQxsIghBDGsiDkEMbkEBakEHcSIWBEADQCAGQgA3AgAgBkEAOgAIIAZBDGohBiAUQQFqIhQgFkcNAAsLIAggCmohCCAOQdQATwRAA0AgBkIANwIAIAZCADcCDCAGQgA3AhggBkIANwIkIAZCADcCMCAGQgA3AjwgBkIANwJIIAZCADcCVCAGQQA6AAggBkEAOgAUIAZBADoAICAGQQA6ACwgBkEAOgA4IAZBADoARCAGQQA6AFAgBkEAOgBcIAZB4ABqIgYgCEcNAAsLIAsoAiAiBiACRwRAA0AgCkEMayIKIAJBDGsiAikCADcCACAKIAIoAgg2AgggAiAGRw0ACyALKAIgIQILIAsgDSAMQQxsajYCKCALIAg2AiQgCyAKNgIgIAIEQCACEC8LDAILDBQLEEcACyALKAIgIQ4gCygCJCECDAELIAYgCE8NACALIA4gBkEMbGoiAjYCJAsgAiAORwRAIAFBDGohCiABKAIAIQ0gASgCBCEUQQAhAgNAIA4gAkEMbCIGaiADIAYgDWogCiAUIA1rQQxtIAJLGygCACAEa2o2AgAgCygCICAGaiADIAEoAgAiCCAGaiAKIAEoAgQgCGtBDG0gAksbKAIEIARrajYCBCALKAIgIg4gBmogASgCACINIAZqIAogASgCBCIUIA1rQQxtIAJLGy0ACDoACCACQQFqIgIgCygCJCAOa0EMbUkNAAsLIAsgCTYCLCALQQA6ADQgCyAJNgIwIAsgAyABKAIYIARraiICNgI4IAsgAyABKAIcIARrajYCPCALIAEtACA6AEAgCyADIAEoAiQgBGtqNgJEIAsgAyABKAIoIARrajYCSCALIAEtACw6AEwgCyACNgJUIAsgAS0AMDoAUCALKAKoASIBBEAgCyABNgKsASABEC8LIAsoAiAhCgJAAn8CQAJAAkAgFQRAIAogCygCJCIORg0FA0ACQAJAAkACQAJAIAotAAgEQCAKKAIEIgwgCigCACIDayIIQfD///8HTw0BAkAgCEEKTQRAIAsgCDoAswEgC0GoAWohAgwBCyAIQQ9yQQFqIgEQMiECIAsgAUGAgICAeHI2ArABIAsgAjYCqAEgCyAINgKsAQsgAyAMRgRAIAJBADoAAAwGCyAIQRBJDQIgAiADa0EQSQ0CIAMgCEFwcSIJaiEEIAIgCWohAUEAIQYDQCACIAZqIAMgBmr9AAAA/QsAACAGQRBqIgYgCUcNAAsgCCAJRg0EDAMLIAtBADYCsAEgC0IANwOoAQwECxBNAAsgAiEBIAMhBAsgBEF/cyAMaiE5QQAhBiAMIARrQQdxIgMEQANAIAEgBC0AADoAACABQQFqIQEgBEEBaiEEIAZBAWoiBiADRw0ACwsgOUEHSQ0AA0AgASAELQAAOgAAIAEgBC0AAToAASABIAQtAAI6AAIgASAELQADOgADIAEgBC0ABDoABCABIAQtAAU6AAUgASAELQAGOgAGIAEgBC0ABzoAByABQQhqIQEgBEEIaiIEIAxHDQALCyABQQA6AAALAkACQCALKAKgASIEIAsoAqQBIgNJBEAgBCALKQOoATcCACAEIAsoArABNgIIIAtBADYCsAEgC0IANwOoASALIARBDGo2AqABDAELIAQgCygCnAEiAmtBDG0iBkEBaiIBQdaq1aoBTw0BQdWq1aoBIAMgAmtBDG0iA0EBdCIIIAEgASAISRsgA0Gq1arVAE8bIgMEfyADQdaq1aoBTw0FIANBDGwQMgVBAAsiCCAGQQxsaiIBIAspA6gBNwIAIAEgCygCsAE2AgggC0EANgKwASALQgA3A6gBIAggA0EMbGohAyABQQxqIQgCQCACIARGBEAgCyADNgKkASALIAg2AqABIAsgATYCnAEMAQsDQCABQQxrIgEgBEEMayIEKQIANwIAIAEgBCgCCDYCCCAEQgA3AgAgBEEANgIIIAIgBEcNAAsgCyADNgKkASALKAKgASEGIAsgCDYCoAEgCygCnAEhBCALIAE2ApwBIAQgBkYNAANAIAZBDGshASAGQQFrLAAAQQBIBEAgASgCABAvCyABIgYgBEcNAAsLIARFDQAgBBAvCyALLACzAUEASARAIAsoAqgBEC8LIApBDGoiCiAORg0HDAELCwwWCyAKBEAgCyAKNgIkIAoQLwsCQCAHKAIgIgFFDQAgAUF//h4CBA0AIAEgASgCACgCCBEBAAJAIAFBCGoiAv4QAgAEQCACQX/+HgIADQELIAEgASgCACgCEBEBAAsLIAcoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAssAI8BQQBIBEAgCygChAEQLwsgCywAmwFBAEgEQCALKAKQARAvCyALQQA2AiggC0EANgIgIAsoApwBIgggCygCoAEiDUYNASATQcgBaiEJQQAhA0EAIQ5BACEUQQAhBiAIIQwDQEEAIQogDCgCBCAMLQALIgEgAcBBAEgbIhUhASAVQQBKBEADQCABIApKBEAjAEEQayIWJAAgFiABIAprNgIMIAoCfyAMLQALQQd2BEAgDCgCBAwBCyAMLQALQf8AcQsiAksEQBDsAgALAn8gDC0AC0EHdgRAIAwoAgAMAQsgDAshOiAWIAIgCms2AgQgOiAKaiEcIwBBEGsiAiQAIBZBBGoiBCgCACAWQQxqIgcoAgBJIRMgAkEQaiQAIAQgByATGygCACECIwBBEGsiBCQAIAtBqAFqIQcCQCACQe////8HTQRAAkAgAkELSQRAIAcgBy0AC0GAAXEgAnI6AAsgByAHLQALQf8AcToACyAHIRMMAQsgBEEIaiAHIAJBC08EfyACQRBqQXBxIhMgE0EBayITIBNBC0YbBUEKC0EBahDDASAEKAIMGiAHIAQoAggiEzYCACAHIAcoAghBgICAgHhxIAQoAgxB/////wdxcjYCCCAHIAcoAghBgICAgHhyNgIIIAcgAjYCBAsgEyAcIAIQdyAEQQA6AAcgAiATaiAELQAHOgAAIARBEGokAAwBCxBNAAsgFkEQaiQAAkACQCAJKAIAIgRFDQAgBygCACAHIAstALMBIgLAQQBIIhMbIRwgCygCrAEgAiATGyEWIAkhAgNAAkAgFiAEKAIUIAQtABsiEyATwEEASCITGyIeIBYgHkkiJhsiJwRAIAQoAhAgBEEQaiATGyAcICcQSiITDQELQX8gJiAWIB5LGyETCyACIAQgE0EASCITGyECIARBBGogBCATGygCACIEDQALIAIgCUYNAAJAAkAgAigCFCACLQAbIgQgBMBBAEgiExsiBCAWIAQgFkkbIh4EQCAcIAIoAhAgAkEQaiATGyAeEEoiEw0BCyAEIBZNDQEMAgsgE0EASA0BCwJAIAYgFEcEQCAGIAIoAhw2AgAgBkEEaiEGQQEhF0EAIQQMAQsCQCAUIA5rIgRBAnUiE0EBaiIKQYCAgIAESQRAQf////8DIARBAXUiAyAKIAMgCksbIARB/P///wdPGyIDBH8gA0GAgICABE8NCiADQQJ0EDIFQQALIgogE0ECdGoiBiACKAIcNgIAIANBAnQhEyAGIQMCQCAOIBRGDQACQCAEQQRrIhZB/AFJDQAgBCAKaiIEQQRrIhcgFEEEayICIA5rQXxxIhxrIBdLDQAgAiACIBxrSQ0AIBQgBGtBEEkNACADQRBrIRcgFEEQayEcIBQgFkECdkEBaiIWQfz///8HcSICQQJ0IgRrIRQgAyAEayEDQQAhBANAIBcgBEECdCIeayAcIB5r/QACAP0LAgAgBEEEaiIEIAJHDQALIAIgFkYNAQsDQCADQQRrIgMgFEEEayIUKAIANgIAIA4gFEcNAAsLIAogE2ohFCAGQQRqIQZBASEXQQAhBCAORQ0BIA4QLwwBCyALIBQ2AiggCyADNgIgIAsgBjYCJAwdCyADIQ4LIAEhCgwBCyABQQFrIQFBASEECyALLACzAUEASARAIAcoAgAQLwsgBA0BCyAXRQRAQQJBpuUAQQAQNCAKQQFqIQoLQQAhFyAVIgEgCkoNAAsLIAxBDGoiDCANRw0ACyALIBQ2AiggCyADNgIgIAsgBjYCJCAGIAhFDQMaA0AgDUEMayEBIA1BAWssAABBAEgEQCABKAIAEC8LIAEiDSAIRw0ACwwCCxBHAAtBACEDQQAhBkEAIAhFDQEaCyAIEC8gBgshASALLAAfQQBIBEAgCygCFBAvCwJAAkACQCAiIAEgA2siAUECdSIGTgRAIAFBAEwNAUEAIRNBACEEAkBBASAGIAZBAUwbIgJBBEkNACAPIANrQRBJDQAgAkH8////B3EhBEEAIQEDQCAPIAFBAnQiCmogAyAKav0AAgD9CwIAIAFBBGoiASAERw0ACyACIARGDQMLIAIgBEF/c2ohOyACQQNxIgoEQANAIA8gBEECdCIHaiADIAdqKAIANgIAIARBAWohBCATQQFqIhMgCkcNAAsLIDtBA0kNAgNAIA8gBEECdCIBaiABIANqKAIANgIAIA8gAUEEaiIKaiADIApqKAIANgIAIA8gAUEIaiIKaiADIApqKAIANgIAIA8gAUEMaiIBaiABIANqKAIANgIAIARBBGoiBCACRw0ACwwCCyALICI2AgggCyAGNgIEIAtB/h42AgBBAkG09wAgCxA0QX8hBgsgA0UNAQsgCyADNgIkIAMQLwsgC0HgAWokACAGDAQLAkACQAJAAkACQCALLQBMBEAgCygCSCIIIAsoAkQiA2siCkHw////B08NAQJAIApBCk0EQCALIAo6ALMBIAtBqAFqIQIMAQsgCkEPckEBaiIBEDIhAiALIAFBgICAgHhyNgKwASALIAI2AqgBIAsgCjYCrAELIAMgCEYEQCACQQA6AAAMBgsgCkEQSQ0CIAIgA2tBEEkNAiADIApBcHEiDGohBCACIAxqIQFBACEGA0AgAiAGaiADIAZq/QAAAP0LAAAgBkEQaiIGIAxHDQALIAogDEYNBAwDCyALQQA2ArABIAtCADcDqAEMBAsQTQALIAIhASADIQQLIARBf3MgCGohPEEAIQYgCCAEa0EHcSIDBEADQCABIAQtAAA6AAAgAUEBaiEBIARBAWohBCAGQQFqIgYgA0cNAAsLIDxBB0kNAANAIAEgBC0AADoAACABIAQtAAE6AAEgASAELQACOgACIAEgBC0AAzoAAyABIAQtAAQ6AAQgASAELQAFOgAFIAEgBC0ABjoABiABIAQtAAc6AAcgAUEIaiEBIARBCGoiBCAIRw0ACwsgAUEAOgAACyALLACbAUEASARAIAsoApABEC8LIAsgCygCsAE2ApgBIAsgCykDqAE3A5ABDAALAAtBDBBeQREQcEHQqQNB0gAQAgALEE0ACyIDIAUoAswGIgIgBSgCyAYiAWtBAnUiBEsEQCAZIAMgBGsQ1wQgBSgCyAYhASAFKALMBiECDAELIAMgBE8NACAFIAEgA0ECdGoiAjYCzAYLIBAgATYCRCAQIAIgAWtBAnUiAjYCSAsgGEGcqAFqIR4gAUUNAiACQQBMDQJBACEIA0AgECgCRCAIQQJ0aiEKAkAgGCgCoKgBIgEgGCgCpKgBRwRAIAEgCigCADYCACAYIAFBBGo2AqCoAQwBCyABIB4oAgAiA2siBEECdSIMQQFqIgJBgICAgARPDQVB/////wMgBEEBdSIGIAIgAiAGSRsgBEH8////B08bIgYEfyAGQYCAgIAETw0HIAZBAnQQMgVBAAsiByAMQQJ0aiICIAooAgA2AgAgByAGQQJ0aiEKIAJBBGohDAJAIAEgA0YNAAJAIARBBGsiBkEsSQ0AIAEgBCAHamtBEEkNACACQRBrIQcgAUEQayEJIAEgBkECdkEBaiIOQfz///8HcSIGQQJ0IgRrIQEgAiAEayECQQAhBANAIAcgBEECdCINayAJIA1r/QACAP0LAgAgBEEEaiIEIAZHDQALIAYgDkYNAQsDQCACQQRrIgIgAUEEayIBKAIANgIAIAEgA0cNAAsLIBggCjYCpKgBIBggDDYCoKgBIBggAjYCnKgBIANFDQAgAxAvCyAIQQFqIgggECgCSCIBSA0ACwwBCyAFQQg2AugFIAUgAzYC5AUgBUHFIDYC4AVBAkG86wAgBUHgBWoQNAwGCyABRQ0AIBgoApyoASIHIBhBoKgBaigCACIKIAFBAnQiA2siAkYNACACIAdBBGpGBEAgBygCACEBIAcgAiAD/AoAACADIAdqIAE2AgAMAQsgCiACQQRqRgRAIApBBGsiASgCACECIAogASAHayIBayAHIAH8CgAAIAcgAjYCAAwBCyACIAdrQQJ1IgYgAUECdEECdSIBRgRAIAIhAQNAIAcoAgAhAyAHIAEoAgA2AgAgASADNgIAIAdBBGoiByACRg0CIAFBBGoiASAKRw0ACwwBCyAGIQQDQCAEIAEiBG8iAQ0ACyAERQ0AIAcgBEECdGohCANAIAhBBGsiCCAGQQJ0IgFqIQMgCCgCACECIAghBANAIAQgAygCADYCACADIgQgAWogByAGIAogA2tBAnUiA2tBAnRqIAMgBkobIgMgCEcNAAsgBCACNgIAIAcgCEcNAAsLIAUoAsgGIgEEQCAFIAE2AswGIAEQLwsgECgCOCIBIAUoAqAaIgIoAiQiA0oEQCAFIAM2AhggBSABNgIUIAVBxSA2AhBBAkGI+AAgBUEQahA0DAULIAUoApwaIAE2AtioASACKALgASEDIAVBBBAyIgE2ArwGIAUgAUEEaiIENgLEBiABIAM2AgAgBSAENgLABgJAAkAgAigCwAFBmZUDSA0AIBAoAkwQ2QQhASAFKAKcGiABNgKoqAEgASAFKAKgGigC4AFqQQFqIQYCQCAFKALABiIBIAUoAsQGIgRJBEAgASAGNgIAIAUgAUEEajYCwAYMAQsgASAFKAK8BiIDayIKQQJ1IghBAWoiAkGAgICABE8NAkH/////AyAEIANrIgRBAXUiByACIAIgB0kbIARB/P///wdPGyIEBH8gBEGAgICABE8NBSAEQQJ0EDIFQQALIgcgCEECdGoiAiAGNgIAIAcgBEECdGohCCACQQRqIQwCQCABIANGDQACQCAKQQRrIgRBLEkNACABIAcgCmprQRBJDQAgAkEQayEKIAFBEGshByABIARBAnZBAWoiCUH8////B3EiBkECdCIEayEBIAIgBGshAkEAIQQDQCAKIARBAnQiDmsgByAOa/0AAgD9CwIAIARBBGoiBCAGRw0ACyAGIAlGDQELA0AgAkEEayICIAFBBGsiASgCADYCACABIANHDQALCyAFIAg2AsQGIAUgDDYCwAYgBSACNgK8BiADRQ0AIAMQLwsgEC0AFARAIAUgBSgCoBooAuQBNgLIBiAFQbwGaiAFQcgGahDWBAwBCyAFIAUoAqAaKALoATYCyAYgBUG8BmogBUHIBmoQ1gQLIBAtABYhAQJAAkACQCAFKAKgGiIIQUBrKAIAQQJGBEAgAQ0BIAVBxSA2AtAFQQNB4OIAIAVB0AVqEDQgEEEBOgAWIAUoAqAaIQgMAQsgAUUNAQsgCCgC+AEhBiAFKALABiIBIAUoAsQGIgRJBEAgASAGNgIAIAUgAUEEajYCwAYMAQsgASAFKAK8BiIDayIKQQJ1IgxBAWoiAkGAgICABE8NAUH/////AyAEIANrIgRBAXUiByACIAIgB0kbIARB/P///wdPGyIEBH8gBEGAgICABE8NBSAEQQJ0EDIFQQALIgcgDEECdGoiAiAGNgIAIAcgBEECdGohDCACQQRqIQkCQCABIANGDQACQCAKQQRrIgRBLEkNACABIAcgCmprQRBJDQAgAkEQayEKIAFBEGshByABIARBAnZBAWoiDkH8////B3EiBkECdCIEayEBIAIgBGshAkEAIQQDQCAKIARBAnQiDWsgByANa/0AAgD9CwIAIARBBGoiBCAGRw0ACyAGIA5GDQELA0AgAkEEayICIAFBBGsiASgCADYCACABIANHDQALCyAFIAw2AsQGIAUgCTYCwAYgBSACNgK8BiADRQ0AIAMQLyAFKAKgGiEICyAFQQA2ArgGIAVCADcCsAYCQCAIKAI0IgEEQCABQYCAgIAETw0BIAUgAUECdCICEDIiATYCtAYgBSABNgKwBiAFIAEgAmo2ArgGCyAbQZCoAWohJiAFIBFBDGwiAhAyIgE2AqQGIAUgASACajYCrAYgAUEAIAJBDGsiAiACQQxwa0EMaiIC/AsAIAUgASACajYCqAYgISAfayEwIAVB2AZqIRZBvMMCKAIAIScgBUHMBmohC0EAIQogHyETA0AgECgChAEiAQRAIAUoAqAaIAUoApwaIBMgH2tB5ABsIDBtIBAoAogBIAERBgALIBNB5ABqIi0gIU4NByAFKAKcGiEBIAUoAqAaIQICQAJAIBAoAowBIgMEQCACIAEgECgCkAEgAxEDAEUNASAFKAKgGiECIAUoApwaIQELIAIgASATIBAoAgQgECgClAEgECgCmAEQ3gRFBEAgBUHFIDYCsAVBAkH36QAgBUGwBWoQNAwKCwJAIBMgH0wNACATQfQDaiAhSA0AIBggGCgCnKgBNgKgqAELQQAhIkEAIRwCQCAFKAKUGiAFKAKQGiICa0EATA0AA0AgBSACIBxBAnRqKgIAIk44AqAGQQEhAQJAAkACQCAQKAIADgIAAQILIE5DAAAAAF5FDQEgECgCcCEBDAELIE5DAAAAAF4EQCAQKAJwIQEMAQsgECgCdCEBCyAFQQEgASABQQFMGzYCnAZBACEEAkADQAJAIAUoApwaIARB2BRsaiIGQQA6ALICIAZBADsBsAIgBkG4FzYCrAIgBkKAgICAgICAeDcDgAIgBv0MAAAAAAAA8P8AAAAAAAAAAP0LA/ABIAb9DAAAAAAAAAAAAAAAAAAA8P/9CwPgASAGQQA2AtwBIAYgBigC0AE2AtQBAkAgECgCpAEiDgRAIBAoAqwBIREgECgCqAEhDCAFQQA2AtAGIAVCADcCyAYgDARAIAxB1qrVqgFPDQMgBSAMQQxsIgIQMiIBNgLIBiAFIAEgAmo2AtAGQQAhByABQQAgAkEMayICIAJBDHBrQQxqIgL8CwAgBSABIAJqNgLMBgNAIA4gB0ECdGooAgAiASgCAARAA0AgASEDAkAgBSgCyAYgB0EMbGoiCCgCBCIBIAgoAghHBEAgASADKQIANwIAIAggAUEIajYCBAwBCyABIAgoAgAiDWsiAkEDdSIVQQFqIglBgICAgAJPDRlB/////wEgAkECdSIUIAkgCSAUSRsgAkH4////B08bIgkEfyAJQYCAgIACTw0UIAlBA3QQMgVBAAsiFCAVQQN0aiICIAMpAgA3AgAgAkEIaiEVIAEgDUcEQANAIAJBCGsiAiABQQhrIgEpAgA3AgAgASANRw0ACyAIKAIAIQELIAggAjYCACAIIBU2AgQgCCAUIAlBA3RqNgIIIAFFDQAgARAvCyADQQhqIQEgAygCCA0ACwsCQCAFKALIBiAHQQxsaiIDKAIEIgEgAygCCCIJSQRAIAFCADcCACADIAFBCGo2AgQMAQsgASADKAIAIghrQQN1Ig1BAWoiAkGAgICAAk8NF0H/////ASAJIAhrIglBAnUiFSACIAIgFUkbIAlB+P///wdPGyIJBH8gCUGAgICAAk8NEiAJQQN0EDIFQQALIhUgDUEDdGoiAkIANwIAIAJBCGohDSABIAhHBEADQCACQQhrIgIgAUEIayIBKQIANwIAIAEgCEcNAAsgAygCACEBCyADIAI2AgAgAyANNgIEIAMgFSAJQQN0ajYCCCABRQ0AIAEQLwsgB0EBaiIHIAxHDQALCyAFQQA2ArgaIAVCADcCsBogDiARQQJ0aigCACEBA0AgBUEANgKsGiAFQgA3AqQaIAEoAgBBAk8EQEEEEDIiAiABNgIAIAUgAkEEaiIDNgKsGiAFIAM2AqgaIAUgAjYCpBoLIAVByAZqIAVBpBpqIAVBsBpqEIsCA0AgASICQQhqIQEgAigCACIDQQFLDQALIANBAUYiA0EDdCEHIAUoAqQaIgEEQCAFIAE2AqgaIAEQLwsgAiAHaiEBIAMNAAsgBSgCuBohCCAFKAK0GiEMIAUoArAaIQkgBSgC0AYhDiAFKALMBiENIAUoAsgGIREgBigCiAIiAwRAIAYoAowCIgIgAyIBRwRAA0AgAkEMayIBKAIAIgcEQCACQQhrIAc2AgAgBxAvCyABIgIgA0cNAAsgBigCiAIhAQsgBiADNgKMAiABEC8LIAYgDjYCkAIgBiANNgKMAiAGIBE2AogCIAYoApQCIgMEQCAGKAKYAiICIAMiAUcEQANAIAJBDGsiASgCACIHBEAgAkEIayAHNgIAIAcQLwsgASICIANHDQALIAYoApQCIQELIAYgAzYCmAIgARAvCyAGIAk2ApQCIAZCADcCoAIgBiAINgKcAiAGIAw2ApgCDAELIAYoAogCIgMEQCAGKAKMAiICIAMiAUcEQANAIAJBDGsiASgCACIHBEAgAkEIayAHNgIAIAcQLwsgASICIANHDQALIAYoAogCIQELIAYgAzYCjAIgARAvCyAGQQA2ApACIAZCADcCiAIgBigClAIiAwRAIAYoApgCIgIgAyIBRwRAA0AgAkEMayIBKAIAIgcEQCACQQhrIAc2AgAgBxAvCyABIgIgA0cNAAsgBigClAIhAQsgBiADNgKYAiABEC8gBkEANgKcAiAGQQA2ApgCCyAG/QwAAAAAAAAAAAAAAAAAAAAA/QsClAIgBkEANgKkAgsgBEEBaiIEIAUoApwGSA0BDAILCwwQCyAFIAUoArAGIgE2ArQGAkAgGCgCnKgBIgwgGCgCoKgBIglGDQAgBSoCoAZDAAAAP11FDQAgECgCCCIIQQBMDQAgBSgCoBoiASgCNCEOIAUgASgC8AE2AsgGAkAgCyAFQcgGaiIEayIGQQJ1IgMgBUGwBmoiASgCCCIHIAEoAgAiAmtBAnVNBEAgASgCBCACayIHQQJ1Ig0gA0kEQCACIAQgB/wKAAAgASgCBCICIAQgDUECdGoiAyALIANrIgP8CgAAIAEgAiADajYCBAwCCyACIAQgBvwKAAAgASACIANBAnRqNgIEDAELIAIEQCABIAI2AgQgAhAvIAFBADYCCCABQgA3AgBBACEHCwJAIAZBAEgNAEH/////AyAHQQF1IgIgAyACIANLGyAHQfz///8HTxsiAkGAgICABE8NACABIAJBAnQiBxAyIgI2AgQgASACNgIAIAEgAiAHajYCCCACIAQgBvwKAAAgASACIANBAnRqNgIEDAELDBELIAEgBSgCsAZBBGogGCgCoKgBIgEgCSAMa0ECdSICIA5BAm0iAyAIIAMgCEgbIgMgAiADSBtBAnRrIAEQ6gIgBSgCtAYhAQsgBUGwBmogASAFKAK8BiAFKALABhDqAiAFKAKcGiIDIQEgAygCUEEASgRAQQAhAgNAIAMoAlggAkEEdGoiAUF/NgIAIAFBBGogAUEIaiIEKAIAEJYBIAEgBDYCBCABQgA3AgggAkEBaiICIAMoAlBIDQALIAUoApwaIQELIANBADYCTCABIAUoArQGIAUoArAGIgZrIgRBAnUiAzYCtAECQCAEQQBMDQBBACECIAZFBEAgA0EBRwRAIANBfnEhDEEAIQgDQCACQQJ0IgYgASgCvAFqIAI2AgAgASgCwAEgBmpBATYCACABKALEASAGaigCAEEANgIAIAEoAsgBIAJqQQA6AAAgAkEBciIGQQJ0IgcgASgCvAFqIAY2AgAgASgCwAEgB2pBATYCACABKALEASAHaigCAEEANgIAIAEoAsgBIAZqQQA6AAAgAkECaiECIAhBAmoiCCAMRw0ACwsgBEEEcUUNASACQQJ0IgQgASgCvAFqIAI2AgAgASgCwAEgBGpBATYCACABKALEASAEaigCAEEANgIAIAEoAsgBIAJqQQA6AAAMAQsDQCACQQJ0IgQgASgCuAFqIAQgBmooAgA2AgAgASgCvAEgBGogAjYCACABKALAASAEakEBNgIAIAEoAsQBIARqKAIAQQA2AgAgASgCyAEgAmpBADoAACACQQFqIgIgA0cNAAsLIAMgASgCyAFqQQFrQQE6AAAgBSgCoBogBSgCnBoiASABQbQBaiAQKAIEIBAoApQBIBAoApgBEO4CRQ0KEGkhTSAFKAKcGiIBIAUoArQGIAUoArAGa0ECdUEBazYCqAIgBSoCoAYhTiAFKAKgGiE9IAVB/ANqIgMgEEG0AfwKAAAgPSABIAFB0AFqIAMgThCKAkEBIQwgBSgCnAZBAUoEQANAIAUoApwaIgZBADYCTCAGIgIoAlAiDwRAQQAhBwNAIAYoAlggB0EEdGoiCEEIaiIDIQIgAygCACIEIQECQCAERQ0AA0AgAiABIAEoAhBBAEgiCRshAiABQQRqIAEgCRsoAgAiAQ0ACyACIANGDQAgAigCEEEASg0AIAgoAgBB/v///wdLDQACfwNAIAwgBCIBKAIQIgJIBEAgASgCACIEDQEgAQwCCyACIAxODQIgASgCBCIEDQALIAFBBGoLIQJBFBAyIgMgATYCCCADQgA3AgAgAyAMNgIQIAIgAzYCACAIKAIEKAIAIgEEQCAIIAE2AgQgAigCACEDCyADIAMgCCgCCCIJRiIBOgAMAkAgAQ0AA0AgAygCCCICLQAMDQECQCACIAIoAggiASgCACIERgRAAkAgASgCBCIERQ0AIAQtAAwNAAwCCwJAIAMgAigCAEYEQCACIQMMAQsgAiACKAIEIgMoAgAiBDYCBCADIAQEfyAEIAI2AgggAigCCAUgAQs2AgggAigCCCIBIAEoAgAgAkdBAnRqIAM2AgAgAyACNgIAIAIgAzYCCCADKAIIIgEoAgAhAgsgA0EBOgAMIAFBADoADCABIAIoAgQiAzYCACADBEAgAyABNgIICyACIAEoAgg2AgggASgCCCIDIAMoAgAgAUdBAnRqIAI2AgAgAiABNgIEIAEgAjYCCAwDCwJAIARFDQAgBC0ADA0ADAELAkAgAyACKAIARwRAIAIhAwwBCyACIAMoAgQiBDYCACADIAQEfyAEIAI2AgggAigCCAUgAQs2AgggAigCCCIBIAEoAgAgAkdBAnRqIAM2AgAgAyACNgIEIAIgAzYCCCADKAIIIQELIANBAToADCABQQA6AAwgASABKAIEIgIoAgAiAzYCBCADBEAgAyABNgIICyACIAEoAgg2AgggASgCCCIDIAMoAgAgAUdBAnRqIAI2AgAgAiABNgIAIAEgAjYCCAwCCyACQQE6AAwgASABIAlGOgAMIARBAToADCAJIAEiA0cNAAsLIAggCCgCDEEBajYCDCAGKAJQIQ8LIAdBAWoiByAPSQ0ACyAFKAKcGiECCyAGIAxB2BRsaiIBKAK0AiIDIAIoArQCIAEoArgCIANr/AoAACABKALAAiICIAUoApwaKALAAiABKALEAiACa/wKAAAgASgCzAIiAiAFKAKcGigCzAIgASgC0AIgAmv8CgAAIAxBAWoiDCAFKAKcBkgNAAsLEGkhTCAFKAKcGiIBIAEpAwAgTCBNfXw3AwACQAJAIAUoAqAaKAI0IgFBCkgNACABQQF2QQVrIS5BACEOA0AgDiEVEGkhTQJAIBAoAgBBAUcNACAFKAKkBiIPIAUoAqgGIgxGDQADQCAPKAIEIgcgDygCACIGRwRAA0AgB0HoAGsiBCgCVCIDBEAgB0EQayIJKAIAIgIgAyIBRwRAA0AgAkEMayIBKAIAIggEQCACQQhrIAg2AgAgCBAvCyABIgIgA0cNAAsgBCgCVCEBCyAJIAM2AgAgARAvCyAHQSBrIgkoAgAiAwRAIAdBHGsiDigCACICIAMiAUcEQANAIAJBDGsiASgCACIIBEAgAkEIayAINgIAIAgQLwsgASICIANHDQALIAkoAgAhAQsgDiADNgIAIAEQLwsgB0HYAGsoAgAiAQRAIAdB1ABrIAE2AgAgARAvCyAEIgcgBkcNAAsLIA8gBjYCBCAPQQxqIg8gDEcNAAsLIAVBADYCpBogBSAFQaQGajYC4AYgBSAFQaAaajYC3AYgBSAFQaAGajYC2AYgBSAFQZwaajYC0AYgBSAFQZwGajYCzAYgBSAFQaQaajYCyAYgBSAQNgLUBgJAAkACQAJAAkAgBSgCnAYiASAQKAIEIgIgASACSBsiB0EBRgRAIAVByAZqEIkCDAELIAVBADYCuBogBUIANwKwGgJAAkACQCAHQQFrIgZBgICAgARJBEAgBSAGQQJ0IgEQMiIDNgKwGiAFIAEgA2oiBDYCuBpBACECIANBACAB/AsAIAUgBDYCtBogBkUEQCAFQcgGahCJAgwECwNAQQQQMhCaAiEIQSAQMiIBIAg2AgAgASAFKQLIBjcCBCABIAX9AALQBv0LAgwgASAFKALgBjYCHCAFQZgGaiIIQdQAIAEQ9gENAiADIAJBAnRqIgEoAgANBiABIAUoApgGNgIAIAVBADYCmAYgCBC4ARogBiACQQFqIgJHDQALDAILDBoLEEYACyAFQcgGahCJAkEAIQEgB0EBSgRAA0AgAyABQQJ0ahCdAiABQQFqIgEgBkcNAAsLA0AgBEEEaxC4ASIEIANHDQALCyADEC8LIAogGkcEQANAIApB6ABrIgQoAlQiAwRAIApBEGsiBygCACICIAMiAUcEQANAIAJBDGsiASgCACIGBEAgAkEIayAGNgIAIAYQLwsgASICIANHDQALIAQoAlQhAQsgByADNgIAIAEQLwsgCkEgayIHKAIAIgMEQCAKQRxrIggoAgAiAiADIgFHBEADQCACQQxrIgEoAgAiBgRAIAJBCGsgBjYCACAGEC8LIAEiAiADRw0ACyAHKAIAIQELIAggAzYCACABEC8LIApB2ABrKAIAIgEEQCAKQdQAayABNgIAIAEQLwsgBCIKIBpHDQALCyAaIQoCQCAFKAKkBiIGIAUoAqgGIiVGDQADQAJAIAYiFCgCBCIJIAYoAgAiEWsiAkHoAG0hFyAKIBprQegAbSEBAkAgAkEATA0AIBogAUHoAGwiAmohDgJAICggCmtB6ABtIBdOBEAgCiAOayIjQegAbSIBIBdOBEAgCiEGIAkhDAwCCyARIAFB6ABsaiIMIQ0gCiEGAkAgCSAMRwRAA0AgBiANKQMANwMAIAYgDS0ACDoACCAGQQA2AhggBkIANwMQIA0oAhQiASANKAIQIgNrIgRBMG0hAiABIANHBEAgAkHWqtUqTw0DIAYgBBAyIgE2AhQgBiABNgIQIAYgASACQTBsajYCGCANKAIQIgIgDSgCFCIDRwRAA0AgASAC/QADAP0LAwAgASAC/QADIP0LAyAgASAC/QADEP0LAxAgAUEwaiEBIAJBMGoiAiADRw0ACwsgBiABNgIUCyAN/QACHCFPIA39AAIsIVAgDSkCPCFMIA0oAkQhASAGQQA2AlAgBkIANwJIIAYgATYCRCAGIEw3AjwgBiBQ/QsCLCAGIE/9CwIcIA0oAkwiAiANKAJIIgNrIgRBDG0hASACIANHBEAgAUHWqtWqAU8NICAGIAQQMiIHNgJMIAYgBzYCSCAGIAcgAUEMbGo2AlAgDSgCSCIPIA0oAkwiIEcEQANAIAdBADYCCCAHQgA3AgAgDygCBCIBIA8oAgAiAkcEQAJAAkACQCABIAJrIgFBAE4EQCAHIAEQMiIENgIEIAcgBDYCACAHIAQgAUF4cWo2AgggDygCACIIIA8oAgQiGUYEQCAEIQEMBAsgGSAIa0EIayIBQRhJDQEgBCAIa0EQSQ0BIAQgAUEDdkEBaiIkQf7///8DcSIdQQN0IgJqIQEgAiAIaiECQQAhAwNAIAQgA0EDdCIqaiAIICpq/QACAP0LAgAgA0ECaiIDIB1HDQALIB0gJEYNAwwCCwwmCyAIIQIgBCEBCwNAIAEgAikCADcCACABQQhqIQEgAkEIaiICIBlHDQALCyAHIAE2AgQLIAdBDGohByAPQQxqIg8gIEcNAAsLIAYgBzYCTAsgBkEANgJcIAZCADcCVCANKAJYIgIgDSgCVCIDayIEQQxtIQEgAiADRwRAIAFB1qrVqgFPDSAgBiAEEDIiBzYCWCAGIAc2AlQgBiAHIAFBDGxqNgJcIA0oAlQiDyANKAJYIiBHBEADQCAHQQA2AgggB0IANwIAIA8oAgQiASAPKAIAIgJHBEACQAJAAkAgASACayIBQQBOBEAgByABEDIiBDYCBCAHIAQ2AgAgByAEIAFBfHFqNgIIIA8oAgAiCCAPKAIEIhlGBEAgBCEBDAQLIBkgCGtBBGsiAUEMSQ0BIAQgCGtBEEkNASAEIAFBAnZBAWoiJEH8////B3EiHUECdCICaiEBIAIgCGohAkEAIQMDQCAEIANBAnQiKmogCCAqav0AAgD9CwIAIANBBGoiAyAdRw0ACyAdICRGDQMMAgsMJgsgCCECIAQhAQsDQCABIAIoAgA2AgAgAUEEaiEBIAJBBGoiAiAZRw0ACwsgByABNgIECyAHQQxqIQcgD0EMaiIPICBHDQALCyAGIAc2AlgLIAYgDSkDYDcDYCAGQegAaiEGIA1B6ABqIg0gCUcNAAsLICNBAEoNAiAGIQoMAwsMHAsgASAXaiIBQePO2BNJBEAgAkHiztgTICggGmtB6ABtIgNBAXQiBCABIAEgBEkbIANBsafsCU8bIhkEfyAZQePO2BNPDRcgGUHoAGwQMgVBAAsiKGoiBCENAkAgCSARRwRAIAQgF0HoAGxqIQ0gBCEGA0AgBiARKQMANwMAIAYgES0ACDoACCAGQQA2AhggBkIANwMQIBEoAhQiASARKAIQIgNrIgdBMG0hAiABIANHBEAgAkHWqtUqTw0DIAYgBxAyIgE2AhQgBiABNgIQIAYgASACQTBsajYCGCARKAIQIgIgESgCFCIDRwRAA0AgASAC/QADAP0LAwAgASAC/QADIP0LAyAgASAC/QADEP0LAxAgAUEwaiEBIAJBMGoiAiADRw0ACwsgBiABNgIUCyAR/QACHCFPIBH9AAIsIVAgESkCPCFMIBEoAkQhASAGQQA2AlAgBkIANwJIIAYgATYCRCAGIEw3AjwgBiBQ/QsCLCAGIE/9CwIcIBEoAkwiAiARKAJIIgNrIgdBDG0hASACIANHBEAgAUHWqtWqAU8NICAGIAcQMiIHNgJMIAYgBzYCSCAGIAcgAUEMbGo2AlAgESgCSCIPIBEoAkwiHUcEQANAIAdBADYCCCAHQgA3AgAgDygCBCIBIA8oAgAiAkcEQAJAAkACQCABIAJrIgFBAE4EQCAHIAEQMiIINgIEIAcgCDYCACAHIAggAUF4cWo2AgggDygCACIMIA8oAgQiCUYEQCAIIQEMBAsgCSAMa0EIayIBQRhJDQEgCCAMa0EQSQ0BIAggAUEDdkEBaiIjQf7///8DcSIXQQN0IgJqIQEgAiAMaiECQQAhAwNAIAggA0EDdCIgaiAMICBq/QACAP0LAgAgA0ECaiIDIBdHDQALIBcgI0YNAwwCCwwmCyAMIQIgCCEBCwNAIAEgAikCADcCACABQQhqIQEgAkEIaiICIAlHDQALCyAHIAE2AgQLIAdBDGohByAPQQxqIg8gHUcNAAsLIAYgBzYCTAsgBkEANgJcIAZCADcCVCARKAJYIgIgESgCVCIDayIHQQxtIQEgAiADRwRAIAFB1qrVqgFPDSAgBiAHEDIiBzYCWCAGIAc2AlQgBiAHIAFBDGxqNgJcIBEoAlQiDyARKAJYIh1HBEADQCAHQQA2AgggB0IANwIAIA8oAgQiASAPKAIAIgJHBEACQAJAAkAgASACayIBQQBOBEAgByABEDIiCDYCBCAHIAg2AgAgByAIIAFBfHFqNgIIIA8oAgAiDCAPKAIEIglGBEAgCCEBDAQLIAkgDGtBBGsiAUEMSQ0BIAggDGtBEEkNASAIIAFBAnZBAWoiI0H8////B3EiF0ECdCICaiEBIAIgDGohAkEAIQMDQCAIIANBAnQiIGogDCAgav0AAgD9CwIAIANBBGoiAyAXRw0ACyAXICNGDQMMAgsMJgsgDCECIAghAQsDQCABIAIoAgA2AgAgAUEEaiEBIAJBBGoiAiAJRw0ACwsgByABNgIECyAHQQxqIQcgD0EMaiIPIB1HDQALCyAGIAc2AlgLIAYgESkDYDcDYCARQegAaiERIAZB6ABqIgYgDUcNAAsLIA4hASAKIBpGIgJFBEADQCAEQegAayIEIAFB6ABrIgEpAwA3AwAgBCABLQAIOgAIIARBADYCGCAEQgA3AxAgBCABKAIQNgIQIAQgASgCFDYCFCAEIAEoAhg2AhggAUEANgIYIAFCADcDECAB/QACHCFPIAH9AAIsIVAgASkCPCFMIAEoAkQhAyAEQQA2AlAgBEIANwNIIAQgAzYCRCAEIEw3AjwgBCBQ/QsCLCAEIE/9CwIcIAQgASgCSDYCSCAEIAEoAkw2AkwgBCABKAJQNgJQIAFBADYCUCABQgA3A0ggBEEANgJcIARCADcCVCAEIAEoAlQ2AlQgBCABKAJYNgJYIAQgASgCXDYCXCABQQA2AlwgAUIANwJUIAQgASkDYDcDYCABIBpHDQALCyAKIA5HBEADQCANIA4pAwA3AwAgDSAOLQAIOgAIIA1BADYCGCANQgA3AxAgDSAOKAIQNgIQIA0gDigCFDYCFCANIA4oAhg2AhggDkEANgIYIA5CADcDECAO/QACHCFPIA79AAIsIVAgDikCPCFMIA4oAkQhASANQQA2AlAgDUIANwNIIA0gATYCRCANIEw3AjwgDSBQ/QsCLCANIE/9CwIcIA0gDigCSDYCSCANIA4oAkw2AkwgDSAOKAJQNgJQIA5BADYCUCAOQgA3A0ggDUEANgJcIA1CADcCVCANIA4oAlQ2AlQgDSAOKAJYNgJYIA0gDigCXDYCXCAOQQA2AlwgDkIANwJUIA0gDikDYDcDYCANQegAaiENIA5B6ABqIg4gCkcNAAsLIAJFBEADQCAKQegAayIGKAJUIgMEQCAKQRBrIggoAgAiAiADIgFHBEADQCACQQxrIgEoAgAiBwRAIAJBCGsgBzYCACAHEC8LIAEiAiADRw0ACyAGKAJUIQELIAggAzYCACABEC8LIApBIGsiCCgCACIDBEAgCkEcayIMKAIAIgIgAyIBRwRAA0AgAkEMayIBKAIAIgcEQCACQQhrIAc2AgAgBxAvCyABIgIgA0cNAAsgCCgCACEBCyAMIAM2AgAgARAvCyAKQdgAaygCACIBBEAgCkHUAGsgATYCACABEC8LIAYiCiAaRw0ACwsgGUHoAGwgKGohKCAaBEAgGhAvCyAEIRogDSEKDAMLDBwLEDYACyAOIAYiBCAOIBdB6ABsaiICa0HoAG1B6ABsaiIHIQEgByAKSQRAA0AgBCABKQMANwMAIAQgAS0ACDoACCAEQQA2AhggBEIANwMQIAQgASgCEDYCECAEIAEoAhQ2AhQgBCABKAIYNgIYIAFBADYCGCABQgA3AxAgAf0AAhwhTyAB/QACLCFQIAEpAjwhTCABKAJEIQMgBEEANgJQIARCADcDSCAEIAM2AkQgBCBMNwI8IAQgUP0LAiwgBCBP/QsCHCAEIAEoAkg2AkggBCABKAJMNgJMIAQgASgCUDYCUCABQQA2AlAgAUIANwNIIARBADYCXCAEQgA3AlQgBCABKAJUNgJUIAQgASgCWDYCWCAEIAEoAlw2AlwgAUEANgJcIAFCADcCVCAEIAEpA2A3A2AgBEHoAGohBCABQegAaiIBIApJDQALCyACIAZHBEADQCAGQegAayIDIAdB6ABrIggpAwA3AwAgAyAILQAIOgAIIAMoAhAiAQRAIAZB1ABrIgIgATYCACABEC8gAkIANwIAIANBADYCEAsgAyAHQegAayINIgEoAhA2AhAgBkHoAGsiCSANKAIUNgIUIAkgDSgCGDYCGCANQQA2AhggAUIANwMQIAkgDSgCRDYCRCAJIA0pAjw3AjwgCSAN/QACLP0LAiwgCSAN/QACHP0LAhwgCSgCSCIKBEAgBkEcayIPKAIAIgIgCiIBRwRAA0AgAkEMayIBKAIAIhcEQCACQQhrIBc2AgAgFxAvCyABIgIgCkcNAAsgCSgCSCEBCyAPIAo2AgAgARAvIAZBGGtBADYCACAPQQA2AgAgCUEANgJICyAJIAdBIGsiASgCADYCSCAJIA0oAkw2AkwgCSANKAJQNgJQIA1CADcCTCABQQA2AgAgCSgCVCIKBEAgBkEQayIHKAIAIgIgCiIBRwRAA0AgAkEMayIBKAIAIg8EQCACQQhrIA82AgAgDxAvCyABIgIgCkcNAAsgCSgCVCEBCyAHIAo2AgAgARAvIAZBDGtBADYCACAHQQA2AgAgCUEANgJUCyAJIA0oAlQ2AlQgCSANKAJYNgJYIAkgDSgCXDYCXCANQQA2AlwgDUIANwJUIAkgDSkCYDcCYCADIQYgCCIHIA5HDQALCyAMIBFGBEAgBCEKDAELA0AgDiARKQMANwMAIA4gES0ACDoACCAOIBFHBEACQCARKAIUIgYgESgCECICayIKQTBtIgMgDigCGCIIIA4oAhAiAWtBMG1NBEAgDigCFCABayIHQTBtIgggA0kEQCABIAIgB/wKAAAgDigCFCEBIAYgAiAIQTBsaiICRwRAA0AgASAC/QADAP0LAwAgASAC/QADIP0LAyAgASAC/QADEP0LAxAgAUEwaiEBIAJBMGoiAiAGRw0ACwsgDiABNgIUDAILIAEgAiAK/AoAACAOIAEgA0EwbGo2AhQMAQsgAQRAIA4gATYCFCABEC8gDkEANgIYIA5CADcCEEEAIQgLIANB1qrVKk8NBEHVqtUqIAhBMG0iAUEBdCIHIAMgAyAHSRsgAUGq1aoVTxsiAUHWqtUqTw0EIA4gAUEwbCIDEDIiATYCFCAOIAE2AhAgDiABIANqNgIYIA4gAiAGRwR/IAEgAiAKQTBrIgIgAkEwcGtBMGoiAvwKAAAgASACagUgAQs2AhQLIA4gESgCRDYCRCAOIBEpAjw3AjwgDiAR/QACLP0LAiwgDiAR/QACHP0LAhwgDkHIAGogESgCSCARKAJMENQEIA5B1ABqIBEoAlQgESgCWBDTBAsgDiARKQNgNwNgIA5B6ABqIQ4gEUHoAGoiESAMRw0ACyAEIQoLIBQoAgAgFCgCBEcEQCAFKAKcGiIBIAEoAjBBAWo2AjALIBRBDGoiBiAlRw0BDAILCwwXCwJAIBAoAgBBAUcEQCAFKAKcBiEBDAELIBogCkE+IAogGmtB6ABtIg5nQQF0a0EAIAogGkcbEOkCIAUoApwGQQBMDQZBACEJQQAhBgJAA0ACQAJAIAUoApwaIAZB2BRsaiIILQCxAg0AIAgtALACDQAgCEHQAWohAyAJQQAgCSAOSRsiCUEBaiEBIBogCUHoAGxqIQQCQCAVRQRAIAEhCQwBCyAOIAEgASAOSRshAQNAIAlBAWoiCSAOTwRAIAEhCQwCCyAaIAlB6ABsaisDICAEKwMgYQ0ACwsgCCAEKAIENgKsAiAIIAQtAAg6ALICAkAgAyAEQRBqRg0AIAQoAhQiDSAEKAIQIgJrIhFBMG0iDCADKAIIIgcgAygCACIBa0EwbU0EQCADKAIEIAFrIgdBMG0iFCAMSQRAIAEgAiAH/AoAACADKAIEIQEgDSACIBRBMGxqIgJHBEADQCABIAL9AAMA/QsDACABIAL9AAMg/QsDICABIAL9AAMQ/QsDECABQTBqIQEgAkEwaiICIA1HDQALCyADIAE2AgQMAgsgASACIBH8CgAAIAMgASAMQTBsajYCBAwBCyABBEAgAyABNgIEIAEQLyADQQA2AgggA0IANwIAQQAhBwsgDEHWqtUqTw0CQdWq1SogB0EwbSIBQQF0IgcgDCAHIAxLGyABQarVqhVPGyIBQdaq1SpPDQIgAyABQTBsIgcQMiIBNgIEIAMgATYCACADIAEgB2o2AgggAyACIA1HBH8gASACIBFBMGsiAiACQTBwa0EwaiIC/AoAACABIAJqBSABCzYCBAsgAyAEKAJENgI0IAMgBCkCPDcCLCADIAT9AAIs/QsCHCADIAT9AAIc/QsCDCAIQYgCaiIBIARByABqRwRAIAEgBCgCSCAEKAJMENQEIAhBlAJqIAQoAlQgBCgCWBDTBAsgCCAEKQJgNwKgAiAEKAIAIRQgBSgCnBoiCEEANgJMIAgoAlAiEUUNACAGQQhqIQ1BACEMA0AgCCgCWCAMQQR0aiIEQQhqIgMhAiADKAIAIgchAQJAIAdFDQADQCACIAEgASgCECAUSCIPGyECIAFBBGogASAPGygCACIBDQALIAIgA0YNACACKAIQIBRKDQAgBCgCAEH+////B0sNAAJ/A0AgDSAHIgEoAhAiAkgEQCABKAIAIgcNASABDAILIAIgDU4NAiABKAIEIgcNAAsgAUEEagshAkEUEDIiAyABNgIIIANCADcCACADIA02AhAgAiADNgIAIAQoAgQoAgAiAQRAIAQgATYCBCACKAIAIQMLIAMgAyAEKAIIIhFGIgE6AAwCQCABDQADQCADKAIIIgItAAwNAQJAIAIgAigCCCIBKAIAIgdGBEACQCABKAIEIgdFDQAgBy0ADA0ADAILAkAgAyACKAIARgRAIAIhAwwBCyACIAIoAgQiAygCACIHNgIEIAMgBwR/IAcgAjYCCCACKAIIBSABCzYCCCACKAIIIgEgASgCACACR0ECdGogAzYCACADIAI2AgAgAiADNgIIIAMoAggiASgCACECCyADQQE6AAwgAUEAOgAMIAEgAigCBCIDNgIAIAMEQCADIAE2AggLIAIgASgCCDYCCCABKAIIIgMgAygCACABR0ECdGogAjYCACACIAE2AgQgASACNgIIDAMLAkAgB0UNACAHLQAMDQAMAQsCQCADIAIoAgBHBEAgAiEDDAELIAIgAygCBCIHNgIAIAMgBwR/IAcgAjYCCCACKAIIBSABCzYCCCACKAIIIgEgASgCACACR0ECdGogAzYCACADIAI2AgQgAiADNgIIIAMoAgghAQsgA0EBOgAMIAFBADoADCABIAEoAgQiAigCACIDNgIEIAMEQCADIAE2AggLIAIgASgCCDYCCCABKAIIIgMgAygCACABR0ECdGogAjYCACACIAE2AgAgASACNgIIDAILIAJBAToADCABIAEgEUY6AAwgB0EBOgAMIBEgASIDRw0ACwsgBCAEKAIMQQFqNgIMIAgoAlAhEQsgDEEBaiIMIBFJDQALCyAGQQFqIgYgBSgCnAYiAUgNAQwCCwsMGAtBACEIIAFBAEwNBgNAAkAgBSgCnBoiAiAIQdgUbGoiAy0AsQINACADLQCwAg0AIAJBzABqIAhBfxDvAiAFKAKcGiIGQQA2AkwgCEEIaiEJIAYoAlAiDgR/QQAhDANAIAYoAlggDEEEdGoiBEEIaiIDIQIgAygCACIHIQECQCAHRQ0AA0AgAiABIAEoAhAgCUgiDRshAiABQQRqIAEgDRsoAgAiAQ0ACyACIANGDQAgAigCECAJSg0AIAQoAgBB/v///wdLDQACfwNAIAggByIBKAIQIgJIBEAgASgCACIHDQEgAQwCCyACIAhODQIgASgCBCIHDQALIAFBBGoLIQJBFBAyIgMgATYCCCADQgA3AgAgAyAINgIQIAIgAzYCACAEKAIEKAIAIgEEQCAEIAE2AgQgAigCACEDCyADIAMgBCgCCCIORiIBOgAMAkAgAQ0AA0AgAygCCCICLQAMDQECQCACIAIoAggiASgCACIHRgRAAkAgASgCBCIHRQ0AIActAAwNAAwCCwJAIAMgAigCAEYEQCACIQMMAQsgAiACKAIEIgMoAgAiBzYCBCADIAcEfyAHIAI2AgggAigCCAUgAQs2AgggAigCCCIBIAEoAgAgAkdBAnRqIAM2AgAgAyACNgIAIAIgAzYCCCADKAIIIgEoAgAhAgsgA0EBOgAMIAFBADoADCABIAIoAgQiAzYCACADBEAgAyABNgIICyACIAEoAgg2AgggASgCCCIDIAMoAgAgAUdBAnRqIAI2AgAgAiABNgIEIAEgAjYCCAwDCwJAIAdFDQAgBy0ADA0ADAELAkAgAyACKAIARwRAIAIhAwwBCyACIAMoAgQiBzYCACADIAcEfyAHIAI2AgggAigCCAUgAQs2AgggAigCCCIBIAEoAgAgAkdBAnRqIAM2AgAgAyACNgIEIAIgAzYCCCADKAIIIQELIANBAToADCABQQA6AAwgASABKAIEIgIoAgAiAzYCBCADBEAgAyABNgIICyACIAEoAgg2AgggASgCCCIDIAMoAgAgAUdBAnRqIAI2AgAgAiABNgIAIAEgAjYCCAwCCyACQQE6AAwgASABIA5GOgAMIAdBAToADCAOIAEiA0cNAAsLIAQgBCgCDEEBajYCDCAGKAJQIQ4LIAxBAWoiDCAOSQ0ACyAFKAKcGgUgBgtBzABqIAlBfxDvAiAFKAKcBiEBCyAIQQFqIgggAUgNAAsLIAFBAEwNBSAVQQFqIQ5BACERA0ACQCAFKAKcGiARQdgUbGoiD0GxAmoiFC0AAA0AIA9BsAJqIgYtAAANAAJAIA8oAtQBQTBrIh0oAgAiBCAFKAKgGiIDKAL8ASIBSgRAIAQgAWtBAXQhAQJAIA8tALICRQ0AIA8oAqwCIAFMDQAgDygC3AEgFUgNAgsgDyABNgKsAiAPIA42AtwBIA9BAToAsgIgHSgCACEECwJAIA9BiAJqIiMoAgAgDygCjAJGDQAgDygClAIgDygCmAJGDQAgA0HUAWoiByEBAkACQCADKALUASICRQ0AA0AgBCACIgEoAhAiB0gEQCABIQcgASgCACICDQEMAgsgBCAHTA0CIAEoAgQiAg0ACyABQQRqIQcLQSAQMiICIAQ2AhAgAiABNgIIIAJCADcCACACQgA3AhQgAkEANgIcIAcgAjYCACACIQEgAygC0AEoAgAiBARAIAMgBDYC0AEgBygCACEBCyADKALUASABEKQBIAMgAygC2AFBAWo2AtgBCyACQRRqIgEoAgAgASACLQAfIgTAQQBIIgcbIgMhASADQQIgAigCGCAEIAcbIgIgAkECTxsiB2oiBCEIIAMhAiAHBEADQAJAIAFBAWohAiABLQAAQdsARgRAIAIgBEYNASABIAggAS0AAUHfAEYbIQggAiEBDAILIAIiASAERw0BCwsgCCECCyACIANGIAIgBEdxDQAgBSAPKQKgAiJMNwPwAyAFIEw3A7AaIAVByAZqIAMgBUHwA2oQ2AQCQCAFKALIBiIHIAUoAswGQQRrIiBGDQAgDygCmAIhDCAPKAKUAiEIIAchDQJAA0ACQCANKAIAIRcgBUEANgKsGiAFQgA3AqQaIAggDEcEQANAAkAgCCgCACIJIAgoAgQiAUYNAEEAIQMgAUEEayIEKAIAIgEoAgAiJUEDa0ECTwRAIAVB2Cs2AugDIAVB/x42AuQDIAVBphU2AuADQQJBsOQAIAVB4ANqEDQQAAALA0ACfyABKAIIIgJBBUYEQAJ/QQEgA0EBcQ0AGkEAIAEoAgQgF0sNABogASgCDCAXTwshAyABKAIQIQIgAUEQagwBCyABQQhqIRkgA0EBcQRAQQEhAyAZDAELIAEoAgQgF0YhAyAZCyEBIAJBBkYNAAsgJUEDRiADcw0AIAVBADYCuBogBUIANwKwGgJAAkAgBCAJRwRAIAQgCWsiAkEASA0GIAUgAhAyIgM2ArQaIAUgAzYCsBogBSADIAJBfHEiAmoiBDYCuBogAyAJIAL8CgAAIAUgAiADaiICNgK0GiABKAIAQQJJDQIgAiAERg0BIAIgATYCACAFIAJBBGo2ArQaDAILQQAhA0EAIQQgAkECSQ0BCyAEIANrIgJBAnUiJEEBaiIJQYCAgIAETw0GQf////8DIAJBAXUiGSAJIAkgGUkbIAJB/P///wdPGyIZBH8gGUGAgICABE8NHSAZQQJ0EDIFQQALIiUgJEECdGoiCSABNgIAIAlBBGohJCADIARHBEAgJSACIAJBBGtBfHEiAWtqQQRrIgkgBCABa0EEayABQQRq/AoAAAsgBSAlIBlBAnRqNgK4GiAFICQ2ArQaIAUgCTYCsBogA0UNACADEC8LICMgBUGwGmogBUGkGmoQiwIgBSgCsBoiAUUNACAFIAE2ArQaIAEQLwsgCEEMaiIIIAxHDQALIA8oApQCIQwLIAwEQCAPKAKYAiICIAwiAUcEQANAIAJBDGsiASgCACIDBEAgAkEIayADNgIAIAMQLwsgASICIAxHDQALIA8oApQCIQELIA8gDDYCmAIgARAvCyAPIAUoAqQaIgg2ApQCIA8gBSgCqBoiDDYCmAIgDyAFKAKsGjYCnAIgDUEEaiINICBHDQEMAwsLDBwLDBsLIA8gBSkC1AY3AqACIAdFDQAgBxAvCwJAAkACQAJAIB0oAgAgBSgCoBoiASgC3AFGDQAgECgCMCICQQBKIAIgFUxxDQAgDy0AsgJFDQEgLSAPKAKsAmogIUgNAQsgDygC3AFFBEAgLSAPKAKsAmogIUgNBCAPIA42AtwBCyAQLQAXRQ0CIA8gDjYC3AEMAQsgASgCsAFFDQAgFSAuRw0DIA8oAtwBRQ0CIA8oAqwCQdwLTg0DDAILIA9BuBc2AqwCCyAUIQYLIAZBAToAAAsgEUEBaiIRIAUoApwGIgJIDQALIAJBAEwNBkEBIQQgAkEBcSEGQQAhASAFKAKcGiEDIAJBAUcEQCACQX5xIQdBACECA0AgAyABQdgUbGoiCC0AsQJFBEAgCC0AsAJBAEcgBHEhBAsgAyABQQFyQdgUbGoiCC0AsQJFBEAgCC0AsAJBAEcgBHEhBAsgAUECaiEBIAJBAmoiAiAHRw0ACwsCQCAGRQ0AIAMgAUHYFGxqIgEtALECDQAgAS0AsAJBAEcgBHEhBAsgBEEBcQ0FEGkhTCAFKAKcGiICQQA2ArQBIAIgAikDACBMIE19fDcDACAFKAKcBiIEQQBKBEAgBSgCtAYgBSgCsAZrQQJ1IBVqIQZBACEHQQAhAQNAAkAgBSgCnBogAUHYFGxqIgMtALACDQAgAy0AsQINACADIAc2AqgCIAIoArgBIAdBAnRqIAMoAtQBQTBrKAIANgIAIAIoArwBIAIoArQBQQJ0aiAGNgIAIAIoAsABIAIoArQBQQJ0akEBNgIAIAIoAsQBIAIoArQBQQJ0aigCACABNgIAIAIoAsgBIAIoArQBakEBOgAAIAIgAigCtAFBAWoiBzYCtAEgBSgCnAYhBAsgAUEBaiIBIARIDQALIAUoApwaIQILIAUoAqAaIAIgAkG0AWogECgCBCAQKAKUASAQKAKYARDuAkUNERBpIU0gBUEANgKwGiAFKAKcBiIBIBAoAgQiAiABIAJIGyIGQQFGBEAgBUEB/h4CsBoiASAFKAKcBk4NBANAAkAgBSgCnBoiAiABQdgUbGoiAS0AsAINACABLQCxAg0AIAUqAqAGIU4gBSgCoBohPiAFQegAaiIEIBBBtAH8CgAAID4gAiABQdABaiAEIE4QigILIAVBAf4eArAaIgEgBSgCnAZIDQALDAQLIAVBADYC0AYgBUIANwLIBiAGQQFrIgNBgICAgARPDRYgBSADQQJ0IgEQMiIENgLIBiAFIAEgBGoiBzYC0AZBACECIARBACAB/AsAIAUgBzYCzAYgA0UNAgNAQQQQMhCaAiEHQRwQMiIBIBA2AhQgASAHNgIAIAEgBUGgBmo2AhggASAFQaAaajYCECABIAVBnBpqNgIMIAEgBUGcBmo2AgggASAFQbAaajYCBCAFQaQaaiIHQdUAIAEQ9gENAiAEIAJBAnRqIgEoAgANASABIAUoAqQaNgIAIAVBADYCpBogBxC4ARogAyACQQFqIgJHDQALDAILEJkCAAsQRgALIAVBAf4eArAaIgEgBSgCnAZIBEADQAJAIAUoApwaIgIgAUHYFGxqIgEtALACDQAgAS0AsQINACAFKgKgBiFOIAUoAqAaIT8gBUGcAmoiByAQQbQB/AoAACA/IAIgAUHQAWogByBOEIoCCyAFQQH+HgKwGiIBIAUoApwGSA0ACwtBACEBIAUoAsgGIQIgBkEBSgRAA0AgAiABQQJ0ahCdAiABQQFqIgEgA0cNAAsLIAJFDQAgAiAFKALMBiIBRwRAA0AgAUEEaxC4ASIBIAJHDQALCyACEC8LEGkhTCAFKAKcGiIBIAEpAwAgTCBNfXw3AwAgFSAuRw0ACwsgBSgCnAYhAgtBACEORAAAAAAAAPD/IUQgAkEASgRAA0ACQCAFKAKcGiAOQdgUbGoiASINLQCwAg0AAkAgAUHQAWoiBigCDCIIIAYoAgQgBigCACICa0EwbSIBSwRAIAYgCCABaxDSBCAGKAIMIQgMAQsgASAITQ0AIAYgAiAIQTBsajYCBAsCQCAIRQ0AAkAgCEEATARARAAAAAAAAAAAIUMMAQsgBigCACECQQAhA0QAAAAAAAAAACFDQQAhASAIQQRPBEAgCEF8cSEHQQAhBANAIEMgAiABQTBsaioCDLugIAIgAUEBckEwbGoqAgy7oCACIAFBAnJBMGxqKgIMu6AgAiABQQNyQTBsaioCDLugIUMgAUEEaiEBIARBBGoiBCAHRw0ACwsgCEEDcSIERQ0AA0AgQyACIAFBMGxqKgIMu6AhQyABQQFqIQEgA0EBaiIDIARHDQALCyAQKgJcIU4gBiBDOQMYIAYgQyAItyJCoyJFOQMgIAYgTkMAAAAAXgR8IEMgQkQAAAAAAAAUQKBEAAAAAAAAGECjIE67EM0CowUgRQs5AzAgBSALNgLIBiAFQgA3AswGRAAAAAAAAAAAIUNBACEPQQAhCQJAQSAgCCAIQSBMG0EgayIMIAhODQADQCAGKAIAIAxBMGxqKAIAIQMgCyIEIQECfwJAAkAgBSgCzAYiAkUNAANAIAIiASgCECICIANKBEAgASEEIAEoAgAiAg0BDAILIAIgA04NAiABKAIEIgINAAsgAUEEaiEEC0EYEDIiCCADNgIQIAggATYCCCAIQgA3AgAgCEEANgIUIAQgCDYCACAIIQMgBSgCyAYoAgAiAQRAIAUgATYCyAYgBCgCACEDCyADIAMgBSgCzAYiB0YiAToADAJAIAENAANAIAMoAggiAi0ADA0BAkAgAiACKAIIIgEoAgAiBEYEQAJAIAEoAgQiBEUNACAELQAMDQAMAgsCQCADIAIoAgBGBEAgAiEDDAELIAIgAigCBCIDKAIAIgQ2AgQgAyAEBH8gBCACNgIIIAIoAggFIAELNgIIIAIoAggiASABKAIAIAJHQQJ0aiADNgIAIAMgAjYCACACIAM2AgggAygCCCIBKAIAIQILIANBAToADCABQQA6AAwgASACKAIEIgM2AgAgAwRAIAMgATYCCAsgAiABKAIINgIIIAEoAggiAyADKAIAIAFHQQJ0aiACNgIAIAIgATYCBCABIAI2AggMAwsCQCAERQ0AIAQtAAwNAAwBCwJAIAMgAigCAEcEQCACIQMMAQsgAiADKAIEIgQ2AgAgAyAEBH8gBCACNgIIIAIoAggFIAELNgIIIAIoAggiASABKAIAIAJHQQJ0aiADNgIAIAMgAjYCBCACIAM2AgggAygCCCEBCyADQQE6AAwgAUEAOgAMIAEgASgCBCICKAIAIgM2AgQgAwRAIAMgATYCCAsgAiABKAIINgIIIAEoAggiAyADKAIAIAFHQQJ0aiACNgIAIAIgATYCACABIAI2AggMAgsgAkEBOgAMIAEgASAHRjoADCAEQQE6AAwgByABIgNHDQALCyAFIA9BAWoiDzYC0AZBAAwBCyABIggoAhQLIQEgCCABQQFqNgIUIAlBAWohCSAMQQFqIgwgBigCDEgNAAsgBSgCyAYiAyALRg0AIAm3IUkDQCADKAIUtyBJoyJCAnwgQr0iTUIwiKchASBNQoCAgICAgID3P31C//////+fwgFYBEBEAAAAAAAAAAAgTUKAgICAgICA+D9RDQEaIEJEAAAAAAAA8L+gIkIgQiBCRAAAAAAAAKBBoiJFoCBFoSJFIEWiQdD9ASsDACJIoiJKoCJLIEIgQiBCoiJGoiJHIEcgRyBHQaD+ASsDAKIgRkGY/gErAwCiIEJBkP4BKwMAokGI/gErAwCgoKCiIEZBgP4BKwMAoiBCQfj9ASsDAKJB8P0BKwMAoKCgoiBGQej9ASsDAKIgQkHg/QErAwCiQdj9ASsDAKCgoKIgQiBFoSBIoiBCIEWgoiBKIEIgS6GgoKCgDAELAkAgAUHw/wFrQZ+Afk0EQCBNQv///////////wCDUARAIwBBEGsiAUQAAAAAAADwvzkDCCABKwMIRAAAAAAAAAAAowwDCyBNQoCAgICAgID4/wBRDQEgAUHw/wFxQfD/AUcgAUH//wFNcUUEQCBCIEKhIkIgQqMMAwsgQkQAAAAAAAAwQ6K9QoCAgICAgICgA30hTQsgTUKAgICAgICA8z99IkxCNIentyJGQZj9ASsDAKIgTEItiKdB/wBxQQR0IgFBsP4BaisDAKAiRyABQaj+AWorAwAgTSBMQoCAgICAgIB4g32/IAFBqI4CaisDAKEgAUGwjgJqKwMAoaIiQqAiSCBCIEIgQqIiRaIgRSBCQcj9ASsDAKJBwP0BKwMAoKIgQkG4/QErAwCiQbD9ASsDAKCgoiBFQaj9ASsDAKIgRkGg/QErAwCiIEIgRyBIoaCgoKCgIUILIEILoiFCAkAgAygCBCICBEADQCACIgEoAgAiAg0ADAILAAsDQCADKAIIIgEoAgAgA0chQCABIQMgQA0ACwsgQyBCoSFDIAsgASIDRw0ACwsgBiBDOQMoIAVByAZqIAUoAswGEOgCIAYoAgxBIUgNACAGKwMoIBAqAmS7Y0UNACANQQE6ALACIAUoApwaIgEgASgCSEEBajYCSAwBCyAOICIgRCAGKwMwIkNjIgEbISIgQyBEIAEbIUQLIA5BAWoiDiAFKAKcBkgNAAsLIBwgBSgClBogBSgCkBoiAmtBAnUiA0EBa0YNASAFKAKcGiIBICJB2BRsaiIELQCwAkUEQCAEKwPwASAQKgJou2NFDQILIAEgASgCREEBajYCRCAcQQFqIhwgA0gNAAsLIAUoApwaICJB2BRsaiIMKALcASEHIAwoAqwCIQ4gGCAYKAKcqAEiATYCoKgBIAUoArAGIgIoAgAgBSgCoBooAvABRgRAIB4gASACQQRqIAUoArQGIAUoArwGIAUoAsAGa2oQ6gILQQAhCCAHQQBMDQEDQCAMKALQASAIQTBsaiEJAkAgGCgCoKgBIgEgGCgCpKgBRwRAIAEgCSgCADYCACAYIAFBBGo2AqCoAQwBCyABIB4oAgAiA2siBEECdSIRQQFqIgJBgICAgARPDQhB/////wMgBEEBdSIGIAIgAiAGSRsgBEH8////B08bIgYEfyAGQYCAgIAETw0KIAZBAnQQMgVBAAsiDSARQQJ0aiICIAkoAgA2AgAgDSAGQQJ0aiEJIAJBBGohEQJAIAEgA0YNAAJAIARBBGsiBkEsSQ0AIAEgBCANamtBEEkNACACQRBrIQ0gAUEQayEVIAEgBkECdkEBaiIUQfz///8HcSIGQQJ0IgRrIQEgAiAEayECQQAhBANAIA0gBEECdCIPayAVIA9r/QACAP0LAgAgBEEEaiIEIAZHDQALIAYgFEYNAQsDQCACQQRrIgIgAUEEayIBKAIANgIAIAEgA0cNAAsLIBggCTYCpKgBIBggETYCoKgBIBggAjYCnKgBIANFDQAgAxAvCyAHIAhBAWoiCEcNAAsMAQsgBUHFIDYCwAVBAkGT5wAgBUHABWoQNAwICwJAIAwoAtABIgEgDCgC1AFGDQAgBSgCoBoiAigCsAFBAEwNACACKAL8ASECIAEoAgQhQUEAIQQgBUG4GmoiDUEANgIAIAVCADcDsBogQSACa0EBdCATaiEJQQAhAkEAIQNBACEIQQAhBwJAIAwoAtQBIAwoAtABIgFrQQBKBH8DQCABIARBMGwiBmooAgAhAiAFKAKgGiEDAkAgEC0AGEUEQCACIAMoAtwBTg0BCwJAAkAgAygC1AEiAUUNAANAIAIgASgCECIDSARAIAEoAgAiAQ0BDAILIAIgA0wNAiABKAIEIgENAAsLQfEjELcBAAsgBUGwGmogAUEUaiICKAIAIAIgASwAH0EASBsQNxogDCgC0AEhAQsgASAGaiIGKAIAIQIgBSgCoBohAyAQLQA8BEAgAiADKALsAUYgCHIhCAsCQAJAIAIgAygC/AEiA0oEQCAQLQAXRQ0BCyAMKALUASABa0EwbSECIAkhBgwBCyAGKAIEIANrQQF0IBNqIQYCQCAFKAK0GiAFLQC7GiIBIAHAQQBIIgMbRQ0AIAYgEC0ANCICdCEBIAkgAnQhAgJAIBAtABpFDQAgEC0AGwRAIAVByAZqIgMgAqwQiAIgBSgCyAYhCSAFLADTBiERIAVBpBpqIhUgAawQiAIgBSAJIAMgEUEASBs2AlAgBSAFKAKkGiAVIAUsAK8aQQBIGzYCVCAFIAUoArAaIAVBsBpqIAUsALsaQQBIGzYCWEHj5AAgBUHQAGoQswEgBSwArxpBAEgEQCAFKAKkGhAvCyAFLADTBkEATg0BIAUoAsgGEC8MAQsgBSAFKAKwGiAFQbAaaiADGzYCQEG5EyAFQUBrELMBICcQMBoLIAUgAaw3A9AGIAUgAqw3A8gGAkAgBSwAuxpBAE4EQCAWIAUpA7AaNwIAIBYgDSgCADYCCAwBCyAWIAUoArAaIAUoArQaEGsLIAVBADYC7AYgBUIANwLkBiAFIAhBAXE6APAGAkAgGygClKgBIgEgGygCmKgBSQRAIAEgBf0AA8gG/QsDACABIBYpAwA3AxAgASAWKAIINgIYIBZBADYCCCAWQgA3AwAgAUEANgIkIAFCADcCHCABIAUoAuQGNgIcIAEgBSgC6AY2AiAgASAFKALsBjYCJCAFQQA2AuwGIAVCADcC5AYgASAFLQDwBjoAKCAbIAFBMGo2ApSoAQwBCyAmIAVByAZqEOcCIAUoAuQGIgFFDQAgBSABNgLoBiABEC8LIAUsAOMGQQBIBEAgBSgC2AYQLwsCQCAEIAdIDQADQAJAIAwoAtABIAciA0EwbGohBwJAIBsoApSoASICQTBrIggoAiAiASAIKAIkRwRAIAEgB/0AAwD9CwMAIAEgB/0AAyD9CwMgIAEgB/0AAxD9CwMQIAggAUEwajYCIAwBCyABIAJBFGsiESgCACIVa0EwbSICQQFqIglB1qrVKk8NAUHVqtUqIAJBAXQiFCAJIAkgFEkbIAJBqtWqFU8bIgkEfyAJQdaq1SpPDRAgCUEwbBAyBUEACyIUIAJBMGxqIgIgB/0AAwD9CwMAIAIgB/0AAyD9CwMgIAIgB/0AAxD9CwMQIAJBMGohByABIBVHBEADQCACQTBrIgIgAUEwayIB/QADAP0LAwAgAiAB/QADIP0LAyAgAiAB/QADEP0LAxAgASAVRw0ACyARKAIAIQELIBEgAjYCACAIIAc2AiAgCCAUIAlBMGxqNgIkIAFFDQAgARAvCyADQQFqIQcgAyAERw0BDAILCwwSCwJ/QQEgEC0AHEUNABogBSgCoBogBSgCnBogGygClKgBIBsoApCoAWtBMG1BAWsgECoCICAQKgIkENEEQQEgECgCKCIBQQBMDQAaIAUoAqAaIAUoApwaIAEgEC0ALBDQBAshASAQKAJ8IgJFDQAgBSgCoBogBSgCnBogASAQKAKAASACEQYACwJ/IAUsALsaQQBIBEAgBUEANgK0GiAFKAKwGgwBCyAFQQA6ALsaIAVBsBpqC0EAOgAAAn8gBCAMKALUASAMKALQASIBa0EwbSICIARMDQAaIAUoAqAaKAL8ASEDA0AgBCADIAEgBEEwbGooAgBODQEaIARBAWoiBCACRw0ACyACCyIHQQFrIQRBACEIIAYhCQsgBEEBaiIEIAJIDQALIAUtALsaIQIgBiEJIAghAyAFKAK0GgVBAAsgAkH/AXEgAsBBAEgiBBtFDQAgDiATaiAQLQA0IgJ0IQEgCSACdCECAkAgEC0AGkUNACAQLQAbBEAgBUHIBmoiBCACrBCIAiAFKALIBiEGIAUsANMGIQggBUGkGmoiCSABrBCIAiAFIAYgBCAIQQBIGzYCMCAFIAUoAqQaIAkgBSwArxpBAEgbNgI0IAUgBSgCsBogBUGwGmogBSwAuxpBAEgbNgI4QePkACAFQTBqELMBIAUsAK8aQQBIBEAgBSgCpBoQLwsgBSwA0wZBAE4NASAFKALIBhAvDAELIAUgBSgCsBogBUGwGmogBBs2AiBBuRMgBUEgahCzASAnEDAaCyAFIAGsNwPQBiAFIAKsNwPIBgJAIAUsALsaQQBOBEAgFiAFKQOwGjcCACAWIA0oAgA2AggMAQsgFiAFKAKwGiAFKAK0GhBrCyAFQQA2AuwGIAVCADcC5AYgBSADQQFxOgDwBgJAIBsoApSoASIBIBsoApioAUkEQCABIAX9AAPIBv0LAwAgASAWKQMANwMQIAEgFigCCDYCGCAWQQA2AgggFkIANwMAIAFBADYCJCABQgA3AhwgASAFKALkBjYCHCABIAUoAugGNgIgIAEgBSgC7AY2AiQgBUEANgLsBiAFQgA3AuQGIAEgBS0A8AY6ACggGyABQTBqNgKUqAEMAQsgJiAFQcgGahDnAiAFKALkBiIBRQ0AIAUgATYC6AYgARAvCyAFLADjBkEASARAIAUoAtgGEC8LAkAgByAMKALUASAMKALQASIBa0EwbU4NAANAAkAgASAHQTBsaiEDAkAgGygClKgBIgJBMGsiBCgCICIBIAQoAiRHBEAgASAD/QADAP0LAwAgASAD/QADIP0LAyAgASAD/QADEP0LAxAgBCABQTBqNgIgDAELIAEgAkEUayIIKAIAIglrQTBtIgJBAWoiBkHWqtUqTw0BQdWq1SogAkEBdCINIAYgBiANSRsgAkGq1aoVTxsiBgR/IAZB1qrVKk8NDCAGQTBsEDIFQQALIg0gAkEwbGoiAiAD/QADAP0LAwAgAiAD/QADIP0LAyAgAiAD/QADEP0LAxAgAkEwaiEDIAEgCUcEQANAIAJBMGsiAiABQTBrIgH9AAMA/QsDACACIAH9AAMg/QsDICACIAH9AAMQ/QsDECABIAlHDQALIAgoAgAhAQsgCCACNgIAIAQgAzYCICAEIA0gBkEwbGo2AiQgAUUNACABEC8LIAdBAWoiByAMKALUASAMKALQASIBa0EwbUgNAQwCCwsMDgsCf0EBIBAtABxFDQAaIAUoAqAaIAUoApwaIBsoApSoASAbKAKQqAFrQTBtQQFrIBAqAiAgECoCJBDRBEEBIBAoAigiAUEATA0AGiAFKAKgGiAFKAKcGiABIBAtACwQ0AQLIQEgECgCfCICRQ0AIAUoAqAaIAUoApwaIAEgECgCgAEgAhEGAAsgBSwAuxpBAE4NACAFKAKwGhAvCyAOIBNqIRMMAAsACwwJCwwICwwHCwwGCxBHAAsgBUHFIDYC0ANBAkGN6gAgBUHQA2oQNAsgGgRAIAogGkcEQANAIApB6ABrIgQoAlQiAwRAIApBEGsiBygCACICIAMiAUcEQANAIAJBDGsiASgCACIGBEAgAkEIayAGNgIAIAYQLwsgASICIANHDQALIAQoAlQhAQsgByADNgIAIAEQLwsgCkEgayIHKAIAIgMEQCAKQRxrIggoAgAiAiADIgFHBEADQCACQQxrIgEoAgAiBgRAIAJBCGsgBjYCACAGEC8LIAEiAiADRw0ACyAHKAIAIQELIAggAzYCACABEC8LIApB2ABrKAIAIgEEQCAKQdQAayABNgIAIAEQLwsgBCIKIBpHDQALCyAaEC8LIAUoAqQGIgMEQCAFKAKoBiIBIAMiAkcEQANAIAFBDGsiCigCACIEBEAgAUEIayIJKAIAIgcgBCICRwRAA0AgB0HoAGsiCCgCVCIGBEAgB0EQayIOKAIAIgIgBiIBRwRAA0AgAkEMayIBKAIAIgwEQCACQQhrIAw2AgAgDBAvCyABIgIgBkcNAAsgCCgCVCEBCyAOIAY2AgAgARAvCyAHQSBrIg4oAgAiBgRAIAdBHGsiDSgCACICIAYiAUcEQANAIAJBDGsiASgCACIMBEAgAkEIayAMNgIAIAwQLwsgASICIAZHDQALIA4oAgAhAQsgDSAGNgIAIAEQLwsgB0HYAGsoAgAiAQRAIAdB1ABrIAE2AgAgARAvCyAIIgcgBEcNAAsgCigCACECCyAJIAQ2AgAgAhAvCyAKIgEgA0cNAAsgBSgCpAYhAgsgBSADNgKoBiACEC8LIAUoArAGIgEEQCAFIAE2ArQGIAEQLwsgBSgCvAYiAUUNACAFIAE2AsAGIAEQLwsgBSgCkBoiAUUNACABEC8LIAVBwBpqJAAgLEHAAWokAEHQuwMoAgAhAiMAQeABayIBJAAQaSFNQQRB4PkAQQAQNCACKQMAIUwgAUGAETYC0AEgASBMtEMAAHpElbs5A9gBQQRB6+MAIAFB0AFqEDQgAigCgAIiAwRAIAMoAkAhBCADKAI8IQYgAygCOCEKIAMoAjQhByADKAIwIQggASADKQJENwLEASABQYARNgLAAUEEQaHmACABQcABahA0IAIoAoACKQMoIUwgAUGAETYCsAEgASBMtEMAAHpElbs5A7gBQQRBr+MAIAFBsAFqEDQgAigCgAIpAwAhTCABQQEgCCAIQQFMGyIDNgKgASABIEy0Q28SgzqUIk4gA7KVuzkDqAEgAUGAETYCkAEgASBOuzkDmAFBBEGb9QAgAUGQAWoQNCACKAKAAikDCCFMIAFBASAHIAdBAUwbIgM2AoABIAEgTLRDbxKDOpQiTiADspW7OQOIASABQYARNgJwIAEgTrs5A3hBBEHX9QAgAUHwAGoQNCACKAKAAikDECFMIAFBASAKIApBAUwbIgM2AmAgASBMtENvEoM6lCJOIAOylbs5A2ggAUGAETYCUCABIE67OQNYQQRBk/YAIAFB0ABqEDQgAigCgAIpAxghTCABQUBrQQEgBiAGQQFMGyIDNgIAIAEgTLRDbxKDOpQiTiADspW7OQNIIAFBgBE2AjAgASBOuzkDOEEEQc/2ACABQTBqEDQgAigCgAIpAyAhTCABQQEgBCAEQQFMGyIDNgIgIAEgTLRDbxKDOpQiTiADspW7OQMoIAFBgBE2AhAgASBOuzkDGEEEQd/0ACABQRBqEDQLIAIpAwghTCABQYARNgIAIAEgTSBMfbRDAAB6RJW7OQMIQQRBzeMAIAEQNCABQeABaiQAICsQBCApEAQgLxAEIBIoAvgBIgFFDQAgEiABNgL8ASABEC8LIBJB0ANqJAAgAA8LEDYACxsAQaCRNSEAA0AgAEEMaxAzIgBBgI81Rw0ACwuqAgACQEH4ijX+EgAAQQFxDQBB+Io1EFRFDQACQEGgkTX+EgAAQQFxDQBBoJE1EFRFDQBBoJE1EFMLQYCPNUHqCBBBQYyPNUHhCBBBQZiPNUH7HBBBQaSPNUGqGhBBQbCPNUG2CRBBQbyPNUHZIhBBQciPNUHyCBBBQdSPNUHXChBBQeCPNUHhFBBBQeyPNUHQFBBBQfiPNUHYFBBBQYSQNUHrFBBBQZCQNUHOGRBBQZyQNUGdKBBBQaiQNUGgFRBBQbSQNUHSExBBQcCQNUG2CRBBQcyQNUGhFhBBQdiQNUHpGRBBQeSQNUGOHRBBQfCQNUHsFRBBQfyQNUHODxBBQYiRNUG+ChBBQZSRNUHgJhBBQfSKNUGAjzU2AgBB+Io1EFMLQfSKNSgCAAsbAEH4jjUhAANAIABBDGsQUCIAQdCNNUcNAAsL3gEAAkBB8Io1/hIAAEEBcQ0AQfCKNRBURQ0AAkBB+I41/hIAAEEBcQ0AQfiONRBURQ0AQfiONRBTC0HQjTVB9JgDEEBB3I01QZCZAxBAQeiNNUGsmQMQQEH0jTVBzJkDEEBBgI41QfSZAxBAQYyONUGYmgMQQEGYjjVBtJoDEEBBpI41QdiaAxBAQbCONUHomgMQQEG8jjVB+JoDEEBByI41QYibAxBAQdSONUGYmwMQQEHgjjVBqJsDEEBB7I41QbibAxBAQeyKNUHQjTU2AgBB8Io1EFMLQeyKNSgCAAsbAEHIjTUhAANAIABBDGsQMyIAQaCMNUcNAAsL0AEAAkBB6Io1/hIAAEEBcQ0AQeiKNRBURQ0AAkBByI01/hIAAEEBcQ0AQciNNRBURQ0AQciNNRBTC0GgjDVBoQkQQUGsjDVBqAkQQUG4jDVBhgkQQUHEjDVBjgkQQUHQjDVB/QgQQUHcjDVBrwkQQUHojDVBmAkQQUH0jDVBnRYQQUGAjTVBixcQQUGMjTVBvB8QQUGYjTVBxCUQQUGkjTVBzAoQQUGwjTVBrBsQQUG8jTVB2A8QQUHkijVBoIw1NgIAQeiKNRBTC0HkijUoAgALCwAgAEGk9gIQugELCwAgAEGbIRCmARoLCwAgAEGQ9gIQugELCwAgAEGYHxCmARoLDAAgACABQRBqEOcBCwwAIAAgAUEMahDnAQsHACAALAAJCwcAIAAsAAgLDAAgABCgAxogABAvCwwAIAAQoQMaIAAQLwsVACAAKAIIIgBFBEBBAQ8LIAAQqAMLkQEBB38DQAJAIAQgCE0NACACIANGDQBBASEHIAAoAgghBSMAQRBrIgYkACAGIAU2AgwgBkEIaiAGQQxqEH8hC0EAIAIgAyACayABQcyINSABGxDwASEFIAsQfiAGQRBqJAACQAJAIAVBAmoOAwICAQALIAUhBwsgCEEBaiEIIAcgCWohCSACIAdqIQIMAQsLIAkLRgECfyAAKAIIIQIjAEEQayIBJAAgASACNgIMIAFBCGogAUEMahB/EH4gAUEQaiQAIAAoAggiAEUEQEEBDwsgABCoA0EBRguNAQEDfyMAQRBrIgYkACAEIAI2AgACf0ECIAZBDGoiBUEAIAAoAggQrAIiAEEBakECSQ0AGkEBIABBAWsiAiADIAQoAgBrSw0AGgN/IAIEfyAFLQAAIQAgBCAEKAIAIgFBAWo2AgAgASAAOgAAIAJBAWshAiAFQQFqIQUMAQVBAAsLCyEHIAZBEGokACAHC+IGAQ9/IwBBEGsiESQAIAIhCANAAkAgAyAIRgRAIAMhCAwBCyAILQAARQ0AIAhBAWohCAwBCwsgByAFNgIAIAQgAjYCAANAAkACfwJAIAIgA0YNACAFIAZGDQAgESABKQIANwMIIAAoAgghCiMAQRBrIhAkACAQIAo2AgwgEEEIaiAQQQxqEH8hFSAIIAJrIQ5BACELIwBBkAhrIg0kACANIAQoAgAiCjYCDCAGIAVrQQJ1QYACIAUbIQwgBSANQRBqIAUbIQ8CQAJAAkACQCAKRQ0AIAxFDQADQCAOQQJ2IQkCQCAOQYMBSw0AIAkgDE8NACAKIQkMBAsgDyANQQxqIAkgDCAJIAxJGyABEOIDIRIgDSgCDCEJIBJBf0YEQEEAIQxBfyELDAMLIAwgEkEAIA8gDUEQakcbIhRrIQwgDyAUQQJ0aiEPIAogDmogCWtBACAJGyEOIAsgEmohCyAJRQ0CIAkhCiAMDQALDAELIAohCQsgCUUNAQsgDEUNACAORQ0AIAshCgNAAkACQCAPIAkgDiABEPABIgtBAmpBAk0EQAJAAkAgC0EBag4CBgABCyANQQA2AgwMAgsgAUEANgIADAELIA0gDSgCDCALaiIJNgIMIApBAWohCiAMQQFrIgwNAQsgCiELDAILIA9BBGohDyAOIAtrIQ4gCiELIA4NAAsLIAUEQCAEIA0oAgw2AgALIA1BkAhqJAAgFRB+IBBBEGokAAJAAkACQAJAIAtBf0YEQANAAkAgByAFNgIAIAIgBCgCAEYNAEEBIQYCQAJAAkAgBSACIAggAmsgEUEIaiAAKAIIEKkDIgFBAmoOAwgAAgELIAQgAjYCAAwFCyABIQYLIAIgBmohAiAHKAIAQQRqIQUMAQsLIAQgAjYCAAwFCyAHIAcoAgAgC0ECdGoiBTYCACAFIAZGDQMgBCgCACECIAMgCEYEQCADIQgMCAsgBSACQQEgASAAKAIIEKkDRQ0BC0ECDAQLIAcgBygCAEEEajYCACAEIAQoAgBBAWoiAjYCACACIQgDQCADIAhGBEAgAyEIDAYLIAgtAABFDQUgCEEBaiEIDAALAAsgBCACNgIAQQEMAgsgBCgCACECCyACIANHCyEWIBFBEGokACAWDwsgBygCACEFDAALAAu4BQENfyMAQRBrIg4kACACIQgDQAJAIAMgCEYEQCADIQgMAQsgCCgCAEUNACAIQQRqIQgMAQsLIAcgBTYCACAEIAI2AgADQAJAAkACQCACIANGDQAgBSAGRg0AIA4gASkCADcDCEEBIRAgACgCCCEJIwBBEGsiDyQAIA8gCTYCDCAPQQhqIA9BDGoQfyEUIAUhCSAIIAJrQQJ1IREgBiAFayEKQQAhDCMAQRBrIhIkAAJAIAQoAgAiC0UNACARRQ0AIApBACAFGyEKA0AgEkEMaiAJIApBBEkbIAsoAgAQwgIiDUF/RgRAQX8hDAwCCyAJBH8gCkEDTQRAIAogDUkNAyAJIBJBDGogDRB7GgsgCiANayEKIAkgDWoFQQALIQkgCygCAEUEQEEAIQsMAgsgDCANaiEMIAtBBGohCyARQQFrIhENAAsLIAkEQCAEIAs2AgALIBJBEGokACAUEH4gD0EQaiQAAkACQAJAAkACQCAMQQFqDgIABgELIAcgBTYCAANAAkAgAiAEKAIARg0AIAUgAigCACAAKAIIEKwCIgFBf0YNACAHIAcoAgAgAWoiBTYCACACQQRqIQIMAQsLIAQgAjYCAAwBCyAHIAcoAgAgDGoiBTYCACAFIAZGDQIgAyAIRgRAIAQoAgAhAiADIQgMBwsgDkEEakEAIAAoAggQrAIiCEF/Rw0BC0ECIRAMAwsgDkEEaiECIAYgBygCAGsgCEkNAgNAIAgEQCACLQAAIQUgByAHKAIAIglBAWo2AgAgCSAFOgAAIAhBAWshCCACQQFqIQIMAQsLIAQgBCgCAEEEaiICNgIAIAIhCANAIAMgCEYEQCADIQgMBQsgCCgCAEUNBCAIQQRqIQgMAAsACyAEKAIAIQILIAIgA0chEAsgDkEQaiQAIBAPCyAHKAIAIQUMAAsACwwAIAAQsgMaIAAQLwtaAQF/IwBBEGsiACQAIAAgBDYCDCAAIAMgAms2AggjAEEQayIBJAAgAEEIaiICKAIAIABBDGoiAygCAEkhBCABQRBqJAAgAiADIAQbKAIAIQUgAEEQaiQAIAULNAADQCABIAJGRQRAIAQgAyABLAAAIgAgAEEASBs6AAAgBEEBaiEEIAFBAWohAQwBCwsgAgsMACACIAEgAUEASBsLKgADQCABIAJGRQRAIAMgAS0AADoAACADQQFqIQMgAUEBaiEBDAELCyACCz0AA0AgASACRwRAIAEgASwAACIAQQBOBH9BwN4CKAIAIABBAnRqKAIABSAACzoAACABQQFqIQEMAQsLIAILIgAgAUEATgR/QcDeAigCACABQf8BcUECdGooAgAFIAELwAs9AANAIAEgAkcEQCABIAEsAAAiAEEATgR/QbjSAigCACAAQQJ0aigCAAUgAAs6AAAgAUEBaiEBDAELCyACCyIAIAFBAE4Ef0G40gIoAgAgAUH/AXFBAnRqKAIABSABC8ALDAAgABCrAxogABAvCzUAA0AgASACRkUEQCAEIAEoAgAiACADIABBgAFJGzoAACAEQQFqIQQgAUEEaiEBDAELCyACCw4AIAEgAiABQYABSRvACyoAA0AgASACRkUEQCADIAEsAAA2AgAgA0EEaiEDIAFBAWohAQwBCwsgAgs+AANAIAEgAkcEQCABIAEoAgAiAEH/AE0Ef0HA3gIoAgAgAEECdGooAgAFIAALNgIAIAFBBGohAQwBCwsgAgseACABQf8ATQR/QcDeAigCACABQQJ0aigCAAUgAQsLPgADQCABIAJHBEAgASABKAIAIgBB/wBNBH9BuNICKAIAIABBAnRqKAIABSAACzYCACABQQRqIQEMAQsLIAILHgAgAUH/AE0Ef0G40gIoAgAgAUECdGooAgAFIAELC0EAAkADQCACIANGDQECQCACKAIAIgBB/wBLDQAgAEECdEGQ7QJqKAIAIAFxRQ0AIAJBBGohAgwBCwsgAiEDCyADC0AAA0ACQCACIANHBH8gAigCACIAQf8ASw0BIABBAnRBkO0CaigCACABcUUNASACBSADCw8LIAJBBGohAgwACwALSQEBfwNAIAEgAkZFBEBBACEAIAMgASgCACIEQf8ATQR/IARBAnRBkO0CaigCAAVBAAs2AgAgA0EEaiEDIAFBBGohAQwBCwsgAgslAEEAIQAgAkH/AE0EfyACQQJ0QZDtAmooAgAgAXFBAEcFQQALCw8AIAAgACgCACgCBBEBAAsUACAAQQBBAf4eApCKNUEBajYCBAsMACAAEK8DGiAAEC8LsBEBA39B7JY1QQA2AgBB6JY1QbigAzYCAEHoljVBkPgCNgIAQeiWNUHI7AI2AgAjAEEQayIAJABB8JY1QgA3AwAgAEEANgIMQfiWNUEANgIAQfiXNUEAOgAAIABB8JY1NgIEIAAoAgQaIABBADoACiMAQRBrIgEkAEHwljUQmANBHkkEQBA2AAsgAUEIakGAlzVBHhCXA0H0ljUgASgCCCICNgIAQfCWNSACNgIAQfiWNSACIAEoAgxBAnRqNgIAQfiWNSgCABpB8JY1KAIAGiABQRBqJABB8JY1QR4QsQMgAEEBOgAKIABBEGokAEGAmDVB5TMQpgEaQfSWNSgCABpB8JY1KAIAGkHwljUQsANB+JY1KAIAGkH0ljUoAgAaQfCWNSgCABpBpJQ1QQA2AgBBoJQ1QbigAzYCAEGglDVBkPgCNgIAQaCUNUHkgAM2AgBB6JY1QaCUNUHQiDUQTBBOQayUNUEANgIAQaiUNUG4oAM2AgBBqJQ1QZD4AjYCAEGolDVBhIEDNgIAQeiWNUGolDVB2Ig1EEwQTkG0lDVBADYCAEGwlDVBuKADNgIAQbCUNUGQ+AI2AgBBvJQ1QQA6AABBuJQ1QQA2AgBBsJQ1QdzsAjYCAEG4lDVBkO0CNgIAQeiWNUGwlDVBnIo1EEwQTkHElDVBADYCAEHAlDVBuKADNgIAQcCUNUGQ+AI2AgBBwJQ1Qcj4AjYCAEHoljVBwJQ1QZSKNRBMEE5BzJQ1QQA2AgBByJQ1QbigAzYCAEHIlDVBkPgCNgIAQciUNUHc+QI2AgBB6JY1QciUNUGkijUQTBBOQdSUNUEANgIAQdCUNUG4oAM2AgBB0JQ1QZD4AjYCAEHQlDVBmPUCNgIAQdiUNRBENgIAQeiWNUHQlDVBrIo1EEwQTkHklDVBADYCAEHglDVBuKADNgIAQeCUNUGQ+AI2AgBB4JQ1QfD6AjYCAEHoljVB4JQ1QbSKNRBMEE5B7JQ1QQA2AgBB6JQ1QbigAzYCAEHolDVBkPgCNgIAQeiUNUHY/AI2AgBB6JY1QeiUNUHEijUQTBBOQfSUNUEANgIAQfCUNUG4oAM2AgBB8JQ1QZD4AjYCAEHwlDVB5PsCNgIAQeiWNUHwlDVBvIo1EEwQTkH8lDVBADYCAEH4lDVBuKADNgIAQfiUNUGQ+AI2AgBB+JQ1Qcz9AjYCAEHoljVB+JQ1QcyKNRBMEE5BhJU1QQA2AgBBgJU1QbigAzYCAEGAlTVBkPgCNgIAQYiVNUGu2AA7AQBBgJU1Qcj1AjYCACMAQRBrIgAkAEGMlTVCADcCAEGUlTVBADYCACAAQRBqJABB6JY1QYCVNUHUijUQTBBOQZyVNUEANgIAQZiVNUG4oAM2AgBBmJU1QZD4AjYCAEGglTVCroCAgMAFNwIAQZiVNUHw9QI2AgAjAEEQayIAJABBqJU1QgA3AgBBsJU1QQA2AgAgAEEQaiQAQeiWNUGYlTVB3Io1EEwQTkG8lTVBADYCAEG4lTVBuKADNgIAQbiVNUGQ+AI2AgBBuJU1QaSBAzYCAEHoljVBuJU1QeCINRBMEE5BxJU1QQA2AgBBwJU1QbigAzYCAEHAlTVBkPgCNgIAQcCVNUGYgwM2AgBB6JY1QcCVNUHoiDUQTBBOQcyVNUEANgIAQciVNUG4oAM2AgBByJU1QZD4AjYCAEHIlTVB7IQDNgIAQeiWNUHIlTVB8Ig1EEwQTkHUlTVBADYCAEHQlTVBuKADNgIAQdCVNUGQ+AI2AgBB0JU1QdSGAzYCAEHoljVB0JU1QfiINRBMEE5B3JU1QQA2AgBB2JU1QbigAzYCAEHYlTVBkPgCNgIAQdiVNUGsjgM2AgBB6JY1QdiVNUGgiTUQTBBOQeSVNUEANgIAQeCVNUG4oAM2AgBB4JU1QZD4AjYCAEHglTVBwI8DNgIAQeiWNUHglTVBqIk1EEwQTkHslTVBADYCAEHolTVBuKADNgIAQeiVNUGQ+AI2AgBB6JU1QbSQAzYCAEHoljVB6JU1QbCJNRBMEE5B9JU1QQA2AgBB8JU1QbigAzYCAEHwlTVBkPgCNgIAQfCVNUGokQM2AgBB6JY1QfCVNUG4iTUQTBBOQfyVNUEANgIAQfiVNUG4oAM2AgBB+JU1QZD4AjYCAEH4lTVBnJIDNgIAQeiWNUH4lTVBwIk1EEwQTkGEljVBADYCAEGAljVBuKADNgIAQYCWNUGQ+AI2AgBBgJY1QcCTAzYCAEHoljVBgJY1QciJNRBMEE5BjJY1QQA2AgBBiJY1QbigAzYCAEGIljVBkPgCNgIAQYiWNUHklAM2AgBB6JY1QYiWNUHQiTUQTBBOQZSWNUEANgIAQZCWNUG4oAM2AgBBkJY1QZD4AjYCAEGQljVBiJYDNgIAQeiWNUGQljVB2Ik1EEwQTkGcljVBADYCAEGYljVBuKADNgIAQZiWNUGQ+AI2AgBBoJY1QfCfAzYCAEGYljVBnIgDNgIAQaCWNUHMiAM2AgBB6JY1QZiWNUGAiTUQTBBOQayWNUEANgIAQaiWNUG4oAM2AgBBqJY1QZD4AjYCAEGwljVBlKADNgIAQaiWNUGkigM2AgBBsJY1QdSKAzYCAEHoljVBqJY1QYiJNRBMEE5BvJY1QQA2AgBBuJY1QbigAzYCAEG4ljVBkPgCNgIAQcCWNRCSA0G4ljVBkIwDNgIAQeiWNUG4ljVBkIk1EEwQTkHMljVBADYCAEHIljVBuKADNgIAQciWNUGQ+AI2AgBB0JY1EJIDQciWNUGsjQM2AgBB6JY1QciWNUGYiTUQTBBOQdyWNUEANgIAQdiWNUG4oAM2AgBB2JY1QZD4AjYCAEHYljVBrJcDNgIAQeiWNUHYljVB4Ik1EEwQTkHkljVBADYCAEHgljVBuKADNgIAQeCWNUGQ+AI2AgBB4JY1QaSYAzYCAEHoljVB4JY1QeiJNRBMEE4LnAIAIwBBEGsiAyQAAkAgBS0AC0EHdkUEQCAAIAUoAgg2AgggACAFKQIANwIADAELIAUoAgAhAiAFKAIEIQUjAEEQayIEJAACQAJAAkAgBUECSQRAIAAiASAALQALQYABcSAFcjoACyAAIAAtAAtB/wBxOgALDAELIAVB7////wNLDQEgBEEIaiAAIAVBAk8EfyAFQQRqQXxxIgEgAUEBayIBIAFBAkYbBUEBC0EBahC5ASAEKAIMGiAAIAQoAggiATYCACAAIAAoAghBgICAgHhxIAQoAgxB/////wdxcjYCCCAAIAAoAghBgICAgHhyNgIIIAAgBTYCBAsgASACIAVBAWoQngEgBEEQaiQADAELEE0ACwsgA0EQaiQACwkAIAAgBRDnAQvfBgEPfyMAQeADayIAJAAgAEHcA2oiBiADKAIcIgc2AgAgB0EEakEB/h4CABogBhBsIQoCfyAFLQALQQd2BEAgBSgCBAwBCyAFLQALQf8AcQsEQAJ/IAUtAAtBB3YEQCAFKAIADAELIAULKAIAIApBLSAKKAIAKAIsEQQARiELCyACIAsgAEHcA2ogAEHYA2ogAEHUA2ohFCAAQdADaiEQIwBBEGsiBiQAIABBxANqIgJCADcCACACQQA2AgggBkEQaiQAIBQgECETIAIiDCEPIwBBEGsiAiQAIABBuANqIgZCADcCACAGQQA2AgggAkEQaiQAIBMgDyESIAYhDiMAQRBrIgIkACAAQawDaiIHQgA3AgAgB0EANgIIIAJBEGokACASIA4gByAAQagDahC2AyAAQfYBNgIQIABBCGpBACAAQRBqIgIQUSEIAkACfwJ/IAUtAAtBB3YEQCAFKAIEDAELIAUtAAtB/wBxCyAAKAKoA0oEQAJ/IAUtAAtBB3YEQCAFKAIEDAELIAUtAAtB/wBxCyEJIAAoAqgDIg0CfyAGLQALQQd2BEAgBigCBAwBCyAGLQALQf8AcQsCfyAHLQALQQd2BEAgBygCBAwBCyAHLQALQf8AcQsgCSANa0EBdGpqakEBagwBCyAAKAKoAwJ/IActAAtBB3YEQCAHKAIEDAELIActAAtB/wBxCwJ/IAYtAAtBB3YEQCAGKAIEDAELIAYtAAtB/wBxC2pqQQJqCyIJQeUASQ0AIAlBAnQQOyEJIAgoAgAhAiAIIAk2AgAgAgRAIAIgCCgCBBEBAAsgCCgCACICDQAQRgALIAIgAEEEaiAAIAMoAgQCfyAFLQALQQd2BEAgBSgCAAwBCyAFCwJ/IAUtAAtBB3YEQCAFKAIADAELIAULAn8gBS0AC0EHdgRAIAUoAgQMAQsgBS0AC0H/AHELQQJ0aiAKIAsgAEHYA2ogACgC1AMgACgC0AMgDCAGIAcgACgCqAMQtQMgASACIAAoAgQgACgCACADIAQQlAEhESAIKAIAIQEgCEEANgIAIAEEQCABIAgoAgQRAQALIAcQUBogBhBQGiAMEDMaIAAoAtwDIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAQeADaiQAIBEL6AcBEn8jAEGgCGsiACQAIAAgBTcDECAAIAY3AxggACAAQbAHaiIHNgKsByAHQeQAQbYeIABBEGoQxQIhCSAAQfYBNgKQBCAAQYgEakEAIABBkARqIg4QUSEMIABB9gE2ApAEIABBgARqQQAgDhBRIQoCQCAJQeQATwRAEEQhByAAIAU3AwAgACAGNwMIIABBrAdqIAdBth4gABCLASIJQX9GDQEgDCgCACEHIAwgACgCrAc2AgAgBwRAIAcgDCgCBBEBAAsgCUECdBA7IQggCigCACEHIAogCDYCACAHBEAgByAKKAIEEQEACyAKKAIAIg5FDQELIABB/ANqIgcgAygCHCIINgIAIAhBBGpBAf4eAgAaIAcQbCIRIgcgACgCrAciCCAIIAlqIA4gBygCACgCMBEIABogCUEASgRAIAAoAqwHLQAAQS1GIQ8LIAIgDyAAQfwDaiAAQfgDaiAAQfQDaiEYIABB8ANqIRQjAEEQayIHJAAgAEHkA2oiAkIANwIAIAJBADYCCCAHQRBqJAAgGCAUIRcgAiIQIRMjAEEQayIHJAAgAEHYA2oiAkIANwIAIAJBADYCCCAHQRBqJAAgFyATIRYgAiIHIRIjAEEQayIIJAAgAEHMA2oiAkIANwIAIAJBADYCCCAIQRBqJAAgFiASIAIiCCAAQcgDahC2AyAAQfYBNgIwIABBKGpBACAAQTBqIgIQUSELAn8gACgCyAMiDSAJSARAAn8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELAn8gCC0AC0EHdgRAIAgoAgQMAQsgCC0AC0H/AHELIAkgDWtBAXRqaiANakEBagwBCyAAKALIAwJ/IAgtAAtBB3YEQCAIKAIEDAELIAgtAAtB/wBxCwJ/IActAAtBB3YEQCAHKAIEDAELIActAAtB/wBxC2pqQQJqCyINQeUATwRAIA1BAnQQOyENIAsoAgAhAiALIA02AgAgAgRAIAIgCygCBBEBAAsgCygCACICRQ0BCyACIABBJGogAEEgaiADKAIEIA4gDiAJQQJ0aiARIA8gAEH4A2ogACgC9AMgACgC8AMgECAHIAggACgCyAMQtQMgASACIAAoAiQgACgCICADIAQQlAEhFSALKAIAIQEgC0EANgIAIAEEQCABIAsoAgQRAQALIAgQUBogBxBQGiAQEDMaIAAoAvwDIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAKKAIAIQEgCkEANgIAIAEEQCABIAooAgQRAQALIAwoAgAhASAMQQA2AgAgAQRAIAEgDCgCBBEBAAsgAEGgCGokACAVDwsQRgAL2QYBD38jAEGwAWsiACQAIABBrAFqIgYgAygCHCIHNgIAIAdBBGpBAf4eAgAaIAYQciEKAn8gBS0AC0EHdgRAIAUoAgQMAQsgBS0AC0H/AHELBEACfyAFLQALQQd2BEAgBSgCAAwBCyAFCy0AACAKQS0gCigCACgCHBEEAEH/AXFGIQsLIAIgCyAAQawBaiAAQagBaiAAQacBaiEUIABBpgFqIRAjAEEQayIGJAAgAEGYAWoiAkIANwIAIAJBADYCCCAGQRBqJAAgFCAQIRMgAiIMIQ8jAEEQayICJAAgAEGMAWoiBkIANwIAIAZBADYCCCACQRBqJAAgEyAPIRIgBiEOIwBBEGsiAiQAIABBgAFqIgdCADcCACAHQQA2AgggAkEQaiQAIBIgDiAHIABB/ABqELkDIABB9gE2AhAgAEEIakEAIABBEGoiAhBRIQgCQAJ/An8gBS0AC0EHdgRAIAUoAgQMAQsgBS0AC0H/AHELIAAoAnxKBEACfyAFLQALQQd2BEAgBSgCBAwBCyAFLQALQf8AcQshCSAAKAJ8Ig0CfyAGLQALQQd2BEAgBigCBAwBCyAGLQALQf8AcQsCfyAHLQALQQd2BEAgBygCBAwBCyAHLQALQf8AcQsgCSANa0EBdGpqakEBagwBCyAAKAJ8An8gBy0AC0EHdgRAIAcoAgQMAQsgBy0AC0H/AHELAn8gBi0AC0EHdgRAIAYoAgQMAQsgBi0AC0H/AHELampBAmoLIglB5QBJDQAgCRA7IQkgCCgCACECIAggCTYCACACBEAgAiAIKAIEEQEACyAIKAIAIgINABBGAAsgAiAAQQRqIAAgAygCBAJ/IAUtAAtBB3YEQCAFKAIADAELIAULAn8gBS0AC0EHdgRAIAUoAgAMAQsgBQsCfyAFLQALQQd2BEAgBSgCBAwBCyAFLQALQf8AcQtqIAogCyAAQagBaiAALACnASAALACmASAMIAYgByAAKAJ8ELgDIAEgAiAAKAIEIAAoAgAgAyAEEJUBIREgCCgCACEBIAhBADYCACABBEAgASAIKAIEEQEACyAHEDMaIAYQMxogDBAzGiAAKAKsASIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgAEGwAWokACARC98HARJ/IwBBwANrIgAkACAAIAU3AxAgACAGNwMYIAAgAEHQAmoiBzYCzAIgB0HkAEG2HiAAQRBqEMUCIQkgAEH2ATYC4AEgAEHYAWpBACAAQeABaiIOEFEhDCAAQfYBNgLgASAAQdABakEAIA4QUSEKAkAgCUHkAE8EQBBEIQcgACAFNwMAIAAgBjcDCCAAQcwCaiAHQbYeIAAQiwEiCUF/Rg0BIAwoAgAhByAMIAAoAswCNgIAIAcEQCAHIAwoAgQRAQALIAkQOyEIIAooAgAhByAKIAg2AgAgBwRAIAcgCigCBBEBAAsgCigCACIORQ0BCyAAQcwBaiIHIAMoAhwiCDYCACAIQQRqQQH+HgIAGiAHEHIiESIHIAAoAswCIgggCCAJaiAOIAcoAgAoAiARCAAaIAlBAEoEQCAAKALMAi0AAEEtRiEPCyACIA8gAEHMAWogAEHIAWogAEHHAWohGCAAQcYBaiEUIwBBEGsiByQAIABBuAFqIgJCADcCACACQQA2AgggB0EQaiQAIBggFCEXIAIiECETIwBBEGsiByQAIABBrAFqIgJCADcCACACQQA2AgggB0EQaiQAIBcgEyEWIAIiByESIwBBEGsiCCQAIABBoAFqIgJCADcCACACQQA2AgggCEEQaiQAIBYgEiACIgggAEGcAWoQuQMgAEH2ATYCMCAAQShqQQAgAEEwaiICEFEhCwJ/IAAoApwBIg0gCUgEQAJ/IActAAtBB3YEQCAHKAIEDAELIActAAtB/wBxCwJ/IAgtAAtBB3YEQCAIKAIEDAELIAgtAAtB/wBxCyAJIA1rQQF0amogDWpBAWoMAQsgACgCnAECfyAILQALQQd2BEAgCCgCBAwBCyAILQALQf8AcQsCfyAHLQALQQd2BEAgBygCBAwBCyAHLQALQf8AcQtqakECagsiDUHlAE8EQCANEDshDSALKAIAIQIgCyANNgIAIAIEQCACIAsoAgQRAQALIAsoAgAiAkUNAQsgAiAAQSRqIABBIGogAygCBCAOIAkgDmogESAPIABByAFqIAAsAMcBIAAsAMYBIBAgByAIIAAoApwBELgDIAEgAiAAKAIkIAAoAiAgAyAEEJUBIRUgCygCACEBIAtBADYCACABBEAgASALKAIEEQEACyAIEDMaIAcQMxogEBAzGiAAKALMASIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgCigCACEBIApBADYCACABBEAgASAKKAIEEQEACyAMKAIAIQEgDEEANgIAIAEEQCABIAwoAgQRAQALIABBwANqJAAgFQ8LEEYAC6UIAQZ/IwBBwANrIgAkACAAIAI2ArgDIAAgATYCvAMgAEH3ATYCFCAAQRhqIABBIGogAEEUaiIHEFEhCSAAQRBqIgggBCgCHCIBNgIAIAFBBGpBAf4eAgAaIAgQbCEBIABBADoADyAAQbwDaiACIAMgCCAEKAIEIAUgAEEPaiABIAkgByAAQbADahC9AwRAIwBBEGsiAiQAAkAgBi0AC0EHdgRAIAYoAgAhCyACQQA2AgwgCyACKAIMNgIAIAZBADYCBAwBCyACQQA2AgggBiACKAIINgIAIAYgBi0AC0GAAXE6AAsgBiAGLQALQf8AcToACwsgAkEQaiQAIAAtAA8EQCAGIAFBLSABKAIAKAIsEQQAEKECCyABQTAgASgCACgCLBEEACEBIAkoAgAhAiAAKAIUIgNBBGshBANAAkAgAiAETw0AIAIoAgAgAUcNACACQQRqIQIMAQsLIwBBEGsiCCQAAn8gBi0AC0EHdgRAIAYoAgQMAQsgBi0AC0H/AHELIQEgBi0AC0EHdgR/IAYoAghB/////wdxQQFrBUEBCyEEAkAgAyACa0ECdSIHRQ0AAn8gBi0AC0EHdgRAIAYoAgAMAQsgBgsgAk0EfwJ/IAYtAAtBB3YEQCAGKAIADAELIAYLAn8gBi0AC0EHdgRAIAYoAgQMAQsgBi0AC0H/AHELQQJ0aiACTwVBAAtFBEAgByAEIAFrSwRAIAYgBCABIARrIAdqIAEgARCGAwsCfyAGLQALQQd2BEAgBigCAAwBCyAGCyABQQJ0aiEEA0AgAiADRwRAIAQgAigCADYCACACQQRqIQIgBEEEaiEEDAELCyAIQQA2AgQgBCAIKAIENgIAIAYgASAHahCZAQwBCyMAQRBrIgQkACAIQQRqIgEgAiADEN4DIARBEGokAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIQcCfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshAiMAQRBrIgQkAAJAIAIgBi0AC0EHdgR/IAYoAghB/////wdxQQFrBUEBCyIKAn8gBi0AC0EHdgRAIAYoAgQMAQsgBi0AC0H/AHELIgNrTQRAIAJFDQECfyAGLQALQQd2BEAgBigCAAwBCyAGCyIKIANBAnRqIAcgAhCeASAGIAIgA2oiAhCZASAEQQA2AgwgCiACQQJ0aiAEKAIMNgIADAELIAYgCiACIAprIANqIAMgA0EAIAIgBxCHAwsgBEEQaiQAIAEQUBoLIAhBEGokAAsgAEG8A2ogAEG4A2oQQgRAIAUgBSgCAEECcjYCAAsgACgCvAMhDCAAKAIQIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAJKAIAIQEgCUEANgIAIAEEQCABIAkoAgQRAQALIABBwANqJAAgDAvQBAEEfyMAQfAEayIAJAAgACACNgLoBCAAIAE2AuwEIABB9wE2AhAgAEHIAWogAEHQAWogAEEQaiIBEFEhByAAQcABaiIJIAQoAhwiCDYCACAIQQRqQQH+HgIAGiAJEGwhCCAAQQA6AL8BAkAgAEHsBGogAiADIAkgBCgCBCAFIABBvwFqIAggByAAQcQBaiAAQeAEahC9A0UNACAAQZ85KAAANgC3ASAAQZg5KQAANwOwASAIIABBsAFqIABBugFqIABBgAFqIAgoAgAoAjARCAAaIABB9gE2AhAgAEEIakEAIAEQUSEDIAEhBAJAIAAoAsQBIAcoAgBrIgFBiQNOBEAgAUECdUECahA7IQIgAygCACEBIAMgAjYCACABBEAgASADKAIEEQEACyADKAIAIgRFDQELIAAtAL8BBEAgBEEtOgAAIARBAWohBAsgBygCACECA0AgACgCxAEgAk0EQAJAIARBADoAACAAIAY2AgAgAEEQaiAAEOQDQQFHDQAgAygCACEBIANBADYCACABBEAgASADKAIEEQEACwwECwUgBCAAQbABaiAAQYABaiIBIAFBKGogAhCxAiABa0ECdWotAAA6AAAgBEEBaiEEIAJBBGohAgwBCwsQRgALEEYACyAAQewEaiAAQegEahBCBEAgBSAFKAIAQQJyNgIACyAAKALsBCEKIAAoAsABIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAHKAIAIQEgB0EANgIAIAEEQCABIAcoAgQRAQALIABB8ARqJAAgCgu0AQEEfyMAQRBrIgQkACAEIAE2AgwgAigCACIFQfD///8HSQRAAkAgBUEKTQRAIAQgBToACyAEIQEMAQsgBUEPckEBaiIGEDIhASAEIAZBgICAgHhyNgIIIAQgATYCACAEIAU2AgQLIAEgAkEEaiAF/AoAACABIAVqQQA6AAAgBEEMaiAEIAMgABEDACEHIAQsAAtBAEgEQCAEKAIAEC8LIAQoAgwQBCAEQRBqJAAgBw8LEE0AC94GAQZ/IwBBkAFrIgAkACAAIAI2AogBIAAgATYCjAEgAEH3ATYCFCAAQRhqIABBIGogAEEUaiIIEFEhCSAAQRBqIgcgBCgCHCIBNgIAIAFBBGpBAf4eAgAaIAcQciEBIABBADoADyAAQYwBaiACIAMgByAEKAIEIAUgAEEPaiABIAkgCCAAQYQBahDDAwRAIwBBEGsiAiQAAkAgBi0AC0EHdgRAIAYoAgAhCiACQQA6AA8gCiACLQAPOgAAIAZBADYCBAwBCyACQQA6AA4gBiACLQAOOgAAIAYgBi0AC0GAAXE6AAsgBiAGLQALQf8AcToACwsgAkEQaiQAIAAtAA8EQCAGIAFBLSABKAIAKAIcEQQAEKICCyABQTAgASgCACgCHBEEACELIAkoAgAhAiAAKAIUIgdBAWshAyALQf8BcSEBA0ACQCACIANPDQAgAi0AACABRw0AIAJBAWohAgwBCwsjAEEQayIBJAACfyAGLQALQQd2BEAgBigCBAwBCyAGLQALQf8AcQshAyAGLQALQQd2BH8gBigCCEH/////B3FBAWsFQQoLIQQCQCAHIAJrIghFDQACfyAGLQALQQd2BEAgBigCAAwBCyAGCyACTQR/An8gBi0AC0EHdgRAIAYoAgAMAQsgBgsCfyAGLQALQQd2BEAgBigCBAwBCyAGLQALQf8AcQtqIAJPBUEAC0UEQCAIIAQgA2tLBEAgBiAEIAMgBGsgCGogAyADEKUCCwJ/IAYtAAtBB3YEQCAGKAIADAELIAYLIANqIQQDQCACIAdHBEAgBCACLQAAOgAAIAJBAWohAiAEQQFqIQQMAQsLIAFBADoADyAEIAEtAA86AAAgBiADIAhqEJkBDAELIwBBEGsiAyQAIAEgAiAHEPkDIANBEGokACAGAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsCfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsQXxogARAzGgsgAUEQaiQACyAAQYwBaiAAQYgBahBDBEAgBSAFKAIAQQJyNgIACyAAKAKMASEMIAAoAhAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAkoAgAhASAJQQA2AgAgAQRAIAEgCSgCBBEBAAsgAEGQAWokACAMC8YEAQR/IwBBkAJrIgAkACAAIAI2AogCIAAgATYCjAIgAEH3ATYCECAAQZgBaiAAQaABaiAAQRBqIgEQUSEHIABBkAFqIgkgBCgCHCIINgIAIAhBBGpBAf4eAgAaIAkQciEIIABBADoAjwECQCAAQYwCaiACIAMgCSAEKAIEIAUgAEGPAWogCCAHIABBlAFqIABBhAJqEMMDRQ0AIABBnzkoAAA2AIcBIABBmDkpAAA3A4ABIAggAEGAAWogAEGKAWogAEH2AGogCCgCACgCIBEIABogAEH2ATYCECAAQQhqQQAgARBRIQMgASEEAkAgACgClAEgBygCAGsiAUHjAE4EQCABQQJqEDshAiADKAIAIQEgAyACNgIAIAEEQCABIAMoAgQRAQALIAMoAgAiBEUNAQsgAC0AjwEEQCAEQS06AAAgBEEBaiEECyAHKAIAIQIDQCAAKAKUASACTQRAAkAgBEEAOgAAIAAgBjYCACAAQRBqIAAQ5ANBAUcNACADKAIAIQEgA0EANgIAIAEEQCABIAMoAgQRAQALDAQLBSAEIABB9gBqIgEgAUEKaiACELQCIABrIABqLQAKOgAAIARBAWohBCACQQFqIQIMAQsLEEYACxBGAAsgAEGMAmogAEGIAmoQQwRAIAUgBSgCAEECcjYCAAsgACgCjAIhCiAAKAKQASIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgBygCACEBIAdBADYCACABBEAgASAHKAIEEQEACyAAQZACaiQAIAoLoQMBBH8jAEGgA2siByQAIAcgB0GgA2oiAzYCDCMAQZABayICJAAgAiACQYQBajYCHCAAQQhqIAJBIGoiCCACQRxqIAQgBSAGEMcDIAJCADcDECACIAg2AgwgBygCDCAHQRBqIgRrQQJ1IQUgACgCCCEGIwBBEGsiACQAIAAgBjYCDCAAQQhqIABBDGoQfyEJIAQgAkEMaiAFIAJBEGoQ4gMhBSAJEH4gAEEQaiQAIAVBf0YEQBBGAAsgByAEIAVBAnRqNgIMIAJBkAFqJAAgBygCDCECIwBBEGsiBiQAIwBBIGsiACQAIABBGGogBCACEL0CIAAoAhghBSAAKAIcIQcjAEEQayICJAAgAiAFNgIIIAIgATYCDANAIAUgB0cEQCACQQxqIAUoAgAQ+gMgAiAFQQRqIgU2AggMAQsLIAAgAigCCDYCECAAIAIoAgw2AhQgAkEQaiQAIAAgBCAAKAIQIARrajYCDCAAIAAoAhQ2AgggBiAAKAIMNgIIIAYgACgCCDYCDCAAQSBqJAAgBigCDCEKIAZBEGokACADJAAgCguLAgECfyMAQYABayICJAAgAiACQfQAajYCDCAAQQhqIAJBEGoiAyACQQxqIAQgBSAGEMcDIAIoAgwhBCMAQRBrIgYkACMAQSBrIgAkACAAQRhqIAMgBBC9AiAAKAIYIQUgACgCHCEHIwBBEGsiBCQAIAQgBTYCCCAEIAE2AgwDQCAFIAdHBEAgBEEMaiAFLAAAEPwDIAQgBUEBaiIFNgIIDAELCyAAIAQoAgg2AhAgACAEKAIMNgIUIARBEGokACAAIAMgACgCECADa2o2AgwgACAAKAIUNgIIIAYgACgCDDYCCCAGIAAoAgg2AgwgAEEgaiQAIAYoAgwhCCAGQRBqJAAgAkGAAWokACAIC9gPAQN/IwBBMGsiByQAIAcgATYCLCAEQQA2AgAgByADKAIcIgg2AgAgCEEEakEB/h4CABogBxBsIQggBygCACIJQQRqQX/+HgIARQRAIAkgCSgCACgCCBEBAAsCfwJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkAgBkHBAGsOOQABFwQXBRcGBxcXFwoXFxcXDg8QFxcXExUXFxcXFxcXAAECAwMXFwEXCBcXCQsXDBcNFwsXFxESFBYLIAAgBUEYaiAHQSxqIAIgBCAIEMoDDBgLIAAgBUEQaiAHQSxqIAIgBCAIEMkDDBcLIAcgACABIAIgAyAEIAUCfyAAQQhqIAAoAggoAgwRAAAiAC0AC0EHdgRAIAAoAgAMAQsgAAsCfyAALQALQQd2BEAgACgCAAwBCyAACwJ/IAAtAAtBB3YEQCAAKAIEDAELIAAtAAtB/wBxC0ECdGoQkgE2AiwMFgsgB0EsaiACIAQgCEECEIkBIQAgBCgCACEBAkACQCAAQQFrQR5LDQAgAUEEcQ0AIAUgADYCDAwBCyAEIAFBBHI2AgALDBULIAdBqOsCKQMANwMYIAdBoOsCKQMANwMQIAdBmOsCKQMANwMIIAdBkOsCKQMANwMAIAcgACABIAIgAyAEIAUgByAHQSBqEJIBNgIsDBQLIAdByOsCKQMANwMYIAdBwOsCKQMANwMQIAdBuOsCKQMANwMIIAdBsOsCKQMANwMAIAcgACABIAIgAyAEIAUgByAHQSBqEJIBNgIsDBMLIAdBLGogAiAEIAhBAhCJASEAIAQoAgAhAQJAAkAgAEEXSg0AIAFBBHENACAFIAA2AggMAQsgBCABQQRyNgIACwwSCyAHQSxqIAIgBCAIQQIQiQEhACAEKAIAIQECQAJAIABBAWtBC0sNACABQQRxDQAgBSAANgIIDAELIAQgAUEEcjYCAAsMEQsgB0EsaiACIAQgCEEDEIkBIQAgBCgCACEBAkACQCAAQe0CSg0AIAFBBHENACAFIAA2AhwMAQsgBCABQQRyNgIACwwQCyAHQSxqIAIgBCAIQQIQiQEhASAEKAIAIQACQAJAIAFBAWsiAUELSw0AIABBBHENACAFIAE2AhAMAQsgBCAAQQRyNgIACwwPCyAHQSxqIAIgBCAIQQIQiQEhACAEKAIAIQECQAJAIABBO0oNACABQQRxDQAgBSAANgIEDAELIAQgAUEEcjYCAAsMDgsgB0EsaiEAIwBBEGsiASQAIAEgAjYCDANAAkAgACABQQxqEEINACAIQQECfyAAKAIAIgIoAgwiAyACKAIQRgRAIAIgAigCACgCJBEAAAwBCyADKAIACyAIKAIAKAIMEQMARQ0AIAAQXBoMAQsLIAAgAUEMahBCBEAgBCAEKAIAQQJyNgIACyABQRBqJAAMDQsgB0EsaiEBAkACfyAAQQhqIAAoAggoAggRAAAiAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELQQACfyAALQAXQQd2BEAgACgCEAwBCyAALQAXQf8AcQtrRgRAIAQgBCgCAEEEcjYCAAwBCyABIAIgACAAQRhqIAggBEEAEO4BIQIgBSgCCCEBAkAgACACRw0AIAFBDEcNACAFQQA2AggMAQsCQCACIABrQQxHDQAgAUELSg0AIAUgAUEMajYCCAsLDAwLIAdB0OsCQSz8CgAAIAcgACABIAIgAyAEIAUgByAHQSxqEJIBNgIsDAsLIAdBkOwCKAIANgIQIAdBiOwCKQMANwMIIAdBgOwCKQMANwMAIAcgACABIAIgAyAEIAUgByAHQRRqEJIBNgIsDAoLIAdBLGogAiAEIAhBAhCJASEAIAQoAgAhAQJAAkAgAEE8Sg0AIAFBBHENACAFIAA2AgAMAQsgBCABQQRyNgIACwwJCyAHQbjsAikDADcDGCAHQbDsAikDADcDECAHQajsAikDADcDCCAHQaDsAikDADcDACAHIAAgASACIAMgBCAFIAcgB0EgahCSATYCLAwICyAHQSxqIAIgBCAIQQEQiQEhACAEKAIAIQECQAJAIABBBkoNACABQQRxDQAgBSAANgIYDAELIAQgAUEEcjYCAAsMBwsgACABIAIgAyAEIAUgACgCACgCFBEHAAwHCyAHIAAgASACIAMgBCAFAn8gAEEIaiAAKAIIKAIYEQAAIgAtAAtBB3YEQCAAKAIADAELIAALAn8gAC0AC0EHdgRAIAAoAgAMAQsgAAsCfyAALQALQQd2BEAgACgCBAwBCyAALQALQf8AcQtBAnRqEJIBNgIsDAULIAVBFGogB0EsaiACIAQgCBDIAwwECyAHQSxqIAIgBCAIQQQQiQEhACAELQAAQQRxRQRAIAUgAEHsDms2AhQLDAMLIAZBJUYNAQsgBCAEKAIAQQRyNgIADAELIwBBEGsiACQAIAAgAjYCDEEGIQECQAJAIAdBLGoiAyAAQQxqIgUQQg0AQQQhASAIAn8gAygCACICKAIMIgYgAigCEEYEQCACIAIoAgAoAiQRAAAMAQsgBigCAAtBACAIKAIAKAI0EQMAQSVHDQBBAiEBIAMQXCAFEEJFDQELIAQgBCgCACABcjYCAAsgAEEQaiQACyAHKAIsCyEKIAdBMGokACAKC3sBAX8jAEEQayIAJAAgACABNgIMIABBCGoiASADKAIcIgM2AgAgA0EEakEB/h4CABogARBsIQMgASgCACIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgBUEUaiAAQQxqIAIgBCADEMgDIAAoAgwhBiAAQRBqJAAgBgt9AQJ/IwBBEGsiBiQAIAYgATYCDCAGQQhqIgEgAygCHCIDNgIAIANBBGpBAf4eAgAaIAEQbCEDIAEoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAAgBUEQaiAGQQxqIAIgBCADEMkDIAYoAgwhByAGQRBqJAAgBwt9AQJ/IwBBEGsiBiQAIAYgATYCDCAGQQhqIgEgAygCHCIDNgIAIANBBGpBAf4eAgAaIAEQbCEDIAEoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAAgBUEYaiAGQQxqIAIgBCADEMoDIAYoAgwhByAGQRBqJAAgBwtxACAAIAEgAiADIAQgBQJ/IABBCGogACgCCCgCFBEAACIALQALQQd2BEAgACgCAAwBCyAACwJ/IAAtAAtBB3YEQCAAKAIADAELIAALAn8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELQQJ0ahCSAQtdAQJ/IwBBIGsiBiQAIAZBuOwCKQMANwMYIAZBsOwCKQMANwMQIAZBqOwCKQMANwMIIAZBoOwCKQMANwMAIAAgASACIAMgBCAFIAYgBkEgaiIBEJIBIQcgASQAIAcLhQ8BA38jAEEQayIHJAAgByABNgIMIARBADYCACAHIAMoAhwiCDYCACAIQQRqQQH+HgIAGiAHEHIhCCAHKAIAIglBBGpBf/4eAgBFBEAgCSAJKAIAKAIIEQEACwJ/AkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQAJAAkACQCAGQcEAaw45AAEXBBcFFwYHFxcXChcXFxcODxAXFxcTFRcXFxcXFxcAAQIDAxcXARcIFxcJCxcMFw0XCxcXERIUFgsgACAFQRhqIAdBDGogAiAEIAgQzQMMGAsgACAFQRBqIAdBDGogAiAEIAgQzAMMFwsgByAAIAEgAiADIAQgBQJ/IABBCGogACgCCCgCDBEAACIALQALQQd2BEAgACgCAAwBCyAACwJ/IAAtAAtBB3YEQCAAKAIADAELIAALAn8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELahCTATYCDAwWCyAHQQxqIAIgBCAIQQIQigEhACAEKAIAIQECQAJAIABBAWtBHksNACABQQRxDQAgBSAANgIMDAELIAQgAUEEcjYCAAsMFQsgB0Kl2r2pwuzLkvkANwMAIAcgACABIAIgAyAEIAUgByAHQQhqEJMBNgIMDBQLIAdCpbK1qdKty5LkADcDACAHIAAgASACIAMgBCAFIAcgB0EIahCTATYCDAwTCyAHQQxqIAIgBCAIQQIQigEhACAEKAIAIQECQAJAIABBF0oNACABQQRxDQAgBSAANgIIDAELIAQgAUEEcjYCAAsMEgsgB0EMaiACIAQgCEECEIoBIQAgBCgCACEBAkACQCAAQQFrQQtLDQAgAUEEcQ0AIAUgADYCCAwBCyAEIAFBBHI2AgALDBELIAdBDGogAiAEIAhBAxCKASEAIAQoAgAhAQJAAkAgAEHtAkoNACABQQRxDQAgBSAANgIcDAELIAQgAUEEcjYCAAsMEAsgB0EMaiACIAQgCEECEIoBIQEgBCgCACEAAkACQCABQQFrIgFBC0sNACAAQQRxDQAgBSABNgIQDAELIAQgAEEEcjYCAAsMDwsgB0EMaiACIAQgCEECEIoBIQAgBCgCACEBAkACQCAAQTtKDQAgAUEEcQ0AIAUgADYCBAwBCyAEIAFBBHI2AgALDA4LIAdBDGohACMAQRBrIgEkACABIAI2AgwDQAJAIAAgAUEMahBDDQACfyAAKAIAIgIoAgwiAyACKAIQRgRAIAIgAigCACgCJBEAAAwBCyADLQAAC8AiAkEATgR/IAgoAgggAkH/AXFBAnRqKAIAQQFxBUEAC0UNACAAEF0aDAELCyAAIAFBDGoQQwRAIAQgBCgCAEECcjYCAAsgAUEQaiQADA0LIAdBDGohAQJAAn8gAEEIaiAAKAIIKAIIEQAAIgAtAAtBB3YEQCAAKAIEDAELIAAtAAtB/wBxC0EAAn8gAC0AF0EHdgRAIAAoAhAMAQsgAC0AF0H/AHELa0YEQCAEIAQoAgBBBHI2AgAMAQsgASACIAAgAEEYaiAIIARBABDvASECIAUoAgghAQJAIAAgAkcNACABQQxHDQAgBUEANgIIDAELAkAgAiAAa0EMRw0AIAFBC0oNACAFIAFBDGo2AggLCwwMCyAHQfjqAigAADYAByAHQfHqAikAADcDACAHIAAgASACIAMgBCAFIAcgB0ELahCTATYCDAwLCyAHQYDrAi0AADoABCAHQfzqAigAADYCACAHIAAgASACIAMgBCAFIAcgB0EFahCTATYCDAwKCyAHQQxqIAIgBCAIQQIQigEhACAEKAIAIQECQAJAIABBPEoNACABQQRxDQAgBSAANgIADAELIAQgAUEEcjYCAAsMCQsgB0KlkOmp0snOktMANwMAIAcgACABIAIgAyAEIAUgByAHQQhqEJMBNgIMDAgLIAdBDGogAiAEIAhBARCKASEAIAQoAgAhAQJAAkAgAEEGSg0AIAFBBHENACAFIAA2AhgMAQsgBCABQQRyNgIACwwHCyAAIAEgAiADIAQgBSAAKAIAKAIUEQcADAcLIAcgACABIAIgAyAEIAUCfyAAQQhqIAAoAggoAhgRAAAiAC0AC0EHdgRAIAAoAgAMAQsgAAsCfyAALQALQQd2BEAgACgCAAwBCyAACwJ/IAAtAAtBB3YEQCAAKAIEDAELIAAtAAtB/wBxC2oQkwE2AgwMBQsgBUEUaiAHQQxqIAIgBCAIEMsDDAQLIAdBDGogAiAEIAhBBBCKASEAIAQtAABBBHFFBEAgBSAAQewOazYCFAsMAwsgBkElRg0BCyAEIAQoAgBBBHI2AgAMAQsjAEEQayIAJAAgACACNgIMQQYhAQJAAkAgB0EMaiIDIABBDGoiBRBDDQBBBCEBIAgCfyADKAIAIgIoAgwiBiACKAIQRgRAIAIgAigCACgCJBEAAAwBCyAGLQAAC8BBACAIKAIAKAIkEQMAQSVHDQBBAiEBIAMQXSAFEENFDQELIAQgBCgCACABcjYCAAsgAEEQaiQACyAHKAIMCyEKIAdBEGokACAKC3sBAX8jAEEQayIAJAAgACABNgIMIABBCGoiASADKAIcIgM2AgAgA0EEakEB/h4CABogARByIQMgASgCACIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgBUEUaiAAQQxqIAIgBCADEMsDIAAoAgwhBiAAQRBqJAAgBgt9AQJ/IwBBEGsiBiQAIAYgATYCDCAGQQhqIgEgAygCHCIDNgIAIANBBGpBAf4eAgAaIAEQciEDIAEoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAAgBUEQaiAGQQxqIAIgBCADEMwDIAYoAgwhByAGQRBqJAAgBwt9AQJ/IwBBEGsiBiQAIAYgATYCDCAGQQhqIgEgAygCHCIDNgIAIANBBGpBAf4eAgAaIAEQciEDIAEoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIAAgBUEYaiAGQQxqIAIgBCADEM0DIAYoAgwhByAGQRBqJAAgBwtuACAAIAEgAiADIAQgBQJ/IABBCGogACgCCCgCFBEAACIALQALQQd2BEAgACgCAAwBCyAACwJ/IAAtAAtBB3YEQCAAKAIADAELIAALAn8gAC0AC0EHdgRAIAAoAgQMAQsgAC0AC0H/AHELahCTAQtAAQJ/IwBBEGsiBiQAIAZCpZDpqdLJzpLTADcDCCAAIAEgAiADIAQgBSAGQQhqIAZBEGoiARCTASEHIAEkACAHC8sBAQd/IwBB0AFrIgAkABBEIQUgACAENgIAIABBsAFqIgcgByAHQRQgBUH5FSAAEGMiCmoiBSACEHQhCCAAQRBqIgQgAigCHCIGNgIAIAZBBGpBAf4eAgAaIAQQbCEGIAQoAgAiCUEEakF//h4CAEUEQCAJIAkoAgAoAggRAQALIAYgByAFIAQgBigCACgCMBEIABogASAEIApBAnQgBGoiASAIIABrQQJ0IABqQbAFayAFIAhGGyABIAIgAxCUASELIABB0AFqJAAgCwuSBQEJfwJ/IwBBoANrIgYkACAGQiU3A5gDIAZBmANqIgdBAXJB0zAgAigCBBDsASEIIAYgBkHwAmoiCTYC7AIQRCEAAn8gCARAIAIoAgghCiAGQUBrIAU3AwAgBiAENwM4IAYgCjYCMCAJQR4gACAHIAZBMGoQYwwBCyAGIAQ3A1AgBiAFNwNYIAZB8AJqQR4gACAGQZgDaiAGQdAAahBjCyEAIAZB9gE2AoABIAZB5AJqQQAgBkGAAWoQUSEJIAZB8AJqIgohBwJAIABBHk4EQBBEIQACfyAIBEAgAigCCCEHIAYgBTcDECAGIAQ3AwggBiAHNgIAIAZB7AJqIAAgBkGYA2ogBhCLAQwBCyAGIAQ3AyAgBiAFNwMoIAZB7AJqIAAgBkGYA2ogBkEgahCLAQsiAEF/Rg0BIAkoAgAhByAJIAYoAuwCNgIAIAcEQCAHIAkoAgQRAQALIAYoAuwCIQcLIAcgACAHaiILIAIQdCEMIAZB9gE2AoABIAZB+ABqQQAgBkGAAWoiBxBRIQgCQCAGKALsAiAGQfACakYEQCAHIQAMAQsgAEEDdBA7IgBFDQEgCCgCACEHIAggADYCACAHBEAgByAIKAIEEQEACyAGKALsAiEKCyAGQewAaiIHIAIoAhwiDTYCACANQQRqQQH+HgIAGiAKIAwgCyAAIAZB9ABqIAZB8ABqIAcQ0AMgBygCACIHQQRqQX/+HgIARQRAIAcgBygCACgCCBEBAAsgASAAIAYoAnQgBigCcCACIAMQlAEhDiAIKAIAIQAgCEEANgIAIAAEQCAAIAgoAgQRAQALIAkoAgAhACAJQQA2AgAgAARAIAAgCSgCBBEBAAsgBkGgA2okACAODAELEEYACwvvBAEJfwJ/IwBB8AJrIgUkACAFQiU3A+gCIAVB6AJqIgZBAXJB4fkAIAIoAgQQ7AEhByAFIAVBwAJqIgg2ArwCEEQhAAJ/IAcEQCACKAIIIQkgBSAEOQMoIAUgCTYCICAIQR4gACAGIAVBIGoQYwwBCyAFIAQ5AzAgBUHAAmpBHiAAIAVB6AJqIAVBMGoQYwshACAFQfYBNgJQIAVBtAJqQQAgBUHQAGoQUSEIIAVBwAJqIgkhBgJAIABBHk4EQBBEIQACfyAHBEAgAigCCCEGIAUgBDkDCCAFIAY2AgAgBUG8AmogACAFQegCaiAFEIsBDAELIAUgBDkDECAFQbwCaiAAIAVB6AJqIAVBEGoQiwELIgBBf0YNASAIKAIAIQYgCCAFKAK8AjYCACAGBEAgBiAIKAIEEQEACyAFKAK8AiEGCyAGIAAgBmoiCiACEHQhCyAFQfYBNgJQIAVByABqQQAgBUHQAGoiBhBRIQcCQCAFKAK8AiAFQcACakYEQCAGIQAMAQsgAEEDdBA7IgBFDQEgBygCACEGIAcgADYCACAGBEAgBiAHKAIEEQEACyAFKAK8AiEJCyAFQTxqIgYgAigCHCIMNgIAIAxBBGpBAf4eAgAaIAkgCyAKIAAgBUHEAGogBUFAayAGENADIAYoAgAiBkEEakF//h4CAEUEQCAGIAYoAgAoAggRAQALIAEgACAFKAJEIAUoAkAgAiADEJQBIQ0gBygCACEAIAdBADYCACAABEAgACAHKAIEEQEACyAIKAIAIQAgCEEANgIAIAAEQCAAIAgoAgQRAQALIAVB8AJqJAAgDQwBCxBGAAsL0wEBBn8jAEGAAmsiACQAIABCJTcD+AEgAEH4AWoiB0EBckGnGkEAIAIoAgQQnAEQRCEGIAAgBDcDACAAQeABaiIFIAVBGCAGIAcgABBjIAVqIgYgAhB0IQggAEEUaiIHIAIoAhwiCTYCACAJQQRqQQH+HgIAGiAFIAggBiAAQSBqIgYgAEEcaiAAQRhqIAcQ6wEgBygCACIFQQRqQX/+HgIARQRAIAUgBSgCACgCCBEBAAsgASAGIAAoAhwgACgCGCACIAMQlAEhCiAAQYACaiQAIAoL0wEBBX8jAEGQAWsiACQAIABCJTcDiAEgAEGIAWoiBkEBckHEGkEAIAIoAgQQnAEQRCEFIAAgBDYCACAAQfsAaiIEIARBDSAFIAYgABBjIARqIgUgAhB0IQcgAEEEaiIGIAIoAhwiCDYCACAIQQRqQQH+HgIAGiAEIAcgBSAAQRBqIgUgAEEMaiAAQQhqIAYQ6wEgBigCACIEQQRqQX/+HgIARQRAIAQgBCgCACgCCBEBAAsgASAFIAAoAgwgACgCCCACIAMQlAEhCSAAQZABaiQAIAkL0wEBBn8jAEGAAmsiACQAIABCJTcD+AEgAEH4AWoiB0EBckGnGkEBIAIoAgQQnAEQRCEGIAAgBDcDACAAQeABaiIFIAVBGCAGIAcgABBjIAVqIgYgAhB0IQggAEEUaiIHIAIoAhwiCTYCACAJQQRqQQH+HgIAGiAFIAggBiAAQSBqIgYgAEEcaiAAQRhqIAcQ6wEgBygCACIFQQRqQX/+HgIARQRAIAUgBSgCACgCCBEBAAsgASAGIAAoAhwgACgCGCACIAMQlAEhCiAAQYACaiQAIAoL0wEBBX8jAEGQAWsiACQAIABCJTcDiAEgAEGIAWoiBkEBckHEGkEBIAIoAgQQnAEQRCEFIAAgBDYCACAAQfsAaiIEIARBDSAFIAYgABBjIARqIgUgAhB0IQcgAEEEaiIGIAIoAhwiCDYCACAIQQRqQQH+HgIAGiAEIAcgBSAAQRBqIgUgAEEMaiAAQQhqIAYQ6wEgBigCACIEQQRqQX/+HgIARQRAIAQgBCgCACgCCBEBAAsgASAFIAAoAgwgACgCCCACIAMQlAEhCSAAQZABaiQAIAkLjAIBAX8jAEEgayIFJAAgBSABNgIcAkAgAigCBEEBcUUEQCAAIAEgAiADIAQgACgCACgCGBEKACECDAELIAVBEGoiASACKAIcIgA2AgAgAEEEakEB/h4CABogARC/ASEAIAEoAgAiAkEEakF//h4CAEUEQCACIAIoAgAoAggRAQALAkAgBARAIAEgACAAKAIAKAIYEQIADAELIAVBEGogACAAKAIAKAIcEQIACyAFIAVBEGoQdTYCDANAIAUgBUEQaiIAEJsBNgIIIAUoAgwiASAFKAIIRwRAIAVBHGogASgCABD6AyAFIAUoAgxBBGo2AgwMAQUgBSgCHCECIAAQUBoLCwsgBUEgaiQAIAILwwEBB38jAEHgAGsiACQAEEQhBSAAIAQ2AgAgAEFAayIHIAcgB0EUIAVB+RUgABBjIgpqIgUgAhB0IQggAEEQaiIEIAIoAhwiBjYCACAGQQRqQQH+HgIAGiAEEHIhBiAEKAIAIglBBGpBf/4eAgBFBEAgCSAJKAIAKAIIEQEACyAGIAcgBSAEIAYoAgAoAiARCAAaIAEgBCAEIApqIgEgCCAAayAAakEwayAFIAhGGyABIAIgAxCVASELIABB4ABqJAAgCwuSBQEJfwJ/IwBBgAJrIgYkACAGQiU3A/gBIAZB+AFqIgdBAXJB0zAgAigCBBDsASEIIAYgBkHQAWoiCTYCzAEQRCEAAn8gCARAIAIoAgghCiAGQUBrIAU3AwAgBiAENwM4IAYgCjYCMCAJQR4gACAHIAZBMGoQYwwBCyAGIAQ3A1AgBiAFNwNYIAZB0AFqQR4gACAGQfgBaiAGQdAAahBjCyEAIAZB9gE2AoABIAZBxAFqQQAgBkGAAWoQUSEJIAZB0AFqIgohBwJAIABBHk4EQBBEIQACfyAIBEAgAigCCCEHIAYgBTcDECAGIAQ3AwggBiAHNgIAIAZBzAFqIAAgBkH4AWogBhCLAQwBCyAGIAQ3AyAgBiAFNwMoIAZBzAFqIAAgBkH4AWogBkEgahCLAQsiAEF/Rg0BIAkoAgAhByAJIAYoAswBNgIAIAcEQCAHIAkoAgQRAQALIAYoAswBIQcLIAcgACAHaiILIAIQdCEMIAZB9gE2AoABIAZB+ABqQQAgBkGAAWoiBxBRIQgCQCAGKALMASAGQdABakYEQCAHIQAMAQsgAEEBdBA7IgBFDQEgCCgCACEHIAggADYCACAHBEAgByAIKAIEEQEACyAGKALMASEKCyAGQewAaiIHIAIoAhwiDTYCACANQQRqQQH+HgIAGiAKIAwgCyAAIAZB9ABqIAZB8ABqIAcQ0gMgBygCACIHQQRqQX/+HgIARQRAIAcgBygCACgCCBEBAAsgASAAIAYoAnQgBigCcCACIAMQlQEhDiAIKAIAIQAgCEEANgIAIAAEQCAAIAgoAgQRAQALIAkoAgAhACAJQQA2AgAgAARAIAAgCSgCBBEBAAsgBkGAAmokACAODAELEEYACwvvBAEJfwJ/IwBB0AFrIgUkACAFQiU3A8gBIAVByAFqIgZBAXJB4fkAIAIoAgQQ7AEhByAFIAVBoAFqIgg2ApwBEEQhAAJ/IAcEQCACKAIIIQkgBSAEOQMoIAUgCTYCICAIQR4gACAGIAVBIGoQYwwBCyAFIAQ5AzAgBUGgAWpBHiAAIAVByAFqIAVBMGoQYwshACAFQfYBNgJQIAVBlAFqQQAgBUHQAGoQUSEIIAVBoAFqIgkhBgJAIABBHk4EQBBEIQACfyAHBEAgAigCCCEGIAUgBDkDCCAFIAY2AgAgBUGcAWogACAFQcgBaiAFEIsBDAELIAUgBDkDECAFQZwBaiAAIAVByAFqIAVBEGoQiwELIgBBf0YNASAIKAIAIQYgCCAFKAKcATYCACAGBEAgBiAIKAIEEQEACyAFKAKcASEGCyAGIAAgBmoiCiACEHQhCyAFQfYBNgJQIAVByABqQQAgBUHQAGoiBhBRIQcCQCAFKAKcASAFQaABakYEQCAGIQAMAQsgAEEBdBA7IgBFDQEgBygCACEGIAcgADYCACAGBEAgBiAHKAIEEQEACyAFKAKcASEJCyAFQTxqIgYgAigCHCIMNgIAIAxBBGpBAf4eAgAaIAkgCyAKIAAgBUHEAGogBUFAayAGENIDIAYoAgAiBkEEakF//h4CAEUEQCAGIAYoAgAoAggRAQALIAEgACAFKAJEIAUoAkAgAiADEJUBIQ0gBygCACEAIAdBADYCACAABEAgACAHKAIEEQEACyAIKAIAIQAgCEEANgIAIAAEQCAAIAgoAgQRAQALIAVB0AFqJAAgDQwBCxBGAAsL0gEBBn8jAEHwAGsiACQAIABCJTcDaCAAQegAaiIHQQFyQacaQQAgAigCBBCcARBEIQYgACAENwMAIABB0ABqIgUgBUEYIAYgByAAEGMgBWoiBiACEHQhCCAAQRRqIgcgAigCHCIJNgIAIAlBBGpBAf4eAgAaIAUgCCAGIABBIGoiBiAAQRxqIABBGGogBxDtASAHKAIAIgVBBGpBf/4eAgBFBEAgBSAFKAIAKAIIEQEACyABIAYgACgCHCAAKAIYIAIgAxCVASEKIABB8ABqJAAgCgvOAQEFfyMAQUBqIgAkACAAQiU3AzggAEE4aiIGQQFyQcQaQQAgAigCBBCcARBEIQUgACAENgIAIABBK2oiBCAEQQ0gBSAGIAAQYyAEaiIFIAIQdCEHIABBBGoiBiACKAIcIgg2AgAgCEEEakEB/h4CABogBCAHIAUgAEEQaiIFIABBDGogAEEIaiAGEO0BIAYoAgAiBEEEakF//h4CAEUEQCAEIAQoAgAoAggRAQALIAEgBSAAKAIMIAAoAgggAiADEJUBIQkgAEFAayQAIAkL0gEBBn8jAEHwAGsiACQAIABCJTcDaCAAQegAaiIHQQFyQacaQQEgAigCBBCcARBEIQYgACAENwMAIABB0ABqIgUgBUEYIAYgByAAEGMgBWoiBiACEHQhCCAAQRRqIgcgAigCHCIJNgIAIAlBBGpBAf4eAgAaIAUgCCAGIABBIGoiBiAAQRxqIABBGGogBxDtASAHKAIAIgVBBGpBf/4eAgBFBEAgBSAFKAIAKAIIEQEACyABIAYgACgCHCAAKAIYIAIgAxCVASEKIABB8ABqJAAgCgvOAQEFfyMAQUBqIgAkACAAQiU3AzggAEE4aiIGQQFyQcQaQQEgAigCBBCcARBEIQUgACAENgIAIABBK2oiBCAEQQ0gBSAGIAAQYyAEaiIFIAIQdCEHIABBBGoiBiACKAIcIgg2AgAgCEEEakEB/h4CABogBCAHIAUgAEEQaiIFIABBDGogAEEIaiAGEO0BIAYoAgAiBEEEakF//h4CAEUEQCAEIAQoAgAoAggRAQALIAEgBSAAKAIMIAAoAgggAiADEJUBIQkgAEFAayQAIAkLjAIBAX8jAEEgayIFJAAgBSABNgIcAkAgAigCBEEBcUUEQCAAIAEgAiADIAQgACgCACgCGBEKACECDAELIAVBEGoiASACKAIcIgA2AgAgAEEEakEB/h4CABogARDBASEAIAEoAgAiAkEEakF//h4CAEUEQCACIAIoAgAoAggRAQALAkAgBARAIAEgACAAKAIAKAIYEQIADAELIAVBEGogACAAKAIAKAIcEQIACyAFIAVBEGoQdTYCDANAIAUgBUEQaiIAEJ0BNgIIIAUoAgwiASAFKAIIRwRAIAVBHGogASwAABD8AyAFIAUoAgxBAWo2AgwMAQUgBSgCHCECIAAQMxoLCwsgBUEgaiQAIAILwQUBBn8jAEHAAmsiACQAIAAgAjYCuAIgACABNgK8AiMAQRBrIgIkACAAQcQBaiIBQgA3AgAgAUEANgIIIAJBEGokACAAQRBqIgYgAygCHCICNgIAIAJBBGpBAf4eAgAaIAYQbCICQdDqAkHq6gIgAEHQAWogAigCACgCMBEIABogBigCACICQQRqQX/+HgIARQRAIAIgAigCACgCCBEBAAsgASEDIwBBEGsiASQAIABBuAFqIgJCADcCACACQQA2AgggAUEQaiQAIAIgAi0AC0EHdgR/IAIoAghB/////wdxQQFrBUEKCxA6IAACfyACLQALQQd2BEAgAigCAAwBCyACCyIBNgK0ASAAIAY2AgwgAEEANgIIA0ACQCAAQbwCaiAAQbgCahBCDQAgACgCtAECfyACLQALQQd2BEAgAigCBAwBCyACLQALQf8AcQsgAWpGBEACfyACLQALQQd2BEAgAigCBAwBCyACLQALQf8AcQshBiACAn8gAi0AC0EHdgRAIAIoAgQMAQsgAi0AC0H/AHELQQF0EDogAiACLQALQQd2BH8gAigCCEH/////B3FBAWsFQQoLEDogACAGAn8gAi0AC0EHdgRAIAIoAgAMAQsgAgsiAWo2ArQBCwJ/IABBvAJqIgcoAgAiBigCDCIIIAYoAhBGBEAgBiAGKAIAKAIkEQAADAELIAgoAgALQRAgASAAQbQBaiAAQQhqQQAgAyAAQRBqIABBDGogAEHQAWoQvQENACAHEFwaDAELCyACIAAoArQBIAFrEDoCfyACLQALQQd2BEAgAigCAAwBCyACCyELEEQhCSAAIAU2AgAgCyAJIAAQ1ANBAUcEQCAEQQQ2AgALIABBvAJqIABBuAJqEEIEQCAEIAQoAgBBAnI2AgALIAAoArwCIQogAhAzGiADEDMaIABBwAJqJAAgCgvQBQIDfwF+IwBBgANrIgAkACAAIAI2AvgCIAAgATYC/AIgAEHcAWogAyAAQfABaiAAQewBaiAAQegBahCzAiMAQRBrIgIkACAAQdABaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCzAEgACAAQSBqNgIcIABBADYCGCAAQQE6ABcgAEHFADoAFgNAAkAgAEH8AmogAEH4AmoQQg0AIAAoAswBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgLMAQsCfyAAQfwCaiIGKAIAIgMoAgwiByADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAHKAIACyAAQRdqIABBFmogAiAAQcwBaiAAKALsASAAKALoASAAQdwBaiAAQSBqIABBHGogAEEYaiAAQfABahCyAg0AIAYQXBoMAQsLAkACfyAALQDnAUEHdgRAIAAoAuABDAELIAAtAOcBQf8AcQtFDQAgAC0AF0UNACAAKAIcIgMgAEEgamtBnwFKDQAgACADQQRqNgIcIAMgACgCGDYCAAsgACACIAAoAswBIAQQ1QMgACkDACEJIAUgACkDCDcDCCAFIAk3AwAgAEHcAWogAEEgaiAAKAIcIAQQZCAAQfwCaiAAQfgCahBCBEAgBCAEKAIAQQJyNgIACyAAKAL8AiEIIAEQMxogAEHcAWoQMxogAEGAA2okACAIC7kFAQN/IwBB8AJrIgAkACAAIAI2AugCIAAgATYC7AIgAEHMAWogAyAAQeABaiAAQdwBaiAAQdgBahCzAiMAQRBrIgIkACAAQcABaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCvAEgACAAQRBqNgIMIABBADYCCCAAQQE6AAcgAEHFADoABgNAAkAgAEHsAmogAEHoAmoQQg0AIAAoArwBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgK8AQsCfyAAQewCaiIGKAIAIgMoAgwiByADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAHKAIACyAAQQdqIABBBmogAiAAQbwBaiAAKALcASAAKALYASAAQcwBaiAAQRBqIABBDGogAEEIaiAAQeABahCyAg0AIAYQXBoMAQsLAkACfyAALQDXAUEHdgRAIAAoAtABDAELIAAtANcBQf8AcQtFDQAgAC0AB0UNACAAKAIMIgMgAEEQamtBnwFKDQAgACADQQRqNgIMIAMgACgCCDYCAAsgBSACIAAoArwBIAQQ1gM5AwAgAEHMAWogAEEQaiAAKAIMIAQQZCAAQewCaiAAQegCahBCBEAgBCAEKAIAQQJyNgIACyAAKALsAiEIIAEQMxogAEHMAWoQMxogAEHwAmokACAIC7kFAQN/IwBB8AJrIgAkACAAIAI2AugCIAAgATYC7AIgAEHMAWogAyAAQeABaiAAQdwBaiAAQdgBahCzAiMAQRBrIgIkACAAQcABaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCvAEgACAAQRBqNgIMIABBADYCCCAAQQE6AAcgAEHFADoABgNAAkAgAEHsAmogAEHoAmoQQg0AIAAoArwBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgK8AQsCfyAAQewCaiIGKAIAIgMoAgwiByADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAHKAIACyAAQQdqIABBBmogAiAAQbwBaiAAKALcASAAKALYASAAQcwBaiAAQRBqIABBDGogAEEIaiAAQeABahCyAg0AIAYQXBoMAQsLAkACfyAALQDXAUEHdgRAIAAoAtABDAELIAAtANcBQf8AcQtFDQAgAC0AB0UNACAAKAIMIgMgAEEQamtBnwFKDQAgACADQQRqNgIMIAMgACgCCDYCAAsgBSACIAAoArwBIAQQ1wM4AgAgAEHMAWogAEEQaiAAKAIMIAQQZCAAQewCaiAAQegCahBCBEAgBCAEKAIAQQJyNgIACyAAKALsAiEIIAEQMxogAEHMAWoQMxogAEHwAmokACAIC5oFAQV/IwBB0AJrIgAkACAAIAI2AsgCIAAgATYCzAIgAxCMASEGIAMgAEHQAWoQ0AEhByAAQcQBaiADIABBxAJqEM8BIwBBEGsiAiQAIABBuAFqIgFCADcCACABQQA2AgggAkEQaiQAIAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAACfyABLQALQQd2BEAgASgCAAwBCyABCyICNgK0ASAAIABBEGo2AgwgAEEANgIIA0ACQCAAQcwCaiAAQcgCahBCDQAgACgCtAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsgAmpGBEACfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshAyABAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELQQF0EDogASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogACADAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAmo2ArQBCwJ/IABBzAJqIggoAgAiAygCDCIJIAMoAhBGBEAgAyADKAIAKAIkEQAADAELIAkoAgALIAYgAiAAQbQBaiAAQQhqIAAoAsQCIABBxAFqIABBEGogAEEMaiAHEL0BDQAgCBBcGgwBCwsCQAJ/IAAtAM8BQQd2BEAgACgCyAEMAQsgAC0AzwFB/wBxC0UNACAAKAIMIgMgAEEQamtBnwFKDQAgACADQQRqNgIMIAMgACgCCDYCAAsgBSACIAAoArQBIAQgBhDYAzcDACAAQcQBaiAAQRBqIAAoAgwgBBBkIABBzAJqIABByAJqEEIEQCAEIAQoAgBBAnI2AgALIAAoAswCIQogARAzGiAAQcQBahAzGiAAQdACaiQAIAoLmgUBBX8jAEHQAmsiACQAIAAgAjYCyAIgACABNgLMAiADEIwBIQYgAyAAQdABahDQASEHIABBxAFqIAMgAEHEAmoQzwEjAEEQayICJAAgAEG4AWoiAUIANwIAIAFBADYCCCACQRBqJAAgASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIgI2ArQBIAAgAEEQajYCDCAAQQA2AggDQAJAIABBzAJqIABByAJqEEINACAAKAK0AQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyACakYEQAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyEDIAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtBAXQQOiABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAIAMCfyABLQALQQd2BEAgASgCAAwBCyABCyICajYCtAELAn8gAEHMAmoiCCgCACIDKAIMIgkgAygCEEYEQCADIAMoAgAoAiQRAAAMAQsgCSgCAAsgBiACIABBtAFqIABBCGogACgCxAIgAEHEAWogAEEQaiAAQQxqIAcQvQENACAIEFwaDAELCwJAAn8gAC0AzwFBB3YEQCAAKALIAQwBCyAALQDPAUH/AHELRQ0AIAAoAgwiAyAAQRBqa0GfAUoNACAAIANBBGo2AgwgAyAAKAIINgIACyAFIAIgACgCtAEgBCAGENsDOwEAIABBxAFqIABBEGogACgCDCAEEGQgAEHMAmogAEHIAmoQQgRAIAQgBCgCAEECcjYCAAsgACgCzAIhCiABEDMaIABBxAFqEDMaIABB0AJqJAAgCguaBQEFfyMAQdACayIAJAAgACACNgLIAiAAIAE2AswCIAMQjAEhBiADIABB0AFqENABIQcgAEHEAWogAyAAQcQCahDPASMAQRBrIgIkACAAQbgBaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCtAEgACAAQRBqNgIMIABBADYCCANAAkAgAEHMAmogAEHIAmoQQg0AIAAoArQBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgK0AQsCfyAAQcwCaiIIKAIAIgMoAgwiCSADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAJKAIACyAGIAIgAEG0AWogAEEIaiAAKALEAiAAQcQBaiAAQRBqIABBDGogBxC9AQ0AIAgQXBoMAQsLAkACfyAALQDPAUEHdgRAIAAoAsgBDAELIAAtAM8BQf8AcQtFDQAgACgCDCIDIABBEGprQZ8BSg0AIAAgA0EEajYCDCADIAAoAgg2AgALIAUgAiAAKAK0ASAEIAYQ3AM3AwAgAEHEAWogAEEQaiAAKAIMIAQQZCAAQcwCaiAAQcgCahBCBEAgBCAEKAIAQQJyNgIACyAAKALMAiEKIAEQMxogAEHEAWoQMxogAEHQAmokACAKC5oFAQV/IwBB0AJrIgAkACAAIAI2AsgCIAAgATYCzAIgAxCMASEGIAMgAEHQAWoQ0AEhByAAQcQBaiADIABBxAJqEM8BIwBBEGsiAiQAIABBuAFqIgFCADcCACABQQA2AgggAkEQaiQAIAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAACfyABLQALQQd2BEAgASgCAAwBCyABCyICNgK0ASAAIABBEGo2AgwgAEEANgIIA0ACQCAAQcwCaiAAQcgCahBCDQAgACgCtAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsgAmpGBEACfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshAyABAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELQQF0EDogASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogACADAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAmo2ArQBCwJ/IABBzAJqIggoAgAiAygCDCIJIAMoAhBGBEAgAyADKAIAKAIkEQAADAELIAkoAgALIAYgAiAAQbQBaiAAQQhqIAAoAsQCIABBxAFqIABBEGogAEEMaiAHEL0BDQAgCBBcGgwBCwsCQAJ/IAAtAM8BQQd2BEAgACgCyAEMAQsgAC0AzwFB/wBxC0UNACAAKAIMIgMgAEEQamtBnwFKDQAgACADQQRqNgIMIAMgACgCCDYCAAsgBSACIAAoArQBIAQgBhDdAzYCACAAQcQBaiAAQRBqIAAoAgwgBBBkIABBzAJqIABByAJqEEIEQCAEIAQoAgBBAnI2AgALIAAoAswCIQogARAzGiAAQcQBahAzGiAAQdACaiQAIAoL2gIBAX8jAEEgayIGJAAgBiABNgIcAkAgAygCBEEBcUUEQCAGQX82AgAgACABIAIgAyAEIAYgACgCACgCEBEHACEBAkACQAJAIAYoAgAOAgABAgsgBUEAOgAADAMLIAVBAToAAAwCCyAFQQE6AAAgBEEENgIADAELIAYgAygCHCIANgIAIABBBGpBAf4eAgAaIAYQbCEBIAYoAgAiAEEEakF//h4CAEUEQCAAIAAoAgAoAggRAQALIAYgAygCHCIANgIAIABBBGpBAf4eAgAaIAYQvwEhAyAGKAIAIgBBBGpBf/4eAgBFBEAgACAAKAIAKAIIEQEACyAGIAMgAygCACgCGBECACAGQQxyIAMgAygCACgCHBECACAFIAZBHGogAiAGIAZBGGoiAyABIARBARDuASAGRjoAACAGKAIcIQEDQCADQQxrEFAiAyAGRw0ACwsgBkEgaiQAIAELwgUBBn8jAEGAAmsiACQAIAAgAjYC+AEgACABNgL8ASMAQRBrIgIkACAAQcQBaiIBQgA3AgAgAUEANgIIIAJBEGokACAAQRBqIgYgAygCHCICNgIAIAJBBGpBAf4eAgAaIAYQciICQdDqAkHq6gIgAEHQAWogAigCACgCIBEIABogBigCACICQQRqQX/+HgIARQRAIAIgAigCACgCCBEBAAsgASEDIwBBEGsiASQAIABBuAFqIgJCADcCACACQQA2AgggAUEQaiQAIAIgAi0AC0EHdgR/IAIoAghB/////wdxQQFrBUEKCxA6IAACfyACLQALQQd2BEAgAigCAAwBCyACCyIBNgK0ASAAIAY2AgwgAEEANgIIA0ACQCAAQfwBaiAAQfgBahBDDQAgACgCtAECfyACLQALQQd2BEAgAigCBAwBCyACLQALQf8AcQsgAWpGBEACfyACLQALQQd2BEAgAigCBAwBCyACLQALQf8AcQshBiACAn8gAi0AC0EHdgRAIAIoAgQMAQsgAi0AC0H/AHELQQF0EDogAiACLQALQQd2BH8gAigCCEH/////B3FBAWsFQQoLEDogACAGAn8gAi0AC0EHdgRAIAIoAgAMAQsgAgsiAWo2ArQBCwJ/IABB/AFqIgcoAgAiBigCDCIIIAYoAhBGBEAgBiAGKAIAKAIkEQAADAELIAgtAAALwEEQIAEgAEG0AWogAEEIakEAIAMgAEEQaiAAQQxqIABB0AFqEMABDQAgBxBdGgwBCwsgAiAAKAK0ASABaxA6An8gAi0AC0EHdgRAIAIoAgAMAQsgAgshCxBEIQkgACAFNgIAIAsgCSAAENQDQQFHBEAgBEEENgIACyAAQfwBaiAAQfgBahBDBEAgBCAEKAIAQQJyNgIACyAAKAL8ASEKIAIQMxogAxAzGiAAQYACaiQAIAoLHgEBf0HQuwMoAgAiAARAIAAQ4ARB0LsDQQA2AgALC9EFAgN/AX4jAEGQAmsiACQAIAAgAjYCiAIgACABNgKMAiAAQdABaiADIABB4AFqIABB3wFqIABB3gFqELYCIwBBEGsiAiQAIABBxAFqIgFCADcCACABQQA2AgggAkEQaiQAIAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAACfyABLQALQQd2BEAgASgCAAwBCyABCyICNgLAASAAIABBIGo2AhwgAEEANgIYIABBAToAFyAAQcUAOgAWA0ACQCAAQYwCaiAAQYgCahBDDQAgACgCwAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsgAmpGBEACfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshAyABAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELQQF0EDogASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogACADAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAmo2AsABCwJ/IABBjAJqIgYoAgAiAygCDCIHIAMoAhBGBEAgAyADKAIAKAIkEQAADAELIActAAALwCAAQRdqIABBFmogAiAAQcABaiAALADfASAALADeASAAQdABaiAAQSBqIABBHGogAEEYaiAAQeABahC1Ag0AIAYQXRoMAQsLAkACfyAALQDbAUEHdgRAIAAoAtQBDAELIAAtANsBQf8AcQtFDQAgAC0AF0UNACAAKAIcIgMgAEEgamtBnwFKDQAgACADQQRqNgIcIAMgACgCGDYCAAsgACACIAAoAsABIAQQ1QMgACkDACEJIAUgACkDCDcDCCAFIAk3AwAgAEHQAWogAEEgaiAAKAIcIAQQZCAAQYwCaiAAQYgCahBDBEAgBCAEKAIAQQJyNgIACyAAKAKMAiEIIAEQMxogAEHQAWoQMxogAEGQAmokACAIC7oFAQN/IwBBgAJrIgAkACAAIAI2AvgBIAAgATYC/AEgAEHAAWogAyAAQdABaiAAQc8BaiAAQc4BahC2AiMAQRBrIgIkACAAQbQBaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCsAEgACAAQRBqNgIMIABBADYCCCAAQQE6AAcgAEHFADoABgNAAkAgAEH8AWogAEH4AWoQQw0AIAAoArABAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgKwAQsCfyAAQfwBaiIGKAIAIgMoAgwiByADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAHLQAAC8AgAEEHaiAAQQZqIAIgAEGwAWogACwAzwEgACwAzgEgAEHAAWogAEEQaiAAQQxqIABBCGogAEHQAWoQtQINACAGEF0aDAELCwJAAn8gAC0AywFBB3YEQCAAKALEAQwBCyAALQDLAUH/AHELRQ0AIAAtAAdFDQAgACgCDCIDIABBEGprQZ8BSg0AIAAgA0EEajYCDCADIAAoAgg2AgALIAUgAiAAKAKwASAEENYDOQMAIABBwAFqIABBEGogACgCDCAEEGQgAEH8AWogAEH4AWoQQwRAIAQgBCgCAEECcjYCAAsgACgC/AEhCCABEDMaIABBwAFqEDMaIABBgAJqJAAgCAu6BQEDfyMAQYACayIAJAAgACACNgL4ASAAIAE2AvwBIABBwAFqIAMgAEHQAWogAEHPAWogAEHOAWoQtgIjAEEQayICJAAgAEG0AWoiAUIANwIAIAFBADYCCCACQRBqJAAgASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIgI2ArABIAAgAEEQajYCDCAAQQA2AgggAEEBOgAHIABBxQA6AAYDQAJAIABB/AFqIABB+AFqEEMNACAAKAKwAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyACakYEQAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyEDIAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtBAXQQOiABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAIAMCfyABLQALQQd2BEAgASgCAAwBCyABCyICajYCsAELAn8gAEH8AWoiBigCACIDKAIMIgcgAygCEEYEQCADIAMoAgAoAiQRAAAMAQsgBy0AAAvAIABBB2ogAEEGaiACIABBsAFqIAAsAM8BIAAsAM4BIABBwAFqIABBEGogAEEMaiAAQQhqIABB0AFqELUCDQAgBhBdGgwBCwsCQAJ/IAAtAMsBQQd2BEAgACgCxAEMAQsgAC0AywFB/wBxC0UNACAALQAHRQ0AIAAoAgwiAyAAQRBqa0GfAUoNACAAIANBBGo2AgwgAyAAKAIINgIACyAFIAIgACgCsAEgBBDXAzgCACAAQcABaiAAQRBqIAAoAgwgBBBkIABB/AFqIABB+AFqEEMEQCAEIAQoAgBBAnI2AgALIAAoAvwBIQggARAzGiAAQcABahAzGiAAQYACaiQAIAgLkAUBBH8jAEGAAmsiACQAIAAgAjYC+AEgACABNgL8ASADEIwBIQYgAEHEAWogAyAAQfcBahDRASMAQRBrIgIkACAAQbgBaiIBQgA3AgAgAUEANgIIIAJBEGokACABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAjYCtAEgACAAQRBqNgIMIABBADYCCANAAkAgAEH8AWogAEH4AWoQQw0AIAAoArQBAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIAJqRgRAAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELIQMgAQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxC0EBdBA6IAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAAgAwJ/IAEtAAtBB3YEQCABKAIADAELIAELIgJqNgK0AQsCfyAAQfwBaiIHKAIAIgMoAgwiCCADKAIQRgRAIAMgAygCACgCJBEAAAwBCyAILQAAC8AgBiACIABBtAFqIABBCGogACwA9wEgAEHEAWogAEEQaiAAQQxqQdDqAhDAAQ0AIAcQXRoMAQsLAkACfyAALQDPAUEHdgRAIAAoAsgBDAELIAAtAM8BQf8AcQtFDQAgACgCDCIDIABBEGprQZ8BSg0AIAAgA0EEajYCDCADIAAoAgg2AgALIAUgAiAAKAK0ASAEIAYQ2AM3AwAgAEHEAWogAEEQaiAAKAIMIAQQZCAAQfwBaiAAQfgBahBDBEAgBCAEKAIAQQJyNgIACyAAKAL8ASEJIAEQMxogAEHEAWoQMxogAEGAAmokACAJC5AFAQR/IwBBgAJrIgAkACAAIAI2AvgBIAAgATYC/AEgAxCMASEGIABBxAFqIAMgAEH3AWoQ0QEjAEEQayICJAAgAEG4AWoiAUIANwIAIAFBADYCCCACQRBqJAAgASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIgI2ArQBIAAgAEEQajYCDCAAQQA2AggDQAJAIABB/AFqIABB+AFqEEMNACAAKAK0AQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyACakYEQAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyEDIAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtBAXQQOiABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAIAMCfyABLQALQQd2BEAgASgCAAwBCyABCyICajYCtAELAn8gAEH8AWoiBygCACIDKAIMIgggAygCEEYEQCADIAMoAgAoAiQRAAAMAQsgCC0AAAvAIAYgAiAAQbQBaiAAQQhqIAAsAPcBIABBxAFqIABBEGogAEEMakHQ6gIQwAENACAHEF0aDAELCwJAAn8gAC0AzwFBB3YEQCAAKALIAQwBCyAALQDPAUH/AHELRQ0AIAAoAgwiAyAAQRBqa0GfAUoNACAAIANBBGo2AgwgAyAAKAIINgIACyAFIAIgACgCtAEgBCAGENsDOwEAIABBxAFqIABBEGogACgCDCAEEGQgAEH8AWogAEH4AWoQQwRAIAQgBCgCAEECcjYCAAsgACgC/AEhCSABEDMaIABBxAFqEDMaIABBgAJqJAAgCQuQBQEEfyMAQYACayIAJAAgACACNgL4ASAAIAE2AvwBIAMQjAEhBiAAQcQBaiADIABB9wFqENEBIwBBEGsiAiQAIABBuAFqIgFCADcCACABQQA2AgggAkEQaiQAIAEgAS0AC0EHdgR/IAEoAghB/////wdxQQFrBUEKCxA6IAACfyABLQALQQd2BEAgASgCAAwBCyABCyICNgK0ASAAIABBEGo2AgwgAEEANgIIA0ACQCAAQfwBaiAAQfgBahBDDQAgACgCtAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQsgAmpGBEACfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQshAyABAn8gAS0AC0EHdgRAIAEoAgQMAQsgAS0AC0H/AHELQQF0EDogASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogACADAn8gAS0AC0EHdgRAIAEoAgAMAQsgAQsiAmo2ArQBCwJ/IABB/AFqIgcoAgAiAygCDCIIIAMoAhBGBEAgAyADKAIAKAIkEQAADAELIAgtAAALwCAGIAIgAEG0AWogAEEIaiAALAD3ASAAQcQBaiAAQRBqIABBDGpB0OoCEMABDQAgBxBdGgwBCwsCQAJ/IAAtAM8BQQd2BEAgACgCyAEMAQsgAC0AzwFB/wBxC0UNACAAKAIMIgMgAEEQamtBnwFKDQAgACADQQRqNgIMIAMgACgCCDYCAAsgBSACIAAoArQBIAQgBhDcAzcDACAAQcQBaiAAQRBqIAAoAgwgBBBkIABB/AFqIABB+AFqEEMEQCAEIAQoAgBBAnI2AgALIAAoAvwBIQkgARAzGiAAQcQBahAzGiAAQYACaiQAIAkL6AMDBH8DewF9AkAgAkEATA0AIAJBBE8EQCACQXxxIQMDQCABIARBAXRq/QwAfgAAAH4AAAB+AAAAfgAAIAAgBEECdGr9AAIAIgf94AH9DAAAgHcAAIB3AACAdwAAgHf95gH9DAAAgAgAAIAIAACACAAAgAj95gEgB0EB/asBIgj9DAAAAP8AAAD/AAAA/wAAAP/9Tv0MAAAAcQAAAHEAAABxAAAAcf25AUEB/a0B/QwAAIAHAACABwAAgAcAAIAH/a4B/eQBIglBDf2tAf0MAHwAAAB8AAAAfAAAAHwAAP1OIAn9DP8PAAD/DwAA/w8AAP8PAAD9Tv2uASAI/QwAAAD/AAAA/wAAAP8AAAD//Tz9UiAHQRD9rQH9DACAAAAAgAAAAIAAAACAAAD9Tv1QIAf9DQABBAUICQwNAAEAAQABAAH9WwEAACAEQQRqIgQgA0cNAAsgAiADRg0BCwNAIAEgA0EBdGpBgPwBIAAgA0ECdGoqAgAiCotDAACAd5RDAACACJRBgICAiAcgCrwiBEEBdCIGQYCAgHhxIgUgBUGAgICIB00bQQF2QYCAgDxqvpK8IgVBDXZBgPgBcSAFQf8fcWogBkGAgIB4SxsgBEEQdkGAgAJxcjsBACADQQFqIgMgAkcNAAsLC5AFAQR/IwBBgAJrIgAkACAAIAI2AvgBIAAgATYC/AEgAxCMASEGIABBxAFqIAMgAEH3AWoQ0QEjAEEQayICJAAgAEG4AWoiAUIANwIAIAFBADYCCCACQRBqJAAgASABLQALQQd2BH8gASgCCEH/////B3FBAWsFQQoLEDogAAJ/IAEtAAtBB3YEQCABKAIADAELIAELIgI2ArQBIAAgAEEQajYCDCAAQQA2AggDQAJAIABB/AFqIABB+AFqEEMNACAAKAK0AQJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyACakYEQAJ/IAEtAAtBB3YEQCABKAIEDAELIAEtAAtB/wBxCyEDIAECfyABLQALQQd2BEAgASgCBAwBCyABLQALQf8AcQtBAXQQOiABIAEtAAtBB3YEfyABKAIIQf////8HcUEBawVBCgsQOiAAIAMCfyABLQALQQd2BEAgASgCAAwBCyABCyICajYCtAELAn8gAEH8AWoiBygCACIDKAIMIgggAygCEEYEQCADIAMoAgAoAiQRAAAMAQsgCC0AAAvAIAYgAiAAQbQBaiAAQQhqIAAsAPcBIABBxAFqIABBEGogAEEMakHQ6gIQwAENACAHEF0aDAELCwJAAn8gAC0AzwFBB3YEQCAAKALIAQwBCyAALQDPAUH/AHELRQ0AIAAoAgwiAyAAQRBqa0GfAUoNACAAIANBBGo2AgwgAyAAKAIINgIACyAFIAIgACgCtAEgBCAGEN0DNgIAIABBxAFqIABBEGogACgCDCAEEGQgAEH8AWogAEH4AWoQQwRAIAQgBCgCAEECcjYCAAsgACgC/AEhCSABEDMaIABBxAFqEDMaIABBgAJqJAAgCQvaAgEBfyMAQSBrIgYkACAGIAE2AhwCQCADKAIEQQFxRQRAIAZBfzYCACAAIAEgAiADIAQgBiAAKAIAKAIQEQcAIQECQAJAAkAgBigCAA4CAAECCyAFQQA6AAAMAwsgBUEBOgAADAILIAVBAToAACAEQQQ2AgAMAQsgBiADKAIcIgA2AgAgAEEEakEB/h4CABogBhByIQEgBigCACIAQQRqQX/+HgIARQRAIAAgACgCACgCCBEBAAsgBiADKAIcIgA2AgAgAEEEakEB/h4CABogBhDBASEDIAYoAgAiAEEEakF//h4CAEUEQCAAIAAoAgAoAggRAQALIAYgAyADKAIAKAIYEQIAIAZBDHIgAyADKAIAKAIcEQIAIAUgBkEcaiACIAYgBkEYaiIDIAEgBEEBEO8BIAZGOgAAIAYoAhwhAQNAIANBDGsQMyIDIAZHDQALCyAGQSBqJAAgAQtAAQF/QQAhAAN/IAEgAkYEfyAABSABKAIAIABBBHRqIgBBgICAgH9xIgNBGHYgA3IgAHMhACABQQRqIQEMAQsLCxsAIwBBEGsiASQAIAAgAiADEN4DIAFBEGokAAtUAQJ/AkADQCADIARHBEBBfyEAIAEgAkYNAiABKAIAIgUgAygCACIGSA0CIAUgBkoEQEEBDwUgA0EEaiEDIAFBBGohAQwCCwALCyABIAJHIQALIAALQAEBf0EAIQADfyABIAJGBH8gAAUgASwAACAAQQR0aiIAQYCAgIB/cSIDQRh2IANyIABzIQAgAUEBaiEBDAELCwuNAgEFfwJAIAJBAEwNACACQQNxIQUgAkEETwRAIAJBfHEhB0EAIQIDQCABIANBAnRqIAAgA0EBdGovAQBBAnRBkNYEaioCADgCACABIANBAXIiBEECdGogACAEQQF0ai8BAEECdEGQ1gRqKgIAOAIAIAEgA0ECciIEQQJ0aiAAIARBAXRqLwEAQQJ0QZDWBGoqAgA4AgAgASADQQNyIgRBAnRqIAAgBEEBdGovAQBBAnRBkNYEaioCADgCACADQQRqIQMgAkEEaiICIAdHDQALCyAFRQ0AA0AgASADQQJ0aiAAIANBAXRqLwEAQQJ0QZDWBGoqAgA4AgAgA0EBaiEDIAZBAWoiBiAFRw0ACwsLCwAgACACIAMQ3wMLXgEDfyABIAQgA2tqIQUCQANAIAMgBEcEQEF/IQAgASACRg0CIAEsAAAiBiADLAAAIgdIDQIgBiAHSgRAQQEPBSADQQFqIQMgAUEBaiEBDAILAAsLIAIgBUchAAsgAAvmDAMGfwl7AX0jAEGAAmsiBCQAIAEgAEGAAk4EfSAAQYACbSEIA0AgBCACIAZB0gFsaiIF/QAAgAEiCkEE/Wv9DDAwMDAwMDAwMDAwMDAwMDAiDf1OIAX9AAAAIgz9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79CwQAIAX9AAAgIQsgBCAK/QwwMDAwMDAwMDAwMDAwMDAw/U4gDEEE/W39UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEQCAEIApBAv1t/QwwMDAwMDAwMDAwMDAwMDAw/U4gC0EE/W39UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEYCAEIApBAv1r/QwwMDAwMDAwMDAwMDAwMDAw/U4gC/0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9DODg4ODg4ODg4ODg4ODg4OD9bv0LBCAgBCAF/QAAkAEiCkEE/Wv9DDAwMDAwMDAwMDAwMDAwMDD9TiAF/QAAECIM/QwPDw8PDw8PDw8PDw8PDw8P/U79UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEECAF/QAAMCELIAQgCv0MMDAwMDAwMDAwMDAwMDAwMP1OIAxBBP1t/VD9DODg4ODg4ODg4ODg4ODg4OD9bv0LBFAgBCAKQQL9bf0MMDAwMDAwMDAwMDAwMDAwMP1OIAtBBP1t/VD9DODg4ODg4ODg4ODg4ODg4OD9bv0LBHAgBCAKQQL9a/0MMDAwMDAwMDAwMDAwMDAwMP1OIAv9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79CwQwIAQgBf0AAKABIgpBBP1r/QwwMDAwMDAwMDAwMDAwMDAw/U4gBf0AAEAiDP0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9DODg4ODg4ODg4ODg4ODg4OD9bv0LBIABIAX9AABgIQsgBCAK/QwwMDAwMDAwMDAwMDAwMDAw/U4gDEEE/W39UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEwAEgBCAKQQL9bf0MMDAwMDAwMDAwMDAwMDAwMP1OIAtBBP1t/VD9DODg4ODg4ODg4ODg4ODg4OD9bv0LBOABIAQgCkEC/Wv9DDAwMDAwMDAwMDAwMDAwMDD9TiAL/QwPDw8PDw8PDw8PDw8PDw8P/U79UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEoAEgBCAF/QAAsAEiCkEE/Wv9DDAwMDAwMDAwMDAwMDAwMDD9TiAF/QAAUCIM/QwPDw8PDw8PDw8PDw8PDw8P/U79UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsEkAEgBf0AAHAhCyAEIAr9DDAwMDAwMDAwMDAwMDAwMDD9TiAMQQT9bf1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79CwTQASAEIApBAv1t/QwwMDAwMDAwMDAwMDAwMDAw/U4gC0EE/W39UP0M4ODg4ODg4ODg4ODg4ODg4P1u/QsE8AEgBCAKQQL9a/0MMDAwMDAwMDAwMDAwMDAwMP1OIAv9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79CwSwASADIAZBpAJsaiIJQQRqIQAgBCEBQQAhB/0MAAAAAAAAAAAAAAAAAAAAACIQIREDQCAB/V0ACCISIA39DQQFBgcAAAAAAAAAAAAAAAD9hwEgAP1dAAgiCiAN/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwBIAUgB2osAMAB/REiDf21ASAB/V0AACILIA39DQQFBgcAAAAAAAAAAAAAAAD9hwEgAP1dAAAiDCAN/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwBIA39tQEgEf2uAf2uASERIBL9hwEgCv2HAf28ASAN/bUBIAv9hwEgDP2HAf28ASAN/bUBIBD9rgH9rgEhECABQRBqIQEgAEEQaiEAIAdBAWoiB0EQRw0ACyAFLwHQAUECdEGQ1gRqKgIAIAkqAgCU/RMiDCAR/foB/eYBIA/95AEhDyAMIBD9+gH95gEgDv3kASEOIAZBAWoiBiAIRw0ACyAO/R8AQwAAAACSIA79HwGSIA79HwKSIA79HwOSIA/9HwCSIA/9HwGSIA/9HwKSIA/9HwOSBUMAAAAACzgCACAEQYACaiQAC1IBAn8gASAAKAJUIgEgASACQYACaiIDEIsEIgQgAWsgAyAEGyIDIAIgAiADSxsiAhB7GiAAIAEgA2oiAzYCVCAAIAM2AgggACABIAJqNgIEIAILgg0EF38NewJ9AX4jAEGQAmsiBiQAIAEgAEGAAk4EfSAAQYACbSEKA0AgAyAJQaQCbGohBSACIAlBsAFsaiIHQTBqIQAgB/0AABAhHiAH/QAAICEfIAYhAUEAIQhBASEEA0AgASAA/QAAACIh/QwPDw8PDw8PDw8PDw8PDw8P/U4iIv0LAAAgAP0AABAhICAB/QwAAAAAAAAAAAAAAAAAAAAAIhv9DBAQEBAQEBAQEBAQEBAQEBD9DAAAAAAAAAAAAAAAAAAAAAAgHiAE/Q8iI/1O/SP9UiAi/VD9CwAAIAH9DAAAAAAAAAAAAAAAAAAAAAD9DBAQEBAQEBAQEBAQEBAQEBD9DAAAAAAAAAAAAAAAAAAAAAAgHyAj/U79I/1SICD9DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/QsAECABIABBDGogAEELaiAAQQpqIABBCWogAEEIaiAA/V0AAP1UAAAI/VQAAAn9VAAACv1UAAAL/VQAAAwgIf0NAAECAwQFBgcICQoLDB0eH0EE/W0iIf0LACAgAEEfaiAAQR5qIABBHWogIP1UAAAN/VQAAA79VAAADyEgIAH9DAAAAAAAAAAAAAAAAAAAAAD9DBAQEBAQEBAQEBAQEBAQEBD9DAAAAAAAAAAAAAAAAAAAAAAgHiAEQQF0/Q8iIv1O/SP9UiAh/VD9CwAgIAH9DAAAAAAAAAAAAAAAAAAAAAD9DBAQEBAQEBAQEBAQEBAQEBD9DAAAAAAAAAAAAAAAAAAAAAAgHyAi/U79I/1SICBBBP1t/VD9CwAwIABBIGohACAEQQJ0IQQgAUFAayEBIAhBAWoiCEEERw0ACyAGIAcpAQQiKjcDgAIgBiAHKAEMIgA2AogCIAYgKqciBEG//vz5A3E2AoACIAYoAoQCIQEgBiAEQQJ2QbDgwIEDcSAAQY+evPgAcXI2AoQCIAYgAUECdkGw4MCBA3EgAEEEdkGPnrz4AHFyIgA2AowCIAYgAUG//vz5A3EiBDYCiAIgAEE/cSIIIAUuAZYCbCELIAggBS4BlAJsIQggBEEYdiIEIAUuAZICbCEMIAQgBS4BkAJsIQ0gAUEQdkE/cSIEIAUuAY4CbCEOIAQgBS4BjAJsIQ8gAUEIdkE/cSIEIAUuAYoCbCEQIAQgBS4BiAJsIREgAEEQdkE/cSIEIAUuAZ4CbCESIAQgBS4BnAJsIRMgAEEIdkE/cSIAIAUuAZoCbCEUIAAgBS4BmAJsIRUgBUEEaiEAIAFBP3EhFiAGLQCPAiIBIAUuAaICbCEXIAEgBS4BoAJsIRggBS4BhgIhGSAFLgGEAiEaIAYhAUEAIQT9DAAAAAAAAAAAAAAAAAAAAAAiHiEfA0AgAf1dABgiICAb/Q0EBQYHAAAAAAAAAAAAAAAA/YcBIAD9XQAYIiEgG/0NBAUGBwAAAAAAAAAAAAAAAP2HAf28ASAGQYACaiAEai0AAP0RIhv9tQEgAf1dABAiIiAb/Q0EBQYHAAAAAAAAAAAAAAAA/YcBIAD9XQAQIiMgG/0NBAUGBwAAAAAAAAAAAAAAAP2HAf28ASAb/bUBIAH9XQAIIiQgG/0NBAUGBwAAAAAAAAAAAAAAAP2HASAA/V0ACCIlIBv9DQQFBgcAAAAAAAAAAAAAAAD9hwH9vAEgG/21ASAB/V0AACImIBv9DQQFBgcAAAAAAAAAAAAAAAD9hwEgAP1dAAAiJyAb/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwBIBv9tQEgH/2uAf2uAf2uAf2uASEfICD9hwEgIf2HAf28ASAb/bUBICL9hwEgI/2HAf28ASAb/bUBICT9hwEgJf2HAf28ASAb/bUBICb9hwEgJ/2HAf28ASAb/bUBIB79rgH9rgH9rgH9rgEhHiABQSBqIQEgAEEgaiEAIARBAWoiBEEIRw0ACyAoIAUqAgAiKSAHLwECQQJ0QZDWBGoqAgCUIBcgGCASIBMgFCAVIAsgCCAMIA0gDiAPIBAgESAWIBkgGmpsampqampqampqampqamqylJMhKCAHLwEAQQJ0QZDWBGoqAgAgKZT9EyIbIB/9+gH95gEgHf3kASEdIBsgHv36Af3mASAc/eQBIRwgCUEBaiIJIApHDQALICggHP0fAJIgHP0fAZIgHP0fApIgHP0fA5IgHf0fAJIgHf0fAZIgHf0fApIgHf0fA5IFQwAAAAALOAIAIAZBkAJqJAAL2AsDF38NewJ9IwBBkAJrIgQkACABIABBgAJOBH0gAEGAAm0hCwNAIAQgAiAJQZABbGoiBf0AABD9DA8PDw8PDw8PDw8PDw8PDw8iG/1O/QsEACAEIAX9AAAg/QwPDw8PDw8PDw8PDw8PDw8P/U79CwQQIAQgBf0AABBBBP1t/QsEICAEIAX9AAAgQQT9bf0LBDAgBCAF/QAAMP0MDw8PDw8PDw8PDw8PDw8PD/1O/QsEQCAEIAVBQGsiAP0AAAD9DA8PDw8PDw8PDw8PDw8PDw/9Tv0LBFAgBCAF/QAAMEEE/W39CwRgIAQgAP0AAABBBP1t/QsEcCAEIAX9AABQ/QwPDw8PDw8PDw8PDw8PDw8P/U79CwSAASAEIAX9AABg/QwPDw8PDw8PDw8PDw8PDw8P/U79CwSQASAEIAX9AABQQQT9bf0LBKABIAQgBf0AAGBBBP1t/QsEsAEgBCAF/QAAcP0MDw8PDw8PDw8PDw8PDw8PD/1O/QsEwAEgBCAF/QAAgAH9DA8PDw8PDw8PDw8PDw8PDw/9Tv0LBNABIAQgBf0AAHBBBP1t/QsE4AEgBCAF/QAAgAFBBP1t/QsE8AEgBCAFKQEENwOAAiAEIAUoAQw2AogCIAQgBCgChAIiB0ECdkGw4MCBA3EgBCgCiAIiAUEEdkGPnrz4AHFyIgg2AowCIAQgBCgCgAIiAEG//vz5A3E2AoACIAQgAEECdkGw4MCBA3EgAUGPnrz4AHFyNgKEAiAEIAdBv/78+QNxIgE2AogCIAhBEHZBP3EiACADIAlBpAJsaiIGLgGeAmwhDCAAIAYuAZwCbCENIAhBCHZBP3EiACAGLgGaAmwhDiAAIAYuAZgCbCEPIAhBP3EiACAGLgGWAmwhECAAIAYuAZQCbCERIAFBGHYiACAGLgGSAmwhEiAAIAYuAZACbCETIAdBEHZBP3EiACAGLgGOAmwhFCAAIAYuAYwCbCEVIAdBCHZBP3EiACAGLgGKAmwhFiAAIAYuAYgCbCEXIAZBBGohACAHQT9xIRggBC0AjwIiASAGLgGiAmwhGSABIAYuAaACbCEaIAYuAYYCIQcgBi4BhAIhCCAEIQFBACEK/QwAAAAAAAAAAAAAAAAAAAAAIh4hHwNAIAH9XQAYIiEgG/0NBAUGBwAAAAAAAAAAAAAAAP2HASAA/V0AGCIiIBv9DQQFBgcAAAAAAAAAAAAAAAD9hwH9vAEgBEGAAmogCmotAAD9ESIb/bUBIAH9XQAQIiMgG/0NBAUGBwAAAAAAAAAAAAAAAP2HASAA/V0AECIkIBv9DQQFBgcAAAAAAAAAAAAAAAD9hwH9vAEgG/21ASAB/V0ACCIlIBv9DQQFBgcAAAAAAAAAAAAAAAD9hwEgAP1dAAgiJiAb/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwBIBv9tQEgAf1dAAAiJyAb/Q0EBQYHAAAAAAAAAAAAAAAA/YcBIAD9XQAAIiAgG/0NBAUGBwAAAAAAAAAAAAAAAP2HAf28ASAb/bUBIB/9rgH9rgH9rgH9rgEhHyAh/YcBICL9hwH9vAEgG/21ASAj/YcBICT9hwH9vAEgG/21ASAl/YcBICb9hwH9vAEgG/21ASAn/YcBICD9hwH9vAEgG/21ASAe/a4B/a4B/a4B/a4BIR4gAUEgaiEBIABBIGohACAKQQFqIgpBCEcNAAsgKCAGKgIAIikgBS8BAkECdEGQ1gRqKgIAlCAZIBogDCANIA4gDyAQIBEgEiATIBQgFSAWIBcgGCAHIAhqbGpqampqampqampqampqspSTISggBS8BAEECdEGQ1gRqKgIAICmU/RMiICAf/foB/eYBIB395AEhHSAgIB79+gH95gEgHP3kASEcIAlBAWoiCSALRw0ACyAoIBz9HwCSIBz9HwGSIBz9HwKSIBz9HwOSIB39HwCSIB39HwGSIB39HwKSIB39HwOSBUMAAAAACzgCACAEQZACaiQACwwAIAAQugIaIAAQLwvFDgQRfwl7AX0BfiMAQZACayIFJAAgASAAQYACTgR9IABBgAJtIQ8DQCADIAhBpAJsaiEMIAIgCEHuAGxqIgdBIGohACAH/QAAECEVIAf9AAAAIRkgBUEQaiEEQQEhAUEBIQYDQCAEIAD9AAAAIhf9DAMDAwMDAwMDAwMDAwMDAwP9TiIY/QsAACAA/QAAECEWIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgGSAG/Q8iHP1O/SP9UiAY/VD9CwAAIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgFSAc/U79I/1SIBb9DAMDAwMDAwMDAwMDAwMDAwP9Tv1Q/QsAECAEIABBCmoiCSAAQQlqIg0gAEEIaiIOIAD9XQAA/VQAAAj9VAAACf1UAAAKIAAtAAsiCv0XCyAALQAMIgv9FwwgF/0NAAECAwQFBgcICQoLDB0eH0EC/W39DAMDAwMDAwMDAwMDAwMDAwP9TiIX/QsAICAAQR9qIhAgAEEeaiIRIABBHWoiEiAAQRxqIABBG2ogAEEaaiAAQRlqIABBGGogAEEXaiAAQRZqIBb9VAAABv1UAAAH/VQAAAj9VAAACf1UAAAK/VQAAAv9VAAADP1UAAAN/VQAAA79VAAADyEWIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgGSAGQQF0/Q8iGP1O/SP9UiAX/VD9CwAgIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgFSAY/U79I/1SIBZBAv1t/QwDAwMDAwMDAwMDAwMDAwMD/U79UP0LADAgBCAJIA0gDiAA/V0AAP1UAAAI/VQAAAn9VAAACiAK/RcLIAv9FwwgAC0ADSIK/RcNIAAtAA4iC/0XDiAALQAPIhP9Fw9BBP1t/QwDAwMDAwMDAwMDAwMDAwMD/U4iF/0LAEAgAP0AABAhFiAE/Qz8/Pz8/Pz8/Pz8/Pz8/Pz8/QwAAAAAAAAAAAAAAAAAAAAA/QwAAAAAAAAAAAAAAAAAAAAAIBkgBkECdP0PIhj9Tv0j/VIgF/1Q/QsAQCAE/Qz8/Pz8/Pz8/Pz8/Pz8/Pz8/QwAAAAAAAAAAAAAAAAAAAAA/QwAAAAAAAAAAAAAAAAAAAAAIBUgGP1O/SP9UiAWQQT9bf0MAwMDAwMDAwMDAwMDAwMDA/1O/VD9CwBQIAQgAEEMaiAAQQtqIAkgDSAOIAD9XQAA/VQAAAj9VAAACf1UAAAK/VQAAAv9VAAADCAK/RcNIAv9Fw4gE/0XD0EG/W0iF/0LAGAgECARIBIgFv1UAAAN/VQAAA79VAAADyEWIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgGSAGQQN0/Q8iGP1O/SP9UiAX/VD9CwBgIAT9DPz8/Pz8/Pz8/Pz8/Pz8/Pz9DAAAAAAAAAAAAAAAAAAAAAD9DAAAAAAAAAAAAAAAAAAAAAAgFSAY/U79I/1SIBZBBv1t/VD9CwBwIABBIGohACAGQQR0IQYgBEGAAWohBCABIRRBACEBIBQNAAsgBSAHKAFoIgA2AgggBSAHKQFgIh83AwAgBSAfpyIBQQR2QY+evPgAcSAAQbDgwIEDcXI2AgggBSABQY+evPgAcSAAQQR0QbDgwIEDcXI2AgAgBSAFKAIEIgFBj568+ABxIABBAnRBsODAgQNxcjYCBCAFIAFBBHZBj568+ABxIABBAnZBsODAgQNxcjYCDCAMQQRqIQAgBUEQaiEEQQAhBv0MAAAAAAAAAAAAAAAAAAAAACIZIRYDQCAFIAZqLAAAQSBr/REiFSAE/V0ACCIXIBX9DQQFBgcAAAAAAAAAAAAAAAD9hwEgAP1dAAgiGCAV/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwB/bUBIBUgBP1dAAAiHCAV/Q0EBQYHAAAAAAAAAAAAAAAA/YcBIAD9XQAAIh0gFf0NBAUGBwAAAAAAAAAAAAAAAP2HAf28Af21ASAW/a4B/a4BIRYgFSAX/YcBIBj9hwH9vAH9tQEgFSAc/YcBIB39hwH9vAH9tQEgGf2uAf2uASEZIARBEGohBCAAQRBqIQAgBkEBaiIGQRBHDQALIAcvAWxBAnRBkNYEaioCACAMKgIAlP0TIhUgFv36Af3mASAb/eQBIRsgFSAZ/foB/eYBIBr95AEhGiAIQQFqIgggD0cNAAsgGv0fAEMAAAAAkiAa/R8BkiAa/R8CkiAa/R8DkiAb/R8AkiAb/R8BkiAb/R8CkiAb/R8DkgVDAAAAAAs4AgAgBUGQAmokAAsTACAAIAAoAgBBDGsoAgBqEPYDCxMAIAAgACgCAEEMaygCAGoQvgILlQoCPX8EfSAAQf8BSgRAIABBgAJtIQ0DQCADIAhBpAJsaiIGLgGiAiACIAhB1ABsaiIHLQAPQQR2bCEOIAYuAaACIActAA5BBHZsIQ8gBi4BngIgBy0ADUEEdmwhECAGLgGcAiAHLQAMQQR2bCERIAYuAZoCIActAAtBBHZsIRIgBi4BmAIgBy0ACkEEdmwhEyAGLgGWAiAHLQAJQQR2bCEUIAYuAZQCIActAAhBBHZsIRUgBi4BkgIgBy0AB0EEdmwhFiAGLgGQAiAHLQAGQQR2bCEXIAYuAY4CIActAAVBBHZsIRggBi4BjAIgBy0ABEEEdmwhGSAGLgGKAiAHLQADQQR2bCEaIAYuAYgCIActAAJBBHZsIRsgBi4BhgIgBy0AAUEEdmwhHCAGQQRqIQAgB0EQaiEEIActAABBBHYhHSAGKgIAIUIgBi4BhAIhHiAHLwFSQQJ0QZDWBGoqAgAhQyAHLwFQQQJ0QZDWBGoqAgAhREEAIQlBASELQQAhCgNAIABBgAFqIUAgBC0AHyEgIAQtAB4hISAELQAdISIgBC0AHCEjIAQtABshJCAELQAaISUgBC0AGSEmIAQtABghJyAELQAXISggBC0AFiEpIAQtABUhKiAELQAUISsgBC0AEyEsIAQtABIhLSAELQARIS4gBC0AECEvIAQtAA8hMCAELQAOITEgBC0ADSEyIAQtAAwhMyAELQALITQgBC0ACiE1IAQtAAkhNiAELQAIITcgBC0AByE4IAQtAAYhOSAELQAFITogBC0ABCE7IAQtAAMhPCAELQACIT0gBC0AASE+IAQtAAAhP0EAIQxBACEFIAkhBgNAIAAsAAEgPiAFdkEDcWwgACwAACA/IAV2QQNxbGogACwAAiA9IAV2QQNxbGogACwAAyA8IAV2QQNxbGogACwABCA7IAV2QQNxbGogACwABSA6IAV2QQNxbGogACwABiA5IAV2QQNxbGogACwAByA4IAV2QQNxbGogACwACCA3IAV2QQNxbGogACwACSA2IAV2QQNxbGogACwACiA1IAV2QQNxbGogACwACyA0IAV2QQNxbGogACwADCAzIAV2QQNxbGogACwADSAyIAV2QQNxbGogACwADiAxIAV2QQNxbGogACwADyAwIAV2QQNxbGogBiAHai0AAEEPcWwgCmogACwAESAuIAV2QQNxbCAALAAQIC8gBXZBA3FsaiAALAASIC0gBXZBA3FsaiAALAATICwgBXZBA3FsaiAALAAUICsgBXZBA3FsaiAALAAVICogBXZBA3FsaiAALAAWICkgBXZBA3FsaiAALAAXICggBXZBA3FsaiAALAAYICcgBXZBA3FsaiAALAAZICYgBXZBA3FsaiAALAAaICUgBXZBA3FsaiAALAAbICQgBXZBA3FsaiAALAAcICMgBXZBA3FsaiAALAAdICIgBXZBA3FsaiAALAAeICEgBXZBA3FsaiAALAAfICAgBXZBA3FsaiAHIAZBAXJqLQAAQQ9xbGohCiAAQSBqIQAgBUECaiEFIAZBAmohBiAMQQFqIgxBBEcNAAsgBEEgaiEEIAlBCGohCSALIQZBACELIEAhACAGDQALIEEgQiBElCAKspQgQyBClCAOIA8gECARIBIgEyAUIBUgFiAXIBggGSAaIBsgHCAdIB5sampqampqampqampqampqspSTkiFBIAhBAWoiCCANRw0ACwsgASBBOAIAC6YCAQF/IAAgACgCACgCGBEAABogACABEPgDIgE2AkQgAC0AYiECIAAgASABKAIAKAIcEQAAIgE6AGIgASACRwRAIABBADYCECAAQQA2AgwgAEEANgIIIABBADYCHCAAQQA2AhQgAEEANgIYIAAtAGAhASAALQBiBEACQCABRQ0AIAAoAiAiAUUNACABEC8LIAAgAC0AYToAYCAAIAAoAjw2AjQgACgCOCEBIABCADcCOCAAIAE2AiAgAEEAOgBhDwsCQCABDQAgACgCICIBIABBLGpGDQAgAEEAOgBhIAAgATYCOCAAIAAoAjQiATYCPCABEDIhASAAQQE6AGAgACABNgIgDwsgACAAKAI0IgE2AjwgARAyIQEgAEEBOgBhIAAgATYCOAsL8QMCBX8BfiMAQRBrIgMkAAJAIAAoAkBFDQACQCAAKAJEIgQEQCAAKAJcIgJBEHEEQCAAKAIYIAAoAhRHBEBBfyEBIABBfyAAKAIAKAI0EQQAQX9GDQQLIABByABqIQEDQCAAKAJEIgQgASAAKAIgIgIgAiAAKAI0aiADQQxqIAQoAgAoAhQRCgAhBCAAKAIgIgIgAygCDCACayICIAAoAkAQ1AEgAkcNAwJAIARBAWsOAgEEAAsLQQAhASAAKAJAEDBFDQMMAgsgAkEIcUUNAiADIAApAlA3AwACfwJAAkAgAC0AYgRAIAAoAhAgACgCDGusIQYMAQsgBCAEKAIAKAIYEQAAIQEgACgCKCAAKAIka6whBiABQQBKBEAgACgCECAAKAIMayABbKwgBnwhBgwBCyAAKAIMIAAoAhBHDQELQQAMAQsgACgCRCIBIAMgACgCICAAKAIkIAAoAgwgACgCCGsgASgCACgCIBEKACEBIAAoAiQgASAAKAIgamusIAZ8IQZBAQshBSAAKAJAQgAgBn1BARDPAg0BIAUEQCAAIAMpAwA3AkgLIAAgACgCICIBNgIoIAAgATYCJEEAIQEgAEEANgIQIABBADYCDCAAQQA2AgggAEEANgJcDAILEEYAC0F/IQELIANBEGokACABC4oBACMAQRBrIgMkAAJAAkAgASgCQARAIAEgASgCACgCGBEAAEUNAQsgAEJ/NwMIIABCADcDAAwBCyABKAJAIAIpAwhBABDPAgRAIABCfzcDCCAAQgA3AwAMAQsgAyACKQMANwIIIAEgAykDCDcCSCAAIAIpAwg3AwggACACKQMANwMACyADQRBqJAALjwIBAn8jAEEQayIEJAAgASgCRCIFBEAgBSAFKAIAKAIYEQAAIQUCQAJAAkAgASgCQEUNACAFQQBMIAJCAFJxDQAgASABKAIAKAIYEQAARQ0BCyAAQn83AwggAEIANwMADAELIANBA08EQCAAQn83AwggAEIANwMADAELIAEoAkAgBa0gAn5CACAFQQBKGyADEM8CBEAgAEJ/NwMIIABCADcDAAwBCyAAAn4gASgCQCIDKAJMQQBIBEAgAxCoBAwBCyADEIIBIQYgAxCoBCECIAYEQCADEIEBCyACCzcDCCAAQgA3AwAgBCABKQJIIgI3AwAgBCACNwMIIAAgBCkCADcDAAsgBEEQaiQADwsQRgAL3wIBBH8jAEEQayIEJAAgBCACNgIMIABBADYCECAAQQA2AgwgAEEANgIIIABBADYCHCAAQQA2AhQgAEEANgIYAkAgAC0AYEUNACAAKAIgIgNFDQAgAxAvCwJAIAAtAGFFDQAgACgCOCIDRQ0AIAMQLwsgACACNgI0IAACfwJAAkAgAkEJTwRAIAAtAGIhAwJAIAFFDQAgA0UNACAAQQA6AGAgACABNgIgDAMLIAIQMiECIABBAToAYCAAIAI2AiAMAQsgAEEAOgBgIABBCDYCNCAAIABBLGo2AiAgAC0AYiEDCyADDQAgBEEINgIIIwBBEGsiAiQAIARBDGoiAygCACAEQQhqIgUoAgBIIQYgAkEQaiQAIAAgBSADIAYbKAIAIgI2AjwgAQRAQQAgAkEHSw0CGgsgAhAyIQFBAQwBC0EAIQEgAEEANgI8QQALOgBhIAAgATYCOCAEQRBqJAAgAAvdBAEHfyMAQRBrIgMkAAJ/AkAgACgCQEUNACAALQBcQRBxRQRAIABBADYCECAAQQA2AgwgAEEANgIIAkAgACgCNCIFQQlPBEAgAC0AYgRAIAAgACgCICICIAVqQQFrNgIcIAAgAjYCFCAAIAI2AhgMAgsgACAAKAI4IgIgACgCPGpBAWs2AhwgACACNgIUIAAgAjYCGAwBCyAAQQA2AhwgAEEANgIUIABBADYCGAsgAEEQNgJcCyAAKAIUIQUgACgCHCEHIAFBf0cEQCAAKAIYRQRAIAAgA0EQajYCHCAAIANBD2oiAjYCFCAAIAI2AhgLIAAoAhggAcA6AAAgACAAKAIYQQFqNgIYCyAAKAIYIgYgACgCFCICRwRAAkAgAC0AYgRAIAIgBiACayICIAAoAkAQ1AEgAkcNAwwBCyADIAAoAiA2AgggAEHIAGohBgNAIAAoAkQiAgRAIAIgBiAAKAIUIAAoAhggA0EEaiAAKAIgIgQgBCAAKAI0aiADQQhqIAIoAgAoAgwRDQAhAiAAKAIUIAMoAgRGDQQgAkEDRgRAIAAoAhQgACgCGCAAKAIUayICIAAoAkAQ1AEgAkcNBQwDCyACQQFLDQQgACgCICIEIAMoAgggBGsiBCAAKAJAENQBIARHDQQgAkEBRw0CIAMoAgQhAiAAIAAoAhg2AhwgACACNgIUIAAgAjYCGCAAIAAoAhggACgCHCAAKAIUa2o2AhgMAQsLEEYACyAAIAc2AhwgACAFNgIUIAAgBTYCGAsgAUEAIAFBf0cbDAELQX8LIQggA0EQaiQAIAgLeAEBfwJAIAAoAkBFDQAgACgCDCICIAAoAghNDQAgAUF/RgRAIAAgAkEBazYCDCABQQAgAUF/RxsPCyAALQBYQRBxRQRAIAAoAgxBAWstAAAgAUH/AXFHDQELIAAgACgCDEEBazYCDCAAKAIMIAHAOgAAIAEPC0F/C88GAQd/IwBBEGsiBCQAAkACQCAAKAJARQRAQX8hBQwBCyAAKAJcQQhxIgVFBEAgAEEANgIcIABBADYCFCAAQQA2AhgCQCAALQBiBEAgACAAKAIgIgEgACgCNGoiAjYCEAwBCyAAIAAoAjgiASAAKAI8aiICNgIQCyAAIAI2AgwgACABNgIIIABBCDYCXAsgACgCDEUEQCAAIARBEGoiATYCECAAIAE2AgwgACAEQQ9qNgIICyAFBEAgACgCECEDIAAoAgghBSAEQQQ2AgQgBCADIAVrQQJtNgIIIwBBEGsiAyQAIARBBGoiBSgCACAEQQhqIgEoAgBJIQIgA0EQaiQAIAUgASACGygCACEDC0F/IQUCQCAAKAIQIgEgACgCDEYEQCAAKAIIIAEgA2sgA/wKAAAgAC0AYgRAIAMgACgCCGogACgCECAAKAIIIANqayAAKAJAEKoEIgFFDQIgACgCCCEFIAMgACgCCGohAiAAIAMgACgCCGogAWo2AhAgACACNgIMIAAgBTYCCCAAKAIMLQAAIQUMAgsCfyAAKAIoIgEgACgCJCICRgRAIAEMAQsgACgCICACIAEgAmv8CgAAIAAoAiQhASAAKAIoCyEGIAAgACgCICICIAYgAWtqIgE2AiQgACACQQggACgCNCACIABBLGpGG2oiAjYCKCAEIAAoAjwgA2s2AgggBCACIAFrNgIEIwBBEGsiASQAIARBBGoiAigCACAEQQhqIgYoAgBJIQcgAUEQaiQAIAIgBiAHGygCACEBIAAgACkCSDcCUCAAKAIkIAEgACgCQBCqBCICRQ0BIAAoAkQiAUUNAyAAIAAoAiQgAmoiAjYCKAJAIAEgAEHIAGogACgCICACIABBJGogAyAAKAIIIgJqIAAoAjwgAmogBiABKAIAKAIQEQ0AQQNGBEAgACgCICEDIAAgACgCKDYCECAAIAM2AgwgACADNgIIDAELIAQoAggiASADIAAoAggiAmoiA0YNAiAAIAE2AhAgACADNgIMIAAgAjYCCAsgACgCDC0AACEFDAELIAAoAgwtAAAhBQsgACgCCCAEQQ9qRw0AIABBADYCECAAQQA2AgwgAEEANgIICyAEQRBqJAAgBQ8LEEYACwwAIAAQ8gEaIAAQLwumBAMEewN/AX0gAEEgTgRAIABBIG0hCkEAIQADQCADIABBImwiCGoiCf0AAAIiBP2HASACIAhqIgj9AAACIgX9hwH9vAEgCf0AABIiBv2HASAI/QAAEiIH/YcB/bwB/a4BIAQgBP0NCAkKCwAAAAAAAAAAAAAAAP2HASAFIAT9DQgJCgsAAAAAAAAAAAAAAAD9hwH9vAEgBiAE/Q0ICQoLAAAAAAAAAAAAAAAA/YcBIAcgBP0NCAkKCwAAAAAAAAAAAAAAAP2HAf28Af2uAf2uASAEIAT9DQQFBgcAAAAAAAAAAAAAAAD9hwEgBSAE/Q0EBQYHAAAAAAAAAAAAAAAA/YcB/bwBIAYgBP0NBAUGBwAAAAAAAAAAAAAAAP2HASAHIAT9DQQFBgcAAAAAAAAAAAAAAAD9hwH9vAH9rgEgBCAE/Q0MDQ4PAAAAAAAAAAAAAAAA/YcBIAUgBP0NDA0ODwAAAAAAAAAAAAAAAP2HAf28ASAGIAT9DQwNDg8AAAAAAAAAAAAAAAD9hwEgByAE/Q0MDQ4PAAAAAAAAAAAAAAAA/YcB/bwB/a4B/a4B/a4BIgQgBCAE/Q0ICQoLDA0ODwABAgMAAQID/a4BIgQgBCAE/Q0EBQYHAAECAwABAgMAAQID/a4B/RsAsiAILwEAQQJ0QZDWBGoqAgAgCS8BAEECdEGQ1gRqKgIAlJQgC5IhCyAAQQFqIgAgCkcNAAsLIAEgCzgCAAsHACAAKAIMCwcAIAAoAggL1wIDA3sBfQR/IABBH0oEQCAAQSBtIQtBACEAA0AgBCACIABBGGxqIggvAQBBAnRBkNYEaioCACADIABBKGxqIgkqAgCU/RMgCP0AAAgiBEEE/W0gCCgBBCIKQRV2QfgPcUGQiwFqIApBDXZB+A9xQZCLAWr9CgMA/VcDAAH9UCIF/YgBIAn9AAAYIgb9iAH9ugEgBf2HASAG/YcB/boB/a4BIAT9DA8PDw8PDw8PDw8PDw8PDw/9TiAKQQV2QfgPcUGQiwFqIApB/wFxQQN0QZCLAWr9CgMA/VcDAAH9UCIE/YcBIAn9AAAIIgX9hwH9ugH9rgEgBP2IASAF/YgB/boB/a4B/foB/eYB/eQBIQQgCC8BAkECdEGQ1gRqKgIAIAkqAgSUIAeSIQcgAEEBaiIAIAtHDQALCyABIAcgBP0fAyAE/R8CIAT9HwAgBP0fAZKSkpI4AgALBwAgABEJAAvCAgIDewN/IABBH0oEQCAAQSBtIQlBACEAA0AgBCACIABBFmxqIgcvAQBBAnRBkNYEaioCACADIABBImxqIggvAQBBAnRBkNYEaioCAJT9EyAH/QAABiIEQQT9bSAHKAECIgdBFXZB+A9xQZD7AGogB0ENdkH4D3FBkPsAav0KAwD9VwMAAf1xIgX9iAEgCP0AABIiBv2IAf26ASAF/YcBIAb9hwH9ugH9rgEgBP0MDw8PDw8PDw8PDw8PDw8PD/1OIAdBBXZB+A9xQZD7AGogB0H/AXFBA3RBkPsAav0KAwD9VwMAAf1xIgT9hwEgCP0AAAIiBf2HAf26Af2uASAE/YgBIAX9iAH9ugH9rgH9+gH95gH95AEhBCAAQQFqIgAgCUcNAAsLIAEgBP0fAyAE/R8CIAT9HwAgBP0fAZKSkjgCAAsTACAAIAAoAgBBDGsoAgBqEP8DCxMAIAAgACgCAEEMaygCAGoQvwILygEBBn8jAEEQayIFJAADQAJAIAIgBEwNACAAKAIYIgMgACgCHCIGTwR/IAAgAS0AACAAKAIAKAI0EQQAQX9GDQEgBEEBaiEEIAFBAWoFIAUgBiADazYCDCAFIAIgBGs2AggjAEEQayIDJAAgBUEIaiIGKAIAIAVBDGoiBygCAEghCCADQRBqJAAgBiAHIAgbIQMgACgCGCABIAMoAgAiAxB3IAAgAyAAKAIYajYCGCADIARqIQQgASADagshAQwBCwsgBUEQaiQAIAQLLAAgACAAKAIAKAIkEQAAQX9GBEBBfw8LIAAgACgCDCIAQQFqNgIMIAAtAAALBABBfwu+BAIEfwF9IABBIE4EQCAAQSBtIQcDQCAIIAIgBkEUbGoiBS8BAEECdEGQ1gRqKgIAIAMgBkEobGoiACoCAJQgACwACCAFLQAEIgRBD3FsIAAsABggBEEEdmxqIAAsAAkgBS0ABSIEQQ9xbGogACwAGSAEQQR2bGogACwACiAFLQAGIgRBD3FsaiAALAAaIARBBHZsaiAALAALIAUtAAciBEEPcWxqIAAsABsgBEEEdmxqIAAsAAwgBS0ACCIEQQ9xbGogACwAHCAEQQR2bGogACwADSAFLQAJIgRBD3FsaiAALAAdIARBBHZsaiAALAAOIAUtAAoiBEEPcWxqIAAsAB4gBEEEdmxqIAAsAA8gBS0ACyIEQQ9xbGogACwAHyAEQQR2bGogACwAECAFLQAMIgRBD3FsaiAALAAgIARBBHZsaiAALAARIAUtAA0iBEEPcWxqIAAsACEgBEEEdmxqIAAsABIgBS0ADiIEQQ9xbGogACwAIiAEQQR2bGogACwAEyAFLQAPIgRBD3FsaiAALAAjIARBBHZsaiAALAAUIAUtABAiBEEPcWxqIAAsACQgBEEEdmxqIAAsABUgBS0AESIEQQ9xbGogACwAJSAEQQR2bGogACwAFiAFLQASIgRBD3FsaiAALAAmIARBBHZsaiAALAAXIAUtABMiBEEPcWxqIAAsACcgBEEEdmxqspQgBS8BAkECdEGQ1gRqKgIAIAAqAgSUkpIhCCAGQQFqIgYgB0cNAAsLIAEgCDgCAAuBAgEGfyMAQRBrIgQkAANAAkAgAiAGTA0AAkAgACgCDCIDIAAoAhAiBUkEQCAEQf////8HNgIMIAQgBSADazYCCCAEIAIgBms2AgQjAEEQayIDJAAgBEEEaiIFKAIAIARBCGoiBygCAEghCCADQRBqJAAgBSAHIAgbIQMjAEEQayIFJAAgAygCACAEQQxqIgcoAgBIIQggBUEQaiQAIAMgByAIGyEDIAEgACgCDCADKAIAIgMQdyAAIAAoAgwgA2o2AgwMAQsgACAAKAIAKAIoEQAAIgNBf0YNASABIAPAOgAAQQEhAwsgASADaiEBIAMgBmohBgwBCwsgBEEQaiQAIAYLEAAgAEJ/NwMIIABCADcDAAsQACAAQn83AwggAEIANwMACwQAIAALMgEBfyAAQeTHAjYCACAAKAIEIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAEC8LMAEBfyAAQeTHAjYCACAAKAIEIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAACwUAQZMaC5IFAgV/AX0gAEEgTgRAIABBIG0hBwNAIAIgBkESbGoiBS8BAEECdEGQ1gRqKgIAIAMgBkEibGoiACwAISAFLQARIghBBHZBCGtsIAAsABIgBS0AAiIEQQR2QQhrbCAALAACIARBD3FBCGtsaiAALAATIAUtAAMiBEEEdkEIa2xqIAAsAAMgBEEPcUEIa2xqIAAsABQgBS0ABCIEQQR2QQhrbGogACwABCAEQQ9xQQhrbGogACwAFSAFLQAFIgRBBHZBCGtsaiAALAAFIARBD3FBCGtsaiAALAAWIAUtAAYiBEEEdkEIa2xqIAAsAAYgBEEPcUEIa2xqIAAsABcgBS0AByIEQQR2QQhrbGogACwAByAEQQ9xQQhrbGogACwAGCAFLQAIIgRBBHZBCGtsaiAALAAIIARBD3FBCGtsaiAALAAZIAUtAAkiBEEEdkEIa2xqIAAsAAkgBEEPcUEIa2xqIAAsABogBS0ACiIEQQR2QQhrbGogACwACiAEQQ9xQQhrbGogACwAGyAFLQALIgRBBHZBCGtsaiAALAALIARBD3FBCGtsaiAALAAcIAUtAAwiBEEEdkEIa2xqIAAsAAwgBEEPcUEIa2xqIAAsAB0gBS0ADSIEQQR2QQhrbGogACwADSAEQQ9xQQhrbGogACwAHiAFLQAOIgRBBHZBCGtsaiAALAAOIARBD3FBCGtsaiAALAAfIAUtAA8iBEEEdkEIa2xqIAAsAA8gBEEPcUEIa2xqIAAsACAgBS0AECIFQQR2QQhrbGogACwAECAFQQ9xQQhrbGpqIAAsABEgCEEPcUEIa2xqspQgAC8BAEECdEGQ1gRqKgIAlCAJkiEJIAZBAWoiBiAHRw0ACwsgASAJOAIACwoAIAAkCCABJAcLmQUDCn0GfwF7IAJBgAJOBEAgAkGAAm0hEgNAQQAhDUMAAAAAIQNDAAAAACEEA0AgACANQQJ0IgJBDHJqKgIAIgWLIgYgACACQQhyaioCACIHiyIIIAAgAkEEcmoqAgAiCYsiCiAAIAJqKgIAIguLIgwgAyADIAxdIgIbIgMgAyAKXSIOGyIDIAMgCF0iEBsiAyADIAZdIhEbIQMgBSAHIAkgCyAEIAIbIA4bIBAbIBEbIQQgDUEEaiINQYACRw0ACwJAIANDAAAAAFsEQCABIA9BpAJsakEAQYQC/AsADAELQwAAAMMgBJUiA/0TIRNBACENQQAhAgNAIAEgD0GkAmxqIg5BBGoiECACaiATIAAgAkECdGr9AAIA/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DH8AQAB/AEAAfwBAAH8AQAD9tgEgE/0NAAQIDAAAAAAAAAAAAAAAAP1aAAAAIBAgAkEEciIRaiATIAAgEUECdGr9AAIA/eYB/QwAAEBLAABASwAAQEsAAEBL/eQB/Qz//38A//9/AP//fwD//38A/U79DH8AQAB/AEAAfwBAAH8AQAD9tgEgE/0NAAQIDAAAAAAAAAAAAAAAAP1aAAAAIAJBCGoiAkGAAkcNAAsDQCAOIA1BAXRqIA4gDUEEdGoiAiwABCACLAAFaiACLAAGaiACLAAHaiACLAAIaiACLAAJaiACLAAKaiACLAALaiACLAAMaiACLAANaiACLAAOaiACLAAPaiACLAAQaiACLAARaiACLAASaiACLAATajsBhAIgDUEBaiINQRBHDQALIA5DAACAPyADlTgCAAsgAEGACGohACAPQQFqIg8gEkcNAAsLC6gBAQV/IAAoAlQiAygCACEFIAMoAgQiBCAAKAIUIAAoAhwiB2siBiAEIAZJGyIGBEAgBSAHIAYQexogAyADKAIAIAZqIgU2AgAgAyADKAIEIAZrIgQ2AgQLIAQgAiACIARLGyIEBEAgBSABIAQQexogAyADKAIAIARqIgU2AgAgAyADKAIEIARrNgIECyAFQQA6AAAgACAAKAIsIgE2AhwgACABNgIUIAILKQAgASABKAIAQQdqQXhxIgFBEGo2AgAgACABKQMAIAEpAwgQwQI5AwALnxgDE38BfAJ+IwBBsARrIgwkACAMQQA2AiwCQCABvSIaQgBTBEBBASEQQaAKIRMgAZoiAb0hGgwBCyAEQYAQcQRAQQEhEEGjCiETDAELQaYKQaEKIARBAXEiEBshEyAQRSEVCwJAIBpCgICAgICAgPj/AINCgICAgICAgPj/AFEEQCAAQSAgAiAQQQNqIgMgBEH//3txEGcgACATIBAQYSAAQZYYQcouIAVBIHEiBRtBsh5BhzIgBRsgASABYhtBAxBhIABBICACIAMgBEGAwABzEGcgAyACIAIgA0gbIQkMAQsgDEEQaiERAkACfwJAIAEgDEEsahCKBCIBIAGgIgFEAAAAAAAAAABiBEAgDCAMKAIsIgZBAWs2AiwgBUEgciIOQeEARw0BDAMLIAVBIHIiDkHhAEYNAiAMKAIsIQpBBiADIANBAEgbDAELIAwgBkEdayIKNgIsIAFEAAAAAAAAsEGiIQFBBiADIANBAEgbCyELIAxBMGpBoAJBACAKQQBOG2oiDSEHA0AgBwJ/IAFEAAAAAAAA8EFjIAFEAAAAAAAAAABmcQRAIAGrDAELQQALIgM2AgAgB0EEaiEHIAEgA7ihRAAAAABlzc1BoiIBRAAAAAAAAAAAYg0ACwJAIApBAEwEQCAKIQMgByEGIA0hCAwBCyANIQggCiEDA0BBHSADIANBHU4bIQMCQCAHQQRrIgYgCEkNACADrSEbQgAhGgNAIAYgGkL/////D4MgBjUCACAbhnwiGiAaQoCU69wDgCIaQoCU69wDfn0+AgAgBkEEayIGIAhPDQALIBqnIgZFDQAgCEEEayIIIAY2AgALA0AgCCAHIgZJBEAgBkEEayIHKAIARQ0BCwsgDCAMKAIsIANrIgM2AiwgBiEHIANBAEoNAAsLIANBAEgEQCALQRlqQQluQQFqIQ8gDkHmAEYhEgNAQQlBACADayIDIANBCU4bIQkCQCAGIAhNBEAgCCgCACEHDAELQYCU69wDIAl2IRRBfyAJdEF/cyEWQQAhAyAIIQcDQCAHIAMgBygCACIXIAl2ajYCACAWIBdxIBRsIQMgB0EEaiIHIAZJDQALIAgoAgAhByADRQ0AIAYgAzYCACAGQQRqIQYLIAwgDCgCLCAJaiIDNgIsIA0gCCAHRUECdGoiCCASGyIHIA9BAnRqIAYgBiAHa0ECdSAPShshBiADQQBIDQALC0EAIQMCQCAGIAhNDQAgDSAIa0ECdUEJbCEDQQohByAIKAIAIglBCkkNAANAIANBAWohAyAJIAdBCmwiB08NAAsLIAsgA0EAIA5B5gBHG2sgDkHnAEYgC0EAR3FrIgcgBiANa0ECdUEJbEEJa0gEQEEEQaQCIApBAEgbIAxqIAdBgMgAaiIJQQltIg9BAnRqQdAfayEKQQohByAJIA9BCWxrIglBB0wEQANAIAdBCmwhByAJQQFqIglBCEcNAAsLAkAgCigCACISIBIgB24iDyAHbGsiCUUgCkEEaiIUIAZGcQ0AAkAgD0EBcUUEQEQAAAAAAABAQyEBIAdBgJTr3ANHDQEgCCAKTw0BIApBBGstAABBAXFFDQELRAEAAAAAAEBDIQELRAAAAAAAAOA/RAAAAAAAAPA/RAAAAAAAAPg/IAYgFEYbRAAAAAAAAPg/IAkgB0EBdiIURhsgCSAUSRshGQJAIBUNACATLQAAQS1HDQAgGZohGSABmiEBCyAKIBIgCWsiCTYCACABIBmgIAFhDQAgCiAHIAlqIgM2AgAgA0GAlOvcA08EQANAIApBADYCACAIIApBBGsiCksEQCAIQQRrIghBADYCAAsgCiAKKAIAQQFqIgM2AgAgA0H/k+vcA0sNAAsLIA0gCGtBAnVBCWwhA0EKIQcgCCgCACIJQQpJDQADQCADQQFqIQMgCSAHQQpsIgdPDQALCyAKQQRqIgcgBiAGIAdLGyEGCwNAIAYiByAITSIJRQRAIAZBBGsiBigCAEUNAQsLAkAgDkHnAEcEQCAEQQhxIQoMAQsgA0F/c0F/IAtBASALGyIGIANKIANBe0pxIgobIAZqIQtBf0F+IAobIAVqIQUgBEEIcSIKDQBBdyEGAkAgCQ0AIAdBBGsoAgAiDkUNAEEKIQlBACEGIA5BCnANAANAIAYiCkEBaiEGIA4gCUEKbCIJcEUNAAsgCkF/cyEGCyAHIA1rQQJ1QQlsIQkgBUFfcUHGAEYEQEEAIQogCyAGIAlqQQlrIgZBACAGQQBKGyIGIAYgC0obIQsMAQtBACEKIAsgAyAJaiAGakEJayIGQQAgBkEAShsiBiAGIAtKGyELC0F/IQkgC0H9////B0H+////ByAKIAtyIhIbSg0BIAsgEkEAR2pBAWohDgJAIAVBX3EiFUHGAEYEQCADIA5B/////wdzSg0DIANBACADQQBKGyEGDAELIBEgAyADQR91IgZzIAZrrSAREMYBIgZrQQFMBEADQCAGQQFrIgZBMDoAACARIAZrQQJIDQALCyAGQQJrIg8gBToAACAGQQFrQS1BKyADQQBIGzoAACARIA9rIgYgDkH/////B3NKDQILIAYgDmoiAyAQQf////8Hc0oNASAAQSAgAiADIBBqIgUgBBBnIAAgEyAQEGEgAEEwIAIgBSAEQYCABHMQZwJAAkACQCAVQcYARgRAIAxBEGoiBkEIciEDIAZBCXIhCiANIAggCCANSxsiCSEIA0AgCDUCACAKEMYBIQYCQCAIIAlHBEAgBiAMQRBqTQ0BA0AgBkEBayIGQTA6AAAgBiAMQRBqSw0ACwwBCyAGIApHDQAgDEEwOgAYIAMhBgsgACAGIAogBmsQYSAIQQRqIgggDU0NAAsgEgRAIABB0coAQQEQYQsgByAITQ0BIAtBAEwNAQNAIAg1AgAgChDGASIGIAxBEGpLBEADQCAGQQFrIgZBMDoAACAGIAxBEGpLDQALCyAAIAZBCSALIAtBCU4bEGEgC0EJayEGIAhBBGoiCCAHTw0DIAtBCUohGCAGIQsgGA0ACwwCCwJAIAtBAEgNACAHIAhBBGogByAISxshCSAMQRBqIgZBCHIhAyAGQQlyIQ0gCCEHA0AgDSAHNQIAIA0QxgEiBkYEQCAMQTA6ABggAyEGCwJAIAcgCEcEQCAGIAxBEGpNDQEDQCAGQQFrIgZBMDoAACAGIAxBEGpLDQALDAELIAAgBkEBEGEgBkEBaiEGIAogC3JFDQAgAEHRygBBARBhCyAAIAYgDSAGayIGIAsgBiALSBsQYSALIAZrIQsgB0EEaiIHIAlPDQEgC0EATg0ACwsgAEEwIAtBEmpBEkEAEGcgACAPIBEgD2sQYQwCCyALIQYLIABBMCAGQQlqQQlBABBnCyAAQSAgAiAFIARBgMAAcxBnIAUgAiACIAVIGyEJDAELIBMgBUEadEEfdUEJcWohCAJAIANBC0sNAEEMIANrIQZEAAAAAAAAMEAhGQNAIBlEAAAAAAAAMECiIRkgBkEBayIGDQALIAgtAABBLUYEQCAZIAGaIBmhoJohAQwBCyABIBmgIBmhIQELIBEgDCgCLCIGIAZBH3UiBnMgBmutIBEQxgEiBkYEQCAMQTA6AA8gDEEPaiEGCyAQQQJyIQsgBUEgcSENIAwoAiwhByAGQQJrIgogBUEPajoAACAGQQFrQS1BKyAHQQBIGzoAACAEQQhxIQYgDEEQaiEHA0AgByIFAn8gAZlEAAAAAAAA4EFjBEAgAaoMAQtBgICAgHgLIgdBkMcCai0AACANcjoAACABIAe3oUQAAAAAAAAwQKIhAQJAIAVBAWoiByAMQRBqa0EBRw0AAkAgBg0AIANBAEoNACABRAAAAAAAAAAAYQ0BCyAFQS46AAEgBUECaiEHCyABRAAAAAAAAAAAYg0AC0F/IQlB/f///wcgCyARIAprIgZqIg1rIANIDQAgAEEgIAIgDSADQQJqIAcgDEEQaiIHayIFIAVBAmsgA0gbIAUgAxsiCWoiAyAEEGcgACAIIAsQYSAAQTAgAiADIARBgIAEcxBnIAAgByAFEGEgAEEwIAkgBWtBAEEAEGcgACAKIAYQYSAAQSAgAiADIARBgMAAcxBnIAMgAiACIANIGyEJCyAMQbAEaiQAIAkLCwAgACABIAIQjwQLIQEBfyMDKAJ4IgBBAf4XAgAgABDMAiAAQQFBAP5IAgAaC70IAgl/BHsgAkH/AUoEQCACQYACbSEKA0AgACAIQdIBbGoiA0HAAWohBSADQYABaiEGIAMvAdABQQJ0QZDWBGoqAgD9EyEMQQAhAgNAIAMgAkEgaiIJav1cAAAhDiABIAJBAnRqIgcgDCAFIAJBBHZqIgT9BwAA/YcB/acB/foB/eYBIAIgA2r9XAAAIg/9DA8PDw8PDw8PDw8PDw8PDw/9Tv0M4ODg4ODg4ODg4ODg4ODg4P1QIAIgBmr9XAAAIg1BBP1r/QwwMDAwMDAwMDAwMDAwMDAw/U79bv2HAf2nAf36Af3mAf0LAgAgASAJQQJ0aiAMIARBAmr9BwAA/YcB/acB/foB/eYBIA1BAv1r/QwwMDAwMDAwMDAwMDAwMDAw/U4gDv0MDw8PDw8PDw8PDw8PDw8PD/1O/VD9DODg4ODg4ODg4ODg4ODg4OD9bv2HAf2nAf36Af3mAf0LAgAgByAMIARBBGr9BwAA/YcB/acB/foB/eYBIA9BBP1t/Qzg4ODg4ODg4ODg4ODg4ODg/VAgDf0MMDAwMDAwMDAwMDAwMDAwMP1O/W79hwH9pwH9+gH95gH9CwKAAiAHIAwgBEEGav0HAAD9hwH9pwH9+gH95gEgDUEC/W39DDAwMDAwMDAwMDAwMDAwMDD9TiAOQQT9bf1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79hwH9pwH9+gH95gH9CwKAAyACQQRqIgJBIEcNAAsgBUEIaiEHIAZBIGohCSADQUBrIQQgAUGABGohBUEAIQIDQCAEIAJBIGoiC2r9XAAAIQ4gBSACQQJ0aiIGIAwgByACQQR2aiID/QcAAP2HAf2nAf36Af3mASACIARq/VwAACIP/QwPDw8PDw8PDw8PDw8PDw8P/U79DODg4ODg4ODg4ODg4ODg4OD9UCACIAlq/VwAACINQQT9a/0MMDAwMDAwMDAwMDAwMDAwMP1O/W79hwH9pwH9+gH95gH9CwIAIAUgC0ECdGogDCADQQJq/QcAAP2HAf2nAf36Af3mASANQQL9a/0MMDAwMDAwMDAwMDAwMDAwMP1OIA79DA8PDw8PDw8PDw8PDw8PDw/9Tv1Q/Qzg4ODg4ODg4ODg4ODg4ODg/W79hwH9pwH9+gH95gH9CwIAIAYgDCADQQRq/QcAAP2HAf2nAf36Af3mASAPQQT9bf0M4ODg4ODg4ODg4ODg4ODg4P1QIA39DDAwMDAwMDAwMDAwMDAwMDD9Tv1u/YcB/acB/foB/eYB/QsCgAIgBiAMIANBBmr9BwAA/YcB/acB/foB/eYBIA1BAv1t/QwwMDAwMDAwMDAwMDAwMDAw/U4gDkEE/W39UP0M4ODg4ODg4ODg4ODg4ODg4P1u/YcB/acB/foB/eYB/QsCgAMgAkEEaiICQSBHDQALIAFBgAhqIQEgCEEBaiIIIApHDQALCwsEAEIACwQAIwMLFQAgAEEA/kECAEEDRgRAIAAQ+gELCwsAIAAgASACEJcEC5EEAQl/IwMiAUEBOgAoIAEgADYCQCABQQA6ACkgAUEB/iUCfEEBayIABEAgAUH8AGohAgNAIAIgAEQAAAAAAADwfxDVARogAv4QAgAiAA0ACwsgASgCeBCeBAJAIAEoAngiAP4QAgBFBEAgABCgBAwBC0HUuAMQVhogAEHQuAM2AjggAEGEuQMoAgA2AjRBhLkDIAA2AgAgACgCNCAANgI4QdS4AxBSGgsjAyECA0AgAigCRCIABEAgACgCBCEJIAAoAgAhByACIAAoAgg2AkQgCSAHEQEADAELC0EAIQICQCMDIgAtACpBAXFFDQADQBCQBCAAIAAtACpB/gFxOgAqQQAhAwNAIANBAnQiBUHQ9zRqKAIAIQQgACgCSCAFaiIGKAIAIQUgBkEANgIAAkAgBUUNACAERQ0AIARByQFGDQAQxwIgBSAEEQEAEJAECyADQQFqIgNBgAFHDQALEMcCIAAtACpBAXFFDQEgAkEDSSEIIAJBAWohAiAIDQALC0Gc8TRBnPE0KAIAQQFrIgA2AgAgAEUEQEGb8TRBADoAAAsQygIgASgCDCIAIAEoAgg2AgggASgCCCAANgIMIAEgATYCCCABIAE2AgwQyQIjBUUEQEEAJANBACQEQQAkBUEBJAYgAUEgaiIAQQJBAf5IAgBBA0YEQCABEAsPCyAAQQD+FwIAIAAQjgEPC0EAECwACxUAIAAoAiwiAEEAQYQBEKABIAAQLwuzFAMSfwN7BX0gAkH/AUoEQCACQYACbSESA0AgC0GwAWwiAkHQAGohEyAAIAJqIgJBBGohByACQRBqIQUgAi8BAEECdEGQ1gRqKgIAIRogAi8BAkECdEGQ1gRqKgIAjCEbQQAhDCACQTBqIg4hBEEAIQ1BAiEIQQEhCUEAIQYDQAJAIAZBA00EQCAGIAdqIgItAABBP3EhAyACLQAEQT9xIQIMAQsgBiAHaiIDLQAAQQJ2QTBxIAMtAAQiCkEEdnIhAiADQQRrLQAAQQJ2QTBxIApBD3FyIQMLIBMgDEEFdGogAGohDyAaIAOzlCEYIBsgArOUIRkCfyAGQQFyIgJBA00EQCAGIAdqLQAFQT9xIRAgAiAHai0AAEE/cQwBCyACIAdqLQAAQQJ2QTBxIAYgB2oiAi0ABSIDQQR2ciEQIAJBA2stAABBAnZBMHEgA0EPcXILsyEcQQAhAgJAAkAgASAOSSAFIAFBgAFqIgpJcQRAIAEhAwwBCyAEIApJIA8gASIDS3ENACADIBj9EyIV/QwAAAAAAAAAAAAAAAAAAAAA/QwQAAAAEAAAABAAAAAQAAAAIAX9XAAAIAn9DyIW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAA/QwPDw8PDw8PDw8PDw8PDw8P/U79iQH9qQH9UP36Af3mASAZ/RMiF/3kAf0LAgAgAyAV/QwAAAAAAAAAAAAAAAAAAAAA/QwQAAAAEAAAABAAAAAQAAAAIAX9XAAEIBb9Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgBP1cAAT9DA8PDw8PDw8PDw8PDw8PDw/9Tv2JAf2pAf1Q/foB/eYBIBf95AH9CwIQIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwACCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAI/QwPDw8PDw8PDw8PDw8PDw8P/U79iQH9qQH9UP36Af3mASAX/eQB/QsCICADIBX9DAAAAAAAAAAAAAAAAAAAAAD9DBAAAAAQAAAAEAAAABAAAAAgBf1cAAwgFv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAE/VwADP0MDw8PDw8PDw8PDw8PDw8PD/1O/YkB/akB/VD9+gH95gEgF/3kAf0LAjAgA0FAayAV/QwAAAAAAAAAAAAAAAAAAAAA/QwQAAAAEAAAABAAAAAQAAAAIAX9XAAQIBb9Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgBP1cABD9DA8PDw8PDw8PDw8PDw8PDw/9Tv2JAf2pAf1Q/foB/eYBIBf95AH9CwIAIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwAFCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAU/QwPDw8PDw8PDw8PDw8PDw8P/U79iQH9qQH9UP36Af3mASAX/eQB/QsCUCADIBX9DAAAAAAAAAAAAAAAAAAAAAD9DBAAAAAQAAAAEAAAABAAAAAgBf1cABggFv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAE/VwAGP0MDw8PDw8PDw8PDw8PDw8PD/1O/YkB/akB/VD9+gH95gEgF/3kAf0LAmAgAyAV/QwAAAAAAAAAAAAAAAAAAAAA/QwQAAAAEAAAABAAAAAQAAAAIAX9XAAcIBb9Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgBP1cABz9DA8PDw8PDw8PDw8PDw8PDw/9Tv2JAf2pAf1Q/foB/eYBIBf95AH9CwJwQR8hESAKIQMMAQsDQCADIBggAiAEai0AAEEPcSAJIAIgBWotAABxQQBHQQR0crKUIBmSOAIAIANBBGohAyACIhFBAWoiAkEgRw0ACwsgGiAclCEYIBsgELOUIRlBACECAkACQCADIA5JIAUgEUECdCABakGEAWoiAUlxDQAgAyAPSSABIARLcQ0AIAMgGP0TIhX9DAAAAAAAAAAAAAAAAAAAAAD9DBAAAAAQAAAAEAAAABAAAAAgBf1cAAAgCP0PIhb9Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgBP1cAABBBP1t/YkB/akB/VD9+gH95gEgGf0TIhf95AH9CwIAIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwABCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAEQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwIQIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwACCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAIQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwIgIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwADCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAMQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwIwIANBQGsgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwAECAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAQQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwIAIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwAFCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAUQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwJQIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwAGCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAYQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwJgIAMgFf0MAAAAAAAAAAAAAAAAAAAAAP0MEAAAABAAAAAQAAAAEAAAACAF/VwAHCAW/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIAT9XAAcQQT9bf2JAf2pAf1Q/foB/eYBIBf95AH9CwJwIANBgAFqIQEMAQsgAyEBA0AgASAYIAggAiAFai0AAHFBAEdBBHQgAiAEai0AAEEEdnKylCAZkjgCACABQQRqIQEgAkEBaiICQSBHDQALCyAMQQFqIQwgCEECdCEIIAlBAnQhCSAGQQJqIQYgBEEgaiEEIA1BwAFJIRQgDUFAayENIBQNAAsgC0EBaiILIBJHDQALCwssAQN/QYDzNCgCAEEAELIBIAAhAQNAIAEoAlghAyABEJkEIAMiASAARw0ACwswACAAIAAoAgAgACgCBCAAKAIIIAAoAgwQIDkDECAALQAYBEAgACgCDBAvIAAQLwsLgdEBBCN/Bn4CewR9QdC7AygCAAR/QQAFQdC7AwJ/IAAoAgAgACAALAALQQBIGyEOIwBB8AFrIgckACAHIA42AhQgB0H1HzYCEEEEQar5ACAHQRBqEDQgB0HoygI2AqABIAdB9MoCKAIAIgA2AjQgB0E0aiICIABBDGsoAgBqQfjKAigCADYCACAHQQA2AjggAiAHKAI0QQxrKAIAaiIAQQA2AhQgACAHQTxqIgk2AhggAEEANgIMIABCgqCAgOAANwIEIAAgCUU2AhAgAEEgakEAQSj8CwAgAEEcahDlARogAEKAgICAcDcCSCAHQejKAjYCoAEgB0HUygI2AjQCfyMAQRBrIgAkACAJQeTHAjYCACAJQQRqEOUBGiAJQgA3AhggCUIANwIQIAlCADcCCCAJQQA2AiggCUIANwIgIAlB1MgCNgIAIAlBNGpBAEEv/AsAIAAgCSgCBCICNgIMIAJBBGpBAf4eAgAaIAAoAgxBpIo1EEwQrgMhGSAAKAIMIgJBBGpBf/4eAgBFBEAgAiACKAIAKAIIEQEACyAZBEAgAEEIaiICIAkoAgQiBDYCACAEQQRqQQH+HgIAGiAJIAIQ+AM2AkQgAigCACICQQRqQX/+HgIARQRAIAIgAigCACgCCBEBAAsgCSAJKAJEIgIgAigCACgCHBEAADoAYgsgCUEAQYAgIAkoAgAoAgwRAwAaIABBEGokAAJAAkAgCSgCQA0AQQAhAiMAQRBrIgMkAAJAAkBB2ChBmigsAAAQxwFFBEAjA0EcNgIcDAELQQIhAEGaKEErEMcBRQRAQZooLQAAQfIARyEACyAAQYABciAAQZooQfgAEMcBGyIAQYCAIHIgAEGaKEHlABDHARsiACAAQcAAckGaKC0AACIAQfIARhsiBEGABHIgBCAAQfcARhsiBEGACHIgBCAAQeEARhshACADQrYDNwMAQZx/IA4gAEGAgAJyIAMQJSIAQYFgTwRAIwNBACAAazYCHEF/IQALIABBAEgNASMAQSBrIgQkAAJ/AkACQEHYKEGaKCwAABDHAUUEQCMDQRw2AhwMAQtBmAkQOyICDQELQQAMAQsgAkEAQZABEKABQZooQSsQxwFFBEAgAkEIQQRBmigtAABB8gBGGzYCAAsCQEGaKC0AAEHhAEcEQCACKAIAIQUMAQsgAEEDQQAQDyIFQYAIcUUEQCAEIAVBgAhyrDcDECAAQQQgBEEQahAPGgsgAiACKAIAQYABciIFNgIACyACQX82AlAgAkGACDYCMCACIAA2AjwgAiACQZgBajYCLAJAIAVBCHENACAEIARBGGqtNwMAIABBk6gBIAQQJA0AIAJBCjYCUAsgAkG6ATYCKCACQbsBNgIkIAJBvAE2AiAgAkG9ATYCDEGZ8TQtAABFBEAgAkF/NgJMCyACEPwBIgUoAgA2AjggBSgCACIBBEAgASACNgI0CyAFIAI2AgBB1PI0ENMBIAILIQIgBEEgaiQAIAINASAAEA0aC0EAIQILIANBEGokACAJIAI2AkAgAkUNACAJQQw2AlgMAQtBAAwBCyAJC0UEQCAHKAI0QQxrKAIAIAdBNGpqIgAgACgCEEEEchC7AgsgB0GgAWohIAJAIAcoAjRBDGsoAgAgB0E0amotABBBBXEEQCAHIA42AgQgB0H1HzYCAEECQZH5ACAHEDRBACEDDAELIAdBzQA2AjAgB0HOADYCLCAHQc8ANgIoIAcgB0E0ajYCJCMAQRBrIg8kAEGYAhAyIgP9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAP9DAAAAACYygAA3AUAAIABAAD9CwIcIANCgYCAgBA3AxAgA0GYlQM2AsABIANBuAFqIgBCADcDACADQQA2AmAgA0IANwNYIANBrIufuQM2AkwgA/0MBgAAAAQAAABQAAAAAQAAAP0LAjwgA/0MBgAAAAQAAADAAQAAgAEAAP0LAiwgA0IANwOQASAD/QwAAAAAAAAAAAAAAAAAAAAA/QsDmAEgA0HIAWoiAkIANwMAIANBtAFqIhUgADYCACADQdQBaiIAQQA2AgAgA0HEAWoiFiACNgIAIANB0AFqIhcgADYCACAD/QwAAAAAUMQAAFHEAAC1xAAA/QsD2AEgA/0MtsQAALfEAAC4xAAAucQAAP0LA+gBIANCuomDgLCXMTcD+AEgA0EANgKQAiAD/QwAAAAAAAAAAAAAAAAAAAAA/QsDgAIgA0EBOgAYAn9BACEAIwBB0ARrIgEkACABQcglNgLgA0EEQeDlACABQeADahA0IAMQaSImNwMIIAcoAiQgAUHwA2pBBCAHKAIoEQMAGgJAAkACQCABKALwA0Hs2p27BkcEQCABQcglNgLQA0ECQcT4ACABQdADahA0DAELIAcoAiQgA0EgakEEIAcoAigRAwAaIAcoAiQgA0EkakEEIAcoAigRAwAaIAcoAiQgA0EoakEEIAcoAigRAwAaIAcoAiQgA0EsakEEIAcoAigRAwAaIAcoAiQgA0EwakEEIAcoAigRAwAaIAcoAiQgA0E0akEEIAcoAigRAwAaIAcoAiQgA0E4akEEIAcoAigRAwAaIAcoAiQgA0E8akEEIAcoAigRAwAaIAcoAiQgA0FAa0EEIAcoAigRAwAaIAcoAiQgA0HEAGpBBCAHKAIoEQMAGiAHKAIkIANByABqQQQgBygCKBEDABogAUEAOgDwAyABQQA6APsDAkACQAJAAkACQAJAIAMoAjBBBGsOHQAFAQUFBQUFAgUFBQUFBQUFBQUFAwUFBQUFBQUEBQsgA0EBNgIcDAQLIANBAjYCHAwDCyADQQM2AhwMAgsgA0EENgIcDAELIANBBTYCHCADKAIgQZqVA0cNACABQQM6APsDIAFBADoA8wMgAUH0OS8AADsB8AMgAUH2OS0AADoA8gMLIAMgAygCSCIAIABB6AdtIgVB6AdsayICNgJIIAMCfyMAQRBrIgAkAAJAIAJBD08NAEGP/wEgAnZBAXFFDQAgAkECdEGApAFqKAIAIRogAEEQaiQAIBoMAQtBvMMCKAIAEDAaIABBkiw2AgggAEG6EDYCBCAAQeQmNgIAQbjDAigCAEHL5AAgABAxEAAACyIENgIQAkAgBEETRgRAIAEgAygCSDYC9AEgAUHIJTYC8AFBAkHg9wAgAUHwAWoQNAwBCyABIAMoAiA2AsQDIAFByCU2AsADQQRBxO0AIAFBwANqEDQgASADKAIkNgK0AyABQcglNgKwA0EEQeTsACABQbADahA0IAEgAygCKDYCpAMgAUHIJTYCoANBBEGE7AAgAUGgA2oQNCABIAMoAiw2ApQDIAFByCU2ApADQQRBzOwAIAFBkANqEDQgASADKAIwNgKEAyABQcglNgKAA0EEQezrACABQYADahA0IAEgAygCNDYC9AIgAUHIJTYC8AJBBEGU7QAgAUHwAmoQNCABIAMoAjg2AuQCIAFByCU2AuACQQRBtOwAIAFB4AJqEDQgASADKAI8NgLUAiABQcglNgLQAkEEQfzsACABQdACahA0IAEgAygCQDYCxAIgAUHIJTYCwAJBBEGc7AAgAUHAAmoQNCABIAMoAkQ2ArQCIAFByCU2ArACQQRB3O0AIAFBsAJqEDQgASADKAJINgKkAiABQcglNgKgAkEEQYzuACABQaACahA0IAEgBTYClAIgAUHIJTYCkAJBBEH07QAgAUGQAmoQNAJAAkBBvNc0KAIAIgBFDQAgAygCHCECA0AgACgCECIFIAJKBEAgACgCACIADQEMAgsgAiAFTA0CIAAoAgQiAA0ACwtB8SMQtwEACyAAQRRqIgUoAgAhCCAALAAfIQAgAUHIJTYCgAIgASACNgKEAiABIAEoAvADIAFB8ANqIAEsAPsDQQBIGzYCjAIgASAIIAUgAEEASBs2AogCQQRBwPQAIAFBgAJqEDQLIAEsAPsDQQBIBEAgASgC8AMQLwtBACEAIARBE0YNACAHKAIkIANB0ABqQQQgBygCKBEDABogBygCJCADQdQAakEEIAcoAigRAwAaAkAgAygCVCADKAJQbCICIAMoAlwiBCADQdgAaiIIKAIAIgBrQQJ1IgVLBEAgCCACIAVrEG4gAygCWCEAIAMoAlwhBAwBCyACIAVPDQAgAyAAIAJBAnRqIgQ2AlwLIAcoAiQgACAEIABrIAcoAigRAwAaIAFBADYCyAQgBygCJCABQcgEakEEIAcoAigRAwAaIAFBADYC+AMgAUIANwPwAyABQYABEDIiADYCtAQgASAANgKwBCABIABBgAFqNgK4BCABKALIBCIAQQBKBEAgA0HUAWohBSADQcQBaiEKA0AgBygCJCABQeQDakEEIAcoAigRAwAaAkAgASgC5AMiAARAAkAgASgCtAQiAiABKAKwBCIEayIIIABJBEAgAUGwBGogACAIaxDxAiABKAKwBCEEIAEoArQEIQIMAQsgACAITw0AIAEgACAEaiICNgK0BAsgBygCJCAEIAIgBGsgBygCKBEDABogAUHwA2ogASgCsAQiACABKAK0BCAAaxDLAQwBCwJ/IAEsAPsDQQBIBEAgAUEANgL0AyABKALwAwwBCyABQQA6APsDIAFB8ANqC0EAOgAACyABIAFB8ANqIgA2ApQEIAFBoARqIAogACABQZQEahDiBCABKAKgBCAGNgIcAkACQCAFIgIiACgCACIERQ0AA0AgBCIAKAIQIgIgBkoEQCAAIgIoAgAiBA0BDAILIAIgBk4NAiAAKAIEIgQNAAsgAEEEaiECC0EgEDIiBCAGNgIQIAQgADYCCCAEQgA3AgAgBEIANwIUIARBADYCHCACIAQ2AgAgBCEAIAMoAtABKAIAIggEQCADIAg2AtABIAIoAgAhAAsgAygC1AEgABCkASADIAMoAtgBQQFqNgLYAQsCQCAEQRRqIgAgAUHwA2pGDQAgAS0A+wMiCMAhAiAELAAfQQBOBEAgAkEATgRAIAAgASkD8AM3AgAgACABKAL4AzYCCAwCCyAAIAEoAvADIAEoAvQDEN4BDAELIAAgASgC8AMgAUHwA2ogAkEASCIAGyABKAL0AyAIIAAbEN8BCyAGQQFqIgYgASgCyAQiAEgNAAsLIAMgAygCICICNgLAASACQZiVA0oEQCADIAMoAtwBQQFqNgLcASADIAMoAuABQQFqNgLgASADIAP9AALkASACQZiVA2siBP0R/a4B/QsC5AEgAyADKAL0ASAEajYC9AEgAyADKAL4ASAEajYC+AEgAyADKAL8ASAEajYC/AELAkAgACACTg0AIAFByCU2AuABIAEgAiAAazYC5AFBBEGT4wAgAUHgAWoQNCABKALIBCIGIAMoAiBODQAgA0HUAWohBSADQcQBaiELIAFB8ANqQQRyIQgDQAJAIAMoAvwBIgIgBkgEQCABQZQEaiIAIAYgAmsQOSABIABBsykQOCIAKAIINgKoBCABIAApAgA3A6AEIABCADcCACAAQQA2AgggAUGgBGpBuyoQNyIAKAIAIQIgASAAKAIENgLkAyABIAAoAAc2AOcDIABCADcCACAALQALIQQgAEEANgIIIAEsAPsDQQBIBEAgASgC8AMQLwsgASACNgLwAyAIIAEoAOcDNgADIAggASgC5AM2AgAgASAEOgD7AyABLACrBEEASARAIAEoAqAEEC8LIAEsAJ8EQQBODQEgASgClAQQLwwBCyADKALcASAGRgRAAn8gASwA+wNBAEgEQCABQQc2AvQDIAEoAvADDAELIAFBBzoA+wMgAUHwA2oLIgBBADoAByAAQdQpKAAANgADIABB0SkoAAA2AAAMAQsgAygC4AEiACAGRgRAAn8gASwA+wNBAEgEQCABQQc2AvQDIAEoAvADDAELIAFBBzoA+wMgAUHwA2oLIgBBADoAByAAQcQpKAAANgADIABBwSkoAAA2AAAMAQsgAygC5AEgBkYEQCABQfADakHhKUENEMsBDAELIAMoAugBIAZGBEAgAUHwA2pB7ylBDhDLAQwBCyADKALsASAGRgRAAn8gASwA+wNBAEgEQCABQQg2AvQDIAEoAvADDAELIAFBCDoA+wMgAUHwA2oLIgBBADoACCAAQtu+zfrEqdOv3QA3AAAMAQsgAygC8AEgBkYEQAJ/IAEsAPsDQQBIBEAgAUEINgL0AyABKALwAwwBCyABQQg6APsDIAFB8ANqCyIAQQA6AAggAELbvsGS1cjVr90ANwAADAELIAMoAvQBIAZGBEACfyABLAD7A0EASARAIAFBCDYC9AMgASgC8AMMAQsgAUEIOgD7AyABQfADagsiAEEAOgAIIABC2765+rSK1K/dADcAAAwBCyADKAL4ASAGRgRAAn8gASwA+wNBAEgEQCABQQc2AvQDIAEoAvADDAELIAFBBzoA+wMgAUHwA2oLIgBBADoAByAAQcwpKAAANgADIABBySkoAAA2AAAMAQsgAiAGRgRAAn8gASwA+wNBAEgEQCABQQc2AvQDIAEoAvADDAELIAFBBzoA+wMgAUHwA2oLIgBBADoAByAAQdwpKAAANgADIABB2SkoAAA2AAAMAQsCQCAAIAZODQAgBiAAIAMoAsABIgJqIAJBmJUDSmtBtZQDa0oNACAGIABBf3NqIQxBACEEIwBBEGsiCiQAAkACQEHE1zQoAgAiAkHI1zRHBEADQCACKAIcIAxGDQICQCACKAIEIgAEQANAIAAiAigCACIADQAMAgsACwNAIAIgAigCCCICKAIARw0ACwsgAkHI1zRHDQALCyAKIAw2AgQgCkHBEzYCAEECQaDrACAKEDQMAQsgAigCECACQRBqIAIsABtBAEgbIQQLIApBEGokACABIAFBlARqIAQQpgEiAkG5KRA4IgAoAgg2AqgEIAEgACkCADcDoAQgAEIANwIAIABBADYCCCABQaAEakG7KhA3IgAoAgAhBCABIAAoAgQ2AuQDIAEgACgABzYA5wMgAEIANwIAIAAtAAshCiAAQQA2AgggASwA+wNBAEgEQCABKALwAxAvCyABIAQ2AvADIAggASgA5wM2AAMgCCABKALkAzYCACABIAo6APsDIAEsAKsEQQBIBEAgASgCoAQQLwsgASwAnwRBAE4NASACKAIAEC8MAQsgAUGUBGoiACAGEDkgASAAQaQpEDgiACgCCDYCqAQgASAAKQIANwOgBCAAQgA3AgAgAEEANgIIIAFBoARqQbsqEDciACgCACECIAEgACgCBDYC5AMgASAAKAAHNgDnAyAAQgA3AgAgAC0ACyEEIABBADYCCCABLAD7A0EASARAIAEoAvADEC8LIAEgAjYC8AMgCCABKADnAzYAAyAIIAEoAuQDNgIAIAEgBDoA+wMgASwAqwRBAEgEQCABKAKgBBAvCyABLACfBEEATg0AIAEoApQEEC8LIAEgAUHwA2oiADYClAQgAUGgBGogCyAAIAFBlARqEOIEIAEoAqAEIAY2AhwCQAJAIAUiAiIAKAIAIgRFDQADQCAEIgAoAhAiAiAGSgRAIAAiAigCACIEDQEMAgsgAiAGTg0CIAAoAgQiBA0ACyAAQQRqIQILQSAQMiIEIAY2AhAgBCAANgIIIARCADcCACAEQgA3AhQgBEEANgIcIAIgBDYCACAEIQAgAygC0AEoAgAiCgRAIAMgCjYC0AEgAigCACEACyADKALUASAAEKQBIAMgAygC2AFBAWo2AtgBCwJAIARBFGoiACABQfADakYNACABLQD7AyIKwCECIAQsAB9BAE4EQCACQQBOBEAgACABKQPwAzcCACAAIAEoAvgDNgIIDAILIAAgASgC8AMgASgC9AMQ3gEMAQsgACABKALwAyABQfADaiACQQBIIgAbIAEoAvQDIAogABsQ3wELIAZBAWoiBiADKAIgSA0ACwsgAygCwAEhACABQcglNgLQASABIAAgAEGYlQNKa0G1lANrNgLUAUEEQaztACABQdABahA0IAEoArAEIgAEQCABIAA2ArQEIAAQLwsgASwA+wNBAEgEQCABKALwAxAvCyADKAIQIQggAygCMCEAIAMoAkAhAiABQQE6APgDIAEgASgC+AM2AsgBIAEgAEEPbCACQRhsakEZakG8Amw2AvADQQAhACABQQA2AvQDIAEgASkC8AM3A8ABIAMgAUHAAWoQvgEiAjYCqAEgAkUEQCABQcglNgIAQQJBo+oAIAEQNAwBCyADNAJEISUgAygCQCEKIAMoAjghEiADNAI0IScgAygCKCETIAM0AiQhKCADNAIgISkCQCADKAIwIgwgAygClAEiACADKAKQASIEa0E8bSICSwRAQQAhBiAMIAJrIgQgAygCmAEiCyAAa0E8bU0EQCADIAQEfyAAQQAgBEE8bEE8ayICIAJBPHBrQTxqIgL8CwAgACACagUgAAs2ApQBDAILAkAgACADKAKQASIFa0E8bSIQIARqIgJBxYiRIkkEQEHEiJEiIAsgBWtBPG0iC0EBdCIRIAIgAiARSRsgC0GixIgRTxsiCwRAIAtBxYiRIk8NAiALQTxsEDIhBgsgEEE8bCAGaiICQQAgBEE8bEE8ayIEIARBPHBrQTxqIgT8CwAgAiAEaiEEIAAgBUcEQANAIAJBPGsiAiAAQTxrIgD9AAIA/QsCACACIAAoAjg2AjggAiAAKQIwNwIwIAIgAP0AAiD9CwIgIAIgAP0AAhD9CwIQIAAgBUcNAAsgAygCkAEhAAsgAyAGIAtBPGxqNgKYASADIAQ2ApQBIAMgAjYCkAEgAARAIAAQLwsMAwsQNgALEEcACyACIAxNDQAgAyAEIAxBPGxqNgKUAQsgCEEARyEQAkAgAygCoAEiACADKAKcASIEa0HgAG0iAiAKSQRAQQAhBSAKIAJrIgQgAygCpAEiCyAAa0HgAG1NBEAgAyAEBH8gAEEAIARB4ABsQeAAayICIAJB4ABwa0HgAGoiAvwLACAAIAJqBSAACzYCoAEMAgsCQCAAIAMoApwBIgZrQeAAbSIRIARqIgJBq9WqFUkEQEGq1aoVIAsgBmtB4ABtIgtBAXQiGCACIAIgGEkbIAtB1arVCk8bIgsEQCALQavVqhVPDQIgC0HgAGwQMiEFCyARQeAAbCAFaiICQQAgBEHgAGxB4ABrIgQgBEHgAHBrQeAAaiIE/AsAIAIgBGohBCAAIAZHBEADQCACQeAAayICIABB4ABrIgBB4AD8CgAAIAAgBkcNAAsgAygCnAEhAAsgAyAFIAtB4ABsajYCpAEgAyAENgKgASADIAI2ApwBIAAEQCAAEC8LDAMLEDYACxBHAAsgAiAKTQ0AIAMgBCAKQeAAbGo2AqABCyADIAMoAqgBQQAgE6wiJCAoEFU2AmQgAyADKAKoASAQQgMgJSAkELwBNgJoIAMgAygCqAFBAEIBICQQVTYCbCADIAMoAqgBIBBCAyAkICQQvAE2AnAgAyADKAKoAUEAQgEgJBBVNgJ0IAMgAygCqAFBACAkEEg2AnggAyADKAKoAUEAICQQSDYCfCADKAJkIQIgAUEgEDIiADYC8AMgAUKcgICAgISAgIB/NwL0AyAAQZAeKAAANgAYIABBiB4pAAA3ABAgAEH4Hf0AAAD9CwAAIABBADoAHCABIAFB8ANqIgA2AqAEIAFBsARqIANBtAFqIgUgACABQaAEahA8IAEoArAEIAI2AhwgASwA+wNBAEgEQCABKALwAxAvCyADKAJoIQIgAUEgEDIiADYC8AMgAUKUgICAgISAgIB/NwL0AyAAQdEOKAAANgAQIABBwQ79AAAA/QsAACAAQQA6ABQgASABQfADaiIANgKgBCABQbAEaiAFIAAgAUGgBGoQPCABKAKwBCACNgIcIAEsAPsDQQBIBEAgASgC8AMQLwsgAygCbCECIAFBIBAyIgA2AvADIAFCkoCAgICEgICAfzcC9AMgAEGNEy8AADsAECAAQf0S/QAAAP0LAAAgAEEAOgASIAEgAUHwA2oiADYCoAQgAUGwBGogBSAAIAFBoARqEDwgASgCsAQgAjYCHCABLAD7A0EASARAIAEoAvADEC8LIAMoAnAhAiABQSAQMiIANgLwAyABQpSAgICAhICAgH83AvQDIABBrg4oAAA2ABAgAEGeDv0AAAD9CwAAIABBADoAFCABIAFB8ANqIgA2AqAEIAFBsARqIAUgACABQaAEahA8IAEoArAEIAI2AhwgASwA+wNBAEgEQCABKALwAxAvCyADKAJ0IQIgAUEgEDIiADYC8AMgAUKSgICAgISAgIB/NwL0AyAAQe4SLwAAOwAQIABB3hL9AAAA/QsAACAAQQA6ABIgASABQfADaiIANgKgBCABQbAEaiAFIAAgAUGgBGoQPCABKAKwBCACNgIcIAEsAPsDQQBIBEAgASgC8AMQLwsgAygCeCECIAFBIBAyIgA2AvADIAFCloCAgICEgICAfzcC9AMgAEGDDSkAADcADiAAQfUM/QAAAP0LAAAgAEEAOgAWIAEgAUHwA2oiADYCoAQgAUGwBGogBSAAIAFBoARqEDwgASgCsAQgAjYCHCABLAD7A0EASARAIAEoAvADEC8LIAMoAnwhAiABQSAQMiIANgLwAyABQpSAgICAhICAgH83AvQDIABB8hEoAAA2ABAgAEHiEf0AAAD9CwAAIABBADoAFCABIAFB8ANqIgA2AqAEIAFBsARqIAUgACABQaAEahA8IAEoArAEIAI2AhwgASwA+wNBAEgEQCABKALwAxAvCyAMQQBKBEAgE0ECdKwhJUEAIQQDQCADKAKQASAEQTxsaiIAIAMoAqgBQQAgJBBINgIkIAAgAygCqAFBACAkEEg2AiggACADKAKoASAIICQgJRBVNgIsIAAgAygCqAFBACAlEEg2AjAgACADKAKoASAIICUgJBBVNgI0IAAgAygCqAFBACAkEEg2AjggACADKAKoAUEAICQQSDYCACAAIAMoAqgBQQAgJBBINgIEIAAgAygCqAEgCCAkICQQVTYCECAAIAMoAqgBQQAgJBBINgIUIAAgAygCqAEgCCAkICQQVTYCGCAAIAMoAqgBIAggJCAkEFU2AhwgACADKAKoAUEAICQQSDYCICAAIAMoAqgBIAggJCAkEFU2AgggACADKAKoAUEAICQQSDYCDCAAKAIkIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQYwNEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIoIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQfcREDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIsIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQdYOEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIwIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQZATEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAI0IQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQbMOEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAI4IQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQfESEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIAIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQbENEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIEIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQZgSEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIQIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQZIMEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIUIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQa0REDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIYIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQbwMEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIcIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQYsOEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIgIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQc0SEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIIIQYgAUGgBGoiAiAEEDkgASACQc7CABA4IgIoAgg2ArgEIAEgAikCADcDsAQgAkIANwIAIAJBADYCCCABIAFBsARqQeQMEDciAigCCDYC+AMgASACKQIANwPwAyACQgA3AgAgAkEANgIIIAEgAUHwA2oiAjYC5AMgAUGUBGogBSACIAFB5ANqEDwgASgClAQgBjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAAKAIMIQIgAUGgBGoiACAEEDkgASAAQc7CABA4IgAoAgg2ArgEIAEgACkCADcDsAQgAEIANwIAIABBADYCCCABIAFBsARqQdMREDciACgCCDYC+AMgASAAKQIANwPwAyAAQgA3AgAgAEEANgIIIAEgAUHwA2oiADYC5AMgAUGUBGogBSAAIAFB5ANqEDwgASgClAQgAjYCHCABLAD7A0EASARAIAEoAvADEC8LIAEsALsEQQBIBEAgASgCsAQQLwsgASwAqwRBAEgEQCABKAKgBBAvCyAEQQFqIgQgDEcNAAsLIAMgAygCqAFBACASrCIkICcQVTYCgAEgAyADKAKoASAIICQgKRBVNgKEASADIAMoAqgBQQAgJBBINgKIASADIAMoAqgBQQAgJBBINgKMASADKAKAASECIAFBIBAyIgA2AvADIAFCnICAgICEgICAfzcC9AMgAEGtHigAADYAGCAAQaUeKQAANwAQIABBlR79AAAA/QsAACAAQQA6ABwgASABQfADaiIANgKgBCABQbAEaiAFIAAgAUGgBGoQPCABKAKwBCACNgIcIAEsAPsDQQBIBEAgASgC8AMQLwsgAygChAEhAiABQSAQMiIANgLwAyABQp6AgICAhICAgH83AvQDIABB6Q0pAAA3ABYgAEHjDSkAADcAECAAQdMN/QAAAP0LAAAgAEEAOgAeIAEgAUHwA2oiADYCoAQgAUGwBGogBSAAIAFBoARqEDwgASgCsAQgAjYCHCABLAD7A0EASARAIAEoAvADEC8LIAMoAogBIQIgAUEgEDIiADYC8AMgAUKRgICAgISAgIB/NwL0AyAAQdENLQAAOgAQIABBwQ39AAAA/QsAACAAQQA6ABEgASABQfADaiIANgKgBCABQbAEaiAFIAAgAUGgBGoQPCABKAKwBCACNgIcIAEsAPsDQQBIBEAgASgC8AMQLwsgAygCjAEhAiABQRAQMiIANgLwAyABQo+AgICAgoCAgH83AvQDIABBrRIpAAA3AAcgAEGmEikAADcAACAAQQA6AA8gASABQfADaiIANgKgBCABQbAEaiAFIAAgAUGgBGoQPCABKAKwBCACNgIcIAEsAPsDQQBIBEAgASgC8AMQLwsgCkEASgRAIBJBAnSsISVBACEEA0AgAygCnAEgBEHgAGxqIgAgAygCqAFBACAkEEg2AkggACADKAKoAUEAICQQSDYCTCAAIAMoAqgBIAggJCAlEFU2AlAgACADKAKoAUEAICUQSDYCVCAAIAMoAqgBIAggJSAkEFU2AlggACADKAKoAUEAICQQSDYCXCAAIAMoAqgBQQAgJBBINgIAIAAgAygCqAFBACAkEEg2AgQgACADKAKoASAIICQgJBBVNgIQIAAgAygCqAFBACAkEEg2AhQgACADKAKoASAIICQgJBBVNgIYIAAgAygCqAEgCCAkICQQVTYCHCAAIAMoAqgBQQAgJBBINgIgIAAgAygCqAEgCCAkICQQVTYCCCAAIAMoAqgBQQAgJBBINgIMIAAgAygCqAFBACAkEEg2AiQgACADKAKoAUEAICQQSDYCKCAAIAMoAqgBIAggJCAkEFU2AjQgACADKAKoAUEAICQQSDYCOCAAIAMoAqgBIAggJCAkEFU2AjwgACADKAKoASAIICQgJBBVNgJAIAAgAygCqAFBACAkEEg2AkQgACADKAKoASAIICQgJBBVNgIsIAAgAygCqAFBACAkEEg2AjAgACgCSCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGMDRA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCTCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakH3ERA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCUCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHWDhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCVCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGQExA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCWCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGzDhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCXCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHxEhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCACEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGxDRA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCBCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGYEhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCECEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGSDBA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCFCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGtERA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCGCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakG8DBA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCHCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGLDhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCICEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHNEhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCCCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHkDBA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCDCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHTERA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCJCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGbDRA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCKCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGEEhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCNCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakH5CxA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCOCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGWERA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCPCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakGlDBA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgAEFAaygCACEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHyDRA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCRCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakG2EhA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCLCEGIAFBoARqIgIgBBA5IAEgAkHewgAQOCICKAIINgK4BCABIAIpAgA3A7AEIAJCADcCACACQQA2AgggASABQbAEakHNDBA3IgIoAgg2AvgDIAEgAikCADcD8AMgAkIANwIAIAJBADYCCCABIAFB8ANqIgI2AuQDIAFBlARqIAUgAiABQeQDahA8IAEoApQEIAY2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgACgCMCECIAFBoARqIgAgBBA5IAEgAEHewgAQOCIAKAIINgK4BCABIAApAgA3A7AEIABCADcCACAAQQA2AgggASABQbAEakG+ERA3IgAoAgg2AvgDIAEgACkCADcD8AMgAEIANwIAIABBADYCCCABIAFB8ANqIgA2AuQDIAFBlARqIAUgACABQeQDahA8IAEoApQEIAI2AhwgASwA+wNBAEgEQCABKALwAxAvCyABLAC7BEEASARAIAEoArAEEC8LIAEsAKsEQQBIBEAgASgCoAQQLwsgBEEBaiIEIApHDQALCyADEOUEIgA2AoQCQQAhBCADKAK0ASICIANBuAFqIgZHBEADQCACKAIcEFsgBGohHAJAIAIoAgQiBARAA0AgBCIAKAIAIgQNAAwCCwALA0AgAigCCCIAKAIAIAJHIRsgACECIBsNAAsLIBxBvAJqIQQgBiAAIgJHDQALIAMoAoQCIQALIAMgACAAKAIIEQAAIgAgBCAAKAIAEQQANgKsASADKAKEAiIABH8gACAAKAIAEQAABUGwMAshACABIAS4RAAAAACAhC5BozkDuAEgASAANgK0ASABQcglNgKwAUEEQa3vACABQbABahA0IAMoAqwBEMkBIQwgBiADKAK0ASICRwRAA0AgDCACKAIcEJABAkAgAigCBCIEBEADQCAEIgAoAgAiBA0ADAILAAsDQCACKAIIIgAoAgAgAkchHSAAIQIgHQ0ACwsgBiAAIgJHDQALCyADQQA2ArABIAFBADYCuAQgAUIANwKwBEEAIQoDQAJAIAcoAiQgAUGQBGpBBCAHKAIoEQMAGiAHKAIkIAFBjARqQQQgBygCKBEDABogBygCJCABQYgEakEEIAcoAigRAwAaAkACQAJAIAcoAiQgBygCLBEAAARAIAFByCU2AjAgASAKuEQAAAAAgIQuQaM5AzhBBEGp8QAgAUEwahA0IAMoArABIgANASABQcglNgIQQQNBxOYAIAFBEGoQNEEAIQIMBAtBACEAIAFB0KUB/QAEAP0LBPADQQEhBCABKAKQBEEASg0BDAILQQAhAiAAIAMoArwBIgRGDQIgASAANgIoIAEgBDYCJCABQcglNgIgQQJB1+oAIAFBIGoQNEEBIQIMAgsDQCAHKAIkIAFB8ANqIABBAnRqIgJBBCAHKAIoEQMAGiACKAIAIARsIQQgAEEBaiIAIAEoApAESA0ACwtBACECIAFBADYCqAQgAUIANwOgBCABQQA2ApwEIAFCADcClARBACEAIAEoAowEIggEQCAIQQBIDQQgASAIEDIiADYClAQgASAAIAhqIgI2ApwEIABBACAI/AsAIAEgAjYCmAQLIAcoAiQgACACIABrIgggBygCKBEDABogAUGgBGoiAiAAIAgQywECfyAFIAIQ8AIgBkYEQCABQcglNgJAQQAhBCABIAEoAqAEIAIgASwAqwRBAEgbNgJEQQJB9+cAIAFBQGsQNEEBDAELIAEoAqAEIAFBoARqIAEsAKsEQQBIGyIIEGgiAEHw////B08NBQJAIABBCk0EQCABIAA6AO8DIAFB5ANqIQIMAQsgAEEPckEBaiILEDIhAiABIAtBgICAgHhyNgLsAyABIAI2AuQDIAEgADYC6AMLIAIgCCAA/AoAACAAIAJqQQA6AAAgASABQeQDaiIANgLEBCABQcgEaiAFIAAgAUHEBGoQPCABKALIBCgCHCEAIAEsAO8DQQBIBEAgASgC5AMQLwsgBKwgACkDKCAAKQMgIAApAxggACkDEH5+flIEQCABQcglNgKgAUEAIQQgASABKAKgBCABQaAEaiABLACrBEEASBs2AqQBQQJByecAIAFBoAFqEDQgACkDICEkIAApAxghJSABIAApAxA+ApABIAEgJT4ClAEgASAkPgKYASABQcglNgKAASABIAEoAvADNgKEASABIAEpAvQDNwOIAUECQfzuACABQYABahA0QQEMAQsgACkDGCEkIAEoAvQDIQgCQAJAIAApAxAiJSABKALwAyILrFIEQCABKAL4AyECDAELIAEoAvgDIQIgJCAIrFINACAAKQMgIAKsUQ0BCyABIAApAyA+AnAgASALNgJ0IAEgCDYCeCABIAI2AnwgAUHIJTYCYEEAIQQgASABKAKgBCABQaAEaiABLACrBEEASBs2AmQgASAlPgJoIAEgJD4CbEECQaTuACABQeAAahA0QQEMAQsgBCABKAKIBEEkbEGYmwFqKAIAbCICIAAoAgBBJGxBlJsBaigCAG4gABBbRwRAIAEoAqAEIQggASwAqwQhCyAAEFshACABIAI2AlwgASAANgJYIAFByCU2AlBBACEEIAEgCCABQaAEaiALQQBIGzYCVEECQZviACABQdAAahA0QQEMAQsCQCADKAKEAigCAEE8RgRAIAcoAighAiAHKAIkIAAoAtABIAAQWyACEQMAGgwBCwJAIAAQWyICIAFBsARqIgQoAgQgBCgCACILayIISwRAIAQgAiAIaxDxAgwBCyACIAhJBEAgBCACIAtqNgIECwsgBygCJCABKAKwBCICIAEoArQEIAJrIAcoAigRAwAaIAAgASgCsARBACAAEFsQlwELIAAQWyEeQQEhBCADIAMoArABQQFqNgKwASAeIApqIQpBAAshAiABKAKUBCIABEAgABAvCyABLACrBEEASARAIAEoAqAEEC8LIAQNAQsLIAEoArAEIgAEQCABIAA2ArQEIAAQLwtBACEAIAINACAMEJEBIAMQaSAmfTcDAEEBIQALIAFB0ARqJAAgAAwCCxA2AAsQTQALIR8gBygCJCAHKAIwEQEAIB9FBEAgD0HTHzYCAEECQfPlACAPEDQgAywAkwJBAEgEQCADKAKIAhAvCyAXIAMoAtQBEI8CIBYgAygCyAEQjgIgFSADKAK4ARCNAiADKAKcASIABEAgAyAANgKgASAAEC8LIAMoApABIgAEQCADIAA2ApQBIAAQLwsgAygCWCIABEAgAyAANgJcIAAQLwsgAxAvQQAhAwsgD0EQaiQAIANFDQAgA0GIAmogDhCIAwsgB0HwygIoAgAiADYCNCAAQQxrKAIAIAdBNGpqQfzKAigCADYCACAJEPIBGiAgEMACIAdB8AFqJAAgAyIABEAgAAJ/QQAhAyMAQdAUayIBJAAgASAAIgk2AswUQfDXNC0AAEUEQP0MAAAAAAEAAAACAAAAAwAAACEqA0AgA0ECdCIAQYDYNGogKv3+Af0MGC1EVPshGUAYLURU+yEZQP3yAf0MAAAAAAAAeUAAAAAAAAB5QP3zASIr/SEAtiIsEHr9EyAr/SEBtiItEHr9IAEgKiAq/Q0ICQoLDA0ODwABAgMAAQID/f4B/QwYLURU+yEZQBgtRFT7IRlA/fIB/QwAAAAAAAB5QAAAAAAAAHlA/fMBIiv9IQC2Ii4Qev0gAiAr/SEBtiIvEHr9IAP9CwQAIABBwOQ0aiAsEGr9EyAtEGr9IAEgLhBq/SACIC8Qav0gA/0LBAAgKv0MBAAAAAQAAAAEAAAABAAAAP2uASEqIANBBGoiA0GQA0cNAAtB8Nc0QQE6AAALQeCoARAyIgX9DAAAAAAAAAAAAAAAAAAAAAD9CwJ0IAVCADcD0AEgBUIANwKEASAFQgA3A4gCIAVBADYC2AEgBUEANgKwASAFQgA3A6gBIAX9DAAAAAAAAAAAAAAAAAAAAAD9CwOQAiAF/QwAAAAAAAAAAAAAAAAAAAAA/QsCtAIgBf0MAAAAAAAAAAAAAAAAAAAAAP0LAsQCIAX9DAAAAAAAAAAAAAAAAAAAAAD9CwLUAiAFQQBB5AD8CwBB8SohAiAFQeQCaiIAQfEqNgIAQQEhAwNAIAAgA0ECdGogAkEediACc0Hlkp7gBmwgA2oiAjYCACAAIANBAWoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACAAIANBAmoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACADQQNqIgRB8ARHBEAgACAEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBBGohAwwBCwsgBUHwFmpCADcDACAFQeAWav0MAAAAAAAAAAAAAAAAAAAAAP0LAwAgBUGMF2r9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVBnBdq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQawXav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUGkFmr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAQfEqIQIgBUG8F2oiAEHxKjYCAEEBIQMDQCAAIANBAnRqIAJBHnYgAnNB5ZKe4AZsIANqIgI2AgAgACADQQFqIgRBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgACADQQJqIgRBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgA0EDaiIEQfAERwRAIAAgBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACADQQRqIQMMAQsLIAVByCtqQgA3AwAgBUG4K2r9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAVB5Ctq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQfQrav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUGELGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVB/Cpq/QwAAAAAAAAAAAAAAAAAAAAA/QsCAEHxKiECIAVBlCxqIgBB8So2AgBBASEDA0AgACADQQJ0aiACQR52IAJzQeWSnuAGbCADaiICNgIAIAAgA0EBaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIAAgA0ECaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBA2oiBEHwBEcEQCAAIARBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgA0EEaiEDDAELCyAFQaDAAGpCADcDACAFQZDAAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAVBvMAAav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUHMwABq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQdzAAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVB1D9q/QwAAAAAAAAAAAAAAAAAAAAA/QsCAEHxKiECIAVB7MAAaiIAQfEqNgIAQQEhAwNAIAAgA0ECdGogAkEediACc0Hlkp7gBmwgA2oiAjYCACAAIANBAWoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACAAIANBAmoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACADQQNqIgRB8ARHBEAgACAEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBBGohAwwBCwsgBUH41ABqQgA3AwAgBUHo1ABq/QwAAAAAAAAAAAAAAAAAAAAA/QsDACAFQZTVAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVBpNUAav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUG01QBq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQazUAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAQfEqIQIgBUHE1QBqIgBB8So2AgBBASEDA0AgACADQQJ0aiACQR52IAJzQeWSnuAGbCADaiICNgIAIAAgA0EBaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIAAgA0ECaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBA2oiBEHwBEcEQCAAIARBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgA0EEaiEDDAELCyAFQdDpAGpCADcDACAFQcDpAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwMAIAVB7OkAav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUH86QBq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQYzqAGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVBhOkAav0MAAAAAAAAAAAAAAAAAAAAAP0LAgBB8SohAiAFQZzqAGoiAEHxKjYCAEEBIQMDQCAAIANBAnRqIAJBHnYgAnNB5ZKe4AZsIANqIgI2AgAgACADQQFqIgRBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgACADQQJqIgRBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgA0EDaiIEQfAERwRAIAAgBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACADQQRqIQMMAQsLIAVBqP4AakIANwMAIAVBmP4Aav0MAAAAAAAAAAAAAAAAAAAAAP0LAwAgBUHE/gBq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQdT+AGr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVB5P4Aav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUHc/QBq/QwAAAAAAAAAAAAAAAAAAAAA/QsCAEHxKiECIAVB9P4AaiIAQfEqNgIAQQEhAwNAIAAgA0ECdGogAkEediACc0Hlkp7gBmwgA2oiAjYCACAAIANBAWoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACAAIANBAmoiBEECdGogAkEediACc0Hlkp7gBmwgBGoiAjYCACADQQNqIgRB8ARHBEAgACAEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBBGohAwwBCwsgBUGAkwFqQgA3AwAgBUHwkgFq/QwAAAAAAAAAAAAAAAAAAAAA/QsDACAFQZyTAWr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVBrJMBav0MAAAAAAAAAAAAAAAAAAAAAP0LAgAgBUG8kwFq/QwAAAAAAAAAAAAAAAAAAAAA/QsCACAFQbSSAWr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAQfEqIQIgBUHMkwFqIgBB8So2AgBBASEDA0AgACADQQJ0aiACQR52IAJzQeWSnuAGbCADaiICNgIAIAAgA0EBaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIAAgA0ECaiIEQQJ0aiACQR52IAJzQeWSnuAGbCAEaiICNgIAIANBA2oiBEHwBEcEQCAAIARBAnRqIAJBHnYgAnNB5ZKe4AZsIARqIgI2AgAgA0EEaiEDDAELCyAF/QwAAAAAAAAAAAAAAAAAAAAA/QsDqKcBIAX9DAAAAAAAAAAAAAAAAAAAAAD9CwK8pwEgBf0MAAAAAAAAAAAAAAAAAAAAAP0LA9CnASAFQYynAWpCADcCACAFQZSnAWr9DAAAAAAAAAAAAAAAAAAAAAD9CwIAIAVB5KcBakEAQeQA/AsAIAX9DAAAAAAAAAAAAAAAAAAAAAD9CwLMqAEgASAFNgLIFCAFEOUENgKQpwECQAJAIAkoAjggCUFAayIAKAIAIAVBzABqIAkoAoQCIAkoAhQgCSgCNEEDbBDjBEUEQCABQcAfNgJwQQJBnukAIAFB8ABqEDQgBRDyAhAvQQAhAwwBCyAFKAJkEFshAiAFKAJoEFshBCABQcAfNgJgIAEgAiAEarhEAAAAAICELkGjOQNoQQRBu/AAIAFB4ABqEDQgCSgCOCAAKAIAIAVB9ABqIAkoAoQCIAkoAhQgCSgCJBDjBEUEQCABQcAfNgJQQQJB6OgAIAFB0ABqEDQgBRDyAhAvQQAhAwwBCyAFKAKMARBbIQAgBSgCkAEQWyECIAFBwB82AkAgASAAIAJquEQAAAAAgIQuQaM5A0hBBEHN7wAgAUFAaxA0AkAgCSgCNCIAIAkoAsABbCICIAVBjKgBaigCACAFKAKEqAEiB2tBAnVNDQAgAkGAgICABE8NAiAFQYioAWooAgAhAyACQQJ0IgIQMiIGIAJqIQogBiADIAdrIghBfHFqIgQhAgJAIAMgB0YNAAJAIAhBBGsiAkEsSQRAIAQhAgwBCyADIAhBfHEgBmprQRBJBEAgBCECDAELIARBEGshCCADQRBrIQwgAyACQQJ2QQFqIg5B/P///wdxIgZBAnQiAmshAyAEIAJrIQIDQCAIIA1BAnQiD2sgDCAPa/0AAgD9CwIAIA1BBGoiDSAGRw0ACyAGIA5GDQELA0AgAkEEayICIANBBGsiAyoCADgCACADIAdHDQALCyAFIAo2AoyoASAFIAQ2AoioASAFIAI2AoSoASAHRQ0AIAcQLyAJKAI0IQALIABBAnQiAhA7IQQgAhA7IQcgAhA7IQYgAkEEahA7IQIgAEEASgRAQQAhAwNAIAIgA0ECdGpBIBA7NgIAIANBAWoiAyAARw0ACwsgAiAAQQJ0akEANgIAIAUgABA7NgLIASAFIAI2AsQBIAUgBjYCwAEgBSAHNgK8ASAFIAQ2ArgBIAVBADYCtAECQCAFKALYASAFKALQASICa0EwbSAATw0AIABB1qrVKk8NAiAFKALUASEDIABBMGwiABAyIgQgAGohByAEIAMgAmtBMG1BMGxqIgQhACACIANHBEADQCAAQTBrIgAgA0EwayID/QADAP0LAwAgACAD/QADIP0LAyAgACAD/QADEP0LAxAgAiADRw0ACwsgBSAHNgLYASAFIAQ2AtQBIAUgADYC0AEgAkUNACACEC8LAkAgCSgCwAEiDSAFKAK8AiAFKAK0AiIHa0ECdU0NACANQYCAgIAETw0CIAUoArgCIQMgDUECdCIAEDIiBCAAaiEIIAQgAyAHayIGQXxxaiICIQACQCADIAdGDQACQCAGQQRrIgBBLEkEQCACIQAMAQsgAyAGQXxxIARqa0EQSQRAIAIhAAwBCyACQRBrIQogA0EQayEMIAMgAEECdkEBaiIOQfz///8HcSIGQQJ0IgBrIQMgAiAAayEAQQAhBANAIAogBEECdCIPayAMIA9r/QACAP0LAgAgBEEEaiIEIAZHDQALIAYgDkYNAQsDQCAAQQRrIgAgA0EEayIDKgIAOAIAIAMgB0cNAAsLIAUgCDYCvAIgBSACNgK4AiAFIAA2ArQCIAdFDQAgBxAvIAkoAsABIQ0LAkAgBSgCyAIgBSgCwAIiB2tBAnUgDU8NACANQYCAgIAETw0CIAUoAsQCIQMgDUECdCIAEDIiBCAAaiEIIAQgAyAHayIGQXxxaiICIQACQCADIAdGDQACQCAGQQRrIgBBLEkEQCACIQAMAQsgAyAGQXxxIARqa0EQSQRAIAIhAAwBCyACQRBrIQogA0EQayEMIAMgAEECdkEBaiIOQfz///8HcSIGQQJ0IgBrIQMgAiAAayEAQQAhBANAIAogBEECdCIPayAMIA9r/QACAP0LAgAgBEEEaiIEIAZHDQALIAYgDkYNAQsDQCAAQQRrIgAgA0EEayIDKgIAOAIAIAMgB0cNAAsLIAUgCDYCyAIgBSACNgLEAiAFIAA2AsACIAdFDQAgBxAvIAkoAsABIQ0LAkAgBSgC1AIgBSgCzAIiB2tBAnUgDU8NACANQYCAgIAETw0CIAUoAtACIQMgDUECdCIAEDIiBCAAaiEIIAQgAyAHayIGQXxxaiICIQACQCADIAdGDQACQCAGQQRrIgBBLEkEQCACIQAMAQsgAyAGQXxxIARqa0EQSQRAIAIhAAwBCyACQRBrIQ0gA0EQayEKIAMgAEECdkEBaiIMQfz///8HcSIGQQJ0IgBrIQMgAiAAayEAQQAhBANAIA0gBEECdCIOayAKIA5r/QACAP0LAgAgBEEEaiIEIAZHDQALIAYgDEYNAQsDQCAAQQRrIgAgA0EEayIDKgIAOAIAIAMgB0cNAAsLIAUgCDYC1AIgBSACNgLQAiAFIAA2AswCIAdFDQAgBxAvCwJAIAkoAiAiACAFKALgAiAFKALYAiICa0EEdU0NACAAQYCAgIABTw0CIAUoAtwCIQMgAEEEdCIAEDIiBCAAaiEHIAQgAyACa2oiBCEAIAIgA0cEQANAIABBEGsiACADQRBrIgMpAwA3AwAgACADKAIINgIIIAIgA0cNAAsLIAUgBzYC4AIgBSAENgLcAiAFIAA2AtgCIAJFDQAgAhAvC0EAIQAgAUEANgKAAUEBIQMDQCABQYABaiICIANBAnRqIABBHnYgAHNB5ZKe4AZsIANqIgA2AgAgA0EBaiIEQQJ0IAJqIABBHnYgAHNB5ZKe4AZsIARqIgA2AgAgA0ECaiIEQQJ0IAJqIABBHnYgAHNB5ZKe4AZsIARqIgA2AgAgA0EDaiIEQfAERwRAIARBAnQgAmogAEEediAAc0Hlkp7gBmwgBGoiADYCACADQQRqIQMMAQsLIAFBADYCwBQgBUHkAmogAUGAAWoiAEHEE/wKAAAgCSgChAIhAiABIAFByBRqNgKIASABQeS1ATYCgAEgASAANgKQASABIAFBzBRqNgKEASAFQZSnAWogAiAAEJACAkACfyAAIAEoApABIgNGBEAgACEDIAEoAoABQRBqDAELIANFDQEgAygCAEEUagshACADIAAoAgARAQALIAEoAsgUIgBBmKcBaigCACECIABBnKcBaigCACEEIAAoApSnASgCACgClBAhACABQcAfNgIwIAEgACAEIAJrarhEAAAAAICELkGjOQM4QQRBgfEAIAFBMGoQNCABKALMFCgChAIhAiABKALIFCEhIAEgAUHIFGo2AogBIAFB1LcBNgKAASABIAFBgAFqIgA2ApABIAEgAUHMFGo2AoQBICFBqKcBaiACIAAQkAICQAJ/IAAgASgCkAEiA0YEQCAAIQMgASgCgAFBEGoMAQsgA0UNASADKAIAQRRqCyEAIAMgACgCABEBAAsgASgCyBQiAEGspwFqKAIAIQIgAEGwpwFqKAIAIQQgACgCqKcBKAIAKAKUECEAIAFBwB82AiAgASAAIAQgAmtquEQAAAAAgIQuQaM5AyhBBEHr7wAgAUEgahA0IAEoAswUKAKEAiECIAEoAsgUISIgASABQcgUajYCiAEgAUGMuQE2AoABIAEgAUGAAWoiADYCkAEgASABQcwUajYChAEgIkG8pwFqIAIgABCQAgJAAn8gACABKAKQASIDRgRAIAAhAyABKAKAAUEQagwBCyADRQ0BIAMoAgBBFGoLIQAgAyAAKAIAEQEACyABKALIFCIAQcCnAWooAgAhAiAAQcSnAWooAgAhBCAAKAK8pwEoAgAoApQQIQAgAUHAHzYCECABIAAgBCACa2q4RAAAAACAhC5BozkDGEEEQdnwACABQRBqEDQgASgCzBQoAoQCIQIgASgCyBQhIyABIAFByBRqNgKIASABQcS6ATYCgAEgASABQYABaiIANgKQASABIAFBzBRqNgKEASAjQdCnAWogAiAAEJACAkACfyAAIAEoApABIgNGBEAgACEDIAEoAoABQRBqDAELIANFDQEgAygCAEEUagshACADIAAoAgARAQALIAEoAsgUIgBB1KcBaigCACECIABB2KcBaigCACEEIAAoAtCnASgCACgClBAhACABQcAfNgIAIAEgACAEIAJrarhEAAAAAICELkGjOQMIQQRBk/AAIAEQNCABKALIFCIDKAKUpwEiAgRAIAEoAswUKAKEAiEAIAIoAgAoApQQIQIgAygClKcBEJEBIANBpKcBaiAAIAAoAggRAAAiACACIAAoAgARBAAiADYCACADIAAQyQE2ApSnASABKALIFCEDCyADKAKopwEiAARAIAEoAswUKAKEAiECIAAoAgAoApQQIQQgABCRASADQbinAWogAiACKAIIEQAAIgAgBCAAKAIAEQQAIgA2AgAgAyAAEMkBNgKopwEgASgCyBQhAwsgAygCvKcBIgAEQCABKALMFCgChAIhAiAAKAIAKAKUECEEIAAQkQEgA0HMpwFqIAIgAigCCBEAACIAIAQgACgCABEEACIANgIAIAMgABDJATYCvKcBIAEoAsgUIQMLIAMoAtCnASIARQ0AIAEoAswUKAKEAiECIAAoAgAoApQQIQQgABCRASADQeCnAWogAiACKAIIEQAAIgAgBCAAKAIAEQQAIgA2AgAgAyAAEMkBNgLQpwEgASgCyBQhAwsgAUHQFGokACADDAELEDYACyIANgKAAiAJIAANARogCRDgBAtBAAsiADYCACAAQQBHCwvqBQIEfwJ8IwBBIGsiBCQAIAQgADYCACAEQQA6ABggBEIANwMQIAQgAjYCDCAEIAE2AgggBCMDNgIEAnwgAwRAIwBBEGsiAyQAIAMgBDYCDCADQQA2AgggA0HIATYCBCADQQRqIQUjAEGAAWsiACQAIwBB4ABrIgEkAAJAQYTzNCgCAEECRgRAIwBBEGtBADYCDAwBCyMAQRBrIgIkAANAAkACQAJAAkACQEGE8zRBAEEB/kgCAA4EAAIBAwULIAJBBGoiBkGE8zQ2AgQgAkHKATYCBCACIwMiBygCRDYCDCAHIAY2AkQQmAQjAyACKAIMNgJEQYTzNEEC/kECAEEDRw0AQYTzNBD6AQsgAkEQaiQADAILQYTzNEEBQQP+SAIAGgtBhPM0QQBBAxChAQwBCwsLIAFBAEHQAPwLACABQccBNgJcIAEgBTYCWCABQQA2AlQgAUEANgJQIABBIGoiAiABKAJcNgIAIAIgASgCWDYCBCACIAEoAlQ2AgggAiABKAJQNgIMIAJBEGogAUHQAPwKAAAgAUHgAGokACAAQcIBNgIYIABBwwE2AhQgACACNgIcIAAgAjYCECAAIAApAhQ3AwhB3PI0QdDxNCAAQQhqEPsBBH8gAEEwaiIBEFYaIAAoAixFBEAgAEHIAGohAgNAIAIgARCUBCAAKAIsRQ0ACwsgARBSGiAAKAIsQQFGBUEACyEBIABBIGoQmgQgAEGAAWokACADQRBqJAAgBCsDEEQAAAAAAAAAACABGwwBC0EgEDsiAyAEKQMYNwMYIAMgBCkDEDcDECADIAQpAwg3AwggAyAEKQMANwMAIANBAToAGCADIAFBA3QiABA7IgE2AgwgASACIAAQexojAEEgayIAJAAgACADNgIcIAAgAzYCECAAQQA2AhggAEHIATYCFCAAIAApAhQ3AwhB3PI0QdDxNCAAQQhqEPsBGiAAQSBqJABEAAAAAAAAAAALIQkgBEEgaiQAIAkLoAEBA38gASgCCCABKAIAEQEAIwBBIGsiASQAAkAgACgCCEUEQCAAQRBqIgIQVhogAEEBNgIMIAAQmwQgAhBSGiAAQShqEJYEDAELIAAQmwQgACgCECECIAAoAgwhBCABIAA2AhwgASAANgIQIAFBwAE2AhggAUHBATYCFCABIAEpAhQ3AwggBCACIAFBCGoQ+wENACAAENIBCyABQSBqJAALFAAgACgCBCAAKAIYEQEAIAAQ0gELbQEBfwJAIwMoAkhBgPM0KAIAQQJ0aigCACIBRQRAIAAgADYCWCAAIAA2AlxBgPM0KAIAIAAQsgEMAQsgACABNgJYIAAgASgCXDYCXCABIAA2AlwgACgCXCAANgJYCyAAIAAoAgQgACgCABECAAsUACAAKAIEIAAoAhQRAQAgABDSAQsLACAAIAEgAhCmBAsaACAAQQH+FwIAIAAQzAIgAEEBQQD+SAIAGgsHACAAEJ4EC40NAgR9CX8gAkH/AUoEQCACQYACbSEOA0AgACALQZABbGoiCEEEaiEKIAhBEGohAiAILwEAQQJ0QZDWBGoqAgAhBSAILwECQQJ0QZDWBGoqAgCMIQZBACEMQQAhCANAAkAgCEEDTQRAIAggCmoiCS0AAEE/cSEHIAktAARBP3EhCQwBCyAIIApqIgctAABBAnZBMHEgBy0ABCINQQR2ciEJIAdBBGstAABBAnZBMHEgDUEPcXIhBwsgBSAHs5QhBCAJsyEDAn8gCEEBciIHQQNNBEAgCCAKai0ABUE/cSEJIAcgCmotAABBP3EMAQsgByAKai0AAEECdkEwcSAIIApqIgctAAUiDUEEdnIhCSAHQQNrLQAAQQJ2QTBxIA1BD3FyCyEHIAEgBCACLQAAQQ9xs5QgBiADlCIDkjgCACABIAQgAi0AAUEPcbOUIAOSOAIEIAEgBCACLQACQQ9xs5QgA5I4AgggASAEIAItAANBD3GzlCADkjgCDCABIAQgAi0ABEEPcbOUIAOSOAIQIAEgBCACLQAFQQ9xs5QgA5I4AhQgASAEIAItAAZBD3GzlCADkjgCGCABIAQgAi0AB0EPcbOUIAOSOAIcIAEgBCACLQAIQQ9xs5QgA5I4AiAgASAEIAItAAlBD3GzlCADkjgCJCABIAQgAi0ACkEPcbOUIAOSOAIoIAEgBCACLQALQQ9xs5QgA5I4AiwgASAEIAItAAxBD3GzlCADkjgCMCABIAQgAi0ADUEPcbOUIAOSOAI0IAEgBCACLQAOQQ9xs5QgA5I4AjggASAEIAItAA9BD3GzlCADkjgCPCABIAQgAi0AEEEPcbOUIAOSOAJAIAEgBCACLQARQQ9xs5QgA5I4AkQgASAEIAItABJBD3GzlCADkjgCSCABIAQgAi0AE0EPcbOUIAOSOAJMIAEgBCACLQAUQQ9xs5QgA5I4AlAgASAEIAItABVBD3GzlCADkjgCVCABIAQgAi0AFkEPcbOUIAOSOAJYIAEgBCACLQAXQQ9xs5QgA5I4AlwgASAEIAItABhBD3GzlCADkjgCYCABIAQgAi0AGUEPcbOUIAOSOAJkIAEgBCACLQAaQQ9xs5QgA5I4AmggASAEIAItABtBD3GzlCADkjgCbCABIAQgAi0AHEEPcbOUIAOSOAJwIAEgBCACLQAdQQ9xs5QgA5I4AnQgASAEIAItAB5BD3GzlCADkjgCeCABIAQgAi0AH0EPcbOUIAOSOAJ8IAEgBSAHs5QiBCACLQAAQQR2s5QgBiAJs5QiA5I4AoABIAEgBCACLQABQQR2s5QgA5I4AoQBIAEgBCACLQACQQR2s5QgA5I4AogBIAEgBCACLQADQQR2s5QgA5I4AowBIAEgBCACLQAEQQR2s5QgA5I4ApABIAEgBCACLQAFQQR2s5QgA5I4ApQBIAEgBCACLQAGQQR2s5QgA5I4ApgBIAEgBCACLQAHQQR2s5QgA5I4ApwBIAEgBCACLQAIQQR2s5QgA5I4AqABIAEgBCACLQAJQQR2s5QgA5I4AqQBIAEgBCACLQAKQQR2s5QgA5I4AqgBIAEgBCACLQALQQR2s5QgA5I4AqwBIAEgBCACLQAMQQR2s5QgA5I4ArABIAEgBCACLQANQQR2s5QgA5I4ArQBIAEgBCACLQAOQQR2s5QgA5I4ArgBIAEgBCACLQAPQQR2s5QgA5I4ArwBIAEgBCACLQAQQQR2s5QgA5I4AsABIAEgBCACLQARQQR2s5QgA5I4AsQBIAEgBCACLQASQQR2s5QgA5I4AsgBIAEgBCACLQATQQR2s5QgA5I4AswBIAEgBCACLQAUQQR2s5QgA5I4AtABIAEgBCACLQAVQQR2s5QgA5I4AtQBIAEgBCACLQAWQQR2s5QgA5I4AtgBIAEgBCACLQAXQQR2s5QgA5I4AtwBIAEgBCACLQAYQQR2s5QgA5I4AuABIAEgBCACLQAZQQR2s5QgA5I4AuQBIAEgBCACLQAaQQR2s5QgA5I4AugBIAEgBCACLQAbQQR2s5QgA5I4AuwBIAEgBCACLQAcQQR2s5QgA5I4AvABIAEgBCACLQAdQQR2s5QgA5I4AvQBIAEgBCACLQAeQQR2s5QgA5I4AvgBIAEgBCACLQAfQQR2s5QgA5I4AvwBIAhBAmohCCACQSBqIQIgAUGAAmohASAMQcABSSEPIAxBQGshDCAPDQALIAtBAWoiCyAORw0ACwsLSgEBfyMBIgAoAgxFBEAgAEEBNgIMQdzyNBBWGkHc8jQjAxCcBCEAQdzyNBBSGgJAIABFDQAgACgCIA0AIAAQzAILIwFBADYCDAsLCwAgACABIAIQrgQLCQAgACgCPBANC+EBAQR/IwBBIGsiBCQAIAQgATYCECAEIAIgACgCMCIDQQBHazYCFCAAKAIsIQUgBCADNgIcIAQgBTYCGAJAAkAgACAAKAI8IARBEGpBAiAEQQxqECMiAwR/IwMgAzYCHEF/BUEACwR/QSAFIAQoAgwiA0EASg0BQSBBECADGwsgACgCAHI2AgAMAQsgBCgCFCIFIAMiBk8NACAAIAAoAiwiAzYCBCAAIAMgBiAFa2o2AgggACgCMARAIAAgA0EBajYCBCABIAJqQQFrIAMtAAA6AAALIAIhBgsgBEEgaiQAIAYL8gIBCH8jAEEgayIDJAAgAyAAKAIcIgQ2AhAgACgCFCEFIAMgAjYCHCADIAE2AhggAyAFIARrIgE2AhQgASACaiEFQQIhBwJ/AkACQAJAIAAoAjwgA0EQaiIBQQIgA0EMahAOIgQEfyMDIAQ2AhxBfwVBAAsEQCABIQQMAQsDQCAFIAMoAgwiBkYNAiAGQQBIBEAgASEEDAQLIAEgBiABKAIEIghLIglBA3RqIgQgBiAIQQAgCRtrIgggBCgCAGo2AgAgAUEMQQQgCRtqIgEgASgCACAIazYCACAFIAZrIQUgACgCPCAEIgEgByAJayIHIANBDGoQDiIGBH8jAyAGNgIcQX8FQQALRQ0ACwsgBUF/Rw0BCyAAIAAoAiwiATYCHCAAIAE2AhQgACABIAAoAjBqNgIQIAIMAQsgAEEANgIcIABCADcDECAAIAAoAgBBIHI2AgBBACAHQQJGDQAaIAIgBCgCBGsLIQogA0EgaiQAIAoLyAsECn8TewF9AX4gAkGAAk4EQCMAQRBrIQQgAkGAAm0hCQNAIAAgCEHuAGxqIgIvAWwhCyAEIAIpAWAiITcDACAEIAIoAWgiBTYCCCAEIAQoAgQiA0GPnrz4AHEgBUECdEGw4MCBA3FyNgIEIAQgIaciB0GPnrz4AHEgBUEEdEGw4MCBA3FyNgIAIAQgB0EEdkGPnrz4AHEgBUGw4MCBA3FyNgIIIAQgA0EEdkGPnrz4AHEgBUECdkGw4MCBA3FyNgIMIAJBIGohAyALQQJ0QZDWBGoqAgAhICAC/VwAHCEQIAL9XAAYIREgAv1cABQhEiAC/VwAECETIAL9XAAMIRQgAv1cAAghFSAC/VwABCEWIAL9XAAAIRdBASEFQQAhAkEBIQcDQP0MAAAAAAAAAAAAAAAAAAAAACAC/RwBIQ8gA/1cABz9iQH9qQEhGCAD/VwAGP2JAf2pASEZIAP9XAAU/YkB/akBIRogA/1cABD9iQH9qQEhGyAD/VwADP2JAf2pASEcIAP9XAAI/YkB/akBIR0gA/1cAAT9iQH9qQEhHiAD/VwAAP2JAf2pASEfQQAhBgNAIAEgICAEIA/9GwFqIgosAABBIGuylP0TIg39DPz////8/////P////z////9DAAAAAAAAAAAAAAAAAAAAAAgFCAH/Q8iDv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAcIA/9GwAiAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/VD9+gH95gH9CwIwIAEgDf0M/P////z////8/////P////0MAAAAAAAAAAAAAAAAAAAAACAVIA79Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgHSAC/a0B/QwDAAAAAwAAAAMAAAADAAAA/U79UP36Af3mAf0LAiAgASAN/Qz8/////P////z////8/////QwAAAAAAAAAAAAAAAAAAAAAIBYgDv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAeIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv1Q/foB/eYB/QsCECABIA39DPz////8/////P////z////9DAAAAAAAAAAAAAAAAAAAAAAgFyAO/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIB8gAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/VD9+gH95gH9CwIAIAEgICAKLAABQSBrspT9EyIN/Qz8/////P////z////8/////QwAAAAAAAAAAAAAAAAAAAAAIBAgDv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAYIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv1Q/foB/eYB/QsCcCABIA39DPz////8/////P////z////9DAAAAAAAAAAAAAAAAAAAAAAgESAO/U79DAAAAAAAAAAAAAAAAAAAAAD9I/2HAf2nAf1SIBkgAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/VD9+gH95gH9CwJgIAEgDf0M/P////z////8/////P////0MAAAAAAAAAAAAAAAAAAAAACASIA79Tv0MAAAAAAAAAAAAAAAAAAAAAP0j/YcB/acB/VIgGiAC/a0B/QwDAAAAAwAAAAMAAAADAAAA/U79UP36Af3mAf0LAlAgASAN/Qz8/////P////z////8/////QwAAAAAAAAAAAAAAAAAAAAAIBMgDv1O/QwAAAAAAAAAAAAAAAAAAAAA/SP9hwH9pwH9UiAbIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv1Q/foB/eYB/QsCQCAHQQF0IQcgD/0MAgAAAAIAAAACAAAAAgAAAP2uASEPIAFBgAFqIQEgBkEBaiIGQQRHDQALIANBIGohAyAFIQwgD/0bASECQQAhBSAMDQALIAhBAWoiCCAJRw0ACwsLVAECfyAAKAI8IQQjAEEQayIAJAAgBCABpyABQiCIpyACQf8BcSAAQQhqEBYiAgR/IwMgAjYCHEF/BUEACyECIAApAwghASAAQRBqJABCfyABIAIbCwwAQYzxNCMD/hcCAAsHACMDQRxqCwsAIAAgASACELMECyQBAX9BgPE0KAIAIgAEQANAIAAoAgARCQAgACgCBCIADQALCwskAQJ/IAAoAgQiABBoQQFqIgEQOyICBH8gAiAAIAEQewVBAAsLhgEBA38jASEAIwMiAigCdCIBBEAgAkEANgJ0IAEkASAB/QwAAAAAAAAAAAAAAAAAAAAA/QsAACMBIgBBBGokCiAAJAsgAQ8LIwJBASAAGwRAQQEkAkEQEDshAAsgACQBIAD9DAAAAAAAAAAAAAAAAAAAAAD9CwAAIwEiAUEEaiQKIAEkCyAAC/IFAwl/C3sCfSACQf8BSgRAIAJBgAJtIQkDQCAAIAVB1ABsaiIGQRBqIQMgBi8BUEECdEGQ1gRqKgIAIRcgBi8BUkECdEGQ1gRqKgIAjCEYQQAhAkEBIQgDQP0MAAAAAAAAAAAAAAAAAAAAACAC/RwBIQ4gA/1cABz9iQH9qQEhDyAD/VwAGP2JAf2pASEQIAP9XAAU/YkB/akBIREgA/1cABD9iQH9qQEhEiAD/VwADP2JAf2pASETIAP9XAAI/YkB/akBIRQgA/1cAAT9iQH9qQEhFSAD/VwAAP2JAf2pASEWQQAhBANAIAEgFyAGIA79GwFqIgctAAAiCkEPcbKU/RMiDCATIA79GwAiAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/foB/eYBIBggCkEEdrKU/RMiDf3kAf0LAjAgASAMIBQgAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/foB/eYBIA395AH9CwIgIAEgDCAVIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv36Af3mASAN/eQB/QsCECABIAwgFiAC/a0B/QwDAAAAAwAAAAMAAAADAAAA/U79+gH95gEgDf3kAf0LAgAgASAXIActAAEiB0EPcbKU/RMiDCAPIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv36Af3mASAYIAdBBHaylP0TIg395AH9CwJwIAEgDCAQIAL9rQH9DAMAAAADAAAAAwAAAAMAAAD9Tv36Af3mASAN/eQB/QsCYCABIAwgESAC/a0B/QwDAAAAAwAAAAMAAAADAAAA/U79+gH95gEgDf3kAf0LAlAgASAMIBIgAv2tAf0MAwAAAAMAAAADAAAAAwAAAP1O/foB/eYBIA395AH9CwJAIA79DAIAAAACAAAAAgAAAAIAAAD9rgEhDiABQYABaiEBIARBAWoiBEEERw0ACyADQSBqIQMgCCELIA79GwEhAkEAIQggCw0ACyAFQQFqIgUgCUcNAAsLCwYAQey7AQsUACAAQQRqQQAgASgCBEHQuwFGGwvHAgEHfyAAKAIIKAIAIgEgACgCBCgCACgCNCIDNgK0AQJAIANBAEwNACADQQFHBEAgA0F+cSEHA0AgAkECdCIFIAEoArwBaiACNgIAIAEoAsABIAVqQQE2AgAgASgCxAEgBWooAgBBADYCACABKALIASACakEAOgAAIAJBAXIiBUECdCIGIAEoArwBaiAFNgIAIAEoAsABIAZqQQE2AgAgASgCxAEgBmooAgBBADYCACABKALIASAFakEAOgAAIAJBAmohAiAEQQJqIgQgB0cNAAsLIANBAXFFDQAgAkECdCIEIAEoArwBaiACNgIAIAEoAsABIARqQQE2AgAgASgCxAEgBGooAgBBADYCACABKALIASACakEAOgAACyADIAEoAsgBakEBa0EBOgAAIAAoAgQoAgAgACgCCCgCACIAIABBtAFqENoECxUAIAFBxLoBNgIAIAEgACkCBDcCBAsdAQF/QQwQMiIBQcS6ATYCACABIAApAgQ3AgQgAQsGAEG0ugELFAAgAEEEakEAIAEoAgRBmLoBRhsLFQAgACgCBCgCACAAKAIIKAIAENsECxUAIAFBjLkBNgIAIAEgACkCBDcCBAsdAQF/QQwQMiIBQYy5ATYCACABIAApAgQ3AgQgAQsGAEH8uAELFAAgAEEEakEAIAEoAgRB4LgBRhsLFQAgACgCBCgCACAAKAIIKAIAENwECxUAIAFB1LcBNgIAIAEgACkCBDcCBAsdAQF/QQwQMiIBQdS3ATYCACABIAApAgQ3AgQgAQsGAEHEtwELFAAgAEEEakEAIAEoAgRBqLcBRhsLFwAgACgCBCgCACAAKAIIKAIAQQAQ3QQLrQICA38BeyACQR9KBEAgAkEgbSEFA0AgASAEQQd0aiIDIAAgBEEibGoiAi8BAEECdEGQ1gRq/QkCACIGIAL9XAAC/YcB/acB/foB/eYB/QsCACADIAYgAkEGav1cAAD9hwH9pwH9+gH95gH9CwIQIAMgBiACQQpq/VwAAP2HAf2nAf36Af3mAf0LAiAgAyAGIAJBDmr9XAAA/YcB/acB/foB/eYB/QsCMCADQUBrIAYgAkESav1cAAD9hwH9pwH9+gH95gH9CwIAIAMgBiACQRZq/VwAAP2HAf2nAf36Af3mAf0LAlAgAyAGIAJBGmr9XAAA/YcB/acB/foB/eYB/QsCYCADIAYgAkEeav1cAAD9hwH9pwH9+gH95gH9CwJwIARBAWoiBCAFRw0ACwsLpQEBBX8jAEEQayICJAAgASgCACIDQfD///8HSQRAAkAgA0EKTQRAIAIgAzoADyACQQRqIQQMAQsgA0EPckEBaiIFEDIhBCACIAVBgICAgHhyNgIMIAIgBDYCBCACIAM2AggLIAQgAUEEaiAD/AoAACADIARqQQA6AAAgAkEEaiAAEQAAIQYgAiwAD0EASARAIAIoAgQQLwsgAkEQaiQAIAYPCxBNAAsVACABQeS1ATYCACABIAApAgQ3AgQLHQEBf0EMEDIiAUHktQE2AgAgASAAKQIENwIEIAELSgEBfwJAIAEoAggiAiABKAIMRg0AIAItAABFDQAgAUGdeDYCACABIAJBAWo2AgggASAAKAIENgIoDwsgAUGfeDYCACABQQA2AigLnwYCBH8FeyACQR9KBEAgAkEgbSEGA0AgASAFQQd0aiIDQUBrIAAgBUEYbGoiBP1cAAj9iQH9qQEiB0EE/a0BIAQoAQQiAkEMdv0RIAJBDXb9HAEgAkEOdv0cAiACQQ92/RwDIgr9DBAAAAAQAAAAEAAAABAAAAD9Tv1Q/foBIAQvAQBBAnRBkNYEav0JAgAiCP3mASAELwECQQJ0QZDWBGr9CQIAIgn95AH9CwIAIAMgCSAIIAf9DA8AAAAPAAAADwAAAA8AAAD9TiACQQR0/REgAkEDdP0cASACQQJ0/RwCIAJBAXT9HAP9DBAAAAAQAAAAEAAAABAAAAD9Tv1Q/foB/eYB/eQB/QsCACADIAkgCCAEQQxq/VwAAP2JAf2pASIHQQT9rQEgAkEQdv0RIAJBEXb9HAEgAkESdv0cAiACQRN2/RwD/QwQAAAAEAAAABAAAAAQAAAA/U79UP36Af3mAf3kAf0LAlAgAyAJIAggB/0MDwAAAA8AAAAPAAAADwAAAP1OIAJBBHb9ESACQQV2/RwBIAJBBnb9HAIgAkEHdv0cAyIHQQT9qwH9DBAAAAAQAAAAEAAAABAAAAD9Tv1Q/foB/eYB/eQB/QsCECADIAkgCCAEQRBq/VwAAP2JAf2pASILQQT9rQEgAkEUdv0RIAJBFXb9HAEgAkEWdv0cAiACQRd2/RwD/QwQAAAAEAAAABAAAAAQAAAA/U79UP36Af3mAf3kAf0LAmAgAyAJIAggC/0MDwAAAA8AAAAPAAAADwAAAP1OIAf9DBAAAAAQAAAAEAAAABAAAAD9Tv1Q/foB/eYB/eQB/QsCICADIAkgCCAEQRRq/VwAAP2JAf2pASIHQQT9rQEgAkEYdv0RIAJBGXb9HAEgAkEadv0cAiACQRt2/RwD/QwQAAAAEAAAABAAAAAQAAAA/U79UP36Af3mAf3kAf0LAnAgAyAJIAggB/0MDwAAAA8AAAAPAAAADwAAAP1OIApBBP2rAf0MEAAAABAAAAAQAAAAEAAAAP1O/VD9+gH95gH95AH9CwIwIAVBAWoiBSAGRw0ACwsLJgEBfyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAALHAAgAkGeeDYCACACIABBCEEEIAEbaigCADYCKAsKACABQaB4NgIAC0YBAX8gAEHAswE2AgAgACgCCCIBBEAgASABKAIAKAIEEQEACyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAALFAAgAUGheDYCACABIAAoAgQ2AigLsgIBBn8gAkGeeDYCAAJAIAEgAC0AIEcEQCACIAAoAgQ2AiggAigCHCAAKAIUQQN0aiACKAIINgIEIAAoAhwiAyAAKAIYIgBGDQEgAEEBayEBIAIoAhAhBCACKAIMIQIgAyAAa0EDcSIHBEADQCAEIAFBDGxqIgZBADoACCAGIAI2AgQgBiACNgIAIAFBAWohASAFQQFqIgUgB0cNAAsLIAMgAEF/c2pBA0kNASADQQVrIQMDQCAEIAFBDGxqIgBBADoACCAAIAI2AgQgACACNgIAIAAgAjYCDCAAIAI2AhAgAEEAOgAUIABBADoAICAAIAI2AhwgACACNgIYIABBADoALCAAIAI2AiggACACNgIkIAEgA0YhCCABQQRqIQEgCEUNAAsMAQsgAiAAKAIINgIoCwv2BQEIfyABKAIcIgQgACgCFCIFQQN0aiECAkAgASgCAEGheEYEQCACIAIoAgBBAWoiAzYCACADIAAoAhAiBkkhAiAAKAIMIQcCQCADIAZPDQAgAyAHSQ0AIAQgBUEDdGooAgQgASgCCEcgAyAGSXEhAgsCQCACRQ0AIAMgB0kNACABQaB4NgIADwsgAUGeeDYCACACBEAgASAAKAIENgIoIAQgBUEDdGogASgCCDYCBCAAKAIcIgMgACgCGCIERg0CIARBAWshAiABKAIQIQUgASgCDCEBIAMgBGtBA3EiBwRAQQAhAANAIAUgAkEMbGoiBkEAOgAIIAYgATYCBCAGIAE2AgAgAkEBaiECIABBAWoiACAHRw0ACwsgAyAEQX9zakEDSQ0CIANBBWshAwNAIAUgAkEMbGoiAEEAOgAIIAAgATYCBCAAIAE2AgAgACABNgIMIAAgATYCECAAQQA6ABQgAEEAOgAgIAAgATYCHCAAIAE2AhggAEEAOgAsIAAgATYCKCAAIAE2AiQgAiADRiEIIAJBBGohAiAIRQ0ACwwCCyABIAAoAgg2AigPCyACQQA2AgACQCAAKAIQIgJFDQAgACgCDA0AIAFBoHg2AgAPCyABQZ54NgIAIAIEQCABIAAoAgQ2AiggBCAFQQN0aiABKAIINgIEIAAoAhwiAyAAKAIYIgRGDQEgBEEBayECIAEoAhAhBSABKAIMIQEgAyAEa0EDcSIHBEBBACEAA0AgBSACQQxsaiIGQQA6AAggBiABNgIEIAYgATYCACACQQFqIQIgAEEBaiIAIAdHDQALCyADIARBf3NqQQNJDQEgA0EFayEDA0AgBSACQQxsaiIAQQA6AAggACABNgIEIAAgATYCACAAIAE2AgwgACABNgIQIABBADoAFCAAQQA6ACAgACABNgIcIAAgATYCGCAAQQA6ACwgACABNgIoIAAgATYCJCACIANGIQkgAkEEaiECIAlFDQALDAELIAEgACgCCDYCKAsLjgcCBH8EeyACQR9KBEAgAkEgbSEGA0AgASAFQQd0aiIDQUBrIAAgBUEWbGoiBC8BAEECdEGQ1gRq/QkCACIHIAQoAQIiAkEMdv0RIAJBDXb9HAEgAkEOdv0cAiACQQ92/RwDIgn9DBAAAAAQAAAAEAAAABAAAAD9TiAE/VwABv2JAf2pASIIQQT9rQH9UP0M8P////D////w////8P////2uAf36Af3mAf0LAgAgAyAHIAJBBHT9ESACQQN0/RwBIAJBAnT9HAIgAkEBdP0cA/0MEAAAABAAAAAQAAAAEAAAAP1OIAj9DA8AAAAPAAAADwAAAA8AAAD9Tv1Q/Qzw////8P////D////w/////a4B/foB/eYB/QsCACADIAcgAkEQdv0RIAJBEXb9HAEgAkESdv0cAiACQRN2/RwD/QwQAAAAEAAAABAAAAAQAAAA/U4gBEEKav1cAAD9iQH9qQEiCEEE/a0B/VD9DPD////w////8P////D////9rgH9+gH95gH9CwJQIAMgByACQQR2/REgAkEFdv0cASACQQZ2/RwCIAJBB3b9HAMiCkEE/asB/QwQAAAAEAAAABAAAAAQAAAA/U4gCP0MDwAAAA8AAAAPAAAADwAAAP1O/VD9DPD////w////8P////D////9rgH9+gH95gH9CwIQIAMgByACQRR2/REgAkEVdv0cASACQRZ2/RwCIAJBF3b9HAP9DBAAAAAQAAAAEAAAABAAAAD9TiAEQQ5q/VwAAP2JAf2pASIIQQT9rQH9UP0M8P////D////w////8P////2uAf36Af3mAf0LAmAgAyAHIAr9DBAAAAAQAAAAEAAAABAAAAD9TiAI/QwPAAAADwAAAA8AAAAPAAAA/U79UP0M8P////D////w////8P////2uAf36Af3mAf0LAiAgAyAHIAJBGHb9ESACQRl2/RwBIAJBGnb9HAIgAkEbdv0cA/0MEAAAABAAAAAQAAAAEAAAAP1OIARBEmr9XAAA/YkB/akBIghBBP2tAf1Q/Qzw////8P////D////w/////a4B/foB/eYB/QsCcCADIAcgCUEE/asB/QwQAAAAEAAAABAAAAAQAAAA/U4gCP0MDwAAAA8AAAAPAAAADwAAAP1O/VD9DPD////w////8P////D////9rgH9+gH95gH9CwIwIAVBAWoiBSAGRw0ACwsLPAECfyABQZ54NgIAIAEoAgghAiABKAIQIAAoAghBDGxqQQxrIgNBAToACCADIAI2AgQgASAAKAIENgIoCy0AIAFBnng2AgAgASgCECAAKAIIQQxsakEMayABKAIINgIAIAEgACgCBDYCKAueAwIDewN/IAJBH0oEQCACQSBtIQgDQCABIAdBB3RqIgJBQGsgACAHQRRsaiIG/VwABP2JAf2pASIDQQT9rQH9+gEgBi8BAEECdEGQ1gRq/QkCACIE/eYBIAYvAQJBAnRBkNYEav0JAgAiBf3kAf0LAgAgAiAFIAQgA/0MDwAAAA8AAAAPAAAADwAAAP1O/foB/eYB/eQB/QsCACACIAUgBCAGQQhq/VwAAP2JAf2pASIDQQT9rQH9+gH95gH95AH9CwJQIAIgBSAEIAP9DA8AAAAPAAAADwAAAA8AAAD9Tv36Af3mAf3kAf0LAhAgAiAFIAQgBkEMav1cAAD9iQH9qQEiA0EE/a0B/foB/eYB/eQB/QsCYCACIAUgBCAD/QwPAAAADwAAAA8AAAAPAAAA/U79+gH95gH95AH9CwIgIAIgBSAEIAZBEGr9XAAA/YkB/akBIgNBBP2tAf36Af3mAf3kAf0LAnAgAiAFIAQgA/0MDwAAAA8AAAAPAAAADwAAAP1O/foB/eYB/eQB/QsCMCAHQQFqIgcgCEcNAAsLC6oWARN/IwBBIGsiBCQAAkACQAJAAkACQAJAIAEoAggiAiABKAIMIgNHBEBBASENIAAtAFtFDQUgAkEBaiADRg0FIAQgAi0AACIFOgAEIAQgAi0AASICOgAFIAAtAFkEQCAEIAAoAgwiAyAFwCADKAIAKAIUEQQAOgAEIAQgACgCDCIFIALAIAUoAgAoAhQRBAA6AAULIARBFGogAEEIaiIKIARBBGogBEEGaiIIENkCIAQoAhggBC0AHyICIALAIgJBAEgbIQUgAkEASARAIAQoAhQQLwsCQAJAIAUEQCAAKAI8IgMgACgCOCIFRg0CQQEhAkEBIAMgBWtBAXUiAyADQQFNGyEDIAQtAAQhByAELQAFIQkMAQtBACEFDAcLA0AgBSAGQQF0aiINLQAAIAdGBEAgCSANLQABRg0GCyAGQQFqIgYgA0cNAAsLAkAgAC0AWkUNACAAKAIsIAAoAjBGDQAgBEEUaiEHIwBBEGsiAiQAAkAgCCAEQQRqIgZrIgVB8P///wdJBEACQCAFQQpNBEAgAiAFOgAPIAJBBGohAwwBCyAFQQ9yQQFqIgkQMiEDIAIgCUGAgICAeHI2AgwgAiADNgIEIAIgBTYCCAsgBiAIRwR/IAMgBiAF/AoAACADIAVqBSADC0EAOgAAIAcgCigCCCIFIAIoAgQgAkEEaiACLQAPIgPAQQBIIgYbIgcgByACKAIIIAMgBhtqIAUoAgAoAhARBgAgAiwAD0EASARAIAIoAgQQLwsgAkEQaiQADAELEE0ACyAAKAIwIgIgACgCLCIHa0EYbSEDIAQtAB8iBcAhDQJ/IAIgB0YEQEEAIQZBAAwBC0EBIAMgA0EBTRshDCAEKAIYIAUgDUEASCIFGyECIAQoAhQgBEEUaiAFGyEOQQEhBkEAIQkDQAJAAkACQCAHIAlBGGxqIgUoAgQgBS0ACyILIAvAQQBIIg8bIgsgAiACIAtLGyIQBEAgDiAFKAIAIAUgDxsgEBBKIg8NAQsgAiALTw0BDAILIA9BAEgNAQsCQAJAIAIgBSgCECAFLQAXIgsgC8BBAEgiDxsiCyACIAtJGyIQBEAgBUEMaiIFKAIAIAUgDxsgDiAQEEoiBQ0BCyACIAtLDQIMAQsgBUEASA0BC0EFDAILIAlBAWoiCSADSSEGIAkgDEcNAAtBAAshAyANQQBIBEAgBCgCFBAvC0EBIQUgBg0FC0EAIQUCQCAAKAJEIAAoAkhGDQBBACEHIARBFGogCiAEQQRqIAgQ2AIgACgCSCICIAAoAkQiDmtBDG0hDSAELQAfwCEKAkAgAiAORwRAIAQoAhgiBiAKQf8BcSICIApBAEgiBRshC0EBIQhBASANIA1BAU0bIQwCQAJAIAUEQCAEKAIUIQkDQCAOIAdBDGxqIgIoAgQgAi0ACyIFIAXAQQBIIg8bIAtGBEBBASEFQQUhAyAGRQ0EIAkgAigCACACIA8bIAYQSkUNBAsgB0EBaiIHIA1JIQggByAMRw0ACwwBCwNAAkAgCyAOIAdBDGxqIgUoAgQgBS0ACyIDIAPAQQBIIgMbRw0AIApFBEBBASEFIAhBAXFFDQdBASECQQIhDQwOCyAFKAIAIAUgAxshBiAEQRRqIQkgAiEDA0AgCS0AACAGLQAARw0BQQEhBSAGQQFqIQYgCUEBaiEJIANBAWsiAw0AC0EFIQMMAwsgB0EBaiIHIA1JIQggByAMRw0ACwtBACEDQQAhBQsgCkEATg0BIAQoAhQQLyAIQQFxRQ0CDAcLIApBAE4NASAEKAIUEC8MAQsgCEEBcQ0FCyAELQAEIgLAIghBAE4EQAJAIAAoAgwoAggiBiACQQJ0aigCACIHIAAoAlAiA3FFBEAgCEHfAEcNASADQYAIcUUNAQsgBC0ABSIJwCIKQQBIDQBBASECIAYgCUECdGooAgAgA3ENBSADQYAIcUUNACAKQd8ARg0FCyAHIAAoAlQiBnENAyAIQd8ARw0CIAZBgAhxDQMMAgsgACgCVCEGDAELIAAtAFhBAEchAgwFC0EBIQIgBC0ABSIDwCIIQQBIDQEgACgCDCgCCCADQQJ0aigCACAGcQ0AIAhB3wBHDQEgBkGACHFFDQELIAUhAgtBAiENDAILQQIhDSAFIQIgAw0BCyAEIAEoAggtAAAiCDoAEyAALQBZBEAgBCAAKAIMIgIgCMAgAigCACgCFBEEACIIOgATCyAAKAIYIgYgACgCFCIDRwRAQQEhAkEBIAYgA2siBiAGQQFNGyEHQQAhBiAIQf8BcSEJA0AgCSADIAZqLQAARg0CIAZBAWoiBiAHRw0ACwsCQCAAKAJUIgJFBEAgACgCICAAKAIkRg0BCwJ/IAjAQQBOBEBBASAAKAIMKAIIIAhB/wFxQQJ0aigCACACcQ0BGgsgAkEKdiAIQf8BcUHfAEZxCyEUAkAgACgCICIGIAAoAiQiA0YNACAIQf8BcSECA0AgBi0AACACRg0BIAZBAWoiBiADRw0ACyADIQYLQQEhAiAUQX9zIAMgBkZxDQELAkAgACgCLCIHIAAoAjAiBkYEQCAFIQIMAQsCQCAALQBaBEAgBEEAOgAVIAQgCDoAFCAEQQE6AB8gBEEEaiAAKAIQIgIgBEEUaiIDIANBAXIgAigCACgCEBEGACAELAAfQQBIBEAgBCgCFBAvCyAAKAIsIQcgACgCMCEGDAELIARBADoABSAEIAg6AAQgBEEBOgAPCyAGIAdrQRhtIQogBC0ADyICwCEOAn8gBiAHRgRAQQAhBiAFDAELQQEgCiAKQQFNGyEPIAQoAgggAiAOQQBIIgMbIQIgBCgCBCAEQQRqIAMbIQtBASEGQQAhCQNAAkACQAJAIAcgCUEYbGoiAygCBCADLQALIgwgDMBBAEgiEBsiDCACIAIgDEsbIhEEQCALIAMoAgAgAyAQGyAREEoiEA0BCyACIAxPDQEMAgsgEEEASA0BCwJAAkAgAiADKAIQIAMtABciDCAMwEEASCIQGyIMIAIgDEkbIhEEQCADQQxqIgMoAgAgAyAQGyALIBEQSiIDDQELIAIgDEsNAgwBCyADQQBIDQELQQEMAgsgCUEBaiIJIApJIQYgCSAPRw0ACyAFCyECIA5BAEgEQCAEKAIEEC8LIAYNAQsgACgCRCAAKAJIRwRAIAIhBUEAIQogBEEUaiICIABBCGogBEETaiACENgCIAAoAkgiAiAAKAJEIgxrQQxtIQ4gBC0AHyIIwCELAkACQAJAAkAgAiAMRwRAQQEgDiAOQQFNGyEQIAQoAhgiDyAIIAtBAEgiERshEiAEKAIUIRNBASEKQQAhBwNAAkAgEiAMIAdBDGxqIgIoAgQgAi0ACyIDIAPAQQBIIgMbRw0AIAIoAgAgAiADGyEGAkAgEUUEQCAEQRRqIQkgCCEDIAsNAUEBIQIgCkUNCAwKC0EBIQIgD0UNBSATIAYgDxBKRQ0FDAELA0AgCS0AACAGLQAARw0BQQEhAiAGQQFqIQYgCUEBaiEJIANBAWsiAw0ACwwDCyAHQQFqIgcgDkkhCiAHIBBHDQALCyAFIQILIAtBAE4NAQsgBCgCFBAvIApFDQEMAwsgCg0CCyAELQATIQgLIAIhBSAAKAJQIQMgCMBBAE4EQEEBIQIgACgCDCgCCCAIQf8BcUECdGooAgAgA3ENAQsgA0GACHFBCnYgCEH/AXFB3wBGcSAFciECCwJ/IAIgAC0AWEYEQEEAIQZBn3gMAQsgASABKAIIIA1qNgIIIAAoAgQhBkGdeAshACABIAY2AiggASAANgIAIARBIGokAAsJACAAEL4EEC8LvgEBBX8gACgCCCIDIAEoAhQgASgCECICa0EMbU0EQCACIANBAWsiA0EMbGotAAgEQAJAIAIgA0EMbGoiAigCBCIFIAIoAgAiBGsiBiABKAIMIAEoAggiA2tKDQAgBCAFRwRAIAMhAgNAIAQtAAAgAi0AAEcNAiACQQFqIQIgBEEBaiIEIAVHDQALCyABQZ54NgIAIAEgAyAGajYCCCABIAAoAgQ2AigPCwsgAUGfeDYCACABQQA2AigPCxD/AQALnQEBBH8CQAJAIAEoAhAgACgCFEEBa0EMbGoiAi0ACEUNASACKAIEIAIoAgAiBWsiAyABKAIMIAEoAggiBGtKDQFBACECIANBAEwNAANAIAIgBWotAAAgAiAEai0AAEcNAiADIAJBAWoiAkcNAAsMAAsgAUGeeDYCACABIAMgBGo2AgggASAAKAIENgIoDwsgAUGfeDYCACABQQA2AigLUgEBfyAAQcyvATYCACAAKAIIIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAAQLwtQAQF/IABBzK8BNgIAIAAoAggiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgAAvPAQEEfwJAAkACQCABKAIQIAAoAhRBDGxqQQxrIgMtAAhFDQIgAygCBCADKAIAayIEIAEoAgwgASgCCCICa0oNAiAEQQBMDQFBACECA0AgACgCDCIFIAMoAgAgAmosAAAgBSgCACgCFBEEACAAKAIMIgUgASgCCCACaiwAACAFKAIAKAIUEQQARw0DIAQgAkEBaiICRw0ACwwACyABKAIIIQILIAFBnng2AgAgASACIARqNgIIIAEgACgCBDYCKA8LIAFBn3g2AgAgAUEANgIoC1IBAX8gAEH0rgE2AgAgACgCCCIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgAEHcqQE2AgAgACgCBCIBBEAgASABKAIAKAIEEQEACyAAEC8LUAEBfyAAQfSuATYCACAAKAIIIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAALjQQCAnsDfyACQR9KBEAgAkEgbSEHA0AgASAFQQd0aiICQUBrIAAgBUESbGoiBi8BAEECdEGQ1gRq/QkCACIDIAb9XAAC/YkB/akBIgRBBP2tAf0M+P////j////4////+P////2uAf36Af3mAf0LAgAgAiADIAT9DA8AAAAPAAAADwAAAA8AAAD9Tv0M+P////j////4////+P////2uAf36Af3mAf0LAgAgAiADIAZBBmr9XAAA/YkB/akBIgRBBP2tAf0M+P////j////4////+P////2uAf36Af3mAf0LAlAgAiADIAT9DA8AAAAPAAAADwAAAA8AAAD9Tv0M+P////j////4////+P////2uAf36Af3mAf0LAhAgAiADIAZBCmr9XAAA/YkB/akBIgRBBP2tAf0M+P////j////4////+P////2uAf36Af3mAf0LAmAgAiADIAT9DA8AAAAPAAAADwAAAA8AAAD9Tv0M+P////j////4////+P////2uAf36Af3mAf0LAiAgAiADIAZBDmr9XAAA/YkB/akBIgRBBP2tAf0M+P////j////4////+P////2uAf36Af3mAf0LAnAgAiADIAT9DA8AAAAPAAAADwAAAA8AAAD9Tv0M+P////j////4////+P////2uAf36Af3mAf0LAjAgBUEBaiIFIAdHDQALCwtPAQF/AkAgASgCCCICIAEoAgxGDQAgAi0AACAALQAIRw0AIAFBnXg2AgAgASACQQFqNgIIIAEgACgCBDYCKA8LIAFBn3g2AgAgAUEANgIoC08BAX8CQCABKAIIIgIgASgCDEYNACACLQAAIAAtABRHDQAgAUGdeDYCACABIAJBAWo2AgggASAAKAIENgIoDwsgAUGfeDYCACABQQA2AigLUgEBfyAAQditATYCACAAKAIIIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAAQLwtQAQF/IABB2K0BNgIAIAAoAggiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgAAtoAQJ/AkAgASgCCCICIAEoAgxGDQAgACgCDCIDIAIsAAAgAygCACgCFBEEAEH/AXEgAC0AFEcNACABQZ14NgIAIAEgASgCCEEBajYCCCABIAAoAgQ2AigPCyABQZ94NgIAIAFBADYCKAtSAQF/IABB/KwBNgIAIAAoAggiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgABAvC1ABAX8gAEH8rAE2AgAgACgCCCIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgAEHcqQE2AgAgACgCBCIBBEAgASABKAIAKAIEEQEACyAAC8wDAwV9BX8DeyACQSBOBEAjAEGAAWshCCACQSBtIQwDQCAIIAAgCkEHdGpBgAH8CgAAIAj9AAQA/eABIAj9AAQQ/eAB/ekBIAj9AAQg/eABIAj9AAQw/eAB/ekB/ekBIAj9AARA/eABIAj9AARQ/eAB/ekBIAj9AARg/eABIAj9AARw/eAB/ekB/ekB/ekBIg39HwAiAyAN/R8BIgQgAyAEXiICGyEFAkACQCAN/R8CIgYgDf0fAyIHXgRAIAMgBiAFIAZeIgkbIQMgAg0CIAkNAQwCCyADIAcgBSAHXiIJGyEDIAINASAJRQ0BCyAEIQMLIAEgCkEobGoiCyADQwAA/kKVIgM4AgBDAACAPyADlUMAAAAAIANDAAAAAFwb/RMhD/0MAAAAAAAAAAAAAAAAAAAAACENQQAhAgNAIAsgAkECdGoiCUEJaiAPIAggAkEEdGr9AAQA/eYB/fgBIg79WAAABCAJQQhqIA79WAAAACAJQQpqIA79WAAACCAJQQtqIA79WAAADCAOIA39rgEhDSACQQFqIgJBCEcNAAsgCyADIA39GwAgDf0bAWogDf0bAmogDf0bA2qylDgCBCAKQQFqIgogDEcNAAsLC+oQAwJ9BH8JeyACQSBOBEAgAkEgbSEIA0AgASAHQShsaiIGIAAgB0EHdGoiBSoCAIsiBCAFKgIEiyIDIAMgBF0bIgQgBSoCCIsiAyADIARdGyIEIAUqAgyLIgMgAyAEXRsiBCAFKgIQiyIDIAMgBF0bIgQgBSoCFIsiAyADIARdGyIEIAUqAhiLIgMgAyAEXRsiBCAFKgIciyIDIAMgBF0bIgQgBSoCIIsiAyADIARdGyIEIAUqAiSLIgMgAyAEXRsiBCAFKgIoiyIDIAMgBF0bIgQgBSoCLIsiAyADIARdGyIEIAUqAjCLIgMgAyAEXRsiBCAFKgI0iyIDIAMgBF0bIgQgBSoCOIsiAyADIARdGyIEIAUqAjyLIgMgAyAEXRsiBCAFQUBrIgIqAgCLIgMgAyAEXRsiBCAFKgJEiyIDIAMgBF0bIgQgBSoCSIsiAyADIARdGyIEIAUqAkyLIgMgAyAEXRsiBCAFKgJQiyIDIAMgBF0bIgQgBSoCVIsiAyADIARdGyIEIAUqAliLIgMgAyAEXRsiBCAFKgJciyIDIAMgBF0bIgQgBSoCYIsiAyADIARdGyIEIAUqAmSLIgMgAyAEXRsiBCAFKgJoiyIDIAMgBF0bIgQgBSoCbIsiAyADIARdGyIEIAUqAnCLIgMgAyAEXRsiBCAFKgJ0iyIDIAMgBF0bIgQgBSoCeIsiAyADIARdGyIEIAUqAnyLIgMgAyAEXRtDAAD+QpUiBDgCAAJ/QwAAgD8gBJVDAAAAACAEQwAAAABcG/0TIgogAv0AAgD95gEiCf0fARA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAJ/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyAC/RcBAn8gCf0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCIQsCfyAJ/R8DEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAF/QACACEJIAZBGGogCyAC/RcDIgv9WgAAAAJ/IAogCf3mASIJ/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAGAn8gCf0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IAn9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAgJ/IAn9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAyIP/VoACAACfyAKIAX9AAJQ/eYBIgn9HwEQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECAn8gCf0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IAn9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAiEMAn8gCf0fAxA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgBf0AAhAhCSAGQRxqIAwgAv0XAyIM/VoAAAACfyAKIAn95gEiCf0fARA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQIgBkEMagJ/IAn9HwAQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAJ/R8CEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwICfyAJ/R8DEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwMiEP1aAAAAAn8gCiAF/QACYP3mASIJ/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAgJ/IAn9HwAQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0PIAL9FwECfyAJ/R8CEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwIhDQJ/IAn9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIAX9AAIgIQkgBkEgaiANIAL9FwMiDf1aAAAAAn8gCiAJ/eYBIgn9HwEQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIAZBEGoCfyAJ/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyAC/RcBAn8gCf0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gCf0fAxA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcDIhH9WgAAAAJ/IAogBf0AAnD95gEiCf0fARA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgLIQICfyAJ/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyAC/RcBAn8gCf0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCIQ4CfyAJ/R8DEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAF/QACMCEJIAZBJGogDiAC/RcDIg79WgAAAAJ/IAogCf3mASIK/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAGQRRqAn8gCv0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IAr9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAgJ/IAr9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAyIK/VoAAAAgBiAEIA/9hwH9pwEgC/2HAf2nAf2uASAQ/YcB/acB/a4BIAz9hwH9pwH9rgEgEf2HAf2nAf2uASAN/YcB/acB/a4BIAr9hwH9pwH9rgEgDv2HAf2nAf2uASIKIAogCv0NCAkKCwwNDg8AAQIDAAECA/2uASIKIAogCv0NBAUGBwABAgMAAQIDAAECA/2uAf0bALKUOAIEIAdBAWoiByAIRw0ACwsLrgQBCn8jAEFAaiICJAAgAkEAOgAoIAJBADoANCACQQA2AhAgAkIANwMgIAJCADcCLCACQQA2AjwgAkEAOgA4IAJCADcDCCAAKAIYIQUgASgCCCEDIAEoAgwhBCACQQA6ABwgAiAENgIYIAIgBDYCFCACQQhqIgQgBUEBaiACQRRqEIICIAJBADoAKCACIAM2AiQgAiACLQAcOgA0IAIgAzYCICACIAM2AjwgAkEBOgA4IAIgAikCFDcCLAJAAkACQCAAQQhqIAEoAggiAyABKAIMIAQgASgCLEG/H3FBwAByIAMgASgCBEYgAS0AMEEAR3EQ3wIgAC0ANEcEQCABQZ54NgIAIAEgACgCBDYCKCACKAIMIAIoAggiA2siBkEMbSIFQQJJDQFBASEEIAVBAWsiB0EBcSELIAEoAhAhBSAAKAIwIQggBkEYRwRAIAdBfnEhCkEAIQYDQCAFIAQgCGpBDGxqIgFBDGsiByADIARBDGxqIgAoAgA2AgAgByAAKAIENgIEIAcgAC0ACDoACCABIAAoAgw2AgAgASAAKAIQNgIEIAEgAC0AFDoACCAEQQJqIQQgBkECaiIGIApHDQALCyALRQ0CIAQgCGpBDGwgBWpBDGsiACADIARBDGxqIgEoAgA2AgAgACABKAIENgIEIAAgAS0ACDoACAwCCyABQQA2AiggAUGfeDYCACACKAIIIQMLIANFDQELIAIgAzYCDCADEC8LIAJBQGskAAulAQEEfyAAQdyrATYCACAAQQhqIQQCQCAAKAIoIgFFDQAgAUF//h4CBA0AIAEgASgCACgCCBEBAAJAIAFBCGoiA/4QAgAEQCADQX/+HgIADQELIAEgASgCACgCEBEBAAsLIAQoAgAiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgABAvC6MBAQR/IABB3KsBNgIAIABBCGohBAJAIAAoAigiAUUNACABQX/+HgIEDQAgASABKAIAKAIIEQEAAkAgAUEIaiID/hACAARAIANBf/4eAgANAQsgASABKAIAKAIQEQEACwsgBCgCACIBQQRqQX/+HgIARQRAIAEgASgCACgCCBEBAAsgAEHcqQE2AgAgACgCBCIBBEAgASABKAIAKAIEEQEACyAAC+sCAQZ/AkAgASgCBCIDIAEoAgwiBEYNACAEIAEoAggiAkYEQEEAIQIgAS0ALEEIcQ0BQQEhAiAEQQFrLQAAIgNB3wBGDQEgA8BBAE4EQCAAKAIMKAIIIANBAnRqLQAAQeAAcQ0CC0EAIQIMAQsCQCACIANHDQAgASgCLCIEQYABcQ0AQQAhAiAEQQRxDQFBASECIAMtAAAiA0HfAEYNASADwEEATgRAIAAoAgwoAgggA0ECdGotAABB4ABxDQILQQAhAgwBCyACLQAAIgbAIQRBASEFAn9BASACQQFrLQAAIgJB3wBGDQAaIALAQQBOBEBBASAAKAIMKAIIIAJBAnRqLQAAQeAAcQ0BGgtBAAshBwJAIARB3wBGDQAgBEEATgRAIAAoAgwoAgggBkECdGotAABB4ABxDQELQQAhBQsgByAFRyECCyABQQAgACgCBCAALQAUIAJGIgAbNgIoIAFBn3hBnnggABs2AgALUgEBfyAAQYSrATYCACAAKAIIIgFBBGpBf/4eAgBFBEAgASABKAIAKAIIEQEACyAAQdypATYCACAAKAIEIgEEQCABIAEoAgAoAgQRAQALIAAQLwtQAQF/IABBhKsBNgIAIAAoAggiAUEEakF//h4CAEUEQCABIAEoAgAoAggRAQALIABB3KkBNgIAIAAoAgQiAQRAIAEgASgCACgCBBEBAAsgAAtmAQF/AkACQCABKAIIIgIgASgCDEcNACABLQAsQQJxDQAMAQsCQCAALQAIRQ0AAkAgAi0AAEEKaw4EAAEBAAELDAELIAFBn3g2AgAgAUEANgIoDwsgAUGeeDYCACABIAAoAgQ2AigL8wMDBX0GfwJ7IAJBIE4EQCMAQYABayEIIAJBIG0hDANAIAggACAKQQd0akGAAfwKAAAgCP0ABAD94AEgCP0ABBD94AH96QEgCP0ABCD94AEgCP0ABDD94AH96QH96QEgCP0ABED94AEgCP0ABFD94AH96QEgCP0ABGD94AEgCP0ABHD94AH96QH96QH96QEiDv0fACIDIA79HwEiBCADIAReIgIbIQUCQAJAIA79HwIiBiAO/R8DIgdeBEAgAyAGIAUgBl4iCRshAyACDQIgCQ0BDAILIAMgByAFIAdeIgkbIQMgAg0BIAlFDQELIAQhAwsgASAKQSJsaiINQYD8ASADQwAA/kKVIgOLQwAAgHeUQwAAgAiUQYCAgIgHIAO8IgJBAXQiCUGAgIB4cSILIAtBgICAiAdNG0EBdkGAgIA8ar6SvCILQQ12QYD4AXEgC0H/H3FqIAlBgICAeEsbIAJBEHZBgIACcXI7AQBDAACAPyADlUMAAAAAIANDAAAAAFwb/RMhD0EAIQIDQCANIAJBAnRqIglBA2ogDyAIIAJBBHRq/QAEAP3mAf34ASIO/VgAAAQgCUECaiAO/VgAAAAgCUEEaiAO/VgAAAggCUEFaiAO/VgAAAwgAkEBaiICQQhHDQALIApBAWoiCiAMRw0ACwsLbQACQAJAIAEtADAEQCABKAIIIAEoAgRHDQEgAS0ALEEBcQ0BDAILIAAtAAhFDQACQCABKAIIQQFrLQAAQQprDgQAAQEAAQsMAQsgAUGfeDYCACABQQA2AigPCyABQZ54NgIAIAEgACgCBDYCKAukDwMCfQd/AnsgAkEgTgRAIAJBIG0hCQNAIAEgBkEibGoiCEGA/AEgACAGQQd0aiIFKgIAiyIDIAUqAgSLIgQgAyAEXhsiAyAFKgIIiyIEIAMgBF4bIgMgBSoCDIsiBCADIAReGyIDIAUqAhCLIgQgAyAEXhsiAyAFKgIUiyIEIAMgBF4bIgMgBSoCGIsiBCADIAReGyIDIAUqAhyLIgQgAyAEXhsiAyAFKgIgiyIEIAMgBF4bIgMgBSoCJIsiBCADIAReGyIDIAUqAiiLIgQgAyAEXhsiAyAFKgIsiyIEIAMgBF4bIgMgBSoCMIsiBCADIAReGyIDIAUqAjSLIgQgAyAEXhsiAyAFKgI4iyIEIAMgBF4bIgMgBSoCPIsiBCADIAReGyIDIAVBQGsiCioCAIsiBCADIAReGyIDIAUqAkSLIgQgAyAEXhsiAyAFKgJIiyIEIAMgBF4bIgMgBSoCTIsiBCADIAReGyIDIAUqAlCLIgQgAyAEXhsiAyAFKgJUiyIEIAMgBF4bIgMgBSoCWIsiBCADIAReGyIDIAUqAlyLIgQgAyAEXhsiAyAFKgJgiyIEIAMgBF4bIgMgBSoCZIsiBCADIAReGyIDIAUqAmiLIgQgAyAEXhsiAyAFKgJsiyIEIAMgBF4bIgMgBSoCcIsiBCADIAReGyIDIAUqAnSLIgQgAyAEXhsiAyAFKgJ4iyIEIAMgBF4bIgMgBSoCfIsiBCADIAReG0MAAP5ClSIDi0MAAIB3lEMAAIAIlEGAgICIByADvCICQQF0IgtBgICAeHEiByAHQYCAgIgHTRtBAXZBgICAPGq+krwiB0ENdkGA+AFxIAdB/x9xaiALQYCAgHhLGyACQRB2QYCAAnFyOwEAAn9DAACAPyADlUMAAAAAIANDAAAAAFwb/RMiDSAF/QACAP3mASIM/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAshAiAIAn8gDP0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/Q8gAv0XAQJ/IAz9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAgJ/IAz9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XAwJ/IA0gBf0AAhD95gEiDP0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcEAn8gDP0fARA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcFAn8gDP0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcGAn8gDP0fAxA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcHAn8gDSAF/QACIP3mASIM/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwgCfyAM/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwkCfyAM/R8CEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwoCfyAM/R8DEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwsCfyANIAX9AAIw/eYBIgz9HwAQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XDAJ/IAz9HwEQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XDQJ/IAz9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XDgJ/IAz9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XD/0LAAICfyANIAr9AAIA/eYBIgz9HwEQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4CyECIAgCfyAM/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9DyAC/RcBAn8gDP0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcCAn8gDP0fAxA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcDAn8gDSAF/QACUP3mASIM/R8AEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwQCfyAM/R8BEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwUCfyAM/R8CEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwYCfyAM/R8DEDUiA4tDAAAAT10EQCADqAwBC0GAgICAeAv9FwcCfyANIAX9AAJg/eYBIgz9HwAQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XCAJ/IAz9HwEQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XCQJ/IAz9HwIQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XCgJ/IAz9HwMQNSIDi0MAAABPXQRAIAOoDAELQYCAgIB4C/0XCwJ/IA0gBf0AAnD95gEiDf0fABA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcMAn8gDf0fARA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcNAn8gDf0fAhA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcOAn8gDf0fAxA1IgOLQwAAAE9dBEAgA6gMAQtBgICAgHgL/RcP/QsAEiAGQQFqIgYgCUcNAAsLCxQAIABBDGpBACABKAIEQeynAUYbCxkAIAAoAgwiAARAIAAgACgCACgCBBEBAAsLAgALCgAgAUGYeDYCAAsLACAAIAEgAhDMBAuLAQEDfxCbAiEDIAAoAgAhAiAAQQA2AgAgAygCACACELIBIAAoAgggACgCDCAAQRBqIAAoAiAgACgCKCAAKAIsIAAoAjAgACgCNCAAKAI4IAAoAgQRKQAgACgCECIBBEAgACABNgIUIAEQLwsgACgCACEBIABBADYCACABBEAgARDdARAvCyAAEC9BAAtBAEHtC0ECQeT5AEG0+gBBAUECQQAQBkGzI0EBQbj6AEG8+gBBA0EEQQAQBkHXC0EEQcD6AEHs+gBBBUEGQQAQBgsWACABQbjDAigCACIAEKsEGiAAEDAaCwsAIAAgASACENUEC+UBAgd/AX0jAEHAAWsiAyQAEJsCIQYgACgCACECIABBADYCACAGKAIAIAIQsgEgACgCBEEB/h4CACIBIAAoAggoAgBIBEADQAJAIAAoAgwoAgAiAiABQdgUbGoiAS0AsAINACABLQCxAg0AIAAoAhgqAgAhCCAAKAIQKAIAIQcgA0EMaiIFIAAoAhRBtAH8CgAAIAcgAiABQdABaiAFIAgQigILIAAoAgRBAf4eAgAiASAAKAIIKAIASA0ACwsgACgCACEBIABBADYCACABBEAgARDdARAvCyAAEC8gA0HAAWokAEEAC0kBA38QmwIhAyAAKAIAIQIgAEEANgIAIAMoAgAgAhCyASAAQQRqEIkCIAAoAgAhASAAQQA2AgAgAQRAIAEQ3QEQLwsgABAvQQALXwECf0Hg1zQoAgAiAARAQeTXNCgCACICIAAiAUcEQANAIAJBDGshASACQQFrLAAAQQBIBEAgASgCABAvCyABIgIgAEcNAAtB4Nc0KAIAIQELQeTXNCAANgIAIAEQLwsLGABB29c0LAAAQQBIBEBB0Nc0KAIAEC8LCwsAIAAgASACEOEEC9QBAQN/IwBBEGsiBCQAIABBADYCBCMAQRBrIgUkACAEQQA6AA8gACAAKAIAQQxrKAIAaiEDAkAgAygCEEUEQCADKAJIBEAgAygCSBD+AwsgBCAAIAAoAgBBDGsoAgBqKAIQRToADwwBCyADQQQQ8wELIAVBEGokAEEEIQMgBC0ADwRAIAAgACAAKAIAQQxrKAIAaigCGCIDIAEgAiADKAIAKAIgEQMAIgE2AgRBBkEAIAEgAkcbIQMLIAAgACgCAEEMaygCAGogAxDzASAEQRBqJAAgAgsZACAAIAAoAgBBDGsoAgBqLQAQQQJxQQF2CykAIABBCGoQ9wNFBEAgACAAKAIAQQxrKAIAaiIAIAAoAhBBBHIQuwILCwsAIAAgASACEOQECxAAQcTXNEHI1zQoAgAQ8wILEABBuNc0QbzXNCgCABD0AgsEAEEBC2gBA38jAEEgayICJAAgAkEMaiABIAAoAjQiACgCABD4AiAAKAIEIQMgAigCDCIEIAAoAghLBEAgACADIAQQxAEiAzYCBCAAIAIoAgw2AggLIAIgAzYCECABIAJBDGoQ9wIgAkEgaiQACwwAIAFBGGogARD3AgsNACABKAIEEC8gARAvC4oBAQJ/IwBBIGsiAiQAIAAoAjQhA0HQABA7IQAgAkEMaiABIAMoAgAQ+AIgACACKAIcNgIQIAAgAv0AAgz9CwIAIAAgASkDMDcDSCAAIAH9AAMg/QsDOCAAIAH9AAMQ/QsDKCAAIAH9AAMA/QsDGCAAKAIAIgEEQCAAIAEQOzYCBAsgAkEgaiQAIAALBgBBuLgDCxgBAX8gACgCNCIBKAIEEC8gARAvIAAQLwsTACACIAEoAtABQQAgARBbEJcBCxMAIAEgAigC0AFBACABEFsQ5gQLqgEAIwBBIGsiACQAAkAgARBbIAMgBGpJBEBBvMMCKAIAEDAaIABBit4ANgIIIABBigM2AgQgAEGbJzYCAEG4wwIoAgBBy+QAIAAQMQwBCyABKALQASIBRQRAQbzDAigCABAwGiAAQZzfADYCGCAAQYsDNgIUIABBmyc2AhBBuMMCKAIAQcvkACAAQRBqEDEMAQsgAiABIANqIAT8CgAAIABBIGokAA8LEAAAC6oBACMAQSBrIgAkAAJAIAEQWyADIARqSQRAQbzDAigCABAwGiAAQcXdADYCCCAAQYEDNgIEIABBmyc2AgBBuMMCKAIAQcvkACAAEDEMAQsgASgC0AEiAUUEQEG8wwIoAgAQMBogAEGc3wA2AhggAEGCAzYCFCAAQZsnNgIQQbjDAigCAEHL5AAgAEEQahAxDAELIAEgA2ogAiAE/AoAACAAQSBqJAAPCxAAAAsFAEGoKwsKACABKAIAQTxGCwUAQcAACwkAIAAoAiAQLwsHACAAKAIgC5sBAQN/IwBBEGsiAiQAIAFBQGsiAxA7IgRFBEBBvMMCKAIAEDAaIAJBzt4ANgIIIAJBuQM2AgQgAkGbJzYCAEG4wwIoAgBBy+QAIAIQMRAAAAtBKBA7IgFBOjYCBCABQTs2AgAgASADNgIkIAEgBDYCICABIAA2AhwgAUH8pAH9AAIA/QsCCCABQYylASgCADYCGCACQRBqJAAgAQuoBgACQAJAAkBBsJk1QQBBAf5IAgAOAgABAgtBgAgkAUGACP0MAAAAAAAAAAAAAAAAAAAAAP0LAABBkAhBAEGSlAH8CAAAQaKcAUEAQR78CwBBwJwBQQBBBvwIAQBBxpwBQQBBHvwLAEHknAFBAEH1AvwIAgBB2Z8BQQBBG/wLAEH0nwFBAEEJ/AgDAEH9nwFBAEEb/AsAQZigAUEAQQn8CAQAQaGgAUEAQR/8CwBBwKABQQBBmJMB/AgFAEHYswJBAEEe/AsAQfazAkEAQQL8CAYAQfizAkEAQR38CwBBlbQCQQBBnRD8CAcAQbLEAkEAQRn8CwBBy8QCQQBBIfwICABB7MQCQQBBGfwLAEGFxQJBAEEh/AgJAEGmxQJBAEEZ/AsAQb/FAkEAQSr8CAoAQenFAkEAQRn8CwBBgsYCQQBBDvwICwBBkMYCQQBBI/wLAEGzxgJBAEEh/AgMAEHUxgJBAEEZ/AsAQe3GAkEAQbYK/AgNAEGj0QJBAEEt/AsAQdDRAkEAQQL8CA4AQdLRAkEAQR78CwBB8NECQQBBygD8CA8AQbrSAkEAQYoE/AsAQcTWAkEAQfkD/AgQAEG92gJBAEGDBPwLAEHA3gJBAEEC/AgRAEHC3gJBAEGSBPwLAEHU4gJBAEH5A/wIEgBBzeYCQQBBgwT8CwBB0OoCQQBBvQb8CBMAQY3xAkEAQYcE/AsAQZT1AkEAQec4/AgUAEH7rQNBAEEf/AsAQZquA0EAQfgA/AgVAEGSrwNBAEHkAPwLAEH2rwNBAEG+CPwIFgBBuLgDQQBBFfwIFwBBzbgDQQBBN/wLAEGEuQNBAEHoAPwIGABB7LkDQQBBPPwLAEGougNBAEHZAPwIGQBBgbsDQQBBP/wLAEHAuwNBAEEM/AgaAEHQuwNBAEHg3TH8CwBBsJk1QQL+FwIAQbCZNUF//gACABoMAQtBsJk1QQFCf/4BAgAaC/wJAPwJAfwJAvwJA/wJBPwJBfwJBvwJB/wJCPwJCfwJCvwJC/wJDPwJDfwJDvwJD/wJEPwJEfwJEvwJE/wJFPwJFfwJFvwJF/wJGPwJGfwJGgsL5pkDGwGSlAHima8A4pmuAOKZrQDimawA4pmrAOKZquKZquKZqgDimakA44CPAOOAjgDjgI0A44CMAHoAaW5maW5pdHkAaW5zdWZmaWNpZW50IG1lbW9yeQBGZWJydWFyeQBKYW51YXJ5AEp1bHkAbWFsYXkAVGh1cnNkYXkAVHVlc2RheQBXZWRuZXNkYXkAU2F0dXJkYXkAU3VuZGF5AE1vbmRheQBGcmlkYXkATWF5ACVtLyVkLyV5ACEha3Zfc2VsZi5jdHgAc2l4ACVzIGZhaWxlZCB0byByZWxlYXNlIG11dGV4ACVzIGZhaWxlZCB0byBhY3F1aXJlIG11dGV4AGNpcmN1bWZsZXgALSsgICAwWDB4AC0wWCswWCAwWC0weCsweCAweABoZWJyZXcAaGF3AE5vdgBlbWJkX2NvbnYAVGh1AHRlbHVndQBBdWd1c3QAJXMgZmFpbGVkIHRvIGJyb2FkY2FzdAB1bnNpZ25lZCBzaG9ydABhbGVydAB3aGlzcGVyX2t2X2NhY2hlX2ZpbmRfc2xvdABwcmludAB1bnNpZ25lZCBpbnQAY2lyY3VtZmxleC1hY2NlbnQAZ3JhdmUtYWNjZW50AGZ1bGxfZGVmYXVsdABrdl9jYWNoZV9pbml0AHhkaWdpdAAuY3Jvc3NfYXR0bi5xdWVyeS53ZWlnaHQALmF0dG4ucXVlcnkud2VpZ2h0AC5jcm9zc19hdHRuLmtleS53ZWlnaHQALmF0dG4ua2V5LndlaWdodAAuY3Jvc3NfYXR0bi5vdXQud2VpZ2h0AC5hdHRuLm91dC53ZWlnaHQAZW5jb2Rlci5sbl9wb3N0LndlaWdodAAubWxwX2xuLndlaWdodAAuY3Jvc3NfYXR0bl9sbi53ZWlnaHQALmF0dG5fbG4ud2VpZ2h0AGRlY29kZXIubG4ud2VpZ2h0AGRlY29kZXIudG9rZW5fZW1iZWRkaW5nLndlaWdodAAuY3Jvc3NfYXR0bi52YWx1ZS53ZWlnaHQALmF0dG4udmFsdWUud2VpZ2h0AGVuY29kZXIuY29udjIud2VpZ2h0AC5tbHAuMi53ZWlnaHQAZW5jb2Rlci5jb252MS53ZWlnaHQALm1scC4wLndlaWdodABzZXQAcmlnaHQtY3VybHktYnJhY2tldABsZWZ0LWN1cmx5LWJyYWNrZXQAcmlnaHQtc3F1YXJlLWJyYWNrZXQAbGVmdC1zcXVhcmUtYnJhY2tldABwdW5jdABnZ21sX25ld19vYmplY3QAT2N0AGZsb2F0AFNhdABjb21tZXJjaWFsLWF0AHVpbnQ2NF90AGh5cGhlbi1taW51cwByZXZlcnNlLXNvbGlkdXMAd2hpc3Blcl9leHBfY29tcHV0ZV90b2tlbl9sZXZlbF90aW1lc3RhbXBzAGFmcmlrYWFucwBwYXJhbXMAZ2dtbF9nZXRfbl90YXNrcwByaWdodC1wYXJlbnRoZXNpcwBsZWZ0LXBhcmVudGhlc2lzAHdoaXNwZXJfcHJpbnRfdGltaW5ncwAuY3Jvc3NfYXR0bi5xdWVyeS5iaWFzAC5hdHRuLnF1ZXJ5LmJpYXMALmNyb3NzX2F0dG4ub3V0LmJpYXMALmF0dG4ub3V0LmJpYXMAZW5jb2Rlci5sbl9wb3N0LmJpYXMALm1scF9sbi5iaWFzAC5jcm9zc19hdHRuX2xuLmJpYXMALmF0dG5fbG4uYmlhcwBkZWNvZGVyLmxuLmJpYXMALmNyb3NzX2F0dG4udmFsdWUuYmlhcwAuYXR0bi52YWx1ZS5iaWFzAGVuY29kZXIuY29udjIuYmlhcwAubWxwLjIuYmlhcwBlbmNvZGVyLmNvbnYxLmJpYXMALm1scC4wLmJpYXMAcm93X2lkID49IDAgJiYgcm93X2lkIDwgbl9hcwAlcwBmb3VyAHdoaXNwZXJfbGFuZ19zdHIAQXByAGNvbnN0cnVjdG9yAHZlY3RvcgBjZ3JhcGgtPm5vZGVzW2NncmFwaC0+bl9ub2RlcyAtIDFdID09IHRlbnNvcgB1bmtub3duIGFsbG9jYXRpb24gZXJyb3IAYmFzaGtpcgBsb3dlcgB1cHBlcgBraG1lcgBidWZmZXIAT2N0b2JlcgBOb3ZlbWJlcgBTZXB0ZW1iZXIARGVjZW1iZXIAdGF0YXIAbXlhbm1hcgB1bnNpZ25lZCBjaGFyAGlvc19iYXNlOjpjbGVhcgBNYXIAcQAvVXNlcnMvZ2dlcmdhbm92L2RldmVsb3BtZW50L2dpdGh1Yi93aGlzcGVyLmNwcC93aGlzcGVyLmNwcABmdWxsLXN0b3AAU2VwACVJOiVNOiVTICVwAHR3bwBhdXRvAHBhc2h0bwB6ZXJvAGxhbwB1bmtub3duAFN1bgBKdW4AY2FycmlhZ2UtcmV0dXJuAGJyZXRvbgBzZW1pY29sb24Ac3RkOjpleGNlcHRpb24AX19jeGFfZ3VhcmRfYWNxdWlyZSBkZXRlY3RlZCByZWN1cnNpdmUgaW5pdGlhbGl6YXRpb24ATW9uAGxhdGluAHBlcmNlbnQtc2lnbgBwbHVzLXNpZ24AZXF1YWxzLXNpZ24AbnVtYmVyLXNpZ24AZG9sbGFyLXNpZ24AbGVzcy10aGFuLXNpZ24AZ3JlYXRlci10aGFuLXNpZ24Ac2V2ZW4AdHVya21lbgBoeXBoZW4Ab2NjaXRhbgB0aWJldGFuAG5hbgBnZXJtYW4AY3BsYW4AY2F0YWxhbgBsYXR2aWFuAGJlbGFydXNpYW4AcnVzc2lhbgBwZXJzaWFuAGluZG9uZXNpYW4AaHVuZ2FyaWFuAGJ1bGdhcmlhbgBib3NuaWFuAG1hY2Vkb25pYW4AdWtyYWluaWFuAHNsb3ZlbmlhbgBsaXRodWFuaWFuAG1vbmdvbGlhbgBpdGFsaWFuAG5vcndlZ2lhbgBzZXJiaWFuAGtvcmVhbgBKYW4AYWxudW0AbWVkaXVtAG1hbGF5YWxhbQBKdWwAY250cmwAZ2dtbF9uZXdfdGVuc29yX2ltcGwAYm9vbABzbWFsbABzdGQ6OmJhZF9mdW5jdGlvbl9jYWxsAEFwcmlsAHRhbWlsAGVtc2NyaXB0ZW46OnZhbABueW5vcnNrAGFzdGVyaXNrAHF1ZXN0aW9uLW1hcmsAcXVvdGF0aW9uLW1hcmsAZXhjbGFtYXRpb24tbWFyawBibGFuawB0YWppawBncmVlawB1emJlawBzbG92YWsAagBtYW9yaQBGcmkAYXplcmJhaWphbmkAc3dhaGlsaQBuZXBhbGkAc29tYWxpAGJlbmdhbGkAbWFyYXRoaQBzaW5kaGkAaGluZGkAcHVuamFiaQBiYWRfYXJyYXlfbmV3X2xlbmd0aAB3ZWxzaABmaW5uaXNoAHNwYW5pc2gAZGFuaXNoAHBvbGlzaABlbmdsaXNoAHR1cmtpc2gAbHV4ZW1ib3VyZ2lzaABzd2VkaXNoAHlpZGRpc2gAYmFja3NsYXNoAGdyYXBoAGthemFraABkdXRjaABNYXJjaABmcmVuY2gAY3plY2gAQXVnAHRhZ2Fsb2cAdW5zaWduZWQgbG9uZwB0ZXJtaW5hdGluZwBzdGQ6OndzdHJpbmcAYmFzaWNfc3RyaW5nAHN0ZDo6c3RyaW5nAHN0ZDo6dTE2c3RyaW5nAHN0ZDo6dTMyc3RyaW5nAGVuY29kZXIucG9zaXRpb25hbF9lbWJlZGRpbmcAZGVjb2Rlci5wb3NpdGlvbmFsX2VtYmVkZGluZwBpbmYAJS4wTGYAJUxmAGNncmFwaC0+bl9sZWFmcyA8IGNncmFwaC0+c2l6ZQBjZ3JhcGgtPm5fbm9kZXMgPCBjZ3JhcGgtPnNpemUAd2hpc3Blcl90b2tlbml6ZQBmaXZlAHl1ZQB0cnVlAGJhc3F1ZQBpbnZhbGlkIGFsaWdubWVudCB2YWx1ZQBUdWUAd2hpc3Blcl9pbml0X3N0YXRlAHdoaXNwZXJfaW5pdF93aXRoX3BhcmFtc19ub19zdGF0ZQB3aGlzcGVyX2luaXRfZnJvbV9maWxlX3dpdGhfcGFyYW1zX25vX3N0YXRlAHdoaXNwZXJfbGFuZ19hdXRvX2RldGVjdF93aXRoX3N0YXRlAHdoaXNwZXJfZnVsbF93aXRoX3N0YXRlAHdoaXNwZXJfZW5jb2RlX3dpdGhfc3RhdGUAd2hpc3Blcl9kZWNvZGVfd2l0aF9zdGF0ZQB0cmFuc2xhdGUAZmFsc2UAcG9ydHVndWVzZQBtYWx0ZXNlAGZhcm9lc2UAY2FudG9uZXNlAGNoaW5lc2UAc3VuZGFuZXNlAHZpZXRuYW1lc2UAX19jeGFfZ3VhcmRfcmVsZWFzZQB1bmRlcnNjb3JlAF9fY3hhX2d1YXJkX2FjcXVpcmUAc3JjMC0+dHlwZSA9PSBkc3QtPnR5cGUAZHN0LT50eXBlID09IHNyYzAtPnR5cGUAVW5rbm93biBlcnJvciB0eXBlAEp1bmUAb25lAG5pbmUAbmV3bGluZQBsb3ctbGluZQB2ZXJ0aWNhbC1saW5lAGhhaXRpYW4gY3Jlb2xlAGRvdWJsZQBhcG9zdHJvcGhlAGxhcmdlAHRocmVlAGZyZWUAdGlsZGUAcmlnaHQtYnJhY2UAbGVmdC1icmFjZQBiYWNrc3BhY2UAdHJhbnNjcmliZQBwZXJpb2QAbWFwOjphdDogIGtleSBub3QgZm91bmQAYW1wZXJzYW5kAHZvaWQAd2hpc3Blcl9sYW5nX2lkAHRlcm1pbmF0ZV9oYW5kbGVyIHVuZXhwZWN0ZWRseSByZXR1cm5lZAB0aHJlYWQgY29uc3RydWN0b3IgZmFpbGVkAF9fdGhyZWFkX3NwZWNpZmljX3B0ciBjb25zdHJ1Y3Rpb24gZmFpbGVkAHRocmVhZDo6am9pbiBmYWlsZWQAbXV0ZXggbG9jayBmYWlsZWQAZm9ybS1mZWVkAFdlZAB3aGlzcGVyX21vZGVsX2xvYWQAJTAyZDolMDJkOiUwMmQlcyUwM2QAbGVhZl8lZABub2RlXyVkAGdnbWxfYWxpZ25lZF9tYWxsb2MAZ2dtbF90YWxsb2NyX2FsbG9jAHN0ZDo6YmFkX2FsbG9jAGRzdC0+bmVbMF0gPT0gbmMAYW1oYXJpYwBpY2VsYW5kaWMAYXJhYmljAERlYwAvVXNlcnMvZ2dlcmdhbm92L2RldmVsb3BtZW50L2dpdGh1Yi93aGlzcGVyLmNwcC9nZ21sLmMAL1VzZXJzL2dnZXJnYW5vdi9kZXZlbG9wbWVudC9naXRodWIvd2hpc3Blci5jcHAvZ2dtbC1iYWNrZW5kLmMAL1VzZXJzL2dnZXJnYW5vdi9kZXZlbG9wbWVudC9naXRodWIvd2hpc3Blci5jcHAvZ2dtbC1hbGxvYy5jAHdiAHJiAEZlYgB2ZXJ0aWNhbC10YWIAbl9sb2dpdHMgPT0gY3R4LnZvY2FiLm5fdm9jYWIAdytiAHIrYgBhK2IAcndhAGNwbGFuLT53b3JrX2RhdGEAaGF1c2EAc2hvbmEAY29tbWEAc2luaGFsYQBsaW5nYWxhAGFscGhhAGthbm5hZGEAeW9ydWJhAFtfZXh0cmFfdG9rZW5fAFtfVFRfAFtfTEFOR18AW19TT1RfXQBbX05PVF9dAFtfRU9UX10AW19CRUdfXQBbX1RSQU5TTEFURV9dAFtfVFJBTlNDUklCRV9dAGEtPm5lWzJdID09IGItPm5lWzJdAGEtPm5lWzJdID09IGItPm5lWzFdAGEtPm5lWzFdID09IGItPm5lWzFdAFoAdGVuc29yLT5vcCA9PSBHR01MX09QX1VOQVJZAE1BUF9VTkFSWQBNQVBfQklOQVJZAENQWQAlYSAlYiAlZCAlSDolTTolUyAlWQBQT1NJWABTT0ZUX01BWABBUkdNQVgAVklFVwBESVYAQ1BVAExFQUtZX1JFTFUAU1FSVABBUkdTT1JUAFdJTl9QQVJUAFdJTl9VTlBBUlQAaXNfcG9zaXRpdmVfY2hhciB8fCBwb3MtPnR5cGUgPT0gV0hJU1BFUl9HUkVUWVBFX0NIQVJfTk9UAHd0eXBlICE9IEdHTUxfVFlQRV9DT1VOVABDT05UAFNFVABNVUxfTUFUAFJFUEVBVABDT05DQVQAR0VUX1JPV1MAU1VNX1JPV1MAQ1JPU1NfRU5UUk9QWV9MT1NTAEdFVF9SRUxfUE9TAEFERF9SRUxfUE9TAGF4aXMzID49IDAgJiYgYXhpczMgPCBHR01MX01BWF9ESU1TAGF4aXMyID49IDAgJiYgYXhpczIgPCBHR01MX01BWF9ESU1TAGF4aXMxID49IDAgJiYgYXhpczEgPCBHR01MX01BWF9ESU1TAGF4aXMwID49IDAgJiYgYXhpczAgPCBHR01MX01BWF9ESU1TACVIOiVNOiVTAFNRUgBRAERVUABDTEFNUABESUFHX01BU0tfWkVSTwBGTEFTSF9BVFROAE5BTgBNRUFOAG5lMSA9PSBOAG5lZDEgPT0gTgBTVU0AUk1TX05PUk0AR1JPVVBfTk9STQBQTQBBTQBuZWIxMCA9PSBNAG5lYzAwID09IE0ATlVMAE1VTABJTTJDT0wAdGVuc29yLT5kYXRhID09IE5VTEwAaGFzaF9zZXQua2V5c1tpXSA9PSBOVUxMAHRlbnNvciAhPSBOVUxMAGN0eC0+bWVtX2J1ZmZlciAhPSBOVUxMAHZpZXctPnZpZXdfc3JjICE9IE5VTEwgJiYgdmlldy0+dmlld19zcmMtPmRhdGEgIT0gTlVMTABpICE9IEdHTUxfSEFTSFRBQkxFX0ZVTEwATENfQUxMAHE4X0sAcTZfSwBxNV9LAHE0X0sAcTNfSwBxMl9LAFNPRlRfTUFYX0JBQ0sAU0lMVV9CQUNLAFJFUEVBVF9CQUNLAEdFVF9ST1dTX0JBQ0sAQ1JPU1NfRU5UUk9QWV9MT1NTX0JBQ0sARkxBU0hfQVRUTl9CQUNLAFJNU19OT1JNX0JBQ0sAUk9QRV9CQUNLAEoAQUxJQkkASABMT0cATEFORwBESUFHAERJQUdfTUFTS19JTkYARkxBU0hfRkYAUEVSTVVURQBUUkFOU1BPU0UAUk9QRQBSRVNIQVBFAE5PTkUAVVBTQ0FMRQBPVVRfUFJPRABNVUxfTUFUX0lEAERFUFJFQ0FURUQAQUREAFBBRABQT09MXzJEAENPTlZfVFJBTlNQT1NFXzJEAFBPT0xfMUQAQ09OVl9UUkFOU1BPU0VfMUQAbmV2MSA9PSBEAG5lYzAxID09IEQAbmVrMCA9PSBEAG5lMCA9PSBEAG5lZDAgPT0gRABuZWMxMCA9PSBEAG5lYjAwID09IEQAQUNDAFNVQgBBAGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PHNob3J0PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzx1bnNpZ25lZCBzaG9ydD4AZW1zY3JpcHRlbjo6bWVtb3J5X3ZpZXc8aW50PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzx1bnNpZ25lZCBpbnQ+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PGZsb2F0PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzx1aW50OF90PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzxpbnQ4X3Q+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PHVpbnQxNl90PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzxpbnQxNl90PgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzx1aW50NjRfdD4AZW1zY3JpcHRlbjo6bWVtb3J5X3ZpZXc8aW50NjRfdD4AZW1zY3JpcHRlbjo6bWVtb3J5X3ZpZXc8dWludDMyX3Q+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PGludDMyX3Q+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PGNoYXI+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PHVuc2lnbmVkIGNoYXI+AHN0ZDo6YmFzaWNfc3RyaW5nPHVuc2lnbmVkIGNoYXI+AGVtc2NyaXB0ZW46Om1lbW9yeV92aWV3PHNpZ25lZCBjaGFyPgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzxsb25nPgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzx1bnNpZ25lZCBsb25nPgBlbXNjcmlwdGVuOjptZW1vcnlfdmlldzxkb3VibGU+AD4+PgA8PDwAMDEyMzQ1Njc4OQBpOABIRUFQVTgAQy5VVEYtOABpMTYAZjE2AGRzdC0+dHlwZSA9PSBHR01MX1RZUEVfRjE2AHNyYzAtPnR5cGUgPT0gR0dNTF9UWVBFX0YxNgAgdjMAYXhpczIgIT0gYXhpczMAYXhpczEgIT0gYXhpczMAYXhpczAgIT0gYXhpczMAbmUwMyA9PSBuZTMAbmIyIDw9IG5iMwBNQVBfQ1VTVE9NMwBuZTMgPT0gbmUxMwBuZTAzID09IG5lMTMAbmUzID09IG5lMDMAYXhpczEgIT0gYXhpczIAYXhpczAgIT0gYXhpczIAZ2dtbF9uZWxlbWVudHMoYSkgPT0gbmUwKm5lMSpuZTIAbl9oZWFkID09IG5lMgBuZTAyID09IG5lMgBuYjEgPD0gbmIyAG5lMiA9PSBuZWEyAE1BUF9DVVNUT00yAGkzMgBmMzIAYi0+dHlwZSA9PSBHR01MX1RZUEVfSTMyAGRzdC0+dHlwZSA9PSBHR01MX1RZUEVfRjMyAHNyYzEtPnR5cGUgPT0gR0dNTF9UWVBFX0YzMgBNQVBfQ1VTVE9NM19GMzIATUFQX0NVU1RPTTJfRjMyAE1BUF9DVVNUT00xX0YzMgBuZTIgPT0gbmUxMgBuZTAyID09IG5lMTIAbmUyID09IG5lMDIAYXhpczAgIT0gYXhpczEAZ2dtbF9uZWxlbWVudHMoYSkgPT0gbmUwKm5lMQBuZTAwID09IG5lMQBuYjAgPD0gbmIxAG5lMSA9PSBuZWExAHE4XzEAcTVfMQBxNF8xAE1BUF9DVVNUT00xAEFERDEAbmUxID09IG5lMTEAbmUxID09IG5lMDEAbmUwID09IG5lMDEAdCA9PSAwIHx8IHQgPT0gMQBtYXNrLT5uZVszXSA9PSAxAGItPm5lWzNdID09IDEAbm9kZS0+c3JjWzFdLT5uZVszXSA9PSAxAG5vZGUtPnNyY1swXS0+bmVbM10gPT0gMQBtYXNrLT5uZVsyXSA9PSAxAG5vZGUtPnNyY1sxXS0+bmVbMl0gPT0gMQBuZWMxMSA9PSAxAG5lYjExID09IDEAbmUwMSA9PSAxAG5lMCA9PSAxAGdnbWxfYmxja19zaXplKHJlc3VsdF90eXBlKSA9PSAxAGswID09IHMwAG5lMDAgPT0gbmUwAG5fZGltcyA8PSBuZTAAbmUwID09IG5lYTAAcThfMABxNV8wAHE0XzAAbmUxID09IG5lMTAAbmUwID09IG5lMDAAY3BsYW4tPm5fdGhyZWFkcyA+IDAAbl9wYXN0ID49IDAAUCA+PSAwAHBhcmFtcy0+aXRoID09IDAAcmMgPT0gMAAoKHVpbnRwdHJfdCkgKG1lbV9idWZmZXIgKyBvYmpfbmV3LT5vZmZzKSklR0dNTF9NRU1fQUxJR04gPT0gMAAoKHVpbnRwdHJfdCkgKGN0eC0+bWVtX2J1ZmZlcikpJUdHTUxfTUVNX0FMSUdOID09IDAAbl9kaW1zICUgMiA9PSAwAHAwID09IDAAVGhlIGV4cHJlc3Npb24gY29udGFpbmVkIG1pc21hdGNoZWQgeyBhbmQgfS4AZW5jb2Rlci5ibG9ja3MuAGRlY29kZXIuYmxvY2tzLgBUaGUgZXhwcmVzc2lvbiBjb250YWluZWQgYW4gaW52YWxpZCBjaGFyYWN0ZXIgcmFuZ2UsIHN1Y2ggYXMgW2ItYV0gaW4gbW9zdCBlbmNvZGluZ3MuAEFuIGVtcHR5IHJlZ2V4IGlzIG5vdCBhbGxvd2VkIGluIHRoZSBQT1NJWCBncmFtbWFyLgBUaGUgZXhwcmVzc2lvbiBjb250YWluZWQgYW4gaW52YWxpZCByYW5nZSBpbiBhIHt9IGV4cHJlc3Npb24uAFRoZSBwYXJzZXIgZGlkIG5vdCBjb25zdW1lIHRoZSBlbnRpcmUgcmVndWxhciBleHByZXNzaW9uLgBPbmUgb2YgKj8reyB3YXMgbm90IHByZWNlZGVkIGJ5IGEgdmFsaWQgcmVndWxhciBleHByZXNzaW9uLgBUaGUgY29tcGxleGl0eSBvZiBhbiBhdHRlbXB0ZWQgbWF0Y2ggYWdhaW5zdCBhIHJlZ3VsYXIgZXhwcmVzc2lvbiBleGNlZWRlZCBhIHByZS1zZXQgbGV2ZWwuAFRoZSBleHByZXNzaW9uIGNvbnRhaW5lZCBhbiBpbnZhbGlkIGVzY2FwZWQgY2hhcmFjdGVyLCBvciBhIHRyYWlsaW5nIGVzY2FwZS4AVGhlcmUgd2FzIGluc3VmZmljaWVudCBtZW1vcnkgdG8gY29udmVydCB0aGUgZXhwcmVzc2lvbiBpbnRvIGEgZmluaXRlIHN0YXRlIG1hY2hpbmUuAFRoZSBleHByZXNzaW9uIGNvbnRhaW5lZCBhbiBpbnZhbGlkIGNvbGxhdGluZyBlbGVtZW50IG5hbWUuAFRoZSBleHByZXNzaW9uIGNvbnRhaW5lZCBhbiBpbnZhbGlkIGNoYXJhY3RlciBjbGFzcyBuYW1lLgBUaGVyZSB3YXMgaW5zdWZmaWNpZW50IG1lbW9yeSB0byBkZXRlcm1pbmUgd2hldGhlciB0aGUgcmVndWxhciBleHByZXNzaW9uIGNvdWxkIG1hdGNoIHRoZSBzcGVjaWZpZWQgY2hhcmFjdGVyIHNlcXVlbmNlLgBUaGUgZXhwcmVzc2lvbiBjb250YWluZWQgYW4gaW52YWxpZCBiYWNrIHJlZmVyZW5jZS4AQW4gaW52YWxpZCByZWdleCBncmFtbWFyIGhhcyBiZWVuIHJlcXVlc3RlZC4AVGhlIGV4cHJlc3Npb24gY29udGFpbmVkIG1pc21hdGNoZWQgWyBhbmQgXS4AVGhlIGV4cHJlc3Npb24gY29udGFpbmVkIG1pc21hdGNoZWQgKCBhbmQgKS4ALS0tAHcrACdzfCd0fCdyZXwndmV8J218J2xsfCdkfCA/W1s6YWxwaGE6XV0rfCA/W1s6ZGlnaXQ6XV0rfCA/W15cc1s6YWxwaGE6XVs6ZGlnaXQ6XV0rfFxzKyg/IVxTKXxccysAcisAYSsAJXMgKGNvcHkpACVzICh2aWV3KQBnZ21sX2JhY2tlbmRfaXNfY3B1KGJhY2tlbmRfY3B1KQBnZ21sX2lzX2NvbnRpZ3VvdXMoZHN0KQBvZmZzZXQgKyBpbTAqbmIwICsgaW0xKm5iMSArIGltMipuYjIgKyBpbTMqbmIzIDw9IGdnbWxfbmJ5dGVzKGRzdCkAb2Zmc2V0ICsgKG5lMTAgPT0gMCA/IDAgOiBuZTEwLTEpKm5iMCArIChuZTExID09IDAgPyAwIDogbmUxMS0xKSpuYjEgKyAobmUxMiA9PSAwID8gMCA6IG5lMTItMSkqbmIyICsgKG5lMTMgPT0gMCA/IDAgOiBuZTEzLTEpKm5iMyA8IGdnbWxfbmJ5dGVzKGRzdCkAZ2dtbF9pc19zY2FsYXIoZHN0KQBnZ21sX2lzX2NvbnRpZ3VvdXNfZXhjZXB0X2RpbV8xKGRzdCkAZ2dtbF9hcmVfc2FtZV9zaGFwZShzcmMxLCBkc3QpAGdnbWxfY2FuX3JlcGVhdChzcmMwLCBkc3QpAGdnbWxfYXJlX3NhbWVfc2hhcGUoc3JjMCwgc3JjMSkgJiYgZ2dtbF9hcmVfc2FtZV9zaGFwZShzcmMwLCBkc3QpAGdnbWxfY2FuX3JlcGVhdChzcmMxLCBzcmMwKSAmJiBnZ21sX2FyZV9zYW1lX3NoYXBlKHNyYzAsIGRzdCkAJXMgKGNvbnQpAGRzdC0+bmJbMF0gPT0gc2l6ZW9mKGZsb2F0KQBzcmMwLT5uYlswXSA9PSBzaXplb2YoZmxvYXQpAG5idjAgPT0gc2l6ZW9mKGZsb2F0KQBuYnEwID09IHNpemVvZihmbG9hdCkAbmJrMCA9PSBzaXplb2YoZmxvYXQpAG5iMCA9PSBzaXplb2YoZmxvYXQpAG5iYzEwID09IHNpemVvZihmbG9hdCkAbmIxMCA9PSBzaXplb2YoZmxvYXQpAG5iYjEwID09IHNpemVvZihmbG9hdCkAbmIwMCA9PSBzaXplb2YoZmxvYXQpAHNyYzAtPm5iWzBdID09IHNpemVvZihnZ21sX2ZwMTZfdCkAbmJ2MCA9PSBzaXplb2YoZ2dtbF9mcDE2X3QpAG5icTAgPT0gc2l6ZW9mKGdnbWxfZnAxNl90KQBuYmswID09IHNpemVvZihnZ21sX2ZwMTZfdCkAbmIwID09IHNpemVvZihnZ21sX2ZwMTZfdCkAbmJhMCA9PSBzaXplb2YoZ2dtbF9mcDE2X3QpAG5iYzAwID09IHNpemVvZihnZ21sX2ZwMTZfdCkAbmIwMCA9PSBzaXplb2YoZ2dtbF9mcDE2X3QpAG5iYjAwID09IHNpemVvZihnZ21sX2ZwMTZfdCkAJXMgKGNvcHkgb2YgJXMpACFnZ21sX2lzX3ZpZXcodGVuc29yKQAobnVsbCkAZ2dtbF9pc19jb250aWd1b3VzKG1hc2spAG5iMTAgPT0gZ2dtbF90eXBlX3NpemUoc3JjMS0+dHlwZSkAZ2dtbF9pc19xdWFudGl6ZWQoc3JjMC0+dHlwZSkAbmIwMCA9PSBnZ21sX3R5cGVfc2l6ZSh0eXBlKQAlcyAocGVybXV0ZWQpACVzICh0cmFuc3Bvc2VkKQAlcyAocmVzaGFwZWQpAGdnbWxfaXNfY29udGlndW91c19leGNlcHRfZGltXzEoZ3JhZCkAZ2dtbF9hcmVfc2FtZV9zaGFwZShzcmMwLCBncmFkKQB2aWV3X3NyYyA9PSBOVUxMIHx8IGRhdGFfc2l6ZSArIHZpZXdfb2ZmcyA8PSBnZ21sX25ieXRlcyh2aWV3X3NyYykAcGFyYW1zLT53c2l6ZSA+PSBzaXplb2YoZmxvYXQpICogKG50aCArIG50aCAqIG5jKQBnZ21sX25lbGVtZW50cyhhKSA9PSBnZ21sX25lbGVtZW50cyhiKQBnZ21sX2lzX3NjYWxhcihiKQBnZ21sX2Nhbl9tdWxfbWF0KGEsIGIpAGdnbWxfYXJlX3NhbWVfc2hhcGUoYSwgYikAZ2dtbF9pc19jb250aWd1b3VzKGEpACFnZ21sX2lzX3RyYW5zcG9zZWQoYSkAZ2dtbF9pc19wYWRkZWRfMWQoYSkAZ2dtbF9jYW5fcmVwZWF0X3Jvd3MobWFzaywgYSkAZ2dtbF9jYW5fcmVwZWF0KGIsIGEpAGdnbWxfaXNfY29udGlndW91cyhzcmMxKQBnZ21sX2lzX3NjYWxhcihzcmMxKQBnZ21sX2FyZV9zYW1lX3NoYXBlKHNyYzAsIGRzdCkgJiYgZ2dtbF9hcmVfc2FtZV9zaGFwZShzcmMwLCBzcmMxKQBnZ21sX2lzX2NvbnRpZ3VvdXMob3B0MCkAZ2dtbF9pc19jb250aWd1b3VzKGRzdCkgJiYgZ2dtbF9pc19jb250aWd1b3VzKHNyYzApAGdnbWxfbmVsZW1lbnRzKGRzdCkgPT0gZ2dtbF9uZWxlbWVudHMoc3JjMCkAb2Zmc2V0ICsgKG5lMTAgPT0gMCA/IDAgOiBuZTEwLTEpKm5iMDAgKyAobmUxMSA9PSAwID8gMCA6IG5lMTEtMSkqbmIwMSArIChuZTEyID09IDAgPyAwIDogbmUxMi0xKSpuYjAyICsgKG5lMTMgPT0gMCA/IDAgOiBuZTEzLTEpKm5iMDMgPCBnZ21sX25ieXRlcyhzcmMwKQBnZ21sX2lzX2NvbnRpZ3VvdXNfZXhjZXB0X2RpbV8xKHNyYzApAGdnbWxfY2FuX3JlcGVhdChkc3QsIHNyYzApACkpKQBvcGVyYXRvcigpACgoKABzcmMxLT50eXBlID09IEdHTUxfVFlQRV9GMzIgJiYgIm9ubHkgZjMyIHNyYzEgc3VwcG9ydGVkIGZvciBub3ciAHRlbnNvci0+YnVmZmVyICE9IE5VTEwgJiYgInRlbnNvciBidWZmZXIgbm90IHNldCIAYWxsb2MtPm5fZnJlZV9ibG9ja3MgPCBNQVhfRlJFRV9CTE9DS1MgJiYgIm91dCBvZiBmcmVlIGJsb2NrcyIAb2Zmc2V0ICsgc2l6ZSA8PSBnZ21sX25ieXRlcyh0ZW5zb3IpICYmICJ0ZW5zb3Igd3JpdGUgb3V0IG9mIGJvdW5kcyIAb2Zmc2V0ICsgc2l6ZSA8PSBnZ21sX25ieXRlcyh0ZW5zb3IpICYmICJ0ZW5zb3IgcmVhZCBvdXQgb2YgYm91bmRzIgBkYXRhICE9IE5VTEwgJiYgImZhaWxlZCB0byBhbGxvY2F0ZSBidWZmZXIiACEibm90IGVub3VnaCBzcGFjZSBpbiB0aGUgYnVmZmVyIgB0ZW5zb3ItPmRhdGEgIT0gTlVMTCAmJiAidGVuc29yIG5vdCBhbGxvY2F0ZWQiAGJhc2UgIT0gTlVMTCAmJiAiYmFja2VuZCBidWZmZXIgYmFzZSBjYW5ub3QgYmUgTlVMTCIAUHVyZSB2aXJ0dWFsIGZ1bmN0aW9uIGNhbGxlZCEAV0FSTklORzogQmVoYXZpb3IgbWF5IGJlIHVuZXhwZWN0ZWQgd2hlbiBhbGxvY2F0aW5nIDAgYnl0ZXMgZm9yIGdnbWxfYWxpZ25lZF9tYWxsb2MhACB8IABBVlggPSAAVlNYID0gAEJMQVMgPSAAT1BFTlZJTk8gPSAATkVPTiA9IABDT1JFTUwgPSAATUVUQUwgPSAAV0FTTV9TSU1EID0gAEYxNkMgPSAARlAxNl9WQSA9IABBUk1fRk1BID0gAENVREEgPSAAU1NTRTMgPSAAQVZYMiA9IABBVlg1MTIgPSAAJXM6IG9wIG5vdCBpbXBsZW1lbnRlZDogACVzOiB0ZW5zb3IgJyVzJyBoYXMgd3Jvbmcgc2l6ZSBpbiBtb2RlbCBmaWxlOiBnb3QgJXp1LCBleHBlY3RlZCAlenUKACVzOiB1c2luZyBkaXN0aWxsZWQgbW9kZWwgLSBmb3JjaW5nIG5vX3RpbWVzdGFtcHMKACVzOiBhZGRpbmcgJWQgZXh0cmEgdG9rZW5zCgAlczogICAgICBtZWwgdGltZSA9ICU4LjJmIG1zCgAlczogICAgdG90YWwgdGltZSA9ICU4LjJmIG1zCgAlczogICAgIGxvYWQgdGltZSA9ICU4LjJmIG1zCgBzeXN0ZW1faW5mbzogbl90aHJlYWRzID0gJWQgLyAlZCB8ICVzCgBXSElTUEVSX0FTU0VSVDogJXM6JWQ6ICVzCgBHR01MX0FTU0VSVDogJXM6JWQ6ICVzCgBbJXMgLS0+ICVzXSAgJXMKACVzOiBvZmZzZXQgJWRtcyBpcyBiZWZvcmUgdGhlIHN0YXJ0IG9mIHRoZSBhdWRpbwoAdW5rbm93biB0b2tlbgoAJXM6IGZhaWxlZCB0byBjb21wdXRlIGxvZyBtZWwgc3BlY3Ryb2dyYW0KACVzOiBsb2FkaW5nIG1vZGVsCgAlczogZmFpbGVkIHRvIGxvYWQgbW9kZWwKACVzOiBmYWlsZWQgdG8gZXZhbAoAJXM6ICAgICBmYWxsYmFja3MgPSAlM2QgcCAvICUzZCBoCgAlczogV0FSTiBubyB0ZW5zb3JzIGxvYWRlZCBmcm9tIG1vZGVsIGZpbGUgLSBhc3N1bWluZyBlbXB0eSBtb2RlbCBmb3IgdGVzdGluZwoAJXM6IGVuY29kZXJfYmVnaW5fY2FsbGJhY2sgcmV0dXJuZWQgZmFsc2UgLSBhYm9ydGluZwoAJXM6IHRlbnNvciAnJXMnIGhhcyB3cm9uZyBzaXplIGluIG1vZGVsIGZpbGUKACVzOiB1bmtub3duIHRlbnNvciAnJXMnIGluIG1vZGVsIGZpbGUKACVzOiBubyBzaWduYWwgZGF0YSBhdmFpbGFibGUKACVzOiBmYWlsZWQgdG8gYWxsb2NhdGUgbWVtb3J5IGZvciBrdiBjYWNoZQoAJXM6IGt2X2NhY2hlX2luaXQoKSBmYWlsZWQgZm9yIGNyb3NzLWF0dGVudGlvbiBjYWNoZQoAJXM6IGt2X2NhY2hlX2luaXQoKSBmYWlsZWQgZm9yIHNlbGYtYXR0ZW50aW9uIGNhY2hlCgAlczogZmFpbGVkIHRvIGF1dG8tZGV0ZWN0IGxhbmd1YWdlCgAlczogZmFpbGVkIHRvIGVuY29kZQoAJXM6IGZhaWxlZCB0byBkZWNvZGUKACVzOiBnZ21sX2luaXQoKSBmYWlsZWQKACVzOiBuX3Rva2Vucz0lZCA+IG5fY3R4PSVkCgAlczogRVJST1Igbm90IGFsbCB0ZW5zb3JzIGxvYWRlZCBmcm9tIG1vZGVsIGZpbGUgLSBleHBlY3RlZCAlenUsIGdvdCAlZAoAJXM6IHVua25vd24gbGFuZ3VhZ2UgaWQgJWQKACVzOiB0b28gbWFueSBkZWNvZGVycyByZXF1ZXN0ZWQgKCVkKSwgbWF4ID0gJWQKACVzOiBuX2F1ZGlvX2xheWVyID0gJWQKACVzOiBuX2F1ZGlvX3N0YXRlID0gJWQKACVzOiBuX3RleHRfbGF5ZXIgID0gJWQKACVzOiBuX3RleHRfc3RhdGUgID0gJWQKACVzOiBuX2F1ZGlvX2hlYWQgID0gJWQKACVzOiBuX2F1ZGlvX2N0eCAgID0gJWQKACVzOiBuX3RleHRfaGVhZCAgID0gJWQKACVzOiBuX3RleHRfY3R4ICAgID0gJWQKACVzOiBuX2xhbmdzICAgICAgID0gJWQKACVzOiBuX3ZvY2FiICAgICAgID0gJWQKACVzOiBuX21lbHMgICAgICAgID0gJWQKACVzOiBxbnR2ciAgICAgICAgID0gJWQKACVzOiBmdHlwZSAgICAgICAgID0gJWQKACVzOiB0ZW5zb3IgJyVzJyBoYXMgd3Jvbmcgc2hhcGUgaW4gbW9kZWwgZmlsZTogZ290IFslZCwgJWQsICVkXSwgZXhwZWN0ZWQgWyVkLCAlZCwgJWRdCgAlczogc2hhcGU6IFslZCwgJWQsICVkXSwgZXhwZWN0ZWQ6IFslZCwgJWQsICVkXQoAJXM6ICU4cyBidWZmZXIgc2l6ZSA9ICU4LjJmIE1CCgAlczoga3YgY3Jvc3Mgc2l6ZSA9ICU3LjJmIE1CCgAlczogY29tcHV0ZSBidWZmZXIgKGVuY29kZSkgPSAlNy4yZiBNQgoAJXM6IGNvbXB1dGUgYnVmZmVyIChkZWNvZGUpID0gJTcuMmYgTUIKACVzOiBrdiBzZWxmIHNpemUgID0gJTcuMmYgTUIKACVzOiBjb21wdXRlIGJ1ZmZlciAoY3Jvc3MpICA9ICU3LjJmIE1CCgAlczogY29tcHV0ZSBidWZmZXIgKGNvbnYpICAgPSAlNy4yZiBNQgoAJXM6IG1vZGVsIHNpemUgICAgPSAlNy4yZiBNQgoAJXM6IHByb2Nlc3NpbmcgJWQgc2FtcGxlcywgJS4xZiBzZWMsICVkIHRocmVhZHMsICVkIHByb2Nlc3NvcnMsIGxhbmcgPSAlcywgdGFzayA9ICVzIC4uLgoAJXM6IG5vdCBlbm91Z2ggc3BhY2UgaW4gdGhlIGJ1ZmZlciAobmVlZGVkICV6dSwgbGFyZ2VzdCBibG9jayBhdmFpbGFibGUgJXp1KQoAJXM6IG5vdCBlbm91Z2ggc3BhY2UgaW4gdGhlIGNvbnRleHQncyBtZW1vcnkgcG9vbCAobmVlZGVkICV6dSwgYXZhaWxhYmxlICV6dSkKACVzOiBub3QgZW5vdWdoIHNwYWNlIGluIHRoZSBzY3JhdGNoIG1lbW9yeSBwb29sIChuZWVkZWQgJXp1LCBhdmFpbGFibGUgJXp1KQoAJXM6IG9mZnNldCAlZG1zIGlzIHBhc3QgdGhlIGVuZCBvZiB0aGUgYXVkaW8gKCVkbXMpCgAlczogdHlwZSAgICAgICAgICA9ICVkICglcyVzKQoAJXM6ICAgcHJvbXB0IHRpbWUgPSAlOC4yZiBtcyAvICU1ZCBydW5zICglOC4yZiBtcyBwZXIgcnVuKQoAJXM6ICAgc2FtcGxlIHRpbWUgPSAlOC4yZiBtcyAvICU1ZCBydW5zICglOC4yZiBtcyBwZXIgcnVuKQoAJXM6ICAgZW5jb2RlIHRpbWUgPSAlOC4yZiBtcyAvICU1ZCBydW5zICglOC4yZiBtcyBwZXIgcnVuKQoAJXM6ICAgZGVjb2RlIHRpbWUgPSAlOC4yZiBtcyAvICU1ZCBydW5zICglOC4yZiBtcyBwZXIgcnVuKQoAJXM6ICAgYmF0Y2hkIHRpbWUgPSAlOC4yZiBtcyAvICU1ZCBydW5zICglOC4yZiBtcyBwZXIgcnVuKQoAJXM6IGF1dG8tZGV0ZWN0ZWQgbGFuZ3VhZ2U6ICVzIChwID0gJWYpCgAlczogdG9vIG1hbnkgcmVzdWx0aW5nIHRva2VuczogJWQgKG1heCAlZCkKACVzOiBpbnZhbGlkIG1vZGVsIChiYWQgZnR5cGUgdmFsdWUgJWQpCgAlczogYXVkaW9fY3R4IGlzIGxhcmdlciB0aGFuIHRoZSBtYXhpbXVtIGFsbG93ZWQgKCVkID4gJWQpCgAlczogaW52YWxpZCBtb2RlbCBkYXRhIChiYWQgbWFnaWMpCgAlczogJXMgKGF0dGVtcHRlZCB0byBhbGxvY2F0ZSAlNi4yZiBNQikKACVzOiBmYWlsZWQgdG8gb3BlbiAnJXMnCgAlczogbG9hZGluZyBtb2RlbCBmcm9tICclcycKACVzOiB1bmtub3duIGxhbmd1YWdlICclcycKAAAANNkAACw9AABOU3QzX18yMTJiYXNpY19zdHJpbmdJY05TXzExY2hhcl90cmFpdHNJY0VFTlNfOWFsbG9jYXRvckljRUVFRQAA4NkAAOw8AABpaWkAKNkAAHZpAAB82QAAZD0AACw9AAA02QAATjEwZW1zY3JpcHRlbjN2YWxFAADg2QAAUD0AAGlpaWlpAAAAZD0AAKDZAAB82QAAKNkAAGQ9AAAAAAAAAAAAABAQEBAQEBAQABAQEBAQEBAQABAQEBAQEAAAEBAQEBAQEBAAEBAQEBAAEAAQEBAQEBAAABAQEBAQAAAAEBAQEBAQEBAAEBAQEAAQEAAQEBAQEAAQABAQEBAAABAAEBAQEBAQAAAQEBAQABAAABAQEBAQAAAAEBAQEAAAAAAQEBAQEBAQEAAQEBAAEBAQABAQEBAAEBAAEBAQAAAQEAAQEBAQEAAQABAQEAAQABAAEBAQEAAAEAAQEBAAAAAQABAQEBAQEAAAEBAQABAQAAAQEBAQABAAABAQEAAAEAAAEBAQEBAAAAAQEBAAEAAAABAQEBAAAAAAEBAQAAAAAAAQEBAQEBAQEAAQEAAQEBAQABAQEAAQEBAAEBAAABAQEAAQEBAQABAQABAQABAAEBAAEBAQAAAQEAAQEAAAABAQABAQEBAQABAAEBAAEBAAEAAQEBAAEAAQABAQAAAQABAAEBAQEAAAEAAQEAAQAAAQABAQEAAAABAAEBAAAAAAEAAQEBAQEBAAABAQABAQEAAAEBAQABAQAAAQEAAAEBAAABAQEBAAEAAAEBAAEAAQAAAQEBAAABAAABAQAAAAEAAAEBAQEBAAAAAQEAAQEAAAABAQEAAQAAAAEBAAABAAAAAQEBAQAAAAABAQABAAAAAAEBAQAAAAAAAQEAAAAAAAABAQEBAQEBAQABAAEBAQEBAAEBAAEBAQEAAQAAAQEBAQABAQEAAQEBAAEAAQABAQEAAQEAAAEBAQABAAAAAQEBAAEBAQEAAQEAAQABAQABAQABAQABAAEBAAEAAAEAAQEAAQEBAAABAQABAAEAAAEBAAEBAAAAAQEAAQAAAAABAQABAQEBAQABAAEAAQEBAAEAAQEAAQEAAQABAAABAQABAAEBAQABAAEAAQABAAEAAQABAQAAAQABAAEAAAABAAEAAQEBAQAAAQABAAEBAAABAAEBAAEAAAEAAQAAAQAAAQABAQEAAAABAAEAAQAAAAEAAQEAAAAAAQABAAAAAAABAAEBAQEBAQAAAQABAQEBAAABAQABAQEAAAEAAAEBAQAAAQEBAAEBAAABAAEAAQEAAAEBAAABAQAAAQAAAAEBAAABAQEBAAEAAAEAAQEAAQAAAQEAAQABAAABAAABAAEAAAEBAQAAAQAAAQABAAABAAABAQAAAAEAAAEAAAAAAQAAAQEBAQEAAAABAAEBAQAAAAEBAAEBAAAAAQAAAQEAAAABAQEAAQAAAAEAAQABAAAAAQEAAAEAAAABAAAAAQAAAAEBAQEAAAAAAQABAQAAAAABAQABAAAAAAEAAAEAAAAAAQEBAAAAAAABAAEAAAAAAAEBAAAAAAAAAQAAAAAAAAABAQEBAQEBAQAAAQEBAQEBAAEAAQEBAQEAAAABAQEBAQABAQABAQEBAAABAAEBAQEAAQAAAQEBAQAAAAABAQEBAAEBAQABAQEAAAEBAAEBAQABAAEAAQEBAAAAAQABAQEAAQEAAAEBAQAAAQAAAQEBAAEAAAABAQEAAAAAAAEBAQABAQEBAAEBAAABAQEAAQEAAQABAQABAQAAAAEBAAEBAAEBAAEAAQEAAAEAAQABAQABAAABAAEBAAAAAAEAAQEAAQEBAAABAQAAAQEAAAEBAAEAAQAAAQEAAAABAAABAQABAQAAAAEBAAABAAAAAQEAAQAAAAABAQAAAAAAAAEBAAEBAQEBAAEAAAEBAQEAAQABAAEBAQABAAAAAQEBAAEAAQEAAQEAAQAAAQABAQABAAEAAAEBAAEAAAAAAQEAAQABAQEAAQABAAABAQABAAEAAQABAAEAAQAAAAEAAQABAAEBAAABAAEAAAEAAAEAAQABAAAAAQABAAAAAAABAAEAAQEBAQAAAQAAAQEBAAABAAEAAQEAAAEAAAABAQAAAQABAQABAAABAAABAAEAAAEAAQAAAQAAAQAAAAABAAABAAEBAQAAAAEAAAEBAAAAAQABAAEAAAABAAAAAQAAAAEAAQEAAAAAAQAAAQAAAAABAAEAAAAAAAEAAAAAAAAAAQABAQEBAQEAAAABAQEBAQAAAQABAQEBAAAAAAEBAQEAAAEBAAEBAQAAAAEAAQEBAAABAAABAQEAAAAAAAEBAQAAAQEBAAEBAAAAAQEAAQEAAAEAAQABAQAAAAABAAEBAAABAQAAAQEAAAABAAABAQAAAQAAAAEBAAAAAAAAAQEAAAEBAQEAAQAAAAEBAQABAAABAAEBAAEAAAAAAQEAAQAAAQEAAQABAAAAAQABAAEAAAEAAAEAAQAAAAAAAQABAAABAQEAAAEAAAABAQAAAQAAAQABAAABAAAAAAEAAAEAAAEBAAAAAQAAAAEAAAABAAABAAAAAAEAAAAAAAAAAQAAAQEBAQEAAAAAAQEBAQAAAAEAAQEBAAAAAAABAQEAAAABAQABAQAAAAABAAEBAAAAAQAAAQEAAAAAAAABAQAAAAEBAQABAAAAAAEBAAEAAAABAAEAAQAAAAAAAQABAAAAAQEAAAEAAAAAAQAAAQAAAAEAAAABAAAAAAAAAAEAAAABAQEBAAAAAAABAQEAAAAAAQABAQAAAAAAAAEBAAAAAAEBAAEAAAAAAAEAAQAAAAABAAABAAAAAAAAAAEAAAAAAQEBAAAAAAAAAQEAAAAAAAEAAQAAAAAAAAABAAAAAAABAQAAAAAAAAABAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAEBAAAAAAAAAAABAAAAAAABAAEAAAAAAAABAQAAAAAAAQEBAAAAAAAAAAABAAAAAAEAAAEAAAAAAAEAAQAAAAABAQABAAAAAAAAAQEAAAAAAQABAQAAAAAAAQEBAAAAAAEBAQEAAAAAAAAAAAEAAAABAAAAAQAAAAABAAABAAAAAQEAAAEAAAAAAAEAAQAAAAEAAQABAAAAAAEBAAEAAAABAQEAAQAAAAAAAAEBAAAAAQAAAQEAAAAAAQABAQAAAAEBAAEBAAAAAAABAQEAAAABAAEBAQAAAAABAQEBAAAAAQEBAQEAAAAAAAAAAAEAAAEAAAAAAQAAAAEAAAABAAABAQAAAAEAAAAAAQAAAQAAAQABAAABAAAAAQEAAAEAAAEBAQAAAQAAAAAAAQABAAABAAABAAEAAAABAAEAAQAAAQEAAQABAAAAAAEBAAEAAAEAAQEAAQAAAAEBAQABAAABAQEBAAEAAAAAAAABAQAAAQAAAAEBAAAAAQAAAQEAAAEBAAABAQAAAAABAAEBAAABAAEAAQEAAAABAQABAQAAAQEBAAEBAAAAAAABAQEAAAEAAAEBAQAAAAEAAQEBAAABAQABAQEAAAAAAQEBAQAAAQABAQEBAAAAAQEBAQEAAAEBAQEBAQAAAAAAAAAAAQABAAAAAAABAAABAAAAAAEAAQEAAAAAAQAAAAEAAAABAAEAAQAAAAEAAAEBAAAAAQABAQEAAAABAAAAAAEAAAEAAQAAAQAAAQAAAQABAAABAAEBAAEAAAEAAAABAQAAAQABAAEBAAABAAABAQEAAAEAAQEBAQAAAQAAAAAAAQABAAEAAAABAAEAAAEAAAEAAQABAQAAAQABAAAAAQABAAEAAQABAAEAAQAAAQEAAQABAAEBAQABAAEAAAAAAQEAAQABAAABAQABAAABAAEBAAEAAQEAAQEAAQAAAAEBAQABAAEAAQEBAAEAAAEBAQEAAQABAQEBAQABAAAAAAAAAQEAAQAAAAABAQAAAQAAAAEBAAEBAAAAAQEAAAABAAABAQABAAEAAAEBAAABAQAAAQEAAQEBAAABAQAAAAABAAEBAAEAAAEAAQEAAAEAAQABAQABAQABAAEBAAAAAQEAAQEAAQABAQABAQAAAQEBAAEBAAEBAQEAAQEAAAAAAAEBAQABAAAAAQEBAAABAAABAQEAAQEAAAEBAQAAAAEAAQEBAAEAAQABAQEAAAEBAAEBAQABAQEAAQEBAAAAAAEBAQEAAQAAAQEBAQAAAQABAQEBAAEBAAEBAQEAAAABAQEBAQABAAEBAQEBAAABAQEBAQEAAQEBAQEBAQAAAAAAAAAAAQEAAAAAAAABAAEAAAAAAAEBAQAAAAAAAQAAAQAAAAABAQABAAAAAAEAAQEAAAAAAQEBAQAAAAABAAAAAQAAAAEBAAABAAAAAQABAAEAAAABAQEAAQAAAAEAAAEBAAAAAQEAAQEAAAABAAEBAQAAAAEBAQEBAAAAAQAAAAABAAABAQAAAAEAAAEAAQAAAQAAAQEBAAABAAABAAABAAEAAAEBAAEAAQAAAQABAQABAAABAQEBAAEAAAEAAAABAQAAAQEAAAEBAAABAAEAAQEAAAEBAQABAQAAAQAAAQEBAAABAQABAQEAAAEAAQEBAQAAAQEBAQEBAAABAAAAAAABAAEBAAAAAAEAAQABAAAAAQABAQEAAAABAAEAAAEAAAEAAQEAAQAAAQABAAEBAAABAAEBAQEAAAEAAQAAAAEAAQABAQAAAQABAAEAAQABAAEAAQEBAAEAAQABAAABAQABAAEBAAEBAAEAAQABAQEAAQABAQEBAQABAAEAAAAAAQEAAQEAAAABAQABAAEAAAEBAAEBAQAAAQEAAQAAAQABAQABAQABAAEBAAEAAQEAAQEAAQEBAQABAQABAAAAAQEBAAEBAAABAQEAAQABAAEBAQABAQEAAQEBAAEAAAEBAQEAAQEAAQEBAQABAAEBAQEBAAEBAQEBAQEAAQAAAAAAAAEBAQAAAAAAAQEAAQAAAAABAQEBAAAAAAEBAAABAAAAAQEBAAEAAAABAQABAQAAAAEBAQEBAAAAAQEAAAABAAABAQEAAAEAAAEBAAEAAQAAAQEBAQABAAABAQAAAQEAAAEBAQABAQAAAQEAAQEBAAABAQEBAQEAAAEBAAAAAAEAAQEBAAAAAQABAQABAAABAAEBAQEAAAEAAQEAAAEAAQABAQEAAQABAAEBAAEBAAEAAQEBAQEAAQABAQAAAAEBAAEBAQAAAQEAAQEAAQABAQABAQEBAAEBAAEBAAABAQEAAQEBAAEBAQABAQABAQEBAAEBAQEBAQEAAQEAAAAAAAEBAQEAAAAAAQEBAAEAAAABAQEBAQAAAAEBAQAAAQAAAQEBAQABAAABAQEAAQEAAAEBAQEBAQAAAQEBAAAAAQABAQEBAAABAAEBAQABAAEAAQEBAQEAAQABAQEAAAEBAAEBAQEAAQEAAQEBAAEBAQABAQEBAQEBAAEBAQAAAAABAQEBAQAAAAEBAQEAAQAAAQEBAQEBAAABAQEBAAABAAEBAQEBAAEAAQEBAQABAQABAQEBAQEBAAEBAQEAAAABAQEBAQEAAAEBAQEBAAEAAQEBAQEBAQABAQEBAQAAAQEBAQEBAQABAQEBAQEAAQEBAQEBAQEBAQEBAQEBDtHQAAAQAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAAAAAC5HAAAAQAAAAIAAAAAAAAADAAAAA0AAAANAAAADgAAAAEAAAA4IAAAIAAAABIAAAABAAAADwAAABAAAAARAAAAEgAAAAgAAADvHgAAIAAAABQAAAABAAAAEwAAABQAAAAVAAAAFgAAAAkAAABUGQEGEwAAAFQZAfUCEwAAADMgAAAgAAAAFgAAAAEAAAAXAAAAGAAAABkAAAAaAAAACAAAAOoeAAAgAAAAGAAAAAEAAAAbAAAAHAAAAB0AAAAeAAAACQAAAC4gAAAgAAAAIgAAAAEAAAAfAAAAIAAAACEAAAAiAAAACAAAAOUeAAAgAAAAKAAAAAEAAAAAAAAAIwAAACQAAAAAAAAACQAAAG4YAAAAAQAAVAAAAAEAAAAlAAAAJgAAACcAAAAoAAAADwAAAGkYAAAAAQAAbgAAAAEAAAApAAAAKgAAACsAAAAsAAAADwAAAGQYAAAAAQAAkAAAAAEAAAAtAAAALgAAAC8AAAAwAAAADwAAAF8YAAAAAQAAsAAAAAEAAAAxAAAAMgAAADMAAAA0AAAADwAAAFoYAAAAAQAA0gAAAAEAAAA1AAAANgAAADcAAAA4AAAADwAAAFUYAAAAAQAAJAEAAAEAAAAAAAAAOQAAAAAAAAAAAAAAAAAAAKMcAAABAAAAAQEJtRwAAAEAAAACAQnpHQAAAQAAAAQBmJMBMxkAACYXAABfGQAAAB8AAOMZAADnGQAAnhcAAKQVAAAgFwAAtxUAAO8YAABmFwAAUxYAAE4XAACYFQAAPBYAAIsYAABDFgAAgRgAAHkXAABqFwAAzRgAAHMXAAA0FgAASRkAAEAZAAA6GQAAMBYAAHAVAAArFgAAKxkAAJ8VAAAUGQAAHBkAAEoWAACXGAAA+BgAAP0YAAAwFwAAjxUAAHMYAAAmGQAA2xgAAOcYAAAqFwAAiRkAAKIXAABvGQAAgRkAAGcZAAA4GQAAYxkAALwVAACsFQAAPxcAAAsZAAC9GAAAxBUAAM0VAABvFgAAexYAAF8VAABbFQAAZRUAAGEeAABRHgAAQR4AAPQeAADdHQAAPB0AAFwWAAClGAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAACAAAAAwAAAAUAAAALAAAAEQAAACUAAABDAAAAgwAAAAEBAAAJAgAABwQAAAUIAAADEAAAESAAABtAAAADgAAAAQABAB0AAgADAAQAFQAIAAcAEAARACAADwBAAAkAgAArAAABIwAAAg8AAAQdAAAIAwAAEAsAACADAABACwAAgAAAAAABAAAAAgAAAAMAAAAAAAAAAAAAAAAAAAAIAAAABgAAAAcAAAAKAAAACwAAAAwAAAANAAAADgAAAAAAAAA8AAAAQAAAAEEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEIAAABDAAAARAAAAEUAAABGAAAAOwAAADoAAAAAAAAARwAAAEgAAABJAAAASgAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAACAAAAAwAAAAQAAAABAAAAAQAAAAEAAAABAAAAAAAAAAAAAAAwUwAAWQAAAFoAAABbAAAAXAAAAE5TdDNfXzIxMV9fZW5kX3N0YXRlSWNFRQBOU3QzX18yNl9fbm9kZUljRUUA4NkAABVTAAAI2gAA/FIAAChTAAAAAAAA4FMAAF0AAABeAAAAXwAAAGAAAABhAAAATlN0M19fMjIwX19zaGFyZWRfcHRyX3BvaW50ZXJJUE5TXzEzX19lbXB0eV9zdGF0ZUljRUVOU18xMHNoYXJlZF9wdHJJUzJfRTI3X19zaGFyZWRfcHRyX2RlZmF1bHRfZGVsZXRlSVMyX1MyX0VFTlNfOWFsbG9jYXRvcklTMl9FRUVFAAAAAAjaAABYUwAAiNAAAE5TdDNfXzIxMHNoYXJlZF9wdHJJTlNfMTNfX2VtcHR5X3N0YXRlSWNFRUUyN19fc2hhcmVkX3B0cl9kZWZhdWx0X2RlbGV0ZUlTMl9TMl9FRQAAAAAAAADIVAAAYgAAAGMAAABkAAAAXAAAAE5TdDNfXzIxM19fZW1wdHlfc3RhdGVJY0VFAE5TdDNfXzIxNl9fb3duc19vbmVfc3RhdGVJY0VFAE5TdDNfXzIxNV9faGFzX29uZV9zdGF0ZUljRUUAAAAI2gAAkVQAAChTAAAI2gAAc1QAALBUAAAI2gAAWFQAALxUAAAAAAAAvFQAAGIAAABlAAAAZgAAAFwAAAAAAAAAKFUAAGIAAABnAAAAaAAAAFwAAABOU3QzX18yMjBfX2xfYW5jaG9yX211bHRpbGluZUljRUUAAAAI2gAABFUAALxUAAAAAAAAcFUAAGIAAABpAAAAagAAAFwAAABOU3QzX18yMjBfX3JfYW5jaG9yX211bHRpbGluZUljRUUAAAAI2gAATFUAALxUAAAAAAAAyFUAAGsAAABsAAAAbQAAAFwAAABOU3QzX18yMTVfX3dvcmRfYm91bmRhcnlJY05TXzEycmVnZXhfdHJhaXRzSWNFRUVFAAAACNoAAJRVAAC8VAAAAAAAABxWAABuAAAAbwAAAHAAAABcAAAATlN0M19fMjExX19sb29rYWhlYWRJY05TXzEycmVnZXhfdHJhaXRzSWNFRUVFAAAACNoAAOxVAAC8VAAAAAAAAGhWAABiAAAAcQAAAHIAAABcAAAATlN0M19fMjIzX19tYXRjaF9hbnlfYnV0X25ld2xpbmVJY0VFAAAAAAjaAABAVgAAvFQAAAAAAADEVgAAcwAAAHQAAAB1AAAAXAAAAE5TdDNfXzIxOF9fbWF0Y2hfY2hhcl9pY2FzZUljTlNfMTJyZWdleF90cmFpdHNJY0VFRUUAAAAACNoAAIxWAAC8VAAAAAAAACBXAAB2AAAAdwAAAHgAAABcAAAATlN0M19fMjIwX19tYXRjaF9jaGFyX2NvbGxhdGVJY05TXzEycmVnZXhfdHJhaXRzSWNFRUVFAAAI2gAA6FYAALxUAAAAAAAAYFcAAGIAAAB5AAAAegAAAFwAAABOU3QzX18yMTJfX21hdGNoX2NoYXJJY0VFAAAACNoAAERXAAC8VAAAAAAAALhXAAB7AAAAfAAAAH0AAABcAAAATlN0M19fMjE2X19iYWNrX3JlZl9pY2FzZUljTlNfMTJyZWdleF90cmFpdHNJY0VFRUUAAAjaAACEVwAAvFQAAAAAAAAUWAAAfgAAAH8AAACAAAAAXAAAAE5TdDNfXzIxOF9fYmFja19yZWZfY29sbGF0ZUljTlNfMTJyZWdleF90cmFpdHNJY0VFRUUAAAAACNoAANxXAAC8VAAAAAAAAFBYAABiAAAAgQAAAIIAAABcAAAATlN0M19fMjEwX19iYWNrX3JlZkljRUUACNoAADhYAAC8VAAAAAAAAKxYAACDAAAAhAAAAIUAAABcAAAATlN0M19fMjIwX19icmFja2V0X2V4cHJlc3Npb25JY05TXzEycmVnZXhfdHJhaXRzSWNFRUVFAAAI2gAAdFgAALxUAAAAAAAA/FgAAGIAAACGAAAAhwAAAFwAAABOU3QzX18yMjhfX2JlZ2luX21hcmtlZF9zdWJleHByZXNzaW9uSWNFRQAAAAjaAADQWAAAvFQAAAAAAABIWQAAYgAAAIgAAACJAAAAXAAAAE5TdDNfXzIyNl9fZW5kX21hcmtlZF9zdWJleHByZXNzaW9uSWNFRQAI2gAAIFkAALxUAAAAAAAArFkAAIoAAACLAAAAjAAAAI0AAABOU3QzX18yNl9fbG9vcEljRUUATlN0M19fMjE3X19vd25zX3R3b19zdGF0ZXNJY0VFAAAACNoAAH9ZAAC8VAAACNoAAGxZAACgWQAAAAAAAKBZAACKAAAAjgAAAGYAAABcAAAAAAAAAAhaAABZAAAAjwAAAJAAAABcAAAATlN0M19fMjE3X19yZXBlYXRfb25lX2xvb3BJY0VFAAAI2gAA6FkAALBUAAAAAAAASFoAAIoAAACRAAAAkgAAAJMAAABOU3QzX18yMTFfX2FsdGVybmF0ZUljRUUAAAAACNoAACxaAACgWQAAAAAAAJBaAABZAAAAlAAAAJUAAABcAAAATlN0M19fMjIxX19lbXB0eV9ub25fb3duX3N0YXRlSWNFRQAACNoAAGxaAACwVAAAAAAAANBaAABiAAAAlgAAAJcAAABcAAAATlN0M19fMjExX19tYXRjaF9hbnlJY0VFAAAAAAjaAAC0WgAAvFQAAAAAAACcWwAAmAAAAJkAAACaAAAAmwAAAJwAAACdAAAAngAAAJ8AAACgAAAATlN0M19fMjEwX19mdW5jdGlvbjZfX2Z1bmNJWjE4d2hpc3Blcl9pbml0X3N0YXRlRTMkXzBOU185YWxsb2NhdG9ySVMyX0VFRlAxMWdnbWxfY2dyYXBodkVFRQBOU3QzX18yMTBfX2Z1bmN0aW9uNl9fYmFzZUlGUDExZ2dtbF9jZ3JhcGh2RUVFAADg2QAAZFsAAAjaAAAIWwAAlFsAAFoxOHdoaXNwZXJfaW5pdF9zdGF0ZUUzJF8wAADg2QAAqFsAAAAAAABUXAAAmAAAAKEAAACiAAAAowAAAKQAAAClAAAApgAAAKcAAACoAAAATlN0M19fMjEwX19mdW5jdGlvbjZfX2Z1bmNJWjE4d2hpc3Blcl9pbml0X3N0YXRlRTMkXzFOU185YWxsb2NhdG9ySVMyX0VFRlAxMWdnbWxfY2dyYXBodkVFRQAI2gAA+FsAAJRbAABaMTh3aGlzcGVyX2luaXRfc3RhdGVFMyRfMQAA4NkAAGBcAAAAAAAADF0AAJgAAACpAAAAqgAAAKsAAACsAAAArQAAAK4AAACvAAAAsAAAAE5TdDNfXzIxMF9fZnVuY3Rpb242X19mdW5jSVoxOHdoaXNwZXJfaW5pdF9zdGF0ZUUzJF8yTlNfOWFsbG9jYXRvcklTMl9FRUZQMTFnZ21sX2NncmFwaHZFRUUACNoAALBcAACUWwAAWjE4d2hpc3Blcl9pbml0X3N0YXRlRTMkXzIAAODZAAAYXQAAAAAAAMRdAACYAAAAsQAAALIAAACzAAAAtAAAALUAAAC2AAAAtwAAALgAAABOU3QzX18yMTBfX2Z1bmN0aW9uNl9fZnVuY0laMTh3aGlzcGVyX2luaXRfc3RhdGVFMyRfM05TXzlhbGxvY2F0b3JJUzJfRUVGUDExZ2dtbF9jZ3JhcGh2RUVFAAjaAABoXQAAlFsAAFoxOHdoaXNwZXJfaW5pdF9zdGF0ZUUzJF8zAADg2QAA0F0AAE5TdDNfXzIxMmJhc2ljX3N0cmluZ0loTlNfMTFjaGFyX3RyYWl0c0loRUVOU185YWxsb2NhdG9ySWhFRUVFAADg2QAA9F0AAE5TdDNfXzIxMmJhc2ljX3N0cmluZ0l3TlNfMTFjaGFyX3RyYWl0c0l3RUVOU185YWxsb2NhdG9ySXdFRUVFAADg2QAAPF4AAE5TdDNfXzIxMmJhc2ljX3N0cmluZ0lEc05TXzExY2hhcl90cmFpdHNJRHNFRU5TXzlhbGxvY2F0b3JJRHNFRUVFAAAA4NkAAIReAABOU3QzX18yMTJiYXNpY19zdHJpbmdJRGlOU18xMWNoYXJfdHJhaXRzSURpRUVOU185YWxsb2NhdG9ySURpRUVFRQAAAODZAADQXgAATjEwZW1zY3JpcHRlbjExbWVtb3J5X3ZpZXdJY0VFAADg2QAAHF8AAE4xMGVtc2NyaXB0ZW4xMW1lbW9yeV92aWV3SWFFRQAA4NkAAERfAABOMTBlbXNjcmlwdGVuMTFtZW1vcnlfdmlld0loRUUAAODZAABsXwAATjEwZW1zY3JpcHRlbjExbWVtb3J5X3ZpZXdJc0VFAADg2QAAlF8AAE4xMGVtc2NyaXB0ZW4xMW1lbW9yeV92aWV3SXRFRQAA4NkAALxfAABOMTBlbXNjcmlwdGVuMTFtZW1vcnlfdmlld0lpRUUAAODZAADkXwAATjEwZW1zY3JpcHRlbjExbWVtb3J5X3ZpZXdJakVFAADg2QAADGAAAE4xMGVtc2NyaXB0ZW4xMW1lbW9yeV92aWV3SWxFRQAA4NkAADRgAABOMTBlbXNjcmlwdGVuMTFtZW1vcnlfdmlld0ltRUUAAODZAABcYAAATjEwZW1zY3JpcHRlbjExbWVtb3J5X3ZpZXdJeEVFAADg2QAAhGAAAE4xMGVtc2NyaXB0ZW4xMW1lbW9yeV92aWV3SXlFRQAA4NkAAKxgAABOMTBlbXNjcmlwdGVuMTFtZW1vcnlfdmlld0lmRUUAAODZAADUYAAATjEwZW1zY3JpcHRlbjExbWVtb3J5X3ZpZXdJZEVFAADg2QAA/GAAAAAAAAAAAAAAAAAAAAMAAAAEAAAABAAAAAYAAACD+aIARE5uAPwpFQDRVycA3TT1AGLbwAA8mZUAQZBDAGNR/gC73qsAt2HFADpuJADSTUIASQbgAAnqLgAcktEA6x3+ACmxHADoPqcA9TWCAES7LgCc6YQAtCZwAEF+XwDWkTkAU4M5AJz0OQCLX4QAKPm9APgfOwDe/5cAD5gFABEv7wAKWosAbR9tAM9+NgAJyycARk+3AJ5mPwAt6l8Auid1AOXrxwA9e/EA9zkHAJJSigD7a+oAH7FfAAhdjQAwA1YAe/xGAPCrawAgvM8ANvSaAOOpHQBeYZEACBvmAIWZZQCgFF8AjUBoAIDY/wAnc00ABgYxAMpWFQDJqHMAe+JgAGuMwAAZxEcAzWfDAAno3ABZgyoAi3bEAKYclgBEr90AGVfRAKU+BQAFB/8AM34/AMIy6ACYT94Au30yACY9wwAea+8An/heADUfOgB/8soA8YcdAHyQIQBqJHwA1W76ADAtdwAVO0MAtRTGAMMZnQCtxMIALE1BAAwAXQCGfUYA43EtAJvGmgAzYgAAtNJ8ALSnlwA3VdUA1z72AKMQGABNdvwAZJ0qAHDXqwBjfPgAerBXABcV5wDASVYAO9bZAKeEOAAkI8sA1op3AFpUIwAAH7kA8QobABnO3wCfMf8AZh5qAJlXYQCs+0cAfn/YACJltwAy6IkA5r9gAO/EzQBsNgkAXT/UABbe1wBYO94A3puSANIiKAAohugA4lhNAMbKMgAI4xYA4H3LABfAUADzHacAGOBbAC4TNACDEmIAg0gBAPWOWwCtsH8AHunyAEhKQwAQZ9MAqt3YAK5fQgBqYc4ACiikANOZtAAGpvIAXHd/AKPCgwBhPIgAinN4AK+MWgBv170ALaZjAPS/ywCNge8AJsFnAFXKRQDK2TYAKKjSAMJhjQASyXcABCYUABJGmwDEWcQAyMVEAE2ykQAAF/MA1EOtAClJ5QD91RAAAL78AB6UzABwzu4AEz71AOzxgACz58MAx/goAJMFlADBcT4ALgmzAAtF8wCIEpwAqyB7AC61nwBHksIAezIvAAxVbQByp5AAa+cfADHLlgB5FkoAQXniAPTfiQDolJcA4uaEAJkxlwCI7WsAX182ALv9DgBImrQAZ6RsAHFyQgCNXTIAnxW4ALzlCQCNMSUA93Q5ADAFHAANDAEASwhoACzuWABHqpAAdOcCAL3WJAD3faYAbkhyAJ8W7wCOlKYAtJH2ANFTUQDPCvIAIJgzAPVLfgCyY2gA3T5fAEBdAwCFiX8AVVIpADdkwABt2BAAMkgyAFtMdQBOcdQARVRuAAsJwQAq9WkAFGbVACcHnQBdBFAAtDvbAOp2xQCH+RcASWt9AB0nugCWaSkAxsysAK0UVACQ4moAiNmJACxyUAAEpL4AdweUAPMwcAAA/CcA6nGoAGbCSQBk4D0Al92DAKM/lwBDlP0ADYaMADFB3gCSOZ0A3XCMABe35wAI3zsAFTcrAFyAoABagJMAEBGSAA/o2ABsgK8A2/9LADiQDwBZGHYAYqUVAGHLuwDHibkAEEC9ANLyBABJdScA67b2ANsiuwAKFKoAiSYvAGSDdgAJOzMADpQaAFE6qgAdo8IAr+2uAFwmEgBtwk0ALXqcAMBWlwADP4MACfD2ACtAjABtMZkAObQHAAwgFQDYw1sA9ZLEAMatSwBOyqUApzfNAOapNgCrkpQA3UJoABlj3gB2jO8AaItSAPzbNwCuoasA3xUxAACuoQAM+9oAZE1mAO0FtwApZTAAV1a/AEf/OgBq+bkAdb7zACiT3wCrgDAAZoz2AATLFQD6IgYA2eQdAD2zpABXG48ANs0JAE5C6QATvqQAMyO1APCqGgBPZagA0sGlAAs/DwBbeM0AI/l2AHuLBACJF3IAxqZTAG9u4gDv6wAAm0pYAMTatwCqZroAds/PANECHQCx8S0AjJnBAMOtdwCGSNoA912gAMaA9ACs8C8A3eyaAD9cvADQ3m0AkMcfACrbtgCjJToAAK+aAK1TkwC2VwQAKS20AEuAfgDaB6cAdqoOAHtZoQAWEioA3LctAPrl/QCJ2/4Aib79AOR2bAAGqfwAPoBwAIVuFQD9h/8AKD4HAGFnMwAqGIYATb3qALPnrwCPbW4AlWc5ADG/WwCE10gAMN8WAMctQwAlYTUAyXDOADDLuAC/bP0ApACiAAVs5ABa3aAAIW9HAGIS0gC5XIQAcGFJAGtW4ACZUgEAUFU3AB7VtwAz8cQAE25fAF0w5ACFLqkAHbLDAKEyNgAIt6QA6rHUABb3IQCPaeQAJ/93AAwDgACNQC0AT82gACClmQCzotMAL10KALT5QgAR2ssAfb7QAJvbwQCrF70AyqKBAAhqXAAuVRcAJwBVAH8U8ADhB4YAFAtkAJZBjQCHvt4A2v0qAGsltgB7iTQABfP+ALm/ngBoak8ASiqoAE/EWgAt+LwA11qYAPTHlQANTY0AIDqmAKRXXwAUP7EAgDiVAMwgAQBx3YYAyd62AL9g9QBNZREAAQdrAIywrACywNAAUVVIAB77DgCVcsMAowY7AMBANQAG3HsA4EXMAE4p+gDWysgA6PNBAHxk3gCbZNgA2b4xAKSXwwB3WNQAaePFAPDaEwC6OjwARhhGAFV1XwDSvfUAbpLGAKwuXQAORO0AHD5CAGHEhwAp/ekA59bzACJ8ygBvkTUACODFAP/XjQBuauIAsP3GAJMIwQB8XXQAa62yAM1unQA+cnsAxhFqAPfPqQApc98Atcm6ALcAUQDisg0AdLokAOV9YAB02IoADRUsAIEYDAB+ZpQAASkWAJ96dgD9/b4AVkXvANl+NgDs2RMAi7q5AMSX/AAxqCcA8W7DAJTFNgDYqFYAtKi1AM/MDgASiS0Ab1c0ACxWiQCZzuMA1iC5AGteqgA+KpwAEV/MAP0LSgDh9PsAjjttAOKGLADp1IQA/LSpAO/u0QAuNckALzlhADghRAAb2cgAgfwKAPtKagAvHNgAU7SEAE6ZjABUIswAKlXcAMDG1gALGZYAGnC4AGmVZAAmWmAAP1LuAH8RDwD0tREA/Mv1ADS8LQA0vO4A6F3MAN1eYABnjpsAkjPvAMkXuABhWJsA4Ve8AFGDxgDYPhAA3XFIAC0c3QCvGKEAISxGAFnz1wDZepgAnlTAAE+G+gBWBvwA5XmuAIkiNgA4rSIAZ5PcAFXoqgCCJjgAyuebAFENpACZM7EAqdcOAGkFSABlsvAAf4inAIhMlwD50TYAIZKzAHuCSgCYzyEAQJ/cANxHVQDhdDoAZ+tCAP6d3wBe1F8Ae2ekALqsegBV9qIAK4gjAEG6VQBZbggAISqGADlHgwCJ4+YA5Z7UAEn7QAD/VukAHA/KAMVZigCU+isA08HFAA/FzwDbWq4AR8WGAIVDYgAhhjsALHmUABBhhwAqTHsAgCwaAEO/EgCIJpAAeDyJAKjE5ADl23sAxDrCACb06gD3Z4oADZK/AGWjKwA9k7EAvXwLAKRR3AAn3WMAaeHdAJqUGQCoKZUAaM4oAAnttABEnyAATpjKAHCCYwB+fCMAD7kyAKf1jgAUVucAIfEIALWdKgBvfk0ApRlRALX5qwCC39YAlt1hABY2AgDEOp8Ag6KhAHLtbQA5jXoAgripAGsyXABGJ1sAADTtANIAdwD89FUAAVlNAOBxgAAAAAAAAAAAAAAAAED7Ifk/AAAAAC1EdD4AAACAmEb4PAAAAGBRzHg7AAAAgIMb8DkAAABAICV6OAAAAIAiguM2AAAAAB3zaTUAAAAAAADwP3SFFdOw2e8/D4n5bFi17z9RWxLQAZPvP3tRfTy4cu8/qrloMYdU7z84YnVuejjvP+HeH/WdHu8/FbcxCv4G7z/LqTo3p/HuPyI0Ekym3u4/LYlhYAjO7j8nKjbV2r/uP4JPnVYrtO4/KVRI3Qer7j+FVTqwfqTuP807f2aeoO4/dF/s6HWf7j+HAetzFKHuPxPOTJmJpe4/26AqQuWs7j/lxc2wN7fuP5Dwo4KRxO4/XSU+sgPV7j+t01qZn+juP0de+/J2/+4/nFKF3ZsZ7z9pkO/cIDfvP4ek+9wYWO8/X5t7M5d87z/akKSir6TvP0BFblt20O8/AAAAAAAA6EKUI5FL+GqsP/PE+lDOv84/1lIM/0Iu5j8AAAAAAAA4Q/6CK2VHFUdAlCORS/hqvD7zxPpQzr8uP9ZSDP9CLpY//oIrZUcVZ0AAAAAAAAA4QwAA+v5CLna/OjuevJr3DL29/f/////fPzxUVVVVVcU/kSsXz1VVpT8X0KRnERGBPwAAAAAAAMhC7zn6/kIu5j8kxIL/vb/OP7X0DNcIa6w/zFBG0quygz+EOk6b4NdVPwAAAAAAAAAAAAAAAAAA8D9uv4gaTzubPDUz+6k99u8/XdzYnBNgcbxhgHc+muzvP9FmhxB6XpC8hX9u6BXj7z8T9mc1UtKMPHSFFdOw2e8/+o75I4DOi7ze9t0pa9DvP2HI5mFO92A8yJt1GEXH7z+Z0zNb5KOQPIPzxso+vu8/bXuDXaaalzwPiflsWLXvP/zv/ZIatY4890dyK5Ks7z/RnC9wPb4+PKLR0zLso+8/C26QiTQDarwb0/6vZpvvPw69LypSVpW8UVsS0AGT7z9V6k6M74BQvMwxbMC9iu8/FvTVuSPJkbzgLamumoLvP69VXOnj04A8UY6lyJh67z9Ik6XqFRuAvHtRfTy4cu8/PTLeVfAfj7zqjYw4+WrvP79TEz+MiYs8dctv61tj7z8m6xF2nNmWvNRcBITgW+8/YC86PvfsmjyquWgxh1TvP504hsuC54+8Hdn8IlBN7z+Nw6ZEQW+KPNaMYog7Ru8/fQTksAV6gDyW3H2RST/vP5SoqOP9jpY8OGJ1bno47z99SHTyGF6HPD+msk/OMe8/8ucfmCtHgDzdfOJlRSvvP14IcT97uJa8gWP14d8k7z8xqwlt4feCPOHeH/WdHu8/+r9vGpshPbyQ2drQfxjvP7QKDHKCN4s8CwPkpoUS7z+Py86JkhRuPFYvPqmvDO8/tquwTXVNgzwVtzEK/gbvP0x0rOIBQoY8MdhM/HAB7z9K+NNdOd2PPP8WZLII/O4/BFuOO4Cjhrzxn5JfxfbuP2hQS8ztSpK8y6k6N6fx7j+OLVEb+AeZvGbYBW2u7O4/0jaUPujRcbz3n+U02+fuPxUbzrMZGZm85agTwy3j7j9tTCqnSJ+FPCI0Ekym3u4/imkoemASk7wcgKwERdruP1uJF0iPp1i8Ki73IQrW7j8bmklnmyx8vJeoUNn10e4/EazCYO1jQzwtiWFgCM7uP+9kBjsJZpY8VwAd7UHK7j95A6Ha4cxuPNA8wbWixu4/MBIPP47/kzze09fwKsPuP7CvervOkHY8Jyo21dq/7j934FTrvR2TPA3d/ZmyvO4/jqNxADSUj7ynLJ12srnuP0mjk9zM3oe8QmbPotq27j9fOA+9xt54vIJPnVYrtO4/9lx77EYShrwPkl3KpLHuP47X/RgFNZM82ie1Nkev7j8Fm4ovt5h7PP3Hl9QSre4/CVQc4uFjkDwpVEjdB6vuP+rGGVCFxzQ8t0ZZiiap7j81wGQr5jKUPEghrRVvp+4/n3aZYUrkjLwJ3Ha54aXuP6hN7zvFM4y8hVU6sH6k7j+u6SuJeFOEvCDDzDRGo+4/WFhWeN3Ok7wlIlWCOKLuP2QZfoCqEFc8c6lM1FWh7j8oIl6/77OTvM07f2aeoO4/grk0h60Sary/2gt1EqDuP+6pbbjvZ2O8LxplPLKf7j9RiOBUPdyAvISUUfl9n+4/zz5afmQfeLx0X+zodZ/uP7B9i8BK7oa8dIGlSJqf7j+K5lUeMhmGvMlnQlbrn+4/09QJXsuckDw/Xd5PaaDuPx2lTbncMnu8hwHrcxSh7j9rwGdU/eyUPDLBMAHtoe4/VWzWq+HrZTxiTs8286LuP0LPsy/FoYi8Eho+VCek7j80NzvxtmmTvBPOTJmJpe4/Hv8ZOoRegLytxyNGGqfuP25XcthQ1JS87ZJEm9mo7j8Aig5bZ62QPJlmitnHqu4/tOrwwS+3jTzboCpC5azuP//nxZxgtmW8jES1FjKv7j9EX/NZg/Z7PDZ3FZmuse4/gz0epx8Jk7zG/5ELW7TuPykebIu4qV285cXNsDe37j9ZuZB8+SNsvA9SyMtEuu4/qvn0IkNDkrxQTt6fgr3uP0uOZtdsyoW8ugfKcPHA7j8nzpEr/K9xPJDwo4KRxO4/u3MK4TXSbTwjI+MZY8juP2MiYiIExYe8ZeVde2bM7j/VMeLjhhyLPDMtSuyb0O4/Fbu809G7kbxdJT6yA9XuP9Ix7pwxzJA8WLMwE57Z7j+zWnNuhGmEPL/9eVVr3u4/tJ2Ol83fgrx689O/a+PuP4czy5J3Gow8rdNamZ/o7j/62dFKj3uQvGa2jSkH7u4/uq7cVtnDVbz7FU+4ovPuP0D2pj0OpJC8OlnljXL57j80k6049NZovEde+/J2/+4/NYpYa+LukbxKBqEwsAXvP83dXwrX/3Q80sFLkB4M7z+smJL6+72RvAke11vCEu8/swyvMK5uczycUoXdmxnvP5T9n1wy4448etD/X6sg7z+sWQnRj+CEPEvRVy7xJ+8/ZxpOOK/NYzy15waUbS/vP2gZkmwsa2c8aZDv3CA37z/StcyDGIqAvPrDXVULP+8/b/r/P12tj7x8iQdKLUfvP0mpdTiuDZC88okNCIdP7z+nBz2mhaN0PIek+9wYWO8/DyJAIJ6RgryYg8kW42DvP6ySwdVQWo48hTLbA+Zp7z9LawGsWTqEPGC0AfMhc+8/Hz60ByHVgrxfm3szl3zvP8kNRzu5Kom8KaH1FEaG7z/TiDpgBLZ0PPY/i+cukO8/cXKdUezFgzyDTMf7UZrvP/CR048S94+82pCkoq+k7z99dCPimK6NvPFnji1Ir+8/CCCqQbzDjjwnWmHuG7rvPzLrqcOUK4Q8l7prNyvF7z/uhdExqWSKPEBFblt20O8/7eM75Lo3jrwUvpyt/dvvP53NkU07iXc82JCegcHn7z+JzGBBwQVTPPFxjyvC8+8/AAAgZUcV9z8Aou8u/AXnPTmDK2VHFee/vgQ63AnH3j/7L3BkRxXXv0hMA1Bsd9I/vJLqKLPHzr8u+RfhJWLKP/6CK2VHFee/9wM63AnH3j8/fCtlRxXXv+Rb8FBsd9I/5Y923QnHzr8258QedmHKP5unZLw/Fce/ShvwVNGExD88OCyn5InCv2buWigvs8A/+Kyxaygk9z8AsM3uXwnhv6HM0mb34fY/ANB2vZSE4L+K1DAOPaH2PwD46K5DAeC/hWzQMuxh9j8AQAs2xf7ev/iYEZX6I/Y/AOC3Gtn93b9sAs+kW+f1PwCQxwyu/9y/uE8hWgWs9T8AoP0ROATcvx5uFg/tcfU/AOA6MmcL2781+AtZCTn1PwCwLVovFdq/3a1h7U8B9T8AYPhafyHZv9B7SI64yvQ/AJBxsE0w2L/uTzO0OZX0PwDgqfmJQde/adWv38tg9D8AkBm1K1XWv1O55E5mLfQ/ABCboiNr1b+m2B0RAfvzPwCgXw9lg9S/NlgMt5XJ8z8AoPY36Z3Tv0r9tkocmfM/AGCNU6G60r+1meAMjmnzPwBAykCD2dG/sucTguQ68z8A4EA6hfrQv7G9hRkZDfM/ADDnMpwd0L/XcbLKJeDyPwBg+qJ9hc6/gs0TzwS08j8AgD1jyNPMv1DLfCywiPI/AKAUTAMmy7/lTZRjIl7yPwDgTy8cfMm/sRWGPVY08j8AAIA/AtbHvzivPuNGC/I/AOAFGqczxr/do8397uLxPwAAV+n1lMS/MDkLWEq78T8AoOAk5PnCvwAif4RTlPE/AMD9Wlliwb8819XABm7xPwCAvXWanL+/wuS3R19I8T8AwPlbV3u8v9GFAK1YI/E/AID0D8Zgub8nIlMP8P7wPwAAtkfiTLa/jzrQdyDb8D8AQAGyeD+zv9mAWdbmt/A/AMBCGn04sL+NQHv+PpXwPwAAtQiSb6q/gzvFyiVz8D8AAHdPlXqkv1wbDeSXUfA/AAAMxagjnb+ijiDBkTDwPwAAeCkmapG/IX6zJRAQ8D8AAOjY+CB3v2unyvl+wO8/AABQsVP+hj+E8fbTZUTvPwCAD+HMHKE/fxCEnwfM7j8AgIuM/E2sP+hal5k6V+4/AEBXHjKqsz/mPb3w1uXtPwCAi9CgGLk/szj/gbZ37T8AQATa6XK+P0PpTXK1DO0/AGB/UNLcwT9jdQ7csqTsPwCg3gOrdsQ/UcvW6I4/7D8AIOJ3QwfHP0wMAk8r3es/AECpi96OyT/KFWAAbH3rPwDg0mq4Dcw/jzMubjYg6z8A4M6vCoTOPzlQKSZwxeo/AIBntAp50D/dMSe8AW3qPwDAAWgFrNE/i/E/vNMW6j8A4P7UEdvSP63+Z0nRwuk/AIDFTkYG1D8CmXz05HDpPwDwOgm+LdU/8ryCOfsg6T8A0FAgkFHWP/FZ94cB0+g/APDqzdJx1z9t9rnr5YboPwCQfYWcjtg/lLlYtpc86D8AYOFVAajZPyIQxv8F9Oc/ANDTbhi+2j/KFRQYIq3nPwDgoK7y0Ns/jP+e+dxn5z8AQL89pODcP44KuRIAIOY/BbZEBqsEiTymNFcEAGDmP6n3Yuqb/2E8xfIlw/+f5j+6kDzLz36CPARauTgA4OY/JpNzVoj/iDzjlJng/x/nP7GCXydA/Yo8EA5ZFQBg5z9BgyO0df1yvNVbZRIAoOc/diskfOYIeDym6VkyAODnP7ci9ibkCGK80rK07f8f6D8vyaUeRgKEvMP8+i0AYOg/H5ryovT3bTxQa4z3/5/oP/2VSQlTBI68ZhVnOQDg6D9Fe8e+8wSKvEUXv+L/H+k/PCAOQDT6d7zRn1zM/1/pP11poAWA/3a8Z0e6OwCg6T8DfuzExPhwPKUtuef/3+k/AkaMR9l/jjyv/S7X/x/qP36uzU1VDGq8lf8E3v9f6j9rsumMqX2GPCuNXsr/n+o/3hNMtcmEgrzqA63d/9/qPzwuYOrIElg8TT0N8f8f6z+ceCet3fqOvFoWIc7/X+s/NxLGGRfLUzx05lDZ/5/rPwDOlEHZ93M8r6icEwDg6z/Am10hxAp1PJnfRlsAIOw/ycHpU6buazyu97lAAGDsP9ZwSiefB3y8iv1VYgCg7D8fTOh2QAt6vF0JTNn/3+w/17Wa+TP5iDzP1nX5/x/tP77hX2YILFi8kxxWov9f7T/zldKbKAR7vAyLIp3/n+0/NqIPNFEChzwWfrxlAODtPwzYpBYeAXW8kUf2AgAg7j/gYu8JL4CJPNim11cAYO4/+vcMWHULfrwMwO0nAKDuPxGYRQmDhIy8fMv1bADg7j/0dhWVJ4CPvMx9K3gAIO8/j1N0ctmBj7wKRQwmAGDvP9z/JycAcUC8M9WM6P+f7z+wqP3h3BtYvImGD9X/3+8/bo6Ryxr5hzxnIykEACDwP4FGMmXzf5s8aNbj4/9f8D97la7dCPqGPFenhQoAoPA/kfvTgN7iV7zMP18aAODwPxTwxQUzgpG89bqv+P8f8T/CuoBmu/qLvK2RTeX/X/E/7+c3FxJ/nbzhNqwRAKDxP//1FgUKAJw8SELIGQDg8T+gXdrk+4KQvG5e/g8AIPI/Q/ucTND9iLyR2J8mAGDyP4LRlHkq/ow82uamKQCg8j/Fi15xcwJwvDk+KeD/3/I/+aay2jl8mzyC8Nz3/x/zP1RS3G4z8X08YIta8P9f8z/rMc1MVgOevMyuDi4AoPM/d6TTS+fwdTw2sjsEAODzPzOInRTLfZw8/4fRAgAg9D8oPS3Prwh+PLF8OA0AYPQ/ppllhTcIgjyJn1YEAKD0P9K8T5Bc+om880M1BADg9D8pUxftJRF4vA9/Asz/H/U/3FR3hNiDmDxvs4f9/1/1Pwco0DHnCYe8uvcd8v+f9T8Ce3Jon/eHPIE0/Ov/3/U/PukwLpCAkbwAOPr+Qi7mPzBnx5NX8y49AQAAAAAA4L9bMFFVVVXVP5BF6////8+/EQHxJLOZyT+fyAbldVXFvwAAAAAAAOC/d1VVVVVV1T/L/f/////PvwzdlZmZmck/p0VnVVVVxb8w3kSjJEnCP2U9QqT//7+/ytYqKIRxvD//aLBD65m5v4XQr/eCgbc/zUXRdRNStb+f3uDD8DT3PwCQ5nl/zNe/H+ksangT9z8AAA3C7m/Xv6C1+ghg8vY/AOBRE+MT1799jBMfptH2PwB4KDhbuNa/0bTFC0mx9j8AeICQVV3Wv7oMLzNHkfY/AAAYdtAC1r8jQiIYn3H2PwCQkIbKqNW/2R6lmU9S9j8AUANWQ0/Vv8Qkj6pWM/Y/AEBrwzf21L8U3J1rsxT2PwBQqP2nndS/TFzGUmT29T8AqIk5kkXUv08skbVn2PU/ALiwOfTt07/ekFvLvLr1PwBwj0TOltO/eBrZ8mGd9T8AoL0XHkDTv4dWRhJWgPU/AIBG7+Lp0r/Ta+fOl2P1PwDgMDgblNK/k3+n4iVH9T8AiNqMxT7Sv4NFBkL/KvU/AJAnKeHp0b/fvbLbIg/1PwD4SCttldG/1940R4/z9D8A+LmaZ0HRv0Ao3s9D2PQ/AJjvlNDt0L/Io3jAPr30PwAQ2xilmtC/iiXgw3+i9D8AuGNS5kfQvzSE1CQFiPQ/APCGRSLrz78LLRkbzm30PwCwF3VKR8+/VBg509lT9D8AMBA9RKTOv1qEtEQnOvQ/ALDpRA0Czr/7+BVBtSD0PwDwdymiYM2/sfQ+2oIH9D8AkJUEAcDMv4/+V12P7vM/ABCJVikgzL/pTAug2dXzPwAQgY0Xgcu/K8EQwGC98z8A0NPMyeLKv7jadSskpfM/AJASLkBFyr8C0J/NIo3zPwDwHWh3qMm/HHqExVt18z8AMEhpbQzJv+I2rUnOXfM/AMBFpiBxyL9A1E2YeUbzPwAwFLSP1se/JMv/zlwv8z8AcGI8uDzHv0kNoXV3GPM/AGA3m5qjxr+QOT43yAHzPwCgt1QxC8a/QfiVu07r8j8AMCR2fXPFv9GpGQIK1fI/ADDCj3vcxL8q/beo+b7yPwAA0lEsRsS/qxsMehyp8j8AAIO8irDDvzC1FGByk/I/AABJa5kbw7/1oVdX+n3yPwBApJBUh8K/vzsdm7No8j8AoHn4ufPBv731j4OdU/I/AKAsJchgwb87CMmqtz7yPwAg91d/zsC/tkCpKwEq8j8AoP5J3DzAvzJBzJZ5FfI/AIBLvL1Xv7+b/NIdIAHyPwBAQJYIN76/C0hNSfTs8T8AQPk+mBe9v2llj1L12PE/AKDYTmf5u798flcRI8XxPwBgLyB53Lq/6SbLdHyx8T8AgCjnw8C5v7YaLAwBnvE/AMBys0amuL+9cLZ7sIrxPwAArLMBjbe/trzvJYp38T8AADhF8XS2v9oxTDWNZPE/AICHbQ5etb/dXyeQuVHxPwDgod5cSLS/TNIypA4/8T8AoGpN2TOzv9r5EHKLLPE/AGDF+Hkgsr8xtewoMBrxPwAgYphGDrG/rzSE2vsH8T8AANJqbPqvv7NrTg/u9fA/AEB3So3arb/OnypdBuTwPwAAheTsvKu/IaUsY0TS8D8AwBJAiaGpvxqY4nynwPA/AMACM1iIp7/RNsaDL6/wPwCA1mdecaW/OROgmNud8D8AgGVJilyjv9/nUq+rjPA/AEAVZONJob/7KE4vn3vwPwCA64LAcp6/GY81jLVq8D8AgFJS8VWavyz57KXuWfA/AICBz2I9lr+QLNHNSUnwPwAAqoz7KJK/qa3wxsY48D8AAPkgezGMv6kyeRNlKPA/AACqXTUZhL9Ic+onJBjwPwAA7MIDEni/lbEUBgQI8D8AACR5CQRgvxr6Jvcf4O8/AACQhPPvbz906mHCHKHvPwAAPTVB3Ic/LpmBsBBj7z8AgMLEo86TP82t7jz2Je8/AACJFMGfmz/nE5EDyOnuPwAAEc7YsKE/q7HLeICu7j8AwAHQW4qlP5sMnaIadO4/AIDYQINcqT+1mQqDkTruPwCAV+9qJ60/VppgCeAB7j8AwJjlmHWwP5i7d+UByu0/ACAN4/VTsj8DkXwL8pLtPwAAOIvdLrQ/zlz7Zqxc7T8AwFeHWQa2P53eXqosJ+0/AABqNXbatz/NLGs+bvLsPwBgHE5Dq7k/Anmnom2+7D8AYA27x3i7P20IN20mi+w/ACDnMhNDvT8EWF29lFjsPwBg3nExCr8/jJ+7M7Um7D8AQJErFWfAPz/n7O6D9es/ALCSgoVHwT/Bltt1/cTrPwAwys1uJsI/KEqGDB6V6z8AUMWm1wPDPyw+78XiZes/ABAzPMPfwz+LiMlnSDfrPwCAems2usQ/SjAdIUsJ6z8A8NEoOZPFP37v8oXo2+o/APAYJM1qxj+iPWAxHa/qPwCQZuz4QMc/p1jTP+aC6j8A8Br1wBXIP4tzCe9AV+o/AID2VCnpyD8nS6uQKizqPwBA+AI2u8k/0fKTE6AB6j8AACwc7YvKPxs82ySf1+k/ANABXFFbyz+QsccFJa7pPwDAvMxnKcw/L86X8i6F6T8AYEjVNfbMP3VLpO66XOk/AMBGNL3BzT84SOedxjTpPwDgz7gBjM4/5lJnL08N6T8AkBfACVXPP53X/45S5ug/ALgfEmwO0D98AMyfzr/oPwDQkw64cdA/DsO+2sCZ6D8AcIaea9TQP/sXI6ondOg/ANBLM4c20T8ImrOsAE/oPwBII2cNmNE/VT5l6Ekq6D8AgMzg//jRP2AC9JUBBug/AGhj119Z0j8po+BjJeLnPwCoFAkwudI/rbXcd7O+5z8AYEMQchjTP8Ill2eqm+c/ABjsbSZ30z9XBhfyB3nnPwAwr/tP1dM/DBPW28pW5z8A4C/j7jLUP2u2TwEAEOY/PFtCkWwCfjyVtE0DADDmP0FdAEjqv408eNSUDQBQ5j+3pdaGp3+OPK1vTgcAcOY/TCVUa+r8YTyuD9/+/4/mP/0OWUwnfny8vMVjBwCw5j8B2txIaMGKvPbBXB4A0OY/EZNJnRw/gzw+9gXr/+/mP1Mt4hoEgH68gJeGDgAQ5z9SeQlxZv97PBLpZ/z/L+c/JIe9JuIAjDxqEYHf/0/nP9IB8W6RAm68kJxnDwBw5z90nFTNcfxnvDXIfvr/j+c/gwT1nsG+gTzmwiD+/6/nP2VkzCkXfnC8AMk/7f/P5z8ci3sIcoCAvHYaJun/7+c/rvmdbSjAjTzoo5wEABDoPzNM5VHSf4k8jyyTFwAw6D+B8zC26f6KvJxzMwYAUOg/vDVla7+/iTzGiUIgAHDoP3V7EfNlv4u8BHn16/+P6D9Xyz2ibgCJvN8EvCIAsOg/CkvgON8AfbyKGwzl/8/oPwWf/0ZxAIi8Q46R/P/v6D84cHrQe4GDPMdf+h4AEOk/A7TfdpE+iTy5e0YTADDpP3YCmEtOgH88bwfu5v9P6T8uYv/Z8H6PvNESPN7/b+k/ujgmlqqCcLwNikX0/4/pP++oZJEbgIe8Pi6Y3f+v6T83k1qK4ECHvGb7Se3/z+k/AOCbwQjOPzxRnPEgAPDpPwpbiCeqP4q8BrBFEQAQ6j9W2liZSP90PPr2uwcAMOo/GG0riqu+jDx5HZcQAFDqPzB5eN3K/og8SC71HQBw6j/bq9g9dkGPvFIzWRwAkOo/EnbChAK/jrxLPk8qALDqP18//zwE/Wm80R6u1//P6j+0cJAS5z6CvHgEUe7/7+o/o94O4D4GajxbDWXb/w/rP7kKHzjIBlo8V8qq/v8v6z8dPCN0HgF5vNy6ldn/T+s/nyqGaBD/ebycZZ4kAHDrPz5PhtBF/4o8QBaH+f+P6z/5w8KWd/58PE/LBNL/r+s/xCvy7if/Y7xFXEHS/8/rPyHqO+63/2y83wlj+P/v6z9cCy6XA0GBvFN2teH/D+w/GWq3lGTBizzjV/rx/y/sP+3GMI3v/mS8JOS/3P9P7D91R+y8aD+EvPe5VO3/b+w/7OBT8KN+hDzVj5nr/4/sP/GS+Y0Gg3M8miElIQCw7D8EDhhkjv1ovJxGlN3/z+w/curHHL5+jjx2xP3q/+/sP/6In605vo48K/iaFgAQ7T9xWrmokX11PB33Dw0AMO0/2sdwaZDBiTzED3nq/0/tPwz+WMU3Dli85YfcLgBw7T9ED8FN1oB/vKqC3CEAkO0/XFz9lI98dLyDAmvY/6/tP35hIcUdf4w8OUdsKQDQ7T9Tsf+yngGIPPWQROX/7+0/icxSxtIAbjyU9qvN/w/uP9JpLSBAg3+83chS2/8v7j9kCBvKwQB7PO8WQvL/T+4/UauUsKj/cjwRXoro/2/uP1m+77Fz9le8Df+eEQCQ7j8ByAtejYCEvEQXpd//r+4/tSBD1QYAeDyhfxIaANDuP5JcVmD4AlC8xLy6BwDw7j8R5jVdRECFvAKNevX/D+8/BZHvOTH7T7zHiuUeADDvP1URc/KsgYo8lDSC9f9P7z9Dx9fUQT+KPGtMqfz/b+8/dXiYHPQCYrxBxPnh/4/vP0vnd/TRfXc8fuPg0v+v7z8xo3yaGQFvvJ7kdxwA0O8/sazOS+6BcTwxw+D3/+/vP1qHcAE3BW68bmBl9P8P8D/aChxJrX6KvFh6hvP/L/A/4LL8w2l/l7wXDfz9/0/wP1uUyzT+v5c8gk3NAwBw8D/LVuTAgwCCPOjL8vn/j/A/GnU3vt//bbxl2gwBALDwP+sm5q5/P5G8ONOkAQDQ8D/3n0h5+n2APP392vr/7/A/wGvWcAUEd7yW/boLABDxP2ILbYTUgI48XfTl+v8v8T/vNv1k+r+dPNma1Q0AUPE/rlAScHcAmjyaVSEPAHDxP+7e4+L5/Y08JlQn/P+P8T9zcjvcMACRPFk8PRIAsPE/iAEDgHl/mTy3nin4/8/xP2eMn6sy+WW8ANSK9P/v8T/rW6edv3+TPKSGiwwAEPI/Ilv9kWuAnzwDQ4UDADDyPzO/n+vC/5M8hPa8//9P8j9yLi5+5wF2PNkhKfX/b/I/YQx/drv8fzw8OpMUAJDyPytBAjzKAnK8E2NVFACw8j8CH/IzgoCSvDtS/uv/z/I/8txPOH7/iLyWrbgLAPDyP8VBMFBR/4W8r+J6+/8P8z+dKF6IcQCBvH9frP7/L/M/Fbe3P13/kbxWZ6YMAFDzP72CiyKCf5U8Iff7EQBw8z/M1Q3EugCAPLkvWfn/j/M/UaeyLZ0/lLxC0t0EALDzP+E4dnBrf4U8V8my9f/P8z8xEr8QOgJ6PBi0sOr/7/M/sFKxZm1/mDz0rzIVABD0PySFGV83+Gc8KYtHFwAw9D9DUdxy5gGDPGO0lef/T/Q/WomyuGn/iTzgdQTo/2/0P1TywpuxwJW858Fv7/+P9D9yKjryCUCbPASnvuX/r/Q/RX0Nv7f/lLzeJxAXAND0Pz1q3HFkwJm84j7wDwDw9D8cU4ULiX+XPNFL3BIAEPU/NqRmcWUEYDx6JwUWADD1PwkyI87Ov5a8THDb7P9P9T/XoQUFcgKJvKlUX+//b/U/EmTJDua/mzwSEOYXAJD1P5Dvr4HFfog8kj7JAwCw9T/ADL8KCEGfvLwZSR0A0PU/KUcl+yqBmLyJerjn/+/1PwRp7YC3fpS8vvP4eexh9j/eqoyA93vVvz2Ir0rtcfU/223Ap/C+0r+wEPDwOZX0P2c6UX+uHtC/hQO4sJXJ8z/pJIKm2DHLv6VkiAwZDfM/WHfACk9Xxr+gjgt7Il7yPwCBnMcrqsG/PzQaSkq78T9eDozOdk66v7rlivBYI/E/zBxhWjyXsb+nAJlBP5XwPx4M4Tj0UqK/AAAAAAAA8D8AAAAAAAAAAKxHmv2MYO4/hFnyXaqlqj+gagIfs6TsP7QuNqpTXrw/5vxqVzYg6z8I2yB35SbFPy2qoWPRwuk/cEciDYbCyz/tQXgD5oboP+F+oMiLBdE/YkhT9dxn5z8J7rZXMATUP+85+v5CLuY/NIO4SKMO0L9qC+ALW1fVPyNBCvL+/9+/ADj6/kIu5j8wZ8eTV/MuPQAAAAAAAOC/YFVVVVVV5b8GAAAAAADgP05VWZmZmek/eqQpVVVV5b/pRUibW0nyv8M/JosrAPA/AAAAAACg9j8AAAAAAAAAAADIufKCLNa/gFY3KCS0+jwAAAAAAID2PwAAAAAAAAAAAAhYv73R1b8g9+DYCKUcvQAAAAAAYPY/AAAAAAAAAAAAWEUXd3bVv21QttWkYiO9AAAAAABA9j8AAAAAAAAAAAD4LYetGtW/1WewnuSE5rwAAAAAACD2PwAAAAAAAAAAAHh3lV++1L/gPimTaRsEvQAAAAAAAPY/AAAAAAAAAAAAYBzCi2HUv8yETEgv2BM9AAAAAADg9T8AAAAAAAAAAACohoYwBNS/OguC7fNC3DwAAAAAAMD1PwAAAAAAAAAAAEhpVUym079glFGGxrEgPQAAAAAAoPU/AAAAAAAAAAAAgJia3UfTv5KAxdRNWSU9AAAAAACA9T8AAAAAAAAAAAAg4bri6NK/2Cu3mR57Jj0AAAAAAGD1PwAAAAAAAAAAAIjeE1qJ0r8/sM+2FMoVPQAAAAAAYPU/AAAAAAAAAAAAiN4TWonSvz+wz7YUyhU9AAAAAABA9T8AAAAAAAAAAAB4z/tBKdK/dtpTKCRaFr0AAAAAACD1PwAAAAAAAAAAAJhpwZjI0b8EVOdovK8fvQAAAAAAAPU/AAAAAAAAAAAAqKurXGfRv/CogjPGHx89AAAAAADg9D8AAAAAAAAAAABIrvmLBdG/ZloF/cSoJr0AAAAAAMD0PwAAAAAAAAAAAJBz4iSj0L8OA/R+7msMvQAAAAAAoPQ/AAAAAAAAAAAA0LSUJUDQv38t9J64NvC8AAAAAACg9D8AAAAAAAAAAADQtJQlQNC/fy30nrg28LwAAAAAAID0PwAAAAAAAAAAAEBebRi5z7+HPJmrKlcNPQAAAAAAYPQ/AAAAAAAAAAAAYNzLrfDOvySvhpy3Jis9AAAAAABA9D8AAAAAAAAAAADwKm4HJ86/EP8/VE8vF70AAAAAACD0PwAAAAAAAAAAAMBPayFczb8baMq7kbohPQAAAAAAAPQ/AAAAAAAAAAAAoJrH94/MvzSEn2hPeSc9AAAAAAAA9D8AAAAAAAAAAACgmsf3j8y/NISfaE95Jz0AAAAAAODzPwAAAAAAAAAAAJAtdIbCy7+Pt4sxsE4ZPQAAAAAAwPM/AAAAAAAAAAAAwIBOyfPKv2aQzT9jTro8AAAAAACg8z8AAAAAAAAAAACw4h+8I8q/6sFG3GSMJb0AAAAAAKDzPwAAAAAAAAAAALDiH7wjyr/qwUbcZIwlvQAAAAAAgPM/AAAAAAAAAAAAUPScWlLJv+PUwQTZ0Sq9AAAAAABg8z8AAAAAAAAAAADQIGWgf8i/Cfrbf7+9Kz0AAAAAAEDzPwAAAAAAAAAAAOAQAomrx79YSlNykNsrPQAAAAAAQPM/AAAAAAAAAAAA4BACiavHv1hKU3KQ2ys9AAAAAAAg8z8AAAAAAAAAAADQGecP1sa/ZuKyo2rkEL0AAAAAAADzPwAAAAAAAAAAAJCncDD/xb85UBCfQ54evQAAAAAAAPM/AAAAAAAAAAAAkKdwMP/FvzlQEJ9Dnh69AAAAAADg8j8AAAAAAAAAAACwoePlJsW/j1sHkIveIL0AAAAAAMDyPwAAAAAAAAAAAIDLbCtNxL88eDVhwQwXPQAAAAAAwPI/AAAAAAAAAAAAgMtsK03Evzx4NWHBDBc9AAAAAACg8j8AAAAAAAAAAACQHiD8ccO/OlQnTYZ48TwAAAAAAIDyPwAAAAAAAAAAAPAf+FKVwr8IxHEXMI0kvQAAAAAAYPI/AAAAAAAAAAAAYC/VKrfBv5ajERikgC69AAAAAABg8j8AAAAAAAAAAABgL9Uqt8G/lqMRGKSALr0AAAAAAEDyPwAAAAAAAAAAAJDQfH7XwL/0W+iIlmkKPQAAAAAAQPI/AAAAAAAAAAAAkNB8ftfAv/Rb6IiWaQo9AAAAAAAg8j8AAAAAAAAAAADg2zGR7L+/8jOjXFR1Jb0AAAAAAADyPwAAAAAAAAAAAAArbgcnvr88APAqLDQqPQAAAAAAAPI/AAAAAAAAAAAAACtuBye+vzwA8CosNCo9AAAAAADg8T8AAAAAAAAAAADAW49UXry/Br5fWFcMHb0AAAAAAMDxPwAAAAAAAAAAAOBKOm2Sur/IqlvoNTklPQAAAAAAwPE/AAAAAAAAAAAA4Eo6bZK6v8iqW+g1OSU9AAAAAACg8T8AAAAAAAAAAACgMdZFw7i/aFYvTSl8Ez0AAAAAAKDxPwAAAAAAAAAAAKAx1kXDuL9oVi9NKXwTPQAAAAAAgPE/AAAAAAAAAAAAYOWK0vC2v9pzM8k3lya9AAAAAABg8T8AAAAAAAAAAAAgBj8HG7W/V17GYVsCHz0AAAAAAGDxPwAAAAAAAAAAACAGPwcbtb9XXsZhWwIfPQAAAAAAQPE/AAAAAAAAAAAA4BuW10Gzv98T+czaXiw9AAAAAABA8T8AAAAAAAAAAADgG5bXQbO/3xP5zNpeLD0AAAAAACDxPwAAAAAAAAAAAICj7jZlsb8Jo492XnwUPQAAAAAAAPE/AAAAAAAAAAAAgBHAMAqvv5GONoOeWS09AAAAAAAA8T8AAAAAAAAAAACAEcAwCq+/kY42g55ZLT0AAAAAAODwPwAAAAAAAAAAAIAZcd1Cq79McNbleoIcPQAAAAAA4PA/AAAAAAAAAAAAgBlx3UKrv0xw1uV6ghw9AAAAAADA8D8AAAAAAAAAAADAMvZYdKe/7qHyNEb8LL0AAAAAAMDwPwAAAAAAAAAAAMAy9lh0p7/uofI0RvwsvQAAAAAAoPA/AAAAAAAAAAAAwP65h56jv6r+JvW3AvU8AAAAAACg8D8AAAAAAAAAAADA/rmHnqO/qv4m9bcC9TwAAAAAAIDwPwAAAAAAAAAAAAB4DpuCn7/kCX58JoApvQAAAAAAgPA/AAAAAAAAAAAAAHgOm4Kfv+QJfnwmgCm9AAAAAABg8D8AAAAAAAAAAACA1QcbuZe/Oab6k1SNKL0AAAAAAEDwPwAAAAAAAAAAAAD8sKjAj7+cptP2fB7fvAAAAAAAQPA/AAAAAAAAAAAAAPywqMCPv5ym0/Z8Ht+8AAAAAAAg8D8AAAAAAAAAAAAAEGsq4H+/5EDaDT/iGb0AAAAAACDwPwAAAAAAAAAAAAAQayrgf7/kQNoNP+IZvQAAAAAAAPA/AQLwPwGdEMDvPwAAAAAAAAAAAACJdRUQgD/oK52Za8cQvQAAAAAAgO8/AAAAAAAAAAAAgJNYViCQP9L34gZb3CO9AAAAAABA7z8AAAAAAAAAAAAAySglSZg/NAxaMrqgKr0AAAAAAADvPwAAAAAAAAAAAEDniV1BoD9T1/FcwBEBPQAAAAAAwO4/AAAAAAAAAAAAAC7UrmakPyj9vXVzFiy9AAAAAACA7j8AAAAAAAAAAADAnxSqlKg/fSZa0JV5Gb0AAAAAAEDuPwAAAAAAAAAAAMDdzXPLrD8HKNhH8mgavQAAAAAAIO4/AAAAAAAAAAAAwAbAMequP3s7yU8+EQ69AAAAAADg7T8AAAAAAAAAAABgRtE7l7E/m54NVl0yJb0AAAAAAKDtPwAAAAAAAAAAAODRp/W9sz/XTtulXsgsPQAAAAAAYO0/AAAAAAAAAAAAoJdNWum1Px4dXTwGaSy9AAAAAABA7T8AAAAAAAAAAADA6grTALc/Mu2dqY0e7DwAAAAAAADtPwAAAAAAAAAAAEBZXV4zuT/aR706XBEjPQAAAAAAwOw/AAAAAAAAAAAAYK2NyGq7P+Vo9yuAkBO9AAAAAACg7D8AAAAAAAAAAABAvAFYiLw/06xaxtFGJj0AAAAAAGDsPwAAAAAAAAAAACAKgznHvj/gReavaMAtvQAAAAAAQOw/AAAAAAAAAAAA4Ns5kei/P/0KoU/WNCW9AAAAAAAA7D8AAAAAAAAAAADgJ4KOF8E/8gctznjvIT0AAAAAAODrPwAAAAAAAAAAAPAjfiuqwT80mThEjqcsPQAAAAAAoOs/AAAAAAAAAAAAgIYMYdHCP6G0gctsnQM9AAAAAACA6z8AAAAAAAAAAACQFbD8ZcM/iXJLI6gvxjwAAAAAAEDrPwAAAAAAAAAAALAzgz2RxD94tv1UeYMlPQAAAAAAIOs/AAAAAAAAAAAAsKHk5SfFP8d9aeXoMyY9AAAAAADg6j8AAAAAAAAAAAAQjL5OV8Y/eC48LIvPGT0AAAAAAMDqPwAAAAAAAAAAAHB1ixLwxj/hIZzljRElvQAAAAAAoOo/AAAAAAAAAAAAUESFjYnHPwVDkXAQZhy9AAAAAABg6j8AAAAAAAAAAAAAOeuvvsg/0SzpqlQ9B70AAAAAAEDqPwAAAAAAAAAAAAD33FpayT9v/6BYKPIHPQAAAAAAAOo/AAAAAAAAAAAA4Io87ZPKP2khVlBDcii9AAAAAADg6T8AAAAAAAAAAADQW1fYMcs/quGsTo01DL0AAAAAAMDpPwAAAAAAAAAAAOA7OIfQyz+2ElRZxEstvQAAAAAAoOk/AAAAAAAAAAAAEPDG+2/MP9IrlsVy7PG8AAAAAABg6T8AAAAAAAAAAACQ1LA9sc0/NbAV9yr/Kr0AAAAAAEDpPwAAAAAAAAAAABDn/w5Tzj8w9EFgJxLCPAAAAAAAIOk/AAAAAAAAAAAAAN3krfXOPxGOu2UVIcq8AAAAAAAA6T8AAAAAAAAAAACws2wcmc8/MN8MyuzLGz0AAAAAAMDoPwAAAAAAAAAAAFhNYDhx0D+RTu0W25z4PAAAAAAAoOg/AAAAAAAAAAAAYGFnLcTQP+nqPBaLGCc9AAAAAACA6D8AAAAAAAAAAADoJ4KOF9E/HPClYw4hLL0AAAAAAGDoPwAAAAAAAAAAAPisy1xr0T+BFqX3zZorPQAAAAAAQOg/AAAAAAAAAAAAaFpjmb/RP7e9R1Htpiw9AAAAAAAg6D8AAAAAAAAAAAC4Dm1FFNI/6rpGut6HCj0AAAAAAODnPwAAAAAAAAAAAJDcfPC+0j/0BFBK+pwqPQAAAAAAwOc/AAAAAAAAAAAAYNPh8RTTP7g8IdN64ii9AAAAAACg5z8AAAAAAAAAAAAQvnZna9M/yHfxsM1uET0AAAAAAIDnPwAAAAAAAAAAADAzd1LC0z9cvQa2VDsYPQAAAAAAYOc/AAAAAAAAAAAA6NUjtBnUP53gkOw25Ag9AAAAAABA5z8AAAAAAAAAAADIccKNcdQ/ddZnCc4nL70AAAAAACDnPwAAAAAAAAAAADAXnuDJ1D+k2AobiSAuvQAAAAAAAOc/AAAAAAAAAAAAoDgHriLVP1nHZIFwvi49AAAAAADg5j8AAAAAAAAAAADQyFP3e9U/70Bd7u2tHz0AAAAAAMDmPwAAAAAAAAAAAGBZ373V1T/cZaQIKgsKvb7z+HnsYfY/GTCWW8b+3r89iK9K7XH1P6T81DJoC9u/sBDw8DmV9D97tx8Ki0HXv4UDuLCVyfM/e89tGumd07+lZIgMGQ3zPzG28vObHdC/oI4LeyJe8j/wejsbHXzJvz80GkpKu/E/nzyvk+P5wr+65YrwWCPxP1yNeL/LYLm/pwCZQT+V8D/OX0e2nW+qvwAAAAAAAPA/AAAAAAAAAACsR5r9jGDuPz31JJ/KOLM/oGoCH7Ok7D+6kThUqXbEP+b8alc2IOs/0uTESguEzj8tqqFj0cLpPxxlxvBFBtQ/7UF4A+aG6D/4nxssnI7YP2JIU/XcZ+c/zHuxTqTg3D8LbknJFnbSP3rGdaBpGde/3bqnbArH3j/I9r5IRxXnvyu4KmVHFfc/mNwAADDdAAAZAAoAGRkZAAAAAAUAAAAAAAAJAAAAAAsAAAAAAAAAABkAEQoZGRkDCgcAAQAJCxgAAAkGCwAACwAGGQAAABkZGQAAAAAAAAAAAAAAAAAAAAAOAAAAAAAAAAAZAAoNGRkZAA0AAAIACQ4AAAAJAA4AAA4BIQwAAAAAAAAAAAAAABMAAAAAEwAAAAAJDAAAAAAADAAADAEhEAAAAAAAAAAAAAAADwAAAAQPAAAAAAkQAAAAAAAQAAAQASoSAAAAAAAAAAAAAAARAAAAABEAAAAACRIAAAAAABIAABIAABoAAAAaGhoBDhoAAAAaGhoAAAAAAAAJASEUAAAAAAAAAAAAAAAXAAAAABcAAAAACRQAAAAAABQAABQBtgoWAAAAAAAAAAAAAAAVAAAAABUAAAAACRYAAAAAABYAABYAADAxMjM0NTY3ODlBQkNERUYAAAAA0KMAAEwAAADQAAAA0QAAAE5TdDNfXzIxN2JhZF9mdW5jdGlvbl9jYWxsRQAI2gAAtKMAAATbAAAAAAAA+KQAANMAAADUAAAA1QAAANYAAADXAAAA2AAAANkAAADaAAAA2wAAANwAAADdAAAA3gAAAN8AAADgAAAACAAAAAAAAAAwpQAA4QAAAOIAAAD4////+P///zClAADjAAAA5AAAACikAAA8pAAAAAAAABSmAADlAAAA5gAAAOcAAADoAAAA6QAAAOoAAADrAAAA2gAAANsAAADsAAAA3QAAAO0AAADfAAAA7gAAAE5TdDNfXzI5YmFzaWNfaW9zSWNOU18xMWNoYXJfdHJhaXRzSWNFRUVFAAAACNoAAIykAABEpgAATlN0M19fMjE1YmFzaWNfc3RyZWFtYnVmSWNOU18xMWNoYXJfdHJhaXRzSWNFRUVFAAAAAODZAADEpAAATlN0M19fMjEzYmFzaWNfaXN0cmVhbUljTlNfMTFjaGFyX3RyYWl0c0ljRUVFRQAAZNoAAAClAAAAAAAAAQAAALikAAAD9P//bAAAAAAAAADYpQAA7wAAAPAAAACU////lP///9ilAADxAAAA8gAAAFSlAACMpQAAoKUAAGilAABsAAAAAAAAADClAADhAAAA4gAAAJT///+U////MKUAAOMAAADkAAAATlN0M19fMjE0YmFzaWNfaWZzdHJlYW1JY05TXzExY2hhcl90cmFpdHNJY0VFRUUACNoAAKilAAAwpQAATlN0M19fMjEzYmFzaWNfZmlsZWJ1ZkljTlNfMTFjaGFyX3RyYWl0c0ljRUVFRQAACNoAAOSlAAD4pAAAAAAAAESmAADzAAAA9AAAAE5TdDNfXzI4aW9zX2Jhc2VFAAAA4NkAADCmAAAAAAAA0XSeAFedvSqAcFIP//8+JwoAAABkAAAA6AMAABAnAACghgEAQEIPAICWmAAA4fUFGAAAADUAAABxAAAAa////877//+Sv///AAAAAAAAAAD/////////////////////////////////////////////////////////////////AAECAwQFBgcICf////////8KCwwNDg8QERITFBUWFxgZGhscHR4fICEiI////////woLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////wABAgQHAwYFAAAAAAAAAAIAAMADAADABAAAwAUAAMAGAADABwAAwAgAAMAJAADACgAAwAsAAMAMAADADQAAwA4AAMAPAADAEAAAwBEAAMASAADAEwAAwBQAAMAVAADAFgAAwBcAAMAYAADAGQAAwBoAAMAbAADAHAAAwB0AAMAeAADAHwAAwAAAALMBAADDAgAAwwMAAMMEAADDBQAAwwYAAMMHAADDCAAAwwkAAMMKAADDCwAAwwwAAMMNAADTDgAAww8AAMMAAAy7AQAMwwIADMMDAAzDBAAM2wAAAADeEgSVAAAAAP///////////////4CoAAAUAAAAQy5VVEYtOAEClKgBSkxDX0NUWVBFAAAAAExDX05VTUVSSUMAAExDX1RJTUUAAAAAAExDX0NPTExBVEUAAExDX01PTkVUQVJZAExDX01FU1NBR0VTAECrAfkDAQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAAgAAAAJAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAABEAAAASAAAAEwAAABQAAAAVAAAAFgAAABcAAAAYAAAAGQAAABoAAAAbAAAAHAAAAB0AAAAeAAAAHwAAACAAAAAhAAAAIgAAACMAAAAkAAAAJQAAACYAAAAnAAAAKAAAACkAAAAqAAAAKwAAACwAAAAtAAAALgAAAC8AAAAwAAAAMQAAADIAAAAzAAAANAAAADUAAAA2AAAANwAAADgAAAA5AAAAOgAAADsAAAA8AAAAPQAAAD4AAAA/AAAAQAAAAEEAAABCAAAAQwAAAEQAAABFAAAARgAAAEcAAABIAAAASQAAAEoAAABLAAAATAAAAE0AAABOAAAATwAAAFAAAABRAAAAUgAAAFMAAABUAAAAVQAAAFYAAABXAAAAWAAAAFkAAABaAAAAWwAAAFwAAABdAAAAXgAAAF8AAABgAAAAQQAAAEIAAABDAAAARAAAAEUAAABGAAAARwAAAEgAAABJAAAASgAAAEsAAABMAAAATQAAAE4AAABPAAAAUAAAAFEAAABSAAAAUwAAAFQAAABVAAAAVgAAAFcAAABYAAAAWQAAAFoAAAB7AAAAfAAAAH0AAAB+AAAAfwECULEB+QMBAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAAEQAAABIAAAATAAAAFAAAABUAAAAWAAAAFwAAABgAAAAZAAAAGgAAABsAAAAcAAAAHQAAAB4AAAAfAAAAIAAAACEAAAAiAAAAIwAAACQAAAAlAAAAJgAAACcAAAAoAAAAKQAAACoAAAArAAAALAAAAC0AAAAuAAAALwAAADAAAAAxAAAAMgAAADMAAAA0AAAANQAAADYAAAA3AAAAOAAAADkAAAA6AAAAOwAAADwAAAA9AAAAPgAAAD8AAABAAAAAYQAAAGIAAABjAAAAZAAAAGUAAABmAAAAZwAAAGgAAABpAAAAagAAAGsAAABsAAAAbQAAAG4AAABvAAAAcAAAAHEAAAByAAAAcwAAAHQAAAB1AAAAdgAAAHcAAAB4AAAAeQAAAHoAAABbAAAAXAAAAF0AAABeAAAAXwAAAGAAAABhAAAAYgAAAGMAAABkAAAAZQAAAGYAAABnAAAAaAAAAGkAAABqAAAAawAAAGwAAABtAAAAbgAAAG8AAABwAAAAcQAAAHIAAABzAAAAdAAAAHUAAAB2AAAAdwAAAHgAAAB5AAAAegAAAHsAAAB8AAAAfQAAAH4AAAB/Ab0GMDEyMzQ1Njc4OWFiY2RlZkFCQ0RFRnhYKy1wUGlJbk4AJUk6JU06JVMgJXAlSDolTQAAAAAAAAAAAAAAAAAAACUAAABtAAAALwAAACUAAABkAAAALwAAACUAAAB5AAAAJQAAAFkAAAAtAAAAJQAAAG0AAAAtAAAAJQAAAGQAAAAlAAAASQAAADoAAAAlAAAATQAAADoAAAAlAAAAUwAAACAAAAAlAAAAcAAAAAAAAAAlAAAASAAAADoAAAAlAAAATQAAAAAAAAAAAAAAAAAAACUAAABIAAAAOgAAACUAAABNAAAAOgAAACUAAABTAAAAAAAAAJS/AAAIAQAACQEAAAoBAAAAAAAA9L8AAAsBAAAMAQAACgEAAA0BAAAOAQAADwEAABABAAARAQAAEgEAABMBAAAUAQAAAAAAAAAAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAFAgAABQAAAAUAAAAFAAAABQAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAMCAACCAAAAggAAAIIAAACCAAAAggAAAIIAAACCAAAAggAAAIIAAACCAAAAggAAAIIAAACCAAAAggAAAIIAAABCAQAAQgEAAEIBAABCAQAAQgEAAEIBAABCAQAAQgEAAEIBAABCAQAAggAAAIIAAACCAAAAggAAAIIAAACCAAAAggAAACoBAAAqAQAAKgEAACoBAAAqAQAAKgEAACoAAAAqAAAAKgAAACoAAAAqAAAAKgAAACoAAAAqAAAAKgAAACoAAAAqAAAAKgAAACoAAAAqAAAAKgAAACoAAAAqAAAAKgAAACoAAAAqAAAAggAAAIIAAACCAAAAggAAAIIAAACCAAAAMgEAADIBAAAyAQAAMgEAADIBAAAyAQAAMgAAADIAAAAyAAAAMgAAADIAAAAyAAAAMgAAADIAAAAyAAAAMgAAADIAAAAyAAAAMgAAADIAAAAyAAAAMgAAADIAAAAyAAAAMgAAADIAAACCAAAAggAAAIIAAACCAAAABAHnOFy/AAAVAQAAFgEAAAoBAAAXAQAAGAEAABkBAAAaAQAAGwEAABwBAAAdAQAAAAAAACzAAAAeAQAAHwEAAAoBAAAgAQAAIQEAACIBAAAjAQAAJAEAAAAAAABQwAAAJQEAACYBAAAKAQAAJwEAACgBAAApAQAAKgEAACsBAAB0AAAAcgAAAHUAAABlAAAAAAAAAGYAAABhAAAAbAAAAHMAAABlAAAAAAAAACUAAABtAAAALwAAACUAAABkAAAALwAAACUAAAB5AAAAAAAAACUAAABIAAAAOgAAACUAAABNAAAAOgAAACUAAABTAAAAAAAAACUAAABhAAAAIAAAACUAAABiAAAAIAAAACUAAABkAAAAIAAAACUAAABIAAAAOgAAACUAAABNAAAAOgAAACUAAABTAAAAIAAAACUAAABZAAAAAAAAACUAAABJAAAAOgAAACUAAABNAAAAOgAAACUAAABTAAAAIAAAACUAAABwAAAAAAAAAAAAAAA0vAAALAEAAC0BAAAKAQAATlN0M19fMjZsb2NhbGU1ZmFjZXRFAAAACNoAABy8AABg0AAAAAAAALS8AAAsAQAALgEAAAoBAAAvAQAAMAEAADEBAAAyAQAAMwEAADQBAAA1AQAANgEAADcBAAA4AQAAOQEAADoBAABOU3QzX18yNWN0eXBlSXdFRQBOU3QzX18yMTBjdHlwZV9iYXNlRQAA4NkAAJa8AABk2gAAhLwAAAAAAAACAAAANLwAAAIAAACsvAAAAgAAAAAAAABIvQAALAEAADsBAAAKAQAAPAEAAD0BAAA+AQAAPwEAAEABAABBAQAAQgEAAE5TdDNfXzI3Y29kZWN2dEljYzExX19tYnN0YXRlX3RFRQBOU3QzX18yMTJjb2RlY3Z0X2Jhc2VFAAAAAODZAAAmvQAAZNoAAAS9AAAAAAAAAgAAADS8AAACAAAAQL0AAAIAAAAAAAAAvL0AACwBAABDAQAACgEAAEQBAABFAQAARgEAAEcBAABIAQAASQEAAEoBAABOU3QzX18yN2NvZGVjdnRJRHNjMTFfX21ic3RhdGVfdEVFAABk2gAAmL0AAAAAAAACAAAANLwAAAIAAABAvQAAAgAAAAAAAAAwvgAALAEAAEsBAAAKAQAATAEAAE0BAABOAQAATwEAAFABAABRAQAAUgEAAE5TdDNfXzI3Y29kZWN2dElEc0R1MTFfX21ic3RhdGVfdEVFAGTaAAAMvgAAAAAAAAIAAAA0vAAAAgAAAEC9AAACAAAAAAAAAKS+AAAsAQAAUwEAAAoBAABUAQAAVQEAAFYBAABXAQAAWAEAAFkBAABaAQAATlN0M19fMjdjb2RlY3Z0SURpYzExX19tYnN0YXRlX3RFRQAAZNoAAIC+AAAAAAAAAgAAADS8AAACAAAAQL0AAAIAAAAAAAAAGL8AACwBAABbAQAACgEAAFwBAABdAQAAXgEAAF8BAABgAQAAYQEAAGIBAABOU3QzX18yN2NvZGVjdnRJRGlEdTExX19tYnN0YXRlX3RFRQBk2gAA9L4AAAAAAAACAAAANLwAAAIAAABAvQAAAgAAAE5TdDNfXzI3Y29kZWN2dEl3YzExX19tYnN0YXRlX3RFRQAAAGTaAAA4vwAAAAAAAAIAAAA0vAAAAgAAAEC9AAACAAAATlN0M19fMjZsb2NhbGU1X19pbXBFAAAACNoAAHy/AAA0vAAATlN0M19fMjdjb2xsYXRlSWNFRQAI2gAAoL8AADS8AABOU3QzX18yN2NvbGxhdGVJd0VFAAjaAADAvwAANLwAAE5TdDNfXzI1Y3R5cGVJY0VFAAAAZNoAAOC/AAAAAAAAAgAAADS8AAACAAAArLwAAAIAAABOU3QzX18yOG51bXB1bmN0SWNFRQAAAAAI2gAAFMAAADS8AABOU3QzX18yOG51bXB1bmN0SXdFRQAAAAAI2gAAOMAAADS8AAAAAAAAtL8AAGMBAABkAQAACgEAAGUBAABmAQAAZwEAAAAAAADUvwAAaAEAAGkBAAAKAQAAagEAAGsBAABsAQAAAAAAAHDBAAAsAQAAbQEAAAoBAABuAQAAbwEAAHABAABxAQAAcgEAAHMBAAB0AQAAdQEAAHYBAAB3AQAAeAEAAE5TdDNfXzI3bnVtX2dldEljTlNfMTlpc3RyZWFtYnVmX2l0ZXJhdG9ySWNOU18xMWNoYXJfdHJhaXRzSWNFRUVFRUUATlN0M19fMjlfX251bV9nZXRJY0VFAE5TdDNfXzIxNF9fbnVtX2dldF9iYXNlRQAA4NkAADbBAABk2gAAIMEAAAAAAAABAAAAUMEAAAAAAABk2gAA3MAAAAAAAAACAAAANLwAAAIAAABYwQAAAAAAAAAAAABEwgAALAEAAHkBAAAKAQAAegEAAHsBAAB8AQAAfQEAAH4BAAB/AQAAgAEAAIEBAACCAQAAgwEAAIQBAABOU3QzX18yN251bV9nZXRJd05TXzE5aXN0cmVhbWJ1Zl9pdGVyYXRvckl3TlNfMTFjaGFyX3RyYWl0c0l3RUVFRUVFAE5TdDNfXzI5X19udW1fZ2V0SXdFRQAAAGTaAAAUwgAAAAAAAAEAAABQwQAAAAAAAGTaAADQwQAAAAAAAAIAAAA0vAAAAgAAACzCAAAAAAAAAAAAACzDAAAsAQAAhQEAAAoBAACGAQAAhwEAAIgBAACJAQAAigEAAIsBAACMAQAAjQEAAE5TdDNfXzI3bnVtX3B1dEljTlNfMTlvc3RyZWFtYnVmX2l0ZXJhdG9ySWNOU18xMWNoYXJfdHJhaXRzSWNFRUVFRUUATlN0M19fMjlfX251bV9wdXRJY0VFAE5TdDNfXzIxNF9fbnVtX3B1dF9iYXNlRQAA4NkAAPLCAABk2gAA3MIAAAAAAAABAAAADMMAAAAAAABk2gAAmMIAAAAAAAACAAAANLwAAAIAAAAUwwAAAAAAAAAAAAD0wwAALAEAAI4BAAAKAQAAjwEAAJABAACRAQAAkgEAAJMBAACUAQAAlQEAAJYBAABOU3QzX18yN251bV9wdXRJd05TXzE5b3N0cmVhbWJ1Zl9pdGVyYXRvckl3TlNfMTFjaGFyX3RyYWl0c0l3RUVFRUVFAE5TdDNfXzI5X19udW1fcHV0SXdFRQAAAGTaAADEwwAAAAAAAAEAAAAMwwAAAAAAAGTaAACAwwAAAAAAAAIAAAA0vAAAAgAAANzDAAAAAAAAAAAAAPTEAACXAQAAmAEAAAoBAACZAQAAmgEAAJsBAACcAQAAnQEAAJ4BAACfAQAA+P////TEAACgAQAAoQEAAKIBAACjAQAApAEAAKUBAACmAQAATlN0M19fMjh0aW1lX2dldEljTlNfMTlpc3RyZWFtYnVmX2l0ZXJhdG9ySWNOU18xMWNoYXJfdHJhaXRzSWNFRUVFRUUATlN0M19fMjl0aW1lX2Jhc2VFAODZAACtxAAATlN0M19fMjIwX190aW1lX2dldF9jX3N0b3JhZ2VJY0VFAAAA4NkAAMjEAABk2gAAaMQAAAAAAAADAAAANLwAAAIAAADAxAAAAgAAAOzEAAAACAAAAAAAAODFAACnAQAAqAEAAAoBAACpAQAAqgEAAKsBAACsAQAArQEAAK4BAACvAQAA+P///+DFAACwAQAAsQEAALIBAACzAQAAtAEAALUBAAC2AQAATlN0M19fMjh0aW1lX2dldEl3TlNfMTlpc3RyZWFtYnVmX2l0ZXJhdG9ySXdOU18xMWNoYXJfdHJhaXRzSXdFRUVFRUUATlN0M19fMjIwX190aW1lX2dldF9jX3N0b3JhZ2VJd0VFAADg2QAAtcUAAGTaAABwxQAAAAAAAAMAAAA0vAAAAgAAAMDEAAACAAAA2MUAAAAIAAAAAAAAhMYAALcBAAC4AQAACgEAALkBAABOU3QzX18yOHRpbWVfcHV0SWNOU18xOW9zdHJlYW1idWZfaXRlcmF0b3JJY05TXzExY2hhcl90cmFpdHNJY0VFRUVFRQBOU3QzX18yMTBfX3RpbWVfcHV0RQAAAODZAABlxgAAZNoAACDGAAAAAAAAAgAAADS8AAACAAAAfMYAAAAIAAAAAAAABMcAALoBAAC7AQAACgEAALwBAABOU3QzX18yOHRpbWVfcHV0SXdOU18xOW9zdHJlYW1idWZfaXRlcmF0b3JJd05TXzExY2hhcl90cmFpdHNJd0VFRUVFRQAAAABk2gAAvMYAAAAAAAACAAAANLwAAAIAAAB8xgAAAAgAAAAAAACYxwAALAEAAL0BAAAKAQAAvgEAAL8BAADAAQAAwQEAAMIBAADDAQAAxAEAAMUBAADGAQAATlN0M19fMjEwbW9uZXlwdW5jdEljTGIwRUVFAE5TdDNfXzIxMG1vbmV5X2Jhc2VFAAAAAODZAAB4xwAAZNoAAFzHAAAAAAAAAgAAADS8AAACAAAAkMcAAAIAAAAAAAAADMgAACwBAADHAQAACgEAAMgBAADJAQAAygEAAMsBAADMAQAAzQEAAM4BAADPAQAA0AEAAE5TdDNfXzIxMG1vbmV5cHVuY3RJY0xiMUVFRQBk2gAA8McAAAAAAAACAAAANLwAAAIAAACQxwAAAgAAAAAAAACAyAAALAEAANEBAAAKAQAA0gEAANMBAADUAQAA1QEAANYBAADXAQAA2AEAANkBAADaAQAATlN0M19fMjEwbW9uZXlwdW5jdEl3TGIwRUVFAGTaAABkyAAAAAAAAAIAAAA0vAAAAgAAAJDHAAACAAAAAAAAAPTIAAAsAQAA2wEAAAoBAADcAQAA3QEAAN4BAADfAQAA4AEAAOEBAADiAQAA4wEAAOQBAABOU3QzX18yMTBtb25leXB1bmN0SXdMYjFFRUUAZNoAANjIAAAAAAAAAgAAADS8AAACAAAAkMcAAAIAAAAAAAAAmMkAACwBAADlAQAACgEAAOYBAADnAQAATlN0M19fMjltb25leV9nZXRJY05TXzE5aXN0cmVhbWJ1Zl9pdGVyYXRvckljTlNfMTFjaGFyX3RyYWl0c0ljRUVFRUVFAE5TdDNfXzIxMV9fbW9uZXlfZ2V0SWNFRQAA4NkAAHbJAABk2gAAMMkAAAAAAAACAAAANLwAAAIAAACQyQAAAAAAAAAAAAA8ygAALAEAAOgBAAAKAQAA6QEAAOoBAABOU3QzX18yOW1vbmV5X2dldEl3TlNfMTlpc3RyZWFtYnVmX2l0ZXJhdG9ySXdOU18xMWNoYXJfdHJhaXRzSXdFRUVFRUUATlN0M19fMjExX19tb25leV9nZXRJd0VFAADg2QAAGsoAAGTaAADUyQAAAAAAAAIAAAA0vAAAAgAAADTKAAAAAAAAAAAAAODKAAAsAQAA6wEAAAoBAADsAQAA7QEAAE5TdDNfXzI5bW9uZXlfcHV0SWNOU18xOW9zdHJlYW1idWZfaXRlcmF0b3JJY05TXzExY2hhcl90cmFpdHNJY0VFRUVFRQBOU3QzX18yMTFfX21vbmV5X3B1dEljRUUAAODZAAC+ygAAZNoAAHjKAAAAAAAAAgAAADS8AAACAAAA2MoAAAAAAAAAAAAAhMsAACwBAADuAQAACgEAAO8BAADwAQAATlN0M19fMjltb25leV9wdXRJd05TXzE5b3N0cmVhbWJ1Zl9pdGVyYXRvckl3TlNfMTFjaGFyX3RyYWl0c0l3RUVFRUVFAE5TdDNfXzIxMV9fbW9uZXlfcHV0SXdFRQAA4NkAAGLLAABk2gAAHMsAAAAAAAACAAAANLwAAAIAAAB8ywAAAAAAAAAAAAD8ywAALAEAAPEBAAAKAQAA8gEAAPMBAAD0AQAATlN0M19fMjhtZXNzYWdlc0ljRUUATlN0M19fMjEzbWVzc2FnZXNfYmFzZUUAAAAA4NkAANnLAABk2gAAxMsAAAAAAAACAAAANLwAAAIAAAD0ywAAAgAAAAAAAABUzAAALAEAAPUBAAAKAQAA9gEAAPcBAAD4AQAATlN0M19fMjhtZXNzYWdlc0l3RUUAAAAAZNoAADzMAAAAAAAAAgAAADS8AAACAAAA9MsAAAIAAABTAAAAdQAAAG4AAABkAAAAYQAAAHkAAAAAAAAATQAAAG8AAABuAAAAZAAAAGEAAAB5AAAAAAAAAFQAAAB1AAAAZQAAAHMAAABkAAAAYQAAAHkAAAAAAAAAVwAAAGUAAABkAAAAbgAAAGUAAABzAAAAZAAAAGEAAAB5AAAAAAAAAFQAAABoAAAAdQAAAHIAAABzAAAAZAAAAGEAAAB5AAAAAAAAAEYAAAByAAAAaQAAAGQAAABhAAAAeQAAAAAAAABTAAAAYQAAAHQAAAB1AAAAcgAAAGQAAABhAAAAeQAAAAAAAABTAAAAdQAAAG4AAAAAAAAATQAAAG8AAABuAAAAAAAAAFQAAAB1AAAAZQAAAAAAAABXAAAAZQAAAGQAAAAAAAAAVAAAAGgAAAB1AAAAAAAAAEYAAAByAAAAaQAAAAAAAABTAAAAYQAAAHQAAAAAAAAASgAAAGEAAABuAAAAdQAAAGEAAAByAAAAeQAAAAAAAABGAAAAZQAAAGIAAAByAAAAdQAAAGEAAAByAAAAeQAAAAAAAABNAAAAYQAAAHIAAABjAAAAaAAAAAAAAABBAAAAcAAAAHIAAABpAAAAbAAAAAAAAABNAAAAYQAAAHkAAAAAAAAASgAAAHUAAABuAAAAZQAAAAAAAABKAAAAdQAAAGwAAAB5AAAAAAAAAEEAAAB1AAAAZwAAAHUAAABzAAAAdAAAAAAAAABTAAAAZQAAAHAAAAB0AAAAZQAAAG0AAABiAAAAZQAAAHIAAAAAAAAATwAAAGMAAAB0AAAAbwAAAGIAAABlAAAAcgAAAAAAAABOAAAAbwAAAHYAAABlAAAAbQAAAGIAAABlAAAAcgAAAAAAAABEAAAAZQAAAGMAAABlAAAAbQAAAGIAAABlAAAAcgAAAAAAAABKAAAAYQAAAG4AAAAAAAAARgAAAGUAAABiAAAAAAAAAE0AAABhAAAAcgAAAAAAAABBAAAAcAAAAHIAAAAAAAAASgAAAHUAAABuAAAAAAAAAEoAAAB1AAAAbAAAAAAAAABBAAAAdQAAAGcAAAAAAAAAUwAAAGUAAABwAAAAAAAAAE8AAABjAAAAdAAAAAAAAABOAAAAbwAAAHYAAAAAAAAARAAAAGUAAABjAAAAAAAAAEEAAABNAAAAAAAAAFAAAABNAAAAAAAAAAAAAADsxAAAoAEAAKEBAACiAQAAowEAAKQBAAClAQAApgEAAAAAAADYxQAAsAEAALEBAACyAQAAswEAALQBAAC1AQAAtgEAAAAAAABg0AAAXQAAAPkBAAD6AQAATlN0M19fMjE0X19zaGFyZWRfY291bnRFAAAAAODZAABE0AAATlN0M19fMjE5X19zaGFyZWRfd2Vha19jb3VudEUAAABk2gAAaNAAAAAAAAABAAAAYNAAAAAAAAAAAAAA0NQAAFIAAAD7AQAA/AEAAAAAAAAAAAAAAAAAAOsZAABBAAAA6RkAAEIAAADlGQAAQwAAAOEZAABEAAAAPhkAAEUAAAASGQAARgAAAPsYAABHAAAA7RgAAEgAAADrGAAASQAAAOUYAABKAAAA4xgAAEsAAABTGAAATAAAAJgXAABNAAAAZBcAAE4AAACaFwAAAAAAAD0XAABPAAAALhcAAFAAAAAkFwAAUQAAACIXAABSAAAAHhcAAFMAAABIFgAAVAAAALUVAABVAAAAphUAAFYAAACiFQAAVwAAAJ0VAABYAAAAhxUAAFkAAAA9FQAAWgAAAKIUAABhAAAAhAUAAAcAAAAJEgAAJgAAAJwRAAAnAAAATg0AACoAAABWFAAAYgAAAF4OAABcAAAA1REAAAgAAAAVFAAAYwAAACULAAANAAAACwUAAF4AAAC4BQAAXgAAAEALAAA6AAAAeRQAACwAAADcBwAAQAAAAP4SAABkAAAAxAsAACQAAADoEQAAZQAAAF4HAAA4AAAArAsAAD0AAAB0DQAAIQAAAD4PAABmAAAAjw8AADUAAAC6EgAADAAAALwJAAA0AAAA4goAAC4AAAAwDwAAZwAAAMoFAABgAAAA3wsAAD4AAACMDgAAaAAAAP8LAAAtAAAA8wcAAC0AAAD1DQAAaQAAAKQNAABqAAAAog0AAGsAAABEDQAAbAAAAMoRAAB7AAAAfAcAAHsAAABvCAAAKAAAAKQHAABbAAAA0AsAADwAAABvEQAAXwAAAOcMAABtAAAA0AwAAG4AAABnEQAACgAAAGIRAAA5AAAAuAsAACMAAAATCwAAbwAAAF4RAAAxAAAA+goAAHAAAACVCwAAJQAAAOoRAAAuAAAAogsAACsAAACkCgAAcQAAAFcNAAA/AAAAZQ0AACIAAACiCgAAcgAAAAAIAABcAAAAvhEAAH0AAABoBwAAfQAAAF0IAAApAAAAjwcAAF0AAAC6CQAAcwAAADwLAAA7AAAA8QsAADcAAADRBAAANgAAAGIOAAAvAAAACAgAAC8AAADZEQAAIAAAAPEHAAB0AAAAKhQAAAkAAACtEQAAMwAAALgRAAB+AAAA/AoAADIAAABVBQAAdQAAAPcQAABfAAAASgUAAHYAAAB4EQAAfAAAACEUAAALAAAAPAUAAHcAAAAxBQAAeAAAAMEEAAB5AAAAQgQAAHoAAAAMCwAAMAAAAAAAAAAAAAAA0gwAAGAAAACPFAAAIAAAAIUNAAAAAgAA7QwAAAQAAAD+EgAAQAAAAPMFAABAAAAAaA4AAOAAAAA3CgAAEAAAAKUFAAACAAAAuAcAAIAAAAC6CQAAAQAAANkRAAABAAAAPQoAAAgAAAA8BQAAAAQAAPIFAAAAAQAATlN0M19fMjExcmVnZXhfZXJyb3JFAAAACNoAALjUAAAQ3AAAqiMAAOYjAAAIIwAAmCQAAPkkAAAmJQAAISEAAPghAABuIQAAVSMAAHAiAACsIgAAICQAAMwkAADEIQAARhEAADYiAAAAAAAACgAAAGQAAADoAwAAECcAAKCGAQBAQg8AgJaYAADh9QUAypo7AAAAAAAAAAAwMDAxMDIwMzA0MDUwNjA3MDgwOTEwMTExMjEzMTQxNTE2MTcxODE5MjAyMTIyMjMyNDI1MjYyNzI4MjkzMDMxMzIzMzM0MzUzNjM3MzgzOTQwNDE0MjQzNDQ0NTQ2NDc0ODQ5NTA1MTUyNTM1NDU1NTY1NzU4NTk2MDYxNjI2MzY0NjU2NjY3Njg2OTcwNzE3MjczNzQ3NTc2Nzc3ODc5ODA4MTgyODM4NDg1ODY4Nzg4ODk5MDkxOTI5Mzk0OTU5Njk3OTg5OQAAAAAAAAAAAv8ABGQAIAAABP//BgABAAEAAQD//wH/Af//////Af8B/wH/Af8B/wH/Af8B//////8K/yAA//8D/wH/BP8eAAABBf//////YwAACGMA6AMCAAAA//////8AAAAB/wH//////////////wAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAB/wH//////wABIAAEAIAAAAj//wH/Af////////8B/wb/B/8I/wn//////7wCvAIBAP//AQABAP//AAD//////////wAAAAAAAAAAAAAAAAAAAAAUAXj//wEACv///////////wH/Af8AAAAAAAAB/wH/Af8AAAAAAAAAAAAAAAAAAAAAAAAB/wAAAAAAAAH/Af8BAAAAAQAAAAH//////wAAAAAB////AAAAAP////////////8oAAr//////wEACv////8A//////////8BvggB/wH///8BAP//////////////////Cv//////DP8N/04xMF9fY3h4YWJpdjExNl9fc2hpbV90eXBlX2luZm9FAAAI2gAAFtgAACzcAABOMTBfX2N4eGFiaXYxMTdfX2NsYXNzX3R5cGVfaW5mb0UAAAAI2gAARNgAADjYAABOMTBfX2N4eGFiaXYxMTdfX3BiYXNlX3R5cGVfaW5mb0UAAAAI2gAAdNgAADjYAABOMTBfX2N4eGFiaXYxMTlfX3BvaW50ZXJfdHlwZV9pbmZvRQAI2gAApNgAAJjYAAAAAAAAGNkAAP8BAAAAAgAAAQIAAAICAAADAgAATjEwX19jeHhhYml2MTIzX19mdW5kYW1lbnRhbF90eXBlX2luZm9FAAjaAADw2AAAONgAAHYAAADc2AAAJNkAAGIAAADc2AAAMNkAAGMAAADc2AAAPNkAAGgAAADc2AAASNkAAGEAAADc2AAAVNkAAHMAAADc2AAAYNkAAHQAAADc2AAAbNkAAGkAAADc2AAAeNkAAGoAAADc2AAAhNkAAGwAAADc2AAAkNkAAG0AAADc2AAAnNkAAHgAAADc2AAAqNkAAHkAAADc2AAAtNkAAGYAAADc2AAAwNkAAGQAAADc2AAAzNkAAAAAAABo2AAA/wEAAAQCAAABAgAAAgIAAAUCAAAGAgAABwIAAAgCAAAAAAAAUNoAAP8BAAAJAgAAAQIAAAICAAAFAgAACgIAAAsCAAAMAgAATjEwX19jeHhhYml2MTIwX19zaV9jbGFzc190eXBlX2luZm9FAAAAAAjaAAAo2gAAaNgAAAAAAACs2gAA/wEAAA0CAAABAgAAAgIAAAUCAAAOAgAADwIAABACAABOMTBfX2N4eGFiaXYxMjFfX3ZtaV9jbGFzc190eXBlX2luZm9FAAAACNoAAITaAABo2AAAAAAAABzbAAAIAAAAEQIAABICAAAAAAAARNsAAAgAAAATAgAAFAIAAAAAAAAE2wAACAAAABUCAAAWAgAAU3Q5ZXhjZXB0aW9uAAAAAODZAAD02gAAU3Q5YmFkX2FsbG9jAAAAAAjaAAAM2wAABNsAAFN0MjBiYWRfYXJyYXlfbmV3X2xlbmd0aAAAAAAI2gAAKNsAABzbAAAAAAAAiNsAAAcAAAAXAgAAGAIAAAAAAAAQ3AAAGQIAABoCAAD8AQAAU3QxMWxvZ2ljX2Vycm9yAAjaAAB42wAABNsAAAAAAAC82wAABwAAABsCAAAYAgAAU3QxMmxlbmd0aF9lcnJvcgAAAAAI2gAAqNsAAIjbAAAAAAAA8NsAAAcAAAAcAgAAGAIAAFN0MTJvdXRfb2ZfcmFuZ2UAAAAACNoAANzbAACI2wAAU3QxM3J1bnRpbWVfZXJyb3IAAAAI2gAA/NsAAATbAABTdDl0eXBlX2luZm8AAAAA4NkAABzcAAABFT0AAAA+AAAAAAAAAD8AAAAAAAAAWAFoUNwAAFDcAAAAAAEAAAIAAAAAAAAFAAAAAAAAAAAAAAC9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC7AAAAugAAAOA9DQAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAA//////////8BWZjcAAAAAAAABQAAAAAAAAAAAAAAywAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuwAAAMwAAADoPQ0AAAQAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAP////8KAQww3QAAwEwOAP4BAAA=\";if(!isDataURI(wasmBinaryFile)){wasmBinaryFile=locateFile(wasmBinaryFile)}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}var binary=tryParseAsDataURI(file);if(binary){return binary}if(readBinary){return readBinary(file)}throw\"both async and sync fetching of the wasm failed\"}function getBinaryPromise(binaryFile){return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(instance=>instance).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){return instantiateArrayBuffer(binaryFile,imports,callback)}function createWasm(){var info={\"a\":wasmImports};function receiveInstance(instance,module){var exports=instance.exports;wasmExports=exports;registerTLSInit(wasmExports[\"_\"]);wasmTable=wasmExports[\"X\"];addOnInit(wasmExports[\"W\"]);wasmModule=module;removeRunDependency(\"wasm-instantiate\");return exports}addRunDependency(\"wasm-instantiate\");function receiveInstantiationResult(result){receiveInstance(result[\"instance\"],result[\"module\"])}if(Module[\"instantiateWasm\"]){try{return Module[\"instantiateWasm\"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);readyPromiseReject(e)}}instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult).catch(readyPromiseReject);return{}}var tempDouble;var tempI64;function ExitStatus(status){this.name=\"ExitStatus\";this.message=`Program terminated with exit(${status})`;this.status=status}var terminateWorker=worker=>{worker.terminate();worker.onmessage=e=>{}};var killThread=pthread_ptr=>{var worker=PThread.pthreads[pthread_ptr];delete PThread.pthreads[pthread_ptr];terminateWorker(worker);__emscripten_thread_free_data(pthread_ptr);PThread.runningWorkers.splice(PThread.runningWorkers.indexOf(worker),1);worker.pthread_ptr=0};var cancelThread=pthread_ptr=>{var worker=PThread.pthreads[pthread_ptr];worker.postMessage({\"cmd\":\"cancel\"})};var cleanupThread=pthread_ptr=>{var worker=PThread.pthreads[pthread_ptr];assert(worker);PThread.returnWorkerToPool(worker)};var spawnThread=threadParams=>{var worker=PThread.getNewWorker();if(!worker){return 6}PThread.runningWorkers.push(worker);PThread.pthreads[threadParams.pthread_ptr]=worker;worker.pthread_ptr=threadParams.pthread_ptr;var msg={\"cmd\":\"run\",\"start_routine\":threadParams.startRoutine,\"arg\":threadParams.arg,\"pthread_ptr\":threadParams.pthread_ptr};if(ENVIRONMENT_IS_NODE){worker.unref()}worker.postMessage(msg,threadParams.transferList);return 0};var PATH={isAbs:path=>path.charAt(0)===\"/\",splitPath:filename=>{var splitPathRe=/^(\\/?|)([\\s\\S]*?)((?:\\.{1,2}|[^\\/]+?|)(\\.[^.\\/]*|))(?:[\\/]*)$/;return splitPathRe.exec(filename).slice(1)},normalizeArray:(parts,allowAboveRoot)=>{var up=0;for(var i=parts.length-1;i>=0;i--){var last=parts[i];if(last===\".\"){parts.splice(i,1)}else if(last===\"..\"){parts.splice(i,1);up++}else if(up){parts.splice(i,1);up--}}if(allowAboveRoot){for(;up;up--){parts.unshift(\"..\")}}return parts},normalize:path=>{var isAbsolute=PATH.isAbs(path),trailingSlash=path.substr(-1)===\"/\";path=PATH.normalizeArray(path.split(\"/\").filter(p=>!!p),!isAbsolute).join(\"/\");if(!path&&!isAbsolute){path=\".\"}if(path&&trailingSlash){path+=\"/\"}return(isAbsolute?\"/\":\"\")+path},dirname:path=>{var result=PATH.splitPath(path),root=result[0],dir=result[1];if(!root&&!dir){return\".\"}if(dir){dir=dir.substr(0,dir.length-1)}return root+dir},basename:path=>{if(path===\"/\")return\"/\";path=PATH.normalize(path);path=path.replace(/\\/$/,\"\");var lastSlash=path.lastIndexOf(\"/\");if(lastSlash===-1)return path;return path.substr(lastSlash+1)},join:function(){var paths=Array.prototype.slice.call(arguments);return PATH.normalize(paths.join(\"/\"))},join2:(l,r)=>PATH.normalize(l+\"/\"+r)};var initRandomFill=()=>{if(typeof crypto==\"object\"&&typeof crypto[\"getRandomValues\"]==\"function\"){return view=>(view.set(crypto.getRandomValues(new Uint8Array(view.byteLength))),view)}else if(ENVIRONMENT_IS_NODE){try{var crypto_module=require(\"crypto\");var randomFillSync=crypto_module[\"randomFillSync\"];if(randomFillSync){return view=>crypto_module[\"randomFillSync\"](view)}var randomBytes=crypto_module[\"randomBytes\"];return view=>(view.set(randomBytes(view.byteLength)),view)}catch(e){}}abort(\"initRandomDevice\")};var randomFill=view=>(randomFill=initRandomFill())(view);var PATH_FS={resolve:function(){var resolvedPath=\"\",resolvedAbsolute=false;for(var i=arguments.length-1;i>=-1&&!resolvedAbsolute;i--){var path=i>=0?arguments[i]:FS.cwd();if(typeof path!=\"string\"){throw new TypeError(\"Arguments to path.resolve must be strings\")}else if(!path){return\"\"}resolvedPath=path+\"/\"+resolvedPath;resolvedAbsolute=PATH.isAbs(path)}resolvedPath=PATH.normalizeArray(resolvedPath.split(\"/\").filter(p=>!!p),!resolvedAbsolute).join(\"/\");return(resolvedAbsolute?\"/\":\"\")+resolvedPath||\".\"},relative:(from,to)=>{from=PATH_FS.resolve(from).substr(1);to=PATH_FS.resolve(to).substr(1);function trim(arr){var start=0;for(;start<arr.length;start++){if(arr[start]!==\"\")break}var end=arr.length-1;for(;end>=0;end--){if(arr[end]!==\"\")break}if(start>end)return[];return arr.slice(start,end-start+1)}var fromParts=trim(from.split(\"/\"));var toParts=trim(to.split(\"/\"));var length=Math.min(fromParts.length,toParts.length);var samePartsLength=length;for(var i=0;i<length;i++){if(fromParts[i]!==toParts[i]){samePartsLength=i;break}}var outputParts=[];for(var i=samePartsLength;i<fromParts.length;i++){outputParts.push(\"..\")}outputParts=outputParts.concat(toParts.slice(samePartsLength));return outputParts.join(\"/\")}};var UTF8Decoder=typeof TextDecoder!=\"undefined\"?new TextDecoder(\"utf8\"):undefined;var UTF8ArrayToString=(heapOrArray,idx,maxBytesToRead)=>{var endIdx=idx+maxBytesToRead;var endPtr=idx;while(heapOrArray[endPtr]&&!(endPtr>=endIdx))++endPtr;if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.buffer instanceof SharedArrayBuffer?heapOrArray.slice(idx,endPtr):heapOrArray.subarray(idx,endPtr))}var str=\"\";while(idx<endPtr){var u0=heapOrArray[idx++];if(!(u0&128)){str+=String.fromCharCode(u0);continue}var u1=heapOrArray[idx++]&63;if((u0&224)==192){str+=String.fromCharCode((u0&31)<<6|u1);continue}var u2=heapOrArray[idx++]&63;if((u0&240)==224){u0=(u0&15)<<12|u1<<6|u2}else{u0=(u0&7)<<18|u1<<12|u2<<6|heapOrArray[idx++]&63}if(u0<65536){str+=String.fromCharCode(u0)}else{var ch=u0-65536;str+=String.fromCharCode(55296|ch>>10,56320|ch&1023)}}return str};var FS_stdin_getChar_buffer=[];var lengthBytesUTF8=str=>{var len=0;for(var i=0;i<str.length;++i){var c=str.charCodeAt(i);if(c<=127){len++}else if(c<=2047){len+=2}else if(c>=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i<str.length;++i){var u=str.charCodeAt(i);if(u>=55296&&u<=57343){var u1=str.charCodeAt(++i);u=65536+((u&1023)<<10)|u1&1023}if(u<=127){if(outIdx>=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}}heap[outIdx]=0;return outIdx-startIdx};function intArrayFromString(stringy,dontAddNull,length){var len=length>0?length:lengthBytesUTF8(stringy)+1;var u8array=new Array(len);var numBytesWritten=stringToUTF8Array(stringy,u8array,0,u8array.length);if(dontAddNull)u8array.length=numBytesWritten;return u8array}var FS_stdin_getChar=()=>{if(!FS_stdin_getChar_buffer.length){var result=null;if(ENVIRONMENT_IS_NODE){var BUFSIZE=256;var buf=Buffer.alloc(BUFSIZE);var bytesRead=0;var fd=process.stdin.fd;try{bytesRead=fs.readSync(fd,buf)}catch(e){if(e.toString().includes(\"EOF\"))bytesRead=0;else throw e}if(bytesRead>0){result=buf.slice(0,bytesRead).toString(\"utf-8\")}else{result=null}}else if(typeof window!=\"undefined\"&&typeof window.prompt==\"function\"){result=window.prompt(\"Input: \");if(result!==null){result+=\"\\n\"}}else if(typeof readline==\"function\"){result=readline();if(result!==null){result+=\"\\n\"}}if(!result){return null}FS_stdin_getChar_buffer=intArrayFromString(result,true)}return FS_stdin_getChar_buffer.shift()};var TTY={ttys:[],init(){},shutdown(){},register(dev,ops){TTY.ttys[dev]={input:[],output:[],ops:ops};FS.registerDevice(dev,TTY.stream_ops)},stream_ops:{open(stream){var tty=TTY.ttys[stream.node.rdev];if(!tty){throw new FS.ErrnoError(43)}stream.tty=tty;stream.seekable=false},close(stream){stream.tty.ops.fsync(stream.tty)},fsync(stream){stream.tty.ops.fsync(stream.tty)},read(stream,buffer,offset,length,pos){if(!stream.tty||!stream.tty.ops.get_char){throw new FS.ErrnoError(60)}var bytesRead=0;for(var i=0;i<length;i++){var result;try{result=stream.tty.ops.get_char(stream.tty)}catch(e){throw new FS.ErrnoError(29)}if(result===undefined&&bytesRead===0){throw new FS.ErrnoError(6)}if(result===null||result===undefined)break;bytesRead++;buffer[offset+i]=result}if(bytesRead){stream.node.timestamp=Date.now()}return bytesRead},write(stream,buffer,offset,length,pos){if(!stream.tty||!stream.tty.ops.put_char){throw new FS.ErrnoError(60)}try{for(var i=0;i<length;i++){stream.tty.ops.put_char(stream.tty,buffer[offset+i])}}catch(e){throw new FS.ErrnoError(29)}if(length){stream.node.timestamp=Date.now()}return i}},default_tty_ops:{get_char(tty){return FS_stdin_getChar()},put_char(tty,val){if(val===null||val===10){out(UTF8ArrayToString(tty.output,0));tty.output=[]}else{if(val!=0)tty.output.push(val)}},fsync(tty){if(tty.output&&tty.output.length>0){out(UTF8ArrayToString(tty.output,0));tty.output=[]}},ioctl_tcgets(tty){return{c_iflag:25856,c_oflag:5,c_cflag:191,c_lflag:35387,c_cc:[3,28,127,21,4,0,1,0,17,19,26,0,18,15,23,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},ioctl_tcsets(tty,optional_actions,data){return 0},ioctl_tiocgwinsz(tty){return[24,80]}},default_tty1_ops:{put_char(tty,val){if(val===null||val===10){err(UTF8ArrayToString(tty.output,0));tty.output=[]}else{if(val!=0)tty.output.push(val)}},fsync(tty){if(tty.output&&tty.output.length>0){err(UTF8ArrayToString(tty.output,0));tty.output=[]}}}};var mmapAlloc=size=>{abort()};var MEMFS={ops_table:null,mount(mount){return MEMFS.createNode(null,\"/\",16384|511,0)},createNode(parent,name,mode,dev){if(FS.isBlkdev(mode)||FS.isFIFO(mode)){throw new FS.ErrnoError(63)}if(!MEMFS.ops_table){MEMFS.ops_table={dir:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,lookup:MEMFS.node_ops.lookup,mknod:MEMFS.node_ops.mknod,rename:MEMFS.node_ops.rename,unlink:MEMFS.node_ops.unlink,rmdir:MEMFS.node_ops.rmdir,readdir:MEMFS.node_ops.readdir,symlink:MEMFS.node_ops.symlink},stream:{llseek:MEMFS.stream_ops.llseek}},file:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:{llseek:MEMFS.stream_ops.llseek,read:MEMFS.stream_ops.read,write:MEMFS.stream_ops.write,allocate:MEMFS.stream_ops.allocate,mmap:MEMFS.stream_ops.mmap,msync:MEMFS.stream_ops.msync}},link:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,readlink:MEMFS.node_ops.readlink},stream:{}},chrdev:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:FS.chrdev_stream_ops}}}var node=FS.createNode(parent,name,mode,dev);if(FS.isDir(node.mode)){node.node_ops=MEMFS.ops_table.dir.node;node.stream_ops=MEMFS.ops_table.dir.stream;node.contents={}}else if(FS.isFile(node.mode)){node.node_ops=MEMFS.ops_table.file.node;node.stream_ops=MEMFS.ops_table.file.stream;node.usedBytes=0;node.contents=null}else if(FS.isLink(node.mode)){node.node_ops=MEMFS.ops_table.link.node;node.stream_ops=MEMFS.ops_table.link.stream}else if(FS.isChrdev(node.mode)){node.node_ops=MEMFS.ops_table.chrdev.node;node.stream_ops=MEMFS.ops_table.chrdev.stream}node.timestamp=Date.now();if(parent){parent.contents[name]=node;parent.timestamp=node.timestamp}return node},getFileDataAsTypedArray(node){if(!node.contents)return new Uint8Array(0);if(node.contents.subarray)return node.contents.subarray(0,node.usedBytes);return new Uint8Array(node.contents)},expandFileStorage(node,newCapacity){var prevCapacity=node.contents?node.contents.length:0;if(prevCapacity>=newCapacity)return;var CAPACITY_DOUBLING_MAX=1024*1024;newCapacity=Math.max(newCapacity,prevCapacity*(prevCapacity<CAPACITY_DOUBLING_MAX?2:1.125)>>>0);if(prevCapacity!=0)newCapacity=Math.max(newCapacity,256);var oldContents=node.contents;node.contents=new Uint8Array(newCapacity);if(node.usedBytes>0)node.contents.set(oldContents.subarray(0,node.usedBytes),0)},resizeFileStorage(node,newSize){if(node.usedBytes==newSize)return;if(newSize==0){node.contents=null;node.usedBytes=0}else{var oldContents=node.contents;node.contents=new Uint8Array(newSize);if(oldContents){node.contents.set(oldContents.subarray(0,Math.min(newSize,node.usedBytes)))}node.usedBytes=newSize}},node_ops:{getattr(node){var attr={};attr.dev=FS.isChrdev(node.mode)?node.id:1;attr.ino=node.id;attr.mode=node.mode;attr.nlink=1;attr.uid=0;attr.gid=0;attr.rdev=node.rdev;if(FS.isDir(node.mode)){attr.size=4096}else if(FS.isFile(node.mode)){attr.size=node.usedBytes}else if(FS.isLink(node.mode)){attr.size=node.link.length}else{attr.size=0}attr.atime=new Date(node.timestamp);attr.mtime=new Date(node.timestamp);attr.ctime=new Date(node.timestamp);attr.blksize=4096;attr.blocks=Math.ceil(attr.size/attr.blksize);return attr},setattr(node,attr){if(attr.mode!==undefined){node.mode=attr.mode}if(attr.timestamp!==undefined){node.timestamp=attr.timestamp}if(attr.size!==undefined){MEMFS.resizeFileStorage(node,attr.size)}},lookup(parent,name){throw FS.genericErrors[44]},mknod(parent,name,mode,dev){return MEMFS.createNode(parent,name,mode,dev)},rename(old_node,new_dir,new_name){if(FS.isDir(old_node.mode)){var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(new_node){for(var i in new_node.contents){throw new FS.ErrnoError(55)}}}delete old_node.parent.contents[old_node.name];old_node.parent.timestamp=Date.now();old_node.name=new_name;new_dir.contents[new_name]=old_node;new_dir.timestamp=old_node.parent.timestamp;old_node.parent=new_dir},unlink(parent,name){delete parent.contents[name];parent.timestamp=Date.now()},rmdir(parent,name){var node=FS.lookupNode(parent,name);for(var i in node.contents){throw new FS.ErrnoError(55)}delete parent.contents[name];parent.timestamp=Date.now()},readdir(node){var entries=[\".\",\"..\"];for(var key in node.contents){if(!node.contents.hasOwnProperty(key)){continue}entries.push(key)}return entries},symlink(parent,newname,oldpath){var node=MEMFS.createNode(parent,newname,511|40960,0);node.link=oldpath;return node},readlink(node){if(!FS.isLink(node.mode)){throw new FS.ErrnoError(28)}return node.link}},stream_ops:{read(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=stream.node.usedBytes)return 0;var size=Math.min(stream.node.usedBytes-position,length);if(size>8&&contents.subarray){buffer.set(contents.subarray(position,position+size),offset)}else{for(var i=0;i<size;i++)buffer[offset+i]=contents[position+i]}return size},write(stream,buffer,offset,length,position,canOwn){if(buffer.buffer===GROWABLE_HEAP_I8().buffer){canOwn=false}if(!length)return 0;var node=stream.node;node.timestamp=Date.now();if(buffer.subarray&&(!node.contents||node.contents.subarray)){if(canOwn){node.contents=buffer.subarray(offset,offset+length);node.usedBytes=length;return length}else if(node.usedBytes===0&&position===0){node.contents=buffer.slice(offset,offset+length);node.usedBytes=length;return length}else if(position+length<=node.usedBytes){node.contents.set(buffer.subarray(offset,offset+length),position);return length}}MEMFS.expandFileStorage(node,position+length);if(node.contents.subarray&&buffer.subarray){node.contents.set(buffer.subarray(offset,offset+length),position)}else{for(var i=0;i<length;i++){node.contents[position+i]=buffer[offset+i]}}node.usedBytes=Math.max(node.usedBytes,position+length);return length},llseek(stream,offset,whence){var position=offset;if(whence===1){position+=stream.position}else if(whence===2){if(FS.isFile(stream.node.mode)){position+=stream.node.usedBytes}}if(position<0){throw new FS.ErrnoError(28)}return position},allocate(stream,offset,length){MEMFS.expandFileStorage(stream.node,offset+length);stream.node.usedBytes=Math.max(stream.node.usedBytes,offset+length)},mmap(stream,length,position,prot,flags){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}var ptr;var allocated;var contents=stream.node.contents;if(!(flags&2)&&contents.buffer===GROWABLE_HEAP_I8().buffer){allocated=false;ptr=contents.byteOffset}else{if(position>0||position+length<contents.length){if(contents.subarray){contents=contents.subarray(position,position+length)}else{contents=Array.prototype.slice.call(contents,position,position+length)}}allocated=true;ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}GROWABLE_HEAP_I8().set(contents,ptr)}return{ptr:ptr,allocated:allocated}},msync(stream,buffer,offset,length,mmapFlags){MEMFS.stream_ops.write(stream,buffer,0,length,offset,false);return 0}}};var asyncLoad=(url,onload,onerror,noRunDep)=>{var dep=!noRunDep?getUniqueRunDependency(`al ${url}`):\"\";readAsync(url,arrayBuffer=>{assert(arrayBuffer,`Loading data file \"${url}\" failed (no arrayBuffer).`);onload(new Uint8Array(arrayBuffer));if(dep)removeRunDependency(dep)},event=>{if(onerror){onerror()}else{throw`Loading data file \"${url}\" failed.`}});if(dep)addRunDependency(dep)};var preloadPlugins=Module[\"preloadPlugins\"]||[];var FS_handledByPreloadPlugin=(byteArray,fullname,finish,onerror)=>{if(typeof Browser!=\"undefined\")Browser.init();var handled=false;preloadPlugins.forEach(plugin=>{if(handled)return;if(plugin[\"canHandle\"](fullname)){plugin[\"handle\"](byteArray,fullname,finish,onerror);handled=true}});return handled};var FS_createPreloadedFile=(parent,name,url,canRead,canWrite,onload,onerror,dontCreateFile,canOwn,preFinish)=>{var fullname=name?PATH_FS.resolve(PATH.join2(parent,name)):parent;var dep=getUniqueRunDependency(`cp ${fullname}`);function processData(byteArray){function finish(byteArray){if(preFinish)preFinish();if(!dontCreateFile){FS.createDataFile(parent,name,byteArray,canRead,canWrite,canOwn)}if(onload)onload();removeRunDependency(dep)}if(FS_handledByPreloadPlugin(byteArray,fullname,finish,()=>{if(onerror)onerror();removeRunDependency(dep)})){return}finish(byteArray)}addRunDependency(dep);if(typeof url==\"string\"){asyncLoad(url,byteArray=>processData(byteArray),onerror)}else{processData(url)}};var FS_modeStringToFlags=str=>{var flagModes={\"r\":0,\"r+\":2,\"w\":512|64|1,\"w+\":512|64|2,\"a\":1024|64|1,\"a+\":1024|64|2};var flags=flagModes[str];if(typeof flags==\"undefined\"){throw new Error(`Unknown file open mode: ${str}`)}return flags};var FS_getMode=(canRead,canWrite)=>{var mode=0;if(canRead)mode|=292|73;if(canWrite)mode|=146;return mode};var FS={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:\"/\",initialized:false,ignorePermissions:true,ErrnoError:null,genericErrors:{},filesystems:null,syncFSRequests:0,lookupPath(path,opts={}){path=PATH_FS.resolve(path);if(!path)return{path:\"\",node:null};var defaults={follow_mount:true,recurse_count:0};opts=Object.assign(defaults,opts);if(opts.recurse_count>8){throw new FS.ErrnoError(32)}var parts=path.split(\"/\").filter(p=>!!p);var current=FS.root;var current_path=\"/\";for(var i=0;i<parts.length;i++){var islast=i===parts.length-1;if(islast&&opts.parent){break}current=FS.lookupNode(current,parts[i]);current_path=PATH.join2(current_path,parts[i]);if(FS.isMountpoint(current)){if(!islast||islast&&opts.follow_mount){current=current.mounted.root}}if(!islast||opts.follow){var count=0;while(FS.isLink(current.mode)){var link=FS.readlink(current_path);current_path=PATH_FS.resolve(PATH.dirname(current_path),link);var lookup=FS.lookupPath(current_path,{recurse_count:opts.recurse_count+1});current=lookup.node;if(count++>40){throw new FS.ErrnoError(32)}}}}return{path:current_path,node:current}},getPath(node){var path;while(true){if(FS.isRoot(node)){var mount=node.mount.mountpoint;if(!path)return mount;return mount[mount.length-1]!==\"/\"?`${mount}/${path}`:mount+path}path=path?`${node.name}/${path}`:node.name;node=node.parent}},hashName(parentid,name){var hash=0;for(var i=0;i<name.length;i++){hash=(hash<<5)-hash+name.charCodeAt(i)|0}return(parentid+hash>>>0)%FS.nameTable.length},hashAddNode(node){var hash=FS.hashName(node.parent.id,node.name);node.name_next=FS.nameTable[hash];FS.nameTable[hash]=node},hashRemoveNode(node){var hash=FS.hashName(node.parent.id,node.name);if(FS.nameTable[hash]===node){FS.nameTable[hash]=node.name_next}else{var current=FS.nameTable[hash];while(current){if(current.name_next===node){current.name_next=node.name_next;break}current=current.name_next}}},lookupNode(parent,name){var errCode=FS.mayLookup(parent);if(errCode){throw new FS.ErrnoError(errCode,parent)}var hash=FS.hashName(parent.id,name);for(var node=FS.nameTable[hash];node;node=node.name_next){var nodeName=node.name;if(node.parent.id===parent.id&&nodeName===name){return node}}return FS.lookup(parent,name)},createNode(parent,name,mode,rdev){var node=new FS.FSNode(parent,name,mode,rdev);FS.hashAddNode(node);return node},destroyNode(node){FS.hashRemoveNode(node)},isRoot(node){return node===node.parent},isMountpoint(node){return!!node.mounted},isFile(mode){return(mode&61440)===32768},isDir(mode){return(mode&61440)===16384},isLink(mode){return(mode&61440)===40960},isChrdev(mode){return(mode&61440)===8192},isBlkdev(mode){return(mode&61440)===24576},isFIFO(mode){return(mode&61440)===4096},isSocket(mode){return(mode&49152)===49152},flagsToPermissionString(flag){var perms=[\"r\",\"w\",\"rw\"][flag&3];if(flag&512){perms+=\"w\"}return perms},nodePermissions(node,perms){if(FS.ignorePermissions){return 0}if(perms.includes(\"r\")&&!(node.mode&292)){return 2}else if(perms.includes(\"w\")&&!(node.mode&146)){return 2}else if(perms.includes(\"x\")&&!(node.mode&73)){return 2}return 0},mayLookup(dir){var errCode=FS.nodePermissions(dir,\"x\");if(errCode)return errCode;if(!dir.node_ops.lookup)return 2;return 0},mayCreate(dir,name){try{var node=FS.lookupNode(dir,name);return 20}catch(e){}return FS.nodePermissions(dir,\"wx\")},mayDelete(dir,name,isdir){var node;try{node=FS.lookupNode(dir,name)}catch(e){return e.errno}var errCode=FS.nodePermissions(dir,\"wx\");if(errCode){return errCode}if(isdir){if(!FS.isDir(node.mode)){return 54}if(FS.isRoot(node)||FS.getPath(node)===FS.cwd()){return 10}}else{if(FS.isDir(node.mode)){return 31}}return 0},mayOpen(node,flags){if(!node){return 44}if(FS.isLink(node.mode)){return 32}else if(FS.isDir(node.mode)){if(FS.flagsToPermissionString(flags)!==\"r\"||flags&512){return 31}}return FS.nodePermissions(node,FS.flagsToPermissionString(flags))},MAX_OPEN_FDS:4096,nextfd(){for(var fd=0;fd<=FS.MAX_OPEN_FDS;fd++){if(!FS.streams[fd]){return fd}}throw new FS.ErrnoError(33)},getStreamChecked(fd){var stream=FS.getStream(fd);if(!stream){throw new FS.ErrnoError(8)}return stream},getStream:fd=>FS.streams[fd],createStream(stream,fd=-1){if(!FS.FSStream){FS.FSStream=function(){this.shared={}};FS.FSStream.prototype={};Object.defineProperties(FS.FSStream.prototype,{object:{get(){return this.node},set(val){this.node=val}},isRead:{get(){return(this.flags&2097155)!==1}},isWrite:{get(){return(this.flags&2097155)!==0}},isAppend:{get(){return this.flags&1024}},flags:{get(){return this.shared.flags},set(val){this.shared.flags=val}},position:{get(){return this.shared.position},set(val){this.shared.position=val}}})}stream=Object.assign(new FS.FSStream,stream);if(fd==-1){fd=FS.nextfd()}stream.fd=fd;FS.streams[fd]=stream;return stream},closeStream(fd){FS.streams[fd]=null},chrdev_stream_ops:{open(stream){var device=FS.getDevice(stream.node.rdev);stream.stream_ops=device.stream_ops;if(stream.stream_ops.open){stream.stream_ops.open(stream)}},llseek(){throw new FS.ErrnoError(70)}},major:dev=>dev>>8,minor:dev=>dev&255,makedev:(ma,mi)=>ma<<8|mi,registerDevice(dev,ops){FS.devices[dev]={stream_ops:ops}},getDevice:dev=>FS.devices[dev],getMounts(mount){var mounts=[];var check=[mount];while(check.length){var m=check.pop();mounts.push(m);check.push.apply(check,m.mounts)}return mounts},syncfs(populate,callback){if(typeof populate==\"function\"){callback=populate;populate=false}FS.syncFSRequests++;if(FS.syncFSRequests>1){err(`warning: ${FS.syncFSRequests} FS.syncfs operations in flight at once, probably just doing extra work`)}var mounts=FS.getMounts(FS.root.mount);var completed=0;function doCallback(errCode){FS.syncFSRequests--;return callback(errCode)}function done(errCode){if(errCode){if(!done.errored){done.errored=true;return doCallback(errCode)}return}if(++completed>=mounts.length){doCallback(null)}}mounts.forEach(mount=>{if(!mount.type.syncfs){return done(null)}mount.type.syncfs(mount,populate,done)})},mount(type,opts,mountpoint){var root=mountpoint===\"/\";var pseudo=!mountpoint;var node;if(root&&FS.root){throw new FS.ErrnoError(10)}else if(!root&&!pseudo){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});mountpoint=lookup.path;node=lookup.node;if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}if(!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}}var mount={type:type,opts:opts,mountpoint:mountpoint,mounts:[]};var mountRoot=type.mount(mount);mountRoot.mount=mount;mount.root=mountRoot;if(root){FS.root=mountRoot}else if(node){node.mounted=mount;if(node.mount){node.mount.mounts.push(mount)}}return mountRoot},unmount(mountpoint){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});if(!FS.isMountpoint(lookup.node)){throw new FS.ErrnoError(28)}var node=lookup.node;var mount=node.mounted;var mounts=FS.getMounts(mount);Object.keys(FS.nameTable).forEach(hash=>{var current=FS.nameTable[hash];while(current){var next=current.name_next;if(mounts.includes(current.mount)){FS.destroyNode(current)}current=next}});node.mounted=null;var idx=node.mount.mounts.indexOf(mount);node.mount.mounts.splice(idx,1)},lookup(parent,name){return parent.node_ops.lookup(parent,name)},mknod(path,mode,dev){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);if(!name||name===\".\"||name===\"..\"){throw new FS.ErrnoError(28)}var errCode=FS.mayCreate(parent,name);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.mknod){throw new FS.ErrnoError(63)}return parent.node_ops.mknod(parent,name,mode,dev)},create(path,mode){mode=mode!==undefined?mode:438;mode&=4095;mode|=32768;return FS.mknod(path,mode,0)},mkdir(path,mode){mode=mode!==undefined?mode:511;mode&=511|512;mode|=16384;return FS.mknod(path,mode,0)},mkdirTree(path,mode){var dirs=path.split(\"/\");var d=\"\";for(var i=0;i<dirs.length;++i){if(!dirs[i])continue;d+=\"/\"+dirs[i];try{FS.mkdir(d,mode)}catch(e){if(e.errno!=20)throw e}}},mkdev(path,mode,dev){if(typeof dev==\"undefined\"){dev=mode;mode=438}mode|=8192;return FS.mknod(path,mode,dev)},symlink(oldpath,newpath){if(!PATH_FS.resolve(oldpath)){throw new FS.ErrnoError(44)}var lookup=FS.lookupPath(newpath,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var newname=PATH.basename(newpath);var errCode=FS.mayCreate(parent,newname);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.symlink){throw new FS.ErrnoError(63)}return parent.node_ops.symlink(parent,newname,oldpath)},rename(old_path,new_path){var old_dirname=PATH.dirname(old_path);var new_dirname=PATH.dirname(new_path);var old_name=PATH.basename(old_path);var new_name=PATH.basename(new_path);var lookup,old_dir,new_dir;lookup=FS.lookupPath(old_path,{parent:true});old_dir=lookup.node;lookup=FS.lookupPath(new_path,{parent:true});new_dir=lookup.node;if(!old_dir||!new_dir)throw new FS.ErrnoError(44);if(old_dir.mount!==new_dir.mount){throw new FS.ErrnoError(75)}var old_node=FS.lookupNode(old_dir,old_name);var relative=PATH_FS.relative(old_path,new_dirname);if(relative.charAt(0)!==\".\"){throw new FS.ErrnoError(28)}relative=PATH_FS.relative(new_path,old_dirname);if(relative.charAt(0)!==\".\"){throw new FS.ErrnoError(55)}var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(old_node===new_node){return}var isdir=FS.isDir(old_node.mode);var errCode=FS.mayDelete(old_dir,old_name,isdir);if(errCode){throw new FS.ErrnoError(errCode)}errCode=new_node?FS.mayDelete(new_dir,new_name,isdir):FS.mayCreate(new_dir,new_name);if(errCode){throw new FS.ErrnoError(errCode)}if(!old_dir.node_ops.rename){throw new FS.ErrnoError(63)}if(FS.isMountpoint(old_node)||new_node&&FS.isMountpoint(new_node)){throw new FS.ErrnoError(10)}if(new_dir!==old_dir){errCode=FS.nodePermissions(old_dir,\"w\");if(errCode){throw new FS.ErrnoError(errCode)}}FS.hashRemoveNode(old_node);try{old_dir.node_ops.rename(old_node,new_dir,new_name)}catch(e){throw e}finally{FS.hashAddNode(old_node)}},rmdir(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,true);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.rmdir){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.rmdir(parent,name);FS.destroyNode(node)},readdir(path){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;if(!node.node_ops.readdir){throw new FS.ErrnoError(54)}return node.node_ops.readdir(node)},unlink(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,false);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.unlink){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.unlink(parent,name);FS.destroyNode(node)},readlink(path){var lookup=FS.lookupPath(path);var link=lookup.node;if(!link){throw new FS.ErrnoError(44)}if(!link.node_ops.readlink){throw new FS.ErrnoError(28)}return PATH_FS.resolve(FS.getPath(link.parent),link.node_ops.readlink(link))},stat(path,dontFollow){var lookup=FS.lookupPath(path,{follow:!dontFollow});var node=lookup.node;if(!node){throw new FS.ErrnoError(44)}if(!node.node_ops.getattr){throw new FS.ErrnoError(63)}return node.node_ops.getattr(node)},lstat(path){return FS.stat(path,true)},chmod(path,mode,dontFollow){var node;if(typeof path==\"string\"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}if(!node.node_ops.setattr){throw new FS.ErrnoError(63)}node.node_ops.setattr(node,{mode:mode&4095|node.mode&~4095,timestamp:Date.now()})},lchmod(path,mode){FS.chmod(path,mode,true)},fchmod(fd,mode){var stream=FS.getStreamChecked(fd);FS.chmod(stream.node,mode)},chown(path,uid,gid,dontFollow){var node;if(typeof path==\"string\"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}if(!node.node_ops.setattr){throw new FS.ErrnoError(63)}node.node_ops.setattr(node,{timestamp:Date.now()})},lchown(path,uid,gid){FS.chown(path,uid,gid,true)},fchown(fd,uid,gid){var stream=FS.getStreamChecked(fd);FS.chown(stream.node,uid,gid)},truncate(path,len){if(len<0){throw new FS.ErrnoError(28)}var node;if(typeof path==\"string\"){var lookup=FS.lookupPath(path,{follow:true});node=lookup.node}else{node=path}if(!node.node_ops.setattr){throw new FS.ErrnoError(63)}if(FS.isDir(node.mode)){throw new FS.ErrnoError(31)}if(!FS.isFile(node.mode)){throw new FS.ErrnoError(28)}var errCode=FS.nodePermissions(node,\"w\");if(errCode){throw new FS.ErrnoError(errCode)}node.node_ops.setattr(node,{size:len,timestamp:Date.now()})},ftruncate(fd,len){var stream=FS.getStreamChecked(fd);if((stream.flags&2097155)===0){throw new FS.ErrnoError(28)}FS.truncate(stream.node,len)},utime(path,atime,mtime){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;node.node_ops.setattr(node,{timestamp:Math.max(atime,mtime)})},open(path,flags,mode){if(path===\"\"){throw new FS.ErrnoError(44)}flags=typeof flags==\"string\"?FS_modeStringToFlags(flags):flags;mode=typeof mode==\"undefined\"?438:mode;if(flags&64){mode=mode&4095|32768}else{mode=0}var node;if(typeof path==\"object\"){node=path}else{path=PATH.normalize(path);try{var lookup=FS.lookupPath(path,{follow:!(flags&131072)});node=lookup.node}catch(e){}}var created=false;if(flags&64){if(node){if(flags&128){throw new FS.ErrnoError(20)}}else{node=FS.mknod(path,mode,0);created=true}}if(!node){throw new FS.ErrnoError(44)}if(FS.isChrdev(node.mode)){flags&=~512}if(flags&65536&&!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}if(!created){var errCode=FS.mayOpen(node,flags);if(errCode){throw new FS.ErrnoError(errCode)}}if(flags&512&&!created){FS.truncate(node,0)}flags&=~(128|512|131072);var stream=FS.createStream({node:node,path:FS.getPath(node),flags:flags,seekable:true,position:0,stream_ops:node.stream_ops,ungotten:[],error:false});if(stream.stream_ops.open){stream.stream_ops.open(stream)}if(Module[\"logReadFiles\"]&&!(flags&1)){if(!FS.readFiles)FS.readFiles={};if(!(path in FS.readFiles)){FS.readFiles[path]=1}}return stream},close(stream){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(stream.getdents)stream.getdents=null;try{if(stream.stream_ops.close){stream.stream_ops.close(stream)}}catch(e){throw e}finally{FS.closeStream(stream.fd)}stream.fd=null},isClosed(stream){return stream.fd===null},llseek(stream,offset,whence){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(!stream.seekable||!stream.stream_ops.llseek){throw new FS.ErrnoError(70)}if(whence!=0&&whence!=1&&whence!=2){throw new FS.ErrnoError(28)}stream.position=stream.stream_ops.llseek(stream,offset,whence);stream.ungotten=[];return stream.position},read(stream,buffer,offset,length,position){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.read){throw new FS.ErrnoError(28)}var seeking=typeof position!=\"undefined\";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesRead=stream.stream_ops.read(stream,buffer,offset,length,position);if(!seeking)stream.position+=bytesRead;return bytesRead},write(stream,buffer,offset,length,position,canOwn){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===0){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.write){throw new FS.ErrnoError(28)}if(stream.seekable&&stream.flags&1024){FS.llseek(stream,0,2)}var seeking=typeof position!=\"undefined\";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesWritten=stream.stream_ops.write(stream,buffer,offset,length,position,canOwn);if(!seeking)stream.position+=bytesWritten;return bytesWritten},allocate(stream,offset,length){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(offset<0||length<=0){throw new FS.ErrnoError(28)}if((stream.flags&2097155)===0){throw new FS.ErrnoError(8)}if(!FS.isFile(stream.node.mode)&&!FS.isDir(stream.node.mode)){throw new FS.ErrnoError(43)}if(!stream.stream_ops.allocate){throw new FS.ErrnoError(138)}stream.stream_ops.allocate(stream,offset,length)},mmap(stream,length,position,prot,flags){if((prot&2)!==0&&(flags&2)===0&&(stream.flags&2097155)!==2){throw new FS.ErrnoError(2)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(2)}if(!stream.stream_ops.mmap){throw new FS.ErrnoError(43)}return stream.stream_ops.mmap(stream,length,position,prot,flags)},msync(stream,buffer,offset,length,mmapFlags){if(!stream.stream_ops.msync){return 0}return stream.stream_ops.msync(stream,buffer,offset,length,mmapFlags)},munmap:stream=>0,ioctl(stream,cmd,arg){if(!stream.stream_ops.ioctl){throw new FS.ErrnoError(59)}return stream.stream_ops.ioctl(stream,cmd,arg)},readFile(path,opts={}){opts.flags=opts.flags||0;opts.encoding=opts.encoding||\"binary\";if(opts.encoding!==\"utf8\"&&opts.encoding!==\"binary\"){throw new Error(`Invalid encoding type \"${opts.encoding}\"`)}var ret;var stream=FS.open(path,opts.flags);var stat=FS.stat(path);var length=stat.size;var buf=new Uint8Array(length);FS.read(stream,buf,0,length,0);if(opts.encoding===\"utf8\"){ret=UTF8ArrayToString(buf,0)}else if(opts.encoding===\"binary\"){ret=buf}FS.close(stream);return ret},writeFile(path,data,opts={}){opts.flags=opts.flags||577;var stream=FS.open(path,opts.flags,opts.mode);if(typeof data==\"string\"){var buf=new Uint8Array(lengthBytesUTF8(data)+1);var actualNumBytes=stringToUTF8Array(data,buf,0,buf.length);FS.write(stream,buf,0,actualNumBytes,undefined,opts.canOwn)}else if(ArrayBuffer.isView(data)){FS.write(stream,data,0,data.byteLength,undefined,opts.canOwn)}else{throw new Error(\"Unsupported data type\")}FS.close(stream)},cwd:()=>FS.currentPath,chdir(path){var lookup=FS.lookupPath(path,{follow:true});if(lookup.node===null){throw new FS.ErrnoError(44)}if(!FS.isDir(lookup.node.mode)){throw new FS.ErrnoError(54)}var errCode=FS.nodePermissions(lookup.node,\"x\");if(errCode){throw new FS.ErrnoError(errCode)}FS.currentPath=lookup.path},createDefaultDirectories(){FS.mkdir(\"/tmp\");FS.mkdir(\"/home\");FS.mkdir(\"/home/web_user\")},createDefaultDevices(){FS.mkdir(\"/dev\");FS.registerDevice(FS.makedev(1,3),{read:()=>0,write:(stream,buffer,offset,length,pos)=>length});FS.mkdev(\"/dev/null\",FS.makedev(1,3));TTY.register(FS.makedev(5,0),TTY.default_tty_ops);TTY.register(FS.makedev(6,0),TTY.default_tty1_ops);FS.mkdev(\"/dev/tty\",FS.makedev(5,0));FS.mkdev(\"/dev/tty1\",FS.makedev(6,0));var randomBuffer=new Uint8Array(1024),randomLeft=0;var randomByte=()=>{if(randomLeft===0){randomLeft=randomFill(randomBuffer).byteLength}return randomBuffer[--randomLeft]};FS.createDevice(\"/dev\",\"random\",randomByte);FS.createDevice(\"/dev\",\"urandom\",randomByte);FS.mkdir(\"/dev/shm\");FS.mkdir(\"/dev/shm/tmp\")},createSpecialDirectories(){FS.mkdir(\"/proc\");var proc_self=FS.mkdir(\"/proc/self\");FS.mkdir(\"/proc/self/fd\");FS.mount({mount(){var node=FS.createNode(proc_self,\"fd\",16384|511,73);node.node_ops={lookup(parent,name){var fd=+name;var stream=FS.getStreamChecked(fd);var ret={parent:null,mount:{mountpoint:\"fake\"},node_ops:{readlink:()=>stream.path}};ret.parent=ret;return ret}};return node}},{},\"/proc/self/fd\")},createStandardStreams(){if(Module[\"stdin\"]){FS.createDevice(\"/dev\",\"stdin\",Module[\"stdin\"])}else{FS.symlink(\"/dev/tty\",\"/dev/stdin\")}if(Module[\"stdout\"]){FS.createDevice(\"/dev\",\"stdout\",null,Module[\"stdout\"])}else{FS.symlink(\"/dev/tty\",\"/dev/stdout\")}if(Module[\"stderr\"]){FS.createDevice(\"/dev\",\"stderr\",null,Module[\"stderr\"])}else{FS.symlink(\"/dev/tty1\",\"/dev/stderr\")}var stdin=FS.open(\"/dev/stdin\",0);var stdout=FS.open(\"/dev/stdout\",1);var stderr=FS.open(\"/dev/stderr\",1)},ensureErrnoError(){if(FS.ErrnoError)return;FS.ErrnoError=function ErrnoError(errno,node){this.name=\"ErrnoError\";this.node=node;this.setErrno=function(errno){this.errno=errno};this.setErrno(errno);this.message=\"FS error\"};FS.ErrnoError.prototype=new Error;FS.ErrnoError.prototype.constructor=FS.ErrnoError;[44].forEach(code=>{FS.genericErrors[code]=new FS.ErrnoError(code);FS.genericErrors[code].stack=\"<generic error, no stack>\"})},staticInit(){FS.ensureErrnoError();FS.nameTable=new Array(4096);FS.mount(MEMFS,{},\"/\");FS.createDefaultDirectories();FS.createDefaultDevices();FS.createSpecialDirectories();FS.filesystems={\"MEMFS\":MEMFS}},init(input,output,error){FS.init.initialized=true;FS.ensureErrnoError();Module[\"stdin\"]=input||Module[\"stdin\"];Module[\"stdout\"]=output||Module[\"stdout\"];Module[\"stderr\"]=error||Module[\"stderr\"];FS.createStandardStreams()},quit(){FS.init.initialized=false;for(var i=0;i<FS.streams.length;i++){var stream=FS.streams[i];if(!stream){continue}FS.close(stream)}},findObject(path,dontResolveLastLink){var ret=FS.analyzePath(path,dontResolveLastLink);if(!ret.exists){return null}return ret.object},analyzePath(path,dontResolveLastLink){try{var lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});path=lookup.path}catch(e){}var ret={isRoot:false,exists:false,error:0,name:null,path:null,object:null,parentExists:false,parentPath:null,parentObject:null};try{var lookup=FS.lookupPath(path,{parent:true});ret.parentExists=true;ret.parentPath=lookup.path;ret.parentObject=lookup.node;ret.name=PATH.basename(path);lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});ret.exists=true;ret.path=lookup.path;ret.object=lookup.node;ret.name=lookup.node.name;ret.isRoot=lookup.path===\"/\"}catch(e){ret.error=e.errno}return ret},createPath(parent,path,canRead,canWrite){parent=typeof parent==\"string\"?parent:FS.getPath(parent);var parts=path.split(\"/\").reverse();while(parts.length){var part=parts.pop();if(!part)continue;var current=PATH.join2(parent,part);try{FS.mkdir(current)}catch(e){}parent=current}return current},createFile(parent,name,properties,canRead,canWrite){var path=PATH.join2(typeof parent==\"string\"?parent:FS.getPath(parent),name);var mode=FS_getMode(canRead,canWrite);return FS.create(path,mode)},createDataFile(parent,name,data,canRead,canWrite,canOwn){var path=name;if(parent){parent=typeof parent==\"string\"?parent:FS.getPath(parent);path=name?PATH.join2(parent,name):parent}var mode=FS_getMode(canRead,canWrite);var node=FS.create(path,mode);if(data){if(typeof data==\"string\"){var arr=new Array(data.length);for(var i=0,len=data.length;i<len;++i)arr[i]=data.charCodeAt(i);data=arr}FS.chmod(node,mode|146);var stream=FS.open(node,577);FS.write(stream,data,0,data.length,0,canOwn);FS.close(stream);FS.chmod(node,mode)}return node},createDevice(parent,name,input,output){var path=PATH.join2(typeof parent==\"string\"?parent:FS.getPath(parent),name);var mode=FS_getMode(!!input,!!output);if(!FS.createDevice.major)FS.createDevice.major=64;var dev=FS.makedev(FS.createDevice.major++,0);FS.registerDevice(dev,{open(stream){stream.seekable=false},close(stream){if(output&&output.buffer&&output.buffer.length){output(10)}},read(stream,buffer,offset,length,pos){var bytesRead=0;for(var i=0;i<length;i++){var result;try{result=input()}catch(e){throw new FS.ErrnoError(29)}if(result===undefined&&bytesRead===0){throw new FS.ErrnoError(6)}if(result===null||result===undefined)break;bytesRead++;buffer[offset+i]=result}if(bytesRead){stream.node.timestamp=Date.now()}return bytesRead},write(stream,buffer,offset,length,pos){for(var i=0;i<length;i++){try{output(buffer[offset+i])}catch(e){throw new FS.ErrnoError(29)}}if(length){stream.node.timestamp=Date.now()}return i}});return FS.mkdev(path,mode,dev)},forceLoadFile(obj){if(obj.isDevice||obj.isFolder||obj.link||obj.contents)return true;if(typeof XMLHttpRequest!=\"undefined\"){throw new Error(\"Lazy loading should have been performed (contents set) in createLazyFile, but it was not. Lazy loading only works in web workers. Use --embed-file or --preload-file in emcc on the main thread.\")}else if(read_){try{obj.contents=intArrayFromString(read_(obj.url),true);obj.usedBytes=obj.contents.length}catch(e){throw new FS.ErrnoError(29)}}else{throw new Error(\"Cannot load without read() or XMLHttpRequest.\")}},createLazyFile(parent,name,url,canRead,canWrite){function LazyUint8Array(){this.lengthKnown=false;this.chunks=[]}LazyUint8Array.prototype.get=function LazyUint8Array_get(idx){if(idx>this.length-1||idx<0){return undefined}var chunkOffset=idx%this.chunkSize;var chunkNum=idx/this.chunkSize|0;return this.getter(chunkNum)[chunkOffset]};LazyUint8Array.prototype.setDataGetter=function LazyUint8Array_setDataGetter(getter){this.getter=getter};LazyUint8Array.prototype.cacheLength=function LazyUint8Array_cacheLength(){var xhr=new XMLHttpRequest;xhr.open(\"HEAD\",url,false);xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))throw new Error(\"Couldn't load \"+url+\". Status: \"+xhr.status);var datalength=Number(xhr.getResponseHeader(\"Content-length\"));var header;var hasByteServing=(header=xhr.getResponseHeader(\"Accept-Ranges\"))&&header===\"bytes\";var usesGzip=(header=xhr.getResponseHeader(\"Content-Encoding\"))&&header===\"gzip\";var chunkSize=1024*1024;if(!hasByteServing)chunkSize=datalength;var doXHR=(from,to)=>{if(from>to)throw new Error(\"invalid range (\"+from+\", \"+to+\") or no bytes requested!\");if(to>datalength-1)throw new Error(\"only \"+datalength+\" bytes available! programmer error!\");var xhr=new XMLHttpRequest;xhr.open(\"GET\",url,false);if(datalength!==chunkSize)xhr.setRequestHeader(\"Range\",\"bytes=\"+from+\"-\"+to);xhr.responseType=\"arraybuffer\";if(xhr.overrideMimeType){xhr.overrideMimeType(\"text/plain; charset=x-user-defined\")}xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))throw new Error(\"Couldn't load \"+url+\". Status: \"+xhr.status);if(xhr.response!==undefined){return new Uint8Array(xhr.response||[])}return intArrayFromString(xhr.responseText||\"\",true)};var lazyArray=this;lazyArray.setDataGetter(chunkNum=>{var start=chunkNum*chunkSize;var end=(chunkNum+1)*chunkSize-1;end=Math.min(end,datalength-1);if(typeof lazyArray.chunks[chunkNum]==\"undefined\"){lazyArray.chunks[chunkNum]=doXHR(start,end)}if(typeof lazyArray.chunks[chunkNum]==\"undefined\")throw new Error(\"doXHR failed!\");return lazyArray.chunks[chunkNum]});if(usesGzip||!datalength){chunkSize=datalength=1;datalength=this.getter(0).length;chunkSize=datalength;out(\"LazyFiles on gzip forces download of the whole file when length is accessed\")}this._length=datalength;this._chunkSize=chunkSize;this.lengthKnown=true};if(typeof XMLHttpRequest!=\"undefined\"){if(!ENVIRONMENT_IS_WORKER)throw\"Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc\";var lazyArray=new LazyUint8Array;Object.defineProperties(lazyArray,{length:{get:function(){if(!this.lengthKnown){this.cacheLength()}return this._length}},chunkSize:{get:function(){if(!this.lengthKnown){this.cacheLength()}return this._chunkSize}}});var properties={isDevice:false,contents:lazyArray}}else{var properties={isDevice:false,url:url}}var node=FS.createFile(parent,name,properties,canRead,canWrite);if(properties.contents){node.contents=properties.contents}else if(properties.url){node.contents=null;node.url=properties.url}Object.defineProperties(node,{usedBytes:{get:function(){return this.contents.length}}});var stream_ops={};var keys=Object.keys(node.stream_ops);keys.forEach(key=>{var fn=node.stream_ops[key];stream_ops[key]=function forceLoadLazyFile(){FS.forceLoadFile(node);return fn.apply(null,arguments)}});function writeChunks(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=contents.length)return 0;var size=Math.min(contents.length-position,length);if(contents.slice){for(var i=0;i<size;i++){buffer[offset+i]=contents[position+i]}}else{for(var i=0;i<size;i++){buffer[offset+i]=contents.get(position+i)}}return size}stream_ops.read=(stream,buffer,offset,length,position)=>{FS.forceLoadFile(node);return writeChunks(stream,buffer,offset,length,position)};stream_ops.mmap=(stream,length,position,prot,flags)=>{FS.forceLoadFile(node);var ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}writeChunks(stream,GROWABLE_HEAP_I8(),ptr,length,position);return{ptr:ptr,allocated:true}};node.stream_ops=stream_ops;return node}};var UTF8ToString=(ptr,maxBytesToRead)=>ptr?UTF8ArrayToString(GROWABLE_HEAP_U8(),ptr,maxBytesToRead):\"\";var SYSCALLS={DEFAULT_POLLMASK:5,calculateAt(dirfd,path,allowEmpty){if(PATH.isAbs(path)){return path}var dir;if(dirfd===-100){dir=FS.cwd()}else{var dirstream=SYSCALLS.getStreamFromFD(dirfd);dir=dirstream.path}if(path.length==0){if(!allowEmpty){throw new FS.ErrnoError(44)}return dir}return PATH.join2(dir,path)},doStat(func,path,buf){try{var stat=func(path)}catch(e){if(e&&e.node&&PATH.normalize(path)!==PATH.normalize(FS.getPath(e.node))){return-54}throw e}GROWABLE_HEAP_I32()[buf>>2]=stat.dev;GROWABLE_HEAP_I32()[buf+4>>2]=stat.mode;GROWABLE_HEAP_U32()[buf+8>>2]=stat.nlink;GROWABLE_HEAP_I32()[buf+12>>2]=stat.uid;GROWABLE_HEAP_I32()[buf+16>>2]=stat.gid;GROWABLE_HEAP_I32()[buf+20>>2]=stat.rdev;tempI64=[stat.size>>>0,(tempDouble=stat.size,+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[buf+24>>2]=tempI64[0],GROWABLE_HEAP_I32()[buf+28>>2]=tempI64[1];GROWABLE_HEAP_I32()[buf+32>>2]=4096;GROWABLE_HEAP_I32()[buf+36>>2]=stat.blocks;var atime=stat.atime.getTime();var mtime=stat.mtime.getTime();var ctime=stat.ctime.getTime();tempI64=[Math.floor(atime/1e3)>>>0,(tempDouble=Math.floor(atime/1e3),+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[buf+40>>2]=tempI64[0],GROWABLE_HEAP_I32()[buf+44>>2]=tempI64[1];GROWABLE_HEAP_U32()[buf+48>>2]=atime%1e3*1e3;tempI64=[Math.floor(mtime/1e3)>>>0,(tempDouble=Math.floor(mtime/1e3),+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[buf+56>>2]=tempI64[0],GROWABLE_HEAP_I32()[buf+60>>2]=tempI64[1];GROWABLE_HEAP_U32()[buf+64>>2]=mtime%1e3*1e3;tempI64=[Math.floor(ctime/1e3)>>>0,(tempDouble=Math.floor(ctime/1e3),+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[buf+72>>2]=tempI64[0],GROWABLE_HEAP_I32()[buf+76>>2]=tempI64[1];GROWABLE_HEAP_U32()[buf+80>>2]=ctime%1e3*1e3;tempI64=[stat.ino>>>0,(tempDouble=stat.ino,+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[buf+88>>2]=tempI64[0],GROWABLE_HEAP_I32()[buf+92>>2]=tempI64[1];return 0},doMsync(addr,stream,len,flags,offset){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}if(flags&2){return 0}var buffer=GROWABLE_HEAP_U8().slice(addr,addr+len);FS.msync(stream,buffer,offset,len,flags)},varargs:undefined,get(){var ret=GROWABLE_HEAP_I32()[SYSCALLS.varargs>>2];SYSCALLS.varargs+=4;return ret},getp(){return SYSCALLS.get()},getStr(ptr){var ret=UTF8ToString(ptr);return ret},getStreamFromFD(fd){var stream=FS.getStreamChecked(fd);return stream}};function _proc_exit(code){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(0,1,code);EXITSTATUS=code;if(!keepRuntimeAlive()){PThread.terminateAllThreads();if(Module[\"onExit\"])Module[\"onExit\"](code);ABORT=true}quit_(code,new ExitStatus(code))}var exitJS=(status,implicit)=>{EXITSTATUS=status;if(ENVIRONMENT_IS_PTHREAD){exitOnMainThread(status);throw\"unwind\"}_proc_exit(status)};var _exit=exitJS;var handleException=e=>{if(e instanceof ExitStatus||e==\"unwind\"){return EXITSTATUS}quit_(1,e)};var PThread={unusedWorkers:[],runningWorkers:[],tlsInitFunctions:[],pthreads:{},init(){if(ENVIRONMENT_IS_PTHREAD){PThread.initWorker()}else{PThread.initMainThread()}},initMainThread(){var pthreadPoolSize=8;while(pthreadPoolSize--){PThread.allocateUnusedWorker()}addOnPreRun(()=>{addRunDependency(\"loading-workers\");PThread.loadWasmModuleToAllWorkers(()=>removeRunDependency(\"loading-workers\"))})},initWorker(){noExitRuntime=false},setExitStatus:status=>{EXITSTATUS=status},terminateAllThreads__deps:[\"$terminateWorker\"],terminateAllThreads:()=>{for(var worker of PThread.runningWorkers){terminateWorker(worker)}for(var worker of PThread.unusedWorkers){terminateWorker(worker)}PThread.unusedWorkers=[];PThread.runningWorkers=[];PThread.pthreads=[]},returnWorkerToPool:worker=>{var pthread_ptr=worker.pthread_ptr;delete PThread.pthreads[pthread_ptr];PThread.unusedWorkers.push(worker);PThread.runningWorkers.splice(PThread.runningWorkers.indexOf(worker),1);worker.pthread_ptr=0;__emscripten_thread_free_data(pthread_ptr)},receiveObjectTransfer(data){},threadInitTLS(){PThread.tlsInitFunctions.forEach(f=>f())},loadWasmModuleToWorker:worker=>new Promise(onFinishedLoading=>{worker.onmessage=e=>{var d=e[\"data\"];var cmd=d[\"cmd\"];if(d[\"targetThread\"]&&d[\"targetThread\"]!=_pthread_self()){var targetWorker=PThread.pthreads[d[\"targetThread\"]];if(targetWorker){targetWorker.postMessage(d,d[\"transferList\"])}else{err(`Internal error! Worker sent a message \"${cmd}\" to target pthread ${d[\"targetThread\"]}, but that thread no longer exists!`)}return}if(cmd===\"checkMailbox\"){checkMailbox()}else if(cmd===\"spawnThread\"){spawnThread(d)}else if(cmd===\"cleanupThread\"){cleanupThread(d[\"thread\"])}else if(cmd===\"killThread\"){killThread(d[\"thread\"])}else if(cmd===\"cancelThread\"){cancelThread(d[\"thread\"])}else if(cmd===\"loaded\"){worker.loaded=true;if(ENVIRONMENT_IS_NODE&&!worker.pthread_ptr){worker.unref()}onFinishedLoading(worker)}else if(cmd===\"alert\"){alert(`Thread ${d[\"threadId\"]}: ${d[\"text\"]}`)}else if(d.target===\"setimmediate\"){worker.postMessage(d)}else if(cmd===\"callHandler\"){Module[d[\"handler\"]](...d[\"args\"])}else if(cmd){err(`worker sent an unknown command ${cmd}`)}};worker.onerror=e=>{var message=\"worker sent an error!\";err(`${message} ${e.filename}:${e.lineno}: ${e.message}`);throw e};if(ENVIRONMENT_IS_NODE){worker.on(\"message\",data=>worker.onmessage({data:data}));worker.on(\"error\",e=>worker.onerror(e))}var handlers=[];var knownHandlers=[\"onExit\",\"onAbort\",\"print\",\"printErr\"];for(var handler of knownHandlers){if(Module.hasOwnProperty(handler)){handlers.push(handler)}}worker.postMessage({\"cmd\":\"load\",\"handlers\":handlers,\"urlOrBlob\":Module[\"mainScriptUrlOrBlob\"]||_scriptDir,\"wasmMemory\":wasmMemory,\"wasmModule\":wasmModule})}),loadWasmModuleToAllWorkers(onMaybeReady){if(ENVIRONMENT_IS_PTHREAD){return onMaybeReady()}let pthreadPoolReady=Promise.all(PThread.unusedWorkers.map(PThread.loadWasmModuleToWorker));pthreadPoolReady.then(onMaybeReady)},allocateUnusedWorker(){var worker;var pthreadMainJs=locateFile(\"libwhisper.worker.js\");worker=new Worker(pthreadMainJs);PThread.unusedWorkers.push(worker)},getNewWorker(){if(PThread.unusedWorkers.length==0){PThread.allocateUnusedWorker();PThread.loadWasmModuleToWorker(PThread.unusedWorkers[0])}return PThread.unusedWorkers.pop()}};Module[\"PThread\"]=PThread;var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var establishStackSpace=()=>{var pthread_ptr=_pthread_self();var stackHigh=GROWABLE_HEAP_I32()[pthread_ptr+52>>2];var stackSize=GROWABLE_HEAP_I32()[pthread_ptr+56>>2];var stackLow=stackHigh-stackSize;_emscripten_stack_set_limits(stackHigh,stackLow);stackRestore(stackHigh)};Module[\"establishStackSpace\"]=establishStackSpace;function exitOnMainThread(returnCode){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(1,0,returnCode);_exit(returnCode)}var wasmTableMirror=[];var getWasmTableEntry=funcPtr=>{var func=wasmTableMirror[funcPtr];if(!func){if(funcPtr>=wasmTableMirror.length)wasmTableMirror.length=funcPtr+1;wasmTableMirror[funcPtr]=func=wasmTable.get(funcPtr)}return func};var invokeEntryPoint=(ptr,arg)=>{var result=getWasmTableEntry(ptr)(arg);function finish(result){if(keepRuntimeAlive()){PThread.setExitStatus(result)}else{__emscripten_thread_exit(result)}}finish(result)};Module[\"invokeEntryPoint\"]=invokeEntryPoint;var registerTLSInit=tlsInitFunc=>{PThread.tlsInitFunctions.push(tlsInitFunc)};function ExceptionInfo(excPtr){this.excPtr=excPtr;this.ptr=excPtr-24;this.set_type=function(type){GROWABLE_HEAP_U32()[this.ptr+4>>2]=type};this.get_type=function(){return GROWABLE_HEAP_U32()[this.ptr+4>>2]};this.set_destructor=function(destructor){GROWABLE_HEAP_U32()[this.ptr+8>>2]=destructor};this.get_destructor=function(){return GROWABLE_HEAP_U32()[this.ptr+8>>2]};this.set_caught=function(caught){caught=caught?1:0;GROWABLE_HEAP_I8()[this.ptr+12>>0]=caught};this.get_caught=function(){return GROWABLE_HEAP_I8()[this.ptr+12>>0]!=0};this.set_rethrown=function(rethrown){rethrown=rethrown?1:0;GROWABLE_HEAP_I8()[this.ptr+13>>0]=rethrown};this.get_rethrown=function(){return GROWABLE_HEAP_I8()[this.ptr+13>>0]!=0};this.init=function(type,destructor){this.set_adjusted_ptr(0);this.set_type(type);this.set_destructor(destructor)};this.set_adjusted_ptr=function(adjustedPtr){GROWABLE_HEAP_U32()[this.ptr+16>>2]=adjustedPtr};this.get_adjusted_ptr=function(){return GROWABLE_HEAP_U32()[this.ptr+16>>2]};this.get_exception_ptr=function(){var isPointer=___cxa_is_pointer_type(this.get_type());if(isPointer){return GROWABLE_HEAP_U32()[this.excPtr>>2]}var adjusted=this.get_adjusted_ptr();if(adjusted!==0)return adjusted;return this.excPtr}}var exceptionLast=0;var uncaughtExceptionCount=0;var ___cxa_throw=(ptr,type,destructor)=>{var info=new ExceptionInfo(ptr);info.init(type,destructor);exceptionLast=ptr;uncaughtExceptionCount++;throw exceptionLast};var ___emscripten_init_main_thread_js=tb=>{__emscripten_thread_init(tb,!ENVIRONMENT_IS_WORKER,1,!ENVIRONMENT_IS_WEB,65536,false);PThread.threadInitTLS()};var ___emscripten_thread_cleanup=thread=>{if(!ENVIRONMENT_IS_PTHREAD)cleanupThread(thread);else postMessage({\"cmd\":\"cleanupThread\",\"thread\":thread})};function pthreadCreateProxied(pthread_ptr,attr,startRoutine,arg){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(2,1,pthread_ptr,attr,startRoutine,arg);return ___pthread_create_js(pthread_ptr,attr,startRoutine,arg)}var ___pthread_create_js=(pthread_ptr,attr,startRoutine,arg)=>{if(typeof SharedArrayBuffer==\"undefined\"){err(\"Current environment does not support SharedArrayBuffer, pthreads are not available!\");return 6}var transferList=[];var error=0;if(ENVIRONMENT_IS_PTHREAD&&(transferList.length===0||error)){return pthreadCreateProxied(pthread_ptr,attr,startRoutine,arg)}if(error)return error;var threadParams={startRoutine:startRoutine,pthread_ptr:pthread_ptr,arg:arg,transferList:transferList};if(ENVIRONMENT_IS_PTHREAD){threadParams.cmd=\"spawnThread\";postMessage(threadParams,transferList);return 0}return spawnThread(threadParams)};var setErrNo=value=>{GROWABLE_HEAP_I32()[___errno_location()>>2]=value;return value};function ___syscall_fcntl64(fd,cmd,varargs){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(3,1,fd,cmd,varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(cmd){case 0:{var arg=SYSCALLS.get();if(arg<0){return-28}while(FS.streams[arg]){arg++}var newStream;newStream=FS.createStream(stream,arg);return newStream.fd}case 1:case 2:return 0;case 3:return stream.flags;case 4:{var arg=SYSCALLS.get();stream.flags|=arg;return 0}case 5:{var arg=SYSCALLS.getp();var offset=0;GROWABLE_HEAP_I16()[arg+offset>>1]=2;return 0}case 6:case 7:return 0;case 16:case 8:return-28;case 9:setErrNo(28);return-1;default:{return-28}}}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return-e.errno}}function ___syscall_ioctl(fd,op,varargs){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(4,1,fd,op,varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(op){case 21509:{if(!stream.tty)return-59;return 0}case 21505:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcgets){var termios=stream.tty.ops.ioctl_tcgets(stream);var argp=SYSCALLS.getp();GROWABLE_HEAP_I32()[argp>>2]=termios.c_iflag||0;GROWABLE_HEAP_I32()[argp+4>>2]=termios.c_oflag||0;GROWABLE_HEAP_I32()[argp+8>>2]=termios.c_cflag||0;GROWABLE_HEAP_I32()[argp+12>>2]=termios.c_lflag||0;for(var i=0;i<32;i++){GROWABLE_HEAP_I8()[argp+i+17>>0]=termios.c_cc[i]||0}return 0}return 0}case 21510:case 21511:case 21512:{if(!stream.tty)return-59;return 0}case 21506:case 21507:case 21508:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcsets){var argp=SYSCALLS.getp();var c_iflag=GROWABLE_HEAP_I32()[argp>>2];var c_oflag=GROWABLE_HEAP_I32()[argp+4>>2];var c_cflag=GROWABLE_HEAP_I32()[argp+8>>2];var c_lflag=GROWABLE_HEAP_I32()[argp+12>>2];var c_cc=[];for(var i=0;i<32;i++){c_cc.push(GROWABLE_HEAP_I8()[argp+i+17>>0])}return stream.tty.ops.ioctl_tcsets(stream.tty,op,{c_iflag:c_iflag,c_oflag:c_oflag,c_cflag:c_cflag,c_lflag:c_lflag,c_cc:c_cc})}return 0}case 21519:{if(!stream.tty)return-59;var argp=SYSCALLS.getp();GROWABLE_HEAP_I32()[argp>>2]=0;return 0}case 21520:{if(!stream.tty)return-59;return-28}case 21531:{var argp=SYSCALLS.getp();return FS.ioctl(stream,op,argp)}case 21523:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tiocgwinsz){var winsize=stream.tty.ops.ioctl_tiocgwinsz(stream.tty);var argp=SYSCALLS.getp();GROWABLE_HEAP_I16()[argp>>1]=winsize[0];GROWABLE_HEAP_I16()[argp+2>>1]=winsize[1]}return 0}case 21524:{if(!stream.tty)return-59;return 0}case 21515:{if(!stream.tty)return-59;return 0}default:return-28}}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return-e.errno}}function ___syscall_openat(dirfd,path,flags,varargs){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(5,1,dirfd,path,flags,varargs);SYSCALLS.varargs=varargs;try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);var mode=varargs?SYSCALLS.get():0;return FS.open(path,flags,mode).fd}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return-e.errno}}var __embind_register_bigint=(primitiveType,name,size,minRange,maxRange)=>{};var embind_init_charCodes=()=>{var codes=new Array(256);for(var i=0;i<256;++i){codes[i]=String.fromCharCode(i)}embind_charCodes=codes};var embind_charCodes=undefined;var readLatin1String=ptr=>{var ret=\"\";var c=ptr;while(GROWABLE_HEAP_U8()[c]){ret+=embind_charCodes[GROWABLE_HEAP_U8()[c++]]}return ret};var awaitingDependencies={};var registeredTypes={};var typeDependencies={};var BindingError=undefined;var throwBindingError=message=>{throw new BindingError(message)};var InternalError=undefined;var throwInternalError=message=>{throw new InternalError(message)};var whenDependentTypesAreResolved=(myTypes,dependentTypes,getTypeConverters)=>{myTypes.forEach(function(type){typeDependencies[type]=dependentTypes});function onComplete(typeConverters){var myTypeConverters=getTypeConverters(typeConverters);if(myTypeConverters.length!==myTypes.length){throwInternalError(\"Mismatched type converter count\")}for(var i=0;i<myTypes.length;++i){registerType(myTypes[i],myTypeConverters[i])}}var typeConverters=new Array(dependentTypes.length);var unregisteredTypes=[];var registered=0;dependentTypes.forEach((dt,i)=>{if(registeredTypes.hasOwnProperty(dt)){typeConverters[i]=registeredTypes[dt]}else{unregisteredTypes.push(dt);if(!awaitingDependencies.hasOwnProperty(dt)){awaitingDependencies[dt]=[]}awaitingDependencies[dt].push(()=>{typeConverters[i]=registeredTypes[dt];++registered;if(registered===unregisteredTypes.length){onComplete(typeConverters)}})}});if(0===unregisteredTypes.length){onComplete(typeConverters)}};function sharedRegisterType(rawType,registeredInstance,options={}){var name=registeredInstance.name;if(!rawType){throwBindingError(`type \"${name}\" must have a positive integer typeid pointer`)}if(registeredTypes.hasOwnProperty(rawType)){if(options.ignoreDuplicateRegistrations){return}else{throwBindingError(`Cannot register type '${name}' twice`)}}registeredTypes[rawType]=registeredInstance;delete typeDependencies[rawType];if(awaitingDependencies.hasOwnProperty(rawType)){var callbacks=awaitingDependencies[rawType];delete awaitingDependencies[rawType];callbacks.forEach(cb=>cb())}}function registerType(rawType,registeredInstance,options={}){if(!(\"argPackAdvance\"in registeredInstance)){throw new TypeError(\"registerType registeredInstance requires argPackAdvance\")}return sharedRegisterType(rawType,registeredInstance,options)}var GenericWireTypeSize=8;var __embind_register_bool=(rawType,name,trueValue,falseValue)=>{name=readLatin1String(name);registerType(rawType,{name:name,\"fromWireType\":function(wt){return!!wt},\"toWireType\":function(destructors,o){return o?trueValue:falseValue},\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":function(pointer){return this[\"fromWireType\"](GROWABLE_HEAP_U8()[pointer])},destructorFunction:null})};function handleAllocatorInit(){Object.assign(HandleAllocator.prototype,{get(id){return this.allocated[id]},has(id){return this.allocated[id]!==undefined},allocate(handle){var id=this.freelist.pop()||this.allocated.length;this.allocated[id]=handle;return id},free(id){this.allocated[id]=undefined;this.freelist.push(id)}})}function HandleAllocator(){this.allocated=[undefined];this.freelist=[]}var emval_handles=new HandleAllocator;var __emval_decref=handle=>{if(handle>=emval_handles.reserved&&0===--emval_handles.get(handle).refcount){emval_handles.free(handle)}};var count_emval_handles=()=>{var count=0;for(var i=emval_handles.reserved;i<emval_handles.allocated.length;++i){if(emval_handles.allocated[i]!==undefined){++count}}return count};var init_emval=()=>{emval_handles.allocated.push({value:undefined},{value:null},{value:true},{value:false});emval_handles.reserved=emval_handles.allocated.length;Module[\"count_emval_handles\"]=count_emval_handles};var Emval={toValue:handle=>{if(!handle){throwBindingError(\"Cannot use deleted val. handle = \"+handle)}return emval_handles.get(handle).value},toHandle:value=>{switch(value){case undefined:return 1;case null:return 2;case true:return 3;case false:return 4;default:{return emval_handles.allocate({refcount:1,value:value})}}}};function simpleReadValueFromPointer(pointer){return this[\"fromWireType\"](GROWABLE_HEAP_I32()[pointer>>2])}var __embind_register_emval=(rawType,name)=>{name=readLatin1String(name);registerType(rawType,{name:name,\"fromWireType\":handle=>{var rv=Emval.toValue(handle);__emval_decref(handle);return rv},\"toWireType\":(destructors,value)=>Emval.toHandle(value),\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":simpleReadValueFromPointer,destructorFunction:null})};var floatReadValueFromPointer=(name,width)=>{switch(width){case 4:return function(pointer){return this[\"fromWireType\"](GROWABLE_HEAP_F32()[pointer>>2])};case 8:return function(pointer){return this[\"fromWireType\"](GROWABLE_HEAP_F64()[pointer>>3])};default:throw new TypeError(`invalid float width (${width}): ${name}`)}};var __embind_register_float=(rawType,name,size)=>{name=readLatin1String(name);registerType(rawType,{name:name,\"fromWireType\":value=>value,\"toWireType\":(destructors,value)=>value,\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":floatReadValueFromPointer(name,size),destructorFunction:null})};var char_0=48;var char_9=57;var makeLegalFunctionName=name=>{if(undefined===name){return\"_unknown\"}name=name.replace(/[^a-zA-Z0-9_]/g,\"$\");var f=name.charCodeAt(0);if(f>=char_0&&f<=char_9){return`_${name}`}return name};var runDestructors=destructors=>{while(destructors.length){var ptr=destructors.pop();var del=destructors.pop();del(ptr)}};function createNamedFunction(name,body){name=makeLegalFunctionName(name);return{[name]:function(){return body.apply(this,arguments)}}[name]}function newFunc(constructor,argumentList){if(!(constructor instanceof Function)){throw new TypeError(`new_ called with constructor type ${typeof constructor} which is not a function`)}var dummy=createNamedFunction(constructor.name||\"unknownFunctionName\",function(){});dummy.prototype=constructor.prototype;var obj=new dummy;var r=constructor.apply(obj,argumentList);return r instanceof Object?r:obj}function craftInvokerFunction(humanName,argTypes,classType,cppInvokerFunc,cppTargetFunc,isAsync){var argCount=argTypes.length;if(argCount<2){throwBindingError(\"argTypes array size mismatch! Must at least get return value and 'this' types!\")}var isClassMethodFunc=argTypes[1]!==null&&classType!==null;var needsDestructorStack=false;for(var i=1;i<argTypes.length;++i){if(argTypes[i]!==null&&argTypes[i].destructorFunction===undefined){needsDestructorStack=true;break}}var returns=argTypes[0].name!==\"void\";var argsList=\"\";var argsListWired=\"\";for(var i=0;i<argCount-2;++i){argsList+=(i!==0?\", \":\"\")+\"arg\"+i;argsListWired+=(i!==0?\", \":\"\")+\"arg\"+i+\"Wired\"}var invokerFnBody=`\\n        return function ${makeLegalFunctionName(humanName)}(${argsList}) {\\n        if (arguments.length !== ${argCount-2}) {\\n          throwBindingError('function ${humanName} called with ' + arguments.length + ' arguments, expected ${argCount-2}');\\n        }`;if(needsDestructorStack){invokerFnBody+=\"var destructors = [];\\n\"}var dtorStack=needsDestructorStack?\"destructors\":\"null\";var args1=[\"throwBindingError\",\"invoker\",\"fn\",\"runDestructors\",\"retType\",\"classParam\"];var args2=[throwBindingError,cppInvokerFunc,cppTargetFunc,runDestructors,argTypes[0],argTypes[1]];if(isClassMethodFunc){invokerFnBody+=\"var thisWired = classParam.toWireType(\"+dtorStack+\", this);\\n\"}for(var i=0;i<argCount-2;++i){invokerFnBody+=\"var arg\"+i+\"Wired = argType\"+i+\".toWireType(\"+dtorStack+\", arg\"+i+\"); // \"+argTypes[i+2].name+\"\\n\";args1.push(\"argType\"+i);args2.push(argTypes[i+2])}if(isClassMethodFunc){argsListWired=\"thisWired\"+(argsListWired.length>0?\", \":\"\")+argsListWired}invokerFnBody+=(returns||isAsync?\"var rv = \":\"\")+\"invoker(fn\"+(argsListWired.length>0?\", \":\"\")+argsListWired+\");\\n\";if(needsDestructorStack){invokerFnBody+=\"runDestructors(destructors);\\n\"}else{for(var i=isClassMethodFunc?1:2;i<argTypes.length;++i){var paramName=i===1?\"thisWired\":\"arg\"+(i-2)+\"Wired\";if(argTypes[i].destructorFunction!==null){invokerFnBody+=paramName+\"_dtor(\"+paramName+\"); // \"+argTypes[i].name+\"\\n\";args1.push(paramName+\"_dtor\");args2.push(argTypes[i].destructorFunction)}}}if(returns){invokerFnBody+=\"var ret = retType.fromWireType(rv);\\n\"+\"return ret;\\n\"}else{}invokerFnBody+=\"}\\n\";args1.push(invokerFnBody);return newFunc(Function,args1).apply(null,args2)}var ensureOverloadTable=(proto,methodName,humanName)=>{if(undefined===proto[methodName].overloadTable){var prevFunc=proto[methodName];proto[methodName]=function(){if(!proto[methodName].overloadTable.hasOwnProperty(arguments.length)){throwBindingError(`Function '${humanName}' called with an invalid number of arguments (${arguments.length}) - expects one of (${proto[methodName].overloadTable})!`)}return proto[methodName].overloadTable[arguments.length].apply(this,arguments)};proto[methodName].overloadTable=[];proto[methodName].overloadTable[prevFunc.argCount]=prevFunc}};var exposePublicSymbol=(name,value,numArguments)=>{if(Module.hasOwnProperty(name)){if(undefined===numArguments||undefined!==Module[name].overloadTable&&undefined!==Module[name].overloadTable[numArguments]){throwBindingError(`Cannot register public name '${name}' twice`)}ensureOverloadTable(Module,name,name);if(Module.hasOwnProperty(numArguments)){throwBindingError(`Cannot register multiple overloads of a function with the same number of arguments (${numArguments})!`)}Module[name].overloadTable[numArguments]=value}else{Module[name]=value;if(undefined!==numArguments){Module[name].numArguments=numArguments}}};var heap32VectorToArray=(count,firstElement)=>{var array=[];for(var i=0;i<count;i++){array.push(GROWABLE_HEAP_U32()[firstElement+i*4>>2])}return array};var replacePublicSymbol=(name,value,numArguments)=>{if(!Module.hasOwnProperty(name)){throwInternalError(\"Replacing nonexistant public symbol\")}if(undefined!==Module[name].overloadTable&&undefined!==numArguments){Module[name].overloadTable[numArguments]=value}else{Module[name]=value;Module[name].argCount=numArguments}};var dynCallLegacy=(sig,ptr,args)=>{var f=Module[\"dynCall_\"+sig];return args&&args.length?f.apply(null,[ptr].concat(args)):f.call(null,ptr)};var dynCall=(sig,ptr,args)=>{if(sig.includes(\"j\")){return dynCallLegacy(sig,ptr,args)}var rtn=getWasmTableEntry(ptr).apply(null,args);return rtn};var getDynCaller=(sig,ptr)=>{var argCache=[];return function(){argCache.length=0;Object.assign(argCache,arguments);return dynCall(sig,ptr,argCache)}};var embind__requireFunction=(signature,rawFunction)=>{signature=readLatin1String(signature);function makeDynCaller(){if(signature.includes(\"j\")){return getDynCaller(signature,rawFunction)}return getWasmTableEntry(rawFunction)}var fp=makeDynCaller();if(typeof fp!=\"function\"){throwBindingError(`unknown function pointer with signature ${signature}: ${rawFunction}`)}return fp};var extendError=(baseErrorType,errorName)=>{var errorClass=createNamedFunction(errorName,function(message){this.name=errorName;this.message=message;var stack=new Error(message).stack;if(stack!==undefined){this.stack=this.toString()+\"\\n\"+stack.replace(/^Error(:[^\\n]*)?\\n/,\"\")}});errorClass.prototype=Object.create(baseErrorType.prototype);errorClass.prototype.constructor=errorClass;errorClass.prototype.toString=function(){if(this.message===undefined){return this.name}else{return`${this.name}: ${this.message}`}};return errorClass};var UnboundTypeError=undefined;var getTypeName=type=>{var ptr=___getTypeName(type);var rv=readLatin1String(ptr);_free(ptr);return rv};var throwUnboundTypeError=(message,types)=>{var unboundTypes=[];var seen={};function visit(type){if(seen[type]){return}if(registeredTypes[type]){return}if(typeDependencies[type]){typeDependencies[type].forEach(visit);return}unboundTypes.push(type);seen[type]=true}types.forEach(visit);throw new UnboundTypeError(`${message}: `+unboundTypes.map(getTypeName).join([\", \"]))};var __embind_register_function=(name,argCount,rawArgTypesAddr,signature,rawInvoker,fn,isAsync)=>{var argTypes=heap32VectorToArray(argCount,rawArgTypesAddr);name=readLatin1String(name);rawInvoker=embind__requireFunction(signature,rawInvoker);exposePublicSymbol(name,function(){throwUnboundTypeError(`Cannot call ${name} due to unbound types`,argTypes)},argCount-1);whenDependentTypesAreResolved([],argTypes,function(argTypes){var invokerArgsArray=[argTypes[0],null].concat(argTypes.slice(1));replacePublicSymbol(name,craftInvokerFunction(name,invokerArgsArray,null,rawInvoker,fn,isAsync),argCount-1);return[]})};var integerReadValueFromPointer=(name,width,signed)=>{switch(width){case 1:return signed?pointer=>GROWABLE_HEAP_I8()[pointer>>0]:pointer=>GROWABLE_HEAP_U8()[pointer>>0];case 2:return signed?pointer=>GROWABLE_HEAP_I16()[pointer>>1]:pointer=>GROWABLE_HEAP_U16()[pointer>>1];case 4:return signed?pointer=>GROWABLE_HEAP_I32()[pointer>>2]:pointer=>GROWABLE_HEAP_U32()[pointer>>2];default:throw new TypeError(`invalid integer width (${width}): ${name}`)}};var __embind_register_integer=(primitiveType,name,size,minRange,maxRange)=>{name=readLatin1String(name);if(maxRange===-1){maxRange=4294967295}var fromWireType=value=>value;if(minRange===0){var bitshift=32-8*size;fromWireType=value=>value<<bitshift>>>bitshift}var isUnsignedType=name.includes(\"unsigned\");var checkAssertions=(value,toTypeName)=>{};var toWireType;if(isUnsignedType){toWireType=function(destructors,value){checkAssertions(value,this.name);return value>>>0}}else{toWireType=function(destructors,value){checkAssertions(value,this.name);return value}}registerType(primitiveType,{name:name,\"fromWireType\":fromWireType,\"toWireType\":toWireType,\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":integerReadValueFromPointer(name,size,minRange!==0),destructorFunction:null})};var __embind_register_memory_view=(rawType,dataTypeIndex,name)=>{var typeMapping=[Int8Array,Uint8Array,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array];var TA=typeMapping[dataTypeIndex];function decodeMemoryView(handle){var size=GROWABLE_HEAP_U32()[handle>>2];var data=GROWABLE_HEAP_U32()[handle+4>>2];return new TA(GROWABLE_HEAP_I8().buffer,data,size)}name=readLatin1String(name);registerType(rawType,{name:name,\"fromWireType\":decodeMemoryView,\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":decodeMemoryView},{ignoreDuplicateRegistrations:true})};function readPointer(pointer){return this[\"fromWireType\"](GROWABLE_HEAP_U32()[pointer>>2])}var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,GROWABLE_HEAP_U8(),outPtr,maxBytesToWrite);var __embind_register_std_string=(rawType,name)=>{name=readLatin1String(name);var stdStringIsUTF8=name===\"std::string\";registerType(rawType,{name:name,\"fromWireType\":value=>{var length=GROWABLE_HEAP_U32()[value>>2];var payload=value+4;var str;if(stdStringIsUTF8){var decodeStartPtr=payload;for(var i=0;i<=length;++i){var currentBytePtr=payload+i;if(i==length||GROWABLE_HEAP_U8()[currentBytePtr]==0){var maxRead=currentBytePtr-decodeStartPtr;var stringSegment=UTF8ToString(decodeStartPtr,maxRead);if(str===undefined){str=stringSegment}else{str+=String.fromCharCode(0);str+=stringSegment}decodeStartPtr=currentBytePtr+1}}}else{var a=new Array(length);for(var i=0;i<length;++i){a[i]=String.fromCharCode(GROWABLE_HEAP_U8()[payload+i])}str=a.join(\"\")}_free(value);return str},\"toWireType\":(destructors,value)=>{if(value instanceof ArrayBuffer){value=new Uint8Array(value)}var length;var valueIsOfTypeString=typeof value==\"string\";if(!(valueIsOfTypeString||value instanceof Uint8Array||value instanceof Uint8ClampedArray||value instanceof Int8Array)){throwBindingError(\"Cannot pass non-string to std::string\")}if(stdStringIsUTF8&&valueIsOfTypeString){length=lengthBytesUTF8(value)}else{length=value.length}var base=_malloc(4+length+1);var ptr=base+4;GROWABLE_HEAP_U32()[base>>2]=length;if(stdStringIsUTF8&&valueIsOfTypeString){stringToUTF8(value,ptr,length+1)}else{if(valueIsOfTypeString){for(var i=0;i<length;++i){var charCode=value.charCodeAt(i);if(charCode>255){_free(ptr);throwBindingError(\"String has UTF-16 code units that do not fit in 8 bits\")}GROWABLE_HEAP_U8()[ptr+i]=charCode}}else{for(var i=0;i<length;++i){GROWABLE_HEAP_U8()[ptr+i]=value[i]}}}if(destructors!==null){destructors.push(_free,base)}return base},\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":readPointer,destructorFunction:ptr=>_free(ptr)})};var UTF16Decoder=typeof TextDecoder!=\"undefined\"?new TextDecoder(\"utf-16le\"):undefined;var UTF16ToString=(ptr,maxBytesToRead)=>{var endPtr=ptr;var idx=endPtr>>1;var maxIdx=idx+maxBytesToRead/2;while(!(idx>=maxIdx)&&GROWABLE_HEAP_U16()[idx])++idx;endPtr=idx<<1;if(endPtr-ptr>32&&UTF16Decoder)return UTF16Decoder.decode(GROWABLE_HEAP_U8().slice(ptr,endPtr));var str=\"\";for(var i=0;!(i>=maxBytesToRead/2);++i){var codeUnit=GROWABLE_HEAP_I16()[ptr+i*2>>1];if(codeUnit==0)break;str+=String.fromCharCode(codeUnit)}return str};var stringToUTF16=(str,outPtr,maxBytesToWrite)=>{if(maxBytesToWrite===undefined){maxBytesToWrite=2147483647}if(maxBytesToWrite<2)return 0;maxBytesToWrite-=2;var startPtr=outPtr;var numCharsToWrite=maxBytesToWrite<str.length*2?maxBytesToWrite/2:str.length;for(var i=0;i<numCharsToWrite;++i){var codeUnit=str.charCodeAt(i);GROWABLE_HEAP_I16()[outPtr>>1]=codeUnit;outPtr+=2}GROWABLE_HEAP_I16()[outPtr>>1]=0;return outPtr-startPtr};var lengthBytesUTF16=str=>str.length*2;var UTF32ToString=(ptr,maxBytesToRead)=>{var i=0;var str=\"\";while(!(i>=maxBytesToRead/4)){var utf32=GROWABLE_HEAP_I32()[ptr+i*4>>2];if(utf32==0)break;++i;if(utf32>=65536){var ch=utf32-65536;str+=String.fromCharCode(55296|ch>>10,56320|ch&1023)}else{str+=String.fromCharCode(utf32)}}return str};var stringToUTF32=(str,outPtr,maxBytesToWrite)=>{if(maxBytesToWrite===undefined){maxBytesToWrite=2147483647}if(maxBytesToWrite<4)return 0;var startPtr=outPtr;var endPtr=startPtr+maxBytesToWrite-4;for(var i=0;i<str.length;++i){var codeUnit=str.charCodeAt(i);if(codeUnit>=55296&&codeUnit<=57343){var trailSurrogate=str.charCodeAt(++i);codeUnit=65536+((codeUnit&1023)<<10)|trailSurrogate&1023}GROWABLE_HEAP_I32()[outPtr>>2]=codeUnit;outPtr+=4;if(outPtr+4>endPtr)break}GROWABLE_HEAP_I32()[outPtr>>2]=0;return outPtr-startPtr};var lengthBytesUTF32=str=>{var len=0;for(var i=0;i<str.length;++i){var codeUnit=str.charCodeAt(i);if(codeUnit>=55296&&codeUnit<=57343)++i;len+=4}return len};var __embind_register_std_wstring=(rawType,charSize,name)=>{name=readLatin1String(name);var decodeString,encodeString,getHeap,lengthBytesUTF,shift;if(charSize===2){decodeString=UTF16ToString;encodeString=stringToUTF16;lengthBytesUTF=lengthBytesUTF16;getHeap=()=>GROWABLE_HEAP_U16();shift=1}else if(charSize===4){decodeString=UTF32ToString;encodeString=stringToUTF32;lengthBytesUTF=lengthBytesUTF32;getHeap=()=>GROWABLE_HEAP_U32();shift=2}registerType(rawType,{name:name,\"fromWireType\":value=>{var length=GROWABLE_HEAP_U32()[value>>2];var HEAP=getHeap();var str;var decodeStartPtr=value+4;for(var i=0;i<=length;++i){var currentBytePtr=value+4+i*charSize;if(i==length||HEAP[currentBytePtr>>shift]==0){var maxReadBytes=currentBytePtr-decodeStartPtr;var stringSegment=decodeString(decodeStartPtr,maxReadBytes);if(str===undefined){str=stringSegment}else{str+=String.fromCharCode(0);str+=stringSegment}decodeStartPtr=currentBytePtr+charSize}}_free(value);return str},\"toWireType\":(destructors,value)=>{if(!(typeof value==\"string\")){throwBindingError(`Cannot pass non-string to C++ string type ${name}`)}var length=lengthBytesUTF(value);var ptr=_malloc(4+length+charSize);GROWABLE_HEAP_U32()[ptr>>2]=length>>shift;encodeString(value,ptr+4,length+charSize);if(destructors!==null){destructors.push(_free,ptr)}return ptr},\"argPackAdvance\":GenericWireTypeSize,\"readValueFromPointer\":simpleReadValueFromPointer,destructorFunction:ptr=>_free(ptr)})};var __embind_register_void=(rawType,name)=>{name=readLatin1String(name);registerType(rawType,{isVoid:true,name:name,\"argPackAdvance\":0,\"fromWireType\":()=>undefined,\"toWireType\":(destructors,o)=>undefined})};var nowIsMonotonic=true;var __emscripten_get_now_is_monotonic=()=>nowIsMonotonic;var maybeExit=()=>{if(!keepRuntimeAlive()){try{if(ENVIRONMENT_IS_PTHREAD)__emscripten_thread_exit(EXITSTATUS);else _exit(EXITSTATUS)}catch(e){handleException(e)}}};var callUserCallback=func=>{if(ABORT){return}try{func();maybeExit()}catch(e){handleException(e)}};var __emscripten_thread_mailbox_await=pthread_ptr=>{if(typeof Atomics.waitAsync===\"function\"){var wait=Atomics.waitAsync(GROWABLE_HEAP_I32(),pthread_ptr>>2,pthread_ptr);wait.value.then(checkMailbox);var waitingAsync=pthread_ptr+128;Atomics.store(GROWABLE_HEAP_I32(),waitingAsync>>2,1)}};Module[\"__emscripten_thread_mailbox_await\"]=__emscripten_thread_mailbox_await;var checkMailbox=()=>{var pthread_ptr=_pthread_self();if(pthread_ptr){__emscripten_thread_mailbox_await(pthread_ptr);callUserCallback(()=>__emscripten_check_mailbox())}};Module[\"checkMailbox\"]=checkMailbox;var __emscripten_notify_mailbox_postmessage=(targetThreadId,currThreadId,mainThreadId)=>{if(targetThreadId==currThreadId){setTimeout(()=>checkMailbox())}else if(ENVIRONMENT_IS_PTHREAD){postMessage({\"targetThread\":targetThreadId,\"cmd\":\"checkMailbox\"})}else{var worker=PThread.pthreads[targetThreadId];if(!worker){return}worker.postMessage({\"cmd\":\"checkMailbox\"})}};var withStackSave=f=>{var stack=stackSave();var ret=f();stackRestore(stack);return ret};var proxyToMainThread=function(index,sync){var numCallArgs=arguments.length-2;var outerArgs=arguments;return withStackSave(()=>{var serializedNumCallArgs=numCallArgs;var args=stackAlloc(serializedNumCallArgs*8);var b=args>>3;for(var i=0;i<numCallArgs;i++){var arg=outerArgs[2+i];GROWABLE_HEAP_F64()[b+i]=arg}return __emscripten_run_on_main_thread_js(index,serializedNumCallArgs,args,sync)})};var proxiedJSCallArgs=[];var __emscripten_receive_on_main_thread_js=(index,callingThread,numCallArgs,args)=>{proxiedJSCallArgs.length=numCallArgs;var b=args>>3;for(var i=0;i<numCallArgs;i++){proxiedJSCallArgs[i]=GROWABLE_HEAP_F64()[b+i]}var func=proxiedFunctionTable[index];PThread.currentProxiedOperationCallerThread=callingThread;var rtn=func.apply(null,proxiedJSCallArgs);PThread.currentProxiedOperationCallerThread=0;return rtn};var __emscripten_thread_set_strongref=thread=>{if(ENVIRONMENT_IS_NODE){PThread.pthreads[thread].ref()}};var requireRegisteredType=(rawType,humanName)=>{var impl=registeredTypes[rawType];if(undefined===impl){throwBindingError(humanName+\" has unknown type \"+getTypeName(rawType))}return impl};var __emval_as=(handle,returnType,destructorsRef)=>{handle=Emval.toValue(handle);returnType=requireRegisteredType(returnType,\"emval::as\");var destructors=[];var rd=Emval.toHandle(destructors);GROWABLE_HEAP_U32()[destructorsRef>>2]=rd;return returnType[\"toWireType\"](destructors,handle)};var emval_symbols={};var getStringOrSymbol=address=>{var symbol=emval_symbols[address];if(symbol===undefined){return readLatin1String(address)}return symbol};var emval_methodCallers=[];var __emval_call_void_method=(caller,handle,methodName,args)=>{caller=emval_methodCallers[caller];handle=Emval.toValue(handle);methodName=getStringOrSymbol(methodName);caller(handle,methodName,null,args)};var emval_addMethodCaller=caller=>{var id=emval_methodCallers.length;emval_methodCallers.push(caller);return id};var emval_lookupTypes=(argCount,argTypes)=>{var a=new Array(argCount);for(var i=0;i<argCount;++i){a[i]=requireRegisteredType(GROWABLE_HEAP_U32()[argTypes+i*4>>2],\"parameter \"+i)}return a};var emval_registeredMethods=[];var __emval_get_method_caller=(argCount,argTypes)=>{var types=emval_lookupTypes(argCount,argTypes);var retType=types[0];var signatureName=retType.name+\"_$\"+types.slice(1).map(function(t){return t.name}).join(\"_\")+\"$\";var returnId=emval_registeredMethods[signatureName];if(returnId!==undefined){return returnId}var params=[\"retType\"];var args=[retType];var argsList=\"\";for(var i=0;i<argCount-1;++i){argsList+=(i!==0?\", \":\"\")+\"arg\"+i;params.push(\"argType\"+i);args.push(types[1+i])}var functionName=makeLegalFunctionName(\"methodCaller_\"+signatureName);var functionBody=\"return function \"+functionName+\"(handle, name, destructors, args) {\\n\";var offset=0;for(var i=0;i<argCount-1;++i){functionBody+=\"    var arg\"+i+\" = argType\"+i+\".readValueFromPointer(args\"+(offset?\"+\"+offset:\"\")+\");\\n\";offset+=types[i+1][\"argPackAdvance\"]}functionBody+=\"    var rv = handle[name](\"+argsList+\");\\n\";for(var i=0;i<argCount-1;++i){if(types[i+1][\"deleteObject\"]){functionBody+=\"    argType\"+i+\".deleteObject(arg\"+i+\");\\n\"}}if(!retType.isVoid){functionBody+=\"    return retType.toWireType(destructors, rv);\\n\"}functionBody+=\"};\\n\";params.push(functionBody);var invokerFunction=newFunc(Function,params).apply(null,args);returnId=emval_addMethodCaller(invokerFunction);emval_registeredMethods[signatureName]=returnId;return returnId};var __emval_get_module_property=name=>{name=getStringOrSymbol(name);return Emval.toHandle(Module[name])};var __emval_get_property=(handle,key)=>{handle=Emval.toValue(handle);key=Emval.toValue(key);return Emval.toHandle(handle[key])};var __emval_incref=handle=>{if(handle>4){emval_handles.get(handle).refcount+=1}};var craftEmvalAllocator=argCount=>{var argsList=\"\";for(var i=0;i<argCount;++i){argsList+=(i!==0?\", \":\"\")+\"arg\"+i}var getMemory=()=>GROWABLE_HEAP_U32();var functionBody=\"return function emval_allocator_\"+argCount+\"(constructor, argTypes, args) {\\n\"+\"  var HEAPU32 = getMemory();\\n\";for(var i=0;i<argCount;++i){functionBody+=\"var argType\"+i+\" = requireRegisteredType(HEAPU32[((argTypes)>>2)], 'parameter \"+i+\"');\\n\"+\"var arg\"+i+\" = argType\"+i+\".readValueFromPointer(args);\\n\"+\"args += argType\"+i+\"['argPackAdvance'];\\n\"+\"argTypes += 4;\\n\"}functionBody+=\"var obj = new constructor(\"+argsList+\");\\n\"+\"return valueToHandle(obj);\\n\"+\"}\\n\";return new Function(\"requireRegisteredType\",\"Module\",\"valueToHandle\",\"getMemory\",functionBody)(requireRegisteredType,Module,Emval.toHandle,getMemory)};var emval_newers={};var __emval_new=(handle,argCount,argTypes,args)=>{handle=Emval.toValue(handle);var newer=emval_newers[argCount];if(!newer){newer=craftEmvalAllocator(argCount);emval_newers[argCount]=newer}return newer(handle,argTypes,args)};var __emval_new_cstring=v=>Emval.toHandle(getStringOrSymbol(v));var __emval_run_destructors=handle=>{var destructors=Emval.toValue(handle);runDestructors(destructors);__emval_decref(handle)};var _abort=()=>{abort(\"\")};var warnOnce=text=>{if(!warnOnce.shown)warnOnce.shown={};if(!warnOnce.shown[text]){warnOnce.shown[text]=1;if(ENVIRONMENT_IS_NODE)text=\"warning: \"+text;err(text)}};var _emscripten_check_blocking_allowed=()=>{};var runtimeKeepalivePush=()=>{runtimeKeepaliveCounter+=1};var _emscripten_exit_with_live_runtime=()=>{runtimeKeepalivePush();throw\"unwind\"};var getHeapMax=()=>2147483648;var _emscripten_get_heap_max=()=>getHeapMax();var _emscripten_get_now;_emscripten_get_now=()=>performance.timeOrigin+performance.now();var _emscripten_num_logical_cores=()=>{if(ENVIRONMENT_IS_NODE)return require(\"os\").cpus().length;return navigator[\"hardwareConcurrency\"]};var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=GROWABLE_HEAP_U8().length;requestedSize>>>=0;if(requestedSize<=oldSize){return false}var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}var alignUp=(x,multiple)=>x+(multiple-x%multiple)%multiple;for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignUp(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var ENV={};var getExecutableName=()=>thisProgram||\"./this.program\";var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(typeof navigator==\"object\"&&navigator.languages&&navigator.languages[0]||\"C\").replace(\"-\",\"_\")+\".UTF-8\";var env={\"USER\":\"web_user\",\"LOGNAME\":\"web_user\",\"PATH\":\"/\",\"PWD\":\"/\",\"HOME\":\"/home/web_user\",\"LANG\":lang,\"_\":getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};var stringToAscii=(str,buffer)=>{for(var i=0;i<str.length;++i){GROWABLE_HEAP_I8()[buffer++>>0]=str.charCodeAt(i)}GROWABLE_HEAP_I8()[buffer>>0]=0};var _environ_get=function(__environ,environ_buf){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(6,1,__environ,environ_buf);var bufSize=0;getEnvStrings().forEach((string,i)=>{var ptr=environ_buf+bufSize;GROWABLE_HEAP_U32()[__environ+i*4>>2]=ptr;stringToAscii(string,ptr);bufSize+=string.length+1});return 0};var _environ_sizes_get=function(penviron_count,penviron_buf_size){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(7,1,penviron_count,penviron_buf_size);var strings=getEnvStrings();GROWABLE_HEAP_U32()[penviron_count>>2]=strings.length;var bufSize=0;strings.forEach(string=>bufSize+=string.length+1);GROWABLE_HEAP_U32()[penviron_buf_size>>2]=bufSize;return 0};function _fd_close(fd){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(8,1,fd);try{var stream=SYSCALLS.getStreamFromFD(fd);FS.close(stream);return 0}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return e.errno}}var doReadv=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i<iovcnt;i++){var ptr=GROWABLE_HEAP_U32()[iov>>2];var len=GROWABLE_HEAP_U32()[iov+4>>2];iov+=8;var curr=FS.read(stream,GROWABLE_HEAP_I8(),ptr,len,offset);if(curr<0)return-1;ret+=curr;if(curr<len)break;if(typeof offset!==\"undefined\"){offset+=curr}}return ret};function _fd_read(fd,iov,iovcnt,pnum){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(9,1,fd,iov,iovcnt,pnum);try{var stream=SYSCALLS.getStreamFromFD(fd);var num=doReadv(stream,iov,iovcnt);GROWABLE_HEAP_U32()[pnum>>2]=num;return 0}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return e.errno}}var convertI32PairToI53Checked=(lo,hi)=>hi+2097152>>>0<4194305-!!lo?(lo>>>0)+hi*4294967296:NaN;function _fd_seek(fd,offset_low,offset_high,whence,newOffset){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(10,1,fd,offset_low,offset_high,whence,newOffset);var offset=convertI32PairToI53Checked(offset_low,offset_high);try{if(isNaN(offset))return 61;var stream=SYSCALLS.getStreamFromFD(fd);FS.llseek(stream,offset,whence);tempI64=[stream.position>>>0,(tempDouble=stream.position,+Math.abs(tempDouble)>=1?tempDouble>0?+Math.floor(tempDouble/4294967296)>>>0:~~+Math.ceil((tempDouble-+(~~tempDouble>>>0))/4294967296)>>>0:0)],GROWABLE_HEAP_I32()[newOffset>>2]=tempI64[0],GROWABLE_HEAP_I32()[newOffset+4>>2]=tempI64[1];if(stream.getdents&&offset===0&&whence===0)stream.getdents=null;return 0}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return e.errno}}var doWritev=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i<iovcnt;i++){var ptr=GROWABLE_HEAP_U32()[iov>>2];var len=GROWABLE_HEAP_U32()[iov+4>>2];iov+=8;var curr=FS.write(stream,GROWABLE_HEAP_I8(),ptr,len,offset);if(curr<0)return-1;ret+=curr;if(typeof offset!==\"undefined\"){offset+=curr}}return ret};function _fd_write(fd,iov,iovcnt,pnum){if(ENVIRONMENT_IS_PTHREAD)return proxyToMainThread(11,1,fd,iov,iovcnt,pnum);try{var stream=SYSCALLS.getStreamFromFD(fd);var num=doWritev(stream,iov,iovcnt);GROWABLE_HEAP_U32()[pnum>>2]=num;return 0}catch(e){if(typeof FS==\"undefined\"||!(e.name===\"ErrnoError\"))throw e;return e.errno}}var isLeapYear=year=>year%4===0&&(year%100!==0||year%400===0);var arraySum=(array,index)=>{var sum=0;for(var i=0;i<=index;sum+=array[i++]){}return sum};var MONTH_DAYS_LEAP=[31,29,31,30,31,30,31,31,30,31,30,31];var MONTH_DAYS_REGULAR=[31,28,31,30,31,30,31,31,30,31,30,31];var addDays=(date,days)=>{var newDate=new Date(date.getTime());while(days>0){var leap=isLeapYear(newDate.getFullYear());var currentMonth=newDate.getMonth();var daysInCurrentMonth=(leap?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR)[currentMonth];if(days>daysInCurrentMonth-newDate.getDate()){days-=daysInCurrentMonth-newDate.getDate()+1;newDate.setDate(1);if(currentMonth<11){newDate.setMonth(currentMonth+1)}else{newDate.setMonth(0);newDate.setFullYear(newDate.getFullYear()+1)}}else{newDate.setDate(newDate.getDate()+days);return newDate}}return newDate};var writeArrayToMemory=(array,buffer)=>{GROWABLE_HEAP_I8().set(array,buffer)};var _strftime=(s,maxsize,format,tm)=>{var tm_zone=GROWABLE_HEAP_U32()[tm+40>>2];var date={tm_sec:GROWABLE_HEAP_I32()[tm>>2],tm_min:GROWABLE_HEAP_I32()[tm+4>>2],tm_hour:GROWABLE_HEAP_I32()[tm+8>>2],tm_mday:GROWABLE_HEAP_I32()[tm+12>>2],tm_mon:GROWABLE_HEAP_I32()[tm+16>>2],tm_year:GROWABLE_HEAP_I32()[tm+20>>2],tm_wday:GROWABLE_HEAP_I32()[tm+24>>2],tm_yday:GROWABLE_HEAP_I32()[tm+28>>2],tm_isdst:GROWABLE_HEAP_I32()[tm+32>>2],tm_gmtoff:GROWABLE_HEAP_I32()[tm+36>>2],tm_zone:tm_zone?UTF8ToString(tm_zone):\"\"};var pattern=UTF8ToString(format);var EXPANSION_RULES_1={\"%c\":\"%a %b %d %H:%M:%S %Y\",\"%D\":\"%m/%d/%y\",\"%F\":\"%Y-%m-%d\",\"%h\":\"%b\",\"%r\":\"%I:%M:%S %p\",\"%R\":\"%H:%M\",\"%T\":\"%H:%M:%S\",\"%x\":\"%m/%d/%y\",\"%X\":\"%H:%M:%S\",\"%Ec\":\"%c\",\"%EC\":\"%C\",\"%Ex\":\"%m/%d/%y\",\"%EX\":\"%H:%M:%S\",\"%Ey\":\"%y\",\"%EY\":\"%Y\",\"%Od\":\"%d\",\"%Oe\":\"%e\",\"%OH\":\"%H\",\"%OI\":\"%I\",\"%Om\":\"%m\",\"%OM\":\"%M\",\"%OS\":\"%S\",\"%Ou\":\"%u\",\"%OU\":\"%U\",\"%OV\":\"%V\",\"%Ow\":\"%w\",\"%OW\":\"%W\",\"%Oy\":\"%y\"};for(var rule in EXPANSION_RULES_1){pattern=pattern.replace(new RegExp(rule,\"g\"),EXPANSION_RULES_1[rule])}var WEEKDAYS=[\"Sunday\",\"Monday\",\"Tuesday\",\"Wednesday\",\"Thursday\",\"Friday\",\"Saturday\"];var MONTHS=[\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"];function leadingSomething(value,digits,character){var str=typeof value==\"number\"?value.toString():value||\"\";while(str.length<digits){str=character[0]+str}return str}function leadingNulls(value,digits){return leadingSomething(value,digits,\"0\")}function compareByDay(date1,date2){function sgn(value){return value<0?-1:value>0?1:0}var compare;if((compare=sgn(date1.getFullYear()-date2.getFullYear()))===0){if((compare=sgn(date1.getMonth()-date2.getMonth()))===0){compare=sgn(date1.getDate()-date2.getDate())}}return compare}function getFirstWeekStartDate(janFourth){switch(janFourth.getDay()){case 0:return new Date(janFourth.getFullYear()-1,11,29);case 1:return janFourth;case 2:return new Date(janFourth.getFullYear(),0,3);case 3:return new Date(janFourth.getFullYear(),0,2);case 4:return new Date(janFourth.getFullYear(),0,1);case 5:return new Date(janFourth.getFullYear()-1,11,31);case 6:return new Date(janFourth.getFullYear()-1,11,30)}}function getWeekBasedYear(date){var thisDate=addDays(new Date(date.tm_year+1900,0,1),date.tm_yday);var janFourthThisYear=new Date(thisDate.getFullYear(),0,4);var janFourthNextYear=new Date(thisDate.getFullYear()+1,0,4);var firstWeekStartThisYear=getFirstWeekStartDate(janFourthThisYear);var firstWeekStartNextYear=getFirstWeekStartDate(janFourthNextYear);if(compareByDay(firstWeekStartThisYear,thisDate)<=0){if(compareByDay(firstWeekStartNextYear,thisDate)<=0){return thisDate.getFullYear()+1}return thisDate.getFullYear()}return thisDate.getFullYear()-1}var EXPANSION_RULES_2={\"%a\":date=>WEEKDAYS[date.tm_wday].substring(0,3),\"%A\":date=>WEEKDAYS[date.tm_wday],\"%b\":date=>MONTHS[date.tm_mon].substring(0,3),\"%B\":date=>MONTHS[date.tm_mon],\"%C\":date=>{var year=date.tm_year+1900;return leadingNulls(year/100|0,2)},\"%d\":date=>leadingNulls(date.tm_mday,2),\"%e\":date=>leadingSomething(date.tm_mday,2,\" \"),\"%g\":date=>getWeekBasedYear(date).toString().substring(2),\"%G\":date=>getWeekBasedYear(date),\"%H\":date=>leadingNulls(date.tm_hour,2),\"%I\":date=>{var twelveHour=date.tm_hour;if(twelveHour==0)twelveHour=12;else if(twelveHour>12)twelveHour-=12;return leadingNulls(twelveHour,2)},\"%j\":date=>leadingNulls(date.tm_mday+arraySum(isLeapYear(date.tm_year+1900)?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR,date.tm_mon-1),3),\"%m\":date=>leadingNulls(date.tm_mon+1,2),\"%M\":date=>leadingNulls(date.tm_min,2),\"%n\":()=>\"\\n\",\"%p\":date=>{if(date.tm_hour>=0&&date.tm_hour<12){return\"AM\"}return\"PM\"},\"%S\":date=>leadingNulls(date.tm_sec,2),\"%t\":()=>\"\\t\",\"%u\":date=>date.tm_wday||7,\"%U\":date=>{var days=date.tm_yday+7-date.tm_wday;return leadingNulls(Math.floor(days/7),2)},\"%V\":date=>{var val=Math.floor((date.tm_yday+7-(date.tm_wday+6)%7)/7);if((date.tm_wday+371-date.tm_yday-2)%7<=2){val++}if(!val){val=52;var dec31=(date.tm_wday+7-date.tm_yday-1)%7;if(dec31==4||dec31==5&&isLeapYear(date.tm_year%400-1)){val++}}else if(val==53){var jan1=(date.tm_wday+371-date.tm_yday)%7;if(jan1!=4&&(jan1!=3||!isLeapYear(date.tm_year)))val=1}return leadingNulls(val,2)},\"%w\":date=>date.tm_wday,\"%W\":date=>{var days=date.tm_yday+7-(date.tm_wday+6)%7;return leadingNulls(Math.floor(days/7),2)},\"%y\":date=>(date.tm_year+1900).toString().substring(2),\"%Y\":date=>date.tm_year+1900,\"%z\":date=>{var off=date.tm_gmtoff;var ahead=off>=0;off=Math.abs(off)/60;off=off/60*100+off%60;return(ahead?\"+\":\"-\")+String(\"0000\"+off).slice(-4)},\"%Z\":date=>date.tm_zone,\"%%\":()=>\"%\"};pattern=pattern.replace(/%%/g,\"\\0\\0\");for(var rule in EXPANSION_RULES_2){if(pattern.includes(rule)){pattern=pattern.replace(new RegExp(rule,\"g\"),EXPANSION_RULES_2[rule](date))}}pattern=pattern.replace(/\\0\\0/g,\"%\");var bytes=intArrayFromString(pattern,false);if(bytes.length>maxsize){return 0}writeArrayToMemory(bytes,s);return bytes.length-1};var _strftime_l=(s,maxsize,format,tm,loc)=>_strftime(s,maxsize,format,tm);PThread.init();var FSNode=function(parent,name,mode,rdev){if(!parent){parent=this}this.parent=parent;this.mount=parent.mount;this.mounted=null;this.id=FS.nextInode++;this.name=name;this.mode=mode;this.node_ops={};this.stream_ops={};this.rdev=rdev};var readMode=292|73;var writeMode=146;Object.defineProperties(FSNode.prototype,{read:{get:function(){return(this.mode&readMode)===readMode},set:function(val){val?this.mode|=readMode:this.mode&=~readMode}},write:{get:function(){return(this.mode&writeMode)===writeMode},set:function(val){val?this.mode|=writeMode:this.mode&=~writeMode}},isFolder:{get:function(){return FS.isDir(this.mode)}},isDevice:{get:function(){return FS.isChrdev(this.mode)}}});FS.FSNode=FSNode;FS.createPreloadedFile=FS_createPreloadedFile;FS.staticInit();Module[\"FS_createPath\"]=FS.createPath;Module[\"FS_createDataFile\"]=FS.createDataFile;Module[\"FS_createPreloadedFile\"]=FS.createPreloadedFile;Module[\"FS_unlink\"]=FS.unlink;Module[\"FS_createLazyFile\"]=FS.createLazyFile;Module[\"FS_createDevice\"]=FS.createDevice;embind_init_charCodes();BindingError=Module[\"BindingError\"]=class BindingError extends Error{constructor(message){super(message);this.name=\"BindingError\"}};InternalError=Module[\"InternalError\"]=class InternalError extends Error{constructor(message){super(message);this.name=\"InternalError\"}};handleAllocatorInit();init_emval();UnboundTypeError=Module[\"UnboundTypeError\"]=extendError(Error,\"UnboundTypeError\");var proxiedFunctionTable=[_proc_exit,exitOnMainThread,pthreadCreateProxied,___syscall_fcntl64,___syscall_ioctl,___syscall_openat,_environ_get,_environ_sizes_get,_fd_close,_fd_read,_fd_seek,_fd_write];var wasmImports={d:___cxa_throw,J:___emscripten_init_main_thread_js,m:___emscripten_thread_cleanup,G:___pthread_create_js,q:___syscall_fcntl64,L:___syscall_ioctl,M:___syscall_openat,y:__embind_register_bigint,R:__embind_register_bool,Q:__embind_register_emval,s:__embind_register_float,h:__embind_register_function,g:__embind_register_integer,c:__embind_register_memory_view,r:__embind_register_std_string,k:__embind_register_std_wstring,S:__embind_register_void,P:__emscripten_get_now_is_monotonic,F:__emscripten_notify_mailbox_postmessage,H:__emscripten_receive_on_main_thread_js,I:__emscripten_thread_mailbox_await,O:__emscripten_thread_set_strongref,w:__emval_as,U:__emval_call_void_method,f:__emval_decref,V:__emval_get_method_caller,u:__emval_get_module_property,i:__emval_get_property,l:__emval_incref,t:__emval_new,j:__emval_new_cstring,v:__emval_run_destructors,b:_abort,n:_emscripten_check_blocking_allowed,N:_emscripten_exit_with_live_runtime,z:_emscripten_get_heap_max,e:_emscripten_get_now,A:_emscripten_num_logical_cores,E:_emscripten_resize_heap,C:_environ_get,D:_environ_sizes_get,T:_exit,o:_fd_close,K:_fd_read,x:_fd_seek,p:_fd_write,a:wasmMemory||Module[\"wasmMemory\"],B:_strftime_l};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports[\"W\"])();var _free=a0=>(_free=wasmExports[\"Y\"])(a0);var _malloc=a0=>(_malloc=wasmExports[\"Z\"])(a0);var __emscripten_tls_init=Module[\"__emscripten_tls_init\"]=()=>(__emscripten_tls_init=Module[\"__emscripten_tls_init\"]=wasmExports[\"_\"])();var _pthread_self=Module[\"_pthread_self\"]=()=>(_pthread_self=Module[\"_pthread_self\"]=wasmExports[\"$\"])();var ___getTypeName=a0=>(___getTypeName=wasmExports[\"aa\"])(a0);var __embind_initialize_bindings=Module[\"__embind_initialize_bindings\"]=()=>(__embind_initialize_bindings=Module[\"__embind_initialize_bindings\"]=wasmExports[\"ba\"])();var ___errno_location=()=>(___errno_location=wasmExports[\"ca\"])();var __emscripten_thread_init=Module[\"__emscripten_thread_init\"]=(a0,a1,a2,a3,a4,a5)=>(__emscripten_thread_init=Module[\"__emscripten_thread_init\"]=wasmExports[\"da\"])(a0,a1,a2,a3,a4,a5);var __emscripten_thread_crashed=Module[\"__emscripten_thread_crashed\"]=()=>(__emscripten_thread_crashed=Module[\"__emscripten_thread_crashed\"]=wasmExports[\"ea\"])();var _emscripten_main_thread_process_queued_calls=()=>(_emscripten_main_thread_process_queued_calls=wasmExports[\"emscripten_main_thread_process_queued_calls\"])();var _emscripten_main_runtime_thread_id=()=>(_emscripten_main_runtime_thread_id=wasmExports[\"emscripten_main_runtime_thread_id\"])();var __emscripten_run_on_main_thread_js=(a0,a1,a2,a3)=>(__emscripten_run_on_main_thread_js=wasmExports[\"fa\"])(a0,a1,a2,a3);var __emscripten_thread_free_data=a0=>(__emscripten_thread_free_data=wasmExports[\"ga\"])(a0);var __emscripten_thread_exit=Module[\"__emscripten_thread_exit\"]=a0=>(__emscripten_thread_exit=Module[\"__emscripten_thread_exit\"]=wasmExports[\"ha\"])(a0);var __emscripten_check_mailbox=Module[\"__emscripten_check_mailbox\"]=()=>(__emscripten_check_mailbox=Module[\"__emscripten_check_mailbox\"]=wasmExports[\"ia\"])();var _emscripten_stack_set_limits=(a0,a1)=>(_emscripten_stack_set_limits=wasmExports[\"ja\"])(a0,a1);var stackSave=()=>(stackSave=wasmExports[\"ka\"])();var stackRestore=a0=>(stackRestore=wasmExports[\"la\"])(a0);var stackAlloc=a0=>(stackAlloc=wasmExports[\"ma\"])(a0);var ___cxa_is_pointer_type=a0=>(___cxa_is_pointer_type=wasmExports[\"na\"])(a0);var dynCall_jiji=Module[\"dynCall_jiji\"]=(a0,a1,a2,a3,a4)=>(dynCall_jiji=Module[\"dynCall_jiji\"]=wasmExports[\"oa\"])(a0,a1,a2,a3,a4);var dynCall_viijii=Module[\"dynCall_viijii\"]=(a0,a1,a2,a3,a4,a5,a6)=>(dynCall_viijii=Module[\"dynCall_viijii\"]=wasmExports[\"pa\"])(a0,a1,a2,a3,a4,a5,a6);var dynCall_iiiiij=Module[\"dynCall_iiiiij\"]=(a0,a1,a2,a3,a4,a5,a6)=>(dynCall_iiiiij=Module[\"dynCall_iiiiij\"]=wasmExports[\"qa\"])(a0,a1,a2,a3,a4,a5,a6);var dynCall_iiiiijj=Module[\"dynCall_iiiiijj\"]=(a0,a1,a2,a3,a4,a5,a6,a7,a8)=>(dynCall_iiiiijj=Module[\"dynCall_iiiiijj\"]=wasmExports[\"ra\"])(a0,a1,a2,a3,a4,a5,a6,a7,a8);var dynCall_iiiiiijj=Module[\"dynCall_iiiiiijj\"]=(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9)=>(dynCall_iiiiiijj=Module[\"dynCall_iiiiiijj\"]=wasmExports[\"sa\"])(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9);function intArrayFromBase64(s){if(typeof ENVIRONMENT_IS_NODE!=\"undefined\"&&ENVIRONMENT_IS_NODE){var buf=Buffer.from(s,\"base64\");return new Uint8Array(buf.buffer,buf.byteOffset,buf.length)}try{var decoded=atob(s);var bytes=new Uint8Array(decoded.length);for(var i=0;i<decoded.length;++i){bytes[i]=decoded.charCodeAt(i)}return bytes}catch(_){throw new Error(\"Converting base64 string to bytes failed.\")}}function tryParseAsDataURI(filename){if(!isDataURI(filename)){return}return intArrayFromBase64(filename.slice(dataURIPrefix.length))}Module[\"addRunDependency\"]=addRunDependency;Module[\"removeRunDependency\"]=removeRunDependency;Module[\"FS_createPath\"]=FS.createPath;Module[\"FS_createDataFile\"]=FS.createDataFile;Module[\"FS_createLazyFile\"]=FS.createLazyFile;Module[\"FS_createDevice\"]=FS.createDevice;Module[\"FS_unlink\"]=FS.unlink;Module[\"keepRuntimeAlive\"]=keepRuntimeAlive;Module[\"wasmMemory\"]=wasmMemory;Module[\"ExitStatus\"]=ExitStatus;Module[\"FS_createPreloadedFile\"]=FS.createPreloadedFile;Module[\"PThread\"]=PThread;var calledRun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(ENVIRONMENT_IS_PTHREAD){readyPromiseResolve(Module);initRuntime();startWorker(Module);return}preRun();if(runDependencies>0){return}function doRun(){if(calledRun)return;calledRun=true;Module[\"calledRun\"]=true;if(ABORT)return;initRuntime();readyPromiseResolve(Module);if(Module[\"onRuntimeInitialized\"])Module[\"onRuntimeInitialized\"]();postRun()}if(Module[\"setStatus\"]){Module[\"setStatus\"](\"Running...\");setTimeout(function(){setTimeout(function(){Module[\"setStatus\"](\"\")},1);doRun()},1)}else{doRun()}}if(Module[\"preInit\"]){if(typeof Module[\"preInit\"]==\"function\")Module[\"preInit\"]=[Module[\"preInit\"]];while(Module[\"preInit\"].length>0){Module[\"preInit\"].pop()()}}run();\n\n\n  return moduleArg.ready\n}\n\n);\n})();\nif (typeof exports === 'object' && typeof module === 'object')\n  module.exports = whisper_factory;\nelse if (typeof define === 'function' && define['amd'])\n  define([], () => whisper_factory);\n"
  },
  {
    "path": "bindings/ruby/.gitignore",
    "content": "LICENSE\npkg/\nlib/whisper.*\next/examples/\next/ggml/\next/include/\next/scripts/\next/src/\ntest/fixtures/\n"
  },
  {
    "path": "bindings/ruby/README.md",
    "content": "whispercpp\n==========\n\n![whisper.cpp](https://user-images.githubusercontent.com/1991296/235238348-05d0f6a4-da44-4900-a1de-d0707e75b763.jpeg)\n\nRuby bindings for [whisper.cpp][], an interface of automatic speech recognition model.\n\nUsage\n-----\n\n```ruby\nrequire \"whisper\"\n\nwhisper = Whisper::Context.new(\"base\")\n\nparams = Whisper::Params.new(\n  language: \"en\",\n  offset: 10_000,\n  duration: 60_000,\n  max_text_tokens: 300,\n  translate: true,\n  print_timestamps: false,\n  initial_prompt: \"Initial prompt here.\",\n  carry_initial_prompt: true\n)\n\nwhisper.transcribe(\"path/to/audio.wav\", params) do |whole_text|\n  puts whole_text\nend\n\n```\n\n### Preparing model ###\n\nSome models are prepared up-front:\n\nYou also can use shorthand for pre-converted models:\n\n```ruby\nwhisper = Whisper::Context.new(\"base.en\")\n```\n\nYou can see the list of prepared model names by `Whisper::Model.pre_converted_models.keys`:\n\n```ruby\nputs Whisper::Model.pre_converted_models.keys\n# tiny\n# tiny.en\n# tiny-q5_1\n# tiny.en-q5_1\n# tiny-q8_0\n# base\n# base.en\n# base-q5_1\n# base.en-q5_1\n# base-q8_0\n#   :\n#   :\n```\n\nYou can also retrieve each model:\n\n```ruby\nbase_en = Whisper::Model.pre_converted_models[\"base.en\"]\nwhisper = Whisper::Context.new(base_en)\n```\n\nAt first time you use a model, it is downloaded automatically. After that, downloaded cached file is used. To clear cache, call `#clear_cache`:\n\n```ruby\nWhisper::Model.pre_converted_models[\"base\"].clear_cache\n```\n\nYou can also use local model files you prepared:\n\n```ruby\nwhisper = Whisper::Context.new(\"path/to/your/model.bin\")\n```\n\nOr, you can download model files:\n\n```ruby\nwhisper = Whisper::Context.new(\"https://example.net/uri/of/your/model.bin\")\n# Or\nuri = URI(\"https://example.net/uri/of/your/model.bin\")\nwhisper = Whisper::Context.new(uri)\n```\n\nSee [models][] page for details.\n\n### Preparing audio file ###\n\nCurrently, whisper.cpp accepts only 16-bit WAV files.\n\n### Voice Activity Detection (VAD) ###\n\nSupport for Voice Activity Detection (VAD) can be enabled by setting `Whisper::Params`'s `vad` argument to `true` and specifying VAD model:\n\n```ruby\nWhisper::Params.new(\n  vad: true,\n  vad_model_path: \"silero-v6.2.0\",\n  # other arguments...\n)\n```\n\nWhen you pass the model name (`\"silero-v6.2.0\"`) or URI (`https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin`), it will be downloaded automatically.\nCurrently, \"silero-v6.2.0\" is registered as pre-converted model like ASR models. You also specify file path or URI of model.\n\nIf you need configure VAD behavior, pass params for that:\n\n```ruby\nWhisper::Params.new(\n  vad: true,\n  vad_model_path: \"silero-v6.2.0\",\n  vad_params: Whisper::VAD::Params.new(\n    threshold: 1.0, # defaults to 0.5\n    min_speech_duration_ms: 500, # defaults to 250\n    min_silence_duration_ms: 200, # defaults to 100\n    max_speech_duration_s: 30000, # default is FLT_MAX,\n    speech_pad_ms: 50, # defaults to 30\n    samples_overlap: 0.5 # defaults to 0.1\n  ),\n  # other arguments...\n)\n```\n\nFor details on VAD, see [whisper.cpp's README](https://github.com/ggml-org/whisper.cpp?tab=readme-ov-file#voice-activity-detection-vad).\n\n### Output ###\n\nwhispercpp supports SRT and WebVTT output:\n\n```ruby\nputs whisper.transcribe(\"path/to/audio.wav\", Whisper::Params.new).to_webvtt\n# =>\nWEBVTT\n\n1\n00:00:00.000 --> 00:00:03.860\n My thought I have nobody by a beauty and will as you poured.\n\n2\n00:00:03.860 --> 00:00:09.840\n Mr. Rochester is sub in that so-don't find simplest, and devoted about, to let might in\n\n3\n00:00:09.840 --> 00:00:09.940\n a\n\n```\n\nYou may call `#to_srt`, too\n\nInstallation\n------------\n\nInstall the gem and add to the application's Gemfile by executing:\n\n    $ bundle add whispercpp\n\nIf bundler is not being used to manage dependencies, install the gem by executing:\n\n    $ gem install whispercpp\n\nYou can pass build options for whisper.cpp, for instance:\n\n    $ bundle config build.whispercpp --enable-ggml-cuda\n\nor,\n\n    $ gem install whispercpp -- --enable-ggml-cuda\n\nSee whisper.cpp's [README](https://github.com/ggml-org/whisper.cpp/blob/master/README.md) for available options. You need convert options present in the README to Ruby-style options, for example:\n\nBoolean options:\n\n* `-DGGML_BLAS=1` -> `--enable-ggml-blas`\n* `-DWHISER_COREML=OFF` -> `--disable-whisper-coreml`\n\nArgument options:\n\n* `-DGGML_CUDA_COMPRESSION_MODE=size` -> `--ggml-cuda-compression-mode=size`\n\nCombination:\n\n* `-DGGML_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES=\"86\"` -> `--enable-ggml-cuda --cmake_cuda-architectures=\"86\"`\n\nFor boolean options like `GGML_CUDA`, the README says `-DGGML_CUDA=1`. You need strip `-D`, prepend `--enable-` for `1` or `ON` (`--disable-` for `0` or `OFF`) and make it kebab-case: `--enable-ggml-cuda`.  \nFor options which require arguments like `CMAKE_CUDA_ARCHITECTURES`, the README says `-DCMAKE_CUDA_ARCHITECTURES=\"86\"`. You need strip `-D`, prepend `--`, make it kebab-case, append `=` and append argument: `--cmake-cuda-architectures=\"86\"`.\n\nAPI\n---\n\n### Transcription ###\n\nBy default, `Whisper::Context#transcribe` works in a single thread. You can make it work in parallel by passing `n_processors` option:\n\n```ruby\nwhisper.transcribe(\"path/to/audio.wav\", params, n_processors: Etc.nprocessors)\n```\n\nNote that transcription occasionally might be low accuracy when it works in parallel.\n\n### Segments ###\n\nOnce `Whisper::Context#transcribe` called, you can retrieve segments by `#each_segment`:\n\n```ruby\ndef format_time(time_ms)\n  sec, decimal_part = time_ms.divmod(1000)\n  min, sec = sec.divmod(60)\n  hour, min = min.divmod(60)\n  \"%02d:%02d:%02d.%03d\" % [hour, min, sec, decimal_part]\nend\n\nwhisper\n  .transcribe(\"path/to/audio.wav\", params)\n  .each_segment.with_index do |segment, index|\n    line = \"[%{nth}: %{st} --> %{ed}] %{text}\" % {\n      nth: index + 1,\n      st: format_time(segment.start_time),\n      ed: format_time(segment.end_time),\n      text: segment.text\n    }\n    line << \" (speaker turned)\" if segment.speaker_turn_next?\n    puts line\n  end\n\n```\n\nYou can also add hook to params called on new segment:\n\n```ruby\n# Add hook before calling #transcribe\nparams.on_new_segment do |segment|\n  line = \"[%{st} --> %{ed}] %{text}\" % {\n    st: format_time(segment.start_time),\n    ed: format_time(segment.end_time),\n    text: segment.text\n  }\n  line << \" (speaker turned)\" if segment.speaker_turn_next?\n  puts line\nend\n\nwhisper.transcribe(\"path/to/audio.wav\", params)\n\n```\n\n### Tokens ###\n\nEach segment has tokens.\n\nTo enable token timestamps, you need to set `Whisper::Params#token_timestamps = true`. Then, retrieve tokens from segments using `Whisper::Segment#each_token`.\n\n```ruby\nwhisper = Whisper::Context.new(\"base.en\")\nparams = Whisper::Params.new(token_timestamps: true)\nwhisper\n  .transcribe(\"path/to/audio.wav\", params)\n  .each_segment do |segment|\n    segment.each_token do |token|\n      token => {start_time:, end_time:, text:, probability:}\n      st = \"%05.2fs\" % (start_time / 1000.0)\n      et = \"%05.2fs\" % (end_time / 1000.0)\n      prob = \"%.1f%%\" % (probability * 100)\n      puts \"[#{st} --> #{et}] #{text} (#{prob})\"\n    end\n  end\n```\n\n```\n[00.00s --> 00.00s] [_BEG_] (84.2%)\n[00.32s --> 00.37s]  And (71.2%)\n[00.37s --> 00.53s]  so (98.5%)\n[00.69s --> 00.85s]  my (70.7%)\n[00.85s --> 01.59s]  fellow (99.5%)\n[01.59s --> 02.10s]  Americans (90.1%)\n[02.85s --> 03.30s] , (28.4%)\n[03.30s --> 04.14s]  ask (79.8%)\n[04.14s --> 04.28s]  not (78.9%)\n[05.03s --> 05.35s]  what (93.3%)\n[05.41s --> 05.74s]  your (98.8%)\n[05.74s --> 06.41s]  country (99.6%)\n[06.41s --> 06.74s]  can (97.7%)\n[06.74s --> 06.92s]  do (99.0%)\n[07.00s --> 07.00s]  for (95.8%)\n[07.01s --> 07.52s]  you (98.5%)\n[07.81s --> 08.05s] , (49.3%)\n[08.19s --> 08.37s]  ask (65.6%)\n[08.37s --> 08.75s]  what (98.8%)\n[08.91s --> 09.04s]  you (98.2%)\n[09.04s --> 09.32s]  can (96.9%)\n[09.32s --> 09.38s]  do (90.3%)\n[09.44s --> 09.76s]  for (91.8%)\n[09.76s --> 09.99s]  your (98.2%)\n[10.02s --> 10.36s]  country (99.6%)\n[10.51s --> 10.99s] . (87.0%)\n[11.00s --> 11.00s] [_TT_550] (7.6%)\n```\n\n### Models ###\n\nYou can see model information:\n\n```ruby\nwhisper = Whisper::Context.new(\"base\")\nmodel = whisper.model\n\nmodel.n_vocab # => 51864\nmodel.n_audio_ctx # => 1500\nmodel.n_audio_state # => 512\nmodel.n_audio_head # => 8\nmodel.n_audio_layer # => 6\nmodel.n_text_ctx # => 448\nmodel.n_text_state # => 512\nmodel.n_text_head # => 8\nmodel.n_text_layer # => 6\nmodel.n_mels # => 80\nmodel.ftype # => 1\nmodel.type # => \"base\"\n\n```\n\n### Logging ###\n\nYou can set log callback:\n\n```ruby\nprefix = \"[MyApp] \"\nlog_callback = ->(level, buffer, user_data) {\n  case level\n  when Whisper::LOG_LEVEL_NONE\n    puts \"#{user_data}none: #{buffer}\"\n  when Whisper::LOG_LEVEL_INFO\n    puts \"#{user_data}info: #{buffer}\"\n  when Whisper::LOG_LEVEL_WARN\n    puts \"#{user_data}warn: #{buffer}\"\n  when Whisper::LOG_LEVEL_ERROR\n    puts \"#{user_data}error: #{buffer}\"\n  when Whisper::LOG_LEVEL_DEBUG\n    puts \"#{user_data}debug: #{buffer}\"\n  when Whisper::LOG_LEVEL_CONT\n    puts \"#{user_data}same to previous: #{buffer}\"\n  end\n}\nWhisper.log_set log_callback, prefix\n```\n\nUsing this feature, you are also able to suppress log:\n\n```ruby\nWhisper.log_set ->(level, buffer, user_data) {\n  # do nothing\n}, nil\nWhisper::Context.new(\"base\")\n```\n\n### Low-level API to transcribe ###\n\nYou can also call `Whisper::Context#full` and `#full_parallel` with a Ruby array as samples. Although `#transcribe` with audio file path is recommended because it extracts PCM samples in C++ and is fast, `#full` and `#full_parallel` give you flexibility.\n\n```ruby\nrequire \"whisper\"\nrequire \"wavefile\"\n\nreader = WaveFile::Reader.new(\"path/to/audio.wav\", WaveFile::Format.new(:mono, :float, 16000))\nsamples = reader.enum_for(:each_buffer).map(&:samples).flatten\n\nwhisper = Whisper::Context.new(\"base\")\nwhisper\n  .full(Whisper::Params.new, samples)\n  .each_segment do |segment|\n    puts segment.text\n  end\n```\n\nThe second argument `samples` may be an array, an object with `length` and `each` method, or a MemoryView.\n\nIf you can prepare audio data as C array and export it as a MemoryView, whispercpp accepts and works with it with zero copy.\n\n```ruby\nrequire \"torchaudio\"\nrequire \"arrow-numo-narray\"\nrequire \"whisper\"\n\nwaveform, sample_rate = TorchAudio.load(\"test/fixtures/jfk.wav\")\n# Convert Torch::Tensor to Arrow::Array via Numo::NArray\nsamples = waveform.squeeze.numo.to_arrow.to_arrow_array\n\nwhisper = Whisper::Context.new(\"base\")\nwhisper\n  # Arrow::Array exports MemoryView\n  .full(Whisper::Params.new, samples)\n```\n\nCustom context params\n---------------------\n\nYou can use customize `Whisper::Context`'s behavior using `Whisper::Context::Params`.\n\n```ruby\ncontext_params = Whisper::Context::Params.new(\n  use_gpu: false,\n  flash_attn: false,\n  # etc\n)\nwhisper = Whisper::Context.new(\"base\", context_params)\n```\n\nUsing VAD separately from ASR\n-----------------------------\n\nVAD feature itself is useful. You can use it separately from ASR:\n\n```ruby\nvad = Whisper::VAD::Context.new(\"silero-v6.2.0\")\nvad\n  .detect(\"path/to/audio.wav\", Whisper::VAD::Params.new)\n  .each.with_index do |segment, index|\n    segment => {start_time: st, end_time: ed} # `Segment` responds to `#deconstruct_keys`\n\n    puts \"[%{nth}: %{st} --> %{ed}]\" % {nth: index + 1, st:, ed:}\n  end\n```\n\nYou may also low level API `Whisper::VAD::Context#segments_from_samples` as such `Whisper::Context#full`:\n\n```ruby\n# Ruby Array\nreader = WaveFile::Reader.new(\"path/to/audio.wav\", WaveFile::Format.new(:mono, :float, 16000))\nsamples = reader.enum_for(:each_buffer).map(&:samples).flatten\n\n# Or, object which exports MemoryView\nwaveform, sample_rate = TorchAudio.load(\"test/fixtures/jfk.wav\")\nsamples = waveform.squeeze.numo.to_arrow.to_arrow_array\n\nsegments = vad.segments_from_samples(Whisper::VAD::Params.new, samples)\n```\n\nDevelopment\n-----------\n\n    % git clone https://github.com/ggml-org/whisper.cpp.git\n    % cd whisper.cpp/bindings/ruby\n    % rake test\n\nFirst call of `rake test` builds an extension and downloads a model for testing. After that, you add tests in `tests` directory and modify `ext/ruby_whisper.cpp`.\n\nIf something seems wrong on build, running `rake clean` solves some cases.\n\n### Need help ###\n\n* Windows support\n* Refinement of C/C++ code, especially memory management\n\nLicense\n-------\n\nThe same to [whisper.cpp][].\n\n[whisper.cpp]: https://github.com/ggml-org/whisper.cpp\n[models]: https://github.com/ggml-org/whisper.cpp/tree/master/models\n"
  },
  {
    "path": "bindings/ruby/Rakefile",
    "content": "require 'rake/clean'\nrequire \"bundler/gem_tasks\"\nrequire \"rake/testtask\"\nrequire_relative \"extsources\"\n\nSOURCES_DIR = \"ext/sources\"\n\nSOURCES = FileList[]\n\nEXTSOURCES.each do |src|\n  basename = src.pathmap(\"%f\")\n  dest = basename == \"LICENSE\" ? basename\n                               : src.pathmap(\"%{\\\\.\\\\./\\\\.\\\\.,#{SOURCES_DIR}}p\")\n                                    .pathmap(\"%{\\\\.\\\\./javascript,#{SOURCES_DIR}/bindings/javascript}p\")\n  dir = dest.pathmap(\"%d\")\n  file src\n  directory dir\n  file dest => [src, dir] do |t|\n    cp t.source, t.name\n  end\n  SOURCES.include dest\nend\n\nCLEAN.include SOURCES\n\nSRC = FileList[\"ext/*.{c,cpp,h}\"]\n\ntask build: SOURCES\n\ndirectory \"pkg\"\nCLOBBER.include \"pkg\"\n\nLIB_NAME = \"whisper\".ext(RbConfig::CONFIG[\"DLEXT\"])\nSO_FILE = File.join(\"ext\", LIB_NAME)\nLIB_FILE = File.join(\"lib\", LIB_NAME)\n\nfile \"ext/Makefile\" => SRC + [\"ext/extconf.rb\"] + SOURCES do |t|\n  chdir \"ext\" do\n    ruby \"extconf.rb\"\n  end\nend\nif File.exist? \"ext/Makefile\"\n  task :make_clean do\n    cd \"ext\" do\n      sh \"make\", \"clean\"\n    end\n  end\n  task clean: :make_clean\n  task :make_distclean do\n    cd \"ext\" do\n      sh \"make\", \"distclean\"\n    end\n  end\n  task clobber: :make_distclean\nend\n\nfile SO_FILE => \"ext/Makefile\" do |t|\n  chdir \"ext\" do\n    sh \"make\"\n  end\nend\nCLEAN.include SO_FILE\n\ndirectory \"lib\"\nfile LIB_FILE => [SO_FILE, \"lib\"] do |t|\n  copy t.source, t.name\nend\nCLEAN.include LIB_FILE\n\nRake::TestTask.new\n\nTEST_FIXTURE_AUDIO = \"test/fixtures/jfk.wav\"\nTEST_FIXTURE_AUDIO_SRC = File.expand_path(File.join(__dir__, \"..\", \"..\", \"samples\", \"jfk.wav\"))\nTEST_FIXTURE_AUDIO_DIR = TEST_FIXTURE_AUDIO.pathmap(\"%d\")\ndirectory TEST_FIXTURE_AUDIO_DIR\nif File.exist? TEST_FIXTURE_AUDIO_SRC\n  file TEST_FIXTURE_AUDIO => [TEST_FIXTURE_AUDIO_SRC, TEST_FIXTURE_AUDIO_DIR] do |t|\n    symlink t.source, t.name\n  end\nelse\n  require \"open-uri\"\n  file TEST_FIXTURE_AUDIO => TEST_FIXTURE_AUDIO_DIR do |t|\n    File.write t.name, URI(\"https://github.com/ggml-org/whisper.cpp/raw/refs/heads/master/samples/jfk.wav\").read\n  end\nend\n\nTEST_MEMORY_VIEW = \"test/jfk_reader/jfk_reader.#{RbConfig::CONFIG['DLEXT']}\"\nfile TEST_MEMORY_VIEW => \"test/jfk_reader/jfk_reader.c\" do |t|\n  chdir \"test/jfk_reader\" do\n    ruby \"extconf.rb\"\n    sh \"make\"\n  end\nend\nCLEAN.include TEST_MEMORY_VIEW\n\ntask test: [LIB_FILE, TEST_MEMORY_VIEW, TEST_FIXTURE_AUDIO]\n"
  },
  {
    "path": "bindings/ruby/ext/.gitignore",
    "content": "Makefile\nwhisper.so\nwhisper.bundle\nwhisper.dll\n*.o\n*.a\nsources/*\n!sources/CMakeGraphVizOptions.cmake\nmkmf.log\n"
  },
  {
    "path": "bindings/ruby/ext/dependencies.rb",
    "content": "require \"tsort\"\n\nclass Dependencies\n  include TSort\n\n  def initialize(cmake, options)\n    @cmake = cmake\n    @options = options\n    @static_lib_shape = nil\n    @nodes = {}\n    @graph = Hash.new {|h, k| h[k] = []}\n\n    generate_dot\n    parse_dot\n  end\n\n  def libs\n    tsort.filter_map {|node|\n      label, shape = @nodes[node]\n      if shape == @static_lib_shape\n        label.gsub(/\\\\n\\([^)]+\\)/, '')\n      else\n        nil\n      end\n    }.reverse.collect {|lib| \"lib#{lib}.a\"}\n  end\n\n  def to_s\n    libs.join(\" \")\n  end\n\n  private\n\n  def dot_path\n    File.join(__dir__, \"build\", \"whisper.cpp.dot\")\n  end\n\n  def generate_dot\n    args = [\"-S\", \"sources\", \"-B\", \"build\", \"--graphviz\", dot_path, \"-D\", \"BUILD_SHARED_LIBS=OFF\"]\n    args << @options.to_s unless @options.to_s.empty?\n    system @cmake, *args, exception: true\n  end\n\n  def parse_dot\n    File.open(dot_path).each_line do |line|\n      case line\n      when /\\[\\s*label\\s*=\\s*\"Static Library\"\\s*,\\s*shape\\s*=\\s*(?<shape>\\w+)\\s*\\]/\n        @static_lib_shape = $~[:shape]\n      when /\\A\\s*\"(?<node>\\w+)\"\\s*\\[\\s*label\\s*=\\s*\"(?<label>\\S+)\"\\s*,\\s*shape\\s*=\\s*(?<shape>\\w+)\\s*\\]\\s*;\\s*\\z/\n        node = $~[:node]\n        label = $~[:label]\n        shape = $~[:shape]\n        @nodes[node] = [label, shape]\n      when /\\A\\s*\"(?<depender>\\w+)\"\\s*->\\s*\"(?<dependee>\\w+)\"/\n        depender = $~[:depender]\n        dependee = $~[:dependee]\n        @graph[depender] << dependee\n      end\n    end\n  end\n\n  def tsort_each_node\n    @nodes.each_key do |node|\n      yield node\n    end\n  end\n\n  def tsort_each_child(node)\n    @graph[node].each do |child|\n      yield child\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/ext/extconf.rb",
    "content": "require \"mkmf\"\nrequire_relative \"options\"\nrequire_relative \"dependencies\"\n\ncmake = find_executable(\"cmake\") || abort\noptions = Options.new(cmake).to_s\nhave_library(\"gomp\") rescue nil\nlibs = Dependencies.new(cmake, options).to_s\n\n$CFLAGS << \" -O3 -march=native\"\n$INCFLAGS << \" -Isources/include -Isources/ggml/include -Isources/examples\"\n$LOCAL_LIBS << \" #{libs}\"\n$cleanfiles << \" build #{libs}\"\n\ncreate_makefile \"whisper\" do |conf|\n  conf << <<~EOF\n    $(TARGET_SO): #{libs}\n    #{libs}: cmake-targets\n    cmake-targets:\n    #{\"\\t\"}#{cmake} -S sources -B build -D BUILD_SHARED_LIBS=OFF -D CMAKE_ARCHIVE_OUTPUT_DIRECTORY=#{__dir__} -D CMAKE_POSITION_INDEPENDENT_CODE=ON #{options}\n    #{\"\\t\"}#{cmake} --build build --config Release --target common whisper\n  EOF\nend\n"
  },
  {
    "path": "bindings/ruby/ext/options.rb",
    "content": "class Options\n  def initialize(cmake=\"cmake\")\n    @cmake = cmake\n    @options = {}\n\n    configure\n  end\n\n  def to_s\n    @options\n      .reject {|name, (type, value)| value.nil?}\n      .collect {|name, (type, value)| \"-D #{name}=#{value == true ? \"ON\" : value == false ? \"OFF\" : value.shellescape}\"}\n      .join(\" \")\n  end\n\n  def cmake_options\n    return @cmake_options if @cmake_options\n\n    output = nil\n    Dir.chdir __dir__ do\n      output = `#{@cmake.shellescape} -S sources -B build -L`\n    end\n    @cmake_options = output.lines.drop_while {|line| line.chomp != \"-- Cache values\"}.drop(1)\n                       .filter_map {|line|\n                         option, value = line.chomp.split(\"=\", 2)\n                         name, type = option.split(\":\", 2)\n                         [\n                           name,\n                           [\n                             type,\n                             type == \"BOOL\" ? value == \"ON\" : value\n                           ]\n                         ]\n                       }.to_h\n  end\n\n  private\n\n  def configure\n    cmake_options.each_pair do |name, (type, default_value)|\n      option = option_name(name)\n      value = type == \"BOOL\" ? enable_config(option) : arg_config(\"--#{option}\")\n      @options[name] = [type, value]\n    end\n\n    configure_accelerate\n    configure_metal\n    configure_coreml\n  end\n\n  # See ggml/src/ggml-cpu/CMakeLists.txt\n  def configure_accelerate\n    if RUBY_PLATFORM.match?(/darwin/) && enabled?(\"GGML_ACCELERATE\")\n      $LDFLAGS << \" -framework Accelerate\"\n    end\n  end\n\n  # See ggml/src/ggml-metal/CMakeLists.txt\n  def configure_metal\n    $LDFLAGS << \" -framework Foundation -framework Metal -framework MetalKit\" if enabled?(\"GGML_METAL\")\n  end\n\n  # See src/CmakeLists.txt\n  def configure_coreml\n    if enabled?(\"WHISPER_COREML\")\n      $LDFLAGS << \" -framework Foundation -framework CoreML\"\n      $defs << \"-DRUBY_WHISPER_USE_COREML\"\n    end\n  end\n\n  def option_name(name)\n    name.downcase.gsub(\"_\", \"-\")\n  end\n\n  def enabled?(option)\n    op = @options[option]\n    raise \"Option not exist: #{option}\" unless op\n    raise \"Option not boolean: #{option}(#{op[0]})\" unless op[0] == \"BOOL\"\n    if op[1].nil?\n      cmake_options[option][1]\n    else\n      op[1]\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper.c",
    "content": "#include \"ruby_whisper.h\"\n\nVALUE mWhisper;\nVALUE mVAD;\nVALUE cContext;\nVALUE cParams;\nVALUE cVADContext;\nVALUE cVADParams;\nVALUE cVADSegments;\nVALUE cVADSegment;\nVALUE eError;\n\nVALUE cSegment;\nVALUE cToken;\nVALUE cModel;\n\nID id_to_s;\nID id_call;\nID id___method__;\nID id_to_enum;\nID id_length;\nID id_next;\nID id_new;\nID id_to_path;\nID id_URI;\nID id_pre_converted_models;\nID id_coreml_compiled_models;\nID id_cache;\nID id_n_processors;\n\nstatic bool is_log_callback_finalized = false;\n\n// High level API\nextern VALUE ruby_whisper_segment_allocate(VALUE klass);\n\nextern VALUE init_ruby_whisper_context(VALUE *mWhisper);\nextern void init_ruby_whisper_context_params(VALUE *cContext);\nextern void init_ruby_whisper_params(VALUE *mWhisper);\nextern void init_ruby_whisper_error(VALUE *mWhisper);\nextern void init_ruby_whisper_segment(VALUE *mWhisper);\nextern void init_ruby_whisper_token(VALUE *mWhisper);\nextern void init_ruby_whisper_model(VALUE *mWhisper);\nextern void init_ruby_whisper_vad_params(VALUE *mVAD);\nextern void init_ruby_whisper_vad_context(VALUE *mVAD);\nextern void init_ruby_whisper_vad_segment(VALUE *mVAD);\nextern void init_ruby_whisper_vad_segments(VALUE *mVAD);\nextern void register_callbacks(ruby_whisper_params *rwp, VALUE *context);\n\n/*\n * call-seq:\n *   lang_max_id -> Integer\n */\nstatic VALUE ruby_whisper_s_lang_max_id(VALUE self) {\n  return INT2NUM(whisper_lang_max_id());\n}\n\n/*\n * call-seq:\n *   lang_id(lang_name) -> Integer\n */\nstatic VALUE ruby_whisper_s_lang_id(VALUE self, VALUE lang) {\n  const char * lang_str = StringValueCStr(lang);\n  const int id = whisper_lang_id(lang_str);\n  if (-1 == id) {\n    rb_raise(rb_eArgError, \"language not found: %s\", lang_str);\n  }\n  return INT2NUM(id);\n}\n\n/*\n * call-seq:\n *   lang_str(lang_id) -> String\n */\nstatic VALUE ruby_whisper_s_lang_str(VALUE self, VALUE id) {\n  const int lang_id = NUM2INT(id);\n  const char * str = whisper_lang_str(lang_id);\n  if (NULL == str) {\n    rb_raise(rb_eIndexError, \"id %d outside of language id\", lang_id);\n  }\n  return rb_str_new2(str);\n}\n\n/*\n * call-seq:\n *   lang_str(lang_id) -> String\n */\nstatic VALUE ruby_whisper_s_lang_str_full(VALUE self, VALUE id) {\n  const int lang_id = NUM2INT(id);\n  const char * str_full = whisper_lang_str_full(lang_id);\n  if (NULL == str_full) {\n    rb_raise(rb_eIndexError, \"id %d outside of language id\", lang_id);\n  }\n  return rb_str_new2(str_full);\n}\n\n/*\n * call-seq:\n *   system_info_str -> String\n */\nstatic VALUE ruby_whisper_s_system_info_str(VALUE self) {\n  return rb_str_new2(whisper_print_system_info());\n}\n\nstatic VALUE ruby_whisper_s_finalize_log_callback(VALUE self, VALUE id) {\n  is_log_callback_finalized = true;\n  return Qnil;\n}\n\nstatic void\nruby_whisper_log_callback(enum ggml_log_level level, const char * buffer, void * user_data) {\n  if (is_log_callback_finalized) {\n    return;\n  }\n  VALUE log_callback = rb_iv_get(mWhisper, \"log_callback\");\n  VALUE udata = rb_iv_get(mWhisper, \"user_data\");\n  rb_funcall(log_callback, id_call, 3, INT2NUM(level), rb_str_new2(buffer), udata);\n}\n\n/*\n * call-seq:\n *   log_set ->(level, buffer, user_data) { ... }, user_data -> nil\n */\nstatic VALUE ruby_whisper_s_log_set(VALUE self, VALUE log_callback, VALUE user_data) {\n  VALUE old_callback = rb_iv_get(self, \"log_callback\");\n  if (!NIL_P(old_callback)) {\n    rb_undefine_finalizer(old_callback);\n  }\n\n  rb_iv_set(self, \"log_callback\", log_callback);\n  rb_iv_set(self, \"user_data\", user_data);\n\n  VALUE finalize_log_callback = rb_funcall(mWhisper, rb_intern(\"method\"), 1, rb_str_new2(\"finalize_log_callback\"));\n  rb_define_finalizer(log_callback, finalize_log_callback);\n\n  whisper_log_set(ruby_whisper_log_callback, NULL);\n\n  return Qnil;\n}\n\nvoid Init_whisper() {\n  id_to_s = rb_intern(\"to_s\");\n  id_call = rb_intern(\"call\");\n  id___method__ = rb_intern(\"__method__\");\n  id_to_enum = rb_intern(\"to_enum\");\n  id_length = rb_intern(\"length\");\n  id_next = rb_intern(\"next\");\n  id_new = rb_intern(\"new\");\n  id_to_path = rb_intern(\"to_path\");\n  id_URI = rb_intern(\"URI\");\n  id_pre_converted_models = rb_intern(\"pre_converted_models\");\n  id_coreml_compiled_models = rb_intern(\"coreml_compiled_models\");\n  id_cache = rb_intern(\"cache\");\n  id_n_processors = rb_intern(\"n_processors\");\n\n  mWhisper = rb_define_module(\"Whisper\");\n  mVAD = rb_define_module_under(mWhisper, \"VAD\");\n\n  rb_define_const(mWhisper, \"VERSION\", rb_str_new2(whisper_version()));\n  rb_define_const(mWhisper, \"LOG_LEVEL_NONE\", INT2NUM(GGML_LOG_LEVEL_NONE));\n  rb_define_const(mWhisper, \"LOG_LEVEL_INFO\", INT2NUM(GGML_LOG_LEVEL_INFO));\n  rb_define_const(mWhisper, \"LOG_LEVEL_WARN\", INT2NUM(GGML_LOG_LEVEL_WARN));\n  rb_define_const(mWhisper, \"LOG_LEVEL_ERROR\", INT2NUM(GGML_LOG_LEVEL_ERROR));\n  rb_define_const(mWhisper, \"LOG_LEVEL_DEBUG\", INT2NUM(GGML_LOG_LEVEL_DEBUG));\n  rb_define_const(mWhisper, \"LOG_LEVEL_CONT\", INT2NUM(GGML_LOG_LEVEL_CONT));\n\n  rb_define_const(mWhisper, \"AHEADS_NONE\", INT2NUM(WHISPER_AHEADS_NONE));\n  rb_define_const(mWhisper, \"AHEADS_N_TOP_MOST\", INT2NUM(WHISPER_AHEADS_N_TOP_MOST));\n  rb_define_const(mWhisper, \"AHEADS_CUSTOM\", INT2NUM(WHISPER_AHEADS_CUSTOM));\n  rb_define_const(mWhisper, \"AHEADS_TINY_EN\", INT2NUM(WHISPER_AHEADS_TINY_EN));\n  rb_define_const(mWhisper, \"AHEADS_TINY\", INT2NUM(WHISPER_AHEADS_TINY));\n  rb_define_const(mWhisper, \"AHEADS_BASE_EN\", INT2NUM(WHISPER_AHEADS_BASE_EN));\n  rb_define_const(mWhisper, \"AHEADS_BASE\", INT2NUM(WHISPER_AHEADS_BASE));\n  rb_define_const(mWhisper, \"AHEADS_SMALL_EN\", INT2NUM(WHISPER_AHEADS_SMALL_EN));\n  rb_define_const(mWhisper, \"AHEADS_SMALL\", INT2NUM(WHISPER_AHEADS_SMALL));\n  rb_define_const(mWhisper, \"AHEADS_MEDIUM_EN\", INT2NUM(WHISPER_AHEADS_MEDIUM_EN));\n  rb_define_const(mWhisper, \"AHEADS_MEDIUM\", INT2NUM(WHISPER_AHEADS_MEDIUM));\n  rb_define_const(mWhisper, \"AHEADS_LARGE_V1\", INT2NUM(WHISPER_AHEADS_LARGE_V1));\n  rb_define_const(mWhisper, \"AHEADS_LARGE_V2\", INT2NUM(WHISPER_AHEADS_LARGE_V2));\n  rb_define_const(mWhisper, \"AHEADS_LARGE_V3\", INT2NUM(WHISPER_AHEADS_LARGE_V3));\n  rb_define_const(mWhisper, \"AHEADS_LARGE_V3_TURBO\", INT2NUM(WHISPER_AHEADS_LARGE_V3_TURBO));\n\n  rb_define_singleton_method(mWhisper, \"lang_max_id\", ruby_whisper_s_lang_max_id, 0);\n  rb_define_singleton_method(mWhisper, \"lang_id\", ruby_whisper_s_lang_id, 1);\n  rb_define_singleton_method(mWhisper, \"lang_str\", ruby_whisper_s_lang_str, 1);\n  rb_define_singleton_method(mWhisper, \"lang_str_full\", ruby_whisper_s_lang_str_full, 1);\n  rb_define_singleton_method(mWhisper, \"system_info_str\", ruby_whisper_s_system_info_str, 0);\n  rb_define_singleton_method(mWhisper, \"log_set\", ruby_whisper_s_log_set, 2);\n  rb_define_private_method(rb_singleton_class(mWhisper), \"finalize_log_callback\", ruby_whisper_s_finalize_log_callback, 1);\n\n  cContext = init_ruby_whisper_context(&mWhisper);\n  init_ruby_whisper_context_params(&cContext);\n  init_ruby_whisper_params(&mWhisper);\n  init_ruby_whisper_error(&mWhisper);\n  init_ruby_whisper_segment(&mWhisper);\n  init_ruby_whisper_token(&mWhisper);\n  init_ruby_whisper_model(&mWhisper);\n  init_ruby_whisper_vad_params(&mVAD);\n  init_ruby_whisper_vad_segment(&mVAD);\n  init_ruby_whisper_vad_segments(&mVAD);\n  init_ruby_whisper_vad_context(&mVAD);\n\n  rb_require(\"whisper/context\");\n  rb_require(\"whisper/segment\");\n  rb_require(\"whisper/model/uri\");\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper.h",
    "content": "#ifndef RUBY_WHISPER_H\n#define RUBY_WHISPER_H\n\n#include <ruby.h>\n#include <ruby/memory_view.h>\n#include \"whisper.h\"\n\ntypedef struct {\n  VALUE *context;\n  VALUE user_data;\n  VALUE callback;\n  VALUE callbacks;\n} ruby_whisper_callback_container;\n\ntypedef struct {\n  struct whisper_context *context;\n} ruby_whisper;\n\ntypedef struct ruby_whisper_context_params {\n  struct whisper_context_params params;\n} ruby_whisper_context_params;\n\ntypedef struct {\n  struct whisper_full_params params;\n  bool diarize;\n  ruby_whisper_callback_container *new_segment_callback_container;\n  ruby_whisper_callback_container *progress_callback_container;\n  ruby_whisper_callback_container *encoder_begin_callback_container;\n  ruby_whisper_callback_container *abort_callback_container;\n  VALUE vad_params;\n} ruby_whisper_params;\n\ntypedef struct {\n  struct whisper_vad_params params;\n} ruby_whisper_vad_params;\n\ntypedef struct {\n  VALUE context;\n  int index;\n} ruby_whisper_segment;\n\ntypedef struct {\n  whisper_token_data *token_data;\n  VALUE text;\n} ruby_whisper_token;\n\ntypedef struct {\n  VALUE context;\n} ruby_whisper_model;\n\ntypedef struct {\n  struct whisper_vad_segments *segments;\n} ruby_whisper_vad_segments;\n\ntypedef struct {\n  VALUE segments;\n  int index;\n} ruby_whisper_vad_segment;\n\ntypedef struct {\n  struct whisper_vad_context *context;\n} ruby_whisper_vad_context;\n\ntypedef struct parsed_samples_t {\n  float *samples;\n  int n_samples;\n  rb_memory_view_t memview;\n  bool memview_exported;\n} parsed_samples_t;\n\n#define GetContext(obj, rw) do { \\\n  TypedData_Get_Struct((obj), ruby_whisper, &ruby_whisper_type, (rw)); \\\n  if ((rw)->context == NULL) { \\\n    rb_raise(rb_eRuntimeError, \"Not initialized\"); \\\n  } \\\n} while (0)\n\n#define GetContextParams(obj, rwcp) do { \\\n  TypedData_Get_Struct((obj), ruby_whisper_context_params, &ruby_whisper_context_params_type, (rwcp)); \\\n} while (0)\n\n#define GetToken(obj, rwt) do { \\\n  TypedData_Get_Struct((obj), ruby_whisper_token, &ruby_whisper_token_type, (rwt)); \\\n  if ((rwt)->token_data == NULL) { \\\n    rb_raise(rb_eRuntimeError, \"Not initialized\"); \\\n  } \\\n} while (0)\n\n#define GetVADContext(obj, rwvc) do { \\\n    TypedData_Get_Struct((obj), ruby_whisper_vad_context, &ruby_whisper_vad_context_type, (rwvc)); \\\n    if ((rwvc)->context == NULL) { \\\n      rb_raise(rb_eRuntimeError, \"Not initialized\"); \\\n    } \\\n} while (0)\n\n#define GetVADParams(obj, rwvp) do { \\\n  TypedData_Get_Struct((obj), ruby_whisper_vad_params, &ruby_whisper_vad_params_type, (rwvp)); \\\n} while (0)\n\n#define GetVADSegments(obj, rwvss) do { \\\n  TypedData_Get_Struct((obj), ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, (rwvss)); \\\n  if ((rwvss)->segments == NULL) { \\\n    rb_raise(rb_eRuntimeError, \"Not initialized\"); \\\n  } \\\n} while (0)\n\n#endif\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_context.c",
    "content": "#include \"ruby_whisper.h\"\n\nextern ID id_to_s;\nextern ID id___method__;\nextern ID id_to_enum;\nextern ID id_length;\nextern ID id_next;\nextern ID id_new;\nextern ID id_to_path;\nextern ID id_URI;\nextern ID id_pre_converted_models;\nextern ID id_coreml_compiled_models;\nextern ID id_cache;\nextern ID id_n_processors;\n\nextern VALUE cContext;\nextern VALUE eError;\nextern VALUE cModel;\n\nextern const rb_data_type_t ruby_whisper_params_type;\nextern const rb_data_type_t ruby_whisper_context_params_type;\nextern VALUE ruby_whisper_transcribe(int argc, VALUE *argv, VALUE self);\nextern VALUE rb_whisper_model_s_new(VALUE context);\nextern VALUE rb_whisper_segment_s_new(VALUE context, int index);\nextern void prepare_transcription(ruby_whisper_params *rwp, VALUE *context);\n\nID transcribe_option_names[1];\n\ntypedef struct fill_samples_args {\n  float *dest;\n  VALUE *src;\n  int n_samples;\n} fill_samples_args;\n\ntypedef struct full_args {\n  VALUE *context;\n  VALUE *params;\n  float *samples;\n  int n_samples;\n} full_args;\n\ntypedef struct full_parallel_args {\n  VALUE *context;\n  VALUE *params;\n  float *samples;\n  int n_samples;\n  int n_processors;\n} full_parallel_args;\n\nstatic void\nruby_whisper_free(ruby_whisper *rw)\n{\n  if (rw->context) {\n    whisper_free(rw->context);\n    rw->context = NULL;\n  }\n}\n\nvoid\nrb_whisper_mark(ruby_whisper *rw)\n{\n  // call rb_gc_mark on any ruby references in rw\n}\n\nvoid\nrb_whisper_free(void *p)\n{\n  ruby_whisper *rw = (ruby_whisper *)p;\n  ruby_whisper_free(rw);\n  free(rw);\n}\n\nstatic size_t\nruby_whisper_memsize(const void *p)\n{\n  const ruby_whisper *rw = (const ruby_whisper *)p;\n  size_t size = sizeof(rw);\n  if (!rw) {\n    return 0;\n  }\n  if (rw->context) {\n    size += sizeof(rw->context);\n  }\n  return size;\n}\n\nconst rb_data_type_t ruby_whisper_type = {\n  \"ruby_whisper\",\n  {0, rb_whisper_free, ruby_whisper_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_allocate(VALUE klass)\n{\n  ruby_whisper *rw;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper, &ruby_whisper_type, rw);\n  rw->context = NULL;\n  return obj;\n}\n\nVALUE\nruby_whisper_normalize_model_path(VALUE model_path)\n{\n  VALUE pre_converted_models = rb_funcall(cModel, id_pre_converted_models, 0);\n  VALUE pre_converted_model = rb_hash_aref(pre_converted_models, model_path);\n  if (!NIL_P(pre_converted_model)) {\n    model_path = pre_converted_model;\n#ifdef RUBY_WHISPER_USE_COREML\n    VALUE coreml_converted_models = rb_funcall(cModel, id_coreml_compiled_models, 0);\n    VALUE coreml_converted_model = rb_hash_aref(coreml_converted_models, pre_converted_model);\n    if (!NIL_P(coreml_converted_model)) {\n      rb_funcall(coreml_converted_model, id_cache, 0);\n    }\n#endif\n  }\n  else if (TYPE(model_path) == T_STRING) {\n    const char * model_path_str = StringValueCStr(model_path);\n    if (strncmp(\"http://\", model_path_str, 7) == 0 || strncmp(\"https://\", model_path_str, 8) == 0) {\n      VALUE uri_class = rb_const_get(cModel, id_URI);\n      model_path = rb_class_new_instance(1, &model_path, uri_class);\n    }\n  }\n  else if (rb_obj_is_kind_of(model_path, rb_path2class(\"URI::HTTP\"))) {\n    VALUE uri_class = rb_const_get(cModel, id_URI);\n    model_path = rb_class_new_instance(1, &model_path, uri_class);\n  }\n  if (rb_respond_to(model_path, id_to_path)) {\n    model_path = rb_funcall(model_path, id_to_path, 0);\n  }\n\n  return model_path;\n}\n\n/*\n * call-seq:\n *   new(\"base.en\") -> Whisper::Context\n *   new(\"path/to/model.bin\") -> Whisper::Context\n *   new(Whisper::Model::URI.new(\"https://example.net/uri/of/model.bin\")) -> Whisper::Context\n */\nstatic VALUE\nruby_whisper_initialize(int argc, VALUE *argv, VALUE self)\n{\n  ruby_whisper *rw;\n  VALUE whisper_model_file_path;\n  VALUE context_params;\n  struct whisper_context_params params;\n\n  // TODO: we can support init from buffer here too maybe another ruby object to expose\n  rb_scan_args(argc, argv, \"11\", &whisper_model_file_path, &context_params);\n  TypedData_Get_Struct(self, ruby_whisper, &ruby_whisper_type, rw);\n\n  whisper_model_file_path = ruby_whisper_normalize_model_path(whisper_model_file_path);\n  if (!rb_respond_to(whisper_model_file_path, id_to_s)) {\n    rb_raise(rb_eRuntimeError, \"Expected file path to model to initialize Whisper::Context\");\n  }\n  if (NIL_P(context_params)) {\n    params = whisper_context_default_params();\n  } else {\n    ruby_whisper_context_params *rwcp;\n    GetContextParams(context_params, rwcp);\n    params = rwcp->params;\n  }\n  rw->context = whisper_init_from_file_with_params(StringValueCStr(whisper_model_file_path), params);\n  if (rw->context == NULL) {\n    rb_raise(rb_eRuntimeError, \"error: failed to initialize whisper context\");\n  }\n  return self;\n}\n\n/*\n * call-seq:\n *   model_n_vocab -> Integer\n */\nVALUE ruby_whisper_model_n_vocab(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_vocab(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_audio_ctx -> Integer\n */\nVALUE ruby_whisper_model_n_audio_ctx(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_audio_ctx(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_audio_state -> Integer\n */\nVALUE ruby_whisper_model_n_audio_state(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_audio_state(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_audio_head -> Integer\n */\nVALUE ruby_whisper_model_n_audio_head(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_audio_head(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_audio_layer -> Integer\n */\nVALUE ruby_whisper_model_n_audio_layer(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_audio_layer(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_text_ctx -> Integer\n */\nVALUE ruby_whisper_model_n_text_ctx(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_text_ctx(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_text_state -> Integer\n */\nVALUE ruby_whisper_model_n_text_state(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_text_state(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_text_head -> Integer\n */\nVALUE ruby_whisper_model_n_text_head(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_text_head(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_text_layer -> Integer\n */\nVALUE ruby_whisper_model_n_text_layer(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_text_layer(rw->context));\n}\n\n/*\n * call-seq:\n *   model_n_mels -> Integer\n */\nVALUE ruby_whisper_model_n_mels(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_n_mels(rw->context));\n}\n\n/*\n * call-seq:\n *   model_ftype -> Integer\n */\nVALUE ruby_whisper_model_ftype(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_model_ftype(rw->context));\n}\n\n/*\n * call-seq:\n *   model_type -> String\n */\nVALUE ruby_whisper_model_type(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return rb_str_new2(whisper_model_type_readable(rw->context));\n}\n\nstatic bool\ncheck_memory_view(rb_memory_view_t *memview)\n{\n  if (memview->format != NULL && strcmp(memview->format, \"f\") != 0) {\n    rb_warn(\"currently only format \\\"f\\\" is supported for MemoryView, but given: %s\", memview->format);\n    return false;\n  }\n  if (memview->format != NULL && memview->ndim != 1) {\n    rb_warn(\"currently only 1 dimensional MemoryView is supported, but given: %zd\", memview->ndim);\n    return false;\n  }\n\n  return true;\n}\n\nstatic VALUE\nfill_samples(VALUE rb_args)\n{\n  fill_samples_args *args = (fill_samples_args *)rb_args;\n\n  if (RB_TYPE_P(*args->src, T_ARRAY)) {\n    for (int i = 0; i < args->n_samples; i++) {\n      args->dest[i] = RFLOAT_VALUE(rb_ary_entry(*args->src, i));\n    }\n  } else {\n    // TODO: use rb_block_call\n    VALUE iter = rb_funcall(*args->src, id_to_enum, 1, rb_str_new2(\"each\"));\n    for (int i = 0; i < args->n_samples; i++) {\n      // TODO: check if iter is exhausted and raise ArgumentError appropriately\n      VALUE sample = rb_funcall(iter, id_next, 0);\n      args->dest[i] = RFLOAT_VALUE(sample);\n    }\n  }\n\n  return Qnil;\n}\n\nstruct parsed_samples_t\nparse_samples(VALUE *samples, VALUE *n_samples)\n{\n  bool memview_available = rb_memory_view_available_p(*samples);\n  struct parsed_samples_t parsed = {0};\n  parsed.memview_exported = false;\n  const bool is_array = RB_TYPE_P(*samples, T_ARRAY);\n\n  if (!NIL_P(*n_samples)) {\n    parsed.n_samples = NUM2INT(*n_samples);\n    if (is_array) {\n      if (RARRAY_LEN(*samples) < parsed.n_samples) {\n        rb_raise(rb_eArgError, \"samples length %ld is less than n_samples %d\", RARRAY_LEN(*samples), parsed.n_samples);\n      }\n    }\n    // Should check when samples.respond_to?(:length)?\n  } else {\n    if (is_array) {\n      if (RARRAY_LEN(*samples) > INT_MAX) {\n        rb_raise(rb_eArgError, \"samples are too long\");\n      }\n      parsed.n_samples = (int)RARRAY_LEN(*samples);\n    } else if (memview_available) {\n      bool memview_got = rb_memory_view_get(*samples, &parsed.memview, RUBY_MEMORY_VIEW_SIMPLE);\n      if (memview_got) {\n        parsed.memview_exported = check_memory_view(&parsed.memview);\n        if (!parsed.memview_exported) {\n          rb_memory_view_release(&parsed.memview);\n          parsed.memview = (rb_memory_view_t){0};\n        }\n      }\n      if (parsed.memview_exported) {\n        ssize_t n_samples_size = parsed.memview.byte_size / parsed.memview.item_size;\n        if (n_samples_size > INT_MAX) {\n          rb_memory_view_release(&parsed.memview);\n          rb_raise(rb_eArgError, \"samples are too long: %zd\", n_samples_size);\n        }\n        parsed.n_samples = (int)n_samples_size;\n      } else {\n        rb_warn(\"unable to get a memory view. falls back to Ruby object\");\n        if (rb_respond_to(*samples, id_length)) {\n          parsed.n_samples = NUM2INT(rb_funcall(*samples, id_length, 0));\n        } else {\n          rb_raise(rb_eArgError, \"samples must respond to :length\");\n        }\n      }\n    } else if (rb_respond_to(*samples, id_length)) {\n      parsed.n_samples = NUM2INT(rb_funcall(*samples, id_length, 0));\n    } else {\n      rb_raise(rb_eArgError, \"samples must respond to :length or be a MemoryView of an array of float when n_samples is not given\");\n    }\n  }\n\n  if (parsed.memview_exported)  {\n    parsed.samples = (float *)parsed.memview.data;\n  } else {\n    parsed.samples = ALLOC_N(float, parsed.n_samples);\n    fill_samples_args args = {\n      parsed.samples,\n      samples,\n      parsed.n_samples,\n    };\n    int state;\n    rb_protect(fill_samples, (VALUE)&args, &state);\n    if (state) {\n      xfree(parsed.samples);\n      rb_jump_tag(state);\n    }\n  }\n\n  return parsed;\n}\n\nVALUE\nrelease_samples(VALUE rb_parsed_args)\n{\n  parsed_samples_t *parsed_args = (parsed_samples_t *)rb_parsed_args;\n\n  if (parsed_args->memview_exported) {\n    rb_memory_view_release(&parsed_args->memview);\n  } else {\n    xfree(parsed_args->samples);\n  }\n  *parsed_args = (parsed_samples_t){0};\n\n  return Qnil;\n}\n\nstatic VALUE\nfull_body(VALUE rb_args)\n{\n  full_args *args = (full_args *)rb_args;\n\n  ruby_whisper *rw;\n  ruby_whisper_params *rwp;\n  GetContext(*args->context, rw);\n  TypedData_Get_Struct(*args->params, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n\n  prepare_transcription(rwp, args->context);\n  int result = whisper_full(rw->context, rwp->params, args->samples, args->n_samples);\n\n  return INT2NUM(result);\n}\n\n/*\n * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n * Not thread safe for same context\n * Uses the specified decoding strategy to obtain the text.\n *\n * call-seq:\n *   full(params, samples, n_samples) -> nil\n *   full(params, samples) -> nil\n *\n * The second argument +samples+ must be an array of samples, respond to :length, or be a MemoryView of an array of float. It must be 32 bit float PCM audio data.\n */\nVALUE ruby_whisper_full(int argc, VALUE *argv, VALUE self)\n{\n  if (argc < 2 || argc > 3) {\n    rb_raise(rb_eArgError, \"wrong number of arguments (given %d, expected 2..3)\", argc);\n  }\n\n  VALUE n_samples = argc == 2 ? Qnil : argv[2];\n\n  struct parsed_samples_t parsed = parse_samples(&argv[1], &n_samples);\n  full_args args = {\n    &self,\n    &argv[0],\n    parsed.samples,\n    parsed.n_samples,\n  };\n  VALUE rb_result = rb_ensure(full_body, (VALUE)&args, release_samples, (VALUE)&parsed);\n  const int result = NUM2INT(rb_result);\n  if (0 == result) {\n    return self;\n  } else {\n    rb_exc_raise(rb_funcall(eError, id_new, 1, result));\n  }\n}\n\nstatic VALUE\nfull_parallel_body(VALUE rb_args)\n{\n  full_parallel_args *args = (full_parallel_args *)rb_args;\n\n  ruby_whisper *rw;\n  ruby_whisper_params *rwp;\n  GetContext(*args->context, rw);\n  TypedData_Get_Struct(*args->params, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n\n  prepare_transcription(rwp, args->context);\n  int result = whisper_full_parallel(rw->context, rwp->params, args->samples, args->n_samples, args->n_processors);\n\n  return INT2NUM(result);\n}\n\n/*\n * Split the input audio in chunks and process each chunk separately using whisper_full_with_state()\n * Result is stored in the default state of the context\n * Not thread safe if executed in parallel on the same context.\n * It seems this approach can offer some speedup in some cases.\n * However, the transcription accuracy can be worse at the beginning and end of each chunk.\n *\n * call-seq:\n *   full_parallel(params, samples) -> nil\n *   full_parallel(params, samples, n_samples) -> nil\n *   full_parallel(params, samples, n_samples, n_processors) -> nil\n *   full_parallel(params, samples, nil, n_processors) -> nil\n */\nstatic VALUE\nruby_whisper_full_parallel(int argc, VALUE *argv,VALUE self)\n{\n  if (argc < 2 || argc > 4) {\n    rb_raise(rb_eArgError, \"wrong number of arguments (given %d, expected 2..4)\", argc);\n  }\n\n  VALUE n_samples = argc == 2 ? Qnil : argv[2];\n  int n_processors;\n  switch (argc) {\n  case 2:\n    n_processors = 1;\n    break;\n  case 3:\n    n_processors = 1;\n    break;\n  case 4:\n    n_processors = NUM2INT(argv[3]);\n    break;\n  }\n  struct parsed_samples_t parsed = parse_samples(&argv[1], &n_samples);\n  const full_parallel_args args = {\n    &self,\n    &argv[0],\n    parsed.samples,\n    parsed.n_samples,\n    n_processors,\n  };\n  const VALUE rb_result = rb_ensure(full_parallel_body, (VALUE)&args, release_samples, (VALUE)&parsed);\n  const int result = NUM2INT(rb_result);\n  if (0 == result) {\n    return self;\n  } else {\n    rb_exc_raise(rb_funcall(eError, id_new, 1, result));\n  }\n}\n\n/*\n * Number of segments.\n *\n * call-seq:\n *   full_n_segments -> Integer\n */\nstatic VALUE\nruby_whisper_full_n_segments(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_full_n_segments(rw->context));\n}\n\n/*\n * Language ID, which can be converted to string by Whisper.lang_str and Whisper.lang_str_full.\n *\n * call-seq:\n *   full_lang_id -> Integer\n */\nstatic VALUE\nruby_whisper_full_lang_id(VALUE self)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  return INT2NUM(whisper_full_lang_id(rw->context));\n}\n\nstatic int ruby_whisper_full_check_segment_index(const ruby_whisper * rw, const VALUE i_segment)\n{\n  const int c_i_segment = NUM2INT(i_segment);\n  if (c_i_segment < 0 || c_i_segment >= whisper_full_n_segments(rw->context)) {\n    rb_raise(rb_eIndexError, \"segment index %d out of range\", c_i_segment);\n  }\n  return c_i_segment;\n}\n\n/*\n * Start time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).\n *\n *   full_get_segment_t0(3) # => 1668 (16680 ms)\n *\n * call-seq:\n *   full_get_segment_t0(segment_index) -> Integer\n */\nstatic VALUE\nruby_whisper_full_get_segment_t0(VALUE self, VALUE i_segment)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);\n  const int64_t t0 = whisper_full_get_segment_t0(rw->context, c_i_segment);\n  return LONG2NUM(t0);\n}\n\n/*\n * End time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).\n *\n *   full_get_segment_t1(3) # => 1668 (16680 ms)\n *\n * call-seq:\n *   full_get_segment_t1(segment_index) -> Integer\n */\nstatic VALUE\nruby_whisper_full_get_segment_t1(VALUE self, VALUE i_segment)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);\n  const int64_t t1 = whisper_full_get_segment_t1(rw->context, c_i_segment);\n  return LONG2NUM(t1);\n}\n\n/*\n * Whether the next segment indexed by +segment_index+ is predicated as a speaker turn.\n *\n *   full_get_segment_speacker_turn_next(3) # => true\n *\n * call-seq:\n *   full_get_segment_speacker_turn_next(segment_index) -> bool\n */\nstatic VALUE\nruby_whisper_full_get_segment_speaker_turn_next(VALUE self, VALUE i_segment)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);\n  const bool speaker_turn_next = whisper_full_get_segment_speaker_turn_next(rw->context, c_i_segment);\n  return speaker_turn_next ? Qtrue : Qfalse;\n}\n\n/*\n * Text of a segment indexed by +segment_index+.\n *\n *   full_get_segment_text(3) # => \"ask not what your country can do for you, ...\"\n *\n * call-seq:\n *   full_get_segment_text(segment_index) -> String\n */\nstatic VALUE\nruby_whisper_full_get_segment_text(VALUE self, VALUE i_segment)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);\n  const char * text = whisper_full_get_segment_text(rw->context, c_i_segment);\n  return rb_str_new2(text);\n}\n\n/*\n * call-seq:\n *   full_get_segment_no_speech_prob(segment_index) -> Float\n */\nstatic VALUE\nruby_whisper_full_get_segment_no_speech_prob(VALUE self, VALUE i_segment)\n{\n  ruby_whisper *rw;\n  GetContext(self, rw);\n  const int c_i_segment = ruby_whisper_full_check_segment_index(rw, i_segment);\n  const float no_speech_prob = whisper_full_get_segment_no_speech_prob(rw->context, c_i_segment);\n  return DBL2NUM(no_speech_prob);\n}\n\n// High level API\n\nstatic VALUE\nruby_whisper_full_get_segment(VALUE self, VALUE i_segment)\n{\n  return rb_whisper_segment_s_new(self, NUM2INT(i_segment));\n}\n\n/*\n * Yields each Whisper::Segment:\n *\n *   whisper.transcribe(\"path/to/audio.wav\", params)\n *   whisper.each_segment do |segment|\n *     puts segment.text\n *   end\n *\n * Returns an Enumerator if no block given:\n *\n *   whisper.transcribe(\"path/to/audio.wav\", params)\n *   enum = whisper.each_segment\n *   enum.to_a # => [#<Whisper::Segment>, ...]\n *\n * call-seq:\n *   each_segment {|segment| ... }\n *   each_segment -> Enumerator\n */\nstatic VALUE\nruby_whisper_each_segment(VALUE self)\n{\n  if (!rb_block_given_p()) {\n    const VALUE method_name = rb_funcall(self, id___method__, 0);\n    return rb_funcall(self, id_to_enum, 1, method_name);\n  }\n\n  ruby_whisper *rw;\n  GetContext(self, rw);\n\n  const int n_segments = whisper_full_n_segments(rw->context);\n  for (int i = 0; i < n_segments; ++i) {\n    rb_yield(rb_whisper_segment_s_new(self, i));\n  }\n\n  return self;\n}\n\n/*\n * call-seq:\n *   model -> Whisper::Model\n */\nstatic VALUE\nruby_whisper_get_model(VALUE self)\n{\n  return rb_whisper_model_s_new(self);\n}\n\nVALUE\ninit_ruby_whisper_context(VALUE *mWhisper)\n{\n  cContext = rb_define_class_under(*mWhisper, \"Context\", rb_cObject);\n\n  transcribe_option_names[0] = id_n_processors;\n\n  rb_define_alloc_func(cContext, ruby_whisper_allocate);\n  rb_define_method(cContext, \"initialize\", ruby_whisper_initialize, -1);\n\n  rb_define_method(cContext, \"transcribe\", ruby_whisper_transcribe, -1);\n  rb_define_method(cContext, \"model_n_vocab\", ruby_whisper_model_n_vocab, 0);\n  rb_define_method(cContext, \"model_n_audio_ctx\", ruby_whisper_model_n_audio_ctx, 0);\n  rb_define_method(cContext, \"model_n_audio_state\", ruby_whisper_model_n_audio_state, 0);\n  rb_define_method(cContext, \"model_n_audio_head\", ruby_whisper_model_n_audio_head, 0);\n  rb_define_method(cContext, \"model_n_audio_layer\", ruby_whisper_model_n_audio_layer, 0);\n  rb_define_method(cContext, \"model_n_text_ctx\", ruby_whisper_model_n_text_ctx, 0);\n  rb_define_method(cContext, \"model_n_text_state\", ruby_whisper_model_n_text_state, 0);\n  rb_define_method(cContext, \"model_n_text_head\", ruby_whisper_model_n_text_head, 0);\n  rb_define_method(cContext, \"model_n_text_layer\", ruby_whisper_model_n_text_layer, 0);\n  rb_define_method(cContext, \"model_n_mels\", ruby_whisper_model_n_mels, 0);\n  rb_define_method(cContext, \"model_ftype\", ruby_whisper_model_ftype, 0);\n  rb_define_method(cContext, \"model_type\", ruby_whisper_model_type, 0);\n  rb_define_method(cContext, \"full_n_segments\", ruby_whisper_full_n_segments, 0);\n  rb_define_method(cContext, \"full_lang_id\", ruby_whisper_full_lang_id, 0);\n  rb_define_method(cContext, \"full_get_segment_t0\", ruby_whisper_full_get_segment_t0, 1);\n  rb_define_method(cContext, \"full_get_segment_t1\", ruby_whisper_full_get_segment_t1, 1);\n  rb_define_method(cContext, \"full_get_segment_speaker_turn_next\", ruby_whisper_full_get_segment_speaker_turn_next, 1);\n  rb_define_method(cContext, \"full_get_segment_text\", ruby_whisper_full_get_segment_text, 1);\n  rb_define_method(cContext, \"full_get_segment_no_speech_prob\", ruby_whisper_full_get_segment_no_speech_prob, 1);\n  rb_define_method(cContext, \"full\", ruby_whisper_full, -1);\n  rb_define_method(cContext, \"full_parallel\", ruby_whisper_full_parallel, -1);\n\n  // High level\n  rb_define_method(cContext, \"full_get_segment\", ruby_whisper_full_get_segment, 1);\n  rb_define_method(cContext, \"each_segment\", ruby_whisper_each_segment, 0);\n\n  rb_define_method(cContext, \"model\", ruby_whisper_get_model, 0);\n\n  return cContext;\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_context_params.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define NUM_PARAMS 6\n\n#define DEF_BOOLEAN_ATTR_METHOD(name) \\\nstatic VALUE \\\nruby_whisper_context_params_get_ ## name(VALUE self) { \\\n  ruby_whisper_context_params *rwcp; \\\n  GetContextParams(self, rwcp); \\\n  return rwcp->params.name ? Qtrue : Qfalse; \\\n} \\\nstatic VALUE \\\nruby_whisper_context_params_set_ ## name(VALUE self, VALUE value) { \\\n  ruby_whisper_context_params *rwcp; \\\n  GetContextParams(self, rwcp); \\\n  rwcp->params.name = RTEST(value); \\\n  return value; \\\n}\n\n#define DEF_INT_ATTR_METHOD(name) \\\nstatic VALUE \\\nruby_whisper_context_params_get_ ## name(VALUE self) { \\\n  ruby_whisper_context_params *rwcp; \\\n  GetContextParams(self, rwcp); \\\n  return INT2NUM(rwcp->params.name); \\\n} \\\nstatic VALUE \\\nruby_whisper_context_params_set_ ## name(VALUE self, VALUE value) { \\\n  ruby_whisper_context_params *rwcp; \\\n  GetContextParams(self, rwcp); \\\n  rwcp->params.name = NUM2INT(value); \\\n  return value; \\\n}\n\n#define DEFINE_PARAM(param_name, nth) \\\n  id_ ## param_name = rb_intern(#param_name); \\\n  param_names[nth] = id_ ## param_name; \\\n  rb_define_method(cContextParams, #param_name, ruby_whisper_context_params_get_ ## param_name, 0); \\\n  rb_define_method(cContextParams, #param_name \"=\", ruby_whisper_context_params_set_ ## param_name, 1);\n\nVALUE cContextParams;\n\nstatic ID param_names[NUM_PARAMS];\nstatic ID id_use_gpu;\nstatic ID id_flash_attn;\nstatic ID id_gpu_device;\nstatic ID id_dtw_token_timestamps;\nstatic ID id_dtw_aheads_preset;\nstatic ID id_dtw_n_top;\n\nstatic size_t\nruby_whisper_context_params_memsize(const void *p)\n{\n  const ruby_whisper_context_params *rwcp = (ruby_whisper_context_params *)p;\n  if (!rwcp) {\n    return 0;\n  }\n  return sizeof(ruby_whisper_context_params);\n}\n\nconst rb_data_type_t ruby_whisper_context_params_type = {\n  \"ruby_whisper_context_params\",\n  {0, RUBY_DEFAULT_FREE, ruby_whisper_context_params_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_context_params_s_allocate(VALUE klass)\n{\n  ruby_whisper_context_params *rwcp;\n  return TypedData_Make_Struct(klass, ruby_whisper_context_params, &ruby_whisper_context_params_type, rwcp);\n}\n\nDEF_BOOLEAN_ATTR_METHOD(use_gpu);\nDEF_BOOLEAN_ATTR_METHOD(flash_attn);\nDEF_INT_ATTR_METHOD(gpu_device);\nDEF_BOOLEAN_ATTR_METHOD(dtw_token_timestamps);\nDEF_INT_ATTR_METHOD(dtw_aheads_preset);\n\nstatic VALUE\nruby_whisper_context_params_get_dtw_n_top(VALUE self) {\n  ruby_whisper_context_params *rwcp;\n  GetContextParams(self, rwcp);\n\n  int dtw_n_top = rwcp->params.dtw_n_top;\n\n  return dtw_n_top == -1 ? Qnil : INT2NUM(dtw_n_top);\n}\n\nstatic VALUE\nruby_whisper_context_params_set_dtw_n_top(VALUE self, VALUE value) {\n  ruby_whisper_context_params *rwcp;\n  GetContextParams(self, rwcp);\n\n  rwcp->params.dtw_n_top = NIL_P(value) ? -1 : NUM2INT(value);\n\n  return value;\n}\n\n#define SET_PARAM_IF_SAME(param_name) \\\n  if (id == id_ ## param_name) { \\\n    ruby_whisper_context_params_set_ ## param_name(self, value); \\\n    continue; \\\n  }\n\nstatic VALUE\nruby_whisper_context_params_initialize(int argc, VALUE *argv, VALUE self)\n{\n  ruby_whisper_context_params *rwcp;\n  TypedData_Get_Struct(self, ruby_whisper_context_params, &ruby_whisper_context_params_type, rwcp);\n  rwcp->params = whisper_context_default_params();\n\n  VALUE kw_hash;\n  rb_scan_args_kw(RB_SCAN_ARGS_KEYWORDS, argc, argv, \":\", &kw_hash);\n  if (NIL_P(kw_hash)) {\n    return Qnil;\n  }\n\n  VALUE values[NUM_PARAMS] = {Qundef};\n  rb_get_kwargs(kw_hash, param_names, 0, NUM_PARAMS, values);\n\n  ID id;\n  VALUE value;\n  for (int i = 0; i < NUM_PARAMS; i++) {\n    id = param_names[i];\n    value = values[i];\n    if (value == Qundef) {\n      continue;\n    }\n    SET_PARAM_IF_SAME(use_gpu)\n    SET_PARAM_IF_SAME(flash_attn)\n    SET_PARAM_IF_SAME(gpu_device)\n    SET_PARAM_IF_SAME(dtw_token_timestamps)\n    SET_PARAM_IF_SAME(dtw_aheads_preset)\n    SET_PARAM_IF_SAME(dtw_n_top)\n  }\n\n  return Qnil;\n}\n\n#undef SET_PARAM_IF_SAME\n\nvoid\ninit_ruby_whisper_context_params(VALUE *cContext)\n{\n  cContextParams = rb_define_class_under(*cContext, \"Params\", rb_cObject);\n\n  rb_define_alloc_func(cContextParams, ruby_whisper_context_params_s_allocate);\n  rb_define_method(cContextParams, \"initialize\", ruby_whisper_context_params_initialize, -1);\n\n  DEFINE_PARAM(use_gpu, 0)\n  DEFINE_PARAM(flash_attn, 1)\n  DEFINE_PARAM(gpu_device, 2)\n  DEFINE_PARAM(dtw_token_timestamps, 3)\n  DEFINE_PARAM(dtw_aheads_preset, 4)\n  DEFINE_PARAM(dtw_n_top, 5)\n}\n\n#undef DEFINE_PARAM\n#undef DEF_INT_ATTR_METHOD\n#undef DEF_BOOLEAN_ATTR_METHOD\n#undef NUM_PARAMS\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_error.c",
    "content": "#include <ruby.h>\n\nextern VALUE eError;\n\nVALUE ruby_whisper_error_initialize(VALUE self, VALUE code)\n{\n  const int c_code = NUM2INT(code);\n  const char *raw_message;\n  switch (c_code) {\n  case -2:\n    raw_message = \"failed to compute log mel spectrogram\";\n    break;\n  case -3:\n    raw_message = \"failed to auto-detect language\";\n    break;\n  case -4:\n    raw_message = \"too many decoders requested\";\n    break;\n  case -5:\n    raw_message = \"audio_ctx is larger than the maximum allowed\";\n    break;\n  case -6:\n    raw_message = \"failed to encode\";\n    break;\n  case -7:\n    raw_message = \"whisper_kv_cache_init() failed for self-attention cache\";\n    break;\n  case -8:\n    raw_message = \"failed to decode\";\n    break;\n  case -9:\n    raw_message = \"failed to decode\";\n    break;\n  default:\n    raw_message = \"unknown error\";\n    break;\n  }\n  const VALUE message = rb_str_new2(raw_message);\n  rb_call_super(1, &message);\n  rb_iv_set(self, \"@code\", code);\n\n  return self;\n}\n\nvoid\ninit_ruby_whisper_error(VALUE *mWhisper)\n{\n  eError = rb_define_class_under(*mWhisper, \"Error\", rb_eStandardError);\n\n  rb_define_attr(eError, \"code\", true, false);\n  rb_define_method(eError, \"initialize\", ruby_whisper_error_initialize, 1);\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_model.c",
    "content": "#include \"ruby_whisper.h\"\n\nextern const rb_data_type_t ruby_whisper_type;\n\nextern VALUE cModel;\n\nstatic void rb_whisper_model_mark(void *p) {\n  ruby_whisper_model *rwm = (ruby_whisper_model *)p;\n  if (rwm->context) {\n    rb_gc_mark(rwm->context);\n  }\n}\n\nstatic size_t\nruby_whisper_model_memsize(const void *p)\n{\n  const ruby_whisper_model *rwm = (const ruby_whisper_model *)p;\n  size_t size = sizeof(rwm);\n  if (!rwm) {\n    return 0;\n  }\n  return size;\n}\n\nstatic const rb_data_type_t rb_whisper_model_type = {\n  \"ruby_whisper_model\",\n  {rb_whisper_model_mark, RUBY_DEFAULT_FREE, ruby_whisper_model_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE ruby_whisper_model_allocate(VALUE klass) {\n  ruby_whisper_model *rwm;\n  return TypedData_Make_Struct(klass, ruby_whisper_model, &rb_whisper_model_type, rwm);\n}\n\nVALUE rb_whisper_model_s_new(VALUE context) {\n  ruby_whisper_model *rwm;\n  const VALUE model = ruby_whisper_model_allocate(cModel);\n  TypedData_Get_Struct(model, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  rwm->context = context;\n  return model;\n};\n\n/*\n * call-seq:\n *   n_vocab -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_vocab(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_vocab(rw->context));\n}\n\n/*\n * call-seq:\n *   n_audio_ctx -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_audio_ctx(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_audio_ctx(rw->context));\n}\n\n/*\n * call-seq:\n *   n_audio_state -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_audio_state(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_audio_state(rw->context));\n}\n\n/*\n * call-seq:\n *   n_audio_head -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_audio_head(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_audio_head(rw->context));\n}\n\n/*\n * call-seq:\n *   n_audio_layer -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_audio_layer(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_audio_layer(rw->context));\n}\n\n/*\n * call-seq:\n *   n_text_ctx -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_text_ctx(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_text_ctx(rw->context));\n}\n\n/*\n * call-seq:\n *   n_text_state -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_text_state(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_text_state(rw->context));\n}\n\n/*\n * call-seq:\n *   n_text_head -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_text_head(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_text_head(rw->context));\n}\n\n/*\n * call-seq:\n *   n_text_layer -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_text_layer(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_text_layer(rw->context));\n}\n\n/*\n * call-seq:\n *   n_mels -> Integer\n */\nstatic VALUE\nruby_whisper_model_n_mels(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_n_mels(rw->context));\n}\n\n/*\n * call-seq:\n *   ftype -> Integer\n */\nstatic VALUE\nruby_whisper_model_ftype(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return INT2NUM(whisper_model_ftype(rw->context));\n}\n\n/*\n * call-seq:\n *   type -> String\n */\nstatic VALUE\nruby_whisper_model_type(VALUE self)\n{\n  ruby_whisper_model *rwm;\n  TypedData_Get_Struct(self, ruby_whisper_model, &rb_whisper_model_type, rwm);\n  ruby_whisper *rw;\n  GetContext(rwm->context, rw);\n  return rb_str_new2(whisper_model_type_readable(rw->context));\n}\n\nvoid\ninit_ruby_whisper_model(VALUE *mWhisper)\n{\n  cModel = rb_define_class_under(*mWhisper, \"Model\", rb_cObject);\n\n  rb_define_alloc_func(cModel, ruby_whisper_model_allocate);\n  rb_define_method(cModel, \"n_vocab\", ruby_whisper_model_n_vocab, 0);\n  rb_define_method(cModel, \"n_audio_ctx\", ruby_whisper_model_n_audio_ctx, 0);\n  rb_define_method(cModel, \"n_audio_state\", ruby_whisper_model_n_audio_state, 0);\n  rb_define_method(cModel, \"n_audio_head\", ruby_whisper_model_n_audio_head, 0);\n  rb_define_method(cModel, \"n_audio_layer\", ruby_whisper_model_n_audio_layer, 0);\n  rb_define_method(cModel, \"n_text_ctx\", ruby_whisper_model_n_text_ctx, 0);\n  rb_define_method(cModel, \"n_text_state\", ruby_whisper_model_n_text_state, 0);\n  rb_define_method(cModel, \"n_text_head\", ruby_whisper_model_n_text_head, 0);\n  rb_define_method(cModel, \"n_text_layer\", ruby_whisper_model_n_text_layer, 0);\n  rb_define_method(cModel, \"n_mels\", ruby_whisper_model_n_mels, 0);\n  rb_define_method(cModel, \"ftype\", ruby_whisper_model_ftype, 0);\n  rb_define_method(cModel, \"type\", ruby_whisper_model_type, 0);\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_params.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define BOOL_PARAMS_SETTER(self, prop, value) \\\n  ruby_whisper_params *rwp; \\\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp); \\\n  if (value == Qfalse || value == Qnil) { \\\n    rwp->params.prop = false; \\\n  } else { \\\n    rwp->params.prop = true; \\\n  } \\\n  return value; \\\n\n#define BOOL_PARAMS_GETTER(self,  prop) \\\n  ruby_whisper_params *rwp; \\\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp); \\\n  if (rwp->params.prop) { \\\n    return Qtrue; \\\n  } else { \\\n    return Qfalse; \\\n  }\n\n#define DEFINE_PARAM(param_name, nth) \\\n  id_ ## param_name = rb_intern(#param_name); \\\n  param_names[nth] = id_ ## param_name; \\\n  rb_define_method(cParams, #param_name, ruby_whisper_params_get_ ## param_name, 0); \\\n  rb_define_method(cParams, #param_name \"=\", ruby_whisper_params_set_ ## param_name, 1);\n\n#define RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT 37\n\nextern VALUE cParams;\nextern VALUE cVADParams;\n\nextern ID id_call;\n\nextern VALUE ruby_whisper_normalize_model_path(VALUE model_path);\nextern VALUE rb_whisper_segment_s_new(VALUE context, int index);\nextern const rb_data_type_t ruby_whisper_vad_params_type;\n\nstatic ID param_names[RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT];\nstatic ID id_language;\nstatic ID id_translate;\nstatic ID id_no_context;\nstatic ID id_single_segment;\nstatic ID id_print_special;\nstatic ID id_print_progress;\nstatic ID id_print_realtime;\nstatic ID id_print_timestamps;\nstatic ID id_carry_initial_prompt;\nstatic ID id_suppress_blank;\nstatic ID id_suppress_nst;\nstatic ID id_token_timestamps;\nstatic ID id_max_len;\nstatic ID id_split_on_word;\nstatic ID id_initial_prompt;\nstatic ID id_diarize;\nstatic ID id_offset;\nstatic ID id_duration;\nstatic ID id_max_text_tokens;\nstatic ID id_temperature;\nstatic ID id_max_initial_ts;\nstatic ID id_length_penalty;\nstatic ID id_temperature_inc;\nstatic ID id_entropy_thold;\nstatic ID id_logprob_thold;\nstatic ID id_no_speech_thold;\nstatic ID id_new_segment_callback;\nstatic ID id_new_segment_callback_user_data;\nstatic ID id_progress_callback;\nstatic ID id_progress_callback_user_data;\nstatic ID id_encoder_begin_callback;\nstatic ID id_encoder_begin_callback_user_data;\nstatic ID id_abort_callback;\nstatic ID id_abort_callback_user_data;\nstatic ID id_vad;\nstatic ID id_vad_model_path;\nstatic ID id_vad_params;\n\nstatic void\nrb_whisper_callbcack_container_mark(ruby_whisper_callback_container *rwc)\n{\n  if (rwc == NULL) return;\n\n  rb_gc_mark(rwc->user_data);\n  rb_gc_mark(rwc->callback);\n  rb_gc_mark(rwc->callbacks);\n}\n\nstatic ruby_whisper_callback_container*\nrb_whisper_callback_container_allocate() {\n  ruby_whisper_callback_container *container;\n  container = ALLOC(ruby_whisper_callback_container);\n  container->context = NULL;\n  container->user_data = Qnil;\n  container->callback = Qnil;\n  container->callbacks = rb_ary_new();\n  return container;\n}\n\nstatic void new_segment_callback(struct whisper_context *ctx, struct whisper_state *state, int n_new, void *user_data) {\n  const ruby_whisper_callback_container *container = (ruby_whisper_callback_container *)user_data;\n\n  // Currently, doesn't support state because\n  // those require to resolve GC-related problems.\n  if (!NIL_P(container->callback)) {\n    rb_funcall(container->callback, id_call, 4, *container->context, Qnil, INT2NUM(n_new), container->user_data);\n  }\n  const long callbacks_len = RARRAY_LEN(container->callbacks);\n  if (0 == callbacks_len) {\n    return;\n  }\n  const int n_segments = whisper_full_n_segments_from_state(state);\n  for (int i = n_new; i > 0; i--) {\n    int i_segment = n_segments - i;\n    VALUE segment = rb_whisper_segment_s_new(*container->context, i_segment);\n    for (int j = 0; j < callbacks_len; j++) {\n      VALUE cb = rb_ary_entry(container->callbacks, j);\n      rb_funcall(cb, id_call, 1, segment);\n    }\n  }\n}\n\nstatic void progress_callback(struct whisper_context *ctx, struct whisper_state *state, int progress_cur, void *user_data) {\n  const ruby_whisper_callback_container *container = (ruby_whisper_callback_container *)user_data;\n  const VALUE progress = INT2NUM(progress_cur);\n  // Currently, doesn't support state because\n  // those require to resolve GC-related problems.\n  if (!NIL_P(container->callback)) {\n    rb_funcall(container->callback, id_call, 4, *container->context, Qnil, progress, container->user_data);\n  }\n  const long callbacks_len = RARRAY_LEN(container->callbacks);\n  if (0 == callbacks_len) {\n    return;\n  }\n  for (int j = 0; j < callbacks_len; j++) {\n    VALUE cb = rb_ary_entry(container->callbacks, j);\n    rb_funcall(cb, id_call, 1, progress);\n  }\n}\n\nstatic bool encoder_begin_callback(struct whisper_context *ctx, struct whisper_state *state, void *user_data) {\n  const ruby_whisper_callback_container *container = (ruby_whisper_callback_container *)user_data;\n  bool is_aborted = false;\n  VALUE result;\n\n  // Currently, doesn't support state because\n  // those require to resolve GC-related problems.\n  if (!NIL_P(container->callback)) {\n    result = rb_funcall(container->callback, id_call, 3, *container->context, Qnil, container->user_data);\n    if (result == Qfalse) {\n      is_aborted = true;\n    }\n  }\n  const long callbacks_len = RARRAY_LEN(container->callbacks);\n  if (0 == callbacks_len) {\n    return !is_aborted;\n  }\n  for (int j = 0; j < callbacks_len; j++) {\n    VALUE cb = rb_ary_entry(container->callbacks, j);\n    result = rb_funcall(cb, id_call, 0);\n    if (result == Qfalse) {\n      is_aborted = true;\n    }\n  }\n  return !is_aborted;\n}\n\nstatic bool abort_callback(void * user_data) {\n  const ruby_whisper_callback_container *container = (ruby_whisper_callback_container *)user_data;\n  if (!NIL_P(container->callback)) {\n    VALUE result = rb_funcall(container->callback, id_call, 1, container->user_data);\n    if (!NIL_P(result) && Qfalse != result) {\n      return true;\n    }\n  }\n  const long callbacks_len = RARRAY_LEN(container->callbacks);\n  if (0 == callbacks_len) {\n    return false;\n  }\n  for (int j = 0; j < callbacks_len; j++) {\n    VALUE cb = rb_ary_entry(container->callbacks, j);\n    VALUE result = rb_funcall(cb, id_call, 1, container->user_data);\n    if (!NIL_P(result) && Qfalse != result) {\n      return true;\n    }\n  }\n  return false;\n}\n\nstatic void register_callbacks(ruby_whisper_params * rwp, VALUE * context) {\n  if (!NIL_P(rwp->new_segment_callback_container->callback) || 0 != RARRAY_LEN(rwp->new_segment_callback_container->callbacks)) {\n    rwp->new_segment_callback_container->context = context;\n    rwp->params.new_segment_callback = new_segment_callback;\n    rwp->params.new_segment_callback_user_data = rwp->new_segment_callback_container;\n  }\n\n  if (!NIL_P(rwp->progress_callback_container->callback) || 0 != RARRAY_LEN(rwp->progress_callback_container->callbacks)) {\n    rwp->progress_callback_container->context = context;\n    rwp->params.progress_callback = progress_callback;\n    rwp->params.progress_callback_user_data = rwp->progress_callback_container;\n  }\n\n  if (!NIL_P(rwp->encoder_begin_callback_container->callback) || 0 != RARRAY_LEN(rwp->encoder_begin_callback_container->callbacks)) {\n    rwp->encoder_begin_callback_container->context = context;\n    rwp->params.encoder_begin_callback = encoder_begin_callback;\n    rwp->params.encoder_begin_callback_user_data = rwp->encoder_begin_callback_container;\n  }\n\n  if (!NIL_P(rwp->abort_callback_container->callback) || 0 != RARRAY_LEN(rwp->abort_callback_container->callbacks)) {\n    rwp->abort_callback_container->context = context;\n    rwp->params.abort_callback = abort_callback;\n    rwp->params.abort_callback_user_data = rwp->abort_callback_container;\n  }\n}\n\nstatic void set_vad_params(ruby_whisper_params *rwp)\n{\n  ruby_whisper_vad_params * rwvp;\n  TypedData_Get_Struct(rwp->vad_params, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwp->params.vad_params = rwvp->params;\n}\n\nvoid\nprepare_transcription(ruby_whisper_params *rwp, VALUE *context)\n{\n  register_callbacks(rwp, context);\n  set_vad_params(rwp);\n}\n\nvoid\nrb_whisper_params_mark(void *p)\n{\n  ruby_whisper_params *rwp = (ruby_whisper_params *)p;\n  rb_whisper_callbcack_container_mark(rwp->new_segment_callback_container);\n  rb_whisper_callbcack_container_mark(rwp->progress_callback_container);\n  rb_whisper_callbcack_container_mark(rwp->encoder_begin_callback_container);\n  rb_whisper_callbcack_container_mark(rwp->abort_callback_container);\n  rb_gc_mark(rwp->vad_params);\n}\n\nvoid\nruby_whisper_params_free(ruby_whisper_params *rwp)\n{\n}\n\nvoid\nrb_whisper_params_free(void *p)\n{\n  ruby_whisper_params *rwp = (ruby_whisper_params *)p;\n  // How to free user_data and callback only when not referred to by others?\n  ruby_whisper_params_free(rwp);\n  free(rwp);\n}\n\nstatic size_t\nruby_whisper_params_memsize(const void *p)\n{\n  const ruby_whisper_params *rwp = (const ruby_whisper_params *)p;\n\n  return sizeof(ruby_whisper_params) + sizeof(rwp->params) + sizeof(rwp->vad_params);\n}\n\nconst rb_data_type_t ruby_whisper_params_type = {\n  \"ruby_whisper_params\",\n  {\n    rb_whisper_params_mark,\n    rb_whisper_params_free,\n    ruby_whisper_params_memsize,\n  },\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_params_allocate(VALUE klass)\n{\n  ruby_whisper_params *rwp;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n  rwp->diarize = false;\n  rwp->vad_params = TypedData_Wrap_Struct(cVADParams, &ruby_whisper_vad_params_type, (void *)&rwp->params.vad_params);\n  rwp->new_segment_callback_container = rb_whisper_callback_container_allocate();\n  rwp->progress_callback_container = rb_whisper_callback_container_allocate();\n  rwp->encoder_begin_callback_container = rb_whisper_callback_container_allocate();\n  rwp->abort_callback_container = rb_whisper_callback_container_allocate();\n  return obj;\n}\n\n/*\n * params.language = \"auto\" | \"en\", etc...\n *\n * call-seq:\n *   language = lang_name -> lang_name\n */\nstatic VALUE\nruby_whisper_params_set_language(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  if (value == Qfalse || value == Qnil) {\n    rwp->params.language = \"auto\";\n  } else {\n    rwp->params.language = StringValueCStr(value);\n  }\n  return value;\n}\n/*\n * call-seq:\n *   language -> String\n */\nstatic VALUE\nruby_whisper_params_get_language(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  if (rwp->params.language) {\n    return rb_str_new2(rwp->params.language);\n  } else {\n    return rb_str_new2(\"auto\");\n  }\n}\n/*\n * call-seq:\n *   translate = do_translate -> do_translate\n */\nstatic VALUE\nruby_whisper_params_set_translate(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, translate, value)\n}\n/*\n * call-seq:\n *   translate -> bool\n */\nstatic VALUE\nruby_whisper_params_get_translate(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, translate)\n}\n/*\n * call-seq:\n *   no_context = dont_use_context -> dont_use_context\n */\nstatic VALUE\nruby_whisper_params_set_no_context(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, no_context, value)\n}\n/*\n * If true, does not use past transcription (if any) as initial prompt for the decoder.\n *\n * call-seq:\n *   no_context -> bool\n */\nstatic VALUE\nruby_whisper_params_get_no_context(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, no_context)\n}\n/*\n * call-seq:\n *   single_segment = force_single -> force_single\n */\nstatic VALUE\nruby_whisper_params_set_single_segment(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, single_segment, value)\n}\n/*\n * If true, forces single segment output (useful for streaming).\n *\n * call-seq:\n *   single_segment -> bool\n */\nstatic VALUE\nruby_whisper_params_get_single_segment(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, single_segment)\n}\n/*\n * call-seq:\n *   print_special = force_print -> force_print\n */\nstatic VALUE\nruby_whisper_params_set_print_special(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, print_special, value)\n}\n/*\n * If true, prints special tokens (e.g. <SOT>, <EOT>, <BEG>, etc.).\n *\n * call-seq:\n *   print_special -> bool\n */\nstatic VALUE\nruby_whisper_params_get_print_special(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, print_special)\n}\n/*\n * call-seq:\n *   print_progress = force_print -> force_print\n */\nstatic VALUE\nruby_whisper_params_set_print_progress(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, print_progress, value)\n}\n/*\n * If true, prints progress information.\n *\n * call-seq:\n *   print_progress -> bool\n */\nstatic VALUE\nruby_whisper_params_get_print_progress(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, print_progress)\n}\n/*\n * call-seq:\n *   print_realtime = force_print -> force_print\n */\nstatic VALUE\nruby_whisper_params_set_print_realtime(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, print_realtime, value)\n}\n/*\n * If true, prints results from within whisper.cpp. (avoid it, use callback instead)\n *\n * call-seq:\n *   print_realtime -> bool\n */\nstatic VALUE\nruby_whisper_params_get_print_realtime(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, print_realtime)\n}\n/*\n * call-seq:\n *   print_timestamps = force_print -> force_print\n */\nstatic VALUE\nruby_whisper_params_set_print_timestamps(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, print_timestamps, value)\n}\n/*\n * If true, prints timestamps for each text segment when printing realtime.\n *\n * call-seq:\n *   print_timestamps -> bool\n */\nstatic VALUE\nruby_whisper_params_get_print_timestamps(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, print_timestamps)\n}\n\n/*\n *  call-seq:\n *    carry_initial_prompt -> true or false\n */\nstatic VALUE\nruby_whisper_params_get_carry_initial_prompt(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, carry_initial_prompt)\n}\n\n/*\n *  call-seq:\n *    carry_initial_prompt = bool -> bool\n */\nstatic VALUE\nruby_whisper_params_set_carry_initial_prompt(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, carry_initial_prompt, value)\n}\n/*\n * call-seq:\n *   suppress_blank = force_suppress -> force_suppress\n */\nstatic VALUE\nruby_whisper_params_set_suppress_blank(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, suppress_blank, value)\n}\n/*\n * If true, suppresses blank outputs.\n *\n * call-seq:\n *   suppress_blank -> bool\n */\nstatic VALUE\nruby_whisper_params_get_suppress_blank(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, suppress_blank)\n}\n/*\n * call-seq:\n *   suppress_nst = force_suppress -> force_suppress\n */\nstatic VALUE\nruby_whisper_params_set_suppress_nst(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, suppress_nst, value)\n}\n/*\n * If true, suppresses non-speech-tokens.\n *\n * call-seq:\n *   suppress_nst -> bool\n */\nstatic VALUE\nruby_whisper_params_get_suppress_nst(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, suppress_nst)\n}\n/*\n * If true, enables token-level timestamps.\n *\n * call-seq:\n *   token_timestamps -> bool\n */\nstatic VALUE\nruby_whisper_params_get_token_timestamps(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, token_timestamps)\n}\n/*\n * call-seq:\n *   token_timestamps = force_timestamps -> force_timestamps\n */\nstatic VALUE\nruby_whisper_params_set_token_timestamps(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, token_timestamps, value)\n}\n\n/*\n * max segment length in characters.\n *\n * call-seq:\n *   max_len -> Integer\n */\nstatic VALUE\nruby_whisper_params_get_max_len(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return INT2NUM(rwp->params.max_len);\n}\n/*\n * call-seq:\n *   max_len = length -> length\n */\nstatic VALUE\nruby_whisper_params_set_max_len(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.max_len = NUM2INT(value);\n  return value;\n}\n\n/*\n * If true, split on word rather than on token (when used with max_len).\n *\n * call-seq:\n *   translate -> bool\n */\nstatic VALUE\nruby_whisper_params_get_split_on_word(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, split_on_word)\n}\n/*\n * call-seq:\n *   split_on_word = force_split -> force_split\n */\nstatic VALUE\nruby_whisper_params_set_split_on_word(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, split_on_word, value)\n}\n/*\n * Tokens to provide to the whisper decoder as initial prompt\n * these are prepended to any existing text context from a previous call\n * use whisper_tokenize() to convert text to tokens.\n * Maximum of whisper_n_text_ctx()/2 tokens are used (typically 224).\n *\n * call-seq:\n *   initial_prompt -> String\n */\nstatic VALUE\nruby_whisper_params_get_initial_prompt(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->params.initial_prompt == NULL ? Qnil : rb_str_new2(rwp->params.initial_prompt);\n}\n/*\n * call-seq:\n *   initial_prompt = prompt -> prompt\n */\nstatic VALUE\nruby_whisper_params_set_initial_prompt(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.initial_prompt = StringValueCStr(value);\n  return value;\n}\n/*\n * If true, enables diarization.\n *\n * call-seq:\n *   diarize -> bool\n */\nstatic VALUE\nruby_whisper_params_get_diarize(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  if (rwp->diarize) {\n    return Qtrue;\n  } else {\n    return Qfalse;\n  }\n}\n/*\n * call-seq:\n *   diarize = force_diarize -> force_diarize\n */\nstatic VALUE\nruby_whisper_params_set_diarize(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  if (value == Qfalse || value == Qnil) {\n    rwp->diarize = false;\n  } else {\n    rwp->diarize = true;\n  } \\\n  return value;\n}\n\n/*\n * Start offset in ms.\n *\n * call-seq:\n *   offset -> Integer\n */\nstatic VALUE\nruby_whisper_params_get_offset(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return INT2NUM(rwp->params.offset_ms);\n}\n/*\n * call-seq:\n *   offset = offset_ms -> offset_ms\n */\nstatic VALUE\nruby_whisper_params_set_offset(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.offset_ms = NUM2INT(value);\n  return value;\n}\n/*\n * Audio duration to process in ms.\n *\n * call-seq:\n *   duration -> Integer\n */\nstatic VALUE\nruby_whisper_params_get_duration(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return INT2NUM(rwp->params.duration_ms);\n}\n/*\n * call-seq:\n *   duration = duration_ms -> duration_ms\n */\nstatic VALUE\nruby_whisper_params_set_duration(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.duration_ms = NUM2INT(value);\n  return value;\n}\n\n/*\n * Max tokens to use from past text as prompt for the decoder.\n *\n * call-seq:\n *   max_text_tokens -> Integer\n */\nstatic VALUE\nruby_whisper_params_get_max_text_tokens(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return INT2NUM(rwp->params.n_max_text_ctx);\n}\n/*\n * call-seq:\n *   max_text_tokens = n_tokens -> n_tokens\n */\nstatic VALUE\nruby_whisper_params_set_max_text_tokens(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.n_max_text_ctx = NUM2INT(value);\n  return value;\n}\n/*\n * call-seq:\n *   temperature -> Float\n */\nstatic VALUE\nruby_whisper_params_get_temperature(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.temperature);\n}\n/*\n * call-seq:\n *   temperature = temp -> temp\n */\nstatic VALUE\nruby_whisper_params_set_temperature(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.temperature = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * See https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L97\n *\n * call-seq:\n *   max_initial_ts -> Flaot\n */\nstatic VALUE\nruby_whisper_params_get_max_initial_ts(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.max_initial_ts);\n}\n/*\n * call-seq:\n *   max_initial_ts = timestamp -> timestamp\n */\nstatic VALUE\nruby_whisper_params_set_max_initial_ts(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.max_initial_ts = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * call-seq:\n *   length_penalty -> Float\n */\nstatic VALUE\nruby_whisper_params_get_length_penalty(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.length_penalty);\n}\n/*\n * call-seq:\n *   length_penalty = penalty -> penalty\n */\nstatic VALUE\nruby_whisper_params_set_length_penalty(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.length_penalty = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * call-seq:\n *   temperature_inc -> Float\n */\nstatic VALUE\nruby_whisper_params_get_temperature_inc(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.temperature_inc);\n}\n/*\n * call-seq:\n *   temperature_inc = inc -> inc\n */\nstatic VALUE\nruby_whisper_params_set_temperature_inc(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.temperature_inc = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * Similar to OpenAI's \"compression_ratio_threshold\"\n *\n * call-seq:\n *   entropy_thold -> Float\n */\nstatic VALUE\nruby_whisper_params_get_entropy_thold(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.entropy_thold);\n}\n/*\n * call-seq:\n *   entropy_thold = threshold -> threshold\n */\nstatic VALUE\nruby_whisper_params_set_entropy_thold(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.entropy_thold = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * call-seq:\n *   logprob_thold -> Float\n */\nstatic VALUE\nruby_whisper_params_get_logprob_thold(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.logprob_thold);\n}\n/*\n * call-seq:\n *   logprob_thold = threshold -> threshold\n */\nstatic VALUE\nruby_whisper_params_set_logprob_thold(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.logprob_thold = RFLOAT_VALUE(value);\n  return value;\n}\n/*\n * call-seq:\n *   no_speech_thold -> Float\n */\nstatic VALUE\nruby_whisper_params_get_no_speech_thold(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return DBL2NUM(rwp->params.no_speech_thold);\n}\n/*\n * call-seq:\n *   no_speech_thold = threshold -> threshold\n */\nstatic VALUE\nruby_whisper_params_set_no_speech_thold(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->params.no_speech_thold = RFLOAT_VALUE(value);\n  return value;\n}\nstatic VALUE\nruby_whisper_params_get_new_segment_callback(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->new_segment_callback_container->callback;\n}\n/*\n * Sets new segment callback, called for every newly generated text segment.\n *\n *   params.new_segment_callback = ->(context, _, n_new, user_data) {\n *     # ...\n *   }\n *\n * call-seq:\n *   new_segment_callback = callback -> callback\n */\nstatic VALUE\nruby_whisper_params_set_new_segment_callback(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->new_segment_callback_container->callback = value;\n  return value;\n}\nstatic VALUE\nruby_whisper_params_get_new_segment_callback_user_data(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->new_segment_callback_container->user_data;\n}\n/*\n * Sets user data passed to the last argument of new segment callback.\n *\n * call-seq:\n *   new_segment_callback_user_data = user_data -> use_data\n */\nstatic VALUE\nruby_whisper_params_set_new_segment_callback_user_data(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->new_segment_callback_container->user_data = value;\n  return value;\n}\nstatic VALUE\nruby_whisper_params_get_progress_callback(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->progress_callback_container->callback;\n}\n/*\n * Sets progress callback, called on each progress update.\n *\n *   params.new_segment_callback = ->(context, _, progress, user_data) {\n *     # ...\n *   }\n *\n * +progress+ is an Integer between 0 and 100.\n *\n * call-seq:\n *   progress_callback = callback -> callback\n */\nstatic VALUE\nruby_whisper_params_set_progress_callback(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->progress_callback_container->callback = value;\n  return value;\n}\nstatic VALUE\nruby_whisper_params_get_progress_callback_user_data(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->progress_callback_container->user_data;\n}\n/*\n * Sets user data passed to the last argument of progress callback.\n *\n * call-seq:\n *   progress_callback_user_data = user_data -> use_data\n */\nstatic VALUE\nruby_whisper_params_set_progress_callback_user_data(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->progress_callback_container->user_data = value;\n  return value;\n}\n\nstatic VALUE\nruby_whisper_params_get_encoder_begin_callback(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->encoder_begin_callback_container->callback;\n}\n\n/*\n * Sets encoder begin callback, called when the encoder starts.\n *\n *   params.encoder_begin_callback = ->(context, _, user_data) {\n *     # ...\n *   }\n *\n * call-seq:\n *   encoder_begin_callback = callback -> callback\n */\nstatic VALUE\nruby_whisper_params_set_encoder_begin_callback(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->encoder_begin_callback_container->callback = value;\n  return value;\n}\n\nstatic VALUE\nruby_whisper_params_get_encoder_begin_callback_user_data(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->encoder_begin_callback_container->user_data;\n}\n\n/*\n * Sets user data passed to the last argument of encoder begin callback.\n *\n * call-seq:\n *   encoder_begin_callback_user_data = user_data -> use_data\n */\nstatic VALUE\nruby_whisper_params_set_encoder_begin_callback_user_data(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->encoder_begin_callback_container->user_data = value;\n  return value;\n}\n\nstatic VALUE\nruby_whisper_params_get_abort_callback(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->abort_callback_container->callback;\n}\n/*\n * Sets abort callback, called to check if the process should be aborted.\n *\n *   params.abort_callback = ->(user_data) {\n *     # ...\n *   }\n *\n * call-seq:\n *   abort_callback = callback -> callback\n */\nstatic VALUE\nruby_whisper_params_set_abort_callback(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->abort_callback_container->callback = value;\n  return value;\n}\nstatic VALUE\nruby_whisper_params_get_abort_callback_user_data(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->abort_callback_container->user_data;\n}\n/*\n * Sets user data passed to the last argument of abort callback.\n *\n * call-seq:\n *   abort_callback_user_data = user_data -> use_data\n */\nstatic VALUE\nruby_whisper_params_set_abort_callback_user_data(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->abort_callback_container->user_data = value;\n  return value;\n}\n\n/*\n * call-seq:\n *   vad = use_vad -> use_vad\n */\nstatic VALUE\nruby_whisper_params_get_vad(VALUE self)\n{\n  BOOL_PARAMS_GETTER(self, vad)\n}\n\nstatic VALUE\nruby_whisper_params_set_vad(VALUE self, VALUE value)\n{\n  BOOL_PARAMS_SETTER(self, vad, value)\n}\n\n/*\n * call-seq:\n *   vad_model_path = model_path -> model_path\n */\nstatic VALUE\nruby_whisper_params_set_vad_model_path(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  if (NIL_P(value)) {\n    rwp->params.vad_model_path = NULL;\n    return value;\n  }\n  VALUE path = ruby_whisper_normalize_model_path(value);\n  rwp->params.vad_model_path = StringValueCStr(path);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_params_get_vad_model_path(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->params.vad_model_path == NULL ? Qnil : rb_str_new2(rwp->params.vad_model_path);\n}\n\n/*\n * call-seq:\n *   vad_params = params -> params\n */\nstatic VALUE\nruby_whisper_params_set_vad_params(VALUE self, VALUE value)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  rwp->vad_params = value;\n  return value;\n}\n\nstatic VALUE\nruby_whisper_params_get_vad_params(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  return rwp->vad_params;\n}\n\n#define SET_PARAM_IF_SAME(param_name) \\\n  if (id == id_ ## param_name) { \\\n    ruby_whisper_params_set_ ## param_name(self, value); \\\n    continue; \\\n  }\n\nstatic VALUE\nruby_whisper_params_initialize(int argc, VALUE *argv, VALUE self)\n{\n  VALUE kw_hash;\n  VALUE values[RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT] = {Qundef};\n  VALUE value;\n  ruby_whisper_params *rwp;\n  ID id;\n  int i;\n\n  rb_scan_args_kw(RB_SCAN_ARGS_KEYWORDS, argc, argv, \":\", &kw_hash);\n  if (NIL_P(kw_hash)) {\n    return self;\n  }\n\n  rb_get_kwargs(kw_hash, param_names, 0, RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT, values);\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n\n  for (i = 0; i < RUBY_WHISPER_PARAMS_PARAM_NAMES_COUNT; i++) {\n    id = param_names[i];\n    value = values[i];\n    if (value == Qundef) {\n      continue;\n    }\n    if (id == id_diarize) {\n      rwp->diarize = value;\n      continue;\n    } else {\n      SET_PARAM_IF_SAME(language)\n      SET_PARAM_IF_SAME(translate)\n      SET_PARAM_IF_SAME(no_context)\n      SET_PARAM_IF_SAME(single_segment)\n      SET_PARAM_IF_SAME(print_special)\n      SET_PARAM_IF_SAME(print_progress)\n      SET_PARAM_IF_SAME(print_realtime)\n      SET_PARAM_IF_SAME(print_timestamps)\n      SET_PARAM_IF_SAME(suppress_blank)\n      SET_PARAM_IF_SAME(suppress_nst)\n      SET_PARAM_IF_SAME(token_timestamps)\n      SET_PARAM_IF_SAME(max_len)\n      SET_PARAM_IF_SAME(split_on_word)\n      SET_PARAM_IF_SAME(initial_prompt)\n      SET_PARAM_IF_SAME(carry_initial_prompt)\n      SET_PARAM_IF_SAME(offset)\n      SET_PARAM_IF_SAME(duration)\n      SET_PARAM_IF_SAME(max_text_tokens)\n      SET_PARAM_IF_SAME(temperature)\n      SET_PARAM_IF_SAME(max_initial_ts)\n      SET_PARAM_IF_SAME(length_penalty)\n      SET_PARAM_IF_SAME(temperature_inc)\n      SET_PARAM_IF_SAME(entropy_thold)\n      SET_PARAM_IF_SAME(logprob_thold)\n      SET_PARAM_IF_SAME(no_speech_thold)\n      SET_PARAM_IF_SAME(new_segment_callback)\n      SET_PARAM_IF_SAME(new_segment_callback_user_data)\n      SET_PARAM_IF_SAME(progress_callback)\n      SET_PARAM_IF_SAME(progress_callback_user_data)\n      SET_PARAM_IF_SAME(encoder_begin_callback)\n      SET_PARAM_IF_SAME(encoder_begin_callback_user_data)\n      SET_PARAM_IF_SAME(abort_callback)\n      SET_PARAM_IF_SAME(abort_callback_user_data)\n      SET_PARAM_IF_SAME(vad)\n      SET_PARAM_IF_SAME(vad_model_path)\n      SET_PARAM_IF_SAME(vad_params)\n    }\n  }\n\n  return self;\n}\n\n#undef SET_PARAM_IF_SAME\n\n/*\n * Hook called on new segment. Yields each Whisper::Segment.\n *\n *   whisper.on_new_segment do |segment|\n *     # ...\n *   end\n *\n * call-seq:\n *   on_new_segment {|segment| ... }\n */\nstatic VALUE\nruby_whisper_params_on_new_segment(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  const VALUE blk = rb_block_proc();\n  rb_ary_push(rwp->new_segment_callback_container->callbacks, blk);\n  return Qnil;\n}\n\n/*\n * Hook called on progress update. Yields each progress Integer between 0 and 100.\n *\n *   whisper.on_progress do |progress|\n *     # ...\n *   end\n *\n * call-seq:\n *   on_progress {|progress| ... }\n */\nstatic VALUE\nruby_whisper_params_on_progress(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  const VALUE blk = rb_block_proc();\n  rb_ary_push(rwp->progress_callback_container->callbacks, blk);\n  return Qnil;\n}\n\n/*\n * Hook called when the encoder starts.\n *\n *   whisper.on_encoder_begin do\n *     # ...\n *   end\n *\n * call-seq:\n *   on_encoder_begin { ... }\n */\nstatic VALUE\nruby_whisper_params_on_encoder_begin(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  const VALUE blk = rb_block_proc();\n  rb_ary_push(rwp->encoder_begin_callback_container->callbacks, blk);\n  return Qnil;\n}\n\n/*\n * Call block to determine whether abort or not. Return +true+ when you want to abort.\n *\n *   params.abort_on do\n *     if some_condition\n *       true # abort\n *     else\n *       false # continue\n *     end\n *   end\n *\n * call-seq:\n *   abort_on { ... }\n */\nstatic VALUE\nruby_whisper_params_abort_on(VALUE self)\n{\n  ruby_whisper_params *rwp;\n  TypedData_Get_Struct(self, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n  const VALUE blk = rb_block_proc();\n  rb_ary_push(rwp->abort_callback_container->callbacks, blk);\n  return Qnil;\n}\n\nvoid\ninit_ruby_whisper_params(VALUE *mWhisper)\n{\n  cParams  = rb_define_class_under(*mWhisper, \"Params\", rb_cObject);\n\n  rb_define_alloc_func(cParams, ruby_whisper_params_allocate);\n  rb_define_method(cParams, \"initialize\", ruby_whisper_params_initialize, -1);\n\n  DEFINE_PARAM(language, 0)\n  DEFINE_PARAM(translate, 1)\n  DEFINE_PARAM(no_context, 2)\n  DEFINE_PARAM(single_segment, 3)\n  DEFINE_PARAM(print_special, 4)\n  DEFINE_PARAM(print_progress, 5)\n  DEFINE_PARAM(print_realtime, 6)\n  DEFINE_PARAM(print_timestamps, 7)\n  DEFINE_PARAM(suppress_blank, 8)\n  DEFINE_PARAM(suppress_nst, 9)\n  DEFINE_PARAM(token_timestamps, 10)\n  DEFINE_PARAM(max_len, 11)\n  DEFINE_PARAM(split_on_word, 12)\n  DEFINE_PARAM(initial_prompt, 13)\n  DEFINE_PARAM(carry_initial_prompt, 14)\n  DEFINE_PARAM(diarize, 15)\n  DEFINE_PARAM(offset, 16)\n  DEFINE_PARAM(duration, 17)\n  DEFINE_PARAM(max_text_tokens, 18)\n  DEFINE_PARAM(temperature, 19)\n  DEFINE_PARAM(max_initial_ts, 20)\n  DEFINE_PARAM(length_penalty, 21)\n  DEFINE_PARAM(temperature_inc, 22)\n  DEFINE_PARAM(entropy_thold, 23)\n  DEFINE_PARAM(logprob_thold, 24)\n  DEFINE_PARAM(no_speech_thold, 25)\n  DEFINE_PARAM(new_segment_callback, 26)\n  DEFINE_PARAM(new_segment_callback_user_data, 27)\n  DEFINE_PARAM(progress_callback, 28)\n  DEFINE_PARAM(progress_callback_user_data, 29)\n  DEFINE_PARAM(encoder_begin_callback, 30)\n  DEFINE_PARAM(encoder_begin_callback_user_data, 31)\n  DEFINE_PARAM(abort_callback, 32)\n  DEFINE_PARAM(abort_callback_user_data, 33)\n  DEFINE_PARAM(vad, 34)\n  DEFINE_PARAM(vad_model_path, 35)\n  DEFINE_PARAM(vad_params, 36)\n\n  rb_define_method(cParams, \"on_new_segment\", ruby_whisper_params_on_new_segment, 0);\n  rb_define_method(cParams, \"on_progress\", ruby_whisper_params_on_progress, 0);\n  rb_define_method(cParams, \"on_encoder_begin\", ruby_whisper_params_on_encoder_begin, 0);\n  rb_define_method(cParams, \"abort_on\", ruby_whisper_params_abort_on, 0);\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_segment.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define N_KEY_NAMES 6\n\nextern ID id___method__;\nextern ID id_to_enum;\nstatic VALUE sym_start_time;\nstatic VALUE sym_end_time;\nstatic VALUE sym_text;\nstatic VALUE sym_no_speech_prob;\nstatic VALUE sym_speaker_turn_next;\nstatic VALUE sym_n_tokens;\n\nextern const rb_data_type_t ruby_whisper_type;\n\nextern VALUE cSegment;\n\nextern VALUE ruby_whisper_token_s_init(struct whisper_context *context, int i_segment, int index);\n\nstatic void\nrb_whisper_segment_mark(void *p)\n{\n  ruby_whisper_segment *rws = (ruby_whisper_segment *)p;\n  rb_gc_mark(rws->context);\n}\n\nstatic size_t\nruby_whisper_segment_memsize(const void *p)\n{\n  const ruby_whisper_segment *rws = (const ruby_whisper_segment *)p;\n  size_t size = sizeof(rws);\n  if (!rws) {\n    return 0;\n  }\n  if (rws->index) {\n    size += sizeof(rws->index);\n  }\n  return size;\n}\n\nstatic const rb_data_type_t ruby_whisper_segment_type = {\n  \"ruby_whisper_segment\",\n  {rb_whisper_segment_mark, RUBY_DEFAULT_FREE, ruby_whisper_segment_memsize,},\n  0, 0,\n  0\n};\n\nVALUE\nruby_whisper_segment_allocate(VALUE klass)\n{\n  ruby_whisper_segment *rws;\n  return TypedData_Make_Struct(klass, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n}\n\nVALUE\nrb_whisper_segment_s_new(VALUE context, int index)\n{\n  ruby_whisper_segment *rws;\n  const VALUE segment = ruby_whisper_segment_allocate(cSegment);\n  TypedData_Get_Struct(segment, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  rws->context = context;\n  rws->index = index;\n  return segment;\n};\n\n/*\n * Start time in milliseconds.\n *\n * call-seq:\n *   start_time -> Integer\n */\nstatic VALUE\nruby_whisper_segment_get_start_time(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  const int64_t t0 = whisper_full_get_segment_t0(rw->context, rws->index);\n  // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it\n  return LONG2NUM(t0 * 10);\n}\n\n/*\n * End time in milliseconds.\n *\n * call-seq:\n *   end_time -> Integer\n */\nstatic VALUE\nruby_whisper_segment_get_end_time(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  const int64_t t1 = whisper_full_get_segment_t1(rw->context, rws->index);\n  // able to multiply 10 without overflow because to_timestamp() in whisper.cpp does it\n  return LONG2NUM(t1 * 10);\n}\n\n/*\n * Whether the next segment is predicted as a speaker turn.\n *\n * call-seq:\n *   speaker_turn_next? -> bool\n */\nstatic VALUE\nruby_whisper_segment_get_speaker_turn_next(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  return whisper_full_get_segment_speaker_turn_next(rw->context, rws->index) ? Qtrue : Qfalse;\n}\n\n/*\n * call-seq:\n *   text -> String\n */\nstatic VALUE\nruby_whisper_segment_get_text(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  const char * text = whisper_full_get_segment_text(rw->context, rws->index);\n  return rb_str_new2(text);\n}\n\n/*\n * call-seq:\n *   no_speech_prob -> Float\n */\nstatic VALUE\nruby_whisper_segment_get_no_speech_prob(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  return DBL2NUM(whisper_full_get_segment_no_speech_prob(rw->context, rws->index));\n}\n\n/*\n * Get number of tokens in the segment\n *\n * call-seq:\n *   n_tokens -> Integer\n */\nstatic VALUE\nruby_whisper_segment_get_n_tokens(VALUE self)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n  return INT2NUM(whisper_full_n_tokens(rw->context, rws->index));\n}\n\n/*\n * Yields each Whisper::Token:\n *\n *   whisper.each_segment.first.each_token do |token|\n *     p token\n *   end\n *\n * Returns an Enumerator if no block is given:\n *\n *   whisper.each_segment.first.each_token.to_a # => [#<Whisper::Token>, ...]\n *\n * call-seq:\n *   each_token {|token| ... }\n *   each_token -> Enumerator\n */\nstatic VALUE\nruby_whisper_segment_each_token(VALUE self)\n{\n  if (!rb_block_given_p()) {\n    const VALUE method_name = rb_funcall(self, id___method__, 0);\n    return rb_funcall(self, id_to_enum, 1, method_name);\n  }\n\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n\n  const int n_tokens = whisper_full_n_tokens(rw->context, rws->index);\n  for (int i = 0; i < n_tokens; ++i) {\n    rb_yield(ruby_whisper_token_s_init(rw->context, rws->index, i));\n  }\n\n  return self;\n}\n\n/*\n * call-seq:\n *   deconstruct_keys(keys) -> hash\n *\n *  Possible keys: :start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next, :n_tokens\n *\n *   whisper.each_segment do |segment|\n *     segment => {start_time:, end_time:, text:, no_speech_prob:, speaker_turn_next:}\n *\n *     puts \"[#{start_time} --> #{end_time}] #{text} (no speech prob: #{no_speech_prob}#{speaker_turn_next ? ', speaker turns next' : ''})\"\n *   end\n */\nstatic VALUE\nruby_whisper_segment_deconstruct_keys(VALUE self, VALUE keys)\n{\n  ruby_whisper_segment *rws;\n  TypedData_Get_Struct(self, ruby_whisper_segment, &ruby_whisper_segment_type, rws);\n  ruby_whisper *rw;\n  GetContext(rws->context, rw);\n\n  VALUE hash = rb_hash_new();\n  long n_keys;\n  if (NIL_P(keys)) {\n    keys = rb_ary_new3(\n      N_KEY_NAMES,\n      sym_start_time,\n      sym_end_time,\n      sym_text,\n      sym_no_speech_prob,\n      sym_speaker_turn_next\n    );\n    n_keys = N_KEY_NAMES;\n  } else {\n    n_keys = RARRAY_LEN(keys);\n    if (n_keys > N_KEY_NAMES) {\n      return hash;\n    }\n  }\n  for (int i = 0; i < n_keys; i++) {\n    VALUE key = rb_ary_entry(keys, i);\n    if (key == sym_start_time) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_start_time(self));\n    }\n    if (key == sym_end_time) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_end_time(self));\n    }\n    if (key == sym_text) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_text(self));\n    }\n    if (key == sym_no_speech_prob) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_no_speech_prob(self));\n    }\n    if (key == sym_speaker_turn_next) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_speaker_turn_next(self));\n    }\n    if (key == sym_n_tokens) {\n      rb_hash_aset(hash, key, ruby_whisper_segment_get_n_tokens(self));\n    }\n  }\n\n  return hash;\n}\n\nvoid\ninit_ruby_whisper_segment(VALUE *mWhisper)\n{\n  cSegment  = rb_define_class_under(*mWhisper, \"Segment\", rb_cObject);\n\n  sym_start_time = ID2SYM(rb_intern(\"start_time\"));\n  sym_end_time = ID2SYM(rb_intern(\"end_time\"));\n  sym_text = ID2SYM(rb_intern(\"text\"));\n  sym_no_speech_prob = ID2SYM(rb_intern(\"no_speech_prob\"));\n  sym_speaker_turn_next = ID2SYM(rb_intern(\"speaker_turn_next\"));\n  sym_n_tokens = ID2SYM(rb_intern(\"n_tokens\"));\n\n  rb_define_alloc_func(cSegment, ruby_whisper_segment_allocate);\n  rb_define_method(cSegment, \"start_time\", ruby_whisper_segment_get_start_time, 0);\n  rb_define_method(cSegment, \"end_time\", ruby_whisper_segment_get_end_time, 0);\n  rb_define_method(cSegment, \"speaker_turn_next?\", ruby_whisper_segment_get_speaker_turn_next, 0);\n  rb_define_method(cSegment, \"text\", ruby_whisper_segment_get_text, 0);\n  rb_define_method(cSegment, \"no_speech_prob\", ruby_whisper_segment_get_no_speech_prob, 0);\n  rb_define_method(cSegment, \"n_tokens\", ruby_whisper_segment_get_n_tokens, 0);\n  rb_define_method(cSegment, \"each_token\", ruby_whisper_segment_each_token, 0);\n  rb_define_method(cSegment, \"deconstruct_keys\", ruby_whisper_segment_deconstruct_keys, 1);\n}\n#undef N_KEY_NAMES\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_token.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define N_KEY_NAMES 11\n\nextern VALUE cToken;\nextern const rb_data_type_t ruby_whisper_type;\n\nstatic VALUE sym_id;\nstatic VALUE sym_tid;\nstatic VALUE sym_probability;\nstatic VALUE sym_log_probability;\nstatic VALUE sym_pt;\nstatic VALUE sym_ptsum;\nstatic VALUE sym_t_dtw;\nstatic VALUE sym_voice_length;\nstatic VALUE sym_start_time;\nstatic VALUE sym_end_time;\nstatic VALUE sym_text;\n\nstatic size_t\nruby_whisper_token_memsize(const void *p)\n{\n  const ruby_whisper_token *rwt = (const ruby_whisper_token *)p;\n  if (!rwt) {\n    return 0;\n  }\n  size_t size = sizeof(*rwt);\n  if (rwt->token_data) {\n    size += sizeof(*rwt->token_data);\n  }\n  return size;\n}\n\nstatic void\nruby_whisper_token_mark(void *p)\n{\n  ruby_whisper_token *rwt = (ruby_whisper_token *)p;\n  rb_gc_mark(rwt->text);\n}\n\nstatic void\nruby_whisper_token_free(void *p)\n{\n  ruby_whisper_token *rwt = (ruby_whisper_token *)p;\n  if (rwt->token_data) {\n    xfree(rwt->token_data);\n    rwt->token_data = NULL;\n  }\n  xfree(rwt);\n}\n\nstatic const rb_data_type_t ruby_whisper_token_type = {\n  \"ruby_whisper_token\",\n  {ruby_whisper_token_mark, ruby_whisper_token_free, ruby_whisper_token_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_token_allocate(VALUE klass)\n{\n  ruby_whisper_token *rwt;\n  VALUE token = TypedData_Make_Struct(klass, ruby_whisper_token, &ruby_whisper_token_type, rwt);\n  rwt->token_data = NULL;\n  rwt->text = Qnil;\n  return token;\n}\n\nVALUE\nruby_whisper_token_s_init(struct whisper_context *context, int i_segment, int i_token)\n{\n  const VALUE token = ruby_whisper_token_allocate(cToken);\n  ruby_whisper_token *rwt;\n  TypedData_Get_Struct(token, ruby_whisper_token, &ruby_whisper_token_type, rwt);\n  rwt->token_data = ALLOC(whisper_token_data);\n  *(rwt->token_data) = whisper_full_get_token_data(context, i_segment, i_token);\n  rwt->text = rb_str_new2(whisper_full_get_token_text(context, i_segment, i_token));\n  return token;\n}\n\n/*\n * Token ID.\n *\n * call-seq:\n *   id -> Integer\n */\nstatic VALUE\nruby_whisper_token_get_id(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return INT2NUM(rwt->token_data->id);\n}\n\n/*\n * Forced timestamp token ID.\n *\n * call-seq:\n *   tid -> Integer\n */\nstatic VALUE\nruby_whisper_token_get_tid(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return INT2NUM(rwt->token_data->tid);\n}\n\n/*\n * Probability of the token.\n *\n * call-seq:\n *   probability -> Float\n */\nstatic VALUE\nruby_whisper_token_get_p(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return DBL2NUM(rwt->token_data->p);\n}\n\n/*\n * Log probability of the token.\n *\n * call-seq:\n *   log_probability -> Float\n */\nstatic VALUE\nruby_whisper_token_get_plog(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return DBL2NUM(rwt->token_data->plog);\n}\n\n/*\n * Probability of the timestamp token.\n *\n * call-seq:\n *   pt -> Float\n */\nstatic VALUE\nruby_whisper_token_get_pt(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return DBL2NUM(rwt->token_data->pt);\n}\n\n/*\n * Sum of probability of all timestamp tokens.\n *\n * call-seq:\n *   ptsum -> Float\n */\nstatic VALUE\nruby_whisper_token_get_ptsum(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return DBL2NUM(rwt->token_data->ptsum);\n}\n\n/*\n * [EXPERIMENTAL] Token-level timestamps with DTW\n *\n * Do not use if you haven't computed token-level timestamps with dtw.\n * Roughly corresponds to the moment in audio in which the token was output.\n *\n * call-seq:\n *   t_dtw -> Integer\n */\nstatic VALUE\nruby_whisper_token_get_t_dtw(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return LONG2NUM(rwt->token_data->t_dtw);\n}\n\n/*\n * Voice length of the token.\n *\n * call-seq:\n *   voice_length -> Float\n */\nstatic VALUE\nruby_whisper_token_get_vlen(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return DBL2NUM(rwt->token_data->vlen);\n}\n\n/*\n * Get the token text of the token.\n *\n * call-seq:\n *   text -> String\n */\nstatic VALUE\nruby_whisper_token_get_text(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return rwt->text;\n}\n\n/*\n * Start time of the token.\n *\n * Token-level timestamp data.\n * Do not use if you haven't computed token-level timestamps.\n *\n * call-seq:\n *   start_time -> Integer\n */\nstatic VALUE\nruby_whisper_token_get_start_time(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return LONG2NUM(rwt->token_data->t0 * 10);\n}\n\n/*\n * End time of the token.\n *\n * Token-level timestamp data.\n * Do not use if you haven't computed token-level timestamps.\n *\n * call-seq:\n *   end_time -> Integer\n */\nstatic VALUE\nruby_whisper_token_get_end_time(VALUE self)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  return LONG2NUM(rwt->token_data->t1 * 10);\n}\n\n/*\n * call-seq:\n *   deconstruct_keys(keys) -> hash\n *\n *  Possible keys: :id, :tid, :probability, :log_probability, :pt, :ptsum,\n *                 :t_dtw, :voice_length, :start_time, :end_time, :text\n *    segment.each_token do |token|\n *      token => {text:, probability:}\n        puts \"#{text} (#{probability})\"\n *    end\n */\nstatic VALUE ruby_whisper_token_deconstruct_keys(VALUE self, VALUE keys)\n{\n  ruby_whisper_token *rwt;\n  GetToken(self, rwt);\n  VALUE hash = rb_hash_new();\n  long n_keys = 0;\n\n  if (NIL_P(keys)) {\n    keys = rb_ary_new3(\n      N_KEY_NAMES,\n      sym_id,\n      sym_tid,\n      sym_probability,\n      sym_log_probability,\n      sym_pt,\n      sym_ptsum,\n      sym_t_dtw,\n      sym_voice_length,\n      sym_start_time,\n      sym_end_time,\n      sym_text\n    );\n    n_keys = N_KEY_NAMES;\n  } else {\n    n_keys = RARRAY_LEN(keys);\n    if (n_keys > N_KEY_NAMES) {\n      return hash;\n    }\n  }\n\n  for (int i = 0; i < n_keys; i++) {\n    VALUE key = rb_ary_entry(keys, i);\n    if (key == sym_start_time) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_start_time(self));\n      continue;\n    }\n    if (key == sym_end_time) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_end_time(self));\n      continue;\n    }\n    if (key == sym_text) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_text(self));\n      continue;\n    }\n    if (key == sym_probability) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_p(self));\n      continue;\n    }\n    if (key == sym_id) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_id(self));\n      continue;\n    }\n    if (key == sym_tid) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_tid(self));\n      continue;\n    }\n    if (key == sym_log_probability) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_plog(self));\n      continue;\n    }\n    if (key == sym_pt) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_pt(self));\n      continue;\n    }\n    if (key == sym_ptsum) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_ptsum(self));\n      continue;\n    }\n    if (key == sym_t_dtw) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_t_dtw(self));\n      continue;\n    }\n    if (key == sym_voice_length) {\n      rb_hash_aset(hash, key, ruby_whisper_token_get_vlen(self));\n      continue;\n    }\n  }\n\n  return hash;\n}\n\n\nvoid\ninit_ruby_whisper_token(VALUE *mWhisper)\n{\n  cToken = rb_define_class_under(*mWhisper, \"Token\", rb_cObject);\n\n  rb_define_alloc_func(cToken, ruby_whisper_token_allocate);\n\n  sym_id = ID2SYM(rb_intern(\"id\"));\n  sym_tid = ID2SYM(rb_intern(\"tid\"));\n  sym_probability = ID2SYM(rb_intern(\"probability\"));\n  sym_log_probability = ID2SYM(rb_intern(\"log_probability\"));\n  sym_pt = ID2SYM(rb_intern(\"pt\"));\n  sym_ptsum = ID2SYM(rb_intern(\"ptsum\"));\n  sym_t_dtw = ID2SYM(rb_intern(\"t_dtw\"));\n  sym_voice_length = ID2SYM(rb_intern(\"voice_length\"));\n  sym_start_time = ID2SYM(rb_intern(\"start_time\"));\n  sym_end_time = ID2SYM(rb_intern(\"end_time\"));\n  sym_text = ID2SYM(rb_intern(\"text\"));\n\n  rb_define_method(cToken, \"id\", ruby_whisper_token_get_id, 0);\n  rb_define_method(cToken, \"tid\", ruby_whisper_token_get_tid, 0);\n  rb_define_method(cToken, \"probability\", ruby_whisper_token_get_p, 0);\n  rb_define_method(cToken, \"log_probability\", ruby_whisper_token_get_plog, 0);\n  rb_define_method(cToken, \"pt\", ruby_whisper_token_get_pt, 0);\n  rb_define_method(cToken, \"ptsum\", ruby_whisper_token_get_ptsum, 0);\n  rb_define_method(cToken, \"t_dtw\", ruby_whisper_token_get_t_dtw, 0);\n  rb_define_method(cToken, \"voice_length\", ruby_whisper_token_get_vlen, 0);\n  rb_define_method(cToken, \"start_time\", ruby_whisper_token_get_start_time, 0);\n  rb_define_method(cToken, \"end_time\", ruby_whisper_token_get_end_time, 0);\n  rb_define_method(cToken, \"text\", ruby_whisper_token_get_text, 0);\n\n  rb_define_method(cToken, \"deconstruct_keys\", ruby_whisper_token_deconstruct_keys, 1);\n}\n\n#undef N_KEY_NAMES\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_transcribe.cpp",
    "content": "#include \"ruby_whisper.h\"\n#include \"common-whisper.h\"\n#include <string>\n#include <vector>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nextern const rb_data_type_t ruby_whisper_type;\nextern const rb_data_type_t ruby_whisper_params_type;\n\nextern ID id_to_s;\nextern ID id_call;\nextern ID id_to_path;\nextern ID transcribe_option_names[1];\n\nextern void\nprepare_transcription(ruby_whisper_params * rwp, VALUE * self);\n\n/*\n * transcribe a single file\n * can emit to a block results\n *\n *   params = Whisper::Params.new\n *   params.duration = 60_000\n *   whisper.transcribe \"path/to/audio.wav\", params do |text|\n *     puts text\n *   end\n *\n * call-seq:\n *   transcribe(path_to_audio, params) {|text| ...}\n **/\nVALUE\nruby_whisper_transcribe(int argc, VALUE *argv, VALUE self) {\n  ruby_whisper *rw;\n  ruby_whisper_params *rwp;\n  VALUE wave_file_path, blk, params, kws;\n  VALUE opts[1];\n\n  rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, \"2:&\", &wave_file_path, &params, &kws, &blk);\n  rb_get_kwargs(kws, transcribe_option_names, 0, 1, opts);\n\n  int n_processors = opts[0] == Qundef ? 1 : NUM2INT(opts[0]);\n\n  GetContext(self, rw);\n  TypedData_Get_Struct(params, ruby_whisper_params, &ruby_whisper_params_type, rwp);\n\n  if (!rb_respond_to(wave_file_path, id_to_s)) {\n    rb_raise(rb_eRuntimeError, \"Expected file path to wave file\");\n  }\n\n  if (rb_respond_to(wave_file_path, id_to_path)) {\n    wave_file_path = rb_funcall(wave_file_path, id_to_path, 0);\n  }\n  std::string fname_inp = StringValueCStr(wave_file_path);\n\n  std::vector<float> pcmf32; // mono-channel F32 PCM\n  std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM\n\n  if (!read_audio_data(fname_inp, pcmf32, pcmf32s, rwp->diarize)) {\n    fprintf(stderr, \"error: failed to open '%s' as WAV file\\n\", fname_inp.c_str());\n    return self;\n  }\n  // Commented out because it is work in progress\n  // {\n  //   static bool is_aborted = false; // NOTE: this should be atomic to avoid data race\n\n  //   rwp->params.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {\n  //     bool is_aborted = *(bool*)user_data;\n  //     return !is_aborted;\n  //   };\n  //   rwp->params.encoder_begin_callback_user_data = &is_aborted;\n  // }\n\n  prepare_transcription(rwp, &self);\n\n  if (whisper_full_parallel(rw->context, rwp->params, pcmf32.data(), pcmf32.size(), n_processors) != 0) {\n    fprintf(stderr, \"failed to process audio\\n\");\n    return self;\n  }\n  if (NIL_P(blk)) {\n    return self;\n  }\n  const int n_segments = whisper_full_n_segments(rw->context);\n  VALUE output = rb_str_new2(\"\");\n  for (int i = 0; i < n_segments; ++i) {\n    const char * text = whisper_full_get_segment_text(rw->context, i);\n    output = rb_str_concat(output, rb_str_new2(text));\n  }\n  rb_funcall(blk, id_call, 1, output);\n  return self;\n}\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_vad_context.c",
    "content": "#include \"ruby_whisper.h\"\n\nextern ID id_to_s;\n\nextern VALUE cVADContext;\n\nextern const rb_data_type_t ruby_whisper_vad_params_type;\nextern VALUE ruby_whisper_vad_detect(VALUE self, VALUE file_path, VALUE params);\nextern VALUE ruby_whisper_normalize_model_path(VALUE model_path);\nextern parsed_samples_t parse_samples(VALUE *samples, VALUE *n_samples);\nextern VALUE release_samples(VALUE parsed);\n\nextern VALUE ruby_whisper_vad_segments_s_init(struct whisper_vad_segments *segments);\n\ntypedef struct segments_from_samples_args {\n  VALUE *context;\n  VALUE *params;\n  float *samples;\n  int n_samples;\n} segments_from_samples_args;\n\nstatic size_t\nruby_whisper_vad_context_memsize(const void *p)\n{\n  const ruby_whisper_vad_context *rwvc = p;\n  size_t size = sizeof(rwvc);\n  if (!rwvc) {\n    return 0;\n  }\n  if (rwvc->context) {\n    size += sizeof(rwvc->context);\n  }\n  return size;\n}\n\nstatic void\nruby_whisper_vad_context_free(void *p)\n{\n  ruby_whisper_vad_context *rwvc = (ruby_whisper_vad_context *)p;\n  if (rwvc->context) {\n    whisper_vad_free(rwvc->context);\n    rwvc->context = NULL;\n  }\n  xfree(rwvc);\n}\n\nconst rb_data_type_t ruby_whisper_vad_context_type = {\n  \"ruby_whisper_vad_context\",\n  {0, ruby_whisper_vad_context_free, ruby_whisper_vad_context_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_vad_context_s_allocate(VALUE klass)\n{\n  ruby_whisper_vad_context *rwvc;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper_vad_context, &ruby_whisper_vad_context_type, rwvc);\n  rwvc->context = NULL;\n  return obj;\n}\n\nstatic VALUE\nruby_whisper_vad_context_initialize(VALUE self, VALUE model_path)\n{\n  ruby_whisper_vad_context *rwvc;\n  struct whisper_vad_context *context;\n\n  model_path = ruby_whisper_normalize_model_path(model_path);\n  context = whisper_vad_init_from_file_with_params(StringValueCStr(model_path), whisper_vad_default_context_params());\n  if (context == NULL) {\n    rb_raise(rb_eRuntimeError, \"Failed to initialize whisper VAD context\");\n  }\n  TypedData_Get_Struct(self, ruby_whisper_vad_context, &ruby_whisper_vad_context_type, rwvc);\n  rwvc->context = context;\n\n  return Qnil;\n}\n\nstatic VALUE\nsegments_from_samples_body(VALUE rb_args)\n{\n  segments_from_samples_args *args = (segments_from_samples_args *)rb_args;\n\n  ruby_whisper_vad_context *rwvc;\n  ruby_whisper_vad_params *rwvp;\n  GetVADContext(*args->context, rwvc);\n  GetVADParams(*args->params, rwvp);\n\n  struct whisper_vad_segments *segments = whisper_vad_segments_from_samples(rwvc->context, rwvp->params, args->samples, args->n_samples);\n\n  return ruby_whisper_vad_segments_s_init(segments);\n}\n\nstatic VALUE\nruby_whisper_vad_segments_from_samples(int argc, VALUE *argv, VALUE self)\n{\n  if (argc < 2 || argc > 3) {\n    rb_raise(rb_eArgError, \"wrong number of arguments (given %d, expected 2..3)\", argc);\n  }\n\n  VALUE n_samples = argc == 2 ? Qnil : argv[2];\n  struct parsed_samples_t parsed = parse_samples(&argv[1], &n_samples);\n  segments_from_samples_args args = {\n    &self,\n    &argv[0],\n    parsed.samples,\n    parsed.n_samples,\n  };\n  VALUE segments = rb_ensure(segments_from_samples_body, (VALUE)&args, release_samples, (VALUE)&parsed);\n\n  return segments;\n}\n\nvoid init_ruby_whisper_vad_context(VALUE *mVAD)\n{\n  cVADContext = rb_define_class_under(*mVAD, \"Context\", rb_cObject);\n  rb_define_alloc_func(cVADContext, ruby_whisper_vad_context_s_allocate);\n  rb_define_method(cVADContext, \"initialize\", ruby_whisper_vad_context_initialize, 1);\n  rb_define_method(cVADContext, \"segments_from_samples\", ruby_whisper_vad_segments_from_samples, -1);\n  rb_define_method(cVADContext, \"detect\", ruby_whisper_vad_detect, 2);\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_vad_context_detect.cpp",
    "content": "#include \"ruby_whisper.h\"\n#include \"common-whisper.h\"\n#include <string>\n#include <vector>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nextern ID id_to_path;\n\nextern VALUE cVADSegments;\n\nextern const rb_data_type_t ruby_whisper_vad_context_type;\nextern const rb_data_type_t ruby_whisper_vad_params_type;\nextern const rb_data_type_t ruby_whisper_vad_segments_type;\n\nextern VALUE ruby_whisper_vad_segments_s_init(struct whisper_vad_segments *segments);\n\nVALUE\nruby_whisper_vad_detect(VALUE self, VALUE file_path, VALUE params) {\n  ruby_whisper_vad_context *rwvc;\n  ruby_whisper_vad_params *rwvp;\n  std::string cpp_file_path;\n  std::vector<float> pcmf32;\n  std::vector<std::vector<float>> pcmf32s;\n  whisper_vad_segments *segments;\n\n  GetVADContext(self, rwvc);\n  TypedData_Get_Struct(params, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n\n  if (rb_respond_to(file_path, id_to_path)) {\n    file_path = rb_funcall(file_path, id_to_path, 0);\n  }\n  cpp_file_path = StringValueCStr(file_path);\n\n  if (!read_audio_data(cpp_file_path, pcmf32, pcmf32s, false)) {\n    rb_raise(rb_eRuntimeError, \"Failed to open '%s' as WAV file\\n\", cpp_file_path.c_str());\n  }\n\n  segments = whisper_vad_segments_from_samples(rwvc->context, rwvp->params, pcmf32.data(), pcmf32.size());\n  if (segments == nullptr) {\n    rb_raise(rb_eRuntimeError, \"Failed to process audio\\n\");\n  }\n\n  return ruby_whisper_vad_segments_s_init(segments);\n}\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_vad_params.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define DEFINE_PARAM(param_name, nth) \\\n  id_ ## param_name = rb_intern(#param_name); \\\n  param_names[nth] = id_ ## param_name; \\\n  rb_define_method(cVADParams, #param_name, ruby_whisper_vad_params_get_ ## param_name, 0); \\\n  rb_define_method(cVADParams, #param_name \"=\", ruby_whisper_vad_params_set_ ## param_name, 1);\n\n#define NUM_PARAMS 6\n\nextern VALUE cVADParams;\n\nstatic size_t\nruby_whisper_vad_params_memsize(const void *p)\n{\n  const struct ruby_whisper_vad_params *params = p;\n  size_t size = sizeof(params);\n  if (!params) {\n    return 0;\n  }\n  return size;\n}\n\nstatic ID param_names[NUM_PARAMS];\nstatic ID id_threshold;\nstatic ID id_min_speech_duration_ms;\nstatic ID id_min_silence_duration_ms;\nstatic ID id_max_speech_duration_s;\nstatic ID id_speech_pad_ms;\nstatic ID id_samples_overlap;\n\nconst rb_data_type_t ruby_whisper_vad_params_type = {\n  \"ruby_whisper_vad_params\",\n  {0, 0, ruby_whisper_vad_params_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_vad_params_s_allocate(VALUE klass)\n{\n  ruby_whisper_vad_params *rwvp;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params = whisper_vad_default_params();\n  return obj;\n}\n\n/*\n * Probability threshold to consider as speech.\n *\n * call-seq:\n *   threshold = th -> th\n */\nstatic VALUE\nruby_whisper_vad_params_set_threshold(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.threshold = RFLOAT_VALUE(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_threshold(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return DBL2NUM(rwvp->params.threshold);\n}\n\n/*\n * Min duration for a valid speech segment.\n *\n * call-seq:\n *   min_speech_duration_ms = duration_ms -> duration_ms\n */\nstatic VALUE\nruby_whisper_vad_params_set_min_speech_duration_ms(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.min_speech_duration_ms = NUM2INT(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_min_speech_duration_ms(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return INT2NUM(rwvp->params.min_speech_duration_ms);\n}\n\n/*\n * Min silence duration to consider speech as ended.\n *\n * call-seq:\n *   min_silence_duration_ms = duration_ms -> duration_ms\n */\nstatic VALUE\nruby_whisper_vad_params_set_min_silence_duration_ms(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.min_silence_duration_ms = NUM2INT(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_min_silence_duration_ms(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return INT2NUM(rwvp->params.min_silence_duration_ms);\n}\n\n/*\n * Max duration of a speech segment before forcing a new segment.\n *\n * call-seq:\n *   max_speech_duration_s = duration_s -> duration_s\n */\nstatic VALUE\nruby_whisper_vad_params_set_max_speech_duration_s(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.max_speech_duration_s = RFLOAT_VALUE(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_max_speech_duration_s(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return DBL2NUM(rwvp->params.max_speech_duration_s);\n}\n\n/*\n * Padding added before and after speech segments.\n *\n * call-seq:\n *   speech_pad_ms = pad_ms -> pad_ms\n */\nstatic VALUE\nruby_whisper_vad_params_set_speech_pad_ms(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.speech_pad_ms = NUM2INT(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_speech_pad_ms(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return INT2NUM(rwvp->params.speech_pad_ms);\n}\n\n/*\n * Overlap in seconds when copying audio samples from speech segment.\n *\n * call-seq:\n *   samples_overlap = overlap -> overlap\n */\nstatic VALUE\nruby_whisper_vad_params_set_samples_overlap(VALUE self, VALUE value)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  rwvp->params.samples_overlap = RFLOAT_VALUE(value);\n  return value;\n}\n\nstatic VALUE\nruby_whisper_vad_params_get_samples_overlap(VALUE self)\n{\n  ruby_whisper_vad_params *rwvp;\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n  return DBL2NUM(rwvp->params.samples_overlap);\n}\n\nstatic VALUE\nruby_whisper_vad_params_equal(VALUE self, VALUE other)\n{\n  ruby_whisper_vad_params *rwvp1;\n  ruby_whisper_vad_params *rwvp2;\n\n  if (self == other) {\n    return Qtrue;\n  }\n\n  if (!rb_obj_is_kind_of(other, cVADParams)) {\n    return Qfalse;\n  }\n\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp1);\n  TypedData_Get_Struct(other, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp2);\n\n  if (rwvp1->params.threshold != rwvp2->params.threshold) {\n    return Qfalse;\n  }\n  if (rwvp1->params.min_speech_duration_ms != rwvp2->params.min_speech_duration_ms) {\n    return Qfalse;\n  }\n  if (rwvp1->params.min_silence_duration_ms != rwvp2->params.min_silence_duration_ms) {\n    return Qfalse;\n  }\n  if (rwvp1->params.max_speech_duration_s != rwvp2->params.max_speech_duration_s) {\n    return Qfalse;\n  }\n  if (rwvp1->params.speech_pad_ms != rwvp2->params.speech_pad_ms) {\n    return Qfalse;\n  }\n  if (rwvp1->params.samples_overlap != rwvp2->params.samples_overlap) {\n    return Qfalse;\n  }\n\n  return Qtrue;\n}\n\n#define SET_PARAM_IF_SAME(param_name) \\\n  if (id == id_ ## param_name) { \\\n    ruby_whisper_vad_params_set_ ## param_name(self, value); \\\n    continue; \\\n  }\n\nVALUE\nruby_whisper_vad_params_initialize(int argc, VALUE *argv, VALUE self)\n{\n  VALUE kw_hash;\n  VALUE values[NUM_PARAMS] = {Qundef};\n  VALUE value;\n  ruby_whisper_vad_params *rwvp;\n  ID id;\n  int i;\n\n  TypedData_Get_Struct(self, ruby_whisper_vad_params, &ruby_whisper_vad_params_type, rwvp);\n\n  rb_scan_args_kw(RB_SCAN_ARGS_KEYWORDS, argc, argv, \":\", &kw_hash);\n  if (NIL_P(kw_hash)) {\n    return self;\n  }\n\n  rb_get_kwargs(kw_hash, param_names, 0, NUM_PARAMS, values);\n\n  for (i = 0; i < NUM_PARAMS; i++) {\n    id = param_names[i];\n    value = values[i];\n    if (value == Qundef) {\n      continue;\n    }\n    SET_PARAM_IF_SAME(threshold)\n    SET_PARAM_IF_SAME(min_speech_duration_ms)\n    SET_PARAM_IF_SAME(min_silence_duration_ms)\n    SET_PARAM_IF_SAME(max_speech_duration_s)\n    SET_PARAM_IF_SAME(speech_pad_ms)\n    SET_PARAM_IF_SAME(samples_overlap)\n  }\n\n  return self;\n}\n\n#undef SET_PARAM_IF_SAME\n\nvoid\ninit_ruby_whisper_vad_params(VALUE *mVAD)\n{\n  cVADParams = rb_define_class_under(*mVAD, \"Params\", rb_cObject);\n  rb_define_alloc_func(cVADParams, ruby_whisper_vad_params_s_allocate);\n  rb_define_method(cVADParams, \"initialize\", ruby_whisper_vad_params_initialize, -1);\n\n  DEFINE_PARAM(threshold, 0)\n  DEFINE_PARAM(min_speech_duration_ms, 1)\n  DEFINE_PARAM(min_silence_duration_ms, 2)\n  DEFINE_PARAM(max_speech_duration_s, 3)\n  DEFINE_PARAM(speech_pad_ms, 4)\n  DEFINE_PARAM(samples_overlap, 5)\n\n  rb_define_method(cVADParams, \"==\", ruby_whisper_vad_params_equal, 1);\n}\n\n#undef DEFINE_PARAM\n#undef NUM_PARAMS\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_vad_segment.c",
    "content": "#include \"ruby_whisper.h\"\n\n#define N_KEY_NAMES 2\n\nextern VALUE cVADSegment;\n\nextern const rb_data_type_t ruby_whisper_vad_segments_type;\n\nstatic VALUE sym_start_time;\nstatic VALUE sym_end_time;\n\nstatic void\nrb_whisper_vad_segment_mark(void *p)\n{\n  ruby_whisper_vad_segment *rwvs = (ruby_whisper_vad_segment *)p;\n  rb_gc_mark(rwvs->segments);\n}\n\nstatic size_t\nruby_whisper_vad_segment_memsize(const void *p)\n{\n  const ruby_whisper_vad_segment *rwvs = p;\n  size_t size = sizeof(rwvs);\n  if (!rwvs) {\n    return 0;\n  }\n  if (rwvs->index) {\n    size += sizeof(rwvs->index);\n  }\n  return size;\n}\n\nstatic const rb_data_type_t ruby_whisper_vad_segment_type = {\n    \"ruby_whisper_vad_segment\",\n    {rb_whisper_vad_segment_mark, RUBY_DEFAULT_FREE, ruby_whisper_vad_segment_memsize,},\n    0, 0,\n    0\n};\n\nstatic VALUE\nruby_whisper_vad_segment_s_allocate(VALUE klass)\n{\n  ruby_whisper_vad_segment *rwvs;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper_vad_segment, &ruby_whisper_vad_segment_type, rwvs);\n  rwvs->segments = Qnil;\n  rwvs->index = -1;\n  return obj;\n}\n\nVALUE\nrb_whisper_vad_segment_s_new(VALUE segments, int index)\n{\n  ruby_whisper_vad_segment *rwvs;\n  const VALUE segment = ruby_whisper_vad_segment_s_allocate(cVADSegment);\n  TypedData_Get_Struct(segment, ruby_whisper_vad_segment, &ruby_whisper_vad_segment_type, rwvs);\n  rwvs->segments = segments;\n  rwvs->index = index;\n  return segment;\n}\n\nstatic VALUE\nruby_whisper_vad_segment_get_start_time(VALUE self)\n{\n  ruby_whisper_vad_segment *rwvs;\n  ruby_whisper_vad_segments *rwvss;\n  float t0;\n\n  TypedData_Get_Struct(self, ruby_whisper_vad_segment, &ruby_whisper_vad_segment_type, rwvs);\n  TypedData_Get_Struct(rwvs->segments, ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, rwvss);\n  t0 = whisper_vad_segments_get_segment_t0(rwvss->segments, rwvs->index);\n  return DBL2NUM(t0 * 10);\n}\n\nstatic VALUE\nruby_whisper_vad_segment_get_end_time(VALUE self)\n{\n  ruby_whisper_vad_segment *rwvs;\n  ruby_whisper_vad_segments *rwvss;\n  float t1;\n\n  TypedData_Get_Struct(self, ruby_whisper_vad_segment, &ruby_whisper_vad_segment_type, rwvs);\n  TypedData_Get_Struct(rwvs->segments, ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, rwvss);\n  t1 = whisper_vad_segments_get_segment_t1(rwvss->segments, rwvs->index);\n  return DBL2NUM(t1 * 10);\n}\n\nstatic VALUE\nruby_whisper_vad_segment_deconstruct_keys(VALUE self, VALUE keys)\n{\n  ruby_whisper_vad_segment *rwvs;\n  ruby_whisper_vad_segments *rwvss;\n  VALUE hash, key;\n  long n_keys;\n  int i;\n\n  TypedData_Get_Struct(self, ruby_whisper_vad_segment, &ruby_whisper_vad_segment_type, rwvs);\n  TypedData_Get_Struct(rwvs->segments, ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, rwvss);\n\n  hash = rb_hash_new();\n  if (NIL_P(keys)) {\n    keys = rb_ary_new3(\n      N_KEY_NAMES,\n      sym_start_time,\n      sym_end_time\n    );\n    n_keys = N_KEY_NAMES;\n  } else {\n    n_keys = RARRAY_LEN(keys);\n    if (n_keys > N_KEY_NAMES) {\n      return hash;\n    }\n  }\n  for (i = 0; i < n_keys; i++) {\n    key = rb_ary_entry(keys, i);\n    if (key == sym_start_time) {\n      rb_hash_aset(hash, key, ruby_whisper_vad_segment_get_start_time(self));\n    }\n    if (key == sym_end_time) {\n      rb_hash_aset(hash, key, ruby_whisper_vad_segment_get_end_time(self));\n    }\n  }\n\n  return hash;\n}\n\nvoid\ninit_ruby_whisper_vad_segment(VALUE *mVAD)\n{\n  cVADSegment = rb_define_class_under(*mVAD, \"Segment\", rb_cObject);\n\n  sym_start_time = ID2SYM(rb_intern(\"start_time\"));\n  sym_end_time = ID2SYM(rb_intern(\"end_time\"));\n\n  rb_define_alloc_func(cVADSegment, ruby_whisper_vad_segment_s_allocate);\n  rb_define_method(cVADSegment, \"start_time\", ruby_whisper_vad_segment_get_start_time, 0);\n  rb_define_method(cVADSegment, \"end_time\", ruby_whisper_vad_segment_get_end_time, 0);\n  rb_define_method(cVADSegment, \"deconstruct_keys\", ruby_whisper_vad_segment_deconstruct_keys, 1);\n}\n"
  },
  {
    "path": "bindings/ruby/ext/ruby_whisper_vad_segments.c",
    "content": "#include \"ruby_whisper.h\"\n\nextern ID id___method__;\nextern ID id_to_enum;\n\nextern VALUE cVADSegments;\n\nextern VALUE rb_whisper_vad_segment_s_new(VALUE segments, int index);\n\nstatic size_t\nruby_whisper_vad_segments_memsize(const void *p)\n{\n  const ruby_whisper_vad_segments *rwvss = p;\n  size_t size = sizeof(rwvss);\n  if (!rwvss) {\n    return 0;\n  }\n  if (rwvss->segments) {\n    size += sizeof(rwvss->segments);\n  }\n  return size;\n}\n\nstatic void\nruby_whisper_vad_segments_free(void *p)\n{\n  ruby_whisper_vad_segments *rwvss = (ruby_whisper_vad_segments *)p;\n  if (rwvss->segments) {\n    whisper_vad_free_segments(rwvss->segments);\n    rwvss->segments = NULL;\n  }\n  xfree(rwvss);\n}\n\nconst rb_data_type_t ruby_whisper_vad_segments_type = {\n  \"ruby_whisper_vad_segments\",\n  {0, ruby_whisper_vad_segments_free, ruby_whisper_vad_segments_memsize,},\n  0, 0,\n  0\n};\n\nstatic VALUE\nruby_whisper_vad_segments_s_allocate(VALUE klass)\n{\n  ruby_whisper_vad_segments *rwvss;\n  VALUE obj = TypedData_Make_Struct(klass, ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, rwvss);\n  rwvss->segments = NULL;\n  return obj;\n}\n\nVALUE\nruby_whisper_vad_segments_s_init(struct whisper_vad_segments *segments)\n{\n  VALUE rb_segments;\n  ruby_whisper_vad_segments *rwvss;\n\n  rb_segments = ruby_whisper_vad_segments_s_allocate(cVADSegments);\n  TypedData_Get_Struct(rb_segments, ruby_whisper_vad_segments, &ruby_whisper_vad_segments_type, rwvss);\n  rwvss->segments = segments;\n\n  return rb_segments;\n}\n\nstatic VALUE\nruby_whisper_vad_segments_each(VALUE self)\n{\n  ruby_whisper_vad_segments *rwvss;\n  VALUE method_name;\n  int n_segments, i;\n\n  if (!rb_block_given_p()) {\n    method_name = rb_funcall(self, id___method__, 0);\n    return rb_funcall(self, id_to_enum, 1, method_name);\n  }\n\n  GetVADSegments(self, rwvss);\n  n_segments = whisper_vad_segments_n_segments(rwvss->segments);\n  for (i = 0; i < n_segments; ++i) {\n    rb_yield(rb_whisper_vad_segment_s_new(self, i));\n  }\n\n  return self;\n}\n\nstatic VALUE\nruby_whisper_vad_segments_get_length(VALUE self)\n{\n  ruby_whisper_vad_segments *rwvss;\n  int n_segments;\n\n  GetVADSegments(self, rwvss);\n  n_segments = whisper_vad_segments_n_segments(rwvss->segments);\n\n  return INT2NUM(n_segments);\n}\n\nvoid\ninit_ruby_whisper_vad_segments(VALUE *mVAD)\n{\n  cVADSegments = rb_define_class_under(*mVAD, \"Segments\", rb_cObject);\n  rb_define_alloc_func(cVADSegments, ruby_whisper_vad_segments_s_allocate);\n  rb_define_method(cVADSegments, \"each\", ruby_whisper_vad_segments_each, 0);\n  rb_define_method(cVADSegments, \"length\", ruby_whisper_vad_segments_get_length, 0);\n  rb_include_module(cVADSegments, rb_path2class(\"Enumerable\"));\n}\n"
  },
  {
    "path": "bindings/ruby/ext/sources/CMakeGraphVizOptions.cmake",
    "content": "set(GRAPHVIZ_EXECUTABLES FALSE)\nset(GRAPHVIZ_STATIC_LIBS TRUE)\nset(GRAPHVIZ_SHARED_LIBS FALSE)\nset(GRAPHVIZ_MODULE_LIBS FALSE)\nset(GRAPHVIZ_INTERFACE_LIBS FALSE)\nset(GRAPHVIZ_OBJECT_LIBS FALSE)\nset(GRAPHVIZ_UNKNOWN_LIBS FALSE)\nset(GRAPHVIZ_GENERATE_DEPENDERS FALSE)\n"
  },
  {
    "path": "bindings/ruby/extsources.rb",
    "content": "require \"pathname\"\n\nroot = Pathname(\"..\")/\"..\"\nignored_dirs = %w[\n  .devops\n  .github\n  ci\n  examples/wchess/wchess.wasm\n  examples/whisper.android\n  examples/whisper.android.java\n  examples/whisper.objc\n  examples/whisper.swiftui\n  grammars\n  models\n  samples\n  scripts\n].collect {|dir| root/dir}\nignored_files = %w[\n  AUTHORS\n  Makefile\n  README.md\n  README_sycl.md\n  .gitignore\n  .gitmodules\n  .dockerignore\n  whisper.nvim\n  twitch.sh\n  yt-wsp.sh\n  close-issue.yml\n  build-xcframework.sh\n]\n\nEXTSOURCES =\n  `git ls-files -z #{root}`.split(\"\\x0\")\n    .collect {|file| Pathname(file)}\n    .reject {|file|\n      ignored_dirs.any? {|dir| file.descend.any? {|desc| desc == dir}} ||\n        ignored_files.include?(file.basename.to_path) ||\n        (file.descend.to_a[1] != root && file.descend.to_a[1] != Pathname(\"..\")/\"javascript\")\n    }\n    .collect(&:to_path)\n"
  },
  {
    "path": "bindings/ruby/lib/whisper/context.rb",
    "content": "module Whisper\n  class Context\n    def to_srt\n      each_segment.with_index.reduce(\"\") {|srt, (segment, index)|\n        srt << \"#{index + 1}\\n#{segment.to_srt_cue}\\n\"\n      }\n    end\n\n    def to_webvtt\n      each_segment.with_index.reduce(\"WEBVTT\\n\\n\") {|webvtt, (segment, index)|\n        webvtt << \"#{index + 1}\\n#{segment.to_webvtt_cue}\\n\"\n      }\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/lib/whisper/model/uri.rb",
    "content": "require \"uri\"\nrequire \"net/http\"\nrequire \"time\"\nrequire \"pathname\"\nrequire \"io/console/size\"\n\nmodule Whisper\n  class Model\n    class URI\n      def initialize(uri)\n        @uri = URI(uri)\n      end\n\n      def to_path\n        cache\n        cache_path.to_path\n      end\n\n      def clear_cache\n        path = cache_path\n        path.delete if path.exist?\n      end\n\n      private\n\n      def cache_path\n        base_cache_dir/@uri.host/@uri.path[1..]\n      end\n\n      def base_cache_dir\n        base = case RUBY_PLATFORM\n               when /mswin|mingw/\n                 ENV.key?(\"LOCALAPPDATA\") ? Pathname(ENV[\"LOCALAPPDATA\"]) : Pathname(Dir.home)/\"AppData/Local\"\n               when /darwin/\n                 Pathname(Dir.home)/\"Library/Caches\"\n               else\n                 ENV.key?(\"XDG_CACHE_HOME\") ? Pathname(ENV[\"XDG_CACHE_HOME\"]) : Pathname(Dir.home)/\".cache\"\n               end\n        base/\"whisper.cpp\"\n      end\n\n      def cache\n        path = cache_path\n        headers = {}\n        headers[\"if-modified-since\"] = path.mtime.httpdate if path.exist?\n        request @uri, headers\n        path\n      end\n\n      def request(uri, headers)\n        Net::HTTP.start uri.host, uri.port, use_ssl: uri.scheme == \"https\" do |http|\n          request = Net::HTTP::Get.new(uri, headers)\n          http.request request do |response|\n            case response\n            when Net::HTTPNotModified\n              # noop\n            when Net::HTTPOK\n              return if !response.key?(\"last-modified\") && cache_path.exist?\n\n              download response\n            when Net::HTTPRedirection\n              request URI(response[\"location\"]), headers\n            else\n              return if headers.key?(\"if-modified-since\") # Use cache file\n\n              raise \"#{response.code} #{response.message}\\n#{response.body}\"\n            end\n          end\n        end\n      rescue => err\n        if cache_path.exist?\n          warn err\n          # Use cache file\n        else\n          raise\n        end\n      end\n\n      def download(response)\n        path = cache_path\n        path.dirname.mkpath unless path.dirname.exist?\n        downloading_path = Pathname(\"#{path}.downloading\")\n        size = response.content_length\n        downloading_path.open \"wb\" do |file|\n          downloaded = 0\n          response.read_body do |chunk|\n            file << chunk\n            downloaded += chunk.bytesize\n            show_progress downloaded, size\n          end\n          $stderr.puts\n        end\n        downloading_path.rename path\n      end\n\n      def show_progress(current, size)\n        line_size = 47\n        progress_rate_available = size && $stderr.tty? && $stderr.winsize[1] >= line_size\n\n        unless @prev\n          @prev = Time.now\n          $stderr.puts \"Downloading #{@uri} to #{cache_path}\"\n        end\n\n        now = Time.now\n\n        if progress_rate_available\n          return if now - @prev < 1 && current < size\n\n          progress_width = 20\n          progress = current.to_f / size\n          arrow_length = progress * progress_width\n          arrow = \"=\" * (arrow_length - 1) + \">\" + \" \" * (progress_width - arrow_length)\n          line = \"[#{arrow}] (#{format_bytesize(current)} / #{format_bytesize(size)})\"\n          padding = ' ' * ($stderr.winsize[1] - line.size)\n          $stderr.print \"\\r#{line}#{padding}\"\n        else\n          return if now - @prev < 1\n\n          $stderr.print \".\"\n        end\n        @prev = now\n      end\n\n      def format_bytesize(bytesize)\n        return \"0.0 B\" if bytesize.zero?\n\n        units = %w[B KiB MiB GiB TiB]\n        exp = (Math.log(bytesize) / Math.log(1024)).to_i\n        format(\"%.1f %s\", bytesize.to_f / 1024 ** exp, units[exp])\n      end\n    end\n\n    class ZipURI < URI\n      def cache\n        zip_path = super\n        dest = unzipped_path\n        return if dest.exist? && dest.mtime >= zip_path.mtime\n        escaping dest do\n          system \"unzip\", \"-q\", \"-d\", zip_path.dirname.to_path, zip_path.to_path, exception: true\n        end\n        zip_path\n      end\n\n      def clear_cache\n        super\n        unzipped_path.rmtree if unzipped_path.exist?\n      end\n\n      private\n\n      def unzipped_path\n        cache_path.sub_ext(\"\")\n      end\n\n      def escaping(path)\n        escaped = Pathname(\"#{path}.removing\")\n        if path.exist?\n          escaped.rmtree if escaped.exist?\n          path.rename escaped\n        end\n        yield\n      ensure\n        if path.exist?\n          escaped.rmtree if escaped.exist?\n        else\n          escaped.rename path if escaped.exist?\n        end\n      end\n    end\n\n    @pre_converted_models = %w[\n      tiny\n      tiny.en\n      tiny-q5_1\n      tiny.en-q5_1\n      tiny-q8_0\n      base\n      base.en\n      base-q5_1\n      base.en-q5_1\n      base-q8_0\n      small\n      small.en\n      small-q5_1\n      small.en-q5_1\n      small-q8_0\n      medium\n      medium.en\n      medium-q5_0\n      medium.en-q5_0\n      medium-q8_0\n      large-v1\n      large-v2\n      large-v2-q5_0\n      large-v2-q8_0\n      large-v3\n      large-v3-q5_0\n      large-v3-turbo\n      large-v3-turbo-q5_0\n      large-v3-turbo-q8_0\n    ].each_with_object({}) {|name, models|\n      models[name] = URI.new(\"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}.bin\")\n    }\n\n    %w[\n      small.en-tdrz\n    ].each do |name|\n      @pre_converted_models[name] = URI.new(\"https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-#{name}.bin\")\n    end\n\n    %w[\n      silero-v5.1.2\n      silero-v6.2.0\n    ].each do |name|\n      @pre_converted_models[name] = URI.new(\"https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-#{name}.bin\")\n    end\n\n    @coreml_compiled_models = @pre_converted_models.each_with_object({}) {|(name, uri), models|\n      next if name.end_with?(\"-tdrz\") || name.start_with?(\"silero-\")\n\n      if matched = name.match(/\\A(?<name>.*)-q\\d_\\d\\z/)\n        name = matched[:name]\n      end\n      models[uri] = ZipURI.new(\"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-#{name}-encoder.mlmodelc.zip\")\n    }\n\n    class << self\n      attr_reader :pre_converted_models, :coreml_compiled_models\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/lib/whisper/segment.rb",
    "content": "module Whisper\n  class Segment\n    SRT_ESCAPES = {\n      \"&\" => \"&amp;\",\n      \"<\" => \"&lt;\",\n      \">\" => \"&gt;\",\n    }\n    SRT_ESCAPES_RE = Regexp.union(SRT_ESCAPES.keys)\n    private_constant :SRT_ESCAPES, :SRT_ESCAPES_RE\n\n    def to_srt_cue\n      \"#{srt_start_time} --> #{srt_end_time}\\n#{srt_text}\\n\"\n    end\n\n    def to_webvtt_cue\n      \"#{webvtt_start_time} --> #{webvtt_end_time}\\n#{webvtt_text}\\n\"\n    end\n\n    private\n\n    def time_to_a(time)\n      sec, decimal_part = time.divmod(1000)\n      min, sec = sec.divmod(60)\n      hour, min = min.divmod(60)\n      [hour, min, sec, decimal_part]\n    end\n\n    def srt_time(time)\n      \"%02d:%02d:%02d,%03d\" % time_to_a(time)\n    end\n\n    def srt_start_time\n      srt_time(start_time)\n    end\n\n    def srt_end_time\n      srt_time(end_time)\n    end\n\n    def srt_text\n      text.gsub(SRT_ESCAPES_RE, SRT_ESCAPES)\n    end\n\n    def webvtt_time(time)\n      \"%02d:%02d:%02d.%03d\" % time_to_a(time)\n    end\n\n    def webvtt_start_time\n      webvtt_time(start_time)\n    end\n\n    def webvtt_end_time\n      webvtt_time(end_time)\n    end\n\n    alias webvtt_text srt_text\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/sig/whisper.rbs",
    "content": "module Whisper\n  interface _Samples\n    def length: () -> Integer\n    def each: { (Float) -> void } -> void\n  end\n\n  type log_callback = ^(Integer level, String message, Object user_data) -> void\n  type new_segment_callback = ^(Whisper::Context, void, Integer n_new, Object user_data) -> void\n  type progress_callback = ^(Whisper::Context, void, Integer progress, Object user_data) -> void\n  type encoder_begin_callback = ^(Whisper::Context, void, Object user_data) -> void\n  type abort_callback = ^(Whisper::Context, void, Object user_data) -> boolish\n\n  VERSION: String\n  LOG_LEVEL_NONE: Integer\n  LOG_LEVEL_INFO: Integer\n  LOG_LEVEL_WARN: Integer\n  LOG_LEVEL_ERROR: Integer\n  LOG_LEVEL_DEBUG: Integer\n  LOG_LEVEL_CONT: Integer\n  AHEADS_NONE: Integer\n  AHEADS_N_TOP_MOST: Integer\n  AHEADS_CUSTOM: Integer\n  AHEADS_TINY_EN: Integer\n  AHEADS_TINY: Integer\n  AHEADS_BASE_EN: Integer\n  AHEADS_BASE: Integer\n  AHEADS_SMALL_EN: Integer\n  AHEADS_SMALL: Integer\n  AHEADS_MEDIUM_EN: Integer\n  AHEADS_MEDIUM: Integer\n  AHEADS_LARGE_V1: Integer\n  AHEADS_LARGE_V2: Integer\n  AHEADS_LARGE_V3: Integer\n  AHEADS_LARGE_V3_TURBO: Integer\n\n  def self.lang_max_id: () -> Integer\n  def self.lang_id: (string name) -> Integer\n  def self.lang_str: (Integer id) -> String\n  def self.lang_str_full: (Integer id) -> String\n  def self.log_set: (log_callback, Object? user_data) -> log_callback\n  def self.system_info_str: () -> String\n\n  class Context\n    def self.new: (String | path | ::URI::HTTP) -> instance\n\n    # transcribe a single file\n    # can emit to a block results\n    #\n    #     params = Whisper::Params.new\n    #     params.duration = 60_000\n    #     whisper.transcribe \"path/to/audio.wav\", params do |text|\n    #       puts text\n    #     end\n    #\n    def transcribe: (path, Params, ?n_processors: Integer) -> self\n                  | (path, Params, ?n_processors: Integer) { (String) -> void } -> self\n\n    def model_n_vocab: () -> Integer\n    def model_n_audio_ctx: () -> Integer\n    def model_n_audio_state: () -> Integer\n    def model_n_text_head: () -> Integer\n    def model_n_text_layer: () -> Integer\n    def model_n_mels: () -> Integer\n    def model_ftype: () -> Integer\n    def model_type: () -> String\n\n    # Yields each Whisper::Segment:\n    #\n    #     whisper.transcribe(\"path/to/audio.wav\", params)\n    #     whisper.each_segment do |segment|\n    #       puts segment.text\n    #     end\n    #\n    # Returns an Enumerator if no block given:\n    #\n    #     whisper.transcribe(\"path/to/audio.wav\", params)\n    #     enum = whisper.each_segment\n    #     enum.to_a # => [#<Whisper::Segment>, ...]\n    #\n    def each_segment: { (Segment) -> void } -> void\n                    | () -> Enumerator[Segment]\n\n    def model: () -> Model\n    def full_get_segment: (Integer nth) -> Segment\n    def full_n_segments: () -> Integer\n\n    # Language ID, which can be converted to string by Whisper.lang_str and Whisper.lang_str_full.\n    #\n    def full_lang_id: () -> Integer\n\n    # Start time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).\n    #\n    #     full_get_segment_t0(3) # => 1668 (16680 ms)\n    #\n    def full_get_segment_t0: (Integer) -> Integer\n\n    # End time of a segment indexed by +segment_index+ in centiseconds (10 times milliseconds).\n    #\n    #     full_get_segment_t1(3) # => 1668 (16680 ms)\n    #\n    def full_get_segment_t1: (Integer) -> Integer\n\n    # Whether the next segment indexed by +segment_index+ is predicated as a speaker turn.\n    #\n    #     full_get_segment_speacker_turn_next(3) # => true\n    #\n    def full_get_segment_speaker_turn_next: (Integer) -> (true | false)\n\n    # Text of a segment indexed by +segment_index+.\n    #\n    #     full_get_segment_text(3) # => \"ask not what your country can do for you, ...\"\n    #\n    def full_get_segment_text: (Integer) -> String\n\n    def full_get_segment_no_speech_prob: (Integer) -> Float\n\n    # Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n    # Not thread safe for same context\n    # Uses the specified decoding strategy to obtain the text.\n    #\n    # The second argument +samples+ must be an array of samples, respond to :length, or be a MemoryView of an array of float. It must be 32 bit float PCM audio data.\n    #\n    def full: (Params, Array[Float] samples, ?Integer n_samples) -> self\n            | (Params, _Samples, ?Integer n_samples) -> self\n\n    # Split the input audio in chunks and process each chunk separately using whisper_full_with_state()\n    # Result is stored in the default state of the context\n    # Not thread safe if executed in parallel on the same context.\n    # It seems this approach can offer some speedup in some cases.\n    # However, the transcription accuracy can be worse at the beginning and end of each chunk.\n    #\n    def full_parallel: (Params, Array[Float], ?Integer n_samples) -> self\n                     | (Params, _Samples, ?Integer n_samples) -> self\n                     | (Params, _Samples, ?Integer? n_samples, Integer n_processors) -> self\n\n    def to_srt: () -> String\n    def to_webvtt: () -> String\n\n    class Params\n      def self.new: (\n        use_gpu: boolish,\n        flash_attn: boolish,\n        gpu_device: Integer,\n        dtw_token_timestamps: boolish,\n        dtw_aheads_preset: Integer,\n        dtw_n_top: Integer | nil,\n      ) -> instance\n\n      def use_gpu=: (boolish) -> boolish\n      def use_gpu: () -> (true | false)\n      def flash_attn=: (boolish) -> boolish\n      def flash_attn: () -> (true | false)\n      def gpu_device=: (Integer) -> Integer\n      def gpu_device: () -> Integer\n      def dtw_token_timestamps=: (boolish) -> boolish\n      def dtw_token_timestamps: () -> (true | false)\n      def dtw_aheads_preset=: (Integer) -> Integer\n      def dtw_aheads_preset: () -> Integer\n      def dtw_n_top=: (Integer | nil) -> (Integer | nil)\n      def dtw_n_top: () -> (Integer | nil)\n    end\n  end\n\n  class Params\n    def self.new: (\n      ?language: string,\n      ?translate: boolish,\n      ?no_context: boolish,\n      ?single_segment: boolish,\n      ?print_special: boolish,\n      ?print_progress: boolish,\n      ?print_realtime: boolish,\n      ?print_timestamps: boolish,\n      ?suppress_blank: boolish,\n      ?suppress_nst: boolish,\n      ?token_timestamps: boolish,\n      ?max_len: Integer,\n      ?split_on_word: boolish,\n      ?initial_prompt: string | nil,\n      ?carry_initial_prompt: boolish,\n      ?diarize: boolish,\n      ?offset: Integer,\n      ?duration: Integer,\n      ?max_text_tokens: Integer,\n      ?temperature: Float,\n      ?max_initial_ts: Float,\n      ?length_penalty: Float,\n      ?temperature_inc: Float,\n      ?entropy_thold: Float,\n      ?logprob_thold: Float,\n      ?no_speech_thold: Float,\n      ?new_segment_callback: new_segment_callback,\n      ?new_segment_callback_user_data: Object,\n      ?progress_callback: progress_callback,\n      ?progress_callback_user_data: Object,\n      ?encoder_begin_callback: encoder_begin_callback,\n      ?encoder_begin_callback_user_data: Object,\n      ?abort_callback: abort_callback,\n      ?abort_callback_user_data: Object,\n      ?vad: boolish,\n      ?vad_model_path: path | URI,\n      ?vad_params: Whisper::VAD::Params\n    ) -> instance\n\n    # params.language = \"auto\" | \"en\", etc...\n    #\n    def language=: (String) -> String # TODO: Enumerate lang names\n\n    def language: () -> String\n    def translate=: (boolish) -> boolish\n    def translate: () -> (true | false)\n    def no_context=: (boolish) -> boolish\n\n    # If true, does not use past transcription (if any) as initial prompt for the decoder.\n    #\n    def no_context: () -> (true | false)\n\n    def single_segment=: (boolish) -> boolish\n\n    # If true, forces single segment output (useful for streaming).\n    #\n    def single_segment: () -> (true | false)\n\n    def print_special=: (boolish) -> boolish\n\n    # If true, prints special tokens (e.g. <SOT>, <EOT>, <BEG>, etc.).\n    #\n    def print_special: () -> (true | false)\n\n    def print_progress=: (boolish) -> boolish\n\n    # If true, prints progress information.\n    #\n    def print_progress: () -> (true | false)\n\n    def print_realtime=: (boolish) -> boolish\n\n    # If true, prints results from within whisper.cpp. (avoid it, use callback instead)\n    #\n    def print_realtime: () -> (true | false)\n\n    # If true, prints timestamps for each text segment when printing realtime.\n    #\n    def print_timestamps=: (boolish) -> boolish\n\n    def print_timestamps: () -> (true | false)\n\n    def suppress_blank=: (boolish) -> boolish\n\n    # If true, suppresses blank outputs.\n    #\n    def suppress_blank: () -> (true | false)\n\n    def suppress_nst=: (boolish) -> boolish\n\n    # If true, suppresses non-speech-tokens.\n    #\n    def suppress_nst: () -> (true | false)\n\n    def token_timestamps=: (boolish) -> boolish\n\n    # If true, enables token-level timestamps.\n    #\n    def token_timestamps: () -> (true | false)\n\n    def max_len=: (Integer) -> Integer\n\n    # max segment length in characters.\n    #\n    def max_len: () -> Integer\n\n    def split_on_word=: (boolish) -> boolish\n\n    # If true, split on word rather than on token (when used with max_len).\n    #\n    def split_on_word: () -> (true | false)\n\n    def initial_prompt=: (_ToS) -> _ToS\n    def carry_initial_prompt=: (boolish) -> boolish\n\n    # Tokens to provide to the whisper decoder as initial prompt\n    # these are prepended to any existing text context from a previous call\n    # use whisper_tokenize() to convert text to tokens.\n    # Maximum of whisper_n_text_ctx()/2 tokens are used (typically 224).\n    #\n    def initial_prompt: () -> (String | nil)\n    def carry_initial_prompt: () -> (true | false)\n\n    def diarize=: (boolish) -> boolish\n\n    # If true, enables diarization.\n    #\n    def diarize: () -> (true | false)\n\n    def offset=: (Integer) -> Integer\n\n    # Start offset in ms.\n    #\n    def offset: () -> Integer\n\n    def duration=: (Integer) -> Integer\n\n    # Audio duration to process in ms.\n    #\n    def duration: () -> Integer\n\n    def max_text_tokens=: (Integer) -> Integer\n\n    # Max tokens to use from past text as prompt for the decoder.\n    #\n    def max_text_tokens: () -> Integer\n\n    def temperature=: (Float) -> Float\n    def temperature: () -> Float\n    def max_initial_ts=: (Float) -> Float\n\n    # See https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L97\n    #\n    def max_initial_ts: () -> Float\n\n    def length_penalty=: (Float) -> Float\n    def length_penalty: () -> Float\n    def temperature_inc=: (Float) -> Float\n    def temperature_inc: () -> Float\n    def entropy_thold=: (Float) -> Float\n\n    # Similar to OpenAI's \"compression_ratio_threshold\"\n    #\n    def entropy_thold: () -> Float\n\n    def logprob_thold=: (Float) -> Float\n    def logprob_thold: () -> Float\n    def no_speech_thold=: (Float) -> Float\n    def no_speech_thold: () -> Float\n\n    # Sets new segment callback, called for every newly generated text segment.\n    #\n    #     params.new_segment_callback = ->(context, _, n_new, user_data) {\n    #       # ...\n    #     }\n    #\n    def new_segment_callback=: (new_segment_callback) -> new_segment_callback\n    def new_segment_callback: () -> (new_segment_callback | nil)\n\n    # Sets user data passed to the last argument of new segment callback.\n    #\n    def new_segment_callback_user_data=: (Object) -> Object\n\n    def new_segment_callback_user_data: () -> Object\n\n    # Sets progress callback, called on each progress update.\n    #\n    #     params.new_segment_callback = ->(context, _, progress, user_data) {\n    #       # ...\n    #     }\n    #\n    # +progress+ is an Integer between 0 and 100.\n    #\n    def progress_callback=: (progress_callback) -> progress_callback\n\n    def progress_callback: () -> (progress_callback | nil)\n\n    # Sets user data passed to the last argument of progress callback.\n    #\n    def progress_callback_user_data=: (Object) -> Object\n\n    def progress_callback_user_data: () -> Object\n\n    # Sets encoder begin callback, called when the encoder starts.\n    #\n    def encoder_begin_callback=: (encoder_begin_callback) -> encoder_begin_callback\n\n    def encoder_begin_callback: () -> (encoder_begin_callback | nil)\n\n    # Sets user data passed to the last argument of encoder begin callback.\n    #\n    def encoder_begin_callback_user_data=: (Object) -> Object\n\n    def encoder_begin_callback_user_data: () -> Object\n\n    # Sets abort callback, called to check if the process should be aborted.\n    #\n    #     params.abort_callback = ->(user_data) {\n    #       # ...\n    #     }\n    #\n    #\n    def abort_callback=: (abort_callback) -> abort_callback\n\n    def abort_callback: () -> (abort_callback | nil)\n\n    # Sets user data passed to the last argument of abort callback.\n    #\n    def abort_callback_user_data=: (Object) -> Object\n\n    def abort_callback_user_data: () -> Object\n\n    # Enable VAD\n    #\n    def vad=: (boolish) -> boolish\n\n    def vad: () -> (true | false)\n\n    # Path to the VAD model\n    def vad_model_path=: (path | URI | nil) -> (path | URI | nil)\n\n    def vad_model_path: () -> (String | nil)\n\n    def vad_params=: (Whisper::VAD::Params) -> Whisper::VAD::Params\n    def vad_params: () -> (Whisper::VAD::Params)\n\n    # Hook called on new segment. Yields each Whisper::Segment.\n    #\n    #     whisper.on_new_segment do |segment|\n    #       # ...\n    #     end\n    #\n    def on_new_segment: { (Segment) -> void } -> void\n\n    # Hook called on progress update. Yields each progress Integer between 0 and 100.\n    #\n    def on_progress: { (Integer progress) -> void } -> void\n\n    # Hook called on encoder starts.\n    #\n    def on_encoder_begin: { () -> void } -> void\n\n    # Call block to determine whether abort or not. Return +true+ when you want to abort.\n    #\n    #     params.abort_on do\n    #       if some_condition\n    #         true # abort\n    #       else\n    #         false # continue\n    #       end\n    #     end\n    #\n    def abort_on: { (Object user_data) -> boolish } -> void\n  end\n\n  class Model\n    def self.pre_converted_models: () -> Hash[String, Model::URI]\n    def self.coreml_compiled_models: () -> Hash[Model::URI, Model::ZipURI]\n    def self.new: () -> instance\n    def n_vocab: () -> Integer\n    def n_audio_ctx: () -> Integer\n    def n_audio_state: () -> Integer\n    def n_audio_head: () -> Integer\n    def n_audio_layer: () -> Integer\n    def n_text_ctx: () -> Integer\n    def n_text_state: () -> Integer\n    def n_text_head: () -> Integer\n    def n_text_layer: () -> Integer\n    def n_mels: () -> Integer\n    def ftype: () -> Integer\n    def type: () -> String\n\n    class URI\n      def self.new: (string | ::URI::HTTP) -> instance\n      def to_path: -> String\n      def clear_cache: -> void\n    end\n\n    class ZipURI < URI\n      def cache: () -> Pathname\n      def clear_cache: () -> void\n    end\n  end\n\n  class Segment\n    type deconstructed_keys = {\n      start_time: (Integer | nil),\n      end_time: (Integer | nil),\n      text: (String | nil),\n      no_speech_prob: (Float | nil),\n      speaker_turn_next: (true | false | nil),\n      n_tokens: (Integer | nil)\n    }\n\n    # Start time in milliseconds.\n    #\n    def start_time: () -> Integer\n\n    # End time in milliseconds.\n    #\n    def end_time: () -> Integer\n\n    # Whether the next segment is predicted as a speaker turn.\n    #\n    def speaker_turn_next?: () -> (true | false)\n\n    def text: () -> String\n    def no_speech_prob: () -> Float\n\n    # Get number of tokens in the segment\n    #\n    def n_tokens: () -> Integer\n\n    # Yields each Whisper::Token:\n    #\n    #   whisper.each_segment.first.each_token do |token|\n    #     p token\n    #   end\n    #\n    # Returns an Enumerator if no block is given:\n    #\n    #   whisper.each_segment.first.each_token.to_a # => [#<Whisper::Token>, ...]\n    #\n    def each_token: { (Token) -> void } -> void\n                  | () -> Enumerator[Token]\n    def to_srt_cue: () -> String\n    def to_webvtt_cue: () -> String\n\n\n    #  Possible keys: :start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next\n    #\n    #      whisper.each_segment do |segment|\n    #        segment => {start_time:, end_time:, text:, no_speech_prob:, speaker_turn_next:}\n    #\n    #        puts \"[#{start_time} --> #{end_time}] #{text} (no speech prob: #{no_speech_prob}#{speaker_turn_next ? ', speaker turns next' : ''})\"\n    #      end\n    def deconstruct_keys: (Array[:start_time | :end_time | :text | :no_speech_prob | :speaker_turn_next | :n_tokens] | nil) -> deconstructed_keys\n  end\n\n  module Token\n    type deconstructed_keys = {\n      id: (Integer | nil),\n      tid: (Integer | nil),\n      probability: (Float | nil),\n      log_probability: (Float | nil),\n      pt: (Float | nil),\n      ptsum: (Float | nil),\n      t_dtw: (Integer | nil),\n      voice_length: (Float | nil),\n      text: (String | nil),\n      start_time: (Integer | nil),\n      end_time: (Integer | nil),\n    }\n\n    # Token ID.\n    #\n    def id: () -> Integer\n\n    # Forced timestamp token ID.\n    #\n    def tid: () -> Integer\n\n    # Probability of the token.\n    #\n    def probability: () -> Float\n\n    # Log probability of the token.\n    #\n    def log_probability: () -> Float\n\n    # Probability of the timestamp token.\n    #\n    def pt: () -> Float\n\n    # Sum of probability of all timestamp tokens.\n    #\n    def ptsum: () -> Float\n\n    # [EXPERIMENTAL] Token-level timestamps with DTW\n    #\n    # Do not use if you haven't computed token-level timestamps with dtw.\n    # Roughly corresponds to the moment in audio in which the token was output.\n    #\n    def t_dtw: () -> Integer\n\n    # Voice length of the token.\n    #\n    def voice_length: () -> Float\n\n    # Start time of the token.\n    #\n    # Token-level timestamp data.\n    # Do not use if you haven't computed token-level timestamps.\n    #\n    def start_time: () -> Integer\n\n    # End time of the token.\n    #\n    # Token-level timestamp data.\n    # Do not use if you haven't computed token-level timestamps.\n    #\n    def end_time: () -> Integer\n\n    # Get the token text of the token.\n    #\n    def text: () -> String\n    def deconstruct_keys: (Array[:id | :tid | :probability | :log_probability | :pt | :ptsum | :t_dtw | :voice_length | :start_time | :end_time | :text] | nil) -> deconstructed_keys\n  end\n\n  module VAD\n    class Params\n      def self.new: (\n        ?threshold: Float,\n        ?min_speech_duration_ms: Integer,\n        ?min_silence_duration_ms: Integer,\n        ?max_speech_duration_s: Float,\n        ?speech_pad_ms: Integer,\n        ?samples_overlap: Float\n      ) -> instance\n\n      # Probability threshold to consider as speech.\n      #\n      def threshold=: (Float) -> Float\n\n      def threshold: () -> Float\n\n      # Min duration for a valid speech segment.\n      #\n      def min_speech_duration_ms=: (Integer) -> Integer\n\n      def min_speech_duration_ms: () -> Integer\n\n      # Min silence duration to consider speech as ended.\n      #\n      def min_silence_duration_ms=: (Integer) -> Integer\n\n      def min_silence_duration_ms: () -> Integer\n\n      # Max duration of a speech segment before forcing a new segment.\n      def max_speech_duration_s=: (Float) -> Float\n\n      def max_speech_duration_s: () -> Float\n\n      # Padding added before and after speech segments.\n      #\n      def speech_pad_ms=: (Integer) -> Integer\n\n      def speech_pad_ms: () -> Integer\n\n      # Overlap in seconds when copying audio samples from speech segment.\n      #\n      def samples_overlap=: (Float) -> Float\n\n      def samples_overlap: () -> Float\n      def ==: (Params) -> (true | false)\n    end\n\n    class Context\n      def self.new: (String | path | ::URI::HTTP model_name_or_path) -> instance\n      def segments_from_samples: (Params, Array[Float] samples, ?Integer n_samples) -> Segments\n                               | (Params, _Samples, ?Integer n_samples) -> Segments\n      def detect: (path wav_file_path, Params) -> Segments\n    end\n\n    class Segments\n      include Enumerable[Segment]\n\n      def each: { (Segment) -> void } -> void\n              | () -> Enumerator[Segment]\n      def length: -> Integer\n    end\n\n    class Segment\n      type deconstructed_keys = {\n        start_time: (Integer | nil),\n        end_time: (Integer | nil),\n      }\n\n      def start_time: () -> Integer\n      def end_time: () -> Integer\n      def deconstruct_keys: (Array[:start_time | :end_time] | nil) -> deconstructed_keys\n    end\n  end\n\n  class Error < StandardError\n    attr_reader code: Integer\n\n    def self.new: (Integer code) -> instance\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/helper.rb",
    "content": "require \"test/unit\"\nrequire \"whisper\"\nrequire_relative \"jfk_reader/jfk_reader\"\n\nclass TestBase < Test::Unit::TestCase\n  AUDIO = File.join(__dir__, \"fixtures\", \"jfk.wav\")\n\n  class << self\n    def whisper\n      return @whisper if @whisper\n\n      @whisper = Whisper::Context.new(\"base.en\")\n      params = Whisper::Params.new\n      params.print_timestamps = false\n      @whisper.transcribe(TestBase::AUDIO, params)\n    end\n  end\n\n  private\n\n  def whisper\n    self.class.whisper\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/jfk_reader/.gitignore",
    "content": "Makefile\njfk_reader.o\njfk_reader.so\njfk_reader.bundle\njfk_reader.dll\n"
  },
  {
    "path": "bindings/ruby/test/jfk_reader/extconf.rb",
    "content": "require \"mkmf\"\n\ncreate_makefile(\"jfk_reader\")\n"
  },
  {
    "path": "bindings/ruby/test/jfk_reader/jfk_reader.c",
    "content": "#include <ruby.h>\n#include <ruby/memory_view.h>\n#include <ruby/encoding.h>\n\nstatic VALUE\njfk_reader_initialize(VALUE self, VALUE audio_path)\n{\n  rb_iv_set(self, \"audio_path\", audio_path);\n  return Qnil;\n}\n\nstatic bool\njfk_reader_get_memory_view(const VALUE obj, rb_memory_view_t *view, int flags)\n{\n  VALUE audio_path = rb_iv_get(obj, \"audio_path\");\n  const char *audio_path_str = StringValueCStr(audio_path);\n  const int n_samples = 176000;\n  float *data = (float *)malloc(n_samples * sizeof(float));\n  short *samples = (short *)malloc(n_samples * sizeof(short));\n  FILE *file = fopen(audio_path_str, \"rb\");\n\n  fseek(file, 78, SEEK_SET);\n  fread(samples, sizeof(short), n_samples, file);\n  fclose(file);\n  for (int i = 0; i < n_samples; i++) {\n    data[i] = samples[i]/32768.0;\n  }\n\n  view->obj = obj;\n  view->data = (void *)data;\n  view->byte_size = sizeof(float) * n_samples;\n  view->readonly = true;\n  view->format = \"f\";\n  view->item_size = sizeof(float);\n  view->item_desc.components = NULL;\n  view->item_desc.length = 0;\n  view->ndim = 1;\n  view->shape = NULL;\n  view->sub_offsets = NULL;\n  view->private_data = NULL;\n\n  return true;\n}\n\nstatic bool\njfk_reader_release_memory_view(const VALUE obj, rb_memory_view_t *view)\n{\n  return true;\n}\n\nstatic bool\njfk_reader_memory_view_available_p(const VALUE obj)\n{\n  return true;\n}\n\nstatic const rb_memory_view_entry_t jfk_reader_view_entry = {\n  jfk_reader_get_memory_view,\n  jfk_reader_release_memory_view,\n  jfk_reader_memory_view_available_p\n};\n\nvoid Init_jfk_reader(void)\n{\n  VALUE cJFKReader = rb_define_class(\"JFKReader\", rb_cObject);\n  rb_memory_view_register(cJFKReader, &jfk_reader_view_entry);\n  rb_define_method(cJFKReader, \"initialize\", jfk_reader_initialize, 1);\n}\n"
  },
  {
    "path": "bindings/ruby/test/test_callback.rb",
    "content": "require_relative \"helper\"\n\nclass TestCallback < TestBase\n  def setup\n    GC.start\n    @params = Whisper::Params.new\n    @whisper = Whisper::Context.new(\"base.en\")\n    @audio = File.join(AUDIO)\n  end\n\n  def test_new_segment_callback\n    @params.new_segment_callback = ->(context, state, n_new, user_data) {\n      assert_kind_of Integer, n_new\n      assert n_new > 0\n      assert_same @whisper, context\n\n      n_segments = context.full_n_segments\n      n_new.times do |i|\n        i_segment = n_segments - 1 + i\n        start_time = context.full_get_segment_t0(i_segment) * 10\n        end_time = context.full_get_segment_t1(i_segment) * 10\n        text = context.full_get_segment_text(i_segment)\n\n        assert_kind_of Integer, start_time\n        assert start_time >= 0\n        assert_kind_of Integer, end_time\n        assert end_time > 0\n        assert_match(/ask not what your country can do for you, ask what you can do for your country/, text) if i_segment == 0\n      end\n    }\n\n    @whisper.transcribe(@audio, @params)\n  end\n\n  def test_new_segment_callback_closure\n    search_word = \"what\"\n    @params.new_segment_callback = ->(context, state, n_new, user_data) {\n      n_segments = context.full_n_segments\n      n_new.times do |i|\n        i_segment = n_segments - 1 + i\n        text = context.full_get_segment_text(i_segment)\n        if text.include?(search_word)\n          t0 = context.full_get_segment_t0(i_segment)\n          t1 = context.full_get_segment_t1(i_segment)\n          raise \"search word '#{search_word}' found at between #{t0} and #{t1}\"\n        end\n      end\n    }\n\n    assert_raise RuntimeError do\n      @whisper.transcribe(@audio, @params)\n    end\n  end\n\n  def test_new_segment_callback_user_data\n    udata = Object.new\n    @params.new_segment_callback_user_data = udata\n    @params.new_segment_callback = ->(context, state, n_new, user_data) {\n      assert_same udata, user_data\n    }\n\n    @whisper.transcribe(@audio, @params)\n  end\n\n  def test_new_segment_callback_user_data_gc\n    @params.new_segment_callback_user_data = \"My user data\"\n    @params.new_segment_callback = ->(context, state, n_new, user_data) {\n      assert_equal \"My user data\", user_data\n    }\n    GC.start\n\n    assert_same @whisper, @whisper.transcribe(@audio, @params)\n  end\n\n  def test_progress_callback\n    first = nil\n    last = nil\n    @params.progress_callback = ->(context, state, progress, user_data) {\n      assert_kind_of Integer, progress\n      assert 0 <= progress && progress <= 100\n      assert_same @whisper, context\n      first = progress if first.nil?\n      last = progress\n    }\n    @whisper.transcribe(@audio, @params)\n    assert_equal 0, first\n    assert_equal 100, last\n  end\n\n  def test_progress_callback_user_data\n    udata = Object.new\n    @params.progress_callback_user_data = udata\n    @params.progress_callback = ->(context, state, n_new, user_data) {\n      assert_same udata, user_data\n    }\n\n    @whisper.transcribe(@audio, @params)\n  end\n\n  def test_on_progress\n    first = nil\n    last = nil\n    @params.on_progress do |progress|\n      assert_kind_of Integer, progress\n      assert 0 <= progress && progress <= 100\n      first = progress if first.nil?\n      last = progress\n    end\n    @whisper.transcribe(@audio, @params)\n    assert_equal 0, first\n    assert_equal 100, last\n  end\n\n  def test_encoder_begin_callback\n    i = 0\n    @params.encoder_begin_callback = ->(context, state, user_data) {\n      i += 1\n    }\n    @whisper.transcribe(@audio, @params)\n    assert i > 0\n  end\n\n  def test_encoder_begin_callback_abort\n    logs = []\n    Whisper.log_set -> (level, buffer, user_data) {\n      logs << buffer if level == Whisper::LOG_LEVEL_ERROR\n    }, logs\n    @params.encoder_begin_callback = ->(context, state, user_data) {\n      return false\n    }\n    @whisper.transcribe(@audio, @params)\n    assert_match(/encoder_begin_callback returned false - aborting/, logs.join)\n    Whisper.log_set ->(level, buffer, user_data) {}, nil\n  end\n\n  def test_encoder_begin_callback_user_data\n    udata = Object.new\n    @params.encoder_begin_callback_user_data = udata\n    yielded = nil\n    @params.encoder_begin_callback = ->(context, state, user_data) {\n      yielded = user_data\n    }\n    @whisper.transcribe(@audio, @params)\n    assert_same udata, yielded\n  end\n\n  def test_on_encoder_begin\n    i = 0\n    @params.on_encoder_begin do\n      i += 1\n    end\n    @whisper.transcribe(@audio, @params)\n    assert i > 0\n  end\n\n  def test_abort_callback\n    i = 0\n    @params.abort_callback = ->(user_data) {\n      assert_nil user_data\n      i += 1\n      return false\n    }\n    @whisper.transcribe(@audio, @params)\n    assert i > 0\n  end\n\n  def test_abort_callback_abort\n    i = 0\n    @params.abort_callback = ->(user_data) {\n      i += 1\n      return i == 3\n    }\n    @whisper.transcribe(@audio, @params)\n    assert_equal 3, i\n  end\n\n  def test_abort_callback_user_data\n    udata = Object.new\n    @params.abort_callback_user_data = udata\n    yielded = nil\n    @params.abort_callback = ->(user_data) {\n      yielded = user_data\n    }\n    @whisper.transcribe(@audio, @params)\n    assert_same udata, yielded\n  end\n\n  def test_abort_on\n    do_abort = false\n    _aborted_from_callback = false\n    @params.on_new_segment do |segment|\n      do_abort = true if segment.text.match?(/ask/)\n    end\n    i = 0\n    @params.abort_on do\n      i += 1\n      do_abort\n    end\n    @whisper.transcribe(@audio, @params)\n    assert i > 0\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_context_params.rb",
    "content": "require_relative \"helper\"\n\nclass TestContextParams < TestBase\n  PARAM_NAMES = [\n    :use_gpu,\n    :flash_attn,\n    :gpu_device,\n    :dtw_token_timestamps,\n    :dtw_aheads_preset,\n    :dtw_n_top\n  ]\n\n  def test_new\n    params = Whisper::Context::Params.new\n    assert_instance_of Whisper::Context::Params, params\n  end\n\n  def test_attributes\n    params = Whisper::Context::Params.new\n\n    assert_true params.use_gpu\n    params.use_gpu = false\n    assert_false params.use_gpu\n\n    assert_true params.flash_attn\n    params.flash_attn = false\n    assert_false params.flash_attn\n\n    assert_equal 0, params.gpu_device\n    params.gpu_device = 1\n    assert_equal 1, params.gpu_device\n\n    assert_false params.dtw_token_timestamps\n    params.dtw_token_timestamps = true\n    assert_true params.dtw_token_timestamps\n\n    assert_equal Whisper::AHEADS_NONE, params.dtw_aheads_preset\n    params.dtw_aheads_preset =Whisper::AHEADS_BASE\n    assert_equal Whisper::AHEADS_BASE, params.dtw_aheads_preset\n\n    assert_nil params.dtw_n_top\n    params.dtw_n_top = 6\n    assert_equal 6, params.dtw_n_top\n    params.dtw_n_top = nil\n    assert_nil params.dtw_n_top\n  end\n\n  def test_new_with_kw_args\n    params = Whisper::Context::Params.new(use_gpu: false)\n    assert_false params.use_gpu\n  end\n\n  def test_new_with_kw_wargs_non_existent\n    assert_raise ArgumentError do\n      Whisper::Context::Params.new(non_existent: \"value\")\n    end\n  end\n\n  data(PARAM_NAMES.collect {|param| [param, param]}.to_h)\n  def test_new_with_kw_args_default_values(param)\n    default_params = Whisper::Context::Params.new\n    default_value = default_params.send(param)\n    value = if param == :dtw_n_top\n              6\n            else\n              case default_value\n              in true | false\n                !default_value\n              in Integer\n                default_value + 1\n              end\n            end\n    params = Whisper::Context::Params.new(param => value)\n    assert_equal value, params.send(param)\n\n    PARAM_NAMES.reject {|name| name == param}.each do |name|\n      expected = default_params.send(name)\n      actual = params.send(name)\n      assert_equal expected, actual\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_error.rb",
    "content": "require_relative \"helper\"\n\nclass TestError < TestBase\n  def test_error\n    error = Whisper::Error.new(-2)\n    assert_equal \"failed to compute log mel spectrogram\", error.message\n    assert_equal(-2, error.code)\n  end\n\n  def test_unknown_error\n    error = Whisper::Error.new(-20)\n    assert_equal \"unknown error\", error.message\n  end\n\n  def test_non_int_code\n    assert_raise TypeError do\n      _error = Whisper::Error.new(\"non int\")\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_model.rb",
    "content": "require_relative \"helper\"\nrequire \"pathname\"\n\nclass TestModel < TestBase\n  def test_model\n    whisper = Whisper::Context.new(\"base.en\")\n    assert_instance_of Whisper::Model, whisper.model\n  end\n\n  def test_attributes\n    whisper = Whisper::Context.new(\"base.en\")\n    model = whisper.model\n\n    assert_equal 51864, model.n_vocab\n    assert_equal 1500, model.n_audio_ctx\n    assert_equal 512, model.n_audio_state\n    assert_equal 8, model.n_audio_head\n    assert_equal 6, model.n_audio_layer\n    assert_equal 448, model.n_text_ctx\n    assert_equal 512, model.n_text_state\n    assert_equal 8, model.n_text_head\n    assert_equal 6, model.n_text_layer\n    assert_equal 80, model.n_mels\n    assert_equal 1, model.ftype\n    assert_equal \"base\", model.type\n  end\n\n  def test_gc\n    model = Whisper::Context.new(\"base.en\").model\n    GC.start\n\n    assert_equal 51864, model.n_vocab\n    assert_equal 1500, model.n_audio_ctx\n    assert_equal 512, model.n_audio_state\n    assert_equal 8, model.n_audio_head\n    assert_equal 6, model.n_audio_layer\n    assert_equal 448, model.n_text_ctx\n    assert_equal 512, model.n_text_state\n    assert_equal 8, model.n_text_head\n    assert_equal 6, model.n_text_layer\n    assert_equal 80, model.n_mels\n    assert_equal 1, model.ftype\n    assert_equal \"base\", model.type\n  end\n\n  def test_pathname\n    path = Pathname(Whisper::Model.pre_converted_models[\"base.en\"].to_path)\n    whisper = Whisper::Context.new(path)\n    model = whisper.model\n\n    assert_equal 51864, model.n_vocab\n    assert_equal 1500, model.n_audio_ctx\n    assert_equal 512, model.n_audio_state\n    assert_equal 8, model.n_audio_head\n    assert_equal 6, model.n_audio_layer\n    assert_equal 448, model.n_text_ctx\n    assert_equal 512, model.n_text_state\n    assert_equal 8, model.n_text_head\n    assert_equal 6, model.n_text_layer\n    assert_equal 80, model.n_mels\n    assert_equal 1, model.ftype\n    assert_equal \"base\", model.type\n  end\n\n  def test_auto_download\n    path = Whisper::Model.pre_converted_models[\"base.en\"].to_path\n\n    assert_path_exist path\n    assert_equal 147964211, File.size(path)\n  end\n\n  def test_uri_string\n    path = \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin\"\n    whisper = Whisper::Context.new(path)\n    model = whisper.model\n\n    assert_equal 51864, model.n_vocab\n    assert_equal 1500, model.n_audio_ctx\n    assert_equal 512, model.n_audio_state\n    assert_equal 8, model.n_audio_head\n    assert_equal 6, model.n_audio_layer\n    assert_equal 448, model.n_text_ctx\n    assert_equal 512, model.n_text_state\n    assert_equal 8, model.n_text_head\n    assert_equal 6, model.n_text_layer\n    assert_equal 80, model.n_mels\n    assert_equal 1, model.ftype\n    assert_equal \"base\", model.type\n  end\n\n  def test_uri\n    path = URI(\"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin\")\n    whisper = Whisper::Context.new(path)\n    model = whisper.model\n\n    assert_equal 51864, model.n_vocab\n    assert_equal 1500, model.n_audio_ctx\n    assert_equal 512, model.n_audio_state\n    assert_equal 8, model.n_audio_head\n    assert_equal 6, model.n_audio_layer\n    assert_equal 448, model.n_text_ctx\n    assert_equal 512, model.n_text_state\n    assert_equal 8, model.n_text_head\n    assert_equal 6, model.n_text_layer\n    assert_equal 80, model.n_mels\n    assert_equal 1, model.ftype\n    assert_equal \"base\", model.type\n  end\n\n  def test_coreml_model_auto_download\n    uri = Whisper::Model.coreml_compiled_models[Whisper::Model.pre_converted_models[\"tiny\"]]\n    model_path = Pathname(uri.to_path).sub_ext(\"\")\n    model_path.rmtree if model_path.exist?\n\n    uri.cache\n    assert_path_exist model_path\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_package.rb",
    "content": "require_relative \"helper\"\nrequire 'tempfile'\nrequire 'tmpdir'\nrequire 'shellwords'\n\nclass TestPackage < TestBase\n  def test_build\n    Tempfile.create do |file|\n      assert system(\"gem\", \"build\", \"whispercpp.gemspec\", \"--output\", file.to_path.shellescape, exception: true)\n      assert file.size > 0\n      assert_path_exist file.to_path\n    end\n  end\n\n  sub_test_case \"Building binary on installation\" do\n    def setup\n      system \"rake\", \"build\", exception: true\n    end\n\n    def test_install\n      gemspec = Gem::Specification.load(\"whispercpp.gemspec\")\n      Dir.mktmpdir do |dir|\n        system \"gem\", \"install\", \"--install-dir\", dir.shellescape, \"--no-document\", \"pkg/#{gemspec.file_name.shellescape}\", exception: true\n        assert_installed dir, gemspec.version\n      end\n    end\n\n    def test_install_with_coreml\n      omit_unless RUBY_PLATFORM.match?(/darwin/) do\n        gemspec = Gem::Specification.load(\"whispercpp.gemspec\")\n        Dir.mktmpdir do |dir|\n          system \"gem\", \"install\", \"--install-dir\", dir.shellescape, \"--no-document\", \"pkg/#{gemspec.file_name.shellescape}\", \"--\", \"--enable-whisper-coreml\", exception: true\n          assert_installed dir, gemspec.version\n          libdir = File.join(dir, \"gems\", \"#{gemspec.name}-#{gemspec.version}\", \"lib\")\n          assert_nothing_raised do\n            system \"ruby\", \"-I\", libdir, \"-r\", \"whisper\", \"-e\", \"Whisper::Context.new('tiny')\", exception: true\n          end\n          assert_match(/COREML = 1/, `ruby -I #{libdir.shellescape} -r whisper -e 'puts Whisper.system_info_str'`)\n        end\n      end\n    end\n\n    private\n\n    def assert_installed(dir, version)\n      assert_path_exist File.join(dir, \"gems/whispercpp-#{version}/lib\", \"whisper.#{RbConfig::CONFIG[\"DLEXT\"]}\")\n      assert_path_exist File.join(dir, \"gems/whispercpp-#{version}/LICENSE\")\n      assert_path_not_exist File.join(dir, \"gems/whispercpp-#{version}/ext/build\")\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_params.rb",
    "content": "require_relative \"helper\"\n\nclass TestParams < TestBase\n  PARAM_NAMES = [\n    :language,\n    :translate,\n    :no_context,\n    :single_segment,\n    :print_special,\n    :print_progress,\n    :print_realtime,\n    :print_timestamps,\n    :suppress_blank,\n    :suppress_nst,\n    :token_timestamps,\n    :max_len,\n    :split_on_word,\n    :initial_prompt,\n    :carry_initial_prompt,\n    :diarize,\n    :offset,\n    :duration,\n    :max_text_tokens,\n    :temperature,\n    :max_initial_ts,\n    :length_penalty,\n    :temperature_inc,\n    :entropy_thold,\n    :logprob_thold,\n    :no_speech_thold,\n    :new_segment_callback,\n    :new_segment_callback_user_data,\n    :progress_callback,\n    :progress_callback_user_data,\n    :abort_callback,\n    :abort_callback_user_data,\n    :vad,\n    :vad_model_path,\n    :vad_params,\n  ]\n\n  def setup\n    @params  = Whisper::Params.new\n  end\n\n  def test_language\n    @params.language = \"en\"\n    assert_equal @params.language, \"en\"\n    @params.language = \"auto\"\n    assert_equal @params.language, \"auto\"\n  end\n\n  def test_offset\n    @params.offset = 10_000\n    assert_equal @params.offset, 10_000\n    @params.offset = 0\n    assert_equal @params.offset, 0\n  end\n\n  def test_duration\n    @params.duration = 60_000\n    assert_equal @params.duration, 60_000\n    @params.duration = 0\n    assert_equal @params.duration, 0\n  end\n\n  def test_max_text_tokens\n    @params.max_text_tokens = 300\n    assert_equal @params.max_text_tokens, 300\n    @params.max_text_tokens = 0\n    assert_equal @params.max_text_tokens, 0\n  end\n\n  def test_translate\n    @params.translate = true\n    assert @params.translate\n    @params.translate = false\n    assert !@params.translate\n  end\n\n  def test_no_context\n    @params.no_context = true\n    assert @params.no_context\n    @params.no_context = false\n    assert !@params.no_context\n  end\n\n  def test_single_segment\n    @params.single_segment = true\n    assert @params.single_segment\n    @params.single_segment = false\n    assert !@params.single_segment\n  end\n\n  def test_print_special\n    @params.print_special = true\n    assert @params.print_special\n    @params.print_special = false\n    assert !@params.print_special\n  end\n\n  def test_print_progress\n    @params.print_progress = true\n    assert @params.print_progress\n    @params.print_progress = false\n    assert !@params.print_progress\n  end\n\n  def test_print_realtime\n    @params.print_realtime = true\n    assert @params.print_realtime\n    @params.print_realtime = false\n    assert !@params.print_realtime\n  end\n\n  def test_print_timestamps\n    @params.print_timestamps = true\n    assert @params.print_timestamps\n    @params.print_timestamps = false\n    assert !@params.print_timestamps\n  end\n\n  def test_carry_initial_prompt\n    @params.carry_initial_prompt = true\n    assert @params.carry_initial_prompt\n    @params.carry_initial_prompt = false\n    assert !@params.carry_initial_prompt\n  end\n\n  def test_suppress_blank\n    @params.suppress_blank = true\n    assert @params.suppress_blank\n    @params.suppress_blank = false\n    assert !@params.suppress_blank\n  end\n\n  def test_suppress_nst\n    @params.suppress_nst = true\n    assert @params.suppress_nst\n    @params.suppress_nst = false\n    assert !@params.suppress_nst\n  end\n\n  def test_token_timestamps\n    @params.token_timestamps = true\n    assert @params.token_timestamps\n    @params.token_timestamps = false\n    assert !@params.token_timestamps\n  end\n\n  def test_max_len\n    @params.max_len = 42\n    assert_equal @params.max_len, 42\n    @params.max_len = 0\n    assert_equal @params.max_len, 0\n  end\n\n  def test_split_on_word\n    @params.split_on_word = true\n    assert @params.split_on_word\n    @params.split_on_word = false\n    assert !@params.split_on_word\n  end\n\n  def test_initial_prompt\n    assert_nil @params.initial_prompt\n    @params.initial_prompt = \"You are a polite person.\"\n    assert_equal \"You are a polite person.\", @params.initial_prompt\n  end\n\n  def test_temperature\n    assert_equal 0.0, @params.temperature\n    @params.temperature = 0.5\n    assert_equal 0.5, @params.temperature\n  end\n\n  def test_max_initial_ts\n    assert_equal 1.0, @params.max_initial_ts\n    @params.max_initial_ts = 600.0\n    assert_equal 600.0, @params.max_initial_ts\n  end\n\n  def test_length_penalty\n    assert_equal(-1.0, @params.length_penalty)\n    @params.length_penalty = 0.5\n    assert_equal 0.5, @params.length_penalty\n  end\n\n  def test_temperature_inc\n    assert_in_delta 0.2, @params.temperature_inc\n    @params.temperature_inc = 0.5\n    assert_in_delta 0.5, @params.temperature_inc\n  end\n\n  def test_entropy_thold\n    assert_in_delta 2.4, @params.entropy_thold\n    @params.entropy_thold = 3.0\n    assert_in_delta 3.0, @params.entropy_thold\n  end\n\n  def test_logprob_thold\n    assert_in_delta(-1.0, @params.logprob_thold)\n    @params.logprob_thold = -0.5\n    assert_in_delta(-0.5, @params.logprob_thold)\n  end\n\n  def test_no_speech_thold\n    assert_in_delta 0.6, @params.no_speech_thold\n    @params.no_speech_thold = 0.2\n    assert_in_delta 0.2, @params.no_speech_thold\n  end\n\n  def test_vad\n    assert_false @params.vad\n    @params.vad = true\n    assert_true @params.vad\n  end\n\n  def test_vad_model_path\n    assert_nil @params.vad_model_path\n    @params.vad_model_path = \"silero-v6.2.0\"\n    assert_equal Whisper::Model.pre_converted_models[\"silero-v6.2.0\"].to_path, @params.vad_model_path\n  end\n\n  def test_vad_model_path_with_nil\n    @params.vad_model_path = \"silero-v6.2.0\"\n    @params.vad_model_path = nil\n    assert_nil @params.vad_model_path\n  end\n\n  def test_vad_model_path_with_invalid\n    assert_raise TypeError do\n      @params.vad_model_path = Object.new\n    end\n  end\n\n  def test_vad_model_path_with_URI_string\n    @params.vad_model_path = \"https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin\"\n    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models[\"silero-v6.2.0\"].to_path\n  end\n\n  def test_vad_model_path_with_URI\n    @params.vad_model_path = URI(\"https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v6.2.0.bin\")\n    assert_equal @params.vad_model_path, Whisper::Model.pre_converted_models[\"silero-v6.2.0\"].to_path\n  end\n\n  def test_vad_params\n    assert_kind_of Whisper::VAD::Params, @params.vad_params\n    default_params = @params.vad_params\n    assert_same default_params, @params.vad_params\n    assert_equal 0.5, default_params.threshold\n    new_params = Whisper::VAD::Params.new\n    @params.vad_params = new_params\n    assert_same new_params, @params.vad_params\n  end\n\n  def test_new_with_kw_args\n    params = Whisper::Params.new(language: \"es\")\n    assert_equal \"es\", params.language\n    assert_equal 1.0, params.max_initial_ts\n  end\n\n  def test_new_with_kw_args_non_existent\n    assert_raise ArgumentError do\n      Whisper::Params.new(non_existent: \"value\")\n    end\n  end\n\n  def test_new_with_kw_args_wrong_type\n    assert_raise TypeError do\n      Whisper::Params.new(language: 3)\n    end\n  end\n\n  data(PARAM_NAMES.collect {|param| [param, param]}.to_h)\n  def test_new_with_kw_args_default_values(param)\n    default_value = @params.send(param)\n    value = case [param, default_value]\n            in [*, true | false]\n              !default_value\n            in [*, Integer | Float]\n              default_value + 1\n            in [:language, *]\n              \"es\"\n            in [:initial_prompt, *]\n              \"Initial prompt\"\n            in [/_callback\\Z/, *]\n              proc {}\n            in [/_user_data\\Z/, *]\n              Object.new\n            in [:vad_model_path, *]\n              Whisper::Model.pre_converted_models[\"silero-v6.2.0\"].to_path\n            in [:vad_params, *]\n              Whisper::VAD::Params.new\n            end\n    params = Whisper::Params.new(param => value)\n    if Float === value\n      assert_in_delta value, params.send(param)\n    else\n      assert_equal value, params.send(param)\n    end\n\n    PARAM_NAMES.reject {|name| name == param}.each do |name|\n      expected = @params.send(name)\n      actual = params.send(name)\n      if Float === expected\n        assert_in_delta expected, actual\n      else\n        assert_equal expected, actual\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_segment.rb",
    "content": "require_relative \"helper\"\n\nclass TestSegment < TestBase\n  def test_iteration\n    whisper.each_segment do |segment|\n      assert_instance_of Whisper::Segment, segment\n    end\n  end\n\n  def test_enumerator\n    enum = whisper.each_segment\n    assert_instance_of Enumerator, enum\n    enum.to_a.each_with_index do |segment, index|\n      assert_instance_of Whisper::Segment, segment\n      assert_kind_of Integer, index\n    end\n  end\n\n  def test_start_time\n    i = 0\n    whisper.each_segment do |segment|\n      assert_equal 0, segment.start_time if i == 0\n      i += 1\n    end\n  end\n\n  def test_end_time\n    i = 0\n    whisper.each_segment do |segment|\n      assert_equal whisper.full_get_segment_t1(i) * 10, segment.end_time\n      i += 1\n    end\n  end\n\n  def test_no_speech_prob\n    no_speech_prob = nil\n    whisper.each_segment do |segment|\n      no_speech_prob = segment.no_speech_prob\n    end\n    assert no_speech_prob > 0.0\n  end\n\n  def test_on_new_segment\n    params = Whisper::Params.new\n    seg = nil\n    index = 0\n    params.on_new_segment do |segment|\n      assert_instance_of Whisper::Segment, segment\n      if index == 0\n        seg = segment\n        assert_equal 0, segment.start_time\n        assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)\n      end\n      index += 1\n    end\n    whisper.transcribe(AUDIO, params)\n    assert_equal 0, seg.start_time\n    assert_match(/ask not what your country can do for you, ask what you can do for your country/, seg.text)\n  end\n\n  def test_on_new_segment_twice\n    params = Whisper::Params.new\n    seg = nil\n    params.on_new_segment do |segment|\n      seg = segment\n      return\n    end\n    params.on_new_segment do |segment|\n      assert_same seg, segment\n      return\n    end\n    whisper.transcribe(AUDIO, params)\n  end\n\n  def test_transcription_after_segment_retrieved\n    segment = whisper.each_segment.first\n    assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)\n\n    whisper.transcribe(AUDIO, Whisper::Params.new(offset: 5000))\n    assert_not_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)\n    assert_match(/what you can do for your country/i, segment.text)\n  end\n\n  def test_pattern_matching\n    segment = whisper.each_segment.first\n    segment => {start_time:, end_time:, text:, no_speech_prob:, speaker_turn_next:}\n\n    assert_equal segment.start_time, start_time\n    assert_equal segment.end_time, end_time\n    assert_equal segment.text, text\n    assert_equal segment.no_speech_prob, no_speech_prob\n    assert_equal segment.speaker_turn_next?, speaker_turn_next\n  end\n\n  def test_pattern_matching_partial\n    segment = whisper.each_segment.first\n    segment => {start_time:, end_time:, text:}\n\n    assert_equal segment.start_time, start_time\n    assert_equal segment.end_time, end_time\n    assert_equal segment.text, text\n  end\n\n  def test_deconstruct_keys\n    segment = whisper.each_segment.first\n    expected = {\n      start_time: segment.start_time,\n      end_time: segment.end_time,\n      text: segment.text,\n      no_speech_prob: segment.no_speech_prob,\n      speaker_turn_next: segment.speaker_turn_next?\n    }\n    assert_equal expected, segment.deconstruct_keys([:start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next])\n  end\n\n  def test_deconstruct_keys_non_existent\n    omit \"Undefined behavior\"\n\n    segment = whisper.each_segment.first\n\n    assert_equal({}, segment.deconstruct_keys([:non_existent]))\n  end\n\n  def test_deconstruct_keys_too_many_keys\n    omit \"Undefined behavior\"\n\n    segment = whisper.each_segment.first\n\n    assert_equal({}, segment.deconstruct_keys([:start_time, :end_time, :text, :no_speech_prob, :speaker_turn_next, :extra_key]))\n  end\n\n  def test_deconstruct_keys_includes_non_existent_keys_not_too_many\n    omit \"Undefined behavior\"\n\n    segment = whisper.each_segment.first\n\n    expected = {\n      start_time: segment.start_time,\n      end_time: segment.end_time,\n      text: segment.text,\n      no_speech_prob: segment.no_speech_prob\n    }\n    assert_equal(expected, segment.deconstruct_keys([:start_time, :end_time, :text, :no_speech_prob, :non_existent]))\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_token.rb",
    "content": "require_relative \"helper\"\n\nclass TestToken < TestBase\n  def setup\n    @segment = whisper.each_segment.first\n    @token = @segment.each_token.first\n  end\n\n  def test_n_tokens\n    assert_equal 27, @segment.n_tokens\n  end\n\n  def test_allocate\n    token = Whisper::Token.allocate\n    assert_raise  do\n      token.id\n    end\n  end\n\n  def test_each_token\n    i = 0\n    @segment.each_token do |token|\n      i += 1\n      assert_instance_of Whisper::Token, token\n    end\n    assert_equal 27, i\n  end\n\n  def test_each_token_without_block\n    assert_instance_of Enumerator, @segment.each_token\n  end\n\n  def test_token\n    assert_instance_of Whisper::Token, @token\n\n    assert_instance_of Integer, @token.id\n    assert_instance_of Float, @token.probability\n    assert_instance_of Float, @token.log_probability\n\n    assert_instance_of Integer, @token.tid\n    assert_instance_of Float, @token.pt\n    assert_instance_of Float, @token.ptsum\n\n    assert_instance_of Integer, @token.start_time\n    assert_instance_of Integer, @token.end_time\n\n    assert_instance_of Integer, @token.t_dtw\n\n    assert_instance_of Float, @token.voice_length\n\n    assert_instance_of String, @token.text\n  end\n\n  def test_text\n    assert_equal [\"[_BEG_]\", \" And\", \" so\", \" my\", \" fellow\", \" Americans\", \",\", \" ask\", \" not\", \" what\", \" your\", \" country\", \" can\", \" do\", \" for\", \" you\", \",\", \" ask\", \" what\", \" you\", \" can\", \" do\", \" for\", \" your\", \" country\", \".\", \"[_TT_550]\"],\n                 @segment.each_token.collect(&:text)\n  end\n\n  def test_token_timestamps\n    params = Whisper::Params.new(token_timestamps: true)\n    whisper.transcribe(TestBase::AUDIO, params)\n    prev = -1\n    whisper.each_segment.first.each_token do |token|\n      assert token.start_time >= prev\n      assert token.end_time >= token.start_time\n      prev = token.end_time\n    end\n  end\n\n  def test_deconstruct_keys_with_nil\n    keys = %i[id tid probability log_probability pt ptsum t_dtw voice_length start_time end_time text]\n    expected = keys.collect {|key| [key, @token.send(key)] }.to_h\n    assert_equal(expected, @token.deconstruct_keys(nil))\n  end\n\n  def test_deconstruct_keys_with_keys\n    keys = %i[id tid probability log_probability pt ptsum t_dtw voice_length start_time end_time text]\n    expected = keys.collect {|key| [key, @token.send(key)] }.to_h\n    assert_equal expected, @token.deconstruct_keys(keys)\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_vad.rb",
    "content": "require_relative \"helper\"\n\nclass TestVAD < TestBase\n  def setup\n    @whisper = Whisper::Context.new(\"base.en\")\n    vad_params = Whisper::VAD::Params.new\n    @params = Whisper::Params.new(\n      vad: true,\n      vad_model_path: \"silero-v6.2.0\",\n      vad_params:\n    )\n  end\n\n  def test_transcribe\n    @whisper.transcribe(TestBase::AUDIO, @params) do |text|\n      assert_match(/ask not what your country can do for you[,.] ask what you can do for your country/i, text)\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_vad_context.rb",
    "content": "require_relative \"helper\"\n\nclass TestVADContext < TestBase\n  def test_initialize\n    context = Whisper::VAD::Context.new(\"silero-v6.2.0\")\n    assert_instance_of Whisper::VAD::Context, context\n  end\n\n  def test_detect\n    context = Whisper::VAD::Context.new(\"silero-v6.2.0\")\n    segments = context.detect(AUDIO, Whisper::VAD::Params.new)\n    assert_segments segments\n  end\n\n  def test_invalid_model_type\n    assert_raise TypeError do\n      Whisper::VAD::Context.new(Object.new)\n    end\n  end\n\n  def test_allocate\n    vad = Whisper::VAD::Context.allocate\n    assert_raise do\n      vad.detect(AUDIO, Whisper::VAD::Params.new)\n    end\n  end\n\n  private\n\n  def assert_segments(segments)\n    assert_instance_of Whisper::VAD::Segments, segments\n\n    i = 0\n    segments.each do |segment|\n      i += 1\n      assert_instance_of Whisper::VAD::Segment, segment\n    end\n    assert i > 0\n\n    segments.each_with_index do |segment, index|\n      assert_instance_of Integer, index\n    end\n\n    assert_instance_of Enumerator, segments.each\n\n    segment = segments.each.first\n    assert_instance_of Float, segment.start_time\n    assert_instance_of Float, segment.end_time\n\n    segment => {start_time:, end_time:}\n    assert_equal segment.start_time, start_time\n    assert_equal segment.end_time, end_time\n\n    assert_equal 4, segments.length\n  end\n\n  sub_test_case \"from samples\" do\n    def setup\n      super\n      @vad = Whisper::VAD::Context.new(\"silero-v6.2.0\")\n      @samples = File.read(AUDIO, nil, 78).unpack(\"s<*\").collect {|i| i.to_f / 2**15}\n    end\n\n    def test_segments_from_samples\n      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, @samples, @samples.length)\n      assert_segments segments\n    end\n\n    def test_segments_from_samples_without_length\n      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, @samples)\n      assert_segments segments\n    end\n\n    def test_segments_from_samples_enumerator\n      samples = @samples.each\n      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, samples, @samples.length)\n      assert_segments segments\n    end\n\n    def test_segments_from_samples_enumerator_without_length\n      samples = @samples.each\n      assert_raise ArgumentError do\n        @vad.segments_from_samples(Whisper::VAD::Params.new, samples)\n      end\n    end\n\n    def test_segments_from_samples_enumerator_with_too_large_length\n      samples = @samples.each.take(10).to_enum\n      assert_raise StopIteration do\n        @vad.segments_from_samples(Whisper::VAD::Params.new, samples, 11)\n      end\n    end\n\n    def test_segments_from_samples_with_memory_view\n      samples = JFKReader.new(AUDIO)\n      segments = @vad.segments_from_samples(Whisper::VAD::Params.new, samples)\n      assert_segments segments\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_vad_params.rb",
    "content": "require_relative \"helper\"\n\nclass TestVADParams < TestBase\n  PARAM_NAMES = [\n    :threshold,\n    :min_speech_duration_ms,\n    :min_silence_duration_ms,\n    :max_speech_duration_s,\n    :speech_pad_ms,\n    :samples_overlap\n  ]\n\n  def setup\n    @params = Whisper::VAD::Params.new\n  end\n\n  def test_new\n    params = Whisper::VAD::Params.new\n    assert_kind_of Whisper::VAD::Params, params\n  end\n\n  def test_threshold\n    assert_in_delta @params.threshold, 0.5\n    @params.threshold = 0.7\n    assert_in_delta @params.threshold, 0.7\n  end\n\n  def test_min_speech_duration\n    pend\n  end\n\n  def test_min_speech_duration_ms\n    assert_equal 250, @params.min_speech_duration_ms\n    @params.min_speech_duration_ms = 500\n    assert_equal 500, @params.min_speech_duration_ms\n  end\n\n  def test_min_silence_duration_ms\n    assert_equal 100, @params.min_silence_duration_ms\n    @params.min_silence_duration_ms = 200\n    assert_equal 200, @params.min_silence_duration_ms\n  end\n\n  def test_max_speech_duration\n    pend\n  end\n\n  def test_max_speech_duration_s\n    assert @params.max_speech_duration_s >= 10e37 # Defaults to FLT_MAX\n    @params.max_speech_duration_s = 60.0\n    assert_equal 60.0, @params.max_speech_duration_s\n  end\n\n  def test_speech_pad_ms\n    assert_equal 30, @params.speech_pad_ms\n    @params.speech_pad_ms = 50\n    assert_equal 50, @params.speech_pad_ms\n  end\n\n  def test_samples_overlap\n    assert_in_delta @params.samples_overlap, 0.1\n    @params.samples_overlap = 0.5\n    assert_in_delta @params.samples_overlap, 0.5\n  end\n\n  def test_equal\n    assert_equal @params, Whisper::VAD::Params.new\n  end\n\n  def test_new_with_kw_args\n    params = Whisper::VAD::Params.new(threshold: 0.7)\n    assert_in_delta params.threshold, 0.7\n    assert_equal 250, params.min_speech_duration_ms\n  end\n\n  def test_new_with_kw_args_non_existent\n    assert_raise ArgumentError do\n      Whisper::VAD::Params.new(non_existent: \"value\")\n    end\n  end\n\n  data(PARAM_NAMES.collect {|param| [param, param]}.to_h)\n  def test_new_with_kw_args_default_values(param)\n    default_value = @params.send(param)\n    value = default_value + 1\n    params = Whisper::VAD::Params.new(param => value)\n    if Float === value\n      assert_in_delta value, params.send(param)\n    else\n      assert_equal value, params.send(param)\n    end\n\n    PARAM_NAMES.reject {|name| name == param}.each do |name|\n      expected = @params.send(name)\n      actual = params.send(name)\n      if Float === expected\n        assert_in_delta expected, actual\n      else\n        assert_equal expected, actual\n      end\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_vad_segment.rb",
    "content": "require_relative \"helper\"\n\nclass TestVADSegment < TestBase\n  def test_initialize\n    segment = Whisper::VAD::Segment.new\n\n    assert_raise do\n      segment.start_time\n    end\n\n    assert_raise do\n      segments.end_time\n    end\n\n    assert_raise do\n      segment => {start_time:, end_time:}\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_vad_segments.rb",
    "content": "require_relative \"helper\"\n\nclass TestVADSegments < TestBase\n  def test_initialize\n    segments = Whisper::VAD::Segments.new\n\n    assert_raise do\n      segments.each do |segment|\n      end\n    end\n\n    assert_raise do\n      segments.length\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/test/test_whisper.rb",
    "content": "require_relative \"helper\"\nrequire \"stringio\"\nrequire \"etc\"\nrequire \"pathname\"\n\n# Exists to detect memory-related bug\nWhisper.log_set ->(level, buffer, user_data) {}, nil\n\nclass TestWhisper < TestBase\n  def setup\n    @params  = Whisper::Params.new\n  end\n\n  def test_whisper\n    @whisper = Whisper::Context.new(\"base.en\")\n    params  = Whisper::Params.new\n    params.print_timestamps = false\n\n    @whisper.transcribe(AUDIO, params) {|text|\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, text)\n    }\n  end\n\n  def test_whisper_pathname\n    @whisper = Whisper::Context.new(\"base.en\")\n    params  = Whisper::Params.new\n\n    @whisper.transcribe(Pathname(AUDIO), params) {|text|\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, text)\n    }\n  end\n\n  def test_transcribe_non_parallel\n    @whisper = Whisper::Context.new(\"base.en\")\n    params  = Whisper::Params.new\n\n    @whisper.transcribe(AUDIO, params, n_processors: 1) {|text|\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, text)\n    }\n  end\n\n  def test_transcribe_n_processors\n    @whisper = Whisper::Context.new(\"base.en\")\n    params  = Whisper::Params.new\n\n    @whisper.transcribe(AUDIO, params, n_processors: 4) {|text|\n      assert_match(/what you can do for your country/i, text)\n    }\n  end\n\n  sub_test_case \"After transcription\" do\n    def test_full_n_segments\n      assert_equal 1, whisper.full_n_segments\n    end\n\n    def test_full_lang_id\n      assert_equal 0, whisper.full_lang_id\n    end\n\n    def test_full_get_segment\n      segment = whisper.full_get_segment(0)\n      assert_equal 0, segment.start_time\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, segment.text)\n    end\n\n    def test_full_get_segment_t0\n      assert_equal 0, whisper.full_get_segment_t0(0)\n      assert_raise IndexError do\n        whisper.full_get_segment_t0(whisper.full_n_segments)\n      end\n      assert_raise IndexError do\n        whisper.full_get_segment_t0(-1)\n      end\n    end\n\n    def test_full_get_segment_t1\n      t1 = whisper.full_get_segment_t1(0)\n      assert_kind_of Integer, t1\n      assert t1 > 0\n      assert_raise IndexError do\n        whisper.full_get_segment_t1(whisper.full_n_segments)\n      end\n    end\n\n    def test_full_get_segment_speaker_turn_next\n      assert_false whisper.full_get_segment_speaker_turn_next(0)\n    end\n\n    def test_full_get_segment_text\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, whisper.full_get_segment_text(0))\n    end\n\n    def test_full_get_segment_no_speech_prob\n      prob = whisper.full_get_segment_no_speech_prob(0)\n      assert prob > 0.0\n      assert prob < 1.0\n    end\n  end\n\n  def test_lang_max_id\n    assert_kind_of Integer, Whisper.lang_max_id\n  end\n\n  def test_lang_id\n    assert_equal 0, Whisper.lang_id(\"en\")\n    assert_raise ArgumentError do\n      Whisper.lang_id(\"non existing language\")\n    end\n  end\n\n  def test_lang_str\n    assert_equal \"en\", Whisper.lang_str(0)\n    assert_raise IndexError do\n      Whisper.lang_str(Whisper.lang_max_id + 1)\n    end\n  end\n\n  def test_lang_str_full\n    assert_equal \"english\", Whisper.lang_str_full(0)\n    assert_raise IndexError do\n      Whisper.lang_str_full(Whisper.lang_max_id + 1)\n    end\n  end\n\n  def test_system_info_str\n    assert_match(/\\AWHISPER : COREML = \\d | OPENVINO = \\d |/, Whisper.system_info_str)\n  end\n\n  def test_version\n    assert_kind_of String, Whisper::VERSION\n  end\n\n  def test_log_set\n    user_data = Object.new\n    logs = []\n    log_callback = ->(level, buffer, udata) {\n      logs << [level, buffer, udata]\n    }\n    Whisper.log_set log_callback, user_data\n    Whisper::Context.new(\"base.en\")\n\n    assert logs.length > 30\n    logs.each do |log|\n      assert_include [Whisper::LOG_LEVEL_DEBUG, Whisper::LOG_LEVEL_INFO, Whisper::LOG_LEVEL_WARN], log[0]\n      assert_same user_data, log[2]\n    end\n  end\n\n  def test_log_suppress\n    stderr = $stderr\n    Whisper.log_set ->(level, buffer, user_data) {\n      # do nothing\n    }, nil\n    dev = StringIO.new(\"\")\n    $stderr = dev\n    Whisper::Context.new(\"base.en\")\n    assert_empty dev.string\n  ensure\n    $stderr = stderr\n  end\n\n  def test_access_attribute_without_initialization\n    whisper = Whisper::Context.allocate\n    assert_raise do\n      whisper.model_type\n    end\n  end\n\n  sub_test_case \"full\" do\n    def setup\n      super\n      @whisper = Whisper::Context.new(\"base.en\")\n      @samples = File.read(AUDIO, nil, 78).unpack(\"s<*\").collect {|i| i.to_f / 2**15}\n    end\n\n    def test_full\n      @whisper.full(@params, @samples, @samples.length)\n\n      assert_equal 1, @whisper.full_n_segments\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)\n    end\n\n    def test_full_without_length\n      @whisper.full(@params, @samples)\n\n      assert_equal 1, @whisper.full_n_segments\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)\n    end\n\n    def test_full_enumerator\n      samples = @samples.each\n      @whisper.full(@params, samples, @samples.length)\n\n      assert_equal 1, @whisper.full_n_segments\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)\n    end\n\n    def test_full_enumerator_without_length\n      samples = @samples.each\n      assert_raise ArgumentError do\n        @whisper.full(@params, samples)\n      end\n    end\n\n    def test_full_enumerator_with_too_large_length\n      samples = @samples.each.take(10).to_enum\n      assert_raise StopIteration do\n        @whisper.full(@params, samples, 11)\n      end\n    end\n\n    def test_full_with_memory_view\n      samples = JFKReader.new(AUDIO)\n      @whisper.full(@params, samples)\n\n      assert_equal 1, @whisper.full_n_segments\n      assert_match(/ask not what your country can do for you, ask what you can do for your country/, @whisper.each_segment.first.text)\n    end\n\n    def test_full_with_memroy_view_gc\n      samples = JFKReader.new(AUDIO)\n      @whisper.full(@params, samples)\n      GC.start\n      require \"fiddle\"\n      Fiddle::MemoryView.export samples do |view|\n        assert_equal 176000, view.to_s.unpack(\"#{view.format}*\").length\n      end\n    end\n\n    def test_full_parallel\n      nprocessors = 2\n      @whisper.full_parallel(@params, @samples, @samples.length, nprocessors)\n\n      assert_equal nprocessors, @whisper.full_n_segments\n      text = @whisper.each_segment.collect(&:text).join\n      assert_match(/ask what you can do/i, text)\n      assert_match(/for your country/i, text)\n    end\n\n    def test_full_parallel_with_memory_view\n      nprocessors = 2\n      samples = JFKReader.new(AUDIO)\n      @whisper.full_parallel(@params, samples, nil, nprocessors)\n\n      assert_equal nprocessors, @whisper.full_n_segments\n      text = @whisper.each_segment.collect(&:text).join\n      assert_match(/ask what you can do/i, text)\n      assert_match(/for your country/i, text)\n    end\n\n    def test_full_parallel_without_length_and_n_processors\n      @whisper.full_parallel(@params, @samples)\n\n      assert_equal 1, @whisper.full_n_segments\n      text = @whisper.each_segment.collect(&:text).join\n      assert_match(/ask what you can do/i, text)\n      assert_match(/for your country/i, text)\n    end\n\n    def test_full_parallel_without_length\n      nprocessors = 2\n      @whisper.full_parallel(@params, @samples, nil, nprocessors)\n\n      assert_equal nprocessors, @whisper.full_n_segments\n      text = @whisper.each_segment.collect(&:text).join\n      assert_match(/ask what you can do/i, text)\n      assert_match(/for your country/i, text)\n    end\n\n    def test_full_parallel_without_n_processors\n      @whisper.full_parallel(@params, @samples, @samples.length)\n\n      assert_equal 1, @whisper.full_n_segments\n      text = @whisper.each_segment.collect(&:text).join\n      assert_match(/ask what you can do/i, text)\n      assert_match(/for your country/i, text)\n    end\n  end\n\n  def test_to_srt\n    whisper = Whisper::Context.new(\"base.en\")\n    whisper.transcribe AUDIO, @params\n\n    lines = whisper.to_srt.lines\n    assert_match(/\\A\\d+\\n/, lines[0])\n    assert_match(/\\d{2}:\\d{2}:\\d{2},\\d{3} --> \\d{2}:\\d{2}:\\d{2},\\d{3}\\n/, lines[1])\n    assert_match(/ask not what your country can do for you, ask what you can do for your country/, lines[2])\n  end\n\n  def test_to_webvtt\n    whisper = Whisper::Context.new(\"base.en\")\n    whisper.transcribe AUDIO, @params\n\n    lines = whisper.to_webvtt.lines\n    assert_equal \"WEBVTT\\n\", lines[0]\n    assert_equal \"\\n\", lines[1]\n    assert_match(/\\A\\d+\\n/, lines[2])\n    assert_match(/\\d{2}:\\d{2}:\\d{2}\\.\\d{3} --> \\d{2}:\\d{2}:\\d{2}\\.\\d{3}\\n/, lines[3])\n    assert_match(/ask not what your country can do for you, ask what you can do for your country/, lines[4])\n  end\n\n  sub_test_case \"Format needs escape\" do\n    def setup\n      @whisper = Whisper::Context.new(\"base.en\")\n      @whisper.transcribe AUDIO, Whisper::Params.new\n      segment = @whisper.each_segment.first\n      segment.define_singleton_method :text do\n        \"& so my fellow Americans --> ask not what your country can do for you <-- ask what you can do for your country.\"\n      end\n      @whisper.define_singleton_method :each_segment do\n        Enumerator.new(3) {|yielder| 3.times {yielder << segment}}\n      end\n    end\n\n    def test_to_srt_escape\n      assert_equal \"&amp; so my fellow Americans --&gt; ask not what your country can do for you &lt;-- ask what you can do for your country.\\n\", @whisper.to_srt.lines[2]\n    end\n\n    def test_to_webvtt_escape\n      assert_equal \"&amp; so my fellow Americans --&gt; ask not what your country can do for you &lt;-- ask what you can do for your country.\\n\", @whisper.to_webvtt.lines[4]\n    end\n  end\nend\n"
  },
  {
    "path": "bindings/ruby/whispercpp.gemspec",
    "content": "require_relative \"extsources\"\n\nGem::Specification.new do |s|\n  s.name    = \"whispercpp\"\n  s.authors = [\"Georgi Gerganov\", \"Todd A. Fisher\"]\n  s.version = '1.3.6'\n  s.description = %q{High-performance inference of OpenAI's Whisper automatic speech recognition (ASR) model via Ruby}\n  s.email   = 'todd.fisher@gmail.com'\n  s.extra_rdoc_files = ['LICENSE', 'README.md']\n\n  s.files = `git ls-files . -z`.split(\"\\x0\") +\n              EXTSOURCES.collect {|file|\n                basename = File.basename(file)\n                if s.extra_rdoc_files.include?(basename)\n                  basename\n                else\n                  file.sub(\"../..\", \"ext/sources\")\n                      .sub(\"../javascript\", \"ext/sources/bindings/javascript\")\n                end\n              }\n\n  s.summary = %q{Ruby whisper.cpp bindings}\n  s.test_files = s.files.select {|file| file.start_with? \"test/\"}\n\n  s.extensions << 'ext/extconf.rb'\n  s.required_ruby_version = '>= 3.1.0'\n\n  #### Documentation and testing.\n  s.homepage = 'https://github.com/ggml-org/whisper.cpp'\n  s.rdoc_options = ['--main', 'README.md']\n\n\n    s.platform = Gem::Platform::RUBY\n\n  s.licenses = ['MIT']\nend\n"
  },
  {
    "path": "build-xcframework.sh",
    "content": "#!/bin/bash\n#\n# Options\nIOS_MIN_OS_VERSION=16.4\nMACOS_MIN_OS_VERSION=13.3\nVISIONOS_MIN_OS_VERSION=1.0\nTVOS_MIN_OS_VERSION=16.4\n\nBUILD_SHARED_LIBS=OFF\nWHISPER_BUILD_EXAMPLES=OFF\nWHISPER_BUILD_TESTS=OFF\nWHISPER_BUILD_SERVER=OFF\nGGML_METAL=ON\nGGML_METAL_EMBED_LIBRARY=ON\nGGML_BLAS_DEFAULT=ON\nGGML_METAL_USE_BF16=ON\nGGML_OPENMP=OFF\nBUILD_STATIC_XCFRAMEWORK=${BUILD_STATIC_XCFRAMEWORK:-OFF}\n\nCOMMON_C_FLAGS=\"-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g\"\nCOMMON_CXX_FLAGS=\"-Wno-macro-redefined -Wno-shorten-64-to-32 -Wno-unused-command-line-argument -g\"\n\n# Common options for all builds\nCOMMON_CMAKE_ARGS=(\n    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=NO\n    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY=\"\"\n    -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO\n    -DCMAKE_XCODE_ATTRIBUTE_DEBUG_INFORMATION_FORMAT=\"dwarf-with-dsym\"\n    -DCMAKE_XCODE_ATTRIBUTE_GCC_GENERATE_DEBUGGING_SYMBOLS=YES\n    -DCMAKE_XCODE_ATTRIBUTE_COPY_PHASE_STRIP=NO\n    -DCMAKE_XCODE_ATTRIBUTE_STRIP_INSTALLED_PRODUCT=NO\n    -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml\n    -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}\n    -DWHISPER_BUILD_EXAMPLES=${WHISPER_BUILD_EXAMPLES}\n    -DWHISPER_BUILD_TESTS=${WHISPER_BUILD_TESTS}\n    -DWHISPER_BUILD_SERVER=${WHISPER_BUILD_SERVER}\n    -DGGML_METAL_EMBED_LIBRARY=${GGML_METAL_EMBED_LIBRARY}\n    -DGGML_BLAS_DEFAULT=${GGML_BLAS_DEFAULT}\n    -DGGML_METAL=${GGML_METAL}\n    -DGGML_METAL_USE_BF16=${GGML_METAL_USE_BF16}\n    -DGGML_NATIVE=OFF\n    -DGGML_OPENMP=${GGML_OPENMP}\n)\n\nXCODE_VERSION=$(xcodebuild -version 2>/dev/null | head -n1 | awk '{ print $2 }')\nMAJOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f1)\nMINOR_VERSION=$(echo $XCODE_VERSION | cut -d. -f2)\necho \"Detected Xcode version: $XCODE_VERSION\"\n\ncheck_required_tool() {\n    local tool=$1\n    local install_message=$2\n\n    if ! command -v $tool &> /dev/null; then\n        echo \"Error: $tool is required but not found.\"\n        echo \"$install_message\"\n        exit 1\n    fi\n}\necho \"Checking for required tools...\"\ncheck_required_tool \"cmake\" \"Please install CMake 3.28.0 or later (brew install cmake)\"\ncheck_required_tool \"xcodebuild\" \"Please install Xcode and Xcode Command Line Tools (xcode-select --install)\"\ncheck_required_tool \"libtool\" \"Please install libtool which should be available with Xcode Command Line Tools (CLT). Make sure Xcode CLT is installed (xcode-select --install)\"\ncheck_required_tool \"dsymutil\" \"Please install Xcode and Xcode Command Line Tools (xcode-select --install)\"\n\nset -e\n\n## Clean up previous builds\nrm -rf build-apple\nrm -rf build-ios-sim\nrm -rf build-ios-device\nrm -rf build-macos\nrm -rf build-visionos\nrm -rf build-visionos-sim\nrm -rf build-tvos-sim\nrm -rf build-tvos-device\n\n# Setup the xcframework build directory structure\nsetup_framework_structure() {\n    local build_dir=$1\n    local min_os_version=$2\n    local platform=$3  # \"ios\", \"macos\", \"visionos\", or \"tvos\"\n    local framework_name=\"whisper\"\n\n    echo \"Creating ${platform}-style framework structure for ${build_dir}\"\n\n    if [[ \"$platform\" == \"macos\" ]]; then\n        # macOS versioned structure uses versioned directories\n        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Headers\n        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Modules\n        mkdir -p ${build_dir}/framework/${framework_name}.framework/Versions/A/Resources\n\n        # Create symbolic links\n        ln -sf A ${build_dir}/framework/${framework_name}.framework/Versions/Current\n        ln -sf Versions/Current/Headers ${build_dir}/framework/${framework_name}.framework/Headers\n        ln -sf Versions/Current/Modules ${build_dir}/framework/${framework_name}.framework/Modules\n        ln -sf Versions/Current/Resources ${build_dir}/framework/${framework_name}.framework/Resources\n        ln -sf Versions/Current/${framework_name} ${build_dir}/framework/${framework_name}.framework/${framework_name}\n\n        # Set header and module paths\n        local header_path=${build_dir}/framework/${framework_name}.framework/Versions/A/Headers/\n        local module_path=${build_dir}/framework/${framework_name}.framework/Versions/A/Modules/\n    else\n        # iOS/VisionOS/tvOS use a flat structure\n        mkdir -p ${build_dir}/framework/${framework_name}.framework/Headers\n        mkdir -p ${build_dir}/framework/${framework_name}.framework/Modules\n\n        # Remove any existing structure to ensure clean build\n        rm -rf ${build_dir}/framework/${framework_name}.framework/Versions\n\n        # Set header and module paths\n        local header_path=${build_dir}/framework/${framework_name}.framework/Headers/\n        local module_path=${build_dir}/framework/${framework_name}.framework/Modules/\n    fi\n\n    # Copy all required headers (common for all platforms)\n    cp include/whisper.h           ${header_path}\n    cp ggml/include/ggml.h         ${header_path}\n    cp ggml/include/ggml-alloc.h   ${header_path}\n    cp ggml/include/ggml-backend.h ${header_path}\n    cp ggml/include/ggml-metal.h   ${header_path}\n    cp ggml/include/ggml-cpu.h     ${header_path}\n    cp ggml/include/ggml-blas.h    ${header_path}\n    cp ggml/include/gguf.h         ${header_path}\n\n    # Create module map (common for all platforms)\n    cat > ${module_path}module.modulemap << EOF\nframework module whisper {\n    header \"whisper.h\"\n    header \"ggml.h\"\n    header \"ggml-alloc.h\"\n    header \"ggml-backend.h\"\n    header \"ggml-metal.h\"\n    header \"ggml-cpu.h\"\n    header \"ggml-blas.h\"\n    header \"gguf.h\"\n\n    link \"c++\"\n    link framework \"Accelerate\"\n    link framework \"Metal\"\n    link framework \"Foundation\"\n\n    export *\n}\nEOF\n\n    # Platform-specific settings for Info.plist\n    local platform_name=\"\"\n    local sdk_name=\"\"\n    local supported_platform=\"\"\n\n    case \"$platform\" in\n        \"ios\")\n            platform_name=\"iphoneos\"\n            sdk_name=\"iphoneos${min_os_version}\"\n            supported_platform=\"iPhoneOS\"\n            local plist_path=\"${build_dir}/framework/${framework_name}.framework/Info.plist\"\n            local device_family='    <key>UIDeviceFamily</key>\n    <array>\n        <integer>1</integer>\n        <integer>2</integer>\n    </array>'\n            ;;\n        \"macos\")\n            platform_name=\"macosx\"\n            sdk_name=\"macosx${min_os_version}\"\n            supported_platform=\"MacOSX\"\n            local plist_path=\"${build_dir}/framework/${framework_name}.framework/Versions/A/Resources/Info.plist\"\n            local device_family=\"\"\n            ;;\n        \"visionos\")\n            platform_name=\"xros\"\n            sdk_name=\"xros${min_os_version}\"\n            supported_platform=\"XRPlatform\"\n            local plist_path=\"${build_dir}/framework/${framework_name}.framework/Info.plist\"\n            local device_family=\"\"\n            ;;\n        \"tvos\")\n            platform_name=\"appletvos\"\n            sdk_name=\"appletvos${min_os_version}\"\n            supported_platform=\"AppleTVOS\"\n            local plist_path=\"${build_dir}/framework/${framework_name}.framework/Info.plist\"\n            local device_family='    <key>UIDeviceFamily</key>\n    <array>\n        <integer>3</integer>\n    </array>'\n            ;;\n    esac\n\n    # Create Info.plist\n    cat > ${plist_path} << EOF\n<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n    <key>CFBundleDevelopmentRegion</key>\n    <string>en</string>\n    <key>CFBundleExecutable</key>\n    <string>whisper</string>\n    <key>CFBundleIdentifier</key>\n    <string>org.ggml.whisper</string>\n    <key>CFBundleInfoDictionaryVersion</key>\n    <string>6.0</string>\n    <key>CFBundleName</key>\n    <string>whisper</string>\n    <key>CFBundlePackageType</key>\n    <string>FMWK</string>\n    <key>CFBundleShortVersionString</key>\n    <string>1.0</string>\n    <key>CFBundleVersion</key>\n    <string>1</string>\n    <key>MinimumOSVersion</key>\n    <string>${min_os_version}</string>\n    <key>CFBundleSupportedPlatforms</key>\n    <array>\n        <string>${supported_platform}</string>\n    </array>${device_family}\n    <key>DTPlatformName</key>\n    <string>${platform_name}</string>\n    <key>DTSDKName</key>\n    <string>${sdk_name}</string>\n</dict>\n</plist>\nEOF\n}\n\n# Create dynamic libraries from static libraries.\ncombine_static_libraries() {\n    local build_dir=\"$1\"\n    local release_dir=\"$2\"\n    local platform=\"$3\"  # \"ios\", \"macos\", \"visionos\", or \"tvos\"\n    local is_simulator=\"$4\"\n    local base_dir=\"$(pwd)\"\n    local framework_name=\"whisper\"\n\n    # Determine output path based on platform\n    local output_lib=\"\"\n    if [[ \"$platform\" == \"macos\" ]]; then\n        # macOS uses versioned structure\n        output_lib=\"${build_dir}/framework/${framework_name}.framework/Versions/A/${framework_name}\"\n    else\n        # iOS, visionOS, and tvOS use a directory flat structure\n        output_lib=\"${build_dir}/framework/${framework_name}.framework/${framework_name}\"\n    fi\n\n    local libs=(\n        \"${base_dir}/${build_dir}/src/${release_dir}/libwhisper.a\"\n        \"${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml.a\"\n        \"${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml-base.a\"\n        \"${base_dir}/${build_dir}/ggml/src/${release_dir}/libggml-cpu.a\"\n        \"${base_dir}/${build_dir}/ggml/src/ggml-metal/${release_dir}/libggml-metal.a\"\n        \"${base_dir}/${build_dir}/ggml/src/ggml-blas/${release_dir}/libggml-blas.a\"\n    )\n    if [[ \"$platform\" == \"macos\" || \"$platform\" == \"ios\" ]]; then\n        echo \"Adding libwhisper.coreml library to the build.\"\n        libs+=(\n            \"${base_dir}/${build_dir}/src/${release_dir}/libwhisper.coreml.a\"\n        )\n    fi\n\n    # Create temporary directory for processing\n    local temp_dir=\"${base_dir}/${build_dir}/temp\"\n    echo \"Creating temporary directory: ${temp_dir}\"\n    mkdir -p \"${temp_dir}\"\n\n    # Since we have multiple architectures libtool will find object files that do not\n    # match the target architecture. We suppress these warnings.\n    libtool -static -o \"${temp_dir}/combined.a\" \"${libs[@]}\" 2> /dev/null\n\n    # Determine SDK, architectures, and install_name based on platform and simulator flag.\n    local sdk=\"\"\n    local archs=\"\"\n    local min_version_flag=\"\"\n    local install_name=\"\"\n    local frameworks=\"-framework Foundation -framework Metal -framework Accelerate\"\n\n    case \"$platform\" in\n        \"ios\")\n            if [[ \"$is_simulator\" == \"true\" ]]; then\n                sdk=\"iphonesimulator\"\n                archs=\"arm64 x86_64\"\n                min_version_flag=\"-mios-simulator-version-min=${IOS_MIN_OS_VERSION}\"\n            else\n                sdk=\"iphoneos\"\n                archs=\"arm64\"\n                min_version_flag=\"-mios-version-min=${IOS_MIN_OS_VERSION}\"\n            fi\n            install_name=\"@rpath/whisper.framework/whisper\"\n            frameworks+=\" -framework CoreML\"\n            ;;\n        \"macos\")\n            sdk=\"macosx\"\n            archs=\"arm64 x86_64\"\n            min_version_flag=\"-mmacosx-version-min=${MACOS_MIN_OS_VERSION}\"\n            install_name=\"@rpath/whisper.framework/Versions/Current/whisper\"\n            frameworks+=\" -framework CoreML\"\n            ;;\n        \"visionos\")\n            if [[ \"$is_simulator\" == \"true\" ]]; then\n                sdk=\"xrsimulator\"\n                archs=\"arm64 x86_64\"\n                min_version_flag=\"-mtargetos=xros${VISIONOS_MIN_OS_VERSION}-simulator\"\n            else\n                sdk=\"xros\"\n                archs=\"arm64\"\n                min_version_flag=\"-mtargetos=xros${VISIONOS_MIN_OS_VERSION}\"\n            fi\n            # Use flat structure for visionOS, same as iOS\n            install_name=\"@rpath/whisper.framework/whisper\"\n            ;;\n        \"tvos\")\n            if [[ \"$is_simulator\" == \"true\" ]]; then\n                sdk=\"appletvsimulator\"\n                archs=\"arm64 x86_64\"\n                min_version_flag=\"-mtvos-simulator-version-min=${TVOS_MIN_OS_VERSION}\"\n            else\n                sdk=\"appletvos\"\n                archs=\"arm64\"\n                min_version_flag=\"-mtvos-version-min=${TVOS_MIN_OS_VERSION}\"\n            fi\n            install_name=\"@rpath/whisper.framework/whisper\"\n            ;;\n    esac\n\n    # Build architecture flags\n    local arch_flags=\"\"\n    for arch in $archs; do\n        arch_flags+=\" -arch $arch\"\n    done\n\n\n    if [[ \"${BUILD_STATIC_XCFRAMEWORK}\" == \"ON\" ]]; then\n        echo \"Packaging static framework for ${platform}.\"\n        mkdir -p \"$(dirname \"${base_dir}/${output_lib}\")\"\n        cp \"${temp_dir}/combined.a\" \"${base_dir}/${output_lib}\"\n        rm -rf \"${temp_dir}\"\n        return\n    fi\n\n    # Create dynamic library\n    echo \"Creating dynamic library for ${platform}.\"\n    xcrun -sdk $sdk clang++ -dynamiclib \\\n        -isysroot $(xcrun --sdk $sdk --show-sdk-path) \\\n        $arch_flags \\\n        $min_version_flag \\\n        -Wl,-force_load,\"${temp_dir}/combined.a\" \\\n        $frameworks \\\n        -install_name \"$install_name\" \\\n        -o \"${base_dir}/${output_lib}\"\n\n    # Platform-specific post-processing for device builds\n    if [[ \"$is_simulator\" == \"false\" ]]; then\n        if command -v xcrun vtool &>/dev/null; then\n            case \"$platform\" in\n                \"ios\")\n                    echo \"Marking binary as a framework binary for iOS...\"\n                    xcrun vtool -set-build-version ios ${IOS_MIN_OS_VERSION} ${IOS_MIN_OS_VERSION} -replace \\\n                        -output \"${base_dir}/${output_lib}\" \"${base_dir}/${output_lib}\"\n                    ;;\n                \"visionos\")\n                    echo \"Marking binary as a framework binary for visionOS...\"\n                    if [[ \"$MAJOR_VERSION\" -gt 16 ]] || [[ \"$MAJOR_VERSION\" -eq 16 && \"$MINOR_VERSION\" -gt 2 ]]; then\n                        echo \"Xcode version greater than 16.2, using visionOS.\"\n                        VISION_OS_BUILD_VERSION=\"visionos\"\n                    else\n                        echo \"Xcode version less than or equal to 16.2, using xros.\"\n                        VISION_OS_BUILD_VERSION=\"xros\"\n                    fi\n                    xcrun vtool -set-build-version ${VISION_OS_BUILD_VERSION} ${VISIONOS_MIN_OS_VERSION} ${VISIONOS_MIN_OS_VERSION} -replace \\\n                        -output \"${base_dir}/${output_lib}\" \"${base_dir}/${output_lib}\"\n                    ;;\n                \"tvos\")\n                    echo \"Marking binary as a framework binary for tvOS...\"\n                    xcrun vtool -set-build-version tvos ${TVOS_MIN_OS_VERSION} ${TVOS_MIN_OS_VERSION} -replace \\\n                        -output \"${base_dir}/${output_lib}\" \"${base_dir}/${output_lib}\"\n                    ;;\n            esac\n        else\n            echo \"Warning: vtool not found. Binary may not pass App Store validation.\"\n        fi\n    fi\n\n    echo \"Creating properly formatted dSYM...\"\n    # Create a separate directory for dSYMs for all platforms\n    mkdir -p \"${base_dir}/${build_dir}/dSYMs\"\n\n    # iOS and visionOS style dSYM (flat structure)\n    if [[ \"$platform\" == \"ios\" || \"$platform\" == \"visionos\" || \"$platform\" == \"tvos\" ]]; then\n        # Generate dSYM in the dSYMs directory\n        xcrun dsymutil \"${base_dir}/${output_lib}\" -o \"${base_dir}/${build_dir}/dSYMs/whisper.dSYM\"\n\n        # Create a copy of the binary that will be stripped\n        cp \"${base_dir}/${output_lib}\" \"${temp_dir}/binary_to_strip\"\n\n        # Strip debug symbols from the copy\n        xcrun strip -S \"${temp_dir}/binary_to_strip\" -o \"${temp_dir}/stripped_lib\"\n\n        # Replace the original with the stripped version\n        mv \"${temp_dir}/stripped_lib\" \"${base_dir}/${output_lib}\"\n    else\n        # macOS style dSYM\n        # First strip debug info to a separate file\n        xcrun strip -S \"${base_dir}/${output_lib}\" -o \"${temp_dir}/stripped_lib\"\n\n        # Generate dSYM in the dSYMs directory\n        xcrun dsymutil \"${base_dir}/${output_lib}\" -o \"${base_dir}/${build_dir}/dSYMs/whisper.dSYM\"\n\n        # Replace original binary with stripped version\n        mv \"${temp_dir}/stripped_lib\" \"${base_dir}/${output_lib}\"\n    fi\n\n    # Remove any automatically generated dSYM files in the framework structure as they will\n    # otherwise case Invalid Bundle Structure validation errors.\n    if [ -d \"${base_dir}/${output_lib}.dSYM\" ]; then\n        echo \"Removing generated dSYM file in framework structure: ${base_dir}/${output_lib}.dSYM\"\n        rm -rf \"${base_dir}/${output_lib}.dSYM\"\n    fi\n\n    # Clean up\n    rm -rf \"${temp_dir}\"\n}\n\necho \"Building for iOS simulator...\"\ncmake -B build-ios-sim -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \\\n    -DIOS=ON \\\n    -DCMAKE_SYSTEM_NAME=iOS \\\n    -DCMAKE_OSX_SYSROOT=iphonesimulator \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\" \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphonesimulator \\\n    -DCMAKE_C_FLAGS=\"${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"${COMMON_CXX_FLAGS}\" \\\n    -DWHISPER_COREML=\"ON\" \\\n    -DWHISPER_COREML_ALLOW_FALLBACK=\"ON\" \\\n    -S .\ncmake --build build-ios-sim --config Release -- -quiet\n\necho \"Building for iOS devices...\"\ncmake -B build-ios-device -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${IOS_MIN_OS_VERSION} \\\n    -DCMAKE_OSX_SYSROOT=iphoneos \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64\" \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=iphoneos \\\n    -DCMAKE_C_FLAGS=\"${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"${COMMON_CXX_FLAGS}\" \\\n    -DWHISPER_COREML=\"ON\" \\\n    -DWHISPER_COREML_ALLOW_FALLBACK=\"ON\" \\\n    -S .\ncmake --build build-ios-device --config Release -- -quiet\n\necho \"Building for macOS...\"\ncmake -B build-macos -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOS_MIN_OS_VERSION} \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\" \\\n    -DCMAKE_C_FLAGS=\"${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"${COMMON_CXX_FLAGS}\" \\\n    -DWHISPER_COREML=\"ON\" \\\n    -DWHISPER_COREML_ALLOW_FALLBACK=\"ON\" \\\n    -S .\ncmake --build build-macos --config Release -- -quiet\n\necho \"Building for visionOS...\"\ncmake -B build-visionos -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64\" \\\n    -DCMAKE_SYSTEM_NAME=visionOS \\\n    -DCMAKE_OSX_SYSROOT=xros \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xros \\\n    -DCMAKE_C_FLAGS=\"-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}\" \\\n    -S .\ncmake --build build-visionos --config Release -- -quiet\n\necho \"Building for visionOS simulator...\"\ncmake -B build-visionos-sim -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${VISIONOS_MIN_OS_VERSION} \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\" \\\n    -DCMAKE_SYSTEM_NAME=visionOS \\\n    -DCMAKE_OSX_SYSROOT=xrsimulator \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=xrsimulator \\\n    -DCMAKE_C_FLAGS=\"-D_XOPEN_SOURCE=700 ${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"-D_XOPEN_SOURCE=700 ${COMMON_CXX_FLAGS}\" \\\n    -S .\ncmake --build build-visionos-sim --config Release -- -quiet\n\n# Add tvOS builds (might need the same u_int definitions as watchOS and visionOS)\necho \"Building for tvOS simulator...\"\ncmake -B build-tvos-sim -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \\\n    -DCMAKE_SYSTEM_NAME=tvOS \\\n    -DCMAKE_OSX_SYSROOT=appletvsimulator \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64;x86_64\" \\\n    -DGGML_METAL=ON \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvsimulator \\\n    -DCMAKE_C_FLAGS=\"${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"${COMMON_CXX_FLAGS}\" \\\n    -S .\ncmake --build build-tvos-sim --config Release -- -quiet\n\necho \"Building for tvOS devices...\"\ncmake -B build-tvos-device -G Xcode \\\n    \"${COMMON_CMAKE_ARGS[@]}\" \\\n    -DCMAKE_OSX_DEPLOYMENT_TARGET=${TVOS_MIN_OS_VERSION} \\\n    -DCMAKE_SYSTEM_NAME=tvOS \\\n    -DCMAKE_OSX_SYSROOT=appletvos \\\n    -DCMAKE_OSX_ARCHITECTURES=\"arm64\" \\\n    -DGGML_METAL=ON \\\n    -DCMAKE_XCODE_ATTRIBUTE_SUPPORTED_PLATFORMS=appletvos \\\n    -DCMAKE_C_FLAGS=\"${COMMON_C_FLAGS}\" \\\n    -DCMAKE_CXX_FLAGS=\"${COMMON_CXX_FLAGS}\" \\\n    -S .\ncmake --build build-tvos-device --config Release -- -quiet\n\n# Setup frameworks and copy binaries and headers\necho \"Setting up framework structures...\"\nsetup_framework_structure \"build-ios-sim\" ${IOS_MIN_OS_VERSION} \"ios\"\nsetup_framework_structure \"build-ios-device\" ${IOS_MIN_OS_VERSION} \"ios\"\nsetup_framework_structure \"build-macos\" ${MACOS_MIN_OS_VERSION} \"macos\"\nsetup_framework_structure \"build-visionos\" ${VISIONOS_MIN_OS_VERSION} \"visionos\"\nsetup_framework_structure \"build-visionos-sim\" ${VISIONOS_MIN_OS_VERSION} \"visionos\"\nsetup_framework_structure \"build-tvos-sim\" ${TVOS_MIN_OS_VERSION} \"tvos\"\nsetup_framework_structure \"build-tvos-device\" ${TVOS_MIN_OS_VERSION} \"tvos\"\n\n# Create dynamic libraries from static libraries\necho \"Creating dynamic libraries from static libraries...\"\ncombine_static_libraries \"build-ios-sim\" \"Release-iphonesimulator\" \"ios\" \"true\"\ncombine_static_libraries \"build-ios-device\" \"Release-iphoneos\" \"ios\" \"false\"\ncombine_static_libraries \"build-macos\" \"Release\" \"macos\" \"false\"\ncombine_static_libraries \"build-visionos\" \"Release-xros\" \"visionos\" \"false\"\ncombine_static_libraries \"build-visionos-sim\" \"Release-xrsimulator\" \"visionos\" \"true\"\ncombine_static_libraries \"build-tvos-sim\" \"Release-appletvsimulator\" \"tvos\" \"true\"\ncombine_static_libraries \"build-tvos-device\" \"Release-appletvos\" \"tvos\" \"false\"\n\n# Create XCFramework with correct debug symbols paths\necho \"Creating XCFramework...\"\n\nif [[ \"${BUILD_STATIC_XCFRAMEWORK}\" == \"ON\" ]]; then\n    xcodebuild -create-xcframework \\\n        -framework $(pwd)/build-ios-sim/framework/whisper.framework \\\n        -framework $(pwd)/build-ios-device/framework/whisper.framework \\\n        -framework $(pwd)/build-macos/framework/whisper.framework \\\n        -framework $(pwd)/build-visionos/framework/whisper.framework \\\n        -framework $(pwd)/build-visionos-sim/framework/whisper.framework \\\n        -framework $(pwd)/build-tvos-device/framework/whisper.framework \\\n        -framework $(pwd)/build-tvos-sim/framework/whisper.framework \\\n        -output $(pwd)/build-apple/whisper.xcframework\n    exit 0\nfi\n\nxcodebuild -create-xcframework \\\n    -framework $(pwd)/build-ios-sim/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-ios-sim/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-ios-device/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-ios-device/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-macos/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-macos/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-visionos/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-visionos/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-visionos-sim/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-visionos-sim/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-tvos-device/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-tvos-device/dSYMs/whisper.dSYM \\\n    -framework $(pwd)/build-tvos-sim/framework/whisper.framework \\\n    -debug-symbols $(pwd)/build-tvos-sim/dSYMs/whisper.dSYM \\\n    -output $(pwd)/build-apple/whisper.xcframework\n"
  },
  {
    "path": "ci/README.md",
    "content": "# CI\n\nIn addition to [Github Actions](https://github.com/ggerganov/whisper.cpp/actions) `whisper.cpp` uses a custom CI framework:\n\nhttps://github.com/ggml-org/ci\n\nIt monitors the `master` branch for new commits and runs the\n[ci/run.sh](https://github.com/ggerganov/whisper.cpp/blob/master/ci/run.sh) script on dedicated cloud instances. This allows us\nto execute heavier workloads compared to just using Github Actions. Also with time, the cloud instances will be scaled\nto cover various hardware architectures, including GPU and Apple Silicon instances.\n\nCollaborators can optionally trigger the CI run by adding the `ggml-ci` keyword to their commit message.\nOnly the branches of this repo are monitored for this keyword.\n\nIt is a good practice, before publishing changes to execute the full CI locally on your machine:\n\n```bash\nmkdir tmp\n\n# CPU-only build\nbash ./ci/run.sh ./tmp/results ./tmp/mnt\n\n# with CUDA support\nGG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n```\n\n## Environment Variables\n\nThe CI script supports several environment variables to control the build:\n\n| Variable | Description |\n|----------|-------------|\n| `GG_BUILD_CUDA` | Enable NVIDIA CUDA GPU acceleration |\n| `GG_BUILD_SYCL` | Enable Intel SYCL acceleration |\n| `GG_BUILD_VULKAN` | Enable Vulkan GPU acceleration |\n| `GG_BUILD_METAL` | Enable Metal acceleration on Apple Silicon |\n| `GG_BUILD_BLAS` | Enable BLAS CPU acceleration |\n| `GG_BUILD_OPENVINO` | Enable OpenVINO support |\n| `GG_BUILD_COREML` | Enable Core ML support for Apple Neural Engine |\n| `GG_BUILD_LOW_PERF` | Limit tests for low-performance hardware |\n| `GG_BUILD_TEST_MODELS` | Comma-separated list of models to test (e.g. \"tiny.en,tiny,base,medium\", defaults to all models unless `GG_BUILD_LOW_PERF` is set) |\n"
  },
  {
    "path": "ci/run.sh",
    "content": "#!/bin/bash\n#\n# sample usage:\n#\n# mkdir tmp\n#\n# # CPU-only build\n# bash ./ci/run.sh ./tmp/results ./tmp/mnt\n#\n# # with CUDA support\n# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n#\n# # with SYCL support\n# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt\n\nif [ -z \"$2\" ]; then\n    echo \"usage: $0 <output-dir> <mnt-dir>\"\n    exit 1\nfi\n\nmkdir -p \"$1\"\nmkdir -p \"$2\"\n\nOUT=$(realpath \"$1\")\nMNT=$(realpath \"$2\")\n\nrm -vf $OUT/*.log\nrm -vf $OUT/*.exit\nrm -vf $OUT/*.md\n\nsd=`dirname $0`\ncd $sd/../\nSRC=`pwd`\n\nALL_MODELS=( \"tiny.en\" \"tiny\" \"base.en\" \"base\" \"small.en\" \"small\" \"medium.en\" \"medium\" \"large-v1\" \"large-v2\" \"large-v3\" \"large-v3-turbo\" )\nBENCH_N_THREADS=4\nBENCH_ENCODER_ONLY=0\nBENCH_FLASH_ATTN=0\n\n# check for user-specified models first. if not specified, use fast models\nif [ ! -z ${GG_BUILD_TEST_MODELS} ]; then\n    IFS=',' read -r -a MODELS <<< \"${GG_BUILD_TEST_MODELS}\"\nelse\n    if [ ! -z ${GG_BUILD_LOW_PERF} ]; then\n        MODELS=( \"tiny\" \"base\" \"small\" )\n    else\n        MODELS=(\"${ALL_MODELS[@]}\")\n    fi\nfi\n\nCMAKE_EXTRA=\"-DWHISPER_FATAL_WARNINGS=ON\"\n\nif [ ! -z ${GG_BUILD_METAL} ]; then\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_METAL=ON\"\nfi\n\nif [ ! -z ${GG_BUILD_CUDA} ]; then\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_CUDA=ON\"\n\n    if command -v nvidia-smi >/dev/null 2>&1; then\n        CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')\n        if [[ -n \"$CUDA_ARCH\" && \"$CUDA_ARCH\" =~ ^[0-9]+$ ]]; then\n            CMAKE_EXTRA=\"${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}\"\n        else\n            echo \"Warning: Using fallback CUDA architectures\"\n            CMAKE_EXTRA=\"${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89\"\n        fi\n    else\n        echo \"Error: nvidia-smi not found, cannot build with CUDA\"\n        exit 1\n    fi\nfi\n\nif [ ! -z ${GG_BUILD_ROCM} ]; then\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_HIP=ON\"\n    if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then\n        echo \"Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)\"\n        exit 1\n    fi\n\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}\"\nfi\n\nif [ ! -z ${GG_BUILD_SYCL} ]; then\n    if [ -z ${ONEAPI_ROOT} ]; then\n        echo \"Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:\"\n        echo \"source /opt/intel/oneapi/setvars.sh\"\n        exit 1\n    fi\n    # Use only main GPU\n    export ONEAPI_DEVICE_SELECTOR=\"level_zero:0\"\n    # Enable sysman for correct memory reporting\n    export ZES_ENABLE_SYSMAN=1\n    # to circumvent precision issues on CPY operations\n    export SYCL_PROGRAM_COMPILE_OPTIONS=\"-cl-fp32-correctly-rounded-divide-sqrt\"\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON\"\nfi\n\nif [ ! -z ${GG_BUILD_VULKAN} ]; then\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_VULKAN=1\"\n\n    # if on Mac, disable METAL\n    if [[ \"$OSTYPE\" == \"darwin\"* ]]; then\n        CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF\"\n    fi\n\nfi\n\nif [ ! -z ${GG_BUILD_WEBGPU} ]; then\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_WEBGPU=1\"\nfi\n\nif [ ! -z ${GG_BUILD_MUSA} ]; then\n    # Use qy1 by default (MTT S80)\n    MUSA_ARCH=${MUSA_ARCH:-21}\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}\"\nfi\n\nif [ ! -z ${GG_BUILD_NO_SVE} ]; then\n    # arm 9 and newer enables sve by default, adjust these flags depending on the cpu used\n    CMAKE_EXTRA=\"${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm\"\nfi\n\n## helpers\n\n# download a file if it does not exist or if it is outdated\nfunction gg_wget {\n    local out=$1\n    local url=$2\n\n    local cwd=`pwd`\n\n    mkdir -p $out\n    cd $out\n\n    # should not re-download if file is the same\n    wget -nv -N $url\n\n    cd $cwd\n}\n\nfunction gg_download_model {\n    local model_name=$1\n    local model_file=\"$MNT/models/ggml-${model_name}.bin\"\n\n    if [ ! -f ${model_file} ]; then\n        local cwd=`pwd`\n        mkdir -p \"$MNT/models\"\n        cd \"$MNT/models\"\n        bash \"$cwd/models/download-ggml-model.sh\" ${model_name} .\n        cd \"$cwd\"\n    fi\n}\n\nfunction gg_printf {\n    printf -- \"$@\" >> $OUT/README.md\n}\n\n# Helper function to check command exit status\nfunction gg_check_last_command_status {\n    local exit_file=$1\n    local command_name=$2\n\n    local exit_status=$?\n    echo \"$exit_status\" > \"$exit_file\"\n\n    if [ $exit_status -ne 0 ]; then\n        echo \"Error: Command $command_name failed with exit status $exit_status\"\n        return 1\n    fi\n\n    return 0\n}\n\n# Usage: gg_run <test_name> [additional_args...]\n#\n# Parameters:\n#   test_name       - Name of the test to run (calls gg_run_<test_name>)\n#   additional_args - Any additional arguments to pass to the test function (first argument is appended to the log filename)\nfunction gg_run {\n    ci=$1\n\n    if [ $# -gt 1 ]; then\n        ci=\"${ci}_${2}\"\n    fi\n\n    set -o pipefail\n    set -x\n\n    gg_run_$1 \"$@\" | tee $OUT/$ci.log\n    cur=$?\n    echo \"$cur\" > $OUT/$ci.exit\n\n    set +x\n    set +o pipefail\n\n    gg_sum_$1 \"$@\"\n\n    ret=$((ret | cur))\n}\n\nfunction gg_check_build_requirements {\n    if ! command -v cmake &> /dev/null; then\n        gg_printf 'cmake not found, please install'\n    fi\n\n    if ! command -v make &> /dev/null; then\n        gg_printf 'make not found, please install'\n    fi\n}\n\n## ci\n\nfunction gg_run_ctest {\n    mode=$2\n\n    cd ${SRC}\n\n    rm -rf build-ci-${mode} && mkdir build-ci-${mode} && cd build-ci-${mode}\n\n    set -e\n\n    gg_check_build_requirements\n\n    (time cmake -DCMAKE_BUILD_TYPE=${mode} ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log\n    (time make -j$(nproc)                                    ) 2>&1 | tee -a $OUT/${ci}-make.log\n\n    (time ctest --output-on-failure -L main -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log\n\n    set +e\n}\n\nfunction gg_sum_ctest {\n    mode=$2\n\n    gg_printf '### %s\\n\\n' \"${ci}\"\n\n    gg_printf 'Runs ctest in '${mode}' mode\\n'\n    gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n    gg_printf '```\\n'\n    gg_printf '%s\\n' \"$(cat $OUT/${ci}-ctest.log)\"\n    gg_printf '```\\n'\n}\n\nfunction gg_run_bench {\n    cd ${SRC}\n\n    # set flash attention flag if enabled\n    fattn=\"-nfa\"\n    if [ \"$BENCH_FLASH_ATTN\" -eq 1 ]; then\n        fattn=\"-fa\"\n    fi\n\n    # run memcpy benchmark if not encoder-only mode\n    if [ \"$BENCH_ENCODER_ONLY\" -eq 0 ]; then\n        echo \"Running memcpy benchmark\"\n        (time ./build-ci-release/bin/whisper-bench -w 1 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-memcpy.log\n        gg_check_last_command_status \"$OUT/${ci}-memcpy.exit\" \"memcpy benchmark\"\n\n        echo \"Running ggml_mul_mat benchmark with $BENCH_N_THREADS threads\"\n        (time ./build-ci-release/bin/whisper-bench -w 2 -t $BENCH_N_THREADS 2>&1) | tee -a $OUT/${ci}-mul_mat.log\n        gg_check_last_command_status \"$OUT/${ci}-mul_mat.exit\" \"ggml_mul_mat benchmark\"\n    fi\n\n    echo \"Running benchmark for all models\"\n\n    # generate header for the benchmark table\n    {\n        printf \"| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\\n\" \"Config\" \"Model\" \"Th\" \"FA\" \"Enc.\" \"Dec.\" \"Bch5\" \"PP\" \"Commit\"\n        printf \"| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\\n\" \"---\" \"---\" \"---\" \"---\" \"---\" \"---\" \"---\" \"---\" \"---\"\n    } | tee -a $OUT/${ci}-models-table.log\n\n    res=0\n\n    # run benchmark for each model\n    for model in \"${MODELS[@]}\"; do\n        echo \"Benchmarking model: $model\"\n\n        # run the benchmark and capture output\n        output=$(./build-ci-release/bin/whisper-bench -m $MNT/models/ggml-$model.bin -t $BENCH_N_THREADS $fattn 2>&1)\n        ret=$?\n\n        # save the raw output\n        echo \"$output\" > $OUT/${ci}-bench-$model.log\n\n        if [ $ret -eq 0 ]; then\n            # parse the benchmark results\n            encode_time=$(echo \"$output\" | grep \"encode time\" | awk '{print $11}')\n            decode_time=$(echo \"$output\" | grep \"decode time\" | awk '{print $11}')\n            batchd_time=$(echo \"$output\" | grep \"batchd time\" | awk '{print $11}')\n            prompt_time=$(echo \"$output\" | grep \"prompt time\" | awk '{print $11}')\n            system_info=$(echo \"$output\" | grep \"system_info\")\n            actual_threads=$(echo \"$output\" | grep \"system_info\" | awk '{print $4}')\n\n            # determine configuration\n            config=\"\"\n            if [[ $system_info == *\"AVX2 = 1\"* ]]; then\n                config=\"$config AVX2\"\n            fi\n            if [[ $system_info == *\"NEON = 1\"* ]]; then\n                config=\"$config NEON\"\n            fi\n            if [[ $system_info == *\"BLAS = 1\"* ]]; then\n                config=\"$config BLAS\"\n            fi\n            if [[ $system_info == *\"COREML = 1\"* ]]; then\n                config=\"$config COREML\"\n            fi\n            if [[ $system_info == *\"CUDA = 1\"* ]]; then\n                config=\"$config CUDA\"\n            fi\n            if [[ $system_info == *\"METAL = 1\"* ]]; then\n                config=\"$config METAL\"\n            fi\n\n            # get commit hash\n            commit=$(git rev-parse --short HEAD)\n\n            # add row to benchmark table\n            printf \"| %16s | %13s | %3s | %3s | %7s | %7s | %7s | %7s | %7s |\\n\" \\\n                \"$config\" \"$model\" \"$actual_threads\" \"$BENCH_FLASH_ATTN\" \"$encode_time\" \"$decode_time\" \"$batchd_time\" \"$prompt_time\" \"$commit\" \\\n                | tee -a $OUT/${ci}-models-table.log\n        else\n            echo \"Benchmark failed for model: $model\" | tee -a $OUT/${ci}-bench-errors.log\n            res=1\n        fi\n    done\n\n    return $res\n}\n\nfunction gg_sum_bench {\n    gg_printf '### %s\\n\\n' \"${ci}\"\n\n    gg_printf 'Whisper Benchmark Results\\n'\n    gg_printf '- status: %s\\n' \"$(cat $OUT/${ci}.exit)\"\n\n    # show memcpy and ggml_mul_mat benchmark results if available\n    if [ \"$BENCH_ENCODER_ONLY\" -eq 0 ]; then\n        if [ -f \"$OUT/${ci}-memcpy.log\" ]; then\n            gg_printf '#### memcpy Benchmark\\n\\n'\n            gg_printf '```\\n%s\\n```\\n\\n' \"$(cat $OUT/${ci}-memcpy.log)\"\n        fi\n\n        if [ -f \"$OUT/${ci}-mul_mat.log\" ]; then\n            gg_printf '#### ggml_mul_mat Benchmark\\n\\n'\n            gg_printf '```\\n%s\\n```\\n\\n' \"$(cat $OUT/${ci}-mul_mat.log)\"\n        fi\n    fi\n\n    # show model benchmark results\n    gg_printf '#### Model Benchmarks\\n\\n'\n    if [ -f \"$OUT/${ci}-models-table.log\" ]; then\n        gg_printf '%s\\n\\n' \"$(cat $OUT/${ci}-models-table.log)\"\n    else\n        gg_printf 'No model benchmark results available.\\n\\n'\n    fi\n\n    # show any errors that occurred\n    if [ -f \"$OUT/${ci}-bench-errors.log\" ]; then\n        gg_printf '#### Benchmark Errors\\n\\n'\n        gg_printf '```\\n%s\\n```\\n\\n' \"$(cat $OUT/${ci}-bench-errors.log)\"\n    fi\n}\n\nret=0\n\nfor model in \"${MODELS[@]}\"; do\n    test $ret -eq 0 && gg_download_model ${model}\ndone\n\ntest $ret -eq 0 && gg_run ctest debug\ntest $ret -eq 0 && gg_run ctest release\n\ntest $ret -eq 0 && gg_run bench\n\ncat $OUT/README.md\n\nexit $ret\n"
  },
  {
    "path": "close-issue.yml",
    "content": "name: Close inactive issues\non:\n  schedule:\n    - cron: \"42 0 * * *\"\n\n# Fine-grant permission\n# https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token\npermissions:\n  issues: write\n\njobs:\n  close-issues:\n    runs-on: ubuntu-latest\n    permissions:\n      issues: write\n      pull-requests: write\n    steps:\n      - uses: actions/stale@v10\n        with:\n          exempt-issue-labels: \"refactor,help wanted,good first issue,research,bug,roadmap\"\n          days-before-issue-stale: 30\n          days-before-issue-close: 14\n          stale-issue-label: \"stale\"\n          close-issue-message: \"This issue was closed because it has been inactive for 14 days since being marked as stale.\"\n          days-before-pr-stale: -1\n          days-before-pr-close: -1\n          operations-per-run: 10000\n          repo-token: ${{ secrets.GITHUB_TOKEN }}\n"
  },
  {
    "path": "cmake/DefaultTargetOptions.cmake",
    "content": "# Set the default compile features and properties for a target.\n\nif (NOT TARGET)\n    message(FATAL_ERROR \"TARGET not set before including DefaultTargetOptions\")\nendif()\n\ntarget_compile_features(${TARGET}\n    PRIVATE\n        cxx_std_11\n    )\n\nset_target_properties(${TARGET}\n    PROPERTIES\n        EXPORT_COMPILE_COMMANDS ON\n        RUNTIME_OUTPUT_DIRECTORY \"${CMAKE_BINARY_DIR}/bin\"\n)\n"
  },
  {
    "path": "cmake/FindFFmpeg.cmake",
    "content": "# From\n# https://github.com/snikulov/cmake-modules/blob/master/FindFFmpeg.cmake\n#\n# vim: ts=2 sw=2\n# - Try to find the required ffmpeg components(default: AVFORMAT, AVUTIL, AVCODEC)\n#\n# Once done this will define\n#  FFMPEG_FOUND         - System has the all required components.\n#  FFMPEG_INCLUDE_DIRS  - Include directory necessary for using the required components headers.\n#  FFMPEG_LIBRARIES     - Link these to use the required ffmpeg components.\n#  FFMPEG_DEFINITIONS   - Compiler switches required for using the required ffmpeg components.\n#\n# For each of the components it will additionally set.\n#   - AVCODEC\n#   - AVDEVICE\n#   - AVFORMAT\n#   - AVFILTER\n#   - AVUTIL\n#   - POSTPROC\n#   - SWSCALE\n# the following variables will be defined\n#  <component>_FOUND        - System has <component>\n#  <component>_INCLUDE_DIRS - Include directory necessary for using the <component> headers\n#  <component>_LIBRARIES    - Link these to use <component>\n#  <component>_DEFINITIONS  - Compiler switches required for using <component>\n#  <component>_VERSION      - The components version\n#\n# Copyright (c) 2006, Matthias Kretz, <kretz@kde.org>\n# Copyright (c) 2008, Alexander Neundorf, <neundorf@kde.org>\n# Copyright (c) 2011, Michael Jansen, <kde@michael-jansen.biz>\n#\n# Redistribution and use is allowed according to the terms of the BSD license.\n# For details see the accompanying COPYING-CMAKE-SCRIPTS file.\n\ninclude(FindPackageHandleStandardArgs)\n\n# The default components were taken from a survey over other FindFFMPEG.cmake files\nif (NOT FFmpeg_FIND_COMPONENTS)\n  set(FFmpeg_FIND_COMPONENTS AVFORMAT AVCODEC AVUTIL SWRESAMPLE)\nendif()\n\n#\n### Macro: set_component_found\n#\n# Marks the given component as found if both *_LIBRARIES AND *_INCLUDE_DIRS is present.\n#\nmacro(set_component_found _component )\n  if (${_component}_LIBRARIES AND ${_component}_INCLUDE_DIRS)\n    message(DEBUG \"  - ${_component} found.\")\n    set(${_component}_FOUND TRUE)\n  else ()\n  message(DEBUG \"  - ${_component} not found.\")\n  endif ()\nendmacro()\n\n#\n### Macro: find_component\n#\n# Checks for the given component by invoking pkgconfig and then looking up the libraries and\n# include directories.\n#\nmacro(find_component _component _pkgconfig _library _header)\n\n  if (NOT WIN32)\n     # use pkg-config to get the directories and then use these values\n     # in the FIND_PATH() and FIND_LIBRARY() calls\n     find_package(PkgConfig)\n     if (PKG_CONFIG_FOUND)\n       pkg_check_modules(PC_${_component} ${_pkgconfig})\n       message(STATUS \"Pkgconfig found: ${PC_${_component}_INCLUDEDIR}\")\n       message(STATUS \"Pkgconfig found: ${PC_${_component}_INCLUDE_DIRS}\")\n       message(STATUS \"${PC_${_component}_CFLAGS}\")\n     endif ()\n  endif (NOT WIN32)\n\n\n  find_path(${_component}_INCLUDE_DIRS ${_header}\n    HINTS\n      ${PC_${_component}_INCLUDEDIR}\n      ${PC_${_component}_INCLUDE_DIRS}\n    PATH_SUFFIXES\n      ffmpeg\n  )\n\n  # CMake's default is to search first for shared libraries and then for static libraries.\n  # Todo later: add option to prefer static libs over dynamic:\n  find_library(${_component}_LIBRARIES NAMES ${_library} lib${_library}.a\n      HINTS\n      ${PC_${_component}_LIBDIR}\n      ${PC_${_component}_LIBRARY_DIRS}\n  )\n\n  set(${_component}_DEFINITIONS  ${PC_${_component}_CFLAGS_OTHER} CACHE STRING \"The ${_component} CFLAGS.\")\n  set(${_component}_VERSION      ${PC_${_component}_VERSION}      CACHE STRING \"The ${_component} version number.\")\n\n  set_component_found(${_component})\n\n  mark_as_advanced(\n    ${_component}_INCLUDE_DIRS\n    ${_component}_LIBRARIES\n    ${_component}_DEFINITIONS\n    ${_component}_VERSION)\n\nendmacro()\n\n\n# Check for cached results. If there are skip the costly part.\nif (NOT FFMPEG_LIBRARIES)\n\n  # Check for all possible component.\n  find_component(AVCODEC    libavcodec    avcodec  libavcodec/avcodec.h)\n  find_component(AVFORMAT   libavformat   avformat libavformat/avformat.h)\n  find_component(AVDEVICE   libavdevice   avdevice libavdevice/avdevice.h)\n  #find_component(AVRESAMPLE libavresample avresample libavresample/avresample.h) # old name for swresample\n  find_component(AVUTIL     libavutil     avutil   libavutil/avutil.h)\n  find_component(AVFILTER   libavfilter   avfilter libavfilter/avfilter.h)\n  find_component(SWSCALE    libswscale    swscale  libswscale/swscale.h)\n  find_component(POSTPROC   libpostproc   postproc libpostproc/postprocess.h)\n  find_component(SWRESAMPLE libswresample swresample libswresample/swresample.h)\n\n  # Check if the required components were found and add their stuff to the FFMPEG_* vars.\n  foreach (_component ${FFmpeg_FIND_COMPONENTS})\n    if (${_component}_FOUND)\n      # message(STATUS \"Required component ${_component} present.\")\n      set(FFMPEG_LIBRARIES   ${FFMPEG_LIBRARIES}   ${${_component}_LIBRARIES})\n      set(FFMPEG_DEFINITIONS ${FFMPEG_DEFINITIONS} ${${_component}_DEFINITIONS})\n      list(APPEND FFMPEG_INCLUDE_DIRS ${${_component}_INCLUDE_DIRS})\n    else ()\n      # message(STATUS \"Required component ${_component} missing.\")\n    endif ()\n  endforeach ()\n\n  # Build the include path with duplicates removed.\n  if (FFMPEG_INCLUDE_DIRS)\n    list(REMOVE_DUPLICATES FFMPEG_INCLUDE_DIRS)\n  endif ()\n\n  # cache the vars.\n  set(FFMPEG_INCLUDE_DIRS ${FFMPEG_INCLUDE_DIRS} CACHE STRING \"The FFmpeg include directories.\" FORCE)\n  set(FFMPEG_LIBRARIES    ${FFMPEG_LIBRARIES}    CACHE STRING \"The FFmpeg libraries.\" FORCE)\n  set(FFMPEG_DEFINITIONS  ${FFMPEG_DEFINITIONS}  CACHE STRING \"The FFmpeg cflags.\" FORCE)\n\n  mark_as_advanced(FFMPEG_INCLUDE_DIRS\n                   FFMPEG_LIBRARIES\n                   FFMPEG_DEFINITIONS)\n\nendif ()\n\n# Now set the noncached _FOUND vars for the components.\n# whisper.cpp does not need SWSCALE\nforeach (_component AVCODEC AVDEVICE AVFORMAT AVRESAMPLE AVUTIL POSTPROCESS)\n  set_component_found(${_component})\nendforeach ()\n\n# Compile the list of required vars\nset(_FFmpeg_REQUIRED_VARS FFMPEG_LIBRARIES FFMPEG_INCLUDE_DIRS)\nforeach (_component ${FFmpeg_FIND_COMPONENTS})\n  list(APPEND _FFmpeg_REQUIRED_VARS ${_component}_LIBRARIES ${_component}_INCLUDE_DIRS)\nendforeach ()\n\n# Give a nice error message if some of the required vars are missing.\nfind_package_handle_standard_args(FFmpeg DEFAULT_MSG ${_FFmpeg_REQUIRED_VARS})\n\n"
  },
  {
    "path": "cmake/arm64-apple-clang.cmake",
    "content": "set( CMAKE_SYSTEM_NAME Darwin )\nset( CMAKE_SYSTEM_PROCESSOR arm64 )\n\nset( target arm64-apple-darwin-macho )\n\nset( CMAKE_C_COMPILER    clang )\nset( CMAKE_CXX_COMPILER  clang++ )\n\nset( CMAKE_C_COMPILER_TARGET   ${target} )\nset( CMAKE_CXX_COMPILER_TARGET ${target} )\n\nset( arch_c_flags \"-march=armv8.4-a -fvectorize -ffp-model=fast -fno-finite-math-only\" )\nset( warn_c_flags \"-Wno-format -Wno-unused-variable -Wno-unused-function\" )\n\nset( CMAKE_C_FLAGS_INIT   \"${arch_c_flags} ${warn_c_flags}\" )\nset( CMAKE_CXX_FLAGS_INIT \"${arch_c_flags} ${warn_c_flags}\" )\n"
  },
  {
    "path": "cmake/arm64-windows-llvm.cmake",
    "content": "set( CMAKE_SYSTEM_NAME Windows )\nset( CMAKE_SYSTEM_PROCESSOR arm64 )\n\nset( target arm64-pc-windows-msvc )\n\nset( CMAKE_C_COMPILER    clang )\nset( CMAKE_CXX_COMPILER  clang++ )\n\nset( CMAKE_C_COMPILER_TARGET   ${target} )\nset( CMAKE_CXX_COMPILER_TARGET ${target} )\n\nset( arch_c_flags \"-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only\" )\nset( warn_c_flags \"-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments\" )\n\nset( CMAKE_C_FLAGS_INIT   \"${arch_c_flags} ${warn_c_flags}\" )\nset( CMAKE_CXX_FLAGS_INIT \"${arch_c_flags} ${warn_c_flags}\" )\n"
  },
  {
    "path": "cmake/build-info.cmake",
    "content": "set(BUILD_NUMBER 0)\nset(BUILD_COMMIT \"unknown\")\nset(BUILD_COMPILER \"unknown\")\nset(BUILD_TARGET \"unknown\")\n\n# Look for git\nfind_package(Git)\nif(NOT Git_FOUND)\n    find_program(GIT_EXECUTABLE NAMES git git.exe)\n    if(GIT_EXECUTABLE)\n        set(Git_FOUND TRUE)\n        message(STATUS \"Found Git: ${GIT_EXECUTABLE}\")\n    else()\n        message(WARNING \"Git not found. Build info will not be accurate.\")\n    endif()\nendif()\n\n# Get the commit count and hash\nif(Git_FOUND)\n    execute_process(\n        COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n        OUTPUT_VARIABLE HEAD\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n        RESULT_VARIABLE RES\n    )\n    if (RES EQUAL 0)\n        set(BUILD_COMMIT ${HEAD})\n    endif()\n    execute_process(\n        COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n        OUTPUT_VARIABLE COUNT\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n        RESULT_VARIABLE RES\n    )\n    if (RES EQUAL 0)\n        set(BUILD_NUMBER ${COUNT})\n    endif()\nendif()\n\nif(MSVC)\n    set(BUILD_COMPILER \"${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}\")\n    set(BUILD_TARGET ${CMAKE_VS_PLATFORM_NAME})\n    add_compile_options(\"$<$<COMPILE_LANGUAGE:C>:/utf-8>\")\n    add_compile_options(\"$<$<COMPILE_LANGUAGE:CXX>:/utf-8>\")\nelse()\n    execute_process(\n        COMMAND sh -c \"$@ --version | head -1\" _ ${CMAKE_C_COMPILER}\n        OUTPUT_VARIABLE OUT\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n    )\n    set(BUILD_COMPILER ${OUT})\n    execute_process(\n        COMMAND ${CMAKE_C_COMPILER} -dumpmachine\n        OUTPUT_VARIABLE OUT\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n    )\n    set(BUILD_TARGET ${OUT})\nendif()\n"
  },
  {
    "path": "cmake/git-vars.cmake",
    "content": "find_package(Git)\n\n# the commit's SHA1\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" describe --match=NeVeRmAtCh --always --abbrev=8\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_SHA1\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the date of the commit\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" log -1 --format=%ad --date=local\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_DATE\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the subject of the commit\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" log -1 --format=%s\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_COMMIT_SUBJECT\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n"
  },
  {
    "path": "cmake/riscv64-spacemit-linux-gnu-gcc.cmake",
    "content": "set(CMAKE_SYSTEM_NAME Linux)\nset(CMAKE_SYSTEM_PROCESSOR riscv64)\nset(CMAKE_SYSTEM_VERSION 1)\n\nif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES \"^(riscv)\")\n    message(STATUS \"HOST SYSTEM ${CMAKE_HOST_SYSTEM_PROCESSOR}\")\nelse()\n    set(GNU_MACHINE riscv64-unknown-linux-gnu CACHE STRING \"GNU compiler triple\")\n    if (DEFINED ENV{RISCV_ROOT_PATH})\n        file(TO_CMAKE_PATH $ENV{RISCV_ROOT_PATH} RISCV_ROOT_PATH)\n    else()\n        message(FATAL_ERROR \"RISCV_ROOT_PATH env must be defined\")\n    endif()\n\n    set(RISCV_ROOT_PATH ${RISCV_ROOT_PATH} CACHE STRING \"root path to riscv toolchain\")\n    set(CMAKE_C_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-gcc)\n    set(CMAKE_CXX_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-g++)\n    set(CMAKE_STRIP ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-strip)\n    set(CMAKE_FIND_ROOT_PATH \"${RISCV_ROOT_PATH}/riscv64-unknown-linux-gnu\")\n    set(CMAKE_SYSROOT \"${RISCV_ROOT_PATH}/sysroot\")\nendif()\n\nset(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)\nset(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)\nset(CMAKE_C_FLAGS \"-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CMAKE_C_FLAGS}\")\nset(CMAKE_CXX_FLAGS \"-march=rv64gcv_zfh_zba_zicbop -mabi=lp64d ${CXX_FLAGS}\")\nset(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} -latomic\")\n"
  },
  {
    "path": "cmake/whisper-config.cmake.in",
    "content": "set(WHISPER_VERSION      @WHISPER_INSTALL_VERSION@)\nset(WHISPER_BUILD_COMMIT @WHISPER_BUILD_COMMIT@)\nset(WHISPER_BUILD_NUMBER @WHISPER_BUILD_NUMBER@)\nset(WHISPER_SHARED_LIB   @BUILD_SHARED_LIBS@)\n\n@PACKAGE_INIT@\n\nset_and_check(WHISPER_INCLUDE_DIR \"@PACKAGE_WHISPER_INCLUDE_INSTALL_DIR@\")\nset_and_check(WHISPER_LIB_DIR     \"@PACKAGE_WHISPER_LIB_INSTALL_DIR@\")\nset_and_check(WHISPER_BIN_DIR     \"@PACKAGE_WHISPER_BIN_INSTALL_DIR@\")\n\nfind_package(ggml REQUIRED HINTS ${LLAMA_LIB_DIR}/cmake)\n\nfind_library(whisper_LIBRARY whisper\n    REQUIRED\n    HINTS ${WHISPER_LIB_DIR}\n    NO_CMAKE_FIND_ROOT_PATH\n)\n\nadd_library(whisper UNKNOWN IMPORTED)\nset_target_properties(whisper\n    PROPERTIES\n    INTERFACE_INCLUDE_DIRECTORIES \"${WHISPER_INCLUDE_DIR}\"\n        INTERFACE_LINK_LIBRARIES \"ggml::ggml;ggml::ggml-base;\"\n        IMPORTED_LINK_INTERFACE_LANGUAGES \"CXX\"\n        IMPORTED_LOCATION \"${whisper_LIBRARY}\"\n        INTERFACE_COMPILE_FEATURES cxx_std_11\n        POSITION_INDEPENDENT_CODE ON )\n\ncheck_required_components(whisper)\n"
  },
  {
    "path": "cmake/whisper.pc.in",
    "content": "prefix=@CMAKE_INSTALL_PREFIX@\nexec_prefix=${prefix}\nlibdir=${exec_prefix}/lib\nincludedir=${prefix}/include\n\nName: whisper\nDescription: Port of OpenAI's Whisper model in C/C++\nVersion: @PROJECT_VERSION@\nLibs: -L${libdir} -lggml  -lggml-base -lwhisper\nCflags: -I${includedir}\n"
  },
  {
    "path": "cmake/x64-windows-llvm.cmake",
    "content": "set( CMAKE_SYSTEM_NAME Windows )\nset( CMAKE_SYSTEM_PROCESSOR x86_64 )\n\nset( CMAKE_C_COMPILER    clang )\nset( CMAKE_CXX_COMPILER  clang++ )\n"
  },
  {
    "path": "examples/CMakeLists.txt",
    "content": "# dependencies\n\nfind_package(Threads REQUIRED)\n\n# third-party\n\nif (WHISPER_SDL2)\n    # SDL2\n    find_package(SDL2 REQUIRED)\n\n    string(STRIP \"${SDL2_LIBRARIES}\" SDL2_LIBRARIES)\n\n    message(STATUS \"SDL2_INCLUDE_DIRS = ${SDL2_INCLUDE_DIRS}\")\n    message(STATUS \"SDL2_LIBRARIES    = ${SDL2_LIBRARIES}\")\nendif()\n\n# common\n\nset(TARGET common)\n\nunset(COMMON_EXTRA_LIBS)\n\nif (WHISPER_FFMPEG)\n    # As of cmake 3.27, there is no official cmake support for FindFFmpeg.\n    # Consequnelty we added a FindFFmpeg.cmake script the cmake subfolder:\n    # whisper.cpp does not need the full ffmpeg libs, just AVFORMAT AVCODEC AVUTIL SWRESAMPLE\n    # libswresample  performs highly optimized audio resampling, rematrixing and sample format conversion operations\n    # libavcodec provides a generic encoding/decoding framework and contains multiple decoders and encoders for audio, video and subtitle streams, and several bitstream filters.\n    # libavformat provides a generic framework for multiplexing and demultiplexing (muxing and demuxing) audio, video and subtitle streams.\n    find_package(FFmpeg REQUIRED)\n\n    if (NOT ${FFMPEG_FOUND})\n        message(FATAL_ERROR \"Cannot find ffmpeg libs/headers\")\n    endif()\n\n    message(STATUS \"Found ffmpeg libs:       ${FFMPEG_LIBRARIES}\")\n    message(STATUS \"Found ffmpeg headers in: ${FFMPEG_INCLUDE_DIRS}\")\n    message(STATUS \"ffmpeg definitions:      ${FFMPEG_DEFINITIONS}\")\n    message(STATUS \"Found avformat           ${AVFORMAT_VERSION}\")\n\n    include_directories(${FFMPEG_INCLUDE_DIRS})\n    add_compile_definitions(WHISPER_FFMPEG)\n\n    list(APPEND COMMON_EXTRA_LIBS ${FFMPEG_LIBRARIES})\n\n    set(COMMON_SOURCES_FFMPEG ffmpeg-transcode.cpp)\nendif()\n\n\nadd_library(${TARGET} STATIC\n    common.h\n    common.cpp\n    common-ggml.h\n    common-ggml.cpp\n    common-whisper.h\n    common-whisper.cpp\n    grammar-parser.h\n    grammar-parser.cpp\n    ${COMMON_SOURCES_FFMPEG}\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE whisper ${COMMON_EXTRA_LIBS} ${CMAKE_DL_LIBS})\n\nset_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)\nset_target_properties(${TARGET} PROPERTIES FOLDER \"libs\")\n\nif (WHISPER_SDL2)\n    # common-sdl\n\n    set(TARGET common-sdl)\n\n    add_library(${TARGET} STATIC\n        common-sdl.h\n        common-sdl.cpp\n        )\n\n    include(DefaultTargetOptions)\n\n    target_include_directories(${TARGET} PUBLIC  ${SDL2_INCLUDE_DIRS})\n    target_link_libraries     (${TARGET} PRIVATE ${SDL2_LIBRARIES})\n\n    set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)\n    set_target_properties(${TARGET} PROPERTIES FOLDER \"libs\")\nendif()\n\n# add json lib\nadd_library(json_cpp INTERFACE)\ntarget_include_directories(json_cpp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})\n\n# examples\n\ninclude_directories(${CMAKE_CURRENT_SOURCE_DIR})\n\nif (EMSCRIPTEN)\n    add_subdirectory(whisper.wasm)\n    add_subdirectory(stream.wasm)\n    add_subdirectory(command.wasm)\n    add_subdirectory(bench.wasm)\n    add_subdirectory(wchess)\nelseif(CMAKE_JS_VERSION)\n    add_subdirectory(addon.node)\nelse()\n    add_subdirectory(cli)\n    add_subdirectory(bench)\n    add_subdirectory(server)\n    add_subdirectory(quantize)\n    add_subdirectory(vad-speech-segments)\n    if (WHISPER_SDL2)\n        add_subdirectory(stream)\n        add_subdirectory(command)\n        add_subdirectory(talk-llama)\n        add_subdirectory(lsp)\n        if (GGML_SYCL)\n            add_subdirectory(sycl)\n        endif()\n    endif (WHISPER_SDL2)\n\n    add_subdirectory(deprecation-warning)\nendif()\n\nif (WHISPER_SDL2)\n    add_subdirectory(wchess)\nendif (WHISPER_SDL2)\n"
  },
  {
    "path": "examples/addon.node/.gitignore",
    "content": ".idea\nnode_modules\nbuild\n"
  },
  {
    "path": "examples/addon.node/CMakeLists.txt",
    "content": "set(TARGET addon.node)\n\n# Base settings\n#==================================================================\n# env var supported by cmake-js\nadd_definitions(-DNAPI_VERSION=4)\ninclude_directories(${CMAKE_JS_INC})\n#==================================================================\n\nadd_library(${TARGET} SHARED ${CMAKE_JS_SRC} addon.cpp)\nset_target_properties(${TARGET} PROPERTIES PREFIX \"\" SUFFIX \".node\")\n\ninclude(DefaultTargetOptions)\n\n# Include N-API wrappers\n#==================================================================\nexecute_process(COMMAND node -p \"require('node-addon-api').include\"\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n        OUTPUT_VARIABLE NODE_ADDON_API_DIR\n        )\nstring(REPLACE \"\\n\" \"\" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})\nstring(REPLACE \"\\\"\" \"\" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})\ntarget_include_directories(${TARGET} PRIVATE ${NODE_ADDON_API_DIR})\n#==================================================================\n\ntarget_link_libraries(${TARGET} ${CMAKE_JS_LIB} common whisper ${CMAKE_THREAD_LIBS_INIT})\n\nif(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)\n    # Generate node.lib\n    execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})\nendif()\n"
  },
  {
    "path": "examples/addon.node/README.md",
    "content": "# whisper.cpp Node.js addon\n\nThis is an addon demo that can **perform whisper model reasoning in `node` and `electron` environments**, based on [cmake-js](https://github.com/cmake-js/cmake-js).\nIt can be used as a reference for using the whisper.cpp project in other node projects.\n\nThis addon now supports **Voice Activity Detection (VAD)** for improved transcription performance.\n\n## Install\n\n```shell\nnpm install\n```\n\n## Compile\n\nMake sure it is in the project root directory and compiled with make-js.\n\n```shell\nnpx cmake-js compile -T addon.node -B Release\n```\n\nFor Electron addon and cmake-js options, you can see [cmake-js](https://github.com/cmake-js/cmake-js) and make very few configuration changes.\n\n> Such as appointing special cmake path:\n> ```shell\n> npx cmake-js compile -c 'xxx/cmake' -T addon.node -B Release\n> ```\n\n## Run\n\n### Basic Usage\n\n```shell\ncd examples/addon.node\n\nnode index.js --language='language' --model='model-path' --fname_inp='file-path'\n```\n\n### VAD (Voice Activity Detection) Usage\n\nRun the VAD example with performance comparison:\n\n```shell\nnode vad-example.js\n```\n\n## Voice Activity Detection (VAD) Support\n\nVAD can significantly improve transcription performance by only processing speech segments, which is especially beneficial for audio files with long periods of silence.\n\n### VAD Model Setup\n\nBefore using VAD, download a VAD model:\n\n```shell\n# From the whisper.cpp root directory\n./models/download-vad-model.sh silero-v6.2.0\n```\n\n### VAD Parameters\n\nAll VAD parameters are optional and have sensible defaults:\n\n- `vad`: Enable VAD (default: false)\n- `vad_model`: Path to VAD model file (required when VAD enabled)\n- `vad_threshold`: Speech detection threshold 0.0-1.0 (default: 0.5)\n- `vad_min_speech_duration_ms`: Min speech duration in ms (default: 250)\n- `vad_min_silence_duration_ms`: Min silence duration in ms (default: 100)\n- `vad_max_speech_duration_s`: Max speech duration in seconds (default: FLT_MAX)\n- `vad_speech_pad_ms`: Speech padding in ms (default: 30)\n- `vad_samples_overlap`: Sample overlap 0.0-1.0 (default: 0.1)\n\n### JavaScript API Example\n\n```javascript\nconst path = require(\"path\");\nconst { whisper } = require(path.join(__dirname, \"../../build/Release/addon.node\"));\nconst { promisify } = require(\"util\");\n\nconst whisperAsync = promisify(whisper);\n\n// With VAD enabled\nconst vadParams = {\n  language: \"en\",\n  model: path.join(__dirname, \"../../models/ggml-base.en.bin\"),\n  fname_inp: path.join(__dirname, \"../../samples/jfk.wav\"),\n  vad: true,\n  vad_model: path.join(__dirname, \"../../models/ggml-silero-v6.2.0.bin\"),\n  vad_threshold: 0.5,\n  progress_callback: (progress) => console.log(`Progress: ${progress}%`)\n};\n\nwhisperAsync(vadParams).then(result => console.log(result));\n```\n\n## Supported Parameters\n\nBoth traditional whisper.cpp parameters and new VAD parameters are supported:\n\n- `language`: Language code (e.g., \"en\", \"es\", \"fr\")\n- `model`: Path to whisper model file\n- `fname_inp`: Path to input audio file\n- `use_gpu`: Enable GPU acceleration (default: true)\n- `flash_attn`: Enable flash attention (default: false)\n- `no_prints`: Disable console output (default: false)\n- `no_timestamps`: Disable timestamps (default: false)\n- `detect_language`: Auto-detect language (default: false)\n- `audio_ctx`: Audio context size (default: 0)\n- `max_len`: Maximum segment length (default: 0)\n- `max_context`: Maximum context size (default: -1)\n- `prompt`: Initial prompt for decoder\n- `comma_in_time`: Use comma in timestamps (default: true)\n- `print_progress`: Print progress info (default: false)\n- `progress_callback`: Progress callback function\n- VAD parameters (see above section)\n"
  },
  {
    "path": "examples/addon.node/__test__/whisper.spec.js",
    "content": "const { join } = require('path');\nconst { whisper } = require('../../../build/Release/addon.node');\nconst { promisify } = require('util');\n\nconst whisperAsync = promisify(whisper);\n\nconst commonParams = {\n  language: 'en',\n  model: join(__dirname, '../../../models/ggml-base.en.bin'),\n  fname_inp: join(__dirname, '../../../samples/jfk.wav'),\n  use_gpu: true,\n  flash_attn: false,\n  no_prints: true,\n  no_timestamps: false,\n  detect_language: false,\n  audio_ctx: 0,\n  max_len: 0\n};\n\ndescribe('Whisper.cpp Node.js addon with VAD support', () => {\n  test('Basic whisper transcription without VAD', async () => {\n    const params = {\n      ...commonParams,\n      vad: false\n    };\n\n    const result = await whisperAsync(params);\n    \n    expect(typeof result).toBe('object');\n    expect(Array.isArray(result.transcription)).toBe(true);\n    expect(result.transcription.length).toBeGreaterThan(0);\n    \n    // Check that we got some transcription text\n    const text = result.transcription.map(segment => segment[2]).join(' ');\n    expect(text.length).toBeGreaterThan(0);\n    expect(text.toLowerCase()).toContain('ask not');\n  }, 30000);\n\n  test('VAD parameters validation', async () => {\n    // Test with invalid VAD model - should return empty transcription\n    const invalidParams = {\n      ...commonParams,\n      vad: true,\n      vad_model: 'non-existent-model.bin',\n      vad_threshold: 0.5\n    };\n\n    // This should handle the error gracefully and return empty transcription\n    const result = await whisperAsync(invalidParams);\n    expect(typeof result).toBe('object');\n    expect(Array.isArray(result.transcription)).toBe(true);\n    // When VAD model doesn't exist, it should return empty transcription\n    expect(result.transcription.length).toBe(0);\n  }, 10000);\n\n  test('VAD parameter parsing', async () => {\n    // Test that VAD parameters are properly parsed (even if VAD model doesn't exist)\n    const vadParams = {\n      ...commonParams,\n      vad: false, // Disabled so no model required\n      vad_threshold: 0.7,\n      vad_min_speech_duration_ms: 300,\n      vad_min_silence_duration_ms: 150,\n      vad_max_speech_duration_s: 45.0,\n      vad_speech_pad_ms: 50,\n      vad_samples_overlap: 0.15\n    };\n\n    const result = await whisperAsync(vadParams);\n    \n    expect(typeof result).toBe('object');\n    expect(Array.isArray(result.transcription)).toBe(true);\n  }, 30000);\n\n  test('Progress callback with VAD disabled', async () => {\n    let progressCalled = false;\n    let lastProgress = 0;\n\n    const params = {\n      ...commonParams,\n      vad: false,\n      progress_callback: (progress) => {\n        progressCalled = true;\n        lastProgress = progress;\n        expect(progress).toBeGreaterThanOrEqual(0);\n        expect(progress).toBeLessThanOrEqual(100);\n      }\n    };\n\n    const result = await whisperAsync(params);\n    \n    expect(progressCalled).toBe(true);\n    expect(lastProgress).toBe(100);\n    expect(typeof result).toBe('object');\n  }, 30000);\n\n  test('Language detection without VAD', async () => {\n    const params = {\n      ...commonParams,\n      vad: false,\n      detect_language: true,\n      language: 'auto'\n    };\n\n    const result = await whisperAsync(params);\n    \n    expect(typeof result).toBe('object');\n    expect(typeof result.language).toBe('string');\n    expect(result.language.length).toBeGreaterThan(0);\n  }, 30000);\n\n  test('Basic transcription with all VAD parameters set', async () => {\n    // Test with VAD disabled but all parameters set to ensure no crashes\n    const params = {\n      ...commonParams,\n      vad: false, // Disabled so it works without VAD model\n      vad_model: '', // Empty model path\n      vad_threshold: 0.6,\n      vad_min_speech_duration_ms: 200,\n      vad_min_silence_duration_ms: 80,\n      vad_max_speech_duration_s: 25.0,\n      vad_speech_pad_ms: 40,\n      vad_samples_overlap: 0.08\n    };\n\n    const result = await whisperAsync(params);\n    \n    expect(typeof result).toBe('object');\n    expect(Array.isArray(result.transcription)).toBe(true);\n    expect(result.transcription.length).toBeGreaterThan(0);\n  }, 30000);\n});\n\n"
  },
  {
    "path": "examples/addon.node/addon.cpp",
    "content": "#include \"napi.h\"\n#include \"common.h\"\n#include \"common-whisper.h\"\n\n#include \"whisper.h\"\n\n#include <string>\n#include <thread>\n#include <vector>\n#include <cmath>\n#include <cstdint>\n#include <cfloat>\n\nstruct whisper_params {\n    int32_t n_threads    = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t n_processors = 1;\n    int32_t offset_t_ms  = 0;\n    int32_t offset_n     = 0;\n    int32_t duration_ms  = 0;\n    int32_t max_context  = -1;\n    int32_t max_len      = 0;\n    int32_t best_of      = 5;\n    int32_t beam_size    = -1;\n    int32_t audio_ctx    = 0;\n\n    float word_thold    = 0.01f;\n    float entropy_thold = 2.4f;\n    float logprob_thold = -1.0f;\n\n    bool translate      = false;\n    bool diarize        = false;\n    bool output_txt     = false;\n    bool output_vtt     = false;\n    bool output_srt     = false;\n    bool output_wts     = false;\n    bool output_csv     = false;\n    bool print_special  = false;\n    bool print_colors   = false;\n    bool print_progress = false;\n    bool no_timestamps  = false;\n    bool no_prints      = false;\n    bool detect_language= false;\n    bool use_gpu        = true;\n    bool flash_attn     = false;\n    bool comma_in_time  = true;\n\n    std::string language = \"en\";\n    std::string prompt;\n    std::string model    = \"../../ggml-large.bin\";\n\n    std::vector<std::string> fname_inp = {};\n    std::vector<std::string> fname_out = {};\n\n    std::vector<float> pcmf32 = {}; // mono-channel F32 PCM\n\n    // Voice Activity Detection (VAD) parameters\n    bool        vad           = false;\n    std::string vad_model     = \"\";\n    float       vad_threshold = 0.5f;\n    int         vad_min_speech_duration_ms = 250;\n    int         vad_min_silence_duration_ms = 100;\n    float       vad_max_speech_duration_s = FLT_MAX;\n    int         vad_speech_pad_ms = 30;\n    float       vad_samples_overlap = 0.1f;\n};\n\nstruct whisper_print_user_data {\n    const whisper_params * params;\n\n    const std::vector<std::vector<float>> * pcmf32s;\n};\n\nvoid whisper_print_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data) {\n    const auto & params  = *((whisper_print_user_data *) user_data)->params;\n    const auto & pcmf32s = *((whisper_print_user_data *) user_data)->pcmf32s;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n\n    std::string speaker = \"\";\n\n    int64_t t0;\n    int64_t t1;\n\n    // print the last n_new segments\n    const int s0 = n_segments - n_new;\n\n    if (s0 == 0) {\n        printf(\"\\n\");\n    }\n\n    for (int i = s0; i < n_segments; i++) {\n        if (!params.no_timestamps || params.diarize) {\n            t0 = whisper_full_get_segment_t0(ctx, i);\n            t1 = whisper_full_get_segment_t1(ctx, i);\n        }\n\n        if (!params.no_timestamps && !params.no_prints) {\n            printf(\"[%s --> %s]  \", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());\n        }\n\n        if (params.diarize && pcmf32s.size() == 2) {\n            const int64_t n_samples = pcmf32s[0].size();\n\n            const int64_t is0 = timestamp_to_sample(t0, n_samples, WHISPER_SAMPLE_RATE);\n            const int64_t is1 = timestamp_to_sample(t1, n_samples, WHISPER_SAMPLE_RATE);\n\n            double energy0 = 0.0f;\n            double energy1 = 0.0f;\n\n            for (int64_t j = is0; j < is1; j++) {\n                energy0 += fabs(pcmf32s[0][j]);\n                energy1 += fabs(pcmf32s[1][j]);\n            }\n\n            if (energy0 > 1.1*energy1) {\n                speaker = \"(speaker 0)\";\n            } else if (energy1 > 1.1*energy0) {\n                speaker = \"(speaker 1)\";\n            } else {\n                speaker = \"(speaker ?)\";\n            }\n\n            //printf(\"is0 = %lld, is1 = %lld, energy0 = %f, energy1 = %f, %s\\n\", is0, is1, energy0, energy1, speaker.c_str());\n        }\n\n        // colorful print bug\n        //\n        if (!params.no_prints) {\n            const char * text = whisper_full_get_segment_text(ctx, i);\n            printf(\"%s%s\", speaker.c_str(), text);\n        }\n\n\n        // with timestamps or speakers: each segment on new line\n        if ((!params.no_timestamps || params.diarize) && !params.no_prints) {\n            printf(\"\\n\");\n        }\n\n        fflush(stdout);\n    }\n}\n\nvoid cb_log_disable(enum ggml_log_level, const char *, void *) {}\n\nstruct whisper_result {\n    std::vector<std::vector<std::string>> segments;\n    std::string language;\n};\n\nclass ProgressWorker : public Napi::AsyncWorker {\n public:\n    ProgressWorker(Napi::Function& callback, whisper_params params, Napi::Function progress_callback, Napi::Env env)\n        : Napi::AsyncWorker(callback), params(params), env(env) {\n        // Create thread-safe function\n        if (!progress_callback.IsEmpty()) {\n            tsfn = Napi::ThreadSafeFunction::New(\n                env,\n                progress_callback,\n                \"Progress Callback\",\n                0,\n                1\n            );\n        }\n    }\n\n    ~ProgressWorker() {\n        if (tsfn) {\n            // Make sure to release the thread-safe function on destruction\n            tsfn.Release();\n        }\n    }\n\n    void Execute() override {\n        // Use custom run function with progress callback support\n        run_with_progress(params, result);\n    }\n\n    void OnOK() override {\n        Napi::HandleScope scope(Env());\n\n        if (params.detect_language) {\n            Napi::Object resultObj = Napi::Object::New(Env());\n            resultObj.Set(\"language\", Napi::String::New(Env(), result.language));\n            Callback().Call({Env().Null(), resultObj});\n        }\n\n        Napi::Object returnObj = Napi::Object::New(Env());\n        if (!result.language.empty()) {\n            returnObj.Set(\"language\", Napi::String::New(Env(), result.language));\n        }\n        Napi::Array transcriptionArray = Napi::Array::New(Env(), result.segments.size());\n        for (uint64_t i = 0; i < result.segments.size(); ++i) {\n            Napi::Object tmp = Napi::Array::New(Env(), 3);\n            for (uint64_t j = 0; j < 3; ++j) {\n                tmp[j] = Napi::String::New(Env(), result.segments[i][j]);\n            }\n            transcriptionArray[i] = tmp;\n         }\n         returnObj.Set(\"transcription\", transcriptionArray);\n         Callback().Call({Env().Null(), returnObj});\n    }\n\n    // Progress callback function - using thread-safe function\n    void OnProgress(int progress) {\n        if (tsfn) {\n            // Use thread-safe function to call JavaScript callback\n            auto callback = [progress](Napi::Env env, Napi::Function jsCallback) {\n                jsCallback.Call({Napi::Number::New(env, progress)});\n            };\n\n            tsfn.BlockingCall(callback);\n        }\n    }\n\n private:\n    whisper_params params;\n    whisper_result result;\n    Napi::Env env;\n    Napi::ThreadSafeFunction tsfn;\n\n    // Custom run function with progress callback support\n    int run_with_progress(whisper_params &params, whisper_result & result) {\n        if (params.no_prints) {\n            whisper_log_set(cb_log_disable, NULL);\n        }\n\n        if (params.fname_inp.empty() && params.pcmf32.empty()) {\n            fprintf(stderr, \"error: no input files or audio buffer specified\\n\");\n            return 2;\n        }\n\n        if (params.language != \"auto\" && whisper_lang_id(params.language.c_str()) == -1) {\n            fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n            exit(0);\n        }\n\n        // whisper init\n        struct whisper_context_params cparams = whisper_context_default_params();\n        cparams.use_gpu = params.use_gpu;\n        cparams.flash_attn = params.flash_attn;\n        struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n\n        if (ctx == nullptr) {\n            fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n            return 3;\n        }\n\n        // If params.pcmf32 provides, set params.fname_inp as \"buffer\"\n        if (!params.pcmf32.empty()) {\n            fprintf(stderr, \"info: using audio buffer as input\\n\");\n            params.fname_inp.clear();\n            params.fname_inp.emplace_back(\"buffer\");\n        }\n\n        for (int f = 0; f < (int) params.fname_inp.size(); ++f) {\n            const auto fname_inp = params.fname_inp[f];\n            const auto fname_out = f < (int)params.fname_out.size() && !params.fname_out[f].empty() ? params.fname_out[f] : params.fname_inp[f];\n\n            std::vector<float> pcmf32; // mono-channel F32 PCM\n            std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM\n\n            // If params.pcmf32 is empty, read input audio file\n            if (params.pcmf32.empty()) {\n                if (!::read_audio_data(fname_inp, pcmf32, pcmf32s, params.diarize)) {\n                    fprintf(stderr, \"error: failed to read audio file '%s'\\n\", fname_inp.c_str());\n                    continue;\n                }\n            } else {\n                pcmf32 = params.pcmf32;\n            }\n\n            // Print system info\n            if (!params.no_prints) {\n                fprintf(stderr, \"\\n\");\n                fprintf(stderr, \"system_info: n_threads = %d / %d | %s\\n\",\n                        params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());\n            }\n\n            // Print processing info\n            if (!params.no_prints) {\n                fprintf(stderr, \"\\n\");\n                if (!whisper_is_multilingual(ctx)) {\n                    if (params.language != \"en\" || params.translate) {\n                        params.language = \"en\";\n                        params.translate = false;\n                        fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n                    }\n                }\n                fprintf(stderr, \"%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, timestamps = %d, audio_ctx = %d ...\\n\",\n                        __func__, fname_inp.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,\n                        params.n_threads, params.n_processors,\n                        params.language.c_str(),\n                        params.translate ? \"translate\" : \"transcribe\",\n                        params.no_timestamps ? 0 : 1,\n                        params.audio_ctx);\n\n                fprintf(stderr, \"\\n\");\n            }\n\n            // Run inference\n            {\n                whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n                wparams.strategy = params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;\n\n                wparams.print_realtime   = false;\n                wparams.print_progress   = params.print_progress;\n                wparams.print_timestamps = !params.no_timestamps;\n                wparams.print_special    = params.print_special;\n                wparams.translate        = params.translate;\n                wparams.language         = params.detect_language ? \"auto\" : params.language.c_str();\n                wparams.detect_language  = params.detect_language;\n                wparams.n_threads        = params.n_threads;\n                wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;\n                wparams.offset_ms        = params.offset_t_ms;\n                wparams.duration_ms      = params.duration_ms;\n\n                wparams.token_timestamps = params.output_wts || params.max_len > 0;\n                wparams.thold_pt         = params.word_thold;\n                wparams.entropy_thold    = params.entropy_thold;\n                wparams.logprob_thold    = params.logprob_thold;\n                wparams.max_len          = params.output_wts && params.max_len == 0 ? 60 : params.max_len;\n                wparams.audio_ctx        = params.audio_ctx;\n\n                wparams.greedy.best_of        = params.best_of;\n                wparams.beam_search.beam_size = params.beam_size;\n\n                wparams.initial_prompt   = params.prompt.c_str();\n\n                wparams.no_timestamps    = params.no_timestamps;\n\n                whisper_print_user_data user_data = { &params, &pcmf32s };\n\n                // This callback is called for each new segment\n                if (!wparams.print_realtime) {\n                    wparams.new_segment_callback           = whisper_print_segment_callback;\n                    wparams.new_segment_callback_user_data = &user_data;\n                }\n\n                // Set progress callback\n                wparams.progress_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {\n                    ProgressWorker* worker = static_cast<ProgressWorker*>(user_data);\n                    worker->OnProgress(progress);\n                };\n                wparams.progress_callback_user_data = this;\n\n                // Set VAD parameters\n                wparams.vad            = params.vad;\n                wparams.vad_model_path = params.vad_model.c_str();\n\n                wparams.vad_params.threshold               = params.vad_threshold;\n                wparams.vad_params.min_speech_duration_ms  = params.vad_min_speech_duration_ms;\n                wparams.vad_params.min_silence_duration_ms = params.vad_min_silence_duration_ms;\n                wparams.vad_params.max_speech_duration_s   = params.vad_max_speech_duration_s;\n                wparams.vad_params.speech_pad_ms           = params.vad_speech_pad_ms;\n                wparams.vad_params.samples_overlap         = params.vad_samples_overlap;\n\n                if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {\n                    fprintf(stderr, \"failed to process audio\\n\");\n                    return 10;\n                }\n            }\n        }\n\n        if (params.detect_language || params.language == \"auto\") {\n            result.language = whisper_lang_str(whisper_full_lang_id(ctx));\n        }\n        const int n_segments = whisper_full_n_segments(ctx);\n        result.segments.resize(n_segments);\n\n        for (int i = 0; i < n_segments; ++i) {\n            const char * text = whisper_full_get_segment_text(ctx, i);\n            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n\n            result.segments[i].emplace_back(to_timestamp(t0, params.comma_in_time));\n            result.segments[i].emplace_back(to_timestamp(t1, params.comma_in_time));\n            result.segments[i].emplace_back(text);\n        }\n\n        whisper_print_timings(ctx);\n        whisper_free(ctx);\n\n        return 0;\n    }\n};\n\nNapi::Value whisper(const Napi::CallbackInfo& info) {\n  Napi::Env env = info.Env();\n  if (info.Length() <= 0 || !info[0].IsObject()) {\n    Napi::TypeError::New(env, \"object expected\").ThrowAsJavaScriptException();\n  }\n  whisper_params params;\n\n  Napi::Object whisper_params = info[0].As<Napi::Object>();\n  std::string language = whisper_params.Get(\"language\").As<Napi::String>();\n  std::string model = whisper_params.Get(\"model\").As<Napi::String>();\n  std::string input = whisper_params.Get(\"fname_inp\").As<Napi::String>();\n\n  bool use_gpu = true;\n  if (whisper_params.Has(\"use_gpu\") && whisper_params.Get(\"use_gpu\").IsBoolean()) {\n    use_gpu = whisper_params.Get(\"use_gpu\").As<Napi::Boolean>();\n  }\n\n  bool flash_attn = false;\n  if (whisper_params.Has(\"flash_attn\") && whisper_params.Get(\"flash_attn\").IsBoolean()) {\n    flash_attn = whisper_params.Get(\"flash_attn\").As<Napi::Boolean>();\n  }\n\n  bool no_prints = false;\n  if (whisper_params.Has(\"no_prints\") && whisper_params.Get(\"no_prints\").IsBoolean()) {\n    no_prints = whisper_params.Get(\"no_prints\").As<Napi::Boolean>();\n  }\n\n  bool no_timestamps = false;\n  if (whisper_params.Has(\"no_timestamps\") && whisper_params.Get(\"no_timestamps\").IsBoolean()) {\n    no_timestamps = whisper_params.Get(\"no_timestamps\").As<Napi::Boolean>();\n  }\n\n  bool detect_language = false;\n  if (whisper_params.Has(\"detect_language\") && whisper_params.Get(\"detect_language\").IsBoolean()) {\n    detect_language = whisper_params.Get(\"detect_language\").As<Napi::Boolean>();\n  }\n\n  int32_t audio_ctx = 0;\n  if (whisper_params.Has(\"audio_ctx\") && whisper_params.Get(\"audio_ctx\").IsNumber()) {\n    audio_ctx = whisper_params.Get(\"audio_ctx\").As<Napi::Number>();\n  }\n\n  bool comma_in_time = true;\n  if (whisper_params.Has(\"comma_in_time\") && whisper_params.Get(\"comma_in_time\").IsBoolean()) {\n    comma_in_time = whisper_params.Get(\"comma_in_time\").As<Napi::Boolean>();\n  }\n\n  int32_t max_len = 0;\n  if (whisper_params.Has(\"max_len\") && whisper_params.Get(\"max_len\").IsNumber()) {\n    max_len = whisper_params.Get(\"max_len\").As<Napi::Number>();\n  }\n\n  // Add support for max_context\n  int32_t max_context = -1;\n  if (whisper_params.Has(\"max_context\") && whisper_params.Get(\"max_context\").IsNumber()) {\n    max_context = whisper_params.Get(\"max_context\").As<Napi::Number>();\n  }\n\n  // support prompt\n  std::string prompt = \"\";\n  if (whisper_params.Has(\"prompt\") && whisper_params.Get(\"prompt\").IsString()) {\n    prompt = whisper_params.Get(\"prompt\").As<Napi::String>();\n  }\n\n  // Add support for print_progress\n  bool print_progress = false;\n  if (whisper_params.Has(\"print_progress\") && whisper_params.Get(\"print_progress\").IsBoolean()) {\n    print_progress = whisper_params.Get(\"print_progress\").As<Napi::Boolean>();\n  }\n  // Add support for progress_callback\n  Napi::Function progress_callback;\n  if (whisper_params.Has(\"progress_callback\") && whisper_params.Get(\"progress_callback\").IsFunction()) {\n    progress_callback = whisper_params.Get(\"progress_callback\").As<Napi::Function>();\n  }\n\n  // Add support for VAD parameters\n  bool vad = false;\n  if (whisper_params.Has(\"vad\") && whisper_params.Get(\"vad\").IsBoolean()) {\n    vad = whisper_params.Get(\"vad\").As<Napi::Boolean>();\n  }\n\n  std::string vad_model = \"\";\n  if (whisper_params.Has(\"vad_model\") && whisper_params.Get(\"vad_model\").IsString()) {\n    vad_model = whisper_params.Get(\"vad_model\").As<Napi::String>();\n  }\n\n  float vad_threshold = 0.5f;\n  if (whisper_params.Has(\"vad_threshold\") && whisper_params.Get(\"vad_threshold\").IsNumber()) {\n    vad_threshold = whisper_params.Get(\"vad_threshold\").As<Napi::Number>();\n  }\n\n  int vad_min_speech_duration_ms = 250;\n  if (whisper_params.Has(\"vad_min_speech_duration_ms\") && whisper_params.Get(\"vad_min_speech_duration_ms\").IsNumber()) {\n    vad_min_speech_duration_ms = whisper_params.Get(\"vad_min_speech_duration_ms\").As<Napi::Number>();\n  }\n\n  int vad_min_silence_duration_ms = 100;\n  if (whisper_params.Has(\"vad_min_silence_duration_ms\") && whisper_params.Get(\"vad_min_silence_duration_ms\").IsNumber()) {\n    vad_min_silence_duration_ms = whisper_params.Get(\"vad_min_silence_duration_ms\").As<Napi::Number>();\n  }\n\n  float vad_max_speech_duration_s = FLT_MAX;\n  if (whisper_params.Has(\"vad_max_speech_duration_s\") && whisper_params.Get(\"vad_max_speech_duration_s\").IsNumber()) {\n    vad_max_speech_duration_s = whisper_params.Get(\"vad_max_speech_duration_s\").As<Napi::Number>();\n  }\n\n  int vad_speech_pad_ms = 30;\n  if (whisper_params.Has(\"vad_speech_pad_ms\") && whisper_params.Get(\"vad_speech_pad_ms\").IsNumber()) {\n    vad_speech_pad_ms = whisper_params.Get(\"vad_speech_pad_ms\").As<Napi::Number>();\n  }\n\n  float vad_samples_overlap = 0.1f;\n  if (whisper_params.Has(\"vad_samples_overlap\") && whisper_params.Get(\"vad_samples_overlap\").IsNumber()) {\n    vad_samples_overlap = whisper_params.Get(\"vad_samples_overlap\").As<Napi::Number>();\n  }\n\n  Napi::Value pcmf32Value = whisper_params.Get(\"pcmf32\");\n  std::vector<float> pcmf32_vec;\n  if (pcmf32Value.IsTypedArray()) {\n    Napi::Float32Array pcmf32 = pcmf32Value.As<Napi::Float32Array>();\n    size_t length = pcmf32.ElementLength();\n    pcmf32_vec.reserve(length);\n    for (size_t i = 0; i < length; i++) {\n      pcmf32_vec.push_back(pcmf32[i]);\n    }\n  }\n\n  params.language = language;\n  params.model = model;\n  params.fname_inp.emplace_back(input);\n  params.use_gpu = use_gpu;\n  params.flash_attn = flash_attn;\n  params.no_prints = no_prints;\n  params.no_timestamps = no_timestamps;\n  params.audio_ctx = audio_ctx;\n  params.pcmf32 = pcmf32_vec;\n  params.comma_in_time = comma_in_time;\n  params.max_len = max_len;\n  params.max_context = max_context;\n  params.print_progress = print_progress;\n  params.prompt = prompt;\n  params.detect_language = detect_language;\n\n  // Set VAD parameters\n  params.vad = vad;\n  params.vad_model = vad_model;\n  params.vad_threshold = vad_threshold;\n  params.vad_min_speech_duration_ms = vad_min_speech_duration_ms;\n  params.vad_min_silence_duration_ms = vad_min_silence_duration_ms;\n  params.vad_max_speech_duration_s = vad_max_speech_duration_s;\n  params.vad_speech_pad_ms = vad_speech_pad_ms;\n  params.vad_samples_overlap = vad_samples_overlap;\n\n  Napi::Function callback = info[1].As<Napi::Function>();\n  // Create a new Worker class with progress callback support\n  ProgressWorker* worker = new ProgressWorker(callback, params, progress_callback, env);\n  worker->Queue();\n  return env.Undefined();\n}\n\n\nNapi::Object Init(Napi::Env env, Napi::Object exports) {\n  exports.Set(\n      Napi::String::New(env, \"whisper\"),\n      Napi::Function::New(env, whisper)\n  );\n  return exports;\n}\n\nNODE_API_MODULE(whisper, Init);\n"
  },
  {
    "path": "examples/addon.node/index.js",
    "content": "const path = require('path');\nconst os = require('os');\n\nconst isWindows = os.platform() === 'win32';\nconst buildPath = isWindows ? \"../../build/bin/Release/addon.node\" : \"../../build/Release/addon.node\";\n\nconst { whisper } = require(path.join(__dirname, buildPath));\nconst { promisify } = require(\"util\");\n\nconst whisperAsync = promisify(whisper);\n\nconst whisperParams = {\n  language: \"en\",\n  model: path.join(__dirname, \"../../models/ggml-base.en.bin\"),\n  fname_inp: path.join(__dirname, \"../../samples/jfk.wav\"),\n  use_gpu: true,\n  flash_attn: false,\n  no_prints: true,\n  comma_in_time: false,\n  translate: true,\n  no_timestamps: false,\n  detect_language: false,\n  audio_ctx: 0,\n  max_len: 0,\n  progress_callback: (progress) => {\n      console.log(`progress: ${progress}%`);\n    }\n};\n\nconst arguments = process.argv.slice(2);\nconst params = Object.fromEntries(\n  arguments.reduce((pre, item) => {\n    if (item.startsWith(\"--\")) {\n      const [key, value] = item.slice(2).split(\"=\");\n      if (key === \"audio_ctx\") {\n        whisperParams[key] = parseInt(value);\n      } else if (key === \"detect_language\") {\n        whisperParams[key] = value === \"true\";\n      } else {\n        whisperParams[key] = value;\n      }\n      return pre;\n    }\n    return pre;\n  }, [])\n);\n\nfor (const key in params) {\n  if (whisperParams.hasOwnProperty(key)) {\n    whisperParams[key] = params[key];\n  }\n}\n\nconsole.log(\"whisperParams =\", whisperParams);\n\nwhisperAsync(whisperParams).then((result) => {\n  console.log();\n  console.log(result);\n});\n"
  },
  {
    "path": "examples/addon.node/package.json",
    "content": "{\n  \"name\": \"addon.node\",\n  \"version\": \"0.0.0\",\n  \"description\": \"\",\n  \"main\": \"index.js\",\n  \"author\": \"Qanhe Chen\",\n  \"license\": \"MIT\",\n  \"scripts\": {\n    \"test\": \"jest\"\n  },\n  \"devDependencies\": {\n    \"cmake-js\": \"^7.1.1\",\n    \"jest\": \"^29.4.0\",\n    \"node-addon-api\": \"^5.0.0\"\n  }\n}\n"
  },
  {
    "path": "examples/addon.node/vad-example.js",
    "content": "const path = require(\"path\");\nconst { whisper } = require(path.join(\n  __dirname,\n  \"../../build/Release/addon.node\"\n));\nconst { promisify } = require(\"util\");\n\nconst whisperAsync = promisify(whisper);\n\n// Example with VAD enabled\nconst vadParams = {\n  language: \"en\",\n  model: path.join(__dirname, \"../../models/ggml-base.en.bin\"),\n  fname_inp: path.join(__dirname, \"../../samples/jfk.wav\"),\n  use_gpu: true,\n  flash_attn: false,\n  no_prints: false,\n  comma_in_time: true,\n  translate: false,\n  no_timestamps: false,\n  detect_language: false,\n  audio_ctx: 0,\n  max_len: 0,\n  // VAD parameters\n  vad: true,\n  vad_model: path.join(__dirname, \"../../models/ggml-silero-v6.2.0.bin\"), // You need to download this model\n  vad_threshold: 0.5,\n  vad_min_speech_duration_ms: 250,\n  vad_min_silence_duration_ms: 100,\n  vad_max_speech_duration_s: 30.0,\n  vad_speech_pad_ms: 30,\n  vad_samples_overlap: 0.1,\n  progress_callback: (progress) => {\n    console.log(`VAD Transcription progress: ${progress}%`);\n  }\n};\n\n// Example without VAD (traditional approach)\nconst traditionalParams = {\n  language: \"en\",\n  model: path.join(__dirname, \"../../models/ggml-base.en.bin\"),\n  fname_inp: path.join(__dirname, \"../../samples/jfk.wav\"),\n  use_gpu: true,\n  flash_attn: false,\n  no_prints: false,\n  comma_in_time: true,\n  translate: false,\n  no_timestamps: false,\n  detect_language: false,\n  audio_ctx: 0,\n  max_len: 0,\n  vad: false, // Explicitly disable VAD\n  progress_callback: (progress) => {\n    console.log(`Traditional transcription progress: ${progress}%`);\n  }\n};\n\nasync function runVADExample() {\n  try {\n    console.log(\"=== Whisper.cpp Node.js VAD Example ===\\n\");\n    \n    // Check if VAD model exists\n    const fs = require('fs');\n    if (!fs.existsSync(vadParams.vad_model)) {\n      console.log(\"⚠️  VAD model not found. Please download the VAD model first:\");\n      console.log(\"   ./models/download-vad-model.sh silero-v6.2.0\");\n      console.log(\"   Or run: python models/convert-silero-vad-to-ggml.py\");\n      console.log(\"\\n   Falling back to traditional transcription without VAD...\\n\");\n      \n      // Run without VAD\n      console.log(\"🎵 Running traditional transcription...\");\n      const traditionalResult = await whisperAsync(traditionalParams);\n      console.log(\"\\n📝 Traditional transcription result:\");\n      console.log(traditionalResult);\n      return;\n    }\n    \n    console.log(\"🎵 Running transcription with VAD enabled...\");\n    console.log(\"VAD Parameters:\");\n    console.log(`  - Threshold: ${vadParams.vad_threshold}`);\n    console.log(`  - Min speech duration: ${vadParams.vad_min_speech_duration_ms}ms`);\n    console.log(`  - Min silence duration: ${vadParams.vad_min_silence_duration_ms}ms`);\n    console.log(`  - Max speech duration: ${vadParams.vad_max_speech_duration_s}s`);\n    console.log(`  - Speech padding: ${vadParams.vad_speech_pad_ms}ms`);\n    console.log(`  - Samples overlap: ${vadParams.vad_samples_overlap}\\n`);\n    \n    const startTime = Date.now();\n    const vadResult = await whisperAsync(vadParams);\n    const vadDuration = Date.now() - startTime;\n    \n    console.log(\"\\n✅ VAD transcription completed!\");\n    console.log(`⏱️  Processing time: ${vadDuration}ms`);\n    console.log(\"\\n📝 VAD transcription result:\");\n    console.log(vadResult);\n    \n    // Compare with traditional approach\n    console.log(\"\\n🔄 Running traditional transcription for comparison...\");\n    const traditionalStartTime = Date.now();\n    const traditionalResult = await whisperAsync(traditionalParams);\n    const traditionalDuration = Date.now() - traditionalStartTime;\n    \n    console.log(\"\\n✅ Traditional transcription completed!\");\n    console.log(`⏱️  Processing time: ${traditionalDuration}ms`);\n    console.log(\"\\n📝 Traditional transcription result:\");\n    console.log(traditionalResult);\n    \n    // Performance comparison\n    console.log(\"\\n📊 Performance Comparison:\");\n    console.log(`VAD:         ${vadDuration}ms`);\n    console.log(`Traditional: ${traditionalDuration}ms`);\n    const speedup = traditionalDuration / vadDuration;\n    if (speedup > 1) {\n      console.log(`🚀 VAD is ${speedup.toFixed(2)}x faster!`);\n    } else {\n      console.log(`ℹ️  Traditional approach was ${(1/speedup).toFixed(2)}x faster in this case.`);\n    }\n    \n  } catch (error) {\n    console.error(\"❌ Error during transcription:\", error);\n  }\n}\n\n// Run the example\nif (require.main === module) {\n  runVADExample();\n}\n\nmodule.exports = {\n  runVADExample,\n  vadParams,\n  traditionalParams\n}; "
  },
  {
    "path": "examples/bench/CMakeLists.txt",
    "content": "set(TARGET whisper-bench)\nadd_executable(${TARGET} bench.cpp)\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE whisper ${CMAKE_THREAD_LIBS_INIT})\n\ninstall(TARGETS ${TARGET} RUNTIME)\n"
  },
  {
    "path": "examples/bench/README.md",
    "content": "# whisper.cpp/examples/bench\n\nA very basic tool for benchmarking the inference performance on your device. The tool simply runs the Encoder part of\nthe transformer on some random audio data and records the execution time. This way we can have an objective comparison\nof the performance of the model for various setups.\n\nBenchmark results are tracked in the following Github issue: https://github.com/ggml-org/whisper.cpp/issues/89\n\n```bash\n# run the bench too on the small.en model using 4 threads\n$ ./build/bin/whisper-bench -m ./models/ggml-small.en.bin -t 4\n\nwhisper_model_load: loading model from './models/ggml-small.en.bin'\nwhisper_model_load: n_vocab       = 51864\nwhisper_model_load: n_audio_ctx   = 1500\nwhisper_model_load: n_audio_state = 768\nwhisper_model_load: n_audio_head  = 12\nwhisper_model_load: n_audio_layer = 12\nwhisper_model_load: n_text_ctx    = 448\nwhisper_model_load: n_text_state  = 768\nwhisper_model_load: n_text_head   = 12\nwhisper_model_load: n_text_layer  = 12\nwhisper_model_load: n_mels        = 80\nwhisper_model_load: f16           = 1\nwhisper_model_load: type          = 3\nwhisper_model_load: mem_required  = 1048.00 MB\nwhisper_model_load: adding 1607 extra tokens\nwhisper_model_load: ggml ctx size = 533.05 MB\nwhisper_model_load: memory size =    68.48 MB \nwhisper_model_load: model size  =   464.44 MB\n\nwhisper_print_timings:     load time =   240.82 ms\nwhisper_print_timings:      mel time =     0.00 ms\nwhisper_print_timings:   sample time =     0.00 ms\nwhisper_print_timings:   encode time =  1062.21 ms / 88.52 ms per layer\nwhisper_print_timings:   decode time =     0.00 ms / 0.00 ms per layer\nwhisper_print_timings:    total time =  1303.04 ms\n\nsystem_info: n_threads = 4 | AVX2 = 0 | AVX512 = 0 | NEON = 1 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | \n\nIf you wish, you can submit these results here:\n\n  https://github.com/ggml-org/whisper.cpp/issues/89\n\nPlease include the following information:\n\n  - CPU model\n  - Operating system\n  - Compiler\n\n```\n"
  },
  {
    "path": "examples/bench/bench.cpp",
    "content": "#include \"whisper.h\"\n\n#include <cstdio>\n#include <cstring>\n#include <string>\n#include <thread>\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t what = 0; // what to benchmark: 0 - whisper encoder, 1 - memcpy, 2 - ggml_mul_mat\n\n    std::string model = \"models/ggml-base.en.bin\";\n\n    bool use_gpu    = true;\n    bool flash_attn = true;\n};\n\nvoid whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"     || arg == \"--threads\")       { params.n_threads  = std::stoi(argv[++i]); }\n        else if (arg == \"-m\"     || arg == \"--model\")         { params.model      = argv[++i]; }\n        else if (arg == \"-w\"     || arg == \"--what\")          { params.what       = atoi(argv[++i]); }\n        else if (arg == \"-ng\"    || arg == \"--no-gpu\")        { params.use_gpu    = false; }\n        else if (arg == \"-fa\"    || arg == \"--flash-attn\")    { params.flash_attn = true; }\n        else if (arg == \"-nfa\"   || arg == \"--no-flash-attn\") { params.flash_attn = false; }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,       --help          [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,     --threads N     [%-7d] number of threads to use during computation\\n\", params.n_threads);\n    fprintf(stderr, \"  -m FNAME, --model FNAME   [%-7s] model path\\n\",                                  params.model.c_str());\n    fprintf(stderr, \"  -w N,     --what N        [%-7d] what to benchmark:\\n\",                          params.what);\n    fprintf(stderr, \"                             %-7s  0 - whisper\\n\",                                 \"\");\n    fprintf(stderr, \"                             %-7s  1 - memcpy\\n\",                                  \"\");\n    fprintf(stderr, \"                             %-7s  2 - ggml_mul_mat\\n\",                            \"\");\n    fprintf(stderr, \"  -ng,      --no-gpu        [%-7s] disable GPU\\n\",                                 params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,      --flash-attn    [%-7s] enable flash attention\\n\",                      params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,     --no-flash-attn [%-7s] disable flash attention\\n\",                     params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"\\n\");\n}\n\nstatic int whisper_bench_full(const whisper_params & params) {\n    // whisper init\n\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    {\n        fprintf(stderr, \"\\n\");\n        fprintf(stderr, \"system_info: n_threads = %d / %d | %s\\n\", params.n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());\n    }\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n    if (ctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 2;\n    }\n\n    const int n_mels = whisper_model_n_mels(ctx);\n\n    if (int ret = whisper_set_mel(ctx, nullptr, 0, n_mels)) {\n        fprintf(stderr, \"error: failed to set mel: %d\\n\", ret);\n        return 3;\n    }\n\n    whisper_token tokens[512];\n    memset(tokens, 0, sizeof(tokens));\n\n    // TODO: need 2 loops because of the current graph capture logic in the CUDA backend\n    //       https://github.com/ggml-org/llama.cpp/pull/19754\n    for (int h = 0; h < 2; ++h) {\n        // heat encoder\n        if (int ret = whisper_encode(ctx, 0, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to encode: %d\\n\", ret);\n            return 4;\n        }\n\n        // prompt heat\n        if (int ret = whisper_decode(ctx, tokens, 256, 0, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n            return 4;\n        }\n\n        // text-generation heat\n        for (int i = 0; i < 256; i++) {\n            if (int ret = whisper_decode(ctx, tokens, 1, i, params.n_threads) != 0) {\n                fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n                return 4;\n            }\n        }\n\n        // batched heat\n        if (int ret = whisper_decode(ctx, tokens, 5, 0, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n            return 4;\n        }\n    }\n\n    whisper_reset_timings(ctx);\n\n    // actual run\n    if (int ret = whisper_encode(ctx, 0, params.n_threads) != 0) {\n        fprintf(stderr, \"error: failed to encode: %d\\n\", ret);\n        return 4;\n    }\n\n    // text-generation\n    for (int i = 0; i < 256; i++) {\n        if (int ret = whisper_decode(ctx, tokens, 1, i, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n            return 4;\n        }\n    }\n\n    // batched decoding\n    for (int i = 0; i < 64; i++) {\n        if (int ret = whisper_decode(ctx, tokens, 5, 0, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n            return 4;\n        }\n    }\n\n    // prompt processing\n    for (int i = 0; i < 16; i++) {\n        if (int ret = whisper_decode(ctx, tokens, 256, 0, params.n_threads) != 0) {\n            fprintf(stderr, \"error: failed to decode: %d\\n\", ret);\n            return 4;\n        }\n    }\n\n    whisper_print_timings(ctx);\n    whisper_free(ctx);\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"If you wish, you can submit these results here:\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"  https://github.com/ggerganov/whisper.cpp/issues/89\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"Please include the following information:\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"  - CPU model\\n\");\n    fprintf(stderr, \"  - Operating system\\n\");\n    fprintf(stderr, \"  - Compiler\\n\");\n    fprintf(stderr, \"\\n\");\n\n    return 0;\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    int ret = -1;\n\n    switch (params.what) {\n        case 0: ret = whisper_bench_full(params);                break;\n        case 1: ret = whisper_bench_memcpy(params.n_threads);       break;\n        case 2: ret = whisper_bench_ggml_mul_mat(params.n_threads); break;\n        default: fprintf(stderr, \"error: unknown benchmark: %d\\n\", params.what); break;\n    }\n\n    return ret;\n}\n"
  },
  {
    "path": "examples/bench.wasm/CMakeLists.txt",
    "content": "#\n# libbench\n#\n\nset(TARGET libbench)\n\nadd_executable(${TARGET}\n    emscripten.cpp\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE\n    whisper\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside bench.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libbench.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/bench.wasm/bench.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE_STRICT=0 \\\n    -s INITIAL_MEMORY=2000MB \\\n    -s TOTAL_MEMORY=2000MB \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s EXPORTED_RUNTIME_METHODS=\\\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\\\" \\\n    ${EXTRA_FLAGS} \\\n    \")\n\n#\n# bench.wasm\n#\n\nset(TARGET bench.wasm)\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)\n"
  },
  {
    "path": "examples/bench.wasm/README.md",
    "content": "# bench.wasm\n\nBenchmark the performance of whisper.cpp in the browser using WebAssembly\n\nLink: https://ggml.ai/whisper.cpp/bench.wasm/\n\nTerminal version: [examples/bench](/examples/bench)\n\n## Build instructions\n\n```bash\n# build using Emscripten (v3.1.2)\ngit clone https://github.com/ggerganov/whisper.cpp\ncd whisper.cpp\nmkdir build-em && cd build-em\nemcmake cmake ..\nmake -j\n```\nThe example can then be started by running a local HTTP server:\n```console\npython3 examples/server.py\n```\nAnd then opening a browser to the following URL:\nhttp://localhost:8000/bench.wasm\n\nTo run the example in a different server, you need to copy the following files\nto the server's HTTP path:\n```\n# copy the produced page to your HTTP path\ncp bin/bench.wasm/*       /path/to/html/\ncp bin/libbench.js        /path/to/html/\ncp bin/libbench.worker.js /path/to/html/\n```\n\n> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`\n> which means that that a separate .wasm file will not be generated. Instead, the\n> WASM module is embedded in the main JS file as a base64 encoded string. To\n> generate a separate .wasm file, you need to disable this option by passing\n> `-DWHISPER_WASM_SINGLE_FILE=OFF`:\n> ```console\n> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF\n> ```\n> This will generate a `libbench.wasm` file in the build/bin directory.\n\n> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no\n> longer generated and the worker is embedded in the main JS file. So the worker\n> file will not be geneated for versions later than `3.1.58`.\n"
  },
  {
    "path": "examples/bench.wasm/emscripten.cpp",
    "content": "#include \"whisper.h\"\n\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <cmath>\n#include <string>\n#include <thread>\n#include <vector>\n\nconstexpr int N_THREAD = 8;\n\n// TODO: get rid of this vector of contexts - bad idea in the first place\nstd::vector<struct whisper_context *> g_contexts(4, nullptr);\n\nstd::thread g_worker;\n\nvoid bench_main(size_t index) {\n    const int n_threads = std::min(N_THREAD, (int) std::thread::hardware_concurrency());\n\n    // whisper context\n    auto & ctx = g_contexts[index];\n\n    fprintf(stderr, \"%s: running benchmark with %d threads - please wait...\\n\", __func__, n_threads);\n\n    const int n_mels = whisper_model_n_mels(ctx);\n\n    if (int ret = whisper_set_mel(ctx, nullptr, 0, n_mels)) {\n        fprintf(stderr, \"error: failed to set mel: %d\\n\", ret);\n        return;\n    }\n\n    {\n        fprintf(stderr, \"\\n\");\n        fprintf(stderr, \"system_info: n_threads = %d / %d | %s\\n\", n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());\n    }\n\n    if (int ret = whisper_encode(ctx, 0, n_threads) != 0) {\n        fprintf(stderr, \"error: failed to encode model: %d\\n\", ret);\n        return;\n    }\n\n    whisper_print_timings(ctx);\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"If you wish, you can submit these results here:\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"  https://github.com/ggerganov/whisper.cpp/issues/89\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"Please include the following information:\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"  - CPU model\\n\");\n    fprintf(stderr, \"  - Operating system\\n\");\n    fprintf(stderr, \"  - Browser\\n\");\n    fprintf(stderr, \"\\n\");\n}\n\nEMSCRIPTEN_BINDINGS(bench) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model) {\n        for (size_t i = 0; i < g_contexts.size(); ++i) {\n            if (g_contexts[i] == nullptr) {\n                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n                if (g_contexts[i] != nullptr) {\n                    if (g_worker.joinable()) {\n                        g_worker.join();\n                    }\n                    g_worker = std::thread([i]() {\n                        bench_main(i);\n                    });\n\n                    return i + 1;\n                } else {\n                    return (size_t) 0;\n                }\n            }\n        }\n\n        return (size_t) 0;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([](size_t index) {\n        if (index < g_contexts.size()) {\n            whisper_free(g_contexts[index]);\n            g_contexts[index] = nullptr;\n        }\n    }));\n}\n"
  },
  {
    "path": "examples/bench.wasm/index-tmpl.html",
    "content": "<!doctype html>\n<html lang=\"en-us\">\n    <head>\n        <title>bench : Benchmark whisper.cpp performance in the browser</title>\n\n        <style>\n            #output {\n                width: 100%;\n                height: 100%;\n                margin: 0 auto;\n                margin-top: 10px;\n                border-left: 0px;\n                border-right: 0px;\n                padding-left: 0px;\n                padding-right: 0px;\n                display: block;\n                background-color: black;\n                color: white;\n                font-size: 10px;\n                font-family: 'Lucida Console', Monaco, monospace;\n                outline: none;\n                white-space: pre;\n                overflow-wrap: normal;\n                overflow-x: scroll;\n            }\n        </style>\n        <script src=\"../coi-serviceworker.js\"></script>\n        <link rel=\"icon\" href=\"data:,\">\n    </head>\n    <body>\n        <div id=\"main-container\">\n            <b>bench : Benchmark whisper.cpp performance in the browser</b>\n\n            <br><br>\n\n            You can find more about this project on <a href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/bench.wasm\">GitHub</a>.\n\n            <br><br>\n\n            <b>More examples:</b>\n                <a href=\"../\">main</a> |\n                <a href=\"../bench.wasm/\">bench</a> |\n                <a href=\"../stream.wasm\">stream</a> |\n                <a href=\"../command.wasm/\">command</a> |\n                <a href=\"../wchess.wasm/\">wchess</a> |\n\n            <br><br>\n\n            <hr>\n\n            Select the model you would like to use and click the \"Bench\" button.<br>\n            The results will be displayed in the textarea below.\n\n            <br><br>\n\n            <div id=\"model-whisper\">\n                Whisper model: <span id=\"model-whisper-status\"></span>\n                <button id=\"fetch-whisper-tiny-en\"  onclick=\"loadWhisper('tiny.en')\">tiny.en (75 MB)</button>\n                <button id=\"fetch-whisper-base-en\"  onclick=\"loadWhisper('base.en')\">base.en (142 MB)</button>\n                <button id=\"fetch-whisper-small-en\" onclick=\"loadWhisper('small.en')\">small.en (466 MB)</button>\n                <input type=\"file\" id=\"whisper-file\" name=\"file\" onchange=\"loadFile(event, 'whisper.bin')\" />\n                <br><br>\n                Quantized models:<br><br>\n                <button id=\"fetch-whisper-tiny-en-q5_1\"   onclick=\"loadWhisper('tiny-en-q5_1')\">tiny.en (Q5_1, 31 MB)</button>\n                <button id=\"fetch-whisper-base-en-q5_1\"   onclick=\"loadWhisper('base-en-q5_1')\">base.en (Q5_1, 57 MB)</button>\n                <button id=\"fetch-whisper-small-en-q5_1\"  onclick=\"loadWhisper('small-en-q5_1')\">small.en (Q5_1, 182 MB)</button>\n                <button id=\"fetch-whisper-medium-en-q5_0\" onclick=\"loadWhisper('medium-en-q5_0')\">medium.en (Q5_0, 515 MB)</button>\n                <button id=\"fetch-whisper-large-q5_0\"     onclick=\"loadWhisper('large-q5_0')\">large (Q5_0, 1030 MB)</button>\n                <span id=\"fetch-whisper-progress\"></span>\n            </div>\n\n            <br>\n\n            <div id=\"input\">\n                <button id=\"bench\" onclick=\"onBench()\" disabled>Bench</button>\n                <button id=\"clear\" onclick=\"clearCache()\">Clear Cache</button>\n            </div>\n\n            <hr>\n\n            Debug output:\n            <textarea id=\"output\" rows=\"20\"></textarea>\n\n            <br>\n\n            <b>Troubleshooting</b>\n\n            <br><br>\n\n            The page does some heavy computations, so make sure:\n\n            <ul>\n                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>\n                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>\n                <li>Your browser supports WASM <a href=\"https://webassembly.org/roadmap/\">Fixed-width SIMD</a></li>\n            </ul>\n\n            <div class=\"cell-version\">\n                <span>\n                    |\n                    Build time: <span class=\"nav-link\">@GIT_DATE@</span> |\n                    Commit hash: <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@\">@GIT_SHA1@</a> |\n                    Commit subject: <span class=\"nav-link\">@GIT_COMMIT_SUBJECT@</span> |\n                    <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/bench.wasm\">Source Code</a> |\n                </span>\n            </div>\n        </div>\n\n        <script type=\"text/javascript\" src=\"helpers.js\"></script>\n        <script type='text/javascript'>\n            // the bench instance\n            var instance = null;\n\n            // model name\n            var model_whisper = null;\n\n            var Module = {\n                print: printTextarea,\n                printErr: printTextarea,\n                setStatus: function(text) {\n                    printTextarea('js: ' + text);\n                },\n                monitorRunDependencies: function(left) {\n                },\n                preRun: function() {\n                    printTextarea('js: Preparing ...');\n                },\n                postRun: function() {\n                    printTextarea('js: Initialized successfully!');\n                }\n            };\n\n            //\n            // fetch models\n            //\n\n            let dbVersion = 1\n            let dbName    = 'whisper.ggerganov.com';\n            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB\n\n            function storeFS(fname, buf) {\n                // write to WASM file using FS_createDataFile\n                // if the file exists, delete it\n                try {\n                    Module.FS_unlink(fname);\n                } catch (e) {\n                    // ignore\n                }\n\n                Module.FS_createDataFile(\"/\", fname, buf, true, true);\n\n                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);\n\n                model_whisper = fname;\n\n                document.getElementById('model-whisper-status').innerHTML = 'loaded \"' + model_whisper + '\"!';\n\n                if (model_whisper != null) {\n                    document.getElementById('bench').disabled = false;\n                }\n            }\n\n            function loadFile(event, fname) {\n                var file = event.target.files[0] || null;\n                if (file == null) {\n                    return;\n                }\n\n                printTextarea(\"loadFile: loading model: \" + file.name + \", size: \" + file.size + \" bytes\");\n                printTextarea('loadFile: please wait ...');\n\n                var reader = new FileReader();\n                reader.onload = function(event) {\n                    var buf = new Uint8Array(reader.result);\n                    storeFS(fname, buf);\n                }\n                reader.readAsArrayBuffer(file);\n\n                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';\n                document.getElementById('fetch-whisper-base-en').style.display = 'none';\n                document.getElementById('fetch-whisper-small-en').style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';\n                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';\n                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';\n\n                document.getElementById('whisper-file'         ).style.display = 'none';\n                document.getElementById('model-whisper-status' ).innerHTML = 'loaded model: ' + file.name;\n            }\n\n            function loadWhisper(model) {\n                let urls = {\n                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',\n                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',\n                    'small.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin',\n\n                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',\n                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',\n                    'small-en-q5_1': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en-q5_1.bin',\n                    'medium-en-q5_0':'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en-q5_0.bin',\n                    'large-q5_0':    'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-q5_0.bin',\n                };\n\n                let sizes = {\n                    'tiny.en': 75,\n                    'base.en': 142,\n                    'small.en': 466,\n\n                    'tiny-en-q5_1':   31,\n                    'base-en-q5_1':   57,\n                    'small-en-q5_1':  182,\n                    'medium-en-q5_0': 515,\n                    'large-q5_0':     1030,\n                };\n\n                let url     = urls[model];\n                let dst     = 'whisper.bin';\n                let size_mb = sizes[model];\n\n                document.getElementById('fetch-whisper-tiny-en').style.display  = 'none';\n                document.getElementById('fetch-whisper-base-en').style.display  = 'none';\n                document.getElementById('fetch-whisper-small-en').style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';\n                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';\n                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';\n\n                document.getElementById('whisper-file'        ).style.display = 'none';\n                document.getElementById('model-whisper-status').innerHTML = 'loading \"' + model + '\" ... ';\n\n                cbProgress = function(p) {\n                    let el = document.getElementById('fetch-whisper-progress');\n                    el.innerHTML = Math.round(100*p) + '%';\n                };\n\n                cbCancel = function() {\n                    var el;\n                    el = document.getElementById('fetch-whisper-tiny-en');  if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en');  if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small-en'); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'  ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en-q5_1'  ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small-en-q5_1' ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-medium-en-q5_0'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-large-q5_0'    ); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('whisper-file'        ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('model-whisper-status'); if (el) el.innerHTML = '';\n                };\n\n                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);\n            }\n\n            //\n            // main\n            //\n\n            function onBench() {\n                if (instance) {\n                    Module.free(instance);\n                }\n\n                instance = Module.init('whisper.bin');\n\n                if (instance) {\n                    printTextarea(\"js: whisper initialized, instance: \" + instance);\n                }\n\n                document.getElementById('bench').disabled = true;\n\n                if (!instance) {\n                    printTextarea(\"js: failed to initialize whisper\");\n                    return;\n                }\n            }\n\n        </script>\n        <script type=\"text/javascript\" src=\"bench.js\"></script>\n    </body>\n</html>\n"
  },
  {
    "path": "examples/cli/CMakeLists.txt",
    "content": "set(TARGET whisper-cli)\nadd_executable(${TARGET} cli.cpp)\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})\n\ninstall(TARGETS ${TARGET} RUNTIME)\n"
  },
  {
    "path": "examples/cli/README.md",
    "content": "# whisper.cpp/examples/cli\r\n\r\nThis is the main example demonstrating most of the functionality of the Whisper model.\r\nIt can be used as a reference for using the `whisper.cpp` library in other projects.\r\n\r\n```\r\n./build/bin/whisper-cli -h\r\n\r\nusage: ./build/bin/whisper-cli [options] file0 file1 ...\r\nsupported audio formats: flac, mp3, ogg, wav\r\n\r\noptions:\r\n  -h,        --help              [default] show this help message and exit\r\n  -t N,      --threads N         [4      ] number of threads to use during computation\r\n  -p N,      --processors N      [1      ] number of processors to use during computation\r\n  -ot N,     --offset-t N        [0      ] time offset in milliseconds\r\n  -on N,     --offset-n N        [0      ] segment index offset\r\n  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds\r\n  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store\r\n  -ml N,     --max-len N         [0      ] maximum segment length in characters\r\n  -sow,      --split-on-word     [false  ] split on word rather than on token\r\n  -bo N,     --best-of N         [5      ] number of best candidates to keep\r\n  -bs N,     --beam-size N       [5      ] beam size for beam search\r\n  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)\r\n  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold\r\n  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail\r\n  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail\r\n  -nth N,    --no-speech-thold N [0.60   ] no speech threshold\r\n  -tp,       --temperature N     [0.00   ] The sampling temperature, between 0 and 1\r\n  -tpi,      --temperature-inc N [0.20   ] The increment of temperature, between 0 and 1\r\n  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)\r\n  -tr,       --translate         [false  ] translate from source language to english\r\n  -di,       --diarize           [false  ] stereo audio diarization\r\n  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)\r\n  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding\r\n  -otxt,     --output-txt        [false  ] output result in a text file\r\n  -ovtt,     --output-vtt        [false  ] output result in a vtt file\r\n  -osrt,     --output-srt        [false  ] output result in a srt file\r\n  -olrc,     --output-lrc        [false  ] output result in a lrc file\r\n  -owts,     --output-words      [false  ] output script for generating karaoke video\r\n  -fp,       --font-path         [/System/Library/Fonts/Supplemental/Courier New Bold.ttf] path to a monospace font for karaoke video\r\n  -ocsv,     --output-csv        [false  ] output result in a CSV file\r\n  -oj,       --output-json       [false  ] output result in a JSON file\r\n  -ojf,      --output-json-full  [false  ] include more information in the JSON file\r\n  -of FNAME, --output-file FNAME [       ] output file path (without file extension)\r\n  -np,       --no-prints         [false  ] do not print anything other than the results\r\n  -ps,       --print-special     [false  ] print special tokens\r\n  -pc,       --print-colors      [false  ] print colors\r\n  -pp,       --print-progress    [false  ] print progress\r\n  -nt,       --no-timestamps     [false  ] do not print timestamps\r\n  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)\r\n  -dl,       --detect-language   [false  ] exit after automatically detecting language\r\n             --prompt PROMPT     [       ] initial prompt (max n_text_ctx/2 tokens)\r\n  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path\r\n  -f FNAME,  --file FNAME        [       ] input audio file path\r\n  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference\r\n  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps\r\n  -ls,       --log-score         [false  ] log best decoder scores of tokens\r\n  -ng,       --no-gpu            [false  ] disable GPU\r\n  -fa,       --flash-attn        [false  ] flash attention\r\n  -sns,      --suppress-nst      [false  ] suppress non-speech tokens\r\n  --suppress-regex REGEX         [       ] regular expression matching tokens to suppress\r\n  --grammar GRAMMAR              [       ] GBNF grammar to guide decoding\r\n  --grammar-rule RULE            [       ] top-level GBNF grammar rule name\r\n  --grammar-penalty N            [100.0  ] scales down logits of nongrammar tokens\r\n```\r\n"
  },
  {
    "path": "examples/cli/cli.cpp",
    "content": "#include \"common.h\"\n#include \"common-whisper.h\"\n\n#include \"whisper.h\"\n#include \"grammar-parser.h\"\n\n#include <cmath>\n#include <algorithm>\n#include <fstream>\n#include <cstdio>\n#include <string>\n#include <thread>\n#include <vector>\n#include <cstring>\n#include <cfloat>\n\n#if defined(_WIN32)\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif\n#include <windows.h>\n#endif\n\n// helper function to replace substrings\nstatic void replace_all(std::string & s, const std::string & search, const std::string & replace) {\n    for (size_t pos = 0; ; pos += replace.length()) {\n        pos = s.find(search, pos);\n        if (pos == std::string::npos) break;\n        s.erase(pos, search.length());\n        s.insert(pos, replace);\n    }\n}\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads     = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t n_processors  = 1;\n    int32_t offset_t_ms   = 0;\n    int32_t offset_n      = 0;\n    int32_t duration_ms   = 0;\n    int32_t progress_step = 5;\n    int32_t max_context   = -1;\n    int32_t max_len       = 0;\n    int32_t best_of       = whisper_full_default_params(WHISPER_SAMPLING_GREEDY).greedy.best_of;\n    int32_t beam_size     = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH).beam_search.beam_size;\n    int32_t audio_ctx     = 0;\n\n    float word_thold      =  0.01f;\n    float entropy_thold   =  2.40f;\n    float logprob_thold   = -1.00f;\n    float no_speech_thold =  0.6f;\n    float grammar_penalty = 100.0f;\n    float temperature     = 0.0f;\n    float temperature_inc = 0.2f;\n\n    bool debug_mode      = false;\n    bool translate       = false;\n    bool detect_language = false;\n    bool diarize         = false;\n    bool tinydiarize     = false;\n    bool split_on_word   = false;\n    bool no_fallback     = false;\n    bool output_txt      = false;\n    bool output_vtt      = false;\n    bool output_srt      = false;\n    bool output_wts      = false;\n    bool output_csv      = false;\n    bool output_jsn      = false;\n    bool output_jsn_full = false;\n    bool output_lrc      = false;\n    bool no_prints       = false;\n    bool print_special   = false;\n    bool print_colors    = false;\n    bool print_confidence= false;\n    bool print_progress  = false;\n    bool no_timestamps   = false;\n    bool log_score       = false;\n    bool use_gpu         = true;\n    bool flash_attn      = true;\n    int32_t gpu_device   = 0;\n    bool suppress_nst    = false;\n    bool carry_initial_prompt = false;\n\n    std::string language  = \"en\";\n    std::string prompt;\n    std::string font_path = \"/System/Library/Fonts/Supplemental/Courier New Bold.ttf\";\n    std::string model     = \"models/ggml-base.en.bin\";\n    std::string grammar;\n    std::string grammar_rule;\n\n    // [TDRZ] speaker turn string\n    std::string tdrz_speaker_turn = \" [SPEAKER_TURN]\"; // TODO: set from command line\n\n    // A regular expression that matches tokens to suppress\n    std::string suppress_regex;\n\n    std::string openvino_encode_device = \"CPU\";\n\n    std::string dtw = \"\";\n\n    std::vector<std::string> fname_inp = {};\n    std::vector<std::string> fname_out = {};\n\n    grammar_parser::parse_state grammar_parsed;\n\n    // Voice Activity Detection (VAD) parameters\n    bool        vad           = false;\n    std::string vad_model     = \"\";\n    float       vad_threshold = 0.5f;\n    int         vad_min_speech_duration_ms = 250;\n    int         vad_min_silence_duration_ms = 100;\n    float       vad_max_speech_duration_s = FLT_MAX;\n    int         vad_speech_pad_ms = 30;\n    float       vad_samples_overlap = 0.1f;\n};\n\nstatic void whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic char * whisper_param_turn_lowercase(char * in){\n    int string_len = strlen(in);\n    for (int i = 0; i < string_len; i++){\n        *(in+i) = tolower((unsigned char)*(in+i));\n    }\n    return in;\n}\n\nstatic char * requires_value_error(const std::string & arg) {\n    fprintf(stderr, \"error: argument %s requires value\\n\", arg.c_str());\n    exit(0);\n}\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    if (const char * env_device = std::getenv(\"WHISPER_ARG_DEVICE\")) {\n        params.gpu_device = std::stoi(env_device);\n    }\n\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-\"){\n            params.fname_inp.push_back(arg);\n            continue;\n        }\n\n        if (arg[0] != '-') {\n            params.fname_inp.push_back(arg);\n            continue;\n        }\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))\n        else if (arg == \"-t\"    || arg == \"--threads\")              { params.n_threads       = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-p\"    || arg == \"--processors\")           { params.n_processors    = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-ot\"   || arg == \"--offset-t\")             { params.offset_t_ms     = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-on\"   || arg == \"--offset-n\")             { params.offset_n        = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-d\"    || arg == \"--duration\")             { params.duration_ms     = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-mc\"   || arg == \"--max-context\")          { params.max_context     = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-ml\"   || arg == \"--max-len\")              { params.max_len         = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-bo\"   || arg == \"--best-of\")              { params.best_of         = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-bs\"   || arg == \"--beam-size\")            { params.beam_size       = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-ac\"   || arg == \"--audio-ctx\")            { params.audio_ctx       = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-wt\"   || arg == \"--word-thold\")           { params.word_thold      = std::stof(ARGV_NEXT); }\n        else if (arg == \"-et\"   || arg == \"--entropy-thold\")        { params.entropy_thold   = std::stof(ARGV_NEXT); }\n        else if (arg == \"-lpt\"  || arg == \"--logprob-thold\")        { params.logprob_thold   = std::stof(ARGV_NEXT); }\n        else if (arg == \"-nth\"  || arg == \"--no-speech-thold\")      { params.no_speech_thold = std::stof(ARGV_NEXT); }\n        else if (arg == \"-tp\"   || arg == \"--temperature\")          { params.temperature     = std::stof(ARGV_NEXT); }\n        else if (arg == \"-tpi\"  || arg == \"--temperature-inc\")      { params.temperature_inc = std::stof(ARGV_NEXT); }\n        else if (arg == \"-debug\"|| arg == \"--debug-mode\")           { params.debug_mode      = true; }\n        else if (arg == \"-tr\"   || arg == \"--translate\")            { params.translate       = true; }\n        else if (arg == \"-di\"   || arg == \"--diarize\")              { params.diarize         = true; }\n        else if (arg == \"-tdrz\" || arg == \"--tinydiarize\")          { params.tinydiarize     = true; }\n        else if (arg == \"-sow\"  || arg == \"--split-on-word\")        { params.split_on_word   = true; }\n        else if (arg == \"-nf\"   || arg == \"--no-fallback\")          { params.no_fallback     = true; }\n        else if (arg == \"-otxt\" || arg == \"--output-txt\")           { params.output_txt      = true; }\n        else if (arg == \"-ovtt\" || arg == \"--output-vtt\")           { params.output_vtt      = true; }\n        else if (arg == \"-osrt\" || arg == \"--output-srt\")           { params.output_srt      = true; }\n        else if (arg == \"-owts\" || arg == \"--output-words\")         { params.output_wts      = true; }\n        else if (arg == \"-olrc\" || arg == \"--output-lrc\")           { params.output_lrc      = true; }\n        else if (arg == \"-fp\"   || arg == \"--font-path\")            { params.font_path       = ARGV_NEXT; }\n        else if (arg == \"-ocsv\" || arg == \"--output-csv\")           { params.output_csv      = true; }\n        else if (arg == \"-oj\"   || arg == \"--output-json\")          { params.output_jsn      = true; }\n        else if (arg == \"-ojf\"  || arg == \"--output-json-full\")     { params.output_jsn_full = params.output_jsn = true; }\n        else if (arg == \"-of\"   || arg == \"--output-file\")          { params.fname_out.emplace_back(ARGV_NEXT); }\n        else if (arg == \"-np\"   || arg == \"--no-prints\")            { params.no_prints       = true; }\n        else if (arg == \"-ps\"   || arg == \"--print-special\")        { params.print_special   = true; }\n        else if (arg == \"-pc\"   || arg == \"--print-colors\")         { params.print_colors    = true; }\n        else if (                  arg == \"--print-confidence\")     { params.print_confidence= true; }\n        else if (arg == \"-pp\"   || arg == \"--print-progress\")       { params.print_progress  = true; }\n        else if (arg == \"-nt\"   || arg == \"--no-timestamps\")        { params.no_timestamps   = true; }\n        else if (arg == \"-l\"    || arg == \"--language\")             { params.language        = whisper_param_turn_lowercase(ARGV_NEXT); }\n        else if (arg == \"-dl\"   || arg == \"--detect-language\")      { params.detect_language = true; }\n        else if (                  arg == \"--prompt\")               { params.prompt          = ARGV_NEXT; }\n        else if (                  arg == \"--carry-initial-prompt\") { params.carry_initial_prompt = true; }\n        else if (arg == \"-m\"    || arg == \"--model\")                { params.model           = ARGV_NEXT; }\n        else if (arg == \"-f\"    || arg == \"--file\")                 { params.fname_inp.emplace_back(ARGV_NEXT); }\n        else if (arg == \"-oved\" || arg == \"--ov-e-device\")          { params.openvino_encode_device = ARGV_NEXT; }\n        else if (arg == \"-dtw\"  || arg == \"--dtw\")                  { params.dtw             = ARGV_NEXT; }\n        else if (arg == \"-ls\"   || arg == \"--log-score\")            { params.log_score       = true; }\n        else if (arg == \"-ng\"   || arg == \"--no-gpu\")               { params.use_gpu         = false; }\n        else if (arg == \"-dev\"  || arg == \"--device\")               { params.gpu_device      = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-fa\"   || arg == \"--flash-attn\")           { params.flash_attn      = true; }\n        else if (arg == \"-nfa\"  || arg == \"--no-flash-attn\")        { params.flash_attn      = false; }\n        else if (arg == \"-sns\"  || arg == \"--suppress-nst\")         { params.suppress_nst    = true; }\n        else if (                  arg == \"--suppress-regex\")       { params.suppress_regex  = ARGV_NEXT; }\n        else if (                  arg == \"--grammar\")              { params.grammar         = ARGV_NEXT; }\n        else if (                  arg == \"--grammar-rule\")         { params.grammar_rule    = ARGV_NEXT; }\n        else if (                  arg == \"--grammar-penalty\")      { params.grammar_penalty = std::stof(ARGV_NEXT); }\n        // Voice Activity Detection (VAD)\n        else if (                  arg == \"--vad\")                         { params.vad                         = true; }\n        else if (arg == \"-vm\"   || arg == \"--vad-model\")                   { params.vad_model                   = ARGV_NEXT; }\n        else if (arg == \"-vt\"   || arg == \"--vad-threshold\")               { params.vad_threshold               = std::stof(ARGV_NEXT); }\n        else if (arg == \"-vspd\" || arg == \"--vad-min-speech-duration-ms\")  { params.vad_min_speech_duration_ms  = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vsd\"  || arg == \"--vad-min-silence-duration-ms\") { params.vad_min_silence_duration_ms = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vmsd\" || arg == \"--vad-max-speech-duration-s\")   { params.vad_max_speech_duration_s   = std::stof(ARGV_NEXT); }\n        else if (arg == \"-vp\"   || arg == \"--vad-speech-pad-ms\")           { params.vad_speech_pad_ms           = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vo\"   || arg == \"--vad-samples-overlap\")         { params.vad_samples_overlap         = std::stof(ARGV_NEXT); }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nstatic void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options] file0 file1 ...\\n\", argv[0]);\n    fprintf(stderr, \"supported audio formats: flac, mp3, ogg, wav\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,        --help                 [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,      --threads N            [%-7d] number of threads to use during computation\\n\",    params.n_threads);\n    fprintf(stderr, \"  -p N,      --processors N         [%-7d] number of processors to use during computation\\n\", params.n_processors);\n    fprintf(stderr, \"  -ot N,     --offset-t N           [%-7d] time offset in milliseconds\\n\",                    params.offset_t_ms);\n    fprintf(stderr, \"  -on N,     --offset-n N           [%-7d] segment index offset\\n\",                           params.offset_n);\n    fprintf(stderr, \"  -d  N,     --duration N           [%-7d] duration of audio to process in milliseconds\\n\",   params.duration_ms);\n    fprintf(stderr, \"  -mc N,     --max-context N        [%-7d] maximum number of text context tokens to store\\n\", params.max_context);\n    fprintf(stderr, \"  -ml N,     --max-len N            [%-7d] maximum segment length in characters\\n\",           params.max_len);\n    fprintf(stderr, \"  -sow,      --split-on-word        [%-7s] split on word rather than on token\\n\",             params.split_on_word ? \"true\" : \"false\");\n    fprintf(stderr, \"  -bo N,     --best-of N            [%-7d] number of best candidates to keep\\n\",              params.best_of);\n    fprintf(stderr, \"  -bs N,     --beam-size N          [%-7d] beam size for beam search\\n\",                      params.beam_size);\n    fprintf(stderr, \"  -ac N,     --audio-ctx N          [%-7d] audio context size (0 - all)\\n\",                   params.audio_ctx);\n    fprintf(stderr, \"  -wt N,     --word-thold N         [%-7.2f] word timestamp probability threshold\\n\",         params.word_thold);\n    fprintf(stderr, \"  -et N,     --entropy-thold N      [%-7.2f] entropy threshold for decoder fail\\n\",           params.entropy_thold);\n    fprintf(stderr, \"  -lpt N,    --logprob-thold N      [%-7.2f] log probability threshold for decoder fail\\n\",   params.logprob_thold);\n    fprintf(stderr, \"  -nth N,    --no-speech-thold N    [%-7.2f] no speech threshold\\n\",                          params.no_speech_thold);\n    fprintf(stderr, \"  -tp,       --temperature N        [%-7.2f] The sampling temperature, between 0 and 1\\n\",    params.temperature);\n    fprintf(stderr, \"  -tpi,      --temperature-inc N    [%-7.2f] The increment of temperature, between 0 and 1\\n\",params.temperature_inc);\n    fprintf(stderr, \"  -debug,    --debug-mode           [%-7s] enable debug mode (eg. dump log_mel)\\n\",           params.debug_mode ? \"true\" : \"false\");\n    fprintf(stderr, \"  -tr,       --translate            [%-7s] translate from source language to english\\n\",      params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -di,       --diarize              [%-7s] stereo audio diarization\\n\",                       params.diarize ? \"true\" : \"false\");\n    fprintf(stderr, \"  -tdrz,     --tinydiarize          [%-7s] enable tinydiarize (requires a tdrz model)\\n\",     params.tinydiarize ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nf,       --no-fallback          [%-7s] do not use temperature fallback while decoding\\n\", params.no_fallback ? \"true\" : \"false\");\n    fprintf(stderr, \"  -otxt,     --output-txt           [%-7s] output result in a text file\\n\",                   params.output_txt ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ovtt,     --output-vtt           [%-7s] output result in a vtt file\\n\",                    params.output_vtt ? \"true\" : \"false\");\n    fprintf(stderr, \"  -osrt,     --output-srt           [%-7s] output result in a srt file\\n\",                    params.output_srt ? \"true\" : \"false\");\n    fprintf(stderr, \"  -olrc,     --output-lrc           [%-7s] output result in a lrc file\\n\",                    params.output_lrc ? \"true\" : \"false\");\n    fprintf(stderr, \"  -owts,     --output-words         [%-7s] output script for generating karaoke video\\n\",     params.output_wts ? \"true\" : \"false\");\n    fprintf(stderr, \"  -fp,       --font-path            [%-7s] path to a monospace font for karaoke video\\n\",     params.font_path.c_str());\n    fprintf(stderr, \"  -ocsv,     --output-csv           [%-7s] output result in a CSV file\\n\",                    params.output_csv ? \"true\" : \"false\");\n    fprintf(stderr, \"  -oj,       --output-json          [%-7s] output result in a JSON file\\n\",                   params.output_jsn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ojf,      --output-json-full     [%-7s] include more information in the JSON file\\n\",      params.output_jsn_full ? \"true\" : \"false\");\n    fprintf(stderr, \"  -of FNAME, --output-file FNAME    [%-7s] output file path (without file extension)\\n\",      \"\");\n    fprintf(stderr, \"  -np,       --no-prints            [%-7s] do not print anything other than the results\\n\",   params.no_prints ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,       --print-special        [%-7s] print special tokens\\n\",                           params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pc,       --print-colors         [%-7s] print colors\\n\",                                   params.print_colors ? \"true\" : \"false\");\n    fprintf(stderr, \"             --print-confidence     [%-7s] print confidence\\n\",                               params.print_confidence ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pp,       --print-progress       [%-7s] print progress\\n\",                                 params.print_progress ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nt,       --no-timestamps        [%-7s] do not print timestamps\\n\",                        params.no_timestamps ? \"true\" : \"false\");\n    fprintf(stderr, \"  -l LANG,   --language LANG        [%-7s] spoken language ('auto' for auto-detect)\\n\",       params.language.c_str());\n    fprintf(stderr, \"  -dl,       --detect-language      [%-7s] exit after automatically detecting language\\n\",    params.detect_language ? \"true\" : \"false\");\n    fprintf(stderr, \"             --prompt PROMPT        [%-7s] initial prompt (max n_text_ctx/2 tokens)\\n\",       params.prompt.c_str());\n    fprintf(stderr, \"             --carry-initial-prompt [%-7s] always prepend initial prompt\\n\",                  params.carry_initial_prompt ? \"true\" : \"false\");\n    fprintf(stderr, \"  -m FNAME,  --model FNAME          [%-7s] model path\\n\",                                     params.model.c_str());\n    fprintf(stderr, \"  -f FNAME,  --file FNAME           [%-7s] input audio file path\\n\",                          \"\");\n    fprintf(stderr, \"  -oved D,   --ov-e-device DNAME    [%-7s] the OpenVINO device used for encode inference\\n\",  params.openvino_encode_device.c_str());\n    fprintf(stderr, \"  -dtw MODEL --dtw MODEL            [%-7s] compute token-level timestamps\\n\",                 params.dtw.c_str());\n    fprintf(stderr, \"  -ls,       --log-score            [%-7s] log best decoder scores of tokens\\n\",              params.log_score?\"true\":\"false\");\n    fprintf(stderr, \"  -ng,       --no-gpu               [%-7s] disable GPU\\n\",                                    params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -dev N,    --device N             [%-7d] GPU device ID (default: 0)\\n\",                     params.gpu_device);\n    fprintf(stderr, \"  -fa,       --flash-attn           [%-7s] enable flash attention\\n\",                         params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,      --no-flash-attn        [%-7s] disable flash attention\\n\",                        params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -sns,      --suppress-nst         [%-7s] suppress non-speech tokens\\n\",                     params.suppress_nst ? \"true\" : \"false\");\n    fprintf(stderr, \"  --suppress-regex REGEX            [%-7s] regular expression matching tokens to suppress\\n\", params.suppress_regex.c_str());\n    fprintf(stderr, \"  --grammar GRAMMAR                 [%-7s] GBNF grammar to guide decoding\\n\",                 params.grammar.c_str());\n    fprintf(stderr, \"  --grammar-rule RULE               [%-7s] top-level GBNF grammar rule name\\n\",               params.grammar_rule.c_str());\n    fprintf(stderr, \"  --grammar-penalty N               [%-7.1f] scales down logits of nongrammar tokens\\n\",      params.grammar_penalty);\n    // Voice Activity Detection (VAD) parameters\n    fprintf(stderr, \"\\nVoice Activity Detection (VAD) options:\\n\");\n    fprintf(stderr, \"             --vad                           [%-7s] enable Voice Activity Detection (VAD)\\n\",            params.vad ? \"true\" : \"false\");\n    fprintf(stderr, \"  -vm FNAME, --vad-model FNAME               [%-7s] VAD model path\\n\",                                   params.vad_model.c_str());\n    fprintf(stderr, \"  -vt N,     --vad-threshold N               [%-7.2f] VAD threshold for speech recognition\\n\",           params.vad_threshold);\n    fprintf(stderr, \"  -vspd N,   --vad-min-speech-duration-ms  N [%-7d] VAD min speech duration (0.0-1.0)\\n\",                params.vad_min_speech_duration_ms);\n    fprintf(stderr, \"  -vsd N,    --vad-min-silence-duration-ms N [%-7d] VAD min silence duration (to split segments)\\n\",      params.vad_min_silence_duration_ms);\n    fprintf(stderr, \"  -vmsd N,   --vad-max-speech-duration-s   N [%-7s] VAD max speech duration (auto-split longer)\\n\",      params.vad_max_speech_duration_s == FLT_MAX ?\n                                                                                                                                  std::string(\"FLT_MAX\").c_str() :\n                                                                                                                                  std::to_string(params.vad_max_speech_duration_s).c_str());\n    fprintf(stderr, \"  -vp N,     --vad-speech-pad-ms           N [%-7d] VAD speech padding (extend segments)\\n\",             params.vad_speech_pad_ms);\n    fprintf(stderr, \"  -vo N,     --vad-samples-overlap         N [%-7.2f] VAD samples overlap (seconds between segments)\\n\", params.vad_samples_overlap);\n    fprintf(stderr, \"\\n\");\n}\n\nstruct whisper_print_user_data {\n    const whisper_params * params;\n\n    const std::vector<std::vector<float>> * pcmf32s;\n    int progress_prev;\n};\n\nstatic std::string estimate_diarization_speaker(std::vector<std::vector<float>> pcmf32s, int64_t t0, int64_t t1, bool id_only = false) {\n    std::string speaker = \"\";\n    const int64_t n_samples = pcmf32s[0].size();\n\n    const int64_t is0 = timestamp_to_sample(t0, n_samples, WHISPER_SAMPLE_RATE);\n    const int64_t is1 = timestamp_to_sample(t1, n_samples, WHISPER_SAMPLE_RATE);\n\n    double energy0 = 0.0f;\n    double energy1 = 0.0f;\n\n    for (int64_t j = is0; j < is1; j++) {\n        energy0 += fabs(pcmf32s[0][j]);\n        energy1 += fabs(pcmf32s[1][j]);\n    }\n\n    if (energy0 > 1.1*energy1) {\n        speaker = \"0\";\n    } else if (energy1 > 1.1*energy0) {\n        speaker = \"1\";\n    } else {\n        speaker = \"?\";\n    }\n\n    //printf(\"is0 = %lld, is1 = %lld, energy0 = %f, energy1 = %f, speaker = %s\\n\", is0, is1, energy0, energy1, speaker.c_str());\n\n    if (!id_only) {\n        speaker.insert(0, \"(speaker \");\n        speaker.append(\")\");\n    }\n\n    return speaker;\n}\n\nstatic void whisper_print_progress_callback(struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {\n    int progress_step = ((whisper_print_user_data *) user_data)->params->progress_step;\n    int * progress_prev  = &(((whisper_print_user_data *) user_data)->progress_prev);\n    if (progress >= *progress_prev + progress_step) {\n        *progress_prev += progress_step;\n        fprintf(stderr, \"%s: progress = %3d%%\\n\", __func__, progress);\n    }\n}\n\nstatic void whisper_print_segment_callback(struct whisper_context * ctx, struct whisper_state * /*state*/, int n_new, void * user_data) {\n    const auto & params  = *((whisper_print_user_data *) user_data)->params;\n    const auto & pcmf32s = *((whisper_print_user_data *) user_data)->pcmf32s;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n\n    std::string speaker = \"\";\n\n    int64_t t0 = 0;\n    int64_t t1 = 0;\n\n    // print the last n_new segments\n    const int s0 = n_segments - n_new;\n\n    if (s0 == 0) {\n        printf(\"\\n\");\n    }\n\n    for (int i = s0; i < n_segments; i++) {\n        if (!params.no_timestamps || params.diarize) {\n            t0 = whisper_full_get_segment_t0(ctx, i);\n            t1 = whisper_full_get_segment_t1(ctx, i);\n        }\n\n        if (!params.no_timestamps) {\n            printf(\"[%s --> %s]  \", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());\n        }\n\n        if (params.diarize && pcmf32s.size() == 2) {\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        if (params.print_colors) {\n            for (int j = 0; j < whisper_full_n_tokens(ctx, i); ++j) {\n                if (params.print_special == false) {\n                    const whisper_token id = whisper_full_get_token_id(ctx, i, j);\n                    if (id >= whisper_token_eot(ctx)) {\n                        continue;\n                    }\n                }\n\n                const char * text = whisper_full_get_token_text(ctx, i, j);\n                const float  p    = whisper_full_get_token_p   (ctx, i, j);\n\n                const int n_colors = (int) k_colors.size();\n                int raw_col = (int) (std::pow(p, 3)*float(n_colors));\n                if (raw_col < 0) raw_col = 0;\n                if (raw_col > n_colors - 1) raw_col = n_colors - 1;\n                const int col = raw_col;\n\n                printf(\"%s%s%s%s\", speaker.c_str(), k_colors[col].c_str(), text, \"\\033[0m\");\n            }\n        } else if (params.print_confidence) {\n            for (int j = 0; j < whisper_full_n_tokens(ctx, i); ++j) {\n                if (params.print_special == false) {\n                    const whisper_token id = whisper_full_get_token_id(ctx, i, j);\n                    if (id >= whisper_token_eot(ctx)) {\n                        continue;\n                    }\n                }\n\n                const char * text = whisper_full_get_token_text(ctx, i, j);\n                const float  p    = whisper_full_get_token_p   (ctx, i, j);\n\n                int style_idx = 2;     // High confidence - dim\n                if (p < 0.33) {\n                    style_idx = 0;     // Low confidence - inverse (highlighted)\n                } else if (p < 0.66) {\n                    style_idx = 1;     // Medium confidence - underlined\n                }\n                printf(\"%s%s%s%s\", speaker.c_str(), k_styles[style_idx].c_str(), text, \"\\033[0m\");\n            }\n        } else {\n            const char * text = whisper_full_get_segment_text(ctx, i);\n\n            printf(\"%s%s\", speaker.c_str(), text);\n        }\n\n        if (params.tinydiarize) {\n            if (whisper_full_get_segment_speaker_turn_next(ctx, i)) {\n                printf(\"%s\", params.tdrz_speaker_turn.c_str());\n            }\n        }\n\n        // with timestamps or speakers: each segment on new line\n        if (!params.no_timestamps || params.diarize) {\n            printf(\"\\n\");\n        }\n\n        fflush(stdout);\n    }\n}\n\nstatic void output_txt(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        fout << speaker << text << \"\\n\";\n    }\n}\n\nstatic void output_vtt(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    fout << \"WEBVTT\\n\\n\";\n\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1, true);\n            speaker.insert(0, \"<v Speaker\");\n            speaker.append(\">\");\n        }\n\n        fout << to_timestamp(t0) << \" --> \" << to_timestamp(t1) << \"\\n\";\n        fout << speaker << text << \"\\n\\n\";\n    }\n}\n\nstatic void output_srt(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        fout << i + 1 + params.offset_n << \"\\n\";\n        fout << to_timestamp(t0, true) << \" --> \" << to_timestamp(t1, true) << \"\\n\";\n        fout << speaker << text << \"\\n\\n\";\n    }\n}\n\nstatic char * escape_double_quotes_and_backslashes(const char * str) {\n    if (str == NULL) {\n        return NULL;\n    }\n\n    size_t escaped_length = strlen(str) + 1;\n\n    for (size_t i = 0; str[i] != '\\0'; i++) {\n        if (str[i] == '\"' || str[i] == '\\\\') {\n            escaped_length++;\n        }\n    }\n\n    char * escaped = (char *)calloc(escaped_length, 1); // pre-zeroed\n    if (escaped == NULL) {\n        return NULL;\n    }\n\n    size_t pos = 0;\n    for (size_t i = 0; str[i] != '\\0'; i++) {\n        if (str[i] == '\"' || str[i] == '\\\\') {\n            escaped[pos++] = '\\\\';\n        }\n        escaped[pos++] = str[i];\n    }\n\n    // no need to set zero due to calloc() being used prior\n\n    return escaped;\n}\n\n// double quote should be escaped by another double quote. (rfc4180)\nstatic char * escape_double_quotes_in_csv(const char * str) {\n    if (str == NULL) {\n        return NULL;\n    }\n\n    size_t escaped_length = strlen(str) + 1;\n\n    for (size_t i = 0; str[i] != '\\0'; i++) {\n        if (str[i] == '\"') {\n            escaped_length++;\n        }\n    }\n\n    char *escaped = (char *)calloc(escaped_length, 1); // pre-zeroed\n    if (escaped == NULL) {\n        return NULL;\n    }\n\n    size_t pos = 0;\n    for (size_t i = 0; str[i] != '\\0'; i++) {\n        if (str[i] == '\"') {\n            escaped[pos++] = '\"';\n        }\n        escaped[pos++] = str[i];\n    }\n\n    // no need to set zero due to calloc() being used prior\n\n    return escaped;\n}\n\nstatic void output_csv(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    const int n_segments = whisper_full_n_segments(ctx);\n    fout << \"start,end,\";\n    if (params.diarize && pcmf32s.size() == 2)\n    {\n        fout << \"speaker,\";\n    }\n    fout << \"text\\n\";\n\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n        char * text_escaped = escape_double_quotes_in_csv(text);\n\n        //need to multiply times returned from whisper_full_get_segment_t{0,1}() by 10 to get milliseconds.\n        fout << 10 * t0 << \",\" << 10 * t1 << \",\";\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            fout << estimate_diarization_speaker(pcmf32s, t0, t1, true) << \",\";\n        }\n        fout << \"\\\"\" << text_escaped << \"\\\"\\n\";\n    }\n}\n\nstatic void output_score(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & /*params*/, std::vector<std::vector<float>> /*pcmf32s*/) {\n    const int n_segments = whisper_full_n_segments(ctx);\n    // fprintf(stderr,\"segments: %d\\n\",n_segments);\n    for (int i = 0; i < n_segments; ++i) {\n        const int n_tokens = whisper_full_n_tokens(ctx, i);\n        // fprintf(stderr,\"tokens: %d\\n\",n_tokens);\n        for (int j = 0; j < n_tokens; j++) {\n            auto token = whisper_full_get_token_text(ctx, i, j);\n            auto probability = whisper_full_get_token_p(ctx, i, j);\n            fout << token << '\\t' << probability << std::endl;\n            // fprintf(stderr,\"token: %s %f\\n\",token,probability);\n\t    }\n    }\n}\n\nstatic void output_json(\n             struct whisper_context * ctx,\n                      std::ofstream & fout,\n               const whisper_params & params,\n    std::vector<std::vector<float>>   pcmf32s) {\n    const bool full = params.output_jsn_full;\n    int indent = 0;\n\n    auto doindent = [&]() {\n        for (int i = 0; i < indent; i++) fout << \"\\t\";\n    };\n\n    auto start_arr = [&](const char *name) {\n        doindent();\n        fout << \"\\\"\" << name << \"\\\": [\\n\";\n        indent++;\n    };\n\n    auto end_arr = [&](bool end) {\n        indent--;\n        doindent();\n        fout << (end ? \"]\\n\" : \"],\\n\");\n    };\n\n    auto start_obj = [&](const char *name) {\n        doindent();\n        if (name) {\n            fout << \"\\\"\" << name << \"\\\": {\\n\";\n        } else {\n            fout << \"{\\n\";\n        }\n        indent++;\n    };\n\n    auto end_obj = [&](bool end) {\n        indent--;\n        doindent();\n        fout << (end ? \"}\\n\" : \"},\\n\");\n    };\n\n    auto start_value = [&](const char *name) {\n        doindent();\n        fout << \"\\\"\" << name << \"\\\": \";\n    };\n\n    auto value_s = [&](const char *name, const char *val, bool end) {\n        start_value(name);\n        char * val_escaped = escape_double_quotes_and_backslashes(val);\n        fout << \"\\\"\" << val_escaped << (end ? \"\\\"\\n\" : \"\\\",\\n\");\n        free(val_escaped);\n    };\n\n    auto end_value = [&](bool end) {\n        fout << (end ? \"\\n\" : \",\\n\");\n    };\n\n    auto value_i = [&](const char *name, const int64_t val, bool end) {\n        start_value(name);\n        fout << val;\n        end_value(end);\n    };\n\n    auto value_f = [&](const char *name, const float val, bool end) {\n        start_value(name);\n        fout << val;\n        end_value(end);\n    };\n\n    auto value_b = [&](const char *name, const bool val, bool end) {\n        start_value(name);\n        fout << (val ? \"true\" : \"false\");\n        end_value(end);\n    };\n\n    auto times_o = [&](int64_t t0, int64_t t1, bool end) {\n        start_obj(\"timestamps\");\n        value_s(\"from\", to_timestamp(t0, true).c_str(), false);\n        value_s(\"to\", to_timestamp(t1, true).c_str(), true);\n        end_obj(false);\n        start_obj(\"offsets\");\n        value_i(\"from\", t0 * 10, false);\n        value_i(\"to\", t1 * 10, true);\n        end_obj(end);\n    };\n\n    start_obj(nullptr);\n        value_s(\"systeminfo\", whisper_print_system_info(), false);\n        start_obj(\"model\");\n            value_s(\"type\", whisper_model_type_readable(ctx), false);\n            value_b(\"multilingual\", whisper_is_multilingual(ctx), false);\n            value_i(\"vocab\", whisper_model_n_vocab(ctx), false);\n            start_obj(\"audio\");\n                value_i(\"ctx\", whisper_model_n_audio_ctx(ctx), false);\n                value_i(\"state\", whisper_model_n_audio_state(ctx), false);\n                value_i(\"head\", whisper_model_n_audio_head(ctx), false);\n                value_i(\"layer\", whisper_model_n_audio_layer(ctx), true);\n            end_obj(false);\n            start_obj(\"text\");\n                value_i(\"ctx\", whisper_model_n_text_ctx(ctx), false);\n                value_i(\"state\", whisper_model_n_text_state(ctx), false);\n                value_i(\"head\", whisper_model_n_text_head(ctx), false);\n                value_i(\"layer\", whisper_model_n_text_layer(ctx), true);\n            end_obj(false);\n            value_i(\"mels\", whisper_model_n_mels(ctx), false);\n            value_i(\"ftype\", whisper_model_ftype(ctx), true);\n        end_obj(false);\n        start_obj(\"params\");\n            value_s(\"model\", params.model.c_str(), false);\n            value_s(\"language\", params.language.c_str(), false);\n            value_b(\"translate\", params.translate, true);\n        end_obj(false);\n        start_obj(\"result\");\n            value_s(\"language\", whisper_lang_str(whisper_full_lang_id(ctx)), true);\n        end_obj(false);\n        start_arr(\"transcription\");\n\n            const int n_segments = whisper_full_n_segments(ctx);\n            for (int i = 0; i < n_segments; ++i) {\n                const char * text = whisper_full_get_segment_text(ctx, i);\n\n                const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n                const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n\n                start_obj(nullptr);\n                    times_o(t0, t1, false);\n                    value_s(\"text\", text, !params.diarize && !params.tinydiarize && !full);\n\n                    if (full) {\n                        start_arr(\"tokens\");\n                        const int n = whisper_full_n_tokens(ctx, i);\n                        for (int j = 0; j < n; ++j) {\n                            auto token = whisper_full_get_token_data(ctx, i, j);\n                            start_obj(nullptr);\n                                value_s(\"text\", whisper_token_to_str(ctx, token.id), false);\n                                if(token.t0 > -1 && token.t1 > -1) {\n                                    // If we have per-token timestamps, write them out\n                                    times_o(token.t0, token.t1, false);\n                                }\n                                value_i(\"id\", token.id, false);\n                                value_f(\"p\", token.p, false);\n                                value_f(\"t_dtw\", token.t_dtw, true);\n                            end_obj(j == (n - 1));\n                        }\n                        end_arr(!params.diarize && !params.tinydiarize);\n                    }\n\n                    if (params.diarize && pcmf32s.size() == 2) {\n                        value_s(\"speaker\", estimate_diarization_speaker(pcmf32s, t0, t1, true).c_str(), true);\n                    }\n\n                    if (params.tinydiarize) {\n                        value_b(\"speaker_turn_next\", whisper_full_get_segment_speaker_turn_next(ctx, i), true);\n                    }\n                end_obj(i == (n_segments - 1));\n            }\n\n        end_arr(true);\n    end_obj(true);\n}\n\n// karaoke video generation\n// outputs a bash script that uses ffmpeg to generate a video with the subtitles\n// TODO: font parameter adjustments\nstatic bool output_wts(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s, const char * fname_inp, float t_sec, const char * fname_out) {\n    static const char * font = params.font_path.c_str();\n\n    std::ifstream fin(font);\n    if (!fin.is_open()) {\n        fprintf(stderr, \"%s: font not found at '%s', please specify a monospace font with -fp\\n\", __func__, font);\n        return false;\n    }\n\n    fout << \"#!/bin/bash\" << \"\\n\";\n    fout << \"\\n\";\n\n    fout << \"ffmpeg -i \" << fname_inp << \" -f lavfi -i color=size=1200x120:duration=\" << t_sec << \":rate=25:color=black -vf \\\"\";\n\n    for (int i = 0; i < whisper_full_n_segments(ctx); i++) {\n        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n\n        const int n = whisper_full_n_tokens(ctx, i);\n\n        std::vector<whisper_token_data> tokens(n);\n        for (int j = 0; j < n; ++j) {\n            tokens[j] = whisper_full_get_token_data(ctx, i, j);\n        }\n\n        if (i > 0) {\n            fout << \",\";\n        }\n\n        // background text\n        fout << \"drawtext=fontfile='\" << font << \"':fontsize=24:fontcolor=gray:x=(w-text_w)/2:y=h/2:text='':enable='between(t,\" << t0/100.0 << \",\" << t0/100.0 << \")'\";\n\n        bool is_first = true;\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2) {\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        for (int j = 0; j < n; ++j) {\n            const auto & token = tokens[j];\n\n            if (tokens[j].id >= whisper_token_eot(ctx)) {\n                continue;\n            }\n\n            std::string txt_bg = \"\";\n            std::string txt_fg = \"\"; // highlight token\n            std::string txt_ul = \"\"; // underline\n\n            if (params.diarize && pcmf32s.size() == 2) {\n                txt_bg = speaker;\n                txt_fg = speaker;\n                txt_ul = \"\\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \\\\ \";\n            }\n\n            txt_bg.append(\"> \");\n            txt_fg.append(\"> \");\n            txt_ul.append(\"\\\\ \\\\ \");\n\n            {\n                for (int k = 0; k < n; ++k) {\n                    const auto & token2 = tokens[k];\n\n                    if (tokens[k].id >= whisper_token_eot(ctx)) {\n                        continue;\n                    }\n\n                    const std::string txt = whisper_token_to_str(ctx, token2.id);\n\n                    txt_bg += txt;\n\n                    if (k == j) {\n                        for (int l = 0; l < (int) txt.size(); ++l) {\n                            txt_fg += txt[l];\n                            txt_ul += \"_\";\n                        }\n                        txt_fg += \"|\";\n                    } else {\n                        for (int l = 0; l < (int) txt.size(); ++l) {\n                            txt_fg += \"\\\\ \";\n                            txt_ul += \"\\\\ \";\n                        }\n                    }\n                }\n\n                ::replace_all(txt_bg, \"'\", \"\\u2019\");\n                ::replace_all(txt_bg, \"\\\"\", \"\\\\\\\"\");\n                ::replace_all(txt_fg, \"'\", \"\\u2019\");\n                ::replace_all(txt_fg, \"\\\"\", \"\\\\\\\"\");\n            }\n\n            if (is_first) {\n                // background text\n                fout << \",drawtext=fontfile='\" << font << \"':fontsize=24:fontcolor=gray:x=(w-text_w)/2:y=h/2:text='\" << txt_bg << \"':enable='between(t,\" << t0/100.0 << \",\" << t1/100.0 << \")'\";\n                is_first = false;\n            }\n\n            // foreground text\n            fout << \",drawtext=fontfile='\" << font << \"':fontsize=24:fontcolor=lightgreen:x=(w-text_w)/2+8:y=h/2:text='\" << txt_fg << \"':enable='between(t,\" << token.t0/100.0 << \",\" << token.t1/100.0 << \")'\";\n\n            // underline\n            fout << \",drawtext=fontfile='\" << font << \"':fontsize=24:fontcolor=lightgreen:x=(w-text_w)/2+8:y=h/2+16:text='\" << txt_ul << \"':enable='between(t,\" << token.t0/100.0 << \",\" << token.t1/100.0 << \")'\";\n        }\n    }\n\n    fout << \"\\\" -c:v libx264 -pix_fmt yuv420p -y \" << fname_inp << \".mp4\" << \"\\n\";\n\n    fout << \"\\n\\n\";\n    fout << \"echo \\\"Your video has been saved to \" << fname_inp << \".mp4\\\"\" << \"\\n\";\n    fout << \"\\n\";\n    fout << \"echo \\\"  ffplay \" << fname_inp << \".mp4\\\"\\n\";\n    fout << \"\\n\";\n\n    fout.close();\n\n    fprintf(stderr, \"# %s: run 'source %s' to generate karaoke video\\n\", __func__, fname_out);\n\n    return true;\n}\n\nstatic void output_lrc(struct whisper_context * ctx, std::ofstream & fout, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    fout << \"[by:whisper.cpp]\\n\";\n\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        const int64_t t = whisper_full_get_segment_t0(ctx, i);\n\n        int64_t msec = t * 10;\n        int64_t min = msec / (1000 * 60);\n        msec = msec - min * (1000 * 60);\n        int64_t sec = msec / 1000;\n        msec = msec - sec * 1000;\n\n        char buf[16];\n        snprintf(buf, sizeof(buf), \"%02d:%02d.%02d\", (int) min, (int) sec, (int) ( msec / 10));\n        std::string timestamp_lrc = std::string(buf);\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        fout <<  '[' << timestamp_lrc << ']' << speaker << text << \"\\n\";\n    }\n}\n\n\nstatic void cb_log_disable(enum ggml_log_level , const char * , void * ) { }\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n#if defined(_WIN32)\n    // Set the console output code page to UTF-8, while command line arguments\n    // are still encoded in the system's code page. In this way, we can print\n    // non-ASCII characters to the console, and access files with non-ASCII paths.\n    SetConsoleOutputCP(CP_UTF8);\n#endif\n\n    whisper_params params;\n\n    // If the only argument starts with \"@\", read arguments line-by-line\n    // from the given file.\n    std::vector<std::string> vec_args;\n    if (argc == 2 && argv != nullptr && argv[1] != nullptr && argv[1][0] == '@') {\n        // Save the name of the executable.\n        vec_args.push_back(argv[0]);\n\n        // Open the response file.\n        char const * rspfile = argv[1] + sizeof(char);\n        std::ifstream fin(rspfile);\n        if (fin.is_open() == false) {\n            fprintf(stderr, \"error: response file '%s' not found\\n\", rspfile);\n            return 1;\n        }\n\n        // Read the entire response file.\n        std::string line;\n        while (std::getline(fin, line)) {\n            vec_args.push_back(line);\n        }\n\n        // Use the contents of the response file as the command-line arguments.\n        argc = static_cast<int>(vec_args.size());\n        argv = static_cast<char **>(alloca(argc * sizeof (char *)));\n        for (int i = 0; i < argc; ++i) {\n            argv[i] = const_cast<char *>(vec_args[i].c_str());\n        }\n    }\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        whisper_print_usage(argc, argv, params);\n        return 1;\n    }\n\n    // remove non-existent files\n    for (auto it = params.fname_inp.begin(); it != params.fname_inp.end();) {\n        const auto fname_inp = it->c_str();\n\n        if (*it != \"-\" && !is_file_exist(fname_inp)) {\n            fprintf(stderr, \"error: input file not found '%s'\\n\", fname_inp);\n            it = params.fname_inp.erase(it);\n            continue;\n        }\n\n        it++;\n    }\n\n    if (params.fname_inp.empty()) {\n        fprintf(stderr, \"error: no input files specified\\n\");\n        whisper_print_usage(argc, argv, params);\n        return 2;\n    }\n\n    if (params.language != \"auto\" && whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    if (params.diarize && params.tinydiarize) {\n        fprintf(stderr, \"error: cannot use both --diarize and --tinydiarize\\n\");\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    if (params.no_prints) {\n        whisper_log_set(cb_log_disable, NULL);\n    }\n\n    // whisper init\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.gpu_device = params.gpu_device;\n    cparams.flash_attn = params.flash_attn;\n\n    if (!params.dtw.empty()) {\n        cparams.dtw_token_timestamps = true;\n        cparams.dtw_aheads_preset = WHISPER_AHEADS_NONE;\n\n        if (params.dtw == \"tiny\")      cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY;\n        if (params.dtw == \"tiny.en\")   cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY_EN;\n        if (params.dtw == \"base\")      cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE;\n        if (params.dtw == \"base.en\")   cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE_EN;\n        if (params.dtw == \"small\")     cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL;\n        if (params.dtw == \"small.en\")  cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL_EN;\n        if (params.dtw == \"medium\")    cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM;\n        if (params.dtw == \"medium.en\") cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM_EN;\n        if (params.dtw == \"large.v1\")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V1;\n        if (params.dtw == \"large.v2\")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V2;\n        if (params.dtw == \"large.v3\")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3;\n        if (params.dtw == \"large.v3.turbo\")  cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3_TURBO;\n\n        if (cparams.dtw_aheads_preset == WHISPER_AHEADS_NONE) {\n            fprintf(stderr, \"error: unknown DTW preset '%s'\\n\", params.dtw.c_str());\n            return 3;\n        }\n    }\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n\n    if (ctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 3;\n    }\n\n    // initialize openvino encoder. this has no effect on whisper.cpp builds that don't have OpenVINO configured\n    whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr);\n\n    if (!params.grammar.empty()) {\n        auto & grammar = params.grammar_parsed;\n        if (is_file_exist(params.grammar.c_str())) {\n            // read grammar from file\n            std::ifstream ifs(params.grammar.c_str());\n            const std::string txt = std::string((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());\n            grammar = grammar_parser::parse(txt.c_str());\n        } else {\n            // read grammar from string\n            grammar = grammar_parser::parse(params.grammar.c_str());\n        }\n\n        // will be empty (default) if there are parse errors\n        if (grammar.rules.empty()) {\n            fprintf(stderr, \"error: failed to parse grammar \\\"%s\\\"\\n\", params.grammar.c_str());\n            return 4;\n        } else {\n            fprintf(stderr, \"%s: grammar:\\n\", __func__);\n            grammar_parser::print_grammar(stderr, grammar);\n            fprintf(stderr, \"\\n\");\n        }\n    }\n\n    for (int f = 0; f < (int) params.fname_inp.size(); ++f) {\n        const auto & fname_inp = params.fname_inp[f];\n        struct fout_factory {\n            std::string fname_out;\n            const size_t basename_length;\n            const bool is_stdout;\n            bool used_stdout;\n            decltype(whisper_print_segment_callback) * const print_segment_callback;\n            std::ofstream fout;\n\n            fout_factory (const std::string & fname_out_, const std::string & fname_inp, whisper_params & params) :\n                    fname_out{!fname_out_.empty() ? fname_out_ : fname_inp},\n                    basename_length{fname_out.size()},\n                    is_stdout{fname_out == \"-\"},\n                    used_stdout{},\n                    print_segment_callback{is_stdout ? nullptr : whisper_print_segment_callback} {\n                if (!print_segment_callback) {\n                    params.print_progress = false;\n                }\n            }\n\n            bool open(const char * ext, const char * function) {\n                if (is_stdout) {\n                    if (used_stdout) {\n                        fprintf(stderr, \"warning: Not appending multiple file formats to stdout\\n\");\n                        return false;\n                    }\n\n                    used_stdout = true;\n#ifdef _WIN32\n                    fout = std::ofstream{\"CON\"};\n#else\n                    fout = std::ofstream{\"/dev/stdout\"};\n#endif\n                    // Not using fprintf stderr here because it might equal stdout\n                    // Also assuming /dev is mounted\n                    return true;\n                }\n\n                fname_out.resize(basename_length);\n                fname_out += ext;\n                fout = std::ofstream{fname_out};\n                if (!fout.is_open()) {\n                    fprintf(stderr, \"%s: failed to open '%s' for writing\\n\", __func__, fname_out.c_str());\n                    return false;\n                }\n                fprintf(stderr, \"%s: saving output to '%s'\\n\", function, fname_out.c_str());\n                return true;\n            }\n        } fout_factory{f < (int) params.fname_out.size() ? params.fname_out[f] : \"\", fname_inp, params};\n\n        std::vector<float> pcmf32;               // mono-channel F32 PCM\n        std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM\n\n        if (!::read_audio_data(fname_inp, pcmf32, pcmf32s, params.diarize)) {\n            fprintf(stderr, \"error: failed to read audio file '%s'\\n\", fname_inp.c_str());\n            continue;\n        }\n\n        if (!whisper_is_multilingual(ctx)) {\n            if (params.language != \"en\" || params.translate) {\n                params.language = \"en\";\n                params.translate = false;\n                fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n            }\n        }\n        if (params.detect_language) {\n            params.language = \"auto\";\n        }\n\n        if (!params.no_prints) {\n            // print system information\n            fprintf(stderr, \"\\n\");\n            fprintf(stderr, \"system_info: n_threads = %d / %d | %s\\n\",\n                    params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());\n\n            // print some info about the processing\n            fprintf(stderr, \"\\n\");\n            fprintf(stderr, \"%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, %d beams + best of %d, lang = %s, task = %s, %stimestamps = %d ...\\n\",\n                    __func__, fname_inp.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,\n                    params.n_threads, params.n_processors, params.beam_size, params.best_of,\n                    params.language.c_str(),\n                    params.translate ? \"translate\" : \"transcribe\",\n                    params.tinydiarize ? \"tdrz = 1, \" : \"\",\n                    params.no_timestamps ? 0 : 1);\n\n            if (params.print_colors) {\n                fprintf(stderr, \"%s: color scheme: red (low confidence), yellow (medium), green (high confidence)\\n\", __func__);\n            } else if (params.print_confidence) {\n                fprintf(stderr, \"%s: confidence: highlighted (low confidence), underlined (medium), dim (high confidence)\\n\", __func__);\n            }\n            fprintf(stderr, \"\\n\");\n        }\n\n        // run the inference\n        {\n            whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n            const bool use_grammar = (!params.grammar_parsed.rules.empty() && !params.grammar_rule.empty());\n            wparams.strategy = (params.beam_size > 1 || use_grammar) ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;\n\n            wparams.print_realtime   = false;\n            wparams.print_progress   = params.print_progress;\n            wparams.print_timestamps = !params.no_timestamps;\n            wparams.print_special    = params.print_special;\n            wparams.translate        = params.translate;\n            wparams.language         = params.language.c_str();\n            wparams.detect_language  = params.detect_language;\n            wparams.n_threads        = params.n_threads;\n            wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;\n            wparams.offset_ms        = params.offset_t_ms;\n            wparams.duration_ms      = params.duration_ms;\n\n            wparams.token_timestamps = params.output_wts || params.output_jsn_full || params.max_len > 0;\n            wparams.thold_pt         = params.word_thold;\n            wparams.max_len          = params.output_wts && params.max_len == 0 ? 60 : params.max_len;\n            wparams.split_on_word    = params.split_on_word;\n            wparams.audio_ctx        = params.audio_ctx;\n\n            wparams.debug_mode       = params.debug_mode;\n\n            wparams.tdrz_enable      = params.tinydiarize; // [TDRZ]\n\n            wparams.suppress_regex   = params.suppress_regex.empty() ? nullptr : params.suppress_regex.c_str();\n\n            wparams.initial_prompt       = params.prompt.c_str();\n            wparams.carry_initial_prompt = params.carry_initial_prompt;\n\n            wparams.greedy.best_of        = params.best_of;\n            wparams.beam_search.beam_size = params.beam_size;\n\n            wparams.temperature_inc  = params.no_fallback ? 0.0f : params.temperature_inc;\n            wparams.temperature      = params.temperature;\n\n            wparams.entropy_thold    = params.entropy_thold;\n            wparams.logprob_thold    = params.logprob_thold;\n            wparams.no_speech_thold  = params.no_speech_thold;\n\n            wparams.no_timestamps    = params.no_timestamps;\n\n            wparams.suppress_nst     = params.suppress_nst;\n\n            wparams.vad            = params.vad;\n            wparams.vad_model_path = params.vad_model.c_str();\n\n            wparams.vad_params.threshold               = params.vad_threshold;\n            wparams.vad_params.min_speech_duration_ms  = params.vad_min_speech_duration_ms;\n            wparams.vad_params.min_silence_duration_ms = params.vad_min_silence_duration_ms;\n            wparams.vad_params.max_speech_duration_s   = params.vad_max_speech_duration_s;\n            wparams.vad_params.speech_pad_ms           = params.vad_speech_pad_ms;\n            wparams.vad_params.samples_overlap         = params.vad_samples_overlap;\n\n            whisper_print_user_data user_data = { &params, &pcmf32s, 0 };\n\n            const auto & grammar_parsed = params.grammar_parsed;\n            auto grammar_rules = grammar_parsed.c_rules();\n\n            if (use_grammar) {\n                if (grammar_parsed.symbol_ids.find(params.grammar_rule) == grammar_parsed.symbol_ids.end()) {\n                    fprintf(stderr, \"%s: warning: grammar rule '%s' not found - skipping grammar sampling\\n\", __func__, params.grammar_rule.c_str());\n                } else {\n                    wparams.grammar_rules = grammar_rules.data();\n                    wparams.n_grammar_rules = grammar_rules.size();\n                    wparams.i_start_rule = grammar_parsed.symbol_ids.at(params.grammar_rule);\n                    wparams.grammar_penalty = params.grammar_penalty;\n                }\n            }\n\n            // this callback is called on each new segment\n            if (!wparams.print_realtime) {\n                wparams.new_segment_callback           = fout_factory.print_segment_callback;\n                wparams.new_segment_callback_user_data = &user_data;\n            }\n\n            if (wparams.print_progress) {\n                wparams.progress_callback           = whisper_print_progress_callback;\n                wparams.progress_callback_user_data = &user_data;\n            }\n\n            // examples for abort mechanism\n            // in examples below, we do not abort the processing, but we could if the flag is set to true\n\n            // the callback is called before every encoder run - if it returns false, the processing is aborted\n            {\n                static bool is_aborted = false; // NOTE: this should be atomic to avoid data race\n\n                wparams.encoder_begin_callback = [](struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, void * user_data) {\n                    bool is_aborted = *(bool*)user_data;\n                    return !is_aborted;\n                };\n                wparams.encoder_begin_callback_user_data = &is_aborted;\n            }\n\n            // the callback is called before every computation - if it returns true, the computation is aborted\n            {\n                static bool is_aborted = false; // NOTE: this should be atomic to avoid data race\n\n                wparams.abort_callback = [](void * user_data) {\n                    bool is_aborted = *(bool*)user_data;\n                    return is_aborted;\n                };\n                wparams.abort_callback_user_data = &is_aborted;\n            }\n\n            if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {\n                fprintf(stderr, \"%s: failed to process audio\\n\", argv[0]);\n                return 10;\n            }\n        }\n\n        // output stuff\n        {\n            // macros to stringify function name\n#define output_func(func, ext, param, ...) if (param && fout_factory.open(ext, #func)) {\\\n    func(ctx, fout_factory.fout, params, __VA_ARGS__); \\\n}\n#define output_ext(ext, ...) output_func(output_##ext, \".\" #ext, params.output_##ext, __VA_ARGS__)\n\n            output_ext(txt, pcmf32s);\n            output_ext(vtt, pcmf32s);\n            output_ext(srt, pcmf32s);\n            output_ext(wts, pcmf32s, fname_inp.c_str(), float(pcmf32.size() + 1000)/WHISPER_SAMPLE_RATE, fout_factory.fname_out.c_str());\n            output_ext(csv, pcmf32s);\n            output_func(output_json, \".json\", params.output_jsn, pcmf32s);\n            output_ext(lrc, pcmf32s);\n            output_func(output_score, \".score.txt\", params.log_score, pcmf32s);\n\n#undef output_ext\n#undef output_func\n\n            if (fout_factory.is_stdout && !fout_factory.used_stdout) {\n                fprintf(stderr, \"warning: '--output-file -' used without any other '--output-*'\");\n            }\n        }\n    }\n\n    if (!params.no_prints) {\n        whisper_print_timings(ctx);\n    }\n    whisper_free(ctx);\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/coi-serviceworker.js",
    "content": "/*! coi-serviceworker v0.1.7 - Guido Zuidhof and contributors, licensed under MIT */\nlet coepCredentialless = false;\nif (typeof window === 'undefined') {\n    self.addEventListener(\"install\", () => self.skipWaiting());\n    self.addEventListener(\"activate\", (event) => event.waitUntil(self.clients.claim()));\n\n    self.addEventListener(\"message\", (ev) => {\n        if (!ev.data) {\n            return;\n        } else if (ev.data.type === \"deregister\") {\n            self.registration\n                .unregister()\n                .then(() => {\n                    return self.clients.matchAll();\n                })\n                .then(clients => {\n                    clients.forEach((client) => client.navigate(client.url));\n                });\n        } else if (ev.data.type === \"coepCredentialless\") {\n            coepCredentialless = ev.data.value;\n        }\n    });\n\n    self.addEventListener(\"fetch\", function (event) {\n        const r = event.request;\n        if (r.cache === \"only-if-cached\" && r.mode !== \"same-origin\") {\n            return;\n        }\n\n        const request = (coepCredentialless && r.mode === \"no-cors\")\n            ? new Request(r, {\n                credentials: \"omit\",\n            })\n            : r;\n        event.respondWith(\n            fetch(request)\n                .then((response) => {\n                    if (response.status === 0) {\n                        return response;\n                    }\n\n                    const newHeaders = new Headers(response.headers);\n                    newHeaders.set(\"Cross-Origin-Embedder-Policy\",\n                        coepCredentialless ? \"credentialless\" : \"require-corp\"\n                    );\n                    if (!coepCredentialless) {\n                        newHeaders.set(\"Cross-Origin-Resource-Policy\", \"cross-origin\");\n                    }\n                    newHeaders.set(\"Cross-Origin-Opener-Policy\", \"same-origin\");\n\n                    return new Response(response.body, {\n                        status: response.status,\n                        statusText: response.statusText,\n                        headers: newHeaders,\n                    });\n                })\n                .catch((e) => console.error(e))\n        );\n    });\n\n} else {\n    (() => {\n        const reloadedBySelf = window.sessionStorage.getItem(\"coiReloadedBySelf\");\n        window.sessionStorage.removeItem(\"coiReloadedBySelf\");\n        const coepDegrading = (reloadedBySelf == \"coepdegrade\");\n\n        // You can customize the behavior of this script through a global `coi` variable.\n        const coi = {\n            shouldRegister: () => !reloadedBySelf,\n            shouldDeregister: () => false,\n            coepCredentialless: () => true,\n            coepDegrade: () => true,\n            doReload: () => window.location.reload(),\n            quiet: false,\n            ...window.coi\n        };\n\n        const n = navigator;\n        const controlling = n.serviceWorker && n.serviceWorker.controller;\n\n        // Record the failure if the page is served by serviceWorker.\n        if (controlling && !window.crossOriginIsolated) {\n            window.sessionStorage.setItem(\"coiCoepHasFailed\", \"true\");\n        }\n        const coepHasFailed = window.sessionStorage.getItem(\"coiCoepHasFailed\");\n\n        if (controlling) {\n            // Reload only on the first failure.\n            const reloadToDegrade = coi.coepDegrade() && !(\n                coepDegrading || window.crossOriginIsolated\n            );\n            n.serviceWorker.controller.postMessage({\n                type: \"coepCredentialless\",\n                value: (reloadToDegrade || coepHasFailed && coi.coepDegrade())\n                    ? false\n                    : coi.coepCredentialless(),\n            });\n            if (reloadToDegrade) {\n                !coi.quiet && console.log(\"Reloading page to degrade COEP.\");\n                window.sessionStorage.setItem(\"coiReloadedBySelf\", \"coepdegrade\");\n                coi.doReload(\"coepdegrade\");\n            }\n\n            if (coi.shouldDeregister()) {\n                n.serviceWorker.controller.postMessage({ type: \"deregister\" });\n            }\n        }\n\n        // If we're already coi: do nothing. Perhaps it's due to this script doing its job, or COOP/COEP are\n        // already set from the origin server. Also if the browser has no notion of crossOriginIsolated, just give up here.\n        if (window.crossOriginIsolated !== false || !coi.shouldRegister()) return;\n\n        if (!window.isSecureContext) {\n            !coi.quiet && console.log(\"COOP/COEP Service Worker not registered, a secure context is required.\");\n            return;\n        }\n\n        // In some environments (e.g. Firefox private mode) this won't be available\n        if (!n.serviceWorker) {\n            !coi.quiet && console.error(\"COOP/COEP Service Worker not registered, perhaps due to private mode.\");\n            return;\n        }\n\n        n.serviceWorker.register(window.document.currentScript.src).then(\n            (registration) => {\n                !coi.quiet && console.log(\"COOP/COEP Service Worker registered\", registration.scope);\n\n                registration.addEventListener(\"updatefound\", () => {\n                    !coi.quiet && console.log(\"Reloading page to make use of updated COOP/COEP Service Worker.\");\n                    window.sessionStorage.setItem(\"coiReloadedBySelf\", \"updatefound\");\n                    coi.doReload();\n                });\n\n                // If the registration is active, but it's not controlling the page\n                if (registration.active && !n.serviceWorker.controller) {\n                    !coi.quiet && console.log(\"Reloading page to make use of COOP/COEP Service Worker.\");\n                    window.sessionStorage.setItem(\"coiReloadedBySelf\", \"notcontrolling\");\n                    coi.doReload();\n                }\n            },\n            (err) => {\n                !coi.quiet && console.error(\"COOP/COEP Service Worker failed to register:\", err);\n            }\n        );\n    })();\n}\n"
  },
  {
    "path": "examples/command/CMakeLists.txt",
    "content": "if (WHISPER_SDL2)\n    set(TARGET whisper-command)\n    add_executable(${TARGET} command.cpp)\n\n    include(DefaultTargetOptions)\n\n    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})\n\n    install(TARGETS ${TARGET} RUNTIME)\nendif ()\n"
  },
  {
    "path": "examples/command/README.md",
    "content": "# whisper.cpp/examples/command\r\n\r\nThis is a basic Voice Assistant example that accepts voice commands from the microphone.\r\nMore info is available in [issue #171](https://github.com/ggerganov/whisper.cpp/issues/171).\r\n\r\n```bash\r\n# Run with default arguments and small model\r\n./whisper-command -m ./models/ggml-small.en.bin -t 8\r\n\r\n# On Raspberry Pi, use tiny or base models + \"-ac 768\" for better performance\r\n./whisper-command -m ./models/ggml-tiny.en.bin -ac 768 -t 3 -c 0\r\n```\r\n\r\nhttps://user-images.githubusercontent.com/1991296/204038393-2f846eae-c255-4099-a76d-5735c25c49da.mp4\r\n\r\nWeb version: [examples/command.wasm](/examples/command.wasm)\r\n\r\n## Guided mode\r\n\r\n\"Guided mode\" allows you to specify a list of commands (i.e. strings) and the transcription will be guided to classify your command into one from the list. This can be useful in situations where a device is listening only for a small subset of commands.\r\n\r\nInitial tests show that this approach might be extremely efficient in terms of performance, since it integrates very well with the \"partial Encoder\" idea from #137.\r\n\r\n```bash\r\n# Run in guided mode, the list of allowed commands is in commands.txt\r\n./whisper-command -m ./models/ggml-base.en.bin -cmd ./examples/command/commands.txt\r\n\r\n# On Raspberry Pi, in guided mode you can use \"-ac 128\" for extra performance\r\n./whisper-command -m ./models/ggml-tiny.en.bin -cmd ./examples/command/commands.txt -ac 128 -t 3 -c 0\r\n```\r\n\r\nhttps://user-images.githubusercontent.com/1991296/207435352-8fc4ed3f-bde5-4555-9b8b-aeeb76bee969.mp4\r\n\r\n\r\n## Building\r\n\r\nThe `whisper-command` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:\r\n\r\n```bash\r\n# Install SDL2\r\n# On Debian based linux distributions:\r\nsudo apt-get install libsdl2-dev\r\n\r\n# On Fedora Linux:\r\nsudo dnf install SDL2 SDL2-devel\r\n\r\n# Install SDL2 on Mac OS\r\nbrew install sdl2\r\n\r\ncmake -B build -DWHISPER_SDL2=ON\r\ncmake --build build --config Release\r\n```\r\n"
  },
  {
    "path": "examples/command/command.cpp",
    "content": "// Voice assistant example\n//\n// Speak short text commands to the microphone.\n// This program will detect your voice command and convert them to text.\n//\n// ref: https://github.com/ggml-org/whisper.cpp/issues/171\n//\n\n#include \"common-sdl.h\"\n#include \"common.h\"\n#include \"whisper.h\"\n#include \"grammar-parser.h\"\n\n#include <algorithm>\n#include <chrono>\n#include <cstdio>\n#include <fstream>\n#include <map>\n#include <sstream>\n#include <string>\n#include <thread>\n#include <vector>\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t prompt_ms  = 5000;\n    int32_t command_ms = 8000;\n    int32_t capture_id = -1;\n    int32_t max_tokens = 32;\n    int32_t audio_ctx  = 0;\n\n    float vad_thold  = 0.6f;\n    float freq_thold = 100.0f;\n\n    float grammar_penalty = 100.0f;\n\n    grammar_parser::parse_state grammar_parsed;\n\n    bool translate     = false;\n    bool print_special = false;\n    bool print_energy  = false;\n    bool no_timestamps = true;\n    bool use_gpu       = true;\n    bool flash_attn    = true;\n\n    std::string language  = \"en\";\n    std::string model     = \"models/ggml-base.en.bin\";\n    std::string fname_out;\n    std::string commands;\n    std::string prompt;\n    std::string context;\n    std::string grammar;\n\n    // A regular expression that matches tokens to suppress\n    std::string suppress_regex;\n};\n\nvoid whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"     || arg == \"--threads\")       { params.n_threads     = std::stoi(argv[++i]); }\n        else if (arg == \"-pms\"   || arg == \"--prompt-ms\")     { params.prompt_ms     = std::stoi(argv[++i]); }\n        else if (arg == \"-cms\"   || arg == \"--command-ms\")    { params.command_ms    = std::stoi(argv[++i]); }\n        else if (arg == \"-c\"     || arg == \"--capture\")       { params.capture_id    = std::stoi(argv[++i]); }\n        else if (arg == \"-mt\"    || arg == \"--max-tokens\")    { params.max_tokens    = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"    || arg == \"--audio-ctx\")     { params.audio_ctx     = std::stoi(argv[++i]); }\n        else if (arg == \"-vth\"   || arg == \"--vad-thold\")     { params.vad_thold     = std::stof(argv[++i]); }\n        else if (arg == \"-fth\"   || arg == \"--freq-thold\")    { params.freq_thold    = std::stof(argv[++i]); }\n        else if (arg == \"-tr\"    || arg == \"--translate\")     { params.translate     = true; }\n        else if (arg == \"-ps\"    || arg == \"--print-special\") { params.print_special = true; }\n        else if (arg == \"-pe\"    || arg == \"--print-energy\")  { params.print_energy  = true; }\n        else if (arg == \"-ng\"    || arg == \"--no-gpu\")        { params.use_gpu       = false; }\n        else if (arg == \"-fa\"    || arg == \"--flash-attn\")    { params.flash_attn    = true; }\n        else if (arg == \"-nfa\"   || arg == \"--no-flash-attn\") { params.flash_attn    = false; }\n        else if (arg == \"-l\"     || arg == \"--language\")      { params.language      = argv[++i]; }\n        else if (arg == \"-m\"     || arg == \"--model\")         { params.model         = argv[++i]; }\n        else if (arg == \"-f\"     || arg == \"--file\")          { params.fname_out     = argv[++i]; }\n        else if (arg == \"-cmd\"   || arg == \"--commands\")      { params.commands      = argv[++i]; }\n        else if (arg == \"-p\"     || arg == \"--prompt\")        { params.prompt        = argv[++i]; }\n        else if (arg == \"-ctx\"   || arg == \"--context\")       { params.context       = argv[++i]; }\n        else if (                   arg == \"--grammar\")       { params.grammar       = argv[++i]; }\n        else if (                   arg == \"--grammar-penalty\") { params.grammar_penalty = std::stof(argv[++i]); }\n        else if (                   arg == \"--suppress-regex\") { params.suppress_regex = argv[++i]; }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,         --help           [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,       --threads N      [%-7d] number of threads to use during computation\\n\", params.n_threads);\n    fprintf(stderr, \"  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\\n\",             params.prompt_ms);\n    fprintf(stderr, \"  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\\n\",            params.command_ms);\n    fprintf(stderr, \"  -c ID,      --capture ID     [%-7d] capture device ID\\n\",                           params.capture_id);\n    fprintf(stderr, \"  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\\n\",    params.max_tokens);\n    fprintf(stderr, \"  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\\n\",                params.audio_ctx);\n    fprintf(stderr, \"  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\\n\",        params.vad_thold);\n    fprintf(stderr, \"  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\\n\",                params.freq_thold);\n    fprintf(stderr, \"  -tr,        --translate      [%-7s] translate from source language to english\\n\",   params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,        --print-special  [%-7s] print special tokens\\n\",                        params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\\n\",          params.print_energy ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ng,        --no-gpu         [%-7s] disable GPU\\n\",                                 params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,        --flash-attn     [%-7s] enbale flash attention\\n\",                      params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,       --no-flash-attn  [%-7s] disable flash attention\\n\",                     params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -l LANG,    --language LANG  [%-7s] spoken language\\n\",                             params.language.c_str());\n    fprintf(stderr, \"  -m FNAME,   --model FNAME    [%-7s] model path\\n\",                                  params.model.c_str());\n    fprintf(stderr, \"  -f FNAME,   --file FNAME     [%-7s] text output file name\\n\",                       params.fname_out.c_str());\n    fprintf(stderr, \"  -cmd FNAME, --commands FNAME [%-7s] text file with allowed commands\\n\",             params.commands.c_str());\n    fprintf(stderr, \"  -p,         --prompt         [%-7s] the required activation prompt\\n\",              params.prompt.c_str());\n    fprintf(stderr, \"  -ctx,       --context        [%-7s] sample text to help the transcription\\n\",       params.context.c_str());\n    fprintf(stderr, \"  --grammar GRAMMAR            [%-7s] GBNF grammar to guide decoding\\n\",              params.grammar.c_str());\n    fprintf(stderr, \"  --grammar-penalty N          [%-7.1f] scales down logits of nongrammar tokens\\n\",   params.grammar_penalty);\n    fprintf(stderr, \"  --suppress-regex REGEX       [%-7s] regular expression matching tokens to suppress\\n\", params.suppress_regex.c_str());\n    fprintf(stderr, \"\\n\");\n}\n\nstatic std::string transcribe(\n                 whisper_context * ctx,\n            const whisper_params & params,\n        const std::vector<float> & pcmf32,\n               const std::string & grammar_rule,\n                           float & logprob_min,\n                           float & logprob_sum,\n                             int & n_tokens,\n                         int64_t & t_ms) {\n    const auto t_start = std::chrono::high_resolution_clock::now();\n\n    logprob_min = 0.0f;\n    logprob_sum = 0.0f;\n    n_tokens    = 0;\n    t_ms = 0;\n\n    //whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_BEAM_SEARCH);\n\n    wparams.print_progress   = false;\n    wparams.print_special    = params.print_special;\n    wparams.print_realtime   = false;\n    wparams.print_timestamps = !params.no_timestamps;\n    wparams.translate        = params.translate;\n    wparams.no_context       = true;\n    wparams.no_timestamps    = params.no_timestamps;\n    wparams.single_segment   = true;\n    wparams.max_tokens       = params.max_tokens;\n    wparams.language         = params.language.c_str();\n    wparams.n_threads        = params.n_threads;\n\n    wparams.audio_ctx = params.audio_ctx;\n\n    wparams.temperature     = 0.4f;\n    wparams.temperature_inc = 1.0f;\n    wparams.greedy.best_of  = 5;\n\n    wparams.beam_search.beam_size = 5;\n\n    wparams.initial_prompt = params.context.data();\n\n    wparams.suppress_regex = params.suppress_regex.c_str();\n\n    const auto & grammar_parsed = params.grammar_parsed;\n    auto grammar_rules = grammar_parsed.c_rules();\n\n    if (!params.grammar_parsed.rules.empty() && !grammar_rule.empty()) {\n        if (grammar_parsed.symbol_ids.find(grammar_rule) == grammar_parsed.symbol_ids.end()) {\n            fprintf(stderr, \"%s: warning: grammar rule '%s' not found - skipping grammar sampling\\n\", __func__, grammar_rule.c_str());\n        } else {\n            wparams.grammar_rules   = grammar_rules.data();\n            wparams.n_grammar_rules = grammar_rules.size();\n            wparams.i_start_rule    = grammar_parsed.symbol_ids.at(grammar_rule);\n            wparams.grammar_penalty = params.grammar_penalty;\n        }\n    }\n\n    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        return \"\";\n    }\n\n    std::string result;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n\n        result += text;\n\n        const int n = whisper_full_n_tokens(ctx, i);\n        for (int j = 0; j < n; ++j) {\n            const auto token = whisper_full_get_token_data(ctx, i, j);\n\n            if(token.plog > 0.0f) exit(0);\n            logprob_min = std::min(logprob_min, token.plog);\n            logprob_sum += token.plog;\n            ++n_tokens;\n        }\n    }\n\n    const auto t_end = std::chrono::high_resolution_clock::now();\n    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();\n\n    return result;\n}\n\nstatic std::vector<std::string> read_allowed_commands(const std::string & fname) {\n    std::vector<std::string> allowed_commands;\n\n    std::ifstream ifs(fname);\n    if (!ifs.is_open()) {\n        return allowed_commands;\n    }\n\n    std::string line;\n    while (std::getline(ifs, line)) {\n        line = ::trim(line);\n        if (line.empty()) {\n            continue;\n        }\n\n        std::transform(line.begin(), line.end(),line.begin(), ::tolower);\n        allowed_commands.push_back(std::move(line));\n    }\n\n    return allowed_commands;\n}\n\nstatic std::vector<std::string> get_words(const std::string &txt) {\n    std::vector<std::string> words;\n\n    std::istringstream iss(txt);\n    std::string word;\n    while (iss >> word) {\n        words.push_back(word);\n    }\n\n    return words;\n}\n\n// command-list mode\n// guide the transcription to match the most likely command from a provided list\nstatic int process_command_list(struct whisper_context * ctx, audio_async &audio, const whisper_params &params, std::ofstream &fout) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"%s: guided mode\\n\", __func__);\n\n    std::vector<std::string> allowed_commands = read_allowed_commands(params.commands);\n\n    if (allowed_commands.empty()) {\n        fprintf(stderr, \"%s: error: failed to read allowed commands from '%s'\\n\", __func__, params.commands.c_str());\n        return 2;\n    }\n\n    int max_len = 0;\n\n    std::vector<std::vector<whisper_token>> allowed_tokens;\n\n    for (const auto & cmd : allowed_commands) {\n        whisper_token tokens[1024];\n        allowed_tokens.emplace_back();\n\n        for (int l = 0; l < (int) cmd.size(); ++l) {\n            // NOTE: very important to add the whitespace !\n            //       the reason is that the first decoded token starts with a whitespace too!\n            std::string ss = std::string(\" \") + cmd.substr(0, l + 1);\n\n            const int n = whisper_tokenize(ctx, ss.c_str(), tokens, 1024);\n            if (n < 0) {\n                fprintf(stderr, \"%s: error: failed to tokenize command '%s'\\n\", __func__, cmd.c_str());\n                return 3;\n            }\n\n            if (n == 1) {\n                allowed_tokens.back().push_back(tokens[0]);\n            }\n        }\n\n        max_len = std::max(max_len, (int) cmd.size());\n    }\n\n    fprintf(stderr, \"%s: allowed commands [ tokens ]:\\n\", __func__);\n    fprintf(stderr, \"\\n\");\n    for (int i = 0; i < (int) allowed_commands.size(); ++i) {\n        fprintf(stderr, \"  - \\033[1m%-*s\\033[0m = [\", max_len, allowed_commands[i].c_str());\n        for (const auto & token : allowed_tokens[i]) {\n            fprintf(stderr, \" %5d\", token);\n        }\n        fprintf(stderr, \" ]\\n\");\n    }\n\n    std::string k_prompt = \"select one from the available words: \";\n    for (int i = 0; i < (int) allowed_commands.size(); ++i) {\n        if (i > 0) {\n            k_prompt += \", \";\n        }\n        k_prompt += allowed_commands[i];\n    }\n    k_prompt += \". selected word: \";\n\n    // tokenize prompt\n    std::vector<whisper_token> k_tokens;\n    {\n        k_tokens.resize(1024);\n        const int n = whisper_tokenize(ctx, k_prompt.c_str(), k_tokens.data(), 1024);\n        if (n < 0) {\n            fprintf(stderr, \"%s: error: failed to tokenize prompt '%s'\\n\", __func__, k_prompt.c_str());\n            return 4;\n        }\n        k_tokens.resize(n);\n    }\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"%s: prompt: '%s'\\n\", __func__, k_prompt.c_str());\n    fprintf(stderr, \"%s: tokens: [\", __func__);\n    for (const auto & token : k_tokens) {\n        fprintf(stderr, \" %d\", token);\n    }\n    fprintf(stderr, \" ]\\n\");\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"%s: listening for a command ...\\n\", __func__);\n    fprintf(stderr, \"\\n\");\n\n    bool is_running  = true;\n\n    std::vector<float> pcmf32_cur;\n    std::vector<float> pcmf32_prompt;\n\n    // main loop\n    while (is_running) {\n        // handle Ctrl + C\n        is_running = sdl_poll_events();\n\n        // delay\n        std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n        audio.get(2000, pcmf32_cur);\n\n        if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {\n            fprintf(stdout, \"%s: Speech detected! Processing ...\\n\", __func__);\n\n            const auto t_start = std::chrono::high_resolution_clock::now();\n\n            whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n            wparams.print_progress   = false;\n            wparams.print_special    = params.print_special;\n            wparams.print_realtime   = false;\n            wparams.print_timestamps = !params.no_timestamps;\n            wparams.translate        = params.translate;\n            wparams.no_context       = true;\n            wparams.single_segment   = true;\n            wparams.max_tokens       = 1;\n            wparams.language         = params.language.c_str();\n            wparams.n_threads        = params.n_threads;\n\n            wparams.audio_ctx        = params.audio_ctx;\n\n            wparams.prompt_tokens    = k_tokens.data();\n            wparams.prompt_n_tokens  = k_tokens.size();\n\n            // run the transformer and a single decoding pass\n            if (whisper_full(ctx, wparams, pcmf32_cur.data(), pcmf32_cur.size()) != 0) {\n                fprintf(stderr, \"%s: ERROR: whisper_full() failed\\n\", __func__);\n                break;\n            }\n\n            // estimate command probability\n            // NOTE: not optimal\n            {\n                const auto * logits = whisper_get_logits(ctx);\n\n                std::vector<float> probs(whisper_n_vocab(ctx), 0.0f);\n\n                // compute probs from logits via softmax\n                {\n                    float max = -1e9;\n                    for (int i = 0; i < (int) probs.size(); ++i) {\n                        max = std::max(max, logits[i]);\n                    }\n\n                    float sum = 0.0f;\n                    for (int i = 0; i < (int) probs.size(); ++i) {\n                        probs[i] = expf(logits[i] - max);\n                        sum += probs[i];\n                    }\n\n                    for (int i = 0; i < (int) probs.size(); ++i) {\n                        probs[i] /= sum;\n                    }\n                }\n\n                std::vector<std::pair<float, int>> probs_id;\n\n                double psum = 0.0;\n                for (int i = 0; i < (int) allowed_commands.size(); ++i) {\n                    probs_id.emplace_back(probs[allowed_tokens[i][0]], i);\n                    for (int j = 1; j < (int) allowed_tokens[i].size(); ++j) {\n                        probs_id.back().first += probs[allowed_tokens[i][j]];\n                    }\n                    probs_id.back().first /= allowed_tokens[i].size();\n                    psum += probs_id.back().first;\n                }\n\n                // normalize\n                for (auto & p : probs_id) {\n                    p.first /= psum;\n                }\n\n                // sort descending\n                {\n                    using pair_type = decltype(probs_id)::value_type;\n                    std::sort(probs_id.begin(), probs_id.end(), [](const pair_type & a, const pair_type & b) {\n                        return a.first > b.first;\n                    });\n                }\n\n                // print the commands and the respective probabilities\n                {\n                    fprintf(stdout, \"\\n\");\n                    for (const auto & cmd : probs_id) {\n                        fprintf(stdout, \"%s: %s%-*s%s = %f | \", __func__, \"\\033[1m\", max_len, allowed_commands[cmd.second].c_str(), \"\\033[0m\", cmd.first);\n                        for (int token : allowed_tokens[cmd.second]) {\n                            fprintf(stdout, \"'%4s' %f \", whisper_token_to_str(ctx, token), probs[token]);\n                        }\n                        fprintf(stdout, \"\\n\");\n                    }\n                }\n\n                // best command\n                {\n                    const auto t_end = std::chrono::high_resolution_clock::now();\n\n                    const float prob = probs_id[0].first;\n                    const int index = probs_id[0].second;\n                    const char * best_command = allowed_commands[index].c_str();\n\n                    fprintf(stdout, \"\\n\");\n                    fprintf(stdout, \"%s: detected command: %s%s%s | p = %f | t = %d ms\\n\", __func__,\n                            \"\\033[1m\", best_command, \"\\033[0m\", prob,\n                            (int) std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count());\n                    fprintf(stdout, \"\\n\");\n                    if (fout.is_open()) {\n                        fout << best_command << std::endl;\n                    }\n                }\n            }\n\n            audio.clear();\n        }\n    }\n\n    return 0;\n}\n\n// always-prompt mode\n// transcribe the voice into text after valid prompt\nstatic int always_prompt_transcription(struct whisper_context * ctx, audio_async & audio, const whisper_params & params, std::ofstream & fout) {\n    bool is_running = true;\n    bool ask_prompt = true;\n\n    float logprob_min = 0.0f;\n    float logprob_sum = 0.0f;\n    int   n_tokens    = 0;\n\n    std::vector<float> pcmf32_cur;\n\n    const std::string k_prompt = params.prompt;\n\n    const int k_prompt_length = get_words(k_prompt).size();\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"%s: always-prompt mode\\n\", __func__);\n\n    // main loop\n    while (is_running) {\n        // handle Ctrl + C\n        is_running = sdl_poll_events();\n\n        // delay\n        std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n        if (ask_prompt) {\n            fprintf(stdout, \"\\n\");\n            fprintf(stdout, \"%s: The prompt is: '%s%s%s'\\n\", __func__, \"\\033[1m\", k_prompt.c_str(), \"\\033[0m\");\n            fprintf(stdout, \"\\n\");\n\n            ask_prompt = false;\n        }\n\n        {\n            audio.get(2000, pcmf32_cur);\n\n            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {\n                fprintf(stdout, \"%s: Speech detected! Processing ...\\n\", __func__);\n\n                int64_t t_ms = 0;\n\n                // detect the commands\n                audio.get(params.command_ms, pcmf32_cur);\n\n                const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, \"\", logprob_min, logprob_sum, n_tokens, t_ms));\n\n                const auto words = get_words(txt);\n\n                std::string prompt;\n                std::string command;\n\n                for (int i = 0; i < (int) words.size(); ++i) {\n                    if (i < k_prompt_length) {\n                        prompt += words[i] + \" \";\n                    } else {\n                        command += words[i] + \" \";\n                    }\n                }\n\n                const float sim = similarity(prompt, k_prompt);\n\n                //debug\n                //fprintf(stdout, \"command size: %i\\n\", command_length);\n\n                if ((sim > 0.7f) && (command.size() > 0)) {\n                    fprintf(stdout, \"%s: Command '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", command.c_str(), \"\\033[0m\", (int) t_ms);\n                    if (fout.is_open()) {\n                        fout << command << std::endl;\n                    }\n                }\n\n                fprintf(stdout, \"\\n\");\n\n                audio.clear();\n            }\n        }\n    }\n\n    return 0;\n}\n\n// general-purpose mode\n// freely transcribe the voice into text\nstatic int process_general_transcription(struct whisper_context * ctx, audio_async & audio, const whisper_params & params, std::ofstream & fout) {\n    bool is_running  = true;\n    bool have_prompt = false;\n    bool ask_prompt  = true;\n\n    float logprob_min0 = 0.0f;\n    float logprob_min  = 0.0f;\n\n    float logprob_sum0 = 0.0f;\n    float logprob_sum  = 0.0f;\n\n    int n_tokens0 = 0;\n    int n_tokens  = 0;\n\n    std::vector<float> pcmf32_cur;\n    std::vector<float> pcmf32_prompt;\n\n    std::string k_prompt = \"Ok Whisper, start listening for commands.\";\n    if (!params.prompt.empty()) {\n        k_prompt = params.prompt;\n    }\n\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"%s: general-purpose mode\\n\", __func__);\n\n    // main loop\n    while (is_running) {\n        // handle Ctrl + C\n        is_running = sdl_poll_events();\n\n        // delay\n        std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n        if (ask_prompt) {\n            fprintf(stdout, \"\\n\");\n            fprintf(stdout, \"%s: Say the following phrase: '%s%s%s'\\n\", __func__, \"\\033[1m\", k_prompt.c_str(), \"\\033[0m\");\n            fprintf(stdout, \"\\n\");\n\n            ask_prompt = false;\n        }\n\n        {\n            audio.get(2000, pcmf32_cur);\n\n            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {\n                fprintf(stdout, \"%s: Speech detected! Processing ...\\n\", __func__);\n\n                int64_t t_ms = 0;\n\n                if (!have_prompt) {\n                    // wait for activation phrase\n                    audio.get(params.prompt_ms, pcmf32_cur);\n\n                    const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, \"prompt\", logprob_min0, logprob_sum0, n_tokens0, t_ms));\n\n                    const float p = 100.0f * std::exp(logprob_min0);\n\n                    fprintf(stdout, \"%s: Heard '%s%s%s', (t = %d ms, p = %.2f%%)\\n\", __func__, \"\\033[1m\", txt.c_str(), \"\\033[0m\", (int) t_ms, p);\n\n                    const float sim = similarity(txt, k_prompt);\n\n                    if (txt.length() < 0.8*k_prompt.length() || txt.length() > 1.2*k_prompt.length() || sim < 0.8f) {\n                        fprintf(stdout, \"%s: WARNING: prompt not recognized, try again\\n\", __func__);\n                        ask_prompt = true;\n                    } else {\n                        fprintf(stdout, \"\\n\");\n                        fprintf(stdout, \"%s: The prompt has been recognized!\\n\", __func__);\n                        fprintf(stdout, \"%s: Waiting for voice commands ...\\n\", __func__);\n                        fprintf(stdout, \"\\n\");\n\n                        // save the audio for the prompt\n                        pcmf32_prompt = pcmf32_cur;\n                        have_prompt = true;\n                    }\n                } else {\n                    // we have heard the activation phrase, now detect the commands\n                    audio.get(params.command_ms, pcmf32_cur);\n\n                    //printf(\"len prompt:  %.4f\\n\", pcmf32_prompt.size() / (float) WHISPER_SAMPLE_RATE);\n                    //printf(\"len command: %.4f\\n\", pcmf32_cur.size() / (float) WHISPER_SAMPLE_RATE);\n\n                    // prepend 3 second of silence\n                    pcmf32_cur.insert(pcmf32_cur.begin(), 3.0f*WHISPER_SAMPLE_RATE, 0.0f);\n\n                    // prepend the prompt audio\n                    pcmf32_cur.insert(pcmf32_cur.begin(), pcmf32_prompt.begin(), pcmf32_prompt.end());\n\n                    const auto txt = ::trim(::transcribe(ctx, params, pcmf32_cur, \"root\", logprob_min, logprob_sum, n_tokens, t_ms));\n\n                    //const float p = 100.0f * std::exp((logprob - logprob0) / (n_tokens - n_tokens0));\n                    const float p = 100.0f * std::exp(logprob_min);\n\n                    //fprintf(stdout, \"%s: heard '%s'\\n\", __func__, txt.c_str());\n\n                    // find the prompt in the text\n                    float best_sim = 0.0f;\n                    size_t best_len = 0;\n                    for (size_t n = 0.8*k_prompt.size(); n <= 1.2*k_prompt.size(); ++n) {\n                        if (n >= txt.size()) {\n                            break;\n                        }\n\n                        const auto prompt = txt.substr(0, n);\n\n                        const float sim = similarity(prompt, k_prompt);\n\n                        //fprintf(stderr, \"%s: prompt = '%s', sim = %f\\n\", __func__, prompt.c_str(), sim);\n\n                        if (sim > best_sim) {\n                            best_sim = sim;\n                            best_len = n;\n                        }\n                    }\n\n                    fprintf(stdout, \"%s:   DEBUG: txt = '%s', prob = %.2f%%\\n\", __func__, txt.c_str(), p);\n                    if (best_len == 0) {\n                        fprintf(stdout, \"%s: WARNING: command not recognized, try again\\n\", __func__);\n                    } else {\n                        // cut the prompt from the decoded text\n                        const std::string command = ::trim(txt.substr(best_len));\n                        fprintf(stdout, \"%s: Command '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", command.c_str(), \"\\033[0m\", (int) t_ms);\n                        if (fout.is_open()) {\n                            fout << command << std::endl;\n                        }\n                    }\n\n                    fprintf(stdout, \"\\n\");\n                }\n\n                audio.clear();\n            }\n        }\n    }\n\n    return 0;\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    if (whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    // whisper init\n\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n    if (ctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 2;\n    }\n\n    // print some info about the processing\n    {\n        fprintf(stderr, \"\\n\");\n        if (!whisper_is_multilingual(ctx)) {\n            if (params.language != \"en\" || params.translate) {\n                params.language = \"en\";\n                params.translate = false;\n                fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n            }\n        }\n        fprintf(stderr, \"%s: processing, %d threads, lang = %s, task = %s, timestamps = %d ...\\n\",\n                __func__,\n                params.n_threads,\n                params.language.c_str(),\n                params.translate ? \"translate\" : \"transcribe\",\n                params.no_timestamps ? 0 : 1);\n\n        fprintf(stderr, \"\\n\");\n    }\n\n    // init audio\n\n    audio_async audio(30*1000);\n    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {\n        fprintf(stderr, \"%s: audio.init() failed!\\n\", __func__);\n        return 1;\n    }\n\n    audio.resume();\n\n    // wait for 1 second to avoid any buffered noise\n    std::this_thread::sleep_for(std::chrono::milliseconds(1000));\n    audio.clear();\n\n    int  ret_val = 0;\n\n    if (!params.grammar.empty()) {\n        auto & grammar = params.grammar_parsed;\n        if (is_file_exist(params.grammar.c_str())) {\n            // read grammar from file\n            std::ifstream ifs(params.grammar.c_str());\n            const std::string txt = std::string((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());\n            grammar = grammar_parser::parse(txt.c_str());\n        } else {\n            // read grammar from string\n            grammar = grammar_parser::parse(params.grammar.c_str());\n        }\n\n        // will be empty (default) if there are parse errors\n        if (grammar.rules.empty()) {\n            ret_val = 1;\n        } else {\n            fprintf(stderr, \"%s: grammar:\\n\", __func__);\n            grammar_parser::print_grammar(stderr, grammar);\n            fprintf(stderr, \"\\n\");\n        }\n    }\n\n    std::ofstream fout;\n    if (params.fname_out.length() > 0) {\n        fout.open(params.fname_out);\n        if (!fout.is_open()) {\n            fprintf(stderr, \"%s: failed to open output file '%s'!\\n\", __func__, params.fname_out.c_str());\n            return 1;\n        }\n    }\n\n    if (ret_val == 0) {\n        if (!params.commands.empty()) {\n            ret_val = process_command_list(ctx, audio, params, fout);\n        } else if (!params.prompt.empty() && params.grammar_parsed.rules.empty()) {\n            ret_val = always_prompt_transcription(ctx, audio, params, fout);\n        } else {\n            ret_val = process_general_transcription(ctx, audio, params, fout);\n        }\n    }\n\n    audio.pause();\n\n    whisper_print_timings(ctx);\n    whisper_free(ctx);\n\n    return ret_val;\n}\n"
  },
  {
    "path": "examples/command/commands.txt",
    "content": "enable\ndisable\ncat\ndog\napple\nred\nblue\ngreen\nlightblue\n"
  },
  {
    "path": "examples/command.wasm/CMakeLists.txt",
    "content": "#\n# libcommand\n#\n\nset(TARGET libcommand)\n\nadd_executable(${TARGET}\n    emscripten.cpp\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE\n    common\n    whisper\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside command.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libcommand.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/command.wasm/command.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE=8 \\\n    -s INITIAL_MEMORY=1024MB \\\n    -s TOTAL_MEMORY=1024MB \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s EXPORTED_RUNTIME_METHODS=\\\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\\\" \\\n    ${EXTRA_FLAGS} \\\n    \")\n\n#\n# command.wasm\n#\n\nset(TARGET command.wasm)\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)\n"
  },
  {
    "path": "examples/command.wasm/README.md",
    "content": "# command.wasm\n\nThis is a basic Voice Assistant example that accepts voice commands from the microphone.\nIt runs in fully in the browser via WebAseembly.\n\nOnline demo: https://ggml.ai/whisper.cpp/command.wasm/\n\nTerminal version: [examples/command](/examples/command)\n\n## Build instructions\n\n```bash\n# build using Emscripten (v3.1.2)\ngit clone https://github.com/ggerganov/whisper.cpp\ncd whisper.cpp\nmkdir build-em && cd build-em\nemcmake cmake ..\nmake -j libcommand\n```\nThe example can then be started by running a local HTTP server:\n```console\npython3 examples/server.py\n```\nAnd then opening a browser to the following URL:\nhttp://localhost:8000/command.wasm/\n\nTo run the example in a different server, you need to copy the following files\nto the server's HTTP path:\n```\ncp bin/command.wasm/*       /path/to/html/\ncp bin/libcommand.js        /path/to/html/\ncp bin/libcommand.worker.js /path/to/html/\n```\n\n> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`\n> which means that that a separate .wasm file will not be generated. Instead, the\n> WASM module is embedded in the main JS file as a base64 encoded string. To\n> generate a separate .wasm file, you need to disable this option by passing\n> `-DWHISPER_WASM_SINGLE_FILE=OFF`:\n> ```console\n> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF\n> ```\n> This will generate a `libcommand.wasm` file in the build/bin directory.\n\n> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no\n> longer generated and the worker is embedded in the main JS file. So the worker\n> file will not be geneated for versions later than `3.1.58`.\n"
  },
  {
    "path": "examples/command.wasm/emscripten.cpp",
    "content": "#include \"ggml.h\"\n#include \"common.h\"\n#include \"whisper.h\"\n\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <atomic>\n#include <cmath>\n#include <mutex>\n#include <string>\n#include <thread>\n#include <vector>\n#include <regex>\n\nconstexpr int N_THREAD = 8;\n\nstd::vector<struct whisper_context *> g_contexts(4, nullptr);\n\nstd::mutex  g_mutex;\nstd::thread g_worker;\n\nstd::atomic<bool> g_running(false);\n\nstd::string g_status        = \"\";\nstd::string g_status_forced = \"\";\nstd::string g_transcribed   = \"\";\n\nstd::vector<float> g_pcmf32;\n\nvoid command_set_status(const std::string & status) {\n    std::lock_guard<std::mutex> lock(g_mutex);\n    g_status = status;\n}\n\nstd::string command_transcribe(whisper_context * ctx, const whisper_full_params & wparams, const std::vector<float> & pcmf32, float & prob, int64_t & t_ms) {\n    const auto t_start = std::chrono::high_resolution_clock::now();\n\n    prob = 0.0f;\n    t_ms = 0;\n\n    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        return \"\";\n    }\n\n    int prob_n = 0;\n    std::string result;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n\n        result += text;\n\n        const int n_tokens = whisper_full_n_tokens(ctx, i);\n        for (int j = 0; j < n_tokens; ++j) {\n            const auto token = whisper_full_get_token_data(ctx, i, j);\n\n            prob += token.p;\n            ++prob_n;\n        }\n    }\n\n    if (prob_n > 0) {\n        prob /= prob_n;\n    }\n\n    const auto t_end = std::chrono::high_resolution_clock::now();\n    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();\n\n    return result;\n}\n\nvoid command_get_audio(int ms, int sample_rate, std::vector<float> & audio) {\n    const int64_t n_samples = (ms * sample_rate) / 1000;\n\n    int64_t n_take = 0;\n    if (n_samples > (int) g_pcmf32.size()) {\n        n_take = g_pcmf32.size();\n    } else {\n        n_take = n_samples;\n    }\n\n    audio.resize(n_take);\n    std::copy(g_pcmf32.end() - n_take, g_pcmf32.end(), audio.begin());\n}\n\nvoid command_main(size_t index) {\n    command_set_status(\"loading data ...\");\n\n    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n\n    wparams.n_threads        = std::min(N_THREAD, (int) std::thread::hardware_concurrency());\n    wparams.offset_ms        = 0;\n    wparams.translate        = false;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.print_realtime   = false;\n    wparams.print_progress   = false;\n    wparams.print_timestamps = true;\n    wparams.print_special    = false;\n\n    wparams.max_tokens       = 32;\n    wparams.audio_ctx        = 768; // partial encoder context for better performance\n\n    wparams.language         = \"en\";\n\n    printf(\"command: using %d threads\\n\", wparams.n_threads);\n\n    bool have_prompt  = false;\n    bool ask_prompt   = true;\n    bool print_energy = false;\n\n    float prob0 = 0.0f;\n    float prob  = 0.0f;\n\n    std::vector<float> pcmf32_cur;\n    std::vector<float> pcmf32_prompt;\n\n    const std::string k_prompt = \"Ok Whisper, start listening for commands.\";\n\n    // whisper context\n    auto & ctx = g_contexts[index];\n\n    const int32_t vad_ms     = 2000;\n    const int32_t prompt_ms  = 5000;\n    const int32_t command_ms = 4000;\n\n    const float vad_thold  = 0.1f;\n    const float freq_thold = -1.0f;\n\n    while (g_running) {\n        // delay\n        std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n        if (ask_prompt) {\n            fprintf(stdout, \"\\n\");\n            fprintf(stdout, \"%s: Say the following phrase: '%s%s%s'\\n\", __func__, \"\\033[1m\", k_prompt.c_str(), \"\\033[0m\");\n            fprintf(stdout, \"\\n\");\n\n            {\n                char txt[1024];\n                snprintf(txt, sizeof(txt), \"Say the following phrase: '%s'\", k_prompt.c_str());\n                command_set_status(txt);\n            }\n\n            ask_prompt = false;\n        }\n\n        int64_t t_ms = 0;\n\n        {\n            command_get_audio(vad_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);\n\n            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1000, vad_thold, freq_thold, print_energy)) {\n                fprintf(stdout, \"%s: Speech detected! Processing ...\\n\", __func__);\n                command_set_status(\"Speech detected! Processing ...\");\n\n                if (!have_prompt) {\n                    command_get_audio(prompt_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);\n\n                    const auto txt = ::trim(::command_transcribe(ctx, wparams, pcmf32_cur, prob0, t_ms));\n\n                    fprintf(stdout, \"%s: Heard '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", txt.c_str(), \"\\033[0m\", (int) t_ms);\n\n                    const float sim = similarity(txt, k_prompt);\n\n                    if (txt.length() < 0.8*k_prompt.length() || txt.length() > 1.2*k_prompt.length() || sim < 0.8f) {\n                        fprintf(stdout, \"%s: WARNING: prompt not recognized, try again\\n\", __func__);\n                        ask_prompt = true;\n                    } else {\n                        fprintf(stdout, \"\\n\");\n                        fprintf(stdout, \"%s: The prompt has been recognized!\\n\", __func__);\n                        fprintf(stdout, \"%s: Waiting for voice commands ...\\n\", __func__);\n                        fprintf(stdout, \"\\n\");\n\n                        {\n                            char txt[1024];\n                            snprintf(txt, sizeof(txt), \"Success! Waiting for voice commands ...\");\n                            command_set_status(txt);\n                        }\n\n                        // save the audio for the prompt\n                        pcmf32_prompt = pcmf32_cur;\n                        have_prompt = true;\n                    }\n                } else {\n                    command_get_audio(command_ms, WHISPER_SAMPLE_RATE, pcmf32_cur);\n\n                    // prepend the prompt audio\n                    pcmf32_cur.insert(pcmf32_cur.begin(), pcmf32_prompt.begin(), pcmf32_prompt.end());\n\n                    const auto txt = ::trim(::command_transcribe(ctx, wparams, pcmf32_cur, prob, t_ms));\n\n                    prob = 100.0f*(prob - prob0);\n\n                    fprintf(stdout, \"%s: heard '%s'\\n\", __func__, txt.c_str());\n\n                    // find the prompt in the text\n                    float best_sim = 0.0f;\n                    size_t best_len = 0;\n                    for (int n = 0.8*k_prompt.size(); n <= 1.2*k_prompt.size(); ++n) {\n                        const auto prompt = txt.substr(0, n);\n\n                        const float sim = similarity(prompt, k_prompt);\n\n                        //fprintf(stderr, \"%s: prompt = '%s', sim = %f\\n\", __func__, prompt.c_str(), sim);\n\n                        if (sim > best_sim) {\n                            best_sim = sim;\n                            best_len = n;\n                        }\n                    }\n\n                    const std::string command = ::trim(txt.substr(best_len));\n\n                    fprintf(stdout, \"%s: Command '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", command.c_str(), \"\\033[0m\", (int) t_ms);\n                    fprintf(stdout, \"\\n\");\n\n                    {\n                        char txt[1024];\n                        snprintf(txt, sizeof(txt), \"Command '%s', (t = %d ms)\", command.c_str(), (int) t_ms);\n                        command_set_status(txt);\n                    }\n                    {\n                        std::lock_guard<std::mutex> lock(g_mutex);\n                        g_transcribed = command;\n                    }\n                }\n\n                g_pcmf32.clear();\n            }\n        }\n    }\n\n    if (index < g_contexts.size()) {\n        whisper_free(g_contexts[index]);\n        g_contexts[index] = nullptr;\n    }\n}\n\nEMSCRIPTEN_BINDINGS(command) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model) {\n        for (size_t i = 0; i < g_contexts.size(); ++i) {\n            if (g_contexts[i] == nullptr) {\n                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n                if (g_contexts[i] != nullptr) {\n                    g_running = true;\n                    if (g_worker.joinable()) {\n                        g_worker.join();\n                    }\n                    g_worker = std::thread([i]() {\n                        command_main(i);\n                    });\n\n                    return i + 1;\n                } else {\n                    return (size_t) 0;\n                }\n            }\n        }\n\n        return (size_t) 0;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([](size_t index) {\n        if (g_running) {\n            g_running = false;\n        }\n    }));\n\n    emscripten::function(\"set_audio\", emscripten::optional_override([](size_t index, const emscripten::val & audio) {\n        --index;\n\n        if (index >= g_contexts.size()) {\n            return -1;\n        }\n\n        if (g_contexts[index] == nullptr) {\n            return -2;\n        }\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            const int n = audio[\"length\"].as<int>();\n\n            emscripten::val heap = emscripten::val::module_property(\"HEAPU8\");\n            emscripten::val memory = heap[\"buffer\"];\n\n            g_pcmf32.resize(n);\n\n            emscripten::val memoryView = audio[\"constructor\"].new_(memory, reinterpret_cast<uintptr_t>(g_pcmf32.data()), n);\n            memoryView.call<void>(\"set\", audio);\n        }\n\n        return 0;\n    }));\n\n    emscripten::function(\"get_transcribed\", emscripten::optional_override([]() {\n        std::string transcribed;\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            transcribed = std::move(g_transcribed);\n        }\n\n        return transcribed;\n    }));\n\n    emscripten::function(\"get_status\", emscripten::optional_override([]() {\n        std::string status;\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            status = g_status_forced.empty() ? g_status : g_status_forced;\n        }\n\n        return status;\n    }));\n\n    emscripten::function(\"set_status\", emscripten::optional_override([](const std::string & status) {\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            g_status_forced = status;\n        }\n    }));\n}\n"
  },
  {
    "path": "examples/command.wasm/index-tmpl.html",
    "content": "<!doctype html>\n<html lang=\"en-us\">\n    <head>\n        <title>command : Voice assistant example using Whisper + WebAssembly</title>\n\n        <style>\n            #output {\n                width: 100%;\n                height: 100%;\n                margin: 0 auto;\n                margin-top: 10px;\n                border-left: 0px;\n                border-right: 0px;\n                padding-left: 0px;\n                padding-right: 0px;\n                display: block;\n                background-color: black;\n                color: white;\n                font-size: 10px;\n                font-family: 'Lucida Console', Monaco, monospace;\n                outline: none;\n                white-space: pre;\n                overflow-wrap: normal;\n                overflow-x: scroll;\n            }\n        </style>\n        <script src=\"../coi-serviceworker.js\"></script>\n        <link rel=\"icon\" href=\"data:,\">\n    </head>\n    <body>\n        <div id=\"main-container\">\n            <b>command : Voice assistant example using Whisper + WebAssembly</b>\n\n            <br><br>\n\n            You can find more about this project on <a href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/command.wasm\">GitHub</a>.\n\n            <br><br>\n\n            <b>More examples:</b>\n                <a href=\"../\">main</a> |\n                <a href=\"../bench.wasm/\">bench</a> |\n                <a href=\"../stream.wasm\">stream</a> |\n                <a href=\"../command.wasm/\">command</a> |\n                <a href=\"../wchess.wasm/\">wchess</a> |\n\n            <br><br>\n\n            <hr>\n\n            Select the model you would like to use, click the \"Start\" button and follow the instructions.\n\n            <br><br>\n\n            <div id=\"model-whisper\">\n                Whisper model: <span id=\"model-whisper-status\"></span>\n                <button id=\"fetch-whisper-tiny-en\" onclick=\"loadWhisper('tiny.en')\">tiny.en (75 MB)</button>\n                <button id=\"fetch-whisper-base-en\" onclick=\"loadWhisper('base.en')\">base.en (142 MB)</button>\n                <br><br>\n                Quantized models:<br><br>\n                <button id=\"fetch-whisper-tiny-en-q5_1\"   onclick=\"loadWhisper('tiny-en-q5_1')\">tiny.en (Q5_1, 31 MB)</button>\n                <button id=\"fetch-whisper-base-en-q5_1\"   onclick=\"loadWhisper('base-en-q5_1')\">base.en (Q5_1, 57 MB)</button>\n                <span id=\"fetch-whisper-progress\"></span>\n\n                <!--\n                    <input type=\"file\" id=\"file\" name=\"file\" onchange=\"loadFile(event, 'whisper.bin')\" />\n                -->\n            </div>\n\n            <br>\n\n            <div id=\"input\">\n                <button id=\"start\" onclick=\"onStart()\" disabled>Start</button>\n                <button id=\"stop\"  onclick=\"onStop()\" disabled>Stop</button>\n                <button id=\"clear\" onclick=\"clearCache()\">Clear Cache</button>\n            </div>\n\n            <br>\n\n            <div id=\"state\">\n                Status: <b><span id=\"state-status\">not started</span></b>\n\n                <pre id=\"state-transcribed\">[The recognized voice commands will be displayed here]</pre>\n            </div>\n\n            <hr>\n\n            Debug output:\n            <textarea id=\"output\" rows=\"20\"></textarea>\n\n            <br>\n\n            <b>Troubleshooting</b>\n\n            <br><br>\n\n            The page does some heavy computations, so make sure:\n\n            <ul>\n                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>\n                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>\n                <li>Your browser supports WASM <a href=\"https://webassembly.org/roadmap/\">Fixed-width SIMD</a></li>\n            </ul>\n\n            <div class=\"cell-version\">\n                <span>\n                    |\n                    Build time: <span class=\"nav-link\">@GIT_DATE@</span> |\n                    Commit hash: <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@\">@GIT_SHA1@</a> |\n                    Commit subject: <span class=\"nav-link\">@GIT_COMMIT_SUBJECT@</span> |\n                    <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/command.wasm\">Source Code</a> |\n                </span>\n            </div>\n        </div>\n\n        <script type=\"text/javascript\" src=\"helpers.js\"></script>\n        <script type='text/javascript'>\n            // web audio context\n            var context = null;\n\n            // audio data\n            var audio = null;\n            var audio0 = null;\n\n            // the command instance\n            var instance = null;\n\n            // model name\n            var model_whisper = null;\n\n            var Module = {\n                print: printTextarea,\n                printErr: printTextarea,\n                setStatus: function(text) {\n                    printTextarea('js: ' + text);\n                },\n                monitorRunDependencies: function(left) {\n                },\n                preRun: function() {\n                    printTextarea('js: Preparing ...');\n                },\n                postRun: function() {\n                    printTextarea('js: Initialized successfully!');\n                }\n            };\n\n            //\n            // fetch models\n            //\n\n            let dbVersion = 1\n            let dbName    = 'whisper.ggerganov.com';\n            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB\n\n            function storeFS(fname, buf) {\n                // write to WASM file using FS_createDataFile\n                // if the file exists, delete it\n                try {\n                    Module.FS_unlink(fname);\n                } catch (e) {\n                    // ignore\n                }\n\n                Module.FS_createDataFile(\"/\", fname, buf, true, true);\n\n                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);\n\n                document.getElementById('model-whisper-status').innerHTML = 'loaded \"' + model_whisper + '\"!';\n\n                if (model_whisper != null) {\n                    document.getElementById('start').disabled = false;\n                    document.getElementById('stop' ).disabled = true;\n                }\n            }\n\n            function loadWhisper(model) {\n                let urls = {\n                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',\n                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',\n\n                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',\n                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',\n                };\n\n                let sizes = {\n                    'tiny.en': 75,\n                    'base.en': 142,\n\n                    'tiny-en-q5_1':   31,\n                    'base-en-q5_1':   57,\n                };\n\n                let url     = urls[model];\n                let dst     = 'whisper.bin';\n                let size_mb = sizes[model];\n\n                model_whisper = model;\n\n                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';\n                document.getElementById('fetch-whisper-base-en').style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1').style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1').style.display = 'none';\n\n                document.getElementById('model-whisper-status').innerHTML = 'loading \"' + model + '\" ... ';\n\n                cbProgress = function(p) {\n                    let el = document.getElementById('fetch-whisper-progress');\n                    el.innerHTML = Math.round(100*p) + '%';\n                };\n\n                cbCancel = function() {\n                    var el;\n                    el = document.getElementById('fetch-whisper-tiny-en'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en'); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en-q5_1'); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('model-whisper-status');  if (el) el.innerHTML = '';\n                };\n\n                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);\n            }\n\n            //\n            // microphone\n            //\n\n            const kSampleRate = 16000;\n            const kRestartRecording_s = 120;\n            const kIntervalAudio_ms = 250; // pass the recorded audio to the C++ instance at this rate\n\n            var mediaRecorder = null;\n            var doRecording = false;\n            var startTime = 0;\n\n            window.AudioContext = window.AudioContext || window.webkitAudioContext;\n            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;\n\n            function stopRecording() {\n                Module.set_status(\"paused\");\n                doRecording = false;\n                audio0 = null;\n                audio = null;\n                context = null;\n            }\n\n            function startRecording() {\n                if (!context) {\n                    context = new AudioContext({\n                        sampleRate: kSampleRate,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                }\n\n                Module.set_status(\"\");\n\n                document.getElementById('start').disabled = true;\n                document.getElementById('stop').disabled = false;\n\n                doRecording = true;\n                startTime = Date.now();\n\n                var chunks = [];\n                var stream = null;\n\n                navigator.mediaDevices.getUserMedia({audio: true, video: false})\n                    .then(function(s) {\n                        stream = s;\n                        mediaRecorder = new MediaRecorder(stream);\n                        mediaRecorder.ondataavailable = function(e) {\n                            chunks.push(e.data);\n\n                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });\n                            var reader = new FileReader();\n\n                            reader.onload = function(event) {\n                                var buf = new Uint8Array(reader.result);\n\n                                if (!context) {\n                                    return;\n                                }\n                                context.decodeAudioData(buf.buffer, function(audioBuffer) {\n                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);\n                                    var source = offlineContext.createBufferSource();\n                                    source.buffer = audioBuffer;\n                                    source.connect(offlineContext.destination);\n                                    source.start(0);\n\n                                    offlineContext.startRendering().then(function(renderedBuffer) {\n                                        audio = renderedBuffer.getChannelData(0);\n\n                                        //printTextarea('js: audio recorded, size: ' + audio.length + ', old size: ' + (audio0 == null ? 0 : audio0.length));\n\n                                        var audioAll = new Float32Array(audio0 == null ? audio.length : audio0.length + audio.length);\n                                        if (audio0 != null) {\n                                            audioAll.set(audio0, 0);\n                                        }\n                                        audioAll.set(audio, audio0 == null ? 0 : audio0.length);\n\n                                        if (instance) {\n                                            Module.set_audio(instance, audioAll);\n                                        }\n                                    });\n                                }, function(e) {\n                                    audio = null;\n                                });\n                            }\n\n                            reader.readAsArrayBuffer(blob);\n                        };\n\n                        mediaRecorder.onstop = function(e) {\n                            if (doRecording) {\n                                setTimeout(function() {\n                                    startRecording();\n                                });\n                            }\n                        };\n\n                        mediaRecorder.start(kIntervalAudio_ms);\n                    })\n                    .catch(function(err) {\n                        printTextarea('js: error getting audio stream: ' + err);\n                    });\n\n                var interval = setInterval(function() {\n                    if (!doRecording) {\n                        clearInterval(interval);\n                        mediaRecorder.stop();\n                        stream.getTracks().forEach(function(track) {\n                            track.stop();\n                        });\n\n                        document.getElementById('start').disabled = false;\n                        document.getElementById('stop').disabled  = true;\n\n                        mediaRecorder = null;\n                    }\n\n                    // if audio length is more than kRestartRecording_s seconds, restart recording\n                    if (audio != null && audio.length > kSampleRate*kRestartRecording_s) {\n                        if (doRecording) {\n                            //printTextarea('js: restarting recording');\n\n                            clearInterval(interval);\n                            audio0 = audio;\n                            audio = null;\n                            mediaRecorder.stop();\n                            stream.getTracks().forEach(function(track) {\n                                track.stop();\n                            });\n                        }\n                    }\n                }, 100);\n            }\n\n            //\n            // main\n            //\n\n            var nLines = 0;\n            var intervalUpdate = null;\n            var transcribedAll = '';\n\n            function onStart() {\n                if (!instance) {\n                    instance = Module.init('whisper.bin');\n\n                    if (instance) {\n                        printTextarea(\"js: whisper initialized, instance: \" + instance);\n                    }\n                }\n\n                if (!instance) {\n                    printTextarea(\"js: failed to initialize whisper\");\n                    return;\n                }\n\n                startRecording();\n\n                intervalUpdate = setInterval(function() {\n                    var transcribed = Module.get_transcribed();\n\n                    if (transcribed != null && transcribed.length > 1) {\n                        transcribedAll += transcribed + '<br>';\n                        nLines++;\n\n                        // if more than 10 lines, remove the first line\n                        if (nLines > 10) {\n                            var i = transcribedAll.indexOf('<br>');\n                            if (i > 0) {\n                                transcribedAll = transcribedAll.substring(i + 4);\n                                nLines--;\n                            }\n                        }\n                    }\n\n                    document.getElementById('state-status').innerHTML = Module.get_status();\n                    document.getElementById('state-transcribed').innerHTML = transcribedAll;\n                }, 100);\n            }\n\n            function onStop() {\n                stopRecording();\n            }\n\n        </script>\n        <script type=\"text/javascript\" src=\"command.js\"></script>\n    </body>\n</html>\n"
  },
  {
    "path": "examples/common-ggml.cpp",
    "content": "#include \"common-ggml.h\"\n\n#include <regex>\n#include <map>\n\nstatic const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {\n    {\"q4_0\", GGML_FTYPE_MOSTLY_Q4_0},\n    {\"q4_1\", GGML_FTYPE_MOSTLY_Q4_1},\n    {\"q5_0\", GGML_FTYPE_MOSTLY_Q5_0},\n    {\"q5_1\", GGML_FTYPE_MOSTLY_Q5_1},\n    {\"q8_0\", GGML_FTYPE_MOSTLY_Q8_0},\n    {\"q2_k\", GGML_FTYPE_MOSTLY_Q2_K},\n    {\"q3_k\", GGML_FTYPE_MOSTLY_Q3_K},\n    {\"q4_k\", GGML_FTYPE_MOSTLY_Q4_K},\n    {\"q5_k\", GGML_FTYPE_MOSTLY_Q5_K},\n    {\"q6_k\", GGML_FTYPE_MOSTLY_Q6_K},\n};\n\nvoid ggml_print_ftypes(FILE * fp) {\n    for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {\n        fprintf(fp, \"  type = \\\"%s\\\" or %d\\n\", it->first.c_str(), it->second);\n    }\n}\n\nenum ggml_ftype ggml_parse_ftype(const char * str) {\n    enum ggml_ftype ftype;\n    if (str[0] == 'q') {\n        const auto it = GGML_FTYPE_MAP.find(str);\n        if (it == GGML_FTYPE_MAP.end()) {\n            fprintf(stderr, \"%s: unknown ftype '%s'\\n\", __func__, str);\n            return GGML_FTYPE_UNKNOWN;\n        }\n        ftype = it->second;\n    } else {\n        ftype = (enum ggml_ftype) atoi(str);\n    }\n\n    return ftype;\n}\n\nbool ggml_common_quantize_0(\n        std::ifstream & finp,\n        std::ofstream & fout,\n        const ggml_ftype ftype,\n        const std::vector<std::string> & to_quant,\n        const std::vector<std::string> & to_skip) {\n\n    ggml_type qtype = GGML_TYPE_F32;\n\n    switch (ftype) {\n        case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;\n        case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;\n        case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;\n        case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;\n        case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;\n        case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;\n        case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;\n        case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;\n        case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;\n        case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;\n        case GGML_FTYPE_UNKNOWN:\n        case GGML_FTYPE_ALL_F32:\n        case GGML_FTYPE_MOSTLY_F16:\n        case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:\n        case GGML_FTYPE_MOSTLY_IQ2_XXS:\n        case GGML_FTYPE_MOSTLY_IQ2_XS:\n        case GGML_FTYPE_MOSTLY_IQ2_S:\n        case GGML_FTYPE_MOSTLY_IQ3_XXS:\n        case GGML_FTYPE_MOSTLY_IQ3_S:\n        case GGML_FTYPE_MOSTLY_IQ1_S:\n        case GGML_FTYPE_MOSTLY_IQ4_NL:\n        case GGML_FTYPE_MOSTLY_IQ4_XS:\n        case GGML_FTYPE_MOSTLY_IQ1_M:\n        case GGML_FTYPE_MOSTLY_BF16:\n        case GGML_FTYPE_MOSTLY_MXFP4:\n        case GGML_FTYPE_MOSTLY_NVFP4:\n                {\n                    fprintf(stderr, \"%s: invalid model type %d\\n\", __func__, ftype);\n                    return false;\n                }\n    };\n\n    if (!ggml_is_quantized(qtype)) {\n        fprintf(stderr, \"%s: invalid quantization type %d (%s)\\n\", __func__, qtype, ggml_type_name(qtype));\n        return false;\n    }\n\n    size_t total_size_org = 0;\n    size_t total_size_new = 0;\n\n    std::vector<float> work;\n\n    std::vector<uint8_t>     data_u8;\n    std::vector<ggml_fp16_t> data_f16;\n    std::vector<float>       data_f32;\n\n    while (true) {\n        int32_t n_dims;\n        int32_t length;\n        int32_t ttype;\n\n        finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));\n        finp.read(reinterpret_cast<char *>(&length), sizeof(length));\n        finp.read(reinterpret_cast<char *>(&ttype),  sizeof(ttype));\n\n        if (finp.eof()) {\n            break;\n        }\n\n        int32_t nelements = 1;\n        int32_t ne[4] = { 1, 1, 1, 1 };\n        for (int i = 0; i < n_dims; ++i) {\n            finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));\n            nelements *= ne[i];\n        }\n\n        std::string name(length, 0);\n        finp.read (&name[0], length);\n\n        printf(\"%64s - [%5d, %5d, %5d], type = %6s \", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));\n\n        bool quantize = false;\n\n        // check if we should quantize this tensor\n        for (const auto & s : to_quant) {\n            if (std::regex_match(name, std::regex(s))) {\n                quantize = true;\n                break;\n            }\n        }\n\n        // check if we should skip this tensor\n        for (const auto & s : to_skip) {\n            if (std::regex_match(name, std::regex(s))) {\n                quantize = false;\n                break;\n            }\n        }\n\n        // quantize only 2D tensors\n        quantize &= (n_dims == 2);\n\n        if (quantize) {\n            if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {\n                fprintf(stderr, \"%s: unsupported ttype %d (%s) for integer quantization\\n\", __func__, ttype, ggml_type_name((ggml_type) ttype));\n                return false;\n            }\n\n            if (ttype == GGML_TYPE_F16) {\n                data_f16.resize(nelements);\n                finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));\n                data_f32.resize(nelements);\n                for (int i = 0; i < nelements; ++i) {\n                    data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);\n                }\n            } else {\n                data_f32.resize(nelements);\n                finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));\n            }\n\n            ttype = qtype;\n        } else {\n            const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);\n\n            data_u8.resize(nelements*bpe);\n            finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);\n        }\n\n        fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));\n        fout.write(reinterpret_cast<char *>(&length), sizeof(length));\n        fout.write(reinterpret_cast<char *>(&ttype),  sizeof(ttype));\n        for (int i = 0; i < n_dims; ++i) {\n            fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));\n        }\n        fout.write(&name[0], length);\n\n        if (quantize) {\n            work.resize(nelements); // for quantization\n\n            size_t cur_size = 0;\n            switch ((ggml_type) ttype) {\n                case GGML_TYPE_Q4_0:\n                case GGML_TYPE_Q4_1:\n                case GGML_TYPE_Q5_0:\n                case GGML_TYPE_Q5_1:\n                case GGML_TYPE_Q8_0:\n                case GGML_TYPE_Q2_K:\n                case GGML_TYPE_Q3_K:\n                case GGML_TYPE_Q4_K:\n                case GGML_TYPE_Q5_K:\n                case GGML_TYPE_Q6_K:\n                    {\n                        cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);\n                    } break;\n                case GGML_TYPE_F32:\n                case GGML_TYPE_F16:\n                case GGML_TYPE_I8:\n                case GGML_TYPE_I16:\n                case GGML_TYPE_I32:\n                case GGML_TYPE_I64:\n                case GGML_TYPE_F64:\n                case GGML_TYPE_Q8_1:\n                case GGML_TYPE_Q8_K:\n                case GGML_TYPE_IQ2_XXS:\n                case GGML_TYPE_IQ2_XS:\n                case GGML_TYPE_IQ2_S:\n                case GGML_TYPE_IQ3_XXS:\n                case GGML_TYPE_IQ3_S:\n                case GGML_TYPE_IQ1_S:\n                case GGML_TYPE_IQ4_NL:\n                case GGML_TYPE_IQ4_XS:\n                case GGML_TYPE_IQ1_M:\n                case GGML_TYPE_BF16:\n                case GGML_TYPE_TQ1_0:\n                case GGML_TYPE_TQ2_0:\n                case GGML_TYPE_MXFP4:\n                case GGML_TYPE_NVFP4:\n                case GGML_TYPE_COUNT:\n                    {\n                        fprintf(stderr, \"%s: unsupported quantization type %d (%s)\\n\", __func__, ttype, ggml_type_name((ggml_type) ttype));\n                        return false;\n                    }\n            }\n\n            fout.write(reinterpret_cast<char *>(work.data()), cur_size);\n            total_size_new += cur_size;\n\n            printf(\"size = %8.2f MB -> %8.2f MB\\n\", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);\n        } else {\n            printf(\"size = %8.3f MB\\n\", data_u8.size()/1024.0/1024.0);\n            fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());\n            total_size_new += data_u8.size();\n        }\n\n        total_size_org += nelements * sizeof(float);\n    }\n\n    printf(\"%s: model size  = %8.2f MB\\n\", __func__, total_size_org/1024.0/1024.0);\n    printf(\"%s: quant size  = %8.2f MB | ftype = %d (%s)\\n\", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));\n\n    return true;\n}\n"
  },
  {
    "path": "examples/common-ggml.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n\n#include <fstream>\n#include <vector>\n#include <string>\n\nenum ggml_ftype ggml_parse_ftype(const char * str);\n\nvoid ggml_print_ftypes(FILE * fp = stderr);\n\nbool ggml_common_quantize_0(\n        std::ifstream & finp,\n        std::ofstream & fout,\n        const ggml_ftype ftype,\n        const std::vector<std::string> & to_quant,\n        const std::vector<std::string> & to_skip);\n"
  },
  {
    "path": "examples/common-sdl.cpp",
    "content": "#include \"common-sdl.h\"\n\n#include <cstdio>\n\naudio_async::audio_async(int len_ms) {\n    m_len_ms = len_ms;\n\n    m_running = false;\n}\n\naudio_async::~audio_async() {\n    if (m_dev_id_in) {\n        SDL_CloseAudioDevice(m_dev_id_in);\n    }\n}\n\nbool audio_async::init(int capture_id, int sample_rate) {\n    SDL_LogSetPriority(SDL_LOG_CATEGORY_APPLICATION, SDL_LOG_PRIORITY_INFO);\n\n    if (SDL_Init(SDL_INIT_AUDIO) < 0) {\n        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, \"Couldn't initialize SDL: %s\\n\", SDL_GetError());\n        return false;\n    }\n\n    SDL_SetHintWithPriority(SDL_HINT_AUDIO_RESAMPLING_MODE, \"medium\", SDL_HINT_OVERRIDE);\n\n    {\n        int nDevices = SDL_GetNumAudioDevices(SDL_TRUE);\n        fprintf(stderr, \"%s: found %d capture devices:\\n\", __func__, nDevices);\n        for (int i = 0; i < nDevices; i++) {\n            fprintf(stderr, \"%s:    - Capture device #%d: '%s'\\n\", __func__, i, SDL_GetAudioDeviceName(i, SDL_TRUE));\n        }\n    }\n\n    SDL_AudioSpec capture_spec_requested;\n    SDL_AudioSpec capture_spec_obtained;\n\n    SDL_zero(capture_spec_requested);\n    SDL_zero(capture_spec_obtained);\n\n    capture_spec_requested.freq     = sample_rate;\n    capture_spec_requested.format   = AUDIO_F32;\n    capture_spec_requested.channels = 1;\n    capture_spec_requested.samples  = 1024;\n    capture_spec_requested.callback = [](void * userdata, uint8_t * stream, int len) {\n        audio_async * audio = (audio_async *) userdata;\n        audio->callback(stream, len);\n    };\n    capture_spec_requested.userdata = this;\n\n    if (capture_id >= 0) {\n        fprintf(stderr, \"%s: attempt to open capture device %d : '%s' ...\\n\", __func__, capture_id, SDL_GetAudioDeviceName(capture_id, SDL_TRUE));\n        m_dev_id_in = SDL_OpenAudioDevice(SDL_GetAudioDeviceName(capture_id, SDL_TRUE), SDL_TRUE, &capture_spec_requested, &capture_spec_obtained, 0);\n    } else {\n        fprintf(stderr, \"%s: attempt to open default capture device ...\\n\", __func__);\n        m_dev_id_in = SDL_OpenAudioDevice(nullptr, SDL_TRUE, &capture_spec_requested, &capture_spec_obtained, 0);\n    }\n\n    if (!m_dev_id_in) {\n        fprintf(stderr, \"%s: couldn't open an audio device for capture: %s!\\n\", __func__, SDL_GetError());\n        m_dev_id_in = 0;\n\n        return false;\n    } else {\n        fprintf(stderr, \"%s: obtained spec for input device (SDL Id = %d):\\n\", __func__, m_dev_id_in);\n        fprintf(stderr, \"%s:     - sample rate:       %d\\n\",                   __func__, capture_spec_obtained.freq);\n        fprintf(stderr, \"%s:     - format:            %d (required: %d)\\n\",    __func__, capture_spec_obtained.format,\n                capture_spec_requested.format);\n        fprintf(stderr, \"%s:     - channels:          %d (required: %d)\\n\",    __func__, capture_spec_obtained.channels,\n                capture_spec_requested.channels);\n        fprintf(stderr, \"%s:     - samples per frame: %d\\n\",                   __func__, capture_spec_obtained.samples);\n    }\n\n    m_sample_rate = capture_spec_obtained.freq;\n\n    m_audio.resize((m_sample_rate*m_len_ms)/1000);\n\n    return true;\n}\n\nbool audio_async::resume() {\n    if (!m_dev_id_in) {\n        fprintf(stderr, \"%s: no audio device to resume!\\n\", __func__);\n        return false;\n    }\n\n    if (m_running) {\n        fprintf(stderr, \"%s: already running!\\n\", __func__);\n        return false;\n    }\n\n    SDL_PauseAudioDevice(m_dev_id_in, 0);\n\n    m_running = true;\n\n    return true;\n}\n\nbool audio_async::pause() {\n    if (!m_dev_id_in) {\n        fprintf(stderr, \"%s: no audio device to pause!\\n\", __func__);\n        return false;\n    }\n\n    if (!m_running) {\n        fprintf(stderr, \"%s: already paused!\\n\", __func__);\n        return false;\n    }\n\n    SDL_PauseAudioDevice(m_dev_id_in, 1);\n\n    m_running = false;\n\n    return true;\n}\n\nbool audio_async::clear() {\n    if (!m_dev_id_in) {\n        fprintf(stderr, \"%s: no audio device to clear!\\n\", __func__);\n        return false;\n    }\n\n    if (!m_running) {\n        fprintf(stderr, \"%s: not running!\\n\", __func__);\n        return false;\n    }\n\n    {\n        std::lock_guard<std::mutex> lock(m_mutex);\n\n        m_audio_pos = 0;\n        m_audio_len = 0;\n    }\n\n    return true;\n}\n\n// callback to be called by SDL\nvoid audio_async::callback(uint8_t * stream, int len) {\n    if (!m_running) {\n        return;\n    }\n\n    size_t n_samples = len / sizeof(float);\n\n    if (n_samples > m_audio.size()) {\n        n_samples = m_audio.size();\n\n        stream += (len - (n_samples * sizeof(float)));\n    }\n\n    //fprintf(stderr, \"%s: %zu samples, pos %zu, len %zu\\n\", __func__, n_samples, m_audio_pos, m_audio_len);\n\n    {\n        std::lock_guard<std::mutex> lock(m_mutex);\n\n        if (m_audio_pos + n_samples > m_audio.size()) {\n            const size_t n0 = m_audio.size() - m_audio_pos;\n\n            memcpy(&m_audio[m_audio_pos], stream, n0 * sizeof(float));\n            memcpy(&m_audio[0], stream + n0 * sizeof(float), (n_samples - n0) * sizeof(float));\n        } else {\n            memcpy(&m_audio[m_audio_pos], stream, n_samples * sizeof(float));\n        }\n        m_audio_pos = (m_audio_pos + n_samples) % m_audio.size();\n        m_audio_len = std::min(m_audio_len + n_samples, m_audio.size());\n    }\n}\n\nvoid audio_async::get(int ms, std::vector<float> & result) {\n    if (!m_dev_id_in) {\n        fprintf(stderr, \"%s: no audio device to get audio from!\\n\", __func__);\n        return;\n    }\n\n    if (!m_running) {\n        fprintf(stderr, \"%s: not running!\\n\", __func__);\n        return;\n    }\n\n    result.clear();\n\n    {\n        std::lock_guard<std::mutex> lock(m_mutex);\n\n        if (ms <= 0) {\n            ms = m_len_ms;\n        }\n\n        size_t n_samples = (m_sample_rate * ms) / 1000;\n        if (n_samples > m_audio_len) {\n            n_samples = m_audio_len;\n        }\n\n        result.resize(n_samples);\n\n        int s0 = m_audio_pos - n_samples;\n        if (s0 < 0) {\n            s0 += m_audio.size();\n        }\n\n        if (s0 + n_samples > m_audio.size()) {\n            const size_t n0 = m_audio.size() - s0;\n\n            memcpy(result.data(), &m_audio[s0], n0 * sizeof(float));\n            memcpy(&result[n0], &m_audio[0], (n_samples - n0) * sizeof(float));\n        } else {\n            memcpy(result.data(), &m_audio[s0], n_samples * sizeof(float));\n        }\n    }\n}\n\nbool sdl_poll_events() {\n    SDL_Event event;\n    while (SDL_PollEvent(&event)) {\n        switch (event.type) {\n            case SDL_QUIT:\n                {\n                    return false;\n                }\n            default:\n                break;\n        }\n    }\n\n    return true;\n}\n"
  },
  {
    "path": "examples/common-sdl.h",
    "content": "#pragma once\n\n#include <SDL.h>\n#include <SDL_audio.h>\n\n#include <atomic>\n#include <cstdint>\n#include <vector>\n#include <mutex>\n\n//\n// SDL Audio capture\n//\n\nclass audio_async {\npublic:\n    audio_async(int len_ms);\n    ~audio_async();\n\n    bool init(int capture_id, int sample_rate);\n\n    // start capturing audio via the provided SDL callback\n    // keep last len_ms seconds of audio in a circular buffer\n    bool resume();\n    bool pause();\n    bool clear();\n\n    // callback to be called by SDL\n    void callback(uint8_t * stream, int len);\n\n    // get audio data from the circular buffer\n    void get(int ms, std::vector<float> & audio);\n\nprivate:\n    SDL_AudioDeviceID m_dev_id_in = 0;\n\n    int m_len_ms = 0;\n    int m_sample_rate = 0;\n\n    std::atomic_bool m_running;\n    std::mutex       m_mutex;\n\n    std::vector<float> m_audio;\n    size_t             m_audio_pos = 0;\n    size_t             m_audio_len = 0;\n};\n\n// Return false if need to quit\nbool sdl_poll_events();\n"
  },
  {
    "path": "examples/common-whisper.cpp",
    "content": "#define _USE_MATH_DEFINES // for M_PI\n\n#include \"common-whisper.h\"\n\n#include \"common.h\"\n\n#include \"whisper.h\"\n\n// third-party utilities\n// use your favorite implementations\n#define STB_VORBIS_HEADER_ONLY\n#include \"stb_vorbis.c\"    /* Enables Vorbis decoding. */\n\n#ifdef _WIN32\n#ifndef NOMINMAX\n    #define NOMINMAX\n#endif\n#endif\n\n#define MA_NO_DEVICE_IO\n#define MA_NO_THREADING\n#define MA_NO_ENCODING\n#define MA_NO_GENERATION\n#define MA_NO_RESOURCE_MANAGER\n#define MA_NO_NODE_GRAPH\n#define MINIAUDIO_IMPLEMENTATION\n#include \"miniaudio.h\"\n\n#ifdef _WIN32\n#include <fcntl.h>\n#include <io.h>\n#endif\n\n#include <cstring>\n#include <fstream>\n\n#ifdef WHISPER_FFMPEG\n// as implemented in ffmpeg_trancode.cpp only embedded in common lib if whisper built with ffmpeg support\nextern bool ffmpeg_decode_audio(const std::string & ifname, std::vector<uint8_t> & wav_data);\n#endif\n\nbool read_audio_data(const std::string & fname, std::vector<float>& pcmf32, std::vector<std::vector<float>>& pcmf32s, bool stereo) {\n    std::vector<uint8_t> audio_data; // used for pipe input from stdin or ffmpeg decoding output\n\n    ma_result result;\n    ma_decoder_config decoder_config;\n    ma_decoder decoder;\n\n    decoder_config = ma_decoder_config_init(ma_format_f32, stereo ? 2 : 1, WHISPER_SAMPLE_RATE);\n\n    if (fname == \"-\") {\n\t\t#ifdef _WIN32\n\t\t_setmode(_fileno(stdin), _O_BINARY);\n\t\t#endif\n\n\t\tuint8_t buf[1024];\n\t\twhile (true)\n\t\t{\n\t\t\tconst size_t n = fread(buf, 1, sizeof(buf), stdin);\n\t\t\tif (n == 0) {\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\taudio_data.insert(audio_data.end(), buf, buf + n);\n\t\t}\n\n\t\tif ((result = ma_decoder_init_memory(audio_data.data(), audio_data.size(), &decoder_config, &decoder)) != MA_SUCCESS) {\n\n\t\t\tfprintf(stderr, \"Error: failed to open audio data from stdin (%s)\\n\", ma_result_description(result));\n\n\t\t\treturn false;\n\t\t}\n\n\t\tfprintf(stderr, \"%s: read %zu bytes from stdin\\n\", __func__, audio_data.size());\n    }\n    else if (((result = ma_decoder_init_file(fname.c_str(), &decoder_config, &decoder)) != MA_SUCCESS)) {\n#if defined(WHISPER_FFMPEG)\n\t\tif (ffmpeg_decode_audio(fname, audio_data) != 0) {\n\t\t\tfprintf(stderr, \"error: failed to ffmpeg decode '%s'\\n\", fname.c_str());\n\n\t\t\treturn false;\n\t\t}\n\n\t\tif ((result = ma_decoder_init_memory(audio_data.data(), audio_data.size(), &decoder_config, &decoder)) != MA_SUCCESS) {\n\t\t\tfprintf(stderr, \"error: failed to read audio data as wav (%s)\\n\", ma_result_description(result));\n\n\t\t\treturn false;\n\t\t}\n#else\n\t\tif ((result = ma_decoder_init_memory(fname.c_str(), fname.size(), &decoder_config, &decoder)) != MA_SUCCESS) {\n\t\t\tfprintf(stderr, \"error: failed to read audio data as wav (%s)\\n\", ma_result_description(result));\n\n\t\t\treturn false;\n\t\t}\n#endif\n    }\n\n    ma_uint64 frame_count;\n    ma_uint64 frames_read;\n\n    if ((result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count)) != MA_SUCCESS) {\n\t\tfprintf(stderr, \"error: failed to retrieve the length of the audio data (%s)\\n\", ma_result_description(result));\n\n\t\treturn false;\n    }\n\n    pcmf32.resize(stereo ? frame_count*2 : frame_count);\n\n    if ((result = ma_decoder_read_pcm_frames(&decoder, pcmf32.data(), frame_count, &frames_read)) != MA_SUCCESS) {\n\t\tfprintf(stderr, \"error: failed to read the frames of the audio data (%s)\\n\", ma_result_description(result));\n\n\t\treturn false;\n    }\n\n    if (stereo) {\n        std::vector<float> stereo_data = pcmf32;\n        pcmf32.resize(frame_count);\n\n        for (uint64_t i = 0; i < frame_count; i++) {\n            pcmf32[i] = (stereo_data[2*i] + stereo_data[2*i + 1]);\n        }\n\n        pcmf32s.resize(2);\n        pcmf32s[0].resize(frame_count);\n        pcmf32s[1].resize(frame_count);\n        for (uint64_t i = 0; i < frame_count; i++) {\n            pcmf32s[0][i] = stereo_data[2*i];\n            pcmf32s[1][i] = stereo_data[2*i + 1];\n        }\n    }\n\n    ma_decoder_uninit(&decoder);\n\n    return true;\n}\n\n//  500 -> 00:05.000\n// 6000 -> 01:00.000\nstd::string to_timestamp(int64_t t, bool comma) {\n    int64_t msec = t * 10;\n    int64_t hr = msec / (1000 * 60 * 60);\n    msec = msec - hr * (1000 * 60 * 60);\n    int64_t min = msec / (1000 * 60);\n    msec = msec - min * (1000 * 60);\n    int64_t sec = msec / 1000;\n    msec = msec - sec * 1000;\n\n    char buf[32];\n    snprintf(buf, sizeof(buf), \"%02d:%02d:%02d%s%03d\", (int) hr, (int) min, (int) sec, comma ? \",\" : \".\", (int) msec);\n\n    return std::string(buf);\n}\n\nint timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate) {\n    return std::max(0, std::min((int) n_samples - 1, (int) ((t*whisper_sample_rate)/100)));\n}\n\nbool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id) {\n    std::ofstream speak_file(path.c_str());\n    if (speak_file.fail()) {\n        fprintf(stderr, \"%s: failed to open speak_file\\n\", __func__);\n        return false;\n    } else {\n        speak_file.write(text.c_str(), text.size());\n        speak_file.close();\n        int ret = system((command + \" \" + std::to_string(voice_id) + \" \" + path).c_str());\n        if (ret != 0) {\n            fprintf(stderr, \"%s: failed to speak\\n\", __func__);\n            return false;\n        }\n    }\n    return true;\n}\n\n#undef STB_VORBIS_HEADER_ONLY\n#include \"stb_vorbis.c\"\n"
  },
  {
    "path": "examples/common-whisper.h",
    "content": "#pragma once\n\n#include <string>\n#include <vector>\n#include <cstdint>\n\n// Read WAV audio file and store the PCM data into pcmf32\n// fname can be a buffer of WAV data instead of a filename\n// The sample rate of the audio must be equal to COMMON_SAMPLE_RATE\n// If stereo flag is set and the audio has 2 channels, the pcmf32s will contain 2 channel PCM\nbool read_audio_data(\n        const std::string & fname,\n        std::vector<float> & pcmf32,\n        std::vector<std::vector<float>> & pcmf32s,\n        bool stereo);\n\n// convert timestamp to string, 6000 -> 01:00.000\nstd::string to_timestamp(int64_t t, bool comma = false);\n\n// given a timestamp get the sample\nint timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate);\n\n// write text to file, and call system(\"command voice_id file\")\nbool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id);\n"
  },
  {
    "path": "examples/common.cpp",
    "content": "#define _USE_MATH_DEFINES // for M_PI\n\n#include \"common.h\"\n\n#include <cmath>\n#include <codecvt>\n#include <cstring>\n#include <fstream>\n#include <locale>\n#include <regex>\n#include <sstream>\n\n// Function to check if the next argument exists\nstatic std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {\n    if (i + 1 < argc && argv[i + 1][0] != '-') {\n        return argv[++i];\n    } else {\n        fprintf(stderr, \"error: %s requires one argument.\\n\", flag.c_str());\n        gpt_print_usage(argc, argv, params);\n        exit(0);\n    }\n}\n\nbool gpt_params_parse(int argc, char ** argv, gpt_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-s\" || arg == \"--seed\") {\n            params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-t\" || arg == \"--threads\") {\n            params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-p\" || arg == \"--prompt\") {\n            params.prompt = get_next_arg(i, argc, argv, arg, params);\n        } else if (arg == \"-n\" || arg == \"--n_predict\") {\n            params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-np\" || arg == \"--n_parallel\") {\n            params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--top_k\") {\n            params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--top_p\") {\n            params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--temp\") {\n            params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--repeat-last-n\") {\n            params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--repeat-penalty\") {\n            params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-b\" || arg == \"--batch_size\") {\n            params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-c\" || arg == \"--context\") {\n            params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-ngl\" || arg == \"--gpu-layers\" || arg == \"--n-gpu-layers\") {\n            params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"--ignore-eos\") {\n            params.ignore_eos = true;\n        } else if (arg == \"-m\" || arg == \"--model\") {\n            params.model = get_next_arg(i, argc, argv, arg, params);\n        } else if (arg == \"-i\" || arg == \"--interactive\") {\n            params.interactive = true;\n        } else if (arg == \"-ip\" || arg == \"--interactive-port\") {\n            params.interactive = true;\n            params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));\n        } else if (arg == \"-h\" || arg == \"--help\") {\n            gpt_print_usage(argc, argv, params);\n            exit(0);\n        } else if (arg == \"-f\" || arg == \"--file\") {\n            get_next_arg(i, argc, argv, arg, params);\n            std::ifstream file(argv[i]);\n            if (!file) {\n                fprintf(stderr, \"error: failed to open file '%s'\\n\", argv[i]);\n                break;\n            }\n            std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));\n            if (params.prompt.back() == '\\n') {\n                params.prompt.pop_back();\n            }\n        } else if (arg == \"-tt\" || arg == \"--token_test\") {\n            params.token_test = get_next_arg(i, argc, argv, arg, params);\n        }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            gpt_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h, --help            show this help message and exit\\n\");\n    fprintf(stderr, \"  -s SEED, --seed SEED  RNG seed (default: -1)\\n\");\n    fprintf(stderr, \"  -t N, --threads N     number of threads to use during computation (default: %d)\\n\", params.n_threads);\n    fprintf(stderr, \"  -p PROMPT, --prompt PROMPT\\n\");\n    fprintf(stderr, \"                        prompt to start generation with (default: random)\\n\");\n    fprintf(stderr, \"  -f FNAME, --file FNAME\\n\");\n    fprintf(stderr, \"                        load prompt from a file\\n\");\n    fprintf(stderr, \"  -tt TOKEN_TEST, --token_test TOKEN_TEST\\n\");\n    fprintf(stderr, \"                        test tokenization\\n\");\n    fprintf(stderr, \"  -n N, --n_predict N   number of tokens to predict (default: %d)\\n\", params.n_predict);\n    fprintf(stderr, \"  --top_k N             top-k sampling (default: %d)\\n\", params.top_k);\n    fprintf(stderr, \"  --top_p N             top-p sampling (default: %.1f)\\n\", params.top_p);\n    fprintf(stderr, \"  --temp N              temperature (default: %.1f)\\n\", params.temp);\n    fprintf(stderr, \"  --repeat-last-n N     last n tokens to consider for penalize (default: %d, 0 = disabled)\\n\", params.repeat_last_n);\n    fprintf(stderr, \"  --repeat-penalty N    penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\\n\", (double)params.repeat_penalty);\n    fprintf(stderr, \"  -b N, --batch_size N  batch size for prompt processing (default: %d)\\n\", params.n_batch);\n    fprintf(stderr, \"  -c N, --context N     context / KV cache size (default: %d)\\n\", params.n_ctx);\n    fprintf(stderr, \"  --ignore-eos          ignore EOS token during generation\\n\");\n    fprintf(stderr, \"  -ngl N, --gpu-layers N  number of layers to offload to GPU on supported models (default: %d)\\n\", params.n_gpu_layers);\n    fprintf(stderr, \"  -m FNAME, --model FNAME\\n\");\n    fprintf(stderr, \"                        model path (default: %s)\\n\", params.model.c_str());\n    fprintf(stderr, \"\\n\");\n}\n\nstd::string gpt_random_prompt(std::mt19937 & rng) {\n    const int r = rng() % 10;\n    switch (r) {\n        case 0: return \"So\";\n        case 1: return \"Once upon a time\";\n        case 2: return \"When\";\n        case 3: return \"The\";\n        case 4: return \"After\";\n        case 5: return \"If\";\n        case 6: return \"import\";\n        case 7: return \"He\";\n        case 8: return \"She\";\n        case 9: return \"They\";\n    }\n\n    return \"The\";\n}\n\nstd::string trim(const std::string & s) {\n    std::regex e(\"^\\\\s+|\\\\s+$\");\n    return std::regex_replace(s, e, \"\");\n}\n\nstd::string replace(const std::string & s, const std::string & from, const std::string & to) {\n    std::string result = s;\n    size_t pos = 0;\n    while ((pos = result.find(from, pos)) != std::string::npos) {\n        result.replace(pos, from.length(), to);\n        pos += to.length();\n    }\n    return result;\n}\n\nvoid gpt_vocab::add_special_token(const std::string & token) {\n    special_tokens.push_back(token);\n}\n\nstd::map<std::string, int32_t> json_parse(const std::string & fname) {\n    std::map<std::string, int32_t> result;\n\n    // read file into string\n    std::string json;\n    {\n        std::ifstream ifs(fname);\n        if (!ifs) {\n            fprintf(stderr, \"Failed to open %s\\n\", fname.c_str());\n            exit(1);\n        }\n\n        json = std::string((std::istreambuf_iterator<char>(ifs)),\n                (std::istreambuf_iterator<char>()));\n    }\n\n    if (json[0] != '{') {\n        return result;\n    }\n\n    // parse json\n    {\n        bool has_key  = false;\n        bool in_token = false;\n\n        std::string str_key = \"\";\n        std::string str_val = \"\";\n\n        int n = json.size();\n        for (int i = 1; i < n; ++i) {\n            if (!in_token) {\n                if (json[i] == ' ') continue;\n                if (json[i] == '\"') {\n                    in_token = true;\n                    continue;\n                }\n            } else {\n                if (json[i] == '\\\\' && i+1 < n) {\n                    if (has_key == false) {\n                        str_key += json[i];\n                    } else {\n                        str_val += json[i];\n                    }\n                    ++i;\n                } else if (json[i] == '\"') {\n                    if (has_key == false) {\n                        has_key = true;\n                        ++i;\n                        while (json[i] == ' ') ++i;\n                        ++i; // :\n                        while (json[i] == ' ') ++i;\n                        if (json[i] != '\\\"') {\n                            while (json[i] != ',' && json[i] != '}') {\n                                str_val += json[i++];\n                            }\n                            has_key = false;\n                        } else {\n                            in_token = true;\n                            continue;\n                        }\n                    } else {\n                        has_key = false;\n                    }\n\n                    str_key = ::replace(str_key, \"\\\\u0120\", \" \" ); // \\u0120 -> space\n                    str_key = ::replace(str_key, \"\\\\u010a\", \"\\n\"); // \\u010a -> new line\n                    str_key = ::replace(str_key, \"\\\\\\\"\",    \"\\\"\"); // \\\\\\\"   -> \"\n\n                    try {\n                        result[str_key] = std::stoi(str_val);\n                    } catch (...) {\n                        //fprintf(stderr, \"%s: ignoring key '%s' with value '%s'\\n\", fname.c_str(), str_key.c_str(), str_val.c_str());\n\n                    }\n                    str_key = \"\";\n                    str_val = \"\";\n                    in_token = false;\n                    continue;\n                }\n                if (has_key == false) {\n                    str_key += json[i];\n                } else {\n                    str_val += json[i];\n                }\n            }\n        }\n    }\n\n    return result;\n}\n\nvoid gpt_split_words(std::string str, std::vector<std::string>& words) {\n    const std::string pattern = R\"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\\s[:alpha:][:digit:]]+|\\s+(?!\\S)|\\s+)\";\n    const std::regex re(pattern);\n    std::smatch m;\n\n    while (std::regex_search(str, m, re)) {\n        for (auto x : m) {\n            words.push_back(x);\n        }\n        str = m.suffix();\n    }\n}\n\nstd::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {\n    std::vector<std::string> words;\n\n    // first split the text into words\n    {\n        std::string str = text;\n\n        // Generate the subpattern from the special_tokens vector if it's not empty\n        if (!vocab.special_tokens.empty()) {\n            const std::regex escape(R\"([\\[\\\\\\^\\$\\.\\|\\?\\*\\+\\(\\)\\{\\}])\");\n            std::string special_tokens_subpattern;\n            for (const auto & token : vocab.special_tokens) {\n                if (!special_tokens_subpattern.empty()) {\n                    special_tokens_subpattern += \"|\";\n                }\n                special_tokens_subpattern += std::regex_replace(token, escape, R\"(\\$&)\");\n            }\n\n            std::regex re(special_tokens_subpattern);\n            std::smatch m;\n            // Split the text by special tokens.\n            while (std::regex_search(str, m, re)) {\n                // Split the substrings in-between special tokens into words.\n                gpt_split_words(m.prefix(), words);\n                // Add matched special tokens as words.\n                for (auto x : m) {\n                    words.push_back(x);\n                }\n                str = m.suffix();\n            }\n            // Remaining text without special tokens will be handled below.\n        }\n\n        gpt_split_words(str, words);\n    }\n\n    // find the longest token that forms each word in words:\n    std::vector<gpt_vocab::id> tokens;\n    for (const auto & word : words) {\n        for (int i = 0; i < (int) word.size(); ){\n            for (int j = word.size() - 1; j >= i; j--){\n                auto cand = word.substr(i, j-i+1);\n                auto it = vocab.token_to_id.find(cand);\n                if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab\n                    tokens.push_back(it->second);\n                    i = j + 1;\n                    break;\n                }\n                else if (j == i){ // word.substr(i, 1) has no matching\n                    fprintf(stderr, \"%s: unknown token '%s'\\n\", __func__, word.substr(i, 1).data());\n                    i++;\n                }\n            }\n        }\n    }\n\n    return tokens;\n}\n\nstatic std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {\n    std::vector<gpt_vocab::id> output;\n    std::stringstream ss(input);\n    std::string token;\n\n    while (std::getline(ss, token, delimiter)) {\n        output.push_back(std::stoi(token));\n    }\n\n    return output;\n}\n\nstatic std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){\n    if (fpath_test.empty()){\n        fprintf(stderr, \"%s : No test file found.\\n\", __func__);\n        return std::map<std::string, std::vector<gpt_vocab::id>>();\n    }\n\n    std::map<std::string, std::vector<gpt_vocab::id>> tests;\n\n    auto fin = std::ifstream(fpath_test, std::ios_base::in);\n    const char * delimeter = \" => \";\n    const char del_tok = ',';\n    std::string line;\n    while (std::getline(fin, line)) {\n        size_t delimiterPos = line.find(delimeter);\n        if (delimiterPos != std::string::npos) {\n            std::string text = line.substr(0, delimiterPos);\n            std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));\n            tests[text] = parse_tokens_from_string(s_tokens, del_tok);\n        }\n    }\n    return tests;\n}\n\nvoid test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){\n    std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);\n\n    size_t n_fails = 0;\n\n    for (const auto & test : tests) {\n        std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);\n\n        if (tokens != test.second){\n            n_fails++;\n\n            // print out failure cases\n            fprintf(stderr, \"%s : failed test: '%s'\\n\", __func__, test.first.c_str());\n            fprintf(stderr, \"%s : tokens in hf:   \", __func__);\n            for (const auto & t : test.second) {\n                fprintf(stderr, \"%s(%d), \", vocab.id_to_token[t].c_str(), t);\n            }\n            fprintf(stderr, \"\\n\");\n            fprintf(stderr, \"%s : tokens in ggml: \", __func__);\n            for (const auto & t : tokens) {\n                fprintf(stderr, \"%s(%d), \", vocab.id_to_token[t].c_str(), t);\n            }\n            fprintf(stderr, \"\\n\");\n        }\n    }\n\n    fprintf(stderr, \"%s : %zu tests failed out of %zu tests.\\n\", __func__, n_fails, tests.size());\n}\n\nbool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {\n    printf(\"%s: loading vocab from '%s'\\n\", __func__, fname.c_str());\n\n    vocab.token_to_id = ::json_parse(fname);\n\n    for (const auto & kv : vocab.token_to_id) {\n        vocab.id_to_token[kv.second] = kv.first;\n    }\n\n    printf(\"%s: vocab size = %d\\n\", __func__, (int) vocab.token_to_id.size());\n\n    // print the vocabulary\n    //for (auto kv : vocab.token_to_id) {\n    //    printf(\"'%s' -> %d\\n\", kv.first.data(), kv.second);\n    //}\n\n    return true;\n}\n\ngpt_vocab::id gpt_sample_top_k_top_p(\n        const gpt_vocab & vocab,\n        const float * logits,\n        int    top_k,\n        double top_p,\n        double temp,\n        std::mt19937 & rng) {\n    int n_logits = vocab.id_to_token.size();\n\n    std::vector<std::pair<double, gpt_vocab::id>> logits_id;\n    logits_id.reserve(n_logits);\n\n    {\n        const double scale = 1.0/temp;\n        for (int i = 0; i < n_logits; ++i) {\n            logits_id.push_back(std::make_pair(logits[i]*scale, i));\n        }\n    }\n\n    // find the top K tokens\n    std::partial_sort(\n            logits_id.begin(),\n            logits_id.begin() + top_k, logits_id.end(),\n            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {\n        return a.first > b.first;\n    });\n\n    logits_id.resize(top_k);\n\n    double maxl = -INFINITY;\n    for (const auto & kv : logits_id) {\n        maxl = std::max(maxl, kv.first);\n    }\n\n    // compute probs for the top K tokens\n    std::vector<double> probs;\n    probs.reserve(logits_id.size());\n\n    double sum = 0.0;\n    for (const auto & kv : logits_id) {\n        double p = exp(kv.first - maxl);\n        probs.push_back(p);\n        sum += p;\n    }\n\n    // normalize the probs\n    for (auto & p : probs) {\n        p /= sum;\n    }\n\n    if (top_p < 1.0f) {\n        double cumsum = 0.0f;\n        for (int i = 0; i < top_k; i++) {\n            cumsum += probs[i];\n            if (cumsum >= top_p) {\n                top_k = i + 1;\n                probs.resize(top_k);\n                logits_id.resize(top_k);\n                break;\n            }\n        }\n\n        cumsum = 1.0/cumsum;\n        for (int i = 0; i < (int) probs.size(); i++) {\n            probs[i] *= cumsum;\n        }\n    }\n\n    //printf(\"\\n\");\n    //for (int i = 0; i < (int) probs.size(); i++) {\n    //    printf(\"%d: '%s' %f\\n\", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);\n    //}\n    //exit(0);\n\n    std::discrete_distribution<> dist(probs.begin(), probs.end());\n    int idx = dist(rng);\n\n    return logits_id[idx].second;\n}\n\ngpt_vocab::id gpt_sample_top_k_top_p_repeat(\n        const gpt_vocab & vocab,\n        const float * logits,\n        const int32_t * last_n_tokens_data,\n        size_t last_n_tokens_data_size,\n        int    top_k,\n        double top_p,\n        double temp,\n        int repeat_last_n,\n        float repeat_penalty,\n        std::mt19937 & rng) {\n\n    int n_logits = vocab.id_to_token.size();\n\n    const auto * plogits = logits;\n\n    const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);\n\n    if (temp <= 0) {\n        // select the token with the highest logit directly\n        float max_logit = plogits[0];\n        gpt_vocab::id max_id = 0;\n\n        for (int i = 1; i < n_logits; ++i) {\n            if (plogits[i] > max_logit) {\n                max_logit = plogits[i];\n                max_id = i;\n            }\n        }\n        return max_id;\n    }\n\n\n    std::vector<std::pair<double, gpt_vocab::id>> logits_id;\n    logits_id.reserve(n_logits);\n\n    {\n        const float scale = 1.0f/temp;\n        for (int i = 0; i < n_logits; ++i) {\n            // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)\n            // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main\n            if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {\n                // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability\n                if (plogits[i] < 0.0f) {\n                    logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));\n                } else {\n                    logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));\n                }\n            } else {\n                logits_id.push_back(std::make_pair(plogits[i]*scale, i));\n            }\n        }\n    }\n\n    // find the top K tokens\n    std::partial_sort(\n            logits_id.begin(),\n            logits_id.begin() + top_k, logits_id.end(),\n            [](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {\n        return a.first > b.first;\n    });\n\n    logits_id.resize(top_k);\n\n    double maxl = -INFINITY;\n    for (const auto & kv : logits_id) {\n        maxl = std::max(maxl, kv.first);\n    }\n\n    // compute probs for the top K tokens\n    std::vector<double> probs;\n    probs.reserve(logits_id.size());\n\n    double sum = 0.0;\n    for (const auto & kv : logits_id) {\n        double p = exp(kv.first - maxl);\n        probs.push_back(p);\n        sum += p;\n    }\n\n    // normalize the probs\n    for (auto & p : probs) {\n        p /= sum;\n    }\n\n    if (top_p < 1.0f) {\n        double cumsum = 0.0f;\n        for (int i = 0; i < top_k; i++) {\n            cumsum += probs[i];\n            if (cumsum >= top_p) {\n                top_k = i + 1;\n                probs.resize(top_k);\n                logits_id.resize(top_k);\n                break;\n            }\n        }\n\n        cumsum = 1.0/cumsum;\n        for (int i = 0; i < (int) probs.size(); i++) {\n            probs[i] *= cumsum;\n        }\n    }\n\n//    printf(\"\\n\");\n//    for (int i = 0; i < (int) probs.size(); i++) {\n//    for (int i = 0; i < 10; i++) {\n//        printf(\"%d: '%s' %f\\n\", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);\n//    }\n\n    std::discrete_distribution<> dist(probs.begin(), probs.end());\n    int idx = dist(rng);\n\n    return logits_id[idx].second;\n\n}\n\nvoid high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {\n    const float rc = 1.0f / (2.0f * M_PI * cutoff);\n    const float dt = 1.0f / sample_rate;\n    const float alpha = dt / (rc + dt);\n\n    float y = data[0];\n\n    for (size_t i = 1; i < data.size(); i++) {\n        y = alpha * (y + data[i] - data[i - 1]);\n        data[i] = y;\n    }\n}\n\nbool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {\n    const int n_samples      = pcmf32.size();\n    const int n_samples_last = (sample_rate * last_ms) / 1000;\n\n    if (n_samples_last >= n_samples) {\n        // not enough samples - assume no speech\n        return false;\n    }\n\n    if (freq_thold > 0.0f) {\n        high_pass_filter(pcmf32, freq_thold, sample_rate);\n    }\n\n    float energy_all  = 0.0f;\n    float energy_last = 0.0f;\n\n    for (int i = 0; i < n_samples; i++) {\n        energy_all += fabsf(pcmf32[i]);\n\n        if (i >= n_samples - n_samples_last) {\n            energy_last += fabsf(pcmf32[i]);\n        }\n    }\n\n    energy_all  /= n_samples;\n    energy_last /= n_samples_last;\n\n    if (verbose) {\n        fprintf(stderr, \"%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\\n\", __func__, energy_all, energy_last, vad_thold, freq_thold);\n    }\n\n    if (energy_last > vad_thold*energy_all) {\n        return false;\n    }\n\n    return true;\n}\n\nfloat similarity(const std::string & s0, const std::string & s1) {\n    const size_t len0 = s0.size() + 1;\n    const size_t len1 = s1.size() + 1;\n\n    std::vector<int> col(len1, 0);\n    std::vector<int> prevCol(len1, 0);\n\n    for (size_t i = 0; i < len1; i++) {\n        prevCol[i] = i;\n    }\n\n    for (size_t i = 0; i < len0; i++) {\n        col[0] = i;\n        for (size_t j = 1; j < len1; j++) {\n            col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));\n        }\n        col.swap(prevCol);\n    }\n\n    const float dist = prevCol[len1 - 1];\n\n    return 1.0f - (dist / std::max(s0.size(), s1.size()));\n}\n\nbool is_file_exist(const char * filename) {\n    std::ifstream infile(filename);\n    return infile.good();\n}\n"
  },
  {
    "path": "examples/common.h",
    "content": "// Various helper functions and utilities\n\n#pragma once\n\n#include <string>\n#include <map>\n#include <vector>\n#include <random>\n#include <thread>\n#include <ctime>\n#include <fstream>\n#include <sstream>\n\n//\n// GPT CLI argument parsing\n//\n\nstruct gpt_params {\n    int32_t seed         = -1;   // RNG seed\n    int32_t n_threads    = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t n_predict    = 200;  // new tokens to predict\n    int32_t n_parallel   = 1;    // number of parallel streams\n    int32_t n_batch      = 32;   // batch size for prompt processing\n    int32_t n_ctx        = 2048; // context size (this is the KV cache max size)\n    int32_t n_gpu_layers = 0;    // number of layers to offlload to the GPU\n\n    bool ignore_eos = false; // ignore EOS token when generating text\n\n    // sampling parameters\n    int32_t top_k          = 40;\n    float   top_p          = 0.9f;\n    float   temp           = 0.9f;\n    int32_t repeat_last_n  = 64;\n    float   repeat_penalty = 1.00f;\n\n    std::string model      = \"models/gpt-2-117M/ggml-model.bin\"; // model path\n    std::string prompt     = \"\";\n    std::string token_test = \"\";\n\n    bool    interactive      = false;\n    int32_t interactive_port = -1;\n};\n\nbool gpt_params_parse(int argc, char ** argv, gpt_params & params);\n\nvoid gpt_print_usage(int argc, char ** argv, const gpt_params & params);\n\nstd::string gpt_random_prompt(std::mt19937 & rng);\n\n//\n// Vocab utils\n//\n\nstd::string trim(const std::string & s);\n\nstd::string replace(\n        const std::string & s,\n        const std::string & from,\n        const std::string & to);\n\nstruct gpt_vocab {\n    using id    = int32_t;\n    using token = std::string;\n\n    std::map<token, id> token_to_id;\n    std::map<id, token> id_to_token;\n    std::vector<std::string> special_tokens;\n\n    void add_special_token(const std::string & token);\n};\n\n// poor-man's JSON parsing\nstd::map<std::string, int32_t> json_parse(const std::string & fname);\n\nstd::string convert_to_utf8(const std::wstring & input);\n\nstd::wstring convert_to_wstring(const std::string & input);\n\nvoid gpt_split_words(std::string str, std::vector<std::string>& words);\n\n// split text into tokens\n//\n// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53\n//\n// Regex (Python):\n// r\"\"\"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\"\"\"\n//\n// Regex (C++):\n// R\"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\\s[:alpha:][:digit:]]+|\\s+(?!\\S)|\\s+)\"\n//\nstd::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);\n\n// test outputs of gpt_tokenize\n//\n//   - compare with tokens generated by the huggingface tokenizer\n//   - test cases are chosen based on the model's main language (under 'prompt' directory)\n//   - if all sentences are tokenized identically, print 'All tests passed.'\n//   - otherwise, print sentence, huggingface tokens, ggml tokens\n//\nvoid test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);\n\n// load the tokens from encoder.json\nbool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);\n\n// sample next token given probabilities for each embedding\n//\n//   - consider only the top K tokens\n//   - from them, consider only the top tokens with cumulative probability > P\n//\n// TODO: not sure if this implementation is correct\n// TODO: temperature is not implemented\n//\ngpt_vocab::id gpt_sample_top_k_top_p(\n        const gpt_vocab & vocab,\n        const float * logits,\n        int    top_k,\n        double top_p,\n        double temp,\n        std::mt19937 & rng);\n\ngpt_vocab::id gpt_sample_top_k_top_p_repeat(\n        const gpt_vocab & vocab,\n        const float * logits,\n        const int32_t * last_n_tokens_data,\n        size_t last_n_tokens_data_size,\n        int    top_k,\n        double top_p,\n        double temp,\n        int repeat_last_n,\n        float repeat_penalty,\n        std::mt19937 & rng);\n\n//\n// Audio utils\n//\n\n// Write PCM data into WAV audio file\nclass wav_writer {\nprivate:\n    std::ofstream file;\n    uint32_t dataSize = 0;\n    std::string wav_filename;\n\n    bool write_header(const uint32_t sample_rate,\n                      const uint16_t bits_per_sample,\n                      const uint16_t channels) {\n\n        file.write(\"RIFF\", 4);\n        file.write(\"\\0\\0\\0\\0\", 4);    // Placeholder for file size\n        file.write(\"WAVE\", 4);\n        file.write(\"fmt \", 4);\n\n        const uint32_t sub_chunk_size = 16;\n        const uint16_t audio_format = 1;      // PCM format\n        const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;\n        const uint16_t block_align = channels * bits_per_sample / 8;\n\n        file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);\n        file.write(reinterpret_cast<const char *>(&audio_format), 2);\n        file.write(reinterpret_cast<const char *>(&channels), 2);\n        file.write(reinterpret_cast<const char *>(&sample_rate), 4);\n        file.write(reinterpret_cast<const char *>(&byte_rate), 4);\n        file.write(reinterpret_cast<const char *>(&block_align), 2);\n        file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);\n        file.write(\"data\", 4);\n        file.write(\"\\0\\0\\0\\0\", 4);    // Placeholder for data size\n\n        return true;\n    }\n\n    // It is assumed that PCM data is normalized to a range from -1 to 1\n    bool write_audio(const float * data, size_t length) {\n        for (size_t i = 0; i < length; ++i) {\n            const int16_t intSample = int16_t(data[i] * 32767);\n            file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));\n            dataSize += sizeof(int16_t);\n        }\n        if (file.is_open()) {\n            file.seekp(4, std::ios::beg);\n            uint32_t fileSize = 36 + dataSize;\n            file.write(reinterpret_cast<char *>(&fileSize), 4);\n            file.seekp(40, std::ios::beg);\n            file.write(reinterpret_cast<char *>(&dataSize), 4);\n            file.seekp(0, std::ios::end);\n        }\n        return true;\n    }\n\n    bool open_wav(const std::string & filename) {\n        if (filename != wav_filename) {\n            if (file.is_open()) {\n                file.close();\n            }\n        }\n        if (!file.is_open()) {\n            file.open(filename, std::ios::binary);\n            wav_filename = filename;\n            dataSize = 0;\n        }\n        return file.is_open();\n    }\n\npublic:\n    bool open(const std::string & filename,\n              const    uint32_t   sample_rate,\n              const    uint16_t   bits_per_sample,\n              const    uint16_t   channels) {\n\n        if (open_wav(filename)) {\n            write_header(sample_rate, bits_per_sample, channels);\n        } else {\n            return false;\n        }\n\n        return true;\n    }\n\n    bool close() {\n        file.close();\n        return true;\n    }\n\n    bool write(const float * data, size_t length) {\n        return write_audio(data, length);\n    }\n\n    ~wav_writer() {\n        if (file.is_open()) {\n            file.close();\n        }\n    }\n};\n\n\n// Apply a high-pass frequency filter to PCM audio\n// Suppresses frequencies below cutoff Hz\nvoid high_pass_filter(\n        std::vector<float> & data,\n        float cutoff,\n        float sample_rate);\n\n// Basic voice activity detection (VAD) using audio energy adaptive threshold\nbool vad_simple(\n        std::vector<float> & pcmf32,\n        int   sample_rate,\n        int   last_ms,\n        float vad_thold,\n        float freq_thold,\n        bool  verbose);\n\n// compute similarity between two strings using Levenshtein distance\nfloat similarity(const std::string & s0, const std::string & s1);\n\n//\n// Terminal utils\n//\n\n#define SQR(X)    ((X) * (X))\n#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40\n\n/**\n * Quantizes 24-bit RGB to xterm256 code range [16,256).\n */\nstatic int rgb2xterm256(int r, int g, int b) {\n    unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};\n    int av, ir, ig, ib, il, qr, qg, qb, ql;\n    av = r * .299 + g * .587 + b * .114 + .5;\n    ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;\n    qr = cube[(ir = UNCUBE(r))];\n    qg = cube[(ig = UNCUBE(g))];\n    qb = cube[(ib = UNCUBE(b))];\n    if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=\n        SQR(ql - r) + SQR(ql - g) + SQR(ql - b))\n        return ir * 36 + ig * 6 + ib + 020;\n    return il + 0350;\n}\n\nstatic std::string set_xterm256_foreground(int r, int g, int b) {\n    int x = rgb2xterm256(r, g, b);\n    std::ostringstream oss;\n    oss << \"\\033[38;5;\" << x << \"m\";\n    return oss.str();\n}\n\n// Lowest is red, middle is yellow, highest is green. Color scheme from\n// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault\nconst std::vector<std::string> k_colors = {\n    set_xterm256_foreground(220,   5,  12),\n    set_xterm256_foreground(232,  96,  28),\n    set_xterm256_foreground(241, 147,  45),\n    set_xterm256_foreground(246, 193,  65),\n    set_xterm256_foreground(247, 240,  86),\n    set_xterm256_foreground(144, 201, 135),\n    set_xterm256_foreground( 78, 178, 101),\n};\n\n// ANSI formatting codes\nstatic std::string set_inverse() {\n    return \"\\033[7m\";\n}\n\nstatic std::string set_underline() {\n    return \"\\033[4m\";\n}\n\nstatic std::string set_dim() {\n    return \"\\033[2m\";\n}\n\n// Style scheme for different confidence levels\nconst std::vector<std::string> k_styles = {\n    set_inverse(),   // Low confidence - inverse (highlighted)\n    set_underline(), // Medium confidence - underlined\n    set_dim(),       // High confidence - dim\n};\n\n//\n// Other utils\n//\n\n// check if file exists using ifstream\nbool is_file_exist(const char * filename);\n"
  },
  {
    "path": "examples/deprecation-warning/CMakeLists.txt",
    "content": "add_executable(main ./deprecation-warning.cpp)\nadd_executable(bench ./deprecation-warning.cpp)\nif (WHISPER_SDL2)\n    add_executable(stream ./deprecation-warning.cpp)\n    add_executable(command ./deprecation-warning.cpp)\nendif()\n"
  },
  {
    "path": "examples/deprecation-warning/README.md",
    "content": "# Migration notice for binary filenames\n\n> [!IMPORTANT]\n[2024 Dec 20] Binaries have been renamed w/ a `whisper-` prefix. `main` is now `whisper-cli`, `server` is `whisper-server`, etc (https://github.com/ggerganov/whisper.cpp/pull/2648)\n\nThis migration was important, but it is a breaking change that may not always be immediately obvious to users.\n\nPlease update all scripts and workflows to use the new binary names.\n\n| Old Filename | New Filename |\n| ---- | ---- |\n| main | whisper-cli |\n| bench | whisper-bench |\n| stream | whisper-stream |\n| command | whisper-command |\n| server | whisper-server |\n| talk-llama | whisper-talk-llama |\n"
  },
  {
    "path": "examples/deprecation-warning/deprecation-warning.cpp",
    "content": "// Warns users that this filename was deprecated, and provides a link for more information.\n\n#include <cstdio>\n#include <string>\n\n// Main\nint main(int argc, char** argv) {\n    std::string filename = \"main\";\n    if (argc >= 1) {\n        filename = argv[0];\n    }\n\n    // Get only the program name from the full path\n    size_t pos = filename.find_last_of(\"/\\\\\");\n    if (pos != std::string::npos) {\n        filename = filename.substr(pos+1);\n    }\n\n    // Append \"whisper-\" to the beginning of filename to get the replacemnt filename\n    std::string replacement_filename = \"whisper-\" + filename;\n\n    // The exception is if the filename is \"main\", then our replacement filename is \"whisper-cli\"\n    if (filename == \"main\") {\n        replacement_filename = \"whisper-cli\";\n    }\n\n    if (filename == \"main.exe\") {\n        replacement_filename = \"whisper-cli.exe\";\n    }\n\n    fprintf(stdout, \"\\n\");\n    fprintf(stdout, \"WARNING: The binary '%s' is deprecated.\\n\", filename.c_str());\n    fprintf(stdout, \" Please use '%s' instead.\\n\", replacement_filename.c_str());\n    fprintf(stdout, \" See https://github.com/ggerganov/whisper.cpp/tree/master/examples/deprecation-warning/README.md for more information.\\n\");\n    fprintf(stdout, \"\\n\");\n\n    return EXIT_FAILURE;\n}\n"
  },
  {
    "path": "examples/ffmpeg-transcode.cpp",
    "content": "/* SPDX-License-Identifier: GPL-2.0 */\n\n/*\n * transcode.c - convert audio file to WAVE\n *\n * Copyright (C) 2019\t\tAndrew Clayton <andrew@digital-domain.net>\n * Copyright (C) 2024       William Tambellini <william.tambellini@gmail.com>\n */\n\n// Just for conveninent C++ API\n#include <vector>\n#include <string>\n\n// C\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <stdbool.h>\n#include <stdint.h>\n#include <sys/types.h>\n#include <sys/stat.h>\n#include <fcntl.h>\n#include <unistd.h>\n#include <sys/mman.h>\n\nextern \"C\" {\n#include <libavutil/opt.h>\n#include <libavcodec/avcodec.h>\n#include <libavformat/avformat.h>\n#include <libswresample/swresample.h>\n}\n\ntypedef uint64_t u64;\ntypedef int64_t  s64;\ntypedef uint32_t u32;\ntypedef int32_t  s32;\ntypedef uint16_t u16;\ntypedef int16_t  s16;\ntypedef uint8_t   u8;\ntypedef int8_t    s8;\n\n#define WAVE_SAMPLE_RATE\t16000\n#define AVIO_CTX_BUF_SZ\t\t 4096\n\nstatic const char* ffmpegLog = getenv(\"FFMPEG_LOG\");\n// Todo: add __FILE__ __LINE__\n#define LOG(...) \\\n  do { if (ffmpegLog) fprintf(stderr, __VA_ARGS__); } while(0) // C99\n\n/*\n * WAVE file header based on definition from\n * https://gist.github.com/Jon-Schneider/8b7c53d27a7a13346a643dac9c19d34f\n *\n * We must ensure this structure doesn't have any holes or\n * padding so we can just map it straight to the WAVE data.\n */\nstruct wave_hdr {\n\t/* RIFF Header: \"RIFF\" */\n\tchar riff_header[4];\n\t/* size of audio data + sizeof(struct wave_hdr) - 8 */\n\tint wav_size;\n\t/* \"WAVE\" */\n\tchar wav_header[4];\n\n\t/* Format Header */\n\t/* \"fmt \" (includes trailing space) */\n\tchar fmt_header[4];\n\t/* Should be 16 for PCM */\n\tint fmt_chunk_size;\n\t/* Should be 1 for PCM. 3 for IEEE Float */\n\ts16 audio_format;\n\ts16 num_channels;\n\tint sample_rate;\n\t/*\n\t * Number of bytes per second\n\t * sample_rate * num_channels * bit_depth/8\n\t */\n\tint byte_rate;\n\t/* num_channels * bytes per sample */\n\ts16 sample_alignment;\n\t/* bits per sample */\n\ts16 bit_depth;\n\n\t/* Data Header */\n\t/* \"data\" */\n\tchar data_header[4];\n\t/*\n\t * size of audio\n\t * number of samples * num_channels * bit_depth/8\n\t */\n\tint data_bytes;\n} __attribute__((__packed__));\n\nstruct audio_buffer {\n\tu8 *ptr;\n\tint size; /* size left in the buffer */\n};\n\nstatic void set_wave_hdr(wave_hdr& wh, size_t size) {\n    memcpy(&wh.riff_header, \"RIFF\", 4);\n    wh.wav_size = size + sizeof(struct wave_hdr) - 8;\n    memcpy(&wh.wav_header, \"WAVE\", 4);\n    memcpy(&wh.fmt_header, \"fmt \", 4);\n    wh.fmt_chunk_size = 16;\n    wh.audio_format = 1;\n    wh.num_channels = 1;\n    wh.sample_rate = WAVE_SAMPLE_RATE;\n    wh.sample_alignment = 2;\n    wh.bit_depth = 16;\n    wh.byte_rate = wh.sample_rate * wh.sample_alignment;\n    memcpy(&wh.data_header, \"data\", 4);\n    wh.data_bytes = size;\n}\n\nstatic void write_wave_hdr(int fd, size_t size) {\n\tstruct wave_hdr wh;\n    set_wave_hdr(wh, size);\n\twrite(fd, &wh, sizeof(struct wave_hdr));\n}\n\nstatic int map_file(int fd, u8 **ptr, size_t *size)\n{\n\tstruct stat sb;\n\n\tfstat(fd, &sb);\n\t*size = sb.st_size;\n\n    *ptr = (u8*)mmap(NULL, *size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);\n\tif (*ptr == MAP_FAILED) {\n\t\tperror(\"mmap\");\n\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nstatic int read_packet(void *opaque, u8 *buf, int buf_size)\n{\n    struct audio_buffer *audio_buf = (audio_buffer*)opaque;\n\n\tbuf_size = FFMIN(buf_size, audio_buf->size);\n\n\t/* copy internal buffer data to buf */\n\tmemcpy(buf, audio_buf->ptr, buf_size);\n\taudio_buf->ptr += buf_size;\n\taudio_buf->size -= buf_size;\n\n\treturn buf_size;\n}\n\nstatic void convert_frame(struct SwrContext *swr, AVCodecContext *codec,\n\t\t\t  AVFrame *frame, s16 **data, int *size, bool flush)\n{\n\tint nr_samples;\n\ts64 delay;\n\tu8 *buffer;\n\n\tdelay = swr_get_delay(swr, codec->sample_rate);\n\tnr_samples = av_rescale_rnd(delay + frame->nb_samples,\n\t\t\t\t    WAVE_SAMPLE_RATE, codec->sample_rate,\n\t\t\t\t    AV_ROUND_UP);\n\tav_samples_alloc(&buffer, NULL, 1, nr_samples, AV_SAMPLE_FMT_S16, 0);\n\n\t/*\n\t * !flush is used to check if we are flushing any remaining\n\t * conversion buffers...\n\t */\n\tnr_samples = swr_convert(swr, &buffer, nr_samples,\n\t\t\t\t !flush ? (const u8 **)frame->data : NULL,\n\t\t\t\t !flush ? frame->nb_samples : 0);\n\n    *data = (s16*)realloc(*data, (*size + nr_samples) * sizeof(s16));\n\tmemcpy(*data + *size, buffer, nr_samples * sizeof(s16));\n\t*size += nr_samples;\n\tav_freep(&buffer);\n}\n\nstatic bool is_audio_stream(const AVStream *stream)\n{\n\tif (stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)\n\t\treturn true;\n\n\treturn false;\n}\n\n// Return non zero on error, 0 on success\n// audio_buffer: input memory\n// data: decoded output audio data (wav file)\n// size: size of output data\nstatic int decode_audio(struct audio_buffer *audio_buf, s16 **data, int *size)\n{\n    LOG(\"decode_audio: input size: %d\\n\", audio_buf->size);\n\tAVFormatContext *fmt_ctx;\n\tAVIOContext *avio_ctx;\n\tAVStream *stream;\n\tAVCodecContext *codec;\n\tAVPacket *packet;\n\tAVFrame *frame;\n\tstruct SwrContext *swr;\n\tu8 *avio_ctx_buffer;\n\tunsigned int i;\n\tint stream_index = -1;\n\tint err;\n    const size_t errbuffsize = 1024;\n    char errbuff[errbuffsize];\n\n    fmt_ctx = avformat_alloc_context();\n    avio_ctx_buffer = (u8*)av_malloc(AVIO_CTX_BUF_SZ);\n    LOG(\"Creating an avio context: AVIO_CTX_BUF_SZ=%d\\n\", AVIO_CTX_BUF_SZ);\n    avio_ctx = avio_alloc_context(avio_ctx_buffer, AVIO_CTX_BUF_SZ, 0, audio_buf, &read_packet, NULL, NULL);\n\tfmt_ctx->pb = avio_ctx;\n\n    // open the input stream and read header\n\terr = avformat_open_input(&fmt_ctx, NULL, NULL, NULL);\n\tif (err) {\n        LOG(\"Could not read audio buffer: %d: %s\\n\", err, av_make_error_string(errbuff, errbuffsize, err));\n        return err;\n\t}\n\n\terr = avformat_find_stream_info(fmt_ctx, NULL);\n\tif (err < 0) {\n        LOG(\"Could not retrieve stream info from audio buffer: %d\\n\", err);\n        return err;\n\t}\n\n\tfor (i = 0; i < fmt_ctx->nb_streams; i++) {\n\t\tif (is_audio_stream(fmt_ctx->streams[i])) {\n\t\t\tstream_index = i;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (stream_index == -1) {\n        LOG(\"Could not retrieve audio stream from buffer\\n\");\n\t\treturn -1;\n\t}\n\n\tstream = fmt_ctx->streams[stream_index];\n\tcodec = avcodec_alloc_context3(\n\t\t\tavcodec_find_decoder(stream->codecpar->codec_id));\n\tavcodec_parameters_to_context(codec, stream->codecpar);\n\terr = avcodec_open2(codec, avcodec_find_decoder(codec->codec_id),\n\t\t\t\t\t\t\tNULL);\n\tif (err) {\n        LOG(\"Failed to open decoder for stream #%d in audio buffer\\n\", stream_index);\n        return err;\n\t}\n\n\t/* prepare resampler */\n\tswr = swr_alloc();\n\n#if LIBAVCODEC_VERSION_MAJOR > 60\n\tAVChannelLayout in_ch_layout = codec->ch_layout;\n\tAVChannelLayout out_ch_layout = AV_CHANNEL_LAYOUT_MONO;\n\n\t/* Set the source audio layout as-is */\n\tav_opt_set_chlayout(swr, \"in_chlayout\", &in_ch_layout, 0);\n\tav_opt_set_int(swr, \"in_sample_rate\", codec->sample_rate, 0);\n\tav_opt_set_sample_fmt(swr, \"in_sample_fmt\", codec->sample_fmt, 0);\n\n\t/* Convert it into 16khz Mono */\n\tav_opt_set_chlayout(swr, \"out_chlayout\", &out_ch_layout, 0);\n\tav_opt_set_int(swr, \"out_sample_rate\", WAVE_SAMPLE_RATE, 0);\n\tav_opt_set_sample_fmt(swr, \"out_sample_fmt\", AV_SAMPLE_FMT_S16, 0);\n#else\n\tav_opt_set_int(swr, \"in_channel_count\", codec->channels, 0);\n\tav_opt_set_int(swr, \"out_channel_count\", 1, 0);\n\tav_opt_set_int(swr, \"in_channel_layout\", codec->channel_layout, 0);\n\tav_opt_set_int(swr, \"out_channel_layout\", AV_CH_LAYOUT_MONO, 0);\n\tav_opt_set_int(swr, \"in_sample_rate\", codec->sample_rate, 0);\n\tav_opt_set_int(swr, \"out_sample_rate\", WAVE_SAMPLE_RATE, 0);\n\tav_opt_set_sample_fmt(swr, \"in_sample_fmt\", codec->sample_fmt, 0);\n\tav_opt_set_sample_fmt(swr, \"out_sample_fmt\", AV_SAMPLE_FMT_S16, 0);\n#endif\n\n\tswr_init(swr);\n\tif (!swr_is_initialized(swr)) {\n        LOG(\"Resampler has not been properly initialized\\n\");\n\t\treturn -1;\n\t}\n\n\tpacket=av_packet_alloc();\n\tif (!packet) {\n\t\tLOG(\"Error allocating the packet\\n\");\n\t\treturn -1;\n\t}\n\tframe = av_frame_alloc();\n\tif (!frame) {\n        LOG(\"Error allocating the frame\\n\");\n\t\treturn -1;\n\t}\n\n\t/* iterate through frames */\n\t*data = NULL;\n\t*size = 0;\n\twhile (av_read_frame(fmt_ctx, packet) >= 0) {\n\t\tavcodec_send_packet(codec, packet);\n\n\t\terr = avcodec_receive_frame(codec, frame);\n\t\tif (err == AVERROR(EAGAIN))\n\t\t\tcontinue;\n\n\t\tconvert_frame(swr, codec, frame, data, size, false);\n\t}\n\t/* Flush any remaining conversion buffers... */\n\tconvert_frame(swr, codec, frame, data, size, true);\n\n\tav_packet_free(&packet);\n\tav_frame_free(&frame);\n\tswr_free(&swr);\n    //avio_context_free(); // todo?\n\tavcodec_free_context(&codec);\n\tavformat_close_input(&fmt_ctx);\n\tavformat_free_context(fmt_ctx);\n\n\tif (avio_ctx) {\n\t\tav_freep(&avio_ctx->buffer);\n\t\tav_freep(&avio_ctx);\n\t}\n\n\treturn 0;\n}\n\n// in mem decoding/conversion/resampling:\n// ifname: input file path\n// owav_data: in mem wav file. Can be forwarded as it to whisper/drwav\n// return 0 on success\nint ffmpeg_decode_audio(const std::string &ifname, std::vector<uint8_t>& owav_data) {\n    LOG(\"ffmpeg_decode_audio: %s\\n\", ifname.c_str());\n    int ifd = open(ifname.c_str(), O_RDONLY);\n    if (ifd == -1) {\n        fprintf(stderr, \"Couldn't open input file %s\\n\", ifname.c_str());\n        return -1;\n    }\n    u8 *ibuf = NULL;\n    size_t ibuf_size;\n    int err = map_file(ifd, &ibuf, &ibuf_size);\n    if (err) {\n        LOG(\"Couldn't map input file %s\\n\", ifname.c_str());\n        return err;\n    }\n    LOG(\"Mapped input file: %s size: %d\\n\", ibuf, (int) ibuf_size);\n    struct audio_buffer inaudio_buf;\n    inaudio_buf.ptr = ibuf;\n    inaudio_buf.size = ibuf_size;\n\n    s16 *odata=NULL;\n    int osize=0;\n\n    err = decode_audio(&inaudio_buf, &odata, &osize);\n    LOG(\"decode_audio returned %d \\n\", err);\n    if (err != 0) {\n        LOG(\"decode_audio failed\\n\");\n        return err;\n    }\n    LOG(\"decode_audio output size: %d\\n\", osize);\n\n    wave_hdr wh;\n    const size_t outdatasize = osize * sizeof(s16);\n    set_wave_hdr(wh, outdatasize);\n    owav_data.resize(sizeof(wave_hdr) + outdatasize);\n    // header:\n    memcpy(owav_data.data(), &wh, sizeof(wave_hdr));\n    // the data:\n    memcpy(owav_data.data() + sizeof(wave_hdr), odata, osize* sizeof(s16));\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/generate-karaoke.sh",
    "content": "#!/bin/bash\n\n# Simple tool to record audio from the microphone and generate a karaoke video\n# Usage:\n#\n#  cd whisper.cpp\n#  make\n#\n#  ./examples/generate-karaoke.sh [model] [step_ms]\n#\n# Press Ctrl+C to stop recording\n#\n\nexecutable=\"./build/bin/whisper-cli\"\nmodel=\"base.en\"\nmodel_path=\"models/ggml-$model.bin\"\n\n# require sox and ffmpeg to be installed\nif ! command -v sox &> /dev/null\nthen\n    echo \"sox could not be found\"\n    exit 1\nfi\n\nif ! command -v ffmpeg &> /dev/null\nthen\n    echo \"ffmpeg could not be found\"\n    exit 2\nfi\n\nif [ ! -f \"$executable\" ]; then\n    echo \"'$executable' does not exist. Please build it first.\"\n    exit 3\nfi\n\nif [ ! -f \"$model_path\" ]; then\n    echo \"'$model_path' does not exist. Please download it first.\"\n    exit 4\nfi\n\n# record some raw audio\nsox -d rec.wav\n\n# run Whisper\necho \"Processing ...\"\n${executable} -m models/ggml-base.en.bin rec.wav -owts > /dev/null 2>&1\n\n# generate Karaoke video\necho \"Generating video ...\"\nsource rec.wav.wts > /dev/null 2>&1\n\n# play the video\necho \"Playing ./rec16.wav.mp4 ...\"\nffplay -loglevel 0 -autoexit ./rec.wav.mp4\n\necho \"Done\"\nexit 0\n"
  },
  {
    "path": "examples/grammar-parser.cpp",
    "content": "#include \"grammar-parser.h\"\n#include <cstdint>\n#include <cwchar>\n#include <string>\n#include <utility>\n#include <stdexcept>\n#include <exception>\n\nnamespace grammar_parser {\n    // NOTE: assumes valid utf8 (but checks for overrun)\n    // copied from whisper.cpp\n    static std::pair<uint32_t, const char *> decode_utf8(const char * src) {\n        static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };\n        uint8_t  first_byte = static_cast<uint8_t>(*src);\n        uint8_t  highbits   = first_byte >> 4;\n        int      len        = lookup[highbits];\n        uint8_t  mask       = (1 << (8 - len)) - 1;\n        uint32_t value      = first_byte & mask;\n        const char * end    = src + len; // may overrun!\n        const char * pos    = src + 1;\n        for ( ; pos < end && *pos; pos++) {\n            value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);\n        }\n        return std::make_pair(value, pos);\n    }\n\n    static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {\n        uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());\n        auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id));\n        return result.first->second;\n    }\n\n    static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) {\n        uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());\n        state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id;\n        return next_id;\n    }\n\n    static void add_rule(\n            parse_state & state,\n            uint32_t      rule_id,\n            const std::vector<whisper_grammar_element> & rule) {\n        if (state.rules.size() <= rule_id) {\n            state.rules.resize(rule_id + 1);\n        }\n        state.rules[rule_id] = rule;\n    }\n\n    static bool is_word_char(char c) {\n        return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || ('0' <= c && c <= '9');\n    }\n\n    static std::pair<uint32_t, const char *> parse_hex(const char * src, int size) {\n        const char * pos   = src;\n        const char * end   = src + size;\n        uint32_t     value = 0;\n        for ( ; pos < end && *pos; pos++) {\n            value <<= 4;\n            char c = *pos;\n            if ('a' <= c && c <= 'f') {\n                value += c - 'a' + 10;\n            } else if ('A' <= c && c <= 'F') {\n                value += c - 'A' + 10;\n            } else if ('0' <= c && c <= '9') {\n                value += c - '0';\n            } else {\n                break;\n            }\n        }\n        if (pos != end) {\n            throw std::runtime_error(\"expecting \" + std::to_string(size) + \" hex chars at \" + src);\n        }\n        return std::make_pair(value, pos);\n    }\n\n    static const char * parse_space(const char * src, bool newline_ok) {\n        const char * pos = src;\n        while (*pos == ' ' || *pos == '\\t' || *pos == '#' ||\n                (newline_ok && (*pos == '\\r' || *pos == '\\n'))) {\n            if (*pos == '#') {\n                while (*pos && *pos != '\\r' && *pos != '\\n') {\n                    pos++;\n                }\n            } else {\n                pos++;\n            }\n        }\n        return pos;\n    }\n\n    static const char * parse_name(const char * src) {\n        const char * pos = src;\n        while (is_word_char(*pos)) {\n            pos++;\n        }\n        if (pos == src) {\n            throw std::runtime_error(std::string(\"expecting name at \") + src);\n        }\n        return pos;\n    }\n\n    static std::pair<uint32_t, const char *> parse_char(const char * src) {\n        if (*src == '\\\\') {\n            switch (src[1]) {\n                case 'x': return parse_hex(src + 2, 2);\n                case 'u': return parse_hex(src + 2, 4);\n                case 'U': return parse_hex(src + 2, 8);\n                case 't': return std::make_pair('\\t', src + 2);\n                case 'r': return std::make_pair('\\r', src + 2);\n                case 'n': return std::make_pair('\\n', src + 2);\n                case '\\\\':\n                case '\"':\n                case '[':\n                case ']':\n                    return std::make_pair(src[1], src + 2);\n                default:\n                    throw std::runtime_error(std::string(\"unknown escape at \") + src);\n            }\n        } else if (*src) {\n            return decode_utf8(src);\n        }\n        throw std::runtime_error(\"unexpected end of input\");\n    }\n\n    static const char * parse_alternates(\n            parse_state       & state,\n            const char        * src,\n            const std::string & rule_name,\n            uint32_t            rule_id,\n            bool                is_nested);\n\n    static const char * parse_sequence(\n            parse_state                        & state,\n            const char                         * src,\n            const std::string                  & rule_name,\n            std::vector<whisper_grammar_element> & out_elements,\n            bool                                 is_nested) {\n        size_t last_sym_start = out_elements.size();\n        const char * pos = src;\n        while (*pos) {\n            if (*pos == '\"') { // literal string\n                pos++;\n                last_sym_start = out_elements.size();\n                while (*pos != '\"') {\n                    auto char_pair = parse_char(pos);\n                         pos       = char_pair.second;\n                    out_elements.push_back({WHISPER_GRETYPE_CHAR, char_pair.first});\n                }\n                pos = parse_space(pos + 1, is_nested);\n            } else if (*pos == '[') { // char range(s)\n                pos++;\n                enum whisper_gretype start_type = WHISPER_GRETYPE_CHAR;\n                if (*pos == '^') {\n                    pos++;\n                    start_type = WHISPER_GRETYPE_CHAR_NOT;\n                }\n                last_sym_start = out_elements.size();\n                while (*pos != ']') {\n                    auto char_pair = parse_char(pos);\n                         pos       = char_pair.second;\n                    enum whisper_gretype type = last_sym_start < out_elements.size()\n                        ? WHISPER_GRETYPE_CHAR_ALT\n                        : start_type;\n\n                    out_elements.push_back({type, char_pair.first});\n                    if (pos[0] == '-' && pos[1] != ']') {\n                        auto endchar_pair = parse_char(pos + 1);\n                             pos          = endchar_pair.second;\n                        out_elements.push_back({WHISPER_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});\n                    }\n                }\n                pos = parse_space(pos + 1, is_nested);\n            } else if (is_word_char(*pos)) { // rule reference\n                const char * name_end    = parse_name(pos);\n                uint32_t     ref_rule_id = get_symbol_id(state, pos, name_end - pos);\n                pos = parse_space(name_end, is_nested);\n                last_sym_start = out_elements.size();\n                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, ref_rule_id});\n            } else if (*pos == '(') { // grouping\n                // parse nested alternates into synthesized rule\n                pos = parse_space(pos + 1, true);\n                uint32_t sub_rule_id = generate_symbol_id(state, rule_name);\n                pos = parse_alternates(state, pos, rule_name, sub_rule_id, true);\n                last_sym_start = out_elements.size();\n                // output reference to synthesized rule\n                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});\n                if (*pos != ')') {\n                    throw std::runtime_error(std::string(\"expecting ')' at \") + pos);\n                }\n                pos = parse_space(pos + 1, is_nested);\n            } else if (*pos == '*' || *pos == '+' || *pos == '?') { // repetition operator\n                if (last_sym_start == out_elements.size()) {\n                    throw std::runtime_error(std::string(\"expecting preceding item to */+/? at \") + pos);\n                }\n\n                // apply transformation to previous symbol (last_sym_start to end) according to\n                // rewrite rules:\n                // S* --> S' ::= S S' |\n                // S+ --> S' ::= S S' | S\n                // S? --> S' ::= S |\n                uint32_t sub_rule_id = generate_symbol_id(state, rule_name);\n                std::vector<whisper_grammar_element> sub_rule;\n                // add preceding symbol to generated rule\n                sub_rule.insert(\n                    sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());\n                if (*pos == '*' || *pos == '+') {\n                    // cause generated rule to recurse\n                    sub_rule.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});\n                }\n                // mark start of alternate def\n                sub_rule.push_back({WHISPER_GRETYPE_ALT, 0});\n                if (*pos == '+') {\n                    // add preceding symbol as alternate only for '+' (otherwise empty)\n                    sub_rule.insert(\n                        sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());\n                }\n                sub_rule.push_back({WHISPER_GRETYPE_END, 0});\n                add_rule(state, sub_rule_id, sub_rule);\n\n                // in original rule, replace previous symbol with reference to generated rule\n                out_elements.resize(last_sym_start);\n                out_elements.push_back({WHISPER_GRETYPE_RULE_REF, sub_rule_id});\n\n                pos = parse_space(pos + 1, is_nested);\n            } else {\n                break;\n            }\n        }\n        return pos;\n    }\n\n    static const char * parse_alternates(\n            parse_state       & state,\n            const char        * src,\n            const std::string & rule_name,\n            uint32_t            rule_id,\n            bool                is_nested) {\n        std::vector<whisper_grammar_element> rule;\n        const char * pos = parse_sequence(state, src, rule_name, rule, is_nested);\n        while (*pos == '|') {\n            rule.push_back({WHISPER_GRETYPE_ALT, 0});\n            pos = parse_space(pos + 1, true);\n            pos = parse_sequence(state, pos, rule_name, rule, is_nested);\n        }\n        rule.push_back({WHISPER_GRETYPE_END, 0});\n        add_rule(state, rule_id, rule);\n        return pos;\n    }\n\n    static const char * parse_rule(parse_state & state, const char * src) {\n        const char * name_end = parse_name(src);\n        const char * pos      = parse_space(name_end, false);\n        size_t       name_len = name_end - src;\n        uint32_t     rule_id  = get_symbol_id(state, src, name_len);\n        const std::string name(src, name_len);\n\n        if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) {\n            throw std::runtime_error(std::string(\"expecting ::= at \") + pos);\n        }\n        pos = parse_space(pos + 3, true);\n\n        pos = parse_alternates(state, pos, name, rule_id, false);\n\n        if (*pos == '\\r') {\n            pos += pos[1] == '\\n' ? 2 : 1;\n        } else if (*pos == '\\n') {\n            pos++;\n        } else if (*pos) {\n            throw std::runtime_error(std::string(\"expecting newline or end at \") + pos);\n        }\n        return parse_space(pos, true);\n    }\n\n    parse_state parse(const char * src) {\n        try {\n            parse_state state;\n            const char * pos = parse_space(src, true);\n            while (*pos) {\n                pos = parse_rule(state, pos);\n            }\n            return state;\n        } catch (const std::exception & err) {\n            fprintf(stderr, \"%s: error parsing grammar: %s\\n\", __func__, err.what());\n            return parse_state();\n        }\n    }\n\n    static void print_grammar_char(FILE * file, uint32_t c) {\n        if (0x20 <= c && c <= 0x7f) {\n            fprintf(file, \"%c\", static_cast<char>(c));\n        } else {\n            // cop out of encoding UTF-8\n            fprintf(file, \"<U+%04X>\", c);\n        }\n    }\n\n    static bool is_char_element(whisper_grammar_element elem) {\n        switch (elem.type) {\n            case WHISPER_GRETYPE_CHAR:           return true;\n            case WHISPER_GRETYPE_CHAR_NOT:       return true;\n            case WHISPER_GRETYPE_CHAR_ALT:       return true;\n            case WHISPER_GRETYPE_CHAR_RNG_UPPER: return true;\n            default:                           return false;\n        }\n    }\n\n    static void print_rule_binary(FILE * file, const std::vector<whisper_grammar_element> & rule) {\n        for (auto elem : rule) {\n            switch (elem.type) {\n                case WHISPER_GRETYPE_END:            fprintf(file, \"END\");            break;\n                case WHISPER_GRETYPE_ALT:            fprintf(file, \"ALT\");            break;\n                case WHISPER_GRETYPE_RULE_REF:       fprintf(file, \"RULE_REF\");       break;\n                case WHISPER_GRETYPE_CHAR:           fprintf(file, \"CHAR\");           break;\n                case WHISPER_GRETYPE_CHAR_NOT:       fprintf(file, \"CHAR_NOT\");       break;\n                case WHISPER_GRETYPE_CHAR_RNG_UPPER: fprintf(file, \"CHAR_RNG_UPPER\"); break;\n                case WHISPER_GRETYPE_CHAR_ALT:       fprintf(file, \"CHAR_ALT\");       break;\n            }\n            switch (elem.type) {\n                case WHISPER_GRETYPE_END:\n                case WHISPER_GRETYPE_ALT:\n                case WHISPER_GRETYPE_RULE_REF:\n                    fprintf(file, \"(%u) \", elem.value);\n                    break;\n                case WHISPER_GRETYPE_CHAR:\n                case WHISPER_GRETYPE_CHAR_NOT:\n                case WHISPER_GRETYPE_CHAR_RNG_UPPER:\n                case WHISPER_GRETYPE_CHAR_ALT:\n                    fprintf(file, \"(\\\"\");\n                    print_grammar_char(file, elem.value);\n                    fprintf(file, \"\\\") \");\n                    break;\n            }\n        }\n        fprintf(file, \"\\n\");\n    }\n\n    static void print_rule(\n            FILE     * file,\n            uint32_t   rule_id,\n            const std::vector<whisper_grammar_element> & rule,\n            const std::map<uint32_t, std::string>    & symbol_id_names) {\n        if (rule.empty() || rule.back().type != WHISPER_GRETYPE_END) {\n            throw std::runtime_error(\n                \"malformed rule, does not end with WHISPER_GRETYPE_END: \" + std::to_string(rule_id));\n        }\n        fprintf(file, \"%s ::= \", symbol_id_names.at(rule_id).c_str());\n        for (size_t i = 0, end = rule.size() - 1; i < end; i++) {\n            whisper_grammar_element elem = rule[i];\n            switch (elem.type) {\n                case WHISPER_GRETYPE_END:\n                    throw std::runtime_error(\n                        \"unexpected end of rule: \" + std::to_string(rule_id) + \",\" +\n                        std::to_string(i));\n                case WHISPER_GRETYPE_ALT:\n                    fprintf(file, \"| \");\n                    break;\n                case WHISPER_GRETYPE_RULE_REF:\n                    fprintf(file, \"%s \", symbol_id_names.at(elem.value).c_str());\n                    break;\n                case WHISPER_GRETYPE_CHAR:\n                    fprintf(file, \"[\");\n                    print_grammar_char(file, elem.value);\n                    break;\n                case WHISPER_GRETYPE_CHAR_NOT:\n                    fprintf(file, \"[^\");\n                    print_grammar_char(file, elem.value);\n                    break;\n                case WHISPER_GRETYPE_CHAR_RNG_UPPER:\n                    if (i == 0 || !is_char_element(rule[i - 1])) {\n                        throw std::runtime_error(\n                            \"WHISPER_GRETYPE_CHAR_RNG_UPPER without preceding char: \" +\n                            std::to_string(rule_id) + \",\" + std::to_string(i));\n                    }\n                    fprintf(file, \"-\");\n                    print_grammar_char(file, elem.value);\n                    break;\n                case WHISPER_GRETYPE_CHAR_ALT:\n                    if (i == 0 || !is_char_element(rule[i - 1])) {\n                        throw std::runtime_error(\n                            \"WHISPER_GRETYPE_CHAR_ALT without preceding char: \" +\n                            std::to_string(rule_id) + \",\" + std::to_string(i));\n                    }\n                    print_grammar_char(file, elem.value);\n                    break;\n            }\n            if (is_char_element(elem)) {\n                switch (rule[i + 1].type) {\n                    case WHISPER_GRETYPE_CHAR_ALT:\n                    case WHISPER_GRETYPE_CHAR_RNG_UPPER:\n                        break;\n                    default:\n                        fprintf(file, \"] \");\n                }\n            }\n        }\n        fprintf(file, \"\\n\");\n    }\n\n    void print_grammar(FILE * file, const parse_state & state) {\n        try {\n            std::map<uint32_t, std::string> symbol_id_names;\n            for (auto kv : state.symbol_ids) {\n                symbol_id_names[kv.second] = kv.first;\n            }\n            for (size_t i = 0, end = state.rules.size(); i < end; i++) {\n                // fprintf(file, \"%zu: \", i);\n                // print_rule_binary(file, state.rules[i]);\n                print_rule(file, uint32_t(i), state.rules[i], symbol_id_names);\n                // fprintf(file, \"\\n\");\n            }\n        } catch (const std::exception & err) {\n            fprintf(stderr, \"\\n%s: error printing grammar: %s\\n\", __func__, err.what());\n        }\n    }\n\n    std::vector<const whisper_grammar_element *> parse_state::c_rules() const {\n        std::vector<const whisper_grammar_element *> ret;\n        for (const auto & rule : rules) {\n            ret.push_back(rule.data());\n        }\n        return ret;\n    }\n}\n"
  },
  {
    "path": "examples/grammar-parser.h",
    "content": "// Implements a parser for an extended Backus-Naur form (BNF), producing the\n// binary context-free grammar format specified by whisper.h. Supports character\n// ranges, grouping, and repetition operators. As an example, a grammar for\n// arithmetic might look like:\n//\n// root  ::= expr\n// expr  ::= term ([-+*/] term)*\n// term  ::= num | \"(\" space expr \")\" space\n// num   ::= [0-9]+ space\n// space ::= [ \\t\\n]*\n\n#pragma once\n#include \"whisper.h\"\n#include <vector>\n#include <map>\n#include <cstdint>\n#include <string>\n\nnamespace grammar_parser {\n    struct parse_state {\n        std::map<std::string, uint32_t>                   symbol_ids;\n        std::vector<std::vector<whisper_grammar_element>> rules;\n\n        std::vector<const whisper_grammar_element *>      c_rules() const;\n    };\n\n    parse_state parse(const char * src);\n    void print_grammar(FILE * file, const parse_state & state);\n}\n"
  },
  {
    "path": "examples/helpers.js",
    "content": "// Common Javascript functions used by the examples\n\nfunction convertTypedArray(src, type) {\n    var buffer = new ArrayBuffer(src.byteLength);\n    var baseView = new src.constructor(buffer).set(src);\n    return new type(buffer);\n}\n\nvar printTextarea = (function() {\n    var element = document.getElementById('output');\n    if (element) element.value = ''; // clear browser cache\n    return function(text) {\n        if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');\n        console.log(text);\n        if (element) {\n            element.value += text + \"\\n\";\n            element.scrollTop = element.scrollHeight; // focus on bottom\n        }\n    };\n})();\n\nasync function clearCache() {\n    if (confirm('Are you sure you want to clear the cache?\\nAll the models will be downloaded again.')) {\n        indexedDB.deleteDatabase(dbName);\n        location.reload();\n    }\n}\n\n// fetch a remote file from remote URL using the Fetch API\nasync function fetchRemote(url, cbProgress, cbPrint) {\n    cbPrint('fetchRemote: downloading with fetch()...');\n\n    const response = await fetch(\n        url,\n        {\n            method: 'GET',\n        }\n    );\n\n    if (!response.ok) {\n        cbPrint('fetchRemote: failed to fetch ' + url);\n        return;\n    }\n\n    const contentLength = response.headers.get('content-length');\n    const total = parseInt(contentLength, 10);\n    const reader = response.body.getReader();\n\n    var chunks = [];\n    var receivedLength = 0;\n    var progressLast = -1;\n\n    while (true) {\n        const { done, value } = await reader.read();\n\n        if (done) {\n            break;\n        }\n\n        chunks.push(value);\n        receivedLength += value.length;\n\n        if (contentLength) {\n            cbProgress(receivedLength/total);\n\n            var progressCur = Math.round((receivedLength / total) * 10);\n            if (progressCur != progressLast) {\n                cbPrint('fetchRemote: fetching ' + 10*progressCur + '% ...');\n                progressLast = progressCur;\n            }\n        }\n    }\n\n    var position = 0;\n    var chunksAll = new Uint8Array(receivedLength);\n\n    for (var chunk of chunks) {\n        chunksAll.set(chunk, position);\n        position += chunk.length;\n    }\n\n    return chunksAll;\n}\n\n// load remote data\n// - check if the data is already in the IndexedDB\n// - if not, fetch it from the remote URL and store it in the IndexedDB\nfunction loadRemote(url, dst, size_mb, cbProgress, cbReady, cbCancel, cbPrint) {\n    if (!navigator.storage || !navigator.storage.estimate) {\n        cbPrint('loadRemote: navigator.storage.estimate() is not supported');\n    } else {\n        // query the storage quota and print it\n        navigator.storage.estimate().then(function (estimate) {\n            cbPrint('loadRemote: storage quota: ' + estimate.quota + ' bytes');\n            cbPrint('loadRemote: storage usage: ' + estimate.usage + ' bytes');\n        });\n    }\n\n    // check if the data is already in the IndexedDB\n    var rq = indexedDB.open(dbName, dbVersion);\n\n    rq.onupgradeneeded = function (event) {\n        var db = event.target.result;\n        if (db.version == 1) {\n            var os = db.createObjectStore('models', { autoIncrement: false });\n            cbPrint('loadRemote: created IndexedDB ' + db.name + ' version ' + db.version);\n        } else {\n            // clear the database\n            var os = event.currentTarget.transaction.objectStore('models');\n            os.clear();\n            cbPrint('loadRemote: cleared IndexedDB ' + db.name + ' version ' + db.version);\n        }\n    };\n\n    rq.onsuccess = function (event) {\n        var db = event.target.result;\n        var tx = db.transaction(['models'], 'readonly');\n        var os = tx.objectStore('models');\n        var rq = os.get(url);\n\n        rq.onsuccess = function (event) {\n            if (rq.result) {\n                cbPrint('loadRemote: \"' + url + '\" is already in the IndexedDB');\n                cbReady(dst, rq.result);\n            } else {\n                // data is not in the IndexedDB\n                cbPrint('loadRemote: \"' + url + '\" is not in the IndexedDB');\n\n                // alert and ask the user to confirm\n                if (!confirm(\n                    'You are about to download ' + size_mb + ' MB of data.\\n' +\n                    'The model data will be cached in the browser for future use.\\n\\n' +\n                    'Press OK to continue.')) {\n                    cbCancel();\n                    return;\n                }\n\n                fetchRemote(url, cbProgress, cbPrint).then(function (data) {\n                    if (data) {\n                        // store the data in the IndexedDB\n                        var rq = indexedDB.open(dbName, dbVersion);\n                        rq.onsuccess = function (event) {\n                            var db = event.target.result;\n                            var tx = db.transaction(['models'], 'readwrite');\n                            var os = tx.objectStore('models');\n\n                            var rq = null;\n                            try {\n                                var rq = os.put(data, url);\n                            } catch (e) {\n                                cbPrint('loadRemote: failed to store \"' + url + '\" in the IndexedDB: \\n' + e);\n                                cbCancel();\n                                return;\n                            }\n\n                            rq.onsuccess = function (event) {\n                                cbPrint('loadRemote: \"' + url + '\" stored in the IndexedDB');\n                                cbReady(dst, data);\n                            };\n\n                            rq.onerror = function (event) {\n                                cbPrint('loadRemote: failed to store \"' + url + '\" in the IndexedDB');\n                                cbCancel();\n                            };\n                        };\n                    }\n                });\n            }\n        };\n\n        rq.onerror = function (event) {\n            cbPrint('loadRemote: failed to get data from the IndexedDB');\n            cbCancel();\n        };\n    };\n\n    rq.onerror = function (event) {\n        cbPrint('loadRemote: failed to open IndexedDB');\n        cbCancel();\n    };\n\n    rq.onblocked = function (event) {\n        cbPrint('loadRemote: failed to open IndexedDB: blocked');\n        cbCancel();\n    };\n\n    rq.onabort = function (event) {\n        cbPrint('loadRemote: failed to open IndexedDB: abort');\n        cbCancel();\n    };\n}\n"
  },
  {
    "path": "examples/json.hpp",
    "content": "//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n/****************************************************************************\\\n * Note on documentation: The source files contain links to the online      *\n * documentation of the public API at https://json.nlohmann.me. This URL    *\n * contains the most recent documentation and should also be applicable to  *\n * previous versions; documentation for deprecated functions is not         *\n * removed, but marked deprecated. See \"Generate documentation\" section in  *\n * file docs/README.md.                                                     *\n\\****************************************************************************/\n\n#ifndef INCLUDE_NLOHMANN_JSON_HPP_\n#define INCLUDE_NLOHMANN_JSON_HPP_\n\n#include <algorithm> // all_of, find, for_each\n#include <cstddef> // nullptr_t, ptrdiff_t, size_t\n#include <functional> // hash, less\n#include <initializer_list> // initializer_list\n#ifndef JSON_NO_IO\n    #include <iosfwd> // istream, ostream\n#endif  // JSON_NO_IO\n#include <iterator> // random_access_iterator_tag\n#include <memory> // unique_ptr\n#include <numeric> // accumulate\n#include <string> // string, stoi, to_string\n#include <utility> // declval, forward, move, pair, swap\n#include <vector> // vector\n\n// #include <nlohmann/adl_serializer.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <utility>\n\n// #include <nlohmann/detail/abi_macros.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// This file contains all macro definitions affecting or depending on the ABI\n\n#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK\n    #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH)\n        #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 2\n            #warning \"Already included a different version of the library!\"\n        #endif\n    #endif\n#endif\n\n#define NLOHMANN_JSON_VERSION_MAJOR 3   // NOLINT(modernize-macro-to-enum)\n#define NLOHMANN_JSON_VERSION_MINOR 11  // NOLINT(modernize-macro-to-enum)\n#define NLOHMANN_JSON_VERSION_PATCH 2   // NOLINT(modernize-macro-to-enum)\n\n#ifndef JSON_DIAGNOSTICS\n    #define JSON_DIAGNOSTICS 0\n#endif\n\n#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON\n    #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0\n#endif\n\n#if JSON_DIAGNOSTICS\n    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag\n#else\n    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS\n#endif\n\n#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON\n    #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp\n#else\n    #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON\n#endif\n\n#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION\n    #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0\n#endif\n\n// Construct the namespace ABI tags component\n#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b\n#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \\\n    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b)\n\n#define NLOHMANN_JSON_ABI_TAGS                                       \\\n    NLOHMANN_JSON_ABI_TAGS_CONCAT(                                   \\\n            NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS,                       \\\n            NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON)\n\n// Construct the namespace version component\n#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \\\n    _v ## major ## _ ## minor ## _ ## patch\n#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \\\n    NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch)\n\n#if NLOHMANN_JSON_NAMESPACE_NO_VERSION\n#define NLOHMANN_JSON_NAMESPACE_VERSION\n#else\n#define NLOHMANN_JSON_NAMESPACE_VERSION                                 \\\n    NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \\\n                                           NLOHMANN_JSON_VERSION_MINOR, \\\n                                           NLOHMANN_JSON_VERSION_PATCH)\n#endif\n\n// Combine namespace components\n#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b\n#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \\\n    NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b)\n\n#ifndef NLOHMANN_JSON_NAMESPACE\n#define NLOHMANN_JSON_NAMESPACE               \\\n    nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \\\n            NLOHMANN_JSON_ABI_TAGS,           \\\n            NLOHMANN_JSON_NAMESPACE_VERSION)\n#endif\n\n#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN\n#define NLOHMANN_JSON_NAMESPACE_BEGIN                \\\n    namespace nlohmann                               \\\n    {                                                \\\n    inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \\\n                NLOHMANN_JSON_ABI_TAGS,              \\\n                NLOHMANN_JSON_NAMESPACE_VERSION)     \\\n    {\n#endif\n\n#ifndef NLOHMANN_JSON_NAMESPACE_END\n#define NLOHMANN_JSON_NAMESPACE_END                                     \\\n    }  /* namespace (inline namespace) NOLINT(readability/namespace) */ \\\n    }  // namespace nlohmann\n#endif\n\n// #include <nlohmann/detail/conversions/from_json.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // transform\n#include <array> // array\n#include <forward_list> // forward_list\n#include <iterator> // inserter, front_inserter, end\n#include <map> // map\n#include <string> // string\n#include <tuple> // tuple, make_tuple\n#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible\n#include <unordered_map> // unordered_map\n#include <utility> // pair, declval\n#include <valarray> // valarray\n\n// #include <nlohmann/detail/exceptions.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef> // nullptr_t\n#include <exception> // exception\n#include <stdexcept> // runtime_error\n#include <string> // to_string\n#include <vector> // vector\n\n// #include <nlohmann/detail/value_t.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <array> // array\n#include <cstddef> // size_t\n#include <cstdint> // uint8_t\n#include <string> // string\n\n// #include <nlohmann/detail/macro_scope.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <utility> // declval, pair\n// #include <nlohmann/detail/meta/detected.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <type_traits>\n\n// #include <nlohmann/detail/meta/void_t.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename ...Ts> struct make_void\n{\n    using type = void;\n};\ntemplate<typename ...Ts> using void_t = typename make_void<Ts...>::type;\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n// https://en.cppreference.com/w/cpp/experimental/is_detected\nstruct nonesuch\n{\n    nonesuch() = delete;\n    ~nonesuch() = delete;\n    nonesuch(nonesuch const&) = delete;\n    nonesuch(nonesuch const&&) = delete;\n    void operator=(nonesuch const&) = delete;\n    void operator=(nonesuch&&) = delete;\n};\n\ntemplate<class Default,\n         class AlwaysVoid,\n         template<class...> class Op,\n         class... Args>\nstruct detector\n{\n    using value_t = std::false_type;\n    using type = Default;\n};\n\ntemplate<class Default, template<class...> class Op, class... Args>\nstruct detector<Default, void_t<Op<Args...>>, Op, Args...>\n{\n    using value_t = std::true_type;\n    using type = Op<Args...>;\n};\n\ntemplate<template<class...> class Op, class... Args>\nusing is_detected = typename detector<nonesuch, void, Op, Args...>::value_t;\n\ntemplate<template<class...> class Op, class... Args>\nstruct is_detected_lazy : is_detected<Op, Args...> { };\n\ntemplate<template<class...> class Op, class... Args>\nusing detected_t = typename detector<nonesuch, void, Op, Args...>::type;\n\ntemplate<class Default, template<class...> class Op, class... Args>\nusing detected_or = detector<Default, void, Op, Args...>;\n\ntemplate<class Default, template<class...> class Op, class... Args>\nusing detected_or_t = typename detected_or<Default, Op, Args...>::type;\n\ntemplate<class Expected, template<class...> class Op, class... Args>\nusing is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>;\n\ntemplate<class To, template<class...> class Op, class... Args>\nusing is_detected_convertible =\n    std::is_convertible<detected_t<Op, Args...>, To>;\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/thirdparty/hedley/hedley.hpp>\n\n\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-FileCopyrightText: 2016-2021 Evan Nemerson <evan@nemerson.com>\n// SPDX-License-Identifier: MIT\n\n/* Hedley - https://nemequ.github.io/hedley\n * Created by Evan Nemerson <evan@nemerson.com>\n */\n\n#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15)\n#if defined(JSON_HEDLEY_VERSION)\n    #undef JSON_HEDLEY_VERSION\n#endif\n#define JSON_HEDLEY_VERSION 15\n\n#if defined(JSON_HEDLEY_STRINGIFY_EX)\n    #undef JSON_HEDLEY_STRINGIFY_EX\n#endif\n#define JSON_HEDLEY_STRINGIFY_EX(x) #x\n\n#if defined(JSON_HEDLEY_STRINGIFY)\n    #undef JSON_HEDLEY_STRINGIFY\n#endif\n#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x)\n\n#if defined(JSON_HEDLEY_CONCAT_EX)\n    #undef JSON_HEDLEY_CONCAT_EX\n#endif\n#define JSON_HEDLEY_CONCAT_EX(a,b) a##b\n\n#if defined(JSON_HEDLEY_CONCAT)\n    #undef JSON_HEDLEY_CONCAT\n#endif\n#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b)\n\n#if defined(JSON_HEDLEY_CONCAT3_EX)\n    #undef JSON_HEDLEY_CONCAT3_EX\n#endif\n#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c\n\n#if defined(JSON_HEDLEY_CONCAT3)\n    #undef JSON_HEDLEY_CONCAT3\n#endif\n#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c)\n\n#if defined(JSON_HEDLEY_VERSION_ENCODE)\n    #undef JSON_HEDLEY_VERSION_ENCODE\n#endif\n#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision))\n\n#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR)\n    #undef JSON_HEDLEY_VERSION_DECODE_MAJOR\n#endif\n#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000)\n\n#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR)\n    #undef JSON_HEDLEY_VERSION_DECODE_MINOR\n#endif\n#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000)\n\n#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION)\n    #undef JSON_HEDLEY_VERSION_DECODE_REVISION\n#endif\n#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000)\n\n#if defined(JSON_HEDLEY_GNUC_VERSION)\n    #undef JSON_HEDLEY_GNUC_VERSION\n#endif\n#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__)\n    #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)\n#elif defined(__GNUC__)\n    #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK)\n    #undef JSON_HEDLEY_GNUC_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_GNUC_VERSION)\n    #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_MSVC_VERSION)\n    #undef JSON_HEDLEY_MSVC_VERSION\n#endif\n#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL)\n    #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100)\n#elif defined(_MSC_FULL_VER) && !defined(__ICL)\n    #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10)\n#elif defined(_MSC_VER) && !defined(__ICL)\n    #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0)\n#endif\n\n#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK)\n    #undef JSON_HEDLEY_MSVC_VERSION_CHECK\n#endif\n#if !defined(JSON_HEDLEY_MSVC_VERSION)\n    #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0)\n#elif defined(_MSC_VER) && (_MSC_VER >= 1400)\n    #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch)))\n#elif defined(_MSC_VER) && (_MSC_VER >= 1200)\n    #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch)))\n#else\n    #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor)))\n#endif\n\n#if defined(JSON_HEDLEY_INTEL_VERSION)\n    #undef JSON_HEDLEY_INTEL_VERSION\n#endif\n#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL)\n    #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE)\n#elif defined(__INTEL_COMPILER) && !defined(__ICL)\n    #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0)\n#endif\n\n#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK)\n    #undef JSON_HEDLEY_INTEL_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_INTEL_VERSION)\n    #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_INTEL_CL_VERSION)\n    #undef JSON_HEDLEY_INTEL_CL_VERSION\n#endif\n#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL)\n    #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0)\n#endif\n\n#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK)\n    #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_INTEL_CL_VERSION)\n    #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_PGI_VERSION)\n    #undef JSON_HEDLEY_PGI_VERSION\n#endif\n#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__)\n    #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__)\n#endif\n\n#if defined(JSON_HEDLEY_PGI_VERSION_CHECK)\n    #undef JSON_HEDLEY_PGI_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_PGI_VERSION)\n    #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_SUNPRO_VERSION)\n    #undef JSON_HEDLEY_SUNPRO_VERSION\n#endif\n#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000)\n    #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10)\n#elif defined(__SUNPRO_C)\n    #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf)\n#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000)\n    #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10)\n#elif defined(__SUNPRO_CC)\n    #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf)\n#endif\n\n#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK)\n    #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_SUNPRO_VERSION)\n    #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION)\n    #undef JSON_HEDLEY_EMSCRIPTEN_VERSION\n#endif\n#if defined(__EMSCRIPTEN__)\n    #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__)\n#endif\n\n#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK)\n    #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION)\n    #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_ARM_VERSION)\n    #undef JSON_HEDLEY_ARM_VERSION\n#endif\n#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION)\n    #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100)\n#elif defined(__CC_ARM) && defined(__ARMCC_VERSION)\n    #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100)\n#endif\n\n#if defined(JSON_HEDLEY_ARM_VERSION_CHECK)\n    #undef JSON_HEDLEY_ARM_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_ARM_VERSION)\n    #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_IBM_VERSION)\n    #undef JSON_HEDLEY_IBM_VERSION\n#endif\n#if defined(__ibmxl__)\n    #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__)\n#elif defined(__xlC__) && defined(__xlC_ver__)\n    #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff)\n#elif defined(__xlC__)\n    #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0)\n#endif\n\n#if defined(JSON_HEDLEY_IBM_VERSION_CHECK)\n    #undef JSON_HEDLEY_IBM_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_IBM_VERSION)\n    #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_VERSION)\n    #undef JSON_HEDLEY_TI_VERSION\n#endif\n#if \\\n    defined(__TI_COMPILER_VERSION__) && \\\n    ( \\\n      defined(__TMS470__) || defined(__TI_ARM__) || \\\n      defined(__MSP430__) || \\\n      defined(__TMS320C2000__) \\\n    )\n#if (__TI_COMPILER_VERSION__ >= 16000000)\n    #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n#endif\n\n#if defined(JSON_HEDLEY_TI_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_VERSION)\n    #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL2000_VERSION)\n    #undef JSON_HEDLEY_TI_CL2000_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__)\n    #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_CL2000_VERSION)\n    #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL430_VERSION)\n    #undef JSON_HEDLEY_TI_CL430_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__)\n    #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_CL430_VERSION)\n    #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_ARMCL_VERSION)\n    #undef JSON_HEDLEY_TI_ARMCL_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__))\n    #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_ARMCL_VERSION)\n    #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL6X_VERSION)\n    #undef JSON_HEDLEY_TI_CL6X_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__)\n    #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_CL6X_VERSION)\n    #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL7X_VERSION)\n    #undef JSON_HEDLEY_TI_CL7X_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__)\n    #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_CL7X_VERSION)\n    #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TI_CLPRU_VERSION)\n    #undef JSON_HEDLEY_TI_CLPRU_VERSION\n#endif\n#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__)\n    #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000))\n#endif\n\n#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK)\n    #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TI_CLPRU_VERSION)\n    #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_CRAY_VERSION)\n    #undef JSON_HEDLEY_CRAY_VERSION\n#endif\n#if defined(_CRAYC)\n    #if defined(_RELEASE_PATCHLEVEL)\n        #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL)\n    #else\n        #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0)\n    #endif\n#endif\n\n#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK)\n    #undef JSON_HEDLEY_CRAY_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_CRAY_VERSION)\n    #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_IAR_VERSION)\n    #undef JSON_HEDLEY_IAR_VERSION\n#endif\n#if defined(__IAR_SYSTEMS_ICC__)\n    #if __VER__ > 1000\n        #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000))\n    #else\n        #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0)\n    #endif\n#endif\n\n#if defined(JSON_HEDLEY_IAR_VERSION_CHECK)\n    #undef JSON_HEDLEY_IAR_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_IAR_VERSION)\n    #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_TINYC_VERSION)\n    #undef JSON_HEDLEY_TINYC_VERSION\n#endif\n#if defined(__TINYC__)\n    #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100)\n#endif\n\n#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK)\n    #undef JSON_HEDLEY_TINYC_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_TINYC_VERSION)\n    #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_DMC_VERSION)\n    #undef JSON_HEDLEY_DMC_VERSION\n#endif\n#if defined(__DMC__)\n    #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf)\n#endif\n\n#if defined(JSON_HEDLEY_DMC_VERSION_CHECK)\n    #undef JSON_HEDLEY_DMC_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_DMC_VERSION)\n    #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_COMPCERT_VERSION)\n    #undef JSON_HEDLEY_COMPCERT_VERSION\n#endif\n#if defined(__COMPCERT_VERSION__)\n    #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100)\n#endif\n\n#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK)\n    #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_COMPCERT_VERSION)\n    #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_PELLES_VERSION)\n    #undef JSON_HEDLEY_PELLES_VERSION\n#endif\n#if defined(__POCC__)\n    #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0)\n#endif\n\n#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK)\n    #undef JSON_HEDLEY_PELLES_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_PELLES_VERSION)\n    #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_MCST_LCC_VERSION)\n    #undef JSON_HEDLEY_MCST_LCC_VERSION\n#endif\n#if defined(__LCC__) && defined(__LCC_MINOR__)\n    #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__)\n#endif\n\n#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK)\n    #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_MCST_LCC_VERSION)\n    #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_VERSION)\n    #undef JSON_HEDLEY_GCC_VERSION\n#endif\n#if \\\n    defined(JSON_HEDLEY_GNUC_VERSION) && \\\n    !defined(__clang__) && \\\n    !defined(JSON_HEDLEY_INTEL_VERSION) && \\\n    !defined(JSON_HEDLEY_PGI_VERSION) && \\\n    !defined(JSON_HEDLEY_ARM_VERSION) && \\\n    !defined(JSON_HEDLEY_CRAY_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_CL430_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \\\n    !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \\\n    !defined(__COMPCERT__) && \\\n    !defined(JSON_HEDLEY_MCST_LCC_VERSION)\n    #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION\n#endif\n\n#if defined(JSON_HEDLEY_GCC_VERSION_CHECK)\n    #undef JSON_HEDLEY_GCC_VERSION_CHECK\n#endif\n#if defined(JSON_HEDLEY_GCC_VERSION)\n    #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch))\n#else\n    #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_ATTRIBUTE)\n    #undef JSON_HEDLEY_HAS_ATTRIBUTE\n#endif\n#if \\\n  defined(__has_attribute) && \\\n  ( \\\n    (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \\\n  )\n#  define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute)\n#else\n#  define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE)\n    #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE\n#endif\n#if defined(__has_attribute)\n    #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE)\n    #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE\n#endif\n#if defined(__has_attribute)\n    #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)\n#else\n    #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE)\n    #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE\n#endif\n#if \\\n    defined(__has_cpp_attribute) && \\\n    defined(__cplusplus) && \\\n    (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0))\n    #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute)\n#else\n    #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS)\n    #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS\n#endif\n#if !defined(__cplusplus) || !defined(__has_cpp_attribute)\n    #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0)\n#elif \\\n    !defined(JSON_HEDLEY_PGI_VERSION) && \\\n    !defined(JSON_HEDLEY_IAR_VERSION) && \\\n    (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \\\n    (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0))\n    #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute)\n#else\n    #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE)\n    #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE\n#endif\n#if defined(__has_cpp_attribute) && defined(__cplusplus)\n    #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE)\n    #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE\n#endif\n#if defined(__has_cpp_attribute) && defined(__cplusplus)\n    #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute)\n#else\n    #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_BUILTIN)\n    #undef JSON_HEDLEY_HAS_BUILTIN\n#endif\n#if defined(__has_builtin)\n    #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin)\n#else\n    #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN)\n    #undef JSON_HEDLEY_GNUC_HAS_BUILTIN\n#endif\n#if defined(__has_builtin)\n    #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN)\n    #undef JSON_HEDLEY_GCC_HAS_BUILTIN\n#endif\n#if defined(__has_builtin)\n    #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin)\n#else\n    #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_FEATURE)\n    #undef JSON_HEDLEY_HAS_FEATURE\n#endif\n#if defined(__has_feature)\n    #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature)\n#else\n    #define JSON_HEDLEY_HAS_FEATURE(feature) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE)\n    #undef JSON_HEDLEY_GNUC_HAS_FEATURE\n#endif\n#if defined(__has_feature)\n    #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_FEATURE)\n    #undef JSON_HEDLEY_GCC_HAS_FEATURE\n#endif\n#if defined(__has_feature)\n    #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature)\n#else\n    #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_EXTENSION)\n    #undef JSON_HEDLEY_HAS_EXTENSION\n#endif\n#if defined(__has_extension)\n    #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension)\n#else\n    #define JSON_HEDLEY_HAS_EXTENSION(extension) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION)\n    #undef JSON_HEDLEY_GNUC_HAS_EXTENSION\n#endif\n#if defined(__has_extension)\n    #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION)\n    #undef JSON_HEDLEY_GCC_HAS_EXTENSION\n#endif\n#if defined(__has_extension)\n    #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension)\n#else\n    #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE)\n    #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE\n#endif\n#if defined(__has_declspec_attribute)\n    #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute)\n#else\n    #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE)\n    #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE\n#endif\n#if defined(__has_declspec_attribute)\n    #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE)\n    #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE\n#endif\n#if defined(__has_declspec_attribute)\n    #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute)\n#else\n    #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_HAS_WARNING)\n    #undef JSON_HEDLEY_HAS_WARNING\n#endif\n#if defined(__has_warning)\n    #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning)\n#else\n    #define JSON_HEDLEY_HAS_WARNING(warning) (0)\n#endif\n\n#if defined(JSON_HEDLEY_GNUC_HAS_WARNING)\n    #undef JSON_HEDLEY_GNUC_HAS_WARNING\n#endif\n#if defined(__has_warning)\n    #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning)\n#else\n    #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_GCC_HAS_WARNING)\n    #undef JSON_HEDLEY_GCC_HAS_WARNING\n#endif\n#if defined(__has_warning)\n    #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning)\n#else\n    #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if \\\n    (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \\\n    defined(__clang__) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \\\n    JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \\\n    (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR))\n    #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value)\n#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)\n    #define JSON_HEDLEY_PRAGMA(value) __pragma(value)\n#else\n    #define JSON_HEDLEY_PRAGMA(value)\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH)\n    #undef JSON_HEDLEY_DIAGNOSTIC_PUSH\n#endif\n#if defined(JSON_HEDLEY_DIAGNOSTIC_POP)\n    #undef JSON_HEDLEY_DIAGNOSTIC_POP\n#endif\n#if defined(__clang__)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"clang diagnostic push\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"clang diagnostic pop\")\n#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"warning(push)\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"warning(pop)\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"GCC diagnostic push\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"GCC diagnostic pop\")\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push))\n    #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop))\n#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"push\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"pop\")\n#elif \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"diag_push\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"diag_pop\")\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma(\"warning(push)\")\n    #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma(\"warning(pop)\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_PUSH\n    #define JSON_HEDLEY_DIAGNOSTIC_POP\n#endif\n\n/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for\n   HEDLEY INTERNAL USE ONLY.  API subject to change without notice. */\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_\n#endif\n#if defined(__cplusplus)\n#  if JSON_HEDLEY_HAS_WARNING(\"-Wc++98-compat\")\n#    if JSON_HEDLEY_HAS_WARNING(\"-Wc++17-extensions\")\n#      if JSON_HEDLEY_HAS_WARNING(\"-Wc++1z-extensions\")\n#        define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++98-compat\\\"\") \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++17-extensions\\\"\") \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++1z-extensions\\\"\") \\\n    xpr \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#      else\n#        define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++98-compat\\\"\") \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++17-extensions\\\"\") \\\n    xpr \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#      endif\n#    else\n#      define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wc++98-compat\\\"\") \\\n    xpr \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#    endif\n#  endif\n#endif\n#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x\n#endif\n\n#if defined(JSON_HEDLEY_CONST_CAST)\n    #undef JSON_HEDLEY_CONST_CAST\n#endif\n#if defined(__cplusplus)\n#  define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast<T>(expr))\n#elif \\\n  JSON_HEDLEY_HAS_WARNING(\"-Wcast-qual\") || \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n#  define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \\\n        JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n        JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \\\n        ((T) (expr)); \\\n        JSON_HEDLEY_DIAGNOSTIC_POP \\\n    }))\n#else\n#  define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr))\n#endif\n\n#if defined(JSON_HEDLEY_REINTERPRET_CAST)\n    #undef JSON_HEDLEY_REINTERPRET_CAST\n#endif\n#if defined(__cplusplus)\n    #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast<T>(expr))\n#else\n    #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr))\n#endif\n\n#if defined(JSON_HEDLEY_STATIC_CAST)\n    #undef JSON_HEDLEY_STATIC_CAST\n#endif\n#if defined(__cplusplus)\n    #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast<T>(expr))\n#else\n    #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr))\n#endif\n\n#if defined(JSON_HEDLEY_CPP_CAST)\n    #undef JSON_HEDLEY_CPP_CAST\n#endif\n#if defined(__cplusplus)\n#  if JSON_HEDLEY_HAS_WARNING(\"-Wold-style-cast\")\n#    define JSON_HEDLEY_CPP_CAST(T, expr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wold-style-cast\\\"\") \\\n    ((T) (expr)) \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#  elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0)\n#    define JSON_HEDLEY_CPP_CAST(T, expr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"diag_suppress=Pe137\") \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#  else\n#    define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr))\n#  endif\n#else\n#  define JSON_HEDLEY_CPP_CAST(T, expr) (expr)\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wdeprecated-declarations\")\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"clang diagnostic ignored \\\"-Wdeprecated-declarations\\\"\")\n#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"warning(disable:1478 1786)\")\n#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786))\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"diag_suppress 1215,1216,1444,1445\")\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"diag_suppress 1215,1444\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"GCC diagnostic ignored \\\"-Wdeprecated-declarations\\\"\")\n#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996))\n#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"diag_suppress 1215,1444\")\n#elif \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"diag_suppress 1291,1718\")\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)\")\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"error_messages(off,symdeprecated,symdeprecated2)\")\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"diag_suppress=Pe1444,Pe1215\")\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma(\"warn(disable:2241)\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wunknown-pragmas\")\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"clang diagnostic ignored \\\"-Wunknown-pragmas\\\"\")\n#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"warning(disable:161)\")\n#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161))\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"diag_suppress 1675\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"GCC diagnostic ignored \\\"-Wunknown-pragmas\\\"\")\n#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068))\n#elif \\\n    JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"diag_suppress 163\")\n#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"diag_suppress 163\")\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"diag_suppress=Pe161\")\n#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma(\"diag_suppress 161\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wunknown-attributes\")\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"clang diagnostic ignored \\\"-Wunknown-attributes\\\"\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"GCC diagnostic ignored \\\"-Wdeprecated-declarations\\\"\")\n#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"warning(disable:1292)\")\n#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292))\n#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030))\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"diag_suppress 1097,1098\")\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"diag_suppress 1097\")\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"error_messages(off,attrskipunsup)\")\n#elif \\\n    JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"diag_suppress 1173\")\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"diag_suppress=Pe1097\")\n#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma(\"diag_suppress 1097\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wcast-qual\")\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma(\"clang diagnostic ignored \\\"-Wcast-qual\\\"\")\n#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma(\"warning(disable:2203 2331)\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma(\"GCC diagnostic ignored \\\"-Wcast-qual\\\"\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL\n#endif\n\n#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION)\n    #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wunused-function\")\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma(\"clang diagnostic ignored \\\"-Wunused-function\\\"\")\n#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma(\"GCC diagnostic ignored \\\"-Wunused-function\\\"\")\n#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505))\n#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma(\"diag_suppress 3142\")\n#else\n    #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION\n#endif\n\n#if defined(JSON_HEDLEY_DEPRECATED)\n    #undef JSON_HEDLEY_DEPRECATED\n#endif\n#if defined(JSON_HEDLEY_DEPRECATED_FOR)\n    #undef JSON_HEDLEY_DEPRECATED_FOR\n#endif\n#if \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated(\"Since \" # since))\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated(\"Since \" #since \"; use \" #replacement))\n#elif \\\n    (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__(\"Since \" #since)))\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__(\"Since \" #since \"; use \" #replacement)))\n#elif defined(__cplusplus) && (__cplusplus >= 201402L)\n    #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated(\"Since \" #since)]])\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated(\"Since \" #since \"; use \" #replacement)]])\n#elif \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \\\n    JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)\n    #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__))\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__))\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \\\n    JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated)\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated)\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_DEPRECATED(since) _Pragma(\"deprecated\")\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma(\"deprecated\")\n#else\n    #define JSON_HEDLEY_DEPRECATED(since)\n    #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement)\n#endif\n\n#if defined(JSON_HEDLEY_UNAVAILABLE)\n    #undef JSON_HEDLEY_UNAVAILABLE\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__(\"Not available until \" #available_since)))\n#else\n    #define JSON_HEDLEY_UNAVAILABLE(available_since)\n#endif\n\n#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT)\n    #undef JSON_HEDLEY_WARN_UNUSED_RESULT\n#endif\n#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG)\n    #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__))\n#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L)\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]])\n#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard)\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]])\n#elif defined(_Check_return_) /* SAL */\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_\n#else\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT\n    #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg)\n#endif\n\n#if defined(JSON_HEDLEY_SENTINEL)\n    #undef JSON_HEDLEY_SENTINEL\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position)))\n#else\n    #define JSON_HEDLEY_SENTINEL(position)\n#endif\n\n#if defined(JSON_HEDLEY_NO_RETURN)\n    #undef JSON_HEDLEY_NO_RETURN\n#endif\n#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_NO_RETURN __noreturn\n#elif \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__))\n#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L\n    #define JSON_HEDLEY_NO_RETURN _Noreturn\n#elif defined(__cplusplus) && (__cplusplus >= 201103L)\n    #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]])\n#elif \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)\n    #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__))\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)\n    #define JSON_HEDLEY_NO_RETURN _Pragma(\"does_not_return\")\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_NO_RETURN __declspec(noreturn)\n#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus)\n    #define JSON_HEDLEY_NO_RETURN _Pragma(\"FUNC_NEVER_RETURNS;\")\n#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0)\n    #define JSON_HEDLEY_NO_RETURN __attribute((noreturn))\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0)\n    #define JSON_HEDLEY_NO_RETURN __declspec(noreturn)\n#else\n    #define JSON_HEDLEY_NO_RETURN\n#endif\n\n#if defined(JSON_HEDLEY_NO_ESCAPE)\n    #undef JSON_HEDLEY_NO_ESCAPE\n#endif\n#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape)\n    #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__))\n#else\n    #define JSON_HEDLEY_NO_ESCAPE\n#endif\n\n#if defined(JSON_HEDLEY_UNREACHABLE)\n    #undef JSON_HEDLEY_UNREACHABLE\n#endif\n#if defined(JSON_HEDLEY_UNREACHABLE_RETURN)\n    #undef JSON_HEDLEY_UNREACHABLE_RETURN\n#endif\n#if defined(JSON_HEDLEY_ASSUME)\n    #undef JSON_HEDLEY_ASSUME\n#endif\n#if \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_ASSUME(expr) __assume(expr)\n#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume)\n    #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr)\n#elif \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0)\n    #if defined(__cplusplus)\n        #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr)\n    #else\n        #define JSON_HEDLEY_ASSUME(expr) _nassert(expr)\n    #endif\n#endif\n#if \\\n    (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \\\n    JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable()\n#elif defined(JSON_HEDLEY_ASSUME)\n    #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0)\n#endif\n#if !defined(JSON_HEDLEY_ASSUME)\n    #if defined(JSON_HEDLEY_UNREACHABLE)\n        #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1)))\n    #else\n        #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr)\n    #endif\n#endif\n#if defined(JSON_HEDLEY_UNREACHABLE)\n    #if  \\\n        JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \\\n        JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0)\n        #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value))\n    #else\n        #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE()\n    #endif\n#else\n    #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value)\n#endif\n#if !defined(JSON_HEDLEY_UNREACHABLE)\n    #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0)\n#endif\n\nJSON_HEDLEY_DIAGNOSTIC_PUSH\n#if JSON_HEDLEY_HAS_WARNING(\"-Wpedantic\")\n    #pragma clang diagnostic ignored \"-Wpedantic\"\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wc++98-compat-pedantic\") && defined(__cplusplus)\n    #pragma clang diagnostic ignored \"-Wc++98-compat-pedantic\"\n#endif\n#if JSON_HEDLEY_GCC_HAS_WARNING(\"-Wvariadic-macros\",4,0,0)\n    #if defined(__clang__)\n        #pragma clang diagnostic ignored \"-Wvariadic-macros\"\n    #elif defined(JSON_HEDLEY_GCC_VERSION)\n        #pragma GCC diagnostic ignored \"-Wvariadic-macros\"\n    #endif\n#endif\n#if defined(JSON_HEDLEY_NON_NULL)\n    #undef JSON_HEDLEY_NON_NULL\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0)\n    #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__)))\n#else\n    #define JSON_HEDLEY_NON_NULL(...)\n#endif\nJSON_HEDLEY_DIAGNOSTIC_POP\n\n#if defined(JSON_HEDLEY_PRINTF_FORMAT)\n    #undef JSON_HEDLEY_PRINTF_FORMAT\n#endif\n#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO)\n    #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check)))\n#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO)\n    #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check)))\n#elif \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(format) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check)))\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0)\n    #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check))\n#else\n    #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check)\n#endif\n\n#if defined(JSON_HEDLEY_CONSTEXPR)\n    #undef JSON_HEDLEY_CONSTEXPR\n#endif\n#if defined(__cplusplus)\n    #if __cplusplus >= 201103L\n        #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr)\n    #endif\n#endif\n#if !defined(JSON_HEDLEY_CONSTEXPR)\n    #define JSON_HEDLEY_CONSTEXPR\n#endif\n\n#if defined(JSON_HEDLEY_PREDICT)\n    #undef JSON_HEDLEY_PREDICT\n#endif\n#if defined(JSON_HEDLEY_LIKELY)\n    #undef JSON_HEDLEY_LIKELY\n#endif\n#if defined(JSON_HEDLEY_UNLIKELY)\n    #undef JSON_HEDLEY_UNLIKELY\n#endif\n#if defined(JSON_HEDLEY_UNPREDICTABLE)\n    #undef JSON_HEDLEY_UNPREDICTABLE\n#endif\n#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable)\n    #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr))\n#endif\n#if \\\n  (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \\\n  JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n#  define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability(  (expr), (value), (probability))\n#  define JSON_HEDLEY_PREDICT_TRUE(expr, probability)   __builtin_expect_with_probability(!!(expr),    1   , (probability))\n#  define JSON_HEDLEY_PREDICT_FALSE(expr, probability)  __builtin_expect_with_probability(!!(expr),    0   , (probability))\n#  define JSON_HEDLEY_LIKELY(expr)                      __builtin_expect                 (!!(expr),    1                  )\n#  define JSON_HEDLEY_UNLIKELY(expr)                    __builtin_expect                 (!!(expr),    0                  )\n#elif \\\n  (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n  (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \\\n  JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n  JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n  JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n  JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \\\n  JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \\\n  JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \\\n  JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \\\n  JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n  JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n  JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \\\n  JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \\\n  JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n#  define JSON_HEDLEY_PREDICT(expr, expected, probability) \\\n    (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)))\n#  define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \\\n    (__extension__ ({ \\\n        double hedley_probability_ = (probability); \\\n        ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \\\n    }))\n#  define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \\\n    (__extension__ ({ \\\n        double hedley_probability_ = (probability); \\\n        ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \\\n    }))\n#  define JSON_HEDLEY_LIKELY(expr)   __builtin_expect(!!(expr), 1)\n#  define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0)\n#else\n#  define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))\n#  define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr))\n#  define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr))\n#  define JSON_HEDLEY_LIKELY(expr) (!!(expr))\n#  define JSON_HEDLEY_UNLIKELY(expr) (!!(expr))\n#endif\n#if !defined(JSON_HEDLEY_UNPREDICTABLE)\n    #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5)\n#endif\n\n#if defined(JSON_HEDLEY_MALLOC)\n    #undef JSON_HEDLEY_MALLOC\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_MALLOC __attribute__((__malloc__))\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)\n    #define JSON_HEDLEY_MALLOC _Pragma(\"returns_new_memory\")\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_MALLOC __declspec(restrict)\n#else\n    #define JSON_HEDLEY_MALLOC\n#endif\n\n#if defined(JSON_HEDLEY_PURE)\n    #undef JSON_HEDLEY_PURE\n#endif\n#if \\\n  JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n  JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n  JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n  JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n  JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n  (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n  (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n  (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n  (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n  JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n  JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n  JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \\\n  JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n#  define JSON_HEDLEY_PURE __attribute__((__pure__))\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)\n#  define JSON_HEDLEY_PURE _Pragma(\"does_not_write_global_data\")\n#elif defined(__cplusplus) && \\\n    ( \\\n      JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \\\n      JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \\\n      JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \\\n    )\n#  define JSON_HEDLEY_PURE _Pragma(\"FUNC_IS_PURE;\")\n#else\n#  define JSON_HEDLEY_PURE\n#endif\n\n#if defined(JSON_HEDLEY_CONST)\n    #undef JSON_HEDLEY_CONST\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(const) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_CONST __attribute__((__const__))\n#elif \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0)\n    #define JSON_HEDLEY_CONST _Pragma(\"no_side_effect\")\n#else\n    #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE\n#endif\n\n#if defined(JSON_HEDLEY_RESTRICT)\n    #undef JSON_HEDLEY_RESTRICT\n#endif\n#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus)\n    #define JSON_HEDLEY_RESTRICT restrict\n#elif \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n    JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \\\n    JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \\\n    defined(__clang__) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_RESTRICT __restrict\n#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus)\n    #define JSON_HEDLEY_RESTRICT _Restrict\n#else\n    #define JSON_HEDLEY_RESTRICT\n#endif\n\n#if defined(JSON_HEDLEY_INLINE)\n    #undef JSON_HEDLEY_INLINE\n#endif\n#if \\\n    (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \\\n    (defined(__cplusplus) && (__cplusplus >= 199711L))\n    #define JSON_HEDLEY_INLINE inline\n#elif \\\n    defined(JSON_HEDLEY_GCC_VERSION) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0)\n    #define JSON_HEDLEY_INLINE __inline__\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_INLINE __inline\n#else\n    #define JSON_HEDLEY_INLINE\n#endif\n\n#if defined(JSON_HEDLEY_ALWAYS_INLINE)\n    #undef JSON_HEDLEY_ALWAYS_INLINE\n#endif\n#if \\\n  JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n  JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n  JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n  JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n  JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n  (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n  (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n  (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n  (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n  JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n  JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n  JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n  JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \\\n  JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)\n#  define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE\n#elif \\\n  JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \\\n  JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n#  define JSON_HEDLEY_ALWAYS_INLINE __forceinline\n#elif defined(__cplusplus) && \\\n    ( \\\n      JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n      JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n      JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n      JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \\\n      JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n      JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \\\n    )\n#  define JSON_HEDLEY_ALWAYS_INLINE _Pragma(\"FUNC_ALWAYS_INLINE;\")\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n#  define JSON_HEDLEY_ALWAYS_INLINE _Pragma(\"inline=forced\")\n#else\n#  define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE\n#endif\n\n#if defined(JSON_HEDLEY_NEVER_INLINE)\n    #undef JSON_HEDLEY_NEVER_INLINE\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \\\n    JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \\\n    (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \\\n    (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \\\n    (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \\\n    (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \\\n    JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \\\n    JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \\\n    JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0)\n    #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__))\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline)\n#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0)\n    #define JSON_HEDLEY_NEVER_INLINE _Pragma(\"noinline\")\n#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus)\n    #define JSON_HEDLEY_NEVER_INLINE _Pragma(\"FUNC_CANNOT_INLINE;\")\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n    #define JSON_HEDLEY_NEVER_INLINE _Pragma(\"inline=never\")\n#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0)\n    #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline))\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0)\n    #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline)\n#else\n    #define JSON_HEDLEY_NEVER_INLINE\n#endif\n\n#if defined(JSON_HEDLEY_PRIVATE)\n    #undef JSON_HEDLEY_PRIVATE\n#endif\n#if defined(JSON_HEDLEY_PUBLIC)\n    #undef JSON_HEDLEY_PUBLIC\n#endif\n#if defined(JSON_HEDLEY_IMPORT)\n    #undef JSON_HEDLEY_IMPORT\n#endif\n#if defined(_WIN32) || defined(__CYGWIN__)\n#  define JSON_HEDLEY_PRIVATE\n#  define JSON_HEDLEY_PUBLIC   __declspec(dllexport)\n#  define JSON_HEDLEY_IMPORT   __declspec(dllimport)\n#else\n#  if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \\\n    JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \\\n    ( \\\n      defined(__TI_EABI__) && \\\n      ( \\\n        (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \\\n        JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \\\n      ) \\\n    ) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n#    define JSON_HEDLEY_PRIVATE __attribute__((__visibility__(\"hidden\")))\n#    define JSON_HEDLEY_PUBLIC  __attribute__((__visibility__(\"default\")))\n#  else\n#    define JSON_HEDLEY_PRIVATE\n#    define JSON_HEDLEY_PUBLIC\n#  endif\n#  define JSON_HEDLEY_IMPORT    extern\n#endif\n\n#if defined(JSON_HEDLEY_NO_THROW)\n    #undef JSON_HEDLEY_NO_THROW\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__))\n#elif \\\n    JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0)\n    #define JSON_HEDLEY_NO_THROW __declspec(nothrow)\n#else\n    #define JSON_HEDLEY_NO_THROW\n#endif\n\n#if defined(JSON_HEDLEY_FALL_THROUGH)\n    #undef JSON_HEDLEY_FALL_THROUGH\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__))\n#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough)\n    #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]])\n#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough)\n    #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]])\n#elif defined(__fallthrough) /* SAL */\n    #define JSON_HEDLEY_FALL_THROUGH __fallthrough\n#else\n    #define JSON_HEDLEY_FALL_THROUGH\n#endif\n\n#if defined(JSON_HEDLEY_RETURNS_NON_NULL)\n    #undef JSON_HEDLEY_RETURNS_NON_NULL\n#endif\n#if \\\n    JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__))\n#elif defined(_Ret_notnull_) /* SAL */\n    #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_\n#else\n    #define JSON_HEDLEY_RETURNS_NON_NULL\n#endif\n\n#if defined(JSON_HEDLEY_ARRAY_PARAM)\n    #undef JSON_HEDLEY_ARRAY_PARAM\n#endif\n#if \\\n    defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \\\n    !defined(__STDC_NO_VLA__) && \\\n    !defined(__cplusplus) && \\\n    !defined(JSON_HEDLEY_PGI_VERSION) && \\\n    !defined(JSON_HEDLEY_TINYC_VERSION)\n    #define JSON_HEDLEY_ARRAY_PARAM(name) (name)\n#else\n    #define JSON_HEDLEY_ARRAY_PARAM(name)\n#endif\n\n#if defined(JSON_HEDLEY_IS_CONSTANT)\n    #undef JSON_HEDLEY_IS_CONSTANT\n#endif\n#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR)\n    #undef JSON_HEDLEY_REQUIRE_CONSTEXPR\n#endif\n/* JSON_HEDLEY_IS_CONSTEXPR_ is for\n   HEDLEY INTERNAL USE ONLY.  API subject to change without notice. */\n#if defined(JSON_HEDLEY_IS_CONSTEXPR_)\n    #undef JSON_HEDLEY_IS_CONSTEXPR_\n#endif\n#if \\\n    JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \\\n    JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \\\n    JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n    JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \\\n    JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \\\n    JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \\\n    JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \\\n    (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \\\n    JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \\\n    JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10)\n    #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr)\n#endif\n#if !defined(__cplusplus)\n#  if \\\n       JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \\\n       JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \\\n       JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n       JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \\\n       JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \\\n       JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \\\n       JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24)\n#if defined(__INTPTR_TYPE__)\n    #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*)\n#else\n    #include <stdint.h>\n    #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*)\n#endif\n#  elif \\\n       ( \\\n          defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \\\n          !defined(JSON_HEDLEY_SUNPRO_VERSION) && \\\n          !defined(JSON_HEDLEY_PGI_VERSION) && \\\n          !defined(JSON_HEDLEY_IAR_VERSION)) || \\\n       (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \\\n       JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \\\n       JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \\\n       JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \\\n       JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0)\n#if defined(__INTPTR_TYPE__)\n    #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0)\n#else\n    #include <stdint.h>\n    #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0)\n#endif\n#  elif \\\n       defined(JSON_HEDLEY_GCC_VERSION) || \\\n       defined(JSON_HEDLEY_INTEL_VERSION) || \\\n       defined(JSON_HEDLEY_TINYC_VERSION) || \\\n       defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \\\n       JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \\\n       defined(JSON_HEDLEY_TI_CL2000_VERSION) || \\\n       defined(JSON_HEDLEY_TI_CL6X_VERSION) || \\\n       defined(JSON_HEDLEY_TI_CL7X_VERSION) || \\\n       defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \\\n       defined(__clang__)\n#    define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \\\n        sizeof(void) != \\\n        sizeof(*( \\\n                  1 ? \\\n                  ((void*) ((expr) * 0L) ) : \\\n((struct { char v[sizeof(void) * 2]; } *) 1) \\\n                ) \\\n              ) \\\n                                            )\n#  endif\n#endif\n#if defined(JSON_HEDLEY_IS_CONSTEXPR_)\n    #if !defined(JSON_HEDLEY_IS_CONSTANT)\n        #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr)\n    #endif\n    #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1))\n#else\n    #if !defined(JSON_HEDLEY_IS_CONSTANT)\n        #define JSON_HEDLEY_IS_CONSTANT(expr) (0)\n    #endif\n    #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr)\n#endif\n\n#if defined(JSON_HEDLEY_BEGIN_C_DECLS)\n    #undef JSON_HEDLEY_BEGIN_C_DECLS\n#endif\n#if defined(JSON_HEDLEY_END_C_DECLS)\n    #undef JSON_HEDLEY_END_C_DECLS\n#endif\n#if defined(JSON_HEDLEY_C_DECL)\n    #undef JSON_HEDLEY_C_DECL\n#endif\n#if defined(__cplusplus)\n    #define JSON_HEDLEY_BEGIN_C_DECLS extern \"C\" {\n    #define JSON_HEDLEY_END_C_DECLS }\n    #define JSON_HEDLEY_C_DECL extern \"C\"\n#else\n    #define JSON_HEDLEY_BEGIN_C_DECLS\n    #define JSON_HEDLEY_END_C_DECLS\n    #define JSON_HEDLEY_C_DECL\n#endif\n\n#if defined(JSON_HEDLEY_STATIC_ASSERT)\n    #undef JSON_HEDLEY_STATIC_ASSERT\n#endif\n#if \\\n  !defined(__cplusplus) && ( \\\n      (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \\\n      (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \\\n      JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \\\n      JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \\\n      defined(_Static_assert) \\\n    )\n#  define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message)\n#elif \\\n  (defined(__cplusplus) && (__cplusplus >= 201103L)) || \\\n  JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \\\n  JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n#  define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message))\n#else\n#  define JSON_HEDLEY_STATIC_ASSERT(expr, message)\n#endif\n\n#if defined(JSON_HEDLEY_NULL)\n    #undef JSON_HEDLEY_NULL\n#endif\n#if defined(__cplusplus)\n    #if __cplusplus >= 201103L\n        #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr)\n    #elif defined(NULL)\n        #define JSON_HEDLEY_NULL NULL\n    #else\n        #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0)\n    #endif\n#elif defined(NULL)\n    #define JSON_HEDLEY_NULL NULL\n#else\n    #define JSON_HEDLEY_NULL ((void*) 0)\n#endif\n\n#if defined(JSON_HEDLEY_MESSAGE)\n    #undef JSON_HEDLEY_MESSAGE\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wunknown-pragmas\")\n#  define JSON_HEDLEY_MESSAGE(msg) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \\\n    JSON_HEDLEY_PRAGMA(message msg) \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#elif \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n#  define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg)\n#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0)\n#  define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg)\n#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0)\n#  define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg))\n#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0)\n#  define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg))\n#else\n#  define JSON_HEDLEY_MESSAGE(msg)\n#endif\n\n#if defined(JSON_HEDLEY_WARNING)\n    #undef JSON_HEDLEY_WARNING\n#endif\n#if JSON_HEDLEY_HAS_WARNING(\"-Wunknown-pragmas\")\n#  define JSON_HEDLEY_WARNING(msg) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \\\n    JSON_HEDLEY_PRAGMA(clang warning msg) \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#elif \\\n  JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \\\n  JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \\\n  JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0)\n#  define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg)\n#elif \\\n  JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \\\n  JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n#  define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg))\n#else\n#  define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg)\n#endif\n\n#if defined(JSON_HEDLEY_REQUIRE)\n    #undef JSON_HEDLEY_REQUIRE\n#endif\n#if defined(JSON_HEDLEY_REQUIRE_MSG)\n    #undef JSON_HEDLEY_REQUIRE_MSG\n#endif\n#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if)\n#  if JSON_HEDLEY_HAS_WARNING(\"-Wgcc-compat\")\n#    define JSON_HEDLEY_REQUIRE(expr) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wgcc-compat\\\"\") \\\n    __attribute__((diagnose_if(!(expr), #expr, \"error\"))) \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#    define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \\\n    JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n    _Pragma(\"clang diagnostic ignored \\\"-Wgcc-compat\\\"\") \\\n    __attribute__((diagnose_if(!(expr), msg, \"error\"))) \\\n    JSON_HEDLEY_DIAGNOSTIC_POP\n#  else\n#    define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, \"error\")))\n#    define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, \"error\")))\n#  endif\n#else\n#  define JSON_HEDLEY_REQUIRE(expr)\n#  define JSON_HEDLEY_REQUIRE_MSG(expr,msg)\n#endif\n\n#if defined(JSON_HEDLEY_FLAGS)\n    #undef JSON_HEDLEY_FLAGS\n#endif\n#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING(\"-Wbitfield-enum-conversion\"))\n    #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__))\n#else\n    #define JSON_HEDLEY_FLAGS\n#endif\n\n#if defined(JSON_HEDLEY_FLAGS_CAST)\n    #undef JSON_HEDLEY_FLAGS_CAST\n#endif\n#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0)\n#  define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \\\n        JSON_HEDLEY_DIAGNOSTIC_PUSH \\\n        _Pragma(\"warning(disable:188)\") \\\n        ((T) (expr)); \\\n        JSON_HEDLEY_DIAGNOSTIC_POP \\\n    }))\n#else\n#  define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr)\n#endif\n\n#if defined(JSON_HEDLEY_EMPTY_BASES)\n    #undef JSON_HEDLEY_EMPTY_BASES\n#endif\n#if \\\n    (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \\\n    JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0)\n    #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases)\n#else\n    #define JSON_HEDLEY_EMPTY_BASES\n#endif\n\n/* Remaining macros are deprecated. */\n\n#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK)\n    #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK\n#endif\n#if defined(__clang__)\n    #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0)\n#else\n    #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch)\n#endif\n\n#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE)\n    #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE\n#endif\n#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE)\n    #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE\n#endif\n#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN)\n    #undef JSON_HEDLEY_CLANG_HAS_BUILTIN\n#endif\n#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE)\n    #undef JSON_HEDLEY_CLANG_HAS_FEATURE\n#endif\n#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION)\n    #undef JSON_HEDLEY_CLANG_HAS_EXTENSION\n#endif\n#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE)\n    #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE\n#endif\n#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute)\n\n#if defined(JSON_HEDLEY_CLANG_HAS_WARNING)\n    #undef JSON_HEDLEY_CLANG_HAS_WARNING\n#endif\n#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning)\n\n#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */\n\n\n// This file contains all internal macro definitions (except those affecting ABI)\n// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\n// exclude unsupported compilers\n#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK)\n    #if defined(__clang__)\n        #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400\n            #error \"unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers\"\n        #endif\n    #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER))\n        #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800\n            #error \"unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers\"\n        #endif\n    #endif\n#endif\n\n// C++ language standard detection\n// if the user manually specified the used c++ version this is skipped\n#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11)\n    #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)\n        #define JSON_HAS_CPP_20\n        #define JSON_HAS_CPP_17\n        #define JSON_HAS_CPP_14\n    #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464\n        #define JSON_HAS_CPP_17\n        #define JSON_HAS_CPP_14\n    #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1)\n        #define JSON_HAS_CPP_14\n    #endif\n    // the cpp 11 flag is always specified because it is the minimal required version\n    #define JSON_HAS_CPP_11\n#endif\n\n#ifdef __has_include\n    #if __has_include(<version>)\n        #include <version>\n    #endif\n#endif\n\n#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM)\n    #ifdef JSON_HAS_CPP_17\n        #if defined(__cpp_lib_filesystem)\n            #define JSON_HAS_FILESYSTEM 1\n        #elif defined(__cpp_lib_experimental_filesystem)\n            #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1\n        #elif !defined(__has_include)\n            #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1\n        #elif __has_include(<filesystem>)\n            #define JSON_HAS_FILESYSTEM 1\n        #elif __has_include(<experimental/filesystem>)\n            #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1\n        #endif\n\n        // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/\n        #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n\n        // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support\n        #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n\n        // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support\n        #if defined(__clang_major__) && __clang_major__ < 7\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n\n        // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support\n        #if defined(_MSC_VER) && _MSC_VER < 1914\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n\n        // no filesystem support before iOS 13\n        #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n\n        // no filesystem support before macOS Catalina\n        #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500\n            #undef JSON_HAS_FILESYSTEM\n            #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n        #endif\n    #endif\n#endif\n\n#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n    #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0\n#endif\n\n#ifndef JSON_HAS_FILESYSTEM\n    #define JSON_HAS_FILESYSTEM 0\n#endif\n\n#ifndef JSON_HAS_THREE_WAY_COMPARISON\n    #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \\\n        && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L\n        #define JSON_HAS_THREE_WAY_COMPARISON 1\n    #else\n        #define JSON_HAS_THREE_WAY_COMPARISON 0\n    #endif\n#endif\n\n#ifndef JSON_HAS_RANGES\n    // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has syntax error\n    #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427\n        #define JSON_HAS_RANGES 0\n    #elif defined(__cpp_lib_ranges)\n        #define JSON_HAS_RANGES 1\n    #else\n        #define JSON_HAS_RANGES 0\n    #endif\n#endif\n\n#ifdef JSON_HAS_CPP_17\n    #define JSON_INLINE_VARIABLE inline\n#else\n    #define JSON_INLINE_VARIABLE\n#endif\n\n#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address)\n    #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]]\n#else\n    #define JSON_NO_UNIQUE_ADDRESS\n#endif\n\n// disable documentation warnings on clang\n#if defined(__clang__)\n    #pragma clang diagnostic push\n    #pragma clang diagnostic ignored \"-Wdocumentation\"\n    #pragma clang diagnostic ignored \"-Wdocumentation-unknown-command\"\n#endif\n\n// allow disabling exceptions\n#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)\n    #define JSON_THROW(exception) throw exception\n    #define JSON_TRY try\n    #define JSON_CATCH(exception) catch(exception)\n    #define JSON_INTERNAL_CATCH(exception) catch(exception)\n#else\n    #include <cstdlib>\n    #define JSON_THROW(exception) std::abort()\n    #define JSON_TRY if(true)\n    #define JSON_CATCH(exception) if(false)\n    #define JSON_INTERNAL_CATCH(exception) if(false)\n#endif\n\n// override exception macros\n#if defined(JSON_THROW_USER)\n    #undef JSON_THROW\n    #define JSON_THROW JSON_THROW_USER\n#endif\n#if defined(JSON_TRY_USER)\n    #undef JSON_TRY\n    #define JSON_TRY JSON_TRY_USER\n#endif\n#if defined(JSON_CATCH_USER)\n    #undef JSON_CATCH\n    #define JSON_CATCH JSON_CATCH_USER\n    #undef JSON_INTERNAL_CATCH\n    #define JSON_INTERNAL_CATCH JSON_CATCH_USER\n#endif\n#if defined(JSON_INTERNAL_CATCH_USER)\n    #undef JSON_INTERNAL_CATCH\n    #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER\n#endif\n\n// allow overriding assert\n#if !defined(JSON_ASSERT)\n    #include <cassert> // assert\n    #define JSON_ASSERT(x) assert(x)\n#endif\n\n// allow to access some private functions (needed by the test suite)\n#if defined(JSON_TESTS_PRIVATE)\n    #define JSON_PRIVATE_UNLESS_TESTED public\n#else\n    #define JSON_PRIVATE_UNLESS_TESTED private\n#endif\n\n/*!\n@brief macro to briefly define a mapping between an enum and JSON\n@def NLOHMANN_JSON_SERIALIZE_ENUM\n@since version 3.4.0\n*/\n#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...)                                            \\\n    template<typename BasicJsonType>                                                            \\\n    inline void to_json(BasicJsonType& j, const ENUM_TYPE& e)                                   \\\n    {                                                                                           \\\n        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE \" must be an enum!\");          \\\n        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                     \\\n        auto it = std::find_if(std::begin(m), std::end(m),                                      \\\n                               [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool  \\\n        {                                                                                       \\\n            return ej_pair.first == e;                                                          \\\n        });                                                                                     \\\n        j = ((it != std::end(m)) ? it : std::begin(m))->second;                                 \\\n    }                                                                                           \\\n    template<typename BasicJsonType>                                                            \\\n    inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                 \\\n    {                                                                                           \\\n        static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE \" must be an enum!\");          \\\n        static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__;                     \\\n        auto it = std::find_if(std::begin(m), std::end(m),                                      \\\n                               [&j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \\\n        {                                                                                       \\\n            return ej_pair.second == j;                                                         \\\n        });                                                                                     \\\n        e = ((it != std::end(m)) ? it : std::begin(m))->first;                                  \\\n    }\n\n// Ugly macros to avoid uglier copy-paste when specializing basic_json. They\n// may be removed in the future once the class is split.\n\n#define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \\\n    template<template<typename, typename, typename...> class ObjectType,   \\\n             template<typename, typename...> class ArrayType,              \\\n             class StringType, class BooleanType, class NumberIntegerType, \\\n             class NumberUnsignedType, class NumberFloatType,              \\\n             template<typename> class AllocatorType,                       \\\n             template<typename, typename = void> class JSONSerializer,     \\\n             class BinaryType>\n\n#define NLOHMANN_BASIC_JSON_TPL                                            \\\n    basic_json<ObjectType, ArrayType, StringType, BooleanType,             \\\n    NumberIntegerType, NumberUnsignedType, NumberFloatType,                \\\n    AllocatorType, JSONSerializer, BinaryType>\n\n// Macros to simplify conversion from/to types\n\n#define NLOHMANN_JSON_EXPAND( x ) x\n#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME\n#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \\\n        NLOHMANN_JSON_PASTE64, \\\n        NLOHMANN_JSON_PASTE63, \\\n        NLOHMANN_JSON_PASTE62, \\\n        NLOHMANN_JSON_PASTE61, \\\n        NLOHMANN_JSON_PASTE60, \\\n        NLOHMANN_JSON_PASTE59, \\\n        NLOHMANN_JSON_PASTE58, \\\n        NLOHMANN_JSON_PASTE57, \\\n        NLOHMANN_JSON_PASTE56, \\\n        NLOHMANN_JSON_PASTE55, \\\n        NLOHMANN_JSON_PASTE54, \\\n        NLOHMANN_JSON_PASTE53, \\\n        NLOHMANN_JSON_PASTE52, \\\n        NLOHMANN_JSON_PASTE51, \\\n        NLOHMANN_JSON_PASTE50, \\\n        NLOHMANN_JSON_PASTE49, \\\n        NLOHMANN_JSON_PASTE48, \\\n        NLOHMANN_JSON_PASTE47, \\\n        NLOHMANN_JSON_PASTE46, \\\n        NLOHMANN_JSON_PASTE45, \\\n        NLOHMANN_JSON_PASTE44, \\\n        NLOHMANN_JSON_PASTE43, \\\n        NLOHMANN_JSON_PASTE42, \\\n        NLOHMANN_JSON_PASTE41, \\\n        NLOHMANN_JSON_PASTE40, \\\n        NLOHMANN_JSON_PASTE39, \\\n        NLOHMANN_JSON_PASTE38, \\\n        NLOHMANN_JSON_PASTE37, \\\n        NLOHMANN_JSON_PASTE36, \\\n        NLOHMANN_JSON_PASTE35, \\\n        NLOHMANN_JSON_PASTE34, \\\n        NLOHMANN_JSON_PASTE33, \\\n        NLOHMANN_JSON_PASTE32, \\\n        NLOHMANN_JSON_PASTE31, \\\n        NLOHMANN_JSON_PASTE30, \\\n        NLOHMANN_JSON_PASTE29, \\\n        NLOHMANN_JSON_PASTE28, \\\n        NLOHMANN_JSON_PASTE27, \\\n        NLOHMANN_JSON_PASTE26, \\\n        NLOHMANN_JSON_PASTE25, \\\n        NLOHMANN_JSON_PASTE24, \\\n        NLOHMANN_JSON_PASTE23, \\\n        NLOHMANN_JSON_PASTE22, \\\n        NLOHMANN_JSON_PASTE21, \\\n        NLOHMANN_JSON_PASTE20, \\\n        NLOHMANN_JSON_PASTE19, \\\n        NLOHMANN_JSON_PASTE18, \\\n        NLOHMANN_JSON_PASTE17, \\\n        NLOHMANN_JSON_PASTE16, \\\n        NLOHMANN_JSON_PASTE15, \\\n        NLOHMANN_JSON_PASTE14, \\\n        NLOHMANN_JSON_PASTE13, \\\n        NLOHMANN_JSON_PASTE12, \\\n        NLOHMANN_JSON_PASTE11, \\\n        NLOHMANN_JSON_PASTE10, \\\n        NLOHMANN_JSON_PASTE9, \\\n        NLOHMANN_JSON_PASTE8, \\\n        NLOHMANN_JSON_PASTE7, \\\n        NLOHMANN_JSON_PASTE6, \\\n        NLOHMANN_JSON_PASTE5, \\\n        NLOHMANN_JSON_PASTE4, \\\n        NLOHMANN_JSON_PASTE3, \\\n        NLOHMANN_JSON_PASTE2, \\\n        NLOHMANN_JSON_PASTE1)(__VA_ARGS__))\n#define NLOHMANN_JSON_PASTE2(func, v1) func(v1)\n#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2)\n#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3)\n#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4)\n#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5)\n#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6)\n#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7)\n#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8)\n#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9)\n#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10)\n#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11)\n#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12)\n#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13)\n#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14)\n#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15)\n#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16)\n#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17)\n#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18)\n#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19)\n#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20)\n#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21)\n#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22)\n#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23)\n#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24)\n#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25)\n#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26)\n#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27)\n#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28)\n#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29)\n#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30)\n#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31)\n#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32)\n#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33)\n#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34)\n#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35)\n#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36)\n#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37)\n#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38)\n#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39)\n#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40)\n#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41)\n#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42)\n#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43)\n#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44)\n#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45)\n#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46)\n#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47)\n#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48)\n#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49)\n#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50)\n#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51)\n#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52)\n#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53)\n#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54)\n#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55)\n#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56)\n#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57)\n#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58)\n#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59)\n#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60)\n#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61)\n#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62)\n#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63)\n\n#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1;\n#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1);\n#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1);\n\n/*!\n@brief macro\n@def NLOHMANN_DEFINE_TYPE_INTRUSIVE\n@since version 3.9.0\n*/\n#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...)  \\\n    friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \\\n    friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) }\n\n#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...)  \\\n    friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \\\n    friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }\n\n/*!\n@brief macro\n@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE\n@since version 3.9.0\n*/\n#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...)  \\\n    inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \\\n    inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) }\n\n#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...)  \\\n    inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \\\n    inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) }\n\n\n// inspired from https://stackoverflow.com/a/26745591\n// allows to call any std function as if (e.g. with begin):\n// using std::begin; begin(x);\n//\n// it allows using the detected idiom to retrieve the return type\n// of such an expression\n#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name)                                 \\\n    namespace detail {                                                            \\\n    using std::std_name;                                                          \\\n    \\\n    template<typename... T>                                                       \\\n    using result_of_##std_name = decltype(std_name(std::declval<T>()...));        \\\n    }                                                                             \\\n    \\\n    namespace detail2 {                                                           \\\n    struct std_name##_tag                                                         \\\n    {                                                                             \\\n    };                                                                            \\\n    \\\n    template<typename... T>                                                       \\\n    std_name##_tag std_name(T&&...);                                              \\\n    \\\n    template<typename... T>                                                       \\\n    using result_of_##std_name = decltype(std_name(std::declval<T>()...));        \\\n    \\\n    template<typename... T>                                                       \\\n    struct would_call_std_##std_name                                              \\\n    {                                                                             \\\n        static constexpr auto const value = ::nlohmann::detail::                  \\\n                                            is_detected_exact<std_name##_tag, result_of_##std_name, T...>::value; \\\n    };                                                                            \\\n    } /* namespace detail2 */ \\\n    \\\n    template<typename... T>                                                       \\\n    struct would_call_std_##std_name : detail2::would_call_std_##std_name<T...>   \\\n    {                                                                             \\\n    }\n\n#ifndef JSON_USE_IMPLICIT_CONVERSIONS\n    #define JSON_USE_IMPLICIT_CONVERSIONS 1\n#endif\n\n#if JSON_USE_IMPLICIT_CONVERSIONS\n    #define JSON_EXPLICIT\n#else\n    #define JSON_EXPLICIT explicit\n#endif\n\n#ifndef JSON_DISABLE_ENUM_SERIALIZATION\n    #define JSON_DISABLE_ENUM_SERIALIZATION 0\n#endif\n\n#ifndef JSON_USE_GLOBAL_UDLS\n    #define JSON_USE_GLOBAL_UDLS 1\n#endif\n\n#if JSON_HAS_THREE_WAY_COMPARISON\n    #include <compare> // partial_ordering\n#endif\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n///////////////////////////\n// JSON type enumeration //\n///////////////////////////\n\n/*!\n@brief the JSON type enumeration\n\nThis enumeration collects the different JSON types. It is internally used to\ndistinguish the stored values, and the functions @ref basic_json::is_null(),\n@ref basic_json::is_object(), @ref basic_json::is_array(),\n@ref basic_json::is_string(), @ref basic_json::is_boolean(),\n@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),\n@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),\n@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and\n@ref basic_json::is_structured() rely on it.\n\n@note There are three enumeration entries (number_integer, number_unsigned, and\nnumber_float), because the library distinguishes these three types for numbers:\n@ref basic_json::number_unsigned_t is used for unsigned integers,\n@ref basic_json::number_integer_t is used for signed integers, and\n@ref basic_json::number_float_t is used for floating-point numbers or to\napproximate integers which do not fit in the limits of their respective type.\n\n@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON\nvalue with the default value for a given type\n\n@since version 1.0.0\n*/\nenum class value_t : std::uint8_t\n{\n    null,             ///< null value\n    object,           ///< object (unordered set of name/value pairs)\n    array,            ///< array (ordered collection of values)\n    string,           ///< string value\n    boolean,          ///< boolean value\n    number_integer,   ///< number value (signed integer)\n    number_unsigned,  ///< number value (unsigned integer)\n    number_float,     ///< number value (floating-point)\n    binary,           ///< binary array (ordered collection of bytes)\n    discarded         ///< discarded by the parser callback function\n};\n\n/*!\n@brief comparison operator for JSON types\n\nReturns an ordering that is similar to Python:\n- order: null < boolean < number < object < array < string < binary\n- furthermore, each type is not smaller than itself\n- discarded values are not comparable\n- binary is represented as a b\"\" string in python and directly comparable to a\n  string; however, making a binary array directly comparable with a string would\n  be surprising behavior in a JSON file.\n\n@since version 1.0.0\n*/\n#if JSON_HAS_THREE_WAY_COMPARISON\n    inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD*\n#else\n    inline bool operator<(const value_t lhs, const value_t rhs) noexcept\n#endif\n{\n    static constexpr std::array<std::uint8_t, 9> order = {{\n            0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,\n            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */,\n            6 /* binary */\n        }\n    };\n\n    const auto l_index = static_cast<std::size_t>(lhs);\n    const auto r_index = static_cast<std::size_t>(rhs);\n#if JSON_HAS_THREE_WAY_COMPARISON\n    if (l_index < order.size() && r_index < order.size())\n    {\n        return order[l_index] <=> order[r_index]; // *NOPAD*\n    }\n    return std::partial_ordering::unordered;\n#else\n    return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index];\n#endif\n}\n\n// GCC selects the built-in operator< over an operator rewritten from\n// a user-defined spaceship operator\n// Clang, MSVC, and ICC select the rewritten candidate\n// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200)\n#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__)\ninline bool operator<(const value_t lhs, const value_t rhs) noexcept\n{\n    return std::is_lt(lhs <=> rhs); // *NOPAD*\n}\n#endif\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/string_escape.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/*!\n@brief replace all occurrences of a substring by another string\n\n@param[in,out] s  the string to manipulate; changed so that all\n               occurrences of @a f are replaced with @a t\n@param[in]     f  the substring to replace with @a t\n@param[in]     t  the string to replace @a f\n\n@pre The search string @a f must not be empty. **This precondition is\nenforced with an assertion.**\n\n@since version 2.0.0\n*/\ntemplate<typename StringType>\ninline void replace_substring(StringType& s, const StringType& f,\n                              const StringType& t)\n{\n    JSON_ASSERT(!f.empty());\n    for (auto pos = s.find(f);                // find first occurrence of f\n            pos != StringType::npos;          // make sure f was found\n            s.replace(pos, f.size(), t),      // replace with t, and\n            pos = s.find(f, pos + t.size()))  // find next occurrence of f\n    {}\n}\n\n/*!\n * @brief string escaping as described in RFC 6901 (Sect. 4)\n * @param[in] s string to escape\n * @return    escaped string\n *\n * Note the order of escaping \"~\" to \"~0\" and \"/\" to \"~1\" is important.\n */\ntemplate<typename StringType>\ninline StringType escape(StringType s)\n{\n    replace_substring(s, StringType{\"~\"}, StringType{\"~0\"});\n    replace_substring(s, StringType{\"/\"}, StringType{\"~1\"});\n    return s;\n}\n\n/*!\n * @brief string unescaping as described in RFC 6901 (Sect. 4)\n * @param[in] s string to unescape\n * @return    unescaped string\n *\n * Note the order of escaping \"~1\" to \"/\" and \"~0\" to \"~\" is important.\n */\ntemplate<typename StringType>\nstatic void unescape(StringType& s)\n{\n    replace_substring(s, StringType{\"~1\"}, StringType{\"/\"});\n    replace_substring(s, StringType{\"~0\"}, StringType{\"~\"});\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/input/position_t.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef> // size_t\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/// struct to capture the start position of the current token\nstruct position_t\n{\n    /// the total number of characters read\n    std::size_t chars_read_total = 0;\n    /// the number of characters read in the current line\n    std::size_t chars_read_current_line = 0;\n    /// the number of lines read\n    std::size_t lines_read = 0;\n\n    /// conversion to size_t to preserve SAX interface\n    constexpr operator size_t() const\n    {\n        return chars_read_total;\n    }\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-FileCopyrightText: 2018 The Abseil Authors\n// SPDX-License-Identifier: MIT\n\n\n\n#include <array> // array\n#include <cstddef> // size_t\n#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type\n#include <utility> // index_sequence, make_index_sequence, index_sequence_for\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename T>\nusing uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;\n\n#ifdef JSON_HAS_CPP_14\n\n// the following utilities are natively available in C++14\nusing std::enable_if_t;\nusing std::index_sequence;\nusing std::make_index_sequence;\nusing std::index_sequence_for;\n\n#else\n\n// alias templates to reduce boilerplate\ntemplate<bool B, typename T = void>\nusing enable_if_t = typename std::enable_if<B, T>::type;\n\n// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h\n// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0.\n\n//// START OF CODE FROM GOOGLE ABSEIL\n\n// integer_sequence\n//\n// Class template representing a compile-time integer sequence. An instantiation\n// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its\n// type through its template arguments (which is a common need when\n// working with C++11 variadic templates). `absl::integer_sequence` is designed\n// to be a drop-in replacement for C++14's `std::integer_sequence`.\n//\n// Example:\n//\n//   template< class T, T... Ints >\n//   void user_function(integer_sequence<T, Ints...>);\n//\n//   int main()\n//   {\n//     // user_function's `T` will be deduced to `int` and `Ints...`\n//     // will be deduced to `0, 1, 2, 3, 4`.\n//     user_function(make_integer_sequence<int, 5>());\n//   }\ntemplate <typename T, T... Ints>\nstruct integer_sequence\n{\n    using value_type = T;\n    static constexpr std::size_t size() noexcept\n    {\n        return sizeof...(Ints);\n    }\n};\n\n// index_sequence\n//\n// A helper template for an `integer_sequence` of `size_t`,\n// `absl::index_sequence` is designed to be a drop-in replacement for C++14's\n// `std::index_sequence`.\ntemplate <size_t... Ints>\nusing index_sequence = integer_sequence<size_t, Ints...>;\n\nnamespace utility_internal\n{\n\ntemplate <typename Seq, size_t SeqSize, size_t Rem>\nstruct Extend;\n\n// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency.\ntemplate <typename T, T... Ints, size_t SeqSize>\nstruct Extend<integer_sequence<T, Ints...>, SeqSize, 0>\n{\n    using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >;\n};\n\ntemplate <typename T, T... Ints, size_t SeqSize>\nstruct Extend<integer_sequence<T, Ints...>, SeqSize, 1>\n{\n    using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >;\n};\n\n// Recursion helper for 'make_integer_sequence<T, N>'.\n// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'.\ntemplate <typename T, size_t N>\nstruct Gen\n{\n    using type =\n        typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type;\n};\n\ntemplate <typename T>\nstruct Gen<T, 0>\n{\n    using type = integer_sequence<T>;\n};\n\n}  // namespace utility_internal\n\n// Compile-time sequences of integers\n\n// make_integer_sequence\n//\n// This template alias is equivalent to\n// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in\n// replacement for C++14's `std::make_integer_sequence`.\ntemplate <typename T, T N>\nusing make_integer_sequence = typename utility_internal::Gen<T, N>::type;\n\n// make_index_sequence\n//\n// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`,\n// and is designed to be a drop-in replacement for C++14's\n// `std::make_index_sequence`.\ntemplate <size_t N>\nusing make_index_sequence = make_integer_sequence<size_t, N>;\n\n// index_sequence_for\n//\n// Converts a typename pack into an index sequence of the same length, and\n// is designed to be a drop-in replacement for C++14's\n// `std::index_sequence_for()`\ntemplate <typename... Ts>\nusing index_sequence_for = make_index_sequence<sizeof...(Ts)>;\n\n//// END OF CODE FROM GOOGLE ABSEIL\n\n#endif\n\n// dispatch utility (taken from ranges-v3)\ntemplate<unsigned N> struct priority_tag : priority_tag < N - 1 > {};\ntemplate<> struct priority_tag<0> {};\n\n// taken from ranges-v3\ntemplate<typename T>\nstruct static_const\n{\n    static JSON_INLINE_VARIABLE constexpr T value{};\n};\n\n#ifndef JSON_HAS_CPP_17\n    template<typename T>\n    constexpr T static_const<T>::value;\n#endif\n\ntemplate<typename T, typename... Args>\ninline constexpr std::array<T, sizeof...(Args)> make_array(Args&& ... args)\n{\n    return std::array<T, sizeof...(Args)> {{static_cast<T>(std::forward<Args>(args))...}};\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <limits> // numeric_limits\n#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type\n#include <utility> // declval\n#include <tuple> // tuple\n\n// #include <nlohmann/detail/iterators/iterator_traits.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <iterator> // random_access_iterator_tag\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/meta/void_t.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename It, typename = void>\nstruct iterator_types {};\n\ntemplate<typename It>\nstruct iterator_types <\n    It,\n    void_t<typename It::difference_type, typename It::value_type, typename It::pointer,\n    typename It::reference, typename It::iterator_category >>\n{\n    using difference_type = typename It::difference_type;\n    using value_type = typename It::value_type;\n    using pointer = typename It::pointer;\n    using reference = typename It::reference;\n    using iterator_category = typename It::iterator_category;\n};\n\n// This is required as some compilers implement std::iterator_traits in a way that\n// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341.\ntemplate<typename T, typename = void>\nstruct iterator_traits\n{\n};\n\ntemplate<typename T>\nstruct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >>\n            : iterator_types<T>\n{\n};\n\ntemplate<typename T>\nstruct iterator_traits<T*, enable_if_t<std::is_object<T>::value>>\n{\n    using iterator_category = std::random_access_iterator_tag;\n    using value_type = T;\n    using difference_type = ptrdiff_t;\n    using pointer = T*;\n    using reference = T&;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/call_std/begin.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\nNLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin);\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/call_std/end.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\nNLOHMANN_CAN_CALL_STD_FUNC_IMPL(end);\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/detected.hpp>\n\n// #include <nlohmann/json_fwd.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_\n    #define INCLUDE_NLOHMANN_JSON_FWD_HPP_\n\n    #include <cstdint> // int64_t, uint64_t\n    #include <map> // map\n    #include <memory> // allocator\n    #include <string> // string\n    #include <vector> // vector\n\n    // #include <nlohmann/detail/abi_macros.hpp>\n\n\n    /*!\n    @brief namespace for Niels Lohmann\n    @see https://github.com/nlohmann\n    @since version 1.0.0\n    */\n    NLOHMANN_JSON_NAMESPACE_BEGIN\n\n    /*!\n    @brief default JSONSerializer template argument\n\n    This serializer ignores the template arguments and uses ADL\n    ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))\n    for serialization.\n    */\n    template<typename T = void, typename SFINAE = void>\n    struct adl_serializer;\n\n    /// a class to store JSON values\n    /// @sa https://json.nlohmann.me/api/basic_json/\n    template<template<typename U, typename V, typename... Args> class ObjectType =\n    std::map,\n    template<typename U, typename... Args> class ArrayType = std::vector,\n    class StringType = std::string, class BooleanType = bool,\n    class NumberIntegerType = std::int64_t,\n    class NumberUnsignedType = std::uint64_t,\n    class NumberFloatType = double,\n    template<typename U> class AllocatorType = std::allocator,\n    template<typename T, typename SFINAE = void> class JSONSerializer =\n    adl_serializer,\n    class BinaryType = std::vector<std::uint8_t>>\n    class basic_json;\n\n    /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document\n    /// @sa https://json.nlohmann.me/api/json_pointer/\n    template<typename RefStringType>\n    class json_pointer;\n\n    /*!\n    @brief default specialization\n    @sa https://json.nlohmann.me/api/json/\n    */\n    using json = basic_json<>;\n\n    /// @brief a minimal map-like container that preserves insertion order\n    /// @sa https://json.nlohmann.me/api/ordered_map/\n    template<class Key, class T, class IgnoredLess, class Allocator>\n    struct ordered_map;\n\n    /// @brief specialization that maintains the insertion order of object keys\n    /// @sa https://json.nlohmann.me/api/ordered_json/\n    using ordered_json = basic_json<nlohmann::ordered_map>;\n\n    NLOHMANN_JSON_NAMESPACE_END\n\n#endif  // INCLUDE_NLOHMANN_JSON_FWD_HPP_\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n/*!\n@brief detail namespace with internal helper functions\n\nThis namespace collects functions that should not be exposed,\nimplementations of some @ref basic_json methods, and meta-programming helpers.\n\n@since version 2.1.0\n*/\nnamespace detail\n{\n\n/////////////\n// helpers //\n/////////////\n\n// Note to maintainers:\n//\n// Every trait in this file expects a non CV-qualified type.\n// The only exceptions are in the 'aliases for detected' section\n// (i.e. those of the form: decltype(T::member_function(std::declval<T>())))\n//\n// In this case, T has to be properly CV-qualified to constraint the function arguments\n// (e.g. to_json(BasicJsonType&, const T&))\n\ntemplate<typename> struct is_basic_json : std::false_type {};\n\nNLOHMANN_BASIC_JSON_TPL_DECLARATION\nstruct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};\n\n// used by exceptions create() member functions\n// true_type for pointer to possibly cv-qualified basic_json or std::nullptr_t\n// false_type otherwise\ntemplate<typename BasicJsonContext>\nstruct is_basic_json_context :\n    std::integral_constant < bool,\n    is_basic_json<typename std::remove_cv<typename std::remove_pointer<BasicJsonContext>::type>::type>::value\n    || std::is_same<BasicJsonContext, std::nullptr_t>::value >\n{};\n\n//////////////////////\n// json_ref helpers //\n//////////////////////\n\ntemplate<typename>\nclass json_ref;\n\ntemplate<typename>\nstruct is_json_ref : std::false_type {};\n\ntemplate<typename T>\nstruct is_json_ref<json_ref<T>> : std::true_type {};\n\n//////////////////////////\n// aliases for detected //\n//////////////////////////\n\ntemplate<typename T>\nusing mapped_type_t = typename T::mapped_type;\n\ntemplate<typename T>\nusing key_type_t = typename T::key_type;\n\ntemplate<typename T>\nusing value_type_t = typename T::value_type;\n\ntemplate<typename T>\nusing difference_type_t = typename T::difference_type;\n\ntemplate<typename T>\nusing pointer_t = typename T::pointer;\n\ntemplate<typename T>\nusing reference_t = typename T::reference;\n\ntemplate<typename T>\nusing iterator_category_t = typename T::iterator_category;\n\ntemplate<typename T, typename... Args>\nusing to_json_function = decltype(T::to_json(std::declval<Args>()...));\n\ntemplate<typename T, typename... Args>\nusing from_json_function = decltype(T::from_json(std::declval<Args>()...));\n\ntemplate<typename T, typename U>\nusing get_template_function = decltype(std::declval<T>().template get<U>());\n\n// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists\ntemplate<typename BasicJsonType, typename T, typename = void>\nstruct has_from_json : std::false_type {};\n\n// trait checking if j.get<T> is valid\n// use this trait instead of std::is_constructible or std::is_convertible,\n// both rely on, or make use of implicit conversions, and thus fail when T\n// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958)\ntemplate <typename BasicJsonType, typename T>\nstruct is_getable\n{\n    static constexpr bool value = is_detected<get_template_function, const BasicJsonType&, T>::value;\n};\n\ntemplate<typename BasicJsonType, typename T>\nstruct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>\n{\n    using serializer = typename BasicJsonType::template json_serializer<T, void>;\n\n    static constexpr bool value =\n        is_detected_exact<void, from_json_function, serializer,\n        const BasicJsonType&, T&>::value;\n};\n\n// This trait checks if JSONSerializer<T>::from_json(json const&) exists\n// this overload is used for non-default-constructible user-defined-types\ntemplate<typename BasicJsonType, typename T, typename = void>\nstruct has_non_default_from_json : std::false_type {};\n\ntemplate<typename BasicJsonType, typename T>\nstruct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>\n{\n    using serializer = typename BasicJsonType::template json_serializer<T, void>;\n\n    static constexpr bool value =\n        is_detected_exact<T, from_json_function, serializer,\n        const BasicJsonType&>::value;\n};\n\n// This trait checks if BasicJsonType::json_serializer<T>::to_json exists\n// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion.\ntemplate<typename BasicJsonType, typename T, typename = void>\nstruct has_to_json : std::false_type {};\n\ntemplate<typename BasicJsonType, typename T>\nstruct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>\n{\n    using serializer = typename BasicJsonType::template json_serializer<T, void>;\n\n    static constexpr bool value =\n        is_detected_exact<void, to_json_function, serializer, BasicJsonType&,\n        T>::value;\n};\n\ntemplate<typename T>\nusing detect_key_compare = typename T::key_compare;\n\ntemplate<typename T>\nstruct has_key_compare : std::integral_constant<bool, is_detected<detect_key_compare, T>::value> {};\n\n// obtains the actual object key comparator\ntemplate<typename BasicJsonType>\nstruct actual_object_comparator\n{\n    using object_t = typename BasicJsonType::object_t;\n    using object_comparator_t = typename BasicJsonType::default_object_comparator_t;\n    using type = typename std::conditional < has_key_compare<object_t>::value,\n          typename object_t::key_compare, object_comparator_t>::type;\n};\n\ntemplate<typename BasicJsonType>\nusing actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;\n\n///////////////////\n// is_ functions //\n///////////////////\n\n// https://en.cppreference.com/w/cpp/types/conjunction\ntemplate<class...> struct conjunction : std::true_type { };\ntemplate<class B> struct conjunction<B> : B { };\ntemplate<class B, class... Bn>\nstruct conjunction<B, Bn...>\n: std::conditional<static_cast<bool>(B::value), conjunction<Bn...>, B>::type {};\n\n// https://en.cppreference.com/w/cpp/types/negation\ntemplate<class B> struct negation : std::integral_constant < bool, !B::value > { };\n\n// Reimplementation of is_constructible and is_default_constructible, due to them being broken for\n// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367).\n// This causes compile errors in e.g. clang 3.5 or gcc 4.9.\ntemplate <typename T>\nstruct is_default_constructible : std::is_default_constructible<T> {};\n\ntemplate <typename T1, typename T2>\nstruct is_default_constructible<std::pair<T1, T2>>\n            : conjunction<is_default_constructible<T1>, is_default_constructible<T2>> {};\n\ntemplate <typename T1, typename T2>\nstruct is_default_constructible<const std::pair<T1, T2>>\n            : conjunction<is_default_constructible<T1>, is_default_constructible<T2>> {};\n\ntemplate <typename... Ts>\nstruct is_default_constructible<std::tuple<Ts...>>\n            : conjunction<is_default_constructible<Ts>...> {};\n\ntemplate <typename... Ts>\nstruct is_default_constructible<const std::tuple<Ts...>>\n            : conjunction<is_default_constructible<Ts>...> {};\n\n\ntemplate <typename T, typename... Args>\nstruct is_constructible : std::is_constructible<T, Args...> {};\n\ntemplate <typename T1, typename T2>\nstruct is_constructible<std::pair<T1, T2>> : is_default_constructible<std::pair<T1, T2>> {};\n\ntemplate <typename T1, typename T2>\nstruct is_constructible<const std::pair<T1, T2>> : is_default_constructible<const std::pair<T1, T2>> {};\n\ntemplate <typename... Ts>\nstruct is_constructible<std::tuple<Ts...>> : is_default_constructible<std::tuple<Ts...>> {};\n\ntemplate <typename... Ts>\nstruct is_constructible<const std::tuple<Ts...>> : is_default_constructible<const std::tuple<Ts...>> {};\n\n\ntemplate<typename T, typename = void>\nstruct is_iterator_traits : std::false_type {};\n\ntemplate<typename T>\nstruct is_iterator_traits<iterator_traits<T>>\n{\n  private:\n    using traits = iterator_traits<T>;\n\n  public:\n    static constexpr auto value =\n        is_detected<value_type_t, traits>::value &&\n        is_detected<difference_type_t, traits>::value &&\n        is_detected<pointer_t, traits>::value &&\n        is_detected<iterator_category_t, traits>::value &&\n        is_detected<reference_t, traits>::value;\n};\n\ntemplate<typename T>\nstruct is_range\n{\n  private:\n    using t_ref = typename std::add_lvalue_reference<T>::type;\n\n    using iterator = detected_t<result_of_begin, t_ref>;\n    using sentinel = detected_t<result_of_end, t_ref>;\n\n    // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator\n    // and https://en.cppreference.com/w/cpp/iterator/sentinel_for\n    // but reimplementing these would be too much work, as a lot of other concepts are used underneath\n    static constexpr auto is_iterator_begin =\n        is_iterator_traits<iterator_traits<iterator>>::value;\n\n  public:\n    static constexpr bool value = !std::is_same<iterator, nonesuch>::value && !std::is_same<sentinel, nonesuch>::value && is_iterator_begin;\n};\n\ntemplate<typename R>\nusing iterator_t = enable_if_t<is_range<R>::value, result_of_begin<decltype(std::declval<R&>())>>;\n\ntemplate<typename T>\nusing range_value_t = value_type_t<iterator_traits<iterator_t<T>>>;\n\n// The following implementation of is_complete_type is taken from\n// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/\n// and is written by Xiang Fan who agreed to using it in this library.\n\ntemplate<typename T, typename = void>\nstruct is_complete_type : std::false_type {};\n\ntemplate<typename T>\nstruct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {};\n\ntemplate<typename BasicJsonType, typename CompatibleObjectType,\n         typename = void>\nstruct is_compatible_object_type_impl : std::false_type {};\n\ntemplate<typename BasicJsonType, typename CompatibleObjectType>\nstruct is_compatible_object_type_impl <\n    BasicJsonType, CompatibleObjectType,\n    enable_if_t < is_detected<mapped_type_t, CompatibleObjectType>::value&&\n    is_detected<key_type_t, CompatibleObjectType>::value >>\n{\n    using object_t = typename BasicJsonType::object_t;\n\n    // macOS's is_constructible does not play well with nonesuch...\n    static constexpr bool value =\n        is_constructible<typename object_t::key_type,\n        typename CompatibleObjectType::key_type>::value &&\n        is_constructible<typename object_t::mapped_type,\n        typename CompatibleObjectType::mapped_type>::value;\n};\n\ntemplate<typename BasicJsonType, typename CompatibleObjectType>\nstruct is_compatible_object_type\n    : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {};\n\ntemplate<typename BasicJsonType, typename ConstructibleObjectType,\n         typename = void>\nstruct is_constructible_object_type_impl : std::false_type {};\n\ntemplate<typename BasicJsonType, typename ConstructibleObjectType>\nstruct is_constructible_object_type_impl <\n    BasicJsonType, ConstructibleObjectType,\n    enable_if_t < is_detected<mapped_type_t, ConstructibleObjectType>::value&&\n    is_detected<key_type_t, ConstructibleObjectType>::value >>\n{\n    using object_t = typename BasicJsonType::object_t;\n\n    static constexpr bool value =\n        (is_default_constructible<ConstructibleObjectType>::value &&\n         (std::is_move_assignable<ConstructibleObjectType>::value ||\n          std::is_copy_assignable<ConstructibleObjectType>::value) &&\n         (is_constructible<typename ConstructibleObjectType::key_type,\n          typename object_t::key_type>::value &&\n          std::is_same <\n          typename object_t::mapped_type,\n          typename ConstructibleObjectType::mapped_type >::value)) ||\n        (has_from_json<BasicJsonType,\n         typename ConstructibleObjectType::mapped_type>::value ||\n         has_non_default_from_json <\n         BasicJsonType,\n         typename ConstructibleObjectType::mapped_type >::value);\n};\n\ntemplate<typename BasicJsonType, typename ConstructibleObjectType>\nstruct is_constructible_object_type\n    : is_constructible_object_type_impl<BasicJsonType,\n      ConstructibleObjectType> {};\n\ntemplate<typename BasicJsonType, typename CompatibleStringType>\nstruct is_compatible_string_type\n{\n    static constexpr auto value =\n        is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value;\n};\n\ntemplate<typename BasicJsonType, typename ConstructibleStringType>\nstruct is_constructible_string_type\n{\n    // launder type through decltype() to fix compilation failure on ICPC\n#ifdef __INTEL_COMPILER\n    using laundered_type = decltype(std::declval<ConstructibleStringType>());\n#else\n    using laundered_type = ConstructibleStringType;\n#endif\n\n    static constexpr auto value =\n        conjunction <\n        is_constructible<laundered_type, typename BasicJsonType::string_t>,\n        is_detected_exact<typename BasicJsonType::string_t::value_type,\n        value_type_t, laundered_type >>::value;\n};\n\ntemplate<typename BasicJsonType, typename CompatibleArrayType, typename = void>\nstruct is_compatible_array_type_impl : std::false_type {};\n\ntemplate<typename BasicJsonType, typename CompatibleArrayType>\nstruct is_compatible_array_type_impl <\n    BasicJsonType, CompatibleArrayType,\n    enable_if_t <\n    is_detected<iterator_t, CompatibleArrayType>::value&&\n    is_iterator_traits<iterator_traits<detected_t<iterator_t, CompatibleArrayType>>>::value&&\n// special case for types like std::filesystem::path whose iterator's value_type are themselves\n// c.f. https://github.com/nlohmann/json/pull/3073\n    !std::is_same<CompatibleArrayType, detected_t<range_value_t, CompatibleArrayType>>::value >>\n{\n    static constexpr bool value =\n        is_constructible<BasicJsonType,\n        range_value_t<CompatibleArrayType>>::value;\n};\n\ntemplate<typename BasicJsonType, typename CompatibleArrayType>\nstruct is_compatible_array_type\n    : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {};\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType, typename = void>\nstruct is_constructible_array_type_impl : std::false_type {};\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType>\nstruct is_constructible_array_type_impl <\n    BasicJsonType, ConstructibleArrayType,\n    enable_if_t<std::is_same<ConstructibleArrayType,\n    typename BasicJsonType::value_type>::value >>\n            : std::true_type {};\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType>\nstruct is_constructible_array_type_impl <\n    BasicJsonType, ConstructibleArrayType,\n    enable_if_t < !std::is_same<ConstructibleArrayType,\n    typename BasicJsonType::value_type>::value&&\n    !is_compatible_string_type<BasicJsonType, ConstructibleArrayType>::value&&\n    is_default_constructible<ConstructibleArrayType>::value&&\n(std::is_move_assignable<ConstructibleArrayType>::value ||\n std::is_copy_assignable<ConstructibleArrayType>::value)&&\nis_detected<iterator_t, ConstructibleArrayType>::value&&\nis_iterator_traits<iterator_traits<detected_t<iterator_t, ConstructibleArrayType>>>::value&&\nis_detected<range_value_t, ConstructibleArrayType>::value&&\n// special case for types like std::filesystem::path whose iterator's value_type are themselves\n// c.f. https://github.com/nlohmann/json/pull/3073\n!std::is_same<ConstructibleArrayType, detected_t<range_value_t, ConstructibleArrayType>>::value&&\n        is_complete_type <\n        detected_t<range_value_t, ConstructibleArrayType >>::value >>\n{\n    using value_type = range_value_t<ConstructibleArrayType>;\n\n    static constexpr bool value =\n        std::is_same<value_type,\n        typename BasicJsonType::array_t::value_type>::value ||\n        has_from_json<BasicJsonType,\n        value_type>::value ||\n        has_non_default_from_json <\n        BasicJsonType,\n        value_type >::value;\n};\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType>\nstruct is_constructible_array_type\n    : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {};\n\ntemplate<typename RealIntegerType, typename CompatibleNumberIntegerType,\n         typename = void>\nstruct is_compatible_integer_type_impl : std::false_type {};\n\ntemplate<typename RealIntegerType, typename CompatibleNumberIntegerType>\nstruct is_compatible_integer_type_impl <\n    RealIntegerType, CompatibleNumberIntegerType,\n    enable_if_t < std::is_integral<RealIntegerType>::value&&\n    std::is_integral<CompatibleNumberIntegerType>::value&&\n    !std::is_same<bool, CompatibleNumberIntegerType>::value >>\n{\n    // is there an assert somewhere on overflows?\n    using RealLimits = std::numeric_limits<RealIntegerType>;\n    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;\n\n    static constexpr auto value =\n        is_constructible<RealIntegerType,\n        CompatibleNumberIntegerType>::value &&\n        CompatibleLimits::is_integer &&\n        RealLimits::is_signed == CompatibleLimits::is_signed;\n};\n\ntemplate<typename RealIntegerType, typename CompatibleNumberIntegerType>\nstruct is_compatible_integer_type\n    : is_compatible_integer_type_impl<RealIntegerType,\n      CompatibleNumberIntegerType> {};\n\ntemplate<typename BasicJsonType, typename CompatibleType, typename = void>\nstruct is_compatible_type_impl: std::false_type {};\n\ntemplate<typename BasicJsonType, typename CompatibleType>\nstruct is_compatible_type_impl <\n    BasicJsonType, CompatibleType,\n    enable_if_t<is_complete_type<CompatibleType>::value >>\n{\n    static constexpr bool value =\n        has_to_json<BasicJsonType, CompatibleType>::value;\n};\n\ntemplate<typename BasicJsonType, typename CompatibleType>\nstruct is_compatible_type\n    : is_compatible_type_impl<BasicJsonType, CompatibleType> {};\n\ntemplate<typename T1, typename T2>\nstruct is_constructible_tuple : std::false_type {};\n\ntemplate<typename T1, typename... Args>\nstruct is_constructible_tuple<T1, std::tuple<Args...>> : conjunction<is_constructible<T1, Args>...> {};\n\ntemplate<typename BasicJsonType, typename T>\nstruct is_json_iterator_of : std::false_type {};\n\ntemplate<typename BasicJsonType>\nstruct is_json_iterator_of<BasicJsonType, typename BasicJsonType::iterator> : std::true_type {};\n\ntemplate<typename BasicJsonType>\nstruct is_json_iterator_of<BasicJsonType, typename BasicJsonType::const_iterator> : std::true_type\n{};\n\n// checks if a given type T is a template specialization of Primary\ntemplate<template <typename...> class Primary, typename T>\nstruct is_specialization_of : std::false_type {};\n\ntemplate<template <typename...> class Primary, typename... Args>\nstruct is_specialization_of<Primary, Primary<Args...>> : std::true_type {};\n\ntemplate<typename T>\nusing is_json_pointer = is_specialization_of<::nlohmann::json_pointer, uncvref_t<T>>;\n\n// checks if A and B are comparable using Compare functor\ntemplate<typename Compare, typename A, typename B, typename = void>\nstruct is_comparable : std::false_type {};\n\ntemplate<typename Compare, typename A, typename B>\nstruct is_comparable<Compare, A, B, void_t<\ndecltype(std::declval<Compare>()(std::declval<A>(), std::declval<B>())),\ndecltype(std::declval<Compare>()(std::declval<B>(), std::declval<A>()))\n>> : std::true_type {};\n\ntemplate<typename T>\nusing detect_is_transparent = typename T::is_transparent;\n\n// type trait to check if KeyType can be used as object key (without a BasicJsonType)\n// see is_usable_as_basic_json_key_type below\ntemplate<typename Comparator, typename ObjectKeyType, typename KeyTypeCVRef, bool RequireTransparentComparator = true,\n         bool ExcludeObjectKeyType = RequireTransparentComparator, typename KeyType = uncvref_t<KeyTypeCVRef>>\nusing is_usable_as_key_type = typename std::conditional <\n                              is_comparable<Comparator, ObjectKeyType, KeyTypeCVRef>::value\n                              && !(ExcludeObjectKeyType && std::is_same<KeyType,\n                                   ObjectKeyType>::value)\n                              && (!RequireTransparentComparator\n                                  || is_detected <detect_is_transparent, Comparator>::value)\n                              && !is_json_pointer<KeyType>::value,\n                              std::true_type,\n                              std::false_type >::type;\n\n// type trait to check if KeyType can be used as object key\n// true if:\n//   - KeyType is comparable with BasicJsonType::object_t::key_type\n//   - if ExcludeObjectKeyType is true, KeyType is not BasicJsonType::object_t::key_type\n//   - the comparator is transparent or RequireTransparentComparator is false\n//   - KeyType is not a JSON iterator or json_pointer\ntemplate<typename BasicJsonType, typename KeyTypeCVRef, bool RequireTransparentComparator = true,\n         bool ExcludeObjectKeyType = RequireTransparentComparator, typename KeyType = uncvref_t<KeyTypeCVRef>>\nusing is_usable_as_basic_json_key_type = typename std::conditional <\n        is_usable_as_key_type<typename BasicJsonType::object_comparator_t,\n        typename BasicJsonType::object_t::key_type, KeyTypeCVRef,\n        RequireTransparentComparator, ExcludeObjectKeyType>::value\n        && !is_json_iterator_of<BasicJsonType, KeyType>::value,\n        std::true_type,\n        std::false_type >::type;\n\ntemplate<typename ObjectType, typename KeyType>\nusing detect_erase_with_key_type = decltype(std::declval<ObjectType&>().erase(std::declval<KeyType>()));\n\n// type trait to check if object_t has an erase() member functions accepting KeyType\ntemplate<typename BasicJsonType, typename KeyType>\nusing has_erase_with_key_type = typename std::conditional <\n                                is_detected <\n                                detect_erase_with_key_type,\n                                typename BasicJsonType::object_t, KeyType >::value,\n                                std::true_type,\n                                std::false_type >::type;\n\n// a naive helper to check if a type is an ordered_map (exploits the fact that\n// ordered_map inherits capacity() from std::vector)\ntemplate <typename T>\nstruct is_ordered_map\n{\n    using one = char;\n\n    struct two\n    {\n        char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n    };\n\n    template <typename C> static one test( decltype(&C::capacity) ) ;\n    template <typename C> static two test(...);\n\n    enum { value = sizeof(test<T>(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n};\n\n// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324)\ntemplate < typename T, typename U, enable_if_t < !std::is_same<T, U>::value, int > = 0 >\nT conditional_static_cast(U value)\n{\n    return static_cast<T>(value);\n}\n\ntemplate<typename T, typename U, enable_if_t<std::is_same<T, U>::value, int> = 0>\nT conditional_static_cast(U value)\n{\n    return value;\n}\n\ntemplate<typename... Types>\nusing all_integral = conjunction<std::is_integral<Types>...>;\n\ntemplate<typename... Types>\nusing all_signed = conjunction<std::is_signed<Types>...>;\n\ntemplate<typename... Types>\nusing all_unsigned = conjunction<std::is_unsigned<Types>...>;\n\n// there's a disjunction trait in another PR; replace when merged\ntemplate<typename... Types>\nusing same_sign = std::integral_constant < bool,\n      all_signed<Types...>::value || all_unsigned<Types...>::value >;\n\ntemplate<typename OfType, typename T>\nusing never_out_of_range = std::integral_constant < bool,\n      (std::is_signed<OfType>::value && (sizeof(T) < sizeof(OfType)))\n      || (same_sign<OfType, T>::value && sizeof(OfType) == sizeof(T)) >;\n\ntemplate<typename OfType, typename T,\n         bool OfTypeSigned = std::is_signed<OfType>::value,\n         bool TSigned = std::is_signed<T>::value>\nstruct value_in_range_of_impl2;\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl2<OfType, T, false, false>\n{\n    static constexpr bool test(T val)\n    {\n        using CommonType = typename std::common_type<OfType, T>::type;\n        return static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());\n    }\n};\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl2<OfType, T, true, false>\n{\n    static constexpr bool test(T val)\n    {\n        using CommonType = typename std::common_type<OfType, T>::type;\n        return static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());\n    }\n};\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl2<OfType, T, false, true>\n{\n    static constexpr bool test(T val)\n    {\n        using CommonType = typename std::common_type<OfType, T>::type;\n        return val >= 0 && static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());\n    }\n};\n\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl2<OfType, T, true, true>\n{\n    static constexpr bool test(T val)\n    {\n        using CommonType = typename std::common_type<OfType, T>::type;\n        return static_cast<CommonType>(val) >= static_cast<CommonType>((std::numeric_limits<OfType>::min)())\n               && static_cast<CommonType>(val) <= static_cast<CommonType>((std::numeric_limits<OfType>::max)());\n    }\n};\n\ntemplate<typename OfType, typename T,\n         bool NeverOutOfRange = never_out_of_range<OfType, T>::value,\n         typename = detail::enable_if_t<all_integral<OfType, T>::value>>\nstruct value_in_range_of_impl1;\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl1<OfType, T, false>\n{\n    static constexpr bool test(T val)\n    {\n        return value_in_range_of_impl2<OfType, T>::test(val);\n    }\n};\n\ntemplate<typename OfType, typename T>\nstruct value_in_range_of_impl1<OfType, T, true>\n{\n    static constexpr bool test(T /*val*/)\n    {\n        return true;\n    }\n};\n\ntemplate<typename OfType, typename T>\ninline constexpr bool value_in_range_of(T val)\n{\n    return value_in_range_of_impl1<OfType, T>::test(val);\n}\n\ntemplate<bool Value>\nusing bool_constant = std::integral_constant<bool, Value>;\n\n///////////////////////////////////////////////////////////////////////////////\n// is_c_string\n///////////////////////////////////////////////////////////////////////////////\n\nnamespace impl\n{\n\ntemplate<typename T>\ninline constexpr bool is_c_string()\n{\n    using TUnExt = typename std::remove_extent<T>::type;\n    using TUnCVExt = typename std::remove_cv<TUnExt>::type;\n    using TUnPtr = typename std::remove_pointer<T>::type;\n    using TUnCVPtr = typename std::remove_cv<TUnPtr>::type;\n    return\n        (std::is_array<T>::value && std::is_same<TUnCVExt, char>::value)\n        || (std::is_pointer<T>::value && std::is_same<TUnCVPtr, char>::value);\n}\n\n}  // namespace impl\n\n// checks whether T is a [cv] char */[cv] char[] C string\ntemplate<typename T>\nstruct is_c_string : bool_constant<impl::is_c_string<T>()> {};\n\ntemplate<typename T>\nusing is_c_string_uncvref = is_c_string<uncvref_t<T>>;\n\n///////////////////////////////////////////////////////////////////////////////\n// is_transparent\n///////////////////////////////////////////////////////////////////////////////\n\nnamespace impl\n{\n\ntemplate<typename T>\ninline constexpr bool is_transparent()\n{\n    return is_detected<detect_is_transparent, T>::value;\n}\n\n}  // namespace impl\n\n// checks whether T has a member named is_transparent\ntemplate<typename T>\nstruct is_transparent : bool_constant<impl::is_transparent<T>()> {};\n\n///////////////////////////////////////////////////////////////////////////////\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/string_concat.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstring> // strlen\n#include <string> // string\n#include <utility> // forward\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/detected.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ninline std::size_t concat_length()\n{\n    return 0;\n}\n\ntemplate<typename... Args>\ninline std::size_t concat_length(const char* cstr, Args&& ... rest);\n\ntemplate<typename StringType, typename... Args>\ninline std::size_t concat_length(const StringType& str, Args&& ... rest);\n\ntemplate<typename... Args>\ninline std::size_t concat_length(const char /*c*/, Args&& ... rest)\n{\n    return 1 + concat_length(std::forward<Args>(rest)...);\n}\n\ntemplate<typename... Args>\ninline std::size_t concat_length(const char* cstr, Args&& ... rest)\n{\n    // cppcheck-suppress ignoredReturnValue\n    return ::strlen(cstr) + concat_length(std::forward<Args>(rest)...);\n}\n\ntemplate<typename StringType, typename... Args>\ninline std::size_t concat_length(const StringType& str, Args&& ... rest)\n{\n    return str.size() + concat_length(std::forward<Args>(rest)...);\n}\n\ntemplate<typename OutStringType>\ninline void concat_into(OutStringType& /*out*/)\n{}\n\ntemplate<typename StringType, typename Arg>\nusing string_can_append = decltype(std::declval<StringType&>().append(std::declval < Arg && > ()));\n\ntemplate<typename StringType, typename Arg>\nusing detect_string_can_append = is_detected<string_can_append, StringType, Arg>;\n\ntemplate<typename StringType, typename Arg>\nusing string_can_append_op = decltype(std::declval<StringType&>() += std::declval < Arg && > ());\n\ntemplate<typename StringType, typename Arg>\nusing detect_string_can_append_op = is_detected<string_can_append_op, StringType, Arg>;\n\ntemplate<typename StringType, typename Arg>\nusing string_can_append_iter = decltype(std::declval<StringType&>().append(std::declval<const Arg&>().begin(), std::declval<const Arg&>().end()));\n\ntemplate<typename StringType, typename Arg>\nusing detect_string_can_append_iter = is_detected<string_can_append_iter, StringType, Arg>;\n\ntemplate<typename StringType, typename Arg>\nusing string_can_append_data = decltype(std::declval<StringType&>().append(std::declval<const Arg&>().data(), std::declval<const Arg&>().size()));\n\ntemplate<typename StringType, typename Arg>\nusing detect_string_can_append_data = is_detected<string_can_append_data, StringType, Arg>;\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && detect_string_can_append_op<OutStringType, Arg>::value, int > = 0 >\ninline void concat_into(OutStringType& out, Arg && arg, Args && ... rest);\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && !detect_string_can_append_op<OutStringType, Arg>::value\n                         && detect_string_can_append_iter<OutStringType, Arg>::value, int > = 0 >\ninline void concat_into(OutStringType& out, const Arg& arg, Args && ... rest);\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && !detect_string_can_append_op<OutStringType, Arg>::value\n                         && !detect_string_can_append_iter<OutStringType, Arg>::value\n                         && detect_string_can_append_data<OutStringType, Arg>::value, int > = 0 >\ninline void concat_into(OutStringType& out, const Arg& arg, Args && ... rest);\n\ntemplate<typename OutStringType, typename Arg, typename... Args,\n         enable_if_t<detect_string_can_append<OutStringType, Arg>::value, int> = 0>\ninline void concat_into(OutStringType& out, Arg && arg, Args && ... rest)\n{\n    out.append(std::forward<Arg>(arg));\n    concat_into(out, std::forward<Args>(rest)...);\n}\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && detect_string_can_append_op<OutStringType, Arg>::value, int > >\ninline void concat_into(OutStringType& out, Arg&& arg, Args&& ... rest)\n{\n    out += std::forward<Arg>(arg);\n    concat_into(out, std::forward<Args>(rest)...);\n}\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && !detect_string_can_append_op<OutStringType, Arg>::value\n                         && detect_string_can_append_iter<OutStringType, Arg>::value, int > >\ninline void concat_into(OutStringType& out, const Arg& arg, Args&& ... rest)\n{\n    out.append(arg.begin(), arg.end());\n    concat_into(out, std::forward<Args>(rest)...);\n}\n\ntemplate < typename OutStringType, typename Arg, typename... Args,\n           enable_if_t < !detect_string_can_append<OutStringType, Arg>::value\n                         && !detect_string_can_append_op<OutStringType, Arg>::value\n                         && !detect_string_can_append_iter<OutStringType, Arg>::value\n                         && detect_string_can_append_data<OutStringType, Arg>::value, int > >\ninline void concat_into(OutStringType& out, const Arg& arg, Args&& ... rest)\n{\n    out.append(arg.data(), arg.size());\n    concat_into(out, std::forward<Args>(rest)...);\n}\n\ntemplate<typename OutStringType = std::string, typename... Args>\ninline OutStringType concat(Args && ... args)\n{\n    OutStringType str;\n    str.reserve(concat_length(std::forward<Args>(args)...));\n    concat_into(str, std::forward<Args>(args)...);\n    return str;\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n////////////////\n// exceptions //\n////////////////\n\n/// @brief general exception of the @ref basic_json class\n/// @sa https://json.nlohmann.me/api/basic_json/exception/\nclass exception : public std::exception\n{\n  public:\n    /// returns the explanatory string\n    const char* what() const noexcept override\n    {\n        return m.what();\n    }\n\n    /// the id of the exception\n    const int id; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes)\n\n  protected:\n    JSON_HEDLEY_NON_NULL(3)\n    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} // NOLINT(bugprone-throw-keyword-missing)\n\n    static std::string name(const std::string& ename, int id_)\n    {\n        return concat(\"[json.exception.\", ename, '.', std::to_string(id_), \"] \");\n    }\n\n    static std::string diagnostics(std::nullptr_t /*leaf_element*/)\n    {\n        return \"\";\n    }\n\n    template<typename BasicJsonType>\n    static std::string diagnostics(const BasicJsonType* leaf_element)\n    {\n#if JSON_DIAGNOSTICS\n        std::vector<std::string> tokens;\n        for (const auto* current = leaf_element; current != nullptr && current->m_parent != nullptr; current = current->m_parent)\n        {\n            switch (current->m_parent->type())\n            {\n                case value_t::array:\n                {\n                    for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i)\n                    {\n                        if (&current->m_parent->m_value.array->operator[](i) == current)\n                        {\n                            tokens.emplace_back(std::to_string(i));\n                            break;\n                        }\n                    }\n                    break;\n                }\n\n                case value_t::object:\n                {\n                    for (const auto& element : *current->m_parent->m_value.object)\n                    {\n                        if (&element.second == current)\n                        {\n                            tokens.emplace_back(element.first.c_str());\n                            break;\n                        }\n                    }\n                    break;\n                }\n\n                case value_t::null: // LCOV_EXCL_LINE\n                case value_t::string: // LCOV_EXCL_LINE\n                case value_t::boolean: // LCOV_EXCL_LINE\n                case value_t::number_integer: // LCOV_EXCL_LINE\n                case value_t::number_unsigned: // LCOV_EXCL_LINE\n                case value_t::number_float: // LCOV_EXCL_LINE\n                case value_t::binary: // LCOV_EXCL_LINE\n                case value_t::discarded: // LCOV_EXCL_LINE\n                default:   // LCOV_EXCL_LINE\n                    break; // LCOV_EXCL_LINE\n            }\n        }\n\n        if (tokens.empty())\n        {\n            return \"\";\n        }\n\n        auto str = std::accumulate(tokens.rbegin(), tokens.rend(), std::string{},\n                                   [](const std::string & a, const std::string & b)\n        {\n            return concat(a, '/', detail::escape(b));\n        });\n        return concat('(', str, \") \");\n#else\n        static_cast<void>(leaf_element);\n        return \"\";\n#endif\n    }\n\n  private:\n    /// an exception object as storage for error messages\n    std::runtime_error m;\n};\n\n/// @brief exception indicating a parse error\n/// @sa https://json.nlohmann.me/api/basic_json/parse_error/\nclass parse_error : public exception\n{\n  public:\n    /*!\n    @brief create a parse error exception\n    @param[in] id_       the id of the exception\n    @param[in] pos       the position where the error occurred (or with\n                         chars_read_total=0 if the position cannot be\n                         determined)\n    @param[in] what_arg  the explanatory string\n    @return parse_error object\n    */\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static parse_error create(int id_, const position_t& pos, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"parse_error\", id_), \"parse error\",\n                               position_string(pos), \": \", exception::diagnostics(context), what_arg);\n        return {id_, pos.chars_read_total, w.c_str()};\n    }\n\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"parse_error\", id_), \"parse error\",\n                               (byte_ != 0 ? (concat(\" at byte \", std::to_string(byte_))) : \"\"),\n                               \": \", exception::diagnostics(context), what_arg);\n        return {id_, byte_, w.c_str()};\n    }\n\n    /*!\n    @brief byte index of the parse error\n\n    The byte index of the last read character in the input file.\n\n    @note For an input with n bytes, 1 is the index of the first character and\n          n+1 is the index of the terminating null byte or the end of file.\n          This also holds true when reading a byte vector (CBOR or MessagePack).\n    */\n    const std::size_t byte;\n\n  private:\n    parse_error(int id_, std::size_t byte_, const char* what_arg)\n        : exception(id_, what_arg), byte(byte_) {}\n\n    static std::string position_string(const position_t& pos)\n    {\n        return concat(\" at line \", std::to_string(pos.lines_read + 1),\n                      \", column \", std::to_string(pos.chars_read_current_line));\n    }\n};\n\n/// @brief exception indicating errors with iterators\n/// @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/\nclass invalid_iterator : public exception\n{\n  public:\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static invalid_iterator create(int id_, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"invalid_iterator\", id_), exception::diagnostics(context), what_arg);\n        return {id_, w.c_str()};\n    }\n\n  private:\n    JSON_HEDLEY_NON_NULL(3)\n    invalid_iterator(int id_, const char* what_arg)\n        : exception(id_, what_arg) {}\n};\n\n/// @brief exception indicating executing a member function with a wrong type\n/// @sa https://json.nlohmann.me/api/basic_json/type_error/\nclass type_error : public exception\n{\n  public:\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static type_error create(int id_, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"type_error\", id_), exception::diagnostics(context), what_arg);\n        return {id_, w.c_str()};\n    }\n\n  private:\n    JSON_HEDLEY_NON_NULL(3)\n    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}\n};\n\n/// @brief exception indicating access out of the defined range\n/// @sa https://json.nlohmann.me/api/basic_json/out_of_range/\nclass out_of_range : public exception\n{\n  public:\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static out_of_range create(int id_, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"out_of_range\", id_), exception::diagnostics(context), what_arg);\n        return {id_, w.c_str()};\n    }\n\n  private:\n    JSON_HEDLEY_NON_NULL(3)\n    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}\n};\n\n/// @brief exception indicating other library errors\n/// @sa https://json.nlohmann.me/api/basic_json/other_error/\nclass other_error : public exception\n{\n  public:\n    template<typename BasicJsonContext, enable_if_t<is_basic_json_context<BasicJsonContext>::value, int> = 0>\n    static other_error create(int id_, const std::string& what_arg, BasicJsonContext context)\n    {\n        std::string w = concat(exception::name(\"other_error\", id_), exception::diagnostics(context), what_arg);\n        return {id_, w.c_str()};\n    }\n\n  private:\n    JSON_HEDLEY_NON_NULL(3)\n    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/identity_tag.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n// dispatching helper struct\ntemplate <class T> struct identity_tag {};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/std_fs.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\n#if JSON_HAS_EXPERIMENTAL_FILESYSTEM\n#include <experimental/filesystem>\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\nnamespace std_fs = std::experimental::filesystem;\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n#elif JSON_HAS_FILESYSTEM\n#include <filesystem>\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\nnamespace std_fs = std::filesystem;\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n#endif\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename std::nullptr_t& n)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_null()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be null, but is \", j.type_name()), &j));\n    }\n    n = nullptr;\n}\n\n// overloads for basic_json template parameters\ntemplate < typename BasicJsonType, typename ArithmeticType,\n           enable_if_t < std::is_arithmetic<ArithmeticType>::value&&\n                         !std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,\n                         int > = 0 >\nvoid get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)\n{\n    switch (static_cast<value_t>(j))\n    {\n        case value_t::number_unsigned:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());\n            break;\n        }\n        case value_t::number_integer:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());\n            break;\n        }\n        case value_t::number_float:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());\n            break;\n        }\n\n        case value_t::null:\n        case value_t::object:\n        case value_t::array:\n        case value_t::string:\n        case value_t::boolean:\n        case value_t::binary:\n        case value_t::discarded:\n        default:\n            JSON_THROW(type_error::create(302, concat(\"type must be number, but is \", j.type_name()), &j));\n    }\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_boolean()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be boolean, but is \", j.type_name()), &j));\n    }\n    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_string()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be string, but is \", j.type_name()), &j));\n    }\n    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();\n}\n\ntemplate <\n    typename BasicJsonType, typename StringType,\n    enable_if_t <\n        std::is_assignable<StringType&, const typename BasicJsonType::string_t>::value\n        && is_detected_exact<typename BasicJsonType::string_t::value_type, value_type_t, StringType>::value\n        && !std::is_same<typename BasicJsonType::string_t, StringType>::value\n        && !is_json_ref<StringType>::value, int > = 0 >\ninline void from_json(const BasicJsonType& j, StringType& s)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_string()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be string, but is \", j.type_name()), &j));\n    }\n\n    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)\n{\n    get_arithmetic_value(j, val);\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)\n{\n    get_arithmetic_value(j, val);\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)\n{\n    get_arithmetic_value(j, val);\n}\n\n#if !JSON_DISABLE_ENUM_SERIALIZATION\ntemplate<typename BasicJsonType, typename EnumType,\n         enable_if_t<std::is_enum<EnumType>::value, int> = 0>\ninline void from_json(const BasicJsonType& j, EnumType& e)\n{\n    typename std::underlying_type<EnumType>::type val;\n    get_arithmetic_value(j, val);\n    e = static_cast<EnumType>(val);\n}\n#endif  // JSON_DISABLE_ENUM_SERIALIZATION\n\n// forward_list doesn't have an insert method\ntemplate<typename BasicJsonType, typename T, typename Allocator,\n         enable_if_t<is_getable<BasicJsonType, T>::value, int> = 0>\ninline void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n    l.clear();\n    std::transform(j.rbegin(), j.rend(),\n                   std::front_inserter(l), [](const BasicJsonType & i)\n    {\n        return i.template get<T>();\n    });\n}\n\n// valarray doesn't have an insert method\ntemplate<typename BasicJsonType, typename T,\n         enable_if_t<is_getable<BasicJsonType, T>::value, int> = 0>\ninline void from_json(const BasicJsonType& j, std::valarray<T>& l)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n    l.resize(j.size());\n    std::transform(j.begin(), j.end(), std::begin(l),\n                   [](const BasicJsonType & elem)\n    {\n        return elem.template get<T>();\n    });\n}\n\ntemplate<typename BasicJsonType, typename T, std::size_t N>\nauto from_json(const BasicJsonType& j, T (&arr)[N])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n-> decltype(j.template get<T>(), void())\n{\n    for (std::size_t i = 0; i < N; ++i)\n    {\n        arr[i] = j.at(i).template get<T>();\n    }\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/)\n{\n    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();\n}\n\ntemplate<typename BasicJsonType, typename T, std::size_t N>\nauto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,\n                          priority_tag<2> /*unused*/)\n-> decltype(j.template get<T>(), void())\n{\n    for (std::size_t i = 0; i < N; ++i)\n    {\n        arr[i] = j.at(i).template get<T>();\n    }\n}\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType,\n         enable_if_t<\n             std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,\n             int> = 0>\nauto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)\n-> decltype(\n    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),\n    j.template get<typename ConstructibleArrayType::value_type>(),\n    void())\n{\n    using std::end;\n\n    ConstructibleArrayType ret;\n    ret.reserve(j.size());\n    std::transform(j.begin(), j.end(),\n                   std::inserter(ret, end(ret)), [](const BasicJsonType & i)\n    {\n        // get<BasicJsonType>() returns *this, this won't call a from_json\n        // method when value_type is BasicJsonType\n        return i.template get<typename ConstructibleArrayType::value_type>();\n    });\n    arr = std::move(ret);\n}\n\ntemplate<typename BasicJsonType, typename ConstructibleArrayType,\n         enable_if_t<\n             std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,\n             int> = 0>\ninline void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,\n                                 priority_tag<0> /*unused*/)\n{\n    using std::end;\n\n    ConstructibleArrayType ret;\n    std::transform(\n        j.begin(), j.end(), std::inserter(ret, end(ret)),\n        [](const BasicJsonType & i)\n    {\n        // get<BasicJsonType>() returns *this, this won't call a from_json\n        // method when value_type is BasicJsonType\n        return i.template get<typename ConstructibleArrayType::value_type>();\n    });\n    arr = std::move(ret);\n}\n\ntemplate < typename BasicJsonType, typename ConstructibleArrayType,\n           enable_if_t <\n               is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value&&\n               !is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value&&\n               !is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value&&\n               !std::is_same<ConstructibleArrayType, typename BasicJsonType::binary_t>::value&&\n               !is_basic_json<ConstructibleArrayType>::value,\n               int > = 0 >\nauto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)\n-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),\nj.template get<typename ConstructibleArrayType::value_type>(),\nvoid())\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n\n    from_json_array_impl(j, arr, priority_tag<3> {});\n}\n\ntemplate < typename BasicJsonType, typename T, std::size_t... Idx >\nstd::array<T, sizeof...(Idx)> from_json_inplace_array_impl(BasicJsonType&& j,\n        identity_tag<std::array<T, sizeof...(Idx)>> /*unused*/, index_sequence<Idx...> /*unused*/)\n{\n    return { { std::forward<BasicJsonType>(j).at(Idx).template get<T>()... } };\n}\n\ntemplate < typename BasicJsonType, typename T, std::size_t N >\nauto from_json(BasicJsonType&& j, identity_tag<std::array<T, N>> tag)\n-> decltype(from_json_inplace_array_impl(std::forward<BasicJsonType>(j), tag, make_index_sequence<N> {}))\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n\n    return from_json_inplace_array_impl(std::forward<BasicJsonType>(j), tag, make_index_sequence<N> {});\n}\n\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_binary()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be binary, but is \", j.type_name()), &j));\n    }\n\n    bin = *j.template get_ptr<const typename BasicJsonType::binary_t*>();\n}\n\ntemplate<typename BasicJsonType, typename ConstructibleObjectType,\n         enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>\ninline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_object()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be object, but is \", j.type_name()), &j));\n    }\n\n    ConstructibleObjectType ret;\n    const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();\n    using value_type = typename ConstructibleObjectType::value_type;\n    std::transform(\n        inner_object->begin(), inner_object->end(),\n        std::inserter(ret, ret.begin()),\n        [](typename BasicJsonType::object_t::value_type const & p)\n    {\n        return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());\n    });\n    obj = std::move(ret);\n}\n\n// overload for arithmetic types, not chosen for basic_json template arguments\n// (BooleanType, etc..); note: Is it really necessary to provide explicit\n// overloads for boolean_t etc. in case of a custom BooleanType which is not\n// an arithmetic type?\ntemplate < typename BasicJsonType, typename ArithmeticType,\n           enable_if_t <\n               std::is_arithmetic<ArithmeticType>::value&&\n               !std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value&&\n               !std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value&&\n               !std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value&&\n               !std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,\n               int > = 0 >\ninline void from_json(const BasicJsonType& j, ArithmeticType& val)\n{\n    switch (static_cast<value_t>(j))\n    {\n        case value_t::number_unsigned:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());\n            break;\n        }\n        case value_t::number_integer:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());\n            break;\n        }\n        case value_t::number_float:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());\n            break;\n        }\n        case value_t::boolean:\n        {\n            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());\n            break;\n        }\n\n        case value_t::null:\n        case value_t::object:\n        case value_t::array:\n        case value_t::string:\n        case value_t::binary:\n        case value_t::discarded:\n        default:\n            JSON_THROW(type_error::create(302, concat(\"type must be number, but is \", j.type_name()), &j));\n    }\n}\n\ntemplate<typename BasicJsonType, typename... Args, std::size_t... Idx>\nstd::tuple<Args...> from_json_tuple_impl_base(BasicJsonType&& j, index_sequence<Idx...> /*unused*/)\n{\n    return std::make_tuple(std::forward<BasicJsonType>(j).at(Idx).template get<Args>()...);\n}\n\ntemplate < typename BasicJsonType, class A1, class A2 >\nstd::pair<A1, A2> from_json_tuple_impl(BasicJsonType&& j, identity_tag<std::pair<A1, A2>> /*unused*/, priority_tag<0> /*unused*/)\n{\n    return {std::forward<BasicJsonType>(j).at(0).template get<A1>(),\n            std::forward<BasicJsonType>(j).at(1).template get<A2>()};\n}\n\ntemplate<typename BasicJsonType, typename A1, typename A2>\ninline void from_json_tuple_impl(BasicJsonType&& j, std::pair<A1, A2>& p, priority_tag<1> /*unused*/)\n{\n    p = from_json_tuple_impl(std::forward<BasicJsonType>(j), identity_tag<std::pair<A1, A2>> {}, priority_tag<0> {});\n}\n\ntemplate<typename BasicJsonType, typename... Args>\nstd::tuple<Args...> from_json_tuple_impl(BasicJsonType&& j, identity_tag<std::tuple<Args...>> /*unused*/, priority_tag<2> /*unused*/)\n{\n    return from_json_tuple_impl_base<BasicJsonType, Args...>(std::forward<BasicJsonType>(j), index_sequence_for<Args...> {});\n}\n\ntemplate<typename BasicJsonType, typename... Args>\ninline void from_json_tuple_impl(BasicJsonType&& j, std::tuple<Args...>& t, priority_tag<3> /*unused*/)\n{\n    t = from_json_tuple_impl_base<BasicJsonType, Args...>(std::forward<BasicJsonType>(j), index_sequence_for<Args...> {});\n}\n\ntemplate<typename BasicJsonType, typename TupleRelated>\nauto from_json(BasicJsonType&& j, TupleRelated&& t)\n-> decltype(from_json_tuple_impl(std::forward<BasicJsonType>(j), std::forward<TupleRelated>(t), priority_tag<3> {}))\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n\n    return from_json_tuple_impl(std::forward<BasicJsonType>(j), std::forward<TupleRelated>(t), priority_tag<3> {});\n}\n\ntemplate < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,\n           typename = enable_if_t < !std::is_constructible <\n                                        typename BasicJsonType::string_t, Key >::value >>\ninline void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n    m.clear();\n    for (const auto& p : j)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!p.is_array()))\n        {\n            JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", p.type_name()), &j));\n        }\n        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());\n    }\n}\n\ntemplate < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,\n           typename = enable_if_t < !std::is_constructible <\n                                        typename BasicJsonType::string_t, Key >::value >>\ninline void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", j.type_name()), &j));\n    }\n    m.clear();\n    for (const auto& p : j)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!p.is_array()))\n        {\n            JSON_THROW(type_error::create(302, concat(\"type must be array, but is \", p.type_name()), &j));\n        }\n        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());\n    }\n}\n\n#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM\ntemplate<typename BasicJsonType>\ninline void from_json(const BasicJsonType& j, std_fs::path& p)\n{\n    if (JSON_HEDLEY_UNLIKELY(!j.is_string()))\n    {\n        JSON_THROW(type_error::create(302, concat(\"type must be string, but is \", j.type_name()), &j));\n    }\n    p = *j.template get_ptr<const typename BasicJsonType::string_t*>();\n}\n#endif\n\nstruct from_json_fn\n{\n    template<typename BasicJsonType, typename T>\n    auto operator()(const BasicJsonType& j, T&& val) const\n    noexcept(noexcept(from_json(j, std::forward<T>(val))))\n    -> decltype(from_json(j, std::forward<T>(val)))\n    {\n        return from_json(j, std::forward<T>(val));\n    }\n};\n\n}  // namespace detail\n\n#ifndef JSON_HAS_CPP_17\n/// namespace to hold default `from_json` function\n/// to see why this is required:\n/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html\nnamespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces)\n{\n#endif\nJSON_INLINE_VARIABLE constexpr const auto& from_json = // NOLINT(misc-definitions-in-headers)\n    detail::static_const<detail::from_json_fn>::value;\n#ifndef JSON_HAS_CPP_17\n}  // namespace\n#endif\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/conversions/to_json.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // copy\n#include <iterator> // begin, end\n#include <string> // string\n#include <tuple> // tuple, get\n#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type\n#include <utility> // move, forward, declval, pair\n#include <valarray> // valarray\n#include <vector> // vector\n\n// #include <nlohmann/detail/iterators/iteration_proxy.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef> // size_t\n#include <iterator> // input_iterator_tag\n#include <string> // string, to_string\n#include <tuple> // tuple_size, get, tuple_element\n#include <utility> // move\n\n#if JSON_HAS_RANGES\n    #include <ranges> // enable_borrowed_range\n#endif\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename string_type>\nvoid int_to_string( string_type& target, std::size_t value )\n{\n    // For ADL\n    using std::to_string;\n    target = to_string(value);\n}\ntemplate<typename IteratorType> class iteration_proxy_value\n{\n  public:\n    using difference_type = std::ptrdiff_t;\n    using value_type = iteration_proxy_value;\n    using pointer = value_type *;\n    using reference = value_type &;\n    using iterator_category = std::input_iterator_tag;\n    using string_type = typename std::remove_cv< typename std::remove_reference<decltype( std::declval<IteratorType>().key() ) >::type >::type;\n\n  private:\n    /// the iterator\n    IteratorType anchor{};\n    /// an index for arrays (used to create key names)\n    std::size_t array_index = 0;\n    /// last stringified array index\n    mutable std::size_t array_index_last = 0;\n    /// a string representation of the array index\n    mutable string_type array_index_str = \"0\";\n    /// an empty string (to return a reference for primitive values)\n    string_type empty_str{};\n\n  public:\n    explicit iteration_proxy_value() = default;\n    explicit iteration_proxy_value(IteratorType it, std::size_t array_index_ = 0)\n    noexcept(std::is_nothrow_move_constructible<IteratorType>::value\n             && std::is_nothrow_default_constructible<string_type>::value)\n        : anchor(std::move(it))\n        , array_index(array_index_)\n    {}\n\n    iteration_proxy_value(iteration_proxy_value const&) = default;\n    iteration_proxy_value& operator=(iteration_proxy_value const&) = default;\n    // older GCCs are a bit fussy and require explicit noexcept specifiers on defaulted functions\n    iteration_proxy_value(iteration_proxy_value&&)\n    noexcept(std::is_nothrow_move_constructible<IteratorType>::value\n             && std::is_nothrow_move_constructible<string_type>::value) = default;\n    iteration_proxy_value& operator=(iteration_proxy_value&&)\n    noexcept(std::is_nothrow_move_assignable<IteratorType>::value\n             && std::is_nothrow_move_assignable<string_type>::value) = default;\n    ~iteration_proxy_value() = default;\n\n    /// dereference operator (needed for range-based for)\n    const iteration_proxy_value& operator*() const\n    {\n        return *this;\n    }\n\n    /// increment operator (needed for range-based for)\n    iteration_proxy_value& operator++()\n    {\n        ++anchor;\n        ++array_index;\n\n        return *this;\n    }\n\n    iteration_proxy_value operator++(int)& // NOLINT(cert-dcl21-cpp)\n    {\n        auto tmp = iteration_proxy_value(anchor, array_index);\n        ++anchor;\n        ++array_index;\n        return tmp;\n    }\n\n    /// equality operator (needed for InputIterator)\n    bool operator==(const iteration_proxy_value& o) const\n    {\n        return anchor == o.anchor;\n    }\n\n    /// inequality operator (needed for range-based for)\n    bool operator!=(const iteration_proxy_value& o) const\n    {\n        return anchor != o.anchor;\n    }\n\n    /// return key of the iterator\n    const string_type& key() const\n    {\n        JSON_ASSERT(anchor.m_object != nullptr);\n\n        switch (anchor.m_object->type())\n        {\n            // use integer array index as key\n            case value_t::array:\n            {\n                if (array_index != array_index_last)\n                {\n                    int_to_string( array_index_str, array_index );\n                    array_index_last = array_index;\n                }\n                return array_index_str;\n            }\n\n            // use key from the object\n            case value_t::object:\n                return anchor.key();\n\n            // use an empty key for all primitive types\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                return empty_str;\n        }\n    }\n\n    /// return value of the iterator\n    typename IteratorType::reference value() const\n    {\n        return anchor.value();\n    }\n};\n\n/// proxy class for the items() function\ntemplate<typename IteratorType> class iteration_proxy\n{\n  private:\n    /// the container to iterate\n    typename IteratorType::pointer container = nullptr;\n\n  public:\n    explicit iteration_proxy() = default;\n\n    /// construct iteration proxy from a container\n    explicit iteration_proxy(typename IteratorType::reference cont) noexcept\n        : container(&cont) {}\n\n    iteration_proxy(iteration_proxy const&) = default;\n    iteration_proxy& operator=(iteration_proxy const&) = default;\n    iteration_proxy(iteration_proxy&&) noexcept = default;\n    iteration_proxy& operator=(iteration_proxy&&) noexcept = default;\n    ~iteration_proxy() = default;\n\n    /// return iterator begin (needed for range-based for)\n    iteration_proxy_value<IteratorType> begin() const noexcept\n    {\n        return iteration_proxy_value<IteratorType>(container->begin());\n    }\n\n    /// return iterator end (needed for range-based for)\n    iteration_proxy_value<IteratorType> end() const noexcept\n    {\n        return iteration_proxy_value<IteratorType>(container->end());\n    }\n};\n\n// Structured Bindings Support\n// For further reference see https://blog.tartanllama.xyz/structured-bindings/\n// And see https://github.com/nlohmann/json/pull/1391\ntemplate<std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0>\nauto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key())\n{\n    return i.key();\n}\n// Structured Bindings Support\n// For further reference see https://blog.tartanllama.xyz/structured-bindings/\n// And see https://github.com/nlohmann/json/pull/1391\ntemplate<std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0>\nauto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value())\n{\n    return i.value();\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// The Addition to the STD Namespace is required to add\n// Structured Bindings Support to the iteration_proxy_value class\n// For further reference see https://blog.tartanllama.xyz/structured-bindings/\n// And see https://github.com/nlohmann/json/pull/1391\nnamespace std\n{\n\n#if defined(__clang__)\n    // Fix: https://github.com/nlohmann/json/issues/1401\n    #pragma clang diagnostic push\n    #pragma clang diagnostic ignored \"-Wmismatched-tags\"\n#endif\ntemplate<typename IteratorType>\nclass tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>>\n            : public std::integral_constant<std::size_t, 2> {};\n\ntemplate<std::size_t N, typename IteratorType>\nclass tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >>\n{\n  public:\n    using type = decltype(\n                     get<N>(std::declval <\n                            ::nlohmann::detail::iteration_proxy_value<IteratorType >> ()));\n};\n#if defined(__clang__)\n    #pragma clang diagnostic pop\n#endif\n\n}  // namespace std\n\n#if JSON_HAS_RANGES\n    template <typename IteratorType>\n    inline constexpr bool ::std::ranges::enable_borrowed_range<::nlohmann::detail::iteration_proxy<IteratorType>> = true;\n#endif\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/std_fs.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n//////////////////\n// constructors //\n//////////////////\n\n/*\n * Note all external_constructor<>::construct functions need to call\n * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an\n * allocated value (e.g., a string). See bug issue\n * https://github.com/nlohmann/json/issues/2865 for more information.\n */\n\ntemplate<value_t> struct external_constructor;\n\ntemplate<>\nstruct external_constructor<value_t::boolean>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::boolean;\n        j.m_value = b;\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::string>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::string;\n        j.m_value = s;\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::string;\n        j.m_value = std::move(s);\n        j.assert_invariant();\n    }\n\n    template < typename BasicJsonType, typename CompatibleStringType,\n               enable_if_t < !std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value,\n                             int > = 0 >\n    static void construct(BasicJsonType& j, const CompatibleStringType& str)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::string;\n        j.m_value.string = j.template create<typename BasicJsonType::string_t>(str);\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::binary>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::binary;\n        j.m_value = typename BasicJsonType::binary_t(b);\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::binary;\n        j.m_value = typename BasicJsonType::binary_t(std::move(b));\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::number_float>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::number_float;\n        j.m_value = val;\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::number_unsigned>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::number_unsigned;\n        j.m_value = val;\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::number_integer>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::number_integer;\n        j.m_value = val;\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::array>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::array;\n        j.m_value = arr;\n        j.set_parents();\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::array;\n        j.m_value = std::move(arr);\n        j.set_parents();\n        j.assert_invariant();\n    }\n\n    template < typename BasicJsonType, typename CompatibleArrayType,\n               enable_if_t < !std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value,\n                             int > = 0 >\n    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)\n    {\n        using std::begin;\n        using std::end;\n\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::array;\n        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));\n        j.set_parents();\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, const std::vector<bool>& arr)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::array;\n        j.m_value = value_t::array;\n        j.m_value.array->reserve(arr.size());\n        for (const bool x : arr)\n        {\n            j.m_value.array->push_back(x);\n            j.set_parent(j.m_value.array->back());\n        }\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType, typename T,\n             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>\n    static void construct(BasicJsonType& j, const std::valarray<T>& arr)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::array;\n        j.m_value = value_t::array;\n        j.m_value.array->resize(arr.size());\n        if (arr.size() > 0)\n        {\n            std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin());\n        }\n        j.set_parents();\n        j.assert_invariant();\n    }\n};\n\ntemplate<>\nstruct external_constructor<value_t::object>\n{\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::object;\n        j.m_value = obj;\n        j.set_parents();\n        j.assert_invariant();\n    }\n\n    template<typename BasicJsonType>\n    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)\n    {\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::object;\n        j.m_value = std::move(obj);\n        j.set_parents();\n        j.assert_invariant();\n    }\n\n    template < typename BasicJsonType, typename CompatibleObjectType,\n               enable_if_t < !std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int > = 0 >\n    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)\n    {\n        using std::begin;\n        using std::end;\n\n        j.m_value.destroy(j.m_type);\n        j.m_type = value_t::object;\n        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));\n        j.set_parents();\n        j.assert_invariant();\n    }\n};\n\n/////////////\n// to_json //\n/////////////\n\ntemplate<typename BasicJsonType, typename T,\n         enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>\ninline void to_json(BasicJsonType& j, T b) noexcept\n{\n    external_constructor<value_t::boolean>::construct(j, b);\n}\n\ntemplate < typename BasicJsonType, typename BoolRef,\n           enable_if_t <\n               ((std::is_same<std::vector<bool>::reference, BoolRef>::value\n                 && !std::is_same <std::vector<bool>::reference, typename BasicJsonType::boolean_t&>::value)\n                || (std::is_same<std::vector<bool>::const_reference, BoolRef>::value\n                    && !std::is_same <detail::uncvref_t<std::vector<bool>::const_reference>,\n                                      typename BasicJsonType::boolean_t >::value))\n               && std::is_convertible<const BoolRef&, typename BasicJsonType::boolean_t>::value, int > = 0 >\ninline void to_json(BasicJsonType& j, const BoolRef& b) noexcept\n{\n    external_constructor<value_t::boolean>::construct(j, static_cast<typename BasicJsonType::boolean_t>(b));\n}\n\ntemplate<typename BasicJsonType, typename CompatibleString,\n         enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0>\ninline void to_json(BasicJsonType& j, const CompatibleString& s)\n{\n    external_constructor<value_t::string>::construct(j, s);\n}\n\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)\n{\n    external_constructor<value_t::string>::construct(j, std::move(s));\n}\n\ntemplate<typename BasicJsonType, typename FloatType,\n         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>\ninline void to_json(BasicJsonType& j, FloatType val) noexcept\n{\n    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));\n}\n\ntemplate<typename BasicJsonType, typename CompatibleNumberUnsignedType,\n         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0>\ninline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept\n{\n    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));\n}\n\ntemplate<typename BasicJsonType, typename CompatibleNumberIntegerType,\n         enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0>\ninline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept\n{\n    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));\n}\n\n#if !JSON_DISABLE_ENUM_SERIALIZATION\ntemplate<typename BasicJsonType, typename EnumType,\n         enable_if_t<std::is_enum<EnumType>::value, int> = 0>\ninline void to_json(BasicJsonType& j, EnumType e) noexcept\n{\n    using underlying_type = typename std::underlying_type<EnumType>::type;\n    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));\n}\n#endif  // JSON_DISABLE_ENUM_SERIALIZATION\n\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, const std::vector<bool>& e)\n{\n    external_constructor<value_t::array>::construct(j, e);\n}\n\ntemplate < typename BasicJsonType, typename CompatibleArrayType,\n           enable_if_t < is_compatible_array_type<BasicJsonType,\n                         CompatibleArrayType>::value&&\n                         !is_compatible_object_type<BasicJsonType, CompatibleArrayType>::value&&\n                         !is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value&&\n                         !std::is_same<typename BasicJsonType::binary_t, CompatibleArrayType>::value&&\n                         !is_basic_json<CompatibleArrayType>::value,\n                         int > = 0 >\ninline void to_json(BasicJsonType& j, const CompatibleArrayType& arr)\n{\n    external_constructor<value_t::array>::construct(j, arr);\n}\n\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin)\n{\n    external_constructor<value_t::binary>::construct(j, bin);\n}\n\ntemplate<typename BasicJsonType, typename T,\n         enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>\ninline void to_json(BasicJsonType& j, const std::valarray<T>& arr)\n{\n    external_constructor<value_t::array>::construct(j, std::move(arr));\n}\n\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)\n{\n    external_constructor<value_t::array>::construct(j, std::move(arr));\n}\n\ntemplate < typename BasicJsonType, typename CompatibleObjectType,\n           enable_if_t < is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value&& !is_basic_json<CompatibleObjectType>::value, int > = 0 >\ninline void to_json(BasicJsonType& j, const CompatibleObjectType& obj)\n{\n    external_constructor<value_t::object>::construct(j, obj);\n}\n\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)\n{\n    external_constructor<value_t::object>::construct(j, std::move(obj));\n}\n\ntemplate <\n    typename BasicJsonType, typename T, std::size_t N,\n    enable_if_t < !std::is_constructible<typename BasicJsonType::string_t,\n                  const T(&)[N]>::value, // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n                  int > = 0 >\ninline void to_json(BasicJsonType& j, const T(&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n{\n    external_constructor<value_t::array>::construct(j, arr);\n}\n\ntemplate < typename BasicJsonType, typename T1, typename T2, enable_if_t < std::is_constructible<BasicJsonType, T1>::value&& std::is_constructible<BasicJsonType, T2>::value, int > = 0 >\ninline void to_json(BasicJsonType& j, const std::pair<T1, T2>& p)\n{\n    j = { p.first, p.second };\n}\n\n// for https://github.com/nlohmann/json/pull/1134\ntemplate<typename BasicJsonType, typename T,\n         enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0>\ninline void to_json(BasicJsonType& j, const T& b)\n{\n    j = { {b.key(), b.value()} };\n}\n\ntemplate<typename BasicJsonType, typename Tuple, std::size_t... Idx>\ninline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/)\n{\n    j = { std::get<Idx>(t)... };\n}\n\ntemplate<typename BasicJsonType, typename T, enable_if_t<is_constructible_tuple<BasicJsonType, T>::value, int > = 0>\ninline void to_json(BasicJsonType& j, const T& t)\n{\n    to_json_tuple_impl(j, t, make_index_sequence<std::tuple_size<T>::value> {});\n}\n\n#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM\ntemplate<typename BasicJsonType>\ninline void to_json(BasicJsonType& j, const std_fs::path& p)\n{\n    j = p.string();\n}\n#endif\n\nstruct to_json_fn\n{\n    template<typename BasicJsonType, typename T>\n    auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val))))\n    -> decltype(to_json(j, std::forward<T>(val)), void())\n    {\n        return to_json(j, std::forward<T>(val));\n    }\n};\n}  // namespace detail\n\n#ifndef JSON_HAS_CPP_17\n/// namespace to hold default `to_json` function\n/// to see why this is required:\n/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html\nnamespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces)\n{\n#endif\nJSON_INLINE_VARIABLE constexpr const auto& to_json = // NOLINT(misc-definitions-in-headers)\n    detail::static_const<detail::to_json_fn>::value;\n#ifndef JSON_HAS_CPP_17\n}  // namespace\n#endif\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/identity_tag.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/// @sa https://json.nlohmann.me/api/adl_serializer/\ntemplate<typename ValueType, typename>\nstruct adl_serializer\n{\n    /// @brief convert a JSON value to any value type\n    /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/\n    template<typename BasicJsonType, typename TargetType = ValueType>\n    static auto from_json(BasicJsonType && j, TargetType& val) noexcept(\n        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))\n    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void())\n    {\n        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);\n    }\n\n    /// @brief convert a JSON value to any value type\n    /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/\n    template<typename BasicJsonType, typename TargetType = ValueType>\n    static auto from_json(BasicJsonType && j) noexcept(\n    noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {})))\n    -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {}))\n    {\n        return ::nlohmann::from_json(std::forward<BasicJsonType>(j), detail::identity_tag<TargetType> {});\n    }\n\n    /// @brief convert any value type to a JSON value\n    /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/\n    template<typename BasicJsonType, typename TargetType = ValueType>\n    static auto to_json(BasicJsonType& j, TargetType && val) noexcept(\n        noexcept(::nlohmann::to_json(j, std::forward<TargetType>(val))))\n    -> decltype(::nlohmann::to_json(j, std::forward<TargetType>(val)), void())\n    {\n        ::nlohmann::to_json(j, std::forward<TargetType>(val));\n    }\n};\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/byte_container_with_subtype.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstdint> // uint8_t, uint64_t\n#include <tuple> // tie\n#include <utility> // move\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/// @brief an internal type for a backed binary type\n/// @sa https://json.nlohmann.me/api/byte_container_with_subtype/\ntemplate<typename BinaryType>\nclass byte_container_with_subtype : public BinaryType\n{\n  public:\n    using container_type = BinaryType;\n    using subtype_type = std::uint64_t;\n\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/\n    byte_container_with_subtype() noexcept(noexcept(container_type()))\n        : container_type()\n    {}\n\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/\n    byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b)))\n        : container_type(b)\n    {}\n\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/\n    byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b))))\n        : container_type(std::move(b))\n    {}\n\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/\n    byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b)))\n        : container_type(b)\n        , m_subtype(subtype_)\n        , m_has_subtype(true)\n    {}\n\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/\n    byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b))))\n        : container_type(std::move(b))\n        , m_subtype(subtype_)\n        , m_has_subtype(true)\n    {}\n\n    bool operator==(const byte_container_with_subtype& rhs) const\n    {\n        return std::tie(static_cast<const BinaryType&>(*this), m_subtype, m_has_subtype) ==\n               std::tie(static_cast<const BinaryType&>(rhs), rhs.m_subtype, rhs.m_has_subtype);\n    }\n\n    bool operator!=(const byte_container_with_subtype& rhs) const\n    {\n        return !(rhs == *this);\n    }\n\n    /// @brief sets the binary subtype\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/\n    void set_subtype(subtype_type subtype_) noexcept\n    {\n        m_subtype = subtype_;\n        m_has_subtype = true;\n    }\n\n    /// @brief return the binary subtype\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/\n    constexpr subtype_type subtype() const noexcept\n    {\n        return m_has_subtype ? m_subtype : static_cast<subtype_type>(-1);\n    }\n\n    /// @brief return whether the value has a subtype\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/\n    constexpr bool has_subtype() const noexcept\n    {\n        return m_has_subtype;\n    }\n\n    /// @brief clears the binary subtype\n    /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/\n    void clear_subtype() noexcept\n    {\n        m_subtype = 0;\n        m_has_subtype = false;\n    }\n\n  private:\n    subtype_type m_subtype = 0;\n    bool m_has_subtype = false;\n};\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/conversions/from_json.hpp>\n\n// #include <nlohmann/detail/conversions/to_json.hpp>\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/hash.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstdint> // uint8_t\n#include <cstddef> // size_t\n#include <functional> // hash\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n// boost::hash_combine\ninline std::size_t combine(std::size_t seed, std::size_t h) noexcept\n{\n    seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U);\n    return seed;\n}\n\n/*!\n@brief hash a JSON value\n\nThe hash function tries to rely on std::hash where possible. Furthermore, the\ntype of the JSON value is taken into account to have different hash values for\nnull, 0, 0U, and false, etc.\n\n@tparam BasicJsonType basic_json specialization\n@param j JSON value to hash\n@return hash value of j\n*/\ntemplate<typename BasicJsonType>\nstd::size_t hash(const BasicJsonType& j)\n{\n    using string_t = typename BasicJsonType::string_t;\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n\n    const auto type = static_cast<std::size_t>(j.type());\n    switch (j.type())\n    {\n        case BasicJsonType::value_t::null:\n        case BasicJsonType::value_t::discarded:\n        {\n            return combine(type, 0);\n        }\n\n        case BasicJsonType::value_t::object:\n        {\n            auto seed = combine(type, j.size());\n            for (const auto& element : j.items())\n            {\n                const auto h = std::hash<string_t> {}(element.key());\n                seed = combine(seed, h);\n                seed = combine(seed, hash(element.value()));\n            }\n            return seed;\n        }\n\n        case BasicJsonType::value_t::array:\n        {\n            auto seed = combine(type, j.size());\n            for (const auto& element : j)\n            {\n                seed = combine(seed, hash(element));\n            }\n            return seed;\n        }\n\n        case BasicJsonType::value_t::string:\n        {\n            const auto h = std::hash<string_t> {}(j.template get_ref<const string_t&>());\n            return combine(type, h);\n        }\n\n        case BasicJsonType::value_t::boolean:\n        {\n            const auto h = std::hash<bool> {}(j.template get<bool>());\n            return combine(type, h);\n        }\n\n        case BasicJsonType::value_t::number_integer:\n        {\n            const auto h = std::hash<number_integer_t> {}(j.template get<number_integer_t>());\n            return combine(type, h);\n        }\n\n        case BasicJsonType::value_t::number_unsigned:\n        {\n            const auto h = std::hash<number_unsigned_t> {}(j.template get<number_unsigned_t>());\n            return combine(type, h);\n        }\n\n        case BasicJsonType::value_t::number_float:\n        {\n            const auto h = std::hash<number_float_t> {}(j.template get<number_float_t>());\n            return combine(type, h);\n        }\n\n        case BasicJsonType::value_t::binary:\n        {\n            auto seed = combine(type, j.get_binary().size());\n            const auto h = std::hash<bool> {}(j.get_binary().has_subtype());\n            seed = combine(seed, h);\n            seed = combine(seed, static_cast<std::size_t>(j.get_binary().subtype()));\n            for (const auto byte : j.get_binary())\n            {\n                seed = combine(seed, std::hash<std::uint8_t> {}(byte));\n            }\n            return seed;\n        }\n\n        default:                   // LCOV_EXCL_LINE\n            JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n            return 0;              // LCOV_EXCL_LINE\n    }\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/input/binary_reader.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // generate_n\n#include <array> // array\n#include <cmath> // ldexp\n#include <cstddef> // size_t\n#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t\n#include <cstdio> // snprintf\n#include <cstring> // memcpy\n#include <iterator> // back_inserter\n#include <limits> // numeric_limits\n#include <string> // char_traits, string\n#include <utility> // make_pair, move\n#include <vector> // vector\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/input/input_adapters.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <array> // array\n#include <cstddef> // size_t\n#include <cstring> // strlen\n#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next\n#include <memory> // shared_ptr, make_shared, addressof\n#include <numeric> // accumulate\n#include <string> // string, char_traits\n#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer\n#include <utility> // pair, declval\n\n#ifndef JSON_NO_IO\n    #include <cstdio>   // FILE *\n    #include <istream>  // istream\n#endif                  // JSON_NO_IO\n\n// #include <nlohmann/detail/iterators/iterator_traits.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/// the supported input formats\nenum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata };\n\n////////////////////\n// input adapters //\n////////////////////\n\n#ifndef JSON_NO_IO\n/*!\nInput adapter for stdio file access. This adapter read only 1 byte and do not use any\n buffer. This adapter is a very low level adapter.\n*/\nclass file_input_adapter\n{\n  public:\n    using char_type = char;\n\n    JSON_HEDLEY_NON_NULL(2)\n    explicit file_input_adapter(std::FILE* f) noexcept\n        : m_file(f)\n    {\n        JSON_ASSERT(m_file != nullptr);\n    }\n\n    // make class move-only\n    file_input_adapter(const file_input_adapter&) = delete;\n    file_input_adapter(file_input_adapter&&) noexcept = default;\n    file_input_adapter& operator=(const file_input_adapter&) = delete;\n    file_input_adapter& operator=(file_input_adapter&&) = delete;\n    ~file_input_adapter() = default;\n\n    std::char_traits<char>::int_type get_character() noexcept\n    {\n        return std::fgetc(m_file);\n    }\n\n  private:\n    /// the file pointer to read from\n    std::FILE* m_file;\n};\n\n\n/*!\nInput adapter for a (caching) istream. Ignores a UFT Byte Order Mark at\nbeginning of input. Does not support changing the underlying std::streambuf\nin mid-input. Maintains underlying std::istream and std::streambuf to support\nsubsequent use of standard std::istream operations to process any input\ncharacters following those used in parsing the JSON input.  Clears the\nstd::istream flags; any input errors (e.g., EOF) will be detected by the first\nsubsequent call for input from the std::istream.\n*/\nclass input_stream_adapter\n{\n  public:\n    using char_type = char;\n\n    ~input_stream_adapter()\n    {\n        // clear stream flags; we use underlying streambuf I/O, do not\n        // maintain ifstream flags, except eof\n        if (is != nullptr)\n        {\n            is->clear(is->rdstate() & std::ios::eofbit);\n        }\n    }\n\n    explicit input_stream_adapter(std::istream& i)\n        : is(&i), sb(i.rdbuf())\n    {}\n\n    // delete because of pointer members\n    input_stream_adapter(const input_stream_adapter&) = delete;\n    input_stream_adapter& operator=(input_stream_adapter&) = delete;\n    input_stream_adapter& operator=(input_stream_adapter&&) = delete;\n\n    input_stream_adapter(input_stream_adapter&& rhs) noexcept\n        : is(rhs.is), sb(rhs.sb)\n    {\n        rhs.is = nullptr;\n        rhs.sb = nullptr;\n    }\n\n    // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to\n    // ensure that std::char_traits<char>::eof() and the character 0xFF do not\n    // end up as the same value, e.g. 0xFFFFFFFF.\n    std::char_traits<char>::int_type get_character()\n    {\n        auto res = sb->sbumpc();\n        // set eof manually, as we don't use the istream interface.\n        if (JSON_HEDLEY_UNLIKELY(res == std::char_traits<char>::eof()))\n        {\n            is->clear(is->rdstate() | std::ios::eofbit);\n        }\n        return res;\n    }\n\n  private:\n    /// the associated input stream\n    std::istream* is = nullptr;\n    std::streambuf* sb = nullptr;\n};\n#endif  // JSON_NO_IO\n\n// General-purpose iterator-based adapter. It might not be as fast as\n// theoretically possible for some containers, but it is extremely versatile.\ntemplate<typename IteratorType>\nclass iterator_input_adapter\n{\n  public:\n    using char_type = typename std::iterator_traits<IteratorType>::value_type;\n\n    iterator_input_adapter(IteratorType first, IteratorType last)\n        : current(std::move(first)), end(std::move(last))\n    {}\n\n    typename std::char_traits<char_type>::int_type get_character()\n    {\n        if (JSON_HEDLEY_LIKELY(current != end))\n        {\n            auto result = std::char_traits<char_type>::to_int_type(*current);\n            std::advance(current, 1);\n            return result;\n        }\n\n        return std::char_traits<char_type>::eof();\n    }\n\n  private:\n    IteratorType current;\n    IteratorType end;\n\n    template<typename BaseInputAdapter, size_t T>\n    friend struct wide_string_input_helper;\n\n    bool empty() const\n    {\n        return current == end;\n    }\n};\n\n\ntemplate<typename BaseInputAdapter, size_t T>\nstruct wide_string_input_helper;\n\ntemplate<typename BaseInputAdapter>\nstruct wide_string_input_helper<BaseInputAdapter, 4>\n{\n    // UTF-32\n    static void fill_buffer(BaseInputAdapter& input,\n                            std::array<std::char_traits<char>::int_type, 4>& utf8_bytes,\n                            size_t& utf8_bytes_index,\n                            size_t& utf8_bytes_filled)\n    {\n        utf8_bytes_index = 0;\n\n        if (JSON_HEDLEY_UNLIKELY(input.empty()))\n        {\n            utf8_bytes[0] = std::char_traits<char>::eof();\n            utf8_bytes_filled = 1;\n        }\n        else\n        {\n            // get the current character\n            const auto wc = input.get_character();\n\n            // UTF-32 to UTF-8 encoding\n            if (wc < 0x80)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);\n                utf8_bytes_filled = 1;\n            }\n            else if (wc <= 0x7FF)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((static_cast<unsigned int>(wc) >> 6u) & 0x1Fu));\n                utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));\n                utf8_bytes_filled = 2;\n            }\n            else if (wc <= 0xFFFF)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((static_cast<unsigned int>(wc) >> 12u) & 0x0Fu));\n                utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));\n                utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));\n                utf8_bytes_filled = 3;\n            }\n            else if (wc <= 0x10FFFF)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | ((static_cast<unsigned int>(wc) >> 18u) & 0x07u));\n                utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 12u) & 0x3Fu));\n                utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));\n                utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));\n                utf8_bytes_filled = 4;\n            }\n            else\n            {\n                // unknown character\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);\n                utf8_bytes_filled = 1;\n            }\n        }\n    }\n};\n\ntemplate<typename BaseInputAdapter>\nstruct wide_string_input_helper<BaseInputAdapter, 2>\n{\n    // UTF-16\n    static void fill_buffer(BaseInputAdapter& input,\n                            std::array<std::char_traits<char>::int_type, 4>& utf8_bytes,\n                            size_t& utf8_bytes_index,\n                            size_t& utf8_bytes_filled)\n    {\n        utf8_bytes_index = 0;\n\n        if (JSON_HEDLEY_UNLIKELY(input.empty()))\n        {\n            utf8_bytes[0] = std::char_traits<char>::eof();\n            utf8_bytes_filled = 1;\n        }\n        else\n        {\n            // get the current character\n            const auto wc = input.get_character();\n\n            // UTF-16 to UTF-8 encoding\n            if (wc < 0x80)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);\n                utf8_bytes_filled = 1;\n            }\n            else if (wc <= 0x7FF)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xC0u | ((static_cast<unsigned int>(wc) >> 6u)));\n                utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));\n                utf8_bytes_filled = 2;\n            }\n            else if (0xD800 > wc || wc >= 0xE000)\n            {\n                utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xE0u | ((static_cast<unsigned int>(wc) >> 12u)));\n                utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((static_cast<unsigned int>(wc) >> 6u) & 0x3Fu));\n                utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | (static_cast<unsigned int>(wc) & 0x3Fu));\n                utf8_bytes_filled = 3;\n            }\n            else\n            {\n                if (JSON_HEDLEY_UNLIKELY(!input.empty()))\n                {\n                    const auto wc2 = static_cast<unsigned int>(input.get_character());\n                    const auto charcode = 0x10000u + (((static_cast<unsigned int>(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu));\n                    utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(0xF0u | (charcode >> 18u));\n                    utf8_bytes[1] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu));\n                    utf8_bytes[2] = static_cast<std::char_traits<char>::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu));\n                    utf8_bytes[3] = static_cast<std::char_traits<char>::int_type>(0x80u | (charcode & 0x3Fu));\n                    utf8_bytes_filled = 4;\n                }\n                else\n                {\n                    utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);\n                    utf8_bytes_filled = 1;\n                }\n            }\n        }\n    }\n};\n\n// Wraps another input apdater to convert wide character types into individual bytes.\ntemplate<typename BaseInputAdapter, typename WideCharType>\nclass wide_string_input_adapter\n{\n  public:\n    using char_type = char;\n\n    wide_string_input_adapter(BaseInputAdapter base)\n        : base_adapter(base) {}\n\n    typename std::char_traits<char>::int_type get_character() noexcept\n    {\n        // check if buffer needs to be filled\n        if (utf8_bytes_index == utf8_bytes_filled)\n        {\n            fill_buffer<sizeof(WideCharType)>();\n\n            JSON_ASSERT(utf8_bytes_filled > 0);\n            JSON_ASSERT(utf8_bytes_index == 0);\n        }\n\n        // use buffer\n        JSON_ASSERT(utf8_bytes_filled > 0);\n        JSON_ASSERT(utf8_bytes_index < utf8_bytes_filled);\n        return utf8_bytes[utf8_bytes_index++];\n    }\n\n  private:\n    BaseInputAdapter base_adapter;\n\n    template<size_t T>\n    void fill_buffer()\n    {\n        wide_string_input_helper<BaseInputAdapter, T>::fill_buffer(base_adapter, utf8_bytes, utf8_bytes_index, utf8_bytes_filled);\n    }\n\n    /// a buffer for UTF-8 bytes\n    std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};\n\n    /// index to the utf8_codes array for the next valid byte\n    std::size_t utf8_bytes_index = 0;\n    /// number of valid bytes in the utf8_codes array\n    std::size_t utf8_bytes_filled = 0;\n};\n\n\ntemplate<typename IteratorType, typename Enable = void>\nstruct iterator_input_adapter_factory\n{\n    using iterator_type = IteratorType;\n    using char_type = typename std::iterator_traits<iterator_type>::value_type;\n    using adapter_type = iterator_input_adapter<iterator_type>;\n\n    static adapter_type create(IteratorType first, IteratorType last)\n    {\n        return adapter_type(std::move(first), std::move(last));\n    }\n};\n\ntemplate<typename T>\nstruct is_iterator_of_multibyte\n{\n    using value_type = typename std::iterator_traits<T>::value_type;\n    enum\n    {\n        value = sizeof(value_type) > 1\n    };\n};\n\ntemplate<typename IteratorType>\nstruct iterator_input_adapter_factory<IteratorType, enable_if_t<is_iterator_of_multibyte<IteratorType>::value>>\n{\n    using iterator_type = IteratorType;\n    using char_type = typename std::iterator_traits<iterator_type>::value_type;\n    using base_adapter_type = iterator_input_adapter<iterator_type>;\n    using adapter_type = wide_string_input_adapter<base_adapter_type, char_type>;\n\n    static adapter_type create(IteratorType first, IteratorType last)\n    {\n        return adapter_type(base_adapter_type(std::move(first), std::move(last)));\n    }\n};\n\n// General purpose iterator-based input\ntemplate<typename IteratorType>\ntypename iterator_input_adapter_factory<IteratorType>::adapter_type input_adapter(IteratorType first, IteratorType last)\n{\n    using factory_type = iterator_input_adapter_factory<IteratorType>;\n    return factory_type::create(first, last);\n}\n\n// Convenience shorthand from container to iterator\n// Enables ADL on begin(container) and end(container)\n// Encloses the using declarations in namespace for not to leak them to outside scope\n\nnamespace container_input_adapter_factory_impl\n{\n\nusing std::begin;\nusing std::end;\n\ntemplate<typename ContainerType, typename Enable = void>\nstruct container_input_adapter_factory {};\n\ntemplate<typename ContainerType>\nstruct container_input_adapter_factory< ContainerType,\n       void_t<decltype(begin(std::declval<ContainerType>()), end(std::declval<ContainerType>()))>>\n       {\n           using adapter_type = decltype(input_adapter(begin(std::declval<ContainerType>()), end(std::declval<ContainerType>())));\n\n           static adapter_type create(const ContainerType& container)\n{\n    return input_adapter(begin(container), end(container));\n}\n       };\n\n}  // namespace container_input_adapter_factory_impl\n\ntemplate<typename ContainerType>\ntypename container_input_adapter_factory_impl::container_input_adapter_factory<ContainerType>::adapter_type input_adapter(const ContainerType& container)\n{\n    return container_input_adapter_factory_impl::container_input_adapter_factory<ContainerType>::create(container);\n}\n\n#ifndef JSON_NO_IO\n// Special cases with fast paths\ninline file_input_adapter input_adapter(std::FILE* file)\n{\n    return file_input_adapter(file);\n}\n\ninline input_stream_adapter input_adapter(std::istream& stream)\n{\n    return input_stream_adapter(stream);\n}\n\ninline input_stream_adapter input_adapter(std::istream&& stream)\n{\n    return input_stream_adapter(stream);\n}\n#endif  // JSON_NO_IO\n\nusing contiguous_bytes_input_adapter = decltype(input_adapter(std::declval<const char*>(), std::declval<const char*>()));\n\n// Null-delimited strings, and the like.\ntemplate < typename CharT,\n           typename std::enable_if <\n               std::is_pointer<CharT>::value&&\n               !std::is_array<CharT>::value&&\n               std::is_integral<typename std::remove_pointer<CharT>::type>::value&&\n               sizeof(typename std::remove_pointer<CharT>::type) == 1,\n               int >::type = 0 >\ncontiguous_bytes_input_adapter input_adapter(CharT b)\n{\n    auto length = std::strlen(reinterpret_cast<const char*>(b));\n    const auto* ptr = reinterpret_cast<const char*>(b);\n    return input_adapter(ptr, ptr + length);\n}\n\ntemplate<typename T, std::size_t N>\nauto input_adapter(T (&array)[N]) -> decltype(input_adapter(array, array + N)) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n{\n    return input_adapter(array, array + N);\n}\n\n// This class only handles inputs of input_buffer_adapter type.\n// It's required so that expressions like {ptr, len} can be implicitly cast\n// to the correct adapter.\nclass span_input_adapter\n{\n  public:\n    template < typename CharT,\n               typename std::enable_if <\n                   std::is_pointer<CharT>::value&&\n                   std::is_integral<typename std::remove_pointer<CharT>::type>::value&&\n                   sizeof(typename std::remove_pointer<CharT>::type) == 1,\n                   int >::type = 0 >\n    span_input_adapter(CharT b, std::size_t l)\n        : ia(reinterpret_cast<const char*>(b), reinterpret_cast<const char*>(b) + l) {}\n\n    template<class IteratorType,\n             typename std::enable_if<\n                 std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,\n                 int>::type = 0>\n    span_input_adapter(IteratorType first, IteratorType last)\n        : ia(input_adapter(first, last)) {}\n\n    contiguous_bytes_input_adapter&& get()\n    {\n        return std::move(ia); // NOLINT(hicpp-move-const-arg,performance-move-const-arg)\n    }\n\n  private:\n    contiguous_bytes_input_adapter ia;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/input/json_sax.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef>\n#include <string> // string\n#include <utility> // move\n#include <vector> // vector\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/*!\n@brief SAX interface\n\nThis class describes the SAX interface used by @ref nlohmann::json::sax_parse.\nEach function is called in different situations while the input is parsed. The\nboolean return value informs the parser whether to continue processing the\ninput.\n*/\ntemplate<typename BasicJsonType>\nstruct json_sax\n{\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n\n    /*!\n    @brief a null value was read\n    @return whether parsing should proceed\n    */\n    virtual bool null() = 0;\n\n    /*!\n    @brief a boolean value was read\n    @param[in] val  boolean value\n    @return whether parsing should proceed\n    */\n    virtual bool boolean(bool val) = 0;\n\n    /*!\n    @brief an integer number was read\n    @param[in] val  integer value\n    @return whether parsing should proceed\n    */\n    virtual bool number_integer(number_integer_t val) = 0;\n\n    /*!\n    @brief an unsigned integer number was read\n    @param[in] val  unsigned integer value\n    @return whether parsing should proceed\n    */\n    virtual bool number_unsigned(number_unsigned_t val) = 0;\n\n    /*!\n    @brief a floating-point number was read\n    @param[in] val  floating-point value\n    @param[in] s    raw token value\n    @return whether parsing should proceed\n    */\n    virtual bool number_float(number_float_t val, const string_t& s) = 0;\n\n    /*!\n    @brief a string value was read\n    @param[in] val  string value\n    @return whether parsing should proceed\n    @note It is safe to move the passed string value.\n    */\n    virtual bool string(string_t& val) = 0;\n\n    /*!\n    @brief a binary value was read\n    @param[in] val  binary value\n    @return whether parsing should proceed\n    @note It is safe to move the passed binary value.\n    */\n    virtual bool binary(binary_t& val) = 0;\n\n    /*!\n    @brief the beginning of an object was read\n    @param[in] elements  number of object elements or -1 if unknown\n    @return whether parsing should proceed\n    @note binary formats may report the number of elements\n    */\n    virtual bool start_object(std::size_t elements) = 0;\n\n    /*!\n    @brief an object key was read\n    @param[in] val  object key\n    @return whether parsing should proceed\n    @note It is safe to move the passed string.\n    */\n    virtual bool key(string_t& val) = 0;\n\n    /*!\n    @brief the end of an object was read\n    @return whether parsing should proceed\n    */\n    virtual bool end_object() = 0;\n\n    /*!\n    @brief the beginning of an array was read\n    @param[in] elements  number of array elements or -1 if unknown\n    @return whether parsing should proceed\n    @note binary formats may report the number of elements\n    */\n    virtual bool start_array(std::size_t elements) = 0;\n\n    /*!\n    @brief the end of an array was read\n    @return whether parsing should proceed\n    */\n    virtual bool end_array() = 0;\n\n    /*!\n    @brief a parse error occurred\n    @param[in] position    the position in the input where the error occurs\n    @param[in] last_token  the last read token\n    @param[in] ex          an exception object describing the error\n    @return whether parsing should proceed (must return false)\n    */\n    virtual bool parse_error(std::size_t position,\n                             const std::string& last_token,\n                             const detail::exception& ex) = 0;\n\n    json_sax() = default;\n    json_sax(const json_sax&) = default;\n    json_sax(json_sax&&) noexcept = default;\n    json_sax& operator=(const json_sax&) = default;\n    json_sax& operator=(json_sax&&) noexcept = default;\n    virtual ~json_sax() = default;\n};\n\n\nnamespace detail\n{\n/*!\n@brief SAX implementation to create a JSON value from SAX events\n\nThis class implements the @ref json_sax interface and processes the SAX events\nto create a JSON value which makes it basically a DOM parser. The structure or\nhierarchy of the JSON value is managed by the stack `ref_stack` which contains\na pointer to the respective array or object for each recursion depth.\n\nAfter successful parsing, the value that is passed by reference to the\nconstructor contains the parsed value.\n\n@tparam BasicJsonType  the JSON type\n*/\ntemplate<typename BasicJsonType>\nclass json_sax_dom_parser\n{\n  public:\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n\n    /*!\n    @param[in,out] r  reference to a JSON value that is manipulated while\n                       parsing\n    @param[in] allow_exceptions_  whether parse errors yield exceptions\n    */\n    explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true)\n        : root(r), allow_exceptions(allow_exceptions_)\n    {}\n\n    // make class move-only\n    json_sax_dom_parser(const json_sax_dom_parser&) = delete;\n    json_sax_dom_parser(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete;\n    json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    ~json_sax_dom_parser() = default;\n\n    bool null()\n    {\n        handle_value(nullptr);\n        return true;\n    }\n\n    bool boolean(bool val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_integer(number_integer_t val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_unsigned(number_unsigned_t val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_float(number_float_t val, const string_t& /*unused*/)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool string(string_t& val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool binary(binary_t& val)\n    {\n        handle_value(std::move(val));\n        return true;\n    }\n\n    bool start_object(std::size_t len)\n    {\n        ref_stack.push_back(handle_value(BasicJsonType::value_t::object));\n\n        if (JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))\n        {\n            JSON_THROW(out_of_range::create(408, concat(\"excessive object size: \", std::to_string(len)), ref_stack.back()));\n        }\n\n        return true;\n    }\n\n    bool key(string_t& val)\n    {\n        JSON_ASSERT(!ref_stack.empty());\n        JSON_ASSERT(ref_stack.back()->is_object());\n\n        // add null at given key and store the reference for later\n        object_element = &(ref_stack.back()->m_value.object->operator[](val));\n        return true;\n    }\n\n    bool end_object()\n    {\n        JSON_ASSERT(!ref_stack.empty());\n        JSON_ASSERT(ref_stack.back()->is_object());\n\n        ref_stack.back()->set_parents();\n        ref_stack.pop_back();\n        return true;\n    }\n\n    bool start_array(std::size_t len)\n    {\n        ref_stack.push_back(handle_value(BasicJsonType::value_t::array));\n\n        if (JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))\n        {\n            JSON_THROW(out_of_range::create(408, concat(\"excessive array size: \", std::to_string(len)), ref_stack.back()));\n        }\n\n        return true;\n    }\n\n    bool end_array()\n    {\n        JSON_ASSERT(!ref_stack.empty());\n        JSON_ASSERT(ref_stack.back()->is_array());\n\n        ref_stack.back()->set_parents();\n        ref_stack.pop_back();\n        return true;\n    }\n\n    template<class Exception>\n    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,\n                     const Exception& ex)\n    {\n        errored = true;\n        static_cast<void>(ex);\n        if (allow_exceptions)\n        {\n            JSON_THROW(ex);\n        }\n        return false;\n    }\n\n    constexpr bool is_errored() const\n    {\n        return errored;\n    }\n\n  private:\n    /*!\n    @invariant If the ref stack is empty, then the passed value will be the new\n               root.\n    @invariant If the ref stack contains a value, then it is an array or an\n               object to which we can add elements\n    */\n    template<typename Value>\n    JSON_HEDLEY_RETURNS_NON_NULL\n    BasicJsonType* handle_value(Value&& v)\n    {\n        if (ref_stack.empty())\n        {\n            root = BasicJsonType(std::forward<Value>(v));\n            return &root;\n        }\n\n        JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object());\n\n        if (ref_stack.back()->is_array())\n        {\n            ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v));\n            return &(ref_stack.back()->m_value.array->back());\n        }\n\n        JSON_ASSERT(ref_stack.back()->is_object());\n        JSON_ASSERT(object_element);\n        *object_element = BasicJsonType(std::forward<Value>(v));\n        return object_element;\n    }\n\n    /// the parsed JSON value\n    BasicJsonType& root;\n    /// stack to model hierarchy of values\n    std::vector<BasicJsonType*> ref_stack {};\n    /// helper to hold the reference for the next object element\n    BasicJsonType* object_element = nullptr;\n    /// whether a syntax error occurred\n    bool errored = false;\n    /// whether to throw exceptions in case of errors\n    const bool allow_exceptions = true;\n};\n\ntemplate<typename BasicJsonType>\nclass json_sax_dom_callback_parser\n{\n  public:\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n    using parser_callback_t = typename BasicJsonType::parser_callback_t;\n    using parse_event_t = typename BasicJsonType::parse_event_t;\n\n    json_sax_dom_callback_parser(BasicJsonType& r,\n                                 const parser_callback_t cb,\n                                 const bool allow_exceptions_ = true)\n        : root(r), callback(cb), allow_exceptions(allow_exceptions_)\n    {\n        keep_stack.push_back(true);\n    }\n\n    // make class move-only\n    json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete;\n    json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete;\n    json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    ~json_sax_dom_callback_parser() = default;\n\n    bool null()\n    {\n        handle_value(nullptr);\n        return true;\n    }\n\n    bool boolean(bool val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_integer(number_integer_t val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_unsigned(number_unsigned_t val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool number_float(number_float_t val, const string_t& /*unused*/)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool string(string_t& val)\n    {\n        handle_value(val);\n        return true;\n    }\n\n    bool binary(binary_t& val)\n    {\n        handle_value(std::move(val));\n        return true;\n    }\n\n    bool start_object(std::size_t len)\n    {\n        // check callback for object start\n        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded);\n        keep_stack.push_back(keep);\n\n        auto val = handle_value(BasicJsonType::value_t::object, true);\n        ref_stack.push_back(val.second);\n\n        // check object limit\n        if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))\n        {\n            JSON_THROW(out_of_range::create(408, concat(\"excessive object size: \", std::to_string(len)), ref_stack.back()));\n        }\n\n        return true;\n    }\n\n    bool key(string_t& val)\n    {\n        BasicJsonType k = BasicJsonType(val);\n\n        // check callback for key\n        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k);\n        key_keep_stack.push_back(keep);\n\n        // add discarded value at given key and store the reference for later\n        if (keep && ref_stack.back())\n        {\n            object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded);\n        }\n\n        return true;\n    }\n\n    bool end_object()\n    {\n        if (ref_stack.back())\n        {\n            if (!callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back()))\n            {\n                // discard object\n                *ref_stack.back() = discarded;\n            }\n            else\n            {\n                ref_stack.back()->set_parents();\n            }\n        }\n\n        JSON_ASSERT(!ref_stack.empty());\n        JSON_ASSERT(!keep_stack.empty());\n        ref_stack.pop_back();\n        keep_stack.pop_back();\n\n        if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())\n        {\n            // remove discarded value\n            for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)\n            {\n                if (it->is_discarded())\n                {\n                    ref_stack.back()->erase(it);\n                    break;\n                }\n            }\n        }\n\n        return true;\n    }\n\n    bool start_array(std::size_t len)\n    {\n        const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded);\n        keep_stack.push_back(keep);\n\n        auto val = handle_value(BasicJsonType::value_t::array, true);\n        ref_stack.push_back(val.second);\n\n        // check array limit\n        if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast<std::size_t>(-1) && len > ref_stack.back()->max_size()))\n        {\n            JSON_THROW(out_of_range::create(408, concat(\"excessive array size: \", std::to_string(len)), ref_stack.back()));\n        }\n\n        return true;\n    }\n\n    bool end_array()\n    {\n        bool keep = true;\n\n        if (ref_stack.back())\n        {\n            keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());\n            if (keep)\n            {\n                ref_stack.back()->set_parents();\n            }\n            else\n            {\n                // discard array\n                *ref_stack.back() = discarded;\n            }\n        }\n\n        JSON_ASSERT(!ref_stack.empty());\n        JSON_ASSERT(!keep_stack.empty());\n        ref_stack.pop_back();\n        keep_stack.pop_back();\n\n        // remove discarded value\n        if (!keep && !ref_stack.empty() && ref_stack.back()->is_array())\n        {\n            ref_stack.back()->m_value.array->pop_back();\n        }\n\n        return true;\n    }\n\n    template<class Exception>\n    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/,\n                     const Exception& ex)\n    {\n        errored = true;\n        static_cast<void>(ex);\n        if (allow_exceptions)\n        {\n            JSON_THROW(ex);\n        }\n        return false;\n    }\n\n    constexpr bool is_errored() const\n    {\n        return errored;\n    }\n\n  private:\n    /*!\n    @param[in] v  value to add to the JSON value we build during parsing\n    @param[in] skip_callback  whether we should skip calling the callback\n               function; this is required after start_array() and\n               start_object() SAX events, because otherwise we would call the\n               callback function with an empty array or object, respectively.\n\n    @invariant If the ref stack is empty, then the passed value will be the new\n               root.\n    @invariant If the ref stack contains a value, then it is an array or an\n               object to which we can add elements\n\n    @return pair of boolean (whether value should be kept) and pointer (to the\n            passed value in the ref_stack hierarchy; nullptr if not kept)\n    */\n    template<typename Value>\n    std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false)\n    {\n        JSON_ASSERT(!keep_stack.empty());\n\n        // do not handle this value if we know it would be added to a discarded\n        // container\n        if (!keep_stack.back())\n        {\n            return {false, nullptr};\n        }\n\n        // create value\n        auto value = BasicJsonType(std::forward<Value>(v));\n\n        // check callback\n        const bool keep = skip_callback || callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value);\n\n        // do not handle this value if we just learnt it shall be discarded\n        if (!keep)\n        {\n            return {false, nullptr};\n        }\n\n        if (ref_stack.empty())\n        {\n            root = std::move(value);\n            return {true, &root};\n        }\n\n        // skip this value if we already decided to skip the parent\n        // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360)\n        if (!ref_stack.back())\n        {\n            return {false, nullptr};\n        }\n\n        // we now only expect arrays and objects\n        JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object());\n\n        // array\n        if (ref_stack.back()->is_array())\n        {\n            ref_stack.back()->m_value.array->emplace_back(std::move(value));\n            return {true, &(ref_stack.back()->m_value.array->back())};\n        }\n\n        // object\n        JSON_ASSERT(ref_stack.back()->is_object());\n        // check if we should store an element for the current key\n        JSON_ASSERT(!key_keep_stack.empty());\n        const bool store_element = key_keep_stack.back();\n        key_keep_stack.pop_back();\n\n        if (!store_element)\n        {\n            return {false, nullptr};\n        }\n\n        JSON_ASSERT(object_element);\n        *object_element = std::move(value);\n        return {true, object_element};\n    }\n\n    /// the parsed JSON value\n    BasicJsonType& root;\n    /// stack to model hierarchy of values\n    std::vector<BasicJsonType*> ref_stack {};\n    /// stack to manage which values to keep\n    std::vector<bool> keep_stack {};\n    /// stack to manage which object keys to keep\n    std::vector<bool> key_keep_stack {};\n    /// helper to hold the reference for the next object element\n    BasicJsonType* object_element = nullptr;\n    /// whether a syntax error occurred\n    bool errored = false;\n    /// callback function\n    const parser_callback_t callback = nullptr;\n    /// whether to throw exceptions in case of errors\n    const bool allow_exceptions = true;\n    /// a discarded value for the callback\n    BasicJsonType discarded = BasicJsonType::value_t::discarded;\n};\n\ntemplate<typename BasicJsonType>\nclass json_sax_acceptor\n{\n  public:\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n\n    bool null()\n    {\n        return true;\n    }\n\n    bool boolean(bool /*unused*/)\n    {\n        return true;\n    }\n\n    bool number_integer(number_integer_t /*unused*/)\n    {\n        return true;\n    }\n\n    bool number_unsigned(number_unsigned_t /*unused*/)\n    {\n        return true;\n    }\n\n    bool number_float(number_float_t /*unused*/, const string_t& /*unused*/)\n    {\n        return true;\n    }\n\n    bool string(string_t& /*unused*/)\n    {\n        return true;\n    }\n\n    bool binary(binary_t& /*unused*/)\n    {\n        return true;\n    }\n\n    bool start_object(std::size_t /*unused*/ = static_cast<std::size_t>(-1))\n    {\n        return true;\n    }\n\n    bool key(string_t& /*unused*/)\n    {\n        return true;\n    }\n\n    bool end_object()\n    {\n        return true;\n    }\n\n    bool start_array(std::size_t /*unused*/ = static_cast<std::size_t>(-1))\n    {\n        return true;\n    }\n\n    bool end_array()\n    {\n        return true;\n    }\n\n    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/)\n    {\n        return false;\n    }\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/input/lexer.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <array> // array\n#include <clocale> // localeconv\n#include <cstddef> // size_t\n#include <cstdio> // snprintf\n#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull\n#include <initializer_list> // initializer_list\n#include <string> // char_traits, string\n#include <utility> // move\n#include <vector> // vector\n\n// #include <nlohmann/detail/input/input_adapters.hpp>\n\n// #include <nlohmann/detail/input/position_t.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n///////////\n// lexer //\n///////////\n\ntemplate<typename BasicJsonType>\nclass lexer_base\n{\n  public:\n    /// token types for the parser\n    enum class token_type\n    {\n        uninitialized,    ///< indicating the scanner is uninitialized\n        literal_true,     ///< the `true` literal\n        literal_false,    ///< the `false` literal\n        literal_null,     ///< the `null` literal\n        value_string,     ///< a string -- use get_string() for actual value\n        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value\n        value_integer,    ///< a signed integer -- use get_number_integer() for actual value\n        value_float,      ///< an floating point number -- use get_number_float() for actual value\n        begin_array,      ///< the character for array begin `[`\n        begin_object,     ///< the character for object begin `{`\n        end_array,        ///< the character for array end `]`\n        end_object,       ///< the character for object end `}`\n        name_separator,   ///< the name separator `:`\n        value_separator,  ///< the value separator `,`\n        parse_error,      ///< indicating a parse error\n        end_of_input,     ///< indicating the end of the input buffer\n        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)\n    };\n\n    /// return name of values of type token_type (only used for errors)\n    JSON_HEDLEY_RETURNS_NON_NULL\n    JSON_HEDLEY_CONST\n    static const char* token_type_name(const token_type t) noexcept\n    {\n        switch (t)\n        {\n            case token_type::uninitialized:\n                return \"<uninitialized>\";\n            case token_type::literal_true:\n                return \"true literal\";\n            case token_type::literal_false:\n                return \"false literal\";\n            case token_type::literal_null:\n                return \"null literal\";\n            case token_type::value_string:\n                return \"string literal\";\n            case token_type::value_unsigned:\n            case token_type::value_integer:\n            case token_type::value_float:\n                return \"number literal\";\n            case token_type::begin_array:\n                return \"'['\";\n            case token_type::begin_object:\n                return \"'{'\";\n            case token_type::end_array:\n                return \"']'\";\n            case token_type::end_object:\n                return \"'}'\";\n            case token_type::name_separator:\n                return \"':'\";\n            case token_type::value_separator:\n                return \"','\";\n            case token_type::parse_error:\n                return \"<parse error>\";\n            case token_type::end_of_input:\n                return \"end of input\";\n            case token_type::literal_or_value:\n                return \"'[', '{', or a literal\";\n            // LCOV_EXCL_START\n            default: // catch non-enum values\n                return \"unknown token\";\n                // LCOV_EXCL_STOP\n        }\n    }\n};\n/*!\n@brief lexical analysis\n\nThis class organizes the lexical analysis during JSON deserialization.\n*/\ntemplate<typename BasicJsonType, typename InputAdapterType>\nclass lexer : public lexer_base<BasicJsonType>\n{\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using char_type = typename InputAdapterType::char_type;\n    using char_int_type = typename std::char_traits<char_type>::int_type;\n\n  public:\n    using token_type = typename lexer_base<BasicJsonType>::token_type;\n\n    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept\n        : ia(std::move(adapter))\n        , ignore_comments(ignore_comments_)\n        , decimal_point_char(static_cast<char_int_type>(get_decimal_point()))\n    {}\n\n    // delete because of pointer members\n    lexer(const lexer&) = delete;\n    lexer(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    lexer& operator=(lexer&) = delete;\n    lexer& operator=(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    ~lexer() = default;\n\n  private:\n    /////////////////////\n    // locales\n    /////////////////////\n\n    /// return the locale-dependent decimal point\n    JSON_HEDLEY_PURE\n    static char get_decimal_point() noexcept\n    {\n        const auto* loc = localeconv();\n        JSON_ASSERT(loc != nullptr);\n        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);\n    }\n\n    /////////////////////\n    // scan functions\n    /////////////////////\n\n    /*!\n    @brief get codepoint from 4 hex characters following `\\u`\n\n    For input \"\\u c1 c2 c3 c4\" the codepoint is:\n      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4\n    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)\n\n    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'\n    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The\n    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)\n    between the ASCII value of the character and the desired integer value.\n\n    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or\n            non-hex character)\n    */\n    int get_codepoint()\n    {\n        // this function only makes sense after reading `\\u`\n        JSON_ASSERT(current == 'u');\n        int codepoint = 0;\n\n        const auto factors = { 12u, 8u, 4u, 0u };\n        for (const auto factor : factors)\n        {\n            get();\n\n            if (current >= '0' && current <= '9')\n            {\n                codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x30u) << factor);\n            }\n            else if (current >= 'A' && current <= 'F')\n            {\n                codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x37u) << factor);\n            }\n            else if (current >= 'a' && current <= 'f')\n            {\n                codepoint += static_cast<int>((static_cast<unsigned int>(current) - 0x57u) << factor);\n            }\n            else\n            {\n                return -1;\n            }\n        }\n\n        JSON_ASSERT(0x0000 <= codepoint && codepoint <= 0xFFFF);\n        return codepoint;\n    }\n\n    /*!\n    @brief check if the next byte(s) are inside a given range\n\n    Adds the current byte and, for each passed range, reads a new byte and\n    checks if it is inside the range. If a violation was detected, set up an\n    error message and return false. Otherwise, return true.\n\n    @param[in] ranges  list of integers; interpreted as list of pairs of\n                       inclusive lower and upper bound, respectively\n\n    @pre The passed list @a ranges must have 2, 4, or 6 elements; that is,\n         1, 2, or 3 pairs. This precondition is enforced by an assertion.\n\n    @return true if and only if no range violation was detected\n    */\n    bool next_byte_in_range(std::initializer_list<char_int_type> ranges)\n    {\n        JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);\n        add(current);\n\n        for (auto range = ranges.begin(); range != ranges.end(); ++range)\n        {\n            get();\n            if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range)))\n            {\n                add(current);\n            }\n            else\n            {\n                error_message = \"invalid string: ill-formed UTF-8 byte\";\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    /*!\n    @brief scan a string literal\n\n    This function scans a string according to Sect. 7 of RFC 8259. While\n    scanning, bytes are escaped and copied into buffer token_buffer. Then the\n    function returns successfully, token_buffer is *not* null-terminated (as it\n    may contain \\0 bytes), and token_buffer.size() is the number of bytes in the\n    string.\n\n    @return token_type::value_string if string could be successfully scanned,\n            token_type::parse_error otherwise\n\n    @note In case of errors, variable error_message contains a textual\n          description.\n    */\n    token_type scan_string()\n    {\n        // reset token_buffer (ignore opening quote)\n        reset();\n\n        // we entered the function by reading an open quote\n        JSON_ASSERT(current == '\\\"');\n\n        while (true)\n        {\n            // get next character\n            switch (get())\n            {\n                // end of file while parsing string\n                case std::char_traits<char_type>::eof():\n                {\n                    error_message = \"invalid string: missing closing quote\";\n                    return token_type::parse_error;\n                }\n\n                // closing quote\n                case '\\\"':\n                {\n                    return token_type::value_string;\n                }\n\n                // escapes\n                case '\\\\':\n                {\n                    switch (get())\n                    {\n                        // quotation mark\n                        case '\\\"':\n                            add('\\\"');\n                            break;\n                        // reverse solidus\n                        case '\\\\':\n                            add('\\\\');\n                            break;\n                        // solidus\n                        case '/':\n                            add('/');\n                            break;\n                        // backspace\n                        case 'b':\n                            add('\\b');\n                            break;\n                        // form feed\n                        case 'f':\n                            add('\\f');\n                            break;\n                        // line feed\n                        case 'n':\n                            add('\\n');\n                            break;\n                        // carriage return\n                        case 'r':\n                            add('\\r');\n                            break;\n                        // tab\n                        case 't':\n                            add('\\t');\n                            break;\n\n                        // unicode escapes\n                        case 'u':\n                        {\n                            const int codepoint1 = get_codepoint();\n                            int codepoint = codepoint1; // start with codepoint1\n\n                            if (JSON_HEDLEY_UNLIKELY(codepoint1 == -1))\n                            {\n                                error_message = \"invalid string: '\\\\u' must be followed by 4 hex digits\";\n                                return token_type::parse_error;\n                            }\n\n                            // check if code point is a high surrogate\n                            if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF)\n                            {\n                                // expect next \\uxxxx entry\n                                if (JSON_HEDLEY_LIKELY(get() == '\\\\' && get() == 'u'))\n                                {\n                                    const int codepoint2 = get_codepoint();\n\n                                    if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1))\n                                    {\n                                        error_message = \"invalid string: '\\\\u' must be followed by 4 hex digits\";\n                                        return token_type::parse_error;\n                                    }\n\n                                    // check if codepoint2 is a low surrogate\n                                    if (JSON_HEDLEY_LIKELY(0xDC00 <= codepoint2 && codepoint2 <= 0xDFFF))\n                                    {\n                                        // overwrite codepoint\n                                        codepoint = static_cast<int>(\n                                                        // high surrogate occupies the most significant 22 bits\n                                                        (static_cast<unsigned int>(codepoint1) << 10u)\n                                                        // low surrogate occupies the least significant 15 bits\n                                                        + static_cast<unsigned int>(codepoint2)\n                                                        // there is still the 0xD800, 0xDC00 and 0x10000 noise\n                                                        // in the result, so we have to subtract with:\n                                                        // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00\n                                                        - 0x35FDC00u);\n                                    }\n                                    else\n                                    {\n                                        error_message = \"invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF\";\n                                        return token_type::parse_error;\n                                    }\n                                }\n                                else\n                                {\n                                    error_message = \"invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF\";\n                                    return token_type::parse_error;\n                                }\n                            }\n                            else\n                            {\n                                if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF))\n                                {\n                                    error_message = \"invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF\";\n                                    return token_type::parse_error;\n                                }\n                            }\n\n                            // result of the above calculation yields a proper codepoint\n                            JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);\n\n                            // translate codepoint into bytes\n                            if (codepoint < 0x80)\n                            {\n                                // 1-byte characters: 0xxxxxxx (ASCII)\n                                add(static_cast<char_int_type>(codepoint));\n                            }\n                            else if (codepoint <= 0x7FF)\n                            {\n                                // 2-byte characters: 110xxxxx 10xxxxxx\n                                add(static_cast<char_int_type>(0xC0u | (static_cast<unsigned int>(codepoint) >> 6u)));\n                                add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));\n                            }\n                            else if (codepoint <= 0xFFFF)\n                            {\n                                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx\n                                add(static_cast<char_int_type>(0xE0u | (static_cast<unsigned int>(codepoint) >> 12u)));\n                                add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu)));\n                                add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));\n                            }\n                            else\n                            {\n                                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx\n                                add(static_cast<char_int_type>(0xF0u | (static_cast<unsigned int>(codepoint) >> 18u)));\n                                add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 12u) & 0x3Fu)));\n                                add(static_cast<char_int_type>(0x80u | ((static_cast<unsigned int>(codepoint) >> 6u) & 0x3Fu)));\n                                add(static_cast<char_int_type>(0x80u | (static_cast<unsigned int>(codepoint) & 0x3Fu)));\n                            }\n\n                            break;\n                        }\n\n                        // other characters after escape\n                        default:\n                            error_message = \"invalid string: forbidden character after backslash\";\n                            return token_type::parse_error;\n                    }\n\n                    break;\n                }\n\n                // invalid control characters\n                case 0x00:\n                {\n                    error_message = \"invalid string: control character U+0000 (NUL) must be escaped to \\\\u0000\";\n                    return token_type::parse_error;\n                }\n\n                case 0x01:\n                {\n                    error_message = \"invalid string: control character U+0001 (SOH) must be escaped to \\\\u0001\";\n                    return token_type::parse_error;\n                }\n\n                case 0x02:\n                {\n                    error_message = \"invalid string: control character U+0002 (STX) must be escaped to \\\\u0002\";\n                    return token_type::parse_error;\n                }\n\n                case 0x03:\n                {\n                    error_message = \"invalid string: control character U+0003 (ETX) must be escaped to \\\\u0003\";\n                    return token_type::parse_error;\n                }\n\n                case 0x04:\n                {\n                    error_message = \"invalid string: control character U+0004 (EOT) must be escaped to \\\\u0004\";\n                    return token_type::parse_error;\n                }\n\n                case 0x05:\n                {\n                    error_message = \"invalid string: control character U+0005 (ENQ) must be escaped to \\\\u0005\";\n                    return token_type::parse_error;\n                }\n\n                case 0x06:\n                {\n                    error_message = \"invalid string: control character U+0006 (ACK) must be escaped to \\\\u0006\";\n                    return token_type::parse_error;\n                }\n\n                case 0x07:\n                {\n                    error_message = \"invalid string: control character U+0007 (BEL) must be escaped to \\\\u0007\";\n                    return token_type::parse_error;\n                }\n\n                case 0x08:\n                {\n                    error_message = \"invalid string: control character U+0008 (BS) must be escaped to \\\\u0008 or \\\\b\";\n                    return token_type::parse_error;\n                }\n\n                case 0x09:\n                {\n                    error_message = \"invalid string: control character U+0009 (HT) must be escaped to \\\\u0009 or \\\\t\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0A:\n                {\n                    error_message = \"invalid string: control character U+000A (LF) must be escaped to \\\\u000A or \\\\n\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0B:\n                {\n                    error_message = \"invalid string: control character U+000B (VT) must be escaped to \\\\u000B\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0C:\n                {\n                    error_message = \"invalid string: control character U+000C (FF) must be escaped to \\\\u000C or \\\\f\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0D:\n                {\n                    error_message = \"invalid string: control character U+000D (CR) must be escaped to \\\\u000D or \\\\r\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0E:\n                {\n                    error_message = \"invalid string: control character U+000E (SO) must be escaped to \\\\u000E\";\n                    return token_type::parse_error;\n                }\n\n                case 0x0F:\n                {\n                    error_message = \"invalid string: control character U+000F (SI) must be escaped to \\\\u000F\";\n                    return token_type::parse_error;\n                }\n\n                case 0x10:\n                {\n                    error_message = \"invalid string: control character U+0010 (DLE) must be escaped to \\\\u0010\";\n                    return token_type::parse_error;\n                }\n\n                case 0x11:\n                {\n                    error_message = \"invalid string: control character U+0011 (DC1) must be escaped to \\\\u0011\";\n                    return token_type::parse_error;\n                }\n\n                case 0x12:\n                {\n                    error_message = \"invalid string: control character U+0012 (DC2) must be escaped to \\\\u0012\";\n                    return token_type::parse_error;\n                }\n\n                case 0x13:\n                {\n                    error_message = \"invalid string: control character U+0013 (DC3) must be escaped to \\\\u0013\";\n                    return token_type::parse_error;\n                }\n\n                case 0x14:\n                {\n                    error_message = \"invalid string: control character U+0014 (DC4) must be escaped to \\\\u0014\";\n                    return token_type::parse_error;\n                }\n\n                case 0x15:\n                {\n                    error_message = \"invalid string: control character U+0015 (NAK) must be escaped to \\\\u0015\";\n                    return token_type::parse_error;\n                }\n\n                case 0x16:\n                {\n                    error_message = \"invalid string: control character U+0016 (SYN) must be escaped to \\\\u0016\";\n                    return token_type::parse_error;\n                }\n\n                case 0x17:\n                {\n                    error_message = \"invalid string: control character U+0017 (ETB) must be escaped to \\\\u0017\";\n                    return token_type::parse_error;\n                }\n\n                case 0x18:\n                {\n                    error_message = \"invalid string: control character U+0018 (CAN) must be escaped to \\\\u0018\";\n                    return token_type::parse_error;\n                }\n\n                case 0x19:\n                {\n                    error_message = \"invalid string: control character U+0019 (EM) must be escaped to \\\\u0019\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1A:\n                {\n                    error_message = \"invalid string: control character U+001A (SUB) must be escaped to \\\\u001A\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1B:\n                {\n                    error_message = \"invalid string: control character U+001B (ESC) must be escaped to \\\\u001B\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1C:\n                {\n                    error_message = \"invalid string: control character U+001C (FS) must be escaped to \\\\u001C\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1D:\n                {\n                    error_message = \"invalid string: control character U+001D (GS) must be escaped to \\\\u001D\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1E:\n                {\n                    error_message = \"invalid string: control character U+001E (RS) must be escaped to \\\\u001E\";\n                    return token_type::parse_error;\n                }\n\n                case 0x1F:\n                {\n                    error_message = \"invalid string: control character U+001F (US) must be escaped to \\\\u001F\";\n                    return token_type::parse_error;\n                }\n\n                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))\n                case 0x20:\n                case 0x21:\n                case 0x23:\n                case 0x24:\n                case 0x25:\n                case 0x26:\n                case 0x27:\n                case 0x28:\n                case 0x29:\n                case 0x2A:\n                case 0x2B:\n                case 0x2C:\n                case 0x2D:\n                case 0x2E:\n                case 0x2F:\n                case 0x30:\n                case 0x31:\n                case 0x32:\n                case 0x33:\n                case 0x34:\n                case 0x35:\n                case 0x36:\n                case 0x37:\n                case 0x38:\n                case 0x39:\n                case 0x3A:\n                case 0x3B:\n                case 0x3C:\n                case 0x3D:\n                case 0x3E:\n                case 0x3F:\n                case 0x40:\n                case 0x41:\n                case 0x42:\n                case 0x43:\n                case 0x44:\n                case 0x45:\n                case 0x46:\n                case 0x47:\n                case 0x48:\n                case 0x49:\n                case 0x4A:\n                case 0x4B:\n                case 0x4C:\n                case 0x4D:\n                case 0x4E:\n                case 0x4F:\n                case 0x50:\n                case 0x51:\n                case 0x52:\n                case 0x53:\n                case 0x54:\n                case 0x55:\n                case 0x56:\n                case 0x57:\n                case 0x58:\n                case 0x59:\n                case 0x5A:\n                case 0x5B:\n                case 0x5D:\n                case 0x5E:\n                case 0x5F:\n                case 0x60:\n                case 0x61:\n                case 0x62:\n                case 0x63:\n                case 0x64:\n                case 0x65:\n                case 0x66:\n                case 0x67:\n                case 0x68:\n                case 0x69:\n                case 0x6A:\n                case 0x6B:\n                case 0x6C:\n                case 0x6D:\n                case 0x6E:\n                case 0x6F:\n                case 0x70:\n                case 0x71:\n                case 0x72:\n                case 0x73:\n                case 0x74:\n                case 0x75:\n                case 0x76:\n                case 0x77:\n                case 0x78:\n                case 0x79:\n                case 0x7A:\n                case 0x7B:\n                case 0x7C:\n                case 0x7D:\n                case 0x7E:\n                case 0x7F:\n                {\n                    add(current);\n                    break;\n                }\n\n                // U+0080..U+07FF: bytes C2..DF 80..BF\n                case 0xC2:\n                case 0xC3:\n                case 0xC4:\n                case 0xC5:\n                case 0xC6:\n                case 0xC7:\n                case 0xC8:\n                case 0xC9:\n                case 0xCA:\n                case 0xCB:\n                case 0xCC:\n                case 0xCD:\n                case 0xCE:\n                case 0xCF:\n                case 0xD0:\n                case 0xD1:\n                case 0xD2:\n                case 0xD3:\n                case 0xD4:\n                case 0xD5:\n                case 0xD6:\n                case 0xD7:\n                case 0xD8:\n                case 0xD9:\n                case 0xDA:\n                case 0xDB:\n                case 0xDC:\n                case 0xDD:\n                case 0xDE:\n                case 0xDF:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF\n                case 0xE0:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF\n                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF\n                case 0xE1:\n                case 0xE2:\n                case 0xE3:\n                case 0xE4:\n                case 0xE5:\n                case 0xE6:\n                case 0xE7:\n                case 0xE8:\n                case 0xE9:\n                case 0xEA:\n                case 0xEB:\n                case 0xEC:\n                case 0xEE:\n                case 0xEF:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+D000..U+D7FF: bytes ED 80..9F 80..BF\n                case 0xED:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF\n                case 0xF0:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF\n                case 0xF1:\n                case 0xF2:\n                case 0xF3:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF\n                case 0xF4:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))\n                    {\n                        return token_type::parse_error;\n                    }\n                    break;\n                }\n\n                // remaining bytes (80..C1 and F5..FF) are ill-formed\n                default:\n                {\n                    error_message = \"invalid string: ill-formed UTF-8 byte\";\n                    return token_type::parse_error;\n                }\n            }\n        }\n    }\n\n    /*!\n     * @brief scan a comment\n     * @return whether comment could be scanned successfully\n     */\n    bool scan_comment()\n    {\n        switch (get())\n        {\n            // single-line comments skip input until a newline or EOF is read\n            case '/':\n            {\n                while (true)\n                {\n                    switch (get())\n                    {\n                        case '\\n':\n                        case '\\r':\n                        case std::char_traits<char_type>::eof():\n                        case '\\0':\n                            return true;\n\n                        default:\n                            break;\n                    }\n                }\n            }\n\n            // multi-line comments skip input until */ is read\n            case '*':\n            {\n                while (true)\n                {\n                    switch (get())\n                    {\n                        case std::char_traits<char_type>::eof():\n                        case '\\0':\n                        {\n                            error_message = \"invalid comment; missing closing '*/'\";\n                            return false;\n                        }\n\n                        case '*':\n                        {\n                            switch (get())\n                            {\n                                case '/':\n                                    return true;\n\n                                default:\n                                {\n                                    unget();\n                                    continue;\n                                }\n                            }\n                        }\n\n                        default:\n                            continue;\n                    }\n                }\n            }\n\n            // unexpected character after reading '/'\n            default:\n            {\n                error_message = \"invalid comment; expecting '/' or '*' after '/'\";\n                return false;\n            }\n        }\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    static void strtof(float& f, const char* str, char** endptr) noexcept\n    {\n        f = std::strtof(str, endptr);\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    static void strtof(double& f, const char* str, char** endptr) noexcept\n    {\n        f = std::strtod(str, endptr);\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    static void strtof(long double& f, const char* str, char** endptr) noexcept\n    {\n        f = std::strtold(str, endptr);\n    }\n\n    /*!\n    @brief scan a number literal\n\n    This function scans a string according to Sect. 6 of RFC 8259.\n\n    The function is realized with a deterministic finite state machine derived\n    from the grammar described in RFC 8259. Starting in state \"init\", the\n    input is read and used to determined the next state. Only state \"done\"\n    accepts the number. State \"error\" is a trap state to model errors. In the\n    table below, \"anything\" means any character but the ones listed before.\n\n    state    | 0        | 1-9      | e E      | +       | -       | .        | anything\n    ---------|----------|----------|----------|---------|---------|----------|-----------\n    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]\n    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]\n    zero     | done     | done     | exponent | done    | done    | decimal1 | done\n    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done\n    decimal1 | decimal2 | decimal2 | [error]  | [error] | [error] | [error]  | [error]\n    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done\n    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]\n    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]\n    any2     | any2     | any2     | done     | done    | done    | done     | done\n\n    The state machine is realized with one label per state (prefixed with\n    \"scan_number_\") and `goto` statements between them. The state machine\n    contains cycles, but any cycle can be left when EOF is read. Therefore,\n    the function is guaranteed to terminate.\n\n    During scanning, the read bytes are stored in token_buffer. This string is\n    then converted to a signed integer, an unsigned integer, or a\n    floating-point number.\n\n    @return token_type::value_unsigned, token_type::value_integer, or\n            token_type::value_float if number could be successfully scanned,\n            token_type::parse_error otherwise\n\n    @note The scanner is independent of the current locale. Internally, the\n          locale's decimal point is used instead of `.` to work with the\n          locale-dependent converters.\n    */\n    token_type scan_number()  // lgtm [cpp/use-of-goto]\n    {\n        // reset token_buffer to store the number's bytes\n        reset();\n\n        // the type of the parsed number; initially set to unsigned; will be\n        // changed if minus sign, decimal point or exponent is read\n        token_type number_type = token_type::value_unsigned;\n\n        // state (init): we just found out we need to scan a number\n        switch (current)\n        {\n            case '-':\n            {\n                add(current);\n                goto scan_number_minus;\n            }\n\n            case '0':\n            {\n                add(current);\n                goto scan_number_zero;\n            }\n\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any1;\n            }\n\n            // all other characters are rejected outside scan_number()\n            default:            // LCOV_EXCL_LINE\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        }\n\nscan_number_minus:\n        // state: we just parsed a leading minus sign\n        number_type = token_type::value_integer;\n        switch (get())\n        {\n            case '0':\n            {\n                add(current);\n                goto scan_number_zero;\n            }\n\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any1;\n            }\n\n            default:\n            {\n                error_message = \"invalid number; expected digit after '-'\";\n                return token_type::parse_error;\n            }\n        }\n\nscan_number_zero:\n        // state: we just parse a zero (maybe with a leading minus sign)\n        switch (get())\n        {\n            case '.':\n            {\n                add(decimal_point_char);\n                goto scan_number_decimal1;\n            }\n\n            case 'e':\n            case 'E':\n            {\n                add(current);\n                goto scan_number_exponent;\n            }\n\n            default:\n                goto scan_number_done;\n        }\n\nscan_number_any1:\n        // state: we just parsed a number 0-9 (maybe with a leading minus sign)\n        switch (get())\n        {\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any1;\n            }\n\n            case '.':\n            {\n                add(decimal_point_char);\n                goto scan_number_decimal1;\n            }\n\n            case 'e':\n            case 'E':\n            {\n                add(current);\n                goto scan_number_exponent;\n            }\n\n            default:\n                goto scan_number_done;\n        }\n\nscan_number_decimal1:\n        // state: we just parsed a decimal point\n        number_type = token_type::value_float;\n        switch (get())\n        {\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_decimal2;\n            }\n\n            default:\n            {\n                error_message = \"invalid number; expected digit after '.'\";\n                return token_type::parse_error;\n            }\n        }\n\nscan_number_decimal2:\n        // we just parsed at least one number after a decimal point\n        switch (get())\n        {\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_decimal2;\n            }\n\n            case 'e':\n            case 'E':\n            {\n                add(current);\n                goto scan_number_exponent;\n            }\n\n            default:\n                goto scan_number_done;\n        }\n\nscan_number_exponent:\n        // we just parsed an exponent\n        number_type = token_type::value_float;\n        switch (get())\n        {\n            case '+':\n            case '-':\n            {\n                add(current);\n                goto scan_number_sign;\n            }\n\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any2;\n            }\n\n            default:\n            {\n                error_message =\n                    \"invalid number; expected '+', '-', or digit after exponent\";\n                return token_type::parse_error;\n            }\n        }\n\nscan_number_sign:\n        // we just parsed an exponent sign\n        switch (get())\n        {\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any2;\n            }\n\n            default:\n            {\n                error_message = \"invalid number; expected digit after exponent sign\";\n                return token_type::parse_error;\n            }\n        }\n\nscan_number_any2:\n        // we just parsed a number after the exponent or exponent sign\n        switch (get())\n        {\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            {\n                add(current);\n                goto scan_number_any2;\n            }\n\n            default:\n                goto scan_number_done;\n        }\n\nscan_number_done:\n        // unget the character after the number (we only read it to know that\n        // we are done scanning a number)\n        unget();\n\n        char* endptr = nullptr; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n        errno = 0;\n\n        // try to parse integers first and fall back to floats\n        if (number_type == token_type::value_unsigned)\n        {\n            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);\n\n            // we checked the number format before\n            JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());\n\n            if (errno == 0)\n            {\n                value_unsigned = static_cast<number_unsigned_t>(x);\n                if (value_unsigned == x)\n                {\n                    return token_type::value_unsigned;\n                }\n            }\n        }\n        else if (number_type == token_type::value_integer)\n        {\n            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);\n\n            // we checked the number format before\n            JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());\n\n            if (errno == 0)\n            {\n                value_integer = static_cast<number_integer_t>(x);\n                if (value_integer == x)\n                {\n                    return token_type::value_integer;\n                }\n            }\n        }\n\n        // this code is reached if we parse a floating-point number or if an\n        // integer conversion above failed\n        strtof(value_float, token_buffer.data(), &endptr);\n\n        // we checked the number format before\n        JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());\n\n        return token_type::value_float;\n    }\n\n    /*!\n    @param[in] literal_text  the literal text to expect\n    @param[in] length        the length of the passed literal text\n    @param[in] return_type   the token type to return on success\n    */\n    JSON_HEDLEY_NON_NULL(2)\n    token_type scan_literal(const char_type* literal_text, const std::size_t length,\n                            token_type return_type)\n    {\n        JSON_ASSERT(std::char_traits<char_type>::to_char_type(current) == literal_text[0]);\n        for (std::size_t i = 1; i < length; ++i)\n        {\n            if (JSON_HEDLEY_UNLIKELY(std::char_traits<char_type>::to_char_type(get()) != literal_text[i]))\n            {\n                error_message = \"invalid literal\";\n                return token_type::parse_error;\n            }\n        }\n        return return_type;\n    }\n\n    /////////////////////\n    // input management\n    /////////////////////\n\n    /// reset token_buffer; current character is beginning of token\n    void reset() noexcept\n    {\n        token_buffer.clear();\n        token_string.clear();\n        token_string.push_back(std::char_traits<char_type>::to_char_type(current));\n    }\n\n    /*\n    @brief get next character from the input\n\n    This function provides the interface to the used input adapter. It does\n    not throw in case the input reached EOF, but returns a\n    `std::char_traits<char>::eof()` in that case.  Stores the scanned characters\n    for use in error messages.\n\n    @return character read from the input\n    */\n    char_int_type get()\n    {\n        ++position.chars_read_total;\n        ++position.chars_read_current_line;\n\n        if (next_unget)\n        {\n            // just reset the next_unget variable and work with current\n            next_unget = false;\n        }\n        else\n        {\n            current = ia.get_character();\n        }\n\n        if (JSON_HEDLEY_LIKELY(current != std::char_traits<char_type>::eof()))\n        {\n            token_string.push_back(std::char_traits<char_type>::to_char_type(current));\n        }\n\n        if (current == '\\n')\n        {\n            ++position.lines_read;\n            position.chars_read_current_line = 0;\n        }\n\n        return current;\n    }\n\n    /*!\n    @brief unget current character (read it again on next get)\n\n    We implement unget by setting variable next_unget to true. The input is not\n    changed - we just simulate ungetting by modifying chars_read_total,\n    chars_read_current_line, and token_string. The next call to get() will\n    behave as if the unget character is read again.\n    */\n    void unget()\n    {\n        next_unget = true;\n\n        --position.chars_read_total;\n\n        // in case we \"unget\" a newline, we have to also decrement the lines_read\n        if (position.chars_read_current_line == 0)\n        {\n            if (position.lines_read > 0)\n            {\n                --position.lines_read;\n            }\n        }\n        else\n        {\n            --position.chars_read_current_line;\n        }\n\n        if (JSON_HEDLEY_LIKELY(current != std::char_traits<char_type>::eof()))\n        {\n            JSON_ASSERT(!token_string.empty());\n            token_string.pop_back();\n        }\n    }\n\n    /// add a character to token_buffer\n    void add(char_int_type c)\n    {\n        token_buffer.push_back(static_cast<typename string_t::value_type>(c));\n    }\n\n  public:\n    /////////////////////\n    // value getters\n    /////////////////////\n\n    /// return integer value\n    constexpr number_integer_t get_number_integer() const noexcept\n    {\n        return value_integer;\n    }\n\n    /// return unsigned integer value\n    constexpr number_unsigned_t get_number_unsigned() const noexcept\n    {\n        return value_unsigned;\n    }\n\n    /// return floating-point value\n    constexpr number_float_t get_number_float() const noexcept\n    {\n        return value_float;\n    }\n\n    /// return current string value (implicitly resets the token; useful only once)\n    string_t& get_string()\n    {\n        return token_buffer;\n    }\n\n    /////////////////////\n    // diagnostics\n    /////////////////////\n\n    /// return position of last read token\n    constexpr position_t get_position() const noexcept\n    {\n        return position;\n    }\n\n    /// return the last read token (for errors only).  Will never contain EOF\n    /// (an arbitrary value that is not a valid char value, often -1), because\n    /// 255 may legitimately occur.  May contain NUL, which should be escaped.\n    std::string get_token_string() const\n    {\n        // escape control characters\n        std::string result;\n        for (const auto c : token_string)\n        {\n            if (static_cast<unsigned char>(c) <= '\\x1F')\n            {\n                // escape control characters\n                std::array<char, 9> cs{{}};\n                static_cast<void>((std::snprintf)(cs.data(), cs.size(), \"<U+%.4X>\", static_cast<unsigned char>(c))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n                result += cs.data();\n            }\n            else\n            {\n                // add character as is\n                result.push_back(static_cast<std::string::value_type>(c));\n            }\n        }\n\n        return result;\n    }\n\n    /// return syntax error message\n    JSON_HEDLEY_RETURNS_NON_NULL\n    constexpr const char* get_error_message() const noexcept\n    {\n        return error_message;\n    }\n\n    /////////////////////\n    // actual scanner\n    /////////////////////\n\n    /*!\n    @brief skip the UTF-8 byte order mark\n    @return true iff there is no BOM or the correct BOM has been skipped\n    */\n    bool skip_bom()\n    {\n        if (get() == 0xEF)\n        {\n            // check if we completely parse the BOM\n            return get() == 0xBB && get() == 0xBF;\n        }\n\n        // the first character is not the beginning of the BOM; unget it to\n        // process is later\n        unget();\n        return true;\n    }\n\n    void skip_whitespace()\n    {\n        do\n        {\n            get();\n        }\n        while (current == ' ' || current == '\\t' || current == '\\n' || current == '\\r');\n    }\n\n    token_type scan()\n    {\n        // initially, skip the BOM\n        if (position.chars_read_total == 0 && !skip_bom())\n        {\n            error_message = \"invalid BOM; must be 0xEF 0xBB 0xBF if given\";\n            return token_type::parse_error;\n        }\n\n        // read next character and ignore whitespace\n        skip_whitespace();\n\n        // ignore comments\n        while (ignore_comments && current == '/')\n        {\n            if (!scan_comment())\n            {\n                return token_type::parse_error;\n            }\n\n            // skip following whitespace\n            skip_whitespace();\n        }\n\n        switch (current)\n        {\n            // structural characters\n            case '[':\n                return token_type::begin_array;\n            case ']':\n                return token_type::end_array;\n            case '{':\n                return token_type::begin_object;\n            case '}':\n                return token_type::end_object;\n            case ':':\n                return token_type::name_separator;\n            case ',':\n                return token_type::value_separator;\n\n            // literals\n            case 't':\n            {\n                std::array<char_type, 4> true_literal = {{static_cast<char_type>('t'), static_cast<char_type>('r'), static_cast<char_type>('u'), static_cast<char_type>('e')}};\n                return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true);\n            }\n            case 'f':\n            {\n                std::array<char_type, 5> false_literal = {{static_cast<char_type>('f'), static_cast<char_type>('a'), static_cast<char_type>('l'), static_cast<char_type>('s'), static_cast<char_type>('e')}};\n                return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false);\n            }\n            case 'n':\n            {\n                std::array<char_type, 4> null_literal = {{static_cast<char_type>('n'), static_cast<char_type>('u'), static_cast<char_type>('l'), static_cast<char_type>('l')}};\n                return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null);\n            }\n\n            // string\n            case '\\\"':\n                return scan_string();\n\n            // number\n            case '-':\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n                return scan_number();\n\n            // end of input (the null byte is needed when parsing from\n            // string literals)\n            case '\\0':\n            case std::char_traits<char_type>::eof():\n                return token_type::end_of_input;\n\n            // error\n            default:\n                error_message = \"invalid literal\";\n                return token_type::parse_error;\n        }\n    }\n\n  private:\n    /// input adapter\n    InputAdapterType ia;\n\n    /// whether comments should be ignored (true) or signaled as errors (false)\n    const bool ignore_comments = false;\n\n    /// the current character\n    char_int_type current = std::char_traits<char_type>::eof();\n\n    /// whether the next get() call should just return current\n    bool next_unget = false;\n\n    /// the start position of the current token\n    position_t position {};\n\n    /// raw input token string (for error messages)\n    std::vector<char_type> token_string {};\n\n    /// buffer for variable-length tokens (numbers, strings)\n    string_t token_buffer {};\n\n    /// a description of occurred lexer errors\n    const char* error_message = \"\";\n\n    // number values\n    number_integer_t value_integer = 0;\n    number_unsigned_t value_unsigned = 0;\n    number_float_t value_float = 0;\n\n    /// the decimal point\n    const char_int_type decimal_point_char = '.';\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/is_sax.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstdint> // size_t\n#include <utility> // declval\n#include <string> // string\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/meta/detected.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename T>\nusing null_function_t = decltype(std::declval<T&>().null());\n\ntemplate<typename T>\nusing boolean_function_t =\n    decltype(std::declval<T&>().boolean(std::declval<bool>()));\n\ntemplate<typename T, typename Integer>\nusing number_integer_function_t =\n    decltype(std::declval<T&>().number_integer(std::declval<Integer>()));\n\ntemplate<typename T, typename Unsigned>\nusing number_unsigned_function_t =\n    decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>()));\n\ntemplate<typename T, typename Float, typename String>\nusing number_float_function_t = decltype(std::declval<T&>().number_float(\n                                    std::declval<Float>(), std::declval<const String&>()));\n\ntemplate<typename T, typename String>\nusing string_function_t =\n    decltype(std::declval<T&>().string(std::declval<String&>()));\n\ntemplate<typename T, typename Binary>\nusing binary_function_t =\n    decltype(std::declval<T&>().binary(std::declval<Binary&>()));\n\ntemplate<typename T>\nusing start_object_function_t =\n    decltype(std::declval<T&>().start_object(std::declval<std::size_t>()));\n\ntemplate<typename T, typename String>\nusing key_function_t =\n    decltype(std::declval<T&>().key(std::declval<String&>()));\n\ntemplate<typename T>\nusing end_object_function_t = decltype(std::declval<T&>().end_object());\n\ntemplate<typename T>\nusing start_array_function_t =\n    decltype(std::declval<T&>().start_array(std::declval<std::size_t>()));\n\ntemplate<typename T>\nusing end_array_function_t = decltype(std::declval<T&>().end_array());\n\ntemplate<typename T, typename Exception>\nusing parse_error_function_t = decltype(std::declval<T&>().parse_error(\n        std::declval<std::size_t>(), std::declval<const std::string&>(),\n        std::declval<const Exception&>()));\n\ntemplate<typename SAX, typename BasicJsonType>\nstruct is_sax\n{\n  private:\n    static_assert(is_basic_json<BasicJsonType>::value,\n                  \"BasicJsonType must be of type basic_json<...>\");\n\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n    using exception_t = typename BasicJsonType::exception;\n\n  public:\n    static constexpr bool value =\n        is_detected_exact<bool, null_function_t, SAX>::value &&\n        is_detected_exact<bool, boolean_function_t, SAX>::value &&\n        is_detected_exact<bool, number_integer_function_t, SAX, number_integer_t>::value &&\n        is_detected_exact<bool, number_unsigned_function_t, SAX, number_unsigned_t>::value &&\n        is_detected_exact<bool, number_float_function_t, SAX, number_float_t, string_t>::value &&\n        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&\n        is_detected_exact<bool, binary_function_t, SAX, binary_t>::value &&\n        is_detected_exact<bool, start_object_function_t, SAX>::value &&\n        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&\n        is_detected_exact<bool, end_object_function_t, SAX>::value &&\n        is_detected_exact<bool, start_array_function_t, SAX>::value &&\n        is_detected_exact<bool, end_array_function_t, SAX>::value &&\n        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;\n};\n\ntemplate<typename SAX, typename BasicJsonType>\nstruct is_sax_static_asserts\n{\n  private:\n    static_assert(is_basic_json<BasicJsonType>::value,\n                  \"BasicJsonType must be of type basic_json<...>\");\n\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n    using exception_t = typename BasicJsonType::exception;\n\n  public:\n    static_assert(is_detected_exact<bool, null_function_t, SAX>::value,\n                  \"Missing/invalid function: bool null()\");\n    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,\n                  \"Missing/invalid function: bool boolean(bool)\");\n    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,\n                  \"Missing/invalid function: bool boolean(bool)\");\n    static_assert(\n        is_detected_exact<bool, number_integer_function_t, SAX,\n        number_integer_t>::value,\n        \"Missing/invalid function: bool number_integer(number_integer_t)\");\n    static_assert(\n        is_detected_exact<bool, number_unsigned_function_t, SAX,\n        number_unsigned_t>::value,\n        \"Missing/invalid function: bool number_unsigned(number_unsigned_t)\");\n    static_assert(is_detected_exact<bool, number_float_function_t, SAX,\n                  number_float_t, string_t>::value,\n                  \"Missing/invalid function: bool number_float(number_float_t, const string_t&)\");\n    static_assert(\n        is_detected_exact<bool, string_function_t, SAX, string_t>::value,\n        \"Missing/invalid function: bool string(string_t&)\");\n    static_assert(\n        is_detected_exact<bool, binary_function_t, SAX, binary_t>::value,\n        \"Missing/invalid function: bool binary(binary_t&)\");\n    static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value,\n                  \"Missing/invalid function: bool start_object(std::size_t)\");\n    static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value,\n                  \"Missing/invalid function: bool key(string_t&)\");\n    static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value,\n                  \"Missing/invalid function: bool end_object()\");\n    static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value,\n                  \"Missing/invalid function: bool start_array(std::size_t)\");\n    static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value,\n                  \"Missing/invalid function: bool end_array()\");\n    static_assert(\n        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value,\n        \"Missing/invalid function: bool parse_error(std::size_t, const \"\n        \"std::string&, const exception&)\");\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/// how to treat CBOR tags\nenum class cbor_tag_handler_t\n{\n    error,   ///< throw a parse_error exception in case of a tag\n    ignore,  ///< ignore tags\n    store    ///< store tags as binary type\n};\n\n/*!\n@brief determine system byte order\n\n@return true if and only if system's byte order is little endian\n\n@note from https://stackoverflow.com/a/1001328/266378\n*/\nstatic inline bool little_endianness(int num = 1) noexcept\n{\n    return *reinterpret_cast<char*>(&num) == 1;\n}\n\n\n///////////////////\n// binary reader //\n///////////////////\n\n/*!\n@brief deserialization of CBOR, MessagePack, and UBJSON values\n*/\ntemplate<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_parser<BasicJsonType>>\nclass binary_reader\n{\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n    using json_sax_t = SAX;\n    using char_type = typename InputAdapterType::char_type;\n    using char_int_type = typename std::char_traits<char_type>::int_type;\n\n  public:\n    /*!\n    @brief create a binary reader\n\n    @param[in] adapter  input adapter to read from\n    */\n    explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format)\n    {\n        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};\n    }\n\n    // make class move-only\n    binary_reader(const binary_reader&) = delete;\n    binary_reader(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    binary_reader& operator=(const binary_reader&) = delete;\n    binary_reader& operator=(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor)\n    ~binary_reader() = default;\n\n    /*!\n    @param[in] format  the binary format to parse\n    @param[in] sax_    a SAX event processor\n    @param[in] strict  whether to expect the input to be consumed completed\n    @param[in] tag_handler  how to treat CBOR tags\n\n    @return whether parsing was successful\n    */\n    JSON_HEDLEY_NON_NULL(3)\n    bool sax_parse(const input_format_t format,\n                   json_sax_t* sax_,\n                   const bool strict = true,\n                   const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)\n    {\n        sax = sax_;\n        bool result = false;\n\n        switch (format)\n        {\n            case input_format_t::bson:\n                result = parse_bson_internal();\n                break;\n\n            case input_format_t::cbor:\n                result = parse_cbor_internal(true, tag_handler);\n                break;\n\n            case input_format_t::msgpack:\n                result = parse_msgpack_internal();\n                break;\n\n            case input_format_t::ubjson:\n            case input_format_t::bjdata:\n                result = parse_ubjson_internal();\n                break;\n\n            case input_format_t::json: // LCOV_EXCL_LINE\n            default:            // LCOV_EXCL_LINE\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        }\n\n        // strict mode: next byte must be EOF\n        if (result && strict)\n        {\n            if (input_format == input_format_t::ubjson || input_format == input_format_t::bjdata)\n            {\n                get_ignore_noop();\n            }\n            else\n            {\n                get();\n            }\n\n            if (JSON_HEDLEY_UNLIKELY(current != std::char_traits<char_type>::eof()))\n            {\n                return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read,\n                                        exception_message(input_format, concat(\"expected end of input; last byte: 0x\", get_token_string()), \"value\"), nullptr));\n            }\n        }\n\n        return result;\n    }\n\n  private:\n    //////////\n    // BSON //\n    //////////\n\n    /*!\n    @brief Reads in a BSON-object and passes it to the SAX-parser.\n    @return whether a valid BSON-value was passed to the SAX parser\n    */\n    bool parse_bson_internal()\n    {\n        std::int32_t document_size{};\n        get_number<std::int32_t, true>(input_format_t::bson, document_size);\n\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))\n        {\n            return false;\n        }\n\n        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false)))\n        {\n            return false;\n        }\n\n        return sax->end_object();\n    }\n\n    /*!\n    @brief Parses a C-style string from the BSON input.\n    @param[in,out] result  A reference to the string variable where the read\n                            string is to be stored.\n    @return `true` if the \\x00-byte indicating the end of the string was\n             encountered before the EOF; false` indicates an unexpected EOF.\n    */\n    bool get_bson_cstr(string_t& result)\n    {\n        auto out = std::back_inserter(result);\n        while (true)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, \"cstring\")))\n            {\n                return false;\n            }\n            if (current == 0x00)\n            {\n                return true;\n            }\n            *out++ = static_cast<typename string_t::value_type>(current);\n        }\n    }\n\n    /*!\n    @brief Parses a zero-terminated string of length @a len from the BSON\n           input.\n    @param[in] len  The length (including the zero-byte at the end) of the\n                    string to be read.\n    @param[in,out] result  A reference to the string variable where the read\n                            string is to be stored.\n    @tparam NumberType The type of the length @a len\n    @pre len >= 1\n    @return `true` if the string was successfully parsed\n    */\n    template<typename NumberType>\n    bool get_bson_string(const NumberType len, string_t& result)\n    {\n        if (JSON_HEDLEY_UNLIKELY(len < 1))\n        {\n            auto last_token = get_token_string();\n            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                    exception_message(input_format_t::bson, concat(\"string length must be at least 1, is \", std::to_string(len)), \"string\"), nullptr));\n        }\n\n        return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) && get() != std::char_traits<char_type>::eof();\n    }\n\n    /*!\n    @brief Parses a byte array input of length @a len from the BSON input.\n    @param[in] len  The length of the byte array to be read.\n    @param[in,out] result  A reference to the binary variable where the read\n                            array is to be stored.\n    @tparam NumberType The type of the length @a len\n    @pre len >= 0\n    @return `true` if the byte array was successfully parsed\n    */\n    template<typename NumberType>\n    bool get_bson_binary(const NumberType len, binary_t& result)\n    {\n        if (JSON_HEDLEY_UNLIKELY(len < 0))\n        {\n            auto last_token = get_token_string();\n            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                    exception_message(input_format_t::bson, concat(\"byte array length cannot be negative, is \", std::to_string(len)), \"binary\"), nullptr));\n        }\n\n        // All BSON binary values have a subtype\n        std::uint8_t subtype{};\n        get_number<std::uint8_t>(input_format_t::bson, subtype);\n        result.set_subtype(subtype);\n\n        return get_binary(input_format_t::bson, len, result);\n    }\n\n    /*!\n    @brief Read a BSON document element of the given @a element_type.\n    @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html\n    @param[in] element_type_parse_position The position in the input stream,\n               where the `element_type` was read.\n    @warning Not all BSON element types are supported yet. An unsupported\n             @a element_type will give rise to a parse_error.114:\n             Unsupported BSON record type 0x...\n    @return whether a valid BSON-object/array was passed to the SAX parser\n    */\n    bool parse_bson_element_internal(const char_int_type element_type,\n                                     const std::size_t element_type_parse_position)\n    {\n        switch (element_type)\n        {\n            case 0x01: // double\n            {\n                double number{};\n                return get_number<double, true>(input_format_t::bson, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 0x02: // string\n            {\n                std::int32_t len{};\n                string_t value;\n                return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value);\n            }\n\n            case 0x03: // object\n            {\n                return parse_bson_internal();\n            }\n\n            case 0x04: // array\n            {\n                return parse_bson_array();\n            }\n\n            case 0x05: // binary\n            {\n                std::int32_t len{};\n                binary_t value;\n                return get_number<std::int32_t, true>(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value);\n            }\n\n            case 0x08: // boolean\n            {\n                return sax->boolean(get() != 0);\n            }\n\n            case 0x0A: // null\n            {\n                return sax->null();\n            }\n\n            case 0x10: // int32\n            {\n                std::int32_t value{};\n                return get_number<std::int32_t, true>(input_format_t::bson, value) && sax->number_integer(value);\n            }\n\n            case 0x12: // int64\n            {\n                std::int64_t value{};\n                return get_number<std::int64_t, true>(input_format_t::bson, value) && sax->number_integer(value);\n            }\n\n            default: // anything else not supported (yet)\n            {\n                std::array<char, 3> cr{{}};\n                static_cast<void>((std::snprintf)(cr.data(), cr.size(), \"%.2hhX\", static_cast<unsigned char>(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n                std::string cr_str{cr.data()};\n                return sax->parse_error(element_type_parse_position, cr_str,\n                                        parse_error::create(114, element_type_parse_position, concat(\"Unsupported BSON record type 0x\", cr_str), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @brief Read a BSON element list (as specified in the BSON-spec)\n\n    The same binary layout is used for objects and arrays, hence it must be\n    indicated with the argument @a is_array which one is expected\n    (true --> array, false --> object).\n\n    @param[in] is_array Determines if the element list being read is to be\n                        treated as an object (@a is_array == false), or as an\n                        array (@a is_array == true).\n    @return whether a valid BSON-object/array was passed to the SAX parser\n    */\n    bool parse_bson_element_list(const bool is_array)\n    {\n        string_t key;\n\n        while (auto element_type = get())\n        {\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, \"element list\")))\n            {\n                return false;\n            }\n\n            const std::size_t element_type_parse_position = chars_read;\n            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))\n            {\n                return false;\n            }\n\n            if (!is_array && !sax->key(key))\n            {\n                return false;\n            }\n\n            if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))\n            {\n                return false;\n            }\n\n            // get_bson_cstr only appends\n            key.clear();\n        }\n\n        return true;\n    }\n\n    /*!\n    @brief Reads an array from the BSON input and passes it to the SAX-parser.\n    @return whether a valid BSON-array was passed to the SAX parser\n    */\n    bool parse_bson_array()\n    {\n        std::int32_t document_size{};\n        get_number<std::int32_t, true>(input_format_t::bson, document_size);\n\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))\n        {\n            return false;\n        }\n\n        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true)))\n        {\n            return false;\n        }\n\n        return sax->end_array();\n    }\n\n    //////////\n    // CBOR //\n    //////////\n\n    /*!\n    @param[in] get_char  whether a new character should be retrieved from the\n                         input (true) or whether the last read character should\n                         be considered instead (false)\n    @param[in] tag_handler how CBOR tags should be treated\n\n    @return whether a valid CBOR value was passed to the SAX parser\n    */\n    bool parse_cbor_internal(const bool get_char,\n                             const cbor_tag_handler_t tag_handler)\n    {\n        switch (get_char ? get() : current)\n        {\n            // EOF\n            case std::char_traits<char_type>::eof():\n                return unexpect_eof(input_format_t::cbor, \"value\");\n\n            // Integer 0x00..0x17 (0..23)\n            case 0x00:\n            case 0x01:\n            case 0x02:\n            case 0x03:\n            case 0x04:\n            case 0x05:\n            case 0x06:\n            case 0x07:\n            case 0x08:\n            case 0x09:\n            case 0x0A:\n            case 0x0B:\n            case 0x0C:\n            case 0x0D:\n            case 0x0E:\n            case 0x0F:\n            case 0x10:\n            case 0x11:\n            case 0x12:\n            case 0x13:\n            case 0x14:\n            case 0x15:\n            case 0x16:\n            case 0x17:\n                return sax->number_unsigned(static_cast<number_unsigned_t>(current));\n\n            case 0x18: // Unsigned integer (one-byte uint8_t follows)\n            {\n                std::uint8_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);\n            }\n\n            case 0x19: // Unsigned integer (two-byte uint16_t follows)\n            {\n                std::uint16_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);\n            }\n\n            case 0x1A: // Unsigned integer (four-byte uint32_t follows)\n            {\n                std::uint32_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);\n            }\n\n            case 0x1B: // Unsigned integer (eight-byte uint64_t follows)\n            {\n                std::uint64_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_unsigned(number);\n            }\n\n            // Negative integer -1-0x00..-1-0x17 (-1..-24)\n            case 0x20:\n            case 0x21:\n            case 0x22:\n            case 0x23:\n            case 0x24:\n            case 0x25:\n            case 0x26:\n            case 0x27:\n            case 0x28:\n            case 0x29:\n            case 0x2A:\n            case 0x2B:\n            case 0x2C:\n            case 0x2D:\n            case 0x2E:\n            case 0x2F:\n            case 0x30:\n            case 0x31:\n            case 0x32:\n            case 0x33:\n            case 0x34:\n            case 0x35:\n            case 0x36:\n            case 0x37:\n                return sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - current));\n\n            case 0x38: // Negative integer (one-byte uint8_t follows)\n            {\n                std::uint8_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);\n            }\n\n            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)\n            {\n                std::uint16_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);\n            }\n\n            case 0x3A: // Negative integer -1-n (four-byte uint32_t follows)\n            {\n                std::uint32_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1) - number);\n            }\n\n            case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows)\n            {\n                std::uint64_t number{};\n                return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast<number_integer_t>(-1)\n                        - static_cast<number_integer_t>(number));\n            }\n\n            // Binary data (0x00..0x17 bytes follow)\n            case 0x40:\n            case 0x41:\n            case 0x42:\n            case 0x43:\n            case 0x44:\n            case 0x45:\n            case 0x46:\n            case 0x47:\n            case 0x48:\n            case 0x49:\n            case 0x4A:\n            case 0x4B:\n            case 0x4C:\n            case 0x4D:\n            case 0x4E:\n            case 0x4F:\n            case 0x50:\n            case 0x51:\n            case 0x52:\n            case 0x53:\n            case 0x54:\n            case 0x55:\n            case 0x56:\n            case 0x57:\n            case 0x58: // Binary data (one-byte uint8_t for n follows)\n            case 0x59: // Binary data (two-byte uint16_t for n follow)\n            case 0x5A: // Binary data (four-byte uint32_t for n follow)\n            case 0x5B: // Binary data (eight-byte uint64_t for n follow)\n            case 0x5F: // Binary data (indefinite length)\n            {\n                binary_t b;\n                return get_cbor_binary(b) && sax->binary(b);\n            }\n\n            // UTF-8 string (0x00..0x17 bytes follow)\n            case 0x60:\n            case 0x61:\n            case 0x62:\n            case 0x63:\n            case 0x64:\n            case 0x65:\n            case 0x66:\n            case 0x67:\n            case 0x68:\n            case 0x69:\n            case 0x6A:\n            case 0x6B:\n            case 0x6C:\n            case 0x6D:\n            case 0x6E:\n            case 0x6F:\n            case 0x70:\n            case 0x71:\n            case 0x72:\n            case 0x73:\n            case 0x74:\n            case 0x75:\n            case 0x76:\n            case 0x77:\n            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)\n            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)\n            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)\n            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)\n            case 0x7F: // UTF-8 string (indefinite length)\n            {\n                string_t s;\n                return get_cbor_string(s) && sax->string(s);\n            }\n\n            // array (0x00..0x17 data items follow)\n            case 0x80:\n            case 0x81:\n            case 0x82:\n            case 0x83:\n            case 0x84:\n            case 0x85:\n            case 0x86:\n            case 0x87:\n            case 0x88:\n            case 0x89:\n            case 0x8A:\n            case 0x8B:\n            case 0x8C:\n            case 0x8D:\n            case 0x8E:\n            case 0x8F:\n            case 0x90:\n            case 0x91:\n            case 0x92:\n            case 0x93:\n            case 0x94:\n            case 0x95:\n            case 0x96:\n            case 0x97:\n                return get_cbor_array(\n                           conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);\n\n            case 0x98: // array (one-byte uint8_t for n follows)\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0x99: // array (two-byte uint16_t for n follow)\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0x9A: // array (four-byte uint32_t for n follow)\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0x9B: // array (eight-byte uint64_t for n follow)\n            {\n                std::uint64_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0x9F: // array (indefinite length)\n                return get_cbor_array(static_cast<std::size_t>(-1), tag_handler);\n\n            // map (0x00..0x17 pairs of data items follow)\n            case 0xA0:\n            case 0xA1:\n            case 0xA2:\n            case 0xA3:\n            case 0xA4:\n            case 0xA5:\n            case 0xA6:\n            case 0xA7:\n            case 0xA8:\n            case 0xA9:\n            case 0xAA:\n            case 0xAB:\n            case 0xAC:\n            case 0xAD:\n            case 0xAE:\n            case 0xAF:\n            case 0xB0:\n            case 0xB1:\n            case 0xB2:\n            case 0xB3:\n            case 0xB4:\n            case 0xB5:\n            case 0xB6:\n            case 0xB7:\n                return get_cbor_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu), tag_handler);\n\n            case 0xB8: // map (one-byte uint8_t for n follows)\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0xB9: // map (two-byte uint16_t for n follow)\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0xBA: // map (four-byte uint32_t for n follow)\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0xBB: // map (eight-byte uint64_t for n follow)\n            {\n                std::uint64_t len{};\n                return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast<std::size_t>(len), tag_handler);\n            }\n\n            case 0xBF: // map (indefinite length)\n                return get_cbor_object(static_cast<std::size_t>(-1), tag_handler);\n\n            case 0xC6: // tagged item\n            case 0xC7:\n            case 0xC8:\n            case 0xC9:\n            case 0xCA:\n            case 0xCB:\n            case 0xCC:\n            case 0xCD:\n            case 0xCE:\n            case 0xCF:\n            case 0xD0:\n            case 0xD1:\n            case 0xD2:\n            case 0xD3:\n            case 0xD4:\n            case 0xD8: // tagged item (1 bytes follow)\n            case 0xD9: // tagged item (2 bytes follow)\n            case 0xDA: // tagged item (4 bytes follow)\n            case 0xDB: // tagged item (8 bytes follow)\n            {\n                switch (tag_handler)\n                {\n                    case cbor_tag_handler_t::error:\n                    {\n                        auto last_token = get_token_string();\n                        return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                                exception_message(input_format_t::cbor, concat(\"invalid byte: 0x\", last_token), \"value\"), nullptr));\n                    }\n\n                    case cbor_tag_handler_t::ignore:\n                    {\n                        // ignore binary subtype\n                        switch (current)\n                        {\n                            case 0xD8:\n                            {\n                                std::uint8_t subtype_to_ignore{};\n                                get_number(input_format_t::cbor, subtype_to_ignore);\n                                break;\n                            }\n                            case 0xD9:\n                            {\n                                std::uint16_t subtype_to_ignore{};\n                                get_number(input_format_t::cbor, subtype_to_ignore);\n                                break;\n                            }\n                            case 0xDA:\n                            {\n                                std::uint32_t subtype_to_ignore{};\n                                get_number(input_format_t::cbor, subtype_to_ignore);\n                                break;\n                            }\n                            case 0xDB:\n                            {\n                                std::uint64_t subtype_to_ignore{};\n                                get_number(input_format_t::cbor, subtype_to_ignore);\n                                break;\n                            }\n                            default:\n                                break;\n                        }\n                        return parse_cbor_internal(true, tag_handler);\n                    }\n\n                    case cbor_tag_handler_t::store:\n                    {\n                        binary_t b;\n                        // use binary subtype and store in binary container\n                        switch (current)\n                        {\n                            case 0xD8:\n                            {\n                                std::uint8_t subtype{};\n                                get_number(input_format_t::cbor, subtype);\n                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));\n                                break;\n                            }\n                            case 0xD9:\n                            {\n                                std::uint16_t subtype{};\n                                get_number(input_format_t::cbor, subtype);\n                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));\n                                break;\n                            }\n                            case 0xDA:\n                            {\n                                std::uint32_t subtype{};\n                                get_number(input_format_t::cbor, subtype);\n                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));\n                                break;\n                            }\n                            case 0xDB:\n                            {\n                                std::uint64_t subtype{};\n                                get_number(input_format_t::cbor, subtype);\n                                b.set_subtype(detail::conditional_static_cast<typename binary_t::subtype_type>(subtype));\n                                break;\n                            }\n                            default:\n                                return parse_cbor_internal(true, tag_handler);\n                        }\n                        get();\n                        return get_cbor_binary(b) && sax->binary(b);\n                    }\n\n                    default:                 // LCOV_EXCL_LINE\n                        JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n                        return false;        // LCOV_EXCL_LINE\n                }\n            }\n\n            case 0xF4: // false\n                return sax->boolean(false);\n\n            case 0xF5: // true\n                return sax->boolean(true);\n\n            case 0xF6: // null\n                return sax->null();\n\n            case 0xF9: // Half-Precision Float (two-byte IEEE 754)\n            {\n                const auto byte1_raw = get();\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, \"number\")))\n                {\n                    return false;\n                }\n                const auto byte2_raw = get();\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, \"number\")))\n                {\n                    return false;\n                }\n\n                const auto byte1 = static_cast<unsigned char>(byte1_raw);\n                const auto byte2 = static_cast<unsigned char>(byte2_raw);\n\n                // code from RFC 7049, Appendix D, Figure 3:\n                // As half-precision floating-point numbers were only added\n                // to IEEE 754 in 2008, today's programming platforms often\n                // still only have limited support for them. It is very\n                // easy to include at least decoding support for them even\n                // without such support. An example of a small decoder for\n                // half-precision floating-point numbers in the C language\n                // is shown in Fig. 3.\n                const auto half = static_cast<unsigned int>((byte1 << 8u) + byte2);\n                const double val = [&half]\n                {\n                    const int exp = (half >> 10u) & 0x1Fu;\n                    const unsigned int mant = half & 0x3FFu;\n                    JSON_ASSERT(0 <= exp&& exp <= 32);\n                    JSON_ASSERT(mant <= 1024);\n                    switch (exp)\n                    {\n                        case 0:\n                            return std::ldexp(mant, -24);\n                        case 31:\n                            return (mant == 0)\n                            ? std::numeric_limits<double>::infinity()\n                            : std::numeric_limits<double>::quiet_NaN();\n                        default:\n                            return std::ldexp(mant + 1024, exp - 25);\n                    }\n                }();\n                return sax->number_float((half & 0x8000u) != 0\n                                         ? static_cast<number_float_t>(-val)\n                                         : static_cast<number_float_t>(val), \"\");\n            }\n\n            case 0xFA: // Single-Precision Float (four-byte IEEE 754)\n            {\n                float number{};\n                return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 0xFB: // Double-Precision Float (eight-byte IEEE 754)\n            {\n                double number{};\n                return get_number(input_format_t::cbor, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            default: // anything else (0xFF is handled inside the other types)\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                        exception_message(input_format_t::cbor, concat(\"invalid byte: 0x\", last_token), \"value\"), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @brief reads a CBOR string\n\n    This function first reads starting bytes to determine the expected\n    string length and then copies this number of bytes into a string.\n    Additionally, CBOR's strings with indefinite lengths are supported.\n\n    @param[out] result  created string\n\n    @return whether string creation completed\n    */\n    bool get_cbor_string(string_t& result)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, \"string\")))\n        {\n            return false;\n        }\n\n        switch (current)\n        {\n            // UTF-8 string (0x00..0x17 bytes follow)\n            case 0x60:\n            case 0x61:\n            case 0x62:\n            case 0x63:\n            case 0x64:\n            case 0x65:\n            case 0x66:\n            case 0x67:\n            case 0x68:\n            case 0x69:\n            case 0x6A:\n            case 0x6B:\n            case 0x6C:\n            case 0x6D:\n            case 0x6E:\n            case 0x6F:\n            case 0x70:\n            case 0x71:\n            case 0x72:\n            case 0x73:\n            case 0x74:\n            case 0x75:\n            case 0x76:\n            case 0x77:\n            {\n                return get_string(input_format_t::cbor, static_cast<unsigned int>(current) & 0x1Fu, result);\n            }\n\n            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);\n            }\n\n            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);\n            }\n\n            case 0x7A: // UTF-8 string (four-byte uint32_t for n follow)\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);\n            }\n\n            case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow)\n            {\n                std::uint64_t len{};\n                return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result);\n            }\n\n            case 0x7F: // UTF-8 string (indefinite length)\n            {\n                while (get() != 0xFF)\n                {\n                    string_t chunk;\n                    if (!get_cbor_string(chunk))\n                    {\n                        return false;\n                    }\n                    result.append(chunk);\n                }\n                return true;\n            }\n\n            default:\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,\n                                        exception_message(input_format_t::cbor, concat(\"expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x\", last_token), \"string\"), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @brief reads a CBOR byte array\n\n    This function first reads starting bytes to determine the expected\n    byte array length and then copies this number of bytes into the byte array.\n    Additionally, CBOR's byte arrays with indefinite lengths are supported.\n\n    @param[out] result  created byte array\n\n    @return whether byte array creation completed\n    */\n    bool get_cbor_binary(binary_t& result)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, \"binary\")))\n        {\n            return false;\n        }\n\n        switch (current)\n        {\n            // Binary data (0x00..0x17 bytes follow)\n            case 0x40:\n            case 0x41:\n            case 0x42:\n            case 0x43:\n            case 0x44:\n            case 0x45:\n            case 0x46:\n            case 0x47:\n            case 0x48:\n            case 0x49:\n            case 0x4A:\n            case 0x4B:\n            case 0x4C:\n            case 0x4D:\n            case 0x4E:\n            case 0x4F:\n            case 0x50:\n            case 0x51:\n            case 0x52:\n            case 0x53:\n            case 0x54:\n            case 0x55:\n            case 0x56:\n            case 0x57:\n            {\n                return get_binary(input_format_t::cbor, static_cast<unsigned int>(current) & 0x1Fu, result);\n            }\n\n            case 0x58: // Binary data (one-byte uint8_t for n follows)\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::cbor, len) &&\n                       get_binary(input_format_t::cbor, len, result);\n            }\n\n            case 0x59: // Binary data (two-byte uint16_t for n follow)\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::cbor, len) &&\n                       get_binary(input_format_t::cbor, len, result);\n            }\n\n            case 0x5A: // Binary data (four-byte uint32_t for n follow)\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::cbor, len) &&\n                       get_binary(input_format_t::cbor, len, result);\n            }\n\n            case 0x5B: // Binary data (eight-byte uint64_t for n follow)\n            {\n                std::uint64_t len{};\n                return get_number(input_format_t::cbor, len) &&\n                       get_binary(input_format_t::cbor, len, result);\n            }\n\n            case 0x5F: // Binary data (indefinite length)\n            {\n                while (get() != 0xFF)\n                {\n                    binary_t chunk;\n                    if (!get_cbor_binary(chunk))\n                    {\n                        return false;\n                    }\n                    result.insert(result.end(), chunk.begin(), chunk.end());\n                }\n                return true;\n            }\n\n            default:\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,\n                                        exception_message(input_format_t::cbor, concat(\"expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x\", last_token), \"binary\"), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @param[in] len  the length of the array or static_cast<std::size_t>(-1) for an\n                    array of indefinite size\n    @param[in] tag_handler how CBOR tags should be treated\n    @return whether array creation completed\n    */\n    bool get_cbor_array(const std::size_t len,\n                        const cbor_tag_handler_t tag_handler)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len)))\n        {\n            return false;\n        }\n\n        if (len != static_cast<std::size_t>(-1))\n        {\n            for (std::size_t i = 0; i < len; ++i)\n            {\n                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))\n                {\n                    return false;\n                }\n            }\n        }\n        else\n        {\n            while (get() != 0xFF)\n            {\n                if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler)))\n                {\n                    return false;\n                }\n            }\n        }\n\n        return sax->end_array();\n    }\n\n    /*!\n    @param[in] len  the length of the object or static_cast<std::size_t>(-1) for an\n                    object of indefinite size\n    @param[in] tag_handler how CBOR tags should be treated\n    @return whether object creation completed\n    */\n    bool get_cbor_object(const std::size_t len,\n                         const cbor_tag_handler_t tag_handler)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len)))\n        {\n            return false;\n        }\n\n        if (len != 0)\n        {\n            string_t key;\n            if (len != static_cast<std::size_t>(-1))\n            {\n                for (std::size_t i = 0; i < len; ++i)\n                {\n                    get();\n                    if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))\n                    {\n                        return false;\n                    }\n\n                    if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))\n                    {\n                        return false;\n                    }\n                    key.clear();\n                }\n            }\n            else\n            {\n                while (get() != 0xFF)\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key)))\n                    {\n                        return false;\n                    }\n\n                    if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler)))\n                    {\n                        return false;\n                    }\n                    key.clear();\n                }\n            }\n        }\n\n        return sax->end_object();\n    }\n\n    /////////////\n    // MsgPack //\n    /////////////\n\n    /*!\n    @return whether a valid MessagePack value was passed to the SAX parser\n    */\n    bool parse_msgpack_internal()\n    {\n        switch (get())\n        {\n            // EOF\n            case std::char_traits<char_type>::eof():\n                return unexpect_eof(input_format_t::msgpack, \"value\");\n\n            // positive fixint\n            case 0x00:\n            case 0x01:\n            case 0x02:\n            case 0x03:\n            case 0x04:\n            case 0x05:\n            case 0x06:\n            case 0x07:\n            case 0x08:\n            case 0x09:\n            case 0x0A:\n            case 0x0B:\n            case 0x0C:\n            case 0x0D:\n            case 0x0E:\n            case 0x0F:\n            case 0x10:\n            case 0x11:\n            case 0x12:\n            case 0x13:\n            case 0x14:\n            case 0x15:\n            case 0x16:\n            case 0x17:\n            case 0x18:\n            case 0x19:\n            case 0x1A:\n            case 0x1B:\n            case 0x1C:\n            case 0x1D:\n            case 0x1E:\n            case 0x1F:\n            case 0x20:\n            case 0x21:\n            case 0x22:\n            case 0x23:\n            case 0x24:\n            case 0x25:\n            case 0x26:\n            case 0x27:\n            case 0x28:\n            case 0x29:\n            case 0x2A:\n            case 0x2B:\n            case 0x2C:\n            case 0x2D:\n            case 0x2E:\n            case 0x2F:\n            case 0x30:\n            case 0x31:\n            case 0x32:\n            case 0x33:\n            case 0x34:\n            case 0x35:\n            case 0x36:\n            case 0x37:\n            case 0x38:\n            case 0x39:\n            case 0x3A:\n            case 0x3B:\n            case 0x3C:\n            case 0x3D:\n            case 0x3E:\n            case 0x3F:\n            case 0x40:\n            case 0x41:\n            case 0x42:\n            case 0x43:\n            case 0x44:\n            case 0x45:\n            case 0x46:\n            case 0x47:\n            case 0x48:\n            case 0x49:\n            case 0x4A:\n            case 0x4B:\n            case 0x4C:\n            case 0x4D:\n            case 0x4E:\n            case 0x4F:\n            case 0x50:\n            case 0x51:\n            case 0x52:\n            case 0x53:\n            case 0x54:\n            case 0x55:\n            case 0x56:\n            case 0x57:\n            case 0x58:\n            case 0x59:\n            case 0x5A:\n            case 0x5B:\n            case 0x5C:\n            case 0x5D:\n            case 0x5E:\n            case 0x5F:\n            case 0x60:\n            case 0x61:\n            case 0x62:\n            case 0x63:\n            case 0x64:\n            case 0x65:\n            case 0x66:\n            case 0x67:\n            case 0x68:\n            case 0x69:\n            case 0x6A:\n            case 0x6B:\n            case 0x6C:\n            case 0x6D:\n            case 0x6E:\n            case 0x6F:\n            case 0x70:\n            case 0x71:\n            case 0x72:\n            case 0x73:\n            case 0x74:\n            case 0x75:\n            case 0x76:\n            case 0x77:\n            case 0x78:\n            case 0x79:\n            case 0x7A:\n            case 0x7B:\n            case 0x7C:\n            case 0x7D:\n            case 0x7E:\n            case 0x7F:\n                return sax->number_unsigned(static_cast<number_unsigned_t>(current));\n\n            // fixmap\n            case 0x80:\n            case 0x81:\n            case 0x82:\n            case 0x83:\n            case 0x84:\n            case 0x85:\n            case 0x86:\n            case 0x87:\n            case 0x88:\n            case 0x89:\n            case 0x8A:\n            case 0x8B:\n            case 0x8C:\n            case 0x8D:\n            case 0x8E:\n            case 0x8F:\n                return get_msgpack_object(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));\n\n            // fixarray\n            case 0x90:\n            case 0x91:\n            case 0x92:\n            case 0x93:\n            case 0x94:\n            case 0x95:\n            case 0x96:\n            case 0x97:\n            case 0x98:\n            case 0x99:\n            case 0x9A:\n            case 0x9B:\n            case 0x9C:\n            case 0x9D:\n            case 0x9E:\n            case 0x9F:\n                return get_msgpack_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));\n\n            // fixstr\n            case 0xA0:\n            case 0xA1:\n            case 0xA2:\n            case 0xA3:\n            case 0xA4:\n            case 0xA5:\n            case 0xA6:\n            case 0xA7:\n            case 0xA8:\n            case 0xA9:\n            case 0xAA:\n            case 0xAB:\n            case 0xAC:\n            case 0xAD:\n            case 0xAE:\n            case 0xAF:\n            case 0xB0:\n            case 0xB1:\n            case 0xB2:\n            case 0xB3:\n            case 0xB4:\n            case 0xB5:\n            case 0xB6:\n            case 0xB7:\n            case 0xB8:\n            case 0xB9:\n            case 0xBA:\n            case 0xBB:\n            case 0xBC:\n            case 0xBD:\n            case 0xBE:\n            case 0xBF:\n            case 0xD9: // str 8\n            case 0xDA: // str 16\n            case 0xDB: // str 32\n            {\n                string_t s;\n                return get_msgpack_string(s) && sax->string(s);\n            }\n\n            case 0xC0: // nil\n                return sax->null();\n\n            case 0xC2: // false\n                return sax->boolean(false);\n\n            case 0xC3: // true\n                return sax->boolean(true);\n\n            case 0xC4: // bin 8\n            case 0xC5: // bin 16\n            case 0xC6: // bin 32\n            case 0xC7: // ext 8\n            case 0xC8: // ext 16\n            case 0xC9: // ext 32\n            case 0xD4: // fixext 1\n            case 0xD5: // fixext 2\n            case 0xD6: // fixext 4\n            case 0xD7: // fixext 8\n            case 0xD8: // fixext 16\n            {\n                binary_t b;\n                return get_msgpack_binary(b) && sax->binary(b);\n            }\n\n            case 0xCA: // float 32\n            {\n                float number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 0xCB: // float 64\n            {\n                double number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 0xCC: // uint 8\n            {\n                std::uint8_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);\n            }\n\n            case 0xCD: // uint 16\n            {\n                std::uint16_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);\n            }\n\n            case 0xCE: // uint 32\n            {\n                std::uint32_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);\n            }\n\n            case 0xCF: // uint 64\n            {\n                std::uint64_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number);\n            }\n\n            case 0xD0: // int 8\n            {\n                std::int8_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_integer(number);\n            }\n\n            case 0xD1: // int 16\n            {\n                std::int16_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_integer(number);\n            }\n\n            case 0xD2: // int 32\n            {\n                std::int32_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_integer(number);\n            }\n\n            case 0xD3: // int 64\n            {\n                std::int64_t number{};\n                return get_number(input_format_t::msgpack, number) && sax->number_integer(number);\n            }\n\n            case 0xDC: // array 16\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast<std::size_t>(len));\n            }\n\n            case 0xDD: // array 32\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast<std::size_t>(len));\n            }\n\n            case 0xDE: // map 16\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast<std::size_t>(len));\n            }\n\n            case 0xDF: // map 32\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast<std::size_t>(len));\n            }\n\n            // negative fixint\n            case 0xE0:\n            case 0xE1:\n            case 0xE2:\n            case 0xE3:\n            case 0xE4:\n            case 0xE5:\n            case 0xE6:\n            case 0xE7:\n            case 0xE8:\n            case 0xE9:\n            case 0xEA:\n            case 0xEB:\n            case 0xEC:\n            case 0xED:\n            case 0xEE:\n            case 0xEF:\n            case 0xF0:\n            case 0xF1:\n            case 0xF2:\n            case 0xF3:\n            case 0xF4:\n            case 0xF5:\n            case 0xF6:\n            case 0xF7:\n            case 0xF8:\n            case 0xF9:\n            case 0xFA:\n            case 0xFB:\n            case 0xFC:\n            case 0xFD:\n            case 0xFE:\n            case 0xFF:\n                return sax->number_integer(static_cast<std::int8_t>(current));\n\n            default: // anything else\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                        exception_message(input_format_t::msgpack, concat(\"invalid byte: 0x\", last_token), \"value\"), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @brief reads a MessagePack string\n\n    This function first reads starting bytes to determine the expected\n    string length and then copies this number of bytes into a string.\n\n    @param[out] result  created string\n\n    @return whether string creation completed\n    */\n    bool get_msgpack_string(string_t& result)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, \"string\")))\n        {\n            return false;\n        }\n\n        switch (current)\n        {\n            // fixstr\n            case 0xA0:\n            case 0xA1:\n            case 0xA2:\n            case 0xA3:\n            case 0xA4:\n            case 0xA5:\n            case 0xA6:\n            case 0xA7:\n            case 0xA8:\n            case 0xA9:\n            case 0xAA:\n            case 0xAB:\n            case 0xAC:\n            case 0xAD:\n            case 0xAE:\n            case 0xAF:\n            case 0xB0:\n            case 0xB1:\n            case 0xB2:\n            case 0xB3:\n            case 0xB4:\n            case 0xB5:\n            case 0xB6:\n            case 0xB7:\n            case 0xB8:\n            case 0xB9:\n            case 0xBA:\n            case 0xBB:\n            case 0xBC:\n            case 0xBD:\n            case 0xBE:\n            case 0xBF:\n            {\n                return get_string(input_format_t::msgpack, static_cast<unsigned int>(current) & 0x1Fu, result);\n            }\n\n            case 0xD9: // str 8\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);\n            }\n\n            case 0xDA: // str 16\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);\n            }\n\n            case 0xDB: // str 32\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result);\n            }\n\n            default:\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,\n                                        exception_message(input_format_t::msgpack, concat(\"expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x\", last_token), \"string\"), nullptr));\n            }\n        }\n    }\n\n    /*!\n    @brief reads a MessagePack byte array\n\n    This function first reads starting bytes to determine the expected\n    byte array length and then copies this number of bytes into a byte array.\n\n    @param[out] result  created byte array\n\n    @return whether byte array creation completed\n    */\n    bool get_msgpack_binary(binary_t& result)\n    {\n        // helper function to set the subtype\n        auto assign_and_return_true = [&result](std::int8_t subtype)\n        {\n            result.set_subtype(static_cast<std::uint8_t>(subtype));\n            return true;\n        };\n\n        switch (current)\n        {\n            case 0xC4: // bin 8\n            {\n                std::uint8_t len{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_binary(input_format_t::msgpack, len, result);\n            }\n\n            case 0xC5: // bin 16\n            {\n                std::uint16_t len{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_binary(input_format_t::msgpack, len, result);\n            }\n\n            case 0xC6: // bin 32\n            {\n                std::uint32_t len{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_binary(input_format_t::msgpack, len, result);\n            }\n\n            case 0xC7: // ext 8\n            {\n                std::uint8_t len{};\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, len, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xC8: // ext 16\n            {\n                std::uint16_t len{};\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, len, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xC9: // ext 32\n            {\n                std::uint32_t len{};\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, len) &&\n                       get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, len, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xD4: // fixext 1\n            {\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, 1, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xD5: // fixext 2\n            {\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, 2, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xD6: // fixext 4\n            {\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, 4, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xD7: // fixext 8\n            {\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, 8, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            case 0xD8: // fixext 16\n            {\n                std::int8_t subtype{};\n                return get_number(input_format_t::msgpack, subtype) &&\n                       get_binary(input_format_t::msgpack, 16, result) &&\n                       assign_and_return_true(subtype);\n            }\n\n            default:           // LCOV_EXCL_LINE\n                return false;  // LCOV_EXCL_LINE\n        }\n    }\n\n    /*!\n    @param[in] len  the length of the array\n    @return whether array creation completed\n    */\n    bool get_msgpack_array(const std::size_t len)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len)))\n        {\n            return false;\n        }\n\n        for (std::size_t i = 0; i < len; ++i)\n        {\n            if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal()))\n            {\n                return false;\n            }\n        }\n\n        return sax->end_array();\n    }\n\n    /*!\n    @param[in] len  the length of the object\n    @return whether object creation completed\n    */\n    bool get_msgpack_object(const std::size_t len)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len)))\n        {\n            return false;\n        }\n\n        string_t key;\n        for (std::size_t i = 0; i < len; ++i)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key)))\n            {\n                return false;\n            }\n\n            if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal()))\n            {\n                return false;\n            }\n            key.clear();\n        }\n\n        return sax->end_object();\n    }\n\n    ////////////\n    // UBJSON //\n    ////////////\n\n    /*!\n    @param[in] get_char  whether a new character should be retrieved from the\n                         input (true, default) or whether the last read\n                         character should be considered instead\n\n    @return whether a valid UBJSON value was passed to the SAX parser\n    */\n    bool parse_ubjson_internal(const bool get_char = true)\n    {\n        return get_ubjson_value(get_char ? get_ignore_noop() : current);\n    }\n\n    /*!\n    @brief reads a UBJSON string\n\n    This function is either called after reading the 'S' byte explicitly\n    indicating a string, or in case of an object key where the 'S' byte can be\n    left out.\n\n    @param[out] result   created string\n    @param[in] get_char  whether a new character should be retrieved from the\n                         input (true, default) or whether the last read\n                         character should be considered instead\n\n    @return whether string creation completed\n    */\n    bool get_ubjson_string(string_t& result, const bool get_char = true)\n    {\n        if (get_char)\n        {\n            get();  // TODO(niels): may we ignore N here?\n        }\n\n        if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"value\")))\n        {\n            return false;\n        }\n\n        switch (current)\n        {\n            case 'U':\n            {\n                std::uint8_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'i':\n            {\n                std::int8_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'I':\n            {\n                std::int16_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'l':\n            {\n                std::int32_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'L':\n            {\n                std::int64_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'u':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint16_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'm':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint32_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            case 'M':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint64_t len{};\n                return get_number(input_format, len) && get_string(input_format, len, result);\n            }\n\n            default:\n                break;\n        }\n        auto last_token = get_token_string();\n        std::string message;\n\n        if (input_format != input_format_t::bjdata)\n        {\n            message = \"expected length type specification (U, i, I, l, L); last byte: 0x\" + last_token;\n        }\n        else\n        {\n            message = \"expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x\" + last_token;\n        }\n        return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, \"string\"), nullptr));\n    }\n\n    /*!\n    @param[out] dim  an integer vector storing the ND array dimensions\n    @return whether reading ND array size vector is successful\n    */\n    bool get_ubjson_ndarray_size(std::vector<size_t>& dim)\n    {\n        std::pair<std::size_t, char_int_type> size_and_type;\n        size_t dimlen = 0;\n        bool no_ndarray = true;\n\n        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type, no_ndarray)))\n        {\n            return false;\n        }\n\n        if (size_and_type.first != npos)\n        {\n            if (size_and_type.second != 0)\n            {\n                if (size_and_type.second != 'N')\n                {\n                    for (std::size_t i = 0; i < size_and_type.first; ++i)\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))\n                        {\n                            return false;\n                        }\n                        dim.push_back(dimlen);\n                    }\n                }\n            }\n            else\n            {\n                for (std::size_t i = 0; i < size_and_type.first; ++i)\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray)))\n                    {\n                        return false;\n                    }\n                    dim.push_back(dimlen);\n                }\n            }\n        }\n        else\n        {\n            while (current != ']')\n            {\n                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, current)))\n                {\n                    return false;\n                }\n                dim.push_back(dimlen);\n                get_ignore_noop();\n            }\n        }\n        return true;\n    }\n\n    /*!\n    @param[out] result  determined size\n    @param[in,out] is_ndarray  for input, `true` means already inside an ndarray vector\n                               or ndarray dimension is not allowed; `false` means ndarray\n                               is allowed; for output, `true` means an ndarray is found;\n                               is_ndarray can only return `true` when its initial value\n                               is `false`\n    @param[in] prefix  type marker if already read, otherwise set to 0\n\n    @return whether size determination completed\n    */\n    bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0)\n    {\n        if (prefix == 0)\n        {\n            prefix = get_ignore_noop();\n        }\n\n        switch (prefix)\n        {\n            case 'U':\n            {\n                std::uint8_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                result = static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'i':\n            {\n                std::int8_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                if (number < 0)\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,\n                                            exception_message(input_format, \"count in an optimized container must be positive\", \"size\"), nullptr));\n                }\n                result = static_cast<std::size_t>(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char\n                return true;\n            }\n\n            case 'I':\n            {\n                std::int16_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                if (number < 0)\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,\n                                            exception_message(input_format, \"count in an optimized container must be positive\", \"size\"), nullptr));\n                }\n                result = static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'l':\n            {\n                std::int32_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                if (number < 0)\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,\n                                            exception_message(input_format, \"count in an optimized container must be positive\", \"size\"), nullptr));\n                }\n                result = static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'L':\n            {\n                std::int64_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                if (number < 0)\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read,\n                                            exception_message(input_format, \"count in an optimized container must be positive\", \"size\"), nullptr));\n                }\n                if (!value_in_range_of<std::size_t>(number))\n                {\n                    return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408,\n                                            exception_message(input_format, \"integer value overflow\", \"size\"), nullptr));\n                }\n                result = static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'u':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint16_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                result = static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'm':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint32_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                result = conditional_static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case 'M':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint64_t number{};\n                if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number)))\n                {\n                    return false;\n                }\n                if (!value_in_range_of<std::size_t>(number))\n                {\n                    return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408,\n                                            exception_message(input_format, \"integer value overflow\", \"size\"), nullptr));\n                }\n                result = detail::conditional_static_cast<std::size_t>(number);\n                return true;\n            }\n\n            case '[':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                if (is_ndarray) // ndarray dimensional vector can only contain integers, and can not embed another array\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, \"ndarray dimentional vector is not allowed\", \"size\"), nullptr));\n                }\n                std::vector<size_t> dim;\n                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim)))\n                {\n                    return false;\n                }\n                if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector\n                {\n                    result = dim.at(dim.size() - 1);\n                    return true;\n                }\n                if (!dim.empty())  // if ndarray, convert to an object in JData annotated array format\n                {\n                    for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container\n                    {\n                        if ( i == 0 )\n                        {\n                            result = 0;\n                            return true;\n                        }\n                    }\n\n                    string_t key = \"_ArraySize_\";\n                    if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size())))\n                    {\n                        return false;\n                    }\n                    result = 1;\n                    for (auto i : dim)\n                    {\n                        result *= i;\n                        if (result == 0 || result == npos) // because dim elements shall not have zeros, result = 0 means overflow happened; it also can't be npos as it is used to initialize size in get_ubjson_size_type()\n                        {\n                            return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, \"excessive ndarray size caused overflow\", \"size\"), nullptr));\n                        }\n                        if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast<number_unsigned_t>(i))))\n                        {\n                            return false;\n                        }\n                    }\n                    is_ndarray = true;\n                    return sax->end_array();\n                }\n                result = 0;\n                return true;\n            }\n\n            default:\n                break;\n        }\n        auto last_token = get_token_string();\n        std::string message;\n\n        if (input_format != input_format_t::bjdata)\n        {\n            message = \"expected length type specification (U, i, I, l, L) after '#'; last byte: 0x\" + last_token;\n        }\n        else\n        {\n            message = \"expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x\" + last_token;\n        }\n        return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, \"size\"), nullptr));\n    }\n\n    /*!\n    @brief determine the type and size for a container\n\n    In the optimized UBJSON format, a type and a size can be provided to allow\n    for a more compact representation.\n\n    @param[out] result  pair of the size and the type\n    @param[in] inside_ndarray  whether the parser is parsing an ND array dimensional vector\n\n    @return whether pair creation completed\n    */\n    bool get_ubjson_size_type(std::pair<std::size_t, char_int_type>& result, bool inside_ndarray = false)\n    {\n        result.first = npos; // size\n        result.second = 0; // type\n        bool is_ndarray = false;\n\n        get_ignore_noop();\n\n        if (current == '$')\n        {\n            result.second = get();  // must not ignore 'N', because 'N' maybe the type\n            if (input_format == input_format_t::bjdata\n                    && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second)))\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                        exception_message(input_format, concat(\"marker 0x\", last_token, \" is not a permitted optimized array type\"), \"type\"), nullptr));\n            }\n\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"type\")))\n            {\n                return false;\n            }\n\n            get_ignore_noop();\n            if (JSON_HEDLEY_UNLIKELY(current != '#'))\n            {\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"value\")))\n                {\n                    return false;\n                }\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                        exception_message(input_format, concat(\"expected '#' after type information; last byte: 0x\", last_token), \"size\"), nullptr));\n            }\n\n            bool is_error = get_ubjson_size_value(result.first, is_ndarray);\n            if (input_format == input_format_t::bjdata && is_ndarray)\n            {\n                if (inside_ndarray)\n                {\n                    return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,\n                                            exception_message(input_format, \"ndarray can not be recursive\", \"size\"), nullptr));\n                }\n                result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters\n            }\n            return is_error;\n        }\n\n        if (current == '#')\n        {\n            bool is_error = get_ubjson_size_value(result.first, is_ndarray);\n            if (input_format == input_format_t::bjdata && is_ndarray)\n            {\n                return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read,\n                                        exception_message(input_format, \"ndarray requires both type and size\", \"size\"), nullptr));\n            }\n            return is_error;\n        }\n\n        return true;\n    }\n\n    /*!\n    @param prefix  the previously read or set type prefix\n    @return whether value creation completed\n    */\n    bool get_ubjson_value(const char_int_type prefix)\n    {\n        switch (prefix)\n        {\n            case std::char_traits<char_type>::eof():  // EOF\n                return unexpect_eof(input_format, \"value\");\n\n            case 'T':  // true\n                return sax->boolean(true);\n            case 'F':  // false\n                return sax->boolean(false);\n\n            case 'Z':  // null\n                return sax->null();\n\n            case 'U':\n            {\n                std::uint8_t number{};\n                return get_number(input_format, number) && sax->number_unsigned(number);\n            }\n\n            case 'i':\n            {\n                std::int8_t number{};\n                return get_number(input_format, number) && sax->number_integer(number);\n            }\n\n            case 'I':\n            {\n                std::int16_t number{};\n                return get_number(input_format, number) && sax->number_integer(number);\n            }\n\n            case 'l':\n            {\n                std::int32_t number{};\n                return get_number(input_format, number) && sax->number_integer(number);\n            }\n\n            case 'L':\n            {\n                std::int64_t number{};\n                return get_number(input_format, number) && sax->number_integer(number);\n            }\n\n            case 'u':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint16_t number{};\n                return get_number(input_format, number) && sax->number_unsigned(number);\n            }\n\n            case 'm':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint32_t number{};\n                return get_number(input_format, number) && sax->number_unsigned(number);\n            }\n\n            case 'M':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                std::uint64_t number{};\n                return get_number(input_format, number) && sax->number_unsigned(number);\n            }\n\n            case 'h':\n            {\n                if (input_format != input_format_t::bjdata)\n                {\n                    break;\n                }\n                const auto byte1_raw = get();\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"number\")))\n                {\n                    return false;\n                }\n                const auto byte2_raw = get();\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"number\")))\n                {\n                    return false;\n                }\n\n                const auto byte1 = static_cast<unsigned char>(byte1_raw);\n                const auto byte2 = static_cast<unsigned char>(byte2_raw);\n\n                // code from RFC 7049, Appendix D, Figure 3:\n                // As half-precision floating-point numbers were only added\n                // to IEEE 754 in 2008, today's programming platforms often\n                // still only have limited support for them. It is very\n                // easy to include at least decoding support for them even\n                // without such support. An example of a small decoder for\n                // half-precision floating-point numbers in the C language\n                // is shown in Fig. 3.\n                const auto half = static_cast<unsigned int>((byte2 << 8u) + byte1);\n                const double val = [&half]\n                {\n                    const int exp = (half >> 10u) & 0x1Fu;\n                    const unsigned int mant = half & 0x3FFu;\n                    JSON_ASSERT(0 <= exp&& exp <= 32);\n                    JSON_ASSERT(mant <= 1024);\n                    switch (exp)\n                    {\n                        case 0:\n                            return std::ldexp(mant, -24);\n                        case 31:\n                            return (mant == 0)\n                            ? std::numeric_limits<double>::infinity()\n                            : std::numeric_limits<double>::quiet_NaN();\n                        default:\n                            return std::ldexp(mant + 1024, exp - 25);\n                    }\n                }();\n                return sax->number_float((half & 0x8000u) != 0\n                                         ? static_cast<number_float_t>(-val)\n                                         : static_cast<number_float_t>(val), \"\");\n            }\n\n            case 'd':\n            {\n                float number{};\n                return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 'D':\n            {\n                double number{};\n                return get_number(input_format, number) && sax->number_float(static_cast<number_float_t>(number), \"\");\n            }\n\n            case 'H':\n            {\n                return get_ubjson_high_precision_number();\n            }\n\n            case 'C':  // char\n            {\n                get();\n                if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"char\")))\n                {\n                    return false;\n                }\n                if (JSON_HEDLEY_UNLIKELY(current > 127))\n                {\n                    auto last_token = get_token_string();\n                    return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read,\n                                            exception_message(input_format, concat(\"byte after 'C' must be in range 0x00..0x7F; last byte: 0x\", last_token), \"char\"), nullptr));\n                }\n                string_t s(1, static_cast<typename string_t::value_type>(current));\n                return sax->string(s);\n            }\n\n            case 'S':  // string\n            {\n                string_t s;\n                return get_ubjson_string(s) && sax->string(s);\n            }\n\n            case '[':  // array\n                return get_ubjson_array();\n\n            case '{':  // object\n                return get_ubjson_object();\n\n            default: // anything else\n                break;\n        }\n        auto last_token = get_token_string();\n        return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, \"invalid byte: 0x\" + last_token, \"value\"), nullptr));\n    }\n\n    /*!\n    @return whether array creation completed\n    */\n    bool get_ubjson_array()\n    {\n        std::pair<std::size_t, char_int_type> size_and_type;\n        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))\n        {\n            return false;\n        }\n\n        // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):\n        // {\"_ArrayType_\" : \"typeid\", \"_ArraySize_\" : [n1, n2, ...], \"_ArrayData_\" : [v1, v2, ...]}\n\n        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)\n        {\n            size_and_type.second &= ~(static_cast<char_int_type>(1) << 8);  // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker\n            auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t)\n            {\n                return p.first < t;\n            });\n            string_t key = \"_ArrayType_\";\n            if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second))\n            {\n                auto last_token = get_token_string();\n                return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                        exception_message(input_format, \"invalid byte: 0x\" + last_token, \"type\"), nullptr));\n            }\n\n            string_t type = it->second; // sax->string() takes a reference\n            if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))\n            {\n                return false;\n            }\n\n            if (size_and_type.second == 'C')\n            {\n                size_and_type.second = 'U';\n            }\n\n            key = \"_ArrayData_\";\n            if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) ))\n            {\n                return false;\n            }\n\n            for (std::size_t i = 0; i < size_and_type.first; ++i)\n            {\n                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))\n                {\n                    return false;\n                }\n            }\n\n            return (sax->end_array() && sax->end_object());\n        }\n\n        if (size_and_type.first != npos)\n        {\n            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first)))\n            {\n                return false;\n            }\n\n            if (size_and_type.second != 0)\n            {\n                if (size_and_type.second != 'N')\n                {\n                    for (std::size_t i = 0; i < size_and_type.first; ++i)\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))\n                        {\n                            return false;\n                        }\n                    }\n                }\n            }\n            else\n            {\n                for (std::size_t i = 0; i < size_and_type.first; ++i)\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))\n                    {\n                        return false;\n                    }\n                }\n            }\n        }\n        else\n        {\n            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))\n            {\n                return false;\n            }\n\n            while (current != ']')\n            {\n                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false)))\n                {\n                    return false;\n                }\n                get_ignore_noop();\n            }\n        }\n\n        return sax->end_array();\n    }\n\n    /*!\n    @return whether object creation completed\n    */\n    bool get_ubjson_object()\n    {\n        std::pair<std::size_t, char_int_type> size_and_type;\n        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type)))\n        {\n            return false;\n        }\n\n        // do not accept ND-array size in objects in BJData\n        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)\n        {\n            auto last_token = get_token_string();\n            return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,\n                                    exception_message(input_format, \"BJData object does not support ND-array size in optimized format\", \"object\"), nullptr));\n        }\n\n        string_t key;\n        if (size_and_type.first != npos)\n        {\n            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first)))\n            {\n                return false;\n            }\n\n            if (size_and_type.second != 0)\n            {\n                for (std::size_t i = 0; i < size_and_type.first; ++i)\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))\n                    {\n                        return false;\n                    }\n                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))\n                    {\n                        return false;\n                    }\n                    key.clear();\n                }\n            }\n            else\n            {\n                for (std::size_t i = 0; i < size_and_type.first; ++i)\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))\n                    {\n                        return false;\n                    }\n                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))\n                    {\n                        return false;\n                    }\n                    key.clear();\n                }\n            }\n        }\n        else\n        {\n            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))\n            {\n                return false;\n            }\n\n            while (current != '}')\n            {\n                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))\n                {\n                    return false;\n                }\n                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))\n                {\n                    return false;\n                }\n                get_ignore_noop();\n                key.clear();\n            }\n        }\n\n        return sax->end_object();\n    }\n\n    // Note, no reader for UBJSON binary types is implemented because they do\n    // not exist\n\n    bool get_ubjson_high_precision_number()\n    {\n        // get size of following number string\n        std::size_t size{};\n        bool no_ndarray = true;\n        auto res = get_ubjson_size_value(size, no_ndarray);\n        if (JSON_HEDLEY_UNLIKELY(!res))\n        {\n            return res;\n        }\n\n        // get number string\n        std::vector<char> number_vector;\n        for (std::size_t i = 0; i < size; ++i)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, \"number\")))\n            {\n                return false;\n            }\n            number_vector.push_back(static_cast<char>(current));\n        }\n\n        // parse number string\n        using ia_type = decltype(detail::input_adapter(number_vector));\n        auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);\n        const auto result_number = number_lexer.scan();\n        const auto number_string = number_lexer.get_token_string();\n        const auto result_remainder = number_lexer.scan();\n\n        using token_type = typename detail::lexer_base<BasicJsonType>::token_type;\n\n        if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input))\n        {\n            return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read,\n                                    exception_message(input_format, concat(\"invalid number text: \", number_lexer.get_token_string()), \"high-precision number\"), nullptr));\n        }\n\n        switch (result_number)\n        {\n            case token_type::value_integer:\n                return sax->number_integer(number_lexer.get_number_integer());\n            case token_type::value_unsigned:\n                return sax->number_unsigned(number_lexer.get_number_unsigned());\n            case token_type::value_float:\n                return sax->number_float(number_lexer.get_number_float(), std::move(number_string));\n            case token_type::uninitialized:\n            case token_type::literal_true:\n            case token_type::literal_false:\n            case token_type::literal_null:\n            case token_type::value_string:\n            case token_type::begin_array:\n            case token_type::begin_object:\n            case token_type::end_array:\n            case token_type::end_object:\n            case token_type::name_separator:\n            case token_type::value_separator:\n            case token_type::parse_error:\n            case token_type::end_of_input:\n            case token_type::literal_or_value:\n            default:\n                return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read,\n                                        exception_message(input_format, concat(\"invalid number text: \", number_lexer.get_token_string()), \"high-precision number\"), nullptr));\n        }\n    }\n\n    ///////////////////////\n    // Utility functions //\n    ///////////////////////\n\n    /*!\n    @brief get next character from the input\n\n    This function provides the interface to the used input adapter. It does\n    not throw in case the input reached EOF, but returns a -'ve valued\n    `std::char_traits<char_type>::eof()` in that case.\n\n    @return character read from the input\n    */\n    char_int_type get()\n    {\n        ++chars_read;\n        return current = ia.get_character();\n    }\n\n    /*!\n    @return character read from the input after ignoring all 'N' entries\n    */\n    char_int_type get_ignore_noop()\n    {\n        do\n        {\n            get();\n        }\n        while (current == 'N');\n\n        return current;\n    }\n\n    /*\n    @brief read a number from the input\n\n    @tparam NumberType the type of the number\n    @param[in] format   the current format (for diagnostics)\n    @param[out] result  number of type @a NumberType\n\n    @return whether conversion completed\n\n    @note This function needs to respect the system's endianness, because\n          bytes in CBOR, MessagePack, and UBJSON are stored in network order\n          (big endian) and therefore need reordering on little endian systems.\n          On the other hand, BSON and BJData use little endian and should reorder\n          on big endian systems.\n    */\n    template<typename NumberType, bool InputIsLittleEndian = false>\n    bool get_number(const input_format_t format, NumberType& result)\n    {\n        // step 1: read input into array with system's byte order\n        std::array<std::uint8_t, sizeof(NumberType)> vec{};\n        for (std::size_t i = 0; i < sizeof(NumberType); ++i)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, \"number\")))\n            {\n                return false;\n            }\n\n            // reverse byte order prior to conversion if necessary\n            if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata))\n            {\n                vec[sizeof(NumberType) - i - 1] = static_cast<std::uint8_t>(current);\n            }\n            else\n            {\n                vec[i] = static_cast<std::uint8_t>(current); // LCOV_EXCL_LINE\n            }\n        }\n\n        // step 2: convert array into number of type T and return\n        std::memcpy(&result, vec.data(), sizeof(NumberType));\n        return true;\n    }\n\n    /*!\n    @brief create a string by reading characters from the input\n\n    @tparam NumberType the type of the number\n    @param[in] format the current format (for diagnostics)\n    @param[in] len number of characters to read\n    @param[out] result string created by reading @a len bytes\n\n    @return whether string creation completed\n\n    @note We can not reserve @a len bytes for the result, because @a len\n          may be too large. Usually, @ref unexpect_eof() detects the end of\n          the input before we run out of string memory.\n    */\n    template<typename NumberType>\n    bool get_string(const input_format_t format,\n                    const NumberType len,\n                    string_t& result)\n    {\n        bool success = true;\n        for (NumberType i = 0; i < len; i++)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, \"string\")))\n            {\n                success = false;\n                break;\n            }\n            result.push_back(static_cast<typename string_t::value_type>(current));\n        }\n        return success;\n    }\n\n    /*!\n    @brief create a byte array by reading bytes from the input\n\n    @tparam NumberType the type of the number\n    @param[in] format the current format (for diagnostics)\n    @param[in] len number of bytes to read\n    @param[out] result byte array created by reading @a len bytes\n\n    @return whether byte array creation completed\n\n    @note We can not reserve @a len bytes for the result, because @a len\n          may be too large. Usually, @ref unexpect_eof() detects the end of\n          the input before we run out of memory.\n    */\n    template<typename NumberType>\n    bool get_binary(const input_format_t format,\n                    const NumberType len,\n                    binary_t& result)\n    {\n        bool success = true;\n        for (NumberType i = 0; i < len; i++)\n        {\n            get();\n            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, \"binary\")))\n            {\n                success = false;\n                break;\n            }\n            result.push_back(static_cast<std::uint8_t>(current));\n        }\n        return success;\n    }\n\n    /*!\n    @param[in] format   the current format (for diagnostics)\n    @param[in] context  further context information (for diagnostics)\n    @return whether the last read character is not EOF\n    */\n    JSON_HEDLEY_NON_NULL(3)\n    bool unexpect_eof(const input_format_t format, const char* context) const\n    {\n        if (JSON_HEDLEY_UNLIKELY(current == std::char_traits<char_type>::eof()))\n        {\n            return sax->parse_error(chars_read, \"<end of file>\",\n                                    parse_error::create(110, chars_read, exception_message(format, \"unexpected end of input\", context), nullptr));\n        }\n        return true;\n    }\n\n    /*!\n    @return a string representation of the last read byte\n    */\n    std::string get_token_string() const\n    {\n        std::array<char, 3> cr{{}};\n        static_cast<void>((std::snprintf)(cr.data(), cr.size(), \"%.2hhX\", static_cast<unsigned char>(current))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n        return std::string{cr.data()};\n    }\n\n    /*!\n    @param[in] format   the current format\n    @param[in] detail   a detailed error message\n    @param[in] context  further context information\n    @return a message string to use in the parse_error exceptions\n    */\n    std::string exception_message(const input_format_t format,\n                                  const std::string& detail,\n                                  const std::string& context) const\n    {\n        std::string error_msg = \"syntax error while parsing \";\n\n        switch (format)\n        {\n            case input_format_t::cbor:\n                error_msg += \"CBOR\";\n                break;\n\n            case input_format_t::msgpack:\n                error_msg += \"MessagePack\";\n                break;\n\n            case input_format_t::ubjson:\n                error_msg += \"UBJSON\";\n                break;\n\n            case input_format_t::bson:\n                error_msg += \"BSON\";\n                break;\n\n            case input_format_t::bjdata:\n                error_msg += \"BJData\";\n                break;\n\n            case input_format_t::json: // LCOV_EXCL_LINE\n            default:            // LCOV_EXCL_LINE\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        }\n\n        return concat(error_msg, ' ', context, \": \", detail);\n    }\n\n  private:\n    static JSON_INLINE_VARIABLE constexpr std::size_t npos = static_cast<std::size_t>(-1);\n\n    /// input adapter\n    InputAdapterType ia;\n\n    /// the current character\n    char_int_type current = std::char_traits<char_type>::eof();\n\n    /// the number of characters read\n    std::size_t chars_read = 0;\n\n    /// whether we can assume little endianness\n    const bool is_little_endian = little_endianness();\n\n    /// input format\n    const input_format_t input_format = input_format_t::json;\n\n    /// the SAX parser\n    json_sax_t* sax = nullptr;\n\n    // excluded markers in bjdata optimized type\n#define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \\\n    make_array<char_int_type>('F', 'H', 'N', 'S', 'T', 'Z', '[', '{')\n\n#define JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_ \\\n    make_array<bjd_type>(                      \\\n    bjd_type{'C', \"char\"},                     \\\n    bjd_type{'D', \"double\"},                   \\\n    bjd_type{'I', \"int16\"},                    \\\n    bjd_type{'L', \"int64\"},                    \\\n    bjd_type{'M', \"uint64\"},                   \\\n    bjd_type{'U', \"uint8\"},                    \\\n    bjd_type{'d', \"single\"},                   \\\n    bjd_type{'i', \"int8\"},                     \\\n    bjd_type{'l', \"int32\"},                    \\\n    bjd_type{'m', \"uint32\"},                   \\\n    bjd_type{'u', \"uint16\"})\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    // lookup tables\n    // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)\n    const decltype(JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_) bjd_optimized_type_markers =\n        JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_;\n\n    using bjd_type = std::pair<char_int_type, string_t>;\n    // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)\n    const decltype(JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_) bjd_types_map =\n        JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_;\n\n#undef JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_\n#undef JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_\n};\n\n#ifndef JSON_HAS_CPP_17\n    template<typename BasicJsonType, typename InputAdapterType, typename SAX>\n    constexpr std::size_t binary_reader<BasicJsonType, InputAdapterType, SAX>::npos;\n#endif\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/input/input_adapters.hpp>\n\n// #include <nlohmann/detail/input/lexer.hpp>\n\n// #include <nlohmann/detail/input/parser.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cmath> // isfinite\n#include <cstdint> // uint8_t\n#include <functional> // function\n#include <string> // string\n#include <utility> // move\n#include <vector> // vector\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/input/input_adapters.hpp>\n\n// #include <nlohmann/detail/input/json_sax.hpp>\n\n// #include <nlohmann/detail/input/lexer.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/is_sax.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n////////////\n// parser //\n////////////\n\nenum class parse_event_t : std::uint8_t\n{\n    /// the parser read `{` and started to process a JSON object\n    object_start,\n    /// the parser read `}` and finished processing a JSON object\n    object_end,\n    /// the parser read `[` and started to process a JSON array\n    array_start,\n    /// the parser read `]` and finished processing a JSON array\n    array_end,\n    /// the parser read a key of a value in an object\n    key,\n    /// the parser finished reading a JSON value\n    value\n};\n\ntemplate<typename BasicJsonType>\nusing parser_callback_t =\n    std::function<bool(int /*depth*/, parse_event_t /*event*/, BasicJsonType& /*parsed*/)>;\n\n/*!\n@brief syntax analysis\n\nThis class implements a recursive descent parser.\n*/\ntemplate<typename BasicJsonType, typename InputAdapterType>\nclass parser\n{\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using string_t = typename BasicJsonType::string_t;\n    using lexer_t = lexer<BasicJsonType, InputAdapterType>;\n    using token_type = typename lexer_t::token_type;\n\n  public:\n    /// a parser reading from an input adapter\n    explicit parser(InputAdapterType&& adapter,\n                    const parser_callback_t<BasicJsonType> cb = nullptr,\n                    const bool allow_exceptions_ = true,\n                    const bool skip_comments = false)\n        : callback(cb)\n        , m_lexer(std::move(adapter), skip_comments)\n        , allow_exceptions(allow_exceptions_)\n    {\n        // read first token\n        get_token();\n    }\n\n    /*!\n    @brief public parser interface\n\n    @param[in] strict      whether to expect the last token to be EOF\n    @param[in,out] result  parsed JSON value\n\n    @throw parse_error.101 in case of an unexpected token\n    @throw parse_error.102 if to_unicode fails or surrogate error\n    @throw parse_error.103 if to_unicode fails\n    */\n    void parse(const bool strict, BasicJsonType& result)\n    {\n        if (callback)\n        {\n            json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions);\n            sax_parse_internal(&sdp);\n\n            // in strict mode, input must be completely read\n            if (strict && (get_token() != token_type::end_of_input))\n            {\n                sdp.parse_error(m_lexer.get_position(),\n                                m_lexer.get_token_string(),\n                                parse_error::create(101, m_lexer.get_position(),\n                                                    exception_message(token_type::end_of_input, \"value\"), nullptr));\n            }\n\n            // in case of an error, return discarded value\n            if (sdp.is_errored())\n            {\n                result = value_t::discarded;\n                return;\n            }\n\n            // set top-level value to null if it was discarded by the callback\n            // function\n            if (result.is_discarded())\n            {\n                result = nullptr;\n            }\n        }\n        else\n        {\n            json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions);\n            sax_parse_internal(&sdp);\n\n            // in strict mode, input must be completely read\n            if (strict && (get_token() != token_type::end_of_input))\n            {\n                sdp.parse_error(m_lexer.get_position(),\n                                m_lexer.get_token_string(),\n                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, \"value\"), nullptr));\n            }\n\n            // in case of an error, return discarded value\n            if (sdp.is_errored())\n            {\n                result = value_t::discarded;\n                return;\n            }\n        }\n\n        result.assert_invariant();\n    }\n\n    /*!\n    @brief public accept interface\n\n    @param[in] strict  whether to expect the last token to be EOF\n    @return whether the input is a proper JSON text\n    */\n    bool accept(const bool strict = true)\n    {\n        json_sax_acceptor<BasicJsonType> sax_acceptor;\n        return sax_parse(&sax_acceptor, strict);\n    }\n\n    template<typename SAX>\n    JSON_HEDLEY_NON_NULL(2)\n    bool sax_parse(SAX* sax, const bool strict = true)\n    {\n        (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};\n        const bool result = sax_parse_internal(sax);\n\n        // strict mode: next byte must be EOF\n        if (result && strict && (get_token() != token_type::end_of_input))\n        {\n            return sax->parse_error(m_lexer.get_position(),\n                                    m_lexer.get_token_string(),\n                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, \"value\"), nullptr));\n        }\n\n        return result;\n    }\n\n  private:\n    template<typename SAX>\n    JSON_HEDLEY_NON_NULL(2)\n    bool sax_parse_internal(SAX* sax)\n    {\n        // stack to remember the hierarchy of structured values we are parsing\n        // true = array; false = object\n        std::vector<bool> states;\n        // value to avoid a goto (see comment where set to true)\n        bool skip_to_state_evaluation = false;\n\n        while (true)\n        {\n            if (!skip_to_state_evaluation)\n            {\n                // invariant: get_token() was called before each iteration\n                switch (last_token)\n                {\n                    case token_type::begin_object:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast<std::size_t>(-1))))\n                        {\n                            return false;\n                        }\n\n                        // closing } -> we are done\n                        if (get_token() == token_type::end_object)\n                        {\n                            if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))\n                            {\n                                return false;\n                            }\n                            break;\n                        }\n\n                        // parse key\n                        if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))\n                        {\n                            return sax->parse_error(m_lexer.get_position(),\n                                                    m_lexer.get_token_string(),\n                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, \"object key\"), nullptr));\n                        }\n                        if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))\n                        {\n                            return false;\n                        }\n\n                        // parse separator (:)\n                        if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))\n                        {\n                            return sax->parse_error(m_lexer.get_position(),\n                                                    m_lexer.get_token_string(),\n                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, \"object separator\"), nullptr));\n                        }\n\n                        // remember we are now inside an object\n                        states.push_back(false);\n\n                        // parse values\n                        get_token();\n                        continue;\n                    }\n\n                    case token_type::begin_array:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast<std::size_t>(-1))))\n                        {\n                            return false;\n                        }\n\n                        // closing ] -> we are done\n                        if (get_token() == token_type::end_array)\n                        {\n                            if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))\n                            {\n                                return false;\n                            }\n                            break;\n                        }\n\n                        // remember we are now inside an array\n                        states.push_back(true);\n\n                        // parse values (no need to call get_token)\n                        continue;\n                    }\n\n                    case token_type::value_float:\n                    {\n                        const auto res = m_lexer.get_number_float();\n\n                        if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))\n                        {\n                            return sax->parse_error(m_lexer.get_position(),\n                                                    m_lexer.get_token_string(),\n                                                    out_of_range::create(406, concat(\"number overflow parsing '\", m_lexer.get_token_string(), '\\''), nullptr));\n                        }\n\n                        if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))\n                        {\n                            return false;\n                        }\n\n                        break;\n                    }\n\n                    case token_type::literal_false:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->boolean(false)))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::literal_null:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->null()))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::literal_true:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->boolean(true)))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::value_integer:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(m_lexer.get_number_integer())))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::value_string:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string())))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::value_unsigned:\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(m_lexer.get_number_unsigned())))\n                        {\n                            return false;\n                        }\n                        break;\n                    }\n\n                    case token_type::parse_error:\n                    {\n                        // using \"uninitialized\" to avoid \"expected\" message\n                        return sax->parse_error(m_lexer.get_position(),\n                                                m_lexer.get_token_string(),\n                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, \"value\"), nullptr));\n                    }\n\n                    case token_type::uninitialized:\n                    case token_type::end_array:\n                    case token_type::end_object:\n                    case token_type::name_separator:\n                    case token_type::value_separator:\n                    case token_type::end_of_input:\n                    case token_type::literal_or_value:\n                    default: // the last token was unexpected\n                    {\n                        return sax->parse_error(m_lexer.get_position(),\n                                                m_lexer.get_token_string(),\n                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, \"value\"), nullptr));\n                    }\n                }\n            }\n            else\n            {\n                skip_to_state_evaluation = false;\n            }\n\n            // we reached this line after we successfully parsed a value\n            if (states.empty())\n            {\n                // empty stack: we reached the end of the hierarchy: done\n                return true;\n            }\n\n            if (states.back())  // array\n            {\n                // comma -> next value\n                if (get_token() == token_type::value_separator)\n                {\n                    // parse a new value\n                    get_token();\n                    continue;\n                }\n\n                // closing ]\n                if (JSON_HEDLEY_LIKELY(last_token == token_type::end_array))\n                {\n                    if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))\n                    {\n                        return false;\n                    }\n\n                    // We are done with this array. Before we can parse a\n                    // new value, we need to evaluate the new state first.\n                    // By setting skip_to_state_evaluation to false, we\n                    // are effectively jumping to the beginning of this if.\n                    JSON_ASSERT(!states.empty());\n                    states.pop_back();\n                    skip_to_state_evaluation = true;\n                    continue;\n                }\n\n                return sax->parse_error(m_lexer.get_position(),\n                                        m_lexer.get_token_string(),\n                                        parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, \"array\"), nullptr));\n            }\n\n            // states.back() is false -> object\n\n            // comma -> next value\n            if (get_token() == token_type::value_separator)\n            {\n                // parse key\n                if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string))\n                {\n                    return sax->parse_error(m_lexer.get_position(),\n                                            m_lexer.get_token_string(),\n                                            parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, \"object key\"), nullptr));\n                }\n\n                if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))\n                {\n                    return false;\n                }\n\n                // parse separator (:)\n                if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))\n                {\n                    return sax->parse_error(m_lexer.get_position(),\n                                            m_lexer.get_token_string(),\n                                            parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, \"object separator\"), nullptr));\n                }\n\n                // parse values\n                get_token();\n                continue;\n            }\n\n            // closing }\n            if (JSON_HEDLEY_LIKELY(last_token == token_type::end_object))\n            {\n                if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))\n                {\n                    return false;\n                }\n\n                // We are done with this object. Before we can parse a\n                // new value, we need to evaluate the new state first.\n                // By setting skip_to_state_evaluation to false, we\n                // are effectively jumping to the beginning of this if.\n                JSON_ASSERT(!states.empty());\n                states.pop_back();\n                skip_to_state_evaluation = true;\n                continue;\n            }\n\n            return sax->parse_error(m_lexer.get_position(),\n                                    m_lexer.get_token_string(),\n                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, \"object\"), nullptr));\n        }\n    }\n\n    /// get next token from lexer\n    token_type get_token()\n    {\n        return last_token = m_lexer.scan();\n    }\n\n    std::string exception_message(const token_type expected, const std::string& context)\n    {\n        std::string error_msg = \"syntax error \";\n\n        if (!context.empty())\n        {\n            error_msg += concat(\"while parsing \", context, ' ');\n        }\n\n        error_msg += \"- \";\n\n        if (last_token == token_type::parse_error)\n        {\n            error_msg += concat(m_lexer.get_error_message(), \"; last read: '\",\n                                m_lexer.get_token_string(), '\\'');\n        }\n        else\n        {\n            error_msg += concat(\"unexpected \", lexer_t::token_type_name(last_token));\n        }\n\n        if (expected != token_type::uninitialized)\n        {\n            error_msg += concat(\"; expected \", lexer_t::token_type_name(expected));\n        }\n\n        return error_msg;\n    }\n\n  private:\n    /// callback function\n    const parser_callback_t<BasicJsonType> callback = nullptr;\n    /// the type of the last read token\n    token_type last_token = token_type::uninitialized;\n    /// the lexer\n    lexer_t m_lexer;\n    /// whether to throw exceptions in case of errors\n    const bool allow_exceptions = true;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/iterators/internal_iterator.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/iterators/primitive_iterator.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef> // ptrdiff_t\n#include <limits>  // numeric_limits\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/*\n@brief an iterator for primitive JSON types\n\nThis class models an iterator for primitive JSON types (boolean, number,\nstring). It's only purpose is to allow the iterator/const_iterator classes\nto \"iterate\" over primitive values. Internally, the iterator is modeled by\na `difference_type` variable. Value begin_value (`0`) models the begin,\nend_value (`1`) models past the end.\n*/\nclass primitive_iterator_t\n{\n  private:\n    using difference_type = std::ptrdiff_t;\n    static constexpr difference_type begin_value = 0;\n    static constexpr difference_type end_value = begin_value + 1;\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    /// iterator as signed integer type\n    difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)();\n\n  public:\n    constexpr difference_type get_value() const noexcept\n    {\n        return m_it;\n    }\n\n    /// set iterator to a defined beginning\n    void set_begin() noexcept\n    {\n        m_it = begin_value;\n    }\n\n    /// set iterator to a defined past the end\n    void set_end() noexcept\n    {\n        m_it = end_value;\n    }\n\n    /// return whether the iterator can be dereferenced\n    constexpr bool is_begin() const noexcept\n    {\n        return m_it == begin_value;\n    }\n\n    /// return whether the iterator is at end\n    constexpr bool is_end() const noexcept\n    {\n        return m_it == end_value;\n    }\n\n    friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept\n    {\n        return lhs.m_it == rhs.m_it;\n    }\n\n    friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept\n    {\n        return lhs.m_it < rhs.m_it;\n    }\n\n    primitive_iterator_t operator+(difference_type n) noexcept\n    {\n        auto result = *this;\n        result += n;\n        return result;\n    }\n\n    friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept\n    {\n        return lhs.m_it - rhs.m_it;\n    }\n\n    primitive_iterator_t& operator++() noexcept\n    {\n        ++m_it;\n        return *this;\n    }\n\n    primitive_iterator_t operator++(int)& noexcept // NOLINT(cert-dcl21-cpp)\n    {\n        auto result = *this;\n        ++m_it;\n        return result;\n    }\n\n    primitive_iterator_t& operator--() noexcept\n    {\n        --m_it;\n        return *this;\n    }\n\n    primitive_iterator_t operator--(int)& noexcept // NOLINT(cert-dcl21-cpp)\n    {\n        auto result = *this;\n        --m_it;\n        return result;\n    }\n\n    primitive_iterator_t& operator+=(difference_type n) noexcept\n    {\n        m_it += n;\n        return *this;\n    }\n\n    primitive_iterator_t& operator-=(difference_type n) noexcept\n    {\n        m_it -= n;\n        return *this;\n    }\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/*!\n@brief an iterator value\n\n@note This structure could easily be a union, but MSVC currently does not allow\nunions members with complex constructors, see https://github.com/nlohmann/json/pull/105.\n*/\ntemplate<typename BasicJsonType> struct internal_iterator\n{\n    /// iterator for JSON objects\n    typename BasicJsonType::object_t::iterator object_iterator {};\n    /// iterator for JSON arrays\n    typename BasicJsonType::array_t::iterator array_iterator {};\n    /// generic iterator for all other types\n    primitive_iterator_t primitive_iterator {};\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/iterators/iter_impl.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next\n#include <type_traits> // conditional, is_const, remove_const\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/iterators/internal_iterator.hpp>\n\n// #include <nlohmann/detail/iterators/primitive_iterator.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n// forward declare, to be able to friend it later on\ntemplate<typename IteratorType> class iteration_proxy;\ntemplate<typename IteratorType> class iteration_proxy_value;\n\n/*!\n@brief a template for a bidirectional iterator for the @ref basic_json class\nThis class implements a both iterators (iterator and const_iterator) for the\n@ref basic_json class.\n@note An iterator is called *initialized* when a pointer to a JSON value has\n      been set (e.g., by a constructor or a copy assignment). If the iterator is\n      default-constructed, it is *uninitialized* and most methods are undefined.\n      **The library uses assertions to detect calls on uninitialized iterators.**\n@requirement The class satisfies the following concept requirements:\n-\n[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):\n  The iterator that can be moved can be moved in both directions (i.e.\n  incremented and decremented).\n@since version 1.0.0, simplified in version 2.0.9, change to bidirectional\n       iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593)\n*/\ntemplate<typename BasicJsonType>\nclass iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)\n{\n    /// the iterator with BasicJsonType of different const-ness\n    using other_iter_impl = iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;\n    /// allow basic_json to access private members\n    friend other_iter_impl;\n    friend BasicJsonType;\n    friend iteration_proxy<iter_impl>;\n    friend iteration_proxy_value<iter_impl>;\n\n    using object_t = typename BasicJsonType::object_t;\n    using array_t = typename BasicJsonType::array_t;\n    // make sure BasicJsonType is basic_json or const basic_json\n    static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,\n                  \"iter_impl only accepts (const) basic_json\");\n    // superficial check for the LegacyBidirectionalIterator named requirement\n    static_assert(std::is_base_of<std::bidirectional_iterator_tag, std::bidirectional_iterator_tag>::value\n                  &&  std::is_base_of<std::bidirectional_iterator_tag, typename std::iterator_traits<typename array_t::iterator>::iterator_category>::value,\n                  \"basic_json iterator assumes array and object type iterators satisfy the LegacyBidirectionalIterator named requirement.\");\n\n  public:\n    /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.\n    /// The C++ Standard has never required user-defined iterators to derive from std::iterator.\n    /// A user-defined iterator should provide publicly accessible typedefs named\n    /// iterator_category, value_type, difference_type, pointer, and reference.\n    /// Note that value_type is required to be non-const, even for constant iterators.\n    using iterator_category = std::bidirectional_iterator_tag;\n\n    /// the type of the values when the iterator is dereferenced\n    using value_type = typename BasicJsonType::value_type;\n    /// a type to represent differences between iterators\n    using difference_type = typename BasicJsonType::difference_type;\n    /// defines a pointer to the type iterated over (value_type)\n    using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,\n          typename BasicJsonType::const_pointer,\n          typename BasicJsonType::pointer>::type;\n    /// defines a reference to the type iterated over (value_type)\n    using reference =\n        typename std::conditional<std::is_const<BasicJsonType>::value,\n        typename BasicJsonType::const_reference,\n        typename BasicJsonType::reference>::type;\n\n    iter_impl() = default;\n    ~iter_impl() = default;\n    iter_impl(iter_impl&&) noexcept = default;\n    iter_impl& operator=(iter_impl&&) noexcept = default;\n\n    /*!\n    @brief constructor for a given JSON instance\n    @param[in] object  pointer to a JSON object for this iterator\n    @pre object != nullptr\n    @post The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    explicit iter_impl(pointer object) noexcept : m_object(object)\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                m_it.object_iterator = typename object_t::iterator();\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_it.array_iterator = typename array_t::iterator();\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                m_it.primitive_iterator = primitive_iterator_t();\n                break;\n            }\n        }\n    }\n\n    /*!\n    @note The conventional copy constructor and copy assignment are implicitly\n          defined. Combined with the following converting constructor and\n          assignment, they support: (1) copy from iterator to iterator, (2)\n          copy from const iterator to const iterator, and (3) conversion from\n          iterator to const iterator. However conversion from const iterator\n          to iterator is not defined.\n    */\n\n    /*!\n    @brief const copy constructor\n    @param[in] other const iterator to copy from\n    @note This copy constructor had to be defined explicitly to circumvent a bug\n          occurring on msvc v19.0 compiler (VS 2015) debug build. For more\n          information refer to: https://github.com/nlohmann/json/issues/1608\n    */\n    iter_impl(const iter_impl<const BasicJsonType>& other) noexcept\n        : m_object(other.m_object), m_it(other.m_it)\n    {}\n\n    /*!\n    @brief converting assignment\n    @param[in] other const iterator to copy from\n    @return const/non-const iterator\n    @note It is not checked whether @a other is initialized.\n    */\n    iter_impl& operator=(const iter_impl<const BasicJsonType>& other) noexcept\n    {\n        if (&other != this)\n        {\n            m_object = other.m_object;\n            m_it = other.m_it;\n        }\n        return *this;\n    }\n\n    /*!\n    @brief converting constructor\n    @param[in] other  non-const iterator to copy from\n    @note It is not checked whether @a other is initialized.\n    */\n    iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept\n        : m_object(other.m_object), m_it(other.m_it)\n    {}\n\n    /*!\n    @brief converting assignment\n    @param[in] other  non-const iterator to copy from\n    @return const/non-const iterator\n    @note It is not checked whether @a other is initialized.\n    */\n    iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept // NOLINT(cert-oop54-cpp)\n    {\n        m_object = other.m_object;\n        m_it = other.m_it;\n        return *this;\n    }\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    /*!\n    @brief set the iterator to the first value\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    void set_begin() noexcept\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                m_it.object_iterator = m_object->m_value.object->begin();\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_it.array_iterator = m_object->m_value.array->begin();\n                break;\n            }\n\n            case value_t::null:\n            {\n                // set to end so begin()==end() is true: null is empty\n                m_it.primitive_iterator.set_end();\n                break;\n            }\n\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                m_it.primitive_iterator.set_begin();\n                break;\n            }\n        }\n    }\n\n    /*!\n    @brief set the iterator past the last value\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    void set_end() noexcept\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                m_it.object_iterator = m_object->m_value.object->end();\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_it.array_iterator = m_object->m_value.array->end();\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                m_it.primitive_iterator.set_end();\n                break;\n            }\n        }\n    }\n\n  public:\n    /*!\n    @brief return a reference to the value pointed to by the iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    reference operator*() const\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end());\n                return m_it.object_iterator->second;\n            }\n\n            case value_t::array:\n            {\n                JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end());\n                return *m_it.array_iterator;\n            }\n\n            case value_t::null:\n                JSON_THROW(invalid_iterator::create(214, \"cannot get value\", m_object));\n\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin()))\n                {\n                    return *m_object;\n                }\n\n                JSON_THROW(invalid_iterator::create(214, \"cannot get value\", m_object));\n            }\n        }\n    }\n\n    /*!\n    @brief dereference the iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    pointer operator->() const\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end());\n                return &(m_it.object_iterator->second);\n            }\n\n            case value_t::array:\n            {\n                JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end());\n                return &*m_it.array_iterator;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin()))\n                {\n                    return m_object;\n                }\n\n                JSON_THROW(invalid_iterator::create(214, \"cannot get value\", m_object));\n            }\n        }\n    }\n\n    /*!\n    @brief post-increment (it++)\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl operator++(int)& // NOLINT(cert-dcl21-cpp)\n    {\n        auto result = *this;\n        ++(*this);\n        return result;\n    }\n\n    /*!\n    @brief pre-increment (++it)\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl& operator++()\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                std::advance(m_it.object_iterator, 1);\n                break;\n            }\n\n            case value_t::array:\n            {\n                std::advance(m_it.array_iterator, 1);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                ++m_it.primitive_iterator;\n                break;\n            }\n        }\n\n        return *this;\n    }\n\n    /*!\n    @brief post-decrement (it--)\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl operator--(int)& // NOLINT(cert-dcl21-cpp)\n    {\n        auto result = *this;\n        --(*this);\n        return result;\n    }\n\n    /*!\n    @brief pre-decrement (--it)\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl& operator--()\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n            {\n                std::advance(m_it.object_iterator, -1);\n                break;\n            }\n\n            case value_t::array:\n            {\n                std::advance(m_it.array_iterator, -1);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                --m_it.primitive_iterator;\n                break;\n            }\n        }\n\n        return *this;\n    }\n\n    /*!\n    @brief comparison: equal\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    template < typename IterImpl, detail::enable_if_t < (std::is_same<IterImpl, iter_impl>::value || std::is_same<IterImpl, other_iter_impl>::value), std::nullptr_t > = nullptr >\n    bool operator==(const IterImpl& other) const\n    {\n        // if objects are not the same, the comparison is undefined\n        if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(212, \"cannot compare iterators of different containers\", m_object));\n        }\n\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n                return (m_it.object_iterator == other.m_it.object_iterator);\n\n            case value_t::array:\n                return (m_it.array_iterator == other.m_it.array_iterator);\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                return (m_it.primitive_iterator == other.m_it.primitive_iterator);\n        }\n    }\n\n    /*!\n    @brief comparison: not equal\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    template < typename IterImpl, detail::enable_if_t < (std::is_same<IterImpl, iter_impl>::value || std::is_same<IterImpl, other_iter_impl>::value), std::nullptr_t > = nullptr >\n    bool operator!=(const IterImpl& other) const\n    {\n        return !operator==(other);\n    }\n\n    /*!\n    @brief comparison: smaller\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    bool operator<(const iter_impl& other) const\n    {\n        // if objects are not the same, the comparison is undefined\n        if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(212, \"cannot compare iterators of different containers\", m_object));\n        }\n\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n                JSON_THROW(invalid_iterator::create(213, \"cannot compare order of object iterators\", m_object));\n\n            case value_t::array:\n                return (m_it.array_iterator < other.m_it.array_iterator);\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                return (m_it.primitive_iterator < other.m_it.primitive_iterator);\n        }\n    }\n\n    /*!\n    @brief comparison: less than or equal\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    bool operator<=(const iter_impl& other) const\n    {\n        return !other.operator < (*this);\n    }\n\n    /*!\n    @brief comparison: greater than\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    bool operator>(const iter_impl& other) const\n    {\n        return !operator<=(other);\n    }\n\n    /*!\n    @brief comparison: greater than or equal\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    bool operator>=(const iter_impl& other) const\n    {\n        return !operator<(other);\n    }\n\n    /*!\n    @brief add to iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl& operator+=(difference_type i)\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n                JSON_THROW(invalid_iterator::create(209, \"cannot use offsets with object iterators\", m_object));\n\n            case value_t::array:\n            {\n                std::advance(m_it.array_iterator, i);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                m_it.primitive_iterator += i;\n                break;\n            }\n        }\n\n        return *this;\n    }\n\n    /*!\n    @brief subtract from iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl& operator-=(difference_type i)\n    {\n        return operator+=(-i);\n    }\n\n    /*!\n    @brief add to iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl operator+(difference_type i) const\n    {\n        auto result = *this;\n        result += i;\n        return result;\n    }\n\n    /*!\n    @brief addition of distance and iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    friend iter_impl operator+(difference_type i, const iter_impl& it)\n    {\n        auto result = it;\n        result += i;\n        return result;\n    }\n\n    /*!\n    @brief subtract from iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    iter_impl operator-(difference_type i) const\n    {\n        auto result = *this;\n        result -= i;\n        return result;\n    }\n\n    /*!\n    @brief return difference\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    difference_type operator-(const iter_impl& other) const\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n                JSON_THROW(invalid_iterator::create(209, \"cannot use offsets with object iterators\", m_object));\n\n            case value_t::array:\n                return m_it.array_iterator - other.m_it.array_iterator;\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                return m_it.primitive_iterator - other.m_it.primitive_iterator;\n        }\n    }\n\n    /*!\n    @brief access to successor\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    reference operator[](difference_type n) const\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        switch (m_object->m_type)\n        {\n            case value_t::object:\n                JSON_THROW(invalid_iterator::create(208, \"cannot use operator[] for object iterators\", m_object));\n\n            case value_t::array:\n                return *std::next(m_it.array_iterator, n);\n\n            case value_t::null:\n                JSON_THROW(invalid_iterator::create(214, \"cannot get value\", m_object));\n\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.get_value() == -n))\n                {\n                    return *m_object;\n                }\n\n                JSON_THROW(invalid_iterator::create(214, \"cannot get value\", m_object));\n            }\n        }\n    }\n\n    /*!\n    @brief return the key of an object iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    const typename object_t::key_type& key() const\n    {\n        JSON_ASSERT(m_object != nullptr);\n\n        if (JSON_HEDLEY_LIKELY(m_object->is_object()))\n        {\n            return m_it.object_iterator->first;\n        }\n\n        JSON_THROW(invalid_iterator::create(207, \"cannot use key() for non-object iterators\", m_object));\n    }\n\n    /*!\n    @brief return the value of an iterator\n    @pre The iterator is initialized; i.e. `m_object != nullptr`.\n    */\n    reference value() const\n    {\n        return operator*();\n    }\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    /// associated JSON instance\n    pointer m_object = nullptr;\n    /// the actual iterator of the associated instance\n    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it {};\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/iterators/iteration_proxy.hpp>\n\n// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <cstddef> // ptrdiff_t\n#include <iterator> // reverse_iterator\n#include <utility> // declval\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n//////////////////////\n// reverse_iterator //\n//////////////////////\n\n/*!\n@brief a template for a reverse iterator class\n\n@tparam Base the base iterator type to reverse. Valid types are @ref\niterator (to create @ref reverse_iterator) and @ref const_iterator (to\ncreate @ref const_reverse_iterator).\n\n@requirement The class satisfies the following concept requirements:\n-\n[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator):\n  The iterator that can be moved can be moved in both directions (i.e.\n  incremented and decremented).\n- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator):\n  It is possible to write to the pointed-to element (only if @a Base is\n  @ref iterator).\n\n@since version 1.0.0\n*/\ntemplate<typename Base>\nclass json_reverse_iterator : public std::reverse_iterator<Base>\n{\n  public:\n    using difference_type = std::ptrdiff_t;\n    /// shortcut to the reverse iterator adapter\n    using base_iterator = std::reverse_iterator<Base>;\n    /// the reference type for the pointed-to element\n    using reference = typename Base::reference;\n\n    /// create reverse iterator from iterator\n    explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept\n        : base_iterator(it) {}\n\n    /// create reverse iterator from base class\n    explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}\n\n    /// post-increment (it++)\n    json_reverse_iterator operator++(int)& // NOLINT(cert-dcl21-cpp)\n    {\n        return static_cast<json_reverse_iterator>(base_iterator::operator++(1));\n    }\n\n    /// pre-increment (++it)\n    json_reverse_iterator& operator++()\n    {\n        return static_cast<json_reverse_iterator&>(base_iterator::operator++());\n    }\n\n    /// post-decrement (it--)\n    json_reverse_iterator operator--(int)& // NOLINT(cert-dcl21-cpp)\n    {\n        return static_cast<json_reverse_iterator>(base_iterator::operator--(1));\n    }\n\n    /// pre-decrement (--it)\n    json_reverse_iterator& operator--()\n    {\n        return static_cast<json_reverse_iterator&>(base_iterator::operator--());\n    }\n\n    /// add to iterator\n    json_reverse_iterator& operator+=(difference_type i)\n    {\n        return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));\n    }\n\n    /// add to iterator\n    json_reverse_iterator operator+(difference_type i) const\n    {\n        return static_cast<json_reverse_iterator>(base_iterator::operator+(i));\n    }\n\n    /// subtract from iterator\n    json_reverse_iterator operator-(difference_type i) const\n    {\n        return static_cast<json_reverse_iterator>(base_iterator::operator-(i));\n    }\n\n    /// return difference\n    difference_type operator-(const json_reverse_iterator& other) const\n    {\n        return base_iterator(*this) - base_iterator(other);\n    }\n\n    /// access to successor\n    reference operator[](difference_type n) const\n    {\n        return *(this->operator+(n));\n    }\n\n    /// return the key of an object iterator\n    auto key() const -> decltype(std::declval<Base>().key())\n    {\n        auto it = --this->base();\n        return it.key();\n    }\n\n    /// return the value of an iterator\n    reference value() const\n    {\n        auto it = --this->base();\n        return it.operator * ();\n    }\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/iterators/primitive_iterator.hpp>\n\n// #include <nlohmann/detail/json_pointer.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // all_of\n#include <cctype> // isdigit\n#include <cerrno> // errno, ERANGE\n#include <cstdlib> // strtoull\n#ifndef JSON_NO_IO\n    #include <iosfwd> // ostream\n#endif  // JSON_NO_IO\n#include <limits> // max\n#include <numeric> // accumulate\n#include <string> // string\n#include <utility> // move\n#include <vector> // vector\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/string_escape.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document\n/// @sa https://json.nlohmann.me/api/json_pointer/\ntemplate<typename RefStringType>\nclass json_pointer\n{\n    // allow basic_json to access private members\n    NLOHMANN_BASIC_JSON_TPL_DECLARATION\n    friend class basic_json;\n\n    template<typename>\n    friend class json_pointer;\n\n    template<typename T>\n    struct string_t_helper\n    {\n        using type = T;\n    };\n\n    NLOHMANN_BASIC_JSON_TPL_DECLARATION\n    struct string_t_helper<NLOHMANN_BASIC_JSON_TPL>\n    {\n        using type = StringType;\n    };\n\n  public:\n    // for backwards compatibility accept BasicJsonType\n    using string_t = typename string_t_helper<RefStringType>::type;\n\n    /// @brief create JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/json_pointer/\n    explicit json_pointer(const string_t& s = \"\")\n        : reference_tokens(split(s))\n    {}\n\n    /// @brief return a string representation of the JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/to_string/\n    string_t to_string() const\n    {\n        return std::accumulate(reference_tokens.begin(), reference_tokens.end(),\n                               string_t{},\n                               [](const string_t& a, const string_t& b)\n        {\n            return detail::concat(a, '/', detail::escape(b));\n        });\n    }\n\n    /// @brief return a string representation of the JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_string/\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, to_string())\n    operator string_t() const\n    {\n        return to_string();\n    }\n\n#ifndef JSON_NO_IO\n    /// @brief write string representation of the JSON pointer to stream\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/\n    friend std::ostream& operator<<(std::ostream& o, const json_pointer& ptr)\n    {\n        o << ptr.to_string();\n        return o;\n    }\n#endif\n\n    /// @brief append another JSON pointer at the end of this JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/\n    json_pointer& operator/=(const json_pointer& ptr)\n    {\n        reference_tokens.insert(reference_tokens.end(),\n                                ptr.reference_tokens.begin(),\n                                ptr.reference_tokens.end());\n        return *this;\n    }\n\n    /// @brief append an unescaped reference token at the end of this JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/\n    json_pointer& operator/=(string_t token)\n    {\n        push_back(std::move(token));\n        return *this;\n    }\n\n    /// @brief append an array index at the end of this JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/\n    json_pointer& operator/=(std::size_t array_idx)\n    {\n        return *this /= std::to_string(array_idx);\n    }\n\n    /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/\n    friend json_pointer operator/(const json_pointer& lhs,\n                                  const json_pointer& rhs)\n    {\n        return json_pointer(lhs) /= rhs;\n    }\n\n    /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/\n    friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param)\n    {\n        return json_pointer(lhs) /= std::move(token);\n    }\n\n    /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/\n    friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx)\n    {\n        return json_pointer(lhs) /= array_idx;\n    }\n\n    /// @brief returns the parent of this JSON pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/parent_pointer/\n    json_pointer parent_pointer() const\n    {\n        if (empty())\n        {\n            return *this;\n        }\n\n        json_pointer res = *this;\n        res.pop_back();\n        return res;\n    }\n\n    /// @brief remove last reference token\n    /// @sa https://json.nlohmann.me/api/json_pointer/pop_back/\n    void pop_back()\n    {\n        if (JSON_HEDLEY_UNLIKELY(empty()))\n        {\n            JSON_THROW(detail::out_of_range::create(405, \"JSON pointer has no parent\", nullptr));\n        }\n\n        reference_tokens.pop_back();\n    }\n\n    /// @brief return last reference token\n    /// @sa https://json.nlohmann.me/api/json_pointer/back/\n    const string_t& back() const\n    {\n        if (JSON_HEDLEY_UNLIKELY(empty()))\n        {\n            JSON_THROW(detail::out_of_range::create(405, \"JSON pointer has no parent\", nullptr));\n        }\n\n        return reference_tokens.back();\n    }\n\n    /// @brief append an unescaped token at the end of the reference pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/push_back/\n    void push_back(const string_t& token)\n    {\n        reference_tokens.push_back(token);\n    }\n\n    /// @brief append an unescaped token at the end of the reference pointer\n    /// @sa https://json.nlohmann.me/api/json_pointer/push_back/\n    void push_back(string_t&& token)\n    {\n        reference_tokens.push_back(std::move(token));\n    }\n\n    /// @brief return whether pointer points to the root document\n    /// @sa https://json.nlohmann.me/api/json_pointer/empty/\n    bool empty() const noexcept\n    {\n        return reference_tokens.empty();\n    }\n\n  private:\n    /*!\n    @param[in] s  reference token to be converted into an array index\n\n    @return integer representation of @a s\n\n    @throw parse_error.106  if an array index begins with '0'\n    @throw parse_error.109  if an array index begins not with a digit\n    @throw out_of_range.404 if string @a s could not be converted to an integer\n    @throw out_of_range.410 if an array index exceeds size_type\n    */\n    template<typename BasicJsonType>\n    static typename BasicJsonType::size_type array_index(const string_t& s)\n    {\n        using size_type = typename BasicJsonType::size_type;\n\n        // error condition (cf. RFC 6901, Sect. 4)\n        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0'))\n        {\n            JSON_THROW(detail::parse_error::create(106, 0, detail::concat(\"array index '\", s, \"' must not begin with '0'\"), nullptr));\n        }\n\n        // error condition (cf. RFC 6901, Sect. 4)\n        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))\n        {\n            JSON_THROW(detail::parse_error::create(109, 0, detail::concat(\"array index '\", s, \"' is not a number\"), nullptr));\n        }\n\n        const char* p = s.c_str();\n        char* p_end = nullptr;\n        errno = 0; // strtoull doesn't reset errno\n        unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)\n        if (p == p_end // invalid input or empty string\n                || errno == ERANGE // out of range\n                || JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(p_end - p) != s.size())) // incomplete read\n        {\n            JSON_THROW(detail::out_of_range::create(404, detail::concat(\"unresolved reference token '\", s, \"'\"), nullptr));\n        }\n\n        // only triggered on special platforms (like 32bit), see also\n        // https://github.com/nlohmann/json/pull/2203\n        if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))  // NOLINT(runtime/int)\n        {\n            JSON_THROW(detail::out_of_range::create(410, detail::concat(\"array index \", s, \" exceeds size_type\"), nullptr));   // LCOV_EXCL_LINE\n        }\n\n        return static_cast<size_type>(res);\n    }\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    json_pointer top() const\n    {\n        if (JSON_HEDLEY_UNLIKELY(empty()))\n        {\n            JSON_THROW(detail::out_of_range::create(405, \"JSON pointer has no parent\", nullptr));\n        }\n\n        json_pointer result = *this;\n        result.reference_tokens = {reference_tokens[0]};\n        return result;\n    }\n\n  private:\n    /*!\n    @brief create and return a reference to the pointed to value\n\n    @complexity Linear in the number of reference tokens.\n\n    @throw parse_error.109 if array index is not a number\n    @throw type_error.313 if value cannot be unflattened\n    */\n    template<typename BasicJsonType>\n    BasicJsonType& get_and_create(BasicJsonType& j) const\n    {\n        auto* result = &j;\n\n        // in case no reference tokens exist, return a reference to the JSON value\n        // j which will be overwritten by a primitive value\n        for (const auto& reference_token : reference_tokens)\n        {\n            switch (result->type())\n            {\n                case detail::value_t::null:\n                {\n                    if (reference_token == \"0\")\n                    {\n                        // start a new array if reference token is 0\n                        result = &result->operator[](0);\n                    }\n                    else\n                    {\n                        // start a new object otherwise\n                        result = &result->operator[](reference_token);\n                    }\n                    break;\n                }\n\n                case detail::value_t::object:\n                {\n                    // create an entry in the object\n                    result = &result->operator[](reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    // create an entry in the array\n                    result = &result->operator[](array_index<BasicJsonType>(reference_token));\n                    break;\n                }\n\n                /*\n                The following code is only reached if there exists a reference\n                token _and_ the current value is primitive. In this case, we have\n                an error situation, because primitive values may only occur as\n                single value; that is, with an empty list of reference tokens.\n                */\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                    JSON_THROW(detail::type_error::create(313, \"invalid value to unflatten\", &j));\n            }\n        }\n\n        return *result;\n    }\n\n    /*!\n    @brief return a reference to the pointed to value\n\n    @note This version does not throw if a value is not present, but tries to\n          create nested values instead. For instance, calling this function\n          with pointer `\"/this/that\"` on a null value is equivalent to calling\n          `operator[](\"this\").operator[](\"that\")` on that value, effectively\n          changing the null value to an object.\n\n    @param[in] ptr  a JSON value\n\n    @return reference to the JSON value pointed to by the JSON pointer\n\n    @complexity Linear in the length of the JSON pointer.\n\n    @throw parse_error.106   if an array index begins with '0'\n    @throw parse_error.109   if an array index was not a number\n    @throw out_of_range.404  if the JSON pointer can not be resolved\n    */\n    template<typename BasicJsonType>\n    BasicJsonType& get_unchecked(BasicJsonType* ptr) const\n    {\n        for (const auto& reference_token : reference_tokens)\n        {\n            // convert null values to arrays or objects before continuing\n            if (ptr->is_null())\n            {\n                // check if reference token is a number\n                const bool nums =\n                    std::all_of(reference_token.begin(), reference_token.end(),\n                                [](const unsigned char x)\n                {\n                    return std::isdigit(x);\n                });\n\n                // change value to array for numbers or \"-\" or to object otherwise\n                *ptr = (nums || reference_token == \"-\")\n                       ? detail::value_t::array\n                       : detail::value_t::object;\n            }\n\n            switch (ptr->type())\n            {\n                case detail::value_t::object:\n                {\n                    // use unchecked object access\n                    ptr = &ptr->operator[](reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    if (reference_token == \"-\")\n                    {\n                        // explicitly treat \"-\" as index beyond the end\n                        ptr = &ptr->operator[](ptr->m_value.array->size());\n                    }\n                    else\n                    {\n                        // convert array index to number; unchecked access\n                        ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));\n                    }\n                    break;\n                }\n\n                case detail::value_t::null:\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                    JSON_THROW(detail::out_of_range::create(404, detail::concat(\"unresolved reference token '\", reference_token, \"'\"), ptr));\n            }\n        }\n\n        return *ptr;\n    }\n\n    /*!\n    @throw parse_error.106   if an array index begins with '0'\n    @throw parse_error.109   if an array index was not a number\n    @throw out_of_range.402  if the array index '-' is used\n    @throw out_of_range.404  if the JSON pointer can not be resolved\n    */\n    template<typename BasicJsonType>\n    BasicJsonType& get_checked(BasicJsonType* ptr) const\n    {\n        for (const auto& reference_token : reference_tokens)\n        {\n            switch (ptr->type())\n            {\n                case detail::value_t::object:\n                {\n                    // note: at performs range check\n                    ptr = &ptr->at(reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(reference_token == \"-\"))\n                    {\n                        // \"-\" always fails the range check\n                        JSON_THROW(detail::out_of_range::create(402, detail::concat(\n                                \"array index '-' (\", std::to_string(ptr->m_value.array->size()),\n                                \") is out of range\"), ptr));\n                    }\n\n                    // note: at performs range check\n                    ptr = &ptr->at(array_index<BasicJsonType>(reference_token));\n                    break;\n                }\n\n                case detail::value_t::null:\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                    JSON_THROW(detail::out_of_range::create(404, detail::concat(\"unresolved reference token '\", reference_token, \"'\"), ptr));\n            }\n        }\n\n        return *ptr;\n    }\n\n    /*!\n    @brief return a const reference to the pointed to value\n\n    @param[in] ptr  a JSON value\n\n    @return const reference to the JSON value pointed to by the JSON\n    pointer\n\n    @throw parse_error.106   if an array index begins with '0'\n    @throw parse_error.109   if an array index was not a number\n    @throw out_of_range.402  if the array index '-' is used\n    @throw out_of_range.404  if the JSON pointer can not be resolved\n    */\n    template<typename BasicJsonType>\n    const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const\n    {\n        for (const auto& reference_token : reference_tokens)\n        {\n            switch (ptr->type())\n            {\n                case detail::value_t::object:\n                {\n                    // use unchecked object access\n                    ptr = &ptr->operator[](reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(reference_token == \"-\"))\n                    {\n                        // \"-\" cannot be used for const access\n                        JSON_THROW(detail::out_of_range::create(402, detail::concat(\"array index '-' (\", std::to_string(ptr->m_value.array->size()), \") is out of range\"), ptr));\n                    }\n\n                    // use unchecked array access\n                    ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));\n                    break;\n                }\n\n                case detail::value_t::null:\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                    JSON_THROW(detail::out_of_range::create(404, detail::concat(\"unresolved reference token '\", reference_token, \"'\"), ptr));\n            }\n        }\n\n        return *ptr;\n    }\n\n    /*!\n    @throw parse_error.106   if an array index begins with '0'\n    @throw parse_error.109   if an array index was not a number\n    @throw out_of_range.402  if the array index '-' is used\n    @throw out_of_range.404  if the JSON pointer can not be resolved\n    */\n    template<typename BasicJsonType>\n    const BasicJsonType& get_checked(const BasicJsonType* ptr) const\n    {\n        for (const auto& reference_token : reference_tokens)\n        {\n            switch (ptr->type())\n            {\n                case detail::value_t::object:\n                {\n                    // note: at performs range check\n                    ptr = &ptr->at(reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(reference_token == \"-\"))\n                    {\n                        // \"-\" always fails the range check\n                        JSON_THROW(detail::out_of_range::create(402, detail::concat(\n                                \"array index '-' (\", std::to_string(ptr->m_value.array->size()),\n                                \") is out of range\"), ptr));\n                    }\n\n                    // note: at performs range check\n                    ptr = &ptr->at(array_index<BasicJsonType>(reference_token));\n                    break;\n                }\n\n                case detail::value_t::null:\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                    JSON_THROW(detail::out_of_range::create(404, detail::concat(\"unresolved reference token '\", reference_token, \"'\"), ptr));\n            }\n        }\n\n        return *ptr;\n    }\n\n    /*!\n    @throw parse_error.106   if an array index begins with '0'\n    @throw parse_error.109   if an array index was not a number\n    */\n    template<typename BasicJsonType>\n    bool contains(const BasicJsonType* ptr) const\n    {\n        for (const auto& reference_token : reference_tokens)\n        {\n            switch (ptr->type())\n            {\n                case detail::value_t::object:\n                {\n                    if (!ptr->contains(reference_token))\n                    {\n                        // we did not find the key in the object\n                        return false;\n                    }\n\n                    ptr = &ptr->operator[](reference_token);\n                    break;\n                }\n\n                case detail::value_t::array:\n                {\n                    if (JSON_HEDLEY_UNLIKELY(reference_token == \"-\"))\n                    {\n                        // \"-\" always fails the range check\n                        return false;\n                    }\n                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !(\"0\" <= reference_token && reference_token <= \"9\")))\n                    {\n                        // invalid char\n                        return false;\n                    }\n                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1))\n                    {\n                        if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9')))\n                        {\n                            // first char should be between '1' and '9'\n                            return false;\n                        }\n                        for (std::size_t i = 1; i < reference_token.size(); i++)\n                        {\n                            if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9')))\n                            {\n                                // other char should be between '0' and '9'\n                                return false;\n                            }\n                        }\n                    }\n\n                    const auto idx = array_index<BasicJsonType>(reference_token);\n                    if (idx >= ptr->size())\n                    {\n                        // index out of range\n                        return false;\n                    }\n\n                    ptr = &ptr->operator[](idx);\n                    break;\n                }\n\n                case detail::value_t::null:\n                case detail::value_t::string:\n                case detail::value_t::boolean:\n                case detail::value_t::number_integer:\n                case detail::value_t::number_unsigned:\n                case detail::value_t::number_float:\n                case detail::value_t::binary:\n                case detail::value_t::discarded:\n                default:\n                {\n                    // we do not expect primitive values if there is still a\n                    // reference token to process\n                    return false;\n                }\n            }\n        }\n\n        // no reference token left means we found a primitive value\n        return true;\n    }\n\n    /*!\n    @brief split the string input to reference tokens\n\n    @note This function is only called by the json_pointer constructor.\n          All exceptions below are documented there.\n\n    @throw parse_error.107  if the pointer is not empty or begins with '/'\n    @throw parse_error.108  if character '~' is not followed by '0' or '1'\n    */\n    static std::vector<string_t> split(const string_t& reference_string)\n    {\n        std::vector<string_t> result;\n\n        // special case: empty reference string -> no reference tokens\n        if (reference_string.empty())\n        {\n            return result;\n        }\n\n        // check if nonempty reference string begins with slash\n        if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/'))\n        {\n            JSON_THROW(detail::parse_error::create(107, 1, detail::concat(\"JSON pointer must be empty or begin with '/' - was: '\", reference_string, \"'\"), nullptr));\n        }\n\n        // extract the reference tokens:\n        // - slash: position of the last read slash (or end of string)\n        // - start: position after the previous slash\n        for (\n            // search for the first slash after the first character\n            std::size_t slash = reference_string.find_first_of('/', 1),\n            // set the beginning of the first reference token\n            start = 1;\n            // we can stop if start == 0 (if slash == string_t::npos)\n            start != 0;\n            // set the beginning of the next reference token\n            // (will eventually be 0 if slash == string_t::npos)\n            start = (slash == string_t::npos) ? 0 : slash + 1,\n            // find next slash\n            slash = reference_string.find_first_of('/', start))\n        {\n            // use the text between the beginning of the reference token\n            // (start) and the last slash (slash).\n            auto reference_token = reference_string.substr(start, slash - start);\n\n            // check reference tokens are properly escaped\n            for (std::size_t pos = reference_token.find_first_of('~');\n                    pos != string_t::npos;\n                    pos = reference_token.find_first_of('~', pos + 1))\n            {\n                JSON_ASSERT(reference_token[pos] == '~');\n\n                // ~ must be followed by 0 or 1\n                if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 ||\n                                         (reference_token[pos + 1] != '0' &&\n                                          reference_token[pos + 1] != '1')))\n                {\n                    JSON_THROW(detail::parse_error::create(108, 0, \"escape character '~' must be followed with '0' or '1'\", nullptr));\n                }\n            }\n\n            // finally, store the reference token\n            detail::unescape(reference_token);\n            result.push_back(reference_token);\n        }\n\n        return result;\n    }\n\n  private:\n    /*!\n    @param[in] reference_string  the reference string to the current value\n    @param[in] value             the value to consider\n    @param[in,out] result        the result object to insert values to\n\n    @note Empty objects or arrays are flattened to `null`.\n    */\n    template<typename BasicJsonType>\n    static void flatten(const string_t& reference_string,\n                        const BasicJsonType& value,\n                        BasicJsonType& result)\n    {\n        switch (value.type())\n        {\n            case detail::value_t::array:\n            {\n                if (value.m_value.array->empty())\n                {\n                    // flatten empty array as null\n                    result[reference_string] = nullptr;\n                }\n                else\n                {\n                    // iterate array and use index as reference string\n                    for (std::size_t i = 0; i < value.m_value.array->size(); ++i)\n                    {\n                        flatten(detail::concat(reference_string, '/', std::to_string(i)),\n                                value.m_value.array->operator[](i), result);\n                    }\n                }\n                break;\n            }\n\n            case detail::value_t::object:\n            {\n                if (value.m_value.object->empty())\n                {\n                    // flatten empty object as null\n                    result[reference_string] = nullptr;\n                }\n                else\n                {\n                    // iterate object and use keys as reference string\n                    for (const auto& element : *value.m_value.object)\n                    {\n                        flatten(detail::concat(reference_string, '/', detail::escape(element.first)), element.second, result);\n                    }\n                }\n                break;\n            }\n\n            case detail::value_t::null:\n            case detail::value_t::string:\n            case detail::value_t::boolean:\n            case detail::value_t::number_integer:\n            case detail::value_t::number_unsigned:\n            case detail::value_t::number_float:\n            case detail::value_t::binary:\n            case detail::value_t::discarded:\n            default:\n            {\n                // add primitive value with its reference string\n                result[reference_string] = value;\n                break;\n            }\n        }\n    }\n\n    /*!\n    @param[in] value  flattened JSON\n\n    @return unflattened JSON\n\n    @throw parse_error.109 if array index is not a number\n    @throw type_error.314  if value is not an object\n    @throw type_error.315  if object values are not primitive\n    @throw type_error.313  if value cannot be unflattened\n    */\n    template<typename BasicJsonType>\n    static BasicJsonType\n    unflatten(const BasicJsonType& value)\n    {\n        if (JSON_HEDLEY_UNLIKELY(!value.is_object()))\n        {\n            JSON_THROW(detail::type_error::create(314, \"only objects can be unflattened\", &value));\n        }\n\n        BasicJsonType result;\n\n        // iterate the JSON object values\n        for (const auto& element : *value.m_value.object)\n        {\n            if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))\n            {\n                JSON_THROW(detail::type_error::create(315, \"values in object must be primitive\", &element.second));\n            }\n\n            // assign value to reference pointed to by JSON pointer; Note that if\n            // the JSON pointer is \"\" (i.e., points to the whole value), function\n            // get_and_create returns a reference to result itself. An assignment\n            // will then create a primitive value.\n            json_pointer(element.first).get_and_create(result) = element.second;\n        }\n\n        return result;\n    }\n\n    // can't use conversion operator because of ambiguity\n    json_pointer<string_t> convert() const&\n    {\n        json_pointer<string_t> result;\n        result.reference_tokens = reference_tokens;\n        return result;\n    }\n\n    json_pointer<string_t> convert()&&\n    {\n        json_pointer<string_t> result;\n        result.reference_tokens = std::move(reference_tokens);\n        return result;\n    }\n\n  public:\n#if JSON_HAS_THREE_WAY_COMPARISON\n    /// @brief compares two JSON pointers for equality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/\n    template<typename RefStringTypeRhs>\n    bool operator==(const json_pointer<RefStringTypeRhs>& rhs) const noexcept\n    {\n        return reference_tokens == rhs.reference_tokens;\n    }\n\n    /// @brief compares JSON pointer and string for equality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer))\n    bool operator==(const string_t& rhs) const\n    {\n        return *this == json_pointer(rhs);\n    }\n\n    /// @brief 3-way compares two JSON pointers\n    template<typename RefStringTypeRhs>\n    std::strong_ordering operator<=>(const json_pointer<RefStringTypeRhs>& rhs) const noexcept // *NOPAD*\n    {\n        return  reference_tokens <=> rhs.reference_tokens; // *NOPAD*\n    }\n#else\n    /// @brief compares two JSON pointers for equality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/\n    template<typename RefStringTypeLhs, typename RefStringTypeRhs>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator==(const json_pointer<RefStringTypeLhs>& lhs,\n                           const json_pointer<RefStringTypeRhs>& rhs) noexcept;\n\n    /// @brief compares JSON pointer and string for equality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/\n    template<typename RefStringTypeLhs, typename StringType>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator==(const json_pointer<RefStringTypeLhs>& lhs,\n                           const StringType& rhs);\n\n    /// @brief compares string and JSON pointer for equality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_eq/\n    template<typename RefStringTypeRhs, typename StringType>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator==(const StringType& lhs,\n                           const json_pointer<RefStringTypeRhs>& rhs);\n\n    /// @brief compares two JSON pointers for inequality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/\n    template<typename RefStringTypeLhs, typename RefStringTypeRhs>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,\n                           const json_pointer<RefStringTypeRhs>& rhs) noexcept;\n\n    /// @brief compares JSON pointer and string for inequality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/\n    template<typename RefStringTypeLhs, typename StringType>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,\n                           const StringType& rhs);\n\n    /// @brief compares string and JSON pointer for inequality\n    /// @sa https://json.nlohmann.me/api/json_pointer/operator_ne/\n    template<typename RefStringTypeRhs, typename StringType>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator!=(const StringType& lhs,\n                           const json_pointer<RefStringTypeRhs>& rhs);\n\n    /// @brief compares two JSON pointer for less-than\n    template<typename RefStringTypeLhs, typename RefStringTypeRhs>\n    // NOLINTNEXTLINE(readability-redundant-declaration)\n    friend bool operator<(const json_pointer<RefStringTypeLhs>& lhs,\n                          const json_pointer<RefStringTypeRhs>& rhs) noexcept;\n#endif\n\n  private:\n    /// the reference tokens\n    std::vector<string_t> reference_tokens;\n};\n\n#if !JSON_HAS_THREE_WAY_COMPARISON\n// functions cannot be defined inside class due to ODR violations\ntemplate<typename RefStringTypeLhs, typename RefStringTypeRhs>\ninline bool operator==(const json_pointer<RefStringTypeLhs>& lhs,\n                       const json_pointer<RefStringTypeRhs>& rhs) noexcept\n{\n    return lhs.reference_tokens == rhs.reference_tokens;\n}\n\ntemplate<typename RefStringTypeLhs,\n         typename StringType = typename json_pointer<RefStringTypeLhs>::string_t>\nJSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer, json_pointer))\ninline bool operator==(const json_pointer<RefStringTypeLhs>& lhs,\n                       const StringType& rhs)\n{\n    return lhs == json_pointer<RefStringTypeLhs>(rhs);\n}\n\ntemplate<typename RefStringTypeRhs,\n         typename StringType = typename json_pointer<RefStringTypeRhs>::string_t>\nJSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator==(json_pointer, json_pointer))\ninline bool operator==(const StringType& lhs,\n                       const json_pointer<RefStringTypeRhs>& rhs)\n{\n    return json_pointer<RefStringTypeRhs>(lhs) == rhs;\n}\n\ntemplate<typename RefStringTypeLhs, typename RefStringTypeRhs>\ninline bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,\n                       const json_pointer<RefStringTypeRhs>& rhs) noexcept\n{\n    return !(lhs == rhs);\n}\n\ntemplate<typename RefStringTypeLhs,\n         typename StringType = typename json_pointer<RefStringTypeLhs>::string_t>\nJSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator!=(json_pointer, json_pointer))\ninline bool operator!=(const json_pointer<RefStringTypeLhs>& lhs,\n                       const StringType& rhs)\n{\n    return !(lhs == rhs);\n}\n\ntemplate<typename RefStringTypeRhs,\n         typename StringType = typename json_pointer<RefStringTypeRhs>::string_t>\nJSON_HEDLEY_DEPRECATED_FOR(3.11.2, operator!=(json_pointer, json_pointer))\ninline bool operator!=(const StringType& lhs,\n                       const json_pointer<RefStringTypeRhs>& rhs)\n{\n    return !(lhs == rhs);\n}\n\ntemplate<typename RefStringTypeLhs, typename RefStringTypeRhs>\ninline bool operator<(const json_pointer<RefStringTypeLhs>& lhs,\n                      const json_pointer<RefStringTypeRhs>& rhs) noexcept\n{\n    return lhs.reference_tokens < rhs.reference_tokens;\n}\n#endif\n\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/json_ref.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <initializer_list>\n#include <utility>\n\n// #include <nlohmann/detail/abi_macros.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\ntemplate<typename BasicJsonType>\nclass json_ref\n{\n  public:\n    using value_type = BasicJsonType;\n\n    json_ref(value_type&& value)\n        : owned_value(std::move(value))\n    {}\n\n    json_ref(const value_type& value)\n        : value_ref(&value)\n    {}\n\n    json_ref(std::initializer_list<json_ref> init)\n        : owned_value(init)\n    {}\n\n    template <\n        class... Args,\n        enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 >\n    json_ref(Args && ... args)\n        : owned_value(std::forward<Args>(args)...)\n    {}\n\n    // class should be movable only\n    json_ref(json_ref&&) noexcept = default;\n    json_ref(const json_ref&) = delete;\n    json_ref& operator=(const json_ref&) = delete;\n    json_ref& operator=(json_ref&&) = delete;\n    ~json_ref() = default;\n\n    value_type moved_or_copied() const\n    {\n        if (value_ref == nullptr)\n        {\n            return std::move(owned_value);\n        }\n        return *value_ref;\n    }\n\n    value_type const& operator*() const\n    {\n        return value_ref ? *value_ref : owned_value;\n    }\n\n    value_type const* operator->() const\n    {\n        return &** this;\n    }\n\n  private:\n    mutable value_type owned_value = nullptr;\n    value_type const* value_ref = nullptr;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/string_escape.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n// #include <nlohmann/detail/output/binary_writer.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // reverse\n#include <array> // array\n#include <map> // map\n#include <cmath> // isnan, isinf\n#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t\n#include <cstring> // memcpy\n#include <limits> // numeric_limits\n#include <string> // string\n#include <utility> // move\n#include <vector> // vector\n\n// #include <nlohmann/detail/input/binary_reader.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/output/output_adapters.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // copy\n#include <cstddef> // size_t\n#include <iterator> // back_inserter\n#include <memory> // shared_ptr, make_shared\n#include <string> // basic_string\n#include <vector> // vector\n\n#ifndef JSON_NO_IO\n    #include <ios>      // streamsize\n    #include <ostream>  // basic_ostream\n#endif  // JSON_NO_IO\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/// abstract output adapter interface\ntemplate<typename CharType> struct output_adapter_protocol\n{\n    virtual void write_character(CharType c) = 0;\n    virtual void write_characters(const CharType* s, std::size_t length) = 0;\n    virtual ~output_adapter_protocol() = default;\n\n    output_adapter_protocol() = default;\n    output_adapter_protocol(const output_adapter_protocol&) = default;\n    output_adapter_protocol(output_adapter_protocol&&) noexcept = default;\n    output_adapter_protocol& operator=(const output_adapter_protocol&) = default;\n    output_adapter_protocol& operator=(output_adapter_protocol&&) noexcept = default;\n};\n\n/// a type to simplify interfaces\ntemplate<typename CharType>\nusing output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;\n\n/// output adapter for byte vectors\ntemplate<typename CharType, typename AllocatorType = std::allocator<CharType>>\nclass output_vector_adapter : public output_adapter_protocol<CharType>\n{\n  public:\n    explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept\n        : v(vec)\n    {}\n\n    void write_character(CharType c) override\n    {\n        v.push_back(c);\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    void write_characters(const CharType* s, std::size_t length) override\n    {\n        v.insert(v.end(), s, s + length);\n    }\n\n  private:\n    std::vector<CharType, AllocatorType>& v;\n};\n\n#ifndef JSON_NO_IO\n/// output adapter for output streams\ntemplate<typename CharType>\nclass output_stream_adapter : public output_adapter_protocol<CharType>\n{\n  public:\n    explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept\n        : stream(s)\n    {}\n\n    void write_character(CharType c) override\n    {\n        stream.put(c);\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    void write_characters(const CharType* s, std::size_t length) override\n    {\n        stream.write(s, static_cast<std::streamsize>(length));\n    }\n\n  private:\n    std::basic_ostream<CharType>& stream;\n};\n#endif  // JSON_NO_IO\n\n/// output adapter for basic_string\ntemplate<typename CharType, typename StringType = std::basic_string<CharType>>\nclass output_string_adapter : public output_adapter_protocol<CharType>\n{\n  public:\n    explicit output_string_adapter(StringType& s) noexcept\n        : str(s)\n    {}\n\n    void write_character(CharType c) override\n    {\n        str.push_back(c);\n    }\n\n    JSON_HEDLEY_NON_NULL(2)\n    void write_characters(const CharType* s, std::size_t length) override\n    {\n        str.append(s, length);\n    }\n\n  private:\n    StringType& str;\n};\n\ntemplate<typename CharType, typename StringType = std::basic_string<CharType>>\nclass output_adapter\n{\n  public:\n    template<typename AllocatorType = std::allocator<CharType>>\n    output_adapter(std::vector<CharType, AllocatorType>& vec)\n        : oa(std::make_shared<output_vector_adapter<CharType, AllocatorType>>(vec)) {}\n\n#ifndef JSON_NO_IO\n    output_adapter(std::basic_ostream<CharType>& s)\n        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}\n#endif  // JSON_NO_IO\n\n    output_adapter(StringType& s)\n        : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {}\n\n    operator output_adapter_t<CharType>()\n    {\n        return oa;\n    }\n\n  private:\n    output_adapter_t<CharType> oa = nullptr;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n///////////////////\n// binary writer //\n///////////////////\n\n/*!\n@brief serialization to CBOR and MessagePack values\n*/\ntemplate<typename BasicJsonType, typename CharType>\nclass binary_writer\n{\n    using string_t = typename BasicJsonType::string_t;\n    using binary_t = typename BasicJsonType::binary_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n\n  public:\n    /*!\n    @brief create a binary writer\n\n    @param[in] adapter  output adapter to write to\n    */\n    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(std::move(adapter))\n    {\n        JSON_ASSERT(oa);\n    }\n\n    /*!\n    @param[in] j  JSON value to serialize\n    @pre       j.type() == value_t::object\n    */\n    void write_bson(const BasicJsonType& j)\n    {\n        switch (j.type())\n        {\n            case value_t::object:\n            {\n                write_bson_object(*j.m_value.object);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::array:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                JSON_THROW(type_error::create(317, concat(\"to serialize to BSON, top-level type must be object, but is \", j.type_name()), &j));\n            }\n        }\n    }\n\n    /*!\n    @param[in] j  JSON value to serialize\n    */\n    void write_cbor(const BasicJsonType& j)\n    {\n        switch (j.type())\n        {\n            case value_t::null:\n            {\n                oa->write_character(to_char_type(0xF6));\n                break;\n            }\n\n            case value_t::boolean:\n            {\n                oa->write_character(j.m_value.boolean\n                                    ? to_char_type(0xF5)\n                                    : to_char_type(0xF4));\n                break;\n            }\n\n            case value_t::number_integer:\n            {\n                if (j.m_value.number_integer >= 0)\n                {\n                    // CBOR does not differentiate between positive signed\n                    // integers and unsigned integers. Therefore, we used the\n                    // code from the value_t::number_unsigned case here.\n                    if (j.m_value.number_integer <= 0x17)\n                    {\n                        write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x18));\n                        write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x19));\n                        write_number(static_cast<std::uint16_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x1A));\n                        write_number(static_cast<std::uint32_t>(j.m_value.number_integer));\n                    }\n                    else\n                    {\n                        oa->write_character(to_char_type(0x1B));\n                        write_number(static_cast<std::uint64_t>(j.m_value.number_integer));\n                    }\n                }\n                else\n                {\n                    // The conversions below encode the sign in the first\n                    // byte, and the value is converted to a positive number.\n                    const auto positive_number = -1 - j.m_value.number_integer;\n                    if (j.m_value.number_integer >= -24)\n                    {\n                        write_number(static_cast<std::uint8_t>(0x20 + positive_number));\n                    }\n                    else if (positive_number <= (std::numeric_limits<std::uint8_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x38));\n                        write_number(static_cast<std::uint8_t>(positive_number));\n                    }\n                    else if (positive_number <= (std::numeric_limits<std::uint16_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x39));\n                        write_number(static_cast<std::uint16_t>(positive_number));\n                    }\n                    else if (positive_number <= (std::numeric_limits<std::uint32_t>::max)())\n                    {\n                        oa->write_character(to_char_type(0x3A));\n                        write_number(static_cast<std::uint32_t>(positive_number));\n                    }\n                    else\n                    {\n                        oa->write_character(to_char_type(0x3B));\n                        write_number(static_cast<std::uint64_t>(positive_number));\n                    }\n                }\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                if (j.m_value.number_unsigned <= 0x17)\n                {\n                    write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x18));\n                    write_number(static_cast<std::uint8_t>(j.m_value.number_unsigned));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x19));\n                    write_number(static_cast<std::uint16_t>(j.m_value.number_unsigned));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x1A));\n                    write_number(static_cast<std::uint32_t>(j.m_value.number_unsigned));\n                }\n                else\n                {\n                    oa->write_character(to_char_type(0x1B));\n                    write_number(static_cast<std::uint64_t>(j.m_value.number_unsigned));\n                }\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                if (std::isnan(j.m_value.number_float))\n                {\n                    // NaN is 0xf97e00 in CBOR\n                    oa->write_character(to_char_type(0xF9));\n                    oa->write_character(to_char_type(0x7E));\n                    oa->write_character(to_char_type(0x00));\n                }\n                else if (std::isinf(j.m_value.number_float))\n                {\n                    // Infinity is 0xf97c00, -Infinity is 0xf9fc00\n                    oa->write_character(to_char_type(0xf9));\n                    oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC));\n                    oa->write_character(to_char_type(0x00));\n                }\n                else\n                {\n                    write_compact_float(j.m_value.number_float, detail::input_format_t::cbor);\n                }\n                break;\n            }\n\n            case value_t::string:\n            {\n                // step 1: write control byte and the string length\n                const auto N = j.m_value.string->size();\n                if (N <= 0x17)\n                {\n                    write_number(static_cast<std::uint8_t>(0x60 + N));\n                }\n                else if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x78));\n                    write_number(static_cast<std::uint8_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x79));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x7A));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n                // LCOV_EXCL_START\n                else if (N <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x7B));\n                    write_number(static_cast<std::uint64_t>(N));\n                }\n                // LCOV_EXCL_STOP\n\n                // step 2: write the string\n                oa->write_characters(\n                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),\n                    j.m_value.string->size());\n                break;\n            }\n\n            case value_t::array:\n            {\n                // step 1: write control byte and the array size\n                const auto N = j.m_value.array->size();\n                if (N <= 0x17)\n                {\n                    write_number(static_cast<std::uint8_t>(0x80 + N));\n                }\n                else if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x98));\n                    write_number(static_cast<std::uint8_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x99));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x9A));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n                // LCOV_EXCL_START\n                else if (N <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x9B));\n                    write_number(static_cast<std::uint64_t>(N));\n                }\n                // LCOV_EXCL_STOP\n\n                // step 2: write each element\n                for (const auto& el : *j.m_value.array)\n                {\n                    write_cbor(el);\n                }\n                break;\n            }\n\n            case value_t::binary:\n            {\n                if (j.m_value.binary->has_subtype())\n                {\n                    if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint8_t>::max)())\n                    {\n                        write_number(static_cast<std::uint8_t>(0xd8));\n                        write_number(static_cast<std::uint8_t>(j.m_value.binary->subtype()));\n                    }\n                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint16_t>::max)())\n                    {\n                        write_number(static_cast<std::uint8_t>(0xd9));\n                        write_number(static_cast<std::uint16_t>(j.m_value.binary->subtype()));\n                    }\n                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint32_t>::max)())\n                    {\n                        write_number(static_cast<std::uint8_t>(0xda));\n                        write_number(static_cast<std::uint32_t>(j.m_value.binary->subtype()));\n                    }\n                    else if (j.m_value.binary->subtype() <= (std::numeric_limits<std::uint64_t>::max)())\n                    {\n                        write_number(static_cast<std::uint8_t>(0xdb));\n                        write_number(static_cast<std::uint64_t>(j.m_value.binary->subtype()));\n                    }\n                }\n\n                // step 1: write control byte and the binary array size\n                const auto N = j.m_value.binary->size();\n                if (N <= 0x17)\n                {\n                    write_number(static_cast<std::uint8_t>(0x40 + N));\n                }\n                else if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x58));\n                    write_number(static_cast<std::uint8_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x59));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x5A));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n                // LCOV_EXCL_START\n                else if (N <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    oa->write_character(to_char_type(0x5B));\n                    write_number(static_cast<std::uint64_t>(N));\n                }\n                // LCOV_EXCL_STOP\n\n                // step 2: write each element\n                oa->write_characters(\n                    reinterpret_cast<const CharType*>(j.m_value.binary->data()),\n                    N);\n\n                break;\n            }\n\n            case value_t::object:\n            {\n                // step 1: write control byte and the object size\n                const auto N = j.m_value.object->size();\n                if (N <= 0x17)\n                {\n                    write_number(static_cast<std::uint8_t>(0xA0 + N));\n                }\n                else if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    oa->write_character(to_char_type(0xB8));\n                    write_number(static_cast<std::uint8_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    oa->write_character(to_char_type(0xB9));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    oa->write_character(to_char_type(0xBA));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n                // LCOV_EXCL_START\n                else if (N <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    oa->write_character(to_char_type(0xBB));\n                    write_number(static_cast<std::uint64_t>(N));\n                }\n                // LCOV_EXCL_STOP\n\n                // step 2: write each element\n                for (const auto& el : *j.m_value.object)\n                {\n                    write_cbor(el.first);\n                    write_cbor(el.second);\n                }\n                break;\n            }\n\n            case value_t::discarded:\n            default:\n                break;\n        }\n    }\n\n    /*!\n    @param[in] j  JSON value to serialize\n    */\n    void write_msgpack(const BasicJsonType& j)\n    {\n        switch (j.type())\n        {\n            case value_t::null: // nil\n            {\n                oa->write_character(to_char_type(0xC0));\n                break;\n            }\n\n            case value_t::boolean: // true and false\n            {\n                oa->write_character(j.m_value.boolean\n                                    ? to_char_type(0xC3)\n                                    : to_char_type(0xC2));\n                break;\n            }\n\n            case value_t::number_integer:\n            {\n                if (j.m_value.number_integer >= 0)\n                {\n                    // MessagePack does not differentiate between positive\n                    // signed integers and unsigned integers. Therefore, we used\n                    // the code from the value_t::number_unsigned case here.\n                    if (j.m_value.number_unsigned < 128)\n                    {\n                        // positive fixnum\n                        write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())\n                    {\n                        // uint 8\n                        oa->write_character(to_char_type(0xCC));\n                        write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())\n                    {\n                        // uint 16\n                        oa->write_character(to_char_type(0xCD));\n                        write_number(static_cast<std::uint16_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())\n                    {\n                        // uint 32\n                        oa->write_character(to_char_type(0xCE));\n                        write_number(static_cast<std::uint32_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())\n                    {\n                        // uint 64\n                        oa->write_character(to_char_type(0xCF));\n                        write_number(static_cast<std::uint64_t>(j.m_value.number_integer));\n                    }\n                }\n                else\n                {\n                    if (j.m_value.number_integer >= -32)\n                    {\n                        // negative fixnum\n                        write_number(static_cast<std::int8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&\n                             j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())\n                    {\n                        // int 8\n                        oa->write_character(to_char_type(0xD0));\n                        write_number(static_cast<std::int8_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&\n                             j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())\n                    {\n                        // int 16\n                        oa->write_character(to_char_type(0xD1));\n                        write_number(static_cast<std::int16_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&\n                             j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())\n                    {\n                        // int 32\n                        oa->write_character(to_char_type(0xD2));\n                        write_number(static_cast<std::int32_t>(j.m_value.number_integer));\n                    }\n                    else if (j.m_value.number_integer >= (std::numeric_limits<std::int64_t>::min)() &&\n                             j.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())\n                    {\n                        // int 64\n                        oa->write_character(to_char_type(0xD3));\n                        write_number(static_cast<std::int64_t>(j.m_value.number_integer));\n                    }\n                }\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                if (j.m_value.number_unsigned < 128)\n                {\n                    // positive fixnum\n                    write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    // uint 8\n                    oa->write_character(to_char_type(0xCC));\n                    write_number(static_cast<std::uint8_t>(j.m_value.number_integer));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    // uint 16\n                    oa->write_character(to_char_type(0xCD));\n                    write_number(static_cast<std::uint16_t>(j.m_value.number_integer));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    // uint 32\n                    oa->write_character(to_char_type(0xCE));\n                    write_number(static_cast<std::uint32_t>(j.m_value.number_integer));\n                }\n                else if (j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    // uint 64\n                    oa->write_character(to_char_type(0xCF));\n                    write_number(static_cast<std::uint64_t>(j.m_value.number_integer));\n                }\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack);\n                break;\n            }\n\n            case value_t::string:\n            {\n                // step 1: write control byte and the string length\n                const auto N = j.m_value.string->size();\n                if (N <= 31)\n                {\n                    // fixstr\n                    write_number(static_cast<std::uint8_t>(0xA0 | N));\n                }\n                else if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    // str 8\n                    oa->write_character(to_char_type(0xD9));\n                    write_number(static_cast<std::uint8_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    // str 16\n                    oa->write_character(to_char_type(0xDA));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    // str 32\n                    oa->write_character(to_char_type(0xDB));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n\n                // step 2: write the string\n                oa->write_characters(\n                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),\n                    j.m_value.string->size());\n                break;\n            }\n\n            case value_t::array:\n            {\n                // step 1: write control byte and the array size\n                const auto N = j.m_value.array->size();\n                if (N <= 15)\n                {\n                    // fixarray\n                    write_number(static_cast<std::uint8_t>(0x90 | N));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    // array 16\n                    oa->write_character(to_char_type(0xDC));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    // array 32\n                    oa->write_character(to_char_type(0xDD));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n\n                // step 2: write each element\n                for (const auto& el : *j.m_value.array)\n                {\n                    write_msgpack(el);\n                }\n                break;\n            }\n\n            case value_t::binary:\n            {\n                // step 0: determine if the binary type has a set subtype to\n                // determine whether or not to use the ext or fixext types\n                const bool use_ext = j.m_value.binary->has_subtype();\n\n                // step 1: write control byte and the byte string length\n                const auto N = j.m_value.binary->size();\n                if (N <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    std::uint8_t output_type{};\n                    bool fixed = true;\n                    if (use_ext)\n                    {\n                        switch (N)\n                        {\n                            case 1:\n                                output_type = 0xD4; // fixext 1\n                                break;\n                            case 2:\n                                output_type = 0xD5; // fixext 2\n                                break;\n                            case 4:\n                                output_type = 0xD6; // fixext 4\n                                break;\n                            case 8:\n                                output_type = 0xD7; // fixext 8\n                                break;\n                            case 16:\n                                output_type = 0xD8; // fixext 16\n                                break;\n                            default:\n                                output_type = 0xC7; // ext 8\n                                fixed = false;\n                                break;\n                        }\n\n                    }\n                    else\n                    {\n                        output_type = 0xC4; // bin 8\n                        fixed = false;\n                    }\n\n                    oa->write_character(to_char_type(output_type));\n                    if (!fixed)\n                    {\n                        write_number(static_cast<std::uint8_t>(N));\n                    }\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    std::uint8_t output_type = use_ext\n                                               ? 0xC8 // ext 16\n                                               : 0xC5; // bin 16\n\n                    oa->write_character(to_char_type(output_type));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    std::uint8_t output_type = use_ext\n                                               ? 0xC9 // ext 32\n                                               : 0xC6; // bin 32\n\n                    oa->write_character(to_char_type(output_type));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n\n                // step 1.5: if this is an ext type, write the subtype\n                if (use_ext)\n                {\n                    write_number(static_cast<std::int8_t>(j.m_value.binary->subtype()));\n                }\n\n                // step 2: write the byte string\n                oa->write_characters(\n                    reinterpret_cast<const CharType*>(j.m_value.binary->data()),\n                    N);\n\n                break;\n            }\n\n            case value_t::object:\n            {\n                // step 1: write control byte and the object size\n                const auto N = j.m_value.object->size();\n                if (N <= 15)\n                {\n                    // fixmap\n                    write_number(static_cast<std::uint8_t>(0x80 | (N & 0xF)));\n                }\n                else if (N <= (std::numeric_limits<std::uint16_t>::max)())\n                {\n                    // map 16\n                    oa->write_character(to_char_type(0xDE));\n                    write_number(static_cast<std::uint16_t>(N));\n                }\n                else if (N <= (std::numeric_limits<std::uint32_t>::max)())\n                {\n                    // map 32\n                    oa->write_character(to_char_type(0xDF));\n                    write_number(static_cast<std::uint32_t>(N));\n                }\n\n                // step 2: write each element\n                for (const auto& el : *j.m_value.object)\n                {\n                    write_msgpack(el.first);\n                    write_msgpack(el.second);\n                }\n                break;\n            }\n\n            case value_t::discarded:\n            default:\n                break;\n        }\n    }\n\n    /*!\n    @param[in] j  JSON value to serialize\n    @param[in] use_count   whether to use '#' prefixes (optimized format)\n    @param[in] use_type    whether to use '$' prefixes (optimized format)\n    @param[in] add_prefix  whether prefixes need to be used for this value\n    @param[in] use_bjdata  whether write in BJData format, default is false\n    */\n    void write_ubjson(const BasicJsonType& j, const bool use_count,\n                      const bool use_type, const bool add_prefix = true,\n                      const bool use_bjdata = false)\n    {\n        switch (j.type())\n        {\n            case value_t::null:\n            {\n                if (add_prefix)\n                {\n                    oa->write_character(to_char_type('Z'));\n                }\n                break;\n            }\n\n            case value_t::boolean:\n            {\n                if (add_prefix)\n                {\n                    oa->write_character(j.m_value.boolean\n                                        ? to_char_type('T')\n                                        : to_char_type('F'));\n                }\n                break;\n            }\n\n            case value_t::number_integer:\n            {\n                write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix, use_bjdata);\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix, use_bjdata);\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix, use_bjdata);\n                break;\n            }\n\n            case value_t::string:\n            {\n                if (add_prefix)\n                {\n                    oa->write_character(to_char_type('S'));\n                }\n                write_number_with_ubjson_prefix(j.m_value.string->size(), true, use_bjdata);\n                oa->write_characters(\n                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),\n                    j.m_value.string->size());\n                break;\n            }\n\n            case value_t::array:\n            {\n                if (add_prefix)\n                {\n                    oa->write_character(to_char_type('['));\n                }\n\n                bool prefix_required = true;\n                if (use_type && !j.m_value.array->empty())\n                {\n                    JSON_ASSERT(use_count);\n                    const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);\n                    const bool same_prefix = std::all_of(j.begin() + 1, j.end(),\n                                                         [this, first_prefix, use_bjdata](const BasicJsonType & v)\n                    {\n                        return ubjson_prefix(v, use_bjdata) == first_prefix;\n                    });\n\n                    std::vector<CharType> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type\n\n                    if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))\n                    {\n                        prefix_required = false;\n                        oa->write_character(to_char_type('$'));\n                        oa->write_character(first_prefix);\n                    }\n                }\n\n                if (use_count)\n                {\n                    oa->write_character(to_char_type('#'));\n                    write_number_with_ubjson_prefix(j.m_value.array->size(), true, use_bjdata);\n                }\n\n                for (const auto& el : *j.m_value.array)\n                {\n                    write_ubjson(el, use_count, use_type, prefix_required, use_bjdata);\n                }\n\n                if (!use_count)\n                {\n                    oa->write_character(to_char_type(']'));\n                }\n\n                break;\n            }\n\n            case value_t::binary:\n            {\n                if (add_prefix)\n                {\n                    oa->write_character(to_char_type('['));\n                }\n\n                if (use_type && !j.m_value.binary->empty())\n                {\n                    JSON_ASSERT(use_count);\n                    oa->write_character(to_char_type('$'));\n                    oa->write_character('U');\n                }\n\n                if (use_count)\n                {\n                    oa->write_character(to_char_type('#'));\n                    write_number_with_ubjson_prefix(j.m_value.binary->size(), true, use_bjdata);\n                }\n\n                if (use_type)\n                {\n                    oa->write_characters(\n                        reinterpret_cast<const CharType*>(j.m_value.binary->data()),\n                        j.m_value.binary->size());\n                }\n                else\n                {\n                    for (size_t i = 0; i < j.m_value.binary->size(); ++i)\n                    {\n                        oa->write_character(to_char_type('U'));\n                        oa->write_character(j.m_value.binary->data()[i]);\n                    }\n                }\n\n                if (!use_count)\n                {\n                    oa->write_character(to_char_type(']'));\n                }\n\n                break;\n            }\n\n            case value_t::object:\n            {\n                if (use_bjdata && j.m_value.object->size() == 3 && j.m_value.object->find(\"_ArrayType_\") != j.m_value.object->end() && j.m_value.object->find(\"_ArraySize_\") != j.m_value.object->end() && j.m_value.object->find(\"_ArrayData_\") != j.m_value.object->end())\n                {\n                    if (!write_bjdata_ndarray(*j.m_value.object, use_count, use_type))  // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata)\n                    {\n                        break;\n                    }\n                }\n\n                if (add_prefix)\n                {\n                    oa->write_character(to_char_type('{'));\n                }\n\n                bool prefix_required = true;\n                if (use_type && !j.m_value.object->empty())\n                {\n                    JSON_ASSERT(use_count);\n                    const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata);\n                    const bool same_prefix = std::all_of(j.begin(), j.end(),\n                                                         [this, first_prefix, use_bjdata](const BasicJsonType & v)\n                    {\n                        return ubjson_prefix(v, use_bjdata) == first_prefix;\n                    });\n\n                    std::vector<CharType> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type\n\n                    if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))\n                    {\n                        prefix_required = false;\n                        oa->write_character(to_char_type('$'));\n                        oa->write_character(first_prefix);\n                    }\n                }\n\n                if (use_count)\n                {\n                    oa->write_character(to_char_type('#'));\n                    write_number_with_ubjson_prefix(j.m_value.object->size(), true, use_bjdata);\n                }\n\n                for (const auto& el : *j.m_value.object)\n                {\n                    write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);\n                    oa->write_characters(\n                        reinterpret_cast<const CharType*>(el.first.c_str()),\n                        el.first.size());\n                    write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata);\n                }\n\n                if (!use_count)\n                {\n                    oa->write_character(to_char_type('}'));\n                }\n\n                break;\n            }\n\n            case value_t::discarded:\n            default:\n                break;\n        }\n    }\n\n  private:\n    //////////\n    // BSON //\n    //////////\n\n    /*!\n    @return The size of a BSON document entry header, including the id marker\n            and the entry name size (and its null-terminator).\n    */\n    static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)\n    {\n        const auto it = name.find(static_cast<typename string_t::value_type>(0));\n        if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos))\n        {\n            JSON_THROW(out_of_range::create(409, concat(\"BSON key cannot contain code point U+0000 (at byte \", std::to_string(it), \")\"), &j));\n            static_cast<void>(j);\n        }\n\n        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;\n    }\n\n    /*!\n    @brief Writes the given @a element_type and @a name to the output adapter\n    */\n    void write_bson_entry_header(const string_t& name,\n                                 const std::uint8_t element_type)\n    {\n        oa->write_character(to_char_type(element_type)); // boolean\n        oa->write_characters(\n            reinterpret_cast<const CharType*>(name.c_str()),\n            name.size() + 1u);\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and boolean value @a value\n    */\n    void write_bson_boolean(const string_t& name,\n                            const bool value)\n    {\n        write_bson_entry_header(name, 0x08);\n        oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00));\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and double value @a value\n    */\n    void write_bson_double(const string_t& name,\n                           const double value)\n    {\n        write_bson_entry_header(name, 0x01);\n        write_number<double>(value, true);\n    }\n\n    /*!\n    @return The size of the BSON-encoded string in @a value\n    */\n    static std::size_t calc_bson_string_size(const string_t& value)\n    {\n        return sizeof(std::int32_t) + value.size() + 1ul;\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and string value @a value\n    */\n    void write_bson_string(const string_t& name,\n                           const string_t& value)\n    {\n        write_bson_entry_header(name, 0x02);\n\n        write_number<std::int32_t>(static_cast<std::int32_t>(value.size() + 1ul), true);\n        oa->write_characters(\n            reinterpret_cast<const CharType*>(value.c_str()),\n            value.size() + 1);\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and null value\n    */\n    void write_bson_null(const string_t& name)\n    {\n        write_bson_entry_header(name, 0x0A);\n    }\n\n    /*!\n    @return The size of the BSON-encoded integer @a value\n    */\n    static std::size_t calc_bson_integer_size(const std::int64_t value)\n    {\n        return (std::numeric_limits<std::int32_t>::min)() <= value && value <= (std::numeric_limits<std::int32_t>::max)()\n               ? sizeof(std::int32_t)\n               : sizeof(std::int64_t);\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and integer @a value\n    */\n    void write_bson_integer(const string_t& name,\n                            const std::int64_t value)\n    {\n        if ((std::numeric_limits<std::int32_t>::min)() <= value && value <= (std::numeric_limits<std::int32_t>::max)())\n        {\n            write_bson_entry_header(name, 0x10); // int32\n            write_number<std::int32_t>(static_cast<std::int32_t>(value), true);\n        }\n        else\n        {\n            write_bson_entry_header(name, 0x12); // int64\n            write_number<std::int64_t>(static_cast<std::int64_t>(value), true);\n        }\n    }\n\n    /*!\n    @return The size of the BSON-encoded unsigned integer in @a j\n    */\n    static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept\n    {\n        return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))\n               ? sizeof(std::int32_t)\n               : sizeof(std::int64_t);\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and unsigned @a value\n    */\n    void write_bson_unsigned(const string_t& name,\n                             const BasicJsonType& j)\n    {\n        if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))\n        {\n            write_bson_entry_header(name, 0x10 /* int32 */);\n            write_number<std::int32_t>(static_cast<std::int32_t>(j.m_value.number_unsigned), true);\n        }\n        else if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))\n        {\n            write_bson_entry_header(name, 0x12 /* int64 */);\n            write_number<std::int64_t>(static_cast<std::int64_t>(j.m_value.number_unsigned), true);\n        }\n        else\n        {\n            JSON_THROW(out_of_range::create(407, concat(\"integer number \", std::to_string(j.m_value.number_unsigned), \" cannot be represented by BSON as it does not fit int64\"), &j));\n        }\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and object @a value\n    */\n    void write_bson_object_entry(const string_t& name,\n                                 const typename BasicJsonType::object_t& value)\n    {\n        write_bson_entry_header(name, 0x03); // object\n        write_bson_object(value);\n    }\n\n    /*!\n    @return The size of the BSON-encoded array @a value\n    */\n    static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value)\n    {\n        std::size_t array_index = 0ul;\n\n        const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast<std::size_t>(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el)\n        {\n            return result + calc_bson_element_size(std::to_string(array_index++), el);\n        });\n\n        return sizeof(std::int32_t) + embedded_document_size + 1ul;\n    }\n\n    /*!\n    @return The size of the BSON-encoded binary array @a value\n    */\n    static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value)\n    {\n        return sizeof(std::int32_t) + value.size() + 1ul;\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and array @a value\n    */\n    void write_bson_array(const string_t& name,\n                          const typename BasicJsonType::array_t& value)\n    {\n        write_bson_entry_header(name, 0x04); // array\n        write_number<std::int32_t>(static_cast<std::int32_t>(calc_bson_array_size(value)), true);\n\n        std::size_t array_index = 0ul;\n\n        for (const auto& el : value)\n        {\n            write_bson_element(std::to_string(array_index++), el);\n        }\n\n        oa->write_character(to_char_type(0x00));\n    }\n\n    /*!\n    @brief Writes a BSON element with key @a name and binary value @a value\n    */\n    void write_bson_binary(const string_t& name,\n                           const binary_t& value)\n    {\n        write_bson_entry_header(name, 0x05);\n\n        write_number<std::int32_t>(static_cast<std::int32_t>(value.size()), true);\n        write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));\n\n        oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());\n    }\n\n    /*!\n    @brief Calculates the size necessary to serialize the JSON value @a j with its @a name\n    @return The calculated size for the BSON document entry for @a j with the given @a name.\n    */\n    static std::size_t calc_bson_element_size(const string_t& name,\n            const BasicJsonType& j)\n    {\n        const auto header_size = calc_bson_entry_header_size(name, j);\n        switch (j.type())\n        {\n            case value_t::object:\n                return header_size + calc_bson_object_size(*j.m_value.object);\n\n            case value_t::array:\n                return header_size + calc_bson_array_size(*j.m_value.array);\n\n            case value_t::binary:\n                return header_size + calc_bson_binary_size(*j.m_value.binary);\n\n            case value_t::boolean:\n                return header_size + 1ul;\n\n            case value_t::number_float:\n                return header_size + 8ul;\n\n            case value_t::number_integer:\n                return header_size + calc_bson_integer_size(j.m_value.number_integer);\n\n            case value_t::number_unsigned:\n                return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned);\n\n            case value_t::string:\n                return header_size + calc_bson_string_size(*j.m_value.string);\n\n            case value_t::null:\n                return header_size + 0ul;\n\n            // LCOV_EXCL_START\n            case value_t::discarded:\n            default:\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)\n                return 0ul;\n                // LCOV_EXCL_STOP\n        }\n    }\n\n    /*!\n    @brief Serializes the JSON value @a j to BSON and associates it with the\n           key @a name.\n    @param name The name to associate with the JSON entity @a j within the\n                current BSON document\n    */\n    void write_bson_element(const string_t& name,\n                            const BasicJsonType& j)\n    {\n        switch (j.type())\n        {\n            case value_t::object:\n                return write_bson_object_entry(name, *j.m_value.object);\n\n            case value_t::array:\n                return write_bson_array(name, *j.m_value.array);\n\n            case value_t::binary:\n                return write_bson_binary(name, *j.m_value.binary);\n\n            case value_t::boolean:\n                return write_bson_boolean(name, j.m_value.boolean);\n\n            case value_t::number_float:\n                return write_bson_double(name, j.m_value.number_float);\n\n            case value_t::number_integer:\n                return write_bson_integer(name, j.m_value.number_integer);\n\n            case value_t::number_unsigned:\n                return write_bson_unsigned(name, j);\n\n            case value_t::string:\n                return write_bson_string(name, *j.m_value.string);\n\n            case value_t::null:\n                return write_bson_null(name);\n\n            // LCOV_EXCL_START\n            case value_t::discarded:\n            default:\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)\n                return;\n                // LCOV_EXCL_STOP\n        }\n    }\n\n    /*!\n    @brief Calculates the size of the BSON serialization of the given\n           JSON-object @a j.\n    @param[in] value  JSON value to serialize\n    @pre       value.type() == value_t::object\n    */\n    static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value)\n    {\n        std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast<std::size_t>(0),\n                                    [](size_t result, const typename BasicJsonType::object_t::value_type & el)\n        {\n            return result += calc_bson_element_size(el.first, el.second);\n        });\n\n        return sizeof(std::int32_t) + document_size + 1ul;\n    }\n\n    /*!\n    @param[in] value  JSON value to serialize\n    @pre       value.type() == value_t::object\n    */\n    void write_bson_object(const typename BasicJsonType::object_t& value)\n    {\n        write_number<std::int32_t>(static_cast<std::int32_t>(calc_bson_object_size(value)), true);\n\n        for (const auto& el : value)\n        {\n            write_bson_element(el.first, el.second);\n        }\n\n        oa->write_character(to_char_type(0x00));\n    }\n\n    //////////\n    // CBOR //\n    //////////\n\n    static constexpr CharType get_cbor_float_prefix(float /*unused*/)\n    {\n        return to_char_type(0xFA);  // Single-Precision Float\n    }\n\n    static constexpr CharType get_cbor_float_prefix(double /*unused*/)\n    {\n        return to_char_type(0xFB);  // Double-Precision Float\n    }\n\n    /////////////\n    // MsgPack //\n    /////////////\n\n    static constexpr CharType get_msgpack_float_prefix(float /*unused*/)\n    {\n        return to_char_type(0xCA);  // float 32\n    }\n\n    static constexpr CharType get_msgpack_float_prefix(double /*unused*/)\n    {\n        return to_char_type(0xCB);  // float 64\n    }\n\n    ////////////\n    // UBJSON //\n    ////////////\n\n    // UBJSON: write number (floating point)\n    template<typename NumberType, typename std::enable_if<\n                 std::is_floating_point<NumberType>::value, int>::type = 0>\n    void write_number_with_ubjson_prefix(const NumberType n,\n                                         const bool add_prefix,\n                                         const bool use_bjdata)\n    {\n        if (add_prefix)\n        {\n            oa->write_character(get_ubjson_float_prefix(n));\n        }\n        write_number(n, use_bjdata);\n    }\n\n    // UBJSON: write number (unsigned integer)\n    template<typename NumberType, typename std::enable_if<\n                 std::is_unsigned<NumberType>::value, int>::type = 0>\n    void write_number_with_ubjson_prefix(const NumberType n,\n                                         const bool add_prefix,\n                                         const bool use_bjdata)\n    {\n        if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('i'));  // int8\n            }\n            write_number(static_cast<std::uint8_t>(n), use_bjdata);\n        }\n        else if (n <= (std::numeric_limits<std::uint8_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('U'));  // uint8\n            }\n            write_number(static_cast<std::uint8_t>(n), use_bjdata);\n        }\n        else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('I'));  // int16\n            }\n            write_number(static_cast<std::int16_t>(n), use_bjdata);\n        }\n        else if (use_bjdata && n <= static_cast<uint64_t>((std::numeric_limits<uint16_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('u'));  // uint16 - bjdata only\n            }\n            write_number(static_cast<std::uint16_t>(n), use_bjdata);\n        }\n        else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('l'));  // int32\n            }\n            write_number(static_cast<std::int32_t>(n), use_bjdata);\n        }\n        else if (use_bjdata && n <= static_cast<uint64_t>((std::numeric_limits<uint32_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('m'));  // uint32 - bjdata only\n            }\n            write_number(static_cast<std::uint32_t>(n), use_bjdata);\n        }\n        else if (n <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('L'));  // int64\n            }\n            write_number(static_cast<std::int64_t>(n), use_bjdata);\n        }\n        else if (use_bjdata && n <= (std::numeric_limits<uint64_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('M'));  // uint64 - bjdata only\n            }\n            write_number(static_cast<std::uint64_t>(n), use_bjdata);\n        }\n        else\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('H'));  // high-precision number\n            }\n\n            const auto number = BasicJsonType(n).dump();\n            write_number_with_ubjson_prefix(number.size(), true, use_bjdata);\n            for (std::size_t i = 0; i < number.size(); ++i)\n            {\n                oa->write_character(to_char_type(static_cast<std::uint8_t>(number[i])));\n            }\n        }\n    }\n\n    // UBJSON: write number (signed integer)\n    template < typename NumberType, typename std::enable_if <\n                   std::is_signed<NumberType>::value&&\n                   !std::is_floating_point<NumberType>::value, int >::type = 0 >\n    void write_number_with_ubjson_prefix(const NumberType n,\n                                         const bool add_prefix,\n                                         const bool use_bjdata)\n    {\n        if ((std::numeric_limits<std::int8_t>::min)() <= n && n <= (std::numeric_limits<std::int8_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('i'));  // int8\n            }\n            write_number(static_cast<std::int8_t>(n), use_bjdata);\n        }\n        else if (static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint8_t>::max)()))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('U'));  // uint8\n            }\n            write_number(static_cast<std::uint8_t>(n), use_bjdata);\n        }\n        else if ((std::numeric_limits<std::int16_t>::min)() <= n && n <= (std::numeric_limits<std::int16_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('I'));  // int16\n            }\n            write_number(static_cast<std::int16_t>(n), use_bjdata);\n        }\n        else if (use_bjdata && (static_cast<std::int64_t>((std::numeric_limits<std::uint16_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint16_t>::max)())))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('u'));  // uint16 - bjdata only\n            }\n            write_number(static_cast<uint16_t>(n), use_bjdata);\n        }\n        else if ((std::numeric_limits<std::int32_t>::min)() <= n && n <= (std::numeric_limits<std::int32_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('l'));  // int32\n            }\n            write_number(static_cast<std::int32_t>(n), use_bjdata);\n        }\n        else if (use_bjdata && (static_cast<std::int64_t>((std::numeric_limits<std::uint32_t>::min)()) <= n && n <= static_cast<std::int64_t>((std::numeric_limits<std::uint32_t>::max)())))\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('m'));  // uint32 - bjdata only\n            }\n            write_number(static_cast<uint32_t>(n), use_bjdata);\n        }\n        else if ((std::numeric_limits<std::int64_t>::min)() <= n && n <= (std::numeric_limits<std::int64_t>::max)())\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('L'));  // int64\n            }\n            write_number(static_cast<std::int64_t>(n), use_bjdata);\n        }\n        // LCOV_EXCL_START\n        else\n        {\n            if (add_prefix)\n            {\n                oa->write_character(to_char_type('H'));  // high-precision number\n            }\n\n            const auto number = BasicJsonType(n).dump();\n            write_number_with_ubjson_prefix(number.size(), true, use_bjdata);\n            for (std::size_t i = 0; i < number.size(); ++i)\n            {\n                oa->write_character(to_char_type(static_cast<std::uint8_t>(number[i])));\n            }\n        }\n        // LCOV_EXCL_STOP\n    }\n\n    /*!\n    @brief determine the type prefix of container values\n    */\n    CharType ubjson_prefix(const BasicJsonType& j, const bool use_bjdata) const noexcept\n    {\n        switch (j.type())\n        {\n            case value_t::null:\n                return 'Z';\n\n            case value_t::boolean:\n                return j.m_value.boolean ? 'T' : 'F';\n\n            case value_t::number_integer:\n            {\n                if ((std::numeric_limits<std::int8_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())\n                {\n                    return 'i';\n                }\n                if ((std::numeric_limits<std::uint8_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint8_t>::max)())\n                {\n                    return 'U';\n                }\n                if ((std::numeric_limits<std::int16_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())\n                {\n                    return 'I';\n                }\n                if (use_bjdata && ((std::numeric_limits<std::uint16_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint16_t>::max)()))\n                {\n                    return 'u';\n                }\n                if ((std::numeric_limits<std::int32_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())\n                {\n                    return 'l';\n                }\n                if (use_bjdata && ((std::numeric_limits<std::uint32_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::uint32_t>::max)()))\n                {\n                    return 'm';\n                }\n                if ((std::numeric_limits<std::int64_t>::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits<std::int64_t>::max)())\n                {\n                    return 'L';\n                }\n                // anything else is treated as high-precision number\n                return 'H'; // LCOV_EXCL_LINE\n            }\n\n            case value_t::number_unsigned:\n            {\n                if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int8_t>::max)()))\n                {\n                    return 'i';\n                }\n                if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint8_t>::max)()))\n                {\n                    return 'U';\n                }\n                if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int16_t>::max)()))\n                {\n                    return 'I';\n                }\n                if (use_bjdata && j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint16_t>::max)()))\n                {\n                    return 'u';\n                }\n                if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)()))\n                {\n                    return 'l';\n                }\n                if (use_bjdata && j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::uint32_t>::max)()))\n                {\n                    return 'm';\n                }\n                if (j.m_value.number_unsigned <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)()))\n                {\n                    return 'L';\n                }\n                if (use_bjdata && j.m_value.number_unsigned <= (std::numeric_limits<std::uint64_t>::max)())\n                {\n                    return 'M';\n                }\n                // anything else is treated as high-precision number\n                return 'H'; // LCOV_EXCL_LINE\n            }\n\n            case value_t::number_float:\n                return get_ubjson_float_prefix(j.m_value.number_float);\n\n            case value_t::string:\n                return 'S';\n\n            case value_t::array: // fallthrough\n            case value_t::binary:\n                return '[';\n\n            case value_t::object:\n                return '{';\n\n            case value_t::discarded:\n            default:  // discarded values\n                return 'N';\n        }\n    }\n\n    static constexpr CharType get_ubjson_float_prefix(float /*unused*/)\n    {\n        return 'd';  // float 32\n    }\n\n    static constexpr CharType get_ubjson_float_prefix(double /*unused*/)\n    {\n        return 'D';  // float 64\n    }\n\n    /*!\n    @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid\n    */\n    bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type)\n    {\n        std::map<string_t, CharType> bjdtype = {{\"uint8\", 'U'},  {\"int8\", 'i'},  {\"uint16\", 'u'}, {\"int16\", 'I'},\n            {\"uint32\", 'm'}, {\"int32\", 'l'}, {\"uint64\", 'M'}, {\"int64\", 'L'}, {\"single\", 'd'}, {\"double\", 'D'}, {\"char\", 'C'}\n        };\n\n        string_t key = \"_ArrayType_\";\n        auto it = bjdtype.find(static_cast<string_t>(value.at(key)));\n        if (it == bjdtype.end())\n        {\n            return true;\n        }\n        CharType dtype = it->second;\n\n        key = \"_ArraySize_\";\n        std::size_t len = (value.at(key).empty() ? 0 : 1);\n        for (const auto& el : value.at(key))\n        {\n            len *= static_cast<std::size_t>(el.m_value.number_unsigned);\n        }\n\n        key = \"_ArrayData_\";\n        if (value.at(key).size() != len)\n        {\n            return true;\n        }\n\n        oa->write_character('[');\n        oa->write_character('$');\n        oa->write_character(dtype);\n        oa->write_character('#');\n\n        key = \"_ArraySize_\";\n        write_ubjson(value.at(key), use_count, use_type, true,  true);\n\n        key = \"_ArrayData_\";\n        if (dtype == 'U' || dtype == 'C')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::uint8_t>(el.m_value.number_unsigned), true);\n            }\n        }\n        else if (dtype == 'i')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::int8_t>(el.m_value.number_integer), true);\n            }\n        }\n        else if (dtype == 'u')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::uint16_t>(el.m_value.number_unsigned), true);\n            }\n        }\n        else if (dtype == 'I')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::int16_t>(el.m_value.number_integer), true);\n            }\n        }\n        else if (dtype == 'm')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::uint32_t>(el.m_value.number_unsigned), true);\n            }\n        }\n        else if (dtype == 'l')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::int32_t>(el.m_value.number_integer), true);\n            }\n        }\n        else if (dtype == 'M')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::uint64_t>(el.m_value.number_unsigned), true);\n            }\n        }\n        else if (dtype == 'L')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<std::int64_t>(el.m_value.number_integer), true);\n            }\n        }\n        else if (dtype == 'd')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<float>(el.m_value.number_float), true);\n            }\n        }\n        else if (dtype == 'D')\n        {\n            for (const auto& el : value.at(key))\n            {\n                write_number(static_cast<double>(el.m_value.number_float), true);\n            }\n        }\n        return false;\n    }\n\n    ///////////////////////\n    // Utility functions //\n    ///////////////////////\n\n    /*\n    @brief write a number to output input\n    @param[in] n number of type @a NumberType\n    @param[in] OutputIsLittleEndian Set to true if output data is\n                                 required to be little endian\n    @tparam NumberType the type of the number\n\n    @note This function needs to respect the system's endianness, because bytes\n          in CBOR, MessagePack, and UBJSON are stored in network order (big\n          endian) and therefore need reordering on little endian systems.\n          On the other hand, BSON and BJData use little endian and should reorder\n          on big endian systems.\n    */\n    template<typename NumberType>\n    void write_number(const NumberType n, const bool OutputIsLittleEndian = false)\n    {\n        // step 1: write number to array of length NumberType\n        std::array<CharType, sizeof(NumberType)> vec{};\n        std::memcpy(vec.data(), &n, sizeof(NumberType));\n\n        // step 2: write array to output (with possible reordering)\n        if (is_little_endian != OutputIsLittleEndian)\n        {\n            // reverse byte order prior to conversion if necessary\n            std::reverse(vec.begin(), vec.end());\n        }\n\n        oa->write_characters(vec.data(), sizeof(NumberType));\n    }\n\n    void write_compact_float(const number_float_t n, detail::input_format_t format)\n    {\n#ifdef __GNUC__\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wfloat-equal\"\n#endif\n        if (static_cast<double>(n) >= static_cast<double>(std::numeric_limits<float>::lowest()) &&\n                static_cast<double>(n) <= static_cast<double>((std::numeric_limits<float>::max)()) &&\n                static_cast<double>(static_cast<float>(n)) == static_cast<double>(n))\n        {\n            oa->write_character(format == detail::input_format_t::cbor\n                                ? get_cbor_float_prefix(static_cast<float>(n))\n                                : get_msgpack_float_prefix(static_cast<float>(n)));\n            write_number(static_cast<float>(n));\n        }\n        else\n        {\n            oa->write_character(format == detail::input_format_t::cbor\n                                ? get_cbor_float_prefix(n)\n                                : get_msgpack_float_prefix(n));\n            write_number(n);\n        }\n#ifdef __GNUC__\n#pragma GCC diagnostic pop\n#endif\n    }\n\n  public:\n    // The following to_char_type functions are implement the conversion\n    // between uint8_t and CharType. In case CharType is not unsigned,\n    // such a conversion is required to allow values greater than 128.\n    // See <https://github.com/nlohmann/json/issues/1286> for a discussion.\n    template < typename C = CharType,\n               enable_if_t < std::is_signed<C>::value && std::is_signed<char>::value > * = nullptr >\n    static constexpr CharType to_char_type(std::uint8_t x) noexcept\n    {\n        return *reinterpret_cast<char*>(&x);\n    }\n\n    template < typename C = CharType,\n               enable_if_t < std::is_signed<C>::value && std::is_unsigned<char>::value > * = nullptr >\n    static CharType to_char_type(std::uint8_t x) noexcept\n    {\n        static_assert(sizeof(std::uint8_t) == sizeof(CharType), \"size of CharType must be equal to std::uint8_t\");\n        static_assert(std::is_trivial<CharType>::value, \"CharType must be trivial\");\n        CharType result;\n        std::memcpy(&result, &x, sizeof(x));\n        return result;\n    }\n\n    template<typename C = CharType,\n             enable_if_t<std::is_unsigned<C>::value>* = nullptr>\n    static constexpr CharType to_char_type(std::uint8_t x) noexcept\n    {\n        return x;\n    }\n\n    template < typename InputCharType, typename C = CharType,\n               enable_if_t <\n                   std::is_signed<C>::value &&\n                   std::is_signed<char>::value &&\n                   std::is_same<char, typename std::remove_cv<InputCharType>::type>::value\n                   > * = nullptr >\n    static constexpr CharType to_char_type(InputCharType x) noexcept\n    {\n        return x;\n    }\n\n  private:\n    /// whether we can assume little endianness\n    const bool is_little_endian = little_endianness();\n\n    /// the output\n    output_adapter_t<CharType> oa = nullptr;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/output/output_adapters.hpp>\n\n// #include <nlohmann/detail/output/serializer.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2008-2009 Björn Hoehrmann <bjoern@hoehrmann.de>\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <algorithm> // reverse, remove, fill, find, none_of\n#include <array> // array\n#include <clocale> // localeconv, lconv\n#include <cmath> // labs, isfinite, isnan, signbit\n#include <cstddef> // size_t, ptrdiff_t\n#include <cstdint> // uint8_t\n#include <cstdio> // snprintf\n#include <limits> // numeric_limits\n#include <string> // string, char_traits\n#include <iomanip> // setfill, setw\n#include <type_traits> // is_same\n#include <utility> // move\n\n// #include <nlohmann/detail/conversions/to_chars.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2009 Florian Loitsch <https://florian.loitsch.com/>\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <array> // array\n#include <cmath>   // signbit, isfinite\n#include <cstdint> // intN_t, uintN_t\n#include <cstring> // memcpy, memmove\n#include <limits> // numeric_limits\n#include <type_traits> // conditional\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n/*!\n@brief implements the Grisu2 algorithm for binary to decimal floating-point\nconversion.\n\nThis implementation is a slightly modified version of the reference\nimplementation which may be obtained from\nhttp://florian.loitsch.com/publications (bench.tar.gz).\n\nThe code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch.\n\nFor a detailed description of the algorithm see:\n\n[1] Loitsch, \"Printing Floating-Point Numbers Quickly and Accurately with\n    Integers\", Proceedings of the ACM SIGPLAN 2010 Conference on Programming\n    Language Design and Implementation, PLDI 2010\n[2] Burger, Dybvig, \"Printing Floating-Point Numbers Quickly and Accurately\",\n    Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language\n    Design and Implementation, PLDI 1996\n*/\nnamespace dtoa_impl\n{\n\ntemplate<typename Target, typename Source>\nTarget reinterpret_bits(const Source source)\n{\n    static_assert(sizeof(Target) == sizeof(Source), \"size mismatch\");\n\n    Target target;\n    std::memcpy(&target, &source, sizeof(Source));\n    return target;\n}\n\nstruct diyfp // f * 2^e\n{\n    static constexpr int kPrecision = 64; // = q\n\n    std::uint64_t f = 0;\n    int e = 0;\n\n    constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {}\n\n    /*!\n    @brief returns x - y\n    @pre x.e == y.e and x.f >= y.f\n    */\n    static diyfp sub(const diyfp& x, const diyfp& y) noexcept\n    {\n        JSON_ASSERT(x.e == y.e);\n        JSON_ASSERT(x.f >= y.f);\n\n        return {x.f - y.f, x.e};\n    }\n\n    /*!\n    @brief returns x * y\n    @note The result is rounded. (Only the upper q bits are returned.)\n    */\n    static diyfp mul(const diyfp& x, const diyfp& y) noexcept\n    {\n        static_assert(kPrecision == 64, \"internal error\");\n\n        // Computes:\n        //  f = round((x.f * y.f) / 2^q)\n        //  e = x.e + y.e + q\n\n        // Emulate the 64-bit * 64-bit multiplication:\n        //\n        // p = u * v\n        //   = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi)\n        //   = (u_lo v_lo         ) + 2^32 ((u_lo v_hi         ) + (u_hi v_lo         )) + 2^64 (u_hi v_hi         )\n        //   = (p0                ) + 2^32 ((p1                ) + (p2                )) + 2^64 (p3                )\n        //   = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3                )\n        //   = (p0_lo             ) + 2^32 (p0_hi + p1_lo + p2_lo                      ) + 2^64 (p1_hi + p2_hi + p3)\n        //   = (p0_lo             ) + 2^32 (Q                                          ) + 2^64 (H                 )\n        //   = (p0_lo             ) + 2^32 (Q_lo + 2^32 Q_hi                           ) + 2^64 (H                 )\n        //\n        // (Since Q might be larger than 2^32 - 1)\n        //\n        //   = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H)\n        //\n        // (Q_hi + H does not overflow a 64-bit int)\n        //\n        //   = p_lo + 2^64 p_hi\n\n        const std::uint64_t u_lo = x.f & 0xFFFFFFFFu;\n        const std::uint64_t u_hi = x.f >> 32u;\n        const std::uint64_t v_lo = y.f & 0xFFFFFFFFu;\n        const std::uint64_t v_hi = y.f >> 32u;\n\n        const std::uint64_t p0 = u_lo * v_lo;\n        const std::uint64_t p1 = u_lo * v_hi;\n        const std::uint64_t p2 = u_hi * v_lo;\n        const std::uint64_t p3 = u_hi * v_hi;\n\n        const std::uint64_t p0_hi = p0 >> 32u;\n        const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu;\n        const std::uint64_t p1_hi = p1 >> 32u;\n        const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu;\n        const std::uint64_t p2_hi = p2 >> 32u;\n\n        std::uint64_t Q = p0_hi + p1_lo + p2_lo;\n\n        // The full product might now be computed as\n        //\n        // p_hi = p3 + p2_hi + p1_hi + (Q >> 32)\n        // p_lo = p0_lo + (Q << 32)\n        //\n        // But in this particular case here, the full p_lo is not required.\n        // Effectively we only need to add the highest bit in p_lo to p_hi (and\n        // Q_hi + 1 does not overflow).\n\n        Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up\n\n        const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u);\n\n        return {h, x.e + y.e + 64};\n    }\n\n    /*!\n    @brief normalize x such that the significand is >= 2^(q-1)\n    @pre x.f != 0\n    */\n    static diyfp normalize(diyfp x) noexcept\n    {\n        JSON_ASSERT(x.f != 0);\n\n        while ((x.f >> 63u) == 0)\n        {\n            x.f <<= 1u;\n            x.e--;\n        }\n\n        return x;\n    }\n\n    /*!\n    @brief normalize x such that the result has the exponent E\n    @pre e >= x.e and the upper e - x.e bits of x.f must be zero.\n    */\n    static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept\n    {\n        const int delta = x.e - target_exponent;\n\n        JSON_ASSERT(delta >= 0);\n        JSON_ASSERT(((x.f << delta) >> delta) == x.f);\n\n        return {x.f << delta, target_exponent};\n    }\n};\n\nstruct boundaries\n{\n    diyfp w;\n    diyfp minus;\n    diyfp plus;\n};\n\n/*!\nCompute the (normalized) diyfp representing the input number 'value' and its\nboundaries.\n\n@pre value must be finite and positive\n*/\ntemplate<typename FloatType>\nboundaries compute_boundaries(FloatType value)\n{\n    JSON_ASSERT(std::isfinite(value));\n    JSON_ASSERT(value > 0);\n\n    // Convert the IEEE representation into a diyfp.\n    //\n    // If v is denormal:\n    //      value = 0.F * 2^(1 - bias) = (          F) * 2^(1 - bias - (p-1))\n    // If v is normalized:\n    //      value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1))\n\n    static_assert(std::numeric_limits<FloatType>::is_iec559,\n                  \"internal error: dtoa_short requires an IEEE-754 floating-point implementation\");\n\n    constexpr int      kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit)\n    constexpr int      kBias      = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1);\n    constexpr int      kMinExp    = 1 - kBias;\n    constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1)\n\n    using bits_type = typename std::conditional<kPrecision == 24, std::uint32_t, std::uint64_t >::type;\n\n    const auto bits = static_cast<std::uint64_t>(reinterpret_bits<bits_type>(value));\n    const std::uint64_t E = bits >> (kPrecision - 1);\n    const std::uint64_t F = bits & (kHiddenBit - 1);\n\n    const bool is_denormal = E == 0;\n    const diyfp v = is_denormal\n                    ? diyfp(F, kMinExp)\n                    : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias);\n\n    // Compute the boundaries m- and m+ of the floating-point value\n    // v = f * 2^e.\n    //\n    // Determine v- and v+, the floating-point predecessor and successor if v,\n    // respectively.\n    //\n    //      v- = v - 2^e        if f != 2^(p-1) or e == e_min                (A)\n    //         = v - 2^(e-1)    if f == 2^(p-1) and e > e_min                (B)\n    //\n    //      v+ = v + 2^e\n    //\n    // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_\n    // between m- and m+ round to v, regardless of how the input rounding\n    // algorithm breaks ties.\n    //\n    //      ---+-------------+-------------+-------------+-------------+---  (A)\n    //         v-            m-            v             m+            v+\n    //\n    //      -----------------+------+------+-------------+-------------+---  (B)\n    //                       v-     m-     v             m+            v+\n\n    const bool lower_boundary_is_closer = F == 0 && E > 1;\n    const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1);\n    const diyfp m_minus = lower_boundary_is_closer\n                          ? diyfp(4 * v.f - 1, v.e - 2)  // (B)\n                          : diyfp(2 * v.f - 1, v.e - 1); // (A)\n\n    // Determine the normalized w+ = m+.\n    const diyfp w_plus = diyfp::normalize(m_plus);\n\n    // Determine w- = m- such that e_(w-) = e_(w+).\n    const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e);\n\n    return {diyfp::normalize(v), w_minus, w_plus};\n}\n\n// Given normalized diyfp w, Grisu needs to find a (normalized) cached\n// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies\n// within a certain range [alpha, gamma] (Definition 3.2 from [1])\n//\n//      alpha <= e = e_c + e_w + q <= gamma\n//\n// or\n//\n//      f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q\n//                          <= f_c * f_w * 2^gamma\n//\n// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies\n//\n//      2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma\n//\n// or\n//\n//      2^(q - 2 + alpha) <= c * w < 2^(q + gamma)\n//\n// The choice of (alpha,gamma) determines the size of the table and the form of\n// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well\n// in practice:\n//\n// The idea is to cut the number c * w = f * 2^e into two parts, which can be\n// processed independently: An integral part p1, and a fractional part p2:\n//\n//      f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e\n//              = (f div 2^-e) + (f mod 2^-e) * 2^e\n//              = p1 + p2 * 2^e\n//\n// The conversion of p1 into decimal form requires a series of divisions and\n// modulos by (a power of) 10. These operations are faster for 32-bit than for\n// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be\n// achieved by choosing\n//\n//      -e >= 32   or   e <= -32 := gamma\n//\n// In order to convert the fractional part\n//\n//      p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ...\n//\n// into decimal form, the fraction is repeatedly multiplied by 10 and the digits\n// d[-i] are extracted in order:\n//\n//      (10 * p2) div 2^-e = d[-1]\n//      (10 * p2) mod 2^-e = d[-2] / 10^1 + ...\n//\n// The multiplication by 10 must not overflow. It is sufficient to choose\n//\n//      10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64.\n//\n// Since p2 = f mod 2^-e < 2^-e,\n//\n//      -e <= 60   or   e >= -60 := alpha\n\nconstexpr int kAlpha = -60;\nconstexpr int kGamma = -32;\n\nstruct cached_power // c = f * 2^e ~= 10^k\n{\n    std::uint64_t f;\n    int e;\n    int k;\n};\n\n/*!\nFor a normalized diyfp w = f * 2^e, this function returns a (normalized) cached\npower-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c\nsatisfies (Definition 3.2 from [1])\n\n     alpha <= e_c + e + q <= gamma.\n*/\ninline cached_power get_cached_power_for_binary_exponent(int e)\n{\n    // Now\n    //\n    //      alpha <= e_c + e + q <= gamma                                    (1)\n    //      ==> f_c * 2^alpha <= c * 2^e * 2^q\n    //\n    // and since the c's are normalized, 2^(q-1) <= f_c,\n    //\n    //      ==> 2^(q - 1 + alpha) <= c * 2^(e + q)\n    //      ==> 2^(alpha - e - 1) <= c\n    //\n    // If c were an exact power of ten, i.e. c = 10^k, one may determine k as\n    //\n    //      k = ceil( log_10( 2^(alpha - e - 1) ) )\n    //        = ceil( (alpha - e - 1) * log_10(2) )\n    //\n    // From the paper:\n    // \"In theory the result of the procedure could be wrong since c is rounded,\n    //  and the computation itself is approximated [...]. In practice, however,\n    //  this simple function is sufficient.\"\n    //\n    // For IEEE double precision floating-point numbers converted into\n    // normalized diyfp's w = f * 2^e, with q = 64,\n    //\n    //      e >= -1022      (min IEEE exponent)\n    //           -52        (p - 1)\n    //           -52        (p - 1, possibly normalize denormal IEEE numbers)\n    //           -11        (normalize the diyfp)\n    //         = -1137\n    //\n    // and\n    //\n    //      e <= +1023      (max IEEE exponent)\n    //           -52        (p - 1)\n    //           -11        (normalize the diyfp)\n    //         = 960\n    //\n    // This binary exponent range [-1137,960] results in a decimal exponent\n    // range [-307,324]. One does not need to store a cached power for each\n    // k in this range. For each such k it suffices to find a cached power\n    // such that the exponent of the product lies in [alpha,gamma].\n    // This implies that the difference of the decimal exponents of adjacent\n    // table entries must be less than or equal to\n    //\n    //      floor( (gamma - alpha) * log_10(2) ) = 8.\n    //\n    // (A smaller distance gamma-alpha would require a larger table.)\n\n    // NB:\n    // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34.\n\n    constexpr int kCachedPowersMinDecExp = -300;\n    constexpr int kCachedPowersDecStep = 8;\n\n    static constexpr std::array<cached_power, 79> kCachedPowers =\n    {\n        {\n            { 0xAB70FE17C79AC6CA, -1060, -300 },\n            { 0xFF77B1FCBEBCDC4F, -1034, -292 },\n            { 0xBE5691EF416BD60C, -1007, -284 },\n            { 0x8DD01FAD907FFC3C,  -980, -276 },\n            { 0xD3515C2831559A83,  -954, -268 },\n            { 0x9D71AC8FADA6C9B5,  -927, -260 },\n            { 0xEA9C227723EE8BCB,  -901, -252 },\n            { 0xAECC49914078536D,  -874, -244 },\n            { 0x823C12795DB6CE57,  -847, -236 },\n            { 0xC21094364DFB5637,  -821, -228 },\n            { 0x9096EA6F3848984F,  -794, -220 },\n            { 0xD77485CB25823AC7,  -768, -212 },\n            { 0xA086CFCD97BF97F4,  -741, -204 },\n            { 0xEF340A98172AACE5,  -715, -196 },\n            { 0xB23867FB2A35B28E,  -688, -188 },\n            { 0x84C8D4DFD2C63F3B,  -661, -180 },\n            { 0xC5DD44271AD3CDBA,  -635, -172 },\n            { 0x936B9FCEBB25C996,  -608, -164 },\n            { 0xDBAC6C247D62A584,  -582, -156 },\n            { 0xA3AB66580D5FDAF6,  -555, -148 },\n            { 0xF3E2F893DEC3F126,  -529, -140 },\n            { 0xB5B5ADA8AAFF80B8,  -502, -132 },\n            { 0x87625F056C7C4A8B,  -475, -124 },\n            { 0xC9BCFF6034C13053,  -449, -116 },\n            { 0x964E858C91BA2655,  -422, -108 },\n            { 0xDFF9772470297EBD,  -396, -100 },\n            { 0xA6DFBD9FB8E5B88F,  -369,  -92 },\n            { 0xF8A95FCF88747D94,  -343,  -84 },\n            { 0xB94470938FA89BCF,  -316,  -76 },\n            { 0x8A08F0F8BF0F156B,  -289,  -68 },\n            { 0xCDB02555653131B6,  -263,  -60 },\n            { 0x993FE2C6D07B7FAC,  -236,  -52 },\n            { 0xE45C10C42A2B3B06,  -210,  -44 },\n            { 0xAA242499697392D3,  -183,  -36 },\n            { 0xFD87B5F28300CA0E,  -157,  -28 },\n            { 0xBCE5086492111AEB,  -130,  -20 },\n            { 0x8CBCCC096F5088CC,  -103,  -12 },\n            { 0xD1B71758E219652C,   -77,   -4 },\n            { 0x9C40000000000000,   -50,    4 },\n            { 0xE8D4A51000000000,   -24,   12 },\n            { 0xAD78EBC5AC620000,     3,   20 },\n            { 0x813F3978F8940984,    30,   28 },\n            { 0xC097CE7BC90715B3,    56,   36 },\n            { 0x8F7E32CE7BEA5C70,    83,   44 },\n            { 0xD5D238A4ABE98068,   109,   52 },\n            { 0x9F4F2726179A2245,   136,   60 },\n            { 0xED63A231D4C4FB27,   162,   68 },\n            { 0xB0DE65388CC8ADA8,   189,   76 },\n            { 0x83C7088E1AAB65DB,   216,   84 },\n            { 0xC45D1DF942711D9A,   242,   92 },\n            { 0x924D692CA61BE758,   269,  100 },\n            { 0xDA01EE641A708DEA,   295,  108 },\n            { 0xA26DA3999AEF774A,   322,  116 },\n            { 0xF209787BB47D6B85,   348,  124 },\n            { 0xB454E4A179DD1877,   375,  132 },\n            { 0x865B86925B9BC5C2,   402,  140 },\n            { 0xC83553C5C8965D3D,   428,  148 },\n            { 0x952AB45CFA97A0B3,   455,  156 },\n            { 0xDE469FBD99A05FE3,   481,  164 },\n            { 0xA59BC234DB398C25,   508,  172 },\n            { 0xF6C69A72A3989F5C,   534,  180 },\n            { 0xB7DCBF5354E9BECE,   561,  188 },\n            { 0x88FCF317F22241E2,   588,  196 },\n            { 0xCC20CE9BD35C78A5,   614,  204 },\n            { 0x98165AF37B2153DF,   641,  212 },\n            { 0xE2A0B5DC971F303A,   667,  220 },\n            { 0xA8D9D1535CE3B396,   694,  228 },\n            { 0xFB9B7CD9A4A7443C,   720,  236 },\n            { 0xBB764C4CA7A44410,   747,  244 },\n            { 0x8BAB8EEFB6409C1A,   774,  252 },\n            { 0xD01FEF10A657842C,   800,  260 },\n            { 0x9B10A4E5E9913129,   827,  268 },\n            { 0xE7109BFBA19C0C9D,   853,  276 },\n            { 0xAC2820D9623BF429,   880,  284 },\n            { 0x80444B5E7AA7CF85,   907,  292 },\n            { 0xBF21E44003ACDD2D,   933,  300 },\n            { 0x8E679C2F5E44FF8F,   960,  308 },\n            { 0xD433179D9C8CB841,   986,  316 },\n            { 0x9E19DB92B4E31BA9,  1013,  324 },\n        }\n    };\n\n    // This computation gives exactly the same results for k as\n    //      k = ceil((kAlpha - e - 1) * 0.30102999566398114)\n    // for |e| <= 1500, but doesn't require floating-point operations.\n    // NB: log_10(2) ~= 78913 / 2^18\n    JSON_ASSERT(e >= -1500);\n    JSON_ASSERT(e <=  1500);\n    const int f = kAlpha - e - 1;\n    const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0);\n\n    const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep;\n    JSON_ASSERT(index >= 0);\n    JSON_ASSERT(static_cast<std::size_t>(index) < kCachedPowers.size());\n\n    const cached_power cached = kCachedPowers[static_cast<std::size_t>(index)];\n    JSON_ASSERT(kAlpha <= cached.e + e + 64);\n    JSON_ASSERT(kGamma >= cached.e + e + 64);\n\n    return cached;\n}\n\n/*!\nFor n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k.\nFor n == 0, returns 1 and sets pow10 := 1.\n*/\ninline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10)\n{\n    // LCOV_EXCL_START\n    if (n >= 1000000000)\n    {\n        pow10 = 1000000000;\n        return 10;\n    }\n    // LCOV_EXCL_STOP\n    if (n >= 100000000)\n    {\n        pow10 = 100000000;\n        return  9;\n    }\n    if (n >= 10000000)\n    {\n        pow10 = 10000000;\n        return  8;\n    }\n    if (n >= 1000000)\n    {\n        pow10 = 1000000;\n        return  7;\n    }\n    if (n >= 100000)\n    {\n        pow10 = 100000;\n        return  6;\n    }\n    if (n >= 10000)\n    {\n        pow10 = 10000;\n        return  5;\n    }\n    if (n >= 1000)\n    {\n        pow10 = 1000;\n        return  4;\n    }\n    if (n >= 100)\n    {\n        pow10 = 100;\n        return  3;\n    }\n    if (n >= 10)\n    {\n        pow10 = 10;\n        return  2;\n    }\n\n    pow10 = 1;\n    return 1;\n}\n\ninline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta,\n                         std::uint64_t rest, std::uint64_t ten_k)\n{\n    JSON_ASSERT(len >= 1);\n    JSON_ASSERT(dist <= delta);\n    JSON_ASSERT(rest <= delta);\n    JSON_ASSERT(ten_k > 0);\n\n    //               <--------------------------- delta ---->\n    //                                  <---- dist --------->\n    // --------------[------------------+-------------------]--------------\n    //               M-                 w                   M+\n    //\n    //                                  ten_k\n    //                                <------>\n    //                                       <---- rest ---->\n    // --------------[------------------+----+--------------]--------------\n    //                                  w    V\n    //                                       = buf * 10^k\n    //\n    // ten_k represents a unit-in-the-last-place in the decimal representation\n    // stored in buf.\n    // Decrement buf by ten_k while this takes buf closer to w.\n\n    // The tests are written in this order to avoid overflow in unsigned\n    // integer arithmetic.\n\n    while (rest < dist\n            && delta - rest >= ten_k\n            && (rest + ten_k < dist || dist - rest > rest + ten_k - dist))\n    {\n        JSON_ASSERT(buf[len - 1] != '0');\n        buf[len - 1]--;\n        rest += ten_k;\n    }\n}\n\n/*!\nGenerates V = buffer * 10^decimal_exponent, such that M- <= V <= M+.\nM- and M+ must be normalized and share the same exponent -60 <= e <= -32.\n*/\ninline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent,\n                             diyfp M_minus, diyfp w, diyfp M_plus)\n{\n    static_assert(kAlpha >= -60, \"internal error\");\n    static_assert(kGamma <= -32, \"internal error\");\n\n    // Generates the digits (and the exponent) of a decimal floating-point\n    // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's\n    // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma.\n    //\n    //               <--------------------------- delta ---->\n    //                                  <---- dist --------->\n    // --------------[------------------+-------------------]--------------\n    //               M-                 w                   M+\n    //\n    // Grisu2 generates the digits of M+ from left to right and stops as soon as\n    // V is in [M-,M+].\n\n    JSON_ASSERT(M_plus.e >= kAlpha);\n    JSON_ASSERT(M_plus.e <= kGamma);\n\n    std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e)\n    std::uint64_t dist  = diyfp::sub(M_plus, w      ).f; // (significand of (M+ - w ), implicit exponent is e)\n\n    // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0):\n    //\n    //      M+ = f * 2^e\n    //         = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e\n    //         = ((p1        ) * 2^-e + (p2        )) * 2^e\n    //         = p1 + p2 * 2^e\n\n    const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e);\n\n    auto p1 = static_cast<std::uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.)\n    std::uint64_t p2 = M_plus.f & (one.f - 1);                    // p2 = f mod 2^-e\n\n    // 1)\n    //\n    // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0]\n\n    JSON_ASSERT(p1 > 0);\n\n    std::uint32_t pow10{};\n    const int k = find_largest_pow10(p1, pow10);\n\n    //      10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1)\n    //\n    //      p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1))\n    //         = (d[k-1]         ) * 10^(k-1) + (p1 mod 10^(k-1))\n    //\n    //      M+ = p1                                             + p2 * 2^e\n    //         = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1))          + p2 * 2^e\n    //         = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e\n    //         = d[k-1] * 10^(k-1) + (                         rest) * 2^e\n    //\n    // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0)\n    //\n    //      p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0]\n    //\n    // but stop as soon as\n    //\n    //      rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e\n\n    int n = k;\n    while (n > 0)\n    {\n        // Invariants:\n        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)    (buffer = 0 for n = k)\n        //      pow10 = 10^(n-1) <= p1 < 10^n\n        //\n        const std::uint32_t d = p1 / pow10;  // d = p1 div 10^(n-1)\n        const std::uint32_t r = p1 % pow10;  // r = p1 mod 10^(n-1)\n        //\n        //      M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e\n        //         = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e)\n        //\n        JSON_ASSERT(d <= 9);\n        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d\n        //\n        //      M+ = buffer * 10^(n-1) + (r + p2 * 2^e)\n        //\n        p1 = r;\n        n--;\n        //\n        //      M+ = buffer * 10^n + (p1 + p2 * 2^e)\n        //      pow10 = 10^n\n        //\n\n        // Now check if enough digits have been generated.\n        // Compute\n        //\n        //      p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e\n        //\n        // Note:\n        // Since rest and delta share the same exponent e, it suffices to\n        // compare the significands.\n        const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2;\n        if (rest <= delta)\n        {\n            // V = buffer * 10^n, with M- <= V <= M+.\n\n            decimal_exponent += n;\n\n            // We may now just stop. But instead look if the buffer could be\n            // decremented to bring V closer to w.\n            //\n            // pow10 = 10^n is now 1 ulp in the decimal representation V.\n            // The rounding procedure works with diyfp's with an implicit\n            // exponent of e.\n            //\n            //      10^n = (10^n * 2^-e) * 2^e = ulp * 2^e\n            //\n            const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e;\n            grisu2_round(buffer, length, dist, delta, rest, ten_n);\n\n            return;\n        }\n\n        pow10 /= 10;\n        //\n        //      pow10 = 10^(n-1) <= p1 < 10^n\n        // Invariants restored.\n    }\n\n    // 2)\n    //\n    // The digits of the integral part have been generated:\n    //\n    //      M+ = d[k-1]...d[1]d[0] + p2 * 2^e\n    //         = buffer            + p2 * 2^e\n    //\n    // Now generate the digits of the fractional part p2 * 2^e.\n    //\n    // Note:\n    // No decimal point is generated: the exponent is adjusted instead.\n    //\n    // p2 actually represents the fraction\n    //\n    //      p2 * 2^e\n    //          = p2 / 2^-e\n    //          = d[-1] / 10^1 + d[-2] / 10^2 + ...\n    //\n    // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...)\n    //\n    //      p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m\n    //                      + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...)\n    //\n    // using\n    //\n    //      10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e)\n    //                = (                   d) * 2^-e + (                   r)\n    //\n    // or\n    //      10^m * p2 * 2^e = d + r * 2^e\n    //\n    // i.e.\n    //\n    //      M+ = buffer + p2 * 2^e\n    //         = buffer + 10^-m * (d + r * 2^e)\n    //         = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e\n    //\n    // and stop as soon as 10^-m * r * 2^e <= delta * 2^e\n\n    JSON_ASSERT(p2 > delta);\n\n    int m = 0;\n    for (;;)\n    {\n        // Invariant:\n        //      M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e\n        //         = buffer * 10^-m + 10^-m * (p2                                 ) * 2^e\n        //         = buffer * 10^-m + 10^-m * (1/10 * (10 * p2)                   ) * 2^e\n        //         = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e\n        //\n        JSON_ASSERT(p2 <= (std::numeric_limits<std::uint64_t>::max)() / 10);\n        p2 *= 10;\n        const std::uint64_t d = p2 >> -one.e;     // d = (10 * p2) div 2^-e\n        const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e\n        //\n        //      M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e\n        //         = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e))\n        //         = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e\n        //\n        JSON_ASSERT(d <= 9);\n        buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d\n        //\n        //      M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e\n        //\n        p2 = r;\n        m++;\n        //\n        //      M+ = buffer * 10^-m + 10^-m * p2 * 2^e\n        // Invariant restored.\n\n        // Check if enough digits have been generated.\n        //\n        //      10^-m * p2 * 2^e <= delta * 2^e\n        //              p2 * 2^e <= 10^m * delta * 2^e\n        //                    p2 <= 10^m * delta\n        delta *= 10;\n        dist  *= 10;\n        if (p2 <= delta)\n        {\n            break;\n        }\n    }\n\n    // V = buffer * 10^-m, with M- <= V <= M+.\n\n    decimal_exponent -= m;\n\n    // 1 ulp in the decimal representation is now 10^-m.\n    // Since delta and dist are now scaled by 10^m, we need to do the\n    // same with ulp in order to keep the units in sync.\n    //\n    //      10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e\n    //\n    const std::uint64_t ten_m = one.f;\n    grisu2_round(buffer, length, dist, delta, p2, ten_m);\n\n    // By construction this algorithm generates the shortest possible decimal\n    // number (Loitsch, Theorem 6.2) which rounds back to w.\n    // For an input number of precision p, at least\n    //\n    //      N = 1 + ceil(p * log_10(2))\n    //\n    // decimal digits are sufficient to identify all binary floating-point\n    // numbers (Matula, \"In-and-Out conversions\").\n    // This implies that the algorithm does not produce more than N decimal\n    // digits.\n    //\n    //      N = 17 for p = 53 (IEEE double precision)\n    //      N = 9  for p = 24 (IEEE single precision)\n}\n\n/*!\nv = buf * 10^decimal_exponent\nlen is the length of the buffer (number of decimal digits)\nThe buffer must be large enough, i.e. >= max_digits10.\n*/\nJSON_HEDLEY_NON_NULL(1)\ninline void grisu2(char* buf, int& len, int& decimal_exponent,\n                   diyfp m_minus, diyfp v, diyfp m_plus)\n{\n    JSON_ASSERT(m_plus.e == m_minus.e);\n    JSON_ASSERT(m_plus.e == v.e);\n\n    //  --------(-----------------------+-----------------------)--------    (A)\n    //          m-                      v                       m+\n    //\n    //  --------------------(-----------+-----------------------)--------    (B)\n    //                      m-          v                       m+\n    //\n    // First scale v (and m- and m+) such that the exponent is in the range\n    // [alpha, gamma].\n\n    const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e);\n\n    const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k\n\n    // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma]\n    const diyfp w       = diyfp::mul(v,       c_minus_k);\n    const diyfp w_minus = diyfp::mul(m_minus, c_minus_k);\n    const diyfp w_plus  = diyfp::mul(m_plus,  c_minus_k);\n\n    //  ----(---+---)---------------(---+---)---------------(---+---)----\n    //          w-                      w                       w+\n    //          = c*m-                  = c*v                   = c*m+\n    //\n    // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and\n    // w+ are now off by a small amount.\n    // In fact:\n    //\n    //      w - v * 10^k < 1 ulp\n    //\n    // To account for this inaccuracy, add resp. subtract 1 ulp.\n    //\n    //  --------+---[---------------(---+---)---------------]---+--------\n    //          w-  M-                  w                   M+  w+\n    //\n    // Now any number in [M-, M+] (bounds included) will round to w when input,\n    // regardless of how the input rounding algorithm breaks ties.\n    //\n    // And digit_gen generates the shortest possible such number in [M-, M+].\n    // Note that this does not mean that Grisu2 always generates the shortest\n    // possible number in the interval (m-, m+).\n    const diyfp M_minus(w_minus.f + 1, w_minus.e);\n    const diyfp M_plus (w_plus.f  - 1, w_plus.e );\n\n    decimal_exponent = -cached.k; // = -(-k) = k\n\n    grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus);\n}\n\n/*!\nv = buf * 10^decimal_exponent\nlen is the length of the buffer (number of decimal digits)\nThe buffer must be large enough, i.e. >= max_digits10.\n*/\ntemplate<typename FloatType>\nJSON_HEDLEY_NON_NULL(1)\nvoid grisu2(char* buf, int& len, int& decimal_exponent, FloatType value)\n{\n    static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3,\n                  \"internal error: not enough precision\");\n\n    JSON_ASSERT(std::isfinite(value));\n    JSON_ASSERT(value > 0);\n\n    // If the neighbors (and boundaries) of 'value' are always computed for double-precision\n    // numbers, all float's can be recovered using strtod (and strtof). However, the resulting\n    // decimal representations are not exactly \"short\".\n    //\n    // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars)\n    // says \"value is converted to a string as if by std::sprintf in the default (\"C\") locale\"\n    // and since sprintf promotes floats to doubles, I think this is exactly what 'std::to_chars'\n    // does.\n    // On the other hand, the documentation for 'std::to_chars' requires that \"parsing the\n    // representation using the corresponding std::from_chars function recovers value exactly\". That\n    // indicates that single precision floating-point numbers should be recovered using\n    // 'std::strtof'.\n    //\n    // NB: If the neighbors are computed for single-precision numbers, there is a single float\n    //     (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision\n    //     value is off by 1 ulp.\n#if 0\n    const boundaries w = compute_boundaries(static_cast<double>(value));\n#else\n    const boundaries w = compute_boundaries(value);\n#endif\n\n    grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus);\n}\n\n/*!\n@brief appends a decimal representation of e to buf\n@return a pointer to the element following the exponent.\n@pre -1000 < e < 1000\n*/\nJSON_HEDLEY_NON_NULL(1)\nJSON_HEDLEY_RETURNS_NON_NULL\ninline char* append_exponent(char* buf, int e)\n{\n    JSON_ASSERT(e > -1000);\n    JSON_ASSERT(e <  1000);\n\n    if (e < 0)\n    {\n        e = -e;\n        *buf++ = '-';\n    }\n    else\n    {\n        *buf++ = '+';\n    }\n\n    auto k = static_cast<std::uint32_t>(e);\n    if (k < 10)\n    {\n        // Always print at least two digits in the exponent.\n        // This is for compatibility with printf(\"%g\").\n        *buf++ = '0';\n        *buf++ = static_cast<char>('0' + k);\n    }\n    else if (k < 100)\n    {\n        *buf++ = static_cast<char>('0' + k / 10);\n        k %= 10;\n        *buf++ = static_cast<char>('0' + k);\n    }\n    else\n    {\n        *buf++ = static_cast<char>('0' + k / 100);\n        k %= 100;\n        *buf++ = static_cast<char>('0' + k / 10);\n        k %= 10;\n        *buf++ = static_cast<char>('0' + k);\n    }\n\n    return buf;\n}\n\n/*!\n@brief prettify v = buf * 10^decimal_exponent\n\nIf v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point\nnotation. Otherwise it will be printed in exponential notation.\n\n@pre min_exp < 0\n@pre max_exp > 0\n*/\nJSON_HEDLEY_NON_NULL(1)\nJSON_HEDLEY_RETURNS_NON_NULL\ninline char* format_buffer(char* buf, int len, int decimal_exponent,\n                           int min_exp, int max_exp)\n{\n    JSON_ASSERT(min_exp < 0);\n    JSON_ASSERT(max_exp > 0);\n\n    const int k = len;\n    const int n = len + decimal_exponent;\n\n    // v = buf * 10^(n-k)\n    // k is the length of the buffer (number of decimal digits)\n    // n is the position of the decimal point relative to the start of the buffer.\n\n    if (k <= n && n <= max_exp)\n    {\n        // digits[000]\n        // len <= max_exp + 2\n\n        std::memset(buf + k, '0', static_cast<size_t>(n) - static_cast<size_t>(k));\n        // Make it look like a floating-point number (#362, #378)\n        buf[n + 0] = '.';\n        buf[n + 1] = '0';\n        return buf + (static_cast<size_t>(n) + 2);\n    }\n\n    if (0 < n && n <= max_exp)\n    {\n        // dig.its\n        // len <= max_digits10 + 1\n\n        JSON_ASSERT(k > n);\n\n        std::memmove(buf + (static_cast<size_t>(n) + 1), buf + n, static_cast<size_t>(k) - static_cast<size_t>(n));\n        buf[n] = '.';\n        return buf + (static_cast<size_t>(k) + 1U);\n    }\n\n    if (min_exp < n && n <= 0)\n    {\n        // 0.[000]digits\n        // len <= 2 + (-min_exp - 1) + max_digits10\n\n        std::memmove(buf + (2 + static_cast<size_t>(-n)), buf, static_cast<size_t>(k));\n        buf[0] = '0';\n        buf[1] = '.';\n        std::memset(buf + 2, '0', static_cast<size_t>(-n));\n        return buf + (2U + static_cast<size_t>(-n) + static_cast<size_t>(k));\n    }\n\n    if (k == 1)\n    {\n        // dE+123\n        // len <= 1 + 5\n\n        buf += 1;\n    }\n    else\n    {\n        // d.igitsE+123\n        // len <= max_digits10 + 1 + 5\n\n        std::memmove(buf + 2, buf + 1, static_cast<size_t>(k) - 1);\n        buf[1] = '.';\n        buf += 1 + static_cast<size_t>(k);\n    }\n\n    *buf++ = 'e';\n    return append_exponent(buf, n - 1);\n}\n\n}  // namespace dtoa_impl\n\n/*!\n@brief generates a decimal representation of the floating-point number value in [first, last).\n\nThe format of the resulting decimal representation is similar to printf's %g\nformat. Returns an iterator pointing past-the-end of the decimal representation.\n\n@note The input number must be finite, i.e. NaN's and Inf's are not supported.\n@note The buffer must be large enough.\n@note The result is NOT null-terminated.\n*/\ntemplate<typename FloatType>\nJSON_HEDLEY_NON_NULL(1, 2)\nJSON_HEDLEY_RETURNS_NON_NULL\nchar* to_chars(char* first, const char* last, FloatType value)\n{\n    static_cast<void>(last); // maybe unused - fix warning\n    JSON_ASSERT(std::isfinite(value));\n\n    // Use signbit(value) instead of (value < 0) since signbit works for -0.\n    if (std::signbit(value))\n    {\n        value = -value;\n        *first++ = '-';\n    }\n\n#ifdef __GNUC__\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wfloat-equal\"\n#endif\n    if (value == 0) // +-0\n    {\n        *first++ = '0';\n        // Make it look like a floating-point number (#362, #378)\n        *first++ = '.';\n        *first++ = '0';\n        return first;\n    }\n#ifdef __GNUC__\n#pragma GCC diagnostic pop\n#endif\n\n    JSON_ASSERT(last - first >= std::numeric_limits<FloatType>::max_digits10);\n\n    // Compute v = buffer * 10^decimal_exponent.\n    // The decimal digits are stored in the buffer, which needs to be interpreted\n    // as an unsigned decimal integer.\n    // len is the length of the buffer, i.e. the number of decimal digits.\n    int len = 0;\n    int decimal_exponent = 0;\n    dtoa_impl::grisu2(first, len, decimal_exponent, value);\n\n    JSON_ASSERT(len <= std::numeric_limits<FloatType>::max_digits10);\n\n    // Format the buffer like printf(\"%.*g\", prec, value)\n    constexpr int kMinExp = -4;\n    // Use digits10 here to increase compatibility with version 2.\n    constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10;\n\n    JSON_ASSERT(last - first >= kMaxExp + 2);\n    JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10);\n    JSON_ASSERT(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6);\n\n    return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp);\n}\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/exceptions.hpp>\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/cpp_future.hpp>\n\n// #include <nlohmann/detail/output/binary_writer.hpp>\n\n// #include <nlohmann/detail/output/output_adapters.hpp>\n\n// #include <nlohmann/detail/string_concat.hpp>\n\n// #include <nlohmann/detail/value_t.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\nnamespace detail\n{\n\n///////////////////\n// serialization //\n///////////////////\n\n/// how to treat decoding errors\nenum class error_handler_t\n{\n    strict,  ///< throw a type_error exception in case of invalid UTF-8\n    replace, ///< replace invalid UTF-8 sequences with U+FFFD\n    ignore   ///< ignore invalid UTF-8 sequences\n};\n\ntemplate<typename BasicJsonType>\nclass serializer\n{\n    using string_t = typename BasicJsonType::string_t;\n    using number_float_t = typename BasicJsonType::number_float_t;\n    using number_integer_t = typename BasicJsonType::number_integer_t;\n    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n    using binary_char_t = typename BasicJsonType::binary_t::value_type;\n    static constexpr std::uint8_t UTF8_ACCEPT = 0;\n    static constexpr std::uint8_t UTF8_REJECT = 1;\n\n  public:\n    /*!\n    @param[in] s  output stream to serialize to\n    @param[in] ichar  indentation character to use\n    @param[in] error_handler_  how to react on decoding errors\n    */\n    serializer(output_adapter_t<char> s, const char ichar,\n               error_handler_t error_handler_ = error_handler_t::strict)\n        : o(std::move(s))\n        , loc(std::localeconv())\n        , thousands_sep(loc->thousands_sep == nullptr ? '\\0' : std::char_traits<char>::to_char_type(* (loc->thousands_sep)))\n        , decimal_point(loc->decimal_point == nullptr ? '\\0' : std::char_traits<char>::to_char_type(* (loc->decimal_point)))\n        , indent_char(ichar)\n        , indent_string(512, indent_char)\n        , error_handler(error_handler_)\n    {}\n\n    // delete because of pointer members\n    serializer(const serializer&) = delete;\n    serializer& operator=(const serializer&) = delete;\n    serializer(serializer&&) = delete;\n    serializer& operator=(serializer&&) = delete;\n    ~serializer() = default;\n\n    /*!\n    @brief internal implementation of the serialization function\n\n    This function is called by the public member function dump and organizes\n    the serialization internally. The indentation level is propagated as\n    additional parameter. In case of arrays and objects, the function is\n    called recursively.\n\n    - strings and object keys are escaped using `escape_string()`\n    - integer numbers are converted implicitly via `operator<<`\n    - floating-point numbers are converted to a string using `\"%g\"` format\n    - binary values are serialized as objects containing the subtype and the\n      byte array\n\n    @param[in] val               value to serialize\n    @param[in] pretty_print      whether the output shall be pretty-printed\n    @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters\n    in the output are escaped with `\\uXXXX` sequences, and the result consists\n    of ASCII characters only.\n    @param[in] indent_step       the indent level\n    @param[in] current_indent    the current indent level (only used internally)\n    */\n    void dump(const BasicJsonType& val,\n              const bool pretty_print,\n              const bool ensure_ascii,\n              const unsigned int indent_step,\n              const unsigned int current_indent = 0)\n    {\n        switch (val.m_type)\n        {\n            case value_t::object:\n            {\n                if (val.m_value.object->empty())\n                {\n                    o->write_characters(\"{}\", 2);\n                    return;\n                }\n\n                if (pretty_print)\n                {\n                    o->write_characters(\"{\\n\", 2);\n\n                    // variable to hold indentation for recursive calls\n                    const auto new_indent = current_indent + indent_step;\n                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))\n                    {\n                        indent_string.resize(indent_string.size() * 2, ' ');\n                    }\n\n                    // first n-1 elements\n                    auto i = val.m_value.object->cbegin();\n                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)\n                    {\n                        o->write_characters(indent_string.c_str(), new_indent);\n                        o->write_character('\\\"');\n                        dump_escaped(i->first, ensure_ascii);\n                        o->write_characters(\"\\\": \", 3);\n                        dump(i->second, true, ensure_ascii, indent_step, new_indent);\n                        o->write_characters(\",\\n\", 2);\n                    }\n\n                    // last element\n                    JSON_ASSERT(i != val.m_value.object->cend());\n                    JSON_ASSERT(std::next(i) == val.m_value.object->cend());\n                    o->write_characters(indent_string.c_str(), new_indent);\n                    o->write_character('\\\"');\n                    dump_escaped(i->first, ensure_ascii);\n                    o->write_characters(\"\\\": \", 3);\n                    dump(i->second, true, ensure_ascii, indent_step, new_indent);\n\n                    o->write_character('\\n');\n                    o->write_characters(indent_string.c_str(), current_indent);\n                    o->write_character('}');\n                }\n                else\n                {\n                    o->write_character('{');\n\n                    // first n-1 elements\n                    auto i = val.m_value.object->cbegin();\n                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)\n                    {\n                        o->write_character('\\\"');\n                        dump_escaped(i->first, ensure_ascii);\n                        o->write_characters(\"\\\":\", 2);\n                        dump(i->second, false, ensure_ascii, indent_step, current_indent);\n                        o->write_character(',');\n                    }\n\n                    // last element\n                    JSON_ASSERT(i != val.m_value.object->cend());\n                    JSON_ASSERT(std::next(i) == val.m_value.object->cend());\n                    o->write_character('\\\"');\n                    dump_escaped(i->first, ensure_ascii);\n                    o->write_characters(\"\\\":\", 2);\n                    dump(i->second, false, ensure_ascii, indent_step, current_indent);\n\n                    o->write_character('}');\n                }\n\n                return;\n            }\n\n            case value_t::array:\n            {\n                if (val.m_value.array->empty())\n                {\n                    o->write_characters(\"[]\", 2);\n                    return;\n                }\n\n                if (pretty_print)\n                {\n                    o->write_characters(\"[\\n\", 2);\n\n                    // variable to hold indentation for recursive calls\n                    const auto new_indent = current_indent + indent_step;\n                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))\n                    {\n                        indent_string.resize(indent_string.size() * 2, ' ');\n                    }\n\n                    // first n-1 elements\n                    for (auto i = val.m_value.array->cbegin();\n                            i != val.m_value.array->cend() - 1; ++i)\n                    {\n                        o->write_characters(indent_string.c_str(), new_indent);\n                        dump(*i, true, ensure_ascii, indent_step, new_indent);\n                        o->write_characters(\",\\n\", 2);\n                    }\n\n                    // last element\n                    JSON_ASSERT(!val.m_value.array->empty());\n                    o->write_characters(indent_string.c_str(), new_indent);\n                    dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);\n\n                    o->write_character('\\n');\n                    o->write_characters(indent_string.c_str(), current_indent);\n                    o->write_character(']');\n                }\n                else\n                {\n                    o->write_character('[');\n\n                    // first n-1 elements\n                    for (auto i = val.m_value.array->cbegin();\n                            i != val.m_value.array->cend() - 1; ++i)\n                    {\n                        dump(*i, false, ensure_ascii, indent_step, current_indent);\n                        o->write_character(',');\n                    }\n\n                    // last element\n                    JSON_ASSERT(!val.m_value.array->empty());\n                    dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);\n\n                    o->write_character(']');\n                }\n\n                return;\n            }\n\n            case value_t::string:\n            {\n                o->write_character('\\\"');\n                dump_escaped(*val.m_value.string, ensure_ascii);\n                o->write_character('\\\"');\n                return;\n            }\n\n            case value_t::binary:\n            {\n                if (pretty_print)\n                {\n                    o->write_characters(\"{\\n\", 2);\n\n                    // variable to hold indentation for recursive calls\n                    const auto new_indent = current_indent + indent_step;\n                    if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))\n                    {\n                        indent_string.resize(indent_string.size() * 2, ' ');\n                    }\n\n                    o->write_characters(indent_string.c_str(), new_indent);\n\n                    o->write_characters(\"\\\"bytes\\\": [\", 10);\n\n                    if (!val.m_value.binary->empty())\n                    {\n                        for (auto i = val.m_value.binary->cbegin();\n                                i != val.m_value.binary->cend() - 1; ++i)\n                        {\n                            dump_integer(*i);\n                            o->write_characters(\", \", 2);\n                        }\n                        dump_integer(val.m_value.binary->back());\n                    }\n\n                    o->write_characters(\"],\\n\", 3);\n                    o->write_characters(indent_string.c_str(), new_indent);\n\n                    o->write_characters(\"\\\"subtype\\\": \", 11);\n                    if (val.m_value.binary->has_subtype())\n                    {\n                        dump_integer(val.m_value.binary->subtype());\n                    }\n                    else\n                    {\n                        o->write_characters(\"null\", 4);\n                    }\n                    o->write_character('\\n');\n                    o->write_characters(indent_string.c_str(), current_indent);\n                    o->write_character('}');\n                }\n                else\n                {\n                    o->write_characters(\"{\\\"bytes\\\":[\", 10);\n\n                    if (!val.m_value.binary->empty())\n                    {\n                        for (auto i = val.m_value.binary->cbegin();\n                                i != val.m_value.binary->cend() - 1; ++i)\n                        {\n                            dump_integer(*i);\n                            o->write_character(',');\n                        }\n                        dump_integer(val.m_value.binary->back());\n                    }\n\n                    o->write_characters(\"],\\\"subtype\\\":\", 12);\n                    if (val.m_value.binary->has_subtype())\n                    {\n                        dump_integer(val.m_value.binary->subtype());\n                        o->write_character('}');\n                    }\n                    else\n                    {\n                        o->write_characters(\"null}\", 5);\n                    }\n                }\n                return;\n            }\n\n            case value_t::boolean:\n            {\n                if (val.m_value.boolean)\n                {\n                    o->write_characters(\"true\", 4);\n                }\n                else\n                {\n                    o->write_characters(\"false\", 5);\n                }\n                return;\n            }\n\n            case value_t::number_integer:\n            {\n                dump_integer(val.m_value.number_integer);\n                return;\n            }\n\n            case value_t::number_unsigned:\n            {\n                dump_integer(val.m_value.number_unsigned);\n                return;\n            }\n\n            case value_t::number_float:\n            {\n                dump_float(val.m_value.number_float);\n                return;\n            }\n\n            case value_t::discarded:\n            {\n                o->write_characters(\"<discarded>\", 11);\n                return;\n            }\n\n            case value_t::null:\n            {\n                o->write_characters(\"null\", 4);\n                return;\n            }\n\n            default:            // LCOV_EXCL_LINE\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        }\n    }\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    /*!\n    @brief dump escaped string\n\n    Escape a string by replacing certain special characters by a sequence of an\n    escape character (backslash) and another character and other control\n    characters by a sequence of \"\\u\" followed by a four-digit hex\n    representation. The escaped string is written to output stream @a o.\n\n    @param[in] s  the string to escape\n    @param[in] ensure_ascii  whether to escape non-ASCII characters with\n                             \\uXXXX sequences\n\n    @complexity Linear in the length of string @a s.\n    */\n    void dump_escaped(const string_t& s, const bool ensure_ascii)\n    {\n        std::uint32_t codepoint{};\n        std::uint8_t state = UTF8_ACCEPT;\n        std::size_t bytes = 0;  // number of bytes written to string_buffer\n\n        // number of bytes written at the point of the last valid byte\n        std::size_t bytes_after_last_accept = 0;\n        std::size_t undumped_chars = 0;\n\n        for (std::size_t i = 0; i < s.size(); ++i)\n        {\n            const auto byte = static_cast<std::uint8_t>(s[i]);\n\n            switch (decode(state, codepoint, byte))\n            {\n                case UTF8_ACCEPT:  // decode found a new code point\n                {\n                    switch (codepoint)\n                    {\n                        case 0x08: // backspace\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = 'b';\n                            break;\n                        }\n\n                        case 0x09: // horizontal tab\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = 't';\n                            break;\n                        }\n\n                        case 0x0A: // newline\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = 'n';\n                            break;\n                        }\n\n                        case 0x0C: // formfeed\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = 'f';\n                            break;\n                        }\n\n                        case 0x0D: // carriage return\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = 'r';\n                            break;\n                        }\n\n                        case 0x22: // quotation mark\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = '\\\"';\n                            break;\n                        }\n\n                        case 0x5C: // reverse solidus\n                        {\n                            string_buffer[bytes++] = '\\\\';\n                            string_buffer[bytes++] = '\\\\';\n                            break;\n                        }\n\n                        default:\n                        {\n                            // escape control characters (0x00..0x1F) or, if\n                            // ensure_ascii parameter is used, non-ASCII characters\n                            if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))\n                            {\n                                if (codepoint <= 0xFFFF)\n                                {\n                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 7, \"\\\\u%04x\",\n                                                                      static_cast<std::uint16_t>(codepoint)));\n                                    bytes += 6;\n                                }\n                                else\n                                {\n                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 13, \"\\\\u%04x\\\\u%04x\",\n                                                                      static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)),\n                                                                      static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu))));\n                                    bytes += 12;\n                                }\n                            }\n                            else\n                            {\n                                // copy byte to buffer (all previous bytes\n                                // been copied have in default case above)\n                                string_buffer[bytes++] = s[i];\n                            }\n                            break;\n                        }\n                    }\n\n                    // write buffer and reset index; there must be 13 bytes\n                    // left, as this is the maximal number of bytes to be\n                    // written (\"\\uxxxx\\uxxxx\\0\") for one code point\n                    if (string_buffer.size() - bytes < 13)\n                    {\n                        o->write_characters(string_buffer.data(), bytes);\n                        bytes = 0;\n                    }\n\n                    // remember the byte position of this accept\n                    bytes_after_last_accept = bytes;\n                    undumped_chars = 0;\n                    break;\n                }\n\n                case UTF8_REJECT:  // decode found invalid UTF-8 byte\n                {\n                    switch (error_handler)\n                    {\n                        case error_handler_t::strict:\n                        {\n                            JSON_THROW(type_error::create(316, concat(\"invalid UTF-8 byte at index \", std::to_string(i), \": 0x\", hex_bytes(byte | 0)), nullptr));\n                        }\n\n                        case error_handler_t::ignore:\n                        case error_handler_t::replace:\n                        {\n                            // in case we saw this character the first time, we\n                            // would like to read it again, because the byte\n                            // may be OK for itself, but just not OK for the\n                            // previous sequence\n                            if (undumped_chars > 0)\n                            {\n                                --i;\n                            }\n\n                            // reset length buffer to the last accepted index;\n                            // thus removing/ignoring the invalid characters\n                            bytes = bytes_after_last_accept;\n\n                            if (error_handler == error_handler_t::replace)\n                            {\n                                // add a replacement character\n                                if (ensure_ascii)\n                                {\n                                    string_buffer[bytes++] = '\\\\';\n                                    string_buffer[bytes++] = 'u';\n                                    string_buffer[bytes++] = 'f';\n                                    string_buffer[bytes++] = 'f';\n                                    string_buffer[bytes++] = 'f';\n                                    string_buffer[bytes++] = 'd';\n                                }\n                                else\n                                {\n                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\\xEF');\n                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\\xBF');\n                                    string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\\xBD');\n                                }\n\n                                // write buffer and reset index; there must be 13 bytes\n                                // left, as this is the maximal number of bytes to be\n                                // written (\"\\uxxxx\\uxxxx\\0\") for one code point\n                                if (string_buffer.size() - bytes < 13)\n                                {\n                                    o->write_characters(string_buffer.data(), bytes);\n                                    bytes = 0;\n                                }\n\n                                bytes_after_last_accept = bytes;\n                            }\n\n                            undumped_chars = 0;\n\n                            // continue processing the string\n                            state = UTF8_ACCEPT;\n                            break;\n                        }\n\n                        default:            // LCOV_EXCL_LINE\n                            JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n                    }\n                    break;\n                }\n\n                default:  // decode found yet incomplete multi-byte code point\n                {\n                    if (!ensure_ascii)\n                    {\n                        // code point will not be escaped - copy byte to buffer\n                        string_buffer[bytes++] = s[i];\n                    }\n                    ++undumped_chars;\n                    break;\n                }\n            }\n        }\n\n        // we finished processing the string\n        if (JSON_HEDLEY_LIKELY(state == UTF8_ACCEPT))\n        {\n            // write buffer\n            if (bytes > 0)\n            {\n                o->write_characters(string_buffer.data(), bytes);\n            }\n        }\n        else\n        {\n            // we finish reading, but do not accept: string was incomplete\n            switch (error_handler)\n            {\n                case error_handler_t::strict:\n                {\n                    JSON_THROW(type_error::create(316, concat(\"incomplete UTF-8 string; last byte: 0x\", hex_bytes(static_cast<std::uint8_t>(s.back() | 0))), nullptr));\n                }\n\n                case error_handler_t::ignore:\n                {\n                    // write all accepted bytes\n                    o->write_characters(string_buffer.data(), bytes_after_last_accept);\n                    break;\n                }\n\n                case error_handler_t::replace:\n                {\n                    // write all accepted bytes\n                    o->write_characters(string_buffer.data(), bytes_after_last_accept);\n                    // add a replacement character\n                    if (ensure_ascii)\n                    {\n                        o->write_characters(\"\\\\ufffd\", 6);\n                    }\n                    else\n                    {\n                        o->write_characters(\"\\xEF\\xBF\\xBD\", 3);\n                    }\n                    break;\n                }\n\n                default:            // LCOV_EXCL_LINE\n                    JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n            }\n        }\n    }\n\n  private:\n    /*!\n    @brief count digits\n\n    Count the number of decimal (base 10) digits for an input unsigned integer.\n\n    @param[in] x  unsigned integer number to count its digits\n    @return    number of decimal digits\n    */\n    inline unsigned int count_digits(number_unsigned_t x) noexcept\n    {\n        unsigned int n_digits = 1;\n        for (;;)\n        {\n            if (x < 10)\n            {\n                return n_digits;\n            }\n            if (x < 100)\n            {\n                return n_digits + 1;\n            }\n            if (x < 1000)\n            {\n                return n_digits + 2;\n            }\n            if (x < 10000)\n            {\n                return n_digits + 3;\n            }\n            x = x / 10000u;\n            n_digits += 4;\n        }\n    }\n\n    /*!\n     * @brief convert a byte to a uppercase hex representation\n     * @param[in] byte byte to represent\n     * @return representation (\"00\"..\"FF\")\n     */\n    static std::string hex_bytes(std::uint8_t byte)\n    {\n        std::string result = \"FF\";\n        constexpr const char* nibble_to_hex = \"0123456789ABCDEF\";\n        result[0] = nibble_to_hex[byte / 16];\n        result[1] = nibble_to_hex[byte % 16];\n        return result;\n    }\n\n    // templates to avoid warnings about useless casts\n    template <typename NumberType, enable_if_t<std::is_signed<NumberType>::value, int> = 0>\n    bool is_negative_number(NumberType x)\n    {\n        return x < 0;\n    }\n\n    template < typename NumberType, enable_if_t <std::is_unsigned<NumberType>::value, int > = 0 >\n    bool is_negative_number(NumberType /*unused*/)\n    {\n        return false;\n    }\n\n    /*!\n    @brief dump an integer\n\n    Dump a given integer to output stream @a o. Works internally with\n    @a number_buffer.\n\n    @param[in] x  integer number (signed or unsigned) to dump\n    @tparam NumberType either @a number_integer_t or @a number_unsigned_t\n    */\n    template < typename NumberType, detail::enable_if_t <\n                   std::is_integral<NumberType>::value ||\n                   std::is_same<NumberType, number_unsigned_t>::value ||\n                   std::is_same<NumberType, number_integer_t>::value ||\n                   std::is_same<NumberType, binary_char_t>::value,\n                   int > = 0 >\n    void dump_integer(NumberType x)\n    {\n        static constexpr std::array<std::array<char, 2>, 100> digits_to_99\n        {\n            {\n                {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}},\n                {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}},\n                {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}},\n                {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}},\n                {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}},\n                {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}},\n                {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}},\n                {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}},\n                {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}},\n                {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}},\n            }\n        };\n\n        // special case for \"0\"\n        if (x == 0)\n        {\n            o->write_character('0');\n            return;\n        }\n\n        // use a pointer to fill the buffer\n        auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n\n        number_unsigned_t abs_value;\n\n        unsigned int n_chars{};\n\n        if (is_negative_number(x))\n        {\n            *buffer_ptr = '-';\n            abs_value = remove_sign(static_cast<number_integer_t>(x));\n\n            // account one more byte for the minus sign\n            n_chars = 1 + count_digits(abs_value);\n        }\n        else\n        {\n            abs_value = static_cast<number_unsigned_t>(x);\n            n_chars = count_digits(abs_value);\n        }\n\n        // spare 1 byte for '\\0'\n        JSON_ASSERT(n_chars < number_buffer.size() - 1);\n\n        // jump to the end to generate the string from backward,\n        // so we later avoid reversing the result\n        buffer_ptr += n_chars;\n\n        // Fast int2ascii implementation inspired by \"Fastware\" talk by Andrei Alexandrescu\n        // See: https://www.youtube.com/watch?v=o4-CwDo2zpg\n        while (abs_value >= 100)\n        {\n            const auto digits_index = static_cast<unsigned>((abs_value % 100));\n            abs_value /= 100;\n            *(--buffer_ptr) = digits_to_99[digits_index][1];\n            *(--buffer_ptr) = digits_to_99[digits_index][0];\n        }\n\n        if (abs_value >= 10)\n        {\n            const auto digits_index = static_cast<unsigned>(abs_value);\n            *(--buffer_ptr) = digits_to_99[digits_index][1];\n            *(--buffer_ptr) = digits_to_99[digits_index][0];\n        }\n        else\n        {\n            *(--buffer_ptr) = static_cast<char>('0' + abs_value);\n        }\n\n        o->write_characters(number_buffer.data(), n_chars);\n    }\n\n    /*!\n    @brief dump a floating-point number\n\n    Dump a given floating-point number to output stream @a o. Works internally\n    with @a number_buffer.\n\n    @param[in] x  floating-point number to dump\n    */\n    void dump_float(number_float_t x)\n    {\n        // NaN / inf\n        if (!std::isfinite(x))\n        {\n            o->write_characters(\"null\", 4);\n            return;\n        }\n\n        // If number_float_t is an IEEE-754 single or double precision number,\n        // use the Grisu2 algorithm to produce short numbers which are\n        // guaranteed to round-trip, using strtof and strtod, resp.\n        //\n        // NB: The test below works if <long double> == <double>.\n        static constexpr bool is_ieee_single_or_double\n            = (std::numeric_limits<number_float_t>::is_iec559 && std::numeric_limits<number_float_t>::digits == 24 && std::numeric_limits<number_float_t>::max_exponent == 128) ||\n              (std::numeric_limits<number_float_t>::is_iec559 && std::numeric_limits<number_float_t>::digits == 53 && std::numeric_limits<number_float_t>::max_exponent == 1024);\n\n        dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>());\n    }\n\n    void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/)\n    {\n        auto* begin = number_buffer.data();\n        auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x);\n\n        o->write_characters(begin, static_cast<size_t>(end - begin));\n    }\n\n    void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/)\n    {\n        // get number of digits for a float -> text -> float round-trip\n        static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10;\n\n        // the actual conversion\n        // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)\n        std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), \"%.*g\", d, x);\n\n        // negative value indicates an error\n        JSON_ASSERT(len > 0);\n        // check if buffer was large enough\n        JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());\n\n        // erase thousands separator\n        if (thousands_sep != '\\0')\n        {\n            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081\n            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);\n            std::fill(end, number_buffer.end(), '\\0');\n            JSON_ASSERT((end - number_buffer.begin()) <= len);\n            len = (end - number_buffer.begin());\n        }\n\n        // convert decimal point to '.'\n        if (decimal_point != '\\0' && decimal_point != '.')\n        {\n            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081\n            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);\n            if (dec_pos != number_buffer.end())\n            {\n                *dec_pos = '.';\n            }\n        }\n\n        o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));\n\n        // determine if we need to append \".0\"\n        const bool value_is_int_like =\n            std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,\n                         [](char c)\n        {\n            return c == '.' || c == 'e';\n        });\n\n        if (value_is_int_like)\n        {\n            o->write_characters(\".0\", 2);\n        }\n    }\n\n    /*!\n    @brief check whether a string is UTF-8 encoded\n\n    The function checks each byte of a string whether it is UTF-8 encoded. The\n    result of the check is stored in the @a state parameter. The function must\n    be called initially with state 0 (accept). State 1 means the string must\n    be rejected, because the current byte is not allowed. If the string is\n    completely processed, but the state is non-zero, the string ended\n    prematurely; that is, the last byte indicated more bytes should have\n    followed.\n\n    @param[in,out] state  the state of the decoding\n    @param[in,out] codep  codepoint (valid only if resulting state is UTF8_ACCEPT)\n    @param[in] byte       next byte to decode\n    @return               new state\n\n    @note The function has been edited: a std::array is used.\n\n    @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>\n    @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/\n    */\n    static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept\n    {\n        static const std::array<std::uint8_t, 400> utf8d =\n        {\n            {\n                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F\n                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F\n                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F\n                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F\n                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F\n                7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF\n                8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF\n                0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF\n                0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF\n                0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0\n                1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2\n                1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4\n                1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6\n                1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8\n            }\n        };\n\n        JSON_ASSERT(byte < utf8d.size());\n        const std::uint8_t type = utf8d[byte];\n\n        codep = (state != UTF8_ACCEPT)\n                ? (byte & 0x3fu) | (codep << 6u)\n                : (0xFFu >> type) & (byte);\n\n        std::size_t index = 256u + static_cast<size_t>(state) * 16u + static_cast<size_t>(type);\n        JSON_ASSERT(index < 400);\n        state = utf8d[index];\n        return state;\n    }\n\n    /*\n     * Overload to make the compiler happy while it is instantiating\n     * dump_integer for number_unsigned_t.\n     * Must never be called.\n     */\n    number_unsigned_t remove_sign(number_unsigned_t x)\n    {\n        JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        return x; // LCOV_EXCL_LINE\n    }\n\n    /*\n     * Helper function for dump_integer\n     *\n     * This function takes a negative signed integer and returns its absolute\n     * value as unsigned integer. The plus/minus shuffling is necessary as we can\n     * not directly remove the sign of an arbitrary signed integer as the\n     * absolute values of INT_MIN and INT_MAX are usually not the same. See\n     * #1708 for details.\n     */\n    inline number_unsigned_t remove_sign(number_integer_t x) noexcept\n    {\n        JSON_ASSERT(x < 0 && x < (std::numeric_limits<number_integer_t>::max)()); // NOLINT(misc-redundant-expression)\n        return static_cast<number_unsigned_t>(-(x + 1)) + 1;\n    }\n\n  private:\n    /// the output of the serializer\n    output_adapter_t<char> o = nullptr;\n\n    /// a (hopefully) large enough character buffer\n    std::array<char, 64> number_buffer{{}};\n\n    /// the locale\n    const std::lconv* loc = nullptr;\n    /// the locale's thousand separator character\n    const char thousands_sep = '\\0';\n    /// the locale's decimal point character\n    const char decimal_point = '\\0';\n\n    /// string buffer\n    std::array<char, 512> string_buffer{{}};\n\n    /// the indentation character\n    const char indent_char;\n    /// the indentation string\n    string_t indent_string;\n\n    /// error_handler how to react on decoding errors\n    const error_handler_t error_handler;\n};\n\n}  // namespace detail\nNLOHMANN_JSON_NAMESPACE_END\n\n// #include <nlohmann/detail/value_t.hpp>\n\n// #include <nlohmann/json_fwd.hpp>\n\n// #include <nlohmann/ordered_map.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#include <functional> // equal_to, less\n#include <initializer_list> // initializer_list\n#include <iterator> // input_iterator_tag, iterator_traits\n#include <memory> // allocator\n#include <stdexcept> // for out_of_range\n#include <type_traits> // enable_if, is_convertible\n#include <utility> // pair\n#include <vector> // vector\n\n// #include <nlohmann/detail/macro_scope.hpp>\n\n// #include <nlohmann/detail/meta/type_traits.hpp>\n\n\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/// ordered_map: a minimal map-like container that preserves insertion order\n/// for use within nlohmann::basic_json<ordered_map>\ntemplate <class Key, class T, class IgnoredLess = std::less<Key>,\n          class Allocator = std::allocator<std::pair<const Key, T>>>\n                  struct ordered_map : std::vector<std::pair<const Key, T>, Allocator>\n{\n    using key_type = Key;\n    using mapped_type = T;\n    using Container = std::vector<std::pair<const Key, T>, Allocator>;\n    using iterator = typename Container::iterator;\n    using const_iterator = typename Container::const_iterator;\n    using size_type = typename Container::size_type;\n    using value_type = typename Container::value_type;\n#ifdef JSON_HAS_CPP_14\n    using key_compare = std::equal_to<>;\n#else\n    using key_compare = std::equal_to<Key>;\n#endif\n\n    // Explicit constructors instead of `using Container::Container`\n    // otherwise older compilers choke on it (GCC <= 5.5, xcode <= 9.4)\n    ordered_map() noexcept(noexcept(Container())) : Container{} {}\n    explicit ordered_map(const Allocator& alloc) noexcept(noexcept(Container(alloc))) : Container{alloc} {}\n    template <class It>\n    ordered_map(It first, It last, const Allocator& alloc = Allocator())\n        : Container{first, last, alloc} {}\n    ordered_map(std::initializer_list<value_type> init, const Allocator& alloc = Allocator() )\n        : Container{init, alloc} {}\n\n    std::pair<iterator, bool> emplace(const key_type& key, T&& t)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return {it, false};\n            }\n        }\n        Container::emplace_back(key, std::forward<T>(t));\n        return {std::prev(this->end()), true};\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    std::pair<iterator, bool> emplace(KeyType && key, T && t)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return {it, false};\n            }\n        }\n        Container::emplace_back(std::forward<KeyType>(key), std::forward<T>(t));\n        return {std::prev(this->end()), true};\n    }\n\n    T& operator[](const key_type& key)\n    {\n        return emplace(key, T{}).first->second;\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    T & operator[](KeyType && key)\n    {\n        return emplace(std::forward<KeyType>(key), T{}).first->second;\n    }\n\n    const T& operator[](const key_type& key) const\n    {\n        return at(key);\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    const T & operator[](KeyType && key) const\n    {\n        return at(std::forward<KeyType>(key));\n    }\n\n    T& at(const key_type& key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it->second;\n            }\n        }\n\n        JSON_THROW(std::out_of_range(\"key not found\"));\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    T & at(KeyType && key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it->second;\n            }\n        }\n\n        JSON_THROW(std::out_of_range(\"key not found\"));\n    }\n\n    const T& at(const key_type& key) const\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it->second;\n            }\n        }\n\n        JSON_THROW(std::out_of_range(\"key not found\"));\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    const T & at(KeyType && key) const\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it->second;\n            }\n        }\n\n        JSON_THROW(std::out_of_range(\"key not found\"));\n    }\n\n    size_type erase(const key_type& key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                // Since we cannot move const Keys, re-construct them in place\n                for (auto next = it; ++next != this->end(); ++it)\n                {\n                    it->~value_type(); // Destroy but keep allocation\n                    new (&*it) value_type{std::move(*next)};\n                }\n                Container::pop_back();\n                return 1;\n            }\n        }\n        return 0;\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    size_type erase(KeyType && key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                // Since we cannot move const Keys, re-construct them in place\n                for (auto next = it; ++next != this->end(); ++it)\n                {\n                    it->~value_type(); // Destroy but keep allocation\n                    new (&*it) value_type{std::move(*next)};\n                }\n                Container::pop_back();\n                return 1;\n            }\n        }\n        return 0;\n    }\n\n    iterator erase(iterator pos)\n    {\n        return erase(pos, std::next(pos));\n    }\n\n    iterator erase(iterator first, iterator last)\n    {\n        if (first == last)\n        {\n            return first;\n        }\n\n        const auto elements_affected = std::distance(first, last);\n        const auto offset = std::distance(Container::begin(), first);\n\n        // This is the start situation. We need to delete elements_affected\n        // elements (3 in this example: e, f, g), and need to return an\n        // iterator past the last deleted element (h in this example).\n        // Note that offset is the distance from the start of the vector\n        // to first. We will need this later.\n\n        // [ a, b, c, d, e, f, g, h, i, j ]\n        //               ^        ^\n        //             first    last\n\n        // Since we cannot move const Keys, we re-construct them in place.\n        // We start at first and re-construct (viz. copy) the elements from\n        // the back of the vector. Example for first iteration:\n\n        //               ,--------.\n        //               v        |   destroy e and re-construct with h\n        // [ a, b, c, d, e, f, g, h, i, j ]\n        //               ^        ^\n        //               it       it + elements_affected\n\n        for (auto it = first; std::next(it, elements_affected) != Container::end(); ++it)\n        {\n            it->~value_type(); // destroy but keep allocation\n            new (&*it) value_type{std::move(*std::next(it, elements_affected))}; // \"move\" next element to it\n        }\n\n        // [ a, b, c, d, h, i, j, h, i, j ]\n        //               ^        ^\n        //             first    last\n\n        // remove the unneeded elements at the end of the vector\n        Container::resize(this->size() - static_cast<size_type>(elements_affected));\n\n        // [ a, b, c, d, h, i, j ]\n        //               ^        ^\n        //             first    last\n\n        // first is now pointing past the last deleted element, but we cannot\n        // use this iterator, because it may have been invalidated by the\n        // resize call. Instead, we can return begin() + offset.\n        return Container::begin() + offset;\n    }\n\n    size_type count(const key_type& key) const\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return 1;\n            }\n        }\n        return 0;\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    size_type count(KeyType && key) const\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return 1;\n            }\n        }\n        return 0;\n    }\n\n    iterator find(const key_type& key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it;\n            }\n        }\n        return Container::end();\n    }\n\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>\n    iterator find(KeyType && key)\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it;\n            }\n        }\n        return Container::end();\n    }\n\n    const_iterator find(const key_type& key) const\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, key))\n            {\n                return it;\n            }\n        }\n        return Container::end();\n    }\n\n    std::pair<iterator, bool> insert( value_type&& value )\n    {\n        return emplace(value.first, std::move(value.second));\n    }\n\n    std::pair<iterator, bool> insert( const value_type& value )\n    {\n        for (auto it = this->begin(); it != this->end(); ++it)\n        {\n            if (m_compare(it->first, value.first))\n            {\n                return {it, false};\n            }\n        }\n        Container::push_back(value);\n        return {--this->end(), true};\n    }\n\n    template<typename InputIt>\n    using require_input_iter = typename std::enable_if<std::is_convertible<typename std::iterator_traits<InputIt>::iterator_category,\n            std::input_iterator_tag>::value>::type;\n\n    template<typename InputIt, typename = require_input_iter<InputIt>>\n    void insert(InputIt first, InputIt last)\n    {\n        for (auto it = first; it != last; ++it)\n        {\n            insert(*it);\n        }\n    }\n\nprivate:\n    JSON_NO_UNIQUE_ADDRESS key_compare m_compare = key_compare();\n};\n\nNLOHMANN_JSON_NAMESPACE_END\n\n\n#if defined(JSON_HAS_CPP_17)\n    #include <any>\n    #include <string_view>\n#endif\n\n/*!\n@brief namespace for Niels Lohmann\n@see https://github.com/nlohmann\n@since version 1.0.0\n*/\nNLOHMANN_JSON_NAMESPACE_BEGIN\n\n/*!\n@brief a class to store JSON values\n\n@internal\n@invariant The member variables @a m_value and @a m_type have the following\nrelationship:\n- If `m_type == value_t::object`, then `m_value.object != nullptr`.\n- If `m_type == value_t::array`, then `m_value.array != nullptr`.\n- If `m_type == value_t::string`, then `m_value.string != nullptr`.\nThe invariants are checked by member function assert_invariant().\n\n@note ObjectType trick from https://stackoverflow.com/a/9860911\n@endinternal\n\n@since version 1.0.0\n\n@nosubgrouping\n*/\nNLOHMANN_BASIC_JSON_TPL_DECLARATION\nclass basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions)\n{\n  private:\n    template<detail::value_t> friend struct detail::external_constructor;\n\n    template<typename>\n    friend class ::nlohmann::json_pointer;\n    // can be restored when json_pointer backwards compatibility is removed\n    // friend ::nlohmann::json_pointer<StringType>;\n\n    template<typename BasicJsonType, typename InputType>\n    friend class ::nlohmann::detail::parser;\n    friend ::nlohmann::detail::serializer<basic_json>;\n    template<typename BasicJsonType>\n    friend class ::nlohmann::detail::iter_impl;\n    template<typename BasicJsonType, typename CharType>\n    friend class ::nlohmann::detail::binary_writer;\n    template<typename BasicJsonType, typename InputType, typename SAX>\n    friend class ::nlohmann::detail::binary_reader;\n    template<typename BasicJsonType>\n    friend class ::nlohmann::detail::json_sax_dom_parser;\n    template<typename BasicJsonType>\n    friend class ::nlohmann::detail::json_sax_dom_callback_parser;\n    friend class ::nlohmann::detail::exception;\n\n    /// workaround type for MSVC\n    using basic_json_t = NLOHMANN_BASIC_JSON_TPL;\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    // convenience aliases for types residing in namespace detail;\n    using lexer = ::nlohmann::detail::lexer_base<basic_json>;\n\n    template<typename InputAdapterType>\n    static ::nlohmann::detail::parser<basic_json, InputAdapterType> parser(\n        InputAdapterType adapter,\n        detail::parser_callback_t<basic_json>cb = nullptr,\n        const bool allow_exceptions = true,\n        const bool ignore_comments = false\n                                 )\n    {\n        return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),\n                std::move(cb), allow_exceptions, ignore_comments);\n    }\n\n  private:\n    using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;\n    template<typename BasicJsonType>\n    using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;\n    template<typename BasicJsonType>\n    using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;\n    template<typename Iterator>\n    using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;\n    template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;\n\n    template<typename CharType>\n    using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;\n\n    template<typename InputType>\n    using binary_reader = ::nlohmann::detail::binary_reader<basic_json, InputType>;\n    template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    using serializer = ::nlohmann::detail::serializer<basic_json>;\n\n  public:\n    using value_t = detail::value_t;\n    /// JSON Pointer, see @ref nlohmann::json_pointer\n    using json_pointer = ::nlohmann::json_pointer<StringType>;\n    template<typename T, typename SFINAE>\n    using json_serializer = JSONSerializer<T, SFINAE>;\n    /// how to treat decoding errors\n    using error_handler_t = detail::error_handler_t;\n    /// how to treat CBOR tags\n    using cbor_tag_handler_t = detail::cbor_tag_handler_t;\n    /// helper type for initializer lists of basic_json values\n    using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;\n\n    using input_format_t = detail::input_format_t;\n    /// SAX interface type, see @ref nlohmann::json_sax\n    using json_sax_t = json_sax<basic_json>;\n\n    ////////////////\n    // exceptions //\n    ////////////////\n\n    /// @name exceptions\n    /// Classes to implement user-defined exceptions.\n    /// @{\n\n    using exception = detail::exception;\n    using parse_error = detail::parse_error;\n    using invalid_iterator = detail::invalid_iterator;\n    using type_error = detail::type_error;\n    using out_of_range = detail::out_of_range;\n    using other_error = detail::other_error;\n\n    /// @}\n\n\n    /////////////////////\n    // container types //\n    /////////////////////\n\n    /// @name container types\n    /// The canonic container types to use @ref basic_json like any other STL\n    /// container.\n    /// @{\n\n    /// the type of elements in a basic_json container\n    using value_type = basic_json;\n\n    /// the type of an element reference\n    using reference = value_type&;\n    /// the type of an element const reference\n    using const_reference = const value_type&;\n\n    /// a type to represent differences between iterators\n    using difference_type = std::ptrdiff_t;\n    /// a type to represent container sizes\n    using size_type = std::size_t;\n\n    /// the allocator type\n    using allocator_type = AllocatorType<basic_json>;\n\n    /// the type of an element pointer\n    using pointer = typename std::allocator_traits<allocator_type>::pointer;\n    /// the type of an element const pointer\n    using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;\n\n    /// an iterator for a basic_json container\n    using iterator = iter_impl<basic_json>;\n    /// a const iterator for a basic_json container\n    using const_iterator = iter_impl<const basic_json>;\n    /// a reverse iterator for a basic_json container\n    using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>;\n    /// a const reverse iterator for a basic_json container\n    using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>;\n\n    /// @}\n\n\n    /// @brief returns the allocator associated with the container\n    /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/\n    static allocator_type get_allocator()\n    {\n        return allocator_type();\n    }\n\n    /// @brief returns version information on the library\n    /// @sa https://json.nlohmann.me/api/basic_json/meta/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json meta()\n    {\n        basic_json result;\n\n        result[\"copyright\"] = \"(C) 2013-2022 Niels Lohmann\";\n        result[\"name\"] = \"JSON for Modern C++\";\n        result[\"url\"] = \"https://github.com/nlohmann/json\";\n        result[\"version\"][\"string\"] =\n            detail::concat(std::to_string(NLOHMANN_JSON_VERSION_MAJOR), '.',\n                           std::to_string(NLOHMANN_JSON_VERSION_MINOR), '.',\n                           std::to_string(NLOHMANN_JSON_VERSION_PATCH));\n        result[\"version\"][\"major\"] = NLOHMANN_JSON_VERSION_MAJOR;\n        result[\"version\"][\"minor\"] = NLOHMANN_JSON_VERSION_MINOR;\n        result[\"version\"][\"patch\"] = NLOHMANN_JSON_VERSION_PATCH;\n\n#ifdef _WIN32\n        result[\"platform\"] = \"win32\";\n#elif defined __linux__\n        result[\"platform\"] = \"linux\";\n#elif defined __APPLE__\n        result[\"platform\"] = \"apple\";\n#elif defined __unix__\n        result[\"platform\"] = \"unix\";\n#else\n        result[\"platform\"] = \"unknown\";\n#endif\n\n#if defined(__ICC) || defined(__INTEL_COMPILER)\n        result[\"compiler\"] = {{\"family\", \"icc\"}, {\"version\", __INTEL_COMPILER}};\n#elif defined(__clang__)\n        result[\"compiler\"] = {{\"family\", \"clang\"}, {\"version\", __clang_version__}};\n#elif defined(__GNUC__) || defined(__GNUG__)\n        result[\"compiler\"] = {{\"family\", \"gcc\"}, {\"version\", detail::concat(\n                    std::to_string(__GNUC__), '.',\n                    std::to_string(__GNUC_MINOR__), '.',\n                    std::to_string(__GNUC_PATCHLEVEL__))\n            }\n        };\n#elif defined(__HP_cc) || defined(__HP_aCC)\n        result[\"compiler\"] = \"hp\"\n#elif defined(__IBMCPP__)\n        result[\"compiler\"] = {{\"family\", \"ilecpp\"}, {\"version\", __IBMCPP__}};\n#elif defined(_MSC_VER)\n        result[\"compiler\"] = {{\"family\", \"msvc\"}, {\"version\", _MSC_VER}};\n#elif defined(__PGI)\n        result[\"compiler\"] = {{\"family\", \"pgcpp\"}, {\"version\", __PGI}};\n#elif defined(__SUNPRO_CC)\n        result[\"compiler\"] = {{\"family\", \"sunpro\"}, {\"version\", __SUNPRO_CC}};\n#else\n        result[\"compiler\"] = {{\"family\", \"unknown\"}, {\"version\", \"unknown\"}};\n#endif\n\n\n#if defined(_MSVC_LANG)\n        result[\"compiler\"][\"c++\"] = std::to_string(_MSVC_LANG);\n#elif defined(__cplusplus)\n        result[\"compiler\"][\"c++\"] = std::to_string(__cplusplus);\n#else\n        result[\"compiler\"][\"c++\"] = \"unknown\";\n#endif\n        return result;\n    }\n\n\n    ///////////////////////////\n    // JSON value data types //\n    ///////////////////////////\n\n    /// @name JSON value data types\n    /// The data types to store a JSON value. These types are derived from\n    /// the template arguments passed to class @ref basic_json.\n    /// @{\n\n    /// @brief default object key comparator type\n    /// The actual object key comparator type (@ref object_comparator_t) may be\n    /// different.\n    /// @sa https://json.nlohmann.me/api/basic_json/default_object_comparator_t/\n#if defined(JSON_HAS_CPP_14)\n    // use of transparent comparator avoids unnecessary repeated construction of temporaries\n    // in functions involving lookup by key with types other than object_t::key_type (aka. StringType)\n    using default_object_comparator_t = std::less<>;\n#else\n    using default_object_comparator_t = std::less<StringType>;\n#endif\n\n    /// @brief a type for an object\n    /// @sa https://json.nlohmann.me/api/basic_json/object_t/\n    using object_t = ObjectType<StringType,\n          basic_json,\n          default_object_comparator_t,\n          AllocatorType<std::pair<const StringType,\n          basic_json>>>;\n\n    /// @brief a type for an array\n    /// @sa https://json.nlohmann.me/api/basic_json/array_t/\n    using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;\n\n    /// @brief a type for a string\n    /// @sa https://json.nlohmann.me/api/basic_json/string_t/\n    using string_t = StringType;\n\n    /// @brief a type for a boolean\n    /// @sa https://json.nlohmann.me/api/basic_json/boolean_t/\n    using boolean_t = BooleanType;\n\n    /// @brief a type for a number (integer)\n    /// @sa https://json.nlohmann.me/api/basic_json/number_integer_t/\n    using number_integer_t = NumberIntegerType;\n\n    /// @brief a type for a number (unsigned)\n    /// @sa https://json.nlohmann.me/api/basic_json/number_unsigned_t/\n    using number_unsigned_t = NumberUnsignedType;\n\n    /// @brief a type for a number (floating-point)\n    /// @sa https://json.nlohmann.me/api/basic_json/number_float_t/\n    using number_float_t = NumberFloatType;\n\n    /// @brief a type for a packed binary type\n    /// @sa https://json.nlohmann.me/api/basic_json/binary_t/\n    using binary_t = nlohmann::byte_container_with_subtype<BinaryType>;\n\n    /// @brief object key comparator type\n    /// @sa https://json.nlohmann.me/api/basic_json/object_comparator_t/\n    using object_comparator_t = detail::actual_object_comparator_t<basic_json>;\n\n    /// @}\n\n  private:\n\n    /// helper for exception-safe object creation\n    template<typename T, typename... Args>\n    JSON_HEDLEY_RETURNS_NON_NULL\n    static T* create(Args&& ... args)\n    {\n        AllocatorType<T> alloc;\n        using AllocatorTraits = std::allocator_traits<AllocatorType<T>>;\n\n        auto deleter = [&](T * obj)\n        {\n            AllocatorTraits::deallocate(alloc, obj, 1);\n        };\n        std::unique_ptr<T, decltype(deleter)> obj(AllocatorTraits::allocate(alloc, 1), deleter);\n        AllocatorTraits::construct(alloc, obj.get(), std::forward<Args>(args)...);\n        JSON_ASSERT(obj != nullptr);\n        return obj.release();\n    }\n\n    ////////////////////////\n    // JSON value storage //\n    ////////////////////////\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    /*!\n    @brief a JSON value\n\n    The actual storage for a JSON value of the @ref basic_json class. This\n    union combines the different storage types for the JSON value types\n    defined in @ref value_t.\n\n    JSON type | value_t type    | used type\n    --------- | --------------- | ------------------------\n    object    | object          | pointer to @ref object_t\n    array     | array           | pointer to @ref array_t\n    string    | string          | pointer to @ref string_t\n    boolean   | boolean         | @ref boolean_t\n    number    | number_integer  | @ref number_integer_t\n    number    | number_unsigned | @ref number_unsigned_t\n    number    | number_float    | @ref number_float_t\n    binary    | binary          | pointer to @ref binary_t\n    null      | null            | *no value is stored*\n\n    @note Variable-length types (objects, arrays, and strings) are stored as\n    pointers. The size of the union should not exceed 64 bits if the default\n    value types are used.\n\n    @since version 1.0.0\n    */\n    union json_value\n    {\n        /// object (stored with pointer to save storage)\n        object_t* object;\n        /// array (stored with pointer to save storage)\n        array_t* array;\n        /// string (stored with pointer to save storage)\n        string_t* string;\n        /// binary (stored with pointer to save storage)\n        binary_t* binary;\n        /// boolean\n        boolean_t boolean;\n        /// number (integer)\n        number_integer_t number_integer;\n        /// number (unsigned integer)\n        number_unsigned_t number_unsigned;\n        /// number (floating-point)\n        number_float_t number_float;\n\n        /// default constructor (for null values)\n        json_value() = default;\n        /// constructor for booleans\n        json_value(boolean_t v) noexcept : boolean(v) {}\n        /// constructor for numbers (integer)\n        json_value(number_integer_t v) noexcept : number_integer(v) {}\n        /// constructor for numbers (unsigned)\n        json_value(number_unsigned_t v) noexcept : number_unsigned(v) {}\n        /// constructor for numbers (floating-point)\n        json_value(number_float_t v) noexcept : number_float(v) {}\n        /// constructor for empty values of a given type\n        json_value(value_t t)\n        {\n            switch (t)\n            {\n                case value_t::object:\n                {\n                    object = create<object_t>();\n                    break;\n                }\n\n                case value_t::array:\n                {\n                    array = create<array_t>();\n                    break;\n                }\n\n                case value_t::string:\n                {\n                    string = create<string_t>(\"\");\n                    break;\n                }\n\n                case value_t::binary:\n                {\n                    binary = create<binary_t>();\n                    break;\n                }\n\n                case value_t::boolean:\n                {\n                    boolean = static_cast<boolean_t>(false);\n                    break;\n                }\n\n                case value_t::number_integer:\n                {\n                    number_integer = static_cast<number_integer_t>(0);\n                    break;\n                }\n\n                case value_t::number_unsigned:\n                {\n                    number_unsigned = static_cast<number_unsigned_t>(0);\n                    break;\n                }\n\n                case value_t::number_float:\n                {\n                    number_float = static_cast<number_float_t>(0.0);\n                    break;\n                }\n\n                case value_t::null:\n                {\n                    object = nullptr;  // silence warning, see #821\n                    break;\n                }\n\n                case value_t::discarded:\n                default:\n                {\n                    object = nullptr;  // silence warning, see #821\n                    if (JSON_HEDLEY_UNLIKELY(t == value_t::null))\n                    {\n                        JSON_THROW(other_error::create(500, \"961c151d2e87f2686a955a9be24d316f1362bf21 3.11.2\", nullptr)); // LCOV_EXCL_LINE\n                    }\n                    break;\n                }\n            }\n        }\n\n        /// constructor for strings\n        json_value(const string_t& value) : string(create<string_t>(value)) {}\n\n        /// constructor for rvalue strings\n        json_value(string_t&& value) : string(create<string_t>(std::move(value))) {}\n\n        /// constructor for objects\n        json_value(const object_t& value) : object(create<object_t>(value)) {}\n\n        /// constructor for rvalue objects\n        json_value(object_t&& value) : object(create<object_t>(std::move(value))) {}\n\n        /// constructor for arrays\n        json_value(const array_t& value) : array(create<array_t>(value)) {}\n\n        /// constructor for rvalue arrays\n        json_value(array_t&& value) : array(create<array_t>(std::move(value))) {}\n\n        /// constructor for binary arrays\n        json_value(const typename binary_t::container_type& value) : binary(create<binary_t>(value)) {}\n\n        /// constructor for rvalue binary arrays\n        json_value(typename binary_t::container_type&& value) : binary(create<binary_t>(std::move(value))) {}\n\n        /// constructor for binary arrays (internal type)\n        json_value(const binary_t& value) : binary(create<binary_t>(value)) {}\n\n        /// constructor for rvalue binary arrays (internal type)\n        json_value(binary_t&& value) : binary(create<binary_t>(std::move(value))) {}\n\n        void destroy(value_t t)\n        {\n            if (t == value_t::array || t == value_t::object)\n            {\n                // flatten the current json_value to a heap-allocated stack\n                std::vector<basic_json> stack;\n\n                // move the top-level items to stack\n                if (t == value_t::array)\n                {\n                    stack.reserve(array->size());\n                    std::move(array->begin(), array->end(), std::back_inserter(stack));\n                }\n                else\n                {\n                    stack.reserve(object->size());\n                    for (auto&& it : *object)\n                    {\n                        stack.push_back(std::move(it.second));\n                    }\n                }\n\n                while (!stack.empty())\n                {\n                    // move the last item to local variable to be processed\n                    basic_json current_item(std::move(stack.back()));\n                    stack.pop_back();\n\n                    // if current_item is array/object, move\n                    // its children to the stack to be processed later\n                    if (current_item.is_array())\n                    {\n                        std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack));\n\n                        current_item.m_value.array->clear();\n                    }\n                    else if (current_item.is_object())\n                    {\n                        for (auto&& it : *current_item.m_value.object)\n                        {\n                            stack.push_back(std::move(it.second));\n                        }\n\n                        current_item.m_value.object->clear();\n                    }\n\n                    // it's now safe that current_item get destructed\n                    // since it doesn't have any children\n                }\n            }\n\n            switch (t)\n            {\n                case value_t::object:\n                {\n                    AllocatorType<object_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, object);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1);\n                    break;\n                }\n\n                case value_t::array:\n                {\n                    AllocatorType<array_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, array);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1);\n                    break;\n                }\n\n                case value_t::string:\n                {\n                    AllocatorType<string_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, string);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1);\n                    break;\n                }\n\n                case value_t::binary:\n                {\n                    AllocatorType<binary_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, binary);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, binary, 1);\n                    break;\n                }\n\n                case value_t::null:\n                case value_t::boolean:\n                case value_t::number_integer:\n                case value_t::number_unsigned:\n                case value_t::number_float:\n                case value_t::discarded:\n                default:\n                {\n                    break;\n                }\n            }\n        }\n    };\n\n  private:\n    /*!\n    @brief checks the class invariants\n\n    This function asserts the class invariants. It needs to be called at the\n    end of every constructor to make sure that created objects respect the\n    invariant. Furthermore, it has to be called each time the type of a JSON\n    value is changed, because the invariant expresses a relationship between\n    @a m_type and @a m_value.\n\n    Furthermore, the parent relation is checked for arrays and objects: If\n    @a check_parents true and the value is an array or object, then the\n    container's elements must have the current value as parent.\n\n    @param[in] check_parents  whether the parent relation should be checked.\n               The value is true by default and should only be set to false\n               during destruction of objects when the invariant does not\n               need to hold.\n    */\n    void assert_invariant(bool check_parents = true) const noexcept\n    {\n        JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr);\n        JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr);\n        JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr);\n        JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr);\n\n#if JSON_DIAGNOSTICS\n        JSON_TRY\n        {\n            // cppcheck-suppress assertWithSideEffect\n            JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j)\n            {\n                return j.m_parent == this;\n            }));\n        }\n        JSON_CATCH(...) {} // LCOV_EXCL_LINE\n#endif\n        static_cast<void>(check_parents);\n    }\n\n    void set_parents()\n    {\n#if JSON_DIAGNOSTICS\n        switch (m_type)\n        {\n            case value_t::array:\n            {\n                for (auto& element : *m_value.array)\n                {\n                    element.m_parent = this;\n                }\n                break;\n            }\n\n            case value_t::object:\n            {\n                for (auto& element : *m_value.object)\n                {\n                    element.second.m_parent = this;\n                }\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                break;\n        }\n#endif\n    }\n\n    iterator set_parents(iterator it, typename iterator::difference_type count_set_parents)\n    {\n#if JSON_DIAGNOSTICS\n        for (typename iterator::difference_type i = 0; i < count_set_parents; ++i)\n        {\n            (it + i)->m_parent = this;\n        }\n#else\n        static_cast<void>(count_set_parents);\n#endif\n        return it;\n    }\n\n    reference set_parent(reference j, std::size_t old_capacity = static_cast<std::size_t>(-1))\n    {\n#if JSON_DIAGNOSTICS\n        if (old_capacity != static_cast<std::size_t>(-1))\n        {\n            // see https://github.com/nlohmann/json/issues/2838\n            JSON_ASSERT(type() == value_t::array);\n            if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity))\n            {\n                // capacity has changed: update all parents\n                set_parents();\n                return j;\n            }\n        }\n\n        // ordered_json uses a vector internally, so pointers could have\n        // been invalidated; see https://github.com/nlohmann/json/issues/2962\n#ifdef JSON_HEDLEY_MSVC_VERSION\n#pragma warning(push )\n#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr\n#endif\n        if (detail::is_ordered_map<object_t>::value)\n        {\n            set_parents();\n            return j;\n        }\n#ifdef JSON_HEDLEY_MSVC_VERSION\n#pragma warning( pop )\n#endif\n\n        j.m_parent = this;\n#else\n        static_cast<void>(j);\n        static_cast<void>(old_capacity);\n#endif\n        return j;\n    }\n\n  public:\n    //////////////////////////\n    // JSON parser callback //\n    //////////////////////////\n\n    /// @brief parser event types\n    /// @sa https://json.nlohmann.me/api/basic_json/parse_event_t/\n    using parse_event_t = detail::parse_event_t;\n\n    /// @brief per-element parser callback type\n    /// @sa https://json.nlohmann.me/api/basic_json/parser_callback_t/\n    using parser_callback_t = detail::parser_callback_t<basic_json>;\n\n    //////////////////\n    // constructors //\n    //////////////////\n\n    /// @name constructors and destructors\n    /// Constructors of class @ref basic_json, copy/move constructor, copy\n    /// assignment, static functions creating objects, and the destructor.\n    /// @{\n\n    /// @brief create an empty value with a given type\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(const value_t v)\n        : m_type(v), m_value(v)\n    {\n        assert_invariant();\n    }\n\n    /// @brief create a null object\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(std::nullptr_t = nullptr) noexcept // NOLINT(bugprone-exception-escape)\n        : basic_json(value_t::null)\n    {\n        assert_invariant();\n    }\n\n    /// @brief create a JSON value from compatible types\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    template < typename CompatibleType,\n               typename U = detail::uncvref_t<CompatibleType>,\n               detail::enable_if_t <\n                   !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value, int > = 0 >\n    basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape)\n                JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),\n                                           std::forward<CompatibleType>(val))))\n    {\n        JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief create a JSON value from an existing one\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    template < typename BasicJsonType,\n               detail::enable_if_t <\n                   detail::is_basic_json<BasicJsonType>::value&& !std::is_same<basic_json, BasicJsonType>::value, int > = 0 >\n    basic_json(const BasicJsonType& val)\n    {\n        using other_boolean_t = typename BasicJsonType::boolean_t;\n        using other_number_float_t = typename BasicJsonType::number_float_t;\n        using other_number_integer_t = typename BasicJsonType::number_integer_t;\n        using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t;\n        using other_string_t = typename BasicJsonType::string_t;\n        using other_object_t = typename BasicJsonType::object_t;\n        using other_array_t = typename BasicJsonType::array_t;\n        using other_binary_t = typename BasicJsonType::binary_t;\n\n        switch (val.type())\n        {\n            case value_t::boolean:\n                JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>());\n                break;\n            case value_t::number_float:\n                JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>());\n                break;\n            case value_t::number_integer:\n                JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>());\n                break;\n            case value_t::number_unsigned:\n                JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>());\n                break;\n            case value_t::string:\n                JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>());\n                break;\n            case value_t::object:\n                JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>());\n                break;\n            case value_t::array:\n                JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>());\n                break;\n            case value_t::binary:\n                JSONSerializer<other_binary_t>::to_json(*this, val.template get_ref<const other_binary_t&>());\n                break;\n            case value_t::null:\n                *this = nullptr;\n                break;\n            case value_t::discarded:\n                m_type = value_t::discarded;\n                break;\n            default:            // LCOV_EXCL_LINE\n                JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n        }\n        JSON_ASSERT(m_type == val.type());\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief create a container (array or object) from an initializer list\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(initializer_list_t init,\n               bool type_deduction = true,\n               value_t manual_type = value_t::array)\n    {\n        // check if each element is an array with two elements whose first\n        // element is a string\n        bool is_an_object = std::all_of(init.begin(), init.end(),\n                                        [](const detail::json_ref<basic_json>& element_ref)\n        {\n            return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string();\n        });\n\n        // adjust type if type deduction is not wanted\n        if (!type_deduction)\n        {\n            // if array is wanted, do not create an object though possible\n            if (manual_type == value_t::array)\n            {\n                is_an_object = false;\n            }\n\n            // if object is wanted but impossible, throw an exception\n            if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object))\n            {\n                JSON_THROW(type_error::create(301, \"cannot create object from initializer list\", nullptr));\n            }\n        }\n\n        if (is_an_object)\n        {\n            // the initializer list is a list of pairs -> create object\n            m_type = value_t::object;\n            m_value = value_t::object;\n\n            for (auto& element_ref : init)\n            {\n                auto element = element_ref.moved_or_copied();\n                m_value.object->emplace(\n                    std::move(*((*element.m_value.array)[0].m_value.string)),\n                    std::move((*element.m_value.array)[1]));\n            }\n        }\n        else\n        {\n            // the initializer list describes an array -> create array\n            m_type = value_t::array;\n            m_value.array = create<array_t>(init.begin(), init.end());\n        }\n\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief explicitly create a binary array (without subtype)\n    /// @sa https://json.nlohmann.me/api/basic_json/binary/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json binary(const typename binary_t::container_type& init)\n    {\n        auto res = basic_json();\n        res.m_type = value_t::binary;\n        res.m_value = init;\n        return res;\n    }\n\n    /// @brief explicitly create a binary array (with subtype)\n    /// @sa https://json.nlohmann.me/api/basic_json/binary/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype)\n    {\n        auto res = basic_json();\n        res.m_type = value_t::binary;\n        res.m_value = binary_t(init, subtype);\n        return res;\n    }\n\n    /// @brief explicitly create a binary array\n    /// @sa https://json.nlohmann.me/api/basic_json/binary/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json binary(typename binary_t::container_type&& init)\n    {\n        auto res = basic_json();\n        res.m_type = value_t::binary;\n        res.m_value = std::move(init);\n        return res;\n    }\n\n    /// @brief explicitly create a binary array (with subtype)\n    /// @sa https://json.nlohmann.me/api/basic_json/binary/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype)\n    {\n        auto res = basic_json();\n        res.m_type = value_t::binary;\n        res.m_value = binary_t(std::move(init), subtype);\n        return res;\n    }\n\n    /// @brief explicitly create an array from an initializer list\n    /// @sa https://json.nlohmann.me/api/basic_json/array/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json array(initializer_list_t init = {})\n    {\n        return basic_json(init, false, value_t::array);\n    }\n\n    /// @brief explicitly create an object from an initializer list\n    /// @sa https://json.nlohmann.me/api/basic_json/object/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json object(initializer_list_t init = {})\n    {\n        return basic_json(init, false, value_t::object);\n    }\n\n    /// @brief construct an array with count copies of given value\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(size_type cnt, const basic_json& val)\n        : m_type(value_t::array)\n    {\n        m_value.array = create<array_t>(cnt, val);\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief construct a JSON container given an iterator range\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    template < class InputIT, typename std::enable_if <\n                   std::is_same<InputIT, typename basic_json_t::iterator>::value ||\n                   std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int >::type = 0 >\n    basic_json(InputIT first, InputIT last)\n    {\n        JSON_ASSERT(first.m_object != nullptr);\n        JSON_ASSERT(last.m_object != nullptr);\n\n        // make sure iterator fits the current value\n        if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(201, \"iterators are not compatible\", nullptr));\n        }\n\n        // copy type from first iterator\n        m_type = first.m_object->m_type;\n\n        // check if iterator range is complete for primitive values\n        switch (m_type)\n        {\n            case value_t::boolean:\n            case value_t::number_float:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::string:\n            {\n                if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin()\n                                         || !last.m_it.primitive_iterator.is_end()))\n                {\n                    JSON_THROW(invalid_iterator::create(204, \"iterators out of range\", first.m_object));\n                }\n                break;\n            }\n\n            case value_t::null:\n            case value_t::object:\n            case value_t::array:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n                break;\n        }\n\n        switch (m_type)\n        {\n            case value_t::number_integer:\n            {\n                m_value.number_integer = first.m_object->m_value.number_integer;\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                m_value.number_unsigned = first.m_object->m_value.number_unsigned;\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                m_value.number_float = first.m_object->m_value.number_float;\n                break;\n            }\n\n            case value_t::boolean:\n            {\n                m_value.boolean = first.m_object->m_value.boolean;\n                break;\n            }\n\n            case value_t::string:\n            {\n                m_value = *first.m_object->m_value.string;\n                break;\n            }\n\n            case value_t::object:\n            {\n                m_value.object = create<object_t>(first.m_it.object_iterator,\n                                                  last.m_it.object_iterator);\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_value.array = create<array_t>(first.m_it.array_iterator,\n                                                last.m_it.array_iterator);\n                break;\n            }\n\n            case value_t::binary:\n            {\n                m_value = *first.m_object->m_value.binary;\n                break;\n            }\n\n            case value_t::null:\n            case value_t::discarded:\n            default:\n                JSON_THROW(invalid_iterator::create(206, detail::concat(\"cannot construct with iterators from \", first.m_object->type_name()), first.m_object));\n        }\n\n        set_parents();\n        assert_invariant();\n    }\n\n\n    ///////////////////////////////////////\n    // other constructors and destructor //\n    ///////////////////////////////////////\n\n    template<typename JsonRef,\n             detail::enable_if_t<detail::conjunction<detail::is_json_ref<JsonRef>,\n                                 std::is_same<typename JsonRef::value_type, basic_json>>::value, int> = 0 >\n    basic_json(const JsonRef& ref) : basic_json(ref.moved_or_copied()) {}\n\n    /// @brief copy constructor\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(const basic_json& other)\n        : m_type(other.m_type)\n    {\n        // check of passed value is valid\n        other.assert_invariant();\n\n        switch (m_type)\n        {\n            case value_t::object:\n            {\n                m_value = *other.m_value.object;\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_value = *other.m_value.array;\n                break;\n            }\n\n            case value_t::string:\n            {\n                m_value = *other.m_value.string;\n                break;\n            }\n\n            case value_t::boolean:\n            {\n                m_value = other.m_value.boolean;\n                break;\n            }\n\n            case value_t::number_integer:\n            {\n                m_value = other.m_value.number_integer;\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                m_value = other.m_value.number_unsigned;\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                m_value = other.m_value.number_float;\n                break;\n            }\n\n            case value_t::binary:\n            {\n                m_value = *other.m_value.binary;\n                break;\n            }\n\n            case value_t::null:\n            case value_t::discarded:\n            default:\n                break;\n        }\n\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief move constructor\n    /// @sa https://json.nlohmann.me/api/basic_json/basic_json/\n    basic_json(basic_json&& other) noexcept\n        : m_type(std::move(other.m_type)),\n          m_value(std::move(other.m_value))\n    {\n        // check that passed value is valid\n        other.assert_invariant(false);\n\n        // invalidate payload\n        other.m_type = value_t::null;\n        other.m_value = {};\n\n        set_parents();\n        assert_invariant();\n    }\n\n    /// @brief copy assignment\n    /// @sa https://json.nlohmann.me/api/basic_json/operator=/\n    basic_json& operator=(basic_json other) noexcept (\n        std::is_nothrow_move_constructible<value_t>::value&&\n        std::is_nothrow_move_assignable<value_t>::value&&\n        std::is_nothrow_move_constructible<json_value>::value&&\n        std::is_nothrow_move_assignable<json_value>::value\n    )\n    {\n        // check that passed value is valid\n        other.assert_invariant();\n\n        using std::swap;\n        swap(m_type, other.m_type);\n        swap(m_value, other.m_value);\n\n        set_parents();\n        assert_invariant();\n        return *this;\n    }\n\n    /// @brief destructor\n    /// @sa https://json.nlohmann.me/api/basic_json/~basic_json/\n    ~basic_json() noexcept\n    {\n        assert_invariant(false);\n        m_value.destroy(m_type);\n    }\n\n    /// @}\n\n  public:\n    ///////////////////////\n    // object inspection //\n    ///////////////////////\n\n    /// @name object inspection\n    /// Functions to inspect the type of a JSON value.\n    /// @{\n\n    /// @brief serialization\n    /// @sa https://json.nlohmann.me/api/basic_json/dump/\n    string_t dump(const int indent = -1,\n                  const char indent_char = ' ',\n                  const bool ensure_ascii = false,\n                  const error_handler_t error_handler = error_handler_t::strict) const\n    {\n        string_t result;\n        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);\n\n        if (indent >= 0)\n        {\n            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));\n        }\n        else\n        {\n            s.dump(*this, false, ensure_ascii, 0);\n        }\n\n        return result;\n    }\n\n    /// @brief return the type of the JSON value (explicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/type/\n    constexpr value_t type() const noexcept\n    {\n        return m_type;\n    }\n\n    /// @brief return whether type is primitive\n    /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/\n    constexpr bool is_primitive() const noexcept\n    {\n        return is_null() || is_string() || is_boolean() || is_number() || is_binary();\n    }\n\n    /// @brief return whether type is structured\n    /// @sa https://json.nlohmann.me/api/basic_json/is_structured/\n    constexpr bool is_structured() const noexcept\n    {\n        return is_array() || is_object();\n    }\n\n    /// @brief return whether value is null\n    /// @sa https://json.nlohmann.me/api/basic_json/is_null/\n    constexpr bool is_null() const noexcept\n    {\n        return m_type == value_t::null;\n    }\n\n    /// @brief return whether value is a boolean\n    /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/\n    constexpr bool is_boolean() const noexcept\n    {\n        return m_type == value_t::boolean;\n    }\n\n    /// @brief return whether value is a number\n    /// @sa https://json.nlohmann.me/api/basic_json/is_number/\n    constexpr bool is_number() const noexcept\n    {\n        return is_number_integer() || is_number_float();\n    }\n\n    /// @brief return whether value is an integer number\n    /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/\n    constexpr bool is_number_integer() const noexcept\n    {\n        return m_type == value_t::number_integer || m_type == value_t::number_unsigned;\n    }\n\n    /// @brief return whether value is an unsigned integer number\n    /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/\n    constexpr bool is_number_unsigned() const noexcept\n    {\n        return m_type == value_t::number_unsigned;\n    }\n\n    /// @brief return whether value is a floating-point number\n    /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/\n    constexpr bool is_number_float() const noexcept\n    {\n        return m_type == value_t::number_float;\n    }\n\n    /// @brief return whether value is an object\n    /// @sa https://json.nlohmann.me/api/basic_json/is_object/\n    constexpr bool is_object() const noexcept\n    {\n        return m_type == value_t::object;\n    }\n\n    /// @brief return whether value is an array\n    /// @sa https://json.nlohmann.me/api/basic_json/is_array/\n    constexpr bool is_array() const noexcept\n    {\n        return m_type == value_t::array;\n    }\n\n    /// @brief return whether value is a string\n    /// @sa https://json.nlohmann.me/api/basic_json/is_string/\n    constexpr bool is_string() const noexcept\n    {\n        return m_type == value_t::string;\n    }\n\n    /// @brief return whether value is a binary array\n    /// @sa https://json.nlohmann.me/api/basic_json/is_binary/\n    constexpr bool is_binary() const noexcept\n    {\n        return m_type == value_t::binary;\n    }\n\n    /// @brief return whether value is discarded\n    /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/\n    constexpr bool is_discarded() const noexcept\n    {\n        return m_type == value_t::discarded;\n    }\n\n    /// @brief return the type of the JSON value (implicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/\n    constexpr operator value_t() const noexcept\n    {\n        return m_type;\n    }\n\n    /// @}\n\n  private:\n    //////////////////\n    // value access //\n    //////////////////\n\n    /// get a boolean (explicit)\n    boolean_t get_impl(boolean_t* /*unused*/) const\n    {\n        if (JSON_HEDLEY_LIKELY(is_boolean()))\n        {\n            return m_value.boolean;\n        }\n\n        JSON_THROW(type_error::create(302, detail::concat(\"type must be boolean, but is \", type_name()), this));\n    }\n\n    /// get a pointer to the value (object)\n    object_t* get_impl_ptr(object_t* /*unused*/) noexcept\n    {\n        return is_object() ? m_value.object : nullptr;\n    }\n\n    /// get a pointer to the value (object)\n    constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept\n    {\n        return is_object() ? m_value.object : nullptr;\n    }\n\n    /// get a pointer to the value (array)\n    array_t* get_impl_ptr(array_t* /*unused*/) noexcept\n    {\n        return is_array() ? m_value.array : nullptr;\n    }\n\n    /// get a pointer to the value (array)\n    constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept\n    {\n        return is_array() ? m_value.array : nullptr;\n    }\n\n    /// get a pointer to the value (string)\n    string_t* get_impl_ptr(string_t* /*unused*/) noexcept\n    {\n        return is_string() ? m_value.string : nullptr;\n    }\n\n    /// get a pointer to the value (string)\n    constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept\n    {\n        return is_string() ? m_value.string : nullptr;\n    }\n\n    /// get a pointer to the value (boolean)\n    boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept\n    {\n        return is_boolean() ? &m_value.boolean : nullptr;\n    }\n\n    /// get a pointer to the value (boolean)\n    constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept\n    {\n        return is_boolean() ? &m_value.boolean : nullptr;\n    }\n\n    /// get a pointer to the value (integer number)\n    number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept\n    {\n        return is_number_integer() ? &m_value.number_integer : nullptr;\n    }\n\n    /// get a pointer to the value (integer number)\n    constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept\n    {\n        return is_number_integer() ? &m_value.number_integer : nullptr;\n    }\n\n    /// get a pointer to the value (unsigned number)\n    number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept\n    {\n        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;\n    }\n\n    /// get a pointer to the value (unsigned number)\n    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept\n    {\n        return is_number_unsigned() ? &m_value.number_unsigned : nullptr;\n    }\n\n    /// get a pointer to the value (floating-point number)\n    number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept\n    {\n        return is_number_float() ? &m_value.number_float : nullptr;\n    }\n\n    /// get a pointer to the value (floating-point number)\n    constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept\n    {\n        return is_number_float() ? &m_value.number_float : nullptr;\n    }\n\n    /// get a pointer to the value (binary)\n    binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept\n    {\n        return is_binary() ? m_value.binary : nullptr;\n    }\n\n    /// get a pointer to the value (binary)\n    constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept\n    {\n        return is_binary() ? m_value.binary : nullptr;\n    }\n\n    /*!\n    @brief helper function to implement get_ref()\n\n    This function helps to implement get_ref() without code duplication for\n    const and non-const overloads\n\n    @tparam ThisType will be deduced as `basic_json` or `const basic_json`\n\n    @throw type_error.303 if ReferenceType does not match underlying value\n    type of the current JSON\n    */\n    template<typename ReferenceType, typename ThisType>\n    static ReferenceType get_ref_impl(ThisType& obj)\n    {\n        // delegate the call to get_ptr<>()\n        auto* ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();\n\n        if (JSON_HEDLEY_LIKELY(ptr != nullptr))\n        {\n            return *ptr;\n        }\n\n        JSON_THROW(type_error::create(303, detail::concat(\"incompatible ReferenceType for get_ref, actual type is \", obj.type_name()), &obj));\n    }\n\n  public:\n    /// @name value access\n    /// Direct access to the stored value of a JSON value.\n    /// @{\n\n    /// @brief get a pointer value (implicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/\n    template<typename PointerType, typename std::enable_if<\n                 std::is_pointer<PointerType>::value, int>::type = 0>\n    auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))\n    {\n        // delegate the call to get_impl_ptr<>()\n        return get_impl_ptr(static_cast<PointerType>(nullptr));\n    }\n\n    /// @brief get a pointer value (implicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/\n    template < typename PointerType, typename std::enable_if <\n                   std::is_pointer<PointerType>::value&&\n                   std::is_const<typename std::remove_pointer<PointerType>::type>::value, int >::type = 0 >\n    constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>()))\n    {\n        // delegate the call to get_impl_ptr<>() const\n        return get_impl_ptr(static_cast<PointerType>(nullptr));\n    }\n\n  private:\n    /*!\n    @brief get a value (explicit)\n\n    Explicit type conversion between the JSON value and a compatible value\n    which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)\n    and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).\n    The value is converted by calling the @ref json_serializer<ValueType>\n    `from_json()` method.\n\n    The function is equivalent to executing\n    @code {.cpp}\n    ValueType ret;\n    JSONSerializer<ValueType>::from_json(*this, ret);\n    return ret;\n    @endcode\n\n    This overloads is chosen if:\n    - @a ValueType is not @ref basic_json,\n    - @ref json_serializer<ValueType> has a `from_json()` method of the form\n      `void from_json(const basic_json&, ValueType&)`, and\n    - @ref json_serializer<ValueType> does not have a `from_json()` method of\n      the form `ValueType from_json(const basic_json&)`\n\n    @tparam ValueType the returned value type\n\n    @return copy of the JSON value, converted to @a ValueType\n\n    @throw what @ref json_serializer<ValueType> `from_json()` method throws\n\n    @liveexample{The example below shows several conversions from JSON values\n    to other types. There a few things to note: (1) Floating-point numbers can\n    be converted to integers\\, (2) A JSON array can be converted to a standard\n    `std::vector<short>`\\, (3) A JSON object can be converted to C++\n    associative containers such as `std::unordered_map<std::string\\,\n    json>`.,get__ValueType_const}\n\n    @since version 2.1.0\n    */\n    template < typename ValueType,\n               detail::enable_if_t <\n                   detail::is_default_constructible<ValueType>::value&&\n                   detail::has_from_json<basic_json_t, ValueType>::value,\n                   int > = 0 >\n    ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept(\n                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))\n    {\n        auto ret = ValueType();\n        JSONSerializer<ValueType>::from_json(*this, ret);\n        return ret;\n    }\n\n    /*!\n    @brief get a value (explicit); special case\n\n    Explicit type conversion between the JSON value and a compatible value\n    which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible)\n    and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible).\n    The value is converted by calling the @ref json_serializer<ValueType>\n    `from_json()` method.\n\n    The function is equivalent to executing\n    @code {.cpp}\n    return JSONSerializer<ValueType>::from_json(*this);\n    @endcode\n\n    This overloads is chosen if:\n    - @a ValueType is not @ref basic_json and\n    - @ref json_serializer<ValueType> has a `from_json()` method of the form\n      `ValueType from_json(const basic_json&)`\n\n    @note If @ref json_serializer<ValueType> has both overloads of\n    `from_json()`, this one is chosen.\n\n    @tparam ValueType the returned value type\n\n    @return copy of the JSON value, converted to @a ValueType\n\n    @throw what @ref json_serializer<ValueType> `from_json()` method throws\n\n    @since version 2.1.0\n    */\n    template < typename ValueType,\n               detail::enable_if_t <\n                   detail::has_non_default_from_json<basic_json_t, ValueType>::value,\n                   int > = 0 >\n    ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept(\n                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>())))\n    {\n        return JSONSerializer<ValueType>::from_json(*this);\n    }\n\n    /*!\n    @brief get special-case overload\n\n    This overloads converts the current @ref basic_json in a different\n    @ref basic_json type\n\n    @tparam BasicJsonType == @ref basic_json\n\n    @return a copy of *this, converted into @a BasicJsonType\n\n    @complexity Depending on the implementation of the called `from_json()`\n                method.\n\n    @since version 3.2.0\n    */\n    template < typename BasicJsonType,\n               detail::enable_if_t <\n                   detail::is_basic_json<BasicJsonType>::value,\n                   int > = 0 >\n    BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const\n    {\n        return *this;\n    }\n\n    /*!\n    @brief get special-case overload\n\n    This overloads avoids a lot of template boilerplate, it can be seen as the\n    identity method\n\n    @tparam BasicJsonType == @ref basic_json\n\n    @return a copy of *this\n\n    @complexity Constant.\n\n    @since version 2.1.0\n    */\n    template<typename BasicJsonType,\n             detail::enable_if_t<\n                 std::is_same<BasicJsonType, basic_json_t>::value,\n                 int> = 0>\n    basic_json get_impl(detail::priority_tag<3> /*unused*/) const\n    {\n        return *this;\n    }\n\n    /*!\n    @brief get a pointer value (explicit)\n    @copydoc get()\n    */\n    template<typename PointerType,\n             detail::enable_if_t<\n                 std::is_pointer<PointerType>::value,\n                 int> = 0>\n    constexpr auto get_impl(detail::priority_tag<4> /*unused*/) const noexcept\n    -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>())\n    {\n        // delegate the call to get_ptr\n        return get_ptr<PointerType>();\n    }\n\n  public:\n    /*!\n    @brief get a (pointer) value (explicit)\n\n    Performs explicit type conversion between the JSON value and a compatible value if required.\n\n    - If the requested type is a pointer to the internally stored JSON value that pointer is returned.\n    No copies are made.\n\n    - If the requested type is the current @ref basic_json, or a different @ref basic_json convertible\n    from the current @ref basic_json.\n\n    - Otherwise the value is converted by calling the @ref json_serializer<ValueType> `from_json()`\n    method.\n\n    @tparam ValueTypeCV the provided value type\n    @tparam ValueType the returned value type\n\n    @return copy of the JSON value, converted to @tparam ValueType if necessary\n\n    @throw what @ref json_serializer<ValueType> `from_json()` method throws if conversion is required\n\n    @since version 2.1.0\n    */\n    template < typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>>\n#if defined(JSON_HAS_CPP_14)\n    constexpr\n#endif\n    auto get() const noexcept(\n    noexcept(std::declval<const basic_json_t&>().template get_impl<ValueType>(detail::priority_tag<4> {})))\n    -> decltype(std::declval<const basic_json_t&>().template get_impl<ValueType>(detail::priority_tag<4> {}))\n    {\n        // we cannot static_assert on ValueTypeCV being non-const, because\n        // there is support for get<const basic_json_t>(), which is why we\n        // still need the uncvref\n        static_assert(!std::is_reference<ValueTypeCV>::value,\n                      \"get() cannot be used with reference types, you might want to use get_ref()\");\n        return get_impl<ValueType>(detail::priority_tag<4> {});\n    }\n\n    /*!\n    @brief get a pointer value (explicit)\n\n    Explicit pointer access to the internally stored JSON value. No copies are\n    made.\n\n    @warning The pointer becomes invalid if the underlying JSON object\n    changes.\n\n    @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref\n    object_t, @ref string_t, @ref boolean_t, @ref number_integer_t,\n    @ref number_unsigned_t, or @ref number_float_t.\n\n    @return pointer to the internally stored JSON value if the requested\n    pointer type @a PointerType fits to the JSON value; `nullptr` otherwise\n\n    @complexity Constant.\n\n    @liveexample{The example below shows how pointers to internal values of a\n    JSON value can be requested. Note that no type conversions are made and a\n    `nullptr` is returned if the value and the requested pointer type does not\n    match.,get__PointerType}\n\n    @sa see @ref get_ptr() for explicit pointer-member access\n\n    @since version 1.0.0\n    */\n    template<typename PointerType, typename std::enable_if<\n                 std::is_pointer<PointerType>::value, int>::type = 0>\n    auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>())\n    {\n        // delegate the call to get_ptr\n        return get_ptr<PointerType>();\n    }\n\n    /// @brief get a value (explicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/get_to/\n    template < typename ValueType,\n               detail::enable_if_t <\n                   !detail::is_basic_json<ValueType>::value&&\n                   detail::has_from_json<basic_json_t, ValueType>::value,\n                   int > = 0 >\n    ValueType & get_to(ValueType& v) const noexcept(noexcept(\n                JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v)))\n    {\n        JSONSerializer<ValueType>::from_json(*this, v);\n        return v;\n    }\n\n    // specialization to allow calling get_to with a basic_json value\n    // see https://github.com/nlohmann/json/issues/2175\n    template<typename ValueType,\n             detail::enable_if_t <\n                 detail::is_basic_json<ValueType>::value,\n                 int> = 0>\n    ValueType & get_to(ValueType& v) const\n    {\n        v = *this;\n        return v;\n    }\n\n    template <\n        typename T, std::size_t N,\n        typename Array = T (&)[N], // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n        detail::enable_if_t <\n            detail::has_from_json<basic_json_t, Array>::value, int > = 0 >\n    Array get_to(T (&v)[N]) const // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)\n    noexcept(noexcept(JSONSerializer<Array>::from_json(\n                          std::declval<const basic_json_t&>(), v)))\n    {\n        JSONSerializer<Array>::from_json(*this, v);\n        return v;\n    }\n\n    /// @brief get a reference value (implicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/get_ref/\n    template<typename ReferenceType, typename std::enable_if<\n                 std::is_reference<ReferenceType>::value, int>::type = 0>\n    ReferenceType get_ref()\n    {\n        // delegate call to get_ref_impl\n        return get_ref_impl<ReferenceType>(*this);\n    }\n\n    /// @brief get a reference value (implicit)\n    /// @sa https://json.nlohmann.me/api/basic_json/get_ref/\n    template < typename ReferenceType, typename std::enable_if <\n                   std::is_reference<ReferenceType>::value&&\n                   std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int >::type = 0 >\n    ReferenceType get_ref() const\n    {\n        // delegate call to get_ref_impl\n        return get_ref_impl<ReferenceType>(*this);\n    }\n\n    /*!\n    @brief get a value (implicit)\n\n    Implicit type conversion between the JSON value and a compatible value.\n    The call is realized by calling @ref get() const.\n\n    @tparam ValueType non-pointer type compatible to the JSON value, for\n    instance `int` for JSON integer numbers, `bool` for JSON booleans, or\n    `std::vector` types for JSON arrays. The character type of @ref string_t\n    as well as an initializer list of this type is excluded to avoid\n    ambiguities as these types implicitly convert to `std::string`.\n\n    @return copy of the JSON value, converted to type @a ValueType\n\n    @throw type_error.302 in case passed type @a ValueType is incompatible\n    to the JSON value type (e.g., the JSON value is of type boolean, but a\n    string is requested); see example below\n\n    @complexity Linear in the size of the JSON value.\n\n    @liveexample{The example below shows several conversions from JSON values\n    to other types. There a few things to note: (1) Floating-point numbers can\n    be converted to integers\\, (2) A JSON array can be converted to a standard\n    `std::vector<short>`\\, (3) A JSON object can be converted to C++\n    associative containers such as `std::unordered_map<std::string\\,\n    json>`.,operator__ValueType}\n\n    @since version 1.0.0\n    */\n    template < typename ValueType, typename std::enable_if <\n                   detail::conjunction <\n                       detail::negation<std::is_pointer<ValueType>>,\n                       detail::negation<std::is_same<ValueType, std::nullptr_t>>,\n                       detail::negation<std::is_same<ValueType, detail::json_ref<basic_json>>>,\n                                        detail::negation<std::is_same<ValueType, typename string_t::value_type>>,\n                                        detail::negation<detail::is_basic_json<ValueType>>,\n                                        detail::negation<std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>>,\n#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914))\n                                                detail::negation<std::is_same<ValueType, std::string_view>>,\n#endif\n#if defined(JSON_HAS_CPP_17)\n                                                detail::negation<std::is_same<ValueType, std::any>>,\n#endif\n                                                detail::is_detected_lazy<detail::get_template_function, const basic_json_t&, ValueType>\n                                                >::value, int >::type = 0 >\n                                        JSON_EXPLICIT operator ValueType() const\n    {\n        // delegate the call to get<>() const\n        return get<ValueType>();\n    }\n\n    /// @brief get a binary value\n    /// @sa https://json.nlohmann.me/api/basic_json/get_binary/\n    binary_t& get_binary()\n    {\n        if (!is_binary())\n        {\n            JSON_THROW(type_error::create(302, detail::concat(\"type must be binary, but is \", type_name()), this));\n        }\n\n        return *get_ptr<binary_t*>();\n    }\n\n    /// @brief get a binary value\n    /// @sa https://json.nlohmann.me/api/basic_json/get_binary/\n    const binary_t& get_binary() const\n    {\n        if (!is_binary())\n        {\n            JSON_THROW(type_error::create(302, detail::concat(\"type must be binary, but is \", type_name()), this));\n        }\n\n        return *get_ptr<const binary_t*>();\n    }\n\n    /// @}\n\n\n    ////////////////////\n    // element access //\n    ////////////////////\n\n    /// @name element access\n    /// Access to the JSON value.\n    /// @{\n\n    /// @brief access specified array element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    reference at(size_type idx)\n    {\n        // at only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            JSON_TRY\n            {\n                return set_parent(m_value.array->at(idx));\n            }\n            JSON_CATCH (std::out_of_range&)\n            {\n                // create better exception explanation\n                JSON_THROW(out_of_range::create(401, detail::concat(\"array index \", std::to_string(idx), \" is out of range\"), this));\n            }\n        }\n        else\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n    }\n\n    /// @brief access specified array element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    const_reference at(size_type idx) const\n    {\n        // at only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            JSON_TRY\n            {\n                return m_value.array->at(idx);\n            }\n            JSON_CATCH (std::out_of_range&)\n            {\n                // create better exception explanation\n                JSON_THROW(out_of_range::create(401, detail::concat(\"array index \", std::to_string(idx), \" is out of range\"), this));\n            }\n        }\n        else\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n    }\n\n    /// @brief access specified object element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    reference at(const typename object_t::key_type& key)\n    {\n        // at only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n\n        auto it = m_value.object->find(key);\n        if (it == m_value.object->end())\n        {\n            JSON_THROW(out_of_range::create(403, detail::concat(\"key '\", key, \"' not found\"), this));\n        }\n        return set_parent(it->second);\n    }\n\n    /// @brief access specified object element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    reference at(KeyType && key)\n    {\n        // at only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n\n        auto it = m_value.object->find(std::forward<KeyType>(key));\n        if (it == m_value.object->end())\n        {\n            JSON_THROW(out_of_range::create(403, detail::concat(\"key '\", string_t(std::forward<KeyType>(key)), \"' not found\"), this));\n        }\n        return set_parent(it->second);\n    }\n\n    /// @brief access specified object element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    const_reference at(const typename object_t::key_type& key) const\n    {\n        // at only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n\n        auto it = m_value.object->find(key);\n        if (it == m_value.object->end())\n        {\n            JSON_THROW(out_of_range::create(403, detail::concat(\"key '\", key, \"' not found\"), this));\n        }\n        return it->second;\n    }\n\n    /// @brief access specified object element with bounds checking\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    const_reference at(KeyType && key) const\n    {\n        // at only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(304, detail::concat(\"cannot use at() with \", type_name()), this));\n        }\n\n        auto it = m_value.object->find(std::forward<KeyType>(key));\n        if (it == m_value.object->end())\n        {\n            JSON_THROW(out_of_range::create(403, detail::concat(\"key '\", string_t(std::forward<KeyType>(key)), \"' not found\"), this));\n        }\n        return it->second;\n    }\n\n    /// @brief access specified array element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    reference operator[](size_type idx)\n    {\n        // implicitly convert null value to an empty array\n        if (is_null())\n        {\n            m_type = value_t::array;\n            m_value.array = create<array_t>();\n            assert_invariant();\n        }\n\n        // operator[] only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            // fill up array with null values if given idx is outside range\n            if (idx >= m_value.array->size())\n            {\n#if JSON_DIAGNOSTICS\n                // remember array size & capacity before resizing\n                const auto old_size = m_value.array->size();\n                const auto old_capacity = m_value.array->capacity();\n#endif\n                m_value.array->resize(idx + 1);\n\n#if JSON_DIAGNOSTICS\n                if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity))\n                {\n                    // capacity has changed: update all parents\n                    set_parents();\n                }\n                else\n                {\n                    // set parent for values added above\n                    set_parents(begin() + static_cast<typename iterator::difference_type>(old_size), static_cast<typename iterator::difference_type>(idx + 1 - old_size));\n                }\n#endif\n                assert_invariant();\n            }\n\n            return m_value.array->operator[](idx);\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a numeric argument with \", type_name()), this));\n    }\n\n    /// @brief access specified array element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    const_reference operator[](size_type idx) const\n    {\n        // const operator[] only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            return m_value.array->operator[](idx);\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a numeric argument with \", type_name()), this));\n    }\n\n    /// @brief access specified object element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    reference operator[](typename object_t::key_type key)\n    {\n        // implicitly convert null value to an empty object\n        if (is_null())\n        {\n            m_type = value_t::object;\n            m_value.object = create<object_t>();\n            assert_invariant();\n        }\n\n        // operator[] only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            auto result = m_value.object->emplace(std::move(key), nullptr);\n            return set_parent(result.first->second);\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a string argument with \", type_name()), this));\n    }\n\n    /// @brief access specified object element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    const_reference operator[](const typename object_t::key_type& key) const\n    {\n        // const operator[] only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            auto it = m_value.object->find(key);\n            JSON_ASSERT(it != m_value.object->end());\n            return it->second;\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a string argument with \", type_name()), this));\n    }\n\n    // these two functions resolve a (const) char * ambiguity affecting Clang and MSVC\n    // (they seemingly cannot be constrained to resolve the ambiguity)\n    template<typename T>\n    reference operator[](T* key)\n    {\n        return operator[](typename object_t::key_type(key));\n    }\n\n    template<typename T>\n    const_reference operator[](T* key) const\n    {\n        return operator[](typename object_t::key_type(key));\n    }\n\n    /// @brief access specified object element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int > = 0 >\n    reference operator[](KeyType && key)\n    {\n        // implicitly convert null value to an empty object\n        if (is_null())\n        {\n            m_type = value_t::object;\n            m_value.object = create<object_t>();\n            assert_invariant();\n        }\n\n        // operator[] only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            auto result = m_value.object->emplace(std::forward<KeyType>(key), nullptr);\n            return set_parent(result.first->second);\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a string argument with \", type_name()), this));\n    }\n\n    /// @brief access specified object element\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int > = 0 >\n    const_reference operator[](KeyType && key) const\n    {\n        // const operator[] only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            auto it = m_value.object->find(std::forward<KeyType>(key));\n            JSON_ASSERT(it != m_value.object->end());\n            return it->second;\n        }\n\n        JSON_THROW(type_error::create(305, detail::concat(\"cannot use operator[] with a string argument with \", type_name()), this));\n    }\n\n  private:\n    template<typename KeyType>\n    using is_comparable_with_object_key = detail::is_comparable <\n        object_comparator_t, const typename object_t::key_type&, KeyType >;\n\n    template<typename ValueType>\n    using value_return_type = std::conditional <\n        detail::is_c_string_uncvref<ValueType>::value,\n        string_t, typename std::decay<ValueType>::type >;\n\n  public:\n    /// @brief access specified object element with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, detail::enable_if_t <\n                   !detail::is_transparent<object_comparator_t>::value\n                   && detail::is_getable<basic_json_t, ValueType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if key is found, return value and given default value otherwise\n            const auto it = find(key);\n            if (it != end())\n            {\n                return it->template get<ValueType>();\n            }\n\n            return default_value;\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    /// @brief access specified object element with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, class ReturnType = typename value_return_type<ValueType>::type,\n               detail::enable_if_t <\n                   !detail::is_transparent<object_comparator_t>::value\n                   && detail::is_getable<basic_json_t, ReturnType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ReturnType value(const typename object_t::key_type& key, ValueType && default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if key is found, return value and given default value otherwise\n            const auto it = find(key);\n            if (it != end())\n            {\n                return it->template get<ReturnType>();\n            }\n\n            return std::forward<ValueType>(default_value);\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    /// @brief access specified object element with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, class KeyType, detail::enable_if_t <\n                   detail::is_transparent<object_comparator_t>::value\n                   && !detail::is_json_pointer<KeyType>::value\n                   && is_comparable_with_object_key<KeyType>::value\n                   && detail::is_getable<basic_json_t, ValueType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ValueType value(KeyType && key, const ValueType& default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if key is found, return value and given default value otherwise\n            const auto it = find(std::forward<KeyType>(key));\n            if (it != end())\n            {\n                return it->template get<ValueType>();\n            }\n\n            return default_value;\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    /// @brief access specified object element via JSON Pointer with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, class KeyType, class ReturnType = typename value_return_type<ValueType>::type,\n               detail::enable_if_t <\n                   detail::is_transparent<object_comparator_t>::value\n                   && !detail::is_json_pointer<KeyType>::value\n                   && is_comparable_with_object_key<KeyType>::value\n                   && detail::is_getable<basic_json_t, ReturnType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ReturnType value(KeyType && key, ValueType && default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if key is found, return value and given default value otherwise\n            const auto it = find(std::forward<KeyType>(key));\n            if (it != end())\n            {\n                return it->template get<ReturnType>();\n            }\n\n            return std::forward<ValueType>(default_value);\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    /// @brief access specified object element via JSON Pointer with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, detail::enable_if_t <\n                   detail::is_getable<basic_json_t, ValueType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ValueType value(const json_pointer& ptr, const ValueType& default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if pointer resolves a value, return it or use default value\n            JSON_TRY\n            {\n                return ptr.get_checked(this).template get<ValueType>();\n            }\n            JSON_INTERNAL_CATCH (out_of_range&)\n            {\n                return default_value;\n            }\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    /// @brief access specified object element via JSON Pointer with default value\n    /// @sa https://json.nlohmann.me/api/basic_json/value/\n    template < class ValueType, class ReturnType = typename value_return_type<ValueType>::type,\n               detail::enable_if_t <\n                   detail::is_getable<basic_json_t, ReturnType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    ReturnType value(const json_pointer& ptr, ValueType && default_value) const\n    {\n        // value only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            // if pointer resolves a value, return it or use default value\n            JSON_TRY\n            {\n                return ptr.get_checked(this).template get<ReturnType>();\n            }\n            JSON_INTERNAL_CATCH (out_of_range&)\n            {\n                return std::forward<ValueType>(default_value);\n            }\n        }\n\n        JSON_THROW(type_error::create(306, detail::concat(\"cannot use value() with \", type_name()), this));\n    }\n\n    template < class ValueType, class BasicJsonType, detail::enable_if_t <\n                   detail::is_basic_json<BasicJsonType>::value\n                   && detail::is_getable<basic_json_t, ValueType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    ValueType value(const ::nlohmann::json_pointer<BasicJsonType>& ptr, const ValueType& default_value) const\n    {\n        return value(ptr.convert(), default_value);\n    }\n\n    template < class ValueType, class BasicJsonType, class ReturnType = typename value_return_type<ValueType>::type,\n               detail::enable_if_t <\n                   detail::is_basic_json<BasicJsonType>::value\n                   && detail::is_getable<basic_json_t, ReturnType>::value\n                   && !std::is_same<value_t, detail::uncvref_t<ValueType>>::value, int > = 0 >\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    ReturnType value(const ::nlohmann::json_pointer<BasicJsonType>& ptr, ValueType && default_value) const\n    {\n        return value(ptr.convert(), std::forward<ValueType>(default_value));\n    }\n\n    /// @brief access the first element\n    /// @sa https://json.nlohmann.me/api/basic_json/front/\n    reference front()\n    {\n        return *begin();\n    }\n\n    /// @brief access the first element\n    /// @sa https://json.nlohmann.me/api/basic_json/front/\n    const_reference front() const\n    {\n        return *cbegin();\n    }\n\n    /// @brief access the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/back/\n    reference back()\n    {\n        auto tmp = end();\n        --tmp;\n        return *tmp;\n    }\n\n    /// @brief access the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/back/\n    const_reference back() const\n    {\n        auto tmp = cend();\n        --tmp;\n        return *tmp;\n    }\n\n    /// @brief remove element given an iterator\n    /// @sa https://json.nlohmann.me/api/basic_json/erase/\n    template < class IteratorType, detail::enable_if_t <\n                   std::is_same<IteratorType, typename basic_json_t::iterator>::value ||\n                   std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int > = 0 >\n    IteratorType erase(IteratorType pos)\n    {\n        // make sure iterator fits the current value\n        if (JSON_HEDLEY_UNLIKELY(this != pos.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(202, \"iterator does not fit current value\", this));\n        }\n\n        IteratorType result = end();\n\n        switch (m_type)\n        {\n            case value_t::boolean:\n            case value_t::number_float:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::string:\n            case value_t::binary:\n            {\n                if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin()))\n                {\n                    JSON_THROW(invalid_iterator::create(205, \"iterator out of range\", this));\n                }\n\n                if (is_string())\n                {\n                    AllocatorType<string_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);\n                    m_value.string = nullptr;\n                }\n                else if (is_binary())\n                {\n                    AllocatorType<binary_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.binary);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.binary, 1);\n                    m_value.binary = nullptr;\n                }\n\n                m_type = value_t::null;\n                assert_invariant();\n                break;\n            }\n\n            case value_t::object:\n            {\n                result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator);\n                break;\n            }\n\n            case value_t::array:\n            {\n                result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::discarded:\n            default:\n                JSON_THROW(type_error::create(307, detail::concat(\"cannot use erase() with \", type_name()), this));\n        }\n\n        return result;\n    }\n\n    /// @brief remove elements given an iterator range\n    /// @sa https://json.nlohmann.me/api/basic_json/erase/\n    template < class IteratorType, detail::enable_if_t <\n                   std::is_same<IteratorType, typename basic_json_t::iterator>::value ||\n                   std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int > = 0 >\n    IteratorType erase(IteratorType first, IteratorType last)\n    {\n        // make sure iterator fits the current value\n        if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(203, \"iterators do not fit current value\", this));\n        }\n\n        IteratorType result = end();\n\n        switch (m_type)\n        {\n            case value_t::boolean:\n            case value_t::number_float:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::string:\n            case value_t::binary:\n            {\n                if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin()\n                                       || !last.m_it.primitive_iterator.is_end()))\n                {\n                    JSON_THROW(invalid_iterator::create(204, \"iterators out of range\", this));\n                }\n\n                if (is_string())\n                {\n                    AllocatorType<string_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1);\n                    m_value.string = nullptr;\n                }\n                else if (is_binary())\n                {\n                    AllocatorType<binary_t> alloc;\n                    std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.binary);\n                    std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.binary, 1);\n                    m_value.binary = nullptr;\n                }\n\n                m_type = value_t::null;\n                assert_invariant();\n                break;\n            }\n\n            case value_t::object:\n            {\n                result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator,\n                                              last.m_it.object_iterator);\n                break;\n            }\n\n            case value_t::array:\n            {\n                result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator,\n                                             last.m_it.array_iterator);\n                break;\n            }\n\n            case value_t::null:\n            case value_t::discarded:\n            default:\n                JSON_THROW(type_error::create(307, detail::concat(\"cannot use erase() with \", type_name()), this));\n        }\n\n        return result;\n    }\n\n  private:\n    template < typename KeyType, detail::enable_if_t <\n                   detail::has_erase_with_key_type<basic_json_t, KeyType>::value, int > = 0 >\n    size_type erase_internal(KeyType && key)\n    {\n        // this erase only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(307, detail::concat(\"cannot use erase() with \", type_name()), this));\n        }\n\n        return m_value.object->erase(std::forward<KeyType>(key));\n    }\n\n    template < typename KeyType, detail::enable_if_t <\n                   !detail::has_erase_with_key_type<basic_json_t, KeyType>::value, int > = 0 >\n    size_type erase_internal(KeyType && key)\n    {\n        // this erase only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(307, detail::concat(\"cannot use erase() with \", type_name()), this));\n        }\n\n        const auto it = m_value.object->find(std::forward<KeyType>(key));\n        if (it != m_value.object->end())\n        {\n            m_value.object->erase(it);\n            return 1;\n        }\n        return 0;\n    }\n\n  public:\n\n    /// @brief remove element from a JSON object given a key\n    /// @sa https://json.nlohmann.me/api/basic_json/erase/\n    size_type erase(const typename object_t::key_type& key)\n    {\n        // the indirection via erase_internal() is added to avoid making this\n        // function a template and thus de-rank it during overload resolution\n        return erase_internal(key);\n    }\n\n    /// @brief remove element from a JSON object given a key\n    /// @sa https://json.nlohmann.me/api/basic_json/erase/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    size_type erase(KeyType && key)\n    {\n        return erase_internal(std::forward<KeyType>(key));\n    }\n\n    /// @brief remove element from a JSON array given an index\n    /// @sa https://json.nlohmann.me/api/basic_json/erase/\n    void erase(const size_type idx)\n    {\n        // this erase only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            if (JSON_HEDLEY_UNLIKELY(idx >= size()))\n            {\n                JSON_THROW(out_of_range::create(401, detail::concat(\"array index \", std::to_string(idx), \" is out of range\"), this));\n            }\n\n            m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));\n        }\n        else\n        {\n            JSON_THROW(type_error::create(307, detail::concat(\"cannot use erase() with \", type_name()), this));\n        }\n    }\n\n    /// @}\n\n\n    ////////////\n    // lookup //\n    ////////////\n\n    /// @name lookup\n    /// @{\n\n    /// @brief find an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/find/\n    iterator find(const typename object_t::key_type& key)\n    {\n        auto result = end();\n\n        if (is_object())\n        {\n            result.m_it.object_iterator = m_value.object->find(key);\n        }\n\n        return result;\n    }\n\n    /// @brief find an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/find/\n    const_iterator find(const typename object_t::key_type& key) const\n    {\n        auto result = cend();\n\n        if (is_object())\n        {\n            result.m_it.object_iterator = m_value.object->find(key);\n        }\n\n        return result;\n    }\n\n    /// @brief find an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/find/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    iterator find(KeyType && key)\n    {\n        auto result = end();\n\n        if (is_object())\n        {\n            result.m_it.object_iterator = m_value.object->find(std::forward<KeyType>(key));\n        }\n\n        return result;\n    }\n\n    /// @brief find an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/find/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    const_iterator find(KeyType && key) const\n    {\n        auto result = cend();\n\n        if (is_object())\n        {\n            result.m_it.object_iterator = m_value.object->find(std::forward<KeyType>(key));\n        }\n\n        return result;\n    }\n\n    /// @brief returns the number of occurrences of a key in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/count/\n    size_type count(const typename object_t::key_type& key) const\n    {\n        // return 0 for all nonobject types\n        return is_object() ? m_value.object->count(key) : 0;\n    }\n\n    /// @brief returns the number of occurrences of a key in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/count/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    size_type count(KeyType && key) const\n    {\n        // return 0 for all nonobject types\n        return is_object() ? m_value.object->count(std::forward<KeyType>(key)) : 0;\n    }\n\n    /// @brief check the existence of an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/contains/\n    bool contains(const typename object_t::key_type& key) const\n    {\n        return is_object() && m_value.object->find(key) != m_value.object->end();\n    }\n\n    /// @brief check the existence of an element in a JSON object\n    /// @sa https://json.nlohmann.me/api/basic_json/contains/\n    template<class KeyType, detail::enable_if_t<\n                 detail::is_usable_as_basic_json_key_type<basic_json_t, KeyType>::value, int> = 0>\n    bool contains(KeyType && key) const\n    {\n        return is_object() && m_value.object->find(std::forward<KeyType>(key)) != m_value.object->end();\n    }\n\n    /// @brief check the existence of an element in a JSON object given a JSON pointer\n    /// @sa https://json.nlohmann.me/api/basic_json/contains/\n    bool contains(const json_pointer& ptr) const\n    {\n        return ptr.contains(this);\n    }\n\n    template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    bool contains(const typename ::nlohmann::json_pointer<BasicJsonType>& ptr) const\n    {\n        return ptr.contains(this);\n    }\n\n    /// @}\n\n\n    ///////////////\n    // iterators //\n    ///////////////\n\n    /// @name iterators\n    /// @{\n\n    /// @brief returns an iterator to the first element\n    /// @sa https://json.nlohmann.me/api/basic_json/begin/\n    iterator begin() noexcept\n    {\n        iterator result(this);\n        result.set_begin();\n        return result;\n    }\n\n    /// @brief returns an iterator to the first element\n    /// @sa https://json.nlohmann.me/api/basic_json/begin/\n    const_iterator begin() const noexcept\n    {\n        return cbegin();\n    }\n\n    /// @brief returns a const iterator to the first element\n    /// @sa https://json.nlohmann.me/api/basic_json/cbegin/\n    const_iterator cbegin() const noexcept\n    {\n        const_iterator result(this);\n        result.set_begin();\n        return result;\n    }\n\n    /// @brief returns an iterator to one past the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/end/\n    iterator end() noexcept\n    {\n        iterator result(this);\n        result.set_end();\n        return result;\n    }\n\n    /// @brief returns an iterator to one past the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/end/\n    const_iterator end() const noexcept\n    {\n        return cend();\n    }\n\n    /// @brief returns an iterator to one past the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/cend/\n    const_iterator cend() const noexcept\n    {\n        const_iterator result(this);\n        result.set_end();\n        return result;\n    }\n\n    /// @brief returns an iterator to the reverse-beginning\n    /// @sa https://json.nlohmann.me/api/basic_json/rbegin/\n    reverse_iterator rbegin() noexcept\n    {\n        return reverse_iterator(end());\n    }\n\n    /// @brief returns an iterator to the reverse-beginning\n    /// @sa https://json.nlohmann.me/api/basic_json/rbegin/\n    const_reverse_iterator rbegin() const noexcept\n    {\n        return crbegin();\n    }\n\n    /// @brief returns an iterator to the reverse-end\n    /// @sa https://json.nlohmann.me/api/basic_json/rend/\n    reverse_iterator rend() noexcept\n    {\n        return reverse_iterator(begin());\n    }\n\n    /// @brief returns an iterator to the reverse-end\n    /// @sa https://json.nlohmann.me/api/basic_json/rend/\n    const_reverse_iterator rend() const noexcept\n    {\n        return crend();\n    }\n\n    /// @brief returns a const reverse iterator to the last element\n    /// @sa https://json.nlohmann.me/api/basic_json/crbegin/\n    const_reverse_iterator crbegin() const noexcept\n    {\n        return const_reverse_iterator(cend());\n    }\n\n    /// @brief returns a const reverse iterator to one before the first\n    /// @sa https://json.nlohmann.me/api/basic_json/crend/\n    const_reverse_iterator crend() const noexcept\n    {\n        return const_reverse_iterator(cbegin());\n    }\n\n  public:\n    /// @brief wrapper to access iterator member functions in range-based for\n    /// @sa https://json.nlohmann.me/api/basic_json/items/\n    /// @deprecated This function is deprecated since 3.1.0 and will be removed in\n    ///             version 4.0.0 of the library. Please use @ref items() instead;\n    ///             that is, replace `json::iterator_wrapper(j)` with `j.items()`.\n    JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items())\n    static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept\n    {\n        return ref.items();\n    }\n\n    /// @brief wrapper to access iterator member functions in range-based for\n    /// @sa https://json.nlohmann.me/api/basic_json/items/\n    /// @deprecated This function is deprecated since 3.1.0 and will be removed in\n    ///         version 4.0.0 of the library. Please use @ref items() instead;\n    ///         that is, replace `json::iterator_wrapper(j)` with `j.items()`.\n    JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items())\n    static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept\n    {\n        return ref.items();\n    }\n\n    /// @brief helper to access iterator member functions in range-based for\n    /// @sa https://json.nlohmann.me/api/basic_json/items/\n    iteration_proxy<iterator> items() noexcept\n    {\n        return iteration_proxy<iterator>(*this);\n    }\n\n    /// @brief helper to access iterator member functions in range-based for\n    /// @sa https://json.nlohmann.me/api/basic_json/items/\n    iteration_proxy<const_iterator> items() const noexcept\n    {\n        return iteration_proxy<const_iterator>(*this);\n    }\n\n    /// @}\n\n\n    //////////////\n    // capacity //\n    //////////////\n\n    /// @name capacity\n    /// @{\n\n    /// @brief checks whether the container is empty.\n    /// @sa https://json.nlohmann.me/api/basic_json/empty/\n    bool empty() const noexcept\n    {\n        switch (m_type)\n        {\n            case value_t::null:\n            {\n                // null values are empty\n                return true;\n            }\n\n            case value_t::array:\n            {\n                // delegate call to array_t::empty()\n                return m_value.array->empty();\n            }\n\n            case value_t::object:\n            {\n                // delegate call to object_t::empty()\n                return m_value.object->empty();\n            }\n\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                // all other types are nonempty\n                return false;\n            }\n        }\n    }\n\n    /// @brief returns the number of elements\n    /// @sa https://json.nlohmann.me/api/basic_json/size/\n    size_type size() const noexcept\n    {\n        switch (m_type)\n        {\n            case value_t::null:\n            {\n                // null values are empty\n                return 0;\n            }\n\n            case value_t::array:\n            {\n                // delegate call to array_t::size()\n                return m_value.array->size();\n            }\n\n            case value_t::object:\n            {\n                // delegate call to object_t::size()\n                return m_value.object->size();\n            }\n\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                // all other types have size 1\n                return 1;\n            }\n        }\n    }\n\n    /// @brief returns the maximum possible number of elements\n    /// @sa https://json.nlohmann.me/api/basic_json/max_size/\n    size_type max_size() const noexcept\n    {\n        switch (m_type)\n        {\n            case value_t::array:\n            {\n                // delegate call to array_t::max_size()\n                return m_value.array->max_size();\n            }\n\n            case value_t::object:\n            {\n                // delegate call to object_t::max_size()\n                return m_value.object->max_size();\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                // all other types have max_size() == size()\n                return size();\n            }\n        }\n    }\n\n    /// @}\n\n\n    ///////////////\n    // modifiers //\n    ///////////////\n\n    /// @name modifiers\n    /// @{\n\n    /// @brief clears the contents\n    /// @sa https://json.nlohmann.me/api/basic_json/clear/\n    void clear() noexcept\n    {\n        switch (m_type)\n        {\n            case value_t::number_integer:\n            {\n                m_value.number_integer = 0;\n                break;\n            }\n\n            case value_t::number_unsigned:\n            {\n                m_value.number_unsigned = 0;\n                break;\n            }\n\n            case value_t::number_float:\n            {\n                m_value.number_float = 0.0;\n                break;\n            }\n\n            case value_t::boolean:\n            {\n                m_value.boolean = false;\n                break;\n            }\n\n            case value_t::string:\n            {\n                m_value.string->clear();\n                break;\n            }\n\n            case value_t::binary:\n            {\n                m_value.binary->clear();\n                break;\n            }\n\n            case value_t::array:\n            {\n                m_value.array->clear();\n                break;\n            }\n\n            case value_t::object:\n            {\n                m_value.object->clear();\n                break;\n            }\n\n            case value_t::null:\n            case value_t::discarded:\n            default:\n                break;\n        }\n    }\n\n    /// @brief add an object to an array\n    /// @sa https://json.nlohmann.me/api/basic_json/push_back/\n    void push_back(basic_json&& val)\n    {\n        // push_back only works for null objects or arrays\n        if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))\n        {\n            JSON_THROW(type_error::create(308, detail::concat(\"cannot use push_back() with \", type_name()), this));\n        }\n\n        // transform null object into an array\n        if (is_null())\n        {\n            m_type = value_t::array;\n            m_value = value_t::array;\n            assert_invariant();\n        }\n\n        // add element to array (move semantics)\n        const auto old_capacity = m_value.array->capacity();\n        m_value.array->push_back(std::move(val));\n        set_parent(m_value.array->back(), old_capacity);\n        // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor\n    }\n\n    /// @brief add an object to an array\n    /// @sa https://json.nlohmann.me/api/basic_json/operator+=/\n    reference operator+=(basic_json&& val)\n    {\n        push_back(std::move(val));\n        return *this;\n    }\n\n    /// @brief add an object to an array\n    /// @sa https://json.nlohmann.me/api/basic_json/push_back/\n    void push_back(const basic_json& val)\n    {\n        // push_back only works for null objects or arrays\n        if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))\n        {\n            JSON_THROW(type_error::create(308, detail::concat(\"cannot use push_back() with \", type_name()), this));\n        }\n\n        // transform null object into an array\n        if (is_null())\n        {\n            m_type = value_t::array;\n            m_value = value_t::array;\n            assert_invariant();\n        }\n\n        // add element to array\n        const auto old_capacity = m_value.array->capacity();\n        m_value.array->push_back(val);\n        set_parent(m_value.array->back(), old_capacity);\n    }\n\n    /// @brief add an object to an array\n    /// @sa https://json.nlohmann.me/api/basic_json/operator+=/\n    reference operator+=(const basic_json& val)\n    {\n        push_back(val);\n        return *this;\n    }\n\n    /// @brief add an object to an object\n    /// @sa https://json.nlohmann.me/api/basic_json/push_back/\n    void push_back(const typename object_t::value_type& val)\n    {\n        // push_back only works for null objects or objects\n        if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object())))\n        {\n            JSON_THROW(type_error::create(308, detail::concat(\"cannot use push_back() with \", type_name()), this));\n        }\n\n        // transform null object into an object\n        if (is_null())\n        {\n            m_type = value_t::object;\n            m_value = value_t::object;\n            assert_invariant();\n        }\n\n        // add element to object\n        auto res = m_value.object->insert(val);\n        set_parent(res.first->second);\n    }\n\n    /// @brief add an object to an object\n    /// @sa https://json.nlohmann.me/api/basic_json/operator+=/\n    reference operator+=(const typename object_t::value_type& val)\n    {\n        push_back(val);\n        return *this;\n    }\n\n    /// @brief add an object to an object\n    /// @sa https://json.nlohmann.me/api/basic_json/push_back/\n    void push_back(initializer_list_t init)\n    {\n        if (is_object() && init.size() == 2 && (*init.begin())->is_string())\n        {\n            basic_json&& key = init.begin()->moved_or_copied();\n            push_back(typename object_t::value_type(\n                          std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));\n        }\n        else\n        {\n            push_back(basic_json(init));\n        }\n    }\n\n    /// @brief add an object to an object\n    /// @sa https://json.nlohmann.me/api/basic_json/operator+=/\n    reference operator+=(initializer_list_t init)\n    {\n        push_back(init);\n        return *this;\n    }\n\n    /// @brief add an object to an array\n    /// @sa https://json.nlohmann.me/api/basic_json/emplace_back/\n    template<class... Args>\n    reference emplace_back(Args&& ... args)\n    {\n        // emplace_back only works for null objects or arrays\n        if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array())))\n        {\n            JSON_THROW(type_error::create(311, detail::concat(\"cannot use emplace_back() with \", type_name()), this));\n        }\n\n        // transform null object into an array\n        if (is_null())\n        {\n            m_type = value_t::array;\n            m_value = value_t::array;\n            assert_invariant();\n        }\n\n        // add element to array (perfect forwarding)\n        const auto old_capacity = m_value.array->capacity();\n        m_value.array->emplace_back(std::forward<Args>(args)...);\n        return set_parent(m_value.array->back(), old_capacity);\n    }\n\n    /// @brief add an object to an object if key does not exist\n    /// @sa https://json.nlohmann.me/api/basic_json/emplace/\n    template<class... Args>\n    std::pair<iterator, bool> emplace(Args&& ... args)\n    {\n        // emplace only works for null objects or arrays\n        if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object())))\n        {\n            JSON_THROW(type_error::create(311, detail::concat(\"cannot use emplace() with \", type_name()), this));\n        }\n\n        // transform null object into an object\n        if (is_null())\n        {\n            m_type = value_t::object;\n            m_value = value_t::object;\n            assert_invariant();\n        }\n\n        // add element to array (perfect forwarding)\n        auto res = m_value.object->emplace(std::forward<Args>(args)...);\n        set_parent(res.first->second);\n\n        // create result iterator and set iterator to the result of emplace\n        auto it = begin();\n        it.m_it.object_iterator = res.first;\n\n        // return pair of iterator and boolean\n        return {it, res.second};\n    }\n\n    /// Helper for insertion of an iterator\n    /// @note: This uses std::distance to support GCC 4.8,\n    ///        see https://github.com/nlohmann/json/pull/1257\n    template<typename... Args>\n    iterator insert_iterator(const_iterator pos, Args&& ... args)\n    {\n        iterator result(this);\n        JSON_ASSERT(m_value.array != nullptr);\n\n        auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator);\n        m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...);\n        result.m_it.array_iterator = m_value.array->begin() + insert_pos;\n\n        // This could have been written as:\n        // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);\n        // but the return value of insert is missing in GCC 4.8, so it is written this way instead.\n\n        set_parents();\n        return result;\n    }\n\n    /// @brief inserts element into array\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    iterator insert(const_iterator pos, const basic_json& val)\n    {\n        // insert only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            // check if iterator pos fits to this JSON value\n            if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))\n            {\n                JSON_THROW(invalid_iterator::create(202, \"iterator does not fit current value\", this));\n            }\n\n            // insert to array and return iterator\n            return insert_iterator(pos, val);\n        }\n\n        JSON_THROW(type_error::create(309, detail::concat(\"cannot use insert() with \", type_name()), this));\n    }\n\n    /// @brief inserts element into array\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    iterator insert(const_iterator pos, basic_json&& val)\n    {\n        return insert(pos, val);\n    }\n\n    /// @brief inserts copies of element into array\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    iterator insert(const_iterator pos, size_type cnt, const basic_json& val)\n    {\n        // insert only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            // check if iterator pos fits to this JSON value\n            if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))\n            {\n                JSON_THROW(invalid_iterator::create(202, \"iterator does not fit current value\", this));\n            }\n\n            // insert to array and return iterator\n            return insert_iterator(pos, cnt, val);\n        }\n\n        JSON_THROW(type_error::create(309, detail::concat(\"cannot use insert() with \", type_name()), this));\n    }\n\n    /// @brief inserts range of elements into array\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    iterator insert(const_iterator pos, const_iterator first, const_iterator last)\n    {\n        // insert only works for arrays\n        if (JSON_HEDLEY_UNLIKELY(!is_array()))\n        {\n            JSON_THROW(type_error::create(309, detail::concat(\"cannot use insert() with \", type_name()), this));\n        }\n\n        // check if iterator pos fits to this JSON value\n        if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))\n        {\n            JSON_THROW(invalid_iterator::create(202, \"iterator does not fit current value\", this));\n        }\n\n        // check if range iterators belong to the same JSON object\n        if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(210, \"iterators do not fit\", this));\n        }\n\n        if (JSON_HEDLEY_UNLIKELY(first.m_object == this))\n        {\n            JSON_THROW(invalid_iterator::create(211, \"passed iterators may not belong to container\", this));\n        }\n\n        // insert to array and return iterator\n        return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);\n    }\n\n    /// @brief inserts elements from initializer list into array\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    iterator insert(const_iterator pos, initializer_list_t ilist)\n    {\n        // insert only works for arrays\n        if (JSON_HEDLEY_UNLIKELY(!is_array()))\n        {\n            JSON_THROW(type_error::create(309, detail::concat(\"cannot use insert() with \", type_name()), this));\n        }\n\n        // check if iterator pos fits to this JSON value\n        if (JSON_HEDLEY_UNLIKELY(pos.m_object != this))\n        {\n            JSON_THROW(invalid_iterator::create(202, \"iterator does not fit current value\", this));\n        }\n\n        // insert to array and return iterator\n        return insert_iterator(pos, ilist.begin(), ilist.end());\n    }\n\n    /// @brief inserts range of elements into object\n    /// @sa https://json.nlohmann.me/api/basic_json/insert/\n    void insert(const_iterator first, const_iterator last)\n    {\n        // insert only works for objects\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(309, detail::concat(\"cannot use insert() with \", type_name()), this));\n        }\n\n        // check if range iterators belong to the same JSON object\n        if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(210, \"iterators do not fit\", this));\n        }\n\n        // passed iterators must belong to objects\n        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))\n        {\n            JSON_THROW(invalid_iterator::create(202, \"iterators first and last must point to objects\", this));\n        }\n\n        m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);\n    }\n\n    /// @brief updates a JSON object from another object, overwriting existing keys\n    /// @sa https://json.nlohmann.me/api/basic_json/update/\n    void update(const_reference j, bool merge_objects = false)\n    {\n        update(j.begin(), j.end(), merge_objects);\n    }\n\n    /// @brief updates a JSON object from another object, overwriting existing keys\n    /// @sa https://json.nlohmann.me/api/basic_json/update/\n    void update(const_iterator first, const_iterator last, bool merge_objects = false)\n    {\n        // implicitly convert null value to an empty object\n        if (is_null())\n        {\n            m_type = value_t::object;\n            m_value.object = create<object_t>();\n            assert_invariant();\n        }\n\n        if (JSON_HEDLEY_UNLIKELY(!is_object()))\n        {\n            JSON_THROW(type_error::create(312, detail::concat(\"cannot use update() with \", type_name()), this));\n        }\n\n        // check if range iterators belong to the same JSON object\n        if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object))\n        {\n            JSON_THROW(invalid_iterator::create(210, \"iterators do not fit\", this));\n        }\n\n        // passed iterators must belong to objects\n        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))\n        {\n            JSON_THROW(type_error::create(312, detail::concat(\"cannot use update() with \", first.m_object->type_name()), first.m_object));\n        }\n\n        for (auto it = first; it != last; ++it)\n        {\n            if (merge_objects && it.value().is_object())\n            {\n                auto it2 = m_value.object->find(it.key());\n                if (it2 != m_value.object->end())\n                {\n                    it2->second.update(it.value(), true);\n                    continue;\n                }\n            }\n            m_value.object->operator[](it.key()) = it.value();\n#if JSON_DIAGNOSTICS\n            m_value.object->operator[](it.key()).m_parent = this;\n#endif\n        }\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(reference other) noexcept (\n        std::is_nothrow_move_constructible<value_t>::value&&\n        std::is_nothrow_move_assignable<value_t>::value&&\n        std::is_nothrow_move_constructible<json_value>::value&&\n        std::is_nothrow_move_assignable<json_value>::value\n    )\n    {\n        std::swap(m_type, other.m_type);\n        std::swap(m_value, other.m_value);\n\n        set_parents();\n        other.set_parents();\n        assert_invariant();\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    friend void swap(reference left, reference right) noexcept (\n        std::is_nothrow_move_constructible<value_t>::value&&\n        std::is_nothrow_move_assignable<value_t>::value&&\n        std::is_nothrow_move_constructible<json_value>::value&&\n        std::is_nothrow_move_assignable<json_value>::value\n    )\n    {\n        left.swap(right);\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(array_t& other) // NOLINT(bugprone-exception-escape)\n    {\n        // swap only works for arrays\n        if (JSON_HEDLEY_LIKELY(is_array()))\n        {\n            using std::swap;\n            swap(*(m_value.array), other);\n        }\n        else\n        {\n            JSON_THROW(type_error::create(310, detail::concat(\"cannot use swap(array_t&) with \", type_name()), this));\n        }\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(object_t& other) // NOLINT(bugprone-exception-escape)\n    {\n        // swap only works for objects\n        if (JSON_HEDLEY_LIKELY(is_object()))\n        {\n            using std::swap;\n            swap(*(m_value.object), other);\n        }\n        else\n        {\n            JSON_THROW(type_error::create(310, detail::concat(\"cannot use swap(object_t&) with \", type_name()), this));\n        }\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(string_t& other) // NOLINT(bugprone-exception-escape)\n    {\n        // swap only works for strings\n        if (JSON_HEDLEY_LIKELY(is_string()))\n        {\n            using std::swap;\n            swap(*(m_value.string), other);\n        }\n        else\n        {\n            JSON_THROW(type_error::create(310, detail::concat(\"cannot use swap(string_t&) with \", type_name()), this));\n        }\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(binary_t& other) // NOLINT(bugprone-exception-escape)\n    {\n        // swap only works for strings\n        if (JSON_HEDLEY_LIKELY(is_binary()))\n        {\n            using std::swap;\n            swap(*(m_value.binary), other);\n        }\n        else\n        {\n            JSON_THROW(type_error::create(310, detail::concat(\"cannot use swap(binary_t&) with \", type_name()), this));\n        }\n    }\n\n    /// @brief exchanges the values\n    /// @sa https://json.nlohmann.me/api/basic_json/swap/\n    void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape)\n    {\n        // swap only works for strings\n        if (JSON_HEDLEY_LIKELY(is_binary()))\n        {\n            using std::swap;\n            swap(*(m_value.binary), other);\n        }\n        else\n        {\n            JSON_THROW(type_error::create(310, detail::concat(\"cannot use swap(binary_t::container_type&) with \", type_name()), this));\n        }\n    }\n\n    /// @}\n\n    //////////////////////////////////////////\n    // lexicographical comparison operators //\n    //////////////////////////////////////////\n\n    /// @name lexicographical comparison operators\n    /// @{\n\n    // note parentheses around operands are necessary; see\n    // https://github.com/nlohmann/json/issues/1530\n#define JSON_IMPLEMENT_OPERATOR(op, null_result, unordered_result, default_result)                       \\\n    const auto lhs_type = lhs.type();                                                                    \\\n    const auto rhs_type = rhs.type();                                                                    \\\n    \\\n    if (lhs_type == rhs_type) /* NOLINT(readability/braces) */                                           \\\n    {                                                                                                    \\\n        switch (lhs_type)                                                                                \\\n        {                                                                                                \\\n            case value_t::array:                                                                         \\\n                return (*lhs.m_value.array) op (*rhs.m_value.array);                                     \\\n                \\\n            case value_t::object:                                                                        \\\n                return (*lhs.m_value.object) op (*rhs.m_value.object);                                   \\\n                \\\n            case value_t::null:                                                                          \\\n                return (null_result);                                                                    \\\n                \\\n            case value_t::string:                                                                        \\\n                return (*lhs.m_value.string) op (*rhs.m_value.string);                                   \\\n                \\\n            case value_t::boolean:                                                                       \\\n                return (lhs.m_value.boolean) op (rhs.m_value.boolean);                                   \\\n                \\\n            case value_t::number_integer:                                                                \\\n                return (lhs.m_value.number_integer) op (rhs.m_value.number_integer);                     \\\n                \\\n            case value_t::number_unsigned:                                                               \\\n                return (lhs.m_value.number_unsigned) op (rhs.m_value.number_unsigned);                   \\\n                \\\n            case value_t::number_float:                                                                  \\\n                return (lhs.m_value.number_float) op (rhs.m_value.number_float);                         \\\n                \\\n            case value_t::binary:                                                                        \\\n                return (*lhs.m_value.binary) op (*rhs.m_value.binary);                                   \\\n                \\\n            case value_t::discarded:                                                                     \\\n            default:                                                                                     \\\n                return (unordered_result);                                                               \\\n        }                                                                                                \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float)                   \\\n    {                                                                                                    \\\n        return static_cast<number_float_t>(lhs.m_value.number_integer) op rhs.m_value.number_float;      \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer)                   \\\n    {                                                                                                    \\\n        return lhs.m_value.number_float op static_cast<number_float_t>(rhs.m_value.number_integer);      \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float)                  \\\n    {                                                                                                    \\\n        return static_cast<number_float_t>(lhs.m_value.number_unsigned) op rhs.m_value.number_float;     \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned)                  \\\n    {                                                                                                    \\\n        return lhs.m_value.number_float op static_cast<number_float_t>(rhs.m_value.number_unsigned);     \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer)                \\\n    {                                                                                                    \\\n        return static_cast<number_integer_t>(lhs.m_value.number_unsigned) op rhs.m_value.number_integer; \\\n    }                                                                                                    \\\n    else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned)                \\\n    {                                                                                                    \\\n        return lhs.m_value.number_integer op static_cast<number_integer_t>(rhs.m_value.number_unsigned); \\\n    }                                                                                                    \\\n    else if(compares_unordered(lhs, rhs))\\\n    {\\\n        return (unordered_result);\\\n    }\\\n    \\\n    return (default_result);\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    // returns true if:\n    // - any operand is NaN and the other operand is of number type\n    // - any operand is discarded\n    // in legacy mode, discarded values are considered ordered if\n    // an operation is computed as an odd number of inverses of others\n    static bool compares_unordered(const_reference lhs, const_reference rhs, bool inverse = false) noexcept\n    {\n        if ((lhs.is_number_float() && std::isnan(lhs.m_value.number_float) && rhs.is_number())\n                || (rhs.is_number_float() && std::isnan(rhs.m_value.number_float) && lhs.is_number()))\n        {\n            return true;\n        }\n#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON\n        return (lhs.is_discarded() || rhs.is_discarded()) && !inverse;\n#else\n        static_cast<void>(inverse);\n        return lhs.is_discarded() || rhs.is_discarded();\n#endif\n    }\n\n  private:\n    bool compares_unordered(const_reference rhs, bool inverse = false) const noexcept\n    {\n        return compares_unordered(*this, rhs, inverse);\n    }\n\n  public:\n#if JSON_HAS_THREE_WAY_COMPARISON\n    /// @brief comparison: equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/\n    bool operator==(const_reference rhs) const noexcept\n    {\n#ifdef __GNUC__\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wfloat-equal\"\n#endif\n        const_reference lhs = *this;\n        JSON_IMPLEMENT_OPERATOR( ==, true, false, false)\n#ifdef __GNUC__\n#pragma GCC diagnostic pop\n#endif\n    }\n\n    /// @brief comparison: equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/\n    template<typename ScalarType>\n    requires std::is_scalar_v<ScalarType>\n    bool operator==(ScalarType rhs) const noexcept\n    {\n        return *this == basic_json(rhs);\n    }\n\n    /// @brief comparison: not equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/\n    bool operator!=(const_reference rhs) const noexcept\n    {\n        if (compares_unordered(rhs, true))\n        {\n            return false;\n        }\n        return !operator==(rhs);\n    }\n\n    /// @brief comparison: 3-way\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/\n    std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD*\n    {\n        const_reference lhs = *this;\n        // default_result is used if we cannot compare values. In that case,\n        // we compare types.\n        JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD*\n                                std::partial_ordering::equivalent,\n                                std::partial_ordering::unordered,\n                                lhs_type <=> rhs_type) // *NOPAD*\n    }\n\n    /// @brief comparison: 3-way\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_spaceship/\n    template<typename ScalarType>\n    requires std::is_scalar_v<ScalarType>\n    std::partial_ordering operator<=>(ScalarType rhs) const noexcept // *NOPAD*\n    {\n        return *this <=> basic_json(rhs); // *NOPAD*\n    }\n\n#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON\n    // all operators that are computed as an odd number of inverses of others\n    // need to be overloaded to emulate the legacy comparison behavior\n\n    /// @brief comparison: less than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_le/\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON)\n    bool operator<=(const_reference rhs) const noexcept\n    {\n        if (compares_unordered(rhs, true))\n        {\n            return false;\n        }\n        return !(rhs < *this);\n    }\n\n    /// @brief comparison: less than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_le/\n    template<typename ScalarType>\n    requires std::is_scalar_v<ScalarType>\n    bool operator<=(ScalarType rhs) const noexcept\n    {\n        return *this <= basic_json(rhs);\n    }\n\n    /// @brief comparison: greater than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON)\n    bool operator>=(const_reference rhs) const noexcept\n    {\n        if (compares_unordered(rhs, true))\n        {\n            return false;\n        }\n        return !(*this < rhs);\n    }\n\n    /// @brief comparison: greater than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/\n    template<typename ScalarType>\n    requires std::is_scalar_v<ScalarType>\n    bool operator>=(ScalarType rhs) const noexcept\n    {\n        return *this >= basic_json(rhs);\n    }\n#endif\n#else\n    /// @brief comparison: equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/\n    friend bool operator==(const_reference lhs, const_reference rhs) noexcept\n    {\n#ifdef __GNUC__\n#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wfloat-equal\"\n#endif\n        JSON_IMPLEMENT_OPERATOR( ==, true, false, false)\n#ifdef __GNUC__\n#pragma GCC diagnostic pop\n#endif\n    }\n\n    /// @brief comparison: equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator==(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs == basic_json(rhs);\n    }\n\n    /// @brief comparison: equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator==(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) == rhs;\n    }\n\n    /// @brief comparison: not equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/\n    friend bool operator!=(const_reference lhs, const_reference rhs) noexcept\n    {\n        if (compares_unordered(lhs, rhs, true))\n        {\n            return false;\n        }\n        return !(lhs == rhs);\n    }\n\n    /// @brief comparison: not equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs != basic_json(rhs);\n    }\n\n    /// @brief comparison: not equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) != rhs;\n    }\n\n    /// @brief comparison: less than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/\n    friend bool operator<(const_reference lhs, const_reference rhs) noexcept\n    {\n        // default_result is used if we cannot compare values. In that case,\n        // we compare types. Note we have to call the operator explicitly,\n        // because MSVC has problems otherwise.\n        JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type))\n    }\n\n    /// @brief comparison: less than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator<(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs < basic_json(rhs);\n    }\n\n    /// @brief comparison: less than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator<(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) < rhs;\n    }\n\n    /// @brief comparison: less than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_le/\n    friend bool operator<=(const_reference lhs, const_reference rhs) noexcept\n    {\n        if (compares_unordered(lhs, rhs, true))\n        {\n            return false;\n        }\n        return !(rhs < lhs);\n    }\n\n    /// @brief comparison: less than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_le/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs <= basic_json(rhs);\n    }\n\n    /// @brief comparison: less than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_le/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) <= rhs;\n    }\n\n    /// @brief comparison: greater than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/\n    friend bool operator>(const_reference lhs, const_reference rhs) noexcept\n    {\n        // double inverse\n        if (compares_unordered(lhs, rhs))\n        {\n            return false;\n        }\n        return !(lhs <= rhs);\n    }\n\n    /// @brief comparison: greater than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator>(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs > basic_json(rhs);\n    }\n\n    /// @brief comparison: greater than\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator>(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) > rhs;\n    }\n\n    /// @brief comparison: greater than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/\n    friend bool operator>=(const_reference lhs, const_reference rhs) noexcept\n    {\n        if (compares_unordered(lhs, rhs, true))\n        {\n            return false;\n        }\n        return !(lhs < rhs);\n    }\n\n    /// @brief comparison: greater than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept\n    {\n        return lhs >= basic_json(rhs);\n    }\n\n    /// @brief comparison: greater than or equal\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/\n    template<typename ScalarType, typename std::enable_if<\n                 std::is_scalar<ScalarType>::value, int>::type = 0>\n    friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept\n    {\n        return basic_json(lhs) >= rhs;\n    }\n#endif\n\n#undef JSON_IMPLEMENT_OPERATOR\n\n    /// @}\n\n    ///////////////////\n    // serialization //\n    ///////////////////\n\n    /// @name serialization\n    /// @{\n#ifndef JSON_NO_IO\n    /// @brief serialize to stream\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/\n    friend std::ostream& operator<<(std::ostream& o, const basic_json& j)\n    {\n        // read width member and use it as indentation parameter if nonzero\n        const bool pretty_print = o.width() > 0;\n        const auto indentation = pretty_print ? o.width() : 0;\n\n        // reset width to 0 for subsequent calls to this stream\n        o.width(0);\n\n        // do the actual serialization\n        serializer s(detail::output_adapter<char>(o), o.fill());\n        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));\n        return o;\n    }\n\n    /// @brief serialize to stream\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/\n    /// @deprecated This function is deprecated since 3.0.0 and will be removed in\n    ///             version 4.0.0 of the library. Please use\n    ///             operator<<(std::ostream&, const basic_json&) instead; that is,\n    ///             replace calls like `j >> o;` with `o << j;`.\n    JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator<<(std::ostream&, const basic_json&))\n    friend std::ostream& operator>>(const basic_json& j, std::ostream& o)\n    {\n        return o << j;\n    }\n#endif  // JSON_NO_IO\n    /// @}\n\n\n    /////////////////////\n    // deserialization //\n    /////////////////////\n\n    /// @name deserialization\n    /// @{\n\n    /// @brief deserialize from a compatible input\n    /// @sa https://json.nlohmann.me/api/basic_json/parse/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json parse(InputType&& i,\n                            const parser_callback_t cb = nullptr,\n                            const bool allow_exceptions = true,\n                            const bool ignore_comments = false)\n    {\n        basic_json result;\n        parser(detail::input_adapter(std::forward<InputType>(i)), cb, allow_exceptions, ignore_comments).parse(true, result);\n        return result;\n    }\n\n    /// @brief deserialize from a pair of character iterators\n    /// @sa https://json.nlohmann.me/api/basic_json/parse/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json parse(IteratorType first,\n                            IteratorType last,\n                            const parser_callback_t cb = nullptr,\n                            const bool allow_exceptions = true,\n                            const bool ignore_comments = false)\n    {\n        basic_json result;\n        parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result);\n        return result;\n    }\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len))\n    static basic_json parse(detail::span_input_adapter&& i,\n                            const parser_callback_t cb = nullptr,\n                            const bool allow_exceptions = true,\n                            const bool ignore_comments = false)\n    {\n        basic_json result;\n        parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result);\n        return result;\n    }\n\n    /// @brief check if the input is valid JSON\n    /// @sa https://json.nlohmann.me/api/basic_json/accept/\n    template<typename InputType>\n    static bool accept(InputType&& i,\n                       const bool ignore_comments = false)\n    {\n        return parser(detail::input_adapter(std::forward<InputType>(i)), nullptr, false, ignore_comments).accept(true);\n    }\n\n    /// @brief check if the input is valid JSON\n    /// @sa https://json.nlohmann.me/api/basic_json/accept/\n    template<typename IteratorType>\n    static bool accept(IteratorType first, IteratorType last,\n                       const bool ignore_comments = false)\n    {\n        return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true);\n    }\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len))\n    static bool accept(detail::span_input_adapter&& i,\n                       const bool ignore_comments = false)\n    {\n        return parser(i.get(), nullptr, false, ignore_comments).accept(true);\n    }\n\n    /// @brief generate SAX events\n    /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/\n    template <typename InputType, typename SAX>\n    JSON_HEDLEY_NON_NULL(2)\n    static bool sax_parse(InputType&& i, SAX* sax,\n                          input_format_t format = input_format_t::json,\n                          const bool strict = true,\n                          const bool ignore_comments = false)\n    {\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        return format == input_format_t::json\n               ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)\n               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);\n    }\n\n    /// @brief generate SAX events\n    /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/\n    template<class IteratorType, class SAX>\n    JSON_HEDLEY_NON_NULL(3)\n    static bool sax_parse(IteratorType first, IteratorType last, SAX* sax,\n                          input_format_t format = input_format_t::json,\n                          const bool strict = true,\n                          const bool ignore_comments = false)\n    {\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        return format == input_format_t::json\n               ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)\n               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);\n    }\n\n    /// @brief generate SAX events\n    /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/\n    /// @deprecated This function is deprecated since 3.8.0 and will be removed in\n    ///             version 4.0.0 of the library. Please use\n    ///             sax_parse(ptr, ptr + len) instead.\n    template <typename SAX>\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, sax_parse(ptr, ptr + len, ...))\n    JSON_HEDLEY_NON_NULL(2)\n    static bool sax_parse(detail::span_input_adapter&& i, SAX* sax,\n                          input_format_t format = input_format_t::json,\n                          const bool strict = true,\n                          const bool ignore_comments = false)\n    {\n        auto ia = i.get();\n        return format == input_format_t::json\n               // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n               ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict)\n               // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(format, sax, strict);\n    }\n#ifndef JSON_NO_IO\n    /// @brief deserialize from stream\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/\n    /// @deprecated This stream operator is deprecated since 3.0.0 and will be removed in\n    ///             version 4.0.0 of the library. Please use\n    ///             operator>>(std::istream&, basic_json&) instead; that is,\n    ///             replace calls like `j << i;` with `i >> j;`.\n    JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator>>(std::istream&, basic_json&))\n    friend std::istream& operator<<(basic_json& j, std::istream& i)\n    {\n        return operator>>(i, j);\n    }\n\n    /// @brief deserialize from stream\n    /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/\n    friend std::istream& operator>>(std::istream& i, basic_json& j)\n    {\n        parser(detail::input_adapter(i)).parse(false, j);\n        return i;\n    }\n#endif  // JSON_NO_IO\n    /// @}\n\n    ///////////////////////////\n    // convenience functions //\n    ///////////////////////////\n\n    /// @brief return the type as string\n    /// @sa https://json.nlohmann.me/api/basic_json/type_name/\n    JSON_HEDLEY_RETURNS_NON_NULL\n    const char* type_name() const noexcept\n    {\n        switch (m_type)\n        {\n            case value_t::null:\n                return \"null\";\n            case value_t::object:\n                return \"object\";\n            case value_t::array:\n                return \"array\";\n            case value_t::string:\n                return \"string\";\n            case value_t::boolean:\n                return \"boolean\";\n            case value_t::binary:\n                return \"binary\";\n            case value_t::discarded:\n                return \"discarded\";\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            default:\n                return \"number\";\n        }\n    }\n\n\n  JSON_PRIVATE_UNLESS_TESTED:\n    //////////////////////\n    // member variables //\n    //////////////////////\n\n    /// the type of the current element\n    value_t m_type = value_t::null;\n\n    /// the value of the current element\n    json_value m_value = {};\n\n#if JSON_DIAGNOSTICS\n    /// a pointer to a parent value (for debugging purposes)\n    basic_json* m_parent = nullptr;\n#endif\n\n    //////////////////////////////////////////\n    // binary serialization/deserialization //\n    //////////////////////////////////////////\n\n    /// @name binary serialization/deserialization support\n    /// @{\n\n  public:\n    /// @brief create a CBOR serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/\n    static std::vector<std::uint8_t> to_cbor(const basic_json& j)\n    {\n        std::vector<std::uint8_t> result;\n        to_cbor(j, result);\n        return result;\n    }\n\n    /// @brief create a CBOR serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/\n    static void to_cbor(const basic_json& j, detail::output_adapter<std::uint8_t> o)\n    {\n        binary_writer<std::uint8_t>(o).write_cbor(j);\n    }\n\n    /// @brief create a CBOR serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/\n    static void to_cbor(const basic_json& j, detail::output_adapter<char> o)\n    {\n        binary_writer<char>(o).write_cbor(j);\n    }\n\n    /// @brief create a MessagePack serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/\n    static std::vector<std::uint8_t> to_msgpack(const basic_json& j)\n    {\n        std::vector<std::uint8_t> result;\n        to_msgpack(j, result);\n        return result;\n    }\n\n    /// @brief create a MessagePack serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/\n    static void to_msgpack(const basic_json& j, detail::output_adapter<std::uint8_t> o)\n    {\n        binary_writer<std::uint8_t>(o).write_msgpack(j);\n    }\n\n    /// @brief create a MessagePack serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/\n    static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)\n    {\n        binary_writer<char>(o).write_msgpack(j);\n    }\n\n    /// @brief create a UBJSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/\n    static std::vector<std::uint8_t> to_ubjson(const basic_json& j,\n            const bool use_size = false,\n            const bool use_type = false)\n    {\n        std::vector<std::uint8_t> result;\n        to_ubjson(j, result, use_size, use_type);\n        return result;\n    }\n\n    /// @brief create a UBJSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/\n    static void to_ubjson(const basic_json& j, detail::output_adapter<std::uint8_t> o,\n                          const bool use_size = false, const bool use_type = false)\n    {\n        binary_writer<std::uint8_t>(o).write_ubjson(j, use_size, use_type);\n    }\n\n    /// @brief create a UBJSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/\n    static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,\n                          const bool use_size = false, const bool use_type = false)\n    {\n        binary_writer<char>(o).write_ubjson(j, use_size, use_type);\n    }\n\n    /// @brief create a BJData serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/\n    static std::vector<std::uint8_t> to_bjdata(const basic_json& j,\n            const bool use_size = false,\n            const bool use_type = false)\n    {\n        std::vector<std::uint8_t> result;\n        to_bjdata(j, result, use_size, use_type);\n        return result;\n    }\n\n    /// @brief create a BJData serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/\n    static void to_bjdata(const basic_json& j, detail::output_adapter<std::uint8_t> o,\n                          const bool use_size = false, const bool use_type = false)\n    {\n        binary_writer<std::uint8_t>(o).write_ubjson(j, use_size, use_type, true, true);\n    }\n\n    /// @brief create a BJData serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/\n    static void to_bjdata(const basic_json& j, detail::output_adapter<char> o,\n                          const bool use_size = false, const bool use_type = false)\n    {\n        binary_writer<char>(o).write_ubjson(j, use_size, use_type, true, true);\n    }\n\n    /// @brief create a BSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bson/\n    static std::vector<std::uint8_t> to_bson(const basic_json& j)\n    {\n        std::vector<std::uint8_t> result;\n        to_bson(j, result);\n        return result;\n    }\n\n    /// @brief create a BSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bson/\n    static void to_bson(const basic_json& j, detail::output_adapter<std::uint8_t> o)\n    {\n        binary_writer<std::uint8_t>(o).write_bson(j);\n    }\n\n    /// @brief create a BSON serialization of a given JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/to_bson/\n    static void to_bson(const basic_json& j, detail::output_adapter<char> o)\n    {\n        binary_writer<char>(o).write_bson(j);\n    }\n\n    /// @brief create a JSON value from an input in CBOR format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_cbor(InputType&& i,\n                                const bool strict = true,\n                                const bool allow_exceptions = true,\n                                const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in CBOR format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_cbor(IteratorType first, IteratorType last,\n                                const bool strict = true,\n                                const bool allow_exceptions = true,\n                                const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    template<typename T>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len))\n    static basic_json from_cbor(const T* ptr, std::size_t len,\n                                const bool strict = true,\n                                const bool allow_exceptions = true,\n                                const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)\n    {\n        return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler);\n    }\n\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len))\n    static basic_json from_cbor(detail::span_input_adapter&& i,\n                                const bool strict = true,\n                                const bool allow_exceptions = true,\n                                const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = i.get();\n        // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in MessagePack format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_msgpack(InputType&& i,\n                                   const bool strict = true,\n                                   const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in MessagePack format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_msgpack(IteratorType first, IteratorType last,\n                                   const bool strict = true,\n                                   const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    template<typename T>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len))\n    static basic_json from_msgpack(const T* ptr, std::size_t len,\n                                   const bool strict = true,\n                                   const bool allow_exceptions = true)\n    {\n        return from_msgpack(ptr, ptr + len, strict, allow_exceptions);\n    }\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len))\n    static basic_json from_msgpack(detail::span_input_adapter&& i,\n                                   const bool strict = true,\n                                   const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = i.get();\n        // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in UBJSON format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_ubjson(InputType&& i,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in UBJSON format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_ubjson(IteratorType first, IteratorType last,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    template<typename T>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len))\n    static basic_json from_ubjson(const T* ptr, std::size_t len,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        return from_ubjson(ptr, ptr + len, strict, allow_exceptions);\n    }\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len))\n    static basic_json from_ubjson(detail::span_input_adapter&& i,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = i.get();\n        // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n\n    /// @brief create a JSON value from an input in BJData format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_bjdata(InputType&& i,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in BJData format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_bjdata(IteratorType first, IteratorType last,\n                                  const bool strict = true,\n                                  const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in BSON format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_bson/\n    template<typename InputType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_bson(InputType&& i,\n                                const bool strict = true,\n                                const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::forward<InputType>(i));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    /// @brief create a JSON value from an input in BSON format\n    /// @sa https://json.nlohmann.me/api/basic_json/from_bson/\n    template<typename IteratorType>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json from_bson(IteratorType first, IteratorType last,\n                                const bool strict = true,\n                                const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = detail::input_adapter(std::move(first), std::move(last));\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n\n    template<typename T>\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len))\n    static basic_json from_bson(const T* ptr, std::size_t len,\n                                const bool strict = true,\n                                const bool allow_exceptions = true)\n    {\n        return from_bson(ptr, ptr + len, strict, allow_exceptions);\n    }\n\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len))\n    static basic_json from_bson(detail::span_input_adapter&& i,\n                                const bool strict = true,\n                                const bool allow_exceptions = true)\n    {\n        basic_json result;\n        detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions);\n        auto ia = i.get();\n        // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)\n        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict);\n        return res ? result : basic_json(value_t::discarded);\n    }\n    /// @}\n\n    //////////////////////////\n    // JSON Pointer support //\n    //////////////////////////\n\n    /// @name JSON Pointer functions\n    /// @{\n\n    /// @brief access specified element via JSON Pointer\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    reference operator[](const json_pointer& ptr)\n    {\n        return ptr.get_unchecked(this);\n    }\n\n    template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    reference operator[](const ::nlohmann::json_pointer<BasicJsonType>& ptr)\n    {\n        return ptr.get_unchecked(this);\n    }\n\n    /// @brief access specified element via JSON Pointer\n    /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/\n    const_reference operator[](const json_pointer& ptr) const\n    {\n        return ptr.get_unchecked(this);\n    }\n\n    template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    const_reference operator[](const ::nlohmann::json_pointer<BasicJsonType>& ptr) const\n    {\n        return ptr.get_unchecked(this);\n    }\n\n    /// @brief access specified element via JSON Pointer\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    reference at(const json_pointer& ptr)\n    {\n        return ptr.get_checked(this);\n    }\n\n    template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    reference at(const ::nlohmann::json_pointer<BasicJsonType>& ptr)\n    {\n        return ptr.get_checked(this);\n    }\n\n    /// @brief access specified element via JSON Pointer\n    /// @sa https://json.nlohmann.me/api/basic_json/at/\n    const_reference at(const json_pointer& ptr) const\n    {\n        return ptr.get_checked(this);\n    }\n\n    template<typename BasicJsonType, detail::enable_if_t<detail::is_basic_json<BasicJsonType>::value, int> = 0>\n    JSON_HEDLEY_DEPRECATED_FOR(3.11.0, basic_json::json_pointer or nlohmann::json_pointer<basic_json::string_t>) // NOLINT(readability/alt_tokens)\n    const_reference at(const ::nlohmann::json_pointer<BasicJsonType>& ptr) const\n    {\n        return ptr.get_checked(this);\n    }\n\n    /// @brief return flattened JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/flatten/\n    basic_json flatten() const\n    {\n        basic_json result(value_t::object);\n        json_pointer::flatten(\"\", *this, result);\n        return result;\n    }\n\n    /// @brief unflatten a previously flattened JSON value\n    /// @sa https://json.nlohmann.me/api/basic_json/unflatten/\n    basic_json unflatten() const\n    {\n        return json_pointer::unflatten(*this);\n    }\n\n    /// @}\n\n    //////////////////////////\n    // JSON Patch functions //\n    //////////////////////////\n\n    /// @name JSON Patch functions\n    /// @{\n\n    /// @brief applies a JSON patch in-place without copying the object\n    /// @sa https://json.nlohmann.me/api/basic_json/patch/\n    void patch_inplace(const basic_json& json_patch)\n    {\n        basic_json& result = *this;\n        // the valid JSON Patch operations\n        enum class patch_operations {add, remove, replace, move, copy, test, invalid};\n\n        const auto get_op = [](const std::string & op)\n        {\n            if (op == \"add\")\n            {\n                return patch_operations::add;\n            }\n            if (op == \"remove\")\n            {\n                return patch_operations::remove;\n            }\n            if (op == \"replace\")\n            {\n                return patch_operations::replace;\n            }\n            if (op == \"move\")\n            {\n                return patch_operations::move;\n            }\n            if (op == \"copy\")\n            {\n                return patch_operations::copy;\n            }\n            if (op == \"test\")\n            {\n                return patch_operations::test;\n            }\n\n            return patch_operations::invalid;\n        };\n\n        // wrapper for \"add\" operation; add value at ptr\n        const auto operation_add = [&result](json_pointer & ptr, basic_json val)\n        {\n            // adding to the root of the target document means replacing it\n            if (ptr.empty())\n            {\n                result = val;\n                return;\n            }\n\n            // make sure the top element of the pointer exists\n            json_pointer top_pointer = ptr.top();\n            if (top_pointer != ptr)\n            {\n                result.at(top_pointer);\n            }\n\n            // get reference to parent of JSON pointer ptr\n            const auto last_path = ptr.back();\n            ptr.pop_back();\n            // parent must exist when performing patch add per RFC6902 specs\n            basic_json& parent = result.at(ptr);\n\n            switch (parent.m_type)\n            {\n                case value_t::null:\n                case value_t::object:\n                {\n                    // use operator[] to add value\n                    parent[last_path] = val;\n                    break;\n                }\n\n                case value_t::array:\n                {\n                    if (last_path == \"-\")\n                    {\n                        // special case: append to back\n                        parent.push_back(val);\n                    }\n                    else\n                    {\n                        const auto idx = json_pointer::template array_index<basic_json_t>(last_path);\n                        if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))\n                        {\n                            // avoid undefined behavior\n                            JSON_THROW(out_of_range::create(401, detail::concat(\"array index \", std::to_string(idx), \" is out of range\"), &parent));\n                        }\n\n                        // default case: insert add offset\n                        parent.insert(parent.begin() + static_cast<difference_type>(idx), val);\n                    }\n                    break;\n                }\n\n                // if there exists a parent it cannot be primitive\n                case value_t::string: // LCOV_EXCL_LINE\n                case value_t::boolean: // LCOV_EXCL_LINE\n                case value_t::number_integer: // LCOV_EXCL_LINE\n                case value_t::number_unsigned: // LCOV_EXCL_LINE\n                case value_t::number_float: // LCOV_EXCL_LINE\n                case value_t::binary: // LCOV_EXCL_LINE\n                case value_t::discarded: // LCOV_EXCL_LINE\n                default:            // LCOV_EXCL_LINE\n                    JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE\n            }\n        };\n\n        // wrapper for \"remove\" operation; remove value at ptr\n        const auto operation_remove = [this, &result](json_pointer & ptr)\n        {\n            // get reference to parent of JSON pointer ptr\n            const auto last_path = ptr.back();\n            ptr.pop_back();\n            basic_json& parent = result.at(ptr);\n\n            // remove child\n            if (parent.is_object())\n            {\n                // perform range check\n                auto it = parent.find(last_path);\n                if (JSON_HEDLEY_LIKELY(it != parent.end()))\n                {\n                    parent.erase(it);\n                }\n                else\n                {\n                    JSON_THROW(out_of_range::create(403, detail::concat(\"key '\", last_path, \"' not found\"), this));\n                }\n            }\n            else if (parent.is_array())\n            {\n                // note erase performs range check\n                parent.erase(json_pointer::template array_index<basic_json_t>(last_path));\n            }\n        };\n\n        // type check: top level value must be an array\n        if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array()))\n        {\n            JSON_THROW(parse_error::create(104, 0, \"JSON patch must be an array of objects\", &json_patch));\n        }\n\n        // iterate and apply the operations\n        for (const auto& val : json_patch)\n        {\n            // wrapper to get a value for an operation\n            const auto get_value = [&val](const std::string & op,\n                                          const std::string & member,\n                                          bool string_type) -> basic_json &\n            {\n                // find value\n                auto it = val.m_value.object->find(member);\n\n                // context-sensitive error message\n                const auto error_msg = (op == \"op\") ? \"operation\" : detail::concat(\"operation '\", op, '\\'');\n\n                // check if desired value is present\n                if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end()))\n                {\n                    // NOLINTNEXTLINE(performance-inefficient-string-concatenation)\n                    JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, \" must have member '\", member, \"'\"), &val));\n                }\n\n                // check if result is of type string\n                if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string()))\n                {\n                    // NOLINTNEXTLINE(performance-inefficient-string-concatenation)\n                    JSON_THROW(parse_error::create(105, 0, detail::concat(error_msg, \" must have string member '\", member, \"'\"), &val));\n                }\n\n                // no error: return value\n                return it->second;\n            };\n\n            // type check: every element of the array must be an object\n            if (JSON_HEDLEY_UNLIKELY(!val.is_object()))\n            {\n                JSON_THROW(parse_error::create(104, 0, \"JSON patch must be an array of objects\", &val));\n            }\n\n            // collect mandatory members\n            const auto op = get_value(\"op\", \"op\", true).template get<std::string>();\n            const auto path = get_value(op, \"path\", true).template get<std::string>();\n            json_pointer ptr(path);\n\n            switch (get_op(op))\n            {\n                case patch_operations::add:\n                {\n                    operation_add(ptr, get_value(\"add\", \"value\", false));\n                    break;\n                }\n\n                case patch_operations::remove:\n                {\n                    operation_remove(ptr);\n                    break;\n                }\n\n                case patch_operations::replace:\n                {\n                    // the \"path\" location must exist - use at()\n                    result.at(ptr) = get_value(\"replace\", \"value\", false);\n                    break;\n                }\n\n                case patch_operations::move:\n                {\n                    const auto from_path = get_value(\"move\", \"from\", true).template get<std::string>();\n                    json_pointer from_ptr(from_path);\n\n                    // the \"from\" location must exist - use at()\n                    basic_json v = result.at(from_ptr);\n\n                    // The move operation is functionally identical to a\n                    // \"remove\" operation on the \"from\" location, followed\n                    // immediately by an \"add\" operation at the target\n                    // location with the value that was just removed.\n                    operation_remove(from_ptr);\n                    operation_add(ptr, v);\n                    break;\n                }\n\n                case patch_operations::copy:\n                {\n                    const auto from_path = get_value(\"copy\", \"from\", true).template get<std::string>();\n                    const json_pointer from_ptr(from_path);\n\n                    // the \"from\" location must exist - use at()\n                    basic_json v = result.at(from_ptr);\n\n                    // The copy is functionally identical to an \"add\"\n                    // operation at the target location using the value\n                    // specified in the \"from\" member.\n                    operation_add(ptr, v);\n                    break;\n                }\n\n                case patch_operations::test:\n                {\n                    bool success = false;\n                    JSON_TRY\n                    {\n                        // check if \"value\" matches the one at \"path\"\n                        // the \"path\" location must exist - use at()\n                        success = (result.at(ptr) == get_value(\"test\", \"value\", false));\n                    }\n                    JSON_INTERNAL_CATCH (out_of_range&)\n                    {\n                        // ignore out of range errors: success remains false\n                    }\n\n                    // throw an exception if test fails\n                    if (JSON_HEDLEY_UNLIKELY(!success))\n                    {\n                        JSON_THROW(other_error::create(501, detail::concat(\"unsuccessful: \", val.dump()), &val));\n                    }\n\n                    break;\n                }\n\n                case patch_operations::invalid:\n                default:\n                {\n                    // op must be \"add\", \"remove\", \"replace\", \"move\", \"copy\", or\n                    // \"test\"\n                    JSON_THROW(parse_error::create(105, 0, detail::concat(\"operation value '\", op, \"' is invalid\"), &val));\n                }\n            }\n        }\n    }\n\n    /// @brief applies a JSON patch to a copy of the current object\n    /// @sa https://json.nlohmann.me/api/basic_json/patch/\n    basic_json patch(const basic_json& json_patch) const\n    {\n        basic_json result = *this;\n        result.patch_inplace(json_patch);\n        return result;\n    }\n\n    /// @brief creates a diff as a JSON patch\n    /// @sa https://json.nlohmann.me/api/basic_json/diff/\n    JSON_HEDLEY_WARN_UNUSED_RESULT\n    static basic_json diff(const basic_json& source, const basic_json& target,\n                           const std::string& path = \"\")\n    {\n        // the patch\n        basic_json result(value_t::array);\n\n        // if the values are the same, return empty patch\n        if (source == target)\n        {\n            return result;\n        }\n\n        if (source.type() != target.type())\n        {\n            // different types: replace value\n            result.push_back(\n            {\n                {\"op\", \"replace\"}, {\"path\", path}, {\"value\", target}\n            });\n            return result;\n        }\n\n        switch (source.type())\n        {\n            case value_t::array:\n            {\n                // first pass: traverse common elements\n                std::size_t i = 0;\n                while (i < source.size() && i < target.size())\n                {\n                    // recursive call to compare array values at index i\n                    auto temp_diff = diff(source[i], target[i], detail::concat(path, '/', std::to_string(i)));\n                    result.insert(result.end(), temp_diff.begin(), temp_diff.end());\n                    ++i;\n                }\n\n                // We now reached the end of at least one array\n                // in a second pass, traverse the remaining elements\n\n                // remove my remaining elements\n                const auto end_index = static_cast<difference_type>(result.size());\n                while (i < source.size())\n                {\n                    // add operations in reverse order to avoid invalid\n                    // indices\n                    result.insert(result.begin() + end_index, object(\n                    {\n                        {\"op\", \"remove\"},\n                        {\"path\", detail::concat(path, '/', std::to_string(i))}\n                    }));\n                    ++i;\n                }\n\n                // add other remaining elements\n                while (i < target.size())\n                {\n                    result.push_back(\n                    {\n                        {\"op\", \"add\"},\n                        {\"path\", detail::concat(path, \"/-\")},\n                        {\"value\", target[i]}\n                    });\n                    ++i;\n                }\n\n                break;\n            }\n\n            case value_t::object:\n            {\n                // first pass: traverse this object's elements\n                for (auto it = source.cbegin(); it != source.cend(); ++it)\n                {\n                    // escape the key name to be used in a JSON patch\n                    const auto path_key = detail::concat(path, '/', detail::escape(it.key()));\n\n                    if (target.find(it.key()) != target.end())\n                    {\n                        // recursive call to compare object values at key it\n                        auto temp_diff = diff(it.value(), target[it.key()], path_key);\n                        result.insert(result.end(), temp_diff.begin(), temp_diff.end());\n                    }\n                    else\n                    {\n                        // found a key that is not in o -> remove it\n                        result.push_back(object(\n                        {\n                            {\"op\", \"remove\"}, {\"path\", path_key}\n                        }));\n                    }\n                }\n\n                // second pass: traverse other object's elements\n                for (auto it = target.cbegin(); it != target.cend(); ++it)\n                {\n                    if (source.find(it.key()) == source.end())\n                    {\n                        // found a key that is not in this -> add it\n                        const auto path_key = detail::concat(path, '/', detail::escape(it.key()));\n                        result.push_back(\n                        {\n                            {\"op\", \"add\"}, {\"path\", path_key},\n                            {\"value\", it.value()}\n                        });\n                    }\n                }\n\n                break;\n            }\n\n            case value_t::null:\n            case value_t::string:\n            case value_t::boolean:\n            case value_t::number_integer:\n            case value_t::number_unsigned:\n            case value_t::number_float:\n            case value_t::binary:\n            case value_t::discarded:\n            default:\n            {\n                // both primitive type: replace value\n                result.push_back(\n                {\n                    {\"op\", \"replace\"}, {\"path\", path}, {\"value\", target}\n                });\n                break;\n            }\n        }\n\n        return result;\n    }\n    /// @}\n\n    ////////////////////////////////\n    // JSON Merge Patch functions //\n    ////////////////////////////////\n\n    /// @name JSON Merge Patch functions\n    /// @{\n\n    /// @brief applies a JSON Merge Patch\n    /// @sa https://json.nlohmann.me/api/basic_json/merge_patch/\n    void merge_patch(const basic_json& apply_patch)\n    {\n        if (apply_patch.is_object())\n        {\n            if (!is_object())\n            {\n                *this = object();\n            }\n            for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it)\n            {\n                if (it.value().is_null())\n                {\n                    erase(it.key());\n                }\n                else\n                {\n                    operator[](it.key()).merge_patch(it.value());\n                }\n            }\n        }\n        else\n        {\n            *this = apply_patch;\n        }\n    }\n\n    /// @}\n};\n\n/// @brief user-defined to_string function for JSON values\n/// @sa https://json.nlohmann.me/api/basic_json/to_string/\nNLOHMANN_BASIC_JSON_TPL_DECLARATION\nstd::string to_string(const NLOHMANN_BASIC_JSON_TPL& j)\n{\n    return j.dump();\n}\n\ninline namespace literals\n{\ninline namespace json_literals\n{\n\n/// @brief user-defined string literal for JSON values\n/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/\nJSON_HEDLEY_NON_NULL(1)\ninline nlohmann::json operator \"\" _json(const char* s, std::size_t n)\n{\n    return nlohmann::json::parse(s, s + n);\n}\n\n/// @brief user-defined string literal for JSON pointer\n/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/\nJSON_HEDLEY_NON_NULL(1)\ninline nlohmann::json::json_pointer operator \"\" _json_pointer(const char* s, std::size_t n)\n{\n    return nlohmann::json::json_pointer(std::string(s, n));\n}\n\n}  // namespace json_literals\n}  // namespace literals\nNLOHMANN_JSON_NAMESPACE_END\n\n///////////////////////\n// nonmember support //\n///////////////////////\n\nnamespace std // NOLINT(cert-dcl58-cpp)\n{\n\n/// @brief hash value for JSON objects\n/// @sa https://json.nlohmann.me/api/basic_json/std_hash/\nNLOHMANN_BASIC_JSON_TPL_DECLARATION\nstruct hash<nlohmann::NLOHMANN_BASIC_JSON_TPL>\n{\n    std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const\n    {\n        return nlohmann::detail::hash(j);\n    }\n};\n\n// specialization for std::less<value_t>\ntemplate<>\nstruct less< ::nlohmann::detail::value_t> // do not remove the space after '<', see https://github.com/nlohmann/json/pull/679\n{\n    /*!\n    @brief compare two value_t enum values\n    @since version 3.0.0\n    */\n    bool operator()(::nlohmann::detail::value_t lhs,\n                    ::nlohmann::detail::value_t rhs) const noexcept\n    {\n#if JSON_HAS_THREE_WAY_COMPARISON\n        return std::is_lt(lhs <=> rhs); // *NOPAD*\n#else\n        return ::nlohmann::detail::operator<(lhs, rhs);\n#endif\n    }\n};\n\n// C++20 prohibit function specialization in the std namespace.\n#ifndef JSON_HAS_CPP_20\n\n/// @brief exchanges the values of two JSON objects\n/// @sa https://json.nlohmann.me/api/basic_json/std_swap/\nNLOHMANN_BASIC_JSON_TPL_DECLARATION\ninline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept(  // NOLINT(readability-inconsistent-declaration-parameter-name)\n    is_nothrow_move_constructible<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value&&                          // NOLINT(misc-redundant-expression)\n    is_nothrow_move_assignable<nlohmann::NLOHMANN_BASIC_JSON_TPL>::value)\n{\n    j1.swap(j2);\n}\n\n#endif\n\n}  // namespace std\n\n#if JSON_USE_GLOBAL_UDLS\n    using nlohmann::literals::json_literals::operator \"\" _json; // NOLINT(misc-unused-using-decls,google-global-names-in-headers)\n    using nlohmann::literals::json_literals::operator \"\" _json_pointer; //NOLINT(misc-unused-using-decls,google-global-names-in-headers)\n#endif\n\n// #include <nlohmann/detail/macro_unscope.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n// restore clang diagnostic settings\n#if defined(__clang__)\n    #pragma clang diagnostic pop\n#endif\n\n// clean up\n#undef JSON_ASSERT\n#undef JSON_INTERNAL_CATCH\n#undef JSON_THROW\n#undef JSON_PRIVATE_UNLESS_TESTED\n#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION\n#undef NLOHMANN_BASIC_JSON_TPL\n#undef JSON_EXPLICIT\n#undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL\n#undef JSON_INLINE_VARIABLE\n#undef JSON_NO_UNIQUE_ADDRESS\n#undef JSON_DISABLE_ENUM_SERIALIZATION\n#undef JSON_USE_GLOBAL_UDLS\n\n#ifndef JSON_TEST_KEEP_MACROS\n    #undef JSON_CATCH\n    #undef JSON_TRY\n    #undef JSON_HAS_CPP_11\n    #undef JSON_HAS_CPP_14\n    #undef JSON_HAS_CPP_17\n    #undef JSON_HAS_CPP_20\n    #undef JSON_HAS_FILESYSTEM\n    #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM\n    #undef JSON_HAS_THREE_WAY_COMPARISON\n    #undef JSON_HAS_RANGES\n    #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON\n#endif\n\n// #include <nlohmann/thirdparty/hedley/hedley_undef.hpp>\n//     __ _____ _____ _____\n//  __|  |   __|     |   | |  JSON for Modern C++\n// |  |  |__   |  |  | | | |  version 3.11.2\n// |_____|_____|_____|_|___|  https://github.com/nlohmann/json\n//\n// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann <https://nlohmann.me>\n// SPDX-License-Identifier: MIT\n\n\n\n#undef JSON_HEDLEY_ALWAYS_INLINE\n#undef JSON_HEDLEY_ARM_VERSION\n#undef JSON_HEDLEY_ARM_VERSION_CHECK\n#undef JSON_HEDLEY_ARRAY_PARAM\n#undef JSON_HEDLEY_ASSUME\n#undef JSON_HEDLEY_BEGIN_C_DECLS\n#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE\n#undef JSON_HEDLEY_CLANG_HAS_BUILTIN\n#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE\n#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE\n#undef JSON_HEDLEY_CLANG_HAS_EXTENSION\n#undef JSON_HEDLEY_CLANG_HAS_FEATURE\n#undef JSON_HEDLEY_CLANG_HAS_WARNING\n#undef JSON_HEDLEY_COMPCERT_VERSION\n#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK\n#undef JSON_HEDLEY_CONCAT\n#undef JSON_HEDLEY_CONCAT3\n#undef JSON_HEDLEY_CONCAT3_EX\n#undef JSON_HEDLEY_CONCAT_EX\n#undef JSON_HEDLEY_CONST\n#undef JSON_HEDLEY_CONSTEXPR\n#undef JSON_HEDLEY_CONST_CAST\n#undef JSON_HEDLEY_CPP_CAST\n#undef JSON_HEDLEY_CRAY_VERSION\n#undef JSON_HEDLEY_CRAY_VERSION_CHECK\n#undef JSON_HEDLEY_C_DECL\n#undef JSON_HEDLEY_DEPRECATED\n#undef JSON_HEDLEY_DEPRECATED_FOR\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS\n#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION\n#undef JSON_HEDLEY_DIAGNOSTIC_POP\n#undef JSON_HEDLEY_DIAGNOSTIC_PUSH\n#undef JSON_HEDLEY_DMC_VERSION\n#undef JSON_HEDLEY_DMC_VERSION_CHECK\n#undef JSON_HEDLEY_EMPTY_BASES\n#undef JSON_HEDLEY_EMSCRIPTEN_VERSION\n#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK\n#undef JSON_HEDLEY_END_C_DECLS\n#undef JSON_HEDLEY_FLAGS\n#undef JSON_HEDLEY_FLAGS_CAST\n#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE\n#undef JSON_HEDLEY_GCC_HAS_BUILTIN\n#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE\n#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE\n#undef JSON_HEDLEY_GCC_HAS_EXTENSION\n#undef JSON_HEDLEY_GCC_HAS_FEATURE\n#undef JSON_HEDLEY_GCC_HAS_WARNING\n#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK\n#undef JSON_HEDLEY_GCC_VERSION\n#undef JSON_HEDLEY_GCC_VERSION_CHECK\n#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE\n#undef JSON_HEDLEY_GNUC_HAS_BUILTIN\n#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE\n#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE\n#undef JSON_HEDLEY_GNUC_HAS_EXTENSION\n#undef JSON_HEDLEY_GNUC_HAS_FEATURE\n#undef JSON_HEDLEY_GNUC_HAS_WARNING\n#undef JSON_HEDLEY_GNUC_VERSION\n#undef JSON_HEDLEY_GNUC_VERSION_CHECK\n#undef JSON_HEDLEY_HAS_ATTRIBUTE\n#undef JSON_HEDLEY_HAS_BUILTIN\n#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE\n#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS\n#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE\n#undef JSON_HEDLEY_HAS_EXTENSION\n#undef JSON_HEDLEY_HAS_FEATURE\n#undef JSON_HEDLEY_HAS_WARNING\n#undef JSON_HEDLEY_IAR_VERSION\n#undef JSON_HEDLEY_IAR_VERSION_CHECK\n#undef JSON_HEDLEY_IBM_VERSION\n#undef JSON_HEDLEY_IBM_VERSION_CHECK\n#undef JSON_HEDLEY_IMPORT\n#undef JSON_HEDLEY_INLINE\n#undef JSON_HEDLEY_INTEL_CL_VERSION\n#undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK\n#undef JSON_HEDLEY_INTEL_VERSION\n#undef JSON_HEDLEY_INTEL_VERSION_CHECK\n#undef JSON_HEDLEY_IS_CONSTANT\n#undef JSON_HEDLEY_IS_CONSTEXPR_\n#undef JSON_HEDLEY_LIKELY\n#undef JSON_HEDLEY_MALLOC\n#undef JSON_HEDLEY_MCST_LCC_VERSION\n#undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK\n#undef JSON_HEDLEY_MESSAGE\n#undef JSON_HEDLEY_MSVC_VERSION\n#undef JSON_HEDLEY_MSVC_VERSION_CHECK\n#undef JSON_HEDLEY_NEVER_INLINE\n#undef JSON_HEDLEY_NON_NULL\n#undef JSON_HEDLEY_NO_ESCAPE\n#undef JSON_HEDLEY_NO_RETURN\n#undef JSON_HEDLEY_NO_THROW\n#undef JSON_HEDLEY_NULL\n#undef JSON_HEDLEY_PELLES_VERSION\n#undef JSON_HEDLEY_PELLES_VERSION_CHECK\n#undef JSON_HEDLEY_PGI_VERSION\n#undef JSON_HEDLEY_PGI_VERSION_CHECK\n#undef JSON_HEDLEY_PREDICT\n#undef JSON_HEDLEY_PRINTF_FORMAT\n#undef JSON_HEDLEY_PRIVATE\n#undef JSON_HEDLEY_PUBLIC\n#undef JSON_HEDLEY_PURE\n#undef JSON_HEDLEY_REINTERPRET_CAST\n#undef JSON_HEDLEY_REQUIRE\n#undef JSON_HEDLEY_REQUIRE_CONSTEXPR\n#undef JSON_HEDLEY_REQUIRE_MSG\n#undef JSON_HEDLEY_RESTRICT\n#undef JSON_HEDLEY_RETURNS_NON_NULL\n#undef JSON_HEDLEY_SENTINEL\n#undef JSON_HEDLEY_STATIC_ASSERT\n#undef JSON_HEDLEY_STATIC_CAST\n#undef JSON_HEDLEY_STRINGIFY\n#undef JSON_HEDLEY_STRINGIFY_EX\n#undef JSON_HEDLEY_SUNPRO_VERSION\n#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK\n#undef JSON_HEDLEY_TINYC_VERSION\n#undef JSON_HEDLEY_TINYC_VERSION_CHECK\n#undef JSON_HEDLEY_TI_ARMCL_VERSION\n#undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK\n#undef JSON_HEDLEY_TI_CL2000_VERSION\n#undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK\n#undef JSON_HEDLEY_TI_CL430_VERSION\n#undef JSON_HEDLEY_TI_CL430_VERSION_CHECK\n#undef JSON_HEDLEY_TI_CL6X_VERSION\n#undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK\n#undef JSON_HEDLEY_TI_CL7X_VERSION\n#undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK\n#undef JSON_HEDLEY_TI_CLPRU_VERSION\n#undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK\n#undef JSON_HEDLEY_TI_VERSION\n#undef JSON_HEDLEY_TI_VERSION_CHECK\n#undef JSON_HEDLEY_UNAVAILABLE\n#undef JSON_HEDLEY_UNLIKELY\n#undef JSON_HEDLEY_UNPREDICTABLE\n#undef JSON_HEDLEY_UNREACHABLE\n#undef JSON_HEDLEY_UNREACHABLE_RETURN\n#undef JSON_HEDLEY_VERSION\n#undef JSON_HEDLEY_VERSION_DECODE_MAJOR\n#undef JSON_HEDLEY_VERSION_DECODE_MINOR\n#undef JSON_HEDLEY_VERSION_DECODE_REVISION\n#undef JSON_HEDLEY_VERSION_ENCODE\n#undef JSON_HEDLEY_WARNING\n#undef JSON_HEDLEY_WARN_UNUSED_RESULT\n#undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG\n#undef JSON_HEDLEY_FALL_THROUGH\n\n\n\n#endif  // INCLUDE_NLOHMANN_JSON_HPP_\n"
  },
  {
    "path": "examples/livestream.sh",
    "content": "#!/bin/bash\n#\n# Transcribe audio livestream by feeding ffmpeg output to whisper.cpp at regular intervals\n# Idea by @semiformal-net\n# ref: https://github.com/ggml-org/whisper.cpp/issues/185\n#\n\nset -eo pipefail\n\nurl=\"http://a.files.bbci.co.uk/media/live/manifesto/audio/simulcast/hls/nonuk/sbr_low/ak/bbc_world_service.m3u8\"\nfmt=aac # the audio format extension of the stream (TODO: auto detect)\nstep_s=30\nmodel=\"base.en\"\n\ncheck_requirements()\n{\n    if ! command -v ./build/bin/whisper-cli &>/dev/null; then\n        echo \"whisper.cpp main executable is required (make)\"\n        exit 1\n    fi\n\n    if ! command -v ffmpeg &>/dev/null; then\n        echo \"ffmpeg is required (https://ffmpeg.org)\"\n        exit 1\n    fi\n}\n\ncheck_requirements\n\n\nif [ -z \"$1\" ]; then\n    echo \"Usage: $0 stream_url [step_s] [model]\"\n    echo \"\"\n    echo \"  Example:\"\n    echo \"    $0 $url $step_s $model\"\n    echo \"\"\n    echo \"No url specified, using default: $url\"\nelse\n    url=\"$1\"\nfi\n\nif [ -n \"$2\" ]; then\n    step_s=\"$2\"\nfi\n\nif [ -n \"$3\" ]; then\n    model=\"$3\"\nfi\n\n# Whisper models\nmodels=( \"tiny.en\" \"tiny\" \"base.en\" \"base\" \"small.en\" \"small\" \"medium.en\" \"medium\" \"large-v1\" \"large-v2\" \"large-v3\" \"large-v3-turbo\" )\n\n# list available models\nfunction list_models {\n    printf \"\\n\"\n    printf \"  Available models:\"\n    for model in \"${models[@]}\"; do\n        printf \" $model\"\n    done\n    printf \"\\n\\n\"\n}\n\nif [[ ! \" ${models[@]} \" =~ \" ${model} \" ]]; then\n    printf \"Invalid model: $model\\n\"\n    list_models\n\n    exit 1\nfi\n\nrunning=1\n\ntrap \"running=0\" SIGINT SIGTERM\n\nprintf \"[+] Transcribing stream with model '$model', step_s $step_s (press Ctrl+C to stop):\\n\\n\"\n\n# continuous stream in native fmt (this file will grow forever!)\nffmpeg -loglevel quiet -y -re -probesize 32 -i $url -c copy /tmp/whisper-live0.${fmt} &\nif [ $? -ne 0 ]; then\n    printf \"Error: ffmpeg failed to capture audio stream\\n\"\n    exit 1\nfi\n\nprintf \"Buffering audio. Please wait...\\n\\n\"\nsleep $(($step_s))\n\n# do not stop script on error\nset +e\n\ni=0\nSECONDS=0\nwhile [ $running -eq 1 ]; do\n    # extract the next piece from the main file above and transcode to wav. -ss sets start time and nudges it by -0.5s to catch missing words (??)\n    err=1\n    while [ $err -ne 0 ]; do\n        if [ $i -gt 0 ]; then\n            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.${fmt} -y -ar 16000 -ac 1 -c:a pcm_s16le -ss $(($i*$step_s-1)).5 -t $step_s /tmp/whisper-live.wav 2> /tmp/whisper-live.err\n        else\n            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.${fmt} -y -ar 16000 -ac 1 -c:a pcm_s16le -ss $(($i*$step_s)) -t $step_s /tmp/whisper-live.wav 2> /tmp/whisper-live.err\n        fi\n        err=$(cat /tmp/whisper-live.err | wc -l)\n    done\n\n    ./build/bin/whisper-cli -t 8 -m ./models/ggml-${model}.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1\n\n    while [ $SECONDS -lt $((($i+1)*$step_s)) ]; do\n        sleep 1\n    done\n    ((i=i+1))\ndone\n\nkillall -v ffmpeg\nkillall -v whisper-cli\n"
  },
  {
    "path": "examples/lsp/CMakeLists.txt",
    "content": "if (WHISPER_SDL2)\n    # stream\n    set(TARGET whisper-lsp)\n    add_executable(${TARGET} lsp.cpp)\n\n    include(DefaultTargetOptions)\n\n    target_link_libraries(${TARGET} PRIVATE common json_cpp common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})\n    install(TARGETS ${TARGET} RUNTIME)\nendif ()\n"
  },
  {
    "path": "examples/lsp/README.md",
    "content": "# Language Server\r\n\r\nThis example consists of a simple language server to expose both unguided\r\nand guided (command) transcriptions by sending json messages over stdout/stdin\r\nas well as a rather robust vim plugin that makes use of the language server.\r\n\r\n## Vim plugin quick start\r\n\r\nCompile the language server with\r\n\r\n```bash\r\nmake lsp\r\n```\r\nInstall the plugin itself by copying or symlinking whisper.vim into ~/.vim/autoload/\r\n\r\nIn your vimrc, set the path of your whisper.cpp directory and optionally add some keybinds.\r\n\r\n```vim\r\nlet g:whisper_dir = \"~/whisper.cpp\"\r\n\" Start listening for commands when Ctrl - g is pressed in normal mode\r\nnnoremap <C-G> call whisper#requestCommands()<CR>\r\n\" Start unguided transcription when Ctrl - g is pressed in insert mode\r\ninoremap <C-G> <Cmd>call whisper#doTranscription()<CR>\r\n```\r\n\r\n## Vim plugin usage\r\n\r\nThe vim plugin was designed to closely follow the mnemonics of vim\r\n\r\n`s:spoken_dict` is used to translate keys to their spoken form.\r\n\r\n\r\nKeys corresponding to a string use that spoken value normally and when a motion is expected, but use the key itself when a character is expected.  \r\nKeys corresponding to a dict, like `i`, can have manual difinitions given to each possible commandset.\r\n\r\n0 is normal (insert), 1 is motion (inside), 2 is it's usage as a single key ([till] i), and 3 is it's usage in an area selection (s -> [around] sentence)\r\n\r\nSome punctuation items, like `-` are explicitly given pronunciations to prevent them from being picked as punctuation instead of an actual command word.\r\n\r\nNot all commands will tokenize to a single token and this can interfere with interpretation. \"yank\" as an example, takes multiple tokens and correspondingly, will give more accurate detection when only the first \"ya\" is used. While it could be changed to something else that is a single token (copy), value was placed on maintaining vim mnemonics.\r\n\r\nCommands that would normally move the editor into insert mode (insert, append, open, change) will begin unguided transcription.\r\nUnguided transcription will end when a speech segment ends in exit.\r\nPresence of punctuation can be designated by whether or not you add a pause between the previous speech segment and exit.\r\nExiting only occurs if exit is the last word, so \"Take the first exit on your right\" would not cause transcription to end.\r\n\r\nAfter a command is evaluated, the plugin will continue listening for the next command.\r\n\r\nWhile in command mode, \"Exit\" will end listening.\r\n\r\nA best effort approach is taken to keep track of audio that is recorded while a previous chunk is still processing and immediately interpret it afterwards, but the current voice detection still needs a fairly sizable gap to determine when a command has been spoken.\r\n\r\nLog information is sent to a special `whisper_log` buffer and can be accessed with\r\n```vim\r\n:e whisper_log\r\n```\r\n\r\n## Vim plugin configuration\r\n\r\n`g:whisper_dir`  \r\nA full path to the whisper.cpp repo. It can be expanded in the definition like so:\r\n```vim\r\nlet g:whisper_dir = expand(\"~/whisper.cpp/\")\r\n```\r\n(The WHISPER_CPP_HOME environment variable is also checked for users of the existing whisper.nvim script)\r\n\r\n`g:whisper_lsp_path`  \r\nCan be used to manually set the path to the language server.\r\nIf not defined, it will be inferred from the above whisper_dir\r\n\r\n`g:whisper_model_path`  \r\nA full path to the model to load. If not defined, it will default to ggml-base.en.bin\r\n\r\n`g:whisper_user_commands`  \r\nA dictionary of spoken commands that correspond to either strings or funcrefs.\r\nThis can be used to create connections with other user plugins, for example\r\n```vim\r\nlet g:whisper_user_commands = {\"gen\": \"llama#doLlamaGen\"}\r\n```\r\nwill trigger the llama.cpp plugin to begin generation when \"gen\" is spoken\r\n\r\n## Language server methods\r\n\r\n`registerCommandset`  \r\n`params` is a list of strings that should be checked for with this commandset. The server prepends a space to these strings before tokenizing.  \r\nResponds with  \r\n`result.index` an integer index for the commandset registered, which should be included when initiating a guided transcription to select this commandset.\r\nWill return an error if any of the commands in the commandset have duplicate tokenizations\r\n\r\n`guided`  \r\n`params.commandset_index` An index returned by a corresponding commandset registration. If not set, the most recently registered commandset is used.\r\n`params.timestamp` A positive unsigned integer which designates a point in time which audio should begin processing from. If left blank, the start point of audio processing will be the moment the message is recieved. This should be left blank unless you have a timestamp from a previous response.  \r\nResponds with  \r\n`result.command_index` The numerical index (starting from 0) of the detected command in the selected commandset\r\n`result.command_text` A string containing the command as provided in the commandset\r\n`result.timestamp` A positive unsigned integer that designates the point in time which audio stopped being processed at. Pass this timestamp back in a subsequent message to mask the latency of transcription.\r\n\r\n`unguided`  \r\n`params.no_context` Sets the corresponding whisper `no_context` param. Defaults to true. Might provide more accurate results for consecutive unguided transcriptions if those after the first are set to false.\r\n`params.prompt` If provided, sets the initial prompt used during transcription.\r\n`params.timestamp` A positive unsigned integer which designates a point in time which audio should begin processing from. If left blank, the start point of audio processing will be the moment the message is recieved. This should be left blank unless you have a timestamp from a previous response.  \r\nResponds with  \r\n`result.transcription` A string containing the transcribed text.  N.B. This will almost always start with a space due to how text is tokenized.\r\n`result.timestamp` A positive unsigned integer that designates the point in time which audio stopped being processed at. Pass this timestamp back in a subsequent message to mask the latency of transcription.\r\n"
  },
  {
    "path": "examples/lsp/lsp.cpp",
    "content": "#include \"common.h\"\n#include \"common-sdl.h\"\n#include \"whisper.h\"\n#include \"json.hpp\"\n\n#include <cassert>\n#include <chrono>\n#include <cstdio>\n#include <deque>\n#include <iostream>\n#include <set>\n#include <string>\n#include <thread>\n#include <vector>\n\nusing json = nlohmann::json;\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t prompt_ms  = 5000;\n    int32_t command_ms = 8000;\n    int32_t capture_id = -1;\n    int32_t max_tokens = 32;\n    int32_t audio_ctx  = 0;\n\n    float vad_thold    = 0.6f;\n    float freq_thold   = 100.0f;\n\n    bool translate     = false;\n    bool print_special = false;\n    bool print_energy  = false;\n    bool use_gpu       = true;\n    bool flash_attn    = true;\n\n    std::string language  = \"en\";\n    std::string model     = \"models/ggml-base.en.bin\";\n};\nstruct command {\n    std::vector<whisper_token> tokens;\n    std::string plaintext;\n};\nstruct commandset {\n    std::vector<struct command> commands;\n    std::vector<whisper_token> prompt_tokens;\n    // TODO: Store longest command?\n    // Multi-token commands should have probabilities of subsequent logits\n    // given that the prior logit is correct.\n    // In this case, all commands must be iterated.\n    // This however, is likely highly involved as different tokens\n    // almost certainly have different spoken lengths\n    // It would also have performance implications equivalent to a beam search\n};\n\nvoid whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"      || arg == \"--threads\")       { params.n_threads     = std::stoi(argv[++i]); }\n        else if (arg == \"-pms\"    || arg == \"--prompt-ms\")     { params.prompt_ms     = std::stoi(argv[++i]); }\n        else if (arg == \"-cms\"    || arg == \"--command-ms\")    { params.command_ms    = std::stoi(argv[++i]); }\n        else if (arg == \"-c\"      || arg == \"--capture\")       { params.capture_id    = std::stoi(argv[++i]); }\n        else if (arg == \"-mt\"     || arg == \"--max-tokens\")    { params.max_tokens    = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"     || arg == \"--audio-ctx\")     { params.audio_ctx     = std::stoi(argv[++i]); }\n        else if (arg == \"-vth\"    || arg == \"--vad-thold\")     { params.vad_thold     = std::stof(argv[++i]); }\n        else if (arg == \"-fth\"    || arg == \"--freq-thold\")    { params.freq_thold    = std::stof(argv[++i]); }\n        else if (arg == \"-tr\"     || arg == \"--translate\")     { params.translate     = true; }\n        else if (arg == \"-ps\"     || arg == \"--print-special\") { params.print_special = true; }\n        else if (arg == \"-pe\"     || arg == \"--print-energy\")  { params.print_energy  = true; }\n        else if (arg == \"-ng\"     || arg == \"--no-gpu\")        { params.use_gpu       = false; }\n        else if (arg == \"-fa\"     || arg == \"--flash-attn\")    { params.flash_attn    = true; }\n        else if (arg == \"-nfa\"    || arg == \"--no-flash-attn\") { params.flash_attn    = false; }\n        else if (arg == \"-l\"      || arg == \"--language\")      { params.language      = argv[++i]; }\n        else if (arg == \"-m\"      || arg == \"--model\")         { params.model         = argv[++i]; }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,         --help           [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,       --threads N      [%-7d] number of threads to use during computation\\n\", params.n_threads);\n    fprintf(stderr, \"  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\\n\",             params.prompt_ms);\n    fprintf(stderr, \"  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\\n\",            params.command_ms);\n    fprintf(stderr, \"  -c ID,      --capture ID     [%-7d] capture device ID\\n\",                           params.capture_id);\n    fprintf(stderr, \"  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\\n\",    params.max_tokens);\n    fprintf(stderr, \"  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\\n\",                params.audio_ctx);\n    fprintf(stderr, \"  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\\n\",        params.vad_thold);\n    fprintf(stderr, \"  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\\n\",                params.freq_thold);\n    fprintf(stderr, \"  -tr,        --translate      [%-7s] translate from source language to english\\n\",   params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,        --print-special  [%-7s] print special tokens\\n\",                        params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\\n\",          params.print_energy ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ng,        --no-gpu         [%-7s] disable GPU\\n\",                                 params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,        --flash-attn     [%-7s] enable flash attention\\n\",                      params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,       --no-flash-attn  [%-7s] disable flash attention\\n\",                     params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -l LANG,    --language LANG  [%-7s] spoken language\\n\",                             params.language.c_str());\n    fprintf(stderr, \"  -m FNAME,   --model FNAME    [%-7s] model path\\n\",                                  params.model.c_str());\n    fprintf(stderr, \"\\n\");\n}\nstatic uint64_t wait_for_vad(audio_async & audio, json jparams, const whisper_params & params, uint64_t maxlength_ms, std::vector<float> & pcmf32) {\n    using namespace std::chrono;\n    uint64_t time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();\n    uint64_t start_time = time_now;\n    if (jparams.contains(\"timestamp\")) {\n        start_time = jparams.at(\"timestamp\");\n    }\n    if(time_now - start_time < 500) {\n        //wait for a backlog of audio\n        std::this_thread::sleep_for(milliseconds(500 - (time_now - start_time)));\n        time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();\n    } else if (time_now - start_time > 1000) {\n        audio.get(time_now-start_time, pcmf32);\n        size_t max_offset = pcmf32.size() - WHISPER_SAMPLE_RATE;\n        for(size_t offset=0;offset < max_offset;offset+=WHISPER_SAMPLE_RATE/10) {\n            std::vector<float> audio_chunk(&pcmf32[offset], &pcmf32[offset+WHISPER_SAMPLE_RATE]);\n            if(::vad_simple(audio_chunk, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {\n                pcmf32.resize(offset+WHISPER_SAMPLE_RATE);\n                if (offset*1000/WHISPER_SAMPLE_RATE+1000 > maxlength_ms) {\n                    //remove samples from the beginning\n                    pcmf32.erase(pcmf32.begin(),pcmf32.end()-(maxlength_ms*WHISPER_SAMPLE_RATE/1000));\n                    fprintf(stderr, \"Shortened samples\");\n                }\n                return start_time + offset*1000/WHISPER_SAMPLE_RATE+1000;\n            }\n        }\n    }\n    size_t window_duration = std::max((uint64_t)1000, time_now-start_time);\n    audio.get(window_duration, pcmf32);\n    while (!::vad_simple(pcmf32, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, params.print_energy)) {\n        std::this_thread::sleep_for(milliseconds(100));\n        time_now = time_point_cast<milliseconds>(system_clock::now()).time_since_epoch().count();\n        window_duration = std::max((uint64_t)1000,time_now-start_time);\n        audio.get(window_duration, pcmf32);\n    }\n    if (time_now - start_time > maxlength_ms) {\n        audio.get(maxlength_ms, pcmf32);\n    } else {\n        audio.get(time_now - start_time, pcmf32);\n    }\n\n    return time_now;\n}\n\nstatic json unguided_transcription(struct whisper_context * ctx, audio_async &audio, json jparams, const whisper_params &params) {\n    std::vector<whisper_token> prompt_tokens;\n    std::vector<float> pcmf32;\n    uint64_t unprocessed_audio_timestamp = wait_for_vad(audio, jparams, params, 10000U, pcmf32);\n\n    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n    if (jparams.contains(\"prompt\")) {\n        // unlikely to see much use. Under normal circumstances, no_context would be set to false\n        std::string prompt = jparams.at(\"prompt\");\n        prompt_tokens.resize(1024);\n        int n = whisper_tokenize(ctx, prompt.c_str(), prompt_tokens.data(), 1024);\n        prompt_tokens.resize(n);\n\n        wparams.prompt_tokens    = prompt_tokens.data();\n        wparams.prompt_n_tokens  = prompt_tokens.size();\n    }\n    wparams.print_progress   = false;\n    wparams.print_special    = params.print_special;\n    wparams.print_realtime   = false;\n    wparams.print_timestamps = false;\n    wparams.translate        = params.translate;\n    wparams.no_context       = jparams.value(\"no_context\", true);\n    wparams.single_segment   = true;\n    wparams.max_tokens       = params.max_tokens;\n    wparams.language         = params.language.c_str();\n    wparams.n_threads        = params.n_threads;\n\n    wparams.audio_ctx        = params.audio_ctx;\n    wparams.suppress_nst     = true;\n    // run the transformer and a single decoding pass\n    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        fprintf(stderr, \"%s: ERROR: whisper_full() failed\\n\", __func__);\n        throw json{\n            {\"code\", -32803},\n            {\"message\", \"ERROR: whisper_full() failed\"}\n        };\n    }\n    std::string result = whisper_full_get_segment_text(ctx,0);\n    return json {\n        {\"transcription\", result},\n        {\"timestamp\", unprocessed_audio_timestamp}\n    };\n}\n\n// command-list mode\n// guide the transcription to match the most likely command from a provided list\nstatic json guided_transcription(struct whisper_context * ctx, audio_async &audio, const whisper_params &params, json jparams, std::vector<struct commandset> commandset_list) {\n    struct commandset cs = commandset_list[jparams.value(\"commandset_index\", commandset_list.size()-1)];\n    std::vector<float> pcmf32;\n    uint64_t unprocessed_audio_timestamp = wait_for_vad(audio, jparams, params, 2000U, pcmf32);\n\n    fprintf(stderr, \"%s: Speech detected! Processing ...\\n\", __func__);\n    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n    wparams.print_progress   = false;\n    wparams.print_special    = params.print_special;\n    wparams.print_realtime   = false;\n    wparams.print_timestamps = false;\n    wparams.translate        = params.translate;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.max_tokens       = 1;\n    wparams.language         = params.language.c_str();\n    wparams.n_threads        = params.n_threads;\n\n    wparams.audio_ctx        = params.audio_ctx;\n\n    // TODO: Do some time testing. Does an overly long prompt slow down processing?\n    // Set up command sets/precompute prompts\n    wparams.prompt_tokens    = cs.prompt_tokens.data();\n    wparams.prompt_n_tokens  = cs.prompt_tokens.size();\n    // TODO: properly expose as option\n    wparams.suppress_nst     = true;\n\n    // run the transformer and a single decoding pass\n    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        fprintf(stderr, \"%s: ERROR: whisper_full() failed\\n\", __func__);\n        throw json{\n            {\"code\", -32803},\n            {\"message\", \"ERROR: whisper_full() failed\"}//TODO: format string (sprintf?)\n        };\n    }\n\n    // estimate command probability\n    // NOTE: not optimal\n    {\n        const auto * logits = whisper_get_logits(ctx);\n\n        std::vector<float> probs(whisper_n_vocab(ctx), 0.0f);\n\n        // compute probs from logits via softmax\n        {\n            float max = -1e9;\n            for (int i = 0; i < (int) probs.size(); ++i) {\n                max = std::max(max, logits[i]);\n            }\n\n            float sum = 0.0f;\n            for (int i = 0; i < (int) probs.size(); ++i) {\n                probs[i] = expf(logits[i] - max);\n                sum += probs[i];\n            }\n\n            for (int i = 0; i < (int) probs.size(); ++i) {\n                probs[i] /= sum;\n            }\n        }\n\n        std::vector<std::pair<float, int>> probs_id;\n\n        // In my testing, the most verbose token is always the desired.\n        // TODO: Trim commandset struct once efficacy has been verified\n        for (int i = 0; i < (int) cs.commands.size(); ++i) {\n            probs_id.emplace_back(probs[cs.commands[i].tokens[0]], i);\n        }\n\n        // sort descending\n        {\n            using pair_type = decltype(probs_id)::value_type;\n            std::sort(probs_id.begin(), probs_id.end(), [](const pair_type & a, const pair_type & b) {\n                    return a.first > b.first;\n                    });\n        }\n        int id = probs_id[0].second;\n        return json{\n            {\"command_index\", id},\n                {\"command_text\", cs.commands[id].plaintext},\n                {\"timestamp\", unprocessed_audio_timestamp},\n        };\n    }\n}\n\nstatic json register_commandset(struct whisper_context * ctx, json jparams, std::vector<struct commandset> &commandset_list) {\n    // TODO: check for token collision\n    struct commandset cs;\n\n    std::string  k_prompt = \" select one from the available words: \";\n    std::set<whisper_token> token_set;\n    whisper_token tokens[32];\n    for (std::string s : jparams) {\n        std::vector<whisper_token> token_vec;\n        // The existing command implementation uses a nested for loop to tokenize single characters\n        // I fail to see the purpose of this when ' a' has a wholly different pronunciation than the start of ' apple'\n        const int n = whisper_tokenize(ctx, (\" \" + s).c_str(), tokens, 32);\n        if (n < 0) {\n            fprintf(stderr, \"%s: error: failed to tokenize command '%s'\\n\", __func__, s.c_str());\n            return 3;\n        }\n        token_vec.push_back(tokens[0]);\n        if (!token_set.insert(tokens[0]).second) {\n            fprintf(stderr, \"%s: warning: %s is a duplicate of an existing token\\n\", __func__, s.c_str());\n            throw json{\n                {\"code\",-31000},\n                {\"message\", \"Duplicate token in token set: \" + s}\n            };\n        }\n        if (n > 1) {// empty string if n=0? Should never occur\n            fprintf(stderr, \"%s: error: command is more than a single token: %s\\n\", __func__, s.c_str());\n        }\n        struct command command = {token_vec, s};\n        cs.commands.push_back(command);\n        k_prompt += s;\n    }\n    k_prompt = k_prompt.substr(0,k_prompt.length()-2) + \". Selected word:\";\n    cs.prompt_tokens.resize(1024);\n    int n = whisper_tokenize(ctx, k_prompt.c_str(), cs.prompt_tokens.data(), 1024);\n    cs.prompt_tokens.resize(n);\n    // prepare response\n    int index = commandset_list.size();\n    commandset_list.push_back(cs);\n    return json{{\"index\",index}};\n}\n\nstatic json seek(struct whisper_context * /*ctx*/, audio_async & /*audio*/, json /*params*/) {\n    // whisper_state has the pertinent offsets, but there also seem to be a large\n    // number of scratch buffers that would prevent rewinding context in a manner similar to llama\n    // I'll give this a another pass once everything else is implemented,\n    // but for now, it's unsupported\n    throw json {\n        {\"code\", -32601},\n            {\"message\", \"Seeking is not yet supported.\"}\n    };\n}\n\nstatic json parse_job(const json &body, struct whisper_context * ctx, audio_async &audio, const whisper_params &params, std::vector<struct commandset> &commandset_list) {\n    // See: https://www.jsonrpc.org/specification\n    json id = body.at(\"id\");\n    try {\n        std::string version = body.at(\"jsonrpc\");\n        if (version != \"2.0\") {\n            // unsupported version\n            throw json{\n                {\"code\", -3260},\n                {\"message\", \"invalid jsonrpc version\"}\n            };\n        }\n        std::string method = body.at(\"method\");\n        json jparams = json{{\"dummy\", \"dummy\"}};\n        if (body.contains(\"params\"))\n            jparams = body.at(\"params\");\n        json res;\n        // TODO: be consistent about argument order\n        fprintf(stderr, \"Dispatching a job\\n\");\n        if (method == \"unguided\")                { res = unguided_transcription(ctx, audio, jparams, params); }\n        else if (method == \"guided\")             { res = guided_transcription(ctx, audio, params, jparams, commandset_list); }\n        else if (method == \"seek\")               { res = seek(ctx, audio, jparams); }\n        else if (method == \"registerCommandset\") { res = register_commandset(ctx, jparams, commandset_list); }\n        else if (method == \"echo\")               { res = jparams; }\n\n\n        return json{\n            {\"jsonrpc\", \"2.0\"},\n                {\"result\", res},\n                {\"id\", id}\n        };\n    } catch(json ex) {\n        return json {\n            {\"jsonrpc\", \"2.0\"},\n                {\"error\", ex},\n                {\"id\", id}\n        };\n    }\n}\n\nstatic void process_loop(struct whisper_context * ctx, audio_async &audio, const whisper_params &params) {\n    std::deque<json> jobqueue;\n    std::vector<struct commandset> commandset_list;\n    while (true) {\n        // For eventual cancellation support, shouldn't block if job exists\n        if (std::cin.rdbuf()->in_avail() > 22 || jobqueue.size() == 0) {\n            int content_length;\n            if (scanf(\"Content-Length: %d\", &content_length) != 1) {\n                fprintf(stderr, \"Could not read input: %d\", std::cin.peek());\n                return;\n            }\n            // scanf leaves the new lines intact\n            std::cin.ignore(2);\n            if (std::cin.peek() != 13) {\n                // Content-Type. jsonrpc necessitates utf8.\n                std::cin.ignore(200,10);\n            }\n            std::cin.ignore(2);\n            // A message is being sent and blocking is acceptable\n            std::string content(content_length,'\\0');\n            std::cin.read(&content[0], content_length);\n            json job = json::parse(content);\n            // TODO: Some messages(cancellation) should skip queue here\n            if (job.is_array()) {\n                // response must also be batched. Will implement later\n                // for (subjob : job.begin())\n                // TODO: At the very least respond with an unsupported error.\n            } else {\n                jobqueue.push_back(job);\n            }\n        }\n        assert(jobqueue.size() > 0);\n        json job = jobqueue.front();\n        json resp = parse_job(job, ctx, audio, params, commandset_list);\n        if (resp != \"unfinished\") {\n            jobqueue.pop_front();\n            // send response\n            std::string data = resp.dump(-1, ' ', false, json::error_handler_t::replace);\n            fprintf(stdout, \"Content-Length: %d\\r\\n\\r\\n%s\\n\", (int)data.length()+1, data.c_str());\n            std::cout.flush();\n\n        }\n    }\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    if (whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    // whisper init\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n    // init audio\n\n    audio_async audio(30*1000);\n    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {\n        fprintf(stderr, \"%s: audio.init() failed!\\n\", __func__);\n        return 1;\n    }\n\n    audio.resume();\n    // TODO: Investigate why this is required. An extra second of startup latency is not great\n    // wait for 1 second to avoid any buffered noise\n    std::this_thread::sleep_for(std::chrono::milliseconds(1000));\n    audio.clear();\n    // TODO: consider some sort of indicator to designate loading has finished?\n    // Potentially better for the client to just start with a non-blocking message (register commands)\n    process_loop(ctx, audio, params);\n\n    audio.pause();\n    whisper_print_timings(ctx);\n    whisper_free(ctx);\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/lsp/whisper.vim",
    "content": "if !exists(\"g:whisper_dir\")\n    let g:whisper_dir = expand($WHISPER_CPP_HOME)\n    if g:whisper_dir == \"\"\n        echoerr \"Please provide a path to the whisper.cpp repo in either the $WHISPER_CPP_HOME environment variable, or g:whisper_dir\"\n    endif\nendif\nif !exists(\"g:whisper_lsp_path\")\n    let g:whisper_lsp_path = g:whisper_dir .. \"lsp\"\n    if !filereadable(g:whisper_lsp_path)\n        echoerr \"Was not able to locate a lsp executable at: \" .. g:whisper_lsp_path\n        throw \"Executable not found\"\n    endif\nendif\nif !exists(\"g:whisper_model_path\")\n    \" TODO: allow custom paths relative to the repo dir\n    let g:whisper_model_path = g:whisper_dir .. \"models/ggml-base.en.bin\"\n    if !filereadable(g:whisper_model_path)\n        echoerr \"Could not find model at: \" .. g:whisper_model_path\n        throw \"Model not found\"\n    endif\nendif\nlet s:output_buffer = bufnr(\"whisper_log\", v:true)\ncall setbufvar(s:output_buffer,\"&buftype\",\"nofile\")\nlet s:lsp_command = [g:whisper_lsp_path,\"-m\",g:whisper_model_path]\n\" For faster execution. TODO: server load multiple models/run multiple servers?\n\" let s:lsp_command = [g:whisper_lsp_path, \"-m\", g:whisper_dir .. \"models/ggml-tiny.en.bin\", \"-ac\", \"128\"]\n\n\" requestCommands([params_dict])\nfunc whisper#requestCommands(...)\n    let l:req = {\"method\": \"guided\", \"params\": {\"commandset_index\": 0}}\n    if a:0 > 0\n        call extend(l:req.params, a:1)\n    endif\n    let resp = ch_sendexpr(g:lsp_job, l:req, {\"callback\": function(\"s:commandCallback\", [l:req.params, 0])})\nendfunction\n\n\" doTranscription([params_dict])\nfunc whisper#doTranscription(...)\n    let l:req = {\"method\": \"unguided\", \"params\": {}}\n    if a:0 > 0\n        call extend(l:req.params, a:1)\n    endif\n    let resp = ch_sendexpr(g:lsp_job, l:req, {\"callback\": function(\"s:transcriptionCallback\", [function(\"s:insertText\"),function(\"s:endTranscription\")])})\nendfunction\n\n\" For testing\nfunc whisper#uppertest(cha)\n    echo tr(a:cha, s:c_lowerkeys, s:c_upperkeys)\nendfunction\n\n\n\" (upper, exit, count, motion, command, insert/append, save run) \"base\"\n\" (upper, exit, count, motion, command, inside/around)           \"motion/visual\"\n\" (upper, exit, count, motion, line,    inside/around)           \"command already entered\"\n\" (upper, exit, key,                                 )           \"from/till\"\n\n\" upper and lower keys is used to translate between cases with tr\n\" Must be sunchronized\nlet s:c_lowerkeys = \"1234567890-=qwertyuiop[]\\\\asdfghjkl;'zxcvbnm,./\\\"\"\nlet s:c_upperkeys = \"!@#$%^&*()_+QWERTYUIOP{}|ASDFGHJKL:\\\"ZXCVBNM<>?'\"\nlet s:c_count = split(\"1234567890\\\"\",'\\zs')\nlet s:c_command = split(\"ryuogpdxcv.iam\", '\\zs')\nlet s:c_motion = split(\"wetf'hjklnb$^)\",'\\zs')\n\" object words: Word, Sentence, Paragraph, [, (, <, Tag, {. \", '\nlet s:c_area = split(\"wsp])>t}\\\"'\",'\\zs')\n\"Special commands.\nlet s:c_special_always = [\"exit\", \"upper\"]\nlet s:c_special_normal = [\"save\", \"run\", \"space\"]\n\n\" If not in dict, key is spoken word,\n\" If key resolves to string, value is used for normal/motion, but key for chars\n\" If key resolves to dict, {0: \"normal\",1: \"motion\",2:\"single char\",3: \"area\"}\n\" Missing entries fall back as follows {0: \"required\", 1: 0, 2: \"key\", 3: 0}\nlet s:spoken_dict = {\"w\": \"word\", \"e\": \"end\", \"r\": \"replace\", \"t\": {0: \"till\", 3: \"tag\"}, \"y\": \"yank\", \"u\": \"undo\", \"i\": {0: \"insert\", 1: \"inside\"}, \"o\": \"open\", \"p\": {0: \"paste\", 3: \"paragraph\"},  \"a\": {0: \"append\", 1: \"around\"}, \"s\": {0: \"substitute\", 3: \"sentence\"}, \"d\": \"delete\", \"f\": \"from\", \"g\": \"go\", \"h\": \"left\", \"j\": \"down\", \"k\": \"up\", \"l\": \"right\", \"c\": \"change\", \"v\": \"visual\", \"b\": \"back\", \"n\": \"next\", \"m\": \"mark\", \".\": {0: \"repeat\", 2: \"period\"}, \"]\": {0: \"bracket\", 2: \"bracket\"}, \"'\": {0: \"jump\", 2: \"apostrophe\", 3:  \"apostrophe\"}, '\"': {0: 'register', 2: \"quotation\", 3: \"quotation\"}, \"-\": {0: \"minus\", 2: \"minus\"}, \"$\": {0: \"dollar\", 2: \"dollar\"}, \"^\": {0: \"carrot\", 2: \"carrot\"}, \")\": {0: \"sentence\", 2: \"parenthesis\",  3: \"parenthesis\"}, \"}\": {0: \"paragraph\", 2: \"brace\", 3: \"brace\"}, \">\": {0: \"indent\", 2: \"angle\", 3: \"angle\"}}\n\n\" Give this another pass. This seems overly hacky even if it's functional\nlet s:sub_tran_msg = \"\"\nfunc s:subTranProg(msg)\n    if s:sub_tran_msg != \"\"\n        let s:sub_tran_msg = s:sub_tran_msg .. a:msg\n        if mode() !=? 'v'\n            exe \"normal\" \"u\" .. s:sub_tran_msg\n        endif\n    else\n        if s:command_backlog == \"\"\n            \" this should not occur\n            call s:logCallback(0, \"Warning: Encountered sub transcription without prior command\")\n            let s:command_backlog = \"a\"\n        endif\n        if a:msg[0] == ' '\n            let s:sub_tran_msg = s:command_backlog .. a:msg[1:-1]\n        else\n            let s:sub_tran_msg = s:command_backlog  .. a:msg\n        endif\n        if mode() !=? 'v'\n            exe \"normal\" s:sub_tran_msg\n        endif\n    endif\n    call appendbufline(s:output_buffer, \"$\", s:sub_tran_msg ..  \":\" .. string(a:msg ))\nendfunction\n\nfunc s:subTranFinish(params, timestamp)\n    let s:repeat_command = s:sub_tran_msg\n    \" Visual selection is lot if used with streaming, so streaming of partial\n    \" transcriptions is disabled in visual mode\n    if mode() ==? 'v'\n        exe \"normal\" s:sub_tran_msg\n    endif\n    let s:sub_tran_msg = \"\"\n    let s:command_backlog = \"\"\n    exe \"normal a\\<C-G>u\"\n    let l:params = a:params\n    let l:params.timestamp = a:timestamp\n    if exists(\"l:params.commandset_index\")\n        unlet l:params.commandset_index\n    endif\n    call whisper#requestCommands(a:params)\nendfunction\n\nfunc s:logCallback(channel, msg)\n    call appendbufline(s:output_buffer,\"$\",a:msg)\nendfunction\n\n\nfunc s:transcriptionCallback(progressCallback, finishedCallback, channel, msg)\n    let l:tr = a:msg.result.transcription\n\n    let l:ex_ind = match(tolower(l:tr),\"exit\", len(l:tr)-6)\n    \" The worst case I've observed so far is \" Exit.\", which is 6 characters\n    if l:ex_ind != -1\n        call a:progressCallback(strpart(l:tr,0,l:ex_ind-1))\n        call a:finishedCallback(a:msg.result.timestamp)\n    else\n        call a:progressCallback(l:tr)\n        let req = {\"method\": \"unguided\", \"params\": {\"timestamp\": a:msg.result.timestamp, \"no_context\": v:true}}\n        let resp = ch_sendexpr(g:lsp_job, req, {\"callback\": function(\"s:transcriptionCallback\", [a:progressCallback, a:finishedCallback])})\n    endif\nendfunc\nfunc s:insertText(msg)\n    exe \"normal a\" .. a:msg\nendfunction\nfunc s:endTranscription(timestamp)\n    call appendbufline(s:output_buffer, \"$\", \"Ending unguided transcription\")\nendfunction\n\n\n\n\" If a command does not include a whole actionable step, attempting to execute\n\" it discards the remainder of things. There is likely a simpler solution,\n\" but it can be made functional now by storing a backbuffer until actionable\nlet s:command_backlog = \"\"\nlet s:repeat_command = \"\"\nlet s:preceeding_upper = v:false\nfunc s:commandCallback(params, commandset_index, channel, msg)\n    let l:command_index = a:msg.result.command_index\n    let l:do_execute = v:false\n    let l:next_mode = a:commandset_index\n    let l:command = s:commandset_list[a:commandset_index][l:command_index]\n    call s:logCallback(0, string(a:msg) .. \" \" .. a:commandset_index .. \" \" .. l:command)\n    if l:command_index == 0\n        \"exit\n        \"if s:command_backlog == \"\"\n        call s:logCallback(0,\"Stopping command mode\")\n        echo \"No longer listening\"\n        let s:command_backlog = \"\"\n        return\n        \"else\n        \" Legacy code to clear an existing buffer with exit.\n        \" Was found to be rarely desired and is better introduced as a\n        \" standalone command (clear?)\n        \"   call s:logCallback(0,\"Clearing command_backlog\" .. s:command_backlog)\n        \"   let s:command_backlog = \"\"\n        \"   let s:preceeding_upper = v:false\n        \" endif\n    elseif l:command_index == 1\n        \" upper\n        let s:preceeding_upper = !s:preceeding_upper\n    elseif l:command == \"save\"\n        \" save and run can only happen in commandset 0,\n        exe \"w\"\n    elseif l:command == \"run\"\n        exe \"make run\"\n    elseif l:command == \"space\"\n        exe \"normal i \\<ESC>l\"\n    elseif has_key(s:c_user, l:command)\n        let Userfunc = s:c_user[l:command]\n        if type(Userfunc) == v:t_string\n            let Userfunc = function(Userfunc)\n        endif\n        call Userfunc()\n    else\n        if s:preceeding_upper\n            \" Upper should keep commandset\n            let s:preceeding_upper = v:false\n            let l:visual_command = tr(l:command, s:c_lowerkeys, s:c_upperkeys)\n        else\n            let l:visual_command = l:command\n        endif\n        echo s:command_backlog .. \" - \" .. l:visual_command\n        let s:command_backlog = s:command_backlog .. l:visual_command\n        if a:commandset_index == 2 || a:commandset_index == 3\n            \" single key, either completes motion, replace, or register\n            \" Should move to execute unless part of a register\n            \" Change will be caught at execute\n            if s:command_backlog[-2:-2] !=# '\"'\n                call s:logCallback(0,\"not register\")\n                let l:do_execute = v:true\n            end\n            let l:next_mode = 0\n            \" commandset index only matters for a/i\n        elseif (l:command == \"a\" || l:command == \"i\") && a:commandset_index == 1\n            \" inside/around. Is commandset 3\n            let l:next_mode = 3\n        elseif l:command ==# '\"'\n            let l:next_mode = 2\n        elseif index(s:c_count, l:command) != -1\n            let l:next_mode = a:commandset_index\n        elseif index(s:c_motion, l:command) != -1\n            if l:command == 't' || l:command == 'f' || l:command == \"'\"\n                \" prompt single key\n                let l:next_mode = 2\n            else\n                let l:do_execute = v:true\n                let l:next_mode = 0\n            endif\n        elseif index(s:c_command, l:command) != -1\n            if index([\"y\",\"g\",\"d\",\"c\"], s:command_backlog[-1:-1]) != -1 && s:command_backlog[-1:-1] != s:command_backlog[-2:-2] && mode() !=? 'v'\n                \" need motion or repeated command\n                \" Potential for bad state here if disparaging command keys are\n                \" entered (i.e. yd), but vim can handle checks for this at exe\n                \" And checking for cases like y123d would complicate things\n                let l:next_mode = 1\n            elseif index([\"i\",\"a\",\"c\", \"o\", \"s\"], l:command) != -1 || s:command_backlog[-1:-1] ==# 'R'\n                \"'Insert' mode, do general transcription\n                let l:req = {\"method\": \"unguided\", \"params\": a:params}\n                let l:req.params.timestamp = a:msg.result.timestamp\n                let l:req.params.no_context = v:true\n                let resp = ch_sendexpr(g:lsp_job, req, {\"callback\": function(\"s:transcriptionCallback\", [function(\"s:subTranProg\"), function(\"s:subTranFinish\", [a:params])])})\n                return\n            elseif l:command == 'r' || l:command == 'm'\n                let l:next_mode = 2\n            elseif l:command == '.'\n                let l:next_mode = 0\n                let l:do_execute = v:true\n                let s:command_backlog = s:command_backlog[0:-2] .. s:repeat_command\n            else\n                if l:command ==? 'v'\n                    let l:next_mode = 1\n                else\n                    let l:next_mode = 0\n                endif\n                let l:do_execute = v:true\n            endif\n        else\n            throw \"Invalid command state: \" .. l:command .. \" \" .. a:commandset_index .. \" \" .. s:command_backlog\n        endif\n    endif\n    if l:do_execute\n        if mode() ==?'v' && l:next_mode == 0\n            let l:next_mode = 1\n        elseif match(s:command_backlog, 'c') != -1\n            let l:req = {\"method\": \"unguided\", \"params\": a:params}\n            let l:req.params.timestamp = a:msg.result.timestamp\n            let l:req.params.no_context = v:true\n            let resp = ch_sendexpr(g:lsp_job, req, {\"callback\": function(\"s:transcriptionCallback\", [function(\"s:subTranProg\"), function(\"s:subTranFinish\", [a:params])])})\n            return\n        endif\n        exe \"normal\" s:command_backlog\n        if index(s:c_motion + [\"u\"],l:command) == -1\n            exe \"normal a\\<C-G>u\"\n            let s:repeat_command = s:command_backlog\n            call s:logCallback(0, s:command_backlog)\n        endif\n        let s:command_backlog = \"\"\n    endif\n    let l:req = {\"method\": \"guided\", \"params\": a:params}\n    let l:req.params.timestamp = a:msg.result.timestamp\n    let l:req.params.commandset_index = l:next_mode\n    let resp = ch_sendexpr(g:lsp_job, l:req, {\"callback\": function(\"s:commandCallback\",[a:params, l:next_mode])})\nendfunction\n\nfunc s:loadedCallback(channel, msg)\n    echo \"Loading complete\"\n    call s:logCallback(a:channel, a:msg)\nendfunction\n\nfunc s:registerCommandset(commandlist, is_final)\n    let req = {\"method\": \"registerCommandset\"}\n    let req.params = a:commandlist\n    call s:logCallback(0, join(a:commandlist))\n    call add(g:whisper_commandlist_spoken, a:commandlist)\n    if a:is_final\n        let resp = ch_sendexpr(g:lsp_job, req, {\"callback\": \"s:loadedCallback\"})\n    else\n        let resp = ch_sendexpr(g:lsp_job, req, {\"callback\": \"s:logCallback\"})\n    endif\nendfunction\n\nfunc s:registerAllCommands()\n    let l:normal = s:c_special_always + s:c_special_normal + s:c_count + s:c_command + s:c_motion + keys(s:c_user)\n    let l:visual = s:c_special_always + s:c_count + s:c_command + s:c_motion\n    \" Currently the same as visual.\n    \" let l:post_command = s:c_special_always + s:c_count + s:c_command + s:c_motion\n    let l:single_key = s:c_special_always + split(s:c_lowerkeys, '\\zs')\n    let l:area = s:c_special_always + s:c_area\n\n    \" Used only for compatibility with the testing script\n    let g:whisper_commandlist_spoken = []\n\n    let s:commandset_list = [l:normal, l:visual, l:single_key, l:area]\n    call s:registerCommandset(s:commandsetToSpoken(l:normal, 0), v:false)\n    call s:registerCommandset(s:commandsetToSpoken(l:visual, 1), v:false)\n    call s:registerCommandset(s:commandsetToSpoken(l:single_key, 2), v:false)\n    call s:registerCommandset(s:commandsetToSpoken(l:area, 3), v:true)\nendfunction\n\nfunc s:commandsetToSpoken(commandset, spoken_index)\n    let l:spoken_list = []\n    for l:command in a:commandset\n        if has_key(s:spoken_dict, l:command)\n            let l:spoken_value = s:spoken_dict[l:command]\n            if type(l:spoken_value) == v:t_dict\n                if has_key(l:spoken_value, a:spoken_index)\n                    let l:spoken_value = l:spoken_value[a:spoken_index]\n                else\n                    if a:spoken_index == 2\n                        let l:spoken_value = l:command\n                    else\n                        let l:spoken_value = l:spoken_value[0]\n                    endif\n                endif\n            else\n                if a:spoken_index == 2\n                    let l:spoken_value = l:command\n                endif\n            endif\n        else\n            let l:spoken_value = l:command\n        endif\n        call add(l:spoken_list, l:spoken_value)\n    endfor\n    return l:spoken_list\nendfunction\n\n\" TODO: Check lifetime. If the script is resourced, is the existing\n\" s:lsp_job dropped and therefore killed?\n\" This seems to not be the case and I've had to deal with zombie processes\n\" that survive exiting vim, even though said behavior conflicts with my\n\" understanding of the provided documentation\nlet s:lsp_opts = {\"in_mode\": \"lsp\", \"out_mode\": \"lsp\", \"err_mode\": \"nl\", \"err_io\": \"buffer\", \"err_buf\": s:output_buffer}\nif !exists(\"g:lsp_job\")\n    if exists(\"g:whisper_user_commands\")\n        let s:c_user = g:whisper_user_commands\n    else\n        let s:c_user = {}\n    endif\n    let g:lsp_job = job_start(s:lsp_command, s:lsp_opts)\n    if job_status(g:lsp_job) == \"fail\"\n        echoerr \"Failed to start whisper job\"\n    endif\n    call s:registerAllCommands()\nendif\n"
  },
  {
    "path": "examples/miniaudio.h",
    "content": "/*\nAudio playback and capture library. Choice of public domain or MIT-0. See license statements at the end of this file.\nminiaudio - v0.11.24 - 2026-01-17\n\nDavid Reid - mackron@gmail.com\n\nWebsite:       https://miniaud.io\nDocumentation: https://miniaud.io/docs\nGitHub:        https://github.com/mackron/miniaudio\n*/\n\n/*\n1. Introduction\n===============\nTo use miniaudio, just include \"miniaudio.h\" like any other header and add \"miniaudio.c\" to your\nsource tree. If you don't want to add it to your source tree you can compile and link to it like\nany other library. Note that ABI compatibility is not guaranteed between versions, even with bug\nfix releases, so take care if compiling as a shared object.\n\nminiaudio includes both low level and high level APIs. The low level API is good for those who want\nto do all of their mixing themselves and only require a light weight interface to the underlying\naudio device. The high level API is good for those who have complex mixing and effect requirements.\n\nIn miniaudio, objects are transparent structures. Unlike many other libraries, there are no handles\nto opaque objects which means you need to allocate memory for objects yourself. In the examples\npresented in this documentation you will often see objects declared on the stack. You need to be\ncareful when translating these examples to your own code so that you don't accidentally declare\nyour objects on the stack and then cause them to become invalid once the function returns. In\naddition, you must ensure the memory address of your objects remain the same throughout their\nlifetime. You therefore cannot be making copies of your objects.\n\nA config/init pattern is used throughout the entire library. The idea is that you set up a config\nobject and pass that into the initialization routine. The advantage to this system is that the\nconfig object can be initialized with logical defaults and new properties added to it without\nbreaking the API. The config object can be allocated on the stack and does not need to be\nmaintained after initialization of the corresponding object.\n\n\n1.1. Low Level API\n------------------\nThe low level API gives you access to the raw audio data of an audio device. It supports playback,\ncapture, full-duplex and loopback (WASAPI only). You can enumerate over devices to determine which\nphysical device(s) you want to connect to.\n\nThe low level API uses the concept of a \"device\" as the abstraction for physical devices. The idea\nis that you choose a physical device to emit or capture audio from, and then move data to/from the\ndevice when miniaudio tells you to. Data is delivered to and from devices asynchronously via a\ncallback which you specify when initializing the device.\n\nWhen initializing the device you first need to configure it. The device configuration allows you to\nspecify things like the format of the data delivered via the callback, the size of the internal\nbuffer and the ID of the device you want to emit or capture audio from.\n\nOnce you have the device configuration set up you can initialize the device. When initializing a\ndevice you need to allocate memory for the device object beforehand. This gives the application\ncomplete control over how the memory is allocated. In the example below we initialize a playback\ndevice on the stack, but you could allocate it on the heap if that suits your situation better.\n\n    ```c\n    void data_callback(ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount)\n    {\n        // In playback mode copy data to pOutput. In capture mode read data from pInput. In full-duplex mode, both\n        // pOutput and pInput will be valid and you can move data from pInput into pOutput. Never process more than\n        // frameCount frames.\n    }\n\n    int main()\n    {\n        ma_device_config config = ma_device_config_init(ma_device_type_playback);\n        config.playback.format   = ma_format_f32;   // Set to ma_format_unknown to use the device's native format.\n        config.playback.channels = 2;               // Set to 0 to use the device's native channel count.\n        config.sampleRate        = 48000;           // Set to 0 to use the device's native sample rate.\n        config.dataCallback      = data_callback;   // This function will be called when miniaudio needs more data.\n        config.pUserData         = pMyCustomData;   // Can be accessed from the device object (device.pUserData).\n\n        ma_device device;\n        if (ma_device_init(NULL, &config, &device) != MA_SUCCESS) {\n            return -1;  // Failed to initialize the device.\n        }\n\n        ma_device_start(&device);     // The device is sleeping by default so you'll need to start it manually.\n\n        // Do something here. Probably your program's main loop.\n\n        ma_device_uninit(&device);\n        return 0;\n    }\n    ```\n\nIn the example above, `data_callback()` is where audio data is written and read from the device.\nThe idea is in playback mode you cause sound to be emitted from the speakers by writing audio data\nto the output buffer (`pOutput` in the example). In capture mode you read data from the input\nbuffer (`pInput`) to extract sound captured by the microphone. The `frameCount` parameter tells you\nhow many frames can be written to the output buffer and read from the input buffer. A \"frame\" is\none sample for each channel. For example, in a stereo stream (2 channels), one frame is 2\nsamples: one for the left, one for the right. The channel count is defined by the device config.\nThe size in bytes of an individual sample is defined by the sample format which is also specified\nin the device config. Multi-channel audio data is always interleaved, which means the samples for\neach frame are stored next to each other in memory. For example, in a stereo stream the first pair\nof samples will be the left and right samples for the first frame, the second pair of samples will\nbe the left and right samples for the second frame, etc.\n\nThe configuration of the device is defined by the `ma_device_config` structure. The config object\nis always initialized with `ma_device_config_init()`. It's important to always initialize the\nconfig with this function as it initializes it with logical defaults and ensures your program\ndoesn't break when new members are added to the `ma_device_config` structure. The example above\nuses a fairly simple and standard device configuration. The call to `ma_device_config_init()` takes\na single parameter, which is whether or not the device is a playback, capture, duplex or loopback\ndevice (loopback devices are not supported on all backends). The `config.playback.format` member\nsets the sample format which can be one of the following (all formats are native-endian):\n\n    +---------------+----------------------------------------+---------------------------+\n    | Symbol        | Description                            | Range                     |\n    +---------------+----------------------------------------+---------------------------+\n    | ma_format_f32 | 32-bit floating point                  | [-1, 1]                   |\n    | ma_format_s16 | 16-bit signed integer                  | [-32768, 32767]           |\n    | ma_format_s24 | 24-bit signed integer (tightly packed) | [-8388608, 8388607]       |\n    | ma_format_s32 | 32-bit signed integer                  | [-2147483648, 2147483647] |\n    | ma_format_u8  | 8-bit unsigned integer                 | [0, 255]                  |\n    +---------------+----------------------------------------+---------------------------+\n\nThe `config.playback.channels` member sets the number of channels to use with the device. The\nchannel count cannot exceed MA_MAX_CHANNELS. The `config.sampleRate` member sets the sample rate\n(which must be the same for both playback and capture in full-duplex configurations). This is\nusually set to 44100 or 48000, but can be set to anything. It's recommended to keep this between\n8000 and 384000, however.\n\nNote that leaving the format, channel count and/or sample rate at their default values will result\nin the internal device's native configuration being used which is useful if you want to avoid the\noverhead of miniaudio's automatic data conversion.\n\nIn addition to the sample format, channel count and sample rate, the data callback and user data\npointer are also set via the config. The user data pointer is not passed into the callback as a\nparameter, but is instead set to the `pUserData` member of `ma_device` which you can access\ndirectly since all miniaudio structures are transparent.\n\nInitializing the device is done with `ma_device_init()`. This will return a result code telling you\nwhat went wrong, if anything. On success it will return `MA_SUCCESS`. After initialization is\ncomplete the device will be in a stopped state. To start it, use `ma_device_start()`.\nUninitializing the device will stop it, which is what the example above does, but you can also stop\nthe device with `ma_device_stop()`. To resume the device simply call `ma_device_start()` again.\nNote that it's important to never stop or start the device from inside the callback. This will\nresult in a deadlock. Instead you set a variable or signal an event indicating that the device\nneeds to stop and handle it in a different thread. The following APIs must never be called inside\nthe callback:\n\n    ```c\n    ma_device_init()\n    ma_device_init_ex()\n    ma_device_uninit()\n    ma_device_start()\n    ma_device_stop()\n    ```\n\nYou must never try uninitializing and reinitializing a device inside the callback. You must also\nnever try to stop and start it from inside the callback. There are a few other things you shouldn't\ndo in the callback depending on your requirements, however this isn't so much a thread-safety\nthing, but rather a real-time processing thing which is beyond the scope of this introduction.\n\nThe example above demonstrates the initialization of a playback device, but it works exactly the\nsame for capture. All you need to do is change the device type from `ma_device_type_playback` to\n`ma_device_type_capture` when setting up the config, like so:\n\n    ```c\n    ma_device_config config = ma_device_config_init(ma_device_type_capture);\n    config.capture.format   = MY_FORMAT;\n    config.capture.channels = MY_CHANNEL_COUNT;\n    ```\n\nIn the data callback you just read from the input buffer (`pInput` in the example above) and leave\nthe output buffer alone (it will be set to NULL when the device type is set to\n`ma_device_type_capture`).\n\nThese are the available device types and how you should handle the buffers in the callback:\n\n    +-------------------------+--------------------------------------------------------+\n    | Device Type             | Callback Behavior                                      |\n    +-------------------------+--------------------------------------------------------+\n    | ma_device_type_playback | Write to output buffer, leave input buffer untouched.  |\n    | ma_device_type_capture  | Read from input buffer, leave output buffer untouched. |\n    | ma_device_type_duplex   | Read from input buffer, write to output buffer.        |\n    | ma_device_type_loopback | Read from input buffer, leave output buffer untouched. |\n    +-------------------------+--------------------------------------------------------+\n\nYou will notice in the example above that the sample format and channel count is specified\nseparately for playback and capture. This is to support different data formats between the playback\nand capture devices in a full-duplex system. An example may be that you want to capture audio data\nas a monaural stream (one channel), but output sound to a stereo speaker system. Note that if you\nuse different formats between playback and capture in a full-duplex configuration you will need to\nconvert the data yourself. There are functions available to help you do this which will be\nexplained later.\n\nThe example above did not specify a physical device to connect to which means it will use the\noperating system's default device. If you have multiple physical devices connected and you want to\nuse a specific one you will need to specify the device ID in the configuration, like so:\n\n    ```c\n    config.playback.pDeviceID = pMyPlaybackDeviceID;    // Only if requesting a playback or duplex device.\n    config.capture.pDeviceID = pMyCaptureDeviceID;      // Only if requesting a capture, duplex or loopback device.\n    ```\n\nTo retrieve the device ID you will need to perform device enumeration, however this requires the\nuse of a new concept called the \"context\". Conceptually speaking the context sits above the device.\nThere is one context to many devices. The purpose of the context is to represent the backend at a\nmore global level and to perform operations outside the scope of an individual device. Mainly it is\nused for performing run-time linking against backend libraries, initializing backends and\nenumerating devices. The example below shows how to enumerate devices.\n\n    ```c\n    ma_context context;\n    if (ma_context_init(NULL, 0, NULL, &context) != MA_SUCCESS) {\n        // Error.\n    }\n\n    ma_device_info* pPlaybackInfos;\n    ma_uint32 playbackCount;\n    ma_device_info* pCaptureInfos;\n    ma_uint32 captureCount;\n    if (ma_context_get_devices(&context, &pPlaybackInfos, &playbackCount, &pCaptureInfos, &captureCount) != MA_SUCCESS) {\n        // Error.\n    }\n\n    // Loop over each device info and do something with it. Here we just print the name with their index. You may want\n    // to give the user the opportunity to choose which device they'd prefer.\n    for (ma_uint32 iDevice = 0; iDevice < playbackCount; iDevice += 1) {\n        printf(\"%d - %s\\n\", iDevice, pPlaybackInfos[iDevice].name);\n    }\n\n    ma_device_config config = ma_device_config_init(ma_device_type_playback);\n    config.playback.pDeviceID = &pPlaybackInfos[chosenPlaybackDeviceIndex].id;\n    config.playback.format    = MY_FORMAT;\n    config.playback.channels  = MY_CHANNEL_COUNT;\n    config.sampleRate         = MY_SAMPLE_RATE;\n    config.dataCallback       = data_callback;\n    config.pUserData          = pMyCustomData;\n\n    ma_device device;\n    if (ma_device_init(&context, &config, &device) != MA_SUCCESS) {\n        // Error\n    }\n\n    ...\n\n    ma_device_uninit(&device);\n    ma_context_uninit(&context);\n    ```\n\nThe first thing we do in this example is initialize a `ma_context` object with `ma_context_init()`.\nThe first parameter is a pointer to a list of `ma_backend` values which are used to override the\ndefault backend priorities. When this is NULL, as in this example, miniaudio's default priorities\nare used. The second parameter is the number of backends listed in the array pointed to by the\nfirst parameter. The third parameter is a pointer to a `ma_context_config` object which can be\nNULL, in which case defaults are used. The context configuration is used for setting the logging\ncallback, custom memory allocation callbacks, user-defined data and some backend-specific\nconfigurations.\n\nOnce the context has been initialized you can enumerate devices. In the example above we use the\nsimpler `ma_context_get_devices()`, however you can also use a callback for handling devices by\nusing `ma_context_enumerate_devices()`. When using `ma_context_get_devices()` you provide a pointer\nto a pointer that will, upon output, be set to a pointer to a buffer containing a list of\n`ma_device_info` structures. You also provide a pointer to an unsigned integer that will receive\nthe number of items in the returned buffer. Do not free the returned buffers as their memory is\nmanaged internally by miniaudio.\n\nThe `ma_device_info` structure contains an `id` member which is the ID you pass to the device\nconfig. It also contains the name of the device which is useful for presenting a list of devices\nto the user via the UI.\n\nWhen creating your own context you will want to pass it to `ma_device_init()` when initializing the\ndevice. Passing in NULL, like we do in the first example, will result in miniaudio creating the\ncontext for you, which you don't want to do since you've already created a context. Note that\ninternally the context is only tracked by it's pointer which means you must not change the location\nof the `ma_context` object. If this is an issue, consider using `malloc()` to allocate memory for\nthe context.\n\n\n1.2. High Level API\n-------------------\nThe high level API consists of three main parts:\n\n  * Resource management for loading and streaming sounds.\n  * A node graph for advanced mixing and effect processing.\n  * A high level \"engine\" that wraps around the resource manager and node graph.\n\nThe resource manager (`ma_resource_manager`) is used for loading sounds. It supports loading sounds\nfully into memory and also streaming. It will also deal with reference counting for you which\navoids the same sound being loaded multiple times.\n\nThe node graph is used for mixing and effect processing. The idea is that you connect a number of\nnodes into the graph by connecting each node's outputs to another node's inputs. Each node can\nimplement its own effect. By chaining nodes together, advanced mixing and effect processing can\nbe achieved.\n\nThe engine encapsulates both the resource manager and the node graph to create a simple, easy to\nuse high level API. The resource manager and node graph APIs are covered in more later sections of\nthis manual.\n\nThe code below shows how you can initialize an engine using its default configuration.\n\n    ```c\n    ma_result result;\n    ma_engine engine;\n\n    result = ma_engine_init(NULL, &engine);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to initialize the engine.\n    }\n    ```\n\nThis creates an engine instance which will initialize a device internally which you can access with\n`ma_engine_get_device()`. It will also initialize a resource manager for you which can be accessed\nwith `ma_engine_get_resource_manager()`. The engine itself is a node graph (`ma_node_graph`) which\nmeans you can pass a pointer to the engine object into any of the `ma_node_graph` APIs (with a\ncast). Alternatively, you can use `ma_engine_get_node_graph()` instead of a cast.\n\nNote that all objects in miniaudio, including the `ma_engine` object in the example above, are\ntransparent structures. There are no handles to opaque structures in miniaudio which means you need\nto be mindful of how you declare them. In the example above we are declaring it on the stack, but\nthis will result in the struct being invalidated once the function encapsulating it returns. If\nallocating the engine on the heap is more appropriate, you can easily do so with a standard call\nto `malloc()` or whatever heap allocation routine you like:\n\n    ```c\n    ma_engine* pEngine = malloc(sizeof(*pEngine));\n    ```\n\nThe `ma_engine` API uses the same config/init pattern used all throughout miniaudio. To configure\nan engine, you can fill out a `ma_engine_config` object and pass it into the first parameter of\n`ma_engine_init()`:\n\n    ```c\n    ma_result result;\n    ma_engine engine;\n    ma_engine_config engineConfig;\n\n    engineConfig = ma_engine_config_init();\n    engineConfig.pResourceManager = &myCustomResourceManager;   // <-- Initialized as some earlier stage.\n\n    result = ma_engine_init(&engineConfig, &engine);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n    ```\n\nThis creates an engine instance using a custom config. In this particular example it's showing how\nyou can specify a custom resource manager rather than having the engine initialize one internally.\nThis is particularly useful if you want to have multiple engine's share the same resource manager.\n\nThe engine must be uninitialized with `ma_engine_uninit()` when it's no longer needed.\n\nBy default the engine will be started, but nothing will be playing because no sounds have been\ninitialized. The easiest but least flexible way of playing a sound is like so:\n\n    ```c\n    ma_engine_play_sound(&engine, \"my_sound.wav\", NULL);\n    ```\n\nThis plays what miniaudio calls an \"inline\" sound. It plays the sound once, and then puts the\ninternal sound up for recycling. The last parameter is used to specify which sound group the sound\nshould be associated with which will be explained later. This particular way of playing a sound is\nsimple, but lacks flexibility and features. A more flexible way of playing a sound is to first\ninitialize a sound:\n\n    ```c\n    ma_result result;\n    ma_sound sound;\n\n    result = ma_sound_init_from_file(&engine, \"my_sound.wav\", 0, NULL, NULL, &sound);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    ma_sound_start(&sound);\n    ```\n\nThis returns a `ma_sound` object which represents a single instance of the specified sound file. If\nyou want to play the same file multiple times simultaneously, you need to create one sound for each\ninstance.\n\nSounds should be uninitialized with `ma_sound_uninit()`.\n\nSounds are not started by default. Start a sound with `ma_sound_start()` and stop it with\n`ma_sound_stop()`. When a sound is stopped, it is not rewound to the start. Use\n`ma_sound_seek_to_pcm_frame(&sound, 0)` to seek back to the start of a sound. By default, starting\nand stopping sounds happens immediately, but sometimes it might be convenient to schedule the sound\nto be started and/or stopped at a specific time. This can be done with the following functions:\n\n    ```c\n    ma_sound_set_start_time_in_pcm_frames()\n    ma_sound_set_start_time_in_milliseconds()\n    ma_sound_set_stop_time_in_pcm_frames()\n    ma_sound_set_stop_time_in_milliseconds()\n    ```\n\nThe start/stop time needs to be specified based on the absolute timer which is controlled by the\nengine. The current global time in PCM frames can be retrieved with\n`ma_engine_get_time_in_pcm_frames()`. The engine's global time can be changed with\n`ma_engine_set_time_in_pcm_frames()` for synchronization purposes if required. Note that scheduling\na start time still requires an explicit call to `ma_sound_start()` before anything will play:\n\n    ```c\n    ma_sound_set_start_time_in_pcm_frames(&sound, ma_engine_get_time_in_pcm_frames(&engine) + (ma_engine_get_sample_rate(&engine) * 2);\n    ma_sound_start(&sound);\n    ```\n\nThe third parameter of `ma_sound_init_from_file()` is a set of flags that control how the sound be\nloaded and a few options on which features should be enabled for that sound. By default, the sound\nis synchronously loaded fully into memory straight from the file system without any kind of\ndecoding. If you want to decode the sound before storing it in memory, you need to specify the\n`MA_SOUND_FLAG_DECODE` flag. This is useful if you want to incur the cost of decoding at an earlier\nstage, such as a loading stage. Without this option, decoding will happen dynamically at mixing\ntime which might be too expensive on the audio thread.\n\nIf you want to load the sound asynchronously, you can specify the `MA_SOUND_FLAG_ASYNC` flag. This\nwill result in `ma_sound_init_from_file()` returning quickly, but the sound will not start playing\nuntil the sound has had some audio decoded.\n\nThe fourth parameter is a pointer to sound group. A sound group is used as a mechanism to organise\nsounds into groups which have their own effect processing and volume control. An example is a game\nwhich might have separate groups for sfx, voice and music. Each of these groups have their own\nindependent volume control. Use `ma_sound_group_init()` or `ma_sound_group_init_ex()` to initialize\na sound group.\n\nSounds and sound groups are nodes in the engine's node graph and can be plugged into any `ma_node`\nAPI. This makes it possible to connect sounds and sound groups to effect nodes to produce complex\neffect chains.\n\nA sound can have its volume changed with `ma_sound_set_volume()`. If you prefer decibel volume\ncontrol you can use `ma_volume_db_to_linear()` to convert from decibel representation to linear.\n\nPanning and pitching is supported with `ma_sound_set_pan()` and `ma_sound_set_pitch()`. If you know\na sound will never have its pitch changed with `ma_sound_set_pitch()` or via the doppler effect,\nyou can specify the `MA_SOUND_FLAG_NO_PITCH` flag when initializing the sound for an optimization.\n\nBy default, sounds and sound groups have spatialization enabled. If you don't ever want to\nspatialize your sounds, initialize the sound with the `MA_SOUND_FLAG_NO_SPATIALIZATION` flag. The\nspatialization model is fairly simple and is roughly on feature parity with OpenAL. HRTF and\nenvironmental occlusion are not currently supported, but planned for the future. The supported\nfeatures include:\n\n  * Sound and listener positioning and orientation with cones\n  * Attenuation models: none, inverse, linear and exponential\n  * Doppler effect\n\nSounds can be faded in and out with `ma_sound_set_fade_in_pcm_frames()`.\n\nTo check if a sound is currently playing, you can use `ma_sound_is_playing()`. To check if a sound\nis at the end, use `ma_sound_at_end()`. Looping of a sound can be controlled with\n`ma_sound_set_looping()`. Use `ma_sound_is_looping()` to check whether or not the sound is looping.\n\n\n\n2. Building\n===========\nminiaudio should work cleanly out of the box without the need to download or install any\ndependencies. See below for platform-specific details.\n\nThis library has been designed to be added directly to your source tree which is the preferred way\nof using it, but you can compile it as a normal library if that's your preference. Be careful if\ncompiling as a shared object because miniaudio is not ABI compatible between any release, including\nbug fix releases. It's recommended you link statically.\n\nNote that GCC and Clang require `-msse2`, `-mavx2`, etc. for SIMD optimizations.\n\nIf you get errors about undefined references to `__sync_val_compare_and_swap_8`, `__atomic_load_8`,\netc. you need to link with `-latomic`.\n\n\n2.1. Windows\n------------\nThe Windows build should compile cleanly on all popular compilers without the need to configure any\ninclude paths nor link to any libraries.\n\nThe UWP build may require linking to mmdevapi.lib if you get errors about an unresolved external\nsymbol for `ActivateAudioInterfaceAsync()`.\n\n\n2.2. macOS and iOS\n------------------\nThe macOS build should compile cleanly without the need to download any dependencies nor link to\nany libraries or frameworks. The iOS build needs to be compiled as Objective-C and will need to\nlink the relevant frameworks but should compile cleanly out of the box with Xcode. Compiling\nthrough the command line requires linking to `-lpthread` and `-lm`.\n\nDue to the way miniaudio links to frameworks at runtime, your application may not pass Apple's\nnotarization process. To fix this there are two options. The first is to compile with\n`-DMA_NO_RUNTIME_LINKING` which in turn will require linking with\n`-framework CoreFoundation -framework CoreAudio -framework AudioToolbox`. If you get errors about\nAudioToolbox, try with `-framework AudioUnit` instead. You may get this when using older versions\nof iOS. Alternatively, if you would rather keep using runtime linking you can add the following to\nyour entitlements.xcent file:\n\n    ```\n    <key>com.apple.security.cs.allow-dyld-environment-variables</key>\n    <true/>\n    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>\n    <true/>\n    ```\n\nSee this discussion for more info: https://github.com/mackron/miniaudio/issues/203.\n\n\n2.3. Linux\n----------\nThe Linux build only requires linking to `-ldl`, `-lpthread` and `-lm`. You do not need any\ndevelopment packages. You may need to link with `-latomic` if you're compiling for 32-bit ARM.\n\n\n2.4. BSD\n--------\nThe BSD build only requires linking to `-lpthread` and `-lm`. NetBSD uses audio(4), OpenBSD uses\nsndio and FreeBSD uses OSS. You may need to link with `-latomic` if you're compiling for 32-bit\nARM.\n\n\n2.5. Android\n------------\nAAudio is the highest priority backend on Android. This should work out of the box without needing\nany kind of compiler configuration. Support for AAudio starts with Android 8 which means older\nversions will fall back to OpenSL|ES which requires API level 16+.\n\nThere have been reports that the OpenSL|ES backend fails to initialize on some Android based\ndevices due to `dlopen()` failing to open \"libOpenSLES.so\". If this happens on your platform\nyou'll need to disable run-time linking with `MA_NO_RUNTIME_LINKING` and link with -lOpenSLES.\n\n\n2.6. Emscripten\n---------------\nThe Emscripten build emits Web Audio JavaScript directly and should compile cleanly out of the box.\nYou cannot use `-std=c*` compiler flags, nor `-ansi`.\n\nYou can enable the use of AudioWorklets by defining `MA_ENABLE_AUDIO_WORKLETS` and then compiling\nwith the following options:\n\n    -sAUDIO_WORKLET=1 -sWASM_WORKERS=1 -sASYNCIFY\n\nAn example for compiling with AudioWorklet support might look like this:\n\n    emcc program.c -o bin/program.html -DMA_ENABLE_AUDIO_WORKLETS -sAUDIO_WORKLET=1 -sWASM_WORKERS=1 -sASYNCIFY\n\nTo run locally, you'll need to use emrun:\n\n    emrun bin/program.html\n\n\n\n2.7. Build Options\n------------------\n`#define` these options before including miniaudio.c, or pass them as compiler flags:\n\n    +----------------------------------+--------------------------------------------------------------------+\n    | Option                           | Description                                                        |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_WASAPI                     | Disables the WASAPI backend.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_DSOUND                     | Disables the DirectSound backend.                                  |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_WINMM                      | Disables the WinMM backend.                                        |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_ALSA                       | Disables the ALSA backend.                                         |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_PULSEAUDIO                 | Disables the PulseAudio backend.                                   |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_JACK                       | Disables the JACK backend.                                         |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_COREAUDIO                  | Disables the Core Audio backend.                                   |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_SNDIO                      | Disables the sndio backend.                                        |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_AUDIO4                     | Disables the audio(4) backend.                                     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_OSS                        | Disables the OSS backend.                                          |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_AAUDIO                     | Disables the AAudio backend.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_OPENSL                     | Disables the OpenSL|ES backend.                                    |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_WEBAUDIO                   | Disables the Web Audio backend.                                    |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_CUSTOM                     | Disables support for custom backends.                              |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_NULL                       | Disables the null backend.                                         |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_ONLY_SPECIFIC_BACKENDS | Disables all backends by default and requires `MA_ENABLE_*` to     |\n    |                                  | enable specific backends.                                          |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_WASAPI                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the WASAPI backend.                                         |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_DSOUND                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the DirectSound backend.                                    |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_WINMM                  | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the WinMM backend.                                          |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_ALSA                   | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the ALSA backend.                                           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_PULSEAUDIO             | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the PulseAudio backend.                                     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_JACK                   | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the JACK backend.                                           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_COREAUDIO              | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the Core Audio backend.                                     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_SNDIO                  | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the sndio backend.                                          |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_AUDIO4                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the audio(4) backend.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_OSS                    | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the OSS backend.                                            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_AAUDIO                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the AAudio backend.                                         |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_OPENSL                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the OpenSL|ES backend.                                      |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_WEBAUDIO               | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the Web Audio backend.                                      |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_CUSTOM                 | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable custom backends.                                            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ENABLE_NULL                   | Used in conjunction with MA_ENABLE_ONLY_SPECIFIC_BACKENDS to       |\n    |                                  | enable the null backend.                                           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_DECODING                   | Disables decoding APIs.                                            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_ENCODING                   | Disables encoding APIs.                                            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_WAV                        | Disables the built-in WAV decoder and encoder.                     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_FLAC                       | Disables the built-in FLAC decoder.                                |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_MP3                        | Disables the built-in MP3 decoder.                                 |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_DEVICE_IO                  | Disables playback and recording. This will disable `ma_context`    |\n    |                                  | and `ma_device` APIs. This is useful if you only want to use       |\n    |                                  | miniaudio's data conversion and/or decoding APIs.                  |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_RESOURCE_MANAGER           | Disables the resource manager. When using the engine this will     |\n    |                                  | also disable the following functions:                              |\n    |                                  |                                                                    |\n    |                                  | ```                                                                |\n    |                                  | ma_sound_init_from_file()                                          |\n    |                                  | ma_sound_init_from_file_w()                                        |\n    |                                  | ma_sound_init_copy()                                               |\n    |                                  | ma_engine_play_sound_ex()                                          |\n    |                                  | ma_engine_play_sound()                                             |\n    |                                  | ```                                                                |\n    |                                  |                                                                    |\n    |                                  | The only way to initialize a `ma_sound` object is to initialize it |\n    |                                  | from a data source.                                                |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_NODE_GRAPH                 | Disables the node graph API. This will also disable the engine API |\n    |                                  | because it depends on the node graph.                              |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_ENGINE                     | Disables the engine API.                                           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_THREADING                  | Disables the `ma_thread`, `ma_mutex`, `ma_semaphore` and           |\n    |                                  | `ma_event` APIs. This option is useful if you only need to use     |\n    |                                  | miniaudio for data conversion, decoding and/or encoding. Some      |\n    |                                  | families of APIs require threading which means the following       |\n    |                                  | options must also be set:                                          |\n    |                                  |                                                                    |\n    |                                  |     ```                                                            |\n    |                                  |     MA_NO_DEVICE_IO                                                |\n    |                                  |     ```                                                            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_GENERATION                 | Disables generation APIs such a `ma_waveform` and `ma_noise`.      |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_SSE2                       | Disables SSE2 optimizations.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_AVX2                       | Disables AVX2 optimizations.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_NEON                       | Disables NEON optimizations.                                       |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_NO_RUNTIME_LINKING            | Disables runtime linking. This is useful for passing Apple's       |\n    |                                  | notarization process. When enabling this, you may need to avoid    |\n    |                                  | using `-std=c89` or `-std=c99` on Linux builds or else you may end |\n    |                                  | up with compilation errors due to conflicts with `timespec` and    |\n    |                                  | `timeval` data types.                                              |\n    |                                  |                                                                    |\n    |                                  | You may need to enable this if your target platform does not allow |\n    |                                  | runtime linking via `dlopen()`.                                    |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_USE_STDINT                    | (Pass this in as a compiler flag. Do not `#define` this before     |\n    |                                  | miniaudio.c) Forces the use of stdint.h for sized types.           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_DEBUG_OUTPUT                  | Enable `printf()` output of debug logs (`MA_LOG_LEVEL_DEBUG`).     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_COINIT_VALUE                  | Windows only. The value to pass to internal calls to               |\n    |                                  | `CoInitializeEx()`. Defaults to `COINIT_MULTITHREADED`.            |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_FORCE_UWP                     | Windows only. Affects only the WASAPI backend. Will force the      |\n    |                                  | WASAPI backend to use the UWP code path instead of the regular     |\n    |                                  | desktop path. This is normally auto-detected and should rarely be  |\n    |                                  | needed to be used explicitly, but can be useful for debugging.     |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ON_THREAD_ENTRY               | Defines some code that will be executed as soon as an internal     |\n    |                                  | miniaudio-managed thread is created. This will be the first thing  |\n    |                                  | to be executed by the thread entry point.                          |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_ON_THREAD_EXIT                | Defines some code that will be executed from the entry point of an |\n    |                                  | internal miniaudio-managed thread upon exit. This will be the last |\n    |                                  | thing to be executed before the thread's entry point exits.        |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_THREAD_DEFAULT_STACK_SIZE     | If set, specifies the default stack size used by miniaudio-managed |\n    |                                  | threads.                                                           |\n    +----------------------------------+--------------------------------------------------------------------+\n    | MA_API                           | Controls how public APIs should be decorated. Default is `extern`. |\n    +----------------------------------+--------------------------------------------------------------------+\n\n\n3. Definitions\n==============\nThis section defines common terms used throughout miniaudio. Unfortunately there is often ambiguity\nin the use of terms throughout the audio space, so this section is intended to clarify how miniaudio\nuses each term.\n\n3.1. Sample\n-----------\nA sample is a single unit of audio data. If the sample format is f32, then one sample is one 32-bit\nfloating point number.\n\n3.2. Frame / PCM Frame\n----------------------\nA frame is a group of samples equal to the number of channels. For a stereo stream a frame is 2\nsamples, a mono frame is 1 sample, a 5.1 surround sound frame is 6 samples, etc. The terms \"frame\"\nand \"PCM frame\" are the same thing in miniaudio. Note that this is different to a compressed frame.\nIf ever miniaudio needs to refer to a compressed frame, such as a FLAC frame, it will always\nclarify what it's referring to with something like \"FLAC frame\".\n\n3.3. Channel\n------------\nA stream of monaural audio that is emitted from an individual speaker in a speaker system, or\nreceived from an individual microphone in a microphone system. A stereo stream has two channels (a\nleft channel, and a right channel), a 5.1 surround sound system has 6 channels, etc. Some audio\nsystems refer to a channel as a complex audio stream that's mixed with other channels to produce\nthe final mix - this is completely different to miniaudio's use of the term \"channel\" and should\nnot be confused.\n\n3.4. Sample Rate\n----------------\nThe sample rate in miniaudio is always expressed in Hz, such as 44100, 48000, etc. It's the number\nof PCM frames that are processed per second.\n\n3.5. Formats\n------------\nThroughout miniaudio you will see references to different sample formats:\n\n    +---------------+----------------------------------------+---------------------------+\n    | Symbol        | Description                            | Range                     |\n    +---------------+----------------------------------------+---------------------------+\n    | ma_format_f32 | 32-bit floating point                  | [-1, 1]                   |\n    | ma_format_s16 | 16-bit signed integer                  | [-32768, 32767]           |\n    | ma_format_s24 | 24-bit signed integer (tightly packed) | [-8388608, 8388607]       |\n    | ma_format_s32 | 32-bit signed integer                  | [-2147483648, 2147483647] |\n    | ma_format_u8  | 8-bit unsigned integer                 | [0, 255]                  |\n    +---------------+----------------------------------------+---------------------------+\n\nAll formats are native-endian.\n\n\n\n4. Data Sources\n===============\nThe data source abstraction in miniaudio is used for retrieving audio data from some source. A few\nexamples include `ma_decoder`, `ma_noise` and `ma_waveform`. You will need to be familiar with data\nsources in order to make sense of some of the higher level concepts in miniaudio.\n\nThe `ma_data_source` API is a generic interface for reading from a data source. Any object that\nimplements the data source interface can be plugged into any `ma_data_source` function.\n\nTo read data from a data source:\n\n    ```c\n    ma_result result;\n    ma_uint64 framesRead;\n\n    result = ma_data_source_read_pcm_frames(pDataSource, pFramesOut, frameCount, &framesRead);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to read data from the data source.\n    }\n    ```\n\nIf you don't need the number of frames that were successfully read you can pass in `NULL` to the\n`pFramesRead` parameter. If this returns a value less than the number of frames requested it means\nthe end of the file has been reached. `MA_AT_END` will be returned only when the number of frames\nread is 0.\n\nWhen calling any data source function, with the exception of `ma_data_source_init()` and\n`ma_data_source_uninit()`, you can pass in any object that implements a data source. For example,\nyou could plug in a decoder like so:\n\n    ```c\n    ma_result result;\n    ma_uint64 framesRead;\n    ma_decoder decoder;   // <-- This would be initialized with `ma_decoder_init_*()`.\n\n    result = ma_data_source_read_pcm_frames(&decoder, pFramesOut, frameCount, &framesRead);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to read data from the decoder.\n    }\n    ```\n\nIf you want to seek forward you can pass in `NULL` to the `pFramesOut` parameter. Alternatively you\ncan use `ma_data_source_seek_pcm_frames()`.\n\nTo seek to a specific PCM frame:\n\n    ```c\n    result = ma_data_source_seek_to_pcm_frame(pDataSource, frameIndex);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to seek to PCM frame.\n    }\n    ```\n\nYou can retrieve the total length of a data source in PCM frames, but note that some data sources\nmay not have the notion of a length, such as noise and waveforms, and others may just not have a\nway of determining the length such as some decoders. To retrieve the length:\n\n    ```c\n    ma_uint64 length;\n\n    result = ma_data_source_get_length_in_pcm_frames(pDataSource, &length);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to retrieve the length.\n    }\n    ```\n\nCare should be taken when retrieving the length of a data source where the underlying decoder is\npulling data from a data stream with an undefined length, such as internet radio or some kind of\nbroadcast. If you do this, `ma_data_source_get_length_in_pcm_frames()` may never return.\n\nThe current position of the cursor in PCM frames can also be retrieved:\n\n    ```c\n    ma_uint64 cursor;\n\n    result = ma_data_source_get_cursor_in_pcm_frames(pDataSource, &cursor);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to retrieve the cursor.\n    }\n    ```\n\nYou will often need to know the data format that will be returned after reading. This can be\nretrieved like so:\n\n    ```c\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_channel channelMap[MA_MAX_CHANNELS];\n\n    result = ma_data_source_get_data_format(pDataSource, &format, &channels, &sampleRate, channelMap, MA_MAX_CHANNELS);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to retrieve data format.\n    }\n    ```\n\nIf you do not need a specific data format property, just pass in NULL to the respective parameter.\n\nThere may be cases where you want to implement something like a sound bank where you only want to\nread data within a certain range of the underlying data. To do this you can use a range:\n\n    ```c\n    result = ma_data_source_set_range_in_pcm_frames(pDataSource, rangeBegInFrames, rangeEndInFrames);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to set the range.\n    }\n    ```\n\nThis is useful if you have a sound bank where many sounds are stored in the same file and you want\nthe data source to only play one of those sub-sounds. Note that once the range is set, everything\nthat takes a position, such as cursors and loop points, should always be relative to the start of\nthe range. When the range is set, any previously defined loop point will be reset.\n\nCustom loop points can also be used with data sources. By default, data sources will loop after\nthey reach the end of the data source, but if you need to loop at a specific location, you can do\nthe following:\n\n    ```c\n    result = ma_data_source_set_loop_point_in_pcm_frames(pDataSource, loopBegInFrames, loopEndInFrames);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to set the loop point.\n    }\n    ```\n\nThe loop point is relative to the current range.\n\nIt's sometimes useful to chain data sources together so that a seamless transition can be achieved.\nTo do this, you can use chaining:\n\n    ```c\n    ma_decoder decoder1;\n    ma_decoder decoder2;\n\n    // ... initialize decoders with ma_decoder_init_*() ...\n\n    result = ma_data_source_set_next(&decoder1, &decoder2);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to set the next data source.\n    }\n\n    result = ma_data_source_read_pcm_frames(&decoder1, pFramesOut, frameCount, pFramesRead);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to read from the decoder.\n    }\n    ```\n\nIn the example above we're using decoders. When reading from a chain, you always want to read from\nthe top level data source in the chain. In the example above, `decoder1` is the top level data\nsource in the chain. When `decoder1` reaches the end, `decoder2` will start seamlessly without any\ngaps.\n\nNote that when looping is enabled, only the current data source will be looped. You can loop the\nentire chain by linking in a loop like so:\n\n    ```c\n    ma_data_source_set_next(&decoder1, &decoder2);  // decoder1 -> decoder2\n    ma_data_source_set_next(&decoder2, &decoder1);  // decoder2 -> decoder1 (loop back to the start).\n    ```\n\nNote that setting up chaining is not thread safe, so care needs to be taken if you're dynamically\nchanging links while the audio thread is in the middle of reading.\n\nDo not use `ma_decoder_seek_to_pcm_frame()` as a means to reuse a data source to play multiple\ninstances of the same sound simultaneously. This can be extremely inefficient depending on the type\nof data source and can result in glitching due to subtle changes to the state of internal filters.\nInstead, initialize multiple data sources for each instance.\n\n\n4.1. Custom Data Sources\n------------------------\nYou can implement a custom data source by implementing the functions in `ma_data_source_vtable`.\nYour custom object must have `ma_data_source_base` as it's first member:\n\n    ```c\n    struct my_data_source\n    {\n        ma_data_source_base base;\n        ...\n    };\n    ```\n\nIn your initialization routine, you need to call `ma_data_source_init()` in order to set up the\nbase object (`ma_data_source_base`):\n\n    ```c\n    static ma_result my_data_source_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n    {\n        // Read data here. Output in the same format returned by my_data_source_get_data_format().\n    }\n\n    static ma_result my_data_source_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n    {\n        // Seek to a specific PCM frame here. Return MA_NOT_IMPLEMENTED if seeking is not supported.\n    }\n\n    static ma_result my_data_source_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n    {\n        // Return the format of the data here.\n    }\n\n    static ma_result my_data_source_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n    {\n        // Retrieve the current position of the cursor here. Return MA_NOT_IMPLEMENTED and set *pCursor to 0 if there is no notion of a cursor.\n    }\n\n    static ma_result my_data_source_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n    {\n        // Retrieve the length in PCM frames here. Return MA_NOT_IMPLEMENTED and set *pLength to 0 if there is no notion of a length or if the length is unknown.\n    }\n\n    static ma_data_source_vtable g_my_data_source_vtable =\n    {\n        my_data_source_read,\n        my_data_source_seek,\n        my_data_source_get_data_format,\n        my_data_source_get_cursor,\n        my_data_source_get_length\n    };\n\n    ma_result my_data_source_init(my_data_source* pMyDataSource)\n    {\n        ma_result result;\n        ma_data_source_config baseConfig;\n\n        baseConfig = ma_data_source_config_init();\n        baseConfig.vtable = &g_my_data_source_vtable;\n\n        result = ma_data_source_init(&baseConfig, &pMyDataSource->base);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        // ... do the initialization of your custom data source here ...\n\n        return MA_SUCCESS;\n    }\n\n    void my_data_source_uninit(my_data_source* pMyDataSource)\n    {\n        // ... do the uninitialization of your custom data source here ...\n\n        // You must uninitialize the base data source.\n        ma_data_source_uninit(&pMyDataSource->base);\n    }\n    ```\n\nNote that `ma_data_source_init()` and `ma_data_source_uninit()` are never called directly outside\nof the custom data source. It's up to the custom data source itself to call these within their own\ninit/uninit functions.\n\n\n\n5. Engine\n=========\nThe `ma_engine` API is a high level API for managing and mixing sounds and effect processing. The\n`ma_engine` object encapsulates a resource manager and a node graph, both of which will be\nexplained in more detail later.\n\nSounds are called `ma_sound` and are created from an engine. Sounds can be associated with a mixing\ngroup called `ma_sound_group` which are also created from the engine. Both `ma_sound` and\n`ma_sound_group` objects are nodes within the engine's node graph.\n\nWhen the engine is initialized, it will normally create a device internally. If you would rather\nmanage the device yourself, you can do so and just pass a pointer to it via the engine config when\nyou initialize the engine. You can also just use the engine without a device, which again can be\nconfigured via the engine config.\n\nThe most basic way to initialize the engine is with a default config, like so:\n\n    ```c\n    ma_result result;\n    ma_engine engine;\n\n    result = ma_engine_init(NULL, &engine);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to initialize the engine.\n    }\n    ```\n\nThis will result in the engine initializing a playback device using the operating system's default\ndevice. This will be sufficient for many use cases, but if you need more flexibility you'll want to\nconfigure the engine with an engine config:\n\n    ```c\n    ma_result result;\n    ma_engine engine;\n    ma_engine_config engineConfig;\n\n    engineConfig = ma_engine_config_init();\n    engineConfig.pDevice = &myDevice;\n\n    result = ma_engine_init(&engineConfig, &engine);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to initialize the engine.\n    }\n    ```\n\nIn the example above we're passing in a pre-initialized device. Since the caller is the one in\ncontrol of the device's data callback, it's their responsibility to manually call\n`ma_engine_read_pcm_frames()` from inside their data callback:\n\n    ```c\n    void playback_data_callback(ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount)\n    {\n        ma_engine_read_pcm_frames(&g_Engine, pOutput, frameCount, NULL);\n    }\n    ```\n\nYou can also use the engine independent of a device entirely:\n\n    ```c\n    ma_result result;\n    ma_engine engine;\n    ma_engine_config engineConfig;\n\n    engineConfig = ma_engine_config_init();\n    engineConfig.noDevice   = MA_TRUE;\n    engineConfig.channels   = 2;        // Must be set when not using a device.\n    engineConfig.sampleRate = 48000;    // Must be set when not using a device.\n\n    result = ma_engine_init(&engineConfig, &engine);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to initialize the engine.\n    }\n    ```\n\nNote that when you're not using a device, you must set the channel count and sample rate in the\nconfig or else miniaudio won't know what to use (miniaudio will use the device to determine this\nnormally). When not using a device, you need to use `ma_engine_read_pcm_frames()` to process audio\ndata from the engine. This kind of setup is useful if you want to do something like offline\nprocessing or want to use a different audio system for playback such as SDL.\n\nWhen a sound is loaded it goes through a resource manager. By default the engine will initialize a\nresource manager internally, but you can also specify a pre-initialized resource manager:\n\n    ```c\n    ma_result result;\n    ma_engine engine1;\n    ma_engine engine2;\n    ma_engine_config engineConfig;\n\n    engineConfig = ma_engine_config_init();\n    engineConfig.pResourceManager = &myResourceManager;\n\n    ma_engine_init(&engineConfig, &engine1);\n    ma_engine_init(&engineConfig, &engine2);\n    ```\n\nIn this example we are initializing two engines, both of which are sharing the same resource\nmanager. This is especially useful for saving memory when loading the same file across multiple\nengines. If you were not to use a shared resource manager, each engine instance would use their own\nwhich would result in any sounds that are used between both engine's being loaded twice. By using\na shared resource manager, it would only be loaded once. Using multiple engine's is useful when you\nneed to output to multiple playback devices, such as in a local multiplayer game where each player\nis using their own set of headphones.\n\nBy default an engine will be in a started state. To make it so the engine is not automatically\nstarted you can configure it as such:\n\n    ```c\n    engineConfig.noAutoStart = MA_TRUE;\n\n    // The engine will need to be started manually.\n    ma_engine_start(&engine);\n\n    // Later on the engine can be stopped with ma_engine_stop().\n    ma_engine_stop(&engine);\n    ```\n\nThe concept of starting or stopping an engine is only relevant when using the engine with a\ndevice. Attempting to start or stop an engine that is not associated with a device will result in\n`MA_INVALID_OPERATION`.\n\nThe master volume of the engine can be controlled with `ma_engine_set_volume()` which takes a\nlinear scale, with 0 resulting in silence and anything above 1 resulting in amplification. If you\nprefer decibel based volume control, use `ma_volume_db_to_linear()` to convert from dB to linear.\n\nWhen a sound is spatialized, it is done so relative to a listener. An engine can be configured to\nhave multiple listeners which can be configured via the config:\n\n    ```c\n    engineConfig.listenerCount = 2;\n    ```\n\nThe maximum number of listeners is restricted to `MA_ENGINE_MAX_LISTENERS`. By default, when a\nsound is spatialized, it will be done so relative to the closest listener. You can also pin a sound\nto a specific listener which will be explained later. Listener's have a position, direction, cone,\nand velocity (for doppler effect). A listener is referenced by an index, the meaning of which is up\nto the caller (the index is 0 based and cannot go beyond the listener count, minus 1). The\nposition, direction and velocity are all specified in absolute terms:\n\n    ```c\n    ma_engine_listener_set_position(&engine, listenerIndex, worldPosX, worldPosY, worldPosZ);\n    ```\n\nThe direction of the listener represents it's forward vector. The listener's up vector can also be\nspecified and defaults to +1 on the Y axis.\n\n    ```c\n    ma_engine_listener_set_direction(&engine, listenerIndex, forwardX, forwardY, forwardZ);\n    ma_engine_listener_set_world_up(&engine, listenerIndex, 0, 1, 0);\n    ```\n\nThe engine supports directional attenuation. The listener can have a cone the controls how sound is\nattenuated based on the listener's direction. When a sound is between the inner and outer cones, it\nwill be attenuated between 1 and the cone's outer gain:\n\n    ```c\n    ma_engine_listener_set_cone(&engine, listenerIndex, innerAngleInRadians, outerAngleInRadians, outerGain);\n    ```\n\nWhen a sound is inside the inner code, no directional attenuation is applied. When the sound is\noutside of the outer cone, the attenuation will be set to `outerGain` in the example above. When\nthe sound is in between the inner and outer cones, the attenuation will be interpolated between 1\nand the outer gain.\n\nThe engine's coordinate system follows the OpenGL coordinate system where positive X points right,\npositive Y points up and negative Z points forward.\n\nThe simplest and least flexible way to play a sound is like so:\n\n    ```c\n    ma_engine_play_sound(&engine, \"my_sound.wav\", pGroup);\n    ```\n\nThis is a \"fire and forget\" style of function. The engine will manage the `ma_sound` object\ninternally. When the sound finishes playing, it'll be put up for recycling. For more flexibility\nyou'll want to initialize a sound object:\n\n    ```c\n    ma_sound sound;\n\n    result = ma_sound_init_from_file(&engine, \"my_sound.wav\", flags, pGroup, NULL, &sound);\n    if (result != MA_SUCCESS) {\n        return result;  // Failed to load sound.\n    }\n    ```\n\nSounds need to be uninitialized with `ma_sound_uninit()`.\n\nThe example above loads a sound from a file. If the resource manager has been disabled you will not\nbe able to use this function and instead you'll need to initialize a sound directly from a data\nsource:\n\n    ```c\n    ma_sound sound;\n\n    result = ma_sound_init_from_data_source(&engine, &dataSource, flags, pGroup, &sound);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n    ```\n\nEach `ma_sound` object represents a single instance of the sound. If you want to play the same\nsound multiple times at the same time, you need to initialize a separate `ma_sound` object.\n\nFor the most flexibility when initializing sounds, use `ma_sound_init_ex()`. This uses miniaudio's\nstandard config/init pattern:\n\n    ```c\n    ma_sound sound;\n    ma_sound_config soundConfig;\n\n    soundConfig = ma_sound_config_init();\n    soundConfig.pFilePath   = NULL; // Set this to load from a file path.\n    soundConfig.pDataSource = NULL; // Set this to initialize from an existing data source.\n    soundConfig.pInitialAttachment = &someNodeInTheNodeGraph;\n    soundConfig.initialAttachmentInputBusIndex = 0;\n    soundConfig.channelsIn  = 1;\n    soundConfig.channelsOut = 0;    // Set to 0 to use the engine's native channel count.\n\n    result = ma_sound_init_ex(&soundConfig, &sound);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n    ```\n\nIn the example above, the sound is being initialized without a file nor a data source. This is\nvalid, in which case the sound acts as a node in the middle of the node graph. This means you can\nconnect other sounds to this sound and allow it to act like a sound group. Indeed, this is exactly\nwhat a `ma_sound_group` is.\n\nWhen loading a sound, you specify a set of flags that control how the sound is loaded and what\nfeatures are enabled for that sound. When no flags are set, the sound will be fully loaded into\nmemory in exactly the same format as how it's stored on the file system. The resource manager will\nallocate a block of memory and then load the file directly into it. When reading audio data, it\nwill be decoded dynamically on the fly. In order to save processing time on the audio thread, it\nmight be beneficial to pre-decode the sound. You can do this with the `MA_SOUND_FLAG_DECODE` flag:\n\n    ```c\n    ma_sound_init_from_file(&engine, \"my_sound.wav\", MA_SOUND_FLAG_DECODE, pGroup, NULL, &sound);\n    ```\n\nBy default, sounds will be loaded synchronously, meaning `ma_sound_init_*()` will not return until\nthe sound has been fully loaded. If this is prohibitive you can instead load sounds asynchronously\nby specifying the `MA_SOUND_FLAG_ASYNC` flag:\n\n    ```c\n    ma_sound_init_from_file(&engine, \"my_sound.wav\", MA_SOUND_FLAG_DECODE | MA_SOUND_FLAG_ASYNC, pGroup, NULL, &sound);\n    ```\n\nThis will result in `ma_sound_init_*()` returning quickly, but the sound won't yet have been fully\nloaded. When you start the sound, it won't output anything until some sound is available. The sound\nwill start outputting audio before the sound has been fully decoded when the `MA_SOUND_FLAG_DECODE`\nis specified.\n\nIf you need to wait for an asynchronously loaded sound to be fully loaded, you can use a fence. A\nfence in miniaudio is a simple synchronization mechanism which simply blocks until it's internal\ncounter hit's zero. You can specify a fence like so:\n\n    ```c\n    ma_result result;\n    ma_fence fence;\n    ma_sound sounds[4];\n\n    result = ma_fence_init(&fence);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    // Load some sounds asynchronously.\n    for (int iSound = 0; iSound < 4; iSound += 1) {\n        ma_sound_init_from_file(&engine, mySoundFilesPaths[iSound], MA_SOUND_FLAG_DECODE | MA_SOUND_FLAG_ASYNC, pGroup, &fence, &sounds[iSound]);\n    }\n\n    // ... do some other stuff here in the mean time ...\n\n    // Wait for all sounds to finish loading.\n    ma_fence_wait(&fence);\n    ```\n\nIf loading the entire sound into memory is prohibitive, you can also configure the engine to stream\nthe audio data:\n\n    ```c\n    ma_sound_init_from_file(&engine, \"my_sound.wav\", MA_SOUND_FLAG_STREAM, pGroup, NULL, &sound);\n    ```\n\nWhen streaming sounds, 2 seconds worth of audio data is stored in memory. Although it should work\nfine, it's inefficient to use streaming for short sounds. Streaming is useful for things like music\ntracks in games.\n\nWhen loading a sound from a file path, the engine will reference count the file to prevent it from\nbeing loaded if it's already in memory. When you uninitialize a sound, the reference count will be\ndecremented, and if it hits zero, the sound will be unloaded from memory. This reference counting\nsystem is not used for streams. The engine will use a 64-bit hash of the file name when comparing\nfile paths which means there's a small chance you might encounter a name collision. If this is an\nissue, you'll need to use a different name for one of the colliding file paths, or just not load\nfrom files and instead load from a data source.\n\nYou can use `ma_sound_init_copy()` to initialize a copy of another sound. Note, however, that this\nonly works for sounds that were initialized with `ma_sound_init_from_file()` and without the\n`MA_SOUND_FLAG_STREAM` flag.\n\nWhen you initialize a sound, if you specify a sound group the sound will be attached to that group\nautomatically. If you set it to NULL, it will be automatically attached to the engine's endpoint.\nIf you would instead rather leave the sound unattached by default, you can specify the\n`MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT` flag. This is useful if you want to set up a complex node\ngraph.\n\nSounds are not started by default. To start a sound, use `ma_sound_start()`. Stop a sound with\n`ma_sound_stop()`.\n\nSounds can have their volume controlled with `ma_sound_set_volume()` in the same way as the\nengine's master volume.\n\nSounds support stereo panning and pitching. Set the pan with `ma_sound_set_pan()`. Setting the pan\nto 0 will result in an unpanned sound. Setting it to -1 will shift everything to the left, whereas\n+1 will shift it to the right. The pitch can be controlled with `ma_sound_set_pitch()`. A larger\nvalue will result in a higher pitch. The pitch must be greater than 0.\n\nThe engine supports 3D spatialization of sounds. By default sounds will have spatialization\nenabled, but if a sound does not need to be spatialized it's best to disable it. There are two ways\nto disable spatialization of a sound:\n\n    ```c\n    // Disable spatialization at initialization time via a flag:\n    ma_sound_init_from_file(&engine, \"my_sound.wav\", MA_SOUND_FLAG_NO_SPATIALIZATION, NULL, NULL, &sound);\n\n    // Dynamically disable or enable spatialization post-initialization:\n    ma_sound_set_spatialization_enabled(&sound, isSpatializationEnabled);\n    ```\n\nBy default sounds will be spatialized based on the closest listener. If a sound should always be\nspatialized relative to a specific listener it can be pinned to one:\n\n    ```c\n    ma_sound_set_pinned_listener_index(&sound, listenerIndex);\n    ```\n\nLike listeners, sounds have a position. By default, the position of a sound is in absolute space,\nbut it can be changed to be relative to a listener:\n\n    ```c\n    ma_sound_set_positioning(&sound, ma_positioning_relative);\n    ```\n\nNote that relative positioning of a sound only makes sense if there is either only one listener, or\nthe sound is pinned to a specific listener. To set the position of a sound:\n\n    ```c\n    ma_sound_set_position(&sound, posX, posY, posZ);\n    ```\n\nThe direction works the same way as a listener and represents the sound's forward direction:\n\n    ```c\n    ma_sound_set_direction(&sound, forwardX, forwardY, forwardZ);\n    ```\n\nSound's also have a cone for controlling directional attenuation. This works exactly the same as\nlisteners:\n\n    ```c\n    ma_sound_set_cone(&sound, innerAngleInRadians, outerAngleInRadians, outerGain);\n    ```\n\nThe velocity of a sound is used for doppler effect and can be set as such:\n\n    ```c\n    ma_sound_set_velocity(&sound, velocityX, velocityY, velocityZ);\n    ```\n\nThe engine supports different attenuation models which can be configured on a per-sound basis. By\ndefault the attenuation model is set to `ma_attenuation_model_inverse` which is the equivalent to\nOpenAL's `AL_INVERSE_DISTANCE_CLAMPED`. Configure the attenuation model like so:\n\n    ```c\n    ma_sound_set_attenuation_model(&sound, ma_attenuation_model_inverse);\n    ```\n\nThe supported attenuation models include the following:\n\n    +----------------------------------+----------------------------------------------+\n    | ma_attenuation_model_none        | No distance attenuation.                     |\n    +----------------------------------+----------------------------------------------+\n    | ma_attenuation_model_inverse     | Equivalent to `AL_INVERSE_DISTANCE_CLAMPED`. |\n    +----------------------------------+----------------------------------------------+\n    | ma_attenuation_model_linear      | Linear attenuation.                          |\n    +----------------------------------+----------------------------------------------+\n    | ma_attenuation_model_exponential | Exponential attenuation.                     |\n    +----------------------------------+----------------------------------------------+\n\nTo control how quickly a sound rolls off as it moves away from the listener, you need to configure\nthe rolloff:\n\n    ```c\n    ma_sound_set_rolloff(&sound, rolloff);\n    ```\n\nYou can control the minimum and maximum gain to apply from spatialization:\n\n    ```c\n    ma_sound_set_min_gain(&sound, minGain);\n    ma_sound_set_max_gain(&sound, maxGain);\n    ```\n\nLikewise, in the calculation of attenuation, you can control the minimum and maximum distances for\nthe attenuation calculation. This is useful if you want to ensure sounds don't drop below a certain\nvolume after the listener moves further away and to have sounds play a maximum volume when the\nlistener is within a certain distance:\n\n    ```c\n    ma_sound_set_min_distance(&sound, minDistance);\n    ma_sound_set_max_distance(&sound, maxDistance);\n    ```\n\nThe engine's spatialization system supports doppler effect. The doppler factor can be configure on\na per-sound basis like so:\n\n    ```c\n    ma_sound_set_doppler_factor(&sound, dopplerFactor);\n    ```\n\nYou can fade sounds in and out with `ma_sound_set_fade_in_pcm_frames()` and\n`ma_sound_set_fade_in_milliseconds()`. Set the volume to -1 to use the current volume as the\nstarting volume:\n\n    ```c\n    // Fade in over 1 second.\n    ma_sound_set_fade_in_milliseconds(&sound, 0, 1, 1000);\n\n    // ... sometime later ...\n\n    // Fade out over 1 second, starting from the current volume.\n    ma_sound_set_fade_in_milliseconds(&sound, -1, 0, 1000);\n    ```\n\nBy default sounds will start immediately, but sometimes for timing and synchronization purposes it\ncan be useful to schedule a sound to start or stop:\n\n    ```c\n    // Start the sound in 1 second from now.\n    ma_sound_set_start_time_in_pcm_frames(&sound, ma_engine_get_time_in_pcm_frames(&engine) + (ma_engine_get_sample_rate(&engine) * 1));\n\n    // Stop the sound in 2 seconds from now.\n    ma_sound_set_stop_time_in_pcm_frames(&sound, ma_engine_get_time_in_pcm_frames(&engine) + (ma_engine_get_sample_rate(&engine) * 2));\n    ```\n\nNote that scheduling a start time still requires an explicit call to `ma_sound_start()` before\nanything will play.\n\nThe time is specified in global time which is controlled by the engine. You can get the engine's\ncurrent time with `ma_engine_get_time_in_pcm_frames()`. The engine's global time is incremented\nautomatically as audio data is read, but it can be reset with `ma_engine_set_time_in_pcm_frames()`\nin case it needs to be resynchronized for some reason.\n\nTo determine whether or not a sound is currently playing, use `ma_sound_is_playing()`. This will\ntake the scheduled start and stop times into account.\n\nWhether or not a sound should loop can be controlled with `ma_sound_set_looping()`. Sounds will not\nbe looping by default. Use `ma_sound_is_looping()` to determine whether or not a sound is looping.\n\nUse `ma_sound_at_end()` to determine whether or not a sound is currently at the end. For a looping\nsound this should never return true. Alternatively, you can configure a callback that will be fired\nwhen the sound reaches the end. Note that the callback is fired from the audio thread which means\nyou cannot be uninitializing sound from the callback. To set the callback you can use\n`ma_sound_set_end_callback()`. Alternatively, if you're using `ma_sound_init_ex()`, you can pass it\ninto the config like so:\n\n    ```c\n    soundConfig.endCallback = my_end_callback;\n    soundConfig.pEndCallbackUserData = pMyEndCallbackUserData;\n    ```\n\nThe end callback is declared like so:\n\n    ```c\n    void my_end_callback(void* pUserData, ma_sound* pSound)\n    {\n        ...\n    }\n    ```\n\nInternally a sound wraps around a data source. Some APIs exist to control the underlying data\nsource, mainly for convenience:\n\n    ```c\n    ma_sound_seek_to_pcm_frame(&sound, frameIndex);\n    ma_sound_get_data_format(&sound, &format, &channels, &sampleRate, pChannelMap, channelMapCapacity);\n    ma_sound_get_cursor_in_pcm_frames(&sound, &cursor);\n    ma_sound_get_length_in_pcm_frames(&sound, &length);\n    ```\n\nSound groups have the same API as sounds, only they are called `ma_sound_group`, and since they do\nnot have any notion of a data source, anything relating to a data source is unavailable.\n\nInternally, sound data is loaded via the `ma_decoder` API which means by default it only supports\nfile formats that have built-in support in miniaudio. You can extend this to support any kind of\nfile format through the use of custom decoders. To do this you'll need to use a self-managed\nresource manager and configure it appropriately. See the \"Resource Management\" section below for\ndetails on how to set this up.\n\n\n6. Resource Management\n======================\nMany programs will want to manage sound resources for things such as reference counting and\nstreaming. This is supported by miniaudio via the `ma_resource_manager` API.\n\nThe resource manager is mainly responsible for the following:\n\n  * Loading of sound files into memory with reference counting.\n  * Streaming of sound data.\n\nWhen loading a sound file, the resource manager will give you back a `ma_data_source` compatible\nobject called `ma_resource_manager_data_source`. This object can be passed into any\n`ma_data_source` API which is how you can read and seek audio data. When loading a sound file, you\nspecify whether or not you want the sound to be fully loaded into memory (and optionally\npre-decoded) or streamed. When loading into memory, you can also specify whether or not you want\nthe data to be loaded asynchronously.\n\nThe example below is how you can initialize a resource manager using it's default configuration:\n\n    ```c\n    ma_resource_manager_config config;\n    ma_resource_manager resourceManager;\n\n    config = ma_resource_manager_config_init();\n    result = ma_resource_manager_init(&config, &resourceManager);\n    if (result != MA_SUCCESS) {\n        ma_device_uninit(&device);\n        printf(\"Failed to initialize the resource manager.\");\n        return -1;\n    }\n    ```\n\nYou can configure the format, channels and sample rate of the decoded audio data. By default it\nwill use the file's native data format, but you can configure it to use a consistent format. This\nis useful for offloading the cost of data conversion to load time rather than dynamically\nconverting at mixing time. To do this, you configure the decoded format, channels and sample rate\nlike the code below:\n\n    ```c\n    config = ma_resource_manager_config_init();\n    config.decodedFormat     = device.playback.format;\n    config.decodedChannels   = device.playback.channels;\n    config.decodedSampleRate = device.sampleRate;\n    ```\n\nIn the code above, the resource manager will be configured so that any decoded audio data will be\npre-converted at load time to the device's native data format. If instead you used defaults and\nthe data format of the file did not match the device's data format, you would need to convert the\ndata at mixing time which may be prohibitive in high-performance and large scale scenarios like\ngames.\n\nInternally the resource manager uses the `ma_decoder` API to load sounds. This means by default it\nonly supports decoders that are built into miniaudio. It's possible to support additional encoding\nformats through the use of custom decoders. To do so, pass in your `ma_decoding_backend_vtable`\nvtables into the resource manager config:\n\n    ```c\n    ma_decoding_backend_vtable* pCustomBackendVTables[] =\n    {\n        &g_ma_decoding_backend_vtable_libvorbis,\n        &g_ma_decoding_backend_vtable_libopus\n    };\n\n    ...\n\n    resourceManagerConfig.ppCustomDecodingBackendVTables = pCustomBackendVTables;\n    resourceManagerConfig.customDecodingBackendCount     = sizeof(pCustomBackendVTables) / sizeof(pCustomBackendVTables[0]);\n    resourceManagerConfig.pCustomDecodingBackendUserData = NULL;\n    ```\n\nThis system can allow you to support any kind of file format. See the \"Decoding\" section for\ndetails on how to implement custom decoders. The miniaudio repository includes examples for Opus\nvia libopus and libopusfile and Vorbis via libvorbis and libvorbisfile.\n\nAsynchronicity is achieved via a job system. When an operation needs to be performed, such as the\ndecoding of a page, a job will be posted to a queue which will then be processed by a job thread.\nBy default there will be only one job thread running, but this can be configured, like so:\n\n    ```c\n    config = ma_resource_manager_config_init();\n    config.jobThreadCount = MY_JOB_THREAD_COUNT;\n    ```\n\nBy default job threads are managed internally by the resource manager, however you can also self\nmanage your job threads if, for example, you want to integrate the job processing into your\nexisting job infrastructure, or if you simply don't like the way the resource manager does it. To\ndo this, just set the job thread count to 0 and process jobs manually. To process jobs, you first\nneed to retrieve a job using `ma_resource_manager_next_job()` and then process it using\n`ma_job_process()`:\n\n    ```c\n    config = ma_resource_manager_config_init();\n    config.jobThreadCount = 0;                            // Don't manage any job threads internally.\n    config.flags = MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING; // Optional. Makes `ma_resource_manager_next_job()` non-blocking.\n\n    // ... Initialize your custom job threads ...\n\n    void my_custom_job_thread(...)\n    {\n        for (;;) {\n            ma_job job;\n            ma_result result = ma_resource_manager_next_job(pMyResourceManager, &job);\n            if (result != MA_SUCCESS) {\n                if (result == MA_NO_DATA_AVAILABLE) {\n                    // No jobs are available. Keep going. Will only get this if the resource manager was initialized\n                    // with MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING.\n                    continue;\n                } else if (result == MA_CANCELLED) {\n                    // MA_JOB_TYPE_QUIT was posted. Exit.\n                    break;\n                } else {\n                    // Some other error occurred.\n                    break;\n                }\n            }\n\n            ma_job_process(&job);\n        }\n    }\n    ```\n\nIn the example above, the `MA_JOB_TYPE_QUIT` event is the used as the termination\nindicator, but you can use whatever you would like to terminate the thread. The call to\n`ma_resource_manager_next_job()` is blocking by default, but can be configured to be non-blocking\nby initializing the resource manager with the `MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING` configuration\nflag. Note that the `MA_JOB_TYPE_QUIT` will never be removed from the job queue. This\nis to give every thread the opportunity to catch the event and terminate naturally.\n\nWhen loading a file, it's sometimes convenient to be able to customize how files are opened and\nread instead of using standard `fopen()`, `fclose()`, etc. which is what miniaudio will use by\ndefault. This can be done by setting `pVFS` member of the resource manager's config:\n\n    ```c\n    // Initialize your custom VFS object. See documentation for VFS for information on how to do this.\n    my_custom_vfs vfs = my_custom_vfs_init();\n\n    config = ma_resource_manager_config_init();\n    config.pVFS = &vfs;\n    ```\n\nThis is particularly useful in programs like games where you want to read straight from an archive\nrather than the normal file system. If you do not specify a custom VFS, the resource manager will\nuse the operating system's normal file operations.\n\nTo load a sound file and create a data source, call `ma_resource_manager_data_source_init()`. When\nloading a sound you need to specify the file path and options for how the sounds should be loaded.\nBy default a sound will be loaded synchronously. The returned data source is owned by the caller\nwhich means the caller is responsible for the allocation and freeing of the data source. Below is\nan example for initializing a data source:\n\n    ```c\n    ma_resource_manager_data_source dataSource;\n    ma_result result = ma_resource_manager_data_source_init(pResourceManager, pFilePath, flags, &dataSource);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    // ...\n\n    // A ma_resource_manager_data_source object is compatible with the `ma_data_source` API. To read data, just call\n    // the `ma_data_source_read_pcm_frames()` like you would with any normal data source.\n    result = ma_data_source_read_pcm_frames(&dataSource, pDecodedData, frameCount, &framesRead);\n    if (result != MA_SUCCESS) {\n        // Failed to read PCM frames.\n    }\n\n    // ...\n\n    ma_resource_manager_data_source_uninit(&dataSource);\n    ```\n\nThe `flags` parameter specifies how you want to perform loading of the sound file. It can be a\ncombination of the following flags:\n\n    ```\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING\n    ```\n\nWhen no flags are specified (set to 0), the sound will be fully loaded into memory, but not\ndecoded, meaning the raw file data will be stored in memory, and then dynamically decoded when\n`ma_data_source_read_pcm_frames()` is called. To instead decode the audio data before storing it in\nmemory, use the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE` flag. By default, the sound file will\nbe loaded synchronously, meaning `ma_resource_manager_data_source_init()` will only return after\nthe entire file has been loaded. This is good for simplicity, but can be prohibitively slow. You\ncan instead load the sound asynchronously using the `MA_RESOURCE_MANAGER_DATA_SOURCE_ASYNC` flag.\nThis will result in `ma_resource_manager_data_source_init()` returning quickly, but no data will be\nreturned by `ma_data_source_read_pcm_frames()` until some data is available. When no data is\navailable because the asynchronous decoding hasn't caught up, `MA_BUSY` will be returned by\n`ma_data_source_read_pcm_frames()`.\n\nFor large sounds, it's often prohibitive to store the entire file in memory. To mitigate this, you\ncan instead stream audio data which you can do by specifying the\n`MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM` flag. When streaming, data will be decoded in 1\nsecond pages. When a new page needs to be decoded, a job will be posted to the job queue and then\nsubsequently processed in a job thread.\n\nThe `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING` flag can be used so that the sound will loop\nwhen it reaches the end by default. It's recommended you use this flag when you want to have a\nlooping streaming sound. If you try loading a very short sound as a stream, you will get a glitch.\nThis is because the resource manager needs to pre-fill the initial buffer at initialization time,\nand if you don't specify the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING` flag, the resource\nmanager will assume the sound is not looping and will stop filling the buffer when it reaches the\nend, therefore resulting in a discontinuous buffer.\n\nFor in-memory sounds, reference counting is used to ensure the data is loaded only once. This means\nmultiple calls to `ma_resource_manager_data_source_init()` with the same file path will result in\nthe file data only being loaded once. Each call to `ma_resource_manager_data_source_init()` must be\nmatched up with a call to `ma_resource_manager_data_source_uninit()`. Sometimes it can be useful\nfor a program to register self-managed raw audio data and associate it with a file path. Use the\n`ma_resource_manager_register_*()` and `ma_resource_manager_unregister_*()` APIs to do this.\n`ma_resource_manager_register_decoded_data()` is used to associate a pointer to raw, self-managed\ndecoded audio data in the specified data format with the specified name. Likewise,\n`ma_resource_manager_register_encoded_data()` is used to associate a pointer to raw self-managed\nencoded audio data (the raw file data) with the specified name. Note that these names need not be\nactual file paths. When `ma_resource_manager_data_source_init()` is called (without the\n`MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM` flag), the resource manager will look for these\nexplicitly registered data buffers and, if found, will use it as the backing data for the data\nsource. Note that the resource manager does *not* make a copy of this data so it is up to the\ncaller to ensure the pointer stays valid for its lifetime. Use\n`ma_resource_manager_unregister_data()` to unregister the self-managed data. You can also use\n`ma_resource_manager_register_file()` and `ma_resource_manager_unregister_file()` to register and\nunregister a file. It does not make sense to use the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM`\nflag with a self-managed data pointer.\n\n\n6.1. Asynchronous Loading and Synchronization\n---------------------------------------------\nWhen loading asynchronously, it can be useful to poll whether or not loading has finished. Use\n`ma_resource_manager_data_source_result()` to determine this. For in-memory sounds, this will\nreturn `MA_SUCCESS` when the file has been *entirely* decoded. If the sound is still being decoded,\n`MA_BUSY` will be returned. Otherwise, some other error code will be returned if the sound failed\nto load. For streaming data sources, `MA_SUCCESS` will be returned when the first page has been\ndecoded and the sound is ready to be played. If the first page is still being decoded, `MA_BUSY`\nwill be returned. Otherwise, some other error code will be returned if the sound failed to load.\n\nIn addition to polling, you can also use a simple synchronization object called a \"fence\" to wait\nfor asynchronously loaded sounds to finish. This is called `ma_fence`. The advantage to using a\nfence is that it can be used to wait for a group of sounds to finish loading rather than waiting\nfor sounds on an individual basis. There are two stages to loading a sound:\n\n  * Initialization of the internal decoder; and\n  * Completion of decoding of the file (the file is fully decoded)\n\nYou can specify separate fences for each of the different stages. Waiting for the initialization\nof the internal decoder is important for when you need to know the sample format, channels and\nsample rate of the file.\n\nThe example below shows how you could use a fence when loading a number of sounds:\n\n    ```c\n    // This fence will be released when all sounds are finished loading entirely.\n    ma_fence fence;\n    ma_fence_init(&fence);\n\n    // This will be passed into the initialization routine for each sound.\n    ma_resource_manager_pipeline_notifications notifications = ma_resource_manager_pipeline_notifications_init();\n    notifications.done.pFence = &fence;\n\n    // Now load a bunch of sounds:\n    for (iSound = 0; iSound < soundCount; iSound += 1) {\n        ma_resource_manager_data_source_init(pResourceManager, pSoundFilePaths[iSound], flags, &notifications, &pSoundSources[iSound]);\n    }\n\n    // ... DO SOMETHING ELSE WHILE SOUNDS ARE LOADING ...\n\n    // Wait for loading of sounds to finish.\n    ma_fence_wait(&fence);\n    ```\n\nIn the example above we used a fence for waiting until the entire file has been fully decoded. If\nyou only need to wait for the initialization of the internal decoder to complete, you can use the\n`init` member of the `ma_resource_manager_pipeline_notifications` object:\n\n    ```c\n    notifications.init.pFence = &fence;\n    ```\n\nIf a fence is not appropriate for your situation, you can instead use a callback that is fired on\nan individual sound basis. This is done in a very similar way to fences:\n\n    ```c\n    typedef struct\n    {\n        ma_async_notification_callbacks cb;\n        void* pMyData;\n    } my_notification;\n\n    void my_notification_callback(ma_async_notification* pNotification)\n    {\n        my_notification* pMyNotification = (my_notification*)pNotification;\n\n        // Do something in response to the sound finishing loading.\n    }\n\n    ...\n\n    my_notification myCallback;\n    myCallback.cb.onSignal = my_notification_callback;\n    myCallback.pMyData     = pMyData;\n\n    ma_resource_manager_pipeline_notifications notifications = ma_resource_manager_pipeline_notifications_init();\n    notifications.done.pNotification = &myCallback;\n\n    ma_resource_manager_data_source_init(pResourceManager, \"my_sound.wav\", flags, &notifications, &mySound);\n    ```\n\nIn the example above we just extend the `ma_async_notification_callbacks` object and pass an\ninstantiation into the `ma_resource_manager_pipeline_notifications` in the same way as we did with\nthe fence, only we set `pNotification` instead of `pFence`. You can set both of these at the same\ntime and they should both work as expected. If using the `pNotification` system, you need to ensure\nyour `ma_async_notification_callbacks` object stays valid.\n\n\n\n6.2. Resource Manager Implementation Details\n--------------------------------------------\nResources are managed in two main ways:\n\n  * By storing the entire sound inside an in-memory buffer (referred to as a data buffer)\n  * By streaming audio data on the fly (referred to as a data stream)\n\nA resource managed data source (`ma_resource_manager_data_source`) encapsulates a data buffer or\ndata stream, depending on whether or not the data source was initialized with the\n`MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM` flag. If so, it will make use of a\n`ma_resource_manager_data_stream` object. Otherwise it will use a `ma_resource_manager_data_buffer`\nobject. Both of these objects are data sources which means they can be used with any\n`ma_data_source_*()` API.\n\nAnother major feature of the resource manager is the ability to asynchronously decode audio files.\nThis relieves the audio thread of time-consuming decoding which can negatively affect scalability\ndue to the audio thread needing to complete it's work extremely quickly to avoid glitching.\nAsynchronous decoding is achieved through a job system. There is a central multi-producer,\nmulti-consumer, fixed-capacity job queue. When some asynchronous work needs to be done, a job is\nposted to the queue which is then read by a job thread. The number of job threads can be\nconfigured for improved scalability, and job threads can all run in parallel without needing to\nworry about the order of execution (how this is achieved is explained below).\n\nWhen a sound is being loaded asynchronously, playback can begin before the sound has been fully\ndecoded. This enables the application to start playback of the sound quickly, while at the same\ntime allowing to resource manager to keep loading in the background. Since there may be less\nthreads than the number of sounds being loaded at a given time, a simple scheduling system is used\nto keep decoding time balanced and fair. The resource manager solves this by splitting decoding\ninto chunks called pages. By default, each page is 1 second long. When a page has been decoded, a\nnew job will be posted to start decoding the next page. By dividing up decoding into pages, an\nindividual sound shouldn't ever delay every other sound from having their first page decoded. Of\ncourse, when loading many sounds at the same time, there will always be an amount of time required\nto process jobs in the queue so in heavy load situations there will still be some delay. To\ndetermine if a data source is ready to have some frames read, use\n`ma_resource_manager_data_source_get_available_frames()`. This will return the number of frames\navailable starting from the current position.\n\n\n6.2.1. Job Queue\n----------------\nThe resource manager uses a job queue which is multi-producer, multi-consumer, and fixed-capacity.\nThis job queue is not currently lock-free, and instead uses a spinlock to achieve thread-safety.\nOnly a fixed number of jobs can be allocated and inserted into the queue which is done through a\nlock-free data structure for allocating an index into a fixed sized array, with reference counting\nfor mitigation of the ABA problem. The reference count is 32-bit.\n\nFor many types of jobs it's important that they execute in a specific order. In these cases, jobs\nare executed serially. For the resource manager, serial execution of jobs is only required on a\nper-object basis (per data buffer or per data stream). Each of these objects stores an execution\ncounter. When a job is posted it is associated with an execution counter. When the job is\nprocessed, it checks if the execution counter of the job equals the execution counter of the\nowning object and if so, processes the job. If the counters are not equal, the job will be posted\nback onto the job queue for later processing. When the job finishes processing the execution order\nof the main object is incremented. This system means the no matter how many job threads are\nexecuting, decoding of an individual sound will always get processed serially. The advantage to\nhaving multiple threads comes into play when loading multiple sounds at the same time.\n\nThe resource manager's job queue is not 100% lock-free and will use a spinlock to achieve\nthread-safety for a very small section of code. This is only relevant when the resource manager\nuses more than one job thread. If only using a single job thread, which is the default, the\nlock should never actually wait in practice. The amount of time spent locking should be quite\nshort, but it's something to be aware of for those who have pedantic lock-free requirements and\nneed to use more than one job thread. There are plans to remove this lock in a future version.\n\nIn addition, posting a job will release a semaphore, which on Win32 is implemented with\n`ReleaseSemaphore` and on POSIX platforms via a condition variable:\n\n    ```c\n    pthread_mutex_lock(&pSemaphore->lock);\n    {\n        pSemaphore->value += 1;\n        pthread_cond_signal(&pSemaphore->cond);\n    }\n    pthread_mutex_unlock(&pSemaphore->lock);\n    ```\n\nAgain, this is relevant for those with strict lock-free requirements in the audio thread. To avoid\nthis, you can use non-blocking mode (via the `MA_JOB_QUEUE_FLAG_NON_BLOCKING`\nflag) and implement your own job processing routine (see the \"Resource Manager\" section above for\ndetails on how to do this).\n\n\n\n6.2.2. Data Buffers\n-------------------\nWhen the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM` flag is excluded at initialization time, the\nresource manager will try to load the data into an in-memory data buffer. Before doing so, however,\nit will first check if the specified file is already loaded. If so, it will increment a reference\ncounter and just use the already loaded data. This saves both time and memory. When the data buffer\nis uninitialized, the reference counter will be decremented. If the counter hits zero, the file\nwill be unloaded. This is a detail to keep in mind because it could result in excessive loading and\nunloading of a sound. For example, the following sequence will result in a file be loaded twice,\nonce after the other:\n\n    ```c\n    ma_resource_manager_data_source_init(pResourceManager, \"my_file\", ..., &myDataBuffer0); // Refcount = 1. Initial load.\n    ma_resource_manager_data_source_uninit(&myDataBuffer0);                                 // Refcount = 0. Unloaded.\n\n    ma_resource_manager_data_source_init(pResourceManager, \"my_file\", ..., &myDataBuffer1); // Refcount = 1. Reloaded because previous uninit() unloaded it.\n    ma_resource_manager_data_source_uninit(&myDataBuffer1);                                 // Refcount = 0. Unloaded.\n    ```\n\nA binary search tree (BST) is used for storing data buffers as it has good balance between\nefficiency and simplicity. The key of the BST is a 64-bit hash of the file path that was passed\ninto `ma_resource_manager_data_source_init()`. The advantage of using a hash is that it saves\nmemory over storing the entire path, has faster comparisons, and results in a mostly balanced BST\ndue to the random nature of the hash. The disadvantages are that file names are case-sensitive and\nthere's a small chance of name collisions. If case-sensitivity is an issue, you should normalize\nyour file names to upper- or lower-case before initializing your data sources. If name collisions\nbecome an issue, you'll need to change the name of one of the colliding names or just not use the\nresource manager.\n\nWhen a sound file has not already been loaded and the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC`\nflag is excluded, the file will be decoded synchronously by the calling thread. There are two\noptions for controlling how the audio is stored in the data buffer - encoded or decoded. When the\n`MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE` option is excluded, the raw file data will be stored\nin memory. Otherwise the sound will be decoded before storing it in memory. Synchronous loading is\na very simple and standard process of simply adding an item to the BST, allocating a block of\nmemory and then decoding (if `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE` is specified).\n\nWhen the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC` flag is specified, loading of the data buffer\nis done asynchronously. In this case, a job is posted to the queue to start loading and then the\nfunction immediately returns, setting an internal result code to `MA_BUSY`. This result code is\nreturned when the program calls `ma_resource_manager_data_source_result()`. When decoding has fully\ncompleted `MA_SUCCESS` will be returned. This can be used to know if loading has fully completed.\n\nWhen loading asynchronously, a single job is posted to the queue of the type\n`MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER_NODE`. This involves making a copy of the file path and\nassociating it with job. When the job is processed by the job thread, it will first load the file\nusing the VFS associated with the resource manager. When using a custom VFS, it's important that it\nbe completely thread-safe because it will be used from one or more job threads at the same time.\nIndividual files should only ever be accessed by one thread at a time, however. After opening the\nfile via the VFS, the job will determine whether or not the file is being decoded. If not, it\nsimply allocates a block of memory and loads the raw file contents into it and returns. On the\nother hand, when the file is being decoded, it will first allocate a decoder on the heap and\ninitialize it. Then it will check if the length of the file is known. If so it will allocate a\nblock of memory to store the decoded output and initialize it to silence. If the size is unknown,\nit will allocate room for one page. After memory has been allocated, the first page will be\ndecoded. If the sound is shorter than a page, the result code will be set to `MA_SUCCESS` and the\ncompletion event will be signalled and loading is now complete. If, however, there is more to\ndecode, a job with the code `MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE` is posted. This job\nwill decode the next page and perform the same process if it reaches the end. If there is more to\ndecode, the job will post another `MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE` job which will\nkeep on happening until the sound has been fully decoded. For sounds of an unknown length, each\npage will be linked together as a linked list. Internally this is implemented via the\n`ma_paged_audio_buffer` object.\n\n\n6.2.3. Data Streams\n-------------------\nData streams only ever store two pages worth of data for each instance. They are most useful for\nlarge sounds like music tracks in games that would consume too much memory if fully decoded in\nmemory. After every frame from a page has been read, a job will be posted to load the next page\nwhich is done from the VFS.\n\nFor data streams, the `MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC` flag will determine whether or\nnot initialization of the data source waits until the two pages have been decoded. When unset,\n`ma_resource_manager_data_source_init()` will wait until the two pages have been loaded, otherwise\nit will return immediately.\n\nWhen frames are read from a data stream using `ma_resource_manager_data_source_read_pcm_frames()`,\n`MA_BUSY` will be returned if there are no frames available. If there are some frames available,\nbut less than the number requested, `MA_SUCCESS` will be returned, but the actual number of frames\nread will be less than the number requested. Due to the asynchronous nature of data streams,\nseeking is also asynchronous. If the data stream is in the middle of a seek, `MA_BUSY` will be\nreturned when trying to read frames.\n\nWhen `ma_resource_manager_data_source_read_pcm_frames()` results in a page getting fully consumed\na job is posted to load the next page. This will be posted from the same thread that called\n`ma_resource_manager_data_source_read_pcm_frames()`.\n\nData streams are uninitialized by posting a job to the queue, but the function won't return until\nthat job has been processed. The reason for this is that the caller owns the data stream object and\ntherefore miniaudio needs to ensure everything completes before handing back control to the caller.\nAlso, if the data stream is uninitialized while pages are in the middle of decoding, they must\ncomplete before destroying any underlying object and the job system handles this cleanly.\n\nNote that when a new page needs to be loaded, a job will be posted to the resource manager's job\nthread from the audio thread. You must keep in mind the details mentioned in the \"Job Queue\"\nsection above regarding locking when posting an event if you require a strictly lock-free audio\nthread.\n\n\n\n7. Node Graph\n=============\nminiaudio's routing infrastructure follows a node graph paradigm. The idea is that you create a\nnode whose outputs are attached to inputs of another node, thereby creating a graph. There are\ndifferent types of nodes, with each node in the graph processing input data to produce output,\nwhich is then fed through the chain. Each node in the graph can apply their own custom effects. At\nthe start of the graph will usually be one or more data source nodes which have no inputs and\ninstead pull their data from a data source. At the end of the graph is an endpoint which represents\nthe end of the chain and is where the final output is ultimately extracted from.\n\nEach node has a number of input buses and a number of output buses. An output bus from a node is\nattached to an input bus of another. Multiple nodes can connect their output buses to another\nnode's input bus, in which case their outputs will be mixed before processing by the node. Below is\na diagram that illustrates a hypothetical node graph setup:\n\n    ```\n    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Data flows left to right >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\n    +---------------+                              +-----------------+\n    | Data Source 1 =----+    +----------+    +----= Low Pass Filter =----+\n    +---------------+    |    |          =----+    +-----------------+    |    +----------+\n                         +----= Splitter |                                +----= ENDPOINT |\n    +---------------+    |    |          =----+    +-----------------+    |    +----------+\n    | Data Source 2 =----+    +----------+    +----=  Echo / Delay   =----+\n    +---------------+                              +-----------------+\n    ```\n\nIn the above graph, it starts with two data sources whose outputs are attached to the input of a\nsplitter node. It's at this point that the two data sources are mixed. After mixing, the splitter\nperforms it's processing routine and produces two outputs which is simply a duplication of the\ninput stream. One output is attached to a low pass filter, whereas the other output is attached to\na echo/delay. The outputs of the low pass filter and the echo are attached to the endpoint, and\nsince they're both connected to the same input bus, they'll be mixed.\n\nEach input bus must be configured to accept the same number of channels, but the number of channels\nused by input buses can be different to the number of channels for output buses in which case\nminiaudio will automatically convert the input data to the output channel count before processing.\nThe number of channels of an output bus of one node must match the channel count of the input bus\nit's attached to. The channel counts cannot be changed after the node has been initialized. If you\nattempt to attach an output bus to an input bus with a different channel count, attachment will\nfail.\n\nTo use a node graph, you first need to initialize a `ma_node_graph` object. This is essentially a\ncontainer around the entire graph. The `ma_node_graph` object is required for some thread-safety\nissues which will be explained later. A `ma_node_graph` object is initialized using miniaudio's\nstandard config/init system:\n\n    ```c\n    ma_node_graph_config nodeGraphConfig = ma_node_graph_config_init(myChannelCount);\n\n    result = ma_node_graph_init(&nodeGraphConfig, NULL, &nodeGraph);    // Second parameter is a pointer to allocation callbacks.\n    if (result != MA_SUCCESS) {\n        // Failed to initialize node graph.\n    }\n    ```\n\nWhen you initialize the node graph, you're specifying the channel count of the endpoint. The\nendpoint is a special node which has one input bus and one output bus, both of which have the\nsame channel count, which is specified in the config. Any nodes that connect directly to the\nendpoint must be configured such that their output buses have the same channel count. When you read\naudio data from the node graph, it'll have the channel count you specified in the config. To read\ndata from the graph:\n\n    ```c\n    ma_uint32 framesRead;\n    result = ma_node_graph_read_pcm_frames(&nodeGraph, pFramesOut, frameCount, &framesRead);\n    if (result != MA_SUCCESS) {\n        // Failed to read data from the node graph.\n    }\n    ```\n\nWhen you read audio data, miniaudio starts at the node graph's endpoint node which then pulls in\ndata from its input attachments, which in turn recursively pull in data from their inputs, and so\non. At the start of the graph there will be some kind of data source node which will have zero\ninputs and will instead read directly from a data source. The base nodes don't literally need to\nread from a `ma_data_source` object, but they will always have some kind of underlying object that\nsources some kind of audio. The `ma_data_source_node` node can be used to read from a\n`ma_data_source`. Data is always in floating-point format and in the number of channels you\nspecified when the graph was initialized. The sample rate is defined by the underlying data sources.\nIt's up to you to ensure they use a consistent and appropriate sample rate.\n\nThe `ma_node` API is designed to allow custom nodes to be implemented with relative ease, but\nminiaudio includes a few stock nodes for common functionality. This is how you would initialize a\nnode which reads directly from a data source (`ma_data_source_node`) which is an example of one\nof the stock nodes that comes with miniaudio:\n\n    ```c\n    ma_data_source_node_config config = ma_data_source_node_config_init(pMyDataSource);\n\n    ma_data_source_node dataSourceNode;\n    result = ma_data_source_node_init(&nodeGraph, &config, NULL, &dataSourceNode);\n    if (result != MA_SUCCESS) {\n        // Failed to create data source node.\n    }\n    ```\n\nThe data source node will use the output channel count to determine the channel count of the output\nbus. There will be 1 output bus and 0 input buses (data will be drawn directly from the data\nsource). The data source must output to floating-point (`ma_format_f32`) or else an error will be\nreturned from `ma_data_source_node_init()`.\n\nBy default the node will not be attached to the graph. To do so, use `ma_node_attach_output_bus()`:\n\n    ```c\n    result = ma_node_attach_output_bus(&dataSourceNode, 0, ma_node_graph_get_endpoint(&nodeGraph), 0);\n    if (result != MA_SUCCESS) {\n        // Failed to attach node.\n    }\n    ```\n\nThe code above connects the data source node directly to the endpoint. Since the data source node\nhas only a single output bus, the index will always be 0. Likewise, the endpoint only has a single\ninput bus which means the input bus index will also always be 0.\n\nTo detach a specific output bus, use `ma_node_detach_output_bus()`. To detach all output buses, use\n`ma_node_detach_all_output_buses()`. If you want to just move the output bus from one attachment to\nanother, you do not need to detach first. You can just call `ma_node_attach_output_bus()` and it'll\ndeal with it for you.\n\nLess frequently you may want to create a specialized node. This will be a node where you implement\nyour own processing callback to apply a custom effect of some kind. This is similar to initializing\none of the stock node types, only this time you need to specify a pointer to a vtable containing a\npointer to the processing function and the number of input and output buses. Example:\n\n    ```c\n    static void my_custom_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n    {\n        // Do some processing of ppFramesIn (one stream of audio data per input bus)\n        const float* pFramesIn_0 = ppFramesIn[0]; // Input bus @ index 0.\n        const float* pFramesIn_1 = ppFramesIn[1]; // Input bus @ index 1.\n        float* pFramesOut_0 = ppFramesOut[0];     // Output bus @ index 0.\n\n        // Do some processing. On input, `pFrameCountIn` will be the number of input frames in each\n        // buffer in `ppFramesIn` and `pFrameCountOut` will be the capacity of each of the buffers\n        // in `ppFramesOut`. On output, `pFrameCountIn` should be set to the number of input frames\n        // your node consumed and `pFrameCountOut` should be set the number of output frames that\n        // were produced.\n        //\n        // You should process as many frames as you can. If your effect consumes input frames at the\n        // same rate as output frames (always the case, unless you're doing resampling), you need\n        // only look at `ppFramesOut` and process that exact number of frames. If you're doing\n        // resampling, you'll need to be sure to set both `pFrameCountIn` and `pFrameCountOut`\n        // properly.\n    }\n\n    static ma_node_vtable my_custom_node_vtable =\n    {\n        my_custom_node_process_pcm_frames, // The function that will be called to process your custom node. This is where you'd implement your effect processing.\n        NULL,   // Optional. A callback for calculating the number of input frames that are required to process a specified number of output frames.\n        2,      // 2 input buses.\n        1,      // 1 output bus.\n        0       // Default flags.\n    };\n\n    ...\n\n    // Each bus needs to have a channel count specified. To do this you need to specify the channel\n    // counts in an array and then pass that into the node config.\n    ma_uint32 inputChannels[2];     // Equal in size to the number of input channels specified in the vtable.\n    ma_uint32 outputChannels[1];    // Equal in size to the number of output channels specified in the vtable.\n\n    inputChannels[0]  = channelsIn;\n    inputChannels[1]  = channelsIn;\n    outputChannels[0] = channelsOut;\n\n    ma_node_config nodeConfig = ma_node_config_init();\n    nodeConfig.vtable          = &my_custom_node_vtable;\n    nodeConfig.pInputChannels  = inputChannels;\n    nodeConfig.pOutputChannels = outputChannels;\n\n    ma_node_base node;\n    result = ma_node_init(&nodeGraph, &nodeConfig, NULL, &node);\n    if (result != MA_SUCCESS) {\n        // Failed to initialize node.\n    }\n    ```\n\nWhen initializing a custom node, as in the code above, you'll normally just place your vtable in\nstatic space. The number of input and output buses are specified as part of the vtable. If you need\na variable number of buses on a per-node bases, the vtable should have the relevant bus count set\nto `MA_NODE_BUS_COUNT_UNKNOWN`. In this case, the bus count should be set in the node config:\n\n    ```c\n    static ma_node_vtable my_custom_node_vtable =\n    {\n        my_custom_node_process_pcm_frames, // The function that will be called process your custom node. This is where you'd implement your effect processing.\n        NULL,   // Optional. A callback for calculating the number of input frames that are required to process a specified number of output frames.\n        MA_NODE_BUS_COUNT_UNKNOWN,  // The number of input buses is determined on a per-node basis.\n        1,      // 1 output bus.\n        0       // Default flags.\n    };\n\n    ...\n\n    ma_node_config nodeConfig = ma_node_config_init();\n    nodeConfig.vtable          = &my_custom_node_vtable;\n    nodeConfig.inputBusCount   = myBusCount;        // <-- Since the vtable specifies MA_NODE_BUS_COUNT_UNKNOWN, the input bus count should be set here.\n    nodeConfig.pInputChannels  = inputChannels;     // <-- Make sure there are nodeConfig.inputBusCount elements in this array.\n    nodeConfig.pOutputChannels = outputChannels;    // <-- The vtable specifies 1 output bus, so there must be 1 element in this array.\n    ```\n\nIn the above example it's important to never set the `inputBusCount` and `outputBusCount` members\nto anything other than their defaults if the vtable specifies an explicit count. They can only be\nset if the vtable specifies MA_NODE_BUS_COUNT_UNKNOWN in the relevant bus count.\n\nMost often you'll want to create a structure to encapsulate your node with some extra data. You\nneed to make sure the `ma_node_base` object is your first member of the structure:\n\n    ```c\n    typedef struct\n    {\n        ma_node_base base; // <-- Make sure this is always the first member.\n        float someCustomData;\n    } my_custom_node;\n    ```\n\nBy doing this, your object will be compatible with all `ma_node` APIs and you can attach it to the\ngraph just like any other node.\n\nIn the custom processing callback (`my_custom_node_process_pcm_frames()` in the example above), the\nnumber of channels for each bus is what was specified by the config when the node was initialized\nwith `ma_node_init()`. In addition, all attachments to each of the input buses will have been\npre-mixed by miniaudio. The config allows you to specify different channel counts for each\nindividual input and output bus. It's up to the effect to handle it appropriate, and if it can't,\nreturn an error in it's initialization routine.\n\nCustom nodes can be assigned some flags to describe their behaviour. These are set via the vtable\nand include the following:\n\n    +-----------------------------------------+---------------------------------------------------+\n    | Flag Name                               | Description                                       |\n    +-----------------------------------------+---------------------------------------------------+\n    | MA_NODE_FLAG_PASSTHROUGH                | Useful for nodes that do not do any kind of audio |\n    |                                         | processing, but are instead used for tracking     |\n    |                                         | time, handling events, etc. Also used by the      |\n    |                                         | internal endpoint node. It reads directly from    |\n    |                                         | the input bus to the output bus. Nodes with this  |\n    |                                         | flag must have exactly 1 input bus and 1 output   |\n    |                                         | bus, and both buses must have the same channel    |\n    |                                         | counts.                                           |\n    +-----------------------------------------+---------------------------------------------------+\n    | MA_NODE_FLAG_CONTINUOUS_PROCESSING      | Causes the processing callback to be called even  |\n    |                                         | when no data is available to be read from input   |\n    |                                         | attachments. When a node has at least one input   |\n    |                                         | bus, but there are no inputs attached or the      |\n    |                                         | inputs do not deliver any data, the node's        |\n    |                                         | processing callback will not get fired. This flag |\n    |                                         | will make it so the callback is always fired      |\n    |                                         | regardless of whether or not any input data is    |\n    |                                         | received. This is useful for effects like         |\n    |                                         | echos where there will be a tail of audio data    |\n    |                                         | that still needs to be processed even when the    |\n    |                                         | original data sources have reached their ends. It |\n    |                                         | may also be useful for nodes that must always     |\n    |                                         | have their processing callback fired when there   |\n    |                                         | are no inputs attached.                           |\n    +-----------------------------------------+---------------------------------------------------+\n    | MA_NODE_FLAG_ALLOW_NULL_INPUT           | Used in conjunction with                          |\n    |                                         | `MA_NODE_FLAG_CONTINUOUS_PROCESSING`. When this   |\n    |                                         | is set, the `ppFramesIn` parameter of the         |\n    |                                         | processing callback will be set to NULL when      |\n    |                                         | there are no input frames are available. When     |\n    |                                         | this is unset, silence will be posted to the      |\n    |                                         | processing callback.                              |\n    +-----------------------------------------+---------------------------------------------------+\n    | MA_NODE_FLAG_DIFFERENT_PROCESSING_RATES | Used to tell miniaudio that input and output      |\n    |                                         | frames are processed at different rates. You      |\n    |                                         | should set this for any nodes that perform        |\n    |                                         | resampling.                                       |\n    +-----------------------------------------+---------------------------------------------------+\n    | MA_NODE_FLAG_SILENT_OUTPUT              | Used to tell miniaudio that a node produces only  |\n    |                                         | silent output. This is useful for nodes where you |\n    |                                         | don't want the output to contribute to the final  |\n    |                                         | mix. An example might be if you want split your   |\n    |                                         | stream and have one branch be output to a file.   |\n    |                                         | When using this flag, you should avoid writing to |\n    |                                         | the output buffer of the node's processing        |\n    |                                         | callback because miniaudio will ignore it anyway. |\n    +-----------------------------------------+---------------------------------------------------+\n\n\nIf you need to make a copy of an audio stream for effect processing you can use a splitter node\ncalled `ma_splitter_node`. This takes has 1 input bus and splits the stream into 2 output buses.\nYou can use it like this:\n\n    ```c\n    ma_splitter_node_config splitterNodeConfig = ma_splitter_node_config_init(channels);\n\n    ma_splitter_node splitterNode;\n    result = ma_splitter_node_init(&nodeGraph, &splitterNodeConfig, NULL, &splitterNode);\n    if (result != MA_SUCCESS) {\n        // Failed to create node.\n    }\n\n    // Attach your output buses to two different input buses (can be on two different nodes).\n    ma_node_attach_output_bus(&splitterNode, 0, ma_node_graph_get_endpoint(&nodeGraph), 0); // Attach directly to the endpoint.\n    ma_node_attach_output_bus(&splitterNode, 1, &myEffectNode,                          0); // Attach to input bus 0 of some effect node.\n    ```\n\nThe volume of an output bus can be configured on a per-bus basis:\n\n    ```c\n    ma_node_set_output_bus_volume(&splitterNode, 0, 0.5f);\n    ma_node_set_output_bus_volume(&splitterNode, 1, 0.5f);\n    ```\n\nIn the code above we're using the splitter node from before and changing the volume of each of the\ncopied streams.\n\nYou can start and stop a node with the following:\n\n    ```c\n    ma_node_set_state(&splitterNode, ma_node_state_started);    // The default state.\n    ma_node_set_state(&splitterNode, ma_node_state_stopped);\n    ```\n\nBy default the node is in a started state, but since it won't be connected to anything won't\nactually be invoked by the node graph until it's connected. When you stop a node, data will not be\nread from any of its input connections. You can use this property to stop a group of sounds\natomically.\n\nYou can configure the initial state of a node in it's config:\n\n    ```c\n    nodeConfig.initialState = ma_node_state_stopped;\n    ```\n\nNote that for the stock specialized nodes, all of their configs will have a `nodeConfig` member\nwhich is the config to use with the base node. This is where the initial state can be configured\nfor specialized nodes:\n\n    ```c\n    dataSourceNodeConfig.nodeConfig.initialState = ma_node_state_stopped;\n    ```\n\nWhen using a specialized node like `ma_data_source_node` or `ma_splitter_node`, be sure to not\nmodify the `vtable` member of the `nodeConfig` object.\n\n\n7.1. Timing\n-----------\nThe node graph supports starting and stopping nodes at scheduled times. This is especially useful\nfor data source nodes where you want to get the node set up, but only start playback at a specific\ntime. There are two clocks: local and global.\n\nA local clock is per-node, whereas the global clock is per graph. Scheduling starts and stops can\nonly be done based on the global clock because the local clock will not be running while the node\nis stopped. The global clocks advances whenever `ma_node_graph_read_pcm_frames()` is called. On the\nother hand, the local clock only advances when the node's processing callback is fired, and is\nadvanced based on the output frame count.\n\nTo retrieve the global time, use `ma_node_graph_get_time()`. The global time can be set with\n`ma_node_graph_set_time()` which might be useful if you want to do seeking on a global timeline.\nGetting and setting the local time is similar. Use `ma_node_get_time()` to retrieve the local time,\nand `ma_node_set_time()` to set the local time. The global and local times will be advanced by the\naudio thread, so care should be taken to avoid data races. Ideally you should avoid calling these\noutside of the node processing callbacks which are always run on the audio thread.\n\nThere is basic support for scheduling the starting and stopping of nodes. You can only schedule one\nstart and one stop at a time. This is mainly intended for putting nodes into a started or stopped\nstate in a frame-exact manner. Without this mechanism, starting and stopping of a node is limited\nto the resolution of a call to `ma_node_graph_read_pcm_frames()` which would typically be in blocks\nof several milliseconds. The following APIs can be used for scheduling node states:\n\n    ```c\n    ma_node_set_state_time()\n    ma_node_get_state_time()\n    ```\n\nThe time is absolute and must be based on the global clock. An example is below:\n\n    ```c\n    ma_node_set_state_time(&myNode, ma_node_state_started, sampleRate*1);   // Delay starting to 1 second.\n    ma_node_set_state_time(&myNode, ma_node_state_stopped, sampleRate*5);   // Delay stopping to 5 seconds.\n    ```\n\nAn example for changing the state using a relative time.\n\n    ```c\n    ma_node_set_state_time(&myNode, ma_node_state_started, sampleRate*1 + ma_node_graph_get_time(&myNodeGraph));\n    ma_node_set_state_time(&myNode, ma_node_state_stopped, sampleRate*5 + ma_node_graph_get_time(&myNodeGraph));\n    ```\n\nNote that due to the nature of multi-threading the times may not be 100% exact. If this is an\nissue, consider scheduling state changes from within a processing callback. An idea might be to\nhave some kind of passthrough trigger node that is used specifically for tracking time and handling\nevents.\n\n\n\n7.2. Thread Safety and Locking\n------------------------------\nWhen processing audio, it's ideal not to have any kind of locking in the audio thread. Since it's\nexpected that `ma_node_graph_read_pcm_frames()` would be run on the audio thread, it does so\nwithout the use of any locks. This section discusses the implementation used by miniaudio and goes\nover some of the compromises employed by miniaudio to achieve this goal. Note that the current\nimplementation may not be ideal - feedback and critiques are most welcome.\n\nThe node graph API is not *entirely* lock-free. Only `ma_node_graph_read_pcm_frames()` is expected\nto be lock-free. Attachment, detachment and uninitialization of nodes use locks to simplify the\nimplementation, but are crafted in a way such that such locking is not required when reading audio\ndata from the graph. Locking in these areas are achieved by means of spinlocks.\n\nThe main complication with keeping `ma_node_graph_read_pcm_frames()` lock-free stems from the fact\nthat a node can be uninitialized, and it's memory potentially freed, while in the middle of being\nprocessed on the audio thread. There are times when the audio thread will be referencing a node,\nwhich means the uninitialization process of a node needs to make sure it delays returning until the\naudio thread is finished so that control is not handed back to the caller thereby giving them a\nchance to free the node's memory.\n\nWhen the audio thread is processing a node, it does so by reading from each of the output buses of\nthe node. In order for a node to process data for one of its output buses, it needs to read from\neach of its input buses, and so on an so forth. It follows that once all output buses of a node\nare detached, the node as a whole will be disconnected and no further processing will occur unless\nit's output buses are reattached, which won't be happening when the node is being uninitialized.\nBy having `ma_node_detach_output_bus()` wait until the audio thread is finished with it, we can\nsimplify a few things, at the expense of making `ma_node_detach_output_bus()` a bit slower. By\ndoing this, the implementation of `ma_node_uninit()` becomes trivial - just detach all output\nnodes, followed by each of the attachments to each of its input nodes, and then do any final clean\nup.\n\nWith the above design, the worst-case scenario is `ma_node_detach_output_bus()` taking as long as\nit takes to process the output bus being detached. This will happen if it's called at just the\nwrong moment where the audio thread has just iterated it and has just started processing. The\ncaller of `ma_node_detach_output_bus()` will stall until the audio thread is finished, which\nincludes the cost of recursively processing its inputs. This is the biggest compromise made with\nthe approach taken by miniaudio for its lock-free processing system. The cost of detaching nodes\nearlier in the pipeline (data sources, for example) will be cheaper than the cost of detaching\nhigher level nodes, such as some kind of final post-processing endpoint. If you need to do mass\ndetachments, detach starting from the lowest level nodes and work your way towards the final\nendpoint node (but don't try detaching the node graph's endpoint). If the audio thread is not\nrunning, detachment will be fast and detachment in any order will be the same. The reason nodes\nneed to wait for their input attachments to complete is due to the potential for desyncs between\ndata sources. If the node was to terminate processing mid way through processing its inputs,\nthere's a chance that some of the underlying data sources will have been read, but then others not.\nThat will then result in a potential desynchronization when detaching and reattaching higher-level\nnodes. A possible solution to this is to have an option when detaching to terminate processing\nbefore processing all input attachments which should be fairly simple.\n\nAnother compromise, albeit less significant, is locking when attaching and detaching nodes. This\nlocking is achieved by means of a spinlock in order to reduce memory overhead. A lock is present\nfor each input bus and output bus. When an output bus is connected to an input bus, both the output\nbus and input bus is locked. This locking is specifically for attaching and detaching across\ndifferent threads and does not affect `ma_node_graph_read_pcm_frames()` in any way. The locking and\nunlocking is mostly self-explanatory, but a slightly less intuitive aspect comes into it when\nconsidering that iterating over attachments must not break as a result of attaching or detaching a\nnode while iteration is occurring.\n\nAttaching and detaching are both quite simple. When an output bus of a node is attached to an input\nbus of another node, it's added to a linked list. Basically, an input bus is a linked list, where\neach item in the list is and output bus. We have some intentional (and convenient) restrictions on\nwhat can done with the linked list in order to simplify the implementation. First of all, whenever\nsomething needs to iterate over the list, it must do so in a forward direction. Backwards iteration\nis not supported. Also, items can only be added to the start of the list.\n\nThe linked list is a doubly-linked list where each item in the list (an output bus) holds a pointer\nto the next item in the list, and another to the previous item. A pointer to the previous item is\nonly required for fast detachment of the node - it is never used in iteration. This is an\nimportant property because it means from the perspective of iteration, attaching and detaching of\nan item can be done with a single atomic assignment. This is exploited by both the attachment and\ndetachment process. When attaching the node, the first thing that is done is the setting of the\nlocal \"next\" and \"previous\" pointers of the node. After that, the item is \"attached\" to the list\nby simply performing an atomic exchange with the head pointer. After that, the node is \"attached\"\nto the list from the perspective of iteration. Even though the \"previous\" pointer of the next item\nhasn't yet been set, from the perspective of iteration it's been attached because iteration will\nonly be happening in a forward direction which means the \"previous\" pointer won't actually ever get\nused. The same general process applies to detachment. See `ma_node_attach_output_bus()` and\n`ma_node_detach_output_bus()` for the implementation of this mechanism.\n\n\n\n8. Decoding\n===========\nThe `ma_decoder` API is used for reading audio files. Decoders are completely decoupled from\ndevices and can be used independently. Built-in support is included for the following formats:\n\n    +---------+\n    | Format  |\n    +---------+\n    | WAV     |\n    | MP3     |\n    | FLAC    |\n    +---------+\n\nYou can disable the built-in decoders by specifying one or more of the following options before the\nminiaudio implementation:\n\n    ```c\n    #define MA_NO_WAV\n    #define MA_NO_MP3\n    #define MA_NO_FLAC\n    ```\n\nminiaudio supports the ability to plug in custom decoders. See the section below for details on how\nto use custom decoders.\n\nA decoder can be initialized from a file with `ma_decoder_init_file()`, a block of memory with\n`ma_decoder_init_memory()`, or from data delivered via callbacks with `ma_decoder_init()`. Here is\nan example for loading a decoder from a file:\n\n    ```c\n    ma_decoder decoder;\n    ma_result result = ma_decoder_init_file(\"MySong.mp3\", NULL, &decoder);\n    if (result != MA_SUCCESS) {\n        return false;   // An error occurred.\n    }\n\n    ...\n\n    ma_decoder_uninit(&decoder);\n    ```\n\nWhen initializing a decoder, you can optionally pass in a pointer to a `ma_decoder_config` object\n(the `NULL` argument in the example above) which allows you to configure the output format, channel\ncount, sample rate and channel map:\n\n    ```c\n    ma_decoder_config config = ma_decoder_config_init(ma_format_f32, 2, 48000);\n    ```\n\nWhen passing in `NULL` for decoder config in `ma_decoder_init*()`, the output format will be the\nsame as that defined by the decoding backend.\n\nData is read from the decoder as PCM frames. This will output the number of PCM frames actually\nread. If this is less than the requested number of PCM frames it means you've reached the end. The\nreturn value will be `MA_AT_END` if no samples have been read and the end has been reached.\n\n    ```c\n    ma_result result = ma_decoder_read_pcm_frames(pDecoder, pFrames, framesToRead, &framesRead);\n    if (framesRead < framesToRead) {\n        // Reached the end.\n    }\n    ```\n\nYou can also seek to a specific frame like so:\n\n    ```c\n    ma_result result = ma_decoder_seek_to_pcm_frame(pDecoder, targetFrame);\n    if (result != MA_SUCCESS) {\n        return false;   // An error occurred.\n    }\n    ```\n\nIf you want to loop back to the start, you can simply seek back to the first PCM frame:\n\n    ```c\n    ma_decoder_seek_to_pcm_frame(pDecoder, 0);\n    ```\n\nWhen loading a decoder, miniaudio uses a trial and error technique to find the appropriate decoding\nbackend. This can be unnecessarily inefficient if the type is already known. In this case you can\nuse `encodingFormat` variable in the device config to specify a specific encoding format you want\nto decode:\n\n    ```c\n    decoderConfig.encodingFormat = ma_encoding_format_wav;\n    ```\n\nSee the `ma_encoding_format` enum for possible encoding formats.\n\nThe `ma_decoder_init_file()` API will try using the file extension to determine which decoding\nbackend to prefer.\n\n\n8.1. Custom Decoders\n--------------------\nIt's possible to implement a custom decoder and plug it into miniaudio. This is extremely useful\nwhen you want to use the `ma_decoder` API, but need to support an encoding format that's not one of\nthe stock formats supported by miniaudio. This can be put to particularly good use when using the\n`ma_engine` and/or `ma_resource_manager` APIs because they use `ma_decoder` internally. If, for\nexample, you wanted to support Opus, you can do so with a custom decoder (there if a reference\nOpus decoder in the \"extras\" folder of the miniaudio repository which uses libopus + libopusfile).\n\nA custom decoder must implement a data source. A vtable called `ma_decoding_backend_vtable` needs\nto be implemented which is then passed into the decoder config:\n\n    ```c\n    ma_decoding_backend_vtable* pCustomBackendVTables[] =\n    {\n        &g_ma_decoding_backend_vtable_libvorbis,\n        &g_ma_decoding_backend_vtable_libopus\n    };\n\n    ...\n\n    decoderConfig = ma_decoder_config_init_default();\n    decoderConfig.pCustomBackendUserData = NULL;\n    decoderConfig.ppCustomBackendVTables = pCustomBackendVTables;\n    decoderConfig.customBackendCount     = sizeof(pCustomBackendVTables) / sizeof(pCustomBackendVTables[0]);\n    ```\n\nThe `ma_decoding_backend_vtable` vtable has the following functions:\n\n    ```\n    onInit\n    onInitFile\n    onInitFileW\n    onInitMemory\n    onUninit\n    ```\n\nThere are only two functions that must be implemented - `onInit` and `onUninit`. The other\nfunctions can be implemented for a small optimization for loading from a file path or memory. If\nthese are not specified, miniaudio will deal with it for you via a generic implementation.\n\nWhen you initialize a custom data source (by implementing the `onInit` function in the vtable) you\nwill need to output a pointer to a `ma_data_source` which implements your custom decoder. See the\nsection about data sources for details on how to implement this. Alternatively, see the\n\"custom_decoders\" example in the miniaudio repository.\n\nThe `onInit` function takes a pointer to some callbacks for the purpose of reading raw audio data\nfrom some arbitrary source. You'll use these functions to read from the raw data and perform the\ndecoding. When you call them, you will pass in the `pReadSeekTellUserData` pointer to the relevant\nparameter.\n\nThe `pConfig` parameter in `onInit` can be used to configure the backend if appropriate. It's only\nused as a hint and can be ignored. However, if any of the properties are relevant to your decoder,\nan optimal implementation will handle the relevant properties appropriately.\n\nIf memory allocation is required, it should be done so via the specified allocation callbacks if\npossible (the `pAllocationCallbacks` parameter).\n\nIf an error occurs when initializing the decoder, you should leave `ppBackend` unset, or set to\nNULL, and make sure everything is cleaned up appropriately and an appropriate result code returned.\nWhen multiple custom backends are specified, miniaudio will cycle through the vtables in the order\nthey're listed in the array that's passed into the decoder config so it's important that your\ninitialization routine is clean.\n\nWhen a decoder is uninitialized, the `onUninit` callback will be fired which will give you an\nopportunity to clean up and internal data.\n\n\n\n9. Encoding\n===========\nThe `ma_encoding` API is used for writing audio files. The only supported output format is WAV.\nThis can be disabled by specifying the following option before the implementation of miniaudio:\n\n    ```c\n    #define MA_NO_WAV\n    ```\n\nAn encoder can be initialized to write to a file with `ma_encoder_init_file()` or from data\ndelivered via callbacks with `ma_encoder_init()`. Below is an example for initializing an encoder\nto output to a file.\n\n    ```c\n    ma_encoder_config config = ma_encoder_config_init(ma_encoding_format_wav, FORMAT, CHANNELS, SAMPLE_RATE);\n    ma_encoder encoder;\n    ma_result result = ma_encoder_init_file(\"my_file.wav\", &config, &encoder);\n    if (result != MA_SUCCESS) {\n        // Error\n    }\n\n    ...\n\n    ma_encoder_uninit(&encoder);\n    ```\n\nWhen initializing an encoder you must specify a config which is initialized with\n`ma_encoder_config_init()`. Here you must specify the file type, the output sample format, output\nchannel count and output sample rate. The following file types are supported:\n\n    +------------------------+-------------+\n    | Enum                   | Description |\n    +------------------------+-------------+\n    | ma_encoding_format_wav | WAV         |\n    +------------------------+-------------+\n\nIf the format, channel count or sample rate is not supported by the output file type an error will\nbe returned. The encoder will not perform data conversion so you will need to convert it before\noutputting any audio data. To output audio data, use `ma_encoder_write_pcm_frames()`, like in the\nexample below:\n\n    ```c\n    ma_uint64 framesWritten;\n    result = ma_encoder_write_pcm_frames(&encoder, pPCMFramesToWrite, framesToWrite, &framesWritten);\n    if (result != MA_SUCCESS) {\n        ... handle error ...\n    }\n    ```\n\nThe `framesWritten` variable will contain the number of PCM frames that were actually written. This\nis optionally and you can pass in `NULL` if you need this.\n\nEncoders must be uninitialized with `ma_encoder_uninit()`.\n\n\n\n10. Data Conversion\n===================\nA data conversion API is included with miniaudio which supports the majority of data conversion\nrequirements. This supports conversion between sample formats, channel counts (with channel\nmapping) and sample rates.\n\n\n10.1. Sample Format Conversion\n------------------------------\nConversion between sample formats is achieved with the `ma_pcm_*_to_*()`, `ma_pcm_convert()` and\n`ma_convert_pcm_frames_format()` APIs. Use `ma_pcm_*_to_*()` to convert between two specific\nformats. Use `ma_pcm_convert()` to convert based on a `ma_format` variable. Use\n`ma_convert_pcm_frames_format()` to convert PCM frames where you want to specify the frame count\nand channel count as a variable instead of the total sample count.\n\n\n10.1.1. Dithering\n-----------------\nDithering can be set using the ditherMode parameter.\n\nThe different dithering modes include the following, in order of efficiency:\n\n    +-----------+--------------------------+\n    | Type      | Enum Token               |\n    +-----------+--------------------------+\n    | None      | ma_dither_mode_none      |\n    | Rectangle | ma_dither_mode_rectangle |\n    | Triangle  | ma_dither_mode_triangle  |\n    +-----------+--------------------------+\n\nNote that even if the dither mode is set to something other than `ma_dither_mode_none`, it will be\nignored for conversions where dithering is not needed. Dithering is available for the following\nconversions:\n\n    ```\n    s16 -> u8\n    s24 -> u8\n    s32 -> u8\n    f32 -> u8\n    s24 -> s16\n    s32 -> s16\n    f32 -> s16\n    ```\n\nNote that it is not an error to pass something other than ma_dither_mode_none for conversions where\ndither is not used. It will just be ignored.\n\n\n\n10.2. Channel Conversion\n------------------------\nChannel conversion is used for channel rearrangement and conversion from one channel count to\nanother. The `ma_channel_converter` API is used for channel conversion. Below is an example of\ninitializing a simple channel converter which converts from mono to stereo.\n\n    ```c\n    ma_channel_converter_config config = ma_channel_converter_config_init(\n        ma_format,                      // Sample format\n        1,                              // Input channels\n        NULL,                           // Input channel map\n        2,                              // Output channels\n        NULL,                           // Output channel map\n        ma_channel_mix_mode_default);   // The mixing algorithm to use when combining channels.\n\n    result = ma_channel_converter_init(&config, NULL, &converter);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n    ```\n\nTo perform the conversion simply call `ma_channel_converter_process_pcm_frames()` like so:\n\n    ```c\n    ma_result result = ma_channel_converter_process_pcm_frames(&converter, pFramesOut, pFramesIn, frameCount);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n    ```\n\nIt is up to the caller to ensure the output buffer is large enough to accommodate the new PCM\nframes.\n\nInput and output PCM frames are always interleaved. Deinterleaved layouts are not supported.\n\n\n10.2.1. Channel Mapping\n-----------------------\nIn addition to converting from one channel count to another, like the example above, the channel\nconverter can also be used to rearrange channels. When initializing the channel converter, you can\noptionally pass in channel maps for both the input and output frames. If the channel counts are the\nsame, and each channel map contains the same channel positions with the exception that they're in\na different order, a simple shuffling of the channels will be performed. If, however, there is not\na 1:1 mapping of channel positions, or the channel counts differ, the input channels will be mixed\nbased on a mixing mode which is specified when initializing the `ma_channel_converter_config`\nobject.\n\nWhen converting from mono to multi-channel, the mono channel is simply copied to each output\nchannel. When going the other way around, the audio of each output channel is simply averaged and\ncopied to the mono channel.\n\nIn more complicated cases blending is used. The `ma_channel_mix_mode_simple` mode will drop excess\nchannels and silence extra channels. For example, converting from 4 to 2 channels, the 3rd and 4th\nchannels will be dropped, whereas converting from 2 to 4 channels will put silence into the 3rd and\n4th channels.\n\nThe `ma_channel_mix_mode_rectangle` mode uses spacial locality based on a rectangle to compute a\nsimple distribution between input and output. Imagine sitting in the middle of a room, with\nspeakers on the walls representing channel positions. The `MA_CHANNEL_FRONT_LEFT` position can be\nthought of as being in the corner of the front and left walls.\n\nFinally, the `ma_channel_mix_mode_custom_weights` mode can be used to use custom user-defined\nweights. Custom weights can be passed in as the last parameter of\n`ma_channel_converter_config_init()`.\n\nPredefined channel maps can be retrieved with `ma_channel_map_init_standard()`. This takes a\n`ma_standard_channel_map` enum as its first parameter, which can be one of the following:\n\n    +-----------------------------------+-----------------------------------------------------------+\n    | Name                              | Description                                               |\n    +-----------------------------------+-----------------------------------------------------------+\n    | ma_standard_channel_map_default   | Default channel map used by miniaudio. See below.         |\n    | ma_standard_channel_map_microsoft | Channel map used by Microsoft's bitfield channel maps.    |\n    | ma_standard_channel_map_alsa      | Default ALSA channel map.                                 |\n    | ma_standard_channel_map_rfc3551   | RFC 3551. Based on AIFF.                                  |\n    | ma_standard_channel_map_flac      | FLAC channel map.                                         |\n    | ma_standard_channel_map_vorbis    | Vorbis channel map.                                       |\n    | ma_standard_channel_map_sound4    | FreeBSD's sound(4).                                       |\n    | ma_standard_channel_map_sndio     | sndio channel map. http://www.sndio.org/tips.html.        |\n    | ma_standard_channel_map_webaudio  | https://webaudio.github.io/web-audio-api/#ChannelOrdering |\n    +-----------------------------------+-----------------------------------------------------------+\n\nBelow are the channel maps used by default in miniaudio (`ma_standard_channel_map_default`):\n\n    +---------------+---------------------------------+\n    | Channel Count | Mapping                         |\n    +---------------+---------------------------------+\n    | 1 (Mono)      | 0: MA_CHANNEL_MONO              |\n    +---------------+---------------------------------+\n    | 2 (Stereo)    | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT       |\n    +---------------+---------------------------------+\n    | 3             | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER      |\n    +---------------+---------------------------------+\n    | 4 (Surround)  | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER <br> |\n    |               | 3: MA_CHANNEL_BACK_CENTER       |\n    +---------------+---------------------------------+\n    | 5             | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER <br> |\n    |               | 3: MA_CHANNEL_BACK_LEFT    <br> |\n    |               | 4: MA_CHANNEL_BACK_RIGHT        |\n    +---------------+---------------------------------+\n    | 6 (5.1)       | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER <br> |\n    |               | 3: MA_CHANNEL_LFE          <br> |\n    |               | 4: MA_CHANNEL_SIDE_LEFT    <br> |\n    |               | 5: MA_CHANNEL_SIDE_RIGHT        |\n    +---------------+---------------------------------+\n    | 7             | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER <br> |\n    |               | 3: MA_CHANNEL_LFE          <br> |\n    |               | 4: MA_CHANNEL_BACK_CENTER  <br> |\n    |               | 4: MA_CHANNEL_SIDE_LEFT    <br> |\n    |               | 5: MA_CHANNEL_SIDE_RIGHT        |\n    +---------------+---------------------------------+\n    | 8 (7.1)       | 0: MA_CHANNEL_FRONT_LEFT   <br> |\n    |               | 1: MA_CHANNEL_FRONT_RIGHT  <br> |\n    |               | 2: MA_CHANNEL_FRONT_CENTER <br> |\n    |               | 3: MA_CHANNEL_LFE          <br> |\n    |               | 4: MA_CHANNEL_BACK_LEFT    <br> |\n    |               | 5: MA_CHANNEL_BACK_RIGHT   <br> |\n    |               | 6: MA_CHANNEL_SIDE_LEFT    <br> |\n    |               | 7: MA_CHANNEL_SIDE_RIGHT        |\n    +---------------+---------------------------------+\n    | Other         | All channels set to 0. This     |\n    |               | is equivalent to the same       |\n    |               | mapping as the device.          |\n    +---------------+---------------------------------+\n\n\n\n10.3. Resampling\n----------------\nResampling is achieved with the `ma_resampler` object. To create a resampler object, do something\nlike the following:\n\n    ```c\n    ma_resampler_config config = ma_resampler_config_init(\n        ma_format_s16,\n        channels,\n        sampleRateIn,\n        sampleRateOut,\n        ma_resample_algorithm_linear);\n\n    ma_resampler resampler;\n    ma_result result = ma_resampler_init(&config, NULL, &resampler);\n    if (result != MA_SUCCESS) {\n        // An error occurred...\n    }\n    ```\n\nDo the following to uninitialize the resampler:\n\n    ```c\n    ma_resampler_uninit(&resampler);\n    ```\n\nThe following example shows how data can be processed\n\n    ```c\n    ma_uint64 frameCountIn  = 1000;\n    ma_uint64 frameCountOut = 2000;\n    ma_result result = ma_resampler_process_pcm_frames(&resampler, pFramesIn, &frameCountIn, pFramesOut, &frameCountOut);\n    if (result != MA_SUCCESS) {\n        // An error occurred...\n    }\n\n    // At this point, frameCountIn contains the number of input frames that were consumed and frameCountOut contains the\n    // number of output frames written.\n    ```\n\nTo initialize the resampler you first need to set up a config (`ma_resampler_config`) with\n`ma_resampler_config_init()`. You need to specify the sample format you want to use, the number of\nchannels, the input and output sample rate, and the algorithm.\n\nThe sample format can be either `ma_format_s16` or `ma_format_f32`. If you need a different format\nyou will need to perform pre- and post-conversions yourself where necessary. Note that the format\nis the same for both input and output. The format cannot be changed after initialization.\n\nThe resampler supports multiple channels and is always interleaved (both input and output). The\nchannel count cannot be changed after initialization.\n\nThe sample rates can be anything other than zero, and are always specified in hertz. They should be\nset to something like 44100, etc. The sample rate is the only configuration property that can be\nchanged after initialization.\n\nThe miniaudio resampler has built-in support for the following algorithms:\n\n    +-----------+------------------------------+\n    | Algorithm | Enum Token                   |\n    +-----------+------------------------------+\n    | Linear    | ma_resample_algorithm_linear |\n    | Custom    | ma_resample_algorithm_custom |\n    +-----------+------------------------------+\n\nThe algorithm cannot be changed after initialization.\n\nProcessing always happens on a per PCM frame basis and always assumes interleaved input and output.\nDe-interleaved processing is not supported. To process frames, use\n`ma_resampler_process_pcm_frames()`. On input, this function takes the number of output frames you\ncan fit in the output buffer and the number of input frames contained in the input buffer. On\noutput these variables contain the number of output frames that were written to the output buffer\nand the number of input frames that were consumed in the process. You can pass in NULL for the\ninput buffer in which case it will be treated as an infinitely large buffer of zeros. The output\nbuffer can also be NULL, in which case the processing will be treated as seek.\n\nThe sample rate can be changed dynamically on the fly. You can change this with explicit sample\nrates with `ma_resampler_set_rate()` and also with a decimal ratio with\n`ma_resampler_set_rate_ratio()`. The ratio is in/out.\n\nSometimes it's useful to know exactly how many input frames will be required to output a specific\nnumber of frames. You can calculate this with `ma_resampler_get_required_input_frame_count()`.\nLikewise, it's sometimes useful to know exactly how many frames would be output given a certain\nnumber of input frames. You can do this with `ma_resampler_get_expected_output_frame_count()`.\n\nDue to the nature of how resampling works, the resampler introduces some latency. This can be\nretrieved in terms of both the input rate and the output rate with\n`ma_resampler_get_input_latency()` and `ma_resampler_get_output_latency()`.\n\n\n10.3.1. Resampling Algorithms\n-----------------------------\nThe choice of resampling algorithm depends on your situation and requirements.\n\n\n10.3.1.1. Linear Resampling\n---------------------------\nThe linear resampler is the fastest, but comes at the expense of poorer quality. There is, however,\nsome control over the quality of the linear resampler which may make it a suitable option depending\non your requirements.\n\nThe linear resampler performs low-pass filtering before or after downsampling or upsampling,\ndepending on the sample rates you're converting between. When decreasing the sample rate, the\nlow-pass filter will be applied before downsampling. When increasing the rate it will be performed\nafter upsampling. By default a fourth order low-pass filter will be applied. This can be configured\nvia the `lpfOrder` configuration variable. Setting this to 0 will disable filtering.\n\nThe low-pass filter has a cutoff frequency which defaults to half the sample rate of the lowest of\nthe input and output sample rates (Nyquist Frequency).\n\nThe API for the linear resampler is the same as the main resampler API, only it's called\n`ma_linear_resampler`.\n\n\n10.3.2. Custom Resamplers\n-------------------------\nYou can implement a custom resampler by using the `ma_resample_algorithm_custom` resampling\nalgorithm and setting a vtable in the resampler config:\n\n    ```c\n    ma_resampler_config config = ma_resampler_config_init(..., ma_resample_algorithm_custom);\n    config.pBackendVTable = &g_customResamplerVTable;\n    ```\n\nCustom resamplers are useful if the stock algorithms are not appropriate for your use case. You\nneed to implement the required functions in `ma_resampling_backend_vtable`. Note that not all\nfunctions in the vtable need to be implemented, but if it's possible to implement, they should be.\n\nYou can use the `ma_linear_resampler` object for an example on how to implement the vtable. The\n`onGetHeapSize` callback is used to calculate the size of any internal heap allocation the custom\nresampler will need to make given the supplied config. When you initialize the resampler via the\n`onInit` callback, you'll be given a pointer to a heap allocation which is where you should store\nthe heap allocated data. You should not free this data in `onUninit` because miniaudio will manage\nit for you.\n\nThe `onProcess` callback is where the actual resampling takes place. On input, `pFrameCountIn`\npoints to a variable containing the number of frames in the `pFramesIn` buffer and\n`pFrameCountOut` points to a variable containing the capacity in frames of the `pFramesOut` buffer.\nOn output, `pFrameCountIn` should be set to the number of input frames that were fully consumed,\nwhereas `pFrameCountOut` should be set to the number of frames that were written to `pFramesOut`.\n\nThe `onSetRate` callback is optional and is used for dynamically changing the sample rate. If\ndynamic rate changes are not supported, you can set this callback to NULL.\n\nThe `onGetInputLatency` and `onGetOutputLatency` functions are used for retrieving the latency in\ninput and output rates respectively. These can be NULL in which case latency calculations will be\nassumed to be NULL.\n\nThe `onGetRequiredInputFrameCount` callback is used to give miniaudio a hint as to how many input\nframes are required to be available to produce the given number of output frames. Likewise, the\n`onGetExpectedOutputFrameCount` callback is used to determine how many output frames will be\nproduced given the specified number of input frames. miniaudio will use these as a hint, but they\nare optional and can be set to NULL if you're unable to implement them.\n\n\n\n10.4. General Data Conversion\n-----------------------------\nThe `ma_data_converter` API can be used to wrap sample format conversion, channel conversion and\nresampling into one operation. This is what miniaudio uses internally to convert between the format\nrequested when the device was initialized and the format of the backend's native device. The API\nfor general data conversion is very similar to the resampling API. Create a `ma_data_converter`\nobject like this:\n\n    ```c\n    ma_data_converter_config config = ma_data_converter_config_init(\n        inputFormat,\n        outputFormat,\n        inputChannels,\n        outputChannels,\n        inputSampleRate,\n        outputSampleRate\n    );\n\n    ma_data_converter converter;\n    ma_result result = ma_data_converter_init(&config, NULL, &converter);\n    if (result != MA_SUCCESS) {\n        // An error occurred...\n    }\n    ```\n\nIn the example above we use `ma_data_converter_config_init()` to initialize the config, however\nthere's many more properties that can be configured, such as channel maps and resampling quality.\nSomething like the following may be more suitable depending on your requirements:\n\n    ```c\n    ma_data_converter_config config = ma_data_converter_config_init_default();\n    config.formatIn = inputFormat;\n    config.formatOut = outputFormat;\n    config.channelsIn = inputChannels;\n    config.channelsOut = outputChannels;\n    config.sampleRateIn = inputSampleRate;\n    config.sampleRateOut = outputSampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_flac, config.channelMapIn, sizeof(config.channelMapIn)/sizeof(config.channelMapIn[0]), config.channelCountIn);\n    config.resampling.linear.lpfOrder = MA_MAX_FILTER_ORDER;\n    ```\n\nDo the following to uninitialize the data converter:\n\n    ```c\n    ma_data_converter_uninit(&converter, NULL);\n    ```\n\nThe following example shows how data can be processed\n\n    ```c\n    ma_uint64 frameCountIn  = 1000;\n    ma_uint64 frameCountOut = 2000;\n    ma_result result = ma_data_converter_process_pcm_frames(&converter, pFramesIn, &frameCountIn, pFramesOut, &frameCountOut);\n    if (result != MA_SUCCESS) {\n        // An error occurred...\n    }\n\n    // At this point, frameCountIn contains the number of input frames that were consumed and frameCountOut contains the number\n    // of output frames written.\n    ```\n\nThe data converter supports multiple channels and is always interleaved (both input and output).\nThe channel count cannot be changed after initialization.\n\nSample rates can be anything other than zero, and are always specified in hertz. They should be set\nto something like 44100, etc. The sample rate is the only configuration property that can be\nchanged after initialization, but only if the `resampling.allowDynamicSampleRate` member of\n`ma_data_converter_config` is set to `MA_TRUE`. To change the sample rate, use\n`ma_data_converter_set_rate()` or `ma_data_converter_set_rate_ratio()`. The ratio must be in/out.\nThe resampling algorithm cannot be changed after initialization.\n\nProcessing always happens on a per PCM frame basis and always assumes interleaved input and output.\nDe-interleaved processing is not supported. To process frames, use\n`ma_data_converter_process_pcm_frames()`. On input, this function takes the number of output frames\nyou can fit in the output buffer and the number of input frames contained in the input buffer. On\noutput these variables contain the number of output frames that were written to the output buffer\nand the number of input frames that were consumed in the process. You can pass in NULL for the\ninput buffer in which case it will be treated as an infinitely large\nbuffer of zeros. The output buffer can also be NULL, in which case the processing will be treated\nas seek.\n\nSometimes it's useful to know exactly how many input frames will be required to output a specific\nnumber of frames. You can calculate this with `ma_data_converter_get_required_input_frame_count()`.\nLikewise, it's sometimes useful to know exactly how many frames would be output given a certain\nnumber of input frames. You can do this with `ma_data_converter_get_expected_output_frame_count()`.\n\nDue to the nature of how resampling works, the data converter introduces some latency if resampling\nis required. This can be retrieved in terms of both the input rate and the output rate with\n`ma_data_converter_get_input_latency()` and `ma_data_converter_get_output_latency()`.\n\n\n\n11. Filtering\n=============\n\n11.1. Biquad Filtering\n----------------------\nBiquad filtering is achieved with the `ma_biquad` API. Example:\n\n    ```c\n    ma_biquad_config config = ma_biquad_config_init(ma_format_f32, channels, b0, b1, b2, a0, a1, a2);\n    ma_result result = ma_biquad_init(&config, NULL, &biquad);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    ...\n\n    ma_biquad_process_pcm_frames(&biquad, pFramesOut, pFramesIn, frameCount);\n    ```\n\nBiquad filtering is implemented using transposed direct form 2. The numerator coefficients are b0,\nb1 and b2, and the denominator coefficients are a0, a1 and a2. The a0 coefficient is required and\ncoefficients must not be pre-normalized.\n\nSupported formats are `ma_format_s16` and `ma_format_f32`. If you need to use a different format\nyou need to convert it yourself beforehand. When using `ma_format_s16` the biquad filter will use\nfixed point arithmetic. When using `ma_format_f32`, floating point arithmetic will be used.\n\nInput and output frames are always interleaved.\n\nFiltering can be applied in-place by passing in the same pointer for both the input and output\nbuffers, like so:\n\n    ```c\n    ma_biquad_process_pcm_frames(&biquad, pMyData, pMyData, frameCount);\n    ```\n\nIf you need to change the values of the coefficients, but maintain the values in the registers you\ncan do so with `ma_biquad_reinit()`. This is useful if you need to change the properties of the\nfilter while keeping the values of registers valid to avoid glitching. Do not use\n`ma_biquad_init()` for this as it will do a full initialization which involves clearing the\nregisters to 0. Note that changing the format or channel count after initialization is invalid and\nwill result in an error.\n\n\n11.2. Low-Pass Filtering\n------------------------\nLow-pass filtering is achieved with the following APIs:\n\n    +---------+------------------------------------------+\n    | API     | Description                              |\n    +---------+------------------------------------------+\n    | ma_lpf1 | First order low-pass filter              |\n    | ma_lpf2 | Second order low-pass filter             |\n    | ma_lpf  | High order low-pass filter (Butterworth) |\n    +---------+------------------------------------------+\n\nLow-pass filter example:\n\n    ```c\n    ma_lpf_config config = ma_lpf_config_init(ma_format_f32, channels, sampleRate, cutoffFrequency, order);\n    ma_result result = ma_lpf_init(&config, &lpf);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    ...\n\n    ma_lpf_process_pcm_frames(&lpf, pFramesOut, pFramesIn, frameCount);\n    ```\n\nSupported formats are `ma_format_s16` and` ma_format_f32`. If you need to use a different format\nyou need to convert it yourself beforehand. Input and output frames are always interleaved.\n\nFiltering can be applied in-place by passing in the same pointer for both the input and output\nbuffers, like so:\n\n    ```c\n    ma_lpf_process_pcm_frames(&lpf, pMyData, pMyData, frameCount);\n    ```\n\nThe maximum filter order is limited to `MA_MAX_FILTER_ORDER` which is set to 8. If you need more,\nyou can chain first and second order filters together.\n\n    ```c\n    for (iFilter = 0; iFilter < filterCount; iFilter += 1) {\n        ma_lpf2_process_pcm_frames(&lpf2[iFilter], pMyData, pMyData, frameCount);\n    }\n    ```\n\nIf you need to change the configuration of the filter, but need to maintain the state of internal\nregisters you can do so with `ma_lpf_reinit()`. This may be useful if you need to change the sample\nrate and/or cutoff frequency dynamically while maintaining smooth transitions. Note that changing the\nformat or channel count after initialization is invalid and will result in an error.\n\nThe `ma_lpf` object supports a configurable order, but if you only need a first order filter you\nmay want to consider using `ma_lpf1`. Likewise, if you only need a second order filter you can use\n`ma_lpf2`. The advantage of this is that they're lighter weight and a bit more efficient.\n\nIf an even filter order is specified, a series of second order filters will be processed in a\nchain. If an odd filter order is specified, a first order filter will be applied, followed by a\nseries of second order filters in a chain.\n\n\n11.3. High-Pass Filtering\n-------------------------\nHigh-pass filtering is achieved with the following APIs:\n\n    +---------+-------------------------------------------+\n    | API     | Description                               |\n    +---------+-------------------------------------------+\n    | ma_hpf1 | First order high-pass filter              |\n    | ma_hpf2 | Second order high-pass filter             |\n    | ma_hpf  | High order high-pass filter (Butterworth) |\n    +---------+-------------------------------------------+\n\nHigh-pass filters work exactly the same as low-pass filters, only the APIs are called `ma_hpf1`,\n`ma_hpf2` and `ma_hpf`. See example code for low-pass filters for example usage.\n\n\n11.4. Band-Pass Filtering\n-------------------------\nBand-pass filtering is achieved with the following APIs:\n\n    +---------+-------------------------------+\n    | API     | Description                   |\n    +---------+-------------------------------+\n    | ma_bpf2 | Second order band-pass filter |\n    | ma_bpf  | High order band-pass filter   |\n    +---------+-------------------------------+\n\nBand-pass filters work exactly the same as low-pass filters, only the APIs are called `ma_bpf2` and\n`ma_hpf`. See example code for low-pass filters for example usage. Note that the order for\nband-pass filters must be an even number which means there is no first order band-pass filter,\nunlike low-pass and high-pass filters.\n\n\n11.5. Notch Filtering\n---------------------\nNotch filtering is achieved with the following APIs:\n\n    +-----------+------------------------------------------+\n    | API       | Description                              |\n    +-----------+------------------------------------------+\n    | ma_notch2 | Second order notching filter             |\n    +-----------+------------------------------------------+\n\n\n11.6. Peaking EQ Filtering\n-------------------------\nPeaking filtering is achieved with the following APIs:\n\n    +----------+------------------------------------------+\n    | API      | Description                              |\n    +----------+------------------------------------------+\n    | ma_peak2 | Second order peaking filter              |\n    +----------+------------------------------------------+\n\n\n11.7. Low Shelf Filtering\n-------------------------\nLow shelf filtering is achieved with the following APIs:\n\n    +-------------+------------------------------------------+\n    | API         | Description                              |\n    +-------------+------------------------------------------+\n    | ma_loshelf2 | Second order low shelf filter            |\n    +-------------+------------------------------------------+\n\nWhere a high-pass filter is used to eliminate lower frequencies, a low shelf filter can be used to\njust turn them down rather than eliminate them entirely.\n\n\n11.8. High Shelf Filtering\n--------------------------\nHigh shelf filtering is achieved with the following APIs:\n\n    +-------------+------------------------------------------+\n    | API         | Description                              |\n    +-------------+------------------------------------------+\n    | ma_hishelf2 | Second order high shelf filter           |\n    +-------------+------------------------------------------+\n\nThe high shelf filter has the same API as the low shelf filter, only you would use `ma_hishelf`\ninstead of `ma_loshelf`. Where a low shelf filter is used to adjust the volume of low frequencies,\nthe high shelf filter does the same thing for high frequencies.\n\n\n\n\n12. Waveform and Noise Generation\n=================================\n\n12.1. Waveforms\n---------------\nminiaudio supports generation of sine, square, triangle and sawtooth waveforms. This is achieved\nwith the `ma_waveform` API. Example:\n\n    ```c\n    ma_waveform_config config = ma_waveform_config_init(\n        FORMAT,\n        CHANNELS,\n        SAMPLE_RATE,\n        ma_waveform_type_sine,\n        amplitude,\n        frequency);\n\n    ma_waveform waveform;\n    ma_result result = ma_waveform_init(&config, &waveform);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    ...\n\n    ma_waveform_read_pcm_frames(&waveform, pOutput, frameCount);\n    ```\n\nThe amplitude, frequency, type, and sample rate can be changed dynamically with\n`ma_waveform_set_amplitude()`, `ma_waveform_set_frequency()`, `ma_waveform_set_type()`, and\n`ma_waveform_set_sample_rate()` respectively.\n\nYou can invert the waveform by setting the amplitude to a negative value. You can use this to\ncontrol whether or not a sawtooth has a positive or negative ramp, for example.\n\nBelow are the supported waveform types:\n\n    +---------------------------+\n    | Enum Name                 |\n    +---------------------------+\n    | ma_waveform_type_sine     |\n    | ma_waveform_type_square   |\n    | ma_waveform_type_triangle |\n    | ma_waveform_type_sawtooth |\n    +---------------------------+\n\n\n\n12.2. Noise\n-----------\nminiaudio supports generation of white, pink and Brownian noise via the `ma_noise` API. Example:\n\n    ```c\n    ma_noise_config config = ma_noise_config_init(\n        FORMAT,\n        CHANNELS,\n        ma_noise_type_white,\n        SEED,\n        amplitude);\n\n    ma_noise noise;\n    ma_result result = ma_noise_init(&config, &noise);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    ...\n\n    ma_noise_read_pcm_frames(&noise, pOutput, frameCount);\n    ```\n\nThe noise API uses simple LCG random number generation. It supports a custom seed which is useful\nfor things like automated testing requiring reproducibility. Setting the seed to zero will default\nto `MA_DEFAULT_LCG_SEED`.\n\nThe amplitude and seed can be changed dynamically with `ma_noise_set_amplitude()` and\n`ma_noise_set_seed()` respectively.\n\nBy default, the noise API will use different values for different channels. So, for example, the\nleft side in a stereo stream will be different to the right side. To instead have each channel use\nthe same random value, set the `duplicateChannels` member of the noise config to true, like so:\n\n    ```c\n    config.duplicateChannels = MA_TRUE;\n    ```\n\nBelow are the supported noise types.\n\n    +------------------------+\n    | Enum Name              |\n    +------------------------+\n    | ma_noise_type_white    |\n    | ma_noise_type_pink     |\n    | ma_noise_type_brownian |\n    +------------------------+\n\n\n\n13. Audio Buffers\n=================\nminiaudio supports reading from a buffer of raw audio data via the `ma_audio_buffer` API. This can\nread from memory that's managed by the application, but can also handle the memory management for\nyou internally. Memory management is flexible and should support most use cases.\n\nAudio buffers are initialized using the standard configuration system used everywhere in miniaudio:\n\n    ```c\n    ma_audio_buffer_config config = ma_audio_buffer_config_init(\n        format,\n        channels,\n        sizeInFrames,\n        pExistingData,\n        &allocationCallbacks);\n\n    ma_audio_buffer buffer;\n    result = ma_audio_buffer_init(&config, &buffer);\n    if (result != MA_SUCCESS) {\n        // Error.\n    }\n\n    ...\n\n    ma_audio_buffer_uninit(&buffer);\n    ```\n\nIn the example above, the memory pointed to by `pExistingData` will *not* be copied and is how an\napplication can do self-managed memory allocation. If you would rather make a copy of the data, use\n`ma_audio_buffer_init_copy()`. To uninitialize the buffer, use `ma_audio_buffer_uninit()`.\n\nSometimes it can be convenient to allocate the memory for the `ma_audio_buffer` structure and the\nraw audio data in a contiguous block of memory. That is, the raw audio data will be located\nimmediately after the `ma_audio_buffer` structure. To do this, use\n`ma_audio_buffer_alloc_and_init()`:\n\n    ```c\n    ma_audio_buffer_config config = ma_audio_buffer_config_init(\n        format,\n        channels,\n        sizeInFrames,\n        pExistingData,\n        &allocationCallbacks);\n\n    ma_audio_buffer* pBuffer\n    result = ma_audio_buffer_alloc_and_init(&config, &pBuffer);\n    if (result != MA_SUCCESS) {\n        // Error\n    }\n\n    ...\n\n    ma_audio_buffer_uninit_and_free(&buffer);\n    ```\n\nIf you initialize the buffer with `ma_audio_buffer_alloc_and_init()` you should uninitialize it\nwith `ma_audio_buffer_uninit_and_free()`. In the example above, the memory pointed to by\n`pExistingData` will be copied into the buffer, which is contrary to the behavior of\n`ma_audio_buffer_init()`.\n\nAn audio buffer has a playback cursor just like a decoder. As you read frames from the buffer, the\ncursor moves forward. The last parameter (`loop`) can be used to determine if the buffer should\nloop. The return value is the number of frames actually read. If this is less than the number of\nframes requested it means the end has been reached. This should never happen if the `loop`\nparameter is set to true. If you want to manually loop back to the start, you can do so with with\n`ma_audio_buffer_seek_to_pcm_frame(pAudioBuffer, 0)`. Below is an example for reading data from an\naudio buffer.\n\n    ```c\n    ma_uint64 framesRead = ma_audio_buffer_read_pcm_frames(pAudioBuffer, pFramesOut, desiredFrameCount, isLooping);\n    if (framesRead < desiredFrameCount) {\n        // If not looping, this means the end has been reached. This should never happen in looping mode with valid input.\n    }\n    ```\n\nSometimes you may want to avoid the cost of data movement between the internal buffer and the\noutput buffer. Instead you can use memory mapping to retrieve a pointer to a segment of data:\n\n    ```c\n    void* pMappedFrames;\n    ma_uint64 frameCount = frameCountToTryMapping;\n    ma_result result = ma_audio_buffer_map(pAudioBuffer, &pMappedFrames, &frameCount);\n    if (result == MA_SUCCESS) {\n        // Map was successful. The value in frameCount will be how many frames were _actually_ mapped, which may be\n        // less due to the end of the buffer being reached.\n        ma_copy_pcm_frames(pFramesOut, pMappedFrames, frameCount, pAudioBuffer->format, pAudioBuffer->channels);\n\n        // You must unmap the buffer.\n        ma_audio_buffer_unmap(pAudioBuffer, frameCount);\n    }\n    ```\n\nWhen you use memory mapping, the read cursor is increment by the frame count passed in to\n`ma_audio_buffer_unmap()`. If you decide not to process every frame you can pass in a value smaller\nthan the value returned by `ma_audio_buffer_map()`. The disadvantage to using memory mapping is\nthat it does not handle looping for you. You can determine if the buffer is at the end for the\npurpose of looping with `ma_audio_buffer_at_end()` or by inspecting the return value of\n`ma_audio_buffer_unmap()` and checking if it equals `MA_AT_END`. You should not treat `MA_AT_END`\nas an error when returned by `ma_audio_buffer_unmap()`.\n\n\n\n14. Ring Buffers\n================\nminiaudio supports lock free (single producer, single consumer) ring buffers which are exposed via\nthe `ma_rb` and `ma_pcm_rb` APIs. The `ma_rb` API operates on bytes, whereas the `ma_pcm_rb`\noperates on PCM frames. They are otherwise identical as `ma_pcm_rb` is just a wrapper around\n`ma_rb`.\n\nUnlike most other APIs in miniaudio, ring buffers support both interleaved and deinterleaved\nstreams. The caller can also allocate their own backing memory for the ring buffer to use\ninternally for added flexibility. Otherwise the ring buffer will manage it's internal memory for\nyou.\n\nThe examples below use the PCM frame variant of the ring buffer since that's most likely the one\nyou will want to use. To initialize a ring buffer, do something like the following:\n\n    ```c\n    ma_pcm_rb rb;\n    ma_result result = ma_pcm_rb_init(FORMAT, CHANNELS, BUFFER_SIZE_IN_FRAMES, NULL, NULL, &rb);\n    if (result != MA_SUCCESS) {\n        // Error\n    }\n    ```\n\nThe `ma_pcm_rb_init()` function takes the sample format and channel count as parameters because\nit's the PCM variant of the ring buffer API. For the regular ring buffer that operates on bytes you\nwould call `ma_rb_init()` which leaves these out and just takes the size of the buffer in bytes\ninstead of frames. The fourth parameter is an optional pre-allocated buffer and the fifth parameter\nis a pointer to a `ma_allocation_callbacks` structure for custom memory allocation routines.\nPassing in `NULL` for this results in `MA_MALLOC()` and `MA_FREE()` being used.\n\nUse `ma_pcm_rb_init_ex()` if you need a deinterleaved buffer. The data for each sub-buffer is\noffset from each other based on the stride. To manage your sub-buffers you can use\n`ma_pcm_rb_get_subbuffer_stride()`, `ma_pcm_rb_get_subbuffer_offset()` and\n`ma_pcm_rb_get_subbuffer_ptr()`.\n\nUse `ma_pcm_rb_acquire_read()` and `ma_pcm_rb_acquire_write()` to retrieve a pointer to a section\nof the ring buffer. You specify the number of frames you need, and on output it will set to what\nwas actually acquired. If the read or write pointer is positioned such that the number of frames\nrequested will require a loop, it will be clamped to the end of the buffer. Therefore, the number\nof frames you're given may be less than the number you requested.\n\nAfter calling `ma_pcm_rb_acquire_read()` or `ma_pcm_rb_acquire_write()`, you do your work on the\nbuffer and then \"commit\" it with `ma_pcm_rb_commit_read()` or `ma_pcm_rb_commit_write()`. This is\nwhere the read/write pointers are updated. When you commit you need to pass in the buffer that was\nreturned by the earlier call to `ma_pcm_rb_acquire_read()` or `ma_pcm_rb_acquire_write()` and is\nonly used for validation. The number of frames passed to `ma_pcm_rb_commit_read()` and\n`ma_pcm_rb_commit_write()` is what's used to increment the pointers, and can be less that what was\noriginally requested.\n\nIf you want to correct for drift between the write pointer and the read pointer you can use a\ncombination of `ma_pcm_rb_pointer_distance()`, `ma_pcm_rb_seek_read()` and\n`ma_pcm_rb_seek_write()`. Note that you can only move the pointers forward, and you should only\nmove the read pointer forward via the consumer thread, and the write pointer forward by the\nproducer thread. If there is too much space between the pointers, move the read pointer forward. If\nthere is too little space between the pointers, move the write pointer forward.\n\nYou can use a ring buffer at the byte level instead of the PCM frame level by using the `ma_rb`\nAPI. This is exactly the same, only you will use the `ma_rb` functions instead of `ma_pcm_rb` and\ninstead of frame counts you will pass around byte counts.\n\nThe maximum size of the buffer in bytes is `0x7FFFFFFF-(MA_SIMD_ALIGNMENT-1)` due to the most\nsignificant bit being used to encode a loop flag and the internally managed buffers always being\naligned to `MA_SIMD_ALIGNMENT`.\n\nNote that the ring buffer is only thread safe when used by a single consumer thread and single\nproducer thread.\n\n\n\n15. Backends\n============\nThe following backends are supported by miniaudio. These are listed in order of default priority.\nWhen no backend is specified when initializing a context or device, miniaudio will attempt to use\neach of these backends in the order listed in the table below.\n\nNote that backends that are not usable by the build target will not be included in the build. For\nexample, ALSA, which is specific to Linux, will not be included in the Windows build.\n\n    +-------------+-----------------------+--------------------------------------------------------+\n    | Name        | Enum Name             | Supported Operating Systems                            |\n    +-------------+-----------------------+--------------------------------------------------------+\n    | WASAPI      | ma_backend_wasapi     | Windows Vista+                                         |\n    | DirectSound | ma_backend_dsound     | Windows XP+                                            |\n    | WinMM       | ma_backend_winmm      | Windows 95+                                            |\n    | Core Audio  | ma_backend_coreaudio  | macOS, iOS                                             |\n    | sndio       | ma_backend_sndio      | OpenBSD                                                |\n    | audio(4)    | ma_backend_audio4     | NetBSD, OpenBSD                                        |\n    | OSS         | ma_backend_oss        | FreeBSD                                                |\n    | PulseAudio  | ma_backend_pulseaudio | Cross Platform (disabled on Windows, BSD and Android)  |\n    | ALSA        | ma_backend_alsa       | Linux                                                  |\n    | JACK        | ma_backend_jack       | Cross Platform (disabled on BSD and Android)           |\n    | AAudio      | ma_backend_aaudio     | Android 8+                                             |\n    | OpenSL ES   | ma_backend_opensl     | Android (API level 16+)                                |\n    | Web Audio   | ma_backend_webaudio   | Web (via Emscripten)                                   |\n    | Custom      | ma_backend_custom     | Cross Platform                                         |\n    | Null        | ma_backend_null       | Cross Platform (not used on Web)                       |\n    +-------------+-----------------------+--------------------------------------------------------+\n\nSome backends have some nuance details you may want to be aware of.\n\n15.1. WASAPI\n------------\n- Low-latency shared mode will be disabled when using an application-defined sample rate which is\n  different to the device's native sample rate. To work around this, set `wasapi.noAutoConvertSRC`\n  to true in the device config. This is due to IAudioClient3_InitializeSharedAudioStream() failing\n  when the `AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM` flag is specified. Setting wasapi.noAutoConvertSRC\n  will result in miniaudio's internal resampler being used instead which will in turn enable the\n  use of low-latency shared mode.\n\n15.2. PulseAudio\n----------------\n- If you experience bad glitching/noise on Arch Linux, consider this fix from the Arch wiki:\n  https://wiki.archlinux.org/index.php/PulseAudio/Troubleshooting#Glitches,_skips_or_crackling.\n  Alternatively, consider using a different backend such as ALSA.\n\n15.3. Android\n-------------\n- To capture audio on Android, remember to add the RECORD_AUDIO permission to your manifest:\n  `<uses-permission android:name=\"android.permission.RECORD_AUDIO\" />`\n- With OpenSL|ES, only a single ma_context can be active at any given time. This is due to a\n  limitation with OpenSL|ES.\n- With AAudio, only default devices are enumerated. This is due to AAudio not having an enumeration\n  API (devices are enumerated through Java). You can however perform your own device enumeration\n  through Java and then set the ID in the ma_device_id structure (ma_device_id.aaudio) and pass it\n  to ma_device_init().\n- The backend API will perform resampling where possible. The reason for this as opposed to using\n  miniaudio's built-in resampler is to take advantage of any potential device-specific\n  optimizations the driver may implement.\n\nBSD\n---\n- The sndio backend is currently only enabled on OpenBSD builds.\n- The audio(4) backend is supported on OpenBSD, but you may need to disable sndiod before you can\n  use it.\n\n15.4. UWP\n---------\n- UWP only supports default playback and capture devices.\n- UWP requires the Microphone capability to be enabled in the application's manifest (Package.appxmanifest):\n\n    ```\n    <Package ...>\n        ...\n        <Capabilities>\n            <DeviceCapability Name=\"microphone\" />\n        </Capabilities>\n    </Package>\n    ```\n\n15.5. Web Audio / Emscripten\n----------------------------\n- You cannot use `-std=c*` compiler flags, nor `-ansi`. This only applies to the Emscripten build.\n- The first time a context is initialized it will create a global object called \"miniaudio\" whose\n  primary purpose is to act as a factory for device objects.\n- Currently the Web Audio backend uses ScriptProcessorNode's, but this may need to change later as\n  they've been deprecated.\n- Google has implemented a policy in their browsers that prevent automatic media output without\n  first receiving some kind of user input. The following web page has additional details:\n  https://developers.google.com/web/updates/2017/09/autoplay-policy-changes. Starting the device\n  may fail if you try to start playback without first handling some kind of user input.\n\n\n\n16. Optimization Tips\n=====================\nSee below for some tips on improving performance.\n\n16.1. Low Level API\n-------------------\n- In the data callback, if your data is already clipped prior to copying it into the output buffer,\n  set the `noClip` config option in the device config to true. This will disable miniaudio's built\n  in clipping function.\n- By default, miniaudio will pre-silence the data callback's output buffer. If you know that you\n  will always write valid data to the output buffer you can disable pre-silencing by setting the\n  `noPreSilence` config option in the device config to true.\n\n16.2. High Level API\n--------------------\n- If a sound does not require doppler or pitch shifting, consider disabling pitching by\n  initializing the sound with the `MA_SOUND_FLAG_NO_PITCH` flag.\n- If a sound does not require spatialization, disable it by initializing the sound with the\n  `MA_SOUND_FLAG_NO_SPATIALIZATION` flag. It can be re-enabled again post-initialization with\n  `ma_sound_set_spatialization_enabled()`.\n- If you know all of your sounds will always be the same sample rate, set the engine's sample\n  rate to match that of the sounds. Likewise, if you're using a self-managed resource manager,\n  consider setting the decoded sample rate to match your sounds. By configuring everything to\n  use a consistent sample rate, sample rate conversion can be avoided.\n\n\n\n17. Miscellaneous Notes\n=======================\n- Automatic stream routing is enabled on a per-backend basis. Support is explicitly enabled for\n  WASAPI and Core Audio, however other backends such as PulseAudio may naturally support it, though\n  not all have been tested.\n- When compiling with VC6 and earlier, decoding is restricted to files less than 2GB in size. This\n  is due to 64-bit file APIs not being available.\n*/\n\n#ifndef miniaudio_h\n#define miniaudio_h\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define MA_STRINGIFY(x)     #x\n#define MA_XSTRINGIFY(x)    MA_STRINGIFY(x)\n\n#define MA_VERSION_MAJOR    0\n#define MA_VERSION_MINOR    11\n#define MA_VERSION_REVISION 24\n#define MA_VERSION_STRING   MA_XSTRINGIFY(MA_VERSION_MAJOR) \".\" MA_XSTRINGIFY(MA_VERSION_MINOR) \".\" MA_XSTRINGIFY(MA_VERSION_REVISION)\n\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(push)\n    #pragma warning(disable:4201)   /* nonstandard extension used: nameless struct/union */\n    #pragma warning(disable:4214)   /* nonstandard extension used: bit field types other than int */\n    #pragma warning(disable:4324)   /* structure was padded due to alignment specifier */\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wpedantic\" /* For ISO C99 doesn't support unnamed structs/unions [-Wpedantic] */\n    #if defined(__clang__)\n        #pragma GCC diagnostic ignored \"-Wc11-extensions\"   /* anonymous unions are a C11 extension */\n    #endif\n#endif\n\n\n#if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__)) || defined(_M_X64) || defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(_M_ARM64) || defined(__powerpc64__) || defined(__ppc64__)\n    #define MA_SIZEOF_PTR   8\n#else\n    #define MA_SIZEOF_PTR   4\n#endif\n\n#include <stddef.h> /* For size_t. */\n\n/* Sized types. */\n#if defined(MA_USE_STDINT)\n    #include <stdint.h>\n    typedef int8_t   ma_int8;\n    typedef uint8_t  ma_uint8;\n    typedef int16_t  ma_int16;\n    typedef uint16_t ma_uint16;\n    typedef int32_t  ma_int32;\n    typedef uint32_t ma_uint32;\n    typedef int64_t  ma_int64;\n    typedef uint64_t ma_uint64;\n#else\n    typedef   signed char           ma_int8;\n    typedef unsigned char           ma_uint8;\n    typedef   signed short          ma_int16;\n    typedef unsigned short          ma_uint16;\n    typedef   signed int            ma_int32;\n    typedef unsigned int            ma_uint32;\n    #if defined(_MSC_VER) && !defined(__clang__)\n        typedef   signed __int64    ma_int64;\n        typedef unsigned __int64    ma_uint64;\n    #else\n        #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n            #pragma GCC diagnostic push\n            #pragma GCC diagnostic ignored \"-Wlong-long\"\n            #if defined(__clang__)\n                #pragma GCC diagnostic ignored \"-Wc++11-long-long\"\n            #endif\n        #endif\n        typedef   signed long long  ma_int64;\n        typedef unsigned long long  ma_uint64;\n        #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n            #pragma GCC diagnostic pop\n        #endif\n    #endif\n#endif  /* MA_USE_STDINT */\n\n#if MA_SIZEOF_PTR == 8\n    typedef ma_uint64           ma_uintptr;\n#else\n    typedef ma_uint32           ma_uintptr;\n#endif\n\ntypedef ma_uint8    ma_bool8;\ntypedef ma_uint32   ma_bool32;\n#define MA_TRUE     1\n#define MA_FALSE    0\n\n/* These float types are not used universally by miniaudio. It's to simplify some macro expansion for atomic types. */\ntypedef float       ma_float;\ntypedef double      ma_double;\n\ntypedef void* ma_handle;\ntypedef void* ma_ptr;\n\n/*\nma_proc is annoying because when compiling with GCC we get pedantic warnings about converting\nbetween `void*` and `void (*)()`. We can't use `void (*)()` with MSVC however, because we'll get\nwarning C4191 about \"type cast between incompatible function types\". To work around this I'm going\nto use a different data type depending on the compiler.\n*/\n#if defined(__GNUC__)\ntypedef void (*ma_proc)(void);\n#else\ntypedef void* ma_proc;\n#endif\n\n#if defined(_MSC_VER) && !defined(_WCHAR_T_DEFINED)\ntypedef ma_uint16 wchar_t;\n#endif\n\n/* Define NULL for some compilers. */\n#ifndef NULL\n#define NULL 0\n#endif\n\n#if defined(SIZE_MAX)\n    #define MA_SIZE_MAX    SIZE_MAX\n#else\n    #define MA_SIZE_MAX    0xFFFFFFFF  /* When SIZE_MAX is not defined by the standard library just default to the maximum 32-bit unsigned integer. */\n#endif\n\n#define MA_UINT64_MAX      (((ma_uint64)0xFFFFFFFF << 32) | (ma_uint64)0xFFFFFFFF)   /* Weird shifting syntax is for VC6 compatibility. */\n\n\n/* Platform/backend detection. */\n#if defined(_WIN32)\n    #define MA_WIN32\n    #if defined(MA_FORCE_UWP) || (defined(WINAPI_FAMILY) && ((defined(WINAPI_FAMILY_PC_APP) && WINAPI_FAMILY == WINAPI_FAMILY_PC_APP) || (defined(WINAPI_FAMILY_PHONE_APP) && WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP)))\n        #define MA_WIN32_UWP\n    #elif defined(WINAPI_FAMILY) && (defined(WINAPI_FAMILY_GAMES) && WINAPI_FAMILY == WINAPI_FAMILY_GAMES)\n        #define MA_WIN32_GDK\n    #elif defined(NXDK)\n        #define MA_WIN32_NXDK\n    #else\n        #define MA_WIN32_DESKTOP\n    #endif\n\n    /* The original Xbox. */\n    #if defined(NXDK)   /* <-- Add other Xbox compiler toolchains here, and then add a toolchain-specific define in case we need to discriminate between them later. */\n        #define MA_XBOX\n\n        #if defined(NXDK)\n            #define MA_XBOX_NXDK\n        #endif\n    #endif\n#endif\n#if defined(__MSDOS__) || defined(MSDOS) || defined(_MSDOS) || defined(__DOS__)\n    #define MA_DOS\n\n    /* No threading allowed on DOS. */\n    #ifndef MA_NO_THREADING\n    #define MA_NO_THREADING\n    #endif\n\n    /* No runtime linking allowed on DOS. */\n    #ifndef MA_NO_RUNTIME_LINKING\n    #define MA_NO_RUNTIME_LINKING\n    #endif\n#endif\n#if !defined(MA_WIN32) && !defined(MA_DOS)    /* If it's not Win32, assume POSIX. */\n    #define MA_POSIX\n\n    #if !defined(MA_NO_THREADING)\n        /*\n        Use the MA_NO_PTHREAD_IN_HEADER option at your own risk. This is intentionally undocumented.\n        You can use this to avoid including pthread.h in the header section. The downside is that it\n        results in some fixed sized structures being declared for the various types that are used in\n        miniaudio. The risk here is that these types might be too small for a given platform. This\n        risk is yours to take and no support will be offered if you enable this option.\n        */\n        #ifndef MA_NO_PTHREAD_IN_HEADER\n            #include <pthread.h>    /* Unfortunate #include, but needed for pthread_t, pthread_mutex_t and pthread_cond_t types. */\n            typedef pthread_t       ma_pthread_t;\n            typedef pthread_mutex_t ma_pthread_mutex_t;\n            typedef pthread_cond_t  ma_pthread_cond_t;\n        #else\n            typedef ma_uintptr      ma_pthread_t;\n            typedef union           ma_pthread_mutex_t { char __data[40]; ma_uint64 __alignment; } ma_pthread_mutex_t;\n            typedef union           ma_pthread_cond_t  { char __data[48]; ma_uint64 __alignment; } ma_pthread_cond_t;\n        #endif\n    #endif\n\n    #if defined(__unix__)\n        #define MA_UNIX\n    #endif\n    #if defined(__linux__)\n        #define MA_LINUX\n    #endif\n    #if defined(__APPLE__)\n        #define MA_APPLE\n    #endif\n    #if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)\n        #define MA_BSD\n    #endif\n    #if defined(__ANDROID__)\n        #define MA_ANDROID\n    #endif\n    #if defined(__EMSCRIPTEN__)\n        #define MA_EMSCRIPTEN\n    #endif\n    #if defined(__ORBIS__)\n        #define MA_ORBIS\n    #endif\n    #if defined(__PROSPERO__)\n        #define MA_PROSPERO\n    #endif\n    #if defined(__3DS__)\n        #define MA_3DS\n    #endif\n    #if defined(__SWITCH__) || defined(__NX__)\n        #define MA_SWITCH\n    #endif\n    #if defined(__BEOS__) || defined(__HAIKU__)\n        #define MA_BEOS\n    #endif\n    #if defined(__HAIKU__)\n        #define MA_HAIKU\n    #endif\n#endif\n\n#if !defined(MA_FALLTHROUGH) && defined(__cplusplus) && __cplusplus >= 201703L\n    #define MA_FALLTHROUGH [[fallthrough]]\n#endif\n#if !defined(MA_FALLTHROUGH) && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000L\n    #define MA_FALLTHROUGH [[fallthrough]]\n#endif\n#if !defined(MA_FALLTHROUGH) && defined(__has_attribute)\n    #if __has_attribute(fallthrough)\n        #define MA_FALLTHROUGH __attribute__((fallthrough))\n    #endif\n#endif\n#if !defined(MA_FALLTHROUGH)\n    #define MA_FALLTHROUGH ((void)0)\n#endif\n\n#ifdef _MSC_VER\n    #define MA_INLINE __forceinline\n\n    /* noinline was introduced in Visual Studio 2005. */\n    #if _MSC_VER >= 1400\n        #define MA_NO_INLINE __declspec(noinline)\n    #else\n        #define MA_NO_INLINE\n    #endif\n#elif defined(__GNUC__)\n    /*\n    I've had a bug report where GCC is emitting warnings about functions possibly not being inlineable. This warning happens when\n    the __attribute__((always_inline)) attribute is defined without an \"inline\" statement. I think therefore there must be some\n    case where \"__inline__\" is not always defined, thus the compiler emitting these warnings. When using -std=c89 or -ansi on the\n    command line, we cannot use the \"inline\" keyword and instead need to use \"__inline__\". In an attempt to work around this issue\n    I am using \"__inline__\" only when we're compiling in strict ANSI mode.\n    */\n    #if defined(__STRICT_ANSI__)\n        #define MA_GNUC_INLINE_HINT __inline__\n    #else\n        #define MA_GNUC_INLINE_HINT inline\n    #endif\n\n    #if (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 2)) || defined(__clang__)\n        #define MA_INLINE MA_GNUC_INLINE_HINT __attribute__((always_inline))\n        #define MA_NO_INLINE __attribute__((noinline))\n    #else\n        #define MA_INLINE MA_GNUC_INLINE_HINT\n        #define MA_NO_INLINE\n    #endif\n#elif defined(__WATCOMC__)\n    #define MA_INLINE __inline\n    #define MA_NO_INLINE\n#else\n    #define MA_INLINE\n    #define MA_NO_INLINE\n#endif\n\n/* MA_DLL is not officially supported. You're on your own if you want to use this. */\n#if defined(MA_DLL)\n    #if defined(_WIN32)\n        #define MA_DLL_IMPORT  __declspec(dllimport)\n        #define MA_DLL_EXPORT  __declspec(dllexport)\n        #define MA_DLL_PRIVATE static\n    #else\n        #if defined(__GNUC__) && __GNUC__ >= 4\n            #define MA_DLL_IMPORT  __attribute__((visibility(\"default\")))\n            #define MA_DLL_EXPORT  __attribute__((visibility(\"default\")))\n            #define MA_DLL_PRIVATE __attribute__((visibility(\"hidden\")))\n        #else\n            #define MA_DLL_IMPORT\n            #define MA_DLL_EXPORT\n            #define MA_DLL_PRIVATE static\n        #endif\n    #endif\n#endif\n\n#if !defined(MA_API)\n    #if defined(MA_DLL)\n        #if defined(MINIAUDIO_IMPLEMENTATION) || defined(MA_IMPLEMENTATION)\n            #define MA_API  MA_DLL_EXPORT\n        #else\n            #define MA_API  MA_DLL_IMPORT\n        #endif\n    #else\n        #define MA_API extern\n    #endif\n#endif\n\n#if !defined(MA_STATIC)\n    #if defined(MA_DLL)\n        #define MA_PRIVATE MA_DLL_PRIVATE\n    #else\n        #define MA_PRIVATE static\n    #endif\n#endif\n\n\n/* SIMD alignment in bytes. Currently set to 32 bytes in preparation for future AVX optimizations. */\n#define MA_SIMD_ALIGNMENT  32\n\n/*\nSpecial wchar_t type to ensure any structures in the public sections that reference it have a\nconsistent size across all platforms.\n\nOn Windows, wchar_t is 2 bytes, whereas everywhere else it's 4 bytes. Since Windows likes to use\nwchar_t for its IDs, we need a special explicitly sized wchar type that is always 2 bytes on all\nplatforms.\n*/\n#if !defined(MA_POSIX) && defined(MA_WIN32)\ntypedef wchar_t     ma_wchar_win32;\n#else\ntypedef ma_uint16   ma_wchar_win32;\n#endif\n\n\n\n/*\nLogging Levels\n==============\nLog levels are only used to give logging callbacks some context as to the severity of a log message\nso they can do filtering. All log levels will be posted to registered logging callbacks. If you\ndon't want to output a certain log level you can discriminate against the log level in the callback.\n\nMA_LOG_LEVEL_DEBUG\n    Used for debugging. Useful for debug and test builds, but should be disabled in release builds.\n\nMA_LOG_LEVEL_INFO\n    Informational logging. Useful for debugging. This will never be called from within the data\n    callback.\n\nMA_LOG_LEVEL_WARNING\n    Warnings. You should enable this in you development builds and action them when encountered. These\n    logs usually indicate a potential problem or misconfiguration, but still allow you to keep\n    running. This will never be called from within the data callback.\n\nMA_LOG_LEVEL_ERROR\n    Error logging. This will be fired when an operation fails and is subsequently aborted. This can\n    be fired from within the data callback, in which case the device will be stopped. You should\n    always have this log level enabled.\n*/\ntypedef enum\n{\n    MA_LOG_LEVEL_DEBUG   = 4,\n    MA_LOG_LEVEL_INFO    = 3,\n    MA_LOG_LEVEL_WARNING = 2,\n    MA_LOG_LEVEL_ERROR   = 1\n} ma_log_level;\n\n/*\nVariables needing to be accessed atomically should be declared with this macro for two reasons:\n\n    1) It allows people who read the code to identify a variable as such; and\n    2) It forces alignment on platforms where it's required or optimal.\n\nNote that for x86/64, alignment is not strictly necessary, but does have some performance\nimplications. Where supported by the compiler, alignment will be used, but otherwise if the CPU\narchitecture does not require it, it will simply leave it unaligned. This is the case with old\nversions of Visual Studio, which I've confirmed with at least VC6.\n*/\n#if !defined(_MSC_VER) && defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)\n    #include <stdalign.h>\n    #define MA_ATOMIC(alignment, type)            _Alignas(alignment) type\n#else\n    #if defined(__GNUC__)\n        /* GCC-style compilers. */\n        #define MA_ATOMIC(alignment, type)        type __attribute__((aligned(alignment)))\n    #elif defined(_MSC_VER) && _MSC_VER > 1200  /* 1200 = VC6. Alignment not supported, but not necessary because x86 is the only supported target. */\n        /* MSVC. */\n        #define MA_ATOMIC(alignment, type)        __declspec(align(alignment)) type\n    #else\n        /* Other compilers. */\n        #define MA_ATOMIC(alignment, type)        type\n    #endif\n#endif\n\ntypedef struct ma_context ma_context;\ntypedef struct ma_device ma_device;\n\ntypedef ma_uint8 ma_channel;\ntypedef enum\n{\n    MA_CHANNEL_NONE               = 0,\n    MA_CHANNEL_MONO               = 1,\n    MA_CHANNEL_FRONT_LEFT         = 2,\n    MA_CHANNEL_FRONT_RIGHT        = 3,\n    MA_CHANNEL_FRONT_CENTER       = 4,\n    MA_CHANNEL_LFE                = 5,\n    MA_CHANNEL_BACK_LEFT          = 6,\n    MA_CHANNEL_BACK_RIGHT         = 7,\n    MA_CHANNEL_FRONT_LEFT_CENTER  = 8,\n    MA_CHANNEL_FRONT_RIGHT_CENTER = 9,\n    MA_CHANNEL_BACK_CENTER        = 10,\n    MA_CHANNEL_SIDE_LEFT          = 11,\n    MA_CHANNEL_SIDE_RIGHT         = 12,\n    MA_CHANNEL_TOP_CENTER         = 13,\n    MA_CHANNEL_TOP_FRONT_LEFT     = 14,\n    MA_CHANNEL_TOP_FRONT_CENTER   = 15,\n    MA_CHANNEL_TOP_FRONT_RIGHT    = 16,\n    MA_CHANNEL_TOP_BACK_LEFT      = 17,\n    MA_CHANNEL_TOP_BACK_CENTER    = 18,\n    MA_CHANNEL_TOP_BACK_RIGHT     = 19,\n    MA_CHANNEL_AUX_0              = 20,\n    MA_CHANNEL_AUX_1              = 21,\n    MA_CHANNEL_AUX_2              = 22,\n    MA_CHANNEL_AUX_3              = 23,\n    MA_CHANNEL_AUX_4              = 24,\n    MA_CHANNEL_AUX_5              = 25,\n    MA_CHANNEL_AUX_6              = 26,\n    MA_CHANNEL_AUX_7              = 27,\n    MA_CHANNEL_AUX_8              = 28,\n    MA_CHANNEL_AUX_9              = 29,\n    MA_CHANNEL_AUX_10             = 30,\n    MA_CHANNEL_AUX_11             = 31,\n    MA_CHANNEL_AUX_12             = 32,\n    MA_CHANNEL_AUX_13             = 33,\n    MA_CHANNEL_AUX_14             = 34,\n    MA_CHANNEL_AUX_15             = 35,\n    MA_CHANNEL_AUX_16             = 36,\n    MA_CHANNEL_AUX_17             = 37,\n    MA_CHANNEL_AUX_18             = 38,\n    MA_CHANNEL_AUX_19             = 39,\n    MA_CHANNEL_AUX_20             = 40,\n    MA_CHANNEL_AUX_21             = 41,\n    MA_CHANNEL_AUX_22             = 42,\n    MA_CHANNEL_AUX_23             = 43,\n    MA_CHANNEL_AUX_24             = 44,\n    MA_CHANNEL_AUX_25             = 45,\n    MA_CHANNEL_AUX_26             = 46,\n    MA_CHANNEL_AUX_27             = 47,\n    MA_CHANNEL_AUX_28             = 48,\n    MA_CHANNEL_AUX_29             = 49,\n    MA_CHANNEL_AUX_30             = 50,\n    MA_CHANNEL_AUX_31             = 51,\n\n    /* Count. */\n    MA_CHANNEL_POSITION_COUNT,\n\n    /* Aliases. */\n    MA_CHANNEL_LEFT               = MA_CHANNEL_FRONT_LEFT,\n    MA_CHANNEL_RIGHT              = MA_CHANNEL_FRONT_RIGHT,\n} _ma_channel_position; /* Do not use `_ma_channel_position` directly. Use `ma_channel` instead. */\n\ntypedef enum\n{\n    MA_SUCCESS                        =  0,\n    MA_ERROR                          = -1,  /* A generic error. */\n    MA_INVALID_ARGS                   = -2,\n    MA_INVALID_OPERATION              = -3,\n    MA_OUT_OF_MEMORY                  = -4,\n    MA_OUT_OF_RANGE                   = -5,\n    MA_ACCESS_DENIED                  = -6,\n    MA_DOES_NOT_EXIST                 = -7,\n    MA_ALREADY_EXISTS                 = -8,\n    MA_TOO_MANY_OPEN_FILES            = -9,\n    MA_INVALID_FILE                   = -10,\n    MA_TOO_BIG                        = -11,\n    MA_PATH_TOO_LONG                  = -12,\n    MA_NAME_TOO_LONG                  = -13,\n    MA_NOT_DIRECTORY                  = -14,\n    MA_IS_DIRECTORY                   = -15,\n    MA_DIRECTORY_NOT_EMPTY            = -16,\n    MA_AT_END                         = -17,\n    MA_NO_SPACE                       = -18,\n    MA_BUSY                           = -19,\n    MA_IO_ERROR                       = -20,\n    MA_INTERRUPT                      = -21,\n    MA_UNAVAILABLE                    = -22,\n    MA_ALREADY_IN_USE                 = -23,\n    MA_BAD_ADDRESS                    = -24,\n    MA_BAD_SEEK                       = -25,\n    MA_BAD_PIPE                       = -26,\n    MA_DEADLOCK                       = -27,\n    MA_TOO_MANY_LINKS                 = -28,\n    MA_NOT_IMPLEMENTED                = -29,\n    MA_NO_MESSAGE                     = -30,\n    MA_BAD_MESSAGE                    = -31,\n    MA_NO_DATA_AVAILABLE              = -32,\n    MA_INVALID_DATA                   = -33,\n    MA_TIMEOUT                        = -34,\n    MA_NO_NETWORK                     = -35,\n    MA_NOT_UNIQUE                     = -36,\n    MA_NOT_SOCKET                     = -37,\n    MA_NO_ADDRESS                     = -38,\n    MA_BAD_PROTOCOL                   = -39,\n    MA_PROTOCOL_UNAVAILABLE           = -40,\n    MA_PROTOCOL_NOT_SUPPORTED         = -41,\n    MA_PROTOCOL_FAMILY_NOT_SUPPORTED  = -42,\n    MA_ADDRESS_FAMILY_NOT_SUPPORTED   = -43,\n    MA_SOCKET_NOT_SUPPORTED           = -44,\n    MA_CONNECTION_RESET               = -45,\n    MA_ALREADY_CONNECTED              = -46,\n    MA_NOT_CONNECTED                  = -47,\n    MA_CONNECTION_REFUSED             = -48,\n    MA_NO_HOST                        = -49,\n    MA_IN_PROGRESS                    = -50,\n    MA_CANCELLED                      = -51,\n    MA_MEMORY_ALREADY_MAPPED          = -52,\n\n    /* General non-standard errors. */\n    MA_CRC_MISMATCH                   = -100,\n\n    /* General miniaudio-specific errors. */\n    MA_FORMAT_NOT_SUPPORTED           = -200,\n    MA_DEVICE_TYPE_NOT_SUPPORTED      = -201,\n    MA_SHARE_MODE_NOT_SUPPORTED       = -202,\n    MA_NO_BACKEND                     = -203,\n    MA_NO_DEVICE                      = -204,\n    MA_API_NOT_FOUND                  = -205,\n    MA_INVALID_DEVICE_CONFIG          = -206,\n    MA_LOOP                           = -207,\n    MA_BACKEND_NOT_ENABLED            = -208,\n\n    /* State errors. */\n    MA_DEVICE_NOT_INITIALIZED         = -300,\n    MA_DEVICE_ALREADY_INITIALIZED     = -301,\n    MA_DEVICE_NOT_STARTED             = -302,\n    MA_DEVICE_NOT_STOPPED             = -303,\n\n    /* Operation errors. */\n    MA_FAILED_TO_INIT_BACKEND         = -400,\n    MA_FAILED_TO_OPEN_BACKEND_DEVICE  = -401,\n    MA_FAILED_TO_START_BACKEND_DEVICE = -402,\n    MA_FAILED_TO_STOP_BACKEND_DEVICE  = -403\n} ma_result;\n\n\n#define MA_MIN_CHANNELS                 1\n#ifndef MA_MAX_CHANNELS\n#define MA_MAX_CHANNELS                 254\n#endif\n\n#ifndef MA_MAX_FILTER_ORDER\n#define MA_MAX_FILTER_ORDER             8\n#endif\n\ntypedef enum\n{\n    ma_stream_format_pcm = 0\n} ma_stream_format;\n\ntypedef enum\n{\n    ma_stream_layout_interleaved = 0,\n    ma_stream_layout_deinterleaved\n} ma_stream_layout;\n\ntypedef enum\n{\n    ma_dither_mode_none = 0,\n    ma_dither_mode_rectangle,\n    ma_dither_mode_triangle\n} ma_dither_mode;\n\ntypedef enum\n{\n    /*\n    I like to keep these explicitly defined because they're used as a key into a lookup table. When items are\n    added to this, make sure there are no gaps and that they're added to the lookup table in ma_get_bytes_per_sample().\n    */\n    ma_format_unknown = 0,     /* Mainly used for indicating an error, but also used as the default for the output format for decoders. */\n    ma_format_u8      = 1,\n    ma_format_s16     = 2,     /* Seems to be the most widely supported format. */\n    ma_format_s24     = 3,     /* Tightly packed. 3 bytes per sample. */\n    ma_format_s32     = 4,\n    ma_format_f32     = 5,\n    ma_format_count\n} ma_format;\n\ntypedef enum\n{\n    /* Standard rates need to be in priority order. */\n    ma_standard_sample_rate_48000  = 48000,     /* Most common */\n    ma_standard_sample_rate_44100  = 44100,\n\n    ma_standard_sample_rate_32000  = 32000,     /* Lows */\n    ma_standard_sample_rate_24000  = 24000,\n    ma_standard_sample_rate_22050  = 22050,\n\n    ma_standard_sample_rate_88200  = 88200,     /* Highs */\n    ma_standard_sample_rate_96000  = 96000,\n    ma_standard_sample_rate_176400 = 176400,\n    ma_standard_sample_rate_192000 = 192000,\n\n    ma_standard_sample_rate_16000  = 16000,     /* Extreme lows */\n    ma_standard_sample_rate_11025  = 11025,\n    ma_standard_sample_rate_8000   = 8000,\n\n    ma_standard_sample_rate_352800 = 352800,    /* Extreme highs */\n    ma_standard_sample_rate_384000 = 384000,\n\n    ma_standard_sample_rate_min    = ma_standard_sample_rate_8000,\n    ma_standard_sample_rate_max    = ma_standard_sample_rate_384000,\n    ma_standard_sample_rate_count  = 14         /* Need to maintain the count manually. Make sure this is updated if items are added to enum. */\n} ma_standard_sample_rate;\n\n\ntypedef enum\n{\n    ma_channel_mix_mode_rectangular = 0,   /* Simple averaging based on the plane(s) the channel is sitting on. */\n    ma_channel_mix_mode_simple,            /* Drop excess channels; zeroed out extra channels. */\n    ma_channel_mix_mode_custom_weights,    /* Use custom weights specified in ma_channel_converter_config. */\n    ma_channel_mix_mode_default = ma_channel_mix_mode_rectangular\n} ma_channel_mix_mode;\n\ntypedef enum\n{\n    ma_standard_channel_map_microsoft,\n    ma_standard_channel_map_alsa,\n    ma_standard_channel_map_rfc3551,   /* Based off AIFF. */\n    ma_standard_channel_map_flac,\n    ma_standard_channel_map_vorbis,\n    ma_standard_channel_map_sound4,    /* FreeBSD's sound(4). */\n    ma_standard_channel_map_sndio,     /* www.sndio.org/tips.html */\n    ma_standard_channel_map_webaudio = ma_standard_channel_map_flac, /* https://webaudio.github.io/web-audio-api/#ChannelOrdering. Only 1, 2, 4 and 6 channels are defined, but can fill in the gaps with logical assumptions. */\n    ma_standard_channel_map_default = ma_standard_channel_map_microsoft\n} ma_standard_channel_map;\n\ntypedef enum\n{\n    ma_performance_profile_low_latency = 0,\n    ma_performance_profile_conservative\n} ma_performance_profile;\n\n\ntypedef struct\n{\n    void* pUserData;\n    void* (* onMalloc)(size_t sz, void* pUserData);\n    void* (* onRealloc)(void* p, size_t sz, void* pUserData);\n    void  (* onFree)(void* p, void* pUserData);\n} ma_allocation_callbacks;\n\ntypedef struct\n{\n    ma_uint32 state;\n} ma_lcg;\n\n\n/*\nAtomics.\n\nThese are typesafe structures to prevent errors as a result of forgetting to reference variables atomically. It's too\neasy to introduce subtle bugs where you accidentally do a regular assignment instead of an atomic load/store, etc. By\nusing a struct we can enforce the use of atomics at compile time.\n\nThese types are declared in the header section because we need to reference them in structs below, but functions for\nusing them are only exposed in the implementation section. I do not want these to be part of the public API.\n\nThere's a few downsides to this system. The first is that you need to declare a new struct for each type. Below are\nsome macros to help with the declarations. They will be named like so:\n\n    ma_atomic_uint32 - atomic ma_uint32\n    ma_atomic_int32  - atomic ma_int32\n    ma_atomic_uint64 - atomic ma_uint64\n    ma_atomic_float  - atomic float\n    ma_atomic_bool32 - atomic ma_bool32\n\nThe other downside is that atomic pointers are extremely messy. You need to declare a new struct for each specific\ntype of pointer you need to make atomic. For example, an atomic ma_node* will look like this:\n\n    MA_ATOMIC_SAFE_TYPE_IMPL_PTR(node)\n\nWhich will declare a type struct that's named like so:\n\n    ma_atomic_ptr_node\n\nFunctions to use the atomic types are declared in the implementation section. All atomic functions are prefixed with\nthe name of the struct. For example:\n\n    ma_atomic_uint32_set() - Atomic store of ma_uint32\n    ma_atomic_uint32_get() - Atomic load of ma_uint32\n    etc.\n\nFor pointer types it's the same, which makes them a bit messy to use due to the length of each function name, but in\nreturn you get type safety and enforcement of atomic operations.\n*/\n#define MA_ATOMIC_SAFE_TYPE_DECL(c89TypeExtension, typeSize, type) \\\n    typedef struct \\\n    { \\\n        MA_ATOMIC(typeSize, ma_##type) value; \\\n    } ma_atomic_##type; \\\n\n#define MA_ATOMIC_SAFE_TYPE_DECL_PTR(type) \\\n    typedef struct \\\n    { \\\n        MA_ATOMIC(MA_SIZEOF_PTR, ma_##type*) value; \\\n    } ma_atomic_ptr_##type; \\\n\nMA_ATOMIC_SAFE_TYPE_DECL(32,  4, uint32)\nMA_ATOMIC_SAFE_TYPE_DECL(i32, 4, int32)\nMA_ATOMIC_SAFE_TYPE_DECL(64,  8, uint64)\nMA_ATOMIC_SAFE_TYPE_DECL(f32, 4, float)\nMA_ATOMIC_SAFE_TYPE_DECL(32,  4, bool32)\n\n\n/* Spinlocks are 32-bit for compatibility reasons. */\ntypedef ma_uint32 ma_spinlock;\n\n#ifndef MA_NO_THREADING\n    /* Thread priorities should be ordered such that the default priority of the worker thread is 0. */\n    typedef enum\n    {\n        ma_thread_priority_idle     = -5,\n        ma_thread_priority_lowest   = -4,\n        ma_thread_priority_low      = -3,\n        ma_thread_priority_normal   = -2,\n        ma_thread_priority_high     = -1,\n        ma_thread_priority_highest  =  0,\n        ma_thread_priority_realtime =  1,\n        ma_thread_priority_default  =  0\n    } ma_thread_priority;\n\n    #if defined(MA_POSIX)\n        typedef ma_pthread_t ma_thread;\n    #elif defined(MA_WIN32)\n        typedef ma_handle ma_thread;\n    #endif\n\n    #if defined(MA_POSIX)\n        typedef ma_pthread_mutex_t ma_mutex;\n    #elif defined(MA_WIN32)\n        typedef ma_handle ma_mutex;\n    #endif\n\n    #if defined(MA_POSIX)\n        typedef struct\n        {\n            ma_uint32 value;\n            ma_pthread_mutex_t lock;\n            ma_pthread_cond_t cond;\n        } ma_event;\n    #elif defined(MA_WIN32)\n        typedef ma_handle ma_event;\n    #endif\n\n    #if defined(MA_POSIX)\n        typedef struct\n        {\n            int value;\n            ma_pthread_mutex_t lock;\n            ma_pthread_cond_t cond;\n        } ma_semaphore;\n    #elif defined(MA_WIN32)\n        typedef ma_handle ma_semaphore;\n    #endif\n#else\n    /* MA_NO_THREADING is set which means threading is disabled. Threading is required by some API families. If any of these are enabled we need to throw an error. */\n    #ifndef MA_NO_DEVICE_IO\n        #error \"MA_NO_THREADING cannot be used without MA_NO_DEVICE_IO\";\n    #endif\n#endif  /* MA_NO_THREADING */\n\n\n/*\nRetrieves the version of miniaudio as separated integers. Each component can be NULL if it's not required.\n*/\nMA_API void ma_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision);\n\n/*\nRetrieves the version of miniaudio as a string which can be useful for logging purposes.\n*/\nMA_API const char* ma_version_string(void);\n\n\n/**************************************************************************************************************************************************************\n\nLogging\n\n**************************************************************************************************************************************************************/\n#include <stdarg.h> /* For va_list. */\n\n#if defined(__has_attribute)\n    #if __has_attribute(format)\n        #define MA_ATTRIBUTE_FORMAT(fmt, va) __attribute__((format(printf, fmt, va)))\n    #endif\n#endif\n#ifndef MA_ATTRIBUTE_FORMAT\n#define MA_ATTRIBUTE_FORMAT(fmt, va)\n#endif\n\n#ifndef MA_MAX_LOG_CALLBACKS\n#define MA_MAX_LOG_CALLBACKS    4\n#endif\n\n\n/*\nThe callback for handling log messages.\n\n\nParameters\n----------\npUserData (in)\n    The user data pointer that was passed into ma_log_register_callback().\n\nlogLevel (in)\n    The log level. This can be one of the following:\n\n    +----------------------+\n    | Log Level            |\n    +----------------------+\n    | MA_LOG_LEVEL_DEBUG   |\n    | MA_LOG_LEVEL_INFO    |\n    | MA_LOG_LEVEL_WARNING |\n    | MA_LOG_LEVEL_ERROR   |\n    +----------------------+\n\npMessage (in)\n    The log message.\n*/\ntypedef void (* ma_log_callback_proc)(void* pUserData, ma_uint32 level, const char* pMessage);\n\ntypedef struct\n{\n    ma_log_callback_proc onLog;\n    void* pUserData;\n} ma_log_callback;\n\nMA_API ma_log_callback ma_log_callback_init(ma_log_callback_proc onLog, void* pUserData);\n\n\ntypedef struct\n{\n    ma_log_callback callbacks[MA_MAX_LOG_CALLBACKS];\n    ma_uint32 callbackCount;\n    ma_allocation_callbacks allocationCallbacks;    /* Need to store these persistently because ma_log_postv() might need to allocate a buffer on the heap. */\n#ifndef MA_NO_THREADING\n    ma_mutex lock;  /* For thread safety just to make it easier and safer for the logging implementation. */\n#endif\n} ma_log;\n\nMA_API ma_result ma_log_init(const ma_allocation_callbacks* pAllocationCallbacks, ma_log* pLog);\nMA_API void ma_log_uninit(ma_log* pLog);\nMA_API ma_result ma_log_register_callback(ma_log* pLog, ma_log_callback callback);\nMA_API ma_result ma_log_unregister_callback(ma_log* pLog, ma_log_callback callback);\nMA_API ma_result ma_log_post(ma_log* pLog, ma_uint32 level, const char* pMessage);\nMA_API ma_result ma_log_postv(ma_log* pLog, ma_uint32 level, const char* pFormat, va_list args);\nMA_API ma_result ma_log_postf(ma_log* pLog, ma_uint32 level, const char* pFormat, ...) MA_ATTRIBUTE_FORMAT(3, 4);\n\n\n/**************************************************************************************************************************************************************\n\nBiquad Filtering\n\n**************************************************************************************************************************************************************/\ntypedef union\n{\n    float    f32;\n    ma_int32 s32;\n} ma_biquad_coefficient;\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    double b0;\n    double b1;\n    double b2;\n    double a0;\n    double a1;\n    double a2;\n} ma_biquad_config;\n\nMA_API ma_biquad_config ma_biquad_config_init(ma_format format, ma_uint32 channels, double b0, double b1, double b2, double a0, double a1, double a2);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_biquad_coefficient b0;\n    ma_biquad_coefficient b1;\n    ma_biquad_coefficient b2;\n    ma_biquad_coefficient a1;\n    ma_biquad_coefficient a2;\n    ma_biquad_coefficient* pR1;\n    ma_biquad_coefficient* pR2;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_biquad;\n\nMA_API ma_result ma_biquad_get_heap_size(const ma_biquad_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_biquad_init_preallocated(const ma_biquad_config* pConfig, void* pHeap, ma_biquad* pBQ);\nMA_API ma_result ma_biquad_init(const ma_biquad_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_biquad* pBQ);\nMA_API void ma_biquad_uninit(ma_biquad* pBQ, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_biquad_reinit(const ma_biquad_config* pConfig, ma_biquad* pBQ);\nMA_API ma_result ma_biquad_clear_cache(ma_biquad* pBQ);\nMA_API ma_result ma_biquad_process_pcm_frames(ma_biquad* pBQ, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_biquad_get_latency(const ma_biquad* pBQ);\n\n\n/**************************************************************************************************************************************************************\n\nLow-Pass Filtering\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    double q;\n} ma_lpf1_config, ma_lpf2_config;\n\nMA_API ma_lpf1_config ma_lpf1_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency);\nMA_API ma_lpf2_config ma_lpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_biquad_coefficient a;\n    ma_biquad_coefficient* pR1;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_lpf1;\n\nMA_API ma_result ma_lpf1_get_heap_size(const ma_lpf1_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_lpf1_init_preallocated(const ma_lpf1_config* pConfig, void* pHeap, ma_lpf1* pLPF);\nMA_API ma_result ma_lpf1_init(const ma_lpf1_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf1* pLPF);\nMA_API void ma_lpf1_uninit(ma_lpf1* pLPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_lpf1_reinit(const ma_lpf1_config* pConfig, ma_lpf1* pLPF);\nMA_API ma_result ma_lpf1_clear_cache(ma_lpf1* pLPF);\nMA_API ma_result ma_lpf1_process_pcm_frames(ma_lpf1* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_lpf1_get_latency(const ma_lpf1* pLPF);\n\ntypedef struct\n{\n    ma_biquad bq;   /* The second order low-pass filter is implemented as a biquad filter. */\n} ma_lpf2;\n\nMA_API ma_result ma_lpf2_get_heap_size(const ma_lpf2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_lpf2_init_preallocated(const ma_lpf2_config* pConfig, void* pHeap, ma_lpf2* pHPF);\nMA_API ma_result ma_lpf2_init(const ma_lpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf2* pLPF);\nMA_API void ma_lpf2_uninit(ma_lpf2* pLPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_lpf2_reinit(const ma_lpf2_config* pConfig, ma_lpf2* pLPF);\nMA_API ma_result ma_lpf2_clear_cache(ma_lpf2* pLPF);\nMA_API ma_result ma_lpf2_process_pcm_frames(ma_lpf2* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_lpf2_get_latency(const ma_lpf2* pLPF);\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    ma_uint32 order;    /* If set to 0, will be treated as a passthrough (no filtering will be applied). */\n} ma_lpf_config;\n\nMA_API ma_lpf_config ma_lpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint32 lpf1Count;\n    ma_uint32 lpf2Count;\n    ma_lpf1* pLPF1;\n    ma_lpf2* pLPF2;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_lpf;\n\nMA_API ma_result ma_lpf_get_heap_size(const ma_lpf_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_lpf_init_preallocated(const ma_lpf_config* pConfig, void* pHeap, ma_lpf* pLPF);\nMA_API ma_result ma_lpf_init(const ma_lpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf* pLPF);\nMA_API void ma_lpf_uninit(ma_lpf* pLPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_lpf_reinit(const ma_lpf_config* pConfig, ma_lpf* pLPF);\nMA_API ma_result ma_lpf_clear_cache(ma_lpf* pLPF);\nMA_API ma_result ma_lpf_process_pcm_frames(ma_lpf* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_lpf_get_latency(const ma_lpf* pLPF);\n\n\n/**************************************************************************************************************************************************************\n\nHigh-Pass Filtering\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    double q;\n} ma_hpf1_config, ma_hpf2_config;\n\nMA_API ma_hpf1_config ma_hpf1_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency);\nMA_API ma_hpf2_config ma_hpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_biquad_coefficient a;\n    ma_biquad_coefficient* pR1;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_hpf1;\n\nMA_API ma_result ma_hpf1_get_heap_size(const ma_hpf1_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_hpf1_init_preallocated(const ma_hpf1_config* pConfig, void* pHeap, ma_hpf1* pLPF);\nMA_API ma_result ma_hpf1_init(const ma_hpf1_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf1* pHPF);\nMA_API void ma_hpf1_uninit(ma_hpf1* pHPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_hpf1_reinit(const ma_hpf1_config* pConfig, ma_hpf1* pHPF);\nMA_API ma_result ma_hpf1_process_pcm_frames(ma_hpf1* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_hpf1_get_latency(const ma_hpf1* pHPF);\n\ntypedef struct\n{\n    ma_biquad bq;   /* The second order high-pass filter is implemented as a biquad filter. */\n} ma_hpf2;\n\nMA_API ma_result ma_hpf2_get_heap_size(const ma_hpf2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_hpf2_init_preallocated(const ma_hpf2_config* pConfig, void* pHeap, ma_hpf2* pHPF);\nMA_API ma_result ma_hpf2_init(const ma_hpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf2* pHPF);\nMA_API void ma_hpf2_uninit(ma_hpf2* pHPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_hpf2_reinit(const ma_hpf2_config* pConfig, ma_hpf2* pHPF);\nMA_API ma_result ma_hpf2_process_pcm_frames(ma_hpf2* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_hpf2_get_latency(const ma_hpf2* pHPF);\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    ma_uint32 order;    /* If set to 0, will be treated as a passthrough (no filtering will be applied). */\n} ma_hpf_config;\n\nMA_API ma_hpf_config ma_hpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint32 hpf1Count;\n    ma_uint32 hpf2Count;\n    ma_hpf1* pHPF1;\n    ma_hpf2* pHPF2;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_hpf;\n\nMA_API ma_result ma_hpf_get_heap_size(const ma_hpf_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_hpf_init_preallocated(const ma_hpf_config* pConfig, void* pHeap, ma_hpf* pLPF);\nMA_API ma_result ma_hpf_init(const ma_hpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf* pHPF);\nMA_API void ma_hpf_uninit(ma_hpf* pHPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_hpf_reinit(const ma_hpf_config* pConfig, ma_hpf* pHPF);\nMA_API ma_result ma_hpf_process_pcm_frames(ma_hpf* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_hpf_get_latency(const ma_hpf* pHPF);\n\n\n/**************************************************************************************************************************************************************\n\nBand-Pass Filtering\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    double q;\n} ma_bpf2_config;\n\nMA_API ma_bpf2_config ma_bpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q);\n\ntypedef struct\n{\n    ma_biquad bq;   /* The second order band-pass filter is implemented as a biquad filter. */\n} ma_bpf2;\n\nMA_API ma_result ma_bpf2_get_heap_size(const ma_bpf2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_bpf2_init_preallocated(const ma_bpf2_config* pConfig, void* pHeap, ma_bpf2* pBPF);\nMA_API ma_result ma_bpf2_init(const ma_bpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf2* pBPF);\nMA_API void ma_bpf2_uninit(ma_bpf2* pBPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_bpf2_reinit(const ma_bpf2_config* pConfig, ma_bpf2* pBPF);\nMA_API ma_result ma_bpf2_process_pcm_frames(ma_bpf2* pBPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_bpf2_get_latency(const ma_bpf2* pBPF);\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double cutoffFrequency;\n    ma_uint32 order;    /* If set to 0, will be treated as a passthrough (no filtering will be applied). */\n} ma_bpf_config;\n\nMA_API ma_bpf_config ma_bpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 bpf2Count;\n    ma_bpf2* pBPF2;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_bpf;\n\nMA_API ma_result ma_bpf_get_heap_size(const ma_bpf_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_bpf_init_preallocated(const ma_bpf_config* pConfig, void* pHeap, ma_bpf* pBPF);\nMA_API ma_result ma_bpf_init(const ma_bpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf* pBPF);\nMA_API void ma_bpf_uninit(ma_bpf* pBPF, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_bpf_reinit(const ma_bpf_config* pConfig, ma_bpf* pBPF);\nMA_API ma_result ma_bpf_process_pcm_frames(ma_bpf* pBPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_bpf_get_latency(const ma_bpf* pBPF);\n\n\n/**************************************************************************************************************************************************************\n\nNotching Filter\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double q;\n    double frequency;\n} ma_notch2_config, ma_notch_config;\n\nMA_API ma_notch2_config ma_notch2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double q, double frequency);\n\ntypedef struct\n{\n    ma_biquad bq;\n} ma_notch2;\n\nMA_API ma_result ma_notch2_get_heap_size(const ma_notch2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_notch2_init_preallocated(const ma_notch2_config* pConfig, void* pHeap, ma_notch2* pFilter);\nMA_API ma_result ma_notch2_init(const ma_notch2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_notch2* pFilter);\nMA_API void ma_notch2_uninit(ma_notch2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_notch2_reinit(const ma_notch2_config* pConfig, ma_notch2* pFilter);\nMA_API ma_result ma_notch2_process_pcm_frames(ma_notch2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_notch2_get_latency(const ma_notch2* pFilter);\n\n\n/**************************************************************************************************************************************************************\n\nPeaking EQ Filter\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double gainDB;\n    double q;\n    double frequency;\n} ma_peak2_config, ma_peak_config;\n\nMA_API ma_peak2_config ma_peak2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency);\n\ntypedef struct\n{\n    ma_biquad bq;\n} ma_peak2;\n\nMA_API ma_result ma_peak2_get_heap_size(const ma_peak2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_peak2_init_preallocated(const ma_peak2_config* pConfig, void* pHeap, ma_peak2* pFilter);\nMA_API ma_result ma_peak2_init(const ma_peak2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_peak2* pFilter);\nMA_API void ma_peak2_uninit(ma_peak2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_peak2_reinit(const ma_peak2_config* pConfig, ma_peak2* pFilter);\nMA_API ma_result ma_peak2_process_pcm_frames(ma_peak2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_peak2_get_latency(const ma_peak2* pFilter);\n\n\n/**************************************************************************************************************************************************************\n\nLow Shelf Filter\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double gainDB;\n    double shelfSlope;\n    double frequency;\n} ma_loshelf2_config, ma_loshelf_config;\n\nMA_API ma_loshelf2_config ma_loshelf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double shelfSlope, double frequency);\n\ntypedef struct\n{\n    ma_biquad bq;\n} ma_loshelf2;\n\nMA_API ma_result ma_loshelf2_get_heap_size(const ma_loshelf2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_loshelf2_init_preallocated(const ma_loshelf2_config* pConfig, void* pHeap, ma_loshelf2* pFilter);\nMA_API ma_result ma_loshelf2_init(const ma_loshelf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_loshelf2* pFilter);\nMA_API void ma_loshelf2_uninit(ma_loshelf2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_loshelf2_reinit(const ma_loshelf2_config* pConfig, ma_loshelf2* pFilter);\nMA_API ma_result ma_loshelf2_process_pcm_frames(ma_loshelf2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_loshelf2_get_latency(const ma_loshelf2* pFilter);\n\n\n/**************************************************************************************************************************************************************\n\nHigh Shelf Filter\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double gainDB;\n    double shelfSlope;\n    double frequency;\n} ma_hishelf2_config, ma_hishelf_config;\n\nMA_API ma_hishelf2_config ma_hishelf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double shelfSlope, double frequency);\n\ntypedef struct\n{\n    ma_biquad bq;\n} ma_hishelf2;\n\nMA_API ma_result ma_hishelf2_get_heap_size(const ma_hishelf2_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_hishelf2_init_preallocated(const ma_hishelf2_config* pConfig, void* pHeap, ma_hishelf2* pFilter);\nMA_API ma_result ma_hishelf2_init(const ma_hishelf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hishelf2* pFilter);\nMA_API void ma_hishelf2_uninit(ma_hishelf2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_hishelf2_reinit(const ma_hishelf2_config* pConfig, ma_hishelf2* pFilter);\nMA_API ma_result ma_hishelf2_process_pcm_frames(ma_hishelf2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_uint32 ma_hishelf2_get_latency(const ma_hishelf2* pFilter);\n\n\n\n/*\nDelay\n*/\ntypedef struct\n{\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint32 delayInFrames;\n    ma_bool32 delayStart;       /* Set to true to delay the start of the output; false otherwise. */\n    float wet;                  /* 0..1. Default = 1. */\n    float dry;                  /* 0..1. Default = 1. */\n    float decay;                /* 0..1. Default = 0 (no feedback). Feedback decay. Use this for echo. */\n} ma_delay_config;\n\nMA_API ma_delay_config ma_delay_config_init(ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 delayInFrames, float decay);\n\n\ntypedef struct\n{\n    ma_delay_config config;\n    ma_uint32 cursor;               /* Feedback is written to this cursor. Always equal or in front of the read cursor. */\n    ma_uint32 bufferSizeInFrames;\n    float* pBuffer;\n} ma_delay;\n\nMA_API ma_result ma_delay_init(const ma_delay_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_delay* pDelay);\nMA_API void ma_delay_uninit(ma_delay* pDelay, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_delay_process_pcm_frames(ma_delay* pDelay, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount);\nMA_API void ma_delay_set_wet(ma_delay* pDelay, float value);\nMA_API float ma_delay_get_wet(const ma_delay* pDelay);\nMA_API void ma_delay_set_dry(ma_delay* pDelay, float value);\nMA_API float ma_delay_get_dry(const ma_delay* pDelay);\nMA_API void ma_delay_set_decay(ma_delay* pDelay, float value);\nMA_API float ma_delay_get_decay(const ma_delay* pDelay);\n\n\n/* Gainer for smooth volume changes. */\ntypedef struct\n{\n    ma_uint32 channels;\n    ma_uint32 smoothTimeInFrames;\n} ma_gainer_config;\n\nMA_API ma_gainer_config ma_gainer_config_init(ma_uint32 channels, ma_uint32 smoothTimeInFrames);\n\n\ntypedef struct\n{\n    ma_gainer_config config;\n    ma_uint32 t;\n    float masterVolume;\n    float* pOldGains;\n    float* pNewGains;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_gainer;\n\nMA_API ma_result ma_gainer_get_heap_size(const ma_gainer_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_gainer_init_preallocated(const ma_gainer_config* pConfig, void* pHeap, ma_gainer* pGainer);\nMA_API ma_result ma_gainer_init(const ma_gainer_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_gainer* pGainer);\nMA_API void ma_gainer_uninit(ma_gainer* pGainer, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_gainer_process_pcm_frames(ma_gainer* pGainer, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_result ma_gainer_set_gain(ma_gainer* pGainer, float newGain);\nMA_API ma_result ma_gainer_set_gains(ma_gainer* pGainer, float* pNewGains);\nMA_API ma_result ma_gainer_set_master_volume(ma_gainer* pGainer, float volume);\nMA_API ma_result ma_gainer_get_master_volume(const ma_gainer* pGainer, float* pVolume);\n\n\n\n/* Stereo panner. */\ntypedef enum\n{\n    ma_pan_mode_balance = 0,    /* Does not blend one side with the other. Technically just a balance. Compatible with other popular audio engines and therefore the default. */\n    ma_pan_mode_pan             /* A true pan. The sound from one side will \"move\" to the other side and blend with it. */\n} ma_pan_mode;\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_pan_mode mode;\n    float pan;\n} ma_panner_config;\n\nMA_API ma_panner_config ma_panner_config_init(ma_format format, ma_uint32 channels);\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_pan_mode mode;\n    float pan;  /* -1..1 where 0 is no pan, -1 is left side, +1 is right side. Defaults to 0. */\n} ma_panner;\n\nMA_API ma_result ma_panner_init(const ma_panner_config* pConfig, ma_panner* pPanner);\nMA_API ma_result ma_panner_process_pcm_frames(ma_panner* pPanner, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API void ma_panner_set_mode(ma_panner* pPanner, ma_pan_mode mode);\nMA_API ma_pan_mode ma_panner_get_mode(const ma_panner* pPanner);\nMA_API void ma_panner_set_pan(ma_panner* pPanner, float pan);\nMA_API float ma_panner_get_pan(const ma_panner* pPanner);\n\n\n\n/* Fader. */\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n} ma_fader_config;\n\nMA_API ma_fader_config ma_fader_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate);\n\ntypedef struct\n{\n    ma_fader_config config;\n    float volumeBeg;            /* If volumeBeg and volumeEnd is equal to 1, no fading happens (ma_fader_process_pcm_frames() will run as a passthrough). */\n    float volumeEnd;\n    ma_uint64 lengthInFrames;   /* The total length of the fade. */\n    ma_int64  cursorInFrames;   /* The current time in frames. Incremented by ma_fader_process_pcm_frames(). Signed because it'll be offset by startOffsetInFrames in set_fade_ex(). */\n} ma_fader;\n\nMA_API ma_result ma_fader_init(const ma_fader_config* pConfig, ma_fader* pFader);\nMA_API ma_result ma_fader_process_pcm_frames(ma_fader* pFader, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API void ma_fader_get_data_format(const ma_fader* pFader, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate);\nMA_API void ma_fader_set_fade(ma_fader* pFader, float volumeBeg, float volumeEnd, ma_uint64 lengthInFrames);\nMA_API void ma_fader_set_fade_ex(ma_fader* pFader, float volumeBeg, float volumeEnd, ma_uint64 lengthInFrames, ma_int64 startOffsetInFrames);\nMA_API float ma_fader_get_current_volume(const ma_fader* pFader);\n\n\n\n/* Spatializer. */\ntypedef struct\n{\n    float x;\n    float y;\n    float z;\n} ma_vec3f;\n\ntypedef struct\n{\n    ma_vec3f v;\n    ma_spinlock lock;\n} ma_atomic_vec3f;\n\ntypedef enum\n{\n    ma_attenuation_model_none,          /* No distance attenuation and no spatialization. */\n    ma_attenuation_model_inverse,       /* Equivalent to OpenAL's AL_INVERSE_DISTANCE_CLAMPED. */\n    ma_attenuation_model_linear,        /* Linear attenuation. Equivalent to OpenAL's AL_LINEAR_DISTANCE_CLAMPED. */\n    ma_attenuation_model_exponential    /* Exponential attenuation. Equivalent to OpenAL's AL_EXPONENT_DISTANCE_CLAMPED. */\n} ma_attenuation_model;\n\ntypedef enum\n{\n    ma_positioning_absolute,\n    ma_positioning_relative\n} ma_positioning;\n\ntypedef enum\n{\n    ma_handedness_right,\n    ma_handedness_left\n} ma_handedness;\n\n\ntypedef struct\n{\n    ma_uint32 channelsOut;\n    ma_channel* pChannelMapOut;\n    ma_handedness handedness;   /* Defaults to right. Forward is -1 on the Z axis. In a left handed system, forward is +1 on the Z axis. */\n    float coneInnerAngleInRadians;\n    float coneOuterAngleInRadians;\n    float coneOuterGain;\n    float speedOfSound;\n    ma_vec3f worldUp;\n} ma_spatializer_listener_config;\n\nMA_API ma_spatializer_listener_config ma_spatializer_listener_config_init(ma_uint32 channelsOut);\n\n\ntypedef struct\n{\n    ma_spatializer_listener_config config;\n    ma_atomic_vec3f position;  /* The absolute position of the listener. */\n    ma_atomic_vec3f direction; /* The direction the listener is facing. The world up vector is config.worldUp. */\n    ma_atomic_vec3f velocity;\n    ma_bool32 isEnabled;\n\n    /* Memory management. */\n    ma_bool32 _ownsHeap;\n    void* _pHeap;\n} ma_spatializer_listener;\n\nMA_API ma_result ma_spatializer_listener_get_heap_size(const ma_spatializer_listener_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_spatializer_listener_init_preallocated(const ma_spatializer_listener_config* pConfig, void* pHeap, ma_spatializer_listener* pListener);\nMA_API ma_result ma_spatializer_listener_init(const ma_spatializer_listener_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_uninit(ma_spatializer_listener* pListener, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_channel* ma_spatializer_listener_get_channel_map(ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_cone(ma_spatializer_listener* pListener, float innerAngleInRadians, float outerAngleInRadians, float outerGain);\nMA_API void ma_spatializer_listener_get_cone(const ma_spatializer_listener* pListener, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain);\nMA_API void ma_spatializer_listener_set_position(ma_spatializer_listener* pListener, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_listener_get_position(const ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_direction(ma_spatializer_listener* pListener, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_listener_get_direction(const ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_velocity(ma_spatializer_listener* pListener, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_listener_get_velocity(const ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_speed_of_sound(ma_spatializer_listener* pListener, float speedOfSound);\nMA_API float ma_spatializer_listener_get_speed_of_sound(const ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_world_up(ma_spatializer_listener* pListener, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_listener_get_world_up(const ma_spatializer_listener* pListener);\nMA_API void ma_spatializer_listener_set_enabled(ma_spatializer_listener* pListener, ma_bool32 isEnabled);\nMA_API ma_bool32 ma_spatializer_listener_is_enabled(const ma_spatializer_listener* pListener);\n\n\ntypedef struct\n{\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_channel* pChannelMapIn;\n    ma_attenuation_model attenuationModel;\n    ma_positioning positioning;\n    ma_handedness handedness;           /* Defaults to right. Forward is -1 on the Z axis. In a left handed system, forward is +1 on the Z axis. */\n    float minGain;\n    float maxGain;\n    float minDistance;\n    float maxDistance;\n    float rolloff;\n    float coneInnerAngleInRadians;\n    float coneOuterAngleInRadians;\n    float coneOuterGain;\n    float dopplerFactor;                /* Set to 0 to disable doppler effect. */\n    float directionalAttenuationFactor; /* Set to 0 to disable directional attenuation. */\n    float minSpatializationChannelGain; /* The minimal scaling factor to apply to channel gains when accounting for the direction of the sound relative to the listener. Must be in the range of 0..1. Smaller values means more aggressive directional panning, larger values means more subtle directional panning. */\n    ma_uint32 gainSmoothTimeInFrames;   /* When the gain of a channel changes during spatialization, the transition will be linearly interpolated over this number of frames. */\n} ma_spatializer_config;\n\nMA_API ma_spatializer_config ma_spatializer_config_init(ma_uint32 channelsIn, ma_uint32 channelsOut);\n\n\ntypedef struct\n{\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_channel* pChannelMapIn;\n    ma_attenuation_model attenuationModel;\n    ma_positioning positioning;\n    ma_handedness handedness;           /* Defaults to right. Forward is -1 on the Z axis. In a left handed system, forward is +1 on the Z axis. */\n    float minGain;\n    float maxGain;\n    float minDistance;\n    float maxDistance;\n    float rolloff;\n    float coneInnerAngleInRadians;\n    float coneOuterAngleInRadians;\n    float coneOuterGain;\n    float dopplerFactor;                /* Set to 0 to disable doppler effect. */\n    float directionalAttenuationFactor; /* Set to 0 to disable directional attenuation. */\n    ma_uint32 gainSmoothTimeInFrames;   /* When the gain of a channel changes during spatialization, the transition will be linearly interpolated over this number of frames. */\n    ma_atomic_vec3f position;\n    ma_atomic_vec3f direction;\n    ma_atomic_vec3f velocity;  /* For doppler effect. */\n    float dopplerPitch; /* Will be updated by ma_spatializer_process_pcm_frames() and can be used by higher level functions to apply a pitch shift for doppler effect. */\n    float minSpatializationChannelGain;\n    ma_gainer gainer;   /* For smooth gain transitions. */\n    float* pNewChannelGainsOut; /* An offset of _pHeap. Used by ma_spatializer_process_pcm_frames() to store new channel gains. The number of elements in this array is equal to config.channelsOut. */\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_spatializer;\n\nMA_API ma_result ma_spatializer_get_heap_size(const ma_spatializer_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_spatializer_init_preallocated(const ma_spatializer_config* pConfig, void* pHeap, ma_spatializer* pSpatializer);\nMA_API ma_result ma_spatializer_init(const ma_spatializer_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_uninit(ma_spatializer* pSpatializer, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_spatializer_process_pcm_frames(ma_spatializer* pSpatializer, ma_spatializer_listener* pListener, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_result ma_spatializer_set_master_volume(ma_spatializer* pSpatializer, float volume);\nMA_API ma_result ma_spatializer_get_master_volume(const ma_spatializer* pSpatializer, float* pVolume);\nMA_API ma_uint32 ma_spatializer_get_input_channels(const ma_spatializer* pSpatializer);\nMA_API ma_uint32 ma_spatializer_get_output_channels(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_attenuation_model(ma_spatializer* pSpatializer, ma_attenuation_model attenuationModel);\nMA_API ma_attenuation_model ma_spatializer_get_attenuation_model(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_positioning(ma_spatializer* pSpatializer, ma_positioning positioning);\nMA_API ma_positioning ma_spatializer_get_positioning(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_rolloff(ma_spatializer* pSpatializer, float rolloff);\nMA_API float ma_spatializer_get_rolloff(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_min_gain(ma_spatializer* pSpatializer, float minGain);\nMA_API float ma_spatializer_get_min_gain(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_max_gain(ma_spatializer* pSpatializer, float maxGain);\nMA_API float ma_spatializer_get_max_gain(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_min_distance(ma_spatializer* pSpatializer, float minDistance);\nMA_API float ma_spatializer_get_min_distance(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_max_distance(ma_spatializer* pSpatializer, float maxDistance);\nMA_API float ma_spatializer_get_max_distance(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_cone(ma_spatializer* pSpatializer, float innerAngleInRadians, float outerAngleInRadians, float outerGain);\nMA_API void ma_spatializer_get_cone(const ma_spatializer* pSpatializer, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain);\nMA_API void ma_spatializer_set_doppler_factor(ma_spatializer* pSpatializer, float dopplerFactor);\nMA_API float ma_spatializer_get_doppler_factor(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_directional_attenuation_factor(ma_spatializer* pSpatializer, float directionalAttenuationFactor);\nMA_API float ma_spatializer_get_directional_attenuation_factor(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_position(ma_spatializer* pSpatializer, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_get_position(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_direction(ma_spatializer* pSpatializer, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_get_direction(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_set_velocity(ma_spatializer* pSpatializer, float x, float y, float z);\nMA_API ma_vec3f ma_spatializer_get_velocity(const ma_spatializer* pSpatializer);\nMA_API void ma_spatializer_get_relative_position_and_direction(const ma_spatializer* pSpatializer, const ma_spatializer_listener* pListener, ma_vec3f* pRelativePos, ma_vec3f* pRelativeDir);\n\n\n\n/************************************************************************************************************************************************************\n*************************************************************************************************************************************************************\n\nDATA CONVERSION\n===============\n\nThis section contains the APIs for data conversion. You will find everything here for channel mapping, sample format conversion, resampling, etc.\n\n*************************************************************************************************************************************************************\n************************************************************************************************************************************************************/\n\n/**************************************************************************************************************************************************************\n\nResampling\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRateIn;\n    ma_uint32 sampleRateOut;\n    ma_uint32 lpfOrder;         /* The low-pass filter order. Setting this to 0 will disable low-pass filtering. */\n    double    lpfNyquistFactor; /* 0..1. Defaults to 1. 1 = Half the sampling frequency (Nyquist Frequency), 0.5 = Quarter the sampling frequency (half Nyquest Frequency), etc. */\n} ma_linear_resampler_config;\n\nMA_API ma_linear_resampler_config ma_linear_resampler_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);\n\ntypedef struct\n{\n    ma_linear_resampler_config config;\n    ma_uint32 inAdvanceInt;\n    ma_uint32 inAdvanceFrac;\n    ma_uint32 inTimeInt;\n    ma_uint32 inTimeFrac;\n    union\n    {\n        float* f32;\n        ma_int16* s16;\n    } x0; /* The previous input frame. */\n    union\n    {\n        float* f32;\n        ma_int16* s16;\n    } x1; /* The next input frame. */\n    ma_lpf lpf;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_linear_resampler;\n\nMA_API ma_result ma_linear_resampler_get_heap_size(const ma_linear_resampler_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_linear_resampler_init_preallocated(const ma_linear_resampler_config* pConfig, void* pHeap, ma_linear_resampler* pResampler);\nMA_API ma_result ma_linear_resampler_init(const ma_linear_resampler_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_linear_resampler* pResampler);\nMA_API void ma_linear_resampler_uninit(ma_linear_resampler* pResampler, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_linear_resampler_process_pcm_frames(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut);\nMA_API ma_result ma_linear_resampler_set_rate(ma_linear_resampler* pResampler, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);\nMA_API ma_result ma_linear_resampler_set_rate_ratio(ma_linear_resampler* pResampler, float ratioInOut);\nMA_API ma_uint64 ma_linear_resampler_get_input_latency(const ma_linear_resampler* pResampler);\nMA_API ma_uint64 ma_linear_resampler_get_output_latency(const ma_linear_resampler* pResampler);\nMA_API ma_result ma_linear_resampler_get_required_input_frame_count(const ma_linear_resampler* pResampler, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount);\nMA_API ma_result ma_linear_resampler_get_expected_output_frame_count(const ma_linear_resampler* pResampler, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount);\nMA_API ma_result ma_linear_resampler_reset(ma_linear_resampler* pResampler);\n\n\ntypedef struct ma_resampler_config ma_resampler_config;\n\ntypedef void ma_resampling_backend;\ntypedef struct\n{\n    ma_result (* onGetHeapSize                )(void* pUserData, const ma_resampler_config* pConfig, size_t* pHeapSizeInBytes);\n    ma_result (* onInit                       )(void* pUserData, const ma_resampler_config* pConfig, void* pHeap, ma_resampling_backend** ppBackend);\n    void      (* onUninit                     )(void* pUserData, ma_resampling_backend* pBackend, const ma_allocation_callbacks* pAllocationCallbacks);\n    ma_result (* onProcess                    )(void* pUserData, ma_resampling_backend* pBackend, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut);\n    ma_result (* onSetRate                    )(void* pUserData, ma_resampling_backend* pBackend, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);                 /* Optional. Rate changes will be disabled. */\n    ma_uint64 (* onGetInputLatency            )(void* pUserData, const ma_resampling_backend* pBackend);                                                            /* Optional. Latency will be reported as 0. */\n    ma_uint64 (* onGetOutputLatency           )(void* pUserData, const ma_resampling_backend* pBackend);                                                            /* Optional. Latency will be reported as 0. */\n    ma_result (* onGetRequiredInputFrameCount )(void* pUserData, const ma_resampling_backend* pBackend, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount);   /* Optional. Latency mitigation will be disabled. */\n    ma_result (* onGetExpectedOutputFrameCount)(void* pUserData, const ma_resampling_backend* pBackend, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount);   /* Optional. Latency mitigation will be disabled. */\n    ma_result (* onReset                      )(void* pUserData, ma_resampling_backend* pBackend);\n} ma_resampling_backend_vtable;\n\ntypedef enum\n{\n    ma_resample_algorithm_linear = 0,    /* Fastest, lowest quality. Optional low-pass filtering. Default. */\n    ma_resample_algorithm_custom,\n} ma_resample_algorithm;\n\nstruct ma_resampler_config\n{\n    ma_format format;   /* Must be either ma_format_f32 or ma_format_s16. */\n    ma_uint32 channels;\n    ma_uint32 sampleRateIn;\n    ma_uint32 sampleRateOut;\n    ma_resample_algorithm algorithm;    /* When set to ma_resample_algorithm_custom, pBackendVTable will be used. */\n    ma_resampling_backend_vtable* pBackendVTable;\n    void* pBackendUserData;\n    struct\n    {\n        ma_uint32 lpfOrder;\n    } linear;\n};\n\nMA_API ma_resampler_config ma_resampler_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut, ma_resample_algorithm algorithm);\n\ntypedef struct\n{\n    ma_resampling_backend* pBackend;\n    ma_resampling_backend_vtable* pBackendVTable;\n    void* pBackendUserData;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRateIn;\n    ma_uint32 sampleRateOut;\n    union\n    {\n        ma_linear_resampler linear;\n    } state;    /* State for stock resamplers so we can avoid a malloc. For stock resamplers, pBackend will point here. */\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_resampler;\n\nMA_API ma_result ma_resampler_get_heap_size(const ma_resampler_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_resampler_init_preallocated(const ma_resampler_config* pConfig, void* pHeap, ma_resampler* pResampler);\n\n/*\nInitializes a new resampler object from a config.\n*/\nMA_API ma_result ma_resampler_init(const ma_resampler_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_resampler* pResampler);\n\n/*\nUninitializes a resampler.\n*/\nMA_API void ma_resampler_uninit(ma_resampler* pResampler, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nConverts the given input data.\n\nBoth the input and output frames must be in the format specified in the config when the resampler was initialized.\n\nOn input, [pFrameCountOut] contains the number of output frames to process. On output it contains the number of output frames that\nwere actually processed, which may be less than the requested amount which will happen if there's not enough input data. You can use\nma_resampler_get_expected_output_frame_count() to know how many output frames will be processed for a given number of input frames.\n\nOn input, [pFrameCountIn] contains the number of input frames contained in [pFramesIn]. On output it contains the number of whole\ninput frames that were actually processed. You can use ma_resampler_get_required_input_frame_count() to know how many input frames\nyou should provide for a given number of output frames. [pFramesIn] can be NULL, in which case zeroes will be used instead.\n\nIf [pFramesOut] is NULL, a seek is performed. In this case, if [pFrameCountOut] is not NULL it will seek by the specified number of\noutput frames. Otherwise, if [pFramesCountOut] is NULL and [pFrameCountIn] is not NULL, it will seek by the specified number of input\nframes. When seeking, [pFramesIn] is allowed to NULL, in which case the internal timing state will be updated, but no input will be\nprocessed. In this case, any internal filter state will be updated as if zeroes were passed in.\n\nIt is an error for [pFramesOut] to be non-NULL and [pFrameCountOut] to be NULL.\n\nIt is an error for both [pFrameCountOut] and [pFrameCountIn] to be NULL.\n*/\nMA_API ma_result ma_resampler_process_pcm_frames(ma_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut);\n\n\n/*\nSets the input and output sample rate.\n*/\nMA_API ma_result ma_resampler_set_rate(ma_resampler* pResampler, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);\n\n/*\nSets the input and output sample rate as a ratio.\n\nThe ration is in/out.\n*/\nMA_API ma_result ma_resampler_set_rate_ratio(ma_resampler* pResampler, float ratio);\n\n/*\nRetrieves the latency introduced by the resampler in input frames.\n*/\nMA_API ma_uint64 ma_resampler_get_input_latency(const ma_resampler* pResampler);\n\n/*\nRetrieves the latency introduced by the resampler in output frames.\n*/\nMA_API ma_uint64 ma_resampler_get_output_latency(const ma_resampler* pResampler);\n\n/*\nCalculates the number of whole input frames that would need to be read from the client in order to output the specified\nnumber of output frames.\n\nThe returned value does not include cached input frames. It only returns the number of extra frames that would need to be\nread from the input buffer in order to output the specified number of output frames.\n*/\nMA_API ma_result ma_resampler_get_required_input_frame_count(const ma_resampler* pResampler, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount);\n\n/*\nCalculates the number of whole output frames that would be output after fully reading and consuming the specified number of\ninput frames.\n*/\nMA_API ma_result ma_resampler_get_expected_output_frame_count(const ma_resampler* pResampler, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount);\n\n/*\nResets the resampler's timer and clears its internal cache.\n*/\nMA_API ma_result ma_resampler_reset(ma_resampler* pResampler);\n\n\n/**************************************************************************************************************************************************************\n\nChannel Conversion\n\n**************************************************************************************************************************************************************/\ntypedef enum\n{\n    ma_channel_conversion_path_unknown,\n    ma_channel_conversion_path_passthrough,\n    ma_channel_conversion_path_mono_out,    /* Converting to mono. */\n    ma_channel_conversion_path_mono_in,     /* Converting from mono. */\n    ma_channel_conversion_path_shuffle,     /* Simple shuffle. Will use this when all channels are present in both input and output channel maps, but just in a different order. */\n    ma_channel_conversion_path_weights      /* Blended based on weights. */\n} ma_channel_conversion_path;\n\ntypedef enum\n{\n    ma_mono_expansion_mode_duplicate = 0,   /* The default. */\n    ma_mono_expansion_mode_average,         /* Average the mono channel across all channels. */\n    ma_mono_expansion_mode_stereo_only,     /* Duplicate to the left and right channels only and ignore the others. */\n    ma_mono_expansion_mode_default = ma_mono_expansion_mode_duplicate\n} ma_mono_expansion_mode;\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    const ma_channel* pChannelMapIn;\n    const ma_channel* pChannelMapOut;\n    ma_channel_mix_mode mixingMode;\n    ma_bool32 calculateLFEFromSpatialChannels;  /* When an output LFE channel is present, but no input LFE, set to true to set the output LFE to the average of all spatial channels (LR, FR, etc.). Ignored when an input LFE is present. */\n    float** ppWeights;  /* [in][out]. Only used when mixingMode is set to ma_channel_mix_mode_custom_weights. */\n} ma_channel_converter_config;\n\nMA_API ma_channel_converter_config ma_channel_converter_config_init(ma_format format, ma_uint32 channelsIn, const ma_channel* pChannelMapIn, ma_uint32 channelsOut, const ma_channel* pChannelMapOut, ma_channel_mix_mode mixingMode);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_channel_mix_mode mixingMode;\n    ma_channel_conversion_path conversionPath;\n    ma_channel* pChannelMapIn;\n    ma_channel* pChannelMapOut;\n    ma_uint8* pShuffleTable;    /* Indexed by output channel index. */\n    union\n    {\n        float**    f32;\n        ma_int32** s16;\n    } weights;  /* [in][out] */\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_channel_converter;\n\nMA_API ma_result ma_channel_converter_get_heap_size(const ma_channel_converter_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_channel_converter_init_preallocated(const ma_channel_converter_config* pConfig, void* pHeap, ma_channel_converter* pConverter);\nMA_API ma_result ma_channel_converter_init(const ma_channel_converter_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_channel_converter* pConverter);\nMA_API void ma_channel_converter_uninit(ma_channel_converter* pConverter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_channel_converter_process_pcm_frames(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount);\nMA_API ma_result ma_channel_converter_get_input_channel_map(const ma_channel_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_channel_converter_get_output_channel_map(const ma_channel_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap);\n\n\n/**************************************************************************************************************************************************************\n\nData Conversion\n\n**************************************************************************************************************************************************************/\ntypedef struct\n{\n    ma_format formatIn;\n    ma_format formatOut;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_uint32 sampleRateIn;\n    ma_uint32 sampleRateOut;\n    ma_channel* pChannelMapIn;\n    ma_channel* pChannelMapOut;\n    ma_dither_mode ditherMode;\n    ma_channel_mix_mode channelMixMode;\n    ma_bool32 calculateLFEFromSpatialChannels;  /* When an output LFE channel is present, but no input LFE, set to true to set the output LFE to the average of all spatial channels (LR, FR, etc.). Ignored when an input LFE is present. */\n    float** ppChannelWeights;  /* [in][out]. Only used when mixingMode is set to ma_channel_mix_mode_custom_weights. */\n    ma_bool32 allowDynamicSampleRate;\n    ma_resampler_config resampling;\n} ma_data_converter_config;\n\nMA_API ma_data_converter_config ma_data_converter_config_init_default(void);\nMA_API ma_data_converter_config ma_data_converter_config_init(ma_format formatIn, ma_format formatOut, ma_uint32 channelsIn, ma_uint32 channelsOut, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);\n\n\ntypedef enum\n{\n    ma_data_converter_execution_path_passthrough,       /* No conversion. */\n    ma_data_converter_execution_path_format_only,       /* Only format conversion. */\n    ma_data_converter_execution_path_channels_only,     /* Only channel conversion. */\n    ma_data_converter_execution_path_resample_only,     /* Only resampling. */\n    ma_data_converter_execution_path_resample_first,    /* All conversions, but resample as the first step. */\n    ma_data_converter_execution_path_channels_first     /* All conversions, but channels as the first step. */\n} ma_data_converter_execution_path;\n\ntypedef struct\n{\n    ma_format formatIn;\n    ma_format formatOut;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_uint32 sampleRateIn;\n    ma_uint32 sampleRateOut;\n    ma_dither_mode ditherMode;\n    ma_data_converter_execution_path executionPath; /* The execution path the data converter will follow when processing. */\n    ma_channel_converter channelConverter;\n    ma_resampler resampler;\n    ma_bool8 hasPreFormatConversion;\n    ma_bool8 hasPostFormatConversion;\n    ma_bool8 hasChannelConverter;\n    ma_bool8 hasResampler;\n    ma_bool8 isPassthrough;\n\n    /* Memory management. */\n    ma_bool8 _ownsHeap;\n    void* _pHeap;\n} ma_data_converter;\n\nMA_API ma_result ma_data_converter_get_heap_size(const ma_data_converter_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_data_converter_init_preallocated(const ma_data_converter_config* pConfig, void* pHeap, ma_data_converter* pConverter);\nMA_API ma_result ma_data_converter_init(const ma_data_converter_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_converter* pConverter);\nMA_API void ma_data_converter_uninit(ma_data_converter* pConverter, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_data_converter_process_pcm_frames(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut);\nMA_API ma_result ma_data_converter_set_rate(ma_data_converter* pConverter, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut);\nMA_API ma_result ma_data_converter_set_rate_ratio(ma_data_converter* pConverter, float ratioInOut);\nMA_API ma_uint64 ma_data_converter_get_input_latency(const ma_data_converter* pConverter);\nMA_API ma_uint64 ma_data_converter_get_output_latency(const ma_data_converter* pConverter);\nMA_API ma_result ma_data_converter_get_required_input_frame_count(const ma_data_converter* pConverter, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount);\nMA_API ma_result ma_data_converter_get_expected_output_frame_count(const ma_data_converter* pConverter, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount);\nMA_API ma_result ma_data_converter_get_input_channel_map(const ma_data_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_data_converter_get_output_channel_map(const ma_data_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_data_converter_reset(ma_data_converter* pConverter);\n\n\n/************************************************************************************************************************************************************\n\nFormat Conversion\n\n************************************************************************************************************************************************************/\nMA_API void ma_pcm_u8_to_s16(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_u8_to_s24(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_u8_to_s32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_u8_to_f32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s16_to_u8(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s16_to_s24(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s16_to_s32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s16_to_f32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s24_to_u8(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s24_to_s16(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s24_to_s32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s24_to_f32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s32_to_u8(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s32_to_s16(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s32_to_s24(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_s32_to_f32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_f32_to_u8(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_f32_to_s16(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_f32_to_s24(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_f32_to_s32(void* pOut, const void* pIn, ma_uint64 count, ma_dither_mode ditherMode);\nMA_API void ma_pcm_convert(void* pOut, ma_format formatOut, const void* pIn, ma_format formatIn, ma_uint64 sampleCount, ma_dither_mode ditherMode);\nMA_API void ma_convert_pcm_frames_format(void* pOut, ma_format formatOut, const void* pIn, ma_format formatIn, ma_uint64 frameCount, ma_uint32 channels, ma_dither_mode ditherMode);\n\n/*\nDeinterleaves an interleaved buffer.\n*/\nMA_API void ma_deinterleave_pcm_frames(ma_format format, ma_uint32 channels, ma_uint64 frameCount, const void* pInterleavedPCMFrames, void** ppDeinterleavedPCMFrames);\n\n/*\nInterleaves a group of deinterleaved buffers.\n*/\nMA_API void ma_interleave_pcm_frames(ma_format format, ma_uint32 channels, ma_uint64 frameCount, const void** ppDeinterleavedPCMFrames, void* pInterleavedPCMFrames);\n\n\n/************************************************************************************************************************************************************\n\nChannel Maps\n\n************************************************************************************************************************************************************/\n/*\nThis is used in the shuffle table to indicate that the channel index is undefined and should be ignored.\n*/\n#define MA_CHANNEL_INDEX_NULL   255\n\n/*\nRetrieves the channel position of the specified channel in the given channel map.\n\nThe pChannelMap parameter can be null, in which case miniaudio's default channel map will be assumed.\n*/\nMA_API ma_channel ma_channel_map_get_channel(const ma_channel* pChannelMap, ma_uint32 channelCount, ma_uint32 channelIndex);\n\n/*\nInitializes a blank channel map.\n\nWhen a blank channel map is specified anywhere it indicates that the native channel map should be used.\n*/\nMA_API void ma_channel_map_init_blank(ma_channel* pChannelMap, ma_uint32 channels);\n\n/*\nHelper for retrieving a standard channel map.\n\nThe output channel map buffer must have a capacity of at least `channelMapCap`.\n*/\nMA_API void ma_channel_map_init_standard(ma_standard_channel_map standardChannelMap, ma_channel* pChannelMap, size_t channelMapCap, ma_uint32 channels);\n\n/*\nCopies a channel map.\n\nBoth input and output channel map buffers must have a capacity of at least `channels`.\n*/\nMA_API void ma_channel_map_copy(ma_channel* pOut, const ma_channel* pIn, ma_uint32 channels);\n\n/*\nCopies a channel map if one is specified, otherwise copies the default channel map.\n\nThe output buffer must have a capacity of at least `channels`. If not NULL, the input channel map must also have a capacity of at least `channels`.\n*/\nMA_API void ma_channel_map_copy_or_default(ma_channel* pOut, size_t channelMapCapOut, const ma_channel* pIn, ma_uint32 channels);\n\n\n/*\nDetermines whether or not a channel map is valid.\n\nA blank channel map is valid (all channels set to MA_CHANNEL_NONE). The way a blank channel map is handled is context specific, but\nis usually treated as a passthrough.\n\nInvalid channel maps:\n  - A channel map with no channels\n  - A channel map with more than one channel and a mono channel\n\nThe channel map buffer must have a capacity of at least `channels`.\n*/\nMA_API ma_bool32 ma_channel_map_is_valid(const ma_channel* pChannelMap, ma_uint32 channels);\n\n/*\nHelper for comparing two channel maps for equality.\n\nThis assumes the channel count is the same between the two.\n\nBoth channels map buffers must have a capacity of at least `channels`.\n*/\nMA_API ma_bool32 ma_channel_map_is_equal(const ma_channel* pChannelMapA, const ma_channel* pChannelMapB, ma_uint32 channels);\n\n/*\nHelper for determining if a channel map is blank (all channels set to MA_CHANNEL_NONE).\n\nThe channel map buffer must have a capacity of at least `channels`.\n*/\nMA_API ma_bool32 ma_channel_map_is_blank(const ma_channel* pChannelMap, ma_uint32 channels);\n\n/*\nHelper for determining whether or not a channel is present in the given channel map.\n\nThe channel map buffer must have a capacity of at least `channels`.\n*/\nMA_API ma_bool32 ma_channel_map_contains_channel_position(ma_uint32 channels, const ma_channel* pChannelMap, ma_channel channelPosition);\n\n/*\nFind a channel position in the given channel map. Returns MA_TRUE if the channel is found; MA_FALSE otherwise. The\nindex of the channel is output to `pChannelIndex`.\n\nThe channel map buffer must have a capacity of at least `channels`.\n*/\nMA_API ma_bool32 ma_channel_map_find_channel_position(ma_uint32 channels, const ma_channel* pChannelMap, ma_channel channelPosition, ma_uint32* pChannelIndex);\n\n/*\nGenerates a string representing the given channel map.\n\nThis is for printing and debugging purposes, not serialization/deserialization.\n\nReturns the length of the string, not including the null terminator.\n*/\nMA_API size_t ma_channel_map_to_string(const ma_channel* pChannelMap, ma_uint32 channels, char* pBufferOut, size_t bufferCap);\n\n/*\nRetrieves a human readable version of a channel position.\n*/\nMA_API const char* ma_channel_position_to_string(ma_channel channel);\n\n\n/************************************************************************************************************************************************************\n\nConversion Helpers\n\n************************************************************************************************************************************************************/\n\n/*\nHigh-level helper for doing a full format conversion in one go. Returns the number of output frames. Call this with pOut set to NULL to\ndetermine the required size of the output buffer. frameCountOut should be set to the capacity of pOut. If pOut is NULL, frameCountOut is\nignored.\n\nA return value of 0 indicates an error.\n\nThis function is useful for one-off bulk conversions, but if you're streaming data you should use the ma_data_converter APIs instead.\n*/\nMA_API ma_uint64 ma_convert_frames(void* pOut, ma_uint64 frameCountOut, ma_format formatOut, ma_uint32 channelsOut, ma_uint32 sampleRateOut, const void* pIn, ma_uint64 frameCountIn, ma_format formatIn, ma_uint32 channelsIn, ma_uint32 sampleRateIn);\nMA_API ma_uint64 ma_convert_frames_ex(void* pOut, ma_uint64 frameCountOut, const void* pIn, ma_uint64 frameCountIn, const ma_data_converter_config* pConfig);\n\n\n/************************************************************************************************************************************************************\n\nData Source\n\n************************************************************************************************************************************************************/\ntypedef void ma_data_source;\n\n#define MA_DATA_SOURCE_SELF_MANAGED_RANGE_AND_LOOP_POINT    0x00000001\n\ntypedef struct\n{\n    ma_result (* onRead)(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\n    ma_result (* onSeek)(ma_data_source* pDataSource, ma_uint64 frameIndex);\n    ma_result (* onGetDataFormat)(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\n    ma_result (* onGetCursor)(ma_data_source* pDataSource, ma_uint64* pCursor);\n    ma_result (* onGetLength)(ma_data_source* pDataSource, ma_uint64* pLength);\n    ma_result (* onSetLooping)(ma_data_source* pDataSource, ma_bool32 isLooping);\n    ma_uint32 flags;\n} ma_data_source_vtable;\n\ntypedef ma_data_source* (* ma_data_source_get_next_proc)(ma_data_source* pDataSource);\n\ntypedef struct\n{\n    const ma_data_source_vtable* vtable;\n} ma_data_source_config;\n\nMA_API ma_data_source_config ma_data_source_config_init(void);\n\n\ntypedef struct\n{\n    const ma_data_source_vtable* vtable;\n    ma_uint64 rangeBegInFrames;\n    ma_uint64 rangeEndInFrames;             /* Set to -1 for unranged (default). */\n    ma_uint64 loopBegInFrames;              /* Relative to rangeBegInFrames. */\n    ma_uint64 loopEndInFrames;              /* Relative to rangeBegInFrames. Set to -1 for the end of the range. */\n    ma_data_source* pCurrent;               /* When non-NULL, the data source being initialized will act as a proxy and will route all operations to pCurrent. Used in conjunction with pNext/onGetNext for seamless chaining. */\n    ma_data_source* pNext;                  /* When set to NULL, onGetNext will be used. */\n    ma_data_source_get_next_proc onGetNext; /* Will be used when pNext is NULL. If both are NULL, no next will be used. */\n    MA_ATOMIC(4, ma_bool32) isLooping;\n} ma_data_source_base;\n\nMA_API ma_result ma_data_source_init(const ma_data_source_config* pConfig, ma_data_source* pDataSource);\nMA_API void ma_data_source_uninit(ma_data_source* pDataSource);\nMA_API ma_result ma_data_source_read_pcm_frames(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);   /* Must support pFramesOut = NULL in which case a forward seek should be performed. */\nMA_API ma_result ma_data_source_seek_pcm_frames(ma_data_source* pDataSource, ma_uint64 frameCount, ma_uint64* pFramesSeeked); /* Can only seek forward. Equivalent to ma_data_source_read_pcm_frames(pDataSource, NULL, frameCount, &framesRead); */\nMA_API ma_result ma_data_source_seek_to_pcm_frame(ma_data_source* pDataSource, ma_uint64 frameIndex);\nMA_API ma_result ma_data_source_seek_seconds(ma_data_source* pDataSource, float secondCount, float* pSecondsSeeked); /* Can only seek forward. Abstraction to ma_data_source_seek_pcm_frames() */\nMA_API ma_result ma_data_source_seek_to_second(ma_data_source* pDataSource, float seekPointInSeconds); /* Abstraction to ma_data_source_seek_to_pcm_frame() */\nMA_API ma_result ma_data_source_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_data_source_get_cursor_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pCursor);\nMA_API ma_result ma_data_source_get_length_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pLength);    /* Returns MA_NOT_IMPLEMENTED if the length is unknown or cannot be determined. Decoders can return this. */\nMA_API ma_result ma_data_source_get_cursor_in_seconds(ma_data_source* pDataSource, float* pCursor);\nMA_API ma_result ma_data_source_get_length_in_seconds(ma_data_source* pDataSource, float* pLength);\nMA_API ma_result ma_data_source_set_looping(ma_data_source* pDataSource, ma_bool32 isLooping);\nMA_API ma_bool32 ma_data_source_is_looping(const ma_data_source* pDataSource);\nMA_API ma_result ma_data_source_set_range_in_pcm_frames(ma_data_source* pDataSource, ma_uint64 rangeBegInFrames, ma_uint64 rangeEndInFrames);\nMA_API void ma_data_source_get_range_in_pcm_frames(const ma_data_source* pDataSource, ma_uint64* pRangeBegInFrames, ma_uint64* pRangeEndInFrames);\nMA_API ma_result ma_data_source_set_loop_point_in_pcm_frames(ma_data_source* pDataSource, ma_uint64 loopBegInFrames, ma_uint64 loopEndInFrames);\nMA_API void ma_data_source_get_loop_point_in_pcm_frames(const ma_data_source* pDataSource, ma_uint64* pLoopBegInFrames, ma_uint64* pLoopEndInFrames);\nMA_API ma_result ma_data_source_set_current(ma_data_source* pDataSource, ma_data_source* pCurrentDataSource);\nMA_API ma_data_source* ma_data_source_get_current(const ma_data_source* pDataSource);\nMA_API ma_result ma_data_source_set_next(ma_data_source* pDataSource, ma_data_source* pNextDataSource);\nMA_API ma_data_source* ma_data_source_get_next(const ma_data_source* pDataSource);\nMA_API ma_result ma_data_source_set_next_callback(ma_data_source* pDataSource, ma_data_source_get_next_proc onGetNext);\nMA_API ma_data_source_get_next_proc ma_data_source_get_next_callback(const ma_data_source* pDataSource);\n\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint64 cursor;\n    ma_uint64 sizeInFrames;\n    const void* pData;\n} ma_audio_buffer_ref;\n\nMA_API ma_result ma_audio_buffer_ref_init(ma_format format, ma_uint32 channels, const void* pData, ma_uint64 sizeInFrames, ma_audio_buffer_ref* pAudioBufferRef);\nMA_API void ma_audio_buffer_ref_uninit(ma_audio_buffer_ref* pAudioBufferRef);\nMA_API ma_result ma_audio_buffer_ref_set_data(ma_audio_buffer_ref* pAudioBufferRef, const void* pData, ma_uint64 sizeInFrames);\nMA_API ma_uint64 ma_audio_buffer_ref_read_pcm_frames(ma_audio_buffer_ref* pAudioBufferRef, void* pFramesOut, ma_uint64 frameCount, ma_bool32 loop);\nMA_API ma_result ma_audio_buffer_ref_seek_to_pcm_frame(ma_audio_buffer_ref* pAudioBufferRef, ma_uint64 frameIndex);\nMA_API ma_result ma_audio_buffer_ref_map(ma_audio_buffer_ref* pAudioBufferRef, void** ppFramesOut, ma_uint64* pFrameCount);\nMA_API ma_result ma_audio_buffer_ref_unmap(ma_audio_buffer_ref* pAudioBufferRef, ma_uint64 frameCount);    /* Returns MA_AT_END if the end has been reached. This should be considered successful. */\nMA_API ma_bool32 ma_audio_buffer_ref_at_end(const ma_audio_buffer_ref* pAudioBufferRef);\nMA_API ma_result ma_audio_buffer_ref_get_cursor_in_pcm_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pCursor);\nMA_API ma_result ma_audio_buffer_ref_get_length_in_pcm_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pLength);\nMA_API ma_result ma_audio_buffer_ref_get_available_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pAvailableFrames);\n\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint64 sizeInFrames;\n    const void* pData;  /* If set to NULL, will allocate a block of memory for you. */\n    ma_allocation_callbacks allocationCallbacks;\n} ma_audio_buffer_config;\n\nMA_API ma_audio_buffer_config ma_audio_buffer_config_init(ma_format format, ma_uint32 channels, ma_uint64 sizeInFrames, const void* pData, const ma_allocation_callbacks* pAllocationCallbacks);\n\ntypedef struct\n{\n    ma_audio_buffer_ref ref;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_bool32 ownsData;             /* Used to control whether or not miniaudio owns the data buffer. If set to true, pData will be freed in ma_audio_buffer_uninit(). */\n    ma_uint8 _pExtraData[1];        /* For allocating a buffer with the memory located directly after the other memory of the structure. */\n} ma_audio_buffer;\n\nMA_API ma_result ma_audio_buffer_init(const ma_audio_buffer_config* pConfig, ma_audio_buffer* pAudioBuffer);\nMA_API ma_result ma_audio_buffer_init_copy(const ma_audio_buffer_config* pConfig, ma_audio_buffer* pAudioBuffer);\nMA_API ma_result ma_audio_buffer_alloc_and_init(const ma_audio_buffer_config* pConfig, ma_audio_buffer** ppAudioBuffer);  /* Always copies the data. Doesn't make sense to use this otherwise. Use ma_audio_buffer_uninit_and_free() to uninit. */\nMA_API void ma_audio_buffer_uninit(ma_audio_buffer* pAudioBuffer);\nMA_API void ma_audio_buffer_uninit_and_free(ma_audio_buffer* pAudioBuffer);\nMA_API ma_uint64 ma_audio_buffer_read_pcm_frames(ma_audio_buffer* pAudioBuffer, void* pFramesOut, ma_uint64 frameCount, ma_bool32 loop);\nMA_API ma_result ma_audio_buffer_seek_to_pcm_frame(ma_audio_buffer* pAudioBuffer, ma_uint64 frameIndex);\nMA_API ma_result ma_audio_buffer_map(ma_audio_buffer* pAudioBuffer, void** ppFramesOut, ma_uint64* pFrameCount);\nMA_API ma_result ma_audio_buffer_unmap(ma_audio_buffer* pAudioBuffer, ma_uint64 frameCount);    /* Returns MA_AT_END if the end has been reached. This should be considered successful. */\nMA_API ma_bool32 ma_audio_buffer_at_end(const ma_audio_buffer* pAudioBuffer);\nMA_API ma_result ma_audio_buffer_get_cursor_in_pcm_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pCursor);\nMA_API ma_result ma_audio_buffer_get_length_in_pcm_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pLength);\nMA_API ma_result ma_audio_buffer_get_available_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pAvailableFrames);\n\n\n/*\nPaged Audio Buffer\n==================\nA paged audio buffer is made up of a linked list of pages. It's expandable, but not shrinkable. It\ncan be used for cases where audio data is streamed in asynchronously while allowing data to be read\nat the same time.\n\nThis is lock-free, but not 100% thread safe. You can append a page and read from the buffer across\nsimultaneously across different threads, however only one thread at a time can append, and only one\nthread at a time can read and seek.\n*/\ntypedef struct ma_paged_audio_buffer_page ma_paged_audio_buffer_page;\nstruct ma_paged_audio_buffer_page\n{\n    MA_ATOMIC(MA_SIZEOF_PTR, ma_paged_audio_buffer_page*) pNext;\n    ma_uint64 sizeInFrames;\n    ma_uint8 pAudioData[1];\n};\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_paged_audio_buffer_page head;                                /* Dummy head for the lock-free algorithm. Always has a size of 0. */\n    MA_ATOMIC(MA_SIZEOF_PTR, ma_paged_audio_buffer_page*) pTail;    /* Never null. Initially set to &head. */\n} ma_paged_audio_buffer_data;\n\nMA_API ma_result ma_paged_audio_buffer_data_init(ma_format format, ma_uint32 channels, ma_paged_audio_buffer_data* pData);\nMA_API void ma_paged_audio_buffer_data_uninit(ma_paged_audio_buffer_data* pData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_paged_audio_buffer_page* ma_paged_audio_buffer_data_get_head(ma_paged_audio_buffer_data* pData);\nMA_API ma_paged_audio_buffer_page* ma_paged_audio_buffer_data_get_tail(ma_paged_audio_buffer_data* pData);\nMA_API ma_result ma_paged_audio_buffer_data_get_length_in_pcm_frames(ma_paged_audio_buffer_data* pData, ma_uint64* pLength);\nMA_API ma_result ma_paged_audio_buffer_data_allocate_page(ma_paged_audio_buffer_data* pData, ma_uint64 pageSizeInFrames, const void* pInitialData, const ma_allocation_callbacks* pAllocationCallbacks, ma_paged_audio_buffer_page** ppPage);\nMA_API ma_result ma_paged_audio_buffer_data_free_page(ma_paged_audio_buffer_data* pData, ma_paged_audio_buffer_page* pPage, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_paged_audio_buffer_data_append_page(ma_paged_audio_buffer_data* pData, ma_paged_audio_buffer_page* pPage);\nMA_API ma_result ma_paged_audio_buffer_data_allocate_and_append_page(ma_paged_audio_buffer_data* pData, ma_uint32 pageSizeInFrames, const void* pInitialData, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\ntypedef struct\n{\n    ma_paged_audio_buffer_data* pData;  /* Must not be null. */\n} ma_paged_audio_buffer_config;\n\nMA_API ma_paged_audio_buffer_config ma_paged_audio_buffer_config_init(ma_paged_audio_buffer_data* pData);\n\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_paged_audio_buffer_data* pData;              /* Audio data is read from here. Cannot be null. */\n    ma_paged_audio_buffer_page* pCurrent;\n    ma_uint64 relativeCursor;                       /* Relative to the current page. */\n    ma_uint64 absoluteCursor;\n} ma_paged_audio_buffer;\n\nMA_API ma_result ma_paged_audio_buffer_init(const ma_paged_audio_buffer_config* pConfig, ma_paged_audio_buffer* pPagedAudioBuffer);\nMA_API void ma_paged_audio_buffer_uninit(ma_paged_audio_buffer* pPagedAudioBuffer);\nMA_API ma_result ma_paged_audio_buffer_read_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);   /* Returns MA_AT_END if no more pages available. */\nMA_API ma_result ma_paged_audio_buffer_seek_to_pcm_frame(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64 frameIndex);\nMA_API ma_result ma_paged_audio_buffer_get_cursor_in_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64* pCursor);\nMA_API ma_result ma_paged_audio_buffer_get_length_in_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64* pLength);\n\n\n\n/************************************************************************************************************************************************************\n\nRing Buffer\n\n************************************************************************************************************************************************************/\ntypedef struct\n{\n    void* pBuffer;\n    ma_uint32 subbufferSizeInBytes;\n    ma_uint32 subbufferCount;\n    ma_uint32 subbufferStrideInBytes;\n    MA_ATOMIC(4, ma_uint32) encodedReadOffset;  /* Most significant bit is the loop flag. Lower 31 bits contains the actual offset in bytes. Must be used atomically. */\n    MA_ATOMIC(4, ma_uint32) encodedWriteOffset; /* Most significant bit is the loop flag. Lower 31 bits contains the actual offset in bytes. Must be used atomically. */\n    ma_bool8 ownsBuffer;                        /* Used to know whether or not miniaudio is responsible for free()-ing the buffer. */\n    ma_bool8 clearOnWriteAcquire;               /* When set, clears the acquired write buffer before returning from ma_rb_acquire_write(). */\n    ma_allocation_callbacks allocationCallbacks;\n} ma_rb;\n\nMA_API ma_result ma_rb_init_ex(size_t subbufferSizeInBytes, size_t subbufferCount, size_t subbufferStrideInBytes, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_rb* pRB);\nMA_API ma_result ma_rb_init(size_t bufferSizeInBytes, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_rb* pRB);\nMA_API void ma_rb_uninit(ma_rb* pRB);\nMA_API void ma_rb_reset(ma_rb* pRB);\nMA_API ma_result ma_rb_acquire_read(ma_rb* pRB, size_t* pSizeInBytes, void** ppBufferOut);\nMA_API ma_result ma_rb_commit_read(ma_rb* pRB, size_t sizeInBytes);\nMA_API ma_result ma_rb_acquire_write(ma_rb* pRB, size_t* pSizeInBytes, void** ppBufferOut);\nMA_API ma_result ma_rb_commit_write(ma_rb* pRB, size_t sizeInBytes);\nMA_API ma_result ma_rb_seek_read(ma_rb* pRB, size_t offsetInBytes);\nMA_API ma_result ma_rb_seek_write(ma_rb* pRB, size_t offsetInBytes);\nMA_API ma_int32 ma_rb_pointer_distance(ma_rb* pRB);    /* Returns the distance between the write pointer and the read pointer. Should never be negative for a correct program. Will return the number of bytes that can be read before the read pointer hits the write pointer. */\nMA_API ma_uint32 ma_rb_available_read(ma_rb* pRB);\nMA_API ma_uint32 ma_rb_available_write(ma_rb* pRB);\nMA_API size_t ma_rb_get_subbuffer_size(ma_rb* pRB);\nMA_API size_t ma_rb_get_subbuffer_stride(ma_rb* pRB);\nMA_API size_t ma_rb_get_subbuffer_offset(ma_rb* pRB, size_t subbufferIndex);\nMA_API void* ma_rb_get_subbuffer_ptr(ma_rb* pRB, size_t subbufferIndex, void* pBuffer);\n\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_rb rb;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate; /* Not required for the ring buffer itself, but useful for associating the data with some sample rate, particularly for data sources. */\n} ma_pcm_rb;\n\nMA_API ma_result ma_pcm_rb_init_ex(ma_format format, ma_uint32 channels, ma_uint32 subbufferSizeInFrames, ma_uint32 subbufferCount, ma_uint32 subbufferStrideInFrames, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_pcm_rb* pRB);\nMA_API ma_result ma_pcm_rb_init(ma_format format, ma_uint32 channels, ma_uint32 bufferSizeInFrames, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_pcm_rb* pRB);\nMA_API void ma_pcm_rb_uninit(ma_pcm_rb* pRB);\nMA_API void ma_pcm_rb_reset(ma_pcm_rb* pRB);\nMA_API ma_result ma_pcm_rb_acquire_read(ma_pcm_rb* pRB, ma_uint32* pSizeInFrames, void** ppBufferOut);\nMA_API ma_result ma_pcm_rb_commit_read(ma_pcm_rb* pRB, ma_uint32 sizeInFrames);\nMA_API ma_result ma_pcm_rb_acquire_write(ma_pcm_rb* pRB, ma_uint32* pSizeInFrames, void** ppBufferOut);\nMA_API ma_result ma_pcm_rb_commit_write(ma_pcm_rb* pRB, ma_uint32 sizeInFrames);\nMA_API ma_result ma_pcm_rb_seek_read(ma_pcm_rb* pRB, ma_uint32 offsetInFrames);\nMA_API ma_result ma_pcm_rb_seek_write(ma_pcm_rb* pRB, ma_uint32 offsetInFrames);\nMA_API ma_int32 ma_pcm_rb_pointer_distance(ma_pcm_rb* pRB); /* Return value is in frames. */\nMA_API ma_uint32 ma_pcm_rb_available_read(ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_available_write(ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_size(ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_stride(ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_offset(ma_pcm_rb* pRB, ma_uint32 subbufferIndex);\nMA_API void* ma_pcm_rb_get_subbuffer_ptr(ma_pcm_rb* pRB, ma_uint32 subbufferIndex, void* pBuffer);\nMA_API ma_format ma_pcm_rb_get_format(const ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_get_channels(const ma_pcm_rb* pRB);\nMA_API ma_uint32 ma_pcm_rb_get_sample_rate(const ma_pcm_rb* pRB);\nMA_API void ma_pcm_rb_set_sample_rate(ma_pcm_rb* pRB, ma_uint32 sampleRate);\n\n\n/*\nThe idea of the duplex ring buffer is to act as the intermediary buffer when running two asynchronous devices in a duplex set up. The\ncapture device writes to it, and then a playback device reads from it.\n\nAt the moment this is just a simple naive implementation, but in the future I want to implement some dynamic resampling to seamlessly\nhandle desyncs. Note that the API is work in progress and may change at any time in any version.\n\nThe size of the buffer is based on the capture side since that's what'll be written to the buffer. It is based on the capture period size\nin frames. The internal sample rate of the capture device is also needed in order to calculate the size.\n*/\ntypedef struct\n{\n    ma_pcm_rb rb;\n} ma_duplex_rb;\n\nMA_API ma_result ma_duplex_rb_init(ma_format captureFormat, ma_uint32 captureChannels, ma_uint32 sampleRate, ma_uint32 captureInternalSampleRate, ma_uint32 captureInternalPeriodSizeInFrames, const ma_allocation_callbacks* pAllocationCallbacks, ma_duplex_rb* pRB);\nMA_API ma_result ma_duplex_rb_uninit(ma_duplex_rb* pRB);\n\n\n/************************************************************************************************************************************************************\n\nMiscellaneous Helpers\n\n************************************************************************************************************************************************************/\n/*\nRetrieves a human readable description of the given result code.\n*/\nMA_API const char* ma_result_description(ma_result result);\n\n/*\nmalloc()\n*/\nMA_API void* ma_malloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\ncalloc()\n*/\nMA_API void* ma_calloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nrealloc()\n*/\nMA_API void* ma_realloc(void* p, size_t sz, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nfree()\n*/\nMA_API void ma_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nPerforms an aligned malloc, with the assumption that the alignment is a power of 2.\n*/\nMA_API void* ma_aligned_malloc(size_t sz, size_t alignment, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nFree's an aligned malloc'd buffer.\n*/\nMA_API void ma_aligned_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks);\n\n/*\nRetrieves a friendly name for a format.\n*/\nMA_API const char* ma_get_format_name(ma_format format);\n\n/*\nBlends two frames in floating point format.\n*/\nMA_API void ma_blend_f32(float* pOut, float* pInA, float* pInB, float factor, ma_uint32 channels);\n\n/*\nRetrieves the size of a sample in bytes for the given format.\n\nThis API is efficient and is implemented using a lookup table.\n\nThread Safety: SAFE\n  This API is pure.\n*/\nMA_API ma_uint32 ma_get_bytes_per_sample(ma_format format);\nstatic MA_INLINE ma_uint32 ma_get_bytes_per_frame(ma_format format, ma_uint32 channels) { return ma_get_bytes_per_sample(format) * channels; }\n\n/*\nConverts a log level to a string.\n*/\nMA_API const char* ma_log_level_to_string(ma_uint32 logLevel);\n\n\n\n\n/************************************************************************************************************************************************************\n\nSynchronization\n\n************************************************************************************************************************************************************/\n/*\nLocks a spinlock.\n*/\nMA_API ma_result ma_spinlock_lock(volatile ma_spinlock* pSpinlock);\n\n/*\nLocks a spinlock, but does not yield() when looping.\n*/\nMA_API ma_result ma_spinlock_lock_noyield(volatile ma_spinlock* pSpinlock);\n\n/*\nUnlocks a spinlock.\n*/\nMA_API ma_result ma_spinlock_unlock(volatile ma_spinlock* pSpinlock);\n\n\n#ifndef MA_NO_THREADING\n\n/*\nCreates a mutex.\n\nA mutex must be created from a valid context. A mutex is initially unlocked.\n*/\nMA_API ma_result ma_mutex_init(ma_mutex* pMutex);\n\n/*\nDeletes a mutex.\n*/\nMA_API void ma_mutex_uninit(ma_mutex* pMutex);\n\n/*\nLocks a mutex with an infinite timeout.\n*/\nMA_API void ma_mutex_lock(ma_mutex* pMutex);\n\n/*\nUnlocks a mutex.\n*/\nMA_API void ma_mutex_unlock(ma_mutex* pMutex);\n\n\n/*\nInitializes an auto-reset event.\n*/\nMA_API ma_result ma_event_init(ma_event* pEvent);\n\n/*\nUninitializes an auto-reset event.\n*/\nMA_API void ma_event_uninit(ma_event* pEvent);\n\n/*\nWaits for the specified auto-reset event to become signalled.\n*/\nMA_API ma_result ma_event_wait(ma_event* pEvent);\n\n/*\nSignals the specified auto-reset event.\n*/\nMA_API ma_result ma_event_signal(ma_event* pEvent);\n\n\nMA_API ma_result ma_semaphore_init(int initialValue, ma_semaphore* pSemaphore);\nMA_API void ma_semaphore_uninit(ma_semaphore* pSemaphore);\nMA_API ma_result ma_semaphore_wait(ma_semaphore* pSemaphore);\nMA_API ma_result ma_semaphore_release(ma_semaphore* pSemaphore);\n#endif  /* MA_NO_THREADING */\n\n\n/*\nFence\n=====\nThis locks while the counter is larger than 0. Counter can be incremented and decremented by any\nthread, but care needs to be taken when waiting. It is possible for one thread to acquire the\nfence just as another thread returns from ma_fence_wait().\n\nThe idea behind a fence is to allow you to wait for a group of operations to complete. When an\noperation starts, the counter is incremented which locks the fence. When the operation completes,\nthe fence will be released which decrements the counter. ma_fence_wait() will block until the\ncounter hits zero.\n\nIf threading is disabled, ma_fence_wait() will spin on the counter.\n*/\ntypedef struct\n{\n#ifndef MA_NO_THREADING\n    ma_event e;\n#endif\n    ma_uint32 counter;\n} ma_fence;\n\nMA_API ma_result ma_fence_init(ma_fence* pFence);\nMA_API void ma_fence_uninit(ma_fence* pFence);\nMA_API ma_result ma_fence_acquire(ma_fence* pFence);    /* Increment counter. */\nMA_API ma_result ma_fence_release(ma_fence* pFence);    /* Decrement counter. */\nMA_API ma_result ma_fence_wait(ma_fence* pFence);       /* Wait for counter to reach 0. */\n\n\n\n/*\nNotification callback for asynchronous operations.\n*/\ntypedef void ma_async_notification;\n\ntypedef struct\n{\n    void (* onSignal)(ma_async_notification* pNotification);\n} ma_async_notification_callbacks;\n\nMA_API ma_result ma_async_notification_signal(ma_async_notification* pNotification);\n\n\n/*\nSimple polling notification.\n\nThis just sets a variable when the notification has been signalled which is then polled with ma_async_notification_poll_is_signalled()\n*/\ntypedef struct\n{\n    ma_async_notification_callbacks cb;\n    ma_bool32 signalled;\n} ma_async_notification_poll;\n\nMA_API ma_result ma_async_notification_poll_init(ma_async_notification_poll* pNotificationPoll);\nMA_API ma_bool32 ma_async_notification_poll_is_signalled(const ma_async_notification_poll* pNotificationPoll);\n\n\n/*\nEvent Notification\n\nThis uses an ma_event. If threading is disabled (MA_NO_THREADING), initialization will fail.\n*/\ntypedef struct\n{\n    ma_async_notification_callbacks cb;\n#ifndef MA_NO_THREADING\n    ma_event e;\n#endif\n} ma_async_notification_event;\n\nMA_API ma_result ma_async_notification_event_init(ma_async_notification_event* pNotificationEvent);\nMA_API ma_result ma_async_notification_event_uninit(ma_async_notification_event* pNotificationEvent);\nMA_API ma_result ma_async_notification_event_wait(ma_async_notification_event* pNotificationEvent);\nMA_API ma_result ma_async_notification_event_signal(ma_async_notification_event* pNotificationEvent);\n\n\n\n\n/************************************************************************************************************************************************************\n\nJob Queue\n\n************************************************************************************************************************************************************/\n\n/*\nSlot Allocator\n--------------\nThe idea of the slot allocator is for it to be used in conjunction with a fixed sized buffer. You use the slot allocator to allocate an index that can be used\nas the insertion point for an object.\n\nSlots are reference counted to help mitigate the ABA problem in the lock-free queue we use for tracking jobs.\n\nThe slot index is stored in the low 32 bits. The reference counter is stored in the high 32 bits:\n\n    +-----------------+-----------------+\n    | 32 Bits         | 32 Bits         |\n    +-----------------+-----------------+\n    | Reference Count | Slot Index      |\n    +-----------------+-----------------+\n*/\ntypedef struct\n{\n    ma_uint32 capacity;    /* The number of slots to make available. */\n} ma_slot_allocator_config;\n\nMA_API ma_slot_allocator_config ma_slot_allocator_config_init(ma_uint32 capacity);\n\n\ntypedef struct\n{\n    MA_ATOMIC(4, ma_uint32) bitfield;   /* Must be used atomically because the allocation and freeing routines need to make copies of this which must never be optimized away by the compiler. */\n} ma_slot_allocator_group;\n\ntypedef struct\n{\n    ma_slot_allocator_group* pGroups;   /* Slots are grouped in chunks of 32. */\n    ma_uint32* pSlots;                  /* 32 bits for reference counting for ABA mitigation. */\n    ma_uint32 count;                    /* Allocation count. */\n    ma_uint32 capacity;\n\n    /* Memory management. */\n    ma_bool32 _ownsHeap;\n    void* _pHeap;\n} ma_slot_allocator;\n\nMA_API ma_result ma_slot_allocator_get_heap_size(const ma_slot_allocator_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_slot_allocator_init_preallocated(const ma_slot_allocator_config* pConfig, void* pHeap, ma_slot_allocator* pAllocator);\nMA_API ma_result ma_slot_allocator_init(const ma_slot_allocator_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_slot_allocator* pAllocator);\nMA_API void ma_slot_allocator_uninit(ma_slot_allocator* pAllocator, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_slot_allocator_alloc(ma_slot_allocator* pAllocator, ma_uint64* pSlot);\nMA_API ma_result ma_slot_allocator_free(ma_slot_allocator* pAllocator, ma_uint64 slot);\n\n\ntypedef struct ma_job ma_job;\n\n/*\nCallback for processing a job. Each job type will have their own processing callback which will be\ncalled by ma_job_process().\n*/\ntypedef ma_result (* ma_job_proc)(ma_job* pJob);\n\n/* When a job type is added here an callback needs to be added go \"g_jobVTable\" in the implementation section. */\ntypedef enum\n{\n    /* Miscellaneous. */\n    MA_JOB_TYPE_QUIT = 0,\n    MA_JOB_TYPE_CUSTOM,\n\n    /* Resource Manager. */\n    MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER_NODE,\n    MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER_NODE,\n    MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE,\n    MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER,\n    MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER,\n    MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_STREAM,\n    MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_STREAM,\n    MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_STREAM,\n    MA_JOB_TYPE_RESOURCE_MANAGER_SEEK_DATA_STREAM,\n\n    /* Device. */\n    MA_JOB_TYPE_DEVICE_AAUDIO_REROUTE,\n\n    /* Count. Must always be last. */\n    MA_JOB_TYPE_COUNT\n} ma_job_type;\n\nstruct ma_job\n{\n    union\n    {\n        struct\n        {\n            ma_uint16 code;         /* Job type. */\n            ma_uint16 slot;         /* Index into a ma_slot_allocator. */\n            ma_uint32 refcount;\n        } breakup;\n        ma_uint64 allocation;\n    } toc;  /* 8 bytes. We encode the job code into the slot allocation data to save space. */\n    MA_ATOMIC(8, ma_uint64) next; /* refcount + slot for the next item. Does not include the job code. */\n    ma_uint32 order;    /* Execution order. Used to create a data dependency and ensure a job is executed in order. Usage is contextual depending on the job type. */\n\n    union\n    {\n        /* Miscellaneous. */\n        struct\n        {\n            ma_job_proc proc;\n            ma_uintptr data0;\n            ma_uintptr data1;\n        } custom;\n\n        /* Resource Manager */\n        union\n        {\n            struct\n            {\n                /*ma_resource_manager**/ void* pResourceManager;\n                /*ma_resource_manager_data_buffer_node**/ void* pDataBufferNode;\n                char* pFilePath;\n                wchar_t* pFilePathW;\n                ma_uint32 flags;                                /* Resource manager data source flags that were used when initializing the data buffer. */\n                ma_async_notification* pInitNotification;       /* Signalled when the data buffer has been initialized and the format/channels/rate can be retrieved. */\n                ma_async_notification* pDoneNotification;       /* Signalled when the data buffer has been fully decoded. Will be passed through to MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE when decoding. */\n                ma_fence* pInitFence;                           /* Released when initialization of the decoder is complete. */\n                ma_fence* pDoneFence;                           /* Released if initialization of the decoder fails. Passed through to PAGE_DATA_BUFFER_NODE untouched if init is successful. */\n            } loadDataBufferNode;\n            struct\n            {\n                /*ma_resource_manager**/ void* pResourceManager;\n                /*ma_resource_manager_data_buffer_node**/ void* pDataBufferNode;\n                ma_async_notification* pDoneNotification;\n                ma_fence* pDoneFence;\n            } freeDataBufferNode;\n            struct\n            {\n                /*ma_resource_manager**/ void* pResourceManager;\n                /*ma_resource_manager_data_buffer_node**/ void* pDataBufferNode;\n                /*ma_decoder**/ void* pDecoder;\n                ma_async_notification* pDoneNotification;       /* Signalled when the data buffer has been fully decoded. */\n                ma_fence* pDoneFence;                           /* Passed through from LOAD_DATA_BUFFER_NODE and released when the data buffer completes decoding or an error occurs. */\n            } pageDataBufferNode;\n\n            struct\n            {\n                /*ma_resource_manager_data_buffer**/ void* pDataBuffer;\n                ma_async_notification* pInitNotification;       /* Signalled when the data buffer has been initialized and the format/channels/rate can be retrieved. */\n                ma_async_notification* pDoneNotification;       /* Signalled when the data buffer has been fully decoded. */\n                ma_fence* pInitFence;                           /* Released when the data buffer has been initialized and the format/channels/rate can be retrieved. */\n                ma_fence* pDoneFence;                           /* Released when the data buffer has been fully decoded. */\n                ma_uint64 rangeBegInPCMFrames;\n                ma_uint64 rangeEndInPCMFrames;\n                ma_uint64 loopPointBegInPCMFrames;\n                ma_uint64 loopPointEndInPCMFrames;\n                ma_uint32 isLooping;\n            } loadDataBuffer;\n            struct\n            {\n                /*ma_resource_manager_data_buffer**/ void* pDataBuffer;\n                ma_async_notification* pDoneNotification;\n                ma_fence* pDoneFence;\n            } freeDataBuffer;\n\n            struct\n            {\n                /*ma_resource_manager_data_stream**/ void* pDataStream;\n                char* pFilePath;                            /* Allocated when the job is posted, freed by the job thread after loading. */\n                wchar_t* pFilePathW;                        /* ^ As above ^. Only used if pFilePath is NULL. */\n                ma_uint64 initialSeekPoint;\n                ma_async_notification* pInitNotification;   /* Signalled after the first two pages have been decoded and frames can be read from the stream. */\n                ma_fence* pInitFence;\n            } loadDataStream;\n            struct\n            {\n                /*ma_resource_manager_data_stream**/ void* pDataStream;\n                ma_async_notification* pDoneNotification;\n                ma_fence* pDoneFence;\n            } freeDataStream;\n            struct\n            {\n                /*ma_resource_manager_data_stream**/ void* pDataStream;\n                ma_uint32 pageIndex;                    /* The index of the page to decode into. */\n            } pageDataStream;\n            struct\n            {\n                /*ma_resource_manager_data_stream**/ void* pDataStream;\n                ma_uint64 frameIndex;\n            } seekDataStream;\n        } resourceManager;\n\n        /* Device. */\n        union\n        {\n            union\n            {\n                struct\n                {\n                    /*ma_device**/ void* pDevice;\n                    /*ma_device_type*/ ma_uint32 deviceType;\n                } reroute;\n            } aaudio;\n        } device;\n    } data;\n};\n\nMA_API ma_job ma_job_init(ma_uint16 code);\nMA_API ma_result ma_job_process(ma_job* pJob);\n\n\n/*\nWhen set, ma_job_queue_next() will not wait and no semaphore will be signaled in\nma_job_queue_post(). ma_job_queue_next() will return MA_NO_DATA_AVAILABLE if nothing is available.\n\nThis flag should always be used for platforms that do not support multithreading.\n*/\ntypedef enum\n{\n    MA_JOB_QUEUE_FLAG_NON_BLOCKING = 0x00000001\n} ma_job_queue_flags;\n\ntypedef struct\n{\n    ma_uint32 flags;\n    ma_uint32 capacity; /* The maximum number of jobs that can fit in the queue at a time. */\n} ma_job_queue_config;\n\nMA_API ma_job_queue_config ma_job_queue_config_init(ma_uint32 flags, ma_uint32 capacity);\n\n\ntypedef struct\n{\n    ma_uint32 flags;                /* Flags passed in at initialization time. */\n    ma_uint32 capacity;             /* The maximum number of jobs that can fit in the queue at a time. Set by the config. */\n    MA_ATOMIC(8, ma_uint64) head;   /* The first item in the list. Required for removing from the top of the list. */\n    MA_ATOMIC(8, ma_uint64) tail;   /* The last item in the list. Required for appending to the end of the list. */\n#ifndef MA_NO_THREADING\n    ma_semaphore sem;               /* Only used when MA_JOB_QUEUE_FLAG_NON_BLOCKING is unset. */\n#endif\n    ma_slot_allocator allocator;\n    ma_job* pJobs;\n#ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n    ma_spinlock lock;\n#endif\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_job_queue;\n\nMA_API ma_result ma_job_queue_get_heap_size(const ma_job_queue_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_job_queue_init_preallocated(const ma_job_queue_config* pConfig, void* pHeap, ma_job_queue* pQueue);\nMA_API ma_result ma_job_queue_init(const ma_job_queue_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_job_queue* pQueue);\nMA_API void ma_job_queue_uninit(ma_job_queue* pQueue, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_job_queue_post(ma_job_queue* pQueue, const ma_job* pJob);\nMA_API ma_result ma_job_queue_next(ma_job_queue* pQueue, ma_job* pJob); /* Returns MA_CANCELLED if the next job is a quit job. */\n\n\n\n/************************************************************************************************************************************************************\n*************************************************************************************************************************************************************\n\nDEVICE I/O\n==========\n\nThis section contains the APIs for device playback and capture. Here is where you'll find ma_device_init(), etc.\n\n*************************************************************************************************************************************************************\n************************************************************************************************************************************************************/\n#ifndef MA_NO_DEVICE_IO\n/* Some backends are only supported on certain platforms. */\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n    #define MA_SUPPORT_WASAPI\n\n    #if defined(MA_WIN32_DESKTOP)   /* DirectSound and WinMM backends are only supported on desktops. */\n        #define MA_SUPPORT_DSOUND\n        #define MA_SUPPORT_WINMM\n        #define MA_SUPPORT_JACK     /* JACK is technically supported on Windows, but I don't know how many people use it in practice... */\n    #endif\n#endif\n#if defined(MA_UNIX) && !defined(MA_ORBIS) && !defined(MA_PROSPERO)\n    #if defined(MA_LINUX)\n        #if !defined(MA_ANDROID) && !defined(MA_EMSCRIPTEN)   /* ALSA is not supported on Android. */\n            #define MA_SUPPORT_ALSA\n        #endif\n    #endif\n    #if !defined(MA_BSD) && !defined(MA_ANDROID) && !defined(MA_EMSCRIPTEN)\n        #define MA_SUPPORT_PULSEAUDIO\n        #define MA_SUPPORT_JACK\n    #endif\n    #if defined(__OpenBSD__)        /* <-- Change this to \"#if defined(MA_BSD)\" to enable sndio on all BSD flavors. */\n        #define MA_SUPPORT_SNDIO    /* sndio is only supported on OpenBSD for now. May be expanded later if there's demand. */\n    #endif\n    #if defined(__NetBSD__) || defined(__OpenBSD__)\n        #define MA_SUPPORT_AUDIO4   /* Only support audio(4) on platforms with known support. */\n    #endif\n    #if defined(__FreeBSD__) || defined(__DragonFly__)\n        #define MA_SUPPORT_OSS      /* Only support OSS on specific platforms with known support. */\n    #endif\n#endif\n#if defined(MA_ANDROID)\n    #define MA_SUPPORT_AAUDIO\n    #define MA_SUPPORT_OPENSL\n#endif\n#if defined(MA_APPLE)\n    #define MA_SUPPORT_COREAUDIO\n#endif\n#if defined(MA_EMSCRIPTEN)\n    #define MA_SUPPORT_WEBAUDIO\n#endif\n\n/* All platforms should support custom backends. */\n#define MA_SUPPORT_CUSTOM\n\n/* Explicitly disable the Null backend for Emscripten because it uses a background thread which is not properly supported right now. */\n#if !defined(MA_EMSCRIPTEN)\n#define MA_SUPPORT_NULL\n#endif\n\n\n#if defined(MA_SUPPORT_WASAPI) && !defined(MA_NO_WASAPI) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_WASAPI))\n    #define MA_HAS_WASAPI\n#endif\n#if defined(MA_SUPPORT_DSOUND) && !defined(MA_NO_DSOUND) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_DSOUND))\n    #define MA_HAS_DSOUND\n#endif\n#if defined(MA_SUPPORT_WINMM) && !defined(MA_NO_WINMM) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_WINMM))\n    #define MA_HAS_WINMM\n#endif\n#if defined(MA_SUPPORT_ALSA) && !defined(MA_NO_ALSA) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_ALSA))\n    #define MA_HAS_ALSA\n#endif\n#if defined(MA_SUPPORT_PULSEAUDIO) && !defined(MA_NO_PULSEAUDIO) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_PULSEAUDIO))\n    #define MA_HAS_PULSEAUDIO\n#endif\n#if defined(MA_SUPPORT_JACK) && !defined(MA_NO_JACK) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_JACK))\n    #define MA_HAS_JACK\n#endif\n#if defined(MA_SUPPORT_COREAUDIO) && !defined(MA_NO_COREAUDIO) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_COREAUDIO))\n    #define MA_HAS_COREAUDIO\n#endif\n#if defined(MA_SUPPORT_SNDIO) && !defined(MA_NO_SNDIO) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_SNDIO))\n    #define MA_HAS_SNDIO\n#endif\n#if defined(MA_SUPPORT_AUDIO4) && !defined(MA_NO_AUDIO4) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_AUDIO4))\n    #define MA_HAS_AUDIO4\n#endif\n#if defined(MA_SUPPORT_OSS) && !defined(MA_NO_OSS) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_OSS))\n    #define MA_HAS_OSS\n#endif\n#if defined(MA_SUPPORT_AAUDIO) && !defined(MA_NO_AAUDIO) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_AAUDIO))\n    #define MA_HAS_AAUDIO\n#endif\n#if defined(MA_SUPPORT_OPENSL) && !defined(MA_NO_OPENSL) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_OPENSL))\n    #define MA_HAS_OPENSL\n#endif\n#if defined(MA_SUPPORT_WEBAUDIO) && !defined(MA_NO_WEBAUDIO) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_WEBAUDIO))\n    #define MA_HAS_WEBAUDIO\n#endif\n#if defined(MA_SUPPORT_CUSTOM) && !defined(MA_NO_CUSTOM) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_CUSTOM))\n    #define MA_HAS_CUSTOM\n#endif\n#if defined(MA_SUPPORT_NULL) && !defined(MA_NO_NULL) && (!defined(MA_ENABLE_ONLY_SPECIFIC_BACKENDS) || defined(MA_ENABLE_NULL))\n    #define MA_HAS_NULL\n#endif\n\ntypedef enum\n{\n    ma_device_state_uninitialized = 0,\n    ma_device_state_stopped       = 1,  /* The device's default state after initialization. */\n    ma_device_state_started       = 2,  /* The device is started and is requesting and/or delivering audio data. */\n    ma_device_state_starting      = 3,  /* Transitioning from a stopped state to started. */\n    ma_device_state_stopping      = 4   /* Transitioning from a started state to stopped. */\n} ma_device_state;\n\nMA_ATOMIC_SAFE_TYPE_DECL(i32, 4, device_state)\n\n\n#ifdef MA_SUPPORT_WASAPI\n/* We need a IMMNotificationClient object for WASAPI. */\ntypedef struct\n{\n    void* lpVtbl;\n    ma_uint32 counter;\n    ma_device* pDevice;\n} ma_IMMNotificationClient;\n#endif\n\n/* Backend enums must be in priority order. */\ntypedef enum\n{\n    ma_backend_wasapi,\n    ma_backend_dsound,\n    ma_backend_winmm,\n    ma_backend_coreaudio,\n    ma_backend_sndio,\n    ma_backend_audio4,\n    ma_backend_oss,\n    ma_backend_pulseaudio,\n    ma_backend_alsa,\n    ma_backend_jack,\n    ma_backend_aaudio,\n    ma_backend_opensl,\n    ma_backend_webaudio,\n    ma_backend_custom,  /* <-- Custom backend, with callbacks defined by the context config. */\n    ma_backend_null     /* <-- Must always be the last item. Lowest priority, and used as the terminator for backend enumeration. */\n} ma_backend;\n\n#define MA_BACKEND_COUNT (ma_backend_null+1)\n\n\n/*\nDevice job thread. This is used by backends that require asynchronous processing of certain\noperations. It is not used by all backends.\n\nThe device job thread is made up of a thread and a job queue. You can post a job to the thread with\nma_device_job_thread_post(). The thread will do the processing of the job.\n*/\ntypedef struct\n{\n    ma_bool32 noThread; /* Set this to true if you want to process jobs yourself. */\n    ma_uint32 jobQueueCapacity;\n    ma_uint32 jobQueueFlags;\n} ma_device_job_thread_config;\n\nMA_API ma_device_job_thread_config ma_device_job_thread_config_init(void);\n\ntypedef struct\n{\n    ma_thread thread;\n    ma_job_queue jobQueue;\n    ma_bool32 _hasThread;\n} ma_device_job_thread;\n\nMA_API ma_result ma_device_job_thread_init(const ma_device_job_thread_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_device_job_thread* pJobThread);\nMA_API void ma_device_job_thread_uninit(ma_device_job_thread* pJobThread, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_device_job_thread_post(ma_device_job_thread* pJobThread, const ma_job* pJob);\nMA_API ma_result ma_device_job_thread_next(ma_device_job_thread* pJobThread, ma_job* pJob);\n\n\n\n/* Device notification types. */\ntypedef enum\n{\n    ma_device_notification_type_started,\n    ma_device_notification_type_stopped,\n    ma_device_notification_type_rerouted,\n    ma_device_notification_type_interruption_began,\n    ma_device_notification_type_interruption_ended,\n    ma_device_notification_type_unlocked\n} ma_device_notification_type;\n\ntypedef struct\n{\n    ma_device* pDevice;\n    ma_device_notification_type type;\n    union\n    {\n        struct\n        {\n            int _unused;\n        } started;\n        struct\n        {\n            int _unused;\n        } stopped;\n        struct\n        {\n            int _unused;\n        } rerouted;\n        struct\n        {\n            int _unused;\n        } interruption;\n    } data;\n} ma_device_notification;\n\n/*\nThe notification callback for when the application should be notified of a change to the device.\n\nThis callback is used for notifying the application of changes such as when the device has started,\nstopped, rerouted or an interruption has occurred. Note that not all backends will post all\nnotification types. For example, some backends will perform automatic stream routing without any\nkind of notification to the host program which means miniaudio will never know about it and will\nnever be able to fire the rerouted notification. You should keep this in mind when designing your\nprogram.\n\nThe stopped notification will *not* get fired when a device is rerouted.\n\n\nParameters\n----------\npNotification (in)\n    A pointer to a structure containing information about the event. Use the `pDevice` member of\n    this object to retrieve the relevant device. The `type` member can be used to discriminate\n    against each of the notification types.\n\n\nRemarks\n-------\nDo not restart or uninitialize the device from the callback.\n\nNot all notifications will be triggered by all backends, however the started and stopped events\nshould be reliable for all backends. Some backends do not have a good way to detect device\nstoppages due to unplugging the device which may result in the stopped callback not getting\nfired. This has been observed with at least one BSD variant.\n\nThe rerouted notification is fired *after* the reroute has occurred. The stopped notification will\n*not* get fired when a device is rerouted. The following backends are known to do automatic stream\nrerouting, but do not have a way to be notified of the change:\n\n  * DirectSound\n\nThe interruption notifications are used on mobile platforms for detecting when audio is interrupted\ndue to things like an incoming phone call. Currently this is only implemented on iOS. None of the\nAndroid backends will report this notification.\n*/\ntypedef void (* ma_device_notification_proc)(const ma_device_notification* pNotification);\n\n\n/*\nThe callback for processing audio data from the device.\n\nThe data callback is fired by miniaudio whenever the device needs to have more data delivered to a playback device, or when a capture device has some data\navailable. This is called as soon as the backend asks for more data which means it may be called with inconsistent frame counts. You cannot assume the\ncallback will be fired with a consistent frame count.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the relevant device.\n\npOutput (out)\n    A pointer to the output buffer that will receive audio data that will later be played back through the speakers. This will be non-null for a playback or\n    full-duplex device and null for a capture and loopback device.\n\npInput (in)\n    A pointer to the buffer containing input data from a recording device. This will be non-null for a capture, full-duplex or loopback device and null for a\n    playback device.\n\nframeCount (in)\n    The number of PCM frames to process. Note that this will not necessarily be equal to what you requested when you initialized the device. The\n    `periodSizeInFrames` and `periodSizeInMilliseconds` members of the device config are just hints, and are not necessarily exactly what you'll get. You must\n    not assume this will always be the same value each time the callback is fired.\n\n\nRemarks\n-------\nYou cannot stop and start the device from inside the callback or else you'll get a deadlock. You must also not uninitialize the device from inside the\ncallback. The following APIs cannot be called from inside the callback:\n\n    ma_device_init()\n    ma_device_init_ex()\n    ma_device_uninit()\n    ma_device_start()\n    ma_device_stop()\n\nThe proper way to stop the device is to call `ma_device_stop()` from a different thread, normally the main application thread.\n*/\ntypedef void (* ma_device_data_proc)(ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount);\n\n\n\n\n/*\nDEPRECATED. Use ma_device_notification_proc instead.\n\nThe callback for when the device has been stopped.\n\nThis will be called when the device is stopped explicitly with `ma_device_stop()` and also called implicitly when the device is stopped through external forces\nsuch as being unplugged or an internal error occurring.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device that has just stopped.\n\n\nRemarks\n-------\nDo not restart or uninitialize the device from the callback.\n*/\ntypedef void (* ma_stop_proc)(ma_device* pDevice);  /* DEPRECATED. Use ma_device_notification_proc instead. */\n\ntypedef enum\n{\n    ma_device_type_playback = 1,\n    ma_device_type_capture  = 2,\n    ma_device_type_duplex   = ma_device_type_playback | ma_device_type_capture, /* 3 */\n    ma_device_type_loopback = 4\n} ma_device_type;\n\ntypedef enum\n{\n    ma_share_mode_shared = 0,\n    ma_share_mode_exclusive\n} ma_share_mode;\n\n/* iOS/tvOS/watchOS session categories. */\ntypedef enum\n{\n    ma_ios_session_category_default = 0,        /* AVAudioSessionCategoryPlayAndRecord. */\n    ma_ios_session_category_none,               /* Leave the session category unchanged. */\n    ma_ios_session_category_ambient,            /* AVAudioSessionCategoryAmbient */\n    ma_ios_session_category_solo_ambient,       /* AVAudioSessionCategorySoloAmbient */\n    ma_ios_session_category_playback,           /* AVAudioSessionCategoryPlayback */\n    ma_ios_session_category_record,             /* AVAudioSessionCategoryRecord */\n    ma_ios_session_category_play_and_record,    /* AVAudioSessionCategoryPlayAndRecord */\n    ma_ios_session_category_multi_route         /* AVAudioSessionCategoryMultiRoute */\n} ma_ios_session_category;\n\n/* iOS/tvOS/watchOS session category options */\ntypedef enum\n{\n    ma_ios_session_category_option_mix_with_others                            = 0x01,   /* AVAudioSessionCategoryOptionMixWithOthers */\n    ma_ios_session_category_option_duck_others                                = 0x02,   /* AVAudioSessionCategoryOptionDuckOthers */\n    ma_ios_session_category_option_allow_bluetooth                            = 0x04,   /* AVAudioSessionCategoryOptionAllowBluetooth */\n    ma_ios_session_category_option_default_to_speaker                         = 0x08,   /* AVAudioSessionCategoryOptionDefaultToSpeaker */\n    ma_ios_session_category_option_interrupt_spoken_audio_and_mix_with_others = 0x11,   /* AVAudioSessionCategoryOptionInterruptSpokenAudioAndMixWithOthers */\n    ma_ios_session_category_option_allow_bluetooth_a2dp                       = 0x20,   /* AVAudioSessionCategoryOptionAllowBluetoothA2DP */\n    ma_ios_session_category_option_allow_air_play                             = 0x40,   /* AVAudioSessionCategoryOptionAllowAirPlay */\n} ma_ios_session_category_option;\n\n/* OpenSL stream types. */\ntypedef enum\n{\n    ma_opensl_stream_type_default = 0,              /* Leaves the stream type unset. */\n    ma_opensl_stream_type_voice,                    /* SL_ANDROID_STREAM_VOICE */\n    ma_opensl_stream_type_system,                   /* SL_ANDROID_STREAM_SYSTEM */\n    ma_opensl_stream_type_ring,                     /* SL_ANDROID_STREAM_RING */\n    ma_opensl_stream_type_media,                    /* SL_ANDROID_STREAM_MEDIA */\n    ma_opensl_stream_type_alarm,                    /* SL_ANDROID_STREAM_ALARM */\n    ma_opensl_stream_type_notification              /* SL_ANDROID_STREAM_NOTIFICATION */\n} ma_opensl_stream_type;\n\n/* OpenSL recording presets. */\ntypedef enum\n{\n    ma_opensl_recording_preset_default = 0,         /* Leaves the input preset unset. */\n    ma_opensl_recording_preset_generic,             /* SL_ANDROID_RECORDING_PRESET_GENERIC */\n    ma_opensl_recording_preset_camcorder,           /* SL_ANDROID_RECORDING_PRESET_CAMCORDER */\n    ma_opensl_recording_preset_voice_recognition,   /* SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION */\n    ma_opensl_recording_preset_voice_communication, /* SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION */\n    ma_opensl_recording_preset_voice_unprocessed    /* SL_ANDROID_RECORDING_PRESET_UNPROCESSED */\n} ma_opensl_recording_preset;\n\n/* WASAPI audio thread priority characteristics. */\ntypedef enum\n{\n    ma_wasapi_usage_default = 0,\n    ma_wasapi_usage_games,\n    ma_wasapi_usage_pro_audio,\n} ma_wasapi_usage;\n\n/* AAudio usage types. */\ntypedef enum\n{\n    ma_aaudio_usage_default = 0,                    /* Leaves the usage type unset. */\n    ma_aaudio_usage_media,                          /* AAUDIO_USAGE_MEDIA */\n    ma_aaudio_usage_voice_communication,            /* AAUDIO_USAGE_VOICE_COMMUNICATION */\n    ma_aaudio_usage_voice_communication_signalling, /* AAUDIO_USAGE_VOICE_COMMUNICATION_SIGNALLING */\n    ma_aaudio_usage_alarm,                          /* AAUDIO_USAGE_ALARM */\n    ma_aaudio_usage_notification,                   /* AAUDIO_USAGE_NOTIFICATION */\n    ma_aaudio_usage_notification_ringtone,          /* AAUDIO_USAGE_NOTIFICATION_RINGTONE */\n    ma_aaudio_usage_notification_event,             /* AAUDIO_USAGE_NOTIFICATION_EVENT */\n    ma_aaudio_usage_assistance_accessibility,       /* AAUDIO_USAGE_ASSISTANCE_ACCESSIBILITY */\n    ma_aaudio_usage_assistance_navigation_guidance, /* AAUDIO_USAGE_ASSISTANCE_NAVIGATION_GUIDANCE */\n    ma_aaudio_usage_assistance_sonification,        /* AAUDIO_USAGE_ASSISTANCE_SONIFICATION */\n    ma_aaudio_usage_game,                           /* AAUDIO_USAGE_GAME */\n    ma_aaudio_usage_assitant,                       /* AAUDIO_USAGE_ASSISTANT */\n    ma_aaudio_usage_emergency,                      /* AAUDIO_SYSTEM_USAGE_EMERGENCY */\n    ma_aaudio_usage_safety,                         /* AAUDIO_SYSTEM_USAGE_SAFETY */\n    ma_aaudio_usage_vehicle_status,                 /* AAUDIO_SYSTEM_USAGE_VEHICLE_STATUS */\n    ma_aaudio_usage_announcement                    /* AAUDIO_SYSTEM_USAGE_ANNOUNCEMENT */\n} ma_aaudio_usage;\n\n/* AAudio content types. */\ntypedef enum\n{\n    ma_aaudio_content_type_default = 0,             /* Leaves the content type unset. */\n    ma_aaudio_content_type_speech,                  /* AAUDIO_CONTENT_TYPE_SPEECH */\n    ma_aaudio_content_type_music,                   /* AAUDIO_CONTENT_TYPE_MUSIC */\n    ma_aaudio_content_type_movie,                   /* AAUDIO_CONTENT_TYPE_MOVIE */\n    ma_aaudio_content_type_sonification             /* AAUDIO_CONTENT_TYPE_SONIFICATION */\n} ma_aaudio_content_type;\n\n/* AAudio input presets. */\ntypedef enum\n{\n    ma_aaudio_input_preset_default = 0,             /* Leaves the input preset unset. */\n    ma_aaudio_input_preset_generic,                 /* AAUDIO_INPUT_PRESET_GENERIC */\n    ma_aaudio_input_preset_camcorder,               /* AAUDIO_INPUT_PRESET_CAMCORDER */\n    ma_aaudio_input_preset_voice_recognition,       /* AAUDIO_INPUT_PRESET_VOICE_RECOGNITION */\n    ma_aaudio_input_preset_voice_communication,     /* AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION */\n    ma_aaudio_input_preset_unprocessed,             /* AAUDIO_INPUT_PRESET_UNPROCESSED */\n    ma_aaudio_input_preset_voice_performance        /* AAUDIO_INPUT_PRESET_VOICE_PERFORMANCE */\n} ma_aaudio_input_preset;\n\ntypedef enum\n{\n    ma_aaudio_allow_capture_default = 0,            /* Leaves the allowed capture policy unset. */\n    ma_aaudio_allow_capture_by_all,                 /* AAUDIO_ALLOW_CAPTURE_BY_ALL */\n    ma_aaudio_allow_capture_by_system,              /* AAUDIO_ALLOW_CAPTURE_BY_SYSTEM */\n    ma_aaudio_allow_capture_by_none                 /* AAUDIO_ALLOW_CAPTURE_BY_NONE */\n} ma_aaudio_allowed_capture_policy;\n\ntypedef union\n{\n    ma_int64 counter;\n    double counterD;\n} ma_timer;\n\ntypedef union\n{\n    ma_wchar_win32 wasapi[64];      /* WASAPI uses a wchar_t string for identification. */\n    ma_uint8 dsound[16];            /* DirectSound uses a GUID for identification. */\n    /*UINT_PTR*/ ma_uint32 winmm;   /* When creating a device, WinMM expects a Win32 UINT_PTR for device identification. In practice it's actually just a UINT. */\n    char alsa[256];                 /* ALSA uses a name string for identification. */\n    char pulse[256];                /* PulseAudio uses a name string for identification. */\n    int jack;                       /* JACK always uses default devices. */\n    char coreaudio[256];            /* Core Audio uses a string for identification. */\n    char sndio[256];                /* \"snd/0\", etc. */\n    char audio4[256];               /* \"/dev/audio\", etc. */\n    char oss[64];                   /* \"dev/dsp0\", etc. \"dev/dsp\" for the default device. */\n    ma_int32 aaudio;                /* AAudio uses a 32-bit integer for identification. */\n    ma_uint32 opensl;               /* OpenSL|ES uses a 32-bit unsigned integer for identification. */\n    char webaudio[32];              /* Web Audio always uses default devices for now, but if this changes it'll be a GUID. */\n    union\n    {\n        int i;\n        char s[256];\n        void* p;\n    } custom;                       /* The custom backend could be anything. Give them a few options. */\n    int nullbackend;                /* The null backend uses an integer for device IDs. */\n} ma_device_id;\n\nMA_API ma_bool32 ma_device_id_equal(const ma_device_id* pA, const ma_device_id* pB);\n\n\ntypedef struct ma_context_config    ma_context_config;\ntypedef struct ma_device_config     ma_device_config;\ntypedef struct ma_backend_callbacks ma_backend_callbacks;\n\n#define MA_DATA_FORMAT_FLAG_EXCLUSIVE_MODE (1U << 1)    /* If set, this is supported in exclusive mode. Otherwise not natively supported by exclusive mode. */\n\n#ifndef MA_MAX_DEVICE_NAME_LENGTH\n#define MA_MAX_DEVICE_NAME_LENGTH   255\n#endif\n\ntypedef struct\n{\n    /* Basic info. This is the only information guaranteed to be filled in during device enumeration. */\n    ma_device_id id;\n    char name[MA_MAX_DEVICE_NAME_LENGTH + 1];   /* +1 for null terminator. */\n    ma_bool32 isDefault;\n\n    ma_uint32 nativeDataFormatCount;\n    struct\n    {\n        ma_format format;       /* Sample format. If set to ma_format_unknown, all sample formats are supported. */\n        ma_uint32 channels;     /* If set to 0, all channels are supported. */\n        ma_uint32 sampleRate;   /* If set to 0, all sample rates are supported. */\n        ma_uint32 flags;        /* A combination of MA_DATA_FORMAT_FLAG_* flags. */\n    } nativeDataFormats[/*ma_format_count * ma_standard_sample_rate_count * MA_MAX_CHANNELS*/ 64];  /* Not sure how big to make this. There can be *many* permutations for virtual devices which can support anything. */\n} ma_device_info;\n\nstruct ma_device_config\n{\n    ma_device_type deviceType;\n    ma_uint32 sampleRate;\n    ma_uint32 periodSizeInFrames;\n    ma_uint32 periodSizeInMilliseconds;\n    ma_uint32 periods;\n    ma_performance_profile performanceProfile;\n    ma_bool8 noPreSilencedOutputBuffer; /* When set to true, the contents of the output buffer passed into the data callback will be left undefined rather than initialized to silence. */\n    ma_bool8 noClip;                    /* When set to true, the contents of the output buffer passed into the data callback will not be clipped after returning. Only applies when the playback sample format is f32. */\n    ma_bool8 noDisableDenormals;        /* Do not disable denormals when firing the data callback. */\n    ma_bool8 noFixedSizedCallback;      /* Disables strict fixed-sized data callbacks. Setting this to true will result in the period size being treated only as a hint to the backend. This is an optimization for those who don't need fixed sized callbacks. */\n    ma_device_data_proc dataCallback;\n    ma_device_notification_proc notificationCallback;\n    ma_stop_proc stopCallback;\n    void* pUserData;\n    ma_resampler_config resampling;\n    struct\n    {\n        const ma_device_id* pDeviceID;\n        ma_format format;\n        ma_uint32 channels;\n        ma_channel* pChannelMap;\n        ma_channel_mix_mode channelMixMode;\n        ma_bool32 calculateLFEFromSpatialChannels;  /* When an output LFE channel is present, but no input LFE, set to true to set the output LFE to the average of all spatial channels (LR, FR, etc.). Ignored when an input LFE is present. */\n        ma_share_mode shareMode;\n    } playback;\n    struct\n    {\n        const ma_device_id* pDeviceID;\n        ma_format format;\n        ma_uint32 channels;\n        ma_channel* pChannelMap;\n        ma_channel_mix_mode channelMixMode;\n        ma_bool32 calculateLFEFromSpatialChannels;  /* When an output LFE channel is present, but no input LFE, set to true to set the output LFE to the average of all spatial channels (LR, FR, etc.). Ignored when an input LFE is present. */\n        ma_share_mode shareMode;\n    } capture;\n\n    struct\n    {\n        ma_wasapi_usage usage;              /* When configured, uses Avrt APIs to set the thread characteristics. */\n        ma_bool8 noAutoConvertSRC;          /* When set to true, disables the use of AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM. */\n        ma_bool8 noDefaultQualitySRC;       /* When set to true, disables the use of AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY. */\n        ma_bool8 noAutoStreamRouting;       /* Disables automatic stream routing. */\n        ma_bool8 noHardwareOffloading;      /* Disables WASAPI's hardware offloading feature. */\n        ma_uint32 loopbackProcessID;        /* The process ID to include or exclude for loopback mode. Set to 0 to capture audio from all processes. Ignored when an explicit device ID is specified. */\n        ma_bool8 loopbackProcessExclude;    /* When set to true, excludes the process specified by loopbackProcessID. By default, the process will be included. */\n    } wasapi;\n    struct\n    {\n        ma_bool32 noMMap;           /* Disables MMap mode. */\n        ma_bool32 noAutoFormat;     /* Opens the ALSA device with SND_PCM_NO_AUTO_FORMAT. */\n        ma_bool32 noAutoChannels;   /* Opens the ALSA device with SND_PCM_NO_AUTO_CHANNELS. */\n        ma_bool32 noAutoResample;   /* Opens the ALSA device with SND_PCM_NO_AUTO_RESAMPLE. */\n    } alsa;\n    struct\n    {\n        const char* pStreamNamePlayback;\n        const char* pStreamNameCapture;\n        int channelMap;\n    } pulse;\n    struct\n    {\n        ma_bool32 allowNominalSampleRateChange; /* Desktop only. When enabled, allows changing of the sample rate at the operating system level. */\n    } coreaudio;\n    struct\n    {\n        ma_opensl_stream_type streamType;\n        ma_opensl_recording_preset recordingPreset;\n        ma_bool32 enableCompatibilityWorkarounds;\n    } opensl;\n    struct\n    {\n        ma_aaudio_usage usage;\n        ma_aaudio_content_type contentType;\n        ma_aaudio_input_preset inputPreset;\n        ma_aaudio_allowed_capture_policy allowedCapturePolicy;\n        ma_bool32 noAutoStartAfterReroute;\n        ma_bool32 enableCompatibilityWorkarounds;\n        ma_bool32 allowSetBufferCapacity;\n    } aaudio;\n};\n\n\n/*\nThe callback for handling device enumeration. This is fired from `ma_context_enumerate_devices()`.\n\n\nParameters\n----------\npContext (in)\n    A pointer to the context performing the enumeration.\n\ndeviceType (in)\n    The type of the device being enumerated. This will always be either `ma_device_type_playback` or `ma_device_type_capture`.\n\npInfo (in)\n    A pointer to a `ma_device_info` containing the ID and name of the enumerated device. Note that this will not include detailed information about the device,\n    only basic information (ID and name). The reason for this is that it would otherwise require opening the backend device to probe for the information which\n    is too inefficient.\n\npUserData (in)\n    The user data pointer passed into `ma_context_enumerate_devices()`.\n*/\ntypedef ma_bool32 (* ma_enum_devices_callback_proc)(ma_context* pContext, ma_device_type deviceType, const ma_device_info* pInfo, void* pUserData);\n\n\n/*\nDescribes some basic details about a playback or capture device.\n*/\ntypedef struct\n{\n    const ma_device_id* pDeviceID;\n    ma_share_mode shareMode;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_channel channelMap[MA_MAX_CHANNELS];\n    ma_uint32 periodSizeInFrames;\n    ma_uint32 periodSizeInMilliseconds;\n    ma_uint32 periodCount;\n} ma_device_descriptor;\n\n/*\nThese are the callbacks required to be implemented for a backend. These callbacks are grouped into two parts: context and device. There is one context\nto many devices. A device is created from a context.\n\nThe general flow goes like this:\n\n  1) A context is created with `onContextInit()`\n     1a) Available devices can be enumerated with `onContextEnumerateDevices()` if required.\n     1b) Detailed information about a device can be queried with `onContextGetDeviceInfo()` if required.\n  2) A device is created from the context that was created in the first step using `onDeviceInit()`, and optionally a device ID that was\n     selected from device enumeration via `onContextEnumerateDevices()`.\n  3) A device is started or stopped with `onDeviceStart()` / `onDeviceStop()`\n  4) Data is delivered to and from the device by the backend. This is always done based on the native format returned by the prior call\n     to `onDeviceInit()`. Conversion between the device's native format and the format requested by the application will be handled by\n     miniaudio internally.\n\nInitialization of the context is quite simple. You need to do any necessary initialization of internal objects and then output the\ncallbacks defined in this structure.\n\nOnce the context has been initialized you can initialize a device. Before doing so, however, the application may want to know which\nphysical devices are available. This is where `onContextEnumerateDevices()` comes in. This is fairly simple. For each device, fire the\ngiven callback with, at a minimum, the basic information filled out in `ma_device_info`. When the callback returns `MA_FALSE`, enumeration\nneeds to stop and the `onContextEnumerateDevices()` function returns with a success code.\n\nDetailed device information can be retrieved from a device ID using `onContextGetDeviceInfo()`. This takes as input the device type and ID,\nand on output returns detailed information about the device in `ma_device_info`. The `onContextGetDeviceInfo()` callback must handle the\ncase when the device ID is NULL, in which case information about the default device needs to be retrieved.\n\nOnce the context has been created and the device ID retrieved (if using anything other than the default device), the device can be created.\nThis is a little bit more complicated than initialization of the context due to its more complicated configuration. When initializing a\ndevice, a duplex device may be requested. This means a separate data format needs to be specified for both playback and capture. On input,\nthe data format is set to what the application wants. On output it's set to the native format which should match as closely as possible to\nthe requested format. The conversion between the format requested by the application and the device's native format will be handled\ninternally by miniaudio.\n\nOn input, if the sample format is set to `ma_format_unknown`, the backend is free to use whatever sample format it desires, so long as it's\nsupported by miniaudio. When the channel count is set to 0, the backend should use the device's native channel count. The same applies for\nsample rate. For the channel map, the default should be used when `ma_channel_map_is_blank()` returns true (all channels set to\n`MA_CHANNEL_NONE`). On input, the `periodSizeInFrames` or `periodSizeInMilliseconds` option should always be set. The backend should\ninspect both of these variables. If `periodSizeInFrames` is set, it should take priority, otherwise it needs to be derived from the period\nsize in milliseconds (`periodSizeInMilliseconds`) and the sample rate, keeping in mind that the sample rate may be 0, in which case the\nsample rate will need to be determined before calculating the period size in frames. On output, all members of the `ma_device_descriptor`\nobject should be set to a valid value, except for `periodSizeInMilliseconds` which is optional (`periodSizeInFrames` *must* be set).\n\nStarting and stopping of the device is done with `onDeviceStart()` and `onDeviceStop()` and should be self-explanatory. If the backend uses\nasynchronous reading and writing, `onDeviceStart()` and `onDeviceStop()` should always be implemented.\n\nThe handling of data delivery between the application and the device is the most complicated part of the process. To make this a bit\neasier, some helper callbacks are available. If the backend uses a blocking read/write style of API, the `onDeviceRead()` and\n`onDeviceWrite()` callbacks can optionally be implemented. These are blocking and work just like reading and writing from a file. If the\nbackend uses a callback for data delivery, that callback must call `ma_device_handle_backend_data_callback()` from within its callback.\nThis allows miniaudio to then process any necessary data conversion and then pass it to the miniaudio data callback.\n\nIf the backend requires absolute flexibility with its data delivery, it can optionally implement the `onDeviceDataLoop()` callback\nwhich will allow it to implement the logic that will run on the audio thread. This is much more advanced and is completely optional.\n\nThe audio thread should run data delivery logic in a loop while `ma_device_get_state() == ma_device_state_started` and no errors have been\nencountered. Do not start or stop the device here. That will be handled from outside the `onDeviceDataLoop()` callback.\n\nThe invocation of the `onDeviceDataLoop()` callback will be handled by miniaudio. When you start the device, miniaudio will fire this\ncallback. When the device is stopped, the `ma_device_get_state() == ma_device_state_started` condition will fail and the loop will be terminated\nwhich will then fall through to the part that stops the device. For an example on how to implement the `onDeviceDataLoop()` callback,\nlook at `ma_device_audio_thread__default_read_write()`. Implement the `onDeviceDataLoopWakeup()` callback if you need a mechanism to\nwake up the audio thread.\n\nIf the backend supports an optimized retrieval of device information from an initialized `ma_device` object, it should implement the\n`onDeviceGetInfo()` callback. This is optional, in which case it will fall back to `onContextGetDeviceInfo()` which is less efficient.\n*/\nstruct ma_backend_callbacks\n{\n    ma_result (* onContextInit)(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks);\n    ma_result (* onContextUninit)(ma_context* pContext);\n    ma_result (* onContextEnumerateDevices)(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData);\n    ma_result (* onContextGetDeviceInfo)(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo);\n    ma_result (* onDeviceInit)(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture);\n    ma_result (* onDeviceUninit)(ma_device* pDevice);\n    ma_result (* onDeviceStart)(ma_device* pDevice);\n    ma_result (* onDeviceStop)(ma_device* pDevice);\n    ma_result (* onDeviceRead)(ma_device* pDevice, void* pFrames, ma_uint32 frameCount, ma_uint32* pFramesRead);\n    ma_result (* onDeviceWrite)(ma_device* pDevice, const void* pFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten);\n    ma_result (* onDeviceDataLoop)(ma_device* pDevice);\n    ma_result (* onDeviceDataLoopWakeup)(ma_device* pDevice);\n    ma_result (* onDeviceGetInfo)(ma_device* pDevice, ma_device_type type, ma_device_info* pDeviceInfo);\n};\n\nstruct ma_context_config\n{\n    ma_log* pLog;\n    ma_thread_priority threadPriority;\n    size_t threadStackSize;\n    void* pUserData;\n    ma_allocation_callbacks allocationCallbacks;\n    struct\n    {\n        ma_handle hWnd; /* HWND. Optional window handle to pass into SetCooperativeLevel(). Will default to the foreground window, and if that fails, the desktop window. */\n    } dsound;\n    struct\n    {\n        ma_bool32 useVerboseDeviceEnumeration;\n    } alsa;\n    struct\n    {\n        const char* pApplicationName;\n        const char* pServerName;\n        ma_bool32 tryAutoSpawn; /* Enables autospawning of the PulseAudio daemon if necessary. */\n    } pulse;\n    struct\n    {\n        ma_ios_session_category sessionCategory;\n        ma_uint32 sessionCategoryOptions;\n        ma_bool32 noAudioSessionActivate;   /* iOS only. When set to true, does not perform an explicit [[AVAudioSession sharedInstace] setActive:true] on initialization. */\n        ma_bool32 noAudioSessionDeactivate; /* iOS only. When set to true, does not perform an explicit [[AVAudioSession sharedInstace] setActive:false] on uninitialization. */\n    } coreaudio;\n    struct\n    {\n        const char* pClientName;\n        ma_bool32 tryStartServer;\n    } jack;\n    ma_backend_callbacks custom;\n};\n\n/* WASAPI specific structure for some commands which must run on a common thread due to bugs in WASAPI. */\ntypedef struct\n{\n    int code;\n    ma_event* pEvent;   /* This will be signalled when the event is complete. */\n    union\n    {\n        struct\n        {\n            int _unused;\n        } quit;\n        struct\n        {\n            ma_device_type deviceType;\n            void* pAudioClient;\n            void** ppAudioClientService;\n            ma_result* pResult; /* The result from creating the audio client service. */\n        } createAudioClient;\n        struct\n        {\n            ma_device* pDevice;\n            ma_device_type deviceType;\n        } releaseAudioClient;\n    } data;\n} ma_context_command__wasapi;\n\nstruct ma_context\n{\n    ma_backend_callbacks callbacks;\n    ma_backend backend;                 /* DirectSound, ALSA, etc. */\n    ma_log* pLog;\n    ma_log log; /* Only used if the log is owned by the context. The pLog member will be set to &log in this case. */\n    ma_thread_priority threadPriority;\n    size_t threadStackSize;\n    void* pUserData;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_mutex deviceEnumLock;            /* Used to make ma_context_get_devices() thread safe. */\n    ma_mutex deviceInfoLock;            /* Used to make ma_context_get_device_info() thread safe. */\n    ma_uint32 deviceInfoCapacity;       /* Total capacity of pDeviceInfos. */\n    ma_uint32 playbackDeviceInfoCount;\n    ma_uint32 captureDeviceInfoCount;\n    ma_device_info* pDeviceInfos;       /* Playback devices first, then capture. */\n\n    union\n    {\n#ifdef MA_SUPPORT_WASAPI\n        struct\n        {\n            ma_thread commandThread;\n            ma_mutex commandLock;\n            ma_semaphore commandSem;\n            ma_uint32 commandIndex;\n            ma_uint32 commandCount;\n            ma_context_command__wasapi commands[4];\n            ma_handle hAvrt;\n            ma_proc AvSetMmThreadCharacteristicsA;\n            ma_proc AvRevertMmThreadcharacteristics;\n            ma_handle hMMDevapi;\n            ma_proc ActivateAudioInterfaceAsync;\n        } wasapi;\n#endif\n#ifdef MA_SUPPORT_DSOUND\n        struct\n        {\n            ma_handle hWnd; /* Can be null. */\n            ma_handle hDSoundDLL;\n            ma_proc DirectSoundCreate;\n            ma_proc DirectSoundEnumerateA;\n            ma_proc DirectSoundCaptureCreate;\n            ma_proc DirectSoundCaptureEnumerateA;\n        } dsound;\n#endif\n#ifdef MA_SUPPORT_WINMM\n        struct\n        {\n            ma_handle hWinMM;\n            ma_proc waveOutGetNumDevs;\n            ma_proc waveOutGetDevCapsA;\n            ma_proc waveOutOpen;\n            ma_proc waveOutClose;\n            ma_proc waveOutPrepareHeader;\n            ma_proc waveOutUnprepareHeader;\n            ma_proc waveOutWrite;\n            ma_proc waveOutReset;\n            ma_proc waveInGetNumDevs;\n            ma_proc waveInGetDevCapsA;\n            ma_proc waveInOpen;\n            ma_proc waveInClose;\n            ma_proc waveInPrepareHeader;\n            ma_proc waveInUnprepareHeader;\n            ma_proc waveInAddBuffer;\n            ma_proc waveInStart;\n            ma_proc waveInReset;\n        } winmm;\n#endif\n#ifdef MA_SUPPORT_ALSA\n        struct\n        {\n            ma_handle asoundSO;\n            ma_proc snd_pcm_open;\n            ma_proc snd_pcm_close;\n            ma_proc snd_pcm_hw_params_sizeof;\n            ma_proc snd_pcm_hw_params_any;\n            ma_proc snd_pcm_hw_params_set_format;\n            ma_proc snd_pcm_hw_params_set_format_first;\n            ma_proc snd_pcm_hw_params_get_format_mask;\n            ma_proc snd_pcm_hw_params_set_channels;\n            ma_proc snd_pcm_hw_params_set_channels_near;\n            ma_proc snd_pcm_hw_params_set_channels_minmax;\n            ma_proc snd_pcm_hw_params_set_rate_resample;\n            ma_proc snd_pcm_hw_params_set_rate;\n            ma_proc snd_pcm_hw_params_set_rate_near;\n            ma_proc snd_pcm_hw_params_set_rate_minmax;\n            ma_proc snd_pcm_hw_params_set_buffer_size_near;\n            ma_proc snd_pcm_hw_params_set_periods_near;\n            ma_proc snd_pcm_hw_params_set_access;\n            ma_proc snd_pcm_hw_params_get_format;\n            ma_proc snd_pcm_hw_params_get_channels;\n            ma_proc snd_pcm_hw_params_get_channels_min;\n            ma_proc snd_pcm_hw_params_get_channels_max;\n            ma_proc snd_pcm_hw_params_get_rate;\n            ma_proc snd_pcm_hw_params_get_rate_min;\n            ma_proc snd_pcm_hw_params_get_rate_max;\n            ma_proc snd_pcm_hw_params_get_buffer_size;\n            ma_proc snd_pcm_hw_params_get_periods;\n            ma_proc snd_pcm_hw_params_get_access;\n            ma_proc snd_pcm_hw_params_test_format;\n            ma_proc snd_pcm_hw_params_test_channels;\n            ma_proc snd_pcm_hw_params_test_rate;\n            ma_proc snd_pcm_hw_params;\n            ma_proc snd_pcm_sw_params_sizeof;\n            ma_proc snd_pcm_sw_params_current;\n            ma_proc snd_pcm_sw_params_get_boundary;\n            ma_proc snd_pcm_sw_params_set_avail_min;\n            ma_proc snd_pcm_sw_params_set_start_threshold;\n            ma_proc snd_pcm_sw_params_set_stop_threshold;\n            ma_proc snd_pcm_sw_params;\n            ma_proc snd_pcm_format_mask_sizeof;\n            ma_proc snd_pcm_format_mask_test;\n            ma_proc snd_pcm_get_chmap;\n            ma_proc snd_pcm_state;\n            ma_proc snd_pcm_prepare;\n            ma_proc snd_pcm_start;\n            ma_proc snd_pcm_drop;\n            ma_proc snd_pcm_drain;\n            ma_proc snd_pcm_reset;\n            ma_proc snd_device_name_hint;\n            ma_proc snd_device_name_get_hint;\n            ma_proc snd_card_get_index;\n            ma_proc snd_device_name_free_hint;\n            ma_proc snd_pcm_mmap_begin;\n            ma_proc snd_pcm_mmap_commit;\n            ma_proc snd_pcm_recover;\n            ma_proc snd_pcm_readi;\n            ma_proc snd_pcm_writei;\n            ma_proc snd_pcm_avail;\n            ma_proc snd_pcm_avail_update;\n            ma_proc snd_pcm_wait;\n            ma_proc snd_pcm_nonblock;\n            ma_proc snd_pcm_info;\n            ma_proc snd_pcm_info_sizeof;\n            ma_proc snd_pcm_info_get_name;\n            ma_proc snd_pcm_poll_descriptors;\n            ma_proc snd_pcm_poll_descriptors_count;\n            ma_proc snd_pcm_poll_descriptors_revents;\n            ma_proc snd_config_update_free_global;\n\n            ma_mutex internalDeviceEnumLock;\n            ma_bool32 useVerboseDeviceEnumeration;\n        } alsa;\n#endif\n#ifdef MA_SUPPORT_PULSEAUDIO\n        struct\n        {\n            ma_handle pulseSO;\n            ma_proc pa_mainloop_new;\n            ma_proc pa_mainloop_free;\n            ma_proc pa_mainloop_quit;\n            ma_proc pa_mainloop_get_api;\n            ma_proc pa_mainloop_iterate;\n            ma_proc pa_mainloop_wakeup;\n            ma_proc pa_threaded_mainloop_new;\n            ma_proc pa_threaded_mainloop_free;\n            ma_proc pa_threaded_mainloop_start;\n            ma_proc pa_threaded_mainloop_stop;\n            ma_proc pa_threaded_mainloop_lock;\n            ma_proc pa_threaded_mainloop_unlock;\n            ma_proc pa_threaded_mainloop_wait;\n            ma_proc pa_threaded_mainloop_signal;\n            ma_proc pa_threaded_mainloop_accept;\n            ma_proc pa_threaded_mainloop_get_retval;\n            ma_proc pa_threaded_mainloop_get_api;\n            ma_proc pa_threaded_mainloop_in_thread;\n            ma_proc pa_threaded_mainloop_set_name;\n            ma_proc pa_context_new;\n            ma_proc pa_context_unref;\n            ma_proc pa_context_connect;\n            ma_proc pa_context_disconnect;\n            ma_proc pa_context_set_state_callback;\n            ma_proc pa_context_get_state;\n            ma_proc pa_context_get_sink_info_list;\n            ma_proc pa_context_get_source_info_list;\n            ma_proc pa_context_get_sink_info_by_name;\n            ma_proc pa_context_get_source_info_by_name;\n            ma_proc pa_operation_unref;\n            ma_proc pa_operation_get_state;\n            ma_proc pa_channel_map_init_extend;\n            ma_proc pa_channel_map_valid;\n            ma_proc pa_channel_map_compatible;\n            ma_proc pa_stream_new;\n            ma_proc pa_stream_unref;\n            ma_proc pa_stream_connect_playback;\n            ma_proc pa_stream_connect_record;\n            ma_proc pa_stream_disconnect;\n            ma_proc pa_stream_get_state;\n            ma_proc pa_stream_get_sample_spec;\n            ma_proc pa_stream_get_channel_map;\n            ma_proc pa_stream_get_buffer_attr;\n            ma_proc pa_stream_set_buffer_attr;\n            ma_proc pa_stream_get_device_name;\n            ma_proc pa_stream_set_write_callback;\n            ma_proc pa_stream_set_read_callback;\n            ma_proc pa_stream_set_suspended_callback;\n            ma_proc pa_stream_set_moved_callback;\n            ma_proc pa_stream_is_suspended;\n            ma_proc pa_stream_flush;\n            ma_proc pa_stream_drain;\n            ma_proc pa_stream_is_corked;\n            ma_proc pa_stream_cork;\n            ma_proc pa_stream_trigger;\n            ma_proc pa_stream_begin_write;\n            ma_proc pa_stream_write;\n            ma_proc pa_stream_peek;\n            ma_proc pa_stream_drop;\n            ma_proc pa_stream_writable_size;\n            ma_proc pa_stream_readable_size;\n\n            /*pa_mainloop**/ ma_ptr pMainLoop;\n            /*pa_context**/ ma_ptr pPulseContext;\n            char* pApplicationName; /* Set when the context is initialized. Used by devices for their local pa_context objects. */\n            char* pServerName;      /* Set when the context is initialized. Used by devices for their local pa_context objects. */\n        } pulse;\n#endif\n#ifdef MA_SUPPORT_JACK\n        struct\n        {\n            ma_handle jackSO;\n            ma_proc jack_client_open;\n            ma_proc jack_client_close;\n            ma_proc jack_client_name_size;\n            ma_proc jack_set_process_callback;\n            ma_proc jack_set_buffer_size_callback;\n            ma_proc jack_on_shutdown;\n            ma_proc jack_get_sample_rate;\n            ma_proc jack_get_buffer_size;\n            ma_proc jack_get_ports;\n            ma_proc jack_activate;\n            ma_proc jack_deactivate;\n            ma_proc jack_connect;\n            ma_proc jack_port_register;\n            ma_proc jack_port_name;\n            ma_proc jack_port_get_buffer;\n            ma_proc jack_free;\n\n            char* pClientName;\n            ma_bool32 tryStartServer;\n        } jack;\n#endif\n#ifdef MA_SUPPORT_COREAUDIO\n        struct\n        {\n            ma_handle hCoreFoundation;\n            ma_proc CFStringGetCString;\n            ma_proc CFRelease;\n\n            ma_handle hCoreAudio;\n            ma_proc AudioObjectGetPropertyData;\n            ma_proc AudioObjectGetPropertyDataSize;\n            ma_proc AudioObjectSetPropertyData;\n            ma_proc AudioObjectAddPropertyListener;\n            ma_proc AudioObjectRemovePropertyListener;\n\n            ma_handle hAudioUnit;  /* Could possibly be set to AudioToolbox on later versions of macOS. */\n            ma_proc AudioComponentFindNext;\n            ma_proc AudioComponentInstanceDispose;\n            ma_proc AudioComponentInstanceNew;\n            ma_proc AudioOutputUnitStart;\n            ma_proc AudioOutputUnitStop;\n            ma_proc AudioUnitAddPropertyListener;\n            ma_proc AudioUnitGetPropertyInfo;\n            ma_proc AudioUnitGetProperty;\n            ma_proc AudioUnitSetProperty;\n            ma_proc AudioUnitInitialize;\n            ma_proc AudioUnitRender;\n\n            /*AudioComponent*/ ma_ptr component;\n            ma_bool32 noAudioSessionDeactivate; /* For tracking whether or not the iOS audio session should be explicitly deactivated. Set from the config in ma_context_init__coreaudio(). */\n        } coreaudio;\n#endif\n#ifdef MA_SUPPORT_SNDIO\n        struct\n        {\n            ma_handle sndioSO;\n            ma_proc sio_open;\n            ma_proc sio_close;\n            ma_proc sio_setpar;\n            ma_proc sio_getpar;\n            ma_proc sio_getcap;\n            ma_proc sio_start;\n            ma_proc sio_stop;\n            ma_proc sio_read;\n            ma_proc sio_write;\n            ma_proc sio_onmove;\n            ma_proc sio_nfds;\n            ma_proc sio_pollfd;\n            ma_proc sio_revents;\n            ma_proc sio_eof;\n            ma_proc sio_setvol;\n            ma_proc sio_onvol;\n            ma_proc sio_initpar;\n        } sndio;\n#endif\n#ifdef MA_SUPPORT_AUDIO4\n        struct\n        {\n            int _unused;\n        } audio4;\n#endif\n#ifdef MA_SUPPORT_OSS\n        struct\n        {\n            int versionMajor;\n            int versionMinor;\n        } oss;\n#endif\n#ifdef MA_SUPPORT_AAUDIO\n        struct\n        {\n            ma_handle hAAudio; /* libaaudio.so */\n            ma_proc AAudio_createStreamBuilder;\n            ma_proc AAudioStreamBuilder_delete;\n            ma_proc AAudioStreamBuilder_setDeviceId;\n            ma_proc AAudioStreamBuilder_setDirection;\n            ma_proc AAudioStreamBuilder_setSharingMode;\n            ma_proc AAudioStreamBuilder_setFormat;\n            ma_proc AAudioStreamBuilder_setChannelCount;\n            ma_proc AAudioStreamBuilder_setSampleRate;\n            ma_proc AAudioStreamBuilder_setBufferCapacityInFrames;\n            ma_proc AAudioStreamBuilder_setFramesPerDataCallback;\n            ma_proc AAudioStreamBuilder_setDataCallback;\n            ma_proc AAudioStreamBuilder_setErrorCallback;\n            ma_proc AAudioStreamBuilder_setPerformanceMode;\n            ma_proc AAudioStreamBuilder_setUsage;\n            ma_proc AAudioStreamBuilder_setContentType;\n            ma_proc AAudioStreamBuilder_setInputPreset;\n            ma_proc AAudioStreamBuilder_setAllowedCapturePolicy;\n            ma_proc AAudioStreamBuilder_openStream;\n            ma_proc AAudioStream_close;\n            ma_proc AAudioStream_getState;\n            ma_proc AAudioStream_waitForStateChange;\n            ma_proc AAudioStream_getFormat;\n            ma_proc AAudioStream_getChannelCount;\n            ma_proc AAudioStream_getSampleRate;\n            ma_proc AAudioStream_getBufferCapacityInFrames;\n            ma_proc AAudioStream_getFramesPerDataCallback;\n            ma_proc AAudioStream_getFramesPerBurst;\n            ma_proc AAudioStream_requestStart;\n            ma_proc AAudioStream_requestStop;\n            ma_device_job_thread jobThread; /* For processing operations outside of the error callback, specifically device disconnections and rerouting. */\n        } aaudio;\n#endif\n#ifdef MA_SUPPORT_OPENSL\n        struct\n        {\n            ma_handle libOpenSLES;\n            ma_handle SL_IID_ENGINE;\n            ma_handle SL_IID_AUDIOIODEVICECAPABILITIES;\n            ma_handle SL_IID_ANDROIDSIMPLEBUFFERQUEUE;\n            ma_handle SL_IID_RECORD;\n            ma_handle SL_IID_PLAY;\n            ma_handle SL_IID_OUTPUTMIX;\n            ma_handle SL_IID_ANDROIDCONFIGURATION;\n            ma_proc   slCreateEngine;\n        } opensl;\n#endif\n#ifdef MA_SUPPORT_WEBAUDIO\n        struct\n        {\n            int _unused;\n        } webaudio;\n#endif\n#ifdef MA_SUPPORT_NULL\n        struct\n        {\n            int _unused;\n        } null_backend;\n#endif\n    };\n\n    union\n    {\n#if defined(MA_WIN32)\n        struct\n        {\n            /*HMODULE*/ ma_handle hOle32DLL;\n            ma_proc CoInitialize;\n            ma_proc CoInitializeEx;\n            ma_proc CoUninitialize;\n            ma_proc CoCreateInstance;\n            ma_proc CoTaskMemFree;\n            ma_proc PropVariantClear;\n            ma_proc StringFromGUID2;\n\n            /*HMODULE*/ ma_handle hUser32DLL;\n            ma_proc GetForegroundWindow;\n            ma_proc GetDesktopWindow;\n\n            /*HMODULE*/ ma_handle hAdvapi32DLL;\n            ma_proc RegOpenKeyExA;\n            ma_proc RegCloseKey;\n            ma_proc RegQueryValueExA;\n\n            /*HRESULT*/ long CoInitializeResult;\n        } win32;\n#endif\n#ifdef MA_POSIX\n        struct\n        {\n            int _unused;\n        } posix;\n#endif\n        int _unused;\n    };\n};\n\nstruct ma_device\n{\n    ma_context* pContext;\n    ma_device_type type;\n    ma_uint32 sampleRate;\n    ma_atomic_device_state state;               /* The state of the device is variable and can change at any time on any thread. Must be used atomically. */\n    ma_device_data_proc onData;                 /* Set once at initialization time and should not be changed after. */\n    ma_device_notification_proc onNotification; /* Set once at initialization time and should not be changed after. */\n    ma_stop_proc onStop;                        /* DEPRECATED. Use the notification callback instead. Set once at initialization time and should not be changed after. */\n    void* pUserData;                            /* Application defined data. */\n    ma_mutex startStopLock;\n    ma_event wakeupEvent;\n    ma_event startEvent;\n    ma_event stopEvent;\n    ma_thread thread;\n    ma_result workResult;                       /* This is set by the worker thread after it's finished doing a job. */\n    ma_bool8 isOwnerOfContext;                  /* When set to true, uninitializing the device will also uninitialize the context. Set to true when NULL is passed into ma_device_init(). */\n    ma_bool8 noPreSilencedOutputBuffer;\n    ma_bool8 noClip;\n    ma_bool8 noDisableDenormals;\n    ma_bool8 noFixedSizedCallback;\n    ma_atomic_float masterVolumeFactor;         /* Linear 0..1. Can be read and written simultaneously by different threads. Must be used atomically. */\n    ma_duplex_rb duplexRB;                      /* Intermediary buffer for duplex device on asynchronous backends. */\n    struct\n    {\n        ma_resample_algorithm algorithm;\n        ma_resampling_backend_vtable* pBackendVTable;\n        void* pBackendUserData;\n        struct\n        {\n            ma_uint32 lpfOrder;\n        } linear;\n    } resampling;\n    struct\n    {\n        ma_device_id* pID;                  /* Set to NULL if using default ID, otherwise set to the address of \"id\". */\n        ma_device_id id;                    /* If using an explicit device, will be set to a copy of the ID used for initialization. Otherwise cleared to 0. */\n        char name[MA_MAX_DEVICE_NAME_LENGTH + 1];                     /* Maybe temporary. Likely to be replaced with a query API. */\n        ma_share_mode shareMode;            /* Set to whatever was passed in when the device was initialized. */\n        ma_format format;\n        ma_uint32 channels;\n        ma_channel channelMap[MA_MAX_CHANNELS];\n        ma_format internalFormat;\n        ma_uint32 internalChannels;\n        ma_uint32 internalSampleRate;\n        ma_channel internalChannelMap[MA_MAX_CHANNELS];\n        ma_uint32 internalPeriodSizeInFrames;\n        ma_uint32 internalPeriods;\n        ma_channel_mix_mode channelMixMode;\n        ma_bool32 calculateLFEFromSpatialChannels;\n        ma_data_converter converter;\n        void* pIntermediaryBuffer;          /* For implementing fixed sized buffer callbacks. Will be null if using variable sized callbacks. */\n        ma_uint32 intermediaryBufferCap;\n        ma_uint32 intermediaryBufferLen;    /* How many valid frames are sitting in the intermediary buffer. */\n        void* pInputCache;                  /* In external format. Can be null. */\n        ma_uint64 inputCacheCap;\n        ma_uint64 inputCacheConsumed;\n        ma_uint64 inputCacheRemaining;\n    } playback;\n    struct\n    {\n        ma_device_id* pID;                  /* Set to NULL if using default ID, otherwise set to the address of \"id\". */\n        ma_device_id id;                    /* If using an explicit device, will be set to a copy of the ID used for initialization. Otherwise cleared to 0. */\n        char name[MA_MAX_DEVICE_NAME_LENGTH + 1];                     /* Maybe temporary. Likely to be replaced with a query API. */\n        ma_share_mode shareMode;            /* Set to whatever was passed in when the device was initialized. */\n        ma_format format;\n        ma_uint32 channels;\n        ma_channel channelMap[MA_MAX_CHANNELS];\n        ma_format internalFormat;\n        ma_uint32 internalChannels;\n        ma_uint32 internalSampleRate;\n        ma_channel internalChannelMap[MA_MAX_CHANNELS];\n        ma_uint32 internalPeriodSizeInFrames;\n        ma_uint32 internalPeriods;\n        ma_channel_mix_mode channelMixMode;\n        ma_bool32 calculateLFEFromSpatialChannels;\n        ma_data_converter converter;\n        void* pIntermediaryBuffer;          /* For implementing fixed sized buffer callbacks. Will be null if using variable sized callbacks. */\n        ma_uint32 intermediaryBufferCap;\n        ma_uint32 intermediaryBufferLen;    /* How many valid frames are sitting in the intermediary buffer. */\n    } capture;\n\n    union\n    {\n#ifdef MA_SUPPORT_WASAPI\n        struct\n        {\n            /*IAudioClient**/ ma_ptr pAudioClientPlayback;\n            /*IAudioClient**/ ma_ptr pAudioClientCapture;\n            /*IAudioRenderClient**/ ma_ptr pRenderClient;\n            /*IAudioCaptureClient**/ ma_ptr pCaptureClient;\n            /*IMMDeviceEnumerator**/ ma_ptr pDeviceEnumerator;      /* Used for IMMNotificationClient notifications. Required for detecting default device changes. */\n            ma_IMMNotificationClient notificationClient;\n            /*HANDLE*/ ma_handle hEventPlayback;                    /* Auto reset. Initialized to signaled. */\n            /*HANDLE*/ ma_handle hEventCapture;                     /* Auto reset. Initialized to unsignaled. */\n            ma_uint32 actualBufferSizeInFramesPlayback;             /* Value from GetBufferSize(). internalPeriodSizeInFrames is not set to the _actual_ buffer size when low-latency shared mode is being used due to the way the IAudioClient3 API works. */\n            ma_uint32 actualBufferSizeInFramesCapture;\n            ma_uint32 originalPeriodSizeInFrames;\n            ma_uint32 originalPeriodSizeInMilliseconds;\n            ma_uint32 originalPeriods;\n            ma_performance_profile originalPerformanceProfile;\n            ma_uint32 periodSizeInFramesPlayback;\n            ma_uint32 periodSizeInFramesCapture;\n            void* pMappedBufferCapture;\n            ma_uint32 mappedBufferCaptureCap;\n            ma_uint32 mappedBufferCaptureLen;\n            void* pMappedBufferPlayback;\n            ma_uint32 mappedBufferPlaybackCap;\n            ma_uint32 mappedBufferPlaybackLen;\n            ma_atomic_bool32 isStartedCapture;                      /* Can be read and written simultaneously across different threads. Must be used atomically, and must be 32-bit. */\n            ma_atomic_bool32 isStartedPlayback;                     /* Can be read and written simultaneously across different threads. Must be used atomically, and must be 32-bit. */\n            ma_uint32 loopbackProcessID;\n            ma_bool8 loopbackProcessExclude;\n            ma_bool8 noAutoConvertSRC;                              /* When set to true, disables the use of AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM. */\n            ma_bool8 noDefaultQualitySRC;                           /* When set to true, disables the use of AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY. */\n            ma_bool8 noHardwareOffloading;\n            ma_bool8 allowCaptureAutoStreamRouting;\n            ma_bool8 allowPlaybackAutoStreamRouting;\n            ma_bool8 isDetachedPlayback;\n            ma_bool8 isDetachedCapture;\n            ma_wasapi_usage usage;\n            void* hAvrtHandle;\n            ma_mutex rerouteLock;\n        } wasapi;\n#endif\n#ifdef MA_SUPPORT_DSOUND\n        struct\n        {\n            /*LPDIRECTSOUND*/ ma_ptr pPlayback;\n            /*LPDIRECTSOUNDBUFFER*/ ma_ptr pPlaybackPrimaryBuffer;\n            /*LPDIRECTSOUNDBUFFER*/ ma_ptr pPlaybackBuffer;\n            /*LPDIRECTSOUNDCAPTURE*/ ma_ptr pCapture;\n            /*LPDIRECTSOUNDCAPTUREBUFFER*/ ma_ptr pCaptureBuffer;\n        } dsound;\n#endif\n#ifdef MA_SUPPORT_WINMM\n        struct\n        {\n            /*HWAVEOUT*/ ma_handle hDevicePlayback;\n            /*HWAVEIN*/ ma_handle hDeviceCapture;\n            /*HANDLE*/ ma_handle hEventPlayback;\n            /*HANDLE*/ ma_handle hEventCapture;\n            ma_uint32 fragmentSizeInFrames;\n            ma_uint32 iNextHeaderPlayback;             /* [0,periods). Used as an index into pWAVEHDRPlayback. */\n            ma_uint32 iNextHeaderCapture;              /* [0,periods). Used as an index into pWAVEHDRCapture. */\n            ma_uint32 headerFramesConsumedPlayback;    /* The number of PCM frames consumed in the buffer in pWAVEHEADER[iNextHeader]. */\n            ma_uint32 headerFramesConsumedCapture;     /* ^^^ */\n            /*WAVEHDR**/ ma_uint8* pWAVEHDRPlayback;   /* One instantiation for each period. */\n            /*WAVEHDR**/ ma_uint8* pWAVEHDRCapture;    /* One instantiation for each period. */\n            ma_uint8* pIntermediaryBufferPlayback;\n            ma_uint8* pIntermediaryBufferCapture;\n            ma_uint8* _pHeapData;                      /* Used internally and is used for the heap allocated data for the intermediary buffer and the WAVEHDR structures. */\n        } winmm;\n#endif\n#ifdef MA_SUPPORT_ALSA\n        struct\n        {\n            /*snd_pcm_t**/ ma_ptr pPCMPlayback;\n            /*snd_pcm_t**/ ma_ptr pPCMCapture;\n            /*struct pollfd**/ void* pPollDescriptorsPlayback;\n            /*struct pollfd**/ void* pPollDescriptorsCapture;\n            int pollDescriptorCountPlayback;\n            int pollDescriptorCountCapture;\n            int wakeupfdPlayback;   /* eventfd for waking up from poll() when the playback device is stopped. */\n            int wakeupfdCapture;    /* eventfd for waking up from poll() when the capture device is stopped. */\n            ma_bool8 isUsingMMapPlayback;\n            ma_bool8 isUsingMMapCapture;\n        } alsa;\n#endif\n#ifdef MA_SUPPORT_PULSEAUDIO\n        struct\n        {\n            /*pa_mainloop**/ ma_ptr pMainLoop;\n            /*pa_context**/ ma_ptr pPulseContext;\n            /*pa_stream**/ ma_ptr pStreamPlayback;\n            /*pa_stream**/ ma_ptr pStreamCapture;\n        } pulse;\n#endif\n#ifdef MA_SUPPORT_JACK\n        struct\n        {\n            /*jack_client_t**/ ma_ptr pClient;\n            /*jack_port_t**/ ma_ptr* ppPortsPlayback;\n            /*jack_port_t**/ ma_ptr* ppPortsCapture;\n            float* pIntermediaryBufferPlayback; /* Typed as a float because JACK is always floating point. */\n            float* pIntermediaryBufferCapture;\n        } jack;\n#endif\n#ifdef MA_SUPPORT_COREAUDIO\n        struct\n        {\n            ma_uint32 deviceObjectIDPlayback;\n            ma_uint32 deviceObjectIDCapture;\n            /*AudioUnit*/ ma_ptr audioUnitPlayback;\n            /*AudioUnit*/ ma_ptr audioUnitCapture;\n            /*AudioBufferList**/ ma_ptr pAudioBufferList;   /* Only used for input devices. */\n            ma_uint32 audioBufferCapInFrames;               /* Only used for input devices. The capacity in frames of each buffer in pAudioBufferList. */\n            ma_event stopEvent;\n            ma_uint32 originalPeriodSizeInFrames;\n            ma_uint32 originalPeriodSizeInMilliseconds;\n            ma_uint32 originalPeriods;\n            ma_performance_profile originalPerformanceProfile;\n            ma_bool32 isDefaultPlaybackDevice;\n            ma_bool32 isDefaultCaptureDevice;\n            ma_bool32 isSwitchingPlaybackDevice;   /* <-- Set to true when the default device has changed and miniaudio is in the process of switching. */\n            ma_bool32 isSwitchingCaptureDevice;    /* <-- Set to true when the default device has changed and miniaudio is in the process of switching. */\n            void* pNotificationHandler;             /* Only used on mobile platforms. Obj-C object for handling route changes. */\n        } coreaudio;\n#endif\n#ifdef MA_SUPPORT_SNDIO\n        struct\n        {\n            ma_ptr handlePlayback;\n            ma_ptr handleCapture;\n            ma_bool32 isStartedPlayback;\n            ma_bool32 isStartedCapture;\n        } sndio;\n#endif\n#ifdef MA_SUPPORT_AUDIO4\n        struct\n        {\n            int fdPlayback;\n            int fdCapture;\n        } audio4;\n#endif\n#ifdef MA_SUPPORT_OSS\n        struct\n        {\n            int fdPlayback;\n            int fdCapture;\n        } oss;\n#endif\n#ifdef MA_SUPPORT_AAUDIO\n        struct\n        {\n            /*AAudioStream**/ ma_ptr pStreamPlayback;\n            /*AAudioStream**/ ma_ptr pStreamCapture;\n            ma_mutex rerouteLock;\n            ma_atomic_bool32 isTearingDown;\n            ma_aaudio_usage usage;\n            ma_aaudio_content_type contentType;\n            ma_aaudio_input_preset inputPreset;\n            ma_aaudio_allowed_capture_policy allowedCapturePolicy;\n            ma_bool32 noAutoStartAfterReroute;\n        } aaudio;\n#endif\n#ifdef MA_SUPPORT_OPENSL\n        struct\n        {\n            /*SLObjectItf*/ ma_ptr pOutputMixObj;\n            /*SLOutputMixItf*/ ma_ptr pOutputMix;\n            /*SLObjectItf*/ ma_ptr pAudioPlayerObj;\n            /*SLPlayItf*/ ma_ptr pAudioPlayer;\n            /*SLObjectItf*/ ma_ptr pAudioRecorderObj;\n            /*SLRecordItf*/ ma_ptr pAudioRecorder;\n            /*SLAndroidSimpleBufferQueueItf*/ ma_ptr pBufferQueuePlayback;\n            /*SLAndroidSimpleBufferQueueItf*/ ma_ptr pBufferQueueCapture;\n            ma_bool32 isDrainingCapture;\n            ma_bool32 isDrainingPlayback;\n            ma_uint32 currentBufferIndexPlayback;\n            ma_uint32 currentBufferIndexCapture;\n            ma_uint8* pBufferPlayback;      /* This is malloc()'d and is used for storing audio data. Typed as ma_uint8 for easy offsetting. */\n            ma_uint8* pBufferCapture;\n        } opensl;\n#endif\n#ifdef MA_SUPPORT_WEBAUDIO\n        struct\n        {\n            /* AudioWorklets path. */\n            /* EMSCRIPTEN_WEBAUDIO_T */ int audioContext;\n            /* EMSCRIPTEN_WEBAUDIO_T */ int audioWorklet;\n            float* pIntermediaryBuffer;\n            void* pStackBuffer;\n            ma_result initResult;   /* Set to MA_BUSY while initialization is in progress. */\n            int deviceIndex;        /* We store the device in a list on the JavaScript side. This is used to map our C object to the JS object. */\n        } webaudio;\n#endif\n#ifdef MA_SUPPORT_NULL\n        struct\n        {\n            ma_thread deviceThread;\n            ma_event operationEvent;\n            ma_event operationCompletionEvent;\n            ma_semaphore operationSemaphore;\n            ma_uint32 operation;\n            ma_result operationResult;\n            ma_timer timer;\n            double priorRunTime;\n            ma_uint32 currentPeriodFramesRemainingPlayback;\n            ma_uint32 currentPeriodFramesRemainingCapture;\n            ma_uint64 lastProcessedFramePlayback;\n            ma_uint64 lastProcessedFrameCapture;\n            ma_atomic_bool32 isStarted; /* Read and written by multiple threads. Must be used atomically, and must be 32-bit for compiler compatibility. */\n        } null_device;\n#endif\n    };\n};\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(pop)\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic pop  /* For ISO C99 doesn't support unnamed structs/unions [-Wpedantic] */\n#endif\n\n/*\nInitializes a `ma_context_config` object.\n\n\nReturn Value\n------------\nA `ma_context_config` initialized to defaults.\n\n\nRemarks\n-------\nYou must always use this to initialize the default state of the `ma_context_config` object. Not using this will result in your program breaking when miniaudio\nis updated and new members are added to `ma_context_config`. It also sets logical defaults.\n\nYou can override members of the returned object by changing it's members directly.\n\n\nSee Also\n--------\nma_context_init()\n*/\nMA_API ma_context_config ma_context_config_init(void);\n\n/*\nInitializes a context.\n\nThe context is used for selecting and initializing an appropriate backend and to represent the backend at a more global level than that of an individual\ndevice. There is one context to many devices, and a device is created from a context. A context is required to enumerate devices.\n\n\nParameters\n----------\nbackends (in, optional)\n    A list of backends to try initializing, in priority order. Can be NULL, in which case it uses default priority order.\n\nbackendCount (in, optional)\n    The number of items in `backend`. Ignored if `backend` is NULL.\n\npConfig (in, optional)\n    The context configuration.\n\npContext (in)\n    A pointer to the context object being initialized.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. Do not call this function across multiple threads as some backends read and write to global state.\n\n\nRemarks\n-------\nWhen `backends` is NULL, the default priority order will be used. Below is a list of backends in priority order:\n\n    |-------------|-----------------------|--------------------------------------------------------|\n    | Name        | Enum Name             | Supported Operating Systems                            |\n    |-------------|-----------------------|--------------------------------------------------------|\n    | WASAPI      | ma_backend_wasapi     | Windows Vista+                                         |\n    | DirectSound | ma_backend_dsound     | Windows XP+                                            |\n    | WinMM       | ma_backend_winmm      | Windows XP+ (may work on older versions, but untested) |\n    | Core Audio  | ma_backend_coreaudio  | macOS, iOS                                             |\n    | ALSA        | ma_backend_alsa       | Linux                                                  |\n    | PulseAudio  | ma_backend_pulseaudio | Cross Platform (disabled on Windows, BSD and Android)  |\n    | JACK        | ma_backend_jack       | Cross Platform (disabled on BSD and Android)           |\n    | sndio       | ma_backend_sndio      | OpenBSD                                                |\n    | audio(4)    | ma_backend_audio4     | NetBSD, OpenBSD                                        |\n    | OSS         | ma_backend_oss        | FreeBSD                                                |\n    | AAudio      | ma_backend_aaudio     | Android 8+                                             |\n    | OpenSL|ES   | ma_backend_opensl     | Android (API level 16+)                                |\n    | Web Audio   | ma_backend_webaudio   | Web (via Emscripten)                                   |\n    | Null        | ma_backend_null       | Cross Platform (not used on Web)                       |\n    |-------------|-----------------------|--------------------------------------------------------|\n\nThe context can be configured via the `pConfig` argument. The config object is initialized with `ma_context_config_init()`. Individual configuration settings\ncan then be set directly on the structure. Below are the members of the `ma_context_config` object.\n\n    pLog\n        A pointer to the `ma_log` to post log messages to. Can be NULL if the application does not\n        require logging. See the `ma_log` API for details on how to use the logging system.\n\n    threadPriority\n        The desired priority to use for the audio thread. Allowable values include the following:\n\n        |--------------------------------------|\n        | Thread Priority                      |\n        |--------------------------------------|\n        | ma_thread_priority_idle              |\n        | ma_thread_priority_lowest            |\n        | ma_thread_priority_low               |\n        | ma_thread_priority_normal            |\n        | ma_thread_priority_high              |\n        | ma_thread_priority_highest (default) |\n        | ma_thread_priority_realtime          |\n        | ma_thread_priority_default           |\n        |--------------------------------------|\n\n    threadStackSize\n        The desired size of the stack for the audio thread. Defaults to the operating system's default.\n\n    pUserData\n        A pointer to application-defined data. This can be accessed from the context object directly such as `context.pUserData`.\n\n    allocationCallbacks\n        Structure containing custom allocation callbacks. Leaving this at defaults will cause it to use MA_MALLOC, MA_REALLOC and MA_FREE. These allocation\n        callbacks will be used for anything tied to the context, including devices.\n\n    alsa.useVerboseDeviceEnumeration\n        ALSA will typically enumerate many different devices which can be intrusive and not user-friendly. To combat this, miniaudio will enumerate only unique\n        card/device pairs by default. The problem with this is that you lose a bit of flexibility and control. Setting alsa.useVerboseDeviceEnumeration makes\n        it so the ALSA backend includes all devices. Defaults to false.\n\n    pulse.pApplicationName\n        PulseAudio only. The application name to use when initializing the PulseAudio context with `pa_context_new()`.\n\n    pulse.pServerName\n        PulseAudio only. The name of the server to connect to with `pa_context_connect()`.\n\n    pulse.tryAutoSpawn\n        PulseAudio only. Whether or not to try automatically starting the PulseAudio daemon. Defaults to false. If you set this to true, keep in mind that\n        miniaudio uses a trial and error method to find the most appropriate backend, and this will result in the PulseAudio daemon starting which may be\n        intrusive for the end user.\n\n    coreaudio.sessionCategory\n        iOS only. The session category to use for the shared AudioSession instance. Below is a list of allowable values and their Core Audio equivalents.\n\n        |-----------------------------------------|-------------------------------------|\n        | miniaudio Token                         | Core Audio Token                    |\n        |-----------------------------------------|-------------------------------------|\n        | ma_ios_session_category_ambient         | AVAudioSessionCategoryAmbient       |\n        | ma_ios_session_category_solo_ambient    | AVAudioSessionCategorySoloAmbient   |\n        | ma_ios_session_category_playback        | AVAudioSessionCategoryPlayback      |\n        | ma_ios_session_category_record          | AVAudioSessionCategoryRecord        |\n        | ma_ios_session_category_play_and_record | AVAudioSessionCategoryPlayAndRecord |\n        | ma_ios_session_category_multi_route     | AVAudioSessionCategoryMultiRoute    |\n        | ma_ios_session_category_none            | AVAudioSessionCategoryAmbient       |\n        | ma_ios_session_category_default         | AVAudioSessionCategoryAmbient       |\n        |-----------------------------------------|-------------------------------------|\n\n    coreaudio.sessionCategoryOptions\n        iOS only. Session category options to use with the shared AudioSession instance. Below is a list of allowable values and their Core Audio equivalents.\n\n        |---------------------------------------------------------------------------|------------------------------------------------------------------|\n        | miniaudio Token                                                           | Core Audio Token                                                 |\n        |---------------------------------------------------------------------------|------------------------------------------------------------------|\n        | ma_ios_session_category_option_mix_with_others                            | AVAudioSessionCategoryOptionMixWithOthers                        |\n        | ma_ios_session_category_option_duck_others                                | AVAudioSessionCategoryOptionDuckOthers                           |\n        | ma_ios_session_category_option_allow_bluetooth                            | AVAudioSessionCategoryOptionAllowBluetooth                       |\n        | ma_ios_session_category_option_default_to_speaker                         | AVAudioSessionCategoryOptionDefaultToSpeaker                     |\n        | ma_ios_session_category_option_interrupt_spoken_audio_and_mix_with_others | AVAudioSessionCategoryOptionInterruptSpokenAudioAndMixWithOthers |\n        | ma_ios_session_category_option_allow_bluetooth_a2dp                       | AVAudioSessionCategoryOptionAllowBluetoothA2DP                   |\n        | ma_ios_session_category_option_allow_air_play                             | AVAudioSessionCategoryOptionAllowAirPlay                         |\n        |---------------------------------------------------------------------------|------------------------------------------------------------------|\n\n    coreaudio.noAudioSessionActivate\n        iOS only. When set to true, does not perform an explicit [[AVAudioSession sharedInstace] setActive:true] on initialization.\n\n    coreaudio.noAudioSessionDeactivate\n        iOS only. When set to true, does not perform an explicit [[AVAudioSession sharedInstace] setActive:false] on uninitialization.\n\n    jack.pClientName\n        The name of the client to pass to `jack_client_open()`.\n\n    jack.tryStartServer\n        Whether or not to try auto-starting the JACK server. Defaults to false.\n\n\nIt is recommended that only a single context is active at any given time because it's a bulky data structure which performs run-time linking for the\nrelevant backends every time it's initialized.\n\nThe location of the context cannot change throughout it's lifetime. Consider allocating the `ma_context` object with `malloc()` if this is an issue. The\nreason for this is that a pointer to the context is stored in the `ma_device` structure.\n\n\nExample 1 - Default Initialization\n----------------------------------\nThe example below shows how to initialize the context using the default configuration.\n\n```c\nma_context context;\nma_result result = ma_context_init(NULL, 0, NULL, &context);\nif (result != MA_SUCCESS) {\n    // Error.\n}\n```\n\n\nExample 2 - Custom Configuration\n--------------------------------\nThe example below shows how to initialize the context using custom backend priorities and a custom configuration. In this hypothetical example, the program\nwants to prioritize ALSA over PulseAudio on Linux. They also want to avoid using the WinMM backend on Windows because it's latency is too high. They also\nwant an error to be returned if no valid backend is available which they achieve by excluding the Null backend.\n\nFor the configuration, the program wants to capture any log messages so they can, for example, route it to a log file and user interface.\n\n```c\nma_backend backends[] = {\n    ma_backend_alsa,\n    ma_backend_pulseaudio,\n    ma_backend_wasapi,\n    ma_backend_dsound\n};\n\nma_log log;\nma_log_init(&log);\nma_log_register_callback(&log, ma_log_callback_init(my_log_callbac, pMyLogUserData));\n\nma_context_config config = ma_context_config_init();\nconfig.pLog = &log; // Specify a custom log object in the config so any logs that are posted from ma_context_init() are captured.\n\nma_context context;\nma_result result = ma_context_init(backends, sizeof(backends)/sizeof(backends[0]), &config, &context);\nif (result != MA_SUCCESS) {\n    // Error.\n    if (result == MA_NO_BACKEND) {\n        // Couldn't find an appropriate backend.\n    }\n}\n\n// You could also attach a log callback post-initialization:\nma_log_register_callback(ma_context_get_log(&context), ma_log_callback_init(my_log_callback, pMyLogUserData));\n```\n\n\nSee Also\n--------\nma_context_config_init()\nma_context_uninit()\n*/\nMA_API ma_result ma_context_init(const ma_backend backends[], ma_uint32 backendCount, const ma_context_config* pConfig, ma_context* pContext);\n\n/*\nUninitializes a context.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. Do not call this function across multiple threads as some backends read and write to global state.\n\n\nRemarks\n-------\nResults are undefined if you call this while any device created by this context is still active.\n\n\nSee Also\n--------\nma_context_init()\n*/\nMA_API ma_result ma_context_uninit(ma_context* pContext);\n\n/*\nRetrieves the size of the ma_context object.\n\nThis is mainly for the purpose of bindings to know how much memory to allocate.\n*/\nMA_API size_t ma_context_sizeof(void);\n\n/*\nRetrieves a pointer to the log object associated with this context.\n\n\nRemarks\n-------\nPass the returned pointer to `ma_log_post()`, `ma_log_postv()` or `ma_log_postf()` to post a log\nmessage.\n\nYou can attach your own logging callback to the log with `ma_log_register_callback()`\n\n\nReturn Value\n------------\nA pointer to the `ma_log` object that the context uses to post log messages. If some error occurs,\nNULL will be returned.\n*/\nMA_API ma_log* ma_context_get_log(ma_context* pContext);\n\n/*\nEnumerates over every device (both playback and capture).\n\nThis is a lower-level enumeration function to the easier to use `ma_context_get_devices()`. Use `ma_context_enumerate_devices()` if you would rather not incur\nan internal heap allocation, or it simply suits your code better.\n\nNote that this only retrieves the ID and name/description of the device. The reason for only retrieving basic information is that it would otherwise require\nopening the backend device in order to probe it for more detailed information which can be inefficient. Consider using `ma_context_get_device_info()` for this,\nbut don't call it from within the enumeration callback.\n\nReturning false from the callback will stop enumeration. Returning true will continue enumeration.\n\n\nParameters\n----------\npContext (in)\n    A pointer to the context performing the enumeration.\n\ncallback (in)\n    The callback to fire for each enumerated device.\n\npUserData (in)\n    A pointer to application-defined data passed to the callback.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nSafe. This is guarded using a simple mutex lock.\n\n\nRemarks\n-------\nDo _not_ assume the first enumerated device of a given type is the default device.\n\nSome backends and platforms may only support default playback and capture devices.\n\nIn general, you should not do anything complicated from within the callback. In particular, do not try initializing a device from within the callback. Also,\ndo not try to call `ma_context_get_device_info()` from within the callback.\n\nConsider using `ma_context_get_devices()` for a simpler and safer API, albeit at the expense of an internal heap allocation.\n\n\nExample 1 - Simple Enumeration\n------------------------------\nma_bool32 ma_device_enum_callback(ma_context* pContext, ma_device_type deviceType, const ma_device_info* pInfo, void* pUserData)\n{\n    printf(\"Device Name: %s\\n\", pInfo->name);\n    return MA_TRUE;\n}\n\nma_result result = ma_context_enumerate_devices(&context, my_device_enum_callback, pMyUserData);\nif (result != MA_SUCCESS) {\n    // Error.\n}\n\n\nSee Also\n--------\nma_context_get_devices()\n*/\nMA_API ma_result ma_context_enumerate_devices(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData);\n\n/*\nRetrieves basic information about every active playback and/or capture device.\n\nThis function will allocate memory internally for the device lists and return a pointer to them through the `ppPlaybackDeviceInfos` and `ppCaptureDeviceInfos`\nparameters. If you do not want to incur the overhead of these allocations consider using `ma_context_enumerate_devices()` which will instead use a callback.\n\nNote that this only retrieves the ID and name/description of the device. The reason for only retrieving basic information is that it would otherwise require\nopening the backend device in order to probe it for more detailed information which can be inefficient. Consider using `ma_context_get_device_info()` for this,\nbut don't call it from within the enumeration callback.\n\n\nParameters\n----------\npContext (in)\n    A pointer to the context performing the enumeration.\n\nppPlaybackDeviceInfos (out)\n    A pointer to a pointer that will receive the address of a buffer containing the list of `ma_device_info` structures for playback devices.\n\npPlaybackDeviceCount (out)\n    A pointer to an unsigned integer that will receive the number of playback devices.\n\nppCaptureDeviceInfos (out)\n    A pointer to a pointer that will receive the address of a buffer containing the list of `ma_device_info` structures for capture devices.\n\npCaptureDeviceCount (out)\n    A pointer to an unsigned integer that will receive the number of capture devices.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. Since each call to this function invalidates the pointers from the previous call, you should not be calling this simultaneously across multiple\nthreads. Instead, you need to make a copy of the returned data with your own higher level synchronization.\n\n\nRemarks\n-------\nIt is _not_ safe to assume the first device in the list is the default device.\n\nYou can pass in NULL for the playback or capture lists in which case they'll be ignored.\n\nThe returned pointers will become invalid upon the next call this this function, or when the context is uninitialized. Do not free the returned pointers.\n\n\nSee Also\n--------\nma_context_enumerate_devices()\n*/\nMA_API ma_result ma_context_get_devices(ma_context* pContext, ma_device_info** ppPlaybackDeviceInfos, ma_uint32* pPlaybackDeviceCount, ma_device_info** ppCaptureDeviceInfos, ma_uint32* pCaptureDeviceCount);\n\n/*\nRetrieves information about a device of the given type, with the specified ID and share mode.\n\n\nParameters\n----------\npContext (in)\n    A pointer to the context performing the query.\n\ndeviceType (in)\n    The type of the device being queried. Must be either `ma_device_type_playback` or `ma_device_type_capture`.\n\npDeviceID (in)\n    The ID of the device being queried.\n\npDeviceInfo (out)\n    A pointer to the `ma_device_info` structure that will receive the device information.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nSafe. This is guarded using a simple mutex lock.\n\n\nRemarks\n-------\nDo _not_ call this from within the `ma_context_enumerate_devices()` callback.\n\nIt's possible for a device to have different information and capabilities depending on whether or not it's opened in shared or exclusive mode. For example, in\nshared mode, WASAPI always uses floating point samples for mixing, but in exclusive mode it can be anything. Therefore, this function allows you to specify\nwhich share mode you want information for. Note that not all backends and devices support shared or exclusive mode, in which case this function will fail if\nthe requested share mode is unsupported.\n\nThis leaves pDeviceInfo unmodified in the result of an error.\n*/\nMA_API ma_result ma_context_get_device_info(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo);\n\n/*\nDetermines if the given context supports loopback mode.\n\n\nParameters\n----------\npContext (in)\n    A pointer to the context getting queried.\n\n\nReturn Value\n------------\nMA_TRUE if the context supports loopback mode; MA_FALSE otherwise.\n*/\nMA_API ma_bool32 ma_context_is_loopback_supported(ma_context* pContext);\n\n\n\n/*\nInitializes a device config with default settings.\n\n\nParameters\n----------\ndeviceType (in)\n    The type of the device this config is being initialized for. This must set to one of the following:\n\n    |-------------------------|\n    | Device Type             |\n    |-------------------------|\n    | ma_device_type_playback |\n    | ma_device_type_capture  |\n    | ma_device_type_duplex   |\n    | ma_device_type_loopback |\n    |-------------------------|\n\n\nReturn Value\n------------\nA new device config object with default settings. You will typically want to adjust the config after this function returns. See remarks.\n\n\nThread Safety\n-------------\nSafe.\n\n\nCallback Safety\n---------------\nSafe, but don't try initializing a device in a callback.\n\n\nRemarks\n-------\nThe returned config will be initialized to defaults. You will normally want to customize a few variables before initializing the device. See Example 1 for a\ntypical configuration which sets the sample format, channel count, sample rate, data callback and user data. These are usually things you will want to change\nbefore initializing the device.\n\nSee `ma_device_init()` for details on specific configuration options.\n\n\nExample 1 - Simple Configuration\n--------------------------------\nThe example below is what a program will typically want to configure for each device at a minimum. Notice how `ma_device_config_init()` is called first, and\nthen the returned object is modified directly. This is important because it ensures that your program continues to work as new configuration options are added\nto the `ma_device_config` structure.\n\n```c\nma_device_config config = ma_device_config_init(ma_device_type_playback);\nconfig.playback.format   = ma_format_f32;\nconfig.playback.channels = 2;\nconfig.sampleRate        = 48000;\nconfig.dataCallback      = ma_data_callback;\nconfig.pUserData         = pMyUserData;\n```\n\n\nSee Also\n--------\nma_device_init()\nma_device_init_ex()\n*/\nMA_API ma_device_config ma_device_config_init(ma_device_type deviceType);\n\n\n/*\nInitializes a device.\n\nA device represents a physical audio device. The idea is you send or receive audio data from the device to either play it back through a speaker, or capture it\nfrom a microphone. Whether or not you should send or receive data from the device (or both) depends on the type of device you are initializing which can be\nplayback, capture, full-duplex or loopback. (Note that loopback mode is only supported on select backends.) Sending and receiving audio data to and from the\ndevice is done via a callback which is fired by miniaudio at periodic time intervals.\n\nThe frequency at which data is delivered to and from a device depends on the size of its period. The size of the period can be defined in terms of PCM frames\nor milliseconds, whichever is more convenient. Generally speaking, the smaller the period, the lower the latency at the expense of higher CPU usage and\nincreased risk of glitching due to the more frequent and granular data deliver intervals. The size of a period will depend on your requirements, but\nminiaudio's defaults should work fine for most scenarios. If you're building a game you should leave this fairly small, whereas if you're building a simple\nmedia player you can make it larger. Note that the period size you request is actually just a hint - miniaudio will tell the backend what you want, but the\nbackend is ultimately responsible for what it gives you. You cannot assume you will get exactly what you ask for.\n\nWhen delivering data to and from a device you need to make sure it's in the correct format which you can set through the device configuration. You just set the\nformat that you want to use and miniaudio will perform all of the necessary conversion for you internally. When delivering data to and from the callback you\ncan assume the format is the same as what you requested when you initialized the device. See Remarks for more details on miniaudio's data conversion pipeline.\n\n\nParameters\n----------\npContext (in, optional)\n    A pointer to the context that owns the device. This can be null, in which case it creates a default context internally.\n\npConfig (in)\n    A pointer to the device configuration. Cannot be null. See remarks for details.\n\npDevice (out)\n    A pointer to the device object being initialized.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. It is not safe to call this function simultaneously for different devices because some backends depend on and mutate global state. The same applies to\ncalling this at the same time as `ma_device_uninit()`.\n\n\nCallback Safety\n---------------\nUnsafe. It is not safe to call this inside any callback.\n\n\nRemarks\n-------\nSetting `pContext` to NULL will result in miniaudio creating a default context internally and is equivalent to passing in a context initialized like so:\n\n    ```c\n    ma_context_init(NULL, 0, NULL, &context);\n    ```\n\nDo not set `pContext` to NULL if you are needing to open multiple devices. You can, however, use NULL when initializing the first device, and then use\ndevice.pContext for the initialization of other devices.\n\nThe device can be configured via the `pConfig` argument. The config object is initialized with `ma_device_config_init()`. Individual configuration settings can\nthen be set directly on the structure. Below are the members of the `ma_device_config` object.\n\n    deviceType\n        Must be `ma_device_type_playback`, `ma_device_type_capture`, `ma_device_type_duplex` of `ma_device_type_loopback`.\n\n    sampleRate\n        The sample rate, in hertz. The most common sample rates are 48000 and 44100. Setting this to 0 will use the device's native sample rate.\n\n    periodSizeInFrames\n        The desired size of a period in PCM frames. If this is 0, `periodSizeInMilliseconds` will be used instead. If both are 0 the default buffer size will\n        be used depending on the selected performance profile. This value affects latency. See below for details.\n\n    periodSizeInMilliseconds\n        The desired size of a period in milliseconds. If this is 0, `periodSizeInFrames` will be used instead. If both are 0 the default buffer size will be\n        used depending on the selected performance profile. The value affects latency. See below for details.\n\n    periods\n        The number of periods making up the device's entire buffer. The total buffer size is `periodSizeInFrames` or `periodSizeInMilliseconds` multiplied by\n        this value. This is just a hint as backends will be the ones who ultimately decide how your periods will be configured.\n\n    performanceProfile\n        A hint to miniaudio as to the performance requirements of your program. Can be either `ma_performance_profile_low_latency` (default) or\n        `ma_performance_profile_conservative`. This mainly affects the size of default buffers and can usually be left at its default value.\n\n    noPreSilencedOutputBuffer\n        When set to true, the contents of the output buffer passed into the data callback will be left undefined. When set to false (default), the contents of\n        the output buffer will be cleared the zero. You can use this to avoid the overhead of zeroing out the buffer if you can guarantee that your data\n        callback will write to every sample in the output buffer, or if you are doing your own clearing.\n\n    noClip\n        When set to true, the contents of the output buffer are left alone after returning and it will be left up to the backend itself to decide whether or\n        not to clip. When set to false (default), the contents of the output buffer passed into the data callback will be clipped after returning. This only\n        applies when the playback sample format is f32.\n\n    noDisableDenormals\n        By default, miniaudio will disable denormals when the data callback is called. Setting this to true will prevent the disabling of denormals.\n\n    noFixedSizedCallback\n        Allows miniaudio to fire the data callback with any frame count. When this is set to false (the default), the data callback will be fired with a\n        consistent frame count as specified by `periodSizeInFrames` or `periodSizeInMilliseconds`. When set to true, miniaudio will fire the callback with\n        whatever the backend requests, which could be anything.\n\n    dataCallback\n        The callback to fire whenever data is ready to be delivered to or from the device.\n\n    notificationCallback\n        The callback to fire when something has changed with the device, such as whether or not it has been started or stopped.\n\n    pUserData\n        The user data pointer to use with the device. You can access this directly from the device object like `device.pUserData`.\n\n    resampling.algorithm\n        The resampling algorithm to use when miniaudio needs to perform resampling between the rate specified by `sampleRate` and the device's native rate. The\n        default value is `ma_resample_algorithm_linear`, and the quality can be configured with `resampling.linear.lpfOrder`.\n\n    resampling.pBackendVTable\n        A pointer to an optional vtable that can be used for plugging in a custom resampler.\n\n    resampling.pBackendUserData\n        A pointer that will passed to callbacks in pBackendVTable.\n\n    resampling.linear.lpfOrder\n        The linear resampler applies a low-pass filter as part of its processing for anti-aliasing. This setting controls the order of the filter. The higher\n        the value, the better the quality, in general. Setting this to 0 will disable low-pass filtering altogether. The maximum value is\n        `MA_MAX_FILTER_ORDER`. The default value is `min(4, MA_MAX_FILTER_ORDER)`.\n\n    playback.pDeviceID\n        A pointer to a `ma_device_id` structure containing the ID of the playback device to initialize. Setting this NULL (default) will use the system's\n        default playback device. Retrieve the device ID from the `ma_device_info` structure, which can be retrieved using device enumeration.\n\n    playback.format\n        The sample format to use for playback. When set to `ma_format_unknown` the device's native format will be used. This can be retrieved after\n        initialization from the device object directly with `device.playback.format`.\n\n    playback.channels\n        The number of channels to use for playback. When set to 0 the device's native channel count will be used. This can be retrieved after initialization\n        from the device object directly with `device.playback.channels`.\n\n    playback.pChannelMap\n        The channel map to use for playback. When left empty, the device's native channel map will be used. This can be retrieved after initialization from the\n        device object direct with `device.playback.pChannelMap`. When set, the buffer should contain `channels` items.\n\n    playback.shareMode\n        The preferred share mode to use for playback. Can be either `ma_share_mode_shared` (default) or `ma_share_mode_exclusive`. Note that if you specify\n        exclusive mode, but it's not supported by the backend, initialization will fail. You can then fall back to shared mode if desired by changing this to\n        ma_share_mode_shared and reinitializing.\n\n    capture.pDeviceID\n        A pointer to a `ma_device_id` structure containing the ID of the capture device to initialize. Setting this NULL (default) will use the system's\n        default capture device. Retrieve the device ID from the `ma_device_info` structure, which can be retrieved using device enumeration.\n\n    capture.format\n        The sample format to use for capture. When set to `ma_format_unknown` the device's native format will be used. This can be retrieved after\n        initialization from the device object directly with `device.capture.format`.\n\n    capture.channels\n        The number of channels to use for capture. When set to 0 the device's native channel count will be used. This can be retrieved after initialization\n        from the device object directly with `device.capture.channels`.\n\n    capture.pChannelMap\n        The channel map to use for capture. When left empty, the device's native channel map will be used. This can be retrieved after initialization from the\n        device object direct with `device.capture.pChannelMap`. When set, the buffer should contain `channels` items.\n\n    capture.shareMode\n        The preferred share mode to use for capture. Can be either `ma_share_mode_shared` (default) or `ma_share_mode_exclusive`. Note that if you specify\n        exclusive mode, but it's not supported by the backend, initialization will fail. You can then fall back to shared mode if desired by changing this to\n        ma_share_mode_shared and reinitializing.\n\n    wasapi.noAutoConvertSRC\n        WASAPI only. When set to true, disables WASAPI's automatic resampling and forces the use of miniaudio's resampler. Defaults to false.\n\n    wasapi.noDefaultQualitySRC\n        WASAPI only. Only used when `wasapi.noAutoConvertSRC` is set to false. When set to true, disables the use of `AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY`.\n        You should usually leave this set to false, which is the default.\n\n    wasapi.noAutoStreamRouting\n        WASAPI only. When set to true, disables automatic stream routing on the WASAPI backend. Defaults to false.\n\n    wasapi.noHardwareOffloading\n        WASAPI only. When set to true, disables the use of WASAPI's hardware offloading feature. Defaults to false.\n\n    alsa.noMMap\n        ALSA only. When set to true, disables MMap mode. Defaults to false.\n\n    alsa.noAutoFormat\n        ALSA only. When set to true, disables ALSA's automatic format conversion by including the SND_PCM_NO_AUTO_FORMAT flag. Defaults to false.\n\n    alsa.noAutoChannels\n        ALSA only. When set to true, disables ALSA's automatic channel conversion by including the SND_PCM_NO_AUTO_CHANNELS flag. Defaults to false.\n\n    alsa.noAutoResample\n        ALSA only. When set to true, disables ALSA's automatic resampling by including the SND_PCM_NO_AUTO_RESAMPLE flag. Defaults to false.\n\n    pulse.pStreamNamePlayback\n        PulseAudio only. Sets the stream name for playback.\n\n    pulse.pStreamNameCapture\n        PulseAudio only. Sets the stream name for capture.\n\n    pulse.channelMap\n        PulseAudio only. Sets the channel map that is requested from PulseAudio. See MA_PA_CHANNEL_MAP_* constants. Defaults to MA_PA_CHANNEL_MAP_AIFF.\n\n    coreaudio.allowNominalSampleRateChange\n        Core Audio only. Desktop only. When enabled, allows the sample rate of the device to be changed at the operating system level. This\n        is disabled by default in order to prevent intrusive changes to the user's system. This is useful if you want to use a sample rate\n        that is known to be natively supported by the hardware thereby avoiding the cost of resampling. When set to true, miniaudio will\n        find the closest match between the sample rate requested in the device config and the sample rates natively supported by the\n        hardware. When set to false, the sample rate currently set by the operating system will always be used.\n\n    opensl.streamType\n        OpenSL only. Explicitly sets the stream type. If left unset (`ma_opensl_stream_type_default`), the\n        stream type will be left unset. Think of this as the type of audio you're playing.\n\n    opensl.recordingPreset\n        OpenSL only. Explicitly sets the type of recording your program will be doing. When left\n        unset, the recording preset will be left unchanged.\n\n    aaudio.usage\n        AAudio only. Explicitly sets the nature of the audio the program will be consuming. When\n        left unset, the usage will be left unchanged.\n\n    aaudio.contentType\n        AAudio only. Sets the content type. When left unset, the content type will be left unchanged.\n\n    aaudio.inputPreset\n        AAudio only. Explicitly sets the type of recording your program will be doing. When left\n        unset, the input preset will be left unchanged.\n\n    aaudio.noAutoStartAfterReroute\n        AAudio only. Controls whether or not the device should be automatically restarted after a\n        stream reroute. When set to false (default) the device will be restarted automatically;\n        otherwise the device will be stopped.\n\n\nOnce initialized, the device's config is immutable. If you need to change the config you will need to initialize a new device.\n\nAfter initializing the device it will be in a stopped state. To start it, use `ma_device_start()`.\n\nIf both `periodSizeInFrames` and `periodSizeInMilliseconds` are set to zero, it will default to `MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY` or\n`MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE`, depending on whether or not `performanceProfile` is set to `ma_performance_profile_low_latency` or\n`ma_performance_profile_conservative`.\n\nIf you request exclusive mode and the backend does not support it an error will be returned. For robustness, you may want to first try initializing the device\nin exclusive mode, and then fall back to shared mode if required. Alternatively you can just request shared mode (the default if you leave it unset in the\nconfig) which is the most reliable option. Some backends do not have a practical way of choosing whether or not the device should be exclusive or not (ALSA,\nfor example) in which case it just acts as a hint. Unless you have special requirements you should try avoiding exclusive mode as it's intrusive to the user.\nStarting with Windows 10, miniaudio will use low-latency shared mode where possible which may make exclusive mode unnecessary.\n\nWhen sending or receiving data to/from a device, miniaudio will internally perform a format conversion to convert between the format specified by the config\nand the format used internally by the backend. If you pass in 0 for the sample format, channel count, sample rate _and_ channel map, data transmission will run\non an optimized pass-through fast path. You can retrieve the format, channel count and sample rate by inspecting the `playback/capture.format`,\n`playback/capture.channels` and `sampleRate` members of the device object.\n\nWhen compiling for UWP you must ensure you call this function on the main UI thread because the operating system may need to present the user with a message\nasking for permissions. Please refer to the official documentation for ActivateAudioInterfaceAsync() for more information.\n\nALSA Specific: When initializing the default device, requesting shared mode will try using the \"dmix\" device for playback and the \"dsnoop\" device for capture.\nIf these fail it will try falling back to the \"hw\" device.\n\n\nExample 1 - Simple Initialization\n---------------------------------\nThis example shows how to initialize a simple playback device using a standard configuration. If you are just needing to do simple playback from the default\nplayback device this is usually all you need.\n\n```c\nma_device_config config = ma_device_config_init(ma_device_type_playback);\nconfig.playback.format   = ma_format_f32;\nconfig.playback.channels = 2;\nconfig.sampleRate        = 48000;\nconfig.dataCallback      = ma_data_callback;\nconfig.pMyUserData       = pMyUserData;\n\nma_device device;\nma_result result = ma_device_init(NULL, &config, &device);\nif (result != MA_SUCCESS) {\n    // Error\n}\n```\n\n\nExample 2 - Advanced Initialization\n-----------------------------------\nThis example shows how you might do some more advanced initialization. In this hypothetical example we want to control the latency by setting the buffer size\nand period count. We also want to allow the user to be able to choose which device to output from which means we need a context so we can perform device\nenumeration.\n\n```c\nma_context context;\nma_result result = ma_context_init(NULL, 0, NULL, &context);\nif (result != MA_SUCCESS) {\n    // Error\n}\n\nma_device_info* pPlaybackDeviceInfos;\nma_uint32 playbackDeviceCount;\nresult = ma_context_get_devices(&context, &pPlaybackDeviceInfos, &playbackDeviceCount, NULL, NULL);\nif (result != MA_SUCCESS) {\n    // Error\n}\n\n// ... choose a device from pPlaybackDeviceInfos ...\n\nma_device_config config = ma_device_config_init(ma_device_type_playback);\nconfig.playback.pDeviceID       = pMyChosenDeviceID;    // <-- Get this from the `id` member of one of the `ma_device_info` objects returned by ma_context_get_devices().\nconfig.playback.format          = ma_format_f32;\nconfig.playback.channels        = 2;\nconfig.sampleRate               = 48000;\nconfig.dataCallback             = ma_data_callback;\nconfig.pUserData                = pMyUserData;\nconfig.periodSizeInMilliseconds = 10;\nconfig.periods                  = 3;\n\nma_device device;\nresult = ma_device_init(&context, &config, &device);\nif (result != MA_SUCCESS) {\n    // Error\n}\n```\n\n\nSee Also\n--------\nma_device_config_init()\nma_device_uninit()\nma_device_start()\nma_context_init()\nma_context_get_devices()\nma_context_enumerate_devices()\n*/\nMA_API ma_result ma_device_init(ma_context* pContext, const ma_device_config* pConfig, ma_device* pDevice);\n\n/*\nInitializes a device without a context, with extra parameters for controlling the configuration of the internal self-managed context.\n\nThis is the same as `ma_device_init()`, only instead of a context being passed in, the parameters from `ma_context_init()` are passed in instead. This function\nallows you to configure the internally created context.\n\n\nParameters\n----------\nbackends (in, optional)\n    A list of backends to try initializing, in priority order. Can be NULL, in which case it uses default priority order.\n\nbackendCount (in, optional)\n    The number of items in `backend`. Ignored if `backend` is NULL.\n\npContextConfig (in, optional)\n    The context configuration.\n\npConfig (in)\n    A pointer to the device configuration. Cannot be null. See remarks for details.\n\npDevice (out)\n    A pointer to the device object being initialized.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. It is not safe to call this function simultaneously for different devices because some backends depend on and mutate global state. The same applies to\ncalling this at the same time as `ma_device_uninit()`.\n\n\nCallback Safety\n---------------\nUnsafe. It is not safe to call this inside any callback.\n\n\nRemarks\n-------\nYou only need to use this function if you want to configure the context differently to its defaults. You should never use this function if you want to manage\nyour own context.\n\nSee the documentation for `ma_context_init()` for information on the different context configuration options.\n\n\nSee Also\n--------\nma_device_init()\nma_device_uninit()\nma_device_config_init()\nma_context_init()\n*/\nMA_API ma_result ma_device_init_ex(const ma_backend backends[], ma_uint32 backendCount, const ma_context_config* pContextConfig, const ma_device_config* pConfig, ma_device* pDevice);\n\n/*\nUninitializes a device.\n\nThis will explicitly stop the device. You do not need to call `ma_device_stop()` beforehand, but it's harmless if you do.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device to stop.\n\n\nReturn Value\n------------\nNothing\n\n\nThread Safety\n-------------\nUnsafe. As soon as this API is called the device should be considered undefined.\n\n\nCallback Safety\n---------------\nUnsafe. It is not safe to call this inside any callback. Doing this will result in a deadlock.\n\n\nSee Also\n--------\nma_device_init()\nma_device_stop()\n*/\nMA_API void ma_device_uninit(ma_device* pDevice);\n\n\n/*\nRetrieves a pointer to the context that owns the given device.\n*/\nMA_API ma_context* ma_device_get_context(ma_device* pDevice);\n\n/*\nHelper function for retrieving the log object associated with the context that owns this device.\n*/\nMA_API ma_log* ma_device_get_log(ma_device* pDevice);\n\n\n/*\nRetrieves information about the device.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose information is being retrieved.\n\ntype (in)\n    The device type. This parameter is required for duplex devices. When retrieving device\n    information, you are doing so for an individual playback or capture device.\n\npDeviceInfo (out)\n    A pointer to the `ma_device_info` that will receive the device information.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. This should be considered unsafe because it may be calling into the backend which may or\nmay not be safe.\n\n\nCallback Safety\n---------------\nUnsafe. You should avoid calling this in the data callback because it may call into the backend\nwhich may or may not be safe.\n*/\nMA_API ma_result ma_device_get_info(ma_device* pDevice, ma_device_type type, ma_device_info* pDeviceInfo);\n\n\n/*\nRetrieves the name of the device.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose information is being retrieved.\n\ntype (in)\n    The device type. This parameter is required for duplex devices. When retrieving device\n    information, you are doing so for an individual playback or capture device.\n\npName (out)\n    A pointer to the buffer that will receive the name.\n\nnameCap (in)\n    The capacity of the output buffer, including space for the null terminator.\n\npLengthNotIncludingNullTerminator (out, optional)\n    A pointer to the variable that will receive the length of the name, not including the null\n    terminator.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nUnsafe. This should be considered unsafe because it may be calling into the backend which may or\nmay not be safe.\n\n\nCallback Safety\n---------------\nUnsafe. You should avoid calling this in the data callback because it may call into the backend\nwhich may or may not be safe.\n\n\nRemarks\n-------\nIf the name does not fully fit into the output buffer, it'll be truncated. You can pass in NULL to\n`pName` if you want to first get the length of the name for the purpose of memory allocation of the\noutput buffer. Allocating a buffer of size `MA_MAX_DEVICE_NAME_LENGTH + 1` should be enough for\nmost cases and will avoid the need for the inefficiency of calling this function twice.\n\nThis is implemented in terms of `ma_device_get_info()`.\n*/\nMA_API ma_result ma_device_get_name(ma_device* pDevice, ma_device_type type, char* pName, size_t nameCap, size_t* pLengthNotIncludingNullTerminator);\n\n\n/*\nStarts the device. For playback devices this begins playback. For capture devices it begins recording.\n\nUse `ma_device_stop()` to stop the device.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device to start.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nSafe. It's safe to call this from any thread with the exception of the callback thread.\n\n\nCallback Safety\n---------------\nUnsafe. It is not safe to call this inside any callback.\n\n\nRemarks\n-------\nFor a playback device, this will retrieve an initial chunk of audio data from the client before returning. The reason for this is to ensure there is valid\naudio data in the buffer, which needs to be done before the device begins playback.\n\nThis API waits until the backend device has been started for real by the worker thread. It also waits on a mutex for thread-safety.\n\nDo not call this in any callback.\n\n\nSee Also\n--------\nma_device_stop()\n*/\nMA_API ma_result ma_device_start(ma_device* pDevice);\n\n/*\nStops the device. For playback devices this stops playback. For capture devices it stops recording.\n\nUse `ma_device_start()` to start the device again.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device to stop.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error code otherwise.\n\n\nThread Safety\n-------------\nSafe. It's safe to call this from any thread with the exception of the callback thread.\n\n\nCallback Safety\n---------------\nUnsafe. It is not safe to call this inside any callback. Doing this will result in a deadlock.\n\n\nRemarks\n-------\nThis API needs to wait on the worker thread to stop the backend device properly before returning. It also waits on a mutex for thread-safety. In addition, some\nbackends need to wait for the device to finish playback/recording of the current fragment which can take some time (usually proportionate to the buffer size\nthat was specified at initialization time).\n\nBackends are required to either pause the stream in-place or drain the buffer if pausing is not possible. The reason for this is that stopping the device and\nthe resuming it with ma_device_start() (which you might do when your program loses focus) may result in a situation where those samples are never output to the\nspeakers or received from the microphone which can in turn result in de-syncs.\n\nDo not call this in any callback.\n\n\nSee Also\n--------\nma_device_start()\n*/\nMA_API ma_result ma_device_stop(ma_device* pDevice);\n\n/*\nDetermines whether or not the device is started.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose start state is being retrieved.\n\n\nReturn Value\n------------\nTrue if the device is started, false otherwise.\n\n\nThread Safety\n-------------\nSafe. If another thread calls `ma_device_start()` or `ma_device_stop()` at this same time as this function is called, there's a very small chance the return\nvalue will be out of sync.\n\n\nCallback Safety\n---------------\nSafe. This is implemented as a simple accessor.\n\n\nSee Also\n--------\nma_device_start()\nma_device_stop()\n*/\nMA_API ma_bool32 ma_device_is_started(const ma_device* pDevice);\n\n\n/*\nRetrieves the state of the device.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose state is being retrieved.\n\n\nReturn Value\n------------\nThe current state of the device. The return value will be one of the following:\n\n    +-------------------------------+------------------------------------------------------------------------------+\n    | ma_device_state_uninitialized | Will only be returned if the device is in the middle of initialization.      |\n    +-------------------------------+------------------------------------------------------------------------------+\n    | ma_device_state_stopped       | The device is stopped. The initial state of the device after initialization. |\n    +-------------------------------+------------------------------------------------------------------------------+\n    | ma_device_state_started       | The device started and requesting and/or delivering audio data.              |\n    +-------------------------------+------------------------------------------------------------------------------+\n    | ma_device_state_starting      | The device is in the process of starting.                                    |\n    +-------------------------------+------------------------------------------------------------------------------+\n    | ma_device_state_stopping      | The device is in the process of stopping.                                    |\n    +-------------------------------+------------------------------------------------------------------------------+\n\n\nThread Safety\n-------------\nSafe. This is implemented as a simple accessor. Note that if the device is started or stopped at the same time as this function is called,\nthere's a possibility the return value could be out of sync. See remarks.\n\n\nCallback Safety\n---------------\nSafe. This is implemented as a simple accessor.\n\n\nRemarks\n-------\nThe general flow of a devices state goes like this:\n\n    ```\n    ma_device_init()  -> ma_device_state_uninitialized -> ma_device_state_stopped\n    ma_device_start() -> ma_device_state_starting      -> ma_device_state_started\n    ma_device_stop()  -> ma_device_state_stopping      -> ma_device_state_stopped\n    ```\n\nWhen the state of the device is changed with `ma_device_start()` or `ma_device_stop()` at this same time as this function is called, the\nvalue returned by this function could potentially be out of sync. If this is significant to your program you need to implement your own\nsynchronization.\n*/\nMA_API ma_device_state ma_device_get_state(const ma_device* pDevice);\n\n\n/*\nPerforms post backend initialization routines for setting up internal data conversion.\n\nThis should be called whenever the backend is initialized. The only time this should be called from\noutside of miniaudio is if you're implementing a custom backend, and you would only do it if you\nare reinitializing the backend due to rerouting or reinitializing for some reason.\n\n\nParameters\n----------\npDevice [in]\n    A pointer to the device.\n\ndeviceType [in]\n    The type of the device that was just reinitialized.\n\npPlaybackDescriptor [in]\n    The descriptor of the playback device containing the internal data format and buffer sizes.\n\npPlaybackDescriptor [in]\n    The descriptor of the capture device containing the internal data format and buffer sizes.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other error otherwise.\n\n\nThread Safety\n-------------\nUnsafe. This will be reinitializing internal data converters which may be in use by another thread.\n\n\nCallback Safety\n---------------\nUnsafe. This will be reinitializing internal data converters which may be in use by the callback.\n\n\nRemarks\n-------\nFor a duplex device, you can call this for only one side of the system. This is why the deviceType\nis specified as a parameter rather than deriving it from the device.\n\nYou do not need to call this manually unless you are doing a custom backend, in which case you need\nonly do it if you're manually performing rerouting or reinitialization.\n*/\nMA_API ma_result ma_device_post_init(ma_device* pDevice, ma_device_type deviceType, const ma_device_descriptor* pPlaybackDescriptor, const ma_device_descriptor* pCaptureDescriptor);\n\n\n/*\nSets the master volume factor for the device.\n\nThe volume factor must be between 0 (silence) and 1 (full volume). Use `ma_device_set_master_volume_db()` to use decibel notation, where 0 is full volume and\nvalues less than 0 decreases the volume.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose volume is being set.\n\nvolume (in)\n    The new volume factor. Must be >= 0.\n\n\nReturn Value\n------------\nMA_SUCCESS if the volume was set successfully.\nMA_INVALID_ARGS if pDevice is NULL.\nMA_INVALID_ARGS if volume is negative.\n\n\nThread Safety\n-------------\nSafe. This just sets a local member of the device object.\n\n\nCallback Safety\n---------------\nSafe. If you set the volume in the data callback, that data written to the output buffer will have the new volume applied.\n\n\nRemarks\n-------\nThis applies the volume factor across all channels.\n\nThis does not change the operating system's volume. It only affects the volume for the given `ma_device` object's audio stream.\n\n\nSee Also\n--------\nma_device_get_master_volume()\nma_device_set_master_volume_db()\nma_device_get_master_volume_db()\n*/\nMA_API ma_result ma_device_set_master_volume(ma_device* pDevice, float volume);\n\n/*\nRetrieves the master volume factor for the device.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose volume factor is being retrieved.\n\npVolume (in)\n    A pointer to the variable that will receive the volume factor. The returned value will be in the range of [0, 1].\n\n\nReturn Value\n------------\nMA_SUCCESS if successful.\nMA_INVALID_ARGS if pDevice is NULL.\nMA_INVALID_ARGS if pVolume is NULL.\n\n\nThread Safety\n-------------\nSafe. This just a simple member retrieval.\n\n\nCallback Safety\n---------------\nSafe.\n\n\nRemarks\n-------\nIf an error occurs, `*pVolume` will be set to 0.\n\n\nSee Also\n--------\nma_device_set_master_volume()\nma_device_set_master_volume_gain_db()\nma_device_get_master_volume_gain_db()\n*/\nMA_API ma_result ma_device_get_master_volume(ma_device* pDevice, float* pVolume);\n\n/*\nSets the master volume for the device as gain in decibels.\n\nA gain of 0 is full volume, whereas a gain of < 0 will decrease the volume.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose gain is being set.\n\ngainDB (in)\n    The new volume as gain in decibels. Must be less than or equal to 0, where 0 is full volume and anything less than 0 decreases the volume.\n\n\nReturn Value\n------------\nMA_SUCCESS if the volume was set successfully.\nMA_INVALID_ARGS if pDevice is NULL.\nMA_INVALID_ARGS if the gain is > 0.\n\n\nThread Safety\n-------------\nSafe. This just sets a local member of the device object.\n\n\nCallback Safety\n---------------\nSafe. If you set the volume in the data callback, that data written to the output buffer will have the new volume applied.\n\n\nRemarks\n-------\nThis applies the gain across all channels.\n\nThis does not change the operating system's volume. It only affects the volume for the given `ma_device` object's audio stream.\n\n\nSee Also\n--------\nma_device_get_master_volume_gain_db()\nma_device_set_master_volume()\nma_device_get_master_volume()\n*/\nMA_API ma_result ma_device_set_master_volume_db(ma_device* pDevice, float gainDB);\n\n/*\nRetrieves the master gain in decibels.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to the device whose gain is being retrieved.\n\npGainDB (in)\n    A pointer to the variable that will receive the gain in decibels. The returned value will be <= 0.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful.\nMA_INVALID_ARGS if pDevice is NULL.\nMA_INVALID_ARGS if pGainDB is NULL.\n\n\nThread Safety\n-------------\nSafe. This just a simple member retrieval.\n\n\nCallback Safety\n---------------\nSafe.\n\n\nRemarks\n-------\nIf an error occurs, `*pGainDB` will be set to 0.\n\n\nSee Also\n--------\nma_device_set_master_volume_db()\nma_device_set_master_volume()\nma_device_get_master_volume()\n*/\nMA_API ma_result ma_device_get_master_volume_db(ma_device* pDevice, float* pGainDB);\n\n\n/*\nCalled from the data callback of asynchronous backends to allow miniaudio to process the data and fire the miniaudio data callback.\n\n\nParameters\n----------\npDevice (in)\n    A pointer to device whose processing the data callback.\n\npOutput (out)\n    A pointer to the buffer that will receive the output PCM frame data. On a playback device this must not be NULL. On a duplex device\n    this can be NULL, in which case pInput must not be NULL.\n\npInput (in)\n    A pointer to the buffer containing input PCM frame data. On a capture device this must not be NULL. On a duplex device this can be\n    NULL, in which case `pOutput` must not be NULL.\n\nframeCount (in)\n    The number of frames being processed.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful; any other result code otherwise.\n\n\nThread Safety\n-------------\nThis function should only ever be called from the internal data callback of the backend. It is safe to call this simultaneously between a\nplayback and capture device in duplex setups.\n\n\nCallback Safety\n---------------\nDo not call this from the miniaudio data callback. It should only ever be called from the internal data callback of the backend.\n\n\nRemarks\n-------\nIf both `pOutput` and `pInput` are NULL, and error will be returned. In duplex scenarios, both `pOutput` and `pInput` can be non-NULL, in\nwhich case `pInput` will be processed first, followed by `pOutput`.\n\nIf you are implementing a custom backend, and that backend uses a callback for data delivery, you'll need to call this from inside that\ncallback.\n*/\nMA_API ma_result ma_device_handle_backend_data_callback(ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount);\n\n\n/*\nCalculates an appropriate buffer size from a descriptor, native sample rate and performance profile.\n\nThis function is used by backends for helping determine an appropriately sized buffer to use with\nthe device depending on the values of `periodSizeInFrames` and `periodSizeInMilliseconds` in the\n`pDescriptor` object. Since buffer size calculations based on time depends on the sample rate, a\nbest guess at the device's native sample rate is also required which is where `nativeSampleRate`\ncomes in. In addition, the performance profile is also needed for cases where both the period size\nin frames and milliseconds are both zero.\n\n\nParameters\n----------\npDescriptor (in)\n    A pointer to device descriptor whose `periodSizeInFrames` and `periodSizeInMilliseconds` members\n    will be used for the calculation of the buffer size.\n\nnativeSampleRate (in)\n    The device's native sample rate. This is only ever used when the `periodSizeInFrames` member of\n    `pDescriptor` is zero. In this case, `periodSizeInMilliseconds` will be used instead, in which\n    case a sample rate is required to convert to a size in frames.\n\nperformanceProfile (in)\n    When both the `periodSizeInFrames` and `periodSizeInMilliseconds` members of `pDescriptor` are\n    zero, miniaudio will fall back to a buffer size based on the performance profile. The profile\n    to use for this calculation is determine by this parameter.\n\n\nReturn Value\n------------\nThe calculated buffer size in frames.\n\n\nThread Safety\n-------------\nThis is safe so long as nothing modifies `pDescriptor` at the same time. However, this function\nshould only ever be called from within the backend's device initialization routine and therefore\nshouldn't have any multithreading concerns.\n\n\nCallback Safety\n---------------\nThis is safe to call within the data callback, but there is no reason to ever do this.\n\n\nRemarks\n-------\nIf `nativeSampleRate` is zero, this function will fall back to `pDescriptor->sampleRate`. If that\nis also zero, `MA_DEFAULT_SAMPLE_RATE` will be used instead.\n*/\nMA_API ma_uint32 ma_calculate_buffer_size_in_frames_from_descriptor(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile);\n\n\n\n/*\nRetrieves a friendly name for a backend.\n*/\nMA_API const char* ma_get_backend_name(ma_backend backend);\n\n/*\nRetrieves the backend enum from the given name.\n*/\nMA_API ma_result ma_get_backend_from_name(const char* pBackendName, ma_backend* pBackend);\n\n/*\nDetermines whether or not the given backend is available by the compilation environment.\n*/\nMA_API ma_bool32 ma_is_backend_enabled(ma_backend backend);\n\n/*\nRetrieves compile-time enabled backends.\n\n\nParameters\n----------\npBackends (out, optional)\n    A pointer to the buffer that will receive the enabled backends. Set to NULL to retrieve the backend count. Setting\n    the capacity of the buffer to `MA_BACKEND_COUNT` will guarantee it's large enough for all backends.\n\nbackendCap (in)\n    The capacity of the `pBackends` buffer.\n\npBackendCount (out)\n    A pointer to the variable that will receive the enabled backend count.\n\n\nReturn Value\n------------\nMA_SUCCESS if successful.\nMA_INVALID_ARGS if `pBackendCount` is NULL.\nMA_NO_SPACE if the capacity of `pBackends` is not large enough.\n\nIf `MA_NO_SPACE` is returned, the `pBackends` buffer will be filled with `*pBackendCount` values.\n\n\nThread Safety\n-------------\nSafe.\n\n\nCallback Safety\n---------------\nSafe.\n\n\nRemarks\n-------\nIf you want to retrieve the number of backends so you can determine the capacity of `pBackends` buffer, you can call\nthis function with `pBackends` set to NULL.\n\nThis will also enumerate the null backend. If you don't want to include this you need to check for `ma_backend_null`\nwhen you enumerate over the returned backends and handle it appropriately. Alternatively, you can disable it at\ncompile time with `MA_NO_NULL`.\n\nThe returned backends are determined based on compile time settings, not the platform it's currently running on. For\nexample, PulseAudio will be returned if it was enabled at compile time, even when the user doesn't actually have\nPulseAudio installed.\n\n\nExample 1\n---------\nThe example below retrieves the enabled backend count using a fixed sized buffer allocated on the stack. The buffer is\ngiven a capacity of `MA_BACKEND_COUNT` which will guarantee it'll be large enough to store all available backends.\nSince `MA_BACKEND_COUNT` is always a relatively small value, this should be suitable for most scenarios.\n\n```\nma_backend enabledBackends[MA_BACKEND_COUNT];\nsize_t enabledBackendCount;\n\nresult = ma_get_enabled_backends(enabledBackends, MA_BACKEND_COUNT, &enabledBackendCount);\nif (result != MA_SUCCESS) {\n    // Failed to retrieve enabled backends. Should never happen in this example since all inputs are valid.\n}\n```\n\n\nSee Also\n--------\nma_is_backend_enabled()\n*/\nMA_API ma_result ma_get_enabled_backends(ma_backend* pBackends, size_t backendCap, size_t* pBackendCount);\n\n/*\nDetermines whether or not loopback mode is support by a backend.\n*/\nMA_API ma_bool32 ma_is_loopback_supported(ma_backend backend);\n\n#endif  /* MA_NO_DEVICE_IO */\n\n\n\n/************************************************************************************************************************************************************\n\nUtilities\n\n************************************************************************************************************************************************************/\n\n/*\nCalculates a buffer size in milliseconds (rounded up) from the specified number of frames and sample rate.\n*/\nMA_API ma_uint32 ma_calculate_buffer_size_in_milliseconds_from_frames(ma_uint32 bufferSizeInFrames, ma_uint32 sampleRate);\n\n/*\nCalculates a buffer size in frames from the specified number of milliseconds and sample rate.\n*/\nMA_API ma_uint32 ma_calculate_buffer_size_in_frames_from_milliseconds(ma_uint32 bufferSizeInMilliseconds, ma_uint32 sampleRate);\n\n/*\nCopies PCM frames from one buffer to another.\n*/\nMA_API void ma_copy_pcm_frames(void* dst, const void* src, ma_uint64 frameCount, ma_format format, ma_uint32 channels);\n\n/*\nCopies silent frames into the given buffer.\n\nRemarks\n-------\nFor all formats except `ma_format_u8`, the output buffer will be filled with 0. For `ma_format_u8` it will be filled with 128. The reason for this is that it\nmakes more sense for the purpose of mixing to initialize it to the center point.\n*/\nMA_API void ma_silence_pcm_frames(void* p, ma_uint64 frameCount, ma_format format, ma_uint32 channels);\n\n\n/*\nOffsets a pointer by the specified number of PCM frames.\n*/\nMA_API void* ma_offset_pcm_frames_ptr(void* p, ma_uint64 offsetInFrames, ma_format format, ma_uint32 channels);\nMA_API const void* ma_offset_pcm_frames_const_ptr(const void* p, ma_uint64 offsetInFrames, ma_format format, ma_uint32 channels);\nstatic MA_INLINE float* ma_offset_pcm_frames_ptr_f32(float* p, ma_uint64 offsetInFrames, ma_uint32 channels) { return (float*)ma_offset_pcm_frames_ptr((void*)p, offsetInFrames, ma_format_f32, channels); }\nstatic MA_INLINE const float* ma_offset_pcm_frames_const_ptr_f32(const float* p, ma_uint64 offsetInFrames, ma_uint32 channels) { return (const float*)ma_offset_pcm_frames_const_ptr((const void*)p, offsetInFrames, ma_format_f32, channels); }\n\n\n/*\nClips samples.\n*/\nMA_API void ma_clip_samples_u8(ma_uint8* pDst, const ma_int16* pSrc, ma_uint64 count);\nMA_API void ma_clip_samples_s16(ma_int16* pDst, const ma_int32* pSrc, ma_uint64 count);\nMA_API void ma_clip_samples_s24(ma_uint8* pDst, const ma_int64* pSrc, ma_uint64 count);\nMA_API void ma_clip_samples_s32(ma_int32* pDst, const ma_int64* pSrc, ma_uint64 count);\nMA_API void ma_clip_samples_f32(float* pDst, const float* pSrc, ma_uint64 count);\nMA_API void ma_clip_pcm_frames(void* pDst, const void* pSrc, ma_uint64 frameCount, ma_format format, ma_uint32 channels);\n\n/*\nHelper for applying a volume factor to samples.\n\nNote that the source and destination buffers can be the same, in which case it'll perform the operation in-place.\n*/\nMA_API void ma_copy_and_apply_volume_factor_u8(ma_uint8* pSamplesOut, const ma_uint8* pSamplesIn, ma_uint64 sampleCount, float factor);\nMA_API void ma_copy_and_apply_volume_factor_s16(ma_int16* pSamplesOut, const ma_int16* pSamplesIn, ma_uint64 sampleCount, float factor);\nMA_API void ma_copy_and_apply_volume_factor_s24(void* pSamplesOut, const void* pSamplesIn, ma_uint64 sampleCount, float factor);\nMA_API void ma_copy_and_apply_volume_factor_s32(ma_int32* pSamplesOut, const ma_int32* pSamplesIn, ma_uint64 sampleCount, float factor);\nMA_API void ma_copy_and_apply_volume_factor_f32(float* pSamplesOut, const float* pSamplesIn, ma_uint64 sampleCount, float factor);\n\nMA_API void ma_apply_volume_factor_u8(ma_uint8* pSamples, ma_uint64 sampleCount, float factor);\nMA_API void ma_apply_volume_factor_s16(ma_int16* pSamples, ma_uint64 sampleCount, float factor);\nMA_API void ma_apply_volume_factor_s24(void* pSamples, ma_uint64 sampleCount, float factor);\nMA_API void ma_apply_volume_factor_s32(ma_int32* pSamples, ma_uint64 sampleCount, float factor);\nMA_API void ma_apply_volume_factor_f32(float* pSamples, ma_uint64 sampleCount, float factor);\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_u8(ma_uint8* pFramesOut, const ma_uint8* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s16(ma_int16* pFramesOut, const ma_int16* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s24(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s32(ma_int32* pFramesOut, const ma_int32* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float factor);\n\nMA_API void ma_apply_volume_factor_pcm_frames_u8(ma_uint8* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_apply_volume_factor_pcm_frames_s16(ma_int16* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_apply_volume_factor_pcm_frames_s24(void* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_apply_volume_factor_pcm_frames_s32(ma_int32* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_apply_volume_factor_pcm_frames_f32(float* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor);\nMA_API void ma_apply_volume_factor_pcm_frames(void* pFrames, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float factor);\n\nMA_API void ma_copy_and_apply_volume_factor_per_channel_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float* pChannelGains);\n\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_u8(ma_uint8* pDst, const ma_int16* pSrc, ma_uint64 count, float volume);\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s16(ma_int16* pDst, const ma_int32* pSrc, ma_uint64 count, float volume);\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s24(ma_uint8* pDst, const ma_int64* pSrc, ma_uint64 count, float volume);\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s32(ma_int32* pDst, const ma_int64* pSrc, ma_uint64 count, float volume);\nMA_API void ma_copy_and_apply_volume_and_clip_samples_f32(float* pDst, const float* pSrc, ma_uint64 count, float volume);\nMA_API void ma_copy_and_apply_volume_and_clip_pcm_frames(void* pDst, const void* pSrc, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float volume);\n\n\n/*\nHelper for converting a linear factor to gain in decibels.\n*/\nMA_API float ma_volume_linear_to_db(float factor);\n\n/*\nHelper for converting gain in decibels to a linear factor.\n*/\nMA_API float ma_volume_db_to_linear(float gain);\n\n\n/*\nMixes the specified number of frames in floating point format with a volume factor.\n\nThis will run on an optimized path when the volume is equal to 1.\n*/\nMA_API ma_result ma_mix_pcm_frames_f32(float* pDst, const float* pSrc, ma_uint64 frameCount, ma_uint32 channels, float volume);\n\n\n\n\n/************************************************************************************************************************************************************\n\nVFS\n===\n\nThe VFS object (virtual file system) is what's used to customize file access. This is useful in cases where stdio FILE* based APIs may not be entirely\nappropriate for a given situation.\n\n************************************************************************************************************************************************************/\ntypedef void      ma_vfs;\ntypedef ma_handle ma_vfs_file;\n\ntypedef enum\n{\n    MA_OPEN_MODE_READ  = 0x00000001,\n    MA_OPEN_MODE_WRITE = 0x00000002\n} ma_open_mode_flags;\n\ntypedef enum\n{\n    ma_seek_origin_start,\n    ma_seek_origin_current,\n    ma_seek_origin_end  /* Not used by decoders. */\n} ma_seek_origin;\n\ntypedef struct\n{\n    ma_uint64 sizeInBytes;\n} ma_file_info;\n\ntypedef struct\n{\n    ma_result (* onOpen) (ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile);\n    ma_result (* onOpenW)(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile);\n    ma_result (* onClose)(ma_vfs* pVFS, ma_vfs_file file);\n    ma_result (* onRead) (ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead);\n    ma_result (* onWrite)(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten);\n    ma_result (* onSeek) (ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin);\n    ma_result (* onTell) (ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor);\n    ma_result (* onInfo) (ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo);\n} ma_vfs_callbacks;\n\nMA_API ma_result ma_vfs_open(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile);\nMA_API ma_result ma_vfs_open_w(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile);\nMA_API ma_result ma_vfs_close(ma_vfs* pVFS, ma_vfs_file file);\nMA_API ma_result ma_vfs_read(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead);\nMA_API ma_result ma_vfs_write(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten);\nMA_API ma_result ma_vfs_seek(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin);\nMA_API ma_result ma_vfs_tell(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor);\nMA_API ma_result ma_vfs_info(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo);\nMA_API ma_result ma_vfs_open_and_read_file(ma_vfs* pVFS, const char* pFilePath, void** ppData, size_t* pSize, const ma_allocation_callbacks* pAllocationCallbacks);\n\ntypedef struct\n{\n    ma_vfs_callbacks cb;\n    ma_allocation_callbacks allocationCallbacks;    /* Only used for the wchar_t version of open() on non-Windows platforms. */\n} ma_default_vfs;\n\nMA_API ma_result ma_default_vfs_init(ma_default_vfs* pVFS, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n\ntypedef ma_result (* ma_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead);\ntypedef ma_result (* ma_seek_proc)(void* pUserData, ma_int64 offset, ma_seek_origin origin);\ntypedef ma_result (* ma_tell_proc)(void* pUserData, ma_int64* pCursor);\n\n\n\n#if !defined(MA_NO_DECODING) || !defined(MA_NO_ENCODING)\ntypedef enum\n{\n    ma_encoding_format_unknown = 0,\n    ma_encoding_format_wav,\n    ma_encoding_format_flac,\n    ma_encoding_format_mp3,\n    ma_encoding_format_vorbis\n} ma_encoding_format;\n#endif\n\n/************************************************************************************************************************************************************\n\nDecoding\n========\n\nDecoders are independent of the main device API. Decoding APIs can be called freely inside the device's data callback, but they are not thread safe unless\nyou do your own synchronization.\n\n************************************************************************************************************************************************************/\n#ifndef MA_NO_DECODING\ntypedef struct ma_decoder ma_decoder;\n\n\ntypedef struct\n{\n    ma_format preferredFormat;\n    ma_uint32 seekPointCount;   /* Set to > 0 to generate a seektable if the decoding backend supports it. */\n} ma_decoding_backend_config;\n\nMA_API ma_decoding_backend_config ma_decoding_backend_config_init(ma_format preferredFormat, ma_uint32 seekPointCount);\n\n\ntypedef struct\n{\n    ma_result (* onInit      )(void* pUserData, ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend);\n    ma_result (* onInitFile  )(void* pUserData, const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend);               /* Optional. */\n    ma_result (* onInitFileW )(void* pUserData, const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend);            /* Optional. */\n    ma_result (* onInitMemory)(void* pUserData, const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend);  /* Optional. */\n    void      (* onUninit    )(void* pUserData, ma_data_source* pBackend, const ma_allocation_callbacks* pAllocationCallbacks);\n} ma_decoding_backend_vtable;\n\n\ntypedef ma_result (* ma_decoder_read_proc)(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead);         /* Returns the number of bytes read. */\ntypedef ma_result (* ma_decoder_seek_proc)(ma_decoder* pDecoder, ma_int64 byteOffset, ma_seek_origin origin);\ntypedef ma_result (* ma_decoder_tell_proc)(ma_decoder* pDecoder, ma_int64* pCursor);\n\ntypedef struct\n{\n    ma_format format;      /* Set to 0 or ma_format_unknown to use the stream's internal format. */\n    ma_uint32 channels;    /* Set to 0 to use the stream's internal channels. */\n    ma_uint32 sampleRate;  /* Set to 0 to use the stream's internal sample rate. */\n    ma_channel* pChannelMap;\n    ma_channel_mix_mode channelMixMode;\n    ma_dither_mode ditherMode;\n    ma_resampler_config resampling;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_encoding_format encodingFormat;\n    ma_uint32 seekPointCount;   /* When set to > 0, specifies the number of seek points to use for the generation of a seek table. Not all decoding backends support this. */\n    ma_decoding_backend_vtable** ppCustomBackendVTables;\n    ma_uint32 customBackendCount;\n    void* pCustomBackendUserData;\n} ma_decoder_config;\n\nstruct ma_decoder\n{\n    ma_data_source_base ds;\n    ma_data_source* pBackend;                   /* The decoding backend we'll be pulling data from. */\n    const ma_decoding_backend_vtable* pBackendVTable; /* The vtable for the decoding backend. This needs to be stored so we can access the onUninit() callback. */\n    void* pBackendUserData;\n    ma_decoder_read_proc onRead;\n    ma_decoder_seek_proc onSeek;\n    ma_decoder_tell_proc onTell;\n    void* pUserData;\n    ma_uint64 readPointerInPCMFrames;      /* In output sample rate. Used for keeping track of how many frames are available for decoding. */\n    ma_format outputFormat;\n    ma_uint32 outputChannels;\n    ma_uint32 outputSampleRate;\n    ma_data_converter converter;    /* Data conversion is achieved by running frames through this. */\n    void* pInputCache;              /* In input format. Can be null if it's not needed. */\n    ma_uint64 inputCacheCap;        /* The capacity of the input cache. */\n    ma_uint64 inputCacheConsumed;   /* The number of frames that have been consumed in the cache. Used for determining the next valid frame. */\n    ma_uint64 inputCacheRemaining;  /* The number of valid frames remaining in the cache. */\n    ma_allocation_callbacks allocationCallbacks;\n    union\n    {\n        struct\n        {\n            ma_vfs* pVFS;\n            ma_vfs_file file;\n        } vfs;\n        struct\n        {\n            const ma_uint8* pData;\n            size_t dataSize;\n            size_t currentReadPos;\n        } memory;               /* Only used for decoders that were opened against a block of memory. */\n    } data;\n};\n\nMA_API ma_decoder_config ma_decoder_config_init(ma_format outputFormat, ma_uint32 outputChannels, ma_uint32 outputSampleRate);\nMA_API ma_decoder_config ma_decoder_config_init_default(void);\n\nMA_API ma_result ma_decoder_init(ma_decoder_read_proc onRead, ma_decoder_seek_proc onSeek, void* pUserData, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\nMA_API ma_result ma_decoder_init_memory(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\nMA_API ma_result ma_decoder_init_vfs(ma_vfs* pVFS, const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\nMA_API ma_result ma_decoder_init_vfs_w(ma_vfs* pVFS, const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\nMA_API ma_result ma_decoder_init_file(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\nMA_API ma_result ma_decoder_init_file_w(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder);\n\n/*\nUninitializes a decoder.\n*/\nMA_API ma_result ma_decoder_uninit(ma_decoder* pDecoder);\n\n/*\nReads PCM frames from the given decoder.\n\nThis is not thread safe without your own synchronization.\n*/\nMA_API ma_result ma_decoder_read_pcm_frames(ma_decoder* pDecoder, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\n\n/*\nSeeks to a PCM frame based on its absolute index.\n\nThis is not thread safe without your own synchronization.\n*/\nMA_API ma_result ma_decoder_seek_to_pcm_frame(ma_decoder* pDecoder, ma_uint64 frameIndex);\n\n/*\nRetrieves the decoder's output data format.\n*/\nMA_API ma_result ma_decoder_get_data_format(ma_decoder* pDecoder, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\n\n/*\nRetrieves the current position of the read cursor in PCM frames.\n*/\nMA_API ma_result ma_decoder_get_cursor_in_pcm_frames(ma_decoder* pDecoder, ma_uint64* pCursor);\n\n/*\nRetrieves the length of the decoder in PCM frames.\n\nDo not call this on streams of an undefined length, such as internet radio.\n\nIf the length is unknown or an error occurs, 0 will be returned.\n\nThis will always return 0 for Vorbis decoders. This is due to a limitation with stb_vorbis in push mode which is what miniaudio\nuses internally.\n\nFor MP3's, this will decode the entire file. Do not call this in time critical scenarios.\n\nThis function is not thread safe without your own synchronization.\n*/\nMA_API ma_result ma_decoder_get_length_in_pcm_frames(ma_decoder* pDecoder, ma_uint64* pLength);\n\n/*\nRetrieves the number of frames that can be read before reaching the end.\n\nThis calls `ma_decoder_get_length_in_pcm_frames()` so you need to be aware of the rules for that function, in\nparticular ensuring you do not call it on streams of an undefined length, such as internet radio.\n\nIf the total length of the decoder cannot be retrieved, such as with Vorbis decoders, `MA_NOT_IMPLEMENTED` will be\nreturned.\n*/\nMA_API ma_result ma_decoder_get_available_frames(ma_decoder* pDecoder, ma_uint64* pAvailableFrames);\n\n/*\nHelper for opening and decoding a file into a heap allocated block of memory. Free the returned pointer with ma_free(). On input,\npConfig should be set to what you want. On output it will be set to what you got.\n*/\nMA_API ma_result ma_decode_from_vfs(ma_vfs* pVFS, const char* pFilePath, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut);\nMA_API ma_result ma_decode_file(const char* pFilePath, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut);\nMA_API ma_result ma_decode_memory(const void* pData, size_t dataSize, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut);\n\n#endif  /* MA_NO_DECODING */\n\n\n/************************************************************************************************************************************************************\n\nEncoding\n========\n\nEncoders do not perform any format conversion for you. If your target format does not support the format, and error will be returned.\n\n************************************************************************************************************************************************************/\n#ifndef MA_NO_ENCODING\ntypedef struct ma_encoder ma_encoder;\n\ntypedef ma_result (* ma_encoder_write_proc)           (ma_encoder* pEncoder, const void* pBufferIn, size_t bytesToWrite, size_t* pBytesWritten);\ntypedef ma_result (* ma_encoder_seek_proc)            (ma_encoder* pEncoder, ma_int64 offset, ma_seek_origin origin);\ntypedef ma_result (* ma_encoder_init_proc)            (ma_encoder* pEncoder);\ntypedef void      (* ma_encoder_uninit_proc)          (ma_encoder* pEncoder);\ntypedef ma_result (* ma_encoder_write_pcm_frames_proc)(ma_encoder* pEncoder, const void* pFramesIn, ma_uint64 frameCount, ma_uint64* pFramesWritten);\n\ntypedef struct\n{\n    ma_encoding_format encodingFormat;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_allocation_callbacks allocationCallbacks;\n} ma_encoder_config;\n\nMA_API ma_encoder_config ma_encoder_config_init(ma_encoding_format encodingFormat, ma_format format, ma_uint32 channels, ma_uint32 sampleRate);\n\nstruct ma_encoder\n{\n    ma_encoder_config config;\n    ma_encoder_write_proc onWrite;\n    ma_encoder_seek_proc onSeek;\n    ma_encoder_init_proc onInit;\n    ma_encoder_uninit_proc onUninit;\n    ma_encoder_write_pcm_frames_proc onWritePCMFrames;\n    void* pUserData;\n    void* pInternalEncoder;\n    union\n    {\n        struct\n        {\n            ma_vfs* pVFS;\n            ma_vfs_file file;\n        } vfs;\n    } data;\n};\n\nMA_API ma_result ma_encoder_init(ma_encoder_write_proc onWrite, ma_encoder_seek_proc onSeek, void* pUserData, const ma_encoder_config* pConfig, ma_encoder* pEncoder);\nMA_API ma_result ma_encoder_init_vfs(ma_vfs* pVFS, const char* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder);\nMA_API ma_result ma_encoder_init_vfs_w(ma_vfs* pVFS, const wchar_t* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder);\nMA_API ma_result ma_encoder_init_file(const char* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder);\nMA_API ma_result ma_encoder_init_file_w(const wchar_t* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder);\nMA_API void ma_encoder_uninit(ma_encoder* pEncoder);\nMA_API ma_result ma_encoder_write_pcm_frames(ma_encoder* pEncoder, const void* pFramesIn, ma_uint64 frameCount, ma_uint64* pFramesWritten);\n\n#endif /* MA_NO_ENCODING */\n\n\n/************************************************************************************************************************************************************\n\nGeneration\n\n************************************************************************************************************************************************************/\n#ifndef MA_NO_GENERATION\ntypedef enum\n{\n    ma_waveform_type_sine,\n    ma_waveform_type_square,\n    ma_waveform_type_triangle,\n    ma_waveform_type_sawtooth\n} ma_waveform_type;\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_waveform_type type;\n    double amplitude;\n    double frequency;\n} ma_waveform_config;\n\nMA_API ma_waveform_config ma_waveform_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, ma_waveform_type type, double amplitude, double frequency);\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_waveform_config config;\n    double advance;\n    double time;\n} ma_waveform;\n\nMA_API ma_result ma_waveform_init(const ma_waveform_config* pConfig, ma_waveform* pWaveform);\nMA_API void ma_waveform_uninit(ma_waveform* pWaveform);\nMA_API ma_result ma_waveform_read_pcm_frames(ma_waveform* pWaveform, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_waveform_seek_to_pcm_frame(ma_waveform* pWaveform, ma_uint64 frameIndex);\nMA_API ma_result ma_waveform_set_amplitude(ma_waveform* pWaveform, double amplitude);\nMA_API ma_result ma_waveform_set_frequency(ma_waveform* pWaveform, double frequency);\nMA_API ma_result ma_waveform_set_type(ma_waveform* pWaveform, ma_waveform_type type);\nMA_API ma_result ma_waveform_set_sample_rate(ma_waveform* pWaveform, ma_uint32 sampleRate);\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    double dutyCycle;\n    double amplitude;\n    double frequency;\n} ma_pulsewave_config;\n\nMA_API ma_pulsewave_config ma_pulsewave_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double dutyCycle, double amplitude, double frequency);\n\ntypedef struct\n{\n    ma_waveform waveform;\n    ma_pulsewave_config config;\n} ma_pulsewave;\n\nMA_API ma_result ma_pulsewave_init(const ma_pulsewave_config* pConfig, ma_pulsewave* pWaveform);\nMA_API void ma_pulsewave_uninit(ma_pulsewave* pWaveform);\nMA_API ma_result ma_pulsewave_read_pcm_frames(ma_pulsewave* pWaveform, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_pulsewave_seek_to_pcm_frame(ma_pulsewave* pWaveform, ma_uint64 frameIndex);\nMA_API ma_result ma_pulsewave_set_amplitude(ma_pulsewave* pWaveform, double amplitude);\nMA_API ma_result ma_pulsewave_set_frequency(ma_pulsewave* pWaveform, double frequency);\nMA_API ma_result ma_pulsewave_set_sample_rate(ma_pulsewave* pWaveform, ma_uint32 sampleRate);\nMA_API ma_result ma_pulsewave_set_duty_cycle(ma_pulsewave* pWaveform, double dutyCycle);\n\ntypedef enum\n{\n    ma_noise_type_white,\n    ma_noise_type_pink,\n    ma_noise_type_brownian\n} ma_noise_type;\n\n\ntypedef struct\n{\n    ma_format format;\n    ma_uint32 channels;\n    ma_noise_type type;\n    ma_int32 seed;\n    double amplitude;\n    ma_bool32 duplicateChannels;\n} ma_noise_config;\n\nMA_API ma_noise_config ma_noise_config_init(ma_format format, ma_uint32 channels, ma_noise_type type, ma_int32 seed, double amplitude);\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_noise_config config;\n    ma_lcg lcg;\n    union\n    {\n        struct\n        {\n            double** bin;\n            double* accumulation;\n            ma_uint32* counter;\n        } pink;\n        struct\n        {\n            double* accumulation;\n        } brownian;\n    } state;\n\n    /* Memory management. */\n    void* _pHeap;\n    ma_bool32 _ownsHeap;\n} ma_noise;\n\nMA_API ma_result ma_noise_get_heap_size(const ma_noise_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_noise_init_preallocated(const ma_noise_config* pConfig, void* pHeap, ma_noise* pNoise);\nMA_API ma_result ma_noise_init(const ma_noise_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_noise* pNoise);\nMA_API void ma_noise_uninit(ma_noise* pNoise, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_noise_read_pcm_frames(ma_noise* pNoise, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_noise_set_amplitude(ma_noise* pNoise, double amplitude);\nMA_API ma_result ma_noise_set_seed(ma_noise* pNoise, ma_int32 seed);\nMA_API ma_result ma_noise_set_type(ma_noise* pNoise, ma_noise_type type);\n\n#endif  /* MA_NO_GENERATION */\n\n\n\n/************************************************************************************************************************************************************\n\nResource Manager\n\n************************************************************************************************************************************************************/\n/* The resource manager cannot be enabled if there is no decoder. */\n#if !defined(MA_NO_RESOURCE_MANAGER) && defined(MA_NO_DECODING)\n#define MA_NO_RESOURCE_MANAGER\n#endif\n\n#ifndef MA_NO_RESOURCE_MANAGER\ntypedef struct ma_resource_manager                  ma_resource_manager;\ntypedef struct ma_resource_manager_data_buffer_node ma_resource_manager_data_buffer_node;\ntypedef struct ma_resource_manager_data_buffer      ma_resource_manager_data_buffer;\ntypedef struct ma_resource_manager_data_stream      ma_resource_manager_data_stream;\ntypedef struct ma_resource_manager_data_source      ma_resource_manager_data_source;\n\ntypedef enum\n{\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM         = 0x00000001,   /* When set, does not load the entire data source in memory. Disk I/O will happen on job threads. */\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE         = 0x00000002,   /* Decode data before storing in memory. When set, decoding is done at the resource manager level rather than the mixing thread. Results in faster mixing, but higher memory usage. */\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC          = 0x00000004,   /* When set, the resource manager will load the data source asynchronously. */\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT      = 0x00000008,   /* When set, waits for initialization of the underlying data source before returning from ma_resource_manager_data_source_init(). */\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_UNKNOWN_LENGTH = 0x00000010,   /* Gives the resource manager a hint that the length of the data source is unknown and calling `ma_data_source_get_length_in_pcm_frames()` should be avoided. */\n    MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING        = 0x00000020    /* When set, configures the data source to loop by default. */\n} ma_resource_manager_data_source_flags;\n\n\n/*\nPipeline notifications used by the resource manager. Made up of both an async notification and a fence, both of which are optional.\n*/\ntypedef struct\n{\n    ma_async_notification* pNotification;\n    ma_fence* pFence;\n} ma_resource_manager_pipeline_stage_notification;\n\ntypedef struct\n{\n    ma_resource_manager_pipeline_stage_notification init;    /* Initialization of the decoder. */\n    ma_resource_manager_pipeline_stage_notification done;    /* Decoding fully completed. */\n} ma_resource_manager_pipeline_notifications;\n\nMA_API ma_resource_manager_pipeline_notifications ma_resource_manager_pipeline_notifications_init(void);\n\n\n\n/* BEGIN BACKWARDS COMPATIBILITY */\n/* TODO: Remove this block in version 0.12. */\n#if 1\n#define ma_resource_manager_job                         ma_job\n#define ma_resource_manager_job_init                    ma_job_init\n#define MA_JOB_TYPE_RESOURCE_MANAGER_QUEUE_FLAG_NON_BLOCKING MA_JOB_QUEUE_FLAG_NON_BLOCKING\n#define ma_resource_manager_job_queue_config            ma_job_queue_config\n#define ma_resource_manager_job_queue_config_init       ma_job_queue_config_init\n#define ma_resource_manager_job_queue                   ma_job_queue\n#define ma_resource_manager_job_queue_get_heap_size     ma_job_queue_get_heap_size\n#define ma_resource_manager_job_queue_init_preallocated ma_job_queue_init_preallocated\n#define ma_resource_manager_job_queue_init              ma_job_queue_init\n#define ma_resource_manager_job_queue_uninit            ma_job_queue_uninit\n#define ma_resource_manager_job_queue_post              ma_job_queue_post\n#define ma_resource_manager_job_queue_next              ma_job_queue_next\n#endif\n/* END BACKWARDS COMPATIBILITY */\n\n\n\n\n/* Maximum job thread count will be restricted to this, but this may be removed later and replaced with a heap allocation thereby removing any limitation. */\n#ifndef MA_RESOURCE_MANAGER_MAX_JOB_THREAD_COUNT\n#define MA_RESOURCE_MANAGER_MAX_JOB_THREAD_COUNT    64\n#endif\n\ntypedef enum\n{\n    /* Indicates ma_resource_manager_next_job() should not block. Only valid when the job thread count is 0. */\n    MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING = 0x00000001,\n\n    /* Disables any kind of multithreading. Implicitly enables MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING. */\n    MA_RESOURCE_MANAGER_FLAG_NO_THREADING = 0x00000002\n} ma_resource_manager_flags;\n\ntypedef struct\n{\n    const char* pFilePath;\n    const wchar_t* pFilePathW;\n    const ma_resource_manager_pipeline_notifications* pNotifications;\n    ma_uint64 initialSeekPointInPCMFrames;\n    ma_uint64 rangeBegInPCMFrames;\n    ma_uint64 rangeEndInPCMFrames;\n    ma_uint64 loopPointBegInPCMFrames;\n    ma_uint64 loopPointEndInPCMFrames;\n    ma_uint32 flags;\n    ma_bool32 isLooping;    /* Deprecated. Use the MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING flag in `flags` instead. */\n} ma_resource_manager_data_source_config;\n\nMA_API ma_resource_manager_data_source_config ma_resource_manager_data_source_config_init(void);\n\n\ntypedef enum\n{\n    ma_resource_manager_data_supply_type_unknown = 0,   /* Used for determining whether or the data supply has been initialized. */\n    ma_resource_manager_data_supply_type_encoded,       /* Data supply is an encoded buffer. Connector is ma_decoder. */\n    ma_resource_manager_data_supply_type_decoded,       /* Data supply is a decoded buffer. Connector is ma_audio_buffer. */\n    ma_resource_manager_data_supply_type_decoded_paged  /* Data supply is a linked list of decoded buffers. Connector is ma_paged_audio_buffer. */\n} ma_resource_manager_data_supply_type;\n\ntypedef struct\n{\n    MA_ATOMIC(4, ma_resource_manager_data_supply_type) type;    /* Read and written from different threads so needs to be accessed atomically. */\n    union\n    {\n        struct\n        {\n            const void* pData;\n            size_t sizeInBytes;\n        } encoded;\n        struct\n        {\n            const void* pData;\n            ma_uint64 totalFrameCount;\n            ma_uint64 decodedFrameCount;\n            ma_format format;\n            ma_uint32 channels;\n            ma_uint32 sampleRate;\n        } decoded;\n        struct\n        {\n            ma_paged_audio_buffer_data data;\n            ma_uint64 decodedFrameCount;\n            ma_uint32 sampleRate;\n        } decodedPaged;\n    } backend;\n} ma_resource_manager_data_supply;\n\nstruct ma_resource_manager_data_buffer_node\n{\n    ma_uint32 hashedName32;                         /* The hashed name. This is the key. */\n    ma_uint32 refCount;\n    MA_ATOMIC(4, ma_result) result;                 /* Result from asynchronous loading. When loading set to MA_BUSY. When fully loaded set to MA_SUCCESS. When deleting set to MA_UNAVAILABLE. */\n    MA_ATOMIC(4, ma_uint32) executionCounter;       /* For allocating execution orders for jobs. */\n    MA_ATOMIC(4, ma_uint32) executionPointer;       /* For managing the order of execution for asynchronous jobs relating to this object. Incremented as jobs complete processing. */\n    ma_bool32 isDataOwnedByResourceManager;         /* Set to true when the underlying data buffer was allocated the resource manager. Set to false if it is owned by the application (via ma_resource_manager_register_*()). */\n    ma_resource_manager_data_supply data;\n    ma_resource_manager_data_buffer_node* pParent;\n    ma_resource_manager_data_buffer_node* pChildLo;\n    ma_resource_manager_data_buffer_node* pChildHi;\n};\n\nstruct ma_resource_manager_data_buffer\n{\n    ma_data_source_base ds;                         /* Base data source. A data buffer is a data source. */\n    ma_resource_manager* pResourceManager;          /* A pointer to the resource manager that owns this buffer. */\n    ma_resource_manager_data_buffer_node* pNode;    /* The data node. This is reference counted and is what supplies the data. */\n    ma_uint32 flags;                                /* The flags that were passed used to initialize the buffer. */\n    MA_ATOMIC(4, ma_uint32) executionCounter;       /* For allocating execution orders for jobs. */\n    MA_ATOMIC(4, ma_uint32) executionPointer;       /* For managing the order of execution for asynchronous jobs relating to this object. Incremented as jobs complete processing. */\n    ma_uint64 seekTargetInPCMFrames;                /* Only updated by the public API. Never written nor read from the job thread. */\n    ma_bool32 seekToCursorOnNextRead;               /* On the next read we need to seek to the frame cursor. */\n    MA_ATOMIC(4, ma_result) result;                 /* Keeps track of a result of decoding. Set to MA_BUSY while the buffer is still loading. Set to MA_SUCCESS when loading is finished successfully. Otherwise set to some other code. */\n    MA_ATOMIC(4, ma_bool32) isLooping;              /* Can be read and written by different threads at the same time. Must be used atomically. */\n    ma_atomic_bool32 isConnectorInitialized;        /* Used for asynchronous loading to ensure we don't try to initialize the connector multiple times while waiting for the node to fully load. */\n    union\n    {\n        ma_decoder decoder;                 /* Supply type is ma_resource_manager_data_supply_type_encoded */\n        ma_audio_buffer buffer;             /* Supply type is ma_resource_manager_data_supply_type_decoded */\n        ma_paged_audio_buffer pagedBuffer;  /* Supply type is ma_resource_manager_data_supply_type_decoded_paged */\n    } connector;    /* Connects this object to the node's data supply. */\n};\n\nstruct ma_resource_manager_data_stream\n{\n    ma_data_source_base ds;                     /* Base data source. A data stream is a data source. */\n    ma_resource_manager* pResourceManager;      /* A pointer to the resource manager that owns this data stream. */\n    ma_uint32 flags;                            /* The flags that were passed used to initialize the stream. */\n    ma_decoder decoder;                         /* Used for filling pages with data. This is only ever accessed by the job thread. The public API should never touch this. */\n    ma_bool32 isDecoderInitialized;             /* Required for determining whether or not the decoder should be uninitialized in MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_STREAM. */\n    ma_uint64 totalLengthInPCMFrames;           /* This is calculated when first loaded by the MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_STREAM. */\n    ma_uint32 relativeCursor;                   /* The playback cursor, relative to the current page. Only ever accessed by the public API. Never accessed by the job thread. */\n    MA_ATOMIC(8, ma_uint64) absoluteCursor;     /* The playback cursor, in absolute position starting from the start of the file. */\n    ma_uint32 currentPageIndex;                 /* Toggles between 0 and 1. Index 0 is the first half of pPageData. Index 1 is the second half. Only ever accessed by the public API. Never accessed by the job thread. */\n    MA_ATOMIC(4, ma_uint32) executionCounter;   /* For allocating execution orders for jobs. */\n    MA_ATOMIC(4, ma_uint32) executionPointer;   /* For managing the order of execution for asynchronous jobs relating to this object. Incremented as jobs complete processing. */\n\n    /* Written by the public API, read by the job thread. */\n    MA_ATOMIC(4, ma_bool32) isLooping;          /* Whether or not the stream is looping. It's important to set the looping flag at the data stream level for smooth loop transitions. */\n\n    /* Written by the job thread, read by the public API. */\n    void* pPageData;                            /* Buffer containing the decoded data of each page. Allocated once at initialization time. */\n    MA_ATOMIC(4, ma_uint32) pageFrameCount[2];  /* The number of valid PCM frames in each page. Used to determine the last valid frame. */\n\n    /* Written and read by both the public API and the job thread. These must be atomic. */\n    MA_ATOMIC(4, ma_result) result;             /* Result from asynchronous loading. When loading set to MA_BUSY. When initialized set to MA_SUCCESS. When deleting set to MA_UNAVAILABLE. If an error occurs when loading, set to an error code. */\n    MA_ATOMIC(4, ma_bool32) isDecoderAtEnd;     /* Whether or not the decoder has reached the end. */\n    MA_ATOMIC(4, ma_bool32) isPageValid[2];     /* Booleans to indicate whether or not a page is valid. Set to false by the public API, set to true by the job thread. Set to false as the pages are consumed, true when they are filled. */\n    MA_ATOMIC(4, ma_bool32) seekCounter;        /* When 0, no seeking is being performed. When > 0, a seek is being performed and reading should be delayed with MA_BUSY. */\n};\n\nstruct ma_resource_manager_data_source\n{\n    union\n    {\n        ma_resource_manager_data_buffer buffer;\n        ma_resource_manager_data_stream stream;\n    } backend;  /* Must be the first item because we need the first item to be the data source callbacks for the buffer or stream. */\n\n    ma_uint32 flags;                          /* The flags that were passed in to ma_resource_manager_data_source_init(). */\n    MA_ATOMIC(4, ma_uint32) executionCounter;     /* For allocating execution orders for jobs. */\n    MA_ATOMIC(4, ma_uint32) executionPointer;     /* For managing the order of execution for asynchronous jobs relating to this object. Incremented as jobs complete processing. */\n};\n\ntypedef struct\n{\n    ma_allocation_callbacks allocationCallbacks;\n    ma_log* pLog;\n    ma_format decodedFormat;        /* The decoded format to use. Set to ma_format_unknown (default) to use the file's native format. */\n    ma_uint32 decodedChannels;      /* The decoded channel count to use. Set to 0 (default) to use the file's native channel count. */\n    ma_uint32 decodedSampleRate;    /* the decoded sample rate to use. Set to 0 (default) to use the file's native sample rate. */\n    ma_uint32 jobThreadCount;       /* Set to 0 if you want to self-manage your job threads. Defaults to 1. */\n    size_t jobThreadStackSize;\n    ma_uint32 jobQueueCapacity;     /* The maximum number of jobs that can fit in the queue at a time. Defaults to MA_JOB_TYPE_RESOURCE_MANAGER_QUEUE_CAPACITY. Cannot be zero. */\n    ma_uint32 flags;\n    ma_vfs* pVFS;                   /* Can be NULL in which case defaults will be used. */\n    ma_decoding_backend_vtable** ppCustomDecodingBackendVTables;\n    ma_uint32 customDecodingBackendCount;\n    void* pCustomDecodingBackendUserData;\n    ma_resampler_config resampling;\n} ma_resource_manager_config;\n\nMA_API ma_resource_manager_config ma_resource_manager_config_init(void);\n\nstruct ma_resource_manager\n{\n    ma_resource_manager_config config;\n    ma_resource_manager_data_buffer_node* pRootDataBufferNode;      /* The root buffer in the binary tree. */\n#ifndef MA_NO_THREADING\n    ma_mutex dataBufferBSTLock;                                     /* For synchronizing access to the data buffer binary tree. */\n    ma_thread jobThreads[MA_RESOURCE_MANAGER_MAX_JOB_THREAD_COUNT]; /* The threads for executing jobs. */\n#endif\n    ma_job_queue jobQueue;                                          /* Multi-consumer, multi-producer job queue for managing jobs for asynchronous decoding and streaming. */\n    ma_default_vfs defaultVFS;                                      /* Only used if a custom VFS is not specified. */\n    ma_log log;                                                     /* Only used if no log was specified in the config. */\n};\n\n/* Init. */\nMA_API ma_result ma_resource_manager_init(const ma_resource_manager_config* pConfig, ma_resource_manager* pResourceManager);\nMA_API void ma_resource_manager_uninit(ma_resource_manager* pResourceManager);\nMA_API ma_log* ma_resource_manager_get_log(ma_resource_manager* pResourceManager);\n\n/* Registration. */\nMA_API ma_result ma_resource_manager_register_file(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags);\nMA_API ma_result ma_resource_manager_register_file_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags);\nMA_API ma_result ma_resource_manager_register_decoded_data(ma_resource_manager* pResourceManager, const char* pName, const void* pData, ma_uint64 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate);  /* Does not copy. Increments the reference count if already exists and returns MA_SUCCESS. */\nMA_API ma_result ma_resource_manager_register_decoded_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName, const void* pData, ma_uint64 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate);\nMA_API ma_result ma_resource_manager_register_encoded_data(ma_resource_manager* pResourceManager, const char* pName, const void* pData, size_t sizeInBytes);    /* Does not copy. Increments the reference count if already exists and returns MA_SUCCESS. */\nMA_API ma_result ma_resource_manager_register_encoded_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName, const void* pData, size_t sizeInBytes);\nMA_API ma_result ma_resource_manager_unregister_file(ma_resource_manager* pResourceManager, const char* pFilePath);\nMA_API ma_result ma_resource_manager_unregister_file_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath);\nMA_API ma_result ma_resource_manager_unregister_data(ma_resource_manager* pResourceManager, const char* pName);\nMA_API ma_result ma_resource_manager_unregister_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName);\n\n/* Data Buffers. */\nMA_API ma_result ma_resource_manager_data_buffer_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_init(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_init_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_init_copy(ma_resource_manager* pResourceManager, const ma_resource_manager_data_buffer* pExistingDataBuffer, ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_uninit(ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_read_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_resource_manager_data_buffer_seek_to_pcm_frame(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64 frameIndex);\nMA_API ma_result ma_resource_manager_data_buffer_get_data_format(ma_resource_manager_data_buffer* pDataBuffer, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_resource_manager_data_buffer_get_cursor_in_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pCursor);\nMA_API ma_result ma_resource_manager_data_buffer_get_length_in_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pLength);\nMA_API ma_result ma_resource_manager_data_buffer_result(const ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_set_looping(ma_resource_manager_data_buffer* pDataBuffer, ma_bool32 isLooping);\nMA_API ma_bool32 ma_resource_manager_data_buffer_is_looping(const ma_resource_manager_data_buffer* pDataBuffer);\nMA_API ma_result ma_resource_manager_data_buffer_get_available_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pAvailableFrames);\n\n/* Data Streams. */\nMA_API ma_result ma_resource_manager_data_stream_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_init(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_init_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_uninit(ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_read_pcm_frames(ma_resource_manager_data_stream* pDataStream, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_resource_manager_data_stream_seek_to_pcm_frame(ma_resource_manager_data_stream* pDataStream, ma_uint64 frameIndex);\nMA_API ma_result ma_resource_manager_data_stream_get_data_format(ma_resource_manager_data_stream* pDataStream, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_resource_manager_data_stream_get_cursor_in_pcm_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pCursor);\nMA_API ma_result ma_resource_manager_data_stream_get_length_in_pcm_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pLength);\nMA_API ma_result ma_resource_manager_data_stream_result(const ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_set_looping(ma_resource_manager_data_stream* pDataStream, ma_bool32 isLooping);\nMA_API ma_bool32 ma_resource_manager_data_stream_is_looping(const ma_resource_manager_data_stream* pDataStream);\nMA_API ma_result ma_resource_manager_data_stream_get_available_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pAvailableFrames);\n\n/* Data Sources. */\nMA_API ma_result ma_resource_manager_data_source_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_init(ma_resource_manager* pResourceManager, const char* pName, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_init_w(ma_resource_manager* pResourceManager, const wchar_t* pName, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_init_copy(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source* pExistingDataSource, ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_uninit(ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_read_pcm_frames(ma_resource_manager_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_resource_manager_data_source_seek_to_pcm_frame(ma_resource_manager_data_source* pDataSource, ma_uint64 frameIndex);\nMA_API ma_result ma_resource_manager_data_source_get_data_format(ma_resource_manager_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_resource_manager_data_source_get_cursor_in_pcm_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pCursor);\nMA_API ma_result ma_resource_manager_data_source_get_length_in_pcm_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pLength);\nMA_API ma_result ma_resource_manager_data_source_result(const ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_set_looping(ma_resource_manager_data_source* pDataSource, ma_bool32 isLooping);\nMA_API ma_bool32 ma_resource_manager_data_source_is_looping(const ma_resource_manager_data_source* pDataSource);\nMA_API ma_result ma_resource_manager_data_source_get_available_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pAvailableFrames);\n\n/* Job management. */\nMA_API ma_result ma_resource_manager_post_job(ma_resource_manager* pResourceManager, const ma_job* pJob);\nMA_API ma_result ma_resource_manager_post_job_quit(ma_resource_manager* pResourceManager);  /* Helper for posting a quit job. */\nMA_API ma_result ma_resource_manager_next_job(ma_resource_manager* pResourceManager, ma_job* pJob);\nMA_API ma_result ma_resource_manager_process_job(ma_resource_manager* pResourceManager, ma_job* pJob);  /* DEPRECATED. Use ma_job_process(). Will be removed in version 0.12. */\nMA_API ma_result ma_resource_manager_process_next_job(ma_resource_manager* pResourceManager);   /* Returns MA_CANCELLED if a MA_JOB_TYPE_QUIT job is found. In non-blocking mode, returns MA_NO_DATA_AVAILABLE if no jobs are available. */\n#endif  /* MA_NO_RESOURCE_MANAGER */\n\n\n\n/************************************************************************************************************************************************************\n\nNode Graph\n\n************************************************************************************************************************************************************/\n#ifndef MA_NO_NODE_GRAPH\n/* Must never exceed 254. */\n#ifndef MA_MAX_NODE_BUS_COUNT\n#define MA_MAX_NODE_BUS_COUNT       254\n#endif\n\n/* Used internally by miniaudio for memory management. Must never exceed MA_MAX_NODE_BUS_COUNT. */\n#ifndef MA_MAX_NODE_LOCAL_BUS_COUNT\n#define MA_MAX_NODE_LOCAL_BUS_COUNT 2\n#endif\n\n/* Use this when the bus count is determined by the node instance rather than the vtable. */\n#define MA_NODE_BUS_COUNT_UNKNOWN   255\n\n\n/* For some internal memory management of ma_node_graph. */\ntypedef struct\n{\n    size_t offset;\n    size_t sizeInBytes;\n    unsigned char _data[1];\n} ma_stack;\n\n\ntypedef struct ma_node_graph ma_node_graph;\ntypedef void ma_node;\n\n\n/* Node flags. */\ntypedef enum\n{\n    MA_NODE_FLAG_PASSTHROUGH                = 0x00000001,\n    MA_NODE_FLAG_CONTINUOUS_PROCESSING      = 0x00000002,\n    MA_NODE_FLAG_ALLOW_NULL_INPUT           = 0x00000004,\n    MA_NODE_FLAG_DIFFERENT_PROCESSING_RATES = 0x00000008,\n    MA_NODE_FLAG_SILENT_OUTPUT              = 0x00000010\n} ma_node_flags;\n\n\n/* The playback state of a node. Either started or stopped. */\ntypedef enum\n{\n    ma_node_state_started = 0,\n    ma_node_state_stopped = 1\n} ma_node_state;\n\n\ntypedef struct\n{\n    /*\n    Extended processing callback. This callback is used for effects that process input and output\n    at different rates (i.e. they perform resampling). This is similar to the simple version, only\n    they take two separate frame counts: one for input, and one for output.\n\n    On input, `pFrameCountOut` is equal to the capacity of the output buffer for each bus, whereas\n    `pFrameCountIn` will be equal to the number of PCM frames in each of the buffers in `ppFramesIn`.\n\n    On output, set `pFrameCountOut` to the number of PCM frames that were actually output and set\n    `pFrameCountIn` to the number of input frames that were consumed.\n    */\n    void (* onProcess)(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut);\n\n    /*\n    A callback for retrieving the number of input frames that are required to output the\n    specified number of output frames. You would only want to implement this when the node performs\n    resampling. This is optional, even for nodes that perform resampling, but it does offer a\n    small reduction in latency as it allows miniaudio to calculate the exact number of input frames\n    to read at a time instead of having to estimate.\n    */\n    ma_result (* onGetRequiredInputFrameCount)(ma_node* pNode, ma_uint32 outputFrameCount, ma_uint32* pInputFrameCount);\n\n    /*\n    The number of input buses. This is how many sub-buffers will be contained in the `ppFramesIn`\n    parameters of the callbacks above.\n    */\n    ma_uint8 inputBusCount;\n\n    /*\n    The number of output buses. This is how many sub-buffers will be contained in the `ppFramesOut`\n    parameters of the callbacks above.\n    */\n    ma_uint8 outputBusCount;\n\n    /*\n    Flags describing characteristics of the node. This is currently just a placeholder for some\n    ideas for later on.\n    */\n    ma_uint32 flags;\n} ma_node_vtable;\n\ntypedef struct\n{\n    const ma_node_vtable* vtable;       /* Should never be null. Initialization of the node will fail if so. */\n    ma_node_state initialState;         /* Defaults to ma_node_state_started. */\n    ma_uint32 inputBusCount;            /* Only used if the vtable specifies an input bus count of `MA_NODE_BUS_COUNT_UNKNOWN`, otherwise must be set to `MA_NODE_BUS_COUNT_UNKNOWN` (default). */\n    ma_uint32 outputBusCount;           /* Only used if the vtable specifies an output bus count of `MA_NODE_BUS_COUNT_UNKNOWN`, otherwise  be set to `MA_NODE_BUS_COUNT_UNKNOWN` (default). */\n    const ma_uint32* pInputChannels;    /* The number of elements are determined by the input bus count as determined by the vtable, or `inputBusCount` if the vtable specifies `MA_NODE_BUS_COUNT_UNKNOWN`. */\n    const ma_uint32* pOutputChannels;   /* The number of elements are determined by the output bus count as determined by the vtable, or `outputBusCount` if the vtable specifies `MA_NODE_BUS_COUNT_UNKNOWN`. */\n} ma_node_config;\n\nMA_API ma_node_config ma_node_config_init(void);\n\n\n/*\nA node has multiple output buses. An output bus is attached to an input bus as an item in a linked\nlist. Think of the input bus as a linked list, with the output bus being an item in that list.\n*/\ntypedef struct ma_node_output_bus ma_node_output_bus;\nstruct ma_node_output_bus\n{\n    /* Immutable. */\n    ma_node* pNode;                                         /* The node that owns this output bus. The input node. Will be null for dummy head and tail nodes. */\n    ma_uint8 outputBusIndex;                                /* The index of the output bus on pNode that this output bus represents. */\n    ma_uint8 channels;                                      /* The number of channels in the audio stream for this bus. */\n\n    /* Mutable via multiple threads. Must be used atomically. The weird ordering here is for packing reasons. */\n    ma_uint8 inputNodeInputBusIndex;                        /* The index of the input bus on the input. Required for detaching. Will only be used within the spinlock so does not need to be atomic. */\n    MA_ATOMIC(4, ma_uint32) flags;                          /* Some state flags for tracking the read state of the output buffer. A combination of MA_NODE_OUTPUT_BUS_FLAG_*. */\n    MA_ATOMIC(4, ma_uint32) refCount;                       /* Reference count for some thread-safety when detaching. */\n    MA_ATOMIC(4, ma_bool32) isAttached;                     /* This is used to prevent iteration of nodes that are in the middle of being detached. Used for thread safety. */\n    MA_ATOMIC(4, ma_spinlock) lock;                         /* Unfortunate lock, but significantly simplifies the implementation. Required for thread-safe attaching and detaching. */\n    MA_ATOMIC(4, float) volume;                             /* Linear. */\n    MA_ATOMIC(MA_SIZEOF_PTR, ma_node_output_bus*) pNext;    /* If null, it's the tail node or detached. */\n    MA_ATOMIC(MA_SIZEOF_PTR, ma_node_output_bus*) pPrev;    /* If null, it's the head node or detached. */\n    MA_ATOMIC(MA_SIZEOF_PTR, ma_node*) pInputNode;          /* The node that this output bus is attached to. Required for detaching. */\n};\n\n/*\nA node has multiple input buses. The output buses of a node are connecting to the input busses of\nanother. An input bus is essentially just a linked list of output buses.\n*/\ntypedef struct ma_node_input_bus ma_node_input_bus;\nstruct ma_node_input_bus\n{\n    /* Mutable via multiple threads. */\n    ma_node_output_bus head;                /* Dummy head node for simplifying some lock-free thread-safety stuff. */\n    MA_ATOMIC(4, ma_uint32) nextCounter;    /* This is used to determine whether or not the input bus is finding the next node in the list. Used for thread safety when detaching output buses. */\n    MA_ATOMIC(4, ma_spinlock) lock;         /* Unfortunate lock, but significantly simplifies the implementation. Required for thread-safe attaching and detaching. */\n\n    /* Set once at startup. */\n    ma_uint8 channels;                      /* The number of channels in the audio stream for this bus. */\n};\n\n\ntypedef struct ma_node_base ma_node_base;\nstruct ma_node_base\n{\n    /* These variables are set once at startup. */\n    ma_node_graph* pNodeGraph;                  /* The graph this node belongs to. */\n    const ma_node_vtable* vtable;\n    ma_uint32 inputBusCount;\n    ma_uint32 outputBusCount;\n    ma_node_input_bus* pInputBuses;\n    ma_node_output_bus* pOutputBuses;\n    float* pCachedData;                         /* Allocated on the heap. Fixed size. Needs to be stored on the heap because reading from output buses is done in separate function calls. */\n    ma_uint16 cachedDataCapInFramesPerBus;      /* The capacity of the input data cache in frames, per bus. */\n\n    /* These variables are read and written only from the audio thread. */\n    ma_uint16 cachedFrameCountOut;\n    ma_uint16 cachedFrameCountIn;\n    ma_uint16 consumedFrameCountIn;\n\n    /* These variables are read and written between different threads. */\n    MA_ATOMIC(4, ma_node_state) state;          /* When set to stopped, nothing will be read, regardless of the times in stateTimes. */\n    MA_ATOMIC(8, ma_uint64) stateTimes[2];      /* Indexed by ma_node_state. Specifies the time based on the global clock that a node should be considered to be in the relevant state. */\n    MA_ATOMIC(8, ma_uint64) localTime;          /* The node's local clock. This is just a running sum of the number of output frames that have been processed. Can be modified by any thread with `ma_node_set_time()`. */\n\n    /* Memory management. */\n    ma_node_input_bus _inputBuses[MA_MAX_NODE_LOCAL_BUS_COUNT];\n    ma_node_output_bus _outputBuses[MA_MAX_NODE_LOCAL_BUS_COUNT];\n    void* _pHeap;   /* A heap allocation for internal use only. pInputBuses and/or pOutputBuses will point to this if the bus count exceeds MA_MAX_NODE_LOCAL_BUS_COUNT. */\n    ma_bool32 _ownsHeap;    /* If set to true, the node owns the heap allocation and _pHeap will be freed in ma_node_uninit(). */\n};\n\nMA_API ma_result ma_node_get_heap_size(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_node_init_preallocated(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, void* pHeap, ma_node* pNode);\nMA_API ma_result ma_node_init(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_node* pNode);\nMA_API void ma_node_uninit(ma_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_node_graph* ma_node_get_node_graph(const ma_node* pNode);\nMA_API ma_uint32 ma_node_get_input_bus_count(const ma_node* pNode);\nMA_API ma_uint32 ma_node_get_output_bus_count(const ma_node* pNode);\nMA_API ma_uint32 ma_node_get_input_channels(const ma_node* pNode, ma_uint32 inputBusIndex);\nMA_API ma_uint32 ma_node_get_output_channels(const ma_node* pNode, ma_uint32 outputBusIndex);\nMA_API ma_result ma_node_attach_output_bus(ma_node* pNode, ma_uint32 outputBusIndex, ma_node* pOtherNode, ma_uint32 otherNodeInputBusIndex);\nMA_API ma_result ma_node_detach_output_bus(ma_node* pNode, ma_uint32 outputBusIndex);\nMA_API ma_result ma_node_detach_all_output_buses(ma_node* pNode);\nMA_API ma_result ma_node_set_output_bus_volume(ma_node* pNode, ma_uint32 outputBusIndex, float volume);\nMA_API float ma_node_get_output_bus_volume(const ma_node* pNode, ma_uint32 outputBusIndex);\nMA_API ma_result ma_node_set_state(ma_node* pNode, ma_node_state state);\nMA_API ma_node_state ma_node_get_state(const ma_node* pNode);\nMA_API ma_result ma_node_set_state_time(ma_node* pNode, ma_node_state state, ma_uint64 globalTime);\nMA_API ma_uint64 ma_node_get_state_time(const ma_node* pNode, ma_node_state state);\nMA_API ma_node_state ma_node_get_state_by_time(const ma_node* pNode, ma_uint64 globalTime);\nMA_API ma_node_state ma_node_get_state_by_time_range(const ma_node* pNode, ma_uint64 globalTimeBeg, ma_uint64 globalTimeEnd);\nMA_API ma_uint64 ma_node_get_time(const ma_node* pNode);\nMA_API ma_result ma_node_set_time(ma_node* pNode, ma_uint64 localTime);\n\n\ntypedef struct\n{\n    ma_uint32 channels;\n    ma_uint32 processingSizeInFrames;   /* This is the preferred processing size for node processing callbacks unless overridden by a node itself. Can be 0 in which case it will be based on the frame count passed into ma_node_graph_read_pcm_frames(), but will not be well defined. */\n    size_t preMixStackSizeInBytes;      /* Defaults to 512KB per channel. Reducing this will save memory, but the depth of your node graph will be more restricted. */\n} ma_node_graph_config;\n\nMA_API ma_node_graph_config ma_node_graph_config_init(ma_uint32 channels);\n\n\nstruct ma_node_graph\n{\n    /* Immutable. */\n    ma_node_base base;                  /* The node graph itself is a node so it can be connected as an input to different node graph. This has zero inputs and calls ma_node_graph_read_pcm_frames() to generate it's output. */\n    ma_node_base endpoint;              /* Special node that all nodes eventually connect to. Data is read from this node in ma_node_graph_read_pcm_frames(). */\n    float* pProcessingCache;            /* This will be allocated when processingSizeInFrames is non-zero. This is needed because ma_node_graph_read_pcm_frames() can be called with a variable number of frames, and we may need to do some buffering in situations where the caller requests a frame count that's not a multiple of processingSizeInFrames. */\n    ma_uint32 processingCacheFramesRemaining;\n    ma_uint32 processingSizeInFrames;\n\n    /* Read and written by multiple threads. */\n    MA_ATOMIC(4, ma_bool32) isReading;\n\n    /* Modified only by the audio thread. */\n    ma_stack* pPreMixStack;\n};\n\nMA_API ma_result ma_node_graph_init(const ma_node_graph_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_node_graph* pNodeGraph);\nMA_API void ma_node_graph_uninit(ma_node_graph* pNodeGraph, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_node* ma_node_graph_get_endpoint(ma_node_graph* pNodeGraph);\nMA_API ma_result ma_node_graph_read_pcm_frames(ma_node_graph* pNodeGraph, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_uint32 ma_node_graph_get_channels(const ma_node_graph* pNodeGraph);\nMA_API ma_uint64 ma_node_graph_get_time(const ma_node_graph* pNodeGraph);\nMA_API ma_result ma_node_graph_set_time(ma_node_graph* pNodeGraph, ma_uint64 globalTime);\nMA_API ma_uint32 ma_node_graph_get_processing_size_in_frames(const ma_node_graph* pNodeGraph);\n\n\n\n/* Data source node. 0 input buses, 1 output bus. Used for reading from a data source. */\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_data_source* pDataSource;\n} ma_data_source_node_config;\n\nMA_API ma_data_source_node_config ma_data_source_node_config_init(ma_data_source* pDataSource);\n\n\ntypedef struct\n{\n    ma_node_base base;\n    ma_data_source* pDataSource;\n} ma_data_source_node;\n\nMA_API ma_result ma_data_source_node_init(ma_node_graph* pNodeGraph, const ma_data_source_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source_node* pDataSourceNode);\nMA_API void ma_data_source_node_uninit(ma_data_source_node* pDataSourceNode, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_data_source_node_set_looping(ma_data_source_node* pDataSourceNode, ma_bool32 isLooping);\nMA_API ma_bool32 ma_data_source_node_is_looping(ma_data_source_node* pDataSourceNode);\n\n\n/* Splitter Node. 1 input, many outputs. Used for splitting/copying a stream so it can be as input into two separate output nodes. */\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_uint32 channels;\n    ma_uint32 outputBusCount;\n} ma_splitter_node_config;\n\nMA_API ma_splitter_node_config ma_splitter_node_config_init(ma_uint32 channels);\n\n\ntypedef struct\n{\n    ma_node_base base;\n} ma_splitter_node;\n\nMA_API ma_result ma_splitter_node_init(ma_node_graph* pNodeGraph, const ma_splitter_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_splitter_node* pSplitterNode);\nMA_API void ma_splitter_node_uninit(ma_splitter_node* pSplitterNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nBiquad Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_biquad_config biquad;\n} ma_biquad_node_config;\n\nMA_API ma_biquad_node_config ma_biquad_node_config_init(ma_uint32 channels, float b0, float b1, float b2, float a0, float a1, float a2);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_biquad biquad;\n} ma_biquad_node;\n\nMA_API ma_result ma_biquad_node_init(ma_node_graph* pNodeGraph, const ma_biquad_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_biquad_node* pNode);\nMA_API ma_result ma_biquad_node_reinit(const ma_biquad_config* pConfig, ma_biquad_node* pNode);\nMA_API void ma_biquad_node_uninit(ma_biquad_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nLow Pass Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_lpf_config lpf;\n} ma_lpf_node_config;\n\nMA_API ma_lpf_node_config ma_lpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_lpf lpf;\n} ma_lpf_node;\n\nMA_API ma_result ma_lpf_node_init(ma_node_graph* pNodeGraph, const ma_lpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf_node* pNode);\nMA_API ma_result ma_lpf_node_reinit(const ma_lpf_config* pConfig, ma_lpf_node* pNode);\nMA_API void ma_lpf_node_uninit(ma_lpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nHigh Pass Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_hpf_config hpf;\n} ma_hpf_node_config;\n\nMA_API ma_hpf_node_config ma_hpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_hpf hpf;\n} ma_hpf_node;\n\nMA_API ma_result ma_hpf_node_init(ma_node_graph* pNodeGraph, const ma_hpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf_node* pNode);\nMA_API ma_result ma_hpf_node_reinit(const ma_hpf_config* pConfig, ma_hpf_node* pNode);\nMA_API void ma_hpf_node_uninit(ma_hpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nBand Pass Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_bpf_config bpf;\n} ma_bpf_node_config;\n\nMA_API ma_bpf_node_config ma_bpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_bpf bpf;\n} ma_bpf_node;\n\nMA_API ma_result ma_bpf_node_init(ma_node_graph* pNodeGraph, const ma_bpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf_node* pNode);\nMA_API ma_result ma_bpf_node_reinit(const ma_bpf_config* pConfig, ma_bpf_node* pNode);\nMA_API void ma_bpf_node_uninit(ma_bpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nNotching Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_notch_config notch;\n} ma_notch_node_config;\n\nMA_API ma_notch_node_config ma_notch_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double q, double frequency);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_notch2 notch;\n} ma_notch_node;\n\nMA_API ma_result ma_notch_node_init(ma_node_graph* pNodeGraph, const ma_notch_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_notch_node* pNode);\nMA_API ma_result ma_notch_node_reinit(const ma_notch_config* pConfig, ma_notch_node* pNode);\nMA_API void ma_notch_node_uninit(ma_notch_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nPeaking Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_peak_config peak;\n} ma_peak_node_config;\n\nMA_API ma_peak_node_config ma_peak_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_peak2 peak;\n} ma_peak_node;\n\nMA_API ma_result ma_peak_node_init(ma_node_graph* pNodeGraph, const ma_peak_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_peak_node* pNode);\nMA_API ma_result ma_peak_node_reinit(const ma_peak_config* pConfig, ma_peak_node* pNode);\nMA_API void ma_peak_node_uninit(ma_peak_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nLow Shelf Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_loshelf_config loshelf;\n} ma_loshelf_node_config;\n\nMA_API ma_loshelf_node_config ma_loshelf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_loshelf2 loshelf;\n} ma_loshelf_node;\n\nMA_API ma_result ma_loshelf_node_init(ma_node_graph* pNodeGraph, const ma_loshelf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_loshelf_node* pNode);\nMA_API ma_result ma_loshelf_node_reinit(const ma_loshelf_config* pConfig, ma_loshelf_node* pNode);\nMA_API void ma_loshelf_node_uninit(ma_loshelf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n/*\nHigh Shelf Filter Node\n*/\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_hishelf_config hishelf;\n} ma_hishelf_node_config;\n\nMA_API ma_hishelf_node_config ma_hishelf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_hishelf2 hishelf;\n} ma_hishelf_node;\n\nMA_API ma_result ma_hishelf_node_init(ma_node_graph* pNodeGraph, const ma_hishelf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hishelf_node* pNode);\nMA_API ma_result ma_hishelf_node_reinit(const ma_hishelf_config* pConfig, ma_hishelf_node* pNode);\nMA_API void ma_hishelf_node_uninit(ma_hishelf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\ntypedef struct\n{\n    ma_node_config nodeConfig;\n    ma_delay_config delay;\n} ma_delay_node_config;\n\nMA_API ma_delay_node_config ma_delay_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 delayInFrames, float decay);\n\n\ntypedef struct\n{\n    ma_node_base baseNode;\n    ma_delay delay;\n} ma_delay_node;\n\nMA_API ma_result ma_delay_node_init(ma_node_graph* pNodeGraph, const ma_delay_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_delay_node* pDelayNode);\nMA_API void ma_delay_node_uninit(ma_delay_node* pDelayNode, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API void ma_delay_node_set_wet(ma_delay_node* pDelayNode, float value);\nMA_API float ma_delay_node_get_wet(const ma_delay_node* pDelayNode);\nMA_API void ma_delay_node_set_dry(ma_delay_node* pDelayNode, float value);\nMA_API float ma_delay_node_get_dry(const ma_delay_node* pDelayNode);\nMA_API void ma_delay_node_set_decay(ma_delay_node* pDelayNode, float value);\nMA_API float ma_delay_node_get_decay(const ma_delay_node* pDelayNode);\n#endif  /* MA_NO_NODE_GRAPH */\n\n\n/* SECTION: miniaudio_engine.h */\n/************************************************************************************************************************************************************\n\nEngine\n\n************************************************************************************************************************************************************/\n#if !defined(MA_NO_ENGINE) && !defined(MA_NO_NODE_GRAPH)\ntypedef struct ma_engine ma_engine;\ntypedef struct ma_sound  ma_sound;\n\n\n/* Sound flags. */\ntypedef enum\n{\n    /* Resource manager flags. */\n    MA_SOUND_FLAG_STREAM                = 0x00000001,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM */\n    MA_SOUND_FLAG_DECODE                = 0x00000002,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE */\n    MA_SOUND_FLAG_ASYNC                 = 0x00000004,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC */\n    MA_SOUND_FLAG_WAIT_INIT             = 0x00000008,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT */\n    MA_SOUND_FLAG_UNKNOWN_LENGTH        = 0x00000010,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_UNKNOWN_LENGTH */\n    MA_SOUND_FLAG_LOOPING               = 0x00000020,   /* MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING */\n\n    /* ma_sound specific flags. */\n    MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT = 0x00001000,   /* Do not attach to the endpoint by default. Useful for when setting up nodes in a complex graph system. */\n    MA_SOUND_FLAG_NO_PITCH              = 0x00002000,   /* Disable pitch shifting with ma_sound_set_pitch() and ma_sound_group_set_pitch(). This is an optimization. */\n    MA_SOUND_FLAG_NO_SPATIALIZATION     = 0x00004000    /* Disable spatialization. */\n} ma_sound_flags;\n\n#ifndef MA_ENGINE_MAX_LISTENERS\n#define MA_ENGINE_MAX_LISTENERS             4\n#endif\n\n#define MA_LISTENER_INDEX_CLOSEST           ((ma_uint8)-1)\n\ntypedef enum\n{\n    ma_engine_node_type_sound,\n    ma_engine_node_type_group\n} ma_engine_node_type;\n\ntypedef struct\n{\n    ma_engine* pEngine;\n    ma_engine_node_type type;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_uint32 sampleRate;               /* Only used when the type is set to ma_engine_node_type_sound. */\n    ma_uint32 volumeSmoothTimeInPCMFrames;  /* The number of frames to smooth over volume changes. Defaults to 0 in which case no smoothing is used. */\n    ma_mono_expansion_mode monoExpansionMode;\n    ma_bool8 isPitchDisabled;           /* Pitching can be explicitly disabled with MA_SOUND_FLAG_NO_PITCH to optimize processing. */\n    ma_bool8 isSpatializationDisabled;  /* Spatialization can be explicitly disabled with MA_SOUND_FLAG_NO_SPATIALIZATION. */\n    ma_uint8 pinnedListenerIndex;       /* The index of the listener this node should always use for spatialization. If set to MA_LISTENER_INDEX_CLOSEST the engine will use the closest listener. */\n    ma_resampler_config resampling;\n} ma_engine_node_config;\n\nMA_API ma_engine_node_config ma_engine_node_config_init(ma_engine* pEngine, ma_engine_node_type type, ma_uint32 flags);\n\n\n/* Base node object for both ma_sound and ma_sound_group. */\ntypedef struct\n{\n    ma_node_base baseNode;                              /* Must be the first member for compatibility with the ma_node API. */\n    ma_engine* pEngine;                                 /* A pointer to the engine. Set based on the value from the config. */\n    ma_uint32 sampleRate;                               /* The sample rate of the input data. For sounds backed by a data source, this will be the data source's sample rate. Otherwise it'll be the engine's sample rate. */\n    ma_uint32 volumeSmoothTimeInPCMFrames;\n    ma_mono_expansion_mode monoExpansionMode;\n    ma_fader fader;\n    ma_resampler resampler;                             /* For pitch shift. */\n    ma_spatializer spatializer;\n    ma_panner panner;\n    ma_gainer volumeGainer;                             /* This will only be used if volumeSmoothTimeInPCMFrames is > 0. */\n    ma_atomic_float volume;                             /* Defaults to 1. */\n    MA_ATOMIC(4, float) pitch;\n    float oldPitch;                                     /* For determining whether or not the resampler needs to be updated to reflect the new pitch. The resampler will be updated on the mixing thread. */\n    float oldDopplerPitch;                              /* For determining whether or not the resampler needs to be updated to take a new doppler pitch into account. */\n    MA_ATOMIC(4, ma_bool32) isPitchDisabled;            /* When set to true, pitching will be disabled which will allow the resampler to be bypassed to save some computation. */\n    MA_ATOMIC(4, ma_bool32) isSpatializationDisabled;   /* Set to false by default. When set to false, will not have spatialisation applied. */\n    MA_ATOMIC(4, ma_uint32) pinnedListenerIndex;        /* The index of the listener this node should always use for spatialization. If set to MA_LISTENER_INDEX_CLOSEST the engine will use the closest listener. */\n\n    /* When setting a fade, it's not done immediately in ma_sound_set_fade(). It's deferred to the audio thread which means we need to store the settings here. */\n    struct\n    {\n        ma_atomic_float volumeBeg;\n        ma_atomic_float volumeEnd;\n        ma_atomic_uint64 fadeLengthInFrames;            /* <-- Defaults to (~(ma_uint64)0) which is used to indicate that no fade should be applied. */\n        ma_atomic_uint64 absoluteGlobalTimeInFrames;    /* <-- The time to start the fade. */\n    } fadeSettings;\n\n    /* Memory management. */\n    ma_bool8 _ownsHeap;\n    void* _pHeap;\n} ma_engine_node;\n\nMA_API ma_result ma_engine_node_get_heap_size(const ma_engine_node_config* pConfig, size_t* pHeapSizeInBytes);\nMA_API ma_result ma_engine_node_init_preallocated(const ma_engine_node_config* pConfig, void* pHeap, ma_engine_node* pEngineNode);\nMA_API ma_result ma_engine_node_init(const ma_engine_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_engine_node* pEngineNode);\nMA_API void ma_engine_node_uninit(ma_engine_node* pEngineNode, const ma_allocation_callbacks* pAllocationCallbacks);\n\n\n#define MA_SOUND_SOURCE_CHANNEL_COUNT   0xFFFFFFFF\n\n/* Callback for when a sound reaches the end. */\ntypedef void (* ma_sound_end_proc)(void* pUserData, ma_sound* pSound);\n\ntypedef struct\n{\n    const char* pFilePath;                      /* Set this to load from the resource manager. */\n    const wchar_t* pFilePathW;                  /* Set this to load from the resource manager. */\n    ma_data_source* pDataSource;                /* Set this to load from an existing data source. */\n    ma_node* pInitialAttachment;                /* If set, the sound will be attached to an input of this node. This can be set to a ma_sound. If set to NULL, the sound will be attached directly to the endpoint unless MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT is set in `flags`. */\n    ma_uint32 initialAttachmentInputBusIndex;   /* The index of the input bus of pInitialAttachment to attach the sound to. */\n    ma_uint32 channelsIn;                       /* Ignored if using a data source as input (the data source's channel count will be used always). Otherwise, setting to 0 will cause the engine's channel count to be used. */\n    ma_uint32 channelsOut;                      /* Set this to 0 (default) to use the engine's channel count. Set to MA_SOUND_SOURCE_CHANNEL_COUNT to use the data source's channel count (only used if using a data source as input). */\n    ma_mono_expansion_mode monoExpansionMode;   /* Controls how the mono channel should be expanded to other channels when spatialization is disabled on a sound. */\n    ma_uint32 flags;                            /* A combination of MA_SOUND_FLAG_* flags. */\n    ma_uint32 volumeSmoothTimeInPCMFrames;      /* The number of frames to smooth over volume changes. Defaults to 0 in which case no smoothing is used. */\n    ma_uint64 initialSeekPointInPCMFrames;      /* Initializes the sound such that it's seeked to this location by default. */\n    ma_uint64 rangeBegInPCMFrames;\n    ma_uint64 rangeEndInPCMFrames;\n    ma_uint64 loopPointBegInPCMFrames;\n    ma_uint64 loopPointEndInPCMFrames;\n    ma_sound_end_proc endCallback;              /* Fired when the sound reaches the end. Will be fired from the audio thread. Do not restart, uninitialize or otherwise change the state of the sound from here. Instead fire an event or set a variable to indicate to a different thread to change the start of the sound. Will not be fired in response to a scheduled stop with ma_sound_set_stop_time_*(). */\n    void* pEndCallbackUserData;\n    ma_resampler_config pitchResampling;\n#ifndef MA_NO_RESOURCE_MANAGER\n    ma_resource_manager_pipeline_notifications initNotifications;\n#endif\n    ma_fence* pDoneFence;                       /* Deprecated. Use initNotifications instead. Released when the resource manager has finished decoding the entire sound. Not used with streams. */\n    ma_bool32 isLooping;                        /* Deprecated. Use the MA_SOUND_FLAG_LOOPING flag in `flags` instead. */\n} ma_sound_config;\n\nMA_API ma_sound_config ma_sound_config_init(void);                  /* Deprecated. Will be removed in version 0.12. Use ma_sound_config_2() instead. */\nMA_API ma_sound_config ma_sound_config_init_2(ma_engine* pEngine);  /* Will be renamed to ma_sound_config_init() in version 0.12. */\n\nstruct ma_sound\n{\n    ma_engine_node engineNode;          /* Must be the first member for compatibility with the ma_node API. */\n    ma_data_source* pDataSource;\n    MA_ATOMIC(8, ma_uint64) seekTarget; /* The PCM frame index to seek to in the mixing thread. Set to (~(ma_uint64)0) to not perform any seeking. */\n    MA_ATOMIC(4, ma_bool32) atEnd;\n    ma_sound_end_proc endCallback;\n    void* pEndCallbackUserData;\n    float* pProcessingCache;            /* Will be null if pDataSource is null. */\n    ma_uint32 processingCacheFramesRemaining;\n    ma_uint32 processingCacheCap;\n    ma_bool8 ownsDataSource;    \n\n    /*\n    We're declaring a resource manager data source object here to save us a malloc when loading a\n    sound via the resource manager, which I *think* will be the most common scenario.\n    */\n#ifndef MA_NO_RESOURCE_MANAGER\n    ma_resource_manager_data_source* pResourceManagerDataSource;\n#endif\n};\n\n/* Structure specifically for sounds played with ma_engine_play_sound(). Making this a separate structure to reduce overhead. */\ntypedef struct ma_sound_inlined ma_sound_inlined;\nstruct ma_sound_inlined\n{\n    ma_sound sound;\n    ma_sound_inlined* pNext;\n    ma_sound_inlined* pPrev;\n};\n\n/* A sound group is just a sound. */\ntypedef ma_sound_config ma_sound_group_config;\ntypedef ma_sound        ma_sound_group;\n\nMA_API ma_sound_group_config ma_sound_group_config_init(void);                  /* Deprecated. Will be removed in version 0.12. Use ma_sound_config_2() instead. */\nMA_API ma_sound_group_config ma_sound_group_config_init_2(ma_engine* pEngine);  /* Will be renamed to ma_sound_config_init() in version 0.12. */\n\ntypedef void (* ma_engine_process_proc)(void* pUserData, float* pFramesOut, ma_uint64 frameCount);\n\ntypedef struct\n{\n#if !defined(MA_NO_RESOURCE_MANAGER)\n    ma_resource_manager* pResourceManager;          /* Can be null in which case a resource manager will be created for you. */\n#endif\n#if !defined(MA_NO_DEVICE_IO)\n    ma_context* pContext;\n    ma_device* pDevice;                             /* If set, the caller is responsible for calling ma_engine_data_callback() in the device's data callback. */\n    ma_device_id* pPlaybackDeviceID;                /* The ID of the playback device to use with the default listener. */\n    ma_device_data_proc dataCallback;               /* Can be null. Can be used to provide a custom device data callback. */\n    ma_device_notification_proc notificationCallback;\n#endif\n    ma_log* pLog;                                   /* When set to NULL, will use the context's log. */\n    ma_uint32 listenerCount;                        /* Must be between 1 and MA_ENGINE_MAX_LISTENERS. */\n    ma_uint32 channels;                             /* The number of channels to use when mixing and spatializing. When set to 0, will use the native channel count of the device. */\n    ma_uint32 sampleRate;                           /* The sample rate. When set to 0 will use the native sample rate of the device. */\n    ma_uint32 periodSizeInFrames;                   /* If set to something other than 0, updates will always be exactly this size. The underlying device may be a different size, but from the perspective of the mixer that won't matter.*/\n    ma_uint32 periodSizeInMilliseconds;             /* Used if periodSizeInFrames is unset. */\n    ma_uint32 gainSmoothTimeInFrames;               /* The number of frames to interpolate the gain of spatialized sounds across. If set to 0, will use gainSmoothTimeInMilliseconds. */\n    ma_uint32 gainSmoothTimeInMilliseconds;         /* When set to 0, gainSmoothTimeInFrames will be used. If both are set to 0, a default value will be used. */\n    ma_uint32 defaultVolumeSmoothTimeInPCMFrames;   /* Defaults to 0. Controls the default amount of smoothing to apply to volume changes to sounds. High values means more smoothing at the expense of high latency (will take longer to reach the new volume). */\n    ma_uint32 preMixStackSizeInBytes;               /* A stack is used for internal processing in the node graph. This allows you to configure the size of this stack. Smaller values will reduce the maximum depth of your node graph. You should rarely need to modify this. */\n    ma_allocation_callbacks allocationCallbacks;\n    ma_bool32 noAutoStart;                          /* When set to true, requires an explicit call to ma_engine_start(). This is false by default, meaning the engine will be started automatically in ma_engine_init(). */\n    ma_bool32 noDevice;                             /* When set to true, don't create a default device. ma_engine_read_pcm_frames() can be called manually to read data. */\n    ma_mono_expansion_mode monoExpansionMode;       /* Controls how the mono channel should be expanded to other channels when spatialization is disabled on a sound. */\n    ma_vfs* pResourceManagerVFS;                    /* A pointer to a pre-allocated VFS object to use with the resource manager. This is ignored if pResourceManager is not NULL. */\n    ma_engine_process_proc onProcess;               /* Fired at the end of each call to ma_engine_read_pcm_frames(). For engine's that manage their own internal device (the default configuration), this will be fired from the audio thread, and you do not need to call ma_engine_read_pcm_frames() manually in order to trigger this. */\n    void* pProcessUserData;                         /* User data that's passed into onProcess. */\n    ma_resampler_config resourceManagerResampling;  /* The resampling config to use with the resource manager. */\n    ma_resampler_config pitchResampling;            /* The resampling config for the pitch and Doppler effects. You will typically want this to be a fast resampler. For high quality stuff, it's recommended that you pre-resample. */\n} ma_engine_config;\n\nMA_API ma_engine_config ma_engine_config_init(void);\n\n\nstruct ma_engine\n{\n    ma_node_graph nodeGraph;                        /* An engine is a node graph. It should be able to be plugged into any ma_node_graph API (with a cast) which means this must be the first member of this struct. */\n#if !defined(MA_NO_RESOURCE_MANAGER)\n    ma_resource_manager* pResourceManager;\n#endif\n#if !defined(MA_NO_DEVICE_IO)\n    ma_device* pDevice;                             /* Optionally set via the config, otherwise allocated by the engine in ma_engine_init(). */\n#endif\n    ma_log* pLog;\n    ma_uint32 sampleRate;\n    ma_uint32 listenerCount;\n    ma_spatializer_listener listeners[MA_ENGINE_MAX_LISTENERS];\n    ma_allocation_callbacks allocationCallbacks;\n    ma_bool8 ownsResourceManager;\n    ma_bool8 ownsDevice;\n    ma_spinlock inlinedSoundLock;                   /* For synchronizing access to the inlined sound list. */\n    ma_sound_inlined* pInlinedSoundHead;            /* The first inlined sound. Inlined sounds are tracked in a linked list. */\n    MA_ATOMIC(4, ma_uint32) inlinedSoundCount;      /* The total number of allocated inlined sound objects. Used for debugging. */\n    ma_uint32 gainSmoothTimeInFrames;               /* The number of frames to interpolate the gain of spatialized sounds across. */\n    ma_uint32 defaultVolumeSmoothTimeInPCMFrames;\n    ma_mono_expansion_mode monoExpansionMode;\n    ma_engine_process_proc onProcess;\n    void* pProcessUserData;\n    ma_resampler_config pitchResamplingConfig;\n};\n\nMA_API ma_result ma_engine_init(const ma_engine_config* pConfig, ma_engine* pEngine);\nMA_API void ma_engine_uninit(ma_engine* pEngine);\nMA_API ma_result ma_engine_read_pcm_frames(ma_engine* pEngine, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_node_graph* ma_engine_get_node_graph(ma_engine* pEngine);\n#if !defined(MA_NO_RESOURCE_MANAGER)\nMA_API ma_resource_manager* ma_engine_get_resource_manager(ma_engine* pEngine);\n#endif\nMA_API ma_device* ma_engine_get_device(ma_engine* pEngine);\nMA_API ma_log* ma_engine_get_log(ma_engine* pEngine);\nMA_API ma_node* ma_engine_get_endpoint(ma_engine* pEngine);\nMA_API ma_uint64 ma_engine_get_time_in_pcm_frames(const ma_engine* pEngine);\nMA_API ma_uint64 ma_engine_get_time_in_milliseconds(const ma_engine* pEngine);\nMA_API ma_result ma_engine_set_time_in_pcm_frames(ma_engine* pEngine, ma_uint64 globalTime);\nMA_API ma_result ma_engine_set_time_in_milliseconds(ma_engine* pEngine, ma_uint64 globalTime);\nMA_API ma_uint64 ma_engine_get_time(const ma_engine* pEngine);                  /* Deprecated. Use ma_engine_get_time_in_pcm_frames(). Will be removed in version 0.12. */\nMA_API ma_result ma_engine_set_time(ma_engine* pEngine, ma_uint64 globalTime);  /* Deprecated. Use ma_engine_set_time_in_pcm_frames(). Will be removed in version 0.12. */\nMA_API ma_uint32 ma_engine_get_channels(const ma_engine* pEngine);\nMA_API ma_uint32 ma_engine_get_sample_rate(const ma_engine* pEngine);\n\nMA_API ma_result ma_engine_start(ma_engine* pEngine);\nMA_API ma_result ma_engine_stop(ma_engine* pEngine);\nMA_API ma_result ma_engine_set_volume(ma_engine* pEngine, float volume);\nMA_API float ma_engine_get_volume(ma_engine* pEngine);\nMA_API ma_result ma_engine_set_gain_db(ma_engine* pEngine, float gainDB);\nMA_API float ma_engine_get_gain_db(ma_engine* pEngine);\n\nMA_API ma_uint32 ma_engine_get_listener_count(const ma_engine* pEngine);\nMA_API ma_uint32 ma_engine_find_closest_listener(const ma_engine* pEngine, float absolutePosX, float absolutePosY, float absolutePosZ);\nMA_API void ma_engine_listener_set_position(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z);\nMA_API ma_vec3f ma_engine_listener_get_position(const ma_engine* pEngine, ma_uint32 listenerIndex);\nMA_API void ma_engine_listener_set_direction(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z);\nMA_API ma_vec3f ma_engine_listener_get_direction(const ma_engine* pEngine, ma_uint32 listenerIndex);\nMA_API void ma_engine_listener_set_velocity(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z);\nMA_API ma_vec3f ma_engine_listener_get_velocity(const ma_engine* pEngine, ma_uint32 listenerIndex);\nMA_API void ma_engine_listener_set_cone(ma_engine* pEngine, ma_uint32 listenerIndex, float innerAngleInRadians, float outerAngleInRadians, float outerGain);\nMA_API void ma_engine_listener_get_cone(const ma_engine* pEngine, ma_uint32 listenerIndex, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain);\nMA_API void ma_engine_listener_set_world_up(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z);\nMA_API ma_vec3f ma_engine_listener_get_world_up(const ma_engine* pEngine, ma_uint32 listenerIndex);\nMA_API void ma_engine_listener_set_enabled(ma_engine* pEngine, ma_uint32 listenerIndex, ma_bool32 isEnabled);\nMA_API ma_bool32 ma_engine_listener_is_enabled(const ma_engine* pEngine, ma_uint32 listenerIndex);\n\n#ifndef MA_NO_RESOURCE_MANAGER\nMA_API ma_result ma_engine_play_sound_ex(ma_engine* pEngine, const char* pFilePath, ma_node* pNode, ma_uint32 nodeInputBusIndex);\nMA_API ma_result ma_engine_play_sound(ma_engine* pEngine, const char* pFilePath, ma_sound_group* pGroup);   /* Fire and forget. */\n#endif\n\n#ifndef MA_NO_RESOURCE_MANAGER\nMA_API ma_result ma_sound_init_from_file(ma_engine* pEngine, const char* pFilePath, ma_uint32 flags, ma_sound_group* pGroup, ma_fence* pDoneFence, ma_sound* pSound);\nMA_API ma_result ma_sound_init_from_file_w(ma_engine* pEngine, const wchar_t* pFilePath, ma_uint32 flags, ma_sound_group* pGroup, ma_fence* pDoneFence, ma_sound* pSound);\nMA_API ma_result ma_sound_init_copy(ma_engine* pEngine, const ma_sound* pExistingSound, ma_uint32 flags, ma_sound_group* pGroup, ma_sound* pSound);\n#endif\nMA_API ma_result ma_sound_init_from_data_source(ma_engine* pEngine, ma_data_source* pDataSource, ma_uint32 flags, ma_sound_group* pGroup, ma_sound* pSound);\nMA_API ma_result ma_sound_init_ex(ma_engine* pEngine, const ma_sound_config* pConfig, ma_sound* pSound);\nMA_API void ma_sound_uninit(ma_sound* pSound);\nMA_API ma_engine* ma_sound_get_engine(const ma_sound* pSound);\nMA_API ma_data_source* ma_sound_get_data_source(const ma_sound* pSound);\nMA_API ma_result ma_sound_start(ma_sound* pSound);\nMA_API ma_result ma_sound_stop(ma_sound* pSound);\nMA_API ma_result ma_sound_stop_with_fade_in_pcm_frames(ma_sound* pSound, ma_uint64 fadeLengthInFrames);     /* Will overwrite any scheduled stop and fade. If you want to restart the sound, first reset it with `ma_sound_reset_stop_time_and_fade()`. There are plans to make this less awkward in the future. */\nMA_API ma_result ma_sound_stop_with_fade_in_milliseconds(ma_sound* pSound, ma_uint64 fadeLengthInFrames);   /* Will overwrite any scheduled stop and fade. If you want to restart the sound, first reset it with `ma_sound_reset_stop_time_and_fade()`. There are plans to make this less awkward in the future. */\nMA_API void ma_sound_reset_start_time(ma_sound* pSound);\nMA_API void ma_sound_reset_stop_time(ma_sound* pSound);\nMA_API void ma_sound_reset_fade(ma_sound* pSound);\nMA_API void ma_sound_reset_stop_time_and_fade(ma_sound* pSound);  /* Resets fades and scheduled stop time. Does not seek back to the start. */\nMA_API void ma_sound_set_volume(ma_sound* pSound, float volume);\nMA_API float ma_sound_get_volume(const ma_sound* pSound);\nMA_API void ma_sound_set_pan(ma_sound* pSound, float pan);\nMA_API float ma_sound_get_pan(const ma_sound* pSound);\nMA_API void ma_sound_set_pan_mode(ma_sound* pSound, ma_pan_mode panMode);\nMA_API ma_pan_mode ma_sound_get_pan_mode(const ma_sound* pSound);\nMA_API void ma_sound_set_pitch(ma_sound* pSound, float pitch);\nMA_API float ma_sound_get_pitch(const ma_sound* pSound);\nMA_API void ma_sound_set_spatialization_enabled(ma_sound* pSound, ma_bool32 enabled);\nMA_API ma_bool32 ma_sound_is_spatialization_enabled(const ma_sound* pSound);\nMA_API void ma_sound_set_pinned_listener_index(ma_sound* pSound, ma_uint32 listenerIndex);\nMA_API ma_uint32 ma_sound_get_pinned_listener_index(const ma_sound* pSound);\nMA_API ma_uint32 ma_sound_get_listener_index(const ma_sound* pSound);\nMA_API ma_vec3f ma_sound_get_direction_to_listener(const ma_sound* pSound);\nMA_API void ma_sound_set_position(ma_sound* pSound, float x, float y, float z);\nMA_API ma_vec3f ma_sound_get_position(const ma_sound* pSound);\nMA_API void ma_sound_set_direction(ma_sound* pSound, float x, float y, float z);\nMA_API ma_vec3f ma_sound_get_direction(const ma_sound* pSound);\nMA_API void ma_sound_set_velocity(ma_sound* pSound, float x, float y, float z);\nMA_API ma_vec3f ma_sound_get_velocity(const ma_sound* pSound);\nMA_API void ma_sound_set_attenuation_model(ma_sound* pSound, ma_attenuation_model attenuationModel);\nMA_API ma_attenuation_model ma_sound_get_attenuation_model(const ma_sound* pSound);\nMA_API void ma_sound_set_positioning(ma_sound* pSound, ma_positioning positioning);\nMA_API ma_positioning ma_sound_get_positioning(const ma_sound* pSound);\nMA_API void ma_sound_set_rolloff(ma_sound* pSound, float rolloff);\nMA_API float ma_sound_get_rolloff(const ma_sound* pSound);\nMA_API void ma_sound_set_min_gain(ma_sound* pSound, float minGain);\nMA_API float ma_sound_get_min_gain(const ma_sound* pSound);\nMA_API void ma_sound_set_max_gain(ma_sound* pSound, float maxGain);\nMA_API float ma_sound_get_max_gain(const ma_sound* pSound);\nMA_API void ma_sound_set_min_distance(ma_sound* pSound, float minDistance);\nMA_API float ma_sound_get_min_distance(const ma_sound* pSound);\nMA_API void ma_sound_set_max_distance(ma_sound* pSound, float maxDistance);\nMA_API float ma_sound_get_max_distance(const ma_sound* pSound);\nMA_API void ma_sound_set_cone(ma_sound* pSound, float innerAngleInRadians, float outerAngleInRadians, float outerGain);\nMA_API void ma_sound_get_cone(const ma_sound* pSound, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain);\nMA_API void ma_sound_set_doppler_factor(ma_sound* pSound, float dopplerFactor);\nMA_API float ma_sound_get_doppler_factor(const ma_sound* pSound);\nMA_API void ma_sound_set_directional_attenuation_factor(ma_sound* pSound, float directionalAttenuationFactor);\nMA_API float ma_sound_get_directional_attenuation_factor(const ma_sound* pSound);\nMA_API void ma_sound_set_fade_in_pcm_frames(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames);\nMA_API void ma_sound_set_fade_in_milliseconds(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds);\nMA_API void ma_sound_set_fade_start_in_pcm_frames(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames, ma_uint64 absoluteGlobalTimeInFrames);\nMA_API void ma_sound_set_fade_start_in_milliseconds(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds, ma_uint64 absoluteGlobalTimeInMilliseconds);\nMA_API float ma_sound_get_current_fade_volume(const ma_sound* pSound);\nMA_API void ma_sound_set_start_time_in_pcm_frames(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInFrames);\nMA_API void ma_sound_set_start_time_in_milliseconds(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInMilliseconds);\nMA_API void ma_sound_set_stop_time_in_pcm_frames(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInFrames);\nMA_API void ma_sound_set_stop_time_in_milliseconds(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInMilliseconds);\nMA_API void ma_sound_set_stop_time_with_fade_in_pcm_frames(ma_sound* pSound, ma_uint64 stopAbsoluteGlobalTimeInFrames, ma_uint64 fadeLengthInFrames);\nMA_API void ma_sound_set_stop_time_with_fade_in_milliseconds(ma_sound* pSound, ma_uint64 stopAbsoluteGlobalTimeInMilliseconds, ma_uint64 fadeLengthInMilliseconds);\nMA_API ma_bool32 ma_sound_is_playing(const ma_sound* pSound);\nMA_API ma_uint64 ma_sound_get_time_in_pcm_frames(const ma_sound* pSound);\nMA_API ma_uint64 ma_sound_get_time_in_milliseconds(const ma_sound* pSound);\nMA_API void ma_sound_set_looping(ma_sound* pSound, ma_bool32 isLooping);\nMA_API ma_bool32 ma_sound_is_looping(const ma_sound* pSound);\nMA_API ma_bool32 ma_sound_at_end(const ma_sound* pSound);\nMA_API ma_result ma_sound_seek_to_pcm_frame(ma_sound* pSound, ma_uint64 frameIndex); /* Just a wrapper around ma_data_source_seek_to_pcm_frame(). */\nMA_API ma_result ma_sound_seek_to_second(ma_sound* pSound, float seekPointInSeconds); /* Abstraction to ma_sound_seek_to_pcm_frame() */\nMA_API ma_result ma_sound_get_data_format(const ma_sound* pSound, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_sound_get_cursor_in_pcm_frames(const ma_sound* pSound, ma_uint64* pCursor);\nMA_API ma_result ma_sound_get_length_in_pcm_frames(const ma_sound* pSound, ma_uint64* pLength);\nMA_API ma_result ma_sound_get_cursor_in_seconds(const ma_sound* pSound, float* pCursor);\nMA_API ma_result ma_sound_get_length_in_seconds(const ma_sound* pSound, float* pLength);\nMA_API ma_result ma_sound_set_end_callback(ma_sound* pSound, ma_sound_end_proc callback, void* pUserData);\n\nMA_API ma_result ma_sound_group_init(ma_engine* pEngine, ma_uint32 flags, ma_sound_group* pParentGroup, ma_sound_group* pGroup);\nMA_API ma_result ma_sound_group_init_ex(ma_engine* pEngine, const ma_sound_group_config* pConfig, ma_sound_group* pGroup);\nMA_API void ma_sound_group_uninit(ma_sound_group* pGroup);\nMA_API ma_engine* ma_sound_group_get_engine(const ma_sound_group* pGroup);\nMA_API ma_result ma_sound_group_start(ma_sound_group* pGroup);\nMA_API ma_result ma_sound_group_stop(ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_volume(ma_sound_group* pGroup, float volume);\nMA_API float ma_sound_group_get_volume(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_pan(ma_sound_group* pGroup, float pan);\nMA_API float ma_sound_group_get_pan(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_pan_mode(ma_sound_group* pGroup, ma_pan_mode panMode);\nMA_API ma_pan_mode ma_sound_group_get_pan_mode(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_pitch(ma_sound_group* pGroup, float pitch);\nMA_API float ma_sound_group_get_pitch(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_spatialization_enabled(ma_sound_group* pGroup, ma_bool32 enabled);\nMA_API ma_bool32 ma_sound_group_is_spatialization_enabled(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_pinned_listener_index(ma_sound_group* pGroup, ma_uint32 listenerIndex);\nMA_API ma_uint32 ma_sound_group_get_pinned_listener_index(const ma_sound_group* pGroup);\nMA_API ma_uint32 ma_sound_group_get_listener_index(const ma_sound_group* pGroup);\nMA_API ma_vec3f ma_sound_group_get_direction_to_listener(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_position(ma_sound_group* pGroup, float x, float y, float z);\nMA_API ma_vec3f ma_sound_group_get_position(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_direction(ma_sound_group* pGroup, float x, float y, float z);\nMA_API ma_vec3f ma_sound_group_get_direction(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_velocity(ma_sound_group* pGroup, float x, float y, float z);\nMA_API ma_vec3f ma_sound_group_get_velocity(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_attenuation_model(ma_sound_group* pGroup, ma_attenuation_model attenuationModel);\nMA_API ma_attenuation_model ma_sound_group_get_attenuation_model(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_positioning(ma_sound_group* pGroup, ma_positioning positioning);\nMA_API ma_positioning ma_sound_group_get_positioning(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_rolloff(ma_sound_group* pGroup, float rolloff);\nMA_API float ma_sound_group_get_rolloff(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_min_gain(ma_sound_group* pGroup, float minGain);\nMA_API float ma_sound_group_get_min_gain(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_max_gain(ma_sound_group* pGroup, float maxGain);\nMA_API float ma_sound_group_get_max_gain(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_min_distance(ma_sound_group* pGroup, float minDistance);\nMA_API float ma_sound_group_get_min_distance(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_max_distance(ma_sound_group* pGroup, float maxDistance);\nMA_API float ma_sound_group_get_max_distance(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_cone(ma_sound_group* pGroup, float innerAngleInRadians, float outerAngleInRadians, float outerGain);\nMA_API void ma_sound_group_get_cone(const ma_sound_group* pGroup, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain);\nMA_API void ma_sound_group_set_doppler_factor(ma_sound_group* pGroup, float dopplerFactor);\nMA_API float ma_sound_group_get_doppler_factor(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_directional_attenuation_factor(ma_sound_group* pGroup, float directionalAttenuationFactor);\nMA_API float ma_sound_group_get_directional_attenuation_factor(const ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_fade_in_pcm_frames(ma_sound_group* pGroup, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames);\nMA_API void ma_sound_group_set_fade_in_milliseconds(ma_sound_group* pGroup, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds);\nMA_API float ma_sound_group_get_current_fade_volume(ma_sound_group* pGroup);\nMA_API void ma_sound_group_set_start_time_in_pcm_frames(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInFrames);\nMA_API void ma_sound_group_set_start_time_in_milliseconds(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInMilliseconds);\nMA_API void ma_sound_group_set_stop_time_in_pcm_frames(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInFrames);\nMA_API void ma_sound_group_set_stop_time_in_milliseconds(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInMilliseconds);\nMA_API ma_bool32 ma_sound_group_is_playing(const ma_sound_group* pGroup);\nMA_API ma_uint64 ma_sound_group_get_time_in_pcm_frames(const ma_sound_group* pGroup);\n#endif  /* MA_NO_ENGINE */\n/* END SECTION: miniaudio_engine.h */\n\n#ifdef __cplusplus\n}\n#endif\n#endif  /* miniaudio_h */\n\n\n/*\nThis is for preventing greying out of the implementation section.\n*/\n#if defined(Q_CREATOR_RUN) || defined(__INTELLISENSE__) || defined(__CDT_PARSER__)\n#define MINIAUDIO_IMPLEMENTATION\n#endif\n\n/************************************************************************************************************************************************************\n*************************************************************************************************************************************************************\n\nIMPLEMENTATION\n\n*************************************************************************************************************************************************************\n************************************************************************************************************************************************************/\n#if defined(MINIAUDIO_IMPLEMENTATION) || defined(MA_IMPLEMENTATION)\n#ifndef miniaudio_c\n#define miniaudio_c\n\n#include <assert.h>\n#include <limits.h>         /* For INT_MAX */\n#include <math.h>           /* sin(), etc. */\n#include <stdlib.h>         /* For malloc(), free(), wcstombs(). */\n#include <string.h>         /* For memset() */\n\n#include <stdarg.h>\n#include <stdio.h>\n#if !defined(_MSC_VER) && !defined(__DMC__)\n    #include <strings.h>    /* For strcasecmp(). */\n    #include <wchar.h>      /* For wcslen(), wcsrtombs() */\n#endif\n#ifdef _MSC_VER\n    #include <float.h>      /* For _controlfp_s constants */\n#endif\n\n#if defined(MA_WIN32)\n    #include <windows.h>\n\n    /*\n    There's a possibility that WIN32_LEAN_AND_MEAN has been defined which will exclude some symbols\n    such as STGM_READ and CLSCTL_ALL. We need to check these and define them ourselves if they're\n    unavailable.\n    */\n    #ifndef STGM_READ\n    #define STGM_READ   0x00000000L\n    #endif\n    #ifndef CLSCTX_ALL\n    #define CLSCTX_ALL  23\n    #endif\n\n    /* IUnknown is used by both the WASAPI and DirectSound backends. It easier to just declare our version here. */\n    typedef struct ma_IUnknown  ma_IUnknown;\n#endif\n\n#if !defined(MA_WIN32)\n    #if !defined(MA_NO_THREADING)\n        #include <sched.h>\n        #include <pthread.h>     /* For pthreads. */\n    #endif\n\n    #include <sys/time.h>   /* select() (used for ma_sleep()). */\n    #include <time.h>       /* For nanosleep() */\n    #include <unistd.h> \n#endif\n\n/* For fstat(), etc. */\n#if defined(MA_XBOX_NXDK)\n    #include <stat.h>       /* Suggestion for NXDK: Add a sys/stat.h wrapper for compatibility. */\n#else\n    #include <sys/stat.h>\n#endif\n\n#ifdef MA_EMSCRIPTEN\n#include <emscripten/emscripten.h>\n#endif\n\n\n/* Architecture Detection */\n#if !defined(MA_64BIT) && !defined(MA_32BIT)\n#ifdef _WIN32\n#ifdef _WIN64\n#define MA_64BIT\n#else\n#define MA_32BIT\n#endif\n#endif\n#endif\n\n#if !defined(MA_64BIT) && !defined(MA_32BIT)\n#ifdef __GNUC__\n#ifdef __LP64__\n#define MA_64BIT\n#else\n#define MA_32BIT\n#endif\n#endif\n#endif\n\n#if !defined(MA_64BIT) && !defined(MA_32BIT)\n#include <stdint.h>\n#if INTPTR_MAX == INT64_MAX\n#define MA_64BIT\n#else\n#define MA_32BIT\n#endif\n#endif\n\n#if defined(__arm__) || defined(_M_ARM)\n#define MA_ARM32\n#endif\n#if defined(__arm64) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64)\n#define MA_ARM64\n#endif\n\n#if defined(__x86_64__) || defined(_M_X64)\n#define MA_X64\n#elif defined(__i386) || defined(_M_IX86)\n#define MA_X86\n#elif defined(MA_ARM32) || defined(MA_ARM64)\n#define MA_ARM\n#endif\n\n/* Intrinsics Support */\n#if defined(MA_X64) || defined(MA_X86)\n    #if defined(_MSC_VER) && !defined(__clang__)\n        /* MSVC. */\n        #if _MSC_VER >= 1400 && !defined(MA_NO_SSE2)   /* 2005 */\n            #define MA_SUPPORT_SSE2\n        #endif\n        /*#if _MSC_VER >= 1600 && !defined(MA_NO_AVX)*/    /* 2010 */\n        /*    #define MA_SUPPORT_AVX*/\n        /*#endif*/\n        #if _MSC_VER >= 1700 && !defined(MA_NO_AVX2)   /* 2012 */\n            #define MA_SUPPORT_AVX2\n        #endif\n    #else\n        /* Assume GNUC-style. */\n        #if defined(__SSE2__) && !defined(MA_NO_SSE2)\n            #define MA_SUPPORT_SSE2\n        #endif\n        /*#if defined(__AVX__) && !defined(MA_NO_AVX)*/\n        /*    #define MA_SUPPORT_AVX*/\n        /*#endif*/\n        #if defined(__AVX2__) && !defined(MA_NO_AVX2)\n            #define MA_SUPPORT_AVX2\n        #endif\n    #endif\n\n    /* If at this point we still haven't determined compiler support for the intrinsics just fall back to __has_include. */\n    #if !defined(__GNUC__) && !defined(__clang__) && defined(__has_include)\n        #if !defined(MA_SUPPORT_SSE2)   && !defined(MA_NO_SSE2)   && __has_include(<emmintrin.h>)\n            #define MA_SUPPORT_SSE2\n        #endif\n        /*#if !defined(MA_SUPPORT_AVX)    && !defined(MA_NO_AVX)    && __has_include(<immintrin.h>)*/\n        /*    #define MA_SUPPORT_AVX*/\n        /*#endif*/\n        #if !defined(MA_SUPPORT_AVX2)   && !defined(MA_NO_AVX2)   && __has_include(<immintrin.h>)\n            #define MA_SUPPORT_AVX2\n        #endif\n    #endif\n\n    #if defined(MA_SUPPORT_AVX2) || defined(MA_SUPPORT_AVX)\n        #include <immintrin.h>\n    #elif defined(MA_SUPPORT_SSE2)\n        #include <emmintrin.h>\n    #endif\n#endif\n\n#if defined(MA_ARM)\n    #if !defined(MA_NO_NEON) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64))\n        #define MA_SUPPORT_NEON\n        #include <arm_neon.h>\n    #endif\n#endif\n\n/* Begin globally disabled warnings. */\n#if defined(_MSC_VER)\n    #pragma warning(push)\n    #pragma warning(disable:4752)   /* found Intel(R) Advanced Vector Extensions; consider using /arch:AVX */\n    #pragma warning(disable:4049)   /* compiler limit : terminating line number emission */\n#endif\n\n#if defined(MA_X64) || defined(MA_X86)\n    #if defined(_MSC_VER) && !defined(__clang__)\n        #if _MSC_VER >= 1400\n            #include <intrin.h>\n            static MA_INLINE void ma_cpuid(int info[4], int fid)\n            {\n                __cpuid(info, fid);\n            }\n        #else\n            #define MA_NO_CPUID\n        #endif\n\n        #if _MSC_VER >= 1600 && (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 160040219)\n            static MA_INLINE unsigned __int64 ma_xgetbv(int reg)\n            {\n                return _xgetbv(reg);\n            }\n        #else\n            #define MA_NO_XGETBV\n        #endif\n    #elif (defined(__GNUC__) || defined(__clang__)) && !defined(MA_ANDROID)\n        static MA_INLINE void ma_cpuid(int info[4], int fid)\n        {\n            /*\n            It looks like the -fPIC option uses the ebx register which GCC complains about. We can work around this by just using a different register, the\n            specific register of which I'm letting the compiler decide on. The \"k\" prefix is used to specify a 32-bit register. The {...} syntax is for\n            supporting different assembly dialects.\n\n            What's basically happening is that we're saving and restoring the ebx register manually.\n            */\n            #if defined(MA_X86) && defined(__PIC__)\n                __asm__ __volatile__ (\n                    \"xchg{l} {%%}ebx, %k1;\"\n                    \"cpuid;\"\n                    \"xchg{l} {%%}ebx, %k1;\"\n                    : \"=a\"(info[0]), \"=&r\"(info[1]), \"=c\"(info[2]), \"=d\"(info[3]) : \"a\"(fid), \"c\"(0)\n                );\n            #else\n                __asm__ __volatile__ (\n                    \"cpuid\" : \"=a\"(info[0]), \"=b\"(info[1]), \"=c\"(info[2]), \"=d\"(info[3]) : \"a\"(fid), \"c\"(0)\n                );\n            #endif\n        }\n\n        static MA_INLINE ma_uint64 ma_xgetbv(int reg)\n        {\n            unsigned int hi;\n            unsigned int lo;\n\n            __asm__ __volatile__ (\n                \"xgetbv\" : \"=a\"(lo), \"=d\"(hi) : \"c\"(reg)\n            );\n\n            return ((ma_uint64)hi << 32) | (ma_uint64)lo;\n        }\n    #else\n        #define MA_NO_CPUID\n        #define MA_NO_XGETBV\n    #endif\n#else\n    #define MA_NO_CPUID\n    #define MA_NO_XGETBV\n#endif\n\nstatic MA_INLINE ma_bool32 ma_has_sse2(void)\n{\n#if defined(MA_SUPPORT_SSE2)\n    #if (defined(MA_X64) || defined(MA_X86)) && !defined(MA_NO_SSE2)\n        #if defined(MA_X64)\n            return MA_TRUE;    /* 64-bit targets always support SSE2. */\n        #elif (defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__)\n            return MA_TRUE;    /* If the compiler is allowed to freely generate SSE2 code we can assume support. */\n        #else\n            #if defined(MA_NO_CPUID)\n                return MA_FALSE;\n            #else\n                int info[4];\n                ma_cpuid(info, 1);\n                return (info[3] & (1 << 26)) != 0;\n            #endif\n        #endif\n    #else\n        return MA_FALSE;       /* SSE2 is only supported on x86 and x64 architectures. */\n    #endif\n#else\n    return MA_FALSE;           /* No compiler support. */\n#endif\n}\n\n#if 0\nstatic MA_INLINE ma_bool32 ma_has_avx()\n{\n#if defined(MA_SUPPORT_AVX)\n    #if (defined(MA_X64) || defined(MA_X86)) && !defined(MA_NO_AVX)\n        #if defined(_AVX_) || defined(__AVX__)\n            return MA_TRUE;    /* If the compiler is allowed to freely generate AVX code we can assume support. */\n        #else\n            /* AVX requires both CPU and OS support. */\n            #if defined(MA_NO_CPUID) || defined(MA_NO_XGETBV)\n                return MA_FALSE;\n            #else\n                int info[4];\n                ma_cpuid(info, 1);\n                if (((info[2] & (1 << 27)) != 0) && ((info[2] & (1 << 28)) != 0)) {\n                    ma_uint64 xrc = ma_xgetbv(0);\n                    if ((xrc & 0x06) == 0x06) {\n                        return MA_TRUE;\n                    } else {\n                        return MA_FALSE;\n                    }\n                } else {\n                    return MA_FALSE;\n                }\n            #endif\n        #endif\n    #else\n        return MA_FALSE;       /* AVX is only supported on x86 and x64 architectures. */\n    #endif\n#else\n    return MA_FALSE;           /* No compiler support. */\n#endif\n}\n#endif\n\nstatic MA_INLINE ma_bool32 ma_has_avx2(void)\n{\n#if defined(MA_SUPPORT_AVX2)\n    #if (defined(MA_X64) || defined(MA_X86)) && !defined(MA_NO_AVX2)\n        #if defined(_AVX2_) || defined(__AVX2__)\n            return MA_TRUE;    /* If the compiler is allowed to freely generate AVX2 code we can assume support. */\n        #else\n            /* AVX2 requires both CPU and OS support. */\n            #if defined(MA_NO_CPUID) || defined(MA_NO_XGETBV)\n                return MA_FALSE;\n            #else\n                int info1[4];\n                int info7[4];\n                ma_cpuid(info1, 1);\n                ma_cpuid(info7, 7);\n                if (((info1[2] & (1 << 27)) != 0) && ((info7[1] & (1 << 5)) != 0)) {\n                    ma_uint64 xrc = ma_xgetbv(0);\n                    if ((xrc & 0x06) == 0x06) {\n                        return MA_TRUE;\n                    } else {\n                        return MA_FALSE;\n                    }\n                } else {\n                    return MA_FALSE;\n                }\n            #endif\n        #endif\n    #else\n        return MA_FALSE;       /* AVX2 is only supported on x86 and x64 architectures. */\n    #endif\n#else\n    return MA_FALSE;           /* No compiler support. */\n#endif\n}\n\nstatic MA_INLINE ma_bool32 ma_has_neon(void)\n{\n#if defined(MA_SUPPORT_NEON)\n    #if defined(MA_ARM) && !defined(MA_NO_NEON)\n        #if (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64))\n            return MA_TRUE;    /* If the compiler is allowed to freely generate NEON code we can assume support. */\n        #else\n            /* TODO: Runtime check. */\n            return MA_FALSE;\n        #endif\n    #else\n        return MA_FALSE;       /* NEON is only supported on ARM architectures. */\n    #endif\n#else\n    return MA_FALSE;           /* No compiler support. */\n#endif\n}\n\n#if defined(__has_builtin)\n    #define MA_COMPILER_HAS_BUILTIN(x) __has_builtin(x)\n#else\n    #define MA_COMPILER_HAS_BUILTIN(x) 0\n#endif\n\n#ifndef MA_ASSUME\n    #if MA_COMPILER_HAS_BUILTIN(__builtin_assume)\n        #define MA_ASSUME(x) __builtin_assume(x)\n    #elif MA_COMPILER_HAS_BUILTIN(__builtin_unreachable)\n        #define MA_ASSUME(x) do { if (!(x)) __builtin_unreachable(); } while (0)\n    #elif defined(_MSC_VER)\n        #define MA_ASSUME(x) __assume(x)\n    #else\n        #define MA_ASSUME(x) (void)(x)\n    #endif\n#endif\n\n#ifndef MA_RESTRICT\n    #if defined(__clang__) || defined(_MSC_VER) || (defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 95)))\n        #define MA_RESTRICT __restrict\n    #else\n        #define MA_RESTRICT\n    #endif\n#endif\n\n#if defined(_MSC_VER) && _MSC_VER >= 1400\n    #define MA_HAS_BYTESWAP16_INTRINSIC\n    #define MA_HAS_BYTESWAP32_INTRINSIC\n    #define MA_HAS_BYTESWAP64_INTRINSIC\n#elif defined(__clang__)\n    #if MA_COMPILER_HAS_BUILTIN(__builtin_bswap16)\n        #define MA_HAS_BYTESWAP16_INTRINSIC\n    #endif\n    #if MA_COMPILER_HAS_BUILTIN(__builtin_bswap32)\n        #define MA_HAS_BYTESWAP32_INTRINSIC\n    #endif\n    #if MA_COMPILER_HAS_BUILTIN(__builtin_bswap64)\n        #define MA_HAS_BYTESWAP64_INTRINSIC\n    #endif\n#elif defined(__GNUC__)\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))\n        #define MA_HAS_BYTESWAP32_INTRINSIC\n        #define MA_HAS_BYTESWAP64_INTRINSIC\n    #endif\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))\n        #define MA_HAS_BYTESWAP16_INTRINSIC\n    #endif\n#endif\n\n\nstatic MA_INLINE ma_bool32 ma_is_little_endian(void)\n{\n#if defined(MA_X86) || defined(MA_X64)\n    return MA_TRUE;\n#else\n    int n = 1;\n    return (*(char*)&n) == 1;\n#endif\n}\n\nstatic MA_INLINE ma_bool32 ma_is_big_endian(void)\n{\n    return !ma_is_little_endian();\n}\n\n\nstatic MA_INLINE ma_uint32 ma_swap_endian_uint32(ma_uint32 n)\n{\n#ifdef MA_HAS_BYTESWAP32_INTRINSIC\n    #if defined(_MSC_VER)\n        return _byteswap_ulong(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        #if defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 6) && !defined(MA_64BIT)   /* <-- 64-bit inline assembly has not been tested, so disabling for now. */\n            /* Inline assembly optimized implementation for ARM. In my testing, GCC does not generate optimized code with __builtin_bswap32(). */\n            ma_uint32 r;\n            __asm__ __volatile__ (\n            #if defined(MA_64BIT)\n                \"rev %w[out], %w[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)   /* <-- This is untested. If someone in the community could test this, that would be appreciated! */\n            #else\n                \"rev %[out], %[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)\n            #endif\n            );\n            return r;\n        #else\n            return __builtin_bswap32(n);\n        #endif\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & 0xFF000000) >> 24) |\n           ((n & 0x00FF0000) >>  8) |\n           ((n & 0x0000FF00) <<  8) |\n           ((n & 0x000000FF) << 24);\n#endif\n}\n\n\n#if !defined(MA_EMSCRIPTEN)\n#ifdef MA_WIN32\nstatic void ma_sleep__win32(ma_uint32 milliseconds)\n{\n    Sleep((DWORD)milliseconds);\n}\n#endif\n#ifdef MA_POSIX\nstatic void ma_sleep__posix(ma_uint32 milliseconds)\n{\n#ifdef MA_EMSCRIPTEN\n    (void)milliseconds;\n    MA_ASSERT(MA_FALSE);  /* The Emscripten build should never sleep. */\n#else\n    #if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 199309L) || defined(MA_SWITCH)\n        struct timespec ts;\n        ts.tv_sec  = milliseconds / 1000;\n        ts.tv_nsec = milliseconds % 1000 * 1000000;\n        nanosleep(&ts, NULL);\n    #else\n        struct timeval tv;\n        tv.tv_sec  = milliseconds / 1000;\n        tv.tv_usec = milliseconds % 1000 * 1000;\n        select(0, NULL, NULL, NULL, &tv);\n    #endif\n#endif\n}\n#endif\n\nstatic MA_INLINE void ma_sleep(ma_uint32 milliseconds)\n{\n#ifdef MA_WIN32\n    ma_sleep__win32(milliseconds);\n#endif\n#ifdef MA_POSIX\n    ma_sleep__posix(milliseconds);\n#endif\n}\n#endif\n\nstatic MA_INLINE void ma_yield(void)\n{\n#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)\n    /* x86/x64 */\n    #if (defined(_MSC_VER) || defined(__WATCOMC__) || defined(__DMC__)) && !defined(__clang__)\n        #if _MSC_VER >= 1400\n            _mm_pause();\n        #else\n            #if defined(__DMC__)\n                /* Digital Mars does not recognize the PAUSE opcode. Fall back to NOP. */\n                __asm nop;\n            #else\n                __asm pause;\n            #endif\n        #endif\n    #else\n        __asm__ __volatile__ (\"rep; nop\");\n    #endif\n#elif (defined(__arm__) && defined(__ARM_ARCH) && __ARM_ARCH >= 7) || defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6T2__)\n    /* ARM */\n    #if defined(_MSC_VER)\n        /* Apparently there is a __yield() intrinsic that's compatible with ARM, but I cannot find documentation for it nor can I find where it's declared. */\n        __yield();\n    #else\n        __asm__ __volatile__ (\"yield\"); /* ARMv6K/ARMv6T2 and above. */\n    #endif\n#else\n    /* Unknown or unsupported architecture. No-op. */\n#endif\n}\n\n\n#define MA_MM_DENORMALS_ZERO_MASK   0x0040\n#define MA_MM_FLUSH_ZERO_MASK       0x8000\n\nstatic MA_INLINE unsigned int ma_disable_denormals(void)\n{\n    unsigned int prevState;\n\n    #if defined(_MSC_VER) && !defined(MA_XBOX_NXDK)\n    {\n        /*\n        Older versions of Visual Studio don't support the \"safe\" versions of _controlfp_s(). I don't\n        know which version of Visual Studio first added support for _controlfp_s(), but I do know\n        that VC6 lacks support. _MSC_VER = 1200 is VC6, but if you get compilation errors on older\n        versions of Visual Studio, let me know and I'll make the necessary adjustment.\n        */\n        #if _MSC_VER <= 1200\n        {\n            prevState = _statusfp();\n            _controlfp(prevState | _DN_FLUSH, _MCW_DN);\n        }\n        #else\n        {\n            unsigned int unused;\n            _controlfp_s(&prevState, 0, 0);\n            _controlfp_s(&unused, prevState | _DN_FLUSH, _MCW_DN);\n        }\n        #endif\n    }\n    #elif defined(MA_X86) || defined(MA_X64)\n    {\n        #if defined(MA_SUPPORT_SSE2) && defined(__SSE2__) && !(defined(__TINYC__) || defined(__WATCOMC__)) /* <-- Add compilers that lack support for _mm_getcsr() and _mm_setcsr() to this list. */\n        {\n            prevState = _mm_getcsr();\n            _mm_setcsr(prevState | MA_MM_DENORMALS_ZERO_MASK | MA_MM_FLUSH_ZERO_MASK);\n        }\n        #else\n        {\n            /* x88/64, but no support for _mm_getcsr()/_mm_setcsr(). May need to fall back to inlined assembly here. */\n            prevState = 0;\n        }\n        #endif\n    }\n    #else\n    {\n        /* Unknown or unsupported architecture. No-op. */\n        prevState = 0;\n    }\n    #endif\n\n    return prevState;\n}\n\nstatic MA_INLINE void ma_restore_denormals(unsigned int prevState)\n{\n    #if defined(_MSC_VER) && !defined(MA_XBOX_NXDK)\n    {\n        /* Older versions of Visual Studio do not support _controlfp_s(). See ma_disable_denormals(). */\n        #if _MSC_VER <= 1200\n        {\n            _controlfp(prevState, _MCW_DN);\n        }\n        #else\n        {\n            unsigned int unused;\n            _controlfp_s(&unused, prevState, _MCW_DN);\n        }\n        #endif\n    }\n    #elif defined(MA_X86) || defined(MA_X64)\n    {\n        #if defined(MA_SUPPORT_SSE2) && defined(__SSE2__) && !(defined(__TINYC__) || defined(__WATCOMC__))   /* <-- Add compilers that lack support for _mm_getcsr() and _mm_setcsr() to this list. */\n        {\n            _mm_setcsr(prevState);\n        }\n        #else\n        {\n            /* x88/64, but no support for _mm_getcsr()/_mm_setcsr(). May need to fall back to inlined assembly here. */\n            (void)prevState;\n        }\n        #endif\n    }\n    #else\n    {\n        /* Unknown or unsupported architecture. No-op. */\n        (void)prevState;\n    }\n    #endif\n}\n\n\n#ifdef MA_ANDROID\n#include <sys/system_properties.h>\n\nint ma_android_sdk_version()\n{\n    char sdkVersion[PROP_VALUE_MAX + 1] = {0, };\n    if (__system_property_get(\"ro.build.version.sdk\", sdkVersion)) {\n        return atoi(sdkVersion);\n    }\n\n    return 0;\n}\n#endif\n\n\n#ifndef MA_COINIT_VALUE\n#define MA_COINIT_VALUE    0   /* 0 = COINIT_MULTITHREADED */\n#endif\n\n\n#ifndef MA_FLT_MAX\n    #ifdef FLT_MAX\n        #define MA_FLT_MAX FLT_MAX\n    #else\n        #define MA_FLT_MAX 3.402823466e+38F\n    #endif\n#endif\n\n\n#ifndef MA_PI\n#define MA_PI      3.14159265358979323846264f\n#endif\n#ifndef MA_PI_D\n#define MA_PI_D    3.14159265358979323846264\n#endif\n#ifndef MA_TAU\n#define MA_TAU     6.28318530717958647693f\n#endif\n#ifndef MA_TAU_D\n#define MA_TAU_D   6.28318530717958647693\n#endif\n\n\n/* The default format when ma_format_unknown (0) is requested when initializing a device. */\n#ifndef MA_DEFAULT_FORMAT\n#define MA_DEFAULT_FORMAT                                   ma_format_f32\n#endif\n\n/* The default channel count to use when 0 is used when initializing a device. */\n#ifndef MA_DEFAULT_CHANNELS\n#define MA_DEFAULT_CHANNELS                                 2\n#endif\n\n/* The default sample rate to use when 0 is used when initializing a device. */\n#ifndef MA_DEFAULT_SAMPLE_RATE\n#define MA_DEFAULT_SAMPLE_RATE                              48000\n#endif\n\n/* Default periods when none is specified in ma_device_init(). More periods means more work on the CPU. */\n#ifndef MA_DEFAULT_PERIODS\n#define MA_DEFAULT_PERIODS                                  3\n#endif\n\n/* The default period size in milliseconds for low latency mode. */\n#ifndef MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY\n#define MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY  10\n#endif\n\n/* The default buffer size in milliseconds for conservative mode. */\n#ifndef MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE\n#define MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE 100\n#endif\n\n/* The default LPF filter order for linear resampling. Note that this is clamped to MA_MAX_FILTER_ORDER. */\n#ifndef MA_DEFAULT_RESAMPLER_LPF_ORDER\n    #if MA_MAX_FILTER_ORDER >= 4\n        #define MA_DEFAULT_RESAMPLER_LPF_ORDER  4\n    #else\n        #define MA_DEFAULT_RESAMPLER_LPF_ORDER  MA_MAX_FILTER_ORDER\n    #endif\n#endif\n\n\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wunused-variable\"\n#endif\n\n/* Standard sample rates, in order of priority. */\nstatic ma_uint32 g_maStandardSampleRatePriorities[] = {\n    (ma_uint32)ma_standard_sample_rate_48000,\n    (ma_uint32)ma_standard_sample_rate_44100,\n\n    (ma_uint32)ma_standard_sample_rate_32000,\n    (ma_uint32)ma_standard_sample_rate_24000,\n    (ma_uint32)ma_standard_sample_rate_22050,\n\n    (ma_uint32)ma_standard_sample_rate_88200,\n    (ma_uint32)ma_standard_sample_rate_96000,\n    (ma_uint32)ma_standard_sample_rate_176400,\n    (ma_uint32)ma_standard_sample_rate_192000,\n\n    (ma_uint32)ma_standard_sample_rate_16000,\n    (ma_uint32)ma_standard_sample_rate_11025,\n    (ma_uint32)ma_standard_sample_rate_8000,\n\n    (ma_uint32)ma_standard_sample_rate_352800,\n    (ma_uint32)ma_standard_sample_rate_384000\n};\n\nstatic MA_INLINE ma_bool32 ma_is_standard_sample_rate(ma_uint32 sampleRate)\n{\n    ma_uint32 iSampleRate;\n\n    for (iSampleRate = 0; iSampleRate < sizeof(g_maStandardSampleRatePriorities) / sizeof(g_maStandardSampleRatePriorities[0]); iSampleRate += 1) {\n        if (g_maStandardSampleRatePriorities[iSampleRate] == sampleRate) {\n            return MA_TRUE;\n        }\n    }\n\n    /* Getting here means the sample rate is not supported. */\n    return MA_FALSE;\n}\n\n\nstatic ma_format g_maFormatPriorities[] = {\n    ma_format_s16,         /* Most common */\n    ma_format_f32,\n\n    /*ma_format_s24_32,*/    /* Clean alignment */\n    ma_format_s32,\n\n    ma_format_s24,         /* Unclean alignment */\n\n    ma_format_u8           /* Low quality */\n};\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic pop\n#endif\n\n\nMA_API void ma_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision)\n{\n    if (pMajor) {\n        *pMajor = MA_VERSION_MAJOR;\n    }\n\n    if (pMinor) {\n        *pMinor = MA_VERSION_MINOR;\n    }\n\n    if (pRevision) {\n        *pRevision = MA_VERSION_REVISION;\n    }\n}\n\nMA_API const char* ma_version_string(void)\n{\n    return MA_VERSION_STRING;\n}\n\n\n/******************************************************************************\n\nStandard Library Stuff\n\n******************************************************************************/\n#ifndef MA_ASSERT\n#define MA_ASSERT(condition)            assert(condition)\n#endif\n\n#ifndef MA_MALLOC\n#define MA_MALLOC(sz)                   malloc((sz))\n#endif\n#ifndef MA_REALLOC\n#define MA_REALLOC(p, sz)               realloc((p), (sz))\n#endif\n#ifndef MA_FREE\n#define MA_FREE(p)                      free((p))\n#endif\n\nstatic MA_INLINE void ma_zero_memory_default(void* p, size_t sz)\n{\n    if (p == NULL) {\n        MA_ASSERT(sz == 0); /* If this is triggered there's an error with the calling code. */\n        return;\n    }\n\n    if (sz > 0) {\n        memset(p, 0, sz);\n    }\n}\n\n\n#ifndef MA_ZERO_MEMORY\n#define MA_ZERO_MEMORY(p, sz)           ma_zero_memory_default((p), (sz))\n#endif\n#ifndef MA_COPY_MEMORY\n#define MA_COPY_MEMORY(dst, src, sz)    memcpy((dst), (src), (sz))\n#endif\n#ifndef MA_MOVE_MEMORY\n#define MA_MOVE_MEMORY(dst, src, sz)    memmove((dst), (src), (sz))\n#endif\n\n#define MA_ZERO_OBJECT(p)               MA_ZERO_MEMORY((p), sizeof(*(p)))\n\n#define ma_countof(x)                   (sizeof(x) / sizeof(x[0]))\n#define ma_max(x, y)                    (((x) > (y)) ? (x) : (y))\n#define ma_min(x, y)                    (((x) < (y)) ? (x) : (y))\n#define ma_abs(x)                       (((x) > 0) ? (x) : -(x))\n#define ma_clamp(x, lo, hi)             (ma_max(lo, ma_min(x, hi)))\n#define ma_offset_ptr(p, offset)        (((ma_uint8*)(p)) + (offset))\n#define ma_align(x, a)                  (((x) + ((a)-1)) & ~((a)-1))\n#define ma_align_64(x)                  ma_align(x, 8)\n\n#define ma_buffer_frame_capacity(buffer, channels, format) (sizeof(buffer) / ma_get_bytes_per_sample(format) / (channels))\n\nstatic MA_INLINE double ma_sind(double x)\n{\n    /* TODO: Implement custom sin(x). */\n    return sin(x);\n}\n\nstatic MA_INLINE double ma_expd(double x)\n{\n    /* TODO: Implement custom exp(x). */\n    return exp(x);\n}\n\nstatic MA_INLINE double ma_logd(double x)\n{\n    /* TODO: Implement custom log(x). */\n    return log(x);\n}\n\nstatic MA_INLINE double ma_powd(double x, double y)\n{\n    /* TODO: Implement custom pow(x, y). */\n    return pow(x, y);\n}\n\nstatic MA_INLINE double ma_sqrtd(double x)\n{\n    /* TODO: Implement custom sqrt(x). */\n    return sqrt(x);\n}\n\n\nstatic MA_INLINE float ma_rsqrtf(float x)\n{\n    #if defined(MA_SUPPORT_SSE2) && !defined(MA_NO_SSE2) && (defined(MA_X64) || (defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__))\n    {\n        /*\n        For SSE we can use RSQRTSS.\n\n        This Stack Overflow post suggests that compilers don't necessarily generate optimal code\n        when using intrinsics:\n\n            https://web.archive.org/web/20221211012522/https://stackoverflow.com/questions/32687079/getting-fewest-instructions-for-rsqrtss-wrapper\n\n        I'm going to do something similar here, but a bit simpler.\n        */\n        #if defined(__GNUC__) || defined(__clang__)\n        {\n            float result;\n            __asm__ __volatile__(\"rsqrtss %1, %0\" : \"=x\"(result) : \"x\"(x));\n            return result;\n        }\n        #else\n        {\n            return _mm_cvtss_f32(_mm_rsqrt_ss(_mm_set_ps1(x)));\n        }\n        #endif\n    }\n    #else\n    {\n        return 1 / (float)ma_sqrtd(x);\n    }\n    #endif\n}\n\n\nstatic MA_INLINE float ma_sinf(float x)\n{\n    return (float)ma_sind((float)x);\n}\n\nstatic MA_INLINE double ma_cosd(double x)\n{\n    return ma_sind((MA_PI_D*0.5) - x);\n}\n\nstatic MA_INLINE float ma_cosf(float x)\n{\n    return (float)ma_cosd((float)x);\n}\n\nstatic MA_INLINE double ma_log10d(double x)\n{\n    return ma_logd(x) * 0.43429448190325182765;\n}\n\nstatic MA_INLINE float ma_powf(float x, float y)\n{\n    return (float)ma_powd((double)x, (double)y);\n}\n\nstatic MA_INLINE float ma_log10f(float x)\n{\n    return (float)ma_log10d((double)x);\n}\n\n\nstatic MA_INLINE double ma_degrees_to_radians(double degrees)\n{\n    return degrees * 0.01745329252;\n}\n\nstatic MA_INLINE double ma_radians_to_degrees(double radians)\n{\n    return radians * 57.295779512896;\n}\n\nstatic MA_INLINE float ma_degrees_to_radians_f(float degrees)\n{\n    return degrees * 0.01745329252f;\n}\n\nstatic MA_INLINE float ma_radians_to_degrees_f(float radians)\n{\n    return radians * 57.295779512896f;\n}\n\n\n/*\nReturn Values:\n  0:  Success\n  22: EINVAL\n  34: ERANGE\n\nNot using symbolic constants for errors because I want to avoid #including errno.h\n\nThese are marked as no-inline because of some bad code generation by Clang. None of these functions\nare used in any performance-critical code within miniaudio.\n*/\nMA_API MA_NO_INLINE int ma_strcpy_s(char* dst, size_t dstSizeInBytes, const char* src)\n{\n    size_t i;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstSizeInBytes == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    for (i = 0; i < dstSizeInBytes && src[i] != '\\0'; ++i) {\n        dst[i] = src[i];\n    }\n\n    if (i < dstSizeInBytes) {\n        dst[i] = '\\0';\n        return 0;\n    }\n\n    dst[0] = '\\0';\n    return 34;\n}\n\nMA_API MA_NO_INLINE int ma_wcscpy_s(wchar_t* dst, size_t dstCap, const wchar_t* src)\n{\n    size_t i;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstCap == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    for (i = 0; i < dstCap && src[i] != '\\0'; ++i) {\n        dst[i] = src[i];\n    }\n\n    if (i < dstCap) {\n        dst[i] = '\\0';\n        return 0;\n    }\n\n    dst[0] = '\\0';\n    return 34;\n}\n\n\nMA_API MA_NO_INLINE int ma_strncpy_s(char* dst, size_t dstSizeInBytes, const char* src, size_t count)\n{\n    size_t maxcount;\n    size_t i;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstSizeInBytes == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    maxcount = count;\n    if (count == ((size_t)-1) || count >= dstSizeInBytes) {        /* -1 = _TRUNCATE */\n        maxcount = dstSizeInBytes - 1;\n    }\n\n    for (i = 0; i < maxcount && src[i] != '\\0'; ++i) {\n        dst[i] = src[i];\n    }\n\n    if (src[i] == '\\0' || i == count || count == ((size_t)-1)) {\n        dst[i] = '\\0';\n        return 0;\n    }\n\n    dst[0] = '\\0';\n    return 34;\n}\n\nMA_API MA_NO_INLINE int ma_strcat_s(char* dst, size_t dstSizeInBytes, const char* src)\n{\n    char* dstorig;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstSizeInBytes == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    dstorig = dst;\n\n    while (dstSizeInBytes > 0 && dst[0] != '\\0') {\n        dst += 1;\n        dstSizeInBytes -= 1;\n    }\n\n    if (dstSizeInBytes == 0) {\n        return 22;  /* Unterminated. */\n    }\n\n\n    while (dstSizeInBytes > 0 && src[0] != '\\0') {\n        *dst++ = *src++;\n        dstSizeInBytes -= 1;\n    }\n\n    if (dstSizeInBytes > 0) {\n        dst[0] = '\\0';\n    } else {\n        dstorig[0] = '\\0';\n        return 34;\n    }\n\n    return 0;\n}\n\nMA_API MA_NO_INLINE int ma_strncat_s(char* dst, size_t dstSizeInBytes, const char* src, size_t count)\n{\n    char* dstorig;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstSizeInBytes == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        return 22;\n    }\n\n    dstorig = dst;\n\n    while (dstSizeInBytes > 0 && dst[0] != '\\0') {\n        dst += 1;\n        dstSizeInBytes -= 1;\n    }\n\n    if (dstSizeInBytes == 0) {\n        return 22;  /* Unterminated. */\n    }\n\n\n    if (count == ((size_t)-1)) {        /* _TRUNCATE */\n        count = dstSizeInBytes - 1;\n    }\n\n    while (dstSizeInBytes > 0 && src[0] != '\\0' && count > 0) {\n        *dst++ = *src++;\n        dstSizeInBytes -= 1;\n        count -= 1;\n    }\n\n    if (dstSizeInBytes > 0) {\n        dst[0] = '\\0';\n    } else {\n        dstorig[0] = '\\0';\n        return 34;\n    }\n\n    return 0;\n}\n\nMA_API MA_NO_INLINE int ma_itoa_s(int value, char* dst, size_t dstSizeInBytes, int radix)\n{\n    int sign;\n    unsigned int valueU;\n    char* dstEnd;\n\n    if (dst == NULL || dstSizeInBytes == 0) {\n        return 22;\n    }\n    if (radix < 2 || radix > 36) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    sign = (value < 0 && radix == 10) ? -1 : 1;     /* The negative sign is only used when the base is 10. */\n\n    if (value < 0) {\n        valueU = -value;\n    } else {\n        valueU = value;\n    }\n\n    dstEnd = dst;\n    do\n    {\n        int remainder = valueU % radix;\n        if (remainder > 9) {\n            *dstEnd = (char)((remainder - 10) + 'a');\n        } else {\n            *dstEnd = (char)(remainder + '0');\n        }\n\n        dstEnd += 1;\n        dstSizeInBytes -= 1;\n        valueU /= radix;\n    } while (dstSizeInBytes > 0 && valueU > 0);\n\n    if (dstSizeInBytes == 0) {\n        dst[0] = '\\0';\n        return 22;  /* Ran out of room in the output buffer. */\n    }\n\n    if (sign < 0) {\n        *dstEnd++ = '-';\n        dstSizeInBytes -= 1;\n    }\n\n    if (dstSizeInBytes == 0) {\n        dst[0] = '\\0';\n        return 22;  /* Ran out of room in the output buffer. */\n    }\n\n    *dstEnd = '\\0';\n\n\n    /* At this point the string will be reversed. */\n    dstEnd -= 1;\n    while (dst < dstEnd) {\n        char temp = *dst;\n        *dst = *dstEnd;\n        *dstEnd = temp;\n\n        dst += 1;\n        dstEnd -= 1;\n    }\n\n    return 0;\n}\n\nMA_API MA_NO_INLINE int ma_strcmp(const char* str1, const char* str2)\n{\n    if (str1 == str2) return  0;\n\n    /* These checks differ from the standard implementation. It's not important, but I prefer it just for sanity. */\n    if (str1 == NULL) return -1;\n    if (str2 == NULL) return  1;\n\n    for (;;) {\n        if (str1[0] == '\\0') {\n            break;\n        }\n        if (str1[0] != str2[0]) {\n            break;\n        }\n\n        str1 += 1;\n        str2 += 1;\n    }\n\n    return ((unsigned char*)str1)[0] - ((unsigned char*)str2)[0];\n}\n\nMA_API MA_NO_INLINE int ma_wcscmp(const wchar_t* str1, const wchar_t* str2)\n{\n    if (str1 == str2) return  0;\n\n    /* These checks differ from the standard implementation. It's not important, but I prefer it just for sanity. */\n    if (str1 == NULL) return -1;\n    if (str2 == NULL) return  1;\n\n    for (;;) {\n        if (str1[0] == L'\\0') {\n            break;\n        }\n        if (str1[0] != str2[0]) {\n            break;\n        }\n\n        str1 += 1;\n        str2 += 1;\n    }\n\n    return ((unsigned short*)str1)[0] - ((unsigned short*)str2)[0];\n}\n\nMA_API MA_NO_INLINE int ma_strappend(char* dst, size_t dstSize, const char* srcA, const char* srcB)\n{\n    int result;\n\n    result = ma_strncpy_s(dst, dstSize, srcA, (size_t)-1);\n    if (result != 0) {\n        return result;\n    }\n\n    result = ma_strncat_s(dst, dstSize, srcB, (size_t)-1);\n    if (result != 0) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API MA_NO_INLINE size_t ma_wcslen(const wchar_t* str)\n{\n    const wchar_t* end;\n\n    if (str == NULL) {\n        return 0;\n    }\n\n    end = str;\n    while (end[0] != '\\0') {\n        end += 1;\n    }\n\n    return end - str;\n}\n\nMA_API MA_NO_INLINE char* ma_copy_string(const char* src, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    size_t sz;\n    char* dst;\n\n    if (src == NULL) {\n        return NULL;\n    }\n\n    sz = strlen(src)+1;\n    dst = (char*)ma_malloc(sz, pAllocationCallbacks);\n    if (dst == NULL) {\n        return NULL;\n    }\n\n    ma_strcpy_s(dst, sz, src);\n\n    return dst;\n}\n\nMA_API MA_NO_INLINE wchar_t* ma_copy_string_w(const wchar_t* src, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    size_t sz = ma_wcslen(src)+1;\n    wchar_t* dst = (wchar_t*)ma_malloc(sz * sizeof(*dst), pAllocationCallbacks);\n    if (dst == NULL) {\n        return NULL;\n    }\n\n    ma_wcscpy_s(dst, sz, src);\n\n    return dst;\n}\n\n\n\n#include <errno.h>\nstatic ma_result ma_result_from_errno(int e)\n{\n    if (e == 0) {\n        return MA_SUCCESS;\n    }\n#ifdef EPERM\n    else if (e == EPERM) { return MA_INVALID_OPERATION; }\n#endif\n#ifdef ENOENT\n    else if (e == ENOENT) { return MA_DOES_NOT_EXIST; }\n#endif\n#ifdef ESRCH\n    else if (e == ESRCH) { return MA_DOES_NOT_EXIST; }\n#endif\n#ifdef EINTR\n    else if (e == EINTR) { return MA_INTERRUPT; }\n#endif\n#ifdef EIO\n    else if (e == EIO) { return MA_IO_ERROR; }\n#endif\n#ifdef ENXIO\n    else if (e == ENXIO) { return MA_DOES_NOT_EXIST; }\n#endif\n#ifdef E2BIG\n    else if (e == E2BIG) { return MA_INVALID_ARGS; }\n#endif\n#ifdef ENOEXEC\n    else if (e == ENOEXEC) { return MA_INVALID_FILE; }\n#endif\n#ifdef EBADF\n    else if (e == EBADF) { return MA_INVALID_FILE; }\n#endif\n#ifdef ECHILD\n    else if (e == ECHILD) { return MA_ERROR; }\n#endif\n#ifdef EAGAIN\n    else if (e == EAGAIN) { return MA_UNAVAILABLE; }\n#endif\n#ifdef ENOMEM\n    else if (e == ENOMEM) { return MA_OUT_OF_MEMORY; }\n#endif\n#ifdef EACCES\n    else if (e == EACCES) { return MA_ACCESS_DENIED; }\n#endif\n#ifdef EFAULT\n    else if (e == EFAULT) { return MA_BAD_ADDRESS; }\n#endif\n#ifdef ENOTBLK\n    else if (e == ENOTBLK) { return MA_ERROR; }\n#endif\n#ifdef EBUSY\n    else if (e == EBUSY) { return MA_BUSY; }\n#endif\n#ifdef EEXIST\n    else if (e == EEXIST) { return MA_ALREADY_EXISTS; }\n#endif\n#ifdef EXDEV\n    else if (e == EXDEV) { return MA_ERROR; }\n#endif\n#ifdef ENODEV\n    else if (e == ENODEV) { return MA_DOES_NOT_EXIST; }\n#endif\n#ifdef ENOTDIR\n    else if (e == ENOTDIR) { return MA_NOT_DIRECTORY; }\n#endif\n#ifdef EISDIR\n    else if (e == EISDIR) { return MA_IS_DIRECTORY; }\n#endif\n#ifdef EINVAL\n    else if (e == EINVAL) { return MA_INVALID_ARGS; }\n#endif\n#ifdef ENFILE\n    else if (e == ENFILE) { return MA_TOO_MANY_OPEN_FILES; }\n#endif\n#ifdef EMFILE\n    else if (e == EMFILE) { return MA_TOO_MANY_OPEN_FILES; }\n#endif\n#ifdef ENOTTY\n    else if (e == ENOTTY) { return MA_INVALID_OPERATION; }\n#endif\n#ifdef ETXTBSY\n    else if (e == ETXTBSY) { return MA_BUSY; }\n#endif\n#ifdef EFBIG\n    else if (e == EFBIG) { return MA_TOO_BIG; }\n#endif\n#ifdef ENOSPC\n    else if (e == ENOSPC) { return MA_NO_SPACE; }\n#endif\n#ifdef ESPIPE\n    else if (e == ESPIPE) { return MA_BAD_SEEK; }\n#endif\n#ifdef EROFS\n    else if (e == EROFS) { return MA_ACCESS_DENIED; }\n#endif\n#ifdef EMLINK\n    else if (e == EMLINK) { return MA_TOO_MANY_LINKS; }\n#endif\n#ifdef EPIPE\n    else if (e == EPIPE) { return MA_BAD_PIPE; }\n#endif\n#ifdef EDOM\n    else if (e == EDOM) { return MA_OUT_OF_RANGE; }\n#endif\n#ifdef ERANGE\n    else if (e == ERANGE) { return MA_OUT_OF_RANGE; }\n#endif\n#ifdef EDEADLK\n    else if (e == EDEADLK) { return MA_DEADLOCK; }\n#endif\n#ifdef ENAMETOOLONG\n    else if (e == ENAMETOOLONG) { return MA_PATH_TOO_LONG; }\n#endif\n#ifdef ENOLCK\n    else if (e == ENOLCK) { return MA_ERROR; }\n#endif\n#ifdef ENOSYS\n    else if (e == ENOSYS) { return MA_NOT_IMPLEMENTED; }\n#endif\n#ifdef ENOTEMPTY\n    else if (e == ENOTEMPTY) { return MA_DIRECTORY_NOT_EMPTY; }\n#endif\n#ifdef ELOOP\n    else if (e == ELOOP) { return MA_TOO_MANY_LINKS; }\n#endif\n#ifdef ENOMSG\n    else if (e == ENOMSG) { return MA_NO_MESSAGE; }\n#endif\n#ifdef EIDRM\n    else if (e == EIDRM) { return MA_ERROR; }\n#endif\n#ifdef ECHRNG\n    else if (e == ECHRNG) { return MA_ERROR; }\n#endif\n#ifdef EL2NSYNC\n    else if (e == EL2NSYNC) { return MA_ERROR; }\n#endif\n#ifdef EL3HLT\n    else if (e == EL3HLT) { return MA_ERROR; }\n#endif\n#ifdef EL3RST\n    else if (e == EL3RST) { return MA_ERROR; }\n#endif\n#ifdef ELNRNG\n    else if (e == ELNRNG) { return MA_OUT_OF_RANGE; }\n#endif\n#ifdef EUNATCH\n    else if (e == EUNATCH) { return MA_ERROR; }\n#endif\n#ifdef ENOCSI\n    else if (e == ENOCSI) { return MA_ERROR; }\n#endif\n#ifdef EL2HLT\n    else if (e == EL2HLT) { return MA_ERROR; }\n#endif\n#ifdef EBADE\n    else if (e == EBADE) { return MA_ERROR; }\n#endif\n#ifdef EBADR\n    else if (e == EBADR) { return MA_ERROR; }\n#endif\n#ifdef EXFULL\n    else if (e == EXFULL) { return MA_ERROR; }\n#endif\n#ifdef ENOANO\n    else if (e == ENOANO) { return MA_ERROR; }\n#endif\n#ifdef EBADRQC\n    else if (e == EBADRQC) { return MA_ERROR; }\n#endif\n#ifdef EBADSLT\n    else if (e == EBADSLT) { return MA_ERROR; }\n#endif\n#ifdef EBFONT\n    else if (e == EBFONT) { return MA_INVALID_FILE; }\n#endif\n#ifdef ENOSTR\n    else if (e == ENOSTR) { return MA_ERROR; }\n#endif\n#ifdef ENODATA\n    else if (e == ENODATA) { return MA_NO_DATA_AVAILABLE; }\n#endif\n#ifdef ETIME\n    else if (e == ETIME) { return MA_TIMEOUT; }\n#endif\n#ifdef ENOSR\n    else if (e == ENOSR) { return MA_NO_DATA_AVAILABLE; }\n#endif\n#ifdef ENONET\n    else if (e == ENONET) { return MA_NO_NETWORK; }\n#endif\n#ifdef ENOPKG\n    else if (e == ENOPKG) { return MA_ERROR; }\n#endif\n#ifdef EREMOTE\n    else if (e == EREMOTE) { return MA_ERROR; }\n#endif\n#ifdef ENOLINK\n    else if (e == ENOLINK) { return MA_ERROR; }\n#endif\n#ifdef EADV\n    else if (e == EADV) { return MA_ERROR; }\n#endif\n#ifdef ESRMNT\n    else if (e == ESRMNT) { return MA_ERROR; }\n#endif\n#ifdef ECOMM\n    else if (e == ECOMM) { return MA_ERROR; }\n#endif\n#ifdef EPROTO\n    else if (e == EPROTO) { return MA_ERROR; }\n#endif\n#ifdef EMULTIHOP\n    else if (e == EMULTIHOP) { return MA_ERROR; }\n#endif\n#ifdef EDOTDOT\n    else if (e == EDOTDOT) { return MA_ERROR; }\n#endif\n#ifdef EBADMSG\n    else if (e == EBADMSG) { return MA_BAD_MESSAGE; }\n#endif\n#ifdef EOVERFLOW\n    else if (e == EOVERFLOW) { return MA_TOO_BIG; }\n#endif\n#ifdef ENOTUNIQ\n    else if (e == ENOTUNIQ) { return MA_NOT_UNIQUE; }\n#endif\n#ifdef EBADFD\n    else if (e == EBADFD) { return MA_ERROR; }\n#endif\n#ifdef EREMCHG\n    else if (e == EREMCHG) { return MA_ERROR; }\n#endif\n#ifdef ELIBACC\n    else if (e == ELIBACC) { return MA_ACCESS_DENIED; }\n#endif\n#ifdef ELIBBAD\n    else if (e == ELIBBAD) { return MA_INVALID_FILE; }\n#endif\n#ifdef ELIBSCN\n    else if (e == ELIBSCN) { return MA_INVALID_FILE; }\n#endif\n#ifdef ELIBMAX\n    else if (e == ELIBMAX) { return MA_ERROR; }\n#endif\n#ifdef ELIBEXEC\n    else if (e == ELIBEXEC) { return MA_ERROR; }\n#endif\n#ifdef EILSEQ\n    else if (e == EILSEQ) { return MA_INVALID_DATA; }\n#endif\n#ifdef ERESTART\n    else if (e == ERESTART) { return MA_ERROR; }\n#endif\n#ifdef ESTRPIPE\n    else if (e == ESTRPIPE) { return MA_ERROR; }\n#endif\n#ifdef EUSERS\n    else if (e == EUSERS) { return MA_ERROR; }\n#endif\n#ifdef ENOTSOCK\n    else if (e == ENOTSOCK) { return MA_NOT_SOCKET; }\n#endif\n#ifdef EDESTADDRREQ\n    else if (e == EDESTADDRREQ) { return MA_NO_ADDRESS; }\n#endif\n#ifdef EMSGSIZE\n    else if (e == EMSGSIZE) { return MA_TOO_BIG; }\n#endif\n#ifdef EPROTOTYPE\n    else if (e == EPROTOTYPE) { return MA_BAD_PROTOCOL; }\n#endif\n#ifdef ENOPROTOOPT\n    else if (e == ENOPROTOOPT) { return MA_PROTOCOL_UNAVAILABLE; }\n#endif\n#ifdef EPROTONOSUPPORT\n    else if (e == EPROTONOSUPPORT) { return MA_PROTOCOL_NOT_SUPPORTED; }\n#endif\n#ifdef ESOCKTNOSUPPORT\n    else if (e == ESOCKTNOSUPPORT) { return MA_SOCKET_NOT_SUPPORTED; }\n#endif\n#ifdef EOPNOTSUPP\n    else if (e == EOPNOTSUPP) { return MA_INVALID_OPERATION; }\n#endif\n#ifdef EPFNOSUPPORT\n    else if (e == EPFNOSUPPORT) { return MA_PROTOCOL_FAMILY_NOT_SUPPORTED; }\n#endif\n#ifdef EAFNOSUPPORT\n    else if (e == EAFNOSUPPORT) { return MA_ADDRESS_FAMILY_NOT_SUPPORTED; }\n#endif\n#ifdef EADDRINUSE\n    else if (e == EADDRINUSE) { return MA_ALREADY_IN_USE; }\n#endif\n#ifdef EADDRNOTAVAIL\n    else if (e == EADDRNOTAVAIL) { return MA_ERROR; }\n#endif\n#ifdef ENETDOWN\n    else if (e == ENETDOWN) { return MA_NO_NETWORK; }\n#endif\n#ifdef ENETUNREACH\n    else if (e == ENETUNREACH) { return MA_NO_NETWORK; }\n#endif\n#ifdef ENETRESET\n    else if (e == ENETRESET) { return MA_NO_NETWORK; }\n#endif\n#ifdef ECONNABORTED\n    else if (e == ECONNABORTED) { return MA_NO_NETWORK; }\n#endif\n#ifdef ECONNRESET\n    else if (e == ECONNRESET) { return MA_CONNECTION_RESET; }\n#endif\n#ifdef ENOBUFS\n    else if (e == ENOBUFS) { return MA_NO_SPACE; }\n#endif\n#ifdef EISCONN\n    else if (e == EISCONN) { return MA_ALREADY_CONNECTED; }\n#endif\n#ifdef ENOTCONN\n    else if (e == ENOTCONN) { return MA_NOT_CONNECTED; }\n#endif\n#ifdef ESHUTDOWN\n    else if (e == ESHUTDOWN) { return MA_ERROR; }\n#endif\n#ifdef ETOOMANYREFS\n    else if (e == ETOOMANYREFS) { return MA_ERROR; }\n#endif\n#ifdef ETIMEDOUT\n    else if (e == ETIMEDOUT) { return MA_TIMEOUT; }\n#endif\n#ifdef ECONNREFUSED\n    else if (e == ECONNREFUSED) { return MA_CONNECTION_REFUSED; }\n#endif\n#ifdef EHOSTDOWN\n    else if (e == EHOSTDOWN) { return MA_NO_HOST; }\n#endif\n#ifdef EHOSTUNREACH\n    else if (e == EHOSTUNREACH) { return MA_NO_HOST; }\n#endif\n#ifdef EALREADY\n    else if (e == EALREADY) { return MA_IN_PROGRESS; }\n#endif\n#ifdef EINPROGRESS\n    else if (e == EINPROGRESS) { return MA_IN_PROGRESS; }\n#endif\n#ifdef ESTALE\n    else if (e == ESTALE) { return MA_INVALID_FILE; }\n#endif\n#ifdef EUCLEAN\n    else if (e == EUCLEAN) { return MA_ERROR; }\n#endif\n#ifdef ENOTNAM\n    else if (e == ENOTNAM) { return MA_ERROR; }\n#endif\n#ifdef ENAVAIL\n    else if (e == ENAVAIL) { return MA_ERROR; }\n#endif\n#ifdef EISNAM\n    else if (e == EISNAM) { return MA_ERROR; }\n#endif\n#ifdef EREMOTEIO\n    else if (e == EREMOTEIO) { return MA_IO_ERROR; }\n#endif\n#ifdef EDQUOT\n    else if (e == EDQUOT) { return MA_NO_SPACE; }\n#endif\n#ifdef ENOMEDIUM\n    else if (e == ENOMEDIUM) { return MA_DOES_NOT_EXIST; }\n#endif\n#ifdef EMEDIUMTYPE\n    else if (e == EMEDIUMTYPE) { return MA_ERROR; }\n#endif\n#ifdef ECANCELED\n    else if (e == ECANCELED) { return MA_CANCELLED; }\n#endif\n#ifdef ENOKEY\n    else if (e == ENOKEY) { return MA_ERROR; }\n#endif\n#ifdef EKEYEXPIRED\n    else if (e == EKEYEXPIRED) { return MA_ERROR; }\n#endif\n#ifdef EKEYREVOKED\n    else if (e == EKEYREVOKED) { return MA_ERROR; }\n#endif\n#ifdef EKEYREJECTED\n    else if (e == EKEYREJECTED) { return MA_ERROR; }\n#endif\n#ifdef EOWNERDEAD\n    else if (e == EOWNERDEAD) { return MA_ERROR; }\n#endif\n#ifdef ENOTRECOVERABLE\n    else if (e == ENOTRECOVERABLE) { return MA_ERROR; }\n#endif\n#ifdef ERFKILL\n    else if (e == ERFKILL) { return MA_ERROR; }\n#endif\n#ifdef EHWPOISON\n    else if (e == EHWPOISON) { return MA_ERROR; }\n#endif\n    else {\n        return MA_ERROR;\n    }\n}\n\nMA_API ma_result ma_fopen(FILE** ppFile, const char* pFilePath, const char* pOpenMode)\n{\n#if defined(_MSC_VER) && _MSC_VER >= 1400\n    errno_t err;\n#endif\n\n    if (ppFile != NULL) {\n        *ppFile = NULL;  /* Safety. */\n    }\n\n    if (pFilePath == NULL || pOpenMode == NULL || ppFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if (defined(_MSC_VER) && _MSC_VER >= 1400) && !defined(MA_XBOX_NXDK)\n    err = fopen_s(ppFile, pFilePath, pOpenMode);\n    if (err != 0) {\n        return ma_result_from_errno(err);\n    }\n#else\n#if defined(_WIN32) || defined(__APPLE__)\n    *ppFile = fopen(pFilePath, pOpenMode);\n#else\n    #if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS == 64 && defined(_LARGEFILE64_SOURCE)\n        *ppFile = fopen64(pFilePath, pOpenMode);\n    #else\n        *ppFile = fopen(pFilePath, pOpenMode);\n    #endif\n#endif\n    if (*ppFile == NULL) {\n        ma_result result = ma_result_from_errno(errno);\n        if (result == MA_SUCCESS) {\n            result = MA_ERROR;   /* Just a safety check to make sure we never ever return success when pFile == NULL. */\n        }\n\n        return result;\n    }\n#endif\n\n    return MA_SUCCESS;\n}\n\n\n\n/*\n_wfopen() isn't always available in all compilation environments.\n\n    * Windows only.\n    * MSVC seems to support it universally as far back as VC6 from what I can tell (haven't checked further back).\n    * MinGW-64 (both 32- and 64-bit) seems to support it.\n    * MinGW wraps it in !defined(__STRICT_ANSI__).\n    * OpenWatcom wraps it in !defined(_NO_EXT_KEYS).\n\nThis can be reviewed as compatibility issues arise. The preference is to use _wfopen_s() and _wfopen() as opposed to the wcsrtombs()\nfallback, so if you notice your compiler not detecting this properly I'm happy to look at adding support.\n*/\n#if defined(_WIN32) && !defined(MA_XBOX_NXDK)\n    #if defined(_MSC_VER) || defined(__MINGW64__) || (!defined(__STRICT_ANSI__) && !defined(_NO_EXT_KEYS))\n        #define MA_HAS_WFOPEN\n    #endif\n#endif\n\nMA_API ma_result ma_wfopen(FILE** ppFile, const wchar_t* pFilePath, const wchar_t* pOpenMode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (ppFile != NULL) {\n        *ppFile = NULL;  /* Safety. */\n    }\n\n    if (pFilePath == NULL || pOpenMode == NULL || ppFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if defined(MA_HAS_WFOPEN)\n    {\n        /* Use _wfopen() on Windows. */\n        #if defined(_MSC_VER) && _MSC_VER >= 1400\n        {\n            errno_t err = _wfopen_s(ppFile, pFilePath, pOpenMode);\n            if (err != 0) {\n                return ma_result_from_errno(err);\n            }\n        }\n        #else\n        {\n            *ppFile = _wfopen(pFilePath, pOpenMode);\n            if (*ppFile == NULL) {\n                return ma_result_from_errno(errno);\n            }\n        }\n        #endif\n\n        (void)pAllocationCallbacks;\n    }\n    #elif !defined(MA_XBOX_NXDK) && !defined(MA_DOS)    /* If your compiler does not support wcsrtombs(), add it here. */\n    {\n        /*\n        Use fopen() on anything other than Windows. Requires a conversion. This is annoying because fopen() is locale specific. The only real way I can\n        think of to do this is with wcsrtombs(). Note that wcstombs() is apparently not thread-safe because it uses a static global mbstate_t object for\n        maintaining state. I've checked this with -std=c89 and it works, but if somebody get's a compiler error I'll look into improving compatibility.\n        */\n        mbstate_t mbs;\n        size_t lenMB;\n        const wchar_t* pFilePathTemp = pFilePath;\n        char* pFilePathMB = NULL;\n        char pOpenModeMB[32] = {0};\n\n        /* Get the length first. */\n        MA_ZERO_OBJECT(&mbs);\n        lenMB = wcsrtombs(NULL, &pFilePathTemp, 0, &mbs);\n        if (lenMB == (size_t)-1) {\n            return ma_result_from_errno(errno);\n        }\n\n        pFilePathMB = (char*)ma_malloc(lenMB + 1, pAllocationCallbacks);\n        if (pFilePathMB == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        pFilePathTemp = pFilePath;\n        MA_ZERO_OBJECT(&mbs);\n        wcsrtombs(pFilePathMB, &pFilePathTemp, lenMB + 1, &mbs);\n\n        /* The open mode should always consist of ASCII characters so we should be able to do a trivial conversion. */\n        {\n            size_t i = 0;\n            for (;;) {\n                if (pOpenMode[i] == 0) {\n                    pOpenModeMB[i] = '\\0';\n                    break;\n                }\n\n                pOpenModeMB[i] = (char)pOpenMode[i];\n                i += 1;\n            }\n        }\n\n        *ppFile = fopen(pFilePathMB, pOpenModeMB);\n\n        ma_free(pFilePathMB, pAllocationCallbacks);\n    }\n    #else\n    {\n        /* Getting here means there is no way to open the file with a wide character string. */\n        *ppFile = NULL;\n    }\n    #endif\n\n    if (*ppFile == NULL) {\n        return MA_ERROR;\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\nstatic MA_INLINE void ma_copy_memory_64(void* dst, const void* src, ma_uint64 sizeInBytes)\n{\n#if MA_SIZE_MAX > 0xFFFFFFFF\n    MA_COPY_MEMORY(dst, src, (size_t)sizeInBytes);\n#else\n    while (sizeInBytes > 0) {\n        ma_uint64 bytesToCopyNow = sizeInBytes;\n        if (bytesToCopyNow > MA_SIZE_MAX) {\n            bytesToCopyNow = MA_SIZE_MAX;\n        }\n\n        MA_COPY_MEMORY(dst, src, (size_t)bytesToCopyNow);  /* Safe cast to size_t. */\n\n        sizeInBytes -= bytesToCopyNow;\n        dst = (      void*)((      ma_uint8*)dst + bytesToCopyNow);\n        src = (const void*)((const ma_uint8*)src + bytesToCopyNow);\n    }\n#endif\n}\n\nstatic MA_INLINE void ma_zero_memory_64(void* dst, ma_uint64 sizeInBytes)\n{\n#if MA_SIZE_MAX > 0xFFFFFFFF\n    MA_ZERO_MEMORY(dst, (size_t)sizeInBytes);\n#else\n    while (sizeInBytes > 0) {\n        ma_uint64 bytesToZeroNow = sizeInBytes;\n        if (bytesToZeroNow > MA_SIZE_MAX) {\n            bytesToZeroNow = MA_SIZE_MAX;\n        }\n\n        MA_ZERO_MEMORY(dst, (size_t)bytesToZeroNow);  /* Safe cast to size_t. */\n\n        sizeInBytes -= bytesToZeroNow;\n        dst = (void*)((ma_uint8*)dst + bytesToZeroNow);\n    }\n#endif\n}\n\n\n/* Thanks to good old Bit Twiddling Hacks for this one: http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 */\nstatic MA_INLINE unsigned int ma_next_power_of_2(unsigned int x)\n{\n    x--;\n    x |= x >> 1;\n    x |= x >> 2;\n    x |= x >> 4;\n    x |= x >> 8;\n    x |= x >> 16;\n    x++;\n\n    return x;\n}\n\nstatic MA_INLINE unsigned int ma_prev_power_of_2(unsigned int x)\n{\n    return ma_next_power_of_2(x) >> 1;\n}\n\nstatic MA_INLINE unsigned int ma_round_to_power_of_2(unsigned int x)\n{\n    unsigned int prev = ma_prev_power_of_2(x);\n    unsigned int next = ma_next_power_of_2(x);\n    if ((next - x) > (x - prev)) {\n        return prev;\n    } else {\n        return next;\n    }\n}\n\nstatic MA_INLINE unsigned int ma_count_set_bits(unsigned int x)\n{\n    unsigned int count = 0;\n    while (x != 0) {\n        if (x & 1) {\n            count += 1;\n        }\n\n        x = x >> 1;\n    }\n\n    return count;\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nAllocation Callbacks\n\n**************************************************************************************************************************************************************/\nstatic void* ma__malloc_default(size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_MALLOC(sz);\n}\n\nstatic void* ma__realloc_default(void* p, size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_REALLOC(p, sz);\n}\n\nstatic void ma__free_default(void* p, void* pUserData)\n{\n    (void)pUserData;\n    MA_FREE(p);\n}\n\nstatic ma_allocation_callbacks ma_allocation_callbacks_init_default(void)\n{\n    ma_allocation_callbacks callbacks;\n    callbacks.pUserData = NULL;\n    callbacks.onMalloc  = ma__malloc_default;\n    callbacks.onRealloc = ma__realloc_default;\n    callbacks.onFree    = ma__free_default;\n\n    return callbacks;\n}\n\nstatic ma_result ma_allocation_callbacks_init_copy(ma_allocation_callbacks* pDst, const ma_allocation_callbacks* pSrc)\n{\n    if (pDst == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pSrc == NULL) {\n        *pDst = ma_allocation_callbacks_init_default();\n    } else {\n        if (pSrc->pUserData == NULL && pSrc->onFree == NULL && pSrc->onMalloc == NULL && pSrc->onRealloc == NULL) {\n            *pDst = ma_allocation_callbacks_init_default();\n        } else {\n            if (pSrc->onFree == NULL || (pSrc->onMalloc == NULL && pSrc->onRealloc == NULL)) {\n                return MA_INVALID_ARGS;    /* Invalid allocation callbacks. */\n            } else {\n                *pDst = *pSrc;\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\n\n/**************************************************************************************************************************************************************\n\nLogging\n\n**************************************************************************************************************************************************************/\n#ifndef ma_va_copy\n    #if !defined(_MSC_VER) || _MSC_VER >= 1800\n        #if (defined(__GNUC__) && __GNUC__ < 3)\n            #define ma_va_copy(dst, src) ((dst) = (src))    /* This is untested. Not sure if this is correct for old GCC. */\n        #else\n            #define ma_va_copy(dst, src) va_copy((dst), (src))\n        #endif\n    #else\n        #define ma_va_copy(dst, src) ((dst) = (src))\n    #endif\n#endif\n\nMA_API const char* ma_log_level_to_string(ma_uint32 logLevel)\n{\n    switch (logLevel)\n    {\n        case MA_LOG_LEVEL_DEBUG:   return \"DEBUG\";\n        case MA_LOG_LEVEL_INFO:    return \"INFO\";\n        case MA_LOG_LEVEL_WARNING: return \"WARNING\";\n        case MA_LOG_LEVEL_ERROR:   return \"ERROR\";\n        default:                   return \"ERROR\";\n    }\n}\n\n#if defined(MA_DEBUG_OUTPUT)\n#if defined(MA_ANDROID)\n    #include <android/log.h>\n#endif\n\n/* Customize this to use a specific tag in __android_log_print() for debug output messages. */\n#ifndef MA_ANDROID_LOG_TAG\n#define MA_ANDROID_LOG_TAG  \"miniaudio\"\n#endif\n\nvoid ma_log_callback_debug(void* pUserData, ma_uint32 level, const char* pMessage)\n{\n    (void)pUserData;\n\n    /* Special handling for some platforms. */\n    #if defined(MA_ANDROID)\n    {\n        /* Android. */\n        __android_log_print(ANDROID_LOG_DEBUG, MA_ANDROID_LOG_TAG, \"%s: %s\", ma_log_level_to_string(level), pMessage);\n    }\n    #else\n    {\n        /* Everything else. */\n        printf(\"%s: %s\", ma_log_level_to_string(level), pMessage);\n    }\n    #endif\n}\n#endif\n\nMA_API ma_log_callback ma_log_callback_init(ma_log_callback_proc onLog, void* pUserData)\n{\n    ma_log_callback callback;\n\n    MA_ZERO_OBJECT(&callback);\n    callback.onLog     = onLog;\n    callback.pUserData = pUserData;\n\n    return callback;\n}\n\n\nMA_API ma_result ma_log_init(const ma_allocation_callbacks* pAllocationCallbacks, ma_log* pLog)\n{\n    if (pLog == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLog);\n    ma_allocation_callbacks_init_copy(&pLog->allocationCallbacks, pAllocationCallbacks);\n\n    /* We need a mutex for thread safety. */\n    #ifndef MA_NO_THREADING\n    {\n        ma_result result = ma_mutex_init(&pLog->lock);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n    #endif\n\n    /* If we're using debug output, enable it. */\n    #if defined(MA_DEBUG_OUTPUT)\n    {\n        ma_log_register_callback(pLog, ma_log_callback_init(ma_log_callback_debug, NULL)); /* Doesn't really matter if this fails. */\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_log_uninit(ma_log* pLog)\n{\n    if (pLog == NULL) {\n        return;\n    }\n\n#ifndef MA_NO_THREADING\n    ma_mutex_uninit(&pLog->lock);\n#endif\n}\n\nstatic void ma_log_lock(ma_log* pLog)\n{\n#ifndef MA_NO_THREADING\n    ma_mutex_lock(&pLog->lock);\n#else\n    (void)pLog;\n#endif\n}\n\nstatic void ma_log_unlock(ma_log* pLog)\n{\n#ifndef MA_NO_THREADING\n    ma_mutex_unlock(&pLog->lock);\n#else\n    (void)pLog;\n#endif\n}\n\nMA_API ma_result ma_log_register_callback(ma_log* pLog, ma_log_callback callback)\n{\n    ma_result result = MA_SUCCESS;\n\n    if (pLog == NULL || callback.onLog == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_log_lock(pLog);\n    {\n        if (pLog->callbackCount == ma_countof(pLog->callbacks)) {\n            result = MA_OUT_OF_MEMORY;  /* Reached the maximum allowed log callbacks. */\n        } else {\n            pLog->callbacks[pLog->callbackCount] = callback;\n            pLog->callbackCount += 1;\n        }\n    }\n    ma_log_unlock(pLog);\n\n    return result;\n}\n\nMA_API ma_result ma_log_unregister_callback(ma_log* pLog, ma_log_callback callback)\n{\n    if (pLog == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_log_lock(pLog);\n    {\n        ma_uint32 iLog;\n        for (iLog = 0; iLog < pLog->callbackCount; ) {\n            if (pLog->callbacks[iLog].onLog == callback.onLog) {\n                /* Found. Move everything down a slot. */\n                ma_uint32 jLog;\n                for (jLog = iLog; jLog < pLog->callbackCount-1; jLog += 1) {\n                    pLog->callbacks[jLog] = pLog->callbacks[jLog + 1];\n                }\n\n                pLog->callbackCount -= 1;\n            } else {\n                /* Not found. */\n                iLog += 1;\n            }\n        }\n    }\n    ma_log_unlock(pLog);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_log_post(ma_log* pLog, ma_uint32 level, const char* pMessage)\n{\n    if (pLog == NULL || pMessage == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_log_lock(pLog);\n    {\n        ma_uint32 iLog;\n        for (iLog = 0; iLog < pLog->callbackCount; iLog += 1) {\n            if (pLog->callbacks[iLog].onLog) {\n                pLog->callbacks[iLog].onLog(pLog->callbacks[iLog].pUserData, level, pMessage);\n            }\n        }\n    }\n    ma_log_unlock(pLog);\n\n    return MA_SUCCESS;\n}\n\n\n/*\nWe need to emulate _vscprintf() for the VC6 build. This can be more efficient, but since it's only VC6, and it's just a\nlogging function, I'm happy to keep this simple. In the VC6 build we can implement this in terms of _vsnprintf().\n*/\n#if defined(_MSC_VER) && _MSC_VER < 1900\nstatic int ma_vscprintf(const ma_allocation_callbacks* pAllocationCallbacks, const char* format, va_list args)\n{\n#if _MSC_VER > 1200\n    return _vscprintf(format, args);\n#else\n    int result;\n    char* pTempBuffer = NULL;\n    size_t tempBufferCap = 1024;\n\n    if (format == NULL) {\n        errno = EINVAL;\n        return -1;\n    }\n\n    for (;;) {\n        char* pNewTempBuffer = (char*)ma_realloc(pTempBuffer, tempBufferCap, pAllocationCallbacks);\n        if (pNewTempBuffer == NULL) {\n            ma_free(pTempBuffer, pAllocationCallbacks);\n            errno = ENOMEM;\n            return -1;  /* Out of memory. */\n        }\n\n        pTempBuffer = pNewTempBuffer;\n\n        result = _vsnprintf(pTempBuffer, tempBufferCap, format, args);\n        ma_free(pTempBuffer, NULL);\n\n        if (result != -1) {\n            break;  /* Got it. */\n        }\n\n        /* Buffer wasn't big enough. Ideally it'd be nice to use an error code to know the reason for sure, but this is reliable enough. */\n        tempBufferCap *= 2;\n    }\n\n    return result;\n#endif\n}\n#endif\n\nMA_API ma_result ma_log_postv(ma_log* pLog, ma_uint32 level, const char* pFormat, va_list args)\n{\n    if (pLog == NULL || pFormat == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || ((!defined(_MSC_VER) || _MSC_VER >= 1900) && !defined(__STRICT_ANSI__) && !defined(_NO_EXT_KEYS)) || (defined(__cplusplus) && __cplusplus >= 201103L)\n    {\n        ma_result result;\n        int length;\n        char  pFormattedMessageStack[1024];\n        char* pFormattedMessageHeap = NULL;\n        va_list args2;\n\n        /* First try formatting into our fixed sized stack allocated buffer. If this is too small we'll fallback to a heap allocation. */\n        ma_va_copy(args2, args);\n        {\n            length = vsnprintf(pFormattedMessageStack, sizeof(pFormattedMessageStack), pFormat, args2);\n        }\n        va_end(args2);\n\n        if (length < 0) {\n            return MA_INVALID_OPERATION;    /* An error occurred when trying to convert the buffer. */\n        }\n\n        if ((size_t)length < sizeof(pFormattedMessageStack)) {\n            /* The string was written to the stack. */\n            result = ma_log_post(pLog, level, pFormattedMessageStack);\n        } else {\n            /* The stack buffer was too small, try the heap. */\n            pFormattedMessageHeap = (char*)ma_malloc(length + 1, &pLog->allocationCallbacks);\n            if (pFormattedMessageHeap == NULL) {\n                return MA_OUT_OF_MEMORY;\n            }\n\n            length = vsnprintf(pFormattedMessageHeap, length + 1, pFormat, args);\n            if (length < 0) {\n                ma_free(pFormattedMessageHeap, &pLog->allocationCallbacks);\n                return MA_INVALID_OPERATION;\n            }\n\n            result = ma_log_post(pLog, level, pFormattedMessageHeap);\n            ma_free(pFormattedMessageHeap, &pLog->allocationCallbacks);\n        }\n\n        return result;\n    }\n    #else\n    {\n        /*\n        Without snprintf() we need to first measure the string and then heap allocate it. I'm only aware of Visual Studio having support for this without snprintf(), so we'll\n        need to restrict this branch to Visual Studio. For other compilers we need to just not support formatted logging because I don't want the security risk of overflowing\n        a fixed sized stack allocated buffer.\n        */\n        #if defined(_MSC_VER) && _MSC_VER >= 1200   /* 1200 = VC6 */\n        {\n            ma_result result;\n            int formattedLen;\n            char* pFormattedMessage = NULL;\n            va_list args2;\n\n            ma_va_copy(args2, args);\n            {\n                formattedLen = ma_vscprintf(&pLog->allocationCallbacks, pFormat, args2);\n            }\n            va_end(args2);\n\n            if (formattedLen <= 0) {\n                return MA_INVALID_OPERATION;\n            }\n\n            pFormattedMessage = (char*)ma_malloc(formattedLen + 1, &pLog->allocationCallbacks);\n            if (pFormattedMessage == NULL) {\n                return MA_OUT_OF_MEMORY;\n            }\n\n            /* We'll get errors on newer versions of Visual Studio if we try to use vsprintf().  */\n            #if _MSC_VER >= 1400    /* 1400 = Visual Studio 2005 */\n            {\n                vsprintf_s(pFormattedMessage, formattedLen + 1, pFormat, args);\n            }\n            #else\n            {\n                vsprintf(pFormattedMessage, pFormat, args);\n            }\n            #endif\n\n            result = ma_log_post(pLog, level, pFormattedMessage);\n            ma_free(pFormattedMessage, &pLog->allocationCallbacks);\n\n            return result;\n        }\n        #else\n        {\n            /* Can't do anything because we don't have a safe way of to emulate vsnprintf() without a manual solution. */\n            (void)level;\n            (void)args;\n\n            return MA_INVALID_OPERATION;\n        }\n        #endif\n    }\n    #endif\n}\n\nMA_API ma_result ma_log_postf(ma_log* pLog, ma_uint32 level, const char* pFormat, ...)\n{\n    ma_result result;\n    va_list args;\n\n    if (pLog == NULL || pFormat == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    va_start(args, pFormat);\n    {\n        result = ma_log_postv(pLog, level, pFormat, args);\n    }\n    va_end(args);\n\n    return result;\n}\n\n\n\nstatic MA_INLINE ma_uint8 ma_clip_u8(ma_int32 x)\n{\n    return (ma_uint8)(ma_clamp(x, -128, 127) + 128);\n}\n\nstatic MA_INLINE ma_int16 ma_clip_s16(ma_int32 x)\n{\n    return (ma_int16)ma_clamp(x, -32768, 32767);\n}\n\nstatic MA_INLINE ma_int64 ma_clip_s24(ma_int64 x)\n{\n    return (ma_int64)ma_clamp(x, -8388608, 8388607);\n}\n\nstatic MA_INLINE ma_int32 ma_clip_s32(ma_int64 x)\n{\n    /* This dance is to silence warnings with -std=c89. A good compiler should be able to optimize this away. */\n    ma_int64 clipMin;\n    ma_int64 clipMax;\n    clipMin = -((ma_int64)2147483647 + 1);\n    clipMax =   (ma_int64)2147483647;\n\n    return (ma_int32)ma_clamp(x, clipMin, clipMax);\n}\n\nstatic MA_INLINE float ma_clip_f32(float x)\n{\n    if (x < -1) return -1;\n    if (x > +1) return +1;\n    return x;\n}\n\n\nstatic MA_INLINE float ma_mix_f32(float x, float y, float a)\n{\n    return x*(1-a) + y*a;\n}\nstatic MA_INLINE float ma_mix_f32_fast(float x, float y, float a)\n{\n    float r0 = (y - x);\n    float r1 = r0*a;\n    return x + r1;\n    /*return x + (y - x)*a;*/\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE __m128 ma_mix_f32_fast__sse2(__m128 x, __m128 y, __m128 a)\n{\n    return _mm_add_ps(x, _mm_mul_ps(_mm_sub_ps(y, x), a));\n}\n#endif\n#if defined(MA_SUPPORT_AVX2)\nstatic MA_INLINE __m256 ma_mix_f32_fast__avx2(__m256 x, __m256 y, __m256 a)\n{\n    return _mm256_add_ps(x, _mm256_mul_ps(_mm256_sub_ps(y, x), a));\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE float32x4_t ma_mix_f32_fast__neon(float32x4_t x, float32x4_t y, float32x4_t a)\n{\n    return vaddq_f32(x, vmulq_f32(vsubq_f32(y, x), a));\n}\n#endif\n\n\nstatic MA_INLINE double ma_mix_f64(double x, double y, double a)\n{\n    return x*(1-a) + y*a;\n}\nstatic MA_INLINE double ma_mix_f64_fast(double x, double y, double a)\n{\n    return x + (y - x)*a;\n}\n\nstatic MA_INLINE float ma_scale_to_range_f32(float x, float lo, float hi)\n{\n    return lo + x*(hi-lo);\n}\n\n\n/*\nGreatest common factor using Euclid's algorithm iteratively.\n*/\nstatic MA_INLINE ma_uint32 ma_gcf_u32(ma_uint32 a, ma_uint32 b)\n{\n    for (;;) {\n        if (b == 0) {\n            break;\n        } else {\n            ma_uint32 t = a;\n            a = b;\n            b = t % a;\n        }\n    }\n\n    return a;\n}\n\n\nstatic ma_uint32 ma_ffs_32(ma_uint32 x)\n{\n    ma_uint32 i;\n\n    /* Just a naive implementation just to get things working for now. Will optimize this later. */\n    for (i = 0; i < 32; i += 1) {\n        if ((x & (1U << i)) != 0) {\n            return i;\n        }\n    }\n\n    return i;\n}\n\nstatic MA_INLINE ma_int16 ma_float_to_fixed_16(float x)\n{\n    return (ma_int16)(x * (1 << 8));\n}\n\n\n\n/*\nRandom Number Generation\n\nminiaudio uses the LCG random number generation algorithm. This is good enough for audio.\n\nNote that miniaudio's global LCG implementation uses global state which is _not_ thread-local. When this is called across\nmultiple threads, results will be unpredictable. However, it won't crash and results will still be random enough for\nminiaudio's purposes.\n*/\n#ifndef MA_DEFAULT_LCG_SEED\n#define MA_DEFAULT_LCG_SEED 4321\n#endif\n\n#define MA_LCG_M   2147483647\n#define MA_LCG_A   48271\n#define MA_LCG_C   0\n\nstatic ma_lcg g_maLCG = {MA_DEFAULT_LCG_SEED}; /* Non-zero initial seed. Use ma_lcg_seed() to use an explicit seed. */\n\nstatic MA_INLINE void ma_lcg_seed(ma_lcg* pLCG, ma_int32 seed)\n{\n    MA_ASSERT(pLCG != NULL);\n    pLCG->state = seed;\n}\n\nstatic MA_INLINE ma_int32 ma_lcg_rand_s32(ma_lcg* pLCG)\n{\n    pLCG->state = (MA_LCG_A * pLCG->state + MA_LCG_C) % MA_LCG_M;\n    return pLCG->state;\n}\n\nstatic MA_INLINE ma_uint32 ma_lcg_rand_u32(ma_lcg* pLCG)\n{\n    return (ma_uint32)ma_lcg_rand_s32(pLCG);\n}\n\nstatic MA_INLINE ma_int16 ma_lcg_rand_s16(ma_lcg* pLCG)\n{\n    return (ma_int16)(ma_lcg_rand_s32(pLCG) & 0xFFFF);\n}\n\nstatic MA_INLINE double ma_lcg_rand_f64(ma_lcg* pLCG)\n{\n    return ma_lcg_rand_s32(pLCG) / (double)0x7FFFFFFF;\n}\n\nstatic MA_INLINE float ma_lcg_rand_f32(ma_lcg* pLCG)\n{\n    return (float)ma_lcg_rand_f64(pLCG);\n}\n\nstatic MA_INLINE float ma_lcg_rand_range_f32(ma_lcg* pLCG, float lo, float hi)\n{\n    return ma_scale_to_range_f32(ma_lcg_rand_f32(pLCG), lo, hi);\n}\n\nstatic MA_INLINE ma_int32 ma_lcg_rand_range_s32(ma_lcg* pLCG, ma_int32 lo, ma_int32 hi)\n{\n    if (lo == hi) {\n        return lo;\n    }\n\n    return lo + ma_lcg_rand_u32(pLCG) / (0xFFFFFFFF / (hi - lo + 1) + 1);\n}\n\n\n#if 0   /* Currently unused. */\nstatic MA_INLINE void ma_seed(ma_int32 seed)\n{\n    ma_lcg_seed(&g_maLCG, seed);\n}\n\nstatic MA_INLINE ma_int32 ma_rand_s32(void)\n{\n    return ma_lcg_rand_s32(&g_maLCG);\n}\n\nstatic MA_INLINE ma_uint32 ma_rand_u32(void)\n{\n    return ma_lcg_rand_u32(&g_maLCG);\n}\n\nstatic MA_INLINE double ma_rand_f64(void)\n{\n    return ma_lcg_rand_f64(&g_maLCG);\n}\n\nstatic MA_INLINE float ma_rand_f32(void)\n{\n    return ma_lcg_rand_f32(&g_maLCG);\n}\n#endif\n\nstatic MA_INLINE float ma_rand_range_f32(float lo, float hi)\n{\n    return ma_lcg_rand_range_f32(&g_maLCG, lo, hi);\n}\n\nstatic MA_INLINE ma_int32 ma_rand_range_s32(ma_int32 lo, ma_int32 hi)\n{\n    return ma_lcg_rand_range_s32(&g_maLCG, lo, hi);\n}\n\n\nstatic MA_INLINE float ma_dither_f32_rectangle(float ditherMin, float ditherMax)\n{\n    return ma_rand_range_f32(ditherMin, ditherMax);\n}\n\nstatic MA_INLINE float ma_dither_f32_triangle(float ditherMin, float ditherMax)\n{\n    float a = ma_rand_range_f32(ditherMin, 0);\n    float b = ma_rand_range_f32(0, ditherMax);\n    return a + b;\n}\n\nstatic MA_INLINE float ma_dither_f32(ma_dither_mode ditherMode, float ditherMin, float ditherMax)\n{\n    if (ditherMode == ma_dither_mode_rectangle) {\n        return ma_dither_f32_rectangle(ditherMin, ditherMax);\n    }\n    if (ditherMode == ma_dither_mode_triangle) {\n        return ma_dither_f32_triangle(ditherMin, ditherMax);\n    }\n\n    return 0;\n}\n\nstatic MA_INLINE ma_int32 ma_dither_s32(ma_dither_mode ditherMode, ma_int32 ditherMin, ma_int32 ditherMax)\n{\n    if (ditherMode == ma_dither_mode_rectangle) {\n        ma_int32 a = ma_rand_range_s32(ditherMin, ditherMax);\n        return a;\n    }\n    if (ditherMode == ma_dither_mode_triangle) {\n        ma_int32 a = ma_rand_range_s32(ditherMin, 0);\n        ma_int32 b = ma_rand_range_s32(0, ditherMax);\n        return a + b;\n    }\n\n    return 0;\n}\n\n\n/**************************************************************************************************************************************************************\n\nAtomics\n\n**************************************************************************************************************************************************************/\n/* c89atomic.h begin */\n#ifndef ma_atomic_h\n#define ma_atomic_h\n#if defined(__cplusplus)\nextern \"C\" {\n#endif\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wlong-long\"\n    #if defined(__clang__)\n        #pragma GCC diagnostic ignored \"-Wc++11-long-long\"\n    #endif\n#endif\ntypedef int ma_atomic_memory_order;\n#if !defined(MA_ATOMIC_MODERN_MSVC) && \\\n    !defined(MA_ATOMIC_LEGACY_MSVC) && \\\n    !defined(MA_ATOMIC_LEGACY_MSVC_ASM) && \\\n    !defined(MA_ATOMIC_MODERN_GCC) && \\\n    !defined(MA_ATOMIC_LEGACY_GCC) && \\\n    !defined(MA_ATOMIC_LEGACY_GCC_ASM)\n    #if defined(_MSC_VER) || defined(__WATCOMC__) || defined(__DMC__) || defined(__BORLANDC__)\n        #if (defined(_MSC_VER) && _MSC_VER > 1600)\n            #define MA_ATOMIC_MODERN_MSVC\n        #else\n            #if defined(MA_X64)\n                #define MA_ATOMIC_LEGACY_MSVC\n            #else\n                #define MA_ATOMIC_LEGACY_MSVC_ASM\n            #endif\n        #endif\n    #elif (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))) || defined(__clang__)\n        #define MA_ATOMIC_MODERN_GCC\n    #else\n        #if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1))\n            #define MA_ATOMIC_LEGACY_GCC\n        #else\n            #define MA_ATOMIC_LEGACY_GCC_ASM\n        #endif\n    #endif\n#endif\n#if defined(MA_ATOMIC_MODERN_MSVC) || defined(MA_ATOMIC_LEGACY_MSVC)\n    #include <intrin.h>\n    #define ma_atomic_memory_order_relaxed  1\n    #define ma_atomic_memory_order_consume  2\n    #define ma_atomic_memory_order_acquire  3\n    #define ma_atomic_memory_order_release  4\n    #define ma_atomic_memory_order_acq_rel  5\n    #define ma_atomic_memory_order_seq_cst  6\n    #define MA_ATOMIC_MSVC_ARM_INTRINSIC_NORETURN(dst, src, order, intrin, ma_atomicType, msvcType)   \\\n        switch (order) \\\n        { \\\n            case ma_atomic_memory_order_relaxed: \\\n            { \\\n                intrin##_nf((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_consume: \\\n            case ma_atomic_memory_order_acquire: \\\n            { \\\n                intrin##_acq((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_release: \\\n            { \\\n                intrin##_rel((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_acq_rel: \\\n            case ma_atomic_memory_order_seq_cst: \\\n            default: \\\n            { \\\n                intrin((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n        }\n    #define MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, intrin, ma_atomicType, msvcType)   \\\n        ma_atomicType result; \\\n        switch (order) \\\n        { \\\n            case ma_atomic_memory_order_relaxed: \\\n            { \\\n                result = (ma_atomicType)intrin##_nf((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_consume: \\\n            case ma_atomic_memory_order_acquire: \\\n            { \\\n                result = (ma_atomicType)intrin##_acq((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_release: \\\n            { \\\n                result = (ma_atomicType)intrin##_rel((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n            case ma_atomic_memory_order_acq_rel: \\\n            case ma_atomic_memory_order_seq_cst: \\\n            default: \\\n            { \\\n                result = (ma_atomicType)intrin((volatile msvcType*)dst, (msvcType)src); \\\n            } break; \\\n        } \\\n        return result;\n    typedef ma_uint32 ma_atomic_flag;\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_test_and_set_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, 1, order, _InterlockedExchange, ma_atomic_flag, long);\n        }\n        #else\n        {\n            (void)order;\n            return (ma_atomic_flag)_InterlockedExchange((volatile long*)dst, (long)1);\n        }\n        #endif\n    }\n    static MA_INLINE void ma_atomic_flag_clear_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC_NORETURN(dst, 0, order, _InterlockedExchange, ma_atomic_flag, long);\n        }\n        #else\n        {\n            (void)order;\n            _InterlockedExchange((volatile long*)dst, (long)0);\n        }\n        #endif\n    }\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_load_explicit(volatile const ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        (void)order;\n        return (ma_uint32)_InterlockedCompareExchange((volatile long*)dst, 0, 0);\n    }\n#endif\n#if defined(MA_ATOMIC_LEGACY_MSVC_ASM)\n    #define ma_atomic_memory_order_relaxed  1\n    #define ma_atomic_memory_order_consume  2\n    #define ma_atomic_memory_order_acquire  3\n    #define ma_atomic_memory_order_release  4\n    #define ma_atomic_memory_order_acq_rel  5\n    #define ma_atomic_memory_order_seq_cst  6\n    typedef ma_uint32 ma_atomic_flag;\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_test_and_set_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        ma_atomic_flag result = 0;\n        (void)order;\n        __asm {\n            mov ecx, dst\n            mov eax, 1\n            xchg [ecx], eax\n            mov result, eax\n        }\n        return result;\n    }\n    static MA_INLINE void ma_atomic_flag_clear_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        if (order == ma_atomic_memory_order_relaxed) {\n            __asm {\n                mov esi, dst\n                mov dword ptr [esi], 0\n            }\n        } else {\n            __asm {\n                mov esi, dst\n                mov eax, 0\n                xchg [esi], eax\n            }\n        }\n    }\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_load_explicit(volatile const ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        ma_atomic_flag result = 0;\n        if (order == ma_atomic_memory_order_relaxed) {\n            __asm {\n                mov esi, dst\n                mov eax, [esi]\n                mov result, eax\n            }\n        } else if (order <= ma_atomic_memory_order_release) {\n            __asm {\n                mov esi, dst\n                mov eax, [esi]\n                lock add dword ptr [esp], 0\n                mov result, eax\n            }\n        } else {\n            __asm {\n                lock add dword ptr [esp], 0\n                mov esi, dst\n                mov eax, [esi]\n                mov result, eax\n                lock add dword ptr [esp], 0\n            }\n        }\n        return result;\n    }\n#endif\n#if defined(MA_ATOMIC_MODERN_GCC)\n    #define ma_atomic_memory_order_relaxed                   __ATOMIC_RELAXED\n    #define ma_atomic_memory_order_consume                   __ATOMIC_CONSUME\n    #define ma_atomic_memory_order_acquire                   __ATOMIC_ACQUIRE\n    #define ma_atomic_memory_order_release                   __ATOMIC_RELEASE\n    #define ma_atomic_memory_order_acq_rel                   __ATOMIC_ACQ_REL\n    #define ma_atomic_memory_order_seq_cst                   __ATOMIC_SEQ_CST\n    typedef ma_uint32 ma_atomic_flag;\n    #define ma_atomic_flag_test_and_set_explicit(dst, order) __atomic_exchange_n(dst, 1, order)\n    #define ma_atomic_flag_clear_explicit(dst, order)        __atomic_store_n(dst, 0, order)\n    #define ma_atomic_flag_load_explicit(dst, order)         __atomic_load_n(dst, order)\n#endif\n#if defined(MA_ATOMIC_LEGACY_GCC)\n    #define ma_atomic_memory_order_relaxed  1\n    #define ma_atomic_memory_order_consume  2\n    #define ma_atomic_memory_order_acquire  3\n    #define ma_atomic_memory_order_release  4\n    #define ma_atomic_memory_order_acq_rel  5\n    #define ma_atomic_memory_order_seq_cst  6\n    typedef ma_uint32 ma_atomic_flag;\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_test_and_set_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        if (order > ma_atomic_memory_order_acquire) {\n            __sync_synchronize();\n        }\n        return __sync_lock_test_and_set(dst, 1);\n    }\n    static MA_INLINE void ma_atomic_flag_clear_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        if (order > ma_atomic_memory_order_release) {\n            __sync_synchronize();\n        }\n        __sync_lock_release(dst);\n    }\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_load_explicit(volatile const ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        (void)order;\n        return __sync_val_compare_and_swap((ma_atomic_flag*)dst, 0, 0);\n    }\n#endif\n#if defined(MA_ATOMIC_LEGACY_GCC_ASM)\n    #define ma_atomic_memory_order_relaxed  1\n    #define ma_atomic_memory_order_consume  2\n    #define ma_atomic_memory_order_acquire  3\n    #define ma_atomic_memory_order_release  4\n    #define ma_atomic_memory_order_acq_rel  5\n    #define ma_atomic_memory_order_seq_cst  6\n    #if defined(MA_X86)\n        #define ma_atomic_thread_fence(order) __asm__ __volatile__(\"lock; addl $0, (%%esp)\" ::: \"memory\")\n    #elif defined(MA_X64)\n        #define ma_atomic_thread_fence(order) __asm__ __volatile__(\"lock; addq $0, (%%rsp)\" ::: \"memory\")\n    #else\n        #error Unsupported architecture.\n    #endif\n    #define MA_ATOMIC_XCHG_GCC_X86(instructionSizeSuffix, result, dst, src) \\\n        __asm__ __volatile__(                    \\\n            \"xchg\"instructionSizeSuffix\" %0, %1\" \\\n            : \"=r\"(result),              \\\n              \"=m\"(*dst)                 \\\n            : \"0\"(src),                  \\\n              \"m\"(*dst)                  \\\n            : \"memory\"                           \\\n        )\n    #define MA_ATOMIC_LOAD_RELAXED_GCC_X86(instructionSizeSuffix, result, dst) \\\n        __asm__ __volatile__(                   \\\n            \"mov\"instructionSizeSuffix\" %1, %0\" \\\n            : \"=r\"(result)              \\\n            : \"m\"(*dst)                 \\\n        )\n    #define MA_ATOMIC_LOAD_RELEASE_GCC_X86(instructionSizeSuffix, result, dst) \\\n        ma_atomic_thread_fence(ma_atomic_memory_order_release); \\\n        __asm__ __volatile__(                   \\\n            \"mov\"instructionSizeSuffix\" %1, %0\" \\\n            : \"=r\"(result)              \\\n            : \"m\"(*dst)                 \\\n            : \"memory\"                          \\\n        )\n    #define MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(instructionSizeSuffix, result, dst) \\\n        ma_atomic_thread_fence(ma_atomic_memory_order_seq_cst); \\\n        __asm__ __volatile__(                   \\\n            \"mov\"instructionSizeSuffix\" %1, %0\" \\\n            : \"=r\"(result)              \\\n            : \"m\"(*dst)                 \\\n            : \"memory\"                          \\\n        );                                      \\\n        ma_atomic_thread_fence(ma_atomic_memory_order_seq_cst)\n    typedef ma_uint32 ma_atomic_flag;\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_test_and_set_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        ma_atomic_flag result;\n        #if defined(MA_X86) || defined(MA_X64)\n        {\n            (void)order;\n            MA_ATOMIC_XCHG_GCC_X86(\"l\", result, dst, 1);\n        }\n        #else\n        {\n            #error Unsupported architecture.\n        }\n        #endif\n        return result;\n    }\n    static MA_INLINE void ma_atomic_flag_clear_explicit(volatile ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_X86) || defined(MA_X64)\n        {\n            if (order == ma_atomic_memory_order_relaxed) {\n                __asm__ __volatile__(\n                    \"movl $0, %0\"\n                    : \"=m\"(*dst)\n                );\n            } else if (order == ma_atomic_memory_order_release) {\n                __asm__ __volatile__(\n                    \"movl $0, %0\"\n                    : \"=m\"(*dst)\n                    :\n                    : \"memory\"\n                );\n            } else {\n                ma_atomic_flag tmp = 0;\n                __asm__ __volatile__(\n                    \"xchgl %0, %1\"\n                    : \"=r\"(tmp),\n                      \"=m\"(*dst)\n                    : \"0\"(tmp),\n                      \"m\"(*dst)\n                    : \"memory\"\n                );\n            }\n        }\n        #else\n        {\n            #error Unsupported architecture.\n        }\n        #endif\n    }\n    static MA_INLINE ma_atomic_flag ma_atomic_flag_load_explicit(volatile const ma_atomic_flag* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_X86) || defined(MA_X64)\n        {\n            ma_atomic_flag result;\n            if (order == ma_atomic_memory_order_relaxed) {\n                MA_ATOMIC_LOAD_RELAXED_GCC_X86(\"l\", result, dst);\n            } else if (order <= ma_atomic_memory_order_release) {\n                MA_ATOMIC_LOAD_RELEASE_GCC_X86(\"l\", result, dst);\n            } else {\n                MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(\"l\", result, dst);\n            }\n            return result;\n        }\n        #else\n        {\n            #error Unsupported architecture.\n        }\n        #endif\n    }\n#endif\n#define ma_atomic_flag_test_and_set(dst) ma_atomic_flag_test_and_set_explicit(dst, ma_atomic_memory_order_acquire)\n#define ma_atomic_flag_clear(dst)        ma_atomic_flag_clear_explicit(dst, ma_atomic_memory_order_release)\ntypedef ma_atomic_flag ma_atomic_spinlock;\nstatic MA_INLINE void ma_atomic_spinlock_lock(volatile ma_atomic_spinlock* pSpinlock)\n{\n    for (;;) {\n        if (ma_atomic_flag_test_and_set_explicit(pSpinlock, ma_atomic_memory_order_acquire) == 0) {\n            break;\n        }\n        while (ma_atomic_flag_load_explicit(pSpinlock, ma_atomic_memory_order_relaxed) == 1) {\n        }\n    }\n}\nstatic MA_INLINE void ma_atomic_spinlock_unlock(volatile ma_atomic_spinlock* pSpinlock)\n{\n    ma_atomic_flag_clear_explicit(pSpinlock, ma_atomic_memory_order_release);\n}\nma_atomic_spinlock ma_atomic_global_lock;\n#if defined(MA_ATOMIC_MODERN_MSVC) || defined(MA_ATOMIC_LEGACY_MSVC) || defined(MA_ATOMIC_LEGACY_MSVC_ASM) || defined(MA_ATOMIC_LEGACY_GCC) || defined(MA_ATOMIC_LEGACY_GCC_ASM)\n    #if defined(MA_X64) || (defined(MA_X86) && ((defined(__GNUC__) && defined(__i486__)) || (defined(_M_IX86) && _M_IX86 >= 400)))\n        #if defined(MA_ATOMIC_LEGACY_MSVC) && defined(MA_X64)\n        #else\n            #define MA_ATOMIC_IS_LOCK_FREE_8  1\n            #define MA_ATOMIC_IS_LOCK_FREE_16 1\n        #endif\n        #define MA_ATOMIC_IS_LOCK_FREE_32     1\n        #if defined(MA_X64) || (defined(MA_X86) && ((defined(__GNUC__) && defined(__i586__)) || (defined(_M_IX86) && _M_IX86 >= 500)))\n            #define MA_ATOMIC_IS_LOCK_FREE_64 1\n        #else\n        #endif\n    #else\n    #endif\n    #if defined(MA_ARM32) || defined(MA_ARM64)\n        #define MA_ATOMIC_IS_LOCK_FREE_8  1\n        #define MA_ATOMIC_IS_LOCK_FREE_16 1\n        #define MA_ATOMIC_IS_LOCK_FREE_32 1\n        #if defined(MA_ARM64) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__)\n            #define MA_ATOMIC_IS_LOCK_FREE_64 1\n        #endif\n    #endif\n    #if defined(MA_ATOMIC_PPC32) || defined(MA_ATOMIC_PPC64)\n        #if (defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7))) && !defined(__clang__)\n        #else\n            #define MA_ATOMIC_IS_LOCK_FREE_8  1\n            #define MA_ATOMIC_IS_LOCK_FREE_16 1\n        #endif\n        #define MA_ATOMIC_IS_LOCK_FREE_32     1\n        #if defined(MA_ATOMIC_PPC64)\n            #define MA_ATOMIC_IS_LOCK_FREE_64 1\n        #endif\n    #endif\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_8(volatile void* ptr)\n    {\n        (void)ptr;\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            return 1;\n        #else\n            return 0;\n        #endif\n    }\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_16(volatile void* ptr)\n    {\n        (void)ptr;\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            return 1;\n        #else\n            return 0;\n        #endif\n    }\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_32(volatile void* ptr)\n    {\n        (void)ptr;\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            return 1;\n        #else\n            return 0;\n        #endif\n    }\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_64(volatile void* ptr)\n    {\n        (void)ptr;\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            return 1;\n        #else\n            return 0;\n        #endif\n    }\n#endif\n#define MA_ATOMIC_COMPARE_AND_SWAP_LOCK(sizeInBits, dst, expected, replacement) \\\n    ma_uint##sizeInBits result; \\\n    ma_atomic_spinlock_lock(&ma_atomic_global_lock); \\\n    { \\\n        result = *dst; \\\n        if (result == expected) { \\\n            *dst = replacement; \\\n        } \\\n    } \\\n    ma_atomic_spinlock_unlock(&ma_atomic_global_lock); \\\n    return result\n#define MA_ATOMIC_LOAD_EXPLICIT_LOCK(sizeInBits, ptr, order) \\\n    ma_uint##sizeInBits result; \\\n    ma_atomic_spinlock_lock(&ma_atomic_global_lock); \\\n    { \\\n        result = *ptr; \\\n        (void)order; \\\n    } \\\n    ma_atomic_spinlock_unlock(&ma_atomic_global_lock); \\\n    return result\n#define MA_ATOMIC_STORE_EXPLICIT_LOCK(sizeInBits, dst, src, order) \\\n    ma_atomic_spinlock_lock(&ma_atomic_global_lock); \\\n    { \\\n        *dst = src; \\\n        (void)order; \\\n    } \\\n    ma_atomic_spinlock_unlock(&ma_atomic_global_lock)\n#define MA_ATOMIC_STORE_EXPLICIT_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, src) != oldValue); \\\n    (void)order\n#define MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits result; \\\n    ma_atomic_spinlock_lock(&ma_atomic_global_lock); \\\n    { \\\n        result = *dst; \\\n        *dst = src; \\\n        (void)order; \\\n    } \\\n    ma_atomic_spinlock_unlock(&ma_atomic_global_lock); \\\n    return result\n#define MA_ATOMIC_EXCHANGE_EXPLICIT_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, src) != oldValue); \\\n    (void)order; \\\n    return oldValue\n#define MA_ATOMIC_FETCH_ADD_LOCK(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits result; \\\n    ma_atomic_spinlock_lock(&ma_atomic_global_lock); \\\n    { \\\n        result = *dst; \\\n        *dst += src; \\\n        (void)order; \\\n    } \\\n    ma_atomic_spinlock_unlock(&ma_atomic_global_lock); \\\n    return result\n#define MA_ATOMIC_FETCH_ADD_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    ma_uint##sizeInBits newValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n        newValue = oldValue + src; \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, newValue) != oldValue); \\\n    (void)order; \\\n    return oldValue\n#define MA_ATOMIC_FETCH_AND_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    ma_uint##sizeInBits newValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n        newValue = (ma_uint##sizeInBits)(oldValue & src); \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, newValue) != oldValue); \\\n    (void)order; \\\n    return oldValue\n#define MA_ATOMIC_FETCH_OR_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    ma_uint##sizeInBits newValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n        newValue = (ma_uint##sizeInBits)(oldValue | src); \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, newValue) != oldValue); \\\n    (void)order; \\\n    return oldValue\n#define MA_ATOMIC_FETCH_XOR_CAS(sizeInBits, dst, src, order) \\\n    ma_uint##sizeInBits oldValue; \\\n    ma_uint##sizeInBits newValue; \\\n    do { \\\n        oldValue = ma_atomic_load_explicit_##sizeInBits(dst, ma_atomic_memory_order_relaxed); \\\n        newValue = (ma_uint##sizeInBits)(oldValue ^ src); \\\n    } while (ma_atomic_compare_and_swap_##sizeInBits(dst, oldValue, newValue) != oldValue); \\\n    (void)order; \\\n    return oldValue\n#if defined(MA_ATOMIC_MODERN_MSVC) || defined(MA_ATOMIC_LEGACY_MSVC)\n    #define MA_ATOMIC_MSVC_ARM_INTRINSIC_COMPARE_EXCHANGE(ptr, expected, replacement, order, intrin, ma_atomicType, msvcType)   \\\n        ma_atomicType result; \\\n        switch (order) \\\n        { \\\n            case ma_atomic_memory_order_relaxed: \\\n            { \\\n                result = (ma_atomicType)intrin##_nf((volatile msvcType*)ptr, (msvcType)expected, (msvcType)replacement); \\\n            } break; \\\n            case ma_atomic_memory_order_consume: \\\n            case ma_atomic_memory_order_acquire: \\\n            { \\\n                result = (ma_atomicType)intrin##_acq((volatile msvcType*)ptr, (msvcType)expected, (msvcType)replacement); \\\n            } break; \\\n            case ma_atomic_memory_order_release: \\\n            { \\\n                result = (ma_atomicType)intrin##_rel((volatile msvcType*)ptr, (msvcType)expected, (msvcType)replacement); \\\n            } break; \\\n            case ma_atomic_memory_order_acq_rel: \\\n            case ma_atomic_memory_order_seq_cst: \\\n            default: \\\n            { \\\n                result = (ma_atomicType)intrin((volatile msvcType*)ptr, (msvcType)expected, (msvcType)replacement); \\\n            } break; \\\n        } \\\n        return result;\n    #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        #define ma_atomic_compare_and_swap_8( dst, expected, replacement) (ma_uint8 )_InterlockedCompareExchange8((volatile char*)dst, (char)replacement, (char)expected)\n    #else\n        static MA_INLINE ma_uint8 __stdcall ma_atomic_compare_and_swap_8(volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement)\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(8, dst, expected, replacement);\n        }\n    #endif\n    #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        #define ma_atomic_compare_and_swap_16(dst, expected, replacement) (ma_uint16)_InterlockedCompareExchange16((volatile short*)dst, (short)replacement, (short)expected)\n    #else\n        static MA_INLINE ma_uint16 __stdcall ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement)\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(16, dst, expected, replacement);\n        }\n    #endif\n    #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        #define ma_atomic_compare_and_swap_32(dst, expected, replacement) (ma_uint32)_InterlockedCompareExchange((volatile long*)dst, (long)replacement, (long)expected)\n    #else\n        static MA_INLINE ma_uint32 __stdcall ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement)\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(32, dst, expected, replacement);\n        }\n    #endif\n    #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        #define ma_atomic_compare_and_swap_64(dst, expected, replacement) (ma_uint64)_InterlockedCompareExchange64((volatile ma_int64*)dst, (ma_int64)replacement, (ma_int64)expected)\n    #else\n        static MA_INLINE ma_uint64 __stdcall ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement)\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(64, dst, expected, replacement);\n        }\n    #endif\n    static MA_INLINE ma_uint8 ma_atomic_load_explicit_8(volatile const ma_uint8* ptr, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC_COMPARE_EXCHANGE(ptr, 0, 0, order, _InterlockedCompareExchange8, ma_uint8, char);\n            }\n            #else\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_8((volatile ma_uint8*)ptr, 0, 0);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(8, ptr, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 ma_atomic_load_explicit_16(volatile const ma_uint16* ptr, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC_COMPARE_EXCHANGE(ptr, 0, 0, order, _InterlockedCompareExchange16, ma_uint16, short);\n            }\n            #else\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_16((volatile ma_uint16*)ptr, 0, 0);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(16, ptr, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 ma_atomic_load_explicit_32(volatile const ma_uint32* ptr, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC_COMPARE_EXCHANGE(ptr, 0, 0, order, _InterlockedCompareExchange, ma_uint32, long);\n            }\n            #else\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_32((volatile ma_uint32*)ptr, 0, 0);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(32, ptr, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 ma_atomic_load_explicit_64(volatile const ma_uint64* ptr, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC_COMPARE_EXCHANGE(ptr, 0, 0, order, _InterlockedCompareExchange64, ma_uint64, long long);\n            }\n            #else\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_64((volatile ma_uint64*)ptr, 0, 0);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(64, ptr, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_exchange_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchange8, ma_uint8, char);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint8)_InterlockedExchange8((volatile char*)dst, (char)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_exchange_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchange16, ma_uint16, short);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint16)_InterlockedExchange16((volatile short*)dst, (short)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_exchange_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchange, ma_uint32, long);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint32)_InterlockedExchange((volatile long*)dst, (long)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_exchange_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            #if defined(MA_32BIT)\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_CAS(64, dst, src, order);\n            }\n            #else\n            {\n                #if defined(MA_ARM)\n                {\n                    MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchange64, ma_uint64, long long);\n                }\n                #else\n                {\n                    (void)order;\n                    return (ma_uint64)_InterlockedExchange64((volatile long long*)dst, (long long)src);\n                }\n                #endif\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_add_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchangeAdd8, ma_uint8, char);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint8)_InterlockedExchangeAdd8((volatile char*)dst, (char)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_add_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchangeAdd16, ma_uint16, short);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint16)_InterlockedExchangeAdd16((volatile short*)dst, (short)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_add_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            #if defined(MA_ARM)\n            {\n                MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchangeAdd, ma_uint32, long);\n            }\n            #else\n            {\n                (void)order;\n                return (ma_uint32)_InterlockedExchangeAdd((volatile long*)dst, (long)src);\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_add_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            #if defined(MA_32BIT)\n            {\n                MA_ATOMIC_FETCH_ADD_CAS(64, dst, src, order);\n            }\n            #else\n            {\n                #if defined(MA_ARM)\n                {\n                    MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedExchangeAdd64, ma_uint64, long long);\n                }\n                #else\n                {\n                    (void)order;\n                    return (ma_uint64)_InterlockedExchangeAdd64((volatile long long*)dst, (long long)src);\n                }\n                #endif\n            }\n            #endif\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(-(ma_int8)src), order);\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(-(ma_int16)src), order);\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(-(ma_int32)src), order);\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(-(ma_int64)src), order);\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedAnd8, ma_uint8, char);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_AND_CAS(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_and_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedAnd16, ma_uint16, short);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_AND_CAS(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_and_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedAnd, ma_uint32, long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_AND_CAS(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_and_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedAnd64, ma_uint64, long long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_AND_CAS(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_or_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedOr8, ma_uint8, char);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_OR_CAS(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_or_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedOr16, ma_uint16, short);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_OR_CAS(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_or_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedOr, ma_uint32, long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_OR_CAS(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_or_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedOr64, ma_uint64, long long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_OR_CAS(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_xor_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedXor8, ma_uint8, char);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_xor_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedXor16, ma_uint16, short);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_xor_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedXor, ma_uint32, long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_xor_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ARM)\n        {\n            MA_ATOMIC_MSVC_ARM_INTRINSIC(dst, src, order, _InterlockedXor64, ma_uint64, long long);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(64, dst, src, order);\n        }\n        #endif\n    }\n    #define ma_atomic_store_explicit_8( dst, src, order) (void)ma_atomic_exchange_explicit_8 (dst, src, order)\n    #define ma_atomic_store_explicit_16(dst, src, order) (void)ma_atomic_exchange_explicit_16(dst, src, order)\n    #define ma_atomic_store_explicit_32(dst, src, order) (void)ma_atomic_exchange_explicit_32(dst, src, order)\n    #define ma_atomic_store_explicit_64(dst, src, order) (void)ma_atomic_exchange_explicit_64(dst, src, order)\n    #if defined(MA_X64)\n        #define ma_atomic_thread_fence(order)   __faststorefence(), (void)order\n    #elif defined(MA_ARM64)\n        #define ma_atomic_thread_fence(order)   __dmb(_ARM64_BARRIER_ISH), (void)order\n    #else\n        static MA_INLINE void ma_atomic_thread_fence(ma_atomic_memory_order order)\n        {\n            volatile ma_uint32 barrier = 0;\n            ma_atomic_fetch_add_explicit_32(&barrier, 0, order);\n        }\n    #endif\n    #define ma_atomic_signal_fence(order)   _ReadWriteBarrier(), (void)order\n#endif\n#if defined(MA_ATOMIC_LEGACY_MSVC_ASM)\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_compare_and_swap_8(volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            ma_uint8 result = 0;\n            __asm {\n                mov ecx, dst\n                mov al,  expected\n                mov dl,  replacement\n                lock cmpxchg [ecx], dl\n                mov result, al\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(8, dst, expected, replacement);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            ma_uint16 result = 0;\n            __asm {\n                mov ecx, dst\n                mov ax,  expected\n                mov dx,  replacement\n                lock cmpxchg [ecx], dx\n                mov result, ax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(16, dst, expected, replacement);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            ma_uint32 result = 0;\n            __asm {\n                mov ecx, dst\n                mov eax, expected\n                mov edx, replacement\n                lock cmpxchg [ecx], edx\n                mov result, eax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(32, dst, expected, replacement);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            ma_uint32 resultEAX = 0;\n            ma_uint32 resultEDX = 0;\n            __asm {\n                mov esi, dst\n                mov eax, dword ptr expected\n                mov edx, dword ptr expected + 4\n                mov ebx, dword ptr replacement\n                mov ecx, dword ptr replacement + 4\n                lock cmpxchg8b qword ptr [esi]\n                mov resultEAX, eax\n                mov resultEDX, edx\n            }\n            return ((ma_uint64)resultEDX << 32) | resultEAX;\n        }\n        #else\n        {\n            MA_ATOMIC_COMPARE_AND_SWAP_LOCK(64, dst, expected, replacement);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 ma_atomic_load_explicit_8(volatile const ma_uint8* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            ma_uint8 result = 0;\n            if (order == ma_atomic_memory_order_relaxed) {\n                __asm {\n                    mov esi, dst\n                    mov al, [esi]\n                    mov result, al\n                }\n            } else if (order <= ma_atomic_memory_order_release) {\n                __asm {\n                    mov esi, dst\n                    mov al, [esi]\n                    lock add dword ptr [esp], 0\n                    mov result, al\n                }\n            } else {\n                __asm {\n                    lock add dword ptr [esp], 0\n                    mov esi, dst\n                    mov al, [esi]\n                    mov result, al\n                    lock add dword ptr [esp], 0\n                }\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(8, dst, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 ma_atomic_load_explicit_16(volatile const ma_uint16* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            ma_uint16 result = 0;\n            if (order == ma_atomic_memory_order_relaxed) {\n                __asm {\n                    mov esi, dst\n                    mov ax, [esi]\n                    mov result, ax\n                }\n            } else if (order <= ma_atomic_memory_order_release) {\n                __asm {\n                    mov esi, dst\n                    mov ax, [esi]\n                    lock add dword ptr [esp], 0\n                    mov result, ax\n                }\n            } else {\n                __asm {\n                    lock add dword ptr [esp], 0\n                    mov esi, dst\n                    mov ax, [esi]\n                    mov result, ax\n                    lock add dword ptr [esp], 0\n                }\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(16, dst, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 ma_atomic_load_explicit_32(volatile const ma_uint32* dst, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            ma_uint32 result = 0;\n            if (order == ma_atomic_memory_order_relaxed) {\n                __asm {\n                    mov esi, dst\n                    mov eax, [esi]\n                    mov result, eax\n                }\n            } else if (order <= ma_atomic_memory_order_release) {\n                __asm {\n                    mov esi, dst\n                    mov eax, [esi]\n                    lock add dword ptr [esp], 0\n                    mov result, eax\n                }\n            } else {\n                __asm {\n                    lock add dword ptr [esp], 0\n                    mov esi, dst\n                    mov eax, [esi]\n                    mov result, eax\n                    lock add dword ptr [esp], 0\n                }\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_LOAD_EXPLICIT_LOCK(32, dst, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 ma_atomic_load_explicit_64(volatile const ma_uint64* dst, ma_atomic_memory_order order)\n    {\n        (void)order;\n        return ma_atomic_compare_and_swap_64((volatile ma_uint64*)dst, 0, 0);\n    }\n    static MA_INLINE void __stdcall ma_atomic_store_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        if (order == ma_atomic_memory_order_relaxed) {\n            __asm {\n                mov esi, dst\n                mov al, src\n                mov [esi], al\n            }\n        } else {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                __asm {\n                    mov esi, dst\n                    mov al, src\n                    xchg [esi], al\n                }\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n    }\n    static MA_INLINE void __stdcall ma_atomic_store_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        if (order == ma_atomic_memory_order_relaxed) {\n            __asm {\n                mov esi, dst\n                mov ax, src\n                mov [esi], ax\n            }\n        } else {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                __asm {\n                    mov esi, dst\n                    mov ax, src\n                    xchg [esi], ax\n                }\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n    }\n    static MA_INLINE void __stdcall ma_atomic_store_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        if (order == ma_atomic_memory_order_relaxed) {\n            __asm {\n                mov esi, dst\n                mov eax, src\n                mov [esi], eax\n            }\n        } else {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                __asm {\n                    mov esi, dst\n                    mov eax, src\n                    xchg [esi], eax\n                }\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n    }\n    static MA_INLINE void __stdcall ma_atomic_store_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            MA_ATOMIC_STORE_EXPLICIT_CAS(64, dst, src, order);\n        }\n        #else\n        {\n            MA_ATOMIC_STORE_EXPLICIT_LOCK(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_exchange_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            ma_uint8 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov al,  src\n                lock xchg [ecx], al\n                mov result, al\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_exchange_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            ma_uint16 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov ax,  src\n                lock xchg [ecx], ax\n                mov result, ax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_exchange_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            ma_uint32 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov eax, src\n                xchg [ecx], eax\n                mov result, eax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_exchange_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_CAS(64, dst, src, order);\n        }\n        #else\n        {\n            MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_add_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            ma_uint8 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov al,  src\n                lock xadd [ecx], al\n                mov result, al\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(8, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_add_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            ma_uint16 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov ax,  src\n                lock xadd [ecx], ax\n                mov result, ax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(16, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_add_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            ma_uint32 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov eax, src\n                lock xadd [ecx], eax\n                mov result, eax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(32, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_add_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n        {\n            MA_ATOMIC_FETCH_ADD_CAS(64, dst, src, order);\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(64, dst, src, order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n        {\n            ma_uint8 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov al,  src\n                neg al\n                lock xadd [ecx], al\n                mov result, al\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(-(ma_int8)src), order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n        {\n            ma_uint16 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov ax,  src\n                neg ax\n                lock xadd [ecx], ax\n                mov result, ax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(-(ma_int16)src), order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n        {\n            ma_uint32 result = 0;\n            (void)order;\n            __asm {\n                mov ecx, dst\n                mov eax, src\n                neg eax\n                lock xadd [ecx], eax\n                mov result, eax\n            }\n            return result;\n        }\n        #else\n        {\n            MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(-(ma_int32)src), order);\n        }\n        #endif\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_ADD_CAS(64, dst, (ma_uint64)(-(ma_int64)src), order);\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_AND_CAS(8, dst, src, order);\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_and_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_AND_CAS(16, dst, src, order);\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_and_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_AND_CAS(32, dst, src, order);\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_and_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_AND_CAS(64, dst, src, order);\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_or_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_OR_CAS(8, dst, src, order);\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_or_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_OR_CAS(16, dst, src, order);\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_or_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_OR_CAS(32, dst, src, order);\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_or_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_OR_CAS(64, dst, src, order);\n    }\n    static MA_INLINE ma_uint8 __stdcall ma_atomic_fetch_xor_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_XOR_CAS(8, dst, src, order);\n    }\n    static MA_INLINE ma_uint16 __stdcall ma_atomic_fetch_xor_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_XOR_CAS(16, dst, src, order);\n    }\n    static MA_INLINE ma_uint32 __stdcall ma_atomic_fetch_xor_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_XOR_CAS(32, dst, src, order);\n    }\n    static MA_INLINE ma_uint64 __stdcall ma_atomic_fetch_xor_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n    {\n        MA_ATOMIC_FETCH_XOR_CAS(64, dst, src, order);\n    }\n    static MA_INLINE void __stdcall ma_atomic_thread_fence(ma_atomic_memory_order order)\n    {\n        (void)order;\n        __asm {\n            lock add dword ptr [esp], 0\n        }\n    }\n    #define ma_atomic_signal_fence(order) __asm {}; (void)order\n#endif\n#if defined(MA_ATOMIC_MODERN_GCC)\n    #define MA_ATOMIC_HAS_NATIVE_COMPARE_EXCHANGE\n    #define ma_atomic_thread_fence(order)                           __atomic_thread_fence(order)\n    #define ma_atomic_signal_fence(order)                           __atomic_signal_fence(order)\n    #define ma_atomic_is_lock_free_8(ptr)                           __atomic_is_lock_free(1, ptr)\n    #define ma_atomic_is_lock_free_16(ptr)                          __atomic_is_lock_free(2, ptr)\n    #define ma_atomic_is_lock_free_32(ptr)                          __atomic_is_lock_free(4, ptr)\n    #define ma_atomic_is_lock_free_64(ptr)                          __atomic_is_lock_free(8, ptr)\n    #define ma_atomic_store_explicit_8( dst, src, order)            __atomic_store_n(dst, src, order)\n    #define ma_atomic_store_explicit_16(dst, src, order)            __atomic_store_n(dst, src, order)\n    #define ma_atomic_store_explicit_32(dst, src, order)            __atomic_store_n(dst, src, order)\n    #define ma_atomic_store_explicit_64(dst, src, order)            __atomic_store_n(dst, src, order)\n    #define ma_atomic_load_explicit_8( dst, order)                  __atomic_load_n(dst, order)\n    #define ma_atomic_load_explicit_16(dst, order)                  __atomic_load_n(dst, order)\n    #define ma_atomic_load_explicit_32(dst, order)                  __atomic_load_n(dst, order)\n    #define ma_atomic_load_explicit_64(dst, order)                  __atomic_load_n(dst, order)\n    #define ma_atomic_exchange_explicit_8( dst, src, order)         __atomic_exchange_n(dst, src, order)\n    #define ma_atomic_exchange_explicit_16(dst, src, order)         __atomic_exchange_n(dst, src, order)\n    #define ma_atomic_exchange_explicit_32(dst, src, order)         __atomic_exchange_n(dst, src, order)\n    #define ma_atomic_exchange_explicit_64(dst, src, order)         __atomic_exchange_n(dst, src, order)\n    #define ma_atomic_compare_exchange_strong_explicit_8( dst, expected, replacement, successOrder, failureOrder)   __atomic_compare_exchange_n(dst, expected, replacement, 0, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_strong_explicit_16(dst, expected, replacement, successOrder, failureOrder)   __atomic_compare_exchange_n(dst, expected, replacement, 0, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_strong_explicit_32(dst, expected, replacement, successOrder, failureOrder)   __atomic_compare_exchange_n(dst, expected, replacement, 0, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_strong_explicit_64(dst, expected, replacement, successOrder, failureOrder)   __atomic_compare_exchange_n(dst, expected, replacement, 0, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_8( dst, expected, replacement, successOrder, failureOrder)     __atomic_compare_exchange_n(dst, expected, replacement, 1, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_16(dst, expected, replacement, successOrder, failureOrder)     __atomic_compare_exchange_n(dst, expected, replacement, 1, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_32(dst, expected, replacement, successOrder, failureOrder)     __atomic_compare_exchange_n(dst, expected, replacement, 1, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_64(dst, expected, replacement, successOrder, failureOrder)     __atomic_compare_exchange_n(dst, expected, replacement, 1, successOrder, failureOrder)\n    #define ma_atomic_fetch_add_explicit_8( dst, src, order)        __atomic_fetch_add(dst, src, order)\n    #define ma_atomic_fetch_add_explicit_16(dst, src, order)        __atomic_fetch_add(dst, src, order)\n    #define ma_atomic_fetch_add_explicit_32(dst, src, order)        __atomic_fetch_add(dst, src, order)\n    #define ma_atomic_fetch_add_explicit_64(dst, src, order)        __atomic_fetch_add(dst, src, order)\n    #define ma_atomic_fetch_sub_explicit_8( dst, src, order)        __atomic_fetch_sub(dst, src, order)\n    #define ma_atomic_fetch_sub_explicit_16(dst, src, order)        __atomic_fetch_sub(dst, src, order)\n    #define ma_atomic_fetch_sub_explicit_32(dst, src, order)        __atomic_fetch_sub(dst, src, order)\n    #define ma_atomic_fetch_sub_explicit_64(dst, src, order)        __atomic_fetch_sub(dst, src, order)\n    #define ma_atomic_fetch_or_explicit_8( dst, src, order)         __atomic_fetch_or(dst, src, order)\n    #define ma_atomic_fetch_or_explicit_16(dst, src, order)         __atomic_fetch_or(dst, src, order)\n    #define ma_atomic_fetch_or_explicit_32(dst, src, order)         __atomic_fetch_or(dst, src, order)\n    #define ma_atomic_fetch_or_explicit_64(dst, src, order)         __atomic_fetch_or(dst, src, order)\n    #define ma_atomic_fetch_xor_explicit_8( dst, src, order)        __atomic_fetch_xor(dst, src, order)\n    #define ma_atomic_fetch_xor_explicit_16(dst, src, order)        __atomic_fetch_xor(dst, src, order)\n    #define ma_atomic_fetch_xor_explicit_32(dst, src, order)        __atomic_fetch_xor(dst, src, order)\n    #define ma_atomic_fetch_xor_explicit_64(dst, src, order)        __atomic_fetch_xor(dst, src, order)\n    #define ma_atomic_fetch_and_explicit_8( dst, src, order)        __atomic_fetch_and(dst, src, order)\n    #define ma_atomic_fetch_and_explicit_16(dst, src, order)        __atomic_fetch_and(dst, src, order)\n    #define ma_atomic_fetch_and_explicit_32(dst, src, order)        __atomic_fetch_and(dst, src, order)\n    #define ma_atomic_fetch_and_explicit_64(dst, src, order)        __atomic_fetch_and(dst, src, order)\n    static MA_INLINE ma_uint8 ma_atomic_compare_and_swap_8(volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement)\n    {\n        __atomic_compare_exchange_n(dst, &expected, replacement, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);\n        return expected;\n    }\n    static MA_INLINE ma_uint16 ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement)\n    {\n        __atomic_compare_exchange_n(dst, &expected, replacement, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);\n        return expected;\n    }\n    static MA_INLINE ma_uint32 ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement)\n    {\n        __atomic_compare_exchange_n(dst, &expected, replacement, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);\n        return expected;\n    }\n    #if defined(__clang__)\n        #pragma clang diagnostic push\n        #if __clang_major__ >= 8\n            #pragma clang diagnostic ignored \"-Watomic-alignment\"\n        #endif\n    #endif\n    static MA_INLINE ma_uint64 ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement)\n    {\n        __atomic_compare_exchange_n(dst, &expected, replacement, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);\n        return expected;\n    }\n    #if defined(__clang__)\n        #pragma clang diagnostic pop\n    #endif\n#endif\n#if defined(MA_ATOMIC_LEGACY_GCC) || defined(MA_ATOMIC_LEGACY_GCC_ASM)\n    #define ma_atomic_signal_fence(order)   __asm__ __volatile__(\"\":::\"memory\")\n    #if defined(MA_ATOMIC_LEGACY_GCC)\n        #define ma_atomic_thread_fence(order) __sync_synchronize(), (void)order\n        static MA_INLINE ma_uint8 ma_atomic_compare_and_swap_8(volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                return __sync_val_compare_and_swap(dst, expected, replacement);\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(8, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                return __sync_val_compare_and_swap(dst, expected, replacement);\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(16, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                return __sync_val_compare_and_swap(dst, expected, replacement);\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(32, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                return __sync_val_compare_and_swap(dst, expected, replacement);\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(64, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_load_explicit_8(volatile const ma_uint8* ptr, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_8((ma_uint8*)ptr, 0, 0);\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(8, ptr, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_load_explicit_16(volatile const ma_uint16* ptr, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_16((ma_uint16*)ptr, 0, 0);\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(16, ptr, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_load_explicit_32(volatile const ma_uint32* ptr, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_32((ma_uint32*)ptr, 0, 0);\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(32, ptr, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_load_explicit_64(volatile const ma_uint64* ptr, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return ma_atomic_compare_and_swap_64((ma_uint64*)ptr, 0, 0);\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(64, ptr, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_exchange_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                if (order > ma_atomic_memory_order_acquire) {\n                    __sync_synchronize();\n                }\n                return __sync_lock_test_and_set(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_exchange_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                if (order > ma_atomic_memory_order_acquire) {\n                    __sync_synchronize();\n                }\n                return __sync_lock_test_and_set(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_exchange_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                if (order > ma_atomic_memory_order_acquire) {\n                    __sync_synchronize();\n                }\n                return __sync_lock_test_and_set(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_exchange_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                if (order > ma_atomic_memory_order_acquire) {\n                    __sync_synchronize();\n                }\n                return __sync_lock_test_and_set(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(64, dst, src, order);\n            }\n            #endif\n        }\n        #define ma_atomic_store_explicit_8( dst, src, order) (void)ma_atomic_exchange_explicit_8 (dst, src, order)\n        #define ma_atomic_store_explicit_16(dst, src, order) (void)ma_atomic_exchange_explicit_16(dst, src, order)\n        #define ma_atomic_store_explicit_32(dst, src, order) (void)ma_atomic_exchange_explicit_32(dst, src, order)\n        #define ma_atomic_store_explicit_64(dst, src, order) (void)ma_atomic_exchange_explicit_64(dst, src, order)\n        static MA_INLINE ma_uint8 ma_atomic_fetch_add_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return __sync_fetch_and_add(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_add_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return __sync_fetch_and_add(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_add_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return __sync_fetch_and_add(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_add_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return __sync_fetch_and_add(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return __sync_fetch_and_sub(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(8, dst, (ma_uint8)(-(ma_int8)src), order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return __sync_fetch_and_sub(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(16, dst, (ma_uint16)(-(ma_int16)src), order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return __sync_fetch_and_sub(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(32, dst, (ma_uint32)(-(ma_int32)src), order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return __sync_fetch_and_sub(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(64, dst, (ma_uint64)(-(ma_int64)src), order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return __sync_fetch_and_and(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_AND_CAS(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_and_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return __sync_fetch_and_and(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_AND_CAS(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_and_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return __sync_fetch_and_and(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_AND_CAS(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_and_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return __sync_fetch_and_and(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_AND_CAS(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_or_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return __sync_fetch_and_or(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_OR_CAS(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_or_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return __sync_fetch_and_or(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_OR_CAS(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_or_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return __sync_fetch_and_or(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_OR_CAS(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_or_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return __sync_fetch_and_or(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_OR_CAS(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_xor_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8)\n            {\n                (void)order;\n                return __sync_fetch_and_xor(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_XOR_CAS(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_xor_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16)\n            {\n                (void)order;\n                return __sync_fetch_and_xor(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_XOR_CAS(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_xor_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32)\n            {\n                (void)order;\n                return __sync_fetch_and_xor(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_XOR_CAS(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_xor_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64)\n            {\n                (void)order;\n                return __sync_fetch_and_xor(dst, src);\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_XOR_CAS(64, dst, src, order);\n            }\n            #endif\n        }\n    #elif defined(MA_ATOMIC_LEGACY_GCC_ASM)\n        #define MA_ATOMIC_CMPXCHG_GCC_X86(instructionSizeSuffix, result, dst, expected, replacement) \\\n            __asm__ __volatile__(                             \\\n                \"lock; cmpxchg\"instructionSizeSuffix\" %2, %1\" \\\n                : \"=a\"(result),                       \\\n                  \"=m\"(*dst)                          \\\n                : \"r\"(replacement),                   \\\n                  \"0\"(expected),                      \\\n                  \"m\"(*dst)                           \\\n                : \"cc\", \"memory\")\n        #define MA_ATOMIC_XADD_GCC_X86(instructionSizeSuffix, result, dst, src) \\\n            __asm__ __volatile__(        \\\n                \"lock; xadd\"instructionSizeSuffix\" %0, %1\"      \\\n                : \"=a\"(result),  \\\n                  \"=m\"(*dst)     \\\n                : \"0\"(src),      \\\n                  \"m\"(*dst)      \\\n                : \"cc\", \"memory\")\n        static MA_INLINE ma_uint8 ma_atomic_compare_and_swap_8(volatile ma_uint8* dst, ma_uint8 expected, ma_uint8 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint8 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_CMPXCHG_GCC_X86(\"b\", result, dst, expected, replacement);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(8, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_compare_and_swap_16(volatile ma_uint16* dst, ma_uint16 expected, ma_uint16 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint16 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_CMPXCHG_GCC_X86(\"w\", result, dst, expected, replacement);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(16, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_compare_and_swap_32(volatile ma_uint32* dst, ma_uint32 expected, ma_uint32 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint32 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_CMPXCHG_GCC_X86(\"l\", result, dst, expected, replacement);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(32, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_compare_and_swap_64(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 replacement)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint64 result;\n                #if defined(MA_X86)\n                {\n                    ma_uint32 resultEAX;\n                    ma_uint32 resultEDX;\n                    __asm__ __volatile__(\n                        \"pushl %%ebx\\n\"\n                        \"movl  %4, %%ebx\\n\"\n                        \"lock  cmpxchg8b (%%edi)\\n\"\n                        \"popl  %%ebx\\n\"\n                        : \"=a\"(resultEAX),\n                          \"=d\"(resultEDX)\n                        : \"a\"((ma_uint32)(expected & 0xFFFFFFFF)),\n                          \"d\"((ma_uint32)(expected >> 32)),\n                          \"r\"((ma_uint32)(replacement & 0xFFFFFFFF)),\n                          \"c\"((ma_uint32)(replacement >> 32)),\n                          \"D\"(dst)\n                        : \"memory\", \"cc\");\n                    result = ((ma_uint64)resultEDX << 32) | resultEAX;\n                }\n                #elif defined(MA_X64)\n                {\n                    MA_ATOMIC_CMPXCHG_GCC_X86(\"q\", result, dst, expected, replacement);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_COMPARE_AND_SWAP_LOCK(64, dst, expected, replacement);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_load_explicit_8(volatile const ma_uint8* dst, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint8 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        MA_ATOMIC_LOAD_RELAXED_GCC_X86(\"b\", result, dst);\n                    } else if (order <= ma_atomic_memory_order_release) {\n                        MA_ATOMIC_LOAD_RELEASE_GCC_X86(\"b\", result, dst);\n                    } else {\n                        MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(\"b\", result, dst);\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(8, dst, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_load_explicit_16(volatile const ma_uint16* dst, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint16 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        MA_ATOMIC_LOAD_RELAXED_GCC_X86(\"w\", result, dst);\n                    } else if (order <= ma_atomic_memory_order_release) {\n                        MA_ATOMIC_LOAD_RELEASE_GCC_X86(\"w\", result, dst);\n                    } else {\n                        MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(\"w\", result, dst);\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(16, dst, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_load_explicit_32(volatile const ma_uint32* dst, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint32 result;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        MA_ATOMIC_LOAD_RELAXED_GCC_X86(\"l\", result, dst);\n                    } else if (order <= ma_atomic_memory_order_release) {\n                        MA_ATOMIC_LOAD_RELEASE_GCC_X86(\"l\", result, dst);\n                    } else {\n                        MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(\"l\", result, dst);\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(32, dst, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_load_explicit_64(volatile const ma_uint64* dst, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint64 result;\n                #if defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        MA_ATOMIC_LOAD_RELAXED_GCC_X86(\"q\", result, dst);\n                    } else if (order <= ma_atomic_memory_order_release) {\n                        MA_ATOMIC_LOAD_RELEASE_GCC_X86(\"q\", result, dst);\n                    } else {\n                        MA_ATOMIC_LOAD_SEQ_CST_GCC_X86(\"q\", result, dst);\n                    }\n                }\n                #elif defined(MA_X86)\n                {\n                    (void)order;\n                    return ma_atomic_compare_and_swap_64((volatile ma_uint64*)dst, 0, 0);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_LOAD_EXPLICIT_LOCK(64, dst, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_exchange_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint8 result;\n                (void)order;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_XCHG_GCC_X86(\"b\", result, dst, src);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_exchange_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint16 result;\n                (void)order;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_XCHG_GCC_X86(\"w\", result, dst, src);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_exchange_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint32 result;\n                (void)order;\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    MA_ATOMIC_XCHG_GCC_X86(\"l\", result, dst, src);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_exchange_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64) && (defined(MA_X86) || defined(MA_X64))\n            {\n                ma_uint64 result;\n                (void)order;\n                #if defined(MA_X86)\n                {\n                    MA_ATOMIC_EXCHANGE_EXPLICIT_CAS(64, dst, src, order);\n                }\n                #elif defined(MA_X64)\n                {\n                    MA_ATOMIC_XCHG_GCC_X86(\"q\", result, dst, src);\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n                return result;\n            }\n            #else\n            {\n                MA_ATOMIC_EXCHANGE_EXPLICIT_LOCK(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE void ma_atomic_store_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        __asm__ __volatile__ (\n                            \"movb %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                        );\n                    } else {\n                        __asm__ __volatile__ (\n                            \"xchgb %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                            : \"memory\"\n                        );\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE void ma_atomic_store_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        __asm__ __volatile__ (\n                            \"movw %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                        );\n                    } else {\n                        __asm__ __volatile__ (\n                            \"xchgw %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                            : \"memory\"\n                        );\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE void ma_atomic_store_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        __asm__ __volatile__ (\n                            \"movl %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                        );\n                    } else {\n                        __asm__ __volatile__ (\n                            \"xchgl %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                            : \"memory\"\n                        );\n                    }\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE void ma_atomic_store_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X64)\n                {\n                    if (order == ma_atomic_memory_order_relaxed) {\n                        __asm__ __volatile__ (\n                            \"movq %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                        );\n                    } else {\n                        __asm__ __volatile__ (\n                            \"xchgq %1, %0\"\n                            : \"=m\"(*dst)\n                            : \"r\"(src)\n                            : \"memory\"\n                        );\n                    }\n                }\n                #else\n                {\n                    MA_ATOMIC_STORE_EXPLICIT_CAS(64, dst, src, order);\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_STORE_EXPLICIT_LOCK(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_add_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_8) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    ma_uint8 result;\n                    (void)order;\n                    MA_ATOMIC_XADD_GCC_X86(\"b\", result, dst, src);\n                    return result;\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(8, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_add_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_16) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    ma_uint16 result;\n                    (void)order;\n                    MA_ATOMIC_XADD_GCC_X86(\"w\", result, dst, src);\n                    return result;\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(16, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_add_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_32) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86) || defined(MA_X64)\n                {\n                    ma_uint32 result;\n                    (void)order;\n                    MA_ATOMIC_XADD_GCC_X86(\"l\", result, dst, src);\n                    return result;\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(32, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_add_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            #if defined(MA_ATOMIC_IS_LOCK_FREE_64) && (defined(MA_X86) || defined(MA_X64))\n            {\n                #if defined(MA_X86)\n                {\n                    MA_ATOMIC_FETCH_ADD_CAS(64, dst, src, order);\n                }\n                #elif defined(MA_X64)\n                {\n                    ma_uint64 result;\n                    MA_ATOMIC_XADD_GCC_X86(\"q\", result, dst, src);\n                    (void)order;\n                    return result;\n                }\n                #else\n                {\n                    #error Unsupported architecture.\n                }\n                #endif\n            }\n            #else\n            {\n                MA_ATOMIC_FETCH_ADD_LOCK(64, dst, src, order);\n            }\n            #endif\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_sub_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            return ma_atomic_fetch_add_explicit_8(dst, (ma_uint8)(-(ma_int8)src), order);\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_sub_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            return ma_atomic_fetch_add_explicit_16(dst, (ma_uint16)(-(ma_int16)src), order);\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_sub_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            return ma_atomic_fetch_add_explicit_32(dst, (ma_uint32)(-(ma_int32)src), order);\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_sub_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            return ma_atomic_fetch_add_explicit_64(dst, (ma_uint64)(-(ma_int64)src), order);\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_and_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_AND_CAS(8, dst, src, order);\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_and_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_AND_CAS(16, dst, src, order);\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_and_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_AND_CAS(32, dst, src, order);\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_and_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_AND_CAS(64, dst, src, order);\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_or_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_OR_CAS(8, dst, src, order);\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_or_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_OR_CAS(16, dst, src, order);\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_or_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_OR_CAS(32, dst, src, order);\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_or_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_OR_CAS(64, dst, src, order);\n        }\n        static MA_INLINE ma_uint8 ma_atomic_fetch_xor_explicit_8(volatile ma_uint8* dst, ma_uint8 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(8, dst, src, order);\n        }\n        static MA_INLINE ma_uint16 ma_atomic_fetch_xor_explicit_16(volatile ma_uint16* dst, ma_uint16 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(16, dst, src, order);\n        }\n        static MA_INLINE ma_uint32 ma_atomic_fetch_xor_explicit_32(volatile ma_uint32* dst, ma_uint32 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(32, dst, src, order);\n        }\n        static MA_INLINE ma_uint64 ma_atomic_fetch_xor_explicit_64(volatile ma_uint64* dst, ma_uint64 src, ma_atomic_memory_order order)\n        {\n            MA_ATOMIC_FETCH_XOR_CAS(64, dst, src, order);\n        }\n    #else\n        #error Unsupported compiler.\n    #endif\n#endif\n#if !defined(MA_ATOMIC_HAS_NATIVE_COMPARE_EXCHANGE)\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_8(volatile ma_uint8* dst, ma_uint8* expected, ma_uint8 replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        ma_uint8 result;\n        (void)successOrder;\n        (void)failureOrder;\n        result = ma_atomic_compare_and_swap_8(dst, *expected, replacement);\n        if (result == *expected) {\n            return 1;\n        } else {\n            *expected = result;\n            return 0;\n        }\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_16(volatile ma_uint16* dst, ma_uint16* expected, ma_uint16 replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        ma_uint16 result;\n        (void)successOrder;\n        (void)failureOrder;\n        result = ma_atomic_compare_and_swap_16(dst, *expected, replacement);\n        if (result == *expected) {\n            return 1;\n        } else {\n            *expected = result;\n            return 0;\n        }\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_32(volatile ma_uint32* dst, ma_uint32* expected, ma_uint32 replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        ma_uint32 result;\n        (void)successOrder;\n        (void)failureOrder;\n        result = ma_atomic_compare_and_swap_32(dst, *expected, replacement);\n        if (result == *expected) {\n            return 1;\n        } else {\n            *expected = result;\n            return 0;\n        }\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_64(volatile ma_uint64* dst, volatile ma_uint64* expected, ma_uint64 replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        ma_uint64 result;\n        (void)successOrder;\n        (void)failureOrder;\n        result = ma_atomic_compare_and_swap_64(dst, *expected, replacement);\n        if (result == *expected) {\n            return 1;\n        } else {\n            *expected = result;\n            return 0;\n        }\n    }\n    #define ma_atomic_compare_exchange_weak_explicit_8( dst, expected, replacement, successOrder, failureOrder) ma_atomic_compare_exchange_strong_explicit_8 (dst, expected, replacement, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_16(dst, expected, replacement, successOrder, failureOrder) ma_atomic_compare_exchange_strong_explicit_16(dst, expected, replacement, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_32(dst, expected, replacement, successOrder, failureOrder) ma_atomic_compare_exchange_strong_explicit_32(dst, expected, replacement, successOrder, failureOrder)\n    #define ma_atomic_compare_exchange_weak_explicit_64(dst, expected, replacement, successOrder, failureOrder) ma_atomic_compare_exchange_strong_explicit_64(dst, expected, replacement, successOrder, failureOrder)\n#endif\n#if defined(MA_64BIT)\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(volatile void** ptr)\n    {\n        return ma_atomic_is_lock_free_64((volatile ma_uint64*)ptr);\n    }\n    static MA_INLINE void* ma_atomic_load_explicit_ptr(volatile void** ptr, ma_atomic_memory_order order)\n    {\n        return (void*)ma_atomic_load_explicit_64((volatile ma_uint64*)ptr, order);\n    }\n    static MA_INLINE void ma_atomic_store_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order)\n    {\n        ma_atomic_store_explicit_64((volatile ma_uint64*)dst, (ma_uint64)src, order);\n    }\n    static MA_INLINE void* ma_atomic_exchange_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order)\n    {\n        return (void*)ma_atomic_exchange_explicit_64((volatile ma_uint64*)dst, (ma_uint64)src, order);\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        return ma_atomic_compare_exchange_strong_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder);\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        return ma_atomic_compare_exchange_weak_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder);\n    }\n    static MA_INLINE void* ma_atomic_compare_and_swap_ptr(volatile void** dst, void* expected, void* replacement)\n    {\n        return (void*)ma_atomic_compare_and_swap_64((volatile ma_uint64*)dst, (ma_uint64)expected, (ma_uint64)replacement);\n    }\n#elif defined(MA_32BIT)\n    static MA_INLINE ma_bool32 ma_atomic_is_lock_free_ptr(volatile void** ptr)\n    {\n        return ma_atomic_is_lock_free_32((volatile ma_uint32*)ptr);\n    }\n    static MA_INLINE void* ma_atomic_load_explicit_ptr(volatile void** ptr, ma_atomic_memory_order order)\n    {\n        return (void*)ma_atomic_load_explicit_32((volatile ma_uint32*)ptr, order);\n    }\n    static MA_INLINE void ma_atomic_store_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order)\n    {\n        ma_atomic_store_explicit_32((volatile ma_uint32*)dst, (ma_uint32)src, order);\n    }\n    static MA_INLINE void* ma_atomic_exchange_explicit_ptr(volatile void** dst, void* src, ma_atomic_memory_order order)\n    {\n        return (void*)ma_atomic_exchange_explicit_32((volatile ma_uint32*)dst, (ma_uint32)src, order);\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        return ma_atomic_compare_exchange_strong_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder);\n    }\n    static MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_ptr(volatile void** dst, void** expected, void* replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n    {\n        return ma_atomic_compare_exchange_weak_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder);\n    }\n    static MA_INLINE void* ma_atomic_compare_and_swap_ptr(volatile void** dst, void* expected, void* replacement)\n    {\n        return (void*)ma_atomic_compare_and_swap_32((volatile ma_uint32*)dst, (ma_uint32)expected, (ma_uint32)replacement);\n    }\n#else\n    #error Unsupported architecture.\n#endif\n#define ma_atomic_store_ptr(dst, src)                                       ma_atomic_store_explicit_ptr((volatile void**)dst, (void*)src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_ptr(ptr)                                             ma_atomic_load_explicit_ptr((volatile void**)ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_ptr(dst, src)                                    ma_atomic_exchange_explicit_ptr((volatile void**)dst, (void*)src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_ptr(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_ptr((volatile void**)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_ptr(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_ptr((volatile void**)dst, (void**)expected, (void*)replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_8( dst, src)                                    ma_atomic_store_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_16(dst, src)                                    ma_atomic_store_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_32(dst, src)                                    ma_atomic_store_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_64(dst, src)                                    ma_atomic_store_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_8( ptr)                                          ma_atomic_load_explicit_8( ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_16(ptr)                                          ma_atomic_load_explicit_16(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_32(ptr)                                          ma_atomic_load_explicit_32(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_64(ptr)                                          ma_atomic_load_explicit_64(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_8( dst, src)                                 ma_atomic_exchange_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_16(dst, src)                                 ma_atomic_exchange_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_32(dst, src)                                 ma_atomic_exchange_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_64(dst, src)                                 ma_atomic_exchange_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_8( dst, expected, replacement)    ma_atomic_compare_exchange_strong_explicit_8( dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_16(dst, expected, replacement)    ma_atomic_compare_exchange_strong_explicit_16(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_32(dst, expected, replacement)    ma_atomic_compare_exchange_strong_explicit_32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_64(dst, expected, replacement)    ma_atomic_compare_exchange_strong_explicit_64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_8(  dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_8( dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_16( dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_16(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_32( dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_64( dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_8( dst, src)                                ma_atomic_fetch_add_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_16(dst, src)                                ma_atomic_fetch_add_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_32(dst, src)                                ma_atomic_fetch_add_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_64(dst, src)                                ma_atomic_fetch_add_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_8( dst, src)                                ma_atomic_fetch_sub_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_16(dst, src)                                ma_atomic_fetch_sub_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_32(dst, src)                                ma_atomic_fetch_sub_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_64(dst, src)                                ma_atomic_fetch_sub_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_8( dst, src)                                 ma_atomic_fetch_or_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_16(dst, src)                                 ma_atomic_fetch_or_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_32(dst, src)                                 ma_atomic_fetch_or_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_64(dst, src)                                 ma_atomic_fetch_or_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_8( dst, src)                                ma_atomic_fetch_xor_explicit_8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_16(dst, src)                                ma_atomic_fetch_xor_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_32(dst, src)                                ma_atomic_fetch_xor_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_64(dst, src)                                ma_atomic_fetch_xor_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_8( dst, src)                                ma_atomic_fetch_and_explicit_8 (dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_16(dst, src)                                ma_atomic_fetch_and_explicit_16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_32(dst, src)                                ma_atomic_fetch_and_explicit_32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_64(dst, src)                                ma_atomic_fetch_and_explicit_64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_explicit_i8( dst, src, order)                   ma_atomic_store_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_store_explicit_i16(dst, src, order)                   ma_atomic_store_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_store_explicit_i32(dst, src, order)                   ma_atomic_store_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_store_explicit_i64(dst, src, order)                   ma_atomic_store_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_load_explicit_i8( ptr, order)                         (ma_int8 )ma_atomic_load_explicit_8( (ma_uint8* )ptr, order)\n#define ma_atomic_load_explicit_i16(ptr, order)                         (ma_int16)ma_atomic_load_explicit_16((ma_uint16*)ptr, order)\n#define ma_atomic_load_explicit_i32(ptr, order)                         (ma_int32)ma_atomic_load_explicit_32((ma_uint32*)ptr, order)\n#define ma_atomic_load_explicit_i64(ptr, order)                         (ma_int64)ma_atomic_load_explicit_64((ma_uint64*)ptr, order)\n#define ma_atomic_exchange_explicit_i8( dst, src, order)                (ma_int8 )ma_atomic_exchange_explicit_8 ((ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_exchange_explicit_i16(dst, src, order)                (ma_int16)ma_atomic_exchange_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_exchange_explicit_i32(dst, src, order)                (ma_int32)ma_atomic_exchange_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_exchange_explicit_i64(dst, src, order)                (ma_int64)ma_atomic_exchange_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_compare_exchange_strong_explicit_i8( dst, expected, replacement, successOrder, failureOrder)  ma_atomic_compare_exchange_strong_explicit_8( (ma_uint8* )dst, (ma_uint8* )expected, (ma_uint8 )replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_strong_explicit_i16(dst, expected, replacement, successOrder, failureOrder)  ma_atomic_compare_exchange_strong_explicit_16((ma_uint16*)dst, (ma_uint16*)expected, (ma_uint16)replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_strong_explicit_i32(dst, expected, replacement, successOrder, failureOrder)  ma_atomic_compare_exchange_strong_explicit_32((ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_strong_explicit_i64(dst, expected, replacement, successOrder, failureOrder)  ma_atomic_compare_exchange_strong_explicit_64((ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_weak_explicit_i8( dst, expected, replacement, successOrder, failureOrder)    ma_atomic_compare_exchange_weak_explicit_8( (ma_uint8* )dst, (ma_uint8* )expected, (ma_uint8 )replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_weak_explicit_i16(dst, expected, replacement, successOrder, failureOrder)    ma_atomic_compare_exchange_weak_explicit_16((ma_uint16*)dst, (ma_uint16*)expected, (ma_uint16)replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_weak_explicit_i32(dst, expected, replacement, successOrder, failureOrder)    ma_atomic_compare_exchange_weak_explicit_32((ma_uint32*)dst, (ma_uint32*)expected, (ma_uint32)replacement, successOrder, failureOrder)\n#define ma_atomic_compare_exchange_weak_explicit_i64(dst, expected, replacement, successOrder, failureOrder)    ma_atomic_compare_exchange_weak_explicit_64((ma_uint64*)dst, (ma_uint64*)expected, (ma_uint64)replacement, successOrder, failureOrder)\n#define ma_atomic_fetch_add_explicit_i8( dst, src, order)               (ma_int8 )ma_atomic_fetch_add_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_fetch_add_explicit_i16(dst, src, order)               (ma_int16)ma_atomic_fetch_add_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_fetch_add_explicit_i32(dst, src, order)               (ma_int32)ma_atomic_fetch_add_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_fetch_add_explicit_i64(dst, src, order)               (ma_int64)ma_atomic_fetch_add_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_fetch_sub_explicit_i8( dst, src, order)               (ma_int8 )ma_atomic_fetch_sub_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_fetch_sub_explicit_i16(dst, src, order)               (ma_int16)ma_atomic_fetch_sub_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_fetch_sub_explicit_i32(dst, src, order)               (ma_int32)ma_atomic_fetch_sub_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_fetch_sub_explicit_i64(dst, src, order)               (ma_int64)ma_atomic_fetch_sub_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_fetch_or_explicit_i8( dst, src, order)                (ma_int8 )ma_atomic_fetch_or_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_fetch_or_explicit_i16(dst, src, order)                (ma_int16)ma_atomic_fetch_or_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_fetch_or_explicit_i32(dst, src, order)                (ma_int32)ma_atomic_fetch_or_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_fetch_or_explicit_i64(dst, src, order)                (ma_int64)ma_atomic_fetch_or_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_fetch_xor_explicit_i8( dst, src, order)               (ma_int8 )ma_atomic_fetch_xor_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_fetch_xor_explicit_i16(dst, src, order)               (ma_int16)ma_atomic_fetch_xor_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_fetch_xor_explicit_i32(dst, src, order)               (ma_int32)ma_atomic_fetch_xor_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_fetch_xor_explicit_i64(dst, src, order)               (ma_int64)ma_atomic_fetch_xor_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_fetch_and_explicit_i8( dst, src, order)               (ma_int8 )ma_atomic_fetch_and_explicit_8( (ma_uint8* )dst, (ma_uint8 )src, order)\n#define ma_atomic_fetch_and_explicit_i16(dst, src, order)               (ma_int16)ma_atomic_fetch_and_explicit_16((ma_uint16*)dst, (ma_uint16)src, order)\n#define ma_atomic_fetch_and_explicit_i32(dst, src, order)               (ma_int32)ma_atomic_fetch_and_explicit_32((ma_uint32*)dst, (ma_uint32)src, order)\n#define ma_atomic_fetch_and_explicit_i64(dst, src, order)               (ma_int64)ma_atomic_fetch_and_explicit_64((ma_uint64*)dst, (ma_uint64)src, order)\n#define ma_atomic_store_i8( dst, src)                                   ma_atomic_store_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_i16(dst, src)                                   ma_atomic_store_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_i32(dst, src)                                   ma_atomic_store_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_i64(dst, src)                                   ma_atomic_store_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_i8( ptr)                                         ma_atomic_load_explicit_i8( ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_i16(ptr)                                         ma_atomic_load_explicit_i16(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_i32(ptr)                                         ma_atomic_load_explicit_i32(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_i64(ptr)                                         ma_atomic_load_explicit_i64(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_i8( dst, src)                                ma_atomic_exchange_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_i16(dst, src)                                ma_atomic_exchange_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_i32(dst, src)                                ma_atomic_exchange_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_i64(dst, src)                                ma_atomic_exchange_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_i8( dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_i8( dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_i16(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_i16(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_i32(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_i32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_i64(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_i64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_i8( dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_i8( dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_i16(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_i16(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_i32(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_i32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_i64(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_i64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_i8( dst, src)                               ma_atomic_fetch_add_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_i16(dst, src)                               ma_atomic_fetch_add_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_i32(dst, src)                               ma_atomic_fetch_add_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_i64(dst, src)                               ma_atomic_fetch_add_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_i8( dst, src)                               ma_atomic_fetch_sub_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_i16(dst, src)                               ma_atomic_fetch_sub_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_i32(dst, src)                               ma_atomic_fetch_sub_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_i64(dst, src)                               ma_atomic_fetch_sub_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_i8( dst, src)                                ma_atomic_fetch_or_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_i16(dst, src)                                ma_atomic_fetch_or_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_i32(dst, src)                                ma_atomic_fetch_or_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_i64(dst, src)                                ma_atomic_fetch_or_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_i8( dst, src)                               ma_atomic_fetch_xor_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_i16(dst, src)                               ma_atomic_fetch_xor_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_i32(dst, src)                               ma_atomic_fetch_xor_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_i64(dst, src)                               ma_atomic_fetch_xor_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_i8( dst, src)                               ma_atomic_fetch_and_explicit_i8( dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_i16(dst, src)                               ma_atomic_fetch_and_explicit_i16(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_i32(dst, src)                               ma_atomic_fetch_and_explicit_i32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_i64(dst, src)                               ma_atomic_fetch_and_explicit_i64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_and_swap_i8( dst, expected, dedsired)         (ma_int8 )ma_atomic_compare_and_swap_8( (ma_uint8* )dst, (ma_uint8 )expected, (ma_uint8 )dedsired)\n#define ma_atomic_compare_and_swap_i16(dst, expected, dedsired)         (ma_int16)ma_atomic_compare_and_swap_16((ma_uint16*)dst, (ma_uint16)expected, (ma_uint16)dedsired)\n#define ma_atomic_compare_and_swap_i32(dst, expected, dedsired)         (ma_int32)ma_atomic_compare_and_swap_32((ma_uint32*)dst, (ma_uint32)expected, (ma_uint32)dedsired)\n#define ma_atomic_compare_and_swap_i64(dst, expected, dedsired)         (ma_int64)ma_atomic_compare_and_swap_64((ma_uint64*)dst, (ma_uint64)expected, (ma_uint64)dedsired)\ntypedef union\n{\n    ma_uint32 i;\n    float f;\n} ma_atomic_if32;\ntypedef union\n{\n    ma_uint64 i;\n    double f;\n} ma_atomic_if64;\n#define ma_atomic_clear_explicit_f32(ptr, order)                        ma_atomic_clear_explicit_32((ma_uint32*)ptr, order)\n#define ma_atomic_clear_explicit_f64(ptr, order)                        ma_atomic_clear_explicit_64((ma_uint64*)ptr, order)\nstatic MA_INLINE void ma_atomic_store_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 x;\n    x.f = src;\n    ma_atomic_store_explicit_32((volatile ma_uint32*)dst, x.i, order);\n}\nstatic MA_INLINE void ma_atomic_store_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 x;\n    x.f = src;\n    ma_atomic_store_explicit_64((volatile ma_uint64*)dst, x.i, order);\n}\nstatic MA_INLINE float ma_atomic_load_explicit_f32(volatile const float* ptr, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    r.i = ma_atomic_load_explicit_32((volatile const ma_uint32*)ptr, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_load_explicit_f64(volatile const double* ptr, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    r.i = ma_atomic_load_explicit_64((volatile const ma_uint64*)ptr, order);\n    return r.f;\n}\nstatic MA_INLINE float ma_atomic_exchange_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_exchange_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_exchange_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_exchange_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_f32(volatile float* dst, float* expected, float replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n{\n    ma_atomic_if32 d;\n    d.f = replacement;\n    return ma_atomic_compare_exchange_strong_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, d.i, successOrder, failureOrder);\n}\nstatic MA_INLINE ma_bool32 ma_atomic_compare_exchange_strong_explicit_f64(volatile double* dst, double* expected, double replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n{\n    ma_atomic_if64 d;\n    d.f = replacement;\n    return ma_atomic_compare_exchange_strong_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, d.i, successOrder, failureOrder);\n}\nstatic MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_f32(volatile float* dst, float* expected, float replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n{\n    ma_atomic_if32 d;\n    d.f = replacement;\n    return ma_atomic_compare_exchange_weak_explicit_32((volatile ma_uint32*)dst, (ma_uint32*)expected, d.i, successOrder, failureOrder);\n}\nstatic MA_INLINE ma_bool32 ma_atomic_compare_exchange_weak_explicit_f64(volatile double* dst, double* expected, double replacement, ma_atomic_memory_order successOrder, ma_atomic_memory_order failureOrder)\n{\n    ma_atomic_if64 d;\n    d.f = replacement;\n    return ma_atomic_compare_exchange_weak_explicit_64((volatile ma_uint64*)dst, (ma_uint64*)expected, d.i, successOrder, failureOrder);\n}\nstatic MA_INLINE float ma_atomic_fetch_add_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_add_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_fetch_add_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_add_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE float ma_atomic_fetch_sub_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_sub_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_fetch_sub_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_sub_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE float ma_atomic_fetch_or_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_or_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_fetch_or_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_or_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE float ma_atomic_fetch_xor_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_xor_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_fetch_xor_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_xor_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE float ma_atomic_fetch_and_explicit_f32(volatile float* dst, float src, ma_atomic_memory_order order)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_and_explicit_32((volatile ma_uint32*)dst, x.i, order);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_fetch_and_explicit_f64(volatile double* dst, double src, ma_atomic_memory_order order)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 x;\n    x.f = src;\n    r.i = ma_atomic_fetch_and_explicit_64((volatile ma_uint64*)dst, x.i, order);\n    return r.f;\n}\n#define ma_atomic_clear_f32(ptr)                                        (float )ma_atomic_clear_explicit_f32(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_clear_f64(ptr)                                        (double)ma_atomic_clear_explicit_f64(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_f32(dst, src)                                   ma_atomic_store_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_store_f64(dst, src)                                   ma_atomic_store_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_f32(ptr)                                         (float )ma_atomic_load_explicit_f32(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_load_f64(ptr)                                         (double)ma_atomic_load_explicit_f64(ptr, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_f32(dst, src)                                (float )ma_atomic_exchange_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_exchange_f64(dst, src)                                (double)ma_atomic_exchange_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_f32(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_f32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_strong_f64(dst, expected, replacement)   ma_atomic_compare_exchange_strong_explicit_f64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_f32(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_f32(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_compare_exchange_weak_f64(dst, expected, replacement)     ma_atomic_compare_exchange_weak_explicit_f64(dst, expected, replacement, ma_atomic_memory_order_seq_cst, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_f32(dst, src)                               ma_atomic_fetch_add_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_add_f64(dst, src)                               ma_atomic_fetch_add_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_f32(dst, src)                               ma_atomic_fetch_sub_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_sub_f64(dst, src)                               ma_atomic_fetch_sub_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_f32(dst, src)                                ma_atomic_fetch_or_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_or_f64(dst, src)                                ma_atomic_fetch_or_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_f32(dst, src)                               ma_atomic_fetch_xor_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_xor_f64(dst, src)                               ma_atomic_fetch_xor_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_f32(dst, src)                               ma_atomic_fetch_and_explicit_f32(dst, src, ma_atomic_memory_order_seq_cst)\n#define ma_atomic_fetch_and_f64(dst, src)                               ma_atomic_fetch_and_explicit_f64(dst, src, ma_atomic_memory_order_seq_cst)\nstatic MA_INLINE float ma_atomic_compare_and_swap_f32(volatile float* dst, float expected, float replacement)\n{\n    ma_atomic_if32 r;\n    ma_atomic_if32 e, d;\n    e.f = expected;\n    d.f = replacement;\n    r.i = ma_atomic_compare_and_swap_32((volatile ma_uint32*)dst, e.i, d.i);\n    return r.f;\n}\nstatic MA_INLINE double ma_atomic_compare_and_swap_f64(volatile double* dst, double expected, double replacement)\n{\n    ma_atomic_if64 r;\n    ma_atomic_if64 e, d;\n    e.f = expected;\n    d.f = replacement;\n    r.i = ma_atomic_compare_and_swap_64((volatile ma_uint64*)dst, e.i, d.i);\n    return r.f;\n}\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic pop\n#endif\n#if defined(__cplusplus)\n}\n#endif\n#endif\n/* c89atomic.h end */\n\n#define MA_ATOMIC_SAFE_TYPE_IMPL(c89TypeExtension, type) \\\n    static MA_INLINE ma_##type ma_atomic_##type##_get(ma_atomic_##type* x) \\\n    { \\\n        return (ma_##type)ma_atomic_load_##c89TypeExtension(&x->value); \\\n    } \\\n    static MA_INLINE void ma_atomic_##type##_set(ma_atomic_##type* x, ma_##type value) \\\n    { \\\n        ma_atomic_store_##c89TypeExtension(&x->value, value); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_exchange(ma_atomic_##type* x, ma_##type value) \\\n    { \\\n        return (ma_##type)ma_atomic_exchange_##c89TypeExtension(&x->value, value); \\\n    } \\\n    static MA_INLINE ma_bool32 ma_atomic_##type##_compare_exchange(ma_atomic_##type* x, ma_##type* expected, ma_##type desired) \\\n    { \\\n        return ma_atomic_compare_exchange_weak_##c89TypeExtension(&x->value, expected, desired); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_fetch_add(ma_atomic_##type* x, ma_##type y) \\\n    { \\\n        return (ma_##type)ma_atomic_fetch_add_##c89TypeExtension(&x->value, y); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_fetch_sub(ma_atomic_##type* x, ma_##type y) \\\n    { \\\n        return (ma_##type)ma_atomic_fetch_sub_##c89TypeExtension(&x->value, y); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_fetch_or(ma_atomic_##type* x, ma_##type y) \\\n    { \\\n        return (ma_##type)ma_atomic_fetch_or_##c89TypeExtension(&x->value, y); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_fetch_xor(ma_atomic_##type* x, ma_##type y) \\\n    { \\\n        return (ma_##type)ma_atomic_fetch_xor_##c89TypeExtension(&x->value, y); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_fetch_and(ma_atomic_##type* x, ma_##type y) \\\n    { \\\n        return (ma_##type)ma_atomic_fetch_and_##c89TypeExtension(&x->value, y); \\\n    } \\\n    static MA_INLINE ma_##type ma_atomic_##type##_compare_and_swap(ma_atomic_##type* x, ma_##type expected, ma_##type desired) \\\n    { \\\n        return (ma_##type)ma_atomic_compare_and_swap_##c89TypeExtension(&x->value, expected, desired); \\\n    } \\\n\n#define MA_ATOMIC_SAFE_TYPE_IMPL_PTR(type) \\\n    static MA_INLINE ma_##type* ma_atomic_ptr_##type##_get(ma_atomic_ptr_##type* x) \\\n    { \\\n        return ma_atomic_load_ptr((void**)&x->value); \\\n    } \\\n    static MA_INLINE void ma_atomic_ptr_##type##_set(ma_atomic_ptr_##type* x, ma_##type* value) \\\n    { \\\n        ma_atomic_store_ptr((void**)&x->value, (void*)value); \\\n    } \\\n    static MA_INLINE ma_##type* ma_atomic_ptr_##type##_exchange(ma_atomic_ptr_##type* x, ma_##type* value) \\\n    { \\\n        return ma_atomic_exchange_ptr((void**)&x->value, (void*)value); \\\n    } \\\n    static MA_INLINE ma_bool32 ma_atomic_ptr_##type##_compare_exchange(ma_atomic_ptr_##type* x, ma_##type** expected, ma_##type* desired) \\\n    { \\\n        return ma_atomic_compare_exchange_weak_ptr((void**)&x->value, (void*)expected, (void*)desired); \\\n    } \\\n    static MA_INLINE ma_##type* ma_atomic_ptr_##type##_compare_and_swap(ma_atomic_ptr_##type* x, ma_##type* expected, ma_##type* desired) \\\n    { \\\n        return (ma_##type*)ma_atomic_compare_and_swap_ptr((void**)&x->value, (void*)expected, (void*)desired); \\\n    } \\\n\nMA_ATOMIC_SAFE_TYPE_IMPL(32,  uint32)\nMA_ATOMIC_SAFE_TYPE_IMPL(i32, int32)\nMA_ATOMIC_SAFE_TYPE_IMPL(64,  uint64)\nMA_ATOMIC_SAFE_TYPE_IMPL(f32, float)\nMA_ATOMIC_SAFE_TYPE_IMPL(32,  bool32)\n\n#if !defined(MA_NO_DEVICE_IO)\nMA_ATOMIC_SAFE_TYPE_IMPL(i32, device_state)\n#endif\n\n\nMA_API ma_uint64 ma_calculate_frame_count_after_resampling(ma_uint32 sampleRateOut, ma_uint32 sampleRateIn, ma_uint64 frameCountIn)\n{\n    /* This is based on the calculation in ma_linear_resampler_get_expected_output_frame_count(). */\n    ma_uint64 outputFrameCount;\n    ma_uint64 preliminaryInputFrameCountFromFrac;\n    ma_uint64 preliminaryInputFrameCount;\n\n    if (sampleRateIn == 0 || sampleRateOut == 0 || frameCountIn == 0) {\n        return 0;\n    }\n\n    if (sampleRateOut == sampleRateIn) {\n        return frameCountIn;\n    }\n\n    outputFrameCount = (frameCountIn * sampleRateOut) / sampleRateIn;\n\n    preliminaryInputFrameCountFromFrac = (outputFrameCount * (sampleRateIn / sampleRateOut)) / sampleRateOut;\n    preliminaryInputFrameCount         = (outputFrameCount * (sampleRateIn % sampleRateOut)) + preliminaryInputFrameCountFromFrac;\n\n    if (preliminaryInputFrameCount <= frameCountIn) {\n        outputFrameCount += 1;\n    }\n\n    return outputFrameCount;\n}\n\n#ifndef MA_DATA_CONVERTER_STACK_BUFFER_SIZE\n#define MA_DATA_CONVERTER_STACK_BUFFER_SIZE     4096\n#endif\n\n\n\n#if defined(MA_WIN32)\nstatic ma_result ma_result_from_GetLastError(DWORD error)\n{\n    switch (error)\n    {\n        case ERROR_SUCCESS:             return MA_SUCCESS;\n        case ERROR_PATH_NOT_FOUND:      return MA_DOES_NOT_EXIST;\n        case ERROR_TOO_MANY_OPEN_FILES: return MA_TOO_MANY_OPEN_FILES;\n        case ERROR_NOT_ENOUGH_MEMORY:   return MA_OUT_OF_MEMORY;\n        case ERROR_DISK_FULL:           return MA_NO_SPACE;\n        case ERROR_HANDLE_EOF:          return MA_AT_END;\n        case ERROR_NEGATIVE_SEEK:       return MA_BAD_SEEK;\n        case ERROR_INVALID_PARAMETER:   return MA_INVALID_ARGS;\n        case ERROR_ACCESS_DENIED:       return MA_ACCESS_DENIED;\n        case ERROR_SEM_TIMEOUT:         return MA_TIMEOUT;\n        case ERROR_FILE_NOT_FOUND:      return MA_DOES_NOT_EXIST;\n        default: break;\n    }\n\n    return MA_ERROR;\n}\n#endif  /* MA_WIN32 */\n\n\n/*******************************************************************************\n\nThreading\n\n*******************************************************************************/\nstatic MA_INLINE ma_result ma_spinlock_lock_ex(volatile ma_spinlock* pSpinlock, ma_bool32 yield)\n{\n    if (pSpinlock == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (;;) {\n        if (ma_atomic_exchange_explicit_32(pSpinlock, 1, ma_atomic_memory_order_acquire) == 0) {\n            break;\n        }\n\n        while (ma_atomic_load_explicit_32(pSpinlock, ma_atomic_memory_order_relaxed) == 1) {\n            if (yield) {\n                ma_yield();\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spinlock_lock(volatile ma_spinlock* pSpinlock)\n{\n    return ma_spinlock_lock_ex(pSpinlock, MA_TRUE);\n}\n\nMA_API ma_result ma_spinlock_lock_noyield(volatile ma_spinlock* pSpinlock)\n{\n    return ma_spinlock_lock_ex(pSpinlock, MA_FALSE);\n}\n\nMA_API ma_result ma_spinlock_unlock(volatile ma_spinlock* pSpinlock)\n{\n    if (pSpinlock == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_store_explicit_32(pSpinlock, 0, ma_atomic_memory_order_release);\n    return MA_SUCCESS;\n}\n\n\n#ifndef MA_NO_THREADING\n#if defined(MA_POSIX)\n    #define MA_THREADCALL\n    typedef void* ma_thread_result;\n#elif defined(MA_WIN32)\n    #define MA_THREADCALL WINAPI\n    typedef unsigned long ma_thread_result;\n#endif\n\ntypedef ma_thread_result (MA_THREADCALL * ma_thread_entry_proc)(void* pData);\n\n#ifdef MA_POSIX\nstatic ma_result ma_thread_create__posix(ma_thread* pThread, ma_thread_priority priority, size_t stackSize, ma_thread_entry_proc entryProc, void* pData)\n{\n    int result;\n    pthread_attr_t* pAttr = NULL;\n\n#if !defined(MA_EMSCRIPTEN) && !defined(MA_3DS) && !defined(MA_SWITCH)\n    /* Try setting the thread priority. It's not critical if anything fails here. */\n    pthread_attr_t attr;\n    if (pthread_attr_init(&attr) == 0) {\n        int scheduler = -1;\n\n        /* We successfully initialized our attributes object so we can assign the pointer so it's passed into pthread_create(). */\n        pAttr = &attr;\n\n        /* We need to set the scheduler policy. Only do this if the OS supports pthread_attr_setschedpolicy() */\n        #if !defined(MA_BEOS)\n        {\n            if (priority == ma_thread_priority_idle) {\n            #ifdef SCHED_IDLE\n                if (pthread_attr_setschedpolicy(&attr, SCHED_IDLE) == 0) {\n                    scheduler = SCHED_IDLE;\n                }\n            #endif\n            } else if (priority == ma_thread_priority_realtime) {\n            #ifdef SCHED_FIFO\n                if (pthread_attr_setschedpolicy(&attr, SCHED_FIFO) == 0) {\n                    scheduler = SCHED_FIFO;\n                }\n            #endif\n            #ifdef MA_LINUX\n            } else {\n                scheduler = sched_getscheduler(0);\n            #endif\n            }\n        }\n        #endif\n\n        #if defined(_POSIX_THREAD_ATTR_STACKSIZE) && _POSIX_THREAD_ATTR_STACKSIZE >= 0\n        {\n            if (stackSize > 0) {\n                pthread_attr_setstacksize(&attr, stackSize);\n            }\n        }\n        #else\n        {\n            (void)stackSize;  /* Suppress unused parameter warning. */\n        }\n        #endif\n        \n\n        if (scheduler != -1) {\n            int priorityMin = sched_get_priority_min(scheduler);\n            int priorityMax = sched_get_priority_max(scheduler);\n            int priorityStep = (priorityMax - priorityMin) / 7;  /* 7 = number of priorities supported by miniaudio. */\n\n            struct sched_param sched;\n            if (priorityMin != -1 && priorityMax != -1 && pthread_attr_getschedparam(&attr, &sched) == 0) {\n                if (priority == ma_thread_priority_idle) {\n                    sched.sched_priority = priorityMin;\n                } else if (priority == ma_thread_priority_realtime) {\n                    #if defined(MA_PTHREAD_REALTIME_THREAD_PRIORITY)\n                    {\n                        sched.sched_priority = MA_PTHREAD_REALTIME_THREAD_PRIORITY;\n                    }\n                    #else\n                    {\n                        sched.sched_priority = priorityMax;\n                    }\n                    #endif\n                } else {\n                    sched.sched_priority += ((int)priority + 5) * priorityStep;  /* +5 because the lowest priority is -5. */\n                }\n\n                if (sched.sched_priority < priorityMin) {\n                    sched.sched_priority = priorityMin;\n                }\n                if (sched.sched_priority > priorityMax) {\n                    sched.sched_priority = priorityMax;\n                }\n\n                /* I'm not treating a failure of setting the priority as a critical error so not aborting on failure here. */\n                if (pthread_attr_setschedparam(&attr, &sched) == 0) {\n                    #if !defined(MA_ANDROID) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 28)\n                    {\n                        pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);\n                    }\n                    #endif\n                }\n            }\n        }\n    }\n#else\n    /* It's the emscripten build. We'll have a few unused parameters. */\n    (void)priority;\n    (void)stackSize;\n#endif\n\n    result = pthread_create((pthread_t*)pThread, pAttr, entryProc, pData);\n\n    /* The thread attributes object is no longer required. */\n    if (pAttr != NULL) {\n        pthread_attr_destroy(pAttr);\n    }\n\n    if (result != 0) {\n        /*\n        There have been reports that attempting to create a realtime thread can sometimes fail. In this case,\n        fall back to a normal priority thread.\n\n        I'm including a compile-time option here to disable this functionality for those who have a hard\n        requirement on realtime threads and would rather an explicit failure.\n        */\n        #ifndef MA_NO_PTHREAD_REALTIME_PRIORITY_FALLBACK\n        {\n            if(result == EPERM && priority == ma_thread_priority_realtime) {\n                return ma_thread_create__posix(pThread, ma_thread_priority_normal, stackSize, entryProc, pData);\n            }\n        }\n        #endif\n\n        return ma_result_from_errno(result);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_thread_wait__posix(ma_thread* pThread)\n{\n    pthread_join((pthread_t)*pThread, NULL);\n}\n\n\nstatic ma_result ma_mutex_init__posix(ma_mutex* pMutex)\n{\n    int result;\n\n    if (pMutex == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pMutex);\n\n    result = pthread_mutex_init((pthread_mutex_t*)pMutex, NULL);\n    if (result != 0) {\n        return ma_result_from_errno(result);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_mutex_uninit__posix(ma_mutex* pMutex)\n{\n    pthread_mutex_destroy((pthread_mutex_t*)pMutex);\n}\n\nstatic void ma_mutex_lock__posix(ma_mutex* pMutex)\n{\n    pthread_mutex_lock((pthread_mutex_t*)pMutex);\n}\n\nstatic void ma_mutex_unlock__posix(ma_mutex* pMutex)\n{\n    pthread_mutex_unlock((pthread_mutex_t*)pMutex);\n}\n\n\nstatic ma_result ma_event_init__posix(ma_event* pEvent)\n{\n    int result;\n\n    result = pthread_mutex_init((pthread_mutex_t*)&pEvent->lock, NULL);\n    if (result != 0) {\n        return ma_result_from_errno(result);\n    }\n\n    result = pthread_cond_init((pthread_cond_t*)&pEvent->cond, NULL);\n    if (result != 0) {\n        pthread_mutex_destroy((pthread_mutex_t*)&pEvent->lock);\n        return ma_result_from_errno(result);\n    }\n\n    pEvent->value = 0;\n    return MA_SUCCESS;\n}\n\nstatic void ma_event_uninit__posix(ma_event* pEvent)\n{\n    pthread_cond_destroy((pthread_cond_t*)&pEvent->cond);\n    pthread_mutex_destroy((pthread_mutex_t*)&pEvent->lock);\n}\n\nstatic ma_result ma_event_wait__posix(ma_event* pEvent)\n{\n    pthread_mutex_lock((pthread_mutex_t*)&pEvent->lock);\n    {\n        while (pEvent->value == 0) {\n            pthread_cond_wait((pthread_cond_t*)&pEvent->cond, (pthread_mutex_t*)&pEvent->lock);\n        }\n        pEvent->value = 0;  /* Auto-reset. */\n    }\n    pthread_mutex_unlock((pthread_mutex_t*)&pEvent->lock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_event_signal__posix(ma_event* pEvent)\n{\n    pthread_mutex_lock((pthread_mutex_t*)&pEvent->lock);\n    {\n        pEvent->value = 1;\n        pthread_cond_signal((pthread_cond_t*)&pEvent->cond);\n    }\n    pthread_mutex_unlock((pthread_mutex_t*)&pEvent->lock);\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_semaphore_init__posix(int initialValue, ma_semaphore* pSemaphore)\n{\n    int result;\n\n    if (pSemaphore == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pSemaphore->value = initialValue;\n\n    result = pthread_mutex_init((pthread_mutex_t*)&pSemaphore->lock, NULL);\n    if (result != 0) {\n        return ma_result_from_errno(result);  /* Failed to create mutex. */\n    }\n\n    result = pthread_cond_init((pthread_cond_t*)&pSemaphore->cond, NULL);\n    if (result != 0) {\n        pthread_mutex_destroy((pthread_mutex_t*)&pSemaphore->lock);\n        return ma_result_from_errno(result);  /* Failed to create condition variable. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_semaphore_uninit__posix(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        return;\n    }\n\n    pthread_cond_destroy((pthread_cond_t*)&pSemaphore->cond);\n    pthread_mutex_destroy((pthread_mutex_t*)&pSemaphore->lock);\n}\n\nstatic ma_result ma_semaphore_wait__posix(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pthread_mutex_lock((pthread_mutex_t*)&pSemaphore->lock);\n    {\n        /* We need to wait on a condition variable before escaping. We can't return from this function until the semaphore has been signaled. */\n        while (pSemaphore->value == 0) {\n            pthread_cond_wait((pthread_cond_t*)&pSemaphore->cond, (pthread_mutex_t*)&pSemaphore->lock);\n        }\n\n        pSemaphore->value -= 1;\n    }\n    pthread_mutex_unlock((pthread_mutex_t*)&pSemaphore->lock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_semaphore_release__posix(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pthread_mutex_lock((pthread_mutex_t*)&pSemaphore->lock);\n    {\n        pSemaphore->value += 1;\n        pthread_cond_signal((pthread_cond_t*)&pSemaphore->cond);\n    }\n    pthread_mutex_unlock((pthread_mutex_t*)&pSemaphore->lock);\n\n    return MA_SUCCESS;\n}\n#elif defined(MA_WIN32)\nstatic int ma_thread_priority_to_win32(ma_thread_priority priority)\n{\n    switch (priority) {\n        case ma_thread_priority_idle:     return THREAD_PRIORITY_IDLE;\n        case ma_thread_priority_lowest:   return THREAD_PRIORITY_LOWEST;\n        case ma_thread_priority_low:      return THREAD_PRIORITY_BELOW_NORMAL;\n        case ma_thread_priority_normal:   return THREAD_PRIORITY_NORMAL;\n        case ma_thread_priority_high:     return THREAD_PRIORITY_ABOVE_NORMAL;\n        case ma_thread_priority_highest:  return THREAD_PRIORITY_HIGHEST;\n        case ma_thread_priority_realtime: return THREAD_PRIORITY_TIME_CRITICAL;\n        default:                          return THREAD_PRIORITY_NORMAL;\n    }\n}\n\nstatic ma_result ma_thread_create__win32(ma_thread* pThread, ma_thread_priority priority, size_t stackSize, ma_thread_entry_proc entryProc, void* pData)\n{\n    DWORD threadID; /* Not used. Only used for passing into CreateThread() so it doesn't fail on Windows 98. */\n\n    *pThread = CreateThread(NULL, stackSize, entryProc, pData, 0, &threadID);\n    if (*pThread == NULL) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    SetThreadPriority((HANDLE)*pThread, ma_thread_priority_to_win32(priority));\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_thread_wait__win32(ma_thread* pThread)\n{\n    WaitForSingleObject((HANDLE)*pThread, INFINITE);\n    CloseHandle((HANDLE)*pThread);\n}\n\n\nstatic ma_result ma_mutex_init__win32(ma_mutex* pMutex)\n{\n    *pMutex = CreateEventA(NULL, FALSE, TRUE, NULL);\n    if (*pMutex == NULL) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_mutex_uninit__win32(ma_mutex* pMutex)\n{\n    CloseHandle((HANDLE)*pMutex);\n}\n\nstatic void ma_mutex_lock__win32(ma_mutex* pMutex)\n{\n    WaitForSingleObject((HANDLE)*pMutex, INFINITE);\n}\n\nstatic void ma_mutex_unlock__win32(ma_mutex* pMutex)\n{\n    SetEvent((HANDLE)*pMutex);\n}\n\n\nstatic ma_result ma_event_init__win32(ma_event* pEvent)\n{\n    *pEvent = CreateEventA(NULL, FALSE, FALSE, NULL);\n    if (*pEvent == NULL) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_event_uninit__win32(ma_event* pEvent)\n{\n    CloseHandle((HANDLE)*pEvent);\n}\n\nstatic ma_result ma_event_wait__win32(ma_event* pEvent)\n{\n    DWORD result = WaitForSingleObject((HANDLE)*pEvent, INFINITE);\n    if (result == WAIT_OBJECT_0) {\n        return MA_SUCCESS;\n    }\n\n    if (result == WAIT_TIMEOUT) {\n        return MA_TIMEOUT;\n    }\n\n    return ma_result_from_GetLastError(GetLastError());\n}\n\nstatic ma_result ma_event_signal__win32(ma_event* pEvent)\n{\n    BOOL result = SetEvent((HANDLE)*pEvent);\n    if (result == 0) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_semaphore_init__win32(int initialValue, ma_semaphore* pSemaphore)\n{\n    *pSemaphore = CreateSemaphore(NULL, (LONG)initialValue, LONG_MAX, NULL);\n    if (*pSemaphore == NULL) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_semaphore_uninit__win32(ma_semaphore* pSemaphore)\n{\n    CloseHandle((HANDLE)*pSemaphore);\n}\n\nstatic ma_result ma_semaphore_wait__win32(ma_semaphore* pSemaphore)\n{\n    DWORD result = WaitForSingleObject((HANDLE)*pSemaphore, INFINITE);\n    if (result == WAIT_OBJECT_0) {\n        return MA_SUCCESS;\n    }\n\n    if (result == WAIT_TIMEOUT) {\n        return MA_TIMEOUT;\n    }\n\n    return ma_result_from_GetLastError(GetLastError());\n}\n\nstatic ma_result ma_semaphore_release__win32(ma_semaphore* pSemaphore)\n{\n    BOOL result = ReleaseSemaphore((HANDLE)*pSemaphore, 1, NULL);\n    if (result == 0) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n#endif\n\ntypedef struct\n{\n    ma_thread_entry_proc entryProc;\n    void* pData;\n    ma_allocation_callbacks allocationCallbacks;\n} ma_thread_proxy_data;\n\nstatic ma_thread_result MA_THREADCALL ma_thread_entry_proxy(void* pData)\n{\n    ma_thread_proxy_data* pProxyData = (ma_thread_proxy_data*)pData;\n    ma_thread_entry_proc entryProc;\n    void* pEntryProcData;\n    ma_thread_result result;\n\n    #if defined(MA_ON_THREAD_ENTRY)\n        MA_ON_THREAD_ENTRY\n    #endif\n\n    entryProc = pProxyData->entryProc;\n    pEntryProcData = pProxyData->pData;\n\n    /* Free the proxy data before getting into the real thread entry proc. */\n    ma_free(pProxyData, &pProxyData->allocationCallbacks);\n\n    result = entryProc(pEntryProcData);\n\n    #if defined(MA_ON_THREAD_EXIT)\n        MA_ON_THREAD_EXIT\n    #endif\n\n    return result;\n}\n\nstatic ma_result ma_thread_create(ma_thread* pThread, ma_thread_priority priority, size_t stackSize, ma_thread_entry_proc entryProc, void* pData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_result result;\n    ma_thread_proxy_data* pProxyData;\n\n    if (pThread == NULL || entryProc == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pProxyData = (ma_thread_proxy_data*)ma_malloc(sizeof(*pProxyData), pAllocationCallbacks);   /* Will be freed by the proxy entry proc. */\n    if (pProxyData == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n#if defined(MA_THREAD_DEFAULT_STACK_SIZE)\n    if (stackSize == 0) {\n        stackSize = MA_THREAD_DEFAULT_STACK_SIZE;\n    }\n#endif\n\n    pProxyData->entryProc = entryProc;\n    pProxyData->pData     = pData;\n    ma_allocation_callbacks_init_copy(&pProxyData->allocationCallbacks, pAllocationCallbacks);\n\n#if defined(MA_POSIX)\n    result = ma_thread_create__posix(pThread, priority, stackSize, ma_thread_entry_proxy, pProxyData);\n#elif defined(MA_WIN32)\n    result = ma_thread_create__win32(pThread, priority, stackSize, ma_thread_entry_proxy, pProxyData);\n#endif\n\n    if (result != MA_SUCCESS) {\n        ma_free(pProxyData, pAllocationCallbacks);\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_thread_wait(ma_thread* pThread)\n{\n    if (pThread == NULL) {\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_thread_wait__posix(pThread);\n#elif defined(MA_WIN32)\n    ma_thread_wait__win32(pThread);\n#endif\n}\n\n\nMA_API ma_result ma_mutex_init(ma_mutex* pMutex)\n{\n    if (pMutex == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_mutex_init__posix(pMutex);\n#elif defined(MA_WIN32)\n    return ma_mutex_init__win32(pMutex);\n#endif\n}\n\nMA_API void ma_mutex_uninit(ma_mutex* pMutex)\n{\n    if (pMutex == NULL) {\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_mutex_uninit__posix(pMutex);\n#elif defined(MA_WIN32)\n    ma_mutex_uninit__win32(pMutex);\n#endif\n}\n\nMA_API void ma_mutex_lock(ma_mutex* pMutex)\n{\n    if (pMutex == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_mutex_lock__posix(pMutex);\n#elif defined(MA_WIN32)\n    ma_mutex_lock__win32(pMutex);\n#endif\n}\n\nMA_API void ma_mutex_unlock(ma_mutex* pMutex)\n{\n    if (pMutex == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_mutex_unlock__posix(pMutex);\n#elif defined(MA_WIN32)\n    ma_mutex_unlock__win32(pMutex);\n#endif\n}\n\n\nMA_API ma_result ma_event_init(ma_event* pEvent)\n{\n    if (pEvent == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_event_init__posix(pEvent);\n#elif defined(MA_WIN32)\n    return ma_event_init__win32(pEvent);\n#endif\n}\n\n#if 0\nstatic ma_result ma_event_alloc_and_init(ma_event** ppEvent, ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_result result;\n    ma_event* pEvent;\n\n    if (ppEvent == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *ppEvent = NULL;\n\n    pEvent = ma_malloc(sizeof(*pEvent), pAllocationCallbacks);\n    if (pEvent == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_event_init(pEvent);\n    if (result != MA_SUCCESS) {\n        ma_free(pEvent, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppEvent = pEvent;\n    return result;\n}\n#endif\n\nMA_API void ma_event_uninit(ma_event* pEvent)\n{\n    if (pEvent == NULL) {\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_event_uninit__posix(pEvent);\n#elif defined(MA_WIN32)\n    ma_event_uninit__win32(pEvent);\n#endif\n}\n\n#if 0\nstatic void ma_event_uninit_and_free(ma_event* pEvent, ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pEvent == NULL) {\n        return;\n    }\n\n    ma_event_uninit(pEvent);\n    ma_free(pEvent, pAllocationCallbacks);\n}\n#endif\n\nMA_API ma_result ma_event_wait(ma_event* pEvent)\n{\n    if (pEvent == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert to the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_event_wait__posix(pEvent);\n#elif defined(MA_WIN32)\n    return ma_event_wait__win32(pEvent);\n#endif\n}\n\nMA_API ma_result ma_event_signal(ma_event* pEvent)\n{\n    if (pEvent == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert to the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_event_signal__posix(pEvent);\n#elif defined(MA_WIN32)\n    return ma_event_signal__win32(pEvent);\n#endif\n}\n\n\nMA_API ma_result ma_semaphore_init(int initialValue, ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_semaphore_init__posix(initialValue, pSemaphore);\n#elif defined(MA_WIN32)\n    return ma_semaphore_init__win32(initialValue, pSemaphore);\n#endif\n}\n\nMA_API void ma_semaphore_uninit(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return;\n    }\n\n#if defined(MA_POSIX)\n    ma_semaphore_uninit__posix(pSemaphore);\n#elif defined(MA_WIN32)\n    ma_semaphore_uninit__win32(pSemaphore);\n#endif\n}\n\nMA_API ma_result ma_semaphore_wait(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_semaphore_wait__posix(pSemaphore);\n#elif defined(MA_WIN32)\n    return ma_semaphore_wait__win32(pSemaphore);\n#endif\n}\n\nMA_API ma_result ma_semaphore_release(ma_semaphore* pSemaphore)\n{\n    if (pSemaphore == NULL) {\n        MA_ASSERT(MA_FALSE);    /* Fire an assert so the caller is aware of this bug. */\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_POSIX)\n    return ma_semaphore_release__posix(pSemaphore);\n#elif defined(MA_WIN32)\n    return ma_semaphore_release__win32(pSemaphore);\n#endif\n}\n#else\n/* MA_NO_THREADING is set which means threading is disabled. Threading is required by some API families. If any of these are enabled we need to throw an error. */\n#ifndef MA_NO_DEVICE_IO\n#error \"MA_NO_THREADING cannot be used without MA_NO_DEVICE_IO\";\n#endif\n#endif  /* MA_NO_THREADING */\n\n\n\n#define MA_FENCE_COUNTER_MAX    0x7FFFFFFF\n\nMA_API ma_result ma_fence_init(ma_fence* pFence)\n{\n    if (pFence == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFence);\n    pFence->counter = 0;\n\n    #ifndef MA_NO_THREADING\n    {\n        ma_result result;\n\n        result = ma_event_init(&pFence->e);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_fence_uninit(ma_fence* pFence)\n{\n    if (pFence == NULL) {\n        return;\n    }\n\n    #ifndef MA_NO_THREADING\n    {\n        ma_event_uninit(&pFence->e);\n    }\n    #endif\n\n    MA_ZERO_OBJECT(pFence);\n}\n\nMA_API ma_result ma_fence_acquire(ma_fence* pFence)\n{\n    if (pFence == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (;;) {\n        ma_uint32 oldCounter = ma_atomic_load_32(&pFence->counter);\n        ma_uint32 newCounter = oldCounter + 1;\n\n        /* Make sure we're not about to exceed our maximum value. */\n        if (newCounter > MA_FENCE_COUNTER_MAX) {\n            MA_ASSERT(MA_FALSE);\n            return MA_OUT_OF_RANGE;\n        }\n\n        if (ma_atomic_compare_exchange_weak_32(&pFence->counter, &oldCounter, newCounter)) {\n            return MA_SUCCESS;\n        } else {\n            if (oldCounter == MA_FENCE_COUNTER_MAX) {\n                MA_ASSERT(MA_FALSE);\n                return MA_OUT_OF_RANGE; /* The other thread took the last available slot. Abort. */\n            }\n        }\n    }\n\n    /* Should never get here. */\n    /*return MA_SUCCESS;*/\n}\n\nMA_API ma_result ma_fence_release(ma_fence* pFence)\n{\n    if (pFence == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (;;) {\n        ma_uint32 oldCounter = ma_atomic_load_32(&pFence->counter);\n        ma_uint32 newCounter = oldCounter - 1;\n\n        if (oldCounter == 0) {\n            MA_ASSERT(MA_FALSE);\n            return MA_INVALID_OPERATION;    /* Acquire/release mismatch. */\n        }\n\n        if (ma_atomic_compare_exchange_weak_32(&pFence->counter, &oldCounter, newCounter)) {\n            #ifndef MA_NO_THREADING\n            {\n                if (newCounter == 0) {\n                    ma_event_signal(&pFence->e);    /* <-- ma_fence_wait() will be waiting on this. */\n                }\n            }\n            #endif\n\n            return MA_SUCCESS;\n        } else {\n            if (oldCounter == 0) {\n                MA_ASSERT(MA_FALSE);\n                return MA_INVALID_OPERATION;    /* Another thread has taken the 0 slot. Acquire/release mismatch. */\n            }\n        }\n    }\n\n    /* Should never get here. */\n    /*return MA_SUCCESS;*/\n}\n\nMA_API ma_result ma_fence_wait(ma_fence* pFence)\n{\n    if (pFence == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (;;) {\n        ma_uint32 counter;\n\n        counter = ma_atomic_load_32(&pFence->counter);\n        if (counter == 0) {\n            /*\n            Counter has hit zero. By the time we get here some other thread may have acquired the\n            fence again, but that is where the caller needs to take care with how they se the fence.\n            */\n            return MA_SUCCESS;\n        }\n\n        /* Getting here means the counter is > 0. We'll need to wait for something to happen. */\n        #ifndef MA_NO_THREADING\n        {\n            ma_result result;\n\n            result = ma_event_wait(&pFence->e);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n        #endif\n    }\n\n    /* Should never get here. */\n    /*return MA_INVALID_OPERATION;*/\n}\n\n\nMA_API ma_result ma_async_notification_signal(ma_async_notification* pNotification)\n{\n    ma_async_notification_callbacks* pNotificationCallbacks = (ma_async_notification_callbacks*)pNotification;\n\n    if (pNotification == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pNotificationCallbacks->onSignal == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    pNotificationCallbacks->onSignal(pNotification);\n    return MA_INVALID_ARGS;\n}\n\n\nstatic void ma_async_notification_poll__on_signal(ma_async_notification* pNotification)\n{\n    ((ma_async_notification_poll*)pNotification)->signalled = MA_TRUE;\n}\n\nMA_API ma_result ma_async_notification_poll_init(ma_async_notification_poll* pNotificationPoll)\n{\n    if (pNotificationPoll == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pNotificationPoll->cb.onSignal = ma_async_notification_poll__on_signal;\n    pNotificationPoll->signalled = MA_FALSE;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_bool32 ma_async_notification_poll_is_signalled(const ma_async_notification_poll* pNotificationPoll)\n{\n    if (pNotificationPoll == NULL) {\n        return MA_FALSE;\n    }\n\n    return pNotificationPoll->signalled;\n}\n\n\nstatic void ma_async_notification_event__on_signal(ma_async_notification* pNotification)\n{\n    ma_async_notification_event_signal((ma_async_notification_event*)pNotification);\n}\n\nMA_API ma_result ma_async_notification_event_init(ma_async_notification_event* pNotificationEvent)\n{\n    if (pNotificationEvent == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pNotificationEvent->cb.onSignal = ma_async_notification_event__on_signal;\n\n    #ifndef MA_NO_THREADING\n    {\n        ma_result result;\n\n        result = ma_event_init(&pNotificationEvent->e);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        return MA_NOT_IMPLEMENTED;  /* Threading is disabled. */\n    }\n    #endif\n}\n\nMA_API ma_result ma_async_notification_event_uninit(ma_async_notification_event* pNotificationEvent)\n{\n    if (pNotificationEvent == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #ifndef MA_NO_THREADING\n    {\n        ma_event_uninit(&pNotificationEvent->e);\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        return MA_NOT_IMPLEMENTED;  /* Threading is disabled. */\n    }\n    #endif\n}\n\nMA_API ma_result ma_async_notification_event_wait(ma_async_notification_event* pNotificationEvent)\n{\n    if (pNotificationEvent == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #ifndef MA_NO_THREADING\n    {\n        return ma_event_wait(&pNotificationEvent->e);\n    }\n    #else\n    {\n        return MA_NOT_IMPLEMENTED;  /* Threading is disabled. */\n    }\n    #endif\n}\n\nMA_API ma_result ma_async_notification_event_signal(ma_async_notification_event* pNotificationEvent)\n{\n    if (pNotificationEvent == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #ifndef MA_NO_THREADING\n    {\n        return ma_event_signal(&pNotificationEvent->e);\n    }\n    #else\n    {\n        return MA_NOT_IMPLEMENTED;  /* Threading is disabled. */\n    }\n    #endif\n}\n\n\n\n/************************************************************************************************************************************************************\n\nJob Queue\n\n************************************************************************************************************************************************************/\nMA_API ma_slot_allocator_config ma_slot_allocator_config_init(ma_uint32 capacity)\n{\n    ma_slot_allocator_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.capacity = capacity;\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_uint32 ma_slot_allocator_calculate_group_capacity(ma_uint32 slotCapacity)\n{\n    ma_uint32 cap = slotCapacity / 32;\n    if ((slotCapacity % 32) != 0) {\n        cap += 1;\n    }\n\n    return cap;\n}\n\nstatic MA_INLINE ma_uint32 ma_slot_allocator_group_capacity(const ma_slot_allocator* pAllocator)\n{\n    return ma_slot_allocator_calculate_group_capacity(pAllocator->capacity);\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t groupsOffset;\n    size_t slotsOffset;\n} ma_slot_allocator_heap_layout;\n\nstatic ma_result ma_slot_allocator_get_heap_layout(const ma_slot_allocator_config* pConfig, ma_slot_allocator_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->capacity == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Groups. */\n    pHeapLayout->groupsOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(ma_slot_allocator_calculate_group_capacity(pConfig->capacity) * sizeof(ma_slot_allocator_group));\n\n    /* Slots. */\n    pHeapLayout->slotsOffset  = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(pConfig->capacity * sizeof(ma_uint32));\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_slot_allocator_get_heap_size(const ma_slot_allocator_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_slot_allocator_heap_layout layout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_slot_allocator_get_heap_layout(pConfig, &layout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = layout.sizeInBytes;\n\n    return result;\n}\n\nMA_API ma_result ma_slot_allocator_init_preallocated(const ma_slot_allocator_config* pConfig, void* pHeap, ma_slot_allocator* pAllocator)\n{\n    ma_result result;\n    ma_slot_allocator_heap_layout heapLayout;\n\n    if (pAllocator == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pAllocator);\n\n    if (pHeap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_slot_allocator_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pAllocator->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pAllocator->pGroups  = (ma_slot_allocator_group*)ma_offset_ptr(pHeap, heapLayout.groupsOffset);\n    pAllocator->pSlots   = (ma_uint32*)ma_offset_ptr(pHeap, heapLayout.slotsOffset);\n    pAllocator->capacity = pConfig->capacity;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_slot_allocator_init(const ma_slot_allocator_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_slot_allocator* pAllocator)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_slot_allocator_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the size of the heap allocation. */\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_slot_allocator_init_preallocated(pConfig, pHeap, pAllocator);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pAllocator->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_slot_allocator_uninit(ma_slot_allocator* pAllocator, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocator == NULL) {\n        return;\n    }\n\n    if (pAllocator->_ownsHeap) {\n        ma_free(pAllocator->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_slot_allocator_alloc(ma_slot_allocator* pAllocator, ma_uint64* pSlot)\n{\n    ma_uint32 iAttempt;\n    const ma_uint32 maxAttempts = 2;    /* The number of iterations to perform until returning MA_OUT_OF_MEMORY if no slots can be found. */\n\n    if (pAllocator == NULL || pSlot == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iAttempt = 0; iAttempt < maxAttempts; iAttempt += 1) {\n        /* We need to acquire a suitable bitfield first. This is a bitfield that's got an available slot within it. */\n        ma_uint32 iGroup;\n        for (iGroup = 0; iGroup < ma_slot_allocator_group_capacity(pAllocator); iGroup += 1) {\n            /* CAS */\n            for (;;) {\n                ma_uint32 oldBitfield;\n                ma_uint32 newBitfield;\n                ma_uint32 bitOffset;\n\n                oldBitfield = ma_atomic_load_32(&pAllocator->pGroups[iGroup].bitfield);  /* <-- This copy must happen. The compiler must not optimize this away. */\n\n                /* Fast check to see if anything is available. */\n                if (oldBitfield == 0xFFFFFFFF) {\n                    break;  /* No available bits in this bitfield. */\n                }\n\n                bitOffset = ma_ffs_32(~oldBitfield);\n                MA_ASSERT(bitOffset < 32);\n\n                newBitfield = oldBitfield | (1 << bitOffset);\n\n                if (ma_atomic_compare_and_swap_32(&pAllocator->pGroups[iGroup].bitfield, oldBitfield, newBitfield) == oldBitfield) {\n                    ma_uint32 slotIndex;\n\n                    /* Increment the counter as soon as possible to have other threads report out-of-memory sooner than later. */\n                    ma_atomic_fetch_add_32(&pAllocator->count, 1);\n\n                    /* The slot index is required for constructing the output value. */\n                    slotIndex = (iGroup << 5) + bitOffset;  /* iGroup << 5 = iGroup * 32 */\n                    if (slotIndex >= pAllocator->capacity) {\n                        return MA_OUT_OF_MEMORY;\n                    }\n\n                    /* Increment the reference count before constructing the output value. */\n                    pAllocator->pSlots[slotIndex] += 1;\n\n                    /* Construct the output value. */\n                    *pSlot = (((ma_uint64)pAllocator->pSlots[slotIndex] << 32) | slotIndex);\n\n                    return MA_SUCCESS;\n                }\n            }\n        }\n\n        /* We weren't able to find a slot. If it's because we've reached our capacity we need to return MA_OUT_OF_MEMORY. Otherwise we need to do another iteration and try again. */\n        if (pAllocator->count < pAllocator->capacity) {\n            ma_yield();\n        } else {\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n    /* We couldn't find a slot within the maximum number of attempts. */\n    return MA_OUT_OF_MEMORY;\n}\n\nMA_API ma_result ma_slot_allocator_free(ma_slot_allocator* pAllocator, ma_uint64 slot)\n{\n    ma_uint32 iGroup;\n    ma_uint32 iBit;\n\n    if (pAllocator == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    iGroup = (ma_uint32)((slot & 0xFFFFFFFF) >> 5);   /* slot / 32 */\n    iBit   = (ma_uint32)((slot & 0xFFFFFFFF) & 31);   /* slot % 32 */\n\n    if (iGroup >= ma_slot_allocator_group_capacity(pAllocator)) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(iBit < 32);   /* This must be true due to the logic we used to actually calculate it. */\n\n    while (ma_atomic_load_32(&pAllocator->count) > 0) {\n        /* CAS */\n        ma_uint32 oldBitfield;\n        ma_uint32 newBitfield;\n\n        oldBitfield = ma_atomic_load_32(&pAllocator->pGroups[iGroup].bitfield);  /* <-- This copy must happen. The compiler must not optimize this away. */\n        newBitfield = oldBitfield & ~(1 << iBit);\n\n        /* Debugging for checking for double-frees. */\n        #if defined(MA_DEBUG_OUTPUT)\n        {\n            if ((oldBitfield & (1 << iBit)) == 0) {\n                MA_ASSERT(MA_FALSE);    /* Double free detected.*/\n            }\n        }\n        #endif\n\n        if (ma_atomic_compare_and_swap_32(&pAllocator->pGroups[iGroup].bitfield, oldBitfield, newBitfield) == oldBitfield) {\n            ma_atomic_fetch_sub_32(&pAllocator->count, 1);\n            return MA_SUCCESS;\n        }\n    }\n\n    /* Getting here means there are no allocations available for freeing. */\n    return MA_INVALID_OPERATION;\n}\n\n\n#define MA_JOB_ID_NONE      ~((ma_uint64)0)\n#define MA_JOB_SLOT_NONE    (ma_uint16)(~0)\n\nstatic MA_INLINE ma_uint32 ma_job_extract_refcount(ma_uint64 toc)\n{\n    return (ma_uint32)(toc >> 32);\n}\n\nstatic MA_INLINE ma_uint16 ma_job_extract_slot(ma_uint64 toc)\n{\n    return (ma_uint16)(toc & 0x0000FFFF);\n}\n\n#if 0   /* Currently unused, but might make use of this later. */\nstatic MA_INLINE ma_uint16 ma_job_extract_code(ma_uint64 toc)\n{\n    return (ma_uint16)((toc & 0xFFFF0000) >> 16);\n}\n#endif\n\nstatic MA_INLINE ma_uint64 ma_job_toc_to_allocation(ma_uint64 toc)\n{\n    return ((ma_uint64)ma_job_extract_refcount(toc) << 32) | (ma_uint64)ma_job_extract_slot(toc);\n}\n\nstatic MA_INLINE ma_uint64 ma_job_set_refcount(ma_uint64 toc, ma_uint32 refcount)\n{\n    /* Clear the reference count first. */\n    toc = toc & ~((ma_uint64)0xFFFFFFFF << 32);\n    toc = toc |  ((ma_uint64)refcount   << 32);\n\n    return toc;\n}\n\n\nMA_API ma_job ma_job_init(ma_uint16 code)\n{\n    ma_job job;\n\n    MA_ZERO_OBJECT(&job);\n    job.toc.breakup.code = code;\n    job.toc.breakup.slot = MA_JOB_SLOT_NONE;    /* Temp value. Will be allocated when posted to a queue. */\n    job.next             = MA_JOB_ID_NONE;\n\n    return job;\n}\n\n\nstatic ma_result ma_job_process__noop(ma_job* pJob);\nstatic ma_result ma_job_process__quit(ma_job* pJob);\nstatic ma_result ma_job_process__custom(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__load_data_buffer_node(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__free_data_buffer_node(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__page_data_buffer_node(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__load_data_buffer(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__free_data_buffer(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__load_data_stream(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__free_data_stream(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__page_data_stream(ma_job* pJob);\nstatic ma_result ma_job_process__resource_manager__seek_data_stream(ma_job* pJob);\n\n#if !defined(MA_NO_DEVICE_IO)\nstatic ma_result ma_job_process__device__aaudio_reroute(ma_job* pJob);\n#endif\n\nstatic ma_job_proc g_jobVTable[MA_JOB_TYPE_COUNT] =\n{\n    /* Miscellaneous. */\n    ma_job_process__quit,                                       /* MA_JOB_TYPE_QUIT */\n    ma_job_process__custom,                                     /* MA_JOB_TYPE_CUSTOM */\n\n    /* Resource Manager. */\n    ma_job_process__resource_manager__load_data_buffer_node,    /* MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER_NODE */\n    ma_job_process__resource_manager__free_data_buffer_node,    /* MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER_NODE */\n    ma_job_process__resource_manager__page_data_buffer_node,    /* MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE */\n    ma_job_process__resource_manager__load_data_buffer,         /* MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER */\n    ma_job_process__resource_manager__free_data_buffer,         /* MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER */\n    ma_job_process__resource_manager__load_data_stream,         /* MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_STREAM */\n    ma_job_process__resource_manager__free_data_stream,         /* MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_STREAM */\n    ma_job_process__resource_manager__page_data_stream,         /* MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_STREAM */\n    ma_job_process__resource_manager__seek_data_stream,         /* MA_JOB_TYPE_RESOURCE_MANAGER_SEEK_DATA_STREAM */\n\n    /* Device. */\n#if !defined(MA_NO_DEVICE_IO)\n    ma_job_process__device__aaudio_reroute                      /* MA_JOB_TYPE_DEVICE_AAUDIO_REROUTE */\n#endif\n};\n\nMA_API ma_result ma_job_process(ma_job* pJob)\n{\n    if (pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pJob->toc.breakup.code >= MA_JOB_TYPE_COUNT) {\n        return MA_INVALID_OPERATION;\n    }\n\n    return g_jobVTable[pJob->toc.breakup.code](pJob);\n}\n\nstatic ma_result ma_job_process__noop(ma_job* pJob)\n{\n    MA_ASSERT(pJob != NULL);\n\n    /* No-op. */\n    (void)pJob;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_job_process__quit(ma_job* pJob)\n{\n    return ma_job_process__noop(pJob);\n}\n\nstatic ma_result ma_job_process__custom(ma_job* pJob)\n{\n    MA_ASSERT(pJob != NULL);\n\n    /* No-op if there's no callback. */\n    if (pJob->data.custom.proc == NULL) {\n        return MA_SUCCESS;\n    }\n\n    return pJob->data.custom.proc(pJob);\n}\n\n\n\nMA_API ma_job_queue_config ma_job_queue_config_init(ma_uint32 flags, ma_uint32 capacity)\n{\n    ma_job_queue_config config;\n\n    config.flags    = flags;\n    config.capacity = capacity;\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t allocatorOffset;\n    size_t jobsOffset;\n} ma_job_queue_heap_layout;\n\nstatic ma_result ma_job_queue_get_heap_layout(const ma_job_queue_config* pConfig, ma_job_queue_heap_layout* pHeapLayout)\n{\n    ma_result result;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->capacity == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Allocator. */\n    {\n        ma_slot_allocator_config allocatorConfig;\n        size_t allocatorHeapSizeInBytes;\n\n        allocatorConfig = ma_slot_allocator_config_init(pConfig->capacity);\n        result = ma_slot_allocator_get_heap_size(&allocatorConfig, &allocatorHeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->allocatorOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes    += allocatorHeapSizeInBytes;\n    }\n\n    /* Jobs. */\n    pHeapLayout->jobsOffset   = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(pConfig->capacity * sizeof(ma_job));\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_job_queue_get_heap_size(const ma_job_queue_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_job_queue_heap_layout layout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_job_queue_get_heap_layout(pConfig, &layout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = layout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_job_queue_init_preallocated(const ma_job_queue_config* pConfig, void* pHeap, ma_job_queue* pQueue)\n{\n    ma_result result;\n    ma_job_queue_heap_layout heapLayout;\n    ma_slot_allocator_config allocatorConfig;\n\n    if (pQueue == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pQueue);\n\n    result = ma_job_queue_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pQueue->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pQueue->flags    = pConfig->flags;\n    pQueue->capacity = pConfig->capacity;\n    pQueue->pJobs    = (ma_job*)ma_offset_ptr(pHeap, heapLayout.jobsOffset);\n\n    allocatorConfig = ma_slot_allocator_config_init(pConfig->capacity);\n    result = ma_slot_allocator_init_preallocated(&allocatorConfig, ma_offset_ptr(pHeap, heapLayout.allocatorOffset), &pQueue->allocator);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need a semaphore if we're running in non-blocking mode. If threading is disabled we need to return an error. */\n    if ((pQueue->flags & MA_JOB_QUEUE_FLAG_NON_BLOCKING) == 0) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_semaphore_init(0, &pQueue->sem);\n        }\n        #else\n        {\n            /* Threading is disabled and we've requested non-blocking mode. */\n            return MA_INVALID_OPERATION;\n        }\n        #endif\n    }\n\n    /*\n    Our queue needs to be initialized with a free standing node. This should always be slot 0. Required for the lock free algorithm. The first job in the queue is\n    just a dummy item for giving us the first item in the list which is stored in the \"next\" member.\n    */\n    ma_slot_allocator_alloc(&pQueue->allocator, &pQueue->head);  /* Will never fail. */\n    pQueue->pJobs[ma_job_extract_slot(pQueue->head)].next = MA_JOB_ID_NONE;\n    pQueue->tail = pQueue->head;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_job_queue_init(const ma_job_queue_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_job_queue* pQueue)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_job_queue_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_job_queue_init_preallocated(pConfig, pHeap, pQueue);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pQueue->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_job_queue_uninit(ma_job_queue* pQueue, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pQueue == NULL) {\n        return;\n    }\n\n    /* All we need to do is uninitialize the semaphore. */\n    if ((pQueue->flags & MA_JOB_QUEUE_FLAG_NON_BLOCKING) == 0) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_semaphore_uninit(&pQueue->sem);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never get here. Should have been checked at initialization time. */\n        }\n        #endif\n    }\n\n    ma_slot_allocator_uninit(&pQueue->allocator, pAllocationCallbacks);\n\n    if (pQueue->_ownsHeap) {\n        ma_free(pQueue->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic ma_bool32 ma_job_queue_cas(volatile ma_uint64* dst, ma_uint64 expected, ma_uint64 desired)\n{\n    /* The new counter is taken from the expected value. */\n    return ma_atomic_compare_and_swap_64(dst, expected, ma_job_set_refcount(desired, ma_job_extract_refcount(expected) + 1)) == expected;\n}\n\nMA_API ma_result ma_job_queue_post(ma_job_queue* pQueue, const ma_job* pJob)\n{\n    /*\n    Lock free queue implementation based on the paper by Michael and Scott: Nonblocking Algorithms and Preemption-Safe Locking on Multiprogrammed Shared Memory Multiprocessors\n    */\n    ma_result result;\n    ma_uint64 slot;\n    ma_uint64 tail;\n    ma_uint64 next;\n\n    if (pQueue == NULL || pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We need a new slot. */\n    result = ma_slot_allocator_alloc(&pQueue->allocator, &slot);\n    if (result != MA_SUCCESS) {\n        return result;  /* Probably ran out of slots. If so, MA_OUT_OF_MEMORY will be returned. */\n    }\n\n    /* At this point we should have a slot to place the job. */\n    MA_ASSERT(ma_job_extract_slot(slot) < pQueue->capacity);\n\n    /* We need to put the job into memory before we do anything. */\n    pQueue->pJobs[ma_job_extract_slot(slot)]                  = *pJob;\n    pQueue->pJobs[ma_job_extract_slot(slot)].toc.allocation   = slot;                    /* This will overwrite the job code. */\n    pQueue->pJobs[ma_job_extract_slot(slot)].toc.breakup.code = pJob->toc.breakup.code;  /* The job code needs to be applied again because the line above overwrote it. */\n    pQueue->pJobs[ma_job_extract_slot(slot)].next             = MA_JOB_ID_NONE;          /* Reset for safety. */\n\n    #ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n    ma_spinlock_lock(&pQueue->lock);\n    #endif\n    {\n        /* The job is stored in memory so now we need to add it to our linked list. We only ever add items to the end of the list. */\n        for (;;) {\n            tail = ma_atomic_load_64(&pQueue->tail);\n            next = ma_atomic_load_64(&pQueue->pJobs[ma_job_extract_slot(tail)].next);\n\n            if (ma_job_toc_to_allocation(tail) == ma_job_toc_to_allocation(ma_atomic_load_64(&pQueue->tail))) {\n                if (ma_job_extract_slot(next) == 0xFFFF) {\n                    if (ma_job_queue_cas(&pQueue->pJobs[ma_job_extract_slot(tail)].next, next, slot)) {\n                        break;\n                    }\n                } else {\n                    ma_job_queue_cas(&pQueue->tail, tail, ma_job_extract_slot(next));\n                }\n            }\n        }\n        ma_job_queue_cas(&pQueue->tail, tail, slot);\n    }\n    #ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n    ma_spinlock_unlock(&pQueue->lock);\n    #endif\n\n\n    /* Signal the semaphore as the last step if we're using synchronous mode. */\n    if ((pQueue->flags & MA_JOB_QUEUE_FLAG_NON_BLOCKING) == 0) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_semaphore_release(&pQueue->sem);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never get here. Should have been checked at initialization time. */\n        }\n        #endif\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_job_queue_next(ma_job_queue* pQueue, ma_job* pJob)\n{\n    ma_uint64 head;\n    ma_uint64 tail;\n    ma_uint64 next;\n\n    if (pQueue == NULL || pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If we're running in synchronous mode we'll need to wait on a semaphore. */\n    if ((pQueue->flags & MA_JOB_QUEUE_FLAG_NON_BLOCKING) == 0) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_semaphore_wait(&pQueue->sem);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never get here. Should have been checked at initialization time. */\n        }\n        #endif\n    }\n\n    #ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n    ma_spinlock_lock(&pQueue->lock);\n    #endif\n    {\n        /*\n        BUG: In lock-free mode, multiple threads can be in this section of code. The \"head\" variable in the loop below\n        is stored. One thread can fall through to the freeing of this item while another is still using \"head\" for the\n        retrieval of the \"next\" variable.\n\n        The slot allocator might need to make use of some reference counting to ensure it's only truly freed when\n        there are no more references to the item. This must be fixed before removing these locks.\n        */\n\n        /* Now we need to remove the root item from the list. */\n        for (;;) {\n            head = ma_atomic_load_64(&pQueue->head);\n            tail = ma_atomic_load_64(&pQueue->tail);\n            next = ma_atomic_load_64(&pQueue->pJobs[ma_job_extract_slot(head)].next);\n\n            if (ma_job_toc_to_allocation(head) == ma_job_toc_to_allocation(ma_atomic_load_64(&pQueue->head))) {\n                if (ma_job_extract_slot(head) == ma_job_extract_slot(tail)) {\n                    if (ma_job_extract_slot(next) == 0xFFFF) {\n                        #ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n                        ma_spinlock_unlock(&pQueue->lock);\n                        #endif\n                        return MA_NO_DATA_AVAILABLE;\n                    }\n                    ma_job_queue_cas(&pQueue->tail, tail, ma_job_extract_slot(next));\n                } else {\n                    *pJob = pQueue->pJobs[ma_job_extract_slot(next)];\n                    if (ma_job_queue_cas(&pQueue->head, head, ma_job_extract_slot(next))) {\n                        break;\n                    }\n                }\n            }\n        }\n    }\n    #ifndef MA_USE_EXPERIMENTAL_LOCK_FREE_JOB_QUEUE\n    ma_spinlock_unlock(&pQueue->lock);\n    #endif\n\n    ma_slot_allocator_free(&pQueue->allocator, head);\n\n    /*\n    If it's a quit job make sure it's put back on the queue to ensure other threads have an opportunity to detect it and terminate naturally. We\n    could instead just leave it on the queue, but that would involve fiddling with the lock-free code above and I want to keep that as simple as\n    possible.\n    */\n    if (pJob->toc.breakup.code == MA_JOB_TYPE_QUIT) {\n        ma_job_queue_post(pQueue, pJob);\n        return MA_CANCELLED;    /* Return a cancelled status just in case the thread is checking return codes and not properly checking for a quit job. */\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\n/*******************************************************************************\n\nDynamic Linking\n\n*******************************************************************************/\n/* Disable run-time linking on certain backends and platforms. */\n#ifndef MA_NO_RUNTIME_LINKING\n    #if defined(MA_EMSCRIPTEN) || defined(MA_ORBIS) || defined(MA_PROSPERO) || defined(MA_SWITCH) || defined(MA_DOS)\n        #define MA_NO_RUNTIME_LINKING\n    #endif\n#endif\n\n#ifdef MA_POSIX\n    /* No need for dlfcn.h if we're not using runtime linking. */\n    #ifndef MA_NO_RUNTIME_LINKING\n        #include <dlfcn.h>\n    #endif\n#endif\n\nMA_API ma_handle ma_dlopen(ma_log* pLog, const char* filename)\n{\n    #ifndef MA_NO_RUNTIME_LINKING\n    {\n        ma_handle handle;\n\n        ma_log_postf(pLog, MA_LOG_LEVEL_DEBUG, \"Loading library: %s\\n\", filename);\n\n        #ifdef MA_WIN32\n            /* From MSDN: Desktop applications cannot use LoadPackagedLibrary; if a desktop application calls this function it fails with APPMODEL_ERROR_NO_PACKAGE.*/\n            #if !defined(MA_WIN32_UWP) || !(defined(WINAPI_FAMILY) && ((defined(WINAPI_FAMILY_PHONE_APP) && WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP)))\n                handle = (ma_handle)LoadLibraryA(filename);\n            #else\n                /* *sigh* It appears there is no ANSI version of LoadPackagedLibrary()... */\n                WCHAR filenameW[4096];\n                if (MultiByteToWideChar(CP_UTF8, 0, filename, -1, filenameW, sizeof(filenameW)) == 0) {\n                    handle = NULL;\n                } else {\n                    handle = (ma_handle)LoadPackagedLibrary(filenameW, 0);\n                }\n            #endif\n        #else\n            handle = (ma_handle)dlopen(filename, RTLD_NOW);\n        #endif\n\n        /*\n        I'm not considering failure to load a library an error nor a warning because seamlessly falling through to a lower-priority\n        backend is a deliberate design choice. Instead I'm logging it as an informational message.\n        */\n        if (handle == NULL) {\n            ma_log_postf(pLog, MA_LOG_LEVEL_INFO, \"Failed to load library: %s\\n\", filename);\n        }\n\n        return handle;\n    }\n    #else\n    {\n        /* Runtime linking is disabled. */\n        (void)pLog;\n        (void)filename;\n        return NULL;\n    }\n    #endif\n}\n\nMA_API void ma_dlclose(ma_log* pLog, ma_handle handle)\n{\n    #ifndef MA_NO_RUNTIME_LINKING\n    {\n        #ifdef MA_WIN32\n        {\n            FreeLibrary((HMODULE)handle);\n        }\n        #else\n        {\n            /* Hack for Android bug (see https://github.com/android/ndk/issues/360). Calling dlclose() pre-API 28 may segfault. */\n            #if !defined(MA_ANDROID) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 28)\n            {\n                dlclose((void*)handle);\n            }\n            #else\n            {\n                (void)handle;\n            }\n            #endif\n        }\n        #endif\n\n        (void)pLog;\n    }\n    #else\n    {\n        /* Runtime linking is disabled. */\n        (void)pLog;\n        (void)handle;\n    }\n    #endif\n}\n\nMA_API ma_proc ma_dlsym(ma_log* pLog, ma_handle handle, const char* symbol)\n{\n    #ifndef MA_NO_RUNTIME_LINKING\n    {\n        ma_proc proc;\n\n        ma_log_postf(pLog, MA_LOG_LEVEL_DEBUG, \"Loading symbol: %s\\n\", symbol);\n\n        #ifdef _WIN32\n        {\n            proc = (ma_proc)GetProcAddress((HMODULE)handle, symbol);\n        }\n        #else\n        {\n            #if (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || defined(__clang__)\n                #pragma GCC diagnostic push\n                #pragma GCC diagnostic ignored \"-Wpedantic\"\n            #endif\n                proc = (ma_proc)dlsym((void*)handle, symbol);\n            #if (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || defined(__clang__)\n                #pragma GCC diagnostic pop\n            #endif\n        }\n        #endif\n\n        if (proc == NULL) {\n            ma_log_postf(pLog, MA_LOG_LEVEL_WARNING, \"Failed to load symbol: %s\\n\", symbol);\n        }\n\n        (void)pLog; /* It's possible for pContext to be unused. */\n        return proc;\n    }\n    #else\n    {\n        /* Runtime linking is disabled. */\n        (void)pLog;\n        (void)handle;\n        (void)symbol;\n        return NULL;\n    }\n    #endif\n}\n\n\n\n/************************************************************************************************************************************************************\n*************************************************************************************************************************************************************\n\nDEVICE I/O\n==========\n\n*************************************************************************************************************************************************************\n************************************************************************************************************************************************************/\n\n#ifdef MA_APPLE\n    #include <AvailabilityMacros.h>\n#endif\n\n#ifndef MA_NO_DEVICE_IO\n\n#if defined(MA_APPLE) && (MAC_OS_X_VERSION_MIN_REQUIRED < 101200)\n    #include <mach/mach_time.h> /* For mach_absolute_time() */\n#endif\n\n#ifdef MA_POSIX\n    #include <sys/types.h>\n#endif\n\n/* This must be set to at least 26. */\n#ifndef MA_AAUDIO_MIN_ANDROID_SDK_VERSION\n#define MA_AAUDIO_MIN_ANDROID_SDK_VERSION 27\n#endif\n\n\nMA_API void ma_device_info_add_native_data_format(ma_device_info* pDeviceInfo, ma_format format, ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 flags)\n{\n    if (pDeviceInfo == NULL) {\n        return;\n    }\n\n    if (pDeviceInfo->nativeDataFormatCount < ma_countof(pDeviceInfo->nativeDataFormats)) {\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = format;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = sampleRate;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = flags;\n        pDeviceInfo->nativeDataFormatCount += 1;\n    }\n}\n\n\ntypedef struct\n{\n    ma_backend backend;\n    const char* pName;\n} ma_backend_info;\n\nstatic ma_backend_info gBackendInfo[] = /* Indexed by the backend enum. Must be in the order backends are declared in the ma_backend enum. */\n{\n    {ma_backend_wasapi,     \"WASAPI\"},\n    {ma_backend_dsound,     \"DirectSound\"},\n    {ma_backend_winmm,      \"WinMM\"},\n    {ma_backend_coreaudio,  \"Core Audio\"},\n    {ma_backend_sndio,      \"sndio\"},\n    {ma_backend_audio4,     \"audio(4)\"},\n    {ma_backend_oss,        \"OSS\"},\n    {ma_backend_pulseaudio, \"PulseAudio\"},\n    {ma_backend_alsa,       \"ALSA\"},\n    {ma_backend_jack,       \"JACK\"},\n    {ma_backend_aaudio,     \"AAudio\"},\n    {ma_backend_opensl,     \"OpenSL|ES\"},\n    {ma_backend_webaudio,   \"Web Audio\"},\n    {ma_backend_custom,     \"Custom\"},\n    {ma_backend_null,       \"Null\"}\n};\n\nMA_API const char* ma_get_backend_name(ma_backend backend)\n{\n    if (backend < 0 || backend >= (int)ma_countof(gBackendInfo)) {\n        return \"Unknown\";\n    }\n\n    return gBackendInfo[backend].pName;\n}\n\nMA_API ma_result ma_get_backend_from_name(const char* pBackendName, ma_backend* pBackend)\n{\n    size_t iBackend;\n\n    if (pBackendName == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iBackend = 0; iBackend < ma_countof(gBackendInfo); iBackend += 1) {\n        if (ma_strcmp(pBackendName, gBackendInfo[iBackend].pName) == 0) {\n            if (pBackend != NULL) {\n                *pBackend = gBackendInfo[iBackend].backend;\n            }\n\n            return MA_SUCCESS;\n        }\n    }\n\n    /* Getting here means the backend name is unknown. */\n    return MA_INVALID_ARGS;\n}\n\nMA_API ma_bool32 ma_is_backend_enabled(ma_backend backend)\n{\n    /*\n    This looks a little bit gross, but we want all backends to be included in the switch to avoid warnings on some compilers\n    about some enums not being handled by the switch statement.\n    */\n    switch (backend)\n    {\n        case ma_backend_wasapi:\n        #if defined(MA_HAS_WASAPI)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_dsound:\n        #if defined(MA_HAS_DSOUND)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_winmm:\n        #if defined(MA_HAS_WINMM)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_coreaudio:\n        #if defined(MA_HAS_COREAUDIO)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_sndio:\n        #if defined(MA_HAS_SNDIO)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_audio4:\n        #if defined(MA_HAS_AUDIO4)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_oss:\n        #if defined(MA_HAS_OSS)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_pulseaudio:\n        #if defined(MA_HAS_PULSEAUDIO)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_alsa:\n        #if defined(MA_HAS_ALSA)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_jack:\n        #if defined(MA_HAS_JACK)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_aaudio:\n        #if defined(MA_HAS_AAUDIO)\n            #if defined(MA_ANDROID)\n            {\n                return ma_android_sdk_version() >= MA_AAUDIO_MIN_ANDROID_SDK_VERSION;\n            }\n            #else\n                return MA_FALSE;\n            #endif\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_opensl:\n        #if defined(MA_HAS_OPENSL)\n            #if defined(MA_ANDROID)\n            {\n                return ma_android_sdk_version() >= 9;\n            }\n            #else\n                return MA_TRUE;\n            #endif\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_webaudio:\n        #if defined(MA_HAS_WEBAUDIO)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_custom:\n        #if defined(MA_HAS_CUSTOM)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n        case ma_backend_null:\n        #if defined(MA_HAS_NULL)\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n\n        default: return MA_FALSE;\n    }\n}\n\nMA_API ma_result ma_get_enabled_backends(ma_backend* pBackends, size_t backendCap, size_t* pBackendCount)\n{\n    size_t backendCount;\n    size_t iBackend;\n    ma_result result = MA_SUCCESS;\n\n    if (pBackendCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    backendCount = 0;\n\n    for (iBackend = 0; iBackend <= ma_backend_null; iBackend += 1) {\n        ma_backend backend = (ma_backend)iBackend;\n\n        if (ma_is_backend_enabled(backend)) {\n            /* The backend is enabled. Try adding it to the list. If there's no room, MA_NO_SPACE needs to be returned. */\n            if (backendCount == backendCap) {\n                result = MA_NO_SPACE;\n                break;\n            } else {\n                pBackends[backendCount] = backend;\n                backendCount += 1;\n            }\n        }\n    }\n\n    if (pBackendCount != NULL) {\n        *pBackendCount = backendCount;\n    }\n\n    return result;\n}\n\nMA_API ma_bool32 ma_is_loopback_supported(ma_backend backend)\n{\n    switch (backend)\n    {\n        case ma_backend_wasapi:     return MA_TRUE;\n        case ma_backend_dsound:     return MA_FALSE;\n        case ma_backend_winmm:      return MA_FALSE;\n        case ma_backend_coreaudio:  return MA_FALSE;\n        case ma_backend_sndio:      return MA_FALSE;\n        case ma_backend_audio4:     return MA_FALSE;\n        case ma_backend_oss:        return MA_FALSE;\n        case ma_backend_pulseaudio: return MA_FALSE;\n        case ma_backend_alsa:       return MA_FALSE;\n        case ma_backend_jack:       return MA_FALSE;\n        case ma_backend_aaudio:     return MA_FALSE;\n        case ma_backend_opensl:     return MA_FALSE;\n        case ma_backend_webaudio:   return MA_FALSE;\n        case ma_backend_custom:     return MA_FALSE;    /* <-- Will depend on the implementation of the backend. */\n        case ma_backend_null:       return MA_FALSE;\n        default:                    return MA_FALSE;\n    }\n}\n\n\n\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n/* WASAPI error codes. */\n#define MA_AUDCLNT_E_NOT_INITIALIZED              ((HRESULT)0x88890001)\n#define MA_AUDCLNT_E_ALREADY_INITIALIZED          ((HRESULT)0x88890002)\n#define MA_AUDCLNT_E_WRONG_ENDPOINT_TYPE          ((HRESULT)0x88890003)\n#define MA_AUDCLNT_E_DEVICE_INVALIDATED           ((HRESULT)0x88890004)\n#define MA_AUDCLNT_E_NOT_STOPPED                  ((HRESULT)0x88890005)\n#define MA_AUDCLNT_E_BUFFER_TOO_LARGE             ((HRESULT)0x88890006)\n#define MA_AUDCLNT_E_OUT_OF_ORDER                 ((HRESULT)0x88890007)\n#define MA_AUDCLNT_E_UNSUPPORTED_FORMAT           ((HRESULT)0x88890008)\n#define MA_AUDCLNT_E_INVALID_SIZE                 ((HRESULT)0x88890009)\n#define MA_AUDCLNT_E_DEVICE_IN_USE                ((HRESULT)0x8889000A)\n#define MA_AUDCLNT_E_BUFFER_OPERATION_PENDING     ((HRESULT)0x8889000B)\n#define MA_AUDCLNT_E_THREAD_NOT_REGISTERED        ((HRESULT)0x8889000C)\n#define MA_AUDCLNT_E_NO_SINGLE_PROCESS            ((HRESULT)0x8889000D)\n#define MA_AUDCLNT_E_EXCLUSIVE_MODE_NOT_ALLOWED   ((HRESULT)0x8889000E)\n#define MA_AUDCLNT_E_ENDPOINT_CREATE_FAILED       ((HRESULT)0x8889000F)\n#define MA_AUDCLNT_E_SERVICE_NOT_RUNNING          ((HRESULT)0x88890010)\n#define MA_AUDCLNT_E_EVENTHANDLE_NOT_EXPECTED     ((HRESULT)0x88890011)\n#define MA_AUDCLNT_E_EXCLUSIVE_MODE_ONLY          ((HRESULT)0x88890012)\n#define MA_AUDCLNT_E_BUFDURATION_PERIOD_NOT_EQUAL ((HRESULT)0x88890013)\n#define MA_AUDCLNT_E_EVENTHANDLE_NOT_SET          ((HRESULT)0x88890014)\n#define MA_AUDCLNT_E_INCORRECT_BUFFER_SIZE        ((HRESULT)0x88890015)\n#define MA_AUDCLNT_E_BUFFER_SIZE_ERROR            ((HRESULT)0x88890016)\n#define MA_AUDCLNT_E_CPUUSAGE_EXCEEDED            ((HRESULT)0x88890017)\n#define MA_AUDCLNT_E_BUFFER_ERROR                 ((HRESULT)0x88890018)\n#define MA_AUDCLNT_E_BUFFER_SIZE_NOT_ALIGNED      ((HRESULT)0x88890019)\n#define MA_AUDCLNT_E_INVALID_DEVICE_PERIOD        ((HRESULT)0x88890020)\n#define MA_AUDCLNT_E_INVALID_STREAM_FLAG          ((HRESULT)0x88890021)\n#define MA_AUDCLNT_E_ENDPOINT_OFFLOAD_NOT_CAPABLE ((HRESULT)0x88890022)\n#define MA_AUDCLNT_E_OUT_OF_OFFLOAD_RESOURCES     ((HRESULT)0x88890023)\n#define MA_AUDCLNT_E_OFFLOAD_MODE_ONLY            ((HRESULT)0x88890024)\n#define MA_AUDCLNT_E_NONOFFLOAD_MODE_ONLY         ((HRESULT)0x88890025)\n#define MA_AUDCLNT_E_RESOURCES_INVALIDATED        ((HRESULT)0x88890026)\n#define MA_AUDCLNT_E_RAW_MODE_UNSUPPORTED         ((HRESULT)0x88890027)\n#define MA_AUDCLNT_E_ENGINE_PERIODICITY_LOCKED    ((HRESULT)0x88890028)\n#define MA_AUDCLNT_E_ENGINE_FORMAT_LOCKED         ((HRESULT)0x88890029)\n#define MA_AUDCLNT_E_HEADTRACKING_ENABLED         ((HRESULT)0x88890030)\n#define MA_AUDCLNT_E_HEADTRACKING_UNSUPPORTED     ((HRESULT)0x88890040)\n#define MA_AUDCLNT_S_BUFFER_EMPTY                 ((HRESULT)0x08890001)\n#define MA_AUDCLNT_S_THREAD_ALREADY_REGISTERED    ((HRESULT)0x08890002)\n#define MA_AUDCLNT_S_POSITION_STALLED             ((HRESULT)0x08890003)\n\n#define MA_DS_OK                                  ((HRESULT)0)\n#define MA_DS_NO_VIRTUALIZATION                   ((HRESULT)0x0878000A)\n#define MA_DSERR_ALLOCATED                        ((HRESULT)0x8878000A)\n#define MA_DSERR_CONTROLUNAVAIL                   ((HRESULT)0x8878001E)\n#define MA_DSERR_INVALIDPARAM                     ((HRESULT)0x80070057) /*E_INVALIDARG*/\n#define MA_DSERR_INVALIDCALL                      ((HRESULT)0x88780032)\n#define MA_DSERR_GENERIC                          ((HRESULT)0x80004005) /*E_FAIL*/\n#define MA_DSERR_PRIOLEVELNEEDED                  ((HRESULT)0x88780046)\n#define MA_DSERR_OUTOFMEMORY                      ((HRESULT)0x8007000E) /*E_OUTOFMEMORY*/\n#define MA_DSERR_BADFORMAT                        ((HRESULT)0x88780064)\n#define MA_DSERR_UNSUPPORTED                      ((HRESULT)0x80004001) /*E_NOTIMPL*/\n#define MA_DSERR_NODRIVER                         ((HRESULT)0x88780078)\n#define MA_DSERR_ALREADYINITIALIZED               ((HRESULT)0x88780082)\n#define MA_DSERR_NOAGGREGATION                    ((HRESULT)0x80040110) /*CLASS_E_NOAGGREGATION*/\n#define MA_DSERR_BUFFERLOST                       ((HRESULT)0x88780096)\n#define MA_DSERR_OTHERAPPHASPRIO                  ((HRESULT)0x887800A0)\n#define MA_DSERR_UNINITIALIZED                    ((HRESULT)0x887800AA)\n#define MA_DSERR_NOINTERFACE                      ((HRESULT)0x80004002) /*E_NOINTERFACE*/\n#define MA_DSERR_ACCESSDENIED                     ((HRESULT)0x80070005) /*E_ACCESSDENIED*/\n#define MA_DSERR_BUFFERTOOSMALL                   ((HRESULT)0x887800B4)\n#define MA_DSERR_DS8_REQUIRED                     ((HRESULT)0x887800BE)\n#define MA_DSERR_SENDLOOP                         ((HRESULT)0x887800C8)\n#define MA_DSERR_BADSENDBUFFERGUID                ((HRESULT)0x887800D2)\n#define MA_DSERR_OBJECTNOTFOUND                   ((HRESULT)0x88781161)\n#define MA_DSERR_FXUNAVAILABLE                    ((HRESULT)0x887800DC)\n\nstatic ma_result ma_result_from_HRESULT(HRESULT hr)\n{\n    switch (hr)\n    {\n        case NOERROR:                                   return MA_SUCCESS;\n        /*case S_OK:                                      return MA_SUCCESS;*/\n\n        case E_POINTER:                                 return MA_INVALID_ARGS;\n        case E_UNEXPECTED:                              return MA_ERROR;\n        case E_NOTIMPL:                                 return MA_NOT_IMPLEMENTED;\n        case E_OUTOFMEMORY:                             return MA_OUT_OF_MEMORY;\n        case E_INVALIDARG:                              return MA_INVALID_ARGS;\n        case E_NOINTERFACE:                             return MA_API_NOT_FOUND;\n        case E_HANDLE:                                  return MA_INVALID_ARGS;\n        case E_ABORT:                                   return MA_ERROR;\n        case E_FAIL:                                    return MA_ERROR;\n        case E_ACCESSDENIED:                            return MA_ACCESS_DENIED;\n\n        /* WASAPI */\n        case MA_AUDCLNT_E_NOT_INITIALIZED:              return MA_DEVICE_NOT_INITIALIZED;\n        case MA_AUDCLNT_E_ALREADY_INITIALIZED:          return MA_DEVICE_ALREADY_INITIALIZED;\n        case MA_AUDCLNT_E_WRONG_ENDPOINT_TYPE:          return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_DEVICE_INVALIDATED:           return MA_UNAVAILABLE;\n        case MA_AUDCLNT_E_NOT_STOPPED:                  return MA_DEVICE_NOT_STOPPED;\n        case MA_AUDCLNT_E_BUFFER_TOO_LARGE:             return MA_TOO_BIG;\n        case MA_AUDCLNT_E_OUT_OF_ORDER:                 return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_UNSUPPORTED_FORMAT:           return MA_FORMAT_NOT_SUPPORTED;\n        case MA_AUDCLNT_E_INVALID_SIZE:                 return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_DEVICE_IN_USE:                return MA_BUSY;\n        case MA_AUDCLNT_E_BUFFER_OPERATION_PENDING:     return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_THREAD_NOT_REGISTERED:        return MA_DOES_NOT_EXIST;\n        case MA_AUDCLNT_E_NO_SINGLE_PROCESS:            return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_EXCLUSIVE_MODE_NOT_ALLOWED:   return MA_SHARE_MODE_NOT_SUPPORTED;\n        case MA_AUDCLNT_E_ENDPOINT_CREATE_FAILED:       return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n        case MA_AUDCLNT_E_SERVICE_NOT_RUNNING:          return MA_NOT_CONNECTED;\n        case MA_AUDCLNT_E_EVENTHANDLE_NOT_EXPECTED:     return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_EXCLUSIVE_MODE_ONLY:          return MA_SHARE_MODE_NOT_SUPPORTED;\n        case MA_AUDCLNT_E_BUFDURATION_PERIOD_NOT_EQUAL: return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_EVENTHANDLE_NOT_SET:          return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_INCORRECT_BUFFER_SIZE:        return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_BUFFER_SIZE_ERROR:            return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_CPUUSAGE_EXCEEDED:            return MA_ERROR;\n        case MA_AUDCLNT_E_BUFFER_ERROR:                 return MA_ERROR;\n        case MA_AUDCLNT_E_BUFFER_SIZE_NOT_ALIGNED:      return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_INVALID_DEVICE_PERIOD:        return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_INVALID_STREAM_FLAG:          return MA_INVALID_ARGS;\n        case MA_AUDCLNT_E_ENDPOINT_OFFLOAD_NOT_CAPABLE: return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_OUT_OF_OFFLOAD_RESOURCES:     return MA_OUT_OF_MEMORY;\n        case MA_AUDCLNT_E_OFFLOAD_MODE_ONLY:            return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_NONOFFLOAD_MODE_ONLY:         return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_RESOURCES_INVALIDATED:        return MA_INVALID_DATA;\n        case MA_AUDCLNT_E_RAW_MODE_UNSUPPORTED:         return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_ENGINE_PERIODICITY_LOCKED:    return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_ENGINE_FORMAT_LOCKED:         return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_HEADTRACKING_ENABLED:         return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_E_HEADTRACKING_UNSUPPORTED:     return MA_INVALID_OPERATION;\n        case MA_AUDCLNT_S_BUFFER_EMPTY:                 return MA_NO_SPACE;\n        case MA_AUDCLNT_S_THREAD_ALREADY_REGISTERED:    return MA_ALREADY_EXISTS;\n        case MA_AUDCLNT_S_POSITION_STALLED:             return MA_ERROR;\n\n        /* DirectSound */\n        /*case MA_DS_OK:                                  return MA_SUCCESS;*/          /* S_OK */\n        case MA_DS_NO_VIRTUALIZATION:                   return MA_SUCCESS;\n        case MA_DSERR_ALLOCATED:                        return MA_ALREADY_IN_USE;\n        case MA_DSERR_CONTROLUNAVAIL:                   return MA_INVALID_OPERATION;\n        /*case MA_DSERR_INVALIDPARAM:                    return MA_INVALID_ARGS;*/      /* E_INVALIDARG */\n        case MA_DSERR_INVALIDCALL:                      return MA_INVALID_OPERATION;\n        /*case MA_DSERR_GENERIC:                          return MA_ERROR;*/            /* E_FAIL */\n        case MA_DSERR_PRIOLEVELNEEDED:                  return MA_INVALID_OPERATION;\n        /*case MA_DSERR_OUTOFMEMORY:                      return MA_OUT_OF_MEMORY;*/    /* E_OUTOFMEMORY */\n        case MA_DSERR_BADFORMAT:                        return MA_FORMAT_NOT_SUPPORTED;\n        /*case MA_DSERR_UNSUPPORTED:                      return MA_NOT_IMPLEMENTED;*/  /* E_NOTIMPL */\n        case MA_DSERR_NODRIVER:                         return MA_FAILED_TO_INIT_BACKEND;\n        case MA_DSERR_ALREADYINITIALIZED:               return MA_DEVICE_ALREADY_INITIALIZED;\n        case MA_DSERR_NOAGGREGATION:                    return MA_ERROR;\n        case MA_DSERR_BUFFERLOST:                       return MA_UNAVAILABLE;\n        case MA_DSERR_OTHERAPPHASPRIO:                  return MA_ACCESS_DENIED;\n        case MA_DSERR_UNINITIALIZED:                    return MA_DEVICE_NOT_INITIALIZED;\n        /*case MA_DSERR_NOINTERFACE:                      return MA_API_NOT_FOUND;*/    /* E_NOINTERFACE */\n        /*case MA_DSERR_ACCESSDENIED:                     return MA_ACCESS_DENIED;*/    /* E_ACCESSDENIED */\n        case MA_DSERR_BUFFERTOOSMALL:                   return MA_NO_SPACE;\n        case MA_DSERR_DS8_REQUIRED:                     return MA_INVALID_OPERATION;\n        case MA_DSERR_SENDLOOP:                         return MA_DEADLOCK;\n        case MA_DSERR_BADSENDBUFFERGUID:                return MA_INVALID_ARGS;\n        case MA_DSERR_OBJECTNOTFOUND:                   return MA_NO_DEVICE;\n        case MA_DSERR_FXUNAVAILABLE:                    return MA_UNAVAILABLE;\n\n        default:                                        return MA_ERROR;\n    }\n}\n\n/* PROPVARIANT */\n#define MA_VT_LPWSTR    31\n#define MA_VT_BLOB      65\n\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(push)\n    #pragma warning(disable:4201)   /* nonstandard extension used: nameless struct/union */\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wpedantic\" /* For ISO C99 doesn't support unnamed structs/unions [-Wpedantic] */\n    #if defined(__clang__)\n        #pragma GCC diagnostic ignored \"-Wc11-extensions\"   /* anonymous unions are a C11 extension */\n    #endif\n#endif\ntypedef struct\n{\n    WORD vt;\n    WORD wReserved1;\n    WORD wReserved2;\n    WORD wReserved3;\n    union\n    {\n        struct\n        {\n            ULONG cbSize;\n            BYTE* pBlobData;\n        } blob;\n        WCHAR* pwszVal;\n        char pad[16];   /* Just to ensure the size of the struct matches the official version. */\n    };\n} MA_PROPVARIANT;\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(pop)\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic pop\n#endif\n\ntypedef HRESULT (WINAPI * MA_PFN_CoInitialize)(void* pvReserved);\ntypedef HRESULT (WINAPI * MA_PFN_CoInitializeEx)(void* pvReserved, DWORD  dwCoInit);\ntypedef void    (WINAPI * MA_PFN_CoUninitialize)(void);\ntypedef HRESULT (WINAPI * MA_PFN_CoCreateInstance)(const IID* rclsid, void* pUnkOuter, DWORD dwClsContext, const IID* riid, void* ppv);\ntypedef void    (WINAPI * MA_PFN_CoTaskMemFree)(void* pv);\ntypedef HRESULT (WINAPI * MA_PFN_PropVariantClear)(MA_PROPVARIANT *pvar);\ntypedef int     (WINAPI * MA_PFN_StringFromGUID2)(const GUID* const rguid, WCHAR* lpsz, int cchMax);\n\ntypedef HWND    (WINAPI * MA_PFN_GetForegroundWindow)(void);\ntypedef HWND    (WINAPI * MA_PFN_GetDesktopWindow)(void);\n\n#if defined(MA_WIN32_DESKTOP)\n/* Microsoft documents these APIs as returning LSTATUS, but the Win32 API shipping with some compilers do not define it. It's just a LONG. */\ntypedef LONG    (WINAPI * MA_PFN_RegOpenKeyExA)(HKEY hKey, const char* lpSubKey, DWORD ulOptions, DWORD samDesired, HKEY* phkResult);\ntypedef LONG    (WINAPI * MA_PFN_RegCloseKey)(HKEY hKey);\ntypedef LONG    (WINAPI * MA_PFN_RegQueryValueExA)(HKEY hKey, const char* lpValueName, DWORD* lpReserved, DWORD* lpType, BYTE* lpData, DWORD* lpcbData);\n#endif  /* MA_WIN32_DESKTOP */\n\nstatic GUID MA_GUID_KSDATAFORMAT_SUBTYPE_PCM        = {0x00000001, 0x0000, 0x0010, {0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}};\nstatic GUID MA_GUID_KSDATAFORMAT_SUBTYPE_IEEE_FLOAT = {0x00000003, 0x0000, 0x0010, {0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}};\n/*static GUID MA_GUID_KSDATAFORMAT_SUBTYPE_ALAW       = {0x00000006, 0x0000, 0x0010, {0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}};*/\n/*static GUID MA_GUID_KSDATAFORMAT_SUBTYPE_MULAW      = {0x00000007, 0x0000, 0x0010, {0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}};*/\n\nMA_API size_t ma_strlen_WCHAR(const WCHAR* str)\n{\n    size_t len = 0;\n    while (str[len] != '\\0') {\n        len += 1;\n    }\n\n    return len;\n}\n\nMA_API int ma_strcmp_WCHAR(const WCHAR *s1, const WCHAR *s2)\n{\n    while (*s1 != '\\0' && *s1 == *s2) {\n        s1 += 1;\n        s2 += 1;\n    }\n\n    return *s1 - *s2;\n}\n\nMA_API int ma_strcpy_s_WCHAR(WCHAR* dst, size_t dstCap, const WCHAR* src)\n{\n    size_t i;\n\n    if (dst == 0) {\n        return 22;\n    }\n    if (dstCap == 0) {\n        return 34;\n    }\n    if (src == 0) {\n        dst[0] = '\\0';\n        return 22;\n    }\n\n    for (i = 0; i < dstCap && src[i] != '\\0'; ++i) {\n        dst[i] = src[i];\n    }\n\n    if (i < dstCap) {\n        dst[i] = '\\0';\n        return 0;\n    }\n\n    dst[0] = '\\0';\n    return 34;\n}\n#endif  /* MA_WIN32 */\n\n\n#define MA_DEFAULT_PLAYBACK_DEVICE_NAME    \"Default Playback Device\"\n#define MA_DEFAULT_CAPTURE_DEVICE_NAME     \"Default Capture Device\"\n\n\n\n\n/*******************************************************************************\n\nTiming\n\n*******************************************************************************/\n#if defined(MA_WIN32) && !defined(MA_POSIX)\n    static LARGE_INTEGER g_ma_TimerFrequency;   /* <-- Initialized to zero since it's static. */\n    static MA_INLINE void ma_timer_init(ma_timer* pTimer)\n    {\n        LARGE_INTEGER counter;\n\n        if (g_ma_TimerFrequency.QuadPart == 0) {\n            QueryPerformanceFrequency(&g_ma_TimerFrequency);\n        }\n\n        QueryPerformanceCounter(&counter);\n        pTimer->counter = counter.QuadPart;\n    }\n\n    static MA_INLINE double ma_timer_get_time_in_seconds(ma_timer* pTimer)\n    {\n        LARGE_INTEGER counter;\n        if (!QueryPerformanceCounter(&counter)) {\n            return 0;\n        }\n\n        return (double)(counter.QuadPart - pTimer->counter) / g_ma_TimerFrequency.QuadPart;\n    }\n#elif defined(MA_APPLE) && (MAC_OS_X_VERSION_MIN_REQUIRED < 101200)\n    static ma_uint64 g_ma_TimerFrequency = 0;\n    static MA_INLINE void ma_timer_init(ma_timer* pTimer)\n    {\n        mach_timebase_info_data_t baseTime;\n        mach_timebase_info(&baseTime);\n        g_ma_TimerFrequency = (baseTime.denom * 1e9) / baseTime.numer;\n\n        pTimer->counter = mach_absolute_time();\n    }\n\n    static MA_INLINE double ma_timer_get_time_in_seconds(ma_timer* pTimer)\n    {\n        ma_uint64 newTimeCounter = mach_absolute_time();\n        ma_uint64 oldTimeCounter = pTimer->counter;\n\n        return (newTimeCounter - oldTimeCounter) / g_ma_TimerFrequency;\n    }\n#elif defined(MA_EMSCRIPTEN)\n    static MA_INLINE void ma_timer_init(ma_timer* pTimer)\n    {\n        pTimer->counterD = emscripten_get_now();\n    }\n\n    static MA_INLINE double ma_timer_get_time_in_seconds(ma_timer* pTimer)\n    {\n        return (emscripten_get_now() - pTimer->counterD) / 1000;    /* Emscripten is in milliseconds. */\n    }\n#else\n    #if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 199309L\n        #if defined(CLOCK_MONOTONIC)\n            #define MA_CLOCK_ID CLOCK_MONOTONIC\n        #else\n            #define MA_CLOCK_ID CLOCK_REALTIME\n        #endif\n\n        static MA_INLINE void ma_timer_init(ma_timer* pTimer)\n        {\n            struct timespec newTime;\n            clock_gettime(MA_CLOCK_ID, &newTime);\n\n            pTimer->counter = ((ma_int64)newTime.tv_sec * 1000000000) + newTime.tv_nsec;\n        }\n\n        static MA_INLINE double ma_timer_get_time_in_seconds(ma_timer* pTimer)\n        {\n            ma_uint64 newTimeCounter;\n            ma_uint64 oldTimeCounter;\n\n            struct timespec newTime;\n            clock_gettime(MA_CLOCK_ID, &newTime);\n\n            newTimeCounter = ((ma_uint64)newTime.tv_sec * 1000000000) + newTime.tv_nsec;\n            oldTimeCounter = pTimer->counter;\n\n            return (newTimeCounter - oldTimeCounter) / 1000000000.0;\n        }\n    #else\n        static MA_INLINE void ma_timer_init(ma_timer* pTimer)\n        {\n            struct timeval newTime;\n            gettimeofday(&newTime, NULL);\n\n            pTimer->counter = ((ma_int64)newTime.tv_sec * 1000000) + newTime.tv_usec;\n        }\n\n        static MA_INLINE double ma_timer_get_time_in_seconds(ma_timer* pTimer)\n        {\n            ma_uint64 newTimeCounter;\n            ma_uint64 oldTimeCounter;\n\n            struct timeval newTime;\n            gettimeofday(&newTime, NULL);\n\n            newTimeCounter = ((ma_uint64)newTime.tv_sec * 1000000) + newTime.tv_usec;\n            oldTimeCounter = pTimer->counter;\n\n            return (newTimeCounter - oldTimeCounter) / 1000000.0;\n        }\n    #endif\n#endif\n\n\n\n#if 0\nstatic ma_uint32 ma_get_closest_standard_sample_rate(ma_uint32 sampleRateIn)\n{\n    ma_uint32 closestRate = 0;\n    ma_uint32 closestDiff = 0xFFFFFFFF;\n    size_t iStandardRate;\n\n    for (iStandardRate = 0; iStandardRate < ma_countof(g_maStandardSampleRatePriorities); ++iStandardRate) {\n        ma_uint32 standardRate = g_maStandardSampleRatePriorities[iStandardRate];\n        ma_uint32 diff;\n\n        if (sampleRateIn > standardRate) {\n            diff = sampleRateIn - standardRate;\n        } else {\n            diff = standardRate - sampleRateIn;\n        }\n\n        if (diff == 0) {\n            return standardRate;    /* The input sample rate is a standard rate. */\n        }\n\n        if (closestDiff > diff) {\n            closestDiff = diff;\n            closestRate = standardRate;\n        }\n    }\n\n    return closestRate;\n}\n#endif\n\n\nstatic MA_INLINE unsigned int ma_device_disable_denormals(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (!pDevice->noDisableDenormals) {\n        return ma_disable_denormals();\n    } else {\n        return 0;\n    }\n}\n\nstatic MA_INLINE void ma_device_restore_denormals(ma_device* pDevice, unsigned int prevState)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (!pDevice->noDisableDenormals) {\n        ma_restore_denormals(prevState);\n    } else {\n        /* Do nothing. */\n        (void)prevState;\n    }\n}\n\nstatic ma_device_notification ma_device_notification_init(ma_device* pDevice, ma_device_notification_type type)\n{\n    ma_device_notification notification;\n\n    MA_ZERO_OBJECT(&notification);\n    notification.pDevice = pDevice;\n    notification.type    = type;\n\n    return notification;\n}\n\nstatic void ma_device__on_notification(ma_device_notification notification)\n{\n    MA_ASSERT(notification.pDevice != NULL);\n\n    if (notification.pDevice->onNotification != NULL) {\n        notification.pDevice->onNotification(&notification);\n    }\n\n    /* TEMP FOR COMPATIBILITY: If it's a stopped notification, fire the onStop callback as well. This is only for backwards compatibility and will be removed. */\n    if (notification.pDevice->onStop != NULL && notification.type == ma_device_notification_type_stopped) {\n        notification.pDevice->onStop(notification.pDevice);\n    }\n}\n\nstatic void ma_device__on_notification_started(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_started));\n}\n\nstatic void ma_device__on_notification_stopped(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_stopped));\n}\n\n/* Not all platforms support reroute notifications. */\n#if !defined(MA_EMSCRIPTEN)\nstatic void ma_device__on_notification_rerouted(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_rerouted));\n}\n#endif\n\n#if defined(MA_EMSCRIPTEN)\n#ifdef __cplusplus\nextern \"C\" {\n#endif\nvoid EMSCRIPTEN_KEEPALIVE ma_device__on_notification_unlocked(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_unlocked));\n}\n#ifdef __cplusplus\n}\n#endif\n#endif\n\n\nstatic void ma_device__on_data_inner(ma_device* pDevice, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount)\n{\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pDevice->onData != NULL);\n\n    if (!pDevice->noPreSilencedOutputBuffer && pFramesOut != NULL) {\n        ma_silence_pcm_frames(pFramesOut, frameCount, pDevice->playback.format, pDevice->playback.channels);\n    }\n\n    pDevice->onData(pDevice, pFramesOut, pFramesIn, frameCount);\n}\n\nstatic void ma_device__on_data(ma_device* pDevice, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* Don't read more data from the client if we're in the process of stopping. */\n    if (ma_device_get_state(pDevice) == ma_device_state_stopping) {\n        return;\n    }\n\n    if (pDevice->noFixedSizedCallback) {\n        /* Fast path. Not using a fixed sized callback. Process directly from the specified buffers. */\n        ma_device__on_data_inner(pDevice, pFramesOut, pFramesIn, frameCount);\n    } else {\n        /* Slow path. Using a fixed sized callback. Need to use the intermediary buffer. */\n        ma_uint32 totalFramesProcessed = 0;\n\n        while (totalFramesProcessed < frameCount) {\n            ma_uint32 totalFramesRemaining = frameCount - totalFramesProcessed;\n            ma_uint32 framesToProcessThisIteration = 0;\n\n            if (pFramesIn != NULL) {\n                /* Capturing. Write to the intermediary buffer. If there's no room, fire the callback to empty it. */\n                if (pDevice->capture.intermediaryBufferLen < pDevice->capture.intermediaryBufferCap) {\n                    /* There's some room left in the intermediary buffer. Write to it without firing the callback. */\n                    framesToProcessThisIteration = totalFramesRemaining;\n                    if (framesToProcessThisIteration > pDevice->capture.intermediaryBufferCap - pDevice->capture.intermediaryBufferLen) {\n                        framesToProcessThisIteration = pDevice->capture.intermediaryBufferCap - pDevice->capture.intermediaryBufferLen;\n                    }\n\n                    ma_copy_pcm_frames(\n                        ma_offset_pcm_frames_ptr(pDevice->capture.pIntermediaryBuffer, pDevice->capture.intermediaryBufferLen, pDevice->capture.format, pDevice->capture.channels),\n                        ma_offset_pcm_frames_const_ptr(pFramesIn, totalFramesProcessed, pDevice->capture.format, pDevice->capture.channels),\n                        framesToProcessThisIteration,\n                        pDevice->capture.format, pDevice->capture.channels);\n\n                    pDevice->capture.intermediaryBufferLen += framesToProcessThisIteration;\n                }\n\n                if (pDevice->capture.intermediaryBufferLen == pDevice->capture.intermediaryBufferCap) {\n                    /* No room left in the intermediary buffer. Fire the data callback. */\n                    if (pDevice->type == ma_device_type_duplex) {\n                        /* We'll do the duplex data callback later after we've processed the playback data. */\n                    } else {\n                        ma_device__on_data_inner(pDevice, NULL, pDevice->capture.pIntermediaryBuffer, pDevice->capture.intermediaryBufferCap);\n\n                        /* The intermediary buffer has just been drained. */\n                        pDevice->capture.intermediaryBufferLen = 0;\n                    }\n                }\n            }\n\n            if (pFramesOut != NULL) {\n                /* Playing back. Read from the intermediary buffer. If there's nothing in it, fire the callback to fill it. */\n                if (pDevice->playback.intermediaryBufferLen > 0) {\n                    /* There's some content in the intermediary buffer. Read from that without firing the callback. */\n                    if (pDevice->type == ma_device_type_duplex) {\n                        /* The frames processed this iteration for a duplex device will always be based on the capture side. Leave it unmodified. */\n                    } else {\n                        framesToProcessThisIteration = totalFramesRemaining;\n                        if (framesToProcessThisIteration > pDevice->playback.intermediaryBufferLen) {\n                            framesToProcessThisIteration = pDevice->playback.intermediaryBufferLen;\n                        }\n                    }\n\n                    ma_copy_pcm_frames(\n                        ma_offset_pcm_frames_ptr(pFramesOut, totalFramesProcessed, pDevice->playback.format, pDevice->playback.channels),\n                        ma_offset_pcm_frames_ptr(pDevice->playback.pIntermediaryBuffer, pDevice->playback.intermediaryBufferCap - pDevice->playback.intermediaryBufferLen, pDevice->playback.format, pDevice->playback.channels),\n                        framesToProcessThisIteration,\n                        pDevice->playback.format, pDevice->playback.channels);\n\n                    pDevice->playback.intermediaryBufferLen -= framesToProcessThisIteration;\n                }\n\n                if (pDevice->playback.intermediaryBufferLen == 0) {\n                    /* There's nothing in the intermediary buffer. Fire the data callback to fill it. */\n                    if (pDevice->type == ma_device_type_duplex) {\n                        /* In duplex mode, the data callback will be fired later. Nothing to do here. */\n                    } else {\n                        ma_device__on_data_inner(pDevice, pDevice->playback.pIntermediaryBuffer, NULL, pDevice->playback.intermediaryBufferCap);\n\n                        /* The intermediary buffer has just been filled. */\n                        pDevice->playback.intermediaryBufferLen = pDevice->playback.intermediaryBufferCap;\n                    }\n                }\n            }\n\n            /* If we're in duplex mode we might need to do a refill of the data. */\n            if (pDevice->type == ma_device_type_duplex) {\n                if (pDevice->capture.intermediaryBufferLen == pDevice->capture.intermediaryBufferCap) {\n                    ma_device__on_data_inner(pDevice, pDevice->playback.pIntermediaryBuffer, pDevice->capture.pIntermediaryBuffer, pDevice->capture.intermediaryBufferCap);\n\n                    pDevice->playback.intermediaryBufferLen = pDevice->playback.intermediaryBufferCap;  /* The playback buffer will have just been filled. */\n                    pDevice->capture.intermediaryBufferLen  = 0;                                        /* The intermediary buffer has just been drained. */\n                }\n            }\n\n            /* Make sure this is only incremented once in the duplex case. */\n            totalFramesProcessed += framesToProcessThisIteration;\n        }\n    }\n}\n\nstatic void ma_device__handle_data_callback(ma_device* pDevice, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount)\n{\n    float masterVolumeFactor;\n\n    ma_device_get_master_volume(pDevice, &masterVolumeFactor);  /* Use ma_device_get_master_volume() to ensure the volume is loaded atomically. */\n\n    if (pDevice->onData) {\n        unsigned int prevDenormalState = ma_device_disable_denormals(pDevice);\n        {\n            /* Volume control of input makes things a bit awkward because the input buffer is read-only. We'll need to use a temp buffer and loop in this case. */\n            if (pFramesIn != NULL && masterVolumeFactor != 1) {\n                ma_uint8 tempFramesIn[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                ma_uint32 bpfCapture  = ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels);\n                ma_uint32 bpfPlayback = ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n                ma_uint32 totalFramesProcessed = 0;\n                while (totalFramesProcessed < frameCount) {\n                    ma_uint32 framesToProcessThisIteration = frameCount - totalFramesProcessed;\n                    if (framesToProcessThisIteration > sizeof(tempFramesIn)/bpfCapture) {\n                        framesToProcessThisIteration = sizeof(tempFramesIn)/bpfCapture;\n                    }\n\n                    ma_copy_and_apply_volume_factor_pcm_frames(tempFramesIn, ma_offset_ptr(pFramesIn, totalFramesProcessed*bpfCapture), framesToProcessThisIteration, pDevice->capture.format, pDevice->capture.channels, masterVolumeFactor);\n\n                    ma_device__on_data(pDevice, ma_offset_ptr(pFramesOut, totalFramesProcessed*bpfPlayback), tempFramesIn, framesToProcessThisIteration);\n\n                    totalFramesProcessed += framesToProcessThisIteration;\n                }\n            } else {\n                ma_device__on_data(pDevice, pFramesOut, pFramesIn, frameCount);\n            }\n\n            /* Volume control and clipping for playback devices. */\n            if (pFramesOut != NULL) {\n                if (masterVolumeFactor != 1) {\n                    if (pFramesIn == NULL) {    /* <-- In full-duplex situations, the volume will have been applied to the input samples before the data callback. Applying it again post-callback will incorrectly compound it. */\n                        ma_apply_volume_factor_pcm_frames(pFramesOut, frameCount, pDevice->playback.format, pDevice->playback.channels, masterVolumeFactor);\n                    }\n                }\n\n                if (!pDevice->noClip && pDevice->playback.format == ma_format_f32) {\n                    ma_clip_samples_f32((float*)pFramesOut, (const float*)pFramesOut, frameCount * pDevice->playback.channels);   /* Intentionally specifying the same pointer for both input and output for in-place processing. */\n                }\n            }\n        }\n        ma_device_restore_denormals(pDevice, prevDenormalState);\n    } else {\n        /* No data callback. Just silence the output. */\n        if (pFramesOut != NULL) {\n            ma_silence_pcm_frames(pFramesOut, frameCount, pDevice->playback.format, pDevice->playback.channels);\n        }\n    }\n}\n\n\n\n/* A helper function for reading sample data from the client. */\nstatic void ma_device__read_frames_from_client(ma_device* pDevice, ma_uint32 frameCount, void* pFramesOut)\n{\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(frameCount > 0);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pDevice->playback.converter.isPassthrough) {\n        ma_device__handle_data_callback(pDevice, pFramesOut, NULL, frameCount);\n    } else {\n        ma_result result;\n        ma_uint64 totalFramesReadOut;\n        void* pRunningFramesOut;\n\n        totalFramesReadOut = 0;\n        pRunningFramesOut  = pFramesOut;\n\n        /*\n        We run slightly different logic depending on whether or not we're using a heap-allocated\n        buffer for caching input data. This will be the case if the data converter does not have\n        the ability to retrieve the required input frame count for a given output frame count.\n        */\n        if (pDevice->playback.pInputCache != NULL) {\n            while (totalFramesReadOut < frameCount) {\n                ma_uint64 framesToReadThisIterationIn;\n                ma_uint64 framesToReadThisIterationOut;\n\n                /* If there's any data available in the cache, that needs to get processed first. */\n                if (pDevice->playback.inputCacheRemaining > 0) {\n                    framesToReadThisIterationOut = (frameCount - totalFramesReadOut);\n                    framesToReadThisIterationIn  = framesToReadThisIterationOut;\n                    if (framesToReadThisIterationIn > pDevice->playback.inputCacheRemaining) {\n                        framesToReadThisIterationIn = pDevice->playback.inputCacheRemaining;\n                    }\n\n                    result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, ma_offset_pcm_frames_ptr(pDevice->playback.pInputCache, pDevice->playback.inputCacheConsumed, pDevice->playback.format, pDevice->playback.channels), &framesToReadThisIterationIn, pRunningFramesOut, &framesToReadThisIterationOut);\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n\n                    pDevice->playback.inputCacheConsumed  += framesToReadThisIterationIn;\n                    pDevice->playback.inputCacheRemaining -= framesToReadThisIterationIn;\n\n                    totalFramesReadOut += framesToReadThisIterationOut;\n                    pRunningFramesOut   = ma_offset_ptr(pRunningFramesOut, framesToReadThisIterationOut * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n\n                    if (framesToReadThisIterationIn == 0 && framesToReadThisIterationOut == 0) {\n                        break;  /* We're done. */\n                    }\n                }\n\n                /* Getting here means there's no data in the cache and we need to fill it up with data from the client. */\n                if (pDevice->playback.inputCacheRemaining == 0) {\n                    ma_device__handle_data_callback(pDevice, pDevice->playback.pInputCache, NULL, (ma_uint32)pDevice->playback.inputCacheCap);\n\n                    pDevice->playback.inputCacheConsumed  = 0;\n                    pDevice->playback.inputCacheRemaining = pDevice->playback.inputCacheCap;\n                }\n            }\n        } else {\n            while (totalFramesReadOut < frameCount) {\n                ma_uint8 pIntermediaryBuffer[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In client format. */\n                ma_uint64 intermediaryBufferCap = sizeof(pIntermediaryBuffer) / ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n                ma_uint64 framesToReadThisIterationIn;\n                ma_uint64 framesReadThisIterationIn;\n                ma_uint64 framesToReadThisIterationOut;\n                ma_uint64 framesReadThisIterationOut;\n                ma_uint64 requiredInputFrameCount;\n\n                framesToReadThisIterationOut = (frameCount - totalFramesReadOut);\n                framesToReadThisIterationIn = framesToReadThisIterationOut;\n                if (framesToReadThisIterationIn > intermediaryBufferCap) {\n                    framesToReadThisIterationIn = intermediaryBufferCap;\n                }\n\n                ma_data_converter_get_required_input_frame_count(&pDevice->playback.converter, framesToReadThisIterationOut, &requiredInputFrameCount);\n                if (framesToReadThisIterationIn > requiredInputFrameCount) {\n                    framesToReadThisIterationIn = requiredInputFrameCount;\n                }\n\n                ma_device__handle_data_callback(pDevice, pIntermediaryBuffer, NULL, (ma_uint32)framesToReadThisIterationIn);\n\n                /*\n                At this point we have our decoded data in input format and now we need to convert to output format. Note that even if we didn't read any\n                input frames, we still want to try processing frames because there may some output frames generated from cached input data.\n                */\n                framesReadThisIterationIn  = framesToReadThisIterationIn;\n                framesReadThisIterationOut = framesToReadThisIterationOut;\n                result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, pIntermediaryBuffer, &framesReadThisIterationIn, pRunningFramesOut, &framesReadThisIterationOut);\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n\n                totalFramesReadOut += framesReadThisIterationOut;\n                pRunningFramesOut   = ma_offset_ptr(pRunningFramesOut, framesReadThisIterationOut * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n\n                if (framesReadThisIterationIn == 0 && framesReadThisIterationOut == 0) {\n                    break;  /* We're done. */\n                }\n            }\n        }\n    }\n}\n\n/* A helper for sending sample data to the client. */\nstatic void ma_device__send_frames_to_client(ma_device* pDevice, ma_uint32 frameCountInDeviceFormat, const void* pFramesInDeviceFormat)\n{\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(frameCountInDeviceFormat > 0);\n    MA_ASSERT(pFramesInDeviceFormat != NULL);\n\n    if (pDevice->capture.converter.isPassthrough) {\n        ma_device__handle_data_callback(pDevice, NULL, pFramesInDeviceFormat, frameCountInDeviceFormat);\n    } else {\n        ma_result result;\n        ma_uint8 pFramesInClientFormat[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n        ma_uint64 framesInClientFormatCap = sizeof(pFramesInClientFormat) / ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels);\n        ma_uint64 totalDeviceFramesProcessed = 0;\n        ma_uint64 totalClientFramesProcessed = 0;\n        const void* pRunningFramesInDeviceFormat = pFramesInDeviceFormat;\n\n        /* We just keep going until we've exhausted all of our input frames and cannot generate any more output frames. */\n        for (;;) {\n            ma_uint64 deviceFramesProcessedThisIteration;\n            ma_uint64 clientFramesProcessedThisIteration;\n\n            deviceFramesProcessedThisIteration = (frameCountInDeviceFormat - totalDeviceFramesProcessed);\n            clientFramesProcessedThisIteration = framesInClientFormatCap;\n\n            result = ma_data_converter_process_pcm_frames(&pDevice->capture.converter, pRunningFramesInDeviceFormat, &deviceFramesProcessedThisIteration, pFramesInClientFormat, &clientFramesProcessedThisIteration);\n            if (result != MA_SUCCESS) {\n                break;\n            }\n\n            if (clientFramesProcessedThisIteration > 0) {\n                ma_device__handle_data_callback(pDevice, NULL, pFramesInClientFormat, (ma_uint32)clientFramesProcessedThisIteration);    /* Safe cast. */\n            }\n\n            pRunningFramesInDeviceFormat = ma_offset_ptr(pRunningFramesInDeviceFormat, deviceFramesProcessedThisIteration * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels));\n            totalDeviceFramesProcessed  += deviceFramesProcessedThisIteration;\n            totalClientFramesProcessed  += clientFramesProcessedThisIteration;\n\n            /* This is just to silence a warning. I might want to use this variable later so leaving in place for now. */\n            (void)totalClientFramesProcessed;\n\n            if (deviceFramesProcessedThisIteration == 0 && clientFramesProcessedThisIteration == 0) {\n                break;  /* We're done. */\n            }\n        }\n    }\n}\n\nstatic ma_result ma_device__handle_duplex_callback_capture(ma_device* pDevice, ma_uint32 frameCountInDeviceFormat, const void* pFramesInDeviceFormat, ma_pcm_rb* pRB)\n{\n    ma_result result;\n    ma_uint32 totalDeviceFramesProcessed = 0;\n    const void* pRunningFramesInDeviceFormat = pFramesInDeviceFormat;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(frameCountInDeviceFormat > 0);\n    MA_ASSERT(pFramesInDeviceFormat != NULL);\n    MA_ASSERT(pRB != NULL);\n\n    /* Write to the ring buffer. The ring buffer is in the client format which means we need to convert. */\n    for (;;) {\n        ma_uint32 framesToProcessInDeviceFormat = (frameCountInDeviceFormat - totalDeviceFramesProcessed);\n        ma_uint32 framesToProcessInClientFormat = MA_DATA_CONVERTER_STACK_BUFFER_SIZE / ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels);\n        ma_uint64 framesProcessedInDeviceFormat;\n        ma_uint64 framesProcessedInClientFormat;\n        void* pFramesInClientFormat;\n\n        result = ma_pcm_rb_acquire_write(pRB, &framesToProcessInClientFormat, &pFramesInClientFormat);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"Failed to acquire capture PCM frames from ring buffer.\");\n            break;\n        }\n\n        if (framesToProcessInClientFormat == 0) {\n            if (ma_pcm_rb_pointer_distance(pRB) == (ma_int32)ma_pcm_rb_get_subbuffer_size(pRB)) {\n                break;  /* Overrun. Not enough room in the ring buffer for input frame. Excess frames are dropped. */\n            }\n        }\n\n        /* Convert. */\n        framesProcessedInDeviceFormat = framesToProcessInDeviceFormat;\n        framesProcessedInClientFormat = framesToProcessInClientFormat;\n        result = ma_data_converter_process_pcm_frames(&pDevice->capture.converter, pRunningFramesInDeviceFormat, &framesProcessedInDeviceFormat, pFramesInClientFormat, &framesProcessedInClientFormat);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        result = ma_pcm_rb_commit_write(pRB, (ma_uint32)framesProcessedInClientFormat);  /* Safe cast. */\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"Failed to commit capture PCM frames to ring buffer.\");\n            break;\n        }\n\n        pRunningFramesInDeviceFormat = ma_offset_ptr(pRunningFramesInDeviceFormat, framesProcessedInDeviceFormat * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels));\n        totalDeviceFramesProcessed += (ma_uint32)framesProcessedInDeviceFormat; /* Safe cast. */\n\n        /* We're done when we're unable to process any client nor device frames. */\n        if (framesProcessedInClientFormat == 0 && framesProcessedInDeviceFormat == 0) {\n            break;  /* Done. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device__handle_duplex_callback_playback(ma_device* pDevice, ma_uint32 frameCount, void* pFramesInInternalFormat, ma_pcm_rb* pRB)\n{\n    ma_result result;\n    ma_uint8 silentInputFrames[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n    ma_uint32 totalFramesReadOut = 0;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(frameCount > 0);\n    MA_ASSERT(pFramesInInternalFormat != NULL);\n    MA_ASSERT(pRB != NULL);\n    MA_ASSERT(pDevice->playback.pInputCache != NULL);\n\n    /*\n    Sitting in the ring buffer should be captured data from the capture callback in external format. If there's not enough data in there for\n    the whole frameCount frames we just use silence instead for the input data.\n    */\n    MA_ZERO_MEMORY(silentInputFrames, sizeof(silentInputFrames));\n\n    while (totalFramesReadOut < frameCount && ma_device_is_started(pDevice)) {\n        /*\n        We should have a buffer allocated on the heap. Any playback frames still sitting in there\n        need to be sent to the internal device before we process any more data from the client.\n        */\n        if (pDevice->playback.inputCacheRemaining > 0) {\n            ma_uint64 framesConvertedIn  = pDevice->playback.inputCacheRemaining;\n            ma_uint64 framesConvertedOut = (frameCount - totalFramesReadOut);\n            ma_data_converter_process_pcm_frames(&pDevice->playback.converter, ma_offset_pcm_frames_ptr(pDevice->playback.pInputCache, pDevice->playback.inputCacheConsumed, pDevice->playback.format, pDevice->playback.channels), &framesConvertedIn, pFramesInInternalFormat, &framesConvertedOut);\n\n            pDevice->playback.inputCacheConsumed  += framesConvertedIn;\n            pDevice->playback.inputCacheRemaining -= framesConvertedIn;\n\n            totalFramesReadOut        += (ma_uint32)framesConvertedOut; /* Safe cast. */\n            pFramesInInternalFormat    = ma_offset_ptr(pFramesInInternalFormat, framesConvertedOut * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n        }\n\n        /* If there's no more data in the cache we'll need to fill it with some. */\n        if (totalFramesReadOut < frameCount && pDevice->playback.inputCacheRemaining == 0) {\n            ma_uint32 inputFrameCount;\n            void* pInputFrames;\n\n            inputFrameCount = (ma_uint32)pDevice->playback.inputCacheCap;\n            result = ma_pcm_rb_acquire_read(pRB, &inputFrameCount, &pInputFrames);\n            if (result == MA_SUCCESS) {\n                if (inputFrameCount > 0) {\n                    ma_device__handle_data_callback(pDevice, pDevice->playback.pInputCache, pInputFrames, inputFrameCount);\n                } else {\n                    if (ma_pcm_rb_pointer_distance(pRB) == 0) {\n                        break;  /* Underrun. */\n                    }\n                }\n            } else {\n                /* No capture data available. Feed in silence. */\n                inputFrameCount = (ma_uint32)ma_min(pDevice->playback.inputCacheCap, sizeof(silentInputFrames) / ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels));\n                ma_device__handle_data_callback(pDevice, pDevice->playback.pInputCache, silentInputFrames, inputFrameCount);\n            }\n\n            pDevice->playback.inputCacheConsumed  = 0;\n            pDevice->playback.inputCacheRemaining = inputFrameCount;\n\n            result = ma_pcm_rb_commit_read(pRB, inputFrameCount);\n            if (result != MA_SUCCESS) {\n                return result;  /* Should never happen. */\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n/* A helper for changing the state of the device. */\nstatic MA_INLINE void ma_device__set_state(ma_device* pDevice, ma_device_state newState)\n{\n    ma_atomic_device_state_set(&pDevice->state, newState);\n}\n\n\n\nMA_API ma_uint32 ma_get_format_priority_index(ma_format format) /* Lower = better. */\n{\n    ma_uint32 i;\n    for (i = 0; i < ma_countof(g_maFormatPriorities); ++i) {\n        if (g_maFormatPriorities[i] == format) {\n            return i;\n        }\n    }\n\n    /* Getting here means the format could not be found or is equal to ma_format_unknown. */\n    return (ma_uint32)-1;\n}\n\nstatic ma_result ma_device__post_init_setup(ma_device* pDevice, ma_device_type deviceType);\n\nstatic ma_bool32 ma_device_descriptor_is_valid(const ma_device_descriptor* pDeviceDescriptor)\n{\n    if (pDeviceDescriptor == NULL) {\n        return MA_FALSE;\n    }\n\n    if (pDeviceDescriptor->format == ma_format_unknown) {\n        return MA_FALSE;\n    }\n\n    if (pDeviceDescriptor->channels == 0 || pDeviceDescriptor->channels > MA_MAX_CHANNELS) {\n        return MA_FALSE;\n    }\n\n    if (pDeviceDescriptor->sampleRate == 0) {\n        return MA_FALSE;\n    }\n\n    return MA_TRUE;\n}\n\n\nstatic ma_result ma_device_audio_thread__default_read_write(ma_device* pDevice)\n{\n    ma_result result = MA_SUCCESS;\n    ma_bool32 exitLoop = MA_FALSE;\n    ma_uint8  capturedDeviceData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n    ma_uint8  playbackDeviceData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n    ma_uint32 capturedDeviceDataCapInFrames = 0;\n    ma_uint32 playbackDeviceDataCapInFrames = 0;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* Just some quick validation on the device type and the available callbacks. */\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n        if (pDevice->pContext->callbacks.onDeviceRead == NULL) {\n            return MA_NOT_IMPLEMENTED;\n        }\n\n        capturedDeviceDataCapInFrames = sizeof(capturedDeviceData) / ma_get_bytes_per_frame(pDevice->capture.internalFormat,  pDevice->capture.internalChannels);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->pContext->callbacks.onDeviceWrite == NULL) {\n            return MA_NOT_IMPLEMENTED;\n        }\n\n        playbackDeviceDataCapInFrames = sizeof(playbackDeviceData) / ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    }\n\n    /* NOTE: The device was started outside of this function, in the worker thread. */\n\n    while (ma_device_get_state(pDevice) == ma_device_state_started && !exitLoop) {\n        switch (pDevice->type) {\n            case ma_device_type_duplex:\n            {\n                /* The process is: onDeviceRead() -> convert -> callback -> convert -> onDeviceWrite() */\n                ma_uint32 totalCapturedDeviceFramesProcessed = 0;\n                ma_uint32 capturedDevicePeriodSizeInFrames = ma_min(pDevice->capture.internalPeriodSizeInFrames, pDevice->playback.internalPeriodSizeInFrames);\n\n                while (totalCapturedDeviceFramesProcessed < capturedDevicePeriodSizeInFrames) {\n                    ma_uint32 capturedDeviceFramesRemaining;\n                    ma_uint32 capturedDeviceFramesProcessed;\n                    ma_uint32 capturedDeviceFramesToProcess;\n                    ma_uint32 capturedDeviceFramesToTryProcessing = capturedDevicePeriodSizeInFrames - totalCapturedDeviceFramesProcessed;\n                    if (capturedDeviceFramesToTryProcessing > capturedDeviceDataCapInFrames) {\n                        capturedDeviceFramesToTryProcessing = capturedDeviceDataCapInFrames;\n                    }\n\n                    result = pDevice->pContext->callbacks.onDeviceRead(pDevice, capturedDeviceData, capturedDeviceFramesToTryProcessing, &capturedDeviceFramesToProcess);\n                    if (result != MA_SUCCESS) {\n                        exitLoop = MA_TRUE;\n                        break;\n                    }\n\n                    capturedDeviceFramesRemaining = capturedDeviceFramesToProcess;\n                    capturedDeviceFramesProcessed = 0;\n\n                    /* At this point we have our captured data in device format and we now need to convert it to client format. */\n                    for (;;) {\n                        ma_uint8  capturedClientData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                        ma_uint8  playbackClientData[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                        ma_uint32 capturedClientDataCapInFrames = sizeof(capturedClientData) / ma_get_bytes_per_frame(pDevice->capture.format,  pDevice->capture.channels);\n                        ma_uint32 playbackClientDataCapInFrames = sizeof(playbackClientData) / ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n                        ma_uint64 capturedClientFramesToProcessThisIteration = ma_min(capturedClientDataCapInFrames, playbackClientDataCapInFrames);\n                        ma_uint64 capturedDeviceFramesToProcessThisIteration = capturedDeviceFramesRemaining;\n                        ma_uint8* pRunningCapturedDeviceFrames = ma_offset_ptr(capturedDeviceData, capturedDeviceFramesProcessed * ma_get_bytes_per_frame(pDevice->capture.internalFormat,  pDevice->capture.internalChannels));\n\n                        /* Convert capture data from device format to client format. */\n                        result = ma_data_converter_process_pcm_frames(&pDevice->capture.converter, pRunningCapturedDeviceFrames, &capturedDeviceFramesToProcessThisIteration, capturedClientData, &capturedClientFramesToProcessThisIteration);\n                        if (result != MA_SUCCESS) {\n                            break;\n                        }\n\n                        /*\n                        If we weren't able to generate any output frames it must mean we've exhausted all of our input. The only time this would not be the case is if capturedClientData was too small\n                        which should never be the case when it's of the size MA_DATA_CONVERTER_STACK_BUFFER_SIZE.\n                        */\n                        if (capturedClientFramesToProcessThisIteration == 0) {\n                            break;\n                        }\n\n                        ma_device__handle_data_callback(pDevice, playbackClientData, capturedClientData, (ma_uint32)capturedClientFramesToProcessThisIteration);    /* Safe cast .*/\n\n                        capturedDeviceFramesProcessed += (ma_uint32)capturedDeviceFramesToProcessThisIteration; /* Safe cast. */\n                        capturedDeviceFramesRemaining -= (ma_uint32)capturedDeviceFramesToProcessThisIteration; /* Safe cast. */\n\n                        /* At this point the playbackClientData buffer should be holding data that needs to be written to the device. */\n                        for (;;) {\n                            ma_uint64 convertedClientFrameCount = capturedClientFramesToProcessThisIteration;\n                            ma_uint64 convertedDeviceFrameCount = playbackDeviceDataCapInFrames;\n                            result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, playbackClientData, &convertedClientFrameCount, playbackDeviceData, &convertedDeviceFrameCount);\n                            if (result != MA_SUCCESS) {\n                                break;\n                            }\n\n                            result = pDevice->pContext->callbacks.onDeviceWrite(pDevice, playbackDeviceData, (ma_uint32)convertedDeviceFrameCount, NULL);   /* Safe cast. */\n                            if (result != MA_SUCCESS) {\n                                exitLoop = MA_TRUE;\n                                break;\n                            }\n\n                            capturedClientFramesToProcessThisIteration -= (ma_uint32)convertedClientFrameCount;  /* Safe cast. */\n                            if (capturedClientFramesToProcessThisIteration == 0) {\n                                break;\n                            }\n                        }\n\n                        /* In case an error happened from ma_device_write__null()... */\n                        if (result != MA_SUCCESS) {\n                            exitLoop = MA_TRUE;\n                            break;\n                        }\n                    }\n\n                    /* Make sure we don't get stuck in the inner loop. */\n                    if (capturedDeviceFramesProcessed == 0) {\n                        break;\n                    }\n\n                    totalCapturedDeviceFramesProcessed += capturedDeviceFramesProcessed;\n                }\n            } break;\n\n            case ma_device_type_capture:\n            case ma_device_type_loopback:\n            {\n                ma_uint32 periodSizeInFrames = pDevice->capture.internalPeriodSizeInFrames;\n                ma_uint32 framesReadThisPeriod = 0;\n                while (framesReadThisPeriod < periodSizeInFrames) {\n                    ma_uint32 framesRemainingInPeriod = periodSizeInFrames - framesReadThisPeriod;\n                    ma_uint32 framesProcessed;\n                    ma_uint32 framesToReadThisIteration = framesRemainingInPeriod;\n                    if (framesToReadThisIteration > capturedDeviceDataCapInFrames) {\n                        framesToReadThisIteration = capturedDeviceDataCapInFrames;\n                    }\n\n                    result = pDevice->pContext->callbacks.onDeviceRead(pDevice, capturedDeviceData, framesToReadThisIteration, &framesProcessed);\n                    if (result != MA_SUCCESS) {\n                        exitLoop = MA_TRUE;\n                        break;\n                    }\n\n                    /* Make sure we don't get stuck in the inner loop. */\n                    if (framesProcessed == 0) {\n                        break;\n                    }\n\n                    ma_device__send_frames_to_client(pDevice, framesProcessed, capturedDeviceData);\n\n                    framesReadThisPeriod += framesProcessed;\n                }\n            } break;\n\n            case ma_device_type_playback:\n            {\n                /* We write in chunks of the period size, but use a stack allocated buffer for the intermediary. */\n                ma_uint32 periodSizeInFrames = pDevice->playback.internalPeriodSizeInFrames;\n                ma_uint32 framesWrittenThisPeriod = 0;\n                while (framesWrittenThisPeriod < periodSizeInFrames) {\n                    ma_uint32 framesRemainingInPeriod = periodSizeInFrames - framesWrittenThisPeriod;\n                    ma_uint32 framesProcessed;\n                    ma_uint32 framesToWriteThisIteration = framesRemainingInPeriod;\n                    if (framesToWriteThisIteration > playbackDeviceDataCapInFrames) {\n                        framesToWriteThisIteration = playbackDeviceDataCapInFrames;\n                    }\n\n                    ma_device__read_frames_from_client(pDevice, framesToWriteThisIteration, playbackDeviceData);\n\n                    result = pDevice->pContext->callbacks.onDeviceWrite(pDevice, playbackDeviceData, framesToWriteThisIteration, &framesProcessed);\n                    if (result != MA_SUCCESS) {\n                        exitLoop = MA_TRUE;\n                        break;\n                    }\n\n                    /* Make sure we don't get stuck in the inner loop. */\n                    if (framesProcessed == 0) {\n                        break;\n                    }\n\n                    framesWrittenThisPeriod += framesProcessed;\n                }\n            } break;\n\n            /* Should never get here. */\n            default: break;\n        }\n    }\n\n    return result;\n}\n\n\n\n/*******************************************************************************\n\nNull Backend\n\n*******************************************************************************/\n#ifdef MA_HAS_NULL\n\n#define MA_DEVICE_OP_NONE__NULL    0\n#define MA_DEVICE_OP_START__NULL   1\n#define MA_DEVICE_OP_SUSPEND__NULL 2\n#define MA_DEVICE_OP_KILL__NULL    3\n\nstatic ma_thread_result MA_THREADCALL ma_device_thread__null(void* pData)\n{\n    ma_device* pDevice = (ma_device*)pData;\n    MA_ASSERT(pDevice != NULL);\n\n    for (;;) {  /* Keep the thread alive until the device is uninitialized. */\n        ma_uint32 operation;\n\n        /* Wait for an operation to be requested. */\n        ma_event_wait(&pDevice->null_device.operationEvent);\n\n        /* At this point an event should have been triggered. */\n        operation = pDevice->null_device.operation;\n\n        /* Starting the device needs to put the thread into a loop. */\n        if (operation == MA_DEVICE_OP_START__NULL) {\n            /* Reset the timer just in case. */\n            ma_timer_init(&pDevice->null_device.timer);\n\n            /* Getting here means a suspend or kill operation has been requested. */\n            pDevice->null_device.operationResult = MA_SUCCESS;\n            ma_event_signal(&pDevice->null_device.operationCompletionEvent);\n            ma_semaphore_release(&pDevice->null_device.operationSemaphore);\n            continue;\n        }\n\n        /* Suspending the device means we need to stop the timer and just continue the loop. */\n        if (operation == MA_DEVICE_OP_SUSPEND__NULL) {\n            /* We need to add the current run time to the prior run time, then reset the timer. */\n            pDevice->null_device.priorRunTime += ma_timer_get_time_in_seconds(&pDevice->null_device.timer);\n            ma_timer_init(&pDevice->null_device.timer);\n\n            /* We're done. */\n            pDevice->null_device.operationResult = MA_SUCCESS;\n            ma_event_signal(&pDevice->null_device.operationCompletionEvent);\n            ma_semaphore_release(&pDevice->null_device.operationSemaphore);\n            continue;\n        }\n\n        /* Killing the device means we need to get out of this loop so that this thread can terminate. */\n        if (operation == MA_DEVICE_OP_KILL__NULL) {\n            pDevice->null_device.operationResult = MA_SUCCESS;\n            ma_event_signal(&pDevice->null_device.operationCompletionEvent);\n            ma_semaphore_release(&pDevice->null_device.operationSemaphore);\n            break;\n        }\n\n        /* Getting a signal on a \"none\" operation probably means an error. Return invalid operation. */\n        if (operation == MA_DEVICE_OP_NONE__NULL) {\n            MA_ASSERT(MA_FALSE);  /* <-- Trigger this in debug mode to ensure developers are aware they're doing something wrong (or there's a bug in a miniaudio). */\n            pDevice->null_device.operationResult = MA_INVALID_OPERATION;\n            ma_event_signal(&pDevice->null_device.operationCompletionEvent);\n            ma_semaphore_release(&pDevice->null_device.operationSemaphore);\n            continue;   /* Continue the loop. Don't terminate. */\n        }\n    }\n\n    return (ma_thread_result)0;\n}\n\nstatic ma_result ma_device_do_operation__null(ma_device* pDevice, ma_uint32 operation)\n{\n    ma_result result;\n\n    /*\n    TODO: Need to review this and consider just using mutual exclusion. I think the original motivation\n    for this was to just post the event to a queue and return immediately, but that has since changed\n    and now this function is synchronous. I think this can be simplified to just use a mutex.\n    */\n\n    /*\n    The first thing to do is wait for an operation slot to become available. We only have a single slot for this, but we could extend this later\n    to support queuing of operations.\n    */\n    result = ma_semaphore_wait(&pDevice->null_device.operationSemaphore);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to wait for the event. */\n    }\n\n    /*\n    When we get here it means the background thread is not referencing the operation code and it can be changed. After changing this we need to\n    signal an event to the worker thread to let it know that it can start work.\n    */\n    pDevice->null_device.operation = operation;\n\n    /* Once the operation code has been set, the worker thread can start work. */\n    if (ma_event_signal(&pDevice->null_device.operationEvent) != MA_SUCCESS) {\n        return MA_ERROR;\n    }\n\n    /* We want everything to be synchronous so we're going to wait for the worker thread to complete it's operation. */\n    if (ma_event_wait(&pDevice->null_device.operationCompletionEvent) != MA_SUCCESS) {\n        return MA_ERROR;\n    }\n\n    return pDevice->null_device.operationResult;\n}\n\nstatic ma_uint64 ma_device_get_total_run_time_in_frames__null(ma_device* pDevice)\n{\n    ma_uint32 internalSampleRate;\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        internalSampleRate = pDevice->capture.internalSampleRate;\n    } else {\n        internalSampleRate = pDevice->playback.internalSampleRate;\n    }\n\n    return (ma_uint64)((pDevice->null_device.priorRunTime + ma_timer_get_time_in_seconds(&pDevice->null_device.timer)) * internalSampleRate);\n}\n\nstatic ma_result ma_context_enumerate_devices__null(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 cbResult = MA_TRUE;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Playback. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), \"NULL Playback Device\", (size_t)-1);\n        deviceInfo.isDefault = MA_TRUE; /* Only one playback and capture device for the null backend, so might as well mark as default. */\n        cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n    }\n\n    /* Capture. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), \"NULL Capture Device\", (size_t)-1);\n        deviceInfo.isDefault = MA_TRUE; /* Only one playback and capture device for the null backend, so might as well mark as default. */\n        cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n    }\n\n    (void)cbResult; /* Silence a static analysis warning. */\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__null(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pContext != NULL);\n\n    if (pDeviceID != NULL && pDeviceID->nullbackend != 0) {\n        return MA_NO_DEVICE;   /* Don't know the device. */\n    }\n\n    /* Name / Description */\n    if (deviceType == ma_device_type_playback) {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), \"NULL Playback Device\", (size_t)-1);\n    } else {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), \"NULL Capture Device\", (size_t)-1);\n    }\n\n    pDeviceInfo->isDefault = MA_TRUE;   /* Only one playback and capture device for the null backend, so might as well mark as default. */\n\n    /* Support everything on the null backend. */\n    pDeviceInfo->nativeDataFormats[0].format     = ma_format_unknown;\n    pDeviceInfo->nativeDataFormats[0].channels   = 0;\n    pDeviceInfo->nativeDataFormats[0].sampleRate = 0;\n    pDeviceInfo->nativeDataFormats[0].flags      = 0;\n    pDeviceInfo->nativeDataFormatCount = 1;\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_uninit__null(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* Keep it clean and wait for the device thread to finish before returning. */\n    ma_device_do_operation__null(pDevice, MA_DEVICE_OP_KILL__NULL);\n\n    /* Wait for the thread to finish before continuing. */\n    ma_thread_wait(&pDevice->null_device.deviceThread);\n\n    /* At this point the loop in the device thread is as good as terminated so we can uninitialize our events. */\n    ma_semaphore_uninit(&pDevice->null_device.operationSemaphore);\n    ma_event_uninit(&pDevice->null_device.operationCompletionEvent);\n    ma_event_uninit(&pDevice->null_device.operationEvent);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__null(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->null_device);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* The null backend supports everything exactly as we specify it. */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        pDescriptorCapture->format     = (pDescriptorCapture->format     != ma_format_unknown) ? pDescriptorCapture->format     : MA_DEFAULT_FORMAT;\n        pDescriptorCapture->channels   = (pDescriptorCapture->channels   != 0)                 ? pDescriptorCapture->channels   : MA_DEFAULT_CHANNELS;\n        pDescriptorCapture->sampleRate = (pDescriptorCapture->sampleRate != 0)                 ? pDescriptorCapture->sampleRate : MA_DEFAULT_SAMPLE_RATE;\n\n        if (pDescriptorCapture->channelMap[0] == MA_CHANNEL_NONE) {\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorCapture->channels);\n        }\n\n        pDescriptorCapture->periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptorCapture, pDescriptorCapture->sampleRate, pConfig->performanceProfile);\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        pDescriptorPlayback->format     = (pDescriptorPlayback->format     != ma_format_unknown) ? pDescriptorPlayback->format     : MA_DEFAULT_FORMAT;\n        pDescriptorPlayback->channels   = (pDescriptorPlayback->channels   != 0)                 ? pDescriptorPlayback->channels   : MA_DEFAULT_CHANNELS;\n        pDescriptorPlayback->sampleRate = (pDescriptorPlayback->sampleRate != 0)                 ? pDescriptorPlayback->sampleRate : MA_DEFAULT_SAMPLE_RATE;\n\n        if (pDescriptorPlayback->channelMap[0] == MA_CHANNEL_NONE) {\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pDescriptorPlayback->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorPlayback->channels);\n        }\n\n        pDescriptorPlayback->periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptorPlayback, pDescriptorPlayback->sampleRate, pConfig->performanceProfile);\n    }\n\n    /*\n    In order to get timing right, we need to create a thread that does nothing but keeps track of the timer. This timer is started when the\n    first period is \"written\" to it, and then stopped in ma_device_stop__null().\n    */\n    result = ma_event_init(&pDevice->null_device.operationEvent);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_event_init(&pDevice->null_device.operationCompletionEvent);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_semaphore_init(1, &pDevice->null_device.operationSemaphore);    /* <-- It's important that the initial value is set to 1. */\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_thread_create(&pDevice->null_device.deviceThread, pDevice->pContext->threadPriority, 0, ma_device_thread__null, pDevice, &pDevice->pContext->allocationCallbacks);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__null(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    ma_device_do_operation__null(pDevice, MA_DEVICE_OP_START__NULL);\n\n    ma_atomic_bool32_set(&pDevice->null_device.isStarted, MA_TRUE);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__null(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    ma_device_do_operation__null(pDevice, MA_DEVICE_OP_SUSPEND__NULL);\n\n    ma_atomic_bool32_set(&pDevice->null_device.isStarted, MA_FALSE);\n    return MA_SUCCESS;\n}\n\nstatic ma_bool32 ma_device_is_started__null(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    return ma_atomic_bool32_get(&pDevice->null_device.isStarted);\n}\n\nstatic ma_result ma_device_write__null(ma_device* pDevice, const void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 totalPCMFramesProcessed;\n    ma_bool32 wasStartedOnEntry;\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    wasStartedOnEntry = ma_device_is_started__null(pDevice);\n\n    /* Keep going until everything has been read. */\n    totalPCMFramesProcessed = 0;\n    while (totalPCMFramesProcessed < frameCount) {\n        ma_uint64 targetFrame;\n\n        /* If there are any frames remaining in the current period, consume those first. */\n        if (pDevice->null_device.currentPeriodFramesRemainingPlayback > 0) {\n            ma_uint32 framesRemaining = (frameCount - totalPCMFramesProcessed);\n            ma_uint32 framesToProcess = pDevice->null_device.currentPeriodFramesRemainingPlayback;\n            if (framesToProcess > framesRemaining) {\n                framesToProcess = framesRemaining;\n            }\n\n            /* We don't actually do anything with pPCMFrames, so just mark it as unused to prevent a warning. */\n            (void)pPCMFrames;\n\n            pDevice->null_device.currentPeriodFramesRemainingPlayback -= framesToProcess;\n            totalPCMFramesProcessed += framesToProcess;\n        }\n\n        /* If we've consumed the current period we'll need to mark it as such an ensure the device is started if it's not already. */\n        if (pDevice->null_device.currentPeriodFramesRemainingPlayback == 0) {\n            pDevice->null_device.currentPeriodFramesRemainingPlayback = 0;\n\n            if (!ma_device_is_started__null(pDevice) && !wasStartedOnEntry) {\n                result = ma_device_start__null(pDevice);\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n            }\n        }\n\n        /* If we've consumed the whole buffer we can return now. */\n        MA_ASSERT(totalPCMFramesProcessed <= frameCount);\n        if (totalPCMFramesProcessed == frameCount) {\n            break;\n        }\n\n        /* Getting here means we've still got more frames to consume, we but need to wait for it to become available. */\n        targetFrame = pDevice->null_device.lastProcessedFramePlayback;\n        for (;;) {\n            ma_uint64 currentFrame;\n\n            /* Stop waiting if the device has been stopped. */\n            if (!ma_device_is_started__null(pDevice)) {\n                break;\n            }\n\n            currentFrame = ma_device_get_total_run_time_in_frames__null(pDevice);\n            if (currentFrame >= targetFrame) {\n                break;\n            }\n\n            /* Getting here means we haven't yet reached the target sample, so continue waiting. */\n            ma_sleep(10);\n        }\n\n        pDevice->null_device.lastProcessedFramePlayback          += pDevice->playback.internalPeriodSizeInFrames;\n        pDevice->null_device.currentPeriodFramesRemainingPlayback = pDevice->playback.internalPeriodSizeInFrames;\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = totalPCMFramesProcessed;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_device_read__null(ma_device* pDevice, void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 totalPCMFramesProcessed;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    /* Keep going until everything has been read. */\n    totalPCMFramesProcessed = 0;\n    while (totalPCMFramesProcessed < frameCount) {\n        ma_uint64 targetFrame;\n\n        /* If there are any frames remaining in the current period, consume those first. */\n        if (pDevice->null_device.currentPeriodFramesRemainingCapture > 0) {\n            ma_uint32 bpf = ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n            ma_uint32 framesRemaining = (frameCount - totalPCMFramesProcessed);\n            ma_uint32 framesToProcess = pDevice->null_device.currentPeriodFramesRemainingCapture;\n            if (framesToProcess > framesRemaining) {\n                framesToProcess = framesRemaining;\n            }\n\n            /* We need to ensure the output buffer is zeroed. */\n            MA_ZERO_MEMORY(ma_offset_ptr(pPCMFrames, totalPCMFramesProcessed*bpf), framesToProcess*bpf);\n\n            pDevice->null_device.currentPeriodFramesRemainingCapture -= framesToProcess;\n            totalPCMFramesProcessed += framesToProcess;\n        }\n\n        /* If we've consumed the current period we'll need to mark it as such an ensure the device is started if it's not already. */\n        if (pDevice->null_device.currentPeriodFramesRemainingCapture == 0) {\n            pDevice->null_device.currentPeriodFramesRemainingCapture = 0;\n        }\n\n        /* If we've consumed the whole buffer we can return now. */\n        MA_ASSERT(totalPCMFramesProcessed <= frameCount);\n        if (totalPCMFramesProcessed == frameCount) {\n            break;\n        }\n\n        /* Getting here means we've still got more frames to consume, we but need to wait for it to become available. */\n        targetFrame = pDevice->null_device.lastProcessedFrameCapture + pDevice->capture.internalPeriodSizeInFrames;\n        for (;;) {\n            ma_uint64 currentFrame;\n\n            /* Stop waiting if the device has been stopped. */\n            if (!ma_device_is_started__null(pDevice)) {\n                break;\n            }\n\n            currentFrame = ma_device_get_total_run_time_in_frames__null(pDevice);\n            if (currentFrame >= targetFrame) {\n                break;\n            }\n\n            /* Getting here means we haven't yet reached the target sample, so continue waiting. */\n            ma_sleep(10);\n        }\n\n        pDevice->null_device.lastProcessedFrameCapture          += pDevice->capture.internalPeriodSizeInFrames;\n        pDevice->null_device.currentPeriodFramesRemainingCapture = pDevice->capture.internalPeriodSizeInFrames;\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalPCMFramesProcessed;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_context_uninit__null(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_null);\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__null(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n    (void)pContext;\n\n    pCallbacks->onContextInit             = ma_context_init__null;\n    pCallbacks->onContextUninit           = ma_context_uninit__null;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__null;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__null;\n    pCallbacks->onDeviceInit              = ma_device_init__null;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__null;\n    pCallbacks->onDeviceStart             = ma_device_start__null;\n    pCallbacks->onDeviceStop              = ma_device_stop__null;\n    pCallbacks->onDeviceRead              = ma_device_read__null;\n    pCallbacks->onDeviceWrite             = ma_device_write__null;\n    pCallbacks->onDeviceDataLoop          = NULL;   /* Our backend is asynchronous with a blocking read-write API which means we can get miniaudio to deal with the audio thread. */\n\n    /* The null backend always works. */\n    return MA_SUCCESS;\n}\n#endif\n\n\n\n/*******************************************************************************\n\nWIN32 COMMON\n\n*******************************************************************************/\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    #define ma_CoInitializeEx(pContext, pvReserved, dwCoInit)                          ((pContext->win32.CoInitializeEx) ? ((MA_PFN_CoInitializeEx)pContext->win32.CoInitializeEx)(pvReserved, dwCoInit) : ((MA_PFN_CoInitialize)pContext->win32.CoInitialize)(pvReserved))\n    #define ma_CoUninitialize(pContext)                                                ((MA_PFN_CoUninitialize)pContext->win32.CoUninitialize)()\n    #define ma_CoCreateInstance(pContext, rclsid, pUnkOuter, dwClsContext, riid, ppv)  ((MA_PFN_CoCreateInstance)pContext->win32.CoCreateInstance)(rclsid, pUnkOuter, dwClsContext, riid, ppv)\n    #define ma_CoTaskMemFree(pContext, pv)                                             ((MA_PFN_CoTaskMemFree)pContext->win32.CoTaskMemFree)(pv)\n    #define ma_PropVariantClear(pContext, pvar)                                        ((MA_PFN_PropVariantClear)pContext->win32.PropVariantClear)(pvar)\n#else\n    #define ma_CoInitializeEx(pContext, pvReserved, dwCoInit)                          CoInitializeEx(pvReserved, dwCoInit)\n    #define ma_CoUninitialize(pContext)                                                CoUninitialize()\n    #define ma_CoCreateInstance(pContext, rclsid, pUnkOuter, dwClsContext, riid, ppv)  CoCreateInstance(rclsid, pUnkOuter, dwClsContext, riid, ppv)\n    #define ma_CoTaskMemFree(pContext, pv)                                             CoTaskMemFree(pv)\n    #define ma_PropVariantClear(pContext, pvar)                                        PropVariantClear(pvar)\n#endif\n\n#if !defined(MAXULONG_PTR) && !defined(__WATCOMC__) && !defined(MA_XBOX_NXDK)\ntypedef size_t DWORD_PTR;\n#endif\n\n#if !defined(WAVE_FORMAT_1M08)\n#define WAVE_FORMAT_1M08    0x00000001\n#define WAVE_FORMAT_1S08    0x00000002\n#define WAVE_FORMAT_1M16    0x00000004\n#define WAVE_FORMAT_1S16    0x00000008\n#define WAVE_FORMAT_2M08    0x00000010\n#define WAVE_FORMAT_2S08    0x00000020\n#define WAVE_FORMAT_2M16    0x00000040\n#define WAVE_FORMAT_2S16    0x00000080\n#define WAVE_FORMAT_4M08    0x00000100\n#define WAVE_FORMAT_4S08    0x00000200\n#define WAVE_FORMAT_4M16    0x00000400\n#define WAVE_FORMAT_4S16    0x00000800\n#endif\n\n#if !defined(WAVE_FORMAT_44M08)\n#define WAVE_FORMAT_44M08   0x00000100\n#define WAVE_FORMAT_44S08   0x00000200\n#define WAVE_FORMAT_44M16   0x00000400\n#define WAVE_FORMAT_44S16   0x00000800\n#define WAVE_FORMAT_48M08   0x00001000\n#define WAVE_FORMAT_48S08   0x00002000\n#define WAVE_FORMAT_48M16   0x00004000\n#define WAVE_FORMAT_48S16   0x00008000\n#define WAVE_FORMAT_96M08   0x00010000\n#define WAVE_FORMAT_96S08   0x00020000\n#define WAVE_FORMAT_96M16   0x00040000\n#define WAVE_FORMAT_96S16   0x00080000\n#endif\n\n#ifndef SPEAKER_FRONT_LEFT\n#define SPEAKER_FRONT_LEFT            0x1\n#define SPEAKER_FRONT_RIGHT           0x2\n#define SPEAKER_FRONT_CENTER          0x4\n#define SPEAKER_LOW_FREQUENCY         0x8\n#define SPEAKER_BACK_LEFT             0x10\n#define SPEAKER_BACK_RIGHT            0x20\n#define SPEAKER_FRONT_LEFT_OF_CENTER  0x40\n#define SPEAKER_FRONT_RIGHT_OF_CENTER 0x80\n#define SPEAKER_BACK_CENTER           0x100\n#define SPEAKER_SIDE_LEFT             0x200\n#define SPEAKER_SIDE_RIGHT            0x400\n#define SPEAKER_TOP_CENTER            0x800\n#define SPEAKER_TOP_FRONT_LEFT        0x1000\n#define SPEAKER_TOP_FRONT_CENTER      0x2000\n#define SPEAKER_TOP_FRONT_RIGHT       0x4000\n#define SPEAKER_TOP_BACK_LEFT         0x8000\n#define SPEAKER_TOP_BACK_CENTER       0x10000\n#define SPEAKER_TOP_BACK_RIGHT        0x20000\n#endif\n\n/*\nImplement our own version of MA_WAVEFORMATEXTENSIBLE so we can avoid a header. Be careful with this\nbecause MA_WAVEFORMATEX has an extra two bytes over standard WAVEFORMATEX due to padding. The\nstandard version uses tight packing, but for compiler compatibility we're not doing that with ours.\n*/\ntypedef struct\n{\n    WORD wFormatTag;\n    WORD nChannels;\n    DWORD nSamplesPerSec;\n    DWORD nAvgBytesPerSec;\n    WORD nBlockAlign;\n    WORD wBitsPerSample;\n    WORD cbSize;\n} MA_WAVEFORMATEX;\n\ntypedef struct\n{\n    WORD wFormatTag;\n    WORD nChannels;\n    DWORD nSamplesPerSec;\n    DWORD nAvgBytesPerSec;\n    WORD nBlockAlign;\n    WORD wBitsPerSample;\n    WORD cbSize;\n    union\n    {\n        WORD wValidBitsPerSample;\n        WORD wSamplesPerBlock;\n        WORD wReserved;\n    } Samples;\n    DWORD dwChannelMask;\n    GUID SubFormat;\n} MA_WAVEFORMATEXTENSIBLE;\n\n\n\n#ifndef WAVE_FORMAT_EXTENSIBLE\n#define WAVE_FORMAT_EXTENSIBLE  0xFFFE\n#endif\n\n#ifndef WAVE_FORMAT_PCM\n#define WAVE_FORMAT_PCM         1\n#endif\n\n#ifndef WAVE_FORMAT_IEEE_FLOAT\n#define WAVE_FORMAT_IEEE_FLOAT  0x0003\n#endif\n\n/* Converts an individual Win32-style channel identifier (SPEAKER_FRONT_LEFT, etc.) to miniaudio. */\nstatic ma_uint8 ma_channel_id_to_ma__win32(DWORD id)\n{\n    switch (id)\n    {\n        case SPEAKER_FRONT_LEFT:            return MA_CHANNEL_FRONT_LEFT;\n        case SPEAKER_FRONT_RIGHT:           return MA_CHANNEL_FRONT_RIGHT;\n        case SPEAKER_FRONT_CENTER:          return MA_CHANNEL_FRONT_CENTER;\n        case SPEAKER_LOW_FREQUENCY:         return MA_CHANNEL_LFE;\n        case SPEAKER_BACK_LEFT:             return MA_CHANNEL_BACK_LEFT;\n        case SPEAKER_BACK_RIGHT:            return MA_CHANNEL_BACK_RIGHT;\n        case SPEAKER_FRONT_LEFT_OF_CENTER:  return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case SPEAKER_FRONT_RIGHT_OF_CENTER: return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case SPEAKER_BACK_CENTER:           return MA_CHANNEL_BACK_CENTER;\n        case SPEAKER_SIDE_LEFT:             return MA_CHANNEL_SIDE_LEFT;\n        case SPEAKER_SIDE_RIGHT:            return MA_CHANNEL_SIDE_RIGHT;\n        case SPEAKER_TOP_CENTER:            return MA_CHANNEL_TOP_CENTER;\n        case SPEAKER_TOP_FRONT_LEFT:        return MA_CHANNEL_TOP_FRONT_LEFT;\n        case SPEAKER_TOP_FRONT_CENTER:      return MA_CHANNEL_TOP_FRONT_CENTER;\n        case SPEAKER_TOP_FRONT_RIGHT:       return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case SPEAKER_TOP_BACK_LEFT:         return MA_CHANNEL_TOP_BACK_LEFT;\n        case SPEAKER_TOP_BACK_CENTER:       return MA_CHANNEL_TOP_BACK_CENTER;\n        case SPEAKER_TOP_BACK_RIGHT:        return MA_CHANNEL_TOP_BACK_RIGHT;\n        default: return 0;\n    }\n}\n\n/* Converts an individual miniaudio channel identifier (MA_CHANNEL_FRONT_LEFT, etc.) to Win32-style. */\nstatic DWORD ma_channel_id_to_win32(DWORD id)\n{\n    switch (id)\n    {\n        case MA_CHANNEL_MONO:               return SPEAKER_FRONT_CENTER;\n        case MA_CHANNEL_FRONT_LEFT:         return SPEAKER_FRONT_LEFT;\n        case MA_CHANNEL_FRONT_RIGHT:        return SPEAKER_FRONT_RIGHT;\n        case MA_CHANNEL_FRONT_CENTER:       return SPEAKER_FRONT_CENTER;\n        case MA_CHANNEL_LFE:                return SPEAKER_LOW_FREQUENCY;\n        case MA_CHANNEL_BACK_LEFT:          return SPEAKER_BACK_LEFT;\n        case MA_CHANNEL_BACK_RIGHT:         return SPEAKER_BACK_RIGHT;\n        case MA_CHANNEL_FRONT_LEFT_CENTER:  return SPEAKER_FRONT_LEFT_OF_CENTER;\n        case MA_CHANNEL_FRONT_RIGHT_CENTER: return SPEAKER_FRONT_RIGHT_OF_CENTER;\n        case MA_CHANNEL_BACK_CENTER:        return SPEAKER_BACK_CENTER;\n        case MA_CHANNEL_SIDE_LEFT:          return SPEAKER_SIDE_LEFT;\n        case MA_CHANNEL_SIDE_RIGHT:         return SPEAKER_SIDE_RIGHT;\n        case MA_CHANNEL_TOP_CENTER:         return SPEAKER_TOP_CENTER;\n        case MA_CHANNEL_TOP_FRONT_LEFT:     return SPEAKER_TOP_FRONT_LEFT;\n        case MA_CHANNEL_TOP_FRONT_CENTER:   return SPEAKER_TOP_FRONT_CENTER;\n        case MA_CHANNEL_TOP_FRONT_RIGHT:    return SPEAKER_TOP_FRONT_RIGHT;\n        case MA_CHANNEL_TOP_BACK_LEFT:      return SPEAKER_TOP_BACK_LEFT;\n        case MA_CHANNEL_TOP_BACK_CENTER:    return SPEAKER_TOP_BACK_CENTER;\n        case MA_CHANNEL_TOP_BACK_RIGHT:     return SPEAKER_TOP_BACK_RIGHT;\n        default: return 0;\n    }\n}\n\n/* Converts a channel mapping to a Win32-style channel mask. */\nstatic DWORD ma_channel_map_to_channel_mask__win32(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    DWORD dwChannelMask = 0;\n    ma_uint32 iChannel;\n\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        dwChannelMask |= ma_channel_id_to_win32(pChannelMap[iChannel]);\n    }\n\n    return dwChannelMask;\n}\n\n/* Converts a Win32-style channel mask to a miniaudio channel map. */\nstatic void ma_channel_mask_to_channel_map__win32(DWORD dwChannelMask, ma_uint32 channels, ma_channel* pChannelMap)\n{\n    /* If the channel mask is set to 0, just assume a default Win32 channel map. */\n    if (dwChannelMask == 0) {\n        ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pChannelMap, channels, channels);\n    } else {\n        if (channels == 1 && (dwChannelMask & SPEAKER_FRONT_CENTER) != 0) {\n            pChannelMap[0] = MA_CHANNEL_MONO;\n        } else {\n            /* Just iterate over each bit. */\n            ma_uint32 iChannel = 0;\n            ma_uint32 iBit;\n\n            for (iBit = 0; iBit < 32 && iChannel < channels; ++iBit) {\n                DWORD bitValue = (dwChannelMask & (1UL << iBit));\n                if (bitValue != 0) {\n                    /* The bit is set. */\n                    pChannelMap[iChannel] = ma_channel_id_to_ma__win32(bitValue);\n                    iChannel += 1;\n                }\n            }\n        }\n    }\n}\n\n#ifdef __cplusplus\nstatic ma_bool32 ma_is_guid_equal(const void* a, const void* b)\n{\n    return IsEqualGUID(*(const GUID*)a, *(const GUID*)b);\n}\n#else\n#define ma_is_guid_equal(a, b) IsEqualGUID((const GUID*)a, (const GUID*)b)\n#endif\n\nstatic MA_INLINE ma_bool32 ma_is_guid_null(const void* guid)\n{\n    static GUID nullguid = {0x00000000, 0x0000, 0x0000, {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}};\n    return ma_is_guid_equal(guid, &nullguid);\n}\n\nstatic ma_format ma_format_from_WAVEFORMATEX(const MA_WAVEFORMATEX* pWF)\n{\n    MA_ASSERT(pWF != NULL);\n\n    if (pWF->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {\n        const MA_WAVEFORMATEXTENSIBLE* pWFEX = (const MA_WAVEFORMATEXTENSIBLE*)pWF;\n        if (ma_is_guid_equal(&pWFEX->SubFormat, &MA_GUID_KSDATAFORMAT_SUBTYPE_PCM)) {\n            if (pWFEX->Samples.wValidBitsPerSample == 32) {\n                return ma_format_s32;\n            }\n            if (pWFEX->Samples.wValidBitsPerSample == 24) {\n                if (pWFEX->wBitsPerSample == 32) {\n                    return ma_format_s32;\n                }\n                if (pWFEX->wBitsPerSample == 24) {\n                    return ma_format_s24;\n                }\n            }\n            if (pWFEX->Samples.wValidBitsPerSample == 16) {\n                return ma_format_s16;\n            }\n            if (pWFEX->Samples.wValidBitsPerSample == 8) {\n                return ma_format_u8;\n            }\n        }\n        if (ma_is_guid_equal(&pWFEX->SubFormat, &MA_GUID_KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)) {\n            if (pWFEX->Samples.wValidBitsPerSample == 32) {\n                return ma_format_f32;\n            }\n            /*\n            if (pWFEX->Samples.wValidBitsPerSample == 64) {\n                return ma_format_f64;\n            }\n            */\n        }\n    } else {\n        if (pWF->wFormatTag == WAVE_FORMAT_PCM) {\n            if (pWF->wBitsPerSample == 32) {\n                return ma_format_s32;\n            }\n            if (pWF->wBitsPerSample == 24) {\n                return ma_format_s24;\n            }\n            if (pWF->wBitsPerSample == 16) {\n                return ma_format_s16;\n            }\n            if (pWF->wBitsPerSample == 8) {\n                return ma_format_u8;\n            }\n        }\n        if (pWF->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) {\n            if (pWF->wBitsPerSample == 32) {\n                return ma_format_f32;\n            }\n            if (pWF->wBitsPerSample == 64) {\n                /*return ma_format_f64;*/\n            }\n        }\n    }\n\n    return ma_format_unknown;\n}\n#endif\n\n\n/*******************************************************************************\n\nWASAPI Backend\n\n*******************************************************************************/\n#ifdef MA_HAS_WASAPI\n#if 0\n#if defined(_MSC_VER)\n    #pragma warning(push)\n    #pragma warning(disable:4091)   /* 'typedef ': ignored on left of '' when no variable is declared */\n#endif\n#include <audioclient.h>\n#include <mmdeviceapi.h>\n#if defined(_MSC_VER)\n    #pragma warning(pop)\n#endif\n#endif  /* 0 */\n\nstatic ma_result ma_device_reroute__wasapi(ma_device* pDevice, ma_device_type deviceType);\n\n/* Some compilers don't define VerifyVersionInfoW. Need to write this ourselves. */\n#define MA_WIN32_WINNT_VISTA    0x0600\n#define MA_VER_MINORVERSION     0x01\n#define MA_VER_MAJORVERSION     0x02\n#define MA_VER_SERVICEPACKMAJOR 0x20\n#define MA_VER_GREATER_EQUAL    0x03\n\ntypedef struct  {\n    DWORD dwOSVersionInfoSize;\n    DWORD dwMajorVersion;\n    DWORD dwMinorVersion;\n    DWORD dwBuildNumber;\n    DWORD dwPlatformId;\n    WCHAR szCSDVersion[128];\n    WORD  wServicePackMajor;\n    WORD  wServicePackMinor;\n    WORD  wSuiteMask;\n    BYTE  wProductType;\n    BYTE  wReserved;\n} ma_OSVERSIONINFOEXW;\n\ntypedef BOOL      (WINAPI * ma_PFNVerifyVersionInfoW) (ma_OSVERSIONINFOEXW* lpVersionInfo, DWORD dwTypeMask, DWORDLONG dwlConditionMask);\ntypedef ULONGLONG (WINAPI * ma_PFNVerSetConditionMask)(ULONGLONG dwlConditionMask, DWORD dwTypeBitMask, BYTE dwConditionMask);\n\n\n#ifndef PROPERTYKEY_DEFINED\n#define PROPERTYKEY_DEFINED\n#ifndef __WATCOMC__\ntypedef struct\n{\n    GUID fmtid;\n    DWORD pid;\n} PROPERTYKEY;\n#endif\n#endif\n\n/* Some compilers don't define PropVariantInit(). We just do this ourselves since it's just a memset(). */\nstatic MA_INLINE void ma_PropVariantInit(MA_PROPVARIANT* pProp)\n{\n    MA_ZERO_OBJECT(pProp);\n}\n\n\nstatic const PROPERTYKEY MA_PKEY_Device_FriendlyName             = {{0xA45C254E, 0xDF1C, 0x4EFD, {0x80, 0x20, 0x67, 0xD1, 0x46, 0xA8, 0x50, 0xE0}}, 14};\nstatic const PROPERTYKEY MA_PKEY_AudioEngine_DeviceFormat        = {{0xF19F064D, 0x82C,  0x4E27, {0xBC, 0x73, 0x68, 0x82, 0xA1, 0xBB, 0x8E, 0x4C}},  0};\n\nstatic const IID MA_IID_IUnknown                                 = {0x00000000, 0x0000, 0x0000, {0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}}; /* 00000000-0000-0000-C000-000000000046 */\n#if !defined(MA_WIN32_DESKTOP) && !defined(MA_WIN32_GDK)\nstatic const IID MA_IID_IAgileObject                             = {0x94EA2B94, 0xE9CC, 0x49E0, {0xC0, 0xFF, 0xEE, 0x64, 0xCA, 0x8F, 0x5B, 0x90}}; /* 94EA2B94-E9CC-49E0-C0FF-EE64CA8F5B90 */\n#endif\n\nstatic const IID MA_IID_IAudioClient                             = {0x1CB9AD4C, 0xDBFA, 0x4C32, {0xB1, 0x78, 0xC2, 0xF5, 0x68, 0xA7, 0x03, 0xB2}}; /* 1CB9AD4C-DBFA-4C32-B178-C2F568A703B2 = __uuidof(IAudioClient) */\nstatic const IID MA_IID_IAudioClient2                            = {0x726778CD, 0xF60A, 0x4EDA, {0x82, 0xDE, 0xE4, 0x76, 0x10, 0xCD, 0x78, 0xAA}}; /* 726778CD-F60A-4EDA-82DE-E47610CD78AA = __uuidof(IAudioClient2) */\nstatic const IID MA_IID_IAudioClient3                            = {0x7ED4EE07, 0x8E67, 0x4CD4, {0x8C, 0x1A, 0x2B, 0x7A, 0x59, 0x87, 0xAD, 0x42}}; /* 7ED4EE07-8E67-4CD4-8C1A-2B7A5987AD42 = __uuidof(IAudioClient3) */\nstatic const IID MA_IID_IAudioRenderClient                       = {0xF294ACFC, 0x3146, 0x4483, {0xA7, 0xBF, 0xAD, 0xDC, 0xA7, 0xC2, 0x60, 0xE2}}; /* F294ACFC-3146-4483-A7BF-ADDCA7C260E2 = __uuidof(IAudioRenderClient) */\nstatic const IID MA_IID_IAudioCaptureClient                      = {0xC8ADBD64, 0xE71E, 0x48A0, {0xA4, 0xDE, 0x18, 0x5C, 0x39, 0x5C, 0xD3, 0x17}}; /* C8ADBD64-E71E-48A0-A4DE-185C395CD317 = __uuidof(IAudioCaptureClient) */\nstatic const IID MA_IID_IMMNotificationClient                    = {0x7991EEC9, 0x7E89, 0x4D85, {0x83, 0x90, 0x6C, 0x70, 0x3C, 0xEC, 0x60, 0xC0}}; /* 7991EEC9-7E89-4D85-8390-6C703CEC60C0 = __uuidof(IMMNotificationClient) */\n#if !defined(MA_WIN32_DESKTOP) && !defined(MA_WIN32_GDK)\nstatic const IID MA_IID_DEVINTERFACE_AUDIO_RENDER                = {0xE6327CAD, 0xDCEC, 0x4949, {0xAE, 0x8A, 0x99, 0x1E, 0x97, 0x6A, 0x79, 0xD2}}; /* E6327CAD-DCEC-4949-AE8A-991E976A79D2 */\nstatic const IID MA_IID_DEVINTERFACE_AUDIO_CAPTURE               = {0x2EEF81BE, 0x33FA, 0x4800, {0x96, 0x70, 0x1C, 0xD4, 0x74, 0x97, 0x2C, 0x3F}}; /* 2EEF81BE-33FA-4800-9670-1CD474972C3F */\nstatic const IID MA_IID_IActivateAudioInterfaceCompletionHandler = {0x41D949AB, 0x9862, 0x444A, {0x80, 0xF6, 0xC2, 0x61, 0x33, 0x4D, 0xA5, 0xEB}}; /* 41D949AB-9862-444A-80F6-C261334DA5EB */\n#endif\n\nstatic const IID MA_CLSID_MMDeviceEnumerator                     = {0xBCDE0395, 0xE52F, 0x467C, {0x8E, 0x3D, 0xC4, 0x57, 0x92, 0x91, 0x69, 0x2E}}; /* BCDE0395-E52F-467C-8E3D-C4579291692E = __uuidof(MMDeviceEnumerator) */\nstatic const IID MA_IID_IMMDeviceEnumerator                      = {0xA95664D2, 0x9614, 0x4F35, {0xA7, 0x46, 0xDE, 0x8D, 0xB6, 0x36, 0x17, 0xE6}}; /* A95664D2-9614-4F35-A746-DE8DB63617E6 = __uuidof(IMMDeviceEnumerator) */\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n#define MA_MM_DEVICE_STATE_ACTIVE                          1\n#define MA_MM_DEVICE_STATE_DISABLED                        2\n#define MA_MM_DEVICE_STATE_NOTPRESENT                      4\n#define MA_MM_DEVICE_STATE_UNPLUGGED                       8\n\ntypedef struct ma_IMMDeviceEnumerator                      ma_IMMDeviceEnumerator;\ntypedef struct ma_IMMDeviceCollection                      ma_IMMDeviceCollection;\ntypedef struct ma_IMMDevice                                ma_IMMDevice;\n#else\ntypedef struct ma_IActivateAudioInterfaceCompletionHandler ma_IActivateAudioInterfaceCompletionHandler;\ntypedef struct ma_IActivateAudioInterfaceAsyncOperation    ma_IActivateAudioInterfaceAsyncOperation;\n#endif\ntypedef struct ma_IPropertyStore                           ma_IPropertyStore;\ntypedef struct ma_IAudioClient                             ma_IAudioClient;\ntypedef struct ma_IAudioClient2                            ma_IAudioClient2;\ntypedef struct ma_IAudioClient3                            ma_IAudioClient3;\ntypedef struct ma_IAudioRenderClient                       ma_IAudioRenderClient;\ntypedef struct ma_IAudioCaptureClient                      ma_IAudioCaptureClient;\n\ntypedef ma_int64                                           MA_REFERENCE_TIME;\n\n#define MA_AUDCLNT_STREAMFLAGS_CROSSPROCESS                0x00010000\n#define MA_AUDCLNT_STREAMFLAGS_LOOPBACK                    0x00020000\n#define MA_AUDCLNT_STREAMFLAGS_EVENTCALLBACK               0x00040000\n#define MA_AUDCLNT_STREAMFLAGS_NOPERSIST                   0x00080000\n#define MA_AUDCLNT_STREAMFLAGS_RATEADJUST                  0x00100000\n#define MA_AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY         0x08000000\n#define MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM              0x80000000\n#define MA_AUDCLNT_SESSIONFLAGS_EXPIREWHENUNOWNED          0x10000000\n#define MA_AUDCLNT_SESSIONFLAGS_DISPLAY_HIDE               0x20000000\n#define MA_AUDCLNT_SESSIONFLAGS_DISPLAY_HIDEWHENEXPIRED    0x40000000\n\n/* Buffer flags. */\n#define MA_AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY          1\n#define MA_AUDCLNT_BUFFERFLAGS_SILENT                      2\n#define MA_AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR             4\n\ntypedef enum\n{\n    ma_eRender  = 0,\n    ma_eCapture = 1,\n    ma_eAll     = 2\n} ma_EDataFlow;\n\ntypedef enum\n{\n    ma_eConsole        = 0,\n    ma_eMultimedia     = 1,\n    ma_eCommunications = 2\n} ma_ERole;\n\ntypedef enum\n{\n    MA_AUDCLNT_SHAREMODE_SHARED,\n    MA_AUDCLNT_SHAREMODE_EXCLUSIVE\n} MA_AUDCLNT_SHAREMODE;\n\ntypedef enum\n{\n    MA_AudioCategory_Other = 0  /* <-- miniaudio is only caring about Other. */\n} MA_AUDIO_STREAM_CATEGORY;\n\ntypedef enum\n{\n    MA_AUDCLNT_STREAMOPTIONS_NONE,\n    MA_AUDCLNT_STREAMOPTIONS_RAW,\n    MA_AUDCLNT_STREAMOPTIONS_MATCH_FORMAT,\n    MA_AUDCLNT_STREAMOPTIONS_AMBISONICS,\n    MA_AUDCLNT_STREAMOPTIONS_POST_VOLUME_LOOPBACK\n} MA_AUDCLNT_STREAMOPTIONS;\n\ntypedef struct\n{\n    ma_uint32 cbSize;\n    BOOL bIsOffload;\n    MA_AUDIO_STREAM_CATEGORY eCategory;\n    MA_AUDCLNT_STREAMOPTIONS Options;\n} ma_AudioClientProperties;\n\n/* IUnknown */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IUnknown* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IUnknown* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IUnknown* pThis);\n} ma_IUnknownVtbl;\nstruct ma_IUnknown\n{\n    ma_IUnknownVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IUnknown_QueryInterface(ma_IUnknown* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IUnknown_AddRef(ma_IUnknown* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IUnknown_Release(ma_IUnknown* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    /* IMMNotificationClient */\n    typedef struct\n    {\n        /* IUnknown */\n        HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IMMNotificationClient* pThis, const IID* const riid, void** ppObject);\n        ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IMMNotificationClient* pThis);\n        ULONG   (STDMETHODCALLTYPE * Release)       (ma_IMMNotificationClient* pThis);\n\n        /* IMMNotificationClient */\n        HRESULT (STDMETHODCALLTYPE * OnDeviceStateChanged)  (ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID, DWORD dwNewState);\n        HRESULT (STDMETHODCALLTYPE * OnDeviceAdded)         (ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID);\n        HRESULT (STDMETHODCALLTYPE * OnDeviceRemoved)       (ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID);\n        HRESULT (STDMETHODCALLTYPE * OnDefaultDeviceChanged)(ma_IMMNotificationClient* pThis, ma_EDataFlow dataFlow, ma_ERole role, const WCHAR* pDefaultDeviceID);\n        HRESULT (STDMETHODCALLTYPE * OnPropertyValueChanged)(ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID, const PROPERTYKEY key);\n    } ma_IMMNotificationClientVtbl;\n\n    /* IMMDeviceEnumerator */\n    typedef struct\n    {\n        /* IUnknown */\n        HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IMMDeviceEnumerator* pThis, const IID* const riid, void** ppObject);\n        ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IMMDeviceEnumerator* pThis);\n        ULONG   (STDMETHODCALLTYPE * Release)       (ma_IMMDeviceEnumerator* pThis);\n\n        /* IMMDeviceEnumerator */\n        HRESULT (STDMETHODCALLTYPE * EnumAudioEndpoints)                    (ma_IMMDeviceEnumerator* pThis, ma_EDataFlow dataFlow, DWORD dwStateMask, ma_IMMDeviceCollection** ppDevices);\n        HRESULT (STDMETHODCALLTYPE * GetDefaultAudioEndpoint)               (ma_IMMDeviceEnumerator* pThis, ma_EDataFlow dataFlow, ma_ERole role, ma_IMMDevice** ppEndpoint);\n        HRESULT (STDMETHODCALLTYPE * GetDevice)                             (ma_IMMDeviceEnumerator* pThis, const WCHAR* pID, ma_IMMDevice** ppDevice);\n        HRESULT (STDMETHODCALLTYPE * RegisterEndpointNotificationCallback)  (ma_IMMDeviceEnumerator* pThis, ma_IMMNotificationClient* pClient);\n        HRESULT (STDMETHODCALLTYPE * UnregisterEndpointNotificationCallback)(ma_IMMDeviceEnumerator* pThis, ma_IMMNotificationClient* pClient);\n    } ma_IMMDeviceEnumeratorVtbl;\n    struct ma_IMMDeviceEnumerator\n    {\n        ma_IMMDeviceEnumeratorVtbl* lpVtbl;\n    };\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_QueryInterface(ma_IMMDeviceEnumerator* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\n    static MA_INLINE ULONG   ma_IMMDeviceEnumerator_AddRef(ma_IMMDeviceEnumerator* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\n    static MA_INLINE ULONG   ma_IMMDeviceEnumerator_Release(ma_IMMDeviceEnumerator* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_EnumAudioEndpoints(ma_IMMDeviceEnumerator* pThis, ma_EDataFlow dataFlow, DWORD dwStateMask, ma_IMMDeviceCollection** ppDevices) { return pThis->lpVtbl->EnumAudioEndpoints(pThis, dataFlow, dwStateMask, ppDevices); }\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_GetDefaultAudioEndpoint(ma_IMMDeviceEnumerator* pThis, ma_EDataFlow dataFlow, ma_ERole role, ma_IMMDevice** ppEndpoint) { return pThis->lpVtbl->GetDefaultAudioEndpoint(pThis, dataFlow, role, ppEndpoint); }\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_GetDevice(ma_IMMDeviceEnumerator* pThis, const WCHAR* pID, ma_IMMDevice** ppDevice) { return pThis->lpVtbl->GetDevice(pThis, pID, ppDevice); }\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_RegisterEndpointNotificationCallback(ma_IMMDeviceEnumerator* pThis, ma_IMMNotificationClient* pClient) { return pThis->lpVtbl->RegisterEndpointNotificationCallback(pThis, pClient); }\n    static MA_INLINE HRESULT ma_IMMDeviceEnumerator_UnregisterEndpointNotificationCallback(ma_IMMDeviceEnumerator* pThis, ma_IMMNotificationClient* pClient) { return pThis->lpVtbl->UnregisterEndpointNotificationCallback(pThis, pClient); }\n\n\n    /* IMMDeviceCollection */\n    typedef struct\n    {\n        /* IUnknown */\n        HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IMMDeviceCollection* pThis, const IID* const riid, void** ppObject);\n        ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IMMDeviceCollection* pThis);\n        ULONG   (STDMETHODCALLTYPE * Release)       (ma_IMMDeviceCollection* pThis);\n\n        /* IMMDeviceCollection */\n        HRESULT (STDMETHODCALLTYPE * GetCount)(ma_IMMDeviceCollection* pThis, UINT* pDevices);\n        HRESULT (STDMETHODCALLTYPE * Item)    (ma_IMMDeviceCollection* pThis, UINT nDevice, ma_IMMDevice** ppDevice);\n    } ma_IMMDeviceCollectionVtbl;\n    struct ma_IMMDeviceCollection\n    {\n        ma_IMMDeviceCollectionVtbl* lpVtbl;\n    };\n    static MA_INLINE HRESULT ma_IMMDeviceCollection_QueryInterface(ma_IMMDeviceCollection* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\n    static MA_INLINE ULONG   ma_IMMDeviceCollection_AddRef(ma_IMMDeviceCollection* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\n    static MA_INLINE ULONG   ma_IMMDeviceCollection_Release(ma_IMMDeviceCollection* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\n    static MA_INLINE HRESULT ma_IMMDeviceCollection_GetCount(ma_IMMDeviceCollection* pThis, UINT* pDevices)                               { return pThis->lpVtbl->GetCount(pThis, pDevices); }\n    static MA_INLINE HRESULT ma_IMMDeviceCollection_Item(ma_IMMDeviceCollection* pThis, UINT nDevice, ma_IMMDevice** ppDevice)            { return pThis->lpVtbl->Item(pThis, nDevice, ppDevice); }\n\n\n    /* IMMDevice */\n    typedef struct\n    {\n        /* IUnknown */\n        HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IMMDevice* pThis, const IID* const riid, void** ppObject);\n        ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IMMDevice* pThis);\n        ULONG   (STDMETHODCALLTYPE * Release)       (ma_IMMDevice* pThis);\n\n        /* IMMDevice */\n        HRESULT (STDMETHODCALLTYPE * Activate)         (ma_IMMDevice* pThis, const IID* const iid, DWORD dwClsCtx, MA_PROPVARIANT* pActivationParams, void** ppInterface);\n        HRESULT (STDMETHODCALLTYPE * OpenPropertyStore)(ma_IMMDevice* pThis, DWORD stgmAccess, ma_IPropertyStore** ppProperties);\n        HRESULT (STDMETHODCALLTYPE * GetId)            (ma_IMMDevice* pThis, WCHAR** pID);\n        HRESULT (STDMETHODCALLTYPE * GetState)         (ma_IMMDevice* pThis, DWORD *pState);\n    } ma_IMMDeviceVtbl;\n    struct ma_IMMDevice\n    {\n        ma_IMMDeviceVtbl* lpVtbl;\n    };\n    static MA_INLINE HRESULT ma_IMMDevice_QueryInterface(ma_IMMDevice* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\n    static MA_INLINE ULONG   ma_IMMDevice_AddRef(ma_IMMDevice* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\n    static MA_INLINE ULONG   ma_IMMDevice_Release(ma_IMMDevice* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\n    static MA_INLINE HRESULT ma_IMMDevice_Activate(ma_IMMDevice* pThis, const IID* const iid, DWORD dwClsCtx, MA_PROPVARIANT* pActivationParams, void** ppInterface) { return pThis->lpVtbl->Activate(pThis, iid, dwClsCtx, pActivationParams, ppInterface); }\n    static MA_INLINE HRESULT ma_IMMDevice_OpenPropertyStore(ma_IMMDevice* pThis, DWORD stgmAccess, ma_IPropertyStore** ppProperties) { return pThis->lpVtbl->OpenPropertyStore(pThis, stgmAccess, ppProperties); }\n    static MA_INLINE HRESULT ma_IMMDevice_GetId(ma_IMMDevice* pThis, WCHAR** pID)                                     { return pThis->lpVtbl->GetId(pThis, pID); }\n    static MA_INLINE HRESULT ma_IMMDevice_GetState(ma_IMMDevice* pThis, DWORD *pState)                                { return pThis->lpVtbl->GetState(pThis, pState); }\n#else\n    /* IActivateAudioInterfaceAsyncOperation */\n    typedef struct\n    {\n        /* IUnknown */\n        HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IActivateAudioInterfaceAsyncOperation* pThis, const IID* const riid, void** ppObject);\n        ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IActivateAudioInterfaceAsyncOperation* pThis);\n        ULONG   (STDMETHODCALLTYPE * Release)       (ma_IActivateAudioInterfaceAsyncOperation* pThis);\n\n        /* IActivateAudioInterfaceAsyncOperation */\n        HRESULT (STDMETHODCALLTYPE * GetActivateResult)(ma_IActivateAudioInterfaceAsyncOperation* pThis, HRESULT *pActivateResult, ma_IUnknown** ppActivatedInterface);\n    } ma_IActivateAudioInterfaceAsyncOperationVtbl;\n    struct ma_IActivateAudioInterfaceAsyncOperation\n    {\n        ma_IActivateAudioInterfaceAsyncOperationVtbl* lpVtbl;\n    };\n    static MA_INLINE HRESULT ma_IActivateAudioInterfaceAsyncOperation_QueryInterface(ma_IActivateAudioInterfaceAsyncOperation* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\n    static MA_INLINE ULONG   ma_IActivateAudioInterfaceAsyncOperation_AddRef(ma_IActivateAudioInterfaceAsyncOperation* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\n    static MA_INLINE ULONG   ma_IActivateAudioInterfaceAsyncOperation_Release(ma_IActivateAudioInterfaceAsyncOperation* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\n    static MA_INLINE HRESULT ma_IActivateAudioInterfaceAsyncOperation_GetActivateResult(ma_IActivateAudioInterfaceAsyncOperation* pThis, HRESULT *pActivateResult, ma_IUnknown** ppActivatedInterface) { return pThis->lpVtbl->GetActivateResult(pThis, pActivateResult, ppActivatedInterface); }\n#endif\n\n/* IPropertyStore */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IPropertyStore* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IPropertyStore* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IPropertyStore* pThis);\n\n    /* IPropertyStore */\n    HRESULT (STDMETHODCALLTYPE * GetCount)(ma_IPropertyStore* pThis, DWORD* pPropCount);\n    HRESULT (STDMETHODCALLTYPE * GetAt)   (ma_IPropertyStore* pThis, DWORD propIndex, PROPERTYKEY* pPropKey);\n    HRESULT (STDMETHODCALLTYPE * GetValue)(ma_IPropertyStore* pThis, const PROPERTYKEY* const pKey, MA_PROPVARIANT* pPropVar);\n    HRESULT (STDMETHODCALLTYPE * SetValue)(ma_IPropertyStore* pThis, const PROPERTYKEY* const pKey, const MA_PROPVARIANT* const pPropVar);\n    HRESULT (STDMETHODCALLTYPE * Commit)  (ma_IPropertyStore* pThis);\n} ma_IPropertyStoreVtbl;\nstruct ma_IPropertyStore\n{\n    ma_IPropertyStoreVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IPropertyStore_QueryInterface(ma_IPropertyStore* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IPropertyStore_AddRef(ma_IPropertyStore* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IPropertyStore_Release(ma_IPropertyStore* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IPropertyStore_GetCount(ma_IPropertyStore* pThis, DWORD* pPropCount)                            { return pThis->lpVtbl->GetCount(pThis, pPropCount); }\nstatic MA_INLINE HRESULT ma_IPropertyStore_GetAt(ma_IPropertyStore* pThis, DWORD propIndex, PROPERTYKEY* pPropKey)          { return pThis->lpVtbl->GetAt(pThis, propIndex, pPropKey); }\nstatic MA_INLINE HRESULT ma_IPropertyStore_GetValue(ma_IPropertyStore* pThis, const PROPERTYKEY* const pKey, MA_PROPVARIANT* pPropVar) { return pThis->lpVtbl->GetValue(pThis, pKey, pPropVar); }\nstatic MA_INLINE HRESULT ma_IPropertyStore_SetValue(ma_IPropertyStore* pThis, const PROPERTYKEY* const pKey, const MA_PROPVARIANT* const pPropVar) { return pThis->lpVtbl->SetValue(pThis, pKey, pPropVar); }\nstatic MA_INLINE HRESULT ma_IPropertyStore_Commit(ma_IPropertyStore* pThis)                                                 { return pThis->lpVtbl->Commit(pThis); }\n\n\n/* IAudioClient */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IAudioClient* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IAudioClient* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IAudioClient* pThis);\n\n    /* IAudioClient */\n    HRESULT (STDMETHODCALLTYPE * Initialize)       (ma_IAudioClient* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid);\n    HRESULT (STDMETHODCALLTYPE * GetBufferSize)    (ma_IAudioClient* pThis, ma_uint32* pNumBufferFrames);\n    HRESULT (STDMETHODCALLTYPE * GetStreamLatency) (ma_IAudioClient* pThis, MA_REFERENCE_TIME* pLatency);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentPadding)(ma_IAudioClient* pThis, ma_uint32* pNumPaddingFrames);\n    HRESULT (STDMETHODCALLTYPE * IsFormatSupported)(ma_IAudioClient* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch);\n    HRESULT (STDMETHODCALLTYPE * GetMixFormat)     (ma_IAudioClient* pThis, MA_WAVEFORMATEX** ppDeviceFormat);\n    HRESULT (STDMETHODCALLTYPE * GetDevicePeriod)  (ma_IAudioClient* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod);\n    HRESULT (STDMETHODCALLTYPE * Start)            (ma_IAudioClient* pThis);\n    HRESULT (STDMETHODCALLTYPE * Stop)             (ma_IAudioClient* pThis);\n    HRESULT (STDMETHODCALLTYPE * Reset)            (ma_IAudioClient* pThis);\n    HRESULT (STDMETHODCALLTYPE * SetEventHandle)   (ma_IAudioClient* pThis, HANDLE eventHandle);\n    HRESULT (STDMETHODCALLTYPE * GetService)       (ma_IAudioClient* pThis, const IID* const riid, void** pp);\n} ma_IAudioClientVtbl;\nstruct ma_IAudioClient\n{\n    ma_IAudioClientVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IAudioClient_QueryInterface(ma_IAudioClient* pThis, const IID* const riid, void** ppObject)    { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IAudioClient_AddRef(ma_IAudioClient* pThis)                                                    { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IAudioClient_Release(ma_IAudioClient* pThis)                                                   { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient_Initialize(ma_IAudioClient* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid) { return pThis->lpVtbl->Initialize(pThis, shareMode, streamFlags, bufferDuration, periodicity, pFormat, pAudioSessionGuid); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetBufferSize(ma_IAudioClient* pThis, ma_uint32* pNumBufferFrames)                { return pThis->lpVtbl->GetBufferSize(pThis, pNumBufferFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetStreamLatency(ma_IAudioClient* pThis, MA_REFERENCE_TIME* pLatency)             { return pThis->lpVtbl->GetStreamLatency(pThis, pLatency); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetCurrentPadding(ma_IAudioClient* pThis, ma_uint32* pNumPaddingFrames)           { return pThis->lpVtbl->GetCurrentPadding(pThis, pNumPaddingFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient_IsFormatSupported(ma_IAudioClient* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch) { return pThis->lpVtbl->IsFormatSupported(pThis, shareMode, pFormat, ppClosestMatch); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetMixFormat(ma_IAudioClient* pThis, MA_WAVEFORMATEX** ppDeviceFormat)            { return pThis->lpVtbl->GetMixFormat(pThis, ppDeviceFormat); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetDevicePeriod(ma_IAudioClient* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod) { return pThis->lpVtbl->GetDevicePeriod(pThis, pDefaultDevicePeriod, pMinimumDevicePeriod); }\nstatic MA_INLINE HRESULT ma_IAudioClient_Start(ma_IAudioClient* pThis)                                                     { return pThis->lpVtbl->Start(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient_Stop(ma_IAudioClient* pThis)                                                      { return pThis->lpVtbl->Stop(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient_Reset(ma_IAudioClient* pThis)                                                     { return pThis->lpVtbl->Reset(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient_SetEventHandle(ma_IAudioClient* pThis, HANDLE eventHandle)                        { return pThis->lpVtbl->SetEventHandle(pThis, eventHandle); }\nstatic MA_INLINE HRESULT ma_IAudioClient_GetService(ma_IAudioClient* pThis, const IID* const riid, void** pp)              { return pThis->lpVtbl->GetService(pThis, riid, pp); }\n\n/* IAudioClient2 */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IAudioClient2* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IAudioClient2* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IAudioClient2* pThis);\n\n    /* IAudioClient */\n    HRESULT (STDMETHODCALLTYPE * Initialize)       (ma_IAudioClient2* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid);\n    HRESULT (STDMETHODCALLTYPE * GetBufferSize)    (ma_IAudioClient2* pThis, ma_uint32* pNumBufferFrames);\n    HRESULT (STDMETHODCALLTYPE * GetStreamLatency) (ma_IAudioClient2* pThis, MA_REFERENCE_TIME* pLatency);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentPadding)(ma_IAudioClient2* pThis, ma_uint32* pNumPaddingFrames);\n    HRESULT (STDMETHODCALLTYPE * IsFormatSupported)(ma_IAudioClient2* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch);\n    HRESULT (STDMETHODCALLTYPE * GetMixFormat)     (ma_IAudioClient2* pThis, MA_WAVEFORMATEX** ppDeviceFormat);\n    HRESULT (STDMETHODCALLTYPE * GetDevicePeriod)  (ma_IAudioClient2* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod);\n    HRESULT (STDMETHODCALLTYPE * Start)            (ma_IAudioClient2* pThis);\n    HRESULT (STDMETHODCALLTYPE * Stop)             (ma_IAudioClient2* pThis);\n    HRESULT (STDMETHODCALLTYPE * Reset)            (ma_IAudioClient2* pThis);\n    HRESULT (STDMETHODCALLTYPE * SetEventHandle)   (ma_IAudioClient2* pThis, HANDLE eventHandle);\n    HRESULT (STDMETHODCALLTYPE * GetService)       (ma_IAudioClient2* pThis, const IID* const riid, void** pp);\n\n    /* IAudioClient2 */\n    HRESULT (STDMETHODCALLTYPE * IsOffloadCapable)   (ma_IAudioClient2* pThis, MA_AUDIO_STREAM_CATEGORY category, BOOL* pOffloadCapable);\n    HRESULT (STDMETHODCALLTYPE * SetClientProperties)(ma_IAudioClient2* pThis, const ma_AudioClientProperties* pProperties);\n    HRESULT (STDMETHODCALLTYPE * GetBufferSizeLimits)(ma_IAudioClient2* pThis, const MA_WAVEFORMATEX* pFormat, BOOL eventDriven, MA_REFERENCE_TIME* pMinBufferDuration, MA_REFERENCE_TIME* pMaxBufferDuration);\n} ma_IAudioClient2Vtbl;\nstruct ma_IAudioClient2\n{\n    ma_IAudioClient2Vtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IAudioClient2_QueryInterface(ma_IAudioClient2* pThis, const IID* const riid, void** ppObject)    { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IAudioClient2_AddRef(ma_IAudioClient2* pThis)                                                    { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IAudioClient2_Release(ma_IAudioClient2* pThis)                                                   { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_Initialize(ma_IAudioClient2* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid) { return pThis->lpVtbl->Initialize(pThis, shareMode, streamFlags, bufferDuration, periodicity, pFormat, pAudioSessionGuid); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetBufferSize(ma_IAudioClient2* pThis, ma_uint32* pNumBufferFrames)                { return pThis->lpVtbl->GetBufferSize(pThis, pNumBufferFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetStreamLatency(ma_IAudioClient2* pThis, MA_REFERENCE_TIME* pLatency)             { return pThis->lpVtbl->GetStreamLatency(pThis, pLatency); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetCurrentPadding(ma_IAudioClient2* pThis, ma_uint32* pNumPaddingFrames)           { return pThis->lpVtbl->GetCurrentPadding(pThis, pNumPaddingFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_IsFormatSupported(ma_IAudioClient2* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch) { return pThis->lpVtbl->IsFormatSupported(pThis, shareMode, pFormat, ppClosestMatch); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetMixFormat(ma_IAudioClient2* pThis, MA_WAVEFORMATEX** ppDeviceFormat)            { return pThis->lpVtbl->GetMixFormat(pThis, ppDeviceFormat); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetDevicePeriod(ma_IAudioClient2* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod) { return pThis->lpVtbl->GetDevicePeriod(pThis, pDefaultDevicePeriod, pMinimumDevicePeriod); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_Start(ma_IAudioClient2* pThis)                                                     { return pThis->lpVtbl->Start(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_Stop(ma_IAudioClient2* pThis)                                                      { return pThis->lpVtbl->Stop(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_Reset(ma_IAudioClient2* pThis)                                                     { return pThis->lpVtbl->Reset(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_SetEventHandle(ma_IAudioClient2* pThis, HANDLE eventHandle)                        { return pThis->lpVtbl->SetEventHandle(pThis, eventHandle); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetService(ma_IAudioClient2* pThis, const IID* const riid, void** pp)              { return pThis->lpVtbl->GetService(pThis, riid, pp); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_IsOffloadCapable(ma_IAudioClient2* pThis, MA_AUDIO_STREAM_CATEGORY category, BOOL* pOffloadCapable) { return pThis->lpVtbl->IsOffloadCapable(pThis, category, pOffloadCapable); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_SetClientProperties(ma_IAudioClient2* pThis, const ma_AudioClientProperties* pProperties)           { return pThis->lpVtbl->SetClientProperties(pThis, pProperties); }\nstatic MA_INLINE HRESULT ma_IAudioClient2_GetBufferSizeLimits(ma_IAudioClient2* pThis, const MA_WAVEFORMATEX* pFormat, BOOL eventDriven, MA_REFERENCE_TIME* pMinBufferDuration, MA_REFERENCE_TIME* pMaxBufferDuration) { return pThis->lpVtbl->GetBufferSizeLimits(pThis, pFormat, eventDriven, pMinBufferDuration, pMaxBufferDuration); }\n\n\n/* IAudioClient3 */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IAudioClient3* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IAudioClient3* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IAudioClient3* pThis);\n\n    /* IAudioClient */\n    HRESULT (STDMETHODCALLTYPE * Initialize)       (ma_IAudioClient3* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid);\n    HRESULT (STDMETHODCALLTYPE * GetBufferSize)    (ma_IAudioClient3* pThis, ma_uint32* pNumBufferFrames);\n    HRESULT (STDMETHODCALLTYPE * GetStreamLatency) (ma_IAudioClient3* pThis, MA_REFERENCE_TIME* pLatency);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentPadding)(ma_IAudioClient3* pThis, ma_uint32* pNumPaddingFrames);\n    HRESULT (STDMETHODCALLTYPE * IsFormatSupported)(ma_IAudioClient3* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch);\n    HRESULT (STDMETHODCALLTYPE * GetMixFormat)     (ma_IAudioClient3* pThis, MA_WAVEFORMATEX** ppDeviceFormat);\n    HRESULT (STDMETHODCALLTYPE * GetDevicePeriod)  (ma_IAudioClient3* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod);\n    HRESULT (STDMETHODCALLTYPE * Start)            (ma_IAudioClient3* pThis);\n    HRESULT (STDMETHODCALLTYPE * Stop)             (ma_IAudioClient3* pThis);\n    HRESULT (STDMETHODCALLTYPE * Reset)            (ma_IAudioClient3* pThis);\n    HRESULT (STDMETHODCALLTYPE * SetEventHandle)   (ma_IAudioClient3* pThis, HANDLE eventHandle);\n    HRESULT (STDMETHODCALLTYPE * GetService)       (ma_IAudioClient3* pThis, const IID* const riid, void** pp);\n\n    /* IAudioClient2 */\n    HRESULT (STDMETHODCALLTYPE * IsOffloadCapable)   (ma_IAudioClient3* pThis, MA_AUDIO_STREAM_CATEGORY category, BOOL* pOffloadCapable);\n    HRESULT (STDMETHODCALLTYPE * SetClientProperties)(ma_IAudioClient3* pThis, const ma_AudioClientProperties* pProperties);\n    HRESULT (STDMETHODCALLTYPE * GetBufferSizeLimits)(ma_IAudioClient3* pThis, const MA_WAVEFORMATEX* pFormat, BOOL eventDriven, MA_REFERENCE_TIME* pMinBufferDuration, MA_REFERENCE_TIME* pMaxBufferDuration);\n\n    /* IAudioClient3 */\n    HRESULT (STDMETHODCALLTYPE * GetSharedModeEnginePeriod)       (ma_IAudioClient3* pThis, const MA_WAVEFORMATEX* pFormat, ma_uint32* pDefaultPeriodInFrames, ma_uint32* pFundamentalPeriodInFrames, ma_uint32* pMinPeriodInFrames, ma_uint32* pMaxPeriodInFrames);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentSharedModeEnginePeriod)(ma_IAudioClient3* pThis, MA_WAVEFORMATEX** ppFormat, ma_uint32* pCurrentPeriodInFrames);\n    HRESULT (STDMETHODCALLTYPE * InitializeSharedAudioStream)     (ma_IAudioClient3* pThis, DWORD streamFlags, ma_uint32 periodInFrames, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid);\n} ma_IAudioClient3Vtbl;\nstruct ma_IAudioClient3\n{\n    ma_IAudioClient3Vtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IAudioClient3_QueryInterface(ma_IAudioClient3* pThis, const IID* const riid, void** ppObject)    { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IAudioClient3_AddRef(ma_IAudioClient3* pThis)                                                    { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IAudioClient3_Release(ma_IAudioClient3* pThis)                                                   { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_Initialize(ma_IAudioClient3* pThis, MA_AUDCLNT_SHAREMODE shareMode, DWORD streamFlags, MA_REFERENCE_TIME bufferDuration, MA_REFERENCE_TIME periodicity, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGuid) { return pThis->lpVtbl->Initialize(pThis, shareMode, streamFlags, bufferDuration, periodicity, pFormat, pAudioSessionGuid); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetBufferSize(ma_IAudioClient3* pThis, ma_uint32* pNumBufferFrames)                { return pThis->lpVtbl->GetBufferSize(pThis, pNumBufferFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetStreamLatency(ma_IAudioClient3* pThis, MA_REFERENCE_TIME* pLatency)             { return pThis->lpVtbl->GetStreamLatency(pThis, pLatency); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetCurrentPadding(ma_IAudioClient3* pThis, ma_uint32* pNumPaddingFrames)           { return pThis->lpVtbl->GetCurrentPadding(pThis, pNumPaddingFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_IsFormatSupported(ma_IAudioClient3* pThis, MA_AUDCLNT_SHAREMODE shareMode, const MA_WAVEFORMATEX* pFormat, MA_WAVEFORMATEX** ppClosestMatch) { return pThis->lpVtbl->IsFormatSupported(pThis, shareMode, pFormat, ppClosestMatch); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetMixFormat(ma_IAudioClient3* pThis, MA_WAVEFORMATEX** ppDeviceFormat)               { return pThis->lpVtbl->GetMixFormat(pThis, ppDeviceFormat); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetDevicePeriod(ma_IAudioClient3* pThis, MA_REFERENCE_TIME* pDefaultDevicePeriod, MA_REFERENCE_TIME* pMinimumDevicePeriod) { return pThis->lpVtbl->GetDevicePeriod(pThis, pDefaultDevicePeriod, pMinimumDevicePeriod); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_Start(ma_IAudioClient3* pThis)                                                     { return pThis->lpVtbl->Start(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_Stop(ma_IAudioClient3* pThis)                                                      { return pThis->lpVtbl->Stop(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_Reset(ma_IAudioClient3* pThis)                                                     { return pThis->lpVtbl->Reset(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_SetEventHandle(ma_IAudioClient3* pThis, HANDLE eventHandle)                        { return pThis->lpVtbl->SetEventHandle(pThis, eventHandle); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetService(ma_IAudioClient3* pThis, const IID* const riid, void** pp)              { return pThis->lpVtbl->GetService(pThis, riid, pp); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_IsOffloadCapable(ma_IAudioClient3* pThis, MA_AUDIO_STREAM_CATEGORY category, BOOL* pOffloadCapable) { return pThis->lpVtbl->IsOffloadCapable(pThis, category, pOffloadCapable); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_SetClientProperties(ma_IAudioClient3* pThis, const ma_AudioClientProperties* pProperties)           { return pThis->lpVtbl->SetClientProperties(pThis, pProperties); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetBufferSizeLimits(ma_IAudioClient3* pThis, const MA_WAVEFORMATEX* pFormat, BOOL eventDriven, MA_REFERENCE_TIME* pMinBufferDuration, MA_REFERENCE_TIME* pMaxBufferDuration) { return pThis->lpVtbl->GetBufferSizeLimits(pThis, pFormat, eventDriven, pMinBufferDuration, pMaxBufferDuration); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetSharedModeEnginePeriod(ma_IAudioClient3* pThis, const MA_WAVEFORMATEX* pFormat, ma_uint32* pDefaultPeriodInFrames, ma_uint32* pFundamentalPeriodInFrames, ma_uint32* pMinPeriodInFrames, ma_uint32* pMaxPeriodInFrames) { return pThis->lpVtbl->GetSharedModeEnginePeriod(pThis, pFormat, pDefaultPeriodInFrames, pFundamentalPeriodInFrames, pMinPeriodInFrames, pMaxPeriodInFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_GetCurrentSharedModeEnginePeriod(ma_IAudioClient3* pThis, MA_WAVEFORMATEX** ppFormat, ma_uint32* pCurrentPeriodInFrames) { return pThis->lpVtbl->GetCurrentSharedModeEnginePeriod(pThis, ppFormat, pCurrentPeriodInFrames); }\nstatic MA_INLINE HRESULT ma_IAudioClient3_InitializeSharedAudioStream(ma_IAudioClient3* pThis, DWORD streamFlags, ma_uint32 periodInFrames, const MA_WAVEFORMATEX* pFormat, const GUID* pAudioSessionGUID) { return pThis->lpVtbl->InitializeSharedAudioStream(pThis, streamFlags, periodInFrames, pFormat, pAudioSessionGUID); }\n\n\n/* IAudioRenderClient */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IAudioRenderClient* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IAudioRenderClient* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IAudioRenderClient* pThis);\n\n    /* IAudioRenderClient */\n    HRESULT (STDMETHODCALLTYPE * GetBuffer)    (ma_IAudioRenderClient* pThis, ma_uint32 numFramesRequested, BYTE** ppData);\n    HRESULT (STDMETHODCALLTYPE * ReleaseBuffer)(ma_IAudioRenderClient* pThis, ma_uint32 numFramesWritten, DWORD dwFlags);\n} ma_IAudioRenderClientVtbl;\nstruct ma_IAudioRenderClient\n{\n    ma_IAudioRenderClientVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IAudioRenderClient_QueryInterface(ma_IAudioRenderClient* pThis, const IID* const riid, void** ppObject)   { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IAudioRenderClient_AddRef(ma_IAudioRenderClient* pThis)                                                   { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IAudioRenderClient_Release(ma_IAudioRenderClient* pThis)                                                  { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioRenderClient_GetBuffer(ma_IAudioRenderClient* pThis, ma_uint32 numFramesRequested, BYTE** ppData)   { return pThis->lpVtbl->GetBuffer(pThis, numFramesRequested, ppData); }\nstatic MA_INLINE HRESULT ma_IAudioRenderClient_ReleaseBuffer(ma_IAudioRenderClient* pThis, ma_uint32 numFramesWritten, DWORD dwFlags) { return pThis->lpVtbl->ReleaseBuffer(pThis, numFramesWritten, dwFlags); }\n\n\n/* IAudioCaptureClient */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IAudioCaptureClient* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IAudioCaptureClient* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IAudioCaptureClient* pThis);\n\n    /* IAudioRenderClient */\n    HRESULT (STDMETHODCALLTYPE * GetBuffer)        (ma_IAudioCaptureClient* pThis, BYTE** ppData, ma_uint32* pNumFramesToRead, DWORD* pFlags, ma_uint64* pDevicePosition, ma_uint64* pQPCPosition);\n    HRESULT (STDMETHODCALLTYPE * ReleaseBuffer)    (ma_IAudioCaptureClient* pThis, ma_uint32 numFramesRead);\n    HRESULT (STDMETHODCALLTYPE * GetNextPacketSize)(ma_IAudioCaptureClient* pThis, ma_uint32* pNumFramesInNextPacket);\n} ma_IAudioCaptureClientVtbl;\nstruct ma_IAudioCaptureClient\n{\n    ma_IAudioCaptureClientVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IAudioCaptureClient_QueryInterface(ma_IAudioCaptureClient* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IAudioCaptureClient_AddRef(ma_IAudioCaptureClient* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IAudioCaptureClient_Release(ma_IAudioCaptureClient* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IAudioCaptureClient_GetBuffer(ma_IAudioCaptureClient* pThis, BYTE** ppData, ma_uint32* pNumFramesToRead, DWORD* pFlags, ma_uint64* pDevicePosition, ma_uint64* pQPCPosition) { return pThis->lpVtbl->GetBuffer(pThis, ppData, pNumFramesToRead, pFlags, pDevicePosition, pQPCPosition); }\nstatic MA_INLINE HRESULT ma_IAudioCaptureClient_ReleaseBuffer(ma_IAudioCaptureClient* pThis, ma_uint32 numFramesRead)                 { return pThis->lpVtbl->ReleaseBuffer(pThis, numFramesRead); }\nstatic MA_INLINE HRESULT ma_IAudioCaptureClient_GetNextPacketSize(ma_IAudioCaptureClient* pThis, ma_uint32* pNumFramesInNextPacket)   { return pThis->lpVtbl->GetNextPacketSize(pThis, pNumFramesInNextPacket); }\n\n#if defined(MA_WIN32_UWP)\n/* mmdevapi Functions */\ntypedef HRESULT (WINAPI * MA_PFN_ActivateAudioInterfaceAsync)(const wchar_t* deviceInterfacePath, const IID* riid, MA_PROPVARIANT* activationParams, ma_IActivateAudioInterfaceCompletionHandler* completionHandler, ma_IActivateAudioInterfaceAsyncOperation** activationOperation);\n#endif\n\n/* Avrt Functions */\ntypedef HANDLE (WINAPI * MA_PFN_AvSetMmThreadCharacteristicsA)(const char* TaskName, DWORD* TaskIndex);\ntypedef BOOL   (WINAPI * MA_PFN_AvRevertMmThreadCharacteristics)(HANDLE AvrtHandle);\n\n#if !defined(MA_WIN32_DESKTOP) && !defined(MA_WIN32_GDK)\ntypedef struct ma_completion_handler_uwp ma_completion_handler_uwp;\n\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_completion_handler_uwp* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_completion_handler_uwp* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_completion_handler_uwp* pThis);\n\n    /* IActivateAudioInterfaceCompletionHandler */\n    HRESULT (STDMETHODCALLTYPE * ActivateCompleted)(ma_completion_handler_uwp* pThis, ma_IActivateAudioInterfaceAsyncOperation* pActivateOperation);\n} ma_completion_handler_uwp_vtbl;\nstruct ma_completion_handler_uwp\n{\n    ma_completion_handler_uwp_vtbl* lpVtbl;\n    MA_ATOMIC(4, ma_uint32) counter;\n    HANDLE hEvent;\n};\n\nstatic HRESULT STDMETHODCALLTYPE ma_completion_handler_uwp_QueryInterface(ma_completion_handler_uwp* pThis, const IID* const riid, void** ppObject)\n{\n    /*\n    We need to \"implement\" IAgileObject which is just an indicator that's used internally by WASAPI for some multithreading management. To\n    \"implement\" this, we just make sure we return pThis when the IAgileObject is requested.\n    */\n    if (!ma_is_guid_equal(riid, &MA_IID_IUnknown) && !ma_is_guid_equal(riid, &MA_IID_IActivateAudioInterfaceCompletionHandler) && !ma_is_guid_equal(riid, &MA_IID_IAgileObject)) {\n        *ppObject = NULL;\n        return E_NOINTERFACE;\n    }\n\n    /* Getting here means the IID is IUnknown or IMMNotificationClient. */\n    *ppObject = (void*)pThis;\n    ((ma_completion_handler_uwp_vtbl*)pThis->lpVtbl)->AddRef(pThis);\n    return S_OK;\n}\n\nstatic ULONG STDMETHODCALLTYPE ma_completion_handler_uwp_AddRef(ma_completion_handler_uwp* pThis)\n{\n    return (ULONG)ma_atomic_fetch_add_32(&pThis->counter, 1) + 1;\n}\n\nstatic ULONG STDMETHODCALLTYPE ma_completion_handler_uwp_Release(ma_completion_handler_uwp* pThis)\n{\n    ma_uint32 newRefCount = ma_atomic_fetch_sub_32(&pThis->counter, 1) - 1;\n    if (newRefCount == 0) {\n        return 0;   /* We don't free anything here because we never allocate the object on the heap. */\n    }\n\n    return (ULONG)newRefCount;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_completion_handler_uwp_ActivateCompleted(ma_completion_handler_uwp* pThis, ma_IActivateAudioInterfaceAsyncOperation* pActivateOperation)\n{\n    (void)pActivateOperation;\n    SetEvent(pThis->hEvent);\n    return S_OK;\n}\n\n\nstatic ma_completion_handler_uwp_vtbl g_maCompletionHandlerVtblInstance = {\n    ma_completion_handler_uwp_QueryInterface,\n    ma_completion_handler_uwp_AddRef,\n    ma_completion_handler_uwp_Release,\n    ma_completion_handler_uwp_ActivateCompleted\n};\n\nstatic ma_result ma_completion_handler_uwp_init(ma_completion_handler_uwp* pHandler)\n{\n    MA_ASSERT(pHandler != NULL);\n    MA_ZERO_OBJECT(pHandler);\n\n    pHandler->lpVtbl = &g_maCompletionHandlerVtblInstance;\n    pHandler->counter = 1;\n    pHandler->hEvent = CreateEventA(NULL, FALSE, FALSE, NULL);\n    if (pHandler->hEvent == NULL) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_completion_handler_uwp_uninit(ma_completion_handler_uwp* pHandler)\n{\n    if (pHandler->hEvent != NULL) {\n        CloseHandle(pHandler->hEvent);\n    }\n}\n\nstatic void ma_completion_handler_uwp_wait(ma_completion_handler_uwp* pHandler)\n{\n    WaitForSingleObject((HANDLE)pHandler->hEvent, INFINITE);\n}\n#endif  /* !MA_WIN32_DESKTOP */\n\n/* We need a virtual table for our notification client object that's used for detecting changes to the default device. */\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_QueryInterface(ma_IMMNotificationClient* pThis, const IID* const riid, void** ppObject)\n{\n    /*\n    We care about two interfaces - IUnknown and IMMNotificationClient. If the requested IID is something else\n    we just return E_NOINTERFACE. Otherwise we need to increment the reference counter and return S_OK.\n    */\n    if (!ma_is_guid_equal(riid, &MA_IID_IUnknown) && !ma_is_guid_equal(riid, &MA_IID_IMMNotificationClient)) {\n        *ppObject = NULL;\n        return E_NOINTERFACE;\n    }\n\n    /* Getting here means the IID is IUnknown or IMMNotificationClient. */\n    *ppObject = (void*)pThis;\n    ((ma_IMMNotificationClientVtbl*)pThis->lpVtbl)->AddRef(pThis);\n    return S_OK;\n}\n\nstatic ULONG STDMETHODCALLTYPE ma_IMMNotificationClient_AddRef(ma_IMMNotificationClient* pThis)\n{\n    return (ULONG)ma_atomic_fetch_add_32(&pThis->counter, 1) + 1;\n}\n\nstatic ULONG STDMETHODCALLTYPE ma_IMMNotificationClient_Release(ma_IMMNotificationClient* pThis)\n{\n    ma_uint32 newRefCount = ma_atomic_fetch_sub_32(&pThis->counter, 1) - 1;\n    if (newRefCount == 0) {\n        return 0;   /* We don't free anything here because we never allocate the object on the heap. */\n    }\n\n    return (ULONG)newRefCount;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_OnDeviceStateChanged(ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID, DWORD dwNewState)\n{\n    ma_bool32 isThisDevice = MA_FALSE;\n    ma_bool32 isCapture    = MA_FALSE;\n    ma_bool32 isPlayback   = MA_FALSE;\n\n#ifdef MA_DEBUG_OUTPUT\n    /*ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"IMMNotificationClient_OnDeviceStateChanged(pDeviceID=%S, dwNewState=%u)\\n\", (pDeviceID != NULL) ? pDeviceID : L\"(NULL)\", (unsigned int)dwNewState);*/\n#endif\n\n    /*\n    There have been reports of a hang when a playback device is disconnected. The idea with this code is to explicitly stop the device if we detect\n    that the device is disabled or has been unplugged.\n    */\n    if (pThis->pDevice->wasapi.allowCaptureAutoStreamRouting && (pThis->pDevice->type == ma_device_type_capture || pThis->pDevice->type == ma_device_type_duplex || pThis->pDevice->type == ma_device_type_loopback)) {\n        isCapture = MA_TRUE;\n        if (ma_strcmp_WCHAR(pThis->pDevice->capture.id.wasapi, pDeviceID) == 0) {\n            isThisDevice = MA_TRUE;\n        }\n    }\n\n    if (pThis->pDevice->wasapi.allowPlaybackAutoStreamRouting && (pThis->pDevice->type == ma_device_type_playback || pThis->pDevice->type == ma_device_type_duplex)) {\n        isPlayback = MA_TRUE;\n        if (ma_strcmp_WCHAR(pThis->pDevice->playback.id.wasapi, pDeviceID) == 0) {\n            isThisDevice = MA_TRUE;\n        }\n    }\n\n\n    /*\n    If the device ID matches our device we need to mark our device as detached and stop it. When a\n    device is added in OnDeviceAdded(), we'll restart it. We only mark it as detached if the device\n    was started at the time of being removed.\n    */\n    if (isThisDevice) {\n        if ((dwNewState & MA_MM_DEVICE_STATE_ACTIVE) == 0) {\n            /*\n            Unplugged or otherwise unavailable. Mark as detached if we were in a playing state. We'll\n            use this to determine whether or not we need to automatically start the device when it's\n            plugged back in again.\n            */\n            if (ma_device_get_state(pThis->pDevice) == ma_device_state_started) {\n                if (isPlayback) {\n                    pThis->pDevice->wasapi.isDetachedPlayback = MA_TRUE;\n                }\n                if (isCapture) {\n                    pThis->pDevice->wasapi.isDetachedCapture = MA_TRUE;\n                }\n\n                ma_device_stop(pThis->pDevice);\n            }\n        }\n\n        if ((dwNewState & MA_MM_DEVICE_STATE_ACTIVE) != 0) {\n            /* The device was activated. If we were detached, we need to start it again. */\n            ma_bool8 tryRestartingDevice = MA_FALSE;\n\n            if (isPlayback) {\n                if (pThis->pDevice->wasapi.isDetachedPlayback) {\n                    pThis->pDevice->wasapi.isDetachedPlayback = MA_FALSE;\n                    ma_device_reroute__wasapi(pThis->pDevice, ma_device_type_playback);\n                    tryRestartingDevice = MA_TRUE;\n                }\n            }\n\n            if (isCapture) {\n                if (pThis->pDevice->wasapi.isDetachedCapture) {\n                    pThis->pDevice->wasapi.isDetachedCapture = MA_FALSE;\n                    ma_device_reroute__wasapi(pThis->pDevice, (pThis->pDevice->type == ma_device_type_loopback) ? ma_device_type_loopback : ma_device_type_capture);\n                    tryRestartingDevice = MA_TRUE;\n                }\n            }\n\n            if (tryRestartingDevice) {\n                if (pThis->pDevice->wasapi.isDetachedPlayback == MA_FALSE && pThis->pDevice->wasapi.isDetachedCapture == MA_FALSE) {\n                    ma_device_start(pThis->pDevice);\n                }\n            }\n        }\n    }\n\n    return S_OK;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_OnDeviceAdded(ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID)\n{\n#ifdef MA_DEBUG_OUTPUT\n    /*ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"IMMNotificationClient_OnDeviceAdded(pDeviceID=%S)\\n\", (pDeviceID != NULL) ? pDeviceID : L\"(NULL)\");*/\n#endif\n\n    /* We don't need to worry about this event for our purposes. */\n    (void)pThis;\n    (void)pDeviceID;\n    return S_OK;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_OnDeviceRemoved(ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID)\n{\n#ifdef MA_DEBUG_OUTPUT\n    /*ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"IMMNotificationClient_OnDeviceRemoved(pDeviceID=%S)\\n\", (pDeviceID != NULL) ? pDeviceID : L\"(NULL)\");*/\n#endif\n\n    /* We don't need to worry about this event for our purposes. */\n    (void)pThis;\n    (void)pDeviceID;\n    return S_OK;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_OnDefaultDeviceChanged(ma_IMMNotificationClient* pThis, ma_EDataFlow dataFlow, ma_ERole role, const WCHAR* pDefaultDeviceID)\n{\n#ifdef MA_DEBUG_OUTPUT\n    /*ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"IMMNotificationClient_OnDefaultDeviceChanged(dataFlow=%d, role=%d, pDefaultDeviceID=%S)\\n\", dataFlow, role, (pDefaultDeviceID != NULL) ? pDefaultDeviceID : L\"(NULL)\");*/\n#endif\n\n    (void)role;\n\n    /* We only care about devices with the same data flow as the current device. */\n    if ((pThis->pDevice->type == ma_device_type_playback && dataFlow != ma_eRender)  ||\n        (pThis->pDevice->type == ma_device_type_capture  && dataFlow != ma_eCapture) ||\n        (pThis->pDevice->type == ma_device_type_loopback && dataFlow != ma_eRender)) {\n        ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Stream rerouting abandoned because dataFlow does match device type.\\n\");\n        return S_OK;\n    }\n\n    /* We need to consider dataFlow as ma_eCapture if device is ma_device_type_loopback */\n    if (pThis->pDevice->type == ma_device_type_loopback) {\n        dataFlow = ma_eCapture;\n    }\n\n    /* Don't do automatic stream routing if we're not allowed. */\n    if ((dataFlow == ma_eRender  && pThis->pDevice->wasapi.allowPlaybackAutoStreamRouting == MA_FALSE) ||\n        (dataFlow == ma_eCapture && pThis->pDevice->wasapi.allowCaptureAutoStreamRouting  == MA_FALSE)) {\n        ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Stream rerouting abandoned because automatic stream routing has been disabled by the device config.\\n\");\n        return S_OK;\n    }\n\n    /*\n    Not currently supporting automatic stream routing in exclusive mode. This is not working correctly on my machine due to\n    AUDCLNT_E_DEVICE_IN_USE errors when reinitializing the device. If this is a bug in miniaudio, we can try re-enabling this once\n    it's fixed.\n    */\n    if ((dataFlow == ma_eRender  && pThis->pDevice->playback.shareMode == ma_share_mode_exclusive) ||\n        (dataFlow == ma_eCapture && pThis->pDevice->capture.shareMode  == ma_share_mode_exclusive)) {\n        ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Stream rerouting abandoned because the device shared mode is exclusive.\\n\");\n        return S_OK;\n    }\n\n\n\n    /*\n    Second attempt at device rerouting. We're going to retrieve the device's state at the time of\n    the route change. We're then going to stop the device, reinitialize the device, and then start\n    it again if the state before stopping was ma_device_state_started.\n    */\n    {\n        ma_uint32 previousState = ma_device_get_state(pThis->pDevice);\n        ma_bool8 restartDevice = MA_FALSE;\n\n        if (previousState == ma_device_state_uninitialized || previousState == ma_device_state_starting) {\n            ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Stream rerouting abandoned because the device is in the process of starting.\\n\");\n            return S_OK;\n        }\n\n        if (previousState == ma_device_state_started) {\n            ma_device_stop(pThis->pDevice);\n            restartDevice = MA_TRUE;\n        }\n\n        if (pDefaultDeviceID != NULL) { /* <-- The input device ID will be null if there's no other device available. */\n            ma_mutex_lock(&pThis->pDevice->wasapi.rerouteLock);\n            {\n                if (dataFlow == ma_eRender) {\n                    ma_device_reroute__wasapi(pThis->pDevice, ma_device_type_playback);\n\n                    if (pThis->pDevice->wasapi.isDetachedPlayback) {\n                        pThis->pDevice->wasapi.isDetachedPlayback = MA_FALSE;\n\n                        if (pThis->pDevice->type == ma_device_type_duplex && pThis->pDevice->wasapi.isDetachedCapture) {\n                            restartDevice = MA_FALSE;   /* It's a duplex device and the capture side is detached. We cannot be restarting the device just yet. */\n                        }\n                        else {\n                            restartDevice = MA_TRUE;    /* It's not a duplex device, or the capture side is also attached so we can go ahead and restart the device. */\n                        }\n                    }\n                }\n                else {\n                    ma_device_reroute__wasapi(pThis->pDevice, (pThis->pDevice->type == ma_device_type_loopback) ? ma_device_type_loopback : ma_device_type_capture);\n\n                    if (pThis->pDevice->wasapi.isDetachedCapture) {\n                        pThis->pDevice->wasapi.isDetachedCapture = MA_FALSE;\n\n                        if (pThis->pDevice->type == ma_device_type_duplex && pThis->pDevice->wasapi.isDetachedPlayback) {\n                            restartDevice = MA_FALSE;   /* It's a duplex device and the playback side is detached. We cannot be restarting the device just yet. */\n                        }\n                        else {\n                            restartDevice = MA_TRUE;    /* It's not a duplex device, or the playback side is also attached so we can go ahead and restart the device. */\n                        }\n                    }\n                }\n            }\n            ma_mutex_unlock(&pThis->pDevice->wasapi.rerouteLock);\n\n            if (restartDevice) {\n                ma_device_start(pThis->pDevice);\n            }\n        }\n    }\n\n    return S_OK;\n}\n\nstatic HRESULT STDMETHODCALLTYPE ma_IMMNotificationClient_OnPropertyValueChanged(ma_IMMNotificationClient* pThis, const WCHAR* pDeviceID, const PROPERTYKEY key)\n{\n#ifdef MA_DEBUG_OUTPUT\n    /*ma_log_postf(ma_device_get_log(pThis->pDevice), MA_LOG_LEVEL_DEBUG, \"IMMNotificationClient_OnPropertyValueChanged(pDeviceID=%S)\\n\", (pDeviceID != NULL) ? pDeviceID : L\"(NULL)\");*/\n#endif\n\n    (void)pThis;\n    (void)pDeviceID;\n    (void)key;\n    return S_OK;\n}\n\nstatic ma_IMMNotificationClientVtbl g_maNotificationCientVtbl = {\n    ma_IMMNotificationClient_QueryInterface,\n    ma_IMMNotificationClient_AddRef,\n    ma_IMMNotificationClient_Release,\n    ma_IMMNotificationClient_OnDeviceStateChanged,\n    ma_IMMNotificationClient_OnDeviceAdded,\n    ma_IMMNotificationClient_OnDeviceRemoved,\n    ma_IMMNotificationClient_OnDefaultDeviceChanged,\n    ma_IMMNotificationClient_OnPropertyValueChanged\n};\n#endif  /* MA_WIN32_DESKTOP */\n\nstatic const char* ma_to_usage_string__wasapi(ma_wasapi_usage usage)\n{\n    switch (usage)\n    {\n        case ma_wasapi_usage_default:   return NULL;\n        case ma_wasapi_usage_games:     return \"Games\";\n        case ma_wasapi_usage_pro_audio: return \"Pro Audio\";\n        default: break;\n    }\n\n    return NULL;\n}\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\ntypedef ma_IMMDevice ma_WASAPIDeviceInterface;\n#else\ntypedef ma_IUnknown ma_WASAPIDeviceInterface;\n#endif\n\n\n#define MA_CONTEXT_COMMAND_QUIT__WASAPI                 1\n#define MA_CONTEXT_COMMAND_CREATE_IAUDIOCLIENT__WASAPI  2\n#define MA_CONTEXT_COMMAND_RELEASE_IAUDIOCLIENT__WASAPI 3\n\nstatic ma_context_command__wasapi ma_context_init_command__wasapi(int code)\n{\n    ma_context_command__wasapi cmd;\n\n    MA_ZERO_OBJECT(&cmd);\n    cmd.code = code;\n\n    return cmd;\n}\n\nstatic ma_result ma_context_post_command__wasapi(ma_context* pContext, const ma_context_command__wasapi* pCmd)\n{\n    /* For now we are doing everything synchronously, but I might relax this later if the need arises. */\n    ma_result result;\n    ma_bool32 isUsingLocalEvent = MA_FALSE;\n    ma_event localEvent;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pCmd     != NULL);\n\n    if (pCmd->pEvent == NULL) {\n        isUsingLocalEvent = MA_TRUE;\n\n        result = ma_event_init(&localEvent);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to create the event for this command. */\n        }\n    }\n\n    /* Here is where we add the command to the list. If there's not enough room we'll spin until there is. */\n    ma_mutex_lock(&pContext->wasapi.commandLock);\n    {\n        ma_uint32 index;\n\n        /* Spin until we've got some space available. */\n        while (pContext->wasapi.commandCount == ma_countof(pContext->wasapi.commands)) {\n            ma_yield();\n        }\n\n        /* Space is now available. Can safely add to the list. */\n        index = (pContext->wasapi.commandIndex + pContext->wasapi.commandCount) % ma_countof(pContext->wasapi.commands);\n        pContext->wasapi.commands[index]        = *pCmd;\n        pContext->wasapi.commands[index].pEvent = &localEvent;\n        pContext->wasapi.commandCount += 1;\n\n        /* Now that the command has been added, release the semaphore so ma_context_next_command__wasapi() can return. */\n        ma_semaphore_release(&pContext->wasapi.commandSem);\n    }\n    ma_mutex_unlock(&pContext->wasapi.commandLock);\n\n    if (isUsingLocalEvent) {\n        ma_event_wait(&localEvent);\n        ma_event_uninit(&localEvent);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_next_command__wasapi(ma_context* pContext, ma_context_command__wasapi* pCmd)\n{\n    ma_result result = MA_SUCCESS;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pCmd     != NULL);\n\n    result = ma_semaphore_wait(&pContext->wasapi.commandSem);\n    if (result == MA_SUCCESS) {\n        ma_mutex_lock(&pContext->wasapi.commandLock);\n        {\n            *pCmd = pContext->wasapi.commands[pContext->wasapi.commandIndex];\n            pContext->wasapi.commandIndex  = (pContext->wasapi.commandIndex + 1) % ma_countof(pContext->wasapi.commands);\n            pContext->wasapi.commandCount -= 1;\n        }\n        ma_mutex_unlock(&pContext->wasapi.commandLock);\n    }\n\n    return result;\n}\n\nstatic ma_thread_result MA_THREADCALL ma_context_command_thread__wasapi(void* pUserData)\n{\n    ma_result result;\n    ma_context* pContext = (ma_context*)pUserData;\n    MA_ASSERT(pContext != NULL);\n\n    for (;;) {\n        ma_context_command__wasapi cmd;\n        result = ma_context_next_command__wasapi(pContext, &cmd);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        switch (cmd.code)\n        {\n            case MA_CONTEXT_COMMAND_QUIT__WASAPI:\n            {\n                /* Do nothing. Handled after the switch. */\n            } break;\n\n            case MA_CONTEXT_COMMAND_CREATE_IAUDIOCLIENT__WASAPI:\n            {\n                if (cmd.data.createAudioClient.deviceType == ma_device_type_playback) {\n                    *cmd.data.createAudioClient.pResult = ma_result_from_HRESULT(ma_IAudioClient_GetService((ma_IAudioClient*)cmd.data.createAudioClient.pAudioClient, &MA_IID_IAudioRenderClient, cmd.data.createAudioClient.ppAudioClientService));\n                } else {\n                    *cmd.data.createAudioClient.pResult = ma_result_from_HRESULT(ma_IAudioClient_GetService((ma_IAudioClient*)cmd.data.createAudioClient.pAudioClient, &MA_IID_IAudioCaptureClient, cmd.data.createAudioClient.ppAudioClientService));\n                }\n            } break;\n\n            case MA_CONTEXT_COMMAND_RELEASE_IAUDIOCLIENT__WASAPI:\n            {\n                if (cmd.data.releaseAudioClient.deviceType == ma_device_type_playback) {\n                    if (cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientPlayback != NULL) {\n                        ma_IAudioClient_Release((ma_IAudioClient*)cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientPlayback);\n                        cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientPlayback = NULL;\n                    }\n                }\n\n                if (cmd.data.releaseAudioClient.deviceType == ma_device_type_capture) {\n                    if (cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientCapture != NULL) {\n                        ma_IAudioClient_Release((ma_IAudioClient*)cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientCapture);\n                        cmd.data.releaseAudioClient.pDevice->wasapi.pAudioClientCapture = NULL;\n                    }\n                }\n            } break;\n\n            default:\n            {\n                /* Unknown command. Ignore it, but trigger an assert in debug mode so we're aware of it. */\n                MA_ASSERT(MA_FALSE);\n            } break;\n        }\n\n        if (cmd.pEvent != NULL) {\n            ma_event_signal(cmd.pEvent);\n        }\n\n        if (cmd.code == MA_CONTEXT_COMMAND_QUIT__WASAPI) {\n            break;  /* Received a quit message. Get out of here. */\n        }\n    }\n\n    return (ma_thread_result)0;\n}\n\nstatic ma_result ma_device_create_IAudioClient_service__wasapi(ma_context* pContext, ma_device_type deviceType, ma_IAudioClient* pAudioClient, void** ppAudioClientService)\n{\n    ma_result result;\n    ma_result cmdResult;\n    ma_context_command__wasapi cmd = ma_context_init_command__wasapi(MA_CONTEXT_COMMAND_CREATE_IAUDIOCLIENT__WASAPI);\n    cmd.data.createAudioClient.deviceType           = deviceType;\n    cmd.data.createAudioClient.pAudioClient         = (void*)pAudioClient;\n    cmd.data.createAudioClient.ppAudioClientService = ppAudioClientService;\n    cmd.data.createAudioClient.pResult              = &cmdResult;   /* Declared locally, but won't be dereferenced after this function returns since execution of the command will wait here. */\n\n    result = ma_context_post_command__wasapi(pContext, &cmd);  /* This will not return until the command has actually been run. */\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return *cmd.data.createAudioClient.pResult;\n}\n\n#if 0   /* Not used at the moment, but leaving here for future use. */\nstatic ma_result ma_device_release_IAudioClient_service__wasapi(ma_device* pDevice, ma_device_type deviceType)\n{\n    ma_result result;\n    ma_context_command__wasapi cmd = ma_context_init_command__wasapi(MA_CONTEXT_COMMAND_RELEASE_IAUDIOCLIENT__WASAPI);\n    cmd.data.releaseAudioClient.pDevice    = pDevice;\n    cmd.data.releaseAudioClient.deviceType = deviceType;\n\n    result = ma_context_post_command__wasapi(pDevice->pContext, &cmd);  /* This will not return until the command has actually been run. */\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n#endif\n\n\nstatic void ma_add_native_data_format_to_device_info_from_WAVEFORMATEX(const MA_WAVEFORMATEX* pWF, ma_share_mode shareMode, ma_device_info* pInfo)\n{\n    MA_ASSERT(pWF != NULL);\n    MA_ASSERT(pInfo != NULL);\n\n    if (pInfo->nativeDataFormatCount >= ma_countof(pInfo->nativeDataFormats)) {\n        return; /* Too many data formats. Need to ignore this one. Don't think this should ever happen with WASAPI. */\n    }\n\n    pInfo->nativeDataFormats[pInfo->nativeDataFormatCount].format     = ma_format_from_WAVEFORMATEX(pWF);\n    pInfo->nativeDataFormats[pInfo->nativeDataFormatCount].channels   = pWF->nChannels;\n    pInfo->nativeDataFormats[pInfo->nativeDataFormatCount].sampleRate = pWF->nSamplesPerSec;\n    pInfo->nativeDataFormats[pInfo->nativeDataFormatCount].flags      = (shareMode == ma_share_mode_exclusive) ? MA_DATA_FORMAT_FLAG_EXCLUSIVE_MODE : 0;\n    pInfo->nativeDataFormatCount += 1;\n}\n\nstatic ma_result ma_context_get_device_info_from_IAudioClient__wasapi(ma_context* pContext, /*ma_IMMDevice**/void* pMMDevice, ma_IAudioClient* pAudioClient, ma_device_info* pInfo)\n{\n    HRESULT hr;\n    MA_WAVEFORMATEX* pWF = NULL;\n\n    MA_ASSERT(pAudioClient != NULL);\n    MA_ASSERT(pInfo != NULL);\n\n    /* Shared Mode. We use GetMixFormat() here. */\n    hr = ma_IAudioClient_GetMixFormat((ma_IAudioClient*)pAudioClient, (MA_WAVEFORMATEX**)&pWF);\n    if (SUCCEEDED(hr)) {\n        ma_add_native_data_format_to_device_info_from_WAVEFORMATEX(pWF, ma_share_mode_shared, pInfo);\n    } else {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to retrieve mix format for device info retrieval.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    /*\n    Exclusive Mode. We repeatedly call IsFormatSupported() here. This is not currently supported on\n    UWP. Failure to retrieve the exclusive mode format is not considered an error, so from here on\n    out, MA_SUCCESS is guaranteed to be returned.\n    */\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    {\n        ma_IPropertyStore *pProperties;\n\n        /*\n        The first thing to do is get the format from PKEY_AudioEngine_DeviceFormat. This should give us a channel count we assume is\n        correct which will simplify our searching.\n        */\n        hr = ma_IMMDevice_OpenPropertyStore((ma_IMMDevice*)pMMDevice, STGM_READ, &pProperties);\n        if (SUCCEEDED(hr)) {\n            MA_PROPVARIANT var;\n            ma_PropVariantInit(&var);\n\n            hr = ma_IPropertyStore_GetValue(pProperties, &MA_PKEY_AudioEngine_DeviceFormat, &var);\n            if (SUCCEEDED(hr)) {\n                pWF = (MA_WAVEFORMATEX*)var.blob.pBlobData;\n\n                /*\n                In my testing, the format returned by PKEY_AudioEngine_DeviceFormat is suitable for exclusive mode so we check this format\n                first. If this fails, fall back to a search.\n                */\n                hr = ma_IAudioClient_IsFormatSupported((ma_IAudioClient*)pAudioClient, MA_AUDCLNT_SHAREMODE_EXCLUSIVE, pWF, NULL);\n                if (SUCCEEDED(hr)) {\n                    /* The format returned by PKEY_AudioEngine_DeviceFormat is supported. */\n                    ma_add_native_data_format_to_device_info_from_WAVEFORMATEX(pWF, ma_share_mode_exclusive, pInfo);\n                } else {\n                    /*\n                    The format returned by PKEY_AudioEngine_DeviceFormat is not supported, so fall back to a search. We assume the channel\n                    count returned by MA_PKEY_AudioEngine_DeviceFormat is valid and correct. For simplicity we're only returning one format.\n                    */\n                    ma_uint32 channels = pWF->nChannels;\n                    ma_channel defaultChannelMap[MA_MAX_CHANNELS];\n                    MA_WAVEFORMATEXTENSIBLE wf;\n                    ma_bool32 found;\n                    ma_uint32 iFormat;\n\n                    /* Make sure we don't overflow the channel map. */\n                    if (channels > MA_MAX_CHANNELS) {\n                        channels = MA_MAX_CHANNELS;\n                    }\n\n                    ma_channel_map_init_standard(ma_standard_channel_map_microsoft, defaultChannelMap, ma_countof(defaultChannelMap), channels);\n\n                    MA_ZERO_OBJECT(&wf);\n                    wf.cbSize     = sizeof(wf);\n                    wf.wFormatTag = WAVE_FORMAT_EXTENSIBLE;\n                    wf.nChannels  = (WORD)channels;\n                    wf.dwChannelMask     = ma_channel_map_to_channel_mask__win32(defaultChannelMap, channels);\n\n                    found = MA_FALSE;\n                    for (iFormat = 0; iFormat < ma_countof(g_maFormatPriorities); ++iFormat) {\n                        ma_format format = g_maFormatPriorities[iFormat];\n                        ma_uint32 iSampleRate;\n\n                        wf.wBitsPerSample       = (WORD)(ma_get_bytes_per_sample(format)*8);\n                        wf.nBlockAlign          = (WORD)(wf.nChannels * wf.wBitsPerSample / 8);\n                        wf.nAvgBytesPerSec      = wf.nBlockAlign * wf.nSamplesPerSec;\n                        wf.Samples.wValidBitsPerSample = /*(format == ma_format_s24_32) ? 24 :*/ wf.wBitsPerSample;\n                        if (format == ma_format_f32) {\n                            wf.SubFormat = MA_GUID_KSDATAFORMAT_SUBTYPE_IEEE_FLOAT;\n                        } else {\n                            wf.SubFormat = MA_GUID_KSDATAFORMAT_SUBTYPE_PCM;\n                        }\n\n                        for (iSampleRate = 0; iSampleRate < ma_countof(g_maStandardSampleRatePriorities); ++iSampleRate) {\n                            wf.nSamplesPerSec = g_maStandardSampleRatePriorities[iSampleRate];\n\n                            hr = ma_IAudioClient_IsFormatSupported((ma_IAudioClient*)pAudioClient, MA_AUDCLNT_SHAREMODE_EXCLUSIVE, (MA_WAVEFORMATEX*)&wf, NULL);\n                            if (SUCCEEDED(hr)) {\n                                ma_add_native_data_format_to_device_info_from_WAVEFORMATEX((MA_WAVEFORMATEX*)&wf, ma_share_mode_exclusive, pInfo);\n                                found = MA_TRUE;\n                                break;\n                            }\n                        }\n\n                        if (found) {\n                            break;\n                        }\n                    }\n\n                    ma_PropVariantClear(pContext, &var);\n\n                    if (!found) {\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"[WASAPI] Failed to find suitable device format for device info retrieval.\");\n                    }\n                }\n            } else {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"[WASAPI] Failed to retrieve device format for device info retrieval.\");\n            }\n\n            ma_IPropertyStore_Release(pProperties);\n        } else {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"[WASAPI] Failed to open property store for device info retrieval.\");\n        }\n    }\n    #else\n    {\n        (void)pMMDevice;    /* Unused. */\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\nstatic ma_EDataFlow ma_device_type_to_EDataFlow(ma_device_type deviceType)\n{\n    if (deviceType == ma_device_type_playback) {\n        return ma_eRender;\n    } else if (deviceType == ma_device_type_capture) {\n        return ma_eCapture;\n    } else {\n        MA_ASSERT(MA_FALSE);\n        return ma_eRender; /* Should never hit this. */\n    }\n}\n\nstatic ma_result ma_context_create_IMMDeviceEnumerator__wasapi(ma_context* pContext, ma_IMMDeviceEnumerator** ppDeviceEnumerator)\n{\n    HRESULT hr;\n    ma_IMMDeviceEnumerator* pDeviceEnumerator;\n\n    MA_ASSERT(pContext           != NULL);\n    MA_ASSERT(ppDeviceEnumerator != NULL);\n\n    *ppDeviceEnumerator = NULL; /* Safety. */\n\n    hr = ma_CoCreateInstance(pContext, &MA_CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, &MA_IID_IMMDeviceEnumerator, (void**)&pDeviceEnumerator);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create device enumerator.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    *ppDeviceEnumerator = pDeviceEnumerator;\n\n    return MA_SUCCESS;\n}\n\nstatic WCHAR* ma_context_get_default_device_id_from_IMMDeviceEnumerator__wasapi(ma_context* pContext, ma_IMMDeviceEnumerator* pDeviceEnumerator, ma_device_type deviceType)\n{\n    HRESULT hr;\n    ma_IMMDevice* pMMDefaultDevice = NULL;\n    WCHAR* pDefaultDeviceID = NULL;\n    ma_EDataFlow dataFlow;\n    ma_ERole role;\n\n    MA_ASSERT(pContext          != NULL);\n    MA_ASSERT(pDeviceEnumerator != NULL);\n\n    (void)pContext;\n\n    /* Grab the EDataFlow type from the device type. */\n    dataFlow = ma_device_type_to_EDataFlow(deviceType);\n\n    /* The role is always eConsole, but we may make this configurable later. */\n    role = ma_eConsole;\n\n    hr = ma_IMMDeviceEnumerator_GetDefaultAudioEndpoint(pDeviceEnumerator, dataFlow, role, &pMMDefaultDevice);\n    if (FAILED(hr)) {\n        return NULL;\n    }\n\n    hr = ma_IMMDevice_GetId(pMMDefaultDevice, &pDefaultDeviceID);\n\n    ma_IMMDevice_Release(pMMDefaultDevice);\n    pMMDefaultDevice = NULL;\n\n    if (FAILED(hr)) {\n        return NULL;\n    }\n\n    return pDefaultDeviceID;\n}\n\nstatic WCHAR* ma_context_get_default_device_id__wasapi(ma_context* pContext, ma_device_type deviceType)    /* Free the returned pointer with ma_CoTaskMemFree() */\n{\n    ma_result result;\n    ma_IMMDeviceEnumerator* pDeviceEnumerator;\n    WCHAR* pDefaultDeviceID = NULL;\n\n    MA_ASSERT(pContext != NULL);\n\n    result = ma_context_create_IMMDeviceEnumerator__wasapi(pContext, &pDeviceEnumerator);\n    if (result != MA_SUCCESS) {\n        return NULL;\n    }\n\n    pDefaultDeviceID = ma_context_get_default_device_id_from_IMMDeviceEnumerator__wasapi(pContext, pDeviceEnumerator, deviceType);\n\n    ma_IMMDeviceEnumerator_Release(pDeviceEnumerator);\n    return pDefaultDeviceID;\n}\n\nstatic ma_result ma_context_get_MMDevice__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_IMMDevice** ppMMDevice)\n{\n    ma_IMMDeviceEnumerator* pDeviceEnumerator;\n    HRESULT hr;\n    HRESULT CoInitializeResult;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppMMDevice != NULL);\n\n    /*\n    This weird COM init/uninit here is a hack to work around a crash when changing devices. What is happening is\n    WASAPI fires a callback from another thread when the device is changed. It's from that thread where this\n    function is getting called. What I'm suspecting is that the other thread is not initializing COM which in turn\n    results in CoCreateInstance() failing.\n\n    The community has reported that this seems to fix the crash. There are future plans to move all WASAPI operation\n    over to a single thread to make everything safer, but in the meantime while we wait for that to come online I'm\n    happy enough to use this hack instead.\n\n    CoUninitialize should only be called if we successfully initialized. S_OK and S_FALSE both mean that we need to\n    call CoUninitialize since the internal ref count was increased. RPC_E_CHANGED_MODE means that CoInitializeEx was\n    called with a different COINIT value, and we don't call CoUninitialize in that case. Other errors are possible,\n    so we check for S_OK and S_FALSE specifically.\n    */\n    CoInitializeResult = ma_CoInitializeEx(pContext, NULL, MA_COINIT_VALUE);\n    {\n        hr = ma_CoCreateInstance(pContext, &MA_CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, &MA_IID_IMMDeviceEnumerator, (void**)&pDeviceEnumerator);\n    }    \n    if (CoInitializeResult == S_OK || CoInitializeResult == S_FALSE) { ma_CoUninitialize(pContext); }\n\n    if (FAILED(hr)) {   /* <-- This is checking the call above to ma_CoCreateInstance(). */\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create IMMDeviceEnumerator.\\n\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    if (pDeviceID == NULL) {\n        hr = ma_IMMDeviceEnumerator_GetDefaultAudioEndpoint(pDeviceEnumerator, (deviceType == ma_device_type_capture) ? ma_eCapture : ma_eRender, ma_eConsole, ppMMDevice);\n    } else {\n        hr = ma_IMMDeviceEnumerator_GetDevice(pDeviceEnumerator, pDeviceID->wasapi, ppMMDevice);\n    }\n\n    ma_IMMDeviceEnumerator_Release(pDeviceEnumerator);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to retrieve IMMDevice.\\n\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_id_from_MMDevice__wasapi(ma_context* pContext, ma_IMMDevice* pMMDevice, ma_device_id* pDeviceID)\n{\n    WCHAR* pDeviceIDString;\n    HRESULT hr;\n\n    MA_ASSERT(pDeviceID != NULL);\n\n    hr = ma_IMMDevice_GetId(pMMDevice, &pDeviceIDString);\n    if (SUCCEEDED(hr)) {\n        size_t idlen = ma_strlen_WCHAR(pDeviceIDString);\n        if (idlen+1 > ma_countof(pDeviceID->wasapi)) {\n            ma_CoTaskMemFree(pContext, pDeviceIDString);\n            MA_ASSERT(MA_FALSE);  /* NOTE: If this is triggered, please report it. It means the format of the ID must have changed and is too long to fit in our fixed sized buffer. */\n            return MA_ERROR;\n        }\n\n        MA_COPY_MEMORY(pDeviceID->wasapi, pDeviceIDString, idlen * sizeof(wchar_t));\n        pDeviceID->wasapi[idlen] = '\\0';\n\n        ma_CoTaskMemFree(pContext, pDeviceIDString);\n\n        return MA_SUCCESS;\n    }\n\n    return MA_ERROR;\n}\n\nstatic ma_result ma_context_get_device_info_from_MMDevice__wasapi(ma_context* pContext, ma_IMMDevice* pMMDevice, WCHAR* pDefaultDeviceID, ma_bool32 onlySimpleInfo, ma_device_info* pInfo)\n{\n    ma_result result;\n    HRESULT hr;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pMMDevice != NULL);\n    MA_ASSERT(pInfo != NULL);\n\n    /* ID. */\n    result = ma_context_get_device_id_from_MMDevice__wasapi(pContext, pMMDevice, &pInfo->id);\n    if (result == MA_SUCCESS) {\n        if (pDefaultDeviceID != NULL) {\n            if (ma_strcmp_WCHAR(pInfo->id.wasapi, pDefaultDeviceID) == 0) {\n                pInfo->isDefault = MA_TRUE;\n            }\n        }\n    }\n\n    /* Description / Friendly Name */\n    {\n        ma_IPropertyStore *pProperties;\n        hr = ma_IMMDevice_OpenPropertyStore(pMMDevice, STGM_READ, &pProperties);\n        if (SUCCEEDED(hr)) {\n            MA_PROPVARIANT var;\n\n            ma_PropVariantInit(&var);\n            hr = ma_IPropertyStore_GetValue(pProperties, &MA_PKEY_Device_FriendlyName, &var);\n            if (SUCCEEDED(hr)) {\n                WideCharToMultiByte(CP_UTF8, 0, var.pwszVal, -1, pInfo->name, sizeof(pInfo->name), 0, FALSE);\n                ma_PropVariantClear(pContext, &var);\n            }\n\n            ma_IPropertyStore_Release(pProperties);\n        }\n    }\n\n    /* Format */\n    if (!onlySimpleInfo) {\n        ma_IAudioClient* pAudioClient;\n        hr = ma_IMMDevice_Activate(pMMDevice, &MA_IID_IAudioClient, CLSCTX_ALL, NULL, (void**)&pAudioClient);\n        if (SUCCEEDED(hr)) {\n            result = ma_context_get_device_info_from_IAudioClient__wasapi(pContext, pMMDevice, pAudioClient, pInfo);\n\n            ma_IAudioClient_Release(pAudioClient);\n            return result;\n        } else {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to activate audio client for device info retrieval.\");\n            return ma_result_from_HRESULT(hr);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_enumerate_devices_by_type__wasapi(ma_context* pContext, ma_IMMDeviceEnumerator* pDeviceEnumerator, ma_device_type deviceType, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_result result = MA_SUCCESS;\n    UINT deviceCount;\n    HRESULT hr;\n    ma_uint32 iDevice;\n    WCHAR* pDefaultDeviceID = NULL;\n    ma_IMMDeviceCollection* pDeviceCollection = NULL;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Grab the default device. We use this to know whether or not flag the returned device info as being the default. */\n    pDefaultDeviceID = ma_context_get_default_device_id_from_IMMDeviceEnumerator__wasapi(pContext, pDeviceEnumerator, deviceType);\n\n    /* We need to enumerate the devices which returns a device collection. */\n    hr = ma_IMMDeviceEnumerator_EnumAudioEndpoints(pDeviceEnumerator, ma_device_type_to_EDataFlow(deviceType), MA_MM_DEVICE_STATE_ACTIVE, &pDeviceCollection);\n    if (SUCCEEDED(hr)) {\n        hr = ma_IMMDeviceCollection_GetCount(pDeviceCollection, &deviceCount);\n        if (FAILED(hr)) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to get device count.\\n\");\n            result = ma_result_from_HRESULT(hr);\n            goto done;\n        }\n\n        for (iDevice = 0; iDevice < deviceCount; ++iDevice) {\n            ma_device_info deviceInfo;\n            ma_IMMDevice* pMMDevice;\n\n            MA_ZERO_OBJECT(&deviceInfo);\n\n            hr = ma_IMMDeviceCollection_Item(pDeviceCollection, iDevice, &pMMDevice);\n            if (SUCCEEDED(hr)) {\n                result = ma_context_get_device_info_from_MMDevice__wasapi(pContext, pMMDevice, pDefaultDeviceID, MA_TRUE, &deviceInfo);   /* MA_TRUE = onlySimpleInfo. */\n\n                ma_IMMDevice_Release(pMMDevice);\n                if (result == MA_SUCCESS) {\n                    ma_bool32 cbResult = callback(pContext, deviceType, &deviceInfo, pUserData);\n                    if (cbResult == MA_FALSE) {\n                        break;\n                    }\n                }\n            }\n        }\n    }\n\ndone:\n    if (pDefaultDeviceID != NULL) {\n        ma_CoTaskMemFree(pContext, pDefaultDeviceID);\n        pDefaultDeviceID = NULL;\n    }\n\n    if (pDeviceCollection != NULL) {\n        ma_IMMDeviceCollection_Release(pDeviceCollection);\n        pDeviceCollection = NULL;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_context_get_IAudioClient_Desktop__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, MA_PROPVARIANT* pActivationParams, ma_IAudioClient** ppAudioClient, ma_IMMDevice** ppMMDevice)\n{\n    ma_result result;\n    HRESULT hr;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppAudioClient != NULL);\n    MA_ASSERT(ppMMDevice != NULL);\n\n    result = ma_context_get_MMDevice__wasapi(pContext, deviceType, pDeviceID, ppMMDevice);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    hr = ma_IMMDevice_Activate(*ppMMDevice, &MA_IID_IAudioClient, CLSCTX_ALL, pActivationParams, (void**)ppAudioClient);\n    if (FAILED(hr)) {\n        return ma_result_from_HRESULT(hr);\n    }\n\n    return MA_SUCCESS;\n}\n#else\nstatic ma_result ma_context_get_IAudioClient_UWP__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, MA_PROPVARIANT* pActivationParams, ma_IAudioClient** ppAudioClient, ma_IUnknown** ppActivatedInterface)\n{\n    ma_IActivateAudioInterfaceAsyncOperation *pAsyncOp = NULL;\n    ma_completion_handler_uwp completionHandler;\n    IID iid;\n    WCHAR* iidStr;\n    HRESULT hr;\n    ma_result result;\n    HRESULT activateResult;\n    ma_IUnknown* pActivatedInterface;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppAudioClient != NULL);\n\n    if (pDeviceID != NULL) {\n        iidStr = (WCHAR*)pDeviceID->wasapi;\n    } else {\n        if (deviceType == ma_device_type_capture) {\n            iid = MA_IID_DEVINTERFACE_AUDIO_CAPTURE;\n        } else {\n            iid = MA_IID_DEVINTERFACE_AUDIO_RENDER;\n        }\n\n    #if defined(__cplusplus)\n        hr = StringFromIID(iid, &iidStr);\n    #else\n        hr = StringFromIID(&iid, &iidStr);\n    #endif\n        if (FAILED(hr)) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to convert device IID to string for ActivateAudioInterfaceAsync(). Out of memory.\\n\");\n            return ma_result_from_HRESULT(hr);\n        }\n    }\n\n    result = ma_completion_handler_uwp_init(&completionHandler);\n    if (result != MA_SUCCESS) {\n        ma_CoTaskMemFree(pContext, iidStr);\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create event for waiting for ActivateAudioInterfaceAsync().\\n\");\n        return result;\n    }\n\n    hr = ((MA_PFN_ActivateAudioInterfaceAsync)pContext->wasapi.ActivateAudioInterfaceAsync)(iidStr, &MA_IID_IAudioClient, pActivationParams, (ma_IActivateAudioInterfaceCompletionHandler*)&completionHandler, (ma_IActivateAudioInterfaceAsyncOperation**)&pAsyncOp);\n    if (FAILED(hr)) {\n        ma_completion_handler_uwp_uninit(&completionHandler);\n        ma_CoTaskMemFree(pContext, iidStr);\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] ActivateAudioInterfaceAsync() failed.\\n\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    if (pDeviceID == NULL) {\n        ma_CoTaskMemFree(pContext, iidStr);\n    }\n\n    /* Wait for the async operation for finish. */\n    ma_completion_handler_uwp_wait(&completionHandler);\n    ma_completion_handler_uwp_uninit(&completionHandler);\n\n    hr = ma_IActivateAudioInterfaceAsyncOperation_GetActivateResult(pAsyncOp, &activateResult, &pActivatedInterface);\n    ma_IActivateAudioInterfaceAsyncOperation_Release(pAsyncOp);\n\n    if (FAILED(hr) || FAILED(activateResult)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to activate device.\\n\");\n        return FAILED(hr) ? ma_result_from_HRESULT(hr) : ma_result_from_HRESULT(activateResult);\n    }\n\n    /* Here is where we grab the IAudioClient interface. */\n    hr = ma_IUnknown_QueryInterface(pActivatedInterface, &MA_IID_IAudioClient, (void**)ppAudioClient);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to query IAudioClient interface.\\n\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    if (ppActivatedInterface) {\n        *ppActivatedInterface = pActivatedInterface;\n    } else {\n        ma_IUnknown_Release(pActivatedInterface);\n    }\n\n    return MA_SUCCESS;\n}\n#endif\n\n\n/* https://docs.microsoft.com/en-us/windows/win32/api/audioclientactivationparams/ne-audioclientactivationparams-audioclient_activation_type */\ntypedef enum\n{\n    MA_AUDIOCLIENT_ACTIVATION_TYPE_DEFAULT,\n    MA_AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK\n} MA_AUDIOCLIENT_ACTIVATION_TYPE;\n\n/* https://docs.microsoft.com/en-us/windows/win32/api/audioclientactivationparams/ne-audioclientactivationparams-process_loopback_mode */\ntypedef enum\n{\n    MA_PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE,\n    MA_PROCESS_LOOPBACK_MODE_EXCLUDE_TARGET_PROCESS_TREE\n} MA_PROCESS_LOOPBACK_MODE;\n\n/* https://docs.microsoft.com/en-us/windows/win32/api/audioclientactivationparams/ns-audioclientactivationparams-audioclient_process_loopback_params */\ntypedef struct\n{\n    DWORD TargetProcessId;\n    MA_PROCESS_LOOPBACK_MODE ProcessLoopbackMode;\n} MA_AUDIOCLIENT_PROCESS_LOOPBACK_PARAMS;\n\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(push)\n    #pragma warning(disable:4201)   /* nonstandard extension used: nameless struct/union */\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wpedantic\" /* For ISO C99 doesn't support unnamed structs/unions [-Wpedantic] */\n    #if defined(__clang__)\n        #pragma GCC diagnostic ignored \"-Wc11-extensions\"   /* anonymous unions are a C11 extension */\n    #endif\n#endif\n/* https://docs.microsoft.com/en-us/windows/win32/api/audioclientactivationparams/ns-audioclientactivationparams-audioclient_activation_params */\ntypedef struct\n{\n    MA_AUDIOCLIENT_ACTIVATION_TYPE ActivationType;\n    union\n    {\n        MA_AUDIOCLIENT_PROCESS_LOOPBACK_PARAMS ProcessLoopbackParams;\n    };\n} MA_AUDIOCLIENT_ACTIVATION_PARAMS;\n#if defined(_MSC_VER) && !defined(__clang__)\n    #pragma warning(pop)\n#elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))\n    #pragma GCC diagnostic pop\n#endif\n\n#define MA_VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK L\"VAD\\\\Process_Loopback\"\n\nstatic ma_result ma_context_get_IAudioClient__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_uint32 loopbackProcessID, ma_bool32 loopbackProcessExclude, ma_IAudioClient** ppAudioClient, ma_WASAPIDeviceInterface** ppDeviceInterface)\n{\n    ma_result result;\n    ma_bool32 usingProcessLoopback = MA_FALSE;\n    MA_AUDIOCLIENT_ACTIVATION_PARAMS audioclientActivationParams;\n    MA_PROPVARIANT activationParams;\n    MA_PROPVARIANT* pActivationParams = NULL;\n    ma_device_id virtualDeviceID;\n\n    /* Activation parameters specific to loopback mode. Note that process-specific loopback will only work when a default device ID is specified. */\n    if (deviceType == ma_device_type_loopback && loopbackProcessID != 0 && pDeviceID == NULL) {\n        usingProcessLoopback = MA_TRUE;\n    }\n\n    if (usingProcessLoopback) {\n        MA_ZERO_OBJECT(&audioclientActivationParams);\n        audioclientActivationParams.ActivationType                            = MA_AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK;\n        audioclientActivationParams.ProcessLoopbackParams.ProcessLoopbackMode = (loopbackProcessExclude) ? MA_PROCESS_LOOPBACK_MODE_EXCLUDE_TARGET_PROCESS_TREE : MA_PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE;\n        audioclientActivationParams.ProcessLoopbackParams.TargetProcessId     = (DWORD)loopbackProcessID;\n\n        ma_PropVariantInit(&activationParams);\n        activationParams.vt             = MA_VT_BLOB;\n        activationParams.blob.cbSize    = sizeof(audioclientActivationParams);\n        activationParams.blob.pBlobData = (BYTE*)&audioclientActivationParams;\n        pActivationParams = &activationParams;\n\n        /* When requesting a specific device ID we need to use a special device ID. */\n        MA_COPY_MEMORY(virtualDeviceID.wasapi, MA_VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, (ma_wcslen(MA_VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK) + 1) * sizeof(wchar_t)); /* +1 for the null terminator. */\n        pDeviceID = &virtualDeviceID;\n    } else {\n        pActivationParams = NULL;   /* No activation parameters required. */\n    }\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    result = ma_context_get_IAudioClient_Desktop__wasapi(pContext, deviceType, pDeviceID, pActivationParams, ppAudioClient, ppDeviceInterface);\n#else\n    result = ma_context_get_IAudioClient_UWP__wasapi(pContext, deviceType, pDeviceID, pActivationParams, ppAudioClient, ppDeviceInterface);\n#endif\n\n    /*\n    If loopback mode was requested with a process ID and initialization failed, it could be because it's\n    trying to run on an older version of Windows where it's not supported. We need to let the caller\n    know about this with a log message.\n    */\n    if (result != MA_SUCCESS) {\n        if (usingProcessLoopback) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Loopback mode requested to %s process ID %u, but initialization failed. Support for this feature begins with Windows 10 Build 20348. Confirm your version of Windows or consider not using process-specific loopback.\\n\", (loopbackProcessExclude) ? \"exclude\" : \"include\", loopbackProcessID);\n        }\n    }\n\n    return result;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__wasapi(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    /* Different enumeration for desktop and UWP. */\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    /* Desktop */\n    HRESULT hr;\n    ma_IMMDeviceEnumerator* pDeviceEnumerator;\n\n    hr = ma_CoCreateInstance(pContext, &MA_CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, &MA_IID_IMMDeviceEnumerator, (void**)&pDeviceEnumerator);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create device enumerator.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    ma_context_enumerate_devices_by_type__wasapi(pContext, pDeviceEnumerator, ma_device_type_playback, callback, pUserData);\n    ma_context_enumerate_devices_by_type__wasapi(pContext, pDeviceEnumerator, ma_device_type_capture,  callback, pUserData);\n\n    ma_IMMDeviceEnumerator_Release(pDeviceEnumerator);\n#else\n    /*\n    UWP\n\n    The MMDevice API is only supported on desktop applications. For now, while I'm still figuring out how to properly enumerate\n    over devices without using MMDevice, I'm restricting devices to defaults.\n\n    Hint: DeviceInformation::FindAllAsync() with DeviceClass.AudioCapture/AudioRender. https://blogs.windows.com/buildingapps/2014/05/15/real-time-audio-in-windows-store-and-windows-phone-apps/\n    */\n    if (callback) {\n        ma_bool32 cbResult = MA_TRUE;\n\n        /* Playback. */\n        if (cbResult) {\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n            deviceInfo.isDefault = MA_TRUE;\n            cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n        }\n\n        /* Capture. */\n        if (cbResult) {\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n            deviceInfo.isDefault = MA_TRUE;\n            cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n        }\n    }\n#endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    ma_result result;\n    ma_IMMDevice* pMMDevice = NULL;\n    WCHAR* pDefaultDeviceID = NULL;\n\n    result = ma_context_get_MMDevice__wasapi(pContext, deviceType, pDeviceID, &pMMDevice);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need the default device ID so we can set the isDefault flag in the device info. */\n    pDefaultDeviceID = ma_context_get_default_device_id__wasapi(pContext, deviceType);\n\n    result = ma_context_get_device_info_from_MMDevice__wasapi(pContext, pMMDevice, pDefaultDeviceID, MA_FALSE, pDeviceInfo);   /* MA_FALSE = !onlySimpleInfo. */\n\n    if (pDefaultDeviceID != NULL) {\n        ma_CoTaskMemFree(pContext, pDefaultDeviceID);\n        pDefaultDeviceID = NULL;\n    }\n\n    ma_IMMDevice_Release(pMMDevice);\n\n    return result;\n#else\n    ma_IAudioClient* pAudioClient;\n    ma_result result;\n\n    /* UWP currently only uses default devices. */\n    if (deviceType == ma_device_type_playback) {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n    } else {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n    }\n\n    result = ma_context_get_IAudioClient_UWP__wasapi(pContext, deviceType, pDeviceID, NULL, &pAudioClient, NULL);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_context_get_device_info_from_IAudioClient__wasapi(pContext, NULL, pAudioClient, pDeviceInfo);\n\n    pDeviceInfo->isDefault = MA_TRUE;  /* UWP only supports default devices. */\n\n    ma_IAudioClient_Release(pAudioClient);\n    return result;\n#endif\n}\n\nstatic ma_result ma_device_uninit__wasapi(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    {\n        if (pDevice->wasapi.pDeviceEnumerator) {\n            ((ma_IMMDeviceEnumerator*)pDevice->wasapi.pDeviceEnumerator)->lpVtbl->UnregisterEndpointNotificationCallback((ma_IMMDeviceEnumerator*)pDevice->wasapi.pDeviceEnumerator, &pDevice->wasapi.notificationClient);\n            ma_IMMDeviceEnumerator_Release((ma_IMMDeviceEnumerator*)pDevice->wasapi.pDeviceEnumerator);\n        }\n\n        ma_mutex_uninit(&pDevice->wasapi.rerouteLock);\n    }\n    #endif\n\n    if (pDevice->wasapi.pRenderClient) {\n        if (pDevice->wasapi.pMappedBufferPlayback != NULL) {\n            ma_IAudioRenderClient_ReleaseBuffer((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient, pDevice->wasapi.mappedBufferPlaybackCap, 0);\n            pDevice->wasapi.pMappedBufferPlayback   = NULL;\n            pDevice->wasapi.mappedBufferPlaybackCap = 0;\n            pDevice->wasapi.mappedBufferPlaybackLen = 0;\n        }\n\n        ma_IAudioRenderClient_Release((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient);\n    }\n    if (pDevice->wasapi.pCaptureClient) {\n        if (pDevice->wasapi.pMappedBufferCapture != NULL) {\n            ma_IAudioCaptureClient_ReleaseBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, pDevice->wasapi.mappedBufferCaptureCap);\n            pDevice->wasapi.pMappedBufferCapture   = NULL;\n            pDevice->wasapi.mappedBufferCaptureCap = 0;\n            pDevice->wasapi.mappedBufferCaptureLen = 0;\n        }\n\n        ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient);\n    }\n\n    if (pDevice->wasapi.pAudioClientPlayback) {\n        ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback);\n    }\n    if (pDevice->wasapi.pAudioClientCapture) {\n        ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n    }\n\n    if (pDevice->wasapi.hEventPlayback) {\n        CloseHandle((HANDLE)pDevice->wasapi.hEventPlayback);\n    }\n    if (pDevice->wasapi.hEventCapture) {\n        CloseHandle((HANDLE)pDevice->wasapi.hEventCapture);\n    }\n\n    return MA_SUCCESS;\n}\n\n\ntypedef struct\n{\n    /* Input. */\n    ma_format formatIn;\n    ma_uint32 channelsIn;\n    ma_uint32 sampleRateIn;\n    ma_channel channelMapIn[MA_MAX_CHANNELS];\n    ma_uint32 periodSizeInFramesIn;\n    ma_uint32 periodSizeInMillisecondsIn;\n    ma_uint32 periodsIn;\n    ma_share_mode shareMode;\n    ma_performance_profile performanceProfile;\n    ma_bool32 noAutoConvertSRC;\n    ma_bool32 noDefaultQualitySRC;\n    ma_bool32 noHardwareOffloading;\n    ma_uint32 loopbackProcessID;\n    ma_bool32 loopbackProcessExclude;\n\n    /* Output. */\n    ma_IAudioClient* pAudioClient;\n    ma_IAudioRenderClient* pRenderClient;\n    ma_IAudioCaptureClient* pCaptureClient;\n    ma_format formatOut;\n    ma_uint32 channelsOut;\n    ma_uint32 sampleRateOut;\n    ma_channel channelMapOut[MA_MAX_CHANNELS];\n    ma_uint32 periodSizeInFramesOut;\n    ma_uint32 periodsOut;\n    ma_bool32 usingAudioClient3;\n    char deviceName[256];\n    ma_device_id id;\n} ma_device_init_internal_data__wasapi;\n\nstatic ma_result ma_device_init_internal__wasapi(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_init_internal_data__wasapi* pData)\n{\n    HRESULT hr;\n    ma_result result = MA_SUCCESS;\n    const char* errorMsg = \"\";\n    MA_AUDCLNT_SHAREMODE shareMode = MA_AUDCLNT_SHAREMODE_SHARED;\n    DWORD streamFlags = 0;\n    MA_REFERENCE_TIME periodDurationInMicroseconds;\n    ma_bool32 wasInitializedUsingIAudioClient3 = MA_FALSE;\n    MA_WAVEFORMATEXTENSIBLE wf;\n    ma_WASAPIDeviceInterface* pDeviceInterface = NULL;\n    ma_IAudioClient2* pAudioClient2;\n    ma_uint32 nativeSampleRate;\n    ma_bool32 usingProcessLoopback = MA_FALSE;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pData != NULL);\n\n    /* This function is only used to initialize one device type: either playback, capture or loopback. Never full-duplex. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    usingProcessLoopback = deviceType == ma_device_type_loopback && pData->loopbackProcessID != 0 && pDeviceID == NULL;\n\n    pData->pAudioClient = NULL;\n    pData->pRenderClient = NULL;\n    pData->pCaptureClient = NULL;\n\n    streamFlags = MA_AUDCLNT_STREAMFLAGS_EVENTCALLBACK;\n    if (!pData->noAutoConvertSRC && pData->sampleRateIn != 0 && pData->shareMode != ma_share_mode_exclusive) {    /* <-- Exclusive streams must use the native sample rate. */\n        streamFlags |= MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM;\n    }\n    if (!pData->noDefaultQualitySRC && pData->sampleRateIn != 0 && (streamFlags & MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM) != 0) {\n        streamFlags |= MA_AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY;\n    }\n    if (deviceType == ma_device_type_loopback) {\n        streamFlags |= MA_AUDCLNT_STREAMFLAGS_LOOPBACK;\n    }\n\n    result = ma_context_get_IAudioClient__wasapi(pContext, deviceType, pDeviceID, pData->loopbackProcessID, pData->loopbackProcessExclude, &pData->pAudioClient, &pDeviceInterface);\n    if (result != MA_SUCCESS) {\n        goto done;\n    }\n\n    MA_ZERO_OBJECT(&wf);\n\n    /* Try enabling hardware offloading. */\n    if (!pData->noHardwareOffloading) {\n        hr = ma_IAudioClient_QueryInterface(pData->pAudioClient, &MA_IID_IAudioClient2, (void**)&pAudioClient2);\n        if (SUCCEEDED(hr)) {\n            BOOL isHardwareOffloadingSupported = 0;\n            hr = ma_IAudioClient2_IsOffloadCapable(pAudioClient2, MA_AudioCategory_Other, &isHardwareOffloadingSupported);\n            if (SUCCEEDED(hr) && isHardwareOffloadingSupported) {\n                ma_AudioClientProperties clientProperties;\n                MA_ZERO_OBJECT(&clientProperties);\n                clientProperties.cbSize = sizeof(clientProperties);\n                clientProperties.bIsOffload = 1;\n                clientProperties.eCategory = MA_AudioCategory_Other;\n                ma_IAudioClient2_SetClientProperties(pAudioClient2, &clientProperties);\n            }\n\n            pAudioClient2->lpVtbl->Release(pAudioClient2);\n        }\n    }\n\n    /* Here is where we try to determine the best format to use with the device. If the client if wanting exclusive mode, first try finding the best format for that. If this fails, fall back to shared mode. */\n    result = MA_FORMAT_NOT_SUPPORTED;\n    if (pData->shareMode == ma_share_mode_exclusive) {\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n        /* In exclusive mode on desktop we always use the backend's native format. */\n        ma_IPropertyStore* pStore = NULL;\n        hr = ma_IMMDevice_OpenPropertyStore(pDeviceInterface, STGM_READ, &pStore);\n        if (SUCCEEDED(hr)) {\n            MA_PROPVARIANT prop;\n            ma_PropVariantInit(&prop);\n            hr = ma_IPropertyStore_GetValue(pStore, &MA_PKEY_AudioEngine_DeviceFormat, &prop);\n            if (SUCCEEDED(hr)) {\n                MA_WAVEFORMATEX* pActualFormat = (MA_WAVEFORMATEX*)prop.blob.pBlobData;\n                hr = ma_IAudioClient_IsFormatSupported((ma_IAudioClient*)pData->pAudioClient, MA_AUDCLNT_SHAREMODE_EXCLUSIVE, pActualFormat, NULL);\n                if (SUCCEEDED(hr)) {\n                    MA_COPY_MEMORY(&wf, pActualFormat, sizeof(MA_WAVEFORMATEXTENSIBLE));\n                }\n\n                ma_PropVariantClear(pContext, &prop);\n            }\n\n            ma_IPropertyStore_Release(pStore);\n        }\n    #else\n        /*\n        I do not know how to query the device's native format on UWP so for now I'm just disabling support for\n        exclusive mode. The alternative is to enumerate over different formats and check IsFormatSupported()\n        until you find one that works.\n\n        TODO: Add support for exclusive mode to UWP.\n        */\n        hr = S_FALSE;\n    #endif\n\n        if (hr == S_OK) {\n            shareMode = MA_AUDCLNT_SHAREMODE_EXCLUSIVE;\n            result = MA_SUCCESS;\n        } else {\n            result = MA_SHARE_MODE_NOT_SUPPORTED;\n        }\n    } else {\n        /* In shared mode we are always using the format reported by the operating system. */\n        MA_WAVEFORMATEXTENSIBLE* pNativeFormat = NULL;\n        hr = ma_IAudioClient_GetMixFormat((ma_IAudioClient*)pData->pAudioClient, (MA_WAVEFORMATEX**)&pNativeFormat);\n        if (hr != S_OK) {\n            /* When using process-specific loopback, GetMixFormat() seems to always fail. */\n            if (usingProcessLoopback) {\n                wf.wFormatTag      = WAVE_FORMAT_IEEE_FLOAT;\n                wf.nChannels       = 2;\n                wf.nSamplesPerSec  = 44100;\n                wf.wBitsPerSample  = 32;\n                wf.nBlockAlign     = wf.nChannels * wf.wBitsPerSample / 8;\n                wf.nAvgBytesPerSec = wf.nSamplesPerSec * wf.nBlockAlign;\n                wf.cbSize          = sizeof(MA_WAVEFORMATEX);\n\n                result = MA_SUCCESS;\n            } else {\n                result = MA_FORMAT_NOT_SUPPORTED;\n            }\n        } else {\n            /*\n            I've seen cases where cbSize will be set to sizeof(WAVEFORMATEX) even though the structure itself\n            is given the format tag of WAVE_FORMAT_EXTENSIBLE. If the format tag is WAVE_FORMAT_EXTENSIBLE\n            want to make sure we copy the whole WAVEFORMATEXTENSIBLE structure. Otherwise we'll have to be\n            safe and only copy the WAVEFORMATEX part.\n            */\n            if (pNativeFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {\n                MA_COPY_MEMORY(&wf, pNativeFormat, sizeof(MA_WAVEFORMATEXTENSIBLE));\n            } else {\n                /* I've seen a case where cbSize was set to 0. Assume sizeof(WAVEFORMATEX) in this case. */\n                size_t cbSize = pNativeFormat->cbSize;\n                if (cbSize == 0) {\n                    cbSize = sizeof(MA_WAVEFORMATEX);\n                }\n\n                /* Make sure we don't copy more than the capacity of `wf`. */\n                if (cbSize > sizeof(wf)) {\n                    cbSize = sizeof(wf);\n                }\n\n                MA_COPY_MEMORY(&wf, pNativeFormat, cbSize);\n            }\n\n            result = MA_SUCCESS;\n        }\n\n        ma_CoTaskMemFree(pContext, pNativeFormat);\n\n        shareMode = MA_AUDCLNT_SHAREMODE_SHARED;\n    }\n\n    /* Return an error if we still haven't found a format. */\n    if (result != MA_SUCCESS) {\n        errorMsg = \"[WASAPI] Failed to find best device mix format.\";\n        goto done;\n    }\n\n    /*\n    Override the native sample rate with the one requested by the caller, but only if we're not using the default sample rate. We'll use\n    WASAPI to perform the sample rate conversion.\n    */\n    nativeSampleRate = wf.nSamplesPerSec;\n    if (streamFlags & MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM) {\n        wf.nSamplesPerSec = (pData->sampleRateIn != 0) ? pData->sampleRateIn : MA_DEFAULT_SAMPLE_RATE;\n        wf.nAvgBytesPerSec = wf.nSamplesPerSec * wf.nBlockAlign;\n    }\n\n    pData->formatOut = ma_format_from_WAVEFORMATEX((MA_WAVEFORMATEX*)&wf);\n    if (pData->formatOut == ma_format_unknown) {\n        /*\n        The format isn't supported. This is almost certainly because the exclusive mode format isn't supported by miniaudio. We need to return MA_SHARE_MODE_NOT_SUPPORTED\n        in this case so that the caller can detect it and fall back to shared mode if desired. We should never get here if shared mode was requested, but just for\n        completeness we'll check for it and return MA_FORMAT_NOT_SUPPORTED.\n        */\n        if (shareMode == MA_AUDCLNT_SHAREMODE_EXCLUSIVE) {\n            result = MA_SHARE_MODE_NOT_SUPPORTED;\n        } else {\n            result = MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        errorMsg = \"[WASAPI] Native format not supported.\";\n        goto done;\n    }\n\n    pData->channelsOut = wf.nChannels;\n    pData->sampleRateOut = wf.nSamplesPerSec;\n\n    /*\n    Get the internal channel map based on the channel mask. There is a possibility that GetMixFormat() returns\n    a WAVEFORMATEX instead of a WAVEFORMATEXTENSIBLE, in which case the channel mask will be undefined. In this\n    case we'll just use the default channel map.\n    */\n    if (wf.wFormatTag == WAVE_FORMAT_EXTENSIBLE || wf.cbSize >= sizeof(MA_WAVEFORMATEXTENSIBLE)) {\n        ma_channel_mask_to_channel_map__win32(wf.dwChannelMask, pData->channelsOut, pData->channelMapOut);\n    } else {\n        ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pData->channelMapOut, ma_countof(pData->channelMapOut), pData->channelsOut);\n    }\n\n    /* Period size. */\n    pData->periodsOut = (pData->periodsIn != 0) ? pData->periodsIn : MA_DEFAULT_PERIODS;\n    pData->periodSizeInFramesOut = pData->periodSizeInFramesIn;\n    if (pData->periodSizeInFramesOut == 0) {\n        if (pData->periodSizeInMillisecondsIn == 0) {\n            if (pData->performanceProfile == ma_performance_profile_low_latency) {\n                pData->periodSizeInFramesOut = ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY, wf.nSamplesPerSec);\n            } else {\n                pData->periodSizeInFramesOut = ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE, wf.nSamplesPerSec);\n            }\n        } else {\n            pData->periodSizeInFramesOut = ma_calculate_buffer_size_in_frames_from_milliseconds(pData->periodSizeInMillisecondsIn, wf.nSamplesPerSec);\n        }\n    }\n\n    periodDurationInMicroseconds = ((ma_uint64)pData->periodSizeInFramesOut * 1000 * 1000) / wf.nSamplesPerSec;\n\n\n    /* Slightly different initialization for shared and exclusive modes. We try exclusive mode first, and if it fails, fall back to shared mode. */\n    if (shareMode == MA_AUDCLNT_SHAREMODE_EXCLUSIVE) {\n        MA_REFERENCE_TIME bufferDuration = periodDurationInMicroseconds * pData->periodsOut * 10;\n\n        /*\n        If the periodicity is too small, Initialize() will fail with AUDCLNT_E_INVALID_DEVICE_PERIOD. In this case we should just keep increasing\n        it and trying it again.\n        */\n        hr = E_FAIL;\n        for (;;) {\n            hr = ma_IAudioClient_Initialize((ma_IAudioClient*)pData->pAudioClient, shareMode, streamFlags, bufferDuration, bufferDuration, (MA_WAVEFORMATEX*)&wf, NULL);\n            if (hr == MA_AUDCLNT_E_INVALID_DEVICE_PERIOD) {\n                if (bufferDuration > 500*10000) {\n                    break;\n                } else {\n                    if (bufferDuration == 0) {  /* <-- Just a sanity check to prevent an infinite loop. Should never happen, but it makes me feel better. */\n                        break;\n                    }\n\n                    bufferDuration = bufferDuration * 2;\n                    continue;\n                }\n            } else {\n                break;\n            }\n        }\n\n        if (hr == MA_AUDCLNT_E_BUFFER_SIZE_NOT_ALIGNED) {\n            ma_uint32 bufferSizeInFrames;\n            hr = ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pData->pAudioClient, &bufferSizeInFrames);\n            if (SUCCEEDED(hr)) {\n                bufferDuration = (MA_REFERENCE_TIME)((10000.0 * 1000 / wf.nSamplesPerSec * bufferSizeInFrames) + 0.5);\n\n                /* Unfortunately we need to release and re-acquire the audio client according to MSDN. Seems silly - why not just call IAudioClient_Initialize() again?! */\n                ma_IAudioClient_Release((ma_IAudioClient*)pData->pAudioClient);\n\n            #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n                hr = ma_IMMDevice_Activate(pDeviceInterface, &MA_IID_IAudioClient, CLSCTX_ALL, NULL, (void**)&pData->pAudioClient);\n            #else\n                hr = ma_IUnknown_QueryInterface(pDeviceInterface, &MA_IID_IAudioClient, (void**)&pData->pAudioClient);\n            #endif\n\n                if (SUCCEEDED(hr)) {\n                    hr = ma_IAudioClient_Initialize((ma_IAudioClient*)pData->pAudioClient, shareMode, streamFlags, bufferDuration, bufferDuration, (MA_WAVEFORMATEX*)&wf, NULL);\n                }\n            }\n        }\n\n        if (FAILED(hr)) {\n            /* Failed to initialize in exclusive mode. Don't fall back to shared mode - instead tell the client about it. They can reinitialize in shared mode if they want. */\n            if (hr == E_ACCESSDENIED) {\n                errorMsg = \"[WASAPI] Failed to initialize device in exclusive mode. Access denied.\", result = MA_ACCESS_DENIED;\n            } else if (hr == MA_AUDCLNT_E_DEVICE_IN_USE) {\n                errorMsg = \"[WASAPI] Failed to initialize device in exclusive mode. Device in use.\", result = MA_BUSY;\n            } else {\n                errorMsg = \"[WASAPI] Failed to initialize device in exclusive mode.\"; result = ma_result_from_HRESULT(hr);\n            }\n            goto done;\n        }\n    }\n\n    if (shareMode == MA_AUDCLNT_SHAREMODE_SHARED) {\n        /*\n        Low latency shared mode via IAudioClient3.\n\n        NOTE\n        ====\n        Contrary to the documentation on MSDN (https://docs.microsoft.com/en-us/windows/win32/api/audioclient/nf-audioclient-iaudioclient3-initializesharedaudiostream), the\n        use of AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM and AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY with IAudioClient3_InitializeSharedAudioStream() absolutely does not work. Using\n        any of these flags will result in HRESULT code 0x88890021. The other problem is that calling IAudioClient3_GetSharedModeEnginePeriod() with a sample rate different to\n        that returned by IAudioClient_GetMixFormat() also results in an error. I'm therefore disabling low-latency shared mode with AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM.\n        */\n        #ifndef MA_WASAPI_NO_LOW_LATENCY_SHARED_MODE\n        {\n            if ((streamFlags & MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM) == 0 || nativeSampleRate == wf.nSamplesPerSec) {\n                ma_IAudioClient3* pAudioClient3 = NULL;\n                hr = ma_IAudioClient_QueryInterface(pData->pAudioClient, &MA_IID_IAudioClient3, (void**)&pAudioClient3);\n                if (SUCCEEDED(hr)) {\n                    ma_uint32 defaultPeriodInFrames;\n                    ma_uint32 fundamentalPeriodInFrames;\n                    ma_uint32 minPeriodInFrames;\n                    ma_uint32 maxPeriodInFrames;\n                    hr = ma_IAudioClient3_GetSharedModeEnginePeriod(pAudioClient3, (MA_WAVEFORMATEX*)&wf, &defaultPeriodInFrames, &fundamentalPeriodInFrames, &minPeriodInFrames, &maxPeriodInFrames);\n                    if (SUCCEEDED(hr)) {\n                        ma_uint32 desiredPeriodInFrames = pData->periodSizeInFramesOut;\n                        ma_uint32 actualPeriodInFrames  = desiredPeriodInFrames;\n\n                        /* Make sure the period size is a multiple of fundamentalPeriodInFrames. */\n                        actualPeriodInFrames = actualPeriodInFrames / fundamentalPeriodInFrames;\n                        actualPeriodInFrames = actualPeriodInFrames * fundamentalPeriodInFrames;\n\n                        /* The period needs to be clamped between minPeriodInFrames and maxPeriodInFrames. */\n                        actualPeriodInFrames = ma_clamp(actualPeriodInFrames, minPeriodInFrames, maxPeriodInFrames);\n\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Trying IAudioClient3_InitializeSharedAudioStream(actualPeriodInFrames=%d)\\n\", actualPeriodInFrames);\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"    defaultPeriodInFrames=%d\\n\", defaultPeriodInFrames);\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"    fundamentalPeriodInFrames=%d\\n\", fundamentalPeriodInFrames);\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"    minPeriodInFrames=%d\\n\", minPeriodInFrames);\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"    maxPeriodInFrames=%d\\n\", maxPeriodInFrames);\n\n                        /* If the client requested a largish buffer than we don't actually want to use low latency shared mode because it forces small buffers. */\n                        if (actualPeriodInFrames >= desiredPeriodInFrames) {\n                            /*\n                            MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | MA_AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY must not be in the stream flags. If either of these are specified,\n                            IAudioClient3_InitializeSharedAudioStream() will fail.\n                            */\n                            hr = ma_IAudioClient3_InitializeSharedAudioStream(pAudioClient3, streamFlags & ~(MA_AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | MA_AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY), actualPeriodInFrames, (MA_WAVEFORMATEX*)&wf, NULL);\n                            if (SUCCEEDED(hr)) {\n                                wasInitializedUsingIAudioClient3 = MA_TRUE;\n                                pData->periodSizeInFramesOut = actualPeriodInFrames;\n\n                                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Using IAudioClient3\\n\");\n                                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"    periodSizeInFramesOut=%d\\n\", pData->periodSizeInFramesOut);\n                            } else {\n                                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] IAudioClient3_InitializeSharedAudioStream failed. Falling back to IAudioClient.\\n\");\n                            }\n                        } else {\n                            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Not using IAudioClient3 because the desired period size is larger than the maximum supported by IAudioClient3.\\n\");\n                        }\n                    } else {\n                        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] IAudioClient3_GetSharedModeEnginePeriod failed. Falling back to IAudioClient.\\n\");\n                    }\n\n                    ma_IAudioClient3_Release(pAudioClient3);\n                    pAudioClient3 = NULL;\n                }\n            }\n        }\n        #else\n        {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Not using IAudioClient3 because MA_WASAPI_NO_LOW_LATENCY_SHARED_MODE is enabled.\\n\");\n        }\n        #endif\n\n        /* If we don't have an IAudioClient3 then we need to use the normal initialization routine. */\n        if (!wasInitializedUsingIAudioClient3) {\n            MA_REFERENCE_TIME bufferDuration = periodDurationInMicroseconds * pData->periodsOut * 10;   /* <-- Multiply by 10 for microseconds to 100-nanoseconds. */\n            hr = ma_IAudioClient_Initialize((ma_IAudioClient*)pData->pAudioClient, shareMode, streamFlags, bufferDuration, 0, (const MA_WAVEFORMATEX*)&wf, NULL);\n            if (FAILED(hr)) {\n                if (hr == E_ACCESSDENIED) {\n                    errorMsg = \"[WASAPI] Failed to initialize device. Access denied.\", result = MA_ACCESS_DENIED;\n                } else if (hr == MA_AUDCLNT_E_DEVICE_IN_USE) {\n                    errorMsg = \"[WASAPI] Failed to initialize device. Device in use.\", result = MA_BUSY;\n                } else {\n                    errorMsg = \"[WASAPI] Failed to initialize device.\", result = ma_result_from_HRESULT(hr);\n                }\n\n                goto done;\n            }\n        }\n    }\n\n    if (!wasInitializedUsingIAudioClient3) {\n        ma_uint32 bufferSizeInFrames = 0;\n        hr = ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pData->pAudioClient, &bufferSizeInFrames);\n        if (FAILED(hr)) {\n            errorMsg = \"[WASAPI] Failed to get audio client's actual buffer size.\", result = ma_result_from_HRESULT(hr);\n            goto done;\n        }\n\n        /*\n        When using process loopback mode, retrieval of the buffer size seems to result in totally\n        incorrect values. In this case we'll just assume it's the same size as what we requested\n        when we initialized the client.\n        */\n        if (usingProcessLoopback) {\n            bufferSizeInFrames = (ma_uint32)((periodDurationInMicroseconds * pData->periodsOut) * pData->sampleRateOut / 1000000);\n        }\n\n        pData->periodSizeInFramesOut = bufferSizeInFrames / pData->periodsOut;\n    }\n\n    pData->usingAudioClient3 = wasInitializedUsingIAudioClient3;\n\n\n    if (deviceType == ma_device_type_playback) {\n        result = ma_device_create_IAudioClient_service__wasapi(pContext, deviceType, (ma_IAudioClient*)pData->pAudioClient, (void**)&pData->pRenderClient);\n    } else {\n        result = ma_device_create_IAudioClient_service__wasapi(pContext, deviceType, (ma_IAudioClient*)pData->pAudioClient, (void**)&pData->pCaptureClient);\n    }\n\n    /*if (FAILED(hr)) {*/\n    if (result != MA_SUCCESS) {\n        errorMsg = \"[WASAPI] Failed to get audio client service.\";\n        goto done;\n    }\n\n\n    /* Grab the name of the device. */\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    {\n        ma_IPropertyStore *pProperties;\n        hr = ma_IMMDevice_OpenPropertyStore(pDeviceInterface, STGM_READ, &pProperties);\n        if (SUCCEEDED(hr)) {\n            MA_PROPVARIANT varName;\n            ma_PropVariantInit(&varName);\n            hr = ma_IPropertyStore_GetValue(pProperties, &MA_PKEY_Device_FriendlyName, &varName);\n            if (SUCCEEDED(hr)) {\n                WideCharToMultiByte(CP_UTF8, 0, varName.pwszVal, -1, pData->deviceName, sizeof(pData->deviceName), 0, FALSE);\n                ma_PropVariantClear(pContext, &varName);\n            }\n\n            ma_IPropertyStore_Release(pProperties);\n        }\n    }\n    #endif\n\n    /*\n    For the WASAPI backend we need to know the actual IDs of the device in order to do automatic\n    stream routing so that IDs can be compared and we can determine which device has been detached\n    and whether or not it matches with our ma_device.\n    */\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    {\n        /* Desktop */\n        ma_context_get_device_id_from_MMDevice__wasapi(pContext, pDeviceInterface, &pData->id);\n    }\n    #else\n    {\n        /* UWP */\n        /* TODO: Implement me. Need to figure out how to get the ID of the default device. */\n    }\n    #endif\n\ndone:\n    /* Clean up. */\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    if (pDeviceInterface != NULL) {\n        ma_IMMDevice_Release(pDeviceInterface);\n    }\n#else\n    if (pDeviceInterface != NULL) {\n        ma_IUnknown_Release(pDeviceInterface);\n    }\n#endif\n\n    if (result != MA_SUCCESS) {\n        if (pData->pRenderClient) {\n            ma_IAudioRenderClient_Release((ma_IAudioRenderClient*)pData->pRenderClient);\n            pData->pRenderClient = NULL;\n        }\n        if (pData->pCaptureClient) {\n            ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pData->pCaptureClient);\n            pData->pCaptureClient = NULL;\n        }\n        if (pData->pAudioClient) {\n            ma_IAudioClient_Release((ma_IAudioClient*)pData->pAudioClient);\n            pData->pAudioClient = NULL;\n        }\n\n        if (errorMsg != NULL && errorMsg[0] != '\\0') {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"%s\\n\", errorMsg);\n        }\n\n        return result;\n    } else {\n        return MA_SUCCESS;\n    }\n}\n\nstatic ma_result ma_device_reinit__wasapi(ma_device* pDevice, ma_device_type deviceType)\n{\n    ma_device_init_internal_data__wasapi data;\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* We only re-initialize the playback or capture device. Never a full-duplex device. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n\n    /*\n    Before reinitializing the device we need to free the previous audio clients.\n\n    There's a known memory leak here. We will be calling this from the routing change callback that\n    is fired by WASAPI. If we attempt to release the IAudioClient we will deadlock. In my opinion\n    this is a bug. I'm not sure what I need to do to handle this cleanly, but I think we'll probably\n    need some system where we post an event, but delay the execution of it until the callback has\n    returned. I'm not sure how to do this reliably, however. I have set up some infrastructure for\n    a command thread which might be useful for this.\n    */\n    if (deviceType == ma_device_type_capture || deviceType == ma_device_type_loopback) {\n        if (pDevice->wasapi.pCaptureClient) {\n            ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient);\n            pDevice->wasapi.pCaptureClient = NULL;\n        }\n\n        if (pDevice->wasapi.pAudioClientCapture) {\n            /*ma_device_release_IAudioClient_service__wasapi(pDevice, ma_device_type_capture);*/\n            pDevice->wasapi.pAudioClientCapture = NULL;\n        }\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        if (pDevice->wasapi.pRenderClient) {\n            ma_IAudioRenderClient_Release((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient);\n            pDevice->wasapi.pRenderClient = NULL;\n        }\n\n        if (pDevice->wasapi.pAudioClientPlayback) {\n            /*ma_device_release_IAudioClient_service__wasapi(pDevice, ma_device_type_playback);*/\n            pDevice->wasapi.pAudioClientPlayback = NULL;\n        }\n    }\n\n\n    if (deviceType == ma_device_type_playback) {\n        data.formatIn               = pDevice->playback.format;\n        data.channelsIn             = pDevice->playback.channels;\n        MA_COPY_MEMORY(data.channelMapIn, pDevice->playback.channelMap, sizeof(pDevice->playback.channelMap));\n        data.shareMode              = pDevice->playback.shareMode;\n    } else {\n        data.formatIn               = pDevice->capture.format;\n        data.channelsIn             = pDevice->capture.channels;\n        MA_COPY_MEMORY(data.channelMapIn, pDevice->capture.channelMap, sizeof(pDevice->capture.channelMap));\n        data.shareMode              = pDevice->capture.shareMode;\n    }\n\n    data.sampleRateIn               = pDevice->sampleRate;\n    data.periodSizeInFramesIn       = pDevice->wasapi.originalPeriodSizeInFrames;\n    data.periodSizeInMillisecondsIn = pDevice->wasapi.originalPeriodSizeInMilliseconds;\n    data.periodsIn                  = pDevice->wasapi.originalPeriods;\n    data.performanceProfile         = pDevice->wasapi.originalPerformanceProfile;\n    data.noAutoConvertSRC           = pDevice->wasapi.noAutoConvertSRC;\n    data.noDefaultQualitySRC        = pDevice->wasapi.noDefaultQualitySRC;\n    data.noHardwareOffloading       = pDevice->wasapi.noHardwareOffloading;\n    data.loopbackProcessID          = pDevice->wasapi.loopbackProcessID;\n    data.loopbackProcessExclude     = pDevice->wasapi.loopbackProcessExclude;\n    result = ma_device_init_internal__wasapi(pDevice->pContext, deviceType, NULL, &data);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* At this point we have some new objects ready to go. We need to uninitialize the previous ones and then set the new ones. */\n    if (deviceType == ma_device_type_capture || deviceType == ma_device_type_loopback) {\n        pDevice->wasapi.pAudioClientCapture         = data.pAudioClient;\n        pDevice->wasapi.pCaptureClient              = data.pCaptureClient;\n\n        pDevice->capture.internalFormat             = data.formatOut;\n        pDevice->capture.internalChannels           = data.channelsOut;\n        pDevice->capture.internalSampleRate         = data.sampleRateOut;\n        MA_COPY_MEMORY(pDevice->capture.internalChannelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDevice->capture.internalPeriodSizeInFrames = data.periodSizeInFramesOut;\n        pDevice->capture.internalPeriods            = data.periodsOut;\n        ma_strcpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), data.deviceName);\n\n        ma_IAudioClient_SetEventHandle((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture, (HANDLE)pDevice->wasapi.hEventCapture);\n\n        pDevice->wasapi.periodSizeInFramesCapture = data.periodSizeInFramesOut;\n        ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture, &pDevice->wasapi.actualBufferSizeInFramesCapture);\n\n        /* We must always have a valid ID. */\n        ma_strcpy_s_WCHAR(pDevice->capture.id.wasapi, sizeof(pDevice->capture.id.wasapi), data.id.wasapi);\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        pDevice->wasapi.pAudioClientPlayback         = data.pAudioClient;\n        pDevice->wasapi.pRenderClient                = data.pRenderClient;\n\n        pDevice->playback.internalFormat             = data.formatOut;\n        pDevice->playback.internalChannels           = data.channelsOut;\n        pDevice->playback.internalSampleRate         = data.sampleRateOut;\n        MA_COPY_MEMORY(pDevice->playback.internalChannelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDevice->playback.internalPeriodSizeInFrames = data.periodSizeInFramesOut;\n        pDevice->playback.internalPeriods            = data.periodsOut;\n        ma_strcpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), data.deviceName);\n\n        ma_IAudioClient_SetEventHandle((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback, (HANDLE)pDevice->wasapi.hEventPlayback);\n\n        pDevice->wasapi.periodSizeInFramesPlayback = data.periodSizeInFramesOut;\n        ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback, &pDevice->wasapi.actualBufferSizeInFramesPlayback);\n\n        /* We must always have a valid ID because rerouting will look at it. */\n        ma_strcpy_s_WCHAR(pDevice->playback.id.wasapi, sizeof(pDevice->playback.id.wasapi), data.id.wasapi);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__wasapi(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result = MA_SUCCESS;\n\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    HRESULT hr;\n    ma_IMMDeviceEnumerator* pDeviceEnumerator;\n#endif\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->wasapi);\n    pDevice->wasapi.usage                  = pConfig->wasapi.usage;\n    pDevice->wasapi.noAutoConvertSRC       = pConfig->wasapi.noAutoConvertSRC;\n    pDevice->wasapi.noDefaultQualitySRC    = pConfig->wasapi.noDefaultQualitySRC;\n    pDevice->wasapi.noHardwareOffloading   = pConfig->wasapi.noHardwareOffloading;\n    pDevice->wasapi.loopbackProcessID      = pConfig->wasapi.loopbackProcessID;\n    pDevice->wasapi.loopbackProcessExclude = pConfig->wasapi.loopbackProcessExclude;\n\n    /* Exclusive mode is not allowed with loopback. */\n    if (pConfig->deviceType == ma_device_type_loopback && pConfig->playback.shareMode == ma_share_mode_exclusive) {\n        return MA_INVALID_DEVICE_CONFIG;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) {\n        ma_device_init_internal_data__wasapi data;\n        data.formatIn                   = pDescriptorCapture->format;\n        data.channelsIn                 = pDescriptorCapture->channels;\n        data.sampleRateIn               = pDescriptorCapture->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDescriptorCapture->channelMap, sizeof(pDescriptorCapture->channelMap));\n        data.periodSizeInFramesIn       = pDescriptorCapture->periodSizeInFrames;\n        data.periodSizeInMillisecondsIn = pDescriptorCapture->periodSizeInMilliseconds;\n        data.periodsIn                  = pDescriptorCapture->periodCount;\n        data.shareMode                  = pDescriptorCapture->shareMode;\n        data.performanceProfile         = pConfig->performanceProfile;\n        data.noAutoConvertSRC           = pConfig->wasapi.noAutoConvertSRC;\n        data.noDefaultQualitySRC        = pConfig->wasapi.noDefaultQualitySRC;\n        data.noHardwareOffloading       = pConfig->wasapi.noHardwareOffloading;\n        data.loopbackProcessID          = pConfig->wasapi.loopbackProcessID;\n        data.loopbackProcessExclude     = pConfig->wasapi.loopbackProcessExclude;\n\n        result = ma_device_init_internal__wasapi(pDevice->pContext, (pConfig->deviceType == ma_device_type_loopback) ? ma_device_type_loopback : ma_device_type_capture, pDescriptorCapture->pDeviceID, &data);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pDevice->wasapi.pAudioClientCapture              = data.pAudioClient;\n        pDevice->wasapi.pCaptureClient                   = data.pCaptureClient;\n        pDevice->wasapi.originalPeriodSizeInMilliseconds = pDescriptorCapture->periodSizeInMilliseconds;\n        pDevice->wasapi.originalPeriodSizeInFrames       = pDescriptorCapture->periodSizeInFrames;\n        pDevice->wasapi.originalPeriods                  = pDescriptorCapture->periodCount;\n        pDevice->wasapi.originalPerformanceProfile       = pConfig->performanceProfile;\n\n        /*\n        The event for capture needs to be manual reset for the same reason as playback. We keep the initial state set to unsignaled,\n        however, because we want to block until we actually have something for the first call to ma_device_read().\n        */\n        pDevice->wasapi.hEventCapture = (ma_handle)CreateEventA(NULL, FALSE, FALSE, NULL);  /* Auto reset, unsignaled by default. */\n        if (pDevice->wasapi.hEventCapture == NULL) {\n            result = ma_result_from_GetLastError(GetLastError());\n\n            if (pDevice->wasapi.pCaptureClient != NULL) {\n                ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient);\n                pDevice->wasapi.pCaptureClient = NULL;\n            }\n            if (pDevice->wasapi.pAudioClientCapture != NULL) {\n                ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n                pDevice->wasapi.pAudioClientCapture = NULL;\n            }\n\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create event for capture.\");\n            return result;\n        }\n        ma_IAudioClient_SetEventHandle((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture, (HANDLE)pDevice->wasapi.hEventCapture);\n\n        pDevice->wasapi.periodSizeInFramesCapture = data.periodSizeInFramesOut;\n        ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture, &pDevice->wasapi.actualBufferSizeInFramesCapture);\n\n        /* We must always have a valid ID. */\n        ma_strcpy_s_WCHAR(pDevice->capture.id.wasapi, sizeof(pDevice->capture.id.wasapi), data.id.wasapi);\n\n        /* The descriptor needs to be updated with actual values. */\n        pDescriptorCapture->format             = data.formatOut;\n        pDescriptorCapture->channels           = data.channelsOut;\n        pDescriptorCapture->sampleRate         = data.sampleRateOut;\n        MA_COPY_MEMORY(pDescriptorCapture->channelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDescriptorCapture->periodSizeInFrames = data.periodSizeInFramesOut;\n        pDescriptorCapture->periodCount        = data.periodsOut;\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_device_init_internal_data__wasapi data;\n        data.formatIn                   = pDescriptorPlayback->format;\n        data.channelsIn                 = pDescriptorPlayback->channels;\n        data.sampleRateIn               = pDescriptorPlayback->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDescriptorPlayback->channelMap, sizeof(pDescriptorPlayback->channelMap));\n        data.periodSizeInFramesIn       = pDescriptorPlayback->periodSizeInFrames;\n        data.periodSizeInMillisecondsIn = pDescriptorPlayback->periodSizeInMilliseconds;\n        data.periodsIn                  = pDescriptorPlayback->periodCount;\n        data.shareMode                  = pDescriptorPlayback->shareMode;\n        data.performanceProfile         = pConfig->performanceProfile;\n        data.noAutoConvertSRC           = pConfig->wasapi.noAutoConvertSRC;\n        data.noDefaultQualitySRC        = pConfig->wasapi.noDefaultQualitySRC;\n        data.noHardwareOffloading       = pConfig->wasapi.noHardwareOffloading;\n        data.loopbackProcessID          = pConfig->wasapi.loopbackProcessID;\n        data.loopbackProcessExclude     = pConfig->wasapi.loopbackProcessExclude;\n\n        result = ma_device_init_internal__wasapi(pDevice->pContext, ma_device_type_playback, pDescriptorPlayback->pDeviceID, &data);\n        if (result != MA_SUCCESS) {\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                if (pDevice->wasapi.pCaptureClient != NULL) {\n                    ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient);\n                    pDevice->wasapi.pCaptureClient = NULL;\n                }\n                if (pDevice->wasapi.pAudioClientCapture != NULL) {\n                    ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n                    pDevice->wasapi.pAudioClientCapture = NULL;\n                }\n\n                CloseHandle((HANDLE)pDevice->wasapi.hEventCapture);\n                pDevice->wasapi.hEventCapture = NULL;\n            }\n            return result;\n        }\n\n        pDevice->wasapi.pAudioClientPlayback             = data.pAudioClient;\n        pDevice->wasapi.pRenderClient                    = data.pRenderClient;\n        pDevice->wasapi.originalPeriodSizeInMilliseconds = pDescriptorPlayback->periodSizeInMilliseconds;\n        pDevice->wasapi.originalPeriodSizeInFrames       = pDescriptorPlayback->periodSizeInFrames;\n        pDevice->wasapi.originalPeriods                  = pDescriptorPlayback->periodCount;\n        pDevice->wasapi.originalPerformanceProfile       = pConfig->performanceProfile;\n\n        /*\n        The event for playback is needs to be manual reset because we want to explicitly control the fact that it becomes signalled\n        only after the whole available space has been filled, never before.\n\n        The playback event also needs to be initially set to a signaled state so that the first call to ma_device_write() is able\n        to get passed WaitForMultipleObjects().\n        */\n        pDevice->wasapi.hEventPlayback = (ma_handle)CreateEventA(NULL, FALSE, TRUE, NULL);  /* Auto reset, signaled by default. */\n        if (pDevice->wasapi.hEventPlayback == NULL) {\n            result = ma_result_from_GetLastError(GetLastError());\n\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                if (pDevice->wasapi.pCaptureClient != NULL) {\n                    ma_IAudioCaptureClient_Release((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient);\n                    pDevice->wasapi.pCaptureClient = NULL;\n                }\n                if (pDevice->wasapi.pAudioClientCapture != NULL) {\n                    ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n                    pDevice->wasapi.pAudioClientCapture = NULL;\n                }\n\n                CloseHandle((HANDLE)pDevice->wasapi.hEventCapture);\n                pDevice->wasapi.hEventCapture = NULL;\n            }\n\n            if (pDevice->wasapi.pRenderClient != NULL) {\n                ma_IAudioRenderClient_Release((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient);\n                pDevice->wasapi.pRenderClient = NULL;\n            }\n            if (pDevice->wasapi.pAudioClientPlayback != NULL) {\n                ma_IAudioClient_Release((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback);\n                pDevice->wasapi.pAudioClientPlayback = NULL;\n            }\n\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create event for playback.\");\n            return result;\n        }\n        ma_IAudioClient_SetEventHandle((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback, (HANDLE)pDevice->wasapi.hEventPlayback);\n\n        pDevice->wasapi.periodSizeInFramesPlayback = data.periodSizeInFramesOut;\n        ma_IAudioClient_GetBufferSize((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback, &pDevice->wasapi.actualBufferSizeInFramesPlayback);\n\n        /* We must always have a valid ID because rerouting will look at it. */\n        ma_strcpy_s_WCHAR(pDevice->playback.id.wasapi, sizeof(pDevice->playback.id.wasapi), data.id.wasapi);\n\n        /* The descriptor needs to be updated with actual values. */\n        pDescriptorPlayback->format             = data.formatOut;\n        pDescriptorPlayback->channels           = data.channelsOut;\n        pDescriptorPlayback->sampleRate         = data.sampleRateOut;\n        MA_COPY_MEMORY(pDescriptorPlayback->channelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDescriptorPlayback->periodSizeInFrames = data.periodSizeInFramesOut;\n        pDescriptorPlayback->periodCount        = data.periodsOut;\n    }\n\n    /*\n    We need to register a notification client to detect when the device has been disabled, unplugged or re-routed (when the default device changes). When\n    we are connecting to the default device we want to do automatic stream routing when the device is disabled or unplugged. Otherwise we want to just\n    stop the device outright and let the application handle it.\n    */\n#if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    if (pConfig->wasapi.noAutoStreamRouting == MA_FALSE) {\n        if ((pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) && pConfig->capture.pDeviceID == NULL) {\n            pDevice->wasapi.allowCaptureAutoStreamRouting = MA_TRUE;\n        }\n        if ((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pConfig->playback.pDeviceID == NULL) {\n            pDevice->wasapi.allowPlaybackAutoStreamRouting = MA_TRUE;\n        }\n    }\n\n    ma_mutex_init(&pDevice->wasapi.rerouteLock);\n\n    hr = ma_CoCreateInstance(pDevice->pContext, &MA_CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL, &MA_IID_IMMDeviceEnumerator, (void**)&pDeviceEnumerator);\n    if (FAILED(hr)) {\n        ma_device_uninit__wasapi(pDevice);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to create device enumerator.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    pDevice->wasapi.notificationClient.lpVtbl  = (void*)&g_maNotificationCientVtbl;\n    pDevice->wasapi.notificationClient.counter = 1;\n    pDevice->wasapi.notificationClient.pDevice = pDevice;\n\n    hr = pDeviceEnumerator->lpVtbl->RegisterEndpointNotificationCallback(pDeviceEnumerator, &pDevice->wasapi.notificationClient);\n    if (SUCCEEDED(hr)) {\n        pDevice->wasapi.pDeviceEnumerator = (ma_ptr)pDeviceEnumerator;\n    } else {\n        /* Not the end of the world if we fail to register the notification callback. We just won't support automatic stream routing. */\n        ma_IMMDeviceEnumerator_Release(pDeviceEnumerator);\n    }\n#endif\n\n    ma_atomic_bool32_set(&pDevice->wasapi.isStartedCapture,  MA_FALSE);\n    ma_atomic_bool32_set(&pDevice->wasapi.isStartedPlayback, MA_FALSE);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device__get_available_frames__wasapi(ma_device* pDevice, ma_IAudioClient* pAudioClient, ma_uint32* pFrameCount)\n{\n    ma_uint32 paddingFramesCount;\n    HRESULT hr;\n    ma_share_mode shareMode;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pFrameCount != NULL);\n\n    *pFrameCount = 0;\n\n    if ((ma_ptr)pAudioClient != pDevice->wasapi.pAudioClientPlayback && (ma_ptr)pAudioClient != pDevice->wasapi.pAudioClientCapture) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /*\n    I've had a report that GetCurrentPadding() is returning a frame count of 0 which is preventing\n    higher level function calls from doing anything because it thinks nothing is available. I have\n    taken a look at the documentation and it looks like this is unnecessary in exclusive mode.\n\n    From Microsoft's documentation:\n\n        For an exclusive-mode rendering or capture stream that was initialized with the\n        AUDCLNT_STREAMFLAGS_EVENTCALLBACK flag, the client typically has no use for the padding\n        value reported by GetCurrentPadding. Instead, the client accesses an entire buffer during\n        each processing pass.\n\n    Considering this, I'm going to skip GetCurrentPadding() for exclusive mode and just report the\n    entire buffer. This depends on the caller making sure they wait on the event handler.\n    */\n    shareMode = ((ma_ptr)pAudioClient == pDevice->wasapi.pAudioClientPlayback) ? pDevice->playback.shareMode : pDevice->capture.shareMode;\n    if (shareMode == ma_share_mode_shared) {\n        /* Shared mode. */\n        hr = ma_IAudioClient_GetCurrentPadding(pAudioClient, &paddingFramesCount);\n        if (FAILED(hr)) {\n            return ma_result_from_HRESULT(hr);\n        }\n\n        if ((ma_ptr)pAudioClient == pDevice->wasapi.pAudioClientPlayback) {\n            *pFrameCount = pDevice->wasapi.actualBufferSizeInFramesPlayback - paddingFramesCount;\n        } else {\n            *pFrameCount = paddingFramesCount;\n        }\n    } else {\n        /* Exclusive mode. */\n        if ((ma_ptr)pAudioClient == pDevice->wasapi.pAudioClientPlayback) {\n            *pFrameCount = pDevice->wasapi.actualBufferSizeInFramesPlayback;\n        } else {\n            *pFrameCount = pDevice->wasapi.actualBufferSizeInFramesCapture;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_reroute__wasapi(ma_device* pDevice, ma_device_type deviceType)\n{\n    ma_result result;\n\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"=== CHANGING DEVICE ===\\n\");\n\n    result = ma_device_reinit__wasapi(pDevice, deviceType);\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[WASAPI] Reinitializing device after route change failed.\\n\");\n        return result;\n    }\n\n    ma_device__post_init_setup(pDevice, deviceType);\n    ma_device__on_notification_rerouted(pDevice);\n\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"=== DEVICE CHANGED ===\\n\");\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__wasapi_nolock(ma_device* pDevice)\n{\n    HRESULT hr;\n\n    if (pDevice->pContext->wasapi.hAvrt) {\n        const char* pTaskName = ma_to_usage_string__wasapi(pDevice->wasapi.usage);\n        if (pTaskName) {\n            DWORD idx = 0;\n            pDevice->wasapi.hAvrtHandle = (ma_handle)((MA_PFN_AvSetMmThreadCharacteristicsA)pDevice->pContext->wasapi.AvSetMmThreadCharacteristicsA)(pTaskName, &idx);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n        hr = ma_IAudioClient_Start((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n        if (FAILED(hr)) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to start internal capture device. HRESULT = %d.\", (int)hr);\n            return ma_result_from_HRESULT(hr);\n        }\n\n        ma_atomic_bool32_set(&pDevice->wasapi.isStartedCapture, MA_TRUE);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        hr = ma_IAudioClient_Start((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback);\n        if (FAILED(hr)) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to start internal playback device. HRESULT = %d.\", (int)hr);\n            return ma_result_from_HRESULT(hr);\n        }\n\n        ma_atomic_bool32_set(&pDevice->wasapi.isStartedPlayback, MA_TRUE);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__wasapi(ma_device* pDevice)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* Wait for any rerouting to finish before attempting to start the device. */\n    ma_mutex_lock(&pDevice->wasapi.rerouteLock);\n    {\n        result = ma_device_start__wasapi_nolock(pDevice);\n    }\n    ma_mutex_unlock(&pDevice->wasapi.rerouteLock);\n\n    return result;\n}\n\nstatic ma_result ma_device_stop__wasapi_nolock(ma_device* pDevice)\n{\n    ma_result result;\n    HRESULT hr;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->wasapi.hAvrtHandle) {\n        ((MA_PFN_AvRevertMmThreadCharacteristics)pDevice->pContext->wasapi.AvRevertMmThreadcharacteristics)((HANDLE)pDevice->wasapi.hAvrtHandle);\n        pDevice->wasapi.hAvrtHandle = NULL;\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n        /* If we have a mapped buffer we need to release it. */\n        if (pDevice->wasapi.pMappedBufferCapture != NULL) {\n            ma_IAudioCaptureClient_ReleaseBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, pDevice->wasapi.mappedBufferCaptureCap);\n            pDevice->wasapi.pMappedBufferCapture = NULL;\n            pDevice->wasapi.mappedBufferCaptureCap = 0;\n            pDevice->wasapi.mappedBufferCaptureLen = 0;\n        }\n\n        hr = ma_IAudioClient_Stop((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to stop internal capture device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* The audio client needs to be reset otherwise restarting will fail. */\n        hr = ma_IAudioClient_Reset((ma_IAudioClient*)pDevice->wasapi.pAudioClientCapture);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to reset internal capture device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        ma_atomic_bool32_set(&pDevice->wasapi.isStartedCapture, MA_FALSE);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->wasapi.pMappedBufferPlayback != NULL) {\n            ma_silence_pcm_frames(\n                ma_offset_pcm_frames_ptr(pDevice->wasapi.pMappedBufferPlayback, pDevice->wasapi.mappedBufferPlaybackLen, pDevice->playback.internalFormat, pDevice->playback.internalChannels),\n                pDevice->wasapi.mappedBufferPlaybackCap - pDevice->wasapi.mappedBufferPlaybackLen,\n                pDevice->playback.internalFormat, pDevice->playback.internalChannels\n            );\n            ma_IAudioRenderClient_ReleaseBuffer((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient, pDevice->wasapi.mappedBufferPlaybackCap, 0);\n            pDevice->wasapi.pMappedBufferPlayback = NULL;\n            pDevice->wasapi.mappedBufferPlaybackCap = 0;\n            pDevice->wasapi.mappedBufferPlaybackLen = 0;\n        }\n\n        /*\n        The buffer needs to be drained before stopping the device. Not doing this will result in the last few frames not getting output to\n        the speakers. This is a problem for very short sounds because it'll result in a significant portion of it not getting played.\n        */\n        if (ma_atomic_bool32_get(&pDevice->wasapi.isStartedPlayback)) {\n            /* We need to make sure we put a timeout here or else we'll risk getting stuck in a deadlock in some cases. */\n            DWORD waitTime = (pDevice->wasapi.actualBufferSizeInFramesPlayback * 1000) / pDevice->playback.internalSampleRate;\n\n            if (pDevice->playback.shareMode == ma_share_mode_exclusive) {\n                WaitForSingleObject((HANDLE)pDevice->wasapi.hEventPlayback, waitTime);\n            } else {\n                ma_uint32 prevFramesAvailablePlayback = (ma_uint32)-1;\n                ma_uint32 framesAvailablePlayback;\n                for (;;) {\n                    result = ma_device__get_available_frames__wasapi(pDevice, (ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback, &framesAvailablePlayback);\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n\n                    if (framesAvailablePlayback >= pDevice->wasapi.actualBufferSizeInFramesPlayback) {\n                        break;\n                    }\n\n                    /*\n                    Just a safety check to avoid an infinite loop. If this iteration results in a situation where the number of available frames\n                    has not changed, get out of the loop. I don't think this should ever happen, but I think it's nice to have just in case.\n                    */\n                    if (framesAvailablePlayback == prevFramesAvailablePlayback) {\n                        break;\n                    }\n                    prevFramesAvailablePlayback = framesAvailablePlayback;\n\n                    ResetEvent((HANDLE)pDevice->wasapi.hEventPlayback); /* Manual reset. */\n                    WaitForSingleObject((HANDLE)pDevice->wasapi.hEventPlayback, waitTime);\n                }\n            }\n        }\n\n        hr = ma_IAudioClient_Stop((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to stop internal playback device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* The audio client needs to be reset otherwise restarting will fail. */\n        {\n            ma_int32 retries = 5;\n\n            while ((hr = ma_IAudioClient_Reset((ma_IAudioClient*)pDevice->wasapi.pAudioClientPlayback)) == MA_AUDCLNT_E_BUFFER_OPERATION_PENDING && retries > 0) {\n                ma_sleep(10);\n                retries -= 1;\n            }\n        }\n\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to reset internal playback device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        ma_atomic_bool32_set(&pDevice->wasapi.isStartedPlayback, MA_FALSE);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__wasapi(ma_device* pDevice)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* Wait for any rerouting to finish before attempting to stop the device. */\n    ma_mutex_lock(&pDevice->wasapi.rerouteLock);\n    {\n        result = ma_device_stop__wasapi_nolock(pDevice);\n    }\n    ma_mutex_unlock(&pDevice->wasapi.rerouteLock);\n\n    return result;\n}\n\n\n#ifndef MA_WASAPI_WAIT_TIMEOUT_MILLISECONDS\n#define MA_WASAPI_WAIT_TIMEOUT_MILLISECONDS 5000\n#endif\n\nstatic ma_result ma_device_read__wasapi(ma_device* pDevice, void* pFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 totalFramesProcessed = 0;\n\n    /*\n    When reading, we need to get a buffer and process all of it before releasing it. Because the\n    frame count (frameCount) can be different to the size of the buffer, we'll need to cache the\n    pointer to the buffer.\n    */\n\n    /* Keep running until we've processed the requested number of frames. */\n    while (ma_device_get_state(pDevice) == ma_device_state_started && totalFramesProcessed < frameCount) {\n        ma_uint32 framesRemaining = frameCount - totalFramesProcessed;\n\n        /* If we have a mapped data buffer, consume that first. */\n        if (pDevice->wasapi.pMappedBufferCapture != NULL) {\n            /* We have a cached data pointer so consume that before grabbing another one from WASAPI. */\n            ma_uint32 framesToProcessNow = framesRemaining;\n            if (framesToProcessNow > pDevice->wasapi.mappedBufferCaptureLen) {\n                framesToProcessNow = pDevice->wasapi.mappedBufferCaptureLen;\n            }\n\n            /* Now just copy the data over to the output buffer. */\n            ma_copy_pcm_frames(\n                ma_offset_pcm_frames_ptr(pFrames, totalFramesProcessed, pDevice->capture.internalFormat, pDevice->capture.internalChannels),\n                ma_offset_pcm_frames_const_ptr(pDevice->wasapi.pMappedBufferCapture, pDevice->wasapi.mappedBufferCaptureCap - pDevice->wasapi.mappedBufferCaptureLen, pDevice->capture.internalFormat, pDevice->capture.internalChannels),\n                framesToProcessNow,\n                pDevice->capture.internalFormat, pDevice->capture.internalChannels\n            );\n\n            totalFramesProcessed                   += framesToProcessNow;\n            pDevice->wasapi.mappedBufferCaptureLen -= framesToProcessNow;\n\n            /* If the data buffer has been fully consumed we need to release it. */\n            if (pDevice->wasapi.mappedBufferCaptureLen == 0) {\n                ma_IAudioCaptureClient_ReleaseBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, pDevice->wasapi.mappedBufferCaptureCap);\n                pDevice->wasapi.pMappedBufferCapture   = NULL;\n                pDevice->wasapi.mappedBufferCaptureCap = 0;\n            }\n        } else {\n            /* We don't have any cached data pointer, so grab another one. */\n            HRESULT hr;\n            DWORD flags = 0;\n\n            /* First just ask WASAPI for a data buffer. If it's not available, we'll wait for more. */\n            hr = ma_IAudioCaptureClient_GetBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, (BYTE**)&pDevice->wasapi.pMappedBufferCapture, &pDevice->wasapi.mappedBufferCaptureCap, &flags, NULL, NULL);\n            if (hr == S_OK) {\n                /* We got a data buffer. Continue to the next loop iteration which will then read from the mapped pointer. */\n                pDevice->wasapi.mappedBufferCaptureLen = pDevice->wasapi.mappedBufferCaptureCap;\n\n                /*\n                There have been reports that indicate that at times the AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY is reported for every\n                call to IAudioCaptureClient_GetBuffer() above which results in spamming of the debug messages below. To partially\n                work around this, I'm only outputting these messages when MA_DEBUG_OUTPUT is explicitly defined. The better solution\n                would be to figure out why the flag is always getting reported.\n                */\n                #if defined(MA_DEBUG_OUTPUT)\n                {\n                    if (flags != 0) {\n                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Capture Flags: %ld\\n\", flags);\n\n                        if ((flags & MA_AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) {\n                            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Data discontinuity (possible overrun). Attempting recovery. mappedBufferCaptureCap=%d\\n\", pDevice->wasapi.mappedBufferCaptureCap);\n                        }\n                    }\n                }\n                #endif\n\n                /* Overrun detection. */\n                if ((flags & MA_AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) {\n                    /* Glitched. Probably due to an overrun. */\n\n                    /*\n                    If we got an overrun it probably means we're straddling the end of the buffer. In normal capture\n                    mode this is the fault of the client application because they're responsible for ensuring data is\n                    processed fast enough. In duplex mode, however, the processing of audio is tied to the playback\n                    device, so this can possibly be the result of a timing de-sync.\n\n                    In capture mode we're not going to do any kind of recovery because the real fix is for the client\n                    application to process faster. In duplex mode, we'll treat this as a desync and reset the buffers\n                    to prevent a never-ending sequence of glitches due to straddling the end of the buffer.\n                    */\n                    if (pDevice->type == ma_device_type_duplex) {\n                        /*\n                        Experiment:\n\n                        If we empty out the *entire* buffer we may end up putting ourselves into an underrun position\n                        which isn't really any better than the overrun we're probably in right now. Instead we'll just\n                        empty out about half.\n                        */\n                        ma_uint32 i;\n                        ma_uint32 periodCount = (pDevice->wasapi.actualBufferSizeInFramesCapture / pDevice->wasapi.periodSizeInFramesCapture);\n                        ma_uint32 iterationCount = periodCount / 2;\n                        if ((periodCount % 2) > 0) {\n                            iterationCount += 1;\n                        }\n\n                        for (i = 0; i < iterationCount; i += 1) {\n                            hr = ma_IAudioCaptureClient_ReleaseBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, pDevice->wasapi.mappedBufferCaptureCap);\n                            if (FAILED(hr)) {\n                                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Data discontinuity recovery: IAudioCaptureClient_ReleaseBuffer() failed with %ld.\\n\", hr);\n                                break;\n                            }\n\n                            flags = 0;\n                            hr = ma_IAudioCaptureClient_GetBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, (BYTE**)&pDevice->wasapi.pMappedBufferCapture, &pDevice->wasapi.mappedBufferCaptureCap, &flags, NULL, NULL);\n                            if (hr == MA_AUDCLNT_S_BUFFER_EMPTY || FAILED(hr)) {\n                                /*\n                                The buffer has been completely emptied or an error occurred. In this case we'll need\n                                to reset the state of the mapped buffer which will trigger the next iteration to get\n                                a fresh buffer from WASAPI.\n                                */\n                                pDevice->wasapi.pMappedBufferCapture   = NULL;\n                                pDevice->wasapi.mappedBufferCaptureCap = 0;\n                                pDevice->wasapi.mappedBufferCaptureLen = 0;\n\n                                if (hr == MA_AUDCLNT_S_BUFFER_EMPTY) {\n                                    if ((flags & MA_AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) {\n                                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Data discontinuity recovery: Buffer emptied, and data discontinuity still reported.\\n\");\n                                    } else {\n                                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Data discontinuity recovery: Buffer emptied.\\n\");\n                                    }\n                                }\n\n                                if (FAILED(hr)) {\n                                    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[WASAPI] Data discontinuity recovery: IAudioCaptureClient_GetBuffer() failed with %ld.\\n\", hr);\n                                }\n\n                                break;\n                            }\n                        }\n\n                        /* If at this point we have a valid buffer mapped, make sure the buffer length is set appropriately. */\n                        if (pDevice->wasapi.pMappedBufferCapture != NULL) {\n                            pDevice->wasapi.mappedBufferCaptureLen = pDevice->wasapi.mappedBufferCaptureCap;\n                        }\n                    }\n                }\n\n                continue;\n            } else {\n                if (hr == MA_AUDCLNT_S_BUFFER_EMPTY || hr == MA_AUDCLNT_E_BUFFER_ERROR) {\n                    /*\n                    No data is available. We need to wait for more. There's two situations to consider\n                    here. The first is normal capture mode. If this times out it probably means the\n                    microphone isn't delivering data for whatever reason. In this case we'll just\n                    abort the read and return whatever we were able to get. The other situations is\n                    loopback mode, in which case a timeout probably just means the nothing is playing\n                    through the speakers.\n                    */\n\n                    /* Experiment: Use a shorter timeout for loopback mode. */\n                    DWORD timeoutInMilliseconds = MA_WASAPI_WAIT_TIMEOUT_MILLISECONDS;\n                    if (pDevice->type == ma_device_type_loopback) {\n                        timeoutInMilliseconds = 10;\n                    }\n\n                    if (WaitForSingleObject((HANDLE)pDevice->wasapi.hEventCapture, timeoutInMilliseconds) != WAIT_OBJECT_0) {\n                        if (pDevice->type == ma_device_type_loopback) {\n                            continue;   /* Keep waiting in loopback mode. */\n                        } else {\n                            result = MA_ERROR;\n                            break;      /* Wait failed. */\n                        }\n                    }\n\n                    /* At this point we should be able to loop back to the start of the loop and try retrieving a data buffer again. */\n                } else {\n                    /* An error occurred and we need to abort. */\n                    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to retrieve internal buffer from capture device in preparation for reading from the device. HRESULT = %d. Stopping device.\\n\", (int)hr);\n                    result = ma_result_from_HRESULT(hr);\n                    break;\n                }\n            }\n        }\n    }\n\n    /*\n    If we were unable to process the entire requested frame count, but we still have a mapped buffer,\n    there's a good chance either an error occurred or the device was stopped mid-read. In this case\n    we'll need to make sure the buffer is released.\n    */\n    if (totalFramesProcessed < frameCount && pDevice->wasapi.pMappedBufferCapture != NULL) {\n        ma_IAudioCaptureClient_ReleaseBuffer((ma_IAudioCaptureClient*)pDevice->wasapi.pCaptureClient, pDevice->wasapi.mappedBufferCaptureCap);\n        pDevice->wasapi.pMappedBufferCapture   = NULL;\n        pDevice->wasapi.mappedBufferCaptureCap = 0;\n        pDevice->wasapi.mappedBufferCaptureLen = 0;\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesProcessed;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_device_write__wasapi(ma_device* pDevice, const void* pFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 totalFramesProcessed = 0;\n\n    /* Keep writing to the device until it's stopped or we've consumed all of our input. */\n    while (ma_device_get_state(pDevice) == ma_device_state_started && totalFramesProcessed < frameCount) {\n        ma_uint32 framesRemaining = frameCount - totalFramesProcessed;\n\n        /*\n        We're going to do this in a similar way to capture. We'll first check if the cached data pointer\n        is valid, and if so, read from that. Otherwise We will call IAudioRenderClient_GetBuffer() with\n        a requested buffer size equal to our actual period size. If it returns AUDCLNT_E_BUFFER_TOO_LARGE\n        it means we need to wait for some data to become available.\n        */\n        if (pDevice->wasapi.pMappedBufferPlayback != NULL) {\n            /* We still have some space available in the mapped data buffer. Write to it. */\n            ma_uint32 framesToProcessNow = framesRemaining;\n            if (framesToProcessNow > (pDevice->wasapi.mappedBufferPlaybackCap - pDevice->wasapi.mappedBufferPlaybackLen)) {\n                framesToProcessNow = (pDevice->wasapi.mappedBufferPlaybackCap - pDevice->wasapi.mappedBufferPlaybackLen);\n            }\n\n            /* Now just copy the data over to the output buffer. */\n            ma_copy_pcm_frames(\n                ma_offset_pcm_frames_ptr(pDevice->wasapi.pMappedBufferPlayback, pDevice->wasapi.mappedBufferPlaybackLen, pDevice->playback.internalFormat, pDevice->playback.internalChannels),\n                ma_offset_pcm_frames_const_ptr(pFrames, totalFramesProcessed, pDevice->playback.internalFormat, pDevice->playback.internalChannels),\n                framesToProcessNow,\n                pDevice->playback.internalFormat, pDevice->playback.internalChannels\n            );\n\n            totalFramesProcessed                    += framesToProcessNow;\n            pDevice->wasapi.mappedBufferPlaybackLen += framesToProcessNow;\n\n            /* If the data buffer has been fully consumed we need to release it. */\n            if (pDevice->wasapi.mappedBufferPlaybackLen == pDevice->wasapi.mappedBufferPlaybackCap) {\n                ma_IAudioRenderClient_ReleaseBuffer((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient, pDevice->wasapi.mappedBufferPlaybackCap, 0);\n                pDevice->wasapi.pMappedBufferPlayback   = NULL;\n                pDevice->wasapi.mappedBufferPlaybackCap = 0;\n                pDevice->wasapi.mappedBufferPlaybackLen = 0;\n\n                /*\n                In exclusive mode we need to wait here. Exclusive mode is weird because GetBuffer() never\n                seems to return AUDCLNT_E_BUFFER_TOO_LARGE, which is what we normally use to determine\n                whether or not we need to wait for more data.\n                */\n                if (pDevice->playback.shareMode == ma_share_mode_exclusive) {\n                    if (WaitForSingleObject((HANDLE)pDevice->wasapi.hEventPlayback, MA_WASAPI_WAIT_TIMEOUT_MILLISECONDS) != WAIT_OBJECT_0) {\n                        result = MA_ERROR;\n                        break;   /* Wait failed. Probably timed out. */\n                    }\n                }\n            }\n        } else {\n            /* We don't have a mapped data buffer so we'll need to get one. */\n            HRESULT hr;\n            ma_uint32 bufferSizeInFrames;\n\n            /* Special rules for exclusive mode. */\n            if (pDevice->playback.shareMode == ma_share_mode_exclusive) {\n                bufferSizeInFrames = pDevice->wasapi.actualBufferSizeInFramesPlayback;\n            } else {\n                bufferSizeInFrames = pDevice->wasapi.periodSizeInFramesPlayback;\n            }\n\n            hr = ma_IAudioRenderClient_GetBuffer((ma_IAudioRenderClient*)pDevice->wasapi.pRenderClient, bufferSizeInFrames, (BYTE**)&pDevice->wasapi.pMappedBufferPlayback);\n            if (hr == S_OK) {\n                /* We have data available. */\n                pDevice->wasapi.mappedBufferPlaybackCap = bufferSizeInFrames;\n                pDevice->wasapi.mappedBufferPlaybackLen = 0;\n            } else {\n                if (hr == MA_AUDCLNT_E_BUFFER_TOO_LARGE || hr == MA_AUDCLNT_E_BUFFER_ERROR) {\n                    /* Not enough data available. We need to wait for more. */\n                    if (WaitForSingleObject((HANDLE)pDevice->wasapi.hEventPlayback, MA_WASAPI_WAIT_TIMEOUT_MILLISECONDS) != WAIT_OBJECT_0) {\n                        result = MA_ERROR;\n                        break;   /* Wait failed. Probably timed out. */\n                    }\n                } else {\n                    /* Some error occurred. We'll need to abort. */\n                    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WASAPI] Failed to retrieve internal buffer from playback device in preparation for writing to the device. HRESULT = %d. Stopping device.\\n\", (int)hr);\n                    result = ma_result_from_HRESULT(hr);\n                    break;\n                }\n            }\n        }\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = totalFramesProcessed;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_device_data_loop_wakeup__wasapi(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n        SetEvent((HANDLE)pDevice->wasapi.hEventCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        SetEvent((HANDLE)pDevice->wasapi.hEventPlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_uninit__wasapi(ma_context* pContext)\n{\n    ma_context_command__wasapi cmd = ma_context_init_command__wasapi(MA_CONTEXT_COMMAND_QUIT__WASAPI);\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_wasapi);\n\n    ma_context_post_command__wasapi(pContext, &cmd);\n    ma_thread_wait(&pContext->wasapi.commandThread);\n\n    if (pContext->wasapi.hAvrt) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->wasapi.hAvrt);\n        pContext->wasapi.hAvrt = NULL;\n    }\n\n    #if defined(MA_WIN32_UWP)\n    {\n        if (pContext->wasapi.hMMDevapi) {\n            ma_dlclose(ma_context_get_log(pContext), pContext->wasapi.hMMDevapi);\n            pContext->wasapi.hMMDevapi = NULL;\n        }\n    }\n    #endif\n\n    /* Only after the thread has been terminated can we uninitialize the sync objects for the command thread. */\n    ma_semaphore_uninit(&pContext->wasapi.commandSem);\n    ma_mutex_uninit(&pContext->wasapi.commandLock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__wasapi(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    ma_result result = MA_SUCCESS;\n\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n#ifdef MA_WIN32_DESKTOP\n    /*\n    WASAPI is only supported in Vista SP1 and newer. The reason for SP1 and not the base version of Vista is that event-driven\n    exclusive mode does not work until SP1.\n\n    Unfortunately older compilers don't define these functions so we need to dynamically load them in order to avoid a link error.\n    */\n    {\n        ma_OSVERSIONINFOEXW osvi;\n        ma_handle kernel32DLL;\n        ma_PFNVerifyVersionInfoW _VerifyVersionInfoW;\n        ma_PFNVerSetConditionMask _VerSetConditionMask;\n\n        kernel32DLL = ma_dlopen(ma_context_get_log(pContext), \"kernel32.dll\");\n        if (kernel32DLL == NULL) {\n            return MA_NO_BACKEND;\n        }\n\n        _VerifyVersionInfoW  = (ma_PFNVerifyVersionInfoW )ma_dlsym(ma_context_get_log(pContext), kernel32DLL, \"VerifyVersionInfoW\");\n        _VerSetConditionMask = (ma_PFNVerSetConditionMask)ma_dlsym(ma_context_get_log(pContext), kernel32DLL, \"VerSetConditionMask\");\n        if (_VerifyVersionInfoW == NULL || _VerSetConditionMask == NULL) {\n            ma_dlclose(ma_context_get_log(pContext), kernel32DLL);\n            return MA_NO_BACKEND;\n        }\n\n        MA_ZERO_OBJECT(&osvi);\n        osvi.dwOSVersionInfoSize = sizeof(osvi);\n        osvi.dwMajorVersion = ((MA_WIN32_WINNT_VISTA >> 8) & 0xFF);\n        osvi.dwMinorVersion = ((MA_WIN32_WINNT_VISTA >> 0) & 0xFF);\n        osvi.wServicePackMajor = 1;\n        if (_VerifyVersionInfoW(&osvi, MA_VER_MAJORVERSION | MA_VER_MINORVERSION | MA_VER_SERVICEPACKMAJOR, _VerSetConditionMask(_VerSetConditionMask(_VerSetConditionMask(0, MA_VER_MAJORVERSION, MA_VER_GREATER_EQUAL), MA_VER_MINORVERSION, MA_VER_GREATER_EQUAL), MA_VER_SERVICEPACKMAJOR, MA_VER_GREATER_EQUAL))) {\n            result = MA_SUCCESS;\n        } else {\n            result = MA_NO_BACKEND;\n        }\n\n        ma_dlclose(ma_context_get_log(pContext), kernel32DLL);\n    }\n#endif\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    MA_ZERO_OBJECT(&pContext->wasapi);\n\n\n    #if defined(MA_WIN32_UWP)\n    {\n        /* Link to mmdevapi so we can get access to ActivateAudioInterfaceAsync(). */\n        pContext->wasapi.hMMDevapi = ma_dlopen(ma_context_get_log(pContext), \"mmdevapi.dll\");\n        if (pContext->wasapi.hMMDevapi) {\n            pContext->wasapi.ActivateAudioInterfaceAsync = ma_dlsym(ma_context_get_log(pContext), pContext->wasapi.hMMDevapi, \"ActivateAudioInterfaceAsync\");\n            if (pContext->wasapi.ActivateAudioInterfaceAsync == NULL) {\n                ma_dlclose(ma_context_get_log(pContext), pContext->wasapi.hMMDevapi);\n                return MA_NO_BACKEND;   /* ActivateAudioInterfaceAsync() could not be loaded. */\n            }\n        } else {\n            return MA_NO_BACKEND;   /* Failed to load mmdevapi.dll which is required for ActivateAudioInterfaceAsync() */\n        }\n    }\n    #endif\n\n    /* Optionally use the Avrt API to specify the audio thread's latency sensitivity requirements */\n    pContext->wasapi.hAvrt = ma_dlopen(ma_context_get_log(pContext), \"avrt.dll\");\n    if (pContext->wasapi.hAvrt) {\n        pContext->wasapi.AvSetMmThreadCharacteristicsA   = ma_dlsym(ma_context_get_log(pContext), pContext->wasapi.hAvrt, \"AvSetMmThreadCharacteristicsA\");\n        pContext->wasapi.AvRevertMmThreadcharacteristics = ma_dlsym(ma_context_get_log(pContext), pContext->wasapi.hAvrt, \"AvRevertMmThreadCharacteristics\");\n\n        /* If either function could not be found, disable use of avrt entirely. */\n        if (!pContext->wasapi.AvSetMmThreadCharacteristicsA || !pContext->wasapi.AvRevertMmThreadcharacteristics) {\n            pContext->wasapi.AvSetMmThreadCharacteristicsA   = NULL;\n            pContext->wasapi.AvRevertMmThreadcharacteristics = NULL;\n            ma_dlclose(ma_context_get_log(pContext), pContext->wasapi.hAvrt);\n            pContext->wasapi.hAvrt = NULL;\n        }\n    }\n\n\n    /*\n    Annoyingly, WASAPI does not allow you to release an IAudioClient object from a different thread\n    than the one that retrieved it with GetService(). This can result in a deadlock in two\n    situations:\n\n        1) When calling ma_device_uninit() from a different thread to ma_device_init(); and\n        2) When uninitializing and reinitializing the internal IAudioClient object in response to\n           automatic stream routing.\n\n    We could define ma_device_uninit() such that it must be called on the same thread as\n    ma_device_init(). We could also just not release the IAudioClient when performing automatic\n    stream routing to avoid the deadlock. Neither of these are acceptable solutions in my view so\n    we're going to have to work around this with a worker thread. This is not ideal, but I can't\n    think of a better way to do this.\n\n    More information about this can be found here:\n\n        https://docs.microsoft.com/en-us/windows/win32/api/audioclient/nn-audioclient-iaudiorenderclient\n\n    Note this section:\n\n        When releasing an IAudioRenderClient interface instance, the client must call the interface's\n        Release method from the same thread as the call to IAudioClient::GetService that created the\n        object.\n    */\n    {\n        result = ma_mutex_init(&pContext->wasapi.commandLock);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_semaphore_init(0, &pContext->wasapi.commandSem);\n        if (result != MA_SUCCESS) {\n            ma_mutex_uninit(&pContext->wasapi.commandLock);\n            return result;\n        }\n\n        result = ma_thread_create(&pContext->wasapi.commandThread, ma_thread_priority_normal, 0, ma_context_command_thread__wasapi, pContext, &pContext->allocationCallbacks);\n        if (result != MA_SUCCESS) {\n            ma_semaphore_uninit(&pContext->wasapi.commandSem);\n            ma_mutex_uninit(&pContext->wasapi.commandLock);\n            return result;\n        }\n    }\n\n\n    pCallbacks->onContextInit             = ma_context_init__wasapi;\n    pCallbacks->onContextUninit           = ma_context_uninit__wasapi;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__wasapi;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__wasapi;\n    pCallbacks->onDeviceInit              = ma_device_init__wasapi;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__wasapi;\n    pCallbacks->onDeviceStart             = ma_device_start__wasapi;\n    pCallbacks->onDeviceStop              = ma_device_stop__wasapi;\n    pCallbacks->onDeviceRead              = ma_device_read__wasapi;\n    pCallbacks->onDeviceWrite             = ma_device_write__wasapi;\n    pCallbacks->onDeviceDataLoop          = NULL;\n    pCallbacks->onDeviceDataLoopWakeup    = ma_device_data_loop_wakeup__wasapi;\n\n    return MA_SUCCESS;\n}\n#endif\n\n/******************************************************************************\n\nDirectSound Backend\n\n******************************************************************************/\n#ifdef MA_HAS_DSOUND\n/*#include <dsound.h>*/\n\n/*static const GUID MA_GUID_IID_DirectSoundNotify = {0xb0210783, 0x89cd, 0x11d0, {0xaf, 0x08, 0x00, 0xa0, 0xc9, 0x25, 0xcd, 0x16}};*/\n\n/* miniaudio only uses priority or exclusive modes. */\n#define MA_DSSCL_NORMAL                 1\n#define MA_DSSCL_PRIORITY               2\n#define MA_DSSCL_EXCLUSIVE              3\n#define MA_DSSCL_WRITEPRIMARY           4\n\n#define MA_DSCAPS_PRIMARYMONO           0x00000001\n#define MA_DSCAPS_PRIMARYSTEREO         0x00000002\n#define MA_DSCAPS_PRIMARY8BIT           0x00000004\n#define MA_DSCAPS_PRIMARY16BIT          0x00000008\n#define MA_DSCAPS_CONTINUOUSRATE        0x00000010\n#define MA_DSCAPS_EMULDRIVER            0x00000020\n#define MA_DSCAPS_CERTIFIED             0x00000040\n#define MA_DSCAPS_SECONDARYMONO         0x00000100\n#define MA_DSCAPS_SECONDARYSTEREO       0x00000200\n#define MA_DSCAPS_SECONDARY8BIT         0x00000400\n#define MA_DSCAPS_SECONDARY16BIT        0x00000800\n\n#define MA_DSBCAPS_PRIMARYBUFFER        0x00000001\n#define MA_DSBCAPS_STATIC               0x00000002\n#define MA_DSBCAPS_LOCHARDWARE          0x00000004\n#define MA_DSBCAPS_LOCSOFTWARE          0x00000008\n#define MA_DSBCAPS_CTRL3D               0x00000010\n#define MA_DSBCAPS_CTRLFREQUENCY        0x00000020\n#define MA_DSBCAPS_CTRLPAN              0x00000040\n#define MA_DSBCAPS_CTRLVOLUME           0x00000080\n#define MA_DSBCAPS_CTRLPOSITIONNOTIFY   0x00000100\n#define MA_DSBCAPS_CTRLFX               0x00000200\n#define MA_DSBCAPS_STICKYFOCUS          0x00004000\n#define MA_DSBCAPS_GLOBALFOCUS          0x00008000\n#define MA_DSBCAPS_GETCURRENTPOSITION2  0x00010000\n#define MA_DSBCAPS_MUTE3DATMAXDISTANCE  0x00020000\n#define MA_DSBCAPS_LOCDEFER             0x00040000\n#define MA_DSBCAPS_TRUEPLAYPOSITION     0x00080000\n\n#define MA_DSBPLAY_LOOPING              0x00000001\n#define MA_DSBPLAY_LOCHARDWARE          0x00000002\n#define MA_DSBPLAY_LOCSOFTWARE          0x00000004\n#define MA_DSBPLAY_TERMINATEBY_TIME     0x00000008\n#define MA_DSBPLAY_TERMINATEBY_DISTANCE 0x00000010\n#define MA_DSBPLAY_TERMINATEBY_PRIORITY 0x00000020\n\n#define MA_DSBSTATUS_PLAYING            0x00000001\n#define MA_DSBSTATUS_BUFFERLOST         0x00000002\n#define MA_DSBSTATUS_LOOPING            0x00000004\n#define MA_DSBSTATUS_LOCHARDWARE        0x00000008\n#define MA_DSBSTATUS_LOCSOFTWARE        0x00000010\n#define MA_DSBSTATUS_TERMINATED         0x00000020\n\n#define MA_DSCBSTART_LOOPING            0x00000001\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwBufferBytes;\n    DWORD dwReserved;\n    MA_WAVEFORMATEX* lpwfxFormat;\n    GUID guid3DAlgorithm;\n} MA_DSBUFFERDESC;\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwBufferBytes;\n    DWORD dwReserved;\n    MA_WAVEFORMATEX* lpwfxFormat;\n    DWORD dwFXCount;\n    void* lpDSCFXDesc;  /* <-- miniaudio doesn't use this, so set to void*. */\n} MA_DSCBUFFERDESC;\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwMinSecondarySampleRate;\n    DWORD dwMaxSecondarySampleRate;\n    DWORD dwPrimaryBuffers;\n    DWORD dwMaxHwMixingAllBuffers;\n    DWORD dwMaxHwMixingStaticBuffers;\n    DWORD dwMaxHwMixingStreamingBuffers;\n    DWORD dwFreeHwMixingAllBuffers;\n    DWORD dwFreeHwMixingStaticBuffers;\n    DWORD dwFreeHwMixingStreamingBuffers;\n    DWORD dwMaxHw3DAllBuffers;\n    DWORD dwMaxHw3DStaticBuffers;\n    DWORD dwMaxHw3DStreamingBuffers;\n    DWORD dwFreeHw3DAllBuffers;\n    DWORD dwFreeHw3DStaticBuffers;\n    DWORD dwFreeHw3DStreamingBuffers;\n    DWORD dwTotalHwMemBytes;\n    DWORD dwFreeHwMemBytes;\n    DWORD dwMaxContigFreeHwMemBytes;\n    DWORD dwUnlockTransferRateHwBuffers;\n    DWORD dwPlayCpuOverheadSwBuffers;\n    DWORD dwReserved1;\n    DWORD dwReserved2;\n} MA_DSCAPS;\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwBufferBytes;\n    DWORD dwUnlockTransferRate;\n    DWORD dwPlayCpuOverhead;\n} MA_DSBCAPS;\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwFormats;\n    DWORD dwChannels;\n} MA_DSCCAPS;\n\ntypedef struct\n{\n    DWORD dwSize;\n    DWORD dwFlags;\n    DWORD dwBufferBytes;\n    DWORD dwReserved;\n} MA_DSCBCAPS;\n\ntypedef struct\n{\n    DWORD  dwOffset;\n    HANDLE hEventNotify;\n} MA_DSBPOSITIONNOTIFY;\n\ntypedef struct ma_IDirectSound              ma_IDirectSound;\ntypedef struct ma_IDirectSoundBuffer        ma_IDirectSoundBuffer;\ntypedef struct ma_IDirectSoundCapture       ma_IDirectSoundCapture;\ntypedef struct ma_IDirectSoundCaptureBuffer ma_IDirectSoundCaptureBuffer;\ntypedef struct ma_IDirectSoundNotify        ma_IDirectSoundNotify;\n\n\n/*\nCOM objects. The way these work is that you have a vtable (a list of function pointers, kind of\nlike how C++ works internally), and then you have a structure with a single member, which is a\npointer to the vtable. The vtable is where the methods of the object are defined. Methods need\nto be in a specific order, and parent classes need to have their methods declared first.\n*/\n\n/* IDirectSound */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IDirectSound* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IDirectSound* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IDirectSound* pThis);\n\n    /* IDirectSound */\n    HRESULT (STDMETHODCALLTYPE * CreateSoundBuffer)   (ma_IDirectSound* pThis, const MA_DSBUFFERDESC* pDSBufferDesc, ma_IDirectSoundBuffer** ppDSBuffer, void* pUnkOuter);\n    HRESULT (STDMETHODCALLTYPE * GetCaps)             (ma_IDirectSound* pThis, MA_DSCAPS* pDSCaps);\n    HRESULT (STDMETHODCALLTYPE * DuplicateSoundBuffer)(ma_IDirectSound* pThis, ma_IDirectSoundBuffer* pDSBufferOriginal, ma_IDirectSoundBuffer** ppDSBufferDuplicate);\n    HRESULT (STDMETHODCALLTYPE * SetCooperativeLevel) (ma_IDirectSound* pThis, HWND hwnd, DWORD dwLevel);\n    HRESULT (STDMETHODCALLTYPE * Compact)             (ma_IDirectSound* pThis);\n    HRESULT (STDMETHODCALLTYPE * GetSpeakerConfig)    (ma_IDirectSound* pThis, DWORD* pSpeakerConfig);\n    HRESULT (STDMETHODCALLTYPE * SetSpeakerConfig)    (ma_IDirectSound* pThis, DWORD dwSpeakerConfig);\n    HRESULT (STDMETHODCALLTYPE * Initialize)          (ma_IDirectSound* pThis, const GUID* pGuidDevice);\n} ma_IDirectSoundVtbl;\nstruct ma_IDirectSound\n{\n    ma_IDirectSoundVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IDirectSound_QueryInterface(ma_IDirectSound* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IDirectSound_AddRef(ma_IDirectSound* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IDirectSound_Release(ma_IDirectSound* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSound_CreateSoundBuffer(ma_IDirectSound* pThis, const MA_DSBUFFERDESC* pDSBufferDesc, ma_IDirectSoundBuffer** ppDSBuffer, void* pUnkOuter) { return pThis->lpVtbl->CreateSoundBuffer(pThis, pDSBufferDesc, ppDSBuffer, pUnkOuter); }\nstatic MA_INLINE HRESULT ma_IDirectSound_GetCaps(ma_IDirectSound* pThis, MA_DSCAPS* pDSCaps)                            { return pThis->lpVtbl->GetCaps(pThis, pDSCaps); }\nstatic MA_INLINE HRESULT ma_IDirectSound_DuplicateSoundBuffer(ma_IDirectSound* pThis, ma_IDirectSoundBuffer* pDSBufferOriginal, ma_IDirectSoundBuffer** ppDSBufferDuplicate) { return pThis->lpVtbl->DuplicateSoundBuffer(pThis, pDSBufferOriginal, ppDSBufferDuplicate); }\nstatic MA_INLINE HRESULT ma_IDirectSound_SetCooperativeLevel(ma_IDirectSound* pThis, HWND hwnd, DWORD dwLevel)          { return pThis->lpVtbl->SetCooperativeLevel(pThis, hwnd, dwLevel); }\nstatic MA_INLINE HRESULT ma_IDirectSound_Compact(ma_IDirectSound* pThis)                                                { return pThis->lpVtbl->Compact(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSound_GetSpeakerConfig(ma_IDirectSound* pThis, DWORD* pSpeakerConfig)                { return pThis->lpVtbl->GetSpeakerConfig(pThis, pSpeakerConfig); }\nstatic MA_INLINE HRESULT ma_IDirectSound_SetSpeakerConfig(ma_IDirectSound* pThis, DWORD dwSpeakerConfig)                { return pThis->lpVtbl->SetSpeakerConfig(pThis, dwSpeakerConfig); }\nstatic MA_INLINE HRESULT ma_IDirectSound_Initialize(ma_IDirectSound* pThis, const GUID* pGuidDevice)                    { return pThis->lpVtbl->Initialize(pThis, pGuidDevice); }\n\n\n/* IDirectSoundBuffer */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IDirectSoundBuffer* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IDirectSoundBuffer* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IDirectSoundBuffer* pThis);\n\n    /* IDirectSoundBuffer */\n    HRESULT (STDMETHODCALLTYPE * GetCaps)           (ma_IDirectSoundBuffer* pThis, MA_DSBCAPS* pDSBufferCaps);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentPosition)(ma_IDirectSoundBuffer* pThis, DWORD* pCurrentPlayCursor, DWORD* pCurrentWriteCursor);\n    HRESULT (STDMETHODCALLTYPE * GetFormat)         (ma_IDirectSoundBuffer* pThis, MA_WAVEFORMATEX* pFormat, DWORD dwSizeAllocated, DWORD* pSizeWritten);\n    HRESULT (STDMETHODCALLTYPE * GetVolume)         (ma_IDirectSoundBuffer* pThis, LONG* pVolume);\n    HRESULT (STDMETHODCALLTYPE * GetPan)            (ma_IDirectSoundBuffer* pThis, LONG* pPan);\n    HRESULT (STDMETHODCALLTYPE * GetFrequency)      (ma_IDirectSoundBuffer* pThis, DWORD* pFrequency);\n    HRESULT (STDMETHODCALLTYPE * GetStatus)         (ma_IDirectSoundBuffer* pThis, DWORD* pStatus);\n    HRESULT (STDMETHODCALLTYPE * Initialize)        (ma_IDirectSoundBuffer* pThis, ma_IDirectSound* pDirectSound, const MA_DSBUFFERDESC* pDSBufferDesc);\n    HRESULT (STDMETHODCALLTYPE * Lock)              (ma_IDirectSoundBuffer* pThis, DWORD dwOffset, DWORD dwBytes, void** ppAudioPtr1, DWORD* pAudioBytes1, void** ppAudioPtr2, DWORD* pAudioBytes2, DWORD dwFlags);\n    HRESULT (STDMETHODCALLTYPE * Play)              (ma_IDirectSoundBuffer* pThis, DWORD dwReserved1, DWORD dwPriority, DWORD dwFlags);\n    HRESULT (STDMETHODCALLTYPE * SetCurrentPosition)(ma_IDirectSoundBuffer* pThis, DWORD dwNewPosition);\n    HRESULT (STDMETHODCALLTYPE * SetFormat)         (ma_IDirectSoundBuffer* pThis, const MA_WAVEFORMATEX* pFormat);\n    HRESULT (STDMETHODCALLTYPE * SetVolume)         (ma_IDirectSoundBuffer* pThis, LONG volume);\n    HRESULT (STDMETHODCALLTYPE * SetPan)            (ma_IDirectSoundBuffer* pThis, LONG pan);\n    HRESULT (STDMETHODCALLTYPE * SetFrequency)      (ma_IDirectSoundBuffer* pThis, DWORD dwFrequency);\n    HRESULT (STDMETHODCALLTYPE * Stop)              (ma_IDirectSoundBuffer* pThis);\n    HRESULT (STDMETHODCALLTYPE * Unlock)            (ma_IDirectSoundBuffer* pThis, void* pAudioPtr1, DWORD dwAudioBytes1, void* pAudioPtr2, DWORD dwAudioBytes2);\n    HRESULT (STDMETHODCALLTYPE * Restore)           (ma_IDirectSoundBuffer* pThis);\n} ma_IDirectSoundBufferVtbl;\nstruct ma_IDirectSoundBuffer\n{\n    ma_IDirectSoundBufferVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_QueryInterface(ma_IDirectSoundBuffer* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IDirectSoundBuffer_AddRef(ma_IDirectSoundBuffer* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IDirectSoundBuffer_Release(ma_IDirectSoundBuffer* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetCaps(ma_IDirectSoundBuffer* pThis, MA_DSBCAPS* pDSBufferCaps)                     { return pThis->lpVtbl->GetCaps(pThis, pDSBufferCaps); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetCurrentPosition(ma_IDirectSoundBuffer* pThis, DWORD* pCurrentPlayCursor, DWORD* pCurrentWriteCursor) { return pThis->lpVtbl->GetCurrentPosition(pThis, pCurrentPlayCursor, pCurrentWriteCursor); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetFormat(ma_IDirectSoundBuffer* pThis, MA_WAVEFORMATEX* pFormat, DWORD dwSizeAllocated, DWORD* pSizeWritten) { return pThis->lpVtbl->GetFormat(pThis, pFormat, dwSizeAllocated, pSizeWritten); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetVolume(ma_IDirectSoundBuffer* pThis, LONG* pVolume)                               { return pThis->lpVtbl->GetVolume(pThis, pVolume); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetPan(ma_IDirectSoundBuffer* pThis, LONG* pPan)                                     { return pThis->lpVtbl->GetPan(pThis, pPan); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetFrequency(ma_IDirectSoundBuffer* pThis, DWORD* pFrequency)                        { return pThis->lpVtbl->GetFrequency(pThis, pFrequency); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_GetStatus(ma_IDirectSoundBuffer* pThis, DWORD* pStatus)                              { return pThis->lpVtbl->GetStatus(pThis, pStatus); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Initialize(ma_IDirectSoundBuffer* pThis, ma_IDirectSound* pDirectSound, const MA_DSBUFFERDESC* pDSBufferDesc) { return pThis->lpVtbl->Initialize(pThis, pDirectSound, pDSBufferDesc); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Lock(ma_IDirectSoundBuffer* pThis, DWORD dwOffset, DWORD dwBytes, void** ppAudioPtr1, DWORD* pAudioBytes1, void** ppAudioPtr2, DWORD* pAudioBytes2, DWORD dwFlags) { return pThis->lpVtbl->Lock(pThis, dwOffset, dwBytes, ppAudioPtr1, pAudioBytes1, ppAudioPtr2, pAudioBytes2, dwFlags); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Play(ma_IDirectSoundBuffer* pThis, DWORD dwReserved1, DWORD dwPriority, DWORD dwFlags) { return pThis->lpVtbl->Play(pThis, dwReserved1, dwPriority, dwFlags); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_SetCurrentPosition(ma_IDirectSoundBuffer* pThis, DWORD dwNewPosition)                { return pThis->lpVtbl->SetCurrentPosition(pThis, dwNewPosition); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_SetFormat(ma_IDirectSoundBuffer* pThis, const MA_WAVEFORMATEX* pFormat)              { return pThis->lpVtbl->SetFormat(pThis, pFormat); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_SetVolume(ma_IDirectSoundBuffer* pThis, LONG volume)                                 { return pThis->lpVtbl->SetVolume(pThis, volume); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_SetPan(ma_IDirectSoundBuffer* pThis, LONG pan)                                       { return pThis->lpVtbl->SetPan(pThis, pan); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_SetFrequency(ma_IDirectSoundBuffer* pThis, DWORD dwFrequency)                        { return pThis->lpVtbl->SetFrequency(pThis, dwFrequency); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Stop(ma_IDirectSoundBuffer* pThis)                                                   { return pThis->lpVtbl->Stop(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Unlock(ma_IDirectSoundBuffer* pThis, void* pAudioPtr1, DWORD dwAudioBytes1, void* pAudioPtr2, DWORD dwAudioBytes2) { return pThis->lpVtbl->Unlock(pThis, pAudioPtr1, dwAudioBytes1, pAudioPtr2, dwAudioBytes2); }\nstatic MA_INLINE HRESULT ma_IDirectSoundBuffer_Restore(ma_IDirectSoundBuffer* pThis)                                                { return pThis->lpVtbl->Restore(pThis); }\n\n\n/* IDirectSoundCapture */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IDirectSoundCapture* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IDirectSoundCapture* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IDirectSoundCapture* pThis);\n\n    /* IDirectSoundCapture */\n    HRESULT (STDMETHODCALLTYPE * CreateCaptureBuffer)(ma_IDirectSoundCapture* pThis, const MA_DSCBUFFERDESC* pDSCBufferDesc, ma_IDirectSoundCaptureBuffer** ppDSCBuffer, void* pUnkOuter);\n    HRESULT (STDMETHODCALLTYPE * GetCaps)            (ma_IDirectSoundCapture* pThis, MA_DSCCAPS* pDSCCaps);\n    HRESULT (STDMETHODCALLTYPE * Initialize)         (ma_IDirectSoundCapture* pThis, const GUID* pGuidDevice);\n} ma_IDirectSoundCaptureVtbl;\nstruct ma_IDirectSoundCapture\n{\n    ma_IDirectSoundCaptureVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IDirectSoundCapture_QueryInterface     (ma_IDirectSoundCapture* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IDirectSoundCapture_AddRef             (ma_IDirectSoundCapture* pThis)                                    { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IDirectSoundCapture_Release            (ma_IDirectSoundCapture* pThis)                                    { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCapture_CreateCaptureBuffer(ma_IDirectSoundCapture* pThis, const MA_DSCBUFFERDESC* pDSCBufferDesc, ma_IDirectSoundCaptureBuffer** ppDSCBuffer, void* pUnkOuter) { return pThis->lpVtbl->CreateCaptureBuffer(pThis, pDSCBufferDesc, ppDSCBuffer, pUnkOuter); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCapture_GetCaps            (ma_IDirectSoundCapture* pThis, MA_DSCCAPS* pDSCCaps)              { return pThis->lpVtbl->GetCaps(pThis, pDSCCaps); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCapture_Initialize         (ma_IDirectSoundCapture* pThis, const GUID* pGuidDevice)           { return pThis->lpVtbl->Initialize(pThis, pGuidDevice); }\n\n\n/* IDirectSoundCaptureBuffer */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IDirectSoundCaptureBuffer* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IDirectSoundCaptureBuffer* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IDirectSoundCaptureBuffer* pThis);\n\n    /* IDirectSoundCaptureBuffer */\n    HRESULT (STDMETHODCALLTYPE * GetCaps)           (ma_IDirectSoundCaptureBuffer* pThis, MA_DSCBCAPS* pDSCBCaps);\n    HRESULT (STDMETHODCALLTYPE * GetCurrentPosition)(ma_IDirectSoundCaptureBuffer* pThis, DWORD* pCapturePosition, DWORD* pReadPosition);\n    HRESULT (STDMETHODCALLTYPE * GetFormat)         (ma_IDirectSoundCaptureBuffer* pThis, MA_WAVEFORMATEX* pFormat, DWORD dwSizeAllocated, DWORD* pSizeWritten);\n    HRESULT (STDMETHODCALLTYPE * GetStatus)         (ma_IDirectSoundCaptureBuffer* pThis, DWORD* pStatus);\n    HRESULT (STDMETHODCALLTYPE * Initialize)        (ma_IDirectSoundCaptureBuffer* pThis, ma_IDirectSoundCapture* pDirectSoundCapture, const MA_DSCBUFFERDESC* pDSCBufferDesc);\n    HRESULT (STDMETHODCALLTYPE * Lock)              (ma_IDirectSoundCaptureBuffer* pThis, DWORD dwOffset, DWORD dwBytes, void** ppAudioPtr1, DWORD* pAudioBytes1, void** ppAudioPtr2, DWORD* pAudioBytes2, DWORD dwFlags);\n    HRESULT (STDMETHODCALLTYPE * Start)             (ma_IDirectSoundCaptureBuffer* pThis, DWORD dwFlags);\n    HRESULT (STDMETHODCALLTYPE * Stop)              (ma_IDirectSoundCaptureBuffer* pThis);\n    HRESULT (STDMETHODCALLTYPE * Unlock)            (ma_IDirectSoundCaptureBuffer* pThis, void* pAudioPtr1, DWORD dwAudioBytes1, void* pAudioPtr2, DWORD dwAudioBytes2);\n} ma_IDirectSoundCaptureBufferVtbl;\nstruct ma_IDirectSoundCaptureBuffer\n{\n    ma_IDirectSoundCaptureBufferVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_QueryInterface(ma_IDirectSoundCaptureBuffer* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IDirectSoundCaptureBuffer_AddRef(ma_IDirectSoundCaptureBuffer* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IDirectSoundCaptureBuffer_Release(ma_IDirectSoundCaptureBuffer* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_GetCaps(ma_IDirectSoundCaptureBuffer* pThis, MA_DSCBCAPS* pDSCBCaps)                        { return pThis->lpVtbl->GetCaps(pThis, pDSCBCaps); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_GetCurrentPosition(ma_IDirectSoundCaptureBuffer* pThis, DWORD* pCapturePosition, DWORD* pReadPosition) { return pThis->lpVtbl->GetCurrentPosition(pThis, pCapturePosition, pReadPosition); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_GetFormat(ma_IDirectSoundCaptureBuffer* pThis, MA_WAVEFORMATEX* pFormat, DWORD dwSizeAllocated, DWORD* pSizeWritten) { return pThis->lpVtbl->GetFormat(pThis, pFormat, dwSizeAllocated, pSizeWritten); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_GetStatus(ma_IDirectSoundCaptureBuffer* pThis, DWORD* pStatus)                              { return pThis->lpVtbl->GetStatus(pThis, pStatus); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_Initialize(ma_IDirectSoundCaptureBuffer* pThis, ma_IDirectSoundCapture* pDirectSoundCapture, const MA_DSCBUFFERDESC* pDSCBufferDesc) { return pThis->lpVtbl->Initialize(pThis, pDirectSoundCapture, pDSCBufferDesc); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_Lock(ma_IDirectSoundCaptureBuffer* pThis, DWORD dwOffset, DWORD dwBytes, void** ppAudioPtr1, DWORD* pAudioBytes1, void** ppAudioPtr2, DWORD* pAudioBytes2, DWORD dwFlags) { return pThis->lpVtbl->Lock(pThis, dwOffset, dwBytes, ppAudioPtr1, pAudioBytes1, ppAudioPtr2, pAudioBytes2, dwFlags); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_Start(ma_IDirectSoundCaptureBuffer* pThis, DWORD dwFlags)                                   { return pThis->lpVtbl->Start(pThis, dwFlags); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_Stop(ma_IDirectSoundCaptureBuffer* pThis)                                                   { return pThis->lpVtbl->Stop(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundCaptureBuffer_Unlock(ma_IDirectSoundCaptureBuffer* pThis, void* pAudioPtr1, DWORD dwAudioBytes1, void* pAudioPtr2, DWORD dwAudioBytes2) { return pThis->lpVtbl->Unlock(pThis, pAudioPtr1, dwAudioBytes1, pAudioPtr2, dwAudioBytes2); }\n\n\n/* IDirectSoundNotify */\ntypedef struct\n{\n    /* IUnknown */\n    HRESULT (STDMETHODCALLTYPE * QueryInterface)(ma_IDirectSoundNotify* pThis, const IID* const riid, void** ppObject);\n    ULONG   (STDMETHODCALLTYPE * AddRef)        (ma_IDirectSoundNotify* pThis);\n    ULONG   (STDMETHODCALLTYPE * Release)       (ma_IDirectSoundNotify* pThis);\n\n    /* IDirectSoundNotify */\n    HRESULT (STDMETHODCALLTYPE * SetNotificationPositions)(ma_IDirectSoundNotify* pThis, DWORD dwPositionNotifies, const MA_DSBPOSITIONNOTIFY* pPositionNotifies);\n} ma_IDirectSoundNotifyVtbl;\nstruct ma_IDirectSoundNotify\n{\n    ma_IDirectSoundNotifyVtbl* lpVtbl;\n};\nstatic MA_INLINE HRESULT ma_IDirectSoundNotify_QueryInterface(ma_IDirectSoundNotify* pThis, const IID* const riid, void** ppObject) { return pThis->lpVtbl->QueryInterface(pThis, riid, ppObject); }\nstatic MA_INLINE ULONG   ma_IDirectSoundNotify_AddRef(ma_IDirectSoundNotify* pThis)                                                 { return pThis->lpVtbl->AddRef(pThis); }\nstatic MA_INLINE ULONG   ma_IDirectSoundNotify_Release(ma_IDirectSoundNotify* pThis)                                                { return pThis->lpVtbl->Release(pThis); }\nstatic MA_INLINE HRESULT ma_IDirectSoundNotify_SetNotificationPositions(ma_IDirectSoundNotify* pThis, DWORD dwPositionNotifies, const MA_DSBPOSITIONNOTIFY* pPositionNotifies) { return pThis->lpVtbl->SetNotificationPositions(pThis, dwPositionNotifies, pPositionNotifies); }\n\n\ntypedef BOOL    (CALLBACK * ma_DSEnumCallbackAProc)             (GUID* pDeviceGUID, const char* pDeviceDescription, const char* pModule, void* pContext);\ntypedef HRESULT (WINAPI   * ma_DirectSoundCreateProc)           (const GUID* pcGuidDevice, ma_IDirectSound** ppDS8, ma_IUnknown* pUnkOuter);\ntypedef HRESULT (WINAPI   * ma_DirectSoundEnumerateAProc)       (ma_DSEnumCallbackAProc pDSEnumCallback, void* pContext);\ntypedef HRESULT (WINAPI   * ma_DirectSoundCaptureCreateProc)    (const GUID* pcGuidDevice, ma_IDirectSoundCapture** ppDSC8, ma_IUnknown* pUnkOuter);\ntypedef HRESULT (WINAPI   * ma_DirectSoundCaptureEnumerateAProc)(ma_DSEnumCallbackAProc pDSEnumCallback, void* pContext);\n\nstatic ma_uint32 ma_get_best_sample_rate_within_range(ma_uint32 sampleRateMin, ma_uint32 sampleRateMax)\n{\n    /* Normalize the range in case we were given something stupid. */\n    if (sampleRateMin < (ma_uint32)ma_standard_sample_rate_min) {\n        sampleRateMin = (ma_uint32)ma_standard_sample_rate_min;\n    }\n    if (sampleRateMax > (ma_uint32)ma_standard_sample_rate_max) {\n        sampleRateMax = (ma_uint32)ma_standard_sample_rate_max;\n    }\n    if (sampleRateMin > sampleRateMax) {\n        sampleRateMin = sampleRateMax;\n    }\n\n    if (sampleRateMin == sampleRateMax) {\n        return sampleRateMax;\n    } else {\n        size_t iStandardRate;\n        for (iStandardRate = 0; iStandardRate < ma_countof(g_maStandardSampleRatePriorities); ++iStandardRate) {\n            ma_uint32 standardRate = g_maStandardSampleRatePriorities[iStandardRate];\n            if (standardRate >= sampleRateMin && standardRate <= sampleRateMax) {\n                return standardRate;\n            }\n        }\n    }\n\n    /* Should never get here. */\n    MA_ASSERT(MA_FALSE);\n    return 0;\n}\n\n/*\nRetrieves the channel count and channel map for the given speaker configuration. If the speaker configuration is unknown,\nthe channel count and channel map will be left unmodified.\n*/\nstatic void ma_get_channels_from_speaker_config__dsound(DWORD speakerConfig, WORD* pChannelsOut, DWORD* pChannelMapOut)\n{\n    WORD  channels;\n    DWORD channelMap;\n\n    channels = 0;\n    if (pChannelsOut != NULL) {\n        channels = *pChannelsOut;\n    }\n\n    channelMap = 0;\n    if (pChannelMapOut != NULL) {\n        channelMap = *pChannelMapOut;\n    }\n\n    /*\n    The speaker configuration is a combination of speaker config and speaker geometry. The lower 8 bits is what we care about. The upper\n    16 bits is for the geometry.\n    */\n    switch ((BYTE)(speakerConfig)) {\n        case 1 /*DSSPEAKER_HEADPHONE*/:                          channels = 2; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT; break;\n        case 2 /*DSSPEAKER_MONO*/:                               channels = 1; channelMap = SPEAKER_FRONT_CENTER; break;\n        case 3 /*DSSPEAKER_QUAD*/:                               channels = 4; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT; break;\n        case 4 /*DSSPEAKER_STEREO*/:                             channels = 2; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT; break;\n        case 5 /*DSSPEAKER_SURROUND*/:                           channels = 4; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_BACK_CENTER; break;\n        case 6 /*DSSPEAKER_5POINT1_BACK*/ /*DSSPEAKER_5POINT1*/: channels = 6; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT; break;\n        case 7 /*DSSPEAKER_7POINT1_WIDE*/ /*DSSPEAKER_7POINT1*/: channels = 8; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT | SPEAKER_FRONT_LEFT_OF_CENTER | SPEAKER_FRONT_RIGHT_OF_CENTER; break;\n        case 8 /*DSSPEAKER_7POINT1_SURROUND*/:                   channels = 8; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_BACK_LEFT | SPEAKER_BACK_RIGHT | SPEAKER_SIDE_LEFT | SPEAKER_SIDE_RIGHT; break;\n        case 9 /*DSSPEAKER_5POINT1_SURROUND*/:                   channels = 6; channelMap = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT | SPEAKER_FRONT_CENTER | SPEAKER_LOW_FREQUENCY | SPEAKER_SIDE_LEFT | SPEAKER_SIDE_RIGHT; break;\n        default: break;\n    }\n\n    if (pChannelsOut != NULL) {\n        *pChannelsOut = channels;\n    }\n\n    if (pChannelMapOut != NULL) {\n        *pChannelMapOut = channelMap;\n    }\n}\n\n\nstatic ma_result ma_context_create_IDirectSound__dsound(ma_context* pContext, ma_share_mode shareMode, const ma_device_id* pDeviceID, ma_IDirectSound** ppDirectSound)\n{\n    ma_IDirectSound* pDirectSound;\n    HWND hWnd;\n    HRESULT hr;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppDirectSound != NULL);\n\n    *ppDirectSound = NULL;\n    pDirectSound = NULL;\n\n    if (FAILED(((ma_DirectSoundCreateProc)pContext->dsound.DirectSoundCreate)((pDeviceID == NULL) ? NULL : (const GUID*)pDeviceID->dsound, &pDirectSound, NULL))) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[DirectSound] DirectSoundCreate() failed for playback device.\");\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n\n    /* The cooperative level must be set before doing anything else. */\n    hWnd = (HWND)pContext->dsound.hWnd;\n    if (hWnd == 0) {\n        hWnd = ((MA_PFN_GetForegroundWindow)pContext->win32.GetForegroundWindow)();\n        if (hWnd == 0) {\n            hWnd = ((MA_PFN_GetDesktopWindow)pContext->win32.GetDesktopWindow)();\n        }\n    }\n\n    hr = ma_IDirectSound_SetCooperativeLevel(pDirectSound, hWnd, (shareMode == ma_share_mode_exclusive) ? MA_DSSCL_EXCLUSIVE : MA_DSSCL_PRIORITY);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSound_SetCooperateiveLevel() failed for playback device.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    *ppDirectSound = pDirectSound;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_create_IDirectSoundCapture__dsound(ma_context* pContext, ma_share_mode shareMode, const ma_device_id* pDeviceID, ma_IDirectSoundCapture** ppDirectSoundCapture)\n{\n    ma_IDirectSoundCapture* pDirectSoundCapture;\n    HRESULT hr;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppDirectSoundCapture != NULL);\n\n    /* DirectSound does not support exclusive mode for capture. */\n    if (shareMode == ma_share_mode_exclusive) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    *ppDirectSoundCapture = NULL;\n    pDirectSoundCapture = NULL;\n\n    hr = ((ma_DirectSoundCaptureCreateProc)pContext->dsound.DirectSoundCaptureCreate)((pDeviceID == NULL) ? NULL : (const GUID*)pDeviceID->dsound, &pDirectSoundCapture, NULL);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[DirectSound] DirectSoundCaptureCreate() failed for capture device.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    *ppDirectSoundCapture = pDirectSoundCapture;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_format_info_for_IDirectSoundCapture__dsound(ma_context* pContext, ma_IDirectSoundCapture* pDirectSoundCapture, WORD* pChannels, WORD* pBitsPerSample, DWORD* pSampleRate)\n{\n    HRESULT hr;\n    MA_DSCCAPS caps;\n    WORD bitsPerSample;\n    DWORD sampleRate;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pDirectSoundCapture != NULL);\n\n    if (pChannels) {\n        *pChannels = 0;\n    }\n    if (pBitsPerSample) {\n        *pBitsPerSample = 0;\n    }\n    if (pSampleRate) {\n        *pSampleRate = 0;\n    }\n\n    MA_ZERO_OBJECT(&caps);\n    caps.dwSize = sizeof(caps);\n    hr = ma_IDirectSoundCapture_GetCaps(pDirectSoundCapture, &caps);\n    if (FAILED(hr)) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundCapture_GetCaps() failed for capture device.\");\n        return ma_result_from_HRESULT(hr);\n    }\n\n    if (pChannels) {\n        *pChannels = (WORD)caps.dwChannels;\n    }\n\n    /* The device can support multiple formats. We just go through the different formats in order of priority and pick the first one. This the same type of system as the WinMM backend. */\n    bitsPerSample = 16;\n    sampleRate = 48000;\n\n    if (caps.dwChannels == 1) {\n        if ((caps.dwFormats & WAVE_FORMAT_48M16) != 0) {\n            sampleRate = 48000;\n        } else if ((caps.dwFormats & WAVE_FORMAT_44M16) != 0) {\n            sampleRate = 44100;\n        } else if ((caps.dwFormats & WAVE_FORMAT_2M16) != 0) {\n            sampleRate = 22050;\n        } else if ((caps.dwFormats & WAVE_FORMAT_1M16) != 0) {\n            sampleRate = 11025;\n        } else if ((caps.dwFormats & WAVE_FORMAT_96M16) != 0) {\n            sampleRate = 96000;\n        } else {\n            bitsPerSample = 8;\n            if ((caps.dwFormats & WAVE_FORMAT_48M08) != 0) {\n                sampleRate = 48000;\n            } else if ((caps.dwFormats & WAVE_FORMAT_44M08) != 0) {\n                sampleRate = 44100;\n            } else if ((caps.dwFormats & WAVE_FORMAT_2M08) != 0) {\n                sampleRate = 22050;\n            } else if ((caps.dwFormats & WAVE_FORMAT_1M08) != 0) {\n                sampleRate = 11025;\n            } else if ((caps.dwFormats & WAVE_FORMAT_96M08) != 0) {\n                sampleRate = 96000;\n            } else {\n                bitsPerSample = 16;  /* Didn't find it. Just fall back to 16-bit. */\n            }\n        }\n    } else if (caps.dwChannels == 2) {\n        if ((caps.dwFormats & WAVE_FORMAT_48S16) != 0) {\n            sampleRate = 48000;\n        } else if ((caps.dwFormats & WAVE_FORMAT_44S16) != 0) {\n            sampleRate = 44100;\n        } else if ((caps.dwFormats & WAVE_FORMAT_2S16) != 0) {\n            sampleRate = 22050;\n        } else if ((caps.dwFormats & WAVE_FORMAT_1S16) != 0) {\n            sampleRate = 11025;\n        } else if ((caps.dwFormats & WAVE_FORMAT_96S16) != 0) {\n            sampleRate = 96000;\n        } else {\n            bitsPerSample = 8;\n            if ((caps.dwFormats & WAVE_FORMAT_48S08) != 0) {\n                sampleRate = 48000;\n            } else if ((caps.dwFormats & WAVE_FORMAT_44S08) != 0) {\n                sampleRate = 44100;\n            } else if ((caps.dwFormats & WAVE_FORMAT_2S08) != 0) {\n                sampleRate = 22050;\n            } else if ((caps.dwFormats & WAVE_FORMAT_1S08) != 0) {\n                sampleRate = 11025;\n            } else if ((caps.dwFormats & WAVE_FORMAT_96S08) != 0) {\n                sampleRate = 96000;\n            } else {\n                bitsPerSample = 16;  /* Didn't find it. Just fall back to 16-bit. */\n            }\n        }\n    }\n\n    if (pBitsPerSample) {\n        *pBitsPerSample = bitsPerSample;\n    }\n    if (pSampleRate) {\n        *pSampleRate = sampleRate;\n    }\n\n    return MA_SUCCESS;\n}\n\n\ntypedef struct\n{\n    ma_context* pContext;\n    ma_device_type deviceType;\n    ma_enum_devices_callback_proc callback;\n    void* pUserData;\n    ma_bool32 terminated;\n} ma_context_enumerate_devices_callback_data__dsound;\n\nstatic BOOL CALLBACK ma_context_enumerate_devices_callback__dsound(GUID* lpGuid, const char* lpcstrDescription, const char* lpcstrModule, void* lpContext)\n{\n    ma_context_enumerate_devices_callback_data__dsound* pData = (ma_context_enumerate_devices_callback_data__dsound*)lpContext;\n    ma_device_info deviceInfo;\n\n    (void)lpcstrModule;\n\n    MA_ZERO_OBJECT(&deviceInfo);\n\n    /* ID. */\n    if (lpGuid != NULL) {\n        MA_COPY_MEMORY(deviceInfo.id.dsound, lpGuid, 16);\n    } else {\n        MA_ZERO_MEMORY(deviceInfo.id.dsound, 16);\n        deviceInfo.isDefault = MA_TRUE;\n    }\n\n    /* Name / Description */\n    ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), lpcstrDescription, (size_t)-1);\n\n\n    /* Call the callback function, but make sure we stop enumerating if the callee requested so. */\n    MA_ASSERT(pData != NULL);\n    pData->terminated = (pData->callback(pData->pContext, pData->deviceType, &deviceInfo, pData->pUserData) == MA_FALSE);\n    if (pData->terminated) {\n        return FALSE;   /* Stop enumeration. */\n    } else {\n        return TRUE;    /* Continue enumeration. */\n    }\n}\n\nstatic ma_result ma_context_enumerate_devices__dsound(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_context_enumerate_devices_callback_data__dsound data;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    data.pContext = pContext;\n    data.callback = callback;\n    data.pUserData = pUserData;\n    data.terminated = MA_FALSE;\n\n    /* Playback. */\n    if (!data.terminated) {\n        data.deviceType = ma_device_type_playback;\n        ((ma_DirectSoundEnumerateAProc)pContext->dsound.DirectSoundEnumerateA)(ma_context_enumerate_devices_callback__dsound, &data);\n    }\n\n    /* Capture. */\n    if (!data.terminated) {\n        data.deviceType = ma_device_type_capture;\n        ((ma_DirectSoundCaptureEnumerateAProc)pContext->dsound.DirectSoundCaptureEnumerateA)(ma_context_enumerate_devices_callback__dsound, &data);\n    }\n\n    return MA_SUCCESS;\n}\n\n\ntypedef struct\n{\n    const ma_device_id* pDeviceID;\n    ma_device_info* pDeviceInfo;\n    ma_bool32 found;\n} ma_context_get_device_info_callback_data__dsound;\n\nstatic BOOL CALLBACK ma_context_get_device_info_callback__dsound(GUID* lpGuid, const char* lpcstrDescription, const char* lpcstrModule, void* lpContext)\n{\n    ma_context_get_device_info_callback_data__dsound* pData = (ma_context_get_device_info_callback_data__dsound*)lpContext;\n    MA_ASSERT(pData != NULL);\n\n    if ((pData->pDeviceID == NULL || ma_is_guid_null(pData->pDeviceID->dsound)) && (lpGuid == NULL || ma_is_guid_null(lpGuid))) {\n        /* Default device. */\n        ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), lpcstrDescription, (size_t)-1);\n        pData->pDeviceInfo->isDefault = MA_TRUE;\n        pData->found = MA_TRUE;\n        return FALSE;   /* Stop enumeration. */\n    } else {\n        /* Not the default device. */\n        if (lpGuid != NULL && pData->pDeviceID != NULL) {\n            if (memcmp(pData->pDeviceID->dsound, lpGuid, sizeof(pData->pDeviceID->dsound)) == 0) {\n                ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), lpcstrDescription, (size_t)-1);\n                pData->found = MA_TRUE;\n                return FALSE;   /* Stop enumeration. */\n            }\n        }\n    }\n\n    (void)lpcstrModule;\n    return TRUE;\n}\n\nstatic ma_result ma_context_get_device_info__dsound(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_result result;\n    HRESULT hr;\n\n    if (pDeviceID != NULL) {\n        ma_context_get_device_info_callback_data__dsound data;\n\n        /* ID. */\n        MA_COPY_MEMORY(pDeviceInfo->id.dsound, pDeviceID->dsound, 16);\n\n        /* Name / Description. This is retrieved by enumerating over each device until we find that one that matches the input ID. */\n        data.pDeviceID = pDeviceID;\n        data.pDeviceInfo = pDeviceInfo;\n        data.found = MA_FALSE;\n        if (deviceType == ma_device_type_playback) {\n            ((ma_DirectSoundEnumerateAProc)pContext->dsound.DirectSoundEnumerateA)(ma_context_get_device_info_callback__dsound, &data);\n        } else {\n            ((ma_DirectSoundCaptureEnumerateAProc)pContext->dsound.DirectSoundCaptureEnumerateA)(ma_context_get_device_info_callback__dsound, &data);\n        }\n\n        if (!data.found) {\n            return MA_NO_DEVICE;\n        }\n    } else {\n        /* I don't think there's a way to get the name of the default device with DirectSound. In this case we just need to use defaults. */\n\n        /* ID */\n        MA_ZERO_MEMORY(pDeviceInfo->id.dsound, 16);\n\n        /* Name / Description */\n        if (deviceType == ma_device_type_playback) {\n            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n        } else {\n            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n        }\n\n        pDeviceInfo->isDefault = MA_TRUE;\n    }\n\n    /* Retrieving detailed information is slightly different depending on the device type. */\n    if (deviceType == ma_device_type_playback) {\n        /* Playback. */\n        ma_IDirectSound* pDirectSound;\n        MA_DSCAPS caps;\n        WORD channels;\n\n        result = ma_context_create_IDirectSound__dsound(pContext, ma_share_mode_shared, pDeviceID, &pDirectSound);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        MA_ZERO_OBJECT(&caps);\n        caps.dwSize = sizeof(caps);\n        hr = ma_IDirectSound_GetCaps(pDirectSound, &caps);\n        if (FAILED(hr)) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSound_GetCaps() failed for playback device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n\n        /* Channels. Only a single channel count is reported for DirectSound. */\n        if ((caps.dwFlags & MA_DSCAPS_PRIMARYSTEREO) != 0) {\n            /* It supports at least stereo, but could support more. */\n            DWORD speakerConfig;\n\n            channels = 2;\n\n            /* Look at the speaker configuration to get a better idea on the channel count. */\n            hr = ma_IDirectSound_GetSpeakerConfig(pDirectSound, &speakerConfig);\n            if (SUCCEEDED(hr)) {\n                ma_get_channels_from_speaker_config__dsound(speakerConfig, &channels, NULL);\n            }\n        } else {\n            /* It does not support stereo, which means we are stuck with mono. */\n            channels = 1;\n        }\n\n\n        /*\n        In DirectSound, our native formats are centered around sample rates. All formats are supported, and we're only reporting a single channel\n        count. However, DirectSound can report a range of supported sample rates. We're only going to include standard rates known by miniaudio\n        in order to keep the size of this within reason.\n        */\n        if ((caps.dwFlags & MA_DSCAPS_CONTINUOUSRATE) != 0) {\n            /* Multiple sample rates are supported. We'll report in order of our preferred sample rates. */\n            size_t iStandardSampleRate;\n            for (iStandardSampleRate = 0; iStandardSampleRate < ma_countof(g_maStandardSampleRatePriorities); iStandardSampleRate += 1) {\n                ma_uint32 sampleRate = g_maStandardSampleRatePriorities[iStandardSampleRate];\n                if (sampleRate >= caps.dwMinSecondarySampleRate && sampleRate <= caps.dwMaxSecondarySampleRate) {\n                    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = ma_format_unknown;\n                    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n                    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = sampleRate;\n                    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = 0;\n                    pDeviceInfo->nativeDataFormatCount += 1;\n                }\n            }\n        } else {\n            /* Only a single sample rate is supported. */\n            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = ma_format_unknown;\n            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = caps.dwMaxSecondarySampleRate;\n            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = 0;\n            pDeviceInfo->nativeDataFormatCount += 1;\n        }\n\n        ma_IDirectSound_Release(pDirectSound);\n    } else {\n        /*\n        Capture. This is a little different to playback due to the say the supported formats are reported. Technically capture\n        devices can support a number of different formats, but for simplicity and consistency with ma_device_init() I'm just\n        reporting the best format.\n        */\n        ma_IDirectSoundCapture* pDirectSoundCapture;\n        WORD channels;\n        WORD bitsPerSample;\n        DWORD sampleRate;\n\n        result = ma_context_create_IDirectSoundCapture__dsound(pContext, ma_share_mode_shared, pDeviceID, &pDirectSoundCapture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_context_get_format_info_for_IDirectSoundCapture__dsound(pContext, pDirectSoundCapture, &channels, &bitsPerSample, &sampleRate);\n        if (result != MA_SUCCESS) {\n            ma_IDirectSoundCapture_Release(pDirectSoundCapture);\n            return result;\n        }\n\n        ma_IDirectSoundCapture_Release(pDirectSoundCapture);\n\n        /* The format is always an integer format and is based on the bits per sample. */\n        if (bitsPerSample == 8) {\n            pDeviceInfo->nativeDataFormats[0].format = ma_format_u8;\n        } else if (bitsPerSample == 16) {\n            pDeviceInfo->nativeDataFormats[0].format = ma_format_s16;\n        } else if (bitsPerSample == 24) {\n            pDeviceInfo->nativeDataFormats[0].format = ma_format_s24;\n        } else if (bitsPerSample == 32) {\n            pDeviceInfo->nativeDataFormats[0].format = ma_format_s32;\n        } else {\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        pDeviceInfo->nativeDataFormats[0].channels   = channels;\n        pDeviceInfo->nativeDataFormats[0].sampleRate = sampleRate;\n        pDeviceInfo->nativeDataFormats[0].flags      = 0;\n        pDeviceInfo->nativeDataFormatCount = 1;\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\nstatic ma_result ma_device_uninit__dsound(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->dsound.pCaptureBuffer != NULL) {\n        ma_IDirectSoundCaptureBuffer_Release((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer);\n    }\n    if (pDevice->dsound.pCapture != NULL) {\n        ma_IDirectSoundCapture_Release((ma_IDirectSoundCapture*)pDevice->dsound.pCapture);\n    }\n\n    if (pDevice->dsound.pPlaybackBuffer != NULL) {\n        ma_IDirectSoundBuffer_Release((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer);\n    }\n    if (pDevice->dsound.pPlaybackPrimaryBuffer != NULL) {\n        ma_IDirectSoundBuffer_Release((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackPrimaryBuffer);\n    }\n    if (pDevice->dsound.pPlayback != NULL) {\n        ma_IDirectSound_Release((ma_IDirectSound*)pDevice->dsound.pPlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_config_to_WAVEFORMATEXTENSIBLE(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, const ma_channel* pChannelMap, MA_WAVEFORMATEXTENSIBLE* pWF)\n{\n    GUID subformat;\n\n    if (format == ma_format_unknown) {\n        format = MA_DEFAULT_FORMAT;\n    }\n\n    if (channels == 0) {\n        channels = MA_DEFAULT_CHANNELS;\n    }\n\n    if (sampleRate == 0) {\n        sampleRate = MA_DEFAULT_SAMPLE_RATE;\n    }\n\n    switch (format)\n    {\n        case ma_format_u8:\n        case ma_format_s16:\n        case ma_format_s24:\n        /*case ma_format_s24_32:*/\n        case ma_format_s32:\n        {\n            subformat = MA_GUID_KSDATAFORMAT_SUBTYPE_PCM;\n        } break;\n\n        case ma_format_f32:\n        {\n            subformat = MA_GUID_KSDATAFORMAT_SUBTYPE_IEEE_FLOAT;\n        } break;\n\n        default:\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    MA_ZERO_OBJECT(pWF);\n    pWF->cbSize                      = sizeof(*pWF);\n    pWF->wFormatTag                  = WAVE_FORMAT_EXTENSIBLE;\n    pWF->nChannels                   = (WORD)channels;\n    pWF->nSamplesPerSec              = (DWORD)sampleRate;\n    pWF->wBitsPerSample              = (WORD)(ma_get_bytes_per_sample(format)*8);\n    pWF->nBlockAlign                 = (WORD)(pWF->nChannels * pWF->wBitsPerSample / 8);\n    pWF->nAvgBytesPerSec             = pWF->nBlockAlign * pWF->nSamplesPerSec;\n    pWF->Samples.wValidBitsPerSample = pWF->wBitsPerSample;\n    pWF->dwChannelMask               = ma_channel_map_to_channel_mask__win32(pChannelMap, channels);\n    pWF->SubFormat                   = subformat;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_uint32 ma_calculate_period_size_in_frames_from_descriptor__dsound(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile)\n{\n    /*\n    DirectSound has a minimum period size of 20ms. In practice, this doesn't seem to be enough for\n    reliable glitch-free processing so going to use 30ms instead.\n    */\n    ma_uint32 minPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(30, nativeSampleRate);\n    ma_uint32 periodSizeInFrames;\n\n    periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, nativeSampleRate, performanceProfile);\n    if (periodSizeInFrames < minPeriodSizeInFrames) {\n        periodSizeInFrames = minPeriodSizeInFrames;\n    }\n\n    return periodSizeInFrames;\n}\n\nstatic ma_result ma_device_init__dsound(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n    HRESULT hr;\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->dsound);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /*\n    Unfortunately DirectSound uses different APIs and data structures for playback and capture devices. We need to initialize\n    the capture device first because we'll want to match its buffer size and period count on the playback side if we're using\n    full-duplex mode.\n    */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        MA_WAVEFORMATEXTENSIBLE wf;\n        MA_DSCBUFFERDESC descDS;\n        ma_uint32 periodSizeInFrames;\n        ma_uint32 periodCount;\n        char rawdata[1024]; /* <-- Ugly hack to avoid a malloc() due to a crappy DirectSound API. */\n        MA_WAVEFORMATEXTENSIBLE* pActualFormat;\n\n        result = ma_config_to_WAVEFORMATEXTENSIBLE(pDescriptorCapture->format, pDescriptorCapture->channels, pDescriptorCapture->sampleRate, pDescriptorCapture->channelMap, &wf);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_context_create_IDirectSoundCapture__dsound(pDevice->pContext, pDescriptorCapture->shareMode, pDescriptorCapture->pDeviceID, (ma_IDirectSoundCapture**)&pDevice->dsound.pCapture);\n        if (result != MA_SUCCESS) {\n            ma_device_uninit__dsound(pDevice);\n            return result;\n        }\n\n        result = ma_context_get_format_info_for_IDirectSoundCapture__dsound(pDevice->pContext, (ma_IDirectSoundCapture*)pDevice->dsound.pCapture, &wf.nChannels, &wf.wBitsPerSample, &wf.nSamplesPerSec);\n        if (result != MA_SUCCESS) {\n            ma_device_uninit__dsound(pDevice);\n            return result;\n        }\n\n        wf.nBlockAlign                 = (WORD)(wf.nChannels * wf.wBitsPerSample / 8);\n        wf.nAvgBytesPerSec             = wf.nBlockAlign * wf.nSamplesPerSec;\n        wf.Samples.wValidBitsPerSample = wf.wBitsPerSample;\n        wf.SubFormat                   = MA_GUID_KSDATAFORMAT_SUBTYPE_PCM;\n\n        /* The size of the buffer must be a clean multiple of the period count. */\n        periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__dsound(pDescriptorCapture, wf.nSamplesPerSec, pConfig->performanceProfile);\n        periodCount = (pDescriptorCapture->periodCount > 0) ? pDescriptorCapture->periodCount : MA_DEFAULT_PERIODS;\n\n        MA_ZERO_OBJECT(&descDS);\n        descDS.dwSize        = sizeof(descDS);\n        descDS.dwFlags       = 0;\n        descDS.dwBufferBytes = periodSizeInFrames * periodCount * wf.nBlockAlign;\n        descDS.lpwfxFormat   = (MA_WAVEFORMATEX*)&wf;\n        hr = ma_IDirectSoundCapture_CreateCaptureBuffer((ma_IDirectSoundCapture*)pDevice->dsound.pCapture, &descDS, (ma_IDirectSoundCaptureBuffer**)&pDevice->dsound.pCaptureBuffer, NULL);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundCapture_CreateCaptureBuffer() failed for capture device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* Get the _actual_ properties of the buffer. */\n        pActualFormat = (MA_WAVEFORMATEXTENSIBLE*)rawdata;\n        hr = ma_IDirectSoundCaptureBuffer_GetFormat((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, (MA_WAVEFORMATEX*)pActualFormat, sizeof(rawdata), NULL);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to retrieve the actual format of the capture device's buffer.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* We can now start setting the output data formats. */\n        pDescriptorCapture->format     = ma_format_from_WAVEFORMATEX((MA_WAVEFORMATEX*)pActualFormat);\n        pDescriptorCapture->channels   = pActualFormat->nChannels;\n        pDescriptorCapture->sampleRate = pActualFormat->nSamplesPerSec;\n\n        /* Get the native channel map based on the channel mask. */\n        if (pActualFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {\n            ma_channel_mask_to_channel_map__win32(pActualFormat->dwChannelMask, pDescriptorCapture->channels, pDescriptorCapture->channelMap);\n        } else {\n            ma_channel_mask_to_channel_map__win32(wf.dwChannelMask, pDescriptorCapture->channels, pDescriptorCapture->channelMap);\n        }\n\n        /*\n        After getting the actual format the size of the buffer in frames may have actually changed. However, we want this to be as close to what the\n        user has asked for as possible, so let's go ahead and release the old capture buffer and create a new one in this case.\n        */\n        if (periodSizeInFrames != (descDS.dwBufferBytes / ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels) / periodCount)) {\n            descDS.dwBufferBytes = periodSizeInFrames * ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels) * periodCount;\n            ma_IDirectSoundCaptureBuffer_Release((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer);\n\n            hr = ma_IDirectSoundCapture_CreateCaptureBuffer((ma_IDirectSoundCapture*)pDevice->dsound.pCapture, &descDS, (ma_IDirectSoundCaptureBuffer**)&pDevice->dsound.pCaptureBuffer, NULL);\n            if (FAILED(hr)) {\n                ma_device_uninit__dsound(pDevice);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Second attempt at IDirectSoundCapture_CreateCaptureBuffer() failed for capture device.\");\n                return ma_result_from_HRESULT(hr);\n            }\n        }\n\n        /* DirectSound should give us a buffer exactly the size we asked for. */\n        pDescriptorCapture->periodSizeInFrames = periodSizeInFrames;\n        pDescriptorCapture->periodCount        = periodCount;\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        MA_WAVEFORMATEXTENSIBLE wf;\n        MA_DSBUFFERDESC descDSPrimary;\n        MA_DSCAPS caps;\n        char rawdata[1024]; /* <-- Ugly hack to avoid a malloc() due to a crappy DirectSound API. */\n        MA_WAVEFORMATEXTENSIBLE* pActualFormat;\n        ma_uint32 periodSizeInFrames;\n        ma_uint32 periodCount;\n        MA_DSBUFFERDESC descDS;\n        WORD nativeChannelCount;\n        DWORD nativeChannelMask = 0;\n\n        result = ma_config_to_WAVEFORMATEXTENSIBLE(pDescriptorPlayback->format, pDescriptorPlayback->channels, pDescriptorPlayback->sampleRate, pDescriptorPlayback->channelMap, &wf);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_context_create_IDirectSound__dsound(pDevice->pContext, pDescriptorPlayback->shareMode, pDescriptorPlayback->pDeviceID, (ma_IDirectSound**)&pDevice->dsound.pPlayback);\n        if (result != MA_SUCCESS) {\n            ma_device_uninit__dsound(pDevice);\n            return result;\n        }\n\n        MA_ZERO_OBJECT(&descDSPrimary);\n        descDSPrimary.dwSize  = sizeof(MA_DSBUFFERDESC);\n        descDSPrimary.dwFlags = MA_DSBCAPS_PRIMARYBUFFER | MA_DSBCAPS_CTRLVOLUME;\n        hr = ma_IDirectSound_CreateSoundBuffer((ma_IDirectSound*)pDevice->dsound.pPlayback, &descDSPrimary, (ma_IDirectSoundBuffer**)&pDevice->dsound.pPlaybackPrimaryBuffer, NULL);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSound_CreateSoundBuffer() failed for playback device's primary buffer.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n\n        /* We may want to make some adjustments to the format if we are using defaults. */\n        MA_ZERO_OBJECT(&caps);\n        caps.dwSize = sizeof(caps);\n        hr = ma_IDirectSound_GetCaps((ma_IDirectSound*)pDevice->dsound.pPlayback, &caps);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSound_GetCaps() failed for playback device.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        if ((caps.dwFlags & MA_DSCAPS_PRIMARYSTEREO) != 0) {\n            DWORD speakerConfig;\n\n            /* It supports at least stereo, but could support more. */\n            nativeChannelCount = 2;\n\n            /* Look at the speaker configuration to get a better idea on the channel count. */\n            if (SUCCEEDED(ma_IDirectSound_GetSpeakerConfig((ma_IDirectSound*)pDevice->dsound.pPlayback, &speakerConfig))) {\n                ma_get_channels_from_speaker_config__dsound(speakerConfig, &nativeChannelCount, &nativeChannelMask);\n            }\n        } else {\n            /* It does not support stereo, which means we are stuck with mono. */\n            nativeChannelCount = 1;\n            nativeChannelMask  = 0x00000001;\n        }\n\n        if (pDescriptorPlayback->channels == 0) {\n            wf.nChannels = nativeChannelCount;\n            wf.dwChannelMask    = nativeChannelMask;\n        }\n\n        if (pDescriptorPlayback->sampleRate == 0) {\n            /* We base the sample rate on the values returned by GetCaps(). */\n            if ((caps.dwFlags & MA_DSCAPS_CONTINUOUSRATE) != 0) {\n                wf.nSamplesPerSec = ma_get_best_sample_rate_within_range(caps.dwMinSecondarySampleRate, caps.dwMaxSecondarySampleRate);\n            } else {\n                wf.nSamplesPerSec = caps.dwMaxSecondarySampleRate;\n            }\n        }\n\n        wf.nBlockAlign     = (WORD)(wf.nChannels * wf.wBitsPerSample / 8);\n        wf.nAvgBytesPerSec = wf.nBlockAlign * wf.nSamplesPerSec;\n\n        /*\n        From MSDN:\n\n        The method succeeds even if the hardware does not support the requested format; DirectSound sets the buffer to the closest\n        supported format. To determine whether this has happened, an application can call the GetFormat method for the primary buffer\n        and compare the result with the format that was requested with the SetFormat method.\n        */\n        hr = ma_IDirectSoundBuffer_SetFormat((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackPrimaryBuffer, (MA_WAVEFORMATEX*)&wf);\n        if (FAILED(hr)) {\n            /*\n            If setting of the format failed we'll try again with some fallback settings. On Windows 98 I have\n            observed that IEEE_FLOAT does not work. We'll therefore enforce PCM. I also had issues where a\n            sample rate of 48000 did not work correctly. Not sure if it was a driver issue or not, but will\n            use 44100 for the sample rate.\n            */\n            wf.cbSize          = 18;    /* NOTE: Don't use sizeof(MA_WAVEFORMATEX) here because it's got an extra 2 bytes due to padding. */\n            wf.wFormatTag      = WAVE_FORMAT_PCM;\n            wf.wBitsPerSample  = 16;\n            wf.nChannels       = nativeChannelCount;\n            wf.nSamplesPerSec  = 44100;\n            wf.nBlockAlign     = wf.nChannels * (wf.wBitsPerSample / 8);\n            wf.nAvgBytesPerSec = wf.nSamplesPerSec * wf.nBlockAlign;\n\n            hr = ma_IDirectSoundBuffer_SetFormat((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackPrimaryBuffer, (MA_WAVEFORMATEX*)&wf);\n            if (FAILED(hr)) {\n                ma_device_uninit__dsound(pDevice);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to set format of playback device's primary buffer.\");\n                return ma_result_from_HRESULT(hr);\n            }\n        }\n\n        /* Get the _actual_ properties of the buffer. */\n        pActualFormat = (MA_WAVEFORMATEXTENSIBLE*)rawdata;\n        hr = ma_IDirectSoundBuffer_GetFormat((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackPrimaryBuffer, (MA_WAVEFORMATEX*)pActualFormat, sizeof(rawdata), NULL);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to retrieve the actual format of the playback device's primary buffer.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* We now have enough information to start setting some output properties. */\n        pDescriptorPlayback->format     = ma_format_from_WAVEFORMATEX((MA_WAVEFORMATEX*)pActualFormat);\n        pDescriptorPlayback->channels   = pActualFormat->nChannels;\n        pDescriptorPlayback->sampleRate = pActualFormat->nSamplesPerSec;\n\n        /* Get the internal channel map based on the channel mask. */\n        if (pActualFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {\n            ma_channel_mask_to_channel_map__win32(pActualFormat->dwChannelMask, pDescriptorPlayback->channels, pDescriptorPlayback->channelMap);\n        } else {\n            ma_channel_mask_to_channel_map__win32(wf.dwChannelMask, pDescriptorPlayback->channels, pDescriptorPlayback->channelMap);\n        }\n\n        /* The size of the buffer must be a clean multiple of the period count. */\n        periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__dsound(pDescriptorPlayback, pDescriptorPlayback->sampleRate, pConfig->performanceProfile);\n        periodCount = (pDescriptorPlayback->periodCount > 0) ? pDescriptorPlayback->periodCount : MA_DEFAULT_PERIODS;\n\n        /*\n        Meaning of dwFlags (from MSDN):\n\n        DSBCAPS_CTRLPOSITIONNOTIFY\n          The buffer has position notification capability.\n\n        DSBCAPS_GLOBALFOCUS\n          With this flag set, an application using DirectSound can continue to play its buffers if the user switches focus to\n          another application, even if the new application uses DirectSound.\n\n        DSBCAPS_GETCURRENTPOSITION2\n          In the first version of DirectSound, the play cursor was significantly ahead of the actual playing sound on emulated\n          sound cards; it was directly behind the write cursor. Now, if the DSBCAPS_GETCURRENTPOSITION2 flag is specified, the\n          application can get a more accurate play cursor.\n        */\n        MA_ZERO_OBJECT(&descDS);\n        descDS.dwSize = sizeof(descDS);\n        descDS.dwFlags = MA_DSBCAPS_CTRLPOSITIONNOTIFY | MA_DSBCAPS_GLOBALFOCUS | MA_DSBCAPS_GETCURRENTPOSITION2;\n        descDS.dwBufferBytes = periodSizeInFrames * periodCount * ma_get_bytes_per_frame(pDescriptorPlayback->format, pDescriptorPlayback->channels);\n        descDS.lpwfxFormat = (MA_WAVEFORMATEX*)pActualFormat;\n        hr = ma_IDirectSound_CreateSoundBuffer((ma_IDirectSound*)pDevice->dsound.pPlayback, &descDS, (ma_IDirectSoundBuffer**)&pDevice->dsound.pPlaybackBuffer, NULL);\n        if (FAILED(hr)) {\n            ma_device_uninit__dsound(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSound_CreateSoundBuffer() failed for playback device's secondary buffer.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        /* DirectSound should give us a buffer exactly the size we asked for. */\n        pDescriptorPlayback->periodSizeInFrames = periodSizeInFrames;\n        pDescriptorPlayback->periodCount        = periodCount;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_data_loop__dsound(ma_device* pDevice)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 bpfDeviceCapture  = ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n    ma_uint32 bpfDevicePlayback = ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    HRESULT hr;\n    DWORD lockOffsetInBytesCapture;\n    DWORD lockSizeInBytesCapture;\n    DWORD mappedSizeInBytesCapture;\n    DWORD mappedDeviceFramesProcessedCapture;\n    void* pMappedDeviceBufferCapture;\n    DWORD lockOffsetInBytesPlayback;\n    DWORD lockSizeInBytesPlayback;\n    DWORD mappedSizeInBytesPlayback;\n    void* pMappedDeviceBufferPlayback;\n    DWORD prevReadCursorInBytesCapture = 0;\n    DWORD prevPlayCursorInBytesPlayback = 0;\n    ma_bool32 physicalPlayCursorLoopFlagPlayback = 0;\n    DWORD virtualWriteCursorInBytesPlayback = 0;\n    ma_bool32 virtualWriteCursorLoopFlagPlayback = 0;\n    ma_bool32 isPlaybackDeviceStarted = MA_FALSE;\n    ma_uint32 framesWrittenToPlaybackDevice = 0;   /* For knowing whether or not the playback device needs to be started. */\n    ma_uint32 waitTimeInMilliseconds = 1;\n    DWORD playbackBufferStatus = 0;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* The first thing to do is start the capture device. The playback device is only started after the first period is written. */\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        hr = ma_IDirectSoundCaptureBuffer_Start((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, MA_DSCBSTART_LOOPING);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundCaptureBuffer_Start() failed.\");\n            return ma_result_from_HRESULT(hr);\n        }\n    }\n\n    while (ma_device_get_state(pDevice) == ma_device_state_started) {\n        switch (pDevice->type)\n        {\n            case ma_device_type_duplex:\n            {\n                DWORD physicalCaptureCursorInBytes;\n                DWORD physicalReadCursorInBytes;\n                hr = ma_IDirectSoundCaptureBuffer_GetCurrentPosition((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, &physicalCaptureCursorInBytes, &physicalReadCursorInBytes);\n                if (FAILED(hr)) {\n                    return ma_result_from_HRESULT(hr);\n                }\n\n                /* If nothing is available we just sleep for a bit and return from this iteration. */\n                if (physicalReadCursorInBytes == prevReadCursorInBytesCapture) {\n                    ma_sleep(waitTimeInMilliseconds);\n                    continue; /* Nothing is available in the capture buffer. */\n                }\n\n                /*\n                The current position has moved. We need to map all of the captured samples and write them to the playback device, making sure\n                we don't return until every frame has been copied over.\n                */\n                if (prevReadCursorInBytesCapture < physicalReadCursorInBytes) {\n                    /* The capture position has not looped. This is the simple case. */\n                    lockOffsetInBytesCapture = prevReadCursorInBytesCapture;\n                    lockSizeInBytesCapture   = (physicalReadCursorInBytes - prevReadCursorInBytesCapture);\n                } else {\n                    /*\n                    The capture position has looped. This is the more complex case. Map to the end of the buffer. If this does not return anything,\n                    do it again from the start.\n                    */\n                    if (prevReadCursorInBytesCapture < pDevice->capture.internalPeriodSizeInFrames*pDevice->capture.internalPeriods*bpfDeviceCapture) {\n                        /* Lock up to the end of the buffer. */\n                        lockOffsetInBytesCapture = prevReadCursorInBytesCapture;\n                        lockSizeInBytesCapture   = (pDevice->capture.internalPeriodSizeInFrames*pDevice->capture.internalPeriods*bpfDeviceCapture) - prevReadCursorInBytesCapture;\n                    } else {\n                        /* Lock starting from the start of the buffer. */\n                        lockOffsetInBytesCapture = 0;\n                        lockSizeInBytesCapture   = physicalReadCursorInBytes;\n                    }\n                }\n\n                if (lockSizeInBytesCapture == 0) {\n                    ma_sleep(waitTimeInMilliseconds);\n                    continue; /* Nothing is available in the capture buffer. */\n                }\n\n                hr = ma_IDirectSoundCaptureBuffer_Lock((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, lockOffsetInBytesCapture, lockSizeInBytesCapture, &pMappedDeviceBufferCapture, &mappedSizeInBytesCapture, NULL, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to map buffer from capture device in preparation for writing to the device.\");\n                    return ma_result_from_HRESULT(hr);\n                }\n\n\n                /* At this point we have some input data that we need to output. We do not return until every mapped frame of the input data is written to the playback device. */\n                mappedDeviceFramesProcessedCapture = 0;\n\n                for (;;) {  /* Keep writing to the playback device. */\n                    ma_uint8  inputFramesInClientFormat[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                    ma_uint32 inputFramesInClientFormatCap = sizeof(inputFramesInClientFormat) / ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels);\n                    ma_uint8  outputFramesInClientFormat[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                    ma_uint32 outputFramesInClientFormatCap = sizeof(outputFramesInClientFormat) / ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n                    ma_uint32 outputFramesInClientFormatCount;\n                    ma_uint32 outputFramesInClientFormatConsumed = 0;\n                    ma_uint64 clientCapturedFramesToProcess = ma_min(inputFramesInClientFormatCap, outputFramesInClientFormatCap);\n                    ma_uint64 deviceCapturedFramesToProcess = (mappedSizeInBytesCapture / bpfDeviceCapture) - mappedDeviceFramesProcessedCapture;\n                    void* pRunningMappedDeviceBufferCapture = ma_offset_ptr(pMappedDeviceBufferCapture, mappedDeviceFramesProcessedCapture * bpfDeviceCapture);\n\n                    result = ma_data_converter_process_pcm_frames(&pDevice->capture.converter, pRunningMappedDeviceBufferCapture, &deviceCapturedFramesToProcess, inputFramesInClientFormat, &clientCapturedFramesToProcess);\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n\n                    outputFramesInClientFormatCount     = (ma_uint32)clientCapturedFramesToProcess;\n                    mappedDeviceFramesProcessedCapture += (ma_uint32)deviceCapturedFramesToProcess;\n\n                    ma_device__handle_data_callback(pDevice, outputFramesInClientFormat, inputFramesInClientFormat, (ma_uint32)clientCapturedFramesToProcess);\n\n                    /* At this point we have input and output data in client format. All we need to do now is convert it to the output device format. This may take a few passes. */\n                    for (;;) {\n                        ma_uint32 framesWrittenThisIteration;\n                        DWORD physicalPlayCursorInBytes;\n                        DWORD physicalWriteCursorInBytes;\n                        DWORD availableBytesPlayback;\n                        DWORD silentPaddingInBytes = 0; /* <-- Must be initialized to 0. */\n\n                        /* We need the physical play and write cursors. */\n                        if (FAILED(ma_IDirectSoundBuffer_GetCurrentPosition((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, &physicalPlayCursorInBytes, &physicalWriteCursorInBytes))) {\n                            break;\n                        }\n\n                        if (physicalPlayCursorInBytes < prevPlayCursorInBytesPlayback) {\n                            physicalPlayCursorLoopFlagPlayback = !physicalPlayCursorLoopFlagPlayback;\n                        }\n                        prevPlayCursorInBytesPlayback  = physicalPlayCursorInBytes;\n\n                        /* If there's any bytes available for writing we can do that now. The space between the virtual cursor position and play cursor. */\n                        if (physicalPlayCursorLoopFlagPlayback == virtualWriteCursorLoopFlagPlayback) {\n                            /* Same loop iteration. The available bytes wraps all the way around from the virtual write cursor to the physical play cursor. */\n                            if (physicalPlayCursorInBytes <= virtualWriteCursorInBytesPlayback) {\n                                availableBytesPlayback  = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - virtualWriteCursorInBytesPlayback;\n                                availableBytesPlayback += physicalPlayCursorInBytes;    /* Wrap around. */\n                            } else {\n                                /* This is an error. */\n                                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[DirectSound] (Duplex/Playback): Play cursor has moved in front of the write cursor (same loop iteration). physicalPlayCursorInBytes=%ld, virtualWriteCursorInBytes=%ld.\\n\", physicalPlayCursorInBytes, virtualWriteCursorInBytesPlayback);\n                                availableBytesPlayback = 0;\n                            }\n                        } else {\n                            /* Different loop iterations. The available bytes only goes from the virtual write cursor to the physical play cursor. */\n                            if (physicalPlayCursorInBytes >= virtualWriteCursorInBytesPlayback) {\n                                availableBytesPlayback = physicalPlayCursorInBytes - virtualWriteCursorInBytesPlayback;\n                            } else {\n                                /* This is an error. */\n                                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[DirectSound] (Duplex/Playback): Write cursor has moved behind the play cursor (different loop iterations). physicalPlayCursorInBytes=%ld, virtualWriteCursorInBytes=%ld.\\n\", physicalPlayCursorInBytes, virtualWriteCursorInBytesPlayback);\n                                availableBytesPlayback = 0;\n                            }\n                        }\n\n                        /* If there's no room available for writing we need to wait for more. */\n                        if (availableBytesPlayback == 0) {\n                            /* If we haven't started the device yet, this will never get beyond 0. In this case we need to get the device started. */\n                            if (!isPlaybackDeviceStarted) {\n                                hr = ma_IDirectSoundBuffer_Play((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0, 0, MA_DSBPLAY_LOOPING);\n                                if (FAILED(hr)) {\n                                    ma_IDirectSoundCaptureBuffer_Stop((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer);\n                                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Play() failed.\");\n                                    return ma_result_from_HRESULT(hr);\n                                }\n                                isPlaybackDeviceStarted = MA_TRUE;\n                            } else {\n                                ma_sleep(waitTimeInMilliseconds);\n                                continue;\n                            }\n                        }\n\n\n                        /* Getting here means there room available somewhere. We limit this to either the end of the buffer or the physical play cursor, whichever is closest. */\n                        lockOffsetInBytesPlayback = virtualWriteCursorInBytesPlayback;\n                        if (physicalPlayCursorLoopFlagPlayback == virtualWriteCursorLoopFlagPlayback) {\n                            /* Same loop iteration. Go up to the end of the buffer. */\n                            lockSizeInBytesPlayback = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - virtualWriteCursorInBytesPlayback;\n                        } else {\n                            /* Different loop iterations. Go up to the physical play cursor. */\n                            lockSizeInBytesPlayback = physicalPlayCursorInBytes - virtualWriteCursorInBytesPlayback;\n                        }\n\n                        hr = ma_IDirectSoundBuffer_Lock((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, lockOffsetInBytesPlayback, lockSizeInBytesPlayback, &pMappedDeviceBufferPlayback, &mappedSizeInBytesPlayback, NULL, NULL, 0);\n                        if (FAILED(hr)) {\n                            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to map buffer from playback device in preparation for writing to the device.\");\n                            result = ma_result_from_HRESULT(hr);\n                            break;\n                        }\n\n                        /*\n                        Experiment: If the playback buffer is being starved, pad it with some silence to get it back in sync. This will cause a glitch, but it may prevent\n                        endless glitching due to it constantly running out of data.\n                        */\n                        if (isPlaybackDeviceStarted) {\n                            DWORD bytesQueuedForPlayback = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - availableBytesPlayback;\n                            if (bytesQueuedForPlayback < (pDevice->playback.internalPeriodSizeInFrames*bpfDevicePlayback)) {\n                                silentPaddingInBytes   = (pDevice->playback.internalPeriodSizeInFrames*2*bpfDevicePlayback) - bytesQueuedForPlayback;\n                                if (silentPaddingInBytes > lockSizeInBytesPlayback) {\n                                    silentPaddingInBytes = lockSizeInBytesPlayback;\n                                }\n\n                                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[DirectSound] (Duplex/Playback) Playback buffer starved. availableBytesPlayback=%ld, silentPaddingInBytes=%ld\\n\", availableBytesPlayback, silentPaddingInBytes);\n                            }\n                        }\n\n                        /* At this point we have a buffer for output. */\n                        if (silentPaddingInBytes > 0) {\n                            MA_ZERO_MEMORY(pMappedDeviceBufferPlayback, silentPaddingInBytes);\n                            framesWrittenThisIteration = silentPaddingInBytes/bpfDevicePlayback;\n                        } else {\n                            ma_uint64 convertedFrameCountIn  = (outputFramesInClientFormatCount - outputFramesInClientFormatConsumed);\n                            ma_uint64 convertedFrameCountOut = mappedSizeInBytesPlayback/bpfDevicePlayback;\n                            void* pConvertedFramesIn  = ma_offset_ptr(outputFramesInClientFormat, outputFramesInClientFormatConsumed * bpfDevicePlayback);\n                            void* pConvertedFramesOut = pMappedDeviceBufferPlayback;\n\n                            result = ma_data_converter_process_pcm_frames(&pDevice->playback.converter, pConvertedFramesIn, &convertedFrameCountIn, pConvertedFramesOut, &convertedFrameCountOut);\n                            if (result != MA_SUCCESS) {\n                                break;\n                            }\n\n                            outputFramesInClientFormatConsumed += (ma_uint32)convertedFrameCountOut;\n                            framesWrittenThisIteration          = (ma_uint32)convertedFrameCountOut;\n                        }\n\n\n                        hr = ma_IDirectSoundBuffer_Unlock((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, pMappedDeviceBufferPlayback, framesWrittenThisIteration*bpfDevicePlayback, NULL, 0);\n                        if (FAILED(hr)) {\n                            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to unlock internal buffer from playback device after writing to the device.\");\n                            result = ma_result_from_HRESULT(hr);\n                            break;\n                        }\n\n                        virtualWriteCursorInBytesPlayback += framesWrittenThisIteration*bpfDevicePlayback;\n                        if ((virtualWriteCursorInBytesPlayback/bpfDevicePlayback) == pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods) {\n                            virtualWriteCursorInBytesPlayback  = 0;\n                            virtualWriteCursorLoopFlagPlayback = !virtualWriteCursorLoopFlagPlayback;\n                        }\n\n                        /*\n                        We may need to start the device. We want two full periods to be written before starting the playback device. Having an extra period adds\n                        a bit of a buffer to prevent the playback buffer from getting starved.\n                        */\n                        framesWrittenToPlaybackDevice += framesWrittenThisIteration;\n                        if (!isPlaybackDeviceStarted && framesWrittenToPlaybackDevice >= (pDevice->playback.internalPeriodSizeInFrames*2)) {\n                            hr = ma_IDirectSoundBuffer_Play((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0, 0, MA_DSBPLAY_LOOPING);\n                            if (FAILED(hr)) {\n                                ma_IDirectSoundCaptureBuffer_Stop((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer);\n                                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Play() failed.\");\n                                return ma_result_from_HRESULT(hr);\n                            }\n                            isPlaybackDeviceStarted = MA_TRUE;\n                        }\n\n                        if (framesWrittenThisIteration < mappedSizeInBytesPlayback/bpfDevicePlayback) {\n                            break;  /* We're finished with the output data.*/\n                        }\n                    }\n\n                    if (clientCapturedFramesToProcess == 0) {\n                        break;  /* We just consumed every input sample. */\n                    }\n                }\n\n\n                /* At this point we're done with the mapped portion of the capture buffer. */\n                hr = ma_IDirectSoundCaptureBuffer_Unlock((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, pMappedDeviceBufferCapture, mappedSizeInBytesCapture, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to unlock internal buffer from capture device after reading from the device.\");\n                    return ma_result_from_HRESULT(hr);\n                }\n                prevReadCursorInBytesCapture = (lockOffsetInBytesCapture + mappedSizeInBytesCapture);\n            } break;\n\n\n\n            case ma_device_type_capture:\n            {\n                DWORD physicalCaptureCursorInBytes;\n                DWORD physicalReadCursorInBytes;\n                hr = ma_IDirectSoundCaptureBuffer_GetCurrentPosition((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, &physicalCaptureCursorInBytes, &physicalReadCursorInBytes);\n                if (FAILED(hr)) {\n                    return MA_ERROR;\n                }\n\n                /* If the previous capture position is the same as the current position we need to wait a bit longer. */\n                if (prevReadCursorInBytesCapture == physicalReadCursorInBytes) {\n                    ma_sleep(waitTimeInMilliseconds);\n                    continue;\n                }\n\n                /* Getting here means we have capture data available. */\n                if (prevReadCursorInBytesCapture < physicalReadCursorInBytes) {\n                    /* The capture position has not looped. This is the simple case. */\n                    lockOffsetInBytesCapture = prevReadCursorInBytesCapture;\n                    lockSizeInBytesCapture   = (physicalReadCursorInBytes - prevReadCursorInBytesCapture);\n                } else {\n                    /*\n                    The capture position has looped. This is the more complex case. Map to the end of the buffer. If this does not return anything,\n                    do it again from the start.\n                    */\n                    if (prevReadCursorInBytesCapture < pDevice->capture.internalPeriodSizeInFrames*pDevice->capture.internalPeriods*bpfDeviceCapture) {\n                        /* Lock up to the end of the buffer. */\n                        lockOffsetInBytesCapture = prevReadCursorInBytesCapture;\n                        lockSizeInBytesCapture   = (pDevice->capture.internalPeriodSizeInFrames*pDevice->capture.internalPeriods*bpfDeviceCapture) - prevReadCursorInBytesCapture;\n                    } else {\n                        /* Lock starting from the start of the buffer. */\n                        lockOffsetInBytesCapture = 0;\n                        lockSizeInBytesCapture   = physicalReadCursorInBytes;\n                    }\n                }\n\n                if (lockSizeInBytesCapture < pDevice->capture.internalPeriodSizeInFrames) {\n                    ma_sleep(waitTimeInMilliseconds);\n                    continue; /* Nothing is available in the capture buffer. */\n                }\n\n                hr = ma_IDirectSoundCaptureBuffer_Lock((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, lockOffsetInBytesCapture, lockSizeInBytesCapture, &pMappedDeviceBufferCapture, &mappedSizeInBytesCapture, NULL, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to map buffer from capture device in preparation for writing to the device.\");\n                    result = ma_result_from_HRESULT(hr);\n                }\n\n                if (lockSizeInBytesCapture != mappedSizeInBytesCapture) {\n                    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[DirectSound] (Capture) lockSizeInBytesCapture=%ld != mappedSizeInBytesCapture=%ld\\n\", lockSizeInBytesCapture, mappedSizeInBytesCapture);\n                }\n\n                ma_device__send_frames_to_client(pDevice, mappedSizeInBytesCapture/bpfDeviceCapture, pMappedDeviceBufferCapture);\n\n                hr = ma_IDirectSoundCaptureBuffer_Unlock((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer, pMappedDeviceBufferCapture, mappedSizeInBytesCapture, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to unlock internal buffer from capture device after reading from the device.\");\n                    return ma_result_from_HRESULT(hr);\n                }\n                prevReadCursorInBytesCapture = lockOffsetInBytesCapture + mappedSizeInBytesCapture;\n\n                if (prevReadCursorInBytesCapture == (pDevice->capture.internalPeriodSizeInFrames*pDevice->capture.internalPeriods*bpfDeviceCapture)) {\n                    prevReadCursorInBytesCapture = 0;\n                }\n            } break;\n\n\n\n            case ma_device_type_playback:\n            {\n                DWORD availableBytesPlayback;\n                DWORD physicalPlayCursorInBytes;\n                DWORD physicalWriteCursorInBytes;\n                hr = ma_IDirectSoundBuffer_GetCurrentPosition((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, &physicalPlayCursorInBytes, &physicalWriteCursorInBytes);\n                if (FAILED(hr)) {\n                    break;\n                }\n\n                hr = ma_IDirectSoundBuffer_GetStatus((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, &playbackBufferStatus);\n                if (SUCCEEDED(hr) && (playbackBufferStatus & MA_DSBSTATUS_PLAYING) == 0 && isPlaybackDeviceStarted) {\n                    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[DirectSound] Attempting to resume audio due to state: %d.\", (int)playbackBufferStatus);\n                    hr = ma_IDirectSoundBuffer_Play((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0, 0, MA_DSBPLAY_LOOPING);\n                    if (FAILED(hr)) {\n                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Play() failed after attempting to resume from state %d.\", (int)playbackBufferStatus);\n                        return ma_result_from_HRESULT(hr);\n                    }\n\n                    isPlaybackDeviceStarted = MA_TRUE;\n                    ma_sleep(waitTimeInMilliseconds);\n                    continue;\n                }\n\n                if (physicalPlayCursorInBytes < prevPlayCursorInBytesPlayback) {\n                    physicalPlayCursorLoopFlagPlayback = !physicalPlayCursorLoopFlagPlayback;\n                }\n                prevPlayCursorInBytesPlayback  = physicalPlayCursorInBytes;\n\n                /* If there's any bytes available for writing we can do that now. The space between the virtual cursor position and play cursor. */\n                if (physicalPlayCursorLoopFlagPlayback == virtualWriteCursorLoopFlagPlayback) {\n                    /* Same loop iteration. The available bytes wraps all the way around from the virtual write cursor to the physical play cursor. */\n                    if (physicalPlayCursorInBytes <= virtualWriteCursorInBytesPlayback) {\n                        availableBytesPlayback  = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - virtualWriteCursorInBytesPlayback;\n                        availableBytesPlayback += physicalPlayCursorInBytes;    /* Wrap around. */\n                    } else {\n                        /* This is an error. */\n                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[DirectSound] (Playback): Play cursor has moved in front of the write cursor (same loop iterations). physicalPlayCursorInBytes=%ld, virtualWriteCursorInBytes=%ld.\\n\", physicalPlayCursorInBytes, virtualWriteCursorInBytesPlayback);\n                        availableBytesPlayback = 0;\n                    }\n                } else {\n                    /* Different loop iterations. The available bytes only goes from the virtual write cursor to the physical play cursor. */\n                    if (physicalPlayCursorInBytes >= virtualWriteCursorInBytesPlayback) {\n                        availableBytesPlayback = physicalPlayCursorInBytes - virtualWriteCursorInBytesPlayback;\n                    } else {\n                        /* This is an error. */\n                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[DirectSound] (Playback): Write cursor has moved behind the play cursor (different loop iterations). physicalPlayCursorInBytes=%ld, virtualWriteCursorInBytes=%ld.\\n\", physicalPlayCursorInBytes, virtualWriteCursorInBytesPlayback);\n                        availableBytesPlayback = 0;\n                    }\n                }\n\n                /* If there's no room available for writing we need to wait for more. */\n                if (availableBytesPlayback < pDevice->playback.internalPeriodSizeInFrames) {\n                    /* If we haven't started the device yet, this will never get beyond 0. In this case we need to get the device started. */\n                    if (availableBytesPlayback == 0 && !isPlaybackDeviceStarted) {\n                        hr = ma_IDirectSoundBuffer_Play((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0, 0, MA_DSBPLAY_LOOPING);\n                        if (FAILED(hr)) {\n                            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Play() failed.\");\n                            return ma_result_from_HRESULT(hr);\n                        }\n                        isPlaybackDeviceStarted = MA_TRUE;\n                    } else {\n                        ma_sleep(waitTimeInMilliseconds);\n                        continue;\n                    }\n                }\n\n                /* Getting here means there room available somewhere. We limit this to either the end of the buffer or the physical play cursor, whichever is closest. */\n                lockOffsetInBytesPlayback = virtualWriteCursorInBytesPlayback;\n                if (physicalPlayCursorLoopFlagPlayback == virtualWriteCursorLoopFlagPlayback) {\n                    /* Same loop iteration. Go up to the end of the buffer. */\n                    lockSizeInBytesPlayback = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - virtualWriteCursorInBytesPlayback;\n                } else {\n                    /* Different loop iterations. Go up to the physical play cursor. */\n                    lockSizeInBytesPlayback = physicalPlayCursorInBytes - virtualWriteCursorInBytesPlayback;\n                }\n\n                hr = ma_IDirectSoundBuffer_Lock((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, lockOffsetInBytesPlayback, lockSizeInBytesPlayback, &pMappedDeviceBufferPlayback, &mappedSizeInBytesPlayback, NULL, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to map buffer from playback device in preparation for writing to the device.\");\n                    result = ma_result_from_HRESULT(hr);\n                    break;\n                }\n\n                /* At this point we have a buffer for output. */\n                ma_device__read_frames_from_client(pDevice, (mappedSizeInBytesPlayback/bpfDevicePlayback), pMappedDeviceBufferPlayback);\n\n                hr = ma_IDirectSoundBuffer_Unlock((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, pMappedDeviceBufferPlayback, mappedSizeInBytesPlayback, NULL, 0);\n                if (FAILED(hr)) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] Failed to unlock internal buffer from playback device after writing to the device.\");\n                    result = ma_result_from_HRESULT(hr);\n                    break;\n                }\n\n                virtualWriteCursorInBytesPlayback += mappedSizeInBytesPlayback;\n                if (virtualWriteCursorInBytesPlayback == pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) {\n                    virtualWriteCursorInBytesPlayback  = 0;\n                    virtualWriteCursorLoopFlagPlayback = !virtualWriteCursorLoopFlagPlayback;\n                }\n\n                /*\n                We may need to start the device. We want two full periods to be written before starting the playback device. Having an extra period adds\n                a bit of a buffer to prevent the playback buffer from getting starved.\n                */\n                framesWrittenToPlaybackDevice += mappedSizeInBytesPlayback/bpfDevicePlayback;\n                if (!isPlaybackDeviceStarted && framesWrittenToPlaybackDevice >= pDevice->playback.internalPeriodSizeInFrames) {\n                    hr = ma_IDirectSoundBuffer_Play((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0, 0, MA_DSBPLAY_LOOPING);\n                    if (FAILED(hr)) {\n                        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Play() failed.\");\n                        return ma_result_from_HRESULT(hr);\n                    }\n                    isPlaybackDeviceStarted = MA_TRUE;\n                }\n            } break;\n\n\n            default: return MA_INVALID_ARGS;   /* Invalid device type. */\n        }\n\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    /* Getting here means the device is being stopped. */\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        hr = ma_IDirectSoundCaptureBuffer_Stop((ma_IDirectSoundCaptureBuffer*)pDevice->dsound.pCaptureBuffer);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundCaptureBuffer_Stop() failed.\");\n            return ma_result_from_HRESULT(hr);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        /* The playback device should be drained before stopping. All we do is wait until the available bytes is equal to the size of the buffer. */\n        if (isPlaybackDeviceStarted) {\n            for (;;) {\n                DWORD availableBytesPlayback = 0;\n                DWORD physicalPlayCursorInBytes;\n                DWORD physicalWriteCursorInBytes;\n                hr = ma_IDirectSoundBuffer_GetCurrentPosition((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, &physicalPlayCursorInBytes, &physicalWriteCursorInBytes);\n                if (FAILED(hr)) {\n                    break;\n                }\n\n                if (physicalPlayCursorInBytes < prevPlayCursorInBytesPlayback) {\n                    physicalPlayCursorLoopFlagPlayback = !physicalPlayCursorLoopFlagPlayback;\n                }\n                prevPlayCursorInBytesPlayback  = physicalPlayCursorInBytes;\n\n                if (physicalPlayCursorLoopFlagPlayback == virtualWriteCursorLoopFlagPlayback) {\n                    /* Same loop iteration. The available bytes wraps all the way around from the virtual write cursor to the physical play cursor. */\n                    if (physicalPlayCursorInBytes <= virtualWriteCursorInBytesPlayback) {\n                        availableBytesPlayback  = (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback) - virtualWriteCursorInBytesPlayback;\n                        availableBytesPlayback += physicalPlayCursorInBytes;    /* Wrap around. */\n                    } else {\n                        break;\n                    }\n                } else {\n                    /* Different loop iterations. The available bytes only goes from the virtual write cursor to the physical play cursor. */\n                    if (physicalPlayCursorInBytes >= virtualWriteCursorInBytesPlayback) {\n                        availableBytesPlayback = physicalPlayCursorInBytes - virtualWriteCursorInBytesPlayback;\n                    } else {\n                        break;\n                    }\n                }\n\n                if (availableBytesPlayback >= (pDevice->playback.internalPeriodSizeInFrames*pDevice->playback.internalPeriods*bpfDevicePlayback)) {\n                    break;\n                }\n\n                ma_sleep(waitTimeInMilliseconds);\n            }\n        }\n\n        hr = ma_IDirectSoundBuffer_Stop((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer);\n        if (FAILED(hr)) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[DirectSound] IDirectSoundBuffer_Stop() failed.\");\n            return ma_result_from_HRESULT(hr);\n        }\n\n        ma_IDirectSoundBuffer_SetCurrentPosition((ma_IDirectSoundBuffer*)pDevice->dsound.pPlaybackBuffer, 0);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__dsound(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_dsound);\n\n    ma_dlclose(ma_context_get_log(pContext), pContext->dsound.hDSoundDLL);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__dsound(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n    pContext->dsound.hDSoundDLL = ma_dlopen(ma_context_get_log(pContext), \"dsound.dll\");\n    if (pContext->dsound.hDSoundDLL == NULL) {\n        return MA_API_NOT_FOUND;\n    }\n\n    pContext->dsound.DirectSoundCreate            = ma_dlsym(ma_context_get_log(pContext), pContext->dsound.hDSoundDLL, \"DirectSoundCreate\");\n    pContext->dsound.DirectSoundEnumerateA        = ma_dlsym(ma_context_get_log(pContext), pContext->dsound.hDSoundDLL, \"DirectSoundEnumerateA\");\n    pContext->dsound.DirectSoundCaptureCreate     = ma_dlsym(ma_context_get_log(pContext), pContext->dsound.hDSoundDLL, \"DirectSoundCaptureCreate\");\n    pContext->dsound.DirectSoundCaptureEnumerateA = ma_dlsym(ma_context_get_log(pContext), pContext->dsound.hDSoundDLL, \"DirectSoundCaptureEnumerateA\");\n\n    /*\n    We need to support all functions or nothing. DirectSound with Windows 95 seems to not work too\n    well in my testing. For example, it's missing DirectSoundCaptureEnumerateA(). This is a convenient\n    place to just disable the DirectSound backend for Windows 95.\n    */\n    if (pContext->dsound.DirectSoundCreate            == NULL ||\n        pContext->dsound.DirectSoundEnumerateA        == NULL ||\n        pContext->dsound.DirectSoundCaptureCreate     == NULL ||\n        pContext->dsound.DirectSoundCaptureEnumerateA == NULL) {\n        return MA_API_NOT_FOUND;\n    }\n\n    pContext->dsound.hWnd = pConfig->dsound.hWnd;\n\n    pCallbacks->onContextInit             = ma_context_init__dsound;\n    pCallbacks->onContextUninit           = ma_context_uninit__dsound;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__dsound;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__dsound;\n    pCallbacks->onDeviceInit              = ma_device_init__dsound;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__dsound;\n    pCallbacks->onDeviceStart             = NULL;   /* Not used. Started in onDeviceDataLoop. */\n    pCallbacks->onDeviceStop              = NULL;   /* Not used. Stopped in onDeviceDataLoop. */\n    pCallbacks->onDeviceRead              = NULL;   /* Not used. Data is read directly in onDeviceDataLoop. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not used. Data is written directly in onDeviceDataLoop. */\n    pCallbacks->onDeviceDataLoop          = ma_device_data_loop__dsound;\n\n    return MA_SUCCESS;\n}\n#endif\n\n\n\n/******************************************************************************\n\nWinMM Backend\n\n******************************************************************************/\n#ifdef MA_HAS_WINMM\n\n/*\nSome build configurations will exclude the WinMM API. An example is when WIN32_LEAN_AND_MEAN\nis defined. We need to define the types and functions we need manually.\n*/\n#define MA_MMSYSERR_NOERROR     0\n#define MA_MMSYSERR_ERROR       1\n#define MA_MMSYSERR_BADDEVICEID 2\n#define MA_MMSYSERR_INVALHANDLE 5\n#define MA_MMSYSERR_NOMEM       7\n#define MA_MMSYSERR_INVALFLAG   10\n#define MA_MMSYSERR_INVALPARAM  11\n#define MA_MMSYSERR_HANDLEBUSY  12\n\n#define MA_CALLBACK_EVENT       0x00050000\n#define MA_WAVE_ALLOWSYNC       0x0002\n\n#define MA_WHDR_DONE            0x00000001\n#define MA_WHDR_PREPARED        0x00000002\n#define MA_WHDR_BEGINLOOP       0x00000004\n#define MA_WHDR_ENDLOOP         0x00000008\n#define MA_WHDR_INQUEUE         0x00000010\n\n#define MA_MAXPNAMELEN          32\n\ntypedef void* MA_HWAVEIN;\ntypedef void* MA_HWAVEOUT;\ntypedef UINT MA_MMRESULT;\ntypedef UINT MA_MMVERSION;\n\ntypedef struct\n{\n    WORD wMid;\n    WORD wPid;\n    MA_MMVERSION vDriverVersion;\n    CHAR szPname[MA_MAXPNAMELEN];\n    DWORD dwFormats;\n    WORD wChannels;\n    WORD wReserved1;\n} MA_WAVEINCAPSA;\n\ntypedef struct\n{\n    WORD wMid;\n    WORD wPid;\n    MA_MMVERSION vDriverVersion;\n    CHAR szPname[MA_MAXPNAMELEN];\n    DWORD dwFormats;\n    WORD wChannels;\n    WORD wReserved1;\n    DWORD dwSupport;\n} MA_WAVEOUTCAPSA;\n\ntypedef struct tagWAVEHDR\n{\n    char* lpData;\n    DWORD dwBufferLength;\n    DWORD dwBytesRecorded;\n    DWORD_PTR dwUser;\n    DWORD dwFlags;\n    DWORD dwLoops;\n    struct tagWAVEHDR* lpNext;\n    DWORD_PTR reserved;\n} MA_WAVEHDR;\n\ntypedef struct\n{\n    WORD wMid;\n    WORD wPid;\n    MA_MMVERSION vDriverVersion;\n    CHAR szPname[MA_MAXPNAMELEN];\n    DWORD dwFormats;\n    WORD wChannels;\n    WORD wReserved1;\n    DWORD dwSupport;\n    GUID ManufacturerGuid;\n    GUID ProductGuid;\n    GUID NameGuid;\n} MA_WAVEOUTCAPS2A;\n\ntypedef struct\n{\n    WORD wMid;\n    WORD wPid;\n    MA_MMVERSION vDriverVersion;\n    CHAR szPname[MA_MAXPNAMELEN];\n    DWORD dwFormats;\n    WORD wChannels;\n    WORD wReserved1;\n    GUID ManufacturerGuid;\n    GUID ProductGuid;\n    GUID NameGuid;\n} MA_WAVEINCAPS2A;\n\ntypedef UINT        (WINAPI * MA_PFN_waveOutGetNumDevs)(void);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutGetDevCapsA)(ma_uintptr uDeviceID, MA_WAVEOUTCAPSA* pwoc, UINT cbwoc);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutOpen)(MA_HWAVEOUT* phwo, UINT uDeviceID, const MA_WAVEFORMATEX* pwfx, DWORD_PTR dwCallback, DWORD_PTR dwInstance, DWORD fdwOpen);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutClose)(MA_HWAVEOUT hwo);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutPrepareHeader)(MA_HWAVEOUT hwo, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutUnprepareHeader)(MA_HWAVEOUT hwo, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutWrite)(MA_HWAVEOUT hwo, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveOutReset)(MA_HWAVEOUT hwo);\ntypedef UINT        (WINAPI * MA_PFN_waveInGetNumDevs)(void);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInGetDevCapsA)(ma_uintptr uDeviceID, MA_WAVEINCAPSA* pwic, UINT cbwic);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInOpen)(MA_HWAVEIN* phwi, UINT uDeviceID, const MA_WAVEFORMATEX* pwfx, DWORD_PTR dwCallback, DWORD_PTR dwInstance, DWORD fdwOpen);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInClose)(MA_HWAVEIN hwi);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInPrepareHeader)(MA_HWAVEIN hwi, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInUnprepareHeader)(MA_HWAVEIN hwi, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInAddBuffer)(MA_HWAVEIN hwi, MA_WAVEHDR* pwh, UINT cbwh);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInStart)(MA_HWAVEIN hwi);\ntypedef MA_MMRESULT (WINAPI * MA_PFN_waveInReset)(MA_HWAVEIN hwi);\n\nstatic ma_result ma_result_from_MMRESULT(MA_MMRESULT resultMM)\n{\n    switch (resultMM)\n    {\n        case MA_MMSYSERR_NOERROR:       return MA_SUCCESS;\n        case MA_MMSYSERR_BADDEVICEID:   return MA_INVALID_ARGS;\n        case MA_MMSYSERR_INVALHANDLE:   return MA_INVALID_ARGS;\n        case MA_MMSYSERR_NOMEM:         return MA_OUT_OF_MEMORY;\n        case MA_MMSYSERR_INVALFLAG:     return MA_INVALID_ARGS;\n        case MA_MMSYSERR_INVALPARAM:    return MA_INVALID_ARGS;\n        case MA_MMSYSERR_HANDLEBUSY:    return MA_BUSY;\n        case MA_MMSYSERR_ERROR:         return MA_ERROR;\n        default:                        return MA_ERROR;\n    }\n}\n\nstatic char* ma_find_last_character(char* str, char ch)\n{\n    char* last;\n\n    if (str == NULL) {\n        return NULL;\n    }\n\n    last = NULL;\n    while (*str != '\\0') {\n        if (*str == ch) {\n            last = str;\n        }\n\n        str += 1;\n    }\n\n    return last;\n}\n\nstatic ma_uint32 ma_get_period_size_in_bytes(ma_uint32 periodSizeInFrames, ma_format format, ma_uint32 channels)\n{\n    return periodSizeInFrames * ma_get_bytes_per_frame(format, channels);\n}\n\n\n/*\nOur own \"WAVECAPS\" structure that contains generic information shared between WAVEOUTCAPS2 and WAVEINCAPS2 so\nwe can do things generically and typesafely. Names are being kept the same for consistency.\n*/\ntypedef struct\n{\n    CHAR szPname[MA_MAXPNAMELEN];\n    DWORD dwFormats;\n    WORD wChannels;\n    GUID NameGuid;\n} MA_WAVECAPSA;\n\nstatic ma_result ma_get_best_info_from_formats_flags__winmm(DWORD dwFormats, WORD channels, WORD* pBitsPerSample, DWORD* pSampleRate)\n{\n    WORD bitsPerSample = 0;\n    DWORD sampleRate = 0;\n\n    if (pBitsPerSample) {\n        *pBitsPerSample = 0;\n    }\n    if (pSampleRate) {\n        *pSampleRate = 0;\n    }\n\n    if (channels == 1) {\n        bitsPerSample = 16;\n        if ((dwFormats & WAVE_FORMAT_48M16) != 0) {\n            sampleRate = 48000;\n        } else if ((dwFormats & WAVE_FORMAT_44M16) != 0) {\n            sampleRate = 44100;\n        } else if ((dwFormats & WAVE_FORMAT_2M16) != 0) {\n            sampleRate = 22050;\n        } else if ((dwFormats & WAVE_FORMAT_1M16) != 0) {\n            sampleRate = 11025;\n        } else if ((dwFormats & WAVE_FORMAT_96M16) != 0) {\n            sampleRate = 96000;\n        } else {\n            bitsPerSample = 8;\n            if ((dwFormats & WAVE_FORMAT_48M08) != 0) {\n                sampleRate = 48000;\n            } else if ((dwFormats & WAVE_FORMAT_44M08) != 0) {\n                sampleRate = 44100;\n            } else if ((dwFormats & WAVE_FORMAT_2M08) != 0) {\n                sampleRate = 22050;\n            } else if ((dwFormats & WAVE_FORMAT_1M08) != 0) {\n                sampleRate = 11025;\n            } else if ((dwFormats & WAVE_FORMAT_96M08) != 0) {\n                sampleRate = 96000;\n            } else {\n                return MA_FORMAT_NOT_SUPPORTED;\n            }\n        }\n    } else {\n        bitsPerSample = 16;\n        if ((dwFormats & WAVE_FORMAT_48S16) != 0) {\n            sampleRate = 48000;\n        } else if ((dwFormats & WAVE_FORMAT_44S16) != 0) {\n            sampleRate = 44100;\n        } else if ((dwFormats & WAVE_FORMAT_2S16) != 0) {\n            sampleRate = 22050;\n        } else if ((dwFormats & WAVE_FORMAT_1S16) != 0) {\n            sampleRate = 11025;\n        } else if ((dwFormats & WAVE_FORMAT_96S16) != 0) {\n            sampleRate = 96000;\n        } else {\n            bitsPerSample = 8;\n            if ((dwFormats & WAVE_FORMAT_48S08) != 0) {\n                sampleRate = 48000;\n            } else if ((dwFormats & WAVE_FORMAT_44S08) != 0) {\n                sampleRate = 44100;\n            } else if ((dwFormats & WAVE_FORMAT_2S08) != 0) {\n                sampleRate = 22050;\n            } else if ((dwFormats & WAVE_FORMAT_1S08) != 0) {\n                sampleRate = 11025;\n            } else if ((dwFormats & WAVE_FORMAT_96S08) != 0) {\n                sampleRate = 96000;\n            } else {\n                return MA_FORMAT_NOT_SUPPORTED;\n            }\n        }\n    }\n\n    if (pBitsPerSample) {\n        *pBitsPerSample = bitsPerSample;\n    }\n    if (pSampleRate) {\n        *pSampleRate = sampleRate;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_formats_flags_to_WAVEFORMATEX__winmm(DWORD dwFormats, WORD channels, MA_WAVEFORMATEX* pWF)\n{\n    ma_result result;\n\n    MA_ASSERT(pWF != NULL);\n\n    MA_ZERO_OBJECT(pWF);\n    pWF->cbSize     = sizeof(*pWF);\n    pWF->wFormatTag = WAVE_FORMAT_PCM;\n    pWF->nChannels  = (WORD)channels;\n    if (pWF->nChannels > 2) {\n        pWF->nChannels = 2;\n    }\n\n    result = ma_get_best_info_from_formats_flags__winmm(dwFormats, channels, &pWF->wBitsPerSample, &pWF->nSamplesPerSec);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pWF->nBlockAlign     = (WORD)(pWF->nChannels * pWF->wBitsPerSample / 8);\n    pWF->nAvgBytesPerSec = pWF->nBlockAlign * pWF->nSamplesPerSec;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info_from_WAVECAPS(ma_context* pContext, MA_WAVECAPSA* pCaps, ma_device_info* pDeviceInfo)\n{\n    WORD bitsPerSample;\n    DWORD sampleRate;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pCaps != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    /*\n    Name / Description\n\n    Unfortunately the name specified in WAVE(OUT/IN)CAPS2 is limited to 31 characters. This results in an unprofessional looking\n    situation where the names of the devices are truncated. To help work around this, we need to look at the name GUID and try\n    looking in the registry for the full name. If we can't find it there, we need to just fall back to the default name.\n    */\n\n    /* Set the default to begin with. */\n    ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), pCaps->szPname, (size_t)-1);\n\n    /*\n    Now try the registry. There's a few things to consider here:\n    - The name GUID can be null, in which we case we just need to stick to the original 31 characters.\n    - If the name GUID is not present in the registry we'll also need to stick to the original 31 characters.\n    - I like consistency, so I want the returned device names to be consistent with those returned by WASAPI and DirectSound. The\n      problem, however is that WASAPI and DirectSound use \"<component> (<name>)\" format (such as \"Speakers (High Definition Audio)\"),\n      but WinMM does not specify the component name. From my admittedly limited testing, I've notice the component name seems to\n      usually fit within the 31 characters of the fixed sized buffer, so what I'm going to do is parse that string for the component\n      name, and then concatenate the name from the registry.\n    */\n    if (!ma_is_guid_null(&pCaps->NameGuid)) {\n        WCHAR guidStrW[256];\n        if (((MA_PFN_StringFromGUID2)pContext->win32.StringFromGUID2)(&pCaps->NameGuid, guidStrW, ma_countof(guidStrW)) > 0) {\n            char guidStr[256];\n            char keyStr[1024];\n            HKEY hKey;\n\n            WideCharToMultiByte(CP_UTF8, 0, guidStrW, -1, guidStr, sizeof(guidStr), 0, FALSE);\n\n            ma_strcpy_s(keyStr, sizeof(keyStr), \"SYSTEM\\\\CurrentControlSet\\\\Control\\\\MediaCategories\\\\\");\n            ma_strcat_s(keyStr, sizeof(keyStr), guidStr);\n\n            if (((MA_PFN_RegOpenKeyExA)pContext->win32.RegOpenKeyExA)(HKEY_LOCAL_MACHINE, keyStr, 0, KEY_READ, &hKey) == ERROR_SUCCESS) {\n                BYTE nameFromReg[512];\n                DWORD nameFromRegSize = sizeof(nameFromReg);\n                LONG resultWin32 = ((MA_PFN_RegQueryValueExA)pContext->win32.RegQueryValueExA)(hKey, \"Name\", 0, NULL, (BYTE*)nameFromReg, (DWORD*)&nameFromRegSize);\n                ((MA_PFN_RegCloseKey)pContext->win32.RegCloseKey)(hKey);\n\n                if (resultWin32 == ERROR_SUCCESS) {\n                    /* We have the value from the registry, so now we need to construct the name string. */\n                    char name[1024];\n                    if (ma_strcpy_s(name, sizeof(name), pDeviceInfo->name) == 0) {\n                        char* nameBeg = ma_find_last_character(name, '(');\n                        if (nameBeg != NULL) {\n                            size_t leadingLen = (nameBeg - name);\n                            ma_strncpy_s(nameBeg + 1, sizeof(name) - leadingLen, (const char*)nameFromReg, (size_t)-1);\n\n                            /* The closing \")\", if it can fit. */\n                            if (leadingLen + nameFromRegSize < sizeof(name)-1) {\n                                ma_strcat_s(name, sizeof(name), \")\");\n                            }\n\n                            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), name, (size_t)-1);\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n\n    result = ma_get_best_info_from_formats_flags__winmm(pCaps->dwFormats, pCaps->wChannels, &bitsPerSample, &sampleRate);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (bitsPerSample == 8) {\n        pDeviceInfo->nativeDataFormats[0].format = ma_format_u8;\n    } else if (bitsPerSample == 16) {\n        pDeviceInfo->nativeDataFormats[0].format = ma_format_s16;\n    } else if (bitsPerSample == 24) {\n        pDeviceInfo->nativeDataFormats[0].format = ma_format_s24;\n    } else if (bitsPerSample == 32) {\n        pDeviceInfo->nativeDataFormats[0].format = ma_format_s32;\n    } else {\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n    pDeviceInfo->nativeDataFormats[0].channels   = pCaps->wChannels;\n    pDeviceInfo->nativeDataFormats[0].sampleRate = sampleRate;\n    pDeviceInfo->nativeDataFormats[0].flags      = 0;\n    pDeviceInfo->nativeDataFormatCount = 1;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info_from_WAVEOUTCAPS2(ma_context* pContext, MA_WAVEOUTCAPS2A* pCaps, ma_device_info* pDeviceInfo)\n{\n    MA_WAVECAPSA caps;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pCaps != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    MA_COPY_MEMORY(caps.szPname, pCaps->szPname, sizeof(caps.szPname));\n    caps.dwFormats = pCaps->dwFormats;\n    caps.wChannels = pCaps->wChannels;\n    caps.NameGuid  = pCaps->NameGuid;\n    return ma_context_get_device_info_from_WAVECAPS(pContext, &caps, pDeviceInfo);\n}\n\nstatic ma_result ma_context_get_device_info_from_WAVEINCAPS2(ma_context* pContext, MA_WAVEINCAPS2A* pCaps, ma_device_info* pDeviceInfo)\n{\n    MA_WAVECAPSA caps;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pCaps != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    MA_COPY_MEMORY(caps.szPname, pCaps->szPname, sizeof(caps.szPname));\n    caps.dwFormats = pCaps->dwFormats;\n    caps.wChannels = pCaps->wChannels;\n    caps.NameGuid  = pCaps->NameGuid;\n    return ma_context_get_device_info_from_WAVECAPS(pContext, &caps, pDeviceInfo);\n}\n\n\nstatic ma_result ma_context_enumerate_devices__winmm(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    UINT playbackDeviceCount;\n    UINT captureDeviceCount;\n    UINT iPlaybackDevice;\n    UINT iCaptureDevice;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Playback. */\n    playbackDeviceCount = ((MA_PFN_waveOutGetNumDevs)pContext->winmm.waveOutGetNumDevs)();\n    for (iPlaybackDevice = 0; iPlaybackDevice < playbackDeviceCount; ++iPlaybackDevice) {\n        MA_MMRESULT result;\n        MA_WAVEOUTCAPS2A caps;\n\n        MA_ZERO_OBJECT(&caps);\n\n        result = ((MA_PFN_waveOutGetDevCapsA)pContext->winmm.waveOutGetDevCapsA)(iPlaybackDevice, (MA_WAVEOUTCAPSA*)&caps, sizeof(caps));\n        if (result == MA_MMSYSERR_NOERROR) {\n            ma_device_info deviceInfo;\n\n            MA_ZERO_OBJECT(&deviceInfo);\n            deviceInfo.id.winmm = iPlaybackDevice;\n\n            /* The first enumerated device is the default device. */\n            if (iPlaybackDevice == 0) {\n                deviceInfo.isDefault = MA_TRUE;\n            }\n\n            if (ma_context_get_device_info_from_WAVEOUTCAPS2(pContext, &caps, &deviceInfo) == MA_SUCCESS) {\n                ma_bool32 cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n                if (cbResult == MA_FALSE) {\n                    return MA_SUCCESS; /* Enumeration was stopped. */\n                }\n            }\n        }\n    }\n\n    /* Capture. */\n    captureDeviceCount = ((MA_PFN_waveInGetNumDevs)pContext->winmm.waveInGetNumDevs)();\n    for (iCaptureDevice = 0; iCaptureDevice < captureDeviceCount; ++iCaptureDevice) {\n        MA_MMRESULT result;\n        MA_WAVEINCAPS2A caps;\n\n        MA_ZERO_OBJECT(&caps);\n\n        result = ((MA_PFN_waveInGetDevCapsA)pContext->winmm.waveInGetDevCapsA)(iCaptureDevice, (MA_WAVEINCAPSA*)&caps, sizeof(caps));\n        if (result == MA_MMSYSERR_NOERROR) {\n            ma_device_info deviceInfo;\n\n            MA_ZERO_OBJECT(&deviceInfo);\n            deviceInfo.id.winmm = iCaptureDevice;\n\n            /* The first enumerated device is the default device. */\n            if (iCaptureDevice == 0) {\n                deviceInfo.isDefault = MA_TRUE;\n            }\n\n            if (ma_context_get_device_info_from_WAVEINCAPS2(pContext, &caps, &deviceInfo) == MA_SUCCESS) {\n                ma_bool32 cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n                if (cbResult == MA_FALSE) {\n                    return MA_SUCCESS; /* Enumeration was stopped. */\n                }\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__winmm(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    UINT winMMDeviceID;\n\n    MA_ASSERT(pContext != NULL);\n\n    winMMDeviceID = 0;\n    if (pDeviceID != NULL) {\n        winMMDeviceID = (UINT)pDeviceID->winmm;\n    }\n\n    pDeviceInfo->id.winmm = winMMDeviceID;\n\n    /* The first ID is the default device. */\n    if (winMMDeviceID == 0) {\n        pDeviceInfo->isDefault = MA_TRUE;\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        MA_MMRESULT result;\n        MA_WAVEOUTCAPS2A caps;\n\n        MA_ZERO_OBJECT(&caps);\n\n        result = ((MA_PFN_waveOutGetDevCapsA)pContext->winmm.waveOutGetDevCapsA)(winMMDeviceID, (MA_WAVEOUTCAPSA*)&caps, sizeof(caps));\n        if (result == MA_MMSYSERR_NOERROR) {\n            return ma_context_get_device_info_from_WAVEOUTCAPS2(pContext, &caps, pDeviceInfo);\n        }\n    } else {\n        MA_MMRESULT result;\n        MA_WAVEINCAPS2A caps;\n\n        MA_ZERO_OBJECT(&caps);\n\n        result = ((MA_PFN_waveInGetDevCapsA)pContext->winmm.waveInGetDevCapsA)(winMMDeviceID, (MA_WAVEINCAPSA*)&caps, sizeof(caps));\n        if (result == MA_MMSYSERR_NOERROR) {\n            return ma_context_get_device_info_from_WAVEINCAPS2(pContext, &caps, pDeviceInfo);\n        }\n    }\n\n    return MA_NO_DEVICE;\n}\n\n\nstatic ma_result ma_device_uninit__winmm(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ((MA_PFN_waveInClose)pDevice->pContext->winmm.waveInClose)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture);\n        CloseHandle((HANDLE)pDevice->winmm.hEventCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ((MA_PFN_waveOutReset)pDevice->pContext->winmm.waveOutReset)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback);\n        ((MA_PFN_waveOutClose)pDevice->pContext->winmm.waveOutClose)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback);\n        CloseHandle((HANDLE)pDevice->winmm.hEventPlayback);\n    }\n\n    ma_free(pDevice->winmm._pHeapData, &pDevice->pContext->allocationCallbacks);\n\n    MA_ZERO_OBJECT(&pDevice->winmm);   /* Safety. */\n\n    return MA_SUCCESS;\n}\n\nstatic ma_uint32 ma_calculate_period_size_in_frames_from_descriptor__winmm(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile)\n{\n    /* WinMM has a minimum period size of 40ms. */\n    ma_uint32 minPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(40, nativeSampleRate);\n    ma_uint32 periodSizeInFrames;\n\n    periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, nativeSampleRate, performanceProfile);\n    if (periodSizeInFrames < minPeriodSizeInFrames) {\n        periodSizeInFrames = minPeriodSizeInFrames;\n    }\n\n    return periodSizeInFrames;\n}\n\nstatic ma_result ma_device_init__winmm(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    const char* errorMsg = \"\";\n    ma_result errorCode = MA_ERROR;\n    ma_result result = MA_SUCCESS;\n    ma_uint32 heapSize;\n    UINT winMMDeviceIDPlayback = 0;\n    UINT winMMDeviceIDCapture  = 0;\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->winmm);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* No exclusive mode with WinMM. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    if (pDescriptorPlayback->pDeviceID != NULL) {\n        winMMDeviceIDPlayback = (UINT)pDescriptorPlayback->pDeviceID->winmm;\n    }\n    if (pDescriptorCapture->pDeviceID != NULL) {\n        winMMDeviceIDCapture = (UINT)pDescriptorCapture->pDeviceID->winmm;\n    }\n\n    /* The capture device needs to be initialized first. */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        MA_WAVEINCAPSA caps;\n        MA_WAVEFORMATEX wf;\n        MA_MMRESULT resultMM;\n\n        /* We use an event to know when a new fragment needs to be enqueued. */\n        pDevice->winmm.hEventCapture = (ma_handle)CreateEventA(NULL, TRUE, TRUE, NULL);\n        if (pDevice->winmm.hEventCapture == NULL) {\n            errorMsg = \"[WinMM] Failed to create event for fragment enqueuing for the capture device.\", errorCode = ma_result_from_GetLastError(GetLastError());\n            goto on_error;\n        }\n\n        /* The format should be based on the device's actual format. */\n        if (((MA_PFN_waveInGetDevCapsA)pDevice->pContext->winmm.waveInGetDevCapsA)(winMMDeviceIDCapture, &caps, sizeof(caps)) != MA_MMSYSERR_NOERROR) {\n            errorMsg = \"[WinMM] Failed to retrieve internal device caps.\", errorCode = MA_FORMAT_NOT_SUPPORTED;\n            goto on_error;\n        }\n\n        result = ma_formats_flags_to_WAVEFORMATEX__winmm(caps.dwFormats, caps.wChannels, &wf);\n        if (result != MA_SUCCESS) {\n            errorMsg = \"[WinMM] Could not find appropriate format for internal device.\", errorCode = result;\n            goto on_error;\n        }\n\n        resultMM = ((MA_PFN_waveInOpen)pDevice->pContext->winmm.waveInOpen)((MA_HWAVEIN*)&pDevice->winmm.hDeviceCapture, winMMDeviceIDCapture, &wf, (DWORD_PTR)pDevice->winmm.hEventCapture, (DWORD_PTR)pDevice, MA_CALLBACK_EVENT | MA_WAVE_ALLOWSYNC);\n        if (resultMM != MA_MMSYSERR_NOERROR) {\n            errorMsg = \"[WinMM] Failed to open capture device.\", errorCode = MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n            goto on_error;\n        }\n\n        pDescriptorCapture->format             = ma_format_from_WAVEFORMATEX(&wf);\n        pDescriptorCapture->channels           = wf.nChannels;\n        pDescriptorCapture->sampleRate         = wf.nSamplesPerSec;\n        ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorCapture->channels);\n        pDescriptorCapture->periodCount        = pDescriptorCapture->periodCount;\n        pDescriptorCapture->periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__winmm(pDescriptorCapture, pDescriptorCapture->sampleRate, pConfig->performanceProfile);\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        MA_WAVEOUTCAPSA caps;\n        MA_WAVEFORMATEX wf;\n        MA_MMRESULT resultMM;\n\n        /* We use an event to know when a new fragment needs to be enqueued. */\n        pDevice->winmm.hEventPlayback = (ma_handle)CreateEventA(NULL, TRUE, TRUE, NULL);\n        if (pDevice->winmm.hEventPlayback == NULL) {\n            errorMsg = \"[WinMM] Failed to create event for fragment enqueuing for the playback device.\", errorCode = ma_result_from_GetLastError(GetLastError());\n            goto on_error;\n        }\n\n        /* The format should be based on the device's actual format. */\n        if (((MA_PFN_waveOutGetDevCapsA)pDevice->pContext->winmm.waveOutGetDevCapsA)(winMMDeviceIDPlayback, &caps, sizeof(caps)) != MA_MMSYSERR_NOERROR) {\n            errorMsg = \"[WinMM] Failed to retrieve internal device caps.\", errorCode = MA_FORMAT_NOT_SUPPORTED;\n            goto on_error;\n        }\n\n        result = ma_formats_flags_to_WAVEFORMATEX__winmm(caps.dwFormats, caps.wChannels, &wf);\n        if (result != MA_SUCCESS) {\n            errorMsg = \"[WinMM] Could not find appropriate format for internal device.\", errorCode = result;\n            goto on_error;\n        }\n\n        resultMM = ((MA_PFN_waveOutOpen)pDevice->pContext->winmm.waveOutOpen)((MA_HWAVEOUT*)&pDevice->winmm.hDevicePlayback, winMMDeviceIDPlayback, &wf, (DWORD_PTR)pDevice->winmm.hEventPlayback, (DWORD_PTR)pDevice, MA_CALLBACK_EVENT | MA_WAVE_ALLOWSYNC);\n        if (resultMM != MA_MMSYSERR_NOERROR) {\n            errorMsg = \"[WinMM] Failed to open playback device.\", errorCode = MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n            goto on_error;\n        }\n\n        pDescriptorPlayback->format             = ma_format_from_WAVEFORMATEX(&wf);\n        pDescriptorPlayback->channels           = wf.nChannels;\n        pDescriptorPlayback->sampleRate         = wf.nSamplesPerSec;\n        ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pDescriptorPlayback->channelMap, ma_countof(pDescriptorPlayback->channelMap), pDescriptorPlayback->channels);\n        pDescriptorPlayback->periodCount        = pDescriptorPlayback->periodCount;\n        pDescriptorPlayback->periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__winmm(pDescriptorPlayback, pDescriptorPlayback->sampleRate, pConfig->performanceProfile);\n    }\n\n    /*\n    The heap allocated data is allocated like so:\n\n    [Capture WAVEHDRs][Playback WAVEHDRs][Capture Intermediary Buffer][Playback Intermediary Buffer]\n    */\n    heapSize = 0;\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        heapSize += sizeof(MA_WAVEHDR)*pDescriptorCapture->periodCount + (pDescriptorCapture->periodSizeInFrames * pDescriptorCapture->periodCount * ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels));\n    }\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        heapSize += sizeof(MA_WAVEHDR)*pDescriptorPlayback->periodCount + (pDescriptorPlayback->periodSizeInFrames * pDescriptorPlayback->periodCount * ma_get_bytes_per_frame(pDescriptorPlayback->format, pDescriptorPlayback->channels));\n    }\n\n    pDevice->winmm._pHeapData = (ma_uint8*)ma_calloc(heapSize, &pDevice->pContext->allocationCallbacks);\n    if (pDevice->winmm._pHeapData == NULL) {\n        errorMsg = \"[WinMM] Failed to allocate memory for the intermediary buffer.\", errorCode = MA_OUT_OF_MEMORY;\n        goto on_error;\n    }\n\n    MA_ZERO_MEMORY(pDevice->winmm._pHeapData, heapSize);\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_uint32 iPeriod;\n\n        if (pConfig->deviceType == ma_device_type_capture) {\n            pDevice->winmm.pWAVEHDRCapture            = pDevice->winmm._pHeapData;\n            pDevice->winmm.pIntermediaryBufferCapture = pDevice->winmm._pHeapData + (sizeof(MA_WAVEHDR)*(pDescriptorCapture->periodCount));\n        } else {\n            pDevice->winmm.pWAVEHDRCapture            = pDevice->winmm._pHeapData;\n            pDevice->winmm.pIntermediaryBufferCapture = pDevice->winmm._pHeapData + (sizeof(MA_WAVEHDR)*(pDescriptorCapture->periodCount + pDescriptorPlayback->periodCount));\n        }\n\n        /* Prepare headers. */\n        for (iPeriod = 0; iPeriod < pDescriptorCapture->periodCount; ++iPeriod) {\n            ma_uint32 periodSizeInBytes = ma_get_period_size_in_bytes(pDescriptorCapture->periodSizeInFrames, pDescriptorCapture->format, pDescriptorCapture->channels);\n\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod].lpData         = (char*)(pDevice->winmm.pIntermediaryBufferCapture + (periodSizeInBytes*iPeriod));\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod].dwBufferLength = periodSizeInBytes;\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod].dwFlags        = 0L;\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod].dwLoops        = 0L;\n            ((MA_PFN_waveInPrepareHeader)pDevice->pContext->winmm.waveInPrepareHeader)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod], sizeof(MA_WAVEHDR));\n\n            /*\n            The user data of the MA_WAVEHDR structure is a single flag the controls whether or not it is ready for writing. Consider it to be named \"isLocked\". A value of 0 means\n            it's unlocked and available for writing. A value of 1 means it's locked.\n            */\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod].dwUser = 0;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_uint32 iPeriod;\n\n        if (pConfig->deviceType == ma_device_type_playback) {\n            pDevice->winmm.pWAVEHDRPlayback            = pDevice->winmm._pHeapData;\n            pDevice->winmm.pIntermediaryBufferPlayback = pDevice->winmm._pHeapData + (sizeof(MA_WAVEHDR)*pDescriptorPlayback->periodCount);\n        } else {\n            pDevice->winmm.pWAVEHDRPlayback            = pDevice->winmm._pHeapData + (sizeof(MA_WAVEHDR)*(pDescriptorCapture->periodCount));\n            pDevice->winmm.pIntermediaryBufferPlayback = pDevice->winmm._pHeapData + (sizeof(MA_WAVEHDR)*(pDescriptorCapture->periodCount + pDescriptorPlayback->periodCount)) + (pDescriptorCapture->periodSizeInFrames*pDescriptorCapture->periodCount*ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels));\n        }\n\n        /* Prepare headers. */\n        for (iPeriod = 0; iPeriod < pDescriptorPlayback->periodCount; ++iPeriod) {\n            ma_uint32 periodSizeInBytes = ma_get_period_size_in_bytes(pDescriptorPlayback->periodSizeInFrames, pDescriptorPlayback->format, pDescriptorPlayback->channels);\n\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod].lpData         = (char*)(pDevice->winmm.pIntermediaryBufferPlayback + (periodSizeInBytes*iPeriod));\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod].dwBufferLength = periodSizeInBytes;\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod].dwFlags        = 0L;\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod].dwLoops        = 0L;\n            ((MA_PFN_waveOutPrepareHeader)pDevice->pContext->winmm.waveOutPrepareHeader)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod], sizeof(MA_WAVEHDR));\n\n            /*\n            The user data of the MA_WAVEHDR structure is a single flag the controls whether or not it is ready for writing. Consider it to be named \"isLocked\". A value of 0 means\n            it's unlocked and available for writing. A value of 1 means it's locked.\n            */\n            ((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod].dwUser = 0;\n        }\n    }\n\n    return MA_SUCCESS;\n\non_error:\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->winmm.pWAVEHDRCapture != NULL) {\n            ma_uint32 iPeriod;\n            for (iPeriod = 0; iPeriod < pDescriptorCapture->periodCount; ++iPeriod) {\n                ((MA_PFN_waveInUnprepareHeader)pDevice->pContext->winmm.waveInUnprepareHeader)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod], sizeof(MA_WAVEHDR));\n            }\n        }\n\n        ((MA_PFN_waveInClose)pDevice->pContext->winmm.waveInClose)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->winmm.pWAVEHDRCapture != NULL) {\n            ma_uint32 iPeriod;\n            for (iPeriod = 0; iPeriod < pDescriptorPlayback->periodCount; ++iPeriod) {\n                ((MA_PFN_waveOutUnprepareHeader)pDevice->pContext->winmm.waveOutUnprepareHeader)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback)[iPeriod], sizeof(MA_WAVEHDR));\n            }\n        }\n\n        ((MA_PFN_waveOutClose)pDevice->pContext->winmm.waveOutClose)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback);\n    }\n\n    ma_free(pDevice->winmm._pHeapData, &pDevice->pContext->allocationCallbacks);\n\n    if (errorMsg != NULL && errorMsg[0] != '\\0') {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"%s\", errorMsg);\n    }\n\n    return errorCode;\n}\n\nstatic ma_result ma_device_start__winmm(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        MA_MMRESULT resultMM;\n        MA_WAVEHDR* pWAVEHDR;\n        ma_uint32 iPeriod;\n\n        pWAVEHDR = (MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture;\n\n        /* Make sure the event is reset to a non-signaled state to ensure we don't prematurely return from WaitForSingleObject(). */\n        ResetEvent((HANDLE)pDevice->winmm.hEventCapture);\n\n        /* To start the device we attach all of the buffers and then start it. As the buffers are filled with data we will get notifications. */\n        for (iPeriod = 0; iPeriod < pDevice->capture.internalPeriods; ++iPeriod) {\n            resultMM = ((MA_PFN_waveInAddBuffer)pDevice->pContext->winmm.waveInAddBuffer)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[iPeriod], sizeof(MA_WAVEHDR));\n            if (resultMM != MA_MMSYSERR_NOERROR) {\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WinMM] Failed to attach input buffers to capture device in preparation for capture.\");\n                return ma_result_from_MMRESULT(resultMM);\n            }\n\n            /* Make sure all of the buffers start out locked. We don't want to access them until the backend tells us we can. */\n            pWAVEHDR[iPeriod].dwUser = 1;   /* 1 = locked. */\n        }\n\n        /* Capture devices need to be explicitly started, unlike playback devices. */\n        resultMM = ((MA_PFN_waveInStart)pDevice->pContext->winmm.waveInStart)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture);\n        if (resultMM != MA_MMSYSERR_NOERROR) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WinMM] Failed to start backend device.\");\n            return ma_result_from_MMRESULT(resultMM);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        /* Don't need to do anything for playback. It'll be started automatically in ma_device_start__winmm(). */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__winmm(ma_device* pDevice)\n{\n    MA_MMRESULT resultMM;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->winmm.hDeviceCapture == NULL) {\n            return MA_INVALID_ARGS;\n        }\n\n        resultMM = ((MA_PFN_waveInReset)pDevice->pContext->winmm.waveInReset)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture);\n        if (resultMM != MA_MMSYSERR_NOERROR) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[WinMM] WARNING: Failed to reset capture device.\");\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_uint32 iPeriod;\n        MA_WAVEHDR* pWAVEHDR;\n\n        if (pDevice->winmm.hDevicePlayback == NULL) {\n            return MA_INVALID_ARGS;\n        }\n\n        /* We need to drain the device. To do this we just loop over each header and if it's locked just wait for the event. */\n        pWAVEHDR = (MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback;\n        for (iPeriod = 0; iPeriod < pDevice->playback.internalPeriods; iPeriod += 1) {\n            if (pWAVEHDR[iPeriod].dwUser == 1) { /* 1 = locked. */\n                if (WaitForSingleObject((HANDLE)pDevice->winmm.hEventPlayback, INFINITE) != WAIT_OBJECT_0) {\n                    break;  /* An error occurred so just abandon ship and stop the device without draining. */\n                }\n\n                pWAVEHDR[iPeriod].dwUser = 0;\n            }\n        }\n\n        resultMM = ((MA_PFN_waveOutReset)pDevice->pContext->winmm.waveOutReset)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback);\n        if (resultMM != MA_MMSYSERR_NOERROR) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[WinMM] WARNING: Failed to reset playback device.\");\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_write__winmm(ma_device* pDevice, const void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    ma_result result = MA_SUCCESS;\n    MA_MMRESULT resultMM;\n    ma_uint32 totalFramesWritten;\n    MA_WAVEHDR* pWAVEHDR;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pPCMFrames != NULL);\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    pWAVEHDR = (MA_WAVEHDR*)pDevice->winmm.pWAVEHDRPlayback;\n\n    /* Keep processing as much data as possible. */\n    totalFramesWritten = 0;\n    while (totalFramesWritten < frameCount) {\n        /* If the current header has some space available we need to write part of it. */\n        if (pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwUser == 0) { /* 0 = unlocked. */\n            /*\n            This header has room in it. We copy as much of it as we can. If we end up fully consuming the buffer we need to\n            write it out and move on to the next iteration.\n            */\n            ma_uint32 bpf = ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n            ma_uint32 framesRemainingInHeader = (pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwBufferLength/bpf) - pDevice->winmm.headerFramesConsumedPlayback;\n\n            ma_uint32 framesToCopy = ma_min(framesRemainingInHeader, (frameCount - totalFramesWritten));\n            const void* pSrc = ma_offset_ptr(pPCMFrames, totalFramesWritten*bpf);\n            void* pDst = ma_offset_ptr(pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].lpData, pDevice->winmm.headerFramesConsumedPlayback*bpf);\n            MA_COPY_MEMORY(pDst, pSrc, framesToCopy*bpf);\n\n            pDevice->winmm.headerFramesConsumedPlayback += framesToCopy;\n            totalFramesWritten += framesToCopy;\n\n            /* If we've consumed the buffer entirely we need to write it out to the device. */\n            if (pDevice->winmm.headerFramesConsumedPlayback == (pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwBufferLength/bpf)) {\n                pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwUser = 1;            /* 1 = locked. */\n                pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwFlags &= ~MA_WHDR_DONE; /* <-- Need to make sure the WHDR_DONE flag is unset. */\n\n                /* Make sure the event is reset to a non-signaled state to ensure we don't prematurely return from WaitForSingleObject(). */\n                ResetEvent((HANDLE)pDevice->winmm.hEventPlayback);\n\n                /* The device will be started here. */\n                resultMM = ((MA_PFN_waveOutWrite)pDevice->pContext->winmm.waveOutWrite)((MA_HWAVEOUT)pDevice->winmm.hDevicePlayback, &pWAVEHDR[pDevice->winmm.iNextHeaderPlayback], sizeof(MA_WAVEHDR));\n                if (resultMM != MA_MMSYSERR_NOERROR) {\n                    result = ma_result_from_MMRESULT(resultMM);\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WinMM] waveOutWrite() failed.\");\n                    break;\n                }\n\n                /* Make sure we move to the next header. */\n                pDevice->winmm.iNextHeaderPlayback = (pDevice->winmm.iNextHeaderPlayback + 1) % pDevice->playback.internalPeriods;\n                pDevice->winmm.headerFramesConsumedPlayback = 0;\n            }\n\n            /* If at this point we have consumed the entire input buffer we can return. */\n            MA_ASSERT(totalFramesWritten <= frameCount);\n            if (totalFramesWritten == frameCount) {\n                break;\n            }\n\n            /* Getting here means there's more to process. */\n            continue;\n        }\n\n        /* Getting here means there isn't enough room in the buffer and we need to wait for one to become available. */\n        if (WaitForSingleObject((HANDLE)pDevice->winmm.hEventPlayback, INFINITE) != WAIT_OBJECT_0) {\n            result = MA_ERROR;\n            break;\n        }\n\n        /* Something happened. If the next buffer has been marked as done we need to reset a bit of state. */\n        if ((pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwFlags & MA_WHDR_DONE) != 0) {\n            pWAVEHDR[pDevice->winmm.iNextHeaderPlayback].dwUser = 0;    /* 0 = unlocked (make it available for writing). */\n            pDevice->winmm.headerFramesConsumedPlayback = 0;\n        }\n\n        /* If the device has been stopped we need to break. */\n        if (ma_device_get_state(pDevice) != ma_device_state_started) {\n            break;\n        }\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = totalFramesWritten;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_device_read__winmm(ma_device* pDevice, void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    MA_MMRESULT resultMM;\n    ma_uint32 totalFramesRead;\n    MA_WAVEHDR* pWAVEHDR;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pPCMFrames != NULL);\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    pWAVEHDR = (MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture;\n\n    /* Keep processing as much data as possible. */\n    totalFramesRead = 0;\n    while (totalFramesRead < frameCount) {\n        /* If the current header has some space available we need to write part of it. */\n        if (pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwUser == 0) { /* 0 = unlocked. */\n            /* The buffer is available for reading. If we fully consume it we need to add it back to the buffer. */\n            ma_uint32 bpf = ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n            ma_uint32 framesRemainingInHeader = (pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwBufferLength/bpf) - pDevice->winmm.headerFramesConsumedCapture;\n\n            ma_uint32 framesToCopy = ma_min(framesRemainingInHeader, (frameCount - totalFramesRead));\n            const void* pSrc = ma_offset_ptr(pWAVEHDR[pDevice->winmm.iNextHeaderCapture].lpData, pDevice->winmm.headerFramesConsumedCapture*bpf);\n            void* pDst = ma_offset_ptr(pPCMFrames, totalFramesRead*bpf);\n            MA_COPY_MEMORY(pDst, pSrc, framesToCopy*bpf);\n\n            pDevice->winmm.headerFramesConsumedCapture += framesToCopy;\n            totalFramesRead += framesToCopy;\n\n            /* If we've consumed the buffer entirely we need to add it back to the device. */\n            if (pDevice->winmm.headerFramesConsumedCapture == (pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwBufferLength/bpf)) {\n                pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwUser = 1;            /* 1 = locked. */\n                pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwFlags &= ~MA_WHDR_DONE; /* <-- Need to make sure the WHDR_DONE flag is unset. */\n\n                /* Make sure the event is reset to a non-signaled state to ensure we don't prematurely return from WaitForSingleObject(). */\n                ResetEvent((HANDLE)pDevice->winmm.hEventCapture);\n\n                /* The device will be started here. */\n                resultMM = ((MA_PFN_waveInAddBuffer)pDevice->pContext->winmm.waveInAddBuffer)((MA_HWAVEIN)pDevice->winmm.hDeviceCapture, &((MA_WAVEHDR*)pDevice->winmm.pWAVEHDRCapture)[pDevice->winmm.iNextHeaderCapture], sizeof(MA_WAVEHDR));\n                if (resultMM != MA_MMSYSERR_NOERROR) {\n                    result = ma_result_from_MMRESULT(resultMM);\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[WinMM] waveInAddBuffer() failed.\");\n                    break;\n                }\n\n                /* Make sure we move to the next header. */\n                pDevice->winmm.iNextHeaderCapture = (pDevice->winmm.iNextHeaderCapture + 1) % pDevice->capture.internalPeriods;\n                pDevice->winmm.headerFramesConsumedCapture = 0;\n            }\n\n            /* If at this point we have filled the entire input buffer we can return. */\n            MA_ASSERT(totalFramesRead <= frameCount);\n            if (totalFramesRead == frameCount) {\n                break;\n            }\n\n            /* Getting here means there's more to process. */\n            continue;\n        }\n\n        /* Getting here means there isn't enough any data left to send to the client which means we need to wait for more. */\n        if (WaitForSingleObject((HANDLE)pDevice->winmm.hEventCapture, INFINITE) != WAIT_OBJECT_0) {\n            result = MA_ERROR;\n            break;\n        }\n\n        /* Something happened. If the next buffer has been marked as done we need to reset a bit of state. */\n        if ((pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwFlags & MA_WHDR_DONE) != 0) {\n            pWAVEHDR[pDevice->winmm.iNextHeaderCapture].dwUser = 0;    /* 0 = unlocked (make it available for reading). */\n            pDevice->winmm.headerFramesConsumedCapture = 0;\n        }\n\n        /* If the device has been stopped we need to break. */\n        if (ma_device_get_state(pDevice) != ma_device_state_started) {\n            break;\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesRead;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_context_uninit__winmm(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_winmm);\n\n    ma_dlclose(ma_context_get_log(pContext), pContext->winmm.hWinMM);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__winmm(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n    pContext->winmm.hWinMM = ma_dlopen(ma_context_get_log(pContext), \"winmm.dll\");\n    if (pContext->winmm.hWinMM == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    pContext->winmm.waveOutGetNumDevs      = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutGetNumDevs\");\n    pContext->winmm.waveOutGetDevCapsA     = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutGetDevCapsA\");\n    pContext->winmm.waveOutOpen            = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutOpen\");\n    pContext->winmm.waveOutClose           = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutClose\");\n    pContext->winmm.waveOutPrepareHeader   = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutPrepareHeader\");\n    pContext->winmm.waveOutUnprepareHeader = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutUnprepareHeader\");\n    pContext->winmm.waveOutWrite           = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutWrite\");\n    pContext->winmm.waveOutReset           = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveOutReset\");\n    pContext->winmm.waveInGetNumDevs       = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInGetNumDevs\");\n    pContext->winmm.waveInGetDevCapsA      = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInGetDevCapsA\");\n    pContext->winmm.waveInOpen             = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInOpen\");\n    pContext->winmm.waveInClose            = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInClose\");\n    pContext->winmm.waveInPrepareHeader    = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInPrepareHeader\");\n    pContext->winmm.waveInUnprepareHeader  = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInUnprepareHeader\");\n    pContext->winmm.waveInAddBuffer        = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInAddBuffer\");\n    pContext->winmm.waveInStart            = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInStart\");\n    pContext->winmm.waveInReset            = ma_dlsym(ma_context_get_log(pContext), pContext->winmm.hWinMM, \"waveInReset\");\n\n    pCallbacks->onContextInit             = ma_context_init__winmm;\n    pCallbacks->onContextUninit           = ma_context_uninit__winmm;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__winmm;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__winmm;\n    pCallbacks->onDeviceInit              = ma_device_init__winmm;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__winmm;\n    pCallbacks->onDeviceStart             = ma_device_start__winmm;\n    pCallbacks->onDeviceStop              = ma_device_stop__winmm;\n    pCallbacks->onDeviceRead              = ma_device_read__winmm;\n    pCallbacks->onDeviceWrite             = ma_device_write__winmm;\n    pCallbacks->onDeviceDataLoop          = NULL;   /* This is a blocking read-write API, so this can be NULL since miniaudio will manage the audio thread for us. */\n\n    return MA_SUCCESS;\n}\n#endif\n\n\n\n\n/******************************************************************************\n\nALSA Backend\n\n******************************************************************************/\n#ifdef MA_HAS_ALSA\n\n#include <poll.h>           /* poll(), struct pollfd */\n#include <sys/eventfd.h>    /* eventfd() */\n\n#ifdef MA_NO_RUNTIME_LINKING\n\n/* asoundlib.h marks some functions with \"inline\" which isn't always supported. Need to emulate it. */\n#if !defined(__cplusplus)\n    #if defined(__STRICT_ANSI__)\n        #if !defined(inline)\n            #define inline __inline__ __attribute__((always_inline))\n            #define MA_INLINE_DEFINED\n        #endif\n    #endif\n#endif\n#include <alsa/asoundlib.h>\n#if defined(MA_INLINE_DEFINED)\n    #undef inline\n    #undef MA_INLINE_DEFINED\n#endif\n\ntypedef snd_pcm_uframes_t                       ma_snd_pcm_uframes_t;\ntypedef snd_pcm_sframes_t                       ma_snd_pcm_sframes_t;\ntypedef snd_pcm_stream_t                        ma_snd_pcm_stream_t;\ntypedef snd_pcm_format_t                        ma_snd_pcm_format_t;\ntypedef snd_pcm_access_t                        ma_snd_pcm_access_t;\ntypedef snd_pcm_t                               ma_snd_pcm_t;\ntypedef snd_pcm_hw_params_t                     ma_snd_pcm_hw_params_t;\ntypedef snd_pcm_sw_params_t                     ma_snd_pcm_sw_params_t;\ntypedef snd_pcm_format_mask_t                   ma_snd_pcm_format_mask_t;\ntypedef snd_pcm_info_t                          ma_snd_pcm_info_t;\ntypedef snd_pcm_channel_area_t                  ma_snd_pcm_channel_area_t;\ntypedef snd_pcm_chmap_t                         ma_snd_pcm_chmap_t;\ntypedef snd_pcm_state_t                         ma_snd_pcm_state_t;\n\n/* snd_pcm_state_t */\n#define MA_SND_PCM_STATE_XRUN                   SND_PCM_STATE_XRUN\n\n/* snd_pcm_stream_t */\n#define MA_SND_PCM_STREAM_PLAYBACK              SND_PCM_STREAM_PLAYBACK\n#define MA_SND_PCM_STREAM_CAPTURE               SND_PCM_STREAM_CAPTURE\n\n/* snd_pcm_format_t */\n#define MA_SND_PCM_FORMAT_UNKNOWN               SND_PCM_FORMAT_UNKNOWN\n#define MA_SND_PCM_FORMAT_U8                    SND_PCM_FORMAT_U8\n#define MA_SND_PCM_FORMAT_S16_LE                SND_PCM_FORMAT_S16_LE\n#define MA_SND_PCM_FORMAT_S16_BE                SND_PCM_FORMAT_S16_BE\n#define MA_SND_PCM_FORMAT_S24_LE                SND_PCM_FORMAT_S24_LE\n#define MA_SND_PCM_FORMAT_S24_BE                SND_PCM_FORMAT_S24_BE\n#define MA_SND_PCM_FORMAT_S32_LE                SND_PCM_FORMAT_S32_LE\n#define MA_SND_PCM_FORMAT_S32_BE                SND_PCM_FORMAT_S32_BE\n#define MA_SND_PCM_FORMAT_FLOAT_LE              SND_PCM_FORMAT_FLOAT_LE\n#define MA_SND_PCM_FORMAT_FLOAT_BE              SND_PCM_FORMAT_FLOAT_BE\n#define MA_SND_PCM_FORMAT_FLOAT64_LE            SND_PCM_FORMAT_FLOAT64_LE\n#define MA_SND_PCM_FORMAT_FLOAT64_BE            SND_PCM_FORMAT_FLOAT64_BE\n#define MA_SND_PCM_FORMAT_MU_LAW                SND_PCM_FORMAT_MU_LAW\n#define MA_SND_PCM_FORMAT_A_LAW                 SND_PCM_FORMAT_A_LAW\n#define MA_SND_PCM_FORMAT_S24_3LE               SND_PCM_FORMAT_S24_3LE\n#define MA_SND_PCM_FORMAT_S24_3BE               SND_PCM_FORMAT_S24_3BE\n\n/* ma_snd_pcm_access_t */\n#define MA_SND_PCM_ACCESS_MMAP_INTERLEAVED      SND_PCM_ACCESS_MMAP_INTERLEAVED\n#define MA_SND_PCM_ACCESS_MMAP_NONINTERLEAVED   SND_PCM_ACCESS_MMAP_NONINTERLEAVED\n#define MA_SND_PCM_ACCESS_MMAP_COMPLEX          SND_PCM_ACCESS_MMAP_COMPLEX\n#define MA_SND_PCM_ACCESS_RW_INTERLEAVED        SND_PCM_ACCESS_RW_INTERLEAVED\n#define MA_SND_PCM_ACCESS_RW_NONINTERLEAVED     SND_PCM_ACCESS_RW_NONINTERLEAVED\n\n/* Channel positions. */\n#define MA_SND_CHMAP_UNKNOWN                    SND_CHMAP_UNKNOWN\n#define MA_SND_CHMAP_NA                         SND_CHMAP_NA\n#define MA_SND_CHMAP_MONO                       SND_CHMAP_MONO\n#define MA_SND_CHMAP_FL                         SND_CHMAP_FL\n#define MA_SND_CHMAP_FR                         SND_CHMAP_FR\n#define MA_SND_CHMAP_RL                         SND_CHMAP_RL\n#define MA_SND_CHMAP_RR                         SND_CHMAP_RR\n#define MA_SND_CHMAP_FC                         SND_CHMAP_FC\n#define MA_SND_CHMAP_LFE                        SND_CHMAP_LFE\n#define MA_SND_CHMAP_SL                         SND_CHMAP_SL\n#define MA_SND_CHMAP_SR                         SND_CHMAP_SR\n#define MA_SND_CHMAP_RC                         SND_CHMAP_RC\n#define MA_SND_CHMAP_FLC                        SND_CHMAP_FLC\n#define MA_SND_CHMAP_FRC                        SND_CHMAP_FRC\n#define MA_SND_CHMAP_RLC                        SND_CHMAP_RLC\n#define MA_SND_CHMAP_RRC                        SND_CHMAP_RRC\n#define MA_SND_CHMAP_FLW                        SND_CHMAP_FLW\n#define MA_SND_CHMAP_FRW                        SND_CHMAP_FRW\n#define MA_SND_CHMAP_FLH                        SND_CHMAP_FLH\n#define MA_SND_CHMAP_FCH                        SND_CHMAP_FCH\n#define MA_SND_CHMAP_FRH                        SND_CHMAP_FRH\n#define MA_SND_CHMAP_TC                         SND_CHMAP_TC\n#define MA_SND_CHMAP_TFL                        SND_CHMAP_TFL\n#define MA_SND_CHMAP_TFR                        SND_CHMAP_TFR\n#define MA_SND_CHMAP_TFC                        SND_CHMAP_TFC\n#define MA_SND_CHMAP_TRL                        SND_CHMAP_TRL\n#define MA_SND_CHMAP_TRR                        SND_CHMAP_TRR\n#define MA_SND_CHMAP_TRC                        SND_CHMAP_TRC\n#define MA_SND_CHMAP_TFLC                       SND_CHMAP_TFLC\n#define MA_SND_CHMAP_TFRC                       SND_CHMAP_TFRC\n#define MA_SND_CHMAP_TSL                        SND_CHMAP_TSL\n#define MA_SND_CHMAP_TSR                        SND_CHMAP_TSR\n#define MA_SND_CHMAP_LLFE                       SND_CHMAP_LLFE\n#define MA_SND_CHMAP_RLFE                       SND_CHMAP_RLFE\n#define MA_SND_CHMAP_BC                         SND_CHMAP_BC\n#define MA_SND_CHMAP_BLC                        SND_CHMAP_BLC\n#define MA_SND_CHMAP_BRC                        SND_CHMAP_BRC\n\n/* Open mode flags. */\n#define MA_SND_PCM_NO_AUTO_RESAMPLE             SND_PCM_NO_AUTO_RESAMPLE\n#define MA_SND_PCM_NO_AUTO_CHANNELS             SND_PCM_NO_AUTO_CHANNELS\n#define MA_SND_PCM_NO_AUTO_FORMAT               SND_PCM_NO_AUTO_FORMAT\n#else\n#include <errno.h>  /* For EPIPE, etc. */\ntypedef unsigned long                           ma_snd_pcm_uframes_t;\ntypedef long                                    ma_snd_pcm_sframes_t;\ntypedef int                                     ma_snd_pcm_stream_t;\ntypedef int                                     ma_snd_pcm_format_t;\ntypedef int                                     ma_snd_pcm_access_t;\ntypedef int                                     ma_snd_pcm_state_t;\ntypedef struct ma_snd_pcm_t                     ma_snd_pcm_t;\ntypedef struct ma_snd_pcm_hw_params_t           ma_snd_pcm_hw_params_t;\ntypedef struct ma_snd_pcm_sw_params_t           ma_snd_pcm_sw_params_t;\ntypedef struct ma_snd_pcm_format_mask_t         ma_snd_pcm_format_mask_t;\ntypedef struct ma_snd_pcm_info_t                ma_snd_pcm_info_t;\ntypedef struct\n{\n    void* addr;\n    unsigned int first;\n    unsigned int step;\n} ma_snd_pcm_channel_area_t;\ntypedef struct\n{\n    unsigned int channels;\n    unsigned int pos[1];\n} ma_snd_pcm_chmap_t;\n\n/* snd_pcm_state_t */\n#define MA_SND_PCM_STATE_OPEN                  0\n#define MA_SND_PCM_STATE_SETUP                 1\n#define MA_SND_PCM_STATE_PREPARED              2\n#define MA_SND_PCM_STATE_RUNNING               3\n#define MA_SND_PCM_STATE_XRUN                  4\n#define MA_SND_PCM_STATE_DRAINING              5\n#define MA_SND_PCM_STATE_PAUSED                6\n#define MA_SND_PCM_STATE_SUSPENDED             7\n#define MA_SND_PCM_STATE_DISCONNECTED          8\n\n/* snd_pcm_stream_t */\n#define MA_SND_PCM_STREAM_PLAYBACK             0\n#define MA_SND_PCM_STREAM_CAPTURE              1\n\n/* snd_pcm_format_t */\n#define MA_SND_PCM_FORMAT_UNKNOWN              -1\n#define MA_SND_PCM_FORMAT_U8                   1\n#define MA_SND_PCM_FORMAT_S16_LE               2\n#define MA_SND_PCM_FORMAT_S16_BE               3\n#define MA_SND_PCM_FORMAT_S24_LE               6\n#define MA_SND_PCM_FORMAT_S24_BE               7\n#define MA_SND_PCM_FORMAT_S32_LE               10\n#define MA_SND_PCM_FORMAT_S32_BE               11\n#define MA_SND_PCM_FORMAT_FLOAT_LE             14\n#define MA_SND_PCM_FORMAT_FLOAT_BE             15\n#define MA_SND_PCM_FORMAT_FLOAT64_LE           16\n#define MA_SND_PCM_FORMAT_FLOAT64_BE           17\n#define MA_SND_PCM_FORMAT_MU_LAW               20\n#define MA_SND_PCM_FORMAT_A_LAW                21\n#define MA_SND_PCM_FORMAT_S24_3LE              32\n#define MA_SND_PCM_FORMAT_S24_3BE              33\n\n/* snd_pcm_access_t */\n#define MA_SND_PCM_ACCESS_MMAP_INTERLEAVED     0\n#define MA_SND_PCM_ACCESS_MMAP_NONINTERLEAVED  1\n#define MA_SND_PCM_ACCESS_MMAP_COMPLEX         2\n#define MA_SND_PCM_ACCESS_RW_INTERLEAVED       3\n#define MA_SND_PCM_ACCESS_RW_NONINTERLEAVED    4\n\n/* Channel positions. */\n#define MA_SND_CHMAP_UNKNOWN                   0\n#define MA_SND_CHMAP_NA                        1\n#define MA_SND_CHMAP_MONO                      2\n#define MA_SND_CHMAP_FL                        3\n#define MA_SND_CHMAP_FR                        4\n#define MA_SND_CHMAP_RL                        5\n#define MA_SND_CHMAP_RR                        6\n#define MA_SND_CHMAP_FC                        7\n#define MA_SND_CHMAP_LFE                       8\n#define MA_SND_CHMAP_SL                        9\n#define MA_SND_CHMAP_SR                        10\n#define MA_SND_CHMAP_RC                        11\n#define MA_SND_CHMAP_FLC                       12\n#define MA_SND_CHMAP_FRC                       13\n#define MA_SND_CHMAP_RLC                       14\n#define MA_SND_CHMAP_RRC                       15\n#define MA_SND_CHMAP_FLW                       16\n#define MA_SND_CHMAP_FRW                       17\n#define MA_SND_CHMAP_FLH                       18\n#define MA_SND_CHMAP_FCH                       19\n#define MA_SND_CHMAP_FRH                       20\n#define MA_SND_CHMAP_TC                        21\n#define MA_SND_CHMAP_TFL                       22\n#define MA_SND_CHMAP_TFR                       23\n#define MA_SND_CHMAP_TFC                       24\n#define MA_SND_CHMAP_TRL                       25\n#define MA_SND_CHMAP_TRR                       26\n#define MA_SND_CHMAP_TRC                       27\n#define MA_SND_CHMAP_TFLC                      28\n#define MA_SND_CHMAP_TFRC                      29\n#define MA_SND_CHMAP_TSL                       30\n#define MA_SND_CHMAP_TSR                       31\n#define MA_SND_CHMAP_LLFE                      32\n#define MA_SND_CHMAP_RLFE                      33\n#define MA_SND_CHMAP_BC                        34\n#define MA_SND_CHMAP_BLC                       35\n#define MA_SND_CHMAP_BRC                       36\n\n/* Open mode flags. */\n#define MA_SND_PCM_NO_AUTO_RESAMPLE            0x00010000\n#define MA_SND_PCM_NO_AUTO_CHANNELS            0x00020000\n#define MA_SND_PCM_NO_AUTO_FORMAT              0x00040000\n#endif\n\ntypedef int                  (* ma_snd_pcm_open_proc)                          (ma_snd_pcm_t **pcm, const char *name, ma_snd_pcm_stream_t stream, int mode);\ntypedef int                  (* ma_snd_pcm_close_proc)                         (ma_snd_pcm_t *pcm);\ntypedef size_t               (* ma_snd_pcm_hw_params_sizeof_proc)              (void);\ntypedef int                  (* ma_snd_pcm_hw_params_any_proc)                 (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params);\ntypedef int                  (* ma_snd_pcm_hw_params_set_format_proc)          (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, ma_snd_pcm_format_t val);\ntypedef int                  (* ma_snd_pcm_hw_params_set_format_first_proc)    (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, ma_snd_pcm_format_t *format);\ntypedef void                 (* ma_snd_pcm_hw_params_get_format_mask_proc)     (ma_snd_pcm_hw_params_t *params, ma_snd_pcm_format_mask_t *mask);\ntypedef int                  (* ma_snd_pcm_hw_params_set_channels_proc)        (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int val);\ntypedef int                  (* ma_snd_pcm_hw_params_set_channels_near_proc)   (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int *val);\ntypedef int                  (* ma_snd_pcm_hw_params_set_channels_minmax_proc) (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int *minimum, unsigned int *maximum);\ntypedef int                  (* ma_snd_pcm_hw_params_set_rate_resample_proc)   (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int val);\ntypedef int                  (* ma_snd_pcm_hw_params_set_rate_proc)            (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int val, int dir);\ntypedef int                  (* ma_snd_pcm_hw_params_set_rate_near_proc)       (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int *val, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_set_rate_minmax_proc)     (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int *min, int *mindir, unsigned int *max, int *maxdir);\ntypedef int                  (* ma_snd_pcm_hw_params_set_buffer_size_near_proc)(ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, ma_snd_pcm_uframes_t *val);\ntypedef int                  (* ma_snd_pcm_hw_params_set_periods_near_proc)    (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int *val, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_set_access_proc)          (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, ma_snd_pcm_access_t _access);\ntypedef int                  (* ma_snd_pcm_hw_params_get_format_proc)          (const ma_snd_pcm_hw_params_t *params, ma_snd_pcm_format_t *format);\ntypedef int                  (* ma_snd_pcm_hw_params_get_channels_proc)        (const ma_snd_pcm_hw_params_t *params, unsigned int *val);\ntypedef int                  (* ma_snd_pcm_hw_params_get_channels_min_proc)    (const ma_snd_pcm_hw_params_t *params, unsigned int *val);\ntypedef int                  (* ma_snd_pcm_hw_params_get_channels_max_proc)    (const ma_snd_pcm_hw_params_t *params, unsigned int *val);\ntypedef int                  (* ma_snd_pcm_hw_params_get_rate_proc)            (const ma_snd_pcm_hw_params_t *params, unsigned int *rate, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_get_rate_min_proc)        (const ma_snd_pcm_hw_params_t *params, unsigned int *rate, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_get_rate_max_proc)        (const ma_snd_pcm_hw_params_t *params, unsigned int *rate, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_get_buffer_size_proc)     (const ma_snd_pcm_hw_params_t *params, ma_snd_pcm_uframes_t *val);\ntypedef int                  (* ma_snd_pcm_hw_params_get_periods_proc)         (const ma_snd_pcm_hw_params_t *params, unsigned int *val, int *dir);\ntypedef int                  (* ma_snd_pcm_hw_params_get_access_proc)          (const ma_snd_pcm_hw_params_t *params, ma_snd_pcm_access_t *_access);\ntypedef int                  (* ma_snd_pcm_hw_params_test_format_proc)         (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, ma_snd_pcm_format_t val);\ntypedef int                  (* ma_snd_pcm_hw_params_test_channels_proc)       (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int val);\ntypedef int                  (* ma_snd_pcm_hw_params_test_rate_proc)           (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params, unsigned int val, int dir);\ntypedef int                  (* ma_snd_pcm_hw_params_proc)                     (ma_snd_pcm_t *pcm, ma_snd_pcm_hw_params_t *params);\ntypedef size_t               (* ma_snd_pcm_sw_params_sizeof_proc)              (void);\ntypedef int                  (* ma_snd_pcm_sw_params_current_proc)             (ma_snd_pcm_t *pcm, ma_snd_pcm_sw_params_t *params);\ntypedef int                  (* ma_snd_pcm_sw_params_get_boundary_proc)        (const ma_snd_pcm_sw_params_t *params, ma_snd_pcm_uframes_t* val);\ntypedef int                  (* ma_snd_pcm_sw_params_set_avail_min_proc)       (ma_snd_pcm_t *pcm, ma_snd_pcm_sw_params_t *params, ma_snd_pcm_uframes_t val);\ntypedef int                  (* ma_snd_pcm_sw_params_set_start_threshold_proc) (ma_snd_pcm_t *pcm, ma_snd_pcm_sw_params_t *params, ma_snd_pcm_uframes_t val);\ntypedef int                  (* ma_snd_pcm_sw_params_set_stop_threshold_proc)  (ma_snd_pcm_t *pcm, ma_snd_pcm_sw_params_t *params, ma_snd_pcm_uframes_t val);\ntypedef int                  (* ma_snd_pcm_sw_params_proc)                     (ma_snd_pcm_t *pcm, ma_snd_pcm_sw_params_t *params);\ntypedef size_t               (* ma_snd_pcm_format_mask_sizeof_proc)            (void);\ntypedef int                  (* ma_snd_pcm_format_mask_test_proc)              (const ma_snd_pcm_format_mask_t *mask, ma_snd_pcm_format_t val);\ntypedef ma_snd_pcm_chmap_t * (* ma_snd_pcm_get_chmap_proc)                     (ma_snd_pcm_t *pcm);\ntypedef ma_snd_pcm_state_t   (* ma_snd_pcm_state_proc)                         (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_prepare_proc)                       (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_start_proc)                         (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_drop_proc)                          (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_drain_proc)                         (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_reset_proc)                         (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_device_name_hint_proc)                  (int card, const char *iface, void ***hints);\ntypedef char *               (* ma_snd_device_name_get_hint_proc)              (const void *hint, const char *id);\ntypedef int                  (* ma_snd_card_get_index_proc)                    (const char *name);\ntypedef int                  (* ma_snd_device_name_free_hint_proc)             (void **hints);\ntypedef int                  (* ma_snd_pcm_mmap_begin_proc)                    (ma_snd_pcm_t *pcm, const ma_snd_pcm_channel_area_t **areas, ma_snd_pcm_uframes_t *offset, ma_snd_pcm_uframes_t *frames);\ntypedef ma_snd_pcm_sframes_t (* ma_snd_pcm_mmap_commit_proc)                   (ma_snd_pcm_t *pcm, ma_snd_pcm_uframes_t offset, ma_snd_pcm_uframes_t frames);\ntypedef int                  (* ma_snd_pcm_recover_proc)                       (ma_snd_pcm_t *pcm, int err, int silent);\ntypedef ma_snd_pcm_sframes_t (* ma_snd_pcm_readi_proc)                         (ma_snd_pcm_t *pcm, void *buffer, ma_snd_pcm_uframes_t size);\ntypedef ma_snd_pcm_sframes_t (* ma_snd_pcm_writei_proc)                        (ma_snd_pcm_t *pcm, const void *buffer, ma_snd_pcm_uframes_t size);\ntypedef ma_snd_pcm_sframes_t (* ma_snd_pcm_avail_proc)                         (ma_snd_pcm_t *pcm);\ntypedef ma_snd_pcm_sframes_t (* ma_snd_pcm_avail_update_proc)                  (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_wait_proc)                          (ma_snd_pcm_t *pcm, int timeout);\ntypedef int                  (* ma_snd_pcm_nonblock_proc)                      (ma_snd_pcm_t *pcm, int nonblock);\ntypedef int                  (* ma_snd_pcm_info_proc)                          (ma_snd_pcm_t *pcm, ma_snd_pcm_info_t* info);\ntypedef size_t               (* ma_snd_pcm_info_sizeof_proc)                   (void);\ntypedef const char*          (* ma_snd_pcm_info_get_name_proc)                 (const ma_snd_pcm_info_t* info);\ntypedef int                  (* ma_snd_pcm_poll_descriptors_proc)              (ma_snd_pcm_t *pcm, struct pollfd *pfds, unsigned int space);\ntypedef int                  (* ma_snd_pcm_poll_descriptors_count_proc)        (ma_snd_pcm_t *pcm);\ntypedef int                  (* ma_snd_pcm_poll_descriptors_revents_proc)      (ma_snd_pcm_t *pcm, struct pollfd *pfds, unsigned int nfds, unsigned short *revents);\ntypedef int                  (* ma_snd_config_update_free_global_proc)         (void);\n\n/* This array specifies each of the common devices that can be used for both playback and capture. */\nstatic const char* g_maCommonDeviceNamesALSA[] = {\n    \"default\",\n    \"null\",\n    \"pulse\",\n    \"jack\"\n};\n\n/* This array allows us to blacklist specific playback devices. */\nstatic const char* g_maBlacklistedPlaybackDeviceNamesALSA[] = {\n    \"\"\n};\n\n/* This array allows us to blacklist specific capture devices. */\nstatic const char* g_maBlacklistedCaptureDeviceNamesALSA[] = {\n    \"\"\n};\n\n\nstatic ma_snd_pcm_format_t ma_convert_ma_format_to_alsa_format(ma_format format)\n{\n    ma_snd_pcm_format_t ALSAFormats[] = {\n        MA_SND_PCM_FORMAT_UNKNOWN,     /* ma_format_unknown */\n        MA_SND_PCM_FORMAT_U8,          /* ma_format_u8 */\n        MA_SND_PCM_FORMAT_S16_LE,      /* ma_format_s16 */\n        MA_SND_PCM_FORMAT_S24_3LE,     /* ma_format_s24 */\n        MA_SND_PCM_FORMAT_S32_LE,      /* ma_format_s32 */\n        MA_SND_PCM_FORMAT_FLOAT_LE     /* ma_format_f32 */\n    };\n\n    if (ma_is_big_endian()) {\n        ALSAFormats[0] = MA_SND_PCM_FORMAT_UNKNOWN;\n        ALSAFormats[1] = MA_SND_PCM_FORMAT_U8;\n        ALSAFormats[2] = MA_SND_PCM_FORMAT_S16_BE;\n        ALSAFormats[3] = MA_SND_PCM_FORMAT_S24_3BE;\n        ALSAFormats[4] = MA_SND_PCM_FORMAT_S32_BE;\n        ALSAFormats[5] = MA_SND_PCM_FORMAT_FLOAT_BE;\n    }\n\n    return ALSAFormats[format];\n}\n\nstatic ma_format ma_format_from_alsa(ma_snd_pcm_format_t formatALSA)\n{\n    if (ma_is_little_endian()) {\n        switch (formatALSA) {\n            case MA_SND_PCM_FORMAT_S16_LE:   return ma_format_s16;\n            case MA_SND_PCM_FORMAT_S24_3LE:  return ma_format_s24;\n            case MA_SND_PCM_FORMAT_S32_LE:   return ma_format_s32;\n            case MA_SND_PCM_FORMAT_FLOAT_LE: return ma_format_f32;\n            default: break;\n        }\n    } else {\n        switch (formatALSA) {\n            case MA_SND_PCM_FORMAT_S16_BE:   return ma_format_s16;\n            case MA_SND_PCM_FORMAT_S24_3BE:  return ma_format_s24;\n            case MA_SND_PCM_FORMAT_S32_BE:   return ma_format_s32;\n            case MA_SND_PCM_FORMAT_FLOAT_BE: return ma_format_f32;\n            default: break;\n        }\n    }\n\n    /* Endian agnostic. */\n    switch (formatALSA) {\n        case MA_SND_PCM_FORMAT_U8: return ma_format_u8;\n        default: return ma_format_unknown;\n    }\n}\n\nstatic ma_channel ma_convert_alsa_channel_position_to_ma_channel(unsigned int alsaChannelPos)\n{\n    switch (alsaChannelPos)\n    {\n        case MA_SND_CHMAP_MONO: return MA_CHANNEL_MONO;\n        case MA_SND_CHMAP_FL:   return MA_CHANNEL_FRONT_LEFT;\n        case MA_SND_CHMAP_FR:   return MA_CHANNEL_FRONT_RIGHT;\n        case MA_SND_CHMAP_RL:   return MA_CHANNEL_BACK_LEFT;\n        case MA_SND_CHMAP_RR:   return MA_CHANNEL_BACK_RIGHT;\n        case MA_SND_CHMAP_FC:   return MA_CHANNEL_FRONT_CENTER;\n        case MA_SND_CHMAP_LFE:  return MA_CHANNEL_LFE;\n        case MA_SND_CHMAP_SL:   return MA_CHANNEL_SIDE_LEFT;\n        case MA_SND_CHMAP_SR:   return MA_CHANNEL_SIDE_RIGHT;\n        case MA_SND_CHMAP_RC:   return MA_CHANNEL_BACK_CENTER;\n        case MA_SND_CHMAP_FLC:  return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case MA_SND_CHMAP_FRC:  return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case MA_SND_CHMAP_RLC:  return 0;\n        case MA_SND_CHMAP_RRC:  return 0;\n        case MA_SND_CHMAP_FLW:  return 0;\n        case MA_SND_CHMAP_FRW:  return 0;\n        case MA_SND_CHMAP_FLH:  return 0;\n        case MA_SND_CHMAP_FCH:  return 0;\n        case MA_SND_CHMAP_FRH:  return 0;\n        case MA_SND_CHMAP_TC:   return MA_CHANNEL_TOP_CENTER;\n        case MA_SND_CHMAP_TFL:  return MA_CHANNEL_TOP_FRONT_LEFT;\n        case MA_SND_CHMAP_TFR:  return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case MA_SND_CHMAP_TFC:  return MA_CHANNEL_TOP_FRONT_CENTER;\n        case MA_SND_CHMAP_TRL:  return MA_CHANNEL_TOP_BACK_LEFT;\n        case MA_SND_CHMAP_TRR:  return MA_CHANNEL_TOP_BACK_RIGHT;\n        case MA_SND_CHMAP_TRC:  return MA_CHANNEL_TOP_BACK_CENTER;\n        default: break;\n    }\n\n    return 0;\n}\n\nstatic ma_bool32 ma_is_common_device_name__alsa(const char* name)\n{\n    size_t iName;\n    for (iName = 0; iName < ma_countof(g_maCommonDeviceNamesALSA); ++iName) {\n        if (ma_strcmp(name, g_maCommonDeviceNamesALSA[iName]) == 0) {\n            return MA_TRUE;\n        }\n    }\n\n    return MA_FALSE;\n}\n\n\nstatic ma_bool32 ma_is_playback_device_blacklisted__alsa(const char* name)\n{\n    size_t iName;\n    for (iName = 0; iName < ma_countof(g_maBlacklistedPlaybackDeviceNamesALSA); ++iName) {\n        if (ma_strcmp(name, g_maBlacklistedPlaybackDeviceNamesALSA[iName]) == 0) {\n            return MA_TRUE;\n        }\n    }\n\n    return MA_FALSE;\n}\n\nstatic ma_bool32 ma_is_capture_device_blacklisted__alsa(const char* name)\n{\n    size_t iName;\n    for (iName = 0; iName < ma_countof(g_maBlacklistedCaptureDeviceNamesALSA); ++iName) {\n        if (ma_strcmp(name, g_maBlacklistedCaptureDeviceNamesALSA[iName]) == 0) {\n            return MA_TRUE;\n        }\n    }\n\n    return MA_FALSE;\n}\n\nstatic ma_bool32 ma_is_device_blacklisted__alsa(ma_device_type deviceType, const char* name)\n{\n    if (deviceType == ma_device_type_playback) {\n        return ma_is_playback_device_blacklisted__alsa(name);\n    } else {\n        return ma_is_capture_device_blacklisted__alsa(name);\n    }\n}\n\n\nstatic const char* ma_find_char(const char* str, char c, int* index)\n{\n    int i = 0;\n    for (;;) {\n        if (str[i] == '\\0') {\n            if (index) *index = -1;\n            return NULL;\n        }\n\n        if (str[i] == c) {\n            if (index) *index = i;\n            return str + i;\n        }\n\n        i += 1;\n    }\n\n    /* Should never get here, but treat it as though the character was not found to make me feel better inside. */\n    if (index) *index = -1;\n    return NULL;\n}\n\nstatic ma_bool32 ma_is_device_name_in_hw_format__alsa(const char* hwid)\n{\n    /* This function is just checking whether or not hwid is in \"hw:%d,%d\" format. */\n\n    int commaPos;\n    const char* dev;\n    int i;\n\n    if (hwid == NULL) {\n        return MA_FALSE;\n    }\n\n    if (hwid[0] != 'h' || hwid[1] != 'w' || hwid[2] != ':') {\n        return MA_FALSE;\n    }\n\n    hwid += 3;\n\n    dev = ma_find_char(hwid, ',', &commaPos);\n    if (dev == NULL) {\n        return MA_FALSE;\n    } else {\n        dev += 1;   /* Skip past the \",\". */\n    }\n\n    /* Check if the part between the \":\" and the \",\" contains only numbers. If not, return false. */\n    for (i = 0; i < commaPos; ++i) {\n        if (hwid[i] < '0' || hwid[i] > '9') {\n            return MA_FALSE;\n        }\n    }\n\n    /* Check if everything after the \",\" is numeric. If not, return false. */\n    i = 0;\n    while (dev[i] != '\\0') {\n        if (dev[i] < '0' || dev[i] > '9') {\n            return MA_FALSE;\n        }\n        i += 1;\n    }\n\n    return MA_TRUE;\n}\n\nstatic int ma_convert_device_name_to_hw_format__alsa(ma_context* pContext, char* dst, size_t dstSize, const char* src)  /* Returns 0 on success, non-0 on error. */\n{\n    /* src should look something like this: \"hw:CARD=I82801AAICH,DEV=0\" */\n\n    int colonPos;\n    int commaPos;\n    char card[256];\n    const char* dev;\n    int cardIndex;\n\n    if (dst == NULL) {\n        return -1;\n    }\n    if (dstSize < 7) {\n        return -1;     /* Absolute minimum size of the output buffer is 7 bytes. */\n    }\n\n    *dst = '\\0';    /* Safety. */\n    if (src == NULL) {\n        return -1;\n    }\n\n    /* If the input name is already in \"hw:%d,%d\" format, just return that verbatim. */\n    if (ma_is_device_name_in_hw_format__alsa(src)) {\n        return ma_strcpy_s(dst, dstSize, src);\n    }\n\n    src = ma_find_char(src, ':', &colonPos);\n    if (src == NULL) {\n        return -1;  /* Couldn't find a colon */\n    }\n\n    dev = ma_find_char(src, ',', &commaPos);\n    if (dev == NULL) {\n        dev = \"0\";\n        ma_strncpy_s(card, sizeof(card), src+6, (size_t)-1);   /* +6 = \":CARD=\" */\n    } else {\n        dev = dev + 5;  /* +5 = \",DEV=\" */\n        ma_strncpy_s(card, sizeof(card), src+6, commaPos-6);   /* +6 = \":CARD=\" */\n    }\n\n    cardIndex = ((ma_snd_card_get_index_proc)pContext->alsa.snd_card_get_index)(card);\n    if (cardIndex < 0) {\n        return -2;  /* Failed to retrieve the card index. */\n    }\n\n\n    /* Construction. */\n    dst[0] = 'h'; dst[1] = 'w'; dst[2] = ':';\n    if (ma_itoa_s(cardIndex, dst+3, dstSize-3, 10) != 0) {\n        return -3;\n    }\n    if (ma_strcat_s(dst, dstSize, \",\") != 0) {\n        return -3;\n    }\n    if (ma_strcat_s(dst, dstSize, dev) != 0) {\n        return -3;\n    }\n\n    return 0;\n}\n\nstatic ma_bool32 ma_does_id_exist_in_list__alsa(ma_device_id* pUniqueIDs, ma_uint32 count, const char* pHWID)\n{\n    ma_uint32 i;\n\n    MA_ASSERT(pHWID != NULL);\n\n    for (i = 0; i < count; ++i) {\n        if (ma_strcmp(pUniqueIDs[i].alsa, pHWID) == 0) {\n            return MA_TRUE;\n        }\n    }\n\n    return MA_FALSE;\n}\n\n\nstatic ma_result ma_context_open_pcm__alsa(ma_context* pContext, ma_share_mode shareMode, ma_device_type deviceType, const ma_device_id* pDeviceID, int openMode, ma_snd_pcm_t** ppPCM)\n{\n    ma_snd_pcm_t* pPCM;\n    ma_snd_pcm_stream_t stream;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppPCM != NULL);\n\n    *ppPCM = NULL;\n    pPCM = NULL;\n\n    stream = (deviceType == ma_device_type_playback) ? MA_SND_PCM_STREAM_PLAYBACK : MA_SND_PCM_STREAM_CAPTURE;\n\n    if (pDeviceID == NULL) {\n        ma_bool32 isDeviceOpen;\n        size_t i;\n\n        /*\n        We're opening the default device. I don't know if trying anything other than \"default\" is necessary, but it makes\n        me feel better to try as hard as we can get to get _something_ working.\n        */\n        const char* defaultDeviceNames[] = {\n            \"default\",\n            NULL,\n            NULL,\n            NULL,\n            NULL,\n            NULL,\n            NULL\n        };\n\n        if (shareMode == ma_share_mode_exclusive) {\n            defaultDeviceNames[1] = \"hw\";\n            defaultDeviceNames[2] = \"hw:0\";\n            defaultDeviceNames[3] = \"hw:0,0\";\n        } else {\n            if (deviceType == ma_device_type_playback) {\n                defaultDeviceNames[1] = \"dmix\";\n                defaultDeviceNames[2] = \"dmix:0\";\n                defaultDeviceNames[3] = \"dmix:0,0\";\n            } else {\n                defaultDeviceNames[1] = \"dsnoop\";\n                defaultDeviceNames[2] = \"dsnoop:0\";\n                defaultDeviceNames[3] = \"dsnoop:0,0\";\n            }\n            defaultDeviceNames[4] = \"hw\";\n            defaultDeviceNames[5] = \"hw:0\";\n            defaultDeviceNames[6] = \"hw:0,0\";\n        }\n\n        isDeviceOpen = MA_FALSE;\n        for (i = 0; i < ma_countof(defaultDeviceNames); ++i) {\n            if (defaultDeviceNames[i] != NULL && defaultDeviceNames[i][0] != '\\0') {\n                if (((ma_snd_pcm_open_proc)pContext->alsa.snd_pcm_open)(&pPCM, defaultDeviceNames[i], stream, openMode) == 0) {\n                    isDeviceOpen = MA_TRUE;\n                    break;\n                }\n            }\n        }\n\n        if (!isDeviceOpen) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[ALSA] snd_pcm_open() failed when trying to open an appropriate default device.\");\n            return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n        }\n    } else {\n        /*\n        We're trying to open a specific device. There's a few things to consider here:\n\n        miniaudio recognizes a special format of device id that excludes the \"hw\", \"dmix\", etc. prefix. It looks like this: \":0,0\", \":0,1\", etc. When\n        an ID of this format is specified, it indicates to miniaudio that it can try different combinations of plugins (\"hw\", \"dmix\", etc.) until it\n        finds an appropriate one that works. This comes in very handy when trying to open a device in shared mode (\"dmix\"), vs exclusive mode (\"hw\").\n        */\n\n        /* May end up needing to make small adjustments to the ID, so make a copy. */\n        ma_device_id deviceID = *pDeviceID;\n        int resultALSA = -ENODEV;\n\n        if (deviceID.alsa[0] != ':') {\n            /* The ID is not in \":0,0\" format. Use the ID exactly as-is. */\n            resultALSA = ((ma_snd_pcm_open_proc)pContext->alsa.snd_pcm_open)(&pPCM, deviceID.alsa, stream, openMode);\n        } else {\n            char hwid[256];\n\n            /* The ID is in \":0,0\" format. Try different plugins depending on the shared mode. */\n            if (deviceID.alsa[1] == '\\0') {\n                deviceID.alsa[0] = '\\0';  /* An ID of \":\" should be converted to \"\". */\n            }\n\n            if (shareMode == ma_share_mode_shared) {\n                if (deviceType == ma_device_type_playback) {\n                    ma_strcpy_s(hwid, sizeof(hwid), \"dmix\");\n                } else {\n                    ma_strcpy_s(hwid, sizeof(hwid), \"dsnoop\");\n                }\n\n                if (ma_strcat_s(hwid, sizeof(hwid), deviceID.alsa) == 0) {\n                    resultALSA = ((ma_snd_pcm_open_proc)pContext->alsa.snd_pcm_open)(&pPCM, hwid, stream, openMode);\n                }\n            }\n\n            /* If at this point we still don't have an open device it means we're either preferencing exclusive mode or opening with \"dmix\"/\"dsnoop\" failed. */\n            if (resultALSA != 0) {\n                ma_strcpy_s(hwid, sizeof(hwid), \"hw\");\n                if (ma_strcat_s(hwid, sizeof(hwid), deviceID.alsa) == 0) {\n                    resultALSA = ((ma_snd_pcm_open_proc)pContext->alsa.snd_pcm_open)(&pPCM, hwid, stream, openMode);\n                }\n            }\n        }\n\n        if (resultALSA < 0) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[ALSA] snd_pcm_open() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n    }\n\n    *ppPCM = pPCM;\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__alsa(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    int resultALSA;\n    ma_bool32 cbResult = MA_TRUE;\n    char** ppDeviceHints;\n    ma_device_id* pUniqueIDs = NULL;\n    ma_uint32 uniqueIDCount = 0;\n    char** ppNextDeviceHint;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    ma_mutex_lock(&pContext->alsa.internalDeviceEnumLock);\n\n    resultALSA = ((ma_snd_device_name_hint_proc)pContext->alsa.snd_device_name_hint)(-1, \"pcm\", (void***)&ppDeviceHints);\n    if (resultALSA < 0) {\n        ma_mutex_unlock(&pContext->alsa.internalDeviceEnumLock);\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    ppNextDeviceHint = ppDeviceHints;\n    while (*ppNextDeviceHint != NULL) {\n        char* NAME = ((ma_snd_device_name_get_hint_proc)pContext->alsa.snd_device_name_get_hint)(*ppNextDeviceHint, \"NAME\");\n        char* DESC = ((ma_snd_device_name_get_hint_proc)pContext->alsa.snd_device_name_get_hint)(*ppNextDeviceHint, \"DESC\");\n        char* IOID = ((ma_snd_device_name_get_hint_proc)pContext->alsa.snd_device_name_get_hint)(*ppNextDeviceHint, \"IOID\");\n        ma_device_type deviceType = ma_device_type_playback;\n        ma_bool32 stopEnumeration = MA_FALSE;\n        char hwid[sizeof(pUniqueIDs->alsa)];\n        ma_device_info deviceInfo;\n\n        if ((IOID == NULL || ma_strcmp(IOID, \"Output\") == 0)) {\n            deviceType = ma_device_type_playback;\n        }\n        if ((IOID != NULL && ma_strcmp(IOID, \"Input\" ) == 0)) {\n            deviceType = ma_device_type_capture;\n        }\n\n        if (NAME != NULL) {\n            if (pContext->alsa.useVerboseDeviceEnumeration) {\n                /* Verbose mode. Use the name exactly as-is. */\n                ma_strncpy_s(hwid, sizeof(hwid), NAME, (size_t)-1);\n            } else {\n                /* Simplified mode. Use \":%d,%d\" format. */\n                if (ma_convert_device_name_to_hw_format__alsa(pContext, hwid, sizeof(hwid), NAME) == 0) {\n                    /*\n                    At this point, hwid looks like \"hw:0,0\". In simplified enumeration mode, we actually want to strip off the\n                    plugin name so it looks like \":0,0\". The reason for this is that this special format is detected at device\n                    initialization time and is used as an indicator to try to use the most appropriate plugin depending on the\n                    device type and sharing mode.\n                    */\n                    char* dst = hwid;\n                    char* src = hwid+2;\n                    while ((*dst++ = *src++));\n                } else {\n                    /* Conversion to \"hw:%d,%d\" failed. Just use the name as-is. */\n                    ma_strncpy_s(hwid, sizeof(hwid), NAME, (size_t)-1);\n                }\n\n                if (ma_does_id_exist_in_list__alsa(pUniqueIDs, uniqueIDCount, hwid)) {\n                    goto next_device;   /* The device has already been enumerated. Move on to the next one. */\n                } else {\n                    /* The device has not yet been enumerated. Make sure it's added to our list so that it's not enumerated again. */\n                    size_t newCapacity = sizeof(*pUniqueIDs) * (uniqueIDCount + 1);\n                    ma_device_id* pNewUniqueIDs = (ma_device_id*)ma_realloc(pUniqueIDs, newCapacity, &pContext->allocationCallbacks);\n                    if (pNewUniqueIDs == NULL) {\n                        goto next_device;   /* Failed to allocate memory. */\n                    }\n\n                    pUniqueIDs = pNewUniqueIDs;\n                    MA_COPY_MEMORY(pUniqueIDs[uniqueIDCount].alsa, hwid, sizeof(hwid));\n                    uniqueIDCount += 1;\n                }\n            }\n        } else {\n            MA_ZERO_MEMORY(hwid, sizeof(hwid));\n        }\n\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.id.alsa, sizeof(deviceInfo.id.alsa), hwid, (size_t)-1);\n\n        /*\n        There's no good way to determine whether or not a device is the default on Linux. We're just going to do something simple and\n        just use the name of \"default\" as the indicator.\n        */\n        if (ma_strcmp(deviceInfo.id.alsa, \"default\") == 0) {\n            deviceInfo.isDefault = MA_TRUE;\n        }\n\n\n        /*\n        DESC is the friendly name. We treat this slightly differently depending on whether or not we are using verbose\n        device enumeration. In verbose mode we want to take the entire description so that the end-user can distinguish\n        between the subdevices of each card/dev pair. In simplified mode, however, we only want the first part of the\n        description.\n\n        The value in DESC seems to be split into two lines, with the first line being the name of the device and the\n        second line being a description of the device. I don't like having the description be across two lines because\n        it makes formatting ugly and annoying. I'm therefore deciding to put it all on a single line with the second line\n        being put into parentheses. In simplified mode I'm just stripping the second line entirely.\n        */\n        if (DESC != NULL) {\n            int lfPos;\n            const char* line2 = ma_find_char(DESC, '\\n', &lfPos);\n            if (line2 != NULL) {\n                line2 += 1; /* Skip past the new-line character. */\n\n                if (pContext->alsa.useVerboseDeviceEnumeration) {\n                    /* Verbose mode. Put the second line in brackets. */\n                    ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), DESC, lfPos);\n                    ma_strcat_s (deviceInfo.name, sizeof(deviceInfo.name), \" (\");\n                    ma_strcat_s (deviceInfo.name, sizeof(deviceInfo.name), line2);\n                    ma_strcat_s (deviceInfo.name, sizeof(deviceInfo.name), \")\");\n                } else {\n                    /* Simplified mode. Strip the second line entirely. */\n                    ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), DESC, lfPos);\n                }\n            } else {\n                /* There's no second line. Just copy the whole description. */\n                ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), DESC, (size_t)-1);\n            }\n        }\n\n        if (!ma_is_device_blacklisted__alsa(deviceType, NAME)) {\n            cbResult = callback(pContext, deviceType, &deviceInfo, pUserData);\n        }\n\n        /*\n        Some devices are both playback and capture, but they are only enumerated by ALSA once. We need to fire the callback\n        again for the other device type in this case. We do this for known devices and where the IOID hint is NULL, which\n        means both Input and Output.\n        */\n        if (cbResult) {\n            if (ma_is_common_device_name__alsa(NAME) || IOID == NULL) {\n                if (deviceType == ma_device_type_playback) {\n                    if (!ma_is_capture_device_blacklisted__alsa(NAME)) {\n                        cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n                    }\n                } else {\n                    if (!ma_is_playback_device_blacklisted__alsa(NAME)) {\n                        cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n                    }\n                }\n            }\n        }\n\n        if (cbResult == MA_FALSE) {\n            stopEnumeration = MA_TRUE;\n        }\n\n    next_device:\n        free(NAME);\n        free(DESC);\n        free(IOID);\n        ppNextDeviceHint += 1;\n\n        /* We need to stop enumeration if the callback returned false. */\n        if (stopEnumeration) {\n            break;\n        }\n    }\n\n    ma_free(pUniqueIDs, &pContext->allocationCallbacks);\n    ((ma_snd_device_name_free_hint_proc)pContext->alsa.snd_device_name_free_hint)((void**)ppDeviceHints);\n\n    ma_mutex_unlock(&pContext->alsa.internalDeviceEnumLock);\n\n    return MA_SUCCESS;\n}\n\n\ntypedef struct\n{\n    ma_device_type deviceType;\n    const ma_device_id* pDeviceID;\n    ma_share_mode shareMode;\n    ma_device_info* pDeviceInfo;\n    ma_bool32 foundDevice;\n} ma_context_get_device_info_enum_callback_data__alsa;\n\nstatic ma_bool32 ma_context_get_device_info_enum_callback__alsa(ma_context* pContext, ma_device_type deviceType, const ma_device_info* pDeviceInfo, void* pUserData)\n{\n    ma_context_get_device_info_enum_callback_data__alsa* pData = (ma_context_get_device_info_enum_callback_data__alsa*)pUserData;\n    MA_ASSERT(pData != NULL);\n\n    (void)pContext;\n\n    if (pData->pDeviceID == NULL && ma_strcmp(pDeviceInfo->id.alsa, \"default\") == 0) {\n        ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), pDeviceInfo->name, (size_t)-1);\n        pData->foundDevice = MA_TRUE;\n    } else {\n        if (pData->deviceType == deviceType && (pData->pDeviceID != NULL && ma_strcmp(pData->pDeviceID->alsa, pDeviceInfo->id.alsa) == 0)) {\n            ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), pDeviceInfo->name, (size_t)-1);\n            pData->foundDevice = MA_TRUE;\n        }\n    }\n\n    /* Keep enumerating until we have found the device. */\n    return !pData->foundDevice;\n}\n\nstatic void ma_context_test_rate_and_add_native_data_format__alsa(ma_context* pContext, ma_snd_pcm_t* pPCM, ma_snd_pcm_hw_params_t* pHWParams, ma_format format, ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 flags, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pPCM        != NULL);\n    MA_ASSERT(pHWParams   != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    if (pDeviceInfo->nativeDataFormatCount < ma_countof(pDeviceInfo->nativeDataFormats) && ((ma_snd_pcm_hw_params_test_rate_proc)pContext->alsa.snd_pcm_hw_params_test_rate)(pPCM, pHWParams, sampleRate, 0) == 0) {\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = format;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = sampleRate;\n        pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = flags;\n        pDeviceInfo->nativeDataFormatCount += 1;\n    }\n}\n\nstatic void ma_context_iterate_rates_and_add_native_data_format__alsa(ma_context* pContext, ma_snd_pcm_t* pPCM, ma_snd_pcm_hw_params_t* pHWParams, ma_format format, ma_uint32 channels, ma_uint32 flags, ma_device_info* pDeviceInfo)\n{\n    ma_uint32 iSampleRate;\n    unsigned int minSampleRate;\n    unsigned int maxSampleRate;\n    int sampleRateDir;  /* Not used. Just passed into snd_pcm_hw_params_get_rate_min/max(). */\n\n    /* There could be a range. */\n    ((ma_snd_pcm_hw_params_get_rate_min_proc)pContext->alsa.snd_pcm_hw_params_get_rate_min)(pHWParams, &minSampleRate, &sampleRateDir);\n    ((ma_snd_pcm_hw_params_get_rate_max_proc)pContext->alsa.snd_pcm_hw_params_get_rate_max)(pHWParams, &maxSampleRate, &sampleRateDir);\n\n    /* Make sure our sample rates are clamped to sane values. Stupid devices like \"pulse\" will reports rates like \"1\" which is ridiculous. */\n    minSampleRate = ma_clamp(minSampleRate, (unsigned int)ma_standard_sample_rate_min, (unsigned int)ma_standard_sample_rate_max);\n    maxSampleRate = ma_clamp(maxSampleRate, (unsigned int)ma_standard_sample_rate_min, (unsigned int)ma_standard_sample_rate_max);\n\n    for (iSampleRate = 0; iSampleRate < ma_countof(g_maStandardSampleRatePriorities); iSampleRate += 1) {\n        ma_uint32 standardSampleRate = g_maStandardSampleRatePriorities[iSampleRate];\n\n        if (standardSampleRate >= minSampleRate && standardSampleRate <= maxSampleRate) {\n            ma_context_test_rate_and_add_native_data_format__alsa(pContext, pPCM, pHWParams, format, channels, standardSampleRate, flags, pDeviceInfo);\n        }\n    }\n\n    /* Now make sure our min and max rates are included just in case they aren't in the range of our standard rates. */\n    if (!ma_is_standard_sample_rate(minSampleRate)) {\n        ma_context_test_rate_and_add_native_data_format__alsa(pContext, pPCM, pHWParams, format, channels, minSampleRate, flags, pDeviceInfo);\n    }\n\n    if (!ma_is_standard_sample_rate(maxSampleRate) && maxSampleRate != minSampleRate) {\n        ma_context_test_rate_and_add_native_data_format__alsa(pContext, pPCM, pHWParams, format, channels, maxSampleRate, flags, pDeviceInfo);\n    }\n}\n\nstatic ma_result ma_context_get_device_info__alsa(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_context_get_device_info_enum_callback_data__alsa data;\n    ma_result result;\n    int resultALSA;\n    ma_snd_pcm_t* pPCM;\n    ma_snd_pcm_hw_params_t* pHWParams;\n    ma_uint32 iFormat;\n    ma_uint32 iChannel;\n\n    MA_ASSERT(pContext != NULL);\n\n    /* We just enumerate to find basic information about the device. */\n    data.deviceType  = deviceType;\n    data.pDeviceID   = pDeviceID;\n    data.pDeviceInfo = pDeviceInfo;\n    data.foundDevice = MA_FALSE;\n    result = ma_context_enumerate_devices__alsa(pContext, ma_context_get_device_info_enum_callback__alsa, &data);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (!data.foundDevice) {\n        return MA_NO_DEVICE;\n    }\n\n    if (ma_strcmp(pDeviceInfo->id.alsa, \"default\") == 0) {\n        pDeviceInfo->isDefault = MA_TRUE;\n    }\n\n    /* For detailed info we need to open the device. */\n    result = ma_context_open_pcm__alsa(pContext, ma_share_mode_shared, deviceType, pDeviceID, 0, &pPCM);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need to initialize a HW parameters object in order to know what formats are supported. */\n    pHWParams = (ma_snd_pcm_hw_params_t*)ma_calloc(((ma_snd_pcm_hw_params_sizeof_proc)pContext->alsa.snd_pcm_hw_params_sizeof)(), &pContext->allocationCallbacks);\n    if (pHWParams == NULL) {\n        ((ma_snd_pcm_close_proc)pContext->alsa.snd_pcm_close)(pPCM);\n        return MA_OUT_OF_MEMORY;\n    }\n\n    resultALSA = ((ma_snd_pcm_hw_params_any_proc)pContext->alsa.snd_pcm_hw_params_any)(pPCM, pHWParams);\n    if (resultALSA < 0) {\n        ma_free(pHWParams, &pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to initialize hardware parameters. snd_pcm_hw_params_any() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    /*\n    Some ALSA devices can support many permutations of formats, channels and rates. We only support\n    a fixed number of permutations which means we need to employ some strategies to ensure the best\n    combinations are returned. An example is the \"pulse\" device which can do its own data conversion\n    in software and as a result can support any combination of format, channels and rate.\n\n    We want to ensure that the first data formats are the best. We have a list of favored sample\n    formats and sample rates, so these will be the basis of our iteration.\n    */\n\n    /* Formats. We just iterate over our standard formats and test them, making sure we reset the configuration space each iteration. */\n    for (iFormat = 0; iFormat < ma_countof(g_maFormatPriorities); iFormat += 1) {\n        ma_format format = g_maFormatPriorities[iFormat];\n\n        /*\n        For each format we need to make sure we reset the configuration space so we don't return\n        channel counts and rates that aren't compatible with a format.\n        */\n        ((ma_snd_pcm_hw_params_any_proc)pContext->alsa.snd_pcm_hw_params_any)(pPCM, pHWParams);\n\n        /* Test the format first. If this fails it means the format is not supported and we can skip it. */\n        if (((ma_snd_pcm_hw_params_test_format_proc)pContext->alsa.snd_pcm_hw_params_test_format)(pPCM, pHWParams, ma_convert_ma_format_to_alsa_format(format)) == 0) {\n            /* The format is supported. */\n            unsigned int minChannels;\n            unsigned int maxChannels;\n\n            /*\n            The configuration space needs to be restricted to this format so we can get an accurate\n            picture of which sample rates and channel counts are support with this format.\n            */\n            ((ma_snd_pcm_hw_params_set_format_proc)pContext->alsa.snd_pcm_hw_params_set_format)(pPCM, pHWParams, ma_convert_ma_format_to_alsa_format(format));\n\n            /* Now we need to check for supported channels. */\n            ((ma_snd_pcm_hw_params_get_channels_min_proc)pContext->alsa.snd_pcm_hw_params_get_channels_min)(pHWParams, &minChannels);\n            ((ma_snd_pcm_hw_params_get_channels_max_proc)pContext->alsa.snd_pcm_hw_params_get_channels_max)(pHWParams, &maxChannels);\n\n            if (minChannels > MA_MAX_CHANNELS) {\n                continue;   /* Too many channels. */\n            }\n            if (maxChannels < MA_MIN_CHANNELS) {\n                continue;   /* Not enough channels. */\n            }\n\n            /*\n            Make sure the channel count is clamped. This is mainly intended for the max channels\n            because some devices can report an unbound maximum.\n            */\n            minChannels = ma_clamp(minChannels, MA_MIN_CHANNELS, MA_MAX_CHANNELS);\n            maxChannels = ma_clamp(maxChannels, MA_MIN_CHANNELS, MA_MAX_CHANNELS);\n\n            if (minChannels == MA_MIN_CHANNELS && maxChannels == MA_MAX_CHANNELS) {\n                /* The device supports all channels. Don't iterate over every single one. Instead just set the channels to 0 which means all channels are supported. */\n                ma_context_iterate_rates_and_add_native_data_format__alsa(pContext, pPCM, pHWParams, format, 0, 0, pDeviceInfo);    /* Intentionally setting the channel count to 0 as that means all channels are supported. */\n            } else {\n                /* The device only supports a specific set of channels. We need to iterate over all of them. */\n                for (iChannel = minChannels; iChannel <= maxChannels; iChannel += 1) {\n                    /* Test the channel before applying it to the configuration space. */\n                    unsigned int channels = iChannel;\n\n                    /* Make sure our channel range is reset before testing again or else we'll always fail the test. */\n                    ((ma_snd_pcm_hw_params_any_proc)pContext->alsa.snd_pcm_hw_params_any)(pPCM, pHWParams);\n                    ((ma_snd_pcm_hw_params_set_format_proc)pContext->alsa.snd_pcm_hw_params_set_format)(pPCM, pHWParams, ma_convert_ma_format_to_alsa_format(format));\n\n                    if (((ma_snd_pcm_hw_params_test_channels_proc)pContext->alsa.snd_pcm_hw_params_test_channels)(pPCM, pHWParams, channels) == 0) {\n                        /* The channel count is supported. */\n\n                        /* The configuration space now needs to be restricted to the channel count before extracting the sample rate. */\n                        ((ma_snd_pcm_hw_params_set_channels_proc)pContext->alsa.snd_pcm_hw_params_set_channels)(pPCM, pHWParams, channels);\n\n                        /* Only after the configuration space has been restricted to the specific channel count should we iterate over our sample rates. */\n                        ma_context_iterate_rates_and_add_native_data_format__alsa(pContext, pPCM, pHWParams, format, channels, 0, pDeviceInfo);\n                    } else {\n                        /* The channel count is not supported. Skip. */\n                    }\n                }\n            }\n        } else {\n            /* The format is not supported. Skip. */\n        }\n    }\n\n    ma_free(pHWParams, &pContext->allocationCallbacks);\n\n    ((ma_snd_pcm_close_proc)pContext->alsa.snd_pcm_close)(pPCM);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_uninit__alsa(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if ((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture);\n        close(pDevice->alsa.wakeupfdCapture);\n        ma_free(pDevice->alsa.pPollDescriptorsCapture, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if ((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback);\n        close(pDevice->alsa.wakeupfdPlayback);\n        ma_free(pDevice->alsa.pPollDescriptorsPlayback, &pDevice->pContext->allocationCallbacks);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init_by_type__alsa(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptor, ma_device_type deviceType)\n{\n    ma_result result;\n    int resultALSA;\n    ma_snd_pcm_t* pPCM;\n    ma_bool32 isUsingMMap;\n    ma_snd_pcm_format_t formatALSA;\n    ma_format internalFormat;\n    ma_uint32 internalChannels;\n    ma_uint32 internalSampleRate;\n    ma_channel internalChannelMap[MA_MAX_CHANNELS];\n    ma_uint32 internalPeriodSizeInFrames;\n    ma_uint32 internalPeriods;\n    int openMode;\n    ma_snd_pcm_hw_params_t* pHWParams;\n    ma_snd_pcm_sw_params_t* pSWParams;\n    ma_snd_pcm_uframes_t bufferBoundary;\n    int pollDescriptorCount;\n    struct pollfd* pPollDescriptors;\n    int wakeupfd;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(deviceType != ma_device_type_duplex); /* This function should only be called for playback _or_ capture, never duplex. */\n    MA_ASSERT(pDevice != NULL);\n\n    formatALSA = ma_convert_ma_format_to_alsa_format(pDescriptor->format);\n\n    openMode = 0;\n    if (pConfig->alsa.noAutoResample) {\n        openMode |= MA_SND_PCM_NO_AUTO_RESAMPLE;\n    }\n    if (pConfig->alsa.noAutoChannels) {\n        openMode |= MA_SND_PCM_NO_AUTO_CHANNELS;\n    }\n    if (pConfig->alsa.noAutoFormat) {\n        openMode |= MA_SND_PCM_NO_AUTO_FORMAT;\n    }\n\n    result = ma_context_open_pcm__alsa(pDevice->pContext, pDescriptor->shareMode, deviceType, pDescriptor->pDeviceID, openMode, &pPCM);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n\n    /* Hardware parameters. */\n    pHWParams = (ma_snd_pcm_hw_params_t*)ma_calloc(((ma_snd_pcm_hw_params_sizeof_proc)pDevice->pContext->alsa.snd_pcm_hw_params_sizeof)(), &pDevice->pContext->allocationCallbacks);\n    if (pHWParams == NULL) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to allocate memory for hardware parameters.\");\n        return MA_OUT_OF_MEMORY;\n    }\n\n    resultALSA = ((ma_snd_pcm_hw_params_any_proc)pDevice->pContext->alsa.snd_pcm_hw_params_any)(pPCM, pHWParams);\n    if (resultALSA < 0) {\n        ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to initialize hardware parameters. snd_pcm_hw_params_any() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    /* MMAP Mode. Try using interleaved MMAP access. If this fails, fall back to standard readi/writei. */\n    isUsingMMap = MA_FALSE;\n#if 0   /* NOTE: MMAP mode temporarily disabled. */\n    if (deviceType != ma_device_type_capture) {    /* <-- Disabling MMAP mode for capture devices because I apparently do not have a device that supports it which means I can't test it... Contributions welcome. */\n        if (!pConfig->alsa.noMMap) {\n            if (((ma_snd_pcm_hw_params_set_access_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_access)(pPCM, pHWParams, MA_SND_PCM_ACCESS_MMAP_INTERLEAVED) == 0) {\n                pDevice->alsa.isUsingMMap = MA_TRUE;\n            }\n        }\n    }\n#endif\n\n    if (!isUsingMMap) {\n        resultALSA = ((ma_snd_pcm_hw_params_set_access_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_access)(pPCM, pHWParams, MA_SND_PCM_ACCESS_RW_INTERLEAVED);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set access mode to neither SND_PCM_ACCESS_MMAP_INTERLEAVED nor SND_PCM_ACCESS_RW_INTERLEAVED. snd_pcm_hw_params_set_access() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n    }\n\n    /*\n    Most important properties first. The documentation for OSS (yes, I know this is ALSA!) recommends format, channels, then sample rate. I can't\n    find any documentation for ALSA specifically, so I'm going to copy the recommendation for OSS.\n    */\n\n    /* Format. */\n    {\n        /*\n        At this point we should have a list of supported formats, so now we need to find the best one. We first check if the requested format is\n        supported, and if so, use that one. If it's not supported, we just run though a list of formats and try to find the best one.\n        */\n        if (formatALSA == MA_SND_PCM_FORMAT_UNKNOWN || ((ma_snd_pcm_hw_params_test_format_proc)pDevice->pContext->alsa.snd_pcm_hw_params_test_format)(pPCM, pHWParams, formatALSA) != 0) {\n            /* We're either requesting the native format or the specified format is not supported. */\n            size_t iFormat;\n\n            formatALSA = MA_SND_PCM_FORMAT_UNKNOWN;\n            for (iFormat = 0; iFormat < ma_countof(g_maFormatPriorities); ++iFormat) {\n                if (((ma_snd_pcm_hw_params_test_format_proc)pDevice->pContext->alsa.snd_pcm_hw_params_test_format)(pPCM, pHWParams, ma_convert_ma_format_to_alsa_format(g_maFormatPriorities[iFormat])) == 0) {\n                    formatALSA = ma_convert_ma_format_to_alsa_format(g_maFormatPriorities[iFormat]);\n                    break;\n                }\n            }\n\n            if (formatALSA == MA_SND_PCM_FORMAT_UNKNOWN) {\n                ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n                ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Format not supported. The device does not support any miniaudio formats.\");\n                return MA_FORMAT_NOT_SUPPORTED;\n            }\n        }\n\n        resultALSA = ((ma_snd_pcm_hw_params_set_format_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_format)(pPCM, pHWParams, formatALSA);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Format not supported. snd_pcm_hw_params_set_format() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        internalFormat = ma_format_from_alsa(formatALSA);\n        if (internalFormat == ma_format_unknown) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] The chosen format is not supported by miniaudio.\");\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n    }\n\n    /* Channels. */\n    {\n        unsigned int channels = pDescriptor->channels;\n        if (channels == 0) {\n            channels = MA_DEFAULT_CHANNELS;\n        }\n\n        resultALSA = ((ma_snd_pcm_hw_params_set_channels_near_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_channels_near)(pPCM, pHWParams, &channels);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set channel count. snd_pcm_hw_params_set_channels_near() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        internalChannels = (ma_uint32)channels;\n    }\n\n    /* Sample Rate */\n    {\n        unsigned int sampleRate;\n\n        /*\n        It appears there's either a bug in ALSA, a bug in some drivers, or I'm doing something silly; but having resampling enabled causes\n        problems with some device configurations when used in conjunction with MMAP access mode. To fix this problem we need to disable\n        resampling.\n\n        To reproduce this problem, open the \"plug:dmix\" device, and set the sample rate to 44100. Internally, it looks like dmix uses a\n        sample rate of 48000. The hardware parameters will get set correctly with no errors, but it looks like the 44100 -> 48000 resampling\n        doesn't work properly - but only with MMAP access mode. You will notice skipping/crackling in the audio, and it'll run at a slightly\n        faster rate.\n\n        miniaudio has built-in support for sample rate conversion (albeit low quality at the moment), so disabling resampling should be fine\n        for us. The only problem is that it won't be taking advantage of any kind of hardware-accelerated resampling and it won't be very\n        good quality until I get a chance to improve the quality of miniaudio's software sample rate conversion.\n\n        I don't currently know if the dmix plugin is the only one with this error. Indeed, this is the only one I've been able to reproduce\n        this error with. In the future, we may want to restrict the disabling of resampling to only known bad plugins.\n        */\n        ((ma_snd_pcm_hw_params_set_rate_resample_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_rate_resample)(pPCM, pHWParams, 0);\n\n        sampleRate = pDescriptor->sampleRate;\n        if (sampleRate == 0) {\n            sampleRate = MA_DEFAULT_SAMPLE_RATE;\n        }\n\n        resultALSA = ((ma_snd_pcm_hw_params_set_rate_near_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_rate_near)(pPCM, pHWParams, &sampleRate, 0);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Sample rate not supported. snd_pcm_hw_params_set_rate_near() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        internalSampleRate = (ma_uint32)sampleRate;\n    }\n\n    /* Periods. */\n    {\n        ma_uint32 periods = pDescriptor->periodCount;\n\n        resultALSA = ((ma_snd_pcm_hw_params_set_periods_near_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_periods_near)(pPCM, pHWParams, &periods, NULL);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set period count. snd_pcm_hw_params_set_periods_near() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        internalPeriods = periods;\n    }\n\n    /* Buffer Size */\n    {\n        ma_snd_pcm_uframes_t actualBufferSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, internalSampleRate, pConfig->performanceProfile) * internalPeriods;\n\n        resultALSA = ((ma_snd_pcm_hw_params_set_buffer_size_near_proc)pDevice->pContext->alsa.snd_pcm_hw_params_set_buffer_size_near)(pPCM, pHWParams, &actualBufferSizeInFrames);\n        if (resultALSA < 0) {\n            ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set buffer size for device. snd_pcm_hw_params_set_buffer_size() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        internalPeriodSizeInFrames = actualBufferSizeInFrames / internalPeriods;\n    }\n\n    /* Apply hardware parameters. */\n    resultALSA = ((ma_snd_pcm_hw_params_proc)pDevice->pContext->alsa.snd_pcm_hw_params)(pPCM, pHWParams);\n    if (resultALSA < 0) {\n        ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set hardware parameters. snd_pcm_hw_params() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    ma_free(pHWParams, &pDevice->pContext->allocationCallbacks);\n    pHWParams = NULL;\n\n\n    /* Software parameters. */\n    pSWParams = (ma_snd_pcm_sw_params_t*)ma_calloc(((ma_snd_pcm_sw_params_sizeof_proc)pDevice->pContext->alsa.snd_pcm_sw_params_sizeof)(), &pDevice->pContext->allocationCallbacks);\n    if (pSWParams == NULL) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to allocate memory for software parameters.\");\n        return MA_OUT_OF_MEMORY;\n    }\n\n    resultALSA = ((ma_snd_pcm_sw_params_current_proc)pDevice->pContext->alsa.snd_pcm_sw_params_current)(pPCM, pSWParams);\n    if (resultALSA < 0) {\n        ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to initialize software parameters. snd_pcm_sw_params_current() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    resultALSA = ((ma_snd_pcm_sw_params_set_avail_min_proc)pDevice->pContext->alsa.snd_pcm_sw_params_set_avail_min)(pPCM, pSWParams, ma_prev_power_of_2(internalPeriodSizeInFrames));\n    if (resultALSA < 0) {\n        ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] snd_pcm_sw_params_set_avail_min() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    resultALSA = ((ma_snd_pcm_sw_params_get_boundary_proc)pDevice->pContext->alsa.snd_pcm_sw_params_get_boundary)(pSWParams, &bufferBoundary);\n    if (resultALSA < 0) {\n        bufferBoundary = internalPeriodSizeInFrames * internalPeriods;\n    }\n\n    if (deviceType == ma_device_type_playback && !isUsingMMap) {   /* Only playback devices in writei/readi mode need a start threshold. */\n        /*\n        Subtle detail here with the start threshold. When in playback-only mode (no full-duplex) we can set the start threshold to\n        the size of a period. But for full-duplex we need to set it such that it is at least two periods.\n        */\n        resultALSA = ((ma_snd_pcm_sw_params_set_start_threshold_proc)pDevice->pContext->alsa.snd_pcm_sw_params_set_start_threshold)(pPCM, pSWParams, internalPeriodSizeInFrames*2);\n        if (resultALSA < 0) {\n            ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set start threshold for playback device. snd_pcm_sw_params_set_start_threshold() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        resultALSA = ((ma_snd_pcm_sw_params_set_stop_threshold_proc)pDevice->pContext->alsa.snd_pcm_sw_params_set_stop_threshold)(pPCM, pSWParams, bufferBoundary);\n        if (resultALSA < 0) { /* Set to boundary to loop instead of stop in the event of an xrun. */\n            ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n            ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set stop threshold for playback device. snd_pcm_sw_params_set_stop_threshold() failed.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n    }\n\n    resultALSA = ((ma_snd_pcm_sw_params_proc)pDevice->pContext->alsa.snd_pcm_sw_params)(pPCM, pSWParams);\n    if (resultALSA < 0) {\n        ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to set software parameters. snd_pcm_sw_params() failed.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n    ma_free(pSWParams, &pDevice->pContext->allocationCallbacks);\n    pSWParams = NULL;\n\n\n    /* Grab the internal channel map. For now we're not going to bother trying to change the channel map and instead just do it ourselves. */\n    {\n        ma_snd_pcm_chmap_t* pChmap = NULL;\n        if (pDevice->pContext->alsa.snd_pcm_get_chmap != NULL) {\n            pChmap = ((ma_snd_pcm_get_chmap_proc)pDevice->pContext->alsa.snd_pcm_get_chmap)(pPCM);\n        }\n\n        if (pChmap != NULL) {\n            ma_uint32 iChannel;\n\n            /* There are cases where the returned channel map can have a different channel count than was returned by snd_pcm_hw_params_set_channels_near(). */\n            if (pChmap->channels >= internalChannels) {\n                /* Drop excess channels. */\n                for (iChannel = 0; iChannel < internalChannels; ++iChannel) {\n                    internalChannelMap[iChannel] = ma_convert_alsa_channel_position_to_ma_channel(pChmap->pos[iChannel]);\n                }\n            } else {\n                ma_uint32 i;\n\n                /*\n                Excess channels use defaults. Do an initial fill with defaults, overwrite the first pChmap->channels, validate to ensure there are no duplicate\n                channels. If validation fails, fall back to defaults.\n                */\n                ma_bool32 isValid = MA_TRUE;\n\n                /* Fill with defaults. */\n                ma_channel_map_init_standard(ma_standard_channel_map_alsa, internalChannelMap, ma_countof(internalChannelMap), internalChannels);\n\n                /* Overwrite first pChmap->channels channels. */\n                for (iChannel = 0; iChannel < pChmap->channels; ++iChannel) {\n                    internalChannelMap[iChannel] = ma_convert_alsa_channel_position_to_ma_channel(pChmap->pos[iChannel]);\n                }\n\n                /* Validate. */\n                for (i = 0; i < internalChannels && isValid; ++i) {\n                    ma_uint32 j;\n                    for (j = i+1; j < internalChannels; ++j) {\n                        if (internalChannelMap[i] == internalChannelMap[j]) {\n                            isValid = MA_FALSE;\n                            break;\n                        }\n                    }\n                }\n\n                /* If our channel map is invalid, fall back to defaults. */\n                if (!isValid) {\n                    ma_channel_map_init_standard(ma_standard_channel_map_alsa, internalChannelMap, ma_countof(internalChannelMap), internalChannels);\n                }\n            }\n\n            free(pChmap);\n            pChmap = NULL;\n        } else {\n            /* Could not retrieve the channel map. Fall back to a hard-coded assumption. */\n            ma_channel_map_init_standard(ma_standard_channel_map_alsa, internalChannelMap, ma_countof(internalChannelMap), internalChannels);\n        }\n    }\n\n\n    /*\n    We need to retrieve the poll descriptors so we can use poll() to wait for data to become\n    available for reading or writing. There's no well defined maximum for this so we're just going\n    to allocate this on the heap.\n    */\n    pollDescriptorCount = ((ma_snd_pcm_poll_descriptors_count_proc)pDevice->pContext->alsa.snd_pcm_poll_descriptors_count)(pPCM);\n    if (pollDescriptorCount <= 0) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to retrieve poll descriptors count.\");\n        return MA_ERROR;\n    }\n\n    pPollDescriptors = (struct pollfd*)ma_malloc(sizeof(*pPollDescriptors) * (pollDescriptorCount + 1), &pDevice->pContext->allocationCallbacks);   /* +1 because we want room for the wakeup descriptor. */\n    if (pPollDescriptors == NULL) {\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to allocate memory for poll descriptors.\");\n        return MA_OUT_OF_MEMORY;\n    }\n\n    /*\n    We need an eventfd to wakeup from poll() and avoid a deadlock in situations where the driver\n    never returns from writei() and readi(). This has been observed with the \"pulse\" device.\n    */\n    wakeupfd = eventfd(0, 0);\n    if (wakeupfd < 0) {\n        ma_free(pPollDescriptors, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to create eventfd for poll wakeup.\");\n        return ma_result_from_errno(errno);\n    }\n\n    /* We'll place the wakeup fd at the start of the buffer. */\n    pPollDescriptors[0].fd      = wakeupfd;\n    pPollDescriptors[0].events  = POLLIN;    /* We only care about waiting to read from the wakeup file descriptor. */\n    pPollDescriptors[0].revents = 0;\n\n    /* We can now extract the PCM poll descriptors which we place after the wakeup descriptor. */\n    pollDescriptorCount = ((ma_snd_pcm_poll_descriptors_proc)pDevice->pContext->alsa.snd_pcm_poll_descriptors)(pPCM, pPollDescriptors + 1, pollDescriptorCount);    /* +1 because we want to place these descriptors after the wakeup descriptor. */\n    if (pollDescriptorCount <= 0) {\n        close(wakeupfd);\n        ma_free(pPollDescriptors, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to retrieve poll descriptors.\");\n        return MA_ERROR;\n    }\n\n    if (deviceType == ma_device_type_capture) {\n        pDevice->alsa.pollDescriptorCountCapture = pollDescriptorCount;\n        pDevice->alsa.pPollDescriptorsCapture = pPollDescriptors;\n        pDevice->alsa.wakeupfdCapture = wakeupfd;\n    } else {\n        pDevice->alsa.pollDescriptorCountPlayback = pollDescriptorCount;\n        pDevice->alsa.pPollDescriptorsPlayback = pPollDescriptors;\n        pDevice->alsa.wakeupfdPlayback = wakeupfd;\n    }\n\n\n    /* We're done. Prepare the device. */\n    resultALSA = ((ma_snd_pcm_prepare_proc)pDevice->pContext->alsa.snd_pcm_prepare)(pPCM);\n    if (resultALSA < 0) {\n        close(wakeupfd);\n        ma_free(pPollDescriptors, &pDevice->pContext->allocationCallbacks);\n        ((ma_snd_pcm_close_proc)pDevice->pContext->alsa.snd_pcm_close)(pPCM);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to prepare device.\");\n        return ma_result_from_errno(-resultALSA);\n    }\n\n\n    if (deviceType == ma_device_type_capture) {\n        pDevice->alsa.pPCMCapture         = (ma_ptr)pPCM;\n        pDevice->alsa.isUsingMMapCapture  = isUsingMMap;\n    } else {\n        pDevice->alsa.pPCMPlayback        = (ma_ptr)pPCM;\n        pDevice->alsa.isUsingMMapPlayback = isUsingMMap;\n    }\n\n    pDescriptor->format             = internalFormat;\n    pDescriptor->channels           = internalChannels;\n    pDescriptor->sampleRate         = internalSampleRate;\n    ma_channel_map_copy(pDescriptor->channelMap, internalChannelMap, ma_min(internalChannels, MA_MAX_CHANNELS));\n    pDescriptor->periodSizeInFrames = internalPeriodSizeInFrames;\n    pDescriptor->periodCount        = internalPeriods;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__alsa(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->alsa);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_by_type__alsa(pDevice, pConfig, pDescriptorCapture, ma_device_type_capture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_by_type__alsa(pDevice, pConfig, pDescriptorPlayback, ma_device_type_playback);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__alsa(ma_device* pDevice)\n{\n    int resultALSA;\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        resultALSA = ((ma_snd_pcm_start_proc)pDevice->pContext->alsa.snd_pcm_start)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture);\n        if (resultALSA < 0) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to start capture device.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        /*        \n        When data is written to the device we wait for the device to get ready to receive data with poll(). In my testing\n        I have observed that poll() can sometimes block forever unless the device is started explicitly with snd_pcm_start()\n        or some data is written with snd_pcm_writei().\n\n        To resolve this I've decided to do an explicit start with snd_pcm_start(). The problem with this is that the device\n        is started without any data in the internal buffer which will result in an immediate underrun. If instead we were\n        to call into snd_pcm_writei() in an attempt to prevent the underrun, we would run the risk of a weird deadlock\n        issue as documented inside ma_device_write__alsa().\n        */\n        resultALSA = ((ma_snd_pcm_start_proc)pDevice->pContext->alsa.snd_pcm_start)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback);\n        if (resultALSA < 0) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to start playback device.\");\n            return ma_result_from_errno(-resultALSA);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__alsa(ma_device* pDevice)\n{\n    /*\n    The stop callback will get called on the worker thread after read/write__alsa() has returned. At this point there is\n    a small chance that our wakeupfd has not been cleared. We'll clear that out now if applicable.\n    */\n    int resultPoll;\n    int resultRead;\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Dropping capture device...\\n\");\n        ((ma_snd_pcm_drop_proc)pDevice->pContext->alsa.snd_pcm_drop)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture);\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Dropping capture device successful.\\n\");\n\n        /* We need to prepare the device again, otherwise we won't be able to restart the device. */\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing capture device...\\n\");\n        if (((ma_snd_pcm_prepare_proc)pDevice->pContext->alsa.snd_pcm_prepare)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture) < 0) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing capture device failed.\\n\");\n        } else {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing capture device successful.\\n\");\n        }\n\n        /* Clear the wakeupfd. */\n        resultPoll = poll((struct pollfd*)pDevice->alsa.pPollDescriptorsCapture, 1, 0);\n        if (resultPoll > 0) {\n            ma_uint64 t;\n            resultRead = read(((struct pollfd*)pDevice->alsa.pPollDescriptorsCapture)[0].fd, &t, sizeof(t));\n            if (resultRead != sizeof(t)) {\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Failed to read from capture wakeupfd. read() = %d\\n\", resultRead);\n            }\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Dropping playback device...\\n\");\n        ((ma_snd_pcm_drop_proc)pDevice->pContext->alsa.snd_pcm_drop)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback);\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Dropping playback device successful.\\n\");\n\n        /* We need to prepare the device again, otherwise we won't be able to restart the device. */\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing playback device...\\n\");\n        if (((ma_snd_pcm_prepare_proc)pDevice->pContext->alsa.snd_pcm_prepare)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback) < 0) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing playback device failed.\\n\");\n        } else {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Preparing playback device successful.\\n\");\n        }\n\n        /* Clear the wakeupfd. */\n        resultPoll = poll((struct pollfd*)pDevice->alsa.pPollDescriptorsPlayback, 1, 0);\n        if (resultPoll > 0) {\n            ma_uint64 t;\n            resultRead = read(((struct pollfd*)pDevice->alsa.pPollDescriptorsPlayback)[0].fd, &t, sizeof(t));\n            if (resultRead != sizeof(t)) {\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Failed to read from playback wakeupfd. read() = %d\\n\", resultRead);\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_wait__alsa(ma_device* pDevice, ma_snd_pcm_t* pPCM, struct pollfd* pPollDescriptors, int pollDescriptorCount, short requiredEvent)\n{\n    for (;;) {\n        unsigned short revents;\n        int resultALSA;\n        int resultPoll = poll(pPollDescriptors, pollDescriptorCount, -1);\n        if (resultPoll < 0) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[ALSA] poll() failed.\\n\");\n\n            /*\n            There have been reports that poll() is returning an error randomly and that instead of\n            returning an error, simply trying again will work. I'm experimenting with adopting this\n            advice.\n            */\n            continue;\n            /*return ma_result_from_errno(errno);*/\n        }\n\n        /*\n        Before checking the ALSA poll descriptor flag we need to check if the wakeup descriptor\n        has had it's POLLIN flag set. If so, we need to actually read the data and then exit the\n        function. The wakeup descriptor will be the first item in the descriptors buffer.\n        */\n        if ((pPollDescriptors[0].revents & POLLIN) != 0) {\n            ma_uint64 t;\n            int resultRead = read(pPollDescriptors[0].fd, &t, sizeof(t));    /* <-- Important that we read here so that the next write() does not block. */\n            if (resultRead < 0) {\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] read() failed.\\n\");\n                return ma_result_from_errno(errno);\n            }\n\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] POLLIN set for wakeupfd\\n\");\n            return MA_DEVICE_NOT_STARTED;\n        }\n\n        /*\n        Getting here means that some data should be able to be read. We need to use ALSA to\n        translate the revents flags for us.\n        */\n        resultALSA = ((ma_snd_pcm_poll_descriptors_revents_proc)pDevice->pContext->alsa.snd_pcm_poll_descriptors_revents)(pPCM, pPollDescriptors + 1, pollDescriptorCount - 1, &revents);   /* +1, -1 to ignore the wakeup descriptor. */\n        if (resultALSA < 0) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] snd_pcm_poll_descriptors_revents() failed.\\n\");\n            return ma_result_from_errno(-resultALSA);\n        }\n\n        if ((revents & POLLERR) != 0) {\n            ma_snd_pcm_state_t state = ((ma_snd_pcm_state_proc)pDevice->pContext->alsa.snd_pcm_state)(pPCM);\n            if (state == MA_SND_PCM_STATE_XRUN) {\n                /* The PCM is in a xrun state. This will be recovered from at a higher level. We can disregard this. */\n            } else {\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[ALSA] POLLERR detected. status = %d\\n\", ((ma_snd_pcm_state_proc)pDevice->pContext->alsa.snd_pcm_state)(pPCM));\n            }\n        }\n\n        if ((revents & requiredEvent) == requiredEvent) {\n            break;  /* We're done. Data available for reading or writing. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_wait_read__alsa(ma_device* pDevice)\n{\n    return ma_device_wait__alsa(pDevice, (ma_snd_pcm_t*)pDevice->alsa.pPCMCapture, (struct pollfd*)pDevice->alsa.pPollDescriptorsCapture, pDevice->alsa.pollDescriptorCountCapture + 1, POLLIN); /* +1 to account for the wakeup descriptor. */\n}\n\nstatic ma_result ma_device_wait_write__alsa(ma_device* pDevice)\n{\n    return ma_device_wait__alsa(pDevice, (ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback, (struct pollfd*)pDevice->alsa.pPollDescriptorsPlayback, pDevice->alsa.pollDescriptorCountPlayback + 1, POLLOUT); /* +1 to account for the wakeup descriptor. */\n}\n\nstatic ma_result ma_device_read__alsa(ma_device* pDevice, void* pFramesOut, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    ma_snd_pcm_sframes_t resultALSA = 0;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    while (ma_device_get_state(pDevice) == ma_device_state_started) {\n        ma_result result;\n\n        /* The first thing to do is wait for data to become available for reading. This will return an error code if the device has been stopped. */\n        result = ma_device_wait_read__alsa(pDevice);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        /* Getting here means we should have data available. */\n        resultALSA = ((ma_snd_pcm_readi_proc)pDevice->pContext->alsa.snd_pcm_readi)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture, pFramesOut, frameCount);\n        if (resultALSA >= 0) {\n            break;  /* Success. */\n        } else {\n            if (resultALSA == -EAGAIN) {\n                /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"EGAIN (read)\\n\");*/\n                continue;   /* Try again. */\n            } else if (resultALSA == -EPIPE) {\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"EPIPE (read)\\n\");\n\n                /* Overrun. Recover and try again. If this fails we need to return an error. */\n                resultALSA = ((ma_snd_pcm_recover_proc)pDevice->pContext->alsa.snd_pcm_recover)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture, resultALSA, MA_TRUE);\n                if (resultALSA < 0) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to recover device after overrun.\");\n                    return ma_result_from_errno((int)-resultALSA);\n                }\n\n                resultALSA = ((ma_snd_pcm_start_proc)pDevice->pContext->alsa.snd_pcm_start)((ma_snd_pcm_t*)pDevice->alsa.pPCMCapture);\n                if (resultALSA < 0) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to start device after underrun.\");\n                    return ma_result_from_errno((int)-resultALSA);\n                }\n\n                continue;   /* Try reading again. */\n            }\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = resultALSA;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_write__alsa(ma_device* pDevice, const void* pFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    ma_snd_pcm_sframes_t resultALSA = 0;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pFrames != NULL);\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    while (ma_device_get_state(pDevice) == ma_device_state_started) {\n        ma_result result;\n\n        /* The first thing to do is wait for space to become available for writing. This will return an error code if the device has been stopped. */\n        result = ma_device_wait_write__alsa(pDevice);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        resultALSA = ((ma_snd_pcm_writei_proc)pDevice->pContext->alsa.snd_pcm_writei)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback, pFrames, frameCount);\n        if (resultALSA >= 0) {\n            break;  /* Success. */\n        } else {\n            if (resultALSA == -EAGAIN) {\n                /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"EGAIN (write)\\n\");*/\n                continue;   /* Try again. */\n            } else if (resultALSA == -EPIPE) {\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"EPIPE (write)\\n\");\n\n                /* Underrun. Recover and try again. If this fails we need to return an error. */\n                resultALSA = ((ma_snd_pcm_recover_proc)pDevice->pContext->alsa.snd_pcm_recover)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback, resultALSA, MA_TRUE);    /* MA_TRUE=silent (don't print anything on error). */\n                if (resultALSA < 0) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to recover device after underrun.\");\n                    return ma_result_from_errno((int)-resultALSA);\n                }\n\n                /*\n                In my testing I have had a situation where writei() does not automatically restart the device even though I've set it\n                up as such in the software parameters. What will happen is writei() will block indefinitely even though the number of\n                frames is well beyond the auto-start threshold. To work around this I've needed to add an explicit start here. Not sure\n                if this is me just being stupid and not recovering the device properly, but this definitely feels like something isn't\n                quite right here.\n                */\n                resultALSA = ((ma_snd_pcm_start_proc)pDevice->pContext->alsa.snd_pcm_start)((ma_snd_pcm_t*)pDevice->alsa.pPCMPlayback);\n                if (resultALSA < 0) {\n                    ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] Failed to start device after underrun.\");\n                    return ma_result_from_errno((int)-resultALSA);\n                }\n\n                continue;   /* Try writing again. */\n            }\n        }\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = resultALSA;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_data_loop_wakeup__alsa(ma_device* pDevice)\n{\n    ma_uint64 t = 1;\n    int resultWrite = 0;\n\n    MA_ASSERT(pDevice != NULL);\n\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Waking up...\\n\");\n\n    /* Write to an eventfd to trigger a wakeup from poll() and abort any reading or writing. */\n    if (pDevice->alsa.pPollDescriptorsCapture != NULL) {\n        resultWrite = write(pDevice->alsa.wakeupfdCapture, &t, sizeof(t));\n    }\n    if (pDevice->alsa.pPollDescriptorsPlayback != NULL) {\n        resultWrite = write(pDevice->alsa.wakeupfdPlayback, &t, sizeof(t));\n    }\n\n    if (resultWrite < 0) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[ALSA] write() failed.\\n\");\n        return ma_result_from_errno(errno);\n    }\n\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[ALSA] Waking up completed successfully.\\n\");\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__alsa(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_alsa);\n\n    /* Clean up memory for memory leak checkers. */\n    ((ma_snd_config_update_free_global_proc)pContext->alsa.snd_config_update_free_global)();\n\n#ifndef MA_NO_RUNTIME_LINKING\n    ma_dlclose(ma_context_get_log(pContext), pContext->alsa.asoundSO);\n#endif\n\n    ma_mutex_uninit(&pContext->alsa.internalDeviceEnumLock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__alsa(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    ma_result result;\n#ifndef MA_NO_RUNTIME_LINKING\n    const char* libasoundNames[] = {\n        \"libasound.so.2\",\n        \"libasound.so\"\n    };\n    size_t i;\n\n    for (i = 0; i < ma_countof(libasoundNames); ++i) {\n        pContext->alsa.asoundSO = ma_dlopen(ma_context_get_log(pContext), libasoundNames[i]);\n        if (pContext->alsa.asoundSO != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->alsa.asoundSO == NULL) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"[ALSA] Failed to open shared object.\\n\");\n        return MA_NO_BACKEND;\n    }\n\n    pContext->alsa.snd_pcm_open                           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_open\");\n    pContext->alsa.snd_pcm_close                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_close\");\n    pContext->alsa.snd_pcm_hw_params_sizeof               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_sizeof\");\n    pContext->alsa.snd_pcm_hw_params_any                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_any\");\n    pContext->alsa.snd_pcm_hw_params_set_format           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_format\");\n    pContext->alsa.snd_pcm_hw_params_set_format_first     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_format_first\");\n    pContext->alsa.snd_pcm_hw_params_get_format_mask      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_format_mask\");\n    pContext->alsa.snd_pcm_hw_params_set_channels         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_channels\");\n    pContext->alsa.snd_pcm_hw_params_set_channels_near    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_channels_near\");\n    pContext->alsa.snd_pcm_hw_params_set_channels_minmax  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_channels_minmax\");\n    pContext->alsa.snd_pcm_hw_params_set_rate_resample    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_rate_resample\");\n    pContext->alsa.snd_pcm_hw_params_set_rate             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_rate\");\n    pContext->alsa.snd_pcm_hw_params_set_rate_near        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_rate_near\");\n    pContext->alsa.snd_pcm_hw_params_set_buffer_size_near = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_buffer_size_near\");\n    pContext->alsa.snd_pcm_hw_params_set_periods_near     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_periods_near\");\n    pContext->alsa.snd_pcm_hw_params_set_access           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_set_access\");\n    pContext->alsa.snd_pcm_hw_params_get_format           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_format\");\n    pContext->alsa.snd_pcm_hw_params_get_channels         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_channels\");\n    pContext->alsa.snd_pcm_hw_params_get_channels_min     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_channels_min\");\n    pContext->alsa.snd_pcm_hw_params_get_channels_max     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_channels_max\");\n    pContext->alsa.snd_pcm_hw_params_get_rate             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_rate\");\n    pContext->alsa.snd_pcm_hw_params_get_rate_min         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_rate_min\");\n    pContext->alsa.snd_pcm_hw_params_get_rate_max         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_rate_max\");\n    pContext->alsa.snd_pcm_hw_params_get_buffer_size      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_buffer_size\");\n    pContext->alsa.snd_pcm_hw_params_get_periods          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_periods\");\n    pContext->alsa.snd_pcm_hw_params_get_access           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_get_access\");\n    pContext->alsa.snd_pcm_hw_params_test_format          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_test_format\");\n    pContext->alsa.snd_pcm_hw_params_test_channels        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_test_channels\");\n    pContext->alsa.snd_pcm_hw_params_test_rate            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params_test_rate\");\n    pContext->alsa.snd_pcm_hw_params                      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_hw_params\");\n    pContext->alsa.snd_pcm_sw_params_sizeof               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_sizeof\");\n    pContext->alsa.snd_pcm_sw_params_current              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_current\");\n    pContext->alsa.snd_pcm_sw_params_get_boundary         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_get_boundary\");\n    pContext->alsa.snd_pcm_sw_params_set_avail_min        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_set_avail_min\");\n    pContext->alsa.snd_pcm_sw_params_set_start_threshold  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_set_start_threshold\");\n    pContext->alsa.snd_pcm_sw_params_set_stop_threshold   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params_set_stop_threshold\");\n    pContext->alsa.snd_pcm_sw_params                      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_sw_params\");\n    pContext->alsa.snd_pcm_format_mask_sizeof             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_format_mask_sizeof\");\n    pContext->alsa.snd_pcm_format_mask_test               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_format_mask_test\");\n    pContext->alsa.snd_pcm_get_chmap                      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_get_chmap\");\n    pContext->alsa.snd_pcm_state                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_state\");\n    pContext->alsa.snd_pcm_prepare                        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_prepare\");\n    pContext->alsa.snd_pcm_start                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_start\");\n    pContext->alsa.snd_pcm_drop                           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_drop\");\n    pContext->alsa.snd_pcm_drain                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_drain\");\n    pContext->alsa.snd_pcm_reset                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_reset\");\n    pContext->alsa.snd_device_name_hint                   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_device_name_hint\");\n    pContext->alsa.snd_device_name_get_hint               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_device_name_get_hint\");\n    pContext->alsa.snd_card_get_index                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_card_get_index\");\n    pContext->alsa.snd_device_name_free_hint              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_device_name_free_hint\");\n    pContext->alsa.snd_pcm_mmap_begin                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_mmap_begin\");\n    pContext->alsa.snd_pcm_mmap_commit                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_mmap_commit\");\n    pContext->alsa.snd_pcm_recover                        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_recover\");\n    pContext->alsa.snd_pcm_readi                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_readi\");\n    pContext->alsa.snd_pcm_writei                         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_writei\");\n    pContext->alsa.snd_pcm_avail                          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_avail\");\n    pContext->alsa.snd_pcm_avail_update                   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_avail_update\");\n    pContext->alsa.snd_pcm_wait                           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_wait\");\n    pContext->alsa.snd_pcm_nonblock                       = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_nonblock\");\n    pContext->alsa.snd_pcm_info                           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_info\");\n    pContext->alsa.snd_pcm_info_sizeof                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_info_sizeof\");\n    pContext->alsa.snd_pcm_info_get_name                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_info_get_name\");\n    pContext->alsa.snd_pcm_poll_descriptors               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_poll_descriptors\");\n    pContext->alsa.snd_pcm_poll_descriptors_count         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_poll_descriptors_count\");\n    pContext->alsa.snd_pcm_poll_descriptors_revents       = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_pcm_poll_descriptors_revents\");\n    pContext->alsa.snd_config_update_free_global          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->alsa.asoundSO, \"snd_config_update_free_global\");\n#else\n    /* The system below is just for type safety. */\n    ma_snd_pcm_open_proc                           _snd_pcm_open                           = snd_pcm_open;\n    ma_snd_pcm_close_proc                          _snd_pcm_close                          = snd_pcm_close;\n    ma_snd_pcm_hw_params_sizeof_proc               _snd_pcm_hw_params_sizeof               = snd_pcm_hw_params_sizeof;\n    ma_snd_pcm_hw_params_any_proc                  _snd_pcm_hw_params_any                  = snd_pcm_hw_params_any;\n    ma_snd_pcm_hw_params_set_format_proc           _snd_pcm_hw_params_set_format           = snd_pcm_hw_params_set_format;\n    ma_snd_pcm_hw_params_set_format_first_proc     _snd_pcm_hw_params_set_format_first     = snd_pcm_hw_params_set_format_first;\n    ma_snd_pcm_hw_params_get_format_mask_proc      _snd_pcm_hw_params_get_format_mask      = snd_pcm_hw_params_get_format_mask;\n    ma_snd_pcm_hw_params_set_channels_proc         _snd_pcm_hw_params_set_channels         = snd_pcm_hw_params_set_channels;\n    ma_snd_pcm_hw_params_set_channels_near_proc    _snd_pcm_hw_params_set_channels_near    = snd_pcm_hw_params_set_channels_near;\n    ma_snd_pcm_hw_params_set_channels_minmax_proc  _snd_pcm_hw_params_set_channels_minmax  = snd_pcm_hw_params_set_channels_minmax;\n    ma_snd_pcm_hw_params_set_rate_resample_proc    _snd_pcm_hw_params_set_rate_resample    = snd_pcm_hw_params_set_rate_resample;\n    ma_snd_pcm_hw_params_set_rate_proc             _snd_pcm_hw_params_set_rate             = snd_pcm_hw_params_set_rate;\n    ma_snd_pcm_hw_params_set_rate_near_proc        _snd_pcm_hw_params_set_rate_near        = snd_pcm_hw_params_set_rate_near;\n    ma_snd_pcm_hw_params_set_rate_minmax_proc      _snd_pcm_hw_params_set_rate_minmax      = snd_pcm_hw_params_set_rate_minmax;\n    ma_snd_pcm_hw_params_set_buffer_size_near_proc _snd_pcm_hw_params_set_buffer_size_near = snd_pcm_hw_params_set_buffer_size_near;\n    ma_snd_pcm_hw_params_set_periods_near_proc     _snd_pcm_hw_params_set_periods_near     = snd_pcm_hw_params_set_periods_near;\n    ma_snd_pcm_hw_params_set_access_proc           _snd_pcm_hw_params_set_access           = snd_pcm_hw_params_set_access;\n    ma_snd_pcm_hw_params_get_format_proc           _snd_pcm_hw_params_get_format           = snd_pcm_hw_params_get_format;\n    ma_snd_pcm_hw_params_get_channels_proc         _snd_pcm_hw_params_get_channels         = snd_pcm_hw_params_get_channels;\n    ma_snd_pcm_hw_params_get_channels_min_proc     _snd_pcm_hw_params_get_channels_min     = snd_pcm_hw_params_get_channels_min;\n    ma_snd_pcm_hw_params_get_channels_max_proc     _snd_pcm_hw_params_get_channels_max     = snd_pcm_hw_params_get_channels_max;\n    ma_snd_pcm_hw_params_get_rate_proc             _snd_pcm_hw_params_get_rate             = snd_pcm_hw_params_get_rate;\n    ma_snd_pcm_hw_params_get_rate_min_proc         _snd_pcm_hw_params_get_rate_min         = snd_pcm_hw_params_get_rate_min;\n    ma_snd_pcm_hw_params_get_rate_max_proc         _snd_pcm_hw_params_get_rate_max         = snd_pcm_hw_params_get_rate_max;\n    ma_snd_pcm_hw_params_get_buffer_size_proc      _snd_pcm_hw_params_get_buffer_size      = snd_pcm_hw_params_get_buffer_size;\n    ma_snd_pcm_hw_params_get_periods_proc          _snd_pcm_hw_params_get_periods          = snd_pcm_hw_params_get_periods;\n    ma_snd_pcm_hw_params_get_access_proc           _snd_pcm_hw_params_get_access           = snd_pcm_hw_params_get_access;\n    ma_snd_pcm_hw_params_test_format_proc          _snd_pcm_hw_params_test_format          = snd_pcm_hw_params_test_format;\n    ma_snd_pcm_hw_params_test_channels_proc        _snd_pcm_hw_params_test_channels        = snd_pcm_hw_params_test_channels;\n    ma_snd_pcm_hw_params_test_rate_proc            _snd_pcm_hw_params_test_rate            = snd_pcm_hw_params_test_rate;\n    ma_snd_pcm_hw_params_proc                      _snd_pcm_hw_params                      = snd_pcm_hw_params;\n    ma_snd_pcm_sw_params_sizeof_proc               _snd_pcm_sw_params_sizeof               = snd_pcm_sw_params_sizeof;\n    ma_snd_pcm_sw_params_current_proc              _snd_pcm_sw_params_current              = snd_pcm_sw_params_current;\n    ma_snd_pcm_sw_params_get_boundary_proc         _snd_pcm_sw_params_get_boundary         = snd_pcm_sw_params_get_boundary;\n    ma_snd_pcm_sw_params_set_avail_min_proc        _snd_pcm_sw_params_set_avail_min        = snd_pcm_sw_params_set_avail_min;\n    ma_snd_pcm_sw_params_set_start_threshold_proc  _snd_pcm_sw_params_set_start_threshold  = snd_pcm_sw_params_set_start_threshold;\n    ma_snd_pcm_sw_params_set_stop_threshold_proc   _snd_pcm_sw_params_set_stop_threshold   = snd_pcm_sw_params_set_stop_threshold;\n    ma_snd_pcm_sw_params_proc                      _snd_pcm_sw_params                      = snd_pcm_sw_params;\n    ma_snd_pcm_format_mask_sizeof_proc             _snd_pcm_format_mask_sizeof             = snd_pcm_format_mask_sizeof;\n    ma_snd_pcm_format_mask_test_proc               _snd_pcm_format_mask_test               = snd_pcm_format_mask_test;\n    ma_snd_pcm_get_chmap_proc                      _snd_pcm_get_chmap                      = snd_pcm_get_chmap;\n    ma_snd_pcm_state_proc                          _snd_pcm_state                          = snd_pcm_state;\n    ma_snd_pcm_prepare_proc                        _snd_pcm_prepare                        = snd_pcm_prepare;\n    ma_snd_pcm_start_proc                          _snd_pcm_start                          = snd_pcm_start;\n    ma_snd_pcm_drop_proc                           _snd_pcm_drop                           = snd_pcm_drop;\n    ma_snd_pcm_drain_proc                          _snd_pcm_drain                          = snd_pcm_drain;\n    ma_snd_pcm_reset_proc                          _snd_pcm_reset                          = snd_pcm_reset;\n    ma_snd_device_name_hint_proc                   _snd_device_name_hint                   = snd_device_name_hint;\n    ma_snd_device_name_get_hint_proc               _snd_device_name_get_hint               = snd_device_name_get_hint;\n    ma_snd_card_get_index_proc                     _snd_card_get_index                     = snd_card_get_index;\n    ma_snd_device_name_free_hint_proc              _snd_device_name_free_hint              = snd_device_name_free_hint;\n    ma_snd_pcm_mmap_begin_proc                     _snd_pcm_mmap_begin                     = snd_pcm_mmap_begin;\n    ma_snd_pcm_mmap_commit_proc                    _snd_pcm_mmap_commit                    = snd_pcm_mmap_commit;\n    ma_snd_pcm_recover_proc                        _snd_pcm_recover                        = snd_pcm_recover;\n    ma_snd_pcm_readi_proc                          _snd_pcm_readi                          = snd_pcm_readi;\n    ma_snd_pcm_writei_proc                         _snd_pcm_writei                         = snd_pcm_writei;\n    ma_snd_pcm_avail_proc                          _snd_pcm_avail                          = snd_pcm_avail;\n    ma_snd_pcm_avail_update_proc                   _snd_pcm_avail_update                   = snd_pcm_avail_update;\n    ma_snd_pcm_wait_proc                           _snd_pcm_wait                           = snd_pcm_wait;\n    ma_snd_pcm_nonblock_proc                       _snd_pcm_nonblock                       = snd_pcm_nonblock;\n    ma_snd_pcm_info_proc                           _snd_pcm_info                           = snd_pcm_info;\n    ma_snd_pcm_info_sizeof_proc                    _snd_pcm_info_sizeof                    = snd_pcm_info_sizeof;\n    ma_snd_pcm_info_get_name_proc                  _snd_pcm_info_get_name                  = snd_pcm_info_get_name;\n    ma_snd_pcm_poll_descriptors_proc               _snd_pcm_poll_descriptors               = snd_pcm_poll_descriptors;\n    ma_snd_pcm_poll_descriptors_count_proc         _snd_pcm_poll_descriptors_count         = snd_pcm_poll_descriptors_count;\n    ma_snd_pcm_poll_descriptors_revents_proc       _snd_pcm_poll_descriptors_revents       = snd_pcm_poll_descriptors_revents;\n    ma_snd_config_update_free_global_proc          _snd_config_update_free_global          = snd_config_update_free_global;\n\n    pContext->alsa.snd_pcm_open                           = (ma_proc)_snd_pcm_open;\n    pContext->alsa.snd_pcm_close                          = (ma_proc)_snd_pcm_close;\n    pContext->alsa.snd_pcm_hw_params_sizeof               = (ma_proc)_snd_pcm_hw_params_sizeof;\n    pContext->alsa.snd_pcm_hw_params_any                  = (ma_proc)_snd_pcm_hw_params_any;\n    pContext->alsa.snd_pcm_hw_params_set_format           = (ma_proc)_snd_pcm_hw_params_set_format;\n    pContext->alsa.snd_pcm_hw_params_set_format_first     = (ma_proc)_snd_pcm_hw_params_set_format_first;\n    pContext->alsa.snd_pcm_hw_params_get_format_mask      = (ma_proc)_snd_pcm_hw_params_get_format_mask;\n    pContext->alsa.snd_pcm_hw_params_set_channels         = (ma_proc)_snd_pcm_hw_params_set_channels;\n    pContext->alsa.snd_pcm_hw_params_set_channels_near    = (ma_proc)_snd_pcm_hw_params_set_channels_near;\n    pContext->alsa.snd_pcm_hw_params_set_channels_minmax  = (ma_proc)_snd_pcm_hw_params_set_channels_minmax;\n    pContext->alsa.snd_pcm_hw_params_set_rate_resample    = (ma_proc)_snd_pcm_hw_params_set_rate_resample;\n    pContext->alsa.snd_pcm_hw_params_set_rate             = (ma_proc)_snd_pcm_hw_params_set_rate;\n    pContext->alsa.snd_pcm_hw_params_set_rate_near        = (ma_proc)_snd_pcm_hw_params_set_rate_near;\n    pContext->alsa.snd_pcm_hw_params_set_rate_minmax      = (ma_proc)_snd_pcm_hw_params_set_rate_minmax;\n    pContext->alsa.snd_pcm_hw_params_set_buffer_size_near = (ma_proc)_snd_pcm_hw_params_set_buffer_size_near;\n    pContext->alsa.snd_pcm_hw_params_set_periods_near     = (ma_proc)_snd_pcm_hw_params_set_periods_near;\n    pContext->alsa.snd_pcm_hw_params_set_access           = (ma_proc)_snd_pcm_hw_params_set_access;\n    pContext->alsa.snd_pcm_hw_params_get_format           = (ma_proc)_snd_pcm_hw_params_get_format;\n    pContext->alsa.snd_pcm_hw_params_get_channels         = (ma_proc)_snd_pcm_hw_params_get_channels;\n    pContext->alsa.snd_pcm_hw_params_get_channels_min     = (ma_proc)_snd_pcm_hw_params_get_channels_min;\n    pContext->alsa.snd_pcm_hw_params_get_channels_max     = (ma_proc)_snd_pcm_hw_params_get_channels_max;\n    pContext->alsa.snd_pcm_hw_params_get_rate             = (ma_proc)_snd_pcm_hw_params_get_rate;\n    pContext->alsa.snd_pcm_hw_params_get_rate_min         = (ma_proc)_snd_pcm_hw_params_get_rate_min;\n    pContext->alsa.snd_pcm_hw_params_get_rate_max         = (ma_proc)_snd_pcm_hw_params_get_rate_max;\n    pContext->alsa.snd_pcm_hw_params_get_buffer_size      = (ma_proc)_snd_pcm_hw_params_get_buffer_size;\n    pContext->alsa.snd_pcm_hw_params_get_periods          = (ma_proc)_snd_pcm_hw_params_get_periods;\n    pContext->alsa.snd_pcm_hw_params_get_access           = (ma_proc)_snd_pcm_hw_params_get_access;\n    pContext->alsa.snd_pcm_hw_params_test_format          = (ma_proc)_snd_pcm_hw_params_test_format;\n    pContext->alsa.snd_pcm_hw_params_test_channels        = (ma_proc)_snd_pcm_hw_params_test_channels;\n    pContext->alsa.snd_pcm_hw_params_test_rate            = (ma_proc)_snd_pcm_hw_params_test_rate;\n    pContext->alsa.snd_pcm_hw_params                      = (ma_proc)_snd_pcm_hw_params;\n    pContext->alsa.snd_pcm_sw_params_sizeof               = (ma_proc)_snd_pcm_sw_params_sizeof;\n    pContext->alsa.snd_pcm_sw_params_current              = (ma_proc)_snd_pcm_sw_params_current;\n    pContext->alsa.snd_pcm_sw_params_get_boundary         = (ma_proc)_snd_pcm_sw_params_get_boundary;\n    pContext->alsa.snd_pcm_sw_params_set_avail_min        = (ma_proc)_snd_pcm_sw_params_set_avail_min;\n    pContext->alsa.snd_pcm_sw_params_set_start_threshold  = (ma_proc)_snd_pcm_sw_params_set_start_threshold;\n    pContext->alsa.snd_pcm_sw_params_set_stop_threshold   = (ma_proc)_snd_pcm_sw_params_set_stop_threshold;\n    pContext->alsa.snd_pcm_sw_params                      = (ma_proc)_snd_pcm_sw_params;\n    pContext->alsa.snd_pcm_format_mask_sizeof             = (ma_proc)_snd_pcm_format_mask_sizeof;\n    pContext->alsa.snd_pcm_format_mask_test               = (ma_proc)_snd_pcm_format_mask_test;\n    pContext->alsa.snd_pcm_get_chmap                      = (ma_proc)_snd_pcm_get_chmap;\n    pContext->alsa.snd_pcm_state                          = (ma_proc)_snd_pcm_state;\n    pContext->alsa.snd_pcm_prepare                        = (ma_proc)_snd_pcm_prepare;\n    pContext->alsa.snd_pcm_start                          = (ma_proc)_snd_pcm_start;\n    pContext->alsa.snd_pcm_drop                           = (ma_proc)_snd_pcm_drop;\n    pContext->alsa.snd_pcm_drain                          = (ma_proc)_snd_pcm_drain;\n    pContext->alsa.snd_pcm_reset                          = (ma_proc)_snd_pcm_reset;\n    pContext->alsa.snd_device_name_hint                   = (ma_proc)_snd_device_name_hint;\n    pContext->alsa.snd_device_name_get_hint               = (ma_proc)_snd_device_name_get_hint;\n    pContext->alsa.snd_card_get_index                     = (ma_proc)_snd_card_get_index;\n    pContext->alsa.snd_device_name_free_hint              = (ma_proc)_snd_device_name_free_hint;\n    pContext->alsa.snd_pcm_mmap_begin                     = (ma_proc)_snd_pcm_mmap_begin;\n    pContext->alsa.snd_pcm_mmap_commit                    = (ma_proc)_snd_pcm_mmap_commit;\n    pContext->alsa.snd_pcm_recover                        = (ma_proc)_snd_pcm_recover;\n    pContext->alsa.snd_pcm_readi                          = (ma_proc)_snd_pcm_readi;\n    pContext->alsa.snd_pcm_writei                         = (ma_proc)_snd_pcm_writei;\n    pContext->alsa.snd_pcm_avail                          = (ma_proc)_snd_pcm_avail;\n    pContext->alsa.snd_pcm_avail_update                   = (ma_proc)_snd_pcm_avail_update;\n    pContext->alsa.snd_pcm_wait                           = (ma_proc)_snd_pcm_wait;\n    pContext->alsa.snd_pcm_nonblock                       = (ma_proc)_snd_pcm_nonblock;\n    pContext->alsa.snd_pcm_info                           = (ma_proc)_snd_pcm_info;\n    pContext->alsa.snd_pcm_info_sizeof                    = (ma_proc)_snd_pcm_info_sizeof;\n    pContext->alsa.snd_pcm_info_get_name                  = (ma_proc)_snd_pcm_info_get_name;\n    pContext->alsa.snd_pcm_poll_descriptors               = (ma_proc)_snd_pcm_poll_descriptors;\n    pContext->alsa.snd_pcm_poll_descriptors_count         = (ma_proc)_snd_pcm_poll_descriptors_count;\n    pContext->alsa.snd_pcm_poll_descriptors_revents       = (ma_proc)_snd_pcm_poll_descriptors_revents;\n    pContext->alsa.snd_config_update_free_global          = (ma_proc)_snd_config_update_free_global;\n#endif\n\n    pContext->alsa.useVerboseDeviceEnumeration = pConfig->alsa.useVerboseDeviceEnumeration;\n\n    result = ma_mutex_init(&pContext->alsa.internalDeviceEnumLock);\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[ALSA] WARNING: Failed to initialize mutex for internal device enumeration.\");\n        return result;\n    }\n\n    pCallbacks->onContextInit             = ma_context_init__alsa;\n    pCallbacks->onContextUninit           = ma_context_uninit__alsa;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__alsa;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__alsa;\n    pCallbacks->onDeviceInit              = ma_device_init__alsa;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__alsa;\n    pCallbacks->onDeviceStart             = ma_device_start__alsa;\n    pCallbacks->onDeviceStop              = ma_device_stop__alsa;\n    pCallbacks->onDeviceRead              = ma_device_read__alsa;\n    pCallbacks->onDeviceWrite             = ma_device_write__alsa;\n    pCallbacks->onDeviceDataLoop          = NULL;\n    pCallbacks->onDeviceDataLoopWakeup    = ma_device_data_loop_wakeup__alsa;\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_ALSA */\n\n\n\n/******************************************************************************\n\nPulseAudio Backend\n\n******************************************************************************/\n#ifdef MA_HAS_PULSEAUDIO\n/*\nThe PulseAudio API, along with Apple's Core Audio, is the worst of the mainstream audio APIs. This is a brief description of what's going on\nin the PulseAudio backend. I apologize if this gets a bit ranty for your liking - you might want to skip this discussion.\n\nPulseAudio has something they call the \"Simple API\", which unfortunately isn't suitable for miniaudio. I've not seen anywhere where it\nallows you to enumerate over devices, nor does it seem to support the ability to stop and start streams. Looking at the documentation, it\nappears as though the stream is constantly running and you prevent sound from being emitted or captured by simply not calling the read or\nwrite functions. This is not a professional solution as it would be much better to *actually* stop the underlying stream. Perhaps the\nsimple API has some smarts to do this automatically, but I'm not sure. Another limitation with the simple API is that it seems inefficient\nwhen you want to have multiple streams to a single context. For these reasons, miniaudio is not using the simple API.\n\nSince we're not using the simple API, we're left with the asynchronous API as our only other option. And boy, is this where it starts to\nget fun, and I don't mean that in a good way...\n\nThe problems start with the very name of the API - \"asynchronous\". Yes, this is an asynchronous oriented API which means your commands\ndon't immediately take effect. You instead need to issue your commands, and then wait for them to complete. The waiting mechanism is\nenabled through the use of a \"main loop\". In the asynchronous API you cannot get away from the main loop, and the main loop is where almost\nall of PulseAudio's problems stem from.\n\nWhen you first initialize PulseAudio you need an object referred to as \"main loop\". You can implement this yourself by defining your own\nvtable, but it's much easier to just use one of the built-in main loop implementations. There's two generic implementations called\npa_mainloop and pa_threaded_mainloop, and another implementation specific to GLib called pa_glib_mainloop. We're using pa_threaded_mainloop\nbecause it simplifies management of the worker thread. The idea of the main loop object is pretty self explanatory - you're supposed to use\nit to implement a worker thread which runs in a loop. The main loop is where operations are actually executed.\n\nTo initialize the main loop, you just use `pa_threaded_mainloop_new()`. This is the first function you'll call. You can then get a pointer\nto the vtable with `pa_threaded_mainloop_get_api()` (the main loop vtable is called `pa_mainloop_api`). Again, you can bypass the threaded\nmain loop object entirely and just implement `pa_mainloop_api` directly, but there's no need for it unless you're doing something extremely\nspecialized such as if you want to integrate it into your application's existing main loop infrastructure.\n\n(EDIT 2021-01-26: miniaudio is no longer using `pa_threaded_mainloop` due to this issue: https://github.com/mackron/miniaudio/issues/262.\nIt is now using `pa_mainloop` which turns out to be a simpler solution anyway. The rest of this rant still applies, however.)\n\nOnce you have your main loop vtable (the `pa_mainloop_api` object) you can create the PulseAudio context. This is very similar to\nminiaudio's context and they map to each other quite well. You have one context to many streams, which is basically the same as miniaudio's\none `ma_context` to many `ma_device`s. Here's where it starts to get annoying, however. When you first create the PulseAudio context, which\nis done with `pa_context_new()`, it's not actually connected to anything. When you connect, you call `pa_context_connect()`. However, if\nyou remember, PulseAudio is an asynchronous API. That means you cannot just assume the context is connected after `pa_context_context()`\nhas returned. You instead need to wait for it to connect. To do this, you need to either wait for a callback to get fired, which you can\nset with `pa_context_set_state_callback()`, or you can continuously poll the context's state. Either way, you need to run this in a loop.\nAll objects from here out are created from the context, and, I believe, you can't be creating these objects until the context is connected.\nThis waiting loop is therefore unavoidable. In order for the waiting to ever complete, however, the main loop needs to be running. Before\nattempting to connect the context, the main loop needs to be started with `pa_threaded_mainloop_start()`.\n\nThe reason for this asynchronous design is to support cases where you're connecting to a remote server, say through a local network or an\ninternet connection. However, the *VAST* majority of cases don't involve this at all - they just connect to a local \"server\" running on the\nhost machine. The fact that this would be the default rather than making `pa_context_connect()` synchronous tends to boggle the mind.\n\nOnce the context has been created and connected you can start creating a stream. A PulseAudio stream is analogous to miniaudio's device.\nThe initialization of a stream is fairly standard - you configure some attributes (analogous to miniaudio's device config) and then call\n`pa_stream_new()` to actually create it. Here is where we start to get into \"operations\". When configuring the stream, you can get\ninformation about the source (such as sample format, sample rate, etc.), however it's not synchronous. Instead, a `pa_operation` object\nis returned from `pa_context_get_source_info_by_name()` (capture) or `pa_context_get_sink_info_by_name()` (playback). Then, you need to\nrun a loop (again!) to wait for the operation to complete which you can determine via a callback or polling, just like we did with the\ncontext. Then, as an added bonus, you need to decrement the reference counter of the `pa_operation` object to ensure memory is cleaned up.\nAll of that just to retrieve basic information about a device!\n\nOnce the basic information about the device has been retrieved, miniaudio can now create the stream with `ma_stream_new()`. Like the\ncontext, this needs to be connected. But we need to be careful here, because we're now about to introduce one of the most horrific design\nchoices in PulseAudio.\n\nPulseAudio allows you to specify a callback that is fired when data can be written to or read from a stream. The language is important here\nbecause PulseAudio takes it literally, specifically the \"can be\". You would think these callbacks would be appropriate as the place for\nwriting and reading data to and from the stream, and that would be right, except when it's not. When you initialize the stream, you can\nset a flag that tells PulseAudio to not start the stream automatically. This is required because miniaudio does not auto-start devices\nstraight after initialization - you need to call `ma_device_start()` manually. The problem is that even when this flag is specified,\nPulseAudio will immediately fire its write or read callback. This is *technically* correct (based on the wording in the documentation)\nbecause indeed, data *can* be written at this point. The problem is that it's not *practical*. It makes sense that the write/read callback\nwould be where a program will want to write or read data to or from the stream, but when it's called before the application has even\nrequested that the stream be started, it's just not practical because the program probably isn't ready for any kind of data delivery at\nthat point (it may still need to load files or whatnot). Instead, this callback should only be fired when the application requests the\nstream be started which is how it works with literally *every* other callback-based audio API. Since miniaudio forbids firing of the data\ncallback until the device has been started (as it should be with *all* callback based APIs), logic needs to be added to ensure miniaudio\ndoesn't just blindly fire the application-defined data callback from within the PulseAudio callback before the stream has actually been\nstarted. The device state is used for this - if the state is anything other than `ma_device_state_starting` or `ma_device_state_started`, the main data\ncallback is not fired.\n\nThis, unfortunately, is not the end of the problems with the PulseAudio write callback. Any normal callback based audio API will\ncontinuously fire the callback at regular intervals based on the size of the internal buffer. This will only ever be fired when the device\nis running, and will be fired regardless of whether or not the user actually wrote anything to the device/stream. This not the case in\nPulseAudio. In PulseAudio, the data callback will *only* be called if you wrote something to it previously. That means, if you don't call\n`pa_stream_write()`, the callback will not get fired. On the surface you wouldn't think this would matter because you should be always\nwriting data, and if you don't have anything to write, just write silence. That's fine until you want to drain the stream. You see, if\nyou're continuously writing data to the stream, the stream will never get drained! That means in order to drain the stream, you need to\n*not* write data to it! But remember, when you don't write data to the stream, the callback won't get fired again! Why is draining\nimportant? Because that's how we've defined stopping to work in miniaudio. In miniaudio, stopping the device requires it to be drained\nbefore returning from ma_device_stop(). So we've stopped the device, which requires us to drain, but draining requires us to *not* write\ndata to the stream (or else it won't ever complete draining), but not writing to the stream means the callback won't get fired again!\n\nThis becomes a problem when stopping and then restarting the device. When the device is stopped, it's drained, which requires us to *not*\nwrite anything to the stream. But then, since we didn't write anything to it, the write callback will *never* get called again if we just\nresume the stream naively. This means that starting the stream requires us to write data to the stream from outside the callback. This\ndisconnect is something PulseAudio has got seriously wrong - there should only ever be a single source of data delivery, that being the\ncallback. (I have tried using `pa_stream_flush()` to trigger the write callback to fire, but this just doesn't work for some reason.)\n\nOnce you've created the stream, you need to connect it which involves the whole waiting procedure. This is the same process as the context,\nonly this time you'll poll for the state with `pa_stream_get_status()`. The starting and stopping of a streaming is referred to as\n\"corking\" in PulseAudio. The analogy is corking a barrel. To start the stream, you uncork it, to stop it you cork it. Personally I think\nit's silly - why would you not just call it \"starting\" and \"stopping\" like any other normal audio API? Anyway, the act of corking is, you\nguessed it, asynchronous. This means you'll need our waiting loop as usual. Again, why this asynchronous design is the default is\nabsolutely beyond me. Would it really be that hard to just make it run synchronously?\n\nTeardown is pretty simple (what?!). It's just a matter of calling the relevant `_unref()` function on each object in reverse order that\nthey were initialized in.\n\nThat's about it from the PulseAudio side. A bit ranty, I know, but they really need to fix that main loop and callback system. They're\nembarrassingly unpractical. The main loop thing is an easy fix - have synchronous versions of all APIs. If an application wants these to\nrun asynchronously, they can execute them in a separate thread themselves. The desire to run these asynchronously is such a niche\nrequirement - it makes no sense to make it the default. The stream write callback needs to be change, or an alternative provided, that is\nconstantly fired, regardless of whether or not `pa_stream_write()` has been called, and it needs to take a pointer to a buffer as a\nparameter which the program just writes to directly rather than having to call `pa_stream_writable_size()` and `pa_stream_write()`. These\nchanges alone will change PulseAudio from one of the worst audio APIs to one of the best.\n*/\n\n\n/*\nIt is assumed pulseaudio.h is available when linking at compile time. When linking at compile time, we use the declarations in the header\nto check for type safety. We cannot do this when linking at run time because the header might not be available.\n*/\n#ifdef MA_NO_RUNTIME_LINKING\n\n/* pulseaudio.h marks some functions with \"inline\" which isn't always supported. Need to emulate it. */\n#if !defined(__cplusplus)\n    #if defined(__STRICT_ANSI__)\n        #if !defined(inline)\n            #define inline __inline__ __attribute__((always_inline))\n            #define MA_INLINE_DEFINED\n        #endif\n    #endif\n#endif\n#include <pulse/pulseaudio.h>\n#if defined(MA_INLINE_DEFINED)\n    #undef inline\n    #undef MA_INLINE_DEFINED\n#endif\n\n#define MA_PA_OK                                       PA_OK\n#define MA_PA_ERR_ACCESS                               PA_ERR_ACCESS\n#define MA_PA_ERR_INVALID                              PA_ERR_INVALID\n#define MA_PA_ERR_NOENTITY                             PA_ERR_NOENTITY\n#define MA_PA_ERR_NOTSUPPORTED                         PA_ERR_NOTSUPPORTED\n\n#define MA_PA_CHANNELS_MAX                             PA_CHANNELS_MAX\n#define MA_PA_RATE_MAX                                 PA_RATE_MAX\n\ntypedef pa_context_flags_t ma_pa_context_flags_t;\n#define MA_PA_CONTEXT_NOFLAGS                          PA_CONTEXT_NOFLAGS\n#define MA_PA_CONTEXT_NOAUTOSPAWN                      PA_CONTEXT_NOAUTOSPAWN\n#define MA_PA_CONTEXT_NOFAIL                           PA_CONTEXT_NOFAIL\n\ntypedef pa_stream_flags_t ma_pa_stream_flags_t;\n#define MA_PA_STREAM_NOFLAGS                           PA_STREAM_NOFLAGS\n#define MA_PA_STREAM_START_CORKED                      PA_STREAM_START_CORKED\n#define MA_PA_STREAM_INTERPOLATE_TIMING                PA_STREAM_INTERPOLATE_TIMING\n#define MA_PA_STREAM_NOT_MONOTONIC                     PA_STREAM_NOT_MONOTONIC\n#define MA_PA_STREAM_AUTO_TIMING_UPDATE                PA_STREAM_AUTO_TIMING_UPDATE\n#define MA_PA_STREAM_NO_REMAP_CHANNELS                 PA_STREAM_NO_REMAP_CHANNELS\n#define MA_PA_STREAM_NO_REMIX_CHANNELS                 PA_STREAM_NO_REMIX_CHANNELS\n#define MA_PA_STREAM_FIX_FORMAT                        PA_STREAM_FIX_FORMAT\n#define MA_PA_STREAM_FIX_RATE                          PA_STREAM_FIX_RATE\n#define MA_PA_STREAM_FIX_CHANNELS                      PA_STREAM_FIX_CHANNELS\n#define MA_PA_STREAM_DONT_MOVE                         PA_STREAM_DONT_MOVE\n#define MA_PA_STREAM_VARIABLE_RATE                     PA_STREAM_VARIABLE_RATE\n#define MA_PA_STREAM_PEAK_DETECT                       PA_STREAM_PEAK_DETECT\n#define MA_PA_STREAM_START_MUTED                       PA_STREAM_START_MUTED\n#define MA_PA_STREAM_ADJUST_LATENCY                    PA_STREAM_ADJUST_LATENCY\n#define MA_PA_STREAM_EARLY_REQUESTS                    PA_STREAM_EARLY_REQUESTS\n#define MA_PA_STREAM_DONT_INHIBIT_AUTO_SUSPEND         PA_STREAM_DONT_INHIBIT_AUTO_SUSPEND\n#define MA_PA_STREAM_START_UNMUTED                     PA_STREAM_START_UNMUTED\n#define MA_PA_STREAM_FAIL_ON_SUSPEND                   PA_STREAM_FAIL_ON_SUSPEND\n#define MA_PA_STREAM_RELATIVE_VOLUME                   PA_STREAM_RELATIVE_VOLUME\n#define MA_PA_STREAM_PASSTHROUGH                       PA_STREAM_PASSTHROUGH\n\ntypedef pa_sink_flags_t ma_pa_sink_flags_t;\n#define MA_PA_SINK_NOFLAGS                             PA_SINK_NOFLAGS\n#define MA_PA_SINK_HW_VOLUME_CTRL                      PA_SINK_HW_VOLUME_CTRL\n#define MA_PA_SINK_LATENCY                             PA_SINK_LATENCY\n#define MA_PA_SINK_HARDWARE                            PA_SINK_HARDWARE\n#define MA_PA_SINK_NETWORK                             PA_SINK_NETWORK\n#define MA_PA_SINK_HW_MUTE_CTRL                        PA_SINK_HW_MUTE_CTRL\n#define MA_PA_SINK_DECIBEL_VOLUME                      PA_SINK_DECIBEL_VOLUME\n#define MA_PA_SINK_FLAT_VOLUME                         PA_SINK_FLAT_VOLUME\n#define MA_PA_SINK_DYNAMIC_LATENCY                     PA_SINK_DYNAMIC_LATENCY\n#define MA_PA_SINK_SET_FORMATS                         PA_SINK_SET_FORMATS\n\ntypedef pa_source_flags_t ma_pa_source_flags_t;\n#define MA_PA_SOURCE_NOFLAGS                           PA_SOURCE_NOFLAGS\n#define MA_PA_SOURCE_HW_VOLUME_CTRL                    PA_SOURCE_HW_VOLUME_CTRL\n#define MA_PA_SOURCE_LATENCY                           PA_SOURCE_LATENCY\n#define MA_PA_SOURCE_HARDWARE                          PA_SOURCE_HARDWARE\n#define MA_PA_SOURCE_NETWORK                           PA_SOURCE_NETWORK\n#define MA_PA_SOURCE_HW_MUTE_CTRL                      PA_SOURCE_HW_MUTE_CTRL\n#define MA_PA_SOURCE_DECIBEL_VOLUME                    PA_SOURCE_DECIBEL_VOLUME\n#define MA_PA_SOURCE_DYNAMIC_LATENCY                   PA_SOURCE_DYNAMIC_LATENCY\n#define MA_PA_SOURCE_FLAT_VOLUME                       PA_SOURCE_FLAT_VOLUME\n\ntypedef pa_context_state_t ma_pa_context_state_t;\n#define MA_PA_CONTEXT_UNCONNECTED                      PA_CONTEXT_UNCONNECTED\n#define MA_PA_CONTEXT_CONNECTING                       PA_CONTEXT_CONNECTING\n#define MA_PA_CONTEXT_AUTHORIZING                      PA_CONTEXT_AUTHORIZING\n#define MA_PA_CONTEXT_SETTING_NAME                     PA_CONTEXT_SETTING_NAME\n#define MA_PA_CONTEXT_READY                            PA_CONTEXT_READY\n#define MA_PA_CONTEXT_FAILED                           PA_CONTEXT_FAILED\n#define MA_PA_CONTEXT_TERMINATED                       PA_CONTEXT_TERMINATED\n\ntypedef pa_stream_state_t ma_pa_stream_state_t;\n#define MA_PA_STREAM_UNCONNECTED                       PA_STREAM_UNCONNECTED\n#define MA_PA_STREAM_CREATING                          PA_STREAM_CREATING\n#define MA_PA_STREAM_READY                             PA_STREAM_READY\n#define MA_PA_STREAM_FAILED                            PA_STREAM_FAILED\n#define MA_PA_STREAM_TERMINATED                        PA_STREAM_TERMINATED\n\ntypedef pa_operation_state_t ma_pa_operation_state_t;\n#define MA_PA_OPERATION_RUNNING                        PA_OPERATION_RUNNING\n#define MA_PA_OPERATION_DONE                           PA_OPERATION_DONE\n#define MA_PA_OPERATION_CANCELLED                      PA_OPERATION_CANCELLED\n\ntypedef pa_sink_state_t ma_pa_sink_state_t;\n#define MA_PA_SINK_INVALID_STATE                       PA_SINK_INVALID_STATE\n#define MA_PA_SINK_RUNNING                             PA_SINK_RUNNING\n#define MA_PA_SINK_IDLE                                PA_SINK_IDLE\n#define MA_PA_SINK_SUSPENDED                           PA_SINK_SUSPENDED\n\ntypedef pa_source_state_t ma_pa_source_state_t;\n#define MA_PA_SOURCE_INVALID_STATE                     PA_SOURCE_INVALID_STATE\n#define MA_PA_SOURCE_RUNNING                           PA_SOURCE_RUNNING\n#define MA_PA_SOURCE_IDLE                              PA_SOURCE_IDLE\n#define MA_PA_SOURCE_SUSPENDED                         PA_SOURCE_SUSPENDED\n\ntypedef pa_seek_mode_t ma_pa_seek_mode_t;\n#define MA_PA_SEEK_RELATIVE                            PA_SEEK_RELATIVE\n#define MA_PA_SEEK_ABSOLUTE                            PA_SEEK_ABSOLUTE\n#define MA_PA_SEEK_RELATIVE_ON_READ                    PA_SEEK_RELATIVE_ON_READ\n#define MA_PA_SEEK_RELATIVE_END                        PA_SEEK_RELATIVE_END\n\ntypedef pa_channel_position_t ma_pa_channel_position_t;\n#define MA_PA_CHANNEL_POSITION_INVALID                 PA_CHANNEL_POSITION_INVALID\n#define MA_PA_CHANNEL_POSITION_MONO                    PA_CHANNEL_POSITION_MONO\n#define MA_PA_CHANNEL_POSITION_FRONT_LEFT              PA_CHANNEL_POSITION_FRONT_LEFT\n#define MA_PA_CHANNEL_POSITION_FRONT_RIGHT             PA_CHANNEL_POSITION_FRONT_RIGHT\n#define MA_PA_CHANNEL_POSITION_FRONT_CENTER            PA_CHANNEL_POSITION_FRONT_CENTER\n#define MA_PA_CHANNEL_POSITION_REAR_CENTER             PA_CHANNEL_POSITION_REAR_CENTER\n#define MA_PA_CHANNEL_POSITION_REAR_LEFT               PA_CHANNEL_POSITION_REAR_LEFT\n#define MA_PA_CHANNEL_POSITION_REAR_RIGHT              PA_CHANNEL_POSITION_REAR_RIGHT\n#define MA_PA_CHANNEL_POSITION_LFE                     PA_CHANNEL_POSITION_LFE\n#define MA_PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER    PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER\n#define MA_PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER   PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER\n#define MA_PA_CHANNEL_POSITION_SIDE_LEFT               PA_CHANNEL_POSITION_SIDE_LEFT\n#define MA_PA_CHANNEL_POSITION_SIDE_RIGHT              PA_CHANNEL_POSITION_SIDE_RIGHT\n#define MA_PA_CHANNEL_POSITION_AUX0                    PA_CHANNEL_POSITION_AUX0\n#define MA_PA_CHANNEL_POSITION_AUX1                    PA_CHANNEL_POSITION_AUX1\n#define MA_PA_CHANNEL_POSITION_AUX2                    PA_CHANNEL_POSITION_AUX2\n#define MA_PA_CHANNEL_POSITION_AUX3                    PA_CHANNEL_POSITION_AUX3\n#define MA_PA_CHANNEL_POSITION_AUX4                    PA_CHANNEL_POSITION_AUX4\n#define MA_PA_CHANNEL_POSITION_AUX5                    PA_CHANNEL_POSITION_AUX5\n#define MA_PA_CHANNEL_POSITION_AUX6                    PA_CHANNEL_POSITION_AUX6\n#define MA_PA_CHANNEL_POSITION_AUX7                    PA_CHANNEL_POSITION_AUX7\n#define MA_PA_CHANNEL_POSITION_AUX8                    PA_CHANNEL_POSITION_AUX8\n#define MA_PA_CHANNEL_POSITION_AUX9                    PA_CHANNEL_POSITION_AUX9\n#define MA_PA_CHANNEL_POSITION_AUX10                   PA_CHANNEL_POSITION_AUX10\n#define MA_PA_CHANNEL_POSITION_AUX11                   PA_CHANNEL_POSITION_AUX11\n#define MA_PA_CHANNEL_POSITION_AUX12                   PA_CHANNEL_POSITION_AUX12\n#define MA_PA_CHANNEL_POSITION_AUX13                   PA_CHANNEL_POSITION_AUX13\n#define MA_PA_CHANNEL_POSITION_AUX14                   PA_CHANNEL_POSITION_AUX14\n#define MA_PA_CHANNEL_POSITION_AUX15                   PA_CHANNEL_POSITION_AUX15\n#define MA_PA_CHANNEL_POSITION_AUX16                   PA_CHANNEL_POSITION_AUX16\n#define MA_PA_CHANNEL_POSITION_AUX17                   PA_CHANNEL_POSITION_AUX17\n#define MA_PA_CHANNEL_POSITION_AUX18                   PA_CHANNEL_POSITION_AUX18\n#define MA_PA_CHANNEL_POSITION_AUX19                   PA_CHANNEL_POSITION_AUX19\n#define MA_PA_CHANNEL_POSITION_AUX20                   PA_CHANNEL_POSITION_AUX20\n#define MA_PA_CHANNEL_POSITION_AUX21                   PA_CHANNEL_POSITION_AUX21\n#define MA_PA_CHANNEL_POSITION_AUX22                   PA_CHANNEL_POSITION_AUX22\n#define MA_PA_CHANNEL_POSITION_AUX23                   PA_CHANNEL_POSITION_AUX23\n#define MA_PA_CHANNEL_POSITION_AUX24                   PA_CHANNEL_POSITION_AUX24\n#define MA_PA_CHANNEL_POSITION_AUX25                   PA_CHANNEL_POSITION_AUX25\n#define MA_PA_CHANNEL_POSITION_AUX26                   PA_CHANNEL_POSITION_AUX26\n#define MA_PA_CHANNEL_POSITION_AUX27                   PA_CHANNEL_POSITION_AUX27\n#define MA_PA_CHANNEL_POSITION_AUX28                   PA_CHANNEL_POSITION_AUX28\n#define MA_PA_CHANNEL_POSITION_AUX29                   PA_CHANNEL_POSITION_AUX29\n#define MA_PA_CHANNEL_POSITION_AUX30                   PA_CHANNEL_POSITION_AUX30\n#define MA_PA_CHANNEL_POSITION_AUX31                   PA_CHANNEL_POSITION_AUX31\n#define MA_PA_CHANNEL_POSITION_TOP_CENTER              PA_CHANNEL_POSITION_TOP_CENTER\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_LEFT          PA_CHANNEL_POSITION_TOP_FRONT_LEFT\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_RIGHT         PA_CHANNEL_POSITION_TOP_FRONT_RIGHT\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_CENTER        PA_CHANNEL_POSITION_TOP_FRONT_CENTER\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_LEFT           PA_CHANNEL_POSITION_TOP_REAR_LEFT\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_RIGHT          PA_CHANNEL_POSITION_TOP_REAR_RIGHT\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_CENTER         PA_CHANNEL_POSITION_TOP_REAR_CENTER\n#define MA_PA_CHANNEL_POSITION_LEFT                    PA_CHANNEL_POSITION_LEFT\n#define MA_PA_CHANNEL_POSITION_RIGHT                   PA_CHANNEL_POSITION_RIGHT\n#define MA_PA_CHANNEL_POSITION_CENTER                  PA_CHANNEL_POSITION_CENTER\n#define MA_PA_CHANNEL_POSITION_SUBWOOFER               PA_CHANNEL_POSITION_SUBWOOFER\n\ntypedef pa_channel_map_def_t ma_pa_channel_map_def_t;\n#define MA_PA_CHANNEL_MAP_AIFF                         PA_CHANNEL_MAP_AIFF\n#define MA_PA_CHANNEL_MAP_ALSA                         PA_CHANNEL_MAP_ALSA\n#define MA_PA_CHANNEL_MAP_AUX                          PA_CHANNEL_MAP_AUX\n#define MA_PA_CHANNEL_MAP_WAVEEX                       PA_CHANNEL_MAP_WAVEEX\n#define MA_PA_CHANNEL_MAP_OSS                          PA_CHANNEL_MAP_OSS\n#define MA_PA_CHANNEL_MAP_DEFAULT                      PA_CHANNEL_MAP_DEFAULT\n\ntypedef pa_sample_format_t ma_pa_sample_format_t;\n#define MA_PA_SAMPLE_INVALID                           PA_SAMPLE_INVALID\n#define MA_PA_SAMPLE_U8                                PA_SAMPLE_U8\n#define MA_PA_SAMPLE_ALAW                              PA_SAMPLE_ALAW\n#define MA_PA_SAMPLE_ULAW                              PA_SAMPLE_ULAW\n#define MA_PA_SAMPLE_S16LE                             PA_SAMPLE_S16LE\n#define MA_PA_SAMPLE_S16BE                             PA_SAMPLE_S16BE\n#define MA_PA_SAMPLE_FLOAT32LE                         PA_SAMPLE_FLOAT32LE\n#define MA_PA_SAMPLE_FLOAT32BE                         PA_SAMPLE_FLOAT32BE\n#define MA_PA_SAMPLE_S32LE                             PA_SAMPLE_S32LE\n#define MA_PA_SAMPLE_S32BE                             PA_SAMPLE_S32BE\n#define MA_PA_SAMPLE_S24LE                             PA_SAMPLE_S24LE\n#define MA_PA_SAMPLE_S24BE                             PA_SAMPLE_S24BE\n#define MA_PA_SAMPLE_S24_32LE                          PA_SAMPLE_S24_32LE\n#define MA_PA_SAMPLE_S24_32BE                          PA_SAMPLE_S24_32BE\n\ntypedef pa_mainloop             ma_pa_mainloop;\ntypedef pa_threaded_mainloop    ma_pa_threaded_mainloop;\ntypedef pa_mainloop_api         ma_pa_mainloop_api;\ntypedef pa_context              ma_pa_context;\ntypedef pa_operation            ma_pa_operation;\ntypedef pa_stream               ma_pa_stream;\ntypedef pa_spawn_api            ma_pa_spawn_api;\ntypedef pa_buffer_attr          ma_pa_buffer_attr;\ntypedef pa_channel_map          ma_pa_channel_map;\ntypedef pa_cvolume              ma_pa_cvolume;\ntypedef pa_sample_spec          ma_pa_sample_spec;\ntypedef pa_sink_info            ma_pa_sink_info;\ntypedef pa_source_info          ma_pa_source_info;\n\ntypedef pa_context_notify_cb_t  ma_pa_context_notify_cb_t;\ntypedef pa_sink_info_cb_t       ma_pa_sink_info_cb_t;\ntypedef pa_source_info_cb_t     ma_pa_source_info_cb_t;\ntypedef pa_stream_success_cb_t  ma_pa_stream_success_cb_t;\ntypedef pa_stream_request_cb_t  ma_pa_stream_request_cb_t;\ntypedef pa_stream_notify_cb_t   ma_pa_stream_notify_cb_t;\ntypedef pa_free_cb_t            ma_pa_free_cb_t;\n#else\n#define MA_PA_OK                                       0\n#define MA_PA_ERR_ACCESS                               1\n#define MA_PA_ERR_INVALID                              2\n#define MA_PA_ERR_NOENTITY                             5\n#define MA_PA_ERR_NOTSUPPORTED                         19\n\n#define MA_PA_CHANNELS_MAX                             32\n#define MA_PA_RATE_MAX                                 384000\n\ntypedef int ma_pa_context_flags_t;\n#define MA_PA_CONTEXT_NOFLAGS                          0x00000000\n#define MA_PA_CONTEXT_NOAUTOSPAWN                      0x00000001\n#define MA_PA_CONTEXT_NOFAIL                           0x00000002\n\ntypedef int ma_pa_stream_flags_t;\n#define MA_PA_STREAM_NOFLAGS                           0x00000000\n#define MA_PA_STREAM_START_CORKED                      0x00000001\n#define MA_PA_STREAM_INTERPOLATE_TIMING                0x00000002\n#define MA_PA_STREAM_NOT_MONOTONIC                     0x00000004\n#define MA_PA_STREAM_AUTO_TIMING_UPDATE                0x00000008\n#define MA_PA_STREAM_NO_REMAP_CHANNELS                 0x00000010\n#define MA_PA_STREAM_NO_REMIX_CHANNELS                 0x00000020\n#define MA_PA_STREAM_FIX_FORMAT                        0x00000040\n#define MA_PA_STREAM_FIX_RATE                          0x00000080\n#define MA_PA_STREAM_FIX_CHANNELS                      0x00000100\n#define MA_PA_STREAM_DONT_MOVE                         0x00000200\n#define MA_PA_STREAM_VARIABLE_RATE                     0x00000400\n#define MA_PA_STREAM_PEAK_DETECT                       0x00000800\n#define MA_PA_STREAM_START_MUTED                       0x00001000\n#define MA_PA_STREAM_ADJUST_LATENCY                    0x00002000\n#define MA_PA_STREAM_EARLY_REQUESTS                    0x00004000\n#define MA_PA_STREAM_DONT_INHIBIT_AUTO_SUSPEND         0x00008000\n#define MA_PA_STREAM_START_UNMUTED                     0x00010000\n#define MA_PA_STREAM_FAIL_ON_SUSPEND                   0x00020000\n#define MA_PA_STREAM_RELATIVE_VOLUME                   0x00040000\n#define MA_PA_STREAM_PASSTHROUGH                       0x00080000\n\ntypedef int ma_pa_sink_flags_t;\n#define MA_PA_SINK_NOFLAGS                             0x00000000\n#define MA_PA_SINK_HW_VOLUME_CTRL                      0x00000001\n#define MA_PA_SINK_LATENCY                             0x00000002\n#define MA_PA_SINK_HARDWARE                            0x00000004\n#define MA_PA_SINK_NETWORK                             0x00000008\n#define MA_PA_SINK_HW_MUTE_CTRL                        0x00000010\n#define MA_PA_SINK_DECIBEL_VOLUME                      0x00000020\n#define MA_PA_SINK_FLAT_VOLUME                         0x00000040\n#define MA_PA_SINK_DYNAMIC_LATENCY                     0x00000080\n#define MA_PA_SINK_SET_FORMATS                         0x00000100\n\ntypedef int ma_pa_source_flags_t;\n#define MA_PA_SOURCE_NOFLAGS                           0x00000000\n#define MA_PA_SOURCE_HW_VOLUME_CTRL                    0x00000001\n#define MA_PA_SOURCE_LATENCY                           0x00000002\n#define MA_PA_SOURCE_HARDWARE                          0x00000004\n#define MA_PA_SOURCE_NETWORK                           0x00000008\n#define MA_PA_SOURCE_HW_MUTE_CTRL                      0x00000010\n#define MA_PA_SOURCE_DECIBEL_VOLUME                    0x00000020\n#define MA_PA_SOURCE_DYNAMIC_LATENCY                   0x00000040\n#define MA_PA_SOURCE_FLAT_VOLUME                       0x00000080\n\ntypedef int ma_pa_context_state_t;\n#define MA_PA_CONTEXT_UNCONNECTED                      0\n#define MA_PA_CONTEXT_CONNECTING                       1\n#define MA_PA_CONTEXT_AUTHORIZING                      2\n#define MA_PA_CONTEXT_SETTING_NAME                     3\n#define MA_PA_CONTEXT_READY                            4\n#define MA_PA_CONTEXT_FAILED                           5\n#define MA_PA_CONTEXT_TERMINATED                       6\n\ntypedef int ma_pa_stream_state_t;\n#define MA_PA_STREAM_UNCONNECTED                       0\n#define MA_PA_STREAM_CREATING                          1\n#define MA_PA_STREAM_READY                             2\n#define MA_PA_STREAM_FAILED                            3\n#define MA_PA_STREAM_TERMINATED                        4\n\ntypedef int ma_pa_operation_state_t;\n#define MA_PA_OPERATION_RUNNING                        0\n#define MA_PA_OPERATION_DONE                           1\n#define MA_PA_OPERATION_CANCELLED                      2\n\ntypedef int ma_pa_sink_state_t;\n#define MA_PA_SINK_INVALID_STATE                       -1\n#define MA_PA_SINK_RUNNING                             0\n#define MA_PA_SINK_IDLE                                1\n#define MA_PA_SINK_SUSPENDED                           2\n\ntypedef int ma_pa_source_state_t;\n#define MA_PA_SOURCE_INVALID_STATE                     -1\n#define MA_PA_SOURCE_RUNNING                           0\n#define MA_PA_SOURCE_IDLE                              1\n#define MA_PA_SOURCE_SUSPENDED                         2\n\ntypedef int ma_pa_seek_mode_t;\n#define MA_PA_SEEK_RELATIVE                            0\n#define MA_PA_SEEK_ABSOLUTE                            1\n#define MA_PA_SEEK_RELATIVE_ON_READ                    2\n#define MA_PA_SEEK_RELATIVE_END                        3\n\ntypedef int ma_pa_channel_position_t;\n#define MA_PA_CHANNEL_POSITION_INVALID                 -1\n#define MA_PA_CHANNEL_POSITION_MONO                    0\n#define MA_PA_CHANNEL_POSITION_FRONT_LEFT              1\n#define MA_PA_CHANNEL_POSITION_FRONT_RIGHT             2\n#define MA_PA_CHANNEL_POSITION_FRONT_CENTER            3\n#define MA_PA_CHANNEL_POSITION_REAR_CENTER             4\n#define MA_PA_CHANNEL_POSITION_REAR_LEFT               5\n#define MA_PA_CHANNEL_POSITION_REAR_RIGHT              6\n#define MA_PA_CHANNEL_POSITION_LFE                     7\n#define MA_PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER    8\n#define MA_PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER   9\n#define MA_PA_CHANNEL_POSITION_SIDE_LEFT               10\n#define MA_PA_CHANNEL_POSITION_SIDE_RIGHT              11\n#define MA_PA_CHANNEL_POSITION_AUX0                    12\n#define MA_PA_CHANNEL_POSITION_AUX1                    13\n#define MA_PA_CHANNEL_POSITION_AUX2                    14\n#define MA_PA_CHANNEL_POSITION_AUX3                    15\n#define MA_PA_CHANNEL_POSITION_AUX4                    16\n#define MA_PA_CHANNEL_POSITION_AUX5                    17\n#define MA_PA_CHANNEL_POSITION_AUX6                    18\n#define MA_PA_CHANNEL_POSITION_AUX7                    19\n#define MA_PA_CHANNEL_POSITION_AUX8                    20\n#define MA_PA_CHANNEL_POSITION_AUX9                    21\n#define MA_PA_CHANNEL_POSITION_AUX10                   22\n#define MA_PA_CHANNEL_POSITION_AUX11                   23\n#define MA_PA_CHANNEL_POSITION_AUX12                   24\n#define MA_PA_CHANNEL_POSITION_AUX13                   25\n#define MA_PA_CHANNEL_POSITION_AUX14                   26\n#define MA_PA_CHANNEL_POSITION_AUX15                   27\n#define MA_PA_CHANNEL_POSITION_AUX16                   28\n#define MA_PA_CHANNEL_POSITION_AUX17                   29\n#define MA_PA_CHANNEL_POSITION_AUX18                   30\n#define MA_PA_CHANNEL_POSITION_AUX19                   31\n#define MA_PA_CHANNEL_POSITION_AUX20                   32\n#define MA_PA_CHANNEL_POSITION_AUX21                   33\n#define MA_PA_CHANNEL_POSITION_AUX22                   34\n#define MA_PA_CHANNEL_POSITION_AUX23                   35\n#define MA_PA_CHANNEL_POSITION_AUX24                   36\n#define MA_PA_CHANNEL_POSITION_AUX25                   37\n#define MA_PA_CHANNEL_POSITION_AUX26                   38\n#define MA_PA_CHANNEL_POSITION_AUX27                   39\n#define MA_PA_CHANNEL_POSITION_AUX28                   40\n#define MA_PA_CHANNEL_POSITION_AUX29                   41\n#define MA_PA_CHANNEL_POSITION_AUX30                   42\n#define MA_PA_CHANNEL_POSITION_AUX31                   43\n#define MA_PA_CHANNEL_POSITION_TOP_CENTER              44\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_LEFT          45\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_RIGHT         46\n#define MA_PA_CHANNEL_POSITION_TOP_FRONT_CENTER        47\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_LEFT           48\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_RIGHT          49\n#define MA_PA_CHANNEL_POSITION_TOP_REAR_CENTER         50\n#define MA_PA_CHANNEL_POSITION_LEFT                    MA_PA_CHANNEL_POSITION_FRONT_LEFT\n#define MA_PA_CHANNEL_POSITION_RIGHT                   MA_PA_CHANNEL_POSITION_FRONT_RIGHT\n#define MA_PA_CHANNEL_POSITION_CENTER                  MA_PA_CHANNEL_POSITION_FRONT_CENTER\n#define MA_PA_CHANNEL_POSITION_SUBWOOFER               MA_PA_CHANNEL_POSITION_LFE\n\ntypedef int ma_pa_channel_map_def_t;\n#define MA_PA_CHANNEL_MAP_AIFF                         0\n#define MA_PA_CHANNEL_MAP_ALSA                         1\n#define MA_PA_CHANNEL_MAP_AUX                          2\n#define MA_PA_CHANNEL_MAP_WAVEEX                       3\n#define MA_PA_CHANNEL_MAP_OSS                          4\n#define MA_PA_CHANNEL_MAP_DEFAULT                      MA_PA_CHANNEL_MAP_AIFF\n\ntypedef int ma_pa_sample_format_t;\n#define MA_PA_SAMPLE_INVALID                           -1\n#define MA_PA_SAMPLE_U8                                0\n#define MA_PA_SAMPLE_ALAW                              1\n#define MA_PA_SAMPLE_ULAW                              2\n#define MA_PA_SAMPLE_S16LE                             3\n#define MA_PA_SAMPLE_S16BE                             4\n#define MA_PA_SAMPLE_FLOAT32LE                         5\n#define MA_PA_SAMPLE_FLOAT32BE                         6\n#define MA_PA_SAMPLE_S32LE                             7\n#define MA_PA_SAMPLE_S32BE                             8\n#define MA_PA_SAMPLE_S24LE                             9\n#define MA_PA_SAMPLE_S24BE                             10\n#define MA_PA_SAMPLE_S24_32LE                          11\n#define MA_PA_SAMPLE_S24_32BE                          12\n\ntypedef struct ma_pa_mainloop           ma_pa_mainloop;\ntypedef struct ma_pa_threaded_mainloop  ma_pa_threaded_mainloop;\ntypedef struct ma_pa_mainloop_api       ma_pa_mainloop_api;\ntypedef struct ma_pa_context            ma_pa_context;\ntypedef struct ma_pa_operation          ma_pa_operation;\ntypedef struct ma_pa_stream             ma_pa_stream;\ntypedef struct ma_pa_spawn_api          ma_pa_spawn_api;\n\ntypedef struct\n{\n    ma_uint32 maxlength;\n    ma_uint32 tlength;\n    ma_uint32 prebuf;\n    ma_uint32 minreq;\n    ma_uint32 fragsize;\n} ma_pa_buffer_attr;\n\ntypedef struct\n{\n    ma_uint8 channels;\n    ma_pa_channel_position_t map[MA_PA_CHANNELS_MAX];\n} ma_pa_channel_map;\n\ntypedef struct\n{\n    ma_uint8 channels;\n    ma_uint32 values[MA_PA_CHANNELS_MAX];\n} ma_pa_cvolume;\n\ntypedef struct\n{\n    ma_pa_sample_format_t format;\n    ma_uint32 rate;\n    ma_uint8 channels;\n} ma_pa_sample_spec;\n\ntypedef struct\n{\n    const char* name;\n    ma_uint32 index;\n    const char* description;\n    ma_pa_sample_spec sample_spec;\n    ma_pa_channel_map channel_map;\n    ma_uint32 owner_module;\n    ma_pa_cvolume volume;\n    int mute;\n    ma_uint32 monitor_source;\n    const char* monitor_source_name;\n    ma_uint64 latency;\n    const char* driver;\n    ma_pa_sink_flags_t flags;\n    void* proplist;\n    ma_uint64 configured_latency;\n    ma_uint32 base_volume;\n    ma_pa_sink_state_t state;\n    ma_uint32 n_volume_steps;\n    ma_uint32 card;\n    ma_uint32 n_ports;\n    void** ports;\n    void* active_port;\n    ma_uint8 n_formats;\n    void** formats;\n} ma_pa_sink_info;\n\ntypedef struct\n{\n    const char *name;\n    ma_uint32 index;\n    const char *description;\n    ma_pa_sample_spec sample_spec;\n    ma_pa_channel_map channel_map;\n    ma_uint32 owner_module;\n    ma_pa_cvolume volume;\n    int mute;\n    ma_uint32 monitor_of_sink;\n    const char *monitor_of_sink_name;\n    ma_uint64 latency;\n    const char *driver;\n    ma_pa_source_flags_t flags;\n    void* proplist;\n    ma_uint64 configured_latency;\n    ma_uint32 base_volume;\n    ma_pa_source_state_t state;\n    ma_uint32 n_volume_steps;\n    ma_uint32 card;\n    ma_uint32 n_ports;\n    void** ports;\n    void* active_port;\n    ma_uint8 n_formats;\n    void** formats;\n} ma_pa_source_info;\n\ntypedef void (* ma_pa_context_notify_cb_t)(ma_pa_context* c, void* userdata);\ntypedef void (* ma_pa_sink_info_cb_t)     (ma_pa_context* c, const ma_pa_sink_info* i, int eol, void* userdata);\ntypedef void (* ma_pa_source_info_cb_t)   (ma_pa_context* c, const ma_pa_source_info* i, int eol, void* userdata);\ntypedef void (* ma_pa_stream_success_cb_t)(ma_pa_stream* s, int success, void* userdata);\ntypedef void (* ma_pa_stream_request_cb_t)(ma_pa_stream* s, size_t nbytes, void* userdata);\ntypedef void (* ma_pa_stream_notify_cb_t) (ma_pa_stream* s, void* userdata);\ntypedef void (* ma_pa_free_cb_t)          (void* p);\n#endif\n\n\ntypedef ma_pa_mainloop*          (* ma_pa_mainloop_new_proc)                   (void);\ntypedef void                     (* ma_pa_mainloop_free_proc)                  (ma_pa_mainloop* m);\ntypedef void                     (* ma_pa_mainloop_quit_proc)                  (ma_pa_mainloop* m, int retval);\ntypedef ma_pa_mainloop_api*      (* ma_pa_mainloop_get_api_proc)               (ma_pa_mainloop* m);\ntypedef int                      (* ma_pa_mainloop_iterate_proc)               (ma_pa_mainloop* m, int block, int* retval);\ntypedef void                     (* ma_pa_mainloop_wakeup_proc)                (ma_pa_mainloop* m);\ntypedef ma_pa_threaded_mainloop* (* ma_pa_threaded_mainloop_new_proc)          (void);\ntypedef void                     (* ma_pa_threaded_mainloop_free_proc)         (ma_pa_threaded_mainloop* m);\ntypedef int                      (* ma_pa_threaded_mainloop_start_proc)        (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_stop_proc)         (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_lock_proc)         (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_unlock_proc)       (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_wait_proc)         (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_signal_proc)       (ma_pa_threaded_mainloop* m, int wait_for_accept);\ntypedef void                     (* ma_pa_threaded_mainloop_accept_proc)       (ma_pa_threaded_mainloop* m);\ntypedef int                      (* ma_pa_threaded_mainloop_get_retval_proc)   (const ma_pa_threaded_mainloop* m);\ntypedef ma_pa_mainloop_api*      (* ma_pa_threaded_mainloop_get_api_proc)      (ma_pa_threaded_mainloop* m);\ntypedef int                      (* ma_pa_threaded_mainloop_in_thread_proc)    (ma_pa_threaded_mainloop* m);\ntypedef void                     (* ma_pa_threaded_mainloop_set_name_proc)     (ma_pa_threaded_mainloop* m, const char* name);\ntypedef ma_pa_context*           (* ma_pa_context_new_proc)                    (ma_pa_mainloop_api* mainloop, const char* name);\ntypedef void                     (* ma_pa_context_unref_proc)                  (ma_pa_context* c);\ntypedef int                      (* ma_pa_context_connect_proc)                (ma_pa_context* c, const char* server, ma_pa_context_flags_t flags, const ma_pa_spawn_api* api);\ntypedef void                     (* ma_pa_context_disconnect_proc)             (ma_pa_context* c);\ntypedef void                     (* ma_pa_context_set_state_callback_proc)     (ma_pa_context* c, ma_pa_context_notify_cb_t cb, void* userdata);\ntypedef ma_pa_context_state_t    (* ma_pa_context_get_state_proc)              (const ma_pa_context* c);\ntypedef ma_pa_operation*         (* ma_pa_context_get_sink_info_list_proc)     (ma_pa_context* c, ma_pa_sink_info_cb_t cb, void* userdata);\ntypedef ma_pa_operation*         (* ma_pa_context_get_source_info_list_proc)   (ma_pa_context* c, ma_pa_source_info_cb_t cb, void* userdata);\ntypedef ma_pa_operation*         (* ma_pa_context_get_sink_info_by_name_proc)  (ma_pa_context* c, const char* name, ma_pa_sink_info_cb_t cb, void* userdata);\ntypedef ma_pa_operation*         (* ma_pa_context_get_source_info_by_name_proc)(ma_pa_context* c, const char* name, ma_pa_source_info_cb_t cb, void* userdata);\ntypedef void                     (* ma_pa_operation_unref_proc)                (ma_pa_operation* o);\ntypedef ma_pa_operation_state_t  (* ma_pa_operation_get_state_proc)            (const ma_pa_operation* o);\ntypedef ma_pa_channel_map*       (* ma_pa_channel_map_init_extend_proc)        (ma_pa_channel_map* m, unsigned channels, ma_pa_channel_map_def_t def);\ntypedef int                      (* ma_pa_channel_map_valid_proc)              (const ma_pa_channel_map* m);\ntypedef int                      (* ma_pa_channel_map_compatible_proc)         (const ma_pa_channel_map* m, const ma_pa_sample_spec* ss);\ntypedef ma_pa_stream*            (* ma_pa_stream_new_proc)                     (ma_pa_context* c, const char* name, const ma_pa_sample_spec* ss, const ma_pa_channel_map* map);\ntypedef void                     (* ma_pa_stream_unref_proc)                   (ma_pa_stream* s);\ntypedef int                      (* ma_pa_stream_connect_playback_proc)        (ma_pa_stream* s, const char* dev, const ma_pa_buffer_attr* attr, ma_pa_stream_flags_t flags, const ma_pa_cvolume* volume, ma_pa_stream* sync_stream);\ntypedef int                      (* ma_pa_stream_connect_record_proc)          (ma_pa_stream* s, const char* dev, const ma_pa_buffer_attr* attr, ma_pa_stream_flags_t flags);\ntypedef int                      (* ma_pa_stream_disconnect_proc)              (ma_pa_stream* s);\ntypedef ma_pa_stream_state_t     (* ma_pa_stream_get_state_proc)               (const ma_pa_stream* s);\ntypedef const ma_pa_sample_spec* (* ma_pa_stream_get_sample_spec_proc)         (ma_pa_stream* s);\ntypedef const ma_pa_channel_map* (* ma_pa_stream_get_channel_map_proc)         (ma_pa_stream* s);\ntypedef const ma_pa_buffer_attr* (* ma_pa_stream_get_buffer_attr_proc)         (ma_pa_stream* s);\ntypedef ma_pa_operation*         (* ma_pa_stream_set_buffer_attr_proc)         (ma_pa_stream* s, const ma_pa_buffer_attr* attr, ma_pa_stream_success_cb_t cb, void* userdata);\ntypedef const char*              (* ma_pa_stream_get_device_name_proc)         (const ma_pa_stream* s);\ntypedef void                     (* ma_pa_stream_set_write_callback_proc)      (ma_pa_stream* s, ma_pa_stream_request_cb_t cb, void* userdata);\ntypedef void                     (* ma_pa_stream_set_read_callback_proc)       (ma_pa_stream* s, ma_pa_stream_request_cb_t cb, void* userdata);\ntypedef void                     (* ma_pa_stream_set_suspended_callback_proc)  (ma_pa_stream* s, ma_pa_stream_notify_cb_t cb, void* userdata);\ntypedef void                     (* ma_pa_stream_set_moved_callback_proc)      (ma_pa_stream* s, ma_pa_stream_notify_cb_t cb, void* userdata);\ntypedef int                      (* ma_pa_stream_is_suspended_proc)            (const ma_pa_stream* s);\ntypedef ma_pa_operation*         (* ma_pa_stream_flush_proc)                   (ma_pa_stream* s, ma_pa_stream_success_cb_t cb, void* userdata);\ntypedef ma_pa_operation*         (* ma_pa_stream_drain_proc)                   (ma_pa_stream* s, ma_pa_stream_success_cb_t cb, void* userdata);\ntypedef int                      (* ma_pa_stream_is_corked_proc)               (const ma_pa_stream* s);\ntypedef ma_pa_operation*         (* ma_pa_stream_cork_proc)                    (ma_pa_stream* s, int b, ma_pa_stream_success_cb_t cb, void* userdata);\ntypedef ma_pa_operation*         (* ma_pa_stream_trigger_proc)                 (ma_pa_stream* s, ma_pa_stream_success_cb_t cb, void* userdata);\ntypedef int                      (* ma_pa_stream_begin_write_proc)             (ma_pa_stream* s, void** data, size_t* nbytes);\ntypedef int                      (* ma_pa_stream_write_proc)                   (ma_pa_stream* s, const void* data, size_t nbytes, ma_pa_free_cb_t free_cb, int64_t offset, ma_pa_seek_mode_t seek);\ntypedef int                      (* ma_pa_stream_peek_proc)                    (ma_pa_stream* s, const void** data, size_t* nbytes);\ntypedef int                      (* ma_pa_stream_drop_proc)                    (ma_pa_stream* s);\ntypedef size_t                   (* ma_pa_stream_writable_size_proc)           (const ma_pa_stream* s);\ntypedef size_t                   (* ma_pa_stream_readable_size_proc)           (const ma_pa_stream* s);\n\ntypedef struct\n{\n    ma_uint32 count;\n    ma_uint32 capacity;\n    ma_device_info* pInfo;\n} ma_pulse_device_enum_data;\n\nstatic ma_result ma_result_from_pulse(int result)\n{\n    if (result < 0) {\n        return MA_ERROR;\n    }\n\n    switch (result) {\n        case MA_PA_OK:           return MA_SUCCESS;\n        case MA_PA_ERR_ACCESS:   return MA_ACCESS_DENIED;\n        case MA_PA_ERR_INVALID:  return MA_INVALID_ARGS;\n        case MA_PA_ERR_NOENTITY: return MA_NO_DEVICE;\n        default:                 return MA_ERROR;\n    }\n}\n\n#if 0\nstatic ma_pa_sample_format_t ma_format_to_pulse(ma_format format)\n{\n    if (ma_is_little_endian()) {\n        switch (format) {\n            case ma_format_s16: return MA_PA_SAMPLE_S16LE;\n            case ma_format_s24: return MA_PA_SAMPLE_S24LE;\n            case ma_format_s32: return MA_PA_SAMPLE_S32LE;\n            case ma_format_f32: return MA_PA_SAMPLE_FLOAT32LE;\n            default: break;\n        }\n    } else {\n        switch (format) {\n            case ma_format_s16: return MA_PA_SAMPLE_S16BE;\n            case ma_format_s24: return MA_PA_SAMPLE_S24BE;\n            case ma_format_s32: return MA_PA_SAMPLE_S32BE;\n            case ma_format_f32: return MA_PA_SAMPLE_FLOAT32BE;\n            default: break;\n        }\n    }\n\n    /* Endian agnostic. */\n    switch (format) {\n        case ma_format_u8: return MA_PA_SAMPLE_U8;\n        default: return MA_PA_SAMPLE_INVALID;\n    }\n}\n#endif\n\nstatic ma_format ma_format_from_pulse(ma_pa_sample_format_t format)\n{\n    if (ma_is_little_endian()) {\n        switch (format) {\n            case MA_PA_SAMPLE_S16LE:     return ma_format_s16;\n            case MA_PA_SAMPLE_S24LE:     return ma_format_s24;\n            case MA_PA_SAMPLE_S32LE:     return ma_format_s32;\n            case MA_PA_SAMPLE_FLOAT32LE: return ma_format_f32;\n            default: break;\n        }\n    } else {\n        switch (format) {\n            case MA_PA_SAMPLE_S16BE:     return ma_format_s16;\n            case MA_PA_SAMPLE_S24BE:     return ma_format_s24;\n            case MA_PA_SAMPLE_S32BE:     return ma_format_s32;\n            case MA_PA_SAMPLE_FLOAT32BE: return ma_format_f32;\n            default: break;\n        }\n    }\n\n    /* Endian agnostic. */\n    switch (format) {\n        case MA_PA_SAMPLE_U8: return ma_format_u8;\n        default: return ma_format_unknown;\n    }\n}\n\nstatic ma_channel ma_channel_position_from_pulse(ma_pa_channel_position_t position)\n{\n    switch (position)\n    {\n        case MA_PA_CHANNEL_POSITION_INVALID:               return MA_CHANNEL_NONE;\n        case MA_PA_CHANNEL_POSITION_MONO:                  return MA_CHANNEL_MONO;\n        case MA_PA_CHANNEL_POSITION_FRONT_LEFT:            return MA_CHANNEL_FRONT_LEFT;\n        case MA_PA_CHANNEL_POSITION_FRONT_RIGHT:           return MA_CHANNEL_FRONT_RIGHT;\n        case MA_PA_CHANNEL_POSITION_FRONT_CENTER:          return MA_CHANNEL_FRONT_CENTER;\n        case MA_PA_CHANNEL_POSITION_REAR_CENTER:           return MA_CHANNEL_BACK_CENTER;\n        case MA_PA_CHANNEL_POSITION_REAR_LEFT:             return MA_CHANNEL_BACK_LEFT;\n        case MA_PA_CHANNEL_POSITION_REAR_RIGHT:            return MA_CHANNEL_BACK_RIGHT;\n        case MA_PA_CHANNEL_POSITION_LFE:                   return MA_CHANNEL_LFE;\n        case MA_PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER:  return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case MA_PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER: return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case MA_PA_CHANNEL_POSITION_SIDE_LEFT:             return MA_CHANNEL_SIDE_LEFT;\n        case MA_PA_CHANNEL_POSITION_SIDE_RIGHT:            return MA_CHANNEL_SIDE_RIGHT;\n        case MA_PA_CHANNEL_POSITION_AUX0:                  return MA_CHANNEL_AUX_0;\n        case MA_PA_CHANNEL_POSITION_AUX1:                  return MA_CHANNEL_AUX_1;\n        case MA_PA_CHANNEL_POSITION_AUX2:                  return MA_CHANNEL_AUX_2;\n        case MA_PA_CHANNEL_POSITION_AUX3:                  return MA_CHANNEL_AUX_3;\n        case MA_PA_CHANNEL_POSITION_AUX4:                  return MA_CHANNEL_AUX_4;\n        case MA_PA_CHANNEL_POSITION_AUX5:                  return MA_CHANNEL_AUX_5;\n        case MA_PA_CHANNEL_POSITION_AUX6:                  return MA_CHANNEL_AUX_6;\n        case MA_PA_CHANNEL_POSITION_AUX7:                  return MA_CHANNEL_AUX_7;\n        case MA_PA_CHANNEL_POSITION_AUX8:                  return MA_CHANNEL_AUX_8;\n        case MA_PA_CHANNEL_POSITION_AUX9:                  return MA_CHANNEL_AUX_9;\n        case MA_PA_CHANNEL_POSITION_AUX10:                 return MA_CHANNEL_AUX_10;\n        case MA_PA_CHANNEL_POSITION_AUX11:                 return MA_CHANNEL_AUX_11;\n        case MA_PA_CHANNEL_POSITION_AUX12:                 return MA_CHANNEL_AUX_12;\n        case MA_PA_CHANNEL_POSITION_AUX13:                 return MA_CHANNEL_AUX_13;\n        case MA_PA_CHANNEL_POSITION_AUX14:                 return MA_CHANNEL_AUX_14;\n        case MA_PA_CHANNEL_POSITION_AUX15:                 return MA_CHANNEL_AUX_15;\n        case MA_PA_CHANNEL_POSITION_AUX16:                 return MA_CHANNEL_AUX_16;\n        case MA_PA_CHANNEL_POSITION_AUX17:                 return MA_CHANNEL_AUX_17;\n        case MA_PA_CHANNEL_POSITION_AUX18:                 return MA_CHANNEL_AUX_18;\n        case MA_PA_CHANNEL_POSITION_AUX19:                 return MA_CHANNEL_AUX_19;\n        case MA_PA_CHANNEL_POSITION_AUX20:                 return MA_CHANNEL_AUX_20;\n        case MA_PA_CHANNEL_POSITION_AUX21:                 return MA_CHANNEL_AUX_21;\n        case MA_PA_CHANNEL_POSITION_AUX22:                 return MA_CHANNEL_AUX_22;\n        case MA_PA_CHANNEL_POSITION_AUX23:                 return MA_CHANNEL_AUX_23;\n        case MA_PA_CHANNEL_POSITION_AUX24:                 return MA_CHANNEL_AUX_24;\n        case MA_PA_CHANNEL_POSITION_AUX25:                 return MA_CHANNEL_AUX_25;\n        case MA_PA_CHANNEL_POSITION_AUX26:                 return MA_CHANNEL_AUX_26;\n        case MA_PA_CHANNEL_POSITION_AUX27:                 return MA_CHANNEL_AUX_27;\n        case MA_PA_CHANNEL_POSITION_AUX28:                 return MA_CHANNEL_AUX_28;\n        case MA_PA_CHANNEL_POSITION_AUX29:                 return MA_CHANNEL_AUX_29;\n        case MA_PA_CHANNEL_POSITION_AUX30:                 return MA_CHANNEL_AUX_30;\n        case MA_PA_CHANNEL_POSITION_AUX31:                 return MA_CHANNEL_AUX_31;\n        case MA_PA_CHANNEL_POSITION_TOP_CENTER:            return MA_CHANNEL_TOP_CENTER;\n        case MA_PA_CHANNEL_POSITION_TOP_FRONT_LEFT:        return MA_CHANNEL_TOP_FRONT_LEFT;\n        case MA_PA_CHANNEL_POSITION_TOP_FRONT_RIGHT:       return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case MA_PA_CHANNEL_POSITION_TOP_FRONT_CENTER:      return MA_CHANNEL_TOP_FRONT_CENTER;\n        case MA_PA_CHANNEL_POSITION_TOP_REAR_LEFT:         return MA_CHANNEL_TOP_BACK_LEFT;\n        case MA_PA_CHANNEL_POSITION_TOP_REAR_RIGHT:        return MA_CHANNEL_TOP_BACK_RIGHT;\n        case MA_PA_CHANNEL_POSITION_TOP_REAR_CENTER:       return MA_CHANNEL_TOP_BACK_CENTER;\n        default: return MA_CHANNEL_NONE;\n    }\n}\n\n#if 0\nstatic ma_pa_channel_position_t ma_channel_position_to_pulse(ma_channel position)\n{\n    switch (position)\n    {\n        case MA_CHANNEL_NONE:               return MA_PA_CHANNEL_POSITION_INVALID;\n        case MA_CHANNEL_FRONT_LEFT:         return MA_PA_CHANNEL_POSITION_FRONT_LEFT;\n        case MA_CHANNEL_FRONT_RIGHT:        return MA_PA_CHANNEL_POSITION_FRONT_RIGHT;\n        case MA_CHANNEL_FRONT_CENTER:       return MA_PA_CHANNEL_POSITION_FRONT_CENTER;\n        case MA_CHANNEL_LFE:                return MA_PA_CHANNEL_POSITION_LFE;\n        case MA_CHANNEL_BACK_LEFT:          return MA_PA_CHANNEL_POSITION_REAR_LEFT;\n        case MA_CHANNEL_BACK_RIGHT:         return MA_PA_CHANNEL_POSITION_REAR_RIGHT;\n        case MA_CHANNEL_FRONT_LEFT_CENTER:  return MA_PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER;\n        case MA_CHANNEL_FRONT_RIGHT_CENTER: return MA_PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER;\n        case MA_CHANNEL_BACK_CENTER:        return MA_PA_CHANNEL_POSITION_REAR_CENTER;\n        case MA_CHANNEL_SIDE_LEFT:          return MA_PA_CHANNEL_POSITION_SIDE_LEFT;\n        case MA_CHANNEL_SIDE_RIGHT:         return MA_PA_CHANNEL_POSITION_SIDE_RIGHT;\n        case MA_CHANNEL_TOP_CENTER:         return MA_PA_CHANNEL_POSITION_TOP_CENTER;\n        case MA_CHANNEL_TOP_FRONT_LEFT:     return MA_PA_CHANNEL_POSITION_TOP_FRONT_LEFT;\n        case MA_CHANNEL_TOP_FRONT_CENTER:   return MA_PA_CHANNEL_POSITION_TOP_FRONT_CENTER;\n        case MA_CHANNEL_TOP_FRONT_RIGHT:    return MA_PA_CHANNEL_POSITION_TOP_FRONT_RIGHT;\n        case MA_CHANNEL_TOP_BACK_LEFT:      return MA_PA_CHANNEL_POSITION_TOP_REAR_LEFT;\n        case MA_CHANNEL_TOP_BACK_CENTER:    return MA_PA_CHANNEL_POSITION_TOP_REAR_CENTER;\n        case MA_CHANNEL_TOP_BACK_RIGHT:     return MA_PA_CHANNEL_POSITION_TOP_REAR_RIGHT;\n        case MA_CHANNEL_19:                 return MA_PA_CHANNEL_POSITION_AUX18;\n        case MA_CHANNEL_20:                 return MA_PA_CHANNEL_POSITION_AUX19;\n        case MA_CHANNEL_21:                 return MA_PA_CHANNEL_POSITION_AUX20;\n        case MA_CHANNEL_22:                 return MA_PA_CHANNEL_POSITION_AUX21;\n        case MA_CHANNEL_23:                 return MA_PA_CHANNEL_POSITION_AUX22;\n        case MA_CHANNEL_24:                 return MA_PA_CHANNEL_POSITION_AUX23;\n        case MA_CHANNEL_25:                 return MA_PA_CHANNEL_POSITION_AUX24;\n        case MA_CHANNEL_26:                 return MA_PA_CHANNEL_POSITION_AUX25;\n        case MA_CHANNEL_27:                 return MA_PA_CHANNEL_POSITION_AUX26;\n        case MA_CHANNEL_28:                 return MA_PA_CHANNEL_POSITION_AUX27;\n        case MA_CHANNEL_29:                 return MA_PA_CHANNEL_POSITION_AUX28;\n        case MA_CHANNEL_30:                 return MA_PA_CHANNEL_POSITION_AUX29;\n        case MA_CHANNEL_31:                 return MA_PA_CHANNEL_POSITION_AUX30;\n        case MA_CHANNEL_32:                 return MA_PA_CHANNEL_POSITION_AUX31;\n        default: return (ma_pa_channel_position_t)position;\n    }\n}\n#endif\n\nstatic ma_result ma_wait_for_operation__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_pa_operation* pOP)\n{\n    int resultPA;\n    ma_pa_operation_state_t state;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pOP != NULL);\n\n    for (;;) {\n        state = ((ma_pa_operation_get_state_proc)pContext->pulse.pa_operation_get_state)(pOP);\n        if (state != MA_PA_OPERATION_RUNNING) {\n            break;  /* Done. */\n        }\n\n        resultPA = ((ma_pa_mainloop_iterate_proc)pContext->pulse.pa_mainloop_iterate)((ma_pa_mainloop*)pMainLoop, 1, NULL);\n        if (resultPA < 0) {\n            return ma_result_from_pulse(resultPA);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_wait_for_operation_and_unref__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_pa_operation* pOP)\n{\n    ma_result result;\n\n    if (pOP == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_wait_for_operation__pulse(pContext, pMainLoop, pOP);\n    ((ma_pa_operation_unref_proc)pContext->pulse.pa_operation_unref)(pOP);\n\n    return result;\n}\n\nstatic ma_result ma_wait_for_pa_context_to_connect__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_ptr pPulseContext)\n{\n    int resultPA;\n    ma_pa_context_state_t state;\n\n    for (;;) {\n        state = ((ma_pa_context_get_state_proc)pContext->pulse.pa_context_get_state)((ma_pa_context*)pPulseContext);\n        if (state == MA_PA_CONTEXT_READY) {\n            break;  /* Done. */\n        }\n\n        if (state == MA_PA_CONTEXT_FAILED || state == MA_PA_CONTEXT_TERMINATED) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] An error occurred while connecting the PulseAudio context.\");\n            return MA_ERROR;\n        }\n\n        resultPA = ((ma_pa_mainloop_iterate_proc)pContext->pulse.pa_mainloop_iterate)((ma_pa_mainloop*)pMainLoop, 1, NULL);\n        if (resultPA < 0) {\n            return ma_result_from_pulse(resultPA);\n        }\n    }\n\n    /* Should never get here. */\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_wait_for_pa_stream_to_connect__pulse(ma_context* pContext, ma_ptr pMainLoop, ma_ptr pStream)\n{\n    int resultPA;\n    ma_pa_stream_state_t state;\n\n    for (;;) {\n        state = ((ma_pa_stream_get_state_proc)pContext->pulse.pa_stream_get_state)((ma_pa_stream*)pStream);\n        if (state == MA_PA_STREAM_READY) {\n            break;  /* Done. */\n        }\n\n        if (state == MA_PA_STREAM_FAILED || state == MA_PA_STREAM_TERMINATED) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] An error occurred while connecting the PulseAudio stream.\");\n            return MA_ERROR;\n        }\n\n        resultPA = ((ma_pa_mainloop_iterate_proc)pContext->pulse.pa_mainloop_iterate)((ma_pa_mainloop*)pMainLoop, 1, NULL);\n        if (resultPA < 0) {\n            return ma_result_from_pulse(resultPA);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_init_pa_mainloop_and_pa_context__pulse(ma_context* pContext, const char* pApplicationName, const char* pServerName, ma_bool32 tryAutoSpawn, ma_ptr* ppMainLoop, ma_ptr* ppPulseContext)\n{\n    ma_result result;\n    ma_ptr pMainLoop;\n    ma_ptr pPulseContext;\n\n    MA_ASSERT(ppMainLoop     != NULL);\n    MA_ASSERT(ppPulseContext != NULL);\n\n    /* The PulseAudio context maps well to miniaudio's notion of a context. The pa_context object will be initialized as part of the ma_context. */\n    pMainLoop = ((ma_pa_mainloop_new_proc)pContext->pulse.pa_mainloop_new)();\n    if (pMainLoop == NULL) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to create mainloop.\");\n        return MA_FAILED_TO_INIT_BACKEND;\n    }\n\n    pPulseContext = ((ma_pa_context_new_proc)pContext->pulse.pa_context_new)(((ma_pa_mainloop_get_api_proc)pContext->pulse.pa_mainloop_get_api)((ma_pa_mainloop*)pMainLoop), pApplicationName);\n    if (pPulseContext == NULL) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to create PulseAudio context.\");\n        ((ma_pa_mainloop_free_proc)pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)(pMainLoop));\n        return MA_FAILED_TO_INIT_BACKEND;\n    }\n\n    /* Now we need to connect to the context. Everything is asynchronous so we need to wait for it to connect before returning. */\n    result = ma_result_from_pulse(((ma_pa_context_connect_proc)pContext->pulse.pa_context_connect)((ma_pa_context*)pPulseContext, pServerName, (tryAutoSpawn) ? MA_PA_CONTEXT_NOFLAGS : MA_PA_CONTEXT_NOAUTOSPAWN, NULL));\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to connect PulseAudio context.\");\n        ((ma_pa_context_unref_proc)pContext->pulse.pa_context_unref)((ma_pa_context*)(pPulseContext));\n        ((ma_pa_mainloop_free_proc)pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)(pMainLoop));\n        return result;\n    }\n\n    /* Since ma_context_init() runs synchronously we need to wait for the PulseAudio context to connect before we return. */\n    result = ma_wait_for_pa_context_to_connect__pulse(pContext, pMainLoop, pPulseContext);\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Waiting for connection failed.\");\n        ((ma_pa_context_unref_proc)pContext->pulse.pa_context_unref)((ma_pa_context*)(pPulseContext));\n        ((ma_pa_mainloop_free_proc)pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)(pMainLoop));\n        return result;\n    }\n\n    *ppMainLoop     = pMainLoop;\n    *ppPulseContext = pPulseContext;\n\n    return MA_SUCCESS;\n}\n\n\nstatic void ma_device_sink_info_callback(ma_pa_context* pPulseContext, const ma_pa_sink_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_pa_sink_info* pInfoOut;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    /*\n    There has been a report that indicates that pInfo can be null which results\n    in a null pointer dereference below. We'll check for this for safety.\n    */\n    if (pInfo == NULL) {\n        return;\n    }\n\n    pInfoOut = (ma_pa_sink_info*)pUserData;\n    MA_ASSERT(pInfoOut != NULL);\n\n    *pInfoOut = *pInfo;\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic void ma_device_source_info_callback(ma_pa_context* pPulseContext, const ma_pa_source_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_pa_source_info* pInfoOut;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    /*\n    There has been a report that indicates that pInfo can be null which results\n    in a null pointer dereference below. We'll check for this for safety.\n    */\n    if (pInfo == NULL) {\n        return;\n    }\n\n    pInfoOut = (ma_pa_source_info*)pUserData;\n    MA_ASSERT(pInfoOut != NULL);\n\n    *pInfoOut = *pInfo;\n\n    (void)pPulseContext; /* Unused. */\n}\n\n#if 0\nstatic void ma_device_sink_name_callback(ma_pa_context* pPulseContext, const ma_pa_sink_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_device* pDevice;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), pInfo->description, (size_t)-1);\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic void ma_device_source_name_callback(ma_pa_context* pPulseContext, const ma_pa_source_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_device* pDevice;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), pInfo->description, (size_t)-1);\n\n    (void)pPulseContext; /* Unused. */\n}\n#endif\n\nstatic ma_result ma_context_get_sink_info__pulse(ma_context* pContext, const char* pDeviceName, ma_pa_sink_info* pSinkInfo)\n{\n    ma_pa_operation* pOP;\n\n    pOP = ((ma_pa_context_get_sink_info_by_name_proc)pContext->pulse.pa_context_get_sink_info_by_name)((ma_pa_context*)pContext->pulse.pPulseContext, pDeviceName, ma_device_sink_info_callback, pSinkInfo);\n    if (pOP == NULL) {\n        return MA_ERROR;\n    }\n\n    return ma_wait_for_operation_and_unref__pulse(pContext, pContext->pulse.pMainLoop, pOP);\n}\n\nstatic ma_result ma_context_get_source_info__pulse(ma_context* pContext, const char* pDeviceName, ma_pa_source_info* pSourceInfo)\n{\n    ma_pa_operation* pOP;\n\n    pOP = ((ma_pa_context_get_source_info_by_name_proc)pContext->pulse.pa_context_get_source_info_by_name)((ma_pa_context*)pContext->pulse.pPulseContext, pDeviceName, ma_device_source_info_callback, pSourceInfo);\n    if (pOP == NULL) {\n        return MA_ERROR;\n    }\n\n    return ma_wait_for_operation_and_unref__pulse(pContext, pContext->pulse.pMainLoop, pOP);\n}\n\nstatic ma_result ma_context_get_default_device_index__pulse(ma_context* pContext, ma_device_type deviceType, ma_uint32* pIndex)\n{\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pIndex   != NULL);\n\n    if (pIndex != NULL) {\n        *pIndex = (ma_uint32)-1;\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        ma_pa_sink_info sinkInfo;\n        result = ma_context_get_sink_info__pulse(pContext, NULL, &sinkInfo);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        if (pIndex != NULL) {\n            *pIndex = sinkInfo.index;\n        }\n    }\n\n    if (deviceType == ma_device_type_capture) {\n        ma_pa_source_info sourceInfo;\n        result = ma_context_get_source_info__pulse(pContext, NULL, &sourceInfo);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        if (pIndex != NULL) {\n            *pIndex = sourceInfo.index;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\ntypedef struct\n{\n    ma_context* pContext;\n    ma_enum_devices_callback_proc callback;\n    void* pUserData;\n    ma_bool32 isTerminated;\n    ma_uint32 defaultDeviceIndexPlayback;\n    ma_uint32 defaultDeviceIndexCapture;\n} ma_context_enumerate_devices_callback_data__pulse;\n\nstatic void ma_context_enumerate_devices_sink_callback__pulse(ma_pa_context* pPulseContext, const ma_pa_sink_info* pSinkInfo, int endOfList, void* pUserData)\n{\n    ma_context_enumerate_devices_callback_data__pulse* pData = (ma_context_enumerate_devices_callback_data__pulse*)pUserData;\n    ma_device_info deviceInfo;\n\n    MA_ASSERT(pData != NULL);\n\n    if (endOfList || pData->isTerminated) {\n        return;\n    }\n\n    MA_ZERO_OBJECT(&deviceInfo);\n\n    /* The name from PulseAudio is the ID for miniaudio. */\n    if (pSinkInfo->name != NULL) {\n        ma_strncpy_s(deviceInfo.id.pulse, sizeof(deviceInfo.id.pulse), pSinkInfo->name, (size_t)-1);\n    }\n\n    /* The description from PulseAudio is the name for miniaudio. */\n    if (pSinkInfo->description != NULL) {\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), pSinkInfo->description, (size_t)-1);\n    }\n\n    if (pSinkInfo->index == pData->defaultDeviceIndexPlayback) {\n        deviceInfo.isDefault = MA_TRUE;\n    }\n\n    pData->isTerminated = !pData->callback(pData->pContext, ma_device_type_playback, &deviceInfo, pData->pUserData);\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic void ma_context_enumerate_devices_source_callback__pulse(ma_pa_context* pPulseContext, const ma_pa_source_info* pSourceInfo, int endOfList, void* pUserData)\n{\n    ma_context_enumerate_devices_callback_data__pulse* pData = (ma_context_enumerate_devices_callback_data__pulse*)pUserData;\n    ma_device_info deviceInfo;\n\n    MA_ASSERT(pData != NULL);\n\n    if (endOfList || pData->isTerminated) {\n        return;\n    }\n\n    MA_ZERO_OBJECT(&deviceInfo);\n\n    /* The name from PulseAudio is the ID for miniaudio. */\n    if (pSourceInfo->name != NULL) {\n        ma_strncpy_s(deviceInfo.id.pulse, sizeof(deviceInfo.id.pulse), pSourceInfo->name, (size_t)-1);\n    }\n\n    /* The description from PulseAudio is the name for miniaudio. */\n    if (pSourceInfo->description != NULL) {\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), pSourceInfo->description, (size_t)-1);\n    }\n\n    if (pSourceInfo->index == pData->defaultDeviceIndexCapture) {\n        deviceInfo.isDefault = MA_TRUE;\n    }\n\n    pData->isTerminated = !pData->callback(pData->pContext, ma_device_type_capture, &deviceInfo, pData->pUserData);\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic ma_result ma_context_enumerate_devices__pulse(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_result result = MA_SUCCESS;\n    ma_context_enumerate_devices_callback_data__pulse callbackData;\n    ma_pa_operation* pOP = NULL;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    callbackData.pContext = pContext;\n    callbackData.callback = callback;\n    callbackData.pUserData = pUserData;\n    callbackData.isTerminated = MA_FALSE;\n    callbackData.defaultDeviceIndexPlayback = (ma_uint32)-1;\n    callbackData.defaultDeviceIndexCapture  = (ma_uint32)-1;\n\n    /* We need to get the index of the default devices. */\n    ma_context_get_default_device_index__pulse(pContext, ma_device_type_playback, &callbackData.defaultDeviceIndexPlayback);\n    ma_context_get_default_device_index__pulse(pContext, ma_device_type_capture,  &callbackData.defaultDeviceIndexCapture);\n\n    /* Playback. */\n    if (!callbackData.isTerminated) {\n        pOP = ((ma_pa_context_get_sink_info_list_proc)pContext->pulse.pa_context_get_sink_info_list)((ma_pa_context*)(pContext->pulse.pPulseContext), ma_context_enumerate_devices_sink_callback__pulse, &callbackData);\n        if (pOP == NULL) {\n            result = MA_ERROR;\n            goto done;\n        }\n\n        result = ma_wait_for_operation__pulse(pContext, pContext->pulse.pMainLoop, pOP);\n        ((ma_pa_operation_unref_proc)pContext->pulse.pa_operation_unref)(pOP);\n\n        if (result != MA_SUCCESS) {\n            goto done;\n        }\n    }\n\n\n    /* Capture. */\n    if (!callbackData.isTerminated) {\n        pOP = ((ma_pa_context_get_source_info_list_proc)pContext->pulse.pa_context_get_source_info_list)((ma_pa_context*)(pContext->pulse.pPulseContext), ma_context_enumerate_devices_source_callback__pulse, &callbackData);\n        if (pOP == NULL) {\n            result = MA_ERROR;\n            goto done;\n        }\n\n        result = ma_wait_for_operation__pulse(pContext, pContext->pulse.pMainLoop, pOP);\n        ((ma_pa_operation_unref_proc)pContext->pulse.pa_operation_unref)(pOP);\n\n        if (result != MA_SUCCESS) {\n            goto done;\n        }\n    }\n\ndone:\n    return result;\n}\n\n\ntypedef struct\n{\n    ma_device_info* pDeviceInfo;\n    ma_uint32 defaultDeviceIndex;\n    ma_bool32 foundDevice;\n} ma_context_get_device_info_callback_data__pulse;\n\nstatic void ma_context_get_device_info_sink_callback__pulse(ma_pa_context* pPulseContext, const ma_pa_sink_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_context_get_device_info_callback_data__pulse* pData = (ma_context_get_device_info_callback_data__pulse*)pUserData;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    MA_ASSERT(pData != NULL);\n    pData->foundDevice = MA_TRUE;\n\n    if (pInfo->name != NULL) {\n        ma_strncpy_s(pData->pDeviceInfo->id.pulse, sizeof(pData->pDeviceInfo->id.pulse), pInfo->name, (size_t)-1);\n    }\n\n    if (pInfo->description != NULL) {\n        ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), pInfo->description, (size_t)-1);\n    }\n\n    /*\n    We're just reporting a single data format here. I think technically PulseAudio might support\n    all formats, but I don't trust that PulseAudio will do *anything* right, so I'm just going to\n    report the \"native\" device format.\n    */\n    pData->pDeviceInfo->nativeDataFormats[0].format     = ma_format_from_pulse(pInfo->sample_spec.format);\n    pData->pDeviceInfo->nativeDataFormats[0].channels   = pInfo->sample_spec.channels;\n    pData->pDeviceInfo->nativeDataFormats[0].sampleRate = pInfo->sample_spec.rate;\n    pData->pDeviceInfo->nativeDataFormats[0].flags      = 0;\n    pData->pDeviceInfo->nativeDataFormatCount = 1;\n\n    if (pData->defaultDeviceIndex == pInfo->index) {\n        pData->pDeviceInfo->isDefault = MA_TRUE;\n    }\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic void ma_context_get_device_info_source_callback__pulse(ma_pa_context* pPulseContext, const ma_pa_source_info* pInfo, int endOfList, void* pUserData)\n{\n    ma_context_get_device_info_callback_data__pulse* pData = (ma_context_get_device_info_callback_data__pulse*)pUserData;\n\n    if (endOfList > 0) {\n        return;\n    }\n\n    MA_ASSERT(pData != NULL);\n    pData->foundDevice = MA_TRUE;\n\n    if (pInfo->name != NULL) {\n        ma_strncpy_s(pData->pDeviceInfo->id.pulse, sizeof(pData->pDeviceInfo->id.pulse), pInfo->name, (size_t)-1);\n    }\n\n    if (pInfo->description != NULL) {\n        ma_strncpy_s(pData->pDeviceInfo->name, sizeof(pData->pDeviceInfo->name), pInfo->description, (size_t)-1);\n    }\n\n    /*\n    We're just reporting a single data format here. I think technically PulseAudio might support\n    all formats, but I don't trust that PulseAudio will do *anything* right, so I'm just going to\n    report the \"native\" device format.\n    */\n    pData->pDeviceInfo->nativeDataFormats[0].format     = ma_format_from_pulse(pInfo->sample_spec.format);\n    pData->pDeviceInfo->nativeDataFormats[0].channels   = pInfo->sample_spec.channels;\n    pData->pDeviceInfo->nativeDataFormats[0].sampleRate = pInfo->sample_spec.rate;\n    pData->pDeviceInfo->nativeDataFormats[0].flags      = 0;\n    pData->pDeviceInfo->nativeDataFormatCount = 1;\n\n    if (pData->defaultDeviceIndex == pInfo->index) {\n        pData->pDeviceInfo->isDefault = MA_TRUE;\n    }\n\n    (void)pPulseContext; /* Unused. */\n}\n\nstatic ma_result ma_context_get_device_info__pulse(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_result result = MA_SUCCESS;\n    ma_context_get_device_info_callback_data__pulse callbackData;\n    ma_pa_operation* pOP = NULL;\n    const char* pDeviceName = NULL;\n\n    MA_ASSERT(pContext != NULL);\n\n    callbackData.pDeviceInfo = pDeviceInfo;\n    callbackData.foundDevice = MA_FALSE;\n\n    if (pDeviceID != NULL) {\n        pDeviceName = pDeviceID->pulse;\n    } else {\n        pDeviceName = NULL;\n    }\n\n    result = ma_context_get_default_device_index__pulse(pContext, deviceType, &callbackData.defaultDeviceIndex);\n\n    if (deviceType == ma_device_type_playback) {\n        pOP = ((ma_pa_context_get_sink_info_by_name_proc)pContext->pulse.pa_context_get_sink_info_by_name)((ma_pa_context*)(pContext->pulse.pPulseContext), pDeviceName, ma_context_get_device_info_sink_callback__pulse, &callbackData);\n    } else {\n        pOP = ((ma_pa_context_get_source_info_by_name_proc)pContext->pulse.pa_context_get_source_info_by_name)((ma_pa_context*)(pContext->pulse.pPulseContext), pDeviceName, ma_context_get_device_info_source_callback__pulse, &callbackData);\n    }\n\n    if (pOP != NULL) {\n        ma_wait_for_operation_and_unref__pulse(pContext, pContext->pulse.pMainLoop, pOP);\n    } else {\n        result = MA_ERROR;\n        goto done;\n    }\n\n    if (!callbackData.foundDevice) {\n        result = MA_NO_DEVICE;\n        goto done;\n    }\n\ndone:\n    return result;\n}\n\nstatic ma_result ma_device_uninit__pulse(ma_device* pDevice)\n{\n    ma_context* pContext;\n\n    MA_ASSERT(pDevice != NULL);\n\n    pContext = pDevice->pContext;\n    MA_ASSERT(pContext != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ((ma_pa_stream_disconnect_proc)pContext->pulse.pa_stream_disconnect)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n        ((ma_pa_stream_unref_proc)pContext->pulse.pa_stream_unref)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ((ma_pa_stream_disconnect_proc)pContext->pulse.pa_stream_disconnect)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n        ((ma_pa_stream_unref_proc)pContext->pulse.pa_stream_unref)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n    }\n\n    if (pDevice->type == ma_device_type_duplex) {\n        ma_duplex_rb_uninit(&pDevice->duplexRB);\n    }\n\n    ((ma_pa_context_disconnect_proc)pContext->pulse.pa_context_disconnect)((ma_pa_context*)pDevice->pulse.pPulseContext);\n    ((ma_pa_context_unref_proc)pContext->pulse.pa_context_unref)((ma_pa_context*)pDevice->pulse.pPulseContext);\n    ((ma_pa_mainloop_free_proc)pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)pDevice->pulse.pMainLoop);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_pa_buffer_attr ma_device__pa_buffer_attr_new(ma_uint32 periodSizeInFrames, ma_uint32 periods, const ma_pa_sample_spec* ss)\n{\n    ma_pa_buffer_attr attr;\n    attr.maxlength = periodSizeInFrames * periods * ma_get_bytes_per_frame(ma_format_from_pulse(ss->format), ss->channels);\n    attr.tlength   = attr.maxlength / periods;\n    attr.prebuf    = (ma_uint32)-1;\n    attr.minreq    = (ma_uint32)-1;\n    attr.fragsize  = attr.maxlength / periods;\n\n    return attr;\n}\n\nstatic ma_pa_stream* ma_device__pa_stream_new__pulse(ma_device* pDevice, const char* pStreamName, const ma_pa_sample_spec* ss, const ma_pa_channel_map* cmap)\n{\n    static ma_atomic_uint32 g_StreamCounter = { 0 };\n    char actualStreamName[256];\n\n    if (pStreamName != NULL) {\n        ma_strncpy_s(actualStreamName, sizeof(actualStreamName), pStreamName, (size_t)-1);\n    } else {\n        const char* pBaseName = \"miniaudio:\";\n        size_t baseNameLen = strlen(pBaseName);\n        ma_strcpy_s(actualStreamName, sizeof(actualStreamName), pBaseName);\n        ma_itoa_s((int)ma_atomic_uint32_get(&g_StreamCounter), actualStreamName + baseNameLen, sizeof(actualStreamName)-baseNameLen, 10);\n    }\n    ma_atomic_uint32_fetch_add(&g_StreamCounter, 1);\n\n    return ((ma_pa_stream_new_proc)pDevice->pContext->pulse.pa_stream_new)((ma_pa_context*)pDevice->pulse.pPulseContext, actualStreamName, ss, cmap);\n}\n\n\nstatic void ma_device_on_read__pulse(ma_pa_stream* pStream, size_t byteCount, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    ma_uint32 bpf;\n    ma_uint32 deviceState;\n    ma_uint64 frameCount;\n    ma_uint64 framesProcessed;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    Don't do anything if the device isn't initialized yet. Yes, this can happen because PulseAudio\n    can fire this callback before the stream has even started. Ridiculous.\n    */\n    deviceState = ma_device_get_state(pDevice);\n    if (deviceState != ma_device_state_starting && deviceState != ma_device_state_started) {\n        return;\n    }\n\n    bpf = ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n    MA_ASSERT(bpf > 0);\n\n    frameCount = byteCount / bpf;\n    framesProcessed = 0;\n\n    while (ma_device_get_state(pDevice) == ma_device_state_started && framesProcessed < frameCount) {\n        const void* pMappedPCMFrames;\n        size_t bytesMapped;\n        ma_uint64 framesMapped;\n\n        int pulseResult = ((ma_pa_stream_peek_proc)pDevice->pContext->pulse.pa_stream_peek)(pStream, &pMappedPCMFrames, &bytesMapped);\n        if (pulseResult < 0) {\n            break; /* Failed to map. Abort. */\n        }\n\n        framesMapped = bytesMapped / bpf;\n        if (framesMapped > 0) {\n            if (pMappedPCMFrames != NULL) {\n                ma_device_handle_backend_data_callback(pDevice, NULL, pMappedPCMFrames, framesMapped);\n            } else {\n                /* It's a hole. */\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[PulseAudio] ma_device_on_read__pulse: Hole.\\n\");\n            }\n\n            pulseResult = ((ma_pa_stream_drop_proc)pDevice->pContext->pulse.pa_stream_drop)(pStream);\n            if (pulseResult < 0) {\n                break;  /* Failed to drop the buffer. */\n            }\n\n            framesProcessed += framesMapped;\n\n        } else {\n            /* Nothing was mapped. Just abort. */\n            break;\n        }\n    }\n}\n\nstatic ma_result ma_device_write_to_stream__pulse(ma_device* pDevice, ma_pa_stream* pStream, ma_uint64* pFramesProcessed)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 framesProcessed = 0;\n    size_t bytesMapped;\n    ma_uint32 bpf;\n    ma_uint32 deviceState;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pStream != NULL);\n\n    bpf = ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    MA_ASSERT(bpf > 0);\n\n    deviceState = ma_device_get_state(pDevice);\n\n    bytesMapped = ((ma_pa_stream_writable_size_proc)pDevice->pContext->pulse.pa_stream_writable_size)(pStream);\n    if (bytesMapped != (size_t)-1) {\n        if (bytesMapped > 0) {\n            ma_uint64 framesMapped;\n            void* pMappedPCMFrames;\n            int pulseResult = ((ma_pa_stream_begin_write_proc)pDevice->pContext->pulse.pa_stream_begin_write)(pStream, &pMappedPCMFrames, &bytesMapped);\n            if (pulseResult < 0) {\n                result = ma_result_from_pulse(pulseResult);\n                goto done;\n            }\n\n            framesMapped = bytesMapped / bpf;\n\n            if (deviceState == ma_device_state_started || deviceState == ma_device_state_starting) {  /* Check for starting state just in case this is being used to do the initial fill. */\n                ma_device_handle_backend_data_callback(pDevice, pMappedPCMFrames, NULL, framesMapped);\n            } else {\n                /* Device is not started. Write silence. */\n                ma_silence_pcm_frames(pMappedPCMFrames, framesMapped, pDevice->playback.format, pDevice->playback.channels);\n            }\n\n            pulseResult = ((ma_pa_stream_write_proc)pDevice->pContext->pulse.pa_stream_write)(pStream, pMappedPCMFrames, bytesMapped, NULL, 0, MA_PA_SEEK_RELATIVE);\n            if (pulseResult < 0) {\n                result = ma_result_from_pulse(pulseResult);\n                goto done;  /* Failed to write data to stream. */\n            }\n\n            framesProcessed += framesMapped;\n        } else {\n            result = MA_SUCCESS;  /* No data available for writing. */\n            goto done;\n        }\n    } else {\n        result = MA_ERROR;  /* Failed to retrieve the writable size. Abort. */\n        goto done;\n    }\n\ndone:\n    if (pFramesProcessed != NULL) {\n        *pFramesProcessed = framesProcessed;\n    }\n\n    return result;\n}\n\nstatic void ma_device_on_write__pulse(ma_pa_stream* pStream, size_t byteCount, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    ma_uint32 bpf;\n    ma_uint64 frameCount;\n    ma_uint64 framesProcessed;\n    ma_uint32 deviceState;\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    Don't do anything if the device isn't initialized yet. Yes, this can happen because PulseAudio\n    can fire this callback before the stream has even started. Ridiculous.\n    */\n    deviceState = ma_device_get_state(pDevice);\n    if (deviceState != ma_device_state_starting && deviceState != ma_device_state_started) {\n        return;\n    }\n\n    bpf = ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    MA_ASSERT(bpf > 0);\n\n    frameCount = byteCount / bpf;\n    framesProcessed = 0;\n\n    while (framesProcessed < frameCount) {\n        ma_uint64 framesProcessedThisIteration;\n\n        /* Don't keep trying to process frames if the device isn't started. */\n        deviceState = ma_device_get_state(pDevice);\n        if (deviceState != ma_device_state_starting && deviceState != ma_device_state_started) {\n            break;\n        }\n\n        result = ma_device_write_to_stream__pulse(pDevice, pStream, &framesProcessedThisIteration);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        framesProcessed += framesProcessedThisIteration;\n    }\n}\n\nstatic void ma_device_on_suspended__pulse(ma_pa_stream* pStream, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    int suspended;\n\n    (void)pStream;\n\n    suspended = ((ma_pa_stream_is_suspended_proc)pDevice->pContext->pulse.pa_stream_is_suspended)(pStream);\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[Pulse] Device suspended state changed. pa_stream_is_suspended() returned %d.\\n\", suspended);\n\n    if (suspended < 0) {\n        return;\n    }\n\n    if (suspended == 1) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[Pulse] Device suspended state changed. Suspended.\\n\");\n        ma_device__on_notification_stopped(pDevice);\n    } else {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"[Pulse] Device suspended state changed. Resumed.\\n\");\n        ma_device__on_notification_started(pDevice);\n    }\n}\n\nstatic void ma_device_on_rerouted__pulse(ma_pa_stream* pStream, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n\n    (void)pStream;\n    (void)pUserData;\n\n    ma_device__on_notification_rerouted(pDevice);\n}\n\nstatic ma_uint32 ma_calculate_period_size_in_frames_from_descriptor__pulse(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile)\n{\n    /*\n    There have been reports from users where buffers of < ~20ms result glitches when running through\n    PipeWire. To work around this we're going to have to use a different default buffer size.\n    */\n    const ma_uint32 defaultPeriodSizeInMilliseconds_LowLatency   = 25;\n    const ma_uint32 defaultPeriodSizeInMilliseconds_Conservative = MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE;\n\n    MA_ASSERT(nativeSampleRate != 0);\n\n    if (pDescriptor->periodSizeInFrames == 0) {\n        if (pDescriptor->periodSizeInMilliseconds == 0) {\n            if (performanceProfile == ma_performance_profile_low_latency) {\n                return ma_calculate_buffer_size_in_frames_from_milliseconds(defaultPeriodSizeInMilliseconds_LowLatency, nativeSampleRate);\n            } else {\n                return ma_calculate_buffer_size_in_frames_from_milliseconds(defaultPeriodSizeInMilliseconds_Conservative, nativeSampleRate);\n            }\n        } else {\n            return ma_calculate_buffer_size_in_frames_from_milliseconds(pDescriptor->periodSizeInMilliseconds, nativeSampleRate);\n        }\n    } else {\n        return pDescriptor->periodSizeInFrames;\n    }\n}\n\nstatic ma_result ma_device_init__pulse(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    /*\n    Notes for PulseAudio:\n\n      - When both the period size in frames and milliseconds are 0, we default to miniaudio's\n        default buffer sizes rather than leaving it up to PulseAudio because I don't trust\n        PulseAudio to give us any kind of reasonable latency by default.\n\n      - Do not ever, *ever* forget to use MA_PA_STREAM_ADJUST_LATENCY. If you don't specify this\n        flag, capture mode will just not work properly until you open another PulseAudio app.\n    */\n\n    ma_result result = MA_SUCCESS;\n    int error = 0;\n    const char* devPlayback = NULL;\n    const char* devCapture  = NULL;\n    ma_format format = ma_format_unknown;\n    ma_uint32 channels = 0;\n    ma_uint32 sampleRate = 0;\n    ma_pa_sink_info sinkInfo;\n    ma_pa_source_info sourceInfo;\n    ma_pa_sample_spec ss;\n    ma_pa_channel_map cmap;\n    ma_pa_buffer_attr attr;\n    const ma_pa_sample_spec* pActualSS   = NULL;\n    const ma_pa_buffer_attr* pActualAttr = NULL;\n    const ma_pa_channel_map* pActualChannelMap = NULL;\n    ma_uint32 iChannel;\n    int streamFlags;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ZERO_OBJECT(&pDevice->pulse);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* No exclusive mode with the PulseAudio backend. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pConfig->playback.shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pConfig->capture.shareMode  == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        if (pDescriptorPlayback->pDeviceID != NULL) {\n            devPlayback = pDescriptorPlayback->pDeviceID->pulse;\n        }\n\n        format     = pDescriptorPlayback->format;\n        channels   = pDescriptorPlayback->channels;\n        sampleRate = pDescriptorPlayback->sampleRate;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) {\n        if (pDescriptorCapture->pDeviceID != NULL) {\n            devCapture = pDescriptorCapture->pDeviceID->pulse;\n        }\n\n        format     = pDescriptorCapture->format;\n        channels   = pDescriptorCapture->channels;\n        sampleRate = pDescriptorCapture->sampleRate;\n    }\n\n\n\n    result = ma_init_pa_mainloop_and_pa_context__pulse(pDevice->pContext, pDevice->pContext->pulse.pApplicationName, pDevice->pContext->pulse.pServerName, MA_FALSE, &pDevice->pulse.pMainLoop, &pDevice->pulse.pPulseContext);\n    if (result != MA_SUCCESS) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to initialize PA mainloop and context for device.\\n\");\n        return result;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        result = ma_context_get_source_info__pulse(pDevice->pContext, devCapture, &sourceInfo);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to retrieve source info for capture device.\");\n            goto on_error0;\n        }\n\n        ss   = sourceInfo.sample_spec;\n        cmap = sourceInfo.channel_map;\n\n        /* Use the requested channel count if we have one. */\n        if (pDescriptorCapture->channels != 0) {\n            ss.channels = pDescriptorCapture->channels;\n        }\n\n        /* PulseAudio has a maximum channel count of 32. We'll get a crash if this is exceeded. */\n        if (ss.channels > 32) {\n            ss.channels = 32;\n        }\n\n        /* Use a default channel map. */\n        ((ma_pa_channel_map_init_extend_proc)pDevice->pContext->pulse.pa_channel_map_init_extend)(&cmap, ss.channels, (ma_pa_channel_map_def_t)pConfig->pulse.channelMap);\n\n        /* Use the requested sample rate if one was specified. */\n        if (pDescriptorCapture->sampleRate != 0) {\n            ss.rate = pDescriptorCapture->sampleRate;\n        }\n        streamFlags = MA_PA_STREAM_START_CORKED | MA_PA_STREAM_ADJUST_LATENCY;\n\n        if (ma_format_from_pulse(ss.format) == ma_format_unknown) {\n            if (ma_is_little_endian()) {\n                ss.format = MA_PA_SAMPLE_FLOAT32LE;\n            } else {\n                ss.format = MA_PA_SAMPLE_FLOAT32BE;\n            }\n            streamFlags |= MA_PA_STREAM_FIX_FORMAT;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.format not supported by miniaudio. Defaulting to PA_SAMPLE_FLOAT32.\\n\");\n        }\n        if (ss.rate == 0) {\n            ss.rate = MA_DEFAULT_SAMPLE_RATE;\n            streamFlags |= MA_PA_STREAM_FIX_RATE;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.rate = 0. Defaulting to %d.\\n\", ss.rate);\n        }\n        if (ss.channels == 0) {\n            ss.channels = MA_DEFAULT_CHANNELS;\n            streamFlags |= MA_PA_STREAM_FIX_CHANNELS;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.channels = 0. Defaulting to %d.\\n\", ss.channels);\n        }\n\n        /* We now have enough information to calculate our actual period size in frames. */\n        pDescriptorCapture->periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__pulse(pDescriptorCapture, ss.rate, pConfig->performanceProfile);\n\n        attr = ma_device__pa_buffer_attr_new(pDescriptorCapture->periodSizeInFrames, pDescriptorCapture->periodCount, &ss);\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Capture attr: maxlength=%d, tlength=%d, prebuf=%d, minreq=%d, fragsize=%d; periodSizeInFrames=%d\\n\", attr.maxlength, attr.tlength, attr.prebuf, attr.minreq, attr.fragsize, pDescriptorCapture->periodSizeInFrames);\n\n        pDevice->pulse.pStreamCapture = ma_device__pa_stream_new__pulse(pDevice, pConfig->pulse.pStreamNameCapture, &ss, &cmap);\n        if (pDevice->pulse.pStreamCapture == NULL) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to create PulseAudio capture stream.\\n\");\n            result = MA_ERROR;\n            goto on_error0;\n        }\n\n\n        /* The callback needs to be set before connecting the stream. */\n        ((ma_pa_stream_set_read_callback_proc)pDevice->pContext->pulse.pa_stream_set_read_callback)((ma_pa_stream*)pDevice->pulse.pStreamCapture, ma_device_on_read__pulse, pDevice);\n\n        /* State callback for checking when the device has been corked. */\n        ((ma_pa_stream_set_suspended_callback_proc)pDevice->pContext->pulse.pa_stream_set_suspended_callback)((ma_pa_stream*)pDevice->pulse.pStreamCapture, ma_device_on_suspended__pulse, pDevice);\n\n        /* Rerouting notification. */\n        ((ma_pa_stream_set_moved_callback_proc)pDevice->pContext->pulse.pa_stream_set_moved_callback)((ma_pa_stream*)pDevice->pulse.pStreamCapture, ma_device_on_rerouted__pulse, pDevice);\n\n\n        /* Connect after we've got all of our internal state set up. */\n        if (devCapture != NULL) {\n            streamFlags |= MA_PA_STREAM_DONT_MOVE;\n        }\n\n        error = ((ma_pa_stream_connect_record_proc)pDevice->pContext->pulse.pa_stream_connect_record)((ma_pa_stream*)pDevice->pulse.pStreamCapture, devCapture, &attr, (ma_pa_stream_flags_t)streamFlags);\n        if (error != MA_PA_OK) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to connect PulseAudio capture stream.\");\n            result = ma_result_from_pulse(error);\n            goto on_error1;\n        }\n\n        result = ma_wait_for_pa_stream_to_connect__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, (ma_pa_stream*)pDevice->pulse.pStreamCapture);\n        if (result != MA_SUCCESS) {\n            goto on_error2;\n        }\n\n\n        /* Internal format. */\n        pActualSS = ((ma_pa_stream_get_sample_spec_proc)pDevice->pContext->pulse.pa_stream_get_sample_spec)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n        if (pActualSS != NULL) {\n            ss = *pActualSS;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Capture sample spec: format=%s, channels=%d, rate=%d\\n\", ma_get_format_name(ma_format_from_pulse(ss.format)), ss.channels, ss.rate);\n        } else {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Failed to retrieve capture sample spec.\\n\");\n        }\n\n        pDescriptorCapture->format     = ma_format_from_pulse(ss.format);\n        pDescriptorCapture->channels   = ss.channels;\n        pDescriptorCapture->sampleRate = ss.rate;\n\n        if (pDescriptorCapture->format == ma_format_unknown || pDescriptorCapture->channels == 0 || pDescriptorCapture->sampleRate == 0) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Capture sample spec is invalid. Device unusable by miniaudio. format=%s, channels=%d, sampleRate=%d.\\n\", ma_get_format_name(pDescriptorCapture->format), pDescriptorCapture->channels, pDescriptorCapture->sampleRate);\n            result = MA_ERROR;\n            goto on_error4;\n        }\n\n\n        /* Internal channel map. */\n        pActualChannelMap = ((ma_pa_stream_get_channel_map_proc)pDevice->pContext->pulse.pa_stream_get_channel_map)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n        if (pActualChannelMap == NULL) {\n            pActualChannelMap = &cmap;  /* Fallback just in case. */\n        }\n\n        /*\n        Bug in PipeWire. There have been reports that PipeWire is returning AUX channels when reporting\n        the channel map. To somewhat workaround this, I'm hacking in a hard coded channel map for mono\n        and stereo. In this case it should be safe to assume mono = MONO and stereo = LEFT/RIGHT. For\n        all other channel counts we need to just put up with whatever PipeWire reports and hope it gets\n        fixed sooner than later. I might remove this hack later.\n        */\n        if (pDescriptorCapture->channels > 2) {\n            for (iChannel = 0; iChannel < pDescriptorCapture->channels; iChannel += 1) {\n                pDescriptorCapture->channelMap[iChannel] = ma_channel_position_from_pulse(pActualChannelMap->map[iChannel]);\n            }\n        } else {\n            /* Hack for mono and stereo. */\n            if (pDescriptorCapture->channels == 1) {\n                pDescriptorCapture->channelMap[0] = MA_CHANNEL_MONO;\n            } else if (pDescriptorCapture->channels == 2) {\n                pDescriptorCapture->channelMap[0] = MA_CHANNEL_FRONT_LEFT;\n                pDescriptorCapture->channelMap[1] = MA_CHANNEL_FRONT_RIGHT;\n            } else {\n                MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n            }\n        }\n\n\n        /* Buffer. */\n        pActualAttr = ((ma_pa_stream_get_buffer_attr_proc)pDevice->pContext->pulse.pa_stream_get_buffer_attr)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n        if (pActualAttr != NULL) {\n            attr = *pActualAttr;\n        }\n\n        if (attr.fragsize > 0) {\n            pDescriptorCapture->periodCount = ma_max(attr.maxlength / attr.fragsize, 1);\n        } else {\n            pDescriptorCapture->periodCount = 1;\n        }\n\n        pDescriptorCapture->periodSizeInFrames = attr.maxlength / ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels) / pDescriptorCapture->periodCount;\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Capture actual attr: maxlength=%d, tlength=%d, prebuf=%d, minreq=%d, fragsize=%d; periodSizeInFrames=%d\\n\", attr.maxlength, attr.tlength, attr.prebuf, attr.minreq, attr.fragsize, pDescriptorCapture->periodSizeInFrames);\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        result = ma_context_get_sink_info__pulse(pDevice->pContext, devPlayback, &sinkInfo);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to retrieve sink info for playback device.\\n\");\n            goto on_error2;\n        }\n\n        ss   = sinkInfo.sample_spec;\n        cmap = sinkInfo.channel_map;\n\n        /* Use the requested channel count if we have one. */\n        if (pDescriptorPlayback->channels != 0) {\n            ss.channels = pDescriptorPlayback->channels;\n        }\n\n        /* PulseAudio has a maximum channel count of 32. We'll get a crash if this is exceeded. */\n        if (ss.channels > 32) {\n            ss.channels = 32;\n        }\n\n        /* Use a default channel map. */\n        ((ma_pa_channel_map_init_extend_proc)pDevice->pContext->pulse.pa_channel_map_init_extend)(&cmap, ss.channels, (ma_pa_channel_map_def_t)pConfig->pulse.channelMap);\n\n\n        /* Use the requested sample rate if one was specified. */\n        if (pDescriptorPlayback->sampleRate != 0) {\n            ss.rate = pDescriptorPlayback->sampleRate;\n        }\n\n        streamFlags = MA_PA_STREAM_START_CORKED | MA_PA_STREAM_ADJUST_LATENCY;\n        if (ma_format_from_pulse(ss.format) == ma_format_unknown) {\n            if (ma_is_little_endian()) {\n                ss.format = MA_PA_SAMPLE_FLOAT32LE;\n            } else {\n                ss.format = MA_PA_SAMPLE_FLOAT32BE;\n            }\n            streamFlags |= MA_PA_STREAM_FIX_FORMAT;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.format not supported by miniaudio. Defaulting to PA_SAMPLE_FLOAT32.\\n\");\n        }\n        if (ss.rate == 0) {\n            ss.rate = MA_DEFAULT_SAMPLE_RATE;\n            streamFlags |= MA_PA_STREAM_FIX_RATE;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.rate = 0. Defaulting to %d.\\n\", ss.rate);\n        }\n        if (ss.channels == 0) {\n            ss.channels = MA_DEFAULT_CHANNELS;\n            streamFlags |= MA_PA_STREAM_FIX_CHANNELS;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] sample_spec.channels = 0. Defaulting to %d.\\n\", ss.channels);\n        }\n\n        /* We now have enough information to calculate the actual buffer size in frames. */\n        pDescriptorPlayback->periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__pulse(pDescriptorPlayback, ss.rate, pConfig->performanceProfile);\n\n        attr = ma_device__pa_buffer_attr_new(pDescriptorPlayback->periodSizeInFrames, pDescriptorPlayback->periodCount, &ss);\n\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Playback attr: maxlength=%d, tlength=%d, prebuf=%d, minreq=%d, fragsize=%d; periodSizeInFrames=%d\\n\", attr.maxlength, attr.tlength, attr.prebuf, attr.minreq, attr.fragsize, pDescriptorPlayback->periodSizeInFrames);\n\n        pDevice->pulse.pStreamPlayback = ma_device__pa_stream_new__pulse(pDevice, pConfig->pulse.pStreamNamePlayback, &ss, &cmap);\n        if (pDevice->pulse.pStreamPlayback == NULL) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to create PulseAudio playback stream.\\n\");\n            result = MA_ERROR;\n            goto on_error2;\n        }\n\n\n        /*\n        Note that this callback will be fired as soon as the stream is connected, even though it's started as corked. The callback needs to handle a\n        device state of ma_device_state_uninitialized.\n        */\n        ((ma_pa_stream_set_write_callback_proc)pDevice->pContext->pulse.pa_stream_set_write_callback)((ma_pa_stream*)pDevice->pulse.pStreamPlayback, ma_device_on_write__pulse, pDevice);\n\n        /* State callback for checking when the device has been corked. */\n        ((ma_pa_stream_set_suspended_callback_proc)pDevice->pContext->pulse.pa_stream_set_suspended_callback)((ma_pa_stream*)pDevice->pulse.pStreamPlayback, ma_device_on_suspended__pulse, pDevice);\n\n        /* Rerouting notification. */\n        ((ma_pa_stream_set_moved_callback_proc)pDevice->pContext->pulse.pa_stream_set_moved_callback)((ma_pa_stream*)pDevice->pulse.pStreamPlayback, ma_device_on_rerouted__pulse, pDevice);\n\n\n        /* Connect after we've got all of our internal state set up. */\n        if (devPlayback != NULL) {\n            streamFlags |= MA_PA_STREAM_DONT_MOVE;\n        }\n\n        error = ((ma_pa_stream_connect_playback_proc)pDevice->pContext->pulse.pa_stream_connect_playback)((ma_pa_stream*)pDevice->pulse.pStreamPlayback, devPlayback, &attr, (ma_pa_stream_flags_t)streamFlags, NULL, NULL);\n        if (error != MA_PA_OK) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to connect PulseAudio playback stream.\");\n            result = ma_result_from_pulse(error);\n            goto on_error3;\n        }\n\n        result = ma_wait_for_pa_stream_to_connect__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, (ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n        if (result != MA_SUCCESS) {\n            goto on_error3;\n        }\n\n\n        /* Internal format. */\n        pActualSS = ((ma_pa_stream_get_sample_spec_proc)pDevice->pContext->pulse.pa_stream_get_sample_spec)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n        if (pActualSS != NULL) {\n            ss = *pActualSS;\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Playback sample spec: format=%s, channels=%d, rate=%d\\n\", ma_get_format_name(ma_format_from_pulse(ss.format)), ss.channels, ss.rate);\n        } else {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Failed to retrieve playback sample spec.\\n\");\n        }\n\n        pDescriptorPlayback->format     = ma_format_from_pulse(ss.format);\n        pDescriptorPlayback->channels   = ss.channels;\n        pDescriptorPlayback->sampleRate = ss.rate;\n\n        if (pDescriptorPlayback->format == ma_format_unknown || pDescriptorPlayback->channels == 0 || pDescriptorPlayback->sampleRate == 0) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Playback sample spec is invalid. Device unusable by miniaudio. format=%s, channels=%d, sampleRate=%d.\\n\", ma_get_format_name(pDescriptorPlayback->format), pDescriptorPlayback->channels, pDescriptorPlayback->sampleRate);\n            result = MA_ERROR;\n            goto on_error4;\n        }\n\n\n        /* Internal channel map. */\n        pActualChannelMap = ((ma_pa_stream_get_channel_map_proc)pDevice->pContext->pulse.pa_stream_get_channel_map)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n        if (pActualChannelMap == NULL) {\n            pActualChannelMap = &cmap;  /* Fallback just in case. */\n        }\n\n        /*\n        Bug in PipeWire. There have been reports that PipeWire is returning AUX channels when reporting\n        the channel map. To somewhat workaround this, I'm hacking in a hard coded channel map for mono\n        and stereo. In this case it should be safe to assume mono = MONO and stereo = LEFT/RIGHT. For\n        all other channel counts we need to just put up with whatever PipeWire reports and hope it gets\n        fixed sooner than later. I might remove this hack later.\n        */\n        if (pDescriptorPlayback->channels > 2) {\n            for (iChannel = 0; iChannel < pDescriptorPlayback->channels; iChannel += 1) {\n                pDescriptorPlayback->channelMap[iChannel] = ma_channel_position_from_pulse(pActualChannelMap->map[iChannel]);\n            }\n        } else {\n            /* Hack for mono and stereo. */\n            if (pDescriptorPlayback->channels == 1) {\n                pDescriptorPlayback->channelMap[0] = MA_CHANNEL_MONO;\n            } else if (pDescriptorPlayback->channels == 2) {\n                pDescriptorPlayback->channelMap[0] = MA_CHANNEL_FRONT_LEFT;\n                pDescriptorPlayback->channelMap[1] = MA_CHANNEL_FRONT_RIGHT;\n            } else {\n                MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n            }\n        }\n\n\n        /* Buffer. */\n        pActualAttr = ((ma_pa_stream_get_buffer_attr_proc)pDevice->pContext->pulse.pa_stream_get_buffer_attr)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n        if (pActualAttr != NULL) {\n            attr = *pActualAttr;\n        }\n\n        if (attr.tlength > 0) {\n            pDescriptorPlayback->periodCount = ma_max(attr.maxlength / attr.tlength, 1);\n        } else {\n            pDescriptorPlayback->periodCount = 1;\n        }\n\n        pDescriptorPlayback->periodSizeInFrames = attr.maxlength / ma_get_bytes_per_frame(pDescriptorPlayback->format, pDescriptorPlayback->channels) / pDescriptorPlayback->periodCount;\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[PulseAudio] Playback actual attr: maxlength=%d, tlength=%d, prebuf=%d, minreq=%d, fragsize=%d; internalPeriodSizeInFrames=%d\\n\", attr.maxlength, attr.tlength, attr.prebuf, attr.minreq, attr.fragsize, pDescriptorPlayback->periodSizeInFrames);\n    }\n\n\n    /*\n    We need a ring buffer for handling duplex mode. We can use the main duplex ring buffer in the main\n    part of the ma_device struct. We cannot, however, depend on ma_device_init() initializing this for\n    us later on because that will only do it if it's a fully asynchronous backend - i.e. the\n    onDeviceDataLoop callback is NULL, which is not the case for PulseAudio.\n    */\n    if (pConfig->deviceType == ma_device_type_duplex) {\n        ma_format rbFormat     = (format != ma_format_unknown) ? format     : pDescriptorCapture->format;\n        ma_uint32 rbChannels   = (channels   > 0)              ? channels   : pDescriptorCapture->channels;\n        ma_uint32 rbSampleRate = (sampleRate > 0)              ? sampleRate : pDescriptorCapture->sampleRate;\n\n        result = ma_duplex_rb_init(rbFormat, rbChannels, rbSampleRate, pDescriptorCapture->sampleRate, pDescriptorCapture->periodSizeInFrames, &pDevice->pContext->allocationCallbacks, &pDevice->duplexRB);\n        if (result != MA_SUCCESS) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to initialize ring buffer. %s.\\n\", ma_result_description(result));\n            goto on_error4;\n        }\n    }\n\n    return MA_SUCCESS;\n\n\non_error4:\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ((ma_pa_stream_disconnect_proc)pDevice->pContext->pulse.pa_stream_disconnect)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n    }\non_error3:\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ((ma_pa_stream_unref_proc)pDevice->pContext->pulse.pa_stream_unref)((ma_pa_stream*)pDevice->pulse.pStreamPlayback);\n    }\non_error2:\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ((ma_pa_stream_disconnect_proc)pDevice->pContext->pulse.pa_stream_disconnect)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n    }\non_error1:\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ((ma_pa_stream_unref_proc)pDevice->pContext->pulse.pa_stream_unref)((ma_pa_stream*)pDevice->pulse.pStreamCapture);\n    }\non_error0:\n    return result;\n}\n\n\nstatic void ma_pulse_operation_complete_callback(ma_pa_stream* pStream, int success, void* pUserData)\n{\n    ma_bool32* pIsSuccessful = (ma_bool32*)pUserData;\n    MA_ASSERT(pIsSuccessful != NULL);\n\n    *pIsSuccessful = (ma_bool32)success;\n\n    (void)pStream; /* Unused. */\n}\n\nstatic ma_result ma_device__cork_stream__pulse(ma_device* pDevice, ma_device_type deviceType, int cork)\n{\n    ma_context* pContext = pDevice->pContext;\n    ma_bool32 wasSuccessful;\n    ma_pa_stream* pStream;\n    ma_pa_operation* pOP;\n    ma_result result;\n\n    /* This should not be called with a duplex device type. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    wasSuccessful = MA_FALSE;\n\n    pStream = (ma_pa_stream*)((deviceType == ma_device_type_capture) ? pDevice->pulse.pStreamCapture : pDevice->pulse.pStreamPlayback);\n    MA_ASSERT(pStream != NULL);\n\n    pOP = ((ma_pa_stream_cork_proc)pContext->pulse.pa_stream_cork)(pStream, cork, ma_pulse_operation_complete_callback, &wasSuccessful);\n    if (pOP == NULL) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to cork PulseAudio stream.\");\n        return MA_ERROR;\n    }\n\n    result = ma_wait_for_operation_and_unref__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, pOP);\n    if (result != MA_SUCCESS) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] An error occurred while waiting for the PulseAudio stream to cork.\");\n        return result;\n    }\n\n    if (!wasSuccessful) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[PulseAudio] Failed to %s PulseAudio stream.\", (cork) ? \"stop\" : \"start\");\n        return MA_ERROR;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__pulse(ma_device* pDevice)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        result = ma_device__cork_stream__pulse(pDevice, ma_device_type_capture, 0);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        /*\n        We need to fill some data before uncorking. Not doing this will result in the write callback\n        never getting fired. We're not going to abort if writing fails because I still want the device\n        to get uncorked.\n        */\n        ma_device_write_to_stream__pulse(pDevice, (ma_pa_stream*)(pDevice->pulse.pStreamPlayback), NULL);   /* No need to check the result here. Always want to fall through an uncork.*/\n\n        result = ma_device__cork_stream__pulse(pDevice, ma_device_type_playback, 0);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__pulse(ma_device* pDevice)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        result = ma_device__cork_stream__pulse(pDevice, ma_device_type_capture, 1);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        /*\n        Ideally we would drain the device here, but there's been cases where PulseAudio seems to be\n        broken on some systems to the point where no audio processing seems to happen. When this\n        happens, draining never completes and we get stuck here. For now I'm disabling draining of\n        the device so we don't just freeze the application.\n        */\n    #if 0\n        ma_pa_operation* pOP = ((ma_pa_stream_drain_proc)pDevice->pContext->pulse.pa_stream_drain)((ma_pa_stream*)pDevice->pulse.pStreamPlayback, ma_pulse_operation_complete_callback, &wasSuccessful);\n        ma_wait_for_operation_and_unref__pulse(pDevice->pContext, pDevice->pulse.pMainLoop, pOP);\n    #endif\n\n        result = ma_device__cork_stream__pulse(pDevice, ma_device_type_playback, 1);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_data_loop__pulse(ma_device* pDevice)\n{\n    int resultPA;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* NOTE: Don't start the device here. It'll be done at a higher level. */\n\n    /*\n    All data is handled through callbacks. All we need to do is iterate over the main loop and let\n    the callbacks deal with it.\n    */\n    while (ma_device_get_state(pDevice) == ma_device_state_started) {\n        resultPA = ((ma_pa_mainloop_iterate_proc)pDevice->pContext->pulse.pa_mainloop_iterate)((ma_pa_mainloop*)pDevice->pulse.pMainLoop, 1, NULL);\n        if (resultPA < 0) {\n            break;\n        }\n    }\n\n    /* NOTE: Don't stop the device here. It'll be done at a higher level. */\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_data_loop_wakeup__pulse(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    ((ma_pa_mainloop_wakeup_proc)pDevice->pContext->pulse.pa_mainloop_wakeup)((ma_pa_mainloop*)pDevice->pulse.pMainLoop);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__pulse(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_pulseaudio);\n\n    ((ma_pa_context_disconnect_proc)pContext->pulse.pa_context_disconnect)((ma_pa_context*)pContext->pulse.pPulseContext);\n    ((ma_pa_context_unref_proc)pContext->pulse.pa_context_unref)((ma_pa_context*)pContext->pulse.pPulseContext);\n    ((ma_pa_mainloop_free_proc)pContext->pulse.pa_mainloop_free)((ma_pa_mainloop*)pContext->pulse.pMainLoop);\n\n    ma_free(pContext->pulse.pServerName, &pContext->allocationCallbacks);\n    ma_free(pContext->pulse.pApplicationName, &pContext->allocationCallbacks);\n\n#ifndef MA_NO_RUNTIME_LINKING\n    ma_dlclose(ma_context_get_log(pContext), pContext->pulse.pulseSO);\n#endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__pulse(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    ma_result result;\n#ifndef MA_NO_RUNTIME_LINKING\n    const char* libpulseNames[] = {\n        \"libpulse.so\",\n        \"libpulse.so.0\"\n    };\n    size_t i;\n\n    for (i = 0; i < ma_countof(libpulseNames); ++i) {\n        pContext->pulse.pulseSO = ma_dlopen(ma_context_get_log(pContext), libpulseNames[i]);\n        if (pContext->pulse.pulseSO != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->pulse.pulseSO == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    pContext->pulse.pa_mainloop_new                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_new\");\n    pContext->pulse.pa_mainloop_free                   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_free\");\n    pContext->pulse.pa_mainloop_quit                   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_quit\");\n    pContext->pulse.pa_mainloop_get_api                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_get_api\");\n    pContext->pulse.pa_mainloop_iterate                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_iterate\");\n    pContext->pulse.pa_mainloop_wakeup                 = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_mainloop_wakeup\");\n    pContext->pulse.pa_threaded_mainloop_new           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_new\");\n    pContext->pulse.pa_threaded_mainloop_free          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_free\");\n    pContext->pulse.pa_threaded_mainloop_start         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_start\");\n    pContext->pulse.pa_threaded_mainloop_stop          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_stop\");\n    pContext->pulse.pa_threaded_mainloop_lock          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_lock\");\n    pContext->pulse.pa_threaded_mainloop_unlock        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_unlock\");\n    pContext->pulse.pa_threaded_mainloop_wait          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_wait\");\n    pContext->pulse.pa_threaded_mainloop_signal        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_signal\");\n    pContext->pulse.pa_threaded_mainloop_accept        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_accept\");\n    pContext->pulse.pa_threaded_mainloop_get_retval    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_get_retval\");\n    pContext->pulse.pa_threaded_mainloop_get_api       = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_get_api\");\n    pContext->pulse.pa_threaded_mainloop_in_thread     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_in_thread\");\n    pContext->pulse.pa_threaded_mainloop_set_name      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_threaded_mainloop_set_name\");\n    pContext->pulse.pa_context_new                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_new\");\n    pContext->pulse.pa_context_unref                   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_unref\");\n    pContext->pulse.pa_context_connect                 = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_connect\");\n    pContext->pulse.pa_context_disconnect              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_disconnect\");\n    pContext->pulse.pa_context_set_state_callback      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_set_state_callback\");\n    pContext->pulse.pa_context_get_state               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_get_state\");\n    pContext->pulse.pa_context_get_sink_info_list      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_get_sink_info_list\");\n    pContext->pulse.pa_context_get_source_info_list    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_get_source_info_list\");\n    pContext->pulse.pa_context_get_sink_info_by_name   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_get_sink_info_by_name\");\n    pContext->pulse.pa_context_get_source_info_by_name = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_context_get_source_info_by_name\");\n    pContext->pulse.pa_operation_unref                 = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_operation_unref\");\n    pContext->pulse.pa_operation_get_state             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_operation_get_state\");\n    pContext->pulse.pa_channel_map_init_extend         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_channel_map_init_extend\");\n    pContext->pulse.pa_channel_map_valid               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_channel_map_valid\");\n    pContext->pulse.pa_channel_map_compatible          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_channel_map_compatible\");\n    pContext->pulse.pa_stream_new                      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_new\");\n    pContext->pulse.pa_stream_unref                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_unref\");\n    pContext->pulse.pa_stream_connect_playback         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_connect_playback\");\n    pContext->pulse.pa_stream_connect_record           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_connect_record\");\n    pContext->pulse.pa_stream_disconnect               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_disconnect\");\n    pContext->pulse.pa_stream_get_state                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_get_state\");\n    pContext->pulse.pa_stream_get_sample_spec          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_get_sample_spec\");\n    pContext->pulse.pa_stream_get_channel_map          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_get_channel_map\");\n    pContext->pulse.pa_stream_get_buffer_attr          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_get_buffer_attr\");\n    pContext->pulse.pa_stream_set_buffer_attr          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_set_buffer_attr\");\n    pContext->pulse.pa_stream_get_device_name          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_get_device_name\");\n    pContext->pulse.pa_stream_set_write_callback       = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_set_write_callback\");\n    pContext->pulse.pa_stream_set_read_callback        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_set_read_callback\");\n    pContext->pulse.pa_stream_set_suspended_callback   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_set_suspended_callback\");\n    pContext->pulse.pa_stream_set_moved_callback       = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_set_moved_callback\");\n    pContext->pulse.pa_stream_is_suspended             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_is_suspended\");\n    pContext->pulse.pa_stream_flush                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_flush\");\n    pContext->pulse.pa_stream_drain                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_drain\");\n    pContext->pulse.pa_stream_is_corked                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_is_corked\");\n    pContext->pulse.pa_stream_cork                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_cork\");\n    pContext->pulse.pa_stream_trigger                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_trigger\");\n    pContext->pulse.pa_stream_begin_write              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_begin_write\");\n    pContext->pulse.pa_stream_write                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_write\");\n    pContext->pulse.pa_stream_peek                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_peek\");\n    pContext->pulse.pa_stream_drop                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_drop\");\n    pContext->pulse.pa_stream_writable_size            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_writable_size\");\n    pContext->pulse.pa_stream_readable_size            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->pulse.pulseSO, \"pa_stream_readable_size\");\n#else\n    /* This strange assignment system is just for type safety. */\n    ma_pa_mainloop_new_proc                    _pa_mainloop_new                   = pa_mainloop_new;\n    ma_pa_mainloop_free_proc                   _pa_mainloop_free                  = pa_mainloop_free;\n    ma_pa_mainloop_quit_proc                   _pa_mainloop_quit                  = pa_mainloop_quit;\n    ma_pa_mainloop_get_api_proc                _pa_mainloop_get_api               = pa_mainloop_get_api;\n    ma_pa_mainloop_iterate_proc                _pa_mainloop_iterate               = pa_mainloop_iterate;\n    ma_pa_mainloop_wakeup_proc                 _pa_mainloop_wakeup                = pa_mainloop_wakeup;\n    ma_pa_threaded_mainloop_new_proc           _pa_threaded_mainloop_new          = pa_threaded_mainloop_new;\n    ma_pa_threaded_mainloop_free_proc          _pa_threaded_mainloop_free         = pa_threaded_mainloop_free;\n    ma_pa_threaded_mainloop_start_proc         _pa_threaded_mainloop_start        = pa_threaded_mainloop_start;\n    ma_pa_threaded_mainloop_stop_proc          _pa_threaded_mainloop_stop         = pa_threaded_mainloop_stop;\n    ma_pa_threaded_mainloop_lock_proc          _pa_threaded_mainloop_lock         = pa_threaded_mainloop_lock;\n    ma_pa_threaded_mainloop_unlock_proc        _pa_threaded_mainloop_unlock       = pa_threaded_mainloop_unlock;\n    ma_pa_threaded_mainloop_wait_proc          _pa_threaded_mainloop_wait         = pa_threaded_mainloop_wait;\n    ma_pa_threaded_mainloop_signal_proc        _pa_threaded_mainloop_signal       = pa_threaded_mainloop_signal;\n    ma_pa_threaded_mainloop_accept_proc        _pa_threaded_mainloop_accept       = pa_threaded_mainloop_accept;\n    ma_pa_threaded_mainloop_get_retval_proc    _pa_threaded_mainloop_get_retval   = pa_threaded_mainloop_get_retval;\n    ma_pa_threaded_mainloop_get_api_proc       _pa_threaded_mainloop_get_api      = pa_threaded_mainloop_get_api;\n    ma_pa_threaded_mainloop_in_thread_proc     _pa_threaded_mainloop_in_thread    = pa_threaded_mainloop_in_thread;\n    ma_pa_threaded_mainloop_set_name_proc      _pa_threaded_mainloop_set_name     = pa_threaded_mainloop_set_name;\n    ma_pa_context_new_proc                     _pa_context_new                    = pa_context_new;\n    ma_pa_context_unref_proc                   _pa_context_unref                  = pa_context_unref;\n    ma_pa_context_connect_proc                 _pa_context_connect                = pa_context_connect;\n    ma_pa_context_disconnect_proc              _pa_context_disconnect             = pa_context_disconnect;\n    ma_pa_context_set_state_callback_proc      _pa_context_set_state_callback     = pa_context_set_state_callback;\n    ma_pa_context_get_state_proc               _pa_context_get_state              = pa_context_get_state;\n    ma_pa_context_get_sink_info_list_proc      _pa_context_get_sink_info_list     = pa_context_get_sink_info_list;\n    ma_pa_context_get_source_info_list_proc    _pa_context_get_source_info_list   = pa_context_get_source_info_list;\n    ma_pa_context_get_sink_info_by_name_proc   _pa_context_get_sink_info_by_name  = pa_context_get_sink_info_by_name;\n    ma_pa_context_get_source_info_by_name_proc _pa_context_get_source_info_by_name= pa_context_get_source_info_by_name;\n    ma_pa_operation_unref_proc                 _pa_operation_unref                = pa_operation_unref;\n    ma_pa_operation_get_state_proc             _pa_operation_get_state            = pa_operation_get_state;\n    ma_pa_channel_map_init_extend_proc         _pa_channel_map_init_extend        = pa_channel_map_init_extend;\n    ma_pa_channel_map_valid_proc               _pa_channel_map_valid              = pa_channel_map_valid;\n    ma_pa_channel_map_compatible_proc          _pa_channel_map_compatible         = pa_channel_map_compatible;\n    ma_pa_stream_new_proc                      _pa_stream_new                     = pa_stream_new;\n    ma_pa_stream_unref_proc                    _pa_stream_unref                   = pa_stream_unref;\n    ma_pa_stream_connect_playback_proc         _pa_stream_connect_playback        = pa_stream_connect_playback;\n    ma_pa_stream_connect_record_proc           _pa_stream_connect_record          = pa_stream_connect_record;\n    ma_pa_stream_disconnect_proc               _pa_stream_disconnect              = pa_stream_disconnect;\n    ma_pa_stream_get_state_proc                _pa_stream_get_state               = pa_stream_get_state;\n    ma_pa_stream_get_sample_spec_proc          _pa_stream_get_sample_spec         = pa_stream_get_sample_spec;\n    ma_pa_stream_get_channel_map_proc          _pa_stream_get_channel_map         = pa_stream_get_channel_map;\n    ma_pa_stream_get_buffer_attr_proc          _pa_stream_get_buffer_attr         = pa_stream_get_buffer_attr;\n    ma_pa_stream_set_buffer_attr_proc          _pa_stream_set_buffer_attr         = pa_stream_set_buffer_attr;\n    ma_pa_stream_get_device_name_proc          _pa_stream_get_device_name         = pa_stream_get_device_name;\n    ma_pa_stream_set_write_callback_proc       _pa_stream_set_write_callback      = pa_stream_set_write_callback;\n    ma_pa_stream_set_read_callback_proc        _pa_stream_set_read_callback       = pa_stream_set_read_callback;\n    ma_pa_stream_set_suspended_callback_proc   _pa_stream_set_suspended_callback  = pa_stream_set_suspended_callback;\n    ma_pa_stream_set_moved_callback_proc       _pa_stream_set_moved_callback      = pa_stream_set_moved_callback;\n    ma_pa_stream_is_suspended_proc             _pa_stream_is_suspended            = pa_stream_is_suspended;\n    ma_pa_stream_flush_proc                    _pa_stream_flush                   = pa_stream_flush;\n    ma_pa_stream_drain_proc                    _pa_stream_drain                   = pa_stream_drain;\n    ma_pa_stream_is_corked_proc                _pa_stream_is_corked               = pa_stream_is_corked;\n    ma_pa_stream_cork_proc                     _pa_stream_cork                    = pa_stream_cork;\n    ma_pa_stream_trigger_proc                  _pa_stream_trigger                 = pa_stream_trigger;\n    ma_pa_stream_begin_write_proc              _pa_stream_begin_write             = pa_stream_begin_write;\n    ma_pa_stream_write_proc                    _pa_stream_write                   = pa_stream_write;\n    ma_pa_stream_peek_proc                     _pa_stream_peek                    = pa_stream_peek;\n    ma_pa_stream_drop_proc                     _pa_stream_drop                    = pa_stream_drop;\n    ma_pa_stream_writable_size_proc            _pa_stream_writable_size           = pa_stream_writable_size;\n    ma_pa_stream_readable_size_proc            _pa_stream_readable_size           = pa_stream_readable_size;\n\n    pContext->pulse.pa_mainloop_new                    = (ma_proc)_pa_mainloop_new;\n    pContext->pulse.pa_mainloop_free                   = (ma_proc)_pa_mainloop_free;\n    pContext->pulse.pa_mainloop_quit                   = (ma_proc)_pa_mainloop_quit;\n    pContext->pulse.pa_mainloop_get_api                = (ma_proc)_pa_mainloop_get_api;\n    pContext->pulse.pa_mainloop_iterate                = (ma_proc)_pa_mainloop_iterate;\n    pContext->pulse.pa_mainloop_wakeup                 = (ma_proc)_pa_mainloop_wakeup;\n    pContext->pulse.pa_threaded_mainloop_new           = (ma_proc)_pa_threaded_mainloop_new;\n    pContext->pulse.pa_threaded_mainloop_free          = (ma_proc)_pa_threaded_mainloop_free;\n    pContext->pulse.pa_threaded_mainloop_start         = (ma_proc)_pa_threaded_mainloop_start;\n    pContext->pulse.pa_threaded_mainloop_stop          = (ma_proc)_pa_threaded_mainloop_stop;\n    pContext->pulse.pa_threaded_mainloop_lock          = (ma_proc)_pa_threaded_mainloop_lock;\n    pContext->pulse.pa_threaded_mainloop_unlock        = (ma_proc)_pa_threaded_mainloop_unlock;\n    pContext->pulse.pa_threaded_mainloop_wait          = (ma_proc)_pa_threaded_mainloop_wait;\n    pContext->pulse.pa_threaded_mainloop_signal        = (ma_proc)_pa_threaded_mainloop_signal;\n    pContext->pulse.pa_threaded_mainloop_accept        = (ma_proc)_pa_threaded_mainloop_accept;\n    pContext->pulse.pa_threaded_mainloop_get_retval    = (ma_proc)_pa_threaded_mainloop_get_retval;\n    pContext->pulse.pa_threaded_mainloop_get_api       = (ma_proc)_pa_threaded_mainloop_get_api;\n    pContext->pulse.pa_threaded_mainloop_in_thread     = (ma_proc)_pa_threaded_mainloop_in_thread;\n    pContext->pulse.pa_threaded_mainloop_set_name      = (ma_proc)_pa_threaded_mainloop_set_name;\n    pContext->pulse.pa_context_new                     = (ma_proc)_pa_context_new;\n    pContext->pulse.pa_context_unref                   = (ma_proc)_pa_context_unref;\n    pContext->pulse.pa_context_connect                 = (ma_proc)_pa_context_connect;\n    pContext->pulse.pa_context_disconnect              = (ma_proc)_pa_context_disconnect;\n    pContext->pulse.pa_context_set_state_callback      = (ma_proc)_pa_context_set_state_callback;\n    pContext->pulse.pa_context_get_state               = (ma_proc)_pa_context_get_state;\n    pContext->pulse.pa_context_get_sink_info_list      = (ma_proc)_pa_context_get_sink_info_list;\n    pContext->pulse.pa_context_get_source_info_list    = (ma_proc)_pa_context_get_source_info_list;\n    pContext->pulse.pa_context_get_sink_info_by_name   = (ma_proc)_pa_context_get_sink_info_by_name;\n    pContext->pulse.pa_context_get_source_info_by_name = (ma_proc)_pa_context_get_source_info_by_name;\n    pContext->pulse.pa_operation_unref                 = (ma_proc)_pa_operation_unref;\n    pContext->pulse.pa_operation_get_state             = (ma_proc)_pa_operation_get_state;\n    pContext->pulse.pa_channel_map_init_extend         = (ma_proc)_pa_channel_map_init_extend;\n    pContext->pulse.pa_channel_map_valid               = (ma_proc)_pa_channel_map_valid;\n    pContext->pulse.pa_channel_map_compatible          = (ma_proc)_pa_channel_map_compatible;\n    pContext->pulse.pa_stream_new                      = (ma_proc)_pa_stream_new;\n    pContext->pulse.pa_stream_unref                    = (ma_proc)_pa_stream_unref;\n    pContext->pulse.pa_stream_connect_playback         = (ma_proc)_pa_stream_connect_playback;\n    pContext->pulse.pa_stream_connect_record           = (ma_proc)_pa_stream_connect_record;\n    pContext->pulse.pa_stream_disconnect               = (ma_proc)_pa_stream_disconnect;\n    pContext->pulse.pa_stream_get_state                = (ma_proc)_pa_stream_get_state;\n    pContext->pulse.pa_stream_get_sample_spec          = (ma_proc)_pa_stream_get_sample_spec;\n    pContext->pulse.pa_stream_get_channel_map          = (ma_proc)_pa_stream_get_channel_map;\n    pContext->pulse.pa_stream_get_buffer_attr          = (ma_proc)_pa_stream_get_buffer_attr;\n    pContext->pulse.pa_stream_set_buffer_attr          = (ma_proc)_pa_stream_set_buffer_attr;\n    pContext->pulse.pa_stream_get_device_name          = (ma_proc)_pa_stream_get_device_name;\n    pContext->pulse.pa_stream_set_write_callback       = (ma_proc)_pa_stream_set_write_callback;\n    pContext->pulse.pa_stream_set_read_callback        = (ma_proc)_pa_stream_set_read_callback;\n    pContext->pulse.pa_stream_set_suspended_callback   = (ma_proc)_pa_stream_set_suspended_callback;\n    pContext->pulse.pa_stream_set_moved_callback       = (ma_proc)_pa_stream_set_moved_callback;\n    pContext->pulse.pa_stream_is_suspended             = (ma_proc)_pa_stream_is_suspended;\n    pContext->pulse.pa_stream_flush                    = (ma_proc)_pa_stream_flush;\n    pContext->pulse.pa_stream_drain                    = (ma_proc)_pa_stream_drain;\n    pContext->pulse.pa_stream_is_corked                = (ma_proc)_pa_stream_is_corked;\n    pContext->pulse.pa_stream_cork                     = (ma_proc)_pa_stream_cork;\n    pContext->pulse.pa_stream_trigger                  = (ma_proc)_pa_stream_trigger;\n    pContext->pulse.pa_stream_begin_write              = (ma_proc)_pa_stream_begin_write;\n    pContext->pulse.pa_stream_write                    = (ma_proc)_pa_stream_write;\n    pContext->pulse.pa_stream_peek                     = (ma_proc)_pa_stream_peek;\n    pContext->pulse.pa_stream_drop                     = (ma_proc)_pa_stream_drop;\n    pContext->pulse.pa_stream_writable_size            = (ma_proc)_pa_stream_writable_size;\n    pContext->pulse.pa_stream_readable_size            = (ma_proc)_pa_stream_readable_size;\n#endif\n\n    /* We need to make a copy of the application and server names so we can pass them to the pa_context of each device. */\n    pContext->pulse.pApplicationName = ma_copy_string(pConfig->pulse.pApplicationName, &pContext->allocationCallbacks);\n    if (pContext->pulse.pApplicationName == NULL && pConfig->pulse.pApplicationName != NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    pContext->pulse.pServerName = ma_copy_string(pConfig->pulse.pServerName, &pContext->allocationCallbacks);\n    if (pContext->pulse.pServerName == NULL && pConfig->pulse.pServerName != NULL) {\n        ma_free(pContext->pulse.pApplicationName, &pContext->allocationCallbacks);\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_init_pa_mainloop_and_pa_context__pulse(pContext, pConfig->pulse.pApplicationName, pConfig->pulse.pServerName, pConfig->pulse.tryAutoSpawn, &pContext->pulse.pMainLoop, &pContext->pulse.pPulseContext);\n    if (result != MA_SUCCESS) {\n        ma_free(pContext->pulse.pServerName, &pContext->allocationCallbacks);\n        ma_free(pContext->pulse.pApplicationName, &pContext->allocationCallbacks);\n    #ifndef MA_NO_RUNTIME_LINKING\n        ma_dlclose(ma_context_get_log(pContext), pContext->pulse.pulseSO);\n    #endif\n        return result;\n    }\n\n    /* With pa_mainloop we run a synchronous backend, but we implement our own main loop. */\n    pCallbacks->onContextInit             = ma_context_init__pulse;\n    pCallbacks->onContextUninit           = ma_context_uninit__pulse;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__pulse;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__pulse;\n    pCallbacks->onDeviceInit              = ma_device_init__pulse;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__pulse;\n    pCallbacks->onDeviceStart             = ma_device_start__pulse;\n    pCallbacks->onDeviceStop              = ma_device_stop__pulse;\n    pCallbacks->onDeviceRead              = NULL;   /* Not used because we're implementing onDeviceDataLoop. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not used because we're implementing onDeviceDataLoop. */\n    pCallbacks->onDeviceDataLoop          = ma_device_data_loop__pulse;\n    pCallbacks->onDeviceDataLoopWakeup    = ma_device_data_loop_wakeup__pulse;\n\n    return MA_SUCCESS;\n}\n#endif\n\n\n/******************************************************************************\n\nJACK Backend\n\n******************************************************************************/\n#ifdef MA_HAS_JACK\n\n/* It is assumed jack.h is available when compile-time linking is being used. */\n#ifdef MA_NO_RUNTIME_LINKING\n#include <jack/jack.h>\n\ntypedef jack_nframes_t              ma_jack_nframes_t;\ntypedef jack_options_t              ma_jack_options_t;\ntypedef jack_status_t               ma_jack_status_t;\ntypedef jack_client_t               ma_jack_client_t;\ntypedef jack_port_t                 ma_jack_port_t;\ntypedef JackProcessCallback         ma_JackProcessCallback;\ntypedef JackBufferSizeCallback      ma_JackBufferSizeCallback;\ntypedef JackShutdownCallback        ma_JackShutdownCallback;\n#define MA_JACK_DEFAULT_AUDIO_TYPE  JACK_DEFAULT_AUDIO_TYPE\n#define ma_JackNullOption           JackNullOption\n#define ma_JackNoStartServer        JackNoStartServer\n#define ma_JackPortIsInput          JackPortIsInput\n#define ma_JackPortIsOutput         JackPortIsOutput\n#define ma_JackPortIsPhysical       JackPortIsPhysical\n#else\ntypedef ma_uint32               ma_jack_nframes_t;\ntypedef int                     ma_jack_options_t;\ntypedef int                     ma_jack_status_t;\ntypedef struct ma_jack_client_t ma_jack_client_t;\ntypedef struct ma_jack_port_t   ma_jack_port_t;\ntypedef int  (* ma_JackProcessCallback)   (ma_jack_nframes_t nframes, void* arg);\ntypedef int  (* ma_JackBufferSizeCallback)(ma_jack_nframes_t nframes, void* arg);\ntypedef void (* ma_JackShutdownCallback)  (void* arg);\n#define MA_JACK_DEFAULT_AUDIO_TYPE \"32 bit float mono audio\"\n#define ma_JackNullOption          0\n#define ma_JackNoStartServer       1\n#define ma_JackPortIsInput         1\n#define ma_JackPortIsOutput        2\n#define ma_JackPortIsPhysical      4\n#endif\n\ntypedef ma_jack_client_t* (* ma_jack_client_open_proc)             (const char* client_name, ma_jack_options_t options, ma_jack_status_t* status, ...);\ntypedef int               (* ma_jack_client_close_proc)            (ma_jack_client_t* client);\ntypedef int               (* ma_jack_client_name_size_proc)        (void);\ntypedef int               (* ma_jack_set_process_callback_proc)    (ma_jack_client_t* client, ma_JackProcessCallback process_callback, void* arg);\ntypedef int               (* ma_jack_set_buffer_size_callback_proc)(ma_jack_client_t* client, ma_JackBufferSizeCallback bufsize_callback, void* arg);\ntypedef void              (* ma_jack_on_shutdown_proc)             (ma_jack_client_t* client, ma_JackShutdownCallback function, void* arg);\ntypedef ma_jack_nframes_t (* ma_jack_get_sample_rate_proc)         (ma_jack_client_t* client);\ntypedef ma_jack_nframes_t (* ma_jack_get_buffer_size_proc)         (ma_jack_client_t* client);\ntypedef const char**      (* ma_jack_get_ports_proc)               (ma_jack_client_t* client, const char* port_name_pattern, const char* type_name_pattern, unsigned long flags);\ntypedef int               (* ma_jack_activate_proc)                (ma_jack_client_t* client);\ntypedef int               (* ma_jack_deactivate_proc)              (ma_jack_client_t* client);\ntypedef int               (* ma_jack_connect_proc)                 (ma_jack_client_t* client, const char* source_port, const char* destination_port);\ntypedef ma_jack_port_t*   (* ma_jack_port_register_proc)           (ma_jack_client_t* client, const char* port_name, const char* port_type, unsigned long flags, unsigned long buffer_size);\ntypedef const char*       (* ma_jack_port_name_proc)               (const ma_jack_port_t* port);\ntypedef void*             (* ma_jack_port_get_buffer_proc)         (ma_jack_port_t* port, ma_jack_nframes_t nframes);\ntypedef void              (* ma_jack_free_proc)                    (void* ptr);\n\nstatic ma_result ma_context_open_client__jack(ma_context* pContext, ma_jack_client_t** ppClient)\n{\n    size_t maxClientNameSize;\n    char clientName[256];\n    ma_jack_status_t status;\n    ma_jack_client_t* pClient;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppClient != NULL);\n\n    if (ppClient) {\n        *ppClient = NULL;\n    }\n\n    maxClientNameSize = ((ma_jack_client_name_size_proc)pContext->jack.jack_client_name_size)(); /* Includes null terminator. */\n    ma_strncpy_s(clientName, ma_min(sizeof(clientName), maxClientNameSize), (pContext->jack.pClientName != NULL) ? pContext->jack.pClientName : \"miniaudio\", (size_t)-1);\n\n    pClient = ((ma_jack_client_open_proc)pContext->jack.jack_client_open)(clientName, (pContext->jack.tryStartServer) ? ma_JackNullOption : ma_JackNoStartServer, &status, NULL);\n    if (pClient == NULL) {\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n\n    if (ppClient) {\n        *ppClient = pClient;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__jack(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 cbResult = MA_TRUE;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Playback. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n        deviceInfo.isDefault = MA_TRUE;    /* JACK only uses default devices. */\n        cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n    }\n\n    /* Capture. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n        deviceInfo.isDefault = MA_TRUE;    /* JACK only uses default devices. */\n        cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n    }\n\n    (void)cbResult; /* For silencing a static analysis warning. */\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__jack(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_jack_client_t* pClient;\n    ma_result result;\n    const char** ppPorts;\n\n    MA_ASSERT(pContext != NULL);\n\n    if (pDeviceID != NULL && pDeviceID->jack != 0) {\n        return MA_NO_DEVICE;   /* Don't know the device. */\n    }\n\n    /* Name / Description */\n    if (deviceType == ma_device_type_playback) {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n    } else {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n    }\n\n    /* Jack only uses default devices. */\n    pDeviceInfo->isDefault = MA_TRUE;\n\n    /* Jack only supports f32 and has a specific channel count and sample rate. */\n    pDeviceInfo->nativeDataFormats[0].format = ma_format_f32;\n\n    /* The channel count and sample rate can only be determined by opening the device. */\n    result = ma_context_open_client__jack(pContext, &pClient);\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to open client.\");\n        return result;\n    }\n\n    pDeviceInfo->nativeDataFormats[0].sampleRate = ((ma_jack_get_sample_rate_proc)pContext->jack.jack_get_sample_rate)((ma_jack_client_t*)pClient);\n    pDeviceInfo->nativeDataFormats[0].channels   = 0;\n\n    ppPorts = ((ma_jack_get_ports_proc)pContext->jack.jack_get_ports)((ma_jack_client_t*)pClient, NULL, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsPhysical | ((deviceType == ma_device_type_playback) ? ma_JackPortIsInput : ma_JackPortIsOutput));\n    if (ppPorts == NULL) {\n        ((ma_jack_client_close_proc)pContext->jack.jack_client_close)((ma_jack_client_t*)pClient);\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to query physical ports.\");\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n\n    while (ppPorts[pDeviceInfo->nativeDataFormats[0].channels] != NULL) {\n        pDeviceInfo->nativeDataFormats[0].channels += 1;\n    }\n\n    pDeviceInfo->nativeDataFormats[0].flags = 0;\n    pDeviceInfo->nativeDataFormatCount = 1;\n\n    ((ma_jack_free_proc)pContext->jack.jack_free)((void*)ppPorts);\n    ((ma_jack_client_close_proc)pContext->jack.jack_client_close)((ma_jack_client_t*)pClient);\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_uninit__jack(ma_device* pDevice)\n{\n    ma_context* pContext;\n\n    MA_ASSERT(pDevice != NULL);\n\n    pContext = pDevice->pContext;\n    MA_ASSERT(pContext != NULL);\n\n    if (pDevice->jack.pClient != NULL) {\n        ((ma_jack_client_close_proc)pContext->jack.jack_client_close)((ma_jack_client_t*)pDevice->jack.pClient);\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_free(pDevice->jack.pIntermediaryBufferCapture, &pDevice->pContext->allocationCallbacks);\n        ma_free(pDevice->jack.ppPortsCapture, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_free(pDevice->jack.pIntermediaryBufferPlayback, &pDevice->pContext->allocationCallbacks);\n        ma_free(pDevice->jack.ppPortsPlayback, &pDevice->pContext->allocationCallbacks);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_device__jack_shutdown_callback(void* pUserData)\n{\n    /* JACK died. Stop the device. */\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    ma_device_stop(pDevice);\n}\n\nstatic int ma_device__jack_buffer_size_callback(ma_jack_nframes_t frameCount, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        size_t newBufferSize = frameCount * (pDevice->capture.internalChannels * ma_get_bytes_per_sample(pDevice->capture.internalFormat));\n        float* pNewBuffer = (float*)ma_calloc(newBufferSize, &pDevice->pContext->allocationCallbacks);\n        if (pNewBuffer == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        ma_free(pDevice->jack.pIntermediaryBufferCapture, &pDevice->pContext->allocationCallbacks);\n\n        pDevice->jack.pIntermediaryBufferCapture = pNewBuffer;\n        pDevice->playback.internalPeriodSizeInFrames = frameCount;\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        size_t newBufferSize = frameCount * (pDevice->playback.internalChannels * ma_get_bytes_per_sample(pDevice->playback.internalFormat));\n        float* pNewBuffer = (float*)ma_calloc(newBufferSize, &pDevice->pContext->allocationCallbacks);\n        if (pNewBuffer == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        ma_free(pDevice->jack.pIntermediaryBufferPlayback, &pDevice->pContext->allocationCallbacks);\n\n        pDevice->jack.pIntermediaryBufferPlayback = pNewBuffer;\n        pDevice->playback.internalPeriodSizeInFrames = frameCount;\n    }\n\n    return 0;\n}\n\nstatic int ma_device__jack_process_callback(ma_jack_nframes_t frameCount, void* pUserData)\n{\n    ma_device* pDevice;\n    ma_context* pContext;\n    ma_uint32 iChannel;\n\n    pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    pContext = pDevice->pContext;\n    MA_ASSERT(pContext != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        /* Channels need to be interleaved. */\n        for (iChannel = 0; iChannel < pDevice->capture.internalChannels; ++iChannel) {\n            const float* pSrc = (const float*)((ma_jack_port_get_buffer_proc)pContext->jack.jack_port_get_buffer)((ma_jack_port_t*)pDevice->jack.ppPortsCapture[iChannel], frameCount);\n            if (pSrc != NULL) {\n                float* pDst = pDevice->jack.pIntermediaryBufferCapture + iChannel;\n                ma_jack_nframes_t iFrame;\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    *pDst = *pSrc;\n\n                    pDst += pDevice->capture.internalChannels;\n                    pSrc += 1;\n                }\n            }\n        }\n\n        ma_device_handle_backend_data_callback(pDevice, NULL, pDevice->jack.pIntermediaryBufferCapture, frameCount);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_device_handle_backend_data_callback(pDevice, pDevice->jack.pIntermediaryBufferPlayback, NULL, frameCount);\n\n        /* Channels need to be deinterleaved. */\n        for (iChannel = 0; iChannel < pDevice->playback.internalChannels; ++iChannel) {\n            float* pDst = (float*)((ma_jack_port_get_buffer_proc)pContext->jack.jack_port_get_buffer)((ma_jack_port_t*)pDevice->jack.ppPortsPlayback[iChannel], frameCount);\n            if (pDst != NULL) {\n                const float* pSrc = pDevice->jack.pIntermediaryBufferPlayback + iChannel;\n                ma_jack_nframes_t iFrame;\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    *pDst = *pSrc;\n\n                    pDst += 1;\n                    pSrc += pDevice->playback.internalChannels;\n                }\n            }\n        }\n    }\n\n    return 0;\n}\n\nstatic ma_result ma_device_init__jack(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n    ma_uint32 periodSizeInFrames;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDevice != NULL);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Loopback mode not supported.\");\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* Only supporting default devices with JACK. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->pDeviceID != NULL && pDescriptorPlayback->pDeviceID->jack != 0) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->pDeviceID  != NULL && pDescriptorCapture->pDeviceID->jack  != 0)) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Only default devices are supported.\");\n        return MA_NO_DEVICE;\n    }\n\n    /* No exclusive mode with the JACK backend. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive)) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Exclusive mode not supported.\");\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    /* Open the client. */\n    result = ma_context_open_client__jack(pDevice->pContext, (ma_jack_client_t**)&pDevice->jack.pClient);\n    if (result != MA_SUCCESS) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to open client.\");\n        return result;\n    }\n\n    /* Callbacks. */\n    if (((ma_jack_set_process_callback_proc)pDevice->pContext->jack.jack_set_process_callback)((ma_jack_client_t*)pDevice->jack.pClient, ma_device__jack_process_callback, pDevice) != 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to set process callback.\");\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n    if (((ma_jack_set_buffer_size_callback_proc)pDevice->pContext->jack.jack_set_buffer_size_callback)((ma_jack_client_t*)pDevice->jack.pClient, ma_device__jack_buffer_size_callback, pDevice) != 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to set buffer size callback.\");\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n\n    ((ma_jack_on_shutdown_proc)pDevice->pContext->jack.jack_on_shutdown)((ma_jack_client_t*)pDevice->jack.pClient, ma_device__jack_shutdown_callback, pDevice);\n\n\n    /* The buffer size in frames can change. */\n    periodSizeInFrames = ((ma_jack_get_buffer_size_proc)pDevice->pContext->jack.jack_get_buffer_size)((ma_jack_client_t*)pDevice->jack.pClient);\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_uint32 iPort;\n        const char** ppPorts;\n\n        pDescriptorCapture->format     = ma_format_f32;\n        pDescriptorCapture->channels   = 0;\n        pDescriptorCapture->sampleRate = ((ma_jack_get_sample_rate_proc)pDevice->pContext->jack.jack_get_sample_rate)((ma_jack_client_t*)pDevice->jack.pClient);\n        ma_channel_map_init_standard(ma_standard_channel_map_alsa, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorCapture->channels);\n\n        ppPorts = ((ma_jack_get_ports_proc)pDevice->pContext->jack.jack_get_ports)((ma_jack_client_t*)pDevice->jack.pClient, NULL, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsPhysical | ma_JackPortIsOutput);\n        if (ppPorts == NULL) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to query physical ports.\");\n            return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n        }\n\n        /* Need to count the number of ports first so we can allocate some memory. */\n        while (ppPorts[pDescriptorCapture->channels] != NULL) {\n            pDescriptorCapture->channels += 1;\n        }\n\n        pDevice->jack.ppPortsCapture = (ma_ptr*)ma_malloc(sizeof(*pDevice->jack.ppPortsCapture) * pDescriptorCapture->channels, &pDevice->pContext->allocationCallbacks);\n        if (pDevice->jack.ppPortsCapture == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        for (iPort = 0; iPort < pDescriptorCapture->channels; iPort += 1) {\n            char name[64];\n            ma_strcpy_s(name, sizeof(name), \"capture\");\n            ma_itoa_s((int)iPort, name+7, sizeof(name)-7, 10); /* 7 = length of \"capture\" */\n\n            pDevice->jack.ppPortsCapture[iPort] = ((ma_jack_port_register_proc)pDevice->pContext->jack.jack_port_register)((ma_jack_client_t*)pDevice->jack.pClient, name, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsInput, 0);\n            if (pDevice->jack.ppPortsCapture[iPort] == NULL) {\n                ((ma_jack_free_proc)pDevice->pContext->jack.jack_free)((void*)ppPorts);\n                ma_device_uninit__jack(pDevice);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to register ports.\");\n                return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n            }\n        }\n\n        ((ma_jack_free_proc)pDevice->pContext->jack.jack_free)((void*)ppPorts);\n\n        pDescriptorCapture->periodSizeInFrames = periodSizeInFrames;\n        pDescriptorCapture->periodCount        = 1; /* There's no notion of a period in JACK. Just set to 1. */\n\n        pDevice->jack.pIntermediaryBufferCapture = (float*)ma_calloc(pDescriptorCapture->periodSizeInFrames * ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels), &pDevice->pContext->allocationCallbacks);\n        if (pDevice->jack.pIntermediaryBufferCapture == NULL) {\n            ma_device_uninit__jack(pDevice);\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_uint32 iPort;\n        const char** ppPorts;\n\n        pDescriptorPlayback->format     = ma_format_f32;\n        pDescriptorPlayback->channels   = 0;\n        pDescriptorPlayback->sampleRate = ((ma_jack_get_sample_rate_proc)pDevice->pContext->jack.jack_get_sample_rate)((ma_jack_client_t*)pDevice->jack.pClient);\n        ma_channel_map_init_standard(ma_standard_channel_map_alsa, pDescriptorPlayback->channelMap, ma_countof(pDescriptorPlayback->channelMap), pDescriptorPlayback->channels);\n\n        ppPorts = ((ma_jack_get_ports_proc)pDevice->pContext->jack.jack_get_ports)((ma_jack_client_t*)pDevice->jack.pClient, NULL, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsPhysical | ma_JackPortIsInput);\n        if (ppPorts == NULL) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to query physical ports.\");\n            return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n        }\n\n        /* Need to count the number of ports first so we can allocate some memory. */\n        while (ppPorts[pDescriptorPlayback->channels] != NULL) {\n            pDescriptorPlayback->channels += 1;\n        }\n\n        pDevice->jack.ppPortsPlayback = (ma_ptr*)ma_malloc(sizeof(*pDevice->jack.ppPortsPlayback) * pDescriptorPlayback->channels, &pDevice->pContext->allocationCallbacks);\n        if (pDevice->jack.ppPortsPlayback == NULL) {\n            ma_free(pDevice->jack.ppPortsCapture, &pDevice->pContext->allocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        for (iPort = 0; iPort < pDescriptorPlayback->channels; iPort += 1) {\n            char name[64];\n            ma_strcpy_s(name, sizeof(name), \"playback\");\n            ma_itoa_s((int)iPort, name+8, sizeof(name)-8, 10); /* 8 = length of \"playback\" */\n\n            pDevice->jack.ppPortsPlayback[iPort] = ((ma_jack_port_register_proc)pDevice->pContext->jack.jack_port_register)((ma_jack_client_t*)pDevice->jack.pClient, name, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsOutput, 0);\n            if (pDevice->jack.ppPortsPlayback[iPort] == NULL) {\n                ((ma_jack_free_proc)pDevice->pContext->jack.jack_free)((void*)ppPorts);\n                ma_device_uninit__jack(pDevice);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to register ports.\");\n                return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n            }\n        }\n\n        ((ma_jack_free_proc)pDevice->pContext->jack.jack_free)((void*)ppPorts);\n\n        pDescriptorPlayback->periodSizeInFrames = periodSizeInFrames;\n        pDescriptorPlayback->periodCount        = 1;   /* There's no notion of a period in JACK. Just set to 1. */\n\n        pDevice->jack.pIntermediaryBufferPlayback = (float*)ma_calloc(pDescriptorPlayback->periodSizeInFrames * ma_get_bytes_per_frame(pDescriptorPlayback->format, pDescriptorPlayback->channels), &pDevice->pContext->allocationCallbacks);\n        if (pDevice->jack.pIntermediaryBufferPlayback == NULL) {\n            ma_device_uninit__jack(pDevice);\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_start__jack(ma_device* pDevice)\n{\n    ma_context* pContext = pDevice->pContext;\n    int resultJACK;\n    size_t i;\n\n    resultJACK = ((ma_jack_activate_proc)pContext->jack.jack_activate)((ma_jack_client_t*)pDevice->jack.pClient);\n    if (resultJACK != 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to activate the JACK client.\");\n        return MA_FAILED_TO_START_BACKEND_DEVICE;\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        const char** ppServerPorts = ((ma_jack_get_ports_proc)pContext->jack.jack_get_ports)((ma_jack_client_t*)pDevice->jack.pClient, NULL, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsPhysical | ma_JackPortIsOutput);\n        if (ppServerPorts == NULL) {\n            ((ma_jack_deactivate_proc)pContext->jack.jack_deactivate)((ma_jack_client_t*)pDevice->jack.pClient);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to retrieve physical ports.\");\n            return MA_ERROR;\n        }\n\n        for (i = 0; ppServerPorts[i] != NULL; ++i) {\n            const char* pServerPort = ppServerPorts[i];\n            const char* pClientPort = ((ma_jack_port_name_proc)pContext->jack.jack_port_name)((ma_jack_port_t*)pDevice->jack.ppPortsCapture[i]);\n\n            resultJACK = ((ma_jack_connect_proc)pContext->jack.jack_connect)((ma_jack_client_t*)pDevice->jack.pClient, pServerPort, pClientPort);\n            if (resultJACK != 0) {\n                ((ma_jack_free_proc)pContext->jack.jack_free)((void*)ppServerPorts);\n                ((ma_jack_deactivate_proc)pContext->jack.jack_deactivate)((ma_jack_client_t*)pDevice->jack.pClient);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to connect ports.\");\n                return MA_ERROR;\n            }\n        }\n\n        ((ma_jack_free_proc)pContext->jack.jack_free)((void*)ppServerPorts);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        const char** ppServerPorts = ((ma_jack_get_ports_proc)pContext->jack.jack_get_ports)((ma_jack_client_t*)pDevice->jack.pClient, NULL, MA_JACK_DEFAULT_AUDIO_TYPE, ma_JackPortIsPhysical | ma_JackPortIsInput);\n        if (ppServerPorts == NULL) {\n            ((ma_jack_deactivate_proc)pContext->jack.jack_deactivate)((ma_jack_client_t*)pDevice->jack.pClient);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to retrieve physical ports.\");\n            return MA_ERROR;\n        }\n\n        for (i = 0; ppServerPorts[i] != NULL; ++i) {\n            const char* pServerPort = ppServerPorts[i];\n            const char* pClientPort = ((ma_jack_port_name_proc)pContext->jack.jack_port_name)((ma_jack_port_t*)pDevice->jack.ppPortsPlayback[i]);\n\n            resultJACK = ((ma_jack_connect_proc)pContext->jack.jack_connect)((ma_jack_client_t*)pDevice->jack.pClient, pClientPort, pServerPort);\n            if (resultJACK != 0) {\n                ((ma_jack_free_proc)pContext->jack.jack_free)((void*)ppServerPorts);\n                ((ma_jack_deactivate_proc)pContext->jack.jack_deactivate)((ma_jack_client_t*)pDevice->jack.pClient);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] Failed to connect ports.\");\n                return MA_ERROR;\n            }\n        }\n\n        ((ma_jack_free_proc)pContext->jack.jack_free)((void*)ppServerPorts);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__jack(ma_device* pDevice)\n{\n    ma_context* pContext = pDevice->pContext;\n\n    if (((ma_jack_deactivate_proc)pContext->jack.jack_deactivate)((ma_jack_client_t*)pDevice->jack.pClient) != 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[JACK] An error occurred when deactivating the JACK client.\");\n        return MA_ERROR;\n    }\n\n    ma_device__on_notification_stopped(pDevice);\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_uninit__jack(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_jack);\n\n    ma_free(pContext->jack.pClientName, &pContext->allocationCallbacks);\n    pContext->jack.pClientName = NULL;\n\n#ifndef MA_NO_RUNTIME_LINKING\n    ma_dlclose(ma_context_get_log(pContext), pContext->jack.jackSO);\n#endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__jack(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n#ifndef MA_NO_RUNTIME_LINKING\n    const char* libjackNames[] = {\n#if defined(MA_WIN32)\n        \"libjack.dll\",\n        \"libjack64.dll\"\n#endif\n#if defined(MA_UNIX)\n        \"libjack.so\",\n        \"libjack.so.0\"\n#endif\n    };\n    size_t i;\n\n    for (i = 0; i < ma_countof(libjackNames); ++i) {\n        pContext->jack.jackSO = ma_dlopen(ma_context_get_log(pContext), libjackNames[i]);\n        if (pContext->jack.jackSO != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->jack.jackSO == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    pContext->jack.jack_client_open              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_client_open\");\n    pContext->jack.jack_client_close             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_client_close\");\n    pContext->jack.jack_client_name_size         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_client_name_size\");\n    pContext->jack.jack_set_process_callback     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_set_process_callback\");\n    pContext->jack.jack_set_buffer_size_callback = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_set_buffer_size_callback\");\n    pContext->jack.jack_on_shutdown              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_on_shutdown\");\n    pContext->jack.jack_get_sample_rate          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_get_sample_rate\");\n    pContext->jack.jack_get_buffer_size          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_get_buffer_size\");\n    pContext->jack.jack_get_ports                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_get_ports\");\n    pContext->jack.jack_activate                 = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_activate\");\n    pContext->jack.jack_deactivate               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_deactivate\");\n    pContext->jack.jack_connect                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_connect\");\n    pContext->jack.jack_port_register            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_port_register\");\n    pContext->jack.jack_port_name                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_port_name\");\n    pContext->jack.jack_port_get_buffer          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_port_get_buffer\");\n    pContext->jack.jack_free                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->jack.jackSO, \"jack_free\");\n#else\n    /*\n    This strange assignment system is here just to ensure type safety of miniaudio's function pointer\n    types. If anything differs slightly the compiler should throw a warning.\n    */\n    ma_jack_client_open_proc              _jack_client_open              = jack_client_open;\n    ma_jack_client_close_proc             _jack_client_close             = jack_client_close;\n    ma_jack_client_name_size_proc         _jack_client_name_size         = jack_client_name_size;\n    ma_jack_set_process_callback_proc     _jack_set_process_callback     = jack_set_process_callback;\n    ma_jack_set_buffer_size_callback_proc _jack_set_buffer_size_callback = jack_set_buffer_size_callback;\n    ma_jack_on_shutdown_proc              _jack_on_shutdown              = jack_on_shutdown;\n    ma_jack_get_sample_rate_proc          _jack_get_sample_rate          = jack_get_sample_rate;\n    ma_jack_get_buffer_size_proc          _jack_get_buffer_size          = jack_get_buffer_size;\n    ma_jack_get_ports_proc                _jack_get_ports                = jack_get_ports;\n    ma_jack_activate_proc                 _jack_activate                 = jack_activate;\n    ma_jack_deactivate_proc               _jack_deactivate               = jack_deactivate;\n    ma_jack_connect_proc                  _jack_connect                  = jack_connect;\n    ma_jack_port_register_proc            _jack_port_register            = jack_port_register;\n    ma_jack_port_name_proc                _jack_port_name                = jack_port_name;\n    ma_jack_port_get_buffer_proc          _jack_port_get_buffer          = jack_port_get_buffer;\n    ma_jack_free_proc                     _jack_free                     = jack_free;\n\n    pContext->jack.jack_client_open              = (ma_proc)_jack_client_open;\n    pContext->jack.jack_client_close             = (ma_proc)_jack_client_close;\n    pContext->jack.jack_client_name_size         = (ma_proc)_jack_client_name_size;\n    pContext->jack.jack_set_process_callback     = (ma_proc)_jack_set_process_callback;\n    pContext->jack.jack_set_buffer_size_callback = (ma_proc)_jack_set_buffer_size_callback;\n    pContext->jack.jack_on_shutdown              = (ma_proc)_jack_on_shutdown;\n    pContext->jack.jack_get_sample_rate          = (ma_proc)_jack_get_sample_rate;\n    pContext->jack.jack_get_buffer_size          = (ma_proc)_jack_get_buffer_size;\n    pContext->jack.jack_get_ports                = (ma_proc)_jack_get_ports;\n    pContext->jack.jack_activate                 = (ma_proc)_jack_activate;\n    pContext->jack.jack_deactivate               = (ma_proc)_jack_deactivate;\n    pContext->jack.jack_connect                  = (ma_proc)_jack_connect;\n    pContext->jack.jack_port_register            = (ma_proc)_jack_port_register;\n    pContext->jack.jack_port_name                = (ma_proc)_jack_port_name;\n    pContext->jack.jack_port_get_buffer          = (ma_proc)_jack_port_get_buffer;\n    pContext->jack.jack_free                     = (ma_proc)_jack_free;\n#endif\n\n    if (pConfig->jack.pClientName != NULL) {\n        pContext->jack.pClientName = ma_copy_string(pConfig->jack.pClientName, &pContext->allocationCallbacks);\n    }\n    pContext->jack.tryStartServer = pConfig->jack.tryStartServer;\n\n    /*\n    Getting here means the JACK library is installed, but it doesn't necessarily mean it's usable. We need to quickly test this by connecting\n    a temporary client.\n    */\n    {\n        ma_jack_client_t* pDummyClient;\n        ma_result result = ma_context_open_client__jack(pContext, &pDummyClient);\n        if (result != MA_SUCCESS) {\n            ma_free(pContext->jack.pClientName, &pContext->allocationCallbacks);\n        #ifndef MA_NO_RUNTIME_LINKING\n            ma_dlclose(ma_context_get_log(pContext), pContext->jack.jackSO);\n        #endif\n            return MA_NO_BACKEND;\n        }\n\n        ((ma_jack_client_close_proc)pContext->jack.jack_client_close)((ma_jack_client_t*)pDummyClient);\n    }\n\n\n    pCallbacks->onContextInit             = ma_context_init__jack;\n    pCallbacks->onContextUninit           = ma_context_uninit__jack;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__jack;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__jack;\n    pCallbacks->onDeviceInit              = ma_device_init__jack;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__jack;\n    pCallbacks->onDeviceStart             = ma_device_start__jack;\n    pCallbacks->onDeviceStop              = ma_device_stop__jack;\n    pCallbacks->onDeviceRead              = NULL;   /* Not used because JACK is asynchronous. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not used because JACK is asynchronous. */\n    pCallbacks->onDeviceDataLoop          = NULL;   /* Not used because JACK is asynchronous. */\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_JACK */\n\n\n\n/******************************************************************************\n\nCore Audio Backend\n\nReferences\n==========\n- Technical Note TN2091: Device input using the HAL Output Audio Unit\n    https://developer.apple.com/library/archive/technotes/tn2091/_index.html\n\n******************************************************************************/\n#ifdef MA_HAS_COREAUDIO\n#include <TargetConditionals.h>\n\n#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1\n    #define MA_APPLE_MOBILE\n    #if defined(TARGET_OS_TV) && TARGET_OS_TV == 1\n        #define MA_APPLE_TV\n    #endif\n    #if defined(TARGET_OS_WATCH) && TARGET_OS_WATCH == 1\n        #define MA_APPLE_WATCH\n    #endif\n    #if __has_feature(objc_arc)\n        #define MA_BRIDGE_TRANSFER  __bridge_transfer\n        #define MA_BRIDGE_RETAINED  __bridge_retained\n    #else\n        #define MA_BRIDGE_TRANSFER\n        #define MA_BRIDGE_RETAINED\n    #endif\n#else\n    #define MA_APPLE_DESKTOP\n#endif\n\n#if defined(MA_APPLE_DESKTOP)\n#include <CoreAudio/CoreAudio.h>\n#else\n#include <AVFoundation/AVFoundation.h>\n#endif\n\n#include <AudioToolbox/AudioToolbox.h>\n\n/* CoreFoundation */\ntypedef Boolean (* ma_CFStringGetCString_proc)(CFStringRef theString, char* buffer, CFIndex bufferSize, CFStringEncoding encoding);\ntypedef void (* ma_CFRelease_proc)(CFTypeRef cf);\n\n/* CoreAudio */\n#if defined(MA_APPLE_DESKTOP)\ntypedef OSStatus (* ma_AudioObjectGetPropertyData_proc)(AudioObjectID inObjectID, const AudioObjectPropertyAddress* inAddress, UInt32 inQualifierDataSize, const void* inQualifierData, UInt32* ioDataSize, void* outData);\ntypedef OSStatus (* ma_AudioObjectGetPropertyDataSize_proc)(AudioObjectID inObjectID, const AudioObjectPropertyAddress* inAddress, UInt32 inQualifierDataSize, const void* inQualifierData, UInt32* outDataSize);\ntypedef OSStatus (* ma_AudioObjectSetPropertyData_proc)(AudioObjectID inObjectID, const AudioObjectPropertyAddress* inAddress, UInt32 inQualifierDataSize, const void* inQualifierData, UInt32 inDataSize, const void* inData);\ntypedef OSStatus (* ma_AudioObjectAddPropertyListener_proc)(AudioObjectID inObjectID, const AudioObjectPropertyAddress* inAddress, AudioObjectPropertyListenerProc inListener, void* inClientData);\ntypedef OSStatus (* ma_AudioObjectRemovePropertyListener_proc)(AudioObjectID inObjectID, const AudioObjectPropertyAddress* inAddress, AudioObjectPropertyListenerProc inListener, void* inClientData);\n#endif\n\n/* AudioToolbox */\ntypedef AudioComponent (* ma_AudioComponentFindNext_proc)(AudioComponent inComponent, const AudioComponentDescription* inDesc);\ntypedef OSStatus (* ma_AudioComponentInstanceDispose_proc)(AudioComponentInstance inInstance);\ntypedef OSStatus (* ma_AudioComponentInstanceNew_proc)(AudioComponent inComponent, AudioComponentInstance* outInstance);\ntypedef OSStatus (* ma_AudioOutputUnitStart_proc)(AudioUnit inUnit);\ntypedef OSStatus (* ma_AudioOutputUnitStop_proc)(AudioUnit inUnit);\ntypedef OSStatus (* ma_AudioUnitAddPropertyListener_proc)(AudioUnit inUnit, AudioUnitPropertyID inID, AudioUnitPropertyListenerProc inProc, void* inProcUserData);\ntypedef OSStatus (* ma_AudioUnitGetPropertyInfo_proc)(AudioUnit inUnit, AudioUnitPropertyID inID, AudioUnitScope inScope, AudioUnitElement inElement, UInt32* outDataSize, Boolean* outWriteable);\ntypedef OSStatus (* ma_AudioUnitGetProperty_proc)(AudioUnit inUnit, AudioUnitPropertyID inID, AudioUnitScope inScope, AudioUnitElement inElement, void* outData, UInt32* ioDataSize);\ntypedef OSStatus (* ma_AudioUnitSetProperty_proc)(AudioUnit inUnit, AudioUnitPropertyID inID, AudioUnitScope inScope, AudioUnitElement inElement, const void* inData, UInt32 inDataSize);\ntypedef OSStatus (* ma_AudioUnitInitialize_proc)(AudioUnit inUnit);\ntypedef OSStatus (* ma_AudioUnitRender_proc)(AudioUnit inUnit, AudioUnitRenderActionFlags* ioActionFlags, const AudioTimeStamp* inTimeStamp, UInt32 inOutputBusNumber, UInt32 inNumberFrames, AudioBufferList* ioData);\n\n\n#define MA_COREAUDIO_OUTPUT_BUS    0\n#define MA_COREAUDIO_INPUT_BUS     1\n\n#if defined(MA_APPLE_DESKTOP)\nstatic ma_result ma_device_reinit_internal__coreaudio(ma_device* pDevice, ma_device_type deviceType, ma_bool32 disposePreviousAudioUnit);\n#endif\n\n/*\nCore Audio\n\nSo far, Core Audio has been the worst backend to work with due to being both unintuitive and having almost no documentation\napart from comments in the headers (which admittedly are quite good). For my own purposes, and for anybody out there whose\nneeding to figure out how this darn thing works, I'm going to outline a few things here.\n\nSince miniaudio is a fairly low-level API, one of the things it needs is control over specific devices, and it needs to be\nable to identify whether or not it can be used as playback and/or capture. The AudioObject API is the only one I've seen\nthat supports this level of detail. There was some public domain sample code I stumbled across that used the AudioComponent\nand AudioUnit APIs, but I couldn't see anything that gave low-level control over device selection and capabilities (the\ndistinction between playback and capture in particular). Therefore, miniaudio is using the AudioObject API.\n\nMost (all?) functions in the AudioObject API take a AudioObjectID as its input. This is the device identifier. When\nretrieving global information, such as the device list, you use kAudioObjectSystemObject. When retrieving device-specific\ndata, you pass in the ID for that device. In order to retrieve device-specific IDs you need to enumerate over each of the\ndevices. This is done using the AudioObjectGetPropertyDataSize() and AudioObjectGetPropertyData() APIs which seem to be\nthe central APIs for retrieving information about the system and specific devices.\n\nTo use the AudioObjectGetPropertyData() API you need to use the notion of a property address. A property address is a\nstructure with three variables and is used to identify which property you are getting or setting. The first is the \"selector\"\nwhich is basically the specific property that you're wanting to retrieve or set. The second is the \"scope\", which is\ntypically set to kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyScopeInput for input-specific properties and\nkAudioObjectPropertyScopeOutput for output-specific properties. The last is the \"element\" which is always set to\nkAudioObjectPropertyElementMain in miniaudio's case. I don't know of any cases where this would be set to anything different.\n\nBack to the earlier issue of device retrieval, you first use the AudioObjectGetPropertyDataSize() API to retrieve the size\nof the raw data which is just a list of AudioDeviceID's. You use the kAudioObjectSystemObject AudioObjectID, and a property\naddress with the kAudioHardwarePropertyDevices selector and the kAudioObjectPropertyScopeGlobal scope. Once you have the\nsize, allocate a block of memory of that size and then call AudioObjectGetPropertyData(). The data is just a list of\nAudioDeviceID's so just do \"dataSize/sizeof(AudioDeviceID)\" to know the device count.\n*/\n\n#if defined(MA_APPLE_MOBILE)\nstatic void ma_device__on_notification_interruption_began(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_interruption_began));\n}\n\nstatic void ma_device__on_notification_interruption_ended(ma_device* pDevice)\n{\n    ma_device__on_notification(ma_device_notification_init(pDevice, ma_device_notification_type_interruption_ended));\n}\n#endif\n\nstatic ma_result ma_result_from_OSStatus(OSStatus status)\n{\n    switch (status)\n    {\n        case noErr:                                   return MA_SUCCESS;\n    #if defined(MA_APPLE_DESKTOP)\n        case kAudioHardwareNotRunningError:           return MA_DEVICE_NOT_STARTED;\n        case kAudioHardwareUnspecifiedError:          return MA_ERROR;\n        case kAudioHardwareUnknownPropertyError:      return MA_INVALID_ARGS;\n        case kAudioHardwareBadPropertySizeError:      return MA_INVALID_OPERATION;\n        case kAudioHardwareIllegalOperationError:     return MA_INVALID_OPERATION;\n        case kAudioHardwareBadObjectError:            return MA_INVALID_ARGS;\n        case kAudioHardwareBadDeviceError:            return MA_INVALID_ARGS;\n        case kAudioHardwareBadStreamError:            return MA_INVALID_ARGS;\n        case kAudioHardwareUnsupportedOperationError: return MA_INVALID_OPERATION;\n        case kAudioDeviceUnsupportedFormatError:      return MA_FORMAT_NOT_SUPPORTED;\n        case kAudioDevicePermissionsError:            return MA_ACCESS_DENIED;\n    #endif\n        default:                                      return MA_ERROR;\n    }\n}\n\n#if 0\nstatic ma_channel ma_channel_from_AudioChannelBitmap(AudioChannelBitmap bit)\n{\n    switch (bit)\n    {\n        case kAudioChannelBit_Left:                 return MA_CHANNEL_LEFT;\n        case kAudioChannelBit_Right:                return MA_CHANNEL_RIGHT;\n        case kAudioChannelBit_Center:               return MA_CHANNEL_FRONT_CENTER;\n        case kAudioChannelBit_LFEScreen:            return MA_CHANNEL_LFE;\n        case kAudioChannelBit_LeftSurround:         return MA_CHANNEL_BACK_LEFT;\n        case kAudioChannelBit_RightSurround:        return MA_CHANNEL_BACK_RIGHT;\n        case kAudioChannelBit_LeftCenter:           return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case kAudioChannelBit_RightCenter:          return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case kAudioChannelBit_CenterSurround:       return MA_CHANNEL_BACK_CENTER;\n        case kAudioChannelBit_LeftSurroundDirect:   return MA_CHANNEL_SIDE_LEFT;\n        case kAudioChannelBit_RightSurroundDirect:  return MA_CHANNEL_SIDE_RIGHT;\n        case kAudioChannelBit_TopCenterSurround:    return MA_CHANNEL_TOP_CENTER;\n        case kAudioChannelBit_VerticalHeightLeft:   return MA_CHANNEL_TOP_FRONT_LEFT;\n        case kAudioChannelBit_VerticalHeightCenter: return MA_CHANNEL_TOP_FRONT_CENTER;\n        case kAudioChannelBit_VerticalHeightRight:  return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case kAudioChannelBit_TopBackLeft:          return MA_CHANNEL_TOP_BACK_LEFT;\n        case kAudioChannelBit_TopBackCenter:        return MA_CHANNEL_TOP_BACK_CENTER;\n        case kAudioChannelBit_TopBackRight:         return MA_CHANNEL_TOP_BACK_RIGHT;\n        default:                                    return MA_CHANNEL_NONE;\n    }\n}\n#endif\n\nstatic ma_result ma_format_from_AudioStreamBasicDescription(const AudioStreamBasicDescription* pDescription, ma_format* pFormatOut)\n{\n    MA_ASSERT(pDescription != NULL);\n    MA_ASSERT(pFormatOut != NULL);\n\n    *pFormatOut = ma_format_unknown;   /* Safety. */\n\n    /* There's a few things miniaudio doesn't support. */\n    if (pDescription->mFormatID != kAudioFormatLinearPCM) {\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    /* We don't support any non-packed formats that are aligned high. */\n    if ((pDescription->mFormatFlags & kLinearPCMFormatFlagIsAlignedHigh) != 0) {\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    /* Only supporting native-endian. */\n    if ((ma_is_little_endian() && (pDescription->mFormatFlags & kAudioFormatFlagIsBigEndian) != 0) || (ma_is_big_endian() && (pDescription->mFormatFlags & kAudioFormatFlagIsBigEndian) == 0)) {\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    /* We are not currently supporting non-interleaved formats (this will be added in a future version of miniaudio). */\n    /*if ((pDescription->mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0) {\n        return MA_FORMAT_NOT_SUPPORTED;\n    }*/\n\n    if ((pDescription->mFormatFlags & kLinearPCMFormatFlagIsFloat) != 0) {\n        if (pDescription->mBitsPerChannel == 32) {\n            *pFormatOut = ma_format_f32;\n            return MA_SUCCESS;\n        }\n    } else {\n        if ((pDescription->mFormatFlags & kLinearPCMFormatFlagIsSignedInteger) != 0) {\n            if (pDescription->mBitsPerChannel == 16) {\n                *pFormatOut = ma_format_s16;\n                return MA_SUCCESS;\n            } else if (pDescription->mBitsPerChannel == 24) {\n                if (pDescription->mBytesPerFrame == (pDescription->mBitsPerChannel/8 * pDescription->mChannelsPerFrame)) {\n                    *pFormatOut = ma_format_s24;\n                    return MA_SUCCESS;\n                } else {\n                    if (pDescription->mBytesPerFrame/pDescription->mChannelsPerFrame == sizeof(ma_int32)) {\n                        /* TODO: Implement ma_format_s24_32. */\n                        /**pFormatOut = ma_format_s24_32;*/\n                        /*return MA_SUCCESS;*/\n                        return MA_FORMAT_NOT_SUPPORTED;\n                    }\n                }\n            } else if (pDescription->mBitsPerChannel == 32) {\n                *pFormatOut = ma_format_s32;\n                return MA_SUCCESS;\n            }\n        } else {\n            if (pDescription->mBitsPerChannel == 8) {\n                *pFormatOut = ma_format_u8;\n                return MA_SUCCESS;\n            }\n        }\n    }\n\n    /* Getting here means the format is not supported. */\n    return MA_FORMAT_NOT_SUPPORTED;\n}\n\n#if defined(MA_APPLE_DESKTOP)\nstatic ma_channel ma_channel_from_AudioChannelLabel(AudioChannelLabel label)\n{\n    switch (label)\n    {\n        case kAudioChannelLabel_Unknown:              return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Unused:               return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_UseCoordinates:       return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Left:                 return MA_CHANNEL_LEFT;\n        case kAudioChannelLabel_Right:                return MA_CHANNEL_RIGHT;\n        case kAudioChannelLabel_Center:               return MA_CHANNEL_FRONT_CENTER;\n        case kAudioChannelLabel_LFEScreen:            return MA_CHANNEL_LFE;\n        case kAudioChannelLabel_LeftSurround:         return MA_CHANNEL_BACK_LEFT;\n        case kAudioChannelLabel_RightSurround:        return MA_CHANNEL_BACK_RIGHT;\n        case kAudioChannelLabel_LeftCenter:           return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case kAudioChannelLabel_RightCenter:          return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case kAudioChannelLabel_CenterSurround:       return MA_CHANNEL_BACK_CENTER;\n        case kAudioChannelLabel_LeftSurroundDirect:   return MA_CHANNEL_SIDE_LEFT;\n        case kAudioChannelLabel_RightSurroundDirect:  return MA_CHANNEL_SIDE_RIGHT;\n        case kAudioChannelLabel_TopCenterSurround:    return MA_CHANNEL_TOP_CENTER;\n        case kAudioChannelLabel_VerticalHeightLeft:   return MA_CHANNEL_TOP_FRONT_LEFT;\n        case kAudioChannelLabel_VerticalHeightCenter: return MA_CHANNEL_TOP_FRONT_CENTER;\n        case kAudioChannelLabel_VerticalHeightRight:  return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case kAudioChannelLabel_TopBackLeft:          return MA_CHANNEL_TOP_BACK_LEFT;\n        case kAudioChannelLabel_TopBackCenter:        return MA_CHANNEL_TOP_BACK_CENTER;\n        case kAudioChannelLabel_TopBackRight:         return MA_CHANNEL_TOP_BACK_RIGHT;\n        case kAudioChannelLabel_RearSurroundLeft:     return MA_CHANNEL_BACK_LEFT;\n        case kAudioChannelLabel_RearSurroundRight:    return MA_CHANNEL_BACK_RIGHT;\n        case kAudioChannelLabel_LeftWide:             return MA_CHANNEL_SIDE_LEFT;\n        case kAudioChannelLabel_RightWide:            return MA_CHANNEL_SIDE_RIGHT;\n        case kAudioChannelLabel_LFE2:                 return MA_CHANNEL_LFE;\n        case kAudioChannelLabel_LeftTotal:            return MA_CHANNEL_LEFT;\n        case kAudioChannelLabel_RightTotal:           return MA_CHANNEL_RIGHT;\n        case kAudioChannelLabel_HearingImpaired:      return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Narration:            return MA_CHANNEL_MONO;\n        case kAudioChannelLabel_Mono:                 return MA_CHANNEL_MONO;\n        case kAudioChannelLabel_DialogCentricMix:     return MA_CHANNEL_MONO;\n        case kAudioChannelLabel_CenterSurroundDirect: return MA_CHANNEL_BACK_CENTER;\n        case kAudioChannelLabel_Haptic:               return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Ambisonic_W:          return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Ambisonic_X:          return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Ambisonic_Y:          return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Ambisonic_Z:          return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_MS_Mid:               return MA_CHANNEL_LEFT;\n        case kAudioChannelLabel_MS_Side:              return MA_CHANNEL_RIGHT;\n        case kAudioChannelLabel_XY_X:                 return MA_CHANNEL_LEFT;\n        case kAudioChannelLabel_XY_Y:                 return MA_CHANNEL_RIGHT;\n        case kAudioChannelLabel_HeadphonesLeft:       return MA_CHANNEL_LEFT;\n        case kAudioChannelLabel_HeadphonesRight:      return MA_CHANNEL_RIGHT;\n        case kAudioChannelLabel_ClickTrack:           return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_ForeignLanguage:      return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Discrete:             return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_Discrete_0:           return MA_CHANNEL_AUX_0;\n        case kAudioChannelLabel_Discrete_1:           return MA_CHANNEL_AUX_1;\n        case kAudioChannelLabel_Discrete_2:           return MA_CHANNEL_AUX_2;\n        case kAudioChannelLabel_Discrete_3:           return MA_CHANNEL_AUX_3;\n        case kAudioChannelLabel_Discrete_4:           return MA_CHANNEL_AUX_4;\n        case kAudioChannelLabel_Discrete_5:           return MA_CHANNEL_AUX_5;\n        case kAudioChannelLabel_Discrete_6:           return MA_CHANNEL_AUX_6;\n        case kAudioChannelLabel_Discrete_7:           return MA_CHANNEL_AUX_7;\n        case kAudioChannelLabel_Discrete_8:           return MA_CHANNEL_AUX_8;\n        case kAudioChannelLabel_Discrete_9:           return MA_CHANNEL_AUX_9;\n        case kAudioChannelLabel_Discrete_10:          return MA_CHANNEL_AUX_10;\n        case kAudioChannelLabel_Discrete_11:          return MA_CHANNEL_AUX_11;\n        case kAudioChannelLabel_Discrete_12:          return MA_CHANNEL_AUX_12;\n        case kAudioChannelLabel_Discrete_13:          return MA_CHANNEL_AUX_13;\n        case kAudioChannelLabel_Discrete_14:          return MA_CHANNEL_AUX_14;\n        case kAudioChannelLabel_Discrete_15:          return MA_CHANNEL_AUX_15;\n        case kAudioChannelLabel_Discrete_65535:       return MA_CHANNEL_NONE;\n\n    #if 0   /* Introduced in a later version of macOS. */\n        case kAudioChannelLabel_HOA_ACN:              return MA_CHANNEL_NONE;\n        case kAudioChannelLabel_HOA_ACN_0:            return MA_CHANNEL_AUX_0;\n        case kAudioChannelLabel_HOA_ACN_1:            return MA_CHANNEL_AUX_1;\n        case kAudioChannelLabel_HOA_ACN_2:            return MA_CHANNEL_AUX_2;\n        case kAudioChannelLabel_HOA_ACN_3:            return MA_CHANNEL_AUX_3;\n        case kAudioChannelLabel_HOA_ACN_4:            return MA_CHANNEL_AUX_4;\n        case kAudioChannelLabel_HOA_ACN_5:            return MA_CHANNEL_AUX_5;\n        case kAudioChannelLabel_HOA_ACN_6:            return MA_CHANNEL_AUX_6;\n        case kAudioChannelLabel_HOA_ACN_7:            return MA_CHANNEL_AUX_7;\n        case kAudioChannelLabel_HOA_ACN_8:            return MA_CHANNEL_AUX_8;\n        case kAudioChannelLabel_HOA_ACN_9:            return MA_CHANNEL_AUX_9;\n        case kAudioChannelLabel_HOA_ACN_10:           return MA_CHANNEL_AUX_10;\n        case kAudioChannelLabel_HOA_ACN_11:           return MA_CHANNEL_AUX_11;\n        case kAudioChannelLabel_HOA_ACN_12:           return MA_CHANNEL_AUX_12;\n        case kAudioChannelLabel_HOA_ACN_13:           return MA_CHANNEL_AUX_13;\n        case kAudioChannelLabel_HOA_ACN_14:           return MA_CHANNEL_AUX_14;\n        case kAudioChannelLabel_HOA_ACN_15:           return MA_CHANNEL_AUX_15;\n        case kAudioChannelLabel_HOA_ACN_65024:        return MA_CHANNEL_NONE;\n    #endif\n\n        default:                                      return MA_CHANNEL_NONE;\n    }\n}\n\nstatic ma_result ma_get_channel_map_from_AudioChannelLayout(AudioChannelLayout* pChannelLayout, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    MA_ASSERT(pChannelLayout != NULL);\n\n    if (pChannelLayout->mChannelLayoutTag == kAudioChannelLayoutTag_UseChannelDescriptions) {\n        UInt32 iChannel;\n        for (iChannel = 0; iChannel < pChannelLayout->mNumberChannelDescriptions && iChannel < channelMapCap; ++iChannel) {\n            pChannelMap[iChannel] = ma_channel_from_AudioChannelLabel(pChannelLayout->mChannelDescriptions[iChannel].mChannelLabel);\n        }\n    } else\n#if 0\n    if (pChannelLayout->mChannelLayoutTag == kAudioChannelLayoutTag_UseChannelBitmap) {\n        /* This is the same kind of system that's used by Windows audio APIs. */\n        UInt32 iChannel = 0;\n        UInt32 iBit;\n        AudioChannelBitmap bitmap = pChannelLayout->mChannelBitmap;\n        for (iBit = 0; iBit < 32 && iChannel < channelMapCap; ++iBit) {\n            AudioChannelBitmap bit = bitmap & (1 << iBit);\n            if (bit != 0) {\n                pChannelMap[iChannel++] = ma_channel_from_AudioChannelBit(bit);\n            }\n        }\n    } else\n#endif\n    {\n        /*\n        Need to use the tag to determine the channel map. For now I'm just assuming a default channel map, but later on this should\n        be updated to determine the mapping based on the tag.\n        */\n        UInt32 channelCount;\n\n        /* Our channel map retrieval APIs below take 32-bit integers, so we'll want to clamp the channel map capacity. */\n        if (channelMapCap > 0xFFFFFFFF) {\n            channelMapCap = 0xFFFFFFFF;\n        }\n\n        channelCount = ma_min(AudioChannelLayoutTag_GetNumberOfChannels(pChannelLayout->mChannelLayoutTag), (UInt32)channelMapCap);\n\n        switch (pChannelLayout->mChannelLayoutTag)\n        {\n            case kAudioChannelLayoutTag_Mono:\n            case kAudioChannelLayoutTag_Stereo:\n            case kAudioChannelLayoutTag_StereoHeadphones:\n            case kAudioChannelLayoutTag_MatrixStereo:\n            case kAudioChannelLayoutTag_MidSide:\n            case kAudioChannelLayoutTag_XY:\n            case kAudioChannelLayoutTag_Binaural:\n            case kAudioChannelLayoutTag_Ambisonic_B_Format:\n            {\n                ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, channelCount);\n            } break;\n\n            case kAudioChannelLayoutTag_Octagonal:\n            {\n                pChannelMap[7] = MA_CHANNEL_SIDE_RIGHT;\n                pChannelMap[6] = MA_CHANNEL_SIDE_LEFT;\n            } MA_FALLTHROUGH; /* Intentional fallthrough. */\n            case kAudioChannelLayoutTag_Hexagonal:\n            {\n                pChannelMap[5] = MA_CHANNEL_BACK_CENTER;\n            } MA_FALLTHROUGH; /* Intentional fallthrough. */\n            case kAudioChannelLayoutTag_Pentagonal:\n            {\n                pChannelMap[4] = MA_CHANNEL_FRONT_CENTER;\n            } MA_FALLTHROUGH; /* Intentional fallthrough. */\n            case kAudioChannelLayoutTag_Quadraphonic:\n            {\n                pChannelMap[3] = MA_CHANNEL_BACK_RIGHT;\n                pChannelMap[2] = MA_CHANNEL_BACK_LEFT;\n                pChannelMap[1] = MA_CHANNEL_RIGHT;\n                pChannelMap[0] = MA_CHANNEL_LEFT;\n            } break;\n\n            /* TODO: Add support for more tags here. */\n\n            default:\n            {\n                ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, channelCount);\n            } break;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n#if (defined(MAC_OS_VERSION_12_0) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_12_0) || \\\n    (defined(__IPHONE_15_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_15_0)\n#define AUDIO_OBJECT_PROPERTY_ELEMENT kAudioObjectPropertyElementMain\n#else\n/* kAudioObjectPropertyElementMaster is deprecated. */\n#define AUDIO_OBJECT_PROPERTY_ELEMENT kAudioObjectPropertyElementMaster\n#endif\n\n/* kAudioDevicePropertyScope* were renamed to kAudioObjectPropertyScope* in 10.8. */\n#if !defined(MAC_OS_X_VERSION_10_8) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_8)\n#define kAudioObjectPropertyScopeInput kAudioDevicePropertyScopeInput\n#define kAudioObjectPropertyScopeOutput kAudioDevicePropertyScopeOutput\n#endif\n\nstatic ma_result ma_get_device_object_ids__coreaudio(ma_context* pContext, UInt32* pDeviceCount, AudioObjectID** ppDeviceObjectIDs) /* NOTE: Free the returned buffer with ma_free(). */\n{\n    AudioObjectPropertyAddress propAddressDevices;\n    UInt32 deviceObjectsDataSize;\n    OSStatus status;\n    AudioObjectID* pDeviceObjectIDs;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pDeviceCount != NULL);\n    MA_ASSERT(ppDeviceObjectIDs != NULL);\n\n    /* Safety. */\n    *pDeviceCount = 0;\n    *ppDeviceObjectIDs = NULL;\n\n    propAddressDevices.mSelector = kAudioHardwarePropertyDevices;\n    propAddressDevices.mScope    = kAudioObjectPropertyScopeGlobal;\n    propAddressDevices.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    status = ((ma_AudioObjectGetPropertyDataSize_proc)pContext->coreaudio.AudioObjectGetPropertyDataSize)(kAudioObjectSystemObject, &propAddressDevices, 0, NULL, &deviceObjectsDataSize);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    pDeviceObjectIDs = (AudioObjectID*)ma_malloc(deviceObjectsDataSize, &pContext->allocationCallbacks);\n    if (pDeviceObjectIDs == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(kAudioObjectSystemObject, &propAddressDevices, 0, NULL, &deviceObjectsDataSize, pDeviceObjectIDs);\n    if (status != noErr) {\n        ma_free(pDeviceObjectIDs, &pContext->allocationCallbacks);\n        return ma_result_from_OSStatus(status);\n    }\n\n    *pDeviceCount = deviceObjectsDataSize / sizeof(AudioObjectID);\n    *ppDeviceObjectIDs = pDeviceObjectIDs;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_get_AudioObject_uid_as_CFStringRef(ma_context* pContext, AudioObjectID objectID, CFStringRef* pUID)\n{\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n\n    MA_ASSERT(pContext != NULL);\n\n    propAddress.mSelector = kAudioDevicePropertyDeviceUID;\n    propAddress.mScope    = kAudioObjectPropertyScopeGlobal;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    dataSize = sizeof(*pUID);\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(objectID, &propAddress, 0, NULL, &dataSize, pUID);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_get_AudioObject_uid(ma_context* pContext, AudioObjectID objectID, size_t bufferSize, char* bufferOut)\n{\n    CFStringRef uid;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n    result = ma_get_AudioObject_uid_as_CFStringRef(pContext, objectID, &uid);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (!((ma_CFStringGetCString_proc)pContext->coreaudio.CFStringGetCString)(uid, bufferOut, bufferSize, kCFStringEncodingUTF8)) {\n        return MA_ERROR;\n    }\n\n    ((ma_CFRelease_proc)pContext->coreaudio.CFRelease)(uid);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_get_AudioObject_name(ma_context* pContext, AudioObjectID objectID, size_t bufferSize, char* bufferOut)\n{\n    AudioObjectPropertyAddress propAddress;\n    CFStringRef deviceName = NULL;\n    UInt32 dataSize;\n    OSStatus status;\n\n    MA_ASSERT(pContext != NULL);\n\n    propAddress.mSelector = kAudioDevicePropertyDeviceNameCFString;\n    propAddress.mScope    = kAudioObjectPropertyScopeGlobal;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    dataSize = sizeof(deviceName);\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(objectID, &propAddress, 0, NULL, &dataSize, &deviceName);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    if (!((ma_CFStringGetCString_proc)pContext->coreaudio.CFStringGetCString)(deviceName, bufferOut, bufferSize, kCFStringEncodingUTF8)) {\n        return MA_ERROR;\n    }\n\n    ((ma_CFRelease_proc)pContext->coreaudio.CFRelease)(deviceName);\n    return MA_SUCCESS;\n}\n\nstatic ma_bool32 ma_does_AudioObject_support_scope(ma_context* pContext, AudioObjectID deviceObjectID, AudioObjectPropertyScope scope)\n{\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n    AudioBufferList* pBufferList;\n    ma_bool32 isSupported;\n\n    MA_ASSERT(pContext != NULL);\n\n    /* To know whether or not a device is an input device we need ot look at the stream configuration. If it has an output channel it's a playback device. */\n    propAddress.mSelector = kAudioDevicePropertyStreamConfiguration;\n    propAddress.mScope    = scope;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    status = ((ma_AudioObjectGetPropertyDataSize_proc)pContext->coreaudio.AudioObjectGetPropertyDataSize)(deviceObjectID, &propAddress, 0, NULL, &dataSize);\n    if (status != noErr) {\n        return MA_FALSE;\n    }\n\n    pBufferList = (AudioBufferList*)ma_malloc(dataSize, &pContext->allocationCallbacks);\n    if (pBufferList == NULL) {\n        return MA_FALSE;   /* Out of memory. */\n    }\n\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, pBufferList);\n    if (status != noErr) {\n        ma_free(pBufferList, &pContext->allocationCallbacks);\n        return MA_FALSE;\n    }\n\n    isSupported = MA_FALSE;\n    if (pBufferList->mNumberBuffers > 0) {\n        isSupported = MA_TRUE;\n    }\n\n    ma_free(pBufferList, &pContext->allocationCallbacks);\n    return isSupported;\n}\n\nstatic ma_bool32 ma_does_AudioObject_support_playback(ma_context* pContext, AudioObjectID deviceObjectID)\n{\n    return ma_does_AudioObject_support_scope(pContext, deviceObjectID, kAudioObjectPropertyScopeOutput);\n}\n\nstatic ma_bool32 ma_does_AudioObject_support_capture(ma_context* pContext, AudioObjectID deviceObjectID)\n{\n    return ma_does_AudioObject_support_scope(pContext, deviceObjectID, kAudioObjectPropertyScopeInput);\n}\n\n\nstatic ma_result ma_get_AudioObject_stream_descriptions(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, UInt32* pDescriptionCount, AudioStreamRangedDescription** ppDescriptions) /* NOTE: Free the returned pointer with ma_free(). */\n{\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n    AudioStreamRangedDescription* pDescriptions;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pDescriptionCount != NULL);\n    MA_ASSERT(ppDescriptions != NULL);\n\n    /*\n    TODO: Experiment with kAudioStreamPropertyAvailablePhysicalFormats instead of (or in addition to) kAudioStreamPropertyAvailableVirtualFormats. My\n          MacBook Pro uses s24/32 format, however, which miniaudio does not currently support.\n    */\n    propAddress.mSelector = kAudioStreamPropertyAvailableVirtualFormats; /*kAudioStreamPropertyAvailablePhysicalFormats;*/\n    propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    status = ((ma_AudioObjectGetPropertyDataSize_proc)pContext->coreaudio.AudioObjectGetPropertyDataSize)(deviceObjectID, &propAddress, 0, NULL, &dataSize);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    pDescriptions = (AudioStreamRangedDescription*)ma_malloc(dataSize, &pContext->allocationCallbacks);\n    if (pDescriptions == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, pDescriptions);\n    if (status != noErr) {\n        ma_free(pDescriptions, &pContext->allocationCallbacks);\n        return ma_result_from_OSStatus(status);\n    }\n\n    *pDescriptionCount = dataSize / sizeof(*pDescriptions);\n    *ppDescriptions = pDescriptions;\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_get_AudioObject_channel_layout(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, AudioChannelLayout** ppChannelLayout)   /* NOTE: Free the returned pointer with ma_free(). */\n{\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n    AudioChannelLayout* pChannelLayout;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(ppChannelLayout != NULL);\n\n    *ppChannelLayout = NULL;    /* Safety. */\n\n    propAddress.mSelector = kAudioDevicePropertyPreferredChannelLayout;\n    propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    status = ((ma_AudioObjectGetPropertyDataSize_proc)pContext->coreaudio.AudioObjectGetPropertyDataSize)(deviceObjectID, &propAddress, 0, NULL, &dataSize);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    pChannelLayout = (AudioChannelLayout*)ma_malloc(dataSize, &pContext->allocationCallbacks);\n    if (pChannelLayout == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, pChannelLayout);\n    if (status != noErr) {\n        ma_free(pChannelLayout, &pContext->allocationCallbacks);\n        return ma_result_from_OSStatus(status);\n    }\n\n    *ppChannelLayout = pChannelLayout;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_get_AudioObject_channel_count(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_uint32* pChannelCount)\n{\n    AudioChannelLayout* pChannelLayout;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pChannelCount != NULL);\n\n    *pChannelCount = 0; /* Safety. */\n\n    result = ma_get_AudioObject_channel_layout(pContext, deviceObjectID, deviceType, &pChannelLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pChannelLayout->mChannelLayoutTag == kAudioChannelLayoutTag_UseChannelDescriptions) {\n        *pChannelCount = pChannelLayout->mNumberChannelDescriptions;\n    } else if (pChannelLayout->mChannelLayoutTag == kAudioChannelLayoutTag_UseChannelBitmap) {\n        *pChannelCount = ma_count_set_bits(pChannelLayout->mChannelBitmap);\n    } else {\n        *pChannelCount = AudioChannelLayoutTag_GetNumberOfChannels(pChannelLayout->mChannelLayoutTag);\n    }\n\n    ma_free(pChannelLayout, &pContext->allocationCallbacks);\n    return MA_SUCCESS;\n}\n\n#if 0\nstatic ma_result ma_get_AudioObject_channel_map(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    AudioChannelLayout* pChannelLayout;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n    result = ma_get_AudioObject_channel_layout(pContext, deviceObjectID, deviceType, &pChannelLayout);\n    if (result != MA_SUCCESS) {\n        return result;  /* Rather than always failing here, would it be more robust to simply assume a default? */\n    }\n\n    result = ma_get_channel_map_from_AudioChannelLayout(pChannelLayout, pChannelMap, channelMapCap);\n    if (result != MA_SUCCESS) {\n        ma_free(pChannelLayout, &pContext->allocationCallbacks);\n        return result;\n    }\n\n    ma_free(pChannelLayout, &pContext->allocationCallbacks);\n    return result;\n}\n#endif\n\nstatic ma_result ma_get_AudioObject_sample_rates(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, UInt32* pSampleRateRangesCount, AudioValueRange** ppSampleRateRanges)   /* NOTE: Free the returned pointer with ma_free(). */\n{\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n    AudioValueRange* pSampleRateRanges;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pSampleRateRangesCount != NULL);\n    MA_ASSERT(ppSampleRateRanges != NULL);\n\n    /* Safety. */\n    *pSampleRateRangesCount = 0;\n    *ppSampleRateRanges = NULL;\n\n    propAddress.mSelector = kAudioDevicePropertyAvailableNominalSampleRates;\n    propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    status = ((ma_AudioObjectGetPropertyDataSize_proc)pContext->coreaudio.AudioObjectGetPropertyDataSize)(deviceObjectID, &propAddress, 0, NULL, &dataSize);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    pSampleRateRanges = (AudioValueRange*)ma_malloc(dataSize, &pContext->allocationCallbacks);\n    if (pSampleRateRanges == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, pSampleRateRanges);\n    if (status != noErr) {\n        ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n        return ma_result_from_OSStatus(status);\n    }\n\n    *pSampleRateRangesCount = dataSize / sizeof(*pSampleRateRanges);\n    *ppSampleRateRanges = pSampleRateRanges;\n    return MA_SUCCESS;\n}\n\n#if 0\nstatic ma_result ma_get_AudioObject_get_closest_sample_rate(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_uint32 sampleRateIn, ma_uint32* pSampleRateOut)\n{\n    UInt32 sampleRateRangeCount;\n    AudioValueRange* pSampleRateRanges;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pSampleRateOut != NULL);\n\n    *pSampleRateOut = 0;    /* Safety. */\n\n    result = ma_get_AudioObject_sample_rates(pContext, deviceObjectID, deviceType, &sampleRateRangeCount, &pSampleRateRanges);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (sampleRateRangeCount == 0) {\n        ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n        return MA_ERROR;   /* Should never hit this case should we? */\n    }\n\n    if (sampleRateIn == 0) {\n        /* Search in order of miniaudio's preferred priority. */\n        UInt32 iMALSampleRate;\n        for (iMALSampleRate = 0; iMALSampleRate < ma_countof(g_maStandardSampleRatePriorities); ++iMALSampleRate) {\n            ma_uint32 malSampleRate = g_maStandardSampleRatePriorities[iMALSampleRate];\n            UInt32 iCASampleRate;\n            for (iCASampleRate = 0; iCASampleRate < sampleRateRangeCount; ++iCASampleRate) {\n                AudioValueRange caSampleRate = pSampleRateRanges[iCASampleRate];\n                if (caSampleRate.mMinimum <= malSampleRate && caSampleRate.mMaximum >= malSampleRate) {\n                    *pSampleRateOut = malSampleRate;\n                    ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n                    return MA_SUCCESS;\n                }\n            }\n        }\n\n        /*\n        If we get here it means none of miniaudio's standard sample rates matched any of the supported sample rates from the device. In this\n        case we just fall back to the first one reported by Core Audio.\n        */\n        MA_ASSERT(sampleRateRangeCount > 0);\n\n        *pSampleRateOut = pSampleRateRanges[0].mMinimum;\n        ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n        return MA_SUCCESS;\n    } else {\n        /* Find the closest match to this sample rate. */\n        UInt32 currentAbsoluteDifference = INT32_MAX;\n        UInt32 iCurrentClosestRange = (UInt32)-1;\n        UInt32 iRange;\n        for (iRange = 0; iRange < sampleRateRangeCount; ++iRange) {\n            if (pSampleRateRanges[iRange].mMinimum <= sampleRateIn && pSampleRateRanges[iRange].mMaximum >= sampleRateIn) {\n                *pSampleRateOut = sampleRateIn;\n                ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n                return MA_SUCCESS;\n            } else {\n                UInt32 absoluteDifference;\n                if (pSampleRateRanges[iRange].mMinimum > sampleRateIn) {\n                    absoluteDifference = pSampleRateRanges[iRange].mMinimum - sampleRateIn;\n                } else {\n                    absoluteDifference = sampleRateIn - pSampleRateRanges[iRange].mMaximum;\n                }\n\n                if (currentAbsoluteDifference > absoluteDifference) {\n                    currentAbsoluteDifference = absoluteDifference;\n                    iCurrentClosestRange = iRange;\n                }\n            }\n        }\n\n        MA_ASSERT(iCurrentClosestRange != (UInt32)-1);\n\n        *pSampleRateOut = pSampleRateRanges[iCurrentClosestRange].mMinimum;\n        ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n        return MA_SUCCESS;\n    }\n\n    /* Should never get here, but it would mean we weren't able to find any suitable sample rates. */\n    /*ma_free(pSampleRateRanges, &pContext->allocationCallbacks);*/\n    /*return MA_ERROR;*/\n}\n#endif\n\nstatic ma_result ma_get_AudioObject_closest_buffer_size_in_frames(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_uint32 bufferSizeInFramesIn, ma_uint32* pBufferSizeInFramesOut)\n{\n    AudioObjectPropertyAddress propAddress;\n    AudioValueRange bufferSizeRange;\n    UInt32 dataSize;\n    OSStatus status;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pBufferSizeInFramesOut != NULL);\n\n    *pBufferSizeInFramesOut = 0;    /* Safety. */\n\n    propAddress.mSelector = kAudioDevicePropertyBufferFrameSizeRange;\n    propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    dataSize = sizeof(bufferSizeRange);\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, &bufferSizeRange);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    /* This is just a clamp. */\n    if (bufferSizeInFramesIn < bufferSizeRange.mMinimum) {\n        *pBufferSizeInFramesOut = (ma_uint32)bufferSizeRange.mMinimum;\n    } else if (bufferSizeInFramesIn > bufferSizeRange.mMaximum) {\n        *pBufferSizeInFramesOut = (ma_uint32)bufferSizeRange.mMaximum;\n    } else {\n        *pBufferSizeInFramesOut = bufferSizeInFramesIn;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_set_AudioObject_buffer_size_in_frames(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_uint32* pPeriodSizeInOut)\n{\n    ma_result result;\n    ma_uint32 chosenBufferSizeInFrames;\n    AudioObjectPropertyAddress propAddress;\n    UInt32 dataSize;\n    OSStatus status;\n\n    MA_ASSERT(pContext != NULL);\n\n    result = ma_get_AudioObject_closest_buffer_size_in_frames(pContext, deviceObjectID, deviceType, *pPeriodSizeInOut, &chosenBufferSizeInFrames);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Try setting the size of the buffer... If this fails we just use whatever is currently set. */\n    propAddress.mSelector = kAudioDevicePropertyBufferFrameSize;\n    propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n    propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n    ((ma_AudioObjectSetPropertyData_proc)pContext->coreaudio.AudioObjectSetPropertyData)(deviceObjectID, &propAddress, 0, NULL, sizeof(chosenBufferSizeInFrames), &chosenBufferSizeInFrames);\n\n    /* Get the actual size of the buffer. */\n    dataSize = sizeof(*pPeriodSizeInOut);\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(deviceObjectID, &propAddress, 0, NULL, &dataSize, &chosenBufferSizeInFrames);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    *pPeriodSizeInOut = chosenBufferSizeInFrames;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_find_default_AudioObjectID(ma_context* pContext, ma_device_type deviceType, AudioObjectID* pDeviceObjectID)\n{\n    AudioObjectPropertyAddress propAddressDefaultDevice;\n    UInt32 defaultDeviceObjectIDSize = sizeof(AudioObjectID);\n    AudioObjectID defaultDeviceObjectID;\n    OSStatus status;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pDeviceObjectID != NULL);\n\n    /* Safety. */\n    *pDeviceObjectID = 0;\n\n    propAddressDefaultDevice.mScope = kAudioObjectPropertyScopeGlobal;\n    propAddressDefaultDevice.mElement = AUDIO_OBJECT_PROPERTY_ELEMENT;\n    if (deviceType == ma_device_type_playback) {\n        propAddressDefaultDevice.mSelector = kAudioHardwarePropertyDefaultOutputDevice;\n    } else {\n        propAddressDefaultDevice.mSelector = kAudioHardwarePropertyDefaultInputDevice;\n    }\n\n    defaultDeviceObjectIDSize = sizeof(AudioObjectID);\n    status = ((ma_AudioObjectGetPropertyData_proc)pContext->coreaudio.AudioObjectGetPropertyData)(kAudioObjectSystemObject, &propAddressDefaultDevice, 0, NULL, &defaultDeviceObjectIDSize, &defaultDeviceObjectID);\n    if (status == noErr) {\n        *pDeviceObjectID = defaultDeviceObjectID;\n        return MA_SUCCESS;\n    }\n\n    /* If we get here it means we couldn't find the device. */\n    return MA_NO_DEVICE;\n}\n\nstatic ma_result ma_find_AudioObjectID(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, AudioObjectID* pDeviceObjectID)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pDeviceObjectID != NULL);\n\n    /* Safety. */\n    *pDeviceObjectID = 0;\n\n    if (pDeviceID == NULL) {\n        /* Default device. */\n        return ma_find_default_AudioObjectID(pContext, deviceType, pDeviceObjectID);\n    } else {\n        /* Explicit device. */\n        UInt32 deviceCount;\n        AudioObjectID* pDeviceObjectIDs;\n        ma_result result;\n        UInt32 iDevice;\n\n        result = ma_get_device_object_ids__coreaudio(pContext, &deviceCount, &pDeviceObjectIDs);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        for (iDevice = 0; iDevice < deviceCount; ++iDevice) {\n            AudioObjectID deviceObjectID = pDeviceObjectIDs[iDevice];\n\n            char uid[256];\n            if (ma_get_AudioObject_uid(pContext, deviceObjectID, sizeof(uid), uid) != MA_SUCCESS) {\n                continue;\n            }\n\n            if (deviceType == ma_device_type_playback) {\n                if (ma_does_AudioObject_support_playback(pContext, deviceObjectID)) {\n                    if (strcmp(uid, pDeviceID->coreaudio) == 0) {\n                        *pDeviceObjectID = deviceObjectID;\n                        ma_free(pDeviceObjectIDs, &pContext->allocationCallbacks);\n                        return MA_SUCCESS;\n                    }\n                }\n            } else {\n                if (ma_does_AudioObject_support_capture(pContext, deviceObjectID)) {\n                    if (strcmp(uid, pDeviceID->coreaudio) == 0) {\n                        *pDeviceObjectID = deviceObjectID;\n                        ma_free(pDeviceObjectIDs, &pContext->allocationCallbacks);\n                        return MA_SUCCESS;\n                    }\n                }\n            }\n        }\n\n        ma_free(pDeviceObjectIDs, &pContext->allocationCallbacks);\n    }\n\n    /* If we get here it means we couldn't find the device. */\n    return MA_NO_DEVICE;\n}\n\n\nstatic ma_result ma_find_best_format__coreaudio(ma_context* pContext, AudioObjectID deviceObjectID, ma_device_type deviceType, ma_format format, ma_uint32 channels, ma_uint32 sampleRate, const AudioStreamBasicDescription* pOrigFormat, AudioStreamBasicDescription* pFormat)\n{\n    UInt32 deviceFormatDescriptionCount;\n    AudioStreamRangedDescription* pDeviceFormatDescriptions;\n    ma_result result;\n    ma_uint32 desiredSampleRate;\n    ma_uint32 desiredChannelCount;\n    ma_format desiredFormat;\n    AudioStreamBasicDescription bestDeviceFormatSoFar;\n    ma_bool32 hasSupportedFormat;\n    UInt32 iFormat;\n\n    result = ma_get_AudioObject_stream_descriptions(pContext, deviceObjectID, deviceType, &deviceFormatDescriptionCount, &pDeviceFormatDescriptions);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    desiredSampleRate = sampleRate;\n    if (desiredSampleRate == 0) {\n        desiredSampleRate = (ma_uint32)pOrigFormat->mSampleRate;\n    }\n\n    desiredChannelCount = channels;\n    if (desiredChannelCount == 0) {\n        desiredChannelCount = pOrigFormat->mChannelsPerFrame;\n    }\n\n    desiredFormat = format;\n    if (desiredFormat == ma_format_unknown) {\n        result = ma_format_from_AudioStreamBasicDescription(pOrigFormat, &desiredFormat);\n        if (result != MA_SUCCESS || desiredFormat == ma_format_unknown) {\n            desiredFormat = g_maFormatPriorities[0];\n        }\n    }\n\n    /*\n    If we get here it means we don't have an exact match to what the client is asking for. We'll need to find the closest one. The next\n    loop will check for formats that have the same sample rate to what we're asking for. If there is, we prefer that one in all cases.\n    */\n    MA_ZERO_OBJECT(&bestDeviceFormatSoFar);\n\n    hasSupportedFormat = MA_FALSE;\n    for (iFormat = 0; iFormat < deviceFormatDescriptionCount; ++iFormat) {\n        ma_format formatFromDescription;\n        ma_result formatResult = ma_format_from_AudioStreamBasicDescription(&pDeviceFormatDescriptions[iFormat].mFormat, &formatFromDescription);\n        if (formatResult == MA_SUCCESS && formatFromDescription != ma_format_unknown) {\n            hasSupportedFormat = MA_TRUE;\n            bestDeviceFormatSoFar = pDeviceFormatDescriptions[iFormat].mFormat;\n            break;\n        }\n    }\n\n    if (!hasSupportedFormat) {\n        ma_free(pDeviceFormatDescriptions, &pContext->allocationCallbacks);\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n\n    for (iFormat = 0; iFormat < deviceFormatDescriptionCount; ++iFormat) {\n        AudioStreamBasicDescription thisDeviceFormat = pDeviceFormatDescriptions[iFormat].mFormat;\n        ma_format thisSampleFormat;\n        ma_result formatResult;\n        ma_format bestSampleFormatSoFar;\n\n        /* If the format is not supported by miniaudio we need to skip this one entirely. */\n        formatResult = ma_format_from_AudioStreamBasicDescription(&pDeviceFormatDescriptions[iFormat].mFormat, &thisSampleFormat);\n        if (formatResult != MA_SUCCESS || thisSampleFormat == ma_format_unknown) {\n            continue;   /* The format is not supported by miniaudio. Skip. */\n        }\n\n        ma_format_from_AudioStreamBasicDescription(&bestDeviceFormatSoFar, &bestSampleFormatSoFar);\n\n        /* Getting here means the format is supported by miniaudio which makes this format a candidate. */\n        if (thisDeviceFormat.mSampleRate != desiredSampleRate) {\n            /*\n            The sample rate does not match, but this format could still be usable, although it's a very low priority. If the best format\n            so far has an equal sample rate we can just ignore this one.\n            */\n            if (bestDeviceFormatSoFar.mSampleRate == desiredSampleRate) {\n                continue;   /* The best sample rate so far has the same sample rate as what we requested which means it's still the best so far. Skip this format. */\n            } else {\n                /* In this case, neither the best format so far nor this one have the same sample rate. Check the channel count next. */\n                if (thisDeviceFormat.mChannelsPerFrame != desiredChannelCount) {\n                    /* This format has a different sample rate _and_ a different channel count. */\n                    if (bestDeviceFormatSoFar.mChannelsPerFrame == desiredChannelCount) {\n                        continue;   /* No change to the best format. */\n                    } else {\n                        /*\n                        Both this format and the best so far have different sample rates and different channel counts. Whichever has the\n                        best format is the new best.\n                        */\n                        if (ma_get_format_priority_index(thisSampleFormat) < ma_get_format_priority_index(bestSampleFormatSoFar)) {\n                            bestDeviceFormatSoFar = thisDeviceFormat;\n                            continue;\n                        } else {\n                            continue;   /* No change to the best format. */\n                        }\n                    }\n                } else {\n                    /* This format has a different sample rate but the desired channel count. */\n                    if (bestDeviceFormatSoFar.mChannelsPerFrame == desiredChannelCount) {\n                        /* Both this format and the best so far have the desired channel count. Whichever has the best format is the new best. */\n                        if (ma_get_format_priority_index(thisSampleFormat) < ma_get_format_priority_index(bestSampleFormatSoFar)) {\n                            bestDeviceFormatSoFar = thisDeviceFormat;\n                            continue;\n                        } else {\n                            continue;   /* No change to the best format for now. */\n                        }\n                    } else {\n                        /* This format has the desired channel count, but the best so far does not. We have a new best. */\n                        bestDeviceFormatSoFar = thisDeviceFormat;\n                        continue;\n                    }\n                }\n            }\n        } else {\n            /*\n            The sample rates match which makes this format a very high priority contender. If the best format so far has a different\n            sample rate it needs to be replaced with this one.\n            */\n            if (bestDeviceFormatSoFar.mSampleRate != desiredSampleRate) {\n                bestDeviceFormatSoFar = thisDeviceFormat;\n                continue;\n            } else {\n                /* In this case both this format and the best format so far have the same sample rate. Check the channel count next. */\n                if (thisDeviceFormat.mChannelsPerFrame == desiredChannelCount) {\n                    /*\n                    In this case this format has the same channel count as what the client is requesting. If the best format so far has\n                    a different count, this one becomes the new best.\n                    */\n                    if (bestDeviceFormatSoFar.mChannelsPerFrame != desiredChannelCount) {\n                        bestDeviceFormatSoFar = thisDeviceFormat;\n                        continue;\n                    } else {\n                        /* In this case both this format and the best so far have the ideal sample rate and channel count. Check the format. */\n                        if (thisSampleFormat == desiredFormat) {\n                            bestDeviceFormatSoFar = thisDeviceFormat;\n                            break;  /* Found the exact match. */\n                        } else {\n                            /* The formats are different. The new best format is the one with the highest priority format according to miniaudio. */\n                            if (ma_get_format_priority_index(thisSampleFormat) < ma_get_format_priority_index(bestSampleFormatSoFar)) {\n                                bestDeviceFormatSoFar = thisDeviceFormat;\n                                continue;\n                            } else {\n                                continue;   /* No change to the best format for now. */\n                            }\n                        }\n                    }\n                } else {\n                    /*\n                    In this case the channel count is different to what the client has requested. If the best so far has the same channel\n                    count as the requested count then it remains the best.\n                    */\n                    if (bestDeviceFormatSoFar.mChannelsPerFrame == desiredChannelCount) {\n                        continue;\n                    } else {\n                        /*\n                        This is the case where both have the same sample rate (good) but different channel counts. Right now both have about\n                        the same priority, but we need to compare the format now.\n                        */\n                        if (thisSampleFormat == bestSampleFormatSoFar) {\n                            if (ma_get_format_priority_index(thisSampleFormat) < ma_get_format_priority_index(bestSampleFormatSoFar)) {\n                                bestDeviceFormatSoFar = thisDeviceFormat;\n                                continue;\n                            } else {\n                                continue;   /* No change to the best format for now. */\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    *pFormat = bestDeviceFormatSoFar;\n\n    ma_free(pDeviceFormatDescriptions, &pContext->allocationCallbacks);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_get_AudioUnit_channel_map(ma_context* pContext, AudioUnit audioUnit, ma_device_type deviceType, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    AudioUnitScope deviceScope;\n    AudioUnitElement deviceBus;\n    UInt32 channelLayoutSize;\n    OSStatus status;\n    AudioChannelLayout* pChannelLayout;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n    if (deviceType == ma_device_type_playback) {\n        deviceScope = kAudioUnitScope_Input;\n        deviceBus = MA_COREAUDIO_OUTPUT_BUS;\n    } else {\n        deviceScope = kAudioUnitScope_Output;\n        deviceBus = MA_COREAUDIO_INPUT_BUS;\n    }\n\n    status = ((ma_AudioUnitGetPropertyInfo_proc)pContext->coreaudio.AudioUnitGetPropertyInfo)(audioUnit, kAudioUnitProperty_AudioChannelLayout, deviceScope, deviceBus, &channelLayoutSize, NULL);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n    pChannelLayout = (AudioChannelLayout*)ma_malloc(channelLayoutSize, &pContext->allocationCallbacks);\n    if (pChannelLayout == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    status = ((ma_AudioUnitGetProperty_proc)pContext->coreaudio.AudioUnitGetProperty)(audioUnit, kAudioUnitProperty_AudioChannelLayout, deviceScope, deviceBus, pChannelLayout, &channelLayoutSize);\n    if (status != noErr) {\n        ma_free(pChannelLayout, &pContext->allocationCallbacks);\n        return ma_result_from_OSStatus(status);\n    }\n\n    result = ma_get_channel_map_from_AudioChannelLayout(pChannelLayout, pChannelMap, channelMapCap);\n    if (result != MA_SUCCESS) {\n        ma_free(pChannelLayout, &pContext->allocationCallbacks);\n        return result;\n    }\n\n    ma_free(pChannelLayout, &pContext->allocationCallbacks);\n    return MA_SUCCESS;\n}\n#endif /* MA_APPLE_DESKTOP */\n\n\n#if !defined(MA_APPLE_DESKTOP)\nstatic void ma_AVAudioSessionPortDescription_to_device_info(AVAudioSessionPortDescription* pPortDesc, ma_device_info* pInfo)\n{\n    MA_ZERO_OBJECT(pInfo);\n    ma_strncpy_s(pInfo->name,         sizeof(pInfo->name),         [pPortDesc.portName UTF8String], (size_t)-1);\n    ma_strncpy_s(pInfo->id.coreaudio, sizeof(pInfo->id.coreaudio), [pPortDesc.UID      UTF8String], (size_t)-1);\n}\n#endif\n\nstatic ma_result ma_context_enumerate_devices__coreaudio(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n#if defined(MA_APPLE_DESKTOP)\n    UInt32 deviceCount;\n    AudioObjectID* pDeviceObjectIDs;\n    AudioObjectID defaultDeviceObjectIDPlayback;\n    AudioObjectID defaultDeviceObjectIDCapture;\n    ma_result result;\n    UInt32 iDevice;\n\n    ma_find_default_AudioObjectID(pContext, ma_device_type_playback, &defaultDeviceObjectIDPlayback);   /* OK if this fails. */\n    ma_find_default_AudioObjectID(pContext, ma_device_type_capture,  &defaultDeviceObjectIDCapture);    /* OK if this fails. */\n\n    result = ma_get_device_object_ids__coreaudio(pContext, &deviceCount, &pDeviceObjectIDs);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    for (iDevice = 0; iDevice < deviceCount; ++iDevice) {\n        AudioObjectID deviceObjectID = pDeviceObjectIDs[iDevice];\n        ma_device_info info;\n\n        MA_ZERO_OBJECT(&info);\n        if (ma_get_AudioObject_uid(pContext, deviceObjectID, sizeof(info.id.coreaudio), info.id.coreaudio) != MA_SUCCESS) {\n            continue;\n        }\n        if (ma_get_AudioObject_name(pContext, deviceObjectID, sizeof(info.name), info.name) != MA_SUCCESS) {\n            continue;\n        }\n\n        if (ma_does_AudioObject_support_playback(pContext, deviceObjectID)) {\n            if (deviceObjectID == defaultDeviceObjectIDPlayback) {\n                info.isDefault = MA_TRUE;\n            }\n\n            if (!callback(pContext, ma_device_type_playback, &info, pUserData)) {\n                break;\n            }\n        }\n        if (ma_does_AudioObject_support_capture(pContext, deviceObjectID)) {\n            if (deviceObjectID == defaultDeviceObjectIDCapture) {\n                info.isDefault = MA_TRUE;\n            }\n\n            if (!callback(pContext, ma_device_type_capture, &info, pUserData)) {\n                break;\n            }\n        }\n    }\n\n    ma_free(pDeviceObjectIDs, &pContext->allocationCallbacks);\n#else\n    ma_device_info info;\n    NSArray *pInputs  = [[[AVAudioSession sharedInstance] currentRoute] inputs];\n    NSArray *pOutputs = [[[AVAudioSession sharedInstance] currentRoute] outputs];\n\n    for (AVAudioSessionPortDescription* pPortDesc in pOutputs) {\n        ma_AVAudioSessionPortDescription_to_device_info(pPortDesc, &info);\n        if (!callback(pContext, ma_device_type_playback, &info, pUserData)) {\n            return MA_SUCCESS;\n        }\n    }\n\n    for (AVAudioSessionPortDescription* pPortDesc in pInputs) {\n        ma_AVAudioSessionPortDescription_to_device_info(pPortDesc, &info);\n        if (!callback(pContext, ma_device_type_capture, &info, pUserData)) {\n            return MA_SUCCESS;\n        }\n    }\n#endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__coreaudio(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n#if defined(MA_APPLE_DESKTOP)\n    /* Desktop */\n    {\n        AudioObjectID deviceObjectID;\n        AudioObjectID defaultDeviceObjectID;\n        UInt32 streamDescriptionCount;\n        AudioStreamRangedDescription* pStreamDescriptions;\n        UInt32 iStreamDescription;\n        UInt32 sampleRateRangeCount;\n        AudioValueRange* pSampleRateRanges;\n\n        ma_find_default_AudioObjectID(pContext, deviceType, &defaultDeviceObjectID);     /* OK if this fails. */\n\n        result = ma_find_AudioObjectID(pContext, deviceType, pDeviceID, &deviceObjectID);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_get_AudioObject_uid(pContext, deviceObjectID, sizeof(pDeviceInfo->id.coreaudio), pDeviceInfo->id.coreaudio);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_get_AudioObject_name(pContext, deviceObjectID, sizeof(pDeviceInfo->name), pDeviceInfo->name);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        if (deviceObjectID == defaultDeviceObjectID) {\n            pDeviceInfo->isDefault = MA_TRUE;\n        }\n\n        /*\n        There could be a large number of permutations here. Fortunately there is only a single channel count\n        being reported which reduces this quite a bit. For sample rates we're only reporting those that are\n        one of miniaudio's recognized \"standard\" rates. If there are still more formats than can fit into\n        our fixed sized array we'll just need to truncate them. This is unlikely and will probably only happen\n        if some driver performs software data conversion and therefore reports every possible format and\n        sample rate.\n        */\n        pDeviceInfo->nativeDataFormatCount = 0;\n\n        /* Formats. */\n        {\n            ma_format uniqueFormats[ma_format_count];\n            ma_uint32 uniqueFormatCount = 0;\n            ma_uint32 channels;\n\n            /* Channels. */\n            result = ma_get_AudioObject_channel_count(pContext, deviceObjectID, deviceType, &channels);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n\n            /* Formats. */\n            result = ma_get_AudioObject_stream_descriptions(pContext, deviceObjectID, deviceType, &streamDescriptionCount, &pStreamDescriptions);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n\n            for (iStreamDescription = 0; iStreamDescription < streamDescriptionCount; ++iStreamDescription) {\n                ma_format format;\n                ma_bool32 hasFormatBeenHandled = MA_FALSE;\n                ma_uint32 iOutputFormat;\n                ma_uint32 iSampleRate;\n\n                result = ma_format_from_AudioStreamBasicDescription(&pStreamDescriptions[iStreamDescription].mFormat, &format);\n                if (result != MA_SUCCESS) {\n                    continue;\n                }\n\n                MA_ASSERT(format != ma_format_unknown);\n\n                /* Make sure the format isn't already in the output list. */\n                for (iOutputFormat = 0; iOutputFormat < uniqueFormatCount; ++iOutputFormat) {\n                    if (uniqueFormats[iOutputFormat] == format) {\n                        hasFormatBeenHandled = MA_TRUE;\n                        break;\n                    }\n                }\n\n                /* If we've already handled this format just skip it. */\n                if (hasFormatBeenHandled) {\n                    continue;\n                }\n\n                uniqueFormats[uniqueFormatCount] = format;\n                uniqueFormatCount += 1;\n\n                /* Sample Rates */\n                result = ma_get_AudioObject_sample_rates(pContext, deviceObjectID, deviceType, &sampleRateRangeCount, &pSampleRateRanges);\n                if (result != MA_SUCCESS) {\n                    return result;\n                }\n\n                /*\n                Annoyingly Core Audio reports a sample rate range. We just get all the standard rates that are\n                between this range.\n                */\n                for (iSampleRate = 0; iSampleRate < sampleRateRangeCount; ++iSampleRate) {\n                    ma_uint32 iStandardSampleRate;\n                    for (iStandardSampleRate = 0; iStandardSampleRate < ma_countof(g_maStandardSampleRatePriorities); iStandardSampleRate += 1) {\n                        ma_uint32 standardSampleRate = g_maStandardSampleRatePriorities[iStandardSampleRate];\n                        if (standardSampleRate >= pSampleRateRanges[iSampleRate].mMinimum && standardSampleRate <= pSampleRateRanges[iSampleRate].mMaximum) {\n                            /* We have a new data format. Add it to the list. */\n                            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = format;\n                            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n                            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = standardSampleRate;\n                            pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = 0;\n                            pDeviceInfo->nativeDataFormatCount += 1;\n\n                            if (pDeviceInfo->nativeDataFormatCount >= ma_countof(pDeviceInfo->nativeDataFormats)) {\n                                break;  /* No more room for any more formats. */\n                            }\n                        }\n                    }\n                }\n\n                ma_free(pSampleRateRanges, &pContext->allocationCallbacks);\n\n                if (pDeviceInfo->nativeDataFormatCount >= ma_countof(pDeviceInfo->nativeDataFormats)) {\n                    break;  /* No more room for any more formats. */\n                }\n            }\n\n            ma_free(pStreamDescriptions, &pContext->allocationCallbacks);\n        }\n    }\n#else\n    /* Mobile */\n    {\n        AudioComponentDescription desc;\n        AudioComponent component;\n        AudioUnit audioUnit;\n        OSStatus status;\n        AudioUnitScope formatScope;\n        AudioUnitElement formatElement;\n        AudioStreamBasicDescription bestFormat;\n        UInt32 propSize;\n\n        /* We want to ensure we use a consistent device name to device enumeration. */\n        if (pDeviceID != NULL && pDeviceID->coreaudio[0] != '\\0') {\n            ma_bool32 found = MA_FALSE;\n            if (deviceType == ma_device_type_playback) {\n                NSArray *pOutputs = [[[AVAudioSession sharedInstance] currentRoute] outputs];\n                for (AVAudioSessionPortDescription* pPortDesc in pOutputs) {\n                    if (strcmp(pDeviceID->coreaudio, [pPortDesc.UID UTF8String]) == 0) {\n                        ma_AVAudioSessionPortDescription_to_device_info(pPortDesc, pDeviceInfo);\n                        found = MA_TRUE;\n                        break;\n                    }\n                }\n            } else {\n                NSArray *pInputs = [[[AVAudioSession sharedInstance] currentRoute] inputs];\n                for (AVAudioSessionPortDescription* pPortDesc in pInputs) {\n                    if (strcmp(pDeviceID->coreaudio, [pPortDesc.UID UTF8String]) == 0) {\n                        ma_AVAudioSessionPortDescription_to_device_info(pPortDesc, pDeviceInfo);\n                        found = MA_TRUE;\n                        break;\n                    }\n                }\n            }\n\n            if (!found) {\n                return MA_DOES_NOT_EXIST;\n            }\n        } else {\n            if (deviceType == ma_device_type_playback) {\n                ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n            } else {\n                ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n            }\n        }\n\n\n        /*\n        Retrieving device information is more annoying on mobile than desktop. For simplicity I'm locking this down to whatever format is\n        reported on a temporary I/O unit. The problem, however, is that this doesn't return a value for the sample rate which we need to\n        retrieve from the AVAudioSession shared instance.\n        */\n        desc.componentType = kAudioUnitType_Output;\n        desc.componentSubType = kAudioUnitSubType_RemoteIO;\n        desc.componentManufacturer = kAudioUnitManufacturer_Apple;\n        desc.componentFlags = 0;\n        desc.componentFlagsMask = 0;\n\n        component = ((ma_AudioComponentFindNext_proc)pContext->coreaudio.AudioComponentFindNext)(NULL, &desc);\n        if (component == NULL) {\n            return MA_FAILED_TO_INIT_BACKEND;\n        }\n\n        status = ((ma_AudioComponentInstanceNew_proc)pContext->coreaudio.AudioComponentInstanceNew)(component, &audioUnit);\n        if (status != noErr) {\n            return ma_result_from_OSStatus(status);\n        }\n\n        formatScope   = (deviceType == ma_device_type_playback) ? kAudioUnitScope_Input : kAudioUnitScope_Output;\n        formatElement = (deviceType == ma_device_type_playback) ? MA_COREAUDIO_OUTPUT_BUS : MA_COREAUDIO_INPUT_BUS;\n\n        propSize = sizeof(bestFormat);\n        status = ((ma_AudioUnitGetProperty_proc)pContext->coreaudio.AudioUnitGetProperty)(audioUnit, kAudioUnitProperty_StreamFormat, formatScope, formatElement, &bestFormat, &propSize);\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(audioUnit);\n        audioUnit = NULL;\n\n        /* Only a single format is being reported for iOS. */\n        pDeviceInfo->nativeDataFormatCount = 1;\n\n        result = ma_format_from_AudioStreamBasicDescription(&bestFormat, &pDeviceInfo->nativeDataFormats[0].format);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pDeviceInfo->nativeDataFormats[0].channels = bestFormat.mChannelsPerFrame;\n\n        /*\n        It looks like Apple are wanting to push the whole AVAudioSession thing. Thus, we need to use that to determine device settings. To do\n        this we just get the shared instance and inspect.\n        */\n        @autoreleasepool {\n            AVAudioSession* pAudioSession = [AVAudioSession sharedInstance];\n            MA_ASSERT(pAudioSession != NULL);\n\n            pDeviceInfo->nativeDataFormats[0].sampleRate = (ma_uint32)pAudioSession.sampleRate;\n        }\n    }\n#endif\n\n    (void)pDeviceInfo; /* Unused. */\n    return MA_SUCCESS;\n}\n\nstatic AudioBufferList* ma_allocate_AudioBufferList__coreaudio(ma_uint32 sizeInFrames, ma_format format, ma_uint32 channels, ma_stream_layout layout, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    AudioBufferList* pBufferList;\n    UInt32 audioBufferSizeInBytes;\n    size_t allocationSize;\n\n    MA_ASSERT(sizeInFrames > 0);\n    MA_ASSERT(format != ma_format_unknown);\n    MA_ASSERT(channels > 0);\n\n    allocationSize = sizeof(AudioBufferList) - sizeof(AudioBuffer);  /* Subtract sizeof(AudioBuffer) because that part is dynamically sized. */\n    if (layout == ma_stream_layout_interleaved) {\n        /* Interleaved case. This is the simple case because we just have one buffer. */\n        allocationSize += sizeof(AudioBuffer) * 1;\n    } else {\n        /* Non-interleaved case. This is the more complex case because there's more than one buffer. */\n        allocationSize += sizeof(AudioBuffer) * channels;\n    }\n\n    allocationSize += sizeInFrames * ma_get_bytes_per_frame(format, channels);\n\n    pBufferList = (AudioBufferList*)ma_malloc(allocationSize, pAllocationCallbacks);\n    if (pBufferList == NULL) {\n        return NULL;\n    }\n\n    audioBufferSizeInBytes = (UInt32)(sizeInFrames * ma_get_bytes_per_sample(format));\n\n    if (layout == ma_stream_layout_interleaved) {\n        pBufferList->mNumberBuffers = 1;\n        pBufferList->mBuffers[0].mNumberChannels = channels;\n        pBufferList->mBuffers[0].mDataByteSize   = audioBufferSizeInBytes * channels;\n        pBufferList->mBuffers[0].mData           = (ma_uint8*)pBufferList + sizeof(AudioBufferList);\n    } else {\n        ma_uint32 iBuffer;\n        pBufferList->mNumberBuffers = channels;\n        for (iBuffer = 0; iBuffer < pBufferList->mNumberBuffers; ++iBuffer) {\n            pBufferList->mBuffers[iBuffer].mNumberChannels = 1;\n            pBufferList->mBuffers[iBuffer].mDataByteSize   = audioBufferSizeInBytes;\n            pBufferList->mBuffers[iBuffer].mData           = (ma_uint8*)pBufferList + ((sizeof(AudioBufferList) - sizeof(AudioBuffer)) + (sizeof(AudioBuffer) * channels)) + (audioBufferSizeInBytes * iBuffer);\n        }\n    }\n\n    return pBufferList;\n}\n\nstatic ma_result ma_device_realloc_AudioBufferList__coreaudio(ma_device* pDevice, ma_uint32 sizeInFrames, ma_format format, ma_uint32 channels, ma_stream_layout layout)\n{\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(format != ma_format_unknown);\n    MA_ASSERT(channels > 0);\n\n    /* Only resize the buffer if necessary. */\n    if (pDevice->coreaudio.audioBufferCapInFrames < sizeInFrames) {\n        AudioBufferList* pNewAudioBufferList;\n\n        pNewAudioBufferList = ma_allocate_AudioBufferList__coreaudio(sizeInFrames, format, channels, layout, &pDevice->pContext->allocationCallbacks);\n        if (pNewAudioBufferList == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        /* At this point we'll have a new AudioBufferList and we can free the old one. */\n        ma_free(pDevice->coreaudio.pAudioBufferList, &pDevice->pContext->allocationCallbacks);\n        pDevice->coreaudio.pAudioBufferList = pNewAudioBufferList;\n        pDevice->coreaudio.audioBufferCapInFrames = sizeInFrames;\n    }\n\n    /* Getting here means the capacity of the audio is fine. */\n    return MA_SUCCESS;\n}\n\n\nstatic OSStatus ma_on_output__coreaudio(void* pUserData, AudioUnitRenderActionFlags* pActionFlags, const AudioTimeStamp* pTimeStamp, UInt32 busNumber, UInt32 frameCount, AudioBufferList* pBufferList)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    ma_stream_layout layout;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"INFO: Output Callback: busNumber=%d, frameCount=%d, mNumberBuffers=%d\\n\", (int)busNumber, (int)frameCount, (int)pBufferList->mNumberBuffers);*/\n\n    /* We need to check whether or not we are outputting interleaved or non-interleaved samples. The way we do this is slightly different for each type. */\n    layout = ma_stream_layout_interleaved;\n    if (pBufferList->mBuffers[0].mNumberChannels != pDevice->playback.internalChannels) {\n        layout = ma_stream_layout_deinterleaved;\n    }\n\n    if (layout == ma_stream_layout_interleaved) {\n        /* For now we can assume everything is interleaved. */\n        UInt32 iBuffer;\n        for (iBuffer = 0; iBuffer < pBufferList->mNumberBuffers; ++iBuffer) {\n            if (pBufferList->mBuffers[iBuffer].mNumberChannels == pDevice->playback.internalChannels) {\n                ma_uint32 frameCountForThisBuffer = pBufferList->mBuffers[iBuffer].mDataByteSize / ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n                if (frameCountForThisBuffer > 0) {\n                    ma_device_handle_backend_data_callback(pDevice, pBufferList->mBuffers[iBuffer].mData, NULL, frameCountForThisBuffer);\n                }\n\n                /*a_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"  frameCount=%d, mNumberChannels=%d, mDataByteSize=%d\\n\", (int)frameCount, (int)pBufferList->mBuffers[iBuffer].mNumberChannels, (int)pBufferList->mBuffers[iBuffer].mDataByteSize);*/\n            } else {\n                /*\n                This case is where the number of channels in the output buffer do not match our internal channels. It could mean that it's\n                not interleaved, in which case we can't handle right now since miniaudio does not yet support non-interleaved streams. We just\n                output silence here.\n                */\n                MA_ZERO_MEMORY(pBufferList->mBuffers[iBuffer].mData, pBufferList->mBuffers[iBuffer].mDataByteSize);\n                /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"  WARNING: Outputting silence. frameCount=%d, mNumberChannels=%d, mDataByteSize=%d\\n\", (int)frameCount, (int)pBufferList->mBuffers[iBuffer].mNumberChannels, (int)pBufferList->mBuffers[iBuffer].mDataByteSize);*/\n            }\n        }\n    } else {\n        /* This is the deinterleaved case. We need to update each buffer in groups of internalChannels. This assumes each buffer is the same size. */\n        MA_ASSERT(pDevice->playback.internalChannels <= MA_MAX_CHANNELS);   /* This should have been validated at initialization time. */\n\n        /*\n        For safety we'll check that the internal channels is a multiple of the buffer count. If it's not it means something\n        very strange has happened and we're not going to support it.\n        */\n        if ((pBufferList->mNumberBuffers % pDevice->playback.internalChannels) == 0) {\n            ma_uint8 tempBuffer[4096];\n            UInt32 iBuffer;\n\n            for (iBuffer = 0; iBuffer < pBufferList->mNumberBuffers; iBuffer += pDevice->playback.internalChannels) {\n                ma_uint32 frameCountPerBuffer = pBufferList->mBuffers[iBuffer].mDataByteSize / ma_get_bytes_per_sample(pDevice->playback.internalFormat);\n                ma_uint32 framesRemaining = frameCountPerBuffer;\n\n                while (framesRemaining > 0) {\n                    void* ppDeinterleavedBuffers[MA_MAX_CHANNELS];\n                    ma_uint32 iChannel;\n                    ma_uint32 framesToRead = sizeof(tempBuffer) / ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n                    if (framesToRead > framesRemaining) {\n                        framesToRead = framesRemaining;\n                    }\n\n                    ma_device_handle_backend_data_callback(pDevice, tempBuffer, NULL, framesToRead);\n\n                    for (iChannel = 0; iChannel < pDevice->playback.internalChannels; ++iChannel) {\n                        ppDeinterleavedBuffers[iChannel] = (void*)ma_offset_ptr(pBufferList->mBuffers[iBuffer+iChannel].mData, (frameCountPerBuffer - framesRemaining) * ma_get_bytes_per_sample(pDevice->playback.internalFormat));\n                    }\n\n                    ma_deinterleave_pcm_frames(pDevice->playback.internalFormat, pDevice->playback.internalChannels, framesToRead, tempBuffer, ppDeinterleavedBuffers);\n\n                    framesRemaining -= framesToRead;\n                }\n            }\n        }\n    }\n\n    (void)pActionFlags;\n    (void)pTimeStamp;\n    (void)busNumber;\n    (void)frameCount;\n\n    return noErr;\n}\n\nstatic OSStatus ma_on_input__coreaudio(void* pUserData, AudioUnitRenderActionFlags* pActionFlags, const AudioTimeStamp* pTimeStamp, UInt32 busNumber, UInt32 frameCount, AudioBufferList* pUnusedBufferList)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    AudioBufferList* pRenderedBufferList;\n    ma_result result;\n    ma_stream_layout layout;\n    ma_uint32 iBuffer;\n    OSStatus status;\n\n    MA_ASSERT(pDevice != NULL);\n\n    pRenderedBufferList = (AudioBufferList*)pDevice->coreaudio.pAudioBufferList;\n    MA_ASSERT(pRenderedBufferList);\n\n    /* We need to check whether or not we are outputting interleaved or non-interleaved samples. The way we do this is slightly different for each type. */\n    layout = ma_stream_layout_interleaved;\n    if (pRenderedBufferList->mBuffers[0].mNumberChannels != pDevice->capture.internalChannels) {\n        layout = ma_stream_layout_deinterleaved;\n    }\n\n    /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"INFO: Input Callback: busNumber=%d, frameCount=%d, mNumberBuffers=%d\\n\", (int)busNumber, (int)frameCount, (int)pRenderedBufferList->mNumberBuffers);*/\n\n    /*\n    There has been a situation reported where frame count passed into this function is greater than the capacity of\n    our capture buffer. There doesn't seem to be a reliable way to determine what the maximum frame count will be,\n    so we need to instead resort to dynamically reallocating our buffer to ensure it's large enough to capture the\n    number of frames requested by this callback.\n    */\n    result = ma_device_realloc_AudioBufferList__coreaudio(pDevice, frameCount, pDevice->capture.internalFormat, pDevice->capture.internalChannels, layout);\n    if (result != MA_SUCCESS) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"Failed to allocate AudioBufferList for capture.\\n\");\n        return noErr;\n    }\n\n    pRenderedBufferList = (AudioBufferList*)pDevice->coreaudio.pAudioBufferList;\n    MA_ASSERT(pRenderedBufferList);\n\n    /*\n    When you call AudioUnitRender(), Core Audio tries to be helpful by setting the mDataByteSize to the number of bytes\n    that were actually rendered. The problem with this is that the next call can fail with -50 due to the size no longer\n    being set to the capacity of the buffer, but instead the size in bytes of the previous render. This will cause a\n    problem when a future call to this callback specifies a larger number of frames.\n\n    To work around this we need to explicitly set the size of each buffer to their respective size in bytes.\n    */\n    for (iBuffer = 0; iBuffer < pRenderedBufferList->mNumberBuffers; ++iBuffer) {\n        pRenderedBufferList->mBuffers[iBuffer].mDataByteSize = pDevice->coreaudio.audioBufferCapInFrames * ma_get_bytes_per_sample(pDevice->capture.internalFormat) * pRenderedBufferList->mBuffers[iBuffer].mNumberChannels;\n        /*printf(\"DEBUG: nDataByteSize = %d\\n\", (int)pRenderedBufferList->mBuffers[iBuffer].mDataByteSize);*/\n    }\n\n    status = ((ma_AudioUnitRender_proc)pDevice->pContext->coreaudio.AudioUnitRender)((AudioUnit)pDevice->coreaudio.audioUnitCapture, pActionFlags, pTimeStamp, busNumber, frameCount, pRenderedBufferList);\n    if (status != noErr) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"ERROR: AudioUnitRender() failed with %d.\\n\", (int)status);\n        return status;\n    }\n\n    if (layout == ma_stream_layout_interleaved) {\n        for (iBuffer = 0; iBuffer < pRenderedBufferList->mNumberBuffers; ++iBuffer) {\n            if (pRenderedBufferList->mBuffers[iBuffer].mNumberChannels == pDevice->capture.internalChannels) {\n                ma_device_handle_backend_data_callback(pDevice, NULL, pRenderedBufferList->mBuffers[iBuffer].mData, frameCount);\n                /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"  mDataByteSize=%d.\\n\", (int)pRenderedBufferList->mBuffers[iBuffer].mDataByteSize);*/\n            } else {\n                /*\n                This case is where the number of channels in the output buffer do not match our internal channels. It could mean that it's\n                not interleaved, in which case we can't handle right now since miniaudio does not yet support non-interleaved streams.\n                */\n                ma_uint8 silentBuffer[4096];\n                ma_uint32 framesRemaining;\n\n                MA_ZERO_MEMORY(silentBuffer, sizeof(silentBuffer));\n\n                framesRemaining = frameCount;\n                while (framesRemaining > 0) {\n                    ma_uint32 framesToSend = sizeof(silentBuffer) / ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n                    if (framesToSend > framesRemaining) {\n                        framesToSend = framesRemaining;\n                    }\n\n                    ma_device_handle_backend_data_callback(pDevice, NULL, silentBuffer, framesToSend);\n\n                    framesRemaining -= framesToSend;\n                }\n\n                /*ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_DEBUG, \"  WARNING: Outputting silence. frameCount=%d, mNumberChannels=%d, mDataByteSize=%d\\n\", (int)frameCount, (int)pRenderedBufferList->mBuffers[iBuffer].mNumberChannels, (int)pRenderedBufferList->mBuffers[iBuffer].mDataByteSize);*/\n            }\n        }\n    } else {\n        /* This is the deinterleaved case. We need to interleave the audio data before sending it to the client. This assumes each buffer is the same size. */\n        MA_ASSERT(pDevice->capture.internalChannels <= MA_MAX_CHANNELS);    /* This should have been validated at initialization time. */\n\n        /*\n        For safety we'll check that the internal channels is a multiple of the buffer count. If it's not it means something\n        very strange has happened and we're not going to support it.\n        */\n        if ((pRenderedBufferList->mNumberBuffers % pDevice->capture.internalChannels) == 0) {\n            ma_uint8 tempBuffer[4096];\n            for (iBuffer = 0; iBuffer < pRenderedBufferList->mNumberBuffers; iBuffer += pDevice->capture.internalChannels) {\n                ma_uint32 framesRemaining = frameCount;\n                while (framesRemaining > 0) {\n                    void* ppDeinterleavedBuffers[MA_MAX_CHANNELS];\n                    ma_uint32 iChannel;\n                    ma_uint32 framesToSend = sizeof(tempBuffer) / ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n                    if (framesToSend > framesRemaining) {\n                        framesToSend = framesRemaining;\n                    }\n\n                    for (iChannel = 0; iChannel < pDevice->capture.internalChannels; ++iChannel) {\n                        ppDeinterleavedBuffers[iChannel] = (void*)ma_offset_ptr(pRenderedBufferList->mBuffers[iBuffer+iChannel].mData, (frameCount - framesRemaining) * ma_get_bytes_per_sample(pDevice->capture.internalFormat));\n                    }\n\n                    ma_interleave_pcm_frames(pDevice->capture.internalFormat, pDevice->capture.internalChannels, framesToSend, (const void**)ppDeinterleavedBuffers, tempBuffer);\n                    ma_device_handle_backend_data_callback(pDevice, NULL, tempBuffer, framesToSend);\n\n                    framesRemaining -= framesToSend;\n                }\n            }\n        }\n    }\n\n    (void)pActionFlags;\n    (void)pTimeStamp;\n    (void)busNumber;\n    (void)frameCount;\n    (void)pUnusedBufferList;\n\n    return noErr;\n}\n\nstatic void on_start_stop__coreaudio(void* pUserData, AudioUnit audioUnit, AudioUnitPropertyID propertyID, AudioUnitScope scope, AudioUnitElement element)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    /* Don't do anything if it looks like we're just reinitializing due to a device switch. */\n    if (((audioUnit == pDevice->coreaudio.audioUnitPlayback) && pDevice->coreaudio.isSwitchingPlaybackDevice) ||\n        ((audioUnit == pDevice->coreaudio.audioUnitCapture)  && pDevice->coreaudio.isSwitchingCaptureDevice)) {\n        return;\n    }\n\n    /*\n    There's been a report of a deadlock here when triggered by ma_device_uninit(). It looks like\n    AudioUnitGetProprty (called below) and AudioComponentInstanceDispose (called in ma_device_uninit)\n    can try waiting on the same lock. I'm going to try working around this by not calling any Core\n    Audio APIs in the callback when the device has been stopped or uninitialized.\n    */\n    if (ma_device_get_state(pDevice) == ma_device_state_uninitialized || ma_device_get_state(pDevice) == ma_device_state_stopping || ma_device_get_state(pDevice) == ma_device_state_stopped) {\n        ma_device__on_notification_stopped(pDevice);\n    } else {\n        UInt32 isRunning;\n        UInt32 isRunningSize = sizeof(isRunning);\n        OSStatus status = ((ma_AudioUnitGetProperty_proc)pDevice->pContext->coreaudio.AudioUnitGetProperty)(audioUnit, kAudioOutputUnitProperty_IsRunning, scope, element, &isRunning, &isRunningSize);\n        if (status != noErr) {\n            goto done; /* Don't really know what to do in this case... just ignore it, I suppose... */\n        }\n\n        if (!isRunning) {\n            /*\n            The stop event is a bit annoying in Core Audio because it will be called when we automatically switch the default device. Some scenarios to consider:\n\n            1) When the device is unplugged, this will be called _before_ the default device change notification.\n            2) When the device is changed via the default device change notification, this will be called _after_ the switch.\n\n            For case #1, we just check if there's a new default device available. If so, we just ignore the stop event. For case #2 we check a flag.\n            */\n            if (((audioUnit == pDevice->coreaudio.audioUnitPlayback) && pDevice->coreaudio.isDefaultPlaybackDevice) ||\n                ((audioUnit == pDevice->coreaudio.audioUnitCapture)  && pDevice->coreaudio.isDefaultCaptureDevice)) {\n                /*\n                It looks like the device is switching through an external event, such as the user unplugging the device or changing the default device\n                via the operating system's sound settings. If we're re-initializing the device, we just terminate because we want the stopping of the\n                device to be seamless to the client (we don't want them receiving the stopped event and thinking that the device has stopped when it\n                hasn't!).\n                */\n                if (((audioUnit == pDevice->coreaudio.audioUnitPlayback) && pDevice->coreaudio.isSwitchingPlaybackDevice) ||\n                    ((audioUnit == pDevice->coreaudio.audioUnitCapture)  && pDevice->coreaudio.isSwitchingCaptureDevice)) {\n                    goto done;\n                }\n\n                /*\n                Getting here means the device is not reinitializing which means it may have been unplugged. From what I can see, it looks like Core Audio\n                will try switching to the new default device seamlessly. We need to somehow find a way to determine whether or not Core Audio will most\n                likely be successful in switching to the new device.\n\n                TODO: Try to predict if Core Audio will switch devices. If not, the stopped callback needs to be posted.\n                */\n                goto done;\n            }\n\n            /* Getting here means we need to stop the device. */\n            ma_device__on_notification_stopped(pDevice);\n        }\n    }\n\n    (void)propertyID; /* Unused. */\n\ndone:\n    /* Always signal the stop event. It's possible for the \"else\" case to get hit which can happen during an interruption. */\n    ma_event_signal(&pDevice->coreaudio.stopEvent);\n}\n\n#if defined(MA_APPLE_DESKTOP)\nstatic ma_spinlock g_DeviceTrackingInitLock_CoreAudio = 0;  /* A spinlock for mutal exclusion of the init/uninit of the global tracking data. Initialization to 0 is what we need. */\nstatic ma_uint32   g_DeviceTrackingInitCounter_CoreAudio = 0;\nstatic ma_mutex    g_DeviceTrackingMutex_CoreAudio;\nstatic ma_device** g_ppTrackedDevices_CoreAudio = NULL;\nstatic ma_uint32   g_TrackedDeviceCap_CoreAudio = 0;\nstatic ma_uint32   g_TrackedDeviceCount_CoreAudio = 0;\n\nstatic OSStatus ma_default_device_changed__coreaudio(AudioObjectID objectID, UInt32 addressCount, const AudioObjectPropertyAddress* pAddresses, void* pUserData)\n{\n    ma_device_type deviceType;\n\n    /* Not sure if I really need to check this, but it makes me feel better. */\n    if (addressCount == 0) {\n        return noErr;\n    }\n\n    if (pAddresses[0].mSelector == kAudioHardwarePropertyDefaultOutputDevice) {\n        deviceType = ma_device_type_playback;\n    } else if (pAddresses[0].mSelector == kAudioHardwarePropertyDefaultInputDevice) {\n        deviceType = ma_device_type_capture;\n    } else {\n        return noErr;   /* Should never hit this. */\n    }\n\n    ma_mutex_lock(&g_DeviceTrackingMutex_CoreAudio);\n    {\n        ma_uint32 iDevice;\n        for (iDevice = 0; iDevice < g_TrackedDeviceCount_CoreAudio; iDevice += 1) {\n            ma_result reinitResult;\n            ma_device* pDevice;\n\n            pDevice = g_ppTrackedDevices_CoreAudio[iDevice];\n            if (pDevice->type == deviceType || pDevice->type == ma_device_type_duplex) {\n                if (deviceType == ma_device_type_playback) {\n                    pDevice->coreaudio.isSwitchingPlaybackDevice = MA_TRUE;\n                    reinitResult = ma_device_reinit_internal__coreaudio(pDevice, deviceType, MA_TRUE);\n                    pDevice->coreaudio.isSwitchingPlaybackDevice = MA_FALSE;\n                } else {\n                    pDevice->coreaudio.isSwitchingCaptureDevice = MA_TRUE;\n                    reinitResult = ma_device_reinit_internal__coreaudio(pDevice, deviceType, MA_TRUE);\n                    pDevice->coreaudio.isSwitchingCaptureDevice = MA_FALSE;\n                }\n\n                if (reinitResult == MA_SUCCESS) {\n                    ma_device__post_init_setup(pDevice, deviceType);\n\n                    /* Restart the device if required. If this fails we need to stop the device entirely. */\n                    if (ma_device_get_state(pDevice) == ma_device_state_started) {\n                        OSStatus status;\n                        if (deviceType == ma_device_type_playback) {\n                            status = ((ma_AudioOutputUnitStart_proc)pDevice->pContext->coreaudio.AudioOutputUnitStart)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n                            if (status != noErr) {\n                                if (pDevice->type == ma_device_type_duplex) {\n                                    ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n                                }\n                                ma_device__set_state(pDevice, ma_device_state_stopped);\n                            }\n                        } else if (deviceType == ma_device_type_capture) {\n                            status = ((ma_AudioOutputUnitStart_proc)pDevice->pContext->coreaudio.AudioOutputUnitStart)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n                            if (status != noErr) {\n                                if (pDevice->type == ma_device_type_duplex) {\n                                    ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n                                }\n                                ma_device__set_state(pDevice, ma_device_state_stopped);\n                            }\n                        }\n                    }\n\n                    ma_device__on_notification_rerouted(pDevice);\n                }\n            }\n        }\n    }\n    ma_mutex_unlock(&g_DeviceTrackingMutex_CoreAudio);\n\n    /* Unused parameters. */\n    (void)objectID;\n    (void)pUserData;\n\n    return noErr;\n}\n\nstatic ma_result ma_context__init_device_tracking__coreaudio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n\n    ma_spinlock_lock(&g_DeviceTrackingInitLock_CoreAudio);\n    {\n        /* Don't do anything if we've already initialized device tracking. */\n        if (g_DeviceTrackingInitCounter_CoreAudio == 0) {\n            AudioObjectPropertyAddress propAddress;\n            propAddress.mScope    = kAudioObjectPropertyScopeGlobal;\n            propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n            ma_mutex_init(&g_DeviceTrackingMutex_CoreAudio);\n\n            propAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice;\n            ((ma_AudioObjectAddPropertyListener_proc)pContext->coreaudio.AudioObjectAddPropertyListener)(kAudioObjectSystemObject, &propAddress, &ma_default_device_changed__coreaudio, NULL);\n\n            propAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice;\n            ((ma_AudioObjectAddPropertyListener_proc)pContext->coreaudio.AudioObjectAddPropertyListener)(kAudioObjectSystemObject, &propAddress, &ma_default_device_changed__coreaudio, NULL);\n\n        }\n        g_DeviceTrackingInitCounter_CoreAudio += 1;\n    }\n    ma_spinlock_unlock(&g_DeviceTrackingInitLock_CoreAudio);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context__uninit_device_tracking__coreaudio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n\n    ma_spinlock_lock(&g_DeviceTrackingInitLock_CoreAudio);\n    {\n        if (g_DeviceTrackingInitCounter_CoreAudio > 0)\n            g_DeviceTrackingInitCounter_CoreAudio -= 1;\n\n        if (g_DeviceTrackingInitCounter_CoreAudio == 0) {\n            AudioObjectPropertyAddress propAddress;\n            propAddress.mScope    = kAudioObjectPropertyScopeGlobal;\n            propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n            propAddress.mSelector = kAudioHardwarePropertyDefaultInputDevice;\n            ((ma_AudioObjectRemovePropertyListener_proc)pContext->coreaudio.AudioObjectRemovePropertyListener)(kAudioObjectSystemObject, &propAddress, &ma_default_device_changed__coreaudio, NULL);\n\n            propAddress.mSelector = kAudioHardwarePropertyDefaultOutputDevice;\n            ((ma_AudioObjectRemovePropertyListener_proc)pContext->coreaudio.AudioObjectRemovePropertyListener)(kAudioObjectSystemObject, &propAddress, &ma_default_device_changed__coreaudio, NULL);\n\n            /* At this point there should be no tracked devices. If not there's an error somewhere. */\n            if (g_ppTrackedDevices_CoreAudio != NULL) {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"You have uninitialized all contexts while an associated device is still active.\");\n                ma_spinlock_unlock(&g_DeviceTrackingInitLock_CoreAudio);\n                return MA_INVALID_OPERATION;\n            }\n\n            ma_mutex_uninit(&g_DeviceTrackingMutex_CoreAudio);\n        }\n    }\n    ma_spinlock_unlock(&g_DeviceTrackingInitLock_CoreAudio);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device__track__coreaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    ma_mutex_lock(&g_DeviceTrackingMutex_CoreAudio);\n    {\n        /* Allocate memory if required. */\n        if (g_TrackedDeviceCap_CoreAudio <= g_TrackedDeviceCount_CoreAudio) {\n            ma_uint32 newCap;\n            ma_device** ppNewDevices;\n\n            newCap = g_TrackedDeviceCap_CoreAudio * 2;\n            if (newCap == 0) {\n                newCap = 1;\n            }\n\n            ppNewDevices = (ma_device**)ma_realloc(g_ppTrackedDevices_CoreAudio, sizeof(*g_ppTrackedDevices_CoreAudio)*newCap, &pDevice->pContext->allocationCallbacks);\n            if (ppNewDevices == NULL) {\n                ma_mutex_unlock(&g_DeviceTrackingMutex_CoreAudio);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            g_ppTrackedDevices_CoreAudio = ppNewDevices;\n            g_TrackedDeviceCap_CoreAudio = newCap;\n        }\n\n        g_ppTrackedDevices_CoreAudio[g_TrackedDeviceCount_CoreAudio] = pDevice;\n        g_TrackedDeviceCount_CoreAudio += 1;\n    }\n    ma_mutex_unlock(&g_DeviceTrackingMutex_CoreAudio);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device__untrack__coreaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    ma_mutex_lock(&g_DeviceTrackingMutex_CoreAudio);\n    {\n        ma_uint32 iDevice;\n        for (iDevice = 0; iDevice < g_TrackedDeviceCount_CoreAudio; iDevice += 1) {\n            if (g_ppTrackedDevices_CoreAudio[iDevice] == pDevice) {\n                /* We've found the device. We now need to remove it from the list. */\n                ma_uint32 jDevice;\n                for (jDevice = iDevice; jDevice < g_TrackedDeviceCount_CoreAudio-1; jDevice += 1) {\n                    g_ppTrackedDevices_CoreAudio[jDevice] = g_ppTrackedDevices_CoreAudio[jDevice+1];\n                }\n\n                g_TrackedDeviceCount_CoreAudio -= 1;\n\n                /* If there's nothing else in the list we need to free memory. */\n                if (g_TrackedDeviceCount_CoreAudio == 0) {\n                    ma_free(g_ppTrackedDevices_CoreAudio, &pDevice->pContext->allocationCallbacks);\n                    g_ppTrackedDevices_CoreAudio = NULL;\n                    g_TrackedDeviceCap_CoreAudio = 0;\n                }\n\n                break;\n            }\n        }\n    }\n    ma_mutex_unlock(&g_DeviceTrackingMutex_CoreAudio);\n\n    return MA_SUCCESS;\n}\n#endif\n\n#if defined(MA_APPLE_MOBILE)\n@interface ma_ios_notification_handler:NSObject {\n    ma_device* m_pDevice;\n}\n@end\n\n@implementation ma_ios_notification_handler\n-(id)init:(ma_device*)pDevice\n{\n    self = [super init];\n    m_pDevice = pDevice;\n\n    /* For route changes. */\n    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(handle_route_change:) name:AVAudioSessionRouteChangeNotification object:[AVAudioSession sharedInstance]];\n\n    /* For interruptions. */\n    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(handle_interruption:) name:AVAudioSessionInterruptionNotification object:[AVAudioSession sharedInstance]];\n\n    return self;\n}\n\n-(void)dealloc\n{\n    [self remove_handler];\n\n    #if defined(__has_feature)\n        #if !__has_feature(objc_arc)\n            [super dealloc];\n        #endif\n    #endif\n}\n\n-(void)remove_handler\n{\n    [[NSNotificationCenter defaultCenter] removeObserver:self name:AVAudioSessionRouteChangeNotification object:nil];\n    [[NSNotificationCenter defaultCenter] removeObserver:self name:AVAudioSessionInterruptionNotification object:nil];\n}\n\n-(void)handle_interruption:(NSNotification*)pNotification\n{\n    NSInteger type = [[[pNotification userInfo] objectForKey:AVAudioSessionInterruptionTypeKey] integerValue];\n    switch (type)\n    {\n        case AVAudioSessionInterruptionTypeBegan:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Interruption: AVAudioSessionInterruptionTypeBegan\\n\");\n\n            /*\n            Core Audio will have stopped the internal device automatically, but we need explicitly\n            stop it at a higher level to ensure miniaudio-specific state is updated for consistency.\n            */\n            ma_device_stop(m_pDevice);\n\n            /*\n            Fire the notification after the device has been stopped to ensure it's in the correct\n            state when the notification handler is invoked.\n            */\n            ma_device__on_notification_interruption_began(m_pDevice);\n        } break;\n\n        case AVAudioSessionInterruptionTypeEnded:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Interruption: AVAudioSessionInterruptionTypeEnded\\n\");\n            ma_device__on_notification_interruption_ended(m_pDevice);\n        } break;\n    }\n}\n\n-(void)handle_route_change:(NSNotification*)pNotification\n{\n    AVAudioSession* pSession = [AVAudioSession sharedInstance];\n\n    NSInteger reason = [[[pNotification userInfo] objectForKey:AVAudioSessionRouteChangeReasonKey] integerValue];\n    switch (reason)\n    {\n        case AVAudioSessionRouteChangeReasonOldDeviceUnavailable:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonOldDeviceUnavailable\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonNewDeviceAvailable:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonNewDeviceAvailable\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonWakeFromSleep:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonWakeFromSleep\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonOverride:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonOverride\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonCategoryChange:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonCategoryChange\\n\");\n        } break;\n\n        case AVAudioSessionRouteChangeReasonUnknown:\n        default:\n        {\n            ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_INFO, \"[Core Audio] Route Changed: AVAudioSessionRouteChangeReasonUnknown\\n\");\n        } break;\n    }\n\n    ma_log_postf(ma_device_get_log(m_pDevice), MA_LOG_LEVEL_DEBUG, \"[Core Audio] Changing Route. inputNumberChannels=%d; outputNumberOfChannels=%d\\n\", (int)pSession.inputNumberOfChannels, (int)pSession.outputNumberOfChannels);\n\n    /* Let the application know about the route change. */\n    ma_device__on_notification_rerouted(m_pDevice);\n}\n@end\n#endif\n\nstatic ma_result ma_device_uninit__coreaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_uninitialized);\n\n#if defined(MA_APPLE_DESKTOP)\n    /*\n    Make sure we're no longer tracking the device. It doesn't matter if we call this for a non-default device because it'll\n    just gracefully ignore it.\n    */\n    ma_device__untrack__coreaudio(pDevice);\n#endif\n#if defined(MA_APPLE_MOBILE)\n    if (pDevice->coreaudio.pNotificationHandler != NULL) {\n        ma_ios_notification_handler* pNotificationHandler = (MA_BRIDGE_TRANSFER ma_ios_notification_handler*)pDevice->coreaudio.pNotificationHandler;\n        [pNotificationHandler remove_handler];\n    }\n#endif\n\n    if (pDevice->coreaudio.audioUnitCapture != NULL) {\n        ((ma_AudioComponentInstanceDispose_proc)pDevice->pContext->coreaudio.AudioComponentInstanceDispose)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n    }\n    if (pDevice->coreaudio.audioUnitPlayback != NULL) {\n        ((ma_AudioComponentInstanceDispose_proc)pDevice->pContext->coreaudio.AudioComponentInstanceDispose)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n    }\n\n    if (pDevice->coreaudio.pAudioBufferList) {\n        ma_free(pDevice->coreaudio.pAudioBufferList, &pDevice->pContext->allocationCallbacks);\n    }\n\n    return MA_SUCCESS;\n}\n\ntypedef struct\n{\n    ma_bool32 allowNominalSampleRateChange;\n\n    /* Input. */\n    ma_format formatIn;\n    ma_uint32 channelsIn;\n    ma_uint32 sampleRateIn;\n    ma_channel channelMapIn[MA_MAX_CHANNELS];\n    ma_uint32 periodSizeInFramesIn;\n    ma_uint32 periodSizeInMillisecondsIn;\n    ma_uint32 periodsIn;\n    ma_share_mode shareMode;\n    ma_performance_profile performanceProfile;\n    ma_bool32 registerStopEvent;\n\n    /* Output. */\n#if defined(MA_APPLE_DESKTOP)\n    AudioObjectID deviceObjectID;\n#endif\n    AudioComponent component;\n    AudioUnit audioUnit;\n    AudioBufferList* pAudioBufferList;  /* Only used for input devices. */\n    ma_format formatOut;\n    ma_uint32 channelsOut;\n    ma_uint32 sampleRateOut;\n    ma_channel channelMapOut[MA_MAX_CHANNELS];\n    ma_uint32 periodSizeInFramesOut;\n    ma_uint32 periodsOut;\n    char deviceName[256];\n} ma_device_init_internal_data__coreaudio;\n\nstatic ma_result ma_device_init_internal__coreaudio(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_init_internal_data__coreaudio* pData, void* pDevice_DoNotReference)   /* <-- pDevice is typed as void* intentionally so as to avoid accidentally referencing it. */\n{\n    ma_result result = MA_SUCCESS;\n    OSStatus status;\n    UInt32 enableIOFlag;\n    AudioStreamBasicDescription bestFormat;\n    ma_uint32 actualPeriodSizeInFrames;\n    AURenderCallbackStruct callbackInfo;\n#if defined(MA_APPLE_DESKTOP)\n    AudioObjectID deviceObjectID;\n#endif\n\n    /* This API should only be used for a single device type: playback or capture. No full-duplex mode. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(deviceType == ma_device_type_playback || deviceType == ma_device_type_capture);\n\n#if defined(MA_APPLE_DESKTOP)\n    pData->deviceObjectID = 0;\n#endif\n    pData->component = NULL;\n    pData->audioUnit = NULL;\n    pData->pAudioBufferList = NULL;\n\n#if defined(MA_APPLE_DESKTOP)\n    result = ma_find_AudioObjectID(pContext, deviceType, pDeviceID, &deviceObjectID);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pData->deviceObjectID = deviceObjectID;\n#endif\n\n    /* Core audio doesn't really use the notion of a period so we can leave this unmodified, but not too over the top. */\n    pData->periodsOut = pData->periodsIn;\n    if (pData->periodsOut == 0) {\n        pData->periodsOut = MA_DEFAULT_PERIODS;\n    }\n    if (pData->periodsOut > 16) {\n        pData->periodsOut = 16;\n    }\n\n\n    /* Audio unit. */\n    status = ((ma_AudioComponentInstanceNew_proc)pContext->coreaudio.AudioComponentInstanceNew)((AudioComponent)pContext->coreaudio.component, (AudioUnit*)&pData->audioUnit);\n    if (status != noErr) {\n        return ma_result_from_OSStatus(status);\n    }\n\n\n    /* The input/output buses need to be explicitly enabled and disabled. We set the flag based on the output unit first, then we just swap it for input. */\n    enableIOFlag = 1;\n    if (deviceType == ma_device_type_capture) {\n        enableIOFlag = 0;\n    }\n\n    status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, MA_COREAUDIO_OUTPUT_BUS, &enableIOFlag, sizeof(enableIOFlag));\n    if (status != noErr) {\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n        return ma_result_from_OSStatus(status);\n    }\n\n    enableIOFlag = (enableIOFlag == 0) ? 1 : 0;\n    status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, MA_COREAUDIO_INPUT_BUS, &enableIOFlag, sizeof(enableIOFlag));\n    if (status != noErr) {\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n        return ma_result_from_OSStatus(status);\n    }\n\n\n    /* Set the device to use with this audio unit. This is only used on desktop since we are using defaults on mobile. */\n#if defined(MA_APPLE_DESKTOP)\n    status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &deviceObjectID, sizeof(deviceObjectID));\n    if (status != noErr) {\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n        return ma_result_from_OSStatus(result);\n    }\n#else\n    /*\n    For some reason it looks like Apple is only allowing selection of the input device. There does not appear to be any way to change\n    the default output route. I have no idea why this is like this, but for now we'll only be able to configure capture devices.\n    */\n    if (pDeviceID != NULL) {\n        if (deviceType == ma_device_type_capture) {\n            ma_bool32 found = MA_FALSE;\n            NSArray *pInputs = [[[AVAudioSession sharedInstance] currentRoute] inputs];\n            for (AVAudioSessionPortDescription* pPortDesc in pInputs) {\n                if (strcmp(pDeviceID->coreaudio, [pPortDesc.UID UTF8String]) == 0) {\n                    [[AVAudioSession sharedInstance] setPreferredInput:pPortDesc error:nil];\n                    found = MA_TRUE;\n                    break;\n                }\n            }\n\n            if (found == MA_FALSE) {\n                return MA_DOES_NOT_EXIST;\n            }\n        }\n    }\n#endif\n\n    /*\n    Format. This is the hardest part of initialization because there's a few variables to take into account.\n      1) The format must be supported by the device.\n      2) The format must be supported miniaudio.\n      3) There's a priority that miniaudio prefers.\n\n    Ideally we would like to use a format that's as close to the hardware as possible so we can get as close to a passthrough as possible. The\n    most important property is the sample rate. miniaudio can do format conversion for any sample rate and channel count, but cannot do the same\n    for the sample data format. If the sample data format is not supported by miniaudio it must be ignored completely.\n\n    On mobile platforms this is a bit different. We just force the use of whatever the audio unit's current format is set to.\n    */\n    {\n        AudioStreamBasicDescription origFormat;\n        UInt32 origFormatSize = sizeof(origFormat);\n        AudioUnitScope   formatScope   = (deviceType == ma_device_type_playback) ? kAudioUnitScope_Input : kAudioUnitScope_Output;\n        AudioUnitElement formatElement = (deviceType == ma_device_type_playback) ? MA_COREAUDIO_OUTPUT_BUS : MA_COREAUDIO_INPUT_BUS;\n\n        if (deviceType == ma_device_type_playback) {\n            status = ((ma_AudioUnitGetProperty_proc)pContext->coreaudio.AudioUnitGetProperty)(pData->audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, MA_COREAUDIO_OUTPUT_BUS, &origFormat, &origFormatSize);\n        } else {\n            status = ((ma_AudioUnitGetProperty_proc)pContext->coreaudio.AudioUnitGetProperty)(pData->audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, MA_COREAUDIO_INPUT_BUS, &origFormat, &origFormatSize);\n        }\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n\n    #if defined(MA_APPLE_DESKTOP)\n        result = ma_find_best_format__coreaudio(pContext, deviceObjectID, deviceType, pData->formatIn, pData->channelsIn, pData->sampleRateIn, &origFormat, &bestFormat);\n        if (result != MA_SUCCESS) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return result;\n        }\n\n        /*\n        Technical Note TN2091: Device input using the HAL Output Audio Unit\n            https://developer.apple.com/library/archive/technotes/tn2091/_index.html\n\n        This documentation says the following:\n\n            The internal AudioConverter can handle any *simple* conversion. Typically, this means that a client can specify ANY\n            variant of the PCM formats. Consequently, the device's sample rate should match the desired sample rate. If sample rate\n            conversion is needed, it can be accomplished by buffering the input and converting the data on a separate thread with\n            another AudioConverter.\n\n        The important part here is the mention that it can handle *simple* conversions, which does *not* include sample rate. We\n        therefore want to ensure the sample rate stays consistent. This document is specifically for input, but I'm going to play it\n        safe and apply the same rule to output as well.\n\n        I have tried going against the documentation by setting the sample rate anyway, but this just results in AudioUnitRender()\n        returning a result code of -10863. I have also tried changing the format directly on the input scope on the input bus, but\n        this just results in `ca_require: IsStreamFormatWritable(inScope, inElement) NotWritable` when trying to set the format.\n\n        Something that does seem to work, however, has been setting the nominal sample rate on the device object. The problem with\n        this, however, is that it actually changes the sample rate at the operating system level and not just the application. This\n        could be intrusive to the user, however, so I don't think it's wise to make this the default. Instead I'm making this a\n        configuration option. When the `coreaudio.allowNominalSampleRateChange` config option is set to true, changing the sample\n        rate will be allowed. Otherwise it'll be fixed to the current sample rate. To check the system-defined sample rate, run\n        the Audio MIDI Setup program that comes installed on macOS and observe how the sample rate changes as the sample rate is\n        changed by miniaudio.\n        */\n        if (pData->allowNominalSampleRateChange) {\n            AudioValueRange sampleRateRange;\n            AudioObjectPropertyAddress propAddress;\n\n            sampleRateRange.mMinimum = bestFormat.mSampleRate;\n            sampleRateRange.mMaximum = bestFormat.mSampleRate;\n\n            propAddress.mSelector = kAudioDevicePropertyNominalSampleRate;\n            propAddress.mScope    = (deviceType == ma_device_type_playback) ? kAudioObjectPropertyScopeOutput : kAudioObjectPropertyScopeInput;\n            propAddress.mElement  = AUDIO_OBJECT_PROPERTY_ELEMENT;\n\n            status = ((ma_AudioObjectSetPropertyData_proc)pContext->coreaudio.AudioObjectSetPropertyData)(deviceObjectID, &propAddress, 0, NULL, sizeof(sampleRateRange), &sampleRateRange);\n            if (status != noErr) {\n                bestFormat.mSampleRate = origFormat.mSampleRate;\n            }\n        } else {\n            bestFormat.mSampleRate = origFormat.mSampleRate;\n        }\n\n        status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioUnitProperty_StreamFormat, formatScope, formatElement, &bestFormat, sizeof(bestFormat));\n        if (status != noErr) {\n            /* We failed to set the format, so fall back to the current format of the audio unit. */\n            bestFormat = origFormat;\n        }\n    #else\n        bestFormat = origFormat;\n\n        /*\n        Sample rate is a little different here because for some reason kAudioUnitProperty_StreamFormat returns 0... Oh well. We need to instead try\n        setting the sample rate to what the user has requested and then just see the results of it. Need to use some Objective-C here for this since\n        it depends on Apple's AVAudioSession API. To do this we just get the shared AVAudioSession instance and then set it. Note that from what I\n        can tell, it looks like the sample rate is shared between playback and capture for everything.\n        */\n        @autoreleasepool {\n            AVAudioSession* pAudioSession = [AVAudioSession sharedInstance];\n            MA_ASSERT(pAudioSession != NULL);\n\n            [pAudioSession setPreferredSampleRate:(double)pData->sampleRateIn error:nil];\n            bestFormat.mSampleRate = pAudioSession.sampleRate;\n\n            /*\n            I've had a report that the channel count returned by AudioUnitGetProperty above is inconsistent with\n            AVAudioSession outputNumberOfChannels. I'm going to try using the AVAudioSession values instead.\n\n            UPDATE 20/02/2025:\n            When testing on the simulator with an iPhone 15 and iOS 17 I get an error when initializing the audio\n            unit if set the input channels to pAudioSession.inputNumberOfChannels. What is happening is the channel\n            count returned from AudioUnitGetProperty() above is set to 2, but pAudioSession is reporting a channel\n            count of 1. When this happens, the call to AudioUnitSetProprty() below just down below will succeed, but\n            AudioUnitInitialize() further down will fail. The only solution I have come up with is to not set the\n            channel count to pAudioSession.inputNumberOfChannels.\n            */\n            if (deviceType == ma_device_type_playback) {\n                bestFormat.mChannelsPerFrame = (UInt32)pAudioSession.outputNumberOfChannels;\n            }\n\n            #if 0\n            if (deviceType == ma_device_type_capture) {\n                /*printf(\"DEBUG: bestFormat.mChannelsPerFrame = %d; pAudioSession.inputNumberOfChannels = %d\\n\", (int)bestFormat.mChannelsPerFrame, (int)pAudioSession.inputNumberOfChannels);*/\n                bestFormat.mChannelsPerFrame = (UInt32)pAudioSession.inputNumberOfChannels;\n            }\n            #endif\n        }\n\n        \n        status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioUnitProperty_StreamFormat, formatScope, formatElement, &bestFormat, sizeof(bestFormat));\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n    #endif\n\n        result = ma_format_from_AudioStreamBasicDescription(&bestFormat, &pData->formatOut);\n        if (result != MA_SUCCESS) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return result;\n        }\n\n        if (pData->formatOut == ma_format_unknown) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        pData->channelsOut   = bestFormat.mChannelsPerFrame;\n        pData->sampleRateOut = (ma_uint32)bestFormat.mSampleRate;\n    }\n\n    /* Clamp the channel count for safety. */\n    if (pData->channelsOut > MA_MAX_CHANNELS) {\n        pData->channelsOut = MA_MAX_CHANNELS;\n    }\n\n    /*\n    Internal channel map. This is weird in my testing. If I use the AudioObject to get the\n    channel map, the channel descriptions are set to \"Unknown\" for some reason. To work around\n    this it looks like retrieving it from the AudioUnit will work. However, and this is where\n    it gets weird, it doesn't seem to work with capture devices, nor at all on iOS... Therefore\n    I'm going to fall back to a default assumption in these cases.\n    */\n#if defined(MA_APPLE_DESKTOP)\n    result = ma_get_AudioUnit_channel_map(pContext, pData->audioUnit, deviceType, pData->channelMapOut, pData->channelsOut);\n    if (result != MA_SUCCESS) {\n    #if 0\n        /* Try falling back to the channel map from the AudioObject. */\n        result = ma_get_AudioObject_channel_map(pContext, deviceObjectID, deviceType, pData->channelMapOut, pData->channelsOut);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    #else\n        /* Fall back to default assumptions. */\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pData->channelMapOut, ma_countof(pData->channelMapOut), pData->channelsOut);\n    #endif\n    }\n#else\n    /* TODO: Figure out how to get the channel map using AVAudioSession. */\n    ma_channel_map_init_standard(ma_standard_channel_map_default, pData->channelMapOut, ma_countof(pData->channelMapOut), pData->channelsOut);\n#endif\n\n\n    /* Buffer size. Not allowing this to be configurable on iOS. */\n    if (pData->periodSizeInFramesIn == 0) {\n        if (pData->periodSizeInMillisecondsIn == 0) {\n            if (pData->performanceProfile == ma_performance_profile_low_latency) {\n                actualPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY, pData->sampleRateOut);\n            } else {\n                actualPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE, pData->sampleRateOut);\n            }\n        } else {\n            actualPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(pData->periodSizeInMillisecondsIn, pData->sampleRateOut);\n        }\n    } else {\n        actualPeriodSizeInFrames = pData->periodSizeInFramesIn;\n    }\n\n#if defined(MA_APPLE_DESKTOP)\n    result = ma_set_AudioObject_buffer_size_in_frames(pContext, deviceObjectID, deviceType, &actualPeriodSizeInFrames);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n#else\n    /*\n    On iOS, the size of the IO buffer needs to be specified in seconds and is a floating point\n    number. I don't trust any potential truncation errors due to converting from float to integer\n    so I'm going to explicitly set the actual period size to the next power of 2.\n    */\n    @autoreleasepool {\n        AVAudioSession* pAudioSession = [AVAudioSession sharedInstance];\n        MA_ASSERT(pAudioSession != NULL);\n\n        [pAudioSession setPreferredIOBufferDuration:((float)actualPeriodSizeInFrames / pAudioSession.sampleRate) error:nil];\n        actualPeriodSizeInFrames = ma_next_power_of_2((ma_uint32)(pAudioSession.IOBufferDuration * pAudioSession.sampleRate));\n    }\n#endif\n\n\n    /*\n    During testing I discovered that the buffer size can be too big. You'll get an error like this:\n\n      kAudioUnitErr_TooManyFramesToProcess : inFramesToProcess=4096, mMaxFramesPerSlice=512\n\n    Note how inFramesToProcess is smaller than mMaxFramesPerSlice. To fix, we need to set kAudioUnitProperty_MaximumFramesPerSlice to that\n    of the size of our buffer, or do it the other way around and set our buffer size to the kAudioUnitProperty_MaximumFramesPerSlice.\n    */\n    status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &actualPeriodSizeInFrames, sizeof(actualPeriodSizeInFrames));\n    if (status != noErr) {\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n        return ma_result_from_OSStatus(status);\n    }\n\n    pData->periodSizeInFramesOut = (ma_uint32)actualPeriodSizeInFrames;\n\n    /* We need a buffer list if this is an input device. We render into this in the input callback. */\n    if (deviceType == ma_device_type_capture) {\n        ma_bool32 isInterleaved = (bestFormat.mFormatFlags & kAudioFormatFlagIsNonInterleaved) == 0;\n        AudioBufferList* pBufferList;\n\n        pBufferList = ma_allocate_AudioBufferList__coreaudio(pData->periodSizeInFramesOut, pData->formatOut, pData->channelsOut, (isInterleaved) ? ma_stream_layout_interleaved : ma_stream_layout_deinterleaved, &pContext->allocationCallbacks);\n        if (pBufferList == NULL) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        pData->pAudioBufferList = pBufferList;\n    }\n\n    /* Callbacks. */\n    callbackInfo.inputProcRefCon = pDevice_DoNotReference;\n    if (deviceType == ma_device_type_playback) {\n        callbackInfo.inputProc = ma_on_output__coreaudio;\n        status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Global, 0, &callbackInfo, sizeof(callbackInfo));\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n    } else {\n        callbackInfo.inputProc = ma_on_input__coreaudio;\n        status = ((ma_AudioUnitSetProperty_proc)pContext->coreaudio.AudioUnitSetProperty)(pData->audioUnit, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Global, 0, &callbackInfo, sizeof(callbackInfo));\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    /* We need to listen for stop events. */\n    if (pData->registerStopEvent) {\n        status = ((ma_AudioUnitAddPropertyListener_proc)pContext->coreaudio.AudioUnitAddPropertyListener)(pData->audioUnit, kAudioOutputUnitProperty_IsRunning, on_start_stop__coreaudio, pDevice_DoNotReference);\n        if (status != noErr) {\n            ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    /* Initialize the audio unit. */\n    status = ((ma_AudioUnitInitialize_proc)pContext->coreaudio.AudioUnitInitialize)(pData->audioUnit);\n    if (status != noErr) {\n        ma_free(pData->pAudioBufferList, &pContext->allocationCallbacks);\n        pData->pAudioBufferList = NULL;\n        ((ma_AudioComponentInstanceDispose_proc)pContext->coreaudio.AudioComponentInstanceDispose)(pData->audioUnit);\n        return ma_result_from_OSStatus(status);\n    }\n\n    /* Grab the name. */\n#if defined(MA_APPLE_DESKTOP)\n    ma_get_AudioObject_name(pContext, deviceObjectID, sizeof(pData->deviceName), pData->deviceName);\n#else\n    if (deviceType == ma_device_type_playback) {\n        ma_strcpy_s(pData->deviceName, sizeof(pData->deviceName), MA_DEFAULT_PLAYBACK_DEVICE_NAME);\n    } else {\n        ma_strcpy_s(pData->deviceName, sizeof(pData->deviceName), MA_DEFAULT_CAPTURE_DEVICE_NAME);\n    }\n#endif\n\n    return result;\n}\n\n#if defined(MA_APPLE_DESKTOP)\nstatic ma_result ma_device_reinit_internal__coreaudio(ma_device* pDevice, ma_device_type deviceType, ma_bool32 disposePreviousAudioUnit)\n{\n    ma_device_init_internal_data__coreaudio data;\n    ma_result result;\n\n    /* This should only be called for playback or capture, not duplex. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    data.allowNominalSampleRateChange = MA_FALSE;   /* Don't change the nominal sample rate when switching devices. */\n\n    if (deviceType == ma_device_type_capture) {\n        data.formatIn               = pDevice->capture.format;\n        data.channelsIn             = pDevice->capture.channels;\n        data.sampleRateIn           = pDevice->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDevice->capture.channelMap, sizeof(pDevice->capture.channelMap));\n        data.shareMode              = pDevice->capture.shareMode;\n        data.performanceProfile     = pDevice->coreaudio.originalPerformanceProfile;\n        data.registerStopEvent      = MA_TRUE;\n\n        if (disposePreviousAudioUnit) {\n            ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n            ((ma_AudioComponentInstanceDispose_proc)pDevice->pContext->coreaudio.AudioComponentInstanceDispose)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n        }\n        if (pDevice->coreaudio.pAudioBufferList) {\n            ma_free(pDevice->coreaudio.pAudioBufferList, &pDevice->pContext->allocationCallbacks);\n        }\n    } else if (deviceType == ma_device_type_playback) {\n        data.formatIn               = pDevice->playback.format;\n        data.channelsIn             = pDevice->playback.channels;\n        data.sampleRateIn           = pDevice->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDevice->playback.channelMap, sizeof(pDevice->playback.channelMap));\n        data.shareMode              = pDevice->playback.shareMode;\n        data.performanceProfile     = pDevice->coreaudio.originalPerformanceProfile;\n        data.registerStopEvent      = (pDevice->type != ma_device_type_duplex);\n\n        if (disposePreviousAudioUnit) {\n            ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n            ((ma_AudioComponentInstanceDispose_proc)pDevice->pContext->coreaudio.AudioComponentInstanceDispose)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n        }\n    }\n    data.periodSizeInFramesIn       = pDevice->coreaudio.originalPeriodSizeInFrames;\n    data.periodSizeInMillisecondsIn = pDevice->coreaudio.originalPeriodSizeInMilliseconds;\n    data.periodsIn                  = pDevice->coreaudio.originalPeriods;\n\n    /* Need at least 3 periods for duplex. */\n    if (data.periodsIn < 3 && pDevice->type == ma_device_type_duplex) {\n        data.periodsIn = 3;\n    }\n\n    result = ma_device_init_internal__coreaudio(pDevice->pContext, deviceType, NULL, &data, (void*)pDevice);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (deviceType == ma_device_type_capture) {\n    #if defined(MA_APPLE_DESKTOP)\n        pDevice->coreaudio.deviceObjectIDCapture     = (ma_uint32)data.deviceObjectID;\n        ma_get_AudioObject_uid(pDevice->pContext, pDevice->coreaudio.deviceObjectIDCapture, sizeof(pDevice->capture.id.coreaudio), pDevice->capture.id.coreaudio);\n    #endif\n        pDevice->coreaudio.audioUnitCapture          = (ma_ptr)data.audioUnit;\n        pDevice->coreaudio.pAudioBufferList          = (ma_ptr)data.pAudioBufferList;\n        pDevice->coreaudio.audioBufferCapInFrames    = data.periodSizeInFramesOut;\n\n        pDevice->capture.internalFormat              = data.formatOut;\n        pDevice->capture.internalChannels            = data.channelsOut;\n        pDevice->capture.internalSampleRate          = data.sampleRateOut;\n        MA_COPY_MEMORY(pDevice->capture.internalChannelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDevice->capture.internalPeriodSizeInFrames  = data.periodSizeInFramesOut;\n        pDevice->capture.internalPeriods             = data.periodsOut;\n    } else if (deviceType == ma_device_type_playback) {\n    #if defined(MA_APPLE_DESKTOP)\n        pDevice->coreaudio.deviceObjectIDPlayback    = (ma_uint32)data.deviceObjectID;\n        ma_get_AudioObject_uid(pDevice->pContext, pDevice->coreaudio.deviceObjectIDPlayback, sizeof(pDevice->playback.id.coreaudio), pDevice->playback.id.coreaudio);\n    #endif\n        pDevice->coreaudio.audioUnitPlayback         = (ma_ptr)data.audioUnit;\n\n        pDevice->playback.internalFormat             = data.formatOut;\n        pDevice->playback.internalChannels           = data.channelsOut;\n        pDevice->playback.internalSampleRate         = data.sampleRateOut;\n        MA_COPY_MEMORY(pDevice->playback.internalChannelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDevice->playback.internalPeriodSizeInFrames = data.periodSizeInFramesOut;\n        pDevice->playback.internalPeriods            = data.periodsOut;\n    }\n\n    return MA_SUCCESS;\n}\n#endif /* MA_APPLE_DESKTOP */\n\nstatic ma_result ma_device_init__coreaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pConfig != NULL);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* No exclusive mode with the Core Audio backend for now. */\n    if (((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    /* Capture needs to be initialized first. */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_device_init_internal_data__coreaudio data;\n        data.allowNominalSampleRateChange = pConfig->coreaudio.allowNominalSampleRateChange;\n        data.formatIn                     = pDescriptorCapture->format;\n        data.channelsIn                   = pDescriptorCapture->channels;\n        data.sampleRateIn                 = pDescriptorCapture->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDescriptorCapture->channelMap, sizeof(pDescriptorCapture->channelMap));\n        data.periodSizeInFramesIn         = pDescriptorCapture->periodSizeInFrames;\n        data.periodSizeInMillisecondsIn   = pDescriptorCapture->periodSizeInMilliseconds;\n        data.periodsIn                    = pDescriptorCapture->periodCount;\n        data.shareMode                    = pDescriptorCapture->shareMode;\n        data.performanceProfile           = pConfig->performanceProfile;\n        data.registerStopEvent            = MA_TRUE;\n\n        /* Need at least 3 periods for duplex. */\n        if (data.periodsIn < 3 && pConfig->deviceType == ma_device_type_duplex) {\n            data.periodsIn = 3;\n        }\n\n        result = ma_device_init_internal__coreaudio(pDevice->pContext, ma_device_type_capture, pDescriptorCapture->pDeviceID, &data, (void*)pDevice);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pDevice->coreaudio.isDefaultCaptureDevice           = (pConfig->capture.pDeviceID == NULL);\n    #if defined(MA_APPLE_DESKTOP)\n        pDevice->coreaudio.deviceObjectIDCapture            = (ma_uint32)data.deviceObjectID;\n    #endif\n        pDevice->coreaudio.audioUnitCapture                 = (ma_ptr)data.audioUnit;\n        pDevice->coreaudio.pAudioBufferList                 = (ma_ptr)data.pAudioBufferList;\n        pDevice->coreaudio.audioBufferCapInFrames           = data.periodSizeInFramesOut;\n        pDevice->coreaudio.originalPeriodSizeInFrames       = pDescriptorCapture->periodSizeInFrames;\n        pDevice->coreaudio.originalPeriodSizeInMilliseconds = pDescriptorCapture->periodSizeInMilliseconds;\n        pDevice->coreaudio.originalPeriods                  = pDescriptorCapture->periodCount;\n        pDevice->coreaudio.originalPerformanceProfile       = pConfig->performanceProfile;\n\n        pDescriptorCapture->format                          = data.formatOut;\n        pDescriptorCapture->channels                        = data.channelsOut;\n        pDescriptorCapture->sampleRate                      = data.sampleRateOut;\n        MA_COPY_MEMORY(pDescriptorCapture->channelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDescriptorCapture->periodSizeInFrames              = data.periodSizeInFramesOut;\n        pDescriptorCapture->periodCount                     = data.periodsOut;\n\n    #if defined(MA_APPLE_DESKTOP)\n        ma_get_AudioObject_uid(pDevice->pContext, pDevice->coreaudio.deviceObjectIDCapture, sizeof(pDevice->capture.id.coreaudio), pDevice->capture.id.coreaudio);\n\n        /*\n        If we are using the default device we'll need to listen for changes to the system's default device so we can seamlessly\n        switch the device in the background.\n        */\n        if (pConfig->capture.pDeviceID == NULL) {\n            ma_device__track__coreaudio(pDevice);\n        }\n    #endif\n    }\n\n    /* Playback. */\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_device_init_internal_data__coreaudio data;\n        data.allowNominalSampleRateChange   = pConfig->coreaudio.allowNominalSampleRateChange;\n        data.formatIn                       = pDescriptorPlayback->format;\n        data.channelsIn                     = pDescriptorPlayback->channels;\n        data.sampleRateIn                   = pDescriptorPlayback->sampleRate;\n        MA_COPY_MEMORY(data.channelMapIn, pDescriptorPlayback->channelMap, sizeof(pDescriptorPlayback->channelMap));\n        data.shareMode                      = pDescriptorPlayback->shareMode;\n        data.performanceProfile             = pConfig->performanceProfile;\n\n        /* In full-duplex mode we want the playback buffer to be the same size as the capture buffer. */\n        if (pConfig->deviceType == ma_device_type_duplex) {\n            data.periodSizeInFramesIn       = pDescriptorCapture->periodSizeInFrames;\n            data.periodsIn                  = pDescriptorCapture->periodCount;\n            data.registerStopEvent          = MA_FALSE;\n        } else {\n            data.periodSizeInFramesIn       = pDescriptorPlayback->periodSizeInFrames;\n            data.periodSizeInMillisecondsIn = pDescriptorPlayback->periodSizeInMilliseconds;\n            data.periodsIn                  = pDescriptorPlayback->periodCount;\n            data.registerStopEvent          = MA_TRUE;\n        }\n\n        result = ma_device_init_internal__coreaudio(pDevice->pContext, ma_device_type_playback, pDescriptorPlayback->pDeviceID, &data, (void*)pDevice);\n        if (result != MA_SUCCESS) {\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                ((ma_AudioComponentInstanceDispose_proc)pDevice->pContext->coreaudio.AudioComponentInstanceDispose)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n                if (pDevice->coreaudio.pAudioBufferList) {\n                    ma_free(pDevice->coreaudio.pAudioBufferList, &pDevice->pContext->allocationCallbacks);\n                }\n            }\n            return result;\n        }\n\n        pDevice->coreaudio.isDefaultPlaybackDevice          = (pConfig->playback.pDeviceID == NULL);\n    #if defined(MA_APPLE_DESKTOP)\n        pDevice->coreaudio.deviceObjectIDPlayback           = (ma_uint32)data.deviceObjectID;\n    #endif\n        pDevice->coreaudio.audioUnitPlayback                = (ma_ptr)data.audioUnit;\n        pDevice->coreaudio.originalPeriodSizeInFrames       = pDescriptorPlayback->periodSizeInFrames;\n        pDevice->coreaudio.originalPeriodSizeInMilliseconds = pDescriptorPlayback->periodSizeInMilliseconds;\n        pDevice->coreaudio.originalPeriods                  = pDescriptorPlayback->periodCount;\n        pDevice->coreaudio.originalPerformanceProfile       = pConfig->performanceProfile;\n\n        pDescriptorPlayback->format                         = data.formatOut;\n        pDescriptorPlayback->channels                       = data.channelsOut;\n        pDescriptorPlayback->sampleRate                     = data.sampleRateOut;\n        MA_COPY_MEMORY(pDescriptorPlayback->channelMap, data.channelMapOut, sizeof(data.channelMapOut));\n        pDescriptorPlayback->periodSizeInFrames             = data.periodSizeInFramesOut;\n        pDescriptorPlayback->periodCount                    = data.periodsOut;\n\n    #if defined(MA_APPLE_DESKTOP)\n        ma_get_AudioObject_uid(pDevice->pContext, pDevice->coreaudio.deviceObjectIDPlayback, sizeof(pDevice->playback.id.coreaudio), pDevice->playback.id.coreaudio);\n\n        /*\n        If we are using the default device we'll need to listen for changes to the system's default device so we can seamlessly\n        switch the device in the background.\n        */\n        if (pDescriptorPlayback->pDeviceID == NULL && (pConfig->deviceType != ma_device_type_duplex || pDescriptorCapture->pDeviceID != NULL)) {\n            ma_device__track__coreaudio(pDevice);\n        }\n    #endif\n    }\n\n\n\n    /*\n    When stopping the device, a callback is called on another thread. We need to wait for this callback\n    before returning from ma_device_stop(). This event is used for this.\n    */\n    ma_event_init(&pDevice->coreaudio.stopEvent);\n\n    /*\n    We need to detect when a route has changed so we can update the data conversion pipeline accordingly. This is done\n    differently on non-Desktop Apple platforms.\n    */\n#if defined(MA_APPLE_MOBILE)\n    pDevice->coreaudio.pNotificationHandler = (MA_BRIDGE_RETAINED void*)[[ma_ios_notification_handler alloc] init:pDevice];\n#endif\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_device_start__coreaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        OSStatus status = ((ma_AudioOutputUnitStart_proc)pDevice->pContext->coreaudio.AudioOutputUnitStart)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n        if (status != noErr) {\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        OSStatus status = ((ma_AudioOutputUnitStart_proc)pDevice->pContext->coreaudio.AudioOutputUnitStart)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n        if (status != noErr) {\n            if (pDevice->type == ma_device_type_duplex) {\n                ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n            }\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__coreaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* It's not clear from the documentation whether or not AudioOutputUnitStop() actually drains the device or not. */\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        OSStatus status = ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitCapture);\n        if (status != noErr) {\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        OSStatus status = ((ma_AudioOutputUnitStop_proc)pDevice->pContext->coreaudio.AudioOutputUnitStop)((AudioUnit)pDevice->coreaudio.audioUnitPlayback);\n        if (status != noErr) {\n            return ma_result_from_OSStatus(status);\n        }\n    }\n\n    /* We need to wait for the callback to finish before returning. */\n    ma_event_wait(&pDevice->coreaudio.stopEvent);\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_uninit__coreaudio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_coreaudio);\n\n#if defined(MA_APPLE_MOBILE)\n    if (!pContext->coreaudio.noAudioSessionDeactivate) {\n        if (![[AVAudioSession sharedInstance] setActive:false error:nil]) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"Failed to deactivate audio session.\");\n            return MA_FAILED_TO_INIT_BACKEND;\n        }\n    }\n#endif\n\n#if !defined(MA_NO_RUNTIME_LINKING) && !defined(MA_APPLE_MOBILE)\n    ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit);\n    ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio);\n    ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n#endif\n\n#if !defined(MA_APPLE_MOBILE)\n    ma_context__uninit_device_tracking__coreaudio(pContext);\n#endif\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\n#if defined(MA_APPLE_MOBILE) && defined(__IPHONE_12_0)\nstatic AVAudioSessionCategory ma_to_AVAudioSessionCategory(ma_ios_session_category category)\n{\n    /* The \"default\" and \"none\" categories are treated different and should not be used as an input into this function. */\n    MA_ASSERT(category != ma_ios_session_category_default);\n    MA_ASSERT(category != ma_ios_session_category_none);\n\n    switch (category) {\n        case ma_ios_session_category_ambient:         return AVAudioSessionCategoryAmbient;\n        case ma_ios_session_category_solo_ambient:    return AVAudioSessionCategorySoloAmbient;\n        case ma_ios_session_category_playback:        return AVAudioSessionCategoryPlayback;\n        case ma_ios_session_category_record:          return AVAudioSessionCategoryRecord;\n        case ma_ios_session_category_play_and_record: return AVAudioSessionCategoryPlayAndRecord;\n        case ma_ios_session_category_multi_route:     return AVAudioSessionCategoryMultiRoute;\n        case ma_ios_session_category_none:            return AVAudioSessionCategoryAmbient;\n        case ma_ios_session_category_default:         return AVAudioSessionCategoryAmbient;\n        default:                                      return AVAudioSessionCategoryAmbient;\n    }\n}\n#endif\n\nstatic ma_result ma_context_init__coreaudio(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n#if !defined(MA_APPLE_MOBILE)\n    ma_result result;\n#endif\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pContext != NULL);\n\n#if defined(MA_APPLE_MOBILE)\n    @autoreleasepool {\n        AVAudioSession* pAudioSession = [AVAudioSession sharedInstance];\n        AVAudioSessionCategoryOptions options = pConfig->coreaudio.sessionCategoryOptions;\n\n        MA_ASSERT(pAudioSession != NULL);\n\n        if (pConfig->coreaudio.sessionCategory == ma_ios_session_category_default) {\n            /*\n            I'm going to use trial and error to determine our default session category. First we'll try PlayAndRecord. If that fails\n            we'll try Playback and if that fails we'll try record. If all of these fail we'll just not set the category.\n            */\n        #if !defined(MA_APPLE_TV) && !defined(MA_APPLE_WATCH)\n            options |= AVAudioSessionCategoryOptionDefaultToSpeaker;\n        #endif\n\n            if ([pAudioSession setCategory: AVAudioSessionCategoryPlayAndRecord withOptions:options error:nil]) {\n                /* Using PlayAndRecord */\n            } else if ([pAudioSession setCategory: AVAudioSessionCategoryPlayback withOptions:options error:nil]) {\n                /* Using Playback */\n            } else if ([pAudioSession setCategory: AVAudioSessionCategoryRecord withOptions:options error:nil]) {\n                /* Using Record */\n            } else {\n                /* Leave as default? */\n            }\n        } else {\n            if (pConfig->coreaudio.sessionCategory != ma_ios_session_category_none) {\n            #if defined(__IPHONE_12_0)\n                if (![pAudioSession setCategory: ma_to_AVAudioSessionCategory(pConfig->coreaudio.sessionCategory) withOptions:options error:nil]) {\n                    return MA_INVALID_OPERATION;    /* Failed to set session category. */\n                }\n            #else\n                /* Ignore the session category on version 11 and older, but post a warning. */\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"Session category only supported in iOS 12 and newer.\");\n            #endif\n            }\n        }\n\n        if (!pConfig->coreaudio.noAudioSessionActivate) {\n            if (![pAudioSession setActive:true error:nil]) {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"Failed to activate audio session.\");\n                return MA_FAILED_TO_INIT_BACKEND;\n            }\n        }\n    }\n#endif\n\n#if !defined(MA_NO_RUNTIME_LINKING) && !defined(MA_APPLE_MOBILE)\n    pContext->coreaudio.hCoreFoundation = ma_dlopen(ma_context_get_log(pContext), \"/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation\");\n    if (pContext->coreaudio.hCoreFoundation == NULL) {\n        return MA_API_NOT_FOUND;\n    }\n\n    pContext->coreaudio.CFStringGetCString = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation, \"CFStringGetCString\");\n    pContext->coreaudio.CFRelease          = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation, \"CFRelease\");\n\n\n    pContext->coreaudio.hCoreAudio = ma_dlopen(ma_context_get_log(pContext), \"/System/Library/Frameworks/CoreAudio.framework/CoreAudio\");\n    if (pContext->coreaudio.hCoreAudio == NULL) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n        return MA_API_NOT_FOUND;\n    }\n\n    pContext->coreaudio.AudioObjectGetPropertyData        = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio, \"AudioObjectGetPropertyData\");\n    pContext->coreaudio.AudioObjectGetPropertyDataSize    = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio, \"AudioObjectGetPropertyDataSize\");\n    pContext->coreaudio.AudioObjectSetPropertyData        = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio, \"AudioObjectSetPropertyData\");\n    pContext->coreaudio.AudioObjectAddPropertyListener    = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio, \"AudioObjectAddPropertyListener\");\n    pContext->coreaudio.AudioObjectRemovePropertyListener = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio, \"AudioObjectRemovePropertyListener\");\n\n    /*\n    It looks like Apple has moved some APIs from AudioUnit into AudioToolbox on more recent versions of macOS. They are still\n    defined in AudioUnit, but just in case they decide to remove them from there entirely I'm going to implement a fallback.\n    The way it'll work is that it'll first try AudioUnit, and if the required symbols are not present there we'll fall back to\n    AudioToolbox.\n    */\n    pContext->coreaudio.hAudioUnit = ma_dlopen(ma_context_get_log(pContext), \"/System/Library/Frameworks/AudioUnit.framework/AudioUnit\");\n    if (pContext->coreaudio.hAudioUnit == NULL) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio);\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n        return MA_API_NOT_FOUND;\n    }\n\n    if (ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioComponentFindNext\") == NULL) {\n        /* Couldn't find the required symbols in AudioUnit, so fall back to AudioToolbox. */\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit);\n        pContext->coreaudio.hAudioUnit = ma_dlopen(ma_context_get_log(pContext), \"/System/Library/Frameworks/AudioToolbox.framework/AudioToolbox\");\n        if (pContext->coreaudio.hAudioUnit == NULL) {\n            ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio);\n            ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n            return MA_API_NOT_FOUND;\n        }\n    }\n\n    pContext->coreaudio.AudioComponentFindNext            = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioComponentFindNext\");\n    pContext->coreaudio.AudioComponentInstanceDispose     = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioComponentInstanceDispose\");\n    pContext->coreaudio.AudioComponentInstanceNew         = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioComponentInstanceNew\");\n    pContext->coreaudio.AudioOutputUnitStart              = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioOutputUnitStart\");\n    pContext->coreaudio.AudioOutputUnitStop               = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioOutputUnitStop\");\n    pContext->coreaudio.AudioUnitAddPropertyListener      = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitAddPropertyListener\");\n    pContext->coreaudio.AudioUnitGetPropertyInfo          = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitGetPropertyInfo\");\n    pContext->coreaudio.AudioUnitGetProperty              = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitGetProperty\");\n    pContext->coreaudio.AudioUnitSetProperty              = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitSetProperty\");\n    pContext->coreaudio.AudioUnitInitialize               = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitInitialize\");\n    pContext->coreaudio.AudioUnitRender                   = ma_dlsym(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit, \"AudioUnitRender\");\n#else\n    pContext->coreaudio.CFStringGetCString                = (ma_proc)CFStringGetCString;\n    pContext->coreaudio.CFRelease                         = (ma_proc)CFRelease;\n\n    #if defined(MA_APPLE_DESKTOP)\n    pContext->coreaudio.AudioObjectGetPropertyData        = (ma_proc)AudioObjectGetPropertyData;\n    pContext->coreaudio.AudioObjectGetPropertyDataSize    = (ma_proc)AudioObjectGetPropertyDataSize;\n    pContext->coreaudio.AudioObjectSetPropertyData        = (ma_proc)AudioObjectSetPropertyData;\n    pContext->coreaudio.AudioObjectAddPropertyListener    = (ma_proc)AudioObjectAddPropertyListener;\n    pContext->coreaudio.AudioObjectRemovePropertyListener = (ma_proc)AudioObjectRemovePropertyListener;\n    #endif\n\n    pContext->coreaudio.AudioComponentFindNext            = (ma_proc)AudioComponentFindNext;\n    pContext->coreaudio.AudioComponentInstanceDispose     = (ma_proc)AudioComponentInstanceDispose;\n    pContext->coreaudio.AudioComponentInstanceNew         = (ma_proc)AudioComponentInstanceNew;\n    pContext->coreaudio.AudioOutputUnitStart              = (ma_proc)AudioOutputUnitStart;\n    pContext->coreaudio.AudioOutputUnitStop               = (ma_proc)AudioOutputUnitStop;\n    pContext->coreaudio.AudioUnitAddPropertyListener      = (ma_proc)AudioUnitAddPropertyListener;\n    pContext->coreaudio.AudioUnitGetPropertyInfo          = (ma_proc)AudioUnitGetPropertyInfo;\n    pContext->coreaudio.AudioUnitGetProperty              = (ma_proc)AudioUnitGetProperty;\n    pContext->coreaudio.AudioUnitSetProperty              = (ma_proc)AudioUnitSetProperty;\n    pContext->coreaudio.AudioUnitInitialize               = (ma_proc)AudioUnitInitialize;\n    pContext->coreaudio.AudioUnitRender                   = (ma_proc)AudioUnitRender;\n#endif\n\n    /* Audio component. */\n    {\n        AudioComponentDescription desc;\n        desc.componentType         = kAudioUnitType_Output;\n    #if defined(MA_APPLE_DESKTOP)\n        desc.componentSubType      = kAudioUnitSubType_HALOutput;\n    #else\n        desc.componentSubType      = kAudioUnitSubType_RemoteIO;\n    #endif\n        desc.componentManufacturer = kAudioUnitManufacturer_Apple;\n        desc.componentFlags        = 0;\n        desc.componentFlagsMask    = 0;\n\n        pContext->coreaudio.component = ((ma_AudioComponentFindNext_proc)pContext->coreaudio.AudioComponentFindNext)(NULL, &desc);\n        if (pContext->coreaudio.component == NULL) {\n        #if !defined(MA_NO_RUNTIME_LINKING) && !defined(MA_APPLE_MOBILE)\n            ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit);\n            ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio);\n            ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n        #endif\n            return MA_FAILED_TO_INIT_BACKEND;\n        }\n    }\n\n#if !defined(MA_APPLE_MOBILE)\n    result = ma_context__init_device_tracking__coreaudio(pContext);\n    if (result != MA_SUCCESS) {\n    #if !defined(MA_NO_RUNTIME_LINKING) && !defined(MA_APPLE_MOBILE)\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hAudioUnit);\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreAudio);\n        ma_dlclose(ma_context_get_log(pContext), pContext->coreaudio.hCoreFoundation);\n    #endif\n        return result;\n    }\n#endif\n\n    pContext->coreaudio.noAudioSessionDeactivate = pConfig->coreaudio.noAudioSessionDeactivate;\n\n    pCallbacks->onContextInit             = ma_context_init__coreaudio;\n    pCallbacks->onContextUninit           = ma_context_uninit__coreaudio;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__coreaudio;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__coreaudio;\n    pCallbacks->onDeviceInit              = ma_device_init__coreaudio;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__coreaudio;\n    pCallbacks->onDeviceStart             = ma_device_start__coreaudio;\n    pCallbacks->onDeviceStop              = ma_device_stop__coreaudio;\n    pCallbacks->onDeviceRead              = NULL;\n    pCallbacks->onDeviceWrite             = NULL;\n    pCallbacks->onDeviceDataLoop          = NULL;\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_COREAUDIO */\n\n\n\n/******************************************************************************\n\nsndio Backend\n\n******************************************************************************/\n#ifdef MA_HAS_SNDIO\n#include <fcntl.h>\n\n/*\nOnly supporting OpenBSD. This did not work very well at all on FreeBSD when I tried it. Not sure if this is due\nto miniaudio's implementation or if it's some kind of system configuration issue, but basically the default device\njust doesn't emit any sound, or at times you'll hear tiny pieces. I will consider enabling this when there's\ndemand for it or if I can get it tested and debugged more thoroughly.\n*/\n#if 0\n#if defined(__NetBSD__) || defined(__OpenBSD__)\n#include <sys/audioio.h>\n#endif\n#if defined(__FreeBSD__) || defined(__DragonFly__)\n#include <sys/soundcard.h>\n#endif\n#endif\n\n#define MA_SIO_DEVANY   \"default\"\n#define MA_SIO_PLAY     1\n#define MA_SIO_REC      2\n#define MA_SIO_NENC     8\n#define MA_SIO_NCHAN    8\n#define MA_SIO_NRATE    16\n#define MA_SIO_NCONF    4\n\nstruct ma_sio_hdl; /* <-- Opaque */\n\nstruct ma_sio_par\n{\n    unsigned int bits;\n    unsigned int bps;\n    unsigned int sig;\n    unsigned int le;\n    unsigned int msb;\n    unsigned int rchan;\n    unsigned int pchan;\n    unsigned int rate;\n    unsigned int bufsz;\n    unsigned int xrun;\n    unsigned int round;\n    unsigned int appbufsz;\n    int __pad[3];\n    unsigned int __magic;\n};\n\nstruct ma_sio_enc\n{\n    unsigned int bits;\n    unsigned int bps;\n    unsigned int sig;\n    unsigned int le;\n    unsigned int msb;\n};\n\nstruct ma_sio_conf\n{\n    unsigned int enc;\n    unsigned int rchan;\n    unsigned int pchan;\n    unsigned int rate;\n};\n\nstruct ma_sio_cap\n{\n    struct ma_sio_enc enc[MA_SIO_NENC];\n    unsigned int rchan[MA_SIO_NCHAN];\n    unsigned int pchan[MA_SIO_NCHAN];\n    unsigned int rate[MA_SIO_NRATE];\n    int __pad[7];\n    unsigned int nconf;\n    struct ma_sio_conf confs[MA_SIO_NCONF];\n};\n\ntypedef struct ma_sio_hdl* (* ma_sio_open_proc)   (const char*, unsigned int, int);\ntypedef void               (* ma_sio_close_proc)  (struct ma_sio_hdl*);\ntypedef int                (* ma_sio_setpar_proc) (struct ma_sio_hdl*, struct ma_sio_par*);\ntypedef int                (* ma_sio_getpar_proc) (struct ma_sio_hdl*, struct ma_sio_par*);\ntypedef int                (* ma_sio_getcap_proc) (struct ma_sio_hdl*, struct ma_sio_cap*);\ntypedef size_t             (* ma_sio_write_proc)  (struct ma_sio_hdl*, const void*, size_t);\ntypedef size_t             (* ma_sio_read_proc)   (struct ma_sio_hdl*, void*, size_t);\ntypedef int                (* ma_sio_start_proc)  (struct ma_sio_hdl*);\ntypedef int                (* ma_sio_stop_proc)   (struct ma_sio_hdl*);\ntypedef int                (* ma_sio_initpar_proc)(struct ma_sio_par*);\n\nstatic ma_uint32 ma_get_standard_sample_rate_priority_index__sndio(ma_uint32 sampleRate)   /* Lower = higher priority */\n{\n    ma_uint32 i;\n    for (i = 0; i < ma_countof(g_maStandardSampleRatePriorities); ++i) {\n        if (g_maStandardSampleRatePriorities[i] == sampleRate) {\n            return i;\n        }\n    }\n\n    return (ma_uint32)-1;\n}\n\nstatic ma_format ma_format_from_sio_enc__sndio(unsigned int bits, unsigned int bps, unsigned int sig, unsigned int le, unsigned int msb)\n{\n    /* We only support native-endian right now. */\n    if ((ma_is_little_endian() && le == 0) || (ma_is_big_endian() && le == 1)) {\n        return ma_format_unknown;\n    }\n\n    if (bits ==  8 && bps == 1 && sig == 0) {\n        return ma_format_u8;\n    }\n    if (bits == 16 && bps == 2 && sig == 1) {\n        return ma_format_s16;\n    }\n    if (bits == 24 && bps == 3 && sig == 1) {\n        return ma_format_s24;\n    }\n    if (bits == 24 && bps == 4 && sig == 1 && msb == 0) {\n        /*return ma_format_s24_32;*/\n    }\n    if (bits == 32 && bps == 4 && sig == 1) {\n        return ma_format_s32;\n    }\n\n    return ma_format_unknown;\n}\n\nstatic ma_format ma_find_best_format_from_sio_cap__sndio(struct ma_sio_cap* caps)\n{\n    ma_format bestFormat;\n    unsigned int iConfig;\n\n    MA_ASSERT(caps != NULL);\n\n    bestFormat = ma_format_unknown;\n    for (iConfig = 0; iConfig < caps->nconf; iConfig += 1) {\n        unsigned int iEncoding;\n        for (iEncoding = 0; iEncoding < MA_SIO_NENC; iEncoding += 1) {\n            unsigned int bits;\n            unsigned int bps;\n            unsigned int sig;\n            unsigned int le;\n            unsigned int msb;\n            ma_format format;\n\n            if ((caps->confs[iConfig].enc & (1UL << iEncoding)) == 0) {\n                continue;\n            }\n\n            bits = caps->enc[iEncoding].bits;\n            bps  = caps->enc[iEncoding].bps;\n            sig  = caps->enc[iEncoding].sig;\n            le   = caps->enc[iEncoding].le;\n            msb  = caps->enc[iEncoding].msb;\n            format = ma_format_from_sio_enc__sndio(bits, bps, sig, le, msb);\n            if (format == ma_format_unknown) {\n                continue;   /* Format not supported. */\n            }\n\n            if (bestFormat == ma_format_unknown) {\n                bestFormat = format;\n            } else {\n                if (ma_get_format_priority_index(bestFormat) > ma_get_format_priority_index(format)) {    /* <-- Lower = better. */\n                    bestFormat = format;\n                }\n            }\n        }\n    }\n\n    return bestFormat;\n}\n\nstatic ma_uint32 ma_find_best_channels_from_sio_cap__sndio(struct ma_sio_cap* caps, ma_device_type deviceType, ma_format requiredFormat)\n{\n    ma_uint32 maxChannels;\n    unsigned int iConfig;\n\n    MA_ASSERT(caps != NULL);\n    MA_ASSERT(requiredFormat != ma_format_unknown);\n\n    /* Just pick whatever configuration has the most channels. */\n    maxChannels = 0;\n    for (iConfig = 0; iConfig < caps->nconf; iConfig += 1) {\n        /* The encoding should be of requiredFormat. */\n        unsigned int iEncoding;\n        for (iEncoding = 0; iEncoding < MA_SIO_NENC; iEncoding += 1) {\n            unsigned int iChannel;\n            unsigned int bits;\n            unsigned int bps;\n            unsigned int sig;\n            unsigned int le;\n            unsigned int msb;\n            ma_format format;\n\n            if ((caps->confs[iConfig].enc & (1UL << iEncoding)) == 0) {\n                continue;\n            }\n\n            bits = caps->enc[iEncoding].bits;\n            bps  = caps->enc[iEncoding].bps;\n            sig  = caps->enc[iEncoding].sig;\n            le   = caps->enc[iEncoding].le;\n            msb  = caps->enc[iEncoding].msb;\n            format = ma_format_from_sio_enc__sndio(bits, bps, sig, le, msb);\n            if (format != requiredFormat) {\n                continue;\n            }\n\n            /* Getting here means the format is supported. Iterate over each channel count and grab the biggest one. */\n            for (iChannel = 0; iChannel < MA_SIO_NCHAN; iChannel += 1) {\n                unsigned int chan = 0;\n                unsigned int channels;\n\n                if (deviceType == ma_device_type_playback) {\n                    chan = caps->confs[iConfig].pchan;\n                } else {\n                    chan = caps->confs[iConfig].rchan;\n                }\n\n                if ((chan & (1UL << iChannel)) == 0) {\n                    continue;\n                }\n\n                if (deviceType == ma_device_type_playback) {\n                    channels = caps->pchan[iChannel];\n                } else {\n                    channels = caps->rchan[iChannel];\n                }\n\n                if (maxChannels < channels) {\n                    maxChannels = channels;\n                }\n            }\n        }\n    }\n\n    return maxChannels;\n}\n\nstatic ma_uint32 ma_find_best_sample_rate_from_sio_cap__sndio(struct ma_sio_cap* caps, ma_device_type deviceType, ma_format requiredFormat, ma_uint32 requiredChannels)\n{\n    ma_uint32 firstSampleRate;\n    ma_uint32 bestSampleRate;\n    unsigned int iConfig;\n\n    MA_ASSERT(caps != NULL);\n    MA_ASSERT(requiredFormat != ma_format_unknown);\n    MA_ASSERT(requiredChannels > 0);\n    MA_ASSERT(requiredChannels <= MA_MAX_CHANNELS);\n\n    firstSampleRate = 0; /* <-- If the device does not support a standard rate we'll fall back to the first one that's found. */\n    bestSampleRate  = 0;\n\n    for (iConfig = 0; iConfig < caps->nconf; iConfig += 1) {\n        /* The encoding should be of requiredFormat. */\n        unsigned int iEncoding;\n        for (iEncoding = 0; iEncoding < MA_SIO_NENC; iEncoding += 1) {\n            unsigned int iChannel;\n            unsigned int bits;\n            unsigned int bps;\n            unsigned int sig;\n            unsigned int le;\n            unsigned int msb;\n            ma_format format;\n\n            if ((caps->confs[iConfig].enc & (1UL << iEncoding)) == 0) {\n                continue;\n            }\n\n            bits = caps->enc[iEncoding].bits;\n            bps  = caps->enc[iEncoding].bps;\n            sig  = caps->enc[iEncoding].sig;\n            le   = caps->enc[iEncoding].le;\n            msb  = caps->enc[iEncoding].msb;\n            format = ma_format_from_sio_enc__sndio(bits, bps, sig, le, msb);\n            if (format != requiredFormat) {\n                continue;\n            }\n\n            /* Getting here means the format is supported. Iterate over each channel count and grab the biggest one. */\n            for (iChannel = 0; iChannel < MA_SIO_NCHAN; iChannel += 1) {\n                unsigned int chan = 0;\n                unsigned int channels;\n                unsigned int iRate;\n\n                if (deviceType == ma_device_type_playback) {\n                    chan = caps->confs[iConfig].pchan;\n                } else {\n                    chan = caps->confs[iConfig].rchan;\n                }\n\n                if ((chan & (1UL << iChannel)) == 0) {\n                    continue;\n                }\n\n                if (deviceType == ma_device_type_playback) {\n                    channels = caps->pchan[iChannel];\n                } else {\n                    channels = caps->rchan[iChannel];\n                }\n\n                if (channels != requiredChannels) {\n                    continue;\n                }\n\n                /* Getting here means we have found a compatible encoding/channel pair. */\n                for (iRate = 0; iRate < MA_SIO_NRATE; iRate += 1) {\n                    ma_uint32 rate = (ma_uint32)caps->rate[iRate];\n                    ma_uint32 ratePriority;\n\n                    if (firstSampleRate == 0) {\n                        firstSampleRate = rate;\n                    }\n\n                    /* Disregard this rate if it's not a standard one. */\n                    ratePriority = ma_get_standard_sample_rate_priority_index__sndio(rate);\n                    if (ratePriority == (ma_uint32)-1) {\n                        continue;\n                    }\n\n                    if (ma_get_standard_sample_rate_priority_index__sndio(bestSampleRate) > ratePriority) {   /* Lower = better. */\n                        bestSampleRate = rate;\n                    }\n                }\n            }\n        }\n    }\n\n    /* If a standard sample rate was not found just fall back to the first one that was iterated. */\n    if (bestSampleRate == 0) {\n        bestSampleRate = firstSampleRate;\n    }\n\n    return bestSampleRate;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__sndio(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 isTerminating = MA_FALSE;\n    struct ma_sio_hdl* handle;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* sndio doesn't seem to have a good device enumeration API, so I'm therefore only enumerating over default devices for now. */\n\n    /* Playback. */\n    if (!isTerminating) {\n        handle = ((ma_sio_open_proc)pContext->sndio.sio_open)(MA_SIO_DEVANY, MA_SIO_PLAY, 0);\n        if (handle != NULL) {\n            /* Supports playback. */\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            ma_strcpy_s(deviceInfo.id.sndio, sizeof(deviceInfo.id.sndio), MA_SIO_DEVANY);\n            ma_strcpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME);\n\n            isTerminating = !callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n\n            ((ma_sio_close_proc)pContext->sndio.sio_close)(handle);\n        }\n    }\n\n    /* Capture. */\n    if (!isTerminating) {\n        handle = ((ma_sio_open_proc)pContext->sndio.sio_open)(MA_SIO_DEVANY, MA_SIO_REC, 0);\n        if (handle != NULL) {\n            /* Supports capture. */\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            ma_strcpy_s(deviceInfo.id.sndio, sizeof(deviceInfo.id.sndio), \"default\");\n            ma_strcpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME);\n\n            isTerminating = !callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n\n            ((ma_sio_close_proc)pContext->sndio.sio_close)(handle);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__sndio(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    char devid[256];\n    struct ma_sio_hdl* handle;\n    struct ma_sio_cap caps;\n    unsigned int iConfig;\n\n    MA_ASSERT(pContext != NULL);\n\n    /* We need to open the device before we can get information about it. */\n    if (pDeviceID == NULL) {\n        ma_strcpy_s(devid, sizeof(devid), MA_SIO_DEVANY);\n        ma_strcpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), (deviceType == ma_device_type_playback) ? MA_DEFAULT_PLAYBACK_DEVICE_NAME : MA_DEFAULT_CAPTURE_DEVICE_NAME);\n    } else {\n        ma_strcpy_s(devid, sizeof(devid), pDeviceID->sndio);\n        ma_strcpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), devid);\n    }\n\n    handle = ((ma_sio_open_proc)pContext->sndio.sio_open)(devid, (deviceType == ma_device_type_playback) ? MA_SIO_PLAY : MA_SIO_REC, 0);\n    if (handle == NULL) {\n        return MA_NO_DEVICE;\n    }\n\n    if (((ma_sio_getcap_proc)pContext->sndio.sio_getcap)(handle, &caps) == 0) {\n        return MA_ERROR;\n    }\n\n    pDeviceInfo->nativeDataFormatCount = 0;\n\n    for (iConfig = 0; iConfig < caps.nconf; iConfig += 1) {\n        /*\n        The main thing we care about is that the encoding is supported by miniaudio. If it is, we want to give\n        preference to some formats over others.\n        */\n        unsigned int iEncoding;\n        unsigned int iChannel;\n        unsigned int iRate;\n\n        for (iEncoding = 0; iEncoding < MA_SIO_NENC; iEncoding += 1) {\n            unsigned int bits;\n            unsigned int bps;\n            unsigned int sig;\n            unsigned int le;\n            unsigned int msb;\n            ma_format format;\n\n            if ((caps.confs[iConfig].enc & (1UL << iEncoding)) == 0) {\n                continue;\n            }\n\n            bits = caps.enc[iEncoding].bits;\n            bps  = caps.enc[iEncoding].bps;\n            sig  = caps.enc[iEncoding].sig;\n            le   = caps.enc[iEncoding].le;\n            msb  = caps.enc[iEncoding].msb;\n            format = ma_format_from_sio_enc__sndio(bits, bps, sig, le, msb);\n            if (format == ma_format_unknown) {\n                continue;   /* Format not supported. */\n            }\n\n\n            /* Channels. */\n            for (iChannel = 0; iChannel < MA_SIO_NCHAN; iChannel += 1) {\n                unsigned int chan = 0;\n                unsigned int channels;\n\n                if (deviceType == ma_device_type_playback) {\n                    chan = caps.confs[iConfig].pchan;\n                } else {\n                    chan = caps.confs[iConfig].rchan;\n                }\n\n                if ((chan & (1UL << iChannel)) == 0) {\n                    continue;\n                }\n\n                if (deviceType == ma_device_type_playback) {\n                    channels = caps.pchan[iChannel];\n                } else {\n                    channels = caps.rchan[iChannel];\n                }\n\n\n                /* Sample Rates. */\n                for (iRate = 0; iRate < MA_SIO_NRATE; iRate += 1) {\n                    if ((caps.confs[iConfig].rate & (1UL << iRate)) != 0) {\n                        ma_device_info_add_native_data_format(pDeviceInfo, format, channels, caps.rate[iRate], 0);\n                    }\n                }\n            }\n        }\n    }\n\n    ((ma_sio_close_proc)pContext->sndio.sio_close)(handle);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_uninit__sndio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_close_proc)pDevice->pContext->sndio.sio_close)((struct ma_sio_hdl*)pDevice->sndio.handleCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_close_proc)pDevice->pContext->sndio.sio_close)((struct ma_sio_hdl*)pDevice->sndio.handlePlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init_handle__sndio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptor, ma_device_type deviceType)\n{\n    const char* pDeviceName;\n    ma_ptr handle;\n    int openFlags = 0;\n    struct ma_sio_cap caps;\n    struct ma_sio_par par;\n    const ma_device_id* pDeviceID;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_format internalFormat;\n    ma_uint32 internalChannels;\n    ma_uint32 internalSampleRate;\n    ma_uint32 internalPeriodSizeInFrames;\n    ma_uint32 internalPeriods;\n\n    MA_ASSERT(pConfig    != NULL);\n    MA_ASSERT(deviceType != ma_device_type_duplex);\n    MA_ASSERT(pDevice    != NULL);\n\n    if (deviceType == ma_device_type_capture) {\n        openFlags = MA_SIO_REC;\n    } else {\n        openFlags = MA_SIO_PLAY;\n    }\n\n    pDeviceID  = pDescriptor->pDeviceID;\n    format     = pDescriptor->format;\n    channels   = pDescriptor->channels;\n    sampleRate = pDescriptor->sampleRate;\n\n    pDeviceName = MA_SIO_DEVANY;\n    if (pDeviceID != NULL) {\n        pDeviceName = pDeviceID->sndio;\n    }\n\n    handle = (ma_ptr)((ma_sio_open_proc)pDevice->pContext->sndio.sio_open)(pDeviceName, openFlags, 0);\n    if (handle == NULL) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to open device.\");\n        return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n    }\n\n    /* We need to retrieve the device caps to determine the most appropriate format to use. */\n    if (((ma_sio_getcap_proc)pDevice->pContext->sndio.sio_getcap)((struct ma_sio_hdl*)handle, &caps) == 0) {\n        ((ma_sio_close_proc)pDevice->pContext->sndio.sio_close)((struct ma_sio_hdl*)handle);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to retrieve device caps.\");\n        return MA_ERROR;\n    }\n\n    /*\n    Note: sndio reports a huge range of available channels. This is inconvenient for us because there's no real\n    way, as far as I can tell, to get the _actual_ channel count of the device. I'm therefore restricting this\n    to the requested channels, regardless of whether or not the default channel count is requested.\n\n    For hardware devices, I'm suspecting only a single channel count will be reported and we can safely use the\n    value returned by ma_find_best_channels_from_sio_cap__sndio().\n    */\n    if (deviceType == ma_device_type_capture) {\n        if (format == ma_format_unknown) {\n            format = ma_find_best_format_from_sio_cap__sndio(&caps);\n        }\n\n        if (channels == 0) {\n            if (strlen(pDeviceName) > strlen(\"rsnd/\") && strncmp(pDeviceName, \"rsnd/\", strlen(\"rsnd/\")) == 0) {\n                channels = ma_find_best_channels_from_sio_cap__sndio(&caps, deviceType, format);\n            } else {\n                channels = MA_DEFAULT_CHANNELS;\n            }\n        }\n    } else {\n        if (format == ma_format_unknown) {\n            format = ma_find_best_format_from_sio_cap__sndio(&caps);\n        }\n\n        if (channels == 0) {\n            if (strlen(pDeviceName) > strlen(\"rsnd/\") && strncmp(pDeviceName, \"rsnd/\", strlen(\"rsnd/\")) == 0) {\n                channels = ma_find_best_channels_from_sio_cap__sndio(&caps, deviceType, format);\n            } else {\n                channels = MA_DEFAULT_CHANNELS;\n            }\n        }\n    }\n\n    if (sampleRate == 0) {\n        sampleRate = ma_find_best_sample_rate_from_sio_cap__sndio(&caps, pConfig->deviceType, format, channels);\n    }\n\n\n    ((ma_sio_initpar_proc)pDevice->pContext->sndio.sio_initpar)(&par);\n    par.msb = 0;\n    par.le  = ma_is_little_endian();\n\n    switch (format) {\n        case ma_format_u8:\n        {\n            par.bits = 8;\n            par.bps  = 1;\n            par.sig  = 0;\n        } break;\n\n        case ma_format_s24:\n        {\n            par.bits = 24;\n            par.bps  = 3;\n            par.sig  = 1;\n        } break;\n\n        case ma_format_s32:\n        {\n            par.bits = 32;\n            par.bps  = 4;\n            par.sig  = 1;\n        } break;\n\n        case ma_format_s16:\n        case ma_format_f32:\n        case ma_format_unknown:\n        default:\n        {\n            par.bits = 16;\n            par.bps  = 2;\n            par.sig  = 1;\n        } break;\n    }\n\n    if (deviceType == ma_device_type_capture) {\n        par.rchan = channels;\n    } else {\n        par.pchan = channels;\n    }\n\n    par.rate = sampleRate;\n\n    internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, par.rate, pConfig->performanceProfile);\n\n    par.round    = internalPeriodSizeInFrames;\n    par.appbufsz = par.round * pDescriptor->periodCount;\n\n    if (((ma_sio_setpar_proc)pDevice->pContext->sndio.sio_setpar)((struct ma_sio_hdl*)handle, &par) == 0) {\n        ((ma_sio_close_proc)pDevice->pContext->sndio.sio_close)((struct ma_sio_hdl*)handle);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to set buffer size.\");\n        return MA_ERROR;\n    }\n\n    if (((ma_sio_getpar_proc)pDevice->pContext->sndio.sio_getpar)((struct ma_sio_hdl*)handle, &par) == 0) {\n        ((ma_sio_close_proc)pDevice->pContext->sndio.sio_close)((struct ma_sio_hdl*)handle);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to retrieve buffer size.\");\n        return MA_ERROR;\n    }\n\n    internalFormat             = ma_format_from_sio_enc__sndio(par.bits, par.bps, par.sig, par.le, par.msb);\n    internalChannels           = (deviceType == ma_device_type_capture) ? par.rchan : par.pchan;\n    internalSampleRate         = par.rate;\n    internalPeriods            = par.appbufsz / par.round;\n    internalPeriodSizeInFrames = par.round;\n\n    if (deviceType == ma_device_type_capture) {\n        pDevice->sndio.handleCapture  = handle;\n    } else {\n        pDevice->sndio.handlePlayback = handle;\n    }\n\n    pDescriptor->format             = internalFormat;\n    pDescriptor->channels           = internalChannels;\n    pDescriptor->sampleRate         = internalSampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_sndio, pDescriptor->channelMap, ma_countof(pDescriptor->channelMap), internalChannels);\n    pDescriptor->periodSizeInFrames = internalPeriodSizeInFrames;\n    pDescriptor->periodCount        = internalPeriods;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__sndio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->sndio);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_handle__sndio(pDevice, pConfig, pDescriptorCapture, ma_device_type_capture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_handle__sndio(pDevice, pConfig, pDescriptorPlayback, ma_device_type_playback);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__sndio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_start_proc)pDevice->pContext->sndio.sio_start)((struct ma_sio_hdl*)pDevice->sndio.handleCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_start_proc)pDevice->pContext->sndio.sio_start)((struct ma_sio_hdl*)pDevice->sndio.handlePlayback);   /* <-- Doesn't actually playback until data is written. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__sndio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    From the documentation:\n\n        The sio_stop() function puts the audio subsystem in the same state as before sio_start() is called. It stops recording, drains the play buffer and then\n        stops playback. If samples to play are queued but playback hasn't started yet then playback is forced immediately; playback will actually stop once the\n        buffer is drained. In no case are samples in the play buffer discarded.\n\n    Therefore, sio_stop() performs all of the necessary draining for us.\n    */\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_stop_proc)pDevice->pContext->sndio.sio_stop)((struct ma_sio_hdl*)pDevice->sndio.handleCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ((ma_sio_stop_proc)pDevice->pContext->sndio.sio_stop)((struct ma_sio_hdl*)pDevice->sndio.handlePlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_write__sndio(ma_device* pDevice, const void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    int result;\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    result = ((ma_sio_write_proc)pDevice->pContext->sndio.sio_write)((struct ma_sio_hdl*)pDevice->sndio.handlePlayback, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n    if (result == 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to send data from the client to the device.\");\n        return MA_IO_ERROR;\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_read__sndio(ma_device* pDevice, void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    int result;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    result = ((ma_sio_read_proc)pDevice->pContext->sndio.sio_read)((struct ma_sio_hdl*)pDevice->sndio.handleCapture, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels));\n    if (result == 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[sndio] Failed to read data from the device to be sent to the device.\");\n        return MA_IO_ERROR;\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__sndio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_sndio);\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__sndio(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n#ifndef MA_NO_RUNTIME_LINKING\n    const char* libsndioNames[] = {\n        \"libsndio.so\"\n    };\n    size_t i;\n\n    for (i = 0; i < ma_countof(libsndioNames); ++i) {\n        pContext->sndio.sndioSO = ma_dlopen(ma_context_get_log(pContext), libsndioNames[i]);\n        if (pContext->sndio.sndioSO != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->sndio.sndioSO == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    pContext->sndio.sio_open    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_open\");\n    pContext->sndio.sio_close   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_close\");\n    pContext->sndio.sio_setpar  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_setpar\");\n    pContext->sndio.sio_getpar  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_getpar\");\n    pContext->sndio.sio_getcap  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_getcap\");\n    pContext->sndio.sio_write   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_write\");\n    pContext->sndio.sio_read    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_read\");\n    pContext->sndio.sio_start   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_start\");\n    pContext->sndio.sio_stop    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_stop\");\n    pContext->sndio.sio_initpar = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->sndio.sndioSO, \"sio_initpar\");\n#else\n    pContext->sndio.sio_open    = sio_open;\n    pContext->sndio.sio_close   = sio_close;\n    pContext->sndio.sio_setpar  = sio_setpar;\n    pContext->sndio.sio_getpar  = sio_getpar;\n    pContext->sndio.sio_getcap  = sio_getcap;\n    pContext->sndio.sio_write   = sio_write;\n    pContext->sndio.sio_read    = sio_read;\n    pContext->sndio.sio_start   = sio_start;\n    pContext->sndio.sio_stop    = sio_stop;\n    pContext->sndio.sio_initpar = sio_initpar;\n#endif\n\n    pCallbacks->onContextInit             = ma_context_init__sndio;\n    pCallbacks->onContextUninit           = ma_context_uninit__sndio;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__sndio;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__sndio;\n    pCallbacks->onDeviceInit              = ma_device_init__sndio;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__sndio;\n    pCallbacks->onDeviceStart             = ma_device_start__sndio;\n    pCallbacks->onDeviceStop              = ma_device_stop__sndio;\n    pCallbacks->onDeviceRead              = ma_device_read__sndio;\n    pCallbacks->onDeviceWrite             = ma_device_write__sndio;\n    pCallbacks->onDeviceDataLoop          = NULL;\n\n    (void)pConfig;\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_SNDIO */\n\n\n\n/******************************************************************************\n\naudio(4) Backend\n\n******************************************************************************/\n#ifdef MA_HAS_AUDIO4\n#include <fcntl.h>\n#include <poll.h>\n#include <errno.h>\n#include <sys/stat.h>\n#include <sys/types.h>\n#include <sys/ioctl.h>\n#include <sys/audioio.h>\n\n#ifdef __NetBSD__\n#include <sys/param.h>\n#endif\n\n#if defined(__OpenBSD__)\n    #include <sys/param.h>\n    #if defined(OpenBSD) && OpenBSD >= 201709\n        #define MA_AUDIO4_USE_NEW_API\n    #endif\n#endif\n\nstatic void ma_construct_device_id__audio4(char* id, size_t idSize, const char* base, int deviceIndex)\n{\n    size_t baseLen;\n\n    MA_ASSERT(id != NULL);\n    MA_ASSERT(idSize > 0);\n    MA_ASSERT(deviceIndex >= 0);\n\n    baseLen = strlen(base);\n    MA_ASSERT(idSize > baseLen);\n\n    ma_strcpy_s(id, idSize, base);\n    ma_itoa_s(deviceIndex, id+baseLen, idSize-baseLen, 10);\n}\n\nstatic ma_result ma_extract_device_index_from_id__audio4(const char* id, const char* base, int* pIndexOut)\n{\n    size_t idLen;\n    size_t baseLen;\n    const char* deviceIndexStr;\n\n    MA_ASSERT(id != NULL);\n    MA_ASSERT(base != NULL);\n    MA_ASSERT(pIndexOut != NULL);\n\n    idLen = strlen(id);\n    baseLen = strlen(base);\n    if (idLen <= baseLen) {\n        return MA_ERROR;   /* Doesn't look like the id starts with the base. */\n    }\n\n    if (strncmp(id, base, baseLen) != 0) {\n        return MA_ERROR;   /* ID does not begin with base. */\n    }\n\n    deviceIndexStr = id + baseLen;\n    if (deviceIndexStr[0] == '\\0') {\n        return MA_ERROR;   /* No index specified in the ID. */\n    }\n\n    if (pIndexOut) {\n        *pIndexOut = atoi(deviceIndexStr);\n    }\n\n    return MA_SUCCESS;\n}\n\n\n#if !defined(MA_AUDIO4_USE_NEW_API)    /* Old API */\nstatic ma_format ma_format_from_encoding__audio4(unsigned int encoding, unsigned int precision)\n{\n    if (precision == 8 && (encoding == AUDIO_ENCODING_ULINEAR || encoding == AUDIO_ENCODING_ULINEAR || encoding == AUDIO_ENCODING_ULINEAR_LE || encoding == AUDIO_ENCODING_ULINEAR_BE)) {\n        return ma_format_u8;\n    } else {\n        if (ma_is_little_endian() && encoding == AUDIO_ENCODING_SLINEAR_LE) {\n            if (precision == 16) {\n                return ma_format_s16;\n            } else if (precision == 24) {\n                return ma_format_s24;\n            } else if (precision == 32) {\n                return ma_format_s32;\n            }\n        } else if (ma_is_big_endian() && encoding == AUDIO_ENCODING_SLINEAR_BE) {\n            if (precision == 16) {\n                return ma_format_s16;\n            } else if (precision == 24) {\n                return ma_format_s24;\n            } else if (precision == 32) {\n                return ma_format_s32;\n            }\n        }\n    }\n\n    return ma_format_unknown;  /* Encoding not supported. */\n}\n\nstatic void ma_encoding_from_format__audio4(ma_format format, unsigned int* pEncoding, unsigned int* pPrecision)\n{\n    MA_ASSERT(pEncoding  != NULL);\n    MA_ASSERT(pPrecision != NULL);\n\n    switch (format)\n    {\n        case ma_format_u8:\n        {\n            *pEncoding = AUDIO_ENCODING_ULINEAR;\n            *pPrecision = 8;\n        } break;\n\n        case ma_format_s24:\n        {\n            *pEncoding = (ma_is_little_endian()) ? AUDIO_ENCODING_SLINEAR_LE : AUDIO_ENCODING_SLINEAR_BE;\n            *pPrecision = 24;\n        } break;\n\n        case ma_format_s32:\n        {\n            *pEncoding = (ma_is_little_endian()) ? AUDIO_ENCODING_SLINEAR_LE : AUDIO_ENCODING_SLINEAR_BE;\n            *pPrecision = 32;\n        } break;\n\n        case ma_format_s16:\n        case ma_format_f32:\n        case ma_format_unknown:\n        default:\n        {\n            *pEncoding = (ma_is_little_endian()) ? AUDIO_ENCODING_SLINEAR_LE : AUDIO_ENCODING_SLINEAR_BE;\n            *pPrecision = 16;\n        } break;\n    }\n}\n\nstatic ma_format ma_format_from_prinfo__audio4(struct audio_prinfo* prinfo)\n{\n    return ma_format_from_encoding__audio4(prinfo->encoding, prinfo->precision);\n}\n\nstatic ma_format ma_best_format_from_fd__audio4(int fd, ma_format preferredFormat)\n{\n    audio_encoding_t encoding;\n    ma_uint32 iFormat;\n    int counter = 0;\n\n    /* First check to see if the preferred format is supported. */\n    if (preferredFormat != ma_format_unknown) {\n        counter = 0;\n        for (;;) {\n            MA_ZERO_OBJECT(&encoding);\n            encoding.index = counter;\n            if (ioctl(fd, AUDIO_GETENC, &encoding) < 0) {\n                break;\n            }\n\n            if (preferredFormat == ma_format_from_encoding__audio4(encoding.encoding, encoding.precision)) {\n                return preferredFormat;  /* Found the preferred format. */\n            }\n\n            /* Getting here means this encoding does not match our preferred format so we need to more on to the next encoding. */\n            counter += 1;\n        }\n    }\n\n    /* Getting here means our preferred format is not supported, so fall back to our standard priorities. */\n    for (iFormat = 0; iFormat < ma_countof(g_maFormatPriorities); iFormat += 1) {\n        ma_format format = g_maFormatPriorities[iFormat];\n\n        counter = 0;\n        for (;;) {\n            MA_ZERO_OBJECT(&encoding);\n            encoding.index = counter;\n            if (ioctl(fd, AUDIO_GETENC, &encoding) < 0) {\n                break;\n            }\n\n            if (format == ma_format_from_encoding__audio4(encoding.encoding, encoding.precision)) {\n                return format;  /* Found a workable format. */\n            }\n\n            /* Getting here means this encoding does not match our preferred format so we need to more on to the next encoding. */\n            counter += 1;\n        }\n    }\n\n    /* Getting here means not appropriate format was found. */\n    return ma_format_unknown;\n}\n#else\nstatic ma_format ma_format_from_swpar__audio4(struct audio_swpar* par)\n{\n    if (par->bits == 8 && par->bps == 1 && par->sig == 0) {\n        return ma_format_u8;\n    }\n    if (par->bits == 16 && par->bps == 2 && par->sig == 1 && par->le == ma_is_little_endian()) {\n        return ma_format_s16;\n    }\n    if (par->bits == 24 && par->bps == 3 && par->sig == 1 && par->le == ma_is_little_endian()) {\n        return ma_format_s24;\n    }\n    if (par->bits == 32 && par->bps == 4 && par->sig == 1 && par->le == ma_is_little_endian()) {\n        return ma_format_f32;\n    }\n\n    /* Format not supported. */\n    return ma_format_unknown;\n}\n#endif\n\nstatic ma_result ma_context_get_device_info_from_fd__audio4(ma_context* pContext, ma_device_type deviceType, int fd, ma_device_info* pDeviceInfo)\n{\n    audio_device_t fdDevice;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(fd >= 0);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    (void)pContext;\n    (void)deviceType;\n\n    if (ioctl(fd, AUDIO_GETDEV, &fdDevice) < 0) {\n        return MA_ERROR;   /* Failed to retrieve device info. */\n    }\n\n    /* Name. */\n    ma_strcpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), fdDevice.name);\n\n    #if !defined(MA_AUDIO4_USE_NEW_API)\n    {\n        audio_info_t fdInfo;\n        int counter = 0;\n        ma_uint32 channels;\n        ma_uint32 sampleRate;\n\n#if defined(__NetBSD__) && (__NetBSD_Version__ >= 900000000)\n        if (ioctl(fd, AUDIO_GETFORMAT, &fdInfo) < 0) {\n            return MA_ERROR;\n        }\n#else\n        if (ioctl(fd, AUDIO_GETINFO, &fdInfo) < 0) {\n            return MA_ERROR;\n        }\n#endif\n\n        if (deviceType == ma_device_type_playback) {\n            channels   = fdInfo.play.channels;\n            sampleRate = fdInfo.play.sample_rate;\n        } else {\n            channels   = fdInfo.record.channels;\n            sampleRate = fdInfo.record.sample_rate;\n        }\n\n        /* Supported formats. We get this by looking at the encodings. */\n        pDeviceInfo->nativeDataFormatCount = 0;\n        for (;;) {\n            audio_encoding_t encoding;\n            ma_format format;\n\n            MA_ZERO_OBJECT(&encoding);\n            encoding.index = counter;\n            if (ioctl(fd, AUDIO_GETENC, &encoding) < 0) {\n                break;\n            }\n\n            format = ma_format_from_encoding__audio4(encoding.encoding, encoding.precision);\n            if (format != ma_format_unknown) {\n                ma_device_info_add_native_data_format(pDeviceInfo, format, channels, sampleRate, 0);\n            }\n\n            counter += 1;\n        }\n    }\n    #else\n    {\n        struct audio_swpar fdPar;\n        ma_format format;\n        ma_uint32 channels;\n        ma_uint32 sampleRate;\n\n        if (ioctl(fd, AUDIO_GETPAR, &fdPar) < 0) {\n            return MA_ERROR;\n        }\n\n        format = ma_format_from_swpar__audio4(&fdPar);\n        if (format == ma_format_unknown) {\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        if (deviceType == ma_device_type_playback) {\n            channels = fdPar.pchan;\n        } else {\n            channels = fdPar.rchan;\n        }\n\n        sampleRate = fdPar.rate;\n\n        pDeviceInfo->nativeDataFormatCount = 0;\n        ma_device_info_add_native_data_format(pDeviceInfo, format, channels, sampleRate, 0);\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_enumerate_devices__audio4(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    const int maxDevices = 64;\n    char devpath[256];\n    int iDevice;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /*\n    Every device will be named \"/dev/audioN\", with a \"/dev/audioctlN\" equivalent. We use the \"/dev/audioctlN\"\n    version here since we can open it even when another process has control of the \"/dev/audioN\" device.\n    */\n    for (iDevice = 0; iDevice < maxDevices; ++iDevice) {\n        struct stat st;\n        int fd;\n        ma_bool32 isTerminating = MA_FALSE;\n\n        ma_strcpy_s(devpath, sizeof(devpath), \"/dev/audioctl\");\n        ma_itoa_s(iDevice, devpath+strlen(devpath), sizeof(devpath)-strlen(devpath), 10);\n\n        if (stat(devpath, &st) < 0) {\n            break;\n        }\n\n        /* The device exists, but we need to check if it's usable as playback and/or capture. */\n\n        /* Playback. */\n        if (!isTerminating) {\n            fd = open(devpath, O_RDONLY, 0);\n            if (fd >= 0) {\n                /* Supports playback. */\n                ma_device_info deviceInfo;\n                MA_ZERO_OBJECT(&deviceInfo);\n                ma_construct_device_id__audio4(deviceInfo.id.audio4, sizeof(deviceInfo.id.audio4), \"/dev/audio\", iDevice);\n                if (ma_context_get_device_info_from_fd__audio4(pContext, ma_device_type_playback, fd, &deviceInfo) == MA_SUCCESS) {\n                    isTerminating = !callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n                }\n\n                close(fd);\n            }\n        }\n\n        /* Capture. */\n        if (!isTerminating) {\n            fd = open(devpath, O_WRONLY, 0);\n            if (fd >= 0) {\n                /* Supports capture. */\n                ma_device_info deviceInfo;\n                MA_ZERO_OBJECT(&deviceInfo);\n                ma_construct_device_id__audio4(deviceInfo.id.audio4, sizeof(deviceInfo.id.audio4), \"/dev/audio\", iDevice);\n                if (ma_context_get_device_info_from_fd__audio4(pContext, ma_device_type_capture, fd, &deviceInfo) == MA_SUCCESS) {\n                    isTerminating = !callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n                }\n\n                close(fd);\n            }\n        }\n\n        if (isTerminating) {\n            break;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__audio4(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    int fd = -1;\n    int deviceIndex = -1;\n    char ctlid[256];\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n    /*\n    We need to open the \"/dev/audioctlN\" device to get the info. To do this we need to extract the number\n    from the device ID which will be in \"/dev/audioN\" format.\n    */\n    if (pDeviceID == NULL) {\n        /* Default device. */\n        ma_strcpy_s(ctlid, sizeof(ctlid), \"/dev/audioctl\");\n    } else {\n        /* Specific device. We need to convert from \"/dev/audioN\" to \"/dev/audioctlN\". */\n        result = ma_extract_device_index_from_id__audio4(pDeviceID->audio4, \"/dev/audio\", &deviceIndex);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        ma_construct_device_id__audio4(ctlid, sizeof(ctlid), \"/dev/audioctl\", deviceIndex);\n    }\n\n    fd = open(ctlid, (deviceType == ma_device_type_playback) ? O_WRONLY : O_RDONLY, 0);\n    if (fd == -1) {\n        return MA_NO_DEVICE;\n    }\n\n    if (deviceIndex == -1) {\n        ma_strcpy_s(pDeviceInfo->id.audio4, sizeof(pDeviceInfo->id.audio4), \"/dev/audio\");\n    } else {\n        ma_construct_device_id__audio4(pDeviceInfo->id.audio4, sizeof(pDeviceInfo->id.audio4), \"/dev/audio\", deviceIndex);\n    }\n\n    result = ma_context_get_device_info_from_fd__audio4(pContext, deviceType, fd, pDeviceInfo);\n\n    close(fd);\n    return result;\n}\n\nstatic ma_result ma_device_uninit__audio4(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        close(pDevice->audio4.fdCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        close(pDevice->audio4.fdPlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init_fd__audio4(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptor, ma_device_type deviceType)\n{\n    const char* pDefaultDeviceNames[] = {\n        \"/dev/audio\",\n        \"/dev/audio0\"\n    };\n    const char* pDefaultDeviceCtlNames[] = {\n        \"/dev/audioctl\",\n        \"/dev/audioctl0\"\n    };\n    int fd;\n    int fdFlags = 0;\n    size_t iDefaultDevice = (size_t)-1;\n    ma_format internalFormat;\n    ma_uint32 internalChannels;\n    ma_uint32 internalSampleRate;\n    ma_uint32 internalPeriodSizeInFrames;\n    ma_uint32 internalPeriods;\n\n    MA_ASSERT(pConfig    != NULL);\n    MA_ASSERT(deviceType != ma_device_type_duplex);\n    MA_ASSERT(pDevice    != NULL);\n\n    /* The first thing to do is open the file. */\n    if (deviceType == ma_device_type_capture) {\n        fdFlags = O_RDONLY;\n    } else {\n        fdFlags = O_WRONLY;\n    }\n    /*fdFlags |= O_NONBLOCK;*/\n\n    /* Find the index of the default device as a start. We'll use this index later. Set it to (size_t)-1 otherwise. */\n    if (pDescriptor->pDeviceID == NULL) {\n        /* Default device. */\n        for (iDefaultDevice = 0; iDefaultDevice < ma_countof(pDefaultDeviceNames); ++iDefaultDevice) {\n            fd = open(pDefaultDeviceNames[iDefaultDevice], fdFlags, 0);\n            if (fd != -1) {\n                break;\n            }\n        }\n    } else {\n        /* Specific device. */\n        fd = open(pDescriptor->pDeviceID->audio4, fdFlags, 0);\n\n        for (iDefaultDevice = 0; iDefaultDevice < ma_countof(pDefaultDeviceNames); iDefaultDevice += 1) {\n            if (ma_strcmp(pDefaultDeviceNames[iDefaultDevice], pDescriptor->pDeviceID->audio4) == 0) {\n                break;\n            }\n        }\n\n        if (iDefaultDevice == ma_countof(pDefaultDeviceNames)) {\n            iDefaultDevice = (size_t)-1;\n        }\n    }\n\n    if (fd == -1) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to open device.\");\n        return ma_result_from_errno(errno);\n    }\n\n    #if !defined(MA_AUDIO4_USE_NEW_API)    /* Old API */\n    {\n        audio_info_t fdInfo;\n        int fdInfoResult = -1;\n\n        /*\n        The documentation is a little bit unclear to me as to how it handles formats. It says the\n        following:\n\n            Regardless of formats supported by underlying driver, the audio driver accepts the\n            following formats.\n\n        By then the next sentence says this:\n\n            `encoding` and `precision` are one of the values obtained by AUDIO_GETENC.\n\n        It sounds like a direct contradiction to me. I'm going to play this safe any only use the\n        best sample format returned by AUDIO_GETENC. If the requested format is supported we'll\n        use that, but otherwise we'll just use our standard format priorities to pick an\n        appropriate one.\n        */\n        AUDIO_INITINFO(&fdInfo);\n\n        /*\n        Get the default format from the audioctl file if we're asking for a default device. If we\n        retrieve it from /dev/audio it'll default to mono 8000Hz.\n        */\n        if (iDefaultDevice != (size_t)-1) {\n            /* We're using a default device. Get the info from the /dev/audioctl file instead of /dev/audio. */\n            int fdctl = open(pDefaultDeviceCtlNames[iDefaultDevice], fdFlags, 0);\n            if (fdctl != -1) {\n#if defined(__NetBSD__) && (__NetBSD_Version__ >= 900000000)\n                fdInfoResult = ioctl(fdctl, AUDIO_GETFORMAT, &fdInfo);\n#else\n                fdInfoResult = ioctl(fdctl, AUDIO_GETINFO, &fdInfo);\n#endif\n                close(fdctl);\n            }\n        }\n\n        if (fdInfoResult == -1) {\n            /* We still don't have the default device info so just retrieve it from the main audio device. */\n            if (ioctl(fd, AUDIO_GETINFO, &fdInfo) < 0) {\n                close(fd);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] AUDIO_GETINFO failed.\");\n                return ma_result_from_errno(errno);\n            }\n        }\n\n        /* We get the driver to do as much of the data conversion as possible. */\n        if (deviceType == ma_device_type_capture) {\n            fdInfo.mode = AUMODE_RECORD;\n            ma_encoding_from_format__audio4(ma_best_format_from_fd__audio4(fd, pDescriptor->format), &fdInfo.record.encoding, &fdInfo.record.precision);\n\n            if (pDescriptor->channels != 0) {\n                fdInfo.record.channels = ma_clamp(pDescriptor->channels, 1, 12);    /* From the documentation: `channels` ranges from 1 to 12. */\n            }\n\n            if (pDescriptor->sampleRate != 0) {\n                fdInfo.record.sample_rate = ma_clamp(pDescriptor->sampleRate, 1000, 192000);    /* From the documentation: `frequency` ranges from 1000Hz to 192000Hz. (They mean `sample_rate` instead of `frequency`.) */\n            }\n        } else {\n            fdInfo.mode = AUMODE_PLAY;\n            ma_encoding_from_format__audio4(ma_best_format_from_fd__audio4(fd, pDescriptor->format), &fdInfo.play.encoding, &fdInfo.play.precision);\n\n            if (pDescriptor->channels != 0) {\n                fdInfo.play.channels = ma_clamp(pDescriptor->channels, 1, 12);    /* From the documentation: `channels` ranges from 1 to 12. */\n            }\n\n            if (pDescriptor->sampleRate != 0) {\n                fdInfo.play.sample_rate = ma_clamp(pDescriptor->sampleRate, 1000, 192000);    /* From the documentation: `frequency` ranges from 1000Hz to 192000Hz. (They mean `sample_rate` instead of `frequency`.) */\n            }\n        }\n\n        if (ioctl(fd, AUDIO_SETINFO, &fdInfo) < 0) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to set device format. AUDIO_SETINFO failed.\");\n            return ma_result_from_errno(errno);\n        }\n\n        if (ioctl(fd, AUDIO_GETINFO, &fdInfo) < 0) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] AUDIO_GETINFO failed.\");\n            return ma_result_from_errno(errno);\n        }\n\n        if (deviceType == ma_device_type_capture) {\n            internalFormat     = ma_format_from_prinfo__audio4(&fdInfo.record);\n            internalChannels   = fdInfo.record.channels;\n            internalSampleRate = fdInfo.record.sample_rate;\n        } else {\n            internalFormat     = ma_format_from_prinfo__audio4(&fdInfo.play);\n            internalChannels   = fdInfo.play.channels;\n            internalSampleRate = fdInfo.play.sample_rate;\n        }\n\n        if (internalFormat == ma_format_unknown) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] The device's internal device format is not supported by miniaudio. The device is unusable.\");\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        /* Buffer. */\n        {\n            ma_uint32 internalPeriodSizeInBytes;\n\n            internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, internalSampleRate, pConfig->performanceProfile);\n\n            internalPeriodSizeInBytes = internalPeriodSizeInFrames * ma_get_bytes_per_frame(internalFormat, internalChannels);\n            if (internalPeriodSizeInBytes < 16) {\n                internalPeriodSizeInBytes = 16;\n            }\n\n            internalPeriods = pDescriptor->periodCount;\n            if (internalPeriods < 2) {\n                internalPeriods = 2;\n            }\n\n            /* What miniaudio calls a period, audio4 calls a block. */\n            AUDIO_INITINFO(&fdInfo);\n            fdInfo.hiwat     = internalPeriods;\n            fdInfo.lowat     = internalPeriods-1;\n            fdInfo.blocksize = internalPeriodSizeInBytes;\n            if (ioctl(fd, AUDIO_SETINFO, &fdInfo) < 0) {\n                close(fd);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to set internal buffer size. AUDIO_SETINFO failed.\");\n                return ma_result_from_errno(errno);\n            }\n\n            internalPeriods            = fdInfo.hiwat;\n            internalPeriodSizeInFrames = fdInfo.blocksize / ma_get_bytes_per_frame(internalFormat, internalChannels);\n        }\n    }\n    #else\n    {\n        struct audio_swpar fdPar;\n\n        /* We need to retrieve the format of the device so we can know the channel count and sample rate. Then we can calculate the buffer size. */\n        if (ioctl(fd, AUDIO_GETPAR, &fdPar) < 0) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to retrieve initial device parameters.\");\n            return ma_result_from_errno(errno);\n        }\n\n        internalFormat     = ma_format_from_swpar__audio4(&fdPar);\n        internalChannels   = (deviceType == ma_device_type_capture) ? fdPar.rchan : fdPar.pchan;\n        internalSampleRate = fdPar.rate;\n\n        if (internalFormat == ma_format_unknown) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] The device's internal device format is not supported by miniaudio. The device is unusable.\");\n            return MA_FORMAT_NOT_SUPPORTED;\n        }\n\n        /* Buffer. */\n        {\n            ma_uint32 internalPeriodSizeInBytes;\n\n            internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, internalSampleRate, pConfig->performanceProfile);\n\n            /* What miniaudio calls a period, audio4 calls a block. */\n            internalPeriodSizeInBytes = internalPeriodSizeInFrames * ma_get_bytes_per_frame(internalFormat, internalChannels);\n            if (internalPeriodSizeInBytes < 16) {\n                internalPeriodSizeInBytes = 16;\n            }\n\n            fdPar.nblks = pDescriptor->periodCount;\n            fdPar.round = internalPeriodSizeInBytes;\n\n            if (ioctl(fd, AUDIO_SETPAR, &fdPar) < 0) {\n                close(fd);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to set device parameters.\");\n                return ma_result_from_errno(errno);\n            }\n\n            if (ioctl(fd, AUDIO_GETPAR, &fdPar) < 0) {\n                close(fd);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to retrieve actual device parameters.\");\n                return ma_result_from_errno(errno);\n            }\n        }\n\n        internalFormat             = ma_format_from_swpar__audio4(&fdPar);\n        internalChannels           = (deviceType == ma_device_type_capture) ? fdPar.rchan : fdPar.pchan;\n        internalSampleRate         = fdPar.rate;\n        internalPeriods            = fdPar.nblks;\n        internalPeriodSizeInFrames = fdPar.round / ma_get_bytes_per_frame(internalFormat, internalChannels);\n    }\n    #endif\n\n    if (internalFormat == ma_format_unknown) {\n        close(fd);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] The device's internal device format is not supported by miniaudio. The device is unusable.\");\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    if (deviceType == ma_device_type_capture) {\n        pDevice->audio4.fdCapture  = fd;\n    } else {\n        pDevice->audio4.fdPlayback = fd;\n    }\n\n    pDescriptor->format             = internalFormat;\n    pDescriptor->channels           = internalChannels;\n    pDescriptor->sampleRate         = internalSampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_sound4, pDescriptor->channelMap, ma_countof(pDescriptor->channelMap), internalChannels);\n    pDescriptor->periodSizeInFrames = internalPeriodSizeInFrames;\n    pDescriptor->periodCount        = internalPeriods;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__audio4(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->audio4);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    pDevice->audio4.fdCapture  = -1;\n    pDevice->audio4.fdPlayback = -1;\n\n    /*\n    The version of the operating system dictates whether or not the device is exclusive or shared. NetBSD\n    introduced in-kernel mixing which means it's shared. All other BSD flavours are exclusive as far as\n    I'm aware.\n    */\n#if defined(__NetBSD_Version__) && __NetBSD_Version__ >= 800000000\n    /* NetBSD 8.0+ */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n#else\n    /* All other flavors. */\n#endif\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_fd__audio4(pDevice, pConfig, pDescriptorCapture, ma_device_type_capture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_fd__audio4(pDevice, pConfig, pDescriptorPlayback, ma_device_type_playback);\n        if (result != MA_SUCCESS) {\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                close(pDevice->audio4.fdCapture);\n            }\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__audio4(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->audio4.fdCapture == -1) {\n            return MA_INVALID_ARGS;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->audio4.fdPlayback == -1) {\n            return MA_INVALID_ARGS;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop_fd__audio4(ma_device* pDevice, int fd)\n{\n    if (fd == -1) {\n        return MA_INVALID_ARGS;\n    }\n\n#if !defined(MA_AUDIO4_USE_NEW_API)\n    if (ioctl(fd, AUDIO_FLUSH, 0) < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to stop device. AUDIO_FLUSH failed.\");\n        return ma_result_from_errno(errno);\n    }\n#else\n    if (ioctl(fd, AUDIO_STOP, 0) < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to stop device. AUDIO_STOP failed.\");\n        return ma_result_from_errno(errno);\n    }\n#endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__audio4(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_result result;\n\n        result = ma_device_stop_fd__audio4(pDevice, pDevice->audio4.fdCapture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_result result;\n\n        /* Drain the device first. If this fails we'll just need to flush without draining. Unfortunately draining isn't available on newer version of OpenBSD. */\n    #if !defined(MA_AUDIO4_USE_NEW_API)\n        ioctl(pDevice->audio4.fdPlayback, AUDIO_DRAIN, 0);\n    #endif\n\n        /* Here is where the device is stopped immediately. */\n        result = ma_device_stop_fd__audio4(pDevice, pDevice->audio4.fdPlayback);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_write__audio4(ma_device* pDevice, const void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    int result;\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    result = write(pDevice->audio4.fdPlayback, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n    if (result < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to write data to the device.\");\n        return ma_result_from_errno(errno);\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = (ma_uint32)result / ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_read__audio4(ma_device* pDevice, void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    int result;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    result = read(pDevice->audio4.fdCapture, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels));\n    if (result < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[audio4] Failed to read data from the device.\");\n        return ma_result_from_errno(errno);\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = (ma_uint32)result / ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__audio4(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_audio4);\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__audio4(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n    pCallbacks->onContextInit             = ma_context_init__audio4;\n    pCallbacks->onContextUninit           = ma_context_uninit__audio4;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__audio4;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__audio4;\n    pCallbacks->onDeviceInit              = ma_device_init__audio4;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__audio4;\n    pCallbacks->onDeviceStart             = ma_device_start__audio4;\n    pCallbacks->onDeviceStop              = ma_device_stop__audio4;\n    pCallbacks->onDeviceRead              = ma_device_read__audio4;\n    pCallbacks->onDeviceWrite             = ma_device_write__audio4;\n    pCallbacks->onDeviceDataLoop          = NULL;\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_AUDIO4 */\n\n\n/******************************************************************************\n\nOSS Backend\n\n******************************************************************************/\n#ifdef MA_HAS_OSS\n#include <sys/ioctl.h>\n#include <unistd.h>\n#include <fcntl.h>\n#include <sys/soundcard.h>\n\n#ifndef SNDCTL_DSP_HALT\n#define SNDCTL_DSP_HALT SNDCTL_DSP_RESET\n#endif\n\n#define MA_OSS_DEFAULT_DEVICE_NAME  \"/dev/dsp\"\n\nstatic int ma_open_temp_device__oss(void)\n{\n    /* The OSS sample code uses \"/dev/mixer\" as the device for getting system properties so I'm going to do the same. */\n    int fd = open(\"/dev/mixer\", O_RDONLY, 0);\n    if (fd >= 0) {\n        return fd;\n    }\n\n    return -1;\n}\n\nstatic ma_result ma_context_open_device__oss(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_share_mode shareMode, int* pfd)\n{\n    const char* deviceName;\n    int flags;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pfd != NULL);\n    (void)pContext;\n\n    *pfd = -1;\n\n    /* This function should only be called for playback or capture, not duplex. */\n    if (deviceType == ma_device_type_duplex) {\n        return MA_INVALID_ARGS;\n    }\n\n    deviceName = MA_OSS_DEFAULT_DEVICE_NAME;\n    if (pDeviceID != NULL) {\n        deviceName = pDeviceID->oss;\n    }\n\n    flags = (deviceType == ma_device_type_playback) ? O_WRONLY : O_RDONLY;\n    if (shareMode == ma_share_mode_exclusive) {\n        flags |= O_EXCL;\n    }\n\n    *pfd = open(deviceName, flags, 0);\n    if (*pfd == -1) {\n        return ma_result_from_errno(errno);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_enumerate_devices__oss(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    int fd;\n    oss_sysinfo si;\n    int result;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    fd = ma_open_temp_device__oss();\n    if (fd == -1) {\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open a temporary device for retrieving system information used for device enumeration.\");\n        return MA_NO_BACKEND;\n    }\n\n    result = ioctl(fd, SNDCTL_SYSINFO, &si);\n    if (result != -1) {\n        int iAudioDevice;\n        for (iAudioDevice = 0; iAudioDevice < si.numaudios; ++iAudioDevice) {\n            oss_audioinfo ai;\n            ai.dev = iAudioDevice;\n            result = ioctl(fd, SNDCTL_AUDIOINFO, &ai);\n            if (result != -1) {\n                if (ai.devnode[0] != '\\0') {    /* <-- Can be blank, according to documentation. */\n                    ma_device_info deviceInfo;\n                    ma_bool32 isTerminating = MA_FALSE;\n\n                    MA_ZERO_OBJECT(&deviceInfo);\n\n                    /* ID */\n                    ma_strncpy_s(deviceInfo.id.oss, sizeof(deviceInfo.id.oss), ai.devnode, (size_t)-1);\n\n                    /*\n                    The human readable device name should be in the \"ai.handle\" variable, but it can\n                    sometimes be empty in which case we just fall back to \"ai.name\" which is less user\n                    friendly, but usually has a value.\n                    */\n                    if (ai.handle[0] != '\\0') {\n                        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), ai.handle, (size_t)-1);\n                    } else {\n                        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), ai.name, (size_t)-1);\n                    }\n\n                    /* The device can be both playback and capture. */\n                    if (!isTerminating && (ai.caps & PCM_CAP_OUTPUT) != 0) {\n                        isTerminating = !callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n                    }\n                    if (!isTerminating && (ai.caps & PCM_CAP_INPUT) != 0) {\n                        isTerminating = !callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n                    }\n\n                    if (isTerminating) {\n                        break;\n                    }\n                }\n            }\n        }\n    } else {\n        close(fd);\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to retrieve system information for device enumeration.\");\n        return MA_NO_BACKEND;\n    }\n\n    close(fd);\n    return MA_SUCCESS;\n}\n\nstatic void ma_context_add_native_data_format__oss(ma_context* pContext, oss_audioinfo* pAudioInfo, ma_format format, ma_device_info* pDeviceInfo)\n{\n    unsigned int minChannels;\n    unsigned int maxChannels;\n    unsigned int iRate;\n\n    MA_ASSERT(pContext    != NULL);\n    MA_ASSERT(pAudioInfo  != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    /* If we support all channels we just report 0. */\n    minChannels = ma_clamp(pAudioInfo->min_channels, MA_MIN_CHANNELS, MA_MAX_CHANNELS);\n    maxChannels = ma_clamp(pAudioInfo->max_channels, MA_MIN_CHANNELS, MA_MAX_CHANNELS);\n\n    /*\n    OSS has this annoying thing where sample rates can be reported in two ways. We prefer explicitness,\n    which OSS has in the form of nrates/rates, however there are times where nrates can be 0, in which\n    case we'll need to use min_rate and max_rate and report only standard rates.\n    */\n    if (pAudioInfo->nrates > 0) {\n        for (iRate = 0; iRate < pAudioInfo->nrates; iRate += 1) {\n            unsigned int rate = pAudioInfo->rates[iRate];\n\n            if (minChannels == MA_MIN_CHANNELS && maxChannels == MA_MAX_CHANNELS) {\n                ma_device_info_add_native_data_format(pDeviceInfo, format, 0, rate, 0);   /* Set the channel count to 0 to indicate that all channel counts are supported. */\n            } else {\n                unsigned int iChannel;\n                for (iChannel = minChannels; iChannel <= maxChannels; iChannel += 1) {\n                     ma_device_info_add_native_data_format(pDeviceInfo, format, iChannel, rate, 0);\n                }\n            }\n        }\n    } else {\n        for (iRate = 0; iRate < ma_countof(g_maStandardSampleRatePriorities); iRate += 1) {\n            ma_uint32 standardRate = g_maStandardSampleRatePriorities[iRate];\n\n            if (standardRate >= (ma_uint32)pAudioInfo->min_rate && standardRate <= (ma_uint32)pAudioInfo->max_rate) {\n                if (minChannels == MA_MIN_CHANNELS && maxChannels == MA_MAX_CHANNELS) {\n                    ma_device_info_add_native_data_format(pDeviceInfo, format, 0, standardRate, 0);   /* Set the channel count to 0 to indicate that all channel counts are supported. */\n                } else {\n                    unsigned int iChannel;\n                    for (iChannel = minChannels; iChannel <= maxChannels; iChannel += 1) {\n                         ma_device_info_add_native_data_format(pDeviceInfo, format, iChannel, standardRate, 0);\n                    }\n                }\n            }\n        }\n    }\n}\n\nstatic ma_result ma_context_get_device_info__oss(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_bool32 foundDevice;\n    int fdTemp;\n    oss_sysinfo si;\n    int result;\n\n    MA_ASSERT(pContext != NULL);\n\n    /* Handle the default device a little differently. */\n    if (pDeviceID == NULL) {\n        if (deviceType == ma_device_type_playback) {\n            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n        } else {\n            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n        }\n\n        return MA_SUCCESS;\n    }\n\n\n    /* If we get here it means we are _not_ using the default device. */\n    foundDevice = MA_FALSE;\n\n    fdTemp = ma_open_temp_device__oss();\n    if (fdTemp == -1) {\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open a temporary device for retrieving system information used for device enumeration.\");\n        return MA_NO_BACKEND;\n    }\n\n    result = ioctl(fdTemp, SNDCTL_SYSINFO, &si);\n    if (result != -1) {\n        int iAudioDevice;\n        for (iAudioDevice = 0; iAudioDevice < si.numaudios; ++iAudioDevice) {\n            oss_audioinfo ai;\n            ai.dev = iAudioDevice;\n            result = ioctl(fdTemp, SNDCTL_AUDIOINFO, &ai);\n            if (result != -1) {\n                if (ma_strcmp(ai.devnode, pDeviceID->oss) == 0) {\n                    /* It has the same name, so now just confirm the type. */\n                    if ((deviceType == ma_device_type_playback && ((ai.caps & PCM_CAP_OUTPUT) != 0)) ||\n                        (deviceType == ma_device_type_capture  && ((ai.caps & PCM_CAP_INPUT)  != 0))) {\n                        unsigned int formatMask;\n\n                        /* ID */\n                        ma_strncpy_s(pDeviceInfo->id.oss, sizeof(pDeviceInfo->id.oss), ai.devnode, (size_t)-1);\n\n                        /*\n                        The human readable device name should be in the \"ai.handle\" variable, but it can\n                        sometimes be empty in which case we just fall back to \"ai.name\" which is less user\n                        friendly, but usually has a value.\n                        */\n                        if (ai.handle[0] != '\\0') {\n                            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), ai.handle, (size_t)-1);\n                        } else {\n                            ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), ai.name, (size_t)-1);\n                        }\n\n\n                        pDeviceInfo->nativeDataFormatCount = 0;\n\n                        if (deviceType == ma_device_type_playback) {\n                            formatMask = ai.oformats;\n                        } else {\n                            formatMask = ai.iformats;\n                        }\n\n                        if (((formatMask & AFMT_S16_LE) != 0 && ma_is_little_endian()) || (AFMT_S16_BE && ma_is_big_endian())) {\n                            ma_context_add_native_data_format__oss(pContext, &ai, ma_format_s16, pDeviceInfo);\n                        }\n                        if (((formatMask & AFMT_S32_LE) != 0 && ma_is_little_endian()) || (AFMT_S32_BE && ma_is_big_endian())) {\n                            ma_context_add_native_data_format__oss(pContext, &ai, ma_format_s32, pDeviceInfo);\n                        }\n                        if ((formatMask & AFMT_U8) != 0) {\n                            ma_context_add_native_data_format__oss(pContext, &ai, ma_format_u8, pDeviceInfo);\n                        }\n\n                        foundDevice = MA_TRUE;\n                        break;\n                    }\n                }\n            }\n        }\n    } else {\n        close(fdTemp);\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to retrieve system information for device enumeration.\");\n        return MA_NO_BACKEND;\n    }\n\n\n    close(fdTemp);\n\n    if (!foundDevice) {\n        return MA_NO_DEVICE;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_uninit__oss(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        close(pDevice->oss.fdCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        close(pDevice->oss.fdPlayback);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic int ma_format_to_oss(ma_format format)\n{\n    int ossFormat = AFMT_U8;\n    switch (format) {\n        case ma_format_s16: ossFormat = (ma_is_little_endian()) ? AFMT_S16_LE : AFMT_S16_BE; break;\n        case ma_format_s24: ossFormat = (ma_is_little_endian()) ? AFMT_S32_LE : AFMT_S32_BE; break;\n        case ma_format_s32: ossFormat = (ma_is_little_endian()) ? AFMT_S32_LE : AFMT_S32_BE; break;\n        case ma_format_f32: ossFormat = (ma_is_little_endian()) ? AFMT_S16_LE : AFMT_S16_BE; break;\n        case ma_format_u8:\n        default: ossFormat = AFMT_U8; break;\n    }\n\n    return ossFormat;\n}\n\nstatic ma_format ma_format_from_oss(int ossFormat)\n{\n    if (ossFormat == AFMT_U8) {\n        return ma_format_u8;\n    } else {\n        if (ma_is_little_endian()) {\n            switch (ossFormat) {\n                case AFMT_S16_LE: return ma_format_s16;\n                case AFMT_S32_LE: return ma_format_s32;\n                default: return ma_format_unknown;\n            }\n        } else {\n            switch (ossFormat) {\n                case AFMT_S16_BE: return ma_format_s16;\n                case AFMT_S32_BE: return ma_format_s32;\n                default: return ma_format_unknown;\n            }\n        }\n    }\n\n    return ma_format_unknown;\n}\n\nstatic ma_result ma_device_init_fd__oss(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptor, ma_device_type deviceType)\n{\n    ma_result result;\n    int ossResult;\n    int fd;\n    const ma_device_id* pDeviceID = NULL;\n    ma_share_mode shareMode;\n    int ossFormat;\n    int ossChannels;\n    int ossSampleRate;\n    int ossFragment;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(deviceType != ma_device_type_duplex);\n\n    pDeviceID     = pDescriptor->pDeviceID;\n    shareMode     = pDescriptor->shareMode;\n    ossFormat     = ma_format_to_oss((pDescriptor->format != ma_format_unknown) ? pDescriptor->format : ma_format_s16); /* Use s16 by default because OSS doesn't like floating point. */\n    ossChannels   = (int)(pDescriptor->channels   > 0) ? pDescriptor->channels   : MA_DEFAULT_CHANNELS;\n    ossSampleRate = (int)(pDescriptor->sampleRate > 0) ? pDescriptor->sampleRate : MA_DEFAULT_SAMPLE_RATE;\n\n    result = ma_context_open_device__oss(pDevice->pContext, deviceType, pDeviceID, shareMode, &fd);\n    if (result != MA_SUCCESS) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open device.\");\n        return result;\n    }\n\n    /*\n    The OSS documentation is very clear about the order we should be initializing the device's properties:\n      1) Format\n      2) Channels\n      3) Sample rate.\n    */\n\n    /* Format. */\n    ossResult = ioctl(fd, SNDCTL_DSP_SETFMT, &ossFormat);\n    if (ossResult == -1) {\n        close(fd);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to set format.\");\n        return ma_result_from_errno(errno);\n    }\n\n    /* Channels. */\n    ossResult = ioctl(fd, SNDCTL_DSP_CHANNELS, &ossChannels);\n    if (ossResult == -1) {\n        close(fd);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to set channel count.\");\n        return ma_result_from_errno(errno);\n    }\n\n    /* Sample Rate. */\n    ossResult = ioctl(fd, SNDCTL_DSP_SPEED, &ossSampleRate);\n    if (ossResult == -1) {\n        close(fd);\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to set sample rate.\");\n        return ma_result_from_errno(errno);\n    }\n\n    /*\n    Buffer.\n\n    The documentation says that the fragment settings should be set as soon as possible, but I'm not sure if\n    it should be done before or after format/channels/rate.\n\n    OSS wants the fragment size in bytes and a power of 2. When setting, we specify the power, not the actual\n    value.\n    */\n    {\n        ma_uint32 periodSizeInFrames;\n        ma_uint32 periodSizeInBytes;\n        ma_uint32 ossFragmentSizePower;\n\n        periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, (ma_uint32)ossSampleRate, pConfig->performanceProfile);\n\n        periodSizeInBytes = ma_round_to_power_of_2(periodSizeInFrames * ma_get_bytes_per_frame(ma_format_from_oss(ossFormat), ossChannels));\n        if (periodSizeInBytes < 16) {\n            periodSizeInBytes = 16;\n        }\n\n        ossFragmentSizePower = 4;\n        periodSizeInBytes >>= 4;\n        while (periodSizeInBytes >>= 1) {\n            ossFragmentSizePower += 1;\n        }\n\n        ossFragment = (int)((pConfig->periods << 16) | ossFragmentSizePower);\n        ossResult = ioctl(fd, SNDCTL_DSP_SETFRAGMENT, &ossFragment);\n        if (ossResult == -1) {\n            close(fd);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to set fragment size and period count.\");\n            return ma_result_from_errno(errno);\n        }\n    }\n\n    /* Internal settings. */\n    if (deviceType == ma_device_type_capture) {\n        pDevice->oss.fdCapture  = fd;\n    } else {\n        pDevice->oss.fdPlayback = fd;\n    }\n\n    pDescriptor->format             = ma_format_from_oss(ossFormat);\n    pDescriptor->channels           = ossChannels;\n    pDescriptor->sampleRate         = ossSampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_sound4, pDescriptor->channelMap, ma_countof(pDescriptor->channelMap), pDescriptor->channels);\n    pDescriptor->periodCount        = (ma_uint32)(ossFragment >> 16);\n    pDescriptor->periodSizeInFrames = (ma_uint32)(1 << (ossFragment & 0xFFFF)) / ma_get_bytes_per_frame(pDescriptor->format, pDescriptor->channels);\n\n    if (pDescriptor->format == ma_format_unknown) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] The device's internal format is not supported by miniaudio.\");\n        return MA_FORMAT_NOT_SUPPORTED;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__oss(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    MA_ASSERT(pDevice  != NULL);\n    MA_ASSERT(pConfig  != NULL);\n\n    MA_ZERO_OBJECT(&pDevice->oss);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_fd__oss(pDevice, pConfig, pDescriptorCapture, ma_device_type_capture);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open device.\");\n            return result;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_result result = ma_device_init_fd__oss(pDevice, pConfig, pDescriptorPlayback, ma_device_type_playback);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open device.\");\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n/*\nNote on Starting and Stopping\n=============================\nIn the past I was using SNDCTL_DSP_HALT to stop the device, however this results in issues when\ntrying to resume the device again. If we use SNDCTL_DSP_HALT, the next write() or read() will\nfail. Instead what we need to do is just not write or read to and from the device when the\ndevice is not running.\n\nAs a result, both the start and stop functions for OSS are just empty stubs. The starting and\nstopping logic is handled by ma_device_write__oss() and ma_device_read__oss(). These will check\nthe device state, and if the device is stopped they will simply not do any kind of processing.\n\nThe downside to this technique is that I've noticed a fairly lengthy delay in stopping the\ndevice, up to a second. This is on a virtual machine, and as such might just be due to the\nvirtual drivers, but I'm not fully sure. I am not sure how to work around this problem so for\nthe moment that's just how it's going to have to be.\n\nWhen starting the device, OSS will automatically start it when write() or read() is called.\n*/\nstatic ma_result ma_device_start__oss(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* The device is automatically started with reading and writing. */\n    (void)pDevice;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__oss(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* See note above on why this is empty. */\n    (void)pDevice;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_write__oss(ma_device* pDevice, const void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesWritten)\n{\n    int resultOSS;\n    ma_uint32 deviceState;\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    /* Don't do any processing if the device is stopped. */\n    deviceState = ma_device_get_state(pDevice);\n    if (deviceState != ma_device_state_started && deviceState != ma_device_state_starting) {\n        return MA_SUCCESS;\n    }\n\n    resultOSS = write(pDevice->oss.fdPlayback, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n    if (resultOSS < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to send data from the client to the device.\");\n        return ma_result_from_errno(errno);\n    }\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = (ma_uint32)resultOSS / ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_read__oss(ma_device* pDevice, void* pPCMFrames, ma_uint32 frameCount, ma_uint32* pFramesRead)\n{\n    int resultOSS;\n    ma_uint32 deviceState;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    /* Don't do any processing if the device is stopped. */\n    deviceState = ma_device_get_state(pDevice);\n    if (deviceState != ma_device_state_started && deviceState != ma_device_state_starting) {\n        return MA_SUCCESS;\n    }\n\n    resultOSS = read(pDevice->oss.fdCapture, pPCMFrames, frameCount * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels));\n    if (resultOSS < 0) {\n        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to read data from the device to be sent to the client.\");\n        return ma_result_from_errno(errno);\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = (ma_uint32)resultOSS / ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__oss(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_oss);\n\n    (void)pContext;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__oss(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    int fd;\n    int ossVersion;\n    int result;\n\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n    /* Try opening a temporary device first so we can get version information. This is closed at the end. */\n    fd = ma_open_temp_device__oss();\n    if (fd == -1) {\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to open temporary device for retrieving system properties.\");   /* Looks liks OSS isn't installed, or there are no available devices. */\n        return MA_NO_BACKEND;\n    }\n\n    /* Grab the OSS version. */\n    ossVersion = 0;\n    result = ioctl(fd, OSS_GETVERSION, &ossVersion);\n    if (result == -1) {\n        close(fd);\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_ERROR, \"[OSS] Failed to retrieve OSS version.\");\n        return MA_NO_BACKEND;\n    }\n\n    /* The file handle to temp device is no longer needed. Close ASAP. */\n    close(fd);\n\n    pContext->oss.versionMajor = ((ossVersion & 0xFF0000) >> 16);\n    pContext->oss.versionMinor = ((ossVersion & 0x00FF00) >> 8);\n\n    pCallbacks->onContextInit             = ma_context_init__oss;\n    pCallbacks->onContextUninit           = ma_context_uninit__oss;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__oss;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__oss;\n    pCallbacks->onDeviceInit              = ma_device_init__oss;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__oss;\n    pCallbacks->onDeviceStart             = ma_device_start__oss;\n    pCallbacks->onDeviceStop              = ma_device_stop__oss;\n    pCallbacks->onDeviceRead              = ma_device_read__oss;\n    pCallbacks->onDeviceWrite             = ma_device_write__oss;\n    pCallbacks->onDeviceDataLoop          = NULL;\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_OSS */\n\n\n\n\n\n/******************************************************************************\n\nAAudio Backend\n\n******************************************************************************/\n#ifdef MA_HAS_AAUDIO\n\n#ifdef MA_NO_RUNTIME_LINKING\n    #include <AAudio/AAudio.h>\n#endif\n\ntypedef int32_t                                         ma_aaudio_result_t;\ntypedef int32_t                                         ma_aaudio_direction_t;\ntypedef int32_t                                         ma_aaudio_sharing_mode_t;\ntypedef int32_t                                         ma_aaudio_format_t;\ntypedef int32_t                                         ma_aaudio_stream_state_t;\ntypedef int32_t                                         ma_aaudio_performance_mode_t;\ntypedef int32_t                                         ma_aaudio_usage_t;\ntypedef int32_t                                         ma_aaudio_content_type_t;\ntypedef int32_t                                         ma_aaudio_input_preset_t;\ntypedef int32_t                                         ma_aaudio_allowed_capture_policy_t;\ntypedef int32_t                                         ma_aaudio_data_callback_result_t;\ntypedef struct ma_AAudioStreamBuilder_t*                ma_AAudioStreamBuilder;\ntypedef struct ma_AAudioStream_t*                       ma_AAudioStream;\n\n#define MA_AAUDIO_UNSPECIFIED                           0\n\n/* Result codes. miniaudio only cares about the success code. */\n#define MA_AAUDIO_OK                                    0\n\n/* Directions. */\n#define MA_AAUDIO_DIRECTION_OUTPUT                      0\n#define MA_AAUDIO_DIRECTION_INPUT                       1\n\n/* Sharing modes. */\n#define MA_AAUDIO_SHARING_MODE_EXCLUSIVE                0\n#define MA_AAUDIO_SHARING_MODE_SHARED                   1\n\n/* Formats. */\n#define MA_AAUDIO_FORMAT_PCM_I16                        1\n#define MA_AAUDIO_FORMAT_PCM_FLOAT                      2\n\n/* Stream states. */\n#define MA_AAUDIO_STREAM_STATE_UNINITIALIZED            0\n#define MA_AAUDIO_STREAM_STATE_UNKNOWN                  1\n#define MA_AAUDIO_STREAM_STATE_OPEN                     2\n#define MA_AAUDIO_STREAM_STATE_STARTING                 3\n#define MA_AAUDIO_STREAM_STATE_STARTED                  4\n#define MA_AAUDIO_STREAM_STATE_PAUSING                  5\n#define MA_AAUDIO_STREAM_STATE_PAUSED                   6\n#define MA_AAUDIO_STREAM_STATE_FLUSHING                 7\n#define MA_AAUDIO_STREAM_STATE_FLUSHED                  8\n#define MA_AAUDIO_STREAM_STATE_STOPPING                 9\n#define MA_AAUDIO_STREAM_STATE_STOPPED                  10\n#define MA_AAUDIO_STREAM_STATE_CLOSING                  11\n#define MA_AAUDIO_STREAM_STATE_CLOSED                   12\n#define MA_AAUDIO_STREAM_STATE_DISCONNECTED             13\n\n/* Performance modes. */\n#define MA_AAUDIO_PERFORMANCE_MODE_NONE                 10\n#define MA_AAUDIO_PERFORMANCE_MODE_POWER_SAVING         11\n#define MA_AAUDIO_PERFORMANCE_MODE_LOW_LATENCY          12\n\n/* Usage types. */\n#define MA_AAUDIO_USAGE_MEDIA                           1\n#define MA_AAUDIO_USAGE_VOICE_COMMUNICATION             2\n#define MA_AAUDIO_USAGE_VOICE_COMMUNICATION_SIGNALLING  3\n#define MA_AAUDIO_USAGE_ALARM                           4\n#define MA_AAUDIO_USAGE_NOTIFICATION                    5\n#define MA_AAUDIO_USAGE_NOTIFICATION_RINGTONE           6\n#define MA_AAUDIO_USAGE_NOTIFICATION_EVENT              10\n#define MA_AAUDIO_USAGE_ASSISTANCE_ACCESSIBILITY        11\n#define MA_AAUDIO_USAGE_ASSISTANCE_NAVIGATION_GUIDANCE  12\n#define MA_AAUDIO_USAGE_ASSISTANCE_SONIFICATION         13\n#define MA_AAUDIO_USAGE_GAME                            14\n#define MA_AAUDIO_USAGE_ASSISTANT                       16\n#define MA_AAUDIO_SYSTEM_USAGE_EMERGENCY                1000\n#define MA_AAUDIO_SYSTEM_USAGE_SAFETY                   1001\n#define MA_AAUDIO_SYSTEM_USAGE_VEHICLE_STATUS           1002\n#define MA_AAUDIO_SYSTEM_USAGE_ANNOUNCEMENT             1003\n\n/* Content types. */\n#define MA_AAUDIO_CONTENT_TYPE_SPEECH                   1\n#define MA_AAUDIO_CONTENT_TYPE_MUSIC                    2\n#define MA_AAUDIO_CONTENT_TYPE_MOVIE                    3\n#define MA_AAUDIO_CONTENT_TYPE_SONIFICATION             4\n\n/* Input presets. */\n#define MA_AAUDIO_INPUT_PRESET_GENERIC                  1\n#define MA_AAUDIO_INPUT_PRESET_CAMCORDER                5\n#define MA_AAUDIO_INPUT_PRESET_VOICE_RECOGNITION        6\n#define MA_AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION      7\n#define MA_AAUDIO_INPUT_PRESET_UNPROCESSED              9\n#define MA_AAUDIO_INPUT_PRESET_VOICE_PERFORMANCE        10\n\n/* Allowed Capture Policies */\n#define MA_AAUDIO_ALLOW_CAPTURE_BY_ALL                  1\n#define MA_AAUDIO_ALLOW_CAPTURE_BY_SYSTEM               2\n#define MA_AAUDIO_ALLOW_CAPTURE_BY_NONE                 3\n\n/* Callback results. */\n#define MA_AAUDIO_CALLBACK_RESULT_CONTINUE              0\n#define MA_AAUDIO_CALLBACK_RESULT_STOP                  1\n\n\ntypedef ma_aaudio_data_callback_result_t (* ma_AAudioStream_dataCallback) (ma_AAudioStream* pStream, void* pUserData, void* pAudioData, int32_t numFrames);\ntypedef void                             (* ma_AAudioStream_errorCallback)(ma_AAudioStream *pStream, void *pUserData, ma_aaudio_result_t error);\n\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudio_createStreamBuilder)                   (ma_AAudioStreamBuilder** ppBuilder);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStreamBuilder_delete)                   (ma_AAudioStreamBuilder* pBuilder);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setDeviceId)              (ma_AAudioStreamBuilder* pBuilder, int32_t deviceId);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setDirection)             (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_direction_t direction);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setSharingMode)           (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_sharing_mode_t sharingMode);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setFormat)                (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_format_t format);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setChannelCount)          (ma_AAudioStreamBuilder* pBuilder, int32_t channelCount);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setSampleRate)            (ma_AAudioStreamBuilder* pBuilder, int32_t sampleRate);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setBufferCapacityInFrames)(ma_AAudioStreamBuilder* pBuilder, int32_t numFrames);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setFramesPerDataCallback) (ma_AAudioStreamBuilder* pBuilder, int32_t numFrames);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setDataCallback)          (ma_AAudioStreamBuilder* pBuilder, ma_AAudioStream_dataCallback callback, void* pUserData);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setErrorCallback)         (ma_AAudioStreamBuilder* pBuilder, ma_AAudioStream_errorCallback callback, void* pUserData);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setPerformanceMode)       (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_performance_mode_t mode);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setUsage)                 (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_usage_t contentType);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setContentType)           (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_content_type_t contentType);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setInputPreset)           (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_input_preset_t inputPreset);\ntypedef void                     (* MA_PFN_AAudioStreamBuilder_setAllowedCapturePolicy)  (ma_AAudioStreamBuilder* pBuilder, ma_aaudio_allowed_capture_policy_t policy);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStreamBuilder_openStream)               (ma_AAudioStreamBuilder* pBuilder, ma_AAudioStream** ppStream);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStream_close)                           (ma_AAudioStream* pStream);\ntypedef ma_aaudio_stream_state_t (* MA_PFN_AAudioStream_getState)                        (ma_AAudioStream* pStream);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStream_waitForStateChange)              (ma_AAudioStream* pStream, ma_aaudio_stream_state_t inputState, ma_aaudio_stream_state_t* pNextState, int64_t timeoutInNanoseconds);\ntypedef ma_aaudio_format_t       (* MA_PFN_AAudioStream_getFormat)                       (ma_AAudioStream* pStream);\ntypedef int32_t                  (* MA_PFN_AAudioStream_getChannelCount)                 (ma_AAudioStream* pStream);\ntypedef int32_t                  (* MA_PFN_AAudioStream_getSampleRate)                   (ma_AAudioStream* pStream);\ntypedef int32_t                  (* MA_PFN_AAudioStream_getBufferCapacityInFrames)       (ma_AAudioStream* pStream);\ntypedef int32_t                  (* MA_PFN_AAudioStream_getFramesPerDataCallback)        (ma_AAudioStream* pStream);\ntypedef int32_t                  (* MA_PFN_AAudioStream_getFramesPerBurst)               (ma_AAudioStream* pStream);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStream_requestStart)                    (ma_AAudioStream* pStream);\ntypedef ma_aaudio_result_t       (* MA_PFN_AAudioStream_requestStop)                     (ma_AAudioStream* pStream);\n\nstatic ma_result ma_result_from_aaudio(ma_aaudio_result_t resultAA)\n{\n    switch (resultAA)\n    {\n        case MA_AAUDIO_OK: return MA_SUCCESS;\n        default: break;\n    }\n\n    return MA_ERROR;\n}\n\nstatic ma_aaudio_usage_t ma_to_usage__aaudio(ma_aaudio_usage usage)\n{\n    switch (usage) {\n        case ma_aaudio_usage_media:                          return MA_AAUDIO_USAGE_MEDIA;\n        case ma_aaudio_usage_voice_communication:            return MA_AAUDIO_USAGE_VOICE_COMMUNICATION;\n        case ma_aaudio_usage_voice_communication_signalling: return MA_AAUDIO_USAGE_VOICE_COMMUNICATION_SIGNALLING;\n        case ma_aaudio_usage_alarm:                          return MA_AAUDIO_USAGE_ALARM;\n        case ma_aaudio_usage_notification:                   return MA_AAUDIO_USAGE_NOTIFICATION;\n        case ma_aaudio_usage_notification_ringtone:          return MA_AAUDIO_USAGE_NOTIFICATION_RINGTONE;\n        case ma_aaudio_usage_notification_event:             return MA_AAUDIO_USAGE_NOTIFICATION_EVENT;\n        case ma_aaudio_usage_assistance_accessibility:       return MA_AAUDIO_USAGE_ASSISTANCE_ACCESSIBILITY;\n        case ma_aaudio_usage_assistance_navigation_guidance: return MA_AAUDIO_USAGE_ASSISTANCE_NAVIGATION_GUIDANCE;\n        case ma_aaudio_usage_assistance_sonification:        return MA_AAUDIO_USAGE_ASSISTANCE_SONIFICATION;\n        case ma_aaudio_usage_game:                           return MA_AAUDIO_USAGE_GAME;\n        case ma_aaudio_usage_assitant:                       return MA_AAUDIO_USAGE_ASSISTANT;\n        case ma_aaudio_usage_emergency:                      return MA_AAUDIO_SYSTEM_USAGE_EMERGENCY;\n        case ma_aaudio_usage_safety:                         return MA_AAUDIO_SYSTEM_USAGE_SAFETY;\n        case ma_aaudio_usage_vehicle_status:                 return MA_AAUDIO_SYSTEM_USAGE_VEHICLE_STATUS;\n        case ma_aaudio_usage_announcement:                   return MA_AAUDIO_SYSTEM_USAGE_ANNOUNCEMENT;\n        default: break;\n    }\n\n    return MA_AAUDIO_USAGE_MEDIA;\n}\n\nstatic ma_aaudio_content_type_t ma_to_content_type__aaudio(ma_aaudio_content_type contentType)\n{\n    switch (contentType) {\n        case ma_aaudio_content_type_speech:       return MA_AAUDIO_CONTENT_TYPE_SPEECH;\n        case ma_aaudio_content_type_music:        return MA_AAUDIO_CONTENT_TYPE_MUSIC;\n        case ma_aaudio_content_type_movie:        return MA_AAUDIO_CONTENT_TYPE_MOVIE;\n        case ma_aaudio_content_type_sonification: return MA_AAUDIO_CONTENT_TYPE_SONIFICATION;\n        default: break;\n    }\n\n    return MA_AAUDIO_CONTENT_TYPE_SPEECH;\n}\n\nstatic ma_aaudio_input_preset_t ma_to_input_preset__aaudio(ma_aaudio_input_preset inputPreset)\n{\n    switch (inputPreset) {\n        case ma_aaudio_input_preset_generic:             return MA_AAUDIO_INPUT_PRESET_GENERIC;\n        case ma_aaudio_input_preset_camcorder:           return MA_AAUDIO_INPUT_PRESET_CAMCORDER;\n        case ma_aaudio_input_preset_voice_recognition:   return MA_AAUDIO_INPUT_PRESET_VOICE_RECOGNITION;\n        case ma_aaudio_input_preset_voice_communication: return MA_AAUDIO_INPUT_PRESET_VOICE_COMMUNICATION;\n        case ma_aaudio_input_preset_unprocessed:         return MA_AAUDIO_INPUT_PRESET_UNPROCESSED;\n        case ma_aaudio_input_preset_voice_performance:   return MA_AAUDIO_INPUT_PRESET_VOICE_PERFORMANCE;\n        default: break;\n    }\n\n    return MA_AAUDIO_INPUT_PRESET_GENERIC;\n}\n\nstatic ma_aaudio_allowed_capture_policy_t ma_to_allowed_capture_policy__aaudio(ma_aaudio_allowed_capture_policy allowedCapturePolicy)\n{\n    switch (allowedCapturePolicy) {\n        case ma_aaudio_allow_capture_by_all:    return MA_AAUDIO_ALLOW_CAPTURE_BY_ALL;\n        case ma_aaudio_allow_capture_by_system: return MA_AAUDIO_ALLOW_CAPTURE_BY_SYSTEM;\n        case ma_aaudio_allow_capture_by_none:   return MA_AAUDIO_ALLOW_CAPTURE_BY_NONE;\n        default: break;\n    }\n\n    return MA_AAUDIO_ALLOW_CAPTURE_BY_ALL;\n}\n\nstatic void ma_stream_error_callback__aaudio(ma_AAudioStream* pStream, void* pUserData, ma_aaudio_result_t error)\n{\n    ma_result result;\n    ma_job job;\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    (void)error;\n    ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[AAudio] ERROR CALLBACK: error=%d, AAudioStream_getState()=%d\\n\", error, ((MA_PFN_AAudioStream_getState)pDevice->pContext->aaudio.AAudioStream_getState)(pStream));\n    \n    /*\n    When we get an error, we'll assume that the stream is in an erroneous state and needs to be restarted. From the documentation,\n    we cannot do this from the error callback. Therefore we are going to use an event thread for the AAudio backend to do this\n    cleanly and safely.\n    */\n    if (ma_atomic_bool32_get(&pDevice->aaudio.isTearingDown)) {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[AAudio] Device Disconnected. Tearing down device.\\n\");\n    }\n    else {\n        job = ma_job_init(MA_JOB_TYPE_DEVICE_AAUDIO_REROUTE);\n        job.data.device.aaudio.reroute.pDevice = pDevice;\n    \n        if (pStream == pDevice->aaudio.pStreamCapture) {\n            job.data.device.aaudio.reroute.deviceType = ma_device_type_capture;\n        } else {\n            job.data.device.aaudio.reroute.deviceType = ma_device_type_playback;\n        }\n    \n        result = ma_device_job_thread_post(&pDevice->pContext->aaudio.jobThread, &job);\n        if (result != MA_SUCCESS) {\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[AAudio] Device Disconnected. Failed to post job for rerouting.\\n\");\n            return;\n        }\n    }\n}\n\nstatic ma_aaudio_data_callback_result_t ma_stream_data_callback_capture__aaudio(ma_AAudioStream* pStream, void* pUserData, void* pAudioData, int32_t frameCount)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    if (frameCount > 0) {\n        ma_device_handle_backend_data_callback(pDevice, NULL, pAudioData, (ma_uint32)frameCount);\n    }\n\n    (void)pStream;\n    return MA_AAUDIO_CALLBACK_RESULT_CONTINUE;\n}\n\nstatic ma_aaudio_data_callback_result_t ma_stream_data_callback_playback__aaudio(ma_AAudioStream* pStream, void* pUserData, void* pAudioData, int32_t frameCount)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    I've had a report that AAudio can sometimes post a frame count of 0. We need to check for that here\n    so we don't get any errors at a deeper level. I'm doing the same with the capture side for safety,\n    though I've not yet had any reports about that one.\n    */\n    if (frameCount > 0) {\n        ma_device_handle_backend_data_callback(pDevice, pAudioData, NULL, (ma_uint32)frameCount);\n    }\n\n    (void)pStream;\n    return MA_AAUDIO_CALLBACK_RESULT_CONTINUE;\n}\n\nstatic ma_result ma_create_and_configure_AAudioStreamBuilder__aaudio(ma_context* pContext, const ma_device_id* pDeviceID, ma_device_type deviceType, ma_share_mode shareMode, const ma_device_descriptor* pDescriptor, const ma_device_config* pConfig, ma_device* pDevice, ma_AAudioStreamBuilder** ppBuilder)\n{\n    ma_AAudioStreamBuilder* pBuilder;\n    ma_aaudio_result_t resultAA;\n\n    /* Safety. */\n    *ppBuilder = NULL;\n\n    resultAA = ((MA_PFN_AAudio_createStreamBuilder)pContext->aaudio.AAudio_createStreamBuilder)(&pBuilder);\n    if (resultAA != MA_AAUDIO_OK) {\n        return ma_result_from_aaudio(resultAA);\n    }\n\n    if (pDeviceID != NULL) {\n        ((MA_PFN_AAudioStreamBuilder_setDeviceId)pContext->aaudio.AAudioStreamBuilder_setDeviceId)(pBuilder, pDeviceID->aaudio);\n    }\n\n    ((MA_PFN_AAudioStreamBuilder_setDirection)pContext->aaudio.AAudioStreamBuilder_setDirection)(pBuilder, (deviceType == ma_device_type_playback) ? MA_AAUDIO_DIRECTION_OUTPUT : MA_AAUDIO_DIRECTION_INPUT);\n    ((MA_PFN_AAudioStreamBuilder_setSharingMode)pContext->aaudio.AAudioStreamBuilder_setSharingMode)(pBuilder, (shareMode == ma_share_mode_shared) ? MA_AAUDIO_SHARING_MODE_SHARED : MA_AAUDIO_SHARING_MODE_EXCLUSIVE);\n\n\n    /* If we have a device descriptor make sure we configure the stream builder to take our requested parameters. */\n    if (pDescriptor != NULL) {\n        MA_ASSERT(pConfig != NULL); /* We must have a device config if we also have a descriptor. The config is required for AAudio specific configuration options. */\n\n        if (pDescriptor->sampleRate != 0) {\n            ((MA_PFN_AAudioStreamBuilder_setSampleRate)pContext->aaudio.AAudioStreamBuilder_setSampleRate)(pBuilder, pDescriptor->sampleRate);\n        }\n\n        if (pDescriptor->channels != 0) {\n            ((MA_PFN_AAudioStreamBuilder_setChannelCount)pContext->aaudio.AAudioStreamBuilder_setChannelCount)(pBuilder, pDescriptor->channels);\n        }\n\n        if (pDescriptor->format != ma_format_unknown) {\n            ((MA_PFN_AAudioStreamBuilder_setFormat)pContext->aaudio.AAudioStreamBuilder_setFormat)(pBuilder, (pDescriptor->format == ma_format_s16) ? MA_AAUDIO_FORMAT_PCM_I16 : MA_AAUDIO_FORMAT_PCM_FLOAT);\n        }\n\n\n        /*\n        There have been reports where setting the frames per data callback results in an error.\n        In particular, re-routing may inadvertently switch from low-latency mode, resulting in a less stable\n        stream from the legacy path (AudioStreamLegacy). To address this, we simply don't set the value. It\n        can still be set if it's explicitly requested via the aaudio.allowSetBufferCapacity variable in the\n        device config.\n        */\n        if ((!pConfig->aaudio.enableCompatibilityWorkarounds || ma_android_sdk_version() > 30) && pConfig->aaudio.allowSetBufferCapacity) {\n            /*\n            AAudio is annoying when it comes to its buffer calculation stuff because it doesn't let you\n            retrieve the actual sample rate until after you've opened the stream. But you need to configure\n            the buffer capacity before you open the stream... :/\n\n            To solve, we're just going to assume MA_DEFAULT_SAMPLE_RATE (48000) and move on.\n            */\n            ma_uint32 bufferCapacityInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptor, pDescriptor->sampleRate, pConfig->performanceProfile) * pDescriptor->periodCount;\n\n            ((MA_PFN_AAudioStreamBuilder_setBufferCapacityInFrames)pContext->aaudio.AAudioStreamBuilder_setBufferCapacityInFrames)(pBuilder, bufferCapacityInFrames);\n            ((MA_PFN_AAudioStreamBuilder_setFramesPerDataCallback)pContext->aaudio.AAudioStreamBuilder_setFramesPerDataCallback)(pBuilder, bufferCapacityInFrames / pDescriptor->periodCount);\n        }\n\n        if (deviceType == ma_device_type_capture) {\n            if (pConfig->aaudio.inputPreset != ma_aaudio_input_preset_default && pContext->aaudio.AAudioStreamBuilder_setInputPreset != NULL) {\n                ((MA_PFN_AAudioStreamBuilder_setInputPreset)pContext->aaudio.AAudioStreamBuilder_setInputPreset)(pBuilder, ma_to_input_preset__aaudio(pConfig->aaudio.inputPreset));\n            }\n\n            ((MA_PFN_AAudioStreamBuilder_setDataCallback)pContext->aaudio.AAudioStreamBuilder_setDataCallback)(pBuilder, ma_stream_data_callback_capture__aaudio, (void*)pDevice);\n        } else {\n            if (pConfig->aaudio.usage != ma_aaudio_usage_default && pContext->aaudio.AAudioStreamBuilder_setUsage != NULL) {\n                ((MA_PFN_AAudioStreamBuilder_setUsage)pContext->aaudio.AAudioStreamBuilder_setUsage)(pBuilder, ma_to_usage__aaudio(pConfig->aaudio.usage));\n            }\n\n            if (pConfig->aaudio.contentType != ma_aaudio_content_type_default && pContext->aaudio.AAudioStreamBuilder_setContentType != NULL) {\n                ((MA_PFN_AAudioStreamBuilder_setContentType)pContext->aaudio.AAudioStreamBuilder_setContentType)(pBuilder, ma_to_content_type__aaudio(pConfig->aaudio.contentType));\n            }\n\n            if (pConfig->aaudio.allowedCapturePolicy != ma_aaudio_allow_capture_default && pContext->aaudio.AAudioStreamBuilder_setAllowedCapturePolicy != NULL) {\n                ((MA_PFN_AAudioStreamBuilder_setAllowedCapturePolicy)pContext->aaudio.AAudioStreamBuilder_setAllowedCapturePolicy)(pBuilder, ma_to_allowed_capture_policy__aaudio(pConfig->aaudio.allowedCapturePolicy));\n            }\n\n            ((MA_PFN_AAudioStreamBuilder_setDataCallback)pContext->aaudio.AAudioStreamBuilder_setDataCallback)(pBuilder, ma_stream_data_callback_playback__aaudio, (void*)pDevice);\n        }\n\n        /*\n        If we set AAUDIO_PERFORMANCE_MODE_LOW_LATENCY, we allow for MMAP (non-legacy path).\n        Since there's a mapping between miniaudio's performance profiles and AAudio's performance modes, let's use it.\n        Beware though, with a conservative performance profile, AAudio will indeed take the legacy path.\n        */\n        ((MA_PFN_AAudioStreamBuilder_setPerformanceMode)pContext->aaudio.AAudioStreamBuilder_setPerformanceMode)(pBuilder, (pConfig->performanceProfile == ma_performance_profile_low_latency) ? MA_AAUDIO_PERFORMANCE_MODE_LOW_LATENCY : MA_AAUDIO_PERFORMANCE_MODE_NONE);\n\n        /* We need to set an error callback to detect device changes. */\n        if (pDevice != NULL) {  /* <-- pDevice should never be null if pDescriptor is not null, which is always the case if we hit this branch. Check anyway for safety. */\n            ((MA_PFN_AAudioStreamBuilder_setErrorCallback)pContext->aaudio.AAudioStreamBuilder_setErrorCallback)(pBuilder, ma_stream_error_callback__aaudio, (void*)pDevice);\n        }\n    }\n\n    *ppBuilder = pBuilder;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_open_stream_and_close_builder__aaudio(ma_context* pContext, ma_AAudioStreamBuilder* pBuilder, ma_AAudioStream** ppStream)\n{\n    ma_result result;\n\n    result = ma_result_from_aaudio(((MA_PFN_AAudioStreamBuilder_openStream)pContext->aaudio.AAudioStreamBuilder_openStream)(pBuilder, ppStream));\n    ((MA_PFN_AAudioStreamBuilder_delete)pContext->aaudio.AAudioStreamBuilder_delete)(pBuilder);\n\n    return result;\n}\n\nstatic ma_result ma_open_stream_basic__aaudio(ma_context* pContext, const ma_device_id* pDeviceID, ma_device_type deviceType, ma_share_mode shareMode, ma_AAudioStream** ppStream)\n{\n    ma_result result;\n    ma_AAudioStreamBuilder* pBuilder;\n\n    *ppStream = NULL;\n\n    result = ma_create_and_configure_AAudioStreamBuilder__aaudio(pContext, pDeviceID, deviceType, shareMode, NULL, NULL, NULL, &pBuilder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Let's give AAudio a hint to avoid the legacy path (AudioStreamLegacy). */\n    ((MA_PFN_AAudioStreamBuilder_setPerformanceMode)pContext->aaudio.AAudioStreamBuilder_setPerformanceMode)(pBuilder, MA_AAUDIO_PERFORMANCE_MODE_LOW_LATENCY);\n\n    return ma_open_stream_and_close_builder__aaudio(pContext, pBuilder, ppStream);\n}\n\nstatic ma_result ma_open_stream__aaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_type deviceType, const ma_device_descriptor* pDescriptor, ma_AAudioStream** ppStream)\n{\n    ma_result result;\n    ma_AAudioStreamBuilder* pBuilder;\n\n    MA_ASSERT(pDevice != NULL);\n    MA_ASSERT(pDescriptor != NULL);\n    MA_ASSERT(deviceType != ma_device_type_duplex);   /* This function should not be called for a full-duplex device type. */\n\n    *ppStream = NULL;\n\n    result = ma_create_and_configure_AAudioStreamBuilder__aaudio(pDevice->pContext, pDescriptor->pDeviceID, deviceType, pDescriptor->shareMode, pDescriptor, pConfig, pDevice, &pBuilder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_open_stream_and_close_builder__aaudio(pDevice->pContext, pBuilder, ppStream);\n}\n\nstatic ma_result ma_close_stream__aaudio(ma_context* pContext, ma_AAudioStream* pStream)\n{\n    if (pStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_result_from_aaudio(((MA_PFN_AAudioStream_close)pContext->aaudio.AAudioStream_close)(pStream));\n}\n\nstatic ma_bool32 ma_has_default_device__aaudio(ma_context* pContext, ma_device_type deviceType)\n{\n    /* The only way to know this is to try creating a stream. */\n    ma_AAudioStream* pStream;\n    ma_result result = ma_open_stream_basic__aaudio(pContext, NULL, deviceType, ma_share_mode_shared, &pStream);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n\n    ma_close_stream__aaudio(pContext, pStream);\n    return MA_TRUE;\n}\n\nstatic ma_result ma_wait_for_simple_state_transition__aaudio(ma_context* pContext, ma_AAudioStream* pStream, ma_aaudio_stream_state_t oldState, ma_aaudio_stream_state_t newState)\n{\n    ma_aaudio_stream_state_t actualNewState;\n    ma_aaudio_result_t resultAA = ((MA_PFN_AAudioStream_waitForStateChange)pContext->aaudio.AAudioStream_waitForStateChange)(pStream, oldState, &actualNewState, 5000000000); /* 5 second timeout. */\n    if (resultAA != MA_AAUDIO_OK) {\n        return ma_result_from_aaudio(resultAA);\n    }\n\n    if (newState != actualNewState) {\n        return MA_ERROR;   /* Failed to transition into the expected state. */\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__aaudio(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 cbResult = MA_TRUE;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Unfortunately AAudio does not have an enumeration API. Therefore I'm only going to report default devices, but only if it can instantiate a stream. */\n\n    /* Playback. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        deviceInfo.id.aaudio = MA_AAUDIO_UNSPECIFIED;\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n\n        if (ma_has_default_device__aaudio(pContext, ma_device_type_playback)) {\n            cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n        }\n    }\n\n    /* Capture. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        deviceInfo.id.aaudio = MA_AAUDIO_UNSPECIFIED;\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n\n        if (ma_has_default_device__aaudio(pContext, ma_device_type_capture)) {\n            cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_context_add_native_data_format_from_AAudioStream_ex__aaudio(ma_context* pContext, ma_AAudioStream* pStream, ma_format format, ma_uint32 flags, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pContext    != NULL);\n    MA_ASSERT(pStream     != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = format;\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = ((MA_PFN_AAudioStream_getChannelCount)pContext->aaudio.AAudioStream_getChannelCount)(pStream);\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = ((MA_PFN_AAudioStream_getSampleRate)pContext->aaudio.AAudioStream_getSampleRate)(pStream);\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = flags;\n    pDeviceInfo->nativeDataFormatCount += 1;\n}\n\nstatic void ma_context_add_native_data_format_from_AAudioStream__aaudio(ma_context* pContext, ma_AAudioStream* pStream, ma_uint32 flags, ma_device_info* pDeviceInfo)\n{\n    /* AAudio supports s16 and f32. */\n    ma_context_add_native_data_format_from_AAudioStream_ex__aaudio(pContext, pStream, ma_format_f32, flags, pDeviceInfo);\n    ma_context_add_native_data_format_from_AAudioStream_ex__aaudio(pContext, pStream, ma_format_s16, flags, pDeviceInfo);\n}\n\nstatic ma_result ma_context_get_device_info__aaudio(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_AAudioStream* pStream;\n    ma_result result;\n\n    MA_ASSERT(pContext != NULL);\n\n    /* ID */\n    if (pDeviceID != NULL) {\n        pDeviceInfo->id.aaudio = pDeviceID->aaudio;\n    } else {\n        pDeviceInfo->id.aaudio = MA_AAUDIO_UNSPECIFIED;\n    }\n\n    /* Name */\n    if (deviceType == ma_device_type_playback) {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n    } else {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n    }\n\n\n    pDeviceInfo->nativeDataFormatCount = 0;\n\n    /* We'll need to open the device to get accurate sample rate and channel count information. */\n    result = ma_open_stream_basic__aaudio(pContext, pDeviceID, deviceType, ma_share_mode_shared, &pStream);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    ma_context_add_native_data_format_from_AAudioStream__aaudio(pContext, pStream, 0, pDeviceInfo);\n\n    ma_close_stream__aaudio(pContext, pStream);\n    pStream = NULL;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_close_streams__aaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /* When rerouting, streams may have been closed and never re-opened. Hence the extra checks below. */\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_close_stream__aaudio(pDevice->pContext, (ma_AAudioStream*)pDevice->aaudio.pStreamCapture);\n        pDevice->aaudio.pStreamCapture = NULL;\n    }\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_close_stream__aaudio(pDevice->pContext, (ma_AAudioStream*)pDevice->aaudio.pStreamPlayback);\n        pDevice->aaudio.pStreamPlayback = NULL;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_uninit__aaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    Note: Closing the streams may cause a timeout error, which would then trigger rerouting in our error callback.\n    We must not schedule a reroute when device is getting destroyed.\n    */\n    ma_atomic_bool32_set(&pDevice->aaudio.isTearingDown, MA_TRUE);\n\n    /* Wait for any rerouting to finish before attempting to close the streams. */\n    ma_mutex_lock(&pDevice->aaudio.rerouteLock);\n    {\n        ma_close_streams__aaudio(pDevice);\n    }\n    ma_mutex_unlock(&pDevice->aaudio.rerouteLock);\n\n    /* Destroy rerouting lock. */\n    ma_mutex_uninit(&pDevice->aaudio.rerouteLock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init_by_type__aaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_type deviceType, ma_device_descriptor* pDescriptor, ma_AAudioStream** ppStream)\n{\n    ma_result result;\n    int32_t bufferCapacityInFrames;\n    int32_t framesPerDataCallback;\n    ma_AAudioStream* pStream;\n\n    MA_ASSERT(pDevice     != NULL);\n    MA_ASSERT(pConfig     != NULL);\n    MA_ASSERT(pDescriptor != NULL);\n\n    *ppStream = NULL;   /* Safety. */\n\n    /* First step is to open the stream. From there we'll be able to extract the internal configuration. */\n    result = ma_open_stream__aaudio(pDevice, pConfig, deviceType, pDescriptor, &pStream);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to open the AAudio stream. */\n    }\n\n    /* Now extract the internal configuration. */\n    pDescriptor->format     = (((MA_PFN_AAudioStream_getFormat)pDevice->pContext->aaudio.AAudioStream_getFormat)(pStream) == MA_AAUDIO_FORMAT_PCM_I16) ? ma_format_s16 : ma_format_f32;\n    pDescriptor->channels   = ((MA_PFN_AAudioStream_getChannelCount)pDevice->pContext->aaudio.AAudioStream_getChannelCount)(pStream);\n    pDescriptor->sampleRate = ((MA_PFN_AAudioStream_getSampleRate)pDevice->pContext->aaudio.AAudioStream_getSampleRate)(pStream);\n\n    /* For the channel map we need to be sure we don't overflow any buffers. */\n    if (pDescriptor->channels <= MA_MAX_CHANNELS) {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pDescriptor->channelMap, ma_countof(pDescriptor->channelMap), pDescriptor->channels); /* <-- Cannot find info on channel order, so assuming a default. */\n    } else {\n        ma_channel_map_init_blank(pDescriptor->channelMap, MA_MAX_CHANNELS); /* Too many channels. Use a blank channel map. */\n    }\n\n    bufferCapacityInFrames = ((MA_PFN_AAudioStream_getBufferCapacityInFrames)pDevice->pContext->aaudio.AAudioStream_getBufferCapacityInFrames)(pStream);\n    framesPerDataCallback = ((MA_PFN_AAudioStream_getFramesPerDataCallback)pDevice->pContext->aaudio.AAudioStream_getFramesPerDataCallback)(pStream);\n\n    if (framesPerDataCallback > 0) {\n        pDescriptor->periodSizeInFrames = framesPerDataCallback;\n        pDescriptor->periodCount        = bufferCapacityInFrames / framesPerDataCallback;\n    } else {\n        pDescriptor->periodSizeInFrames = bufferCapacityInFrames;\n        pDescriptor->periodCount        = 1;\n    }\n\n    *ppStream = pStream;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init_streams__aaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    pDevice->aaudio.usage                   = pConfig->aaudio.usage;\n    pDevice->aaudio.contentType             = pConfig->aaudio.contentType;\n    pDevice->aaudio.inputPreset             = pConfig->aaudio.inputPreset;\n    pDevice->aaudio.allowedCapturePolicy    = pConfig->aaudio.allowedCapturePolicy;\n    pDevice->aaudio.noAutoStartAfterReroute = pConfig->aaudio.noAutoStartAfterReroute;\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        result = ma_device_init_by_type__aaudio(pDevice, pConfig, ma_device_type_capture, pDescriptorCapture, (ma_AAudioStream**)&pDevice->aaudio.pStreamCapture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        result = ma_device_init_by_type__aaudio(pDevice, pConfig, ma_device_type_playback, pDescriptorPlayback, (ma_AAudioStream**)&pDevice->aaudio.pStreamPlayback);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__aaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    result = ma_device_init_streams__aaudio(pDevice, pConfig, pDescriptorPlayback, pDescriptorCapture);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_mutex_init(&pDevice->aaudio.rerouteLock);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start_stream__aaudio(ma_device* pDevice, ma_AAudioStream* pStream)\n{\n    ma_aaudio_result_t resultAA;\n    ma_aaudio_stream_state_t currentState;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    resultAA = ((MA_PFN_AAudioStream_requestStart)pDevice->pContext->aaudio.AAudioStream_requestStart)(pStream);\n    if (resultAA != MA_AAUDIO_OK) {\n        return ma_result_from_aaudio(resultAA);\n    }\n\n    /* Do we actually need to wait for the device to transition into its started state? */\n\n    /* The device should be in either a starting or started state. If it's not set to started we need to wait for it to transition. It should go from starting to started. */\n    currentState = ((MA_PFN_AAudioStream_getState)pDevice->pContext->aaudio.AAudioStream_getState)(pStream);\n    if (currentState != MA_AAUDIO_STREAM_STATE_STARTED) {\n        ma_result result;\n\n        if (currentState != MA_AAUDIO_STREAM_STATE_STARTING) {\n            return MA_ERROR;   /* Expecting the stream to be a starting or started state. */\n        }\n\n        result = ma_wait_for_simple_state_transition__aaudio(pDevice->pContext, pStream, currentState, MA_AAUDIO_STREAM_STATE_STARTED);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop_stream__aaudio(ma_device* pDevice, ma_AAudioStream* pStream)\n{\n    ma_aaudio_result_t resultAA;\n    ma_aaudio_stream_state_t currentState;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (pStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    From the AAudio documentation:\n\n        The stream will stop after all of the data currently buffered has been played.\n\n    This maps with miniaudio's requirement that device's be drained which means we don't need to implement any draining logic.\n    */\n    currentState = ((MA_PFN_AAudioStream_getState)pDevice->pContext->aaudio.AAudioStream_getState)(pStream);\n    if (currentState == MA_AAUDIO_STREAM_STATE_DISCONNECTED) {\n        return MA_SUCCESS;  /* The device is disconnected. Don't try stopping it. */\n    }\n\n    resultAA = ((MA_PFN_AAudioStream_requestStop)pDevice->pContext->aaudio.AAudioStream_requestStop)(pStream);\n    if (resultAA != MA_AAUDIO_OK) {\n        return ma_result_from_aaudio(resultAA);\n    }\n\n    /* The device should be in either a stopping or stopped state. If it's not set to started we need to wait for it to transition. It should go from stopping to stopped. */\n    currentState = ((MA_PFN_AAudioStream_getState)pDevice->pContext->aaudio.AAudioStream_getState)(pStream);\n    if (currentState != MA_AAUDIO_STREAM_STATE_STOPPED) {\n        ma_result result;\n\n        if (currentState != MA_AAUDIO_STREAM_STATE_STOPPING) {\n            return MA_ERROR;   /* Expecting the stream to be a stopping or stopped state. */\n        }\n\n        result = ma_wait_for_simple_state_transition__aaudio(pDevice->pContext, pStream, currentState, MA_AAUDIO_STREAM_STATE_STOPPED);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_start__aaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_result result = ma_device_start_stream__aaudio(pDevice, (ma_AAudioStream*)pDevice->aaudio.pStreamCapture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_result result = ma_device_start_stream__aaudio(pDevice, (ma_AAudioStream*)pDevice->aaudio.pStreamPlayback);\n        if (result != MA_SUCCESS) {\n            if (pDevice->type == ma_device_type_duplex) {\n                ma_device_stop_stream__aaudio(pDevice, (ma_AAudioStream*)pDevice->aaudio.pStreamCapture);\n            }\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__aaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_result result = ma_device_stop_stream__aaudio(pDevice, (ma_AAudioStream*)pDevice->aaudio.pStreamCapture);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_result result = ma_device_stop_stream__aaudio(pDevice, (ma_AAudioStream*)pDevice->aaudio.pStreamPlayback);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    ma_device__on_notification_stopped(pDevice);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_reinit__aaudio(ma_device* pDevice, ma_device_type deviceType)\n{\n    const ma_int32 maxAttempts = 4; /* Reasonable retry limit. */\n\n    ma_result result;\n    ma_int32 iAttempt;\n\n    MA_ASSERT(pDevice != NULL);\n\n    /* We got disconnected! Retry a few times, until we find a connected device! */\n    iAttempt = 0;\n    while (iAttempt++ < maxAttempts) {        \n        /* Device tearing down? No need to reroute! */\n        if (ma_atomic_bool32_get(&pDevice->aaudio.isTearingDown)) {\n            result = MA_SUCCESS; /* Caller should continue as normal. */\n            break;\n        }\n\n        /* The first thing to do is close the streams. */\n        ma_close_streams__aaudio(pDevice);\n\n        /* Now we need to reinitialize each streams. The hardest part with this is just filling output the config and descriptors. */\n        ma_device_config deviceConfig;\n        ma_device_descriptor descriptorPlayback;\n        ma_device_descriptor descriptorCapture;\n\n        deviceConfig = ma_device_config_init(deviceType);\n        deviceConfig.playback.pDeviceID             = NULL; /* Only doing rerouting with default devices. */\n        deviceConfig.playback.shareMode             = pDevice->playback.shareMode;\n        deviceConfig.playback.format                = pDevice->playback.format;\n        deviceConfig.playback.channels              = pDevice->playback.channels;\n        deviceConfig.capture.pDeviceID              = NULL; /* Only doing rerouting with default devices. */\n        deviceConfig.capture.shareMode              = pDevice->capture.shareMode;\n        deviceConfig.capture.format                 = pDevice->capture.format;\n        deviceConfig.capture.channels               = pDevice->capture.channels;\n        deviceConfig.sampleRate                     = pDevice->sampleRate;\n        deviceConfig.aaudio.usage                   = pDevice->aaudio.usage;\n        deviceConfig.aaudio.contentType             = pDevice->aaudio.contentType;\n        deviceConfig.aaudio.inputPreset             = pDevice->aaudio.inputPreset;\n        deviceConfig.aaudio.allowedCapturePolicy    = pDevice->aaudio.allowedCapturePolicy;\n        deviceConfig.aaudio.noAutoStartAfterReroute = pDevice->aaudio.noAutoStartAfterReroute;\n        deviceConfig.periods                        = 1;\n\n        /* Try to get an accurate period size. */\n        if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n            deviceConfig.periodSizeInFrames = pDevice->playback.internalPeriodSizeInFrames;\n        } else {\n            deviceConfig.periodSizeInFrames = pDevice->capture.internalPeriodSizeInFrames;\n        }\n\n        if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n            descriptorCapture.pDeviceID           = deviceConfig.capture.pDeviceID;\n            descriptorCapture.shareMode           = deviceConfig.capture.shareMode;\n            descriptorCapture.format              = deviceConfig.capture.format;\n            descriptorCapture.channels            = deviceConfig.capture.channels;\n            descriptorCapture.sampleRate          = deviceConfig.sampleRate;\n            descriptorCapture.periodSizeInFrames  = deviceConfig.periodSizeInFrames;\n            descriptorCapture.periodCount         = deviceConfig.periods;\n        }\n\n        if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n            descriptorPlayback.pDeviceID          = deviceConfig.playback.pDeviceID;\n            descriptorPlayback.shareMode          = deviceConfig.playback.shareMode;\n            descriptorPlayback.format             = deviceConfig.playback.format;\n            descriptorPlayback.channels           = deviceConfig.playback.channels;\n            descriptorPlayback.sampleRate         = deviceConfig.sampleRate;\n            descriptorPlayback.periodSizeInFrames = deviceConfig.periodSizeInFrames;\n            descriptorPlayback.periodCount        = deviceConfig.periods;\n        }\n\n        result = ma_device_init_streams__aaudio(pDevice, &deviceConfig, &descriptorPlayback, &descriptorCapture);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[AAudio] Failed to create stream after route change.\");\n            /* Reroute failed! */\n            break;\n        }\n\n        result = ma_device_post_init(pDevice, deviceType, &descriptorPlayback, &descriptorCapture);\n        if (result != MA_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_WARNING, \"[AAudio] Failed to initialize device after route change.\");\n            ma_close_streams__aaudio(pDevice);\n            /* Reroute failed! */\n            break;\n        }\n\n        /* We'll only ever do this in response to a reroute. */\n        ma_device__on_notification_rerouted(pDevice);\n\n        /* If the device is started, start the streams. Maybe make this configurable? */\n        if (ma_device_get_state(pDevice) == ma_device_state_started) {\n            if (pDevice->aaudio.noAutoStartAfterReroute == MA_FALSE) {\n                result = ma_device_start__aaudio(pDevice);\n                if (result != MA_SUCCESS) {\n                    if (iAttempt < maxAttempts) {\n                        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[AAudio] Failed to start stream after route change, retrying(%d)\", iAttempt);\n                    } else {\n                        ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[AAudio] Failed to start stream after route change, giving up.\");\n                    }\n                }\n            } else {\n                ma_device_stop(pDevice); /* Do a full device stop so we set internal state correctly. */\n            }\n        }\n\n        if (result == MA_SUCCESS) {\n            /* Reroute successful! */\n            break;\n        }\n    }\n    \n    return result;\n}\n\nstatic ma_result ma_device_get_info__aaudio(ma_device* pDevice, ma_device_type type, ma_device_info* pDeviceInfo)\n{\n    ma_AAudioStream* pStream = NULL;\n\n    MA_ASSERT(pDevice     != NULL);\n    MA_ASSERT(type        != ma_device_type_duplex);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    if (type == ma_device_type_capture) {\n        pStream = (ma_AAudioStream*)pDevice->aaudio.pStreamCapture;\n        pDeviceInfo->id.aaudio = pDevice->capture.id.aaudio;\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);     /* Only supporting default devices. */\n    }\n    if (type == ma_device_type_playback) {\n        pStream = (ma_AAudioStream*)pDevice->aaudio.pStreamPlayback;\n        pDeviceInfo->id.aaudio = pDevice->playback.id.aaudio;\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);    /* Only supporting default devices. */\n    }\n\n    /* Safety. Should never happen. */\n    if (pStream == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    pDeviceInfo->nativeDataFormatCount = 0;\n    ma_context_add_native_data_format_from_AAudioStream__aaudio(pDevice->pContext, pStream, 0, pDeviceInfo);\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_uninit__aaudio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_aaudio);\n\n    ma_device_job_thread_uninit(&pContext->aaudio.jobThread, &pContext->allocationCallbacks);\n\n    ma_dlclose(ma_context_get_log(pContext), pContext->aaudio.hAAudio);\n    pContext->aaudio.hAAudio = NULL;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__aaudio(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n#if !defined(MA_NO_RUNTIME_LINKING)\n    size_t i;\n    const char* libNames[] = {\n        \"libaaudio.so\"\n    };\n\n    for (i = 0; i < ma_countof(libNames); ++i) {\n        pContext->aaudio.hAAudio = ma_dlopen(ma_context_get_log(pContext), libNames[i]);\n        if (pContext->aaudio.hAAudio != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->aaudio.hAAudio == NULL) {\n        return MA_FAILED_TO_INIT_BACKEND;\n    }\n\n    pContext->aaudio.AAudio_createStreamBuilder                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudio_createStreamBuilder\");\n    pContext->aaudio.AAudioStreamBuilder_delete                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_delete\");\n    pContext->aaudio.AAudioStreamBuilder_setDeviceId               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setDeviceId\");\n    pContext->aaudio.AAudioStreamBuilder_setDirection              = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setDirection\");\n    pContext->aaudio.AAudioStreamBuilder_setSharingMode            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setSharingMode\");\n    pContext->aaudio.AAudioStreamBuilder_setFormat                 = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setFormat\");\n    pContext->aaudio.AAudioStreamBuilder_setChannelCount           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setChannelCount\");\n    pContext->aaudio.AAudioStreamBuilder_setSampleRate             = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setSampleRate\");\n    pContext->aaudio.AAudioStreamBuilder_setBufferCapacityInFrames = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setBufferCapacityInFrames\");\n    pContext->aaudio.AAudioStreamBuilder_setFramesPerDataCallback  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setFramesPerDataCallback\");\n    pContext->aaudio.AAudioStreamBuilder_setDataCallback           = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setDataCallback\");\n    pContext->aaudio.AAudioStreamBuilder_setErrorCallback          = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setErrorCallback\");\n    pContext->aaudio.AAudioStreamBuilder_setPerformanceMode        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setPerformanceMode\");\n    pContext->aaudio.AAudioStreamBuilder_setUsage                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setUsage\");\n    pContext->aaudio.AAudioStreamBuilder_setContentType            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setContentType\");\n    pContext->aaudio.AAudioStreamBuilder_setInputPreset            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setInputPreset\");\n    pContext->aaudio.AAudioStreamBuilder_setAllowedCapturePolicy   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_setAllowedCapturePolicy\");\n    pContext->aaudio.AAudioStreamBuilder_openStream                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStreamBuilder_openStream\");\n    pContext->aaudio.AAudioStream_close                            = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_close\");\n    pContext->aaudio.AAudioStream_getState                         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getState\");\n    pContext->aaudio.AAudioStream_waitForStateChange               = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_waitForStateChange\");\n    pContext->aaudio.AAudioStream_getFormat                        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getFormat\");\n    pContext->aaudio.AAudioStream_getChannelCount                  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getChannelCount\");\n    pContext->aaudio.AAudioStream_getSampleRate                    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getSampleRate\");\n    pContext->aaudio.AAudioStream_getBufferCapacityInFrames        = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getBufferCapacityInFrames\");\n    pContext->aaudio.AAudioStream_getFramesPerDataCallback         = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getFramesPerDataCallback\");\n    pContext->aaudio.AAudioStream_getFramesPerBurst                = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_getFramesPerBurst\");\n    pContext->aaudio.AAudioStream_requestStart                     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_requestStart\");\n    pContext->aaudio.AAudioStream_requestStop                      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->aaudio.hAAudio, \"AAudioStream_requestStop\");\n#else\n    pContext->aaudio.AAudio_createStreamBuilder                    = (ma_proc)AAudio_createStreamBuilder;\n    pContext->aaudio.AAudioStreamBuilder_delete                    = (ma_proc)AAudioStreamBuilder_delete;\n    pContext->aaudio.AAudioStreamBuilder_setDeviceId               = (ma_proc)AAudioStreamBuilder_setDeviceId;\n    pContext->aaudio.AAudioStreamBuilder_setDirection              = (ma_proc)AAudioStreamBuilder_setDirection;\n    pContext->aaudio.AAudioStreamBuilder_setSharingMode            = (ma_proc)AAudioStreamBuilder_setSharingMode;\n    pContext->aaudio.AAudioStreamBuilder_setFormat                 = (ma_proc)AAudioStreamBuilder_setFormat;\n    pContext->aaudio.AAudioStreamBuilder_setChannelCount           = (ma_proc)AAudioStreamBuilder_setChannelCount;\n    pContext->aaudio.AAudioStreamBuilder_setSampleRate             = (ma_proc)AAudioStreamBuilder_setSampleRate;\n    pContext->aaudio.AAudioStreamBuilder_setBufferCapacityInFrames = (ma_proc)AAudioStreamBuilder_setBufferCapacityInFrames;\n    pContext->aaudio.AAudioStreamBuilder_setFramesPerDataCallback  = (ma_proc)AAudioStreamBuilder_setFramesPerDataCallback;\n    pContext->aaudio.AAudioStreamBuilder_setDataCallback           = (ma_proc)AAudioStreamBuilder_setDataCallback;\n    pContext->aaudio.AAudioStreamBuilder_setErrorCallback          = (ma_proc)AAudioStreamBuilder_setErrorCallback;\n    pContext->aaudio.AAudioStreamBuilder_setPerformanceMode        = (ma_proc)AAudioStreamBuilder_setPerformanceMode;\n    pContext->aaudio.AAudioStreamBuilder_setUsage                  = (ma_proc)AAudioStreamBuilder_setUsage;\n    pContext->aaudio.AAudioStreamBuilder_setContentType            = (ma_proc)AAudioStreamBuilder_setContentType;\n    pContext->aaudio.AAudioStreamBuilder_setInputPreset            = (ma_proc)AAudioStreamBuilder_setInputPreset;\n    #if defined(__ANDROID_API__) && __ANDROID_API__ >= 29\n    pContext->aaudio.AAudioStreamBuilder_setAllowedCapturePolicy   = (ma_proc)AAudioStreamBuilder_setAllowedCapturePolicy;\n    #endif\n    pContext->aaudio.AAudioStreamBuilder_openStream                = (ma_proc)AAudioStreamBuilder_openStream;\n    pContext->aaudio.AAudioStream_close                            = (ma_proc)AAudioStream_close;\n    pContext->aaudio.AAudioStream_getState                         = (ma_proc)AAudioStream_getState;\n    pContext->aaudio.AAudioStream_waitForStateChange               = (ma_proc)AAudioStream_waitForStateChange;\n    pContext->aaudio.AAudioStream_getFormat                        = (ma_proc)AAudioStream_getFormat;\n    pContext->aaudio.AAudioStream_getChannelCount                  = (ma_proc)AAudioStream_getChannelCount;\n    pContext->aaudio.AAudioStream_getSampleRate                    = (ma_proc)AAudioStream_getSampleRate;\n    pContext->aaudio.AAudioStream_getBufferCapacityInFrames        = (ma_proc)AAudioStream_getBufferCapacityInFrames;\n    pContext->aaudio.AAudioStream_getFramesPerDataCallback         = (ma_proc)AAudioStream_getFramesPerDataCallback;\n    pContext->aaudio.AAudioStream_getFramesPerBurst                = (ma_proc)AAudioStream_getFramesPerBurst;\n    pContext->aaudio.AAudioStream_requestStart                     = (ma_proc)AAudioStream_requestStart;\n    pContext->aaudio.AAudioStream_requestStop                      = (ma_proc)AAudioStream_requestStop;\n#endif\n\n    pCallbacks->onContextInit             = ma_context_init__aaudio;\n    pCallbacks->onContextUninit           = ma_context_uninit__aaudio;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__aaudio;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__aaudio;\n    pCallbacks->onDeviceInit              = ma_device_init__aaudio;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__aaudio;\n    pCallbacks->onDeviceStart             = ma_device_start__aaudio;\n    pCallbacks->onDeviceStop              = ma_device_stop__aaudio;\n    pCallbacks->onDeviceRead              = NULL;   /* Not used because AAudio is asynchronous. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not used because AAudio is asynchronous. */\n    pCallbacks->onDeviceDataLoop          = NULL;   /* Not used because AAudio is asynchronous. */\n    pCallbacks->onDeviceGetInfo           = ma_device_get_info__aaudio;\n\n\n    /* We need a job thread so we can deal with rerouting. */\n    {\n        ma_result result;\n        ma_device_job_thread_config jobThreadConfig;\n\n        jobThreadConfig = ma_device_job_thread_config_init();\n\n        result = ma_device_job_thread_init(&jobThreadConfig, &pContext->allocationCallbacks, &pContext->aaudio.jobThread);\n        if (result != MA_SUCCESS) {\n            ma_dlclose(ma_context_get_log(pContext), pContext->aaudio.hAAudio);\n            pContext->aaudio.hAAudio = NULL;\n            return result;\n        }\n    }\n\n\n    (void)pConfig;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_job_process__device__aaudio_reroute(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_device* pDevice;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDevice = (ma_device*)pJob->data.device.aaudio.reroute.pDevice;\n    MA_ASSERT(pDevice != NULL);\n\n    ma_mutex_lock(&pDevice->aaudio.rerouteLock);\n    {\n        /* Here is where we need to reroute the device. To do this we need to uninitialize the stream and reinitialize it. */\n        result = ma_device_reinit__aaudio(pDevice, (ma_device_type)pJob->data.device.aaudio.reroute.deviceType);\n        if (result != MA_SUCCESS) {\n            /*\n            Getting here means we failed to reroute the device. The best thing I can think of here is to\n            just stop the device.\n            */\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[AAudio] Stopping device due to reroute failure.\");\n            ma_device_stop(pDevice);\n        }\n    }\n    ma_mutex_unlock(&pDevice->aaudio.rerouteLock);\n\n    return result;\n}\n#else\n/* Getting here means there is no AAudio backend so we need a no-op job implementation. */\nstatic ma_result ma_job_process__device__aaudio_reroute(ma_job* pJob)\n{\n    return ma_job_process__noop(pJob);\n}\n#endif  /* AAudio */\n\n\n/******************************************************************************\n\nOpenSL|ES Backend\n\n******************************************************************************/\n#ifdef MA_HAS_OPENSL\n#include <SLES/OpenSLES.h>\n#ifdef MA_ANDROID\n#include <SLES/OpenSLES_Android.h>\n#endif\n\ntypedef SLresult (SLAPIENTRY * ma_slCreateEngine_proc)(SLObjectItf* pEngine, SLuint32 numOptions, SLEngineOption* pEngineOptions, SLuint32 numInterfaces, SLInterfaceID* pInterfaceIds, SLboolean* pInterfaceRequired);\n\n/* OpenSL|ES has one-per-application objects :( */\nstatic SLObjectItf g_maEngineObjectSL    = NULL;\nstatic SLEngineItf g_maEngineSL          = NULL;\nstatic ma_uint32   g_maOpenSLInitCounter = 0;\nstatic ma_spinlock g_maOpenSLSpinlock    = 0;   /* For init/uninit. */\n\n#define MA_OPENSL_OBJ(p)         (*((SLObjectItf)(p)))\n#define MA_OPENSL_OUTPUTMIX(p)   (*((SLOutputMixItf)(p)))\n#define MA_OPENSL_PLAY(p)        (*((SLPlayItf)(p)))\n#define MA_OPENSL_RECORD(p)      (*((SLRecordItf)(p)))\n\n#ifdef MA_ANDROID\n#define MA_OPENSL_BUFFERQUEUE(p) (*((SLAndroidSimpleBufferQueueItf)(p)))\n#else\n#define MA_OPENSL_BUFFERQUEUE(p) (*((SLBufferQueueItf)(p)))\n#endif\n\nstatic ma_result ma_result_from_OpenSL(SLuint32 result)\n{\n    switch (result)\n    {\n        case SL_RESULT_SUCCESS:                 return MA_SUCCESS;\n        case SL_RESULT_PRECONDITIONS_VIOLATED:  return MA_ERROR;\n        case SL_RESULT_PARAMETER_INVALID:       return MA_INVALID_ARGS;\n        case SL_RESULT_MEMORY_FAILURE:          return MA_OUT_OF_MEMORY;\n        case SL_RESULT_RESOURCE_ERROR:          return MA_INVALID_DATA;\n        case SL_RESULT_RESOURCE_LOST:           return MA_ERROR;\n        case SL_RESULT_IO_ERROR:                return MA_IO_ERROR;\n        case SL_RESULT_BUFFER_INSUFFICIENT:     return MA_NO_SPACE;\n        case SL_RESULT_CONTENT_CORRUPTED:       return MA_INVALID_DATA;\n        case SL_RESULT_CONTENT_UNSUPPORTED:     return MA_FORMAT_NOT_SUPPORTED;\n        case SL_RESULT_CONTENT_NOT_FOUND:       return MA_ERROR;\n        case SL_RESULT_PERMISSION_DENIED:       return MA_ACCESS_DENIED;\n        case SL_RESULT_FEATURE_UNSUPPORTED:     return MA_NOT_IMPLEMENTED;\n        case SL_RESULT_INTERNAL_ERROR:          return MA_ERROR;\n        case SL_RESULT_UNKNOWN_ERROR:           return MA_ERROR;\n        case SL_RESULT_OPERATION_ABORTED:       return MA_ERROR;\n        case SL_RESULT_CONTROL_LOST:            return MA_ERROR;\n        default:                                return MA_ERROR;\n    }\n}\n\n/* Converts an individual OpenSL-style channel identifier (SL_SPEAKER_FRONT_LEFT, etc.) to miniaudio. */\nstatic ma_uint8 ma_channel_id_to_ma__opensl(SLuint32 id)\n{\n    switch (id)\n    {\n        case SL_SPEAKER_FRONT_LEFT:            return MA_CHANNEL_FRONT_LEFT;\n        case SL_SPEAKER_FRONT_RIGHT:           return MA_CHANNEL_FRONT_RIGHT;\n        case SL_SPEAKER_FRONT_CENTER:          return MA_CHANNEL_FRONT_CENTER;\n        case SL_SPEAKER_LOW_FREQUENCY:         return MA_CHANNEL_LFE;\n        case SL_SPEAKER_BACK_LEFT:             return MA_CHANNEL_BACK_LEFT;\n        case SL_SPEAKER_BACK_RIGHT:            return MA_CHANNEL_BACK_RIGHT;\n        case SL_SPEAKER_FRONT_LEFT_OF_CENTER:  return MA_CHANNEL_FRONT_LEFT_CENTER;\n        case SL_SPEAKER_FRONT_RIGHT_OF_CENTER: return MA_CHANNEL_FRONT_RIGHT_CENTER;\n        case SL_SPEAKER_BACK_CENTER:           return MA_CHANNEL_BACK_CENTER;\n        case SL_SPEAKER_SIDE_LEFT:             return MA_CHANNEL_SIDE_LEFT;\n        case SL_SPEAKER_SIDE_RIGHT:            return MA_CHANNEL_SIDE_RIGHT;\n        case SL_SPEAKER_TOP_CENTER:            return MA_CHANNEL_TOP_CENTER;\n        case SL_SPEAKER_TOP_FRONT_LEFT:        return MA_CHANNEL_TOP_FRONT_LEFT;\n        case SL_SPEAKER_TOP_FRONT_CENTER:      return MA_CHANNEL_TOP_FRONT_CENTER;\n        case SL_SPEAKER_TOP_FRONT_RIGHT:       return MA_CHANNEL_TOP_FRONT_RIGHT;\n        case SL_SPEAKER_TOP_BACK_LEFT:         return MA_CHANNEL_TOP_BACK_LEFT;\n        case SL_SPEAKER_TOP_BACK_CENTER:       return MA_CHANNEL_TOP_BACK_CENTER;\n        case SL_SPEAKER_TOP_BACK_RIGHT:        return MA_CHANNEL_TOP_BACK_RIGHT;\n        default: return 0;\n    }\n}\n\n/* Converts an individual miniaudio channel identifier (MA_CHANNEL_FRONT_LEFT, etc.) to OpenSL-style. */\nstatic SLuint32 ma_channel_id_to_opensl(ma_uint8 id)\n{\n    switch (id)\n    {\n        case MA_CHANNEL_MONO:               return SL_SPEAKER_FRONT_CENTER;\n        case MA_CHANNEL_FRONT_LEFT:         return SL_SPEAKER_FRONT_LEFT;\n        case MA_CHANNEL_FRONT_RIGHT:        return SL_SPEAKER_FRONT_RIGHT;\n        case MA_CHANNEL_FRONT_CENTER:       return SL_SPEAKER_FRONT_CENTER;\n        case MA_CHANNEL_LFE:                return SL_SPEAKER_LOW_FREQUENCY;\n        case MA_CHANNEL_BACK_LEFT:          return SL_SPEAKER_BACK_LEFT;\n        case MA_CHANNEL_BACK_RIGHT:         return SL_SPEAKER_BACK_RIGHT;\n        case MA_CHANNEL_FRONT_LEFT_CENTER:  return SL_SPEAKER_FRONT_LEFT_OF_CENTER;\n        case MA_CHANNEL_FRONT_RIGHT_CENTER: return SL_SPEAKER_FRONT_RIGHT_OF_CENTER;\n        case MA_CHANNEL_BACK_CENTER:        return SL_SPEAKER_BACK_CENTER;\n        case MA_CHANNEL_SIDE_LEFT:          return SL_SPEAKER_SIDE_LEFT;\n        case MA_CHANNEL_SIDE_RIGHT:         return SL_SPEAKER_SIDE_RIGHT;\n        case MA_CHANNEL_TOP_CENTER:         return SL_SPEAKER_TOP_CENTER;\n        case MA_CHANNEL_TOP_FRONT_LEFT:     return SL_SPEAKER_TOP_FRONT_LEFT;\n        case MA_CHANNEL_TOP_FRONT_CENTER:   return SL_SPEAKER_TOP_FRONT_CENTER;\n        case MA_CHANNEL_TOP_FRONT_RIGHT:    return SL_SPEAKER_TOP_FRONT_RIGHT;\n        case MA_CHANNEL_TOP_BACK_LEFT:      return SL_SPEAKER_TOP_BACK_LEFT;\n        case MA_CHANNEL_TOP_BACK_CENTER:    return SL_SPEAKER_TOP_BACK_CENTER;\n        case MA_CHANNEL_TOP_BACK_RIGHT:     return SL_SPEAKER_TOP_BACK_RIGHT;\n        default: return 0;\n    }\n}\n\n/* Converts a channel mapping to an OpenSL-style channel mask. */\nstatic SLuint32 ma_channel_map_to_channel_mask__opensl(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    SLuint32 channelMask = 0;\n    ma_uint32 iChannel;\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        channelMask |= ma_channel_id_to_opensl(pChannelMap[iChannel]);\n    }\n\n    return channelMask;\n}\n\n/* Converts an OpenSL-style channel mask to a miniaudio channel map. */\nstatic void ma_channel_mask_to_channel_map__opensl(SLuint32 channelMask, ma_uint32 channels, ma_channel* pChannelMap)\n{\n    if (channels == 1 && channelMask == 0) {\n        pChannelMap[0] = MA_CHANNEL_MONO;\n    } else if (channels == 2 && channelMask == 0) {\n        pChannelMap[0] = MA_CHANNEL_FRONT_LEFT;\n        pChannelMap[1] = MA_CHANNEL_FRONT_RIGHT;\n    } else {\n        if (channels == 1 && (channelMask & SL_SPEAKER_FRONT_CENTER) != 0) {\n            pChannelMap[0] = MA_CHANNEL_MONO;\n        } else {\n            /* Just iterate over each bit. */\n            ma_uint32 iChannel = 0;\n            ma_uint32 iBit;\n            for (iBit = 0; iBit < 32 && iChannel < channels; ++iBit) {\n                SLuint32 bitValue = (channelMask & (1UL << iBit));\n                if (bitValue != 0) {\n                    /* The bit is set. */\n                    pChannelMap[iChannel] = ma_channel_id_to_ma__opensl(bitValue);\n                    iChannel += 1;\n                }\n            }\n        }\n    }\n}\n\nstatic SLuint32 ma_round_to_standard_sample_rate__opensl(SLuint32 samplesPerSec)\n{\n    if (samplesPerSec <= SL_SAMPLINGRATE_8) {\n        return SL_SAMPLINGRATE_8;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_11_025) {\n        return SL_SAMPLINGRATE_11_025;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_12) {\n        return SL_SAMPLINGRATE_12;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_16) {\n        return SL_SAMPLINGRATE_16;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_22_05) {\n        return SL_SAMPLINGRATE_22_05;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_24) {\n        return SL_SAMPLINGRATE_24;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_32) {\n        return SL_SAMPLINGRATE_32;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_44_1) {\n        return SL_SAMPLINGRATE_44_1;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_48) {\n        return SL_SAMPLINGRATE_48;\n    }\n\n    /* Android doesn't support more than 48000. */\n#ifndef MA_ANDROID\n    if (samplesPerSec <= SL_SAMPLINGRATE_64) {\n        return SL_SAMPLINGRATE_64;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_88_2) {\n        return SL_SAMPLINGRATE_88_2;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_96) {\n        return SL_SAMPLINGRATE_96;\n    }\n    if (samplesPerSec <= SL_SAMPLINGRATE_192) {\n        return SL_SAMPLINGRATE_192;\n    }\n#endif\n\n    return SL_SAMPLINGRATE_16;\n}\n\n\nstatic SLint32 ma_to_stream_type__opensl(ma_opensl_stream_type streamType)\n{\n    switch (streamType) {\n        case ma_opensl_stream_type_voice:        return SL_ANDROID_STREAM_VOICE;\n        case ma_opensl_stream_type_system:       return SL_ANDROID_STREAM_SYSTEM;\n        case ma_opensl_stream_type_ring:         return SL_ANDROID_STREAM_RING;\n        case ma_opensl_stream_type_media:        return SL_ANDROID_STREAM_MEDIA;\n        case ma_opensl_stream_type_alarm:        return SL_ANDROID_STREAM_ALARM;\n        case ma_opensl_stream_type_notification: return SL_ANDROID_STREAM_NOTIFICATION;\n        default: break;\n    }\n\n    return SL_ANDROID_STREAM_VOICE;\n}\n\nstatic SLint32 ma_to_recording_preset__opensl(ma_opensl_recording_preset recordingPreset)\n{\n    switch (recordingPreset) {\n        case ma_opensl_recording_preset_generic:             return SL_ANDROID_RECORDING_PRESET_GENERIC;\n        case ma_opensl_recording_preset_camcorder:           return SL_ANDROID_RECORDING_PRESET_CAMCORDER;\n        case ma_opensl_recording_preset_voice_recognition:   return SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;\n        case ma_opensl_recording_preset_voice_communication: return SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;\n        case ma_opensl_recording_preset_voice_unprocessed:   return SL_ANDROID_RECORDING_PRESET_UNPROCESSED;\n        default: break;\n    }\n\n    return SL_ANDROID_RECORDING_PRESET_NONE;\n}\n\n\nstatic ma_result ma_context_enumerate_devices__opensl(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 cbResult;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it and then attempted to enumerate devices. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /*\n    TODO: Test Me.\n\n    This is currently untested, so for now we are just returning default devices.\n    */\n#if 0 && !defined(MA_ANDROID)\n    ma_bool32 isTerminated = MA_FALSE;\n\n    SLuint32 pDeviceIDs[128];\n    SLint32 deviceCount = sizeof(pDeviceIDs) / sizeof(pDeviceIDs[0]);\n\n    SLAudioIODeviceCapabilitiesItf deviceCaps;\n    SLresult resultSL = (*g_maEngineObjectSL)->GetInterface(g_maEngineObjectSL, (SLInterfaceID)pContext->opensl.SL_IID_AUDIOIODEVICECAPABILITIES, &deviceCaps);\n    if (resultSL != SL_RESULT_SUCCESS) {\n        /* The interface may not be supported so just report a default device. */\n        goto return_default_device;\n    }\n\n    /* Playback */\n    if (!isTerminated) {\n        resultSL = (*deviceCaps)->GetAvailableAudioOutputs(deviceCaps, &deviceCount, pDeviceIDs);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        for (SLint32 iDevice = 0; iDevice < deviceCount; ++iDevice) {\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            deviceInfo.id.opensl = pDeviceIDs[iDevice];\n\n            SLAudioOutputDescriptor desc;\n            resultSL = (*deviceCaps)->QueryAudioOutputCapabilities(deviceCaps, deviceInfo.id.opensl, &desc);\n            if (resultSL == SL_RESULT_SUCCESS) {\n                ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), (const char*)desc.pDeviceName, (size_t)-1);\n\n                ma_bool32 cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n                if (cbResult == MA_FALSE) {\n                    isTerminated = MA_TRUE;\n                    break;\n                }\n            }\n        }\n    }\n\n    /* Capture */\n    if (!isTerminated) {\n        resultSL = (*deviceCaps)->GetAvailableAudioInputs(deviceCaps, &deviceCount, pDeviceIDs);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        for (SLint32 iDevice = 0; iDevice < deviceCount; ++iDevice) {\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            deviceInfo.id.opensl = pDeviceIDs[iDevice];\n\n            SLAudioInputDescriptor desc;\n            resultSL = (*deviceCaps)->QueryAudioInputCapabilities(deviceCaps, deviceInfo.id.opensl, &desc);\n            if (resultSL == SL_RESULT_SUCCESS) {\n                ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), (const char*)desc.deviceName, (size_t)-1);\n\n                ma_bool32 cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n                if (cbResult == MA_FALSE) {\n                    isTerminated = MA_TRUE;\n                    break;\n                }\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n#else\n    goto return_default_device;\n#endif\n\nreturn_default_device:;\n    cbResult = MA_TRUE;\n\n    /* Playback. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        deviceInfo.id.opensl = SL_DEFAULTDEVICEID_AUDIOOUTPUT;\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n        cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n    }\n\n    /* Capture. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        deviceInfo.id.opensl = SL_DEFAULTDEVICEID_AUDIOINPUT;\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n        cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_context_add_data_format_ex__opensl(ma_context* pContext, ma_format format, ma_uint32 channels, ma_uint32 sampleRate, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pContext    != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].format     = format;\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].channels   = channels;\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].sampleRate = sampleRate;\n    pDeviceInfo->nativeDataFormats[pDeviceInfo->nativeDataFormatCount].flags      = 0;\n    pDeviceInfo->nativeDataFormatCount += 1;\n}\n\nstatic void ma_context_add_data_format__opensl(ma_context* pContext, ma_format format, ma_device_info* pDeviceInfo)\n{\n    ma_uint32 minChannels   = 1;\n    ma_uint32 maxChannels   = 2;\n    ma_uint32 minSampleRate = (ma_uint32)ma_standard_sample_rate_8000;\n    ma_uint32 maxSampleRate = (ma_uint32)ma_standard_sample_rate_48000;\n    ma_uint32 iChannel;\n    ma_uint32 iSampleRate;\n\n    MA_ASSERT(pContext    != NULL);\n    MA_ASSERT(pDeviceInfo != NULL);\n\n    /*\n    Each sample format can support mono and stereo, and we'll support a small subset of standard\n    rates (up to 48000). A better solution would be to somehow find a native sample rate.\n    */\n    for (iChannel = minChannels; iChannel < maxChannels; iChannel += 1) {\n        for (iSampleRate = 0; iSampleRate < ma_countof(g_maStandardSampleRatePriorities); iSampleRate += 1) {\n            ma_uint32 standardSampleRate = g_maStandardSampleRatePriorities[iSampleRate];\n            if (standardSampleRate >= minSampleRate && standardSampleRate <= maxSampleRate) {\n                ma_context_add_data_format_ex__opensl(pContext, format, iChannel, standardSampleRate, pDeviceInfo);\n            }\n        }\n    }\n}\n\nstatic ma_result ma_context_get_device_info__opensl(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pContext != NULL);\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it and then attempted to get device info. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /*\n    TODO: Test Me.\n\n    This is currently untested, so for now we are just returning default devices.\n    */\n#if 0 && !defined(MA_ANDROID)\n    SLAudioIODeviceCapabilitiesItf deviceCaps;\n    SLresult resultSL = (*g_maEngineObjectSL)->GetInterface(g_maEngineObjectSL, (SLInterfaceID)pContext->opensl.SL_IID_AUDIOIODEVICECAPABILITIES, &deviceCaps);\n    if (resultSL != SL_RESULT_SUCCESS) {\n        /* The interface may not be supported so just report a default device. */\n        goto return_default_device;\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        SLAudioOutputDescriptor desc;\n        resultSL = (*deviceCaps)->QueryAudioOutputCapabilities(deviceCaps, pDeviceID->opensl, &desc);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), (const char*)desc.pDeviceName, (size_t)-1);\n    } else {\n        SLAudioInputDescriptor desc;\n        resultSL = (*deviceCaps)->QueryAudioInputCapabilities(deviceCaps, pDeviceID->opensl, &desc);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), (const char*)desc.deviceName, (size_t)-1);\n    }\n\n    goto return_detailed_info;\n#else\n    goto return_default_device;\n#endif\n\nreturn_default_device:\n    if (pDeviceID != NULL) {\n        if ((deviceType == ma_device_type_playback && pDeviceID->opensl != SL_DEFAULTDEVICEID_AUDIOOUTPUT) ||\n            (deviceType == ma_device_type_capture  && pDeviceID->opensl != SL_DEFAULTDEVICEID_AUDIOINPUT)) {\n            return MA_NO_DEVICE;   /* Don't know the device. */\n        }\n    }\n\n    /* ID and Name / Description */\n    if (deviceType == ma_device_type_playback) {\n        pDeviceInfo->id.opensl = SL_DEFAULTDEVICEID_AUDIOOUTPUT;\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n    } else {\n        pDeviceInfo->id.opensl = SL_DEFAULTDEVICEID_AUDIOINPUT;\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n    }\n\n    pDeviceInfo->isDefault = MA_TRUE;\n\n    goto return_detailed_info;\n\n\nreturn_detailed_info:\n\n    /*\n    For now we're just outputting a set of values that are supported by the API but not necessarily supported\n    by the device natively. Later on we should work on this so that it more closely reflects the device's\n    actual native format.\n    */\n    pDeviceInfo->nativeDataFormatCount = 0;\n#if defined(MA_ANDROID) && __ANDROID_API__ >= 21\n    ma_context_add_data_format__opensl(pContext, ma_format_f32, pDeviceInfo);\n#endif\n    ma_context_add_data_format__opensl(pContext, ma_format_s16, pDeviceInfo);\n    ma_context_add_data_format__opensl(pContext, ma_format_u8,  pDeviceInfo);\n\n    return MA_SUCCESS;\n}\n\n\n#ifdef MA_ANDROID\n/*void ma_buffer_queue_callback_capture__opensl_android(SLAndroidSimpleBufferQueueItf pBufferQueue, SLuint32 eventFlags, const void* pBuffer, SLuint32 bufferSize, SLuint32 dataUsed, void* pContext)*/\nstatic void ma_buffer_queue_callback_capture__opensl_android(SLAndroidSimpleBufferQueueItf pBufferQueue, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    size_t periodSizeInBytes;\n    ma_uint8* pBuffer;\n    SLresult resultSL;\n\n    MA_ASSERT(pDevice != NULL);\n\n    (void)pBufferQueue;\n\n    /*\n    For now, don't do anything unless the buffer was fully processed. From what I can tell, it looks like\n    OpenSL|ES 1.1 improves on buffer queues to the point that we could much more intelligently handle this,\n    but unfortunately it looks like Android is only supporting OpenSL|ES 1.0.1 for now :(\n    */\n\n    /* Don't do anything if the device is not started. */\n    if (ma_device_get_state(pDevice) != ma_device_state_started) {\n        return;\n    }\n\n    /* Don't do anything if the device is being drained. */\n    if (pDevice->opensl.isDrainingCapture) {\n        return;\n    }\n\n    periodSizeInBytes = pDevice->capture.internalPeriodSizeInFrames * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n    pBuffer = pDevice->opensl.pBufferCapture + (pDevice->opensl.currentBufferIndexCapture * periodSizeInBytes);\n\n    ma_device_handle_backend_data_callback(pDevice, NULL, pBuffer, pDevice->capture.internalPeriodSizeInFrames);\n\n    resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueueCapture)->Enqueue((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture, pBuffer, periodSizeInBytes);\n    if (resultSL != SL_RESULT_SUCCESS) {\n        return;\n    }\n\n    pDevice->opensl.currentBufferIndexCapture = (pDevice->opensl.currentBufferIndexCapture + 1) % pDevice->capture.internalPeriods;\n}\n\nstatic void ma_buffer_queue_callback_playback__opensl_android(SLAndroidSimpleBufferQueueItf pBufferQueue, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    size_t periodSizeInBytes;\n    ma_uint8* pBuffer;\n    SLresult resultSL;\n\n    MA_ASSERT(pDevice != NULL);\n\n    (void)pBufferQueue;\n\n    /* Don't do anything if the device is not started. */\n    if (ma_device_get_state(pDevice) != ma_device_state_started) {\n        return;\n    }\n\n    /* Don't do anything if the device is being drained. */\n    if (pDevice->opensl.isDrainingPlayback) {\n        return;\n    }\n\n    periodSizeInBytes = pDevice->playback.internalPeriodSizeInFrames * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n    pBuffer = pDevice->opensl.pBufferPlayback + (pDevice->opensl.currentBufferIndexPlayback * periodSizeInBytes);\n\n    ma_device_handle_backend_data_callback(pDevice, pBuffer, NULL, pDevice->playback.internalPeriodSizeInFrames);\n\n    resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueuePlayback)->Enqueue((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueuePlayback, pBuffer, periodSizeInBytes);\n    if (resultSL != SL_RESULT_SUCCESS) {\n        return;\n    }\n\n    pDevice->opensl.currentBufferIndexPlayback = (pDevice->opensl.currentBufferIndexPlayback + 1) % pDevice->playback.internalPeriods;\n}\n#endif\n\nstatic ma_result ma_device_uninit__opensl(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it before uninitializing the device. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->opensl.pAudioRecorderObj) {\n            MA_OPENSL_OBJ(pDevice->opensl.pAudioRecorderObj)->Destroy((SLObjectItf)pDevice->opensl.pAudioRecorderObj);\n        }\n\n        ma_free(pDevice->opensl.pBufferCapture, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        if (pDevice->opensl.pAudioPlayerObj) {\n            MA_OPENSL_OBJ(pDevice->opensl.pAudioPlayerObj)->Destroy((SLObjectItf)pDevice->opensl.pAudioPlayerObj);\n        }\n        if (pDevice->opensl.pOutputMixObj) {\n            MA_OPENSL_OBJ(pDevice->opensl.pOutputMixObj)->Destroy((SLObjectItf)pDevice->opensl.pOutputMixObj);\n        }\n\n        ma_free(pDevice->opensl.pBufferPlayback, &pDevice->pContext->allocationCallbacks);\n    }\n\n    return MA_SUCCESS;\n}\n\n#if defined(MA_ANDROID) && __ANDROID_API__ >= 21\ntypedef SLAndroidDataFormat_PCM_EX  ma_SLDataFormat_PCM;\n#else\ntypedef SLDataFormat_PCM            ma_SLDataFormat_PCM;\n#endif\n\nstatic ma_result ma_SLDataFormat_PCM_init__opensl(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, const ma_channel* channelMap, ma_SLDataFormat_PCM* pDataFormat)\n{\n    /* We need to convert our format/channels/rate so that they aren't set to default. */\n    if (format == ma_format_unknown) {\n        format = MA_DEFAULT_FORMAT;\n    }\n    if (channels == 0) {\n        channels = MA_DEFAULT_CHANNELS;\n    }\n    if (sampleRate == 0) {\n        sampleRate = MA_DEFAULT_SAMPLE_RATE;\n    }\n\n#if defined(MA_ANDROID) && __ANDROID_API__ >= 21\n    if (format == ma_format_f32) {\n        pDataFormat->formatType     = SL_ANDROID_DATAFORMAT_PCM_EX;\n        pDataFormat->representation = SL_ANDROID_PCM_REPRESENTATION_FLOAT;\n    } else {\n        pDataFormat->formatType = SL_DATAFORMAT_PCM;\n    }\n#else\n    pDataFormat->formatType = SL_DATAFORMAT_PCM;\n#endif\n\n    pDataFormat->numChannels   = channels;\n    ((SLDataFormat_PCM*)pDataFormat)->samplesPerSec = ma_round_to_standard_sample_rate__opensl(sampleRate * 1000);  /* In millihertz. Annoyingly, the sample rate variable is named differently between SLAndroidDataFormat_PCM_EX and SLDataFormat_PCM */\n    pDataFormat->bitsPerSample = ma_get_bytes_per_sample(format) * 8;\n    pDataFormat->channelMask   = ma_channel_map_to_channel_mask__opensl(channelMap, channels);\n    pDataFormat->endianness    = (ma_is_little_endian()) ? SL_BYTEORDER_LITTLEENDIAN : SL_BYTEORDER_BIGENDIAN;\n\n    /*\n    Android has a few restrictions on the format as documented here: https://developer.android.com/ndk/guides/audio/opensl-for-android.html\n     - Only mono and stereo is supported.\n     - Only u8 and s16 formats are supported.\n     - Maximum sample rate of 48000.\n    */\n#ifdef MA_ANDROID\n    if (pDataFormat->numChannels > 2) {\n        pDataFormat->numChannels = 2;\n    }\n#if __ANDROID_API__ >= 21\n    if (pDataFormat->formatType == SL_ANDROID_DATAFORMAT_PCM_EX) {\n        /* It's floating point. */\n        MA_ASSERT(pDataFormat->representation == SL_ANDROID_PCM_REPRESENTATION_FLOAT);\n        if (pDataFormat->bitsPerSample > 32) {\n            pDataFormat->bitsPerSample = 32;\n        }\n    } else {\n        if (pDataFormat->bitsPerSample > 16) {\n            pDataFormat->bitsPerSample = 16;\n        }\n    }\n#else\n    if (pDataFormat->bitsPerSample > 16) {\n        pDataFormat->bitsPerSample = 16;\n    }\n#endif\n    if (((SLDataFormat_PCM*)pDataFormat)->samplesPerSec > SL_SAMPLINGRATE_48) {\n        ((SLDataFormat_PCM*)pDataFormat)->samplesPerSec = SL_SAMPLINGRATE_48;\n    }\n#endif\n\n    pDataFormat->containerSize = pDataFormat->bitsPerSample;  /* Always tightly packed for now. */\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_deconstruct_SLDataFormat_PCM__opensl(ma_SLDataFormat_PCM* pDataFormat, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_bool32 isFloatingPoint = MA_FALSE;\n#if defined(MA_ANDROID) && __ANDROID_API__ >= 21\n    if (pDataFormat->formatType == SL_ANDROID_DATAFORMAT_PCM_EX) {\n        MA_ASSERT(pDataFormat->representation == SL_ANDROID_PCM_REPRESENTATION_FLOAT);\n        isFloatingPoint = MA_TRUE;\n    }\n#endif\n    if (isFloatingPoint) {\n        if (pDataFormat->bitsPerSample == 32) {\n            *pFormat = ma_format_f32;\n        }\n    } else {\n        if (pDataFormat->bitsPerSample == 8) {\n            *pFormat = ma_format_u8;\n        } else if (pDataFormat->bitsPerSample == 16) {\n            *pFormat = ma_format_s16;\n        } else if (pDataFormat->bitsPerSample == 24) {\n            *pFormat = ma_format_s24;\n        } else if (pDataFormat->bitsPerSample == 32) {\n            *pFormat = ma_format_s32;\n        }\n    }\n\n    *pChannels   = pDataFormat->numChannels;\n    *pSampleRate = ((SLDataFormat_PCM*)pDataFormat)->samplesPerSec / 1000;\n    ma_channel_mask_to_channel_map__opensl(pDataFormat->channelMask, ma_min(pDataFormat->numChannels, channelMapCap), pChannelMap);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_init__opensl(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n#ifdef MA_ANDROID\n    SLDataLocator_AndroidSimpleBufferQueue queue;\n    SLresult resultSL;\n    size_t bufferSizeInBytes;\n    SLInterfaceID itfIDs[2];\n    const SLboolean itfIDsRequired[] = {\n        SL_BOOLEAN_TRUE,    /* SL_IID_ANDROIDSIMPLEBUFFERQUEUE */\n        SL_BOOLEAN_FALSE    /* SL_IID_ANDROIDCONFIGURATION */\n    };\n#endif\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it and then attempted to initialize a new device. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /*\n    For now, only supporting Android implementations of OpenSL|ES since that's the only one I've\n    been able to test with and I currently depend on Android-specific extensions (simple buffer\n    queues).\n    */\n#ifdef MA_ANDROID\n    itfIDs[0] = (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDSIMPLEBUFFERQUEUE;\n    itfIDs[1] = (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDCONFIGURATION;\n\n    /* No exclusive mode with OpenSL|ES. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    /* Now we can start initializing the device properly. */\n    MA_ASSERT(pDevice != NULL);\n    MA_ZERO_OBJECT(&pDevice->opensl);\n\n    queue.locatorType = SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE;\n\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        ma_SLDataFormat_PCM pcm;\n        SLDataLocator_IODevice locatorDevice;\n        SLDataSource source;\n        SLDataSink sink;\n        SLAndroidConfigurationItf pRecorderConfig;\n\n        ma_SLDataFormat_PCM_init__opensl(pDescriptorCapture->format, pDescriptorCapture->channels, pDescriptorCapture->sampleRate, pDescriptorCapture->channelMap, &pcm);\n\n        locatorDevice.locatorType = SL_DATALOCATOR_IODEVICE;\n        locatorDevice.deviceType  = SL_IODEVICE_AUDIOINPUT;\n        locatorDevice.deviceID    = SL_DEFAULTDEVICEID_AUDIOINPUT;  /* Must always use the default device with Android. */\n        locatorDevice.device      = NULL;\n\n        source.pLocator = &locatorDevice;\n        source.pFormat  = NULL;\n\n        queue.numBuffers = pDescriptorCapture->periodCount;\n\n        sink.pLocator = &queue;\n        sink.pFormat  = (SLDataFormat_PCM*)&pcm;\n\n        resultSL = (*g_maEngineSL)->CreateAudioRecorder(g_maEngineSL, (SLObjectItf*)&pDevice->opensl.pAudioRecorderObj, &source, &sink, ma_countof(itfIDs), itfIDs, itfIDsRequired);\n        if (resultSL == SL_RESULT_CONTENT_UNSUPPORTED || resultSL == SL_RESULT_PARAMETER_INVALID) {\n            /* Unsupported format. Fall back to something safer and try again. If this fails, just abort. */\n            pcm.formatType    = SL_DATAFORMAT_PCM;\n            pcm.numChannels   = 1;\n            ((SLDataFormat_PCM*)&pcm)->samplesPerSec = SL_SAMPLINGRATE_16;  /* The name of the sample rate variable is different between SLAndroidDataFormat_PCM_EX and SLDataFormat_PCM. */\n            pcm.bitsPerSample = 16;\n            pcm.containerSize = pcm.bitsPerSample;  /* Always tightly packed for now. */\n            pcm.channelMask   = 0;\n            resultSL = (*g_maEngineSL)->CreateAudioRecorder(g_maEngineSL, (SLObjectItf*)&pDevice->opensl.pAudioRecorderObj, &source, &sink, ma_countof(itfIDs), itfIDs, itfIDsRequired);\n        }\n\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to create audio recorder.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n\n        /* Set the recording preset before realizing the player. */\n        if (pConfig->opensl.recordingPreset != ma_opensl_recording_preset_default) {\n            resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioRecorderObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioRecorderObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDCONFIGURATION, &pRecorderConfig);\n            if (resultSL == SL_RESULT_SUCCESS) {\n                SLint32 recordingPreset = ma_to_recording_preset__opensl(pConfig->opensl.recordingPreset);\n                resultSL = (*pRecorderConfig)->SetConfiguration(pRecorderConfig, SL_ANDROID_KEY_RECORDING_PRESET, &recordingPreset, sizeof(SLint32));\n                if (resultSL != SL_RESULT_SUCCESS) {\n                    /* Failed to set the configuration. Just keep going. */\n                }\n            }\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioRecorderObj)->Realize((SLObjectItf)pDevice->opensl.pAudioRecorderObj, SL_BOOLEAN_FALSE);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to realize audio recorder.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioRecorderObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioRecorderObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_RECORD, &pDevice->opensl.pAudioRecorder);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to retrieve SL_IID_RECORD interface.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioRecorderObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioRecorderObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &pDevice->opensl.pBufferQueueCapture);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to retrieve SL_IID_ANDROIDSIMPLEBUFFERQUEUE interface.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueueCapture)->RegisterCallback((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture, ma_buffer_queue_callback_capture__opensl_android, pDevice);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to register buffer queue callback.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        /* The internal format is determined by the \"pcm\" object. */\n        ma_deconstruct_SLDataFormat_PCM__opensl(&pcm, &pDescriptorCapture->format, &pDescriptorCapture->channels, &pDescriptorCapture->sampleRate, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap));\n\n        /* Buffer. */\n        pDescriptorCapture->periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptorCapture, pDescriptorCapture->sampleRate, pConfig->performanceProfile);\n        pDevice->opensl.currentBufferIndexCapture = 0;\n\n        bufferSizeInBytes = pDescriptorCapture->periodSizeInFrames * ma_get_bytes_per_frame(pDescriptorCapture->format, pDescriptorCapture->channels) * pDescriptorCapture->periodCount;\n        pDevice->opensl.pBufferCapture = (ma_uint8*)ma_calloc(bufferSizeInBytes, &pDevice->pContext->allocationCallbacks);\n        if (pDevice->opensl.pBufferCapture == NULL) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to allocate memory for data buffer.\");\n            return MA_OUT_OF_MEMORY;\n        }\n        MA_ZERO_MEMORY(pDevice->opensl.pBufferCapture, bufferSizeInBytes);\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        ma_SLDataFormat_PCM pcm;\n        SLDataSource source;\n        SLDataLocator_OutputMix outmixLocator;\n        SLDataSink sink;\n        SLAndroidConfigurationItf pPlayerConfig;\n\n        ma_SLDataFormat_PCM_init__opensl(pDescriptorPlayback->format, pDescriptorPlayback->channels, pDescriptorPlayback->sampleRate, pDescriptorPlayback->channelMap, &pcm);\n\n        resultSL = (*g_maEngineSL)->CreateOutputMix(g_maEngineSL, (SLObjectItf*)&pDevice->opensl.pOutputMixObj, 0, NULL, NULL);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to create output mix.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pOutputMixObj)->Realize((SLObjectItf)pDevice->opensl.pOutputMixObj, SL_BOOLEAN_FALSE);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to realize output mix object.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pOutputMixObj)->GetInterface((SLObjectItf)pDevice->opensl.pOutputMixObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_OUTPUTMIX, &pDevice->opensl.pOutputMix);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to retrieve SL_IID_OUTPUTMIX interface.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        /* Set the output device. */\n        if (pDescriptorPlayback->pDeviceID != NULL) {\n            SLuint32 deviceID_OpenSL = pDescriptorPlayback->pDeviceID->opensl;\n            MA_OPENSL_OUTPUTMIX(pDevice->opensl.pOutputMix)->ReRoute((SLOutputMixItf)pDevice->opensl.pOutputMix, 1, &deviceID_OpenSL);\n        }\n\n        queue.numBuffers = pDescriptorPlayback->periodCount;\n\n        source.pLocator = &queue;\n        source.pFormat  = (SLDataFormat_PCM*)&pcm;\n\n        outmixLocator.locatorType = SL_DATALOCATOR_OUTPUTMIX;\n        outmixLocator.outputMix   = (SLObjectItf)pDevice->opensl.pOutputMixObj;\n\n        sink.pLocator = &outmixLocator;\n        sink.pFormat  = NULL;\n\n        resultSL = (*g_maEngineSL)->CreateAudioPlayer(g_maEngineSL, (SLObjectItf*)&pDevice->opensl.pAudioPlayerObj, &source, &sink, ma_countof(itfIDs), itfIDs, itfIDsRequired);\n        if (resultSL == SL_RESULT_CONTENT_UNSUPPORTED || resultSL == SL_RESULT_PARAMETER_INVALID) {\n            /* Unsupported format. Fall back to something safer and try again. If this fails, just abort. */\n            pcm.formatType = SL_DATAFORMAT_PCM;\n            pcm.numChannels = 2;\n            ((SLDataFormat_PCM*)&pcm)->samplesPerSec = SL_SAMPLINGRATE_16;\n            pcm.bitsPerSample = 16;\n            pcm.containerSize = pcm.bitsPerSample;  /* Always tightly packed for now. */\n            pcm.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;\n            resultSL = (*g_maEngineSL)->CreateAudioPlayer(g_maEngineSL, (SLObjectItf*)&pDevice->opensl.pAudioPlayerObj, &source, &sink, ma_countof(itfIDs), itfIDs, itfIDsRequired);\n        }\n\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to create audio player.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n\n        /* Set the stream type before realizing the player. */\n        if (pConfig->opensl.streamType != ma_opensl_stream_type_default) {\n            resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioPlayerObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioPlayerObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDCONFIGURATION, &pPlayerConfig);\n            if (resultSL == SL_RESULT_SUCCESS) {\n                SLint32 streamType = ma_to_stream_type__opensl(pConfig->opensl.streamType);\n                resultSL = (*pPlayerConfig)->SetConfiguration(pPlayerConfig, SL_ANDROID_KEY_STREAM_TYPE, &streamType, sizeof(SLint32));\n                if (resultSL != SL_RESULT_SUCCESS) {\n                    /* Failed to set the configuration. Just keep going. */\n                }\n            }\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioPlayerObj)->Realize((SLObjectItf)pDevice->opensl.pAudioPlayerObj, SL_BOOLEAN_FALSE);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to realize audio player.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioPlayerObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioPlayerObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_PLAY, &pDevice->opensl.pAudioPlayer);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to retrieve SL_IID_PLAY interface.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_OBJ(pDevice->opensl.pAudioPlayerObj)->GetInterface((SLObjectItf)pDevice->opensl.pAudioPlayerObj, (SLInterfaceID)pDevice->pContext->opensl.SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &pDevice->opensl.pBufferQueuePlayback);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to retrieve SL_IID_ANDROIDSIMPLEBUFFERQUEUE interface.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueuePlayback)->RegisterCallback((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueuePlayback, ma_buffer_queue_callback_playback__opensl_android, pDevice);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to register buffer queue callback.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        /* The internal format is determined by the \"pcm\" object. */\n        ma_deconstruct_SLDataFormat_PCM__opensl(&pcm, &pDescriptorPlayback->format, &pDescriptorPlayback->channels, &pDescriptorPlayback->sampleRate, pDescriptorPlayback->channelMap, ma_countof(pDescriptorPlayback->channelMap));\n\n        /* Buffer. */\n        pDescriptorPlayback->periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_descriptor(pDescriptorPlayback, pDescriptorPlayback->sampleRate, pConfig->performanceProfile);\n        pDevice->opensl.currentBufferIndexPlayback   = 0;\n\n        bufferSizeInBytes = pDescriptorPlayback->periodSizeInFrames * ma_get_bytes_per_frame(pDescriptorPlayback->format, pDescriptorPlayback->channels) * pDescriptorPlayback->periodCount;\n        pDevice->opensl.pBufferPlayback = (ma_uint8*)ma_calloc(bufferSizeInBytes, &pDevice->pContext->allocationCallbacks);\n        if (pDevice->opensl.pBufferPlayback == NULL) {\n            ma_device_uninit__opensl(pDevice);\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to allocate memory for data buffer.\");\n            return MA_OUT_OF_MEMORY;\n        }\n        MA_ZERO_MEMORY(pDevice->opensl.pBufferPlayback, bufferSizeInBytes);\n    }\n\n    return MA_SUCCESS;\n#else\n    return MA_NO_BACKEND;   /* Non-Android implementations are not supported. */\n#endif\n}\n\nstatic ma_result ma_device_start__opensl(ma_device* pDevice)\n{\n    SLresult resultSL;\n    size_t periodSizeInBytes;\n    ma_uint32 iPeriod;\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it and then attempted to start the device. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        resultSL = MA_OPENSL_RECORD(pDevice->opensl.pAudioRecorder)->SetRecordState((SLRecordItf)pDevice->opensl.pAudioRecorder, SL_RECORDSTATE_RECORDING);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to start internal capture device.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        periodSizeInBytes = pDevice->capture.internalPeriodSizeInFrames * ma_get_bytes_per_frame(pDevice->capture.internalFormat, pDevice->capture.internalChannels);\n        for (iPeriod = 0; iPeriod < pDevice->capture.internalPeriods; ++iPeriod) {\n            resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueueCapture)->Enqueue((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture, pDevice->opensl.pBufferCapture + (periodSizeInBytes * iPeriod), periodSizeInBytes);\n            if (resultSL != SL_RESULT_SUCCESS) {\n                MA_OPENSL_RECORD(pDevice->opensl.pAudioRecorder)->SetRecordState((SLRecordItf)pDevice->opensl.pAudioRecorder, SL_RECORDSTATE_STOPPED);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to enqueue buffer for capture device.\");\n                return ma_result_from_OpenSL(resultSL);\n            }\n        }\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        resultSL = MA_OPENSL_PLAY(pDevice->opensl.pAudioPlayer)->SetPlayState((SLPlayItf)pDevice->opensl.pAudioPlayer, SL_PLAYSTATE_PLAYING);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to start internal playback device.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        /* In playback mode (no duplex) we need to load some initial buffers. In duplex mode we need to enqueue silent buffers. */\n        if (pDevice->type == ma_device_type_duplex) {\n            MA_ZERO_MEMORY(pDevice->opensl.pBufferPlayback, pDevice->playback.internalPeriodSizeInFrames * pDevice->playback.internalPeriods * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels));\n        } else {\n            ma_device__read_frames_from_client(pDevice, pDevice->playback.internalPeriodSizeInFrames * pDevice->playback.internalPeriods, pDevice->opensl.pBufferPlayback);\n        }\n\n        periodSizeInBytes = pDevice->playback.internalPeriodSizeInFrames * ma_get_bytes_per_frame(pDevice->playback.internalFormat, pDevice->playback.internalChannels);\n        for (iPeriod = 0; iPeriod < pDevice->playback.internalPeriods; ++iPeriod) {\n            resultSL = MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueuePlayback)->Enqueue((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueuePlayback, pDevice->opensl.pBufferPlayback + (periodSizeInBytes * iPeriod), periodSizeInBytes);\n            if (resultSL != SL_RESULT_SUCCESS) {\n                MA_OPENSL_PLAY(pDevice->opensl.pAudioPlayer)->SetPlayState((SLPlayItf)pDevice->opensl.pAudioPlayer, SL_PLAYSTATE_STOPPED);\n                ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to enqueue buffer for playback device.\");\n                return ma_result_from_OpenSL(resultSL);\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_drain__opensl(ma_device* pDevice, ma_device_type deviceType)\n{\n    SLAndroidSimpleBufferQueueItf pBufferQueue;\n\n    MA_ASSERT(deviceType == ma_device_type_capture || deviceType == ma_device_type_playback);\n\n    if (pDevice->type == ma_device_type_capture) {\n        pBufferQueue = (SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture;\n        pDevice->opensl.isDrainingCapture  = MA_TRUE;\n    } else {\n        pBufferQueue = (SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueuePlayback;\n        pDevice->opensl.isDrainingPlayback = MA_TRUE;\n    }\n\n    for (;;) {\n        SLAndroidSimpleBufferQueueState state;\n\n        MA_OPENSL_BUFFERQUEUE(pBufferQueue)->GetState(pBufferQueue, &state);\n        if (state.count == 0) {\n            break;\n        }\n\n        ma_sleep(10);\n    }\n\n    if (pDevice->type == ma_device_type_capture) {\n        pDevice->opensl.isDrainingCapture  = MA_FALSE;\n    } else {\n        pDevice->opensl.isDrainingPlayback = MA_FALSE;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__opensl(ma_device* pDevice)\n{\n    SLresult resultSL;\n\n    MA_ASSERT(pDevice != NULL);\n\n    MA_ASSERT(g_maOpenSLInitCounter > 0); /* <-- If you trigger this it means you've either not initialized the context, or you've uninitialized it before stopping/uninitializing the device. */\n    if (g_maOpenSLInitCounter == 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex) {\n        ma_device_drain__opensl(pDevice, ma_device_type_capture);\n\n        resultSL = MA_OPENSL_RECORD(pDevice->opensl.pAudioRecorder)->SetRecordState((SLRecordItf)pDevice->opensl.pAudioRecorder, SL_RECORDSTATE_STOPPED);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to stop internal capture device.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueueCapture)->Clear((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueueCapture);\n    }\n\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_device_drain__opensl(pDevice, ma_device_type_playback);\n\n        resultSL = MA_OPENSL_PLAY(pDevice->opensl.pAudioPlayer)->SetPlayState((SLPlayItf)pDevice->opensl.pAudioPlayer, SL_PLAYSTATE_STOPPED);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            ma_log_post(ma_device_get_log(pDevice), MA_LOG_LEVEL_ERROR, \"[OpenSL] Failed to stop internal playback device.\");\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        MA_OPENSL_BUFFERQUEUE(pDevice->opensl.pBufferQueuePlayback)->Clear((SLAndroidSimpleBufferQueueItf)pDevice->opensl.pBufferQueuePlayback);\n    }\n\n    /* Make sure the client is aware that the device has stopped. There may be an OpenSL|ES callback for this, but I haven't found it. */\n    ma_device__on_notification_stopped(pDevice);\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_context_uninit__opensl(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_opensl);\n    (void)pContext;\n\n    /* Uninit global data. */\n    ma_spinlock_lock(&g_maOpenSLSpinlock);\n    {\n        MA_ASSERT(g_maOpenSLInitCounter > 0);   /* If you've triggered this, it means you have ma_context_init/uninit mismatch. Each successful call to ma_context_init() must be matched up with a call to ma_context_uninit(). */\n\n        g_maOpenSLInitCounter -= 1;\n        if (g_maOpenSLInitCounter == 0) {\n            (*g_maEngineObjectSL)->Destroy(g_maEngineObjectSL);\n        }\n    }\n    ma_spinlock_unlock(&g_maOpenSLSpinlock);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_dlsym_SLInterfaceID__opensl(ma_context* pContext, const char* pName, ma_handle* pHandle)\n{\n    /* We need to return an error if the symbol cannot be found. This is important because there have been reports that some symbols do not exist. */\n    ma_handle* p = (ma_handle*)ma_dlsym(ma_context_get_log(pContext), pContext->opensl.libOpenSLES, pName);\n    if (p == NULL) {\n        ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_INFO, \"[OpenSL] Cannot find symbol %s\", pName);\n        return MA_NO_BACKEND;\n    }\n\n    *pHandle = *p;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init_engine_nolock__opensl(ma_context* pContext)\n{\n    g_maOpenSLInitCounter += 1;\n    if (g_maOpenSLInitCounter == 1) {\n        SLresult resultSL;\n\n        resultSL = ((ma_slCreateEngine_proc)pContext->opensl.slCreateEngine)(&g_maEngineObjectSL, 0, NULL, 0, NULL, NULL);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            g_maOpenSLInitCounter -= 1;\n            return ma_result_from_OpenSL(resultSL);\n        }\n\n        (*g_maEngineObjectSL)->Realize(g_maEngineObjectSL, SL_BOOLEAN_FALSE);\n\n        resultSL = (*g_maEngineObjectSL)->GetInterface(g_maEngineObjectSL, (SLInterfaceID)pContext->opensl.SL_IID_ENGINE, &g_maEngineSL);\n        if (resultSL != SL_RESULT_SUCCESS) {\n            (*g_maEngineObjectSL)->Destroy(g_maEngineObjectSL);\n            g_maOpenSLInitCounter -= 1;\n            return ma_result_from_OpenSL(resultSL);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__opensl(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    ma_result result;\n\n#if !defined(MA_NO_RUNTIME_LINKING)\n    size_t i;\n    const char* libOpenSLESNames[] = {\n        \"libOpenSLES.so\"\n    };\n#endif\n\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig;\n\n#if !defined(MA_NO_RUNTIME_LINKING)\n    /*\n    Dynamically link against libOpenSLES.so. I have now had multiple reports that SL_IID_ANDROIDSIMPLEBUFFERQUEUE cannot be found. One\n    report was happening at compile time and another at runtime. To try working around this, I'm going to link to libOpenSLES at runtime\n    and extract the symbols rather than reference them directly. This should, hopefully, fix these issues as the compiler won't see any\n    references to the symbols and will hopefully skip the checks.\n    */\n    for (i = 0; i < ma_countof(libOpenSLESNames); i += 1) {\n        pContext->opensl.libOpenSLES = ma_dlopen(ma_context_get_log(pContext), libOpenSLESNames[i]);\n        if (pContext->opensl.libOpenSLES != NULL) {\n            break;\n        }\n    }\n\n    if (pContext->opensl.libOpenSLES == NULL) {\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_INFO, \"[OpenSL] Could not find libOpenSLES.so\");\n        return MA_NO_BACKEND;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_ENGINE\", &pContext->opensl.SL_IID_ENGINE);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_AUDIOIODEVICECAPABILITIES\", &pContext->opensl.SL_IID_AUDIOIODEVICECAPABILITIES);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_ANDROIDSIMPLEBUFFERQUEUE\", &pContext->opensl.SL_IID_ANDROIDSIMPLEBUFFERQUEUE);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_RECORD\", &pContext->opensl.SL_IID_RECORD);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_PLAY\", &pContext->opensl.SL_IID_PLAY);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_OUTPUTMIX\", &pContext->opensl.SL_IID_OUTPUTMIX);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    result = ma_dlsym_SLInterfaceID__opensl(pContext, \"SL_IID_ANDROIDCONFIGURATION\", &pContext->opensl.SL_IID_ANDROIDCONFIGURATION);\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        return result;\n    }\n\n    pContext->opensl.slCreateEngine = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->opensl.libOpenSLES, \"slCreateEngine\");\n    if (pContext->opensl.slCreateEngine == NULL) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_INFO, \"[OpenSL] Cannot find symbol slCreateEngine.\");\n        return MA_NO_BACKEND;\n    }\n#else\n    pContext->opensl.SL_IID_ENGINE                    = (ma_handle)SL_IID_ENGINE;\n    pContext->opensl.SL_IID_AUDIOIODEVICECAPABILITIES = (ma_handle)SL_IID_AUDIOIODEVICECAPABILITIES;\n    pContext->opensl.SL_IID_ANDROIDSIMPLEBUFFERQUEUE  = (ma_handle)SL_IID_ANDROIDSIMPLEBUFFERQUEUE;\n    pContext->opensl.SL_IID_RECORD                    = (ma_handle)SL_IID_RECORD;\n    pContext->opensl.SL_IID_PLAY                      = (ma_handle)SL_IID_PLAY;\n    pContext->opensl.SL_IID_OUTPUTMIX                 = (ma_handle)SL_IID_OUTPUTMIX;\n    pContext->opensl.SL_IID_ANDROIDCONFIGURATION      = (ma_handle)SL_IID_ANDROIDCONFIGURATION;\n    pContext->opensl.slCreateEngine                   = (ma_proc)slCreateEngine;\n#endif\n\n\n    /* Initialize global data first if applicable. */\n    ma_spinlock_lock(&g_maOpenSLSpinlock);\n    {\n        result = ma_context_init_engine_nolock__opensl(pContext);\n    }\n    ma_spinlock_unlock(&g_maOpenSLSpinlock);\n\n    if (result != MA_SUCCESS) {\n        ma_dlclose(ma_context_get_log(pContext), pContext->opensl.libOpenSLES);\n        ma_log_post(ma_context_get_log(pContext), MA_LOG_LEVEL_INFO, \"[OpenSL] Failed to initialize OpenSL engine.\");\n        return result;\n    }\n\n    pCallbacks->onContextInit             = ma_context_init__opensl;\n    pCallbacks->onContextUninit           = ma_context_uninit__opensl;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__opensl;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__opensl;\n    pCallbacks->onDeviceInit              = ma_device_init__opensl;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__opensl;\n    pCallbacks->onDeviceStart             = ma_device_start__opensl;\n    pCallbacks->onDeviceStop              = ma_device_stop__opensl;\n    pCallbacks->onDeviceRead              = NULL;   /* Not needed because OpenSL|ES is asynchronous. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not needed because OpenSL|ES is asynchronous. */\n    pCallbacks->onDeviceDataLoop          = NULL;   /* Not needed because OpenSL|ES is asynchronous. */\n\n    return MA_SUCCESS;\n}\n#endif  /* OpenSL|ES */\n\n\n/******************************************************************************\n\nWeb Audio Backend\n\n******************************************************************************/\n#ifdef MA_HAS_WEBAUDIO\n#include <emscripten/emscripten.h>\n\n#if (__EMSCRIPTEN_major__ > 3) || (__EMSCRIPTEN_major__ == 3 && (__EMSCRIPTEN_minor__ > 1 || (__EMSCRIPTEN_minor__ == 1 && __EMSCRIPTEN_tiny__ >= 32)))\n    #include <emscripten/webaudio.h>\n    #define MA_SUPPORT_AUDIO_WORKLETS\n\n    #if (__EMSCRIPTEN_major__ > 3) || (__EMSCRIPTEN_major__ == 3 && (__EMSCRIPTEN_minor__ > 1 || (__EMSCRIPTEN_minor__ == 1 && __EMSCRIPTEN_tiny__ >= 70)))\n        #define MA_SUPPORT_AUDIO_WORKLETS_VARIABLE_BUFFER_SIZE\n    #endif\n#endif\n\n/*\nTODO: Version 0.12: Swap this logic around so that AudioWorklets are used by default. Add MA_NO_AUDIO_WORKLETS.\n*/\n#if defined(MA_ENABLE_AUDIO_WORKLETS) && defined(MA_SUPPORT_AUDIO_WORKLETS)\n    #define MA_USE_AUDIO_WORKLETS\n#endif\n\n/* The thread stack size must be a multiple of 16. */\n#ifndef MA_AUDIO_WORKLETS_THREAD_STACK_SIZE\n#define MA_AUDIO_WORKLETS_THREAD_STACK_SIZE     131072\n#endif\n\n#if defined(MA_USE_AUDIO_WORKLETS)\n#define MA_WEBAUDIO_LATENCY_HINT_BALANCED       \"balanced\"\n#define MA_WEBAUDIO_LATENCY_HINT_INTERACTIVE    \"interactive\"\n#define MA_WEBAUDIO_LATENCY_HINT_PLAYBACK       \"playback\"\n#endif\n\nstatic ma_bool32 ma_is_capture_supported__webaudio()\n{\n    return EM_ASM_INT({\n        return (navigator.mediaDevices !== undefined && navigator.mediaDevices.getUserMedia !== undefined);\n    }, 0) != 0; /* Must pass in a dummy argument for C99 compatibility. */\n}\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\nvoid* EMSCRIPTEN_KEEPALIVE ma_malloc_emscripten(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_malloc(sz, pAllocationCallbacks);\n}\n\nvoid EMSCRIPTEN_KEEPALIVE ma_free_emscripten(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_free(p, pAllocationCallbacks);\n}\n\nvoid EMSCRIPTEN_KEEPALIVE ma_device_process_pcm_frames_capture__webaudio(ma_device* pDevice, int frameCount, float* pFrames)\n{\n    ma_device_handle_backend_data_callback(pDevice, NULL, pFrames, (ma_uint32)frameCount);\n}\n\nvoid EMSCRIPTEN_KEEPALIVE ma_device_process_pcm_frames_playback__webaudio(ma_device* pDevice, int frameCount, float* pFrames)\n{\n    ma_device_handle_backend_data_callback(pDevice, pFrames, NULL, (ma_uint32)frameCount);\n}\n#ifdef __cplusplus\n}\n#endif\n\nstatic ma_result ma_context_enumerate_devices__webaudio(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_bool32 cbResult = MA_TRUE;\n\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(callback != NULL);\n\n    /* Only supporting default devices for now. */\n\n    /* Playback. */\n    if (cbResult) {\n        ma_device_info deviceInfo;\n        MA_ZERO_OBJECT(&deviceInfo);\n        ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n        deviceInfo.isDefault = MA_TRUE;    /* Only supporting default devices. */\n        cbResult = callback(pContext, ma_device_type_playback, &deviceInfo, pUserData);\n    }\n\n    /* Capture. */\n    if (cbResult) {\n        if (ma_is_capture_supported__webaudio()) {\n            ma_device_info deviceInfo;\n            MA_ZERO_OBJECT(&deviceInfo);\n            ma_strncpy_s(deviceInfo.name, sizeof(deviceInfo.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n            deviceInfo.isDefault = MA_TRUE;    /* Only supporting default devices. */\n            cbResult = callback(pContext, ma_device_type_capture, &deviceInfo, pUserData);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_get_device_info__webaudio(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    MA_ASSERT(pContext != NULL);\n\n    if (deviceType == ma_device_type_capture && !ma_is_capture_supported__webaudio()) {\n        return MA_NO_DEVICE;\n    }\n\n    MA_ZERO_MEMORY(pDeviceInfo->id.webaudio, sizeof(pDeviceInfo->id.webaudio));\n\n    /* Only supporting default devices for now. */\n    (void)pDeviceID;\n    if (deviceType == ma_device_type_playback) {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n    } else {\n        ma_strncpy_s(pDeviceInfo->name, sizeof(pDeviceInfo->name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n    }\n\n    /* Only supporting default devices. */\n    pDeviceInfo->isDefault = MA_TRUE;\n\n    /* Web Audio can support any number of channels and sample rates. It only supports f32 formats, however. */\n    pDeviceInfo->nativeDataFormats[0].flags      = 0;\n    pDeviceInfo->nativeDataFormats[0].format     = ma_format_unknown;\n    pDeviceInfo->nativeDataFormats[0].channels   = 0; /* All channels are supported. */\n    pDeviceInfo->nativeDataFormats[0].sampleRate = EM_ASM_INT({\n        try {\n            var temp = new (window.AudioContext || window.webkitAudioContext)();\n            var sampleRate = temp.sampleRate;\n            temp.close();\n            return sampleRate;\n        } catch(e) {\n            return 0;\n        }\n    }, 0);  /* Must pass in a dummy argument for C99 compatibility. */\n\n    if (pDeviceInfo->nativeDataFormats[0].sampleRate == 0) {\n        return MA_NO_DEVICE;\n    }\n\n    pDeviceInfo->nativeDataFormatCount = 1;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_uninit__webaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    #if defined(MA_USE_AUDIO_WORKLETS)\n    {\n        EM_ASM({\n            var device = window.miniaudio.get_device_by_index($0);\n\n            if (device.streamNode !== undefined) {\n                device.streamNode.disconnect();\n                device.streamNode = undefined;\n            }\n\n            device.pDevice = undefined;\n        }, pDevice->webaudio.deviceIndex);\n\n        emscripten_destroy_web_audio_node(pDevice->webaudio.audioWorklet);\n        emscripten_destroy_audio_context(pDevice->webaudio.audioContext);\n        ma_free(pDevice->webaudio.pStackBuffer, &pDevice->pContext->allocationCallbacks);\n    }\n    #else\n    {\n        EM_ASM({\n            var device = window.miniaudio.get_device_by_index($0);\n\n            /* Make sure all nodes are disconnected and marked for collection. */\n            if (device.scriptNode !== undefined) {\n                device.scriptNode.onaudioprocess = function(e) {};  /* We want to reset the callback to ensure it doesn't get called after AudioContext.close() has returned. Shouldn't happen since we're disconnecting, but just to be safe... */\n                device.scriptNode.disconnect();\n                device.scriptNode = undefined;\n            }\n\n            if (device.streamNode !== undefined) {\n                device.streamNode.disconnect();\n                device.streamNode = undefined;\n            }\n\n            /*\n            Stop the device. I think there is a chance the callback could get fired after calling this, hence why we want\n            to clear the callback before closing.\n            */\n            device.webaudio.close();\n            device.webaudio = undefined;\n            device.pDevice = undefined;\n        }, pDevice->webaudio.deviceIndex);\n    }\n    #endif\n\n    /* Clean up the device on the JS side. */\n    EM_ASM({\n        window.miniaudio.untrack_device_by_index($0);\n    }, pDevice->webaudio.deviceIndex);\n\n    ma_free(pDevice->webaudio.pIntermediaryBuffer, &pDevice->pContext->allocationCallbacks);\n\n    return MA_SUCCESS;\n}\n\n#if !defined(MA_USE_AUDIO_WORKLETS)\nstatic ma_uint32 ma_calculate_period_size_in_frames_from_descriptor__webaudio(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile)\n{\n    /*\n    There have been reports of the default buffer size being too small on some browsers. If we're using\n    the default buffer size, we'll make sure the period size is bigger than our standard defaults.\n    */\n    ma_uint32 periodSizeInFrames;\n\n    if (nativeSampleRate == 0) {\n        nativeSampleRate = MA_DEFAULT_SAMPLE_RATE;\n    }\n\n    if (pDescriptor->periodSizeInFrames == 0) {\n        if (pDescriptor->periodSizeInMilliseconds == 0) {\n            if (performanceProfile == ma_performance_profile_low_latency) {\n                periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(33, nativeSampleRate);  /* 1 frame @ 30 FPS */\n            } else {\n                periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(333, nativeSampleRate);\n            }\n        } else {\n            periodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(pDescriptor->periodSizeInMilliseconds, nativeSampleRate);\n        }\n    } else {\n        periodSizeInFrames = pDescriptor->periodSizeInFrames;\n    }\n\n    /* The size of the buffer must be a power of 2 and between 256 and 16384. */\n    if (periodSizeInFrames < 256) {\n        periodSizeInFrames = 256;\n    } else if (periodSizeInFrames > 16384) {\n        periodSizeInFrames = 16384;\n    } else {\n        periodSizeInFrames = ma_next_power_of_2(periodSizeInFrames);\n    }\n\n    return periodSizeInFrames;\n}\n#endif\n\n\n#if defined(MA_USE_AUDIO_WORKLETS)\ntypedef struct\n{\n    ma_device* pDevice;\n    const ma_device_config* pConfig;\n    ma_device_descriptor* pDescriptorPlayback;\n    ma_device_descriptor* pDescriptorCapture;\n} ma_audio_worklet_thread_initialized_data;\n\nstatic EM_BOOL ma_audio_worklet_process_callback__webaudio(int inputCount, const AudioSampleFrame* pInputs, int outputCount, AudioSampleFrame* pOutputs, int paramCount, const AudioParamFrame* pParams, void* pUserData)\n{\n    ma_device* pDevice = (ma_device*)pUserData;\n    ma_uint32 frameCount;\n\n    (void)paramCount;\n    (void)pParams;\n\n    /*\n    The Emscripten documentation says that it'll always be 128 frames being passed in. Hard coding it like that feels\n    like a very bad idea to me. Even if it's hard coded in the backend, the API and documentation should always refer\n    to variables instead of a hard coded number. In any case, will follow along for the time being.\n\n    Unfortunately the audio data is not interleaved so we'll need to convert it before we give the data to miniaudio\n    for further processing.\n    */\n    if (pDevice->type == ma_device_type_playback) {\n        frameCount = pDevice->playback.internalPeriodSizeInFrames;\n    } else {\n        frameCount = pDevice->capture.internalPeriodSizeInFrames;\n    }\n\n    /*\n    If this is called by the device has not yet been started we need to return early, making sure we output silence to\n    the output buffer.\n    */\n    if (ma_device_get_state(pDevice) != ma_device_state_started) {\n        for (int i = 0; i < outputCount; i += 1) {\n            MA_ZERO_MEMORY(pOutputs[i].data, pOutputs[i].numberOfChannels * frameCount * sizeof(float));\n        }\n\n        return EM_TRUE;\n    }\n\n    if (inputCount > 0) {\n        /* Input data needs to be interleaved before we hand it to the client. */\n        for (ma_uint32 iChannel = 0; iChannel < pDevice->capture.internalChannels; iChannel += 1) {\n            for (ma_uint32 iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                pDevice->webaudio.pIntermediaryBuffer[iFrame*pDevice->capture.internalChannels + iChannel] = pInputs[0].data[frameCount*iChannel + iFrame];\n            }\n        }\n\n        ma_device_process_pcm_frames_capture__webaudio(pDevice, frameCount, pDevice->webaudio.pIntermediaryBuffer);\n    }\n\n    if (outputCount > 0) {\n        /* If it's a capture-only device, we'll need to output silence. */\n        if (pDevice->type == ma_device_type_capture) {\n            for (int i = 0; i < outputCount; i += 1) {\n                MA_ZERO_MEMORY(pOutputs[i].data, pOutputs[i].numberOfChannels * frameCount * sizeof(float));\n            }\n        } else {\n            ma_device_process_pcm_frames_playback__webaudio(pDevice, frameCount, pDevice->webaudio.pIntermediaryBuffer);\n\n            /* We've read the data from the client. Now we need to deinterleave the buffer and output to the output buffer. */\n            for (ma_uint32 iChannel = 0; iChannel < pDevice->playback.internalChannels; iChannel += 1) {\n                for (ma_uint32 iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                    pOutputs[0].data[frameCount*iChannel + iFrame] = pDevice->webaudio.pIntermediaryBuffer[iFrame*pDevice->playback.internalChannels + iChannel];\n                }\n            }\n\n            /*\n            Just above we output data to the first output buffer. Here we just make sure we're putting silence into any\n            remaining output buffers.\n            */\n            for (int i = 1; i < outputCount; i += 1) {  /* <-- Note that the counter starts at 1 instead of 0. */\n                MA_ZERO_MEMORY(pOutputs[i].data, pOutputs[i].numberOfChannels * frameCount * sizeof(float));\n            }\n        }\n    }\n\n    return EM_TRUE;\n}\n\n\nstatic void ma_audio_worklet_processor_created__webaudio(EMSCRIPTEN_WEBAUDIO_T audioContext, EM_BOOL success, void* pUserData)\n{\n    ma_audio_worklet_thread_initialized_data* pParameters = (ma_audio_worklet_thread_initialized_data*)pUserData;\n    EmscriptenAudioWorkletNodeCreateOptions audioWorkletOptions;\n    int channels = 0;\n    size_t intermediaryBufferSizeInFrames;\n    int sampleRate;\n\n    if (success == EM_FALSE) {\n        pParameters->pDevice->webaudio.initResult = MA_ERROR;\n        ma_free(pParameters, &pParameters->pDevice->pContext->allocationCallbacks);\n        return;\n    }\n\n    /* The next step is to initialize the audio worklet node. */\n    MA_ZERO_OBJECT(&audioWorkletOptions);\n\n    /*\n    The way channel counts work with Web Audio is confusing. As far as I can tell, there's no way to know the channel\n    count from MediaStreamAudioSourceNode (what we use for capture)? The only way to have control is to configure an\n    output channel count on the capture side. This is slightly confusing for capture mode because intuitively you\n    wouldn't actually connect an output to an input-only node, but this is what we'll have to do in order to have\n    proper control over the channel count. In the capture case, we'll have to output silence to its output node.\n    */\n    if (pParameters->pConfig->deviceType == ma_device_type_capture) {\n        channels = (int)((pParameters->pDescriptorCapture->channels > 0) ? pParameters->pDescriptorCapture->channels : MA_DEFAULT_CHANNELS);\n        audioWorkletOptions.numberOfInputs = 1;\n    } else {\n        channels = (int)((pParameters->pDescriptorPlayback->channels > 0) ? pParameters->pDescriptorPlayback->channels : MA_DEFAULT_CHANNELS);\n\n        if (pParameters->pConfig->deviceType == ma_device_type_duplex) {\n            audioWorkletOptions.numberOfInputs = 1;\n        } else {\n            audioWorkletOptions.numberOfInputs = 0;\n        }\n    }\n\n    audioWorkletOptions.numberOfOutputs = 1;\n    audioWorkletOptions.outputChannelCounts = &channels;\n\n\n    /*\n    Now that we know the channel count to use we can allocate the intermediary buffer. The\n    intermediary buffer is used for interleaving and deinterleaving.\n    */\n    #if defined(MA_SUPPORT_AUDIO_WORKLETS_VARIABLE_BUFFER_SIZE)\n    {\n        intermediaryBufferSizeInFrames = (size_t)emscripten_audio_context_quantum_size(audioContext);\n    }\n    #else\n    {\n        intermediaryBufferSizeInFrames = 128;\n    }\n    #endif\n\n    pParameters->pDevice->webaudio.pIntermediaryBuffer = (float*)ma_malloc(intermediaryBufferSizeInFrames * (ma_uint32)channels * sizeof(float), &pParameters->pDevice->pContext->allocationCallbacks);\n    if (pParameters->pDevice->webaudio.pIntermediaryBuffer == NULL) {\n        pParameters->pDevice->webaudio.initResult = MA_OUT_OF_MEMORY;\n        ma_free(pParameters, &pParameters->pDevice->pContext->allocationCallbacks);\n        return;\n    }\n\n    pParameters->pDevice->webaudio.audioWorklet = emscripten_create_wasm_audio_worklet_node(audioContext, \"miniaudio\", &audioWorkletOptions, &ma_audio_worklet_process_callback__webaudio, pParameters->pDevice);\n\n    /* With the audio worklet initialized we can now attach it to the graph. */\n    if (pParameters->pConfig->deviceType == ma_device_type_capture || pParameters->pConfig->deviceType == ma_device_type_duplex) {\n        ma_result attachmentResult = (ma_result)EM_ASM_INT({\n            var getUserMediaResult = 0;\n            var audioWorklet = emscriptenGetAudioObject($0);\n            var audioContext = emscriptenGetAudioObject($1);\n\n            navigator.mediaDevices.getUserMedia({audio:true, video:false})\n                .then(function(stream) {\n                    audioContext.streamNode = audioContext.createMediaStreamSource(stream);\n                    audioContext.streamNode.connect(audioWorklet);\n                    audioWorklet.connect(audioContext.destination);\n                    getUserMediaResult = 0;   /* 0 = MA_SUCCESS */\n                })\n                .catch(function(error) {\n                    console.log(\"navigator.mediaDevices.getUserMedia Failed: \" + error);\n                    getUserMediaResult = -1;  /* -1 = MA_ERROR */\n                });\n\n            return getUserMediaResult;\n        }, pParameters->pDevice->webaudio.audioWorklet, audioContext);\n\n        if (attachmentResult != MA_SUCCESS) {\n            ma_log_postf(ma_device_get_log(pParameters->pDevice), MA_LOG_LEVEL_ERROR, \"Web Audio: Failed to connect capture node.\");\n            emscripten_destroy_web_audio_node(pParameters->pDevice->webaudio.audioWorklet);\n            pParameters->pDevice->webaudio.initResult = attachmentResult;\n            ma_free(pParameters, &pParameters->pDevice->pContext->allocationCallbacks);\n            return;\n        }\n    }\n\n    /* If it's playback only we can now attach the worklet node to the graph. This has already been done for the duplex case. */\n    if (pParameters->pConfig->deviceType == ma_device_type_playback) {\n        ma_result attachmentResult = (ma_result)EM_ASM_INT({\n            var audioWorklet = emscriptenGetAudioObject($0);\n            var audioContext = emscriptenGetAudioObject($1);\n            audioWorklet.connect(audioContext.destination);\n            return 0;   /* 0 = MA_SUCCESS */\n        }, pParameters->pDevice->webaudio.audioWorklet, audioContext);\n\n        if (attachmentResult != MA_SUCCESS) {\n            ma_log_postf(ma_device_get_log(pParameters->pDevice), MA_LOG_LEVEL_ERROR, \"Web Audio: Failed to connect playback node.\");\n            pParameters->pDevice->webaudio.initResult = attachmentResult;\n            ma_free(pParameters, &pParameters->pDevice->pContext->allocationCallbacks);\n            return;\n        }\n    }\n\n    /* We need to update the descriptors so that they reflect the internal data format. Both capture and playback should be the same. */\n    sampleRate = EM_ASM_INT({ return emscriptenGetAudioObject($0).sampleRate; }, audioContext);\n\n    if (pParameters->pDescriptorCapture != NULL) {\n        pParameters->pDescriptorCapture->format              = ma_format_f32;\n        pParameters->pDescriptorCapture->channels            = (ma_uint32)channels;\n        pParameters->pDescriptorCapture->sampleRate          = (ma_uint32)sampleRate;\n        ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pParameters->pDescriptorCapture->channelMap, ma_countof(pParameters->pDescriptorCapture->channelMap), pParameters->pDescriptorCapture->channels);\n        pParameters->pDescriptorCapture->periodSizeInFrames  = intermediaryBufferSizeInFrames;\n        pParameters->pDescriptorCapture->periodCount         = 1;\n    }\n\n    if (pParameters->pDescriptorPlayback != NULL) {\n        pParameters->pDescriptorPlayback->format             = ma_format_f32;\n        pParameters->pDescriptorPlayback->channels           = (ma_uint32)channels;\n        pParameters->pDescriptorPlayback->sampleRate         = (ma_uint32)sampleRate;\n        ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pParameters->pDescriptorPlayback->channelMap, ma_countof(pParameters->pDescriptorPlayback->channelMap), pParameters->pDescriptorPlayback->channels);\n        pParameters->pDescriptorPlayback->periodSizeInFrames = intermediaryBufferSizeInFrames;\n        pParameters->pDescriptorPlayback->periodCount        = 1;\n    }\n\n    /* At this point we're done and we can return. */\n    ma_log_postf(ma_device_get_log(pParameters->pDevice), MA_LOG_LEVEL_DEBUG, \"AudioWorklets: Created worklet node: %d\\n\", pParameters->pDevice->webaudio.audioWorklet);\n    pParameters->pDevice->webaudio.initResult = MA_SUCCESS;\n    ma_free(pParameters, &pParameters->pDevice->pContext->allocationCallbacks);\n}\n\nstatic void ma_audio_worklet_thread_initialized__webaudio(EMSCRIPTEN_WEBAUDIO_T audioContext, EM_BOOL success, void* pUserData)\n{\n    ma_audio_worklet_thread_initialized_data* pParameters = (ma_audio_worklet_thread_initialized_data*)pUserData;\n    WebAudioWorkletProcessorCreateOptions workletProcessorOptions;\n\n    MA_ASSERT(pParameters != NULL);\n\n    if (success == EM_FALSE) {\n        pParameters->pDevice->webaudio.initResult = MA_ERROR;\n        return;\n    }\n\n    MA_ZERO_OBJECT(&workletProcessorOptions);\n    workletProcessorOptions.name = \"miniaudio\"; /* I'm not entirely sure what to call this. Does this need to be globally unique, or does it need only be unique for a given AudioContext? */\n\n    emscripten_create_wasm_audio_worklet_processor_async(audioContext, &workletProcessorOptions, ma_audio_worklet_processor_created__webaudio, pParameters);\n}\n#endif\n\nstatic ma_result ma_device_init__webaudio(ma_device* pDevice, const ma_device_config* pConfig, ma_device_descriptor* pDescriptorPlayback, ma_device_descriptor* pDescriptorCapture)\n{\n    if (pConfig->deviceType == ma_device_type_loopback) {\n        return MA_DEVICE_TYPE_NOT_SUPPORTED;\n    }\n\n    /* No exclusive mode with Web Audio. */\n    if (((pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) && pDescriptorPlayback->shareMode == ma_share_mode_exclusive) ||\n        ((pConfig->deviceType == ma_device_type_capture  || pConfig->deviceType == ma_device_type_duplex) && pDescriptorCapture->shareMode  == ma_share_mode_exclusive)) {\n        return MA_SHARE_MODE_NOT_SUPPORTED;\n    }\n\n    /*\n    With AudioWorklets we'll have just a single AudioContext. I'm not sure why I'm not doing this for ScriptProcessorNode so\n    it might be worthwhile to look into that as well.\n    */\n    #if defined(MA_USE_AUDIO_WORKLETS)\n    {\n        EmscriptenWebAudioCreateAttributes audioContextAttributes;\n        ma_audio_worklet_thread_initialized_data* pInitParameters;\n        void* pStackBuffer;\n\n        if (pConfig->performanceProfile == ma_performance_profile_conservative) {\n            audioContextAttributes.latencyHint = MA_WEBAUDIO_LATENCY_HINT_PLAYBACK;\n        } else {\n            audioContextAttributes.latencyHint = MA_WEBAUDIO_LATENCY_HINT_INTERACTIVE;\n        }\n\n        /*\n        In my testing, Firefox does not seem to capture audio data properly if the sample rate is set\n        to anything other than 48K. This does not seem to be the case for other browsers. For this reason,\n        if the device type is anything other than playback, we'll leave the sample rate as-is and let the\n        browser pick the appropriate rate for us.\n        */\n        if (pConfig->deviceType == ma_device_type_playback) {\n            audioContextAttributes.sampleRate = pDescriptorPlayback->sampleRate;\n        } else {\n            audioContextAttributes.sampleRate = 0;\n        }\n\n        /* It's not clear if this can return an error. None of the tests in the Emscripten repository check for this, so neither am I for now. */\n        pDevice->webaudio.audioContext = emscripten_create_audio_context(&audioContextAttributes);\n\n        /*\n        With the context created we can now create the worklet. We can only have a single worklet per audio\n        context which means we'll need to craft this appropriately to handle duplex devices correctly.\n        */\n\n        /*\n        We now need to create a worker thread. This is a bit weird because we need to allocate our\n        own buffer for the thread's stack. The stack needs to be aligned to 16 bytes. I'm going to\n        allocate this on the heap to keep it simple.\n        */\n        pStackBuffer = ma_aligned_malloc(MA_AUDIO_WORKLETS_THREAD_STACK_SIZE, 16, &pDevice->pContext->allocationCallbacks);\n        if (pStackBuffer == NULL) {\n            emscripten_destroy_audio_context(pDevice->webaudio.audioContext);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        /* Our thread initialization parameters need to be allocated on the heap so they don't go out of scope. */\n        pInitParameters = (ma_audio_worklet_thread_initialized_data*)ma_malloc(sizeof(*pInitParameters), &pDevice->pContext->allocationCallbacks);\n        if (pInitParameters == NULL) {\n            ma_free(pStackBuffer, &pDevice->pContext->allocationCallbacks);\n            emscripten_destroy_audio_context(pDevice->webaudio.audioContext);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        pInitParameters->pDevice = pDevice;\n        pInitParameters->pConfig = pConfig;\n        pInitParameters->pDescriptorPlayback = pDescriptorPlayback;\n        pInitParameters->pDescriptorCapture  = pDescriptorCapture;\n\n        /*\n        We need to flag the device as not yet initialized so we can wait on it later. Unfortunately all of\n        the Emscripten WebAudio stuff is asynchronous.\n        */\n        pDevice->webaudio.initResult = MA_BUSY;\n        {\n            emscripten_start_wasm_audio_worklet_thread_async(pDevice->webaudio.audioContext, pStackBuffer, MA_AUDIO_WORKLETS_THREAD_STACK_SIZE, ma_audio_worklet_thread_initialized__webaudio, pInitParameters);\n        }\n        while (pDevice->webaudio.initResult == MA_BUSY) { emscripten_sleep(1); }    /* We must wait for initialization to complete. We're just spinning here. The emscripten_sleep() call is why we need to build with `-sASYNCIFY`. */\n\n        /* Initialization is now complete. Descriptors were updated when the worklet was initialized. */\n        if (pDevice->webaudio.initResult != MA_SUCCESS) {\n            ma_free(pStackBuffer, &pDevice->pContext->allocationCallbacks);\n            emscripten_destroy_audio_context(pDevice->webaudio.audioContext);\n            return pDevice->webaudio.initResult;\n        }\n\n        /* We need to add an entry to the miniaudio.devices list on the JS side so we can do some JS/C interop. */\n        pDevice->webaudio.deviceIndex = EM_ASM_INT({\n            return window.miniaudio.track_device({\n                webaudio: emscriptenGetAudioObject($0),\n                state:    1, /* 1 = ma_device_state_stopped */\n                pDevice: $1\n            });\n        }, pDevice->webaudio.audioContext, pDevice);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* ScriptProcessorNode. This path requires us to do almost everything in JS, but we'll do as much as we can in C. */\n        ma_uint32 deviceIndex;\n        ma_uint32 channels;\n        ma_uint32 sampleRate;\n        ma_uint32 periodSizeInFrames;\n\n        /* The channel count will depend on the device type. If it's a capture, use its, otherwise use the playback side. */\n        if (pConfig->deviceType == ma_device_type_capture) {\n            channels = (pDescriptorCapture->channels  > 0) ? pDescriptorCapture->channels  : MA_DEFAULT_CHANNELS;\n        } else {\n            channels = (pDescriptorPlayback->channels > 0) ? pDescriptorPlayback->channels : MA_DEFAULT_CHANNELS;\n        }\n\n        /*\n        When testing in Firefox, I've seen it where capture mode fails if the sample rate is changed to anything other than it's\n        native rate. For this reason we're leaving the sample rate untouched for capture devices.\n        */\n        if (pConfig->deviceType == ma_device_type_playback) {\n            sampleRate = pDescriptorPlayback->sampleRate;\n        } else {\n            sampleRate = 0; /* Let the browser decide when capturing. */\n        }\n\n        /* The period size needs to be a power of 2. */\n        if (pConfig->deviceType == ma_device_type_capture) {\n            periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__webaudio(pDescriptorCapture, sampleRate, pConfig->performanceProfile);\n        } else {\n            periodSizeInFrames = ma_calculate_period_size_in_frames_from_descriptor__webaudio(pDescriptorPlayback, sampleRate, pConfig->performanceProfile);\n        }\n\n        /* We need an intermediary buffer for doing interleaving and deinterleaving. */\n        pDevice->webaudio.pIntermediaryBuffer = (float*)ma_malloc(periodSizeInFrames * channels * sizeof(float), &pDevice->pContext->allocationCallbacks);\n        if (pDevice->webaudio.pIntermediaryBuffer == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        deviceIndex = EM_ASM_INT({\n            var deviceType = $0;\n            var channels   = $1;\n            var sampleRate = $2;\n            var bufferSize = $3;\n            var pIntermediaryBuffer = $4;\n            var pDevice    = $5;\n\n            if (typeof(window.miniaudio) === 'undefined') {\n                return -1;  /* Context not initialized. */\n            }\n\n            var device = {};\n\n            /* First thing we need is an AudioContext. */\n            var audioContextOptions = {};\n            if (deviceType == window.miniaudio.device_type.playback && sampleRate != 0) {\n                audioContextOptions.sampleRate = sampleRate;\n            }\n\n            device.webaudio = new (window.AudioContext || window.webkitAudioContext)(audioContextOptions);\n            device.webaudio.suspend();  /* The AudioContext must be created in a suspended state. */\n            device.state = window.miniaudio.device_state.stopped;\n\n            /*\n            We need to create a ScriptProcessorNode. The channel situation is the same as the AudioWorklet path in that we\n            need to specify an output and configure the channel count there.\n            */\n            var channelCountIn  = 0;\n            var channelCountOut = channels;\n            if (deviceType != window.miniaudio.device_type.playback) {\n                channelCountIn  = channels;\n            }\n\n            device.scriptNode = device.webaudio.createScriptProcessor(bufferSize, channelCountIn, channelCountOut);\n\n            /* The node processing callback. */\n            device.scriptNode.onaudioprocess = function(e) {\n                if (device.intermediaryBufferView == null || device.intermediaryBufferView.length == 0) {\n                    device.intermediaryBufferView = new Float32Array(HEAPF32.buffer, pIntermediaryBuffer, bufferSize * channels);\n                }\n\n                /* Do the capture side first. */\n                if (deviceType == window.miniaudio.device_type.capture || deviceType == window.miniaudio.device_type.duplex) {\n                    /* The data must be interleaved before being processed miniaudio. */\n                    for (var iChannel = 0; iChannel < channels; iChannel += 1) {\n                        var inputBuffer = e.inputBuffer.getChannelData(iChannel);\n                        var intermediaryBuffer = device.intermediaryBufferView;\n\n                        for (var iFrame = 0; iFrame < bufferSize; iFrame += 1) {\n                            intermediaryBuffer[iFrame*channels + iChannel] = inputBuffer[iFrame];\n                        }\n                    }\n\n                    _ma_device_process_pcm_frames_capture__webaudio(pDevice, bufferSize, pIntermediaryBuffer);\n                }\n\n                if (deviceType == window.miniaudio.device_type.playback || deviceType == window.miniaudio.device_type.duplex) {\n                    _ma_device_process_pcm_frames_playback__webaudio(pDevice, bufferSize, pIntermediaryBuffer);\n\n                    for (var iChannel = 0; iChannel < e.outputBuffer.numberOfChannels; ++iChannel) {\n                        var outputBuffer = e.outputBuffer.getChannelData(iChannel);\n                        var intermediaryBuffer = device.intermediaryBufferView;\n\n                        for (var iFrame = 0; iFrame < bufferSize; iFrame += 1) {\n                            outputBuffer[iFrame] = intermediaryBuffer[iFrame*channels + iChannel];\n                        }\n                    }\n                } else {\n                    /* It's a capture-only device. Make sure the output is silenced. */\n                    for (var iChannel = 0; iChannel < e.outputBuffer.numberOfChannels; ++iChannel) {\n                        e.outputBuffer.getChannelData(iChannel).fill(0.0);\n                    }\n                }\n            };\n\n            /* Now we need to connect our node to the graph. */\n            if (deviceType == window.miniaudio.device_type.capture || deviceType == window.miniaudio.device_type.duplex) {\n                navigator.mediaDevices.getUserMedia({audio:true, video:false})\n                    .then(function(stream) {\n                        device.streamNode = device.webaudio.createMediaStreamSource(stream);\n                        device.streamNode.connect(device.scriptNode);\n                        device.scriptNode.connect(device.webaudio.destination);\n                    })\n                    .catch(function(error) {\n                        console.log(\"Failed to get user media: \" + error);\n                    });\n            }\n\n            if (deviceType == window.miniaudio.device_type.playback) {\n                device.scriptNode.connect(device.webaudio.destination);\n            }\n\n            device.pDevice = pDevice;\n\n            return window.miniaudio.track_device(device);\n        }, pConfig->deviceType, channels, sampleRate, periodSizeInFrames, pDevice->webaudio.pIntermediaryBuffer, pDevice);\n\n        if (deviceIndex < 0) {\n            return MA_FAILED_TO_OPEN_BACKEND_DEVICE;\n        }\n\n        pDevice->webaudio.deviceIndex = deviceIndex;\n\n        /* Grab the sample rate from the audio context directly. */\n        sampleRate = (ma_uint32)EM_ASM_INT({ return window.miniaudio.get_device_by_index($0).webaudio.sampleRate; }, deviceIndex);\n\n        if (pDescriptorCapture != NULL) {\n            pDescriptorCapture->format              = ma_format_f32;\n            pDescriptorCapture->channels            = channels;\n            pDescriptorCapture->sampleRate          = sampleRate;\n            ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pDescriptorCapture->channelMap, ma_countof(pDescriptorCapture->channelMap), pDescriptorCapture->channels);\n            pDescriptorCapture->periodSizeInFrames  = periodSizeInFrames;\n            pDescriptorCapture->periodCount         = 1;\n        }\n\n        if (pDescriptorPlayback != NULL) {\n            pDescriptorPlayback->format             = ma_format_f32;\n            pDescriptorPlayback->channels           = channels;\n            pDescriptorPlayback->sampleRate         = sampleRate;\n            ma_channel_map_init_standard(ma_standard_channel_map_webaudio, pDescriptorPlayback->channelMap, ma_countof(pDescriptorPlayback->channelMap), pDescriptorPlayback->channels);\n            pDescriptorPlayback->periodSizeInFrames = periodSizeInFrames;\n            pDescriptorPlayback->periodCount        = 1;\n        }\n\n        return MA_SUCCESS;\n    }\n    #endif\n}\n\nstatic ma_result ma_device_start__webaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    EM_ASM({\n        var device = window.miniaudio.get_device_by_index($0);\n        device.webaudio.resume();\n        device.state = window.miniaudio.device_state.started;\n    }, pDevice->webaudio.deviceIndex);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_device_stop__webaudio(ma_device* pDevice)\n{\n    MA_ASSERT(pDevice != NULL);\n\n    /*\n    From the WebAudio API documentation for AudioContext.suspend():\n\n        Suspends the progression of AudioContext's currentTime, allows any current context processing blocks that are already processed to be played to the\n        destination, and then allows the system to release its claim on audio hardware.\n\n    I read this to mean that \"any current context processing blocks\" are processed by suspend() - i.e. They they are drained. We therefore shouldn't need to\n    do any kind of explicit draining.\n    */\n    EM_ASM({\n        var device = window.miniaudio.get_device_by_index($0);\n        device.webaudio.suspend();\n        device.state = window.miniaudio.device_state.stopped;\n    }, pDevice->webaudio.deviceIndex);\n\n    ma_device__on_notification_stopped(pDevice);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_uninit__webaudio(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n    MA_ASSERT(pContext->backend == ma_backend_webaudio);\n\n    (void)pContext; /* Unused. */\n\n    /* Remove the global miniaudio object from window if there are no more references to it. */\n    EM_ASM({\n        if (typeof(window.miniaudio) !== 'undefined') {\n            window.miniaudio.unlock_event_types.map(function(event_type) {\n                document.removeEventListener(event_type, window.miniaudio.unlock, true);\n            });\n\n            window.miniaudio.referenceCount -= 1;\n            if (window.miniaudio.referenceCount === 0) {\n                delete window.miniaudio;\n            }\n        }\n    });\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init__webaudio(ma_context* pContext, const ma_context_config* pConfig, ma_backend_callbacks* pCallbacks)\n{\n    int resultFromJS;\n\n    MA_ASSERT(pContext != NULL);\n\n    (void)pConfig; /* Unused. */\n\n    /* Here is where our global JavaScript object is initialized. */\n    resultFromJS = EM_ASM_INT({\n        if (typeof window === 'undefined' || (window.AudioContext || window.webkitAudioContext) === undefined) {\n            return 0;   /* Web Audio not supported. */\n        }\n\n        if (typeof(window.miniaudio) === 'undefined') {\n            window.miniaudio = {\n                referenceCount: 0\n            };\n\n            /* Device types. */\n            window.miniaudio.device_type = {};\n            window.miniaudio.device_type.playback = $0;\n            window.miniaudio.device_type.capture  = $1;\n            window.miniaudio.device_type.duplex   = $2;\n\n            /* Device states. */\n            window.miniaudio.device_state = {};\n            window.miniaudio.device_state.stopped = $3;\n            window.miniaudio.device_state.started = $4;\n\n            /* Device cache for mapping devices to indexes for JavaScript/C interop. */\n            let miniaudio = window.miniaudio;\n            miniaudio.devices = [];\n\n            miniaudio.track_device = function(device) {\n                /* Try inserting into a free slot first. */\n                for (var iDevice = 0; iDevice < miniaudio.devices.length; ++iDevice) {\n                    if (miniaudio.devices[iDevice] == null) {\n                        miniaudio.devices[iDevice] = device;\n                        return iDevice;\n                    }\n                }\n\n                /* Getting here means there is no empty slots in the array so we just push to the end. */\n                miniaudio.devices.push(device);\n                return miniaudio.devices.length - 1;\n            };\n\n            miniaudio.untrack_device_by_index = function(deviceIndex) {\n                /* We just set the device's slot to null. The slot will get reused in the next call to ma_track_device. */\n                miniaudio.devices[deviceIndex] = null;\n\n                /* Trim the array if possible. */\n                while (miniaudio.devices.length > 0) {\n                    if (miniaudio.devices[miniaudio.devices.length-1] == null) {\n                        miniaudio.devices.pop();\n                    } else {\n                        break;\n                    }\n                }\n            };\n\n            miniaudio.untrack_device = function(device) {\n                for (var iDevice = 0; iDevice < miniaudio.devices.length; ++iDevice) {\n                    if (miniaudio.devices[iDevice] == device) {\n                        return miniaudio.untrack_device_by_index(iDevice);\n                    }\n                }\n            };\n\n            miniaudio.get_device_by_index = function(deviceIndex) {\n                return miniaudio.devices[deviceIndex];\n            };\n\n            miniaudio.unlock_event_types = (function(){\n                return ['touchend', 'click'];\n            })();\n\n            miniaudio.unlock = function() {\n                for(var i = 0; i < miniaudio.devices.length; ++i) {\n                    var device = miniaudio.devices[i];\n                    if (device != null &&\n                        device.webaudio != null &&\n                        device.state === miniaudio.device_state.started) {\n\n                        device.webaudio.resume().then(() => {\n                            _ma_device__on_notification_unlocked(device.pDevice);\n                        },\n                        (error) => {console.error(\"Failed to resume audiocontext\", error);\n                        });\n                    }\n                }\n                miniaudio.unlock_event_types.map(function(event_type) {\n                    document.removeEventListener(event_type, miniaudio.unlock, true);\n                });\n            };\n\n            miniaudio.unlock_event_types.map(function(event_type) {\n                document.addEventListener(event_type, miniaudio.unlock, true);\n            });\n        }\n\n        window.miniaudio.referenceCount += 1;\n\n        return 1;\n    }, ma_device_type_playback, ma_device_type_capture, ma_device_type_duplex, ma_device_state_stopped, ma_device_state_started);\n\n    if (resultFromJS != 1) {\n        return MA_FAILED_TO_INIT_BACKEND;\n    }\n\n    pCallbacks->onContextInit             = ma_context_init__webaudio;\n    pCallbacks->onContextUninit           = ma_context_uninit__webaudio;\n    pCallbacks->onContextEnumerateDevices = ma_context_enumerate_devices__webaudio;\n    pCallbacks->onContextGetDeviceInfo    = ma_context_get_device_info__webaudio;\n    pCallbacks->onDeviceInit              = ma_device_init__webaudio;\n    pCallbacks->onDeviceUninit            = ma_device_uninit__webaudio;\n    pCallbacks->onDeviceStart             = ma_device_start__webaudio;\n    pCallbacks->onDeviceStop              = ma_device_stop__webaudio;\n    pCallbacks->onDeviceRead              = NULL;   /* Not needed because WebAudio is asynchronous. */\n    pCallbacks->onDeviceWrite             = NULL;   /* Not needed because WebAudio is asynchronous. */\n    pCallbacks->onDeviceDataLoop          = NULL;   /* Not needed because WebAudio is asynchronous. */\n\n    return MA_SUCCESS;\n}\n#endif  /* MA_HAS_WEBAUDIO */\n\n\n\nstatic ma_bool32 ma__is_channel_map_valid(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    /* A blank channel map should be allowed, in which case it should use an appropriate default which will depend on context. */\n    if (pChannelMap != NULL && pChannelMap[0] != MA_CHANNEL_NONE) {\n        ma_uint32 iChannel;\n\n        if (channels == 0 || channels > MA_MAX_CHANNELS) {\n            return MA_FALSE;   /* Channel count out of range. */\n        }\n\n        /* A channel cannot be present in the channel map more than once. */\n        for (iChannel = 0; iChannel < channels; ++iChannel) {\n            ma_uint32 jChannel;\n            for (jChannel = iChannel + 1; jChannel < channels; ++jChannel) {\n                if (pChannelMap[iChannel] == pChannelMap[jChannel]) {\n                    return MA_FALSE;\n                }\n            }\n        }\n    }\n\n    return MA_TRUE;\n}\n\n\nstatic ma_bool32 ma_context_is_backend_asynchronous(ma_context* pContext)\n{\n    MA_ASSERT(pContext != NULL);\n\n    if (pContext->callbacks.onDeviceRead == NULL && pContext->callbacks.onDeviceWrite == NULL) {\n        if (pContext->callbacks.onDeviceDataLoop == NULL) {\n            return MA_TRUE;\n        } else {\n            return MA_FALSE;\n        }\n    } else {\n        return MA_FALSE;\n    }\n}\n\n\nstatic ma_result ma_device__post_init_setup(ma_device* pDevice, ma_device_type deviceType)\n{\n    ma_result result;\n\n    MA_ASSERT(pDevice != NULL);\n\n    if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n        if (pDevice->capture.format == ma_format_unknown) {\n            pDevice->capture.format = pDevice->capture.internalFormat;\n        }\n        if (pDevice->capture.channels == 0) {\n            pDevice->capture.channels = pDevice->capture.internalChannels;\n        }\n        if (pDevice->capture.channelMap[0] == MA_CHANNEL_NONE) {\n            MA_ASSERT(pDevice->capture.channels <= MA_MAX_CHANNELS);\n            if (pDevice->capture.internalChannels == pDevice->capture.channels) {\n                ma_channel_map_copy(pDevice->capture.channelMap, pDevice->capture.internalChannelMap, pDevice->capture.channels);\n            } else {\n                if (pDevice->capture.channelMixMode == ma_channel_mix_mode_simple) {\n                    ma_channel_map_init_blank(pDevice->capture.channelMap, pDevice->capture.channels);\n                } else {\n                    ma_channel_map_init_standard(ma_standard_channel_map_default, pDevice->capture.channelMap, ma_countof(pDevice->capture.channelMap), pDevice->capture.channels);\n                }\n            }\n        }\n    }\n\n    if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n        if (pDevice->playback.format == ma_format_unknown) {\n            pDevice->playback.format = pDevice->playback.internalFormat;\n        }\n        if (pDevice->playback.channels == 0) {\n            pDevice->playback.channels = pDevice->playback.internalChannels;\n        }\n        if (pDevice->playback.channelMap[0] == MA_CHANNEL_NONE) {\n            MA_ASSERT(pDevice->playback.channels <= MA_MAX_CHANNELS);\n            if (pDevice->playback.internalChannels == pDevice->playback.channels) {\n                ma_channel_map_copy(pDevice->playback.channelMap, pDevice->playback.internalChannelMap, pDevice->playback.channels);\n            } else {\n                if (pDevice->playback.channelMixMode == ma_channel_mix_mode_simple) {\n                    ma_channel_map_init_blank(pDevice->playback.channelMap, pDevice->playback.channels);\n                } else {\n                    ma_channel_map_init_standard(ma_standard_channel_map_default, pDevice->playback.channelMap, ma_countof(pDevice->playback.channelMap), pDevice->playback.channels);\n                }\n            }\n        }\n    }\n\n    if (pDevice->sampleRate == 0) {\n        if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n            pDevice->sampleRate = pDevice->capture.internalSampleRate;\n        } else {\n            pDevice->sampleRate = pDevice->playback.internalSampleRate;\n        }\n    }\n\n    /* Data converters. */\n    if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n        /* Converting from internal device format to client format. */\n        ma_data_converter_config converterConfig = ma_data_converter_config_init_default();\n        converterConfig.formatIn                        = pDevice->capture.internalFormat;\n        converterConfig.channelsIn                      = pDevice->capture.internalChannels;\n        converterConfig.sampleRateIn                    = pDevice->capture.internalSampleRate;\n        converterConfig.pChannelMapIn                   = pDevice->capture.internalChannelMap;\n        converterConfig.formatOut                       = pDevice->capture.format;\n        converterConfig.channelsOut                     = pDevice->capture.channels;\n        converterConfig.sampleRateOut                   = pDevice->sampleRate;\n        converterConfig.pChannelMapOut                  = pDevice->capture.channelMap;\n        converterConfig.channelMixMode                  = pDevice->capture.channelMixMode;\n        converterConfig.calculateLFEFromSpatialChannels = pDevice->capture.calculateLFEFromSpatialChannels;\n        converterConfig.allowDynamicSampleRate          = MA_FALSE;\n        converterConfig.resampling.algorithm            = pDevice->resampling.algorithm;\n        converterConfig.resampling.linear.lpfOrder      = pDevice->resampling.linear.lpfOrder;\n        converterConfig.resampling.pBackendVTable       = pDevice->resampling.pBackendVTable;\n        converterConfig.resampling.pBackendUserData     = pDevice->resampling.pBackendUserData;\n\n        /* Make sure the old converter is uninitialized first. */\n        if (ma_device_get_state(pDevice) != ma_device_state_uninitialized) {\n            ma_data_converter_uninit(&pDevice->capture.converter, &pDevice->pContext->allocationCallbacks);\n        }\n\n        result = ma_data_converter_init(&converterConfig, &pDevice->pContext->allocationCallbacks, &pDevice->capture.converter);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n        /* Converting from client format to device format. */\n        ma_data_converter_config converterConfig = ma_data_converter_config_init_default();\n        converterConfig.formatIn                        = pDevice->playback.format;\n        converterConfig.channelsIn                      = pDevice->playback.channels;\n        converterConfig.sampleRateIn                    = pDevice->sampleRate;\n        converterConfig.pChannelMapIn                   = pDevice->playback.channelMap;\n        converterConfig.formatOut                       = pDevice->playback.internalFormat;\n        converterConfig.channelsOut                     = pDevice->playback.internalChannels;\n        converterConfig.sampleRateOut                   = pDevice->playback.internalSampleRate;\n        converterConfig.pChannelMapOut                  = pDevice->playback.internalChannelMap;\n        converterConfig.channelMixMode                  = pDevice->playback.channelMixMode;\n        converterConfig.calculateLFEFromSpatialChannels = pDevice->playback.calculateLFEFromSpatialChannels;\n        converterConfig.allowDynamicSampleRate          = MA_FALSE;\n        converterConfig.resampling.algorithm            = pDevice->resampling.algorithm;\n        converterConfig.resampling.linear.lpfOrder      = pDevice->resampling.linear.lpfOrder;\n        converterConfig.resampling.pBackendVTable       = pDevice->resampling.pBackendVTable;\n        converterConfig.resampling.pBackendUserData     = pDevice->resampling.pBackendUserData;\n\n        /* Make sure the old converter is uninitialized first. */\n        if (ma_device_get_state(pDevice) != ma_device_state_uninitialized) {\n            ma_data_converter_uninit(&pDevice->playback.converter, &pDevice->pContext->allocationCallbacks);\n        }\n\n        result = ma_data_converter_init(&converterConfig, &pDevice->pContext->allocationCallbacks, &pDevice->playback.converter);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n\n    /*\n    If the device is doing playback (ma_device_type_playback or ma_device_type_duplex), there's\n    a couple of situations where we'll need a heap allocated cache.\n\n    The first is a duplex device for backends that use a callback for data delivery. The reason\n    this is needed is that the input stage needs to have a buffer to place the input data while it\n    waits for the playback stage, after which the miniaudio data callback will get fired. This is\n    not needed for backends that use a blocking API because miniaudio manages temporary buffers on\n    the stack to achieve this.\n\n    The other situation is when the data converter does not have the ability to query the number\n    of input frames that are required in order to process a given number of output frames. When\n    performing data conversion, it's useful if miniaudio know exactly how many frames it needs\n    from the client in order to generate a given number of output frames. This way, only exactly\n    the number of frames are needed to be read from the client which means no cache is necessary.\n    On the other hand, if miniaudio doesn't know how many frames to read, it is forced to read\n    in fixed sized chunks and then cache any residual unused input frames, those of which will be\n    processed at a later stage.\n    */\n    if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n        ma_uint64 unused;\n\n        pDevice->playback.inputCacheConsumed  = 0;\n        pDevice->playback.inputCacheRemaining = 0;\n\n        if (pDevice->type == ma_device_type_duplex ||                                                                       /* Duplex. backend may decide to use ma_device_handle_backend_data_callback() which will require this cache. */\n            ma_data_converter_get_required_input_frame_count(&pDevice->playback.converter, 1, &unused) != MA_SUCCESS)       /* Data conversion required input frame calculation not supported. */\n        {\n            /* We need a heap allocated cache. We want to size this based on the period size. */\n            void* pNewInputCache;\n            ma_uint64 newInputCacheCap;\n            ma_uint64 newInputCacheSizeInBytes;\n\n            newInputCacheCap = ma_calculate_frame_count_after_resampling(pDevice->playback.internalSampleRate, pDevice->sampleRate, pDevice->playback.internalPeriodSizeInFrames);\n\n            newInputCacheSizeInBytes = newInputCacheCap * ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n            if (newInputCacheSizeInBytes > MA_SIZE_MAX) {\n                ma_free(pDevice->playback.pInputCache, &pDevice->pContext->allocationCallbacks);\n                pDevice->playback.pInputCache   = NULL;\n                pDevice->playback.inputCacheCap = 0;\n                return MA_OUT_OF_MEMORY;    /* Allocation too big. Should never hit this, but makes the cast below safer for 32-bit builds. */\n            }\n\n            pNewInputCache = ma_realloc(pDevice->playback.pInputCache, (size_t)newInputCacheSizeInBytes, &pDevice->pContext->allocationCallbacks);\n            if (pNewInputCache == NULL) {\n                ma_free(pDevice->playback.pInputCache, &pDevice->pContext->allocationCallbacks);\n                pDevice->playback.pInputCache   = NULL;\n                pDevice->playback.inputCacheCap = 0;\n                return MA_OUT_OF_MEMORY;\n            }\n\n            pDevice->playback.pInputCache   = pNewInputCache;\n            pDevice->playback.inputCacheCap = newInputCacheCap;\n        } else {\n            /* Heap allocation not required. Make sure we clear out the old cache just in case this function was called in response to a route change. */\n            ma_free(pDevice->playback.pInputCache, &pDevice->pContext->allocationCallbacks);\n            pDevice->playback.pInputCache   = NULL;\n            pDevice->playback.inputCacheCap = 0;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_device_post_init(ma_device* pDevice, ma_device_type deviceType, const ma_device_descriptor* pDescriptorPlayback, const ma_device_descriptor* pDescriptorCapture)\n{\n    ma_result result;\n\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Capture. */\n    if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n        if (ma_device_descriptor_is_valid(pDescriptorCapture) == MA_FALSE) {\n            return MA_INVALID_ARGS;\n        }\n\n        pDevice->capture.internalFormat             = pDescriptorCapture->format;\n        pDevice->capture.internalChannels           = pDescriptorCapture->channels;\n        pDevice->capture.internalSampleRate         = pDescriptorCapture->sampleRate;\n        MA_COPY_MEMORY(pDevice->capture.internalChannelMap, pDescriptorCapture->channelMap, sizeof(pDescriptorCapture->channelMap));\n        pDevice->capture.internalPeriodSizeInFrames = pDescriptorCapture->periodSizeInFrames;\n        pDevice->capture.internalPeriods            = pDescriptorCapture->periodCount;\n\n        if (pDevice->capture.internalPeriodSizeInFrames == 0) {\n            pDevice->capture.internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(pDescriptorCapture->periodSizeInMilliseconds, pDescriptorCapture->sampleRate);\n        }\n    }\n\n    /* Playback. */\n    if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n        if (ma_device_descriptor_is_valid(pDescriptorPlayback) == MA_FALSE) {\n            return MA_INVALID_ARGS;\n        }\n\n        pDevice->playback.internalFormat             = pDescriptorPlayback->format;\n        pDevice->playback.internalChannels           = pDescriptorPlayback->channels;\n        pDevice->playback.internalSampleRate         = pDescriptorPlayback->sampleRate;\n        MA_COPY_MEMORY(pDevice->playback.internalChannelMap, pDescriptorPlayback->channelMap, sizeof(pDescriptorPlayback->channelMap));\n        pDevice->playback.internalPeriodSizeInFrames = pDescriptorPlayback->periodSizeInFrames;\n        pDevice->playback.internalPeriods            = pDescriptorPlayback->periodCount;\n\n        if (pDevice->playback.internalPeriodSizeInFrames == 0) {\n            pDevice->playback.internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(pDescriptorPlayback->periodSizeInMilliseconds, pDescriptorPlayback->sampleRate);\n        }\n    }\n\n    /*\n    The name of the device can be retrieved from device info. This may be temporary and replaced with a `ma_device_get_info(pDevice, deviceType)` instead.\n    For loopback devices, we need to retrieve the name of the playback device.\n    */\n    {\n        ma_device_info deviceInfo;\n\n        if (deviceType == ma_device_type_capture || deviceType == ma_device_type_duplex || deviceType == ma_device_type_loopback) {\n            result = ma_device_get_info(pDevice, ma_device_type_capture, &deviceInfo);\n            if (result == MA_SUCCESS) {\n                ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), deviceInfo.name, (size_t)-1);\n            } else {\n                /* We failed to retrieve the device info. Fall back to a default name. */\n                if (pDescriptorCapture->pDeviceID == NULL) {\n                    ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n                } else {\n                    ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), \"Capture Device\", (size_t)-1);\n                }\n            }\n        }\n\n        if (deviceType == ma_device_type_playback || deviceType == ma_device_type_duplex) {\n            result = ma_device_get_info(pDevice, ma_device_type_playback, &deviceInfo);\n            if (result == MA_SUCCESS) {\n                ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), deviceInfo.name, (size_t)-1);\n            } else {\n                /* We failed to retrieve the device info. Fall back to a default name. */\n                if (pDescriptorPlayback->pDeviceID == NULL) {\n                    ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n                } else {\n                    ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), \"Playback Device\", (size_t)-1);\n                }\n            }\n        }\n    }\n\n    /* Update data conversion. */\n    return ma_device__post_init_setup(pDevice, deviceType); /* TODO: Should probably rename ma_device__post_init_setup() to something better. */\n}\n\n\nstatic ma_thread_result MA_THREADCALL ma_worker_thread(void* pData)\n{\n    ma_device* pDevice = (ma_device*)pData;\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n    HRESULT CoInitializeResult;\n#endif\n\n    MA_ASSERT(pDevice != NULL);\n\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n    CoInitializeResult = ma_CoInitializeEx(pDevice->pContext, NULL, MA_COINIT_VALUE);\n#endif\n\n    /*\n    When the device is being initialized its initial state is set to ma_device_state_uninitialized. Before returning from\n    ma_device_init(), the state needs to be set to something valid. In miniaudio the device's default state immediately\n    after initialization is stopped, so therefore we need to mark the device as such. miniaudio will wait on the worker\n    thread to signal an event to know when the worker thread is ready for action.\n    */\n    ma_device__set_state(pDevice, ma_device_state_stopped);\n    ma_event_signal(&pDevice->stopEvent);\n\n    for (;;) {  /* <-- This loop just keeps the thread alive. The main audio loop is inside. */\n        ma_result startResult;\n        ma_result stopResult;   /* <-- This will store the result from onDeviceStop(). If it returns an error, we don't fire the stopped notification callback. */\n\n        /* We wait on an event to know when something has requested that the device be started and the main loop entered. */\n        ma_event_wait(&pDevice->wakeupEvent);\n\n        /* Default result code. */\n        pDevice->workResult = MA_SUCCESS;\n\n        /* If the reason for the wake up is that we are terminating, just break from the loop. */\n        if (ma_device_get_state(pDevice) == ma_device_state_uninitialized) {\n            break;\n        }\n\n        /*\n        Getting to this point means the device is wanting to get started. The function that has requested that the device\n        be started will be waiting on an event (pDevice->startEvent) which means we need to make sure we signal the event\n        in both the success and error case. It's important that the state of the device is set _before_ signaling the event.\n        */\n        MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_starting);\n\n        /* If the device has a start callback, start it now. */\n        if (pDevice->pContext->callbacks.onDeviceStart != NULL) {\n            startResult = pDevice->pContext->callbacks.onDeviceStart(pDevice);\n        } else {\n            startResult = MA_SUCCESS;\n        }\n\n        /*\n        If starting was not successful we'll need to loop back to the start and wait for something\n        to happen (pDevice->wakeupEvent).\n        */\n        if (startResult != MA_SUCCESS) {\n            pDevice->workResult = startResult;\n            ma_event_signal(&pDevice->startEvent);  /* <-- Always signal the start event so ma_device_start() can return as it'll be waiting on it. */\n            continue;\n        }\n\n        /* Make sure the state is set appropriately. */\n        ma_device__set_state(pDevice, ma_device_state_started); /* <-- Set this before signaling the event so that the state is always guaranteed to be good after ma_device_start() has returned. */\n        ma_event_signal(&pDevice->startEvent);\n\n        ma_device__on_notification_started(pDevice);\n\n        if (pDevice->pContext->callbacks.onDeviceDataLoop != NULL) {\n            pDevice->pContext->callbacks.onDeviceDataLoop(pDevice);\n        } else {\n            /* The backend is not using a custom main loop implementation, so now fall back to the blocking read-write implementation. */\n            ma_device_audio_thread__default_read_write(pDevice);\n        }\n\n        /* Getting here means we have broken from the main loop which happens the application has requested that device be stopped. */\n        if (pDevice->pContext->callbacks.onDeviceStop != NULL) {\n            stopResult = pDevice->pContext->callbacks.onDeviceStop(pDevice);\n        } else {\n            stopResult = MA_SUCCESS;    /* No stop callback with the backend. Just assume successful. */\n        }\n\n        /*\n        After the device has stopped, make sure an event is posted. Don't post a stopped event if\n        stopping failed. This can happen on some backends when the underlying stream has been\n        stopped due to the device being physically unplugged or disabled via an OS setting.\n        */\n        if (stopResult == MA_SUCCESS) {\n            ma_device__on_notification_stopped(pDevice);\n        }\n\n        /* If we stopped because the device has been uninitialized, abort now. */\n        if (ma_device_get_state(pDevice) == ma_device_state_uninitialized) {\n            break;\n        }\n\n        /* A function somewhere is waiting for the device to have stopped for real so we need to signal an event to allow it to continue. */\n        ma_device__set_state(pDevice, ma_device_state_stopped);\n        ma_event_signal(&pDevice->stopEvent);\n    }\n\n#if defined(MA_WIN32) && !defined(MA_XBOX)\n    if (CoInitializeResult == S_OK || CoInitializeResult == S_FALSE) {\n        ma_CoUninitialize(pDevice->pContext);\n    }\n#endif\n\n    return (ma_thread_result)0;\n}\n\n\n/* Helper for determining whether or not the given device is initialized. */\nstatic ma_bool32 ma_device__is_initialized(ma_device* pDevice)\n{\n    if (pDevice == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_device_get_state(pDevice) != ma_device_state_uninitialized;\n}\n\n\n#ifdef MA_WIN32\nstatic ma_result ma_context_uninit_backend_apis__win32(ma_context* pContext)\n{\n    /* For some reason UWP complains when CoUninitialize() is called. I'm just not going to call it on UWP. */\n    #if defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)\n    {\n        /* TODO: Remove this once the new single threaded backend system is in place in 0.12. */\n        #if !defined(MA_XBOX)\n        {\n            if (pContext->win32.CoInitializeResult == S_OK || pContext->win32.CoInitializeResult == S_FALSE) {\n                ma_CoUninitialize(pContext);    /* TODO: Remove this once the new single threaded backend system is in place in 0.12. */\n            }\n        }\n        #endif\n\n        #if defined(MA_WIN32_DESKTOP)\n            ma_dlclose(ma_context_get_log(pContext), pContext->win32.hUser32DLL);\n            ma_dlclose(ma_context_get_log(pContext), pContext->win32.hAdvapi32DLL);\n        #endif\n\n        ma_dlclose(ma_context_get_log(pContext), pContext->win32.hOle32DLL);\n    }\n    #else\n    {\n        (void)pContext;\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init_backend_apis__win32(ma_context* pContext)\n{\n    /*\n    TODO: Reassess all of this stuff and move everything to the relevant backends. For example, I think\n    GetForegroundWindow() and GetDesktopWindow() are only used by the DirectSound backend.\n    */\n    #if (defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_GDK)) && !defined(MA_XBOX)\n    {\n        #if defined(MA_WIN32_DESKTOP)\n        {\n            /* User32.dll */\n            pContext->win32.hUser32DLL = ma_dlopen(ma_context_get_log(pContext), \"user32.dll\");\n            if (pContext->win32.hUser32DLL == NULL) {\n                return MA_FAILED_TO_INIT_BACKEND;\n            }\n\n            pContext->win32.GetForegroundWindow = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hUser32DLL, \"GetForegroundWindow\");\n            pContext->win32.GetDesktopWindow    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hUser32DLL, \"GetDesktopWindow\");\n\n\n            /* Advapi32.dll */\n            pContext->win32.hAdvapi32DLL = ma_dlopen(ma_context_get_log(pContext), \"advapi32.dll\");\n            if (pContext->win32.hAdvapi32DLL == NULL) {\n                return MA_FAILED_TO_INIT_BACKEND;\n            }\n\n            pContext->win32.RegOpenKeyExA    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hAdvapi32DLL, \"RegOpenKeyExA\");\n            pContext->win32.RegCloseKey      = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hAdvapi32DLL, \"RegCloseKey\");\n            pContext->win32.RegQueryValueExA = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hAdvapi32DLL, \"RegQueryValueExA\");\n        }\n        #endif\n\n        /* Ole32.dll */\n        pContext->win32.hOle32DLL = ma_dlopen(ma_context_get_log(pContext), \"ole32.dll\");\n        if (pContext->win32.hOle32DLL == NULL) {\n            return MA_FAILED_TO_INIT_BACKEND;\n        }\n\n        pContext->win32.CoInitialize     = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"CoInitialize\");\n        pContext->win32.CoInitializeEx   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"CoInitializeEx\");\n        pContext->win32.CoUninitialize   = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"CoUninitialize\");\n        pContext->win32.CoCreateInstance = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"CoCreateInstance\");\n        pContext->win32.CoTaskMemFree    = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"CoTaskMemFree\");\n        pContext->win32.PropVariantClear = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"PropVariantClear\");\n        pContext->win32.StringFromGUID2  = (ma_proc)ma_dlsym(ma_context_get_log(pContext), pContext->win32.hOle32DLL, \"StringFromGUID2\");\n    }\n    #else\n    {\n        (void)pContext; /* Unused. */\n    }\n    #endif\n\n    /* TODO: Remove this once the new single threaded backend system is in place in 0.12. */\n    #if !defined(MA_XBOX)\n    {\n        pContext->win32.CoInitializeResult = ma_CoInitializeEx(pContext, NULL, MA_COINIT_VALUE);\n    }\n    #endif\n\n    return MA_SUCCESS;\n}\n#else\nstatic ma_result ma_context_uninit_backend_apis__nix(ma_context* pContext)\n{\n    (void)pContext;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_context_init_backend_apis__nix(ma_context* pContext)\n{\n    (void)pContext;\n\n    return MA_SUCCESS;\n}\n#endif\n\nstatic ma_result ma_context_init_backend_apis(ma_context* pContext)\n{\n    ma_result result;\n#ifdef MA_WIN32\n    result = ma_context_init_backend_apis__win32(pContext);\n#else\n    result = ma_context_init_backend_apis__nix(pContext);\n#endif\n\n    return result;\n}\n\nstatic ma_result ma_context_uninit_backend_apis(ma_context* pContext)\n{\n    ma_result result;\n#ifdef MA_WIN32\n    result = ma_context_uninit_backend_apis__win32(pContext);\n#else\n    result = ma_context_uninit_backend_apis__nix(pContext);\n#endif\n\n    return result;\n}\n\n\n/* The default capacity doesn't need to be too big. */\n#ifndef MA_DEFAULT_DEVICE_JOB_QUEUE_CAPACITY\n#define MA_DEFAULT_DEVICE_JOB_QUEUE_CAPACITY    32\n#endif\n\nMA_API ma_device_job_thread_config ma_device_job_thread_config_init(void)\n{\n    ma_device_job_thread_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.noThread         = MA_FALSE;\n    config.jobQueueCapacity = MA_DEFAULT_DEVICE_JOB_QUEUE_CAPACITY;\n    config.jobQueueFlags    = 0;\n\n    return config;\n}\n\n\nstatic ma_thread_result MA_THREADCALL ma_device_job_thread_entry(void* pUserData)\n{\n    ma_device_job_thread* pJobThread = (ma_device_job_thread*)pUserData;\n    MA_ASSERT(pJobThread != NULL);\n\n    for (;;) {\n        ma_result result;\n        ma_job job;\n\n        result = ma_device_job_thread_next(pJobThread, &job);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        if (job.toc.breakup.code == MA_JOB_TYPE_QUIT) {\n            break;\n        }\n\n        ma_job_process(&job);\n    }\n\n    return (ma_thread_result)0;\n}\n\nMA_API ma_result ma_device_job_thread_init(const ma_device_job_thread_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_device_job_thread* pJobThread)\n{\n    ma_result result;\n    ma_job_queue_config jobQueueConfig;\n\n    if (pJobThread == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pJobThread);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n\n    /* Initialize the job queue before the thread to ensure it's in a valid state. */\n    jobQueueConfig = ma_job_queue_config_init(pConfig->jobQueueFlags, pConfig->jobQueueCapacity);\n\n    result = ma_job_queue_init(&jobQueueConfig, pAllocationCallbacks, &pJobThread->jobQueue);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize job queue. */\n    }\n\n\n    /* The thread needs to be initialized after the job queue to ensure the thread doesn't try to access it prematurely. */\n    if (pConfig->noThread == MA_FALSE) {\n        result = ma_thread_create(&pJobThread->thread, ma_thread_priority_normal, 0, ma_device_job_thread_entry, pJobThread, pAllocationCallbacks);\n        if (result != MA_SUCCESS) {\n            ma_job_queue_uninit(&pJobThread->jobQueue, pAllocationCallbacks);\n            return result;  /* Failed to create the job thread. */\n        }\n\n        pJobThread->_hasThread = MA_TRUE;\n    } else {\n        pJobThread->_hasThread = MA_FALSE;\n    }\n\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_device_job_thread_uninit(ma_device_job_thread* pJobThread, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pJobThread == NULL) {\n        return;\n    }\n\n    /* The first thing to do is post a quit message to the job queue. If we're using a thread we'll need to wait for it. */\n    {\n        ma_job job = ma_job_init(MA_JOB_TYPE_QUIT);\n        ma_device_job_thread_post(pJobThread, &job);\n    }\n\n    /* Wait for the thread to terminate naturally. */\n    if (pJobThread->_hasThread) {\n        ma_thread_wait(&pJobThread->thread);\n    }\n\n    /* At this point the thread should be terminated so we can safely uninitialize the job queue. */\n    ma_job_queue_uninit(&pJobThread->jobQueue, pAllocationCallbacks);\n}\n\nMA_API ma_result ma_device_job_thread_post(ma_device_job_thread* pJobThread, const ma_job* pJob)\n{\n    if (pJobThread == NULL || pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_job_queue_post(&pJobThread->jobQueue, pJob);\n}\n\nMA_API ma_result ma_device_job_thread_next(ma_device_job_thread* pJobThread, ma_job* pJob)\n{\n    if (pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pJob);\n\n    if (pJobThread == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_job_queue_next(&pJobThread->jobQueue, pJob);\n}\n\n\nMA_API ma_bool32 ma_device_id_equal(const ma_device_id* pA, const ma_device_id* pB)\n{\n    size_t i;\n\n    if (pA == NULL || pB == NULL) {\n        return MA_FALSE;\n    }\n\n    for (i = 0; i < sizeof(ma_device_id); i += 1) {\n        if (((const char*)pA)[i] != ((const char*)pB)[i]) {\n            return MA_FALSE;\n        }\n    }\n\n    return MA_TRUE;\n}\n\n\n\nMA_API ma_context_config ma_context_config_init(void)\n{\n    ma_context_config config;\n    MA_ZERO_OBJECT(&config);\n\n    return config;\n}\n\nMA_API ma_result ma_context_init(const ma_backend backends[], ma_uint32 backendCount, const ma_context_config* pConfig, ma_context* pContext)\n{\n    ma_result result;\n    ma_context_config defaultConfig;\n    ma_backend defaultBackends[ma_backend_null+1];\n    ma_uint32 iBackend;\n    ma_backend* pBackendsToIterate;\n    ma_uint32 backendsToIterateCount;\n\n    if (pContext == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pContext);\n\n    /* Always make sure the config is set first to ensure properties are available as soon as possible. */\n    if (pConfig == NULL) {\n        defaultConfig = ma_context_config_init();\n        pConfig = &defaultConfig;\n    }\n\n    /* Allocation callbacks need to come first because they'll be passed around to other areas. */\n    result = ma_allocation_callbacks_init_copy(&pContext->allocationCallbacks, &pConfig->allocationCallbacks);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Get a lot set up first so we can start logging ASAP. */\n    if (pConfig->pLog != NULL) {\n        pContext->pLog = pConfig->pLog;\n    } else {\n        result = ma_log_init(&pContext->allocationCallbacks, &pContext->log);\n        if (result == MA_SUCCESS) {\n            pContext->pLog = &pContext->log;\n        } else {\n            pContext->pLog = NULL;  /* Logging is not available. */\n        }\n    }\n\n    pContext->threadPriority  = pConfig->threadPriority;\n    pContext->threadStackSize = pConfig->threadStackSize;\n    pContext->pUserData       = pConfig->pUserData;\n\n    /* Backend APIs need to be initialized first. This is where external libraries will be loaded and linked. */\n    result = ma_context_init_backend_apis(pContext);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    for (iBackend = 0; iBackend <= ma_backend_null; ++iBackend) {\n        defaultBackends[iBackend] = (ma_backend)iBackend;\n    }\n\n    pBackendsToIterate = (ma_backend*)backends;\n    backendsToIterateCount = backendCount;\n    if (pBackendsToIterate == NULL) {\n        pBackendsToIterate = (ma_backend*)defaultBackends;\n        backendsToIterateCount = ma_countof(defaultBackends);\n    }\n\n    MA_ASSERT(pBackendsToIterate != NULL);\n\n    for (iBackend = 0; iBackend < backendsToIterateCount; iBackend += 1) {\n        ma_backend backend = pBackendsToIterate[iBackend];\n\n        /* Make sure all callbacks are reset so we don't accidentally drag in any from previously failed initialization attempts. */\n        MA_ZERO_OBJECT(&pContext->callbacks);\n\n        /* These backends are using the new callback system. */\n        switch (backend) {\n        #ifdef MA_HAS_WASAPI\n            case ma_backend_wasapi:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__wasapi;\n            } break;\n        #endif\n        #ifdef MA_HAS_DSOUND\n            case ma_backend_dsound:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__dsound;\n            } break;\n        #endif\n        #ifdef MA_HAS_WINMM\n            case ma_backend_winmm:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__winmm;\n            } break;\n        #endif\n        #ifdef MA_HAS_COREAUDIO\n            case ma_backend_coreaudio:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__coreaudio;\n            } break;\n        #endif\n        #ifdef MA_HAS_SNDIO\n            case ma_backend_sndio:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__sndio;\n            } break;\n        #endif\n        #ifdef MA_HAS_AUDIO4\n            case ma_backend_audio4:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__audio4;\n            } break;\n        #endif\n        #ifdef MA_HAS_OSS\n            case ma_backend_oss:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__oss;\n            } break;\n        #endif\n        #ifdef MA_HAS_PULSEAUDIO\n            case ma_backend_pulseaudio:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__pulse;\n            } break;\n        #endif\n        #ifdef MA_HAS_ALSA\n            case ma_backend_alsa:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__alsa;\n            } break;\n        #endif\n        #ifdef MA_HAS_JACK\n            case ma_backend_jack:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__jack;\n            } break;\n        #endif\n        #ifdef MA_HAS_AAUDIO\n            case ma_backend_aaudio:\n            {\n                if (ma_is_backend_enabled(backend)) {\n                    pContext->callbacks.onContextInit = ma_context_init__aaudio;\n                }\n            } break;\n        #endif\n        #ifdef MA_HAS_OPENSL\n            case ma_backend_opensl:\n            {\n                if (ma_is_backend_enabled(backend)) {\n                    pContext->callbacks.onContextInit = ma_context_init__opensl;\n                }\n            } break;\n        #endif\n        #ifdef MA_HAS_WEBAUDIO\n            case ma_backend_webaudio:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__webaudio;\n            } break;\n        #endif\n        #ifdef MA_HAS_CUSTOM\n            case ma_backend_custom:\n            {\n                /* Slightly different logic for custom backends. Custom backends can optionally set all of their callbacks in the config. */\n                pContext->callbacks = pConfig->custom;\n            } break;\n        #endif\n        #ifdef MA_HAS_NULL\n            case ma_backend_null:\n            {\n                pContext->callbacks.onContextInit = ma_context_init__null;\n            } break;\n        #endif\n\n            default: break;\n        }\n\n        if (pContext->callbacks.onContextInit != NULL) {\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"Attempting to initialize %s backend...\\n\", ma_get_backend_name(backend));\n            result = pContext->callbacks.onContextInit(pContext, pConfig, &pContext->callbacks);\n        } else {\n            /* Getting here means the onContextInit callback is not set which means the backend is not enabled. Special case for the custom backend. */\n            if (backend != ma_backend_custom) {\n                result = MA_BACKEND_NOT_ENABLED;\n            } else {\n            #if !defined(MA_HAS_CUSTOM)\n                result = MA_BACKEND_NOT_ENABLED;\n            #else\n                result = MA_NO_BACKEND;\n            #endif\n            }\n        }\n\n        /* If this iteration was successful, return. */\n        if (result == MA_SUCCESS) {\n            result = ma_mutex_init(&pContext->deviceEnumLock);\n            if (result != MA_SUCCESS) {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"Failed to initialize mutex for device enumeration. ma_context_get_devices() is not thread safe.\\n\");\n            }\n\n            result = ma_mutex_init(&pContext->deviceInfoLock);\n            if (result != MA_SUCCESS) {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_WARNING, \"Failed to initialize mutex for device info retrieval. ma_context_get_device_info() is not thread safe.\\n\");\n            }\n\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"System Architecture:\\n\");\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"  Endian: %s\\n\", ma_is_little_endian() ? \"LE\"  : \"BE\");\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"  SSE2:   %s\\n\", ma_has_sse2()         ? \"YES\" : \"NO\");\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"  AVX2:   %s\\n\", ma_has_avx2()         ? \"YES\" : \"NO\");\n            ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"  NEON:   %s\\n\", ma_has_neon()         ? \"YES\" : \"NO\");\n\n            pContext->backend = backend;\n            return result;\n        } else {\n            if (result == MA_BACKEND_NOT_ENABLED) {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"%s backend is disabled.\\n\", ma_get_backend_name(backend));\n            } else {\n                ma_log_postf(ma_context_get_log(pContext), MA_LOG_LEVEL_DEBUG, \"Failed to initialize %s backend.\\n\", ma_get_backend_name(backend));\n            }\n        }\n    }\n\n    /* If we get here it means an error occurred. */\n    MA_ZERO_OBJECT(pContext);  /* Safety. */\n    return MA_NO_BACKEND;\n}\n\nMA_API ma_result ma_context_uninit(ma_context* pContext)\n{\n    if (pContext == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pContext->callbacks.onContextUninit != NULL) {\n        pContext->callbacks.onContextUninit(pContext);\n    }\n\n    ma_mutex_uninit(&pContext->deviceEnumLock);\n    ma_mutex_uninit(&pContext->deviceInfoLock);\n    ma_free(pContext->pDeviceInfos, &pContext->allocationCallbacks);\n    ma_context_uninit_backend_apis(pContext);\n\n    if (pContext->pLog == &pContext->log) {\n        ma_log_uninit(&pContext->log);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API size_t ma_context_sizeof(void)\n{\n    return sizeof(ma_context);\n}\n\n\nMA_API ma_log* ma_context_get_log(ma_context* pContext)\n{\n    if (pContext == NULL) {\n        return NULL;\n    }\n\n    return pContext->pLog;\n}\n\n\nMA_API ma_result ma_context_enumerate_devices(ma_context* pContext, ma_enum_devices_callback_proc callback, void* pUserData)\n{\n    ma_result result;\n\n    if (pContext == NULL || callback == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pContext->callbacks.onContextEnumerateDevices == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    ma_mutex_lock(&pContext->deviceEnumLock);\n    {\n        result = pContext->callbacks.onContextEnumerateDevices(pContext, callback, pUserData);\n    }\n    ma_mutex_unlock(&pContext->deviceEnumLock);\n\n    return result;\n}\n\n\nstatic ma_bool32 ma_context_get_devices__enum_callback(ma_context* pContext, ma_device_type deviceType, const ma_device_info* pInfo, void* pUserData)\n{\n    /*\n    We need to insert the device info into our main internal buffer. Where it goes depends on the device type. If it's a capture device\n    it's just appended to the end. If it's a playback device it's inserted just before the first capture device.\n    */\n\n    /*\n    First make sure we have room. Since the number of devices we add to the list is usually relatively small I've decided to use a\n    simple fixed size increment for buffer expansion.\n    */\n    const ma_uint32 bufferExpansionCount = 2;\n    const ma_uint32 totalDeviceInfoCount = pContext->playbackDeviceInfoCount + pContext->captureDeviceInfoCount;\n\n    if (totalDeviceInfoCount >= pContext->deviceInfoCapacity) {\n        ma_uint32 newCapacity = pContext->deviceInfoCapacity + bufferExpansionCount;\n        ma_device_info* pNewInfos = (ma_device_info*)ma_realloc(pContext->pDeviceInfos, sizeof(*pContext->pDeviceInfos)*newCapacity, &pContext->allocationCallbacks);\n        if (pNewInfos == NULL) {\n            return MA_FALSE;   /* Out of memory. */\n        }\n\n        pContext->pDeviceInfos = pNewInfos;\n        pContext->deviceInfoCapacity = newCapacity;\n    }\n\n    if (deviceType == ma_device_type_playback) {\n        /* Playback. Insert just before the first capture device. */\n\n        /* The first thing to do is move all of the capture devices down a slot. */\n        ma_uint32 iFirstCaptureDevice = pContext->playbackDeviceInfoCount;\n        size_t iCaptureDevice;\n        for (iCaptureDevice = totalDeviceInfoCount; iCaptureDevice > iFirstCaptureDevice; --iCaptureDevice) {\n            pContext->pDeviceInfos[iCaptureDevice] = pContext->pDeviceInfos[iCaptureDevice-1];\n        }\n\n        /* Now just insert where the first capture device was before moving it down a slot. */\n        pContext->pDeviceInfos[iFirstCaptureDevice] = *pInfo;\n        pContext->playbackDeviceInfoCount += 1;\n    } else {\n        /* Capture. Insert at the end. */\n        pContext->pDeviceInfos[totalDeviceInfoCount] = *pInfo;\n        pContext->captureDeviceInfoCount += 1;\n    }\n\n    (void)pUserData;\n    return MA_TRUE;\n}\n\nMA_API ma_result ma_context_get_devices(ma_context* pContext, ma_device_info** ppPlaybackDeviceInfos, ma_uint32* pPlaybackDeviceCount, ma_device_info** ppCaptureDeviceInfos, ma_uint32* pCaptureDeviceCount)\n{\n    ma_result result;\n\n    /* Safety. */\n    if (ppPlaybackDeviceInfos != NULL) *ppPlaybackDeviceInfos = NULL;\n    if (pPlaybackDeviceCount  != NULL) *pPlaybackDeviceCount  = 0;\n    if (ppCaptureDeviceInfos  != NULL) *ppCaptureDeviceInfos  = NULL;\n    if (pCaptureDeviceCount   != NULL) *pCaptureDeviceCount   = 0;\n\n    if (pContext == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pContext->callbacks.onContextEnumerateDevices == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* Note that we don't use ma_context_enumerate_devices() here because we want to do locking at a higher level. */\n    ma_mutex_lock(&pContext->deviceEnumLock);\n    {\n        /* Reset everything first. */\n        pContext->playbackDeviceInfoCount = 0;\n        pContext->captureDeviceInfoCount = 0;\n\n        /* Now enumerate over available devices. */\n        result = pContext->callbacks.onContextEnumerateDevices(pContext, ma_context_get_devices__enum_callback, NULL);\n        if (result == MA_SUCCESS) {\n            /* Playback devices. */\n            if (ppPlaybackDeviceInfos != NULL) {\n                *ppPlaybackDeviceInfos = pContext->pDeviceInfos;\n            }\n            if (pPlaybackDeviceCount != NULL) {\n                *pPlaybackDeviceCount = pContext->playbackDeviceInfoCount;\n            }\n\n            /* Capture devices. */\n            if (ppCaptureDeviceInfos != NULL) {\n                *ppCaptureDeviceInfos = pContext->pDeviceInfos;\n                /* Capture devices come after playback devices. */\n                if (pContext->playbackDeviceInfoCount > 0) {\n                    /* Conditional, because NULL+0 is undefined behavior. */\n                    *ppCaptureDeviceInfos += pContext->playbackDeviceInfoCount;\n                }\n            }\n            if (pCaptureDeviceCount != NULL) {\n                *pCaptureDeviceCount = pContext->captureDeviceInfoCount;\n            }\n        }\n    }\n    ma_mutex_unlock(&pContext->deviceEnumLock);\n\n    return result;\n}\n\nMA_API ma_result ma_context_get_device_info(ma_context* pContext, ma_device_type deviceType, const ma_device_id* pDeviceID, ma_device_info* pDeviceInfo)\n{\n    ma_result result;\n    ma_device_info deviceInfo;\n\n    /* NOTE: Do not clear pDeviceInfo on entry. The reason is the pDeviceID may actually point to pDeviceInfo->id which will break things. */\n    if (pContext == NULL || pDeviceInfo == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(&deviceInfo);\n\n    /* Help the backend out by copying over the device ID if we have one. */\n    if (pDeviceID != NULL) {\n        MA_COPY_MEMORY(&deviceInfo.id, pDeviceID, sizeof(*pDeviceID));\n    }\n\n    if (pContext->callbacks.onContextGetDeviceInfo == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    ma_mutex_lock(&pContext->deviceInfoLock);\n    {\n        result = pContext->callbacks.onContextGetDeviceInfo(pContext, deviceType, pDeviceID, &deviceInfo);\n    }\n    ma_mutex_unlock(&pContext->deviceInfoLock);\n\n    *pDeviceInfo = deviceInfo;\n    return result;\n}\n\nMA_API ma_bool32 ma_context_is_loopback_supported(ma_context* pContext)\n{\n    if (pContext == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_is_loopback_supported(pContext->backend);\n}\n\n\nMA_API ma_device_config ma_device_config_init(ma_device_type deviceType)\n{\n    ma_device_config config;\n    MA_ZERO_OBJECT(&config);\n    config.deviceType = deviceType;\n    config.resampling = ma_resampler_config_init(ma_format_unknown, 0, 0, 0, ma_resample_algorithm_linear); /* Format/channels/rate don't matter here. */\n\n    return config;\n}\n\nMA_API ma_result ma_device_init(ma_context* pContext, const ma_device_config* pConfig, ma_device* pDevice)\n{\n    ma_result result;\n    ma_device_descriptor descriptorPlayback;\n    ma_device_descriptor descriptorCapture;\n\n    /* The context can be null, in which case we self-manage it. */\n    if (pContext == NULL) {\n        return ma_device_init_ex(NULL, 0, NULL, pConfig, pDevice);\n    }\n\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDevice);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Check that we have our callbacks defined. */\n    if (pContext->callbacks.onDeviceInit == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* Basic config validation. */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex) {\n        if (pConfig->capture.channels > MA_MAX_CHANNELS) {\n            return MA_INVALID_ARGS;\n        }\n\n        if (!ma__is_channel_map_valid(pConfig->capture.pChannelMap, pConfig->capture.channels)) {\n            return MA_INVALID_ARGS;\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) {\n        if (pConfig->playback.channels > MA_MAX_CHANNELS) {\n            return MA_INVALID_ARGS;\n        }\n\n        if (!ma__is_channel_map_valid(pConfig->playback.pChannelMap, pConfig->playback.channels)) {\n            return MA_INVALID_ARGS;\n        }\n    }\n\n    pDevice->pContext = pContext;\n\n    /* Set the user data and log callback ASAP to ensure it is available for the entire initialization process. */\n    pDevice->pUserData      = pConfig->pUserData;\n    pDevice->onData         = pConfig->dataCallback;\n    pDevice->onNotification = pConfig->notificationCallback;\n    pDevice->onStop         = pConfig->stopCallback;\n\n    if (pConfig->playback.pDeviceID != NULL) {\n        MA_COPY_MEMORY(&pDevice->playback.id, pConfig->playback.pDeviceID, sizeof(pDevice->playback.id));\n        pDevice->playback.pID = &pDevice->playback.id;\n    } else {\n        pDevice->playback.pID = NULL;\n    }\n\n    if (pConfig->capture.pDeviceID != NULL) {\n        MA_COPY_MEMORY(&pDevice->capture.id, pConfig->capture.pDeviceID, sizeof(pDevice->capture.id));\n        pDevice->capture.pID = &pDevice->capture.id;\n    } else {\n        pDevice->capture.pID = NULL;\n    }\n\n    pDevice->noPreSilencedOutputBuffer   = pConfig->noPreSilencedOutputBuffer;\n    pDevice->noClip                      = pConfig->noClip;\n    pDevice->noDisableDenormals          = pConfig->noDisableDenormals;\n    pDevice->noFixedSizedCallback        = pConfig->noFixedSizedCallback;\n    ma_atomic_float_set(&pDevice->masterVolumeFactor, 1);\n\n    pDevice->type                        = pConfig->deviceType;\n    pDevice->sampleRate                  = pConfig->sampleRate;\n    pDevice->resampling.algorithm        = pConfig->resampling.algorithm;\n    pDevice->resampling.linear.lpfOrder  = pConfig->resampling.linear.lpfOrder;\n    pDevice->resampling.pBackendVTable   = pConfig->resampling.pBackendVTable;\n    pDevice->resampling.pBackendUserData = pConfig->resampling.pBackendUserData;\n\n    pDevice->capture.shareMode           = pConfig->capture.shareMode;\n    pDevice->capture.format              = pConfig->capture.format;\n    pDevice->capture.channels            = pConfig->capture.channels;\n    ma_channel_map_copy_or_default(pDevice->capture.channelMap, ma_countof(pDevice->capture.channelMap), pConfig->capture.pChannelMap, pConfig->capture.channels);\n    pDevice->capture.channelMixMode      = pConfig->capture.channelMixMode;\n    pDevice->capture.calculateLFEFromSpatialChannels = pConfig->capture.calculateLFEFromSpatialChannels;\n\n    pDevice->playback.shareMode          = pConfig->playback.shareMode;\n    pDevice->playback.format             = pConfig->playback.format;\n    pDevice->playback.channels           = pConfig->playback.channels;\n    ma_channel_map_copy_or_default(pDevice->playback.channelMap, ma_countof(pDevice->playback.channelMap), pConfig->playback.pChannelMap, pConfig->playback.channels);\n    pDevice->playback.channelMixMode     = pConfig->playback.channelMixMode;\n    pDevice->playback.calculateLFEFromSpatialChannels = pConfig->playback.calculateLFEFromSpatialChannels;\n\n    result = ma_mutex_init(&pDevice->startStopLock);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /*\n    When the device is started, the worker thread is the one that does the actual startup of the backend device. We\n    use a semaphore to wait for the background thread to finish the work. The same applies for stopping the device.\n\n    Each of these semaphores is released internally by the worker thread when the work is completed. The start\n    semaphore is also used to wake up the worker thread.\n    */\n    result = ma_event_init(&pDevice->wakeupEvent);\n    if (result != MA_SUCCESS) {\n        ma_mutex_uninit(&pDevice->startStopLock);\n        return result;\n    }\n\n    result = ma_event_init(&pDevice->startEvent);\n    if (result != MA_SUCCESS) {\n        ma_event_uninit(&pDevice->wakeupEvent);\n        ma_mutex_uninit(&pDevice->startStopLock);\n        return result;\n    }\n\n    result = ma_event_init(&pDevice->stopEvent);\n    if (result != MA_SUCCESS) {\n        ma_event_uninit(&pDevice->startEvent);\n        ma_event_uninit(&pDevice->wakeupEvent);\n        ma_mutex_uninit(&pDevice->startStopLock);\n        return result;\n    }\n\n\n    MA_ZERO_OBJECT(&descriptorPlayback);\n    descriptorPlayback.pDeviceID                = pConfig->playback.pDeviceID;\n    descriptorPlayback.shareMode                = pConfig->playback.shareMode;\n    descriptorPlayback.format                   = pConfig->playback.format;\n    descriptorPlayback.channels                 = pConfig->playback.channels;\n    descriptorPlayback.sampleRate               = pConfig->sampleRate;\n    ma_channel_map_copy_or_default(descriptorPlayback.channelMap, ma_countof(descriptorPlayback.channelMap), pConfig->playback.pChannelMap, pConfig->playback.channels);\n    descriptorPlayback.periodSizeInFrames       = pConfig->periodSizeInFrames;\n    descriptorPlayback.periodSizeInMilliseconds = pConfig->periodSizeInMilliseconds;\n    descriptorPlayback.periodCount              = pConfig->periods;\n\n    if (descriptorPlayback.periodCount == 0) {\n        descriptorPlayback.periodCount = MA_DEFAULT_PERIODS;\n    }\n\n\n    MA_ZERO_OBJECT(&descriptorCapture);\n    descriptorCapture.pDeviceID                 = pConfig->capture.pDeviceID;\n    descriptorCapture.shareMode                 = pConfig->capture.shareMode;\n    descriptorCapture.format                    = pConfig->capture.format;\n    descriptorCapture.channels                  = pConfig->capture.channels;\n    descriptorCapture.sampleRate                = pConfig->sampleRate;\n    ma_channel_map_copy_or_default(descriptorCapture.channelMap, ma_countof(descriptorCapture.channelMap), pConfig->capture.pChannelMap, pConfig->capture.channels);\n    descriptorCapture.periodSizeInFrames        = pConfig->periodSizeInFrames;\n    descriptorCapture.periodSizeInMilliseconds  = pConfig->periodSizeInMilliseconds;\n    descriptorCapture.periodCount               = pConfig->periods;\n\n    if (descriptorCapture.periodCount == 0) {\n        descriptorCapture.periodCount = MA_DEFAULT_PERIODS;\n    }\n\n\n    result = pContext->callbacks.onDeviceInit(pDevice, pConfig, &descriptorPlayback, &descriptorCapture);\n    if (result != MA_SUCCESS) {\n        ma_event_uninit(&pDevice->startEvent);\n        ma_event_uninit(&pDevice->wakeupEvent);\n        ma_mutex_uninit(&pDevice->startStopLock);\n        return result;\n    }\n\n#if 0\n    /*\n    On output the descriptors will contain the *actual* data format of the device. We need this to know how to convert the data between\n    the requested format and the internal format.\n    */\n    if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) {\n        if (!ma_device_descriptor_is_valid(&descriptorCapture)) {\n            ma_device_uninit(pDevice);\n            return MA_INVALID_ARGS;\n        }\n\n        pDevice->capture.internalFormat             = descriptorCapture.format;\n        pDevice->capture.internalChannels           = descriptorCapture.channels;\n        pDevice->capture.internalSampleRate         = descriptorCapture.sampleRate;\n        ma_channel_map_copy(pDevice->capture.internalChannelMap, descriptorCapture.channelMap, descriptorCapture.channels);\n        pDevice->capture.internalPeriodSizeInFrames = descriptorCapture.periodSizeInFrames;\n        pDevice->capture.internalPeriods            = descriptorCapture.periodCount;\n\n        if (pDevice->capture.internalPeriodSizeInFrames == 0) {\n            pDevice->capture.internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(descriptorCapture.periodSizeInMilliseconds, descriptorCapture.sampleRate);\n        }\n    }\n\n    if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n        if (!ma_device_descriptor_is_valid(&descriptorPlayback)) {\n            ma_device_uninit(pDevice);\n            return MA_INVALID_ARGS;\n        }\n\n        pDevice->playback.internalFormat             = descriptorPlayback.format;\n        pDevice->playback.internalChannels           = descriptorPlayback.channels;\n        pDevice->playback.internalSampleRate         = descriptorPlayback.sampleRate;\n        ma_channel_map_copy(pDevice->playback.internalChannelMap, descriptorPlayback.channelMap, descriptorPlayback.channels);\n        pDevice->playback.internalPeriodSizeInFrames = descriptorPlayback.periodSizeInFrames;\n        pDevice->playback.internalPeriods            = descriptorPlayback.periodCount;\n\n        if (pDevice->playback.internalPeriodSizeInFrames == 0) {\n            pDevice->playback.internalPeriodSizeInFrames = ma_calculate_buffer_size_in_frames_from_milliseconds(descriptorPlayback.periodSizeInMilliseconds, descriptorPlayback.sampleRate);\n        }\n    }\n\n\n    /*\n    The name of the device can be retrieved from device info. This may be temporary and replaced with a `ma_device_get_info(pDevice, deviceType)` instead.\n    For loopback devices, we need to retrieve the name of the playback device.\n    */\n    {\n        ma_device_info deviceInfo;\n\n        if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) {\n            result = ma_device_get_info(pDevice, (pConfig->deviceType == ma_device_type_loopback) ? ma_device_type_playback : ma_device_type_capture, &deviceInfo);\n            if (result == MA_SUCCESS) {\n                ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), deviceInfo.name, (size_t)-1);\n            } else {\n                /* We failed to retrieve the device info. Fall back to a default name. */\n                if (descriptorCapture.pDeviceID == NULL) {\n                    ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), MA_DEFAULT_CAPTURE_DEVICE_NAME, (size_t)-1);\n                } else {\n                    ma_strncpy_s(pDevice->capture.name, sizeof(pDevice->capture.name), \"Capture Device\", (size_t)-1);\n                }\n            }\n        }\n\n        if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n            result = ma_device_get_info(pDevice, ma_device_type_playback, &deviceInfo);\n            if (result == MA_SUCCESS) {\n                ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), deviceInfo.name, (size_t)-1);\n            } else {\n                /* We failed to retrieve the device info. Fall back to a default name. */\n                if (descriptorPlayback.pDeviceID == NULL) {\n                    ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), MA_DEFAULT_PLAYBACK_DEVICE_NAME, (size_t)-1);\n                } else {\n                    ma_strncpy_s(pDevice->playback.name, sizeof(pDevice->playback.name), \"Playback Device\", (size_t)-1);\n                }\n            }\n        }\n    }\n\n\n    ma_device__post_init_setup(pDevice, pConfig->deviceType);\n#endif\n\n    result = ma_device_post_init(pDevice, pConfig->deviceType, &descriptorPlayback, &descriptorCapture);\n    if (result != MA_SUCCESS) {\n        ma_device_uninit(pDevice);\n        return result;\n    }\n\n\n    /*\n    If we're using fixed sized callbacks we'll need to make use of an intermediary buffer. Needs to\n    be done after post_init_setup() because we'll need access to the sample rate.\n    */\n    if (pConfig->noFixedSizedCallback == MA_FALSE) {\n        /* We're using a fixed sized data callback so we'll need an intermediary buffer. */\n        ma_uint32 intermediaryBufferCap = pConfig->periodSizeInFrames;\n        if (intermediaryBufferCap == 0) {\n            intermediaryBufferCap = ma_calculate_buffer_size_in_frames_from_milliseconds(pConfig->periodSizeInMilliseconds, pDevice->sampleRate);\n        }\n\n        if (pConfig->deviceType == ma_device_type_capture || pConfig->deviceType == ma_device_type_duplex || pConfig->deviceType == ma_device_type_loopback) {\n            ma_uint32 intermediaryBufferSizeInBytes;\n\n            pDevice->capture.intermediaryBufferLen = 0;\n            pDevice->capture.intermediaryBufferCap = intermediaryBufferCap;\n            if (pDevice->capture.intermediaryBufferCap == 0) {\n                pDevice->capture.intermediaryBufferCap = pDevice->capture.internalPeriodSizeInFrames;\n            }\n\n            intermediaryBufferSizeInBytes = pDevice->capture.intermediaryBufferCap * ma_get_bytes_per_frame(pDevice->capture.format, pDevice->capture.channels);\n\n            pDevice->capture.pIntermediaryBuffer = ma_malloc((size_t)intermediaryBufferSizeInBytes, &pContext->allocationCallbacks);\n            if (pDevice->capture.pIntermediaryBuffer == NULL) {\n                ma_device_uninit(pDevice);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            /* Silence the buffer for safety. */\n            ma_silence_pcm_frames(pDevice->capture.pIntermediaryBuffer, pDevice->capture.intermediaryBufferCap, pDevice->capture.format, pDevice->capture.channels);\n            pDevice->capture.intermediaryBufferLen = pDevice->capture.intermediaryBufferCap;\n        }\n\n        if (pConfig->deviceType == ma_device_type_playback || pConfig->deviceType == ma_device_type_duplex) {\n            ma_uint64 intermediaryBufferSizeInBytes;\n\n            pDevice->playback.intermediaryBufferLen = 0;\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                pDevice->playback.intermediaryBufferCap = pDevice->capture.intermediaryBufferCap;   /* In duplex mode, make sure the intermediary buffer is always the same size as the capture side. */\n            } else {\n                pDevice->playback.intermediaryBufferCap = intermediaryBufferCap;\n                if (pDevice->playback.intermediaryBufferCap == 0) {\n                    pDevice->playback.intermediaryBufferCap = pDevice->playback.internalPeriodSizeInFrames;\n                }\n            }\n\n            intermediaryBufferSizeInBytes = pDevice->playback.intermediaryBufferCap * ma_get_bytes_per_frame(pDevice->playback.format, pDevice->playback.channels);\n\n            pDevice->playback.pIntermediaryBuffer = ma_malloc((size_t)intermediaryBufferSizeInBytes, &pContext->allocationCallbacks);\n            if (pDevice->playback.pIntermediaryBuffer == NULL) {\n                ma_device_uninit(pDevice);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            /* Silence the buffer for safety. */\n            ma_silence_pcm_frames(pDevice->playback.pIntermediaryBuffer, pDevice->playback.intermediaryBufferCap, pDevice->playback.format, pDevice->playback.channels);\n            pDevice->playback.intermediaryBufferLen = 0;\n        }\n    } else {\n        /* Not using a fixed sized data callback so no need for an intermediary buffer. */\n    }\n\n\n    /* Some backends don't require the worker thread. */\n    if (!ma_context_is_backend_asynchronous(pContext)) {\n        /* The worker thread. */\n        result = ma_thread_create(&pDevice->thread, pContext->threadPriority, pContext->threadStackSize, ma_worker_thread, pDevice, &pContext->allocationCallbacks);\n        if (result != MA_SUCCESS) {\n            ma_device_uninit(pDevice);\n            return result;\n        }\n\n        /* Wait for the worker thread to put the device into its stopped state for real. */\n        ma_event_wait(&pDevice->stopEvent);\n        MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_stopped);\n    } else {\n        /*\n        If the backend is asynchronous and the device is duplex, we'll need an intermediary ring buffer. Note that this needs to be done\n        after ma_device__post_init_setup().\n        */\n        if (ma_context_is_backend_asynchronous(pContext)) {\n            if (pConfig->deviceType == ma_device_type_duplex) {\n                result = ma_duplex_rb_init(pDevice->capture.format, pDevice->capture.channels, pDevice->sampleRate, pDevice->capture.internalSampleRate, pDevice->capture.internalPeriodSizeInFrames, &pDevice->pContext->allocationCallbacks, &pDevice->duplexRB);\n                if (result != MA_SUCCESS) {\n                    ma_device_uninit(pDevice);\n                    return result;\n                }\n            }\n        }\n\n        ma_device__set_state(pDevice, ma_device_state_stopped);\n    }\n\n    /* Log device information. */\n    {\n        ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"[%s]\\n\", ma_get_backend_name(pDevice->pContext->backend));\n        if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n            char name[MA_MAX_DEVICE_NAME_LENGTH + 1];\n            ma_device_get_name(pDevice, ma_device_type_capture, name, sizeof(name), NULL);\n\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"  %s (%s)\\n\", name, \"Capture\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Format:      %s -> %s\\n\", ma_get_format_name(pDevice->capture.internalFormat), ma_get_format_name(pDevice->capture.format));\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Channels:    %d -> %d\\n\", pDevice->capture.internalChannels, pDevice->capture.channels);\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Sample Rate: %d -> %d\\n\", pDevice->capture.internalSampleRate, pDevice->sampleRate);\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Buffer Size: %d*%d (%d)\\n\", pDevice->capture.internalPeriodSizeInFrames, pDevice->capture.internalPeriods, (pDevice->capture.internalPeriodSizeInFrames * pDevice->capture.internalPeriods));\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Conversion:\\n\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Pre Format Conversion:  %s\\n\", pDevice->capture.converter.hasPreFormatConversion  ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Post Format Conversion: %s\\n\", pDevice->capture.converter.hasPostFormatConversion ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Routing:        %s\\n\", pDevice->capture.converter.hasChannelConverter     ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Resampling:             %s\\n\", pDevice->capture.converter.hasResampler            ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Passthrough:            %s\\n\", pDevice->capture.converter.isPassthrough           ? \"YES\" : \"NO\");\n            {\n                char channelMapStr[1024];\n                ma_channel_map_to_string(pDevice->capture.internalChannelMap, pDevice->capture.internalChannels, channelMapStr, sizeof(channelMapStr));\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Map In:         {%s}\\n\", channelMapStr);\n\n                ma_channel_map_to_string(pDevice->capture.channelMap, pDevice->capture.channels, channelMapStr, sizeof(channelMapStr));\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Map Out:        {%s}\\n\", channelMapStr);\n            }\n        }\n        if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n            char name[MA_MAX_DEVICE_NAME_LENGTH + 1];\n            ma_device_get_name(pDevice, ma_device_type_playback, name, sizeof(name), NULL);\n\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"  %s (%s)\\n\", name, \"Playback\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Format:      %s -> %s\\n\", ma_get_format_name(pDevice->playback.format), ma_get_format_name(pDevice->playback.internalFormat));\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Channels:    %d -> %d\\n\", pDevice->playback.channels, pDevice->playback.internalChannels);\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Sample Rate: %d -> %d\\n\", pDevice->sampleRate, pDevice->playback.internalSampleRate);\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Buffer Size: %d*%d (%d)\\n\", pDevice->playback.internalPeriodSizeInFrames, pDevice->playback.internalPeriods, (pDevice->playback.internalPeriodSizeInFrames * pDevice->playback.internalPeriods));\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"    Conversion:\\n\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Pre Format Conversion:  %s\\n\", pDevice->playback.converter.hasPreFormatConversion  ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Post Format Conversion: %s\\n\", pDevice->playback.converter.hasPostFormatConversion ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Routing:        %s\\n\", pDevice->playback.converter.hasChannelConverter     ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Resampling:             %s\\n\", pDevice->playback.converter.hasResampler            ? \"YES\" : \"NO\");\n            ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Passthrough:            %s\\n\", pDevice->playback.converter.isPassthrough           ? \"YES\" : \"NO\");\n            {\n                char channelMapStr[1024];\n                ma_channel_map_to_string(pDevice->playback.channelMap, pDevice->playback.channels, channelMapStr, sizeof(channelMapStr));\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Map In:         {%s}\\n\", channelMapStr);\n\n                ma_channel_map_to_string(pDevice->playback.internalChannelMap, pDevice->playback.internalChannels, channelMapStr, sizeof(channelMapStr));\n                ma_log_postf(ma_device_get_log(pDevice), MA_LOG_LEVEL_INFO, \"      Channel Map Out:        {%s}\\n\", channelMapStr);\n            }\n        }\n    }\n\n    MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_stopped);\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_device_init_ex(const ma_backend backends[], ma_uint32 backendCount, const ma_context_config* pContextConfig, const ma_device_config* pConfig, ma_device* pDevice)\n{\n    ma_result result;\n    ma_context* pContext;\n    ma_backend defaultBackends[ma_backend_null+1];\n    ma_uint32 iBackend;\n    ma_backend* pBackendsToIterate;\n    ma_uint32 backendsToIterateCount;\n    ma_allocation_callbacks allocationCallbacks;\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pContextConfig != NULL) {\n        result = ma_allocation_callbacks_init_copy(&allocationCallbacks, &pContextConfig->allocationCallbacks);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        allocationCallbacks = ma_allocation_callbacks_init_default();\n    }\n\n    pContext = (ma_context*)ma_malloc(sizeof(*pContext), &allocationCallbacks);\n    if (pContext == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    for (iBackend = 0; iBackend <= ma_backend_null; ++iBackend) {\n        defaultBackends[iBackend] = (ma_backend)iBackend;\n    }\n\n    pBackendsToIterate = (ma_backend*)backends;\n    backendsToIterateCount = backendCount;\n    if (pBackendsToIterate == NULL) {\n        pBackendsToIterate = (ma_backend*)defaultBackends;\n        backendsToIterateCount = ma_countof(defaultBackends);\n    }\n\n    result = MA_NO_BACKEND;\n\n    for (iBackend = 0; iBackend < backendsToIterateCount; ++iBackend) {\n        /*\n        This is a hack for iOS. If the context config is null, there's a good chance the\n        `ma_device_init(NULL, &deviceConfig, pDevice);` pattern is being used. In this\n        case, set the session category based on the device type.\n        */\n    #if defined(MA_APPLE_MOBILE)\n        ma_context_config contextConfig;\n\n        if (pContextConfig == NULL) {\n            contextConfig = ma_context_config_init();\n            switch (pConfig->deviceType) {\n                case ma_device_type_duplex: {\n                    contextConfig.coreaudio.sessionCategory = ma_ios_session_category_play_and_record;\n                } break;\n                case ma_device_type_capture: {\n                    contextConfig.coreaudio.sessionCategory = ma_ios_session_category_record;\n                } break;\n                case ma_device_type_playback:\n                default: {\n                    contextConfig.coreaudio.sessionCategory = ma_ios_session_category_playback;\n                } break;\n            }\n\n            pContextConfig = &contextConfig;\n        }\n    #endif\n\n        result = ma_context_init(&pBackendsToIterate[iBackend], 1, pContextConfig, pContext);\n        if (result == MA_SUCCESS) {\n            result = ma_device_init(pContext, pConfig, pDevice);\n            if (result == MA_SUCCESS) {\n                break;  /* Success. */\n            } else {\n                ma_context_uninit(pContext);   /* Failure. */\n            }\n        }\n    }\n\n    if (result != MA_SUCCESS) {\n        ma_free(pContext, &allocationCallbacks);\n        return result;\n    }\n\n    pDevice->isOwnerOfContext = MA_TRUE;\n    return result;\n}\n\nMA_API void ma_device_uninit(ma_device* pDevice)\n{\n    if (!ma_device__is_initialized(pDevice)) {\n        return;\n    }\n\n    /*\n    It's possible for the miniaudio side of the device and the backend to not be in sync due to\n    system-level situations such as the computer being put into sleep mode and the backend not\n    notifying miniaudio of the fact the device has stopped. It's possible for this to result in a\n    deadlock due to miniaudio thinking the device is in a running state, when in fact it's not\n    running at all. For this reason I am no longer explicitly stopping the device. I don't think\n    this should affect anyone in practice since uninitializing the backend will naturally stop the\n    device anyway.\n    */\n    #if 0\n    {\n        /* Make sure the device is stopped first. The backends will probably handle this naturally, but I like to do it explicitly for my own sanity. */\n        if (ma_device_is_started(pDevice)) {\n            ma_device_stop(pDevice);\n        }\n    }\n    #endif\n\n    /* Putting the device into an uninitialized state will make the worker thread return. */\n    ma_device__set_state(pDevice, ma_device_state_uninitialized);\n\n    /* Wake up the worker thread and wait for it to properly terminate. */\n    if (!ma_context_is_backend_asynchronous(pDevice->pContext)) {\n        ma_event_signal(&pDevice->wakeupEvent);\n        ma_thread_wait(&pDevice->thread);\n    }\n\n    if (pDevice->pContext->callbacks.onDeviceUninit != NULL) {\n        pDevice->pContext->callbacks.onDeviceUninit(pDevice);\n    }\n\n\n    ma_event_uninit(&pDevice->stopEvent);\n    ma_event_uninit(&pDevice->startEvent);\n    ma_event_uninit(&pDevice->wakeupEvent);\n    ma_mutex_uninit(&pDevice->startStopLock);\n\n    if (ma_context_is_backend_asynchronous(pDevice->pContext)) {\n        if (pDevice->type == ma_device_type_duplex) {\n            ma_duplex_rb_uninit(&pDevice->duplexRB);\n        }\n    }\n\n    if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_duplex || pDevice->type == ma_device_type_loopback) {\n        ma_data_converter_uninit(&pDevice->capture.converter, &pDevice->pContext->allocationCallbacks);\n    }\n    if (pDevice->type == ma_device_type_playback || pDevice->type == ma_device_type_duplex) {\n        ma_data_converter_uninit(&pDevice->playback.converter, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if (pDevice->playback.pInputCache != NULL) {\n        ma_free(pDevice->playback.pInputCache, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if (pDevice->capture.pIntermediaryBuffer != NULL) {\n        ma_free(pDevice->capture.pIntermediaryBuffer, &pDevice->pContext->allocationCallbacks);\n    }\n    if (pDevice->playback.pIntermediaryBuffer != NULL) {\n        ma_free(pDevice->playback.pIntermediaryBuffer, &pDevice->pContext->allocationCallbacks);\n    }\n\n    if (pDevice->isOwnerOfContext) {\n        ma_allocation_callbacks allocationCallbacks = pDevice->pContext->allocationCallbacks;\n\n        ma_context_uninit(pDevice->pContext);\n        ma_free(pDevice->pContext, &allocationCallbacks);\n    }\n\n    MA_ZERO_OBJECT(pDevice);\n}\n\nMA_API ma_context* ma_device_get_context(ma_device* pDevice)\n{\n    if (pDevice == NULL) {\n        return NULL;\n    }\n\n    return pDevice->pContext;\n}\n\nMA_API ma_log* ma_device_get_log(ma_device* pDevice)\n{\n    return ma_context_get_log(ma_device_get_context(pDevice));\n}\n\nMA_API ma_result ma_device_get_info(ma_device* pDevice, ma_device_type type, ma_device_info* pDeviceInfo)\n{\n    if (pDeviceInfo == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDeviceInfo);\n\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If the onDeviceGetInfo() callback is set, use that. Otherwise we'll fall back to ma_context_get_device_info(). */\n    if (pDevice->pContext->callbacks.onDeviceGetInfo != NULL) {\n        return pDevice->pContext->callbacks.onDeviceGetInfo(pDevice, type, pDeviceInfo);\n    }\n\n    /* Getting here means onDeviceGetInfo is not implemented so we need to fall back to an alternative. */\n    if (type == ma_device_type_playback) {\n        return ma_context_get_device_info(pDevice->pContext, type, pDevice->playback.pID, pDeviceInfo);\n    } else {\n        /*\n        Here we're getting the capture side, which is the branch we'll be entering for a loopback\n        device, since loopback is capturing. However, if the device is using the default device ID,\n        it won't get the correct information because it'll think we're asking for the default\n        capture device, where in fact for loopback we want the default *playback* device. We'll do\n        a bit of a hack here to make sure we get the correct info.\n        */\n        if (pDevice->type == ma_device_type_loopback && pDevice->capture.pID == NULL) {\n            type = ma_device_type_playback;\n        }\n\n        return ma_context_get_device_info(pDevice->pContext, type, pDevice->capture.pID, pDeviceInfo);\n    }\n}\n\nMA_API ma_result ma_device_get_name(ma_device* pDevice, ma_device_type type, char* pName, size_t nameCap, size_t* pLengthNotIncludingNullTerminator)\n{\n    ma_result result;\n    ma_device_info deviceInfo;\n\n    if (pLengthNotIncludingNullTerminator != NULL) {\n        *pLengthNotIncludingNullTerminator = 0;\n    }\n\n    if (pName != NULL && nameCap > 0) {\n        pName[0] = '\\0';\n    }\n\n    result = ma_device_get_info(pDevice, type, &deviceInfo);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pName != NULL) {\n        ma_strncpy_s(pName, nameCap, deviceInfo.name, (size_t)-1);\n\n        /*\n        For safety, make sure the length is based on the truncated output string rather than the\n        source. Otherwise the caller might assume the output buffer contains more content than it\n        actually does.\n        */\n        if (pLengthNotIncludingNullTerminator != NULL) {\n            *pLengthNotIncludingNullTerminator = strlen(pName);\n        }\n    } else {\n        /* Name not specified. Just report the length of the source string. */\n        if (pLengthNotIncludingNullTerminator != NULL) {\n            *pLengthNotIncludingNullTerminator = strlen(deviceInfo.name);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_device_start(ma_device* pDevice)\n{\n    ma_result result;\n\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_device_get_state(pDevice) == ma_device_state_uninitialized) {\n        return MA_INVALID_OPERATION;    /* Not initialized. */\n    }\n\n    if (ma_device_get_state(pDevice) == ma_device_state_started) {\n        return MA_SUCCESS;  /* Already started. */\n    }\n\n    ma_mutex_lock(&pDevice->startStopLock);\n    {\n        /*\n        We need to check again if the device is in a started state because it's possible for one thread to have started the device\n        while another was waiting on the mutex.\n        */\n        if (ma_device_get_state(pDevice) == ma_device_state_started) {\n            ma_mutex_unlock(&pDevice->startStopLock);\n            return MA_SUCCESS;  /* Already started. */\n        }\n\n        /* Starting and stopping are wrapped in a mutex which means we can assert that the device is in a stopped or paused state. */\n        MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_stopped);\n\n        ma_device__set_state(pDevice, ma_device_state_starting);\n\n        /* Asynchronous backends need to be handled differently. */\n        if (ma_context_is_backend_asynchronous(pDevice->pContext)) {\n            if (pDevice->pContext->callbacks.onDeviceStart != NULL) {\n                result = pDevice->pContext->callbacks.onDeviceStart(pDevice);\n            } else {\n                result = MA_INVALID_OPERATION;\n            }\n\n            if (result == MA_SUCCESS) {\n                ma_device__set_state(pDevice, ma_device_state_started);\n                ma_device__on_notification_started(pDevice);\n            }\n        } else {\n            /*\n            Synchronous backends are started by signaling an event that's being waited on in the worker thread. We first wake up the\n            thread and then wait for the start event.\n            */\n            ma_event_signal(&pDevice->wakeupEvent);\n\n            /*\n            Wait for the worker thread to finish starting the device. Note that the worker thread will be the one who puts the device\n            into the started state. Don't call ma_device__set_state() here.\n            */\n            ma_event_wait(&pDevice->startEvent);\n            result = pDevice->workResult;\n        }\n\n        /* We changed the state from stopped to started, so if we failed, make sure we put the state back to stopped. */\n        if (result != MA_SUCCESS) {\n            ma_device__set_state(pDevice, ma_device_state_stopped);\n        }\n    }\n    ma_mutex_unlock(&pDevice->startStopLock);\n\n    return result;\n}\n\nMA_API ma_result ma_device_stop(ma_device* pDevice)\n{\n    ma_result result;\n\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_device_get_state(pDevice) == ma_device_state_uninitialized) {\n        return MA_INVALID_OPERATION;    /* Not initialized. */\n    }\n\n    if (ma_device_get_state(pDevice) == ma_device_state_stopped) {\n        return MA_SUCCESS;  /* Already stopped. */\n    }\n\n    ma_mutex_lock(&pDevice->startStopLock);\n    {\n        /*\n        We need to check again if the device is in a stopped state because it's possible for one thread to have stopped the device\n        while another was waiting on the mutex.\n        */\n        if (ma_device_get_state(pDevice) == ma_device_state_stopped) {\n            ma_mutex_unlock(&pDevice->startStopLock);\n            return MA_SUCCESS;  /* Already stopped. */\n        }\n\n        /* Starting and stopping are wrapped in a mutex which means we can assert that the device is in a started or paused state. */\n        MA_ASSERT(ma_device_get_state(pDevice) == ma_device_state_started);\n\n        ma_device__set_state(pDevice, ma_device_state_stopping);\n\n        /* Asynchronous backends need to be handled differently. */\n        if (ma_context_is_backend_asynchronous(pDevice->pContext)) {\n            /* Asynchronous backends must have a stop operation. */\n            if (pDevice->pContext->callbacks.onDeviceStop != NULL) {\n                result = pDevice->pContext->callbacks.onDeviceStop(pDevice);\n            } else {\n                result = MA_INVALID_OPERATION;\n            }\n\n            ma_device__set_state(pDevice, ma_device_state_stopped);\n        } else {\n            /*\n            Synchronous backends. The stop callback is always called from the worker thread. Do not call the stop callback here. If\n            the backend is implementing its own audio thread loop we'll need to wake it up if required. Note that we need to make\n            sure the state of the device is *not* playing right now, which it shouldn't be since we set it above. This is super\n            important though, so I'm asserting it here as well for extra safety in case we accidentally change something later.\n            */\n            MA_ASSERT(ma_device_get_state(pDevice) != ma_device_state_started);\n\n            if (pDevice->pContext->callbacks.onDeviceDataLoopWakeup != NULL) {\n                pDevice->pContext->callbacks.onDeviceDataLoopWakeup(pDevice);\n            }\n\n            /*\n            We need to wait for the worker thread to become available for work before returning. Note that the worker thread will be\n            the one who puts the device into the stopped state. Don't call ma_device__set_state() here.\n            */\n            ma_event_wait(&pDevice->stopEvent);\n            result = MA_SUCCESS;\n        }\n\n        /*\n        This is a safety measure to ensure the internal buffer has been cleared so any leftover\n        does not get played the next time the device starts. Ideally this should be drained by\n        the backend first.\n        */\n        pDevice->playback.intermediaryBufferLen = 0;\n        pDevice->playback.inputCacheConsumed    = 0;\n        pDevice->playback.inputCacheRemaining   = 0;\n    }\n    ma_mutex_unlock(&pDevice->startStopLock);\n\n    return result;\n}\n\nMA_API ma_bool32 ma_device_is_started(const ma_device* pDevice)\n{\n    return ma_device_get_state(pDevice) == ma_device_state_started;\n}\n\nMA_API ma_device_state ma_device_get_state(const ma_device* pDevice)\n{\n    if (pDevice == NULL) {\n        return ma_device_state_uninitialized;\n    }\n\n    return ma_atomic_device_state_get((ma_atomic_device_state*)&pDevice->state);   /* Naughty cast to get rid of a const warning. */\n}\n\nMA_API ma_result ma_device_set_master_volume(ma_device* pDevice, float volume)\n{\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (volume < 0.0f) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_float_set(&pDevice->masterVolumeFactor, volume);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_device_get_master_volume(ma_device* pDevice, float* pVolume)\n{\n    if (pVolume == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDevice == NULL) {\n        *pVolume = 0;\n        return MA_INVALID_ARGS;\n    }\n\n    *pVolume = ma_atomic_float_get(&pDevice->masterVolumeFactor);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_device_set_master_volume_db(ma_device* pDevice, float gainDB)\n{\n    if (gainDB > 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_device_set_master_volume(pDevice, ma_volume_db_to_linear(gainDB));\n}\n\nMA_API ma_result ma_device_get_master_volume_db(ma_device* pDevice, float* pGainDB)\n{\n    float factor;\n    ma_result result;\n\n    if (pGainDB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_device_get_master_volume(pDevice, &factor);\n    if (result != MA_SUCCESS) {\n        *pGainDB = 0;\n        return result;\n    }\n\n    *pGainDB = ma_volume_linear_to_db(factor);\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_device_handle_backend_data_callback(ma_device* pDevice, void* pOutput, const void* pInput, ma_uint32 frameCount)\n{\n    if (pDevice == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pOutput == NULL && pInput == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    There is an assert deeper in the code that checks that frameCount > 0. Since this is a public facing\n    API we'll need to check for that here. I've had reports that AAudio can sometimes post a frame count\n    of 0.\n    */\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDevice->type == ma_device_type_duplex) {\n        if (pInput != NULL) {\n            ma_device__handle_duplex_callback_capture(pDevice, frameCount, pInput, &pDevice->duplexRB.rb);\n        }\n\n        if (pOutput != NULL) {\n            ma_device__handle_duplex_callback_playback(pDevice, frameCount, pOutput, &pDevice->duplexRB.rb);\n        }\n    } else {\n        if (pDevice->type == ma_device_type_capture || pDevice->type == ma_device_type_loopback) {\n            if (pInput == NULL) {\n                return MA_INVALID_ARGS;\n            }\n\n            ma_device__send_frames_to_client(pDevice, frameCount, pInput);\n        }\n\n        if (pDevice->type == ma_device_type_playback) {\n            if (pOutput == NULL) {\n                return MA_INVALID_ARGS;\n            }\n\n            ma_device__read_frames_from_client(pDevice, frameCount, pOutput);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_calculate_buffer_size_in_frames_from_descriptor(const ma_device_descriptor* pDescriptor, ma_uint32 nativeSampleRate, ma_performance_profile performanceProfile)\n{\n    if (pDescriptor == NULL) {\n        return 0;\n    }\n\n    /*\n    We must have a non-0 native sample rate, but some backends don't allow retrieval of this at the\n    time when the size of the buffer needs to be determined. In this case we need to just take a best\n    guess and move on. We'll try using the sample rate in pDescriptor first. If that's not set we'll\n    just fall back to MA_DEFAULT_SAMPLE_RATE.\n    */\n    if (nativeSampleRate == 0) {\n        nativeSampleRate = pDescriptor->sampleRate;\n    }\n    if (nativeSampleRate == 0) {\n        nativeSampleRate = MA_DEFAULT_SAMPLE_RATE;\n    }\n\n    MA_ASSERT(nativeSampleRate != 0);\n\n    if (pDescriptor->periodSizeInFrames == 0) {\n        if (pDescriptor->periodSizeInMilliseconds == 0) {\n            if (performanceProfile == ma_performance_profile_low_latency) {\n                return ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_LOW_LATENCY, nativeSampleRate);\n            } else {\n                return ma_calculate_buffer_size_in_frames_from_milliseconds(MA_DEFAULT_PERIOD_SIZE_IN_MILLISECONDS_CONSERVATIVE, nativeSampleRate);\n            }\n        } else {\n            return ma_calculate_buffer_size_in_frames_from_milliseconds(pDescriptor->periodSizeInMilliseconds, nativeSampleRate);\n        }\n    } else {\n        return pDescriptor->periodSizeInFrames;\n    }\n}\n#endif  /* MA_NO_DEVICE_IO */\n\n\nMA_API ma_uint32 ma_calculate_buffer_size_in_milliseconds_from_frames(ma_uint32 bufferSizeInFrames, ma_uint32 sampleRate)\n{\n    /* Prevent a division by zero. */\n    if (sampleRate == 0) {\n        return 0;\n    }\n\n    return (bufferSizeInFrames*1000 + (sampleRate - 1)) / sampleRate;\n}\n\nMA_API ma_uint32 ma_calculate_buffer_size_in_frames_from_milliseconds(ma_uint32 bufferSizeInMilliseconds, ma_uint32 sampleRate)\n{\n    /* Prevent a division by zero. */\n    if (sampleRate == 0) {\n        return 0;\n    }\n\n    return bufferSizeInMilliseconds*sampleRate / 1000;\n}\n\nMA_API void ma_copy_pcm_frames(void* dst, const void* src, ma_uint64 frameCount, ma_format format, ma_uint32 channels)\n{\n    if (dst == src) {\n        return; /* No-op. */\n    }\n\n    ma_copy_memory_64(dst, src, frameCount * ma_get_bytes_per_frame(format, channels));\n}\n\nMA_API void ma_silence_pcm_frames(void* p, ma_uint64 frameCount, ma_format format, ma_uint32 channels)\n{\n    if (format == ma_format_u8) {\n        ma_uint64 sampleCount = frameCount * channels;\n        ma_uint64 iSample;\n        for (iSample = 0; iSample < sampleCount; iSample += 1) {\n            ((ma_uint8*)p)[iSample] = 128;\n        }\n    } else {\n        ma_zero_memory_64(p, frameCount * ma_get_bytes_per_frame(format, channels));\n    }\n}\n\nMA_API void* ma_offset_pcm_frames_ptr(void* p, ma_uint64 offsetInFrames, ma_format format, ma_uint32 channels)\n{\n    return ma_offset_ptr(p, offsetInFrames * ma_get_bytes_per_frame(format, channels));\n}\n\nMA_API const void* ma_offset_pcm_frames_const_ptr(const void* p, ma_uint64 offsetInFrames, ma_format format, ma_uint32 channels)\n{\n    return ma_offset_ptr(p, offsetInFrames * ma_get_bytes_per_frame(format, channels));\n}\n\n\nMA_API void ma_clip_samples_u8(ma_uint8* pDst, const ma_int16* pSrc, ma_uint64 count)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_u8(pSrc[iSample]);\n    }\n}\n\nMA_API void ma_clip_samples_s16(ma_int16* pDst, const ma_int32* pSrc, ma_uint64 count)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_s16(pSrc[iSample]);\n    }\n}\n\nMA_API void ma_clip_samples_s24(ma_uint8* pDst, const ma_int64* pSrc, ma_uint64 count)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        ma_int64 s = ma_clip_s24(pSrc[iSample]);\n        pDst[iSample*3 + 0] = (ma_uint8)((s & 0x000000FF) >>  0);\n        pDst[iSample*3 + 1] = (ma_uint8)((s & 0x0000FF00) >>  8);\n        pDst[iSample*3 + 2] = (ma_uint8)((s & 0x00FF0000) >> 16);\n    }\n}\n\nMA_API void ma_clip_samples_s32(ma_int32* pDst, const ma_int64* pSrc, ma_uint64 count)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_s32(pSrc[iSample]);\n    }\n}\n\nMA_API void ma_clip_samples_f32(float* pDst, const float* pSrc, ma_uint64 count)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_f32(pSrc[iSample]);\n    }\n}\n\nMA_API void ma_clip_pcm_frames(void* pDst, const void* pSrc, ma_uint64 frameCount, ma_format format, ma_uint32 channels)\n{\n    ma_uint64 sampleCount;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    sampleCount = frameCount * channels;\n\n    switch (format) {\n        case ma_format_u8:  ma_clip_samples_u8( (ma_uint8*)pDst, (const ma_int16*)pSrc, sampleCount); break;\n        case ma_format_s16: ma_clip_samples_s16((ma_int16*)pDst, (const ma_int32*)pSrc, sampleCount); break;\n        case ma_format_s24: ma_clip_samples_s24((ma_uint8*)pDst, (const ma_int64*)pSrc, sampleCount); break;\n        case ma_format_s32: ma_clip_samples_s32((ma_int32*)pDst, (const ma_int64*)pSrc, sampleCount); break;\n        case ma_format_f32: ma_clip_samples_f32((   float*)pDst, (const    float*)pSrc, sampleCount); break;\n\n        /* Do nothing if we don't know the format. We're including these here to silence a compiler warning about enums not being handled by the switch. */\n        case ma_format_unknown:\n        case ma_format_count:\n            break;\n    }\n}\n\n\nMA_API void ma_copy_and_apply_volume_factor_u8(ma_uint8* pSamplesOut, const ma_uint8* pSamplesIn, ma_uint64 sampleCount, float factor)\n{\n    ma_uint64 iSample;\n\n    if (pSamplesOut == NULL || pSamplesIn == NULL) {\n        return;\n    }\n\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamplesOut[iSample] = (ma_uint8)(pSamplesIn[iSample] * factor);\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_factor_s16(ma_int16* pSamplesOut, const ma_int16* pSamplesIn, ma_uint64 sampleCount, float factor)\n{\n    ma_uint64 iSample;\n\n    if (pSamplesOut == NULL || pSamplesIn == NULL) {\n        return;\n    }\n\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamplesOut[iSample] = (ma_int16)(pSamplesIn[iSample] * factor);\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_factor_s24(void* pSamplesOut, const void* pSamplesIn, ma_uint64 sampleCount, float factor)\n{\n    ma_uint64 iSample;\n    ma_uint8* pSamplesOut8;\n    ma_uint8* pSamplesIn8;\n\n    if (pSamplesOut == NULL || pSamplesIn == NULL) {\n        return;\n    }\n\n    pSamplesOut8 = (ma_uint8*)pSamplesOut;\n    pSamplesIn8  = (ma_uint8*)pSamplesIn;\n\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        ma_int32 sampleS32;\n\n        sampleS32 = (ma_int32)(((ma_uint32)(pSamplesIn8[iSample*3+0]) << 8) | ((ma_uint32)(pSamplesIn8[iSample*3+1]) << 16) | ((ma_uint32)(pSamplesIn8[iSample*3+2])) << 24);\n        sampleS32 = (ma_int32)(sampleS32 * factor);\n\n        pSamplesOut8[iSample*3+0] = (ma_uint8)(((ma_uint32)sampleS32 & 0x0000FF00) >>  8);\n        pSamplesOut8[iSample*3+1] = (ma_uint8)(((ma_uint32)sampleS32 & 0x00FF0000) >> 16);\n        pSamplesOut8[iSample*3+2] = (ma_uint8)(((ma_uint32)sampleS32 & 0xFF000000) >> 24);\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_factor_s32(ma_int32* pSamplesOut, const ma_int32* pSamplesIn, ma_uint64 sampleCount, float factor)\n{\n    ma_uint64 iSample;\n\n    if (pSamplesOut == NULL || pSamplesIn == NULL) {\n        return;\n    }\n\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamplesOut[iSample] = (ma_int32)(pSamplesIn[iSample] * factor);\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_factor_f32(float* pSamplesOut, const float* pSamplesIn, ma_uint64 sampleCount, float factor)\n{\n    ma_uint64 iSample;\n\n    if (pSamplesOut == NULL || pSamplesIn == NULL) {\n        return;\n    }\n\n    if (factor == 1) {\n        if (pSamplesOut == pSamplesIn) {\n            /* In place. No-op. */\n        } else {\n            /* Just a copy. */\n            for (iSample = 0; iSample < sampleCount; iSample += 1) {\n                pSamplesOut[iSample] = pSamplesIn[iSample];\n            }\n        }\n    } else {\n        for (iSample = 0; iSample < sampleCount; iSample += 1) {\n            pSamplesOut[iSample] = pSamplesIn[iSample] * factor;\n        }\n    }\n}\n\nMA_API void ma_apply_volume_factor_u8(ma_uint8* pSamples, ma_uint64 sampleCount, float factor)\n{\n    ma_copy_and_apply_volume_factor_u8(pSamples, pSamples, sampleCount, factor);\n}\n\nMA_API void ma_apply_volume_factor_s16(ma_int16* pSamples, ma_uint64 sampleCount, float factor)\n{\n    ma_copy_and_apply_volume_factor_s16(pSamples, pSamples, sampleCount, factor);\n}\n\nMA_API void ma_apply_volume_factor_s24(void* pSamples, ma_uint64 sampleCount, float factor)\n{\n    ma_copy_and_apply_volume_factor_s24(pSamples, pSamples, sampleCount, factor);\n}\n\nMA_API void ma_apply_volume_factor_s32(ma_int32* pSamples, ma_uint64 sampleCount, float factor)\n{\n    ma_copy_and_apply_volume_factor_s32(pSamples, pSamples, sampleCount, factor);\n}\n\nMA_API void ma_apply_volume_factor_f32(float* pSamples, ma_uint64 sampleCount, float factor)\n{\n    ma_copy_and_apply_volume_factor_f32(pSamples, pSamples, sampleCount, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_u8(ma_uint8* pFramesOut, const ma_uint8* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_u8(pFramesOut, pFramesIn, frameCount*channels, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s16(ma_int16* pFramesOut, const ma_int16* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_s16(pFramesOut, pFramesIn, frameCount*channels, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s24(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_s24(pFramesOut, pFramesIn, frameCount*channels, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_s32(ma_int32* pFramesOut, const ma_int32* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_s32(pFramesOut, pFramesIn, frameCount*channels, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_f32(pFramesOut, pFramesIn, frameCount*channels, factor);\n}\n\nMA_API void ma_copy_and_apply_volume_factor_pcm_frames(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float factor)\n{\n    switch (format)\n    {\n    case ma_format_u8:  ma_copy_and_apply_volume_factor_pcm_frames_u8 ((ma_uint8*)pFramesOut, (const ma_uint8*)pFramesIn, frameCount, channels, factor); return;\n    case ma_format_s16: ma_copy_and_apply_volume_factor_pcm_frames_s16((ma_int16*)pFramesOut, (const ma_int16*)pFramesIn, frameCount, channels, factor); return;\n    case ma_format_s24: ma_copy_and_apply_volume_factor_pcm_frames_s24(           pFramesOut,                  pFramesIn, frameCount, channels, factor); return;\n    case ma_format_s32: ma_copy_and_apply_volume_factor_pcm_frames_s32((ma_int32*)pFramesOut, (const ma_int32*)pFramesIn, frameCount, channels, factor); return;\n    case ma_format_f32: ma_copy_and_apply_volume_factor_pcm_frames_f32(   (float*)pFramesOut,    (const float*)pFramesIn, frameCount, channels, factor); return;\n    default: return;    /* Do nothing. */\n    }\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames_u8(ma_uint8* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames_u8(pFrames, pFrames, frameCount, channels, factor);\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames_s16(ma_int16* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames_s16(pFrames, pFrames, frameCount, channels, factor);\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames_s24(void* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames_s24(pFrames, pFrames, frameCount, channels, factor);\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames_s32(ma_int32* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames_s32(pFrames, pFrames, frameCount, channels, factor);\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames_f32(float* pFrames, ma_uint64 frameCount, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames_f32(pFrames, pFrames, frameCount, channels, factor);\n}\n\nMA_API void ma_apply_volume_factor_pcm_frames(void* pFramesOut, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float factor)\n{\n    ma_copy_and_apply_volume_factor_pcm_frames(pFramesOut, pFramesOut, frameCount, format, channels, factor);\n}\n\n\nMA_API void ma_copy_and_apply_volume_factor_per_channel_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, ma_uint32 channels, float* pChannelGains)\n{\n    ma_uint64 iFrame;\n\n    if (channels == 2) {\n        /* TODO: Do an optimized implementation for stereo and mono. Can do a SIMD optimized implementation as well. */\n    }\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            pFramesOut[iFrame * channels + iChannel] = pFramesIn[iFrame * channels + iChannel] * pChannelGains[iChannel];\n        }\n    }\n}\n\n\n\nstatic MA_INLINE ma_int16 ma_apply_volume_unclipped_u8(ma_int16 x, ma_int16 volume)\n{\n    return (ma_int16)(((ma_int32)x * (ma_int32)volume) >> 8);\n}\n\nstatic MA_INLINE ma_int32 ma_apply_volume_unclipped_s16(ma_int32 x, ma_int16 volume)\n{\n    return (ma_int32)((x * volume) >> 8);\n}\n\nstatic MA_INLINE ma_int64 ma_apply_volume_unclipped_s24(ma_int64 x, ma_int16 volume)\n{\n    return (ma_int64)((x * volume) >> 8);\n}\n\nstatic MA_INLINE ma_int64 ma_apply_volume_unclipped_s32(ma_int64 x, ma_int16 volume)\n{\n    return (ma_int64)((x * volume) >> 8);\n}\n\nstatic MA_INLINE float ma_apply_volume_unclipped_f32(float x, float volume)\n{\n    return x * volume;\n}\n\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_u8(ma_uint8* pDst, const ma_int16* pSrc, ma_uint64 count, float volume)\n{\n    ma_uint64 iSample;\n    ma_int16  volumeFixed;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    volumeFixed = ma_float_to_fixed_16(volume);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_u8(ma_apply_volume_unclipped_u8(pSrc[iSample], volumeFixed));\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s16(ma_int16* pDst, const ma_int32* pSrc, ma_uint64 count, float volume)\n{\n    ma_uint64 iSample;\n    ma_int16  volumeFixed;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    volumeFixed = ma_float_to_fixed_16(volume);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_s16(ma_apply_volume_unclipped_s16(pSrc[iSample], volumeFixed));\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s24(ma_uint8* pDst, const ma_int64* pSrc, ma_uint64 count, float volume)\n{\n    ma_uint64 iSample;\n    ma_int16  volumeFixed;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    volumeFixed = ma_float_to_fixed_16(volume);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        ma_int64 s = ma_clip_s24(ma_apply_volume_unclipped_s24(pSrc[iSample], volumeFixed));\n        pDst[iSample*3 + 0] = (ma_uint8)((s & 0x000000FF) >>  0);\n        pDst[iSample*3 + 1] = (ma_uint8)((s & 0x0000FF00) >>  8);\n        pDst[iSample*3 + 2] = (ma_uint8)((s & 0x00FF0000) >> 16);\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_s32(ma_int32* pDst, const ma_int64* pSrc, ma_uint64 count, float volume)\n{\n    ma_uint64 iSample;\n    ma_int16  volumeFixed;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    volumeFixed = ma_float_to_fixed_16(volume);\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_s32(ma_apply_volume_unclipped_s32(pSrc[iSample], volumeFixed));\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_and_clip_samples_f32(float* pDst, const float* pSrc, ma_uint64 count, float volume)\n{\n    ma_uint64 iSample;\n\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    /* For the f32 case we need to make sure this supports in-place processing where the input and output buffers are the same. */\n\n    for (iSample = 0; iSample < count; iSample += 1) {\n        pDst[iSample] = ma_clip_f32(ma_apply_volume_unclipped_f32(pSrc[iSample], volume));\n    }\n}\n\nMA_API void ma_copy_and_apply_volume_and_clip_pcm_frames(void* pDst, const void* pSrc, ma_uint64 frameCount, ma_format format, ma_uint32 channels, float volume)\n{\n    MA_ASSERT(pDst != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    if (volume == 1) {\n        ma_clip_pcm_frames(pDst, pSrc, frameCount, format, channels);   /* Optimized case for volume = 1. */\n    } else if (volume == 0) {\n        ma_silence_pcm_frames(pDst, frameCount, format, channels);      /* Optimized case for volume = 0. */\n    } else {\n        ma_uint64 sampleCount = frameCount * channels;\n\n        switch (format) {\n            case ma_format_u8:  ma_copy_and_apply_volume_and_clip_samples_u8( (ma_uint8*)pDst, (const ma_int16*)pSrc, sampleCount, volume); break;\n            case ma_format_s16: ma_copy_and_apply_volume_and_clip_samples_s16((ma_int16*)pDst, (const ma_int32*)pSrc, sampleCount, volume); break;\n            case ma_format_s24: ma_copy_and_apply_volume_and_clip_samples_s24((ma_uint8*)pDst, (const ma_int64*)pSrc, sampleCount, volume); break;\n            case ma_format_s32: ma_copy_and_apply_volume_and_clip_samples_s32((ma_int32*)pDst, (const ma_int64*)pSrc, sampleCount, volume); break;\n            case ma_format_f32: ma_copy_and_apply_volume_and_clip_samples_f32((   float*)pDst, (const    float*)pSrc, sampleCount, volume); break;\n\n            /* Do nothing if we don't know the format. We're including these here to silence a compiler warning about enums not being handled by the switch. */\n            case ma_format_unknown:\n            case ma_format_count:\n                break;\n        }\n    }\n}\n\n\n\nMA_API float ma_volume_linear_to_db(float factor)\n{\n    return 20*ma_log10f(factor);\n}\n\nMA_API float ma_volume_db_to_linear(float gain)\n{\n    return ma_powf(10, gain/20.0f);\n}\n\n\nMA_API ma_result ma_mix_pcm_frames_f32(float* pDst, const float* pSrc, ma_uint64 frameCount, ma_uint32 channels, float volume)\n{\n    ma_uint64 iSample;\n    ma_uint64 sampleCount;\n\n    if (pDst == NULL || pSrc == NULL || channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (volume == 0) {\n        return MA_SUCCESS;  /* No changes if the volume is 0. */\n    }\n\n    sampleCount = frameCount * channels;\n\n    if (volume == 1) {\n        for (iSample = 0; iSample < sampleCount; iSample += 1) {\n            pDst[iSample] += pSrc[iSample];\n        }\n    } else {\n        for (iSample = 0; iSample < sampleCount; iSample += 1) {\n            pDst[iSample] += ma_apply_volume_unclipped_f32(pSrc[iSample], volume);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nFormat Conversion\n\n**************************************************************************************************************************************************************/\n\nstatic MA_INLINE ma_int16 ma_pcm_sample_f32_to_s16(float x)\n{\n    return (ma_int16)(x * 32767.0f);\n}\n\nstatic MA_INLINE ma_int16 ma_pcm_sample_u8_to_s16_no_scale(ma_uint8 x)\n{\n    return (ma_int16)((ma_int16)x - 128);\n}\n\nstatic MA_INLINE ma_int64 ma_pcm_sample_s24_to_s32_no_scale(const ma_uint8* x)\n{\n    return (ma_int64)(((ma_uint64)x[0] << 40) | ((ma_uint64)x[1] << 48) | ((ma_uint64)x[2] << 56)) >> 40;  /* Make sure the sign bits are maintained. */\n}\n\nstatic MA_INLINE void ma_pcm_sample_s32_to_s24_no_scale(ma_int64 x, ma_uint8* s24)\n{\n    s24[0] = (ma_uint8)((x & 0x000000FF) >>  0);\n    s24[1] = (ma_uint8)((x & 0x0000FF00) >>  8);\n    s24[2] = (ma_uint8)((x & 0x00FF0000) >> 16);\n}\n\n\n/* u8 */\nMA_API void ma_pcm_u8_to_u8(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    (void)ditherMode;\n    ma_copy_memory_64(dst, src, count * sizeof(ma_uint8));\n}\n\n\nstatic MA_INLINE void ma_pcm_u8_to_s16__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const ma_uint8* src_u8 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        ma_int16 x = src_u8[i];\n        x = (ma_int16)(x - 128);\n        x = (ma_int16)(x << 8);\n        dst_s16[i] = x;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_u8_to_s16__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s16__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_u8_to_s16__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_u8_to_s16__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_u8_to_s16(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_u8_to_s16__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_u8_to_s16__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_u8_to_s16__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_u8_to_s16__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_u8_to_s24__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_s24 = (ma_uint8*)dst;\n    const ma_uint8* src_u8 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        ma_int16 x = src_u8[i];\n        x = (ma_int16)(x - 128);\n\n        dst_s24[i*3+0] = 0;\n        dst_s24[i*3+1] = 0;\n        dst_s24[i*3+2] = (ma_uint8)((ma_int8)x);\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_u8_to_s24__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s24__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_u8_to_s24__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_u8_to_s24__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_u8_to_s24(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_u8_to_s24__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_u8_to_s24__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_u8_to_s24__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_u8_to_s24__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_u8_to_s32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int32* dst_s32 = (ma_int32*)dst;\n    const ma_uint8* src_u8 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        ma_int32 x = src_u8[i];\n        x = x - 128;\n        x = x << 24;\n        dst_s32[i] = x;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_u8_to_s32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_u8_to_s32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_u8_to_s32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_u8_to_s32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_u8_to_s32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_u8_to_s32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_u8_to_s32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_u8_to_s32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_u8_to_f32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    float* dst_f32 = (float*)dst;\n    const ma_uint8* src_u8 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        float x = (float)src_u8[i];\n        x = x * 0.00784313725490196078f;    /* 0..255 to 0..2 */\n        x = x - 1;                          /* 0..2 to -1..1 */\n\n        dst_f32[i] = x;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_u8_to_f32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_f32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_u8_to_f32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_u8_to_f32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_u8_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_u8_to_f32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_u8_to_f32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_u8_to_f32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_u8_to_f32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_u8_to_f32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\nstatic MA_INLINE void ma_pcm_interleave_u8__reference(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const ma_uint8** src_u8 = (const ma_uint8**)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_u8[iFrame*channels + iChannel] = src_u8[iChannel][iFrame];\n        }\n    }\n}\n#else\nstatic MA_INLINE void ma_pcm_interleave_u8__optimized(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const ma_uint8** src_u8 = (const ma_uint8**)src;\n\n    if (channels == 1) {\n        ma_copy_memory_64(dst, src[0], frameCount * sizeof(ma_uint8));\n    } else if (channels == 2) {\n        ma_uint64 iFrame;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            dst_u8[iFrame*2 + 0] = src_u8[0][iFrame];\n            dst_u8[iFrame*2 + 1] = src_u8[1][iFrame];\n        }\n    } else {\n        ma_uint64 iFrame;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            ma_uint32 iChannel;\n            for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                dst_u8[iFrame*channels + iChannel] = src_u8[iChannel][iFrame];\n            }\n        }\n    }\n}\n#endif\n\nMA_API void ma_pcm_interleave_u8(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_interleave_u8__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_interleave_u8__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_deinterleave_u8__reference(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_uint8** dst_u8 = (ma_uint8**)dst;\n    const ma_uint8* src_u8 = (const ma_uint8*)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_u8[iChannel][iFrame] = src_u8[iFrame*channels + iChannel];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_deinterleave_u8__optimized(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_deinterleave_u8__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_deinterleave_u8(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_deinterleave_u8__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_deinterleave_u8__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\n/* s16 */\nstatic MA_INLINE void ma_pcm_s16_to_u8__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const ma_int16* src_s16 = (const ma_int16*)src;\n\n    if (ditherMode == ma_dither_mode_none) {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int16 x = src_s16[i];\n            x = (ma_int16)(x >> 8);\n            x = (ma_int16)(x + 128);\n            dst_u8[i] = (ma_uint8)x;\n        }\n    } else {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int16 x = src_s16[i];\n\n            /* Dither. Don't overflow. */\n            ma_int32 dither = ma_dither_s32(ditherMode, -0x80, 0x7F);\n            if ((x + dither) <= 0x7FFF) {\n                x = (ma_int16)(x + dither);\n            } else {\n                x = 0x7FFF;\n            }\n\n            x = (ma_int16)(x >> 8);\n            x = (ma_int16)(x + 128);\n            dst_u8[i] = (ma_uint8)x;\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_s16_to_u8__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_u8__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s16_to_u8__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s16_to_u8__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s16_to_u8(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s16_to_u8__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s16_to_u8__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s16_to_u8__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s16_to_u8__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nMA_API void ma_pcm_s16_to_s16(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    (void)ditherMode;\n    ma_copy_memory_64(dst, src, count * sizeof(ma_int16));\n}\n\n\nstatic MA_INLINE void ma_pcm_s16_to_s24__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_s24 = (ma_uint8*)dst;\n    const ma_int16* src_s16 = (const ma_int16*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        dst_s24[i*3+0] = 0;\n        dst_s24[i*3+1] = (ma_uint8)(src_s16[i] & 0xFF);\n        dst_s24[i*3+2] = (ma_uint8)(src_s16[i] >> 8);\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_s16_to_s24__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s24__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s16_to_s24__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s16_to_s24__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s16_to_s24(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s16_to_s24__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s16_to_s24__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s16_to_s24__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s16_to_s24__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s16_to_s32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int32* dst_s32 = (ma_int32*)dst;\n    const ma_int16* src_s16 = (const ma_int16*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        dst_s32[i] = (ma_int32)src_s16[i] << 16;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_s16_to_s32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s16_to_s32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s16_to_s32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s16_to_s32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s16_to_s32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s16_to_s32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s16_to_s32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s16_to_s32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s16_to_f32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    float* dst_f32 = (float*)dst;\n    const ma_int16* src_s16 = (const ma_int16*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        float x = (float)src_s16[i];\n\n#if 0\n        /* The accurate way. */\n        x = x + 32768.0f;                   /* -32768..32767 to 0..65535 */\n        x = x * 0.00003051804379339284f;    /* 0..65535 to 0..2 */\n        x = x - 1;                          /* 0..2 to -1..1 */\n#else\n        /* The fast way. */\n        x = x * 0.000030517578125f;         /* -32768..32767 to -1..0.999969482421875 */\n#endif\n\n        dst_f32[i] = x;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_s16_to_f32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_f32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s16_to_f32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s16_to_f32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s16_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s16_to_f32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s16_to_f32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s16_to_f32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s16_to_f32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s16_to_f32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_interleave_s16__reference(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const ma_int16** src_s16 = (const ma_int16**)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_s16[iFrame*channels + iChannel] = src_s16[iChannel][iFrame];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_interleave_s16__optimized(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_interleave_s16__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_interleave_s16(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_interleave_s16__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_interleave_s16__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_deinterleave_s16__reference(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_int16** dst_s16 = (ma_int16**)dst;\n    const ma_int16* src_s16 = (const ma_int16*)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_s16[iChannel][iFrame] = src_s16[iFrame*channels + iChannel];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_deinterleave_s16__optimized(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_deinterleave_s16__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_deinterleave_s16(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_deinterleave_s16__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_deinterleave_s16__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\n/* s24 */\nstatic MA_INLINE void ma_pcm_s24_to_u8__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const ma_uint8* src_s24 = (const ma_uint8*)src;\n\n    if (ditherMode == ma_dither_mode_none) {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            dst_u8[i] = (ma_uint8)((ma_int8)src_s24[i*3 + 2] + 128);\n        }\n    } else {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = (ma_int32)(((ma_uint32)(src_s24[i*3+0]) << 8) | ((ma_uint32)(src_s24[i*3+1]) << 16) | ((ma_uint32)(src_s24[i*3+2])) << 24);\n\n            /* Dither. Don't overflow. */\n            ma_int32 dither = ma_dither_s32(ditherMode, -0x800000, 0x7FFFFF);\n            if ((ma_int64)x + dither <= 0x7FFFFFFF) {\n                x = x + dither;\n            } else {\n                x = 0x7FFFFFFF;\n            }\n\n            x = x >> 24;\n            x = x + 128;\n            dst_u8[i] = (ma_uint8)x;\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_s24_to_u8__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_u8__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s24_to_u8__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s24_to_u8__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s24_to_u8(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s24_to_u8__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s24_to_u8__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s24_to_u8__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s24_to_u8__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s24_to_s16__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const ma_uint8* src_s24 = (const ma_uint8*)src;\n\n    if (ditherMode == ma_dither_mode_none) {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_uint16 dst_lo =            ((ma_uint16)src_s24[i*3 + 1]);\n            ma_uint16 dst_hi = (ma_uint16)((ma_uint16)src_s24[i*3 + 2] << 8);\n            dst_s16[i] = (ma_int16)(dst_lo | dst_hi);\n        }\n    } else {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = (ma_int32)(((ma_uint32)(src_s24[i*3+0]) << 8) | ((ma_uint32)(src_s24[i*3+1]) << 16) | ((ma_uint32)(src_s24[i*3+2])) << 24);\n\n            /* Dither. Don't overflow. */\n            ma_int32 dither = ma_dither_s32(ditherMode, -0x8000, 0x7FFF);\n            if ((ma_int64)x + dither <= 0x7FFFFFFF) {\n                x = x + dither;\n            } else {\n                x = 0x7FFFFFFF;\n            }\n\n            x = x >> 16;\n            dst_s16[i] = (ma_int16)x;\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_s24_to_s16__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s16__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s24_to_s16__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s24_to_s16__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s24_to_s16(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s24_to_s16__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s24_to_s16__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s24_to_s16__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s24_to_s16__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nMA_API void ma_pcm_s24_to_s24(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    (void)ditherMode;\n\n    ma_copy_memory_64(dst, src, count * 3);\n}\n\n\nstatic MA_INLINE void ma_pcm_s24_to_s32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int32* dst_s32 = (ma_int32*)dst;\n    const ma_uint8* src_s24 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        dst_s32[i] = (ma_int32)(((ma_uint32)(src_s24[i*3+0]) << 8) | ((ma_uint32)(src_s24[i*3+1]) << 16) | ((ma_uint32)(src_s24[i*3+2])) << 24);\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_s24_to_s32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s24_to_s32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s24_to_s32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s24_to_s32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s24_to_s32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s24_to_s32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s24_to_s32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s24_to_s32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s24_to_f32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    float* dst_f32 = (float*)dst;\n    const ma_uint8* src_s24 = (const ma_uint8*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        float x = (float)(((ma_int32)(((ma_uint32)(src_s24[i*3+0]) << 8) | ((ma_uint32)(src_s24[i*3+1]) << 16) | ((ma_uint32)(src_s24[i*3+2])) << 24)) >> 8);\n\n#if 0\n        /* The accurate way. */\n        x = x + 8388608.0f;                 /* -8388608..8388607 to 0..16777215 */\n        x = x * 0.00000011920929665621f;    /* 0..16777215 to 0..2 */\n        x = x - 1;                          /* 0..2 to -1..1 */\n#else\n        /* The fast way. */\n        x = x * 0.00000011920928955078125f; /* -8388608..8388607 to -1..0.999969482421875 */\n#endif\n\n        dst_f32[i] = x;\n    }\n\n    (void)ditherMode;\n}\n\nstatic MA_INLINE void ma_pcm_s24_to_f32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_f32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s24_to_f32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s24_to_f32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s24_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s24_to_f32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s24_to_f32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s24_to_f32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s24_to_f32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s24_to_f32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_interleave_s24__reference(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_uint8* dst8 = (ma_uint8*)dst;\n    const ma_uint8** src8 = (const ma_uint8**)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst8[iFrame*3*channels + iChannel*3 + 0] = src8[iChannel][iFrame*3 + 0];\n            dst8[iFrame*3*channels + iChannel*3 + 1] = src8[iChannel][iFrame*3 + 1];\n            dst8[iFrame*3*channels + iChannel*3 + 2] = src8[iChannel][iFrame*3 + 2];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_interleave_s24__optimized(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_interleave_s24__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_interleave_s24(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_interleave_s24__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_interleave_s24__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_deinterleave_s24__reference(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_uint8** dst8 = (ma_uint8**)dst;\n    const ma_uint8* src8 = (const ma_uint8*)src;\n\n    ma_uint32 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst8[iChannel][iFrame*3 + 0] = src8[iFrame*3*channels + iChannel*3 + 0];\n            dst8[iChannel][iFrame*3 + 1] = src8[iFrame*3*channels + iChannel*3 + 1];\n            dst8[iChannel][iFrame*3 + 2] = src8[iFrame*3*channels + iChannel*3 + 2];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_deinterleave_s24__optimized(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_deinterleave_s24__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_deinterleave_s24(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_deinterleave_s24__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_deinterleave_s24__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\n\n/* s32 */\nstatic MA_INLINE void ma_pcm_s32_to_u8__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const ma_int32* src_s32 = (const ma_int32*)src;\n\n    if (ditherMode == ma_dither_mode_none) {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = src_s32[i];\n            x = x >> 24;\n            x = x + 128;\n            dst_u8[i] = (ma_uint8)x;\n        }\n    } else {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = src_s32[i];\n\n            /* Dither. Don't overflow. */\n            ma_int32 dither = ma_dither_s32(ditherMode, -0x800000, 0x7FFFFF);\n            if ((ma_int64)x + dither <= 0x7FFFFFFF) {\n                x = x + dither;\n            } else {\n                x = 0x7FFFFFFF;\n            }\n\n            x = x >> 24;\n            x = x + 128;\n            dst_u8[i] = (ma_uint8)x;\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_s32_to_u8__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_u8__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s32_to_u8__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s32_to_u8__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s32_to_u8(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s32_to_u8__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s32_to_u8__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s32_to_u8__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s32_to_u8__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s32_to_s16__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const ma_int32* src_s32 = (const ma_int32*)src;\n\n    if (ditherMode == ma_dither_mode_none) {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = src_s32[i];\n            x = x >> 16;\n            dst_s16[i] = (ma_int16)x;\n        }\n    } else {\n        ma_uint64 i;\n        for (i = 0; i < count; i += 1) {\n            ma_int32 x = src_s32[i];\n\n            /* Dither. Don't overflow. */\n            ma_int32 dither = ma_dither_s32(ditherMode, -0x8000, 0x7FFF);\n            if ((ma_int64)x + dither <= 0x7FFFFFFF) {\n                x = x + dither;\n            } else {\n                x = 0x7FFFFFFF;\n            }\n\n            x = x >> 16;\n            dst_s16[i] = (ma_int16)x;\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_s32_to_s16__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s16__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s32_to_s16__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s32_to_s16__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s16__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s32_to_s16(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s32_to_s16__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s32_to_s16__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s32_to_s16__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s32_to_s16__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_s32_to_s24__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_s24 = (ma_uint8*)dst;\n    const ma_int32* src_s32 = (const ma_int32*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        ma_uint32 x = (ma_uint32)src_s32[i];\n        dst_s24[i*3+0] = (ma_uint8)((x & 0x0000FF00) >>  8);\n        dst_s24[i*3+1] = (ma_uint8)((x & 0x00FF0000) >> 16);\n        dst_s24[i*3+2] = (ma_uint8)((x & 0xFF000000) >> 24);\n    }\n\n    (void)ditherMode;   /* No dithering for s32 -> s24. */\n}\n\nstatic MA_INLINE void ma_pcm_s32_to_s24__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s24__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s32_to_s24__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s32_to_s24__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s32_to_s24(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s32_to_s24__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s32_to_s24__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s32_to_s24__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s32_to_s24__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nMA_API void ma_pcm_s32_to_s32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    (void)ditherMode;\n\n    ma_copy_memory_64(dst, src, count * sizeof(ma_int32));\n}\n\n\nstatic MA_INLINE void ma_pcm_s32_to_f32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    float* dst_f32 = (float*)dst;\n    const ma_int32* src_s32 = (const ma_int32*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        double x = src_s32[i];\n\n#if 0\n        x = x + 2147483648.0;\n        x = x * 0.0000000004656612873077392578125;\n        x = x - 1;\n#else\n        x = x / 2147483648.0;\n#endif\n\n        dst_f32[i] = (float)x;\n    }\n\n    (void)ditherMode;   /* No dithering for s32 -> f32. */\n}\n\nstatic MA_INLINE void ma_pcm_s32_to_f32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_f32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_s32_to_f32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_s32_to_f32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_s32_to_f32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_s32_to_f32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_s32_to_f32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_s32_to_f32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_s32_to_f32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_s32_to_f32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_interleave_s32__reference(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_int32* dst_s32 = (ma_int32*)dst;\n    const ma_int32** src_s32 = (const ma_int32**)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_s32[iFrame*channels + iChannel] = src_s32[iChannel][iFrame];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_interleave_s32__optimized(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_interleave_s32__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_interleave_s32(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_interleave_s32__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_interleave_s32__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_deinterleave_s32__reference(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_int32** dst_s32 = (ma_int32**)dst;\n    const ma_int32* src_s32 = (const ma_int32*)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_s32[iChannel][iFrame] = src_s32[iFrame*channels + iChannel];\n        }\n    }\n}\n\nstatic MA_INLINE void ma_pcm_deinterleave_s32__optimized(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_deinterleave_s32__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_deinterleave_s32(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_deinterleave_s32__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_deinterleave_s32__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\n/* f32 */\nstatic MA_INLINE void ma_pcm_f32_to_u8__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint64 i;\n\n    ma_uint8* dst_u8 = (ma_uint8*)dst;\n    const float* src_f32 = (const float*)src;\n\n    float ditherMin = 0;\n    float ditherMax = 0;\n    if (ditherMode != ma_dither_mode_none) {\n        ditherMin = 1.0f / -128;\n        ditherMax = 1.0f /  127;\n    }\n\n    for (i = 0; i < count; i += 1) {\n        float x = src_f32[i];\n        x = x + ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n        x = x + 1;                                  /* -1..1 to 0..2 */\n        x = x * 127.5f;                             /* 0..2 to 0..255 */\n\n        dst_u8[i] = (ma_uint8)x;\n    }\n}\n\nstatic MA_INLINE void ma_pcm_f32_to_u8__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_u8__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_f32_to_u8__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_f32_to_u8__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_u8__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_f32_to_u8(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_f32_to_u8__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_f32_to_u8__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_f32_to_u8__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_f32_to_u8__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\nstatic MA_INLINE void ma_pcm_f32_to_s16__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint64 i;\n\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const float* src_f32 = (const float*)src;\n\n    float ditherMin = 0;\n    float ditherMax = 0;\n    if (ditherMode != ma_dither_mode_none) {\n        ditherMin = 1.0f / -32768;\n        ditherMax = 1.0f /  32767;\n    }\n\n    for (i = 0; i < count; i += 1) {\n        float x = src_f32[i];\n        x = x + ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n\n#if 0\n        /* The accurate way. */\n        x = x + 1;                                  /* -1..1 to 0..2 */\n        x = x * 32767.5f;                           /* 0..2 to 0..65535 */\n        x = x - 32768.0f;                           /* 0...65535 to -32768..32767 */\n#else\n        /* The fast way. */\n        x = x * 32767.0f;                           /* -1..1 to -32767..32767 */\n#endif\n\n        dst_s16[i] = (ma_int16)x;\n    }\n}\n#else\nstatic MA_INLINE void ma_pcm_f32_to_s16__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint64 i;\n    ma_uint64 i4;\n    ma_uint64 count4;\n\n    ma_int16* dst_s16 = (ma_int16*)dst;\n    const float* src_f32 = (const float*)src;\n\n    float ditherMin = 0;\n    float ditherMax = 0;\n    if (ditherMode != ma_dither_mode_none) {\n        ditherMin = 1.0f / -32768;\n        ditherMax = 1.0f /  32767;\n    }\n\n    /* Unrolled. */\n    i = 0;\n    count4 = count >> 2;\n    for (i4 = 0; i4 < count4; i4 += 1) {\n        float d0 = ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        float d1 = ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        float d2 = ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        float d3 = ma_dither_f32(ditherMode, ditherMin, ditherMax);\n\n        float x0 = src_f32[i+0];\n        float x1 = src_f32[i+1];\n        float x2 = src_f32[i+2];\n        float x3 = src_f32[i+3];\n\n        x0 = x0 + d0;\n        x1 = x1 + d1;\n        x2 = x2 + d2;\n        x3 = x3 + d3;\n\n        x0 = ((x0 < -1) ? -1 : ((x0 > 1) ? 1 : x0));\n        x1 = ((x1 < -1) ? -1 : ((x1 > 1) ? 1 : x1));\n        x2 = ((x2 < -1) ? -1 : ((x2 > 1) ? 1 : x2));\n        x3 = ((x3 < -1) ? -1 : ((x3 > 1) ? 1 : x3));\n\n        x0 = x0 * 32767.0f;\n        x1 = x1 * 32767.0f;\n        x2 = x2 * 32767.0f;\n        x3 = x3 * 32767.0f;\n\n        dst_s16[i+0] = (ma_int16)x0;\n        dst_s16[i+1] = (ma_int16)x1;\n        dst_s16[i+2] = (ma_int16)x2;\n        dst_s16[i+3] = (ma_int16)x3;\n\n        i += 4;\n    }\n\n    /* Leftover. */\n    for (; i < count; i += 1) {\n        float x = src_f32[i];\n        x = x + ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n        x = x * 32767.0f;                           /* -1..1 to -32767..32767 */\n\n        dst_s16[i] = (ma_int16)x;\n    }\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_f32_to_s16__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint64 i;\n    ma_uint64 i8;\n    ma_uint64 count8;\n    ma_int16* dst_s16;\n    const float* src_f32;\n    float ditherMin;\n    float ditherMax;\n\n    /* Both the input and output buffers need to be aligned to 16 bytes. */\n    if ((((ma_uintptr)dst & 15) != 0) || (((ma_uintptr)src & 15) != 0)) {\n        ma_pcm_f32_to_s16__optimized(dst, src, count, ditherMode);\n        return;\n    }\n\n    dst_s16 = (ma_int16*)dst;\n    src_f32 = (const float*)src;\n\n    ditherMin = 0;\n    ditherMax = 0;\n    if (ditherMode != ma_dither_mode_none) {\n        ditherMin = 1.0f / -32768;\n        ditherMax = 1.0f /  32767;\n    }\n\n    i = 0;\n\n    /* SSE2. SSE allows us to output 8 s16's at a time which means our loop is unrolled 8 times. */\n    count8 = count >> 3;\n    for (i8 = 0; i8 < count8; i8 += 1) {\n        __m128 d0;\n        __m128 d1;\n        __m128 x0;\n        __m128 x1;\n\n        if (ditherMode == ma_dither_mode_none) {\n            d0 = _mm_set1_ps(0);\n            d1 = _mm_set1_ps(0);\n        } else if (ditherMode == ma_dither_mode_rectangle) {\n            d0 = _mm_set_ps(\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax)\n            );\n            d1 = _mm_set_ps(\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax),\n                ma_dither_f32_rectangle(ditherMin, ditherMax)\n            );\n        } else {\n            d0 = _mm_set_ps(\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax)\n            );\n            d1 = _mm_set_ps(\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax),\n                ma_dither_f32_triangle(ditherMin, ditherMax)\n            );\n        }\n\n        x0 = *((__m128*)(src_f32 + i) + 0);\n        x1 = *((__m128*)(src_f32 + i) + 1);\n\n        x0 = _mm_add_ps(x0, d0);\n        x1 = _mm_add_ps(x1, d1);\n\n        x0 = _mm_mul_ps(x0, _mm_set1_ps(32767.0f));\n        x1 = _mm_mul_ps(x1, _mm_set1_ps(32767.0f));\n\n        _mm_stream_si128(((__m128i*)(dst_s16 + i)), _mm_packs_epi32(_mm_cvttps_epi32(x0), _mm_cvttps_epi32(x1)));\n\n        i += 8;\n    }\n\n\n    /* Leftover. */\n    for (; i < count; i += 1) {\n        float x = src_f32[i];\n        x = x + ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n        x = x * 32767.0f;                           /* -1..1 to -32767..32767 */\n\n        dst_s16[i] = (ma_int16)x;\n    }\n}\n#endif  /* SSE2 */\n\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_f32_to_s16__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint64 i;\n    ma_uint64 i8;\n    ma_uint64 count8;\n    ma_int16* dst_s16;\n    const float* src_f32;\n    float ditherMin;\n    float ditherMax;\n\n    if (!ma_has_neon()) {\n        ma_pcm_f32_to_s16__optimized(dst, src, count, ditherMode);\n        return;\n    }\n\n    /* Both the input and output buffers need to be aligned to 16 bytes. */\n    if ((((ma_uintptr)dst & 15) != 0) || (((ma_uintptr)src & 15) != 0)) {\n        ma_pcm_f32_to_s16__optimized(dst, src, count, ditherMode);\n        return;\n    }\n\n    dst_s16 = (ma_int16*)dst;\n    src_f32 = (const float*)src;\n\n    ditherMin = 0;\n    ditherMax = 0;\n    if (ditherMode != ma_dither_mode_none) {\n        ditherMin = 1.0f / -32768;\n        ditherMax = 1.0f /  32767;\n    }\n\n    i = 0;\n\n    /* NEON. NEON allows us to output 8 s16's at a time which means our loop is unrolled 8 times. */\n    count8 = count >> 3;\n    for (i8 = 0; i8 < count8; i8 += 1) {\n        float32x4_t d0;\n        float32x4_t d1;\n        float32x4_t x0;\n        float32x4_t x1;\n        int32x4_t i0;\n        int32x4_t i1;\n\n        if (ditherMode == ma_dither_mode_none) {\n            d0 = vmovq_n_f32(0);\n            d1 = vmovq_n_f32(0);\n        } else if (ditherMode == ma_dither_mode_rectangle) {\n            float d0v[4];\n            float d1v[4];\n\n            d0v[0] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d0v[1] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d0v[2] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d0v[3] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d0 = vld1q_f32(d0v);\n\n            d1v[0] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d1v[1] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d1v[2] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d1v[3] = ma_dither_f32_rectangle(ditherMin, ditherMax);\n            d1 = vld1q_f32(d1v);\n        } else {\n            float d0v[4];\n            float d1v[4];\n\n            d0v[0] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d0v[1] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d0v[2] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d0v[3] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d0 = vld1q_f32(d0v);\n\n            d1v[0] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d1v[1] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d1v[2] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d1v[3] = ma_dither_f32_triangle(ditherMin, ditherMax);\n            d1 = vld1q_f32(d1v);\n        }\n\n        x0 = *((float32x4_t*)(src_f32 + i) + 0);\n        x1 = *((float32x4_t*)(src_f32 + i) + 1);\n\n        x0 = vaddq_f32(x0, d0);\n        x1 = vaddq_f32(x1, d1);\n\n        x0 = vmulq_n_f32(x0, 32767.0f);\n        x1 = vmulq_n_f32(x1, 32767.0f);\n\n        i0 = vcvtq_s32_f32(x0);\n        i1 = vcvtq_s32_f32(x1);\n        *((int16x8_t*)(dst_s16 + i)) = vcombine_s16(vqmovn_s32(i0), vqmovn_s32(i1));\n\n        i += 8;\n    }\n\n\n    /* Leftover. */\n    for (; i < count; i += 1) {\n        float x = src_f32[i];\n        x = x + ma_dither_f32(ditherMode, ditherMin, ditherMax);\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n        x = x * 32767.0f;                           /* -1..1 to -32767..32767 */\n\n        dst_s16[i] = (ma_int16)x;\n    }\n}\n#endif  /* Neon */\n#endif  /* MA_USE_REFERENCE_CONVERSION_APIS */\n\nMA_API void ma_pcm_f32_to_s16(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_f32_to_s16__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_f32_to_s16__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_f32_to_s16__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_f32_to_s16__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_f32_to_s24__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_uint8* dst_s24 = (ma_uint8*)dst;\n    const float* src_f32 = (const float*)src;\n\n    ma_uint64 i;\n    for (i = 0; i < count; i += 1) {\n        ma_int32 r;\n        float x = src_f32[i];\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n\n#if 0\n        /* The accurate way. */\n        x = x + 1;                                  /* -1..1 to 0..2 */\n        x = x * 8388607.5f;                         /* 0..2 to 0..16777215 */\n        x = x - 8388608.0f;                         /* 0..16777215 to -8388608..8388607 */\n#else\n        /* The fast way. */\n        x = x * 8388607.0f;                         /* -1..1 to -8388607..8388607 */\n#endif\n\n        r = (ma_int32)x;\n        dst_s24[(i*3)+0] = (ma_uint8)((r & 0x0000FF) >>  0);\n        dst_s24[(i*3)+1] = (ma_uint8)((r & 0x00FF00) >>  8);\n        dst_s24[(i*3)+2] = (ma_uint8)((r & 0xFF0000) >> 16);\n    }\n\n    (void)ditherMode;   /* No dithering for f32 -> s24. */\n}\n\nstatic MA_INLINE void ma_pcm_f32_to_s24__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s24__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_f32_to_s24__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_f32_to_s24__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s24__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_f32_to_s24(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_f32_to_s24__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_f32_to_s24__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_f32_to_s24__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_f32_to_s24__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nstatic MA_INLINE void ma_pcm_f32_to_s32__reference(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_int32* dst_s32 = (ma_int32*)dst;\n    const float* src_f32 = (const float*)src;\n\n    ma_uint32 i;\n    for (i = 0; i < count; i += 1) {\n        double x = src_f32[i];\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));    /* clip */\n\n#if 0\n        /* The accurate way. */\n        x = x + 1;                                  /* -1..1 to 0..2 */\n        x = x * 2147483647.5;                       /* 0..2 to 0..4294967295 */\n        x = x - 2147483648.0;                       /* 0...4294967295 to -2147483648..2147483647 */\n#else\n        /* The fast way. */\n        x = x * 2147483647.0;                       /* -1..1 to -2147483647..2147483647 */\n#endif\n\n        dst_s32[i] = (ma_int32)x;\n    }\n\n    (void)ditherMode;   /* No dithering for f32 -> s32. */\n}\n\nstatic MA_INLINE void ma_pcm_f32_to_s32__optimized(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s32__reference(dst, src, count, ditherMode);\n}\n\n#if defined(MA_SUPPORT_SSE2)\nstatic MA_INLINE void ma_pcm_f32_to_s32__sse2(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n#if defined(MA_SUPPORT_NEON)\nstatic MA_INLINE void ma_pcm_f32_to_s32__neon(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    ma_pcm_f32_to_s32__optimized(dst, src, count, ditherMode);\n}\n#endif\n\nMA_API void ma_pcm_f32_to_s32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_f32_to_s32__reference(dst, src, count, ditherMode);\n#else\n    #  if defined(MA_SUPPORT_SSE2)\n        if (ma_has_sse2()) {\n            ma_pcm_f32_to_s32__sse2(dst, src, count, ditherMode);\n        } else\n    #elif defined(MA_SUPPORT_NEON)\n        if (ma_has_neon()) {\n            ma_pcm_f32_to_s32__neon(dst, src, count, ditherMode);\n        } else\n    #endif\n        {\n            ma_pcm_f32_to_s32__optimized(dst, src, count, ditherMode);\n        }\n#endif\n}\n\n\nMA_API void ma_pcm_f32_to_f32(void* dst, const void* src, ma_uint64 count, ma_dither_mode ditherMode)\n{\n    (void)ditherMode;\n\n    ma_copy_memory_64(dst, src, count * sizeof(float));\n}\n\n\nstatic void ma_pcm_interleave_f32__reference(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    float* dst_f32 = (float*)dst;\n    const float** src_f32 = (const float**)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_f32[iFrame*channels + iChannel] = src_f32[iChannel][iFrame];\n        }\n    }\n}\n\nstatic void ma_pcm_interleave_f32__optimized(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_interleave_f32__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_interleave_f32(void* dst, const void** src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_interleave_f32__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_interleave_f32__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nstatic void ma_pcm_deinterleave_f32__reference(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    float** dst_f32 = (float**)dst;\n    const float* src_f32 = (const float*)src;\n\n    ma_uint64 iFrame;\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; iChannel += 1) {\n            dst_f32[iChannel][iFrame] = src_f32[iFrame*channels + iChannel];\n        }\n    }\n}\n\nstatic void ma_pcm_deinterleave_f32__optimized(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n    ma_pcm_deinterleave_f32__reference(dst, src, frameCount, channels);\n}\n\nMA_API void ma_pcm_deinterleave_f32(void** dst, const void* src, ma_uint64 frameCount, ma_uint32 channels)\n{\n#ifdef MA_USE_REFERENCE_CONVERSION_APIS\n    ma_pcm_deinterleave_f32__reference(dst, src, frameCount, channels);\n#else\n    ma_pcm_deinterleave_f32__optimized(dst, src, frameCount, channels);\n#endif\n}\n\n\nMA_API void ma_pcm_convert(void* pOut, ma_format formatOut, const void* pIn, ma_format formatIn, ma_uint64 sampleCount, ma_dither_mode ditherMode)\n{\n    if (formatOut == formatIn) {\n        ma_copy_memory_64(pOut, pIn, sampleCount * ma_get_bytes_per_sample(formatOut));\n        return;\n    }\n\n    switch (formatIn)\n    {\n        case ma_format_u8:\n        {\n            switch (formatOut)\n            {\n                case ma_format_s16: ma_pcm_u8_to_s16(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s24: ma_pcm_u8_to_s24(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s32: ma_pcm_u8_to_s32(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_f32: ma_pcm_u8_to_f32(pOut, pIn, sampleCount, ditherMode); return;\n                default: break;\n            }\n        } break;\n\n        case ma_format_s16:\n        {\n            switch (formatOut)\n            {\n                case ma_format_u8:  ma_pcm_s16_to_u8( pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s24: ma_pcm_s16_to_s24(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s32: ma_pcm_s16_to_s32(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_f32: ma_pcm_s16_to_f32(pOut, pIn, sampleCount, ditherMode); return;\n                default: break;\n            }\n        } break;\n\n        case ma_format_s24:\n        {\n            switch (formatOut)\n            {\n                case ma_format_u8:  ma_pcm_s24_to_u8( pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s16: ma_pcm_s24_to_s16(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s32: ma_pcm_s24_to_s32(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_f32: ma_pcm_s24_to_f32(pOut, pIn, sampleCount, ditherMode); return;\n                default: break;\n            }\n        } break;\n\n        case ma_format_s32:\n        {\n            switch (formatOut)\n            {\n                case ma_format_u8:  ma_pcm_s32_to_u8( pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s16: ma_pcm_s32_to_s16(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s24: ma_pcm_s32_to_s24(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_f32: ma_pcm_s32_to_f32(pOut, pIn, sampleCount, ditherMode); return;\n                default: break;\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            switch (formatOut)\n            {\n                case ma_format_u8:  ma_pcm_f32_to_u8( pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s16: ma_pcm_f32_to_s16(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s24: ma_pcm_f32_to_s24(pOut, pIn, sampleCount, ditherMode); return;\n                case ma_format_s32: ma_pcm_f32_to_s32(pOut, pIn, sampleCount, ditherMode); return;\n                default: break;\n            }\n        } break;\n\n        default: break;\n    }\n}\n\nMA_API void ma_convert_pcm_frames_format(void* pOut, ma_format formatOut, const void* pIn, ma_format formatIn, ma_uint64 frameCount, ma_uint32 channels, ma_dither_mode ditherMode)\n{\n    ma_pcm_convert(pOut, formatOut, pIn, formatIn, frameCount * channels, ditherMode);\n}\n\nMA_API void ma_deinterleave_pcm_frames(ma_format format, ma_uint32 channels, ma_uint64 frameCount, const void* pInterleavedPCMFrames, void** ppDeinterleavedPCMFrames)\n{\n    if (pInterleavedPCMFrames == NULL || ppDeinterleavedPCMFrames == NULL) {\n        return; /* Invalid args. */\n    }\n\n    /* For efficiency we do this per format. */\n    switch (format) {\n        case ma_format_s16:\n        {\n            const ma_int16* pSrcS16 = (const ma_int16*)pInterleavedPCMFrames;\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                    ma_int16* pDstS16 = (ma_int16*)ppDeinterleavedPCMFrames[iChannel];\n                    pDstS16[iPCMFrame] = pSrcS16[iPCMFrame*channels+iChannel];\n                }\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            const float* pSrcF32 = (const float*)pInterleavedPCMFrames;\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                    float* pDstF32 = (float*)ppDeinterleavedPCMFrames[iChannel];\n                    pDstF32[iPCMFrame] = pSrcF32[iPCMFrame*channels+iChannel];\n                }\n            }\n        } break;\n\n        default:\n        {\n            ma_uint32 sampleSizeInBytes = ma_get_bytes_per_sample(format);\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                          void* pDst = ma_offset_ptr(ppDeinterleavedPCMFrames[iChannel], iPCMFrame*sampleSizeInBytes);\n                    const void* pSrc = ma_offset_ptr(pInterleavedPCMFrames, (iPCMFrame*channels+iChannel)*sampleSizeInBytes);\n                    memcpy(pDst, pSrc, sampleSizeInBytes);\n                }\n            }\n        } break;\n    }\n}\n\nMA_API void ma_interleave_pcm_frames(ma_format format, ma_uint32 channels, ma_uint64 frameCount, const void** ppDeinterleavedPCMFrames, void* pInterleavedPCMFrames)\n{\n    switch (format)\n    {\n        case ma_format_s16:\n        {\n            ma_int16* pDstS16 = (ma_int16*)pInterleavedPCMFrames;\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                    const ma_int16* pSrcS16 = (const ma_int16*)ppDeinterleavedPCMFrames[iChannel];\n                    pDstS16[iPCMFrame*channels+iChannel] = pSrcS16[iPCMFrame];\n                }\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            float* pDstF32 = (float*)pInterleavedPCMFrames;\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                    const float* pSrcF32 = (const float*)ppDeinterleavedPCMFrames[iChannel];\n                    pDstF32[iPCMFrame*channels+iChannel] = pSrcF32[iPCMFrame];\n                }\n            }\n        } break;\n\n        default:\n        {\n            ma_uint32 sampleSizeInBytes = ma_get_bytes_per_sample(format);\n            ma_uint64 iPCMFrame;\n            for (iPCMFrame = 0; iPCMFrame < frameCount; ++iPCMFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < channels; ++iChannel) {\n                          void* pDst = ma_offset_ptr(pInterleavedPCMFrames, (iPCMFrame*channels+iChannel)*sampleSizeInBytes);\n                    const void* pSrc = ma_offset_ptr(ppDeinterleavedPCMFrames[iChannel], iPCMFrame*sampleSizeInBytes);\n                    memcpy(pDst, pSrc, sampleSizeInBytes);\n                }\n            }\n        } break;\n    }\n}\n\n\n/**************************************************************************************************************************************************************\n\nBiquad Filter\n\n**************************************************************************************************************************************************************/\n#ifndef MA_BIQUAD_FIXED_POINT_SHIFT\n#define MA_BIQUAD_FIXED_POINT_SHIFT 14\n#endif\n\nstatic ma_int32 ma_biquad_float_to_fp(double x)\n{\n    return (ma_int32)(x * (1 << MA_BIQUAD_FIXED_POINT_SHIFT));\n}\n\nMA_API ma_biquad_config ma_biquad_config_init(ma_format format, ma_uint32 channels, double b0, double b1, double b2, double a0, double a1, double a2)\n{\n    ma_biquad_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.b0 = b0;\n    config.b1 = b1;\n    config.b2 = b2;\n    config.a0 = a0;\n    config.a1 = a1;\n    config.a2 = a2;\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t r1Offset;\n    size_t r2Offset;\n} ma_biquad_heap_layout;\n\nstatic ma_result ma_biquad_get_heap_layout(const ma_biquad_config* pConfig, ma_biquad_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* R0 */\n    pHeapLayout->r1Offset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(ma_biquad_coefficient) * pConfig->channels;\n\n    /* R1 */\n    pHeapLayout->r2Offset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(ma_biquad_coefficient) * pConfig->channels;\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_biquad_get_heap_size(const ma_biquad_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_biquad_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_biquad_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_biquad_init_preallocated(const ma_biquad_config* pConfig, void* pHeap, ma_biquad* pBQ)\n{\n    ma_result result;\n    ma_biquad_heap_layout heapLayout;\n\n    if (pBQ == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pBQ);\n\n    result = ma_biquad_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pBQ->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pBQ->pR1 = (ma_biquad_coefficient*)ma_offset_ptr(pHeap, heapLayout.r1Offset);\n    pBQ->pR2 = (ma_biquad_coefficient*)ma_offset_ptr(pHeap, heapLayout.r2Offset);\n\n    return ma_biquad_reinit(pConfig, pBQ);\n}\n\nMA_API ma_result ma_biquad_init(const ma_biquad_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_biquad* pBQ)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_biquad_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_biquad_init_preallocated(pConfig, pHeap, pBQ);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pBQ->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_biquad_uninit(ma_biquad* pBQ, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pBQ == NULL) {\n        return;\n    }\n\n    if (pBQ->_ownsHeap) {\n        ma_free(pBQ->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_biquad_reinit(const ma_biquad_config* pConfig, ma_biquad* pBQ)\n{\n    if (pBQ == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->a0 == 0) {\n        return MA_INVALID_ARGS; /* Division by zero. */\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pBQ->format != ma_format_unknown && pBQ->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pBQ->channels != 0 && pBQ->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n\n    pBQ->format   = pConfig->format;\n    pBQ->channels = pConfig->channels;\n\n    /* Normalize. */\n    if (pConfig->format == ma_format_f32) {\n        pBQ->b0.f32 = (float)(pConfig->b0 / pConfig->a0);\n        pBQ->b1.f32 = (float)(pConfig->b1 / pConfig->a0);\n        pBQ->b2.f32 = (float)(pConfig->b2 / pConfig->a0);\n        pBQ->a1.f32 = (float)(pConfig->a1 / pConfig->a0);\n        pBQ->a2.f32 = (float)(pConfig->a2 / pConfig->a0);\n    } else {\n        pBQ->b0.s32 = ma_biquad_float_to_fp(pConfig->b0 / pConfig->a0);\n        pBQ->b1.s32 = ma_biquad_float_to_fp(pConfig->b1 / pConfig->a0);\n        pBQ->b2.s32 = ma_biquad_float_to_fp(pConfig->b2 / pConfig->a0);\n        pBQ->a1.s32 = ma_biquad_float_to_fp(pConfig->a1 / pConfig->a0);\n        pBQ->a2.s32 = ma_biquad_float_to_fp(pConfig->a2 / pConfig->a0);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_biquad_clear_cache(ma_biquad* pBQ)\n{\n    if (pBQ == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pBQ->format == ma_format_f32) {\n        pBQ->pR1->f32 = 0;\n        pBQ->pR2->f32 = 0;\n    } else {\n        pBQ->pR1->s32 = 0;\n        pBQ->pR2->s32 = 0;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_biquad_process_pcm_frame_f32__direct_form_2_transposed(ma_biquad* pBQ, float* pY, const float* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pBQ->channels;\n    const float b0 = pBQ->b0.f32;\n    const float b1 = pBQ->b1.f32;\n    const float b2 = pBQ->b2.f32;\n    const float a1 = pBQ->a1.f32;\n    const float a2 = pBQ->a2.f32;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        float r1 = pBQ->pR1[c].f32;\n        float r2 = pBQ->pR2[c].f32;\n        float x  = pX[c];\n        float y;\n\n        y  = b0*x        + r1;\n        r1 = b1*x - a1*y + r2;\n        r2 = b2*x - a2*y;\n\n        pY[c]           = y;\n        pBQ->pR1[c].f32 = r1;\n        pBQ->pR2[c].f32 = r2;\n    }\n}\n\nstatic MA_INLINE void ma_biquad_process_pcm_frame_f32(ma_biquad* pBQ, float* pY, const float* pX)\n{\n    ma_biquad_process_pcm_frame_f32__direct_form_2_transposed(pBQ, pY, pX);\n}\n\nstatic MA_INLINE void ma_biquad_process_pcm_frame_s16__direct_form_2_transposed(ma_biquad* pBQ, ma_int16* pY, const ma_int16* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pBQ->channels;\n    const ma_int32 b0 = pBQ->b0.s32;\n    const ma_int32 b1 = pBQ->b1.s32;\n    const ma_int32 b2 = pBQ->b2.s32;\n    const ma_int32 a1 = pBQ->a1.s32;\n    const ma_int32 a2 = pBQ->a2.s32;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        ma_int32 r1 = pBQ->pR1[c].s32;\n        ma_int32 r2 = pBQ->pR2[c].s32;\n        ma_int32 x  = pX[c];\n        ma_int32 y;\n\n        y  = (b0*x        + r1) >> MA_BIQUAD_FIXED_POINT_SHIFT;\n        r1 = (b1*x - a1*y + r2);\n        r2 = (b2*x - a2*y);\n\n        pY[c]           = (ma_int16)ma_clamp(y, -32768, 32767);\n        pBQ->pR1[c].s32 = r1;\n        pBQ->pR2[c].s32 = r2;\n    }\n}\n\nstatic MA_INLINE void ma_biquad_process_pcm_frame_s16(ma_biquad* pBQ, ma_int16* pY, const ma_int16* pX)\n{\n    ma_biquad_process_pcm_frame_s16__direct_form_2_transposed(pBQ, pY, pX);\n}\n\nMA_API ma_result ma_biquad_process_pcm_frames(ma_biquad* pBQ, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint32 n;\n\n    if (pBQ == NULL || pFramesOut == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Note that the logic below needs to support in-place filtering. That is, it must support the case where pFramesOut and pFramesIn are the same. */\n\n    if (pBQ->format == ma_format_f32) {\n        /* */ float* pY = (      float*)pFramesOut;\n        const float* pX = (const float*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_biquad_process_pcm_frame_f32__direct_form_2_transposed(pBQ, pY, pX);\n            pY += pBQ->channels;\n            pX += pBQ->channels;\n        }\n    } else if (pBQ->format == ma_format_s16) {\n        /* */ ma_int16* pY = (      ma_int16*)pFramesOut;\n        const ma_int16* pX = (const ma_int16*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_biquad_process_pcm_frame_s16__direct_form_2_transposed(pBQ, pY, pX);\n            pY += pBQ->channels;\n            pX += pBQ->channels;\n        }\n    } else {\n        MA_ASSERT(MA_FALSE);\n        return MA_INVALID_ARGS; /* Format not supported. Should never hit this because it's checked in ma_biquad_init() and ma_biquad_reinit(). */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_biquad_get_latency(const ma_biquad* pBQ)\n{\n    if (pBQ == NULL) {\n        return 0;\n    }\n\n    return 2;\n}\n\n\n/**************************************************************************************************************************************************************\n\nLow-Pass Filter\n\n**************************************************************************************************************************************************************/\nMA_API ma_lpf1_config ma_lpf1_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency)\n{\n    ma_lpf1_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.q = 0.5;\n\n    return config;\n}\n\nMA_API ma_lpf2_config ma_lpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q)\n{\n    ma_lpf2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.q = q;\n\n    /* Q cannot be 0 or else it'll result in a division by 0. In this case just default to 0.707107. */\n    if (config.q == 0) {\n        config.q = 0.707107;\n    }\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t r1Offset;\n} ma_lpf1_heap_layout;\n\nstatic ma_result ma_lpf1_get_heap_layout(const ma_lpf1_config* pConfig, ma_lpf1_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* R1 */\n    pHeapLayout->r1Offset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(ma_biquad_coefficient) * pConfig->channels;\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf1_get_heap_size(const ma_lpf1_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_lpf1_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_lpf1_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf1_init_preallocated(const ma_lpf1_config* pConfig, void* pHeap, ma_lpf1* pLPF)\n{\n    ma_result result;\n    ma_lpf1_heap_layout heapLayout;\n\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLPF);\n\n    result = ma_lpf1_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pLPF->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pLPF->pR1 = (ma_biquad_coefficient*)ma_offset_ptr(pHeap, heapLayout.r1Offset);\n\n    return ma_lpf1_reinit(pConfig, pLPF);\n}\n\nMA_API ma_result ma_lpf1_init(const ma_lpf1_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf1* pLPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_lpf1_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_lpf1_init_preallocated(pConfig, pHeap, pLPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pLPF->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_lpf1_uninit(ma_lpf1* pLPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pLPF == NULL) {\n        return;\n    }\n\n    if (pLPF->_ownsHeap) {\n        ma_free(pLPF->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_lpf1_reinit(const ma_lpf1_config* pConfig, ma_lpf1* pLPF)\n{\n    double a;\n\n    if (pLPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pLPF->format != ma_format_unknown && pLPF->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pLPF->channels != 0 && pLPF->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n    pLPF->format   = pConfig->format;\n    pLPF->channels = pConfig->channels;\n\n    a = ma_expd(-2 * MA_PI_D * pConfig->cutoffFrequency / pConfig->sampleRate);\n    if (pConfig->format == ma_format_f32) {\n        pLPF->a.f32 = (float)a;\n    } else {\n        pLPF->a.s32 = ma_biquad_float_to_fp(a);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf1_clear_cache(ma_lpf1* pLPF)\n{\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pLPF->format == ma_format_f32) {\n        pLPF->a.f32 = 0;\n    } else {\n        pLPF->a.s32 = 0;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_lpf1_process_pcm_frame_f32(ma_lpf1* pLPF, float* pY, const float* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pLPF->channels;\n    const float a = pLPF->a.f32;\n    const float b = 1 - a;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        float r1 = pLPF->pR1[c].f32;\n        float x  = pX[c];\n        float y;\n\n        y = b*x + a*r1;\n\n        pY[c]           = y;\n        pLPF->pR1[c].f32 = y;\n    }\n}\n\nstatic MA_INLINE void ma_lpf1_process_pcm_frame_s16(ma_lpf1* pLPF, ma_int16* pY, const ma_int16* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pLPF->channels;\n    const ma_int32 a = pLPF->a.s32;\n    const ma_int32 b = ((1 << MA_BIQUAD_FIXED_POINT_SHIFT) - a);\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        ma_int32 r1 = pLPF->pR1[c].s32;\n        ma_int32 x  = pX[c];\n        ma_int32 y;\n\n        y = (b*x + a*r1) >> MA_BIQUAD_FIXED_POINT_SHIFT;\n\n        pY[c]            = (ma_int16)y;\n        pLPF->pR1[c].s32 = (ma_int32)y;\n    }\n}\n\nMA_API ma_result ma_lpf1_process_pcm_frames(ma_lpf1* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint32 n;\n\n    if (pLPF == NULL || pFramesOut == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Note that the logic below needs to support in-place filtering. That is, it must support the case where pFramesOut and pFramesIn are the same. */\n\n    if (pLPF->format == ma_format_f32) {\n        /* */ float* pY = (      float*)pFramesOut;\n        const float* pX = (const float*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_lpf1_process_pcm_frame_f32(pLPF, pY, pX);\n            pY += pLPF->channels;\n            pX += pLPF->channels;\n        }\n    } else if (pLPF->format == ma_format_s16) {\n        /* */ ma_int16* pY = (      ma_int16*)pFramesOut;\n        const ma_int16* pX = (const ma_int16*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_lpf1_process_pcm_frame_s16(pLPF, pY, pX);\n            pY += pLPF->channels;\n            pX += pLPF->channels;\n        }\n    } else {\n        MA_ASSERT(MA_FALSE);\n        return MA_INVALID_ARGS; /* Format not supported. Should never hit this because it's checked in ma_biquad_init() and ma_biquad_reinit(). */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_lpf1_get_latency(const ma_lpf1* pLPF)\n{\n    if (pLPF == NULL) {\n        return 0;\n    }\n\n    return 1;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_lpf2__get_biquad_config(const ma_lpf2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double q;\n    double w;\n    double s;\n    double c;\n    double a;\n\n    MA_ASSERT(pConfig != NULL);\n\n    q = pConfig->q;\n    w = 2 * MA_PI_D * pConfig->cutoffFrequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    a = s / (2*q);\n\n    bqConfig.b0 = (1 - c) / 2;\n    bqConfig.b1 =  1 - c;\n    bqConfig.b2 = (1 - c) / 2;\n    bqConfig.a0 =  1 + a;\n    bqConfig.a1 = -2 * c;\n    bqConfig.a2 =  1 - a;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_lpf2_get_heap_size(const ma_lpf2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_lpf2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_lpf2_init_preallocated(const ma_lpf2_config* pConfig, void* pHeap, ma_lpf2* pLPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLPF);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_lpf2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pLPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf2_init(const ma_lpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf2* pLPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_lpf2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_lpf2_init_preallocated(pConfig, pHeap, pLPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pLPF->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_lpf2_uninit(ma_lpf2* pLPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pLPF == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pLPF->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_lpf2_reinit(const ma_lpf2_config* pConfig, ma_lpf2* pLPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pLPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_lpf2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pLPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf2_clear_cache(ma_lpf2* pLPF)\n{\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_biquad_clear_cache(&pLPF->bq);\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_lpf2_process_pcm_frame_s16(ma_lpf2* pLPF, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pLPF->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_lpf2_process_pcm_frame_f32(ma_lpf2* pLPF, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pLPF->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_lpf2_process_pcm_frames(ma_lpf2* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pLPF->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_lpf2_get_latency(const ma_lpf2* pLPF)\n{\n    if (pLPF == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pLPF->bq);\n}\n\n\nMA_API ma_lpf_config ma_lpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_lpf_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format          = format;\n    config.channels        = channels;\n    config.sampleRate      = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.order           = ma_min(order, MA_MAX_FILTER_ORDER);\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t lpf1Offset;\n    size_t lpf2Offset;  /* Offset of the first second order filter. Subsequent filters will come straight after, and will each have the same heap size. */\n} ma_lpf_heap_layout;\n\nstatic void ma_lpf_calculate_sub_lpf_counts(ma_uint32 order, ma_uint32* pLPF1Count, ma_uint32* pLPF2Count)\n{\n    MA_ASSERT(pLPF1Count != NULL);\n    MA_ASSERT(pLPF2Count != NULL);\n\n    *pLPF1Count = order % 2;\n    *pLPF2Count = order / 2;\n}\n\nstatic ma_result ma_lpf_get_heap_layout(const ma_lpf_config* pConfig, ma_lpf_heap_layout* pHeapLayout)\n{\n    ma_result result;\n    ma_uint32 lpf1Count;\n    ma_uint32 lpf2Count;\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_lpf_calculate_sub_lpf_counts(pConfig->order, &lpf1Count, &lpf2Count);\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* LPF 1 */\n    pHeapLayout->lpf1Offset = pHeapLayout->sizeInBytes;\n    for (ilpf1 = 0; ilpf1 < lpf1Count; ilpf1 += 1) {\n        size_t lpf1HeapSizeInBytes;\n        ma_lpf1_config lpf1Config = ma_lpf1_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency);\n\n        result = ma_lpf1_get_heap_size(&lpf1Config, &lpf1HeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += sizeof(ma_lpf1) + lpf1HeapSizeInBytes;\n    }\n\n    /* LPF 2*/\n    pHeapLayout->lpf2Offset = pHeapLayout->sizeInBytes;\n    for (ilpf2 = 0; ilpf2 < lpf2Count; ilpf2 += 1) {\n        size_t lpf2HeapSizeInBytes;\n        ma_lpf2_config lpf2Config = ma_lpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, 0.707107);   /* <-- The \"q\" parameter does not matter for the purpose of calculating the heap size. */\n\n        result = ma_lpf2_get_heap_size(&lpf2Config, &lpf2HeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += sizeof(ma_lpf2) + lpf2HeapSizeInBytes;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_lpf_reinit__internal(const ma_lpf_config* pConfig, void* pHeap, ma_lpf* pLPF, ma_bool32 isNew)\n{\n    ma_result result;\n    ma_uint32 lpf1Count;\n    ma_uint32 lpf2Count;\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n    ma_lpf_heap_layout heapLayout;  /* Only used if isNew is true. */\n\n    if (pLPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pLPF->format != ma_format_unknown && pLPF->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pLPF->channels != 0 && pLPF->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_lpf_calculate_sub_lpf_counts(pConfig->order, &lpf1Count, &lpf2Count);\n\n    /* The filter order can't change between reinits. */\n    if (!isNew) {\n        if (pLPF->lpf1Count != lpf1Count || pLPF->lpf2Count != lpf2Count) {\n            return MA_INVALID_OPERATION;\n        }\n    }\n\n    if (isNew) {\n        result = ma_lpf_get_heap_layout(pConfig, &heapLayout);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pLPF->_pHeap = pHeap;\n        MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n        pLPF->pLPF1 = (ma_lpf1*)ma_offset_ptr(pHeap, heapLayout.lpf1Offset);\n        pLPF->pLPF2 = (ma_lpf2*)ma_offset_ptr(pHeap, heapLayout.lpf2Offset);\n    } else {\n        MA_ZERO_OBJECT(&heapLayout);    /* To silence a compiler warning. */\n    }\n\n    for (ilpf1 = 0; ilpf1 < lpf1Count; ilpf1 += 1) {\n        ma_lpf1_config lpf1Config = ma_lpf1_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency);\n\n        if (isNew) {\n            size_t lpf1HeapSizeInBytes;\n\n            result = ma_lpf1_get_heap_size(&lpf1Config, &lpf1HeapSizeInBytes);\n            if (result == MA_SUCCESS) {\n                result = ma_lpf1_init_preallocated(&lpf1Config, ma_offset_ptr(pHeap, heapLayout.lpf1Offset + (sizeof(ma_lpf1) * lpf1Count) + (ilpf1 * lpf1HeapSizeInBytes)), &pLPF->pLPF1[ilpf1]);\n            }\n        } else {\n            result = ma_lpf1_reinit(&lpf1Config, &pLPF->pLPF1[ilpf1]);\n        }\n\n        if (result != MA_SUCCESS) {\n            ma_uint32 jlpf1;\n\n            for (jlpf1 = 0; jlpf1 < ilpf1; jlpf1 += 1) {\n                ma_lpf1_uninit(&pLPF->pLPF1[jlpf1], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            return result;\n        }\n    }\n\n    for (ilpf2 = 0; ilpf2 < lpf2Count; ilpf2 += 1) {\n        ma_lpf2_config lpf2Config;\n        double q;\n        double a;\n\n        /* Tempting to use 0.707107, but won't result in a Butterworth filter if the order is > 2. */\n        if (lpf1Count == 1) {\n            a = (1 + ilpf2*1) * (MA_PI_D/(pConfig->order*1));   /* Odd order. */\n        } else {\n            a = (1 + ilpf2*2) * (MA_PI_D/(pConfig->order*2));   /* Even order. */\n        }\n        q = 1 / (2*ma_cosd(a));\n\n        lpf2Config = ma_lpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, q);\n\n        if (isNew) {\n            size_t lpf2HeapSizeInBytes;\n\n            result = ma_lpf2_get_heap_size(&lpf2Config, &lpf2HeapSizeInBytes);\n            if (result == MA_SUCCESS) {\n                result = ma_lpf2_init_preallocated(&lpf2Config, ma_offset_ptr(pHeap, heapLayout.lpf2Offset + (sizeof(ma_lpf2) * lpf2Count) + (ilpf2 * lpf2HeapSizeInBytes)), &pLPF->pLPF2[ilpf2]);\n            }\n        } else {\n            result = ma_lpf2_reinit(&lpf2Config, &pLPF->pLPF2[ilpf2]);\n        }\n\n        if (result != MA_SUCCESS) {\n            ma_uint32 jlpf1;\n            ma_uint32 jlpf2;\n\n            for (jlpf1 = 0; jlpf1 < lpf1Count; jlpf1 += 1) {\n                ma_lpf1_uninit(&pLPF->pLPF1[jlpf1], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            for (jlpf2 = 0; jlpf2 < ilpf2; jlpf2 += 1) {\n                ma_lpf2_uninit(&pLPF->pLPF2[jlpf2], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            return result;\n        }\n    }\n\n    pLPF->lpf1Count  = lpf1Count;\n    pLPF->lpf2Count  = lpf2Count;\n    pLPF->format     = pConfig->format;\n    pLPF->channels   = pConfig->channels;\n    pLPF->sampleRate = pConfig->sampleRate;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_lpf_get_heap_size(const ma_lpf_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_lpf_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_lpf_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return result;\n}\n\nMA_API ma_result ma_lpf_init_preallocated(const ma_lpf_config* pConfig, void* pHeap, ma_lpf* pLPF)\n{\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLPF);\n\n    return ma_lpf_reinit__internal(pConfig, pHeap, pLPF, /*isNew*/MA_TRUE);\n}\n\nMA_API ma_result ma_lpf_init(const ma_lpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf* pLPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_lpf_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_lpf_init_preallocated(pConfig, pHeap, pLPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pLPF->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_lpf_uninit(ma_lpf* pLPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    if (pLPF == NULL) {\n        return;\n    }\n\n    for (ilpf1 = 0; ilpf1 < pLPF->lpf1Count; ilpf1 += 1) {\n        ma_lpf1_uninit(&pLPF->pLPF1[ilpf1], pAllocationCallbacks);\n    }\n\n    for (ilpf2 = 0; ilpf2 < pLPF->lpf2Count; ilpf2 += 1) {\n        ma_lpf2_uninit(&pLPF->pLPF2[ilpf2], pAllocationCallbacks);\n    }\n\n    if (pLPF->_ownsHeap) {\n        ma_free(pLPF->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_lpf_reinit(const ma_lpf_config* pConfig, ma_lpf* pLPF)\n{\n    return ma_lpf_reinit__internal(pConfig, NULL, pLPF, /*isNew*/MA_FALSE);\n}\n\nMA_API ma_result ma_lpf_clear_cache(ma_lpf* pLPF)\n{\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (ilpf1 = 0; ilpf1 < pLPF->lpf1Count; ilpf1 += 1) {\n        ma_lpf1_clear_cache(&pLPF->pLPF1[ilpf1]);\n    }\n\n    for (ilpf2 = 0; ilpf2 < pLPF->lpf2Count; ilpf2 += 1) {\n        ma_lpf2_clear_cache(&pLPF->pLPF2[ilpf2]);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_lpf_process_pcm_frame_f32(ma_lpf* pLPF, float* pY, const void* pX)\n{\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    MA_ASSERT(pLPF->format == ma_format_f32);\n\n    MA_MOVE_MEMORY(pY, pX, ma_get_bytes_per_frame(pLPF->format, pLPF->channels));\n\n    for (ilpf1 = 0; ilpf1 < pLPF->lpf1Count; ilpf1 += 1) {\n        ma_lpf1_process_pcm_frame_f32(&pLPF->pLPF1[ilpf1], pY, pY);\n    }\n\n    for (ilpf2 = 0; ilpf2 < pLPF->lpf2Count; ilpf2 += 1) {\n        ma_lpf2_process_pcm_frame_f32(&pLPF->pLPF2[ilpf2], pY, pY);\n    }\n}\n\nstatic MA_INLINE void ma_lpf_process_pcm_frame_s16(ma_lpf* pLPF, ma_int16* pY, const ma_int16* pX)\n{\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    MA_ASSERT(pLPF->format == ma_format_s16);\n\n    MA_MOVE_MEMORY(pY, pX, ma_get_bytes_per_frame(pLPF->format, pLPF->channels));\n\n    for (ilpf1 = 0; ilpf1 < pLPF->lpf1Count; ilpf1 += 1) {\n        ma_lpf1_process_pcm_frame_s16(&pLPF->pLPF1[ilpf1], pY, pY);\n    }\n\n    for (ilpf2 = 0; ilpf2 < pLPF->lpf2Count; ilpf2 += 1) {\n        ma_lpf2_process_pcm_frame_s16(&pLPF->pLPF2[ilpf2], pY, pY);\n    }\n}\n\nMA_API ma_result ma_lpf_process_pcm_frames(ma_lpf* pLPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_result result;\n    ma_uint32 ilpf1;\n    ma_uint32 ilpf2;\n\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Faster path for in-place. */\n    if (pFramesOut == pFramesIn) {\n        for (ilpf1 = 0; ilpf1 < pLPF->lpf1Count; ilpf1 += 1) {\n            result = ma_lpf1_process_pcm_frames(&pLPF->pLPF1[ilpf1], pFramesOut, pFramesOut, frameCount);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n\n        for (ilpf2 = 0; ilpf2 < pLPF->lpf2Count; ilpf2 += 1) {\n            result = ma_lpf2_process_pcm_frames(&pLPF->pLPF2[ilpf2], pFramesOut, pFramesOut, frameCount);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n    }\n\n    /* Slightly slower path for copying. */\n    if (pFramesOut != pFramesIn) {\n        ma_uint32 iFrame;\n\n        /*  */ if (pLPF->format == ma_format_f32) {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                ma_lpf_process_pcm_frame_f32(pLPF, pFramesOutF32, pFramesInF32);\n                pFramesOutF32 += pLPF->channels;\n                pFramesInF32  += pLPF->channels;\n            }\n        } else if (pLPF->format == ma_format_s16) {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                ma_lpf_process_pcm_frame_s16(pLPF, pFramesOutS16, pFramesInS16);\n                pFramesOutS16 += pLPF->channels;\n                pFramesInS16  += pLPF->channels;\n            }\n        } else {\n            MA_ASSERT(MA_FALSE);\n            return MA_INVALID_OPERATION;    /* Should never hit this. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_lpf_get_latency(const ma_lpf* pLPF)\n{\n    if (pLPF == NULL) {\n        return 0;\n    }\n\n    return pLPF->lpf2Count*2 + pLPF->lpf1Count;\n}\n\n\n/**************************************************************************************************************************************************************\n\nHigh-Pass Filtering\n\n**************************************************************************************************************************************************************/\nMA_API ma_hpf1_config ma_hpf1_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency)\n{\n    ma_hpf1_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n\n    return config;\n}\n\nMA_API ma_hpf2_config ma_hpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q)\n{\n    ma_hpf2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.q = q;\n\n    /* Q cannot be 0 or else it'll result in a division by 0. In this case just default to 0.707107. */\n    if (config.q == 0) {\n        config.q = 0.707107;\n    }\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t r1Offset;\n} ma_hpf1_heap_layout;\n\nstatic ma_result ma_hpf1_get_heap_layout(const ma_hpf1_config* pConfig, ma_hpf1_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* R1 */\n    pHeapLayout->r1Offset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(ma_biquad_coefficient) * pConfig->channels;\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_hpf1_get_heap_size(const ma_hpf1_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_hpf1_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_hpf1_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_hpf1_init_preallocated(const ma_hpf1_config* pConfig, void* pHeap, ma_hpf1* pLPF)\n{\n    ma_result result;\n    ma_hpf1_heap_layout heapLayout;\n\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLPF);\n\n    result = ma_hpf1_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pLPF->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pLPF->pR1 = (ma_biquad_coefficient*)ma_offset_ptr(pHeap, heapLayout.r1Offset);\n\n    return ma_hpf1_reinit(pConfig, pLPF);\n}\n\nMA_API ma_result ma_hpf1_init(const ma_hpf1_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf1* pLPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_hpf1_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_hpf1_init_preallocated(pConfig, pHeap, pLPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pLPF->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_hpf1_uninit(ma_hpf1* pHPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pHPF == NULL) {\n        return;\n    }\n\n    if (pHPF->_ownsHeap) {\n        ma_free(pHPF->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_hpf1_reinit(const ma_hpf1_config* pConfig, ma_hpf1* pHPF)\n{\n    double a;\n\n    if (pHPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pHPF->format != ma_format_unknown && pHPF->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pHPF->channels != 0 && pHPF->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n    pHPF->format   = pConfig->format;\n    pHPF->channels = pConfig->channels;\n\n    a = ma_expd(-2 * MA_PI_D * pConfig->cutoffFrequency / pConfig->sampleRate);\n    if (pConfig->format == ma_format_f32) {\n        pHPF->a.f32 = (float)a;\n    } else {\n        pHPF->a.s32 = ma_biquad_float_to_fp(a);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_hpf1_process_pcm_frame_f32(ma_hpf1* pHPF, float* pY, const float* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pHPF->channels;\n    const float a = 1 - pHPF->a.f32;\n    const float b = 1 - a;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        float r1 = pHPF->pR1[c].f32;\n        float x  = pX[c];\n        float y;\n\n        y = b*x - a*r1;\n\n        pY[c]            = y;\n        pHPF->pR1[c].f32 = y;\n    }\n}\n\nstatic MA_INLINE void ma_hpf1_process_pcm_frame_s16(ma_hpf1* pHPF, ma_int16* pY, const ma_int16* pX)\n{\n    ma_uint32 c;\n    const ma_uint32 channels = pHPF->channels;\n    const ma_int32 a = ((1 << MA_BIQUAD_FIXED_POINT_SHIFT) - pHPF->a.s32);\n    const ma_int32 b = ((1 << MA_BIQUAD_FIXED_POINT_SHIFT) - a);\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        ma_int32 r1 = pHPF->pR1[c].s32;\n        ma_int32 x  = pX[c];\n        ma_int32 y;\n\n        y = (b*x - a*r1) >> MA_BIQUAD_FIXED_POINT_SHIFT;\n\n        pY[c]            = (ma_int16)y;\n        pHPF->pR1[c].s32 = (ma_int32)y;\n    }\n}\n\nMA_API ma_result ma_hpf1_process_pcm_frames(ma_hpf1* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint32 n;\n\n    if (pHPF == NULL || pFramesOut == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Note that the logic below needs to support in-place filtering. That is, it must support the case where pFramesOut and pFramesIn are the same. */\n\n    if (pHPF->format == ma_format_f32) {\n        /* */ float* pY = (      float*)pFramesOut;\n        const float* pX = (const float*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_hpf1_process_pcm_frame_f32(pHPF, pY, pX);\n            pY += pHPF->channels;\n            pX += pHPF->channels;\n        }\n    } else if (pHPF->format == ma_format_s16) {\n        /* */ ma_int16* pY = (      ma_int16*)pFramesOut;\n        const ma_int16* pX = (const ma_int16*)pFramesIn;\n\n        for (n = 0; n < frameCount; n += 1) {\n            ma_hpf1_process_pcm_frame_s16(pHPF, pY, pX);\n            pY += pHPF->channels;\n            pX += pHPF->channels;\n        }\n    } else {\n        MA_ASSERT(MA_FALSE);\n        return MA_INVALID_ARGS; /* Format not supported. Should never hit this because it's checked in ma_biquad_init() and ma_biquad_reinit(). */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_hpf1_get_latency(const ma_hpf1* pHPF)\n{\n    if (pHPF == NULL) {\n        return 0;\n    }\n\n    return 1;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_hpf2__get_biquad_config(const ma_hpf2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double q;\n    double w;\n    double s;\n    double c;\n    double a;\n\n    MA_ASSERT(pConfig != NULL);\n\n    q = pConfig->q;\n    w = 2 * MA_PI_D * pConfig->cutoffFrequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    a = s / (2*q);\n\n    bqConfig.b0 =  (1 + c) / 2;\n    bqConfig.b1 = -(1 + c);\n    bqConfig.b2 =  (1 + c) / 2;\n    bqConfig.a0 =   1 + a;\n    bqConfig.a1 =  -2 * c;\n    bqConfig.a2 =   1 - a;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_hpf2_get_heap_size(const ma_hpf2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_hpf2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_hpf2_init_preallocated(const ma_hpf2_config* pConfig, void* pHeap, ma_hpf2* pHPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pHPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pHPF);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_hpf2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pHPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_hpf2_init(const ma_hpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf2* pHPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_hpf2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_hpf2_init_preallocated(pConfig, pHeap, pHPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pHPF->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_hpf2_uninit(ma_hpf2* pHPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pHPF == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pHPF->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_hpf2_reinit(const ma_hpf2_config* pConfig, ma_hpf2* pHPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pHPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_hpf2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pHPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_hpf2_process_pcm_frame_s16(ma_hpf2* pHPF, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pHPF->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_hpf2_process_pcm_frame_f32(ma_hpf2* pHPF, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pHPF->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_hpf2_process_pcm_frames(ma_hpf2* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pHPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pHPF->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_hpf2_get_latency(const ma_hpf2* pHPF)\n{\n    if (pHPF == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pHPF->bq);\n}\n\n\nMA_API ma_hpf_config ma_hpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_hpf_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format          = format;\n    config.channels        = channels;\n    config.sampleRate      = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.order           = ma_min(order, MA_MAX_FILTER_ORDER);\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t hpf1Offset;\n    size_t hpf2Offset;  /* Offset of the first second order filter. Subsequent filters will come straight after, and will each have the same heap size. */\n} ma_hpf_heap_layout;\n\nstatic void ma_hpf_calculate_sub_hpf_counts(ma_uint32 order, ma_uint32* pHPF1Count, ma_uint32* pHPF2Count)\n{\n    MA_ASSERT(pHPF1Count != NULL);\n    MA_ASSERT(pHPF2Count != NULL);\n\n    *pHPF1Count = order % 2;\n    *pHPF2Count = order / 2;\n}\n\nstatic ma_result ma_hpf_get_heap_layout(const ma_hpf_config* pConfig, ma_hpf_heap_layout* pHeapLayout)\n{\n    ma_result result;\n    ma_uint32 hpf1Count;\n    ma_uint32 hpf2Count;\n    ma_uint32 ihpf1;\n    ma_uint32 ihpf2;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_hpf_calculate_sub_hpf_counts(pConfig->order, &hpf1Count, &hpf2Count);\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* HPF 1 */\n    pHeapLayout->hpf1Offset = pHeapLayout->sizeInBytes;\n    for (ihpf1 = 0; ihpf1 < hpf1Count; ihpf1 += 1) {\n        size_t hpf1HeapSizeInBytes;\n        ma_hpf1_config hpf1Config = ma_hpf1_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency);\n\n        result = ma_hpf1_get_heap_size(&hpf1Config, &hpf1HeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += sizeof(ma_hpf1) + hpf1HeapSizeInBytes;\n    }\n\n    /* HPF 2*/\n    pHeapLayout->hpf2Offset = pHeapLayout->sizeInBytes;\n    for (ihpf2 = 0; ihpf2 < hpf2Count; ihpf2 += 1) {\n        size_t hpf2HeapSizeInBytes;\n        ma_hpf2_config hpf2Config = ma_hpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, 0.707107);   /* <-- The \"q\" parameter does not matter for the purpose of calculating the heap size. */\n\n        result = ma_hpf2_get_heap_size(&hpf2Config, &hpf2HeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += sizeof(ma_hpf2) + hpf2HeapSizeInBytes;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_hpf_reinit__internal(const ma_hpf_config* pConfig, void* pHeap, ma_hpf* pHPF, ma_bool32 isNew)\n{\n    ma_result result;\n    ma_uint32 hpf1Count;\n    ma_uint32 hpf2Count;\n    ma_uint32 ihpf1;\n    ma_uint32 ihpf2;\n    ma_hpf_heap_layout heapLayout;  /* Only used if isNew is true. */\n\n    if (pHPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pHPF->format != ma_format_unknown && pHPF->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pHPF->channels != 0 && pHPF->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_hpf_calculate_sub_hpf_counts(pConfig->order, &hpf1Count, &hpf2Count);\n\n    /* The filter order can't change between reinits. */\n    if (!isNew) {\n        if (pHPF->hpf1Count != hpf1Count || pHPF->hpf2Count != hpf2Count) {\n            return MA_INVALID_OPERATION;\n        }\n    }\n\n    if (isNew) {\n        result = ma_hpf_get_heap_layout(pConfig, &heapLayout);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHPF->_pHeap = pHeap;\n        MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n        pHPF->pHPF1 = (ma_hpf1*)ma_offset_ptr(pHeap, heapLayout.hpf1Offset);\n        pHPF->pHPF2 = (ma_hpf2*)ma_offset_ptr(pHeap, heapLayout.hpf2Offset);\n    } else {\n        MA_ZERO_OBJECT(&heapLayout);    /* To silence a compiler warning. */\n    }\n\n    for (ihpf1 = 0; ihpf1 < hpf1Count; ihpf1 += 1) {\n        ma_hpf1_config hpf1Config = ma_hpf1_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency);\n\n        if (isNew) {\n            size_t hpf1HeapSizeInBytes;\n\n            result = ma_hpf1_get_heap_size(&hpf1Config, &hpf1HeapSizeInBytes);\n            if (result == MA_SUCCESS) {\n                result = ma_hpf1_init_preallocated(&hpf1Config, ma_offset_ptr(pHeap, heapLayout.hpf1Offset + (sizeof(ma_hpf1) * hpf1Count) + (ihpf1 * hpf1HeapSizeInBytes)), &pHPF->pHPF1[ihpf1]);\n            }\n        } else {\n            result = ma_hpf1_reinit(&hpf1Config, &pHPF->pHPF1[ihpf1]);\n        }\n\n        if (result != MA_SUCCESS) {\n            ma_uint32 jhpf1;\n\n            for (jhpf1 = 0; jhpf1 < ihpf1; jhpf1 += 1) {\n                ma_hpf1_uninit(&pHPF->pHPF1[jhpf1], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            return result;\n        }\n    }\n\n    for (ihpf2 = 0; ihpf2 < hpf2Count; ihpf2 += 1) {\n        ma_hpf2_config hpf2Config;\n        double q;\n        double a;\n\n        /* Tempting to use 0.707107, but won't result in a Butterworth filter if the order is > 2. */\n        if (hpf1Count == 1) {\n            a = (1 + ihpf2*1) * (MA_PI_D/(pConfig->order*1));   /* Odd order. */\n        } else {\n            a = (1 + ihpf2*2) * (MA_PI_D/(pConfig->order*2));   /* Even order. */\n        }\n        q = 1 / (2*ma_cosd(a));\n\n        hpf2Config = ma_hpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, q);\n\n        if (isNew) {\n            size_t hpf2HeapSizeInBytes;\n\n            result = ma_hpf2_get_heap_size(&hpf2Config, &hpf2HeapSizeInBytes);\n            if (result == MA_SUCCESS) {\n                result = ma_hpf2_init_preallocated(&hpf2Config, ma_offset_ptr(pHeap, heapLayout.hpf2Offset + (sizeof(ma_hpf2) * hpf2Count) + (ihpf2 * hpf2HeapSizeInBytes)), &pHPF->pHPF2[ihpf2]);\n            }\n        } else {\n            result = ma_hpf2_reinit(&hpf2Config, &pHPF->pHPF2[ihpf2]);\n        }\n\n        if (result != MA_SUCCESS) {\n            ma_uint32 jhpf1;\n            ma_uint32 jhpf2;\n\n            for (jhpf1 = 0; jhpf1 < hpf1Count; jhpf1 += 1) {\n                ma_hpf1_uninit(&pHPF->pHPF1[jhpf1], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            for (jhpf2 = 0; jhpf2 < ihpf2; jhpf2 += 1) {\n                ma_hpf2_uninit(&pHPF->pHPF2[jhpf2], NULL);  /* No need for allocation callbacks here since we used a preallocated heap allocation. */\n            }\n\n            return result;\n        }\n    }\n\n    pHPF->hpf1Count  = hpf1Count;\n    pHPF->hpf2Count  = hpf2Count;\n    pHPF->format     = pConfig->format;\n    pHPF->channels   = pConfig->channels;\n    pHPF->sampleRate = pConfig->sampleRate;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_hpf_get_heap_size(const ma_hpf_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_hpf_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_hpf_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return result;\n}\n\nMA_API ma_result ma_hpf_init_preallocated(const ma_hpf_config* pConfig, void* pHeap, ma_hpf* pLPF)\n{\n    if (pLPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pLPF);\n\n    return ma_hpf_reinit__internal(pConfig, pHeap, pLPF, /*isNew*/MA_TRUE);\n}\n\nMA_API ma_result ma_hpf_init(const ma_hpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf* pHPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_hpf_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_hpf_init_preallocated(pConfig, pHeap, pHPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pHPF->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_hpf_uninit(ma_hpf* pHPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_uint32 ihpf1;\n    ma_uint32 ihpf2;\n\n    if (pHPF == NULL) {\n        return;\n    }\n\n    for (ihpf1 = 0; ihpf1 < pHPF->hpf1Count; ihpf1 += 1) {\n        ma_hpf1_uninit(&pHPF->pHPF1[ihpf1], pAllocationCallbacks);\n    }\n\n    for (ihpf2 = 0; ihpf2 < pHPF->hpf2Count; ihpf2 += 1) {\n        ma_hpf2_uninit(&pHPF->pHPF2[ihpf2], pAllocationCallbacks);\n    }\n\n    if (pHPF->_ownsHeap) {\n        ma_free(pHPF->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_hpf_reinit(const ma_hpf_config* pConfig, ma_hpf* pHPF)\n{\n    return ma_hpf_reinit__internal(pConfig, NULL, pHPF, /*isNew*/MA_FALSE);\n}\n\nMA_API ma_result ma_hpf_process_pcm_frames(ma_hpf* pHPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_result result;\n    ma_uint32 ihpf1;\n    ma_uint32 ihpf2;\n\n    if (pHPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Faster path for in-place. */\n    if (pFramesOut == pFramesIn) {\n        for (ihpf1 = 0; ihpf1 < pHPF->hpf1Count; ihpf1 += 1) {\n            result = ma_hpf1_process_pcm_frames(&pHPF->pHPF1[ihpf1], pFramesOut, pFramesOut, frameCount);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n\n        for (ihpf2 = 0; ihpf2 < pHPF->hpf2Count; ihpf2 += 1) {\n            result = ma_hpf2_process_pcm_frames(&pHPF->pHPF2[ihpf2], pFramesOut, pFramesOut, frameCount);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n    }\n\n    /* Slightly slower path for copying. */\n    if (pFramesOut != pFramesIn) {\n        ma_uint32 iFrame;\n\n        /*  */ if (pHPF->format == ma_format_f32) {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                MA_COPY_MEMORY(pFramesOutF32, pFramesInF32, ma_get_bytes_per_frame(pHPF->format, pHPF->channels));\n\n                for (ihpf1 = 0; ihpf1 < pHPF->hpf1Count; ihpf1 += 1) {\n                    ma_hpf1_process_pcm_frame_f32(&pHPF->pHPF1[ihpf1], pFramesOutF32, pFramesOutF32);\n                }\n\n                for (ihpf2 = 0; ihpf2 < pHPF->hpf2Count; ihpf2 += 1) {\n                    ma_hpf2_process_pcm_frame_f32(&pHPF->pHPF2[ihpf2], pFramesOutF32, pFramesOutF32);\n                }\n\n                pFramesOutF32 += pHPF->channels;\n                pFramesInF32  += pHPF->channels;\n            }\n        } else if (pHPF->format == ma_format_s16) {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                MA_COPY_MEMORY(pFramesOutS16, pFramesInS16, ma_get_bytes_per_frame(pHPF->format, pHPF->channels));\n\n                for (ihpf1 = 0; ihpf1 < pHPF->hpf1Count; ihpf1 += 1) {\n                    ma_hpf1_process_pcm_frame_s16(&pHPF->pHPF1[ihpf1], pFramesOutS16, pFramesOutS16);\n                }\n\n                for (ihpf2 = 0; ihpf2 < pHPF->hpf2Count; ihpf2 += 1) {\n                    ma_hpf2_process_pcm_frame_s16(&pHPF->pHPF2[ihpf2], pFramesOutS16, pFramesOutS16);\n                }\n\n                pFramesOutS16 += pHPF->channels;\n                pFramesInS16  += pHPF->channels;\n            }\n        } else {\n            MA_ASSERT(MA_FALSE);\n            return MA_INVALID_OPERATION;    /* Should never hit this. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_hpf_get_latency(const ma_hpf* pHPF)\n{\n    if (pHPF == NULL) {\n        return 0;\n    }\n\n    return pHPF->hpf2Count*2 + pHPF->hpf1Count;\n}\n\n\n/**************************************************************************************************************************************************************\n\nBand-Pass Filtering\n\n**************************************************************************************************************************************************************/\nMA_API ma_bpf2_config ma_bpf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, double q)\n{\n    ma_bpf2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.q = q;\n\n    /* Q cannot be 0 or else it'll result in a division by 0. In this case just default to 0.707107. */\n    if (config.q == 0) {\n        config.q = 0.707107;\n    }\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_bpf2__get_biquad_config(const ma_bpf2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double q;\n    double w;\n    double s;\n    double c;\n    double a;\n\n    MA_ASSERT(pConfig != NULL);\n\n    q = pConfig->q;\n    w = 2 * MA_PI_D * pConfig->cutoffFrequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    a = s / (2*q);\n\n    bqConfig.b0 =  q * a;\n    bqConfig.b1 =  0;\n    bqConfig.b2 = -q * a;\n    bqConfig.a0 =  1 + a;\n    bqConfig.a1 = -2 * c;\n    bqConfig.a2 =  1 - a;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_bpf2_get_heap_size(const ma_bpf2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_bpf2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_bpf2_init_preallocated(const ma_bpf2_config* pConfig, void* pHeap, ma_bpf2* pBPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pBPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pBPF);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_bpf2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pBPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_bpf2_init(const ma_bpf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf2* pBPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_bpf2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_bpf2_init_preallocated(pConfig, pHeap, pBPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pBPF->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_bpf2_uninit(ma_bpf2* pBPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pBPF == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pBPF->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_bpf2_reinit(const ma_bpf2_config* pConfig, ma_bpf2* pBPF)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pBPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_bpf2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pBPF->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_bpf2_process_pcm_frame_s16(ma_bpf2* pBPF, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pBPF->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_bpf2_process_pcm_frame_f32(ma_bpf2* pBPF, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pBPF->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_bpf2_process_pcm_frames(ma_bpf2* pBPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pBPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pBPF->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_bpf2_get_latency(const ma_bpf2* pBPF)\n{\n    if (pBPF == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pBPF->bq);\n}\n\n\nMA_API ma_bpf_config ma_bpf_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_bpf_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format          = format;\n    config.channels        = channels;\n    config.sampleRate      = sampleRate;\n    config.cutoffFrequency = cutoffFrequency;\n    config.order           = ma_min(order, MA_MAX_FILTER_ORDER);\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t bpf2Offset;\n} ma_bpf_heap_layout;\n\nstatic ma_result ma_bpf_get_heap_layout(const ma_bpf_config* pConfig, ma_bpf_heap_layout* pHeapLayout)\n{\n    ma_result result;\n    ma_uint32 bpf2Count;\n    ma_uint32 ibpf2;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We must have an even number of order. */\n    if ((pConfig->order & 0x1) != 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    bpf2Count = pConfig->order / 2;\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* BPF 2 */\n    pHeapLayout->bpf2Offset = pHeapLayout->sizeInBytes;\n    for (ibpf2 = 0; ibpf2 < bpf2Count; ibpf2 += 1) {\n        size_t bpf2HeapSizeInBytes;\n        ma_bpf2_config bpf2Config = ma_bpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, 0.707107);   /* <-- The \"q\" parameter does not matter for the purpose of calculating the heap size. */\n\n        result = ma_bpf2_get_heap_size(&bpf2Config, &bpf2HeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += sizeof(ma_bpf2) + bpf2HeapSizeInBytes;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_bpf_reinit__internal(const ma_bpf_config* pConfig, void* pHeap, ma_bpf* pBPF, ma_bool32 isNew)\n{\n    ma_result result;\n    ma_uint32 bpf2Count;\n    ma_uint32 ibpf2;\n    ma_bpf_heap_layout heapLayout;  /* Only used if isNew is true. */\n\n    if (pBPF == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only supporting f32 and s16. */\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The format cannot be changed after initialization. */\n    if (pBPF->format != ma_format_unknown && pBPF->format != pConfig->format) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The channel count cannot be changed after initialization. */\n    if (pBPF->channels != 0 && pBPF->channels != pConfig->channels) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pConfig->order > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We must have an even number of order. */\n    if ((pConfig->order & 0x1) != 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    bpf2Count = pConfig->order / 2;\n\n    /* The filter order can't change between reinits. */\n    if (!isNew) {\n        if (pBPF->bpf2Count != bpf2Count) {\n            return MA_INVALID_OPERATION;\n        }\n    }\n\n    if (isNew) {\n        result = ma_bpf_get_heap_layout(pConfig, &heapLayout);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pBPF->_pHeap = pHeap;\n        MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n        pBPF->pBPF2 = (ma_bpf2*)ma_offset_ptr(pHeap, heapLayout.bpf2Offset);\n    } else {\n        MA_ZERO_OBJECT(&heapLayout);\n    }\n\n    for (ibpf2 = 0; ibpf2 < bpf2Count; ibpf2 += 1) {\n        ma_bpf2_config bpf2Config;\n        double q;\n\n        /* TODO: Calculate Q to make this a proper Butterworth filter. */\n        q = 0.707107;\n\n        bpf2Config = ma_bpf2_config_init(pConfig->format, pConfig->channels, pConfig->sampleRate, pConfig->cutoffFrequency, q);\n\n        if (isNew) {\n            size_t bpf2HeapSizeInBytes;\n\n            result = ma_bpf2_get_heap_size(&bpf2Config, &bpf2HeapSizeInBytes);\n            if (result == MA_SUCCESS) {\n                result = ma_bpf2_init_preallocated(&bpf2Config, ma_offset_ptr(pHeap, heapLayout.bpf2Offset + (sizeof(ma_bpf2) * bpf2Count) + (ibpf2 * bpf2HeapSizeInBytes)), &pBPF->pBPF2[ibpf2]);\n            }\n        } else {\n            result = ma_bpf2_reinit(&bpf2Config, &pBPF->pBPF2[ibpf2]);\n        }\n\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    pBPF->bpf2Count = bpf2Count;\n    pBPF->format    = pConfig->format;\n    pBPF->channels  = pConfig->channels;\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_bpf_get_heap_size(const ma_bpf_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_bpf_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_bpf_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_bpf_init_preallocated(const ma_bpf_config* pConfig, void* pHeap, ma_bpf* pBPF)\n{\n    if (pBPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pBPF);\n\n    return ma_bpf_reinit__internal(pConfig, pHeap, pBPF, /*isNew*/MA_TRUE);\n}\n\nMA_API ma_result ma_bpf_init(const ma_bpf_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf* pBPF)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_bpf_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_bpf_init_preallocated(pConfig, pHeap, pBPF);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pBPF->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_bpf_uninit(ma_bpf* pBPF, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_uint32 ibpf2;\n\n    if (pBPF == NULL) {\n        return;\n    }\n\n    for (ibpf2 = 0; ibpf2 < pBPF->bpf2Count; ibpf2 += 1) {\n        ma_bpf2_uninit(&pBPF->pBPF2[ibpf2], pAllocationCallbacks);\n    }\n\n    if (pBPF->_ownsHeap) {\n        ma_free(pBPF->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_bpf_reinit(const ma_bpf_config* pConfig, ma_bpf* pBPF)\n{\n    return ma_bpf_reinit__internal(pConfig, NULL, pBPF, /*isNew*/MA_FALSE);\n}\n\nMA_API ma_result ma_bpf_process_pcm_frames(ma_bpf* pBPF, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_result result;\n    ma_uint32 ibpf2;\n\n    if (pBPF == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Faster path for in-place. */\n    if (pFramesOut == pFramesIn) {\n        for (ibpf2 = 0; ibpf2 < pBPF->bpf2Count; ibpf2 += 1) {\n            result = ma_bpf2_process_pcm_frames(&pBPF->pBPF2[ibpf2], pFramesOut, pFramesOut, frameCount);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n        }\n    }\n\n    /* Slightly slower path for copying. */\n    if (pFramesOut != pFramesIn) {\n        ma_uint32 iFrame;\n\n        /*  */ if (pBPF->format == ma_format_f32) {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                MA_COPY_MEMORY(pFramesOutF32, pFramesInF32, ma_get_bytes_per_frame(pBPF->format, pBPF->channels));\n\n                for (ibpf2 = 0; ibpf2 < pBPF->bpf2Count; ibpf2 += 1) {\n                    ma_bpf2_process_pcm_frame_f32(&pBPF->pBPF2[ibpf2], pFramesOutF32, pFramesOutF32);\n                }\n\n                pFramesOutF32 += pBPF->channels;\n                pFramesInF32  += pBPF->channels;\n            }\n        } else if (pBPF->format == ma_format_s16) {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                MA_COPY_MEMORY(pFramesOutS16, pFramesInS16, ma_get_bytes_per_frame(pBPF->format, pBPF->channels));\n\n                for (ibpf2 = 0; ibpf2 < pBPF->bpf2Count; ibpf2 += 1) {\n                    ma_bpf2_process_pcm_frame_s16(&pBPF->pBPF2[ibpf2], pFramesOutS16, pFramesOutS16);\n                }\n\n                pFramesOutS16 += pBPF->channels;\n                pFramesInS16  += pBPF->channels;\n            }\n        } else {\n            MA_ASSERT(MA_FALSE);\n            return MA_INVALID_OPERATION;    /* Should never hit this. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint32 ma_bpf_get_latency(const ma_bpf* pBPF)\n{\n    if (pBPF == NULL) {\n        return 0;\n    }\n\n    return pBPF->bpf2Count*2;\n}\n\n\n/**************************************************************************************************************************************************************\n\nNotching Filter\n\n**************************************************************************************************************************************************************/\nMA_API ma_notch2_config ma_notch2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double q, double frequency)\n{\n    ma_notch2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.q          = q;\n    config.frequency  = frequency;\n\n    if (config.q == 0) {\n        config.q = 0.707107;\n    }\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_notch2__get_biquad_config(const ma_notch2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double q;\n    double w;\n    double s;\n    double c;\n    double a;\n\n    MA_ASSERT(pConfig != NULL);\n\n    q = pConfig->q;\n    w = 2 * MA_PI_D * pConfig->frequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    a = s / (2*q);\n\n    bqConfig.b0 =  1;\n    bqConfig.b1 = -2 * c;\n    bqConfig.b2 =  1;\n    bqConfig.a0 =  1 + a;\n    bqConfig.a1 = -2 * c;\n    bqConfig.a2 =  1 - a;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_notch2_get_heap_size(const ma_notch2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_notch2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_notch2_init_preallocated(const ma_notch2_config* pConfig, void* pHeap, ma_notch2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFilter);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_notch2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_notch2_init(const ma_notch2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_notch2* pFilter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_notch2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_notch2_init_preallocated(pConfig, pHeap, pFilter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pFilter->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_notch2_uninit(ma_notch2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFilter == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pFilter->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_notch2_reinit(const ma_notch2_config* pConfig, ma_notch2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_notch2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_notch2_process_pcm_frame_s16(ma_notch2* pFilter, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_notch2_process_pcm_frame_f32(ma_notch2* pFilter, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_notch2_process_pcm_frames(ma_notch2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pFilter->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_notch2_get_latency(const ma_notch2* pFilter)\n{\n    if (pFilter == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pFilter->bq);\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nPeaking EQ Filter\n\n**************************************************************************************************************************************************************/\nMA_API ma_peak2_config ma_peak2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency)\n{\n    ma_peak2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.gainDB     = gainDB;\n    config.q          = q;\n    config.frequency  = frequency;\n\n    if (config.q == 0) {\n        config.q = 0.707107;\n    }\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_peak2__get_biquad_config(const ma_peak2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double q;\n    double w;\n    double s;\n    double c;\n    double a;\n    double A;\n\n    MA_ASSERT(pConfig != NULL);\n\n    q = pConfig->q;\n    w = 2 * MA_PI_D * pConfig->frequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    a = s / (2*q);\n    A = ma_powd(10, (pConfig->gainDB / 40));\n\n    bqConfig.b0 =  1 + (a * A);\n    bqConfig.b1 = -2 * c;\n    bqConfig.b2 =  1 - (a * A);\n    bqConfig.a0 =  1 + (a / A);\n    bqConfig.a1 = -2 * c;\n    bqConfig.a2 =  1 - (a / A);\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_peak2_get_heap_size(const ma_peak2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_peak2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_peak2_init_preallocated(const ma_peak2_config* pConfig, void* pHeap, ma_peak2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFilter);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_peak2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_peak2_init(const ma_peak2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_peak2* pFilter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_peak2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_peak2_init_preallocated(pConfig, pHeap, pFilter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pFilter->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_peak2_uninit(ma_peak2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFilter == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pFilter->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_peak2_reinit(const ma_peak2_config* pConfig, ma_peak2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_peak2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_peak2_process_pcm_frame_s16(ma_peak2* pFilter, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_peak2_process_pcm_frame_f32(ma_peak2* pFilter, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_peak2_process_pcm_frames(ma_peak2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pFilter->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_peak2_get_latency(const ma_peak2* pFilter)\n{\n    if (pFilter == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pFilter->bq);\n}\n\n\n/**************************************************************************************************************************************************************\n\nLow Shelf Filter\n\n**************************************************************************************************************************************************************/\nMA_API ma_loshelf2_config ma_loshelf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double shelfSlope, double frequency)\n{\n    ma_loshelf2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.gainDB     = gainDB;\n    config.shelfSlope = shelfSlope;\n    config.frequency  = frequency;\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_loshelf2__get_biquad_config(const ma_loshelf2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double w;\n    double s;\n    double c;\n    double A;\n    double S;\n    double a;\n    double sqrtA;\n\n    MA_ASSERT(pConfig != NULL);\n\n    w = 2 * MA_PI_D * pConfig->frequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    A = ma_powd(10, (pConfig->gainDB / 40));\n    S = pConfig->shelfSlope;\n    a = s/2 * ma_sqrtd((A + 1/A) * (1/S - 1) + 2);\n    sqrtA = 2*ma_sqrtd(A)*a;\n\n    bqConfig.b0 =  A * ((A + 1) - (A - 1)*c + sqrtA);\n    bqConfig.b1 =  2 * A * ((A - 1) - (A + 1)*c);\n    bqConfig.b2 =  A * ((A + 1) - (A - 1)*c - sqrtA);\n    bqConfig.a0 =  (A + 1) + (A - 1)*c + sqrtA;\n    bqConfig.a1 = -2 * ((A - 1) + (A + 1)*c);\n    bqConfig.a2 =  (A + 1) + (A - 1)*c - sqrtA;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_loshelf2_get_heap_size(const ma_loshelf2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_loshelf2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_loshelf2_init_preallocated(const ma_loshelf2_config* pConfig, void* pHeap, ma_loshelf2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFilter);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_loshelf2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_loshelf2_init(const ma_loshelf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_loshelf2* pFilter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_loshelf2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_loshelf2_init_preallocated(pConfig, pHeap, pFilter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pFilter->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_loshelf2_uninit(ma_loshelf2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFilter == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pFilter->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_loshelf2_reinit(const ma_loshelf2_config* pConfig, ma_loshelf2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_loshelf2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_loshelf2_process_pcm_frame_s16(ma_loshelf2* pFilter, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_loshelf2_process_pcm_frame_f32(ma_loshelf2* pFilter, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_loshelf2_process_pcm_frames(ma_loshelf2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pFilter->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_loshelf2_get_latency(const ma_loshelf2* pFilter)\n{\n    if (pFilter == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pFilter->bq);\n}\n\n\n/**************************************************************************************************************************************************************\n\nHigh Shelf Filter\n\n**************************************************************************************************************************************************************/\nMA_API ma_hishelf2_config ma_hishelf2_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double shelfSlope, double frequency)\n{\n    ma_hishelf2_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.gainDB     = gainDB;\n    config.shelfSlope = shelfSlope;\n    config.frequency  = frequency;\n\n    return config;\n}\n\n\nstatic MA_INLINE ma_biquad_config ma_hishelf2__get_biquad_config(const ma_hishelf2_config* pConfig)\n{\n    ma_biquad_config bqConfig;\n    double w;\n    double s;\n    double c;\n    double A;\n    double S;\n    double a;\n    double sqrtA;\n\n    MA_ASSERT(pConfig != NULL);\n\n    w = 2 * MA_PI_D * pConfig->frequency / pConfig->sampleRate;\n    s = ma_sind(w);\n    c = ma_cosd(w);\n    A = ma_powd(10, (pConfig->gainDB / 40));\n    S = pConfig->shelfSlope;\n    a = s/2 * ma_sqrtd((A + 1/A) * (1/S - 1) + 2);\n    sqrtA = 2*ma_sqrtd(A)*a;\n\n    bqConfig.b0 =  A * ((A + 1) + (A - 1)*c + sqrtA);\n    bqConfig.b1 = -2 * A * ((A - 1) + (A + 1)*c);\n    bqConfig.b2 =  A * ((A + 1) + (A - 1)*c - sqrtA);\n    bqConfig.a0 =  (A + 1) - (A - 1)*c + sqrtA;\n    bqConfig.a1 =  2 * ((A - 1) - (A + 1)*c);\n    bqConfig.a2 =  (A + 1) - (A - 1)*c - sqrtA;\n\n    bqConfig.format   = pConfig->format;\n    bqConfig.channels = pConfig->channels;\n\n    return bqConfig;\n}\n\nMA_API ma_result ma_hishelf2_get_heap_size(const ma_hishelf2_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_biquad_config bqConfig;\n    bqConfig = ma_hishelf2__get_biquad_config(pConfig);\n\n    return ma_biquad_get_heap_size(&bqConfig, pHeapSizeInBytes);\n}\n\nMA_API ma_result ma_hishelf2_init_preallocated(const ma_hishelf2_config* pConfig, void* pHeap, ma_hishelf2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFilter);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_hishelf2__get_biquad_config(pConfig);\n    result = ma_biquad_init_preallocated(&bqConfig, pHeap, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_hishelf2_init(const ma_hishelf2_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hishelf2* pFilter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_hishelf2_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_hishelf2_init_preallocated(pConfig, pHeap, pFilter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pFilter->bq._ownsHeap = MA_TRUE;    /* <-- This will cause the biquad to take ownership of the heap and free it when it's uninitialized. */\n    return MA_SUCCESS;\n}\n\nMA_API void ma_hishelf2_uninit(ma_hishelf2* pFilter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFilter == NULL) {\n        return;\n    }\n\n    ma_biquad_uninit(&pFilter->bq, pAllocationCallbacks);   /* <-- This will free the heap allocation. */\n}\n\nMA_API ma_result ma_hishelf2_reinit(const ma_hishelf2_config* pConfig, ma_hishelf2* pFilter)\n{\n    ma_result result;\n    ma_biquad_config bqConfig;\n\n    if (pFilter == NULL || pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    bqConfig = ma_hishelf2__get_biquad_config(pConfig);\n    result = ma_biquad_reinit(&bqConfig, &pFilter->bq);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic MA_INLINE void ma_hishelf2_process_pcm_frame_s16(ma_hishelf2* pFilter, ma_int16* pFrameOut, const ma_int16* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_s16(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nstatic MA_INLINE void ma_hishelf2_process_pcm_frame_f32(ma_hishelf2* pFilter, float* pFrameOut, const float* pFrameIn)\n{\n    ma_biquad_process_pcm_frame_f32(&pFilter->bq, pFrameOut, pFrameIn);\n}\n\nMA_API ma_result ma_hishelf2_process_pcm_frames(ma_hishelf2* pFilter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pFilter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_biquad_process_pcm_frames(&pFilter->bq, pFramesOut, pFramesIn, frameCount);\n}\n\nMA_API ma_uint32 ma_hishelf2_get_latency(const ma_hishelf2* pFilter)\n{\n    if (pFilter == NULL) {\n        return 0;\n    }\n\n    return ma_biquad_get_latency(&pFilter->bq);\n}\n\n\n\n/*\nDelay\n*/\nMA_API ma_delay_config ma_delay_config_init(ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 delayInFrames, float decay)\n{\n    ma_delay_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.channels      = channels;\n    config.sampleRate    = sampleRate;\n    config.delayInFrames = delayInFrames;\n    config.delayStart    = (decay == 0) ? MA_TRUE : MA_FALSE;   /* Delay the start if it looks like we're not configuring an echo. */\n    config.wet           = 1;\n    config.dry           = 1;\n    config.decay         = decay;\n\n    return config;\n}\n\n\nMA_API ma_result ma_delay_init(const ma_delay_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_delay* pDelay)\n{\n    if (pDelay == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDelay);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->decay < 0 || pConfig->decay > 1) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDelay->config             = *pConfig;\n    pDelay->bufferSizeInFrames = pConfig->delayInFrames;\n    pDelay->cursor             = 0;\n\n    pDelay->pBuffer = (float*)ma_malloc((size_t)(pDelay->bufferSizeInFrames * ma_get_bytes_per_frame(ma_format_f32, pConfig->channels)), pAllocationCallbacks);\n    if (pDelay->pBuffer == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    ma_silence_pcm_frames(pDelay->pBuffer, pDelay->bufferSizeInFrames, ma_format_f32, pConfig->channels);\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_delay_uninit(ma_delay* pDelay, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pDelay == NULL) {\n        return;\n    }\n\n    ma_free(pDelay->pBuffer, pAllocationCallbacks);\n}\n\nMA_API ma_result ma_delay_process_pcm_frames(ma_delay* pDelay, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount)\n{\n    ma_uint32 iFrame;\n    ma_uint32 iChannel;\n    float* pFramesOutF32 = (float*)pFramesOut;\n    const float* pFramesInF32 = (const float*)pFramesIn;\n\n    if (pDelay == NULL || pFramesOut == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannel = 0; iChannel < pDelay->config.channels; iChannel += 1) {\n            ma_uint32 iBuffer = (pDelay->cursor * pDelay->config.channels) + iChannel;\n\n            if (pDelay->config.delayStart) {\n                /* Delayed start. */\n\n                /* Read */\n                pFramesOutF32[iChannel] = pDelay->pBuffer[iBuffer] * pDelay->config.wet;\n\n                /* Feedback */\n                pDelay->pBuffer[iBuffer] = (pDelay->pBuffer[iBuffer] * pDelay->config.decay) + (pFramesInF32[iChannel] * pDelay->config.dry);\n            } else {\n                /* Immediate start */\n\n                /* Feedback */\n                pDelay->pBuffer[iBuffer] = (pDelay->pBuffer[iBuffer] * pDelay->config.decay) + (pFramesInF32[iChannel] * pDelay->config.dry);\n\n                /* Read */\n                pFramesOutF32[iChannel] = pDelay->pBuffer[iBuffer] * pDelay->config.wet;\n            }\n        }\n\n        pDelay->cursor = (pDelay->cursor + 1) % pDelay->bufferSizeInFrames;\n\n        pFramesOutF32 += pDelay->config.channels;\n        pFramesInF32  += pDelay->config.channels;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_delay_set_wet(ma_delay* pDelay, float value)\n{\n    if (pDelay == NULL) {\n        return;\n    }\n\n    pDelay->config.wet = value;\n}\n\nMA_API float ma_delay_get_wet(const ma_delay* pDelay)\n{\n    if (pDelay == NULL) {\n        return 0;\n    }\n\n    return pDelay->config.wet;\n}\n\nMA_API void ma_delay_set_dry(ma_delay* pDelay, float value)\n{\n    if (pDelay == NULL) {\n        return;\n    }\n\n    pDelay->config.dry = value;\n}\n\nMA_API float ma_delay_get_dry(const ma_delay* pDelay)\n{\n    if (pDelay == NULL) {\n        return 0;\n    }\n\n    return pDelay->config.dry;\n}\n\nMA_API void ma_delay_set_decay(ma_delay* pDelay, float value)\n{\n    if (pDelay == NULL) {\n        return;\n    }\n\n    pDelay->config.decay = value;\n}\n\nMA_API float ma_delay_get_decay(const ma_delay* pDelay)\n{\n    if (pDelay == NULL) {\n        return 0;\n    }\n\n    return pDelay->config.decay;\n}\n\n\nMA_API ma_gainer_config ma_gainer_config_init(ma_uint32 channels, ma_uint32 smoothTimeInFrames)\n{\n    ma_gainer_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.channels           = channels;\n    config.smoothTimeInFrames = smoothTimeInFrames;\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t oldGainsOffset;\n    size_t newGainsOffset;\n} ma_gainer_heap_layout;\n\nstatic ma_result ma_gainer_get_heap_layout(const ma_gainer_config* pConfig, ma_gainer_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Old gains. */\n    pHeapLayout->oldGainsOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(float) * pConfig->channels;\n\n    /* New gains. */\n    pHeapLayout->newGainsOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += sizeof(float) * pConfig->channels;\n\n    /* Alignment. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_gainer_get_heap_size(const ma_gainer_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_gainer_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_gainer_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_gainer_init_preallocated(const ma_gainer_config* pConfig, void* pHeap, ma_gainer* pGainer)\n{\n    ma_result result;\n    ma_gainer_heap_layout heapLayout;\n    ma_uint32 iChannel;\n\n    if (pGainer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pGainer);\n\n    if (pConfig == NULL || pHeap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_gainer_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pGainer->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pGainer->pOldGains = (float*)ma_offset_ptr(pHeap, heapLayout.oldGainsOffset);\n    pGainer->pNewGains = (float*)ma_offset_ptr(pHeap, heapLayout.newGainsOffset);\n    pGainer->masterVolume = 1;\n\n    pGainer->config = *pConfig;\n    pGainer->t      = (ma_uint32)-1;  /* No interpolation by default. */\n\n    for (iChannel = 0; iChannel < pConfig->channels; iChannel += 1) {\n        pGainer->pOldGains[iChannel] = 1;\n        pGainer->pNewGains[iChannel] = 1;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_gainer_init(const ma_gainer_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_gainer* pGainer)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_gainer_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the size of the heap allocation. */\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_gainer_init_preallocated(pConfig, pHeap, pGainer);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pGainer->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_gainer_uninit(ma_gainer* pGainer, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pGainer == NULL) {\n        return;\n    }\n\n    if (pGainer->_ownsHeap) {\n        ma_free(pGainer->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic float ma_gainer_calculate_current_gain(const ma_gainer* pGainer, ma_uint32 channel)\n{\n    float a = (float)pGainer->t / pGainer->config.smoothTimeInFrames;\n    return ma_mix_f32_fast(pGainer->pOldGains[channel], pGainer->pNewGains[channel], a);\n}\n\nstatic /*__attribute__((noinline))*/ ma_result ma_gainer_process_pcm_frames_internal(ma_gainer * pGainer, void* MA_RESTRICT pFramesOut, const void* MA_RESTRICT pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannel;\n    ma_uint64 interpolatedFrameCount;\n\n    MA_ASSERT(pGainer != NULL);\n\n    /*\n    We don't necessarily need to apply a linear interpolation for the entire frameCount frames. When\n    linear interpolation is not needed we can do a simple volume adjustment which will be more\n    efficient than a lerp with an alpha value of 1.\n\n    To do this, all we need to do is determine how many frames need to have a lerp applied. Then we\n    just process that number of frames with linear interpolation. After that we run on an optimized\n    path which just applies the new gains without a lerp.\n    */\n    if (pGainer->t >= pGainer->config.smoothTimeInFrames) {\n        interpolatedFrameCount = 0;\n    } else {\n        interpolatedFrameCount = pGainer->t - pGainer->config.smoothTimeInFrames;\n        if (interpolatedFrameCount > frameCount) {\n            interpolatedFrameCount = frameCount;\n        }\n    }\n\n    /*\n    Start off with our interpolated frames. When we do this, we'll adjust frameCount and our pointers\n    so that the fast path can work naturally without consideration of the interpolated path.\n    */\n    if (interpolatedFrameCount > 0) {\n        /* We can allow the input and output buffers to be null in which case we'll just update the internal timer. */\n        if (pFramesOut != NULL && pFramesIn != NULL) {\n            /*\n            All we're really doing here is moving the old gains towards the new gains. We don't want to\n            be modifying the gains inside the ma_gainer object because that will break things. Instead\n            we can make a copy here on the stack. For extreme channel counts we can fall back to a slower\n            implementation which just uses a standard lerp.\n            */\n            float* pFramesOutF32 = (float*)pFramesOut;\n            const float* pFramesInF32 = (const float*)pFramesIn;\n            float a = (float)pGainer->t / pGainer->config.smoothTimeInFrames;\n            float d = 1.0f / pGainer->config.smoothTimeInFrames;\n\n            if (pGainer->config.channels <= 32) {\n                float pRunningGain[32];\n                float pRunningGainDelta[32];    /* Could this be heap-allocated as part of the ma_gainer object? */\n\n                /* Initialize the running gain. */\n                for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                    float t = (pGainer->pNewGains[iChannel] - pGainer->pOldGains[iChannel]) * pGainer->masterVolume;\n                    pRunningGainDelta[iChannel] = t * d;\n                    pRunningGain[iChannel] = (pGainer->pOldGains[iChannel] * pGainer->masterVolume) + (t * a);\n                }\n\n                iFrame = 0;\n\n                /* Optimized paths for common channel counts. This is mostly just experimenting with some SIMD ideas. It's not necessarily final. */\n                if (pGainer->config.channels == 2) {\n                #if defined(MA_SUPPORT_SSE2)\n                    if (ma_has_sse2()) {\n                        ma_uint64 unrolledLoopCount = interpolatedFrameCount >> 1;\n\n                        /* Expand some arrays so we can have a clean SIMD loop below. */\n                        __m128 runningGainDelta0 = _mm_set_ps(pRunningGainDelta[1], pRunningGainDelta[0], pRunningGainDelta[1], pRunningGainDelta[0]);\n                        __m128 runningGain0      = _mm_set_ps(pRunningGain[1] + pRunningGainDelta[1], pRunningGain[0] + pRunningGainDelta[0], pRunningGain[1], pRunningGain[0]);\n\n                        for (; iFrame < unrolledLoopCount; iFrame += 1) {\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*4 + 0], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*4 + 0]), runningGain0));\n                            runningGain0 = _mm_add_ps(runningGain0, runningGainDelta0);\n                        }\n\n                        iFrame = unrolledLoopCount << 1;\n                    } else\n                #endif\n                    {\n                        /*\n                        Two different scalar implementations here. Clang (and I assume GCC) will vectorize\n                        both of these, but the bottom version results in a nicer vectorization with less\n                        instructions emitted. The problem, however, is that the bottom version runs slower\n                        when compiled with MSVC. The top version will be partially vectorized by MSVC.\n                        */\n                    #if defined(_MSC_VER) && !defined(__clang__)\n                        ma_uint64 unrolledLoopCount = interpolatedFrameCount >> 1;\n\n                        /* Expand some arrays so we can have a clean 4x SIMD operation in the loop. */\n                        pRunningGainDelta[2] = pRunningGainDelta[0];\n                        pRunningGainDelta[3] = pRunningGainDelta[1];\n                        pRunningGain[2] = pRunningGain[0] + pRunningGainDelta[0];\n                        pRunningGain[3] = pRunningGain[1] + pRunningGainDelta[1];\n\n                        for (; iFrame < unrolledLoopCount; iFrame += 1) {\n                            pFramesOutF32[iFrame*4 + 0] = pFramesInF32[iFrame*4 + 0] * pRunningGain[0];\n                            pFramesOutF32[iFrame*4 + 1] = pFramesInF32[iFrame*4 + 1] * pRunningGain[1];\n                            pFramesOutF32[iFrame*4 + 2] = pFramesInF32[iFrame*4 + 2] * pRunningGain[2];\n                            pFramesOutF32[iFrame*4 + 3] = pFramesInF32[iFrame*4 + 3] * pRunningGain[3];\n\n                            /* Move the running gain forward towards the new gain. */\n                            pRunningGain[0] += pRunningGainDelta[0];\n                            pRunningGain[1] += pRunningGainDelta[1];\n                            pRunningGain[2] += pRunningGainDelta[2];\n                            pRunningGain[3] += pRunningGainDelta[3];\n                        }\n\n                        iFrame = unrolledLoopCount << 1;\n                    #else\n                        for (; iFrame < interpolatedFrameCount; iFrame += 1) {\n                            for (iChannel = 0; iChannel < 2; iChannel += 1) {\n                                pFramesOutF32[iFrame*2 + iChannel] = pFramesInF32[iFrame*2 + iChannel] * pRunningGain[iChannel];\n                            }\n\n                            for (iChannel = 0; iChannel < 2; iChannel += 1) {\n                                pRunningGain[iChannel] += pRunningGainDelta[iChannel];\n                            }\n                        }\n                    #endif\n                    }\n                } else if (pGainer->config.channels == 6) {\n                #if defined(MA_SUPPORT_SSE2)\n                    if (ma_has_sse2()) {\n                        /*\n                        For 6 channels things are a bit more complicated because 6 isn't cleanly divisible by 4. We need to do 2 frames\n                        at a time, meaning we'll be doing 12 samples in a group. Like the stereo case we'll need to expand some arrays\n                        so we can do clean 4x SIMD operations.\n                        */\n                        ma_uint64 unrolledLoopCount = interpolatedFrameCount >> 1;\n\n                        /* Expand some arrays so we can have a clean SIMD loop below. */\n                        __m128 runningGainDelta0 = _mm_set_ps(pRunningGainDelta[3], pRunningGainDelta[2], pRunningGainDelta[1], pRunningGainDelta[0]);\n                        __m128 runningGainDelta1 = _mm_set_ps(pRunningGainDelta[1], pRunningGainDelta[0], pRunningGainDelta[5], pRunningGainDelta[4]);\n                        __m128 runningGainDelta2 = _mm_set_ps(pRunningGainDelta[5], pRunningGainDelta[4], pRunningGainDelta[3], pRunningGainDelta[2]);\n\n                        __m128 runningGain0      = _mm_set_ps(pRunningGain[3],                        pRunningGain[2],                        pRunningGain[1],                        pRunningGain[0]);\n                        __m128 runningGain1      = _mm_set_ps(pRunningGain[1] + pRunningGainDelta[1], pRunningGain[0] + pRunningGainDelta[0], pRunningGain[5],                        pRunningGain[4]);\n                        __m128 runningGain2      = _mm_set_ps(pRunningGain[5] + pRunningGainDelta[5], pRunningGain[4] + pRunningGainDelta[4], pRunningGain[3] + pRunningGainDelta[3], pRunningGain[2] + pRunningGainDelta[2]);\n\n                        for (; iFrame < unrolledLoopCount; iFrame += 1) {\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*12 + 0], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*12 + 0]), runningGain0));\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*12 + 4], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*12 + 4]), runningGain1));\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*12 + 8], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*12 + 8]), runningGain2));\n\n                            runningGain0 = _mm_add_ps(runningGain0, runningGainDelta0);\n                            runningGain1 = _mm_add_ps(runningGain1, runningGainDelta1);\n                            runningGain2 = _mm_add_ps(runningGain2, runningGainDelta2);\n                        }\n\n                        iFrame = unrolledLoopCount << 1;\n                    } else\n                #endif\n                    {\n                        for (; iFrame < interpolatedFrameCount; iFrame += 1) {\n                            for (iChannel = 0; iChannel < 6; iChannel += 1) {\n                                pFramesOutF32[iFrame*6 + iChannel] = pFramesInF32[iFrame*6 + iChannel] * pRunningGain[iChannel];\n                            }\n\n                            /* Move the running gain forward towards the new gain. */\n                            for (iChannel = 0; iChannel < 6; iChannel += 1) {\n                                pRunningGain[iChannel] += pRunningGainDelta[iChannel];\n                            }\n                        }\n                    }\n                } else if (pGainer->config.channels == 8) {\n                    /* For 8 channels we can just go over frame by frame and do all eight channels as 2 separate 4x SIMD operations. */\n                #if defined(MA_SUPPORT_SSE2)\n                    if (ma_has_sse2()) {\n                        __m128 runningGainDelta0 = _mm_loadu_ps(&pRunningGainDelta[0]);\n                        __m128 runningGainDelta1 = _mm_loadu_ps(&pRunningGainDelta[4]);\n                        __m128 runningGain0      = _mm_loadu_ps(&pRunningGain[0]);\n                        __m128 runningGain1      = _mm_loadu_ps(&pRunningGain[4]);\n\n                        for (; iFrame < interpolatedFrameCount; iFrame += 1) {\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*8 + 0], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*8 + 0]), runningGain0));\n                            _mm_storeu_ps(&pFramesOutF32[iFrame*8 + 4], _mm_mul_ps(_mm_loadu_ps(&pFramesInF32[iFrame*8 + 4]), runningGain1));\n\n                            runningGain0 = _mm_add_ps(runningGain0, runningGainDelta0);\n                            runningGain1 = _mm_add_ps(runningGain1, runningGainDelta1);\n                        }\n                    } else\n                #endif\n                    {\n                        /* This is crafted so that it auto-vectorizes when compiled with Clang. */\n                        for (; iFrame < interpolatedFrameCount; iFrame += 1) {\n                            for (iChannel = 0; iChannel < 8; iChannel += 1) {\n                                pFramesOutF32[iFrame*8 + iChannel] = pFramesInF32[iFrame*8 + iChannel] * pRunningGain[iChannel];\n                            }\n\n                            /* Move the running gain forward towards the new gain. */\n                            for (iChannel = 0; iChannel < 8; iChannel += 1) {\n                                pRunningGain[iChannel] += pRunningGainDelta[iChannel];\n                            }\n                        }\n                    }\n                }\n\n                for (; iFrame < interpolatedFrameCount; iFrame += 1) {\n                    for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                        pFramesOutF32[iFrame*pGainer->config.channels + iChannel] = pFramesInF32[iFrame*pGainer->config.channels + iChannel] * pRunningGain[iChannel];\n                        pRunningGain[iChannel] += pRunningGainDelta[iChannel];\n                    }\n                }\n            } else {\n                /* Slower path for extreme channel counts where we can't fit enough on the stack. We could also move this to the heap as part of the ma_gainer object which might even be better since it'll only be updated when the gains actually change. */\n                for (iFrame = 0; iFrame < interpolatedFrameCount; iFrame += 1) {\n                    for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                        pFramesOutF32[iFrame*pGainer->config.channels + iChannel] = pFramesInF32[iFrame*pGainer->config.channels + iChannel] * ma_mix_f32_fast(pGainer->pOldGains[iChannel], pGainer->pNewGains[iChannel], a) * pGainer->masterVolume;\n                    }\n\n                    a += d;\n                }\n            }\n\n            pFramesOut = ma_offset_ptr(pFramesOut, interpolatedFrameCount * sizeof(float));\n            pFramesIn  = ma_offset_ptr(pFramesIn,  interpolatedFrameCount * sizeof(float));\n        }\n\n        frameCount -= interpolatedFrameCount;\n\n        /* Make sure the timer is updated. */\n        pGainer->t = (ma_uint32)ma_min(pGainer->t + interpolatedFrameCount, pGainer->config.smoothTimeInFrames);\n    }\n\n    /* All we need to do here is apply the new gains using an optimized path. */\n    if (pFramesOut != NULL && pFramesIn != NULL) {\n        if (pGainer->config.channels <= 32) {\n            float gains[32];\n            for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                gains[iChannel] = pGainer->pNewGains[iChannel] * pGainer->masterVolume;\n            }\n\n            ma_copy_and_apply_volume_factor_per_channel_f32((float*)pFramesOut, (const float*)pFramesIn, frameCount, pGainer->config.channels, gains);\n        } else {\n            /* Slow path. Too many channels to fit on the stack. Need to apply a master volume as a separate path. */\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                    ((float*)pFramesOut)[iFrame*pGainer->config.channels + iChannel] = ((const float*)pFramesIn)[iFrame*pGainer->config.channels + iChannel] * pGainer->pNewGains[iChannel] * pGainer->masterVolume;\n                }\n            }\n        }\n    }\n\n    /* Now that some frames have been processed we need to make sure future changes to the gain are interpolated. */\n    if (pGainer->t == (ma_uint32)-1) {\n        pGainer->t  = (ma_uint32)ma_min(pGainer->config.smoothTimeInFrames, frameCount);\n    }\n\n#if 0\n    if (pGainer->t >= pGainer->config.smoothTimeInFrames) {\n        /* Fast path. No gain calculation required. */\n        ma_copy_and_apply_volume_factor_per_channel_f32(pFramesOutF32, pFramesInF32, frameCount, pGainer->config.channels, pGainer->pNewGains);\n        ma_apply_volume_factor_f32(pFramesOutF32, frameCount * pGainer->config.channels, pGainer->masterVolume);\n\n        /* Now that some frames have been processed we need to make sure future changes to the gain are interpolated. */\n        if (pGainer->t == (ma_uint32)-1) {\n            pGainer->t = pGainer->config.smoothTimeInFrames;\n        }\n    } else {\n        /* Slow path. Need to interpolate the gain for each channel individually. */\n\n        /* We can allow the input and output buffers to be null in which case we'll just update the internal timer. */\n        if (pFramesOut != NULL && pFramesIn != NULL) {\n            float a = (float)pGainer->t / pGainer->config.smoothTimeInFrames;\n            float d = 1.0f / pGainer->config.smoothTimeInFrames;\n            ma_uint32 channelCount = pGainer->config.channels;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channelCount; iChannel += 1) {\n                    pFramesOutF32[iChannel] = pFramesInF32[iChannel] * ma_mix_f32_fast(pGainer->pOldGains[iChannel], pGainer->pNewGains[iChannel], a) * pGainer->masterVolume;\n                }\n\n                pFramesOutF32 += channelCount;\n                pFramesInF32  += channelCount;\n\n                a += d;\n                if (a > 1) {\n                    a = 1;\n                }\n            }\n        }\n\n        pGainer->t = (ma_uint32)ma_min(pGainer->t + frameCount, pGainer->config.smoothTimeInFrames);\n\n    #if 0   /* Reference implementation. */\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            /* We can allow the input and output buffers to be null in which case we'll just update the internal timer. */\n            if (pFramesOut != NULL && pFramesIn != NULL) {\n                for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n                    pFramesOutF32[iFrame * pGainer->config.channels + iChannel] = pFramesInF32[iFrame * pGainer->config.channels + iChannel] * ma_gainer_calculate_current_gain(pGainer, iChannel) * pGainer->masterVolume;\n                }\n            }\n\n            /* Move interpolation time forward, but don't go beyond our smoothing time. */\n            pGainer->t = ma_min(pGainer->t + 1, pGainer->config.smoothTimeInFrames);\n        }\n    #endif\n    }\n#endif\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_gainer_process_pcm_frames(ma_gainer* pGainer, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pGainer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    ma_gainer_process_pcm_frames_internal() marks pFramesOut and pFramesIn with MA_RESTRICT which\n    helps with auto-vectorization.\n    */\n    return ma_gainer_process_pcm_frames_internal(pGainer, pFramesOut, pFramesIn, frameCount);\n}\n\nstatic void ma_gainer_set_gain_by_index(ma_gainer* pGainer, float newGain, ma_uint32 iChannel)\n{\n    pGainer->pOldGains[iChannel] = ma_gainer_calculate_current_gain(pGainer, iChannel);\n    pGainer->pNewGains[iChannel] = newGain;\n}\n\nstatic void ma_gainer_reset_smoothing_time(ma_gainer* pGainer)\n{\n    if (pGainer->t == (ma_uint32)-1) {\n        pGainer->t = pGainer->config.smoothTimeInFrames;    /* No smoothing required for initial gains setting. */\n    } else {\n        pGainer->t = 0;\n    }\n}\n\nMA_API ma_result ma_gainer_set_gain(ma_gainer* pGainer, float newGain)\n{\n    ma_uint32 iChannel;\n\n    if (pGainer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n        ma_gainer_set_gain_by_index(pGainer, newGain, iChannel);\n    }\n\n    /* The smoothing time needs to be reset to ensure we always interpolate by the configured smoothing time, but only if it's not the first setting. */\n    ma_gainer_reset_smoothing_time(pGainer);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_gainer_set_gains(ma_gainer* pGainer, float* pNewGains)\n{\n    ma_uint32 iChannel;\n\n    if (pGainer == NULL || pNewGains == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iChannel = 0; iChannel < pGainer->config.channels; iChannel += 1) {\n        ma_gainer_set_gain_by_index(pGainer, pNewGains[iChannel], iChannel);\n    }\n\n    /* The smoothing time needs to be reset to ensure we always interpolate by the configured smoothing time, but only if it's not the first setting. */\n    ma_gainer_reset_smoothing_time(pGainer);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_gainer_set_master_volume(ma_gainer* pGainer, float volume)\n{\n    if (pGainer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pGainer->masterVolume = volume;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_gainer_get_master_volume(const ma_gainer* pGainer, float* pVolume)\n{\n    if (pGainer == NULL || pVolume == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pVolume = pGainer->masterVolume;\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_panner_config ma_panner_config_init(ma_format format, ma_uint32 channels)\n{\n    ma_panner_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format   = format;\n    config.channels = channels;\n    config.mode     = ma_pan_mode_balance;  /* Set to balancing mode by default because it's consistent with other audio engines and most likely what the caller is expecting. */\n    config.pan      = 0;\n\n    return config;\n}\n\n\nMA_API ma_result ma_panner_init(const ma_panner_config* pConfig, ma_panner* pPanner)\n{\n    if (pPanner == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pPanner);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pPanner->format   = pConfig->format;\n    pPanner->channels = pConfig->channels;\n    pPanner->mode     = pConfig->mode;\n    pPanner->pan      = pConfig->pan;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_stereo_balance_pcm_frames_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, float pan)\n{\n    ma_uint64 iFrame;\n\n    if (pan > 0) {\n        float factor = 1.0f - pan;\n        if (pFramesOut == pFramesIn) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                pFramesOut[iFrame*2 + 0] = pFramesIn[iFrame*2 + 0] * factor;\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                pFramesOut[iFrame*2 + 0] = pFramesIn[iFrame*2 + 0] * factor;\n                pFramesOut[iFrame*2 + 1] = pFramesIn[iFrame*2 + 1];\n            }\n        }\n    } else {\n        float factor = 1.0f + pan;\n        if (pFramesOut == pFramesIn) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                pFramesOut[iFrame*2 + 1] = pFramesIn[iFrame*2 + 1] * factor;\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                pFramesOut[iFrame*2 + 0] = pFramesIn[iFrame*2 + 0];\n                pFramesOut[iFrame*2 + 1] = pFramesIn[iFrame*2 + 1] * factor;\n            }\n        }\n    }\n}\n\nstatic void ma_stereo_balance_pcm_frames(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_format format, float pan)\n{\n    if (pan == 0) {\n        /* Fast path. No panning required. */\n        if (pFramesOut == pFramesIn) {\n            /* No-op */\n        } else {\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, format, 2);\n        }\n\n        return;\n    }\n\n    switch (format) {\n        case ma_format_f32: ma_stereo_balance_pcm_frames_f32((float*)pFramesOut, (float*)pFramesIn, frameCount, pan); break;\n\n        /* Unknown format. Just copy. */\n        default:\n        {\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, format, 2);\n        } break;\n    }\n}\n\n\nstatic void ma_stereo_pan_pcm_frames_f32(float* pFramesOut, const float* pFramesIn, ma_uint64 frameCount, float pan)\n{\n    ma_uint64 iFrame;\n\n    if (pan > 0) {\n        float factorL0 = 1.0f - pan;\n        float factorL1 = 0.0f + pan;\n\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float sample0 = (pFramesIn[iFrame*2 + 0] * factorL0);\n            float sample1 = (pFramesIn[iFrame*2 + 0] * factorL1) + pFramesIn[iFrame*2 + 1];\n\n            pFramesOut[iFrame*2 + 0] = sample0;\n            pFramesOut[iFrame*2 + 1] = sample1;\n        }\n    } else {\n        float factorR0 = 0.0f - pan;\n        float factorR1 = 1.0f + pan;\n\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float sample0 = pFramesIn[iFrame*2 + 0] + (pFramesIn[iFrame*2 + 1] * factorR0);\n            float sample1 =                           (pFramesIn[iFrame*2 + 1] * factorR1);\n\n            pFramesOut[iFrame*2 + 0] = sample0;\n            pFramesOut[iFrame*2 + 1] = sample1;\n        }\n    }\n}\n\nstatic void ma_stereo_pan_pcm_frames(void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount, ma_format format, float pan)\n{\n    if (pan == 0) {\n        /* Fast path. No panning required. */\n        if (pFramesOut == pFramesIn) {\n            /* No-op */\n        } else {\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, format, 2);\n        }\n\n        return;\n    }\n\n    switch (format) {\n        case ma_format_f32: ma_stereo_pan_pcm_frames_f32((float*)pFramesOut, (float*)pFramesIn, frameCount, pan); break;\n\n        /* Unknown format. Just copy. */\n        default:\n        {\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, format, 2);\n        } break;\n    }\n}\n\nMA_API ma_result ma_panner_process_pcm_frames(ma_panner* pPanner, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pPanner == NULL || pFramesOut == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pPanner->channels == 2) {\n        /* Stereo case. For now assume channel 0 is left and channel right is 1, but should probably add support for a channel map. */\n        if (pPanner->mode == ma_pan_mode_balance) {\n            ma_stereo_balance_pcm_frames(pFramesOut, pFramesIn, frameCount, pPanner->format, pPanner->pan);\n        } else {\n            ma_stereo_pan_pcm_frames(pFramesOut, pFramesIn, frameCount, pPanner->format, pPanner->pan);\n        }\n    } else {\n        if (pPanner->channels == 1) {\n            /* Panning has no effect on mono streams. */\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, pPanner->format, pPanner->channels);\n        } else {\n            /* For now we're not going to support non-stereo set ups. Not sure how I want to handle this case just yet. */\n            ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, pPanner->format, pPanner->channels);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_panner_set_mode(ma_panner* pPanner, ma_pan_mode mode)\n{\n    if (pPanner == NULL) {\n        return;\n    }\n\n    pPanner->mode = mode;\n}\n\nMA_API ma_pan_mode ma_panner_get_mode(const ma_panner* pPanner)\n{\n    if (pPanner == NULL) {\n        return ma_pan_mode_balance;\n    }\n\n    return pPanner->mode;\n}\n\nMA_API void ma_panner_set_pan(ma_panner* pPanner, float pan)\n{\n    if (pPanner == NULL) {\n        return;\n    }\n\n    pPanner->pan = ma_clamp(pan, -1.0f, 1.0f);\n}\n\nMA_API float ma_panner_get_pan(const ma_panner* pPanner)\n{\n    if (pPanner == NULL) {\n        return 0;\n    }\n\n    return pPanner->pan;\n}\n\n\n\n\nMA_API ma_fader_config ma_fader_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    ma_fader_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n\n    return config;\n}\n\n\nMA_API ma_result ma_fader_init(const ma_fader_config* pConfig, ma_fader* pFader)\n{\n    if (pFader == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFader);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Only f32 is supported for now. */\n    if (pConfig->format != ma_format_f32) {\n        return MA_INVALID_ARGS;\n    }\n\n    pFader->config         = *pConfig;\n    pFader->volumeBeg      = 1;\n    pFader->volumeEnd      = 1;\n    pFader->lengthInFrames = 0;\n    pFader->cursorInFrames = 0;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_fader_process_pcm_frames(ma_fader* pFader, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pFader == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If the cursor is still negative we need to just copy the absolute number of those frames, but no more than frameCount. */\n    if (pFader->cursorInFrames < 0) {\n        ma_uint64 absCursorInFrames = (ma_uint64)0 - pFader->cursorInFrames;\n        if (absCursorInFrames > frameCount) {\n            absCursorInFrames = frameCount;\n        }\n\n        ma_copy_pcm_frames(pFramesOut, pFramesIn, absCursorInFrames, pFader->config.format, pFader->config.channels);\n\n        pFader->cursorInFrames += absCursorInFrames;\n        frameCount -= absCursorInFrames;\n        pFramesOut  = ma_offset_ptr(pFramesOut, ma_get_bytes_per_frame(pFader->config.format, pFader->config.channels)*absCursorInFrames);\n        pFramesIn   = ma_offset_ptr(pFramesIn,  ma_get_bytes_per_frame(pFader->config.format, pFader->config.channels)*absCursorInFrames);\n    }\n\n    if (pFader->cursorInFrames >= 0) {\n        /*\n        For now we need to clamp frameCount so that the cursor never overflows 32-bits. This is required for\n        the conversion to a float which we use for the linear interpolation. This might be changed later.\n        */\n        if (frameCount + pFader->cursorInFrames > UINT_MAX) {\n            frameCount = UINT_MAX - pFader->cursorInFrames;\n        }\n\n        /* Optimized path if volumeBeg and volumeEnd are equal. */\n        if (pFader->volumeBeg == pFader->volumeEnd) {\n            if (pFader->volumeBeg == 1) {\n                /* Straight copy. */\n                ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, pFader->config.format, pFader->config.channels);\n            } else {\n                /* Copy with volume. */\n                ma_copy_and_apply_volume_and_clip_pcm_frames(pFramesOut, pFramesIn, frameCount, pFader->config.format, pFader->config.channels, pFader->volumeBeg);\n            }\n        } else {\n            /* Slower path. Volumes are different, so may need to do an interpolation. */\n            if ((ma_uint64)pFader->cursorInFrames >= pFader->lengthInFrames) {\n                /* Fast path. We've gone past the end of the fade period so just apply the end volume to all samples. */\n                ma_copy_and_apply_volume_and_clip_pcm_frames(pFramesOut, pFramesIn, frameCount, pFader->config.format, pFader->config.channels, pFader->volumeEnd);\n            } else {\n                /* Slow path. This is where we do the actual fading. */\n                ma_uint64 iFrame;\n                ma_uint32 iChannel;\n\n                /* For now we only support f32. Support for other formats might be added later. */\n                if (pFader->config.format == ma_format_f32) {\n                    const float* pFramesInF32  = (const float*)pFramesIn;\n                    /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n\n                    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                        float a = (ma_uint32)ma_min(pFader->cursorInFrames + iFrame, pFader->lengthInFrames) / (float)((ma_uint32)pFader->lengthInFrames);   /* Safe cast due to the frameCount clamp at the top of this function. */\n                        float volume = ma_mix_f32_fast(pFader->volumeBeg, pFader->volumeEnd, a);\n\n                        for (iChannel = 0; iChannel < pFader->config.channels; iChannel += 1) {\n                            pFramesOutF32[iFrame*pFader->config.channels + iChannel] = pFramesInF32[iFrame*pFader->config.channels + iChannel] * volume;\n                        }\n                    }\n                } else {\n                    return MA_NOT_IMPLEMENTED;\n                }\n            }\n        }\n    }\n\n    pFader->cursorInFrames += frameCount;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_fader_get_data_format(const ma_fader* pFader, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate)\n{\n    if (pFader == NULL) {\n        return;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pFader->config.format;\n    }\n\n    if (pChannels != NULL) {\n        *pChannels = pFader->config.channels;\n    }\n\n    if (pSampleRate != NULL) {\n        *pSampleRate = pFader->config.sampleRate;\n    }\n}\n\nMA_API void ma_fader_set_fade(ma_fader* pFader, float volumeBeg, float volumeEnd, ma_uint64 lengthInFrames)\n{\n    ma_fader_set_fade_ex(pFader, volumeBeg, volumeEnd, lengthInFrames, 0);\n}\n\nMA_API void ma_fader_set_fade_ex(ma_fader* pFader, float volumeBeg, float volumeEnd, ma_uint64 lengthInFrames, ma_int64 startOffsetInFrames)\n{\n    if (pFader == NULL) {\n        return;\n    }\n\n    /* If the volume is negative, use current volume. */\n    if (volumeBeg < 0) {\n        volumeBeg = ma_fader_get_current_volume(pFader);\n    }\n\n    /*\n    The length needs to be clamped to 32-bits due to how we convert it to a float for linear\n    interpolation reasons. I might change this requirement later, but for now it's not important.\n    */\n    if (lengthInFrames > UINT_MAX) {\n        lengthInFrames = UINT_MAX;\n    }\n\n    /* The start offset needs to be clamped to ensure it doesn't overflow a signed number. */\n    if (startOffsetInFrames > INT_MAX) {\n        startOffsetInFrames = INT_MAX;\n    }\n\n    pFader->volumeBeg      = volumeBeg;\n    pFader->volumeEnd      = volumeEnd;\n    pFader->lengthInFrames = lengthInFrames;\n    pFader->cursorInFrames = -startOffsetInFrames;\n}\n\nMA_API float ma_fader_get_current_volume(const ma_fader* pFader)\n{\n    if (pFader == NULL) {\n        return 0.0f;\n    }\n\n    /* Any frames prior to the start of the fade period will be at unfaded volume. */\n    if (pFader->cursorInFrames < 0) {\n        return 1.0f;\n    }\n\n    /* The current volume depends on the position of the cursor. */\n    if (pFader->cursorInFrames == 0) {\n        return pFader->volumeBeg;\n    } else if ((ma_uint64)pFader->cursorInFrames >= pFader->lengthInFrames) {   /* Safe case because the < 0 case was checked above. */\n        return pFader->volumeEnd;\n    } else {\n        /* The cursor is somewhere inside the fading period. We can figure this out with a simple linear interpolation between volumeBeg and volumeEnd based on our cursor position. */\n        return ma_mix_f32_fast(pFader->volumeBeg, pFader->volumeEnd, (ma_uint32)pFader->cursorInFrames / (float)((ma_uint32)pFader->lengthInFrames));    /* Safe cast to uint32 because we clamp it in ma_fader_process_pcm_frames(). */\n    }\n}\n\n\n\n\n\nMA_API ma_vec3f ma_vec3f_init_3f(float x, float y, float z)\n{\n    ma_vec3f v;\n\n    v.x = x;\n    v.y = y;\n    v.z = z;\n\n    return v;\n}\n\nMA_API ma_vec3f ma_vec3f_sub(ma_vec3f a, ma_vec3f b)\n{\n    return ma_vec3f_init_3f(\n        a.x - b.x,\n        a.y - b.y,\n        a.z - b.z\n    );\n}\n\nMA_API ma_vec3f ma_vec3f_neg(ma_vec3f a)\n{\n    return ma_vec3f_init_3f(\n        -a.x,\n        -a.y,\n        -a.z\n    );\n}\n\nMA_API float ma_vec3f_dot(ma_vec3f a, ma_vec3f b)\n{\n    return a.x*b.x + a.y*b.y + a.z*b.z;\n}\n\nMA_API float ma_vec3f_len2(ma_vec3f v)\n{\n    return ma_vec3f_dot(v, v);\n}\n\nMA_API float ma_vec3f_len(ma_vec3f v)\n{\n    return (float)ma_sqrtd(ma_vec3f_len2(v));\n}\n\n\n\nMA_API float ma_vec3f_dist(ma_vec3f a, ma_vec3f b)\n{\n    return ma_vec3f_len(ma_vec3f_sub(a, b));\n}\n\nMA_API ma_vec3f ma_vec3f_normalize(ma_vec3f v)\n{\n    float invLen;\n    float len2 = ma_vec3f_len2(v);\n    if (len2 == 0) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    invLen = ma_rsqrtf(len2);\n    v.x *= invLen;\n    v.y *= invLen;\n    v.z *= invLen;\n\n    return v;\n}\n\nMA_API ma_vec3f ma_vec3f_cross(ma_vec3f a, ma_vec3f b)\n{\n    return ma_vec3f_init_3f(\n        a.y*b.z - a.z*b.y,\n        a.z*b.x - a.x*b.z,\n        a.x*b.y - a.y*b.x\n    );\n}\n\n\nMA_API void ma_atomic_vec3f_init(ma_atomic_vec3f* v, ma_vec3f value)\n{\n    v->v = value;\n    v->lock = 0;    /* Important this is initialized to 0. */\n}\n\nMA_API void ma_atomic_vec3f_set(ma_atomic_vec3f* v, ma_vec3f value)\n{\n    ma_spinlock_lock(&v->lock);\n    {\n        v->v = value;\n    }\n    ma_spinlock_unlock(&v->lock);\n}\n\nMA_API ma_vec3f ma_atomic_vec3f_get(ma_atomic_vec3f* v)\n{\n    ma_vec3f r;\n\n    ma_spinlock_lock(&v->lock);\n    {\n        r = v->v;\n    }\n    ma_spinlock_unlock(&v->lock);\n\n    return r;\n}\n\n\n\nstatic void ma_channel_map_apply_f32(float* pFramesOut, const ma_channel* pChannelMapOut, ma_uint32 channelsOut, const float* pFramesIn, const ma_channel* pChannelMapIn, ma_uint32 channelsIn, ma_uint64 frameCount, ma_channel_mix_mode mode, ma_mono_expansion_mode monoExpansionMode);\nstatic ma_bool32 ma_is_spatial_channel_position(ma_channel channelPosition);\n\n\n#ifndef MA_DEFAULT_SPEED_OF_SOUND\n#define MA_DEFAULT_SPEED_OF_SOUND   343.3f\n#endif\n\n/*\nThese vectors represent the direction that speakers are facing from the center point. They're used\nfor panning in the spatializer. Must be normalized.\n*/\nstatic ma_vec3f g_maChannelDirections[MA_CHANNEL_POSITION_COUNT] = {\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_NONE */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_MONO */\n    {-0.7071f,  0.0f,    -0.7071f },  /* MA_CHANNEL_FRONT_LEFT */\n    {+0.7071f,  0.0f,    -0.7071f },  /* MA_CHANNEL_FRONT_RIGHT */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_FRONT_CENTER */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_LFE */\n    {-0.7071f,  0.0f,    +0.7071f },  /* MA_CHANNEL_BACK_LEFT */\n    {+0.7071f,  0.0f,    +0.7071f },  /* MA_CHANNEL_BACK_RIGHT */\n    {-0.3162f,  0.0f,    -0.9487f },  /* MA_CHANNEL_FRONT_LEFT_CENTER */\n    {+0.3162f,  0.0f,    -0.9487f },  /* MA_CHANNEL_FRONT_RIGHT_CENTER */\n    { 0.0f,     0.0f,    +1.0f    },  /* MA_CHANNEL_BACK_CENTER */\n    {-1.0f,     0.0f,     0.0f    },  /* MA_CHANNEL_SIDE_LEFT */\n    {+1.0f,     0.0f,     0.0f    },  /* MA_CHANNEL_SIDE_RIGHT */\n    { 0.0f,    +1.0f,     0.0f    },  /* MA_CHANNEL_TOP_CENTER */\n    {-0.5774f, +0.5774f, -0.5774f },  /* MA_CHANNEL_TOP_FRONT_LEFT */\n    { 0.0f,    +0.7071f, -0.7071f },  /* MA_CHANNEL_TOP_FRONT_CENTER */\n    {+0.5774f, +0.5774f, -0.5774f },  /* MA_CHANNEL_TOP_FRONT_RIGHT */\n    {-0.5774f, +0.5774f, +0.5774f },  /* MA_CHANNEL_TOP_BACK_LEFT */\n    { 0.0f,    +0.7071f, +0.7071f },  /* MA_CHANNEL_TOP_BACK_CENTER */\n    {+0.5774f, +0.5774f, +0.5774f },  /* MA_CHANNEL_TOP_BACK_RIGHT */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_0 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_1 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_2 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_3 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_4 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_5 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_6 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_7 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_8 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_9 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_10 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_11 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_12 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_13 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_14 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_15 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_16 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_17 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_18 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_19 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_20 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_21 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_22 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_23 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_24 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_25 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_26 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_27 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_28 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_29 */\n    { 0.0f,     0.0f,    -1.0f    },  /* MA_CHANNEL_AUX_30 */\n    { 0.0f,     0.0f,    -1.0f    }   /* MA_CHANNEL_AUX_31 */\n};\n\nstatic ma_vec3f ma_get_channel_direction(ma_channel channel)\n{\n    if (channel >= MA_CHANNEL_POSITION_COUNT) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    } else {\n        return g_maChannelDirections[channel];\n    }\n}\n\n\n\nstatic float ma_attenuation_inverse(float distance, float minDistance, float maxDistance, float rolloff)\n{\n    if (minDistance >= maxDistance) {\n        return 1;   /* To avoid division by zero. Do not attenuate. */\n    }\n\n    return minDistance / (minDistance + rolloff * (ma_clamp(distance, minDistance, maxDistance) - minDistance));\n}\n\nstatic float ma_attenuation_linear(float distance, float minDistance, float maxDistance, float rolloff)\n{\n    if (minDistance >= maxDistance) {\n        return 1;   /* To avoid division by zero. Do not attenuate. */\n    }\n\n    return 1 - rolloff * (ma_clamp(distance, minDistance, maxDistance) - minDistance) / (maxDistance - minDistance);\n}\n\nstatic float ma_attenuation_exponential(float distance, float minDistance, float maxDistance, float rolloff)\n{\n    if (minDistance >= maxDistance) {\n        return 1;   /* To avoid division by zero. Do not attenuate. */\n    }\n\n    return (float)ma_powd(ma_clamp(distance, minDistance, maxDistance) / minDistance, -rolloff);\n}\n\n\n/*\nDoppler Effect calculation taken from the OpenAL spec, with two main differences:\n\n  1) The source to listener vector will have already been calculated at an earlier step so we can\n     just use that directly. We need only the position of the source relative to the origin.\n\n  2) We don't scale by a frequency because we actually just want the ratio which we'll plug straight\n     into the resampler directly.\n*/\nstatic float ma_doppler_pitch(ma_vec3f relativePosition, ma_vec3f sourceVelocity, ma_vec3f listenVelocity, float speedOfSound, float dopplerFactor)\n{\n    float len;\n    float vls;\n    float vss;\n\n    len = ma_vec3f_len(relativePosition);\n\n    /*\n    There's a case where the position of the source will be right on top of the listener in which\n    case the length will be 0 and we'll end up with a division by zero. We can just return a ratio\n    of 1.0 in this case. This is not considered in the OpenAL spec, but is necessary.\n    */\n    if (len == 0) {\n        return 1.0;\n    }\n\n    vls = ma_vec3f_dot(relativePosition, listenVelocity) / len;\n    vss = ma_vec3f_dot(relativePosition, sourceVelocity) / len;\n\n    vls = ma_min(vls, speedOfSound / dopplerFactor);\n    vss = ma_min(vss, speedOfSound / dopplerFactor);\n\n    return (speedOfSound - dopplerFactor*vls) / (speedOfSound - dopplerFactor*vss);\n}\n\n\nstatic void ma_get_default_channel_map_for_spatializer(ma_channel* pChannelMap, size_t channelMapCap, ma_uint32 channelCount)\n{\n    /*\n    Special case for stereo. Want to default the left and right speakers to side left and side\n    right so that they're facing directly down the X axis rather than slightly forward. Not\n    doing this will result in sounds being quieter when behind the listener. This might\n    actually be good for some scenarios, but I don't think it's an appropriate default because\n    it can be a bit unexpected.\n    */\n    if (channelCount == 2) {\n        pChannelMap[0] = MA_CHANNEL_SIDE_LEFT;\n        pChannelMap[1] = MA_CHANNEL_SIDE_RIGHT;\n    } else {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, channelCount);\n    }\n}\n\n\nMA_API ma_spatializer_listener_config ma_spatializer_listener_config_init(ma_uint32 channelsOut)\n{\n    ma_spatializer_listener_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.channelsOut             = channelsOut;\n    config.pChannelMapOut          = NULL;\n    config.handedness              = ma_handedness_right;\n    config.worldUp                 = ma_vec3f_init_3f(0, 1,  0);\n    config.coneInnerAngleInRadians = 6.283185f; /* 360 degrees. */\n    config.coneOuterAngleInRadians = 6.283185f; /* 360 degrees. */\n    config.coneOuterGain           = 0;\n    config.speedOfSound            = 343.3f;    /* Same as OpenAL. Used for doppler effect. */\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t channelMapOutOffset;\n} ma_spatializer_listener_heap_layout;\n\nstatic ma_result ma_spatializer_listener_get_heap_layout(const ma_spatializer_listener_config* pConfig, ma_spatializer_listener_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channelsOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Channel map. We always need this, even for passthroughs. */\n    pHeapLayout->channelMapOutOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(sizeof(*pConfig->pChannelMapOut) * pConfig->channelsOut);\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_spatializer_listener_get_heap_size(const ma_spatializer_listener_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_spatializer_listener_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_spatializer_listener_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spatializer_listener_init_preallocated(const ma_spatializer_listener_config* pConfig, void* pHeap, ma_spatializer_listener* pListener)\n{\n    ma_result result;\n    ma_spatializer_listener_heap_layout heapLayout;\n\n    if (pListener == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pListener);\n\n    result = ma_spatializer_listener_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pListener->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pListener->config    = *pConfig;\n    ma_atomic_vec3f_init(&pListener->position,  ma_vec3f_init_3f(0, 0, 0));\n    ma_atomic_vec3f_init(&pListener->direction, ma_vec3f_init_3f(0, 0, -1));\n    ma_atomic_vec3f_init(&pListener->velocity,  ma_vec3f_init_3f(0, 0,  0));\n    pListener->isEnabled = MA_TRUE;\n\n    /* Swap the forward direction if we're left handed (it was initialized based on right handed). */\n    if (pListener->config.handedness == ma_handedness_left) {\n        ma_vec3f negDir = ma_vec3f_neg(ma_spatializer_listener_get_direction(pListener));\n        ma_spatializer_listener_set_direction(pListener, negDir.x, negDir.y, negDir.z);\n    }\n\n\n    /* We must always have a valid channel map. */\n    pListener->config.pChannelMapOut = (ma_channel*)ma_offset_ptr(pHeap, heapLayout.channelMapOutOffset);\n\n    /* Use a slightly different default channel map for stereo. */\n    if (pConfig->pChannelMapOut == NULL) {\n        ma_get_default_channel_map_for_spatializer(pListener->config.pChannelMapOut, pConfig->channelsOut, pConfig->channelsOut);\n    } else {\n        ma_channel_map_copy_or_default(pListener->config.pChannelMapOut, pConfig->channelsOut, pConfig->pChannelMapOut, pConfig->channelsOut);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spatializer_listener_init(const ma_spatializer_listener_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_spatializer_listener* pListener)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_spatializer_listener_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_spatializer_listener_init_preallocated(pConfig, pHeap, pListener);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pListener->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_spatializer_listener_uninit(ma_spatializer_listener* pListener, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    if (pListener->_ownsHeap) {\n        ma_free(pListener->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_channel* ma_spatializer_listener_get_channel_map(ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return NULL;\n    }\n\n    return pListener->config.pChannelMapOut;\n}\n\nMA_API void ma_spatializer_listener_set_cone(ma_spatializer_listener* pListener, float innerAngleInRadians, float outerAngleInRadians, float outerGain)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    pListener->config.coneInnerAngleInRadians = innerAngleInRadians;\n    pListener->config.coneOuterAngleInRadians = outerAngleInRadians;\n    pListener->config.coneOuterGain           = outerGain;\n}\n\nMA_API void ma_spatializer_listener_get_cone(const ma_spatializer_listener* pListener, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    if (pInnerAngleInRadians != NULL) {\n        *pInnerAngleInRadians = pListener->config.coneInnerAngleInRadians;\n    }\n\n    if (pOuterAngleInRadians != NULL) {\n        *pOuterAngleInRadians = pListener->config.coneOuterAngleInRadians;\n    }\n\n    if (pOuterGain != NULL) {\n        *pOuterGain = pListener->config.coneOuterGain;\n    }\n}\n\nMA_API void ma_spatializer_listener_set_position(ma_spatializer_listener* pListener, float x, float y, float z)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pListener->position, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_listener_get_position(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pListener->position); /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_listener_set_direction(ma_spatializer_listener* pListener, float x, float y, float z)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pListener->direction, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_listener_get_direction(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pListener->direction);    /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_listener_set_velocity(ma_spatializer_listener* pListener, float x, float y, float z)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pListener->velocity, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_listener_get_velocity(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pListener->velocity); /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_listener_set_speed_of_sound(ma_spatializer_listener* pListener, float speedOfSound)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    pListener->config.speedOfSound = speedOfSound;\n}\n\nMA_API float ma_spatializer_listener_get_speed_of_sound(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return 0;\n    }\n\n    return pListener->config.speedOfSound;\n}\n\nMA_API void ma_spatializer_listener_set_world_up(ma_spatializer_listener* pListener, float x, float y, float z)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    pListener->config.worldUp = ma_vec3f_init_3f(x, y, z);\n}\n\nMA_API ma_vec3f ma_spatializer_listener_get_world_up(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return ma_vec3f_init_3f(0, 1, 0);\n    }\n\n    return pListener->config.worldUp;\n}\n\nMA_API void ma_spatializer_listener_set_enabled(ma_spatializer_listener* pListener, ma_bool32 isEnabled)\n{\n    if (pListener == NULL) {\n        return;\n    }\n\n    pListener->isEnabled = isEnabled;\n}\n\nMA_API ma_bool32 ma_spatializer_listener_is_enabled(const ma_spatializer_listener* pListener)\n{\n    if (pListener == NULL) {\n        return MA_FALSE;\n    }\n\n    return pListener->isEnabled;\n}\n\n\n\n\nMA_API ma_spatializer_config ma_spatializer_config_init(ma_uint32 channelsIn, ma_uint32 channelsOut)\n{\n    ma_spatializer_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.channelsIn                   = channelsIn;\n    config.channelsOut                  = channelsOut;\n    config.pChannelMapIn                = NULL;\n    config.attenuationModel             = ma_attenuation_model_inverse;\n    config.positioning                  = ma_positioning_absolute;\n    config.handedness                   = ma_handedness_right;\n    config.minGain                      = 0;\n    config.maxGain                      = 1;\n    config.minDistance                  = 1;\n    config.maxDistance                  = MA_FLT_MAX;\n    config.rolloff                      = 1;\n    config.coneInnerAngleInRadians      = 6.283185f; /* 360 degrees. */\n    config.coneOuterAngleInRadians      = 6.283185f; /* 360 degrees. */\n    config.coneOuterGain                = 0.0f;\n    config.dopplerFactor                = 1;\n    config.directionalAttenuationFactor = 1;\n    config.minSpatializationChannelGain = 0.2f;\n    config.gainSmoothTimeInFrames       = 360;       /* 7.5ms @ 48K. */\n\n    return config;\n}\n\n\nstatic ma_gainer_config ma_spatializer_gainer_config_init(const ma_spatializer_config* pConfig)\n{\n    MA_ASSERT(pConfig != NULL);\n    return ma_gainer_config_init(pConfig->channelsOut, pConfig->gainSmoothTimeInFrames);\n}\n\nstatic ma_result ma_spatializer_validate_config(const ma_spatializer_config* pConfig)\n{\n    MA_ASSERT(pConfig != NULL);\n\n    if (pConfig->channelsIn == 0 || pConfig->channelsOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    return MA_SUCCESS;\n}\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t channelMapInOffset;\n    size_t newChannelGainsOffset;\n    size_t gainerOffset;\n} ma_spatializer_heap_layout;\n\nstatic ma_result ma_spatializer_get_heap_layout(const ma_spatializer_config* pConfig, ma_spatializer_heap_layout* pHeapLayout)\n{\n    ma_result result;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_spatializer_validate_config(pConfig);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Channel map. */\n    pHeapLayout->channelMapInOffset = MA_SIZE_MAX;  /* <-- MA_SIZE_MAX indicates no allocation necessary. */\n    if (pConfig->pChannelMapIn != NULL) {\n        pHeapLayout->channelMapInOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(sizeof(*pConfig->pChannelMapIn) * pConfig->channelsIn);\n    }\n\n    /* New channel gains for output. */\n    pHeapLayout->newChannelGainsOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(sizeof(float) * pConfig->channelsOut);\n\n    /* Gainer. */\n    {\n        size_t gainerHeapSizeInBytes;\n        ma_gainer_config gainerConfig;\n\n        gainerConfig = ma_spatializer_gainer_config_init(pConfig);\n\n        result = ma_gainer_get_heap_size(&gainerConfig, &gainerHeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->gainerOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(gainerHeapSizeInBytes);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spatializer_get_heap_size(const ma_spatializer_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_spatializer_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;  /* Safety. */\n\n    result = ma_spatializer_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_spatializer_init_preallocated(const ma_spatializer_config* pConfig, void* pHeap, ma_spatializer* pSpatializer)\n{\n    ma_result result;\n    ma_spatializer_heap_layout heapLayout;\n    ma_gainer_config gainerConfig;\n\n    if (pSpatializer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pSpatializer);\n\n    if (pConfig == NULL || pHeap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_spatializer_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pSpatializer->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pSpatializer->channelsIn                   = pConfig->channelsIn;\n    pSpatializer->channelsOut                  = pConfig->channelsOut;\n    pSpatializer->attenuationModel             = pConfig->attenuationModel;\n    pSpatializer->positioning                  = pConfig->positioning;\n    pSpatializer->handedness                   = pConfig->handedness;\n    pSpatializer->minGain                      = pConfig->minGain;\n    pSpatializer->maxGain                      = pConfig->maxGain;\n    pSpatializer->minDistance                  = pConfig->minDistance;\n    pSpatializer->maxDistance                  = pConfig->maxDistance;\n    pSpatializer->rolloff                      = pConfig->rolloff;\n    pSpatializer->coneInnerAngleInRadians      = pConfig->coneInnerAngleInRadians;\n    pSpatializer->coneOuterAngleInRadians      = pConfig->coneOuterAngleInRadians;\n    pSpatializer->coneOuterGain                = pConfig->coneOuterGain;\n    pSpatializer->dopplerFactor                = pConfig->dopplerFactor;\n    pSpatializer->minSpatializationChannelGain = pConfig->minSpatializationChannelGain;\n    pSpatializer->directionalAttenuationFactor = pConfig->directionalAttenuationFactor;\n    pSpatializer->gainSmoothTimeInFrames       = pConfig->gainSmoothTimeInFrames;\n    ma_atomic_vec3f_init(&pSpatializer->position,  ma_vec3f_init_3f(0, 0,  0));\n    ma_atomic_vec3f_init(&pSpatializer->direction, ma_vec3f_init_3f(0, 0, -1));\n    ma_atomic_vec3f_init(&pSpatializer->velocity,  ma_vec3f_init_3f(0, 0,  0));\n    pSpatializer->dopplerPitch                 = 1;\n\n    /* Swap the forward direction if we're left handed (it was initialized based on right handed). */\n    if (pSpatializer->handedness == ma_handedness_left) {\n        ma_vec3f negDir = ma_vec3f_neg(ma_spatializer_get_direction(pSpatializer));\n        ma_spatializer_set_direction(pSpatializer, negDir.x, negDir.y, negDir.z);\n    }\n\n    /* Channel map. This will be on the heap. */\n    if (pConfig->pChannelMapIn != NULL) {\n        pSpatializer->pChannelMapIn = (ma_channel*)ma_offset_ptr(pHeap, heapLayout.channelMapInOffset);\n        ma_channel_map_copy_or_default(pSpatializer->pChannelMapIn, pSpatializer->channelsIn, pConfig->pChannelMapIn, pSpatializer->channelsIn);\n    }\n\n    /* New channel gains for output channels. */\n    pSpatializer->pNewChannelGainsOut = (float*)ma_offset_ptr(pHeap, heapLayout.newChannelGainsOffset);\n\n    /* Gainer. */\n    gainerConfig = ma_spatializer_gainer_config_init(pConfig);\n\n    result = ma_gainer_init_preallocated(&gainerConfig, ma_offset_ptr(pHeap, heapLayout.gainerOffset), &pSpatializer->gainer);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the gainer. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spatializer_init(const ma_spatializer_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_spatializer* pSpatializer)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    /* We'll need a heap allocation to retrieve the size. */\n    result = ma_spatializer_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_spatializer_init_preallocated(pConfig, pHeap, pSpatializer);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pSpatializer->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_spatializer_uninit(ma_spatializer* pSpatializer, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_gainer_uninit(&pSpatializer->gainer, pAllocationCallbacks);\n\n    if (pSpatializer->_ownsHeap) {\n        ma_free(pSpatializer->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic float ma_calculate_angular_gain(ma_vec3f dirA, ma_vec3f dirB, float coneInnerAngleInRadians, float coneOuterAngleInRadians, float coneOuterGain)\n{\n    /*\n    Angular attenuation.\n\n    Unlike distance gain, the math for this is not specified by the OpenAL spec so we'll just go ahead and figure\n    this out for ourselves at the expense of possibly being inconsistent with other implementations.\n\n    To do cone attenuation, I'm just using the same math that we'd use to implement a basic spotlight in OpenGL. We\n    just need to get the direction from the source to the listener and then do a dot product against that and the\n    direction of the spotlight. Then we just compare that dot product against the cosine of the inner and outer\n    angles. If the dot product is greater than the outer angle, we just use coneOuterGain. If it's less than\n    the inner angle, we just use a gain of 1. Otherwise we linearly interpolate between 1 and coneOuterGain.\n    */\n    if (coneInnerAngleInRadians < 6.283185f) {\n        float angularGain = 1;\n        float cutoffInner = (float)ma_cosd(coneInnerAngleInRadians*0.5f);\n        float cutoffOuter = (float)ma_cosd(coneOuterAngleInRadians*0.5f);\n        float d;\n\n        d = ma_vec3f_dot(dirA, dirB);\n\n        if (d > cutoffInner) {\n            /* It's inside the inner angle. */\n            angularGain = 1;\n        } else {\n            /* It's outside the inner angle. */\n            if (d > cutoffOuter) {\n                /* It's between the inner and outer angle. We need to linearly interpolate between 1 and coneOuterGain. */\n                angularGain = ma_mix_f32(coneOuterGain, 1, (d - cutoffOuter) / (cutoffInner - cutoffOuter));\n            } else {\n                /* It's outside the outer angle. */\n                angularGain = coneOuterGain;\n            }\n        }\n\n        /*printf(\"d = %f; cutoffInner = %f; cutoffOuter = %f; angularGain = %f\\n\", d, cutoffInner, cutoffOuter, angularGain);*/\n        return angularGain;\n    } else {\n        /* Inner angle is 360 degrees so no need to do any attenuation. */\n        return 1;\n    }\n}\n\nMA_API ma_result ma_spatializer_process_pcm_frames(ma_spatializer* pSpatializer, ma_spatializer_listener* pListener, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_channel* pChannelMapIn;\n    ma_channel* pChannelMapOut;\n\n    if (pSpatializer == NULL || pListener == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pChannelMapIn = pSpatializer->pChannelMapIn;\n    pChannelMapOut = pListener->config.pChannelMapOut;\n\n    /* If we're not spatializing we need to run an optimized path. */\n    if (ma_atomic_load_i32(&pSpatializer->attenuationModel) == ma_attenuation_model_none) {\n        if (ma_spatializer_listener_is_enabled(pListener)) {\n            /* No attenuation is required, but we'll need to do some channel conversion. */\n            if (pSpatializer->channelsIn == pSpatializer->channelsOut) {\n                ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, ma_format_f32, pSpatializer->channelsIn);\n            } else {\n                ma_channel_map_apply_f32((float*)pFramesOut, pChannelMapOut, pSpatializer->channelsOut, (const float*)pFramesIn, pChannelMapIn, pSpatializer->channelsIn, frameCount, ma_channel_mix_mode_rectangular, ma_mono_expansion_mode_default);   /* Safe casts to float* because f32 is the only supported format. */\n            }\n        } else {\n            /* The listener is disabled. Output silence. */\n            ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, pSpatializer->channelsOut);\n        }\n\n        /*\n        We're not doing attenuation so don't bother with doppler for now. I'm not sure if this is\n        the correct thinking so might need to review this later.\n        */\n        pSpatializer->dopplerPitch = 1;\n    } else {\n        /*\n        Let's first determine which listener the sound is closest to. Need to keep in mind that we\n        might not have a world or any listeners, in which case we just spatializer based on the\n        listener being positioned at the origin (0, 0, 0).\n        */\n        ma_vec3f relativePosNormalized;\n        ma_vec3f relativePos;   /* The position relative to the listener. */\n        ma_vec3f relativeDir;   /* The direction of the sound, relative to the listener. */\n        ma_vec3f listenerVel;   /* The velocity of the listener. For doppler pitch calculation. */\n        float speedOfSound;\n        float distance = 0;\n        float gain = 1;\n        ma_uint32 iChannel;\n        const ma_uint32 channelsOut = pSpatializer->channelsOut;\n        const ma_uint32 channelsIn  = pSpatializer->channelsIn;\n        float minDistance = ma_spatializer_get_min_distance(pSpatializer);\n        float maxDistance = ma_spatializer_get_max_distance(pSpatializer);\n        float rolloff = ma_spatializer_get_rolloff(pSpatializer);\n        float dopplerFactor = ma_spatializer_get_doppler_factor(pSpatializer);\n\n        /*\n        We'll need the listener velocity for doppler pitch calculations. The speed of sound is\n        defined by the listener, so we'll grab that here too.\n        */\n        listenerVel  = ma_spatializer_listener_get_velocity(pListener);\n        speedOfSound = pListener->config.speedOfSound;\n\n        if (ma_spatializer_get_positioning(pSpatializer) == ma_positioning_relative) {\n            relativePos = ma_spatializer_get_position(pSpatializer);\n            relativeDir = ma_spatializer_get_direction(pSpatializer);\n        } else {\n            /*\n            We're using absolute positioning. We need to transform the sound's position and\n            direction so that it's relative to listener. Later on we'll use this for determining\n            the factors to apply to each channel to apply the panning effect.\n            */\n            ma_spatializer_get_relative_position_and_direction(pSpatializer, pListener, &relativePos, &relativeDir);\n        }\n\n        distance = ma_vec3f_len(relativePos);\n\n        /* We've gathered the data, so now we can apply some spatialization. */\n        switch (ma_spatializer_get_attenuation_model(pSpatializer)) {\n            case ma_attenuation_model_inverse:\n            {\n                gain = ma_attenuation_inverse(distance, minDistance, maxDistance, rolloff);\n            } break;\n            case ma_attenuation_model_linear:\n            {\n                gain = ma_attenuation_linear(distance, minDistance, maxDistance, rolloff);\n            } break;\n            case ma_attenuation_model_exponential:\n            {\n                gain = ma_attenuation_exponential(distance, minDistance, maxDistance, rolloff);\n            } break;\n            case ma_attenuation_model_none:\n            default:\n            {\n                gain = 1;\n            } break;\n        }\n\n        /* Normalize the position. */\n        if (distance > 0.001f) {\n            float distanceInv = 1/distance;\n            relativePosNormalized    = relativePos;\n            relativePosNormalized.x *= distanceInv;\n            relativePosNormalized.y *= distanceInv;\n            relativePosNormalized.z *= distanceInv;\n        } else {\n            distance = 0;\n            relativePosNormalized = ma_vec3f_init_3f(0, 0, 0);\n        }\n\n        /*\n        Angular attenuation.\n\n        Unlike distance gain, the math for this is not specified by the OpenAL spec so we'll just go ahead and figure\n        this out for ourselves at the expense of possibly being inconsistent with other implementations.\n\n        To do cone attenuation, I'm just using the same math that we'd use to implement a basic spotlight in OpenGL. We\n        just need to get the direction from the source to the listener and then do a dot product against that and the\n        direction of the spotlight. Then we just compare that dot product against the cosine of the inner and outer\n        angles. If the dot product is greater than the outer angle, we just use coneOuterGain. If it's less than\n        the inner angle, we just use a gain of 1. Otherwise we linearly interpolate between 1 and coneOuterGain.\n        */\n        if (distance > 0) {\n            /* Source angular gain. */\n            float spatializerConeInnerAngle;\n            float spatializerConeOuterAngle;\n            float spatializerConeOuterGain;\n            ma_spatializer_get_cone(pSpatializer, &spatializerConeInnerAngle, &spatializerConeOuterAngle, &spatializerConeOuterGain);\n\n            gain *= ma_calculate_angular_gain(relativeDir, ma_vec3f_neg(relativePosNormalized), spatializerConeInnerAngle, spatializerConeOuterAngle, spatializerConeOuterGain);\n\n            /*\n            We're supporting angular gain on the listener as well for those who want to reduce the volume of sounds that\n            are positioned behind the listener. On default settings, this will have no effect.\n            */\n            if (pListener != NULL && pListener->config.coneInnerAngleInRadians < 6.283185f) {\n                ma_vec3f listenerDirection;\n                float listenerInnerAngle;\n                float listenerOuterAngle;\n                float listenerOuterGain;\n\n                if (pListener->config.handedness == ma_handedness_right) {\n                    listenerDirection = ma_vec3f_init_3f(0, 0, -1);\n                } else {\n                    listenerDirection = ma_vec3f_init_3f(0, 0, +1);\n                }\n\n                listenerInnerAngle = pListener->config.coneInnerAngleInRadians;\n                listenerOuterAngle = pListener->config.coneOuterAngleInRadians;\n                listenerOuterGain  = pListener->config.coneOuterGain;\n\n                gain *= ma_calculate_angular_gain(listenerDirection, relativePosNormalized, listenerInnerAngle, listenerOuterAngle, listenerOuterGain);\n            }\n        } else {\n            /* The sound is right on top of the listener. Don't do any angular attenuation. */\n        }\n\n\n        /* Clamp the gain. */\n        gain = ma_clamp(gain, ma_spatializer_get_min_gain(pSpatializer), ma_spatializer_get_max_gain(pSpatializer));\n\n        /*\n        The gain needs to be applied per-channel here. The spatialization code below will be changing the per-channel\n        gains which will then eventually be passed into the gainer which will deal with smoothing the gain transitions\n        to avoid harsh changes in gain.\n        */\n        for (iChannel = 0; iChannel < channelsOut; iChannel += 1) {\n            pSpatializer->pNewChannelGainsOut[iChannel] = gain;\n        }\n\n        /*\n        Convert to our output channel count. If the listener is disabled we just output silence here. We cannot ignore\n        the whole section of code here because we need to update some internal spatialization state.\n        */\n        if (ma_spatializer_listener_is_enabled(pListener)) {\n            ma_channel_map_apply_f32((float*)pFramesOut, pChannelMapOut, channelsOut, (const float*)pFramesIn, pChannelMapIn, channelsIn, frameCount, ma_channel_mix_mode_rectangular, ma_mono_expansion_mode_default);\n        } else {\n            ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, pSpatializer->channelsOut);\n        }\n\n\n        /*\n        Panning. This is where we'll apply the gain and convert to the output channel count. We have an optimized path for\n        when we're converting to a mono stream. In that case we don't really need to do any panning - we just apply the\n        gain to the final output.\n        */\n        /*printf(\"distance=%f; gain=%f\\n\", distance, gain);*/\n\n        /* We must have a valid channel map here to ensure we spatialize properly. */\n        MA_ASSERT(pChannelMapOut != NULL);\n\n        /*\n        We're not converting to mono so we'll want to apply some panning. This is where the feeling of something being\n        to the left, right, infront or behind the listener is calculated. I'm just using a basic model here. Note that\n        the code below is not based on any specific algorithm. I'm just implementing this off the top of my head and\n        seeing how it goes. There might be better ways to do this.\n\n        To determine the direction of the sound relative to a speaker I'm using dot products. Each speaker is given a\n        direction. For example, the left channel in a stereo system will be -1 on the X axis and the right channel will\n        be +1 on the X axis. A dot product is performed against the direction vector of the channel and the normalized\n        position of the sound.\n        */\n\n        /*\n        Calculate our per-channel gains. We do this based on the normalized relative position of the sound and it's\n        relation to the direction of the channel.\n        */\n        if (distance > 0) {\n            ma_vec3f unitPos = relativePos;\n            float distanceInv = 1/distance;\n            unitPos.x *= distanceInv;\n            unitPos.y *= distanceInv;\n            unitPos.z *= distanceInv;\n\n            for (iChannel = 0; iChannel < channelsOut; iChannel += 1) {\n                ma_channel channelOut;\n                float d;\n                float dMin;\n\n                channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannel);\n                if (ma_is_spatial_channel_position(channelOut)) {\n                    d = ma_mix_f32_fast(1, ma_vec3f_dot(unitPos, ma_get_channel_direction(channelOut)), ma_spatializer_get_directional_attenuation_factor(pSpatializer));\n                } else {\n                    d = 1;  /* It's not a spatial channel so there's no real notion of direction. */\n                }\n\n                /*\n                In my testing, if the panning effect is too aggressive it makes spatialization feel uncomfortable.\n                The \"dMin\" variable below is used to control the aggressiveness of the panning effect. When set to\n                0, panning will be most extreme and any sounds that are positioned on the opposite side of the\n                speaker will be completely silent from that speaker. Not only does this feel uncomfortable, it\n                doesn't even remotely represent the real world at all because sounds that come from your right side\n                are still clearly audible from your left side. Setting \"dMin\" to 1 will result in no panning at\n                all, which is also not ideal. By setting it to something greater than 0, the spatialization effect\n                becomes much less dramatic and a lot more bearable.\n\n                Summary: 0 = more extreme panning; 1 = no panning.\n                */\n                dMin = pSpatializer->minSpatializationChannelGain;\n\n                /*\n                At this point, \"d\" will be positive if the sound is on the same side as the channel and negative if\n                it's on the opposite side. It will be in the range of -1..1. There's two ways I can think of to\n                calculate a panning value. The first is to simply convert it to 0..1, however this has a problem\n                which I'm not entirely happy with. Considering a stereo system, when a sound is positioned right\n                in front of the listener it'll result in each speaker getting a gain of 0.5. I don't know if I like\n                the idea of having a scaling factor of 0.5 being applied to a sound when it's sitting right in front\n                of the listener. I would intuitively expect that to be played at full volume, or close to it.\n\n                The second idea I think of is to only apply a reduction in gain when the sound is on the opposite\n                side of the speaker. That is, reduce the gain only when the dot product is negative. The problem\n                with this is that there will not be any attenuation as the sound sweeps around the 180 degrees\n                where the dot product is positive. The idea with this option is that you leave the gain at 1 when\n                the sound is being played on the same side as the speaker and then you just reduce the volume when\n                the sound is on the other side.\n\n                The summarize, I think the first option should give a better sense of spatialization, but the second\n                option is better for preserving the sound's power.\n\n                UPDATE: In my testing, I find the first option to sound better. You can feel the sense of space a\n                bit better, but you can also hear the reduction in volume when it's right in front.\n                */\n                #if 1\n                {\n                    /*\n                    Scale the dot product from -1..1 to 0..1. Will result in a sound directly in front losing power\n                    by being played at 0.5 gain.\n                    */\n                    d = (d + 1) * 0.5f;  /* -1..1 to 0..1 */\n                    d = ma_max(d, dMin);\n                    pSpatializer->pNewChannelGainsOut[iChannel] *= d;\n                }\n                #else\n                {\n                    /*\n                    Only reduce the volume of the sound if it's on the opposite side. This path keeps the volume more\n                    consistent, but comes at the expense of a worse sense of space and positioning.\n                    */\n                    if (d < 0) {\n                        d += 1; /* Move into the positive range. */\n                        d = ma_max(d, dMin);\n                        channelGainsOut[iChannel] *= d;\n                    }\n                }\n                #endif\n            }\n        } else {\n            /* Assume the sound is right on top of us. Don't do any panning. */\n        }\n\n        /* Now we need to apply the volume to each channel. This needs to run through the gainer to ensure we get a smooth volume transition. */\n        ma_gainer_set_gains(&pSpatializer->gainer, pSpatializer->pNewChannelGainsOut);\n        ma_gainer_process_pcm_frames(&pSpatializer->gainer, pFramesOut, pFramesOut, frameCount);\n\n        /*\n        Before leaving we'll want to update our doppler pitch so that the caller can apply some\n        pitch shifting if they desire. Note that we need to negate the relative position here\n        because the doppler calculation needs to be source-to-listener, but ours is listener-to-\n        source.\n        */\n        if (dopplerFactor > 0) {\n            pSpatializer->dopplerPitch = ma_doppler_pitch(ma_vec3f_sub(ma_spatializer_listener_get_position(pListener), ma_spatializer_get_position(pSpatializer)), ma_spatializer_get_velocity(pSpatializer), listenerVel, speedOfSound, dopplerFactor);\n        } else {\n            pSpatializer->dopplerPitch = 1;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_spatializer_set_master_volume(ma_spatializer* pSpatializer, float volume)\n{\n    if (pSpatializer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_gainer_set_master_volume(&pSpatializer->gainer, volume);\n}\n\nMA_API ma_result ma_spatializer_get_master_volume(const ma_spatializer* pSpatializer, float* pVolume)\n{\n    if (pSpatializer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_gainer_get_master_volume(&pSpatializer->gainer, pVolume);\n}\n\nMA_API ma_uint32 ma_spatializer_get_input_channels(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return pSpatializer->channelsIn;\n}\n\nMA_API ma_uint32 ma_spatializer_get_output_channels(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return pSpatializer->channelsOut;\n}\n\nMA_API void ma_spatializer_set_attenuation_model(ma_spatializer* pSpatializer, ma_attenuation_model attenuationModel)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_i32(&pSpatializer->attenuationModel, attenuationModel);\n}\n\nMA_API ma_attenuation_model ma_spatializer_get_attenuation_model(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return ma_attenuation_model_none;\n    }\n\n    return (ma_attenuation_model)ma_atomic_load_i32(&pSpatializer->attenuationModel);\n}\n\nMA_API void ma_spatializer_set_positioning(ma_spatializer* pSpatializer, ma_positioning positioning)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_i32(&pSpatializer->positioning, positioning);\n}\n\nMA_API ma_positioning ma_spatializer_get_positioning(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return ma_positioning_absolute;\n    }\n\n    return (ma_positioning)ma_atomic_load_i32(&pSpatializer->positioning);\n}\n\nMA_API void ma_spatializer_set_rolloff(ma_spatializer* pSpatializer, float rolloff)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->rolloff, rolloff);\n}\n\nMA_API float ma_spatializer_get_rolloff(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->rolloff);\n}\n\nMA_API void ma_spatializer_set_min_gain(ma_spatializer* pSpatializer, float minGain)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->minGain, minGain);\n}\n\nMA_API float ma_spatializer_get_min_gain(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->minGain);\n}\n\nMA_API void ma_spatializer_set_max_gain(ma_spatializer* pSpatializer, float maxGain)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->maxGain, maxGain);\n}\n\nMA_API float ma_spatializer_get_max_gain(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->maxGain);\n}\n\nMA_API void ma_spatializer_set_min_distance(ma_spatializer* pSpatializer, float minDistance)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->minDistance, minDistance);\n}\n\nMA_API float ma_spatializer_get_min_distance(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->minDistance);\n}\n\nMA_API void ma_spatializer_set_max_distance(ma_spatializer* pSpatializer, float maxDistance)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->maxDistance, maxDistance);\n}\n\nMA_API float ma_spatializer_get_max_distance(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->maxDistance);\n}\n\nMA_API void ma_spatializer_set_cone(ma_spatializer* pSpatializer, float innerAngleInRadians, float outerAngleInRadians, float outerGain)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->coneInnerAngleInRadians, innerAngleInRadians);\n    ma_atomic_exchange_f32(&pSpatializer->coneOuterAngleInRadians, outerAngleInRadians);\n    ma_atomic_exchange_f32(&pSpatializer->coneOuterGain,           outerGain);\n}\n\nMA_API void ma_spatializer_get_cone(const ma_spatializer* pSpatializer, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    if (pInnerAngleInRadians != NULL) {\n        *pInnerAngleInRadians = ma_atomic_load_f32(&pSpatializer->coneInnerAngleInRadians);\n    }\n\n    if (pOuterAngleInRadians != NULL) {\n        *pOuterAngleInRadians = ma_atomic_load_f32(&pSpatializer->coneOuterAngleInRadians);\n    }\n\n    if (pOuterGain != NULL) {\n        *pOuterGain = ma_atomic_load_f32(&pSpatializer->coneOuterGain);\n    }\n}\n\nMA_API void ma_spatializer_set_doppler_factor(ma_spatializer* pSpatializer, float dopplerFactor)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->dopplerFactor, dopplerFactor);\n}\n\nMA_API float ma_spatializer_get_doppler_factor(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 1;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->dopplerFactor);\n}\n\nMA_API void ma_spatializer_set_directional_attenuation_factor(ma_spatializer* pSpatializer, float directionalAttenuationFactor)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_f32(&pSpatializer->directionalAttenuationFactor, directionalAttenuationFactor);\n}\n\nMA_API float ma_spatializer_get_directional_attenuation_factor(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return 1;\n    }\n\n    return ma_atomic_load_f32(&pSpatializer->directionalAttenuationFactor);\n}\n\nMA_API void ma_spatializer_set_position(ma_spatializer* pSpatializer, float x, float y, float z)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pSpatializer->position, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_get_position(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pSpatializer->position);  /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_set_direction(ma_spatializer* pSpatializer, float x, float y, float z)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pSpatializer->direction, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_get_direction(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pSpatializer->direction); /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_set_velocity(ma_spatializer* pSpatializer, float x, float y, float z)\n{\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    ma_atomic_vec3f_set(&pSpatializer->velocity, ma_vec3f_init_3f(x, y, z));\n}\n\nMA_API ma_vec3f ma_spatializer_get_velocity(const ma_spatializer* pSpatializer)\n{\n    if (pSpatializer == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_atomic_vec3f_get((ma_atomic_vec3f*)&pSpatializer->velocity);  /* Naughty const-cast. It's just for atomically loading the vec3 which should be safe. */\n}\n\nMA_API void ma_spatializer_get_relative_position_and_direction(const ma_spatializer* pSpatializer, const ma_spatializer_listener* pListener, ma_vec3f* pRelativePos, ma_vec3f* pRelativeDir)\n{\n    if (pRelativePos != NULL) {\n        pRelativePos->x = 0;\n        pRelativePos->y = 0;\n        pRelativePos->z = 0;\n    }\n\n    if (pRelativeDir != NULL) {\n        pRelativeDir->x = 0;\n        pRelativeDir->y = 0;\n        pRelativeDir->z = -1;\n    }\n\n    if (pSpatializer == NULL) {\n        return;\n    }\n\n    if (pListener == NULL || ma_spatializer_get_positioning(pSpatializer) == ma_positioning_relative) {\n        /* There's no listener or we're using relative positioning. */\n        if (pRelativePos != NULL) {\n            *pRelativePos = ma_spatializer_get_position(pSpatializer);\n        }\n        if (pRelativeDir != NULL) {\n            *pRelativeDir = ma_spatializer_get_direction(pSpatializer);\n        }\n    } else {\n        ma_vec3f spatializerPosition;\n        ma_vec3f spatializerDirection;\n        ma_vec3f listenerPosition;\n        ma_vec3f listenerDirection;\n        ma_vec3f v;\n        ma_vec3f axisX;\n        ma_vec3f axisY;\n        ma_vec3f axisZ;\n        float m[4][4];\n\n        spatializerPosition  = ma_spatializer_get_position(pSpatializer);\n        spatializerDirection = ma_spatializer_get_direction(pSpatializer);\n        listenerPosition     = ma_spatializer_listener_get_position(pListener);\n        listenerDirection    = ma_spatializer_listener_get_direction(pListener);\n\n        /*\n        We need to calculate the right vector from our forward and up vectors. This is done with\n        a cross product.\n        */\n        axisZ = ma_vec3f_normalize(listenerDirection);                                  /* Normalization required here because we can't trust the caller. */\n        axisX = ma_vec3f_normalize(ma_vec3f_cross(axisZ, pListener->config.worldUp));   /* Normalization required here because the world up vector may not be perpendicular with the forward vector. */\n\n        /*\n        The calculation of axisX above can result in a zero-length vector if the listener is\n        looking straight up on the Y axis. We'll need to fall back to a +X in this case so that\n        the calculations below don't fall apart. This is where a quaternion based listener and\n        sound orientation would come in handy.\n        */\n        if (ma_vec3f_len2(axisX) == 0) {\n            axisX = ma_vec3f_init_3f(1, 0, 0);\n        }\n\n        axisY = ma_vec3f_cross(axisX, axisZ);                                           /* No normalization is required here because axisX and axisZ are unit length and perpendicular. */\n\n        /*\n        We need to swap the X axis if we're left handed because otherwise the cross product above\n        will have resulted in it pointing in the wrong direction (right handed was assumed in the\n        cross products above).\n        */\n        if (pListener->config.handedness == ma_handedness_left) {\n            axisX = ma_vec3f_neg(axisX);\n        }\n\n        /* Lookat. */\n        m[0][0] =  axisX.x; m[1][0] =  axisX.y; m[2][0] =  axisX.z; m[3][0] = -ma_vec3f_dot(axisX,               listenerPosition);\n        m[0][1] =  axisY.x; m[1][1] =  axisY.y; m[2][1] =  axisY.z; m[3][1] = -ma_vec3f_dot(axisY,               listenerPosition);\n        m[0][2] = -axisZ.x; m[1][2] = -axisZ.y; m[2][2] = -axisZ.z; m[3][2] = -ma_vec3f_dot(ma_vec3f_neg(axisZ), listenerPosition);\n        m[0][3] = 0;        m[1][3] = 0;        m[2][3] = 0;        m[3][3] = 1;\n\n        /*\n        Multiply the lookat matrix by the spatializer position to transform it to listener\n        space. This allows calculations to work based on the sound being relative to the\n        origin which makes things simpler.\n        */\n        if (pRelativePos != NULL) {\n            v = spatializerPosition;\n            pRelativePos->x = m[0][0] * v.x + m[1][0] * v.y + m[2][0] * v.z + m[3][0] * 1;\n            pRelativePos->y = m[0][1] * v.x + m[1][1] * v.y + m[2][1] * v.z + m[3][1] * 1;\n            pRelativePos->z = m[0][2] * v.x + m[1][2] * v.y + m[2][2] * v.z + m[3][2] * 1;\n        }\n\n        /*\n        The direction of the sound needs to also be transformed so that it's relative to the\n        rotation of the listener.\n        */\n        if (pRelativeDir != NULL) {\n            v = spatializerDirection;\n            pRelativeDir->x = m[0][0] * v.x + m[1][0] * v.y + m[2][0] * v.z;\n            pRelativeDir->y = m[0][1] * v.x + m[1][1] * v.y + m[2][1] * v.z;\n            pRelativeDir->z = m[0][2] * v.x + m[1][2] * v.y + m[2][2] * v.z;\n        }\n    }\n}\n\n\n\n\n/**************************************************************************************************************************************************************\n\nResampling\n\n**************************************************************************************************************************************************************/\nMA_API ma_linear_resampler_config ma_linear_resampler_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    ma_linear_resampler_config config;\n    MA_ZERO_OBJECT(&config);\n    config.format           = format;\n    config.channels         = channels;\n    config.sampleRateIn     = sampleRateIn;\n    config.sampleRateOut    = sampleRateOut;\n    config.lpfOrder         = ma_min(MA_DEFAULT_RESAMPLER_LPF_ORDER, MA_MAX_FILTER_ORDER);\n    config.lpfNyquistFactor = 1;\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t x0Offset;\n    size_t x1Offset;\n    size_t lpfOffset;\n} ma_linear_resampler_heap_layout;\n\n\nstatic void ma_linear_resampler_adjust_timer_for_new_rate(ma_linear_resampler* pResampler, ma_uint32 oldSampleRateOut, ma_uint32 newSampleRateOut)\n{\n    /*\n    So what's happening here? Basically we need to adjust the fractional component of the time advance based on the new rate. The old time advance will\n    be based on the old sample rate, but we are needing to adjust it to that it's based on the new sample rate.\n    */\n    ma_uint32 oldRateTimeWhole = pResampler->inTimeFrac / oldSampleRateOut;  /* <-- This should almost never be anything other than 0, but leaving it here to make this more general and robust just in case. */\n    ma_uint32 oldRateTimeFract = pResampler->inTimeFrac % oldSampleRateOut;\n\n    pResampler->inTimeFrac =\n         (oldRateTimeWhole * newSampleRateOut) +\n        ((oldRateTimeFract * newSampleRateOut) / oldSampleRateOut);\n\n    /* Make sure the fractional part is less than the output sample rate. */\n    pResampler->inTimeInt += pResampler->inTimeFrac / pResampler->config.sampleRateOut;\n    pResampler->inTimeFrac = pResampler->inTimeFrac % pResampler->config.sampleRateOut;\n}\n\nstatic ma_result ma_linear_resampler_set_rate_internal(ma_linear_resampler* pResampler, void* pHeap, ma_linear_resampler_heap_layout* pHeapLayout, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut, ma_bool32 isResamplerAlreadyInitialized)\n{\n    ma_result result;\n    ma_uint32 gcf;\n    ma_uint32 lpfSampleRate;\n    double lpfCutoffFrequency;\n    ma_lpf_config lpfConfig;\n    ma_uint32 oldSampleRateOut; /* Required for adjusting time advance down the bottom. */\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (sampleRateIn == 0 || sampleRateOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    oldSampleRateOut = pResampler->config.sampleRateOut;\n\n    pResampler->config.sampleRateIn  = sampleRateIn;\n    pResampler->config.sampleRateOut = sampleRateOut;\n\n    /* Simplify the sample rate. */\n    gcf = ma_gcf_u32(pResampler->config.sampleRateIn, pResampler->config.sampleRateOut);\n    pResampler->config.sampleRateIn  /= gcf;\n    pResampler->config.sampleRateOut /= gcf;\n\n    /* Always initialize the low-pass filter, even when the order is 0. */\n    if (pResampler->config.lpfOrder > MA_MAX_FILTER_ORDER) {\n        return MA_INVALID_ARGS;\n    }\n\n    lpfSampleRate      = (ma_uint32)(ma_max(pResampler->config.sampleRateIn, pResampler->config.sampleRateOut));\n    lpfCutoffFrequency = (   double)(ma_min(pResampler->config.sampleRateIn, pResampler->config.sampleRateOut) * 0.5 * pResampler->config.lpfNyquistFactor);\n\n    lpfConfig = ma_lpf_config_init(pResampler->config.format, pResampler->config.channels, lpfSampleRate, lpfCutoffFrequency, pResampler->config.lpfOrder);\n\n    /*\n    If the resampler is already initialized we don't want to do a fresh initialization of the low-pass filter because it will result in the cached frames\n    getting cleared. Instead we re-initialize the filter which will maintain any cached frames.\n    */\n    if (isResamplerAlreadyInitialized) {\n        result = ma_lpf_reinit(&lpfConfig, &pResampler->lpf);\n    } else {\n        result = ma_lpf_init_preallocated(&lpfConfig, ma_offset_ptr(pHeap, pHeapLayout->lpfOffset), &pResampler->lpf);\n    }\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n\n    pResampler->inAdvanceInt  = pResampler->config.sampleRateIn / pResampler->config.sampleRateOut;\n    pResampler->inAdvanceFrac = pResampler->config.sampleRateIn % pResampler->config.sampleRateOut;\n\n    /* Our timer was based on the old rate. We need to adjust it so that it's based on the new rate. */\n    ma_linear_resampler_adjust_timer_for_new_rate(pResampler, oldSampleRateOut, pResampler->config.sampleRateOut);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_linear_resampler_get_heap_layout(const ma_linear_resampler_config* pConfig, ma_linear_resampler_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->format != ma_format_f32 && pConfig->format != ma_format_s16) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* x0 */\n    pHeapLayout->x0Offset = pHeapLayout->sizeInBytes;\n    if (pConfig->format == ma_format_f32) {\n        pHeapLayout->sizeInBytes += sizeof(float) * pConfig->channels;\n    } else {\n        pHeapLayout->sizeInBytes += sizeof(ma_int16) * pConfig->channels;\n    }\n\n    /* x1 */\n    pHeapLayout->x1Offset = pHeapLayout->sizeInBytes;\n    if (pConfig->format == ma_format_f32) {\n        pHeapLayout->sizeInBytes += sizeof(float) * pConfig->channels;\n    } else {\n        pHeapLayout->sizeInBytes += sizeof(ma_int16) * pConfig->channels;\n    }\n\n    /* LPF */\n    pHeapLayout->lpfOffset = ma_align_64(pHeapLayout->sizeInBytes);\n    {\n        ma_result result;\n        size_t lpfHeapSizeInBytes;\n        ma_lpf_config lpfConfig = ma_lpf_config_init(pConfig->format, pConfig->channels, 1, 1, pConfig->lpfOrder);  /* Sample rate and cutoff frequency do not matter. */\n\n        result = ma_lpf_get_heap_size(&lpfConfig, &lpfHeapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += lpfHeapSizeInBytes;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_linear_resampler_get_heap_size(const ma_linear_resampler_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_linear_resampler_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_linear_resampler_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_linear_resampler_init_preallocated(const ma_linear_resampler_config* pConfig, void* pHeap, ma_linear_resampler* pResampler)\n{\n    ma_result result;\n    ma_linear_resampler_heap_layout heapLayout;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pResampler);\n\n    result = ma_linear_resampler_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pResampler->config = *pConfig;\n\n    pResampler->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    if (pConfig->format == ma_format_f32) {\n        pResampler->x0.f32 = (float*)ma_offset_ptr(pHeap, heapLayout.x0Offset);\n        pResampler->x1.f32 = (float*)ma_offset_ptr(pHeap, heapLayout.x1Offset);\n    } else {\n        pResampler->x0.s16 = (ma_int16*)ma_offset_ptr(pHeap, heapLayout.x0Offset);\n        pResampler->x1.s16 = (ma_int16*)ma_offset_ptr(pHeap, heapLayout.x1Offset);\n    }\n\n    /* Setting the rate will set up the filter and time advances for us. */\n    result = ma_linear_resampler_set_rate_internal(pResampler, pHeap, &heapLayout, pConfig->sampleRateIn, pConfig->sampleRateOut, /* isResamplerAlreadyInitialized = */ MA_FALSE);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pResampler->inTimeInt  = 1;  /* Set this to one to force an input sample to always be loaded for the first output frame. */\n    pResampler->inTimeFrac = 0;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_linear_resampler_init(const ma_linear_resampler_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_linear_resampler* pResampler)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_linear_resampler_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_linear_resampler_init_preallocated(pConfig, pHeap, pResampler);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pResampler->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_linear_resampler_uninit(ma_linear_resampler* pResampler, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pResampler == NULL) {\n        return;\n    }\n\n    ma_lpf_uninit(&pResampler->lpf, pAllocationCallbacks);\n\n    if (pResampler->_ownsHeap) {\n        ma_free(pResampler->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic MA_INLINE ma_int16 ma_linear_resampler_mix_s16(ma_int16 x, ma_int16 y, ma_int32 a, const ma_int32 shift)\n{\n    ma_int32 b;\n    ma_int32 c;\n    ma_int32 r;\n\n    MA_ASSERT(a <= (1<<shift));\n\n    b = x * ((1<<shift) - a);\n    c = y * a;\n    r = b + c;\n\n    return (ma_int16)(r >> shift);\n}\n\nstatic void ma_linear_resampler_interpolate_frame_s16(ma_linear_resampler* pResampler, ma_int16* MA_RESTRICT pFrameOut)\n{\n    ma_uint32 c;\n    ma_uint32 a;\n    const ma_uint32 channels = pResampler->config.channels;\n    const ma_uint32 shift = 12;\n\n    MA_ASSERT(pResampler != NULL);\n    MA_ASSERT(pFrameOut  != NULL);\n\n    a = (pResampler->inTimeFrac << shift) / pResampler->config.sampleRateOut;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        ma_int16 s = ma_linear_resampler_mix_s16(pResampler->x0.s16[c], pResampler->x1.s16[c], a, shift);\n        pFrameOut[c] = s;\n    }\n}\n\n\nstatic void ma_linear_resampler_interpolate_frame_f32(ma_linear_resampler* pResampler, float* MA_RESTRICT pFrameOut)\n{\n    ma_uint32 c;\n    float a;\n    const ma_uint32 channels = pResampler->config.channels;\n\n    MA_ASSERT(pResampler != NULL);\n    MA_ASSERT(pFrameOut  != NULL);\n\n    a = (float)pResampler->inTimeFrac / pResampler->config.sampleRateOut;\n\n    MA_ASSUME(channels > 0);\n    for (c = 0; c < channels; c += 1) {\n        float s = ma_mix_f32_fast(pResampler->x0.f32[c], pResampler->x1.f32[c], a);\n        pFrameOut[c] = s;\n    }\n}\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_s16_downsample(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    const ma_int16* pFramesInS16;\n    /* */ ma_int16* pFramesOutS16;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n\n    MA_ASSERT(pResampler     != NULL);\n    MA_ASSERT(pFrameCountIn  != NULL);\n    MA_ASSERT(pFrameCountOut != NULL);\n\n    pFramesInS16       = (const ma_int16*)pFramesIn;\n    pFramesOutS16      = (      ma_int16*)pFramesOut;\n    frameCountIn       = *pFrameCountIn;\n    frameCountOut      = *pFrameCountOut;\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    while (framesProcessedOut < frameCountOut) {\n        /* Before interpolating we need to load the buffers. When doing this we need to ensure we run every input sample through the filter. */\n        while (pResampler->inTimeInt > 0 && frameCountIn > framesProcessedIn) {\n            ma_uint32 iChannel;\n\n            if (pFramesInS16 != NULL) {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.s16[iChannel] = pResampler->x1.s16[iChannel];\n                    pResampler->x1.s16[iChannel] = pFramesInS16[iChannel];\n                }\n                pFramesInS16 += pResampler->config.channels;\n            } else {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.s16[iChannel] = pResampler->x1.s16[iChannel];\n                    pResampler->x1.s16[iChannel] = 0;\n                }\n            }\n\n            /* Filter. Do not apply filtering if sample rates are the same or else you'll get dangerous glitching. */\n            if (pResampler->config.sampleRateIn != pResampler->config.sampleRateOut) {\n                ma_lpf_process_pcm_frame_s16(&pResampler->lpf, pResampler->x1.s16, pResampler->x1.s16);\n            }\n\n            framesProcessedIn     += 1;\n            pResampler->inTimeInt -= 1;\n        }\n\n        if (pResampler->inTimeInt > 0) {\n            break;  /* Ran out of input data. */\n        }\n\n        /* Getting here means the frames have been loaded and filtered and we can generate the next output frame. */\n        if (pFramesOutS16 != NULL) {\n            MA_ASSERT(pResampler->inTimeInt == 0);\n            ma_linear_resampler_interpolate_frame_s16(pResampler, pFramesOutS16);\n\n            pFramesOutS16 += pResampler->config.channels;\n        }\n\n        framesProcessedOut += 1;\n\n        /* Advance time forward. */\n        pResampler->inTimeInt  += pResampler->inAdvanceInt;\n        pResampler->inTimeFrac += pResampler->inAdvanceFrac;\n        if (pResampler->inTimeFrac >= pResampler->config.sampleRateOut) {\n            pResampler->inTimeFrac -= pResampler->config.sampleRateOut;\n            pResampler->inTimeInt  += 1;\n        }\n    }\n\n    *pFrameCountIn  = framesProcessedIn;\n    *pFrameCountOut = framesProcessedOut;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_s16_upsample(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    const ma_int16* pFramesInS16;\n    /* */ ma_int16* pFramesOutS16;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n\n    MA_ASSERT(pResampler     != NULL);\n    MA_ASSERT(pFrameCountIn  != NULL);\n    MA_ASSERT(pFrameCountOut != NULL);\n\n    pFramesInS16       = (const ma_int16*)pFramesIn;\n    pFramesOutS16      = (      ma_int16*)pFramesOut;\n    frameCountIn       = *pFrameCountIn;\n    frameCountOut      = *pFrameCountOut;\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    while (framesProcessedOut < frameCountOut) {\n        /* Before interpolating we need to load the buffers. */\n        while (pResampler->inTimeInt > 0 && frameCountIn > framesProcessedIn) {\n            ma_uint32 iChannel;\n\n            if (pFramesInS16 != NULL) {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.s16[iChannel] = pResampler->x1.s16[iChannel];\n                    pResampler->x1.s16[iChannel] = pFramesInS16[iChannel];\n                }\n                pFramesInS16 += pResampler->config.channels;\n            } else {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.s16[iChannel] = pResampler->x1.s16[iChannel];\n                    pResampler->x1.s16[iChannel] = 0;\n                }\n            }\n\n            framesProcessedIn     += 1;\n            pResampler->inTimeInt -= 1;\n        }\n\n        if (pResampler->inTimeInt > 0) {\n            break;  /* Ran out of input data. */\n        }\n\n        /* Getting here means the frames have been loaded and we can generate the next output frame. */\n        if (pFramesOutS16 != NULL) {\n            MA_ASSERT(pResampler->inTimeInt == 0);\n            ma_linear_resampler_interpolate_frame_s16(pResampler, pFramesOutS16);\n\n            /* Filter. Do not apply filtering if sample rates are the same or else you'll get dangerous glitching. */\n            if (pResampler->config.sampleRateIn != pResampler->config.sampleRateOut) {\n                ma_lpf_process_pcm_frame_s16(&pResampler->lpf, pFramesOutS16, pFramesOutS16);\n            }\n\n            pFramesOutS16 += pResampler->config.channels;\n        }\n\n        framesProcessedOut += 1;\n\n        /* Advance time forward. */\n        pResampler->inTimeInt  += pResampler->inAdvanceInt;\n        pResampler->inTimeFrac += pResampler->inAdvanceFrac;\n        if (pResampler->inTimeFrac >= pResampler->config.sampleRateOut) {\n            pResampler->inTimeFrac -= pResampler->config.sampleRateOut;\n            pResampler->inTimeInt  += 1;\n        }\n    }\n\n    *pFrameCountIn  = framesProcessedIn;\n    *pFrameCountOut = framesProcessedOut;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_s16(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    MA_ASSERT(pResampler != NULL);\n\n    if (pResampler->config.sampleRateIn > pResampler->config.sampleRateOut) {\n        return ma_linear_resampler_process_pcm_frames_s16_downsample(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    } else {\n        return ma_linear_resampler_process_pcm_frames_s16_upsample(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    }\n}\n\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_f32_downsample(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    const float* pFramesInF32;\n    /* */ float* pFramesOutF32;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n\n    MA_ASSERT(pResampler     != NULL);\n    MA_ASSERT(pFrameCountIn  != NULL);\n    MA_ASSERT(pFrameCountOut != NULL);\n\n    pFramesInF32       = (const float*)pFramesIn;\n    pFramesOutF32      = (      float*)pFramesOut;\n    frameCountIn       = *pFrameCountIn;\n    frameCountOut      = *pFrameCountOut;\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    while (framesProcessedOut < frameCountOut) {\n        /* Before interpolating we need to load the buffers. When doing this we need to ensure we run every input sample through the filter. */\n        while (pResampler->inTimeInt > 0 && frameCountIn > framesProcessedIn) {\n            ma_uint32 iChannel;\n\n            if (pFramesInF32 != NULL) {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.f32[iChannel] = pResampler->x1.f32[iChannel];\n                    pResampler->x1.f32[iChannel] = pFramesInF32[iChannel];\n                }\n                pFramesInF32 += pResampler->config.channels;\n            } else {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.f32[iChannel] = pResampler->x1.f32[iChannel];\n                    pResampler->x1.f32[iChannel] = 0;\n                }\n            }\n\n            /* Filter. Do not apply filtering if sample rates are the same or else you'll get dangerous glitching. */\n            if (pResampler->config.sampleRateIn != pResampler->config.sampleRateOut) {\n                ma_lpf_process_pcm_frame_f32(&pResampler->lpf, pResampler->x1.f32, pResampler->x1.f32);\n            }\n\n            framesProcessedIn     += 1;\n            pResampler->inTimeInt -= 1;\n        }\n\n        if (pResampler->inTimeInt > 0) {\n            break;  /* Ran out of input data. */\n        }\n\n        /* Getting here means the frames have been loaded and filtered and we can generate the next output frame. */\n        if (pFramesOutF32 != NULL) {\n            MA_ASSERT(pResampler->inTimeInt == 0);\n            ma_linear_resampler_interpolate_frame_f32(pResampler, pFramesOutF32);\n\n            pFramesOutF32 += pResampler->config.channels;\n        }\n\n        framesProcessedOut += 1;\n\n        /* Advance time forward. */\n        pResampler->inTimeInt  += pResampler->inAdvanceInt;\n        pResampler->inTimeFrac += pResampler->inAdvanceFrac;\n        if (pResampler->inTimeFrac >= pResampler->config.sampleRateOut) {\n            pResampler->inTimeFrac -= pResampler->config.sampleRateOut;\n            pResampler->inTimeInt  += 1;\n        }\n    }\n\n    *pFrameCountIn  = framesProcessedIn;\n    *pFrameCountOut = framesProcessedOut;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_f32_upsample(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    const float* pFramesInF32;\n    /* */ float* pFramesOutF32;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n\n    MA_ASSERT(pResampler     != NULL);\n    MA_ASSERT(pFrameCountIn  != NULL);\n    MA_ASSERT(pFrameCountOut != NULL);\n\n    pFramesInF32       = (const float*)pFramesIn;\n    pFramesOutF32      = (      float*)pFramesOut;\n    frameCountIn       = *pFrameCountIn;\n    frameCountOut      = *pFrameCountOut;\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    while (framesProcessedOut < frameCountOut) {\n        /* Before interpolating we need to load the buffers. */\n        while (pResampler->inTimeInt > 0 && frameCountIn > framesProcessedIn) {\n            ma_uint32 iChannel;\n\n            if (pFramesInF32 != NULL) {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.f32[iChannel] = pResampler->x1.f32[iChannel];\n                    pResampler->x1.f32[iChannel] = pFramesInF32[iChannel];\n                }\n                pFramesInF32 += pResampler->config.channels;\n            } else {\n                for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n                    pResampler->x0.f32[iChannel] = pResampler->x1.f32[iChannel];\n                    pResampler->x1.f32[iChannel] = 0;\n                }\n            }\n\n            framesProcessedIn     += 1;\n            pResampler->inTimeInt -= 1;\n        }\n\n        if (pResampler->inTimeInt > 0) {\n            break;  /* Ran out of input data. */\n        }\n\n        /* Getting here means the frames have been loaded and we can generate the next output frame. */\n        if (pFramesOutF32 != NULL) {\n            MA_ASSERT(pResampler->inTimeInt == 0);\n            ma_linear_resampler_interpolate_frame_f32(pResampler, pFramesOutF32);\n\n            /* Filter. Do not apply filtering if sample rates are the same or else you'll get dangerous glitching. */\n            if (pResampler->config.sampleRateIn != pResampler->config.sampleRateOut) {\n                ma_lpf_process_pcm_frame_f32(&pResampler->lpf, pFramesOutF32, pFramesOutF32);\n            }\n\n            pFramesOutF32 += pResampler->config.channels;\n        }\n\n        framesProcessedOut += 1;\n\n        /* Advance time forward. */\n        pResampler->inTimeInt  += pResampler->inAdvanceInt;\n        pResampler->inTimeFrac += pResampler->inAdvanceFrac;\n        if (pResampler->inTimeFrac >= pResampler->config.sampleRateOut) {\n            pResampler->inTimeFrac -= pResampler->config.sampleRateOut;\n            pResampler->inTimeInt  += 1;\n        }\n    }\n\n    *pFrameCountIn  = framesProcessedIn;\n    *pFrameCountOut = framesProcessedOut;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_linear_resampler_process_pcm_frames_f32(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    MA_ASSERT(pResampler != NULL);\n\n    if (pResampler->config.sampleRateIn > pResampler->config.sampleRateOut) {\n        return ma_linear_resampler_process_pcm_frames_f32_downsample(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    } else {\n        return ma_linear_resampler_process_pcm_frames_f32_upsample(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    }\n}\n\n\nMA_API ma_result ma_linear_resampler_process_pcm_frames(ma_linear_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*  */ if (pResampler->config.format == ma_format_s16) {\n        return ma_linear_resampler_process_pcm_frames_s16(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    } else if (pResampler->config.format == ma_format_f32) {\n        return ma_linear_resampler_process_pcm_frames_f32(pResampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    } else {\n        /* Should never get here. Getting here means the format is not supported and you didn't check the return value of ma_linear_resampler_init(). */\n        MA_ASSERT(MA_FALSE);\n        return MA_INVALID_ARGS;\n    }\n}\n\n\nMA_API ma_result ma_linear_resampler_set_rate(ma_linear_resampler* pResampler, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    return ma_linear_resampler_set_rate_internal(pResampler, NULL, NULL, sampleRateIn, sampleRateOut, /* isResamplerAlreadyInitialized = */ MA_TRUE);\n}\n\nMA_API ma_result ma_linear_resampler_set_rate_ratio(ma_linear_resampler* pResampler, float ratioInOut)\n{\n    ma_uint32 n;\n    ma_uint32 d;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ratioInOut <= 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    d = 1000000;\n    n = (ma_uint32)(ratioInOut * d);\n\n    if (n == 0) {\n        return MA_INVALID_ARGS; /* Ratio too small. */\n    }\n\n    MA_ASSERT(n != 0);\n\n    return ma_linear_resampler_set_rate(pResampler, n, d);\n}\n\nMA_API ma_uint64 ma_linear_resampler_get_input_latency(const ma_linear_resampler* pResampler)\n{\n    if (pResampler == NULL) {\n        return 0;\n    }\n\n    return 1 + ma_lpf_get_latency(&pResampler->lpf);\n}\n\nMA_API ma_uint64 ma_linear_resampler_get_output_latency(const ma_linear_resampler* pResampler)\n{\n    if (pResampler == NULL) {\n        return 0;\n    }\n\n    return ma_linear_resampler_get_input_latency(pResampler) * pResampler->config.sampleRateOut / pResampler->config.sampleRateIn;\n}\n\nMA_API ma_result ma_linear_resampler_get_required_input_frame_count(const ma_linear_resampler* pResampler, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount)\n{\n    ma_uint64 inputFrameCount;\n\n    if (pInputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pInputFrameCount = 0;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (outputFrameCount == 0) {\n        return MA_SUCCESS;\n    }\n\n    /* Any whole input frames are consumed before the first output frame is generated. */\n    inputFrameCount = pResampler->inTimeInt;\n    outputFrameCount -= 1;\n\n    /* The rest of the output frames can be calculated in constant time. */\n    inputFrameCount += outputFrameCount * pResampler->inAdvanceInt;\n    inputFrameCount += (pResampler->inTimeFrac + (outputFrameCount * pResampler->inAdvanceFrac)) / pResampler->config.sampleRateOut;\n\n    *pInputFrameCount = inputFrameCount;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_linear_resampler_get_expected_output_frame_count(const ma_linear_resampler* pResampler, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount)\n{\n    ma_uint64 outputFrameCount;\n    ma_uint64 preliminaryInputFrameCountFromFrac;\n    ma_uint64 preliminaryInputFrameCount;\n\n    if (pOutputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pOutputFrameCount = 0;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    The first step is to get a preliminary output frame count. This will either be exactly equal to what we need, or less by 1. We need to\n    determine how many input frames will be consumed by this value. If it's greater than our original input frame count it means we won't\n    be able to generate an extra frame because we will have run out of input data. Otherwise we will have enough input for the generation\n    of an extra output frame. This add-by-one logic is necessary due to how the data loading logic works when processing frames.\n    */\n    outputFrameCount = (inputFrameCount * pResampler->config.sampleRateOut) / pResampler->config.sampleRateIn;\n\n    /*\n    We need to determine how many *whole* input frames will have been processed to generate our preliminary output frame count. This is\n    used in the logic below to determine whether or not we need to add an extra output frame.\n    */\n    preliminaryInputFrameCountFromFrac = (pResampler->inTimeFrac + outputFrameCount*pResampler->inAdvanceFrac) / pResampler->config.sampleRateOut;\n    preliminaryInputFrameCount         = (pResampler->inTimeInt  + outputFrameCount*pResampler->inAdvanceInt ) + preliminaryInputFrameCountFromFrac;\n\n    /*\n    If the total number of *whole* input frames that would be required to generate our preliminary output frame count is greater than\n    the amount of whole input frames we have available as input we need to *not* add an extra output frame as there won't be enough data\n    to actually process. Otherwise we need to add the extra output frame.\n    */\n    if (preliminaryInputFrameCount <= inputFrameCount) {\n        outputFrameCount += 1;\n    }\n\n    *pOutputFrameCount = outputFrameCount;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_linear_resampler_reset(ma_linear_resampler* pResampler)\n{\n    ma_uint32 iChannel;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Timers need to be cleared back to zero. */\n    pResampler->inTimeInt  = 1;  /* Set this to one to force an input sample to always be loaded for the first output frame. */\n    pResampler->inTimeFrac = 0;\n\n    /* Cached samples need to be cleared. */\n    if (pResampler->config.format == ma_format_f32) {\n        for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n            pResampler->x0.f32[iChannel] = 0;\n            pResampler->x1.f32[iChannel] = 0;\n        }\n    } else {\n        for (iChannel = 0; iChannel < pResampler->config.channels; iChannel += 1) {\n            pResampler->x0.s16[iChannel] = 0;\n            pResampler->x1.s16[iChannel] = 0;\n        }\n    }\n\n    /* The low pass filter needs to have its cache reset. */\n    ma_lpf_clear_cache(&pResampler->lpf);\n\n    return MA_SUCCESS;\n}\n\n\n\n/* Linear resampler backend vtable. */\nstatic ma_linear_resampler_config ma_resampling_backend_get_config__linear(const ma_resampler_config* pConfig)\n{\n    ma_linear_resampler_config linearConfig;\n\n    linearConfig = ma_linear_resampler_config_init(pConfig->format, pConfig->channels, pConfig->sampleRateIn, pConfig->sampleRateOut);\n    linearConfig.lpfOrder = pConfig->linear.lpfOrder;\n\n    return linearConfig;\n}\n\nstatic ma_result ma_resampling_backend_get_heap_size__linear(void* pUserData, const ma_resampler_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_linear_resampler_config linearConfig;\n\n    (void)pUserData;\n\n    linearConfig = ma_resampling_backend_get_config__linear(pConfig);\n\n    return ma_linear_resampler_get_heap_size(&linearConfig, pHeapSizeInBytes);\n}\n\nstatic ma_result ma_resampling_backend_init__linear(void* pUserData, const ma_resampler_config* pConfig, void* pHeap, ma_resampling_backend** ppBackend)\n{\n    ma_resampler* pResampler = (ma_resampler*)pUserData;\n    ma_result result;\n    ma_linear_resampler_config linearConfig;\n\n    (void)pUserData;\n\n    linearConfig = ma_resampling_backend_get_config__linear(pConfig);\n\n    result = ma_linear_resampler_init_preallocated(&linearConfig, pHeap, &pResampler->state.linear);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *ppBackend = &pResampler->state.linear;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_resampling_backend_uninit__linear(void* pUserData, ma_resampling_backend* pBackend, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    (void)pUserData;\n\n    ma_linear_resampler_uninit((ma_linear_resampler*)pBackend, pAllocationCallbacks);\n}\n\nstatic ma_result ma_resampling_backend_process__linear(void* pUserData, ma_resampling_backend* pBackend, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_process_pcm_frames((ma_linear_resampler*)pBackend, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n}\n\nstatic ma_result ma_resampling_backend_set_rate__linear(void* pUserData, ma_resampling_backend* pBackend, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_set_rate((ma_linear_resampler*)pBackend, sampleRateIn, sampleRateOut);\n}\n\nstatic ma_uint64 ma_resampling_backend_get_input_latency__linear(void* pUserData, const ma_resampling_backend* pBackend)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_get_input_latency((const ma_linear_resampler*)pBackend);\n}\n\nstatic ma_uint64 ma_resampling_backend_get_output_latency__linear(void* pUserData, const ma_resampling_backend* pBackend)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_get_output_latency((const ma_linear_resampler*)pBackend);\n}\n\nstatic ma_result ma_resampling_backend_get_required_input_frame_count__linear(void* pUserData, const ma_resampling_backend* pBackend, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_get_required_input_frame_count((const ma_linear_resampler*)pBackend, outputFrameCount, pInputFrameCount);\n}\n\nstatic ma_result ma_resampling_backend_get_expected_output_frame_count__linear(void* pUserData, const ma_resampling_backend* pBackend, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_get_expected_output_frame_count((const ma_linear_resampler*)pBackend, inputFrameCount, pOutputFrameCount);\n}\n\nstatic ma_result ma_resampling_backend_reset__linear(void* pUserData, ma_resampling_backend* pBackend)\n{\n    (void)pUserData;\n\n    return ma_linear_resampler_reset((ma_linear_resampler*)pBackend);\n}\n\nstatic ma_resampling_backend_vtable g_ma_linear_resampler_vtable =\n{\n    ma_resampling_backend_get_heap_size__linear,\n    ma_resampling_backend_init__linear,\n    ma_resampling_backend_uninit__linear,\n    ma_resampling_backend_process__linear,\n    ma_resampling_backend_set_rate__linear,\n    ma_resampling_backend_get_input_latency__linear,\n    ma_resampling_backend_get_output_latency__linear,\n    ma_resampling_backend_get_required_input_frame_count__linear,\n    ma_resampling_backend_get_expected_output_frame_count__linear,\n    ma_resampling_backend_reset__linear\n};\n\n\n\nMA_API ma_resampler_config ma_resampler_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut, ma_resample_algorithm algorithm)\n{\n    ma_resampler_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format = format;\n    config.channels = channels;\n    config.sampleRateIn = sampleRateIn;\n    config.sampleRateOut = sampleRateOut;\n    config.algorithm = algorithm;\n\n    /* Linear. */\n    config.linear.lpfOrder = ma_min(MA_DEFAULT_RESAMPLER_LPF_ORDER, MA_MAX_FILTER_ORDER);\n\n    return config;\n}\n\nstatic ma_result ma_resampler_get_vtable(const ma_resampler_config* pConfig, ma_resampler* pResampler, ma_resampling_backend_vtable** ppVTable, void** ppUserData)\n{\n    MA_ASSERT(pConfig    != NULL);\n    MA_ASSERT(ppVTable   != NULL);\n    MA_ASSERT(ppUserData != NULL);\n\n    /* Safety. */\n    *ppVTable   = NULL;\n    *ppUserData = NULL;\n\n    switch (pConfig->algorithm)\n    {\n        case ma_resample_algorithm_linear:\n        {\n            *ppVTable   = &g_ma_linear_resampler_vtable;\n            *ppUserData = pResampler;\n        } break;\n\n        case ma_resample_algorithm_custom:\n        {\n            *ppVTable   = pConfig->pBackendVTable;\n            *ppUserData = pConfig->pBackendUserData;\n        } break;\n\n        default: return MA_INVALID_ARGS;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resampler_get_heap_size(const ma_resampler_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_resampling_backend_vtable* pVTable;\n    void* pVTableUserData;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_resampler_get_vtable(pConfig, NULL, &pVTable, &pVTableUserData);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pVTable == NULL || pVTable->onGetHeapSize == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    result = pVTable->onGetHeapSize(pVTableUserData, pConfig, pHeapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resampler_init_preallocated(const ma_resampler_config* pConfig, void* pHeap, ma_resampler* pResampler)\n{\n    ma_result result;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pResampler);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pResampler->_pHeap        = pHeap;\n    pResampler->format        = pConfig->format;\n    pResampler->channels      = pConfig->channels;\n    pResampler->sampleRateIn  = pConfig->sampleRateIn;\n    pResampler->sampleRateOut = pConfig->sampleRateOut;\n\n    result = ma_resampler_get_vtable(pConfig, pResampler, &pResampler->pBackendVTable, &pResampler->pBackendUserData);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onInit == NULL) {\n        return MA_NOT_IMPLEMENTED;  /* onInit not implemented. */\n    }\n\n    result = pResampler->pBackendVTable->onInit(pResampler->pBackendUserData, pConfig, pHeap, &pResampler->pBackend);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resampler_init(const ma_resampler_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_resampler* pResampler)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_resampler_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_resampler_init_preallocated(pConfig, pHeap, pResampler);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pResampler->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_resampler_uninit(ma_resampler* pResampler, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pResampler == NULL) {\n        return;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onUninit == NULL) {\n        return;\n    }\n\n    pResampler->pBackendVTable->onUninit(pResampler->pBackendUserData, pResampler->pBackend, pAllocationCallbacks);\n\n    if (pResampler->_ownsHeap) {\n        ma_free(pResampler->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_resampler_process_pcm_frames(ma_resampler* pResampler, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFrameCountOut == NULL && pFrameCountIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onProcess == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pResampler->pBackendVTable->onProcess(pResampler->pBackendUserData, pResampler->pBackend, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n}\n\nMA_API ma_result ma_resampler_set_rate(ma_resampler* pResampler, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    ma_result result;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (sampleRateIn == 0 || sampleRateOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onSetRate == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    result = pResampler->pBackendVTable->onSetRate(pResampler->pBackendUserData, pResampler->pBackend, sampleRateIn, sampleRateOut);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pResampler->sampleRateIn  = sampleRateIn;\n    pResampler->sampleRateOut = sampleRateOut;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resampler_set_rate_ratio(ma_resampler* pResampler, float ratio)\n{\n    ma_uint32 n;\n    ma_uint32 d;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ratio <= 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    d = 1000;\n    n = (ma_uint32)(ratio * d);\n\n    if (n == 0) {\n        return MA_INVALID_ARGS; /* Ratio too small. */\n    }\n\n    MA_ASSERT(n != 0);\n\n    return ma_resampler_set_rate(pResampler, n, d);\n}\n\nMA_API ma_uint64 ma_resampler_get_input_latency(const ma_resampler* pResampler)\n{\n    if (pResampler == NULL) {\n        return 0;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onGetInputLatency == NULL) {\n        return 0;\n    }\n\n    return pResampler->pBackendVTable->onGetInputLatency(pResampler->pBackendUserData, pResampler->pBackend);\n}\n\nMA_API ma_uint64 ma_resampler_get_output_latency(const ma_resampler* pResampler)\n{\n    if (pResampler == NULL) {\n        return 0;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onGetOutputLatency == NULL) {\n        return 0;\n    }\n\n    return pResampler->pBackendVTable->onGetOutputLatency(pResampler->pBackendUserData, pResampler->pBackend);\n}\n\nMA_API ma_result ma_resampler_get_required_input_frame_count(const ma_resampler* pResampler, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount)\n{\n    if (pInputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pInputFrameCount = 0;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onGetRequiredInputFrameCount == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pResampler->pBackendVTable->onGetRequiredInputFrameCount(pResampler->pBackendUserData, pResampler->pBackend, outputFrameCount, pInputFrameCount);\n}\n\nMA_API ma_result ma_resampler_get_expected_output_frame_count(const ma_resampler* pResampler, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount)\n{\n    if (pOutputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pOutputFrameCount = 0;\n\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onGetExpectedOutputFrameCount == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pResampler->pBackendVTable->onGetExpectedOutputFrameCount(pResampler->pBackendUserData, pResampler->pBackend, inputFrameCount, pOutputFrameCount);\n}\n\nMA_API ma_result ma_resampler_reset(ma_resampler* pResampler)\n{\n    if (pResampler == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResampler->pBackendVTable == NULL || pResampler->pBackendVTable->onReset == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pResampler->pBackendVTable->onReset(pResampler->pBackendUserData, pResampler->pBackend);\n}\n\n/**************************************************************************************************************************************************************\n\nChannel Conversion\n\n**************************************************************************************************************************************************************/\n#ifndef MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT\n#define MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT  12\n#endif\n\n#define MA_PLANE_LEFT      0\n#define MA_PLANE_RIGHT     1\n#define MA_PLANE_FRONT     2\n#define MA_PLANE_BACK      3\n#define MA_PLANE_BOTTOM    4\n#define MA_PLANE_TOP       5\n\nstatic float g_maChannelPlaneRatios[MA_CHANNEL_POSITION_COUNT][6] = {\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_NONE */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_MONO */\n    { 0.5f,  0.0f,  0.5f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_FRONT_LEFT */\n    { 0.0f,  0.5f,  0.5f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_FRONT_RIGHT */\n    { 0.0f,  0.0f,  1.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_FRONT_CENTER */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_LFE */\n    { 0.5f,  0.0f,  0.0f,  0.5f,  0.0f,  0.0f},  /* MA_CHANNEL_BACK_LEFT */\n    { 0.0f,  0.5f,  0.0f,  0.5f,  0.0f,  0.0f},  /* MA_CHANNEL_BACK_RIGHT */\n    { 0.25f, 0.0f,  0.75f, 0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_FRONT_LEFT_CENTER */\n    { 0.0f,  0.25f, 0.75f, 0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_FRONT_RIGHT_CENTER */\n    { 0.0f,  0.0f,  0.0f,  1.0f,  0.0f,  0.0f},  /* MA_CHANNEL_BACK_CENTER */\n    { 1.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_SIDE_LEFT */\n    { 0.0f,  1.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_SIDE_RIGHT */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  1.0f},  /* MA_CHANNEL_TOP_CENTER */\n    { 0.33f, 0.0f,  0.33f, 0.0f,  0.0f,  0.34f}, /* MA_CHANNEL_TOP_FRONT_LEFT */\n    { 0.0f,  0.0f,  0.5f,  0.0f,  0.0f,  0.5f},  /* MA_CHANNEL_TOP_FRONT_CENTER */\n    { 0.0f,  0.33f, 0.33f, 0.0f,  0.0f,  0.34f}, /* MA_CHANNEL_TOP_FRONT_RIGHT */\n    { 0.33f, 0.0f,  0.0f,  0.33f, 0.0f,  0.34f}, /* MA_CHANNEL_TOP_BACK_LEFT */\n    { 0.0f,  0.0f,  0.0f,  0.5f,  0.0f,  0.5f},  /* MA_CHANNEL_TOP_BACK_CENTER */\n    { 0.0f,  0.33f, 0.0f,  0.33f, 0.0f,  0.34f}, /* MA_CHANNEL_TOP_BACK_RIGHT */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_0 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_1 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_2 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_3 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_4 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_5 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_6 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_7 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_8 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_9 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_10 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_11 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_12 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_13 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_14 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_15 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_16 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_17 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_18 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_19 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_20 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_21 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_22 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_23 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_24 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_25 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_26 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_27 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_28 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_29 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_30 */\n    { 0.0f,  0.0f,  0.0f,  0.0f,  0.0f,  0.0f},  /* MA_CHANNEL_AUX_31 */\n};\n\nstatic float ma_calculate_channel_position_rectangular_weight(ma_channel channelPositionA, ma_channel channelPositionB)\n{\n    /*\n    Imagine the following simplified example: You have a single input speaker which is the front/left speaker which you want to convert to\n    the following output configuration:\n\n     - front/left\n     - side/left\n     - back/left\n\n    The front/left output is easy - it the same speaker position so it receives the full contribution of the front/left input. The amount\n    of contribution to apply to the side/left and back/left speakers, however, is a bit more complicated.\n\n    Imagine the front/left speaker as emitting audio from two planes - the front plane and the left plane. You can think of the front/left\n    speaker emitting half of its total volume from the front, and the other half from the left. Since part of its volume is being emitted\n    from the left side, and the side/left and back/left channels also emit audio from the left plane, one would expect that they would\n    receive some amount of contribution from front/left speaker. The amount of contribution depends on how many planes are shared between\n    the two speakers. Note that in the examples below I've added a top/front/left speaker as an example just to show how the math works\n    across 3 spatial dimensions.\n\n    The first thing to do is figure out how each speaker's volume is spread over each of plane:\n     - front/left:     2 planes (front and left)      = 1/2 = half its total volume on each plane\n     - side/left:      1 plane (left only)            = 1/1 = entire volume from left plane\n     - back/left:      2 planes (back and left)       = 1/2 = half its total volume on each plane\n     - top/front/left: 3 planes (top, front and left) = 1/3 = one third its total volume on each plane\n\n    The amount of volume each channel contributes to each of its planes is what controls how much it is willing to given and take to other\n    channels on the same plane. The volume that is willing to the given by one channel is multiplied by the volume that is willing to be\n    taken by the other to produce the final contribution.\n    */\n\n    /* Contribution = Sum(Volume to Give * Volume to Take) */\n    float contribution =\n        g_maChannelPlaneRatios[channelPositionA][0] * g_maChannelPlaneRatios[channelPositionB][0] +\n        g_maChannelPlaneRatios[channelPositionA][1] * g_maChannelPlaneRatios[channelPositionB][1] +\n        g_maChannelPlaneRatios[channelPositionA][2] * g_maChannelPlaneRatios[channelPositionB][2] +\n        g_maChannelPlaneRatios[channelPositionA][3] * g_maChannelPlaneRatios[channelPositionB][3] +\n        g_maChannelPlaneRatios[channelPositionA][4] * g_maChannelPlaneRatios[channelPositionB][4] +\n        g_maChannelPlaneRatios[channelPositionA][5] * g_maChannelPlaneRatios[channelPositionB][5];\n\n    return contribution;\n}\n\nMA_API ma_channel_converter_config ma_channel_converter_config_init(ma_format format, ma_uint32 channelsIn, const ma_channel* pChannelMapIn, ma_uint32 channelsOut, const ma_channel* pChannelMapOut, ma_channel_mix_mode mixingMode)\n{\n    ma_channel_converter_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format         = format;\n    config.channelsIn     = channelsIn;\n    config.channelsOut    = channelsOut;\n    config.pChannelMapIn  = pChannelMapIn;\n    config.pChannelMapOut = pChannelMapOut;\n    config.mixingMode     = mixingMode;\n\n    return config;\n}\n\nstatic ma_int32 ma_channel_converter_float_to_fixed(float x)\n{\n    return (ma_int32)(x * (1<<MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT));\n}\n\nstatic ma_uint32 ma_channel_map_get_spatial_channel_count(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    ma_uint32 spatialChannelCount = 0;\n    ma_uint32 iChannel;\n\n    MA_ASSERT(pChannelMap != NULL);\n    MA_ASSERT(channels > 0);\n\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        if (ma_is_spatial_channel_position(ma_channel_map_get_channel(pChannelMap, channels, iChannel))) {\n            spatialChannelCount++;\n        }\n    }\n\n    return spatialChannelCount;\n}\n\nstatic ma_bool32 ma_is_spatial_channel_position(ma_channel channelPosition)\n{\n    int i;\n\n    if (channelPosition == MA_CHANNEL_NONE || channelPosition == MA_CHANNEL_MONO || channelPosition == MA_CHANNEL_LFE) {\n        return MA_FALSE;\n    }\n\n    if (channelPosition >= MA_CHANNEL_AUX_0) {\n        return MA_FALSE;\n    }\n\n    for (i = 0; i < 6; ++i) {   /* Each side of a cube. */\n        if (g_maChannelPlaneRatios[channelPosition][i] != 0) {\n            return MA_TRUE;\n        }\n    }\n\n    return MA_FALSE;\n}\n\n\nstatic ma_bool32 ma_channel_map_is_passthrough(const ma_channel* pChannelMapIn, ma_uint32 channelsIn, const ma_channel* pChannelMapOut, ma_uint32 channelsOut)\n{\n    if (channelsOut == channelsIn) {\n        return ma_channel_map_is_equal(pChannelMapOut, pChannelMapIn, channelsOut);\n    } else {\n        return MA_FALSE;    /* Channel counts differ, so cannot be a passthrough. */\n    }\n}\n\nstatic ma_channel_conversion_path ma_channel_map_get_conversion_path(const ma_channel* pChannelMapIn, ma_uint32 channelsIn, const ma_channel* pChannelMapOut, ma_uint32 channelsOut, ma_channel_mix_mode mode)\n{\n    if (ma_channel_map_is_passthrough(pChannelMapIn, channelsIn, pChannelMapOut, channelsOut)) {\n        return ma_channel_conversion_path_passthrough;\n    }\n\n    if (channelsOut == 1 && (pChannelMapOut == NULL || pChannelMapOut[0] == MA_CHANNEL_MONO)) {\n        return ma_channel_conversion_path_mono_out;\n    }\n\n    if (channelsIn == 1 && (pChannelMapIn == NULL || pChannelMapIn[0] == MA_CHANNEL_MONO)) {\n        return ma_channel_conversion_path_mono_in;\n    }\n\n    if (mode == ma_channel_mix_mode_custom_weights) {\n        return ma_channel_conversion_path_weights;\n    }\n\n    /*\n    We can use a simple shuffle if both channel maps have the same channel count and all channel\n    positions are present in both.\n    */\n    if (channelsIn == channelsOut) {\n        ma_uint32 iChannelIn;\n        ma_bool32 areAllChannelPositionsPresent = MA_TRUE;\n        for (iChannelIn = 0; iChannelIn < channelsIn; ++iChannelIn) {\n            ma_bool32 isInputChannelPositionInOutput = ma_channel_map_contains_channel_position(channelsOut, pChannelMapOut, ma_channel_map_get_channel(pChannelMapIn, channelsIn, iChannelIn));\n            if (!isInputChannelPositionInOutput) {\n                areAllChannelPositionsPresent = MA_FALSE;\n                break;\n            }\n        }\n\n        if (areAllChannelPositionsPresent) {\n            return ma_channel_conversion_path_shuffle;\n        }\n    }\n\n    /* Getting here means we'll need to use weights. */\n    return ma_channel_conversion_path_weights;\n}\n\n\nstatic ma_result ma_channel_map_build_shuffle_table(const ma_channel* pChannelMapIn, ma_uint32 channelCountIn, const ma_channel* pChannelMapOut, ma_uint32 channelCountOut, ma_uint8* pShuffleTable)\n{\n    ma_uint32 iChannelIn;\n    ma_uint32 iChannelOut;\n\n    if (pShuffleTable == NULL || channelCountIn == 0 || channelCountOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    When building the shuffle table we just do a 1:1 mapping based on the first occurrence of a channel. If the\n    input channel has more than one occurrence of a channel position, the second one will be ignored.\n    */\n    for (iChannelOut = 0; iChannelOut < channelCountOut; iChannelOut += 1) {\n        ma_channel channelOut;\n\n        /* Default to MA_CHANNEL_INDEX_NULL so that if a mapping is not found it'll be set appropriately. */\n        pShuffleTable[iChannelOut] = MA_CHANNEL_INDEX_NULL;\n\n        channelOut = ma_channel_map_get_channel(pChannelMapOut, channelCountOut, iChannelOut);\n        for (iChannelIn = 0; iChannelIn < channelCountIn; iChannelIn += 1) {\n            ma_channel channelIn;\n\n            channelIn = ma_channel_map_get_channel(pChannelMapIn, channelCountIn, iChannelIn);\n            if (channelOut == channelIn) {\n                pShuffleTable[iChannelOut] = (ma_uint8)iChannelIn;\n                break;\n            }\n\n            /*\n            Getting here means the channels don't exactly match, but we are going to support some\n            relaxed matching for practicality. If, for example, there are two stereo channel maps,\n            but one uses front left/right and the other uses side left/right, it makes logical\n            sense to just map these. The way we'll do it is we'll check if there is a logical\n            corresponding mapping, and if so, apply it, but we will *not* break from the loop,\n            thereby giving the loop a chance to find an exact match later which will take priority.\n            */\n            switch (channelOut)\n            {\n                /* Left channels. */\n                case MA_CHANNEL_FRONT_LEFT:\n                case MA_CHANNEL_SIDE_LEFT:\n                {\n                    switch (channelIn) {\n                        case MA_CHANNEL_FRONT_LEFT:\n                        case MA_CHANNEL_SIDE_LEFT:\n                        {\n                            pShuffleTable[iChannelOut] = (ma_uint8)iChannelIn;\n                        } break;\n                    }\n                } break;\n\n                /* Right channels. */\n                case MA_CHANNEL_FRONT_RIGHT:\n                case MA_CHANNEL_SIDE_RIGHT:\n                {\n                    switch (channelIn) {\n                        case MA_CHANNEL_FRONT_RIGHT:\n                        case MA_CHANNEL_SIDE_RIGHT:\n                        {\n                            pShuffleTable[iChannelOut] = (ma_uint8)iChannelIn;\n                        } break;\n                    }\n                } break;\n\n                default: break;\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic void ma_channel_map_apply_shuffle_table_u8(ma_uint8* pFramesOut, ma_uint32 channelsOut, const ma_uint8* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n            ma_uint8 iChannelIn = pShuffleTable[iChannelOut];\n            if (iChannelIn < channelsIn) {  /* For safety, and to deal with MA_CHANNEL_INDEX_NULL. */\n                pFramesOut[iChannelOut] = pFramesIn[iChannelIn];\n            } else {\n                pFramesOut[iChannelOut] = 0;\n            }\n        }\n\n        pFramesOut += channelsOut;\n        pFramesIn  += channelsIn;\n    }\n}\n\nstatic void ma_channel_map_apply_shuffle_table_s16(ma_int16* pFramesOut, ma_uint32 channelsOut, const ma_int16* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n            ma_uint8 iChannelIn = pShuffleTable[iChannelOut];\n            if (iChannelIn < channelsIn) {  /* For safety, and to deal with MA_CHANNEL_INDEX_NULL. */\n                pFramesOut[iChannelOut] = pFramesIn[iChannelIn];\n            } else {\n                pFramesOut[iChannelOut] = 0;\n            }\n        }\n\n        pFramesOut += channelsOut;\n        pFramesIn  += channelsIn;\n    }\n}\n\nstatic void ma_channel_map_apply_shuffle_table_s24(ma_uint8* pFramesOut, ma_uint32 channelsOut, const ma_uint8* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n            ma_uint8 iChannelIn = pShuffleTable[iChannelOut];\n            if (iChannelIn < channelsIn) {  /* For safety, and to deal with MA_CHANNEL_INDEX_NULL. */\n                pFramesOut[iChannelOut*3 + 0] = pFramesIn[iChannelIn*3 + 0];\n                pFramesOut[iChannelOut*3 + 1] = pFramesIn[iChannelIn*3 + 1];\n                pFramesOut[iChannelOut*3 + 2] = pFramesIn[iChannelIn*3 + 2];\n            } else {\n                pFramesOut[iChannelOut*3 + 0] = 0;\n            }   pFramesOut[iChannelOut*3 + 1] = 0;\n        }       pFramesOut[iChannelOut*3 + 2] = 0;\n\n        pFramesOut += channelsOut*3;\n        pFramesIn  += channelsIn*3;\n    }\n}\n\nstatic void ma_channel_map_apply_shuffle_table_s32(ma_int32* pFramesOut, ma_uint32 channelsOut, const ma_int32* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n            ma_uint8 iChannelIn = pShuffleTable[iChannelOut];\n            if (iChannelIn < channelsIn) {  /* For safety, and to deal with MA_CHANNEL_INDEX_NULL. */\n                pFramesOut[iChannelOut] = pFramesIn[iChannelIn];\n            } else {\n                pFramesOut[iChannelOut] = 0;\n            }\n        }\n\n        pFramesOut += channelsOut;\n        pFramesIn  += channelsIn;\n    }\n}\n\nstatic void ma_channel_map_apply_shuffle_table_f32(float* pFramesOut, ma_uint32 channelsOut, const float* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n            ma_uint8 iChannelIn = pShuffleTable[iChannelOut];\n            if (iChannelIn < channelsIn) {  /* For safety, and to deal with MA_CHANNEL_INDEX_NULL. */\n                pFramesOut[iChannelOut] = pFramesIn[iChannelIn];\n            } else {\n                pFramesOut[iChannelOut] = 0;\n            }\n        }\n\n        pFramesOut += channelsOut;\n        pFramesIn  += channelsIn;\n    }\n}\n\nstatic ma_result ma_channel_map_apply_shuffle_table(void* pFramesOut, ma_uint32 channelsOut, const void* pFramesIn, ma_uint32 channelsIn, ma_uint64 frameCount, const ma_uint8* pShuffleTable, ma_format format)\n{\n    if (pFramesOut == NULL || pFramesIn == NULL || channelsOut == 0 || pShuffleTable == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    switch (format)\n    {\n        case ma_format_u8:\n        {\n            ma_channel_map_apply_shuffle_table_u8((ma_uint8*)pFramesOut, channelsOut, (const ma_uint8*)pFramesIn, channelsIn, frameCount, pShuffleTable);\n        } break;\n\n        case ma_format_s16:\n        {\n            ma_channel_map_apply_shuffle_table_s16((ma_int16*)pFramesOut, channelsOut, (const ma_int16*)pFramesIn, channelsIn, frameCount, pShuffleTable);\n        } break;\n\n        case ma_format_s24:\n        {\n            ma_channel_map_apply_shuffle_table_s24((ma_uint8*)pFramesOut, channelsOut, (const ma_uint8*)pFramesIn, channelsIn, frameCount, pShuffleTable);\n        } break;\n\n        case ma_format_s32:\n        {\n            ma_channel_map_apply_shuffle_table_s32((ma_int32*)pFramesOut, channelsOut, (const ma_int32*)pFramesIn, channelsIn, frameCount, pShuffleTable);\n        } break;\n\n        case ma_format_f32:\n        {\n            ma_channel_map_apply_shuffle_table_f32((float*)pFramesOut, channelsOut, (const float*)pFramesIn, channelsIn, frameCount, pShuffleTable);\n        } break;\n\n        default: return MA_INVALID_ARGS;    /* Unknown format. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_channel_map_apply_mono_out_f32(float* pFramesOut, const float* pFramesIn, const ma_channel* pChannelMapIn, ma_uint32 channelsIn, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelIn;\n    ma_uint32 accumulationCount;\n\n    if (pFramesOut == NULL || pFramesIn == NULL || channelsIn == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* In this case the output stream needs to be the average of all channels, ignoring NONE. */\n\n    /* A quick pre-processing step to get the accumulation counter since we're ignoring NONE channels. */\n    accumulationCount = 0;\n    for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n        if (ma_channel_map_get_channel(pChannelMapIn, channelsIn, iChannelIn) != MA_CHANNEL_NONE) {\n            accumulationCount += 1;\n        }\n    }\n\n    if (accumulationCount > 0) {    /* <-- Prevent a division by zero. */\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float accumulation = 0;\n\n            for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                ma_channel channelIn = ma_channel_map_get_channel(pChannelMapIn, channelsIn, iChannelIn);\n                if (channelIn != MA_CHANNEL_NONE) {\n                    accumulation += pFramesIn[iChannelIn];\n                }\n            }\n\n            pFramesOut[0] = accumulation / accumulationCount;\n            pFramesOut += 1;\n            pFramesIn  += channelsIn;\n        }\n    } else {\n        ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, 1);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_channel_map_apply_mono_in_f32(float* MA_RESTRICT pFramesOut, const ma_channel* pChannelMapOut, ma_uint32 channelsOut, const float* MA_RESTRICT pFramesIn, ma_uint64 frameCount, ma_mono_expansion_mode monoExpansionMode)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannelOut;\n\n    if (pFramesOut == NULL || channelsOut == 0 || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Note that the MA_CHANNEL_NONE channel must be ignored in all cases. */\n    switch (monoExpansionMode)\n    {\n        case ma_mono_expansion_mode_average:\n        {\n            float weight;\n            ma_uint32 validChannelCount = 0;\n\n            for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                if (channelOut != MA_CHANNEL_NONE) {\n                    validChannelCount += 1;\n                }\n            }\n\n            weight = 1.0f / validChannelCount;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                    ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                    if (channelOut != MA_CHANNEL_NONE) {\n                        pFramesOut[iChannelOut] = pFramesIn[0] * weight;\n                    }\n                }\n\n                pFramesOut += channelsOut;\n                pFramesIn  += 1;\n            }\n        } break;\n\n        case ma_mono_expansion_mode_stereo_only:\n        {\n            if (channelsOut >= 2) {\n                ma_uint32 iChannelLeft  = (ma_uint32)-1;\n                ma_uint32 iChannelRight = (ma_uint32)-1;\n\n                /*\n                We first need to find our stereo channels. We prefer front-left and front-right, but\n                if they're not available, we'll also try side-left and side-right. If neither are\n                available we'll fall through to the default case below.\n                */\n                for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                    ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                    if (channelOut == MA_CHANNEL_SIDE_LEFT) {\n                        iChannelLeft  = iChannelOut;\n                    }\n                    if (channelOut == MA_CHANNEL_SIDE_RIGHT) {\n                        iChannelRight = iChannelOut;\n                    }\n                }\n\n                for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                    ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                    if (channelOut == MA_CHANNEL_FRONT_LEFT) {\n                        iChannelLeft  = iChannelOut;\n                    }\n                    if (channelOut == MA_CHANNEL_FRONT_RIGHT) {\n                        iChannelRight = iChannelOut;\n                    }\n                }\n\n\n                if (iChannelLeft != (ma_uint32)-1 && iChannelRight != (ma_uint32)-1) {\n                    /* We found our stereo channels so we can duplicate the signal across those channels. */\n                    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                            ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                            if (channelOut != MA_CHANNEL_NONE) {\n                                if (iChannelOut == iChannelLeft || iChannelOut == iChannelRight) {\n                                    pFramesOut[iChannelOut] = pFramesIn[0];\n                                } else {\n                                    pFramesOut[iChannelOut] = 0.0f;\n                                }\n                            }\n                        }\n\n                        pFramesOut += channelsOut;\n                        pFramesIn  += 1;\n                    }\n\n                    break;  /* Get out of the switch. */\n                } else {\n                    /* Fallthrough. Does not have left and right channels. */\n                    goto default_handler;\n                }\n            } else {\n                /* Fallthrough. Does not have stereo channels. */\n                goto default_handler;\n            }\n        };  /* Fallthrough. See comments above. */\n\n        case ma_mono_expansion_mode_duplicate:\n        default:\n        {\n            default_handler:\n            {\n                if (channelsOut <= MA_MAX_CHANNELS) {\n                    ma_bool32 hasEmptyChannel = MA_FALSE;\n                    ma_channel channelPositions[MA_MAX_CHANNELS];\n                    for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                        channelPositions[iChannelOut] = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                        if (channelPositions[iChannelOut] == MA_CHANNEL_NONE) {\n                            hasEmptyChannel = MA_TRUE;\n                        }\n                    }\n\n                    if (hasEmptyChannel == MA_FALSE) {\n                        /*\n                        Faster path when there's no MA_CHANNEL_NONE channel positions. This should hopefully\n                        help the compiler with auto-vectorization.m\n                        */\n                        if (channelsOut == 2) {\n                        #if defined(MA_SUPPORT_SSE2)\n                            if (ma_has_sse2()) {\n                                /* We want to do two frames in each iteration. */\n                                ma_uint64 unrolledFrameCount = frameCount >> 1;\n\n                                for (iFrame = 0; iFrame < unrolledFrameCount; iFrame += 1) {\n                                    __m128 in0 = _mm_set1_ps(pFramesIn[iFrame*2 + 0]);\n                                    __m128 in1 = _mm_set1_ps(pFramesIn[iFrame*2 + 1]);\n                                    _mm_storeu_ps(&pFramesOut[iFrame*4 + 0], _mm_shuffle_ps(in0, in1, _MM_SHUFFLE(0, 0, 0, 0)));\n                                }\n\n                                /* Tail. */\n                                iFrame = unrolledFrameCount << 1;\n                                goto generic_on_fastpath;\n                            } else\n                        #endif\n                            {\n                                for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                                    for (iChannelOut = 0; iChannelOut < 2; iChannelOut += 1) {\n                                        pFramesOut[iFrame*2 + iChannelOut] = pFramesIn[iFrame];\n                                    }\n                                }\n                            }\n                        } else if (channelsOut == 6) {\n                        #if defined(MA_SUPPORT_SSE2)\n                            if (ma_has_sse2()) {\n                                /* We want to do two frames in each iteration so we can have a multiple of 4 samples. */\n                                ma_uint64 unrolledFrameCount = frameCount >> 1;\n\n                                for (iFrame = 0; iFrame < unrolledFrameCount; iFrame += 1) {\n                                    __m128 in0 = _mm_set1_ps(pFramesIn[iFrame*2 + 0]);\n                                    __m128 in1 = _mm_set1_ps(pFramesIn[iFrame*2 + 1]);\n\n                                    _mm_storeu_ps(&pFramesOut[iFrame*12 + 0], in0);\n                                    _mm_storeu_ps(&pFramesOut[iFrame*12 + 4], _mm_shuffle_ps(in0, in1, _MM_SHUFFLE(0, 0, 0, 0)));\n                                    _mm_storeu_ps(&pFramesOut[iFrame*12 + 8], in1);\n                                }\n\n                                /* Tail. */\n                                iFrame = unrolledFrameCount << 1;\n                                goto generic_on_fastpath;\n                            } else\n                        #endif\n                            {\n                                for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                                    for (iChannelOut = 0; iChannelOut < 6; iChannelOut += 1) {\n                                        pFramesOut[iFrame*6 + iChannelOut] = pFramesIn[iFrame];\n                                    }\n                                }\n                            }\n                        } else if (channelsOut == 8) {\n                        #if defined(MA_SUPPORT_SSE2)\n                            if (ma_has_sse2()) {\n                                for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                                    __m128 in = _mm_set1_ps(pFramesIn[iFrame]);\n                                    _mm_storeu_ps(&pFramesOut[iFrame*8 + 0], in);\n                                    _mm_storeu_ps(&pFramesOut[iFrame*8 + 4], in);\n                                }\n                            } else\n                        #endif\n                            {\n                                for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                                    for (iChannelOut = 0; iChannelOut < 8; iChannelOut += 1) {\n                                        pFramesOut[iFrame*8 + iChannelOut] = pFramesIn[iFrame];\n                                    }\n                                }\n                            }\n                        } else {\n                            iFrame = 0;\n\n                            #if defined(MA_SUPPORT_SSE2)    /* For silencing a warning with non-x86 builds. */\n                            generic_on_fastpath:\n                            #endif\n                            {\n                                for (; iFrame < frameCount; iFrame += 1) {\n                                    for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                                        pFramesOut[iFrame*channelsOut + iChannelOut] = pFramesIn[iFrame];\n                                    }\n                                }\n                            }\n                        }\n                    } else {\n                        /* Slow path. Need to handle MA_CHANNEL_NONE. */\n                        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                            for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                                if (channelPositions[iChannelOut] != MA_CHANNEL_NONE) {\n                                    pFramesOut[iFrame*channelsOut + iChannelOut] = pFramesIn[iFrame];\n                                }\n                            }\n                        }\n                    }\n                } else {\n                    /* Slow path. Too many channels to store on the stack. */\n                    for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                        for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                            ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                            if (channelOut != MA_CHANNEL_NONE) {\n                                pFramesOut[iFrame*channelsOut + iChannelOut] = pFramesIn[iFrame];\n                            }\n                        }\n                    }\n                }\n            }\n        } break;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_channel_map_apply_f32(float* pFramesOut, const ma_channel* pChannelMapOut, ma_uint32 channelsOut, const float* pFramesIn, const ma_channel* pChannelMapIn, ma_uint32 channelsIn, ma_uint64 frameCount, ma_channel_mix_mode mode, ma_mono_expansion_mode monoExpansionMode)\n{\n    ma_channel_conversion_path conversionPath = ma_channel_map_get_conversion_path(pChannelMapIn, channelsIn, pChannelMapOut, channelsOut, mode);\n\n    /* Optimized Path: Passthrough */\n    if (conversionPath == ma_channel_conversion_path_passthrough) {\n        ma_copy_pcm_frames(pFramesOut, pFramesIn, frameCount, ma_format_f32, channelsOut);\n        return;\n    }\n\n    /* Special Path: Mono Output. */\n    if (conversionPath == ma_channel_conversion_path_mono_out) {\n        ma_channel_map_apply_mono_out_f32(pFramesOut, pFramesIn, pChannelMapIn, channelsIn, frameCount);\n        return;\n    }\n\n    /* Special Path: Mono Input. */\n    if (conversionPath == ma_channel_conversion_path_mono_in) {\n        ma_channel_map_apply_mono_in_f32(pFramesOut, pChannelMapOut, channelsOut, pFramesIn, frameCount, monoExpansionMode);\n        return;\n    }\n\n    /* Getting here means we aren't running on an optimized conversion path. */\n    if (channelsOut <= MA_MAX_CHANNELS) {\n        ma_result result;\n\n        if (mode == ma_channel_mix_mode_simple) {\n            ma_channel shuffleTable[MA_MAX_CHANNELS];\n\n            result = ma_channel_map_build_shuffle_table(pChannelMapIn, channelsIn, pChannelMapOut, channelsOut, shuffleTable);\n            if (result != MA_SUCCESS) {\n                return;\n            }\n\n            result = ma_channel_map_apply_shuffle_table(pFramesOut, channelsOut, pFramesIn, channelsIn, frameCount, shuffleTable, ma_format_f32);\n            if (result != MA_SUCCESS) {\n                return;\n            }\n        } else {\n            ma_uint32 iFrame;\n            ma_uint32 iChannelOut;\n            ma_uint32 iChannelIn;\n            float weights[32][32];  /* Do not use MA_MAX_CHANNELS here! */\n\n            /*\n            If we have a small enough number of channels, pre-compute the weights. Otherwise we'll just need to\n            fall back to a slower path because otherwise we'll run out of stack space.\n            */\n            if (channelsIn <= ma_countof(weights) && channelsOut <= ma_countof(weights)) {\n                /* Pre-compute weights. */\n                for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                    ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n                    for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                        ma_channel channelIn = ma_channel_map_get_channel(pChannelMapIn, channelsIn, iChannelIn);\n                        weights[iChannelOut][iChannelIn] = ma_calculate_channel_position_rectangular_weight(channelOut, channelIn);\n                    }\n                }\n\n                iFrame = 0;\n\n                /* Experiment: Try an optimized unroll for some specific cases to see how it improves performance. RESULT: Good gains. */\n                if (channelsOut == 8) {\n                    /* Experiment 2: Expand the inner loop to see what kind of different it makes. RESULT: Small, but worthwhile gain. */\n                    if (channelsIn == 2) {\n                        for (; iFrame < frameCount; iFrame += 1) {\n                            float accumulation[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };\n\n                            accumulation[0] += pFramesIn[iFrame*2 + 0] * weights[0][0];\n                            accumulation[1] += pFramesIn[iFrame*2 + 0] * weights[1][0];\n                            accumulation[2] += pFramesIn[iFrame*2 + 0] * weights[2][0];\n                            accumulation[3] += pFramesIn[iFrame*2 + 0] * weights[3][0];\n                            accumulation[4] += pFramesIn[iFrame*2 + 0] * weights[4][0];\n                            accumulation[5] += pFramesIn[iFrame*2 + 0] * weights[5][0];\n                            accumulation[6] += pFramesIn[iFrame*2 + 0] * weights[6][0];\n                            accumulation[7] += pFramesIn[iFrame*2 + 0] * weights[7][0];\n\n                            accumulation[0] += pFramesIn[iFrame*2 + 1] * weights[0][1];\n                            accumulation[1] += pFramesIn[iFrame*2 + 1] * weights[1][1];\n                            accumulation[2] += pFramesIn[iFrame*2 + 1] * weights[2][1];\n                            accumulation[3] += pFramesIn[iFrame*2 + 1] * weights[3][1];\n                            accumulation[4] += pFramesIn[iFrame*2 + 1] * weights[4][1];\n                            accumulation[5] += pFramesIn[iFrame*2 + 1] * weights[5][1];\n                            accumulation[6] += pFramesIn[iFrame*2 + 1] * weights[6][1];\n                            accumulation[7] += pFramesIn[iFrame*2 + 1] * weights[7][1];\n\n                            pFramesOut[iFrame*8 + 0] = accumulation[0];\n                            pFramesOut[iFrame*8 + 1] = accumulation[1];\n                            pFramesOut[iFrame*8 + 2] = accumulation[2];\n                            pFramesOut[iFrame*8 + 3] = accumulation[3];\n                            pFramesOut[iFrame*8 + 4] = accumulation[4];\n                            pFramesOut[iFrame*8 + 5] = accumulation[5];\n                            pFramesOut[iFrame*8 + 6] = accumulation[6];\n                            pFramesOut[iFrame*8 + 7] = accumulation[7];\n                        }\n                    } else {\n                        /* When outputting to 8 channels, we can do everything in groups of two 4x SIMD operations. */\n                        for (; iFrame < frameCount; iFrame += 1) {\n                            float accumulation[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };\n\n                            for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                                accumulation[0] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[0][iChannelIn];\n                                accumulation[1] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[1][iChannelIn];\n                                accumulation[2] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[2][iChannelIn];\n                                accumulation[3] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[3][iChannelIn];\n                                accumulation[4] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[4][iChannelIn];\n                                accumulation[5] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[5][iChannelIn];\n                                accumulation[6] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[6][iChannelIn];\n                                accumulation[7] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[7][iChannelIn];\n                            }\n\n                            pFramesOut[iFrame*8 + 0] = accumulation[0];\n                            pFramesOut[iFrame*8 + 1] = accumulation[1];\n                            pFramesOut[iFrame*8 + 2] = accumulation[2];\n                            pFramesOut[iFrame*8 + 3] = accumulation[3];\n                            pFramesOut[iFrame*8 + 4] = accumulation[4];\n                            pFramesOut[iFrame*8 + 5] = accumulation[5];\n                            pFramesOut[iFrame*8 + 6] = accumulation[6];\n                            pFramesOut[iFrame*8 + 7] = accumulation[7];\n                        }\n                    }\n                } else if (channelsOut == 6) {\n                    /*\n                    When outputting to 6 channels we unfortunately don't have a nice multiple of 4 to do 4x SIMD operations. Instead we'll\n                    expand our weights and do two frames at a time.\n                    */\n                    for (; iFrame < frameCount; iFrame += 1) {\n                        float accumulation[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };\n\n                        for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                            accumulation[0] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[0][iChannelIn];\n                            accumulation[1] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[1][iChannelIn];\n                            accumulation[2] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[2][iChannelIn];\n                            accumulation[3] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[3][iChannelIn];\n                            accumulation[4] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[4][iChannelIn];\n                            accumulation[5] += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[5][iChannelIn];\n                        }\n\n                        pFramesOut[iFrame*6 + 0] = accumulation[0];\n                        pFramesOut[iFrame*6 + 1] = accumulation[1];\n                        pFramesOut[iFrame*6 + 2] = accumulation[2];\n                        pFramesOut[iFrame*6 + 3] = accumulation[3];\n                        pFramesOut[iFrame*6 + 4] = accumulation[4];\n                        pFramesOut[iFrame*6 + 5] = accumulation[5];\n                    }\n                }\n\n                /* Leftover frames. */\n                for (; iFrame < frameCount; iFrame += 1) {\n                    for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                        float accumulation = 0;\n\n                        for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                            accumulation += pFramesIn[iFrame*channelsIn + iChannelIn] * weights[iChannelOut][iChannelIn];\n                        }\n\n                        pFramesOut[iFrame*channelsOut + iChannelOut] = accumulation;\n                    }\n                }\n            } else {\n                /* Cannot pre-compute weights because not enough room in stack-allocated buffer. */\n                for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                    for (iChannelOut = 0; iChannelOut < channelsOut; iChannelOut += 1) {\n                        float accumulation = 0;\n                        ma_channel channelOut = ma_channel_map_get_channel(pChannelMapOut, channelsOut, iChannelOut);\n\n                        for (iChannelIn = 0; iChannelIn < channelsIn; iChannelIn += 1) {\n                            ma_channel channelIn = ma_channel_map_get_channel(pChannelMapIn, channelsIn, iChannelIn);\n                            accumulation += pFramesIn[iFrame*channelsIn + iChannelIn] * ma_calculate_channel_position_rectangular_weight(channelOut, channelIn);\n                        }\n\n                        pFramesOut[iFrame*channelsOut + iChannelOut] = accumulation;\n                    }\n                }\n            }\n        }\n    } else {\n        /* Fall back to silence. If you hit this, what are you doing with so many channels?! */\n        ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, channelsOut);\n    }\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t channelMapInOffset;\n    size_t channelMapOutOffset;\n    size_t shuffleTableOffset;\n    size_t weightsOffset;\n} ma_channel_converter_heap_layout;\n\nstatic ma_channel_conversion_path ma_channel_converter_config_get_conversion_path(const ma_channel_converter_config* pConfig)\n{\n    return ma_channel_map_get_conversion_path(pConfig->pChannelMapIn, pConfig->channelsIn, pConfig->pChannelMapOut, pConfig->channelsOut, pConfig->mixingMode);\n}\n\nstatic ma_result ma_channel_converter_get_heap_layout(const ma_channel_converter_config* pConfig, ma_channel_converter_heap_layout* pHeapLayout)\n{\n    ma_channel_conversion_path conversionPath;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channelsIn == 0 || pConfig->channelsOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (!ma_channel_map_is_valid(pConfig->pChannelMapIn, pConfig->channelsIn)) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (!ma_channel_map_is_valid(pConfig->pChannelMapOut, pConfig->channelsOut)) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Input channel map. Only need to allocate this if we have an input channel map (otherwise default channel map is assumed). */\n    pHeapLayout->channelMapInOffset = pHeapLayout->sizeInBytes;\n    if (pConfig->pChannelMapIn != NULL) {\n        pHeapLayout->sizeInBytes += sizeof(ma_channel) * pConfig->channelsIn;\n    }\n\n    /* Output channel map. Only need to allocate this if we have an output channel map (otherwise default channel map is assumed). */\n    pHeapLayout->channelMapOutOffset = pHeapLayout->sizeInBytes;\n    if (pConfig->pChannelMapOut != NULL) {\n        pHeapLayout->sizeInBytes += sizeof(ma_channel) * pConfig->channelsOut;\n    }\n\n    /* Alignment for the next section. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    /* Whether or not we use weights of a shuffle table depends on the channel map themselves and the algorithm we've chosen. */\n    conversionPath = ma_channel_converter_config_get_conversion_path(pConfig);\n\n    /* Shuffle table */\n    pHeapLayout->shuffleTableOffset = pHeapLayout->sizeInBytes;\n    if (conversionPath == ma_channel_conversion_path_shuffle) {\n        pHeapLayout->sizeInBytes += sizeof(ma_uint8) * pConfig->channelsOut;\n    }\n\n    /* Weights */\n    pHeapLayout->weightsOffset = pHeapLayout->sizeInBytes;\n    if (conversionPath == ma_channel_conversion_path_weights) {\n        pHeapLayout->sizeInBytes += sizeof(float*) * pConfig->channelsIn;\n        pHeapLayout->sizeInBytes += sizeof(float ) * pConfig->channelsIn * pConfig->channelsOut;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_channel_converter_get_heap_size(const ma_channel_converter_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_channel_converter_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_channel_converter_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_channel_converter_init_preallocated(const ma_channel_converter_config* pConfig, void* pHeap, ma_channel_converter* pConverter)\n{\n    ma_result result;\n    ma_channel_converter_heap_layout heapLayout;\n\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pConverter);\n\n    result = ma_channel_converter_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pConverter->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pConverter->_pHeap, heapLayout.sizeInBytes);\n\n    pConverter->format      = pConfig->format;\n    pConverter->channelsIn  = pConfig->channelsIn;\n    pConverter->channelsOut = pConfig->channelsOut;\n    pConverter->mixingMode  = pConfig->mixingMode;\n\n    if (pConfig->pChannelMapIn != NULL) {\n        pConverter->pChannelMapIn = (ma_channel*)ma_offset_ptr(pHeap, heapLayout.channelMapInOffset);\n        ma_channel_map_copy_or_default(pConverter->pChannelMapIn, pConfig->channelsIn, pConfig->pChannelMapIn, pConfig->channelsIn);\n    } else {\n        pConverter->pChannelMapIn = NULL;   /* Use default channel map. */\n    }\n\n    if (pConfig->pChannelMapOut != NULL) {\n        pConverter->pChannelMapOut = (ma_channel*)ma_offset_ptr(pHeap, heapLayout.channelMapOutOffset);\n        ma_channel_map_copy_or_default(pConverter->pChannelMapOut, pConfig->channelsOut, pConfig->pChannelMapOut, pConfig->channelsOut);\n    } else {\n        pConverter->pChannelMapOut = NULL;  /* Use default channel map. */\n    }\n\n    pConverter->conversionPath = ma_channel_converter_config_get_conversion_path(pConfig);\n\n    if (pConverter->conversionPath == ma_channel_conversion_path_shuffle) {\n        pConverter->pShuffleTable = (ma_uint8*)ma_offset_ptr(pHeap, heapLayout.shuffleTableOffset);\n        ma_channel_map_build_shuffle_table(pConverter->pChannelMapIn, pConverter->channelsIn, pConverter->pChannelMapOut, pConverter->channelsOut, pConverter->pShuffleTable);\n    }\n\n    if (pConverter->conversionPath == ma_channel_conversion_path_weights) {\n        ma_uint32 iChannelIn;\n        ma_uint32 iChannelOut;\n\n        if (pConverter->format == ma_format_f32) {\n            pConverter->weights.f32 = (float**   )ma_offset_ptr(pHeap, heapLayout.weightsOffset);\n            for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; iChannelIn += 1) {\n                pConverter->weights.f32[iChannelIn] = (float*)ma_offset_ptr(pHeap, heapLayout.weightsOffset + ((sizeof(float*) * pConverter->channelsIn) + (sizeof(float) * pConverter->channelsOut * iChannelIn)));\n            }\n        } else {\n            pConverter->weights.s16 = (ma_int32**)ma_offset_ptr(pHeap, heapLayout.weightsOffset);\n            for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; iChannelIn += 1) {\n                pConverter->weights.s16[iChannelIn] = (ma_int32*)ma_offset_ptr(pHeap, heapLayout.weightsOffset + ((sizeof(ma_int32*) * pConverter->channelsIn) + (sizeof(ma_int32) * pConverter->channelsOut * iChannelIn)));\n            }\n        }\n\n        /* Silence our weights by default. */\n        for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; iChannelIn += 1) {\n            for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; iChannelOut += 1) {\n                if (pConverter->format == ma_format_f32) {\n                    pConverter->weights.f32[iChannelIn][iChannelOut] = 0.0f;\n                } else {\n                    pConverter->weights.s16[iChannelIn][iChannelOut] = 0;\n                }\n            }\n        }\n\n        /*\n        We now need to fill out our weights table. This is determined by the mixing mode.\n        */\n\n        /* In all cases we need to make sure all channels that are present in both channel maps have a 1:1 mapping. */\n        for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n            ma_channel channelPosIn = ma_channel_map_get_channel(pConverter->pChannelMapIn, pConverter->channelsIn, iChannelIn);\n\n            for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                ma_channel channelPosOut = ma_channel_map_get_channel(pConverter->pChannelMapOut, pConverter->channelsOut, iChannelOut);\n\n                if (channelPosIn == channelPosOut) {\n                    float weight = 1;\n\n                    if (pConverter->format == ma_format_f32) {\n                        pConverter->weights.f32[iChannelIn][iChannelOut] = weight;\n                    } else {\n                        pConverter->weights.s16[iChannelIn][iChannelOut] = ma_channel_converter_float_to_fixed(weight);\n                    }\n                }\n            }\n        }\n\n        switch (pConverter->mixingMode)\n        {\n            case ma_channel_mix_mode_custom_weights:\n            {\n                if (pConfig->ppWeights == NULL) {\n                    return MA_INVALID_ARGS; /* Config specified a custom weights mixing mode, but no custom weights have been specified. */\n                }\n\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; iChannelIn += 1) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; iChannelOut += 1) {\n                        float weight = pConfig->ppWeights[iChannelIn][iChannelOut];\n\n                        if (pConverter->format == ma_format_f32) {\n                            pConverter->weights.f32[iChannelIn][iChannelOut] = weight;\n                        } else {\n                            pConverter->weights.s16[iChannelIn][iChannelOut] = ma_channel_converter_float_to_fixed(weight);\n                        }\n                    }\n                }\n            } break;\n\n            case ma_channel_mix_mode_simple:\n            {\n                /*\n                In simple mode, only set weights for channels that have exactly matching types, leave the rest at\n                zero. The 1:1 mappings have already been covered before this switch statement.\n                */\n            } break;\n\n            case ma_channel_mix_mode_rectangular:\n            default:\n            {\n                /* Unmapped input channels. */\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    ma_channel channelPosIn = ma_channel_map_get_channel(pConverter->pChannelMapIn, pConverter->channelsIn, iChannelIn);\n\n                    if (ma_is_spatial_channel_position(channelPosIn)) {\n                        if (!ma_channel_map_contains_channel_position(pConverter->channelsOut, pConverter->pChannelMapOut, channelPosIn)) {\n                            for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                                ma_channel channelPosOut = ma_channel_map_get_channel(pConverter->pChannelMapOut, pConverter->channelsOut, iChannelOut);\n\n                                if (ma_is_spatial_channel_position(channelPosOut)) {\n                                    float weight = 0;\n                                    if (pConverter->mixingMode == ma_channel_mix_mode_rectangular) {\n                                        weight = ma_calculate_channel_position_rectangular_weight(channelPosIn, channelPosOut);\n                                    }\n\n                                    /* Only apply the weight if we haven't already got some contribution from the respective channels. */\n                                    if (pConverter->format == ma_format_f32) {\n                                        if (pConverter->weights.f32[iChannelIn][iChannelOut] == 0) {\n                                            pConverter->weights.f32[iChannelIn][iChannelOut] = weight;\n                                        }\n                                    } else {\n                                        if (pConverter->weights.s16[iChannelIn][iChannelOut] == 0) {\n                                            pConverter->weights.s16[iChannelIn][iChannelOut] = ma_channel_converter_float_to_fixed(weight);\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n\n                /* Unmapped output channels. */\n                for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                    ma_channel channelPosOut = ma_channel_map_get_channel(pConverter->pChannelMapOut, pConverter->channelsOut, iChannelOut);\n\n                    if (ma_is_spatial_channel_position(channelPosOut)) {\n                        if (!ma_channel_map_contains_channel_position(pConverter->channelsIn, pConverter->pChannelMapIn, channelPosOut)) {\n                            for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                                ma_channel channelPosIn = ma_channel_map_get_channel(pConverter->pChannelMapIn, pConverter->channelsIn, iChannelIn);\n\n                                if (ma_is_spatial_channel_position(channelPosIn)) {\n                                    float weight = 0;\n                                    if (pConverter->mixingMode == ma_channel_mix_mode_rectangular) {\n                                        weight = ma_calculate_channel_position_rectangular_weight(channelPosIn, channelPosOut);\n                                    }\n\n                                    /* Only apply the weight if we haven't already got some contribution from the respective channels. */\n                                    if (pConverter->format == ma_format_f32) {\n                                        if (pConverter->weights.f32[iChannelIn][iChannelOut] == 0) {\n                                            pConverter->weights.f32[iChannelIn][iChannelOut] = weight;\n                                        }\n                                    } else {\n                                        if (pConverter->weights.s16[iChannelIn][iChannelOut] == 0) {\n                                            pConverter->weights.s16[iChannelIn][iChannelOut] = ma_channel_converter_float_to_fixed(weight);\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n\n                /* If LFE is in the output channel map but was not present in the input channel map, configure its weight now */\n                if (pConfig->calculateLFEFromSpatialChannels) {\n                    if (!ma_channel_map_contains_channel_position(pConverter->channelsIn, pConverter->pChannelMapIn, MA_CHANNEL_LFE)) {\n                        ma_uint32 spatialChannelCount = ma_channel_map_get_spatial_channel_count(pConverter->pChannelMapIn, pConverter->channelsIn);\n                        ma_uint32 iChannelOutLFE;\n\n                        if (spatialChannelCount > 0 && ma_channel_map_find_channel_position(pConverter->channelsOut, pConverter->pChannelMapOut, MA_CHANNEL_LFE, &iChannelOutLFE)) {\n                            const float weightForLFE = 1.0f / spatialChannelCount;\n                            for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                                const ma_channel channelPosIn = ma_channel_map_get_channel(pConverter->pChannelMapIn, pConverter->channelsIn, iChannelIn);\n                                if (ma_is_spatial_channel_position(channelPosIn)) {\n                                    if (pConverter->format == ma_format_f32) {\n                                        if (pConverter->weights.f32[iChannelIn][iChannelOutLFE] == 0) {\n                                            pConverter->weights.f32[iChannelIn][iChannelOutLFE] = weightForLFE;\n                                        }\n                                    } else {\n                                        if (pConverter->weights.s16[iChannelIn][iChannelOutLFE] == 0) {\n                                            pConverter->weights.s16[iChannelIn][iChannelOutLFE] = ma_channel_converter_float_to_fixed(weightForLFE);\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            } break;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_channel_converter_init(const ma_channel_converter_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_channel_converter* pConverter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_channel_converter_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_channel_converter_init_preallocated(pConfig, pHeap, pConverter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pConverter->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_channel_converter_uninit(ma_channel_converter* pConverter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pConverter == NULL) {\n        return;\n    }\n\n    if (pConverter->_ownsHeap) {\n        ma_free(pConverter->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic ma_result ma_channel_converter_process_pcm_frames__passthrough(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n    MA_ASSERT(pFramesIn  != NULL);\n\n    ma_copy_memory_64(pFramesOut, pFramesIn, frameCount * ma_get_bytes_per_frame(pConverter->format, pConverter->channelsOut));\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_channel_converter_process_pcm_frames__shuffle(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n    MA_ASSERT(pFramesIn  != NULL);\n    MA_ASSERT(pConverter->channelsIn == pConverter->channelsOut);\n\n    return ma_channel_map_apply_shuffle_table(pFramesOut, pConverter->channelsOut, pFramesIn, pConverter->channelsIn, frameCount, pConverter->pShuffleTable, pConverter->format);\n}\n\nstatic ma_result ma_channel_converter_process_pcm_frames__mono_in(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n    MA_ASSERT(pFramesIn  != NULL);\n    MA_ASSERT(pConverter->channelsIn == 1);\n\n    switch (pConverter->format)\n    {\n        case ma_format_u8:\n        {\n            /* */ ma_uint8* pFramesOutU8 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInU8  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < pConverter->channelsOut; iChannel += 1) {\n                    pFramesOutU8[iFrame*pConverter->channelsOut + iChannel] = pFramesInU8[iFrame];\n                }\n            }\n        } break;\n\n        case ma_format_s16:\n        {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            if (pConverter->channelsOut == 2) {\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    pFramesOutS16[iFrame*2 + 0] = pFramesInS16[iFrame];\n                    pFramesOutS16[iFrame*2 + 1] = pFramesInS16[iFrame];\n                }\n            } else {\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    ma_uint32 iChannel;\n                    for (iChannel = 0; iChannel < pConverter->channelsOut; iChannel += 1) {\n                        pFramesOutS16[iFrame*pConverter->channelsOut + iChannel] = pFramesInS16[iFrame];\n                    }\n                }\n            }\n        } break;\n\n        case ma_format_s24:\n        {\n            /* */ ma_uint8* pFramesOutS24 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInS24  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < pConverter->channelsOut; iChannel += 1) {\n                    ma_uint64 iSampleOut = iFrame*pConverter->channelsOut + iChannel;\n                    ma_uint64 iSampleIn  = iFrame;\n                    pFramesOutS24[iSampleOut*3 + 0] = pFramesInS24[iSampleIn*3 + 0];\n                    pFramesOutS24[iSampleOut*3 + 1] = pFramesInS24[iSampleIn*3 + 1];\n                    pFramesOutS24[iSampleOut*3 + 2] = pFramesInS24[iSampleIn*3 + 2];\n                }\n            }\n        } break;\n\n        case ma_format_s32:\n        {\n            /* */ ma_int32* pFramesOutS32 = (      ma_int32*)pFramesOut;\n            const ma_int32* pFramesInS32  = (const ma_int32*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_uint32 iChannel;\n                for (iChannel = 0; iChannel < pConverter->channelsOut; iChannel += 1) {\n                    pFramesOutS32[iFrame*pConverter->channelsOut + iChannel] = pFramesInS32[iFrame];\n                }\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            if (pConverter->channelsOut == 2) {\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    pFramesOutF32[iFrame*2 + 0] = pFramesInF32[iFrame];\n                    pFramesOutF32[iFrame*2 + 1] = pFramesInF32[iFrame];\n                }\n            } else {\n                for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                    ma_uint32 iChannel;\n                    for (iChannel = 0; iChannel < pConverter->channelsOut; iChannel += 1) {\n                        pFramesOutF32[iFrame*pConverter->channelsOut + iChannel] = pFramesInF32[iFrame];\n                    }\n                }\n            }\n        } break;\n\n        default: return MA_INVALID_OPERATION;   /* Unknown format. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_channel_converter_process_pcm_frames__mono_out(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannel;\n\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n    MA_ASSERT(pFramesIn  != NULL);\n    MA_ASSERT(pConverter->channelsOut == 1);\n\n    switch (pConverter->format)\n    {\n        case ma_format_u8:\n        {\n            /* */ ma_uint8* pFramesOutU8 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInU8  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_int32 t = 0;\n                for (iChannel = 0; iChannel < pConverter->channelsIn; iChannel += 1) {\n                    t += ma_pcm_sample_u8_to_s16_no_scale(pFramesInU8[iFrame*pConverter->channelsIn + iChannel]);\n                }\n\n                pFramesOutU8[iFrame] = ma_clip_u8(t / pConverter->channelsOut);\n            }\n        } break;\n\n        case ma_format_s16:\n        {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_int32 t = 0;\n                for (iChannel = 0; iChannel < pConverter->channelsIn; iChannel += 1) {\n                    t += pFramesInS16[iFrame*pConverter->channelsIn + iChannel];\n                }\n\n                pFramesOutS16[iFrame] = (ma_int16)(t / pConverter->channelsIn);\n            }\n        } break;\n\n        case ma_format_s24:\n        {\n            /* */ ma_uint8* pFramesOutS24 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInS24  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_int64 t = 0;\n                for (iChannel = 0; iChannel < pConverter->channelsIn; iChannel += 1) {\n                    t += ma_pcm_sample_s24_to_s32_no_scale(&pFramesInS24[(iFrame*pConverter->channelsIn + iChannel)*3]);\n                }\n\n                ma_pcm_sample_s32_to_s24_no_scale(t / pConverter->channelsIn, &pFramesOutS24[iFrame*3]);\n            }\n        } break;\n\n        case ma_format_s32:\n        {\n            /* */ ma_int32* pFramesOutS32 = (      ma_int32*)pFramesOut;\n            const ma_int32* pFramesInS32  = (const ma_int32*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                ma_int64 t = 0;\n                for (iChannel = 0; iChannel < pConverter->channelsIn; iChannel += 1) {\n                    t += pFramesInS32[iFrame*pConverter->channelsIn + iChannel];\n                }\n\n                pFramesOutS32[iFrame] = (ma_int32)(t / pConverter->channelsIn);\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; ++iFrame) {\n                float t = 0;\n                for (iChannel = 0; iChannel < pConverter->channelsIn; iChannel += 1) {\n                    t += pFramesInF32[iFrame*pConverter->channelsIn + iChannel];\n                }\n\n                pFramesOutF32[iFrame] = t / pConverter->channelsIn;\n            }\n        } break;\n\n        default: return MA_INVALID_OPERATION;   /* Unknown format. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_channel_converter_process_pcm_frames__weights(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    ma_uint32 iFrame;\n    ma_uint32 iChannelIn;\n    ma_uint32 iChannelOut;\n\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n    MA_ASSERT(pFramesIn  != NULL);\n\n    /* This is the more complicated case. Each of the output channels is accumulated with 0 or more input channels. */\n\n    /* Clear. */\n    ma_zero_memory_64(pFramesOut, frameCount * ma_get_bytes_per_frame(pConverter->format, pConverter->channelsOut));\n\n    /* Accumulate. */\n    switch (pConverter->format)\n    {\n        case ma_format_u8:\n        {\n            /* */ ma_uint8* pFramesOutU8 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInU8  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                        ma_int16 u8_O = ma_pcm_sample_u8_to_s16_no_scale(pFramesOutU8[iFrame*pConverter->channelsOut + iChannelOut]);\n                        ma_int16 u8_I = ma_pcm_sample_u8_to_s16_no_scale(pFramesInU8 [iFrame*pConverter->channelsIn  + iChannelIn ]);\n                        ma_int32 s    = (ma_int32)ma_clamp(u8_O + ((u8_I * pConverter->weights.s16[iChannelIn][iChannelOut]) >> MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT), -128, 127);\n                        pFramesOutU8[iFrame*pConverter->channelsOut + iChannelOut] = ma_clip_u8((ma_int16)s);\n                    }\n                }\n            }\n        } break;\n\n        case ma_format_s16:\n        {\n            /* */ ma_int16* pFramesOutS16 = (      ma_int16*)pFramesOut;\n            const ma_int16* pFramesInS16  = (const ma_int16*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                        ma_int32 s = pFramesOutS16[iFrame*pConverter->channelsOut + iChannelOut];\n                        s += (pFramesInS16[iFrame*pConverter->channelsIn + iChannelIn] * pConverter->weights.s16[iChannelIn][iChannelOut]) >> MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT;\n\n                        pFramesOutS16[iFrame*pConverter->channelsOut + iChannelOut] = (ma_int16)ma_clamp(s, -32768, 32767);\n                    }\n                }\n            }\n        } break;\n\n        case ma_format_s24:\n        {\n            /* */ ma_uint8* pFramesOutS24 = (      ma_uint8*)pFramesOut;\n            const ma_uint8* pFramesInS24  = (const ma_uint8*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                        ma_int64 s24_O = ma_pcm_sample_s24_to_s32_no_scale(&pFramesOutS24[(iFrame*pConverter->channelsOut + iChannelOut)*3]);\n                        ma_int64 s24_I = ma_pcm_sample_s24_to_s32_no_scale(&pFramesInS24 [(iFrame*pConverter->channelsIn  + iChannelIn )*3]);\n                        ma_int64 s24   = (ma_int32)ma_clamp(s24_O + ((s24_I * pConverter->weights.s16[iChannelIn][iChannelOut]) >> MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT), -8388608, 8388607);\n                        ma_pcm_sample_s32_to_s24_no_scale(s24, &pFramesOutS24[(iFrame*pConverter->channelsOut + iChannelOut)*3]);\n                    }\n                }\n            }\n        } break;\n\n        case ma_format_s32:\n        {\n            /* */ ma_int32* pFramesOutS32 = (      ma_int32*)pFramesOut;\n            const ma_int32* pFramesInS32  = (const ma_int32*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                        ma_int64 s = pFramesOutS32[iFrame*pConverter->channelsOut + iChannelOut];\n                        s += ((ma_int64)pFramesInS32[iFrame*pConverter->channelsIn + iChannelIn] * pConverter->weights.s16[iChannelIn][iChannelOut]) >> MA_CHANNEL_CONVERTER_FIXED_POINT_SHIFT;\n\n                        pFramesOutS32[iFrame*pConverter->channelsOut + iChannelOut] = ma_clip_s32(s);\n                    }\n                }\n            }\n        } break;\n\n        case ma_format_f32:\n        {\n            /* */ float* pFramesOutF32 = (      float*)pFramesOut;\n            const float* pFramesInF32  = (const float*)pFramesIn;\n\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannelIn = 0; iChannelIn < pConverter->channelsIn; ++iChannelIn) {\n                    for (iChannelOut = 0; iChannelOut < pConverter->channelsOut; ++iChannelOut) {\n                        pFramesOutF32[iFrame*pConverter->channelsOut + iChannelOut] += pFramesInF32[iFrame*pConverter->channelsIn + iChannelIn] * pConverter->weights.f32[iChannelIn][iChannelOut];\n                    }\n                }\n            }\n        } break;\n\n        default: return MA_INVALID_OPERATION;   /* Unknown format. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_channel_converter_process_pcm_frames(ma_channel_converter* pConverter, void* pFramesOut, const void* pFramesIn, ma_uint64 frameCount)\n{\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFramesOut == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFramesIn == NULL) {\n        ma_zero_memory_64(pFramesOut, frameCount * ma_get_bytes_per_frame(pConverter->format, pConverter->channelsOut));\n        return MA_SUCCESS;\n    }\n\n    switch (pConverter->conversionPath)\n    {\n        case ma_channel_conversion_path_passthrough: return ma_channel_converter_process_pcm_frames__passthrough(pConverter, pFramesOut, pFramesIn, frameCount);\n        case ma_channel_conversion_path_mono_out:    return ma_channel_converter_process_pcm_frames__mono_out(pConverter, pFramesOut, pFramesIn, frameCount);\n        case ma_channel_conversion_path_mono_in:     return ma_channel_converter_process_pcm_frames__mono_in(pConverter, pFramesOut, pFramesIn, frameCount);\n        case ma_channel_conversion_path_shuffle:     return ma_channel_converter_process_pcm_frames__shuffle(pConverter, pFramesOut, pFramesIn, frameCount);\n        case ma_channel_conversion_path_weights:\n        default:\n        {\n            return ma_channel_converter_process_pcm_frames__weights(pConverter, pFramesOut, pFramesIn, frameCount);\n        }\n    }\n}\n\nMA_API ma_result ma_channel_converter_get_input_channel_map(const ma_channel_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pConverter == NULL || pChannelMap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_channel_map_copy_or_default(pChannelMap, channelMapCap, pConverter->pChannelMapIn, pConverter->channelsIn);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_channel_converter_get_output_channel_map(const ma_channel_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pConverter == NULL || pChannelMap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_channel_map_copy_or_default(pChannelMap, channelMapCap, pConverter->pChannelMapOut, pConverter->channelsOut);\n\n    return MA_SUCCESS;\n}\n\n\n/**************************************************************************************************************************************************************\n\nData Conversion\n\n**************************************************************************************************************************************************************/\nMA_API ma_data_converter_config ma_data_converter_config_init_default(void)\n{\n    ma_data_converter_config config;\n    MA_ZERO_OBJECT(&config);\n\n    config.ditherMode = ma_dither_mode_none;\n    config.resampling.algorithm = ma_resample_algorithm_linear;\n    config.allowDynamicSampleRate = MA_FALSE; /* Disable dynamic sample rates by default because dynamic rate adjustments should be quite rare and it allows an optimization for cases when the in and out sample rates are the same. */\n\n    /* Linear resampling defaults. */\n    config.resampling.linear.lpfOrder = 1;\n\n    return config;\n}\n\nMA_API ma_data_converter_config ma_data_converter_config_init(ma_format formatIn, ma_format formatOut, ma_uint32 channelsIn, ma_uint32 channelsOut, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    ma_data_converter_config config = ma_data_converter_config_init_default();\n    config.formatIn      = formatIn;\n    config.formatOut     = formatOut;\n    config.channelsIn    = channelsIn;\n    config.channelsOut   = channelsOut;\n    config.sampleRateIn  = sampleRateIn;\n    config.sampleRateOut = sampleRateOut;\n\n    return config;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t channelConverterOffset;\n    size_t resamplerOffset;\n} ma_data_converter_heap_layout;\n\nstatic ma_bool32 ma_data_converter_config_is_resampler_required(const ma_data_converter_config* pConfig)\n{\n    MA_ASSERT(pConfig != NULL);\n\n    return pConfig->allowDynamicSampleRate || pConfig->sampleRateIn != pConfig->sampleRateOut;\n}\n\nstatic ma_format ma_data_converter_config_get_mid_format(const ma_data_converter_config* pConfig)\n{\n    MA_ASSERT(pConfig != NULL);\n\n    /*\n    We want to avoid as much data conversion as possible. The channel converter and linear\n    resampler both support s16 and f32 natively. We need to decide on the format to use for this\n    stage. We call this the mid format because it's used in the middle stage of the conversion\n    pipeline. If the output format is either s16 or f32 we use that one. If that is not the case it\n    will do the same thing for the input format. If it's neither we just use f32. If we are using a\n    custom resampling backend, we can only guarantee that f32 will be supported so we'll be forced\n    to use that if resampling is required.\n    */\n    if (ma_data_converter_config_is_resampler_required(pConfig) && pConfig->resampling.algorithm != ma_resample_algorithm_linear) {\n        return ma_format_f32;  /* <-- Force f32 since that is the only one we can guarantee will be supported by the resampler. */\n    } else {\n        /*  */ if (pConfig->formatOut == ma_format_s16 || pConfig->formatOut == ma_format_f32) {\n            return pConfig->formatOut;\n        } else if (pConfig->formatIn  == ma_format_s16 || pConfig->formatIn  == ma_format_f32) {\n            return pConfig->formatIn;\n        } else {\n            return ma_format_f32;\n        }\n    }\n}\n\nstatic ma_channel_converter_config ma_channel_converter_config_init_from_data_converter_config(const ma_data_converter_config* pConfig)\n{\n    ma_channel_converter_config channelConverterConfig;\n\n    MA_ASSERT(pConfig != NULL);\n\n    channelConverterConfig = ma_channel_converter_config_init(ma_data_converter_config_get_mid_format(pConfig), pConfig->channelsIn, pConfig->pChannelMapIn, pConfig->channelsOut, pConfig->pChannelMapOut, pConfig->channelMixMode);\n    channelConverterConfig.ppWeights = pConfig->ppChannelWeights;\n    channelConverterConfig.calculateLFEFromSpatialChannels = pConfig->calculateLFEFromSpatialChannels;\n\n    return channelConverterConfig;\n}\n\nstatic ma_resampler_config ma_resampler_config_init_from_data_converter_config(const ma_data_converter_config* pConfig)\n{\n    ma_resampler_config resamplerConfig;\n    ma_uint32 resamplerChannels;\n\n    MA_ASSERT(pConfig != NULL);\n\n    /* The resampler is the most expensive part of the conversion process, so we need to do it at the stage where the channel count is at it's lowest. */\n    if (pConfig->channelsIn < pConfig->channelsOut) {\n        resamplerChannels = pConfig->channelsIn;\n    } else {\n        resamplerChannels = pConfig->channelsOut;\n    }\n\n    resamplerConfig = ma_resampler_config_init(ma_data_converter_config_get_mid_format(pConfig), resamplerChannels, pConfig->sampleRateIn, pConfig->sampleRateOut, pConfig->resampling.algorithm);\n    resamplerConfig.linear           = pConfig->resampling.linear;\n    resamplerConfig.pBackendVTable   = pConfig->resampling.pBackendVTable;\n    resamplerConfig.pBackendUserData = pConfig->resampling.pBackendUserData;\n\n    return resamplerConfig;\n}\n\nstatic ma_result ma_data_converter_get_heap_layout(const ma_data_converter_config* pConfig, ma_data_converter_heap_layout* pHeapLayout)\n{\n    ma_result result;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channelsIn == 0 || pConfig->channelsOut == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Channel converter. */\n    pHeapLayout->channelConverterOffset = pHeapLayout->sizeInBytes;\n    {\n        size_t heapSizeInBytes;\n        ma_channel_converter_config channelConverterConfig = ma_channel_converter_config_init_from_data_converter_config(pConfig);\n\n        result = ma_channel_converter_get_heap_size(&channelConverterConfig, &heapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += heapSizeInBytes;\n    }\n\n    /* Resampler. */\n    pHeapLayout->resamplerOffset = pHeapLayout->sizeInBytes;\n    if (ma_data_converter_config_is_resampler_required(pConfig)) {\n        size_t heapSizeInBytes;\n        ma_resampler_config resamplerConfig = ma_resampler_config_init_from_data_converter_config(pConfig);\n\n        result = ma_resampler_get_heap_size(&resamplerConfig, &heapSizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->sizeInBytes += heapSizeInBytes;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_get_heap_size(const ma_data_converter_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_data_converter_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_data_converter_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_init_preallocated(const ma_data_converter_config* pConfig, void* pHeap, ma_data_converter* pConverter)\n{\n    ma_result result;\n    ma_data_converter_heap_layout heapLayout;\n    ma_format midFormat;\n    ma_bool32 isResamplingRequired;\n\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pConverter);\n\n    result = ma_data_converter_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pConverter->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pConverter->formatIn      = pConfig->formatIn;\n    pConverter->formatOut     = pConfig->formatOut;\n    pConverter->channelsIn    = pConfig->channelsIn;\n    pConverter->channelsOut   = pConfig->channelsOut;\n    pConverter->sampleRateIn  = pConfig->sampleRateIn;\n    pConverter->sampleRateOut = pConfig->sampleRateOut;\n    pConverter->ditherMode    = pConfig->ditherMode;\n\n    /*\n    Determine if resampling is required. We need to do this so we can determine an appropriate\n    mid format to use. If resampling is required, the mid format must be ma_format_f32 since\n    that is the only one that is guaranteed to supported by custom resampling backends.\n    */\n    isResamplingRequired = ma_data_converter_config_is_resampler_required(pConfig);\n    midFormat = ma_data_converter_config_get_mid_format(pConfig);\n\n\n    /* Channel converter. We always initialize this, but we check if it configures itself as a passthrough to determine whether or not it's needed. */\n    {\n        ma_channel_converter_config channelConverterConfig = ma_channel_converter_config_init_from_data_converter_config(pConfig);\n\n        result = ma_channel_converter_init_preallocated(&channelConverterConfig, ma_offset_ptr(pHeap, heapLayout.channelConverterOffset), &pConverter->channelConverter);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        /* If the channel converter is not a passthrough we need to enable it. Otherwise we can skip it. */\n        if (pConverter->channelConverter.conversionPath != ma_channel_conversion_path_passthrough) {\n            pConverter->hasChannelConverter = MA_TRUE;\n        }\n    }\n\n\n    /* Resampler. */\n    if (isResamplingRequired) {\n        ma_resampler_config resamplerConfig = ma_resampler_config_init_from_data_converter_config(pConfig);\n\n        result = ma_resampler_init_preallocated(&resamplerConfig, ma_offset_ptr(pHeap, heapLayout.resamplerOffset), &pConverter->resampler);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pConverter->hasResampler = MA_TRUE;\n    }\n\n\n    /* We can simplify pre- and post-format conversion if we have neither channel conversion nor resampling. */\n    if (pConverter->hasChannelConverter == MA_FALSE && pConverter->hasResampler == MA_FALSE) {\n        /* We have neither channel conversion nor resampling so we'll only need one of pre- or post-format conversion, or none if the input and output formats are the same. */\n        if (pConverter->formatIn == pConverter->formatOut) {\n            /* The formats are the same so we can just pass through. */\n            pConverter->hasPreFormatConversion  = MA_FALSE;\n            pConverter->hasPostFormatConversion = MA_FALSE;\n        } else {\n            /* The formats are different so we need to do either pre- or post-format conversion. It doesn't matter which. */\n            pConverter->hasPreFormatConversion  = MA_FALSE;\n            pConverter->hasPostFormatConversion = MA_TRUE;\n        }\n    } else {\n        /* We have a channel converter and/or resampler so we'll need channel conversion based on the mid format. */\n        if (pConverter->formatIn != midFormat) {\n            pConverter->hasPreFormatConversion  = MA_TRUE;\n        }\n        if (pConverter->formatOut != midFormat) {\n            pConverter->hasPostFormatConversion = MA_TRUE;\n        }\n    }\n\n    /* We can enable passthrough optimizations if applicable. Note that we'll only be able to do this if the sample rate is static. */\n    if (pConverter->hasPreFormatConversion  == MA_FALSE &&\n        pConverter->hasPostFormatConversion == MA_FALSE &&\n        pConverter->hasChannelConverter     == MA_FALSE &&\n        pConverter->hasResampler            == MA_FALSE) {\n        pConverter->isPassthrough = MA_TRUE;\n    }\n\n\n    /* We now need to determine our execution path. */\n    if (pConverter->isPassthrough) {\n        pConverter->executionPath = ma_data_converter_execution_path_passthrough;\n    } else {\n        if (pConverter->channelsIn < pConverter->channelsOut) {\n            /* Do resampling first, if necessary. */\n            MA_ASSERT(pConverter->hasChannelConverter == MA_TRUE);\n\n            if (pConverter->hasResampler) {\n                pConverter->executionPath = ma_data_converter_execution_path_resample_first;\n            } else {\n                pConverter->executionPath = ma_data_converter_execution_path_channels_only;\n            }\n        } else {\n            /* Do channel conversion first, if necessary. */\n            if (pConverter->hasChannelConverter) {\n                if (pConverter->hasResampler) {\n                    pConverter->executionPath = ma_data_converter_execution_path_channels_first;\n                } else {\n                    pConverter->executionPath = ma_data_converter_execution_path_channels_only;\n                }\n            } else {\n                /* Channel routing not required. */\n                if (pConverter->hasResampler) {\n                    pConverter->executionPath = ma_data_converter_execution_path_resample_only;\n                } else {\n                    pConverter->executionPath = ma_data_converter_execution_path_format_only;\n                }\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_init(const ma_data_converter_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_converter* pConverter)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_data_converter_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_data_converter_init_preallocated(pConfig, pHeap, pConverter);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pConverter->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_data_converter_uninit(ma_data_converter* pConverter, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pConverter == NULL) {\n        return;\n    }\n\n    if (pConverter->hasResampler) {\n        ma_resampler_uninit(&pConverter->resampler, pAllocationCallbacks);\n    }\n\n    ma_channel_converter_uninit(&pConverter->channelConverter, pAllocationCallbacks);\n\n    if (pConverter->_ownsHeap) {\n        ma_free(pConverter->_pHeap, pAllocationCallbacks);\n    }\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__passthrough(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 frameCount;\n\n    MA_ASSERT(pConverter != NULL);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    frameCount = ma_min(frameCountIn, frameCountOut);\n\n    if (pFramesOut != NULL) {\n        if (pFramesIn != NULL) {\n            ma_copy_memory_64(pFramesOut, pFramesIn, frameCount * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        } else {\n            ma_zero_memory_64(pFramesOut,            frameCount * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = frameCount;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__format_only(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 frameCount;\n\n    MA_ASSERT(pConverter != NULL);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    frameCount = ma_min(frameCountIn, frameCountOut);\n\n    if (pFramesOut != NULL) {\n        if (pFramesIn != NULL) {\n            ma_convert_pcm_frames_format(pFramesOut, pConverter->formatOut, pFramesIn, pConverter->formatIn, frameCount, pConverter->channelsIn, pConverter->ditherMode);\n        } else {\n            ma_zero_memory_64(pFramesOut, frameCount * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = frameCount;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_data_converter_process_pcm_frames__resample_with_format_conversion(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n\n    MA_ASSERT(pConverter != NULL);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    while (framesProcessedOut < frameCountOut) {\n        ma_uint8 pTempBufferOut[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n        const ma_uint32 tempBufferOutCap = sizeof(pTempBufferOut) / ma_get_bytes_per_frame(pConverter->resampler.format, pConverter->resampler.channels);\n        const void* pFramesInThisIteration;\n        /* */ void* pFramesOutThisIteration;\n        ma_uint64 frameCountInThisIteration;\n        ma_uint64 frameCountOutThisIteration;\n\n        if (pFramesIn != NULL) {\n            pFramesInThisIteration = ma_offset_ptr(pFramesIn, framesProcessedIn * ma_get_bytes_per_frame(pConverter->formatIn, pConverter->channelsIn));\n        } else {\n            pFramesInThisIteration = NULL;\n        }\n\n        if (pFramesOut != NULL) {\n            pFramesOutThisIteration = ma_offset_ptr(pFramesOut, framesProcessedOut * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        } else {\n            pFramesOutThisIteration = NULL;\n        }\n\n        /* Do a pre format conversion if necessary. */\n        if (pConverter->hasPreFormatConversion) {\n            ma_uint8 pTempBufferIn[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n            const ma_uint32 tempBufferInCap = sizeof(pTempBufferIn) / ma_get_bytes_per_frame(pConverter->resampler.format, pConverter->resampler.channels);\n\n            frameCountInThisIteration  = (frameCountIn - framesProcessedIn);\n            if (frameCountInThisIteration > tempBufferInCap) {\n                frameCountInThisIteration = tempBufferInCap;\n            }\n\n            if (pConverter->hasPostFormatConversion) {\n               if (frameCountInThisIteration > tempBufferOutCap) {\n                   frameCountInThisIteration = tempBufferOutCap;\n               }\n            }\n\n            if (pFramesInThisIteration != NULL) {\n                ma_convert_pcm_frames_format(pTempBufferIn, pConverter->resampler.format, pFramesInThisIteration, pConverter->formatIn, frameCountInThisIteration, pConverter->channelsIn, pConverter->ditherMode);\n            } else {\n                MA_ZERO_MEMORY(pTempBufferIn, sizeof(pTempBufferIn));\n            }\n\n            frameCountOutThisIteration = (frameCountOut - framesProcessedOut);\n\n            if (pConverter->hasPostFormatConversion) {\n                /* Both input and output conversion required. Output to the temp buffer. */\n                if (frameCountOutThisIteration > tempBufferOutCap) {\n                    frameCountOutThisIteration = tempBufferOutCap;\n                }\n\n                result = ma_resampler_process_pcm_frames(&pConverter->resampler, pTempBufferIn, &frameCountInThisIteration, pTempBufferOut, &frameCountOutThisIteration);\n            } else {\n                /* Only pre-format required. Output straight to the output buffer. */\n                result = ma_resampler_process_pcm_frames(&pConverter->resampler, pTempBufferIn, &frameCountInThisIteration, pFramesOutThisIteration, &frameCountOutThisIteration);\n            }\n\n            if (result != MA_SUCCESS) {\n                break;\n            }\n        } else {\n            /* No pre-format required. Just read straight from the input buffer. */\n            MA_ASSERT(pConverter->hasPostFormatConversion == MA_TRUE);\n\n            frameCountInThisIteration  = (frameCountIn  - framesProcessedIn);\n            frameCountOutThisIteration = (frameCountOut - framesProcessedOut);\n            if (frameCountOutThisIteration > tempBufferOutCap) {\n                frameCountOutThisIteration = tempBufferOutCap;\n            }\n\n            result = ma_resampler_process_pcm_frames(&pConverter->resampler, pFramesInThisIteration, &frameCountInThisIteration, pTempBufferOut, &frameCountOutThisIteration);\n            if (result != MA_SUCCESS) {\n                break;\n            }\n        }\n\n        /* If we are doing a post format conversion we need to do that now. */\n        if (pConverter->hasPostFormatConversion) {\n            if (pFramesOutThisIteration != NULL) {\n                ma_convert_pcm_frames_format(pFramesOutThisIteration, pConverter->formatOut, pTempBufferOut, pConverter->resampler.format, frameCountOutThisIteration, pConverter->resampler.channels, pConverter->ditherMode);\n            }\n        }\n\n        framesProcessedIn  += frameCountInThisIteration;\n        framesProcessedOut += frameCountOutThisIteration;\n\n        MA_ASSERT(framesProcessedIn  <= frameCountIn);\n        MA_ASSERT(framesProcessedOut <= frameCountOut);\n\n        if (frameCountOutThisIteration == 0) {\n            break;  /* Consumed all of our input data. */\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = framesProcessedIn;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = framesProcessedOut;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__resample_only(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    MA_ASSERT(pConverter != NULL);\n\n    if (pConverter->hasPreFormatConversion == MA_FALSE && pConverter->hasPostFormatConversion == MA_FALSE) {\n        /* Neither pre- nor post-format required. This is simple case where only resampling is required. */\n        return ma_resampler_process_pcm_frames(&pConverter->resampler, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    } else {\n        /* Format conversion required. */\n        return ma_data_converter_process_pcm_frames__resample_with_format_conversion(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n    }\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__channels_only(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_result result;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 frameCount;\n\n    MA_ASSERT(pConverter != NULL);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    frameCount = ma_min(frameCountIn, frameCountOut);\n\n    if (pConverter->hasPreFormatConversion == MA_FALSE && pConverter->hasPostFormatConversion == MA_FALSE) {\n        /* No format conversion required. */\n        result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pFramesOut, pFramesIn, frameCount);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        /* Format conversion required. */\n        ma_uint64 framesProcessed = 0;\n\n        while (framesProcessed < frameCount) {\n            ma_uint8 pTempBufferOut[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n            const ma_uint32 tempBufferOutCap = sizeof(pTempBufferOut) / ma_get_bytes_per_frame(pConverter->channelConverter.format, pConverter->channelConverter.channelsOut);\n            const void* pFramesInThisIteration;\n            /* */ void* pFramesOutThisIteration;\n            ma_uint64 frameCountThisIteration;\n\n            if (pFramesIn != NULL) {\n                pFramesInThisIteration = ma_offset_ptr(pFramesIn, framesProcessed * ma_get_bytes_per_frame(pConverter->formatIn, pConverter->channelsIn));\n            } else {\n                pFramesInThisIteration = NULL;\n            }\n\n            if (pFramesOut != NULL) {\n                pFramesOutThisIteration = ma_offset_ptr(pFramesOut, framesProcessed * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n            } else {\n                pFramesOutThisIteration = NULL;\n            }\n\n            /* Do a pre format conversion if necessary. */\n            if (pConverter->hasPreFormatConversion) {\n                ma_uint8 pTempBufferIn[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n                const ma_uint32 tempBufferInCap = sizeof(pTempBufferIn) / ma_get_bytes_per_frame(pConverter->channelConverter.format, pConverter->channelConverter.channelsIn);\n\n                frameCountThisIteration = (frameCount - framesProcessed);\n                if (frameCountThisIteration > tempBufferInCap) {\n                    frameCountThisIteration = tempBufferInCap;\n                }\n\n                if (pConverter->hasPostFormatConversion) {\n                    if (frameCountThisIteration > tempBufferOutCap) {\n                        frameCountThisIteration = tempBufferOutCap;\n                    }\n                }\n\n                if (pFramesInThisIteration != NULL) {\n                    ma_convert_pcm_frames_format(pTempBufferIn, pConverter->channelConverter.format, pFramesInThisIteration, pConverter->formatIn, frameCountThisIteration, pConverter->channelsIn, pConverter->ditherMode);\n                } else {\n                    MA_ZERO_MEMORY(pTempBufferIn, sizeof(pTempBufferIn));\n                }\n\n                if (pConverter->hasPostFormatConversion) {\n                    /* Both input and output conversion required. Output to the temp buffer. */\n                    result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pTempBufferOut, pTempBufferIn, frameCountThisIteration);\n                } else {\n                    /* Only pre-format required. Output straight to the output buffer. */\n                    result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pFramesOutThisIteration, pTempBufferIn, frameCountThisIteration);\n                }\n\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n            } else {\n                /* No pre-format required. Just read straight from the input buffer. */\n                MA_ASSERT(pConverter->hasPostFormatConversion == MA_TRUE);\n\n                frameCountThisIteration = (frameCount - framesProcessed);\n                if (frameCountThisIteration > tempBufferOutCap) {\n                    frameCountThisIteration = tempBufferOutCap;\n                }\n\n                result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pTempBufferOut, pFramesInThisIteration, frameCountThisIteration);\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n            }\n\n            /* If we are doing a post format conversion we need to do that now. */\n            if (pConverter->hasPostFormatConversion) {\n                if (pFramesOutThisIteration != NULL) {\n                    ma_convert_pcm_frames_format(pFramesOutThisIteration, pConverter->formatOut, pTempBufferOut, pConverter->channelConverter.format, frameCountThisIteration, pConverter->channelConverter.channelsOut, pConverter->ditherMode);\n                }\n            }\n\n            framesProcessed += frameCountThisIteration;\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = frameCount;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__resample_first(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_result result;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n    ma_uint8  pTempBufferIn[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];   /* In resampler format. */\n    ma_uint64 tempBufferInCap;\n    ma_uint8  pTempBufferMid[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In resampler format, channel converter input format. */\n    ma_uint64 tempBufferMidCap;\n    ma_uint8  pTempBufferOut[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In channel converter output format. */\n    ma_uint64 tempBufferOutCap;\n\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pConverter->resampler.format   == pConverter->channelConverter.format);\n    MA_ASSERT(pConverter->resampler.channels == pConverter->channelConverter.channelsIn);\n    MA_ASSERT(pConverter->resampler.channels <  pConverter->channelConverter.channelsOut);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    tempBufferInCap  = sizeof(pTempBufferIn)  / ma_get_bytes_per_frame(pConverter->resampler.format, pConverter->resampler.channels);\n    tempBufferMidCap = sizeof(pTempBufferIn)  / ma_get_bytes_per_frame(pConverter->resampler.format, pConverter->resampler.channels);\n    tempBufferOutCap = sizeof(pTempBufferOut) / ma_get_bytes_per_frame(pConverter->channelConverter.format, pConverter->channelConverter.channelsOut);\n\n    while (framesProcessedOut < frameCountOut) {\n        ma_uint64 frameCountInThisIteration;\n        ma_uint64 frameCountOutThisIteration;\n        const void* pRunningFramesIn = NULL;\n        void* pRunningFramesOut = NULL;\n        const void* pResampleBufferIn;\n        void* pChannelsBufferOut;\n\n        if (pFramesIn != NULL) {\n            pRunningFramesIn  = ma_offset_ptr(pFramesIn,  framesProcessedIn  * ma_get_bytes_per_frame(pConverter->formatIn, pConverter->channelsIn));\n        }\n        if (pFramesOut != NULL) {\n            pRunningFramesOut = ma_offset_ptr(pFramesOut, framesProcessedOut * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        }\n\n        /* Run input data through the resampler and output it to the temporary buffer. */\n        frameCountInThisIteration = (frameCountIn - framesProcessedIn);\n\n        if (pConverter->hasPreFormatConversion) {\n            if (frameCountInThisIteration > tempBufferInCap) {\n                frameCountInThisIteration = tempBufferInCap;\n            }\n        }\n\n        frameCountOutThisIteration = (frameCountOut - framesProcessedOut);\n        if (frameCountOutThisIteration > tempBufferMidCap) {\n            frameCountOutThisIteration = tempBufferMidCap;\n        }\n\n        /* We can't read more frames than can fit in the output buffer. */\n        if (pConverter->hasPostFormatConversion) {\n            if (frameCountOutThisIteration > tempBufferOutCap) {\n                frameCountOutThisIteration = tempBufferOutCap;\n            }\n        }\n\n        /* We need to ensure we don't try to process too many input frames that we run out of room in the output buffer. If this happens we'll end up glitching. */\n\n        /*\n        We need to try to predict how many input frames will be required for the resampler. If the\n        resampler can tell us, we'll use that. Otherwise we'll need to make a best guess. The further\n        off we are from this, the more wasted format conversions we'll end up doing.\n        */\n        #if 1\n        {\n            ma_uint64 requiredInputFrameCount;\n\n            result = ma_resampler_get_required_input_frame_count(&pConverter->resampler, frameCountOutThisIteration, &requiredInputFrameCount);\n            if (result != MA_SUCCESS) {\n                /* Fall back to a best guess. */\n                requiredInputFrameCount = (frameCountOutThisIteration * pConverter->resampler.sampleRateIn) / pConverter->resampler.sampleRateOut;\n            }\n\n            if (frameCountInThisIteration > requiredInputFrameCount) {\n                frameCountInThisIteration = requiredInputFrameCount;\n            }\n        }\n        #endif\n\n        if (pConverter->hasPreFormatConversion) {\n            if (pFramesIn != NULL) {\n                ma_convert_pcm_frames_format(pTempBufferIn, pConverter->resampler.format, pRunningFramesIn, pConverter->formatIn, frameCountInThisIteration, pConverter->channelsIn, pConverter->ditherMode);\n                pResampleBufferIn = pTempBufferIn;\n            } else {\n                pResampleBufferIn = NULL;\n            }\n        } else {\n            pResampleBufferIn = pRunningFramesIn;\n        }\n\n        result = ma_resampler_process_pcm_frames(&pConverter->resampler, pResampleBufferIn, &frameCountInThisIteration, pTempBufferMid, &frameCountOutThisIteration);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n\n        /*\n        The input data has been resampled so now we need to run it through the channel converter. The input data is always contained in pTempBufferMid. We only need to do\n        this part if we have an output buffer.\n        */\n        if (pFramesOut != NULL) {\n            if (pConverter->hasPostFormatConversion) {\n                pChannelsBufferOut = pTempBufferOut;\n            } else {\n                pChannelsBufferOut = pRunningFramesOut;\n            }\n\n            result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pChannelsBufferOut, pTempBufferMid, frameCountOutThisIteration);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n\n            /* Finally we do post format conversion. */\n            if (pConverter->hasPostFormatConversion) {\n                ma_convert_pcm_frames_format(pRunningFramesOut, pConverter->formatOut, pChannelsBufferOut, pConverter->channelConverter.format, frameCountOutThisIteration, pConverter->channelConverter.channelsOut, pConverter->ditherMode);\n            }\n        }\n\n\n        framesProcessedIn  += frameCountInThisIteration;\n        framesProcessedOut += frameCountOutThisIteration;\n\n        MA_ASSERT(framesProcessedIn  <= frameCountIn);\n        MA_ASSERT(framesProcessedOut <= frameCountOut);\n\n        if (frameCountOutThisIteration == 0) {\n            break;  /* Consumed all of our input data. */\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = framesProcessedIn;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = framesProcessedOut;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_data_converter_process_pcm_frames__channels_first(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    ma_result result;\n    ma_uint64 frameCountIn;\n    ma_uint64 frameCountOut;\n    ma_uint64 framesProcessedIn;\n    ma_uint64 framesProcessedOut;\n    ma_uint8  pTempBufferIn[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];   /* In resampler format. */\n    ma_uint64 tempBufferInCap;\n    ma_uint8  pTempBufferMid[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In resampler format, channel converter input format. */\n    ma_uint64 tempBufferMidCap;\n    ma_uint8  pTempBufferOut[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In channel converter output format. */\n    ma_uint64 tempBufferOutCap;\n\n    MA_ASSERT(pConverter != NULL);\n    MA_ASSERT(pConverter->resampler.format   == pConverter->channelConverter.format);\n    MA_ASSERT(pConverter->resampler.channels == pConverter->channelConverter.channelsOut);\n    MA_ASSERT(pConverter->resampler.channels <= pConverter->channelConverter.channelsIn);\n\n    frameCountIn = 0;\n    if (pFrameCountIn != NULL) {\n        frameCountIn = *pFrameCountIn;\n    }\n\n    frameCountOut = 0;\n    if (pFrameCountOut != NULL) {\n        frameCountOut = *pFrameCountOut;\n    }\n\n    framesProcessedIn  = 0;\n    framesProcessedOut = 0;\n\n    tempBufferInCap  = sizeof(pTempBufferIn)  / ma_get_bytes_per_frame(pConverter->channelConverter.format, pConverter->channelConverter.channelsIn);\n    tempBufferMidCap = sizeof(pTempBufferIn)  / ma_get_bytes_per_frame(pConverter->channelConverter.format, pConverter->channelConverter.channelsOut);\n    tempBufferOutCap = sizeof(pTempBufferOut) / ma_get_bytes_per_frame(pConverter->resampler.format, pConverter->resampler.channels);\n\n    while (framesProcessedOut < frameCountOut) {\n        ma_uint64 frameCountInThisIteration;\n        ma_uint64 frameCountOutThisIteration;\n        const void* pRunningFramesIn = NULL;\n        void* pRunningFramesOut = NULL;\n        const void* pChannelsBufferIn;\n        void* pResampleBufferOut;\n\n        if (pFramesIn != NULL) {\n            pRunningFramesIn  = ma_offset_ptr(pFramesIn,  framesProcessedIn  * ma_get_bytes_per_frame(pConverter->formatIn, pConverter->channelsIn));\n        }\n        if (pFramesOut != NULL) {\n            pRunningFramesOut = ma_offset_ptr(pFramesOut, framesProcessedOut * ma_get_bytes_per_frame(pConverter->formatOut, pConverter->channelsOut));\n        }\n\n        /*\n        Before doing any processing we need to determine how many frames we should try processing\n        this iteration, for both input and output. The resampler requires us to perform format and\n        channel conversion before passing any data into it. If we get our input count wrong, we'll\n        end up performing redundant pre-processing. This isn't the end of the world, but it does\n        result in some inefficiencies proportionate to how far our estimates are off.\n\n        If the resampler has a means to calculate exactly how much we'll need, we'll use that.\n        Otherwise we'll make a best guess. In order to do this, we'll need to calculate the output\n        frame count first.\n        */\n        frameCountOutThisIteration = (frameCountOut - framesProcessedOut);\n        if (frameCountOutThisIteration > tempBufferMidCap) {\n            frameCountOutThisIteration = tempBufferMidCap;\n        }\n\n        if (pConverter->hasPostFormatConversion) {\n            if (frameCountOutThisIteration > tempBufferOutCap) {\n                frameCountOutThisIteration = tempBufferOutCap;\n            }\n        }\n\n        /* Now that we have the output frame count we can determine the input frame count. */\n        frameCountInThisIteration = (frameCountIn - framesProcessedIn);\n        if (pConverter->hasPreFormatConversion) {\n            if (frameCountInThisIteration > tempBufferInCap) {\n                frameCountInThisIteration = tempBufferInCap;\n            }\n        }\n\n        if (frameCountInThisIteration > tempBufferMidCap) {\n            frameCountInThisIteration = tempBufferMidCap;\n        }\n\n        #if 1\n        {\n            ma_uint64 requiredInputFrameCount;\n\n            result = ma_resampler_get_required_input_frame_count(&pConverter->resampler, frameCountOutThisIteration, &requiredInputFrameCount);\n            if (result != MA_SUCCESS) {\n                /* Fall back to a best guess. */\n                requiredInputFrameCount = (frameCountOutThisIteration * pConverter->resampler.sampleRateIn) / pConverter->resampler.sampleRateOut;\n            }\n\n            if (frameCountInThisIteration > requiredInputFrameCount) {\n                frameCountInThisIteration = requiredInputFrameCount;\n            }\n        }\n        #endif\n\n\n        /* Pre format conversion. */\n        if (pConverter->hasPreFormatConversion) {\n            if (pRunningFramesIn != NULL) {\n                ma_convert_pcm_frames_format(pTempBufferIn, pConverter->channelConverter.format, pRunningFramesIn, pConverter->formatIn, frameCountInThisIteration, pConverter->channelsIn, pConverter->ditherMode);\n                pChannelsBufferIn = pTempBufferIn;\n            } else {\n                pChannelsBufferIn = NULL;\n            }\n        } else {\n            pChannelsBufferIn = pRunningFramesIn;\n        }\n\n\n        /* Channel conversion. */\n        result = ma_channel_converter_process_pcm_frames(&pConverter->channelConverter, pTempBufferMid, pChannelsBufferIn, frameCountInThisIteration);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n\n        /* Resampling. */\n        if (pConverter->hasPostFormatConversion) {\n            pResampleBufferOut = pTempBufferOut;\n        } else {\n            pResampleBufferOut = pRunningFramesOut;\n        }\n\n        result = ma_resampler_process_pcm_frames(&pConverter->resampler, pTempBufferMid, &frameCountInThisIteration, pResampleBufferOut, &frameCountOutThisIteration);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n\n        /* Post format conversion. */\n        if (pConverter->hasPostFormatConversion) {\n            if (pRunningFramesOut != NULL) {\n                ma_convert_pcm_frames_format(pRunningFramesOut, pConverter->formatOut, pResampleBufferOut, pConverter->resampler.format, frameCountOutThisIteration, pConverter->channelsOut, pConverter->ditherMode);\n            }\n        }\n\n\n        framesProcessedIn  += frameCountInThisIteration;\n        framesProcessedOut += frameCountOutThisIteration;\n\n        MA_ASSERT(framesProcessedIn  <= frameCountIn);\n        MA_ASSERT(framesProcessedOut <= frameCountOut);\n\n        if (frameCountOutThisIteration == 0) {\n            break;  /* Consumed all of our input data. */\n        }\n    }\n\n    if (pFrameCountIn != NULL) {\n        *pFrameCountIn = framesProcessedIn;\n    }\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = framesProcessedOut;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_process_pcm_frames(ma_data_converter* pConverter, const void* pFramesIn, ma_uint64* pFrameCountIn, void* pFramesOut, ma_uint64* pFrameCountOut)\n{\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    switch (pConverter->executionPath)\n    {\n        case ma_data_converter_execution_path_passthrough:    return ma_data_converter_process_pcm_frames__passthrough(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        case ma_data_converter_execution_path_format_only:    return ma_data_converter_process_pcm_frames__format_only(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        case ma_data_converter_execution_path_channels_only:  return ma_data_converter_process_pcm_frames__channels_only(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        case ma_data_converter_execution_path_resample_only:  return ma_data_converter_process_pcm_frames__resample_only(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        case ma_data_converter_execution_path_resample_first: return ma_data_converter_process_pcm_frames__resample_first(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        case ma_data_converter_execution_path_channels_first: return ma_data_converter_process_pcm_frames__channels_first(pConverter, pFramesIn, pFrameCountIn, pFramesOut, pFrameCountOut);\n        default: return MA_INVALID_OPERATION;   /* Should never hit this. */\n    }\n}\n\nMA_API ma_result ma_data_converter_set_rate(ma_data_converter* pConverter, ma_uint32 sampleRateIn, ma_uint32 sampleRateOut)\n{\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasResampler == MA_FALSE) {\n        return MA_INVALID_OPERATION;    /* Dynamic resampling not enabled. */\n    }\n\n    return ma_resampler_set_rate(&pConverter->resampler, sampleRateIn, sampleRateOut);\n}\n\nMA_API ma_result ma_data_converter_set_rate_ratio(ma_data_converter* pConverter, float ratioInOut)\n{\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasResampler == MA_FALSE) {\n        return MA_INVALID_OPERATION;    /* Dynamic resampling not enabled. */\n    }\n\n    return ma_resampler_set_rate_ratio(&pConverter->resampler, ratioInOut);\n}\n\nMA_API ma_uint64 ma_data_converter_get_input_latency(const ma_data_converter* pConverter)\n{\n    if (pConverter == NULL) {\n        return 0;\n    }\n\n    if (pConverter->hasResampler) {\n        return ma_resampler_get_input_latency(&pConverter->resampler);\n    }\n\n    return 0;   /* No latency without a resampler. */\n}\n\nMA_API ma_uint64 ma_data_converter_get_output_latency(const ma_data_converter* pConverter)\n{\n    if (pConverter == NULL) {\n        return 0;\n    }\n\n    if (pConverter->hasResampler) {\n        return ma_resampler_get_output_latency(&pConverter->resampler);\n    }\n\n    return 0;   /* No latency without a resampler. */\n}\n\nMA_API ma_result ma_data_converter_get_required_input_frame_count(const ma_data_converter* pConverter, ma_uint64 outputFrameCount, ma_uint64* pInputFrameCount)\n{\n    if (pInputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pInputFrameCount = 0;\n\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasResampler) {\n        return ma_resampler_get_required_input_frame_count(&pConverter->resampler, outputFrameCount, pInputFrameCount);\n    } else {\n        *pInputFrameCount = outputFrameCount;   /* 1:1 */\n        return MA_SUCCESS;\n    }\n}\n\nMA_API ma_result ma_data_converter_get_expected_output_frame_count(const ma_data_converter* pConverter, ma_uint64 inputFrameCount, ma_uint64* pOutputFrameCount)\n{\n    if (pOutputFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pOutputFrameCount = 0;\n\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasResampler) {\n        return ma_resampler_get_expected_output_frame_count(&pConverter->resampler, inputFrameCount, pOutputFrameCount);\n    } else {\n        *pOutputFrameCount = inputFrameCount;   /* 1:1 */\n        return MA_SUCCESS;\n    }\n}\n\nMA_API ma_result ma_data_converter_get_input_channel_map(const ma_data_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pConverter == NULL || pChannelMap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasChannelConverter) {\n        ma_channel_converter_get_output_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap);\n    } else {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pConverter->channelsOut);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_get_output_channel_map(const ma_data_converter* pConverter, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pConverter == NULL || pChannelMap == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConverter->hasChannelConverter) {\n        ma_channel_converter_get_input_channel_map(&pConverter->channelConverter, pChannelMap, channelMapCap);\n    } else {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pConverter->channelsIn);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_converter_reset(ma_data_converter* pConverter)\n{\n    if (pConverter == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* There's nothing to do if we're not resampling. */\n    if (pConverter->hasResampler == MA_FALSE) {\n        return MA_SUCCESS;\n    }\n\n    return ma_resampler_reset(&pConverter->resampler);\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nChannel Maps\n\n**************************************************************************************************************************************************************/\nstatic ma_channel ma_channel_map_init_standard_channel(ma_standard_channel_map standardChannelMap, ma_uint32 channelCount, ma_uint32 channelIndex);\n\nMA_API ma_channel ma_channel_map_get_channel(const ma_channel* pChannelMap, ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    if (pChannelMap == NULL) {\n        return ma_channel_map_init_standard_channel(ma_standard_channel_map_default, channelCount, channelIndex);\n    } else {\n        if (channelIndex >= channelCount) {\n            return MA_CHANNEL_NONE;\n        }\n\n        return pChannelMap[channelIndex];\n    }\n}\n\nMA_API void ma_channel_map_init_blank(ma_channel* pChannelMap, ma_uint32 channels)\n{\n    if (pChannelMap == NULL) {\n        return;\n    }\n\n    MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channels);\n}\n\n\nstatic ma_channel ma_channel_map_init_standard_channel_microsoft(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    if (channelCount == 0 || channelIndex >= channelCount) {\n        return MA_CHANNEL_NONE;\n    }\n\n    /* This is the Microsoft channel map. Based off the speaker configurations mentioned here: https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ksmedia/ns-ksmedia-ksaudio_channel_config */\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3: /* No defined, but best guess. */\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n            #ifndef MA_USE_QUAD_MICROSOFT_CHANNEL_MAP\n                /* Surround. Using the Surround profile has the advantage of the 3rd channel (MA_CHANNEL_FRONT_CENTER) mapping nicely with higher channel counts. */\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_BACK_CENTER;\n            #else\n                /* Quad. */\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            #endif\n            }\n        } break;\n\n        case 5: /* Not defined, but best guess. */\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 6:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_SIDE_LEFT;\n                case 5: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n\n        case 7: /* Not defined, but best guess. */\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_BACK_CENTER;\n                case 5: return MA_CHANNEL_SIDE_LEFT;\n                case 6: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n\n        case 8:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_BACK_LEFT;\n                case 5: return MA_CHANNEL_BACK_RIGHT;\n                case 6: return MA_CHANNEL_SIDE_LEFT;\n                case 7: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n    }\n\n    if (channelCount > 8) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 8));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_alsa(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 5:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 6:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n                case 5: return MA_CHANNEL_LFE;\n            }\n        } break;\n\n        case 7:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n                case 5: return MA_CHANNEL_LFE;\n                case 6: return MA_CHANNEL_BACK_CENTER;\n            }\n        } break;\n\n        case 8:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n                case 5: return MA_CHANNEL_LFE;\n                case 6: return MA_CHANNEL_SIDE_LEFT;\n                case 7: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n    }\n\n    if (channelCount > 8) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 8));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_rfc3551(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_BACK_CENTER;\n            }\n        } break;\n\n        case 5:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 6:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_SIDE_LEFT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_FRONT_RIGHT;\n                case 4: return MA_CHANNEL_SIDE_RIGHT;\n                case 5: return MA_CHANNEL_BACK_CENTER;\n            }\n        } break;\n    }\n\n    if (channelCount > 6) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 6));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_flac(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 5:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 6:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_BACK_LEFT;\n                case 5: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 7:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_BACK_CENTER;\n                case 5: return MA_CHANNEL_SIDE_LEFT;\n                case 6: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n\n        case 8:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_LFE;\n                case 4: return MA_CHANNEL_BACK_LEFT;\n                case 5: return MA_CHANNEL_BACK_RIGHT;\n                case 6: return MA_CHANNEL_SIDE_LEFT;\n                case 7: return MA_CHANNEL_SIDE_RIGHT;\n            }\n        } break;\n    }\n\n    if (channelCount > 8) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 8));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_vorbis(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 5:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 6:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n                case 5: return MA_CHANNEL_LFE;\n            }\n        } break;\n\n        case 7:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_SIDE_LEFT;\n                case 4: return MA_CHANNEL_SIDE_RIGHT;\n                case 5: return MA_CHANNEL_BACK_CENTER;\n                case 6: return MA_CHANNEL_LFE;\n            }\n        } break;\n\n        case 8:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_SIDE_LEFT;\n                case 4: return MA_CHANNEL_SIDE_RIGHT;\n                case 5: return MA_CHANNEL_BACK_LEFT;\n                case 6: return MA_CHANNEL_BACK_RIGHT;\n                case 7: return MA_CHANNEL_LFE;\n            }\n        } break;\n    }\n\n    if (channelCount > 8) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 8));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_sound4(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 5:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 6:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_BACK_LEFT;\n                case 4: return MA_CHANNEL_BACK_RIGHT;\n                case 5: return MA_CHANNEL_LFE;\n            }\n        } break;\n\n        case 7:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_SIDE_LEFT;\n                case 4: return MA_CHANNEL_SIDE_RIGHT;\n                case 5: return MA_CHANNEL_BACK_CENTER;\n                case 6: return MA_CHANNEL_LFE;\n            }\n        } break;\n\n        case 8:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_CENTER;\n                case 2: return MA_CHANNEL_FRONT_RIGHT;\n                case 3: return MA_CHANNEL_SIDE_LEFT;\n                case 4: return MA_CHANNEL_SIDE_RIGHT;\n                case 5: return MA_CHANNEL_BACK_LEFT;\n                case 6: return MA_CHANNEL_BACK_RIGHT;\n                case 7: return MA_CHANNEL_LFE;\n            }\n        } break;\n    }\n\n    if (channelCount > 8) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 8));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\nstatic ma_channel ma_channel_map_init_standard_channel_sndio(ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    switch (channelCount)\n    {\n        case 0: return MA_CHANNEL_NONE;\n\n        case 1:\n        {\n            return MA_CHANNEL_MONO;\n        } break;\n\n        case 2:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n            }\n        } break;\n\n        case 3: /* No defined, but best guess. */\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 4:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n            }\n        } break;\n\n        case 5: /* Not defined, but best guess. */\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n            }\n        } break;\n\n        case 6:\n        default:\n        {\n            switch (channelIndex) {\n                case 0: return MA_CHANNEL_FRONT_LEFT;\n                case 1: return MA_CHANNEL_FRONT_RIGHT;\n                case 2: return MA_CHANNEL_BACK_LEFT;\n                case 3: return MA_CHANNEL_BACK_RIGHT;\n                case 4: return MA_CHANNEL_FRONT_CENTER;\n                case 5: return MA_CHANNEL_LFE;\n            }\n        } break;\n    }\n\n    if (channelCount > 6) {\n        if (channelIndex < 32) {    /* We have 32 AUX channels. */\n            return (ma_channel)(MA_CHANNEL_AUX_0 + (channelIndex - 6));\n        }\n    }\n\n    /* Getting here means we don't know how to map the channel position so just return MA_CHANNEL_NONE. */\n    return MA_CHANNEL_NONE;\n}\n\n\nstatic ma_channel ma_channel_map_init_standard_channel(ma_standard_channel_map standardChannelMap, ma_uint32 channelCount, ma_uint32 channelIndex)\n{\n    if (channelCount == 0 || channelIndex >= channelCount) {\n        return MA_CHANNEL_NONE;\n    }\n\n    switch (standardChannelMap)\n    {\n        case ma_standard_channel_map_alsa:\n        {\n            return ma_channel_map_init_standard_channel_alsa(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_rfc3551:\n        {\n            return ma_channel_map_init_standard_channel_rfc3551(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_flac:\n        {\n            return ma_channel_map_init_standard_channel_flac(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_vorbis:\n        {\n            return ma_channel_map_init_standard_channel_vorbis(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_sound4:\n        {\n            return ma_channel_map_init_standard_channel_sound4(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_sndio:\n        {\n            return ma_channel_map_init_standard_channel_sndio(channelCount, channelIndex);\n        } break;\n\n        case ma_standard_channel_map_microsoft: /* Also default. */\n        /*case ma_standard_channel_map_default;*/\n        default:\n        {\n            return ma_channel_map_init_standard_channel_microsoft(channelCount, channelIndex);\n        } break;\n    }\n}\n\nMA_API void ma_channel_map_init_standard(ma_standard_channel_map standardChannelMap, ma_channel* pChannelMap, size_t channelMapCap, ma_uint32 channels)\n{\n    ma_uint32 iChannel;\n\n    if (pChannelMap == NULL || channelMapCap == 0 || channels == 0) {\n        return;\n    }\n\n    for (iChannel = 0; iChannel < channels; iChannel += 1) {\n        if (channelMapCap == 0) {\n            break;  /* Ran out of room. */\n        }\n\n        pChannelMap[0] = ma_channel_map_init_standard_channel(standardChannelMap, channels, iChannel);\n        pChannelMap   += 1;\n        channelMapCap -= 1;\n    }\n}\n\nMA_API void ma_channel_map_copy(ma_channel* pOut, const ma_channel* pIn, ma_uint32 channels)\n{\n    if (pOut != NULL && pIn != NULL && channels > 0) {\n        MA_COPY_MEMORY(pOut, pIn, sizeof(*pOut) * channels);\n    }\n}\n\nMA_API void ma_channel_map_copy_or_default(ma_channel* pOut, size_t channelMapCapOut, const ma_channel* pIn, ma_uint32 channels)\n{\n    if (pOut == NULL || channels == 0) {\n        return;\n    }\n\n    if (pIn != NULL) {\n        ma_channel_map_copy(pOut, pIn, channels);\n    } else {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pOut, channelMapCapOut, channels);\n    }\n}\n\nMA_API ma_bool32 ma_channel_map_is_valid(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    /* A channel count of 0 is invalid. */\n    if (channels == 0) {\n        return MA_FALSE;\n    }\n\n    /* It does not make sense to have a mono channel when there is more than 1 channel. */\n    if (channels > 1) {\n        ma_uint32 iChannel;\n        for (iChannel = 0; iChannel < channels; ++iChannel) {\n            if (ma_channel_map_get_channel(pChannelMap, channels, iChannel) == MA_CHANNEL_MONO) {\n                return MA_FALSE;\n            }\n        }\n    }\n\n    return MA_TRUE;\n}\n\nMA_API ma_bool32 ma_channel_map_is_equal(const ma_channel* pChannelMapA, const ma_channel* pChannelMapB, ma_uint32 channels)\n{\n    ma_uint32 iChannel;\n\n    if (pChannelMapA == pChannelMapB) {\n        return MA_TRUE;\n    }\n\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        if (ma_channel_map_get_channel(pChannelMapA, channels, iChannel) != ma_channel_map_get_channel(pChannelMapB, channels, iChannel)) {\n            return MA_FALSE;\n        }\n    }\n\n    return MA_TRUE;\n}\n\nMA_API ma_bool32 ma_channel_map_is_blank(const ma_channel* pChannelMap, ma_uint32 channels)\n{\n    ma_uint32 iChannel;\n\n    /* A null channel map is equivalent to the default channel map. */\n    if (pChannelMap == NULL) {\n        return MA_FALSE;\n    }\n\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        if (pChannelMap[iChannel] != MA_CHANNEL_NONE) {\n            return MA_FALSE;\n        }\n    }\n\n    return MA_TRUE;\n}\n\nMA_API ma_bool32 ma_channel_map_contains_channel_position(ma_uint32 channels, const ma_channel* pChannelMap, ma_channel channelPosition)\n{\n    return ma_channel_map_find_channel_position(channels, pChannelMap, channelPosition, NULL);\n}\n\nMA_API ma_bool32 ma_channel_map_find_channel_position(ma_uint32 channels, const ma_channel* pChannelMap, ma_channel channelPosition, ma_uint32* pChannelIndex)\n{\n    ma_uint32 iChannel;\n\n    if (pChannelIndex != NULL) {\n        *pChannelIndex = (ma_uint32)-1;\n    }\n\n    for (iChannel = 0; iChannel < channels; ++iChannel) {\n        if (ma_channel_map_get_channel(pChannelMap, channels, iChannel) == channelPosition) {\n            if (pChannelIndex != NULL) {\n                *pChannelIndex = iChannel;\n            }\n\n            return MA_TRUE;\n        }\n    }\n\n    /* Getting here means the channel position was not found. */\n    return MA_FALSE;\n}\n\nMA_API size_t ma_channel_map_to_string(const ma_channel* pChannelMap, ma_uint32 channels, char* pBufferOut, size_t bufferCap)\n{\n    size_t len;\n    ma_uint32 iChannel;\n\n    len = 0;\n\n    for (iChannel = 0; iChannel < channels; iChannel += 1) {\n        const char* pChannelStr = ma_channel_position_to_string(ma_channel_map_get_channel(pChannelMap, channels, iChannel));\n        size_t channelStrLen = strlen(pChannelStr);\n\n        /* Append the string if necessary. */\n        if (pBufferOut != NULL && bufferCap > len + channelStrLen) {\n            MA_COPY_MEMORY(pBufferOut + len, pChannelStr, channelStrLen);\n        }\n        len += channelStrLen;\n\n        /* Append a space if it's not the last item. */\n        if (iChannel+1 < channels) {\n            if (pBufferOut != NULL && bufferCap > len + 1) {\n                pBufferOut[len] = ' ';\n            }\n            len += 1;\n        }\n    }\n\n    /* Null terminate. Don't increment the length here. */\n    if (pBufferOut != NULL) {\n        if (bufferCap > len) {\n            pBufferOut[len] = '\\0';\n        } else if (bufferCap > 0) {\n            pBufferOut[bufferCap - 1] = '\\0';\n        }\n    }\n\n    return len;\n}\n\nMA_API const char* ma_channel_position_to_string(ma_channel channel)\n{\n    switch (channel)\n    {\n        case MA_CHANNEL_NONE              : return \"CHANNEL_NONE\";\n        case MA_CHANNEL_MONO              : return \"CHANNEL_MONO\";\n        case MA_CHANNEL_FRONT_LEFT        : return \"CHANNEL_FRONT_LEFT\";\n        case MA_CHANNEL_FRONT_RIGHT       : return \"CHANNEL_FRONT_RIGHT\";\n        case MA_CHANNEL_FRONT_CENTER      : return \"CHANNEL_FRONT_CENTER\";\n        case MA_CHANNEL_LFE               : return \"CHANNEL_LFE\";\n        case MA_CHANNEL_BACK_LEFT         : return \"CHANNEL_BACK_LEFT\";\n        case MA_CHANNEL_BACK_RIGHT        : return \"CHANNEL_BACK_RIGHT\";\n        case MA_CHANNEL_FRONT_LEFT_CENTER : return \"CHANNEL_FRONT_LEFT_CENTER\";\n        case MA_CHANNEL_FRONT_RIGHT_CENTER: return \"CHANNEL_FRONT_RIGHT_CENTER\";\n        case MA_CHANNEL_BACK_CENTER       : return \"CHANNEL_BACK_CENTER\";\n        case MA_CHANNEL_SIDE_LEFT         : return \"CHANNEL_SIDE_LEFT\";\n        case MA_CHANNEL_SIDE_RIGHT        : return \"CHANNEL_SIDE_RIGHT\";\n        case MA_CHANNEL_TOP_CENTER        : return \"CHANNEL_TOP_CENTER\";\n        case MA_CHANNEL_TOP_FRONT_LEFT    : return \"CHANNEL_TOP_FRONT_LEFT\";\n        case MA_CHANNEL_TOP_FRONT_CENTER  : return \"CHANNEL_TOP_FRONT_CENTER\";\n        case MA_CHANNEL_TOP_FRONT_RIGHT   : return \"CHANNEL_TOP_FRONT_RIGHT\";\n        case MA_CHANNEL_TOP_BACK_LEFT     : return \"CHANNEL_TOP_BACK_LEFT\";\n        case MA_CHANNEL_TOP_BACK_CENTER   : return \"CHANNEL_TOP_BACK_CENTER\";\n        case MA_CHANNEL_TOP_BACK_RIGHT    : return \"CHANNEL_TOP_BACK_RIGHT\";\n        case MA_CHANNEL_AUX_0             : return \"CHANNEL_AUX_0\";\n        case MA_CHANNEL_AUX_1             : return \"CHANNEL_AUX_1\";\n        case MA_CHANNEL_AUX_2             : return \"CHANNEL_AUX_2\";\n        case MA_CHANNEL_AUX_3             : return \"CHANNEL_AUX_3\";\n        case MA_CHANNEL_AUX_4             : return \"CHANNEL_AUX_4\";\n        case MA_CHANNEL_AUX_5             : return \"CHANNEL_AUX_5\";\n        case MA_CHANNEL_AUX_6             : return \"CHANNEL_AUX_6\";\n        case MA_CHANNEL_AUX_7             : return \"CHANNEL_AUX_7\";\n        case MA_CHANNEL_AUX_8             : return \"CHANNEL_AUX_8\";\n        case MA_CHANNEL_AUX_9             : return \"CHANNEL_AUX_9\";\n        case MA_CHANNEL_AUX_10            : return \"CHANNEL_AUX_10\";\n        case MA_CHANNEL_AUX_11            : return \"CHANNEL_AUX_11\";\n        case MA_CHANNEL_AUX_12            : return \"CHANNEL_AUX_12\";\n        case MA_CHANNEL_AUX_13            : return \"CHANNEL_AUX_13\";\n        case MA_CHANNEL_AUX_14            : return \"CHANNEL_AUX_14\";\n        case MA_CHANNEL_AUX_15            : return \"CHANNEL_AUX_15\";\n        case MA_CHANNEL_AUX_16            : return \"CHANNEL_AUX_16\";\n        case MA_CHANNEL_AUX_17            : return \"CHANNEL_AUX_17\";\n        case MA_CHANNEL_AUX_18            : return \"CHANNEL_AUX_18\";\n        case MA_CHANNEL_AUX_19            : return \"CHANNEL_AUX_19\";\n        case MA_CHANNEL_AUX_20            : return \"CHANNEL_AUX_20\";\n        case MA_CHANNEL_AUX_21            : return \"CHANNEL_AUX_21\";\n        case MA_CHANNEL_AUX_22            : return \"CHANNEL_AUX_22\";\n        case MA_CHANNEL_AUX_23            : return \"CHANNEL_AUX_23\";\n        case MA_CHANNEL_AUX_24            : return \"CHANNEL_AUX_24\";\n        case MA_CHANNEL_AUX_25            : return \"CHANNEL_AUX_25\";\n        case MA_CHANNEL_AUX_26            : return \"CHANNEL_AUX_26\";\n        case MA_CHANNEL_AUX_27            : return \"CHANNEL_AUX_27\";\n        case MA_CHANNEL_AUX_28            : return \"CHANNEL_AUX_28\";\n        case MA_CHANNEL_AUX_29            : return \"CHANNEL_AUX_29\";\n        case MA_CHANNEL_AUX_30            : return \"CHANNEL_AUX_30\";\n        case MA_CHANNEL_AUX_31            : return \"CHANNEL_AUX_31\";\n        default: break;\n    }\n\n    return \"UNKNOWN\";\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nConversion Helpers\n\n**************************************************************************************************************************************************************/\nMA_API ma_uint64 ma_convert_frames(void* pOut, ma_uint64 frameCountOut, ma_format formatOut, ma_uint32 channelsOut, ma_uint32 sampleRateOut, const void* pIn, ma_uint64 frameCountIn, ma_format formatIn, ma_uint32 channelsIn, ma_uint32 sampleRateIn)\n{\n    ma_data_converter_config config;\n\n    config = ma_data_converter_config_init(formatIn, formatOut, channelsIn, channelsOut, sampleRateIn, sampleRateOut);\n    config.resampling.linear.lpfOrder = ma_min(MA_DEFAULT_RESAMPLER_LPF_ORDER, MA_MAX_FILTER_ORDER);\n\n    return ma_convert_frames_ex(pOut, frameCountOut, pIn, frameCountIn, &config);\n}\n\nMA_API ma_uint64 ma_convert_frames_ex(void* pOut, ma_uint64 frameCountOut, const void* pIn, ma_uint64 frameCountIn, const ma_data_converter_config* pConfig)\n{\n    ma_result result;\n    ma_data_converter converter;\n\n    if (frameCountIn == 0 || pConfig == NULL) {\n        return 0;\n    }\n\n    result = ma_data_converter_init(pConfig, NULL, &converter);\n    if (result != MA_SUCCESS) {\n        return 0;   /* Failed to initialize the data converter. */\n    }\n\n    if (pOut == NULL) {\n        result = ma_data_converter_get_expected_output_frame_count(&converter, frameCountIn, &frameCountOut);\n        if (result != MA_SUCCESS) {\n            if (result == MA_NOT_IMPLEMENTED) {\n                /* No way to calculate the number of frames, so we'll need to brute force it and loop. */\n                frameCountOut = 0;\n\n                while (frameCountIn > 0) {\n                    ma_uint64 framesProcessedIn  = frameCountIn;\n                    ma_uint64 framesProcessedOut = 0xFFFFFFFF;\n\n                    result = ma_data_converter_process_pcm_frames(&converter, pIn, &framesProcessedIn, NULL, &framesProcessedOut);\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n\n                    frameCountIn  -= framesProcessedIn;\n                }\n            }\n        }\n    } else {\n        result = ma_data_converter_process_pcm_frames(&converter, pIn, &frameCountIn, pOut, &frameCountOut);\n        if (result != MA_SUCCESS) {\n            frameCountOut = 0;\n        }\n    }\n\n    ma_data_converter_uninit(&converter, NULL);\n    return frameCountOut;\n}\n\n\n/**************************************************************************************************************************************************************\n\nRing Buffer\n\n**************************************************************************************************************************************************************/\nstatic MA_INLINE ma_uint32 ma_rb__extract_offset_in_bytes(ma_uint32 encodedOffset)\n{\n    return encodedOffset & 0x7FFFFFFF;\n}\n\nstatic MA_INLINE ma_uint32 ma_rb__extract_offset_loop_flag(ma_uint32 encodedOffset)\n{\n    return encodedOffset & 0x80000000;\n}\n\nstatic MA_INLINE void* ma_rb__get_read_ptr(ma_rb* pRB)\n{\n    MA_ASSERT(pRB != NULL);\n    return ma_offset_ptr(pRB->pBuffer, ma_rb__extract_offset_in_bytes(ma_atomic_load_32(&pRB->encodedReadOffset)));\n}\n\nstatic MA_INLINE void* ma_rb__get_write_ptr(ma_rb* pRB)\n{\n    MA_ASSERT(pRB != NULL);\n    return ma_offset_ptr(pRB->pBuffer, ma_rb__extract_offset_in_bytes(ma_atomic_load_32(&pRB->encodedWriteOffset)));\n}\n\nstatic MA_INLINE ma_uint32 ma_rb__construct_offset(ma_uint32 offsetInBytes, ma_uint32 offsetLoopFlag)\n{\n    return offsetLoopFlag | offsetInBytes;\n}\n\nstatic MA_INLINE void ma_rb__deconstruct_offset(ma_uint32 encodedOffset, ma_uint32* pOffsetInBytes, ma_uint32* pOffsetLoopFlag)\n{\n    MA_ASSERT(pOffsetInBytes != NULL);\n    MA_ASSERT(pOffsetLoopFlag != NULL);\n\n    *pOffsetInBytes  = ma_rb__extract_offset_in_bytes(encodedOffset);\n    *pOffsetLoopFlag = ma_rb__extract_offset_loop_flag(encodedOffset);\n}\n\n\nMA_API ma_result ma_rb_init_ex(size_t subbufferSizeInBytes, size_t subbufferCount, size_t subbufferStrideInBytes, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_rb* pRB)\n{\n    ma_result result;\n    const ma_uint32 maxSubBufferSize = 0x7FFFFFFF - (MA_SIMD_ALIGNMENT-1);\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (subbufferSizeInBytes == 0 || subbufferCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (subbufferSizeInBytes > maxSubBufferSize) {\n        return MA_INVALID_ARGS;    /* Maximum buffer size is ~2GB. The most significant bit is a flag for use internally. */\n    }\n\n\n    MA_ZERO_OBJECT(pRB);\n\n    result = ma_allocation_callbacks_init_copy(&pRB->allocationCallbacks, pAllocationCallbacks);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pRB->subbufferSizeInBytes = (ma_uint32)subbufferSizeInBytes;\n    pRB->subbufferCount = (ma_uint32)subbufferCount;\n\n    if (pOptionalPreallocatedBuffer != NULL) {\n        pRB->subbufferStrideInBytes = (ma_uint32)subbufferStrideInBytes;\n        pRB->pBuffer = pOptionalPreallocatedBuffer;\n    } else {\n        size_t bufferSizeInBytes;\n\n        /*\n        Here is where we allocate our own buffer. We always want to align this to MA_SIMD_ALIGNMENT for future SIMD optimization opportunity. To do this\n        we need to make sure the stride is a multiple of MA_SIMD_ALIGNMENT.\n        */\n        pRB->subbufferStrideInBytes = ma_align(pRB->subbufferSizeInBytes, MA_SIMD_ALIGNMENT);\n\n        bufferSizeInBytes = (size_t)pRB->subbufferCount*pRB->subbufferStrideInBytes;\n        pRB->pBuffer = ma_aligned_malloc(bufferSizeInBytes, MA_SIMD_ALIGNMENT, &pRB->allocationCallbacks);\n        if (pRB->pBuffer == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        MA_ZERO_MEMORY(pRB->pBuffer, bufferSizeInBytes);\n        pRB->ownsBuffer = MA_TRUE;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_init(size_t bufferSizeInBytes, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_rb* pRB)\n{\n    return ma_rb_init_ex(bufferSizeInBytes, 1, 0, pOptionalPreallocatedBuffer, pAllocationCallbacks, pRB);\n}\n\nMA_API void ma_rb_uninit(ma_rb* pRB)\n{\n    if (pRB == NULL) {\n        return;\n    }\n\n    if (pRB->ownsBuffer) {\n        ma_aligned_free(pRB->pBuffer, &pRB->allocationCallbacks);\n    }\n}\n\nMA_API void ma_rb_reset(ma_rb* pRB)\n{\n    if (pRB == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_32(&pRB->encodedReadOffset, 0);\n    ma_atomic_exchange_32(&pRB->encodedWriteOffset, 0);\n}\n\nMA_API ma_result ma_rb_acquire_read(ma_rb* pRB, size_t* pSizeInBytes, void** ppBufferOut)\n{\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    size_t bytesAvailable;\n    size_t bytesRequested;\n\n    if (pRB == NULL || pSizeInBytes == NULL || ppBufferOut == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The returned buffer should never move ahead of the write pointer. */\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    /*\n    The number of bytes available depends on whether or not the read and write pointers are on the same loop iteration. If so, we\n    can only read up to the write pointer. If not, we can only read up to the end of the buffer.\n    */\n    if (readOffsetLoopFlag == writeOffsetLoopFlag) {\n        bytesAvailable = writeOffsetInBytes - readOffsetInBytes;\n    } else {\n        bytesAvailable = pRB->subbufferSizeInBytes - readOffsetInBytes;\n    }\n\n    bytesRequested = *pSizeInBytes;\n    if (bytesRequested > bytesAvailable) {\n        bytesRequested = bytesAvailable;\n    }\n\n    *pSizeInBytes = bytesRequested;\n    (*ppBufferOut) = ma_rb__get_read_ptr(pRB);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_commit_read(ma_rb* pRB, size_t sizeInBytes)\n{\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    ma_uint32 newReadOffsetInBytes;\n    ma_uint32 newReadOffsetLoopFlag;\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    /* Check that sizeInBytes is correct. It should never go beyond the end of the buffer. */\n    newReadOffsetInBytes = (ma_uint32)(readOffsetInBytes + sizeInBytes);\n    if (newReadOffsetInBytes > pRB->subbufferSizeInBytes) {\n        return MA_INVALID_ARGS;    /* <-- sizeInBytes will cause the read offset to overflow. */\n    }\n\n    /* Move the read pointer back to the start if necessary. */\n    newReadOffsetLoopFlag = readOffsetLoopFlag;\n    if (newReadOffsetInBytes == pRB->subbufferSizeInBytes) {\n        newReadOffsetInBytes = 0;\n        newReadOffsetLoopFlag ^= 0x80000000;\n    }\n\n    ma_atomic_exchange_32(&pRB->encodedReadOffset, ma_rb__construct_offset(newReadOffsetInBytes, newReadOffsetLoopFlag));\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_acquire_write(ma_rb* pRB, size_t* pSizeInBytes, void** ppBufferOut)\n{\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n    size_t bytesAvailable;\n    size_t bytesRequested;\n\n    if (pRB == NULL || pSizeInBytes == NULL || ppBufferOut == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The returned buffer should never overtake the read buffer. */\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    /*\n    In the case of writing, if the write pointer and the read pointer are on the same loop iteration we can only\n    write up to the end of the buffer. Otherwise we can only write up to the read pointer. The write pointer should\n    never overtake the read pointer.\n    */\n    if (writeOffsetLoopFlag == readOffsetLoopFlag) {\n        bytesAvailable = pRB->subbufferSizeInBytes - writeOffsetInBytes;\n    } else {\n        bytesAvailable = readOffsetInBytes - writeOffsetInBytes;\n    }\n\n    bytesRequested = *pSizeInBytes;\n    if (bytesRequested > bytesAvailable) {\n        bytesRequested = bytesAvailable;\n    }\n\n    *pSizeInBytes = bytesRequested;\n    *ppBufferOut  = ma_rb__get_write_ptr(pRB);\n\n    /* Clear the buffer if desired. */\n    if (pRB->clearOnWriteAcquire) {\n        MA_ZERO_MEMORY(*ppBufferOut, *pSizeInBytes);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_commit_write(ma_rb* pRB, size_t sizeInBytes)\n{\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n    ma_uint32 newWriteOffsetInBytes;\n    ma_uint32 newWriteOffsetLoopFlag;\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    /* Check that sizeInBytes is correct. It should never go beyond the end of the buffer. */\n    newWriteOffsetInBytes = (ma_uint32)(writeOffsetInBytes + sizeInBytes);\n    if (newWriteOffsetInBytes > pRB->subbufferSizeInBytes) {\n        return MA_INVALID_ARGS;    /* <-- sizeInBytes will cause the read offset to overflow. */\n    }\n\n    /* Move the read pointer back to the start if necessary. */\n    newWriteOffsetLoopFlag = writeOffsetLoopFlag;\n    if (newWriteOffsetInBytes == pRB->subbufferSizeInBytes) {\n        newWriteOffsetInBytes = 0;\n        newWriteOffsetLoopFlag ^= 0x80000000;\n    }\n\n    ma_atomic_exchange_32(&pRB->encodedWriteOffset, ma_rb__construct_offset(newWriteOffsetInBytes, newWriteOffsetLoopFlag));\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_seek_read(ma_rb* pRB, size_t offsetInBytes)\n{\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n    ma_uint32 newReadOffsetInBytes;\n    ma_uint32 newReadOffsetLoopFlag;\n\n    if (pRB == NULL || offsetInBytes > pRB->subbufferSizeInBytes) {\n        return MA_INVALID_ARGS;\n    }\n\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    newReadOffsetLoopFlag = readOffsetLoopFlag;\n\n    /* We cannot go past the write buffer. */\n    if (readOffsetLoopFlag == writeOffsetLoopFlag) {\n        if ((readOffsetInBytes + offsetInBytes) > writeOffsetInBytes) {\n            newReadOffsetInBytes = writeOffsetInBytes;\n        } else {\n            newReadOffsetInBytes = (ma_uint32)(readOffsetInBytes + offsetInBytes);\n        }\n    } else {\n        /* May end up looping. */\n        if ((readOffsetInBytes + offsetInBytes) >= pRB->subbufferSizeInBytes) {\n            newReadOffsetInBytes = (ma_uint32)(readOffsetInBytes + offsetInBytes) - pRB->subbufferSizeInBytes;\n            newReadOffsetLoopFlag ^= 0x80000000;    /* <-- Looped. */\n        } else {\n            newReadOffsetInBytes = (ma_uint32)(readOffsetInBytes + offsetInBytes);\n        }\n    }\n\n    ma_atomic_exchange_32(&pRB->encodedReadOffset, ma_rb__construct_offset(newReadOffsetInBytes, newReadOffsetLoopFlag));\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_rb_seek_write(ma_rb* pRB, size_t offsetInBytes)\n{\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n    ma_uint32 newWriteOffsetInBytes;\n    ma_uint32 newWriteOffsetLoopFlag;\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    newWriteOffsetLoopFlag = writeOffsetLoopFlag;\n\n    /* We cannot go past the write buffer. */\n    if (readOffsetLoopFlag == writeOffsetLoopFlag) {\n        /* May end up looping. */\n        if ((writeOffsetInBytes + offsetInBytes) >= pRB->subbufferSizeInBytes) {\n            newWriteOffsetInBytes = (ma_uint32)(writeOffsetInBytes + offsetInBytes) - pRB->subbufferSizeInBytes;\n            newWriteOffsetLoopFlag ^= 0x80000000;    /* <-- Looped. */\n        } else {\n            newWriteOffsetInBytes = (ma_uint32)(writeOffsetInBytes + offsetInBytes);\n        }\n    } else {\n        if ((writeOffsetInBytes + offsetInBytes) > readOffsetInBytes) {\n            newWriteOffsetInBytes = readOffsetInBytes;\n        } else {\n            newWriteOffsetInBytes = (ma_uint32)(writeOffsetInBytes + offsetInBytes);\n        }\n    }\n\n    ma_atomic_exchange_32(&pRB->encodedWriteOffset, ma_rb__construct_offset(newWriteOffsetInBytes, newWriteOffsetLoopFlag));\n    return MA_SUCCESS;\n}\n\nMA_API ma_int32 ma_rb_pointer_distance(ma_rb* pRB)\n{\n    ma_uint32 readOffset;\n    ma_uint32 readOffsetInBytes;\n    ma_uint32 readOffsetLoopFlag;\n    ma_uint32 writeOffset;\n    ma_uint32 writeOffsetInBytes;\n    ma_uint32 writeOffsetLoopFlag;\n\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    readOffset = ma_atomic_load_32(&pRB->encodedReadOffset);\n    ma_rb__deconstruct_offset(readOffset, &readOffsetInBytes, &readOffsetLoopFlag);\n\n    writeOffset = ma_atomic_load_32(&pRB->encodedWriteOffset);\n    ma_rb__deconstruct_offset(writeOffset, &writeOffsetInBytes, &writeOffsetLoopFlag);\n\n    if (readOffsetLoopFlag == writeOffsetLoopFlag) {\n        return writeOffsetInBytes - readOffsetInBytes;\n    } else {\n        return writeOffsetInBytes + (pRB->subbufferSizeInBytes - readOffsetInBytes);\n    }\n}\n\nMA_API ma_uint32 ma_rb_available_read(ma_rb* pRB)\n{\n    ma_int32 dist;\n\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    dist = ma_rb_pointer_distance(pRB);\n    if (dist < 0) {\n        return 0;\n    }\n\n    return dist;\n}\n\nMA_API ma_uint32 ma_rb_available_write(ma_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return (ma_uint32)(ma_rb_get_subbuffer_size(pRB) - ma_rb_pointer_distance(pRB));\n}\n\nMA_API size_t ma_rb_get_subbuffer_size(ma_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return pRB->subbufferSizeInBytes;\n}\n\nMA_API size_t ma_rb_get_subbuffer_stride(ma_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    if (pRB->subbufferStrideInBytes == 0) {\n        return (size_t)pRB->subbufferSizeInBytes;\n    }\n\n    return (size_t)pRB->subbufferStrideInBytes;\n}\n\nMA_API size_t ma_rb_get_subbuffer_offset(ma_rb* pRB, size_t subbufferIndex)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return subbufferIndex * ma_rb_get_subbuffer_stride(pRB);\n}\n\nMA_API void* ma_rb_get_subbuffer_ptr(ma_rb* pRB, size_t subbufferIndex, void* pBuffer)\n{\n    if (pRB == NULL) {\n        return NULL;\n    }\n\n    return ma_offset_ptr(pBuffer, ma_rb_get_subbuffer_offset(pRB, subbufferIndex));\n}\n\n\n\nstatic ma_result ma_pcm_rb_data_source__on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    /* Since there's no notion of an end, we don't ever want to return MA_AT_END here. But it is possible to return 0. */\n    ma_pcm_rb* pRB = (ma_pcm_rb*)pDataSource;\n    ma_result result;\n    ma_uint64 totalFramesRead;\n\n    MA_ASSERT(pRB != NULL);\n\n    /* We need to run this in a loop since the ring buffer itself may loop. */\n    totalFramesRead = 0;\n    while (totalFramesRead < frameCount) {\n        void* pMappedBuffer;\n        ma_uint32 mappedFrameCount;\n        ma_uint64 framesToRead = frameCount - totalFramesRead;\n        if (framesToRead > 0xFFFFFFFF) {\n            framesToRead = 0xFFFFFFFF;\n        }\n\n        mappedFrameCount = (ma_uint32)framesToRead;\n        result = ma_pcm_rb_acquire_read(pRB, &mappedFrameCount, &pMappedBuffer);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        if (mappedFrameCount == 0) {\n            break;  /* <-- End of ring buffer. */\n        }\n\n        ma_copy_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, pRB->format, pRB->channels), pMappedBuffer, mappedFrameCount, pRB->format, pRB->channels);\n\n        result = ma_pcm_rb_commit_read(pRB, mappedFrameCount);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        totalFramesRead += mappedFrameCount;\n    }\n\n    /*\n    There is no notion of an \"end\" in a ring buffer. If we didn't have enough data to fill the requested frame\n    count we'll need to pad with silence. If we don't do this, totalFramesRead might equal 0 which will result\n    in the data source layer at a higher level translating this to MA_AT_END which is incorrect for a ring buffer.\n    */\n    if (totalFramesRead < frameCount) {\n        ma_silence_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, pRB->format, pRB->channels), (frameCount - totalFramesRead), pRB->format, pRB->channels);\n        totalFramesRead = frameCount;\n    }\n\n    *pFramesRead = totalFramesRead;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_pcm_rb_data_source__on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_pcm_rb* pRB = (ma_pcm_rb*)pDataSource;\n    MA_ASSERT(pRB != NULL);\n\n    if (pFormat != NULL) {\n        *pFormat = pRB->format;\n    }\n\n    if (pChannels != NULL) {\n        *pChannels = pRB->channels;\n    }\n\n    if (pSampleRate != NULL) {\n        *pSampleRate = pRB->sampleRate;\n    }\n\n    /* Just assume the default channel map. */\n    if (pChannelMap != NULL) {\n        ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pRB->channels);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_data_source_vtable ma_gRBDataSourceVTable =\n{\n    ma_pcm_rb_data_source__on_read,\n    NULL,   /* onSeek */\n    ma_pcm_rb_data_source__on_get_data_format,\n    NULL,   /* onGetCursor */\n    NULL,   /* onGetLength */\n    NULL,   /* onSetLooping */\n    0\n};\n\nstatic MA_INLINE ma_uint32 ma_pcm_rb_get_bpf(ma_pcm_rb* pRB)\n{\n    MA_ASSERT(pRB != NULL);\n\n    return ma_get_bytes_per_frame(pRB->format, pRB->channels);\n}\n\nMA_API ma_result ma_pcm_rb_init_ex(ma_format format, ma_uint32 channels, ma_uint32 subbufferSizeInFrames, ma_uint32 subbufferCount, ma_uint32 subbufferStrideInFrames, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_pcm_rb* pRB)\n{\n    ma_uint32 bpf;\n    ma_result result;\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pRB);\n\n    bpf = ma_get_bytes_per_frame(format, channels);\n    if (bpf == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_rb_init_ex(subbufferSizeInFrames*bpf, subbufferCount, subbufferStrideInFrames*bpf, pOptionalPreallocatedBuffer, pAllocationCallbacks, &pRB->rb);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pRB->format     = format;\n    pRB->channels   = channels;\n    pRB->sampleRate = 0;    /* The sample rate is not passed in as a parameter. */\n\n    /* The PCM ring buffer is a data source. We need to get that set up as well. */\n    {\n        ma_data_source_config dataSourceConfig = ma_data_source_config_init();\n        dataSourceConfig.vtable = &ma_gRBDataSourceVTable;\n\n        result = ma_data_source_init(&dataSourceConfig, &pRB->ds);\n        if (result != MA_SUCCESS) {\n            ma_rb_uninit(&pRB->rb);\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pcm_rb_init(ma_format format, ma_uint32 channels, ma_uint32 bufferSizeInFrames, void* pOptionalPreallocatedBuffer, const ma_allocation_callbacks* pAllocationCallbacks, ma_pcm_rb* pRB)\n{\n    return ma_pcm_rb_init_ex(format, channels, bufferSizeInFrames, 1, 0, pOptionalPreallocatedBuffer, pAllocationCallbacks, pRB);\n}\n\nMA_API void ma_pcm_rb_uninit(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return;\n    }\n\n    ma_data_source_uninit(&pRB->ds);\n    ma_rb_uninit(&pRB->rb);\n}\n\nMA_API void ma_pcm_rb_reset(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return;\n    }\n\n    ma_rb_reset(&pRB->rb);\n}\n\nMA_API ma_result ma_pcm_rb_acquire_read(ma_pcm_rb* pRB, ma_uint32* pSizeInFrames, void** ppBufferOut)\n{\n    size_t sizeInBytes;\n    ma_result result;\n\n    if (pRB == NULL || pSizeInFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    sizeInBytes = *pSizeInFrames * ma_pcm_rb_get_bpf(pRB);\n\n    result = ma_rb_acquire_read(&pRB->rb, &sizeInBytes, ppBufferOut);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pSizeInFrames = (ma_uint32)(sizeInBytes / (size_t)ma_pcm_rb_get_bpf(pRB));\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pcm_rb_commit_read(ma_pcm_rb* pRB, ma_uint32 sizeInFrames)\n{\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_rb_commit_read(&pRB->rb, sizeInFrames * ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_result ma_pcm_rb_acquire_write(ma_pcm_rb* pRB, ma_uint32* pSizeInFrames, void** ppBufferOut)\n{\n    size_t sizeInBytes;\n    ma_result result;\n\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    sizeInBytes = *pSizeInFrames * ma_pcm_rb_get_bpf(pRB);\n\n    result = ma_rb_acquire_write(&pRB->rb, &sizeInBytes, ppBufferOut);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pSizeInFrames = (ma_uint32)(sizeInBytes / ma_pcm_rb_get_bpf(pRB));\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pcm_rb_commit_write(ma_pcm_rb* pRB, ma_uint32 sizeInFrames)\n{\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_rb_commit_write(&pRB->rb, sizeInFrames * ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_result ma_pcm_rb_seek_read(ma_pcm_rb* pRB, ma_uint32 offsetInFrames)\n{\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_rb_seek_read(&pRB->rb, offsetInFrames * ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_result ma_pcm_rb_seek_write(ma_pcm_rb* pRB, ma_uint32 offsetInFrames)\n{\n    if (pRB == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_rb_seek_write(&pRB->rb, offsetInFrames * ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_int32 ma_pcm_rb_pointer_distance(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return ma_rb_pointer_distance(&pRB->rb) / ma_pcm_rb_get_bpf(pRB);\n}\n\nMA_API ma_uint32 ma_pcm_rb_available_read(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return ma_rb_available_read(&pRB->rb) / ma_pcm_rb_get_bpf(pRB);\n}\n\nMA_API ma_uint32 ma_pcm_rb_available_write(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return ma_rb_available_write(&pRB->rb) / ma_pcm_rb_get_bpf(pRB);\n}\n\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_size(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return (ma_uint32)(ma_rb_get_subbuffer_size(&pRB->rb) / ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_stride(ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return (ma_uint32)(ma_rb_get_subbuffer_stride(&pRB->rb) / ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API ma_uint32 ma_pcm_rb_get_subbuffer_offset(ma_pcm_rb* pRB, ma_uint32 subbufferIndex)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return (ma_uint32)(ma_rb_get_subbuffer_offset(&pRB->rb, subbufferIndex) / ma_pcm_rb_get_bpf(pRB));\n}\n\nMA_API void* ma_pcm_rb_get_subbuffer_ptr(ma_pcm_rb* pRB, ma_uint32 subbufferIndex, void* pBuffer)\n{\n    if (pRB == NULL) {\n        return NULL;\n    }\n\n    return ma_rb_get_subbuffer_ptr(&pRB->rb, subbufferIndex, pBuffer);\n}\n\nMA_API ma_format ma_pcm_rb_get_format(const ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return ma_format_unknown;\n    }\n\n    return pRB->format;\n}\n\nMA_API ma_uint32 ma_pcm_rb_get_channels(const ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return pRB->channels;\n}\n\nMA_API ma_uint32 ma_pcm_rb_get_sample_rate(const ma_pcm_rb* pRB)\n{\n    if (pRB == NULL) {\n        return 0;\n    }\n\n    return pRB->sampleRate;\n}\n\nMA_API void ma_pcm_rb_set_sample_rate(ma_pcm_rb* pRB, ma_uint32 sampleRate)\n{\n    if (pRB == NULL) {\n        return;\n    }\n\n    pRB->sampleRate = sampleRate;\n}\n\n\n\nMA_API ma_result ma_duplex_rb_init(ma_format captureFormat, ma_uint32 captureChannels, ma_uint32 sampleRate, ma_uint32 captureInternalSampleRate, ma_uint32 captureInternalPeriodSizeInFrames, const ma_allocation_callbacks* pAllocationCallbacks, ma_duplex_rb* pRB)\n{\n    ma_result result;\n    ma_uint32 sizeInFrames;\n\n    sizeInFrames = (ma_uint32)ma_calculate_frame_count_after_resampling(sampleRate, captureInternalSampleRate, captureInternalPeriodSizeInFrames * 5);\n    if (sizeInFrames == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_pcm_rb_init(captureFormat, captureChannels, sizeInFrames, NULL, pAllocationCallbacks, &pRB->rb);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Seek forward a bit so we have a bit of a buffer in case of desyncs. */\n    ma_pcm_rb_seek_write((ma_pcm_rb*)pRB, captureInternalPeriodSizeInFrames * 2);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_duplex_rb_uninit(ma_duplex_rb* pRB)\n{\n    ma_pcm_rb_uninit((ma_pcm_rb*)pRB);\n    return MA_SUCCESS;\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nMiscellaneous Helpers\n\n**************************************************************************************************************************************************************/\nMA_API const char* ma_result_description(ma_result result)\n{\n    switch (result)\n    {\n        case MA_SUCCESS:                       return \"No error\";\n        case MA_ERROR:                         return \"Unknown error\";\n        case MA_INVALID_ARGS:                  return \"Invalid argument\";\n        case MA_INVALID_OPERATION:             return \"Invalid operation\";\n        case MA_OUT_OF_MEMORY:                 return \"Out of memory\";\n        case MA_OUT_OF_RANGE:                  return \"Out of range\";\n        case MA_ACCESS_DENIED:                 return \"Permission denied\";\n        case MA_DOES_NOT_EXIST:                return \"Resource does not exist\";\n        case MA_ALREADY_EXISTS:                return \"Resource already exists\";\n        case MA_TOO_MANY_OPEN_FILES:           return \"Too many open files\";\n        case MA_INVALID_FILE:                  return \"Invalid file\";\n        case MA_TOO_BIG:                       return \"Too large\";\n        case MA_PATH_TOO_LONG:                 return \"Path too long\";\n        case MA_NAME_TOO_LONG:                 return \"Name too long\";\n        case MA_NOT_DIRECTORY:                 return \"Not a directory\";\n        case MA_IS_DIRECTORY:                  return \"Is a directory\";\n        case MA_DIRECTORY_NOT_EMPTY:           return \"Directory not empty\";\n        case MA_AT_END:                        return \"At end\";\n        case MA_NO_SPACE:                      return \"No space available\";\n        case MA_BUSY:                          return \"Device or resource busy\";\n        case MA_IO_ERROR:                      return \"Input/output error\";\n        case MA_INTERRUPT:                     return \"Interrupted\";\n        case MA_UNAVAILABLE:                   return \"Resource unavailable\";\n        case MA_ALREADY_IN_USE:                return \"Resource already in use\";\n        case MA_BAD_ADDRESS:                   return \"Bad address\";\n        case MA_BAD_SEEK:                      return \"Illegal seek\";\n        case MA_BAD_PIPE:                      return \"Broken pipe\";\n        case MA_DEADLOCK:                      return \"Deadlock\";\n        case MA_TOO_MANY_LINKS:                return \"Too many links\";\n        case MA_NOT_IMPLEMENTED:               return \"Not implemented\";\n        case MA_NO_MESSAGE:                    return \"No message of desired type\";\n        case MA_BAD_MESSAGE:                   return \"Invalid message\";\n        case MA_NO_DATA_AVAILABLE:             return \"No data available\";\n        case MA_INVALID_DATA:                  return \"Invalid data\";\n        case MA_TIMEOUT:                       return \"Timeout\";\n        case MA_NO_NETWORK:                    return \"Network unavailable\";\n        case MA_NOT_UNIQUE:                    return \"Not unique\";\n        case MA_NOT_SOCKET:                    return \"Socket operation on non-socket\";\n        case MA_NO_ADDRESS:                    return \"Destination address required\";\n        case MA_BAD_PROTOCOL:                  return \"Protocol wrong type for socket\";\n        case MA_PROTOCOL_UNAVAILABLE:          return \"Protocol not available\";\n        case MA_PROTOCOL_NOT_SUPPORTED:        return \"Protocol not supported\";\n        case MA_PROTOCOL_FAMILY_NOT_SUPPORTED: return \"Protocol family not supported\";\n        case MA_ADDRESS_FAMILY_NOT_SUPPORTED:  return \"Address family not supported\";\n        case MA_SOCKET_NOT_SUPPORTED:          return \"Socket type not supported\";\n        case MA_CONNECTION_RESET:              return \"Connection reset\";\n        case MA_ALREADY_CONNECTED:             return \"Already connected\";\n        case MA_NOT_CONNECTED:                 return \"Not connected\";\n        case MA_CONNECTION_REFUSED:            return \"Connection refused\";\n        case MA_NO_HOST:                       return \"No host\";\n        case MA_IN_PROGRESS:                   return \"Operation in progress\";\n        case MA_CANCELLED:                     return \"Operation cancelled\";\n        case MA_MEMORY_ALREADY_MAPPED:         return \"Memory already mapped\";\n\n        case MA_FORMAT_NOT_SUPPORTED:          return \"Format not supported\";\n        case MA_DEVICE_TYPE_NOT_SUPPORTED:     return \"Device type not supported\";\n        case MA_SHARE_MODE_NOT_SUPPORTED:      return \"Share mode not supported\";\n        case MA_NO_BACKEND:                    return \"No backend\";\n        case MA_NO_DEVICE:                     return \"No device\";\n        case MA_API_NOT_FOUND:                 return \"API not found\";\n        case MA_INVALID_DEVICE_CONFIG:         return \"Invalid device config\";\n\n        case MA_DEVICE_NOT_INITIALIZED:        return \"Device not initialized\";\n        case MA_DEVICE_NOT_STARTED:            return \"Device not started\";\n\n        case MA_FAILED_TO_INIT_BACKEND:        return \"Failed to initialize backend\";\n        case MA_FAILED_TO_OPEN_BACKEND_DEVICE: return \"Failed to open backend device\";\n        case MA_FAILED_TO_START_BACKEND_DEVICE: return \"Failed to start backend device\";\n        case MA_FAILED_TO_STOP_BACKEND_DEVICE: return \"Failed to stop backend device\";\n\n        default:                               return \"Unknown error\";\n    }\n}\n\nMA_API void* ma_malloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        if (pAllocationCallbacks->onMalloc != NULL) {\n            return pAllocationCallbacks->onMalloc(sz, pAllocationCallbacks->pUserData);\n        } else {\n            return NULL;    /* Do not fall back to the default implementation. */\n        }\n    } else {\n        return ma__malloc_default(sz, NULL);\n    }\n}\n\nMA_API void* ma_calloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    void* p = ma_malloc(sz, pAllocationCallbacks);\n    if (p != NULL) {\n        MA_ZERO_MEMORY(p, sz);\n    }\n\n    return p;\n}\n\nMA_API void* ma_realloc(void* p, size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        if (pAllocationCallbacks->onRealloc != NULL) {\n            return pAllocationCallbacks->onRealloc(p, sz, pAllocationCallbacks->pUserData);\n        } else {\n            return NULL;    /* Do not fall back to the default implementation. */\n        }\n    } else {\n        return ma__realloc_default(p, sz, NULL);\n    }\n}\n\nMA_API void ma_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (p == NULL) {\n        return;\n    }\n\n    if (pAllocationCallbacks != NULL) {\n        if (pAllocationCallbacks->onFree != NULL) {\n            pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n        } else {\n            return; /* Do no fall back to the default implementation. */\n        }\n    } else {\n        ma__free_default(p, NULL);\n    }\n}\n\nMA_API void* ma_aligned_malloc(size_t sz, size_t alignment, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    size_t extraBytes;\n    void* pUnaligned;\n    void* pAligned;\n\n    if (alignment == 0) {\n        return 0;\n    }\n\n    extraBytes = alignment-1 + sizeof(void*);\n\n    pUnaligned = ma_malloc(sz + extraBytes, pAllocationCallbacks);\n    if (pUnaligned == NULL) {\n        return NULL;\n    }\n\n    pAligned = (void*)(((ma_uintptr)pUnaligned + extraBytes) & ~((ma_uintptr)(alignment-1)));\n    ((void**)pAligned)[-1] = pUnaligned;\n\n    return pAligned;\n}\n\nMA_API void ma_aligned_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_free(((void**)p)[-1], pAllocationCallbacks);\n}\n\nMA_API const char* ma_get_format_name(ma_format format)\n{\n    switch (format)\n    {\n        case ma_format_unknown: return \"Unknown\";\n        case ma_format_u8:      return \"8-bit Unsigned Integer\";\n        case ma_format_s16:     return \"16-bit Signed Integer\";\n        case ma_format_s24:     return \"24-bit Signed Integer (Tightly Packed)\";\n        case ma_format_s32:     return \"32-bit Signed Integer\";\n        case ma_format_f32:     return \"32-bit IEEE Floating Point\";\n        default:                return \"Invalid\";\n    }\n}\n\nMA_API void ma_blend_f32(float* pOut, float* pInA, float* pInB, float factor, ma_uint32 channels)\n{\n    ma_uint32 i;\n    for (i = 0; i < channels; ++i) {\n        pOut[i] = ma_mix_f32(pInA[i], pInB[i], factor);\n    }\n}\n\n\nMA_API ma_uint32 ma_get_bytes_per_sample(ma_format format)\n{\n    ma_uint32 sizes[] = {\n        0,  /* unknown */\n        1,  /* u8 */\n        2,  /* s16 */\n        3,  /* s24 */\n        4,  /* s32 */\n        4,  /* f32 */\n    };\n    return sizes[format];\n}\n\n\n\n#define MA_DATA_SOURCE_DEFAULT_RANGE_BEG        0\n#define MA_DATA_SOURCE_DEFAULT_RANGE_END        ~((ma_uint64)0)\n#define MA_DATA_SOURCE_DEFAULT_LOOP_POINT_BEG   0\n#define MA_DATA_SOURCE_DEFAULT_LOOP_POINT_END   ~((ma_uint64)0)\n\nMA_API ma_data_source_config ma_data_source_config_init(void)\n{\n    ma_data_source_config config;\n\n    MA_ZERO_OBJECT(&config);\n\n    return config;\n}\n\n\nMA_API ma_result ma_data_source_init(const ma_data_source_config* pConfig, ma_data_source* pDataSource)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDataSourceBase);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->vtable == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDataSourceBase->vtable           = pConfig->vtable;\n    pDataSourceBase->rangeBegInFrames = MA_DATA_SOURCE_DEFAULT_RANGE_BEG;\n    pDataSourceBase->rangeEndInFrames = MA_DATA_SOURCE_DEFAULT_RANGE_END;\n    pDataSourceBase->loopBegInFrames  = MA_DATA_SOURCE_DEFAULT_LOOP_POINT_BEG;\n    pDataSourceBase->loopEndInFrames  = MA_DATA_SOURCE_DEFAULT_LOOP_POINT_END;\n    pDataSourceBase->pCurrent         = pDataSource;    /* Always read from ourself by default. */\n    pDataSourceBase->pNext            = NULL;\n    pDataSourceBase->onGetNext        = NULL;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_data_source_uninit(ma_data_source* pDataSource)\n{\n    if (pDataSource == NULL) {\n        return;\n    }\n\n    /*\n    This is placeholder in case we need this later. Data sources need to call this in their\n    uninitialization routine to ensure things work later on if something is added here.\n    */\n}\n\nstatic ma_result ma_data_source_resolve_current(ma_data_source* pDataSource, ma_data_source** ppCurrentDataSource)\n{\n    ma_data_source_base* pCurrentDataSource = (ma_data_source_base*)pDataSource;\n\n    MA_ASSERT(pDataSource         != NULL);\n    MA_ASSERT(ppCurrentDataSource != NULL);\n\n    if (pCurrentDataSource->pCurrent == NULL) {\n        /*\n        The current data source is NULL. If we're using this in the context of a chain we need to return NULL\n        here so that we don't end up looping. Otherwise we just return the data source itself.\n        */\n        if (pCurrentDataSource->pNext != NULL || pCurrentDataSource->onGetNext != NULL) {\n            pCurrentDataSource = NULL;\n        } else {\n            pCurrentDataSource = (ma_data_source_base*)pDataSource; /* Not being used in a chain. Make sure we just always read from the data source itself at all times. */\n        }\n    } else {\n        pCurrentDataSource = (ma_data_source_base*)pCurrentDataSource->pCurrent;\n    }\n\n    *ppCurrentDataSource = pCurrentDataSource;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_data_source_read_pcm_frames_from_backend(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    MA_ASSERT(pDataSourceBase                 != NULL);\n    MA_ASSERT(pDataSourceBase->vtable         != NULL);\n    MA_ASSERT(pDataSourceBase->vtable->onRead != NULL);\n    MA_ASSERT(pFramesRead != NULL);\n\n    if (pFramesOut != NULL) {\n        return pDataSourceBase->vtable->onRead(pDataSourceBase, pFramesOut, frameCount, pFramesRead);\n    } else {\n        /*\n        No output buffer. Probably seeking forward. Read and discard. Can probably optimize this in terms of\n        onSeek and onGetCursor, but need to keep in mind that the data source may not implement these functions.\n        */\n        ma_result result;\n        ma_uint64 framesRead;\n        ma_format format;\n        ma_uint32 channels;\n        ma_uint64 discardBufferCapInFrames;\n        ma_uint8  pDiscardBuffer[4096];\n\n        result = ma_data_source_get_data_format(pDataSource, &format, &channels, NULL, NULL, 0);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        discardBufferCapInFrames = sizeof(pDiscardBuffer) / ma_get_bytes_per_frame(format, channels);\n\n        framesRead = 0;\n        while (framesRead < frameCount) {\n            ma_uint64 framesReadThisIteration = 0;\n            ma_uint64 framesToRead = frameCount - framesRead;\n            if (framesToRead > discardBufferCapInFrames) {\n                framesToRead = discardBufferCapInFrames;\n            }\n\n            result = pDataSourceBase->vtable->onRead(pDataSourceBase, pDiscardBuffer, framesToRead, &framesReadThisIteration);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n\n            framesRead += framesReadThisIteration;\n        }\n\n        *pFramesRead = framesRead;\n\n        return MA_SUCCESS;\n    }\n}\n\nstatic ma_result ma_data_source_read_pcm_frames_within_range(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n    ma_result result;\n    ma_uint64 framesRead = 0;\n    ma_bool32 loop = ma_data_source_is_looping(pDataSource);\n\n    if (pDataSourceBase == NULL) {\n        return MA_AT_END;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    if ((pDataSourceBase->vtable->flags & MA_DATA_SOURCE_SELF_MANAGED_RANGE_AND_LOOP_POINT) != 0 || (pDataSourceBase->rangeEndInFrames == ~((ma_uint64)0) && (pDataSourceBase->loopEndInFrames == ~((ma_uint64)0) || loop == MA_FALSE))) {\n        /* Either the data source is self-managing the range, or no range is set - just read like normal. The data source itself will tell us when the end is reached. */\n        result = ma_data_source_read_pcm_frames_from_backend(pDataSource, pFramesOut, frameCount, &framesRead);\n    } else {\n        /* Need to clamp to within the range. */\n        ma_uint64 relativeCursor;\n        ma_uint64 absoluteCursor;\n\n        result = ma_data_source_get_cursor_in_pcm_frames(pDataSourceBase, &relativeCursor);\n        if (result != MA_SUCCESS) {\n            /* Failed to retrieve the cursor. Cannot read within a range or loop points. Just read like normal - this may happen for things like noise data sources where it doesn't really matter. */\n            result = ma_data_source_read_pcm_frames_from_backend(pDataSource, pFramesOut, frameCount, &framesRead);\n        } else {\n            ma_uint64 rangeBeg;\n            ma_uint64 rangeEnd;\n\n            /* We have the cursor. We need to make sure we don't read beyond our range. */\n            rangeBeg = pDataSourceBase->rangeBegInFrames;\n            rangeEnd = pDataSourceBase->rangeEndInFrames;\n\n            absoluteCursor = rangeBeg + relativeCursor;\n\n            /* If looping, make sure we're within range. */\n            if (loop) {\n                if (pDataSourceBase->loopEndInFrames != ~((ma_uint64)0)) {\n                    rangeEnd = ma_min(rangeEnd, pDataSourceBase->rangeBegInFrames + pDataSourceBase->loopEndInFrames);\n                }\n            }\n\n            if (frameCount > (rangeEnd - absoluteCursor) && rangeEnd != ~((ma_uint64)0)) {\n                frameCount = (rangeEnd - absoluteCursor);\n            }\n\n            /*\n            If the cursor is sitting on the end of the range the frame count will be set to 0 which can\n            result in MA_INVALID_ARGS. In this case, we don't want to try reading, but instead return\n            MA_AT_END so the higher level function can know about it.\n            */\n            if (frameCount > 0) {\n                result = ma_data_source_read_pcm_frames_from_backend(pDataSource, pFramesOut, frameCount, &framesRead);\n            } else {\n                result = MA_AT_END; /* The cursor is sitting on the end of the range which means we're at the end. */\n            }\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = framesRead;\n    }\n\n    /* We need to make sure MA_AT_END is returned if we hit the end of the range. */\n    if (result == MA_SUCCESS && framesRead == 0) {\n        result  = MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_data_source_read_pcm_frames(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n    ma_data_source_base* pCurrentDataSource;\n    void* pRunningFramesOut = pFramesOut;\n    ma_uint64 totalFramesProcessed = 0;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 emptyLoopCounter = 0; /* Keeps track of how many times 0 frames have been read. For infinite loop detection of sounds with no audio data. */\n    ma_bool32 loop;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDataSourceBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    loop = ma_data_source_is_looping(pDataSource);\n\n    /*\n    We need to know the data format so we can advance the output buffer as we read frames. If this\n    fails, chaining will not work and we'll just read as much as we can from the current source.\n    */\n    if (ma_data_source_get_data_format(pDataSource, &format, &channels, NULL, NULL, 0) != MA_SUCCESS) {\n        result = ma_data_source_resolve_current(pDataSource, (ma_data_source**)&pCurrentDataSource);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        return ma_data_source_read_pcm_frames_within_range(pCurrentDataSource, pFramesOut, frameCount, pFramesRead);\n    }\n\n    /*\n    Looping is a bit of a special case. When the `loop` argument is true, chaining will not work and\n    only the current data source will be read from.\n    */\n\n    /* Keep reading until we've read as many frames as possible. */\n    while (totalFramesProcessed < frameCount) {\n        ma_uint64 framesProcessed;\n        ma_uint64 framesRemaining = frameCount - totalFramesProcessed;\n\n        /* We need to resolve the data source that we'll actually be reading from. */\n        result = ma_data_source_resolve_current(pDataSource, (ma_data_source**)&pCurrentDataSource);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        if (pCurrentDataSource == NULL) {\n            break;\n        }\n\n        result = ma_data_source_read_pcm_frames_within_range(pCurrentDataSource, pRunningFramesOut, framesRemaining, &framesProcessed);\n        totalFramesProcessed += framesProcessed;\n\n        /*\n        If we encountered an error from the read callback, make sure it's propagated to the caller. The caller may need to know whether or not MA_BUSY is returned which is\n        not necessarily considered an error.\n        */\n        if (result != MA_SUCCESS && result != MA_AT_END) {\n            break;\n        }\n\n        /*\n        We can determine if we've reached the end by checking if ma_data_source_read_pcm_frames_within_range() returned\n        MA_AT_END. To loop back to the start, all we need to do is seek back to the first frame.\n        */\n        if (result == MA_AT_END) {\n            /*\n            The result needs to be reset back to MA_SUCCESS (from MA_AT_END) so that we don't\n            accidentally return MA_AT_END when data has been read in prior loop iterations. at the\n            end of this function, the result will be checked for MA_SUCCESS, and if the total\n            number of frames processed is 0, will be explicitly set to MA_AT_END.\n            */\n            result = MA_SUCCESS;\n\n            /*\n            We reached the end. If we're looping, we just loop back to the start of the current\n            data source. If we're not looping we need to check if we have another in the chain, and\n            if so, switch to it.\n            */\n            if (loop) {\n                if (framesProcessed == 0) {\n                    emptyLoopCounter += 1;\n                    if (emptyLoopCounter > 1) {\n                        break;  /* Infinite loop detected. Get out. */\n                    }\n                } else {\n                    emptyLoopCounter = 0;\n                }\n\n                result = ma_data_source_seek_to_pcm_frame(pCurrentDataSource, pCurrentDataSource->loopBegInFrames);\n                if (result != MA_SUCCESS) {\n                    break;  /* Failed to loop. Abort. */\n                }\n\n                /* Don't return MA_AT_END for looping sounds. */\n                result = MA_SUCCESS;\n            } else {\n                if (pCurrentDataSource->pNext != NULL) {\n                    pDataSourceBase->pCurrent = pCurrentDataSource->pNext;\n                } else if (pCurrentDataSource->onGetNext != NULL) {\n                    pDataSourceBase->pCurrent = pCurrentDataSource->onGetNext(pCurrentDataSource);\n                    if (pDataSourceBase->pCurrent == NULL) {\n                        break;  /* Our callback did not return a next data source. We're done. */\n                    }\n                } else {\n                    /* Reached the end of the chain. We're done. */\n                    break;\n                }\n\n                /* The next data source needs to be rewound to ensure data is read in looping scenarios. */\n                result = ma_data_source_seek_to_pcm_frame(pDataSourceBase->pCurrent, 0);\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n            }\n        }\n\n        if (pRunningFramesOut != NULL) {\n            pRunningFramesOut = ma_offset_ptr(pRunningFramesOut, framesProcessed * ma_get_bytes_per_frame(format, channels));\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesProcessed;\n    }\n\n    MA_ASSERT(!(result == MA_AT_END && totalFramesProcessed > 0));  /* We should never be returning MA_AT_END if we read some data. */\n\n    if (result == MA_SUCCESS && totalFramesProcessed == 0) {\n        result  = MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_data_source_seek_pcm_frames(ma_data_source* pDataSource, ma_uint64 frameCount, ma_uint64* pFramesSeeked)\n{\n    return ma_data_source_read_pcm_frames(pDataSource, NULL, frameCount, pFramesSeeked);\n}\n\nMA_API ma_result ma_data_source_seek_to_pcm_frame(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSourceBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDataSourceBase->vtable->onSeek == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    if (frameIndex > pDataSourceBase->rangeEndInFrames) {\n        return MA_INVALID_OPERATION;    /* Trying to seek too far forward. */\n    }\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    return pDataSourceBase->vtable->onSeek(pDataSource, pDataSourceBase->rangeBegInFrames + frameIndex);\n}\n\nMA_API ma_result ma_data_source_seek_seconds(ma_data_source* pDataSource, float secondCount, float* pSecondsSeeked)\n{\n    ma_uint64 frameCount;\n    ma_uint64 framesSeeked = 0;\n    ma_uint32 sampleRate;\n    ma_result result;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_data_source_get_data_format(pDataSource, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need PCM frames instead of seconds */\n    frameCount = (ma_uint64)(secondCount * sampleRate);\n\n    result = ma_data_source_seek_pcm_frames(pDataSource, frameCount, &framesSeeked);\n\n    /* VC6 doesn't support division between unsigned 64-bit integer and floating point number. Signed integer needed. This shouldn't affect anything in practice */\n    *pSecondsSeeked = (ma_int64)framesSeeked / (float)sampleRate;\n    return result;\n}\n\nMA_API ma_result ma_data_source_seek_to_second(ma_data_source* pDataSource, float seekPointInSeconds)\n{\n    ma_uint64 frameIndex;\n    ma_uint32 sampleRate;\n    ma_result result;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_data_source_get_data_format(pDataSource, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need PCM frames instead of seconds */\n    frameIndex = (ma_uint64)(seekPointInSeconds * sampleRate);\n\n    return ma_data_source_seek_to_pcm_frame(pDataSource, frameIndex);\n}\n\nMA_API ma_result ma_data_source_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n    ma_result result;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n\n    /* Initialize to defaults for safety just in case the data source does not implement this callback. */\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pDataSourceBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    if (pDataSourceBase->vtable->onGetDataFormat == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    result = pDataSourceBase->vtable->onGetDataFormat(pDataSource, &format, &channels, &sampleRate, pChannelMap, channelMapCap);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = format;\n    }\n    if (pChannels != NULL) {\n        *pChannels = channels;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = sampleRate;\n    }\n\n    /* Channel map was passed in directly to the callback. This is safe due to the channelMapCap parameter. */\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_source_get_cursor_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n    ma_result result;\n    ma_uint64 cursor;\n\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    if (pDataSourceBase == NULL) {\n        return MA_SUCCESS;\n    }\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    if (pDataSourceBase->vtable->onGetCursor == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    result = pDataSourceBase->vtable->onGetCursor(pDataSourceBase, &cursor);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* The cursor needs to be made relative to the start of the range. */\n    if (cursor < pDataSourceBase->rangeBegInFrames) {   /* Safety check so we don't return some huge number. */\n        *pCursor = 0;\n    } else {\n        *pCursor = cursor - pDataSourceBase->rangeBegInFrames;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_source_get_length_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    if (pDataSourceBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    /*\n    If we have a range defined we'll use that to determine the length. This is one of rare times\n    where we'll actually trust the caller. If they've set the range, I think it's mostly safe to\n    assume they've set it based on some higher level knowledge of the structure of the sound bank.\n    */\n    if (pDataSourceBase->rangeEndInFrames != ~((ma_uint64)0)) {\n        *pLength = pDataSourceBase->rangeEndInFrames - pDataSourceBase->rangeBegInFrames;\n        return MA_SUCCESS;\n    }\n\n    /*\n    Getting here means a range is not defined so we'll need to get the data source itself to tell\n    us the length.\n    */\n    if (pDataSourceBase->vtable->onGetLength == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pDataSourceBase->vtable->onGetLength(pDataSource, pLength);\n}\n\nMA_API ma_result ma_data_source_get_cursor_in_seconds(ma_data_source* pDataSource, float* pCursor)\n{\n    ma_result result;\n    ma_uint64 cursorInPCMFrames;\n    ma_uint32 sampleRate;\n\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    result = ma_data_source_get_cursor_in_pcm_frames(pDataSource, &cursorInPCMFrames);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_data_source_get_data_format(pDataSource, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* VC6 does not support division of unsigned 64-bit integers with floating point numbers. Need to use a signed number. This shouldn't effect anything in practice. */\n    *pCursor = (ma_int64)cursorInPCMFrames / (float)sampleRate;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_source_get_length_in_seconds(ma_data_source* pDataSource, float* pLength)\n{\n    ma_result result;\n    ma_uint64 lengthInPCMFrames;\n    ma_uint32 sampleRate;\n\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    result = ma_data_source_get_length_in_pcm_frames(pDataSource, &lengthInPCMFrames);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_data_source_get_data_format(pDataSource, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* VC6 does not support division of unsigned 64-bit integers with floating point numbers. Need to use a signed number. This shouldn't effect anything in practice. */\n    *pLength = (ma_int64)lengthInPCMFrames / (float)sampleRate;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_data_source_set_looping(ma_data_source* pDataSource, ma_bool32 isLooping)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_exchange_32(&pDataSourceBase->isLooping, isLooping);\n\n    MA_ASSERT(pDataSourceBase->vtable != NULL);\n\n    /* If there's no callback for this just treat it as a successful no-op. */\n    if (pDataSourceBase->vtable->onSetLooping == NULL) {\n        return MA_SUCCESS;\n    }\n\n    return pDataSourceBase->vtable->onSetLooping(pDataSource, isLooping);\n}\n\nMA_API ma_bool32 ma_data_source_is_looping(const ma_data_source* pDataSource)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_atomic_load_32(&pDataSourceBase->isLooping);\n}\n\nMA_API ma_result ma_data_source_set_range_in_pcm_frames(ma_data_source* pDataSource, ma_uint64 rangeBegInFrames, ma_uint64 rangeEndInFrames)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n    ma_result result;\n    ma_uint64 relativeCursor;\n    ma_uint64 absoluteCursor;\n    ma_bool32 doSeekAdjustment = MA_FALSE;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (rangeEndInFrames < rangeBegInFrames) {\n        return MA_INVALID_ARGS; /* The end of the range must come after the beginning. */\n    }\n\n    /*\n    We may need to adjust the position of the cursor to ensure it's clamped to the range. Grab it now\n    so we can calculate its absolute position before we change the range.\n    */\n    result = ma_data_source_get_cursor_in_pcm_frames(pDataSource, &relativeCursor);\n    if (result == MA_SUCCESS) {\n        doSeekAdjustment = MA_TRUE;\n        absoluteCursor = relativeCursor + pDataSourceBase->rangeBegInFrames;\n    } else {\n        /*\n        We couldn't get the position of the cursor. It probably means the data source has no notion\n        of a cursor. We'll just leave it at position 0. Don't treat this as an error.\n        */\n        doSeekAdjustment = MA_FALSE;\n        relativeCursor = 0;\n        absoluteCursor = 0;\n    }\n\n    pDataSourceBase->rangeBegInFrames = rangeBegInFrames;\n    pDataSourceBase->rangeEndInFrames = rangeEndInFrames;\n\n    /*\n    The commented out logic below was intended to maintain loop points in response to a change in the\n    range. However, this is not useful because it results in the sound breaking when you move the range\n    outside of the old loop points. I'm simplifying this by simply resetting the loop points. The\n    caller is expected to update their loop points if they change the range.\n\n    In practice this should be mostly a non-issue because the majority of the time the range will be\n    set once right after initialization.\n    */\n    pDataSourceBase->loopBegInFrames = 0;\n    pDataSourceBase->loopEndInFrames = ~((ma_uint64)0);\n\n\n    /*\n    Seek to within range. Note that our seek positions here are relative to the new range. We don't want\n    to do this if we failed to retrieve the cursor earlier on because it probably means the data source\n    has no notion of a cursor. In practice the seek would probably fail (which we silently ignore), but\n    I'm just not even going to attempt it.\n    */\n    if (doSeekAdjustment) {\n        if (absoluteCursor < rangeBegInFrames) {\n            ma_data_source_seek_to_pcm_frame(pDataSource, 0);\n        } else if (absoluteCursor > rangeEndInFrames) {\n            ma_data_source_seek_to_pcm_frame(pDataSource, rangeEndInFrames - rangeBegInFrames);\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_data_source_get_range_in_pcm_frames(const ma_data_source* pDataSource, ma_uint64* pRangeBegInFrames, ma_uint64* pRangeEndInFrames)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pRangeBegInFrames != NULL) {\n        *pRangeBegInFrames = 0;\n    }\n    if (pRangeEndInFrames != NULL) {\n        *pRangeEndInFrames = 0;\n    }\n\n    if (pDataSource == NULL) {\n        return;\n    }\n\n    if (pRangeBegInFrames != NULL) {\n        *pRangeBegInFrames = pDataSourceBase->rangeBegInFrames;\n    }\n\n    if (pRangeEndInFrames != NULL) {\n        *pRangeEndInFrames = pDataSourceBase->rangeEndInFrames;\n    }\n}\n\nMA_API ma_result ma_data_source_set_loop_point_in_pcm_frames(ma_data_source* pDataSource, ma_uint64 loopBegInFrames, ma_uint64 loopEndInFrames)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (loopEndInFrames < loopBegInFrames) {\n        return MA_INVALID_ARGS; /* The end of the loop point must come after the beginning. */\n    }\n\n    if (loopEndInFrames > pDataSourceBase->rangeEndInFrames && loopEndInFrames != ~((ma_uint64)0)) {\n        return MA_INVALID_ARGS; /* The end of the loop point must not go beyond the range. */\n    }\n\n    pDataSourceBase->loopBegInFrames = loopBegInFrames;\n    pDataSourceBase->loopEndInFrames = loopEndInFrames;\n\n    /* The end cannot exceed the range. */\n    if (pDataSourceBase->loopEndInFrames > (pDataSourceBase->rangeEndInFrames - pDataSourceBase->rangeBegInFrames) && pDataSourceBase->loopEndInFrames != ~((ma_uint64)0)) {\n        pDataSourceBase->loopEndInFrames = (pDataSourceBase->rangeEndInFrames - pDataSourceBase->rangeBegInFrames);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_data_source_get_loop_point_in_pcm_frames(const ma_data_source* pDataSource, ma_uint64* pLoopBegInFrames, ma_uint64* pLoopEndInFrames)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pLoopBegInFrames != NULL) {\n        *pLoopBegInFrames = 0;\n    }\n    if (pLoopEndInFrames != NULL) {\n        *pLoopEndInFrames = 0;\n    }\n\n    if (pDataSource == NULL) {\n        return;\n    }\n\n    if (pLoopBegInFrames != NULL) {\n        *pLoopBegInFrames = pDataSourceBase->loopBegInFrames;\n    }\n\n    if (pLoopEndInFrames != NULL) {\n        *pLoopEndInFrames = pDataSourceBase->loopEndInFrames;\n    }\n}\n\nMA_API ma_result ma_data_source_set_current(ma_data_source* pDataSource, ma_data_source* pCurrentDataSource)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDataSourceBase->pCurrent = pCurrentDataSource;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_data_source* ma_data_source_get_current(const ma_data_source* pDataSource)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return NULL;\n    }\n\n    return pDataSourceBase->pCurrent;\n}\n\nMA_API ma_result ma_data_source_set_next(ma_data_source* pDataSource, ma_data_source* pNextDataSource)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDataSourceBase->pNext = pNextDataSource;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_data_source* ma_data_source_get_next(const ma_data_source* pDataSource)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return NULL;\n    }\n\n    return pDataSourceBase->pNext;\n}\n\nMA_API ma_result ma_data_source_set_next_callback(ma_data_source* pDataSource, ma_data_source_get_next_proc onGetNext)\n{\n    ma_data_source_base* pDataSourceBase = (ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDataSourceBase->onGetNext = onGetNext;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_data_source_get_next_proc ma_data_source_get_next_callback(const ma_data_source* pDataSource)\n{\n    const ma_data_source_base* pDataSourceBase = (const ma_data_source_base*)pDataSource;\n\n    if (pDataSource == NULL) {\n        return NULL;\n    }\n\n    return pDataSourceBase->onGetNext;\n}\n\n\nstatic ma_result ma_audio_buffer_ref__data_source_on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_audio_buffer_ref* pAudioBufferRef = (ma_audio_buffer_ref*)pDataSource;\n    ma_uint64 framesRead = ma_audio_buffer_ref_read_pcm_frames(pAudioBufferRef, pFramesOut, frameCount, MA_FALSE);\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = framesRead;\n    }\n\n    if (framesRead < frameCount || framesRead == 0) {\n        return MA_AT_END;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_audio_buffer_ref__data_source_on_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_audio_buffer_ref_seek_to_pcm_frame((ma_audio_buffer_ref*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_audio_buffer_ref__data_source_on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_audio_buffer_ref* pAudioBufferRef = (ma_audio_buffer_ref*)pDataSource;\n\n    *pFormat     = pAudioBufferRef->format;\n    *pChannels   = pAudioBufferRef->channels;\n    *pSampleRate = pAudioBufferRef->sampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pAudioBufferRef->channels);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_audio_buffer_ref__data_source_on_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    ma_audio_buffer_ref* pAudioBufferRef = (ma_audio_buffer_ref*)pDataSource;\n\n    *pCursor = pAudioBufferRef->cursor;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_audio_buffer_ref__data_source_on_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    ma_audio_buffer_ref* pAudioBufferRef = (ma_audio_buffer_ref*)pDataSource;\n\n    *pLength = pAudioBufferRef->sizeInFrames;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_data_source_vtable g_ma_audio_buffer_ref_data_source_vtable =\n{\n    ma_audio_buffer_ref__data_source_on_read,\n    ma_audio_buffer_ref__data_source_on_seek,\n    ma_audio_buffer_ref__data_source_on_get_data_format,\n    ma_audio_buffer_ref__data_source_on_get_cursor,\n    ma_audio_buffer_ref__data_source_on_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\nMA_API ma_result ma_audio_buffer_ref_init(ma_format format, ma_uint32 channels, const void* pData, ma_uint64 sizeInFrames, ma_audio_buffer_ref* pAudioBufferRef)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pAudioBufferRef);\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_audio_buffer_ref_data_source_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pAudioBufferRef->ds);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pAudioBufferRef->format       = format;\n    pAudioBufferRef->channels     = channels;\n    pAudioBufferRef->sampleRate   = 0;  /* TODO: Version 0.12. Set this to sampleRate. */\n    pAudioBufferRef->cursor       = 0;\n    pAudioBufferRef->sizeInFrames = sizeInFrames;\n    pAudioBufferRef->pData        = pData;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_audio_buffer_ref_uninit(ma_audio_buffer_ref* pAudioBufferRef)\n{\n    if (pAudioBufferRef == NULL) {\n        return;\n    }\n\n    ma_data_source_uninit(&pAudioBufferRef->ds);\n}\n\nMA_API ma_result ma_audio_buffer_ref_set_data(ma_audio_buffer_ref* pAudioBufferRef, const void* pData, ma_uint64 sizeInFrames)\n{\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pAudioBufferRef->cursor       = 0;\n    pAudioBufferRef->sizeInFrames = sizeInFrames;\n    pAudioBufferRef->pData        = pData;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint64 ma_audio_buffer_ref_read_pcm_frames(ma_audio_buffer_ref* pAudioBufferRef, void* pFramesOut, ma_uint64 frameCount, ma_bool32 loop)\n{\n    ma_uint64 totalFramesRead = 0;\n\n    if (pAudioBufferRef == NULL) {\n        return 0;\n    }\n\n    if (frameCount == 0) {\n        return 0;\n    }\n\n    while (totalFramesRead < frameCount) {\n        ma_uint64 framesAvailable = pAudioBufferRef->sizeInFrames - pAudioBufferRef->cursor;\n        ma_uint64 framesRemaining = frameCount - totalFramesRead;\n        ma_uint64 framesToRead;\n\n        framesToRead = framesRemaining;\n        if (framesToRead > framesAvailable) {\n            framesToRead = framesAvailable;\n        }\n\n        if (pFramesOut != NULL) {\n            ma_copy_pcm_frames(ma_offset_ptr(pFramesOut, totalFramesRead * ma_get_bytes_per_frame(pAudioBufferRef->format, pAudioBufferRef->channels)), ma_offset_ptr(pAudioBufferRef->pData, pAudioBufferRef->cursor * ma_get_bytes_per_frame(pAudioBufferRef->format, pAudioBufferRef->channels)), framesToRead, pAudioBufferRef->format, pAudioBufferRef->channels);\n        }\n\n        totalFramesRead += framesToRead;\n\n        pAudioBufferRef->cursor += framesToRead;\n        if (pAudioBufferRef->cursor == pAudioBufferRef->sizeInFrames) {\n            if (loop) {\n                pAudioBufferRef->cursor = 0;\n            } else {\n                break;  /* We've reached the end and we're not looping. Done. */\n            }\n        }\n\n        MA_ASSERT(pAudioBufferRef->cursor < pAudioBufferRef->sizeInFrames);\n    }\n\n    return totalFramesRead;\n}\n\nMA_API ma_result ma_audio_buffer_ref_seek_to_pcm_frame(ma_audio_buffer_ref* pAudioBufferRef, ma_uint64 frameIndex)\n{\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (frameIndex > pAudioBufferRef->sizeInFrames) {\n        return MA_INVALID_ARGS;\n    }\n\n    pAudioBufferRef->cursor = (size_t)frameIndex;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_audio_buffer_ref_map(ma_audio_buffer_ref* pAudioBufferRef, void** ppFramesOut, ma_uint64* pFrameCount)\n{\n    ma_uint64 framesAvailable;\n    ma_uint64 frameCount = 0;\n\n    if (ppFramesOut != NULL) {\n        *ppFramesOut = NULL;    /* Safety. */\n    }\n\n    if (pFrameCount != NULL) {\n        frameCount = *pFrameCount;\n        *pFrameCount = 0;       /* Safety. */\n    }\n\n    if (pAudioBufferRef == NULL || ppFramesOut == NULL || pFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    framesAvailable = pAudioBufferRef->sizeInFrames - pAudioBufferRef->cursor;\n    if (frameCount > framesAvailable) {\n        frameCount = framesAvailable;\n    }\n\n    *ppFramesOut = ma_offset_ptr(pAudioBufferRef->pData, pAudioBufferRef->cursor * ma_get_bytes_per_frame(pAudioBufferRef->format, pAudioBufferRef->channels));\n    *pFrameCount = frameCount;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_audio_buffer_ref_unmap(ma_audio_buffer_ref* pAudioBufferRef, ma_uint64 frameCount)\n{\n    ma_uint64 framesAvailable;\n\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    framesAvailable = pAudioBufferRef->sizeInFrames - pAudioBufferRef->cursor;\n    if (frameCount > framesAvailable) {\n        return MA_INVALID_ARGS;   /* The frame count was too big. This should never happen in an unmapping. Need to make sure the caller is aware of this. */\n    }\n\n    pAudioBufferRef->cursor += frameCount;\n\n    if (pAudioBufferRef->cursor == pAudioBufferRef->sizeInFrames) {\n        return MA_AT_END;   /* Successful. Need to tell the caller that the end has been reached so that it can loop if desired. */\n    } else {\n        return MA_SUCCESS;\n    }\n}\n\nMA_API ma_bool32 ma_audio_buffer_ref_at_end(const ma_audio_buffer_ref* pAudioBufferRef)\n{\n    if (pAudioBufferRef == NULL) {\n        return MA_FALSE;\n    }\n\n    return pAudioBufferRef->cursor == pAudioBufferRef->sizeInFrames;\n}\n\nMA_API ma_result ma_audio_buffer_ref_get_cursor_in_pcm_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = pAudioBufferRef->cursor;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_audio_buffer_ref_get_length_in_pcm_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = pAudioBufferRef->sizeInFrames;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_audio_buffer_ref_get_available_frames(const ma_audio_buffer_ref* pAudioBufferRef, ma_uint64* pAvailableFrames)\n{\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pAudioBufferRef == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pAudioBufferRef->sizeInFrames <= pAudioBufferRef->cursor) {\n        *pAvailableFrames = 0;\n    } else {\n        *pAvailableFrames = pAudioBufferRef->sizeInFrames - pAudioBufferRef->cursor;\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\n\nMA_API ma_audio_buffer_config ma_audio_buffer_config_init(ma_format format, ma_uint32 channels, ma_uint64 sizeInFrames, const void* pData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_audio_buffer_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format       = format;\n    config.channels     = channels;\n    config.sampleRate   = 0;    /* TODO: Version 0.12. Set this to sampleRate. */\n    config.sizeInFrames = sizeInFrames;\n    config.pData        = pData;\n    ma_allocation_callbacks_init_copy(&config.allocationCallbacks, pAllocationCallbacks);\n\n    return config;\n}\n\nstatic ma_result ma_audio_buffer_init_ex(const ma_audio_buffer_config* pConfig, ma_bool32 doCopy, ma_audio_buffer* pAudioBuffer)\n{\n    ma_result result;\n\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_MEMORY(pAudioBuffer, sizeof(*pAudioBuffer) - sizeof(pAudioBuffer->_pExtraData));   /* Safety. Don't overwrite the extra data. */\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->sizeInFrames == 0) {\n        return MA_INVALID_ARGS; /* Not allowing buffer sizes of 0 frames. */\n    }\n\n    result = ma_audio_buffer_ref_init(pConfig->format, pConfig->channels, NULL, 0, &pAudioBuffer->ref);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* TODO: Version 0.12. Set this in ma_audio_buffer_ref_init() instead of here. */\n    pAudioBuffer->ref.sampleRate = pConfig->sampleRate;\n\n    ma_allocation_callbacks_init_copy(&pAudioBuffer->allocationCallbacks, &pConfig->allocationCallbacks);\n\n    if (doCopy) {\n        ma_uint64 allocationSizeInBytes;\n        void* pData;\n\n        allocationSizeInBytes = pConfig->sizeInFrames * ma_get_bytes_per_frame(pConfig->format, pConfig->channels);\n        if (allocationSizeInBytes > MA_SIZE_MAX) {\n            return MA_OUT_OF_MEMORY;    /* Too big. */\n        }\n\n        pData = ma_malloc((size_t)allocationSizeInBytes, &pAudioBuffer->allocationCallbacks);   /* Safe cast to size_t. */\n        if (pData == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        if (pConfig->pData != NULL) {\n            ma_copy_pcm_frames(pData, pConfig->pData, pConfig->sizeInFrames, pConfig->format, pConfig->channels);\n        } else {\n            ma_silence_pcm_frames(pData, pConfig->sizeInFrames, pConfig->format, pConfig->channels);\n        }\n\n        ma_audio_buffer_ref_set_data(&pAudioBuffer->ref, pData, pConfig->sizeInFrames);\n        pAudioBuffer->ownsData = MA_TRUE;\n    } else {\n        ma_audio_buffer_ref_set_data(&pAudioBuffer->ref, pConfig->pData, pConfig->sizeInFrames);\n        pAudioBuffer->ownsData = MA_FALSE;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_audio_buffer_uninit_ex(ma_audio_buffer* pAudioBuffer, ma_bool32 doFree)\n{\n    if (pAudioBuffer == NULL) {\n        return;\n    }\n\n    if (pAudioBuffer->ownsData && pAudioBuffer->ref.pData != &pAudioBuffer->_pExtraData[0]) {\n        ma_free((void*)pAudioBuffer->ref.pData, &pAudioBuffer->allocationCallbacks);    /* Naugty const cast, but OK in this case since we've guarded it with the ownsData check. */\n    }\n\n    if (doFree) {\n        ma_free(pAudioBuffer, &pAudioBuffer->allocationCallbacks);\n    }\n\n    ma_audio_buffer_ref_uninit(&pAudioBuffer->ref);\n}\n\nMA_API ma_result ma_audio_buffer_init(const ma_audio_buffer_config* pConfig, ma_audio_buffer* pAudioBuffer)\n{\n    return ma_audio_buffer_init_ex(pConfig, MA_FALSE, pAudioBuffer);\n}\n\nMA_API ma_result ma_audio_buffer_init_copy(const ma_audio_buffer_config* pConfig, ma_audio_buffer* pAudioBuffer)\n{\n    return ma_audio_buffer_init_ex(pConfig, MA_TRUE, pAudioBuffer);\n}\n\nMA_API ma_result ma_audio_buffer_alloc_and_init(const ma_audio_buffer_config* pConfig, ma_audio_buffer** ppAudioBuffer)\n{\n    ma_result result;\n    ma_audio_buffer* pAudioBuffer;\n    ma_audio_buffer_config innerConfig; /* We'll be making some changes to the config, so need to make a copy. */\n    ma_uint64 allocationSizeInBytes;\n\n    if (ppAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *ppAudioBuffer = NULL;  /* Safety. */\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    innerConfig = *pConfig;\n    ma_allocation_callbacks_init_copy(&innerConfig.allocationCallbacks, &pConfig->allocationCallbacks);\n\n    allocationSizeInBytes = sizeof(*pAudioBuffer) - sizeof(pAudioBuffer->_pExtraData) + (pConfig->sizeInFrames * ma_get_bytes_per_frame(pConfig->format, pConfig->channels));\n    if (allocationSizeInBytes > MA_SIZE_MAX) {\n        return MA_OUT_OF_MEMORY;    /* Too big. */\n    }\n\n    pAudioBuffer = (ma_audio_buffer*)ma_malloc((size_t)allocationSizeInBytes, &innerConfig.allocationCallbacks);  /* Safe cast to size_t. */\n    if (pAudioBuffer == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    if (pConfig->pData != NULL) {\n        ma_copy_pcm_frames(&pAudioBuffer->_pExtraData[0], pConfig->pData, pConfig->sizeInFrames, pConfig->format, pConfig->channels);\n    } else {\n        ma_silence_pcm_frames(&pAudioBuffer->_pExtraData[0], pConfig->sizeInFrames, pConfig->format, pConfig->channels);\n    }\n\n    innerConfig.pData = &pAudioBuffer->_pExtraData[0];\n\n    result = ma_audio_buffer_init_ex(&innerConfig, MA_FALSE, pAudioBuffer);\n    if (result != MA_SUCCESS) {\n        ma_free(pAudioBuffer, &innerConfig.allocationCallbacks);\n        return result;\n    }\n\n    *ppAudioBuffer = pAudioBuffer;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_audio_buffer_uninit(ma_audio_buffer* pAudioBuffer)\n{\n    ma_audio_buffer_uninit_ex(pAudioBuffer, MA_FALSE);\n}\n\nMA_API void ma_audio_buffer_uninit_and_free(ma_audio_buffer* pAudioBuffer)\n{\n    ma_audio_buffer_uninit_ex(pAudioBuffer, MA_TRUE);\n}\n\nMA_API ma_uint64 ma_audio_buffer_read_pcm_frames(ma_audio_buffer* pAudioBuffer, void* pFramesOut, ma_uint64 frameCount, ma_bool32 loop)\n{\n    if (pAudioBuffer == NULL) {\n        return 0;\n    }\n\n    return ma_audio_buffer_ref_read_pcm_frames(&pAudioBuffer->ref, pFramesOut, frameCount, loop);\n}\n\nMA_API ma_result ma_audio_buffer_seek_to_pcm_frame(ma_audio_buffer* pAudioBuffer, ma_uint64 frameIndex)\n{\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_seek_to_pcm_frame(&pAudioBuffer->ref, frameIndex);\n}\n\nMA_API ma_result ma_audio_buffer_map(ma_audio_buffer* pAudioBuffer, void** ppFramesOut, ma_uint64* pFrameCount)\n{\n    if (ppFramesOut != NULL) {\n        *ppFramesOut = NULL;    /* Safety. */\n    }\n\n    if (pAudioBuffer == NULL) {\n        if (pFrameCount != NULL) {\n            *pFrameCount = 0;\n        }\n\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_map(&pAudioBuffer->ref, ppFramesOut, pFrameCount);\n}\n\nMA_API ma_result ma_audio_buffer_unmap(ma_audio_buffer* pAudioBuffer, ma_uint64 frameCount)\n{\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_unmap(&pAudioBuffer->ref, frameCount);\n}\n\nMA_API ma_bool32 ma_audio_buffer_at_end(const ma_audio_buffer* pAudioBuffer)\n{\n    if (pAudioBuffer == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_audio_buffer_ref_at_end(&pAudioBuffer->ref);\n}\n\nMA_API ma_result ma_audio_buffer_get_cursor_in_pcm_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pCursor)\n{\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_get_cursor_in_pcm_frames(&pAudioBuffer->ref, pCursor);\n}\n\nMA_API ma_result ma_audio_buffer_get_length_in_pcm_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pLength)\n{\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_get_length_in_pcm_frames(&pAudioBuffer->ref, pLength);\n}\n\nMA_API ma_result ma_audio_buffer_get_available_frames(const ma_audio_buffer* pAudioBuffer, ma_uint64* pAvailableFrames)\n{\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_audio_buffer_ref_get_available_frames(&pAudioBuffer->ref, pAvailableFrames);\n}\n\n\n\n\n\nMA_API ma_result ma_paged_audio_buffer_data_init(ma_format format, ma_uint32 channels, ma_paged_audio_buffer_data* pData)\n{\n    if (pData == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pData);\n\n    pData->format   = format;\n    pData->channels = channels;\n    pData->pTail    = &pData->head;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_paged_audio_buffer_data_uninit(ma_paged_audio_buffer_data* pData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_paged_audio_buffer_page* pPage;\n\n    if (pData == NULL) {\n        return;\n    }\n\n    /* All pages need to be freed. */\n    pPage = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pData->head.pNext);\n    while (pPage != NULL) {\n        ma_paged_audio_buffer_page* pNext = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pPage->pNext);\n\n        ma_free(pPage, pAllocationCallbacks);\n        pPage = pNext;\n    }\n}\n\nMA_API ma_paged_audio_buffer_page* ma_paged_audio_buffer_data_get_head(ma_paged_audio_buffer_data* pData)\n{\n    if (pData == NULL) {\n        return NULL;\n    }\n\n    return &pData->head;\n}\n\nMA_API ma_paged_audio_buffer_page* ma_paged_audio_buffer_data_get_tail(ma_paged_audio_buffer_data* pData)\n{\n    if (pData == NULL) {\n        return NULL;\n    }\n\n    return pData->pTail;\n}\n\nMA_API ma_result ma_paged_audio_buffer_data_get_length_in_pcm_frames(ma_paged_audio_buffer_data* pData, ma_uint64* pLength)\n{\n    ma_paged_audio_buffer_page* pPage;\n\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    if (pData == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Calculate the length from the linked list. */\n    for (pPage = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pData->head.pNext); pPage != NULL; pPage = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pPage->pNext)) {\n        *pLength += pPage->sizeInFrames;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_data_allocate_page(ma_paged_audio_buffer_data* pData, ma_uint64 pageSizeInFrames, const void* pInitialData, const ma_allocation_callbacks* pAllocationCallbacks, ma_paged_audio_buffer_page** ppPage)\n{\n    ma_paged_audio_buffer_page* pPage;\n    ma_uint64 allocationSize;\n\n    if (ppPage == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *ppPage = NULL;\n\n    if (pData == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    allocationSize = sizeof(*pPage) + (pageSizeInFrames * ma_get_bytes_per_frame(pData->format, pData->channels));\n    if (allocationSize > MA_SIZE_MAX) {\n        return MA_OUT_OF_MEMORY;    /* Too big. */\n    }\n\n    pPage = (ma_paged_audio_buffer_page*)ma_malloc((size_t)allocationSize, pAllocationCallbacks);   /* Safe cast to size_t. */\n    if (pPage == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    pPage->pNext = NULL;\n    pPage->sizeInFrames = pageSizeInFrames;\n\n    if (pInitialData != NULL) {\n        ma_copy_pcm_frames(pPage->pAudioData, pInitialData, pageSizeInFrames, pData->format, pData->channels);\n    }\n\n    *ppPage = pPage;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_data_free_page(ma_paged_audio_buffer_data* pData, ma_paged_audio_buffer_page* pPage, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pData == NULL || pPage == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* It's assumed the page is not attached to the list. */\n    ma_free(pPage, pAllocationCallbacks);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_data_append_page(ma_paged_audio_buffer_data* pData, ma_paged_audio_buffer_page* pPage)\n{\n    if (pData == NULL || pPage == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* This function assumes the page has been filled with audio data by this point. As soon as we append, the page will be available for reading. */\n\n    /* First thing to do is update the tail. */\n    for (;;) {\n        ma_paged_audio_buffer_page* pOldTail = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pData->pTail);\n        ma_paged_audio_buffer_page* pNewTail = pPage;\n\n        if (ma_atomic_compare_exchange_weak_ptr((volatile void**)&pData->pTail, (void**)&pOldTail, pNewTail)) {\n            /* Here is where we append the page to the list. After this, the page is attached to the list and ready to be read from. */\n            ma_atomic_exchange_ptr(&pOldTail->pNext, pPage);\n            break;  /* Done. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_data_allocate_and_append_page(ma_paged_audio_buffer_data* pData, ma_uint32 pageSizeInFrames, const void* pInitialData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_result result;\n    ma_paged_audio_buffer_page* pPage;\n\n    result = ma_paged_audio_buffer_data_allocate_page(pData, pageSizeInFrames, pInitialData, pAllocationCallbacks, &pPage);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_paged_audio_buffer_data_append_page(pData, pPage);    /* <-- Should never fail. */\n}\n\n\nMA_API ma_paged_audio_buffer_config ma_paged_audio_buffer_config_init(ma_paged_audio_buffer_data* pData)\n{\n    ma_paged_audio_buffer_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.pData = pData;\n\n    return config;\n}\n\n\nstatic ma_result ma_paged_audio_buffer__data_source_on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_paged_audio_buffer_read_pcm_frames((ma_paged_audio_buffer*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_paged_audio_buffer__data_source_on_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_paged_audio_buffer_seek_to_pcm_frame((ma_paged_audio_buffer*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_paged_audio_buffer__data_source_on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_paged_audio_buffer* pPagedAudioBuffer = (ma_paged_audio_buffer*)pDataSource;\n\n    *pFormat     = pPagedAudioBuffer->pData->format;\n    *pChannels   = pPagedAudioBuffer->pData->channels;\n    *pSampleRate = 0;   /* There is no notion of a sample rate with audio buffers. */\n    ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pPagedAudioBuffer->pData->channels);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_paged_audio_buffer__data_source_on_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_paged_audio_buffer_get_cursor_in_pcm_frames((ma_paged_audio_buffer*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_paged_audio_buffer__data_source_on_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_paged_audio_buffer_get_length_in_pcm_frames((ma_paged_audio_buffer*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_paged_audio_buffer_data_source_vtable =\n{\n    ma_paged_audio_buffer__data_source_on_read,\n    ma_paged_audio_buffer__data_source_on_seek,\n    ma_paged_audio_buffer__data_source_on_get_data_format,\n    ma_paged_audio_buffer__data_source_on_get_cursor,\n    ma_paged_audio_buffer__data_source_on_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\nMA_API ma_result ma_paged_audio_buffer_init(const ma_paged_audio_buffer_config* pConfig, ma_paged_audio_buffer* pPagedAudioBuffer)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pPagedAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pPagedAudioBuffer);\n\n    /* A config is required for the format and channel count. */\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->pData == NULL) {\n        return MA_INVALID_ARGS; /* No underlying data specified. */\n    }\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_paged_audio_buffer_data_source_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pPagedAudioBuffer->ds);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pPagedAudioBuffer->pData          = pConfig->pData;\n    pPagedAudioBuffer->pCurrent       = ma_paged_audio_buffer_data_get_head(pConfig->pData);\n    pPagedAudioBuffer->relativeCursor = 0;\n    pPagedAudioBuffer->absoluteCursor = 0;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_paged_audio_buffer_uninit(ma_paged_audio_buffer* pPagedAudioBuffer)\n{\n    if (pPagedAudioBuffer == NULL) {\n        return;\n    }\n\n    /* Nothing to do. The data needs to be deleted separately. */\n}\n\nMA_API ma_result ma_paged_audio_buffer_read_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 totalFramesRead = 0;\n    ma_format format;\n    ma_uint32 channels;\n\n    if (pPagedAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    format   = pPagedAudioBuffer->pData->format;\n    channels = pPagedAudioBuffer->pData->channels;\n\n    while (totalFramesRead < frameCount) {\n        /* Read from the current page. The buffer should never be in a state where this is NULL. */\n        ma_uint64 framesRemainingInCurrentPage;\n        ma_uint64 framesRemainingToRead = frameCount - totalFramesRead;\n        ma_uint64 framesToReadThisIteration;\n\n        MA_ASSERT(pPagedAudioBuffer->pCurrent != NULL);\n\n        framesRemainingInCurrentPage = pPagedAudioBuffer->pCurrent->sizeInFrames - pPagedAudioBuffer->relativeCursor;\n\n        framesToReadThisIteration = ma_min(framesRemainingInCurrentPage, framesRemainingToRead);\n        ma_copy_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, format, channels), ma_offset_pcm_frames_ptr(pPagedAudioBuffer->pCurrent->pAudioData, pPagedAudioBuffer->relativeCursor, format, channels), framesToReadThisIteration, format, channels);\n        totalFramesRead += framesToReadThisIteration;\n\n        pPagedAudioBuffer->absoluteCursor += framesToReadThisIteration;\n        pPagedAudioBuffer->relativeCursor += framesToReadThisIteration;\n\n        /* Move to the next page if necessary. If there's no more pages, we need to return MA_AT_END. */\n        MA_ASSERT(pPagedAudioBuffer->relativeCursor <= pPagedAudioBuffer->pCurrent->sizeInFrames);\n\n        if (pPagedAudioBuffer->relativeCursor == pPagedAudioBuffer->pCurrent->sizeInFrames) {\n            /* We reached the end of the page. Need to move to the next. If there's no more pages, we're done. */\n            ma_paged_audio_buffer_page* pNext = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pPagedAudioBuffer->pCurrent->pNext);\n            if (pNext == NULL) {\n                result = MA_AT_END;\n                break;  /* We've reached the end. */\n            } else {\n                pPagedAudioBuffer->pCurrent       = pNext;\n                pPagedAudioBuffer->relativeCursor = 0;\n            }\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesRead;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_paged_audio_buffer_seek_to_pcm_frame(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64 frameIndex)\n{\n    if (pPagedAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (frameIndex == pPagedAudioBuffer->absoluteCursor) {\n        return MA_SUCCESS;  /* Nothing to do. */\n    }\n\n    if (frameIndex < pPagedAudioBuffer->absoluteCursor) {\n        /* Moving backwards. Need to move the cursor back to the start, and then move forward. */\n        pPagedAudioBuffer->pCurrent       = ma_paged_audio_buffer_data_get_head(pPagedAudioBuffer->pData);\n        pPagedAudioBuffer->absoluteCursor = 0;\n        pPagedAudioBuffer->relativeCursor = 0;\n\n        /* Fall through to the forward seeking section below. */\n    }\n\n    if (frameIndex > pPagedAudioBuffer->absoluteCursor) {\n        /* Moving forward. */\n        ma_paged_audio_buffer_page* pPage;\n        ma_uint64 runningCursor = 0;\n\n        for (pPage = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&ma_paged_audio_buffer_data_get_head(pPagedAudioBuffer->pData)->pNext); pPage != NULL; pPage = (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(&pPage->pNext)) {\n            ma_uint64 pageRangeBeg = runningCursor;\n            ma_uint64 pageRangeEnd = pageRangeBeg + pPage->sizeInFrames;\n\n            if (frameIndex >= pageRangeBeg) {\n                if (frameIndex < pageRangeEnd || (frameIndex == pageRangeEnd && pPage == (ma_paged_audio_buffer_page*)ma_atomic_load_ptr(ma_paged_audio_buffer_data_get_tail(pPagedAudioBuffer->pData)))) {  /* A small edge case - allow seeking to the very end of the buffer. */\n                    /* We found the page. */\n                    pPagedAudioBuffer->pCurrent       = pPage;\n                    pPagedAudioBuffer->absoluteCursor = frameIndex;\n                    pPagedAudioBuffer->relativeCursor = frameIndex - pageRangeBeg;\n                    return MA_SUCCESS;\n                }\n            }\n\n            runningCursor = pageRangeEnd;\n        }\n\n        /* Getting here means we tried seeking too far forward. Don't change any state. */\n        return MA_BAD_SEEK;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_get_cursor_in_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;   /* Safety. */\n\n    if (pPagedAudioBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = pPagedAudioBuffer->absoluteCursor;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_paged_audio_buffer_get_length_in_pcm_frames(ma_paged_audio_buffer* pPagedAudioBuffer, ma_uint64* pLength)\n{\n    return ma_paged_audio_buffer_data_get_length_in_pcm_frames(pPagedAudioBuffer->pData, pLength);\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nVFS\n\n**************************************************************************************************************************************************************/\nMA_API ma_result ma_vfs_open(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pFile = NULL;\n\n    if (pVFS == NULL || pFilePath == NULL || openMode == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onOpen == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onOpen(pVFS, pFilePath, openMode, pFile);\n}\n\nMA_API ma_result ma_vfs_open_w(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pFile = NULL;\n\n    if (pVFS == NULL || pFilePath == NULL || openMode == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onOpenW == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onOpenW(pVFS, pFilePath, openMode, pFile);\n}\n\nMA_API ma_result ma_vfs_close(ma_vfs* pVFS, ma_vfs_file file)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pVFS == NULL || file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onClose == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onClose(pVFS, file);\n}\n\nMA_API ma_result ma_vfs_read(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n    ma_result result;\n    size_t bytesRead = 0;\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = 0;\n    }\n\n    if (pVFS == NULL || file == NULL || pDst == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onRead == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    result = pCallbacks->onRead(pVFS, file, pDst, sizeInBytes, &bytesRead);\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = bytesRead;\n    }\n\n    if (result == MA_SUCCESS && bytesRead == 0 && sizeInBytes > 0) {\n        result  = MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_vfs_write(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pBytesWritten != NULL) {\n        *pBytesWritten = 0;\n    }\n\n    if (pVFS == NULL || file == NULL || pSrc == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onWrite == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onWrite(pVFS, file, pSrc, sizeInBytes, pBytesWritten);\n}\n\nMA_API ma_result ma_vfs_seek(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pVFS == NULL || file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onSeek == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onSeek(pVFS, file, offset, origin);\n}\n\nMA_API ma_result ma_vfs_tell(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    if (pVFS == NULL || file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onTell == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onTell(pVFS, file, pCursor);\n}\n\nMA_API ma_result ma_vfs_info(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo)\n{\n    ma_vfs_callbacks* pCallbacks = (ma_vfs_callbacks*)pVFS;\n\n    if (pInfo == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pInfo);\n\n    if (pVFS == NULL || file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pCallbacks->onInfo == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pCallbacks->onInfo(pVFS, file, pInfo);\n}\n\n\n#if !defined(MA_USE_WIN32_FILEIO) && (defined(MA_WIN32) && (defined(MA_WIN32_DESKTOP) || defined(MA_WIN32_NXDK)) && !defined(MA_NO_WIN32_FILEIO) && !defined(MA_POSIX))\n    #define MA_USE_WIN32_FILEIO\n#endif\n\n#if defined(MA_USE_WIN32_FILEIO)\n/*\nWe need to dynamically load SetFilePointer or SetFilePointerEx because older versions of Windows do\nnot have the Ex version. We therefore need to do some dynamic branching depending on what's available.\n\nWe load these when we load our first file from the default VFS. It's left open for the life of the\nprogram and is left to the OS to uninitialize when the program terminates.\n*/\ntypedef DWORD (__stdcall * ma_SetFilePointer_proc)(HANDLE hFile, LONG lDistanceToMove, LONG* lpDistanceToMoveHigh, DWORD dwMoveMethod);\ntypedef BOOL  (__stdcall * ma_SetFilePointerEx_proc)(HANDLE hFile, LARGE_INTEGER liDistanceToMove, LARGE_INTEGER* lpNewFilePointer, DWORD dwMoveMethod);\n\nstatic ma_handle hKernel32DLL = NULL;\nstatic ma_SetFilePointer_proc   ma_SetFilePointer   = NULL;\nstatic ma_SetFilePointerEx_proc ma_SetFilePointerEx = NULL;\n\nstatic void ma_win32_fileio_init(void)\n{\n    if (hKernel32DLL == NULL) {\n        hKernel32DLL = ma_dlopen(NULL, \"kernel32.dll\");\n        if (hKernel32DLL != NULL) {\n            ma_SetFilePointer   = (ma_SetFilePointer_proc)  ma_dlsym(NULL, hKernel32DLL, \"SetFilePointer\");\n            ma_SetFilePointerEx = (ma_SetFilePointerEx_proc)ma_dlsym(NULL, hKernel32DLL, \"SetFilePointerEx\");\n        }\n    }\n}\n\nstatic void ma_default_vfs__get_open_settings_win32(ma_uint32 openMode, DWORD* pDesiredAccess, DWORD* pShareMode, DWORD* pCreationDisposition)\n{\n    *pDesiredAccess = 0;\n    if ((openMode & MA_OPEN_MODE_READ) != 0) {\n        *pDesiredAccess |= GENERIC_READ;\n    }\n    if ((openMode & MA_OPEN_MODE_WRITE) != 0) {\n        *pDesiredAccess |= GENERIC_WRITE;\n    }\n\n    *pShareMode = 0;\n    if ((openMode & MA_OPEN_MODE_READ) != 0) {\n        *pShareMode |= FILE_SHARE_READ;\n    }\n\n    if ((openMode & MA_OPEN_MODE_WRITE) != 0) {\n        *pCreationDisposition = CREATE_ALWAYS;  /* Opening in write mode. Truncate. */\n    } else {\n        *pCreationDisposition = OPEN_EXISTING;  /* Opening in read mode. File must exist. */\n    }\n}\n\nstatic ma_result ma_default_vfs_open__win32(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    HANDLE hFile;\n    DWORD dwDesiredAccess;\n    DWORD dwShareMode;\n    DWORD dwCreationDisposition;\n\n    (void)pVFS;\n\n    /* Load some Win32 symbols dynamically so we can dynamically check for the existence of SetFilePointerEx. */\n    ma_win32_fileio_init();\n\n    ma_default_vfs__get_open_settings_win32(openMode, &dwDesiredAccess, &dwShareMode, &dwCreationDisposition);\n\n    hFile = CreateFileA(pFilePath, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, FILE_ATTRIBUTE_NORMAL, NULL);\n    if (hFile == INVALID_HANDLE_VALUE) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    *pFile = hFile;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_open_w__win32(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    #if !defined(MA_XBOX_NXDK)\n    {\n        HANDLE hFile;\n        DWORD dwDesiredAccess;\n        DWORD dwShareMode;\n        DWORD dwCreationDisposition;\n\n        (void)pVFS;\n\n        /* Load some Win32 symbols dynamically so we can dynamically check for the existence of SetFilePointerEx. */\n        ma_win32_fileio_init();\n\n        ma_default_vfs__get_open_settings_win32(openMode, &dwDesiredAccess, &dwShareMode, &dwCreationDisposition);\n\n        hFile = CreateFileW(pFilePath, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, FILE_ATTRIBUTE_NORMAL, NULL);\n        if (hFile == INVALID_HANDLE_VALUE) {\n            return ma_result_from_GetLastError(GetLastError());\n        }\n\n        *pFile = hFile;\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* No CreateFileW() available. */\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nstatic ma_result ma_default_vfs_close__win32(ma_vfs* pVFS, ma_vfs_file file)\n{\n    (void)pVFS;\n\n    if (CloseHandle((HANDLE)file) == 0) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_default_vfs_read__win32(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead)\n{\n    ma_result result = MA_SUCCESS;\n    size_t totalBytesRead;\n\n    (void)pVFS;\n\n    totalBytesRead = 0;\n    while (totalBytesRead < sizeInBytes) {\n        size_t bytesRemaining;\n        DWORD bytesToRead;\n        DWORD bytesRead;\n        BOOL readResult;\n\n        bytesRemaining = sizeInBytes - totalBytesRead;\n        if (bytesRemaining >= 0xFFFFFFFF) {\n            bytesToRead = 0xFFFFFFFF;\n        } else {\n            bytesToRead = (DWORD)bytesRemaining;\n        }\n\n        readResult = ReadFile((HANDLE)file, ma_offset_ptr(pDst, totalBytesRead), bytesToRead, &bytesRead, NULL);\n        if (readResult == 1 && bytesRead == 0) {\n            result = MA_AT_END;\n            break;  /* EOF */\n        }\n\n        totalBytesRead += bytesRead;\n\n        if (bytesRead < bytesToRead) {\n            break;  /* EOF */\n        }\n\n        if (readResult == 0) {\n            result = ma_result_from_GetLastError(GetLastError());\n            break;\n        }\n    }\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = totalBytesRead;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_default_vfs_write__win32(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten)\n{\n    ma_result result = MA_SUCCESS;\n    size_t totalBytesWritten;\n\n    (void)pVFS;\n\n    totalBytesWritten = 0;\n    while (totalBytesWritten < sizeInBytes) {\n        size_t bytesRemaining;\n        DWORD bytesToWrite;\n        DWORD bytesWritten;\n        BOOL writeResult;\n\n        bytesRemaining = sizeInBytes - totalBytesWritten;\n        if (bytesRemaining >= 0xFFFFFFFF) {\n            bytesToWrite = 0xFFFFFFFF;\n        } else {\n            bytesToWrite = (DWORD)bytesRemaining;\n        }\n\n        writeResult = WriteFile((HANDLE)file, ma_offset_ptr(pSrc, totalBytesWritten), bytesToWrite, &bytesWritten, NULL);\n        totalBytesWritten += bytesWritten;\n\n        if (writeResult == 0) {\n            result = ma_result_from_GetLastError(GetLastError());\n            break;\n        }\n    }\n\n    if (pBytesWritten != NULL) {\n        *pBytesWritten = totalBytesWritten;\n    }\n\n    return result;\n}\n\n\nstatic ma_result ma_default_vfs_seek__win32(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin)\n{\n    LARGE_INTEGER liDistanceToMove;\n    DWORD dwMoveMethod;\n    BOOL result;\n\n    (void)pVFS;\n\n    liDistanceToMove.QuadPart = offset;\n\n    /*  */ if (origin == ma_seek_origin_current) {\n        dwMoveMethod = FILE_CURRENT;\n    } else if (origin == ma_seek_origin_end) {\n        dwMoveMethod = FILE_END;\n    } else {\n        dwMoveMethod = FILE_BEGIN;\n    }\n\n    if (ma_SetFilePointerEx != NULL) {\n        result = ma_SetFilePointerEx((HANDLE)file, liDistanceToMove, NULL, dwMoveMethod);\n    } else if (ma_SetFilePointer != NULL) {\n        /* No SetFilePointerEx() so restrict to 31 bits. */\n        if (offset > 0x7FFFFFFF) {\n            return MA_OUT_OF_RANGE;\n        }\n\n        result = ma_SetFilePointer((HANDLE)file, (LONG)liDistanceToMove.QuadPart, NULL, dwMoveMethod);\n    } else {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    if (result == 0) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_tell__win32(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor)\n{\n    LARGE_INTEGER liZero;\n    LARGE_INTEGER liTell;\n    BOOL result;\n\n    (void)pVFS;\n\n    liZero.QuadPart = 0;\n\n    if (ma_SetFilePointerEx != NULL) {\n        result = ma_SetFilePointerEx((HANDLE)file, liZero, &liTell, FILE_CURRENT);\n    } else if (ma_SetFilePointer != NULL) {\n        LONG tell;\n\n        result = ma_SetFilePointer((HANDLE)file, (LONG)liZero.QuadPart, &tell, FILE_CURRENT);\n        liTell.QuadPart = tell;\n    } else {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    if (result == 0) {\n        return ma_result_from_GetLastError(GetLastError());\n    }\n\n    if (pCursor != NULL) {\n        *pCursor = liTell.QuadPart;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_info__win32(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo)\n{\n    (void)pVFS;\n\n    #if !defined(MA_XBOX_NXDK)\n    {\n        BY_HANDLE_FILE_INFORMATION fi;\n        BOOL result;\n\n        result = GetFileInformationByHandle((HANDLE)file, &fi);\n        if (result == 0) {\n            return ma_result_from_GetLastError(GetLastError());\n        }\n\n        pInfo->sizeInBytes = ((ma_uint64)fi.nFileSizeHigh << 32) | ((ma_uint64)fi.nFileSizeLow);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* GetFileInformationByHandle() is unavailable. */\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n#else\nstatic ma_result ma_default_vfs_open__stdio(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    ma_result result;\n    FILE* pFileStd;\n    const char* pOpenModeStr;\n\n    MA_ASSERT(pFilePath != NULL);\n    MA_ASSERT(openMode  != 0);\n    MA_ASSERT(pFile     != NULL);\n\n    (void)pVFS;\n\n    if ((openMode & MA_OPEN_MODE_READ) != 0) {\n        if ((openMode & MA_OPEN_MODE_WRITE) != 0) {\n            pOpenModeStr = \"r+\";\n        } else {\n            pOpenModeStr = \"rb\";\n        }\n    } else {\n        pOpenModeStr = \"wb\";\n    }\n\n    result = ma_fopen(&pFileStd, pFilePath, pOpenModeStr);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pFile = pFileStd;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_open_w__stdio(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    ma_result result;\n    FILE* pFileStd;\n    const wchar_t* pOpenModeStr;\n\n    MA_ASSERT(pFilePath != NULL);\n    MA_ASSERT(openMode  != 0);\n    MA_ASSERT(pFile     != NULL);\n\n    (void)pVFS;\n\n    if ((openMode & MA_OPEN_MODE_READ) != 0) {\n        if ((openMode & MA_OPEN_MODE_WRITE) != 0) {\n            pOpenModeStr = L\"r+\";\n        } else {\n            pOpenModeStr = L\"rb\";\n        }\n    } else {\n        pOpenModeStr = L\"wb\";\n    }\n\n    result = ma_wfopen(&pFileStd, pFilePath, pOpenModeStr, (pVFS != NULL) ? &((ma_default_vfs*)pVFS)->allocationCallbacks : NULL);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pFile = pFileStd;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_close__stdio(ma_vfs* pVFS, ma_vfs_file file)\n{\n    MA_ASSERT(file != NULL);\n\n    (void)pVFS;\n\n    fclose((FILE*)file);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_read__stdio(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead)\n{\n    size_t result;\n\n    MA_ASSERT(file != NULL);\n    MA_ASSERT(pDst != NULL);\n\n    (void)pVFS;\n\n    result = fread(pDst, 1, sizeInBytes, (FILE*)file);\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = result;\n    }\n\n    if (result != sizeInBytes) {\n        if (result == 0 && feof((FILE*)file)) {\n            return MA_AT_END;\n        } else {\n            return ma_result_from_errno(ferror((FILE*)file));\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_write__stdio(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten)\n{\n    size_t result;\n\n    MA_ASSERT(file != NULL);\n    MA_ASSERT(pSrc != NULL);\n\n    (void)pVFS;\n\n    result = fwrite(pSrc, 1, sizeInBytes, (FILE*)file);\n\n    if (pBytesWritten != NULL) {\n        *pBytesWritten = result;\n    }\n\n    if (result != sizeInBytes) {\n        return ma_result_from_errno(ferror((FILE*)file));\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_seek__stdio(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin)\n{\n    int result;\n    int whence;\n\n    MA_ASSERT(file != NULL);\n\n    (void)pVFS;\n\n    if (origin == ma_seek_origin_start) {\n        whence = SEEK_SET;\n    } else if (origin == ma_seek_origin_end) {\n        whence = SEEK_END;\n    } else {\n        whence = SEEK_CUR;\n    }\n\n#if defined(_WIN32)\n    #if defined(_MSC_VER) && _MSC_VER > 1200\n        result = _fseeki64((FILE*)file, offset, whence);\n    #else\n        /* No _fseeki64() so restrict to 31 bits. */\n        if (offset > 0x7FFFFFFF) {\n            return MA_OUT_OF_RANGE;\n        }\n\n        result = fseek((FILE*)file, (int)offset, whence);\n    #endif\n#else\n    result = fseek((FILE*)file, (long int)offset, whence);\n#endif\n    if (result != 0) {\n        return MA_ERROR;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_default_vfs_tell__stdio(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor)\n{\n    ma_int64 result;\n\n    MA_ASSERT(file    != NULL);\n    MA_ASSERT(pCursor != NULL);\n\n    (void)pVFS;\n\n#if defined(_WIN32)\n    #if defined(_MSC_VER) && _MSC_VER > 1200\n        result = _ftelli64((FILE*)file);\n    #else\n        result = ftell((FILE*)file);\n    #endif\n#else\n    result = ftell((FILE*)file);\n#endif\n\n    *pCursor = result;\n\n    return MA_SUCCESS;\n}\n\n#if !defined(_MSC_VER) && !((defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)) && !defined(MA_BSD)\nint fileno(FILE *stream);\n#endif\n\nstatic ma_result ma_default_vfs_info__stdio(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo)\n{\n    int fd;\n    struct stat info;\n\n    MA_ASSERT(file  != NULL);\n    MA_ASSERT(pInfo != NULL);\n\n    (void)pVFS;\n\n#if defined(_MSC_VER)\n    fd = _fileno((FILE*)file);\n#else\n    fd =  fileno((FILE*)file);\n#endif\n\n    if (fstat(fd, &info) != 0) {\n        return ma_result_from_errno(errno);\n    }\n\n    pInfo->sizeInBytes = info.st_size;\n\n    return MA_SUCCESS;\n}\n#endif\n\n\nstatic ma_result ma_default_vfs_open(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    if (pFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pFile = NULL;\n\n    if (pFilePath == NULL || openMode == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_open__win32(pVFS, pFilePath, openMode, pFile);\n#else\n    return ma_default_vfs_open__stdio(pVFS, pFilePath, openMode, pFile);\n#endif\n}\n\nstatic ma_result ma_default_vfs_open_w(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    if (pFile == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pFile = NULL;\n\n    if (pFilePath == NULL || openMode == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_open_w__win32(pVFS, pFilePath, openMode, pFile);\n#else\n    return ma_default_vfs_open_w__stdio(pVFS, pFilePath, openMode, pFile);\n#endif\n}\n\nstatic ma_result ma_default_vfs_close(ma_vfs* pVFS, ma_vfs_file file)\n{\n    if (file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_close__win32(pVFS, file);\n#else\n    return ma_default_vfs_close__stdio(pVFS, file);\n#endif\n}\n\nstatic ma_result ma_default_vfs_read(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead)\n{\n    if (pBytesRead != NULL) {\n        *pBytesRead = 0;\n    }\n\n    if (file == NULL || pDst == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_read__win32(pVFS, file, pDst, sizeInBytes, pBytesRead);\n#else\n    return ma_default_vfs_read__stdio(pVFS, file, pDst, sizeInBytes, pBytesRead);\n#endif\n}\n\nstatic ma_result ma_default_vfs_write(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten)\n{\n    if (pBytesWritten != NULL) {\n        *pBytesWritten = 0;\n    }\n\n    if (file == NULL || pSrc == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_write__win32(pVFS, file, pSrc, sizeInBytes, pBytesWritten);\n#else\n    return ma_default_vfs_write__stdio(pVFS, file, pSrc, sizeInBytes, pBytesWritten);\n#endif\n}\n\nstatic ma_result ma_default_vfs_seek(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin)\n{\n    if (file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_seek__win32(pVFS, file, offset, origin);\n#else\n    return ma_default_vfs_seek__stdio(pVFS, file, offset, origin);\n#endif\n}\n\nstatic ma_result ma_default_vfs_tell(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    if (file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    return ma_default_vfs_tell__win32(pVFS, file, pCursor);\n#else\n    return ma_default_vfs_tell__stdio(pVFS, file, pCursor);\n#endif\n}\n\nstatic ma_result ma_default_vfs_info(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo)\n{\n    ma_result result;\n\n    if (pInfo == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pInfo);\n\n    if (file == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n#if defined(MA_USE_WIN32_FILEIO)\n    result = ma_default_vfs_info__win32(pVFS, file, pInfo);\n#else\n    result = ma_default_vfs_info__stdio(pVFS, file, pInfo);\n#endif\n\n    if (result == MA_NOT_IMPLEMENTED) {\n        /* Not implemented. Fall back to seek/tell/seek. */\n        ma_int64 cursor;\n        ma_int64 sizeInBytes;\n        \n        result = ma_default_vfs_tell(pVFS, file, &cursor);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_default_vfs_seek(pVFS, file, 0, ma_seek_origin_end);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_default_vfs_tell(pVFS, file, &sizeInBytes);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pInfo->sizeInBytes = sizeInBytes;\n\n        result = ma_default_vfs_seek(pVFS, file, cursor, ma_seek_origin_start);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        MA_ASSERT(result == MA_SUCCESS);\n    }\n\n    return result;\n}\n\n\nMA_API ma_result ma_default_vfs_init(ma_default_vfs* pVFS, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pVFS == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pVFS->cb.onOpen  = ma_default_vfs_open;\n    pVFS->cb.onOpenW = ma_default_vfs_open_w;\n    pVFS->cb.onClose = ma_default_vfs_close;\n    pVFS->cb.onRead  = ma_default_vfs_read;\n    pVFS->cb.onWrite = ma_default_vfs_write;\n    pVFS->cb.onSeek  = ma_default_vfs_seek;\n    pVFS->cb.onTell  = ma_default_vfs_tell;\n    pVFS->cb.onInfo  = ma_default_vfs_info;\n    ma_allocation_callbacks_init_copy(&pVFS->allocationCallbacks, pAllocationCallbacks);\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_vfs_or_default_open(ma_vfs* pVFS, const char* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_open(pVFS, pFilePath, openMode, pFile);\n    } else {\n        return ma_default_vfs_open(pVFS, pFilePath, openMode, pFile);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_open_w(ma_vfs* pVFS, const wchar_t* pFilePath, ma_uint32 openMode, ma_vfs_file* pFile)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_open_w(pVFS, pFilePath, openMode, pFile);\n    } else {\n        return ma_default_vfs_open_w(pVFS, pFilePath, openMode, pFile);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_close(ma_vfs* pVFS, ma_vfs_file file)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_close(pVFS, file);\n    } else {\n        return ma_default_vfs_close(pVFS, file);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_read(ma_vfs* pVFS, ma_vfs_file file, void* pDst, size_t sizeInBytes, size_t* pBytesRead)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_read(pVFS, file, pDst, sizeInBytes, pBytesRead);\n    } else {\n        return ma_default_vfs_read(pVFS, file, pDst, sizeInBytes, pBytesRead);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_write(ma_vfs* pVFS, ma_vfs_file file, const void* pSrc, size_t sizeInBytes, size_t* pBytesWritten)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_write(pVFS, file, pSrc, sizeInBytes, pBytesWritten);\n    } else {\n        return ma_default_vfs_write(pVFS, file, pSrc, sizeInBytes, pBytesWritten);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_seek(ma_vfs* pVFS, ma_vfs_file file, ma_int64 offset, ma_seek_origin origin)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_seek(pVFS, file, offset, origin);\n    } else {\n        return ma_default_vfs_seek(pVFS, file, offset, origin);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_tell(ma_vfs* pVFS, ma_vfs_file file, ma_int64* pCursor)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_tell(pVFS, file, pCursor);\n    } else {\n        return ma_default_vfs_tell(pVFS, file, pCursor);\n    }\n}\n\nMA_API ma_result ma_vfs_or_default_info(ma_vfs* pVFS, ma_vfs_file file, ma_file_info* pInfo)\n{\n    if (pVFS != NULL) {\n        return ma_vfs_info(pVFS, file, pInfo);\n    } else {\n        return ma_default_vfs_info(pVFS, file, pInfo);\n    }\n}\n\n\n\nstatic ma_result ma_vfs_open_and_read_file_ex(ma_vfs* pVFS, const char* pFilePath, const wchar_t* pFilePathW, void** ppData, size_t* pSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_result result;\n    ma_vfs_file file;\n    ma_file_info info;\n    void* pData;\n    size_t bytesRead;\n\n    if (ppData != NULL) {\n        *ppData = NULL;\n    }\n    if (pSize != NULL) {\n        *pSize = 0;\n    }\n\n    if (ppData == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFilePath != NULL) {\n        result = ma_vfs_or_default_open(pVFS, pFilePath, MA_OPEN_MODE_READ, &file);\n    } else {\n        result = ma_vfs_or_default_open_w(pVFS, pFilePathW, MA_OPEN_MODE_READ, &file);\n    }\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_vfs_or_default_info(pVFS, file, &info);\n    if (result != MA_SUCCESS) {\n        ma_vfs_or_default_close(pVFS, file);\n        return result;\n    }\n\n    if (info.sizeInBytes > MA_SIZE_MAX) {\n        ma_vfs_or_default_close(pVFS, file);\n        return MA_TOO_BIG;\n    }\n\n    pData = ma_malloc((size_t)info.sizeInBytes, pAllocationCallbacks);  /* Safe cast. */\n    if (pData == NULL) {\n        ma_vfs_or_default_close(pVFS, file);\n        return result;\n    }\n\n    result = ma_vfs_or_default_read(pVFS, file, pData, (size_t)info.sizeInBytes, &bytesRead);  /* Safe cast. */\n    ma_vfs_or_default_close(pVFS, file);\n\n    if (result != MA_SUCCESS) {\n        ma_free(pData, pAllocationCallbacks);\n        return result;\n    }\n\n    if (pSize != NULL) {\n        *pSize = bytesRead;\n    }\n\n    MA_ASSERT(ppData != NULL);\n    *ppData = pData;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_vfs_open_and_read_file(ma_vfs* pVFS, const char* pFilePath, void** ppData, size_t* pSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_vfs_open_and_read_file_ex(pVFS, pFilePath, NULL, ppData, pSize, pAllocationCallbacks);\n}\n\nMA_API ma_result ma_vfs_open_and_read_file_w(ma_vfs* pVFS, const wchar_t* pFilePath, void** ppData, size_t* pSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_vfs_open_and_read_file_ex(pVFS, NULL, pFilePath, ppData, pSize, pAllocationCallbacks);\n}\n\n\n\n/**************************************************************************************************************************************************************\n\nDecoding and Encoding Headers. These are auto-generated from a tool.\n\n**************************************************************************************************************************************************************/\n#if !defined(MA_NO_WAV) && (!defined(MA_NO_DECODING) || !defined(MA_NO_ENCODING))\n#define MA_HAS_WAV\n\n/* dr_wav_h begin */\n#ifndef ma_dr_wav_h\n#define ma_dr_wav_h\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n#define MA_DR_WAV_STRINGIFY(x)      #x\n#define MA_DR_WAV_XSTRINGIFY(x)     MA_DR_WAV_STRINGIFY(x)\n#define MA_DR_WAV_VERSION_MAJOR     0\n#define MA_DR_WAV_VERSION_MINOR     14\n#define MA_DR_WAV_VERSION_REVISION  4\n#define MA_DR_WAV_VERSION_STRING    MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_MAJOR) \".\" MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_MINOR) \".\" MA_DR_WAV_XSTRINGIFY(MA_DR_WAV_VERSION_REVISION)\n#include <stddef.h>\n#define MA_DR_WAVE_FORMAT_PCM          0x1\n#define MA_DR_WAVE_FORMAT_ADPCM        0x2\n#define MA_DR_WAVE_FORMAT_IEEE_FLOAT   0x3\n#define MA_DR_WAVE_FORMAT_ALAW         0x6\n#define MA_DR_WAVE_FORMAT_MULAW        0x7\n#define MA_DR_WAVE_FORMAT_DVI_ADPCM    0x11\n#define MA_DR_WAVE_FORMAT_EXTENSIBLE   0xFFFE\n#define MA_DR_WAV_SEQUENTIAL            0x00000001\n#define MA_DR_WAV_WITH_METADATA         0x00000002\nMA_API void ma_dr_wav_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision);\nMA_API const char* ma_dr_wav_version_string(void);\ntypedef enum\n{\n    MA_DR_WAV_SEEK_SET,\n    MA_DR_WAV_SEEK_CUR,\n    MA_DR_WAV_SEEK_END\n} ma_dr_wav_seek_origin;\ntypedef enum\n{\n    ma_dr_wav_container_riff,\n    ma_dr_wav_container_rifx,\n    ma_dr_wav_container_w64,\n    ma_dr_wav_container_rf64,\n    ma_dr_wav_container_aiff\n} ma_dr_wav_container;\ntypedef struct\n{\n    union\n    {\n        ma_uint8 fourcc[4];\n        ma_uint8 guid[16];\n    } id;\n    ma_uint64 sizeInBytes;\n    unsigned int paddingSize;\n} ma_dr_wav_chunk_header;\ntypedef struct\n{\n    ma_uint16 formatTag;\n    ma_uint16 channels;\n    ma_uint32 sampleRate;\n    ma_uint32 avgBytesPerSec;\n    ma_uint16 blockAlign;\n    ma_uint16 bitsPerSample;\n    ma_uint16 extendedSize;\n    ma_uint16 validBitsPerSample;\n    ma_uint32 channelMask;\n    ma_uint8 subFormat[16];\n} ma_dr_wav_fmt;\nMA_API ma_uint16 ma_dr_wav_fmt_get_format(const ma_dr_wav_fmt* pFMT);\ntypedef size_t (* ma_dr_wav_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead);\ntypedef size_t (* ma_dr_wav_write_proc)(void* pUserData, const void* pData, size_t bytesToWrite);\ntypedef ma_bool32 (* ma_dr_wav_seek_proc)(void* pUserData, int offset, ma_dr_wav_seek_origin origin);\ntypedef ma_bool32 (* ma_dr_wav_tell_proc)(void* pUserData, ma_int64* pCursor);\ntypedef ma_uint64 (* ma_dr_wav_chunk_proc)(void* pChunkUserData, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, void* pReadSeekUserData, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_container container, const ma_dr_wav_fmt* pFMT);\ntypedef struct\n{\n    const ma_uint8* data;\n    size_t dataSize;\n    size_t currentReadPos;\n} ma_dr_wav__memory_stream;\ntypedef struct\n{\n    void** ppData;\n    size_t* pDataSize;\n    size_t dataSize;\n    size_t dataCapacity;\n    size_t currentWritePos;\n} ma_dr_wav__memory_stream_write;\ntypedef struct\n{\n    ma_dr_wav_container container;\n    ma_uint32 format;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint32 bitsPerSample;\n} ma_dr_wav_data_format;\ntypedef enum\n{\n    ma_dr_wav_metadata_type_none                        = 0,\n    ma_dr_wav_metadata_type_unknown                     = 1 << 0,\n    ma_dr_wav_metadata_type_smpl                        = 1 << 1,\n    ma_dr_wav_metadata_type_inst                        = 1 << 2,\n    ma_dr_wav_metadata_type_cue                         = 1 << 3,\n    ma_dr_wav_metadata_type_acid                        = 1 << 4,\n    ma_dr_wav_metadata_type_bext                        = 1 << 5,\n    ma_dr_wav_metadata_type_list_label                  = 1 << 6,\n    ma_dr_wav_metadata_type_list_note                   = 1 << 7,\n    ma_dr_wav_metadata_type_list_labelled_cue_region    = 1 << 8,\n    ma_dr_wav_metadata_type_list_info_software          = 1 << 9,\n    ma_dr_wav_metadata_type_list_info_copyright         = 1 << 10,\n    ma_dr_wav_metadata_type_list_info_title             = 1 << 11,\n    ma_dr_wav_metadata_type_list_info_artist            = 1 << 12,\n    ma_dr_wav_metadata_type_list_info_comment           = 1 << 13,\n    ma_dr_wav_metadata_type_list_info_date              = 1 << 14,\n    ma_dr_wav_metadata_type_list_info_genre             = 1 << 15,\n    ma_dr_wav_metadata_type_list_info_album             = 1 << 16,\n    ma_dr_wav_metadata_type_list_info_tracknumber       = 1 << 17,\n    ma_dr_wav_metadata_type_list_info_location          = 1 << 18,\n    ma_dr_wav_metadata_type_list_info_organization      = 1 << 19,\n    ma_dr_wav_metadata_type_list_info_keywords          = 1 << 20,\n    ma_dr_wav_metadata_type_list_info_medium            = 1 << 21,\n    ma_dr_wav_metadata_type_list_info_description       = 1 << 22,\n    ma_dr_wav_metadata_type_list_all_info_strings       = ma_dr_wav_metadata_type_list_info_software\n                                                    | ma_dr_wav_metadata_type_list_info_copyright\n                                                    | ma_dr_wav_metadata_type_list_info_title\n                                                    | ma_dr_wav_metadata_type_list_info_artist\n                                                    | ma_dr_wav_metadata_type_list_info_comment\n                                                    | ma_dr_wav_metadata_type_list_info_date\n                                                    | ma_dr_wav_metadata_type_list_info_genre\n                                                    | ma_dr_wav_metadata_type_list_info_album\n                                                    | ma_dr_wav_metadata_type_list_info_tracknumber\n                                                    | ma_dr_wav_metadata_type_list_info_location\n                                                    | ma_dr_wav_metadata_type_list_info_organization\n                                                    | ma_dr_wav_metadata_type_list_info_keywords\n                                                    | ma_dr_wav_metadata_type_list_info_medium\n                                                    | ma_dr_wav_metadata_type_list_info_description,\n    ma_dr_wav_metadata_type_list_all_adtl               = ma_dr_wav_metadata_type_list_label\n                                                    | ma_dr_wav_metadata_type_list_note\n                                                    | ma_dr_wav_metadata_type_list_labelled_cue_region,\n    ma_dr_wav_metadata_type_all                         = -2,\n    ma_dr_wav_metadata_type_all_including_unknown       = -1\n} ma_dr_wav_metadata_type;\ntypedef enum\n{\n    ma_dr_wav_smpl_loop_type_forward  = 0,\n    ma_dr_wav_smpl_loop_type_pingpong = 1,\n    ma_dr_wav_smpl_loop_type_backward = 2\n} ma_dr_wav_smpl_loop_type;\ntypedef struct\n{\n    ma_uint32 cuePointId;\n    ma_uint32 type;\n    ma_uint32 firstSampleOffset;\n    ma_uint32 lastSampleOffset;\n    ma_uint32 sampleFraction;\n    ma_uint32 playCount;\n} ma_dr_wav_smpl_loop;\ntypedef struct\n{\n    ma_uint32 manufacturerId;\n    ma_uint32 productId;\n    ma_uint32 samplePeriodNanoseconds;\n    ma_uint32 midiUnityNote;\n    ma_uint32 midiPitchFraction;\n    ma_uint32 smpteFormat;\n    ma_uint32 smpteOffset;\n    ma_uint32 sampleLoopCount;\n    ma_uint32 samplerSpecificDataSizeInBytes;\n    ma_dr_wav_smpl_loop* pLoops;\n    ma_uint8* pSamplerSpecificData;\n} ma_dr_wav_smpl;\ntypedef struct\n{\n    ma_int8 midiUnityNote;\n    ma_int8 fineTuneCents;\n    ma_int8 gainDecibels;\n    ma_int8 lowNote;\n    ma_int8 highNote;\n    ma_int8 lowVelocity;\n    ma_int8 highVelocity;\n} ma_dr_wav_inst;\ntypedef struct\n{\n    ma_uint32 id;\n    ma_uint32 playOrderPosition;\n    ma_uint8 dataChunkId[4];\n    ma_uint32 chunkStart;\n    ma_uint32 blockStart;\n    ma_uint32 sampleOffset;\n} ma_dr_wav_cue_point;\ntypedef struct\n{\n    ma_uint32 cuePointCount;\n    ma_dr_wav_cue_point *pCuePoints;\n} ma_dr_wav_cue;\ntypedef enum\n{\n    ma_dr_wav_acid_flag_one_shot      = 1,\n    ma_dr_wav_acid_flag_root_note_set = 2,\n    ma_dr_wav_acid_flag_stretch       = 4,\n    ma_dr_wav_acid_flag_disk_based    = 8,\n    ma_dr_wav_acid_flag_acidizer      = 16\n} ma_dr_wav_acid_flag;\ntypedef struct\n{\n    ma_uint32 flags;\n    ma_uint16 midiUnityNote;\n    ma_uint16 reserved1;\n    float reserved2;\n    ma_uint32 numBeats;\n    ma_uint16 meterDenominator;\n    ma_uint16 meterNumerator;\n    float tempo;\n} ma_dr_wav_acid;\ntypedef struct\n{\n    ma_uint32 cuePointId;\n    ma_uint32 stringLength;\n    char* pString;\n} ma_dr_wav_list_label_or_note;\ntypedef struct\n{\n    char* pDescription;\n    char* pOriginatorName;\n    char* pOriginatorReference;\n    char  pOriginationDate[10];\n    char  pOriginationTime[8];\n    ma_uint64 timeReference;\n    ma_uint16 version;\n    char* pCodingHistory;\n    ma_uint32 codingHistorySize;\n    ma_uint8* pUMID;\n    ma_uint16 loudnessValue;\n    ma_uint16 loudnessRange;\n    ma_uint16 maxTruePeakLevel;\n    ma_uint16 maxMomentaryLoudness;\n    ma_uint16 maxShortTermLoudness;\n} ma_dr_wav_bext;\ntypedef struct\n{\n    ma_uint32 stringLength;\n    char* pString;\n} ma_dr_wav_list_info_text;\ntypedef struct\n{\n    ma_uint32 cuePointId;\n    ma_uint32 sampleLength;\n    ma_uint8 purposeId[4];\n    ma_uint16 country;\n    ma_uint16 language;\n    ma_uint16 dialect;\n    ma_uint16 codePage;\n    ma_uint32 stringLength;\n    char* pString;\n} ma_dr_wav_list_labelled_cue_region;\ntypedef enum\n{\n    ma_dr_wav_metadata_location_invalid,\n    ma_dr_wav_metadata_location_top_level,\n    ma_dr_wav_metadata_location_inside_info_list,\n    ma_dr_wav_metadata_location_inside_adtl_list\n} ma_dr_wav_metadata_location;\ntypedef struct\n{\n    ma_uint8 id[4];\n    ma_dr_wav_metadata_location chunkLocation;\n    ma_uint32 dataSizeInBytes;\n    ma_uint8* pData;\n} ma_dr_wav_unknown_metadata;\ntypedef struct\n{\n    ma_dr_wav_metadata_type type;\n    union\n    {\n        ma_dr_wav_cue cue;\n        ma_dr_wav_smpl smpl;\n        ma_dr_wav_acid acid;\n        ma_dr_wav_inst inst;\n        ma_dr_wav_bext bext;\n        ma_dr_wav_list_label_or_note labelOrNote;\n        ma_dr_wav_list_labelled_cue_region labelledCueRegion;\n        ma_dr_wav_list_info_text infoText;\n        ma_dr_wav_unknown_metadata unknown;\n    } data;\n} ma_dr_wav_metadata;\ntypedef struct\n{\n    ma_dr_wav_read_proc onRead;\n    ma_dr_wav_write_proc onWrite;\n    ma_dr_wav_seek_proc onSeek;\n    ma_dr_wav_tell_proc onTell;\n    void* pUserData;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_dr_wav_container container;\n    ma_dr_wav_fmt fmt;\n    ma_uint32 sampleRate;\n    ma_uint16 channels;\n    ma_uint16 bitsPerSample;\n    ma_uint16 translatedFormatTag;\n    ma_uint64 totalPCMFrameCount;\n    ma_uint64 dataChunkDataSize;\n    ma_uint64 dataChunkDataPos;\n    ma_uint64 bytesRemaining;\n    ma_uint64 readCursorInPCMFrames;\n    ma_uint64 dataChunkDataSizeTargetWrite;\n    ma_bool32 isSequentialWrite;\n    ma_dr_wav_metadata* pMetadata;\n    ma_uint32 metadataCount;\n    ma_dr_wav__memory_stream memoryStream;\n    ma_dr_wav__memory_stream_write memoryStreamWrite;\n    struct\n    {\n        ma_uint32 bytesRemainingInBlock;\n        ma_uint16 predictor[2];\n        ma_int32  delta[2];\n        ma_int32  cachedFrames[4];\n        ma_uint32 cachedFrameCount;\n        ma_int32  prevFrames[2][2];\n    } msadpcm;\n    struct\n    {\n        ma_uint32 bytesRemainingInBlock;\n        ma_int32  predictor[2];\n        ma_int32  stepIndex[2];\n        ma_int32  cachedFrames[16];\n        ma_uint32 cachedFrameCount;\n    } ima;\n    struct\n    {\n        ma_bool8 isLE;\n        ma_bool8 isUnsigned;\n    } aiff;\n} ma_dr_wav;\nMA_API ma_bool32 ma_dr_wav_init(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_ex(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, ma_dr_wav_chunk_proc onChunk, void* pReadSeekTellUserData, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_with_metadata(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_write(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_dr_wav_write_proc onWrite, ma_dr_wav_seek_proc onSeek, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_write_sequential(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_dr_wav_write_proc onWrite, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_write_sequential_pcm_frames(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, ma_dr_wav_write_proc onWrite, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_write_with_metadata(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_dr_wav_write_proc onWrite, ma_dr_wav_seek_proc onSeek, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks, ma_dr_wav_metadata* pMetadata, ma_uint32 metadataCount);\nMA_API ma_uint64 ma_dr_wav_target_write_size_bytes(const ma_dr_wav_data_format* pFormat, ma_uint64 totalFrameCount, ma_dr_wav_metadata* pMetadata, ma_uint32 metadataCount);\nMA_API ma_dr_wav_metadata* ma_dr_wav_take_ownership_of_metadata(ma_dr_wav* pWav);\nMA_API ma_result ma_dr_wav_uninit(ma_dr_wav* pWav);\nMA_API size_t ma_dr_wav_read_raw(ma_dr_wav* pWav, size_t bytesToRead, void* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_be(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut);\nMA_API ma_bool32 ma_dr_wav_seek_to_pcm_frame(ma_dr_wav* pWav, ma_uint64 targetFrameIndex);\nMA_API ma_result ma_dr_wav_get_cursor_in_pcm_frames(ma_dr_wav* pWav, ma_uint64* pCursor);\nMA_API ma_result ma_dr_wav_get_length_in_pcm_frames(ma_dr_wav* pWav, ma_uint64* pLength);\nMA_API size_t ma_dr_wav_write_raw(ma_dr_wav* pWav, size_t bytesToWrite, const void* pData);\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData);\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData);\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames_be(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData);\n#ifndef MA_DR_WAV_NO_CONVERSION_API\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16le(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16be(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut);\nMA_API void ma_dr_wav_u8_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s24_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s32_to_s16(ma_int16* pOut, const ma_int32* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_f32_to_s16(ma_int16* pOut, const float* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_f64_to_s16(ma_int16* pOut, const double* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_alaw_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_mulaw_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32le(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32be(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut);\nMA_API void ma_dr_wav_u8_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s16_to_f32(float* pOut, const ma_int16* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s24_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s32_to_f32(float* pOut, const ma_int32* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_f64_to_f32(float* pOut, const double* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_alaw_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_mulaw_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32le(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut);\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32be(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut);\nMA_API void ma_dr_wav_u8_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s16_to_s32(ma_int32* pOut, const ma_int16* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_s24_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_f32_to_s32(ma_int32* pOut, const float* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_f64_to_s32(ma_int32* pOut, const double* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_alaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount);\nMA_API void ma_dr_wav_mulaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount);\n#endif\n#ifndef MA_DR_WAV_NO_STDIO\nMA_API ma_bool32 ma_dr_wav_init_file(ma_dr_wav* pWav, const char* filename, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_ex(ma_dr_wav* pWav, const char* filename, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_ex_w(ma_dr_wav* pWav, const wchar_t* filename, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_with_metadata(ma_dr_wav* pWav, const char* filename, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_with_metadata_w(ma_dr_wav* pWav, const wchar_t* filename, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_pcm_frames(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_pcm_frames_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API ma_bool32 ma_dr_wav_init_memory(ma_dr_wav* pWav, const void* data, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_memory_ex(ma_dr_wav* pWav, const void* data, size_t dataSize, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_memory_with_metadata(ma_dr_wav* pWav, const void* data, size_t dataSize, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_memory_write(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_memory_write_sequential(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_wav_init_memory_write_sequential_pcm_frames(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifndef MA_DR_WAV_NO_CONVERSION_API\nMA_API ma_int16* ma_dr_wav_open_and_read_pcm_frames_s16(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_wav_open_and_read_pcm_frames_f32(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int32* ma_dr_wav_open_and_read_pcm_frames_s32(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifndef MA_DR_WAV_NO_STDIO\nMA_API ma_int16* ma_dr_wav_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_wav_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int32* ma_dr_wav_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_wav_open_file_and_read_pcm_frames_s16_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_wav_open_file_and_read_pcm_frames_f32_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int32* ma_dr_wav_open_file_and_read_pcm_frames_s32_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API ma_int16* ma_dr_wav_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_wav_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int32* ma_dr_wav_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API void ma_dr_wav_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_uint16 ma_dr_wav_bytes_to_u16(const ma_uint8* data);\nMA_API ma_int16 ma_dr_wav_bytes_to_s16(const ma_uint8* data);\nMA_API ma_uint32 ma_dr_wav_bytes_to_u32(const ma_uint8* data);\nMA_API ma_int32 ma_dr_wav_bytes_to_s32(const ma_uint8* data);\nMA_API ma_uint64 ma_dr_wav_bytes_to_u64(const ma_uint8* data);\nMA_API ma_int64 ma_dr_wav_bytes_to_s64(const ma_uint8* data);\nMA_API float ma_dr_wav_bytes_to_f32(const ma_uint8* data);\nMA_API ma_bool32 ma_dr_wav_guid_equal(const ma_uint8 a[16], const ma_uint8 b[16]);\nMA_API ma_bool32 ma_dr_wav_fourcc_equal(const ma_uint8* a, const char* b);\n#ifdef __cplusplus\n}\n#endif\n#endif\n/* dr_wav_h end */\n#endif  /* MA_NO_WAV */\n\n#if !defined(MA_NO_FLAC) && !defined(MA_NO_DECODING)\n#define MA_HAS_FLAC\n\n/* dr_flac_h begin */\n#ifndef ma_dr_flac_h\n#define ma_dr_flac_h\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n#define MA_DR_FLAC_STRINGIFY(x)      #x\n#define MA_DR_FLAC_XSTRINGIFY(x)     MA_DR_FLAC_STRINGIFY(x)\n#define MA_DR_FLAC_VERSION_MAJOR     0\n#define MA_DR_FLAC_VERSION_MINOR     13\n#define MA_DR_FLAC_VERSION_REVISION  3\n#define MA_DR_FLAC_VERSION_STRING    MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_MAJOR) \".\" MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_MINOR) \".\" MA_DR_FLAC_XSTRINGIFY(MA_DR_FLAC_VERSION_REVISION)\n#include <stddef.h>\n#if defined(_MSC_VER) && _MSC_VER >= 1700\n    #define MA_DR_FLAC_DEPRECATED       __declspec(deprecated)\n#elif (defined(__GNUC__) && __GNUC__ >= 4)\n    #define MA_DR_FLAC_DEPRECATED       __attribute__((deprecated))\n#elif defined(__has_feature)\n    #if __has_feature(attribute_deprecated)\n        #define MA_DR_FLAC_DEPRECATED   __attribute__((deprecated))\n    #else\n        #define MA_DR_FLAC_DEPRECATED\n    #endif\n#else\n    #define MA_DR_FLAC_DEPRECATED\n#endif\nMA_API void ma_dr_flac_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision);\nMA_API const char* ma_dr_flac_version_string(void);\n#ifndef MA_DR_FLAC_BUFFER_SIZE\n#define MA_DR_FLAC_BUFFER_SIZE   4096\n#endif\n#ifdef MA_64BIT\ntypedef ma_uint64 ma_dr_flac_cache_t;\n#else\ntypedef ma_uint32 ma_dr_flac_cache_t;\n#endif\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO       0\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_PADDING          1\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_APPLICATION      2\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_SEEKTABLE        3\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT   4\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_CUESHEET         5\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_PICTURE          6\n#define MA_DR_FLAC_METADATA_BLOCK_TYPE_INVALID          127\n#define MA_DR_FLAC_PICTURE_TYPE_OTHER                   0\n#define MA_DR_FLAC_PICTURE_TYPE_FILE_ICON               1\n#define MA_DR_FLAC_PICTURE_TYPE_OTHER_FILE_ICON         2\n#define MA_DR_FLAC_PICTURE_TYPE_COVER_FRONT             3\n#define MA_DR_FLAC_PICTURE_TYPE_COVER_BACK              4\n#define MA_DR_FLAC_PICTURE_TYPE_LEAFLET_PAGE            5\n#define MA_DR_FLAC_PICTURE_TYPE_MEDIA                   6\n#define MA_DR_FLAC_PICTURE_TYPE_LEAD_ARTIST             7\n#define MA_DR_FLAC_PICTURE_TYPE_ARTIST                  8\n#define MA_DR_FLAC_PICTURE_TYPE_CONDUCTOR               9\n#define MA_DR_FLAC_PICTURE_TYPE_BAND                    10\n#define MA_DR_FLAC_PICTURE_TYPE_COMPOSER                11\n#define MA_DR_FLAC_PICTURE_TYPE_LYRICIST                12\n#define MA_DR_FLAC_PICTURE_TYPE_RECORDING_LOCATION      13\n#define MA_DR_FLAC_PICTURE_TYPE_DURING_RECORDING        14\n#define MA_DR_FLAC_PICTURE_TYPE_DURING_PERFORMANCE      15\n#define MA_DR_FLAC_PICTURE_TYPE_SCREEN_CAPTURE          16\n#define MA_DR_FLAC_PICTURE_TYPE_BRIGHT_COLORED_FISH     17\n#define MA_DR_FLAC_PICTURE_TYPE_ILLUSTRATION            18\n#define MA_DR_FLAC_PICTURE_TYPE_BAND_LOGOTYPE           19\n#define MA_DR_FLAC_PICTURE_TYPE_PUBLISHER_LOGOTYPE      20\ntypedef enum\n{\n    ma_dr_flac_container_native,\n    ma_dr_flac_container_ogg,\n    ma_dr_flac_container_unknown\n} ma_dr_flac_container;\ntypedef enum\n{\n    MA_DR_FLAC_SEEK_SET,\n    MA_DR_FLAC_SEEK_CUR,\n    MA_DR_FLAC_SEEK_END\n} ma_dr_flac_seek_origin;\ntypedef struct\n{\n    ma_uint64 firstPCMFrame;\n    ma_uint64 flacFrameOffset;\n    ma_uint16 pcmFrameCount;\n} ma_dr_flac_seekpoint;\ntypedef struct\n{\n    ma_uint16 minBlockSizeInPCMFrames;\n    ma_uint16 maxBlockSizeInPCMFrames;\n    ma_uint32 minFrameSizeInPCMFrames;\n    ma_uint32 maxFrameSizeInPCMFrames;\n    ma_uint32 sampleRate;\n    ma_uint8  channels;\n    ma_uint8  bitsPerSample;\n    ma_uint64 totalPCMFrameCount;\n    ma_uint8  md5[16];\n} ma_dr_flac_streaminfo;\ntypedef struct\n{\n    ma_uint32 type;\n    ma_uint32 rawDataSize;\n    ma_uint64 rawDataOffset;\n    const void* pRawData;\n    union\n    {\n        ma_dr_flac_streaminfo streaminfo;\n        struct\n        {\n            int unused;\n        } padding;\n        struct\n        {\n            ma_uint32 id;\n            const void* pData;\n            ma_uint32 dataSize;\n        } application;\n        struct\n        {\n            ma_uint32 seekpointCount;\n            const ma_dr_flac_seekpoint* pSeekpoints;\n        } seektable;\n        struct\n        {\n            ma_uint32 vendorLength;\n            const char* vendor;\n            ma_uint32 commentCount;\n            const void* pComments;\n        } vorbis_comment;\n        struct\n        {\n            char catalog[128];\n            ma_uint64 leadInSampleCount;\n            ma_bool32 isCD;\n            ma_uint8 trackCount;\n            const void* pTrackData;\n        } cuesheet;\n        struct\n        {\n            ma_uint32 type;\n            ma_uint32 mimeLength;\n            const char* mime;\n            ma_uint32 descriptionLength;\n            const char* description;\n            ma_uint32 width;\n            ma_uint32 height;\n            ma_uint32 colorDepth;\n            ma_uint32 indexColorCount;\n            ma_uint32 pictureDataSize;\n            ma_uint64 pictureDataOffset;\n            const ma_uint8* pPictureData;\n        } picture;\n    } data;\n} ma_dr_flac_metadata;\ntypedef size_t (* ma_dr_flac_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead);\ntypedef ma_bool32 (* ma_dr_flac_seek_proc)(void* pUserData, int offset, ma_dr_flac_seek_origin origin);\ntypedef ma_bool32 (* ma_dr_flac_tell_proc)(void* pUserData, ma_int64* pCursor);\ntypedef void (* ma_dr_flac_meta_proc)(void* pUserData, ma_dr_flac_metadata* pMetadata);\ntypedef struct\n{\n    const ma_uint8* data;\n    size_t dataSize;\n    size_t currentReadPos;\n} ma_dr_flac__memory_stream;\ntypedef struct\n{\n    ma_dr_flac_read_proc onRead;\n    ma_dr_flac_seek_proc onSeek;\n    ma_dr_flac_tell_proc onTell;\n    void* pUserData;\n    size_t unalignedByteCount;\n    ma_dr_flac_cache_t unalignedCache;\n    ma_uint32 nextL2Line;\n    ma_uint32 consumedBits;\n    ma_dr_flac_cache_t cacheL2[MA_DR_FLAC_BUFFER_SIZE/sizeof(ma_dr_flac_cache_t)];\n    ma_dr_flac_cache_t cache;\n    ma_uint16 crc16;\n    ma_dr_flac_cache_t crc16Cache;\n    ma_uint32 crc16CacheIgnoredBytes;\n} ma_dr_flac_bs;\ntypedef struct\n{\n    ma_uint8 subframeType;\n    ma_uint8 wastedBitsPerSample;\n    ma_uint8 lpcOrder;\n    ma_int32* pSamplesS32;\n} ma_dr_flac_subframe;\ntypedef struct\n{\n    ma_uint64 pcmFrameNumber;\n    ma_uint32 flacFrameNumber;\n    ma_uint32 sampleRate;\n    ma_uint16 blockSizeInPCMFrames;\n    ma_uint8 channelAssignment;\n    ma_uint8 bitsPerSample;\n    ma_uint8 crc8;\n} ma_dr_flac_frame_header;\ntypedef struct\n{\n    ma_dr_flac_frame_header header;\n    ma_uint32 pcmFramesRemaining;\n    ma_dr_flac_subframe subframes[8];\n} ma_dr_flac_frame;\ntypedef struct\n{\n    ma_dr_flac_meta_proc onMeta;\n    void* pUserDataMD;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_uint32 sampleRate;\n    ma_uint8 channels;\n    ma_uint8 bitsPerSample;\n    ma_uint16 maxBlockSizeInPCMFrames;\n    ma_uint64 totalPCMFrameCount;\n    ma_dr_flac_container container;\n    ma_uint32 seekpointCount;\n    ma_dr_flac_frame currentFLACFrame;\n    ma_uint64 currentPCMFrame;\n    ma_uint64 firstFLACFramePosInBytes;\n    ma_dr_flac__memory_stream memoryStream;\n    ma_int32* pDecodedSamples;\n    ma_dr_flac_seekpoint* pSeekpoints;\n    void* _oggbs;\n    ma_bool32 _noSeekTableSeek    : 1;\n    ma_bool32 _noBinarySearchSeek : 1;\n    ma_bool32 _noBruteForceSeek   : 1;\n    ma_dr_flac_bs bs;\n    ma_uint8 pExtraData[1];\n} ma_dr_flac;\nMA_API ma_dr_flac* ma_dr_flac_open(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_relaxed(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_container container, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_with_metadata(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_with_metadata_relaxed(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, ma_dr_flac_container container, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API void ma_dr_flac_close(ma_dr_flac* pFlac);\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_s32(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int32* pBufferOut);\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_s16(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int16* pBufferOut);\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_f32(ma_dr_flac* pFlac, ma_uint64 framesToRead, float* pBufferOut);\nMA_API ma_bool32 ma_dr_flac_seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex);\n#ifndef MA_DR_FLAC_NO_STDIO\nMA_API ma_dr_flac* ma_dr_flac_open_file(const char* pFileName, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_file_w(const wchar_t* pFileName, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_file_with_metadata(const char* pFileName, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_file_with_metadata_w(const wchar_t* pFileName, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API ma_dr_flac* ma_dr_flac_open_memory(const void* pData, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_dr_flac* ma_dr_flac_open_memory_with_metadata(const void* pData, size_t dataSize, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int32* ma_dr_flac_open_and_read_pcm_frames_s32(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_flac_open_and_read_pcm_frames_s16(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_flac_open_and_read_pcm_frames_f32(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifndef MA_DR_FLAC_NO_STDIO\nMA_API ma_int32* ma_dr_flac_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_flac_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_flac_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API ma_int32* ma_dr_flac_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_flac_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_flac_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API void ma_dr_flac_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks);\ntypedef struct\n{\n    ma_uint32 countRemaining;\n    const char* pRunningData;\n} ma_dr_flac_vorbis_comment_iterator;\nMA_API void ma_dr_flac_init_vorbis_comment_iterator(ma_dr_flac_vorbis_comment_iterator* pIter, ma_uint32 commentCount, const void* pComments);\nMA_API const char* ma_dr_flac_next_vorbis_comment(ma_dr_flac_vorbis_comment_iterator* pIter, ma_uint32* pCommentLengthOut);\ntypedef struct\n{\n    ma_uint32 countRemaining;\n    const char* pRunningData;\n} ma_dr_flac_cuesheet_track_iterator;\ntypedef struct\n{\n    ma_uint64 offset;\n    ma_uint8 index;\n    ma_uint8 reserved[3];\n} ma_dr_flac_cuesheet_track_index;\ntypedef struct\n{\n    ma_uint64 offset;\n    ma_uint8 trackNumber;\n    char ISRC[12];\n    ma_bool8 isAudio;\n    ma_bool8 preEmphasis;\n    ma_uint8 indexCount;\n    const ma_dr_flac_cuesheet_track_index* pIndexPoints;\n} ma_dr_flac_cuesheet_track;\nMA_API void ma_dr_flac_init_cuesheet_track_iterator(ma_dr_flac_cuesheet_track_iterator* pIter, ma_uint32 trackCount, const void* pTrackData);\nMA_API ma_bool32 ma_dr_flac_next_cuesheet_track(ma_dr_flac_cuesheet_track_iterator* pIter, ma_dr_flac_cuesheet_track* pCuesheetTrack);\n#ifdef __cplusplus\n}\n#endif\n#endif\n/* dr_flac_h end */\n#endif  /* MA_NO_FLAC */\n\n#if !defined(MA_NO_MP3) && !defined(MA_NO_DECODING)\n#define MA_HAS_MP3\n\n#ifndef MA_DR_MP3_NO_SIMD\n    #if (defined(MA_NO_NEON) && defined(MA_ARM)) || (defined(MA_NO_SSE2) && (defined(MA_X86) || defined(MA_X64)))\n    #define MA_DR_MP3_NO_SIMD\n    #endif\n#endif\n\n/* dr_mp3_h begin */\n#ifndef ma_dr_mp3_h\n#define ma_dr_mp3_h\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n#define MA_DR_MP3_STRINGIFY(x)      #x\n#define MA_DR_MP3_XSTRINGIFY(x)     MA_DR_MP3_STRINGIFY(x)\n#define MA_DR_MP3_VERSION_MAJOR     0\n#define MA_DR_MP3_VERSION_MINOR     7\n#define MA_DR_MP3_VERSION_REVISION  3\n#define MA_DR_MP3_VERSION_STRING    MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_MAJOR) \".\" MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_MINOR) \".\" MA_DR_MP3_XSTRINGIFY(MA_DR_MP3_VERSION_REVISION)\n#include <stddef.h>\n#define MA_DR_MP3_MAX_PCM_FRAMES_PER_MP3_FRAME  1152\n#define MA_DR_MP3_MAX_SAMPLES_PER_FRAME         (MA_DR_MP3_MAX_PCM_FRAMES_PER_MP3_FRAME*2)\nMA_API void ma_dr_mp3_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision);\nMA_API const char* ma_dr_mp3_version_string(void);\n#define MA_DR_MP3_MAX_BITRESERVOIR_BYTES      511\n#define MA_DR_MP3_MAX_FREE_FORMAT_FRAME_SIZE  2304\n#define MA_DR_MP3_MAX_L3_FRAME_PAYLOAD_BYTES  MA_DR_MP3_MAX_FREE_FORMAT_FRAME_SIZE\ntypedef struct\n{\n    int frame_bytes, channels, sample_rate, layer, bitrate_kbps;\n} ma_dr_mp3dec_frame_info;\ntypedef struct\n{\n    const ma_uint8 *buf;\n    int pos, limit;\n} ma_dr_mp3_bs;\ntypedef struct\n{\n    const ma_uint8 *sfbtab;\n    ma_uint16 part_23_length, big_values, scalefac_compress;\n    ma_uint8 global_gain, block_type, mixed_block_flag, n_long_sfb, n_short_sfb;\n    ma_uint8 table_select[3], region_count[3], subblock_gain[3];\n    ma_uint8 preflag, scalefac_scale, count1_table, scfsi;\n} ma_dr_mp3_L3_gr_info;\ntypedef struct\n{\n    ma_dr_mp3_bs bs;\n    ma_uint8 maindata[MA_DR_MP3_MAX_BITRESERVOIR_BYTES + MA_DR_MP3_MAX_L3_FRAME_PAYLOAD_BYTES];\n    ma_dr_mp3_L3_gr_info gr_info[4];\n    float grbuf[2][576], scf[40], syn[18 + 15][2*32];\n    ma_uint8 ist_pos[2][39];\n} ma_dr_mp3dec_scratch;\ntypedef struct\n{\n    float mdct_overlap[2][9*32], qmf_state[15*2*32];\n    int reserv, free_format_bytes;\n    ma_uint8 header[4], reserv_buf[511];\n    ma_dr_mp3dec_scratch scratch;\n} ma_dr_mp3dec;\nMA_API void ma_dr_mp3dec_init(ma_dr_mp3dec *dec);\nMA_API int ma_dr_mp3dec_decode_frame(ma_dr_mp3dec *dec, const ma_uint8 *mp3, int mp3_bytes, void *pcm, ma_dr_mp3dec_frame_info *info);\nMA_API void ma_dr_mp3dec_f32_to_s16(const float *in, ma_int16 *out, size_t num_samples);\ntypedef enum\n{\n    MA_DR_MP3_SEEK_SET,\n    MA_DR_MP3_SEEK_CUR,\n    MA_DR_MP3_SEEK_END\n} ma_dr_mp3_seek_origin;\ntypedef struct\n{\n    ma_uint64 seekPosInBytes;\n    ma_uint64 pcmFrameIndex;\n    ma_uint16 mp3FramesToDiscard;\n    ma_uint16 pcmFramesToDiscard;\n} ma_dr_mp3_seek_point;\ntypedef enum\n{\n    MA_DR_MP3_METADATA_TYPE_ID3V1,\n    MA_DR_MP3_METADATA_TYPE_ID3V2,\n    MA_DR_MP3_METADATA_TYPE_APE,\n    MA_DR_MP3_METADATA_TYPE_XING,\n    MA_DR_MP3_METADATA_TYPE_VBRI\n} ma_dr_mp3_metadata_type;\ntypedef struct\n{\n    ma_dr_mp3_metadata_type type;\n    const void* pRawData;\n    size_t rawDataSize;\n} ma_dr_mp3_metadata;\ntypedef size_t (* ma_dr_mp3_read_proc)(void* pUserData, void* pBufferOut, size_t bytesToRead);\ntypedef ma_bool32 (* ma_dr_mp3_seek_proc)(void* pUserData, int offset, ma_dr_mp3_seek_origin origin);\ntypedef ma_bool32 (* ma_dr_mp3_tell_proc)(void* pUserData, ma_int64* pCursor);\ntypedef void (* ma_dr_mp3_meta_proc)(void* pUserData, const ma_dr_mp3_metadata* pMetadata);\ntypedef struct\n{\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n} ma_dr_mp3_config;\ntypedef struct\n{\n    ma_dr_mp3dec decoder;\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_dr_mp3_read_proc onRead;\n    ma_dr_mp3_seek_proc onSeek;\n    ma_dr_mp3_meta_proc onMeta;\n    void* pUserData;\n    void* pUserDataMeta;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_uint32 mp3FrameChannels;\n    ma_uint32 mp3FrameSampleRate;\n    ma_uint32 pcmFramesConsumedInMP3Frame;\n    ma_uint32 pcmFramesRemainingInMP3Frame;\n    ma_uint8 pcmFrames[sizeof(float)*MA_DR_MP3_MAX_SAMPLES_PER_FRAME];\n    ma_uint64 currentPCMFrame;\n    ma_uint64 streamCursor;\n    ma_uint64 streamLength;\n    ma_uint64 streamStartOffset;\n    ma_dr_mp3_seek_point* pSeekPoints;\n    ma_uint32 seekPointCount;\n    ma_uint32 delayInPCMFrames;\n    ma_uint32 paddingInPCMFrames;\n    ma_uint64 totalPCMFrameCount;\n    ma_bool32 isVBR;\n    ma_bool32 isCBR;\n    size_t dataSize;\n    size_t dataCapacity;\n    size_t dataConsumed;\n    ma_uint8* pData;\n    ma_bool32 atEnd;\n    struct\n    {\n        const ma_uint8* pData;\n        size_t dataSize;\n        size_t currentReadPos;\n    } memory;\n} ma_dr_mp3;\nMA_API ma_bool32 ma_dr_mp3_init(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, ma_dr_mp3_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_mp3_init_memory_with_metadata(ma_dr_mp3* pMP3, const void* pData, size_t dataSize, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_mp3_init_memory(ma_dr_mp3* pMP3, const void* pData, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifndef MA_DR_MP3_NO_STDIO\nMA_API ma_bool32 ma_dr_mp3_init_file_with_metadata(ma_dr_mp3* pMP3, const char* pFilePath, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_mp3_init_file_with_metadata_w(ma_dr_mp3* pMP3, const wchar_t* pFilePath, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_mp3_init_file(ma_dr_mp3* pMP3, const char* pFilePath, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_bool32 ma_dr_mp3_init_file_w(ma_dr_mp3* pMP3, const wchar_t* pFilePath, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API void ma_dr_mp3_uninit(ma_dr_mp3* pMP3);\nMA_API ma_uint64 ma_dr_mp3_read_pcm_frames_f32(ma_dr_mp3* pMP3, ma_uint64 framesToRead, float* pBufferOut);\nMA_API ma_uint64 ma_dr_mp3_read_pcm_frames_s16(ma_dr_mp3* pMP3, ma_uint64 framesToRead, ma_int16* pBufferOut);\nMA_API ma_bool32 ma_dr_mp3_seek_to_pcm_frame(ma_dr_mp3* pMP3, ma_uint64 frameIndex);\nMA_API ma_uint64 ma_dr_mp3_get_pcm_frame_count(ma_dr_mp3* pMP3);\nMA_API ma_uint64 ma_dr_mp3_get_mp3_frame_count(ma_dr_mp3* pMP3);\nMA_API ma_bool32 ma_dr_mp3_get_mp3_and_pcm_frame_count(ma_dr_mp3* pMP3, ma_uint64* pMP3FrameCount, ma_uint64* pPCMFrameCount);\nMA_API ma_bool32 ma_dr_mp3_calculate_seek_points(ma_dr_mp3* pMP3, ma_uint32* pSeekPointCount, ma_dr_mp3_seek_point* pSeekPoints);\nMA_API ma_bool32 ma_dr_mp3_bind_seek_table(ma_dr_mp3* pMP3, ma_uint32 seekPointCount, ma_dr_mp3_seek_point* pSeekPoints);\nMA_API float* ma_dr_mp3_open_and_read_pcm_frames_f32(ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, void* pUserData, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_mp3_open_and_read_pcm_frames_s16(ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, void* pUserData, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API float* ma_dr_mp3_open_memory_and_read_pcm_frames_f32(const void* pData, size_t dataSize, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_mp3_open_memory_and_read_pcm_frames_s16(const void* pData, size_t dataSize, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifndef MA_DR_MP3_NO_STDIO\nMA_API float* ma_dr_mp3_open_file_and_read_pcm_frames_f32(const char* filePath, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_int16* ma_dr_mp3_open_file_and_read_pcm_frames_s16(const char* filePath, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks);\n#endif\nMA_API void* ma_dr_mp3_malloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API void ma_dr_mp3_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks);\n#ifdef __cplusplus\n}\n#endif\n#endif\n/* dr_mp3_h end */\n#endif  /* MA_NO_MP3 */\n\n\n/**************************************************************************************************************************************************************\n\nDecoding\n\n**************************************************************************************************************************************************************/\n#ifndef MA_NO_DECODING\n\nstatic ma_result ma_decoder_read_bytes(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    return pDecoder->onRead(pDecoder, pBufferOut, bytesToRead, pBytesRead);\n}\n\nstatic ma_result ma_decoder_seek_bytes(ma_decoder* pDecoder, ma_int64 byteOffset, ma_seek_origin origin)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    return pDecoder->onSeek(pDecoder, byteOffset, origin);\n}\n\nstatic ma_result ma_decoder_tell_bytes(ma_decoder* pDecoder, ma_int64* pCursor)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pDecoder->onTell == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    return pDecoder->onTell(pDecoder, pCursor);\n}\n\n\nMA_API ma_decoding_backend_config ma_decoding_backend_config_init(ma_format preferredFormat, ma_uint32 seekPointCount)\n{\n    ma_decoding_backend_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.preferredFormat = preferredFormat;\n    config.seekPointCount  = seekPointCount;\n\n    return config;\n}\n\n\nMA_API ma_decoder_config ma_decoder_config_init(ma_format outputFormat, ma_uint32 outputChannels, ma_uint32 outputSampleRate)\n{\n    ma_decoder_config config;\n    MA_ZERO_OBJECT(&config);\n    config.format         = outputFormat;\n    config.channels       = outputChannels;\n    config.sampleRate     = outputSampleRate;\n    config.resampling     = ma_resampler_config_init(ma_format_unknown, 0, 0, 0, ma_resample_algorithm_linear); /* Format/channels/rate doesn't matter here. */\n    config.encodingFormat = ma_encoding_format_unknown;\n\n    /* Note that we are intentionally leaving the channel map empty here which will cause the default channel map to be used. */\n\n    return config;\n}\n\nMA_API ma_decoder_config ma_decoder_config_init_default(void)\n{\n    return ma_decoder_config_init(ma_format_unknown, 0, 0);\n}\n\nMA_API ma_decoder_config ma_decoder_config_init_copy(const ma_decoder_config* pConfig)\n{\n    ma_decoder_config config;\n    if (pConfig != NULL) {\n        config = *pConfig;\n    } else {\n        MA_ZERO_OBJECT(&config);\n    }\n\n    return config;\n}\n\nstatic ma_result ma_decoder__init_data_converter(ma_decoder* pDecoder, const ma_decoder_config* pConfig)\n{\n    ma_result result;\n    ma_data_converter_config converterConfig;\n    ma_format internalFormat;\n    ma_uint32 internalChannels;\n    ma_uint32 internalSampleRate;\n    ma_channel internalChannelMap[MA_MAX_CHANNELS];\n\n    MA_ASSERT(pDecoder != NULL);\n    MA_ASSERT(pConfig  != NULL);\n\n    result = ma_data_source_get_data_format(pDecoder->pBackend, &internalFormat, &internalChannels, &internalSampleRate, internalChannelMap, ma_countof(internalChannelMap));\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the internal data format. */\n    }\n\n\n    /* Make sure we're not asking for too many channels. */\n    if (pConfig->channels > MA_MAX_CHANNELS) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The internal channels should have already been validated at a higher level, but we'll do it again explicitly here for safety. */\n    if (internalChannels > MA_MAX_CHANNELS) {\n        return MA_INVALID_ARGS;\n    }\n\n\n    /* Output format. */\n    if (pConfig->format == ma_format_unknown) {\n        pDecoder->outputFormat = internalFormat;\n    } else {\n        pDecoder->outputFormat = pConfig->format;\n    }\n\n    if (pConfig->channels == 0) {\n        pDecoder->outputChannels = internalChannels;\n    } else {\n        pDecoder->outputChannels = pConfig->channels;\n    }\n\n    if (pConfig->sampleRate == 0) {\n        pDecoder->outputSampleRate = internalSampleRate;\n    } else {\n        pDecoder->outputSampleRate = pConfig->sampleRate;\n    }\n\n    converterConfig = ma_data_converter_config_init(\n        internalFormat,     pDecoder->outputFormat,\n        internalChannels,   pDecoder->outputChannels,\n        internalSampleRate, pDecoder->outputSampleRate\n    );\n    converterConfig.pChannelMapIn          = internalChannelMap;\n    converterConfig.pChannelMapOut         = pConfig->pChannelMap;\n    converterConfig.channelMixMode         = pConfig->channelMixMode;\n    converterConfig.ditherMode             = pConfig->ditherMode;\n    converterConfig.allowDynamicSampleRate = MA_FALSE;   /* Never allow dynamic sample rate conversion. Setting this to true will disable passthrough optimizations. */\n    converterConfig.resampling             = pConfig->resampling;\n\n    result = ma_data_converter_init(&converterConfig, &pDecoder->allocationCallbacks, &pDecoder->converter);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /*\n    Now that we have the decoder we need to determine whether or not we need a heap-allocated cache. We'll\n    need this if the data converter does not support calculation of the required input frame count. To\n    determine support for this we'll just run a test.\n    */\n    {\n        ma_uint64 unused;\n\n        result = ma_data_converter_get_required_input_frame_count(&pDecoder->converter, 1, &unused);\n        if (result != MA_SUCCESS) {\n            /*\n            We were unable to calculate the required input frame count which means we'll need to use\n            a heap-allocated cache.\n            */\n            ma_uint64 inputCacheCapSizeInBytes;\n\n            pDecoder->inputCacheCap = MA_DATA_CONVERTER_STACK_BUFFER_SIZE / ma_get_bytes_per_frame(internalFormat, internalChannels);\n\n            /* Not strictly necessary, but keeping here for safety in case we change the default value of pDecoder->inputCacheCap. */\n            inputCacheCapSizeInBytes = pDecoder->inputCacheCap * ma_get_bytes_per_frame(internalFormat, internalChannels);\n            if (inputCacheCapSizeInBytes > MA_SIZE_MAX) {\n                ma_data_converter_uninit(&pDecoder->converter, &pDecoder->allocationCallbacks);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            pDecoder->pInputCache = ma_malloc((size_t)inputCacheCapSizeInBytes, &pDecoder->allocationCallbacks);    /* Safe cast to size_t. */\n            if (pDecoder->pInputCache == NULL) {\n                ma_data_converter_uninit(&pDecoder->converter, &pDecoder->allocationCallbacks);\n                return MA_OUT_OF_MEMORY;\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\n\nstatic ma_result ma_decoder_internal_on_read__custom(void* pUserData, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead)\n{\n    ma_decoder* pDecoder = (ma_decoder*)pUserData;\n    MA_ASSERT(pDecoder != NULL);\n\n    return ma_decoder_read_bytes(pDecoder, pBufferOut, bytesToRead, pBytesRead);\n}\n\nstatic ma_result ma_decoder_internal_on_seek__custom(void* pUserData, ma_int64 offset, ma_seek_origin origin)\n{\n    ma_decoder* pDecoder = (ma_decoder*)pUserData;\n    MA_ASSERT(pDecoder != NULL);\n\n    return ma_decoder_seek_bytes(pDecoder, offset, origin);\n}\n\nstatic ma_result ma_decoder_internal_on_tell__custom(void* pUserData, ma_int64* pCursor)\n{\n    ma_decoder* pDecoder = (ma_decoder*)pUserData;\n    MA_ASSERT(pDecoder != NULL);\n\n    return ma_decoder_tell_bytes(pDecoder, pCursor);\n}\n\n\nstatic ma_result ma_decoder_init_from_vtable__internal(const ma_decoding_backend_vtable* pVTable, void* pVTableUserData, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoding_backend_config backendConfig;\n    ma_data_source* pBackend;\n\n    MA_ASSERT(pVTable  != NULL);\n    MA_ASSERT(pConfig  != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pVTable->onInit == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    backendConfig = ma_decoding_backend_config_init(pConfig->format, pConfig->seekPointCount);\n\n    result = pVTable->onInit(pVTableUserData, ma_decoder_internal_on_read__custom, ma_decoder_internal_on_seek__custom, ma_decoder_internal_on_tell__custom, pDecoder, &backendConfig, &pDecoder->allocationCallbacks, &pBackend);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the backend from this vtable. */\n    }\n\n    /* Getting here means we were able to initialize the backend so we can now initialize the decoder. */\n    pDecoder->pBackend         = pBackend;\n    pDecoder->pBackendVTable   = pVTable;\n    pDecoder->pBackendUserData = pConfig->pCustomBackendUserData;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder_init_from_file__internal(const ma_decoding_backend_vtable* pVTable, void* pVTableUserData, const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoding_backend_config backendConfig;\n    ma_data_source* pBackend;\n\n    MA_ASSERT(pVTable  != NULL);\n    MA_ASSERT(pConfig  != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pVTable->onInitFile == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    backendConfig = ma_decoding_backend_config_init(pConfig->format, pConfig->seekPointCount);\n\n    result = pVTable->onInitFile(pVTableUserData, pFilePath, &backendConfig, &pDecoder->allocationCallbacks, &pBackend);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the backend from this vtable. */\n    }\n\n    /* Getting here means we were able to initialize the backend so we can now initialize the decoder. */\n    pDecoder->pBackend         = pBackend;\n    pDecoder->pBackendVTable   = pVTable;\n    pDecoder->pBackendUserData = pConfig->pCustomBackendUserData;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder_init_from_file_w__internal(const ma_decoding_backend_vtable* pVTable, void* pVTableUserData, const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoding_backend_config backendConfig;\n    ma_data_source* pBackend;\n\n    MA_ASSERT(pVTable  != NULL);\n    MA_ASSERT(pConfig  != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pVTable->onInitFileW == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    backendConfig = ma_decoding_backend_config_init(pConfig->format, pConfig->seekPointCount);\n\n    result = pVTable->onInitFileW(pVTableUserData, pFilePath, &backendConfig, &pDecoder->allocationCallbacks, &pBackend);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the backend from this vtable. */\n    }\n\n    /* Getting here means we were able to initialize the backend so we can now initialize the decoder. */\n    pDecoder->pBackend         = pBackend;\n    pDecoder->pBackendVTable   = pVTable;\n    pDecoder->pBackendUserData = pConfig->pCustomBackendUserData;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder_init_from_memory__internal(const ma_decoding_backend_vtable* pVTable, void* pVTableUserData, const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoding_backend_config backendConfig;\n    ma_data_source* pBackend;\n\n    MA_ASSERT(pVTable  != NULL);\n    MA_ASSERT(pConfig  != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pVTable->onInitMemory == NULL) {\n        return MA_NOT_IMPLEMENTED;\n    }\n\n    backendConfig = ma_decoding_backend_config_init(pConfig->format, pConfig->seekPointCount);\n\n    result = pVTable->onInitMemory(pVTableUserData, pData, dataSize, &backendConfig, &pDecoder->allocationCallbacks, &pBackend);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the backend from this vtable. */\n    }\n\n    /* Getting here means we were able to initialize the backend so we can now initialize the decoder. */\n    pDecoder->pBackend         = pBackend;\n    pDecoder->pBackendVTable   = pVTable;\n    pDecoder->pBackendUserData = pConfig->pCustomBackendUserData;\n\n    return MA_SUCCESS;\n}\n\n\n\nstatic ma_result ma_decoder_init_custom__internal(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = MA_NO_BACKEND;\n    size_t ivtable;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pConfig->ppCustomBackendVTables == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    /* The order each backend is listed is what defines the priority. */\n    for (ivtable = 0; ivtable < pConfig->customBackendCount; ivtable += 1) {\n        const ma_decoding_backend_vtable* pVTable = pConfig->ppCustomBackendVTables[ivtable];\n        if (pVTable != NULL) {\n            result = ma_decoder_init_from_vtable__internal(pVTable, pConfig->pCustomBackendUserData, pConfig, pDecoder);\n            if (result == MA_SUCCESS) {\n                return MA_SUCCESS;\n            } else {\n                /* Initialization failed. Move on to the next one, but seek back to the start first so the next vtable starts from the first byte of the file. */\n                result = ma_decoder_seek_bytes(pDecoder, 0, ma_seek_origin_start);\n                if (result != MA_SUCCESS) {\n                    return result;  /* Failed to seek back to the start. */\n                }\n            }\n        } else {\n            /* No vtable. */\n        }\n    }\n\n    /* Getting here means we couldn't find a backend. */\n    return MA_NO_BACKEND;\n}\n\nstatic ma_result ma_decoder_init_custom_from_file__internal(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = MA_NO_BACKEND;\n    size_t ivtable;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pConfig->ppCustomBackendVTables == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    /* The order each backend is listed is what defines the priority. */\n    for (ivtable = 0; ivtable < pConfig->customBackendCount; ivtable += 1) {\n        const ma_decoding_backend_vtable* pVTable = pConfig->ppCustomBackendVTables[ivtable];\n        if (pVTable != NULL) {\n            result = ma_decoder_init_from_file__internal(pVTable, pConfig->pCustomBackendUserData, pFilePath, pConfig, pDecoder);\n            if (result == MA_SUCCESS) {\n                return MA_SUCCESS;\n            }\n        } else {\n            /* No vtable. */\n        }\n    }\n\n    /* Getting here means we couldn't find a backend. */\n    return MA_NO_BACKEND;\n}\n\nstatic ma_result ma_decoder_init_custom_from_file_w__internal(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = MA_NO_BACKEND;\n    size_t ivtable;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pConfig->ppCustomBackendVTables == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    /* The order each backend is listed is what defines the priority. */\n    for (ivtable = 0; ivtable < pConfig->customBackendCount; ivtable += 1) {\n        const ma_decoding_backend_vtable* pVTable = pConfig->ppCustomBackendVTables[ivtable];\n        if (pVTable != NULL) {\n            result = ma_decoder_init_from_file_w__internal(pVTable, pConfig->pCustomBackendUserData, pFilePath, pConfig, pDecoder);\n            if (result == MA_SUCCESS) {\n                return MA_SUCCESS;\n            }\n        } else {\n            /* No vtable. */\n        }\n    }\n\n    /* Getting here means we couldn't find a backend. */\n    return MA_NO_BACKEND;\n}\n\nstatic ma_result ma_decoder_init_custom_from_memory__internal(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = MA_NO_BACKEND;\n    size_t ivtable;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pConfig->ppCustomBackendVTables == NULL) {\n        return MA_NO_BACKEND;\n    }\n\n    /* The order each backend is listed is what defines the priority. */\n    for (ivtable = 0; ivtable < pConfig->customBackendCount; ivtable += 1) {\n        const ma_decoding_backend_vtable* pVTable = pConfig->ppCustomBackendVTables[ivtable];\n        if (pVTable != NULL) {\n            result = ma_decoder_init_from_memory__internal(pVTable, pConfig->pCustomBackendUserData, pData, dataSize, pConfig, pDecoder);\n            if (result == MA_SUCCESS) {\n                return MA_SUCCESS;\n            }\n        } else {\n            /* No vtable. */\n        }\n    }\n\n    /* Getting here means we couldn't find a backend. */\n    return MA_NO_BACKEND;\n}\n\n\n/* WAV */\n#ifdef ma_dr_wav_h\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_read_proc onRead;\n    ma_seek_proc onSeek;\n    ma_tell_proc onTell;\n    void* pReadSeekTellUserData;\n    ma_format format;           /* Can be f32, s16 or s32. */\n#if !defined(MA_NO_WAV)\n    ma_dr_wav dr;\n#endif\n} ma_wav;\n\nMA_API ma_result ma_wav_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav);\nMA_API ma_result ma_wav_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav);\nMA_API ma_result ma_wav_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav);\nMA_API ma_result ma_wav_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav);\nMA_API void ma_wav_uninit(ma_wav* pWav, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_wav_read_pcm_frames(ma_wav* pWav, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_wav_seek_to_pcm_frame(ma_wav* pWav, ma_uint64 frameIndex);\nMA_API ma_result ma_wav_get_data_format(ma_wav* pWav, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_wav_get_cursor_in_pcm_frames(ma_wav* pWav, ma_uint64* pCursor);\nMA_API ma_result ma_wav_get_length_in_pcm_frames(ma_wav* pWav, ma_uint64* pLength);\n\n\nstatic ma_result ma_wav_ds_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_wav_read_pcm_frames((ma_wav*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_wav_ds_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_wav_seek_to_pcm_frame((ma_wav*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_wav_ds_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_wav_get_data_format((ma_wav*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_wav_ds_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_wav_get_cursor_in_pcm_frames((ma_wav*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_wav_ds_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_wav_get_length_in_pcm_frames((ma_wav*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_wav_ds_vtable =\n{\n    ma_wav_ds_read,\n    ma_wav_ds_seek,\n    ma_wav_ds_get_data_format,\n    ma_wav_ds_get_cursor,\n    ma_wav_ds_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\n\n#if !defined(MA_NO_WAV)\nstatic size_t ma_wav_dr_callback__read(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    ma_wav* pWav = (ma_wav*)pUserData;\n    ma_result result;\n    size_t bytesRead;\n\n    MA_ASSERT(pWav != NULL);\n\n    result = pWav->onRead(pWav->pReadSeekTellUserData, pBufferOut, bytesToRead, &bytesRead);\n    (void)result;\n\n    return bytesRead;\n}\n\nstatic ma_bool32 ma_wav_dr_callback__seek(void* pUserData, int offset, ma_dr_wav_seek_origin origin)\n{\n    ma_wav* pWav = (ma_wav*)pUserData;\n    ma_result result;\n    ma_seek_origin maSeekOrigin;\n\n    MA_ASSERT(pWav != NULL);\n\n    maSeekOrigin = ma_seek_origin_start;\n    if (origin == MA_DR_WAV_SEEK_CUR) {\n        maSeekOrigin = ma_seek_origin_current;\n    } else if (origin == MA_DR_WAV_SEEK_END) {\n        maSeekOrigin = ma_seek_origin_end;\n    }\n\n    result = pWav->onSeek(pWav->pReadSeekTellUserData, offset, maSeekOrigin);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n\n    return MA_TRUE;\n}\n\nstatic ma_bool32 ma_wav_dr_callback__tell(void* pUserData, ma_int64* pCursor)\n{\n    ma_wav* pWav = (ma_wav*)pUserData;\n    ma_result result;\n\n    MA_ASSERT(pWav != NULL);\n    MA_ASSERT(pCursor != NULL);\n\n    if (pWav->onTell == NULL) {\n        return MA_FALSE;  /* Not implemented. */\n    }\n\n    result = pWav->onTell(pWav->pReadSeekTellUserData, pCursor);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;  /* Failed to tell. */\n    }\n\n    return MA_TRUE;\n}\n#endif\n\nstatic ma_result ma_wav_init_internal(const ma_decoding_backend_config* pConfig, ma_wav* pWav)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pWav);\n    pWav->format = ma_format_unknown;   /* Use closest match to source file by default. */\n\n    if (pConfig != NULL && (pConfig->preferredFormat == ma_format_f32 || pConfig->preferredFormat == ma_format_s16 || pConfig->preferredFormat == ma_format_s32)) {\n        pWav->format = pConfig->preferredFormat;\n    } else {\n        /* Getting here means something other than f32 and s16 was specified. Just leave this unset to use the default format. */\n    }\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_wav_ds_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pWav->ds);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the base data source. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_wav_post_init(ma_wav* pWav)\n{\n    /*\n    If an explicit format was not specified, try picking the closest match based on the internal\n    format. The format needs to be supported by miniaudio.\n    */\n    if (pWav->format == ma_format_unknown) {\n        switch (pWav->dr.translatedFormatTag)\n        {\n            case MA_DR_WAVE_FORMAT_PCM:\n            {\n                if (pWav->dr.bitsPerSample == 8) {\n                    pWav->format = ma_format_u8;\n                } else if (pWav->dr.bitsPerSample == 16) {\n                    pWav->format = ma_format_s16;\n                } else if (pWav->dr.bitsPerSample == 24) {\n                    pWav->format = ma_format_s24;\n                } else if (pWav->dr.bitsPerSample == 32) {\n                    pWav->format = ma_format_s32;\n                }\n            } break;\n\n            case MA_DR_WAVE_FORMAT_IEEE_FLOAT:\n            {\n                if (pWav->dr.bitsPerSample == 32) {\n                    pWav->format = ma_format_f32;\n                }\n            } break;\n\n            default: break;\n        }\n\n        /* Fall back to f32 if we couldn't find anything. */\n        if (pWav->format == ma_format_unknown) {\n            pWav->format =  ma_format_f32;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_wav_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav)\n{\n    ma_result result;\n\n    result = ma_wav_init_internal(pConfig, pWav);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (onRead == NULL || onSeek == NULL) {\n        return MA_INVALID_ARGS; /* onRead and onSeek are mandatory. */\n    }\n\n    pWav->onRead = onRead;\n    pWav->onSeek = onSeek;\n    pWav->onTell = onTell;\n    pWav->pReadSeekTellUserData = pReadSeekTellUserData;\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_bool32 wavResult;\n\n        wavResult = ma_dr_wav_init(&pWav->dr, ma_wav_dr_callback__read, ma_wav_dr_callback__seek, ma_wav_dr_callback__tell, pWav, pAllocationCallbacks);\n        if (wavResult != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_wav_post_init(pWav);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. */\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav)\n{\n    ma_result result;\n\n    result = ma_wav_init_internal(pConfig, pWav);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_bool32 wavResult;\n\n        wavResult = ma_dr_wav_init_file(&pWav->dr, pFilePath, pAllocationCallbacks);\n        if (wavResult != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_wav_post_init(pWav);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav)\n{\n    ma_result result;\n\n    result = ma_wav_init_internal(pConfig, pWav);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_bool32 wavResult;\n\n        wavResult = ma_dr_wav_init_file_w(&pWav->dr, pFilePath, pAllocationCallbacks);\n        if (wavResult != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_wav_post_init(pWav);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_wav* pWav)\n{\n    ma_result result;\n\n    result = ma_wav_init_internal(pConfig, pWav);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_bool32 wavResult;\n\n        wavResult = ma_dr_wav_init_memory(&pWav->dr, pData, dataSize, pAllocationCallbacks);\n        if (wavResult != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_wav_post_init(pWav);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. */\n        (void)pData;\n        (void)dataSize;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API void ma_wav_uninit(ma_wav* pWav, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pWav == NULL) {\n        return;\n    }\n\n    (void)pAllocationCallbacks;\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_dr_wav_uninit(&pWav->dr);\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n    }\n    #endif\n\n    ma_data_source_uninit(&pWav->ds);\n}\n\nMA_API ma_result ma_wav_read_pcm_frames(ma_wav* pWav, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        /* We always use floating point format. */\n        ma_result result = MA_SUCCESS;  /* Must be initialized to MA_SUCCESS. */\n        ma_uint64 totalFramesRead = 0;\n        ma_format format;\n\n        ma_wav_get_data_format(pWav, &format, NULL, NULL, NULL, 0);\n\n        switch (format)\n        {\n            case ma_format_f32:\n            {\n                totalFramesRead = ma_dr_wav_read_pcm_frames_f32(&pWav->dr, frameCount, (float*)pFramesOut);\n            } break;\n\n            case ma_format_s16:\n            {\n                totalFramesRead = ma_dr_wav_read_pcm_frames_s16(&pWav->dr, frameCount, (ma_int16*)pFramesOut);\n            } break;\n\n            case ma_format_s32:\n            {\n                totalFramesRead = ma_dr_wav_read_pcm_frames_s32(&pWav->dr, frameCount, (ma_int32*)pFramesOut);\n            } break;\n\n            /* Fallback to a raw read. */\n            case ma_format_unknown: return MA_INVALID_OPERATION; /* <-- this should never be hit because initialization would just fall back to a supported format. */\n            default:\n            {\n                totalFramesRead = ma_dr_wav_read_pcm_frames(&pWav->dr, frameCount, pFramesOut);\n            } break;\n        }\n\n        /* In the future we'll update ma_dr_wav to return MA_AT_END for us. */\n        if (totalFramesRead == 0) {\n            result = MA_AT_END;\n        }\n\n        if (pFramesRead != NULL) {\n            *pFramesRead = totalFramesRead;\n        }\n\n        if (result == MA_SUCCESS && totalFramesRead == 0) {\n            result  = MA_AT_END;\n        }\n\n        return result;\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)pFramesOut;\n        (void)frameCount;\n        (void)pFramesRead;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_seek_to_pcm_frame(ma_wav* pWav, ma_uint64 frameIndex)\n{\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_bool32 wavResult;\n\n        wavResult = ma_dr_wav_seek_to_pcm_frame(&pWav->dr, frameIndex);\n        if (wavResult != MA_TRUE) {\n            return MA_ERROR;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)frameIndex;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_get_data_format(ma_wav* pWav, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* Defaults for safety. */\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pWav == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pWav->format;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        if (pChannels != NULL) {\n            *pChannels = pWav->dr.channels;\n        }\n\n        if (pSampleRate != NULL) {\n            *pSampleRate = pWav->dr.sampleRate;\n        }\n\n        if (pChannelMap != NULL) {\n            ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pChannelMap, channelMapCap, pWav->dr.channels);\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_get_cursor_in_pcm_frames(ma_wav* pWav, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;   /* Safety. */\n\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_result wavResult = ma_dr_wav_get_cursor_in_pcm_frames(&pWav->dr, pCursor);\n        if (wavResult != MA_SUCCESS) {\n            return (ma_result)wavResult;    /* ma_dr_wav result codes map to miniaudio's. */\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_wav_get_length_in_pcm_frames(ma_wav* pWav, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;   /* Safety. */\n\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_WAV)\n    {\n        ma_result wavResult = ma_dr_wav_get_length_in_pcm_frames(&pWav->dr, pLength);\n        if (wavResult != MA_SUCCESS) {\n            return (ma_result)wavResult;    /* ma_dr_wav result codes map to miniaudio's. */\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* wav is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\n\nstatic ma_result ma_decoding_backend_init__wav(void* pUserData, ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_wav* pWav;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pWav = (ma_wav*)ma_malloc(sizeof(*pWav), pAllocationCallbacks);\n    if (pWav == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_wav_init(onRead, onSeek, onTell, pReadSeekTellUserData, pConfig, pAllocationCallbacks, pWav);\n    if (result != MA_SUCCESS) {\n        ma_free(pWav, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pWav;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file__wav(void* pUserData, const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_wav* pWav;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pWav = (ma_wav*)ma_malloc(sizeof(*pWav), pAllocationCallbacks);\n    if (pWav == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_wav_init_file(pFilePath, pConfig, pAllocationCallbacks, pWav);\n    if (result != MA_SUCCESS) {\n        ma_free(pWav, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pWav;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file_w__wav(void* pUserData, const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_wav* pWav;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pWav = (ma_wav*)ma_malloc(sizeof(*pWav), pAllocationCallbacks);\n    if (pWav == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_wav_init_file_w(pFilePath, pConfig, pAllocationCallbacks, pWav);\n    if (result != MA_SUCCESS) {\n        ma_free(pWav, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pWav;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_memory__wav(void* pUserData, const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_wav* pWav;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pWav = (ma_wav*)ma_malloc(sizeof(*pWav), pAllocationCallbacks);\n    if (pWav == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_wav_init_memory(pData, dataSize, pConfig, pAllocationCallbacks, pWav);\n    if (result != MA_SUCCESS) {\n        ma_free(pWav, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pWav;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_decoding_backend_uninit__wav(void* pUserData, ma_data_source* pBackend, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_wav* pWav = (ma_wav*)pBackend;\n\n    (void)pUserData;\n\n    ma_wav_uninit(pWav, pAllocationCallbacks);\n    ma_free(pWav, pAllocationCallbacks);\n}\n\nstatic ma_decoding_backend_vtable g_ma_decoding_backend_vtable_wav =\n{\n    ma_decoding_backend_init__wav,\n    ma_decoding_backend_init_file__wav,\n    ma_decoding_backend_init_file_w__wav,\n    ma_decoding_backend_init_memory__wav,\n    ma_decoding_backend_uninit__wav\n};\n\nstatic ma_result ma_decoder_init_wav__internal(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_vtable__internal(&g_ma_decoding_backend_vtable_wav, NULL, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_wav_from_file__internal(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file__internal(&g_ma_decoding_backend_vtable_wav, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_wav_from_file_w__internal(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file_w__internal(&g_ma_decoding_backend_vtable_wav, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_wav_from_memory__internal(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_memory__internal(&g_ma_decoding_backend_vtable_wav, NULL, pData, dataSize, pConfig, pDecoder);\n}\n#endif  /* ma_dr_wav_h */\n\n/* FLAC */\n#ifdef ma_dr_flac_h\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_read_proc onRead;\n    ma_seek_proc onSeek;\n    ma_tell_proc onTell;\n    void* pReadSeekTellUserData;\n    ma_format format;           /* Can be f32, s16 or s32. */\n#if !defined(MA_NO_FLAC)\n    ma_dr_flac* dr;\n#endif\n} ma_flac;\n\nMA_API ma_result ma_flac_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac);\nMA_API ma_result ma_flac_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac);\nMA_API ma_result ma_flac_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac);\nMA_API ma_result ma_flac_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac);\nMA_API void ma_flac_uninit(ma_flac* pFlac, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_flac_read_pcm_frames(ma_flac* pFlac, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_flac_seek_to_pcm_frame(ma_flac* pFlac, ma_uint64 frameIndex);\nMA_API ma_result ma_flac_get_data_format(ma_flac* pFlac, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_flac_get_cursor_in_pcm_frames(ma_flac* pFlac, ma_uint64* pCursor);\nMA_API ma_result ma_flac_get_length_in_pcm_frames(ma_flac* pFlac, ma_uint64* pLength);\n\n\nstatic ma_result ma_flac_ds_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_flac_read_pcm_frames((ma_flac*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_flac_ds_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_flac_seek_to_pcm_frame((ma_flac*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_flac_ds_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_flac_get_data_format((ma_flac*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_flac_ds_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_flac_get_cursor_in_pcm_frames((ma_flac*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_flac_ds_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_flac_get_length_in_pcm_frames((ma_flac*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_flac_ds_vtable =\n{\n    ma_flac_ds_read,\n    ma_flac_ds_seek,\n    ma_flac_ds_get_data_format,\n    ma_flac_ds_get_cursor,\n    ma_flac_ds_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\n\n#if !defined(MA_NO_FLAC)\nstatic size_t ma_flac_dr_callback__read(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    ma_flac* pFlac = (ma_flac*)pUserData;\n    ma_result result;\n    size_t bytesRead;\n\n    MA_ASSERT(pFlac != NULL);\n\n    result = pFlac->onRead(pFlac->pReadSeekTellUserData, pBufferOut, bytesToRead, &bytesRead);\n    (void)result;\n\n    return bytesRead;\n}\n\nstatic ma_bool32 ma_flac_dr_callback__seek(void* pUserData, int offset, ma_dr_flac_seek_origin origin)\n{\n    ma_flac* pFlac = (ma_flac*)pUserData;\n    ma_result result;\n    ma_seek_origin maSeekOrigin;\n\n    MA_ASSERT(pFlac != NULL);\n\n    maSeekOrigin = ma_seek_origin_start;\n    if (origin == MA_DR_FLAC_SEEK_CUR) {\n        maSeekOrigin = ma_seek_origin_current;\n    } else if (origin == MA_DR_FLAC_SEEK_END) {\n        maSeekOrigin = ma_seek_origin_end;\n    }\n\n    result = pFlac->onSeek(pFlac->pReadSeekTellUserData, offset, maSeekOrigin);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n\n    return MA_TRUE;\n}\n\nstatic ma_bool32 ma_flac_dr_callback__tell(void* pUserData, ma_int64* pCursor)\n{\n    ma_flac* pFlac = (ma_flac*)pUserData;\n    ma_result result;\n\n    MA_ASSERT(pFlac != NULL);\n    MA_ASSERT(pCursor != NULL);\n\n    if (pFlac->onTell == NULL) {\n        return MA_FALSE;  /* Not implemented. */\n    }\n\n    result = pFlac->onTell(pFlac->pReadSeekTellUserData, pCursor);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;  /* Failed to tell. */\n    }\n\n    return MA_TRUE;\n}\n#endif\n\nstatic ma_result ma_flac_init_internal(const ma_decoding_backend_config* pConfig, ma_flac* pFlac)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pFlac == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pFlac);\n    pFlac->format = ma_format_f32;    /* f32 by default. */\n\n    if (pConfig != NULL && (pConfig->preferredFormat == ma_format_f32 || pConfig->preferredFormat == ma_format_s16 || pConfig->preferredFormat == ma_format_s32)) {\n        pFlac->format = pConfig->preferredFormat;\n    } else {\n        /* Getting here means something other than f32 and s16 was specified. Just leave this unset to use the default format. */\n    }\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_flac_ds_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pFlac->ds);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the base data source. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_flac_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac)\n{\n    ma_result result;\n\n    result = ma_flac_init_internal(pConfig, pFlac);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (onRead == NULL || onSeek == NULL) {\n        return MA_INVALID_ARGS; /* onRead and onSeek are mandatory. */\n    }\n\n    pFlac->onRead = onRead;\n    pFlac->onSeek = onSeek;\n    pFlac->onTell = onTell;\n    pFlac->pReadSeekTellUserData = pReadSeekTellUserData;\n\n    #if !defined(MA_NO_FLAC)\n    {\n        pFlac->dr = ma_dr_flac_open(ma_flac_dr_callback__read, ma_flac_dr_callback__seek, ma_flac_dr_callback__tell, pFlac, pAllocationCallbacks);\n        if (pFlac->dr == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. */\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac)\n{\n    ma_result result;\n\n    result = ma_flac_init_internal(pConfig, pFlac);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        pFlac->dr = ma_dr_flac_open_file(pFilePath, pAllocationCallbacks);\n        if (pFlac->dr == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac)\n{\n    ma_result result;\n\n    result = ma_flac_init_internal(pConfig, pFlac);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        pFlac->dr = ma_dr_flac_open_file_w(pFilePath, pAllocationCallbacks);\n        if (pFlac->dr == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_flac* pFlac)\n{\n    ma_result result;\n\n    result = ma_flac_init_internal(pConfig, pFlac);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        pFlac->dr = ma_dr_flac_open_memory(pData, dataSize, pAllocationCallbacks);\n        if (pFlac->dr == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. */\n        (void)pData;\n        (void)dataSize;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API void ma_flac_uninit(ma_flac* pFlac, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFlac == NULL) {\n        return;\n    }\n\n    (void)pAllocationCallbacks;\n\n    #if !defined(MA_NO_FLAC)\n    {\n        ma_dr_flac_close(pFlac->dr);\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n    }\n    #endif\n\n    ma_data_source_uninit(&pFlac->ds);\n}\n\nMA_API ma_result ma_flac_read_pcm_frames(ma_flac* pFlac, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFlac == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        /* We always use floating point format. */\n        ma_result result = MA_SUCCESS;  /* Must be initialized to MA_SUCCESS. */\n        ma_uint64 totalFramesRead = 0;\n        ma_format format;\n\n        ma_flac_get_data_format(pFlac, &format, NULL, NULL, NULL, 0);\n\n        switch (format)\n        {\n            case ma_format_f32:\n            {\n                totalFramesRead = ma_dr_flac_read_pcm_frames_f32(pFlac->dr, frameCount, (float*)pFramesOut);\n            } break;\n\n            case ma_format_s16:\n            {\n                totalFramesRead = ma_dr_flac_read_pcm_frames_s16(pFlac->dr, frameCount, (ma_int16*)pFramesOut);\n            } break;\n\n            case ma_format_s32:\n            {\n                totalFramesRead = ma_dr_flac_read_pcm_frames_s32(pFlac->dr, frameCount, (ma_int32*)pFramesOut);\n            } break;\n\n            case ma_format_u8:\n            case ma_format_s24:\n            case ma_format_unknown:\n            default:\n            {\n                return MA_INVALID_OPERATION;\n            };\n        }\n\n        /* In the future we'll update ma_dr_flac to return MA_AT_END for us. */\n        if (totalFramesRead == 0) {\n            result = MA_AT_END;\n        }\n\n        if (pFramesRead != NULL) {\n            *pFramesRead = totalFramesRead;\n        }\n\n        if (result == MA_SUCCESS && totalFramesRead == 0) {\n            result  = MA_AT_END;\n        }\n\n        return result;\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)pFramesOut;\n        (void)frameCount;\n        (void)pFramesRead;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_seek_to_pcm_frame(ma_flac* pFlac, ma_uint64 frameIndex)\n{\n    if (pFlac == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        ma_bool32 flacResult;\n\n        flacResult = ma_dr_flac_seek_to_pcm_frame(pFlac->dr, frameIndex);\n        if (flacResult != MA_TRUE) {\n            return MA_ERROR;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)frameIndex;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_get_data_format(ma_flac* pFlac, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* Defaults for safety. */\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pFlac == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pFlac->format;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        if (pChannels != NULL) {\n            *pChannels = pFlac->dr->channels;\n        }\n\n        if (pSampleRate != NULL) {\n            *pSampleRate = pFlac->dr->sampleRate;\n        }\n\n        if (pChannelMap != NULL) {\n            ma_channel_map_init_standard(ma_standard_channel_map_microsoft, pChannelMap, channelMapCap, pFlac->dr->channels);\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_get_cursor_in_pcm_frames(ma_flac* pFlac, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;   /* Safety. */\n\n    if (pFlac == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        *pCursor = pFlac->dr->currentPCMFrame;\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_flac_get_length_in_pcm_frames(ma_flac* pFlac, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;   /* Safety. */\n\n    if (pFlac == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_FLAC)\n    {\n        *pLength = pFlac->dr->totalPCMFrameCount;\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* flac is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\n\nstatic ma_result ma_decoding_backend_init__flac(void* pUserData, ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_flac* pFlac;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pFlac = (ma_flac*)ma_malloc(sizeof(*pFlac), pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_flac_init(onRead, onSeek, onTell, pReadSeekTellUserData, pConfig, pAllocationCallbacks, pFlac);\n    if (result != MA_SUCCESS) {\n        ma_free(pFlac, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pFlac;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file__flac(void* pUserData, const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_flac* pFlac;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pFlac = (ma_flac*)ma_malloc(sizeof(*pFlac), pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_flac_init_file(pFilePath, pConfig, pAllocationCallbacks, pFlac);\n    if (result != MA_SUCCESS) {\n        ma_free(pFlac, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pFlac;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file_w__flac(void* pUserData, const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_flac* pFlac;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pFlac = (ma_flac*)ma_malloc(sizeof(*pFlac), pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_flac_init_file_w(pFilePath, pConfig, pAllocationCallbacks, pFlac);\n    if (result != MA_SUCCESS) {\n        ma_free(pFlac, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pFlac;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_memory__flac(void* pUserData, const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_flac* pFlac;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pFlac = (ma_flac*)ma_malloc(sizeof(*pFlac), pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_flac_init_memory(pData, dataSize, pConfig, pAllocationCallbacks, pFlac);\n    if (result != MA_SUCCESS) {\n        ma_free(pFlac, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pFlac;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_decoding_backend_uninit__flac(void* pUserData, ma_data_source* pBackend, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_flac* pFlac = (ma_flac*)pBackend;\n\n    (void)pUserData;\n\n    ma_flac_uninit(pFlac, pAllocationCallbacks);\n    ma_free(pFlac, pAllocationCallbacks);\n}\n\nstatic ma_decoding_backend_vtable g_ma_decoding_backend_vtable_flac =\n{\n    ma_decoding_backend_init__flac,\n    ma_decoding_backend_init_file__flac,\n    ma_decoding_backend_init_file_w__flac,\n    ma_decoding_backend_init_memory__flac,\n    ma_decoding_backend_uninit__flac\n};\n\nstatic ma_result ma_decoder_init_flac__internal(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_vtable__internal(&g_ma_decoding_backend_vtable_flac, NULL, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_flac_from_file__internal(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file__internal(&g_ma_decoding_backend_vtable_flac, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_flac_from_file_w__internal(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file_w__internal(&g_ma_decoding_backend_vtable_flac, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_flac_from_memory__internal(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_memory__internal(&g_ma_decoding_backend_vtable_flac, NULL, pData, dataSize, pConfig, pDecoder);\n}\n#endif  /* ma_dr_flac_h */\n\n/* MP3 */\n#ifdef ma_dr_mp3_h\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_read_proc onRead;\n    ma_seek_proc onSeek;\n    ma_tell_proc onTell;\n    void* pReadSeekTellUserData;\n    ma_format format;           /* Can be f32 or s16. */\n#if !defined(MA_NO_MP3)\n    ma_dr_mp3 dr;\n    ma_uint32 seekPointCount;\n    ma_dr_mp3_seek_point* pSeekPoints;  /* Only used if seek table generation is used. */\n#endif\n} ma_mp3;\n\nMA_API ma_result ma_mp3_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3);\nMA_API ma_result ma_mp3_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3);\nMA_API ma_result ma_mp3_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3);\nMA_API ma_result ma_mp3_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3);\nMA_API void ma_mp3_uninit(ma_mp3* pMP3, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_mp3_read_pcm_frames(ma_mp3* pMP3, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_mp3_seek_to_pcm_frame(ma_mp3* pMP3, ma_uint64 frameIndex);\nMA_API ma_result ma_mp3_get_data_format(ma_mp3* pMP3, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_mp3_get_cursor_in_pcm_frames(ma_mp3* pMP3, ma_uint64* pCursor);\nMA_API ma_result ma_mp3_get_length_in_pcm_frames(ma_mp3* pMP3, ma_uint64* pLength);\n\n\nstatic ma_result ma_mp3_ds_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_mp3_read_pcm_frames((ma_mp3*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_mp3_ds_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_mp3_seek_to_pcm_frame((ma_mp3*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_mp3_ds_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_mp3_get_data_format((ma_mp3*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_mp3_ds_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_mp3_get_cursor_in_pcm_frames((ma_mp3*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_mp3_ds_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_mp3_get_length_in_pcm_frames((ma_mp3*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_mp3_ds_vtable =\n{\n    ma_mp3_ds_read,\n    ma_mp3_ds_seek,\n    ma_mp3_ds_get_data_format,\n    ma_mp3_ds_get_cursor,\n    ma_mp3_ds_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\n\n#if !defined(MA_NO_MP3)\nstatic size_t ma_mp3_dr_callback__read(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    ma_mp3* pMP3 = (ma_mp3*)pUserData;\n    ma_result result;\n    size_t bytesRead;\n\n    MA_ASSERT(pMP3 != NULL);\n\n    result = pMP3->onRead(pMP3->pReadSeekTellUserData, pBufferOut, bytesToRead, &bytesRead);\n    (void)result;\n\n    return bytesRead;\n}\n\nstatic ma_bool32 ma_mp3_dr_callback__seek(void* pUserData, int offset, ma_dr_mp3_seek_origin origin)\n{\n    ma_mp3* pMP3 = (ma_mp3*)pUserData;\n    ma_result result;\n    ma_seek_origin maSeekOrigin;\n\n    MA_ASSERT(pMP3 != NULL);\n\n    if (origin == MA_DR_MP3_SEEK_SET) {\n        maSeekOrigin = ma_seek_origin_start;\n    } else if (origin == MA_DR_MP3_SEEK_END) {\n        maSeekOrigin = ma_seek_origin_end;\n    } else {\n        maSeekOrigin = ma_seek_origin_current;\n    }\n\n    result = pMP3->onSeek(pMP3->pReadSeekTellUserData, offset, maSeekOrigin);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n\n    return MA_TRUE;\n}\n\nstatic ma_bool32 ma_mp3_dr_callback__tell(void* pUserData, ma_int64* pCursor)\n{\n    ma_mp3* pMP3 = (ma_mp3*)pUserData;\n    ma_result result;\n\n    MA_ASSERT(pMP3 != NULL);\n\n    result = pMP3->onTell(pMP3->pReadSeekTellUserData, pCursor);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n\n    return MA_TRUE;\n}\n#endif\n\nstatic ma_result ma_mp3_init_internal(const ma_decoding_backend_config* pConfig, ma_mp3* pMP3)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pMP3 == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pMP3);\n    pMP3->format = ma_format_f32;    /* f32 by default. */\n\n    if (pConfig != NULL && (pConfig->preferredFormat == ma_format_f32 || pConfig->preferredFormat == ma_format_s16)) {\n        pMP3->format = pConfig->preferredFormat;\n    } else {\n        /* Getting here means something other than f32 and s16 was specified. Just leave this unset to use the default format. */\n    }\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_mp3_ds_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pMP3->ds);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the base data source. */\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_mp3_generate_seek_table(ma_mp3* pMP3, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 mp3Result;\n    ma_uint32 seekPointCount = 0;\n    ma_dr_mp3_seek_point* pSeekPoints = NULL;\n\n    MA_ASSERT(pMP3    != NULL);\n    MA_ASSERT(pConfig != NULL);\n\n    seekPointCount = pConfig->seekPointCount;\n    if (seekPointCount > 0) {\n        pSeekPoints = (ma_dr_mp3_seek_point*)ma_malloc(sizeof(*pMP3->pSeekPoints) * seekPointCount, pAllocationCallbacks);\n        if (pSeekPoints == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n    mp3Result = ma_dr_mp3_calculate_seek_points(&pMP3->dr, &seekPointCount, pSeekPoints);\n    if (mp3Result != MA_TRUE) {\n        ma_free(pSeekPoints, pAllocationCallbacks);\n        return MA_ERROR;\n    }\n\n    mp3Result = ma_dr_mp3_bind_seek_table(&pMP3->dr, seekPointCount, pSeekPoints);\n    if (mp3Result != MA_TRUE) {\n        ma_free(pSeekPoints, pAllocationCallbacks);\n        return MA_ERROR;\n    }\n\n    pMP3->seekPointCount = seekPointCount;\n    pMP3->pSeekPoints    = pSeekPoints;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_mp3_post_init(ma_mp3* pMP3, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_result result;\n\n    result = ma_mp3_generate_seek_table(pMP3, pConfig, pAllocationCallbacks);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_mp3_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3)\n{\n    ma_result result;\n\n    result = ma_mp3_init_internal(pConfig, pMP3);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (onRead == NULL || onSeek == NULL) {\n        return MA_INVALID_ARGS; /* onRead and onSeek are mandatory. */\n    }\n\n    pMP3->onRead = onRead;\n    pMP3->onSeek = onSeek;\n    pMP3->onTell = onTell;\n    pMP3->pReadSeekTellUserData = pReadSeekTellUserData;\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_bool32 mp3Result;\n\n        mp3Result = ma_dr_mp3_init(&pMP3->dr, ma_mp3_dr_callback__read, ma_mp3_dr_callback__seek, ma_mp3_dr_callback__tell, NULL, pMP3, pAllocationCallbacks);\n        if (mp3Result != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_mp3_post_init(pMP3, pConfig, pAllocationCallbacks);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. */\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3)\n{\n    ma_result result;\n\n    result = ma_mp3_init_internal(pConfig, pMP3);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_bool32 mp3Result;\n\n        mp3Result = ma_dr_mp3_init_file(&pMP3->dr, pFilePath, pAllocationCallbacks);\n        if (mp3Result != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_mp3_post_init(pMP3, pConfig, pAllocationCallbacks);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_init_file_w(const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3)\n{\n    ma_result result;\n\n    result = ma_mp3_init_internal(pConfig, pMP3);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_bool32 mp3Result;\n\n        mp3Result = ma_dr_mp3_init_file_w(&pMP3->dr, pFilePath, pAllocationCallbacks);\n        if (mp3Result != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_mp3_post_init(pMP3, pConfig, pAllocationCallbacks);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_mp3* pMP3)\n{\n    ma_result result;\n\n    result = ma_mp3_init_internal(pConfig, pMP3);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_bool32 mp3Result;\n\n        mp3Result = ma_dr_mp3_init_memory(&pMP3->dr, pData, dataSize, pAllocationCallbacks);\n        if (mp3Result != MA_TRUE) {\n            return MA_INVALID_FILE;\n        }\n\n        ma_mp3_post_init(pMP3, pConfig, pAllocationCallbacks);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. */\n        (void)pData;\n        (void)dataSize;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API void ma_mp3_uninit(ma_mp3* pMP3, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pMP3 == NULL) {\n        return;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_dr_mp3_uninit(&pMP3->dr);\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n    }\n    #endif\n\n    /* Seek points need to be freed after the MP3 decoder has been uninitialized to ensure they're no longer being referenced. */\n    ma_free(pMP3->pSeekPoints, pAllocationCallbacks);\n\n    ma_data_source_uninit(&pMP3->ds);\n}\n\nMA_API ma_result ma_mp3_read_pcm_frames(ma_mp3* pMP3, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pMP3 == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        /* We always use floating point format. */\n        ma_result result = MA_SUCCESS;  /* Must be initialized to MA_SUCCESS. */\n        ma_uint64 totalFramesRead = 0;\n        ma_format format;\n\n        ma_mp3_get_data_format(pMP3, &format, NULL, NULL, NULL, 0);\n\n        switch (format)\n        {\n            case ma_format_f32:\n            {\n                totalFramesRead = ma_dr_mp3_read_pcm_frames_f32(&pMP3->dr, frameCount, (float*)pFramesOut);\n            } break;\n\n            case ma_format_s16:\n            {\n                totalFramesRead = ma_dr_mp3_read_pcm_frames_s16(&pMP3->dr, frameCount, (ma_int16*)pFramesOut);\n            } break;\n\n            case ma_format_u8:\n            case ma_format_s24:\n            case ma_format_s32:\n            case ma_format_unknown:\n            default:\n            {\n                return MA_INVALID_OPERATION;\n            };\n        }\n\n        /* In the future we'll update ma_dr_mp3 to return MA_AT_END for us. */\n        if (totalFramesRead == 0) {\n            result = MA_AT_END;\n        }\n\n        if (pFramesRead != NULL) {\n            *pFramesRead = totalFramesRead;\n        }\n\n        return result;\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)pFramesOut;\n        (void)frameCount;\n        (void)pFramesRead;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_seek_to_pcm_frame(ma_mp3* pMP3, ma_uint64 frameIndex)\n{\n    if (pMP3 == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        ma_bool32 mp3Result;\n\n        mp3Result = ma_dr_mp3_seek_to_pcm_frame(&pMP3->dr, frameIndex);\n        if (mp3Result != MA_TRUE) {\n            return MA_ERROR;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)frameIndex;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_get_data_format(ma_mp3* pMP3, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* Defaults for safety. */\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pMP3 == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pMP3->format;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        if (pChannels != NULL) {\n            *pChannels = pMP3->dr.channels;\n        }\n\n        if (pSampleRate != NULL) {\n            *pSampleRate = pMP3->dr.sampleRate;\n        }\n\n        if (pChannelMap != NULL) {\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pMP3->dr.channels);\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_get_cursor_in_pcm_frames(ma_mp3* pMP3, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;   /* Safety. */\n\n    if (pMP3 == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        *pCursor = pMP3->dr.currentPCMFrame;\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_mp3_get_length_in_pcm_frames(ma_mp3* pMP3, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;   /* Safety. */\n\n    if (pMP3 == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_MP3)\n    {\n        *pLength = ma_dr_mp3_get_pcm_frame_count(&pMP3->dr);\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* mp3 is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\n\nstatic ma_result ma_decoding_backend_init__mp3(void* pUserData, ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_mp3* pMP3;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pMP3 = (ma_mp3*)ma_malloc(sizeof(*pMP3), pAllocationCallbacks);\n    if (pMP3 == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_mp3_init(onRead, onSeek, onTell, pReadSeekTellUserData, pConfig, pAllocationCallbacks, pMP3);\n    if (result != MA_SUCCESS) {\n        ma_free(pMP3, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pMP3;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file__mp3(void* pUserData, const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_mp3* pMP3;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pMP3 = (ma_mp3*)ma_malloc(sizeof(*pMP3), pAllocationCallbacks);\n    if (pMP3 == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_mp3_init_file(pFilePath, pConfig, pAllocationCallbacks, pMP3);\n    if (result != MA_SUCCESS) {\n        ma_free(pMP3, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pMP3;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file_w__mp3(void* pUserData, const wchar_t* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_mp3* pMP3;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pMP3 = (ma_mp3*)ma_malloc(sizeof(*pMP3), pAllocationCallbacks);\n    if (pMP3 == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_mp3_init_file_w(pFilePath, pConfig, pAllocationCallbacks, pMP3);\n    if (result != MA_SUCCESS) {\n        ma_free(pMP3, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pMP3;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_memory__mp3(void* pUserData, const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_mp3* pMP3;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pMP3 = (ma_mp3*)ma_malloc(sizeof(*pMP3), pAllocationCallbacks);\n    if (pMP3 == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_mp3_init_memory(pData, dataSize, pConfig, pAllocationCallbacks, pMP3);\n    if (result != MA_SUCCESS) {\n        ma_free(pMP3, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pMP3;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_decoding_backend_uninit__mp3(void* pUserData, ma_data_source* pBackend, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_mp3* pMP3 = (ma_mp3*)pBackend;\n\n    (void)pUserData;\n\n    ma_mp3_uninit(pMP3, pAllocationCallbacks);\n    ma_free(pMP3, pAllocationCallbacks);\n}\n\nstatic ma_decoding_backend_vtable g_ma_decoding_backend_vtable_mp3 =\n{\n    ma_decoding_backend_init__mp3,\n    ma_decoding_backend_init_file__mp3,\n    ma_decoding_backend_init_file_w__mp3,\n    ma_decoding_backend_init_memory__mp3,\n    ma_decoding_backend_uninit__mp3\n};\n\nstatic ma_result ma_decoder_init_mp3__internal(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_vtable__internal(&g_ma_decoding_backend_vtable_mp3, NULL, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_mp3_from_file__internal(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file__internal(&g_ma_decoding_backend_vtable_mp3, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_mp3_from_file_w__internal(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file_w__internal(&g_ma_decoding_backend_vtable_mp3, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_mp3_from_memory__internal(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_memory__internal(&g_ma_decoding_backend_vtable_mp3, NULL, pData, dataSize, pConfig, pDecoder);\n}\n#endif  /* ma_dr_mp3_h */\n\n/* Vorbis */\n#ifdef STB_VORBIS_INCLUDE_STB_VORBIS_H\n#define MA_HAS_VORBIS\n\n/* The size in bytes of each chunk of data to read from the Vorbis stream. */\n#define MA_VORBIS_DATA_CHUNK_SIZE  4096\n\ntypedef struct\n{\n    ma_data_source_base ds;\n    ma_read_proc onRead;\n    ma_seek_proc onSeek;\n    ma_tell_proc onTell;\n    void* pReadSeekTellUserData;\n    ma_allocation_callbacks allocationCallbacks;    /* Store the allocation callbacks within the structure because we may need to dynamically expand a buffer in ma_stbvorbis_read_pcm_frames() when using push mode. */\n    ma_format format;               /* Only f32 is allowed with stb_vorbis. */\n    ma_uint32 channels;\n    ma_uint32 sampleRate;\n    ma_uint64 cursor;\n#if !defined(MA_NO_VORBIS)\n    stb_vorbis* stb;\n    ma_bool32 usingPushMode;\n    struct\n    {\n        ma_uint8* pData;\n        size_t dataSize;\n        size_t dataCapacity;\n        size_t audioStartOffsetInBytes;\n        ma_uint32 framesConsumed;   /* The number of frames consumed in ppPacketData. */\n        ma_uint32 framesRemaining;  /* The number of frames remaining in ppPacketData. */\n        float** ppPacketData;\n    } push;\n#endif\n} ma_stbvorbis;\n\nMA_API ma_result ma_stbvorbis_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis);\nMA_API ma_result ma_stbvorbis_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis);\nMA_API ma_result ma_stbvorbis_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis);\nMA_API void ma_stbvorbis_uninit(ma_stbvorbis* pVorbis, const ma_allocation_callbacks* pAllocationCallbacks);\nMA_API ma_result ma_stbvorbis_read_pcm_frames(ma_stbvorbis* pVorbis, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead);\nMA_API ma_result ma_stbvorbis_seek_to_pcm_frame(ma_stbvorbis* pVorbis, ma_uint64 frameIndex);\nMA_API ma_result ma_stbvorbis_get_data_format(ma_stbvorbis* pVorbis, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap);\nMA_API ma_result ma_stbvorbis_get_cursor_in_pcm_frames(ma_stbvorbis* pVorbis, ma_uint64* pCursor);\nMA_API ma_result ma_stbvorbis_get_length_in_pcm_frames(ma_stbvorbis* pVorbis, ma_uint64* pLength);\n\n\nstatic ma_result ma_stbvorbis_ds_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_stbvorbis_read_pcm_frames((ma_stbvorbis*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_stbvorbis_ds_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_stbvorbis_seek_to_pcm_frame((ma_stbvorbis*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_stbvorbis_ds_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_stbvorbis_get_data_format((ma_stbvorbis*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_stbvorbis_ds_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_stbvorbis_get_cursor_in_pcm_frames((ma_stbvorbis*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_stbvorbis_ds_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_stbvorbis_get_length_in_pcm_frames((ma_stbvorbis*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_stbvorbis_ds_vtable =\n{\n    ma_stbvorbis_ds_read,\n    ma_stbvorbis_ds_seek,\n    ma_stbvorbis_ds_get_data_format,\n    ma_stbvorbis_ds_get_cursor,\n    ma_stbvorbis_ds_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\n\nstatic ma_result ma_stbvorbis_init_internal(const ma_decoding_backend_config* pConfig, ma_stbvorbis* pVorbis)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    (void)pConfig;\n\n    if (pVorbis == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pVorbis);\n    pVorbis->format = ma_format_f32;    /* Only supporting f32. */\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_stbvorbis_ds_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pVorbis->ds);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the base data source. */\n    }\n\n    return MA_SUCCESS;\n}\n\n#if !defined(MA_NO_VORBIS)\nstatic ma_result ma_stbvorbis_post_init(ma_stbvorbis* pVorbis)\n{\n    stb_vorbis_info info;\n\n    MA_ASSERT(pVorbis != NULL);\n\n    info = stb_vorbis_get_info(pVorbis->stb);\n\n    pVorbis->channels   = info.channels;\n    pVorbis->sampleRate = info.sample_rate;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_stbvorbis_init_internal_decoder_push(ma_stbvorbis* pVorbis)\n{\n    ma_result result;\n    stb_vorbis* stb;\n    size_t dataSize = 0;\n    size_t dataCapacity = 0;\n    ma_uint8* pData = NULL; /* <-- Must be initialized to NULL. */\n\n    for (;;) {\n        int vorbisError;\n        int consumedDataSize;   /* <-- Fill by stb_vorbis_open_pushdata(). */\n        size_t bytesRead;\n        ma_uint8* pNewData;\n\n        /* Allocate memory for the new chunk. */\n        dataCapacity += MA_VORBIS_DATA_CHUNK_SIZE;\n        pNewData = (ma_uint8*)ma_realloc(pData, dataCapacity, &pVorbis->allocationCallbacks);\n        if (pNewData == NULL) {\n            ma_free(pData, &pVorbis->allocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        pData = pNewData;\n\n        /* Read in the next chunk. */\n        result = pVorbis->onRead(pVorbis->pReadSeekTellUserData, ma_offset_ptr(pData, dataSize), (dataCapacity - dataSize), &bytesRead);\n        dataSize += bytesRead;\n\n        if (result != MA_SUCCESS) {\n            ma_free(pData, &pVorbis->allocationCallbacks);\n            return result;\n        }\n\n        /* We have a maximum of 31 bits with stb_vorbis. */\n        if (dataSize > INT_MAX) {\n            ma_free(pData, &pVorbis->allocationCallbacks);\n            return MA_TOO_BIG;\n        }\n\n        stb = stb_vorbis_open_pushdata(pData, (int)dataSize, &consumedDataSize, &vorbisError, NULL);\n        if (stb != NULL) {\n            /*\n            Successfully opened the Vorbis decoder. We might have some leftover unprocessed\n            data so we'll need to move that down to the front.\n            */\n            dataSize -= (size_t)consumedDataSize;   /* Consume the data. */\n            MA_MOVE_MEMORY(pData, ma_offset_ptr(pData, consumedDataSize), dataSize);\n\n            /*\n            We need to track the start point so we can seek back to the start of the audio\n            data when seeking.\n            */\n            pVorbis->push.audioStartOffsetInBytes = consumedDataSize;\n\n            break;\n        } else {\n            /* Failed to open the decoder. */\n            if (vorbisError == VORBIS_need_more_data) {\n                continue;\n            } else {\n                ma_free(pData, &pVorbis->allocationCallbacks);\n                return MA_ERROR;   /* Failed to open the stb_vorbis decoder. */\n            }\n        }\n    }\n\n    MA_ASSERT(stb != NULL);\n    pVorbis->stb = stb;\n    pVorbis->push.pData = pData;\n    pVorbis->push.dataSize = dataSize;\n    pVorbis->push.dataCapacity = dataCapacity;\n\n    return MA_SUCCESS;\n}\n#endif\n\nMA_API ma_result ma_stbvorbis_init(ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis)\n{\n    ma_result result;\n\n    result = ma_stbvorbis_init_internal(pConfig, pVorbis);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (onRead == NULL || onSeek == NULL) {\n        return MA_INVALID_ARGS; /* onRead and onSeek are mandatory. */\n    }\n\n    pVorbis->onRead = onRead;\n    pVorbis->onSeek = onSeek;\n    pVorbis->onTell = onTell;\n    pVorbis->pReadSeekTellUserData = pReadSeekTellUserData;\n    ma_allocation_callbacks_init_copy(&pVorbis->allocationCallbacks, pAllocationCallbacks);\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        /*\n        stb_vorbis lacks a callback based API for its pulling API which means we're stuck with the\n        pushing API. In order for us to be able to successfully initialize the decoder we need to\n        supply it with enough data. We need to keep loading data until we have enough.\n        */\n        result = ma_stbvorbis_init_internal_decoder_push(pVorbis);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pVorbis->usingPushMode = MA_TRUE;\n\n        result = ma_stbvorbis_post_init(pVorbis);\n        if (result != MA_SUCCESS) {\n            stb_vorbis_close(pVorbis->stb);\n            ma_free(pVorbis->push.pData, pAllocationCallbacks);\n            return result;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. */\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_init_file(const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis)\n{\n    ma_result result;\n\n    result = ma_stbvorbis_init_internal(pConfig, pVorbis);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        (void)pAllocationCallbacks; /* Don't know how to make use of this with stb_vorbis. */\n\n        /* We can use stb_vorbis' pull mode for file based streams. */\n        pVorbis->stb = stb_vorbis_open_filename(pFilePath, NULL, NULL);\n        if (pVorbis->stb == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        pVorbis->usingPushMode = MA_FALSE;\n\n        result = ma_stbvorbis_post_init(pVorbis);\n        if (result != MA_SUCCESS) {\n            stb_vorbis_close(pVorbis->stb);\n            return result;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. */\n        (void)pFilePath;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_init_memory(const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_stbvorbis* pVorbis)\n{\n    ma_result result;\n\n    result = ma_stbvorbis_init_internal(pConfig, pVorbis);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        (void)pAllocationCallbacks;\n\n        /* stb_vorbis uses an int as its size specifier, restricting it to 32-bit even on 64-bit systems. *sigh*. */\n        if (dataSize > INT_MAX) {\n            return MA_TOO_BIG;\n        }\n\n        pVorbis->stb = stb_vorbis_open_memory((const unsigned char*)pData, (int)dataSize, NULL, NULL);\n        if (pVorbis->stb == NULL) {\n            return MA_INVALID_FILE;\n        }\n\n        pVorbis->usingPushMode = MA_FALSE;\n\n        result = ma_stbvorbis_post_init(pVorbis);\n        if (result != MA_SUCCESS) {\n            stb_vorbis_close(pVorbis->stb);\n            return result;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. */\n        (void)pData;\n        (void)dataSize;\n        (void)pAllocationCallbacks;\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API void ma_stbvorbis_uninit(ma_stbvorbis* pVorbis, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pVorbis == NULL) {\n        return;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        stb_vorbis_close(pVorbis->stb);\n\n        /* We'll have to clear some memory if we're using push mode. */\n        if (pVorbis->usingPushMode) {\n            ma_free(pVorbis->push.pData, pAllocationCallbacks);\n        }\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n    }\n    #endif\n\n    ma_data_source_uninit(&pVorbis->ds);\n}\n\nMA_API ma_result ma_stbvorbis_read_pcm_frames(ma_stbvorbis* pVorbis, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pVorbis == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        /* We always use floating point format. */\n        ma_result result = MA_SUCCESS;  /* Must be initialized to MA_SUCCESS. */\n        ma_uint64 totalFramesRead = 0;\n        ma_format format;\n        ma_uint32 channels;\n\n        ma_stbvorbis_get_data_format(pVorbis, &format, &channels, NULL, NULL, 0);\n\n        if (format == ma_format_f32) {\n            /* We read differently depending on whether or not we're using push mode. */\n            if (pVorbis->usingPushMode) {\n                /* Push mode. This is the complex case. */\n                float* pFramesOutF32 = (float*)pFramesOut;\n\n                while (totalFramesRead < frameCount) {\n                    /* The first thing to do is read from any already-cached frames. */\n                    ma_uint32 framesToReadFromCache = (ma_uint32)ma_min(pVorbis->push.framesRemaining, (frameCount - totalFramesRead));  /* Safe cast because pVorbis->framesRemaining is 32-bit. */\n\n                    /* The output pointer can be null in which case we just treat it as a seek. */\n                    if (pFramesOut != NULL) {\n                        ma_uint64 iFrame;\n                        for (iFrame = 0; iFrame < framesToReadFromCache; iFrame += 1) {\n                            ma_uint32 iChannel;\n                            for (iChannel = 0; iChannel < pVorbis->channels; iChannel += 1) {\n                                pFramesOutF32[iChannel] = pVorbis->push.ppPacketData[iChannel][pVorbis->push.framesConsumed + iFrame];\n                            }\n\n                            pFramesOutF32 += pVorbis->channels;\n                        }\n                    }\n\n                    /* Update pointers and counters. */\n                    pVorbis->push.framesConsumed  += framesToReadFromCache;\n                    pVorbis->push.framesRemaining -= framesToReadFromCache;\n                    totalFramesRead               += framesToReadFromCache;\n\n                    /* Don't bother reading any more frames right now if we've just finished loading. */\n                    if (totalFramesRead == frameCount) {\n                        break;\n                    }\n\n                    MA_ASSERT(pVorbis->push.framesRemaining == 0);\n\n                    /* Getting here means we've run out of cached frames. We'll need to load some more. */\n                    for (;;) {\n                        int samplesRead = 0;\n                        int consumedDataSize;\n\n                        /* We need to case dataSize to an int, so make sure we can do it safely. */\n                        if (pVorbis->push.dataSize > INT_MAX) {\n                            break;  /* Too big. */\n                        }\n\n                        consumedDataSize = stb_vorbis_decode_frame_pushdata(pVorbis->stb, pVorbis->push.pData, (int)pVorbis->push.dataSize, NULL, &pVorbis->push.ppPacketData, &samplesRead);\n                        if (consumedDataSize != 0) {\n                            /* Successfully decoded a Vorbis frame. Consume the data. */\n                            pVorbis->push.dataSize -= (size_t)consumedDataSize;\n                            MA_MOVE_MEMORY(pVorbis->push.pData, ma_offset_ptr(pVorbis->push.pData, consumedDataSize), pVorbis->push.dataSize);\n\n                            pVorbis->push.framesConsumed  = 0;\n                            pVorbis->push.framesRemaining = samplesRead;\n\n                            break;\n                        } else {\n                            /* Not enough data. Read more. */\n                            size_t bytesRead;\n\n                            /* Expand the data buffer if necessary. */\n                            if (pVorbis->push.dataCapacity == pVorbis->push.dataSize) {\n                                size_t newCap = pVorbis->push.dataCapacity + MA_VORBIS_DATA_CHUNK_SIZE;\n                                ma_uint8* pNewData;\n\n                                pNewData = (ma_uint8*)ma_realloc(pVorbis->push.pData, newCap, &pVorbis->allocationCallbacks);\n                                if (pNewData == NULL) {\n                                    result = MA_OUT_OF_MEMORY;\n                                    break;\n                                }\n\n                                pVorbis->push.pData = pNewData;\n                                pVorbis->push.dataCapacity = newCap;\n                            }\n\n                            /* We should have enough room to load some data. */\n                            result = pVorbis->onRead(pVorbis->pReadSeekTellUserData, ma_offset_ptr(pVorbis->push.pData, pVorbis->push.dataSize), (pVorbis->push.dataCapacity - pVorbis->push.dataSize), &bytesRead);\n                            pVorbis->push.dataSize += bytesRead;\n\n                            if (result != MA_SUCCESS) {\n                                break;  /* Failed to read any data. Get out. */\n                            }\n                        }\n                    }\n\n                    /* If we don't have a success code at this point it means we've encountered an error or the end of the file has been reached (probably the latter). */\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n                }\n            } else {\n                /* Pull mode. This is the simple case, but we still need to run in a loop because stb_vorbis loves using 32-bit instead of 64-bit. */\n                while (totalFramesRead < frameCount) {\n                    ma_uint64 framesRemaining = (frameCount - totalFramesRead);\n                    int framesRead;\n\n                    if (framesRemaining > INT_MAX) {\n                        framesRemaining = INT_MAX;\n                    }\n\n                    framesRead = stb_vorbis_get_samples_float_interleaved(pVorbis->stb, channels, (float*)ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, format, channels), (int)framesRemaining * channels);   /* Safe cast. */\n                    totalFramesRead += framesRead;\n\n                    if (framesRead < (int)framesRemaining) {\n                        break;  /* Nothing left to read. Get out. */\n                    }\n                }\n            }\n        } else {\n            result = MA_INVALID_ARGS;\n        }\n\n        pVorbis->cursor += totalFramesRead;\n\n        if (totalFramesRead == 0) {\n            result = MA_AT_END;\n        }\n\n        if (pFramesRead != NULL) {\n            *pFramesRead = totalFramesRead;\n        }\n\n        if (result == MA_SUCCESS && totalFramesRead == 0) {\n            result  = MA_AT_END;\n        }\n\n        return result;\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)pFramesOut;\n        (void)frameCount;\n        (void)pFramesRead;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_seek_to_pcm_frame(ma_stbvorbis* pVorbis, ma_uint64 frameIndex)\n{\n    if (pVorbis == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        /* Different seeking methods depending on whether or not we're using push mode. */\n        if (pVorbis->usingPushMode) {\n            /* Push mode. This is the complex case. */\n            ma_result result;\n            float buffer[4096];\n\n            /* If we're seeking backwards, we need to seek back to the start and then brute-force forward. */\n            if (frameIndex < pVorbis->cursor) {\n                if (frameIndex > 0x7FFFFFFF) {\n                    return MA_INVALID_ARGS; /* Trying to seek beyond the 32-bit maximum of stb_vorbis. */\n                }\n\n                /*\n                This is wildly inefficient due to me having trouble getting sample exact seeking working\n                robustly with stb_vorbis_flush_pushdata(). The only way I can think to make this work\n                perfectly is to reinitialize the decoder. Note that we only enter this path when seeking\n                backwards. This will hopefully be removed once we get our own Vorbis decoder implemented.\n                */\n                stb_vorbis_close(pVorbis->stb);\n                ma_free(pVorbis->push.pData, &pVorbis->allocationCallbacks);\n\n                MA_ZERO_OBJECT(&pVorbis->push);\n\n                /* Seek to the start of the file. */\n                result = pVorbis->onSeek(pVorbis->pReadSeekTellUserData, 0, ma_seek_origin_start);\n                if (result != MA_SUCCESS) {\n                    return result;\n                }\n\n                result = ma_stbvorbis_init_internal_decoder_push(pVorbis);\n                if (result != MA_SUCCESS) {\n                    return result;\n                }\n\n                /* At this point we should be sitting on the first frame. */\n                pVorbis->cursor = 0;\n            }\n\n            /* We're just brute-forcing this for now. */\n            while (pVorbis->cursor < frameIndex) {\n                ma_uint64 framesRead;\n                ma_uint64 framesToRead = ma_countof(buffer)/pVorbis->channels;\n                if (framesToRead > (frameIndex - pVorbis->cursor)) {\n                    framesToRead = (frameIndex - pVorbis->cursor);\n                }\n\n                result = ma_stbvorbis_read_pcm_frames(pVorbis, buffer, framesToRead, &framesRead);\n                if (result != MA_SUCCESS) {\n                    return result;\n                }\n            }\n        } else {\n            /* Pull mode. This is the simple case. */\n            int vorbisResult;\n\n            if (frameIndex > UINT_MAX) {\n                return MA_INVALID_ARGS; /* Trying to seek beyond the 32-bit maximum of stb_vorbis. */\n            }\n\n            vorbisResult = stb_vorbis_seek(pVorbis->stb, (unsigned int)frameIndex);  /* Safe cast. */\n            if (vorbisResult == 0) {\n                return MA_ERROR;    /* See failed. */\n            }\n\n            pVorbis->cursor = frameIndex;\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n\n        (void)frameIndex;\n\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_get_data_format(ma_stbvorbis* pVorbis, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* Defaults for safety. */\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pVorbis == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pVorbis->format;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        if (pChannels != NULL) {\n            *pChannels = pVorbis->channels;\n        }\n\n        if (pSampleRate != NULL) {\n            *pSampleRate = pVorbis->sampleRate;\n        }\n\n        if (pChannelMap != NULL) {\n            ma_channel_map_init_standard(ma_standard_channel_map_vorbis, pChannelMap, channelMapCap, pVorbis->channels);\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_get_cursor_in_pcm_frames(ma_stbvorbis* pVorbis, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;   /* Safety. */\n\n    if (pVorbis == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        *pCursor = pVorbis->cursor;\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\nMA_API ma_result ma_stbvorbis_get_length_in_pcm_frames(ma_stbvorbis* pVorbis, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;   /* Safety. */\n\n    if (pVorbis == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_VORBIS)\n    {\n        if (pVorbis->usingPushMode) {\n            *pLength = 0;   /* I don't know of a good way to determine this reliably with stb_vorbis and push mode. */\n        } else {\n            *pLength = stb_vorbis_stream_length_in_samples(pVorbis->stb);\n        }\n\n        return MA_SUCCESS;\n    }\n    #else\n    {\n        /* vorbis is disabled. Should never hit this since initialization would have failed. */\n        MA_ASSERT(MA_FALSE);\n        return MA_NOT_IMPLEMENTED;\n    }\n    #endif\n}\n\n\nstatic ma_result ma_decoding_backend_init__stbvorbis(void* pUserData, ma_read_proc onRead, ma_seek_proc onSeek, ma_tell_proc onTell, void* pReadSeekTellUserData, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_stbvorbis* pVorbis;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pVorbis = (ma_stbvorbis*)ma_malloc(sizeof(*pVorbis), pAllocationCallbacks);\n    if (pVorbis == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_stbvorbis_init(onRead, onSeek, onTell, pReadSeekTellUserData, pConfig, pAllocationCallbacks, pVorbis);\n    if (result != MA_SUCCESS) {\n        ma_free(pVorbis, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pVorbis;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_file__stbvorbis(void* pUserData, const char* pFilePath, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_stbvorbis* pVorbis;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pVorbis = (ma_stbvorbis*)ma_malloc(sizeof(*pVorbis), pAllocationCallbacks);\n    if (pVorbis == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_stbvorbis_init_file(pFilePath, pConfig, pAllocationCallbacks, pVorbis);\n    if (result != MA_SUCCESS) {\n        ma_free(pVorbis, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pVorbis;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoding_backend_init_memory__stbvorbis(void* pUserData, const void* pData, size_t dataSize, const ma_decoding_backend_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source** ppBackend)\n{\n    ma_result result;\n    ma_stbvorbis* pVorbis;\n\n    (void)pUserData;    /* For now not using pUserData, but once we start storing the vorbis decoder state within the ma_decoder structure this will be set to the decoder so we can avoid a malloc. */\n\n    /* For now we're just allocating the decoder backend on the heap. */\n    pVorbis = (ma_stbvorbis*)ma_malloc(sizeof(*pVorbis), pAllocationCallbacks);\n    if (pVorbis == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_stbvorbis_init_memory(pData, dataSize, pConfig, pAllocationCallbacks, pVorbis);\n    if (result != MA_SUCCESS) {\n        ma_free(pVorbis, pAllocationCallbacks);\n        return result;\n    }\n\n    *ppBackend = pVorbis;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_decoding_backend_uninit__stbvorbis(void* pUserData, ma_data_source* pBackend, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_stbvorbis* pVorbis = (ma_stbvorbis*)pBackend;\n\n    (void)pUserData;\n\n    ma_stbvorbis_uninit(pVorbis, pAllocationCallbacks);\n    ma_free(pVorbis, pAllocationCallbacks);\n}\n\nstatic ma_decoding_backend_vtable g_ma_decoding_backend_vtable_stbvorbis =\n{\n    ma_decoding_backend_init__stbvorbis,\n    ma_decoding_backend_init_file__stbvorbis,\n    NULL, /* onInitFileW() */\n    ma_decoding_backend_init_memory__stbvorbis,\n    ma_decoding_backend_uninit__stbvorbis\n};\n\nstatic ma_result ma_decoder_init_vorbis__internal(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_vtable__internal(&g_ma_decoding_backend_vtable_stbvorbis, NULL, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_vorbis_from_file__internal(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file__internal(&g_ma_decoding_backend_vtable_stbvorbis, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_vorbis_from_file_w__internal(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_file_w__internal(&g_ma_decoding_backend_vtable_stbvorbis, NULL, pFilePath, pConfig, pDecoder);\n}\n\nstatic ma_result ma_decoder_init_vorbis_from_memory__internal(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    return ma_decoder_init_from_memory__internal(&g_ma_decoding_backend_vtable_stbvorbis, NULL, pData, dataSize, pConfig, pDecoder);\n}\n#endif  /* STB_VORBIS_INCLUDE_STB_VORBIS_H */\n\n\n\nstatic ma_result ma_decoder__init_allocation_callbacks(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    if (pConfig != NULL) {\n        return ma_allocation_callbacks_init_copy(&pDecoder->allocationCallbacks, &pConfig->allocationCallbacks);\n    } else {\n        pDecoder->allocationCallbacks = ma_allocation_callbacks_init_default();\n        return MA_SUCCESS;\n    }\n}\n\nstatic ma_result ma_decoder__data_source_on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_decoder_read_pcm_frames((ma_decoder*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_decoder__data_source_on_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_decoder_seek_to_pcm_frame((ma_decoder*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_decoder__data_source_on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_decoder_get_data_format((ma_decoder*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_decoder__data_source_on_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_decoder_get_cursor_in_pcm_frames((ma_decoder*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_decoder__data_source_on_get_length(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_decoder_get_length_in_pcm_frames((ma_decoder*)pDataSource, pLength);\n}\n\nstatic ma_data_source_vtable g_ma_decoder_data_source_vtable =\n{\n    ma_decoder__data_source_on_read,\n    ma_decoder__data_source_on_seek,\n    ma_decoder__data_source_on_get_data_format,\n    ma_decoder__data_source_on_get_cursor,\n    ma_decoder__data_source_on_get_length,\n    NULL,   /* onSetLooping */\n    0\n};\n\nstatic ma_result ma_decoder__preinit(ma_decoder_read_proc onRead, ma_decoder_seek_proc onSeek, ma_decoder_tell_proc onTell, void* pUserData, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    MA_ASSERT(pConfig != NULL);\n\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDecoder);\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_decoder_data_source_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pDecoder->ds);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pDecoder->onRead    = onRead;\n    pDecoder->onSeek    = onSeek;\n    pDecoder->onTell    = onTell;\n    pDecoder->pUserData = pUserData;\n\n    result = ma_decoder__init_allocation_callbacks(pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        ma_data_source_uninit(&pDecoder->ds);\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder__postinit(const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n\n    result = ma_decoder__init_data_converter(pDecoder, pConfig);\n\n    /* If we failed post initialization we need to uninitialize the decoder before returning to prevent a memory leak. */\n    if (result != MA_SUCCESS) {\n        ma_decoder_uninit(pDecoder);\n        return result;\n    }\n\n    return result;\n}\n\n\nstatic ma_result ma_decoder_init__internal(ma_decoder_read_proc onRead, ma_decoder_seek_proc onSeek, void* pUserData, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = MA_NO_BACKEND;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pDecoder != NULL);\n\n    /* Silence some warnings in the case that we don't have any decoder backends enabled. */\n    (void)onRead;\n    (void)onSeek;\n    (void)pUserData;\n\n\n    /* If we've specified a specific encoding type, try that first. */\n    if (pConfig->encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (pConfig->encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav__internal(pConfig, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (pConfig->encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac__internal(pConfig, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (pConfig->encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3__internal(pConfig, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (pConfig->encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis__internal(pConfig, pDecoder);\n        }\n    #endif\n\n        /* If we weren't able to initialize the decoder, seek back to the start to give the next attempts a clean start. */\n        if (result != MA_SUCCESS) {\n            onSeek(pDecoder, 0, ma_seek_origin_start);\n        }\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we couldn't load a specific decoding backend based on the encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom__internal(pConfig, pDecoder);\n        if (result != MA_SUCCESS) {\n            onSeek(pDecoder, 0, ma_seek_origin_start);\n        }\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (pConfig->encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n    #ifdef MA_HAS_WAV\n        if (result != MA_SUCCESS) {\n            result = ma_decoder_init_wav__internal(pConfig, pDecoder);\n            if (result != MA_SUCCESS) {\n                onSeek(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (result != MA_SUCCESS) {\n            result = ma_decoder_init_flac__internal(pConfig, pDecoder);\n            if (result != MA_SUCCESS) {\n                onSeek(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (result != MA_SUCCESS) {\n            result = ma_decoder_init_mp3__internal(pConfig, pDecoder);\n            if (result != MA_SUCCESS) {\n                onSeek(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (result != MA_SUCCESS) {\n            result = ma_decoder_init_vorbis__internal(pConfig, pDecoder);\n            if (result != MA_SUCCESS) {\n                onSeek(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    }\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_decoder__postinit(pConfig, pDecoder);\n}\n\nMA_API ma_result ma_decoder_init(ma_decoder_read_proc onRead, ma_decoder_seek_proc onSeek, void* pUserData, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_decoder_config config;\n    ma_result result;\n\n    config = ma_decoder_config_init_copy(pConfig);\n\n    result = ma_decoder__preinit(onRead, onSeek, NULL, pUserData, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_decoder_init__internal(onRead, onSeek, pUserData, &config, pDecoder);\n}\n\n\nstatic ma_result ma_decoder__on_read_memory(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead)\n{\n    size_t bytesRemaining;\n\n    MA_ASSERT(pDecoder->data.memory.dataSize >= pDecoder->data.memory.currentReadPos);\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = 0;\n    }\n\n    bytesRemaining = pDecoder->data.memory.dataSize - pDecoder->data.memory.currentReadPos;\n    if (bytesToRead > bytesRemaining) {\n        bytesToRead = bytesRemaining;\n    }\n\n    if (bytesRemaining == 0) {\n        return MA_AT_END;\n    }\n\n    if (bytesToRead > 0) {\n        MA_COPY_MEMORY(pBufferOut, pDecoder->data.memory.pData + pDecoder->data.memory.currentReadPos, bytesToRead);\n        pDecoder->data.memory.currentReadPos += bytesToRead;\n    }\n\n    if (pBytesRead != NULL) {\n        *pBytesRead = bytesToRead;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder__on_seek_memory(ma_decoder* pDecoder, ma_int64 byteOffset, ma_seek_origin origin)\n{\n    if (byteOffset > 0 && (ma_uint64)byteOffset > MA_SIZE_MAX) {\n        return MA_BAD_SEEK;\n    }\n\n    if (origin == ma_seek_origin_current) {\n        if (byteOffset > 0) {\n            if (pDecoder->data.memory.currentReadPos + byteOffset > pDecoder->data.memory.dataSize) {\n                byteOffset = (ma_int64)(pDecoder->data.memory.dataSize - pDecoder->data.memory.currentReadPos);  /* Trying to seek too far forward. */\n            }\n\n            pDecoder->data.memory.currentReadPos += (size_t)byteOffset;\n        } else {\n            if (pDecoder->data.memory.currentReadPos < (size_t)-byteOffset) {\n                byteOffset = -(ma_int64)pDecoder->data.memory.currentReadPos;  /* Trying to seek too far backwards. */\n            }\n\n            pDecoder->data.memory.currentReadPos -= (size_t)-byteOffset;\n        }\n    } else {\n        if (origin == ma_seek_origin_end) {\n            if (byteOffset < 0) {\n                byteOffset = -byteOffset;\n            }\n\n            if (byteOffset > (ma_int64)pDecoder->data.memory.dataSize) {\n                pDecoder->data.memory.currentReadPos = 0;   /* Trying to seek too far back. */\n            } else {\n                pDecoder->data.memory.currentReadPos = pDecoder->data.memory.dataSize - (size_t)byteOffset;\n            }\n        } else {\n            if ((size_t)byteOffset <= pDecoder->data.memory.dataSize) {\n                pDecoder->data.memory.currentReadPos = (size_t)byteOffset;\n            } else {\n                pDecoder->data.memory.currentReadPos = pDecoder->data.memory.dataSize;  /* Trying to seek too far forward. */\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder__on_tell_memory(ma_decoder* pDecoder, ma_int64* pCursor)\n{\n    MA_ASSERT(pDecoder != NULL);\n    MA_ASSERT(pCursor  != NULL);\n\n    *pCursor = (ma_int64)pDecoder->data.memory.currentReadPos;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder__preinit_memory_wrapper(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result = ma_decoder__preinit(ma_decoder__on_read_memory, ma_decoder__on_seek_memory, ma_decoder__on_tell_memory, NULL, pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pData == NULL || dataSize == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDecoder->data.memory.pData = (const ma_uint8*)pData;\n    pDecoder->data.memory.dataSize = dataSize;\n    pDecoder->data.memory.currentReadPos = 0;\n\n    (void)pConfig;\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_init_memory(const void* pData, size_t dataSize, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init_copy(pConfig);\n\n    result = ma_decoder__preinit(NULL, NULL, NULL, NULL, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pData == NULL || dataSize == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If the backend has support for loading from a file path we'll want to use that. If that all fails we'll fall back to the VFS path. */\n    result = MA_NO_BACKEND;\n\n    if (config.encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (config.encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav_from_memory__internal(pData, dataSize, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (config.encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac_from_memory__internal(pData, dataSize, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (config.encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3_from_memory__internal(pData, dataSize, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (config.encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis_from_memory__internal(pData, dataSize, &config, pDecoder);\n        }\n    #endif\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we weren't able to initialize a decoder of a specific encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom_from_memory__internal(pData, dataSize, &config, pDecoder);\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (result != MA_SUCCESS && config.encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n        /* Use trial and error for stock decoders. */\n        if (result != MA_SUCCESS) {\n        #ifdef MA_HAS_WAV\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_wav_from_memory__internal(pData, dataSize, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_FLAC\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_flac_from_memory__internal(pData, dataSize, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_MP3\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_mp3_from_memory__internal(pData, dataSize, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_VORBIS\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_vorbis_from_memory__internal(pData, dataSize, &config, pDecoder);\n            }\n        #endif\n        }\n    }\n\n    /*\n    If at this point we still haven't successfully initialized the decoder it most likely means\n    the backend doesn't have an implementation for loading from a file path. We'll try using\n    miniaudio's built-in file IO for loading file.\n    */\n    if (result == MA_SUCCESS) {\n        /* Initialization was successful. Finish up. */\n        result = ma_decoder__postinit(&config, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        /* Probably no implementation for loading from a block of memory. Use miniaudio's abstraction instead. */\n        result = ma_decoder__preinit_memory_wrapper(pData, dataSize, &config, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        result = ma_decoder_init__internal(ma_decoder__on_read_memory, ma_decoder__on_seek_memory, NULL, &config, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n\n#if defined(MA_HAS_WAV)    || \\\n    defined(MA_HAS_MP3)    || \\\n    defined(MA_HAS_FLAC)   || \\\n    defined(MA_HAS_VORBIS)\n#define MA_HAS_PATH_API\n#endif\n\n#if defined(MA_HAS_PATH_API)\nstatic const char* ma_path_file_name(const char* path)\n{\n    const char* fileName;\n\n    if (path == NULL) {\n        return NULL;\n    }\n\n    fileName = path;\n\n    /* We just loop through the path until we find the last slash. */\n    while (path[0] != '\\0') {\n        if (path[0] == '/' || path[0] == '\\\\') {\n            fileName = path;\n        }\n\n        path += 1;\n    }\n\n    /* At this point the file name is sitting on a slash, so just move forward. */\n    while (fileName[0] != '\\0' && (fileName[0] == '/' || fileName[0] == '\\\\')) {\n        fileName += 1;\n    }\n\n    return fileName;\n}\n\nstatic const wchar_t* ma_path_file_name_w(const wchar_t* path)\n{\n    const wchar_t* fileName;\n\n    if (path == NULL) {\n        return NULL;\n    }\n\n    fileName = path;\n\n    /* We just loop through the path until we find the last slash. */\n    while (path[0] != '\\0') {\n        if (path[0] == '/' || path[0] == '\\\\') {\n            fileName = path;\n        }\n\n        path += 1;\n    }\n\n    /* At this point the file name is sitting on a slash, so just move forward. */\n    while (fileName[0] != '\\0' && (fileName[0] == '/' || fileName[0] == '\\\\')) {\n        fileName += 1;\n    }\n\n    return fileName;\n}\n\n\nstatic const char* ma_path_extension(const char* path)\n{\n    const char* extension;\n    const char* lastOccurance;\n\n    if (path == NULL) {\n        path = \"\";\n    }\n\n    extension = ma_path_file_name(path);\n    lastOccurance = NULL;\n\n    /* Just find the last '.' and return. */\n    while (extension[0] != '\\0') {\n        if (extension[0] == '.') {\n            extension += 1;\n            lastOccurance = extension;\n        }\n\n        extension += 1;\n    }\n\n    return (lastOccurance != NULL) ? lastOccurance : extension;\n}\n\nstatic const wchar_t* ma_path_extension_w(const wchar_t* path)\n{\n    const wchar_t* extension;\n    const wchar_t* lastOccurance;\n\n    if (path == NULL) {\n        path = L\"\";\n    }\n\n    extension = ma_path_file_name_w(path);\n    lastOccurance = NULL;\n\n    /* Just find the last '.' and return. */\n    while (extension[0] != '\\0') {\n        if (extension[0] == '.') {\n            extension += 1;\n            lastOccurance = extension;\n        }\n\n        extension += 1;\n    }\n\n    return (lastOccurance != NULL) ? lastOccurance : extension;\n}\n\n\nstatic ma_bool32 ma_path_extension_equal(const char* path, const char* extension)\n{\n    const char* ext1;\n    const char* ext2;\n\n    if (path == NULL || extension == NULL) {\n        return MA_FALSE;\n    }\n\n    ext1 = extension;\n    ext2 = ma_path_extension(path);\n\n#if defined(_MSC_VER) || defined(__DMC__)\n    return _stricmp(ext1, ext2) == 0;\n#else\n    return strcasecmp(ext1, ext2) == 0;\n#endif\n}\n\nstatic ma_bool32 ma_path_extension_equal_w(const wchar_t* path, const wchar_t* extension)\n{\n    const wchar_t* ext1;\n    const wchar_t* ext2;\n\n    if (path == NULL || extension == NULL) {\n        return MA_FALSE;\n    }\n\n    ext1 = extension;\n    ext2 = ma_path_extension_w(path);\n\n    #if (defined(_MSC_VER) || defined(__WATCOMC__) || defined(__DMC__)) && !defined(MA_XBOX_NXDK)\n    {\n        return _wcsicmp(ext1, ext2) == 0;\n    }\n    #elif !defined(MA_XBOX_NXDK) && !defined(MA_DOS)\n    {\n        /*\n        I'm not aware of a wide character version of strcasecmp(). I'm therefore converting the extensions to multibyte strings and comparing those. This\n        isn't the most efficient way to do it, but it should work OK.\n        */\n        char ext1MB[4096];\n        char ext2MB[4096];\n        const wchar_t* pext1 = ext1;\n        const wchar_t* pext2 = ext2;\n        mbstate_t mbs1;\n        mbstate_t mbs2;\n\n        MA_ZERO_OBJECT(&mbs1);\n        MA_ZERO_OBJECT(&mbs2);\n\n        if (wcsrtombs(ext1MB, &pext1, sizeof(ext1MB), &mbs1) == (size_t)-1) {\n            return MA_FALSE;\n        }\n        if (wcsrtombs(ext2MB, &pext2, sizeof(ext2MB), &mbs2) == (size_t)-1) {\n            return MA_FALSE;\n        }\n\n        return strcasecmp(ext1MB, ext2MB) == 0;\n    }\n    #else\n    {\n        /* Getting here means we don't have a way to do a case-sensitive comparison for wide strings. Fall back to a simple case-sensitive comparison. */\n        /* TODO: Implement our own wchar_t-to-char conversion routine and then use the char* version for comparing. */\n        return ma_wcscmp(ext1, ext2) == 0;\n    }\n    #endif\n}\n#endif  /* MA_HAS_PATH_API */\n\n\n\nstatic ma_result ma_decoder__on_read_vfs(ma_decoder* pDecoder, void* pBufferOut, size_t bytesToRead, size_t* pBytesRead)\n{\n    MA_ASSERT(pDecoder   != NULL);\n    MA_ASSERT(pBufferOut != NULL);\n\n    return ma_vfs_or_default_read(pDecoder->data.vfs.pVFS, pDecoder->data.vfs.file, pBufferOut, bytesToRead, pBytesRead);\n}\n\nstatic ma_result ma_decoder__on_seek_vfs(ma_decoder* pDecoder, ma_int64 offset, ma_seek_origin origin)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    return ma_vfs_or_default_seek(pDecoder->data.vfs.pVFS, pDecoder->data.vfs.file, offset, origin);\n}\n\nstatic ma_result ma_decoder__on_tell_vfs(ma_decoder* pDecoder, ma_int64* pCursor)\n{\n    MA_ASSERT(pDecoder != NULL);\n\n    return ma_vfs_or_default_tell(pDecoder->data.vfs.pVFS, pDecoder->data.vfs.file, pCursor);\n}\n\nstatic ma_result ma_decoder__preinit_vfs(ma_vfs* pVFS, const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_vfs_file file;\n\n    result = ma_decoder__preinit(ma_decoder__on_read_vfs, ma_decoder__on_seek_vfs, ma_decoder__on_tell_vfs, NULL, pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFilePath == NULL || pFilePath[0] == '\\0') {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_vfs_or_default_open(pVFS, pFilePath, MA_OPEN_MODE_READ, &file);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pDecoder->data.vfs.pVFS = pVFS;\n    pDecoder->data.vfs.file = file;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_init_vfs(ma_vfs* pVFS, const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init_copy(pConfig);\n    result = ma_decoder__preinit_vfs(pVFS, pFilePath, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = MA_NO_BACKEND;\n\n    if (config.encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (config.encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (config.encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (config.encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (config.encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis__internal(&config, pDecoder);\n        }\n    #endif\n\n        /* Make sure we seek back to the start if we didn't initialize a decoder successfully so the next attempts have a fresh start. */\n        if (result != MA_SUCCESS) {\n            ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n        }\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we weren't able to initialize a decoder of a specific encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom__internal(&config, pDecoder);\n        if (result != MA_SUCCESS) {\n            ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n        }\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (config.encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n    #ifdef MA_HAS_WAV\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"wav\")) {\n            result = ma_decoder_init_wav__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"flac\")) {\n            result = ma_decoder_init_flac__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"mp3\")) {\n            result = ma_decoder_init_mp3__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    }\n\n    /* If we still haven't got a result just use trial and error. Otherwise we can finish up. */\n    if (result != MA_SUCCESS) {\n        result = ma_decoder_init__internal(ma_decoder__on_read_vfs, ma_decoder__on_seek_vfs, NULL, &config, pDecoder);\n    } else {\n        result = ma_decoder__postinit(&config, pDecoder);\n    }\n\n    if (result != MA_SUCCESS) {\n        if (pDecoder->data.vfs.file != NULL) {   /* <-- Will be reset to NULL if ma_decoder_uninit() is called in one of the steps above which allows us to avoid a double close of the file. */\n            ma_vfs_or_default_close(pVFS, pDecoder->data.vfs.file);\n        }\n\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_decoder__preinit_vfs_w(ma_vfs* pVFS, const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_vfs_file file;\n\n    result = ma_decoder__preinit(ma_decoder__on_read_vfs, ma_decoder__on_seek_vfs, ma_decoder__on_tell_vfs, NULL, pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFilePath == NULL || pFilePath[0] == '\\0') {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_vfs_or_default_open_w(pVFS, pFilePath, MA_OPEN_MODE_READ, &file);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pDecoder->data.vfs.pVFS = pVFS;\n    pDecoder->data.vfs.file = file;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_init_vfs_w(ma_vfs* pVFS, const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init_copy(pConfig);\n    result = ma_decoder__preinit_vfs_w(pVFS, pFilePath, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = MA_NO_BACKEND;\n\n    if (config.encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (config.encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (config.encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (config.encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3__internal(&config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (config.encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis__internal(&config, pDecoder);\n        }\n    #endif\n\n        /* Make sure we seek back to the start if we didn't initialize a decoder successfully so the next attempts have a fresh start. */\n        if (result != MA_SUCCESS) {\n            ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n        }\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we weren't able to initialize a decoder of a specific encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom__internal(&config, pDecoder);\n        if (result != MA_SUCCESS) {\n            ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n        }\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (config.encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n    #ifdef MA_HAS_WAV\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"wav\")) {\n            result = ma_decoder_init_wav__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"flac\")) {\n            result = ma_decoder_init_flac__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"mp3\")) {\n            result = ma_decoder_init_mp3__internal(&config, pDecoder);\n            if (result != MA_SUCCESS) {\n                ma_decoder__on_seek_vfs(pDecoder, 0, ma_seek_origin_start);\n            }\n        }\n    #endif\n    }\n\n    /* If we still haven't got a result just use trial and error. Otherwise we can finish up. */\n    if (result != MA_SUCCESS) {\n        result = ma_decoder_init__internal(ma_decoder__on_read_vfs, ma_decoder__on_seek_vfs, NULL, &config, pDecoder);\n    } else {\n        result = ma_decoder__postinit(&config, pDecoder);\n    }\n\n    if (result != MA_SUCCESS) {\n        ma_vfs_or_default_close(pVFS, pDecoder->data.vfs.file);\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_decoder__preinit_file(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n\n    result = ma_decoder__preinit(NULL, NULL, NULL, NULL, pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFilePath == NULL || pFilePath[0] == '\\0') {\n        return MA_INVALID_ARGS;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_init_file(const char* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init_copy(pConfig);\n    result = ma_decoder__preinit_file(pFilePath, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* If the backend has support for loading from a file path we'll want to use that. If that all fails we'll fall back to the VFS path. */\n    result = MA_NO_BACKEND;\n\n    if (config.encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (config.encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (config.encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (config.encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (config.encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we weren't able to initialize a decoder of a specific encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom_from_file__internal(pFilePath, &config, pDecoder);\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (result != MA_SUCCESS && config.encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n        /* First try loading based on the file extension so we don't waste time opening and closing files. */\n    #ifdef MA_HAS_WAV\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"wav\")) {\n            result = ma_decoder_init_wav_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"flac\")) {\n            result = ma_decoder_init_flac_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"mp3\")) {\n            result = ma_decoder_init_mp3_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (result != MA_SUCCESS && ma_path_extension_equal(pFilePath, \"ogg\")) {\n            result = ma_decoder_init_vorbis_from_file__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n\n        /*\n        If we still haven't got a result just use trial and error. Custom decoders have already been attempted, so here we\n        need only iterate over our stock decoders.\n        */\n        if (result != MA_SUCCESS) {\n        #ifdef MA_HAS_WAV\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_wav_from_file__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_FLAC\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_flac_from_file__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_MP3\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_mp3_from_file__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_VORBIS\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_vorbis_from_file__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        }\n    }\n\n    /*\n    If at this point we still haven't successfully initialized the decoder it most likely means\n    the backend doesn't have an implementation for loading from a file path. We'll try using\n    miniaudio's built-in file IO for loading file.\n    */\n    if (result == MA_SUCCESS) {\n        /* Initialization was successful. Finish up. */\n        result = ma_decoder__postinit(&config, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        /* Probably no implementation for loading from a file path. Use miniaudio's file IO instead. */\n        result = ma_decoder_init_vfs(NULL, pFilePath, pConfig, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_decoder__preinit_file_w(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n\n    result = ma_decoder__preinit(NULL, NULL, NULL, NULL, pConfig, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFilePath == NULL || pFilePath[0] == '\\0') {\n        return MA_INVALID_ARGS;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_init_file_w(const wchar_t* pFilePath, const ma_decoder_config* pConfig, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init_copy(pConfig);\n    result = ma_decoder__preinit_file_w(pFilePath, &config, pDecoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* If the backend has support for loading from a file path we'll want to use that. If that all fails we'll fall back to the VFS path. */\n    result = MA_NO_BACKEND;\n\n    if (config.encodingFormat != ma_encoding_format_unknown) {\n    #ifdef MA_HAS_WAV\n        if (config.encodingFormat == ma_encoding_format_wav) {\n            result = ma_decoder_init_wav_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (config.encodingFormat == ma_encoding_format_flac) {\n            result = ma_decoder_init_flac_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (config.encodingFormat == ma_encoding_format_mp3) {\n            result = ma_decoder_init_mp3_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (config.encodingFormat == ma_encoding_format_vorbis) {\n            result = ma_decoder_init_vorbis_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    }\n\n    if (result != MA_SUCCESS) {\n        /* Getting here means we weren't able to initialize a decoder of a specific encoding format. */\n\n        /*\n        We use trial and error to open a decoder. We prioritize custom decoders so that if they\n        implement the same encoding format they take priority over the built-in decoders.\n        */\n        result = ma_decoder_init_custom_from_file_w__internal(pFilePath, &config, pDecoder);\n\n        /*\n        If we get to this point and we still haven't found a decoder, and the caller has requested a\n        specific encoding format, there's no hope for it. Abort.\n        */\n        if (result != MA_SUCCESS && config.encodingFormat != ma_encoding_format_unknown) {\n            return MA_NO_BACKEND;\n        }\n\n        /* First try loading based on the file extension so we don't waste time opening and closing files. */\n    #ifdef MA_HAS_WAV\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"wav\")) {\n            result = ma_decoder_init_wav_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_FLAC\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"flac\")) {\n            result = ma_decoder_init_flac_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_MP3\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"mp3\")) {\n            result = ma_decoder_init_mp3_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n    #ifdef MA_HAS_VORBIS\n        if (result != MA_SUCCESS && ma_path_extension_equal_w(pFilePath, L\"ogg\")) {\n            result = ma_decoder_init_vorbis_from_file_w__internal(pFilePath, &config, pDecoder);\n        }\n    #endif\n\n        /*\n        If we still haven't got a result just use trial and error. Custom decoders have already been attempted, so here we\n        need only iterate over our stock decoders.\n        */\n        if (result != MA_SUCCESS) {\n        #ifdef MA_HAS_WAV\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_wav_from_file_w__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_FLAC\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_flac_from_file_w__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_MP3\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_mp3_from_file_w__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        #ifdef MA_HAS_VORBIS\n            if (result != MA_SUCCESS) {\n                result = ma_decoder_init_vorbis_from_file_w__internal(pFilePath, &config, pDecoder);\n            }\n        #endif\n        }\n    }\n\n    /*\n    If at this point we still haven't successfully initialized the decoder it most likely means\n    the backend doesn't have an implementation for loading from a file path. We'll try using\n    miniaudio's built-in file IO for loading file.\n    */\n    if (result == MA_SUCCESS) {\n        /* Initialization was successful. Finish up. */\n        result = ma_decoder__postinit(&config, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        /* Probably no implementation for loading from a file path. Use miniaudio's file IO instead. */\n        result = ma_decoder_init_vfs_w(NULL, pFilePath, pConfig, pDecoder);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_uninit(ma_decoder* pDecoder)\n{\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDecoder->pBackend != NULL) {\n        if (pDecoder->pBackendVTable != NULL && pDecoder->pBackendVTable->onUninit != NULL) {\n            pDecoder->pBackendVTable->onUninit(pDecoder->pBackendUserData, pDecoder->pBackend, &pDecoder->allocationCallbacks);\n        }\n    }\n\n    if (pDecoder->onRead == ma_decoder__on_read_vfs) {\n        ma_vfs_or_default_close(pDecoder->data.vfs.pVFS, pDecoder->data.vfs.file);\n        pDecoder->data.vfs.file = NULL;\n    }\n\n    ma_data_converter_uninit(&pDecoder->converter, &pDecoder->allocationCallbacks);\n    ma_data_source_uninit(&pDecoder->ds);\n\n    if (pDecoder->pInputCache != NULL) {\n        ma_free(pDecoder->pInputCache, &pDecoder->allocationCallbacks);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_read_pcm_frames(ma_decoder* pDecoder, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 totalFramesReadOut;\n    void* pRunningFramesOut;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;   /* Safety. */\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDecoder->pBackend == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* Fast path. */\n    if (pDecoder->converter.isPassthrough) {\n        result = ma_data_source_read_pcm_frames(pDecoder->pBackend, pFramesOut, frameCount, &totalFramesReadOut);\n    } else {\n        /*\n        Getting here means we need to do data conversion. If we're seeking forward and are _not_ doing resampling we can run this in a fast path. If we're doing resampling we\n        need to run through each sample because we need to ensure its internal cache is updated.\n        */\n        if (pFramesOut == NULL && pDecoder->converter.hasResampler == MA_FALSE) {\n            result = ma_data_source_read_pcm_frames(pDecoder->pBackend, NULL, frameCount, &totalFramesReadOut);\n        } else {\n            /* Slow path. Need to run everything through the data converter. */\n            ma_format internalFormat;\n            ma_uint32 internalChannels;\n\n            totalFramesReadOut = 0;\n            pRunningFramesOut  = pFramesOut;\n\n            result = ma_data_source_get_data_format(pDecoder->pBackend, &internalFormat, &internalChannels, NULL, NULL, 0);\n            if (result != MA_SUCCESS) {\n                return result;   /* Failed to retrieve the internal format and channel count. */\n            }\n\n            /*\n            We run a different path depending on whether or not we are using a heap-allocated\n            intermediary buffer or not. If the data converter does not support the calculation of\n            the required number of input frames, we'll use the heap-allocated path. Otherwise we'll\n            use the stack-allocated path.\n            */\n            if (pDecoder->pInputCache != NULL) {\n                /* We don't have a way of determining the required number of input frames, so need to persistently store input data in a cache. */\n                while (totalFramesReadOut < frameCount) {\n                    ma_uint64 framesToReadThisIterationIn;\n                    ma_uint64 framesToReadThisIterationOut;\n\n                    /* If there's any data available in the cache, that needs to get processed first. */\n                    if (pDecoder->inputCacheRemaining > 0) {\n                        framesToReadThisIterationOut = (frameCount - totalFramesReadOut);\n                        framesToReadThisIterationIn  = framesToReadThisIterationOut;\n                        if (framesToReadThisIterationIn > pDecoder->inputCacheRemaining) {\n                            framesToReadThisIterationIn = pDecoder->inputCacheRemaining;\n                        }\n\n                        result = ma_data_converter_process_pcm_frames(&pDecoder->converter, ma_offset_pcm_frames_ptr(pDecoder->pInputCache, pDecoder->inputCacheConsumed, internalFormat, internalChannels), &framesToReadThisIterationIn, pRunningFramesOut, &framesToReadThisIterationOut);\n                        if (result != MA_SUCCESS) {\n                            break;\n                        }\n\n                        pDecoder->inputCacheConsumed  += framesToReadThisIterationIn;\n                        pDecoder->inputCacheRemaining -= framesToReadThisIterationIn;\n\n                        totalFramesReadOut += framesToReadThisIterationOut;\n\n                        if (pRunningFramesOut != NULL) {\n                            pRunningFramesOut = ma_offset_ptr(pRunningFramesOut, framesToReadThisIterationOut * ma_get_bytes_per_frame(pDecoder->outputFormat, pDecoder->outputChannels));\n                        }\n\n                        if (framesToReadThisIterationIn == 0 && framesToReadThisIterationOut == 0) {\n                            break;  /* We're done. */\n                        }\n                    }\n\n                    /* Getting here means there's no data in the cache and we need to fill it up from the data source. */\n                    if (pDecoder->inputCacheRemaining == 0) {\n                        pDecoder->inputCacheConsumed = 0;\n\n                        result = ma_data_source_read_pcm_frames(pDecoder->pBackend, pDecoder->pInputCache, pDecoder->inputCacheCap, &pDecoder->inputCacheRemaining);\n                        if (result != MA_SUCCESS) {\n                            break;\n                        }\n                    }\n                }\n            } else {\n                /* We have a way of determining the required number of input frames so just use the stack. */\n                while (totalFramesReadOut < frameCount) {\n                    ma_uint8 pIntermediaryBuffer[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];  /* In internal format. */\n                    ma_uint64 intermediaryBufferCap = sizeof(pIntermediaryBuffer) / ma_get_bytes_per_frame(internalFormat, internalChannels);\n                    ma_uint64 framesToReadThisIterationIn;\n                    ma_uint64 framesReadThisIterationIn;\n                    ma_uint64 framesToReadThisIterationOut;\n                    ma_uint64 framesReadThisIterationOut;\n                    ma_uint64 requiredInputFrameCount;\n\n                    framesToReadThisIterationOut = (frameCount - totalFramesReadOut);\n                    framesToReadThisIterationIn = framesToReadThisIterationOut;\n                    if (framesToReadThisIterationIn > intermediaryBufferCap) {\n                        framesToReadThisIterationIn = intermediaryBufferCap;\n                    }\n\n                    ma_data_converter_get_required_input_frame_count(&pDecoder->converter, framesToReadThisIterationOut, &requiredInputFrameCount);\n                    if (framesToReadThisIterationIn > requiredInputFrameCount) {\n                        framesToReadThisIterationIn = requiredInputFrameCount;\n                    }\n\n                    if (requiredInputFrameCount > 0) {\n                        result = ma_data_source_read_pcm_frames(pDecoder->pBackend, pIntermediaryBuffer, framesToReadThisIterationIn, &framesReadThisIterationIn);\n\n                        /*\n                        Note here that even if we've reached the end, we don't want to abort because there might be more output frames needing to be\n                        generated from cached input data, which might happen if resampling is being performed.\n                        */\n                        if (result != MA_SUCCESS && result != MA_AT_END) {\n                            break;\n                        }\n                    } else {\n                        framesReadThisIterationIn = 0;\n                        pIntermediaryBuffer[0] = 0; /* <-- This is just to silence a static analysis warning. */\n                    }\n\n                    /*\n                    At this point we have our decoded data in input format and now we need to convert to output format. Note that even if we didn't read any\n                    input frames, we still want to try processing frames because there may some output frames generated from cached input data.\n                    */\n                    framesReadThisIterationOut = framesToReadThisIterationOut;\n                    result = ma_data_converter_process_pcm_frames(&pDecoder->converter, pIntermediaryBuffer, &framesReadThisIterationIn, pRunningFramesOut, &framesReadThisIterationOut);\n                    if (result != MA_SUCCESS) {\n                        break;\n                    }\n\n                    totalFramesReadOut += framesReadThisIterationOut;\n\n                    if (pRunningFramesOut != NULL) {\n                        pRunningFramesOut = ma_offset_ptr(pRunningFramesOut, framesReadThisIterationOut * ma_get_bytes_per_frame(pDecoder->outputFormat, pDecoder->outputChannels));\n                    }\n\n                    if (framesReadThisIterationIn == 0 && framesReadThisIterationOut == 0) {\n                        break;  /* We're done. */\n                    }\n                }\n            }\n        }\n    }\n\n    pDecoder->readPointerInPCMFrames += totalFramesReadOut;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesReadOut;\n    }\n\n    if (result == MA_SUCCESS && totalFramesReadOut == 0) {\n        result =  MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_decoder_seek_to_pcm_frame(ma_decoder* pDecoder, ma_uint64 frameIndex)\n{\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDecoder->pBackend != NULL) {\n        ma_result result;\n        ma_uint64 internalFrameIndex;\n        ma_uint32 internalSampleRate;\n        ma_uint64 currentFrameIndex;\n\n        result = ma_data_source_get_data_format(pDecoder->pBackend, NULL, NULL, &internalSampleRate, NULL, 0);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to retrieve the internal sample rate. */\n        }\n\n        if (internalSampleRate == pDecoder->outputSampleRate) {\n            internalFrameIndex = frameIndex;\n        } else {\n            internalFrameIndex = ma_calculate_frame_count_after_resampling(internalSampleRate, pDecoder->outputSampleRate, frameIndex);\n        }\n\n        /* Only seek if we're requesting a different frame to what we're currently sitting on. */\n        ma_data_source_get_cursor_in_pcm_frames(pDecoder->pBackend, &currentFrameIndex);\n        if (currentFrameIndex != internalFrameIndex) {\n            result = ma_data_source_seek_to_pcm_frame(pDecoder->pBackend, internalFrameIndex);\n            if (result == MA_SUCCESS) {\n                pDecoder->readPointerInPCMFrames = frameIndex;\n            }\n\n            /* Reset the data converter so that any cached data in the resampler is cleared. */\n            ma_data_converter_reset(&pDecoder->converter);\n        }\n\n        return result;\n    }\n\n    /* Should never get here, but if we do it means onSeekToPCMFrame was not set by the backend. */\n    return MA_INVALID_ARGS;\n}\n\nMA_API ma_result ma_decoder_get_data_format(ma_decoder* pDecoder, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFormat != NULL) {\n        *pFormat = pDecoder->outputFormat;\n    }\n\n    if (pChannels != NULL) {\n        *pChannels = pDecoder->outputChannels;\n    }\n\n    if (pSampleRate != NULL) {\n        *pSampleRate = pDecoder->outputSampleRate;\n    }\n\n    if (pChannelMap != NULL) {\n        ma_data_converter_get_output_channel_map(&pDecoder->converter, pChannelMap, channelMapCap);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_get_cursor_in_pcm_frames(ma_decoder* pDecoder, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = pDecoder->readPointerInPCMFrames;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decoder_get_length_in_pcm_frames(ma_decoder* pDecoder, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDecoder->pBackend != NULL) {\n        ma_result result;\n        ma_uint64 internalLengthInPCMFrames;\n        ma_uint32 internalSampleRate;\n\n        result = ma_data_source_get_length_in_pcm_frames(pDecoder->pBackend, &internalLengthInPCMFrames);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to retrieve the internal length. */\n        }\n\n        result = ma_data_source_get_data_format(pDecoder->pBackend, NULL, NULL, &internalSampleRate, NULL, 0);\n        if (result != MA_SUCCESS) {\n            return result;   /* Failed to retrieve the internal sample rate. */\n        }\n\n        if (internalSampleRate == pDecoder->outputSampleRate) {\n            *pLength = internalLengthInPCMFrames;\n        } else {\n            *pLength = ma_calculate_frame_count_after_resampling(pDecoder->outputSampleRate, internalSampleRate, internalLengthInPCMFrames);\n        }\n\n        return MA_SUCCESS;\n    } else {\n        return MA_NO_BACKEND;\n    }\n}\n\nMA_API ma_result ma_decoder_get_available_frames(ma_decoder* pDecoder, ma_uint64* pAvailableFrames)\n{\n    ma_result result;\n    ma_uint64 totalFrameCount;\n\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pDecoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_decoder_get_length_in_pcm_frames(pDecoder, &totalFrameCount);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (totalFrameCount <= pDecoder->readPointerInPCMFrames) {\n        *pAvailableFrames = 0;\n    } else {\n        *pAvailableFrames = totalFrameCount - pDecoder->readPointerInPCMFrames;\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_decoder__full_decode_and_uninit(ma_decoder* pDecoder, ma_decoder_config* pConfigOut, ma_uint64* pFrameCountOut, void** ppPCMFramesOut)\n{\n    ma_result result;\n    ma_uint64 totalFrameCount;\n    ma_uint64 bpf;\n    ma_uint64 dataCapInFrames;\n    void* pPCMFramesOut;\n\n    MA_ASSERT(pDecoder != NULL);\n\n    totalFrameCount = 0;\n    bpf = ma_get_bytes_per_frame(pDecoder->outputFormat, pDecoder->outputChannels);\n\n    /* The frame count is unknown until we try reading. Thus, we just run in a loop. */\n    dataCapInFrames = 0;\n    pPCMFramesOut = NULL;\n    for (;;) {\n        ma_uint64 frameCountToTryReading;\n        ma_uint64 framesJustRead;\n\n        /* Make room if there's not enough. */\n        if (totalFrameCount == dataCapInFrames) {\n            void* pNewPCMFramesOut;\n            ma_uint64 newDataCapInFrames = dataCapInFrames*2;\n            if (newDataCapInFrames == 0) {\n                newDataCapInFrames = 4096;\n            }\n\n            if ((newDataCapInFrames * bpf) > MA_SIZE_MAX) {\n                ma_free(pPCMFramesOut, &pDecoder->allocationCallbacks);\n                return MA_TOO_BIG;\n            }\n\n            pNewPCMFramesOut = (void*)ma_realloc(pPCMFramesOut, (size_t)(newDataCapInFrames * bpf), &pDecoder->allocationCallbacks);\n            if (pNewPCMFramesOut == NULL) {\n                ma_free(pPCMFramesOut, &pDecoder->allocationCallbacks);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            dataCapInFrames = newDataCapInFrames;\n            pPCMFramesOut = pNewPCMFramesOut;\n        }\n\n        frameCountToTryReading = dataCapInFrames - totalFrameCount;\n        MA_ASSERT(frameCountToTryReading > 0);\n\n        result = ma_decoder_read_pcm_frames(pDecoder, (ma_uint8*)pPCMFramesOut + (totalFrameCount * bpf), frameCountToTryReading, &framesJustRead);\n        totalFrameCount += framesJustRead;\n\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        if (framesJustRead < frameCountToTryReading) {\n            break;\n        }\n    }\n\n\n    if (pConfigOut != NULL) {\n        pConfigOut->format     = pDecoder->outputFormat;\n        pConfigOut->channels   = pDecoder->outputChannels;\n        pConfigOut->sampleRate = pDecoder->outputSampleRate;\n    }\n\n    if (ppPCMFramesOut != NULL) {\n        *ppPCMFramesOut = pPCMFramesOut;\n    } else {\n        ma_free(pPCMFramesOut, &pDecoder->allocationCallbacks);\n    }\n\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = totalFrameCount;\n    }\n\n    ma_decoder_uninit(pDecoder);\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_decode_from_vfs(ma_vfs* pVFS, const char* pFilePath, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut)\n{\n    ma_result result;\n    ma_decoder_config config;\n    ma_decoder decoder;\n\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = 0;\n    }\n    if (ppPCMFramesOut != NULL) {\n        *ppPCMFramesOut = NULL;\n    }\n\n    config = ma_decoder_config_init_copy(pConfig);\n\n    result = ma_decoder_init_vfs(pVFS, pFilePath, &config, &decoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_decoder__full_decode_and_uninit(&decoder, pConfig, pFrameCountOut, ppPCMFramesOut);\n\n    return result;\n}\n\nMA_API ma_result ma_decode_file(const char* pFilePath, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut)\n{\n    return ma_decode_from_vfs(NULL, pFilePath, pConfig, pFrameCountOut, ppPCMFramesOut);\n}\n\nMA_API ma_result ma_decode_memory(const void* pData, size_t dataSize, ma_decoder_config* pConfig, ma_uint64* pFrameCountOut, void** ppPCMFramesOut)\n{\n    ma_decoder_config config;\n    ma_decoder decoder;\n    ma_result result;\n\n    if (pFrameCountOut != NULL) {\n        *pFrameCountOut = 0;\n    }\n    if (ppPCMFramesOut != NULL) {\n        *ppPCMFramesOut = NULL;\n    }\n\n    if (pData == NULL || dataSize == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    config = ma_decoder_config_init_copy(pConfig);\n\n    result = ma_decoder_init_memory(pData, dataSize, &config, &decoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_decoder__full_decode_and_uninit(&decoder, pConfig, pFrameCountOut, ppPCMFramesOut);\n}\n#endif  /* MA_NO_DECODING */\n\n\n#ifndef MA_NO_ENCODING\n\n#if defined(MA_HAS_WAV)\nstatic size_t ma_encoder__internal_on_write_wav(void* pUserData, const void* pData, size_t bytesToWrite)\n{\n    ma_encoder* pEncoder = (ma_encoder*)pUserData;\n    size_t bytesWritten = 0;\n\n    MA_ASSERT(pEncoder != NULL);\n\n    pEncoder->onWrite(pEncoder, pData, bytesToWrite, &bytesWritten);\n    return bytesWritten;\n}\n\nstatic ma_bool32 ma_encoder__internal_on_seek_wav(void* pUserData, int offset, ma_dr_wav_seek_origin origin)\n{\n    ma_encoder* pEncoder = (ma_encoder*)pUserData;\n    ma_result result;\n    ma_seek_origin maSeekOrigin;\n\n    MA_ASSERT(pEncoder != NULL);\n\n    maSeekOrigin = ma_seek_origin_start;\n    if (origin == MA_DR_WAV_SEEK_CUR) {\n        maSeekOrigin = ma_seek_origin_current;\n    } else if (origin == MA_DR_WAV_SEEK_END) {\n        maSeekOrigin = ma_seek_origin_end;\n    }\n\n    result = pEncoder->onSeek(pEncoder, offset, maSeekOrigin);\n    if (result != MA_SUCCESS) {\n        return MA_FALSE;\n    } else {\n        return MA_TRUE;\n    }\n}\n\nstatic ma_result ma_encoder__on_init_wav(ma_encoder* pEncoder)\n{\n    ma_dr_wav_data_format wavFormat;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_dr_wav* pWav;\n\n    MA_ASSERT(pEncoder != NULL);\n\n    pWav = (ma_dr_wav*)ma_malloc(sizeof(*pWav), &pEncoder->config.allocationCallbacks);\n    if (pWav == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    wavFormat.container     = ma_dr_wav_container_riff;\n    wavFormat.channels      = pEncoder->config.channels;\n    wavFormat.sampleRate    = pEncoder->config.sampleRate;\n    wavFormat.bitsPerSample = ma_get_bytes_per_sample(pEncoder->config.format) * 8;\n    if (pEncoder->config.format == ma_format_f32) {\n        wavFormat.format    = MA_DR_WAVE_FORMAT_IEEE_FLOAT;\n    } else {\n        wavFormat.format    = MA_DR_WAVE_FORMAT_PCM;\n    }\n\n    allocationCallbacks.pUserData = pEncoder->config.allocationCallbacks.pUserData;\n    allocationCallbacks.onMalloc  = pEncoder->config.allocationCallbacks.onMalloc;\n    allocationCallbacks.onRealloc = pEncoder->config.allocationCallbacks.onRealloc;\n    allocationCallbacks.onFree    = pEncoder->config.allocationCallbacks.onFree;\n\n    if (!ma_dr_wav_init_write(pWav, &wavFormat, ma_encoder__internal_on_write_wav, ma_encoder__internal_on_seek_wav, pEncoder, &allocationCallbacks)) {\n        return MA_ERROR;\n    }\n\n    pEncoder->pInternalEncoder = pWav;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_encoder__on_uninit_wav(ma_encoder* pEncoder)\n{\n    ma_dr_wav* pWav;\n\n    MA_ASSERT(pEncoder != NULL);\n\n    pWav = (ma_dr_wav*)pEncoder->pInternalEncoder;\n    MA_ASSERT(pWav != NULL);\n\n    ma_dr_wav_uninit(pWav);\n    ma_free(pWav, &pEncoder->config.allocationCallbacks);\n}\n\nstatic ma_result ma_encoder__on_write_pcm_frames_wav(ma_encoder* pEncoder, const void* pFramesIn, ma_uint64 frameCount, ma_uint64* pFramesWritten)\n{\n    ma_dr_wav* pWav;\n    ma_uint64 framesWritten;\n\n    MA_ASSERT(pEncoder != NULL);\n\n    pWav = (ma_dr_wav*)pEncoder->pInternalEncoder;\n    MA_ASSERT(pWav != NULL);\n\n    framesWritten = ma_dr_wav_write_pcm_frames(pWav, frameCount, pFramesIn);\n\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = framesWritten;\n    }\n\n    return MA_SUCCESS;\n}\n#endif\n\nMA_API ma_encoder_config ma_encoder_config_init(ma_encoding_format encodingFormat, ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    ma_encoder_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.encodingFormat = encodingFormat;\n    config.format = format;\n    config.channels = channels;\n    config.sampleRate = sampleRate;\n\n    return config;\n}\n\nMA_API ma_result ma_encoder_preinit(const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    ma_result result;\n\n    if (pEncoder == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pEncoder);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->format == ma_format_unknown || pConfig->channels == 0 || pConfig->sampleRate == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pEncoder->config = *pConfig;\n\n    result = ma_allocation_callbacks_init_copy(&pEncoder->config.allocationCallbacks, &pConfig->allocationCallbacks);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_encoder_init__internal(ma_encoder_write_proc onWrite, ma_encoder_seek_proc onSeek, void* pUserData, ma_encoder* pEncoder)\n{\n    ma_result result = MA_SUCCESS;\n\n    /* This assumes ma_encoder_preinit() has been called prior. */\n    MA_ASSERT(pEncoder != NULL);\n\n    if (onWrite == NULL || onSeek == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pEncoder->onWrite   = onWrite;\n    pEncoder->onSeek    = onSeek;\n    pEncoder->pUserData = pUserData;\n\n    switch (pEncoder->config.encodingFormat)\n    {\n        case ma_encoding_format_wav:\n        {\n        #if defined(MA_HAS_WAV)\n            pEncoder->onInit           = ma_encoder__on_init_wav;\n            pEncoder->onUninit         = ma_encoder__on_uninit_wav;\n            pEncoder->onWritePCMFrames = ma_encoder__on_write_pcm_frames_wav;\n        #else\n            result = MA_NO_BACKEND;\n        #endif\n        } break;\n\n        default:\n        {\n            result = MA_INVALID_ARGS;\n        } break;\n    }\n\n    /* Getting here means we should have our backend callbacks set up. */\n    if (result == MA_SUCCESS) {\n        result = pEncoder->onInit(pEncoder);\n    }\n\n    return result;\n}\n\nstatic ma_result ma_encoder__on_write_vfs(ma_encoder* pEncoder, const void* pBufferIn, size_t bytesToWrite, size_t* pBytesWritten)\n{\n    return ma_vfs_or_default_write(pEncoder->data.vfs.pVFS, pEncoder->data.vfs.file, pBufferIn, bytesToWrite, pBytesWritten);\n}\n\nstatic ma_result ma_encoder__on_seek_vfs(ma_encoder* pEncoder, ma_int64 offset, ma_seek_origin origin)\n{\n    return ma_vfs_or_default_seek(pEncoder->data.vfs.pVFS, pEncoder->data.vfs.file, offset, origin);\n}\n\nMA_API ma_result ma_encoder_init_vfs(ma_vfs* pVFS, const char* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    ma_result result;\n    ma_vfs_file file;\n\n    result = ma_encoder_preinit(pConfig, pEncoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Now open the file. If this fails we don't need to uninitialize the encoder. */\n    result = ma_vfs_or_default_open(pVFS, pFilePath, MA_OPEN_MODE_WRITE, &file);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pEncoder->data.vfs.pVFS = pVFS;\n    pEncoder->data.vfs.file = file;\n\n    result = ma_encoder_init__internal(ma_encoder__on_write_vfs, ma_encoder__on_seek_vfs, NULL, pEncoder);\n    if (result != MA_SUCCESS) {\n        ma_vfs_or_default_close(pVFS, file);\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_encoder_init_vfs_w(ma_vfs* pVFS, const wchar_t* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    ma_result result;\n    ma_vfs_file file;\n\n    result = ma_encoder_preinit(pConfig, pEncoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Now open the file. If this fails we don't need to uninitialize the encoder. */\n    result = ma_vfs_or_default_open_w(pVFS, pFilePath, MA_OPEN_MODE_WRITE, &file);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pEncoder->data.vfs.pVFS = pVFS;\n    pEncoder->data.vfs.file = file;\n\n    result = ma_encoder_init__internal(ma_encoder__on_write_vfs, ma_encoder__on_seek_vfs, NULL, pEncoder);\n    if (result != MA_SUCCESS) {\n        ma_vfs_or_default_close(pVFS, file);\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_encoder_init_file(const char* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    return ma_encoder_init_vfs(NULL, pFilePath, pConfig, pEncoder);\n}\n\nMA_API ma_result ma_encoder_init_file_w(const wchar_t* pFilePath, const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    return ma_encoder_init_vfs_w(NULL, pFilePath, pConfig, pEncoder);\n}\n\nMA_API ma_result ma_encoder_init(ma_encoder_write_proc onWrite, ma_encoder_seek_proc onSeek, void* pUserData, const ma_encoder_config* pConfig, ma_encoder* pEncoder)\n{\n    ma_result result;\n\n    result = ma_encoder_preinit(pConfig, pEncoder);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_encoder_init__internal(onWrite, onSeek, pUserData, pEncoder);\n}\n\n\nMA_API void ma_encoder_uninit(ma_encoder* pEncoder)\n{\n    if (pEncoder == NULL) {\n        return;\n    }\n\n    if (pEncoder->onUninit) {\n        pEncoder->onUninit(pEncoder);\n    }\n\n    /* If we have a file handle, close it. */\n    if (pEncoder->onWrite == ma_encoder__on_write_vfs) {\n        ma_vfs_or_default_close(pEncoder->data.vfs.pVFS, pEncoder->data.vfs.file);\n        pEncoder->data.vfs.file = NULL;\n    }\n}\n\n\nMA_API ma_result ma_encoder_write_pcm_frames(ma_encoder* pEncoder, const void* pFramesIn, ma_uint64 frameCount, ma_uint64* pFramesWritten)\n{\n    if (pFramesWritten != NULL) {\n        *pFramesWritten = 0;\n    }\n\n    if (pEncoder == NULL || pFramesIn == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return pEncoder->onWritePCMFrames(pEncoder, pFramesIn, frameCount, pFramesWritten);\n}\n#endif  /* MA_NO_ENCODING */\n\n\n\n/**************************************************************************************************************************************************************\n\nGeneration\n\n**************************************************************************************************************************************************************/\n#ifndef MA_NO_GENERATION\nMA_API ma_waveform_config ma_waveform_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, ma_waveform_type type, double amplitude, double frequency)\n{\n    ma_waveform_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.type       = type;\n    config.amplitude  = amplitude;\n    config.frequency  = frequency;\n\n    return config;\n}\n\nstatic ma_result ma_waveform__data_source_on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_waveform_read_pcm_frames((ma_waveform*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_waveform__data_source_on_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_waveform_seek_to_pcm_frame((ma_waveform*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_waveform__data_source_on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_waveform* pWaveform = (ma_waveform*)pDataSource;\n\n    *pFormat     = pWaveform->config.format;\n    *pChannels   = pWaveform->config.channels;\n    *pSampleRate = pWaveform->config.sampleRate;\n    ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pWaveform->config.channels);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_waveform__data_source_on_get_cursor(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    ma_waveform* pWaveform = (ma_waveform*)pDataSource;\n\n    *pCursor = (ma_uint64)(pWaveform->time / pWaveform->advance);\n\n    return MA_SUCCESS;\n}\n\nstatic double ma_waveform__calculate_advance(ma_uint32 sampleRate, double frequency)\n{\n    return (1.0 / (sampleRate / frequency));\n}\n\nstatic void ma_waveform__update_advance(ma_waveform* pWaveform)\n{\n    pWaveform->advance = ma_waveform__calculate_advance(pWaveform->config.sampleRate, pWaveform->config.frequency);\n}\n\nstatic ma_data_source_vtable g_ma_waveform_data_source_vtable =\n{\n    ma_waveform__data_source_on_read,\n    ma_waveform__data_source_on_seek,\n    ma_waveform__data_source_on_get_data_format,\n    ma_waveform__data_source_on_get_cursor,\n    NULL,   /* onGetLength. There's no notion of a length in waveforms. */\n    NULL,   /* onSetLooping */\n    0\n};\n\nMA_API ma_result ma_waveform_init(const ma_waveform_config* pConfig, ma_waveform* pWaveform)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pWaveform);\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_waveform_data_source_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pWaveform->ds);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pWaveform->config  = *pConfig;\n    pWaveform->advance = ma_waveform__calculate_advance(pWaveform->config.sampleRate, pWaveform->config.frequency);\n    pWaveform->time    = 0;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_waveform_uninit(ma_waveform* pWaveform)\n{\n    if (pWaveform == NULL) {\n        return;\n    }\n\n    ma_data_source_uninit(&pWaveform->ds);\n}\n\nMA_API ma_result ma_waveform_set_amplitude(ma_waveform* pWaveform, double amplitude)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.amplitude = amplitude;\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_waveform_set_frequency(ma_waveform* pWaveform, double frequency)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.frequency = frequency;\n    ma_waveform__update_advance(pWaveform);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_waveform_set_type(ma_waveform* pWaveform, ma_waveform_type type)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.type = type;\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_waveform_set_sample_rate(ma_waveform* pWaveform, ma_uint32 sampleRate)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.sampleRate = sampleRate;\n    ma_waveform__update_advance(pWaveform);\n\n    return MA_SUCCESS;\n}\n\nstatic float ma_waveform_sine_f32(double time, double amplitude)\n{\n    return (float)(ma_sind(MA_TAU_D * time) * amplitude);\n}\n\nstatic ma_int16 ma_waveform_sine_s16(double time, double amplitude)\n{\n    return ma_pcm_sample_f32_to_s16(ma_waveform_sine_f32(time, amplitude));\n}\n\nstatic float ma_waveform_square_f32(double time, double dutyCycle, double amplitude)\n{\n    double f = time - (ma_int64)time;\n    double r;\n\n    if (f < dutyCycle) {\n        r =  amplitude;\n    } else {\n        r = -amplitude;\n    }\n\n    return (float)r;\n}\n\nstatic ma_int16 ma_waveform_square_s16(double time, double dutyCycle, double amplitude)\n{\n    return ma_pcm_sample_f32_to_s16(ma_waveform_square_f32(time, dutyCycle, amplitude));\n}\n\nstatic float ma_waveform_triangle_f32(double time, double amplitude)\n{\n    double f = time - (ma_int64)time;\n    double r;\n\n    r = 2 * ma_abs(2 * (f - 0.5)) - 1;\n\n    return (float)(r * amplitude);\n}\n\nstatic ma_int16 ma_waveform_triangle_s16(double time, double amplitude)\n{\n    return ma_pcm_sample_f32_to_s16(ma_waveform_triangle_f32(time, amplitude));\n}\n\nstatic float ma_waveform_sawtooth_f32(double time, double amplitude)\n{\n    double f = time - (ma_int64)time;\n    double r;\n\n    r = 2 * (f - 0.5);\n\n    return (float)(r * amplitude);\n}\n\nstatic ma_int16 ma_waveform_sawtooth_s16(double time, double amplitude)\n{\n    return ma_pcm_sample_f32_to_s16(ma_waveform_sawtooth_f32(time, amplitude));\n}\n\nstatic void ma_waveform_read_pcm_frames__sine(ma_waveform* pWaveform, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint64 iChannel;\n    ma_uint32 bps = ma_get_bytes_per_sample(pWaveform->config.format);\n    ma_uint32 bpf = bps * pWaveform->config.channels;\n\n    MA_ASSERT(pWaveform  != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pWaveform->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_sine_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutF32[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else if (pWaveform->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            ma_int16 s = ma_waveform_sine_s16(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutS16[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else {\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_sine_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pWaveform->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n            }\n        }\n    }\n}\n\nstatic void ma_waveform_read_pcm_frames__square(ma_waveform* pWaveform, double dutyCycle, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint64 iChannel;\n    ma_uint32 bps = ma_get_bytes_per_sample(pWaveform->config.format);\n    ma_uint32 bpf = bps * pWaveform->config.channels;\n\n    MA_ASSERT(pWaveform  != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pWaveform->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_square_f32(pWaveform->time, dutyCycle, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutF32[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else if (pWaveform->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            ma_int16 s = ma_waveform_square_s16(pWaveform->time, dutyCycle, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutS16[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else {\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_square_f32(pWaveform->time, dutyCycle, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pWaveform->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n            }\n        }\n    }\n}\n\nstatic void ma_waveform_read_pcm_frames__triangle(ma_waveform* pWaveform, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint64 iChannel;\n    ma_uint32 bps = ma_get_bytes_per_sample(pWaveform->config.format);\n    ma_uint32 bpf = bps * pWaveform->config.channels;\n\n    MA_ASSERT(pWaveform  != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pWaveform->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_triangle_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutF32[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else if (pWaveform->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            ma_int16 s = ma_waveform_triangle_s16(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutS16[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else {\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_triangle_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pWaveform->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n            }\n        }\n    }\n}\n\nstatic void ma_waveform_read_pcm_frames__sawtooth(ma_waveform* pWaveform, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint64 iChannel;\n    ma_uint32 bps = ma_get_bytes_per_sample(pWaveform->config.format);\n    ma_uint32 bpf = bps * pWaveform->config.channels;\n\n    MA_ASSERT(pWaveform  != NULL);\n    MA_ASSERT(pFramesOut != NULL);\n\n    if (pWaveform->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_sawtooth_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutF32[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else if (pWaveform->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            ma_int16 s = ma_waveform_sawtooth_s16(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                pFramesOutS16[iFrame*pWaveform->config.channels + iChannel] = s;\n            }\n        }\n    } else {\n        for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n            float s = ma_waveform_sawtooth_f32(pWaveform->time, pWaveform->config.amplitude);\n            pWaveform->time += pWaveform->advance;\n\n            for (iChannel = 0; iChannel < pWaveform->config.channels; iChannel += 1) {\n                ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pWaveform->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n            }\n        }\n    }\n}\n\nMA_API ma_result ma_waveform_read_pcm_frames(ma_waveform* pWaveform, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFramesOut != NULL) {\n        switch (pWaveform->config.type)\n        {\n            case ma_waveform_type_sine:\n            {\n                ma_waveform_read_pcm_frames__sine(pWaveform, pFramesOut, frameCount);\n            } break;\n\n            case ma_waveform_type_square:\n            {\n                ma_waveform_read_pcm_frames__square(pWaveform, 0.5, pFramesOut, frameCount);\n            } break;\n\n            case ma_waveform_type_triangle:\n            {\n                ma_waveform_read_pcm_frames__triangle(pWaveform, pFramesOut, frameCount);\n            } break;\n\n            case ma_waveform_type_sawtooth:\n            {\n                ma_waveform_read_pcm_frames__sawtooth(pWaveform, pFramesOut, frameCount);\n            } break;\n\n            default: return MA_INVALID_OPERATION;   /* Unknown waveform type. */\n        }\n    } else {\n        pWaveform->time += pWaveform->advance * (ma_int64)frameCount; /* Cast to int64 required for VC6. Won't affect anything in practice. */\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_waveform_seek_to_pcm_frame(ma_waveform* pWaveform, ma_uint64 frameIndex)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->time = pWaveform->advance * (ma_int64)frameIndex;    /* Casting for VC6. Won't be an issue in practice. */\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_pulsewave_config ma_pulsewave_config_init(ma_format format, ma_uint32 channels, ma_uint32 sampleRate, double dutyCycle, double amplitude, double frequency)\n{\n    ma_pulsewave_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.format     = format;\n    config.channels   = channels;\n    config.sampleRate = sampleRate;\n    config.dutyCycle  = dutyCycle;\n    config.amplitude  = amplitude;\n    config.frequency  = frequency;\n\n    return config;\n}\n\nMA_API ma_result ma_pulsewave_init(const ma_pulsewave_config* pConfig, ma_pulsewave* pWaveform)\n{\n    ma_result result;\n    ma_waveform_config config;\n\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pWaveform);\n\n    config = ma_waveform_config_init(\n        pConfig->format,\n        pConfig->channels,\n        pConfig->sampleRate,\n        ma_waveform_type_square,\n        pConfig->amplitude,\n        pConfig->frequency\n    );\n\n    result = ma_waveform_init(&config, &pWaveform->waveform);\n    ma_pulsewave_set_duty_cycle(pWaveform, pConfig->dutyCycle);\n\n    return result;\n}\n\nMA_API void ma_pulsewave_uninit(ma_pulsewave* pWaveform)\n{\n    if (pWaveform == NULL) {\n        return;\n    }\n\n    ma_waveform_uninit(&pWaveform->waveform);\n}\n\nMA_API ma_result ma_pulsewave_read_pcm_frames(ma_pulsewave* pWaveform, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pFramesOut != NULL) {\n        ma_waveform_read_pcm_frames__square(&pWaveform->waveform, pWaveform->config.dutyCycle, pFramesOut, frameCount);\n    } else {\n        pWaveform->waveform.time += pWaveform->waveform.advance * (ma_int64)frameCount; /* Cast to int64 required for VC6. Won't affect anything in practice. */\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = frameCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pulsewave_seek_to_pcm_frame(ma_pulsewave* pWaveform, ma_uint64 frameIndex)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_waveform_seek_to_pcm_frame(&pWaveform->waveform, frameIndex);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pulsewave_set_amplitude(ma_pulsewave* pWaveform, double amplitude)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.amplitude = amplitude;\n    ma_waveform_set_amplitude(&pWaveform->waveform, amplitude);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pulsewave_set_frequency(ma_pulsewave* pWaveform, double frequency)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.frequency = frequency;\n    ma_waveform_set_frequency(&pWaveform->waveform, frequency);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pulsewave_set_sample_rate(ma_pulsewave* pWaveform, ma_uint32 sampleRate)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.sampleRate = sampleRate;\n    ma_waveform_set_sample_rate(&pWaveform->waveform, sampleRate);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_pulsewave_set_duty_cycle(ma_pulsewave* pWaveform, double dutyCycle)\n{\n    if (pWaveform == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pWaveform->config.dutyCycle = dutyCycle;\n\n    return MA_SUCCESS;\n}\n\n\n\nMA_API ma_noise_config ma_noise_config_init(ma_format format, ma_uint32 channels, ma_noise_type type, ma_int32 seed, double amplitude)\n{\n    ma_noise_config config;\n    MA_ZERO_OBJECT(&config);\n\n    config.format    = format;\n    config.channels  = channels;\n    config.type      = type;\n    config.seed      = seed;\n    config.amplitude = amplitude;\n\n    if (config.seed == 0) {\n        config.seed = MA_DEFAULT_LCG_SEED;\n    }\n\n    return config;\n}\n\n\nstatic ma_result ma_noise__data_source_on_read(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_noise_read_pcm_frames((ma_noise*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_noise__data_source_on_seek(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    /* No-op. Just pretend to be successful. */\n    (void)pDataSource;\n    (void)frameIndex;\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_noise__data_source_on_get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    ma_noise* pNoise = (ma_noise*)pDataSource;\n\n    *pFormat     = pNoise->config.format;\n    *pChannels   = pNoise->config.channels;\n    *pSampleRate = 0;   /* There is no notion of sample rate with noise generation. */\n    ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pNoise->config.channels);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_data_source_vtable g_ma_noise_data_source_vtable =\n{\n    ma_noise__data_source_on_read,\n    ma_noise__data_source_on_seek,  /* No-op for noise. */\n    ma_noise__data_source_on_get_data_format,\n    NULL,   /* onGetCursor. No notion of a cursor for noise. */\n    NULL,   /* onGetLength. No notion of a length for noise. */\n    NULL,   /* onSetLooping */\n    0\n};\n\n\n#ifndef MA_PINK_NOISE_BIN_SIZE\n#define MA_PINK_NOISE_BIN_SIZE 16\n#endif\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    struct\n    {\n        size_t binOffset;\n        size_t accumulationOffset;\n        size_t counterOffset;\n    } pink;\n    struct\n    {\n        size_t accumulationOffset;\n    } brownian;\n} ma_noise_heap_layout;\n\nstatic ma_result ma_noise_get_heap_layout(const ma_noise_config* pConfig, ma_noise_heap_layout* pHeapLayout)\n{\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Pink. */\n    if (pConfig->type == ma_noise_type_pink) {\n        /* bin */\n        pHeapLayout->pink.binOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += sizeof(double*) * pConfig->channels;\n        pHeapLayout->sizeInBytes += sizeof(double ) * pConfig->channels * MA_PINK_NOISE_BIN_SIZE;\n\n        /* accumulation */\n        pHeapLayout->pink.accumulationOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += sizeof(double) * pConfig->channels;\n\n        /* counter */\n        pHeapLayout->pink.counterOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += sizeof(ma_uint32) * pConfig->channels;\n    }\n\n    /* Brownian. */\n    if (pConfig->type == ma_noise_type_brownian) {\n        /* accumulation */\n        pHeapLayout->brownian.accumulationOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += sizeof(double) * pConfig->channels;\n    }\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_noise_get_heap_size(const ma_noise_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_noise_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_noise_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_noise_init_preallocated(const ma_noise_config* pConfig, void* pHeap, ma_noise* pNoise)\n{\n    ma_result result;\n    ma_noise_heap_layout heapLayout;\n    ma_data_source_config dataSourceConfig;\n    ma_uint32 iChannel;\n\n    if (pNoise == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNoise);\n\n    result = ma_noise_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pNoise->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pNoise->_pHeap, heapLayout.sizeInBytes);\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_noise_data_source_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pNoise->ds);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pNoise->config = *pConfig;\n    ma_lcg_seed(&pNoise->lcg, pConfig->seed);\n\n    if (pNoise->config.type == ma_noise_type_pink) {\n        pNoise->state.pink.bin          = (double**  )ma_offset_ptr(pHeap, heapLayout.pink.binOffset);\n        pNoise->state.pink.accumulation = (double*   )ma_offset_ptr(pHeap, heapLayout.pink.accumulationOffset);\n        pNoise->state.pink.counter      = (ma_uint32*)ma_offset_ptr(pHeap, heapLayout.pink.counterOffset);\n\n        for (iChannel = 0; iChannel < pConfig->channels; iChannel += 1) {\n            pNoise->state.pink.bin[iChannel]          = (double*)ma_offset_ptr(pHeap, heapLayout.pink.binOffset + (sizeof(double*) * pConfig->channels) + (sizeof(double) * MA_PINK_NOISE_BIN_SIZE * iChannel));\n            pNoise->state.pink.accumulation[iChannel] = 0;\n            pNoise->state.pink.counter[iChannel]      = 1;\n        }\n    }\n\n    if (pNoise->config.type == ma_noise_type_brownian) {\n        pNoise->state.brownian.accumulation = (double*)ma_offset_ptr(pHeap, heapLayout.brownian.accumulationOffset);\n\n        for (iChannel = 0; iChannel < pConfig->channels; iChannel += 1) {\n            pNoise->state.brownian.accumulation[iChannel] = 0;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_noise_init(const ma_noise_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_noise* pNoise)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_noise_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_noise_init_preallocated(pConfig, pHeap, pNoise);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pNoise->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_noise_uninit(ma_noise* pNoise, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pNoise == NULL) {\n        return;\n    }\n\n    ma_data_source_uninit(&pNoise->ds);\n\n    if (pNoise->_ownsHeap) {\n        ma_free(pNoise->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_result ma_noise_set_amplitude(ma_noise* pNoise, double amplitude)\n{\n    if (pNoise == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pNoise->config.amplitude = amplitude;\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_noise_set_seed(ma_noise* pNoise, ma_int32 seed)\n{\n    if (pNoise == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pNoise->lcg.state = seed;\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_noise_set_type(ma_noise* pNoise, ma_noise_type type)\n{\n    if (pNoise == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    This function should never have been implemented in the first place. Changing the type dynamically is not\n    supported. Instead you need to uninitialize and reinitialize a fresh `ma_noise` object. This function\n    will be removed in version 0.12.\n    */\n    MA_ASSERT(MA_FALSE);\n    (void)type;\n\n    return MA_INVALID_OPERATION;\n}\n\nstatic MA_INLINE float ma_noise_f32_white(ma_noise* pNoise)\n{\n    return (float)(ma_lcg_rand_f64(&pNoise->lcg) * pNoise->config.amplitude);\n}\n\nstatic MA_INLINE ma_int16 ma_noise_s16_white(ma_noise* pNoise)\n{\n    return ma_pcm_sample_f32_to_s16(ma_noise_f32_white(pNoise));\n}\n\nstatic MA_INLINE ma_uint64 ma_noise_read_pcm_frames__white(ma_noise* pNoise, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannel;\n    const ma_uint32 channels = pNoise->config.channels;\n    MA_ASSUME(channels > 0);\n\n    if (pNoise->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_white(pNoise);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = ma_noise_f32_white(pNoise);\n                }\n            }\n        }\n    } else if (pNoise->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                ma_int16 s = ma_noise_s16_white(pNoise);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = ma_noise_s16_white(pNoise);\n                }\n            }\n        }\n    } else {\n        const ma_uint32 bps = ma_get_bytes_per_sample(pNoise->config.format);\n        const ma_uint32 bpf = bps * channels;\n\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_white(pNoise);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    float s = ma_noise_f32_white(pNoise);\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        }\n    }\n\n    return frameCount;\n}\n\n\nstatic MA_INLINE unsigned int ma_tzcnt32(unsigned int x)\n{\n    unsigned int n;\n\n    /* Special case for odd numbers since they should happen about half the time. */\n    if (x & 0x1)  {\n        return 0;\n    }\n\n    if (x == 0) {\n        return sizeof(x) << 3;\n    }\n\n    n = 1;\n    if ((x & 0x0000FFFF) == 0) { x >>= 16; n += 16; }\n    if ((x & 0x000000FF) == 0) { x >>=  8; n +=  8; }\n    if ((x & 0x0000000F) == 0) { x >>=  4; n +=  4; }\n    if ((x & 0x00000003) == 0) { x >>=  2; n +=  2; }\n    n -= x & 0x00000001;\n\n    return n;\n}\n\n/*\nPink noise generation based on Tonic (public domain) with modifications. https://github.com/TonicAudio/Tonic/blob/master/src/Tonic/Noise.h\n\nThis is basically _the_ reference for pink noise from what I've found: http://www.firstpr.com.au/dsp/pink-noise/\n*/\nstatic MA_INLINE float ma_noise_f32_pink(ma_noise* pNoise, ma_uint32 iChannel)\n{\n    double result;\n    double binPrev;\n    double binNext;\n    unsigned int ibin;\n\n    ibin = ma_tzcnt32(pNoise->state.pink.counter[iChannel]) & (MA_PINK_NOISE_BIN_SIZE - 1);\n\n    binPrev = pNoise->state.pink.bin[iChannel][ibin];\n    binNext = ma_lcg_rand_f64(&pNoise->lcg);\n    pNoise->state.pink.bin[iChannel][ibin] = binNext;\n\n    pNoise->state.pink.accumulation[iChannel] += (binNext - binPrev);\n    pNoise->state.pink.counter[iChannel]      += 1;\n\n    result = (ma_lcg_rand_f64(&pNoise->lcg) + pNoise->state.pink.accumulation[iChannel]);\n    result /= 10;\n\n    return (float)(result * pNoise->config.amplitude);\n}\n\nstatic MA_INLINE ma_int16 ma_noise_s16_pink(ma_noise* pNoise, ma_uint32 iChannel)\n{\n    return ma_pcm_sample_f32_to_s16(ma_noise_f32_pink(pNoise, iChannel));\n}\n\nstatic MA_INLINE ma_uint64 ma_noise_read_pcm_frames__pink(ma_noise* pNoise, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannel;\n    const ma_uint32 channels = pNoise->config.channels;\n    MA_ASSUME(channels > 0);\n\n    if (pNoise->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_pink(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = ma_noise_f32_pink(pNoise, iChannel);\n                }\n            }\n        }\n    } else if (pNoise->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                ma_int16 s = ma_noise_s16_pink(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = ma_noise_s16_pink(pNoise, iChannel);\n                }\n            }\n        }\n    } else {\n        const ma_uint32 bps = ma_get_bytes_per_sample(pNoise->config.format);\n        const ma_uint32 bpf = bps * channels;\n\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_pink(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    float s = ma_noise_f32_pink(pNoise, iChannel);\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        }\n    }\n\n    return frameCount;\n}\n\n\nstatic MA_INLINE float ma_noise_f32_brownian(ma_noise* pNoise, ma_uint32 iChannel)\n{\n    double result;\n\n    result = (ma_lcg_rand_f64(&pNoise->lcg) + pNoise->state.brownian.accumulation[iChannel]);\n    result /= 1.005; /* Don't escape the -1..1 range on average. */\n\n    pNoise->state.brownian.accumulation[iChannel] = result;\n    result /= 20;\n\n    return (float)(result * pNoise->config.amplitude);\n}\n\nstatic MA_INLINE ma_int16 ma_noise_s16_brownian(ma_noise* pNoise, ma_uint32 iChannel)\n{\n    return ma_pcm_sample_f32_to_s16(ma_noise_f32_brownian(pNoise, iChannel));\n}\n\nstatic MA_INLINE ma_uint64 ma_noise_read_pcm_frames__brownian(ma_noise* pNoise, void* pFramesOut, ma_uint64 frameCount)\n{\n    ma_uint64 iFrame;\n    ma_uint32 iChannel;\n    const ma_uint32 channels = pNoise->config.channels;\n    MA_ASSUME(channels > 0);\n\n    if (pNoise->config.format == ma_format_f32) {\n        float* pFramesOutF32 = (float*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_brownian(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutF32[iFrame*channels + iChannel] = ma_noise_f32_brownian(pNoise, iChannel);\n                }\n            }\n        }\n    } else if (pNoise->config.format == ma_format_s16) {\n        ma_int16* pFramesOutS16 = (ma_int16*)pFramesOut;\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                ma_int16 s = ma_noise_s16_brownian(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = s;\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    pFramesOutS16[iFrame*channels + iChannel] = ma_noise_s16_brownian(pNoise, iChannel);\n                }\n            }\n        }\n    } else {\n        const ma_uint32 bps = ma_get_bytes_per_sample(pNoise->config.format);\n        const ma_uint32 bpf = bps * channels;\n\n        if (pNoise->config.duplicateChannels) {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                float s = ma_noise_f32_brownian(pNoise, 0);\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        } else {\n            for (iFrame = 0; iFrame < frameCount; iFrame += 1) {\n                for (iChannel = 0; iChannel < channels; iChannel += 1) {\n                    float s = ma_noise_f32_brownian(pNoise, iChannel);\n                    ma_pcm_convert(ma_offset_ptr(pFramesOut, iFrame*bpf + iChannel*bps), pNoise->config.format, &s, ma_format_f32, 1, ma_dither_mode_none);\n                }\n            }\n        }\n    }\n\n    return frameCount;\n}\n\nMA_API ma_result ma_noise_read_pcm_frames(ma_noise* pNoise, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_uint64 framesRead = 0;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pNoise == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The output buffer is allowed to be NULL. Since we aren't tracking cursors or anything we can just do nothing and pretend to be successful. */\n    if (pFramesOut == NULL) {\n        framesRead = frameCount;\n    } else {\n        switch (pNoise->config.type) {\n            case ma_noise_type_white:    framesRead = ma_noise_read_pcm_frames__white   (pNoise, pFramesOut, frameCount); break;\n            case ma_noise_type_pink:     framesRead = ma_noise_read_pcm_frames__pink    (pNoise, pFramesOut, frameCount); break;\n            case ma_noise_type_brownian: framesRead = ma_noise_read_pcm_frames__brownian(pNoise, pFramesOut, frameCount); break;\n            default: return MA_INVALID_OPERATION;   /* Unknown noise type. */\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = framesRead;\n    }\n\n    return MA_SUCCESS;\n}\n#endif /* MA_NO_GENERATION */\n\n\n\n#ifndef MA_NO_RESOURCE_MANAGER\n#ifndef MA_RESOURCE_MANAGER_PAGE_SIZE_IN_MILLISECONDS\n#define MA_RESOURCE_MANAGER_PAGE_SIZE_IN_MILLISECONDS   1000\n#endif\n\n#ifndef MA_JOB_TYPE_RESOURCE_MANAGER_QUEUE_CAPACITY\n#define MA_JOB_TYPE_RESOURCE_MANAGER_QUEUE_CAPACITY          1024\n#endif\n\nMA_API ma_resource_manager_pipeline_notifications ma_resource_manager_pipeline_notifications_init(void)\n{\n    ma_resource_manager_pipeline_notifications notifications;\n\n    MA_ZERO_OBJECT(&notifications);\n\n    return notifications;\n}\n\nstatic void ma_resource_manager_pipeline_notifications_signal_all_notifications(const ma_resource_manager_pipeline_notifications* pPipelineNotifications)\n{\n    if (pPipelineNotifications == NULL) {\n        return;\n    }\n\n    if (pPipelineNotifications->init.pNotification) { ma_async_notification_signal(pPipelineNotifications->init.pNotification); }\n    if (pPipelineNotifications->done.pNotification) { ma_async_notification_signal(pPipelineNotifications->done.pNotification); }\n}\n\nstatic void ma_resource_manager_pipeline_notifications_acquire_all_fences(const ma_resource_manager_pipeline_notifications* pPipelineNotifications)\n{\n    if (pPipelineNotifications == NULL) {\n        return;\n    }\n\n    if (pPipelineNotifications->init.pFence != NULL) { ma_fence_acquire(pPipelineNotifications->init.pFence); }\n    if (pPipelineNotifications->done.pFence != NULL) { ma_fence_acquire(pPipelineNotifications->done.pFence); }\n}\n\nstatic void ma_resource_manager_pipeline_notifications_release_all_fences(const ma_resource_manager_pipeline_notifications* pPipelineNotifications)\n{\n    if (pPipelineNotifications == NULL) {\n        return;\n    }\n\n    if (pPipelineNotifications->init.pFence != NULL) { ma_fence_release(pPipelineNotifications->init.pFence); }\n    if (pPipelineNotifications->done.pFence != NULL) { ma_fence_release(pPipelineNotifications->done.pFence); }\n}\n\n\n\n#ifndef MA_DEFAULT_HASH_SEED\n#define MA_DEFAULT_HASH_SEED    42\n#endif\n\n/* MurmurHash3. Based on code from https://github.com/PeterScott/murmur3/blob/master/murmur3.c (public domain). */\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic push\n    #if __GNUC__ >= 7\n    #pragma GCC diagnostic ignored \"-Wimplicit-fallthrough\"\n    #endif\n#endif\n\nstatic MA_INLINE ma_uint32 ma_rotl32(ma_uint32 x, ma_int8 r)\n{\n    return (x << r) | (x >> (32 - r));\n}\n\nstatic MA_INLINE ma_uint32 ma_hash_getblock(const ma_uint32* blocks, int i)\n{\n    ma_uint32 block;\n\n    /* Try silencing a sanitization warning about unaligned access by doing a memcpy() instead of assignment. */\n    MA_COPY_MEMORY(&block, ma_offset_ptr(blocks, i * (int) sizeof(block)), sizeof(block));\n\n    if (ma_is_little_endian()) {\n        return block;\n    } else {\n        return ma_swap_endian_uint32(block);\n    }\n}\n\nstatic MA_INLINE ma_uint32 ma_hash_fmix32(ma_uint32 h)\n{\n    h ^= h >> 16;\n    h *= 0x85ebca6b;\n    h ^= h >> 13;\n    h *= 0xc2b2ae35;\n    h ^= h >> 16;\n\n    return h;\n}\n\nstatic ma_uint32 ma_hash_32(const void* key, int len, ma_uint32 seed)\n{\n    const ma_uint8* data = (const ma_uint8*)key;\n    const ma_uint32* blocks;\n    const ma_uint8* tail;\n    const int nblocks = len / 4;\n    ma_uint32 h1 = seed;\n    ma_uint32 c1 = 0xcc9e2d51;\n    ma_uint32 c2 = 0x1b873593;\n    ma_uint32 k1;\n    int i;\n\n    blocks = (const ma_uint32 *)(data + nblocks*4);\n\n    for(i = -nblocks; i; i++) {\n        k1 = ma_hash_getblock(blocks,i);\n\n        k1 *= c1;\n        k1 = ma_rotl32(k1, 15);\n        k1 *= c2;\n\n        h1 ^= k1;\n        h1 = ma_rotl32(h1, 13);\n        h1 = h1*5 + 0xe6546b64;\n    }\n\n\n    tail = (const ma_uint8*)(data + nblocks*4);\n\n    k1 = 0;\n    switch(len & 3) {\n        case 3: k1 ^= tail[2] << 16;\n        case 2: k1 ^= tail[1] << 8;\n        case 1: k1 ^= tail[0];\n                k1 *= c1; k1 = ma_rotl32(k1, 15); k1 *= c2; h1 ^= k1;\n    };\n\n\n    h1 ^= len;\n    h1  = ma_hash_fmix32(h1);\n\n    return h1;\n}\n\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic push\n#endif\n/* End MurmurHash3 */\n\nstatic ma_uint32 ma_hash_string_32(const char* str)\n{\n    return ma_hash_32(str, (int)strlen(str), MA_DEFAULT_HASH_SEED);\n}\n\nstatic ma_uint32 ma_hash_string_w_32(const wchar_t* str)\n{\n    return ma_hash_32(str, (int)ma_wcslen(str) * sizeof(*str), MA_DEFAULT_HASH_SEED);\n}\n\n\n\n\n/*\nBasic BST Functions\n*/\nstatic ma_result ma_resource_manager_data_buffer_node_search(ma_resource_manager* pResourceManager, ma_uint32 hashedName32, ma_resource_manager_data_buffer_node** ppDataBufferNode)\n{\n    ma_resource_manager_data_buffer_node* pCurrentNode;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(ppDataBufferNode != NULL);\n\n    pCurrentNode = pResourceManager->pRootDataBufferNode;\n    while (pCurrentNode != NULL) {\n        if (hashedName32 == pCurrentNode->hashedName32) {\n            break;  /* Found. */\n        } else if (hashedName32 < pCurrentNode->hashedName32) {\n            pCurrentNode = pCurrentNode->pChildLo;\n        } else {\n            pCurrentNode = pCurrentNode->pChildHi;\n        }\n    }\n\n    *ppDataBufferNode = pCurrentNode;\n\n    if (pCurrentNode == NULL) {\n        return MA_DOES_NOT_EXIST;\n    } else {\n        return MA_SUCCESS;\n    }\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_insert_point(ma_resource_manager* pResourceManager, ma_uint32 hashedName32, ma_resource_manager_data_buffer_node** ppInsertPoint)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager_data_buffer_node* pCurrentNode;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(ppInsertPoint    != NULL);\n\n    *ppInsertPoint = NULL;\n\n    if (pResourceManager->pRootDataBufferNode == NULL) {\n        return MA_SUCCESS;  /* No items. */\n    }\n\n    /* We need to find the node that will become the parent of the new node. If a node is found that already has the same hashed name we need to return MA_ALREADY_EXISTS. */\n    pCurrentNode = pResourceManager->pRootDataBufferNode;\n    while (pCurrentNode != NULL) {\n        if (hashedName32 == pCurrentNode->hashedName32) {\n            result = MA_ALREADY_EXISTS;\n            break;\n        } else {\n            if (hashedName32 < pCurrentNode->hashedName32) {\n                if (pCurrentNode->pChildLo == NULL) {\n                    result = MA_SUCCESS;\n                    break;\n                } else {\n                    pCurrentNode = pCurrentNode->pChildLo;\n                }\n            } else {\n                if (pCurrentNode->pChildHi == NULL) {\n                    result = MA_SUCCESS;\n                    break;\n                } else {\n                    pCurrentNode = pCurrentNode->pChildHi;\n                }\n            }\n        }\n    }\n\n    *ppInsertPoint = pCurrentNode;\n    return result;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_insert_at(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, ma_resource_manager_data_buffer_node* pInsertPoint)\n{\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    /* The key must have been set before calling this function. */\n    MA_ASSERT(pDataBufferNode->hashedName32 != 0);\n\n    if (pInsertPoint == NULL) {\n        /* It's the first node. */\n        pResourceManager->pRootDataBufferNode = pDataBufferNode;\n    } else {\n        /* It's not the first node. It needs to be inserted. */\n        if (pDataBufferNode->hashedName32 < pInsertPoint->hashedName32) {\n            MA_ASSERT(pInsertPoint->pChildLo == NULL);\n            pInsertPoint->pChildLo = pDataBufferNode;\n        } else {\n            MA_ASSERT(pInsertPoint->pChildHi == NULL);\n            pInsertPoint->pChildHi = pDataBufferNode;\n        }\n    }\n\n    pDataBufferNode->pParent = pInsertPoint;\n\n    return MA_SUCCESS;\n}\n\n#if 0   /* Unused for now. */\nstatic ma_result ma_resource_manager_data_buffer_node_insert(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    ma_result result;\n    ma_resource_manager_data_buffer_node* pInsertPoint;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    result = ma_resource_manager_data_buffer_node_insert_point(pResourceManager, pDataBufferNode->hashedName32, &pInsertPoint);\n    if (result != MA_SUCCESS) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_resource_manager_data_buffer_node_insert_at(pResourceManager, pDataBufferNode, pInsertPoint);\n}\n#endif\n\nstatic MA_INLINE ma_resource_manager_data_buffer_node* ma_resource_manager_data_buffer_node_find_min(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    ma_resource_manager_data_buffer_node* pCurrentNode;\n\n    MA_ASSERT(pDataBufferNode != NULL);\n\n    pCurrentNode = pDataBufferNode;\n    while (pCurrentNode->pChildLo != NULL) {\n        pCurrentNode = pCurrentNode->pChildLo;\n    }\n\n    return pCurrentNode;\n}\n\nstatic MA_INLINE ma_resource_manager_data_buffer_node* ma_resource_manager_data_buffer_node_find_max(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    ma_resource_manager_data_buffer_node* pCurrentNode;\n\n    MA_ASSERT(pDataBufferNode != NULL);\n\n    pCurrentNode = pDataBufferNode;\n    while (pCurrentNode->pChildHi != NULL) {\n        pCurrentNode = pCurrentNode->pChildHi;\n    }\n\n    return pCurrentNode;\n}\n\nstatic MA_INLINE ma_resource_manager_data_buffer_node* ma_resource_manager_data_buffer_node_find_inorder_successor(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pDataBufferNode           != NULL);\n    MA_ASSERT(pDataBufferNode->pChildHi != NULL);\n\n    return ma_resource_manager_data_buffer_node_find_min(pDataBufferNode->pChildHi);\n}\n\n#if 0   /* Currently unused, but might make use of this later. */\nstatic MA_INLINE ma_resource_manager_data_buffer_node* ma_resource_manager_data_buffer_node_find_inorder_predecessor(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pDataBufferNode           != NULL);\n    MA_ASSERT(pDataBufferNode->pChildLo != NULL);\n\n    return ma_resource_manager_data_buffer_node_find_max(pDataBufferNode->pChildLo);\n}\n#endif\n\nstatic ma_result ma_resource_manager_data_buffer_node_remove(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    if (pDataBufferNode->pChildLo == NULL) {\n        if (pDataBufferNode->pChildHi == NULL) {\n            /* Simple case - deleting a buffer with no children. */\n            if (pDataBufferNode->pParent == NULL) {\n                MA_ASSERT(pResourceManager->pRootDataBufferNode == pDataBufferNode);    /* There is only a single buffer in the tree which should be equal to the root node. */\n                pResourceManager->pRootDataBufferNode = NULL;\n            } else {\n                if (pDataBufferNode->pParent->pChildLo == pDataBufferNode) {\n                    pDataBufferNode->pParent->pChildLo = NULL;\n                } else {\n                    pDataBufferNode->pParent->pChildHi = NULL;\n                }\n            }\n        } else {\n            /* Node has one child - pChildHi != NULL. */\n            pDataBufferNode->pChildHi->pParent = pDataBufferNode->pParent;\n\n            if (pDataBufferNode->pParent == NULL) {\n                MA_ASSERT(pResourceManager->pRootDataBufferNode == pDataBufferNode);\n                pResourceManager->pRootDataBufferNode = pDataBufferNode->pChildHi;\n            } else {\n                if (pDataBufferNode->pParent->pChildLo == pDataBufferNode) {\n                    pDataBufferNode->pParent->pChildLo = pDataBufferNode->pChildHi;\n                } else {\n                    pDataBufferNode->pParent->pChildHi = pDataBufferNode->pChildHi;\n                }\n            }\n        }\n    } else {\n        if (pDataBufferNode->pChildHi == NULL) {\n            /* Node has one child - pChildLo != NULL. */\n            pDataBufferNode->pChildLo->pParent = pDataBufferNode->pParent;\n\n            if (pDataBufferNode->pParent == NULL) {\n                MA_ASSERT(pResourceManager->pRootDataBufferNode == pDataBufferNode);\n                pResourceManager->pRootDataBufferNode = pDataBufferNode->pChildLo;\n            } else {\n                if (pDataBufferNode->pParent->pChildLo == pDataBufferNode) {\n                    pDataBufferNode->pParent->pChildLo = pDataBufferNode->pChildLo;\n                } else {\n                    pDataBufferNode->pParent->pChildHi = pDataBufferNode->pChildLo;\n                }\n            }\n        } else {\n            /* Complex case - deleting a node with two children. */\n            ma_resource_manager_data_buffer_node* pReplacementDataBufferNode;\n\n            /* For now we are just going to use the in-order successor as the replacement, but we may want to try to keep this balanced by switching between the two. */\n            pReplacementDataBufferNode = ma_resource_manager_data_buffer_node_find_inorder_successor(pDataBufferNode);\n            MA_ASSERT(pReplacementDataBufferNode != NULL);\n\n            /*\n            Now that we have our replacement node we can make the change. The simple way to do this would be to just exchange the values, and then remove the replacement\n            node, however we track specific nodes via pointers which means we can't just swap out the values. We need to instead just change the pointers around. The\n            replacement node should have at most 1 child. Therefore, we can detach it in terms of our simpler cases above. What we're essentially doing is detaching the\n            replacement node and reinserting it into the same position as the deleted node.\n            */\n            MA_ASSERT(pReplacementDataBufferNode->pParent  != NULL);  /* The replacement node should never be the root which means it should always have a parent. */\n            MA_ASSERT(pReplacementDataBufferNode->pChildLo == NULL);  /* Because we used in-order successor. This would be pChildHi == NULL if we used in-order predecessor. */\n\n            if (pReplacementDataBufferNode->pChildHi == NULL) {\n                if (pReplacementDataBufferNode->pParent->pChildLo == pReplacementDataBufferNode) {\n                    pReplacementDataBufferNode->pParent->pChildLo = NULL;\n                } else {\n                    pReplacementDataBufferNode->pParent->pChildHi = NULL;\n                }\n            } else {\n                pReplacementDataBufferNode->pChildHi->pParent = pReplacementDataBufferNode->pParent;\n                if (pReplacementDataBufferNode->pParent->pChildLo == pReplacementDataBufferNode) {\n                    pReplacementDataBufferNode->pParent->pChildLo = pReplacementDataBufferNode->pChildHi;\n                } else {\n                    pReplacementDataBufferNode->pParent->pChildHi = pReplacementDataBufferNode->pChildHi;\n                }\n            }\n\n\n            /* The replacement node has essentially been detached from the binary tree, so now we need to replace the old data buffer with it. The first thing to update is the parent */\n            if (pDataBufferNode->pParent != NULL) {\n                if (pDataBufferNode->pParent->pChildLo == pDataBufferNode) {\n                    pDataBufferNode->pParent->pChildLo = pReplacementDataBufferNode;\n                } else {\n                    pDataBufferNode->pParent->pChildHi = pReplacementDataBufferNode;\n                }\n            }\n\n            /* Now need to update the replacement node's pointers. */\n            pReplacementDataBufferNode->pParent  = pDataBufferNode->pParent;\n            pReplacementDataBufferNode->pChildLo = pDataBufferNode->pChildLo;\n            pReplacementDataBufferNode->pChildHi = pDataBufferNode->pChildHi;\n\n            /* Now the children of the replacement node need to have their parent pointers updated. */\n            if (pReplacementDataBufferNode->pChildLo != NULL) {\n                pReplacementDataBufferNode->pChildLo->pParent = pReplacementDataBufferNode;\n            }\n            if (pReplacementDataBufferNode->pChildHi != NULL) {\n                pReplacementDataBufferNode->pChildHi->pParent = pReplacementDataBufferNode;\n            }\n\n            /* Now the root node needs to be updated. */\n            if (pResourceManager->pRootDataBufferNode == pDataBufferNode) {\n                pResourceManager->pRootDataBufferNode = pReplacementDataBufferNode;\n            }\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\n#if 0   /* Unused for now. */\nstatic ma_result ma_resource_manager_data_buffer_node_remove_by_key(ma_resource_manager* pResourceManager, ma_uint32 hashedName32)\n{\n    ma_result result;\n    ma_resource_manager_data_buffer_node* pDataBufferNode;\n\n    result = ma_resource_manager_data_buffer_search(pResourceManager, hashedName32, &pDataBufferNode);\n    if (result != MA_SUCCESS) {\n        return result;  /* Could not find the data buffer. */\n    }\n\n    return ma_resource_manager_data_buffer_remove(pResourceManager, pDataBufferNode);\n}\n#endif\n\nstatic ma_resource_manager_data_supply_type ma_resource_manager_data_buffer_node_get_data_supply_type(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    return (ma_resource_manager_data_supply_type)ma_atomic_load_i32(&pDataBufferNode->data.type);\n}\n\nstatic void ma_resource_manager_data_buffer_node_set_data_supply_type(ma_resource_manager_data_buffer_node* pDataBufferNode, ma_resource_manager_data_supply_type supplyType)\n{\n    ma_atomic_exchange_i32(&pDataBufferNode->data.type, supplyType);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_increment_ref(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, ma_uint32* pNewRefCount)\n{\n    ma_uint32 refCount;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    (void)pResourceManager;\n\n    refCount = ma_atomic_fetch_add_32(&pDataBufferNode->refCount, 1) + 1;\n\n    if (pNewRefCount != NULL) {\n        *pNewRefCount = refCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_decrement_ref(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, ma_uint32* pNewRefCount)\n{\n    ma_uint32 refCount;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    (void)pResourceManager;\n\n    refCount = ma_atomic_fetch_sub_32(&pDataBufferNode->refCount, 1) - 1;\n\n    if (pNewRefCount != NULL) {\n        *pNewRefCount = refCount;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_resource_manager_data_buffer_node_free(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n\n    if (pDataBufferNode->isDataOwnedByResourceManager) {\n        if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBufferNode) == ma_resource_manager_data_supply_type_encoded) {\n            ma_free((void*)pDataBufferNode->data.backend.encoded.pData, &pResourceManager->config.allocationCallbacks);\n            pDataBufferNode->data.backend.encoded.pData       = NULL;\n            pDataBufferNode->data.backend.encoded.sizeInBytes = 0;\n        } else if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBufferNode) == ma_resource_manager_data_supply_type_decoded) {\n            ma_free((void*)pDataBufferNode->data.backend.decoded.pData, &pResourceManager->config.allocationCallbacks);\n            pDataBufferNode->data.backend.decoded.pData           = NULL;\n            pDataBufferNode->data.backend.decoded.totalFrameCount = 0;\n        } else if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBufferNode) == ma_resource_manager_data_supply_type_decoded_paged) {\n            ma_paged_audio_buffer_data_uninit(&pDataBufferNode->data.backend.decodedPaged.data, &pResourceManager->config.allocationCallbacks);\n        } else {\n            /* Should never hit this if the node was successfully initialized. */\n            MA_ASSERT(pDataBufferNode->result != MA_SUCCESS);\n        }\n    }\n\n    /* The data buffer itself needs to be freed. */\n    ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_result(const ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pDataBufferNode != NULL);\n\n    return (ma_result)ma_atomic_load_i32((ma_result*)&pDataBufferNode->result);    /* Need a naughty const-cast here. */\n}\n\n\nstatic ma_bool32 ma_resource_manager_is_threading_enabled(const ma_resource_manager* pResourceManager)\n{\n    MA_ASSERT(pResourceManager != NULL);\n\n    return (pResourceManager->config.flags & MA_RESOURCE_MANAGER_FLAG_NO_THREADING) == 0;\n}\n\n\ntypedef struct\n{\n    union\n    {\n        ma_async_notification_event e;\n        ma_async_notification_poll p;\n    } backend;  /* Must be the first member. */\n    ma_resource_manager* pResourceManager;\n} ma_resource_manager_inline_notification;\n\nstatic ma_result ma_resource_manager_inline_notification_init(ma_resource_manager* pResourceManager, ma_resource_manager_inline_notification* pNotification)\n{\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pNotification    != NULL);\n\n    pNotification->pResourceManager = pResourceManager;\n\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        return ma_async_notification_event_init(&pNotification->backend.e);\n    } else {\n        return ma_async_notification_poll_init(&pNotification->backend.p);\n    }\n}\n\nstatic void ma_resource_manager_inline_notification_uninit(ma_resource_manager_inline_notification* pNotification)\n{\n    MA_ASSERT(pNotification != NULL);\n\n    if (ma_resource_manager_is_threading_enabled(pNotification->pResourceManager)) {\n        ma_async_notification_event_uninit(&pNotification->backend.e);\n    } else {\n        /* No need to uninitialize a polling notification. */\n    }\n}\n\nstatic void ma_resource_manager_inline_notification_wait(ma_resource_manager_inline_notification* pNotification)\n{\n    MA_ASSERT(pNotification != NULL);\n\n    if (ma_resource_manager_is_threading_enabled(pNotification->pResourceManager)) {\n        ma_async_notification_event_wait(&pNotification->backend.e);\n    } else {\n        while (ma_async_notification_poll_is_signalled(&pNotification->backend.p) == MA_FALSE) {\n            ma_result result = ma_resource_manager_process_next_job(pNotification->pResourceManager);\n            if (result == MA_NO_DATA_AVAILABLE || result == MA_CANCELLED) {\n                break;\n            }\n        }\n    }\n}\n\nstatic void ma_resource_manager_inline_notification_wait_and_uninit(ma_resource_manager_inline_notification* pNotification)\n{\n    ma_resource_manager_inline_notification_wait(pNotification);\n    ma_resource_manager_inline_notification_uninit(pNotification);\n}\n\n\nstatic void ma_resource_manager_data_buffer_bst_lock(ma_resource_manager* pResourceManager)\n{\n    MA_ASSERT(pResourceManager != NULL);\n\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_mutex_lock(&pResourceManager->dataBufferBSTLock);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n        }\n        #endif\n    } else {\n        /* Threading not enabled. Do nothing. */\n    }\n}\n\nstatic void ma_resource_manager_data_buffer_bst_unlock(ma_resource_manager* pResourceManager)\n{\n    MA_ASSERT(pResourceManager != NULL);\n\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_mutex_unlock(&pResourceManager->dataBufferBSTLock);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n        }\n        #endif\n    } else {\n        /* Threading not enabled. Do nothing. */\n    }\n}\n\n#ifndef MA_NO_THREADING\nstatic ma_thread_result MA_THREADCALL ma_resource_manager_job_thread(void* pUserData)\n{\n    ma_resource_manager* pResourceManager = (ma_resource_manager*)pUserData;\n    MA_ASSERT(pResourceManager != NULL);\n\n    for (;;) {\n        ma_result result;\n        ma_job job;\n\n        result = ma_resource_manager_next_job(pResourceManager, &job);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        /* Terminate if we got a quit message. */\n        if (job.toc.breakup.code == MA_JOB_TYPE_QUIT) {\n            break;\n        }\n\n        ma_job_process(&job);\n    }\n\n    return (ma_thread_result)0;\n}\n#endif\n\nMA_API ma_resource_manager_config ma_resource_manager_config_init(void)\n{\n    ma_resource_manager_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.decodedFormat     = ma_format_unknown;\n    config.decodedChannels   = 0;\n    config.decodedSampleRate = 0;\n    config.jobThreadCount    = 1;   /* A single miniaudio-managed job thread by default. */\n    config.jobQueueCapacity  = MA_JOB_TYPE_RESOURCE_MANAGER_QUEUE_CAPACITY;\n    config.resampling        = ma_resampler_config_init(ma_format_unknown, 0, 0, 0, ma_resample_algorithm_linear); /* Format/channels/rate doesn't matter here. */\n\n    /* Flags. */\n    config.flags = 0;\n    #ifdef MA_NO_THREADING\n    {\n        /* Threading is disabled at compile time so disable threading at runtime as well by default. */\n        config.flags |= MA_RESOURCE_MANAGER_FLAG_NO_THREADING;\n        config.jobThreadCount = 0;\n    }\n    #endif\n\n    return config;\n}\n\n\nMA_API ma_result ma_resource_manager_init(const ma_resource_manager_config* pConfig, ma_resource_manager* pResourceManager)\n{\n    ma_result result;\n    ma_job_queue_config jobQueueConfig;\n\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pResourceManager);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #ifndef MA_NO_THREADING\n    {\n        if (pConfig->jobThreadCount > ma_countof(pResourceManager->jobThreads)) {\n            return MA_INVALID_ARGS; /* Requesting too many job threads. */\n        }\n    }\n    #endif\n\n    pResourceManager->config = *pConfig;\n    ma_allocation_callbacks_init_copy(&pResourceManager->config.allocationCallbacks, &pConfig->allocationCallbacks);\n\n    /* Get the log set up early so we can start using it as soon as possible. */\n    if (pResourceManager->config.pLog == NULL) {\n        result = ma_log_init(&pResourceManager->config.allocationCallbacks, &pResourceManager->log);\n        if (result == MA_SUCCESS) {\n            pResourceManager->config.pLog = &pResourceManager->log;\n        } else {\n            pResourceManager->config.pLog = NULL;   /* Logging is unavailable. */\n        }\n    }\n\n    if (pResourceManager->config.pVFS == NULL) {\n        result = ma_default_vfs_init(&pResourceManager->defaultVFS, &pResourceManager->config.allocationCallbacks);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to initialize the default file system. */\n        }\n\n        pResourceManager->config.pVFS = &pResourceManager->defaultVFS;\n    }\n\n    /* If threading has been disabled at compile time, enforce it at run time as well. */\n    #ifdef MA_NO_THREADING\n    {\n        pResourceManager->config.flags |= MA_RESOURCE_MANAGER_FLAG_NO_THREADING;\n    }\n    #endif\n\n    /* We need to force MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING if MA_RESOURCE_MANAGER_FLAG_NO_THREADING is set. */\n    if ((pResourceManager->config.flags & MA_RESOURCE_MANAGER_FLAG_NO_THREADING) != 0) {\n        pResourceManager->config.flags |= MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING;\n\n        /* We cannot allow job threads when MA_RESOURCE_MANAGER_FLAG_NO_THREADING has been set. This is an invalid use case. */\n        if (pResourceManager->config.jobThreadCount > 0) {\n            return MA_INVALID_ARGS;\n        }\n    }\n\n    /* Job queue. */\n    jobQueueConfig.capacity = pResourceManager->config.jobQueueCapacity;\n    jobQueueConfig.flags    = 0;\n    if ((pResourceManager->config.flags & MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING) != 0) {\n        if (pResourceManager->config.jobThreadCount > 0) {\n            return MA_INVALID_ARGS; /* Non-blocking mode is only valid for self-managed job threads. */\n        }\n\n        jobQueueConfig.flags |= MA_JOB_QUEUE_FLAG_NON_BLOCKING;\n    }\n\n    result = ma_job_queue_init(&jobQueueConfig, &pResourceManager->config.allocationCallbacks, &pResourceManager->jobQueue);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n\n    /* Custom decoding backends. */\n    if (pConfig->ppCustomDecodingBackendVTables != NULL && pConfig->customDecodingBackendCount > 0) {\n        size_t sizeInBytes = sizeof(*pResourceManager->config.ppCustomDecodingBackendVTables) * pConfig->customDecodingBackendCount;\n        ma_decoding_backend_vtable** ppCustomDecodingBackendVTables;\n\n        ppCustomDecodingBackendVTables = (ma_decoding_backend_vtable**)ma_malloc(sizeInBytes, &pResourceManager->config.allocationCallbacks);\n        if (pResourceManager->config.ppCustomDecodingBackendVTables == NULL) {\n            ma_job_queue_uninit(&pResourceManager->jobQueue, &pResourceManager->config.allocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        MA_COPY_MEMORY(ppCustomDecodingBackendVTables, pConfig->ppCustomDecodingBackendVTables, sizeInBytes);\n\n        pResourceManager->config.ppCustomDecodingBackendVTables = ppCustomDecodingBackendVTables;\n        pResourceManager->config.customDecodingBackendCount     = pConfig->customDecodingBackendCount;\n        pResourceManager->config.pCustomDecodingBackendUserData = pConfig->pCustomDecodingBackendUserData;\n    }\n\n\n\n    /* Here is where we initialize our threading stuff. We don't do this if we don't support threading. */\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_uint32 iJobThread;\n\n            /* Data buffer lock. */\n            result = ma_mutex_init(&pResourceManager->dataBufferBSTLock);\n            if (result != MA_SUCCESS) {\n                ma_job_queue_uninit(&pResourceManager->jobQueue, &pResourceManager->config.allocationCallbacks);\n                return result;\n            }\n\n            /* Create the job threads last to ensure the threads has access to valid data. */\n            for (iJobThread = 0; iJobThread < pResourceManager->config.jobThreadCount; iJobThread += 1) {\n                result = ma_thread_create(&pResourceManager->jobThreads[iJobThread], ma_thread_priority_normal, pResourceManager->config.jobThreadStackSize, ma_resource_manager_job_thread, pResourceManager, &pResourceManager->config.allocationCallbacks);\n                if (result != MA_SUCCESS) {\n                    ma_mutex_uninit(&pResourceManager->dataBufferBSTLock);\n                    ma_job_queue_uninit(&pResourceManager->jobQueue, &pResourceManager->config.allocationCallbacks);\n                    return result;\n                }\n            }\n        }\n        #else\n        {\n            /* Threading is disabled at compile time. We should never get here because validation checks should have already been performed. */\n            MA_ASSERT(MA_FALSE);\n        }\n        #endif\n    }\n\n    return MA_SUCCESS;\n}\n\n\nstatic void ma_resource_manager_delete_all_data_buffer_nodes(ma_resource_manager* pResourceManager)\n{\n    MA_ASSERT(pResourceManager);\n\n    /* If everything was done properly, there shouldn't be any active data buffers. */\n    while (pResourceManager->pRootDataBufferNode != NULL) {\n        ma_resource_manager_data_buffer_node* pDataBufferNode = pResourceManager->pRootDataBufferNode;\n        ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode);\n\n        /* The data buffer has been removed from the BST, so now we need to free its data. */\n        ma_resource_manager_data_buffer_node_free(pResourceManager, pDataBufferNode);\n    }\n}\n\nMA_API void ma_resource_manager_uninit(ma_resource_manager* pResourceManager)\n{\n    if (pResourceManager == NULL) {\n        return;\n    }\n\n    /*\n    Job threads need to be killed first. To do this we need to post a quit message to the message queue and then wait for the thread. The quit message will never be removed from the\n    queue which means it will never not be returned after being encountered for the first time which means all threads will eventually receive it.\n    */\n    ma_resource_manager_post_job_quit(pResourceManager);\n\n    /* Wait for every job to finish before continuing to ensure nothing is sill trying to access any of our objects below. */\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_uint32 iJobThread;\n\n            for (iJobThread = 0; iJobThread < pResourceManager->config.jobThreadCount; iJobThread += 1) {\n                ma_thread_wait(&pResourceManager->jobThreads[iJobThread]);\n            }\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n        }\n        #endif\n    }\n\n    /* At this point the thread should have returned and no other thread should be accessing our data. We can now delete all data buffers. */\n    ma_resource_manager_delete_all_data_buffer_nodes(pResourceManager);\n\n    /* The job queue is no longer needed. */\n    ma_job_queue_uninit(&pResourceManager->jobQueue, &pResourceManager->config.allocationCallbacks);\n\n    /* We're no longer doing anything with data buffers so the lock can now be uninitialized. */\n    if (ma_resource_manager_is_threading_enabled(pResourceManager)) {\n        #ifndef MA_NO_THREADING\n        {\n            ma_mutex_uninit(&pResourceManager->dataBufferBSTLock);\n        }\n        #else\n        {\n            MA_ASSERT(MA_FALSE);    /* Should never hit this. */\n        }\n        #endif\n    }\n\n    ma_free((ma_decoding_backend_vtable**)pResourceManager->config.ppCustomDecodingBackendVTables, &pResourceManager->config.allocationCallbacks);  /* <-- Naughty const-cast, but this is safe. */\n\n    if (pResourceManager->config.pLog == &pResourceManager->log) {\n        ma_log_uninit(&pResourceManager->log);\n    }\n}\n\nMA_API ma_log* ma_resource_manager_get_log(ma_resource_manager* pResourceManager)\n{\n    if (pResourceManager == NULL) {\n        return NULL;\n    }\n\n    return pResourceManager->config.pLog;\n}\n\n\n\nMA_API ma_resource_manager_data_source_config ma_resource_manager_data_source_config_init(void)\n{\n    ma_resource_manager_data_source_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.rangeBegInPCMFrames     = MA_DATA_SOURCE_DEFAULT_RANGE_BEG;\n    config.rangeEndInPCMFrames     = MA_DATA_SOURCE_DEFAULT_RANGE_END;\n    config.loopPointBegInPCMFrames = MA_DATA_SOURCE_DEFAULT_LOOP_POINT_BEG;\n    config.loopPointEndInPCMFrames = MA_DATA_SOURCE_DEFAULT_LOOP_POINT_END;\n    config.isLooping               = MA_FALSE;\n\n    return config;\n}\n\n\nstatic ma_decoder_config ma_resource_manager__init_decoder_config(ma_resource_manager* pResourceManager)\n{\n    ma_decoder_config config;\n\n    config = ma_decoder_config_init(pResourceManager->config.decodedFormat, pResourceManager->config.decodedChannels, pResourceManager->config.decodedSampleRate);\n    config.allocationCallbacks    = pResourceManager->config.allocationCallbacks;\n    config.ppCustomBackendVTables = pResourceManager->config.ppCustomDecodingBackendVTables;\n    config.customBackendCount     = pResourceManager->config.customDecodingBackendCount;\n    config.pCustomBackendUserData = pResourceManager->config.pCustomDecodingBackendUserData;\n    config.resampling = pResourceManager->config.resampling;\n\n    return config;\n}\n\nstatic ma_result ma_resource_manager__init_decoder(ma_resource_manager* pResourceManager, const char* pFilePath, const wchar_t* pFilePathW, ma_decoder* pDecoder)\n{\n    ma_result result;\n    ma_decoder_config config;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pFilePath        != NULL || pFilePathW != NULL);\n    MA_ASSERT(pDecoder         != NULL);\n\n    config = ma_resource_manager__init_decoder_config(pResourceManager);\n\n    if (pFilePath != NULL) {\n        result = ma_decoder_init_vfs(pResourceManager->config.pVFS, pFilePath, &config, pDecoder);\n        if (result != MA_SUCCESS) {\n            ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to load file \\\"%s\\\". %s.\\n\", pFilePath, ma_result_description(result));\n            return result;\n        }\n    } else {\n        result = ma_decoder_init_vfs_w(pResourceManager->config.pVFS, pFilePathW, &config, pDecoder);\n        if (result != MA_SUCCESS) {\n            #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(_MSC_VER)\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to load file \\\"%ls\\\". %s.\\n\", pFilePathW, ma_result_description(result));\n            #endif\n            return result;\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_bool32 ma_resource_manager_data_buffer_has_connector(ma_resource_manager_data_buffer* pDataBuffer)\n{\n    return ma_atomic_bool32_get(&pDataBuffer->isConnectorInitialized);\n}\n\nstatic ma_data_source* ma_resource_manager_data_buffer_get_connector(ma_resource_manager_data_buffer* pDataBuffer)\n{\n    if (ma_resource_manager_data_buffer_has_connector(pDataBuffer) == MA_FALSE) {\n        return NULL;    /* Connector not yet initialized. */\n    }\n\n    switch (pDataBuffer->pNode->data.type)\n    {\n        case ma_resource_manager_data_supply_type_encoded:       return &pDataBuffer->connector.decoder;\n        case ma_resource_manager_data_supply_type_decoded:       return &pDataBuffer->connector.buffer;\n        case ma_resource_manager_data_supply_type_decoded_paged: return &pDataBuffer->connector.pagedBuffer;\n\n        case ma_resource_manager_data_supply_type_unknown:\n        default:\n        {\n            ma_log_postf(ma_resource_manager_get_log(pDataBuffer->pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to retrieve data buffer connector. Unknown data supply type.\\n\");\n            return NULL;\n        };\n    };\n}\n\nstatic ma_result ma_resource_manager_data_buffer_init_connector(ma_resource_manager_data_buffer* pDataBuffer, const ma_resource_manager_data_source_config* pConfig, ma_async_notification* pInitNotification, ma_fence* pInitFence)\n{\n    ma_result result;\n\n    MA_ASSERT(pDataBuffer != NULL);\n    MA_ASSERT(pConfig     != NULL);\n    MA_ASSERT(ma_resource_manager_data_buffer_has_connector(pDataBuffer) == MA_FALSE);\n\n    /* The underlying data buffer must be initialized before we'll be able to know how to initialize the backend. */\n    result = ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode);\n    if (result != MA_SUCCESS && result != MA_BUSY) {\n        return result;  /* The data buffer is in an erroneous state. */\n    }\n\n    /*\n    We need to initialize either a ma_decoder or an ma_audio_buffer depending on whether or not the backing data is encoded or decoded. These act as the\n    \"instance\" to the data and are used to form the connection between underlying data buffer and the data source. If the data buffer is decoded, we can use\n    an ma_audio_buffer. This enables us to use memory mapping when mixing which saves us a bit of data movement overhead.\n    */\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode))\n    {\n        case ma_resource_manager_data_supply_type_encoded:          /* Connector is a decoder. */\n        {\n            ma_decoder_config config;\n            config = ma_resource_manager__init_decoder_config(pDataBuffer->pResourceManager);\n            result = ma_decoder_init_memory(pDataBuffer->pNode->data.backend.encoded.pData, pDataBuffer->pNode->data.backend.encoded.sizeInBytes, &config, &pDataBuffer->connector.decoder);\n        } break;\n\n        case ma_resource_manager_data_supply_type_decoded:          /* Connector is an audio buffer. */\n        {\n            ma_audio_buffer_config config;\n            config = ma_audio_buffer_config_init(pDataBuffer->pNode->data.backend.decoded.format, pDataBuffer->pNode->data.backend.decoded.channels, pDataBuffer->pNode->data.backend.decoded.totalFrameCount, pDataBuffer->pNode->data.backend.decoded.pData, NULL);\n            result = ma_audio_buffer_init(&config, &pDataBuffer->connector.buffer);\n        } break;\n\n        case ma_resource_manager_data_supply_type_decoded_paged:    /* Connector is a paged audio buffer. */\n        {\n            ma_paged_audio_buffer_config config;\n            config = ma_paged_audio_buffer_config_init(&pDataBuffer->pNode->data.backend.decodedPaged.data);\n            result = ma_paged_audio_buffer_init(&config, &pDataBuffer->connector.pagedBuffer);\n        } break;\n\n        case ma_resource_manager_data_supply_type_unknown:\n        default:\n        {\n            /* Unknown data supply type. Should never happen. Need to post an error here. */\n            return MA_INVALID_ARGS;\n        };\n    }\n\n    /*\n    Initialization of the connector is when we can fire the init notification. This will give the application access to\n    the format/channels/rate of the data source.\n    */\n    if (result == MA_SUCCESS) {\n        /*\n        The resource manager supports the ability to set the range and loop settings via a config at\n        initialization time. This results in an case where the ranges could be set explicitly via\n        ma_data_source_set_*() before we get to this point here. If this happens, we'll end up\n        hitting a case where we just override those settings which results in what feels like a bug.\n\n        To address this we only change the relevant properties if they're not equal to defaults. If\n        they're equal to defaults there's no need to change them anyway. If they're *not* set to the\n        default values, we can assume the user has set the range and loop settings via the config. If\n        they're doing their own calls to ma_data_source_set_*() in addition to setting them via the\n        config, that's entirely on the caller and any synchronization issue becomes their problem.\n        */\n        if (pConfig->rangeBegInPCMFrames != MA_DATA_SOURCE_DEFAULT_RANGE_BEG || pConfig->rangeEndInPCMFrames != MA_DATA_SOURCE_DEFAULT_RANGE_END) {\n            ma_data_source_set_range_in_pcm_frames(pDataBuffer, pConfig->rangeBegInPCMFrames, pConfig->rangeEndInPCMFrames);\n        }\n\n        if (pConfig->loopPointBegInPCMFrames != MA_DATA_SOURCE_DEFAULT_LOOP_POINT_BEG || pConfig->loopPointEndInPCMFrames != MA_DATA_SOURCE_DEFAULT_LOOP_POINT_END) {\n            ma_data_source_set_loop_point_in_pcm_frames(pDataBuffer, pConfig->loopPointBegInPCMFrames, pConfig->loopPointEndInPCMFrames);\n        }\n\n        if (pConfig->isLooping != MA_FALSE) {\n            ma_data_source_set_looping(pDataBuffer, pConfig->isLooping);\n        }\n\n        ma_atomic_bool32_set(&pDataBuffer->isConnectorInitialized, MA_TRUE);\n\n        if (pInitNotification != NULL) {\n            ma_async_notification_signal(pInitNotification);\n        }\n\n        if (pInitFence != NULL) {\n            ma_fence_release(pInitFence);\n        }\n    }\n\n    /* At this point the backend should be initialized. We do *not* want to set pDataSource->result here - that needs to be done at a higher level to ensure it's done as the last step. */\n    return result;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_uninit_connector(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBuffer      != NULL);\n\n    (void)pResourceManager;\n\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode))\n    {\n        case ma_resource_manager_data_supply_type_encoded:          /* Connector is a decoder. */\n        {\n            ma_decoder_uninit(&pDataBuffer->connector.decoder);\n        } break;\n\n        case ma_resource_manager_data_supply_type_decoded:          /* Connector is an audio buffer. */\n        {\n            ma_audio_buffer_uninit(&pDataBuffer->connector.buffer);\n        } break;\n\n        case ma_resource_manager_data_supply_type_decoded_paged:    /* Connector is a paged audio buffer. */\n        {\n            ma_paged_audio_buffer_uninit(&pDataBuffer->connector.pagedBuffer);\n        } break;\n\n        case ma_resource_manager_data_supply_type_unknown:\n        default:\n        {\n            /* Unknown data supply type. Should never happen. Need to post an error here. */\n            return MA_INVALID_ARGS;\n        };\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_uint32 ma_resource_manager_data_buffer_node_next_execution_order(ma_resource_manager_data_buffer_node* pDataBufferNode)\n{\n    MA_ASSERT(pDataBufferNode != NULL);\n    return ma_atomic_fetch_add_32(&pDataBufferNode->executionCounter, 1);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_init_supply_encoded(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, const char* pFilePath, const wchar_t* pFilePathW)\n{\n    ma_result result;\n    size_t dataSizeInBytes;\n    void* pData;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n    MA_ASSERT(pFilePath != NULL || pFilePathW != NULL);\n\n    result = ma_vfs_open_and_read_file_ex(pResourceManager->config.pVFS, pFilePath, pFilePathW, &pData, &dataSizeInBytes, &pResourceManager->config.allocationCallbacks);\n    if (result != MA_SUCCESS) {\n        if (pFilePath != NULL) {\n            ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to load file \\\"%s\\\". %s.\\n\", pFilePath, ma_result_description(result));\n        } else {\n            #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(_MSC_VER)\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to load file \\\"%ls\\\". %s.\\n\", pFilePathW, ma_result_description(result));\n            #endif\n        }\n\n        return result;\n    }\n\n    pDataBufferNode->data.backend.encoded.pData       = pData;\n    pDataBufferNode->data.backend.encoded.sizeInBytes = dataSizeInBytes;\n    ma_resource_manager_data_buffer_node_set_data_supply_type(pDataBufferNode, ma_resource_manager_data_supply_type_encoded);  /* <-- Must be set last. */\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_init_supply_decoded(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, const char* pFilePath, const wchar_t* pFilePathW, ma_uint32 flags, ma_decoder** ppDecoder)\n{\n    ma_result result = MA_SUCCESS;\n    ma_decoder* pDecoder;\n    ma_uint64 totalFrameCount;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n    MA_ASSERT(ppDecoder         != NULL);\n    MA_ASSERT(pFilePath != NULL || pFilePathW != NULL);\n\n    *ppDecoder = NULL;  /* For safety. */\n\n    pDecoder = (ma_decoder*)ma_malloc(sizeof(*pDecoder), &pResourceManager->config.allocationCallbacks);\n    if (pDecoder == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_resource_manager__init_decoder(pResourceManager, pFilePath, pFilePathW, pDecoder);\n    if (result != MA_SUCCESS) {\n        ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n        return result;\n    }\n\n    /*\n    At this point we have the decoder and we now need to initialize the data supply. This will\n    be either a decoded buffer, or a decoded paged buffer. A regular buffer is just one big heap\n    allocated buffer, whereas a paged buffer is a linked list of paged-sized buffers. The latter\n    is used when the length of a sound is unknown until a full decode has been performed.\n    */\n    if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_UNKNOWN_LENGTH) == 0) {\n        result = ma_decoder_get_length_in_pcm_frames(pDecoder, &totalFrameCount);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    } else {\n        totalFrameCount = 0;\n    }\n\n    if (totalFrameCount > 0) {\n        /* It's a known length. The data supply is a regular decoded buffer. */\n        ma_uint64 dataSizeInBytes;\n        void* pData;\n\n        dataSizeInBytes = totalFrameCount * ma_get_bytes_per_frame(pDecoder->outputFormat, pDecoder->outputChannels);\n        if (dataSizeInBytes > MA_SIZE_MAX) {\n            ma_decoder_uninit(pDecoder);\n            ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n            return MA_TOO_BIG;\n        }\n\n        pData = ma_malloc((size_t)dataSizeInBytes, &pResourceManager->config.allocationCallbacks);\n        if (pData == NULL) {\n            ma_decoder_uninit(pDecoder);\n            ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n\n        /* The buffer needs to be initialized to silence in case the caller reads from it. */\n        ma_silence_pcm_frames(pData, totalFrameCount, pDecoder->outputFormat, pDecoder->outputChannels);\n\n        /* Data has been allocated and the data supply can now be initialized. */\n        pDataBufferNode->data.backend.decoded.pData             = pData;\n        pDataBufferNode->data.backend.decoded.totalFrameCount   = totalFrameCount;\n        pDataBufferNode->data.backend.decoded.format            = pDecoder->outputFormat;\n        pDataBufferNode->data.backend.decoded.channels          = pDecoder->outputChannels;\n        pDataBufferNode->data.backend.decoded.sampleRate        = pDecoder->outputSampleRate;\n        pDataBufferNode->data.backend.decoded.decodedFrameCount = 0;\n        ma_resource_manager_data_buffer_node_set_data_supply_type(pDataBufferNode, ma_resource_manager_data_supply_type_decoded);  /* <-- Must be set last. */\n    } else {\n        /*\n        It's an unknown length. The data supply is a paged decoded buffer. Setting this up is\n        actually easier than the non-paged decoded buffer because we just need to initialize\n        a ma_paged_audio_buffer object.\n        */\n        result = ma_paged_audio_buffer_data_init(pDecoder->outputFormat, pDecoder->outputChannels, &pDataBufferNode->data.backend.decodedPaged.data);\n        if (result != MA_SUCCESS) {\n            ma_decoder_uninit(pDecoder);\n            ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n            return result;\n        }\n\n        pDataBufferNode->data.backend.decodedPaged.sampleRate        = pDecoder->outputSampleRate;\n        pDataBufferNode->data.backend.decodedPaged.decodedFrameCount = 0;\n        ma_resource_manager_data_buffer_node_set_data_supply_type(pDataBufferNode, ma_resource_manager_data_supply_type_decoded_paged);  /* <-- Must be set last. */\n    }\n\n    *ppDecoder = pDecoder;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_decode_next_page(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, ma_decoder* pDecoder)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 pageSizeInFrames;\n    ma_uint64 framesToTryReading;\n    ma_uint64 framesRead;\n\n    MA_ASSERT(pResourceManager != NULL);\n    MA_ASSERT(pDataBufferNode  != NULL);\n    MA_ASSERT(pDecoder         != NULL);\n\n    /* We need to know the size of a page in frames to know how many frames to decode. */\n    pageSizeInFrames = MA_RESOURCE_MANAGER_PAGE_SIZE_IN_MILLISECONDS * (pDecoder->outputSampleRate/1000);\n    framesToTryReading = pageSizeInFrames;\n\n    /*\n    Here is where we do the decoding of the next page. We'll run a slightly different path depending\n    on whether or not we're using a flat or paged buffer because the allocation of the page differs\n    between the two. For a flat buffer it's an offset to an already-allocated buffer. For a paged\n    buffer, we need to allocate a new page and attach it to the linked list.\n    */\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBufferNode))\n    {\n        case ma_resource_manager_data_supply_type_decoded:\n        {\n            /* The destination buffer is an offset to the existing buffer. Don't read more than we originally retrieved when we first initialized the decoder. */\n            void* pDst;\n            ma_uint64 framesRemaining = pDataBufferNode->data.backend.decoded.totalFrameCount - pDataBufferNode->data.backend.decoded.decodedFrameCount;\n            if (framesToTryReading > framesRemaining) {\n                framesToTryReading = framesRemaining;\n            }\n\n            if (framesToTryReading > 0) {\n                pDst = ma_offset_ptr(\n                    pDataBufferNode->data.backend.decoded.pData,\n                    pDataBufferNode->data.backend.decoded.decodedFrameCount * ma_get_bytes_per_frame(pDataBufferNode->data.backend.decoded.format, pDataBufferNode->data.backend.decoded.channels)\n                );\n                MA_ASSERT(pDst != NULL);\n\n                result = ma_decoder_read_pcm_frames(pDecoder, pDst, framesToTryReading, &framesRead);\n                if (framesRead > 0) {\n                    pDataBufferNode->data.backend.decoded.decodedFrameCount += framesRead;\n                }\n            } else {\n                framesRead = 0;\n            }\n        } break;\n\n        case ma_resource_manager_data_supply_type_decoded_paged:\n        {\n            /* The destination buffer is a freshly allocated page. */\n            ma_paged_audio_buffer_page* pPage;\n\n            result = ma_paged_audio_buffer_data_allocate_page(&pDataBufferNode->data.backend.decodedPaged.data, framesToTryReading, NULL, &pResourceManager->config.allocationCallbacks, &pPage);\n            if (result != MA_SUCCESS) {\n                return result;\n            }\n\n            result = ma_decoder_read_pcm_frames(pDecoder, pPage->pAudioData, framesToTryReading, &framesRead);\n            if (result == MA_SUCCESS && framesRead > 0) {\n                pPage->sizeInFrames = framesRead;\n\n                result = ma_paged_audio_buffer_data_append_page(&pDataBufferNode->data.backend.decodedPaged.data, pPage);\n                if (result == MA_SUCCESS) {\n                    pDataBufferNode->data.backend.decodedPaged.decodedFrameCount += framesRead;\n                } else {\n                    /* Failed to append the page. Just abort and set the status to MA_AT_END. */\n                    ma_paged_audio_buffer_data_free_page(&pDataBufferNode->data.backend.decodedPaged.data, pPage, &pResourceManager->config.allocationCallbacks);\n                    result = MA_AT_END;\n                }\n            } else {\n                /* No frames were read. Free the page and just set the status to MA_AT_END. */\n                ma_paged_audio_buffer_data_free_page(&pDataBufferNode->data.backend.decodedPaged.data, pPage, &pResourceManager->config.allocationCallbacks);\n                result = MA_AT_END;\n            }\n        } break;\n\n        case ma_resource_manager_data_supply_type_encoded:\n        case ma_resource_manager_data_supply_type_unknown:\n        default:\n        {\n            /* Unexpected data supply type. */\n            ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Unexpected data supply type (%d) when decoding page.\", ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBufferNode));\n            return MA_ERROR;\n        };\n    }\n\n    if (result == MA_SUCCESS && framesRead == 0) {\n        result = MA_AT_END;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_acquire_critical_section(ma_resource_manager* pResourceManager, const char* pFilePath, const wchar_t* pFilePathW, ma_uint32 hashedName32, ma_uint32 flags, const ma_resource_manager_data_supply* pExistingData, ma_fence* pInitFence, ma_fence* pDoneFence, ma_resource_manager_inline_notification* pInitNotification, ma_resource_manager_data_buffer_node** ppDataBufferNode)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager_data_buffer_node* pDataBufferNode = NULL;\n    ma_resource_manager_data_buffer_node* pInsertPoint;\n\n    if (ppDataBufferNode != NULL) {\n        *ppDataBufferNode = NULL;\n    }\n\n    result = ma_resource_manager_data_buffer_node_insert_point(pResourceManager, hashedName32, &pInsertPoint);\n    if (result == MA_ALREADY_EXISTS) {\n        /* The node already exists. We just need to increment the reference count. */\n        pDataBufferNode = pInsertPoint;\n\n        result = ma_resource_manager_data_buffer_node_increment_ref(pResourceManager, pDataBufferNode, NULL);\n        if (result != MA_SUCCESS) {\n            return result;  /* Should never happen. Failed to increment the reference count. */\n        }\n\n        result = MA_ALREADY_EXISTS;\n        goto done;\n    } else {\n        /*\n        The node does not already exist. We need to post a LOAD_DATA_BUFFER_NODE job here. This\n        needs to be done inside the critical section to ensure an uninitialization of the node\n        does not occur before initialization on another thread.\n        */\n        pDataBufferNode = (ma_resource_manager_data_buffer_node*)ma_malloc(sizeof(*pDataBufferNode), &pResourceManager->config.allocationCallbacks);\n        if (pDataBufferNode == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n\n        MA_ZERO_OBJECT(pDataBufferNode);\n        pDataBufferNode->hashedName32 = hashedName32;\n        pDataBufferNode->refCount     = 1;        /* Always set to 1 by default (this is our first reference). */\n\n        if (pExistingData == NULL) {\n            pDataBufferNode->data.type    = ma_resource_manager_data_supply_type_unknown;    /* <-- We won't know this until we start decoding. */\n            pDataBufferNode->result       = MA_BUSY;  /* Must be set to MA_BUSY before we leave the critical section, so might as well do it now. */\n            pDataBufferNode->isDataOwnedByResourceManager = MA_TRUE;\n        } else {\n            pDataBufferNode->data         = *pExistingData;\n            pDataBufferNode->result       = MA_SUCCESS;   /* Not loading asynchronously, so just set the status */\n            pDataBufferNode->isDataOwnedByResourceManager = MA_FALSE;\n        }\n\n        result = ma_resource_manager_data_buffer_node_insert_at(pResourceManager, pDataBufferNode, pInsertPoint);\n        if (result != MA_SUCCESS) {\n            ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks);\n            return result;  /* Should never happen. Failed to insert the data buffer into the BST. */\n        }\n\n        /*\n        Here is where we'll post the job, but only if we're loading asynchronously. If we're\n        loading synchronously we'll defer loading to a later stage, outside of the critical\n        section.\n        */\n        if (pDataBufferNode->isDataOwnedByResourceManager && (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) != 0) {\n            /* Loading asynchronously. Post the job. */\n            ma_job job;\n            char* pFilePathCopy = NULL;\n            wchar_t* pFilePathWCopy = NULL;\n\n            /* We need a copy of the file path. We should probably make this more efficient, but for now we'll do a transient memory allocation. */\n            if (pFilePath != NULL) {\n                pFilePathCopy = ma_copy_string(pFilePath, &pResourceManager->config.allocationCallbacks);\n            } else {\n                pFilePathWCopy = ma_copy_string_w(pFilePathW, &pResourceManager->config.allocationCallbacks);\n            }\n\n            if (pFilePathCopy == NULL && pFilePathWCopy == NULL) {\n                ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode);\n                ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks);\n                return MA_OUT_OF_MEMORY;\n            }\n\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                ma_resource_manager_inline_notification_init(pResourceManager, pInitNotification);\n            }\n\n            /* Acquire init and done fences before posting the job. These will be unacquired by the job thread. */\n            if (pInitFence != NULL) { ma_fence_acquire(pInitFence); }\n            if (pDoneFence != NULL) { ma_fence_acquire(pDoneFence); }\n\n            /* We now have everything we need to post the job to the job thread. */\n            job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER_NODE);\n            job.order = ma_resource_manager_data_buffer_node_next_execution_order(pDataBufferNode);\n            job.data.resourceManager.loadDataBufferNode.pResourceManager  = pResourceManager;\n            job.data.resourceManager.loadDataBufferNode.pDataBufferNode   = pDataBufferNode;\n            job.data.resourceManager.loadDataBufferNode.pFilePath         = pFilePathCopy;\n            job.data.resourceManager.loadDataBufferNode.pFilePathW        = pFilePathWCopy;\n            job.data.resourceManager.loadDataBufferNode.flags             = flags;\n            job.data.resourceManager.loadDataBufferNode.pInitNotification = ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) ? pInitNotification : NULL;\n            job.data.resourceManager.loadDataBufferNode.pDoneNotification = NULL;\n            job.data.resourceManager.loadDataBufferNode.pInitFence        = pInitFence;\n            job.data.resourceManager.loadDataBufferNode.pDoneFence        = pDoneFence;\n\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                result = ma_job_process(&job);\n            } else {\n                result = ma_resource_manager_post_job(pResourceManager, &job);\n            }\n\n            if (result != MA_SUCCESS) {\n                /* Failed to post job. Probably ran out of memory. */\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to post MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER_NODE job. %s.\\n\", ma_result_description(result));\n\n                if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                    ma_resource_manager_inline_notification_uninit(pInitNotification);\n                } else {\n                    /*\n                    Fences were acquired before posting the job, but since the job was not able to\n                    be posted, we need to make sure we release them so nothing gets stuck waiting.\n\n                    In the WAIT_INIT case, these will have already been released in ma_job_process()\n                    so we should only release fences in this branch.\n                    */\n                    if (pInitFence != NULL) { ma_fence_release(pInitFence); }\n                    if (pDoneFence != NULL) { ma_fence_release(pDoneFence); }\n\n                    /* These will have been freed by the job thread, but with WAIT_INIT they will already have happened since the job has already been handled. */\n                    ma_free(pFilePathCopy,  &pResourceManager->config.allocationCallbacks);\n                    ma_free(pFilePathWCopy, &pResourceManager->config.allocationCallbacks);\n                }\n\n                ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode);\n                ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks);\n\n                return result;\n            }\n        }\n    }\n\ndone:\n    if (ppDataBufferNode != NULL) {\n        *ppDataBufferNode = pDataBufferNode;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_acquire(ma_resource_manager* pResourceManager, const char* pFilePath, const wchar_t* pFilePathW, ma_uint32 hashedName32, ma_uint32 flags, const ma_resource_manager_data_supply* pExistingData, ma_fence* pInitFence, ma_fence* pDoneFence, ma_resource_manager_data_buffer_node** ppDataBufferNode)\n{\n    ma_result result = MA_SUCCESS;\n    ma_bool32 nodeAlreadyExists = MA_FALSE;\n    ma_resource_manager_data_buffer_node* pDataBufferNode = NULL;\n    ma_resource_manager_inline_notification initNotification;   /* Used when the WAIT_INIT flag is set. */\n\n    if (ppDataBufferNode != NULL) {\n        *ppDataBufferNode = NULL;   /* Safety. */\n    }\n\n    if (pResourceManager == NULL || (pFilePath == NULL && pFilePathW == NULL && hashedName32 == 0)) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If we're specifying existing data, it must be valid. */\n    if (pExistingData != NULL && pExistingData->type == ma_resource_manager_data_supply_type_unknown) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If we don't support threading, remove the ASYNC flag to make the rest of this a bit simpler. */\n    if (ma_resource_manager_is_threading_enabled(pResourceManager) == MA_FALSE) {\n        flags &= ~MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC;\n    }\n\n    if (hashedName32 == 0) {\n        if (pFilePath != NULL) {\n            hashedName32 = ma_hash_string_32(pFilePath);\n        } else {\n            hashedName32 = ma_hash_string_w_32(pFilePathW);\n        }\n    }\n\n    /*\n    Here is where we either increment the node's reference count or allocate a new one and add it\n    to the BST. When allocating a new node, we need to make sure the LOAD_DATA_BUFFER_NODE job is\n    posted inside the critical section just in case the caller immediately uninitializes the node\n    as this will ensure the FREE_DATA_BUFFER_NODE job is given an execution order such that the\n    node is not uninitialized before initialization.\n    */\n    ma_resource_manager_data_buffer_bst_lock(pResourceManager);\n    {\n        result = ma_resource_manager_data_buffer_node_acquire_critical_section(pResourceManager, pFilePath, pFilePathW, hashedName32, flags, pExistingData, pInitFence, pDoneFence, &initNotification, &pDataBufferNode);\n    }\n    ma_resource_manager_data_buffer_bst_unlock(pResourceManager);\n\n    if (result == MA_ALREADY_EXISTS) {\n        nodeAlreadyExists = MA_TRUE;\n        result = MA_SUCCESS;\n    } else {\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    /*\n    If we're loading synchronously, we'll need to load everything now. When loading asynchronously,\n    a job will have been posted inside the BST critical section so that an uninitialization can be\n    allocated an appropriate execution order thereby preventing it from being uninitialized before\n    the node is initialized by the decoding thread(s).\n    */\n    if (nodeAlreadyExists == MA_FALSE) {    /* Don't need to try loading anything if the node already exists. */\n        if (pFilePath == NULL && pFilePathW == NULL) {\n            /*\n            If this path is hit, it means a buffer is being copied (i.e. initialized from only the\n            hashed name), but that node has been freed in the meantime, probably from some other\n            thread. This is an invalid operation.\n            */\n            ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Cloning data buffer node failed because the source node was released. The source node must remain valid until the cloning has completed.\\n\");\n            result = MA_INVALID_OPERATION;\n            goto done;\n        }\n\n        if (pDataBufferNode->isDataOwnedByResourceManager) {\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) == 0) {\n                /* Loading synchronously. Load the sound in it's entirety here. */\n                if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE) == 0) {\n                    /* No decoding. This is the simple case - just store the file contents in memory. */\n                    result = ma_resource_manager_data_buffer_node_init_supply_encoded(pResourceManager, pDataBufferNode, pFilePath, pFilePathW);\n                    if (result != MA_SUCCESS) {\n                        goto done;\n                    }\n                } else {\n                    /* Decoding. We do this the same way as we do when loading asynchronously. */\n                    ma_decoder* pDecoder;\n                    result = ma_resource_manager_data_buffer_node_init_supply_decoded(pResourceManager, pDataBufferNode, pFilePath, pFilePathW, flags, &pDecoder);\n                    if (result != MA_SUCCESS) {\n                        goto done;\n                    }\n\n                    /* We have the decoder, now decode page by page just like we do when loading asynchronously. */\n                    for (;;) {\n                        /* Decode next page. */\n                        result = ma_resource_manager_data_buffer_node_decode_next_page(pResourceManager, pDataBufferNode, pDecoder);\n                        if (result != MA_SUCCESS) {\n                            break;  /* Will return MA_AT_END when the last page has been decoded. */\n                        }\n                    }\n\n                    /* Reaching the end needs to be considered successful. */\n                    if (result == MA_AT_END) {\n                        result  = MA_SUCCESS;\n                    }\n\n                    /*\n                    At this point the data buffer is either fully decoded or some error occurred. Either\n                    way, the decoder is no longer necessary.\n                    */\n                    ma_decoder_uninit(pDecoder);\n                    ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n                }\n\n                /* Getting here means we were successful. Make sure the status of the node is updated accordingly. */\n                ma_atomic_exchange_i32(&pDataBufferNode->result, result);\n            } else {\n                /* Loading asynchronously. We may need to wait for initialization. */\n                if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                    ma_resource_manager_inline_notification_wait(&initNotification);\n                }\n            }\n        } else {\n            /* The data is not managed by the resource manager so there's nothing else to do. */\n            MA_ASSERT(pExistingData != NULL);\n        }\n    }\n\ndone:\n    /* If we failed to initialize the data buffer we need to free it. */\n    if (result != MA_SUCCESS) {\n        if (nodeAlreadyExists == MA_FALSE) {\n            ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode);\n            ma_free(pDataBufferNode, &pResourceManager->config.allocationCallbacks);\n        }\n    }\n\n    /*\n    The init notification needs to be uninitialized. This will be used if the node does not already\n    exist, and we've specified ASYNC | WAIT_INIT.\n    */\n    if (nodeAlreadyExists == MA_FALSE && pDataBufferNode->isDataOwnedByResourceManager && (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) != 0) {\n        if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n            ma_resource_manager_inline_notification_uninit(&initNotification);\n        }\n    }\n\n    if (ppDataBufferNode != NULL) {\n        *ppDataBufferNode = pDataBufferNode;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_resource_manager_data_buffer_node_unacquire(ma_resource_manager* pResourceManager, ma_resource_manager_data_buffer_node* pDataBufferNode, const char* pName, const wchar_t* pNameW)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 refCount = 0xFFFFFFFF; /* The new reference count of the node after decrementing. Initialize to non-0 to be safe we don't fall into the freeing path. */\n    ma_uint32 hashedName32 = 0;\n\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pDataBufferNode == NULL) {\n        if (pName == NULL && pNameW == NULL) {\n            return MA_INVALID_ARGS;\n        }\n\n        if (pName != NULL) {\n            hashedName32 = ma_hash_string_32(pName);\n        } else {\n            hashedName32 = ma_hash_string_w_32(pNameW);\n        }\n    }\n\n    /*\n    The first thing to do is decrement the reference counter of the node. Then, if the reference\n    count is zero, we need to free the node. If the node is still in the process of loading, we'll\n    need to post a job to the job queue to free the node. Otherwise we'll just do it here.\n    */\n    ma_resource_manager_data_buffer_bst_lock(pResourceManager);\n    {\n        /* Might need to find the node. Must be done inside the critical section. */\n        if (pDataBufferNode == NULL) {\n            result = ma_resource_manager_data_buffer_node_search(pResourceManager, hashedName32, &pDataBufferNode);\n            if (result != MA_SUCCESS) {\n                goto stage2;    /* Couldn't find the node. */\n            }\n        }\n\n        result = ma_resource_manager_data_buffer_node_decrement_ref(pResourceManager, pDataBufferNode, &refCount);\n        if (result != MA_SUCCESS) {\n            goto stage2;    /* Should never happen. */\n        }\n\n        if (refCount == 0) {\n            result = ma_resource_manager_data_buffer_node_remove(pResourceManager, pDataBufferNode);\n            if (result != MA_SUCCESS) {\n                goto stage2;  /* An error occurred when trying to remove the data buffer. This should never happen. */\n            }\n        }\n    }\n    ma_resource_manager_data_buffer_bst_unlock(pResourceManager);\n\nstage2:\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /*\n    Here is where we need to free the node. We don't want to do this inside the critical section\n    above because we want to keep that as small as possible for multi-threaded efficiency.\n    */\n    if (refCount == 0) {\n        if (ma_resource_manager_data_buffer_node_result(pDataBufferNode) == MA_BUSY) {\n            /* The sound is still loading. We need to delay the freeing of the node to a safe time. */\n            ma_job job;\n\n            /* We need to mark the node as unavailable for the sake of the resource manager worker threads. */\n            ma_atomic_exchange_i32(&pDataBufferNode->result, MA_UNAVAILABLE);\n\n            job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER_NODE);\n            job.order = ma_resource_manager_data_buffer_node_next_execution_order(pDataBufferNode);\n            job.data.resourceManager.freeDataBufferNode.pResourceManager = pResourceManager;\n            job.data.resourceManager.freeDataBufferNode.pDataBufferNode  = pDataBufferNode;\n\n            result = ma_resource_manager_post_job(pResourceManager, &job);\n            if (result != MA_SUCCESS) {\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to post MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER_NODE job. %s.\\n\", ma_result_description(result));\n                return result;\n            }\n\n            /* If we don't support threading, process the job queue here. */\n            if (ma_resource_manager_is_threading_enabled(pResourceManager) == MA_FALSE) {\n                while (ma_resource_manager_data_buffer_node_result(pDataBufferNode) == MA_BUSY) {\n                    result = ma_resource_manager_process_next_job(pResourceManager);\n                    if (result == MA_NO_DATA_AVAILABLE || result == MA_CANCELLED) {\n                        result = MA_SUCCESS;\n                        break;\n                    }\n                }\n            } else {\n                /* Threading is enabled. The job queue will deal with the rest of the cleanup from here. */\n            }\n        } else {\n            /* The sound isn't loading so we can just free the node here. */\n            ma_resource_manager_data_buffer_node_free(pResourceManager, pDataBufferNode);\n        }\n    }\n\n    return result;\n}\n\n\n\nstatic ma_uint32 ma_resource_manager_data_buffer_next_execution_order(ma_resource_manager_data_buffer* pDataBuffer)\n{\n    MA_ASSERT(pDataBuffer != NULL);\n    return ma_atomic_fetch_add_32(&pDataBuffer->executionCounter, 1);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__read_pcm_frames(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_resource_manager_data_buffer_read_pcm_frames((ma_resource_manager_data_buffer*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__seek_to_pcm_frame(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_resource_manager_data_buffer_seek_to_pcm_frame((ma_resource_manager_data_buffer*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_resource_manager_data_buffer_get_data_format((ma_resource_manager_data_buffer*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__get_cursor_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_resource_manager_data_buffer_get_cursor_in_pcm_frames((ma_resource_manager_data_buffer*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__get_length_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_resource_manager_data_buffer_get_length_in_pcm_frames((ma_resource_manager_data_buffer*)pDataSource, pLength);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_cb__set_looping(ma_data_source* pDataSource, ma_bool32 isLooping)\n{\n    ma_resource_manager_data_buffer* pDataBuffer = (ma_resource_manager_data_buffer*)pDataSource;\n    MA_ASSERT(pDataBuffer != NULL);\n\n    ma_atomic_exchange_32(&pDataBuffer->isLooping, isLooping);\n\n    /* The looping state needs to be set on the connector as well or else looping won't work when we read audio data. */\n    ma_data_source_set_looping(ma_resource_manager_data_buffer_get_connector(pDataBuffer), isLooping);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_data_source_vtable g_ma_resource_manager_data_buffer_vtable =\n{\n    ma_resource_manager_data_buffer_cb__read_pcm_frames,\n    ma_resource_manager_data_buffer_cb__seek_to_pcm_frame,\n    ma_resource_manager_data_buffer_cb__get_data_format,\n    ma_resource_manager_data_buffer_cb__get_cursor_in_pcm_frames,\n    ma_resource_manager_data_buffer_cb__get_length_in_pcm_frames,\n    ma_resource_manager_data_buffer_cb__set_looping,\n    0\n};\n\nstatic ma_result ma_resource_manager_data_buffer_init_ex_internal(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_uint32 hashedName32, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager_data_buffer_node* pDataBufferNode;\n    ma_data_source_config dataSourceConfig;\n    ma_bool32 async;\n    ma_uint32 flags;\n    ma_resource_manager_pipeline_notifications notifications;\n\n    if (pDataBuffer == NULL) {\n        if (pConfig != NULL && pConfig->pNotifications != NULL) {\n            ma_resource_manager_pipeline_notifications_signal_all_notifications(pConfig->pNotifications);\n        }\n\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDataBuffer);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->pNotifications != NULL) {\n        notifications = *pConfig->pNotifications;   /* From here on out we should be referencing `notifications` instead of `pNotifications`. Set this to NULL to catch errors at testing time. */\n    } else {\n        MA_ZERO_OBJECT(&notifications);\n    }\n\n    /* For safety, always remove the ASYNC flag if threading is disabled on the resource manager. */\n    flags = pConfig->flags;\n    if (ma_resource_manager_is_threading_enabled(pResourceManager) == MA_FALSE) {\n        flags &= ~MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC;\n    }\n\n    if (pConfig->isLooping) {\n        flags |= MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING;\n    }\n\n    async = (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) != 0;\n\n    /*\n    Fences need to be acquired before doing anything. These must be acquired and released outside of\n    the node to ensure there's no holes where ma_fence_wait() could prematurely return before the\n    data buffer has completed initialization.\n\n    When loading asynchronously, the node acquisition routine below will acquire the fences on this\n    thread and then release them on the async thread when the operation is complete.\n\n    These fences are always released at the \"done\" tag at the end of this function. They'll be\n    acquired a second if loading asynchronously. This double acquisition system is just done to\n    simplify code maintenance.\n    */\n    ma_resource_manager_pipeline_notifications_acquire_all_fences(&notifications);\n    {\n        /* We first need to acquire a node. If ASYNC is not set, this will not return until the entire sound has been loaded. */\n        result = ma_resource_manager_data_buffer_node_acquire(pResourceManager, pConfig->pFilePath, pConfig->pFilePathW, hashedName32, flags, NULL, notifications.init.pFence, notifications.done.pFence, &pDataBufferNode);\n        if (result != MA_SUCCESS) {\n            ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n            goto done;\n        }\n\n        dataSourceConfig = ma_data_source_config_init();\n        dataSourceConfig.vtable = &g_ma_resource_manager_data_buffer_vtable;\n\n        result = ma_data_source_init(&dataSourceConfig, &pDataBuffer->ds);\n        if (result != MA_SUCCESS) {\n            ma_resource_manager_data_buffer_node_unacquire(pResourceManager, pDataBufferNode, NULL, NULL);\n            ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n            goto done;\n        }\n\n        pDataBuffer->pResourceManager = pResourceManager;\n        pDataBuffer->pNode  = pDataBufferNode;\n        pDataBuffer->flags  = flags;\n        pDataBuffer->result = MA_BUSY;  /* Always default to MA_BUSY for safety. It'll be overwritten when loading completes or an error occurs. */\n\n        /* If we're loading asynchronously we need to post a job to the job queue to initialize the connector. */\n        if (async == MA_FALSE || ma_resource_manager_data_buffer_node_result(pDataBufferNode) == MA_SUCCESS) {\n            /* Loading synchronously or the data has already been fully loaded. We can just initialize the connector from here without a job. */\n            result = ma_resource_manager_data_buffer_init_connector(pDataBuffer, pConfig, NULL, NULL);\n            ma_atomic_exchange_i32(&pDataBuffer->result, result);\n\n            ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n            goto done;\n        } else {\n            /* The node's data supply isn't initialized yet. The caller has requested that we load asynchronously so we need to post a job to do this. */\n            ma_job job;\n            ma_resource_manager_inline_notification initNotification;   /* Used when the WAIT_INIT flag is set. */\n\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                ma_resource_manager_inline_notification_init(pResourceManager, &initNotification);\n            }\n\n            /*\n            The status of the data buffer needs to be set to MA_BUSY before posting the job so that the\n            worker thread is aware of its busy state. If the LOAD_DATA_BUFFER job sees a status other\n            than MA_BUSY, it'll assume an error and fall through to an early exit.\n            */\n            ma_atomic_exchange_i32(&pDataBuffer->result, MA_BUSY);\n\n            /* Acquire fences a second time. These will be released by the async thread. */\n            ma_resource_manager_pipeline_notifications_acquire_all_fences(&notifications);\n\n            job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER);\n            job.order = ma_resource_manager_data_buffer_next_execution_order(pDataBuffer);\n            job.data.resourceManager.loadDataBuffer.pDataBuffer             = pDataBuffer;\n            job.data.resourceManager.loadDataBuffer.pInitNotification       = ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) ? &initNotification : notifications.init.pNotification;\n            job.data.resourceManager.loadDataBuffer.pDoneNotification       = notifications.done.pNotification;\n            job.data.resourceManager.loadDataBuffer.pInitFence              = notifications.init.pFence;\n            job.data.resourceManager.loadDataBuffer.pDoneFence              = notifications.done.pFence;\n            job.data.resourceManager.loadDataBuffer.rangeBegInPCMFrames     = pConfig->rangeBegInPCMFrames;\n            job.data.resourceManager.loadDataBuffer.rangeEndInPCMFrames     = pConfig->rangeEndInPCMFrames;\n            job.data.resourceManager.loadDataBuffer.loopPointBegInPCMFrames = pConfig->loopPointBegInPCMFrames;\n            job.data.resourceManager.loadDataBuffer.loopPointEndInPCMFrames = pConfig->loopPointEndInPCMFrames;\n            job.data.resourceManager.loadDataBuffer.isLooping               = (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING) != 0;\n\n            /* If we need to wait for initialization to complete we can just process the job in place. */\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                result = ma_job_process(&job);\n            } else {\n                result = ma_resource_manager_post_job(pResourceManager, &job);\n            }\n\n            if (result != MA_SUCCESS) {\n                /* We failed to post the job. Most likely there isn't enough room in the queue's buffer. */\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to post MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_BUFFER job. %s.\\n\", ma_result_description(result));\n                ma_atomic_exchange_i32(&pDataBuffer->result, result);\n\n                /* Release the fences after the result has been set on the data buffer. */\n                ma_resource_manager_pipeline_notifications_release_all_fences(&notifications);\n            } else {\n                if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                    ma_resource_manager_inline_notification_wait(&initNotification);\n\n                    if (notifications.init.pNotification != NULL) {\n                        ma_async_notification_signal(notifications.init.pNotification);\n                    }\n\n                    /* NOTE: Do not release the init fence here. It will have been done by the job. */\n\n                    /* Make sure we return an error if initialization failed on the async thread. */\n                    result = ma_resource_manager_data_buffer_result(pDataBuffer);\n                    if (result == MA_BUSY) {\n                        result  = MA_SUCCESS;\n                    }\n                }\n            }\n\n            if ((flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n                ma_resource_manager_inline_notification_uninit(&initNotification);\n            }\n        }\n\n        if (result != MA_SUCCESS) {\n            ma_resource_manager_data_buffer_node_unacquire(pResourceManager, pDataBufferNode, NULL, NULL);\n            goto done;\n        }\n    }\ndone:\n    if (result == MA_SUCCESS) {\n        if (pConfig->initialSeekPointInPCMFrames > 0) {\n            ma_resource_manager_data_buffer_seek_to_pcm_frame(pDataBuffer, pConfig->initialSeekPointInPCMFrames);\n        }\n    }\n\n    ma_resource_manager_pipeline_notifications_release_all_fences(&notifications);\n\n    return result;\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    return ma_resource_manager_data_buffer_init_ex_internal(pResourceManager, pConfig, 0, pDataBuffer);\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_init(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePath      = pFilePath;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_buffer_init_ex(pResourceManager, &config, pDataBuffer);\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_init_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePathW     = pFilePath;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_buffer_init_ex(pResourceManager, &config, pDataBuffer);\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_init_copy(ma_resource_manager* pResourceManager, const ma_resource_manager_data_buffer* pExistingDataBuffer, ma_resource_manager_data_buffer* pDataBuffer)\n{\n    ma_resource_manager_data_source_config config;\n\n    if (pExistingDataBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ASSERT(pExistingDataBuffer->pNode != NULL);  /* <-- If you've triggered this, you've passed in an invalid existing data buffer. */\n\n    config = ma_resource_manager_data_source_config_init();\n    config.flags = pExistingDataBuffer->flags;\n\n    return ma_resource_manager_data_buffer_init_ex_internal(pResourceManager, &config, pExistingDataBuffer->pNode->hashedName32, pDataBuffer);\n}\n\nstatic ma_result ma_resource_manager_data_buffer_uninit_internal(ma_resource_manager_data_buffer* pDataBuffer)\n{\n    MA_ASSERT(pDataBuffer != NULL);\n\n    /* The connector should be uninitialized first. */\n    ma_resource_manager_data_buffer_uninit_connector(pDataBuffer->pResourceManager, pDataBuffer);\n\n    /* With the connector uninitialized we can unacquire the node. */\n    ma_resource_manager_data_buffer_node_unacquire(pDataBuffer->pResourceManager, pDataBuffer->pNode, NULL, NULL);\n\n    /* The base data source needs to be uninitialized as well. */\n    ma_data_source_uninit(&pDataBuffer->ds);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_uninit(ma_resource_manager_data_buffer* pDataBuffer)\n{\n    ma_result result;\n\n    if (pDataBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_buffer_result(pDataBuffer) == MA_SUCCESS) {\n        /* The data buffer can be deleted synchronously. */\n        return ma_resource_manager_data_buffer_uninit_internal(pDataBuffer);\n    } else {\n        /*\n        The data buffer needs to be deleted asynchronously because it's still loading. With the status set to MA_UNAVAILABLE, no more pages will\n        be loaded and the uninitialization should happen fairly quickly. Since the caller owns the data buffer, we need to wait for this event\n        to get processed before returning.\n        */\n        ma_resource_manager_inline_notification notification;\n        ma_job job;\n\n        /*\n        We need to mark the node as unavailable so we don't try reading from it anymore, but also to\n        let the loading thread know that it needs to abort it's loading procedure.\n        */\n        ma_atomic_exchange_i32(&pDataBuffer->result, MA_UNAVAILABLE);\n\n        result = ma_resource_manager_inline_notification_init(pDataBuffer->pResourceManager, &notification);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to create the notification. This should rarely, if ever, happen. */\n        }\n\n        job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER);\n        job.order = ma_resource_manager_data_buffer_next_execution_order(pDataBuffer);\n        job.data.resourceManager.freeDataBuffer.pDataBuffer       = pDataBuffer;\n        job.data.resourceManager.freeDataBuffer.pDoneNotification = &notification;\n        job.data.resourceManager.freeDataBuffer.pDoneFence        = NULL;\n\n        result = ma_resource_manager_post_job(pDataBuffer->pResourceManager, &job);\n        if (result != MA_SUCCESS) {\n            ma_resource_manager_inline_notification_uninit(&notification);\n            return result;\n        }\n\n        ma_resource_manager_inline_notification_wait_and_uninit(&notification);\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_read_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 framesRead = 0;\n    ma_bool32 isDecodedBufferBusy = MA_FALSE;\n\n    /* Safety. */\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    We cannot be using the data buffer after it's been uninitialized. If you trigger this assert it means you're trying to read from the data buffer after\n    it's been uninitialized or is in the process of uninitializing.\n    */\n    MA_ASSERT(ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) != MA_UNAVAILABLE);\n\n    /* If the node is not initialized we need to abort with a busy code. */\n    if (ma_resource_manager_data_buffer_has_connector(pDataBuffer) == MA_FALSE) {\n        return MA_BUSY; /* Still loading. */\n    }\n\n    /*\n    If we've got a seek scheduled we'll want to do that before reading. However, for paged buffers, there's\n    a chance that the sound hasn't yet been decoded up to the seek point will result in the seek failing. If\n    this happens, we need to keep the seek scheduled and return MA_BUSY.\n    */\n    if (pDataBuffer->seekToCursorOnNextRead) {\n        pDataBuffer->seekToCursorOnNextRead = MA_FALSE;\n\n        result = ma_data_source_seek_to_pcm_frame(ma_resource_manager_data_buffer_get_connector(pDataBuffer), pDataBuffer->seekTargetInPCMFrames);\n        if (result != MA_SUCCESS) {\n            if (result == MA_BAD_SEEK && ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode) == ma_resource_manager_data_supply_type_decoded_paged) {\n                pDataBuffer->seekToCursorOnNextRead = MA_TRUE;  /* Keep the seek scheduled. We just haven't loaded enough data yet to do the seek properly. */\n                return MA_BUSY;\n            }\n\n            return result;\n        }\n    }\n\n    /*\n    For decoded buffers (not paged) we need to check beforehand how many frames we have available. We cannot\n    exceed this amount. We'll read as much as we can, and then return MA_BUSY.\n    */\n    if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode) == ma_resource_manager_data_supply_type_decoded) {\n        ma_uint64 availableFrames;\n\n        isDecodedBufferBusy = (ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) == MA_BUSY);\n\n        if (ma_resource_manager_data_buffer_get_available_frames(pDataBuffer, &availableFrames) == MA_SUCCESS) {\n            /* Don't try reading more than the available frame count if the data buffer node is still loading. */\n            if (isDecodedBufferBusy) {\n                if (frameCount > availableFrames) {\n                    frameCount = availableFrames;\n\n                    /*\n                    If there's no frames available we want to set the status to MA_AT_END. The logic below\n                    will check if the node is busy, and if so, change it to MA_BUSY. The reason we do this\n                    is because we don't want to call `ma_data_source_read_pcm_frames()` if the frame count\n                    is 0 because that'll result in a situation where it's possible MA_AT_END won't get\n                    returned.\n                    */\n                    if (frameCount == 0) {\n                        result = MA_AT_END;\n                    }\n                } else {\n                    isDecodedBufferBusy = MA_FALSE; /* We have enough frames available in the buffer to avoid a MA_BUSY status. */\n                }\n            } else {\n                /*\n                Getting here means the buffer has been fully loaded. We can just pass the frame count straight\n                into ma_data_source_read_pcm_frames() below and let ma_data_source handle it.\n                */\n            }\n        }\n    }\n\n    /* Don't attempt to read anything if we've got no frames available. */\n    if (frameCount > 0) {\n        result = ma_data_source_read_pcm_frames(ma_resource_manager_data_buffer_get_connector(pDataBuffer), pFramesOut, frameCount, &framesRead);\n    }\n\n    /*\n    If we returned MA_AT_END, but the node is still loading, we don't want to return that code or else the caller will interpret the sound\n    as at the end and terminate decoding.\n    */\n    if (result == MA_AT_END) {\n        if (ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) == MA_BUSY) {\n            result = MA_BUSY;\n        }\n    }\n\n    if (isDecodedBufferBusy) {\n        result = MA_BUSY;\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = framesRead;\n    }\n\n    if (result == MA_SUCCESS && framesRead == 0) {\n        result  = MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_seek_to_pcm_frame(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64 frameIndex)\n{\n    ma_result result;\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) != MA_UNAVAILABLE);\n\n    /* If we haven't yet got a connector we need to abort. */\n    if (ma_resource_manager_data_buffer_has_connector(pDataBuffer) == MA_FALSE) {\n        pDataBuffer->seekTargetInPCMFrames = frameIndex;\n        pDataBuffer->seekToCursorOnNextRead = MA_TRUE;\n        return MA_BUSY; /* Still loading. */\n    }\n\n    result = ma_data_source_seek_to_pcm_frame(ma_resource_manager_data_buffer_get_connector(pDataBuffer), frameIndex);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pDataBuffer->seekTargetInPCMFrames = ~(ma_uint64)0; /* <-- For identification purposes. */\n    pDataBuffer->seekToCursorOnNextRead = MA_FALSE;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_get_data_format(ma_resource_manager_data_buffer* pDataBuffer, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) != MA_UNAVAILABLE);\n\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode))\n    {\n        case ma_resource_manager_data_supply_type_encoded:\n        {\n            return ma_data_source_get_data_format(&pDataBuffer->connector.decoder, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n        };\n\n        case ma_resource_manager_data_supply_type_decoded:\n        {\n            *pFormat     = pDataBuffer->pNode->data.backend.decoded.format;\n            *pChannels   = pDataBuffer->pNode->data.backend.decoded.channels;\n            *pSampleRate = pDataBuffer->pNode->data.backend.decoded.sampleRate;\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pDataBuffer->pNode->data.backend.decoded.channels);\n            return MA_SUCCESS;\n        };\n\n        case ma_resource_manager_data_supply_type_decoded_paged:\n        {\n            *pFormat     = pDataBuffer->pNode->data.backend.decodedPaged.data.format;\n            *pChannels   = pDataBuffer->pNode->data.backend.decodedPaged.data.channels;\n            *pSampleRate = pDataBuffer->pNode->data.backend.decodedPaged.sampleRate;\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, pDataBuffer->pNode->data.backend.decoded.channels);\n            return MA_SUCCESS;\n        };\n\n        case ma_resource_manager_data_supply_type_unknown:\n        {\n            return MA_BUSY; /* Still loading. */\n        };\n\n        default:\n        {\n            /* Unknown supply type. Should never hit this. */\n            return MA_INVALID_ARGS;\n        }\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_get_cursor_in_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pCursor)\n{\n    if (pDataBuffer == NULL || pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) != MA_UNAVAILABLE);\n\n    *pCursor = 0;\n\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode))\n    {\n        case ma_resource_manager_data_supply_type_encoded:\n        {\n            return ma_decoder_get_cursor_in_pcm_frames(&pDataBuffer->connector.decoder, pCursor);\n        };\n\n        case ma_resource_manager_data_supply_type_decoded:\n        {\n            return ma_audio_buffer_get_cursor_in_pcm_frames(&pDataBuffer->connector.buffer, pCursor);\n        };\n\n        case ma_resource_manager_data_supply_type_decoded_paged:\n        {\n            return ma_paged_audio_buffer_get_cursor_in_pcm_frames(&pDataBuffer->connector.pagedBuffer, pCursor);\n        };\n\n        case ma_resource_manager_data_supply_type_unknown:\n        {\n            return MA_BUSY;\n        };\n\n        default:\n        {\n            return MA_INVALID_ARGS;\n        }\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_get_length_in_pcm_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pLength)\n{\n    if (pDataBuffer == NULL || pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) != MA_UNAVAILABLE);\n\n    if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode) == ma_resource_manager_data_supply_type_unknown) {\n        return MA_BUSY; /* Still loading. */\n    }\n\n    return ma_data_source_get_length_in_pcm_frames(ma_resource_manager_data_buffer_get_connector(pDataBuffer), pLength);\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_result(const ma_resource_manager_data_buffer* pDataBuffer)\n{\n    if (pDataBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return (ma_result)ma_atomic_load_i32((ma_result*)&pDataBuffer->result);    /* Need a naughty const-cast here. */\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_set_looping(ma_resource_manager_data_buffer* pDataBuffer, ma_bool32 isLooping)\n{\n    return ma_data_source_set_looping(pDataBuffer, isLooping);\n}\n\nMA_API ma_bool32 ma_resource_manager_data_buffer_is_looping(const ma_resource_manager_data_buffer* pDataBuffer)\n{\n    return ma_data_source_is_looping(pDataBuffer);\n}\n\nMA_API ma_result ma_resource_manager_data_buffer_get_available_frames(ma_resource_manager_data_buffer* pDataBuffer, ma_uint64* pAvailableFrames)\n{\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pDataBuffer == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode) == ma_resource_manager_data_supply_type_unknown) {\n        if (ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode) == MA_BUSY) {\n            return MA_BUSY;\n        } else {\n            return MA_INVALID_OPERATION;    /* No connector. */\n        }\n    }\n\n    switch (ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode))\n    {\n        case ma_resource_manager_data_supply_type_encoded:\n        {\n            return ma_decoder_get_available_frames(&pDataBuffer->connector.decoder, pAvailableFrames);\n        };\n\n        case ma_resource_manager_data_supply_type_decoded:\n        {\n            return ma_audio_buffer_get_available_frames(&pDataBuffer->connector.buffer, pAvailableFrames);\n        };\n\n        case ma_resource_manager_data_supply_type_decoded_paged:\n        {\n            ma_uint64 cursor;\n            ma_paged_audio_buffer_get_cursor_in_pcm_frames(&pDataBuffer->connector.pagedBuffer, &cursor);\n\n            if (pDataBuffer->pNode->data.backend.decodedPaged.decodedFrameCount > cursor) {\n                *pAvailableFrames = pDataBuffer->pNode->data.backend.decodedPaged.decodedFrameCount - cursor;\n            } else {\n                *pAvailableFrames = 0;\n            }\n\n            return MA_SUCCESS;\n        };\n\n        case ma_resource_manager_data_supply_type_unknown:\n        default:\n        {\n            /* Unknown supply type. Should never hit this. */\n            return MA_INVALID_ARGS;\n        }\n    }\n}\n\nMA_API ma_result ma_resource_manager_register_file(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags)\n{\n    return ma_resource_manager_data_buffer_node_acquire(pResourceManager, pFilePath, NULL, 0, flags, NULL, NULL, NULL, NULL);\n}\n\nMA_API ma_result ma_resource_manager_register_file_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags)\n{\n    return ma_resource_manager_data_buffer_node_acquire(pResourceManager, NULL, pFilePath, 0, flags, NULL, NULL, NULL, NULL);\n}\n\n\nstatic ma_result ma_resource_manager_register_data(ma_resource_manager* pResourceManager, const char* pName, const wchar_t* pNameW, ma_resource_manager_data_supply* pExistingData)\n{\n    return ma_resource_manager_data_buffer_node_acquire(pResourceManager, pName, pNameW, 0, 0, pExistingData, NULL, NULL, NULL);\n}\n\nstatic ma_result ma_resource_manager_register_decoded_data_internal(ma_resource_manager* pResourceManager, const char* pName, const wchar_t* pNameW, const void* pData, ma_uint64 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    ma_resource_manager_data_supply data;\n    data.type                            = ma_resource_manager_data_supply_type_decoded;\n    data.backend.decoded.pData           = pData;\n    data.backend.decoded.totalFrameCount = frameCount;\n    data.backend.decoded.format          = format;\n    data.backend.decoded.channels        = channels;\n    data.backend.decoded.sampleRate      = sampleRate;\n\n    return ma_resource_manager_register_data(pResourceManager, pName, pNameW, &data);\n}\n\nMA_API ma_result ma_resource_manager_register_decoded_data(ma_resource_manager* pResourceManager, const char* pName, const void* pData, ma_uint64 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    return ma_resource_manager_register_decoded_data_internal(pResourceManager, pName, NULL, pData, frameCount, format, channels, sampleRate);\n}\n\nMA_API ma_result ma_resource_manager_register_decoded_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName, const void* pData, ma_uint64 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    return ma_resource_manager_register_decoded_data_internal(pResourceManager, NULL, pName, pData, frameCount, format, channels, sampleRate);\n}\n\n\nstatic ma_result ma_resource_manager_register_encoded_data_internal(ma_resource_manager* pResourceManager, const char* pName, const wchar_t* pNameW, const void* pData, size_t sizeInBytes)\n{\n    ma_resource_manager_data_supply data;\n    data.type                        = ma_resource_manager_data_supply_type_encoded;\n    data.backend.encoded.pData       = pData;\n    data.backend.encoded.sizeInBytes = sizeInBytes;\n\n    return ma_resource_manager_register_data(pResourceManager, pName, pNameW, &data);\n}\n\nMA_API ma_result ma_resource_manager_register_encoded_data(ma_resource_manager* pResourceManager, const char* pName, const void* pData, size_t sizeInBytes)\n{\n    return ma_resource_manager_register_encoded_data_internal(pResourceManager, pName, NULL, pData, sizeInBytes);\n}\n\nMA_API ma_result ma_resource_manager_register_encoded_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName, const void* pData, size_t sizeInBytes)\n{\n    return ma_resource_manager_register_encoded_data_internal(pResourceManager, NULL, pName, pData, sizeInBytes);\n}\n\n\nMA_API ma_result ma_resource_manager_unregister_file(ma_resource_manager* pResourceManager, const char* pFilePath)\n{\n    return ma_resource_manager_unregister_data(pResourceManager, pFilePath);\n}\n\nMA_API ma_result ma_resource_manager_unregister_file_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath)\n{\n    return ma_resource_manager_unregister_data_w(pResourceManager, pFilePath);\n}\n\nMA_API ma_result ma_resource_manager_unregister_data(ma_resource_manager* pResourceManager, const char* pName)\n{\n    return ma_resource_manager_data_buffer_node_unacquire(pResourceManager, NULL, pName, NULL);\n}\n\nMA_API ma_result ma_resource_manager_unregister_data_w(ma_resource_manager* pResourceManager, const wchar_t* pName)\n{\n    return ma_resource_manager_data_buffer_node_unacquire(pResourceManager, NULL, NULL, pName);\n}\n\n\nstatic ma_uint32 ma_resource_manager_data_stream_next_execution_order(ma_resource_manager_data_stream* pDataStream)\n{\n    MA_ASSERT(pDataStream != NULL);\n    return ma_atomic_fetch_add_32(&pDataStream->executionCounter, 1);\n}\n\nstatic ma_bool32 ma_resource_manager_data_stream_is_decoder_at_end(const ma_resource_manager_data_stream* pDataStream)\n{\n    MA_ASSERT(pDataStream != NULL);\n    return ma_atomic_load_32((ma_bool32*)&pDataStream->isDecoderAtEnd);\n}\n\nstatic ma_uint32 ma_resource_manager_data_stream_seek_counter(const ma_resource_manager_data_stream* pDataStream)\n{\n    MA_ASSERT(pDataStream != NULL);\n    return ma_atomic_load_32((ma_uint32*)&pDataStream->seekCounter);\n}\n\n\nstatic ma_result ma_resource_manager_data_stream_cb__read_pcm_frames(ma_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    return ma_resource_manager_data_stream_read_pcm_frames((ma_resource_manager_data_stream*)pDataSource, pFramesOut, frameCount, pFramesRead);\n}\n\nstatic ma_result ma_resource_manager_data_stream_cb__seek_to_pcm_frame(ma_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    return ma_resource_manager_data_stream_seek_to_pcm_frame((ma_resource_manager_data_stream*)pDataSource, frameIndex);\n}\n\nstatic ma_result ma_resource_manager_data_stream_cb__get_data_format(ma_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    return ma_resource_manager_data_stream_get_data_format((ma_resource_manager_data_stream*)pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nstatic ma_result ma_resource_manager_data_stream_cb__get_cursor_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pCursor)\n{\n    return ma_resource_manager_data_stream_get_cursor_in_pcm_frames((ma_resource_manager_data_stream*)pDataSource, pCursor);\n}\n\nstatic ma_result ma_resource_manager_data_stream_cb__get_length_in_pcm_frames(ma_data_source* pDataSource, ma_uint64* pLength)\n{\n    return ma_resource_manager_data_stream_get_length_in_pcm_frames((ma_resource_manager_data_stream*)pDataSource, pLength);\n}\n\nstatic ma_result ma_resource_manager_data_stream_cb__set_looping(ma_data_source* pDataSource, ma_bool32 isLooping)\n{\n    ma_resource_manager_data_stream* pDataStream = (ma_resource_manager_data_stream*)pDataSource;\n    MA_ASSERT(pDataStream != NULL);\n\n    ma_atomic_exchange_32(&pDataStream->isLooping, isLooping);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_data_source_vtable g_ma_resource_manager_data_stream_vtable =\n{\n    ma_resource_manager_data_stream_cb__read_pcm_frames,\n    ma_resource_manager_data_stream_cb__seek_to_pcm_frame,\n    ma_resource_manager_data_stream_cb__get_data_format,\n    ma_resource_manager_data_stream_cb__get_cursor_in_pcm_frames,\n    ma_resource_manager_data_stream_cb__get_length_in_pcm_frames,\n    ma_resource_manager_data_stream_cb__set_looping,\n    0 /*MA_DATA_SOURCE_SELF_MANAGED_RANGE_AND_LOOP_POINT*/\n};\n\nstatic void ma_resource_manager_data_stream_set_absolute_cursor(ma_resource_manager_data_stream* pDataStream, ma_uint64 absoluteCursor)\n{\n    /* Loop if possible. */\n    if (absoluteCursor > pDataStream->totalLengthInPCMFrames && pDataStream->totalLengthInPCMFrames > 0) {\n        absoluteCursor = absoluteCursor % pDataStream->totalLengthInPCMFrames;\n    }\n\n    ma_atomic_exchange_64(&pDataStream->absoluteCursor, absoluteCursor);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_stream* pDataStream)\n{\n    ma_result result;\n    ma_data_source_config dataSourceConfig;\n    char* pFilePathCopy = NULL;\n    wchar_t* pFilePathWCopy = NULL;\n    ma_job job;\n    ma_bool32 waitBeforeReturning = MA_FALSE;\n    ma_resource_manager_inline_notification waitNotification;\n    ma_resource_manager_pipeline_notifications notifications;\n    ma_uint32 flags;\n\n    if (pDataStream == NULL) {\n        if (pConfig != NULL && pConfig->pNotifications != NULL) {\n            ma_resource_manager_pipeline_notifications_signal_all_notifications(pConfig->pNotifications);\n        }\n\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDataStream);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->pNotifications != NULL) {\n        notifications = *pConfig->pNotifications;    /* From here on out, `notifications` should be used instead of `pNotifications`. Setting this to NULL to catch any errors at testing time. */\n    } else {\n        MA_ZERO_OBJECT(&notifications);\n    }\n\n    dataSourceConfig = ma_data_source_config_init();\n    dataSourceConfig.vtable = &g_ma_resource_manager_data_stream_vtable;\n\n    result = ma_data_source_init(&dataSourceConfig, &pDataStream->ds);\n    if (result != MA_SUCCESS) {\n        ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n        return result;\n    }\n\n    flags = pConfig->flags;\n    if (pConfig->isLooping) {\n        flags |= MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING;\n    }\n\n    pDataStream->pResourceManager = pResourceManager;\n    pDataStream->flags            = pConfig->flags;\n    pDataStream->result           = MA_BUSY;\n\n    ma_data_source_set_range_in_pcm_frames(pDataStream, pConfig->rangeBegInPCMFrames, pConfig->rangeEndInPCMFrames);\n    ma_data_source_set_loop_point_in_pcm_frames(pDataStream, pConfig->loopPointBegInPCMFrames, pConfig->loopPointEndInPCMFrames);\n    ma_data_source_set_looping(pDataStream, (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING) != 0);\n\n    if (pResourceManager == NULL || (pConfig->pFilePath == NULL && pConfig->pFilePathW == NULL)) {\n        ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n        return MA_INVALID_ARGS;\n    }\n\n    /* We want all access to the VFS and the internal decoder to happen on the job thread just to keep things easier to manage for the VFS.  */\n\n    /* We need a copy of the file path. We should probably make this more efficient, but for now we'll do a transient memory allocation. */\n    if (pConfig->pFilePath != NULL) {\n        pFilePathCopy  = ma_copy_string(pConfig->pFilePath, &pResourceManager->config.allocationCallbacks);\n    } else {\n        pFilePathWCopy = ma_copy_string_w(pConfig->pFilePathW, &pResourceManager->config.allocationCallbacks);\n    }\n\n    if (pFilePathCopy == NULL && pFilePathWCopy == NULL) {\n        ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n        return MA_OUT_OF_MEMORY;\n    }\n\n    /*\n    We need to check for the presence of MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC. If it's not set, we need to wait before returning. Otherwise we\n    can return immediately. Likewise, we'll also check for MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT and do the same.\n    */\n    if ((pConfig->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_ASYNC) == 0 || (pConfig->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT) != 0) {\n        waitBeforeReturning = MA_TRUE;\n        ma_resource_manager_inline_notification_init(pResourceManager, &waitNotification);\n    }\n\n    ma_resource_manager_pipeline_notifications_acquire_all_fences(&notifications);\n\n    /* Set the absolute cursor to our initial seek position so retrieval of the cursor returns a good value. */\n    ma_resource_manager_data_stream_set_absolute_cursor(pDataStream, pConfig->initialSeekPointInPCMFrames);\n\n    /* We now have everything we need to post the job. This is the last thing we need to do from here. The rest will be done by the job thread. */\n    job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_LOAD_DATA_STREAM);\n    job.order = ma_resource_manager_data_stream_next_execution_order(pDataStream);\n    job.data.resourceManager.loadDataStream.pDataStream       = pDataStream;\n    job.data.resourceManager.loadDataStream.pFilePath         = pFilePathCopy;\n    job.data.resourceManager.loadDataStream.pFilePathW        = pFilePathWCopy;\n    job.data.resourceManager.loadDataStream.initialSeekPoint  = pConfig->initialSeekPointInPCMFrames;\n    job.data.resourceManager.loadDataStream.pInitNotification = (waitBeforeReturning == MA_TRUE) ? &waitNotification : notifications.init.pNotification;\n    job.data.resourceManager.loadDataStream.pInitFence        = notifications.init.pFence;\n    result = ma_resource_manager_post_job(pResourceManager, &job);\n    if (result != MA_SUCCESS) {\n        ma_resource_manager_pipeline_notifications_signal_all_notifications(&notifications);\n        ma_resource_manager_pipeline_notifications_release_all_fences(&notifications);\n\n        if (waitBeforeReturning) {\n            ma_resource_manager_inline_notification_uninit(&waitNotification);\n        }\n\n        ma_free(pFilePathCopy,  &pResourceManager->config.allocationCallbacks);\n        ma_free(pFilePathWCopy, &pResourceManager->config.allocationCallbacks);\n        return result;\n    }\n\n    /* Wait if needed. */\n    if (waitBeforeReturning) {\n        ma_resource_manager_inline_notification_wait_and_uninit(&waitNotification);\n\n        if (notifications.init.pNotification != NULL) {\n            ma_async_notification_signal(notifications.init.pNotification);\n        }\n\n        /*\n        If there was an error during initialization make sure we return that result here. We don't want to do this\n        if we're not waiting because it will most likely be in a busy state.\n        */\n        if (pDataStream->result != MA_SUCCESS) {\n            return pDataStream->result;\n        }\n\n        /* NOTE: Do not release pInitFence here. That will be done by the job. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_stream_init(ma_resource_manager* pResourceManager, const char* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_stream* pDataStream)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePath      = pFilePath;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_stream_init_ex(pResourceManager, &config, pDataStream);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_init_w(ma_resource_manager* pResourceManager, const wchar_t* pFilePath, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_stream* pDataStream)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePathW     = pFilePath;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_stream_init_ex(pResourceManager, &config, pDataStream);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_uninit(ma_resource_manager_data_stream* pDataStream)\n{\n    ma_resource_manager_inline_notification freeEvent;\n    ma_job job;\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The first thing to do is set the result to unavailable. This will prevent future page decoding. */\n    ma_atomic_exchange_i32(&pDataStream->result, MA_UNAVAILABLE);\n\n    /*\n    We need to post a job to ensure we're not in the middle or decoding or anything. Because the object is owned by the caller, we'll need\n    to wait for it to complete before returning which means we need an event.\n    */\n    ma_resource_manager_inline_notification_init(pDataStream->pResourceManager, &freeEvent);\n\n    job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_STREAM);\n    job.order = ma_resource_manager_data_stream_next_execution_order(pDataStream);\n    job.data.resourceManager.freeDataStream.pDataStream       = pDataStream;\n    job.data.resourceManager.freeDataStream.pDoneNotification = &freeEvent;\n    job.data.resourceManager.freeDataStream.pDoneFence        = NULL;\n    ma_resource_manager_post_job(pDataStream->pResourceManager, &job);\n\n    /* We need to wait for the job to finish processing before we return. */\n    ma_resource_manager_inline_notification_wait_and_uninit(&freeEvent);\n\n    return MA_SUCCESS;\n}\n\n\nstatic ma_uint32 ma_resource_manager_data_stream_get_page_size_in_frames(ma_resource_manager_data_stream* pDataStream)\n{\n    MA_ASSERT(pDataStream != NULL);\n    MA_ASSERT(pDataStream->isDecoderInitialized == MA_TRUE);\n\n    return MA_RESOURCE_MANAGER_PAGE_SIZE_IN_MILLISECONDS * (pDataStream->decoder.outputSampleRate/1000);\n}\n\nstatic void* ma_resource_manager_data_stream_get_page_data_pointer(ma_resource_manager_data_stream* pDataStream, ma_uint32 pageIndex, ma_uint32 relativeCursor)\n{\n    MA_ASSERT(pDataStream != NULL);\n    MA_ASSERT(pDataStream->isDecoderInitialized == MA_TRUE);\n    MA_ASSERT(pageIndex == 0 || pageIndex == 1);\n\n    return ma_offset_ptr(pDataStream->pPageData, ((ma_resource_manager_data_stream_get_page_size_in_frames(pDataStream) * pageIndex) + relativeCursor) * ma_get_bytes_per_frame(pDataStream->decoder.outputFormat, pDataStream->decoder.outputChannels));\n}\n\nstatic void ma_resource_manager_data_stream_fill_page(ma_resource_manager_data_stream* pDataStream, ma_uint32 pageIndex)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 pageSizeInFrames;\n    ma_uint64 totalFramesReadForThisPage = 0;\n    void* pPageData = ma_resource_manager_data_stream_get_page_data_pointer(pDataStream, pageIndex, 0);\n\n    pageSizeInFrames = ma_resource_manager_data_stream_get_page_size_in_frames(pDataStream);\n\n    /* The decoder needs to inherit the stream's looping and range state. */\n    {\n        ma_uint64 rangeBeg;\n        ma_uint64 rangeEnd;\n        ma_uint64 loopPointBeg;\n        ma_uint64 loopPointEnd;\n\n        ma_data_source_set_looping(&pDataStream->decoder, ma_resource_manager_data_stream_is_looping(pDataStream));\n\n        ma_data_source_get_range_in_pcm_frames(pDataStream, &rangeBeg, &rangeEnd);\n        ma_data_source_set_range_in_pcm_frames(&pDataStream->decoder, rangeBeg, rangeEnd);\n\n        ma_data_source_get_loop_point_in_pcm_frames(pDataStream, &loopPointBeg, &loopPointEnd);\n        ma_data_source_set_loop_point_in_pcm_frames(&pDataStream->decoder, loopPointBeg, loopPointEnd);\n    }\n\n    /* Just read straight from the decoder. It will deal with ranges and looping for us. */\n    result = ma_data_source_read_pcm_frames(&pDataStream->decoder, pPageData, pageSizeInFrames, &totalFramesReadForThisPage);\n    if (result == MA_AT_END || totalFramesReadForThisPage < pageSizeInFrames) {\n        ma_atomic_exchange_32(&pDataStream->isDecoderAtEnd, MA_TRUE);\n    }\n\n    ma_atomic_exchange_32(&pDataStream->pageFrameCount[pageIndex], (ma_uint32)totalFramesReadForThisPage);\n    ma_atomic_exchange_32(&pDataStream->isPageValid[pageIndex], MA_TRUE);\n}\n\nstatic void ma_resource_manager_data_stream_fill_pages(ma_resource_manager_data_stream* pDataStream)\n{\n    ma_uint32 iPage;\n\n    MA_ASSERT(pDataStream != NULL);\n\n    for (iPage = 0; iPage < 2; iPage += 1) {\n        ma_resource_manager_data_stream_fill_page(pDataStream, iPage);\n    }\n}\n\n\nstatic ma_result ma_resource_manager_data_stream_map(ma_resource_manager_data_stream* pDataStream, void** ppFramesOut, ma_uint64* pFrameCount)\n{\n    ma_uint64 framesAvailable;\n    ma_uint64 frameCount = 0;\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) != MA_UNAVAILABLE);\n\n    if (pFrameCount != NULL) {\n        frameCount = *pFrameCount;\n        *pFrameCount = 0;\n    }\n    if (ppFramesOut != NULL) {\n        *ppFramesOut = NULL;\n    }\n\n    if (pDataStream == NULL || ppFramesOut == NULL || pFrameCount == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* Don't attempt to read while we're in the middle of seeking. Tell the caller that we're busy. */\n    if (ma_resource_manager_data_stream_seek_counter(pDataStream) > 0) {\n        return MA_BUSY;\n    }\n\n    /* If the page we're on is invalid it means we've caught up to the job thread. */\n    if (ma_atomic_load_32(&pDataStream->isPageValid[pDataStream->currentPageIndex]) == MA_FALSE) {\n        framesAvailable = 0;\n    } else {\n        /*\n        The page we're on is valid so we must have some frames available. We need to make sure that we don't overflow into the next page, even if it's valid. The reason is\n        that the unmap process will only post an update for one page at a time. Keeping mapping tied to page boundaries makes this simpler.\n        */\n        ma_uint32 currentPageFrameCount = ma_atomic_load_32(&pDataStream->pageFrameCount[pDataStream->currentPageIndex]);\n        MA_ASSERT(currentPageFrameCount >= pDataStream->relativeCursor);\n\n        framesAvailable = currentPageFrameCount - pDataStream->relativeCursor;\n    }\n\n    /* If there's no frames available and the result is set to MA_AT_END we need to return MA_AT_END. */\n    if (framesAvailable == 0) {\n        if (ma_resource_manager_data_stream_is_decoder_at_end(pDataStream)) {\n            return MA_AT_END;\n        } else {\n            return MA_BUSY; /* There are no frames available, but we're not marked as EOF so we might have caught up to the job thread. Need to return MA_BUSY and wait for more data. */\n        }\n    }\n\n    MA_ASSERT(framesAvailable > 0);\n\n    if (frameCount > framesAvailable) {\n        frameCount = framesAvailable;\n    }\n\n    *ppFramesOut = ma_resource_manager_data_stream_get_page_data_pointer(pDataStream, pDataStream->currentPageIndex, pDataStream->relativeCursor);\n    *pFrameCount = frameCount;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_resource_manager_data_stream_unmap(ma_resource_manager_data_stream* pDataStream, ma_uint64 frameCount)\n{\n    ma_uint32 newRelativeCursor;\n    ma_uint32 pageSizeInFrames;\n    ma_job job;\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) != MA_UNAVAILABLE);\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* The frame count should always fit inside a 32-bit integer. */\n    if (frameCount > 0xFFFFFFFF) {\n        return MA_INVALID_ARGS;\n    }\n\n    pageSizeInFrames = ma_resource_manager_data_stream_get_page_size_in_frames(pDataStream);\n\n    /* The absolute cursor needs to be updated for ma_resource_manager_data_stream_get_cursor_in_pcm_frames(). */\n    ma_resource_manager_data_stream_set_absolute_cursor(pDataStream, ma_atomic_load_64(&pDataStream->absoluteCursor) + frameCount);\n\n    /* Here is where we need to check if we need to load a new page, and if so, post a job to load it. */\n    newRelativeCursor = pDataStream->relativeCursor + (ma_uint32)frameCount;\n\n    /* If the new cursor has flowed over to the next page we need to mark the old one as invalid and post an event for it. */\n    if (newRelativeCursor >= pageSizeInFrames) {\n        newRelativeCursor -= pageSizeInFrames;\n\n        /* Here is where we post the job start decoding. */\n        job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_STREAM);\n        job.order = ma_resource_manager_data_stream_next_execution_order(pDataStream);\n        job.data.resourceManager.pageDataStream.pDataStream = pDataStream;\n        job.data.resourceManager.pageDataStream.pageIndex   = pDataStream->currentPageIndex;\n\n        /* The page needs to be marked as invalid so that the public API doesn't try reading from it. */\n        ma_atomic_exchange_32(&pDataStream->isPageValid[pDataStream->currentPageIndex], MA_FALSE);\n\n        /* Before posting the job we need to make sure we set some state. */\n        pDataStream->relativeCursor   = newRelativeCursor;\n        pDataStream->currentPageIndex = (pDataStream->currentPageIndex + 1) & 0x01;\n        return ma_resource_manager_post_job(pDataStream->pResourceManager, &job);\n    } else {\n        /* We haven't moved into a new page so we can just move the cursor forward. */\n        pDataStream->relativeCursor = newRelativeCursor;\n        return MA_SUCCESS;\n    }\n}\n\n\nMA_API ma_result ma_resource_manager_data_stream_read_pcm_frames(ma_resource_manager_data_stream* pDataStream, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 totalFramesProcessed;\n    ma_format format;\n    ma_uint32 channels;\n\n    /* Safety. */\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (frameCount == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) != MA_UNAVAILABLE);\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* Don't attempt to read while we're in the middle of seeking. Tell the caller that we're busy. */\n    if (ma_resource_manager_data_stream_seek_counter(pDataStream) > 0) {\n        return MA_BUSY;\n    }\n\n    ma_resource_manager_data_stream_get_data_format(pDataStream, &format, &channels, NULL, NULL, 0);\n\n    /* Reading is implemented in terms of map/unmap. We need to run this in a loop because mapping is clamped against page boundaries. */\n    totalFramesProcessed = 0;\n    while (totalFramesProcessed < frameCount) {\n        void* pMappedFrames;\n        ma_uint64 mappedFrameCount;\n\n        mappedFrameCount = frameCount - totalFramesProcessed;\n        result = ma_resource_manager_data_stream_map(pDataStream, &pMappedFrames, &mappedFrameCount);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n\n        /* Copy the mapped data to the output buffer if we have one. It's allowed for pFramesOut to be NULL in which case a relative forward seek is performed. */\n        if (pFramesOut != NULL) {\n            ma_copy_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, totalFramesProcessed, format, channels), pMappedFrames, mappedFrameCount, format, channels);\n        }\n\n        totalFramesProcessed += mappedFrameCount;\n\n        result = ma_resource_manager_data_stream_unmap(pDataStream, mappedFrameCount);\n        if (result != MA_SUCCESS) {\n            break;  /* This is really bad - will only get an error here if we failed to post a job to the queue for loading the next page. */\n        }\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesProcessed;\n    }\n\n    if (result == MA_SUCCESS && totalFramesProcessed == 0) {\n        result  = MA_AT_END;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_resource_manager_data_stream_seek_to_pcm_frame(ma_resource_manager_data_stream* pDataStream, ma_uint64 frameIndex)\n{\n    ma_job job;\n    ma_result streamResult;\n\n    streamResult = ma_resource_manager_data_stream_result(pDataStream);\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(streamResult != MA_UNAVAILABLE);\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (streamResult != MA_SUCCESS && streamResult != MA_BUSY) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* If we're not already seeking and we're sitting on the same frame, just make this a no-op. */\n    if (ma_atomic_load_32(&pDataStream->seekCounter) == 0) {\n        if (ma_atomic_load_64(&pDataStream->absoluteCursor) == frameIndex) {\n            return MA_SUCCESS;\n        }\n    }\n\n\n    /* Increment the seek counter first to indicate to read_paged_pcm_frames() and map_paged_pcm_frames() that we are in the middle of a seek and MA_BUSY should be returned. */\n    ma_atomic_fetch_add_32(&pDataStream->seekCounter, 1);\n\n    /* Update the absolute cursor so that ma_resource_manager_data_stream_get_cursor_in_pcm_frames() returns the new position. */\n    ma_resource_manager_data_stream_set_absolute_cursor(pDataStream, frameIndex);\n\n    /*\n    We need to clear our currently loaded pages so that the stream starts playback from the new seek point as soon as possible. These are for the purpose of the public\n    API and will be ignored by the seek job. The seek job will operate on the assumption that both pages have been marked as invalid and the cursor is at the start of\n    the first page.\n    */\n    pDataStream->relativeCursor   = 0;\n    pDataStream->currentPageIndex = 0;\n    ma_atomic_exchange_32(&pDataStream->isPageValid[0], MA_FALSE);\n    ma_atomic_exchange_32(&pDataStream->isPageValid[1], MA_FALSE);\n\n    /* Make sure the data stream is not marked as at the end or else if we seek in response to hitting the end, we won't be able to read any more data. */\n    ma_atomic_exchange_32(&pDataStream->isDecoderAtEnd, MA_FALSE);\n\n    /*\n    The public API is not allowed to touch the internal decoder so we need to use a job to perform the seek. When seeking, the job thread will assume both pages\n    are invalid and any content contained within them will be discarded and replaced with newly decoded data.\n    */\n    job = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_SEEK_DATA_STREAM);\n    job.order = ma_resource_manager_data_stream_next_execution_order(pDataStream);\n    job.data.resourceManager.seekDataStream.pDataStream = pDataStream;\n    job.data.resourceManager.seekDataStream.frameIndex  = frameIndex;\n    return ma_resource_manager_post_job(pDataStream->pResourceManager, &job);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_get_data_format(ma_resource_manager_data_stream* pDataStream, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) != MA_UNAVAILABLE);\n\n    if (pFormat != NULL) {\n        *pFormat = ma_format_unknown;\n    }\n\n    if (pChannels != NULL) {\n        *pChannels = 0;\n    }\n\n    if (pSampleRate != NULL) {\n        *pSampleRate = 0;\n    }\n\n    if (pChannelMap != NULL) {\n        MA_ZERO_MEMORY(pChannelMap, sizeof(*pChannelMap) * channelMapCap);\n    }\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /*\n    We're being a little bit naughty here and accessing the internal decoder from the public API. The output data format is constant, and we've defined this function\n    such that the application is responsible for ensuring it's not called while uninitializing so it should be safe.\n    */\n    return ma_data_source_get_data_format(&pDataStream->decoder, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_get_cursor_in_pcm_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pCursor)\n{\n    ma_result result;\n\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pCursor = 0;\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) != MA_UNAVAILABLE);\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    If the stream is in an erroneous state we need to return an invalid operation. We can allow\n    this to be called when the data stream is in a busy state because the caller may have asked\n    for an initial seek position and it's convenient to return that as the cursor position.\n    */\n    result = ma_resource_manager_data_stream_result(pDataStream);\n    if (result != MA_SUCCESS && result != MA_BUSY) {\n        return MA_INVALID_OPERATION;\n    }\n\n    *pCursor = ma_atomic_load_64(&pDataStream->absoluteCursor);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_stream_get_length_in_pcm_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pLength)\n{\n    ma_result streamResult;\n\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pLength = 0;\n\n    streamResult = ma_resource_manager_data_stream_result(pDataStream);\n\n    /* We cannot be using the data source after it's been uninitialized. */\n    MA_ASSERT(streamResult != MA_UNAVAILABLE);\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (streamResult != MA_SUCCESS) {\n        return streamResult;\n    }\n\n    /*\n    We most definitely do not want to be calling ma_decoder_get_length_in_pcm_frames() directly. Instead we want to use a cached value that we\n    calculated when we initialized it on the job thread.\n    */\n    *pLength = pDataStream->totalLengthInPCMFrames;\n    if (*pLength == 0) {\n        return MA_NOT_IMPLEMENTED;  /* Some decoders may not have a known length. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_stream_result(const ma_resource_manager_data_stream* pDataStream)\n{\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return (ma_result)ma_atomic_load_i32(&pDataStream->result);\n}\n\nMA_API ma_result ma_resource_manager_data_stream_set_looping(ma_resource_manager_data_stream* pDataStream, ma_bool32 isLooping)\n{\n    return ma_data_source_set_looping(pDataStream, isLooping);\n}\n\nMA_API ma_bool32 ma_resource_manager_data_stream_is_looping(const ma_resource_manager_data_stream* pDataStream)\n{\n    if (pDataStream == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_atomic_load_32((ma_bool32*)&pDataStream->isLooping);   /* Naughty const-cast. Value won't change from here in practice (maybe from another thread). */\n}\n\nMA_API ma_result ma_resource_manager_data_stream_get_available_frames(ma_resource_manager_data_stream* pDataStream, ma_uint64* pAvailableFrames)\n{\n    ma_uint32 pageIndex0;\n    ma_uint32 pageIndex1;\n    ma_uint32 relativeCursor;\n    ma_uint64 availableFrames;\n\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pDataStream == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pageIndex0     =  pDataStream->currentPageIndex;\n    pageIndex1     = (pDataStream->currentPageIndex + 1) & 0x01;\n    relativeCursor =  pDataStream->relativeCursor;\n\n    availableFrames = 0;\n    if (ma_atomic_load_32(&pDataStream->isPageValid[pageIndex0])) {\n        availableFrames += ma_atomic_load_32(&pDataStream->pageFrameCount[pageIndex0]) - relativeCursor;\n        if (ma_atomic_load_32(&pDataStream->isPageValid[pageIndex1])) {\n            availableFrames += ma_atomic_load_32(&pDataStream->pageFrameCount[pageIndex1]);\n        }\n    }\n\n    *pAvailableFrames = availableFrames;\n    return MA_SUCCESS;\n}\n\n\nstatic ma_result ma_resource_manager_data_source_preinit(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_source* pDataSource)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDataSource);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pDataSource->flags = pConfig->flags;\n    if (pConfig->isLooping) {\n        pDataSource->flags |= MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_resource_manager_data_source_init_ex(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source_config* pConfig, ma_resource_manager_data_source* pDataSource)\n{\n    ma_result result;\n\n    result = ma_resource_manager_data_source_preinit(pResourceManager, pConfig, pDataSource);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* The data source itself is just a data stream or a data buffer. */\n    if ((pConfig->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_init_ex(pResourceManager, pConfig, &pDataSource->backend.stream);\n    } else {\n        return ma_resource_manager_data_buffer_init_ex(pResourceManager, pConfig, &pDataSource->backend.buffer);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_init(ma_resource_manager* pResourceManager, const char* pName, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_source* pDataSource)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePath      = pName;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_source_init_ex(pResourceManager, &config, pDataSource);\n}\n\nMA_API ma_result ma_resource_manager_data_source_init_w(ma_resource_manager* pResourceManager, const wchar_t* pName, ma_uint32 flags, const ma_resource_manager_pipeline_notifications* pNotifications, ma_resource_manager_data_source* pDataSource)\n{\n    ma_resource_manager_data_source_config config;\n\n    config = ma_resource_manager_data_source_config_init();\n    config.pFilePathW     = pName;\n    config.flags          = flags;\n    config.pNotifications = pNotifications;\n\n    return ma_resource_manager_data_source_init_ex(pResourceManager, &config, pDataSource);\n}\n\nMA_API ma_result ma_resource_manager_data_source_init_copy(ma_resource_manager* pResourceManager, const ma_resource_manager_data_source* pExistingDataSource, ma_resource_manager_data_source* pDataSource)\n{\n    ma_result result;\n    ma_resource_manager_data_source_config config;\n\n    if (pExistingDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    config = ma_resource_manager_data_source_config_init();\n    config.flags = pExistingDataSource->flags;\n\n    result = ma_resource_manager_data_source_preinit(pResourceManager, &config, pDataSource);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Copying can only be done from data buffers. Streams cannot be copied. */\n    if ((pExistingDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return MA_INVALID_OPERATION;\n    }\n\n    return ma_resource_manager_data_buffer_init_copy(pResourceManager, &pExistingDataSource->backend.buffer, &pDataSource->backend.buffer);\n}\n\nMA_API ma_result ma_resource_manager_data_source_uninit(ma_resource_manager_data_source* pDataSource)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* All we need to is uninitialize the underlying data buffer or data stream. */\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_uninit(&pDataSource->backend.stream);\n    } else {\n        return ma_resource_manager_data_buffer_uninit(&pDataSource->backend.buffer);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_read_pcm_frames(ma_resource_manager_data_source* pDataSource, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    /* Safety. */\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_read_pcm_frames(&pDataSource->backend.stream, pFramesOut, frameCount, pFramesRead);\n    } else {\n        return ma_resource_manager_data_buffer_read_pcm_frames(&pDataSource->backend.buffer, pFramesOut, frameCount, pFramesRead);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_seek_to_pcm_frame(ma_resource_manager_data_source* pDataSource, ma_uint64 frameIndex)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_seek_to_pcm_frame(&pDataSource->backend.stream, frameIndex);\n    } else {\n        return ma_resource_manager_data_buffer_seek_to_pcm_frame(&pDataSource->backend.buffer, frameIndex);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_map(ma_resource_manager_data_source* pDataSource, void** ppFramesOut, ma_uint64* pFrameCount)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_map(&pDataSource->backend.stream, ppFramesOut, pFrameCount);\n    } else {\n        return MA_NOT_IMPLEMENTED;  /* Mapping not supported with data buffers. */\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_unmap(ma_resource_manager_data_source* pDataSource, ma_uint64 frameCount)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_unmap(&pDataSource->backend.stream, frameCount);\n    } else {\n        return MA_NOT_IMPLEMENTED;  /* Mapping not supported with data buffers. */\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_get_data_format(ma_resource_manager_data_source* pDataSource, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_get_data_format(&pDataSource->backend.stream, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n    } else {\n        return ma_resource_manager_data_buffer_get_data_format(&pDataSource->backend.buffer, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_get_cursor_in_pcm_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pCursor)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_get_cursor_in_pcm_frames(&pDataSource->backend.stream, pCursor);\n    } else {\n        return ma_resource_manager_data_buffer_get_cursor_in_pcm_frames(&pDataSource->backend.buffer, pCursor);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_get_length_in_pcm_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pLength)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_get_length_in_pcm_frames(&pDataSource->backend.stream, pLength);\n    } else {\n        return ma_resource_manager_data_buffer_get_length_in_pcm_frames(&pDataSource->backend.buffer, pLength);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_result(const ma_resource_manager_data_source* pDataSource)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_result(&pDataSource->backend.stream);\n    } else {\n        return ma_resource_manager_data_buffer_result(&pDataSource->backend.buffer);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_set_looping(ma_resource_manager_data_source* pDataSource, ma_bool32 isLooping)\n{\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_set_looping(&pDataSource->backend.stream, isLooping);\n    } else {\n        return ma_resource_manager_data_buffer_set_looping(&pDataSource->backend.buffer, isLooping);\n    }\n}\n\nMA_API ma_bool32 ma_resource_manager_data_source_is_looping(const ma_resource_manager_data_source* pDataSource)\n{\n    if (pDataSource == NULL) {\n        return MA_FALSE;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_is_looping(&pDataSource->backend.stream);\n    } else {\n        return ma_resource_manager_data_buffer_is_looping(&pDataSource->backend.buffer);\n    }\n}\n\nMA_API ma_result ma_resource_manager_data_source_get_available_frames(ma_resource_manager_data_source* pDataSource, ma_uint64* pAvailableFrames)\n{\n    if (pAvailableFrames == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pAvailableFrames = 0;\n\n    if (pDataSource == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if ((pDataSource->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_STREAM) != 0) {\n        return ma_resource_manager_data_stream_get_available_frames(&pDataSource->backend.stream, pAvailableFrames);\n    } else {\n        return ma_resource_manager_data_buffer_get_available_frames(&pDataSource->backend.buffer, pAvailableFrames);\n    }\n}\n\n\nMA_API ma_result ma_resource_manager_post_job(ma_resource_manager* pResourceManager, const ma_job* pJob)\n{\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_job_queue_post(&pResourceManager->jobQueue, pJob);\n}\n\nMA_API ma_result ma_resource_manager_post_job_quit(ma_resource_manager* pResourceManager)\n{\n    ma_job job = ma_job_init(MA_JOB_TYPE_QUIT);\n    return ma_resource_manager_post_job(pResourceManager, &job);\n}\n\nMA_API ma_result ma_resource_manager_next_job(ma_resource_manager* pResourceManager, ma_job* pJob)\n{\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_job_queue_next(&pResourceManager->jobQueue, pJob);\n}\n\n\nstatic ma_result ma_job_process__resource_manager__load_data_buffer_node(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_buffer_node* pDataBufferNode;\n\n    MA_ASSERT(pJob != NULL);\n\n    pResourceManager = (ma_resource_manager*)pJob->data.resourceManager.loadDataBufferNode.pResourceManager;\n    MA_ASSERT(pResourceManager != NULL);\n\n    pDataBufferNode = (ma_resource_manager_data_buffer_node*)pJob->data.resourceManager.loadDataBufferNode.pDataBufferNode;\n    MA_ASSERT(pDataBufferNode != NULL);\n    MA_ASSERT(pDataBufferNode->isDataOwnedByResourceManager == MA_TRUE);  /* The data should always be owned by the resource manager. */\n\n    /* The data buffer is not getting deleted, but we may be getting executed out of order. If so, we need to push the job back onto the queue and return. */\n    if (pJob->order != ma_atomic_load_32(&pDataBufferNode->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Attempting to execute out of order. Probably interleaved with a MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER job. */\n    }\n\n    /* First thing we need to do is check whether or not the data buffer is getting deleted. If so we just abort. */\n    if (ma_resource_manager_data_buffer_node_result(pDataBufferNode) != MA_BUSY) {\n        result = ma_resource_manager_data_buffer_node_result(pDataBufferNode);    /* The data buffer may be getting deleted before it's even been loaded. */\n        goto done;\n    }\n\n    /*\n    We're ready to start loading. Essentially what we're doing here is initializing the data supply\n    of the node. Once this is complete, data buffers can have their connectors initialized which\n    will allow then to have audio data read from them.\n\n    Note that when the data supply type has been moved away from \"unknown\", that is when other threads\n    will determine that the node is available for data delivery and the data buffer connectors can be\n    initialized. Therefore, it's important that it is set after the data supply has been initialized.\n    */\n    if ((pJob->data.resourceManager.loadDataBufferNode.flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_DECODE) != 0) {\n        /*\n        Decoding. This is the complex case because we're not going to be doing the entire decoding\n        process here. Instead it's going to be split of multiple jobs and loaded in pages. The\n        reason for this is to evenly distribute decoding time across multiple sounds, rather than\n        having one huge sound hog all the available processing resources.\n\n        The first thing we do is initialize a decoder. This is allocated on the heap and is passed\n        around to the paging jobs. When the last paging job has completed it's processing, it'll\n        free the decoder for us.\n\n        This job does not do any actual decoding. It instead just posts a PAGE_DATA_BUFFER_NODE job\n        which is where the actual decoding work will be done. However, once this job is complete,\n        the node will be in a state where data buffer connectors can be initialized.\n        */\n        ma_decoder* pDecoder;   /* <-- Free'd on the last page decode. */\n        ma_job pageDataBufferNodeJob;\n\n        /* Allocate the decoder by initializing a decoded data supply. */\n        result = ma_resource_manager_data_buffer_node_init_supply_decoded(pResourceManager, pDataBufferNode, pJob->data.resourceManager.loadDataBufferNode.pFilePath, pJob->data.resourceManager.loadDataBufferNode.pFilePathW, pJob->data.resourceManager.loadDataBufferNode.flags, &pDecoder);\n\n        /*\n        Don't ever propagate an MA_BUSY result code or else the resource manager will think the\n        node is just busy decoding rather than in an error state. This should never happen, but\n        including this logic for safety just in case.\n        */\n        if (result == MA_BUSY) {\n            result  = MA_ERROR;\n        }\n\n        if (result != MA_SUCCESS) {\n            if (pJob->data.resourceManager.loadDataBufferNode.pFilePath != NULL) {\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to initialize data supply for \\\"%s\\\". %s.\\n\", pJob->data.resourceManager.loadDataBufferNode.pFilePath, ma_result_description(result));\n            } else {\n                #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(_MSC_VER)\n                    ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_WARNING, \"Failed to initialize data supply for \\\"%ls\\\", %s.\\n\", pJob->data.resourceManager.loadDataBufferNode.pFilePathW, ma_result_description(result));\n                #endif\n            }\n\n            goto done;\n        }\n\n        /*\n        At this point the node's data supply is initialized and other threads can start initializing\n        their data buffer connectors. However, no data will actually be available until we start to\n        actually decode it. To do this, we need to post a paging job which is where the decoding\n        work is done.\n\n        Note that if an error occurred at an earlier point, this section will have been skipped.\n        */\n        pageDataBufferNodeJob = ma_job_init(MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE);\n        pageDataBufferNodeJob.order = ma_resource_manager_data_buffer_node_next_execution_order(pDataBufferNode);\n        pageDataBufferNodeJob.data.resourceManager.pageDataBufferNode.pResourceManager  = pResourceManager;\n        pageDataBufferNodeJob.data.resourceManager.pageDataBufferNode.pDataBufferNode   = pDataBufferNode;\n        pageDataBufferNodeJob.data.resourceManager.pageDataBufferNode.pDecoder          = pDecoder;\n        pageDataBufferNodeJob.data.resourceManager.pageDataBufferNode.pDoneNotification = pJob->data.resourceManager.loadDataBufferNode.pDoneNotification;\n        pageDataBufferNodeJob.data.resourceManager.pageDataBufferNode.pDoneFence        = pJob->data.resourceManager.loadDataBufferNode.pDoneFence;\n\n        /* The job has been set up so it can now be posted. */\n        result = ma_resource_manager_post_job(pResourceManager, &pageDataBufferNodeJob);\n\n        /*\n        When we get here, we want to make sure the result code is set to MA_BUSY. The reason for\n        this is that the result will be copied over to the node's internal result variable. In\n        this case, since the decoding is still in-progress, we need to make sure the result code\n        is set to MA_BUSY.\n        */\n        if (result != MA_SUCCESS) {\n            ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to post MA_JOB_TYPE_RESOURCE_MANAGER_PAGE_DATA_BUFFER_NODE job. %s\\n\", ma_result_description(result));\n            ma_decoder_uninit(pDecoder);\n            ma_free(pDecoder, &pResourceManager->config.allocationCallbacks);\n        } else {\n            result = MA_BUSY;\n        }\n    } else {\n        /* No decoding. This is the simple case. We need only read the file content into memory and we're done. */\n        result = ma_resource_manager_data_buffer_node_init_supply_encoded(pResourceManager, pDataBufferNode, pJob->data.resourceManager.loadDataBufferNode.pFilePath, pJob->data.resourceManager.loadDataBufferNode.pFilePathW);\n    }\n\n\ndone:\n    /* File paths are no longer needed. */\n    ma_free(pJob->data.resourceManager.loadDataBufferNode.pFilePath,  &pResourceManager->config.allocationCallbacks);\n    ma_free(pJob->data.resourceManager.loadDataBufferNode.pFilePathW, &pResourceManager->config.allocationCallbacks);\n\n    /*\n    We need to set the result to at the very end to ensure no other threads try reading the data before we've fully initialized the object. Other threads\n    are going to be inspecting this variable to determine whether or not they're ready to read data. We can only change the result if it's set to MA_BUSY\n    because otherwise we may be changing away from an error code which would be bad. An example is if the application creates a data buffer, but then\n    immediately deletes it before we've got to this point. In this case, pDataBuffer->result will be MA_UNAVAILABLE, and setting it to MA_SUCCESS or any\n    other error code would cause the buffer to look like it's in a state that it's not.\n    */\n    ma_atomic_compare_and_swap_i32(&pDataBufferNode->result, MA_BUSY, result);\n\n    /* At this point initialization is complete and we can signal the notification if any. */\n    if (pJob->data.resourceManager.loadDataBufferNode.pInitNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.loadDataBufferNode.pInitNotification);\n    }\n    if (pJob->data.resourceManager.loadDataBufferNode.pInitFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.loadDataBufferNode.pInitFence);\n    }\n\n    /* If we have a success result it means we've fully loaded the buffer. This will happen in the non-decoding case. */\n    if (result != MA_BUSY) {\n        if (pJob->data.resourceManager.loadDataBufferNode.pDoneNotification != NULL) {\n            ma_async_notification_signal(pJob->data.resourceManager.loadDataBufferNode.pDoneNotification);\n        }\n        if (pJob->data.resourceManager.loadDataBufferNode.pDoneFence != NULL) {\n            ma_fence_release(pJob->data.resourceManager.loadDataBufferNode.pDoneFence);\n        }\n    }\n\n    /* Increment the node's execution pointer so that the next jobs can be processed. This is how we keep decoding of pages in-order. */\n    ma_atomic_fetch_add_32(&pDataBufferNode->executionPointer, 1);\n\n    /* A busy result should be considered successful from the point of view of the job system. */\n    if (result == MA_BUSY) {\n        result  = MA_SUCCESS;\n    }\n\n    return result;\n}\n\nstatic ma_result ma_job_process__resource_manager__free_data_buffer_node(ma_job* pJob)\n{\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_buffer_node* pDataBufferNode;\n\n    MA_ASSERT(pJob != NULL);\n\n    pResourceManager = (ma_resource_manager*)pJob->data.resourceManager.freeDataBufferNode.pResourceManager;\n    MA_ASSERT(pResourceManager != NULL);\n\n    pDataBufferNode = (ma_resource_manager_data_buffer_node*)pJob->data.resourceManager.freeDataBufferNode.pDataBufferNode;\n    MA_ASSERT(pDataBufferNode != NULL);\n\n    if (pJob->order != ma_atomic_load_32(&pDataBufferNode->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    /* The event needs to be signalled last. */\n    if (pJob->data.resourceManager.freeDataBufferNode.pDoneNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.freeDataBufferNode.pDoneNotification);\n    }\n\n    if (pJob->data.resourceManager.freeDataBufferNode.pDoneFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.freeDataBufferNode.pDoneFence);\n    }\n\n    ma_atomic_fetch_add_32(&pDataBufferNode->executionPointer, 1);\n\n    ma_resource_manager_data_buffer_node_free(pResourceManager, pDataBufferNode);\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_job_process__resource_manager__page_data_buffer_node(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_buffer_node* pDataBufferNode;\n\n    MA_ASSERT(pJob != NULL);\n\n    pResourceManager = (ma_resource_manager*)pJob->data.resourceManager.pageDataBufferNode.pResourceManager;\n    MA_ASSERT(pResourceManager != NULL);\n\n    pDataBufferNode = (ma_resource_manager_data_buffer_node*)pJob->data.resourceManager.pageDataBufferNode.pDataBufferNode;\n    MA_ASSERT(pDataBufferNode != NULL);\n\n    if (pJob->order != ma_atomic_load_32(&pDataBufferNode->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    /* Don't do any more decoding if the data buffer has started the uninitialization process. */\n    result = ma_resource_manager_data_buffer_node_result(pDataBufferNode);\n    if (result != MA_BUSY) {\n        goto done;\n    }\n\n    /* We're ready to decode the next page. */\n    result = ma_resource_manager_data_buffer_node_decode_next_page(pResourceManager, pDataBufferNode, (ma_decoder*)pJob->data.resourceManager.pageDataBufferNode.pDecoder);\n\n    /*\n    If we have a success code by this point, we want to post another job. We're going to set the\n    result back to MA_BUSY to make it clear that there's still more to load.\n    */\n    if (result == MA_SUCCESS) {\n        ma_job newJob;\n        newJob = *pJob; /* Everything is the same as the input job, except the execution order. */\n        newJob.order = ma_resource_manager_data_buffer_node_next_execution_order(pDataBufferNode);   /* We need a fresh execution order. */\n\n        result = ma_resource_manager_post_job(pResourceManager, &newJob);\n\n        /* Since the sound isn't yet fully decoded we want the status to be set to busy. */\n        if (result == MA_SUCCESS) {\n            result  = MA_BUSY;\n        }\n    }\n\ndone:\n    /* If there's still more to decode the result will be set to MA_BUSY. Otherwise we can free the decoder. */\n    if (result != MA_BUSY) {\n        ma_decoder_uninit((ma_decoder*)pJob->data.resourceManager.pageDataBufferNode.pDecoder);\n        ma_free(pJob->data.resourceManager.pageDataBufferNode.pDecoder, &pResourceManager->config.allocationCallbacks);\n    }\n\n    /* If we reached the end we need to treat it as successful. */\n    if (result == MA_AT_END) {\n        result  = MA_SUCCESS;\n    }\n\n    /* Make sure we set the result of node in case some error occurred. */\n    ma_atomic_compare_and_swap_i32(&pDataBufferNode->result, MA_BUSY, result);\n\n    /* Signal the notification after setting the result in case the notification callback wants to inspect the result code. */\n    if (result != MA_BUSY) {\n        if (pJob->data.resourceManager.pageDataBufferNode.pDoneNotification != NULL) {\n            ma_async_notification_signal(pJob->data.resourceManager.pageDataBufferNode.pDoneNotification);\n        }\n\n        if (pJob->data.resourceManager.pageDataBufferNode.pDoneFence != NULL) {\n            ma_fence_release(pJob->data.resourceManager.pageDataBufferNode.pDoneFence);\n        }\n    }\n\n    ma_atomic_fetch_add_32(&pDataBufferNode->executionPointer, 1);\n    return result;\n}\n\n\nstatic ma_result ma_job_process__resource_manager__load_data_buffer(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_buffer* pDataBuffer;\n    ma_resource_manager_data_supply_type dataSupplyType = ma_resource_manager_data_supply_type_unknown;\n    ma_bool32 isConnectorInitialized = MA_FALSE;\n\n    /*\n    All we're doing here is checking if the node has finished loading. If not, we just re-post the job\n    and keep waiting. Otherwise we increment the execution counter and set the buffer's result code.\n    */\n    MA_ASSERT(pJob != NULL);\n\n    pDataBuffer = (ma_resource_manager_data_buffer*)pJob->data.resourceManager.loadDataBuffer.pDataBuffer;\n    MA_ASSERT(pDataBuffer != NULL);\n\n    pResourceManager = pDataBuffer->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataBuffer->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Attempting to execute out of order. Probably interleaved with a MA_JOB_TYPE_RESOURCE_MANAGER_FREE_DATA_BUFFER job. */\n    }\n\n    /*\n    First thing we need to do is check whether or not the data buffer is getting deleted. If so we\n    just abort, but making sure we increment the execution pointer.\n    */\n    result = ma_resource_manager_data_buffer_result(pDataBuffer);\n    if (result != MA_BUSY) {\n        goto done;  /* <-- This will ensure the execution pointer is incremented. */\n    } else {\n        result = MA_SUCCESS;    /* <-- Make sure this is reset. */\n        (void)result;           /* <-- This is to suppress a static analysis diagnostic about \"result\" not being used. But for safety when I do future maintenance I don't want to delete that assignment. */\n    }\n\n    /* Try initializing the connector if we haven't already. */\n    isConnectorInitialized = ma_resource_manager_data_buffer_has_connector(pDataBuffer);\n    if (isConnectorInitialized == MA_FALSE) {\n        dataSupplyType = ma_resource_manager_data_buffer_node_get_data_supply_type(pDataBuffer->pNode);\n\n        if (dataSupplyType != ma_resource_manager_data_supply_type_unknown) {\n            /* We can now initialize the connector. If this fails, we need to abort. It's very rare for this to fail. */\n            ma_resource_manager_data_source_config dataSourceConfig;    /* For setting initial looping state and range. */\n            dataSourceConfig = ma_resource_manager_data_source_config_init();\n            dataSourceConfig.rangeBegInPCMFrames     = pJob->data.resourceManager.loadDataBuffer.rangeBegInPCMFrames;\n            dataSourceConfig.rangeEndInPCMFrames     = pJob->data.resourceManager.loadDataBuffer.rangeEndInPCMFrames;\n            dataSourceConfig.loopPointBegInPCMFrames = pJob->data.resourceManager.loadDataBuffer.loopPointBegInPCMFrames;\n            dataSourceConfig.loopPointEndInPCMFrames = pJob->data.resourceManager.loadDataBuffer.loopPointEndInPCMFrames;\n            dataSourceConfig.isLooping               = pJob->data.resourceManager.loadDataBuffer.isLooping;\n\n            result = ma_resource_manager_data_buffer_init_connector(pDataBuffer, &dataSourceConfig, pJob->data.resourceManager.loadDataBuffer.pInitNotification, pJob->data.resourceManager.loadDataBuffer.pInitFence);\n            if (result != MA_SUCCESS) {\n                ma_log_postf(ma_resource_manager_get_log(pResourceManager), MA_LOG_LEVEL_ERROR, \"Failed to initialize connector for data buffer. %s.\\n\", ma_result_description(result));\n                goto done;\n            }\n        } else {\n            /* Don't have a known data supply type. Most likely the data buffer node is still loading, but it could be that an error occurred. */\n        }\n    } else {\n        /* The connector is already initialized. Nothing to do here. */\n    }\n\n    /*\n    If the data node is still loading, we need to repost the job and *not* increment the execution\n    pointer (i.e. we need to not fall through to the \"done\" label).\n\n    There is a hole between here and the where the data connector is initialized where the data\n    buffer node may have finished initializing. We need to check for this by checking the result of\n    the data buffer node and whether or not we had an unknown data supply type at the time of\n    trying to initialize the data connector.\n    */\n    result = ma_resource_manager_data_buffer_node_result(pDataBuffer->pNode);\n    if (result == MA_BUSY || (result == MA_SUCCESS && isConnectorInitialized == MA_FALSE && dataSupplyType == ma_resource_manager_data_supply_type_unknown)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);\n    }\n\ndone:\n    /* Only move away from a busy code so that we don't trash any existing error codes. */\n    ma_atomic_compare_and_swap_i32(&pDataBuffer->result, MA_BUSY, result);\n\n    /* Only signal the other threads after the result has been set just for cleanliness sake. */\n    if (pJob->data.resourceManager.loadDataBuffer.pDoneNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.loadDataBuffer.pDoneNotification);\n    }\n    if (pJob->data.resourceManager.loadDataBuffer.pDoneFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.loadDataBuffer.pDoneFence);\n    }\n\n    /*\n    If at this point the data buffer has not had it's connector initialized, it means the\n    notification event was never signalled which means we need to signal it here.\n    */\n    if (ma_resource_manager_data_buffer_has_connector(pDataBuffer) == MA_FALSE && result != MA_SUCCESS) {\n        if (pJob->data.resourceManager.loadDataBuffer.pInitNotification != NULL) {\n            ma_async_notification_signal(pJob->data.resourceManager.loadDataBuffer.pInitNotification);\n        }\n        if (pJob->data.resourceManager.loadDataBuffer.pInitFence != NULL) {\n            ma_fence_release(pJob->data.resourceManager.loadDataBuffer.pInitFence);\n        }\n    }\n\n    ma_atomic_fetch_add_32(&pDataBuffer->executionPointer, 1);\n    return result;\n}\n\nstatic ma_result ma_job_process__resource_manager__free_data_buffer(ma_job* pJob)\n{\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_buffer* pDataBuffer;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDataBuffer = (ma_resource_manager_data_buffer*)pJob->data.resourceManager.freeDataBuffer.pDataBuffer;\n    MA_ASSERT(pDataBuffer != NULL);\n\n    pResourceManager = pDataBuffer->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataBuffer->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    ma_resource_manager_data_buffer_uninit_internal(pDataBuffer);\n\n    /* The event needs to be signalled last. */\n    if (pJob->data.resourceManager.freeDataBuffer.pDoneNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.freeDataBuffer.pDoneNotification);\n    }\n\n    if (pJob->data.resourceManager.freeDataBuffer.pDoneFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.freeDataBuffer.pDoneFence);\n    }\n\n    ma_atomic_fetch_add_32(&pDataBuffer->executionPointer, 1);\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_job_process__resource_manager__load_data_stream(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_decoder_config decoderConfig;\n    ma_uint32 pageBufferSizeInBytes;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_stream* pDataStream;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDataStream = (ma_resource_manager_data_stream*)pJob->data.resourceManager.loadDataStream.pDataStream;\n    MA_ASSERT(pDataStream != NULL);\n\n    pResourceManager = pDataStream->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataStream->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_BUSY) {\n        result = MA_INVALID_OPERATION;  /* Most likely the data stream is being uninitialized. */\n        goto done;\n    }\n\n    /* We need to initialize the decoder first so we can determine the size of the pages. */\n    decoderConfig = ma_resource_manager__init_decoder_config(pResourceManager);\n\n    if (pJob->data.resourceManager.loadDataStream.pFilePath != NULL) {\n        result = ma_decoder_init_vfs(pResourceManager->config.pVFS, pJob->data.resourceManager.loadDataStream.pFilePath, &decoderConfig, &pDataStream->decoder);\n    } else {\n        result = ma_decoder_init_vfs_w(pResourceManager->config.pVFS, pJob->data.resourceManager.loadDataStream.pFilePathW, &decoderConfig, &pDataStream->decoder);\n    }\n    if (result != MA_SUCCESS) {\n        goto done;\n    }\n\n    /* Retrieve the total length of the file before marking the decoder as loaded. */\n    if ((pDataStream->flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_UNKNOWN_LENGTH) == 0) {\n        result = ma_decoder_get_length_in_pcm_frames(&pDataStream->decoder, &pDataStream->totalLengthInPCMFrames);\n        if (result != MA_SUCCESS) {\n            goto done;  /* Failed to retrieve the length. */\n        }\n    } else {\n        pDataStream->totalLengthInPCMFrames = 0;\n    }\n\n    /*\n    Only mark the decoder as initialized when the length of the decoder has been retrieved because that can possibly require a scan over the whole file\n    and we don't want to have another thread trying to access the decoder while it's scanning.\n    */\n    pDataStream->isDecoderInitialized = MA_TRUE;\n\n    /* We have the decoder so we can now initialize our page buffer. */\n    pageBufferSizeInBytes = ma_resource_manager_data_stream_get_page_size_in_frames(pDataStream) * 2 * ma_get_bytes_per_frame(pDataStream->decoder.outputFormat, pDataStream->decoder.outputChannels);\n\n    pDataStream->pPageData = ma_malloc(pageBufferSizeInBytes, &pResourceManager->config.allocationCallbacks);\n    if (pDataStream->pPageData == NULL) {\n        ma_decoder_uninit(&pDataStream->decoder);\n        result = MA_OUT_OF_MEMORY;\n        goto done;\n    }\n\n    /* Seek to our initial seek point before filling the initial pages. */\n    ma_decoder_seek_to_pcm_frame(&pDataStream->decoder, pJob->data.resourceManager.loadDataStream.initialSeekPoint);\n\n    /* We have our decoder and our page buffer, so now we need to fill our pages. */\n    ma_resource_manager_data_stream_fill_pages(pDataStream);\n\n    /* And now we're done. We want to make sure the result is MA_SUCCESS. */\n    result = MA_SUCCESS;\n\ndone:\n    ma_free(pJob->data.resourceManager.loadDataStream.pFilePath,  &pResourceManager->config.allocationCallbacks);\n    ma_free(pJob->data.resourceManager.loadDataStream.pFilePathW, &pResourceManager->config.allocationCallbacks);\n\n    /* We can only change the status away from MA_BUSY. If it's set to anything else it means an error has occurred somewhere or the uninitialization process has started (most likely). */\n    ma_atomic_compare_and_swap_i32(&pDataStream->result, MA_BUSY, result);\n\n    /* Only signal the other threads after the result has been set just for cleanliness sake. */\n    if (pJob->data.resourceManager.loadDataStream.pInitNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.loadDataStream.pInitNotification);\n    }\n    if (pJob->data.resourceManager.loadDataStream.pInitFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.loadDataStream.pInitFence);\n    }\n\n    ma_atomic_fetch_add_32(&pDataStream->executionPointer, 1);\n    return result;\n}\n\nstatic ma_result ma_job_process__resource_manager__free_data_stream(ma_job* pJob)\n{\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_stream* pDataStream;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDataStream = (ma_resource_manager_data_stream*)pJob->data.resourceManager.freeDataStream.pDataStream;\n    MA_ASSERT(pDataStream != NULL);\n\n    pResourceManager = pDataStream->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataStream->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    /* If our status is not MA_UNAVAILABLE we have a bug somewhere. */\n    MA_ASSERT(ma_resource_manager_data_stream_result(pDataStream) == MA_UNAVAILABLE);\n\n    if (pDataStream->isDecoderInitialized) {\n        ma_decoder_uninit(&pDataStream->decoder);\n    }\n\n    if (pDataStream->pPageData != NULL) {\n        ma_free(pDataStream->pPageData, &pResourceManager->config.allocationCallbacks);\n        pDataStream->pPageData = NULL;  /* Just in case... */\n    }\n\n    ma_data_source_uninit(&pDataStream->ds);\n\n    /* The event needs to be signalled last. */\n    if (pJob->data.resourceManager.freeDataStream.pDoneNotification != NULL) {\n        ma_async_notification_signal(pJob->data.resourceManager.freeDataStream.pDoneNotification);\n    }\n    if (pJob->data.resourceManager.freeDataStream.pDoneFence != NULL) {\n        ma_fence_release(pJob->data.resourceManager.freeDataStream.pDoneFence);\n    }\n\n    /*ma_atomic_fetch_add_32(&pDataStream->executionPointer, 1);*/\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_job_process__resource_manager__page_data_stream(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_stream* pDataStream;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDataStream = (ma_resource_manager_data_stream*)pJob->data.resourceManager.pageDataStream.pDataStream;\n    MA_ASSERT(pDataStream != NULL);\n\n    pResourceManager = pDataStream->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataStream->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    /* For streams, the status should be MA_SUCCESS. */\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS) {\n        result = MA_INVALID_OPERATION;\n        goto done;\n    }\n\n    ma_resource_manager_data_stream_fill_page(pDataStream, pJob->data.resourceManager.pageDataStream.pageIndex);\n\ndone:\n    ma_atomic_fetch_add_32(&pDataStream->executionPointer, 1);\n    return result;\n}\n\nstatic ma_result ma_job_process__resource_manager__seek_data_stream(ma_job* pJob)\n{\n    ma_result result = MA_SUCCESS;\n    ma_resource_manager* pResourceManager;\n    ma_resource_manager_data_stream* pDataStream;\n\n    MA_ASSERT(pJob != NULL);\n\n    pDataStream = (ma_resource_manager_data_stream*)pJob->data.resourceManager.seekDataStream.pDataStream;\n    MA_ASSERT(pDataStream != NULL);\n\n    pResourceManager = pDataStream->pResourceManager;\n\n    if (pJob->order != ma_atomic_load_32(&pDataStream->executionPointer)) {\n        return ma_resource_manager_post_job(pResourceManager, pJob);    /* Out of order. */\n    }\n\n    /* For streams the status should be MA_SUCCESS for this to do anything. */\n    if (ma_resource_manager_data_stream_result(pDataStream) != MA_SUCCESS || pDataStream->isDecoderInitialized == MA_FALSE) {\n        result = MA_INVALID_OPERATION;\n        goto done;\n    }\n\n    /*\n    With seeking we just assume both pages are invalid and the relative frame cursor at position 0. This is basically exactly the same as loading, except\n    instead of initializing the decoder, we seek to a frame.\n    */\n    ma_decoder_seek_to_pcm_frame(&pDataStream->decoder, pJob->data.resourceManager.seekDataStream.frameIndex);\n\n    /* After seeking we'll need to reload the pages. */\n    ma_resource_manager_data_stream_fill_pages(pDataStream);\n\n    /* We need to let the public API know that we're done seeking. */\n    ma_atomic_fetch_sub_32(&pDataStream->seekCounter, 1);\n\ndone:\n    ma_atomic_fetch_add_32(&pDataStream->executionPointer, 1);\n    return result;\n}\n\nMA_API ma_result ma_resource_manager_process_job(ma_resource_manager* pResourceManager, ma_job* pJob)\n{\n    if (pResourceManager == NULL || pJob == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_job_process(pJob);\n}\n\nMA_API ma_result ma_resource_manager_process_next_job(ma_resource_manager* pResourceManager)\n{\n    ma_result result;\n    ma_job job;\n\n    if (pResourceManager == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* This will return MA_CANCELLED if the next job is a quit job. */\n    result = ma_resource_manager_next_job(pResourceManager, &job);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return ma_job_process(&job);\n}\n#else\n/* We'll get here if the resource manager is being excluded from the build. We need to define the job processing callbacks as no-ops. */\nstatic ma_result ma_job_process__resource_manager__load_data_buffer_node(ma_job* pJob) { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__free_data_buffer_node(ma_job* pJob) { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__page_data_buffer_node(ma_job* pJob) { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__load_data_buffer(ma_job* pJob)      { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__free_data_buffer(ma_job* pJob)      { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__load_data_stream(ma_job* pJob)      { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__free_data_stream(ma_job* pJob)      { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__page_data_stream(ma_job* pJob)      { return ma_job_process__noop(pJob); }\nstatic ma_result ma_job_process__resource_manager__seek_data_stream(ma_job* pJob)      { return ma_job_process__noop(pJob); }\n#endif  /* MA_NO_RESOURCE_MANAGER */\n\n\n#ifndef MA_NO_NODE_GRAPH\n\nstatic ma_stack* ma_stack_init(size_t sizeInBytes, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_stack* pStack;\n\n    if (sizeInBytes == 0) {\n        return NULL;\n    }\n\n    pStack = (ma_stack*)ma_malloc(sizeof(*pStack) - sizeof(pStack->_data) + sizeInBytes, pAllocationCallbacks);\n    if (pStack == NULL) {\n        return NULL;\n    }\n\n    pStack->offset = 0;\n    pStack->sizeInBytes = sizeInBytes;\n\n    return pStack;\n}\n\nstatic void ma_stack_uninit(ma_stack* pStack, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pStack == NULL) {\n        return;\n    }\n\n    ma_free(pStack, pAllocationCallbacks);\n}\n\nstatic void* ma_stack_alloc(ma_stack* pStack, size_t sz)\n{\n    /* The size of the allocation is stored in the memory directly before the pointer. This needs to include padding to keep it aligned to ma_uintptr */\n    void* p = (void*)((char*)pStack->_data + pStack->offset);\n    size_t* pSize = (size_t*)p;\n\n    sz = (sz + (sizeof(ma_uintptr) - 1)) & ~(sizeof(ma_uintptr) - 1);  /* Padding. */\n    if (pStack->offset + sz + sizeof(size_t) > pStack->sizeInBytes) {\n        return NULL;    /* Out of memory. */\n    }\n\n    pStack->offset += sz + sizeof(size_t);\n\n    *pSize = sz;\n    return (void*)((char*)p + sizeof(size_t));\n}\n\nstatic void ma_stack_free(ma_stack* pStack, void* p)\n{\n    size_t* pSize;\n\n    if (p == NULL) {\n        return;\n    }\n\n    pSize = (size_t*)p - 1;\n    pStack->offset -= *pSize + sizeof(size_t);\n}\n\n\n\n/* 10ms @ 48K = 480. Must never exceed 65535. */\n#ifndef MA_DEFAULT_NODE_CACHE_CAP_IN_FRAMES_PER_BUS\n#define MA_DEFAULT_NODE_CACHE_CAP_IN_FRAMES_PER_BUS 480\n#endif\n\n#ifndef MA_DEFAULT_PREMIX_STACK_SIZE_PER_CHANNEL\n#define MA_DEFAULT_PREMIX_STACK_SIZE_PER_CHANNEL    524288\n#endif\n\nstatic ma_result ma_node_read_pcm_frames(ma_node* pNode, ma_uint32 outputBusIndex, float* pFramesOut, ma_uint32 frameCount, ma_uint32* pFramesRead, ma_uint64 globalTime);\n\nMA_API void ma_debug_fill_pcm_frames_with_sine_wave(float* pFramesOut, ma_uint32 frameCount, ma_format format, ma_uint32 channels, ma_uint32 sampleRate)\n{\n    #ifndef MA_NO_GENERATION\n    {\n        ma_waveform_config waveformConfig;\n        ma_waveform waveform;\n\n        waveformConfig = ma_waveform_config_init(format, channels, sampleRate, ma_waveform_type_sine, 1.0, 400);\n        ma_waveform_init(&waveformConfig, &waveform);\n        ma_waveform_read_pcm_frames(&waveform, pFramesOut, frameCount, NULL);\n    }\n    #else\n    {\n        (void)pFramesOut;\n        (void)frameCount;\n        (void)format;\n        (void)channels;\n        (void)sampleRate;\n        #if defined(MA_DEBUG_OUTPUT)\n        {\n            #if _MSC_VER\n                #pragma message (\"ma_debug_fill_pcm_frames_with_sine_wave() will do nothing because MA_NO_GENERATION is enabled.\")\n            #endif\n        }\n        #endif\n    }\n    #endif\n}\n\n\n\nMA_API ma_node_graph_config ma_node_graph_config_init(ma_uint32 channels)\n{\n    ma_node_graph_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.channels               = channels;\n    config.processingSizeInFrames = 0;\n\n    return config;\n}\n\n\nstatic void ma_node_graph_set_is_reading(ma_node_graph* pNodeGraph, ma_bool32 isReading)\n{\n    MA_ASSERT(pNodeGraph != NULL);\n    ma_atomic_exchange_32(&pNodeGraph->isReading, isReading);\n}\n\n#if 0\nstatic ma_bool32 ma_node_graph_is_reading(ma_node_graph* pNodeGraph)\n{\n    MA_ASSERT(pNodeGraph != NULL);\n    return ma_atomic_load_32(&pNodeGraph->isReading);\n}\n#endif\n\n\nstatic void ma_node_graph_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_node_graph* pNodeGraph = (ma_node_graph*)pNode;\n    ma_uint64 framesRead;\n\n    ma_node_graph_read_pcm_frames(pNodeGraph, ppFramesOut[0], *pFrameCountOut, &framesRead);\n\n    *pFrameCountOut = (ma_uint32)framesRead;    /* Safe cast. */\n\n    (void)ppFramesIn;\n    (void)pFrameCountIn;\n}\n\nstatic ma_node_vtable g_node_graph_node_vtable =\n{\n    ma_node_graph_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    0,      /* 0 input buses. */\n    1,      /* 1 output bus. */\n    0       /* Flags. */\n};\n\nstatic void ma_node_graph_endpoint_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    MA_ASSERT(pNode != NULL);\n    MA_ASSERT(ma_node_get_input_bus_count(pNode)  == 1);\n    MA_ASSERT(ma_node_get_output_bus_count(pNode) == 1);\n\n    /* Input channel count needs to be the same as the output channel count. */\n    MA_ASSERT(ma_node_get_input_channels(pNode, 0) == ma_node_get_output_channels(pNode, 0));\n\n    /* We don't need to do anything here because it's a passthrough. */\n    (void)pNode;\n    (void)ppFramesIn;\n    (void)pFrameCountIn;\n    (void)ppFramesOut;\n    (void)pFrameCountOut;\n\n#if 0\n    /* The data has already been mixed. We just need to move it to the output buffer. */\n    if (ppFramesIn != NULL) {\n        ma_copy_pcm_frames(ppFramesOut[0], ppFramesIn[0], *pFrameCountOut, ma_format_f32, ma_node_get_output_channels(pNode, 0));\n    }\n#endif\n}\n\nstatic ma_node_vtable g_node_graph_endpoint_vtable =\n{\n    ma_node_graph_endpoint_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* 1 input bus. */\n    1,      /* 1 output bus. */\n    MA_NODE_FLAG_PASSTHROUGH    /* Flags. The endpoint is a passthrough. */\n};\n\nMA_API ma_result ma_node_graph_init(const ma_node_graph_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_node_graph* pNodeGraph)\n{\n    ma_result result;\n    ma_node_config baseConfig;\n    ma_node_config endpointConfig;\n\n    if (pNodeGraph == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNodeGraph);\n    pNodeGraph->processingSizeInFrames = pConfig->processingSizeInFrames;\n\n    /* Base node so we can use the node graph as a node into another graph. */\n    baseConfig = ma_node_config_init();\n    baseConfig.vtable = &g_node_graph_node_vtable;\n    baseConfig.pOutputChannels = &pConfig->channels;\n\n    result = ma_node_init(pNodeGraph, &baseConfig, pAllocationCallbacks, &pNodeGraph->base);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n\n    /* Endpoint. */\n    endpointConfig = ma_node_config_init();\n    endpointConfig.vtable          = &g_node_graph_endpoint_vtable;\n    endpointConfig.pInputChannels  = &pConfig->channels;\n    endpointConfig.pOutputChannels = &pConfig->channels;\n\n    result = ma_node_init(pNodeGraph, &endpointConfig, pAllocationCallbacks, &pNodeGraph->endpoint);\n    if (result != MA_SUCCESS) {\n        ma_node_uninit(&pNodeGraph->base, pAllocationCallbacks);\n        return result;\n    }\n\n\n    /* Processing cache. */\n    if (pConfig->processingSizeInFrames > 0) {\n        pNodeGraph->pProcessingCache = (float*)ma_malloc(pConfig->processingSizeInFrames * pConfig->channels * sizeof(float), pAllocationCallbacks);\n        if (pNodeGraph->pProcessingCache == NULL) {\n            ma_node_uninit(&pNodeGraph->endpoint, pAllocationCallbacks);\n            ma_node_uninit(&pNodeGraph->base, pAllocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n\n    /*\n    We need a pre-mix stack. The size of this stack is configurable via the config. The default value depends on the channel count.\n    */\n    {\n        size_t preMixStackSizeInBytes = pConfig->preMixStackSizeInBytes;\n        if (preMixStackSizeInBytes == 0) {\n            preMixStackSizeInBytes = pConfig->channels * MA_DEFAULT_PREMIX_STACK_SIZE_PER_CHANNEL;\n        }\n\n        pNodeGraph->pPreMixStack = ma_stack_init(preMixStackSizeInBytes, pAllocationCallbacks);\n        if (pNodeGraph->pPreMixStack == NULL) {\n            ma_node_uninit(&pNodeGraph->endpoint, pAllocationCallbacks);\n            ma_node_uninit(&pNodeGraph->base, pAllocationCallbacks);\n            if (pNodeGraph->pProcessingCache != NULL) {\n                ma_free(pNodeGraph->pProcessingCache, pAllocationCallbacks);\n            }\n\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_node_graph_uninit(ma_node_graph* pNodeGraph, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pNodeGraph == NULL) {\n        return;\n    }\n\n    ma_node_uninit(&pNodeGraph->endpoint, pAllocationCallbacks);\n    ma_node_uninit(&pNodeGraph->base, pAllocationCallbacks);\n\n    if (pNodeGraph->pProcessingCache != NULL) {\n        ma_free(pNodeGraph->pProcessingCache, pAllocationCallbacks);\n        pNodeGraph->pProcessingCache = NULL;\n    }\n\n    if (pNodeGraph->pPreMixStack != NULL) {\n        ma_stack_uninit(pNodeGraph->pPreMixStack, pAllocationCallbacks);\n        pNodeGraph->pPreMixStack = NULL;\n    }\n}\n\nMA_API ma_node* ma_node_graph_get_endpoint(ma_node_graph* pNodeGraph)\n{\n    if (pNodeGraph == NULL) {\n        return NULL;\n    }\n\n    return &pNodeGraph->endpoint;\n}\n\nMA_API ma_result ma_node_graph_read_pcm_frames(ma_node_graph* pNodeGraph, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint64 totalFramesRead;\n    ma_uint32 channels;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;   /* Safety. */\n    }\n\n    if (pNodeGraph == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    channels = ma_node_get_output_channels(&pNodeGraph->endpoint, 0);\n\n\n    /* We'll be nice and try to do a full read of all frameCount frames. */\n    totalFramesRead = 0;\n    while (totalFramesRead < frameCount) {\n        ma_uint32 framesJustRead;\n        ma_uint64 framesToRead;\n        float* pRunningFramesOut;\n\n        framesToRead = frameCount - totalFramesRead;\n        if (framesToRead > 0xFFFFFFFF) {\n            framesToRead = 0xFFFFFFFF;\n        }\n\n        pRunningFramesOut = (float*)ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, ma_format_f32, channels);\n\n        /* If there's anything in the cache, consume that first. */\n        if (pNodeGraph->processingCacheFramesRemaining > 0) {\n            ma_uint32 framesToReadFromCache;\n\n            framesToReadFromCache = (ma_uint32)framesToRead;\n            if (framesToReadFromCache > pNodeGraph->processingCacheFramesRemaining) {\n                framesToReadFromCache = pNodeGraph->processingCacheFramesRemaining;\n            }\n\n            MA_COPY_MEMORY(pRunningFramesOut, pNodeGraph->pProcessingCache, framesToReadFromCache * channels * sizeof(float));\n            MA_MOVE_MEMORY(pNodeGraph->pProcessingCache, pNodeGraph->pProcessingCache + (framesToReadFromCache * channels), (pNodeGraph->processingCacheFramesRemaining - framesToReadFromCache) * channels * sizeof(float));\n            pNodeGraph->processingCacheFramesRemaining -= framesToReadFromCache;\n\n            totalFramesRead += framesToReadFromCache;\n            continue;\n        } else {\n            /*\n            If processingSizeInFrames is non-zero, we need to make sure we always read in chunks of that size. If the frame count is less than\n            that, we need to read into the cache and then continue on.\n            */\n            float* pReadDst = pRunningFramesOut;\n\n            if (pNodeGraph->processingSizeInFrames > 0) {\n                if (framesToRead < pNodeGraph->processingSizeInFrames) {\n                    pReadDst = pNodeGraph->pProcessingCache;    /* We need to read into the cache because otherwise we'll overflow the output buffer. */\n                }\n\n                framesToRead = pNodeGraph->processingSizeInFrames;\n            }\n\n            ma_node_graph_set_is_reading(pNodeGraph, MA_TRUE);\n            {\n                result = ma_node_read_pcm_frames(&pNodeGraph->endpoint, 0, pReadDst, (ma_uint32)framesToRead, &framesJustRead, ma_node_get_time(&pNodeGraph->endpoint));\n            }\n            ma_node_graph_set_is_reading(pNodeGraph, MA_FALSE);\n\n            /*\n            Do not increment the total frames read counter if we read into the cache. We use this to determine how many frames have\n            been written to the final output buffer.\n            */\n            if (pReadDst == pNodeGraph->pProcessingCache) {\n                /* We read into the cache. */\n                pNodeGraph->processingCacheFramesRemaining = framesJustRead;\n            } else {\n                /* We read straight into the output buffer. */\n                totalFramesRead += framesJustRead;\n            }\n\n            if (result != MA_SUCCESS) {\n                break;\n            }\n\n            /* Abort if we weren't able to read any frames or else we risk getting stuck in a loop. */\n            if (framesJustRead == 0) {\n                break;\n            }\n        }\n    }\n\n    /* Let's go ahead and silence any leftover frames just for some added safety to ensure the caller doesn't try emitting garbage out of the speakers. */\n    if (totalFramesRead < frameCount) {\n        ma_silence_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, totalFramesRead, ma_format_f32, channels), (frameCount - totalFramesRead), ma_format_f32, channels);\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = totalFramesRead;\n    }\n\n    return result;\n}\n\nMA_API ma_uint32 ma_node_graph_get_channels(const ma_node_graph* pNodeGraph)\n{\n    if (pNodeGraph == NULL) {\n        return 0;\n    }\n\n    return ma_node_get_output_channels(&pNodeGraph->endpoint, 0);\n}\n\nMA_API ma_uint64 ma_node_graph_get_time(const ma_node_graph* pNodeGraph)\n{\n    if (pNodeGraph == NULL) {\n        return 0;\n    }\n\n    return ma_node_get_time(&pNodeGraph->endpoint); /* Global time is just the local time of the endpoint. */\n}\n\nMA_API ma_result ma_node_graph_set_time(ma_node_graph* pNodeGraph, ma_uint64 globalTime)\n{\n    if (pNodeGraph == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_node_set_time(&pNodeGraph->endpoint, globalTime); /* Global time is just the local time of the endpoint. */\n}\n\nMA_API ma_uint32 ma_node_graph_get_processing_size_in_frames(const ma_node_graph* pNodeGraph)\n{\n    if (pNodeGraph == NULL) {\n        return 0;\n    }\n\n    return pNodeGraph->processingSizeInFrames;\n}\n\n\n#define MA_NODE_OUTPUT_BUS_FLAG_HAS_READ    0x01    /* Whether or not this bus ready to read more data. Only used on nodes with multiple output buses. */\n\nstatic ma_result ma_node_output_bus_init(ma_node* pNode, ma_uint32 outputBusIndex, ma_uint32 channels, ma_node_output_bus* pOutputBus)\n{\n    MA_ASSERT(pOutputBus != NULL);\n    MA_ASSERT(outputBusIndex < MA_MAX_NODE_BUS_COUNT);\n    MA_ASSERT(outputBusIndex < ma_node_get_output_bus_count(pNode));\n    MA_ASSERT(channels < 256);\n\n    MA_ZERO_OBJECT(pOutputBus);\n\n    if (channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pOutputBus->pNode          = pNode;\n    pOutputBus->outputBusIndex = (ma_uint8)outputBusIndex;\n    pOutputBus->channels       = (ma_uint8)channels;\n    pOutputBus->flags          = MA_NODE_OUTPUT_BUS_FLAG_HAS_READ; /* <-- Important that this flag is set by default. */\n    pOutputBus->volume         = 1;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_node_output_bus_lock(ma_node_output_bus* pOutputBus)\n{\n    ma_spinlock_lock(&pOutputBus->lock);\n}\n\nstatic void ma_node_output_bus_unlock(ma_node_output_bus* pOutputBus)\n{\n    ma_spinlock_unlock(&pOutputBus->lock);\n}\n\n\nstatic ma_uint32 ma_node_output_bus_get_channels(const ma_node_output_bus* pOutputBus)\n{\n    return pOutputBus->channels;\n}\n\n\nstatic void ma_node_output_bus_set_has_read(ma_node_output_bus* pOutputBus, ma_bool32 hasRead)\n{\n    if (hasRead) {\n        ma_atomic_fetch_or_32(&pOutputBus->flags, MA_NODE_OUTPUT_BUS_FLAG_HAS_READ);\n    } else {\n        ma_atomic_fetch_and_32(&pOutputBus->flags, (ma_uint32)~MA_NODE_OUTPUT_BUS_FLAG_HAS_READ);\n    }\n}\n\nstatic ma_bool32 ma_node_output_bus_has_read(ma_node_output_bus* pOutputBus)\n{\n    return (ma_atomic_load_32(&pOutputBus->flags) & MA_NODE_OUTPUT_BUS_FLAG_HAS_READ) != 0;\n}\n\n\nstatic void ma_node_output_bus_set_is_attached(ma_node_output_bus* pOutputBus, ma_bool32 isAttached)\n{\n    ma_atomic_exchange_32(&pOutputBus->isAttached, isAttached);\n}\n\nstatic ma_bool32 ma_node_output_bus_is_attached(ma_node_output_bus* pOutputBus)\n{\n    return ma_atomic_load_32(&pOutputBus->isAttached);\n}\n\n\nstatic ma_result ma_node_output_bus_set_volume(ma_node_output_bus* pOutputBus, float volume)\n{\n    MA_ASSERT(pOutputBus != NULL);\n\n    if (volume < 0.0f) {\n        volume = 0.0f;\n    }\n\n    ma_atomic_exchange_f32(&pOutputBus->volume, volume);\n\n    return MA_SUCCESS;\n}\n\nstatic float ma_node_output_bus_get_volume(const ma_node_output_bus* pOutputBus)\n{\n    return ma_atomic_load_f32((float*)&pOutputBus->volume);\n}\n\n\nstatic ma_result ma_node_input_bus_init(ma_uint32 channels, ma_node_input_bus* pInputBus)\n{\n    MA_ASSERT(pInputBus != NULL);\n    MA_ASSERT(channels < 256);\n\n    MA_ZERO_OBJECT(pInputBus);\n\n    if (channels == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pInputBus->channels = (ma_uint8)channels;\n\n    return MA_SUCCESS;\n}\n\nstatic void ma_node_input_bus_lock(ma_node_input_bus* pInputBus)\n{\n    MA_ASSERT(pInputBus != NULL);\n\n    ma_spinlock_lock(&pInputBus->lock);\n}\n\nstatic void ma_node_input_bus_unlock(ma_node_input_bus* pInputBus)\n{\n    MA_ASSERT(pInputBus != NULL);\n\n    ma_spinlock_unlock(&pInputBus->lock);\n}\n\n\nstatic void ma_node_input_bus_next_begin(ma_node_input_bus* pInputBus)\n{\n    ma_atomic_fetch_add_32(&pInputBus->nextCounter, 1);\n}\n\nstatic void ma_node_input_bus_next_end(ma_node_input_bus* pInputBus)\n{\n    ma_atomic_fetch_sub_32(&pInputBus->nextCounter, 1);\n}\n\nstatic ma_uint32 ma_node_input_bus_get_next_counter(ma_node_input_bus* pInputBus)\n{\n    return ma_atomic_load_32(&pInputBus->nextCounter);\n}\n\n\nstatic ma_uint32 ma_node_input_bus_get_channels(const ma_node_input_bus* pInputBus)\n{\n    return pInputBus->channels;\n}\n\n\nstatic void ma_node_input_bus_detach__no_output_bus_lock(ma_node_input_bus* pInputBus, ma_node_output_bus* pOutputBus)\n{\n    MA_ASSERT(pInputBus  != NULL);\n    MA_ASSERT(pOutputBus != NULL);\n\n    /*\n    Mark the output bus as detached first. This will prevent future iterations on the audio thread\n    from iterating this output bus.\n    */\n    ma_node_output_bus_set_is_attached(pOutputBus, MA_FALSE);\n\n    /*\n    We cannot use the output bus lock here since it'll be getting used at a higher level, but we do\n    still need to use the input bus lock since we'll be updating pointers on two different output\n    buses. The same rules apply here as the attaching case. Although we're using a lock here, we're\n    *not* using a lock when iterating over the list in the audio thread. We therefore need to craft\n    this in a way such that the iteration on the audio thread doesn't break.\n\n    The first thing to do is swap out the \"next\" pointer of the previous output bus with the\n    new \"next\" output bus. This is the operation that matters for iteration on the audio thread.\n    After that, the previous pointer on the new \"next\" pointer needs to be updated, after which\n    point the linked list will be in a good state.\n    */\n    ma_node_input_bus_lock(pInputBus);\n    {\n        ma_node_output_bus* pOldPrev = (ma_node_output_bus*)ma_atomic_load_ptr(&pOutputBus->pPrev);\n        ma_node_output_bus* pOldNext = (ma_node_output_bus*)ma_atomic_load_ptr(&pOutputBus->pNext);\n\n        if (pOldPrev != NULL) {\n            ma_atomic_exchange_ptr(&pOldPrev->pNext, pOldNext); /* <-- This is where the output bus is detached from the list. */\n        }\n        if (pOldNext != NULL) {\n            ma_atomic_exchange_ptr(&pOldNext->pPrev, pOldPrev); /* <-- This is required for detachment. */\n        }\n    }\n    ma_node_input_bus_unlock(pInputBus);\n\n    /* At this point the output bus is detached and the linked list is completely unaware of it. Reset some data for safety. */\n    ma_atomic_exchange_ptr(&pOutputBus->pNext, NULL);   /* Using atomic exchanges here, mainly for the benefit of analysis tools which don't always recognize spinlocks. */\n    ma_atomic_exchange_ptr(&pOutputBus->pPrev, NULL);   /* As above. */\n    pOutputBus->pInputNode             = NULL;\n    pOutputBus->inputNodeInputBusIndex = 0;\n\n\n    /*\n    For thread-safety reasons, we don't want to be returning from this straight away. We need to\n    wait for the audio thread to finish with the output bus. There's two things we need to wait\n    for. The first is the part that selects the next output bus in the list, and the other is the\n    part that reads from the output bus. Basically all we're doing is waiting for the input bus\n    to stop referencing the output bus.\n\n    We're doing this part last because we want the section above to run while the audio thread\n    is finishing up with the output bus, just for efficiency reasons. We marked the output bus as\n    detached right at the top of this function which is going to prevent the audio thread from\n    iterating the output bus again.\n    */\n\n    /* Part 1: Wait for the current iteration to complete. */\n    while (ma_node_input_bus_get_next_counter(pInputBus) > 0) {\n        ma_yield();\n    }\n\n    /* Part 2: Wait for any reads to complete. */\n    while (ma_atomic_load_32(&pOutputBus->refCount) > 0) {\n        ma_yield();\n    }\n\n    /*\n    At this point we're done detaching and we can be guaranteed that the audio thread is not going\n    to attempt to reference this output bus again (until attached again).\n    */\n}\n\n#if 0   /* Not used at the moment, but leaving here in case I need it later. */\nstatic void ma_node_input_bus_detach(ma_node_input_bus* pInputBus, ma_node_output_bus* pOutputBus)\n{\n    MA_ASSERT(pInputBus  != NULL);\n    MA_ASSERT(pOutputBus != NULL);\n\n    ma_node_output_bus_lock(pOutputBus);\n    {\n        ma_node_input_bus_detach__no_output_bus_lock(pInputBus, pOutputBus);\n    }\n    ma_node_output_bus_unlock(pOutputBus);\n}\n#endif\n\nstatic void ma_node_input_bus_attach(ma_node_input_bus* pInputBus, ma_node_output_bus* pOutputBus, ma_node* pNewInputNode, ma_uint32 inputNodeInputBusIndex)\n{\n    MA_ASSERT(pInputBus  != NULL);\n    MA_ASSERT(pOutputBus != NULL);\n\n    ma_node_output_bus_lock(pOutputBus);\n    {\n        ma_node_output_bus* pOldInputNode = (ma_node_output_bus*)ma_atomic_load_ptr(&pOutputBus->pInputNode);\n\n        /* Detach from any existing attachment first if necessary. */\n        if (pOldInputNode != NULL) {\n            ma_node_input_bus_detach__no_output_bus_lock(pInputBus, pOutputBus);\n        }\n\n        /*\n        At this point we can be sure the output bus is not attached to anything. The linked list in the\n        old input bus has been updated so that pOutputBus will not get iterated again.\n        */\n        pOutputBus->pInputNode             = pNewInputNode;                     /* No need for an atomic assignment here because modification of this variable always happens within a lock. */\n        pOutputBus->inputNodeInputBusIndex = (ma_uint8)inputNodeInputBusIndex;\n\n        /*\n        Now we need to attach the output bus to the linked list. This involves updating two pointers on\n        two different output buses so I'm going to go ahead and keep this simple and just use a lock.\n        There are ways to do this without a lock, but it's just too hard to maintain for its value.\n\n        Although we're locking here, it's important to remember that we're *not* locking when iterating\n        and reading audio data since that'll be running on the audio thread. As a result we need to be\n        careful how we craft this so that we don't break iteration. What we're going to do is always\n        attach the new item so that it becomes the first item in the list. That way, as we're iterating\n        we won't break any links in the list and iteration will continue safely. The detaching case will\n        also be crafted in a way as to not break list iteration. It's important to remember to use\n        atomic exchanges here since no locking is happening on the audio thread during iteration.\n        */\n        ma_node_input_bus_lock(pInputBus);\n        {\n            ma_node_output_bus* pNewPrev = &pInputBus->head;\n            ma_node_output_bus* pNewNext = (ma_node_output_bus*)ma_atomic_load_ptr(&pInputBus->head.pNext);\n\n            /* Update the local output bus. */\n            ma_atomic_exchange_ptr(&pOutputBus->pPrev, pNewPrev);\n            ma_atomic_exchange_ptr(&pOutputBus->pNext, pNewNext);\n\n            /* Update the other output buses to point back to the local output bus. */\n            ma_atomic_exchange_ptr(&pInputBus->head.pNext, pOutputBus); /* <-- This is where the output bus is actually attached to the input bus. */\n\n            /* Do the previous pointer last. This is only used for detachment. */\n            if (pNewNext != NULL) {\n                ma_atomic_exchange_ptr(&pNewNext->pPrev,  pOutputBus);\n            }\n        }\n        ma_node_input_bus_unlock(pInputBus);\n\n        /*\n        Mark the node as attached last. This is used to controlling whether or the output bus will be\n        iterated on the audio thread. Mainly required for detachment purposes.\n        */\n        ma_node_output_bus_set_is_attached(pOutputBus, MA_TRUE);\n    }\n    ma_node_output_bus_unlock(pOutputBus);\n}\n\nstatic ma_node_output_bus* ma_node_input_bus_next(ma_node_input_bus* pInputBus, ma_node_output_bus* pOutputBus)\n{\n    ma_node_output_bus* pNext;\n\n    MA_ASSERT(pInputBus != NULL);\n\n    if (pOutputBus == NULL) {\n        return NULL;\n    }\n\n    ma_node_input_bus_next_begin(pInputBus);\n    {\n        pNext = pOutputBus;\n        for (;;) {\n            pNext = (ma_node_output_bus*)ma_atomic_load_ptr(&pNext->pNext);\n            if (pNext == NULL) {\n                break;      /* Reached the end. */\n            }\n\n            if (ma_node_output_bus_is_attached(pNext) == MA_FALSE) {\n                continue;   /* The node is not attached. Keep checking. */\n            }\n\n            /* The next node has been selected. */\n            break;\n        }\n\n        /* We need to increment the reference count of the selected node. */\n        if (pNext != NULL) {\n            ma_atomic_fetch_add_32(&pNext->refCount, 1);\n        }\n\n        /* The previous node is no longer being referenced. */\n        ma_atomic_fetch_sub_32(&pOutputBus->refCount, 1);\n    }\n    ma_node_input_bus_next_end(pInputBus);\n\n    return pNext;\n}\n\nstatic ma_node_output_bus* ma_node_input_bus_first(ma_node_input_bus* pInputBus)\n{\n    return ma_node_input_bus_next(pInputBus, &pInputBus->head);\n}\n\n\n\nstatic ma_result ma_node_input_bus_read_pcm_frames(ma_node* pInputNode, ma_node_input_bus* pInputBus, float* pFramesOut, ma_uint32 frameCount, ma_uint32* pFramesRead, ma_uint64 globalTime)\n{\n    ma_result result = MA_SUCCESS;\n    ma_node_output_bus* pOutputBus;\n    ma_node_output_bus* pFirst;\n    ma_uint32 inputChannels;\n    ma_bool32 doesOutputBufferHaveContent = MA_FALSE;\n\n    /*\n    This will be called from the audio thread which means we can't be doing any locking. Basically,\n    this function will not perform any locking, whereas attaching and detaching will, but crafted in\n    such a way that we don't need to perform any locking here. The important thing to remember is\n    to always iterate in a forward direction.\n\n    In order to process any data we need to first read from all input buses. That's where this\n    function comes in. This iterates over each of the attachments and accumulates/mixes them. We\n    also convert the channels to the nodes output channel count before mixing. We want to do this\n    channel conversion so that the caller of this function can invoke the processing callback\n    without having to do it themselves.\n\n    When we iterate over each of the attachments on the input bus, we need to read as much data as\n    we can from each of them so that we don't end up with holes between each of the attachments. To\n    do this, we need to read from each attachment in a loop and read as many frames as we can, up\n    to `frameCount`.\n    */\n    MA_ASSERT(pInputNode  != NULL);\n    MA_ASSERT(pFramesRead != NULL); /* pFramesRead is critical and must always be specified. On input it's undefined and on output it'll be set to the number of frames actually read. */\n\n    *pFramesRead = 0;   /* Safety. */\n\n    inputChannels = ma_node_input_bus_get_channels(pInputBus);\n\n    /*\n    We need to be careful with how we call ma_node_input_bus_first() and ma_node_input_bus_next(). They\n    are both critical to our lock-free thread-safety system. We can only call ma_node_input_bus_first()\n    once per iteration, however we have an optimization to checks whether or not it's the first item in\n    the list. We therefore need to store a pointer to the first item rather than repeatedly calling\n    ma_node_input_bus_first(). It's safe to keep hold of this pointer, so long as we don't dereference it\n    after calling ma_node_input_bus_next(), which we won't be.\n    */\n    pFirst = ma_node_input_bus_first(pInputBus);\n    if (pFirst == NULL) {\n        return MA_SUCCESS;  /* No attachments. Read nothing. */\n    }\n\n    for (pOutputBus = pFirst; pOutputBus != NULL; pOutputBus = ma_node_input_bus_next(pInputBus, pOutputBus)) {\n        ma_uint32 framesProcessed = 0;\n        ma_bool32 isSilentOutput = MA_FALSE;\n\n        MA_ASSERT(pOutputBus->pNode != NULL);\n        MA_ASSERT(((ma_node_base*)pOutputBus->pNode)->vtable != NULL);\n\n        isSilentOutput = (((ma_node_base*)pOutputBus->pNode)->vtable->flags & MA_NODE_FLAG_SILENT_OUTPUT) != 0;\n\n        if (pFramesOut != NULL) {\n            /* Read. */\n            while (framesProcessed < frameCount) {\n                float* pRunningFramesOut;\n                ma_uint32 framesToRead;\n                ma_uint32 framesJustRead = 0;\n\n                framesToRead = frameCount - framesProcessed;\n                pRunningFramesOut = ma_offset_pcm_frames_ptr_f32(pFramesOut, framesProcessed, inputChannels);\n\n                if (doesOutputBufferHaveContent == MA_FALSE) {\n                    /* Fast path. First attachment. We just read straight into the output buffer (no mixing required). */\n                    result = ma_node_read_pcm_frames(pOutputBus->pNode, pOutputBus->outputBusIndex, pRunningFramesOut, framesToRead, &framesJustRead, globalTime + framesProcessed);\n                } else {\n                    /* Slow path. Not the first attachment. Mixing required. */\n                    ma_uint32 preMixBufferCapInFrames = ((ma_node_base*)pInputNode)->cachedDataCapInFramesPerBus;\n                    float* pPreMixBuffer = (float*)ma_stack_alloc(((ma_node_base*)pInputNode)->pNodeGraph->pPreMixStack, preMixBufferCapInFrames * inputChannels * sizeof(float));\n\n                    if (pPreMixBuffer == NULL) {\n                        /*\n                        If you're hitting this assert it means you've got an unusually deep chain of nodes, you've got an excessively large processing\n                        size, or you have a combination of both, and as a result have run out of stack space. You can increase this using the\n                        preMixStackSizeInBytes variable in ma_node_graph_config. If you're using ma_engine, you can do it via the preMixStackSizeInBytes\n                        variable in ma_engine_config. It defaults to 512KB per output channel.\n                        */\n                        MA_ASSERT(MA_FALSE);\n                    } else {\n                        if (framesToRead > preMixBufferCapInFrames) {\n                            framesToRead = preMixBufferCapInFrames;\n                        }\n\n                        result = ma_node_read_pcm_frames(pOutputBus->pNode, pOutputBus->outputBusIndex, pPreMixBuffer, framesToRead, &framesJustRead, globalTime + framesProcessed);\n                        if (result == MA_SUCCESS || result == MA_AT_END) {\n                            if (isSilentOutput == MA_FALSE) {   /* Don't mix if the node outputs silence. */\n                                ma_mix_pcm_frames_f32(pRunningFramesOut, pPreMixBuffer, framesJustRead, inputChannels, /*volume*/1);\n                            }\n                        }\n\n                        /* The pre-mix buffer is no longer required. */\n                        ma_stack_free(((ma_node_base*)pInputNode)->pNodeGraph->pPreMixStack, pPreMixBuffer);\n                        pPreMixBuffer = NULL;\n                    }\n                }\n\n                framesProcessed += framesJustRead;\n\n                /* If we reached the end or otherwise failed to read any data we need to finish up with this output node. */\n                if (result != MA_SUCCESS) {\n                    break;\n                }\n\n                /* If we didn't read anything, abort so we don't get stuck in a loop. */\n                if (framesJustRead == 0) {\n                    break;\n                }\n            }\n\n            /* If it's the first attachment we didn't do any mixing. Any leftover samples need to be silenced. */\n            if (pOutputBus == pFirst && framesProcessed < frameCount) {\n                ma_silence_pcm_frames(ma_offset_pcm_frames_ptr(pFramesOut, framesProcessed, ma_format_f32, inputChannels), (frameCount - framesProcessed), ma_format_f32, inputChannels);\n            }\n\n            if (isSilentOutput == MA_FALSE) {\n                doesOutputBufferHaveContent = MA_TRUE;\n            }\n        } else {\n            /* Seek. */\n            ma_node_read_pcm_frames(pOutputBus->pNode, pOutputBus->outputBusIndex, NULL, frameCount, &framesProcessed, globalTime);\n        }\n    }\n\n    /* If we didn't output anything, output silence. */\n    if (doesOutputBufferHaveContent == MA_FALSE && pFramesOut != NULL) {\n        ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, inputChannels);\n    }\n\n    /* In this path we always \"process\" the entire amount. */\n    *pFramesRead = frameCount;\n\n    return result;\n}\n\n\nMA_API ma_node_config ma_node_config_init(void)\n{\n    ma_node_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.initialState   = ma_node_state_started;    /* Nodes are started by default. */\n    config.inputBusCount  = MA_NODE_BUS_COUNT_UNKNOWN;\n    config.outputBusCount = MA_NODE_BUS_COUNT_UNKNOWN;\n\n    return config;\n}\n\nstatic ma_uint16 ma_node_config_get_cache_size_in_frames(const ma_node_config* pConfig, const ma_node_graph* pNodeGraph)\n{\n    ma_uint32 cacheSizeInFrames;\n\n    (void)pConfig;\n\n    if (pNodeGraph->processingSizeInFrames > 0) {\n        cacheSizeInFrames = pNodeGraph->processingSizeInFrames;\n    } else {\n        cacheSizeInFrames = MA_DEFAULT_NODE_CACHE_CAP_IN_FRAMES_PER_BUS;\n    }\n\n    if (cacheSizeInFrames > 0xFFFF) {\n        cacheSizeInFrames = 0xFFFF;\n    }\n\n    return (ma_uint16)cacheSizeInFrames;\n}\n\n\n\nstatic ma_result ma_node_detach_full(ma_node* pNode);\n\nstatic float* ma_node_get_cached_input_ptr(ma_node* pNode, ma_uint32 inputBusIndex)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_uint32 iInputBus;\n    float* pBasePtr;\n\n    MA_ASSERT(pNodeBase != NULL);\n\n    /* Input data is stored at the front of the buffer. */\n    pBasePtr = pNodeBase->pCachedData;\n    for (iInputBus = 0; iInputBus < inputBusIndex; iInputBus += 1) {\n        pBasePtr += pNodeBase->cachedDataCapInFramesPerBus * ma_node_input_bus_get_channels(&pNodeBase->pInputBuses[iInputBus]);\n    }\n\n    return pBasePtr;\n}\n\nstatic float* ma_node_get_cached_output_ptr(ma_node* pNode, ma_uint32 outputBusIndex)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_uint32 iInputBus;\n    ma_uint32 iOutputBus;\n    float* pBasePtr;\n\n    MA_ASSERT(pNodeBase != NULL);\n\n    /* Cached output data starts after the input data. */\n    pBasePtr = pNodeBase->pCachedData;\n    for (iInputBus = 0; iInputBus < ma_node_get_input_bus_count(pNodeBase); iInputBus += 1) {\n        pBasePtr += pNodeBase->cachedDataCapInFramesPerBus * ma_node_input_bus_get_channels(&pNodeBase->pInputBuses[iInputBus]);\n    }\n\n    for (iOutputBus = 0; iOutputBus < outputBusIndex; iOutputBus += 1) {\n        pBasePtr += pNodeBase->cachedDataCapInFramesPerBus * ma_node_output_bus_get_channels(&pNodeBase->pOutputBuses[iOutputBus]);\n    }\n\n    return pBasePtr;\n}\n\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t inputBusOffset;\n    size_t outputBusOffset;\n    size_t cachedDataOffset;\n    ma_uint32 inputBusCount;    /* So it doesn't have to be calculated twice. */\n    ma_uint32 outputBusCount;   /* So it doesn't have to be calculated twice. */\n} ma_node_heap_layout;\n\nstatic ma_result ma_node_translate_bus_counts(const ma_node_config* pConfig, ma_uint32* pInputBusCount, ma_uint32* pOutputBusCount)\n{\n    ma_uint32 inputBusCount;\n    ma_uint32 outputBusCount;\n\n    MA_ASSERT(pConfig != NULL);\n    MA_ASSERT(pInputBusCount  != NULL);\n    MA_ASSERT(pOutputBusCount != NULL);\n\n    /* Bus counts are determined by the vtable, unless they're set to `MA_NODE_BUS_COUNT_UNKNWON`, in which case they're taken from the config. */\n    if (pConfig->vtable->inputBusCount == MA_NODE_BUS_COUNT_UNKNOWN) {\n        inputBusCount = pConfig->inputBusCount;\n    } else {\n        inputBusCount = pConfig->vtable->inputBusCount;\n\n        if (pConfig->inputBusCount != MA_NODE_BUS_COUNT_UNKNOWN && pConfig->inputBusCount != pConfig->vtable->inputBusCount) {\n            return MA_INVALID_ARGS; /* Invalid configuration. You must not specify a conflicting bus count between the node's config and the vtable. */\n        }\n    }\n\n    if (pConfig->vtable->outputBusCount == MA_NODE_BUS_COUNT_UNKNOWN) {\n        outputBusCount = pConfig->outputBusCount;\n    } else {\n        outputBusCount = pConfig->vtable->outputBusCount;\n\n        if (pConfig->outputBusCount != MA_NODE_BUS_COUNT_UNKNOWN && pConfig->outputBusCount != pConfig->vtable->outputBusCount) {\n            return MA_INVALID_ARGS; /* Invalid configuration. You must not specify a conflicting bus count between the node's config and the vtable. */\n        }\n    }\n\n    /* Bus counts must be within limits. */\n    if (inputBusCount > MA_MAX_NODE_BUS_COUNT || outputBusCount > MA_MAX_NODE_BUS_COUNT) {\n        return MA_INVALID_ARGS;\n    }\n\n\n    /* We must have channel counts for each bus. */\n    if ((inputBusCount > 0 && pConfig->pInputChannels == NULL) || (outputBusCount > 0 && pConfig->pOutputChannels == NULL)) {\n        return MA_INVALID_ARGS; /* You must specify channel counts for each input and output bus. */\n    }\n\n\n    /* Some special rules for passthrough nodes. */\n    if ((pConfig->vtable->flags & MA_NODE_FLAG_PASSTHROUGH) != 0) {\n        if ((pConfig->vtable->inputBusCount != 0 && pConfig->vtable->inputBusCount != 1) || pConfig->vtable->outputBusCount != 1) {\n            return MA_INVALID_ARGS; /* Passthrough nodes must have exactly 1 output bus and either 0 or 1 input bus. */\n        }\n\n        if (pConfig->pInputChannels[0] != pConfig->pOutputChannels[0]) {\n            return MA_INVALID_ARGS; /* Passthrough nodes must have the same number of channels between input and output nodes. */\n        }\n    }\n\n\n    *pInputBusCount  = inputBusCount;\n    *pOutputBusCount = outputBusCount;\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_node_get_heap_layout(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, ma_node_heap_layout* pHeapLayout)\n{\n    ma_result result;\n    ma_uint32 inputBusCount;\n    ma_uint32 outputBusCount;\n\n    MA_ASSERT(pHeapLayout != NULL);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL || pConfig->vtable == NULL || pConfig->vtable->onProcess == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_node_translate_bus_counts(pConfig, &inputBusCount, &outputBusCount);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    /* Input buses. */\n    if (inputBusCount > MA_MAX_NODE_LOCAL_BUS_COUNT) {\n        pHeapLayout->inputBusOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(sizeof(ma_node_input_bus) * inputBusCount);\n    } else {\n        pHeapLayout->inputBusOffset = MA_SIZE_MAX;  /* MA_SIZE_MAX indicates that no heap allocation is required for the input bus. */\n    }\n\n    /* Output buses. */\n    if (outputBusCount > MA_MAX_NODE_LOCAL_BUS_COUNT) {\n        pHeapLayout->outputBusOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(sizeof(ma_node_output_bus) * outputBusCount);\n    } else {\n        pHeapLayout->outputBusOffset = MA_SIZE_MAX;\n    }\n\n    /*\n    Cached audio data.\n\n    We need to allocate memory for caching both input and output data. We have an optimization\n    where no caching is necessary for specific conditions:\n\n        - The node has 0 inputs and 1 output.\n\n    When a node meets the above conditions, no cache is allocated.\n\n    The size choice for this buffer is a little bit finicky. We don't want to be too wasteful by\n    allocating too much, but at the same time we want it be large enough so that enough frames can\n    be processed for each call to ma_node_read_pcm_frames() so that it keeps things efficient. For\n    now I'm going with 10ms @ 48K which is 480 frames per bus. This is configurable at compile\n    time. It might also be worth investigating whether or not this can be configured at run time.\n    */\n    if (inputBusCount == 0 && outputBusCount == 1) {\n        /* Fast path. No cache needed. */\n        pHeapLayout->cachedDataOffset = MA_SIZE_MAX;\n    } else {\n        /* Slow path. Cache needed. */\n        size_t cachedDataSizeInBytes = 0;\n        ma_uint32 cacheCapInFrames;\n        ma_uint32 iBus;\n\n        /* The capacity of the cache is based on our callback processing size. */\n        cacheCapInFrames = ma_node_config_get_cache_size_in_frames(pConfig, pNodeGraph);\n\n        for (iBus = 0; iBus < inputBusCount; iBus += 1) {\n            cachedDataSizeInBytes += cacheCapInFrames * ma_get_bytes_per_frame(ma_format_f32, pConfig->pInputChannels[iBus]);\n        }\n\n        for (iBus = 0; iBus < outputBusCount; iBus += 1) {\n            cachedDataSizeInBytes += cacheCapInFrames * ma_get_bytes_per_frame(ma_format_f32, pConfig->pOutputChannels[iBus]);\n        }\n\n        pHeapLayout->cachedDataOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(cachedDataSizeInBytes);\n    }\n\n\n    /*\n    Not technically part of the heap, but we can output the input and output bus counts so we can\n    avoid a redundant call to ma_node_translate_bus_counts().\n    */\n    pHeapLayout->inputBusCount  = inputBusCount;\n    pHeapLayout->outputBusCount = outputBusCount;\n\n    /* Make sure allocation size is aligned. */\n    pHeapLayout->sizeInBytes = ma_align_64(pHeapLayout->sizeInBytes);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_get_heap_size(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_node_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_node_get_heap_layout(pNodeGraph, pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_init_preallocated(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, void* pHeap, ma_node* pNode)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_result result;\n    ma_node_heap_layout heapLayout;\n    ma_uint32 iInputBus;\n    ma_uint32 iOutputBus;\n\n    if (pNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNodeBase);\n\n    result = ma_node_get_heap_layout(pNodeGraph, pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pNodeBase->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pNodeBase->pNodeGraph     = pNodeGraph;\n    pNodeBase->vtable         = pConfig->vtable;\n    pNodeBase->state          = pConfig->initialState;\n    pNodeBase->stateTimes[ma_node_state_started] = 0;\n    pNodeBase->stateTimes[ma_node_state_stopped] = (ma_uint64)(ma_int64)-1; /* Weird casting for VC6 compatibility. */\n    pNodeBase->inputBusCount  = heapLayout.inputBusCount;\n    pNodeBase->outputBusCount = heapLayout.outputBusCount;\n\n    if (heapLayout.inputBusOffset != MA_SIZE_MAX) {\n        pNodeBase->pInputBuses = (ma_node_input_bus*)ma_offset_ptr(pHeap, heapLayout.inputBusOffset);\n    } else {\n        pNodeBase->pInputBuses = pNodeBase->_inputBuses;\n    }\n\n    if (heapLayout.outputBusOffset != MA_SIZE_MAX) {\n        pNodeBase->pOutputBuses = (ma_node_output_bus*)ma_offset_ptr(pHeap, heapLayout.outputBusOffset);\n    } else {\n        pNodeBase->pOutputBuses = pNodeBase->_outputBuses;\n    }\n\n    if (heapLayout.cachedDataOffset != MA_SIZE_MAX) {\n        pNodeBase->pCachedData = (float*)ma_offset_ptr(pHeap, heapLayout.cachedDataOffset);\n        pNodeBase->cachedDataCapInFramesPerBus = ma_node_config_get_cache_size_in_frames(pConfig, pNodeGraph);\n    } else {\n        pNodeBase->pCachedData = NULL;\n    }\n\n\n    /* We need to run an initialization step for each input and output bus. */\n    for (iInputBus = 0; iInputBus < ma_node_get_input_bus_count(pNodeBase); iInputBus += 1) {\n        result = ma_node_input_bus_init(pConfig->pInputChannels[iInputBus], &pNodeBase->pInputBuses[iInputBus]);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n    for (iOutputBus = 0; iOutputBus < ma_node_get_output_bus_count(pNodeBase); iOutputBus += 1) {\n        result = ma_node_output_bus_init(pNodeBase, iOutputBus, pConfig->pOutputChannels[iOutputBus], &pNodeBase->pOutputBuses[iOutputBus]);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n    }\n\n\n    /* The cached data needs to be initialized to silence (or a sine wave tone if we're debugging). */\n    if (pNodeBase->pCachedData != NULL) {\n        ma_uint32 iBus;\n\n    #if 1   /* Toggle this between 0 and 1 to turn debugging on or off. 1 = fill with a sine wave for debugging; 0 = fill with silence. */\n        /* For safety we'll go ahead and default the buffer to silence. */\n        for (iBus = 0; iBus < ma_node_get_input_bus_count(pNodeBase); iBus += 1) {\n            ma_silence_pcm_frames(ma_node_get_cached_input_ptr(pNode, iBus), pNodeBase->cachedDataCapInFramesPerBus, ma_format_f32, ma_node_input_bus_get_channels(&pNodeBase->pInputBuses[iBus]));\n        }\n        for (iBus = 0; iBus < ma_node_get_output_bus_count(pNodeBase); iBus += 1) {\n            ma_silence_pcm_frames(ma_node_get_cached_output_ptr(pNode, iBus), pNodeBase->cachedDataCapInFramesPerBus, ma_format_f32, ma_node_output_bus_get_channels(&pNodeBase->pOutputBuses[iBus]));\n        }\n    #else\n        /* For debugging. Default to a sine wave. */\n        for (iBus = 0; iBus < ma_node_get_input_bus_count(pNodeBase); iBus += 1) {\n            ma_debug_fill_pcm_frames_with_sine_wave(ma_node_get_cached_input_ptr(pNode, iBus), pNodeBase->cachedDataCapInFramesPerBus, ma_format_f32, ma_node_input_bus_get_channels(&pNodeBase->pInputBuses[iBus]), 48000);\n        }\n        for (iBus = 0; iBus < ma_node_get_output_bus_count(pNodeBase); iBus += 1) {\n            ma_debug_fill_pcm_frames_with_sine_wave(ma_node_get_cached_output_ptr(pNode, iBus), pNodeBase->cachedDataCapInFramesPerBus, ma_format_f32, ma_node_output_bus_get_channels(&pNodeBase->pOutputBuses[iBus]), 48000);\n        }\n    #endif\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_init(ma_node_graph* pNodeGraph, const ma_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_node* pNode)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_node_get_heap_size(pNodeGraph, pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_node_init_preallocated(pNodeGraph, pConfig, pHeap, pNode);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    ((ma_node_base*)pNode)->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_node_uninit(ma_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n\n    if (pNodeBase == NULL) {\n        return;\n    }\n\n    /*\n    The first thing we need to do is fully detach the node. This will detach all inputs and\n    outputs. We need to do this first because it will sever the connection with the node graph and\n    allow us to complete uninitialization without needing to worry about thread-safety with the\n    audio thread. The detachment process will wait for any local processing of the node to finish.\n    */\n    ma_node_detach_full(pNode);\n\n    /*\n    At this point the node should be completely unreferenced by the node graph and we can finish up\n    the uninitialization process without needing to worry about thread-safety.\n    */\n    if (pNodeBase->_ownsHeap) {\n        ma_free(pNodeBase->_pHeap, pAllocationCallbacks);\n    }\n}\n\nMA_API ma_node_graph* ma_node_get_node_graph(const ma_node* pNode)\n{\n    if (pNode == NULL) {\n        return NULL;\n    }\n\n    return ((const ma_node_base*)pNode)->pNodeGraph;\n}\n\nMA_API ma_uint32 ma_node_get_input_bus_count(const ma_node* pNode)\n{\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    return ((ma_node_base*)pNode)->inputBusCount;\n}\n\nMA_API ma_uint32 ma_node_get_output_bus_count(const ma_node* pNode)\n{\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    return ((ma_node_base*)pNode)->outputBusCount;\n}\n\n\nMA_API ma_uint32 ma_node_get_input_channels(const ma_node* pNode, ma_uint32 inputBusIndex)\n{\n    const ma_node_base* pNodeBase = (const ma_node_base*)pNode;\n\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    if (inputBusIndex >= ma_node_get_input_bus_count(pNode)) {\n        return 0;   /* Invalid bus index. */\n    }\n\n    return ma_node_input_bus_get_channels(&pNodeBase->pInputBuses[inputBusIndex]);\n}\n\nMA_API ma_uint32 ma_node_get_output_channels(const ma_node* pNode, ma_uint32 outputBusIndex)\n{\n    const ma_node_base* pNodeBase = (const ma_node_base*)pNode;\n\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNode)) {\n        return 0;   /* Invalid bus index. */\n    }\n\n    return ma_node_output_bus_get_channels(&pNodeBase->pOutputBuses[outputBusIndex]);\n}\n\n\nstatic ma_result ma_node_detach_full(ma_node* pNode)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_uint32 iInputBus;\n\n    if (pNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /*\n    Make sure the node is completely detached first. This will not return until the output bus is\n    guaranteed to no longer be referenced by the audio thread.\n    */\n    ma_node_detach_all_output_buses(pNode);\n\n    /*\n    At this point all output buses will have been detached from the graph and we can be guaranteed\n    that none of its input nodes will be getting processed by the graph. We can detach these\n    without needing to worry about the audio thread touching them.\n    */\n    for (iInputBus = 0; iInputBus < ma_node_get_input_bus_count(pNode); iInputBus += 1) {\n        ma_node_input_bus* pInputBus;\n        ma_node_output_bus* pOutputBus;\n\n        pInputBus = &pNodeBase->pInputBuses[iInputBus];\n\n        /*\n        This is important. We cannot be using ma_node_input_bus_first() or ma_node_input_bus_next(). Those\n        functions are specifically for the audio thread. We'll instead just manually iterate using standard\n        linked list logic. We don't need to worry about the audio thread referencing these because the step\n        above severed the connection to the graph.\n        */\n        for (pOutputBus = (ma_node_output_bus*)ma_atomic_load_ptr(&pInputBus->head.pNext); pOutputBus != NULL; pOutputBus = (ma_node_output_bus*)ma_atomic_load_ptr(&pInputBus->head.pNext)) {\n            ma_node_detach_output_bus(pOutputBus->pNode, pOutputBus->outputBusIndex);   /* This won't do any waiting in practice and should be efficient. */\n        }\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_detach_output_bus(ma_node* pNode, ma_uint32 outputBusIndex)\n{\n    ma_result result = MA_SUCCESS;\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_node_base* pInputNodeBase;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNode)) {\n        return MA_INVALID_ARGS; /* Invalid output bus index. */\n    }\n\n    /* We need to lock the output bus because we need to inspect the input node and grab its input bus. */\n    ma_node_output_bus_lock(&pNodeBase->pOutputBuses[outputBusIndex]);\n    {\n        pInputNodeBase = (ma_node_base*)pNodeBase->pOutputBuses[outputBusIndex].pInputNode;\n        if (pInputNodeBase != NULL) {\n            ma_node_input_bus_detach__no_output_bus_lock(&pInputNodeBase->pInputBuses[pNodeBase->pOutputBuses[outputBusIndex].inputNodeInputBusIndex], &pNodeBase->pOutputBuses[outputBusIndex]);\n        }\n    }\n    ma_node_output_bus_unlock(&pNodeBase->pOutputBuses[outputBusIndex]);\n\n    return result;\n}\n\nMA_API ma_result ma_node_detach_all_output_buses(ma_node* pNode)\n{\n    ma_uint32 iOutputBus;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    for (iOutputBus = 0; iOutputBus < ma_node_get_output_bus_count(pNode); iOutputBus += 1) {\n        ma_node_detach_output_bus(pNode, iOutputBus);\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_attach_output_bus(ma_node* pNode, ma_uint32 outputBusIndex, ma_node* pOtherNode, ma_uint32 otherNodeInputBusIndex)\n{\n    ma_node_base* pNodeBase  = (ma_node_base*)pNode;\n    ma_node_base* pOtherNodeBase = (ma_node_base*)pOtherNode;\n\n    if (pNodeBase == NULL || pOtherNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pNodeBase == pOtherNodeBase) {\n        return MA_INVALID_OPERATION;    /* Cannot attach a node to itself. */\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNode) || otherNodeInputBusIndex >= ma_node_get_input_bus_count(pOtherNode)) {\n        return MA_INVALID_OPERATION;    /* Invalid bus index. */\n    }\n\n    /* The output channel count of the output node must be the same as the input channel count of the input node. */\n    if (ma_node_get_output_channels(pNode, outputBusIndex) != ma_node_get_input_channels(pOtherNode, otherNodeInputBusIndex)) {\n        return MA_INVALID_OPERATION;    /* Channel count is incompatible. */\n    }\n\n    /* This will deal with detaching if the output bus is already attached to something. */\n    ma_node_input_bus_attach(&pOtherNodeBase->pInputBuses[otherNodeInputBusIndex], &pNodeBase->pOutputBuses[outputBusIndex], pOtherNode, otherNodeInputBusIndex);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_node_set_output_bus_volume(ma_node* pNode, ma_uint32 outputBusIndex, float volume)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n\n    if (pNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNode)) {\n        return MA_INVALID_ARGS; /* Invalid bus index. */\n    }\n\n    return ma_node_output_bus_set_volume(&pNodeBase->pOutputBuses[outputBusIndex], volume);\n}\n\nMA_API float ma_node_get_output_bus_volume(const ma_node* pNode, ma_uint32 outputBusIndex)\n{\n    const ma_node_base* pNodeBase = (const ma_node_base*)pNode;\n\n    if (pNodeBase == NULL) {\n        return 0;\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNode)) {\n        return 0;   /* Invalid bus index. */\n    }\n\n    return ma_node_output_bus_get_volume(&pNodeBase->pOutputBuses[outputBusIndex]);\n}\n\nMA_API ma_result ma_node_set_state(ma_node* pNode, ma_node_state state)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n\n    if (pNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_exchange_i32(&pNodeBase->state, state);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_node_state ma_node_get_state(const ma_node* pNode)\n{\n    const ma_node_base* pNodeBase = (const ma_node_base*)pNode;\n\n    if (pNodeBase == NULL) {\n        return ma_node_state_stopped;\n    }\n\n    return (ma_node_state)ma_atomic_load_i32(&pNodeBase->state);\n}\n\nMA_API ma_result ma_node_set_state_time(ma_node* pNode, ma_node_state state, ma_uint64 globalTime)\n{\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Validation check for safety since we'll be using this as an index into stateTimes[]. */\n    if (state != ma_node_state_started && state != ma_node_state_stopped) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_exchange_64(&((ma_node_base*)pNode)->stateTimes[state], globalTime);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_uint64 ma_node_get_state_time(const ma_node* pNode, ma_node_state state)\n{\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    /* Validation check for safety since we'll be using this as an index into stateTimes[]. */\n    if (state != ma_node_state_started && state != ma_node_state_stopped) {\n        return 0;\n    }\n\n    return ma_atomic_load_64(&((ma_node_base*)pNode)->stateTimes[state]);\n}\n\nMA_API ma_node_state ma_node_get_state_by_time(const ma_node* pNode, ma_uint64 globalTime)\n{\n    if (pNode == NULL) {\n        return ma_node_state_stopped;\n    }\n\n    return ma_node_get_state_by_time_range(pNode, globalTime, globalTime);\n}\n\nMA_API ma_node_state ma_node_get_state_by_time_range(const ma_node* pNode, ma_uint64 globalTimeBeg, ma_uint64 globalTimeEnd)\n{\n    ma_node_state state;\n\n    if (pNode == NULL) {\n        return ma_node_state_stopped;\n    }\n\n    state = ma_node_get_state(pNode);\n\n    /* An explicitly stopped node is always stopped. */\n    if (state == ma_node_state_stopped) {\n        return ma_node_state_stopped;\n    }\n\n    /*\n    Getting here means the node is marked as started, but it may still not be truly started due to\n    its start time not having been reached yet. Also, the stop time may have also been reached in\n    which case it'll be considered stopped.\n    */\n    if (ma_node_get_state_time(pNode, ma_node_state_stopped) < globalTimeBeg) {\n        return ma_node_state_stopped;   /* End time is before the start of the range. */\n    }\n\n    if (ma_node_get_state_time(pNode, ma_node_state_started) > globalTimeEnd) {\n        return ma_node_state_stopped;   /* Start time is after the end of the range. */\n    }\n\n    /* Getting here means the node is marked as started and is within its start/stop times. */\n    return ma_node_state_started;\n}\n\nMA_API ma_uint64 ma_node_get_time(const ma_node* pNode)\n{\n    if (pNode == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_64(&((ma_node_base*)pNode)->localTime);\n}\n\nMA_API ma_result ma_node_set_time(ma_node* pNode, ma_uint64 localTime)\n{\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_exchange_64(&((ma_node_base*)pNode)->localTime, localTime);\n\n    return MA_SUCCESS;\n}\n\n\n\nstatic void ma_node_process_pcm_frames_internal(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n\n    if (pNodeBase->vtable->onProcess) {\n        pNodeBase->vtable->onProcess(pNode, ppFramesIn, pFrameCountIn, ppFramesOut, pFrameCountOut);\n    }\n}\n\nstatic ma_result ma_node_read_pcm_frames(ma_node* pNode, ma_uint32 outputBusIndex, float* pFramesOut, ma_uint32 frameCount, ma_uint32* pFramesRead, ma_uint64 globalTime)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_result result = MA_SUCCESS;\n    ma_uint32 iInputBus;\n    ma_uint32 iOutputBus;\n    ma_uint32 inputBusCount;\n    ma_uint32 outputBusCount;\n    ma_uint32 totalFramesRead = 0;\n    float* ppFramesIn[MA_MAX_NODE_BUS_COUNT];\n    float* ppFramesOut[MA_MAX_NODE_BUS_COUNT];\n    ma_uint64 globalTimeBeg;\n    ma_uint64 globalTimeEnd;\n    ma_uint64 startTime;\n    ma_uint64 stopTime;\n    ma_uint32 timeOffsetBeg;\n    ma_uint32 timeOffsetEnd;\n    ma_uint32 frameCountIn;\n    ma_uint32 frameCountOut;\n\n    /*\n    pFramesRead is mandatory. It must be used to determine how many frames were read. It's normal and\n    expected that the number of frames read may be different to that requested. Therefore, the caller\n    must look at this value to correctly determine how many frames were read.\n    */\n    MA_ASSERT(pFramesRead != NULL); /* <-- If you've triggered this assert, you're using this function wrong. You *must* use this variable and inspect it after the call returns. */\n    if (pFramesRead == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pFramesRead = 0;   /* Safety. */\n\n    if (pNodeBase == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (outputBusIndex >= ma_node_get_output_bus_count(pNodeBase)) {\n        return MA_INVALID_ARGS; /* Invalid output bus index. */\n    }\n\n    globalTimeBeg = globalTime;\n    globalTimeEnd = globalTime + frameCount;\n\n    /* Don't do anything if we're in a stopped state. */\n    if (ma_node_get_state_by_time_range(pNode, globalTimeBeg, globalTimeEnd) != ma_node_state_started) {\n        return MA_SUCCESS;  /* We're in a stopped state. This is not an error - we just need to not read anything. */\n    }\n\n    startTime = ma_node_get_state_time(pNode, ma_node_state_started);\n    stopTime  = ma_node_get_state_time(pNode, ma_node_state_stopped);\n\n    /*\n    At this point we know that we are inside our start/stop times. However, we may need to adjust\n    our frame count and output pointer to accommodate since we could be straddling the time period\n    that this function is getting called for.\n\n    It's possible (and likely) that the start time does not line up with the output buffer. We\n    therefore need to offset it by a number of frames to accommodate. The same thing applies for\n    the stop time.\n    */\n    timeOffsetBeg = (globalTimeBeg < startTime) ? (ma_uint32)(startTime - globalTimeBeg) : 0;\n    timeOffsetEnd = (globalTimeEnd > stopTime)  ? (ma_uint32)(globalTimeEnd - stopTime)  : 0;\n\n    /* Trim based on the start offset. We need to silence the start of the buffer. */\n    if (timeOffsetBeg > 0) {\n        MA_ASSERT(timeOffsetBeg <= frameCount);\n        if (timeOffsetBeg > frameCount) {\n            timeOffsetBeg = frameCount;\n        }\n\n        ma_silence_pcm_frames(pFramesOut, timeOffsetBeg, ma_format_f32, ma_node_get_output_channels(pNode, outputBusIndex));\n        pFramesOut += timeOffsetBeg * ma_node_get_output_channels(pNode, outputBusIndex);\n        frameCount -= timeOffsetBeg;\n    }\n\n    /* Trim based on the end offset. We don't need to silence the tail section because we'll just have a reduced value written to pFramesRead. */\n    if (timeOffsetEnd > 0) {\n        MA_ASSERT(timeOffsetEnd <= frameCount);\n        if (timeOffsetEnd > frameCount) {\n            timeOffsetEnd = frameCount;\n        }\n\n        frameCount -= timeOffsetEnd;\n    }\n\n\n    /* We run on different paths depending on the bus counts. */\n    inputBusCount  = ma_node_get_input_bus_count(pNode);\n    outputBusCount = ma_node_get_output_bus_count(pNode);\n\n    /*\n    Run a simplified path when there are no inputs and one output. In this case there's nothing to\n    actually read and we can go straight to output. This is a very common scenario because the vast\n    majority of data source nodes will use this setup so this optimization I think is worthwhile.\n    */\n    if (inputBusCount == 0 && outputBusCount == 1) {\n        /* Fast path. No need to read from input and no need for any caching. */\n        frameCountIn  = 0;\n        frameCountOut = frameCount;    /* Just read as much as we can. The callback will return what was actually read. */\n\n        ppFramesOut[0] = pFramesOut;\n\n        /*\n        If it's a passthrough we won't be expecting the callback to output anything, so we'll\n        need to pre-silence the output buffer.\n        */\n        if ((pNodeBase->vtable->flags & MA_NODE_FLAG_PASSTHROUGH) != 0) {\n            ma_silence_pcm_frames(pFramesOut, frameCount, ma_format_f32, ma_node_get_output_channels(pNode, outputBusIndex));\n        }\n\n        ma_node_process_pcm_frames_internal(pNode, NULL, &frameCountIn, ppFramesOut, &frameCountOut);\n        totalFramesRead = frameCountOut;\n    } else {\n        /* Slow path. Need to read input data. */\n        if ((pNodeBase->vtable->flags & MA_NODE_FLAG_PASSTHROUGH) != 0) {\n            /*\n            Fast path. We're running a passthrough. We need to read directly into the output buffer, but\n            still fire the callback so that event handling and trigger nodes can do their thing. Since\n            it's a passthrough there's no need for any kind of caching logic.\n            */\n            MA_ASSERT(outputBusCount == inputBusCount);\n            MA_ASSERT(outputBusCount == 1);\n            MA_ASSERT(outputBusIndex == 0);\n\n            /* We just read directly from input bus to output buffer, and then afterwards fire the callback. */\n            ppFramesOut[0] = pFramesOut;\n            ppFramesIn[0] = ppFramesOut[0];\n\n            result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[0], ppFramesIn[0], frameCount, &totalFramesRead, globalTime);\n            if (result == MA_SUCCESS) {\n                /* Even though it's a passthrough, we still need to fire the callback. */\n                frameCountIn  = totalFramesRead;\n                frameCountOut = totalFramesRead;\n\n                if (totalFramesRead > 0) {\n                    ma_node_process_pcm_frames_internal(pNode, (const float**)ppFramesIn, &frameCountIn, ppFramesOut, &frameCountOut);  /* From GCC: expected 'const float **' but argument is of type 'float **'. Shouldn't this be implicit? Explicit cast to silence the warning. */\n                }\n\n                /*\n                A passthrough should never have modified the input and output frame counts. If you're\n                triggering these asserts you need to fix your processing callback.\n                */\n                MA_ASSERT(frameCountIn  == totalFramesRead);\n                MA_ASSERT(frameCountOut == totalFramesRead);\n            }\n        } else {\n            /* Slow path. Need to do caching. */\n            ma_uint32 framesToProcessIn;\n            ma_uint32 framesToProcessOut;\n            ma_bool32 consumeNullInput = MA_FALSE;\n\n            /*\n            We use frameCount as a basis for the number of frames to read since that's what's being\n            requested, however we still need to clamp it to whatever can fit in the cache.\n\n            This will also be used as the basis for determining how many input frames to read. This is\n            not ideal because it can result in too many input frames being read which introduces latency.\n            To solve this, nodes can implement an optional callback called onGetRequiredInputFrameCount\n            which is used as hint to miniaudio as to how many input frames it needs to read at a time. This\n            callback is completely optional, and if it's not set, miniaudio will assume `frameCount`.\n\n            This function will be called multiple times for each period of time, once for each output node.\n            We cannot read from each input node each time this function is called. Instead we need to check\n            whether or not this is first output bus to be read from for this time period, and if so, read\n            from our input data.\n\n            To determine whether or not we're ready to read data, we check a flag. There will be one flag\n            for each output. When the flag is set, it means data has been read previously and that we're\n            ready to advance time forward for our input nodes by reading fresh data.\n            */\n            framesToProcessOut = frameCount;\n            if (framesToProcessOut > pNodeBase->cachedDataCapInFramesPerBus) {\n                framesToProcessOut = pNodeBase->cachedDataCapInFramesPerBus;\n            }\n\n            framesToProcessIn  = frameCount;\n            if (pNodeBase->vtable->onGetRequiredInputFrameCount) {\n                pNodeBase->vtable->onGetRequiredInputFrameCount(pNode, framesToProcessOut, &framesToProcessIn); /* <-- It does not matter if this fails. */\n            }\n            if (framesToProcessIn > pNodeBase->cachedDataCapInFramesPerBus) {\n                framesToProcessIn = pNodeBase->cachedDataCapInFramesPerBus;\n            }\n\n\n            MA_ASSERT(framesToProcessIn  <= 0xFFFF);\n            MA_ASSERT(framesToProcessOut <= 0xFFFF);\n\n            if (ma_node_output_bus_has_read(&pNodeBase->pOutputBuses[outputBusIndex])) {\n                /* Getting here means we need to do another round of processing. */\n                pNodeBase->cachedFrameCountOut = 0;\n\n                for (;;) {\n                    frameCountOut = 0;\n\n                    /*\n                    We need to prepare our output frame pointers for processing. In the same iteration we need\n                    to mark every output bus as unread so that future calls to this function for different buses\n                    for the current time period don't pull in data when they should instead be reading from cache.\n                    */\n                    for (iOutputBus = 0; iOutputBus < outputBusCount; iOutputBus += 1) {\n                        ma_node_output_bus_set_has_read(&pNodeBase->pOutputBuses[iOutputBus], MA_FALSE); /* <-- This is what tells the next calls to this function for other output buses for this time period to read from cache instead of pulling in more data. */\n                        ppFramesOut[iOutputBus] = ma_node_get_cached_output_ptr(pNode, iOutputBus);\n                    }\n\n                    /* We only need to read from input buses if there isn't already some data in the cache. */\n                    if (pNodeBase->cachedFrameCountIn == 0) {\n                        ma_uint32 maxFramesReadIn = 0;\n\n                        /* Here is where we pull in data from the input buses. This is what will trigger an advance in time. */\n                        for (iInputBus = 0; iInputBus < inputBusCount; iInputBus += 1) {\n                            ma_uint32 framesRead;\n\n                            /* The first thing to do is get the offset within our bulk allocation to store this input data. */\n                            ppFramesIn[iInputBus] = ma_node_get_cached_input_ptr(pNode, iInputBus);\n\n                            /* Once we've determined our destination pointer we can read. Note that we must inspect the number of frames read and fill any leftovers with silence for safety. */\n                            result = ma_node_input_bus_read_pcm_frames(pNodeBase, &pNodeBase->pInputBuses[iInputBus], ppFramesIn[iInputBus], framesToProcessIn, &framesRead, globalTime);\n                            if (result != MA_SUCCESS) {\n                                /* It doesn't really matter if we fail because we'll just fill with silence. */\n                                framesRead = 0; /* Just for safety, but I don't think it's really needed. */\n                            }\n\n                            /* TODO: Minor optimization opportunity here. If no frames were read and the buffer is already filled with silence, no need to re-silence it. */\n                            /* Any leftover frames need to silenced for safety. */\n                            if (framesRead < framesToProcessIn) {\n                                ma_silence_pcm_frames(ppFramesIn[iInputBus] + (framesRead * ma_node_get_input_channels(pNodeBase, iInputBus)), (framesToProcessIn - framesRead), ma_format_f32, ma_node_get_input_channels(pNodeBase, iInputBus));\n                            }\n\n                            maxFramesReadIn = ma_max(maxFramesReadIn, framesRead);\n                        }\n\n                        /* This was a fresh load of input data so reset our consumption counter. */\n                        pNodeBase->consumedFrameCountIn = 0;\n\n                        /*\n                        We don't want to keep processing if there's nothing to process, so set the number of cached\n                        input frames to the maximum number we read from each attachment (the lesser will be padded\n                        with silence). If we didn't read anything, this will be set to 0 and the entire buffer will\n                        have been assigned to silence. This being equal to 0 is an important property for us because\n                        it allows us to detect when NULL can be passed into the processing callback for the input\n                        buffer for the purpose of continuous processing.\n                        */\n                        pNodeBase->cachedFrameCountIn = (ma_uint16)maxFramesReadIn;\n                    } else {\n                        /* We don't need to read anything, but we do need to prepare our input frame pointers. */\n                        for (iInputBus = 0; iInputBus < inputBusCount; iInputBus += 1) {\n                            ppFramesIn[iInputBus] = ma_node_get_cached_input_ptr(pNode, iInputBus) + (pNodeBase->consumedFrameCountIn * ma_node_get_input_channels(pNodeBase, iInputBus));\n                        }\n                    }\n\n                    /*\n                    At this point we have our input data so now we need to do some processing. Sneaky little\n                    optimization here - we can set the pointer to the output buffer for this output bus so\n                    that the final copy into the output buffer is done directly by onProcess().\n                    */\n                    if (pFramesOut != NULL) {\n                        ppFramesOut[outputBusIndex] = ma_offset_pcm_frames_ptr_f32(pFramesOut, pNodeBase->cachedFrameCountOut, ma_node_get_output_channels(pNode, outputBusIndex));\n                    }\n\n\n                    /* Give the processing function the entire capacity of the output buffer. */\n                    frameCountOut = (framesToProcessOut - pNodeBase->cachedFrameCountOut);\n\n                    /*\n                    We need to treat nodes with continuous processing a little differently. For these ones,\n                    we always want to fire the callback with the requested number of frames, regardless of\n                    pNodeBase->cachedFrameCountIn, which could be 0. Also, we want to check if we can pass\n                    in NULL for the input buffer to the callback.\n                    */\n                    if ((pNodeBase->vtable->flags & MA_NODE_FLAG_CONTINUOUS_PROCESSING) != 0) {\n                        /* We're using continuous processing. Make sure we specify the whole frame count at all times. */\n                        frameCountIn = framesToProcessIn;    /* Give the processing function as much input data as we've got in the buffer, including any silenced padding from short reads. */\n\n                        if ((pNodeBase->vtable->flags & MA_NODE_FLAG_ALLOW_NULL_INPUT) != 0 && pNodeBase->consumedFrameCountIn == 0 && pNodeBase->cachedFrameCountIn == 0) {\n                            consumeNullInput = MA_TRUE;\n                        } else {\n                            consumeNullInput = MA_FALSE;\n                        }\n\n                        /*\n                        Since we're using continuous processing we're always passing in a full frame count\n                        regardless of how much input data was read. If this is greater than what we read as\n                        input, we'll end up with an underflow. We instead need to make sure our cached frame\n                        count is set to the number of frames we'll be passing to the data callback. Not\n                        doing this will result in an underflow when we \"consume\" the cached data later on.\n\n                        Note that this check needs to be done after the \"consumeNullInput\" check above because\n                        we use the property of cachedFrameCountIn being 0 to determine whether or not we\n                        should be passing in a null pointer to the processing callback for when the node is\n                        configured with MA_NODE_FLAG_ALLOW_NULL_INPUT.\n                        */\n                        if (pNodeBase->cachedFrameCountIn < (ma_uint16)frameCountIn) {\n                            pNodeBase->cachedFrameCountIn = (ma_uint16)frameCountIn;\n                        }\n                    } else {\n                        frameCountIn = pNodeBase->cachedFrameCountIn;  /* Give the processing function as much valid input data as we've got. */\n                        consumeNullInput = MA_FALSE;\n                    }\n\n                    /*\n                    Process data slightly differently depending on whether or not we're consuming NULL\n                    input (checked just above).\n                    */\n                    if (consumeNullInput) {\n                        ma_node_process_pcm_frames_internal(pNode, NULL, &frameCountIn, ppFramesOut, &frameCountOut);\n                    } else {\n                        /*\n                        We want to skip processing if there's no input data, but we can only do that safely if\n                        we know that there is no chance of any output frames being produced. If continuous\n                        processing is being used, this won't be a problem because the input frame count will\n                        always be non-0. However, if continuous processing is *not* enabled and input and output\n                        data is processed at different rates, we still need to process that last input frame\n                        because there could be a few excess output frames needing to be produced from cached\n                        data. The `MA_NODE_FLAG_DIFFERENT_PROCESSING_RATES` flag is used as the indicator for\n                        determining whether or not we need to process the node even when there are no input\n                        frames available right now.\n                        */\n                        if (frameCountIn > 0 || (pNodeBase->vtable->flags & MA_NODE_FLAG_DIFFERENT_PROCESSING_RATES) != 0) {\n                            ma_node_process_pcm_frames_internal(pNode, (const float**)ppFramesIn, &frameCountIn, ppFramesOut, &frameCountOut);    /* From GCC: expected 'const float **' but argument is of type 'float **'. Shouldn't this be implicit? Explicit cast to silence the warning. */\n                        } else {\n                            frameCountOut = 0;  /* No data was processed. */\n                        }\n                    }\n\n                    /*\n                    Thanks to our sneaky optimization above we don't need to do any data copying directly into\n                    the output buffer - the onProcess() callback just did that for us. We do, however, need to\n                    apply the number of input and output frames that were processed. Note that due to continuous\n                    processing above, we need to do explicit checks here. If we just consumed a NULL input\n                    buffer it means that no actual input data was processed from the internal buffers and we\n                    don't want to be modifying any counters.\n                    */\n                    if (consumeNullInput == MA_FALSE) {\n                        pNodeBase->consumedFrameCountIn += (ma_uint16)frameCountIn;\n                        pNodeBase->cachedFrameCountIn   -= (ma_uint16)frameCountIn;\n                    }\n\n                    /* The cached output frame count is always equal to what we just read. */\n                    pNodeBase->cachedFrameCountOut += (ma_uint16)frameCountOut;\n\n                    /* If we couldn't process any data, we're done. The loop needs to be terminated here or else we'll get stuck in a loop. */\n                    if (pNodeBase->cachedFrameCountOut == framesToProcessOut || (frameCountOut == 0 && frameCountIn == 0)) {\n                        break;\n                    }\n                }\n            } else {\n                /*\n                We're not needing to read anything from the input buffer so just read directly from our\n                already-processed data.\n                */\n                if (pFramesOut != NULL) {\n                    ma_copy_pcm_frames(pFramesOut, ma_node_get_cached_output_ptr(pNodeBase, outputBusIndex), pNodeBase->cachedFrameCountOut, ma_format_f32, ma_node_get_output_channels(pNodeBase, outputBusIndex));\n                }\n            }\n\n            /* The number of frames read is always equal to the number of cached output frames. */\n            totalFramesRead = pNodeBase->cachedFrameCountOut;\n\n            /* Now that we've read the data, make sure our read flag is set. */\n            ma_node_output_bus_set_has_read(&pNodeBase->pOutputBuses[outputBusIndex], MA_TRUE);\n        }\n    }\n\n    /* Apply volume, if necessary. */\n    ma_apply_volume_factor_f32(pFramesOut, totalFramesRead * ma_node_get_output_channels(pNodeBase, outputBusIndex), ma_node_output_bus_get_volume(&pNodeBase->pOutputBuses[outputBusIndex]));\n\n    /* Advance our local time forward. */\n    ma_atomic_fetch_add_64(&pNodeBase->localTime, (ma_uint64)totalFramesRead);\n\n    *pFramesRead = totalFramesRead + timeOffsetBeg; /* Must include the silenced section at the start of the buffer. */\n    return result;\n}\n\n\n\n\n/* Data source node. */\nMA_API ma_data_source_node_config ma_data_source_node_config_init(ma_data_source* pDataSource)\n{\n    ma_data_source_node_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.nodeConfig  = ma_node_config_init();\n    config.pDataSource = pDataSource;\n\n    return config;\n}\n\n\nstatic void ma_data_source_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_data_source_node* pDataSourceNode = (ma_data_source_node*)pNode;\n    ma_format format;\n    ma_uint32 channels;\n    ma_uint32 frameCount;\n    ma_uint64 framesRead = 0;\n\n    MA_ASSERT(pDataSourceNode != NULL);\n    MA_ASSERT(pDataSourceNode->pDataSource != NULL);\n    MA_ASSERT(ma_node_get_input_bus_count(pDataSourceNode)  == 0);\n    MA_ASSERT(ma_node_get_output_bus_count(pDataSourceNode) == 1);\n\n    /* We don't want to read from ppFramesIn at all. Instead we read from the data source. */\n    (void)ppFramesIn;\n    (void)pFrameCountIn;\n\n    frameCount = *pFrameCountOut;\n\n    /* miniaudio should never be calling this with a frame count of zero. */\n    MA_ASSERT(frameCount > 0);\n\n    if (ma_data_source_get_data_format(pDataSourceNode->pDataSource, &format, &channels, NULL, NULL, 0) == MA_SUCCESS) { /* <-- Don't care about sample rate here. */\n        /* The node graph system requires samples be in floating point format. This is checked in ma_data_source_node_init(). */\n        MA_ASSERT(format == ma_format_f32);\n        (void)format;   /* Just to silence some static analysis tools. */\n\n        ma_data_source_read_pcm_frames(pDataSourceNode->pDataSource, ppFramesOut[0], frameCount, &framesRead);\n    }\n\n    *pFrameCountOut = (ma_uint32)framesRead;\n}\n\nstatic ma_node_vtable g_ma_data_source_node_vtable =\n{\n    ma_data_source_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    0,      /* 0 input buses. */\n    1,      /* 1 output bus. */\n    0\n};\n\nMA_API ma_result ma_data_source_node_init(ma_node_graph* pNodeGraph, const ma_data_source_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_data_source_node* pDataSourceNode)\n{\n    ma_result result;\n    ma_format format;   /* For validating the format, which must be ma_format_f32. */\n    ma_uint32 channels; /* For specifying the channel count of the output bus. */\n    ma_node_config baseConfig;\n\n    if (pDataSourceNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDataSourceNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_data_source_get_data_format(pConfig->pDataSource, &format, &channels, NULL, NULL, 0);    /* Don't care about sample rate. This will check pDataSource for NULL. */\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    MA_ASSERT(format == ma_format_f32); /* <-- If you've triggered this it means your data source is not outputting floating-point samples. You must configure your data source to use ma_format_f32. */\n    if (format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* Invalid format. */\n    }\n\n    /* The channel count is defined by the data source. If the caller has manually changed the channels we just ignore it. */\n    baseConfig = pConfig->nodeConfig;\n    baseConfig.vtable = &g_ma_data_source_node_vtable;  /* Explicitly set the vtable here to prevent callers from setting it incorrectly. */\n\n    /*\n    The channel count is defined by the data source. It is invalid for the caller to manually set\n    the channel counts in the config. `ma_data_source_node_config_init()` will have defaulted the\n    channel count pointer to NULL which is how it must remain. If you trigger any of these asserts\n    it means you're explicitly setting the channel count. Instead, configure the output channel\n    count of your data source to be the necessary channel count.\n    */\n    if (baseConfig.pOutputChannels != NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    baseConfig.pOutputChannels = &channels;\n\n    result = ma_node_init(pNodeGraph, &baseConfig, pAllocationCallbacks, &pDataSourceNode->base);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    pDataSourceNode->pDataSource = pConfig->pDataSource;\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_data_source_node_uninit(ma_data_source_node* pDataSourceNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_node_uninit(&pDataSourceNode->base, pAllocationCallbacks);\n}\n\nMA_API ma_result ma_data_source_node_set_looping(ma_data_source_node* pDataSourceNode, ma_bool32 isLooping)\n{\n    if (pDataSourceNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_data_source_set_looping(pDataSourceNode->pDataSource, isLooping);\n}\n\nMA_API ma_bool32 ma_data_source_node_is_looping(ma_data_source_node* pDataSourceNode)\n{\n    if (pDataSourceNode == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_data_source_is_looping(pDataSourceNode->pDataSource);\n}\n\n\n\n/* Splitter Node. */\nMA_API ma_splitter_node_config ma_splitter_node_config_init(ma_uint32 channels)\n{\n    ma_splitter_node_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.nodeConfig     = ma_node_config_init();\n    config.channels       = channels;\n    config.outputBusCount = 2;\n\n    return config;\n}\n\n\nstatic void ma_splitter_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_node_base* pNodeBase = (ma_node_base*)pNode;\n    ma_uint32 iOutputBus;\n    ma_uint32 channels;\n\n    MA_ASSERT(pNodeBase != NULL);\n    MA_ASSERT(ma_node_get_input_bus_count(pNodeBase) == 1);\n\n    /* We don't need to consider the input frame count - it'll be the same as the output frame count and we process everything. */\n    (void)pFrameCountIn;\n\n    /* NOTE: This assumes the same number of channels for all inputs and outputs. This was checked in ma_splitter_node_init(). */\n    channels = ma_node_get_input_channels(pNodeBase, 0);\n\n    /* Splitting is just copying the first input bus and copying it over to each output bus. */\n    for (iOutputBus = 0; iOutputBus < ma_node_get_output_bus_count(pNodeBase); iOutputBus += 1) {\n        ma_copy_pcm_frames(ppFramesOut[iOutputBus], ppFramesIn[0], *pFrameCountOut, ma_format_f32, channels);\n    }\n}\n\nstatic ma_node_vtable g_ma_splitter_node_vtable =\n{\n    ma_splitter_node_process_pcm_frames,\n    NULL,                       /* onGetRequiredInputFrameCount */\n    1,                          /* 1 input bus. */\n    MA_NODE_BUS_COUNT_UNKNOWN,  /* The output bus count is specified on a per-node basis. */\n    0\n};\n\nMA_API ma_result ma_splitter_node_init(ma_node_graph* pNodeGraph, const ma_splitter_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_splitter_node* pSplitterNode)\n{\n    ma_result result;\n    ma_node_config baseConfig;\n    ma_uint32 pInputChannels[1];\n    ma_uint32 pOutputChannels[MA_MAX_NODE_BUS_COUNT];\n    ma_uint32 iOutputBus;\n\n    if (pSplitterNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pSplitterNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->outputBusCount > MA_MAX_NODE_BUS_COUNT) {\n        return MA_INVALID_ARGS; /* Too many output buses. */\n    }\n\n    /* Splitters require the same number of channels between inputs and outputs. */\n    pInputChannels[0]  = pConfig->channels;\n    for (iOutputBus = 0; iOutputBus < pConfig->outputBusCount; iOutputBus += 1) {\n        pOutputChannels[iOutputBus] = pConfig->channels;\n    }\n\n    baseConfig = pConfig->nodeConfig;\n    baseConfig.vtable = &g_ma_splitter_node_vtable;\n    baseConfig.pInputChannels  = pInputChannels;\n    baseConfig.pOutputChannels = pOutputChannels;\n    baseConfig.outputBusCount  = pConfig->outputBusCount;\n\n    result = ma_node_init(pNodeGraph, &baseConfig, pAllocationCallbacks, &pSplitterNode->base);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to initialize the base node. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API void ma_splitter_node_uninit(ma_splitter_node* pSplitterNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_node_uninit(pSplitterNode, pAllocationCallbacks);\n}\n\n\n/*\nBiquad Node\n*/\nMA_API ma_biquad_node_config ma_biquad_node_config_init(ma_uint32 channels, float b0, float b1, float b2, float a0, float a1, float a2)\n{\n    ma_biquad_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.biquad = ma_biquad_config_init(ma_format_f32, channels, b0, b1, b2, a0, a1, a2);\n\n    return config;\n}\n\nstatic void ma_biquad_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_biquad_node* pLPFNode = (ma_biquad_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_biquad_process_pcm_frames(&pLPFNode->biquad, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_biquad_node_vtable =\n{\n    ma_biquad_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_biquad_node_init(ma_node_graph* pNodeGraph, const ma_biquad_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_biquad_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->biquad.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_biquad_init(&pConfig->biquad, pAllocationCallbacks, &pNode->biquad);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_biquad_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->biquad.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->biquad.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_biquad_node_reinit(const ma_biquad_config* pConfig, ma_biquad_node* pNode)\n{\n    ma_biquad_node* pLPFNode = (ma_biquad_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n\n    return ma_biquad_reinit(pConfig, &pLPFNode->biquad);\n}\n\nMA_API void ma_biquad_node_uninit(ma_biquad_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_biquad_node* pLPFNode = (ma_biquad_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_biquad_uninit(&pLPFNode->biquad, pAllocationCallbacks);\n}\n\n\n\n/*\nLow Pass Filter Node\n*/\nMA_API ma_lpf_node_config ma_lpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_lpf_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.lpf = ma_lpf_config_init(ma_format_f32, channels, sampleRate, cutoffFrequency, order);\n\n    return config;\n}\n\nstatic void ma_lpf_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_lpf_node* pLPFNode = (ma_lpf_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_lpf_process_pcm_frames(&pLPFNode->lpf, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_lpf_node_vtable =\n{\n    ma_lpf_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_lpf_node_init(ma_node_graph* pNodeGraph, const ma_lpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_lpf_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->lpf.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_lpf_init(&pConfig->lpf, pAllocationCallbacks, &pNode->lpf);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_lpf_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->lpf.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->lpf.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_lpf_node_reinit(const ma_lpf_config* pConfig, ma_lpf_node* pNode)\n{\n    ma_lpf_node* pLPFNode = (ma_lpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_lpf_reinit(pConfig, &pLPFNode->lpf);\n}\n\nMA_API void ma_lpf_node_uninit(ma_lpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_lpf_node* pLPFNode = (ma_lpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_lpf_uninit(&pLPFNode->lpf, pAllocationCallbacks);\n}\n\n\n\n/*\nHigh Pass Filter Node\n*/\nMA_API ma_hpf_node_config ma_hpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_hpf_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.hpf = ma_hpf_config_init(ma_format_f32, channels, sampleRate, cutoffFrequency, order);\n\n    return config;\n}\n\nstatic void ma_hpf_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_hpf_node* pHPFNode = (ma_hpf_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_hpf_process_pcm_frames(&pHPFNode->hpf, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_hpf_node_vtable =\n{\n    ma_hpf_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_hpf_node_init(ma_node_graph* pNodeGraph, const ma_hpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hpf_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->hpf.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_hpf_init(&pConfig->hpf, pAllocationCallbacks, &pNode->hpf);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_hpf_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->hpf.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->hpf.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_hpf_node_reinit(const ma_hpf_config* pConfig, ma_hpf_node* pNode)\n{\n    ma_hpf_node* pHPFNode = (ma_hpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_hpf_reinit(pConfig, &pHPFNode->hpf);\n}\n\nMA_API void ma_hpf_node_uninit(ma_hpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_hpf_node* pHPFNode = (ma_hpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_hpf_uninit(&pHPFNode->hpf, pAllocationCallbacks);\n}\n\n\n\n\n/*\nBand Pass Filter Node\n*/\nMA_API ma_bpf_node_config ma_bpf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double cutoffFrequency, ma_uint32 order)\n{\n    ma_bpf_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.bpf = ma_bpf_config_init(ma_format_f32, channels, sampleRate, cutoffFrequency, order);\n\n    return config;\n}\n\nstatic void ma_bpf_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_bpf_node* pBPFNode = (ma_bpf_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_bpf_process_pcm_frames(&pBPFNode->bpf, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_bpf_node_vtable =\n{\n    ma_bpf_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_bpf_node_init(ma_node_graph* pNodeGraph, const ma_bpf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_bpf_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->bpf.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_bpf_init(&pConfig->bpf, pAllocationCallbacks, &pNode->bpf);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_bpf_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->bpf.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->bpf.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_bpf_node_reinit(const ma_bpf_config* pConfig, ma_bpf_node* pNode)\n{\n    ma_bpf_node* pBPFNode = (ma_bpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_bpf_reinit(pConfig, &pBPFNode->bpf);\n}\n\nMA_API void ma_bpf_node_uninit(ma_bpf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bpf_node* pBPFNode = (ma_bpf_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_bpf_uninit(&pBPFNode->bpf, pAllocationCallbacks);\n}\n\n\n\n/*\nNotching Filter Node\n*/\nMA_API ma_notch_node_config ma_notch_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double q, double frequency)\n{\n    ma_notch_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.notch = ma_notch2_config_init(ma_format_f32, channels, sampleRate, q, frequency);\n\n    return config;\n}\n\nstatic void ma_notch_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_notch_node* pBPFNode = (ma_notch_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_notch2_process_pcm_frames(&pBPFNode->notch, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_notch_node_vtable =\n{\n    ma_notch_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_notch_node_init(ma_node_graph* pNodeGraph, const ma_notch_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_notch_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->notch.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_notch2_init(&pConfig->notch, pAllocationCallbacks, &pNode->notch);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_notch_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->notch.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->notch.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_notch_node_reinit(const ma_notch_config* pConfig, ma_notch_node* pNode)\n{\n    ma_notch_node* pNotchNode = (ma_notch_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_notch2_reinit(pConfig, &pNotchNode->notch);\n}\n\nMA_API void ma_notch_node_uninit(ma_notch_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_notch_node* pNotchNode = (ma_notch_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_notch2_uninit(&pNotchNode->notch, pAllocationCallbacks);\n}\n\n\n\n/*\nPeaking Filter Node\n*/\nMA_API ma_peak_node_config ma_peak_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency)\n{\n    ma_peak_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.peak = ma_peak2_config_init(ma_format_f32, channels, sampleRate, gainDB, q, frequency);\n\n    return config;\n}\n\nstatic void ma_peak_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_peak_node* pBPFNode = (ma_peak_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_peak2_process_pcm_frames(&pBPFNode->peak, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_peak_node_vtable =\n{\n    ma_peak_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_peak_node_init(ma_node_graph* pNodeGraph, const ma_peak_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_peak_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->peak.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_peak2_init(&pConfig->peak, pAllocationCallbacks, &pNode->peak);\n    if (result != MA_SUCCESS) {\n        ma_node_uninit(pNode, pAllocationCallbacks);\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_peak_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->peak.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->peak.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_peak_node_reinit(const ma_peak_config* pConfig, ma_peak_node* pNode)\n{\n    ma_peak_node* pPeakNode = (ma_peak_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_peak2_reinit(pConfig, &pPeakNode->peak);\n}\n\nMA_API void ma_peak_node_uninit(ma_peak_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_peak_node* pPeakNode = (ma_peak_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_peak2_uninit(&pPeakNode->peak, pAllocationCallbacks);\n}\n\n\n\n/*\nLow Shelf Filter Node\n*/\nMA_API ma_loshelf_node_config ma_loshelf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency)\n{\n    ma_loshelf_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.loshelf = ma_loshelf2_config_init(ma_format_f32, channels, sampleRate, gainDB, q, frequency);\n\n    return config;\n}\n\nstatic void ma_loshelf_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_loshelf_node* pBPFNode = (ma_loshelf_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_loshelf2_process_pcm_frames(&pBPFNode->loshelf, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_loshelf_node_vtable =\n{\n    ma_loshelf_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_loshelf_node_init(ma_node_graph* pNodeGraph, const ma_loshelf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_loshelf_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->loshelf.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_loshelf2_init(&pConfig->loshelf, pAllocationCallbacks, &pNode->loshelf);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_loshelf_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->loshelf.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->loshelf.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_loshelf_node_reinit(const ma_loshelf_config* pConfig, ma_loshelf_node* pNode)\n{\n    ma_loshelf_node* pLoshelfNode = (ma_loshelf_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_loshelf2_reinit(pConfig, &pLoshelfNode->loshelf);\n}\n\nMA_API void ma_loshelf_node_uninit(ma_loshelf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_loshelf_node* pLoshelfNode = (ma_loshelf_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_loshelf2_uninit(&pLoshelfNode->loshelf, pAllocationCallbacks);\n}\n\n\n\n/*\nHigh Shelf Filter Node\n*/\nMA_API ma_hishelf_node_config ma_hishelf_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, double gainDB, double q, double frequency)\n{\n    ma_hishelf_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.hishelf = ma_hishelf2_config_init(ma_format_f32, channels, sampleRate, gainDB, q, frequency);\n\n    return config;\n}\n\nstatic void ma_hishelf_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_hishelf_node* pBPFNode = (ma_hishelf_node*)pNode;\n\n    MA_ASSERT(pNode != NULL);\n    (void)pFrameCountIn;\n\n    ma_hishelf2_process_pcm_frames(&pBPFNode->hishelf, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_hishelf_node_vtable =\n{\n    ma_hishelf_node_process_pcm_frames,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* One input. */\n    1,      /* One output. */\n    0       /* Default flags. */\n};\n\nMA_API ma_result ma_hishelf_node_init(ma_node_graph* pNodeGraph, const ma_hishelf_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_hishelf_node* pNode)\n{\n    ma_result result;\n    ma_node_config baseNodeConfig;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pNode);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->hishelf.format != ma_format_f32) {\n        return MA_INVALID_ARGS; /* The format must be f32. */\n    }\n\n    result = ma_hishelf2_init(&pConfig->hishelf, pAllocationCallbacks, &pNode->hishelf);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseNodeConfig = ma_node_config_init();\n    baseNodeConfig.vtable          = &g_ma_hishelf_node_vtable;\n    baseNodeConfig.pInputChannels  = &pConfig->hishelf.channels;\n    baseNodeConfig.pOutputChannels = &pConfig->hishelf.channels;\n\n    result = ma_node_init(pNodeGraph, &baseNodeConfig, pAllocationCallbacks, pNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return result;\n}\n\nMA_API ma_result ma_hishelf_node_reinit(const ma_hishelf_config* pConfig, ma_hishelf_node* pNode)\n{\n    ma_hishelf_node* pHishelfNode = (ma_hishelf_node*)pNode;\n\n    if (pNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_hishelf2_reinit(pConfig, &pHishelfNode->hishelf);\n}\n\nMA_API void ma_hishelf_node_uninit(ma_hishelf_node* pNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_hishelf_node* pHishelfNode = (ma_hishelf_node*)pNode;\n\n    if (pNode == NULL) {\n        return;\n    }\n\n    ma_node_uninit(pNode, pAllocationCallbacks);\n    ma_hishelf2_uninit(&pHishelfNode->hishelf, pAllocationCallbacks);\n}\n\n\n\n\nMA_API ma_delay_node_config ma_delay_node_config_init(ma_uint32 channels, ma_uint32 sampleRate, ma_uint32 delayInFrames, float decay)\n{\n    ma_delay_node_config config;\n\n    config.nodeConfig = ma_node_config_init();\n    config.delay = ma_delay_config_init(channels, sampleRate, delayInFrames, decay);\n\n    return config;\n}\n\n\nstatic void ma_delay_node_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_delay_node* pDelayNode = (ma_delay_node*)pNode;\n\n    (void)pFrameCountIn;\n\n    ma_delay_process_pcm_frames(&pDelayNode->delay, ppFramesOut[0], ppFramesIn[0], *pFrameCountOut);\n}\n\nstatic ma_node_vtable g_ma_delay_node_vtable =\n{\n    ma_delay_node_process_pcm_frames,\n    NULL,\n    1,  /* 1 input channels. */\n    1,  /* 1 output channel. */\n    MA_NODE_FLAG_CONTINUOUS_PROCESSING  /* Delay requires continuous processing to ensure the tail get's processed. */\n};\n\nMA_API ma_result ma_delay_node_init(ma_node_graph* pNodeGraph, const ma_delay_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_delay_node* pDelayNode)\n{\n    ma_result result;\n    ma_node_config baseConfig;\n\n    if (pDelayNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pDelayNode);\n\n    result = ma_delay_init(&pConfig->delay, pAllocationCallbacks, &pDelayNode->delay);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    baseConfig = pConfig->nodeConfig;\n    baseConfig.vtable          = &g_ma_delay_node_vtable;\n    baseConfig.pInputChannels  = &pConfig->delay.channels;\n    baseConfig.pOutputChannels = &pConfig->delay.channels;\n\n    result = ma_node_init(pNodeGraph, &baseConfig, pAllocationCallbacks, &pDelayNode->baseNode);\n    if (result != MA_SUCCESS) {\n        ma_delay_uninit(&pDelayNode->delay, pAllocationCallbacks);\n        return result;\n    }\n\n    return result;\n}\n\nMA_API void ma_delay_node_uninit(ma_delay_node* pDelayNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pDelayNode == NULL) {\n        return;\n    }\n\n    /* The base node is always uninitialized first. */\n    ma_node_uninit(pDelayNode, pAllocationCallbacks);\n    ma_delay_uninit(&pDelayNode->delay, pAllocationCallbacks);\n}\n\nMA_API void ma_delay_node_set_wet(ma_delay_node* pDelayNode, float value)\n{\n    if (pDelayNode == NULL) {\n        return;\n    }\n\n    ma_delay_set_wet(&pDelayNode->delay, value);\n}\n\nMA_API float ma_delay_node_get_wet(const ma_delay_node* pDelayNode)\n{\n    if (pDelayNode == NULL) {\n        return 0;\n    }\n\n    return ma_delay_get_wet(&pDelayNode->delay);\n}\n\nMA_API void ma_delay_node_set_dry(ma_delay_node* pDelayNode, float value)\n{\n    if (pDelayNode == NULL) {\n        return;\n    }\n\n    ma_delay_set_dry(&pDelayNode->delay, value);\n}\n\nMA_API float ma_delay_node_get_dry(const ma_delay_node* pDelayNode)\n{\n    if (pDelayNode == NULL) {\n        return 0;\n    }\n\n    return ma_delay_get_dry(&pDelayNode->delay);\n}\n\nMA_API void ma_delay_node_set_decay(ma_delay_node* pDelayNode, float value)\n{\n    if (pDelayNode == NULL) {\n        return;\n    }\n\n    ma_delay_set_decay(&pDelayNode->delay, value);\n}\n\nMA_API float ma_delay_node_get_decay(const ma_delay_node* pDelayNode)\n{\n    if (pDelayNode == NULL) {\n        return 0;\n    }\n\n    return ma_delay_get_decay(&pDelayNode->delay);\n}\n#endif  /* MA_NO_NODE_GRAPH */\n\n\n/* SECTION: miniaudio_engine.c */\n#if !defined(MA_NO_ENGINE) && !defined(MA_NO_NODE_GRAPH)\n/**************************************************************************************************************************************************************\n\nEngine\n\n**************************************************************************************************************************************************************/\n#define MA_SEEK_TARGET_NONE         (~(ma_uint64)0)\n\n\nstatic void ma_sound_set_at_end(ma_sound* pSound, ma_bool32 atEnd)\n{\n    MA_ASSERT(pSound != NULL);\n    ma_atomic_exchange_32(&pSound->atEnd, atEnd);\n\n    /*\n    When this function is called the state of the sound will not yet be in a stopped state. This makes it confusing\n    because an end callback will intuitively expect ma_sound_is_playing() to return false from inside the callback.\n    I'm therefore no longer firing the callback here and will instead fire it manually in the *next* processing step\n    when the state should be set to stopped as expected.\n    */\n    #if 0\n    /* Fire any callbacks or events. */\n    if (atEnd) {\n        if (pSound->endCallback != NULL) {\n            pSound->endCallback(pSound->pEndCallbackUserData, pSound);\n        }\n    }\n    #endif\n}\n\nstatic ma_bool32 ma_sound_get_at_end(const ma_sound* pSound)\n{\n    MA_ASSERT(pSound != NULL);\n    return ma_atomic_load_32(&pSound->atEnd);\n}\n\n\nMA_API ma_engine_node_config ma_engine_node_config_init(ma_engine* pEngine, ma_engine_node_type type, ma_uint32 flags)\n{\n    ma_engine_node_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.pEngine                  = pEngine;\n    config.type                     = type;\n    config.isPitchDisabled          = (flags & MA_SOUND_FLAG_NO_PITCH) != 0;\n    config.isSpatializationDisabled = (flags & MA_SOUND_FLAG_NO_SPATIALIZATION) != 0;\n    config.monoExpansionMode        = pEngine->monoExpansionMode;\n    config.resampling               = pEngine->pitchResamplingConfig;\n\n    return config;\n}\n\n\nstatic void ma_engine_node_update_pitch_if_required(ma_engine_node* pEngineNode)\n{\n    ma_bool32 isUpdateRequired = MA_FALSE;\n    float newPitch;\n\n    MA_ASSERT(pEngineNode != NULL);\n\n    newPitch = ma_atomic_load_explicit_f32(&pEngineNode->pitch, ma_atomic_memory_order_acquire);\n\n    if (pEngineNode->oldPitch != newPitch) {\n        pEngineNode->oldPitch  = newPitch;\n        isUpdateRequired = MA_TRUE;\n    }\n\n    if (pEngineNode->oldDopplerPitch != pEngineNode->spatializer.dopplerPitch) {\n        pEngineNode->oldDopplerPitch  = pEngineNode->spatializer.dopplerPitch;\n        isUpdateRequired = MA_TRUE;\n    }\n\n    if (isUpdateRequired) {\n        float basePitch = (float)pEngineNode->sampleRate / ma_engine_get_sample_rate(pEngineNode->pEngine);\n        ma_resampler_set_rate_ratio(&pEngineNode->resampler, basePitch * pEngineNode->oldPitch * pEngineNode->oldDopplerPitch);\n    }\n}\n\nstatic ma_bool32 ma_engine_node_is_pitching_enabled(const ma_engine_node* pEngineNode)\n{\n    MA_ASSERT(pEngineNode != NULL);\n\n    /* Don't try to be clever by skipping resampling in the pitch=1 case or else you'll glitch when moving away from 1. */\n    return !ma_atomic_load_explicit_32(&pEngineNode->isPitchDisabled, ma_atomic_memory_order_acquire);\n}\n\nstatic ma_bool32 ma_engine_node_is_spatialization_enabled(const ma_engine_node* pEngineNode)\n{\n    MA_ASSERT(pEngineNode != NULL);\n\n    return !ma_atomic_load_explicit_32(&pEngineNode->isSpatializationDisabled, ma_atomic_memory_order_acquire);\n}\n\nstatic ma_result ma_engine_node_set_volume(ma_engine_node* pEngineNode, float volume)\n{\n    if (pEngineNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    ma_atomic_float_set(&pEngineNode->volume, volume);\n\n    /* If we're not smoothing we should bypass the volume gainer entirely. */\n    if (pEngineNode->volumeSmoothTimeInPCMFrames == 0) {\n        /* We should always have an active spatializer because it can be enabled and disabled dynamically. We can just use that for holding our volume. */\n        ma_spatializer_set_master_volume(&pEngineNode->spatializer, volume);\n    } else {\n        /* We're using volume smoothing, so apply the master volume to the gainer. */\n        ma_gainer_set_gain(&pEngineNode->volumeGainer, volume);\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_engine_node_get_volume(const ma_engine_node* pEngineNode, float* pVolume)\n{\n    if (pVolume == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pVolume = 0.0f;\n\n    if (pEngineNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pVolume = ma_atomic_float_get((ma_atomic_float*)&pEngineNode->volume);\n\n    return MA_SUCCESS;\n}\n\n\nstatic void ma_engine_node_process_pcm_frames__general(ma_engine_node* pEngineNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    ma_uint32 frameCountIn;\n    ma_uint32 frameCountOut;\n    ma_uint32 totalFramesProcessedIn;\n    ma_uint32 totalFramesProcessedOut;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_bool32 isPitchingEnabled;\n    ma_bool32 isFadingEnabled;\n    ma_bool32 isSpatializationEnabled;\n    ma_bool32 isPanningEnabled;\n    ma_bool32 isVolumeSmoothingEnabled;\n\n    frameCountIn  = *pFrameCountIn;\n    frameCountOut = *pFrameCountOut;\n\n    channelsIn  = ma_spatializer_get_input_channels(&pEngineNode->spatializer);\n    channelsOut = ma_spatializer_get_output_channels(&pEngineNode->spatializer);\n\n    totalFramesProcessedIn  = 0;\n    totalFramesProcessedOut = 0;\n\n    /* Update the fader if applicable. */\n    {\n        ma_uint64 fadeLengthInFrames = ma_atomic_uint64_get(&pEngineNode->fadeSettings.fadeLengthInFrames);\n        if (fadeLengthInFrames != ~(ma_uint64)0) {\n            float fadeVolumeBeg = ma_atomic_float_get(&pEngineNode->fadeSettings.volumeBeg);\n            float fadeVolumeEnd = ma_atomic_float_get(&pEngineNode->fadeSettings.volumeEnd);\n            ma_int64 fadeStartOffsetInFrames = (ma_int64)ma_atomic_uint64_get(&pEngineNode->fadeSettings.absoluteGlobalTimeInFrames);\n            if (fadeStartOffsetInFrames == (ma_int64)(~(ma_uint64)0)) {\n                fadeStartOffsetInFrames = 0;\n            } else {\n                fadeStartOffsetInFrames -= ma_engine_get_time_in_pcm_frames(pEngineNode->pEngine);\n            }\n\n            ma_fader_set_fade_ex(&pEngineNode->fader, fadeVolumeBeg, fadeVolumeEnd, fadeLengthInFrames, fadeStartOffsetInFrames);\n\n            /* Reset the fade length so we don't erroneously apply it again. */\n            ma_atomic_uint64_set(&pEngineNode->fadeSettings.fadeLengthInFrames, ~(ma_uint64)0);\n        }\n    }\n\n    isPitchingEnabled        = ma_engine_node_is_pitching_enabled(pEngineNode);\n    isFadingEnabled          = pEngineNode->fader.volumeBeg != 1 || pEngineNode->fader.volumeEnd != 1;\n    isSpatializationEnabled  = ma_engine_node_is_spatialization_enabled(pEngineNode);\n    isPanningEnabled         = pEngineNode->panner.pan != 0 && channelsOut != 1;\n    isVolumeSmoothingEnabled = pEngineNode->volumeSmoothTimeInPCMFrames > 0;\n\n    /* Keep going while we've still got data available for processing. */\n    while (totalFramesProcessedOut < frameCountOut) {\n        /*\n        We need to process in a specific order. We always do resampling first because it's likely\n        we're going to be increasing the channel count after spatialization. Also, I want to do\n        fading based on the output sample rate.\n\n        We'll first read into a buffer from the resampler. Then we'll do all processing that\n        operates on the on the input channel count. We'll then get the spatializer to output to\n        the output buffer and then do all effects from that point directly in the output buffer\n        in-place.\n\n        Note that we're always running the resampler if pitching is enabled, even when the pitch\n        is 1. If we try to be clever and skip resampling when the pitch is 1, we'll get a glitch\n        when we move away from 1, back to 1, and then away from 1 again. We'll want to implement\n        any pitch=1 optimizations in the resampler itself.\n\n        There's a small optimization here that we'll utilize since it might be a fairly common\n        case. When the input and output channel counts are the same, we'll read straight into the\n        output buffer from the resampler and do everything in-place.\n        */\n        const float* pRunningFramesIn;\n        float* pRunningFramesOut;\n        float* pWorkingBuffer;   /* This is the buffer that we'll be processing frames in. This is in input channels. */\n        float temp[MA_DATA_CONVERTER_STACK_BUFFER_SIZE / sizeof(float)];\n        ma_uint32 tempCapInFrames = ma_countof(temp) / channelsIn;\n        ma_uint32 framesAvailableIn;\n        ma_uint32 framesAvailableOut;\n        ma_uint32 framesJustProcessedIn;\n        ma_uint32 framesJustProcessedOut;\n        ma_bool32 isWorkingBufferValid = MA_FALSE;\n\n        framesAvailableIn  = frameCountIn  - totalFramesProcessedIn;\n        framesAvailableOut = frameCountOut - totalFramesProcessedOut;\n\n        pRunningFramesIn  = ma_offset_pcm_frames_const_ptr_f32(ppFramesIn[0], totalFramesProcessedIn, channelsIn);\n        pRunningFramesOut = ma_offset_pcm_frames_ptr_f32(ppFramesOut[0], totalFramesProcessedOut, channelsOut);\n\n        if (channelsIn == channelsOut) {\n            /* Fast path. Channel counts are the same. No need for an intermediary input buffer. */\n            pWorkingBuffer = pRunningFramesOut;\n        } else {\n            /* Slow path. Channel counts are different. Need to use an intermediary input buffer. */\n            pWorkingBuffer = temp;\n            if (framesAvailableOut > tempCapInFrames) {\n                framesAvailableOut = tempCapInFrames;\n            }\n        }\n\n        /* First is resampler. */\n        if (isPitchingEnabled) {\n            ma_uint64 resampleFrameCountIn  = framesAvailableIn;\n            ma_uint64 resampleFrameCountOut = framesAvailableOut;\n\n            ma_resampler_process_pcm_frames(&pEngineNode->resampler, pRunningFramesIn, &resampleFrameCountIn, pWorkingBuffer, &resampleFrameCountOut);\n            isWorkingBufferValid = MA_TRUE;\n\n            framesJustProcessedIn  = (ma_uint32)resampleFrameCountIn;\n            framesJustProcessedOut = (ma_uint32)resampleFrameCountOut;\n        } else {\n            framesJustProcessedIn  = ma_min(framesAvailableIn, framesAvailableOut);\n            framesJustProcessedOut = framesJustProcessedIn; /* When no resampling is being performed, the number of output frames is the same as input frames. */\n        }\n\n        /* Fading. */\n        if (isFadingEnabled) {\n            if (isWorkingBufferValid) {\n                ma_fader_process_pcm_frames(&pEngineNode->fader, pWorkingBuffer, pWorkingBuffer, framesJustProcessedOut);   /* In-place processing. */\n            } else {\n                ma_fader_process_pcm_frames(&pEngineNode->fader, pWorkingBuffer, pRunningFramesIn, framesJustProcessedOut);\n                isWorkingBufferValid = MA_TRUE;\n            }\n        }\n\n        /*\n        If we're using smoothing, we won't be applying volume via the spatializer, but instead from a ma_gainer. In this case\n        we'll want to apply our volume now.\n        */\n        if (isVolumeSmoothingEnabled) {\n            if (isWorkingBufferValid) {\n                ma_gainer_process_pcm_frames(&pEngineNode->volumeGainer, pWorkingBuffer, pWorkingBuffer, framesJustProcessedOut);\n            } else {\n                ma_gainer_process_pcm_frames(&pEngineNode->volumeGainer, pWorkingBuffer, pRunningFramesIn, framesJustProcessedOut);\n                isWorkingBufferValid = MA_TRUE;\n            }\n        }\n\n        /*\n        If at this point we still haven't actually done anything with the working buffer we need\n        to just read straight from the input buffer.\n        */\n        if (isWorkingBufferValid == MA_FALSE) {\n            pWorkingBuffer = (float*)pRunningFramesIn;  /* Naughty const cast, but it's safe at this point because we won't ever be writing to it from this point out. */\n        }\n\n        /* Spatialization. */\n        if (isSpatializationEnabled) {\n            ma_uint32 iListener;\n\n            /*\n            When determining the listener to use, we first check to see if the sound is pinned to a\n            specific listener. If so, we use that. Otherwise we just use the closest listener.\n            */\n            if (pEngineNode->pinnedListenerIndex != MA_LISTENER_INDEX_CLOSEST && pEngineNode->pinnedListenerIndex < ma_engine_get_listener_count(pEngineNode->pEngine)) {\n                iListener = pEngineNode->pinnedListenerIndex;\n            } else {\n                ma_vec3f spatializerPosition = ma_spatializer_get_position(&pEngineNode->spatializer);\n                iListener = ma_engine_find_closest_listener(pEngineNode->pEngine, spatializerPosition.x, spatializerPosition.y, spatializerPosition.z);\n            }\n\n            ma_spatializer_process_pcm_frames(&pEngineNode->spatializer, &pEngineNode->pEngine->listeners[iListener], pRunningFramesOut, pWorkingBuffer, framesJustProcessedOut);\n        } else {\n            /* No spatialization, but we still need to do channel conversion and master volume. */\n            float volume;\n            ma_engine_node_get_volume(pEngineNode, &volume);    /* Should never fail. */\n\n            if (channelsIn == channelsOut) {\n                /* No channel conversion required. Just copy straight to the output buffer. */\n                if (isVolumeSmoothingEnabled) {\n                    /* Volume has already been applied. Just copy straight to the output buffer. */\n                    ma_copy_pcm_frames(pRunningFramesOut, pWorkingBuffer, framesJustProcessedOut * channelsOut, ma_format_f32, channelsOut);\n                } else {\n                    /* Volume has not been applied yet. Copy and apply volume in the same pass. */\n                    ma_copy_and_apply_volume_factor_f32(pRunningFramesOut, pWorkingBuffer, framesJustProcessedOut * channelsOut, volume);\n                }\n            } else {\n                /* Channel conversion required. TODO: Add support for channel maps here. */\n                ma_channel_map_apply_f32(pRunningFramesOut, NULL, channelsOut, pWorkingBuffer, NULL, channelsIn, framesJustProcessedOut, ma_channel_mix_mode_simple, pEngineNode->monoExpansionMode);\n\n                /* If we're using smoothing, the volume will have already been applied. */\n                if (!isVolumeSmoothingEnabled) {\n                    ma_apply_volume_factor_f32(pRunningFramesOut, framesJustProcessedOut * channelsOut, volume);\n                }\n            }\n        }\n\n        /* At this point we can guarantee that the output buffer contains valid data. We can process everything in place now. */\n\n        /* Panning. */\n        if (isPanningEnabled) {\n            ma_panner_process_pcm_frames(&pEngineNode->panner, pRunningFramesOut, pRunningFramesOut, framesJustProcessedOut);   /* In-place processing. */\n        }\n\n        /* We're done for this chunk. */\n        totalFramesProcessedIn  += framesJustProcessedIn;\n        totalFramesProcessedOut += framesJustProcessedOut;\n\n        /* If we didn't process any output frames this iteration it means we've either run out of input data, or run out of room in the output buffer. */\n        if (framesJustProcessedOut == 0) {\n            break;\n        }\n    }\n\n    /* At this point we're done processing. */\n    *pFrameCountIn  = totalFramesProcessedIn;\n    *pFrameCountOut = totalFramesProcessedOut;\n}\n\nstatic void ma_engine_node_process_pcm_frames__sound(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    /* For sounds, we need to first read from the data source. Then we need to apply the engine effects (pan, pitch, fades, etc.). */\n    ma_result result = MA_SUCCESS;\n    ma_sound* pSound = (ma_sound*)pNode;\n    ma_uint32 frameCount = *pFrameCountOut;\n    ma_uint32 totalFramesRead = 0;\n    ma_format dataSourceFormat;\n    ma_uint32 dataSourceChannels;\n    ma_uint8 temp[MA_DATA_CONVERTER_STACK_BUFFER_SIZE];\n    ma_uint32 tempCapInFrames;\n    ma_uint64 seekTarget;\n\n    /* This is a data source node which means no input buses. */\n    (void)ppFramesIn;\n    (void)pFrameCountIn;\n\n    /* If we're marked at the end we need to stop the sound and do nothing. */\n    if (ma_sound_at_end(pSound)) {\n        ma_sound_stop(pSound);\n\n        if (pSound->endCallback != NULL) {\n            pSound->endCallback(pSound->pEndCallbackUserData, pSound);\n        }\n\n        *pFrameCountOut = 0;\n        return;\n    }\n\n    /* If we're seeking, do so now before reading. */\n    seekTarget = ma_atomic_load_64(&pSound->seekTarget);\n    if (seekTarget != MA_SEEK_TARGET_NONE) {\n        ma_data_source_seek_to_pcm_frame(pSound->pDataSource, seekTarget);\n\n        /* Any time-dependant effects need to have their times updated. */\n        ma_node_set_time(pSound, seekTarget);\n\n        ma_atomic_exchange_64(&pSound->seekTarget, MA_SEEK_TARGET_NONE);\n    }\n\n    /*\n    We want to update the pitch once. For sounds, this can be either at the start or at the end. If\n    we don't force this to only ever be updating once, we could end up in a situation where\n    retrieving the required input frame count ends up being different to what we actually retrieve.\n    What could happen is that the required input frame count is calculated, the pitch is update,\n    and then this processing function is called resulting in a different number of input frames\n    being processed. Do not call this in ma_engine_node_process_pcm_frames__general() or else\n    you'll hit the aforementioned bug.\n    */\n    ma_engine_node_update_pitch_if_required(&pSound->engineNode);\n\n    /*\n    For the convenience of the caller, we're doing to allow data sources to use non-floating-point formats and channel counts that differ\n    from the main engine.\n    */\n    result = ma_data_source_get_data_format(pSound->pDataSource, &dataSourceFormat, &dataSourceChannels, NULL, NULL, 0);\n    if (result == MA_SUCCESS) {\n        tempCapInFrames = sizeof(temp) / ma_get_bytes_per_frame(dataSourceFormat, dataSourceChannels);\n\n        /* Keep reading until we've read as much as was requested or we reach the end of the data source. */\n        while (totalFramesRead < frameCount) {\n            ma_uint32 framesRemaining = frameCount - totalFramesRead;\n            ma_uint64 framesJustRead;\n            ma_uint32 frameCountIn;\n            ma_uint32 frameCountOut;\n            const float* pRunningFramesIn;\n            float* pRunningFramesOut;\n\n            /* If there's any input frames sitting in the cache get those processed first. */\n            if (pSound->processingCacheFramesRemaining > 0) {\n                pRunningFramesIn = pSound->pProcessingCache;\n                frameCountIn = pSound->processingCacheFramesRemaining;\n\n                pRunningFramesOut = ma_offset_pcm_frames_ptr_f32(ppFramesOut[0], totalFramesRead, ma_node_get_output_channels(pNode, 0));\n                frameCountOut = framesRemaining;\n\n                ma_engine_node_process_pcm_frames__general(&pSound->engineNode, &pRunningFramesIn, &frameCountIn, &pRunningFramesOut, &frameCountOut);\n\n                MA_ASSERT(frameCountIn <= pSound->processingCacheFramesRemaining);\n                pSound->processingCacheFramesRemaining -= frameCountIn;\n\n                /* Move any remaining data in the cache down. */\n                if (pSound->processingCacheFramesRemaining > 0) {\n                    MA_MOVE_MEMORY(pSound->pProcessingCache, ma_offset_pcm_frames_ptr_f32(pSound->pProcessingCache, frameCountIn, dataSourceChannels), pSound->processingCacheFramesRemaining * ma_get_bytes_per_frame(ma_format_f32, dataSourceChannels));\n                }\n                \n                totalFramesRead += (ma_uint32)frameCountOut;   /* Safe cast. */\n\n                if (result != MA_SUCCESS || ma_sound_at_end(pSound)) {\n                    break;  /* Might have reached the end. */\n                }\n            } else {\n                /* Getting here means there's nothing in the cache. Read more data from the data source. */\n                if (dataSourceFormat == ma_format_f32) {\n                    /* Fast path. No conversion to f32 necessary. */\n                    result = ma_data_source_read_pcm_frames(pSound->pDataSource, pSound->pProcessingCache, pSound->processingCacheCap, &framesJustRead);\n                } else {\n                    /* Slow path. Need to convert to f32. */\n                    ma_uint64 totalFramesConverted = 0;\n\n                    while (totalFramesConverted < pSound->processingCacheCap) {\n                        ma_uint64 framesConverted;\n                        ma_uint32 framesToConvertThisIteration = pSound->processingCacheCap - (ma_uint32)totalFramesConverted;\n                        if (framesToConvertThisIteration > tempCapInFrames) {\n                            framesToConvertThisIteration = tempCapInFrames;\n                        }\n\n                        result = ma_data_source_read_pcm_frames(pSound->pDataSource, temp, framesToConvertThisIteration, &framesConverted);\n                        if (result != MA_SUCCESS) {\n                            break;\n                        }\n\n                        ma_convert_pcm_frames_format(ma_offset_pcm_frames_ptr_f32(pSound->pProcessingCache, totalFramesConverted, dataSourceChannels), ma_format_f32, temp, dataSourceFormat, framesConverted, dataSourceChannels, ma_dither_mode_none);\n                        totalFramesConverted += framesConverted;\n                    }\n\n                    framesJustRead = totalFramesConverted;\n                }\n\n                MA_ASSERT(framesJustRead <= pSound->processingCacheCap);\n                pSound->processingCacheFramesRemaining = (ma_uint32)framesJustRead;\n\n                /* If we reached the end of the sound we'll want to mark it as at the end and stop it. This should never be returned for looping sounds. */\n                if (result == MA_AT_END) {\n                    ma_sound_set_at_end(pSound, MA_TRUE);   /* This will be set to false in ma_sound_start(). */\n                }\n\n                if (result != MA_SUCCESS || ma_sound_at_end(pSound)) {\n                    break;\n                }\n            }\n        }\n    }\n\n    *pFrameCountOut = totalFramesRead;\n}\n\nstatic void ma_engine_node_process_pcm_frames__group(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn, float** ppFramesOut, ma_uint32* pFrameCountOut)\n{\n    /*\n    Make sure the pitch is updated before trying to read anything. It's important that this is done\n    only once and not in ma_engine_node_process_pcm_frames__general(). The reason for this is that\n    ma_engine_node_process_pcm_frames__general() will call ma_engine_node_get_required_input_frame_count(),\n    and if another thread modifies the pitch just after that call it can result in a glitch due to\n    the input rate changing.\n    */\n    ma_engine_node_update_pitch_if_required((ma_engine_node*)pNode);\n\n    /* For groups, the input data has already been read and we just need to apply the effect. */\n    ma_engine_node_process_pcm_frames__general((ma_engine_node*)pNode, ppFramesIn, pFrameCountIn, ppFramesOut, pFrameCountOut);\n}\n\n\nstatic ma_node_vtable g_ma_engine_node_vtable__sound =\n{\n    ma_engine_node_process_pcm_frames__sound,\n    NULL,   /* onGetRequiredInputFrameCount */\n    0,      /* Sounds are data source nodes which means they have zero inputs (their input is drawn from the data source itself). */\n    1,      /* Sounds have one output bus. */\n    0       /* Default flags. */\n};\n\nstatic ma_node_vtable g_ma_engine_node_vtable__group =\n{\n    ma_engine_node_process_pcm_frames__group,\n    NULL,   /* onGetRequiredInputFrameCount */\n    1,      /* Groups have one input bus. */\n    1,      /* Groups have one output bus. */\n    MA_NODE_FLAG_DIFFERENT_PROCESSING_RATES /* The engine node does resampling so should let miniaudio know about it. */\n};\n\n\n\nstatic ma_node_config ma_engine_node_base_node_config_init(const ma_engine_node_config* pConfig)\n{\n    ma_node_config baseNodeConfig;\n\n    if (pConfig->type == ma_engine_node_type_sound) {\n        /* Sound. */\n        baseNodeConfig = ma_node_config_init();\n        baseNodeConfig.vtable       = &g_ma_engine_node_vtable__sound;\n        baseNodeConfig.initialState = ma_node_state_stopped;    /* Sounds are stopped by default. */\n    } else {\n        /* Group. */\n        baseNodeConfig = ma_node_config_init();\n        baseNodeConfig.vtable       = &g_ma_engine_node_vtable__group;\n        baseNodeConfig.initialState = ma_node_state_started;    /* Groups are started by default. */\n    }\n\n    return baseNodeConfig;\n}\n\nstatic ma_spatializer_config ma_engine_node_spatializer_config_init(const ma_node_config* pBaseNodeConfig)\n{\n    return ma_spatializer_config_init(pBaseNodeConfig->pInputChannels[0], pBaseNodeConfig->pOutputChannels[0]);\n}\n\ntypedef struct\n{\n    size_t sizeInBytes;\n    size_t baseNodeOffset;\n    size_t resamplerOffset;\n    size_t spatializerOffset;\n    size_t gainerOffset;\n} ma_engine_node_heap_layout;\n\nstatic ma_result ma_engine_node_get_heap_layout(const ma_engine_node_config* pConfig, ma_engine_node_heap_layout* pHeapLayout)\n{\n    ma_result result;\n    size_t tempHeapSize;\n    ma_node_config baseNodeConfig;\n    ma_resampler_config resamplerConfig;\n    ma_spatializer_config spatializerConfig;\n    ma_gainer_config gainerConfig;\n    ma_uint32 sampleRate;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_channel defaultStereoChannelMap[2] = {MA_CHANNEL_SIDE_LEFT, MA_CHANNEL_SIDE_RIGHT};  /* <-- Consistent with the default channel map of a stereo listener. Means channel conversion can run on a fast path. */\n\n    MA_ASSERT(pHeapLayout);\n\n    MA_ZERO_OBJECT(pHeapLayout);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    if (pConfig->pEngine == NULL) {\n        return MA_INVALID_ARGS; /* An engine must be specified. */\n    }\n\n    pHeapLayout->sizeInBytes = 0;\n\n    sampleRate  = (pConfig->sampleRate   > 0) ? pConfig->sampleRate  : ma_engine_get_sample_rate(pConfig->pEngine);\n    channelsIn  = (pConfig->channelsIn  != 0) ? pConfig->channelsIn  : ma_engine_get_channels(pConfig->pEngine);\n    channelsOut = (pConfig->channelsOut != 0) ? pConfig->channelsOut : ma_engine_get_channels(pConfig->pEngine);\n\n\n    /* Base node. */\n    baseNodeConfig = ma_engine_node_base_node_config_init(pConfig);\n    baseNodeConfig.pInputChannels  = &channelsIn;\n    baseNodeConfig.pOutputChannels = &channelsOut;\n\n    result = ma_node_get_heap_size(ma_engine_get_node_graph(pConfig->pEngine), &baseNodeConfig, &tempHeapSize);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the size of the heap for the base node. */\n    }\n\n    pHeapLayout->baseNodeOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(tempHeapSize);\n\n\n    /* Resmapler. */\n    resamplerConfig = pConfig->resampling;\n    resamplerConfig.format        = ma_format_f32;\n    resamplerConfig.channels      = channelsIn;\n    resamplerConfig.sampleRateIn  = sampleRate;\n    resamplerConfig.sampleRateOut = ma_engine_get_sample_rate(pConfig->pEngine);\n\n    result = ma_resampler_get_heap_size(&resamplerConfig, &tempHeapSize);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the size of the heap for the resampler. */\n    }\n\n    pHeapLayout->resamplerOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(tempHeapSize);\n\n\n    /* Spatializer. */\n    spatializerConfig = ma_engine_node_spatializer_config_init(&baseNodeConfig);\n\n    if (spatializerConfig.channelsIn == 2) {\n        spatializerConfig.pChannelMapIn = defaultStereoChannelMap;\n    }\n\n    result = ma_spatializer_get_heap_size(&spatializerConfig, &tempHeapSize);\n    if (result != MA_SUCCESS) {\n        return result;  /* Failed to retrieve the size of the heap for the spatializer. */\n    }\n\n    pHeapLayout->spatializerOffset = pHeapLayout->sizeInBytes;\n    pHeapLayout->sizeInBytes += ma_align_64(tempHeapSize);\n\n\n    /* Gainer. Will not be used if we are not using smoothing. */\n    if (pConfig->volumeSmoothTimeInPCMFrames > 0) {\n        gainerConfig = ma_gainer_config_init(channelsIn, pConfig->volumeSmoothTimeInPCMFrames);\n\n        result = ma_gainer_get_heap_size(&gainerConfig, &tempHeapSize);\n        if (result != MA_SUCCESS) {\n            return result;\n        }\n\n        pHeapLayout->gainerOffset = pHeapLayout->sizeInBytes;\n        pHeapLayout->sizeInBytes += ma_align_64(tempHeapSize);\n    }\n\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_engine_node_get_heap_size(const ma_engine_node_config* pConfig, size_t* pHeapSizeInBytes)\n{\n    ma_result result;\n    ma_engine_node_heap_layout heapLayout;\n\n    if (pHeapSizeInBytes == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    *pHeapSizeInBytes = 0;\n\n    result = ma_engine_node_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    *pHeapSizeInBytes = heapLayout.sizeInBytes;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_engine_node_init_preallocated(const ma_engine_node_config* pConfig, void* pHeap, ma_engine_node* pEngineNode)\n{\n    ma_result result;\n    ma_engine_node_heap_layout heapLayout;\n    ma_node_config baseNodeConfig;\n    ma_resampler_config resamplerConfig;\n    ma_fader_config faderConfig;\n    ma_spatializer_config spatializerConfig;\n    ma_panner_config pannerConfig;\n    ma_gainer_config gainerConfig;\n    ma_uint32 channelsIn;\n    ma_uint32 channelsOut;\n    ma_channel defaultStereoChannelMap[2] = {MA_CHANNEL_SIDE_LEFT, MA_CHANNEL_SIDE_RIGHT};  /* <-- Consistent with the default channel map of a stereo listener. Means channel conversion can run on a fast path. */\n\n    if (pEngineNode == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pEngineNode);\n\n    result = ma_engine_node_get_heap_layout(pConfig, &heapLayout);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pConfig->pinnedListenerIndex != MA_LISTENER_INDEX_CLOSEST && pConfig->pinnedListenerIndex >= ma_engine_get_listener_count(pConfig->pEngine)) {\n        return MA_INVALID_ARGS; /* Invalid listener. */\n    }\n\n    pEngineNode->_pHeap = pHeap;\n    MA_ZERO_MEMORY(pHeap, heapLayout.sizeInBytes);\n\n    pEngineNode->pEngine                     = pConfig->pEngine;\n    pEngineNode->sampleRate                  = (pConfig->sampleRate > 0) ? pConfig->sampleRate : ma_engine_get_sample_rate(pEngineNode->pEngine);\n    pEngineNode->volumeSmoothTimeInPCMFrames = pConfig->volumeSmoothTimeInPCMFrames;\n    pEngineNode->monoExpansionMode           = pConfig->monoExpansionMode;\n    ma_atomic_float_set(&pEngineNode->volume, 1);\n    pEngineNode->pitch                       = 1;\n    pEngineNode->oldPitch                    = 1;\n    pEngineNode->oldDopplerPitch             = 1;\n    pEngineNode->isPitchDisabled             = pConfig->isPitchDisabled;\n    pEngineNode->isSpatializationDisabled    = pConfig->isSpatializationDisabled;\n    pEngineNode->pinnedListenerIndex         = pConfig->pinnedListenerIndex;\n    ma_atomic_float_set(&pEngineNode->fadeSettings.volumeBeg, 1);\n    ma_atomic_float_set(&pEngineNode->fadeSettings.volumeEnd, 1);\n    ma_atomic_uint64_set(&pEngineNode->fadeSettings.fadeLengthInFrames, (~(ma_uint64)0));\n    ma_atomic_uint64_set(&pEngineNode->fadeSettings.absoluteGlobalTimeInFrames, (~(ma_uint64)0));   /* <-- Indicates that the fade should start immediately. */\n\n    channelsIn  = (pConfig->channelsIn  != 0) ? pConfig->channelsIn  : ma_engine_get_channels(pConfig->pEngine);\n    channelsOut = (pConfig->channelsOut != 0) ? pConfig->channelsOut : ma_engine_get_channels(pConfig->pEngine);\n\n    /*\n    If the sample rate of the sound is different to the engine, make sure pitching is enabled so that the resampler\n    is activated. Not doing this will result in the sound not being resampled if MA_SOUND_FLAG_NO_PITCH is used.\n    */\n    if (pEngineNode->sampleRate != ma_engine_get_sample_rate(pEngineNode->pEngine)) {\n        pEngineNode->isPitchDisabled = MA_FALSE;\n    }\n\n\n    /* Base node. */\n    baseNodeConfig = ma_engine_node_base_node_config_init(pConfig);\n    baseNodeConfig.pInputChannels  = &channelsIn;\n    baseNodeConfig.pOutputChannels = &channelsOut;\n\n    result = ma_node_init_preallocated(&pConfig->pEngine->nodeGraph, &baseNodeConfig, ma_offset_ptr(pHeap, heapLayout.baseNodeOffset), &pEngineNode->baseNode);\n    if (result != MA_SUCCESS) {\n        goto error0;\n    }\n\n\n    /*\n    We can now initialize the effects we need in order to implement the engine node. There's a\n    defined order of operations here, mainly centered around when we convert our channels from the\n    data source's native channel count to the engine's channel count. As a rule, we want to do as\n    much computation as possible before spatialization because there's a chance that will increase\n    the channel count, thereby increasing the amount of work needing to be done to process.\n    */\n\n    /* We'll always do resampling first. */\n    resamplerConfig = pConfig->resampling;\n    resamplerConfig.format        = ma_format_f32;\n    resamplerConfig.channels      = baseNodeConfig.pInputChannels[0];\n    resamplerConfig.sampleRateIn  = pEngineNode->sampleRate;\n    resamplerConfig.sampleRateOut = ma_engine_get_sample_rate(pEngineNode->pEngine);\n\n    result = ma_resampler_init_preallocated(&resamplerConfig, ma_offset_ptr(pHeap, heapLayout.resamplerOffset), &pEngineNode->resampler);\n    if (result != MA_SUCCESS) {\n        goto error1;\n    }\n\n\n    /* After resampling will come the fader. */\n    faderConfig = ma_fader_config_init(ma_format_f32, baseNodeConfig.pInputChannels[0], ma_engine_get_sample_rate(pEngineNode->pEngine));\n\n    result = ma_fader_init(&faderConfig, &pEngineNode->fader);\n    if (result != MA_SUCCESS) {\n        goto error2;\n    }\n\n\n    /*\n    Spatialization comes next. We spatialize based on the node's output channel count. It's up the caller to\n    ensure channels counts link up correctly in the node graph.\n    */\n    spatializerConfig = ma_engine_node_spatializer_config_init(&baseNodeConfig);\n    spatializerConfig.gainSmoothTimeInFrames = pEngineNode->pEngine->gainSmoothTimeInFrames;\n\n    if (spatializerConfig.channelsIn == 2) {\n        spatializerConfig.pChannelMapIn = defaultStereoChannelMap;\n    }\n\n    result = ma_spatializer_init_preallocated(&spatializerConfig, ma_offset_ptr(pHeap, heapLayout.spatializerOffset), &pEngineNode->spatializer);\n    if (result != MA_SUCCESS) {\n        goto error2;\n    }\n\n\n    /*\n    After spatialization comes panning. We need to do this after spatialization because otherwise we wouldn't\n    be able to pan mono sounds.\n    */\n    pannerConfig = ma_panner_config_init(ma_format_f32, baseNodeConfig.pOutputChannels[0]);\n\n    result = ma_panner_init(&pannerConfig, &pEngineNode->panner);\n    if (result != MA_SUCCESS) {\n        goto error3;\n    }\n\n\n    /* We'll need a gainer for smoothing out volume changes if we have a non-zero smooth time. We apply this before converting to the output channel count. */\n    if (pConfig->volumeSmoothTimeInPCMFrames > 0) {\n        gainerConfig = ma_gainer_config_init(channelsIn, pConfig->volumeSmoothTimeInPCMFrames);\n\n        result = ma_gainer_init_preallocated(&gainerConfig, ma_offset_ptr(pHeap, heapLayout.gainerOffset), &pEngineNode->volumeGainer);\n        if (result != MA_SUCCESS) {\n            goto error3;\n        }\n    }\n\n\n    return MA_SUCCESS;\n\n    /* No need for allocation callbacks here because we use a preallocated heap. */\nerror3: ma_spatializer_uninit(&pEngineNode->spatializer, NULL);\nerror2: ma_resampler_uninit(&pEngineNode->resampler, NULL);\nerror1: ma_node_uninit(&pEngineNode->baseNode, NULL);\nerror0: return result;\n}\n\nMA_API ma_result ma_engine_node_init(const ma_engine_node_config* pConfig, const ma_allocation_callbacks* pAllocationCallbacks, ma_engine_node* pEngineNode)\n{\n    ma_result result;\n    size_t heapSizeInBytes;\n    void* pHeap;\n\n    result = ma_engine_node_get_heap_size(pConfig, &heapSizeInBytes);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (heapSizeInBytes > 0) {\n        pHeap = ma_malloc(heapSizeInBytes, pAllocationCallbacks);\n        if (pHeap == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n    } else {\n        pHeap = NULL;\n    }\n\n    result = ma_engine_node_init_preallocated(pConfig, pHeap, pEngineNode);\n    if (result != MA_SUCCESS) {\n        ma_free(pHeap, pAllocationCallbacks);\n        return result;\n    }\n\n    pEngineNode->_ownsHeap = MA_TRUE;\n    return MA_SUCCESS;\n}\n\nMA_API void ma_engine_node_uninit(ma_engine_node* pEngineNode, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    /*\n    The base node always needs to be uninitialized first to ensure it's detached from the graph completely before we\n    destroy anything that might be in the middle of being used by the processing function.\n    */\n    ma_node_uninit(&pEngineNode->baseNode, pAllocationCallbacks);\n\n    /* Now that the node has been uninitialized we can safely uninitialize the rest. */\n    if (pEngineNode->volumeSmoothTimeInPCMFrames > 0) {\n        ma_gainer_uninit(&pEngineNode->volumeGainer, pAllocationCallbacks);\n    }\n\n    ma_spatializer_uninit(&pEngineNode->spatializer, pAllocationCallbacks);\n    ma_resampler_uninit(&pEngineNode->resampler, pAllocationCallbacks);\n\n    /* Free the heap last. */\n    if (pEngineNode->_ownsHeap) {\n        ma_free(pEngineNode->_pHeap, pAllocationCallbacks);\n    }\n}\n\n\nMA_API ma_sound_config ma_sound_config_init(void)\n{\n    return ma_sound_config_init_2(NULL);\n}\n\nMA_API ma_sound_config ma_sound_config_init_2(ma_engine* pEngine)\n{\n    ma_sound_config config;\n\n    MA_ZERO_OBJECT(&config);\n\n    if (pEngine != NULL) {\n        config.monoExpansionMode = pEngine->monoExpansionMode;\n        config.pitchResampling = pEngine->pitchResamplingConfig;\n    } else {\n        config.monoExpansionMode = ma_mono_expansion_mode_default;\n\n        config.pitchResampling = ma_resampler_config_init(ma_format_f32, 0, 0, 0, ma_resample_algorithm_linear);\n        config.pitchResampling.linear.lpfOrder = 0; /* <-- Need to disable low-pass filtering for pitch shifting for now because there's cases where the biquads are becoming unstable. Need to figure out a better fix for this. */\n    }\n\n    config.rangeEndInPCMFrames     = ~((ma_uint64)0);\n    config.loopPointEndInPCMFrames = ~((ma_uint64)0);\n\n    return config;\n}\n\nMA_API ma_sound_group_config ma_sound_group_config_init(void)\n{\n    return ma_sound_group_config_init_2(NULL);\n}\n\nMA_API ma_sound_group_config ma_sound_group_config_init_2(ma_engine* pEngine)\n{\n    ma_sound_group_config config;\n\n    MA_ZERO_OBJECT(&config);\n\n    if (pEngine != NULL) {\n        config.monoExpansionMode = pEngine->monoExpansionMode;\n        config.pitchResampling   = pEngine->pitchResamplingConfig;\n    } else {\n        config.monoExpansionMode = ma_mono_expansion_mode_default;\n\n        config.pitchResampling = ma_resampler_config_init(ma_format_f32, 0, 0, 0, ma_resample_algorithm_linear);\n        config.pitchResampling.linear.lpfOrder = 0; /* <-- Need to disable low-pass filtering for pitch shifting for now because there's cases where the biquads are becoming unstable. Need to figure out a better fix for this. */\n    }\n\n    return config;\n}\n\n\nMA_API ma_engine_config ma_engine_config_init(void)\n{\n    ma_engine_config config;\n\n    MA_ZERO_OBJECT(&config);\n    config.listenerCount             = 1;   /* Always want at least one listener. */\n    config.monoExpansionMode         = ma_mono_expansion_mode_default;\n    config.resourceManagerResampling = ma_resampler_config_init(ma_format_unknown, 0, 0, 0, ma_resample_algorithm_linear);\n\n    config.pitchResampling = ma_resampler_config_init(ma_format_f32, 0, 0, 0, ma_resample_algorithm_linear);\n    config.pitchResampling.linear.lpfOrder = 0; /* <-- Need to disable low-pass filtering for pitch shifting for now because there's cases where the biquads are becoming unstable. Need to figure out a better fix for this. */\n\n    return config;\n}\n\n\n#if !defined(MA_NO_DEVICE_IO)\nstatic void ma_engine_data_callback_internal(ma_device* pDevice, void* pFramesOut, const void* pFramesIn, ma_uint32 frameCount)\n{\n    ma_engine* pEngine = (ma_engine*)pDevice->pUserData;\n\n    (void)pFramesIn;\n\n    /*\n    Experiment: Try processing a resource manager job if we're on the Emscripten build.\n\n    This serves two purposes:\n\n        1) It ensures jobs are actually processed at some point since we cannot guarantee that the\n           caller is doing the right thing and calling ma_resource_manager_process_next_job(); and\n\n        2) It's an attempt at working around an issue where processing jobs on the Emscripten main\n           loop doesn't work as well as it should. When trying to load sounds without the `DECODE`\n           flag or with the `ASYNC` flag, the sound data is just not able to be loaded in time\n           before the callback is processed. I think it's got something to do with the single-\n           threaded nature of Web, but I'm not entirely sure.\n    */\n    #if !defined(MA_NO_RESOURCE_MANAGER) && defined(MA_EMSCRIPTEN)\n    {\n        if (pEngine->pResourceManager != NULL) {\n            if ((pEngine->pResourceManager->config.flags & MA_RESOURCE_MANAGER_FLAG_NO_THREADING) != 0) {\n                ma_resource_manager_process_next_job(pEngine->pResourceManager);\n            }\n        }\n    }\n    #endif\n\n    ma_engine_read_pcm_frames(pEngine, pFramesOut, frameCount, NULL);\n}\n\nstatic ma_uint32 ma_device__get_processing_size_in_frames(ma_device* pDevice)\n{\n    /*\n    The processing size is the period size. The device can have a fixed sized processing size, or\n    it can be decided by the backend in which case it can be variable.\n    */\n    if (pDevice->playback.intermediaryBufferCap > 0) {\n        /* Using a fixed sized processing callback. */\n        return pDevice->playback.intermediaryBufferCap;\n    } else {\n        /* Not using a fixed sized processing callback. Need to estimate the processing size based on the backend. */\n        return pDevice->playback.internalPeriodSizeInFrames;\n    }\n}\n#endif\n\nMA_API ma_result ma_engine_init(const ma_engine_config* pConfig, ma_engine* pEngine)\n{\n    ma_result result;\n    ma_node_graph_config nodeGraphConfig;\n    ma_engine_config engineConfig;\n    ma_spatializer_listener_config listenerConfig;\n    ma_uint32 iListener;\n\n    if (pEngine == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pEngine);\n\n    /* The config is allowed to be NULL in which case we use defaults for everything. */\n    if (pConfig != NULL) {\n        engineConfig = *pConfig;\n    } else {\n        engineConfig = ma_engine_config_init();\n    }\n\n    pEngine->monoExpansionMode = engineConfig.monoExpansionMode;\n    pEngine->defaultVolumeSmoothTimeInPCMFrames = engineConfig.defaultVolumeSmoothTimeInPCMFrames;\n    pEngine->onProcess = engineConfig.onProcess;\n    pEngine->pProcessUserData = engineConfig.pProcessUserData;\n    pEngine->pitchResamplingConfig = engineConfig.pitchResampling;\n    ma_allocation_callbacks_init_copy(&pEngine->allocationCallbacks, &engineConfig.allocationCallbacks);\n\n    #if !defined(MA_NO_RESOURCE_MANAGER)\n    {\n        pEngine->pResourceManager = engineConfig.pResourceManager;\n    }\n    #endif\n\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        pEngine->pDevice = engineConfig.pDevice;\n\n        /* If we don't have a device, we need one. */\n        if (pEngine->pDevice == NULL && engineConfig.noDevice == MA_FALSE) {\n            ma_device_config deviceConfig;\n\n            pEngine->pDevice = (ma_device*)ma_malloc(sizeof(*pEngine->pDevice), &pEngine->allocationCallbacks);\n            if (pEngine->pDevice == NULL) {\n                return MA_OUT_OF_MEMORY;\n            }\n\n            deviceConfig = ma_device_config_init(ma_device_type_playback);\n            deviceConfig.playback.pDeviceID        = engineConfig.pPlaybackDeviceID;\n            deviceConfig.playback.format           = ma_format_f32;\n            deviceConfig.playback.channels         = engineConfig.channels;\n            deviceConfig.sampleRate                = engineConfig.sampleRate;\n            deviceConfig.dataCallback              = (engineConfig.dataCallback != NULL) ? engineConfig.dataCallback : ma_engine_data_callback_internal;\n            deviceConfig.pUserData                 = pEngine;\n            deviceConfig.notificationCallback      = engineConfig.notificationCallback;\n            deviceConfig.periodSizeInFrames        = engineConfig.periodSizeInFrames;\n            deviceConfig.periodSizeInMilliseconds  = engineConfig.periodSizeInMilliseconds;\n            deviceConfig.noPreSilencedOutputBuffer = MA_TRUE;    /* We'll always be outputting to every frame in the callback so there's no need for a pre-silenced buffer. */\n            deviceConfig.noClip                    = MA_TRUE;    /* The engine will do clipping itself. */\n\n            if (engineConfig.pContext == NULL) {\n                ma_context_config contextConfig = ma_context_config_init();\n                contextConfig.allocationCallbacks = pEngine->allocationCallbacks;\n                contextConfig.pLog = engineConfig.pLog;\n\n                /* If the engine config does not specify a log, use the resource manager's if we have one. */\n                #ifndef MA_NO_RESOURCE_MANAGER\n                {\n                    if (contextConfig.pLog == NULL && engineConfig.pResourceManager != NULL) {\n                        contextConfig.pLog = ma_resource_manager_get_log(engineConfig.pResourceManager);\n                    }\n                }\n                #endif\n\n                result = ma_device_init_ex(NULL, 0, &contextConfig, &deviceConfig, pEngine->pDevice);\n            } else {\n                result = ma_device_init(engineConfig.pContext, &deviceConfig, pEngine->pDevice);\n            }\n\n            if (result != MA_SUCCESS) {\n                ma_free(pEngine->pDevice, &pEngine->allocationCallbacks);\n                pEngine->pDevice = NULL;\n                return result;\n            }\n\n            pEngine->ownsDevice = MA_TRUE;\n        }\n\n        /* Update the channel count and sample rate of the engine config so we can reference it below. */\n        if (pEngine->pDevice != NULL) {\n            engineConfig.channels   = pEngine->pDevice->playback.channels;\n            engineConfig.sampleRate = pEngine->pDevice->sampleRate;\n\n            /*\n            The processing size used by the engine is determined by engineConfig.periodSizeInFrames. We want\n            to make this equal to what the device is using for it's period size. If we don't do that, it's\n            possible that the node graph will split it's processing into multiple passes which can introduce\n            glitching.\n            */\n            engineConfig.periodSizeInFrames = ma_device__get_processing_size_in_frames(pEngine->pDevice);\n        }\n    }\n    #endif\n\n    if (engineConfig.channels == 0 || engineConfig.sampleRate == 0) {\n        return MA_INVALID_ARGS;\n    }\n\n    pEngine->sampleRate = engineConfig.sampleRate;\n\n    /* The engine always uses either the log that was passed into the config, or the context's log is available. */\n    if (engineConfig.pLog != NULL) {\n        pEngine->pLog = engineConfig.pLog;\n    } else {\n        #if !defined(MA_NO_DEVICE_IO)\n        {\n            pEngine->pLog = ma_device_get_log(pEngine->pDevice);\n        }\n        #else\n        {\n            pEngine->pLog = NULL;\n        }\n        #endif\n    }\n\n\n    /* The engine is a node graph. This needs to be initialized after we have the device so we can determine the channel count. */\n    nodeGraphConfig = ma_node_graph_config_init(engineConfig.channels);\n    nodeGraphConfig.processingSizeInFrames = engineConfig.periodSizeInFrames;\n    nodeGraphConfig.preMixStackSizeInBytes = engineConfig.preMixStackSizeInBytes;\n\n    result = ma_node_graph_init(&nodeGraphConfig, &pEngine->allocationCallbacks, &pEngine->nodeGraph);\n    if (result != MA_SUCCESS) {\n        goto on_error_1;\n    }\n\n\n    /* We need at least one listener. */\n    if (engineConfig.listenerCount == 0) {\n        engineConfig.listenerCount = 1;\n    }\n\n    if (engineConfig.listenerCount > MA_ENGINE_MAX_LISTENERS) {\n        result = MA_INVALID_ARGS;   /* Too many listeners. */\n        goto on_error_1;\n    }\n\n    for (iListener = 0; iListener < engineConfig.listenerCount; iListener += 1) {\n        listenerConfig = ma_spatializer_listener_config_init(ma_node_graph_get_channels(&pEngine->nodeGraph));\n\n        /*\n        If we're using a device, use the device's channel map for the listener. Otherwise just use\n        miniaudio's default channel map.\n        */\n        #if !defined(MA_NO_DEVICE_IO)\n        {\n            if (pEngine->pDevice != NULL) {\n                /*\n                Temporarily disabled. There is a subtle bug here where front-left and front-right\n                will be used by the device's channel map, but this is not what we want to use for\n                spatialization. Instead we want to use side-left and side-right. I need to figure\n                out a better solution for this. For now, disabling the use of device channel maps.\n                */\n                /*listenerConfig.pChannelMapOut = pEngine->pDevice->playback.channelMap;*/\n            }\n        }\n        #endif\n\n        result = ma_spatializer_listener_init(&listenerConfig, &pEngine->allocationCallbacks, &pEngine->listeners[iListener]);  /* TODO: Change this to a pre-allocated heap. */\n        if (result != MA_SUCCESS) {\n            goto on_error_2;\n        }\n\n        pEngine->listenerCount += 1;\n    }\n\n\n    /* Gain smoothing for spatialized sounds. */\n    pEngine->gainSmoothTimeInFrames = engineConfig.gainSmoothTimeInFrames;\n    if (pEngine->gainSmoothTimeInFrames == 0) {\n        ma_uint32 gainSmoothTimeInMilliseconds = engineConfig.gainSmoothTimeInMilliseconds;\n        if (gainSmoothTimeInMilliseconds == 0) {\n            gainSmoothTimeInMilliseconds = 8;\n        }\n\n        pEngine->gainSmoothTimeInFrames = (gainSmoothTimeInMilliseconds * ma_engine_get_sample_rate(pEngine)) / 1000;  /* 8ms by default. */\n    }\n\n\n    /* We need a resource manager. */\n    #ifndef MA_NO_RESOURCE_MANAGER\n    {\n        if (pEngine->pResourceManager == NULL) {\n            ma_resource_manager_config resourceManagerConfig;\n\n            pEngine->pResourceManager = (ma_resource_manager*)ma_malloc(sizeof(*pEngine->pResourceManager), &pEngine->allocationCallbacks);\n            if (pEngine->pResourceManager == NULL) {\n                result = MA_OUT_OF_MEMORY;\n                goto on_error_2;\n            }\n\n            resourceManagerConfig = ma_resource_manager_config_init();\n            resourceManagerConfig.pLog              = pEngine->pLog;    /* Always use the engine's log for internally-managed resource managers. */\n            resourceManagerConfig.decodedFormat     = ma_format_f32;\n            resourceManagerConfig.decodedChannels   = 0;  /* Leave the decoded channel count as 0 so we can get good spatialization. */\n            resourceManagerConfig.decodedSampleRate = ma_engine_get_sample_rate(pEngine);\n            ma_allocation_callbacks_init_copy(&resourceManagerConfig.allocationCallbacks, &pEngine->allocationCallbacks);\n            resourceManagerConfig.pVFS              = engineConfig.pResourceManagerVFS;\n            resourceManagerConfig.resampling        = engineConfig.resourceManagerResampling;\n\n            /* The Emscripten build cannot use threads unless it's targeting pthreads. */\n            #if defined(MA_EMSCRIPTEN) && !defined(__EMSCRIPTEN_PTHREADS__)\n            {\n                resourceManagerConfig.jobThreadCount = 0;\n                resourceManagerConfig.flags |= MA_RESOURCE_MANAGER_FLAG_NO_THREADING;\n            }\n            #endif\n\n            result = ma_resource_manager_init(&resourceManagerConfig, pEngine->pResourceManager);\n            if (result != MA_SUCCESS) {\n                goto on_error_3;\n            }\n\n            pEngine->ownsResourceManager = MA_TRUE;\n        }\n    }\n    #endif\n\n    /* Setup some stuff for inlined sounds. That is sounds played with ma_engine_play_sound(). */\n    pEngine->inlinedSoundLock  = 0;\n    pEngine->pInlinedSoundHead = NULL;\n\n    /* Start the engine if required. This should always be the last step. */\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        if (engineConfig.noAutoStart == MA_FALSE && pEngine->pDevice != NULL) {\n            result = ma_engine_start(pEngine);\n            if (result != MA_SUCCESS) {\n                goto on_error_4;    /* Failed to start the engine. */\n            }\n        }\n    }\n    #endif\n\n    return MA_SUCCESS;\n\n#if !defined(MA_NO_DEVICE_IO)\non_error_4:\n#endif\n#if !defined(MA_NO_RESOURCE_MANAGER)\non_error_3:\n    if (pEngine->ownsResourceManager) {\n        ma_free(pEngine->pResourceManager, &pEngine->allocationCallbacks);\n    }\n#endif  /* MA_NO_RESOURCE_MANAGER */\non_error_2:\n    for (iListener = 0; iListener < pEngine->listenerCount; iListener += 1) {\n        ma_spatializer_listener_uninit(&pEngine->listeners[iListener], &pEngine->allocationCallbacks);\n    }\n\n    ma_node_graph_uninit(&pEngine->nodeGraph, &pEngine->allocationCallbacks);\non_error_1:\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        if (pEngine->ownsDevice) {\n            ma_device_uninit(pEngine->pDevice);\n            ma_free(pEngine->pDevice, &pEngine->allocationCallbacks);\n        }\n    }\n    #endif\n\n    return result;\n}\n\nMA_API void ma_engine_uninit(ma_engine* pEngine)\n{\n    ma_uint32 iListener;\n\n    if (pEngine == NULL) {\n        return;\n    }\n\n    /* The device must be uninitialized before the node graph to ensure the audio thread doesn't try accessing it. */\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        if (pEngine->ownsDevice) {\n            ma_device_uninit(pEngine->pDevice);\n            ma_free(pEngine->pDevice, &pEngine->allocationCallbacks);\n        } else {\n            if (pEngine->pDevice != NULL) {\n                ma_device_stop(pEngine->pDevice);\n            }\n        }\n    }\n    #endif\n\n    /*\n    All inlined sounds need to be deleted. I'm going to use a lock here just to future proof in case\n    I want to do some kind of garbage collection later on.\n    */\n    ma_spinlock_lock(&pEngine->inlinedSoundLock);\n    {\n        for (;;) {\n            ma_sound_inlined* pSoundToDelete = pEngine->pInlinedSoundHead;\n            if (pSoundToDelete == NULL) {\n                break;  /* Done. */\n            }\n\n            pEngine->pInlinedSoundHead = pSoundToDelete->pNext;\n\n            ma_sound_uninit(&pSoundToDelete->sound);\n            ma_free(pSoundToDelete, &pEngine->allocationCallbacks);\n        }\n    }\n    ma_spinlock_unlock(&pEngine->inlinedSoundLock);\n\n    for (iListener = 0; iListener < pEngine->listenerCount; iListener += 1) {\n        ma_spatializer_listener_uninit(&pEngine->listeners[iListener], &pEngine->allocationCallbacks);\n    }\n\n    /* Make sure the node graph is uninitialized after the audio thread has been shutdown to prevent accessing of the node graph after being uninitialized. */\n    ma_node_graph_uninit(&pEngine->nodeGraph, &pEngine->allocationCallbacks);\n\n    /* Uninitialize the resource manager last to ensure we don't have a thread still trying to access it. */\n#ifndef MA_NO_RESOURCE_MANAGER\n    if (pEngine->ownsResourceManager) {\n        ma_resource_manager_uninit(pEngine->pResourceManager);\n        ma_free(pEngine->pResourceManager, &pEngine->allocationCallbacks);\n    }\n#endif\n}\n\nMA_API ma_result ma_engine_read_pcm_frames(ma_engine* pEngine, void* pFramesOut, ma_uint64 frameCount, ma_uint64* pFramesRead)\n{\n    ma_result result;\n    ma_uint64 framesRead = 0;\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = 0;\n    }\n\n    result = ma_node_graph_read_pcm_frames(&pEngine->nodeGraph, pFramesOut, frameCount, &framesRead);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pFramesRead != NULL) {\n        *pFramesRead = framesRead;\n    }\n\n    if (pEngine->onProcess) {\n        pEngine->onProcess(pEngine->pProcessUserData, (float*)pFramesOut, framesRead);  /* Safe cast to float* because the engine always works on floating point samples. */\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_node_graph* ma_engine_get_node_graph(ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return NULL;\n    }\n\n    return &pEngine->nodeGraph;\n}\n\n#if !defined(MA_NO_RESOURCE_MANAGER)\nMA_API ma_resource_manager* ma_engine_get_resource_manager(ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return NULL;\n    }\n\n    #if !defined(MA_NO_RESOURCE_MANAGER)\n    {\n        return pEngine->pResourceManager;\n    }\n    #else\n    {\n        return NULL;\n    }\n    #endif\n}\n#endif\n\nMA_API ma_device* ma_engine_get_device(ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return NULL;\n    }\n\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        return pEngine->pDevice;\n    }\n    #else\n    {\n        return NULL;\n    }\n    #endif\n}\n\nMA_API ma_log* ma_engine_get_log(ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return NULL;\n    }\n\n    if (pEngine->pLog != NULL) {\n        return pEngine->pLog;\n    } else {\n        #if !defined(MA_NO_DEVICE_IO)\n        {\n            return ma_device_get_log(ma_engine_get_device(pEngine));\n        }\n        #else\n        {\n            return NULL;\n        }\n        #endif\n    }\n}\n\nMA_API ma_node* ma_engine_get_endpoint(ma_engine* pEngine)\n{\n    return ma_node_graph_get_endpoint(&pEngine->nodeGraph);\n}\n\nMA_API ma_uint64 ma_engine_get_time_in_pcm_frames(const ma_engine* pEngine)\n{\n    return ma_node_graph_get_time(&pEngine->nodeGraph);\n}\n\nMA_API ma_uint64 ma_engine_get_time_in_milliseconds(const ma_engine* pEngine)\n{\n    return ma_engine_get_time_in_pcm_frames(pEngine) * 1000 / ma_engine_get_sample_rate(pEngine);\n}\n\nMA_API ma_result ma_engine_set_time_in_pcm_frames(ma_engine* pEngine, ma_uint64 globalTime)\n{\n    return ma_node_graph_set_time(&pEngine->nodeGraph, globalTime);\n}\n\nMA_API ma_result ma_engine_set_time_in_milliseconds(ma_engine* pEngine, ma_uint64 globalTime)\n{\n    return ma_engine_set_time_in_pcm_frames(pEngine, globalTime * ma_engine_get_sample_rate(pEngine) / 1000);\n}\n\nMA_API ma_uint64 ma_engine_get_time(const ma_engine* pEngine)\n{\n    return ma_engine_get_time_in_pcm_frames(pEngine);\n}\n\nMA_API ma_result ma_engine_set_time(ma_engine* pEngine, ma_uint64 globalTime)\n{\n    return ma_engine_set_time_in_pcm_frames(pEngine, globalTime);\n}\n\nMA_API ma_uint32 ma_engine_get_channels(const ma_engine* pEngine)\n{\n    return ma_node_graph_get_channels(&pEngine->nodeGraph);\n}\n\nMA_API ma_uint32 ma_engine_get_sample_rate(const ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return 0;\n    }\n\n    return pEngine->sampleRate;\n}\n\n\nMA_API ma_result ma_engine_start(ma_engine* pEngine)\n{\n    ma_result result;\n\n    if (pEngine == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        if (pEngine->pDevice != NULL) {\n            result = ma_device_start(pEngine->pDevice);\n        } else {\n            result = MA_INVALID_OPERATION;  /* The engine is running without a device which means there's no real notion of \"starting\" the engine. */\n        }\n    }\n    #else\n    {\n        result = MA_INVALID_OPERATION;  /* Device IO is disabled, so there's no real notion of \"starting\" the engine. */\n    }\n    #endif\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_engine_stop(ma_engine* pEngine)\n{\n    ma_result result;\n\n    if (pEngine == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    #if !defined(MA_NO_DEVICE_IO)\n    {\n        if (pEngine->pDevice != NULL) {\n            result = ma_device_stop(pEngine->pDevice);\n        } else {\n            result = MA_INVALID_OPERATION;  /* The engine is running without a device which means there's no real notion of \"stopping\" the engine. */\n        }\n    }\n    #else\n    {\n        result = MA_INVALID_OPERATION;  /* Device IO is disabled, so there's no real notion of \"stopping\" the engine. */\n    }\n    #endif\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_engine_set_volume(ma_engine* pEngine, float volume)\n{\n    if (pEngine == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_node_set_output_bus_volume(ma_node_graph_get_endpoint(&pEngine->nodeGraph), 0, volume);\n}\n\nMA_API float ma_engine_get_volume(ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return 0;\n    }\n\n    return ma_node_get_output_bus_volume(ma_node_graph_get_endpoint(&pEngine->nodeGraph), 0);\n}\n\nMA_API ma_result ma_engine_set_gain_db(ma_engine* pEngine, float gainDB)\n{\n    return ma_engine_set_volume(pEngine, ma_volume_db_to_linear(gainDB));\n}\n\nMA_API float ma_engine_get_gain_db(ma_engine* pEngine)\n{\n    return ma_volume_linear_to_db(ma_engine_get_volume(pEngine));\n}\n\n\nMA_API ma_uint32 ma_engine_get_listener_count(const ma_engine* pEngine)\n{\n    if (pEngine == NULL) {\n        return 0;\n    }\n\n    return pEngine->listenerCount;\n}\n\nMA_API ma_uint32 ma_engine_find_closest_listener(const ma_engine* pEngine, float absolutePosX, float absolutePosY, float absolutePosZ)\n{\n    ma_uint32 iListener;\n    ma_uint32 iListenerClosest;\n    float closestLen2 = MA_FLT_MAX;\n\n    if (pEngine == NULL || pEngine->listenerCount == 1) {\n        return 0;\n    }\n\n    iListenerClosest = 0;\n    for (iListener = 0; iListener < pEngine->listenerCount; iListener += 1) {\n        if (ma_engine_listener_is_enabled(pEngine, iListener)) {\n            float len2 = ma_vec3f_len2(ma_vec3f_sub(ma_spatializer_listener_get_position(&pEngine->listeners[iListener]), ma_vec3f_init_3f(absolutePosX, absolutePosY, absolutePosZ)));\n            if (closestLen2 > len2) {\n                closestLen2 = len2;\n                iListenerClosest = iListener;\n            }\n        }\n    }\n\n    MA_ASSERT(iListenerClosest < 255);\n    return iListenerClosest;\n}\n\nMA_API void ma_engine_listener_set_position(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_position(&pEngine->listeners[listenerIndex], x, y, z);\n}\n\nMA_API ma_vec3f ma_engine_listener_get_position(const ma_engine* pEngine, ma_uint32 listenerIndex)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_spatializer_listener_get_position(&pEngine->listeners[listenerIndex]);\n}\n\nMA_API void ma_engine_listener_set_direction(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_direction(&pEngine->listeners[listenerIndex], x, y, z);\n}\n\nMA_API ma_vec3f ma_engine_listener_get_direction(const ma_engine* pEngine, ma_uint32 listenerIndex)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    }\n\n    return ma_spatializer_listener_get_direction(&pEngine->listeners[listenerIndex]);\n}\n\nMA_API void ma_engine_listener_set_velocity(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_velocity(&pEngine->listeners[listenerIndex], x, y, z);\n}\n\nMA_API ma_vec3f ma_engine_listener_get_velocity(const ma_engine* pEngine, ma_uint32 listenerIndex)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_spatializer_listener_get_velocity(&pEngine->listeners[listenerIndex]);\n}\n\nMA_API void ma_engine_listener_set_cone(ma_engine* pEngine, ma_uint32 listenerIndex, float innerAngleInRadians, float outerAngleInRadians, float outerGain)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_cone(&pEngine->listeners[listenerIndex], innerAngleInRadians, outerAngleInRadians, outerGain);\n}\n\nMA_API void ma_engine_listener_get_cone(const ma_engine* pEngine, ma_uint32 listenerIndex, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain)\n{\n    if (pInnerAngleInRadians != NULL) {\n        *pInnerAngleInRadians = 0;\n    }\n\n    if (pOuterAngleInRadians != NULL) {\n        *pOuterAngleInRadians = 0;\n    }\n\n    if (pOuterGain != NULL) {\n        *pOuterGain = 0;\n    }\n\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_get_cone(&pEngine->listeners[listenerIndex], pInnerAngleInRadians, pOuterAngleInRadians, pOuterGain);\n}\n\nMA_API void ma_engine_listener_set_world_up(ma_engine* pEngine, ma_uint32 listenerIndex, float x, float y, float z)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_world_up(&pEngine->listeners[listenerIndex], x, y, z);\n}\n\nMA_API ma_vec3f ma_engine_listener_get_world_up(const ma_engine* pEngine, ma_uint32 listenerIndex)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return ma_vec3f_init_3f(0, 1, 0);\n    }\n\n    return ma_spatializer_listener_get_world_up(&pEngine->listeners[listenerIndex]);\n}\n\nMA_API void ma_engine_listener_set_enabled(ma_engine* pEngine, ma_uint32 listenerIndex, ma_bool32 isEnabled)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return;\n    }\n\n    ma_spatializer_listener_set_enabled(&pEngine->listeners[listenerIndex], isEnabled);\n}\n\nMA_API ma_bool32 ma_engine_listener_is_enabled(const ma_engine* pEngine, ma_uint32 listenerIndex)\n{\n    if (pEngine == NULL || listenerIndex >= pEngine->listenerCount) {\n        return MA_FALSE;\n    }\n\n    return ma_spatializer_listener_is_enabled(&pEngine->listeners[listenerIndex]);\n}\n\n\n#ifndef MA_NO_RESOURCE_MANAGER\nMA_API ma_result ma_engine_play_sound_ex(ma_engine* pEngine, const char* pFilePath, ma_node* pNode, ma_uint32 nodeInputBusIndex)\n{\n    ma_result result = MA_SUCCESS;\n    ma_sound_inlined* pSound = NULL;\n    ma_sound_inlined* pNextSound = NULL;\n\n    if (pEngine == NULL || pFilePath == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Attach to the endpoint node if nothing is specified. */\n    if (pNode == NULL) {\n        pNode = ma_node_graph_get_endpoint(&pEngine->nodeGraph);\n        nodeInputBusIndex = 0;\n    }\n\n    /*\n    We want to check if we can recycle an already-allocated inlined sound. Since this is just a\n    helper I'm not *too* concerned about performance here and I'm happy to use a lock to keep\n    the implementation simple. Maybe this can be optimized later if there's enough demand, but\n    if this function is being used it probably means the caller doesn't really care too much.\n\n    What we do is check the atEnd flag. When this is true, we can recycle the sound. Otherwise\n    we just keep iterating. If we reach the end without finding a sound to recycle we just\n    allocate a new one. This doesn't scale well for a massive number of sounds being played\n    simultaneously as we don't ever actually free the sound objects. Some kind of garbage\n    collection routine might be valuable for this which I'll think about.\n    */\n    ma_spinlock_lock(&pEngine->inlinedSoundLock);\n    {\n        ma_uint32 soundFlags = 0;\n\n        for (pNextSound = pEngine->pInlinedSoundHead; pNextSound != NULL; pNextSound = pNextSound->pNext) {\n            if (ma_sound_at_end(&pNextSound->sound)) {\n                /*\n                The sound is at the end which means it's available for recycling. All we need to do\n                is uninitialize it and reinitialize it. All we're doing is recycling memory.\n                */\n                pSound = pNextSound;\n                ma_atomic_fetch_sub_32(&pEngine->inlinedSoundCount, 1);\n                break;\n            }\n        }\n\n        if (pSound != NULL) {\n            /*\n            We actually want to detach the sound from the list here. The reason is because we want the sound\n            to be in a consistent state at the non-recycled case to simplify the logic below.\n            */\n            if (pEngine->pInlinedSoundHead == pSound) {\n                pEngine->pInlinedSoundHead =  pSound->pNext;\n            }\n\n            if (pSound->pPrev != NULL) {\n                pSound->pPrev->pNext = pSound->pNext;\n            }\n            if (pSound->pNext != NULL) {\n                pSound->pNext->pPrev = pSound->pPrev;\n            }\n\n            /* Now the previous sound needs to be uninitialized. */\n            ma_sound_uninit(&pNextSound->sound);\n        } else {\n            /* No sound available for recycling. Allocate one now. */\n            pSound = (ma_sound_inlined*)ma_malloc(sizeof(*pSound), &pEngine->allocationCallbacks);\n        }\n\n        if (pSound != NULL) {   /* Safety check for the allocation above. */\n            /*\n            At this point we should have memory allocated for the inlined sound. We just need\n            to initialize it like a normal sound now.\n            */\n            soundFlags |= MA_SOUND_FLAG_ASYNC;                 /* For inlined sounds we don't want to be sitting around waiting for stuff to load so force an async load. */\n            soundFlags |= MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT; /* We want specific control over where the sound is attached in the graph. We'll attach it manually just before playing the sound. */\n            soundFlags |= MA_SOUND_FLAG_NO_PITCH;              /* Pitching isn't usable with inlined sounds, so disable it to save on speed. */\n            soundFlags |= MA_SOUND_FLAG_NO_SPATIALIZATION;     /* Not currently doing spatialization with inlined sounds, but this might actually change later. For now disable spatialization. Will be removed if we ever add support for spatialization here. */\n\n            result = ma_sound_init_from_file(pEngine, pFilePath, soundFlags, NULL, NULL, &pSound->sound);\n            if (result == MA_SUCCESS) {\n                /* Now attach the sound to the graph. */\n                result = ma_node_attach_output_bus(pSound, 0, pNode, nodeInputBusIndex);\n                if (result == MA_SUCCESS) {\n                    /* At this point the sound should be loaded and we can go ahead and add it to the list. The new item becomes the new head. */\n                    pSound->pNext = pEngine->pInlinedSoundHead;\n                    pSound->pPrev = NULL;\n\n                    pEngine->pInlinedSoundHead = pSound;    /* <-- This is what attaches the sound to the list. */\n                    if (pSound->pNext != NULL) {\n                        pSound->pNext->pPrev = pSound;\n                    }\n                } else {\n                    ma_free(pSound, &pEngine->allocationCallbacks);\n                }\n            } else {\n                ma_free(pSound, &pEngine->allocationCallbacks);\n            }\n        } else {\n            result = MA_OUT_OF_MEMORY;\n        }\n    }\n    ma_spinlock_unlock(&pEngine->inlinedSoundLock);\n\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* Finally we can start playing the sound. */\n    result = ma_sound_start(&pSound->sound);\n    if (result != MA_SUCCESS) {\n        /* Failed to start the sound. We need to mark it for recycling and return an error. */\n        ma_atomic_exchange_32(&pSound->sound.atEnd, MA_TRUE);\n        return result;\n    }\n\n    ma_atomic_fetch_add_32(&pEngine->inlinedSoundCount, 1);\n    return result;\n}\n\nMA_API ma_result ma_engine_play_sound(ma_engine* pEngine, const char* pFilePath, ma_sound_group* pGroup)\n{\n    return ma_engine_play_sound_ex(pEngine, pFilePath, pGroup, 0);\n}\n#endif\n\n\nstatic ma_result ma_sound_preinit(ma_engine* pEngine, ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pSound);\n    pSound->seekTarget = MA_SEEK_TARGET_NONE;\n\n    if (pEngine == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return MA_SUCCESS;\n}\n\nstatic ma_result ma_sound_init_from_data_source_internal(ma_engine* pEngine, const ma_sound_config* pConfig, ma_sound* pSound)\n{\n    ma_result result;\n    ma_engine_node_config engineNodeConfig;\n    ma_engine_node_type type;   /* Will be set to ma_engine_node_type_group if no data source is specified. */\n\n    /* Do not clear pSound to zero here - that's done at a higher level with ma_sound_preinit(). */\n    MA_ASSERT(pEngine != NULL);\n    MA_ASSERT(pSound  != NULL);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pSound->pDataSource = pConfig->pDataSource;\n\n    if (pConfig->pDataSource != NULL) {\n        type = ma_engine_node_type_sound;\n    } else {\n        type = ma_engine_node_type_group;\n    }\n\n    /*\n    Sounds are engine nodes. Before we can initialize this we need to determine the channel count.\n    If we can't do this we need to abort. It's up to the caller to ensure they're using a data\n    source that provides this information upfront.\n    */\n    engineNodeConfig = ma_engine_node_config_init(pEngine, type, pConfig->flags);\n    engineNodeConfig.channelsIn                  = pConfig->channelsIn;\n    engineNodeConfig.channelsOut                 = pConfig->channelsOut;\n    engineNodeConfig.volumeSmoothTimeInPCMFrames = pConfig->volumeSmoothTimeInPCMFrames;\n    engineNodeConfig.monoExpansionMode           = pConfig->monoExpansionMode;\n\n    if (engineNodeConfig.volumeSmoothTimeInPCMFrames == 0) {\n        engineNodeConfig.volumeSmoothTimeInPCMFrames = pEngine->defaultVolumeSmoothTimeInPCMFrames;\n    }\n\n    /* If we're loading from a data source the input channel count needs to be the data source's native channel count. */\n    if (pConfig->pDataSource != NULL) {\n        result = ma_data_source_get_data_format(pConfig->pDataSource, NULL, &engineNodeConfig.channelsIn, &engineNodeConfig.sampleRate, NULL, 0);\n        if (result != MA_SUCCESS) {\n            return result;  /* Failed to retrieve the channel count. */\n        }\n\n        if (engineNodeConfig.channelsIn == 0) {\n            return MA_INVALID_OPERATION;    /* Invalid channel count. */\n        }\n\n        if (engineNodeConfig.channelsOut == MA_SOUND_SOURCE_CHANNEL_COUNT) {\n            engineNodeConfig.channelsOut = engineNodeConfig.channelsIn;\n        }\n    }\n\n\n    /* Getting here means we should have a valid channel count and we can initialize the engine node. */\n    result = ma_engine_node_init(&engineNodeConfig, &pEngine->allocationCallbacks, &pSound->engineNode);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* If no attachment is specified, attach the sound straight to the endpoint. */\n    if (pConfig->pInitialAttachment == NULL) {\n        /* No group. Attach straight to the endpoint by default, unless the caller has requested that it not. */\n        if ((pConfig->flags & MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT) == 0) {\n            result = ma_node_attach_output_bus(pSound, 0, ma_node_graph_get_endpoint(&pEngine->nodeGraph), 0);\n        }\n    } else {\n        /* An attachment is specified. Attach to it by default. The sound has only a single output bus, and the config will specify which input bus to attach to. */\n        result = ma_node_attach_output_bus(pSound, 0, pConfig->pInitialAttachment, pConfig->initialAttachmentInputBusIndex);\n    }\n\n    if (result != MA_SUCCESS) {\n        ma_engine_node_uninit(&pSound->engineNode, &pEngine->allocationCallbacks);\n        return result;\n    }\n\n\n    /*\n    When pulling data from a data source we need a processing cache to hold onto unprocessed input data from the data source\n    after doing resampling.\n    */\n    if (pSound->pDataSource != NULL) {\n        pSound->processingCacheFramesRemaining = 0;\n        pSound->processingCacheCap = ma_node_graph_get_processing_size_in_frames(&pEngine->nodeGraph);\n        if (pSound->processingCacheCap == 0) {\n            pSound->processingCacheCap = 512;\n        }\n        \n        pSound->pProcessingCache = (float*)ma_calloc(pSound->processingCacheCap * ma_get_bytes_per_frame(ma_format_f32, engineNodeConfig.channelsIn), &pEngine->allocationCallbacks);\n        if (pSound->pProcessingCache == NULL) {\n            ma_engine_node_uninit(&pSound->engineNode, &pEngine->allocationCallbacks);\n            return MA_OUT_OF_MEMORY;\n        }\n    }\n\n\n    /* Apply initial range and looping state to the data source if applicable. */\n    if (pConfig->rangeBegInPCMFrames != 0 || pConfig->rangeEndInPCMFrames != ~((ma_uint64)0)) {\n        ma_data_source_set_range_in_pcm_frames(ma_sound_get_data_source(pSound), pConfig->rangeBegInPCMFrames, pConfig->rangeEndInPCMFrames);\n    }\n\n    if (pConfig->loopPointBegInPCMFrames != 0 || pConfig->loopPointEndInPCMFrames != ~((ma_uint64)0)) {\n        ma_data_source_set_loop_point_in_pcm_frames(ma_sound_get_data_source(pSound), pConfig->loopPointBegInPCMFrames, pConfig->loopPointEndInPCMFrames);\n    }\n\n    ma_sound_set_looping(pSound, pConfig->isLooping || ((pConfig->flags & MA_SOUND_FLAG_LOOPING) != 0));\n\n    return MA_SUCCESS;\n}\n\n#ifndef MA_NO_RESOURCE_MANAGER\nMA_API ma_result ma_sound_init_from_file_internal(ma_engine* pEngine, const ma_sound_config* pConfig, ma_sound* pSound)\n{\n    ma_result result = MA_SUCCESS;\n    ma_uint32 flags;\n    ma_sound_config config;\n    ma_resource_manager_pipeline_notifications notifications;\n\n    /*\n    The engine requires knowledge of the channel count of the underlying data source before it can\n    initialize the sound. Therefore, we need to make the resource manager wait until initialization\n    of the underlying data source to be initialized so we can get access to the channel count. To\n    do this, the MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT is forced.\n\n    Because we're initializing the data source before the sound, there's a chance the notification\n    will get triggered before this function returns. This is OK, so long as the caller is aware of\n    it and can avoid accessing the sound from within the notification.\n    */\n    flags = pConfig->flags | MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_WAIT_INIT;\n    if (pConfig->isLooping) {\n        flags |= MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING;\n    }\n\n    pSound->pResourceManagerDataSource = (ma_resource_manager_data_source*)ma_malloc(sizeof(*pSound->pResourceManagerDataSource), &pEngine->allocationCallbacks);\n    if (pSound->pResourceManagerDataSource == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    /* Removed in 0.12. Set pDoneFence on the notifications. */\n    notifications = pConfig->initNotifications;\n    if (pConfig->pDoneFence != NULL && notifications.done.pFence == NULL) {\n        notifications.done.pFence = pConfig->pDoneFence;\n    }\n\n    /*\n    We must wrap everything around the fence if one was specified. This ensures ma_fence_wait() does\n    not return prematurely before the sound has finished initializing.\n    */\n    if (notifications.done.pFence) { ma_fence_acquire(notifications.done.pFence); }\n    {\n        ma_resource_manager_data_source_config resourceManagerDataSourceConfig = ma_resource_manager_data_source_config_init();\n        resourceManagerDataSourceConfig.pFilePath                   = pConfig->pFilePath;\n        resourceManagerDataSourceConfig.pFilePathW                  = pConfig->pFilePathW;\n        resourceManagerDataSourceConfig.flags                       = flags;\n        resourceManagerDataSourceConfig.pNotifications              = &notifications;\n        resourceManagerDataSourceConfig.initialSeekPointInPCMFrames = pConfig->initialSeekPointInPCMFrames;\n        resourceManagerDataSourceConfig.rangeBegInPCMFrames         = pConfig->rangeBegInPCMFrames;\n        resourceManagerDataSourceConfig.rangeEndInPCMFrames         = pConfig->rangeEndInPCMFrames;\n        resourceManagerDataSourceConfig.loopPointBegInPCMFrames     = pConfig->loopPointBegInPCMFrames;\n        resourceManagerDataSourceConfig.loopPointEndInPCMFrames     = pConfig->loopPointEndInPCMFrames;\n        resourceManagerDataSourceConfig.isLooping                   = (flags & MA_RESOURCE_MANAGER_DATA_SOURCE_FLAG_LOOPING) != 0;\n\n        result = ma_resource_manager_data_source_init_ex(pEngine->pResourceManager, &resourceManagerDataSourceConfig, pSound->pResourceManagerDataSource);\n        if (result != MA_SUCCESS) {\n            ma_free(pSound->pResourceManagerDataSource, &pEngine->allocationCallbacks);\n            goto done;\n        }\n\n        pSound->ownsDataSource = MA_TRUE;   /* <-- Important. Not setting this will result in the resource manager data source never getting uninitialized. */\n\n        /* We need to use a slightly customized version of the config so we'll need to make a copy. */\n        config = *pConfig;\n        config.pFilePath   = NULL;\n        config.pFilePathW  = NULL;\n        config.pDataSource = pSound->pResourceManagerDataSource;\n\n        result = ma_sound_init_from_data_source_internal(pEngine, &config, pSound);\n        if (result != MA_SUCCESS) {\n            ma_resource_manager_data_source_uninit(pSound->pResourceManagerDataSource);\n            ma_free(pSound->pResourceManagerDataSource, &pEngine->allocationCallbacks);\n            MA_ZERO_OBJECT(pSound);\n            goto done;\n        }\n    }\ndone:\n    if (notifications.done.pFence) { ma_fence_release(notifications.done.pFence); }\n    return result;\n}\n\nMA_API ma_result ma_sound_init_from_file(ma_engine* pEngine, const char* pFilePath, ma_uint32 flags, ma_sound_group* pGroup, ma_fence* pDoneFence, ma_sound* pSound)\n{\n    ma_sound_config config;\n\n    if (pFilePath == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    config = ma_sound_config_init_2(pEngine);\n    config.pFilePath          = pFilePath;\n    config.flags              = flags;\n    config.pInitialAttachment = pGroup;\n    config.pDoneFence         = pDoneFence;\n\n    return ma_sound_init_ex(pEngine, &config, pSound);\n}\n\nMA_API ma_result ma_sound_init_from_file_w(ma_engine* pEngine, const wchar_t* pFilePath, ma_uint32 flags, ma_sound_group* pGroup, ma_fence* pDoneFence, ma_sound* pSound)\n{\n    ma_sound_config config;\n\n    if (pFilePath == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    config = ma_sound_config_init_2(pEngine);\n    config.pFilePathW         = pFilePath;\n    config.flags              = flags;\n    config.pInitialAttachment = pGroup;\n    config.pDoneFence         = pDoneFence;\n\n    return ma_sound_init_ex(pEngine, &config, pSound);\n}\n\nMA_API ma_result ma_sound_init_copy(ma_engine* pEngine, const ma_sound* pExistingSound, ma_uint32 flags, ma_sound_group* pGroup, ma_sound* pSound)\n{\n    ma_result result;\n    ma_sound_config config;\n\n    result = ma_sound_preinit(pEngine, pSound);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pExistingSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Cloning only works for data buffers (not streams) that are loaded from the resource manager. */\n    if (pExistingSound->pResourceManagerDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /*\n    We need to make a clone of the data source. If the data source is not a data buffer (i.e. a stream)\n    this will fail.\n    */\n    pSound->pResourceManagerDataSource = (ma_resource_manager_data_source*)ma_malloc(sizeof(*pSound->pResourceManagerDataSource), &pEngine->allocationCallbacks);\n    if (pSound->pResourceManagerDataSource == NULL) {\n        return MA_OUT_OF_MEMORY;\n    }\n\n    result = ma_resource_manager_data_source_init_copy(pEngine->pResourceManager, pExistingSound->pResourceManagerDataSource, pSound->pResourceManagerDataSource);\n    if (result != MA_SUCCESS) {\n        ma_free(pSound->pResourceManagerDataSource, &pEngine->allocationCallbacks);\n        return result;\n    }\n\n    config = ma_sound_config_init_2(pEngine);\n    config.pDataSource                 = pSound->pResourceManagerDataSource;\n    config.flags                       = flags;\n    config.pInitialAttachment          = pGroup;\n    config.monoExpansionMode           = pExistingSound->engineNode.monoExpansionMode;\n    config.volumeSmoothTimeInPCMFrames = pExistingSound->engineNode.volumeSmoothTimeInPCMFrames;\n\n    result = ma_sound_init_from_data_source_internal(pEngine, &config, pSound);\n    if (result != MA_SUCCESS) {\n        ma_resource_manager_data_source_uninit(pSound->pResourceManagerDataSource);\n        ma_free(pSound->pResourceManagerDataSource, &pEngine->allocationCallbacks);\n        MA_ZERO_OBJECT(pSound);\n        return result;\n    }\n\n    /* Make sure the sound is marked as the owner of the data source or else it will never get uninitialized. */\n    pSound->ownsDataSource = MA_TRUE;\n\n    return MA_SUCCESS;\n}\n#endif\n\nMA_API ma_result ma_sound_init_from_data_source(ma_engine* pEngine, ma_data_source* pDataSource, ma_uint32 flags, ma_sound_group* pGroup, ma_sound* pSound)\n{\n    ma_sound_config config = ma_sound_config_init_2(pEngine);\n    config.pDataSource        = pDataSource;\n    config.flags              = flags;\n    config.pInitialAttachment = pGroup;\n    return ma_sound_init_ex(pEngine, &config, pSound);\n}\n\nMA_API ma_result ma_sound_init_ex(ma_engine* pEngine, const ma_sound_config* pConfig, ma_sound* pSound)\n{\n    ma_result result;\n\n    result = ma_sound_preinit(pEngine, pSound);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    pSound->endCallback          = pConfig->endCallback;\n    pSound->pEndCallbackUserData = pConfig->pEndCallbackUserData;\n\n    /* We need to load the sound differently depending on whether or not we're loading from a file. */\n#ifndef MA_NO_RESOURCE_MANAGER\n    if (pConfig->pFilePath != NULL || pConfig->pFilePathW != NULL) {\n        return ma_sound_init_from_file_internal(pEngine, pConfig, pSound);\n    } else\n#endif\n    {\n        /*\n        Getting here means we're not loading from a file. We may be loading from an already-initialized\n        data source, or none at all. If we aren't specifying any data source, we'll be initializing\n        the equivalent to a group. ma_data_source_init_from_data_source_internal() will deal with this\n        for us, so no special treatment required here.\n        */\n        return ma_sound_init_from_data_source_internal(pEngine, pConfig, pSound);\n    }\n}\n\nMA_API void ma_sound_uninit(ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    /*\n    Always uninitialize the node first. This ensures it's detached from the graph and does not return until it has done\n    so which makes thread safety beyond this point trivial.\n    */\n    ma_engine_node_uninit(&pSound->engineNode, &pSound->engineNode.pEngine->allocationCallbacks);\n\n    if (pSound->pProcessingCache != NULL) {\n        ma_free(pSound->pProcessingCache, &pSound->engineNode.pEngine->allocationCallbacks);\n        pSound->pProcessingCache = NULL;\n    }\n\n    /* Once the sound is detached from the group we can guarantee that it won't be referenced by the mixer thread which means it's safe for us to destroy the data source. */\n#ifndef MA_NO_RESOURCE_MANAGER\n    if (pSound->ownsDataSource) {\n        ma_resource_manager_data_source_uninit(pSound->pResourceManagerDataSource);\n        ma_free(pSound->pResourceManagerDataSource, &pSound->engineNode.pEngine->allocationCallbacks);\n        pSound->pDataSource = NULL;\n    }\n#else\n    MA_ASSERT(pSound->ownsDataSource == MA_FALSE);\n#endif\n}\n\nMA_API ma_engine* ma_sound_get_engine(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return NULL;\n    }\n\n    return pSound->engineNode.pEngine;\n}\n\nMA_API ma_data_source* ma_sound_get_data_source(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return NULL;\n    }\n\n    return pSound->pDataSource;\n}\n\nMA_API ma_result ma_sound_start(ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* If the sound is already playing, do nothing. */\n    if (ma_sound_is_playing(pSound)) {\n        return MA_SUCCESS;\n    }\n\n    /* If the sound is at the end it means we want to start from the start again. */\n    if (ma_sound_at_end(pSound)) {\n        ma_result result = ma_data_source_seek_to_pcm_frame(pSound->pDataSource, 0);\n        if (result != MA_SUCCESS && result != MA_NOT_IMPLEMENTED) {\n            return result;  /* Failed to seek back to the start. */\n        }\n\n        /* Make sure we clear the end indicator. */\n        ma_atomic_exchange_32(&pSound->atEnd, MA_FALSE);\n    }\n\n    /* Make sure the sound is started. If there's a start delay, the sound won't actually start until the start time is reached. */\n    ma_node_set_state(pSound, ma_node_state_started);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_sound_stop(ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* This will stop the sound immediately. Use ma_sound_set_stop_time() to stop the sound at a specific time. */\n    ma_node_set_state(pSound, ma_node_state_stopped);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_sound_stop_with_fade_in_pcm_frames(ma_sound* pSound, ma_uint64 fadeLengthInFrames)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Stopping with a fade out requires us to schedule the stop into the future by the fade length. */\n    ma_sound_set_stop_time_with_fade_in_pcm_frames(pSound, ma_engine_get_time_in_pcm_frames(ma_sound_get_engine(pSound)) + fadeLengthInFrames, fadeLengthInFrames);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_sound_stop_with_fade_in_milliseconds(ma_sound* pSound, ma_uint64 fadeLengthInMilliseconds)\n{\n    ma_uint64 sampleRate;\n\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    sampleRate = ma_engine_get_sample_rate(ma_sound_get_engine(pSound));\n\n    return ma_sound_stop_with_fade_in_pcm_frames(pSound, (fadeLengthInMilliseconds * sampleRate) / 1000);\n}\n\nMA_API void ma_sound_reset_start_time(ma_sound* pSound)\n{\n    ma_sound_set_start_time_in_pcm_frames(pSound, 0);\n}\n\nMA_API void ma_sound_reset_stop_time(ma_sound* pSound)\n{\n    ma_sound_set_stop_time_in_pcm_frames(pSound, ~(ma_uint64)0);\n}\n\nMA_API void ma_sound_reset_fade(ma_sound* pSound)\n{\n    ma_sound_set_fade_in_pcm_frames(pSound, 0, 1, 0);\n}\n\nMA_API void ma_sound_reset_stop_time_and_fade(ma_sound* pSound)\n{\n    ma_sound_reset_stop_time(pSound);\n    ma_sound_reset_fade(pSound);\n}\n\nMA_API void ma_sound_set_volume(ma_sound* pSound, float volume)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_engine_node_set_volume(&pSound->engineNode, volume);\n}\n\nMA_API float ma_sound_get_volume(const ma_sound* pSound)\n{\n    float volume = 0;\n\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    ma_engine_node_get_volume(&pSound->engineNode, &volume);\n\n    return volume;\n}\n\nMA_API void ma_sound_set_pan(ma_sound* pSound, float pan)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_panner_set_pan(&pSound->engineNode.panner, pan);\n}\n\nMA_API float ma_sound_get_pan(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_panner_get_pan(&pSound->engineNode.panner);\n}\n\nMA_API void ma_sound_set_pan_mode(ma_sound* pSound, ma_pan_mode panMode)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_panner_set_mode(&pSound->engineNode.panner, panMode);\n}\n\nMA_API ma_pan_mode ma_sound_get_pan_mode(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_pan_mode_balance;\n    }\n\n    return ma_panner_get_mode(&pSound->engineNode.panner);\n}\n\nMA_API void ma_sound_set_pitch(ma_sound* pSound, float pitch)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    if (pitch <= 0) {\n        return;\n    }\n\n    ma_atomic_exchange_explicit_f32(&pSound->engineNode.pitch, pitch, ma_atomic_memory_order_release);\n}\n\nMA_API float ma_sound_get_pitch(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_atomic_load_f32(&pSound->engineNode.pitch);    /* Naughty const-cast for this. */\n}\n\nMA_API void ma_sound_set_spatialization_enabled(ma_sound* pSound, ma_bool32 enabled)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_atomic_exchange_explicit_32(&pSound->engineNode.isSpatializationDisabled, !enabled, ma_atomic_memory_order_release);\n}\n\nMA_API ma_bool32 ma_sound_is_spatialization_enabled(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_engine_node_is_spatialization_enabled(&pSound->engineNode);\n}\n\nMA_API void ma_sound_set_pinned_listener_index(ma_sound* pSound, ma_uint32 listenerIndex)\n{\n    if (pSound == NULL || listenerIndex >= ma_engine_get_listener_count(ma_sound_get_engine(pSound))) {\n        return;\n    }\n\n    ma_atomic_exchange_explicit_32(&pSound->engineNode.pinnedListenerIndex, listenerIndex, ma_atomic_memory_order_release);\n}\n\nMA_API ma_uint32 ma_sound_get_pinned_listener_index(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_LISTENER_INDEX_CLOSEST;\n    }\n\n    return ma_atomic_load_explicit_32(&pSound->engineNode.pinnedListenerIndex, ma_atomic_memory_order_acquire);\n}\n\nMA_API ma_uint32 ma_sound_get_listener_index(const ma_sound* pSound)\n{\n    ma_uint32 listenerIndex;\n\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    listenerIndex = ma_sound_get_pinned_listener_index(pSound);\n    if (listenerIndex == MA_LISTENER_INDEX_CLOSEST) {\n        ma_vec3f position = ma_sound_get_position(pSound);\n        return ma_engine_find_closest_listener(ma_sound_get_engine(pSound), position.x, position.y, position.z);\n    }\n\n    return listenerIndex;\n}\n\nMA_API ma_vec3f ma_sound_get_direction_to_listener(const ma_sound* pSound)\n{\n    ma_vec3f relativePos;\n    ma_engine* pEngine;\n\n    if (pSound == NULL) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    }\n\n    pEngine = ma_sound_get_engine(pSound);\n    if (pEngine == NULL) {\n        return ma_vec3f_init_3f(0, 0, -1);\n    }\n\n    ma_spatializer_get_relative_position_and_direction(&pSound->engineNode.spatializer, &pEngine->listeners[ma_sound_get_listener_index(pSound)], &relativePos, NULL);\n\n    return ma_vec3f_normalize(ma_vec3f_neg(relativePos));\n}\n\nMA_API void ma_sound_set_position(ma_sound* pSound, float x, float y, float z)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_position(&pSound->engineNode.spatializer, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_get_position(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_spatializer_get_position(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_direction(ma_sound* pSound, float x, float y, float z)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_direction(&pSound->engineNode.spatializer, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_get_direction(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_spatializer_get_direction(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_velocity(ma_sound* pSound, float x, float y, float z)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_velocity(&pSound->engineNode.spatializer, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_get_velocity(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_vec3f_init_3f(0, 0, 0);\n    }\n\n    return ma_spatializer_get_velocity(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_attenuation_model(ma_sound* pSound, ma_attenuation_model attenuationModel)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_attenuation_model(&pSound->engineNode.spatializer, attenuationModel);\n}\n\nMA_API ma_attenuation_model ma_sound_get_attenuation_model(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_attenuation_model_none;\n    }\n\n    return ma_spatializer_get_attenuation_model(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_positioning(ma_sound* pSound, ma_positioning positioning)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_positioning(&pSound->engineNode.spatializer, positioning);\n}\n\nMA_API ma_positioning ma_sound_get_positioning(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return ma_positioning_absolute;\n    }\n\n    return ma_spatializer_get_positioning(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_rolloff(ma_sound* pSound, float rolloff)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_rolloff(&pSound->engineNode.spatializer, rolloff);\n}\n\nMA_API float ma_sound_get_rolloff(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_rolloff(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_min_gain(ma_sound* pSound, float minGain)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_min_gain(&pSound->engineNode.spatializer, minGain);\n}\n\nMA_API float ma_sound_get_min_gain(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_min_gain(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_max_gain(ma_sound* pSound, float maxGain)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_max_gain(&pSound->engineNode.spatializer, maxGain);\n}\n\nMA_API float ma_sound_get_max_gain(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_max_gain(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_min_distance(ma_sound* pSound, float minDistance)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_min_distance(&pSound->engineNode.spatializer, minDistance);\n}\n\nMA_API float ma_sound_get_min_distance(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_min_distance(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_max_distance(ma_sound* pSound, float maxDistance)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_max_distance(&pSound->engineNode.spatializer, maxDistance);\n}\n\nMA_API float ma_sound_get_max_distance(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_max_distance(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_cone(ma_sound* pSound, float innerAngleInRadians, float outerAngleInRadians, float outerGain)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_cone(&pSound->engineNode.spatializer, innerAngleInRadians, outerAngleInRadians, outerGain);\n}\n\nMA_API void ma_sound_get_cone(const ma_sound* pSound, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain)\n{\n    if (pInnerAngleInRadians != NULL) {\n        *pInnerAngleInRadians = 0;\n    }\n\n    if (pOuterAngleInRadians != NULL) {\n        *pOuterAngleInRadians = 0;\n    }\n\n    if (pOuterGain != NULL) {\n        *pOuterGain = 0;\n    }\n\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_get_cone(&pSound->engineNode.spatializer, pInnerAngleInRadians, pOuterAngleInRadians, pOuterGain);\n}\n\nMA_API void ma_sound_set_doppler_factor(ma_sound* pSound, float dopplerFactor)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_doppler_factor(&pSound->engineNode.spatializer, dopplerFactor);\n}\n\nMA_API float ma_sound_get_doppler_factor(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_spatializer_get_doppler_factor(&pSound->engineNode.spatializer);\n}\n\nMA_API void ma_sound_set_directional_attenuation_factor(ma_sound* pSound, float directionalAttenuationFactor)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_spatializer_set_directional_attenuation_factor(&pSound->engineNode.spatializer, directionalAttenuationFactor);\n}\n\nMA_API float ma_sound_get_directional_attenuation_factor(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 1;\n    }\n\n    return ma_spatializer_get_directional_attenuation_factor(&pSound->engineNode.spatializer);\n}\n\n\nMA_API void ma_sound_set_fade_in_pcm_frames(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_sound_set_fade_start_in_pcm_frames(pSound, volumeBeg, volumeEnd, fadeLengthInFrames, (~(ma_uint64)0));\n}\n\nMA_API void ma_sound_set_fade_in_milliseconds(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_sound_set_fade_in_pcm_frames(pSound, volumeBeg, volumeEnd, (fadeLengthInMilliseconds * pSound->engineNode.fader.config.sampleRate) / 1000);\n}\n\nMA_API void ma_sound_set_fade_start_in_pcm_frames(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames, ma_uint64 absoluteGlobalTimeInFrames)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    /*\n    We don't want to update the fader at this point because we need to use the engine's current time\n    to derive the fader's start offset. The timer is being updated on the audio thread so in order to\n    do this as accurately as possible we'll need to defer this to the audio thread.\n    */\n    ma_atomic_float_set(&pSound->engineNode.fadeSettings.volumeBeg, volumeBeg);\n    ma_atomic_float_set(&pSound->engineNode.fadeSettings.volumeEnd, volumeEnd);\n    ma_atomic_uint64_set(&pSound->engineNode.fadeSettings.fadeLengthInFrames, fadeLengthInFrames);\n    ma_atomic_uint64_set(&pSound->engineNode.fadeSettings.absoluteGlobalTimeInFrames, absoluteGlobalTimeInFrames);\n}\n\nMA_API void ma_sound_set_fade_start_in_milliseconds(ma_sound* pSound, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds, ma_uint64 absoluteGlobalTimeInMilliseconds)\n{\n    ma_uint32 sampleRate;\n\n    if (pSound == NULL) {\n        return;\n    }\n\n    sampleRate = ma_engine_get_sample_rate(ma_sound_get_engine(pSound));\n\n    ma_sound_set_fade_start_in_pcm_frames(pSound, volumeBeg, volumeEnd, (fadeLengthInMilliseconds * sampleRate) / 1000, (absoluteGlobalTimeInMilliseconds * sampleRate) / 1000);\n}\n\nMA_API float ma_sound_get_current_fade_volume(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    return ma_fader_get_current_volume(&pSound->engineNode.fader);\n}\n\nMA_API void ma_sound_set_start_time_in_pcm_frames(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInFrames)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_node_set_state_time(pSound, ma_node_state_started, absoluteGlobalTimeInFrames);\n}\n\nMA_API void ma_sound_set_start_time_in_milliseconds(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInMilliseconds)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_sound_set_start_time_in_pcm_frames(pSound, absoluteGlobalTimeInMilliseconds * ma_engine_get_sample_rate(ma_sound_get_engine(pSound)) / 1000);\n}\n\nMA_API void ma_sound_set_stop_time_in_pcm_frames(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInFrames)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_sound_set_stop_time_with_fade_in_pcm_frames(pSound, absoluteGlobalTimeInFrames, 0);\n}\n\nMA_API void ma_sound_set_stop_time_in_milliseconds(ma_sound* pSound, ma_uint64 absoluteGlobalTimeInMilliseconds)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    ma_sound_set_stop_time_in_pcm_frames(pSound, absoluteGlobalTimeInMilliseconds * ma_engine_get_sample_rate(ma_sound_get_engine(pSound)) / 1000);\n}\n\nMA_API void ma_sound_set_stop_time_with_fade_in_pcm_frames(ma_sound* pSound, ma_uint64 stopAbsoluteGlobalTimeInFrames, ma_uint64 fadeLengthInFrames)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    if (fadeLengthInFrames > 0) {\n        if (fadeLengthInFrames > stopAbsoluteGlobalTimeInFrames) {\n            fadeLengthInFrames = stopAbsoluteGlobalTimeInFrames;\n        }\n\n        ma_sound_set_fade_start_in_pcm_frames(pSound, -1, 0, fadeLengthInFrames, stopAbsoluteGlobalTimeInFrames - fadeLengthInFrames);\n    }\n\n    ma_node_set_state_time(pSound, ma_node_state_stopped, stopAbsoluteGlobalTimeInFrames);\n}\n\nMA_API void ma_sound_set_stop_time_with_fade_in_milliseconds(ma_sound* pSound, ma_uint64 stopAbsoluteGlobalTimeInMilliseconds, ma_uint64 fadeLengthInMilliseconds)\n{\n    ma_uint32 sampleRate;\n\n    if (pSound == NULL) {\n        return;\n    }\n\n    sampleRate = ma_engine_get_sample_rate(ma_sound_get_engine(pSound));\n\n    ma_sound_set_stop_time_with_fade_in_pcm_frames(pSound, (stopAbsoluteGlobalTimeInMilliseconds * sampleRate) / 1000, (fadeLengthInMilliseconds * sampleRate) / 1000);\n}\n\nMA_API ma_bool32 ma_sound_is_playing(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_node_get_state_by_time(pSound, ma_engine_get_time_in_pcm_frames(ma_sound_get_engine(pSound))) == ma_node_state_started;\n}\n\nMA_API ma_uint64 ma_sound_get_time_in_pcm_frames(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return 0;\n    }\n\n    return ma_node_get_time(pSound);\n}\n\nMA_API ma_uint64 ma_sound_get_time_in_milliseconds(const ma_sound* pSound)\n{\n    ma_uint32 sampleRate = ma_engine_get_sample_rate(ma_sound_get_engine(pSound));\n    if (sampleRate == 0) {\n        return 0;   /* Prevent a division by zero. */\n    }\n\n    return ma_sound_get_time_in_pcm_frames(pSound) * 1000 / sampleRate;\n}\n\nMA_API void ma_sound_set_looping(ma_sound* pSound, ma_bool32 isLooping)\n{\n    if (pSound == NULL) {\n        return;\n    }\n\n    /* Looping is only a valid concept if the sound is backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return;\n    }\n\n    /* The looping state needs to be applied to the data source in order for any looping to actually happen. */\n    ma_data_source_set_looping(pSound->pDataSource, isLooping);\n}\n\nMA_API ma_bool32 ma_sound_is_looping(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_FALSE;\n    }\n\n    /* There is no notion of looping for sounds that are not backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_data_source_is_looping(pSound->pDataSource);\n}\n\nMA_API ma_bool32 ma_sound_at_end(const ma_sound* pSound)\n{\n    if (pSound == NULL) {\n        return MA_FALSE;\n    }\n\n    /* There is no notion of an end of a sound if it's not backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_FALSE;\n    }\n\n    return ma_sound_get_at_end(pSound);\n}\n\nMA_API ma_result ma_sound_seek_to_pcm_frame(ma_sound* pSound, ma_uint64 frameIndex)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* Seeking is only valid for sounds that are backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    /* We can't be seeking while reading at the same time. We just set the seek target and get the mixing thread to do the actual seek. */\n    ma_atomic_exchange_64(&pSound->seekTarget, frameIndex);\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_sound_seek_to_second(ma_sound* pSound, float seekPointInSeconds)\n{\n    ma_uint64 frameIndex;\n    ma_uint32 sampleRate;\n    ma_result result;\n\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    result = ma_sound_get_data_format(pSound, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* We need PCM frames. We need to convert first */\n    frameIndex = (ma_uint64)(seekPointInSeconds * sampleRate);\n\n    return ma_sound_seek_to_pcm_frame(pSound, frameIndex);\n}\n\nMA_API ma_result ma_sound_get_data_format(const ma_sound* pSound, ma_format* pFormat, ma_uint32* pChannels, ma_uint32* pSampleRate, ma_channel* pChannelMap, size_t channelMapCap)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The data format is retrieved directly from the data source if the sound is backed by one. Otherwise we pull it from the node. */\n    if (pSound->pDataSource == NULL) {\n        ma_uint32 channels;\n\n        if (pFormat != NULL) {\n            *pFormat = ma_format_f32;\n        }\n\n        channels = ma_node_get_input_channels(&pSound->engineNode, 0);\n        if (pChannels != NULL) {\n            *pChannels = channels;\n        }\n\n        if (pSampleRate != NULL) {\n            *pSampleRate = pSound->engineNode.resampler.sampleRateIn;\n        }\n\n        if (pChannelMap != NULL) {\n            ma_channel_map_init_standard(ma_standard_channel_map_default, pChannelMap, channelMapCap, channels);\n        }\n\n        return MA_SUCCESS;\n    } else {\n        return ma_data_source_get_data_format(pSound->pDataSource, pFormat, pChannels, pSampleRate, pChannelMap, channelMapCap);\n    }\n}\n\nMA_API ma_result ma_sound_get_cursor_in_pcm_frames(const ma_sound* pSound, ma_uint64* pCursor)\n{\n    ma_uint64 seekTarget;\n\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The notion of a cursor is only valid for sounds that are backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    seekTarget = ma_atomic_load_64(&pSound->seekTarget);\n    if (seekTarget != MA_SEEK_TARGET_NONE) {\n        *pCursor = seekTarget;\n        return MA_SUCCESS;\n    } else {\n        return ma_data_source_get_cursor_in_pcm_frames(pSound->pDataSource, pCursor);\n    }\n}\n\nMA_API ma_result ma_sound_get_length_in_pcm_frames(const ma_sound* pSound, ma_uint64* pLength)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The notion of a sound length is only valid for sounds that are backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    return ma_data_source_get_length_in_pcm_frames(pSound->pDataSource, pLength);\n}\n\nMA_API ma_result ma_sound_get_cursor_in_seconds(const ma_sound* pSound, float* pCursor)\n{\n    ma_result result;\n    ma_uint64 cursorInPCMFrames;\n    ma_uint32 sampleRate;\n\n    if (pCursor != NULL) {\n        *pCursor = 0;\n    }\n\n    result = ma_sound_get_cursor_in_pcm_frames(pSound, &cursorInPCMFrames);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    result = ma_sound_get_data_format(pSound, NULL, NULL, &sampleRate, NULL, 0);\n    if (result != MA_SUCCESS) {\n        return result;\n    }\n\n    /* VC6 does not support division of unsigned 64-bit integers with floating point numbers. Need to use a signed number. This shouldn't effect anything in practice. */\n    *pCursor = (ma_int64)cursorInPCMFrames / (float)sampleRate;\n\n    return MA_SUCCESS;\n}\n\nMA_API ma_result ma_sound_get_length_in_seconds(const ma_sound* pSound, float* pLength)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The notion of a sound length is only valid for sounds that are backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    return ma_data_source_get_length_in_seconds(pSound->pDataSource, pLength);\n}\n\nMA_API ma_result ma_sound_set_end_callback(ma_sound* pSound, ma_sound_end_proc callback, void* pUserData)\n{\n    if (pSound == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* The notion of an end is only valid for sounds that are backed by a data source. */\n    if (pSound->pDataSource == NULL) {\n        return MA_INVALID_OPERATION;\n    }\n\n    pSound->endCallback          = callback;\n    pSound->pEndCallbackUserData = pUserData;\n\n    return MA_SUCCESS;\n}\n\n\nMA_API ma_result ma_sound_group_init(ma_engine* pEngine, ma_uint32 flags, ma_sound_group* pParentGroup, ma_sound_group* pGroup)\n{\n    ma_sound_group_config config = ma_sound_group_config_init_2(pEngine);\n    config.flags              = flags;\n    config.pInitialAttachment = pParentGroup;\n    return ma_sound_group_init_ex(pEngine, &config, pGroup);\n}\n\nMA_API ma_result ma_sound_group_init_ex(ma_engine* pEngine, const ma_sound_group_config* pConfig, ma_sound_group* pGroup)\n{\n    ma_sound_config soundConfig;\n\n    if (pGroup == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    MA_ZERO_OBJECT(pGroup);\n\n    if (pConfig == NULL) {\n        return MA_INVALID_ARGS;\n    }\n\n    /* A sound group is just a sound without a data source. */\n    soundConfig = *pConfig;\n    soundConfig.pFilePath   = NULL;\n    soundConfig.pFilePathW  = NULL;\n    soundConfig.pDataSource = NULL;\n\n    /*\n    Groups need to have spatialization disabled by default because I think it'll be pretty rare\n    that programs will want to spatialize groups (but not unheard of). Certainly it feels like\n    disabling this by default feels like the right option. Spatialization can be enabled with a\n    call to ma_sound_group_set_spatialization_enabled().\n    */\n    soundConfig.flags |= MA_SOUND_FLAG_NO_SPATIALIZATION;\n\n    return ma_sound_init_ex(pEngine, &soundConfig, pGroup);\n}\n\nMA_API void ma_sound_group_uninit(ma_sound_group* pGroup)\n{\n    ma_sound_uninit(pGroup);\n}\n\nMA_API ma_engine* ma_sound_group_get_engine(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_engine(pGroup);\n}\n\nMA_API ma_result ma_sound_group_start(ma_sound_group* pGroup)\n{\n    return ma_sound_start(pGroup);\n}\n\nMA_API ma_result ma_sound_group_stop(ma_sound_group* pGroup)\n{\n    return ma_sound_stop(pGroup);\n}\n\nMA_API void ma_sound_group_set_volume(ma_sound_group* pGroup, float volume)\n{\n    ma_sound_set_volume(pGroup, volume);\n}\n\nMA_API float ma_sound_group_get_volume(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_volume(pGroup);\n}\n\nMA_API void ma_sound_group_set_pan(ma_sound_group* pGroup, float pan)\n{\n    ma_sound_set_pan(pGroup, pan);\n}\n\nMA_API float ma_sound_group_get_pan(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_pan(pGroup);\n}\n\nMA_API void ma_sound_group_set_pan_mode(ma_sound_group* pGroup, ma_pan_mode panMode)\n{\n    ma_sound_set_pan_mode(pGroup, panMode);\n}\n\nMA_API ma_pan_mode ma_sound_group_get_pan_mode(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_pan_mode(pGroup);\n}\n\nMA_API void ma_sound_group_set_pitch(ma_sound_group* pGroup, float pitch)\n{\n    ma_sound_set_pitch(pGroup, pitch);\n}\n\nMA_API float ma_sound_group_get_pitch(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_pitch(pGroup);\n}\n\nMA_API void ma_sound_group_set_spatialization_enabled(ma_sound_group* pGroup, ma_bool32 enabled)\n{\n    ma_sound_set_spatialization_enabled(pGroup, enabled);\n}\n\nMA_API ma_bool32 ma_sound_group_is_spatialization_enabled(const ma_sound_group* pGroup)\n{\n    return ma_sound_is_spatialization_enabled(pGroup);\n}\n\nMA_API void ma_sound_group_set_pinned_listener_index(ma_sound_group* pGroup, ma_uint32 listenerIndex)\n{\n    ma_sound_set_pinned_listener_index(pGroup, listenerIndex);\n}\n\nMA_API ma_uint32 ma_sound_group_get_pinned_listener_index(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_pinned_listener_index(pGroup);\n}\n\nMA_API ma_uint32 ma_sound_group_get_listener_index(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_listener_index(pGroup);\n}\n\nMA_API ma_vec3f ma_sound_group_get_direction_to_listener(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_direction_to_listener(pGroup);\n}\n\nMA_API void ma_sound_group_set_position(ma_sound_group* pGroup, float x, float y, float z)\n{\n    ma_sound_set_position(pGroup, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_group_get_position(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_position(pGroup);\n}\n\nMA_API void ma_sound_group_set_direction(ma_sound_group* pGroup, float x, float y, float z)\n{\n    ma_sound_set_direction(pGroup, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_group_get_direction(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_direction(pGroup);\n}\n\nMA_API void ma_sound_group_set_velocity(ma_sound_group* pGroup, float x, float y, float z)\n{\n    ma_sound_set_velocity(pGroup, x, y, z);\n}\n\nMA_API ma_vec3f ma_sound_group_get_velocity(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_velocity(pGroup);\n}\n\nMA_API void ma_sound_group_set_attenuation_model(ma_sound_group* pGroup, ma_attenuation_model attenuationModel)\n{\n    ma_sound_set_attenuation_model(pGroup, attenuationModel);\n}\n\nMA_API ma_attenuation_model ma_sound_group_get_attenuation_model(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_attenuation_model(pGroup);\n}\n\nMA_API void ma_sound_group_set_positioning(ma_sound_group* pGroup, ma_positioning positioning)\n{\n    ma_sound_set_positioning(pGroup, positioning);\n}\n\nMA_API ma_positioning ma_sound_group_get_positioning(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_positioning(pGroup);\n}\n\nMA_API void ma_sound_group_set_rolloff(ma_sound_group* pGroup, float rolloff)\n{\n    ma_sound_set_rolloff(pGroup, rolloff);\n}\n\nMA_API float ma_sound_group_get_rolloff(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_rolloff(pGroup);\n}\n\nMA_API void ma_sound_group_set_min_gain(ma_sound_group* pGroup, float minGain)\n{\n    ma_sound_set_min_gain(pGroup, minGain);\n}\n\nMA_API float ma_sound_group_get_min_gain(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_min_gain(pGroup);\n}\n\nMA_API void ma_sound_group_set_max_gain(ma_sound_group* pGroup, float maxGain)\n{\n    ma_sound_set_max_gain(pGroup, maxGain);\n}\n\nMA_API float ma_sound_group_get_max_gain(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_max_gain(pGroup);\n}\n\nMA_API void ma_sound_group_set_min_distance(ma_sound_group* pGroup, float minDistance)\n{\n    ma_sound_set_min_distance(pGroup, minDistance);\n}\n\nMA_API float ma_sound_group_get_min_distance(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_min_distance(pGroup);\n}\n\nMA_API void ma_sound_group_set_max_distance(ma_sound_group* pGroup, float maxDistance)\n{\n    ma_sound_set_max_distance(pGroup, maxDistance);\n}\n\nMA_API float ma_sound_group_get_max_distance(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_max_distance(pGroup);\n}\n\nMA_API void ma_sound_group_set_cone(ma_sound_group* pGroup, float innerAngleInRadians, float outerAngleInRadians, float outerGain)\n{\n    ma_sound_set_cone(pGroup, innerAngleInRadians, outerAngleInRadians, outerGain);\n}\n\nMA_API void ma_sound_group_get_cone(const ma_sound_group* pGroup, float* pInnerAngleInRadians, float* pOuterAngleInRadians, float* pOuterGain)\n{\n    ma_sound_get_cone(pGroup, pInnerAngleInRadians, pOuterAngleInRadians, pOuterGain);\n}\n\nMA_API void ma_sound_group_set_doppler_factor(ma_sound_group* pGroup, float dopplerFactor)\n{\n    ma_sound_set_doppler_factor(pGroup, dopplerFactor);\n}\n\nMA_API float ma_sound_group_get_doppler_factor(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_doppler_factor(pGroup);\n}\n\nMA_API void ma_sound_group_set_directional_attenuation_factor(ma_sound_group* pGroup, float directionalAttenuationFactor)\n{\n    ma_sound_set_directional_attenuation_factor(pGroup, directionalAttenuationFactor);\n}\n\nMA_API float ma_sound_group_get_directional_attenuation_factor(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_directional_attenuation_factor(pGroup);\n}\n\nMA_API void ma_sound_group_set_fade_in_pcm_frames(ma_sound_group* pGroup, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInFrames)\n{\n    ma_sound_set_fade_in_pcm_frames(pGroup, volumeBeg, volumeEnd, fadeLengthInFrames);\n}\n\nMA_API void ma_sound_group_set_fade_in_milliseconds(ma_sound_group* pGroup, float volumeBeg, float volumeEnd, ma_uint64 fadeLengthInMilliseconds)\n{\n    ma_sound_set_fade_in_milliseconds(pGroup, volumeBeg, volumeEnd, fadeLengthInMilliseconds);\n}\n\nMA_API float ma_sound_group_get_current_fade_volume(ma_sound_group* pGroup)\n{\n    return ma_sound_get_current_fade_volume(pGroup);\n}\n\nMA_API void ma_sound_group_set_start_time_in_pcm_frames(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInFrames)\n{\n    ma_sound_set_start_time_in_pcm_frames(pGroup, absoluteGlobalTimeInFrames);\n}\n\nMA_API void ma_sound_group_set_start_time_in_milliseconds(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInMilliseconds)\n{\n    ma_sound_set_start_time_in_milliseconds(pGroup, absoluteGlobalTimeInMilliseconds);\n}\n\nMA_API void ma_sound_group_set_stop_time_in_pcm_frames(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInFrames)\n{\n    ma_sound_set_stop_time_in_pcm_frames(pGroup, absoluteGlobalTimeInFrames);\n}\n\nMA_API void ma_sound_group_set_stop_time_in_milliseconds(ma_sound_group* pGroup, ma_uint64 absoluteGlobalTimeInMilliseconds)\n{\n    ma_sound_set_stop_time_in_milliseconds(pGroup, absoluteGlobalTimeInMilliseconds);\n}\n\nMA_API ma_bool32 ma_sound_group_is_playing(const ma_sound_group* pGroup)\n{\n    return ma_sound_is_playing(pGroup);\n}\n\nMA_API ma_uint64 ma_sound_group_get_time_in_pcm_frames(const ma_sound_group* pGroup)\n{\n    return ma_sound_get_time_in_pcm_frames(pGroup);\n}\n#endif  /* MA_NO_ENGINE */\n/* END SECTION: miniaudio_engine.c */\n\n\n\n/**************************************************************************************************************************************************************\n***************************************************************************************************************************************************************\n\nAuto Generated\n==============\nAll code below is auto-generated from a tool. This mostly consists of decoding backend implementations such as ma_dr_wav, ma_dr_flac, etc. If you find a bug in the\ncode below please report the bug to the respective repository for the relevant project (probably dr_libs).\n\n***************************************************************************************************************************************************************\n**************************************************************************************************************************************************************/\n#if !defined(MA_NO_WAV) && (!defined(MA_NO_DECODING) || !defined(MA_NO_ENCODING))\n#if !defined(MA_DR_WAV_IMPLEMENTATION)\n/* dr_wav_c begin */\n#ifndef ma_dr_wav_c\n#define ma_dr_wav_c\n#ifdef __MRC__\n#pragma options opt off\n#endif\n#include <stdlib.h>\n#include <string.h>\n#include <limits.h>\n#ifndef MA_DR_WAV_NO_STDIO\n#include <stdio.h>\n#ifndef MA_DR_WAV_NO_WCHAR\n#include <wchar.h>\n#endif\n#endif\n#ifndef MA_DR_WAV_ASSERT\n#include <assert.h>\n#define MA_DR_WAV_ASSERT(expression)           assert(expression)\n#endif\n#ifndef MA_DR_WAV_MALLOC\n#define MA_DR_WAV_MALLOC(sz)                   malloc((sz))\n#endif\n#ifndef MA_DR_WAV_REALLOC\n#define MA_DR_WAV_REALLOC(p, sz)               realloc((p), (sz))\n#endif\n#ifndef MA_DR_WAV_FREE\n#define MA_DR_WAV_FREE(p)                      free((p))\n#endif\n#ifndef MA_DR_WAV_COPY_MEMORY\n#define MA_DR_WAV_COPY_MEMORY(dst, src, sz)    memcpy((dst), (src), (sz))\n#endif\n#ifndef MA_DR_WAV_ZERO_MEMORY\n#define MA_DR_WAV_ZERO_MEMORY(p, sz)           memset((p), 0, (sz))\n#endif\n#ifndef MA_DR_WAV_ZERO_OBJECT\n#define MA_DR_WAV_ZERO_OBJECT(p)               MA_DR_WAV_ZERO_MEMORY((p), sizeof(*p))\n#endif\n#define ma_dr_wav_countof(x)                   (sizeof(x) / sizeof(x[0]))\n#define ma_dr_wav_align(x, a)                  ((((x) + (a) - 1) / (a)) * (a))\n#define ma_dr_wav_min(a, b)                    (((a) < (b)) ? (a) : (b))\n#define ma_dr_wav_max(a, b)                    (((a) > (b)) ? (a) : (b))\n#define ma_dr_wav_clamp(x, lo, hi)             (ma_dr_wav_max((lo), ma_dr_wav_min((hi), (x))))\n#define ma_dr_wav_offset_ptr(p, offset)        (((ma_uint8*)(p)) + (offset))\n#define MA_DR_WAV_MAX_SIMD_VECTOR_SIZE         32\n#define MA_DR_WAV_INT64_MIN ((ma_int64) ((ma_uint64)0x80000000 << 32))\n#define MA_DR_WAV_INT64_MAX ((ma_int64)(((ma_uint64)0x7FFFFFFF << 32) | 0xFFFFFFFF))\n#if defined(_MSC_VER) && _MSC_VER >= 1400\n    #define MA_DR_WAV_HAS_BYTESWAP16_INTRINSIC\n    #define MA_DR_WAV_HAS_BYTESWAP32_INTRINSIC\n    #define MA_DR_WAV_HAS_BYTESWAP64_INTRINSIC\n#elif defined(__clang__)\n    #if defined(__has_builtin)\n        #if __has_builtin(__builtin_bswap16)\n            #define MA_DR_WAV_HAS_BYTESWAP16_INTRINSIC\n        #endif\n        #if __has_builtin(__builtin_bswap32)\n            #define MA_DR_WAV_HAS_BYTESWAP32_INTRINSIC\n        #endif\n        #if __has_builtin(__builtin_bswap64)\n            #define MA_DR_WAV_HAS_BYTESWAP64_INTRINSIC\n        #endif\n    #endif\n#elif defined(__GNUC__)\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))\n        #define MA_DR_WAV_HAS_BYTESWAP32_INTRINSIC\n        #define MA_DR_WAV_HAS_BYTESWAP64_INTRINSIC\n    #endif\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))\n        #define MA_DR_WAV_HAS_BYTESWAP16_INTRINSIC\n    #endif\n#endif\nMA_API void ma_dr_wav_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision)\n{\n    if (pMajor) {\n        *pMajor = MA_DR_WAV_VERSION_MAJOR;\n    }\n    if (pMinor) {\n        *pMinor = MA_DR_WAV_VERSION_MINOR;\n    }\n    if (pRevision) {\n        *pRevision = MA_DR_WAV_VERSION_REVISION;\n    }\n}\nMA_API const char* ma_dr_wav_version_string(void)\n{\n    return MA_DR_WAV_VERSION_STRING;\n}\n#ifndef MA_DR_WAV_MAX_SAMPLE_RATE\n#define MA_DR_WAV_MAX_SAMPLE_RATE       384000\n#endif\n#ifndef MA_DR_WAV_MAX_CHANNELS\n#define MA_DR_WAV_MAX_CHANNELS          256\n#endif\n#ifndef MA_DR_WAV_MAX_BITS_PER_SAMPLE\n#define MA_DR_WAV_MAX_BITS_PER_SAMPLE   64\n#endif\nstatic const ma_uint8 ma_dr_wavGUID_W64_RIFF[16] = {0x72,0x69,0x66,0x66, 0x2E,0x91, 0xCF,0x11, 0xA5,0xD6, 0x28,0xDB,0x04,0xC1,0x00,0x00};\nstatic const ma_uint8 ma_dr_wavGUID_W64_WAVE[16] = {0x77,0x61,0x76,0x65, 0xF3,0xAC, 0xD3,0x11, 0x8C,0xD1, 0x00,0xC0,0x4F,0x8E,0xDB,0x8A};\nstatic const ma_uint8 ma_dr_wavGUID_W64_FMT [16] = {0x66,0x6D,0x74,0x20, 0xF3,0xAC, 0xD3,0x11, 0x8C,0xD1, 0x00,0xC0,0x4F,0x8E,0xDB,0x8A};\nstatic const ma_uint8 ma_dr_wavGUID_W64_FACT[16] = {0x66,0x61,0x63,0x74, 0xF3,0xAC, 0xD3,0x11, 0x8C,0xD1, 0x00,0xC0,0x4F,0x8E,0xDB,0x8A};\nstatic const ma_uint8 ma_dr_wavGUID_W64_DATA[16] = {0x64,0x61,0x74,0x61, 0xF3,0xAC, 0xD3,0x11, 0x8C,0xD1, 0x00,0xC0,0x4F,0x8E,0xDB,0x8A};\nstatic MA_INLINE int ma_dr_wav__is_little_endian(void)\n{\n#if defined(MA_X86) || defined(MA_X64)\n    return MA_TRUE;\n#elif defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN\n    return MA_TRUE;\n#else\n    int n = 1;\n    return (*(char*)&n) == 1;\n#endif\n}\nstatic MA_INLINE void ma_dr_wav_bytes_to_guid(const ma_uint8* data, ma_uint8* guid)\n{\n    int i;\n    for (i = 0; i < 16; ++i) {\n        guid[i] = data[i];\n    }\n}\nstatic MA_INLINE ma_uint16 ma_dr_wav__bswap16(ma_uint16 n)\n{\n#ifdef MA_DR_WAV_HAS_BYTESWAP16_INTRINSIC\n    #if defined(_MSC_VER)\n        return _byteswap_ushort(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        return __builtin_bswap16(n);\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & 0xFF00) >> 8) |\n           ((n & 0x00FF) << 8);\n#endif\n}\nstatic MA_INLINE ma_uint32 ma_dr_wav__bswap32(ma_uint32 n)\n{\n#ifdef MA_DR_WAV_HAS_BYTESWAP32_INTRINSIC\n    #if defined(_MSC_VER)\n        return _byteswap_ulong(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        #if defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 6) && !defined(MA_64BIT)\n            ma_uint32 r;\n            __asm__ __volatile__ (\n            #if defined(MA_64BIT)\n                \"rev %w[out], %w[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)\n            #else\n                \"rev %[out], %[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)\n            #endif\n            );\n            return r;\n        #else\n            return __builtin_bswap32(n);\n        #endif\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & 0xFF000000) >> 24) |\n           ((n & 0x00FF0000) >>  8) |\n           ((n & 0x0000FF00) <<  8) |\n           ((n & 0x000000FF) << 24);\n#endif\n}\nstatic MA_INLINE ma_uint64 ma_dr_wav__bswap64(ma_uint64 n)\n{\n#ifdef MA_DR_WAV_HAS_BYTESWAP64_INTRINSIC\n    #if defined(_MSC_VER)\n        return _byteswap_uint64(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        return __builtin_bswap64(n);\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & ((ma_uint64)0xFF000000 << 32)) >> 56) |\n           ((n & ((ma_uint64)0x00FF0000 << 32)) >> 40) |\n           ((n & ((ma_uint64)0x0000FF00 << 32)) >> 24) |\n           ((n & ((ma_uint64)0x000000FF << 32)) >>  8) |\n           ((n & ((ma_uint64)0xFF000000      )) <<  8) |\n           ((n & ((ma_uint64)0x00FF0000      )) << 24) |\n           ((n & ((ma_uint64)0x0000FF00      )) << 40) |\n           ((n & ((ma_uint64)0x000000FF      )) << 56);\n#endif\n}\nstatic MA_INLINE ma_int16 ma_dr_wav__bswap_s16(ma_int16 n)\n{\n    return (ma_int16)ma_dr_wav__bswap16((ma_uint16)n);\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples_s16(ma_int16* pSamples, ma_uint64 sampleCount)\n{\n    ma_uint64 iSample;\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamples[iSample] = ma_dr_wav__bswap_s16(pSamples[iSample]);\n    }\n}\nstatic MA_INLINE void ma_dr_wav__bswap_s24(ma_uint8* p)\n{\n    ma_uint8 t;\n    t = p[0];\n    p[0] = p[2];\n    p[2] = t;\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples_s24(ma_uint8* pSamples, ma_uint64 sampleCount)\n{\n    ma_uint64 iSample;\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        ma_uint8* pSample = pSamples + (iSample*3);\n        ma_dr_wav__bswap_s24(pSample);\n    }\n}\nstatic MA_INLINE ma_int32 ma_dr_wav__bswap_s32(ma_int32 n)\n{\n    return (ma_int32)ma_dr_wav__bswap32((ma_uint32)n);\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples_s32(ma_int32* pSamples, ma_uint64 sampleCount)\n{\n    ma_uint64 iSample;\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamples[iSample] = ma_dr_wav__bswap_s32(pSamples[iSample]);\n    }\n}\nstatic MA_INLINE ma_int64 ma_dr_wav__bswap_s64(ma_int64 n)\n{\n    return (ma_int64)ma_dr_wav__bswap64((ma_uint64)n);\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples_s64(ma_int64* pSamples, ma_uint64 sampleCount)\n{\n    ma_uint64 iSample;\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamples[iSample] = ma_dr_wav__bswap_s64(pSamples[iSample]);\n    }\n}\nstatic MA_INLINE float ma_dr_wav__bswap_f32(float n)\n{\n    union {\n        ma_uint32 i;\n        float f;\n    } x;\n    x.f = n;\n    x.i = ma_dr_wav__bswap32(x.i);\n    return x.f;\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples_f32(float* pSamples, ma_uint64 sampleCount)\n{\n    ma_uint64 iSample;\n    for (iSample = 0; iSample < sampleCount; iSample += 1) {\n        pSamples[iSample] = ma_dr_wav__bswap_f32(pSamples[iSample]);\n    }\n}\nstatic MA_INLINE void ma_dr_wav__bswap_samples(void* pSamples, ma_uint64 sampleCount, ma_uint32 bytesPerSample)\n{\n    switch (bytesPerSample)\n    {\n        case 1:\n        {\n        } break;\n        case 2:\n        {\n            ma_dr_wav__bswap_samples_s16((ma_int16*)pSamples, sampleCount);\n        } break;\n        case 3:\n        {\n            ma_dr_wav__bswap_samples_s24((ma_uint8*)pSamples, sampleCount);\n        } break;\n        case 4:\n        {\n            ma_dr_wav__bswap_samples_s32((ma_int32*)pSamples, sampleCount);\n        } break;\n        case 8:\n        {\n            ma_dr_wav__bswap_samples_s64((ma_int64*)pSamples, sampleCount);\n        } break;\n        default:\n        {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n        } break;\n    }\n}\nMA_PRIVATE MA_INLINE ma_bool32 ma_dr_wav_is_container_be(ma_dr_wav_container container)\n{\n    if (container == ma_dr_wav_container_rifx || container == ma_dr_wav_container_aiff) {\n        return MA_TRUE;\n    } else {\n        return MA_FALSE;\n    }\n}\nMA_PRIVATE MA_INLINE ma_uint16 ma_dr_wav_bytes_to_u16_le(const ma_uint8* data)\n{\n    return ((ma_uint16)data[0] << 0) | ((ma_uint16)data[1] << 8);\n}\nMA_PRIVATE MA_INLINE ma_uint16 ma_dr_wav_bytes_to_u16_be(const ma_uint8* data)\n{\n    return ((ma_uint16)data[1] << 0) | ((ma_uint16)data[0] << 8);\n}\nMA_PRIVATE MA_INLINE ma_uint16 ma_dr_wav_bytes_to_u16_ex(const ma_uint8* data, ma_dr_wav_container container)\n{\n    if (ma_dr_wav_is_container_be(container)) {\n        return ma_dr_wav_bytes_to_u16_be(data);\n    } else {\n        return ma_dr_wav_bytes_to_u16_le(data);\n    }\n}\nMA_PRIVATE MA_INLINE ma_uint32 ma_dr_wav_bytes_to_u32_le(const ma_uint8* data)\n{\n    return ((ma_uint32)data[0] << 0) | ((ma_uint32)data[1] << 8) | ((ma_uint32)data[2] << 16) | ((ma_uint32)data[3] << 24);\n}\nMA_PRIVATE MA_INLINE ma_uint32 ma_dr_wav_bytes_to_u32_be(const ma_uint8* data)\n{\n    return ((ma_uint32)data[3] << 0) | ((ma_uint32)data[2] << 8) | ((ma_uint32)data[1] << 16) | ((ma_uint32)data[0] << 24);\n}\nMA_PRIVATE MA_INLINE ma_uint32 ma_dr_wav_bytes_to_u32_ex(const ma_uint8* data, ma_dr_wav_container container)\n{\n    if (ma_dr_wav_is_container_be(container)) {\n        return ma_dr_wav_bytes_to_u32_be(data);\n    } else {\n        return ma_dr_wav_bytes_to_u32_le(data);\n    }\n}\nMA_PRIVATE ma_int64 ma_dr_wav_aiff_extented_to_s64(const ma_uint8* data)\n{\n    ma_uint32 exponent = ((ma_uint32)data[0] << 8) | data[1];\n    ma_uint64 hi = ((ma_uint64)data[2] << 24) | ((ma_uint64)data[3] << 16) | ((ma_uint64)data[4] <<  8) | ((ma_uint64)data[5] <<  0);\n    ma_uint64 lo = ((ma_uint64)data[6] << 24) | ((ma_uint64)data[7] << 16) | ((ma_uint64)data[8] <<  8) | ((ma_uint64)data[9] <<  0);\n    ma_uint64 significand = (hi << 32) | lo;\n    int sign = exponent >> 15;\n    exponent &= 0x7FFF;\n    if (exponent == 0 && significand == 0) {\n        return 0;\n    } else if (exponent == 0x7FFF) {\n        return sign ? MA_DR_WAV_INT64_MIN : MA_DR_WAV_INT64_MAX;\n    }\n    exponent -= 16383;\n    if (exponent > 63) {\n        return sign ? MA_DR_WAV_INT64_MIN : MA_DR_WAV_INT64_MAX;\n    } else if (exponent < 1) {\n        return 0;\n    }\n    significand >>= (63 - exponent);\n    if (sign) {\n        return -(ma_int64)significand;\n    } else {\n        return  (ma_int64)significand;\n    }\n}\nMA_PRIVATE void* ma_dr_wav__malloc_default(size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_WAV_MALLOC(sz);\n}\nMA_PRIVATE void* ma_dr_wav__realloc_default(void* p, size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_WAV_REALLOC(p, sz);\n}\nMA_PRIVATE void ma_dr_wav__free_default(void* p, void* pUserData)\n{\n    (void)pUserData;\n    MA_DR_WAV_FREE(p);\n}\nMA_PRIVATE void* ma_dr_wav__malloc_from_callbacks(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onMalloc != NULL) {\n        return pAllocationCallbacks->onMalloc(sz, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(NULL, sz, pAllocationCallbacks->pUserData);\n    }\n    return NULL;\n}\nMA_PRIVATE void* ma_dr_wav__realloc_from_callbacks(void* p, size_t szNew, size_t szOld, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(p, szNew, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onMalloc != NULL && pAllocationCallbacks->onFree != NULL) {\n        void* p2;\n        p2 = pAllocationCallbacks->onMalloc(szNew, pAllocationCallbacks->pUserData);\n        if (p2 == NULL) {\n            return NULL;\n        }\n        if (p != NULL) {\n            MA_DR_WAV_COPY_MEMORY(p2, p, szOld);\n            pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n        }\n        return p2;\n    }\n    return NULL;\n}\nMA_PRIVATE void ma_dr_wav__free_from_callbacks(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (p == NULL || pAllocationCallbacks == NULL) {\n        return;\n    }\n    if (pAllocationCallbacks->onFree != NULL) {\n        pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n    }\n}\nMA_PRIVATE ma_allocation_callbacks ma_dr_wav_copy_allocation_callbacks_or_defaults(const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        return *pAllocationCallbacks;\n    } else {\n        ma_allocation_callbacks allocationCallbacks;\n        allocationCallbacks.pUserData = NULL;\n        allocationCallbacks.onMalloc  = ma_dr_wav__malloc_default;\n        allocationCallbacks.onRealloc = ma_dr_wav__realloc_default;\n        allocationCallbacks.onFree    = ma_dr_wav__free_default;\n        return allocationCallbacks;\n    }\n}\nstatic MA_INLINE ma_bool32 ma_dr_wav__is_compressed_format_tag(ma_uint16 formatTag)\n{\n    return\n        formatTag == MA_DR_WAVE_FORMAT_ADPCM ||\n        formatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM;\n}\nMA_PRIVATE unsigned int ma_dr_wav__chunk_padding_size_riff(ma_uint64 chunkSize)\n{\n    return (unsigned int)(chunkSize % 2);\n}\nMA_PRIVATE unsigned int ma_dr_wav__chunk_padding_size_w64(ma_uint64 chunkSize)\n{\n    return (unsigned int)(chunkSize % 8);\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_uint64 samplesToRead, ma_int16* pBufferOut);\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__ima(ma_dr_wav* pWav, ma_uint64 samplesToRead, ma_int16* pBufferOut);\nMA_PRIVATE ma_bool32 ma_dr_wav_init_write__internal(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount);\nMA_PRIVATE ma_result ma_dr_wav__read_chunk_header(ma_dr_wav_read_proc onRead, void* pUserData, ma_dr_wav_container container, ma_uint64* pRunningBytesReadOut, ma_dr_wav_chunk_header* pHeaderOut)\n{\n    if (container == ma_dr_wav_container_riff || container == ma_dr_wav_container_rifx || container == ma_dr_wav_container_rf64 || container == ma_dr_wav_container_aiff) {\n        ma_uint8 sizeInBytes[4];\n        if (onRead(pUserData, pHeaderOut->id.fourcc, 4) != 4) {\n            return MA_AT_END;\n        }\n        if (onRead(pUserData, sizeInBytes, 4) != 4) {\n            return MA_INVALID_FILE;\n        }\n        pHeaderOut->sizeInBytes = ma_dr_wav_bytes_to_u32_ex(sizeInBytes, container);\n        pHeaderOut->paddingSize = ma_dr_wav__chunk_padding_size_riff(pHeaderOut->sizeInBytes);\n        *pRunningBytesReadOut += 8;\n    } else if (container == ma_dr_wav_container_w64) {\n        ma_uint8 sizeInBytes[8];\n        if (onRead(pUserData, pHeaderOut->id.guid, 16) != 16) {\n            return MA_AT_END;\n        }\n        if (onRead(pUserData, sizeInBytes, 8) != 8) {\n            return MA_INVALID_FILE;\n        }\n        pHeaderOut->sizeInBytes = ma_dr_wav_bytes_to_u64(sizeInBytes) - 24;\n        pHeaderOut->paddingSize = ma_dr_wav__chunk_padding_size_w64(pHeaderOut->sizeInBytes);\n        *pRunningBytesReadOut += 24;\n    } else {\n        return MA_INVALID_FILE;\n    }\n    return MA_SUCCESS;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__seek_forward(ma_dr_wav_seek_proc onSeek, ma_uint64 offset, void* pUserData)\n{\n    ma_uint64 bytesRemainingToSeek = offset;\n    while (bytesRemainingToSeek > 0) {\n        if (bytesRemainingToSeek > 0x7FFFFFFF) {\n            if (!onSeek(pUserData, 0x7FFFFFFF, MA_DR_WAV_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            bytesRemainingToSeek -= 0x7FFFFFFF;\n        } else {\n            if (!onSeek(pUserData, (int)bytesRemainingToSeek, MA_DR_WAV_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            bytesRemainingToSeek = 0;\n        }\n    }\n    return MA_TRUE;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__seek_from_start(ma_dr_wav_seek_proc onSeek, ma_uint64 offset, void* pUserData)\n{\n    if (offset <= 0x7FFFFFFF) {\n        return onSeek(pUserData, (int)offset, MA_DR_WAV_SEEK_SET);\n    }\n    if (!onSeek(pUserData, 0x7FFFFFFF, MA_DR_WAV_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    offset -= 0x7FFFFFFF;\n    for (;;) {\n        if (offset <= 0x7FFFFFFF) {\n            return onSeek(pUserData, (int)offset, MA_DR_WAV_SEEK_CUR);\n        }\n        if (!onSeek(pUserData, 0x7FFFFFFF, MA_DR_WAV_SEEK_CUR)) {\n            return MA_FALSE;\n        }\n        offset -= 0x7FFFFFFF;\n    }\n}\nMA_PRIVATE size_t ma_dr_wav__on_read(ma_dr_wav_read_proc onRead, void* pUserData, void* pBufferOut, size_t bytesToRead, ma_uint64* pCursor)\n{\n    size_t bytesRead;\n    MA_DR_WAV_ASSERT(onRead != NULL);\n    MA_DR_WAV_ASSERT(pCursor != NULL);\n    bytesRead = onRead(pUserData, pBufferOut, bytesToRead);\n    *pCursor += bytesRead;\n    return bytesRead;\n}\n#if 0\nMA_PRIVATE ma_bool32 ma_dr_wav__on_seek(ma_dr_wav_seek_proc onSeek, void* pUserData, int offset, ma_dr_wav_seek_origin origin, ma_uint64* pCursor)\n{\n    MA_DR_WAV_ASSERT(onSeek != NULL);\n    MA_DR_WAV_ASSERT(pCursor != NULL);\n    if (!onSeek(pUserData, offset, origin)) {\n        return MA_FALSE;\n    }\n    if (origin == MA_DR_WAV_SEEK_SET) {\n        *pCursor = offset;\n    } else {\n        *pCursor += offset;\n    }\n    return MA_TRUE;\n}\n#endif\n#define MA_DR_WAV_SMPL_BYTES                    36\n#define MA_DR_WAV_SMPL_LOOP_BYTES               24\n#define MA_DR_WAV_INST_BYTES                    7\n#define MA_DR_WAV_ACID_BYTES                    24\n#define MA_DR_WAV_CUE_BYTES                     4\n#define MA_DR_WAV_BEXT_BYTES                    602\n#define MA_DR_WAV_BEXT_DESCRIPTION_BYTES        256\n#define MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES    32\n#define MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES     32\n#define MA_DR_WAV_BEXT_RESERVED_BYTES           180\n#define MA_DR_WAV_BEXT_UMID_BYTES               64\n#define MA_DR_WAV_CUE_POINT_BYTES               24\n#define MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES      4\n#define MA_DR_WAV_LIST_LABELLED_TEXT_BYTES      20\n#define MA_DR_WAV_METADATA_ALIGNMENT            8\ntypedef enum\n{\n    ma_dr_wav__metadata_parser_stage_count,\n    ma_dr_wav__metadata_parser_stage_read\n} ma_dr_wav__metadata_parser_stage;\ntypedef struct\n{\n    ma_dr_wav_read_proc onRead;\n    ma_dr_wav_seek_proc onSeek;\n    void *pReadSeekUserData;\n    ma_dr_wav__metadata_parser_stage stage;\n    ma_dr_wav_metadata *pMetadata;\n    ma_uint32 metadataCount;\n    ma_uint8 *pData;\n    ma_uint8 *pDataCursor;\n    ma_uint64 metadataCursor;\n    ma_uint64 extraCapacity;\n} ma_dr_wav__metadata_parser;\nMA_PRIVATE size_t ma_dr_wav__metadata_memory_capacity(ma_dr_wav__metadata_parser* pParser)\n{\n    ma_uint64 cap = sizeof(ma_dr_wav_metadata) * (ma_uint64)pParser->metadataCount + pParser->extraCapacity;\n    if (cap > MA_SIZE_MAX) {\n        return 0;\n    }\n    return (size_t)cap;\n}\nMA_PRIVATE ma_uint8* ma_dr_wav__metadata_get_memory(ma_dr_wav__metadata_parser* pParser, size_t size, size_t align)\n{\n    ma_uint8* pResult;\n    if (align) {\n        ma_uintptr modulo = (ma_uintptr)pParser->pDataCursor % align;\n        if (modulo != 0) {\n            pParser->pDataCursor += align - modulo;\n        }\n    }\n    pResult = pParser->pDataCursor;\n    MA_DR_WAV_ASSERT((pResult + size) <= (pParser->pData + ma_dr_wav__metadata_memory_capacity(pParser)));\n    pParser->pDataCursor += size;\n    return pResult;\n}\nMA_PRIVATE void ma_dr_wav__metadata_request_extra_memory_for_stage_2(ma_dr_wav__metadata_parser* pParser, size_t bytes, size_t align)\n{\n    size_t extra = bytes + (align ? (align - 1) : 0);\n    pParser->extraCapacity += extra;\n}\nMA_PRIVATE ma_result ma_dr_wav__metadata_alloc(ma_dr_wav__metadata_parser* pParser, ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pParser->extraCapacity != 0 || pParser->metadataCount != 0) {\n        pAllocationCallbacks->onFree(pParser->pData, pAllocationCallbacks->pUserData);\n        pParser->pData = (ma_uint8*)pAllocationCallbacks->onMalloc(ma_dr_wav__metadata_memory_capacity(pParser), pAllocationCallbacks->pUserData);\n        pParser->pDataCursor = pParser->pData;\n        if (pParser->pData == NULL) {\n            return MA_OUT_OF_MEMORY;\n        }\n        pParser->pMetadata = (ma_dr_wav_metadata*)ma_dr_wav__metadata_get_memory(pParser, sizeof(ma_dr_wav_metadata) * pParser->metadataCount, 1);\n        pParser->metadataCursor = 0;\n    }\n    return MA_SUCCESS;\n}\nMA_PRIVATE size_t ma_dr_wav__metadata_parser_read(ma_dr_wav__metadata_parser* pParser, void* pBufferOut, size_t bytesToRead, ma_uint64* pCursor)\n{\n    if (pCursor != NULL) {\n        return ma_dr_wav__on_read(pParser->onRead, pParser->pReadSeekUserData, pBufferOut, bytesToRead, pCursor);\n    } else {\n        return pParser->onRead(pParser->pReadSeekUserData, pBufferOut, bytesToRead);\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_smpl_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata)\n{\n    ma_uint8 smplHeaderData[MA_DR_WAV_SMPL_BYTES];\n    ma_uint64 totalBytesRead = 0;\n    size_t bytesJustRead;\n    if (pMetadata == NULL) {\n        return 0;\n    }\n    bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, smplHeaderData, sizeof(smplHeaderData), &totalBytesRead);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    MA_DR_WAV_ASSERT(pChunkHeader != NULL);\n    if (pMetadata != NULL && bytesJustRead == sizeof(smplHeaderData)) {\n        ma_uint32 iSampleLoop;\n        pMetadata->type                                     = ma_dr_wav_metadata_type_smpl;\n        pMetadata->data.smpl.manufacturerId                 = ma_dr_wav_bytes_to_u32(smplHeaderData + 0);\n        pMetadata->data.smpl.productId                      = ma_dr_wav_bytes_to_u32(smplHeaderData + 4);\n        pMetadata->data.smpl.samplePeriodNanoseconds        = ma_dr_wav_bytes_to_u32(smplHeaderData + 8);\n        pMetadata->data.smpl.midiUnityNote                  = ma_dr_wav_bytes_to_u32(smplHeaderData + 12);\n        pMetadata->data.smpl.midiPitchFraction              = ma_dr_wav_bytes_to_u32(smplHeaderData + 16);\n        pMetadata->data.smpl.smpteFormat                    = ma_dr_wav_bytes_to_u32(smplHeaderData + 20);\n        pMetadata->data.smpl.smpteOffset                    = ma_dr_wav_bytes_to_u32(smplHeaderData + 24);\n        pMetadata->data.smpl.sampleLoopCount                = ma_dr_wav_bytes_to_u32(smplHeaderData + 28);\n        pMetadata->data.smpl.samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(smplHeaderData + 32);\n        if (pMetadata->data.smpl.sampleLoopCount == (pChunkHeader->sizeInBytes - MA_DR_WAV_SMPL_BYTES) / MA_DR_WAV_SMPL_LOOP_BYTES) {\n            pMetadata->data.smpl.pLoops = (ma_dr_wav_smpl_loop*)ma_dr_wav__metadata_get_memory(pParser, sizeof(ma_dr_wav_smpl_loop) * pMetadata->data.smpl.sampleLoopCount, MA_DR_WAV_METADATA_ALIGNMENT);\n            for (iSampleLoop = 0; iSampleLoop < pMetadata->data.smpl.sampleLoopCount; ++iSampleLoop) {\n                ma_uint8 smplLoopData[MA_DR_WAV_SMPL_LOOP_BYTES];\n                bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, smplLoopData, sizeof(smplLoopData), &totalBytesRead);\n                if (bytesJustRead == sizeof(smplLoopData)) {\n                    pMetadata->data.smpl.pLoops[iSampleLoop].cuePointId        = ma_dr_wav_bytes_to_u32(smplLoopData + 0);\n                    pMetadata->data.smpl.pLoops[iSampleLoop].type              = ma_dr_wav_bytes_to_u32(smplLoopData + 4);\n                    pMetadata->data.smpl.pLoops[iSampleLoop].firstSampleOffset = ma_dr_wav_bytes_to_u32(smplLoopData + 8);\n                    pMetadata->data.smpl.pLoops[iSampleLoop].lastSampleOffset  = ma_dr_wav_bytes_to_u32(smplLoopData + 12);\n                    pMetadata->data.smpl.pLoops[iSampleLoop].sampleFraction    = ma_dr_wav_bytes_to_u32(smplLoopData + 16);\n                    pMetadata->data.smpl.pLoops[iSampleLoop].playCount         = ma_dr_wav_bytes_to_u32(smplLoopData + 20);\n                } else {\n                    break;\n                }\n            }\n            if (pMetadata->data.smpl.samplerSpecificDataSizeInBytes > 0) {\n                pMetadata->data.smpl.pSamplerSpecificData = ma_dr_wav__metadata_get_memory(pParser, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, 1);\n                MA_DR_WAV_ASSERT(pMetadata->data.smpl.pSamplerSpecificData != NULL);\n                ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.smpl.pSamplerSpecificData, pMetadata->data.smpl.samplerSpecificDataSizeInBytes, &totalBytesRead);\n            }\n        }\n    }\n    return totalBytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_cue_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata* pMetadata)\n{\n    ma_uint8 cueHeaderSectionData[MA_DR_WAV_CUE_BYTES];\n    ma_uint64 totalBytesRead = 0;\n    size_t bytesJustRead;\n    if (pMetadata == NULL) {\n        return 0;\n    }\n    bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, cueHeaderSectionData, sizeof(cueHeaderSectionData), &totalBytesRead);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesJustRead == sizeof(cueHeaderSectionData)) {\n        pMetadata->type                   = ma_dr_wav_metadata_type_cue;\n        pMetadata->data.cue.cuePointCount = ma_dr_wav_bytes_to_u32(cueHeaderSectionData);\n        if (pMetadata->data.cue.cuePointCount == (pChunkHeader->sizeInBytes - MA_DR_WAV_CUE_BYTES) / MA_DR_WAV_CUE_POINT_BYTES) {\n            pMetadata->data.cue.pCuePoints    = (ma_dr_wav_cue_point*)ma_dr_wav__metadata_get_memory(pParser, sizeof(ma_dr_wav_cue_point) * pMetadata->data.cue.cuePointCount, MA_DR_WAV_METADATA_ALIGNMENT);\n            MA_DR_WAV_ASSERT(pMetadata->data.cue.pCuePoints != NULL);\n            if (pMetadata->data.cue.cuePointCount > 0) {\n                ma_uint32 iCuePoint;\n                for (iCuePoint = 0; iCuePoint < pMetadata->data.cue.cuePointCount; ++iCuePoint) {\n                    ma_uint8 cuePointData[MA_DR_WAV_CUE_POINT_BYTES];\n                    bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, cuePointData, sizeof(cuePointData), &totalBytesRead);\n                    if (bytesJustRead == sizeof(cuePointData)) {\n                        pMetadata->data.cue.pCuePoints[iCuePoint].id                = ma_dr_wav_bytes_to_u32(cuePointData + 0);\n                        pMetadata->data.cue.pCuePoints[iCuePoint].playOrderPosition = ma_dr_wav_bytes_to_u32(cuePointData + 4);\n                        pMetadata->data.cue.pCuePoints[iCuePoint].dataChunkId[0]    = cuePointData[8];\n                        pMetadata->data.cue.pCuePoints[iCuePoint].dataChunkId[1]    = cuePointData[9];\n                        pMetadata->data.cue.pCuePoints[iCuePoint].dataChunkId[2]    = cuePointData[10];\n                        pMetadata->data.cue.pCuePoints[iCuePoint].dataChunkId[3]    = cuePointData[11];\n                        pMetadata->data.cue.pCuePoints[iCuePoint].chunkStart        = ma_dr_wav_bytes_to_u32(cuePointData + 12);\n                        pMetadata->data.cue.pCuePoints[iCuePoint].blockStart        = ma_dr_wav_bytes_to_u32(cuePointData + 16);\n                        pMetadata->data.cue.pCuePoints[iCuePoint].sampleOffset      = ma_dr_wav_bytes_to_u32(cuePointData + 20);\n                    } else {\n                        break;\n                    }\n                }\n            }\n        }\n    }\n    return totalBytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_inst_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata)\n{\n    ma_uint8 instData[MA_DR_WAV_INST_BYTES];\n    ma_uint64 bytesRead;\n    if (pMetadata == NULL) {\n        return 0;\n    }\n    bytesRead = ma_dr_wav__metadata_parser_read(pParser, instData, sizeof(instData), NULL);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesRead == sizeof(instData)) {\n        pMetadata->type                    = ma_dr_wav_metadata_type_inst;\n        pMetadata->data.inst.midiUnityNote = (ma_int8)instData[0];\n        pMetadata->data.inst.fineTuneCents = (ma_int8)instData[1];\n        pMetadata->data.inst.gainDecibels  = (ma_int8)instData[2];\n        pMetadata->data.inst.lowNote       = (ma_int8)instData[3];\n        pMetadata->data.inst.highNote      = (ma_int8)instData[4];\n        pMetadata->data.inst.lowVelocity   = (ma_int8)instData[5];\n        pMetadata->data.inst.highVelocity  = (ma_int8)instData[6];\n    }\n    return bytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_acid_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata)\n{\n    ma_uint8 acidData[MA_DR_WAV_ACID_BYTES];\n    ma_uint64 bytesRead;\n    if (pMetadata == NULL) {\n        return 0;\n    }\n    bytesRead = ma_dr_wav__metadata_parser_read(pParser, acidData, sizeof(acidData), NULL);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesRead == sizeof(acidData)) {\n        pMetadata->type                       = ma_dr_wav_metadata_type_acid;\n        pMetadata->data.acid.flags            = ma_dr_wav_bytes_to_u32(acidData + 0);\n        pMetadata->data.acid.midiUnityNote    = ma_dr_wav_bytes_to_u16(acidData + 4);\n        pMetadata->data.acid.reserved1        = ma_dr_wav_bytes_to_u16(acidData + 6);\n        pMetadata->data.acid.reserved2        = ma_dr_wav_bytes_to_f32(acidData + 8);\n        pMetadata->data.acid.numBeats         = ma_dr_wav_bytes_to_u32(acidData + 12);\n        pMetadata->data.acid.meterDenominator = ma_dr_wav_bytes_to_u16(acidData + 16);\n        pMetadata->data.acid.meterNumerator   = ma_dr_wav_bytes_to_u16(acidData + 18);\n        pMetadata->data.acid.tempo            = ma_dr_wav_bytes_to_f32(acidData + 20);\n    }\n    return bytesRead;\n}\nMA_PRIVATE size_t ma_dr_wav__strlen(const char* str)\n{\n    size_t result = 0;\n    while (*str++) {\n        result += 1;\n    }\n    return result;\n}\nMA_PRIVATE size_t ma_dr_wav__strlen_clamped(const char* str, size_t maxToRead)\n{\n    size_t result = 0;\n    while (*str++ && result < maxToRead) {\n        result += 1;\n    }\n    return result;\n}\nMA_PRIVATE char* ma_dr_wav__metadata_copy_string(ma_dr_wav__metadata_parser* pParser, const char* str, size_t maxToRead)\n{\n    size_t len = ma_dr_wav__strlen_clamped(str, maxToRead);\n    if (len) {\n        char* result = (char*)ma_dr_wav__metadata_get_memory(pParser, len + 1, 1);\n        MA_DR_WAV_ASSERT(result != NULL);\n        MA_DR_WAV_COPY_MEMORY(result, str, len);\n        result[len] = '\\0';\n        return result;\n    } else {\n        return NULL;\n    }\n}\ntypedef struct\n{\n    const void* pBuffer;\n    size_t sizeInBytes;\n    size_t cursor;\n} ma_dr_wav_buffer_reader;\nMA_PRIVATE ma_result ma_dr_wav_buffer_reader_init(const void* pBuffer, size_t sizeInBytes, ma_dr_wav_buffer_reader* pReader)\n{\n    MA_DR_WAV_ASSERT(pBuffer != NULL);\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    MA_DR_WAV_ZERO_OBJECT(pReader);\n    pReader->pBuffer     = pBuffer;\n    pReader->sizeInBytes = sizeInBytes;\n    pReader->cursor      = 0;\n    return MA_SUCCESS;\n}\nMA_PRIVATE const void* ma_dr_wav_buffer_reader_ptr(const ma_dr_wav_buffer_reader* pReader)\n{\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    return ma_dr_wav_offset_ptr(pReader->pBuffer, pReader->cursor);\n}\nMA_PRIVATE ma_result ma_dr_wav_buffer_reader_seek(ma_dr_wav_buffer_reader* pReader, size_t bytesToSeek)\n{\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    if (pReader->cursor + bytesToSeek > pReader->sizeInBytes) {\n        return MA_BAD_SEEK;\n    }\n    pReader->cursor += bytesToSeek;\n    return MA_SUCCESS;\n}\nMA_PRIVATE ma_result ma_dr_wav_buffer_reader_read(ma_dr_wav_buffer_reader* pReader, void* pDst, size_t bytesToRead, size_t* pBytesRead)\n{\n    ma_result result = MA_SUCCESS;\n    size_t bytesRemaining;\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    if (pBytesRead != NULL) {\n        *pBytesRead = 0;\n    }\n    bytesRemaining = (pReader->sizeInBytes - pReader->cursor);\n    if (bytesToRead > bytesRemaining) {\n        bytesToRead = bytesRemaining;\n    }\n    if (pDst == NULL) {\n        result = ma_dr_wav_buffer_reader_seek(pReader, bytesToRead);\n    } else {\n        MA_DR_WAV_COPY_MEMORY(pDst, ma_dr_wav_buffer_reader_ptr(pReader), bytesToRead);\n        pReader->cursor += bytesToRead;\n    }\n    MA_DR_WAV_ASSERT(pReader->cursor <= pReader->sizeInBytes);\n    if (result == MA_SUCCESS) {\n        if (pBytesRead != NULL) {\n            *pBytesRead = bytesToRead;\n        }\n    }\n    return MA_SUCCESS;\n}\nMA_PRIVATE ma_result ma_dr_wav_buffer_reader_read_u16(ma_dr_wav_buffer_reader* pReader, ma_uint16* pDst)\n{\n    ma_result result;\n    size_t bytesRead;\n    ma_uint8 data[2];\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    MA_DR_WAV_ASSERT(pDst != NULL);\n    *pDst = 0;\n    result = ma_dr_wav_buffer_reader_read(pReader, data, sizeof(*pDst), &bytesRead);\n    if (result != MA_SUCCESS || bytesRead != sizeof(*pDst)) {\n        return result;\n    }\n    *pDst = ma_dr_wav_bytes_to_u16(data);\n    return MA_SUCCESS;\n}\nMA_PRIVATE ma_result ma_dr_wav_buffer_reader_read_u32(ma_dr_wav_buffer_reader* pReader, ma_uint32* pDst)\n{\n    ma_result result;\n    size_t bytesRead;\n    ma_uint8 data[4];\n    MA_DR_WAV_ASSERT(pReader != NULL);\n    MA_DR_WAV_ASSERT(pDst != NULL);\n    *pDst = 0;\n    result = ma_dr_wav_buffer_reader_read(pReader, data, sizeof(*pDst), &bytesRead);\n    if (result != MA_SUCCESS || bytesRead != sizeof(*pDst)) {\n        return result;\n    }\n    *pDst = ma_dr_wav_bytes_to_u32(data);\n    return MA_SUCCESS;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_bext_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata, ma_uint64 chunkSize)\n{\n    ma_uint8 bextData[MA_DR_WAV_BEXT_BYTES];\n    size_t bytesRead = ma_dr_wav__metadata_parser_read(pParser, bextData, sizeof(bextData), NULL);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesRead == sizeof(bextData)) {\n        ma_dr_wav_buffer_reader reader;\n        ma_uint32 timeReferenceLow;\n        ma_uint32 timeReferenceHigh;\n        size_t extraBytes;\n        pMetadata->type = ma_dr_wav_metadata_type_bext;\n        if (ma_dr_wav_buffer_reader_init(bextData, bytesRead, &reader) == MA_SUCCESS) {\n            pMetadata->data.bext.pDescription = ma_dr_wav__metadata_copy_string(pParser, (const char*)ma_dr_wav_buffer_reader_ptr(&reader), MA_DR_WAV_BEXT_DESCRIPTION_BYTES);\n            ma_dr_wav_buffer_reader_seek(&reader, MA_DR_WAV_BEXT_DESCRIPTION_BYTES);\n            pMetadata->data.bext.pOriginatorName = ma_dr_wav__metadata_copy_string(pParser, (const char*)ma_dr_wav_buffer_reader_ptr(&reader), MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES);\n            ma_dr_wav_buffer_reader_seek(&reader, MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES);\n            pMetadata->data.bext.pOriginatorReference = ma_dr_wav__metadata_copy_string(pParser, (const char*)ma_dr_wav_buffer_reader_ptr(&reader), MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES);\n            ma_dr_wav_buffer_reader_seek(&reader, MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES);\n            ma_dr_wav_buffer_reader_read(&reader, pMetadata->data.bext.pOriginationDate, sizeof(pMetadata->data.bext.pOriginationDate), NULL);\n            ma_dr_wav_buffer_reader_read(&reader, pMetadata->data.bext.pOriginationTime, sizeof(pMetadata->data.bext.pOriginationTime), NULL);\n            ma_dr_wav_buffer_reader_read_u32(&reader, &timeReferenceLow);\n            ma_dr_wav_buffer_reader_read_u32(&reader, &timeReferenceHigh);\n            pMetadata->data.bext.timeReference = ((ma_uint64)timeReferenceHigh << 32) + timeReferenceLow;\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.version);\n            pMetadata->data.bext.pUMID = ma_dr_wav__metadata_get_memory(pParser, MA_DR_WAV_BEXT_UMID_BYTES, 1);\n            ma_dr_wav_buffer_reader_read(&reader, pMetadata->data.bext.pUMID, MA_DR_WAV_BEXT_UMID_BYTES, NULL);\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.loudnessValue);\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.loudnessRange);\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.maxTruePeakLevel);\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.maxMomentaryLoudness);\n            ma_dr_wav_buffer_reader_read_u16(&reader, &pMetadata->data.bext.maxShortTermLoudness);\n            MA_DR_WAV_ASSERT((ma_dr_wav_offset_ptr(ma_dr_wav_buffer_reader_ptr(&reader), MA_DR_WAV_BEXT_RESERVED_BYTES)) == (bextData + MA_DR_WAV_BEXT_BYTES));\n            extraBytes = (size_t)(chunkSize - MA_DR_WAV_BEXT_BYTES);\n            if (extraBytes > 0) {\n                pMetadata->data.bext.pCodingHistory = (char*)ma_dr_wav__metadata_get_memory(pParser, extraBytes + 1, 1);\n                MA_DR_WAV_ASSERT(pMetadata->data.bext.pCodingHistory != NULL);\n                bytesRead += ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.bext.pCodingHistory, extraBytes, NULL);\n                pMetadata->data.bext.codingHistorySize = (ma_uint32)ma_dr_wav__strlen(pMetadata->data.bext.pCodingHistory);\n            } else {\n                pMetadata->data.bext.pCodingHistory    = NULL;\n                pMetadata->data.bext.codingHistorySize = 0;\n            }\n        }\n    }\n    return bytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_list_label_or_note_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata, ma_uint64 chunkSize, ma_dr_wav_metadata_type type)\n{\n    ma_uint8 cueIDBuffer[MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES];\n    ma_uint64 totalBytesRead = 0;\n    size_t bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, cueIDBuffer, sizeof(cueIDBuffer), &totalBytesRead);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesJustRead == sizeof(cueIDBuffer)) {\n        ma_uint32 sizeIncludingNullTerminator;\n        pMetadata->type = type;\n        pMetadata->data.labelOrNote.cuePointId = ma_dr_wav_bytes_to_u32(cueIDBuffer);\n        sizeIncludingNullTerminator = (ma_uint32)chunkSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES;\n        if (sizeIncludingNullTerminator > 0) {\n            pMetadata->data.labelOrNote.stringLength = sizeIncludingNullTerminator - 1;\n            pMetadata->data.labelOrNote.pString      = (char*)ma_dr_wav__metadata_get_memory(pParser, sizeIncludingNullTerminator, 1);\n            MA_DR_WAV_ASSERT(pMetadata->data.labelOrNote.pString != NULL);\n            ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.labelOrNote.pString, sizeIncludingNullTerminator, &totalBytesRead);\n        } else {\n            pMetadata->data.labelOrNote.stringLength = 0;\n            pMetadata->data.labelOrNote.pString      = NULL;\n        }\n    }\n    return totalBytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(ma_dr_wav__metadata_parser* pParser, ma_dr_wav_metadata* pMetadata, ma_uint64 chunkSize)\n{\n    ma_uint8 buffer[MA_DR_WAV_LIST_LABELLED_TEXT_BYTES];\n    ma_uint64 totalBytesRead = 0;\n    size_t bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &totalBytesRead);\n    MA_DR_WAV_ASSERT(pParser->stage == ma_dr_wav__metadata_parser_stage_read);\n    if (bytesJustRead == sizeof(buffer)) {\n        ma_uint32 sizeIncludingNullTerminator;\n        pMetadata->type                                = ma_dr_wav_metadata_type_list_labelled_cue_region;\n        pMetadata->data.labelledCueRegion.cuePointId   = ma_dr_wav_bytes_to_u32(buffer + 0);\n        pMetadata->data.labelledCueRegion.sampleLength = ma_dr_wav_bytes_to_u32(buffer + 4);\n        pMetadata->data.labelledCueRegion.purposeId[0] = buffer[8];\n        pMetadata->data.labelledCueRegion.purposeId[1] = buffer[9];\n        pMetadata->data.labelledCueRegion.purposeId[2] = buffer[10];\n        pMetadata->data.labelledCueRegion.purposeId[3] = buffer[11];\n        pMetadata->data.labelledCueRegion.country      = ma_dr_wav_bytes_to_u16(buffer + 12);\n        pMetadata->data.labelledCueRegion.language     = ma_dr_wav_bytes_to_u16(buffer + 14);\n        pMetadata->data.labelledCueRegion.dialect      = ma_dr_wav_bytes_to_u16(buffer + 16);\n        pMetadata->data.labelledCueRegion.codePage     = ma_dr_wav_bytes_to_u16(buffer + 18);\n        sizeIncludingNullTerminator = (ma_uint32)chunkSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES;\n        if (sizeIncludingNullTerminator > 0) {\n            pMetadata->data.labelledCueRegion.stringLength = sizeIncludingNullTerminator - 1;\n            pMetadata->data.labelledCueRegion.pString      = (char*)ma_dr_wav__metadata_get_memory(pParser, sizeIncludingNullTerminator, 1);\n            MA_DR_WAV_ASSERT(pMetadata->data.labelledCueRegion.pString != NULL);\n            ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.labelledCueRegion.pString, sizeIncludingNullTerminator, &totalBytesRead);\n        } else {\n            pMetadata->data.labelledCueRegion.stringLength = 0;\n            pMetadata->data.labelledCueRegion.pString      = NULL;\n        }\n    }\n    return totalBytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_info_text_chunk(ma_dr_wav__metadata_parser* pParser, ma_uint64 chunkSize, ma_dr_wav_metadata_type type)\n{\n    ma_uint64 bytesRead = 0;\n    ma_uint32 stringSizeWithNullTerminator = (ma_uint32)chunkSize;\n    if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n        pParser->metadataCount += 1;\n        ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, stringSizeWithNullTerminator, 1);\n    } else {\n        ma_dr_wav_metadata* pMetadata = &pParser->pMetadata[pParser->metadataCursor];\n        pMetadata->type = type;\n        if (stringSizeWithNullTerminator > 0) {\n            pMetadata->data.infoText.stringLength = stringSizeWithNullTerminator - 1;\n            pMetadata->data.infoText.pString = (char*)ma_dr_wav__metadata_get_memory(pParser, stringSizeWithNullTerminator, 1);\n            MA_DR_WAV_ASSERT(pMetadata->data.infoText.pString != NULL);\n            bytesRead = ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.infoText.pString, (size_t)stringSizeWithNullTerminator, NULL);\n            if (bytesRead == chunkSize) {\n                pParser->metadataCursor += 1;\n            } else {\n            }\n        } else {\n            pMetadata->data.infoText.stringLength = 0;\n            pMetadata->data.infoText.pString      = NULL;\n            pParser->metadataCursor += 1;\n        }\n    }\n    return bytesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_unknown_chunk(ma_dr_wav__metadata_parser* pParser, const ma_uint8* pChunkId, ma_uint64 chunkSize, ma_dr_wav_metadata_location location)\n{\n    ma_uint64 bytesRead = 0;\n    if (location == ma_dr_wav_metadata_location_invalid) {\n        return 0;\n    }\n    if (ma_dr_wav_fourcc_equal(pChunkId, \"data\") || ma_dr_wav_fourcc_equal(pChunkId, \"fmt \") || ma_dr_wav_fourcc_equal(pChunkId, \"fact\")) {\n        return 0;\n    }\n    if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n        pParser->metadataCount += 1;\n        ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)chunkSize, 1);\n    } else {\n        ma_dr_wav_metadata* pMetadata = &pParser->pMetadata[pParser->metadataCursor];\n        pMetadata->type                         = ma_dr_wav_metadata_type_unknown;\n        pMetadata->data.unknown.chunkLocation   = location;\n        pMetadata->data.unknown.id[0]           = pChunkId[0];\n        pMetadata->data.unknown.id[1]           = pChunkId[1];\n        pMetadata->data.unknown.id[2]           = pChunkId[2];\n        pMetadata->data.unknown.id[3]           = pChunkId[3];\n        pMetadata->data.unknown.dataSizeInBytes = (ma_uint32)chunkSize;\n        pMetadata->data.unknown.pData           = (ma_uint8 *)ma_dr_wav__metadata_get_memory(pParser, (size_t)chunkSize, 1);\n        MA_DR_WAV_ASSERT(pMetadata->data.unknown.pData != NULL);\n        bytesRead = ma_dr_wav__metadata_parser_read(pParser, pMetadata->data.unknown.pData, pMetadata->data.unknown.dataSizeInBytes, NULL);\n        if (bytesRead == pMetadata->data.unknown.dataSizeInBytes) {\n            pParser->metadataCursor += 1;\n        } else {\n        }\n    }\n    return bytesRead;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__chunk_matches(ma_dr_wav_metadata_type allowedMetadataTypes, const ma_uint8* pChunkID, ma_dr_wav_metadata_type type, const char* pID)\n{\n    return (allowedMetadataTypes & type) && ma_dr_wav_fourcc_equal(pChunkID, pID);\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__metadata_process_chunk(ma_dr_wav__metadata_parser* pParser, const ma_dr_wav_chunk_header* pChunkHeader, ma_dr_wav_metadata_type allowedMetadataTypes)\n{\n    const ma_uint8 *pChunkID = pChunkHeader->id.fourcc;\n    ma_uint64 bytesRead = 0;\n    if (ma_dr_wav__chunk_matches(allowedMetadataTypes, pChunkID, ma_dr_wav_metadata_type_smpl, \"smpl\")) {\n        if (pChunkHeader->sizeInBytes >= MA_DR_WAV_SMPL_BYTES) {\n            if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                ma_uint8 buffer[4];\n                size_t bytesJustRead;\n                if (!pParser->onSeek(pParser->pReadSeekUserData, 28, MA_DR_WAV_SEEK_CUR)) {\n                    return bytesRead;\n                }\n                bytesRead += 28;\n                bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &bytesRead);\n                if (bytesJustRead == sizeof(buffer)) {\n                    ma_uint32 loopCount = ma_dr_wav_bytes_to_u32(buffer);\n                    ma_uint64 calculatedLoopCount;\n                    calculatedLoopCount = (pChunkHeader->sizeInBytes - MA_DR_WAV_SMPL_BYTES) / MA_DR_WAV_SMPL_LOOP_BYTES;\n                    if (calculatedLoopCount == loopCount) {\n                        bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, sizeof(buffer), &bytesRead);\n                        if (bytesJustRead == sizeof(buffer)) {\n                            ma_uint32 samplerSpecificDataSizeInBytes = ma_dr_wav_bytes_to_u32(buffer);\n                            pParser->metadataCount += 1;\n                            ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, sizeof(ma_dr_wav_smpl_loop) * loopCount, MA_DR_WAV_METADATA_ALIGNMENT);\n                            ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, samplerSpecificDataSizeInBytes, 1);\n                        }\n                    } else {\n                    }\n                }\n            } else {\n                bytesRead = ma_dr_wav__read_smpl_to_metadata_obj(pParser, pChunkHeader, &pParser->pMetadata[pParser->metadataCursor]);\n                if (bytesRead == pChunkHeader->sizeInBytes) {\n                    pParser->metadataCursor += 1;\n                } else {\n                }\n            }\n        } else {\n        }\n    } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, pChunkID, ma_dr_wav_metadata_type_inst, \"inst\")) {\n        if (pChunkHeader->sizeInBytes == MA_DR_WAV_INST_BYTES) {\n            if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                pParser->metadataCount += 1;\n            } else {\n                bytesRead = ma_dr_wav__read_inst_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor]);\n                if (bytesRead == pChunkHeader->sizeInBytes) {\n                    pParser->metadataCursor += 1;\n                } else {\n                }\n            }\n        } else {\n        }\n    } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, pChunkID, ma_dr_wav_metadata_type_acid, \"acid\")) {\n        if (pChunkHeader->sizeInBytes == MA_DR_WAV_ACID_BYTES) {\n            if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                pParser->metadataCount += 1;\n            } else {\n                bytesRead = ma_dr_wav__read_acid_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor]);\n                if (bytesRead == pChunkHeader->sizeInBytes) {\n                    pParser->metadataCursor += 1;\n                } else {\n                }\n            }\n        } else {\n        }\n    } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, pChunkID, ma_dr_wav_metadata_type_cue, \"cue \")) {\n        if (pChunkHeader->sizeInBytes >= MA_DR_WAV_CUE_BYTES) {\n            if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                size_t cueCount;\n                pParser->metadataCount += 1;\n                cueCount = (size_t)(pChunkHeader->sizeInBytes - MA_DR_WAV_CUE_BYTES) / MA_DR_WAV_CUE_POINT_BYTES;\n                ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, sizeof(ma_dr_wav_cue_point) * cueCount, MA_DR_WAV_METADATA_ALIGNMENT);\n            } else {\n                bytesRead = ma_dr_wav__read_cue_to_metadata_obj(pParser, pChunkHeader, &pParser->pMetadata[pParser->metadataCursor]);\n                if (bytesRead == pChunkHeader->sizeInBytes) {\n                    pParser->metadataCursor += 1;\n                } else {\n                }\n            }\n        } else {\n        }\n    } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, pChunkID, ma_dr_wav_metadata_type_bext, \"bext\")) {\n        if (pChunkHeader->sizeInBytes >= MA_DR_WAV_BEXT_BYTES) {\n            if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                char buffer[MA_DR_WAV_BEXT_DESCRIPTION_BYTES + 1];\n                size_t allocSizeNeeded = MA_DR_WAV_BEXT_UMID_BYTES;\n                size_t bytesJustRead;\n                buffer[MA_DR_WAV_BEXT_DESCRIPTION_BYTES] = '\\0';\n                bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, MA_DR_WAV_BEXT_DESCRIPTION_BYTES, &bytesRead);\n                if (bytesJustRead != MA_DR_WAV_BEXT_DESCRIPTION_BYTES) {\n                    return bytesRead;\n                }\n                allocSizeNeeded += ma_dr_wav__strlen(buffer) + 1;\n                buffer[MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES] = '\\0';\n                bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES, &bytesRead);\n                if (bytesJustRead != MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES) {\n                    return bytesRead;\n                }\n                allocSizeNeeded += ma_dr_wav__strlen(buffer) + 1;\n                buffer[MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES] = '\\0';\n                bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, buffer, MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES, &bytesRead);\n                if (bytesJustRead != MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES) {\n                    return bytesRead;\n                }\n                allocSizeNeeded += ma_dr_wav__strlen(buffer) + 1;\n                allocSizeNeeded += (size_t)pChunkHeader->sizeInBytes - MA_DR_WAV_BEXT_BYTES + 1;\n                ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, allocSizeNeeded, 1);\n                pParser->metadataCount += 1;\n            } else {\n                bytesRead = ma_dr_wav__read_bext_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], pChunkHeader->sizeInBytes);\n                if (bytesRead == pChunkHeader->sizeInBytes) {\n                    pParser->metadataCursor += 1;\n                } else {\n                }\n            }\n        } else {\n        }\n    } else if (ma_dr_wav_fourcc_equal(pChunkID, \"LIST\") || ma_dr_wav_fourcc_equal(pChunkID, \"list\")) {\n        ma_dr_wav_metadata_location listType = ma_dr_wav_metadata_location_invalid;\n        while (bytesRead < pChunkHeader->sizeInBytes) {\n            ma_uint8 subchunkId[4];\n            ma_uint8 subchunkSizeBuffer[4];\n            ma_uint64 subchunkDataSize;\n            ma_uint64 subchunkBytesRead = 0;\n            ma_uint64 bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, subchunkId, sizeof(subchunkId), &bytesRead);\n            if (bytesJustRead != sizeof(subchunkId)) {\n                break;\n            }\n            if (ma_dr_wav_fourcc_equal(subchunkId, \"adtl\")) {\n                listType = ma_dr_wav_metadata_location_inside_adtl_list;\n                continue;\n            } else if (ma_dr_wav_fourcc_equal(subchunkId, \"INFO\")) {\n                listType = ma_dr_wav_metadata_location_inside_info_list;\n                continue;\n            }\n            bytesJustRead = ma_dr_wav__metadata_parser_read(pParser, subchunkSizeBuffer, sizeof(subchunkSizeBuffer), &bytesRead);\n            if (bytesJustRead != sizeof(subchunkSizeBuffer)) {\n                break;\n            }\n            subchunkDataSize = ma_dr_wav_bytes_to_u32(subchunkSizeBuffer);\n            if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_label, \"labl\") || ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_note, \"note\")) {\n                if (subchunkDataSize >= MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES) {\n                    ma_uint64 stringSizeWithNullTerm = subchunkDataSize - MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES;\n                    if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                        pParser->metadataCount += 1;\n                        ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)stringSizeWithNullTerm, 1);\n                    } else {\n                        subchunkBytesRead = ma_dr_wav__read_list_label_or_note_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], subchunkDataSize, ma_dr_wav_fourcc_equal(subchunkId, \"labl\") ? ma_dr_wav_metadata_type_list_label : ma_dr_wav_metadata_type_list_note);\n                        if (subchunkBytesRead == subchunkDataSize) {\n                            pParser->metadataCursor += 1;\n                        } else {\n                        }\n                    }\n                } else {\n                }\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_labelled_cue_region, \"ltxt\")) {\n                if (subchunkDataSize >= MA_DR_WAV_LIST_LABELLED_TEXT_BYTES) {\n                    ma_uint64 stringSizeWithNullTerminator = subchunkDataSize - MA_DR_WAV_LIST_LABELLED_TEXT_BYTES;\n                    if (pParser->stage == ma_dr_wav__metadata_parser_stage_count) {\n                        pParser->metadataCount += 1;\n                        ma_dr_wav__metadata_request_extra_memory_for_stage_2(pParser, (size_t)stringSizeWithNullTerminator, 1);\n                    } else {\n                        subchunkBytesRead = ma_dr_wav__read_list_labelled_cue_region_to_metadata_obj(pParser, &pParser->pMetadata[pParser->metadataCursor], subchunkDataSize);\n                        if (subchunkBytesRead == subchunkDataSize) {\n                            pParser->metadataCursor += 1;\n                        } else {\n                        }\n                    }\n                } else {\n                }\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_software, \"ISFT\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_software);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_copyright, \"ICOP\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_copyright);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_title, \"INAM\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_title);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_artist, \"IART\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_artist);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_comment, \"ICMT\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_comment);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_date, \"ICRD\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_date);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_genre, \"IGNR\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_genre);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_album, \"IPRD\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_album);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_tracknumber, \"ITRK\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_tracknumber);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_location, \"IARL\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_location);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_organization, \"ICMS\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_organization);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_keywords, \"IKEY\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_keywords);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_medium, \"IMED\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_medium);\n            } else if (ma_dr_wav__chunk_matches(allowedMetadataTypes, subchunkId, ma_dr_wav_metadata_type_list_info_description, \"ISBJ\")) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_info_text_chunk(pParser, subchunkDataSize,  ma_dr_wav_metadata_type_list_info_description);\n            } else if ((allowedMetadataTypes & ma_dr_wav_metadata_type_unknown) != 0) {\n                subchunkBytesRead = ma_dr_wav__metadata_process_unknown_chunk(pParser, subchunkId, subchunkDataSize, listType);\n            }\n            bytesRead += subchunkBytesRead;\n            MA_DR_WAV_ASSERT(subchunkBytesRead <= subchunkDataSize);\n            if (subchunkBytesRead < subchunkDataSize) {\n                ma_uint64 bytesToSeek = subchunkDataSize - subchunkBytesRead;\n                if (!pParser->onSeek(pParser->pReadSeekUserData, (int)bytesToSeek, MA_DR_WAV_SEEK_CUR)) {\n                    break;\n                }\n                bytesRead += bytesToSeek;\n            }\n            if ((subchunkDataSize % 2) == 1) {\n                if (!pParser->onSeek(pParser->pReadSeekUserData, 1, MA_DR_WAV_SEEK_CUR)) {\n                    break;\n                }\n                bytesRead += 1;\n            }\n        }\n    } else if ((allowedMetadataTypes & ma_dr_wav_metadata_type_unknown) != 0) {\n        bytesRead = ma_dr_wav__metadata_process_unknown_chunk(pParser, pChunkID, pChunkHeader->sizeInBytes, ma_dr_wav_metadata_location_top_level);\n    }\n    return bytesRead;\n}\nMA_PRIVATE ma_uint32 ma_dr_wav_get_bytes_per_pcm_frame(ma_dr_wav* pWav)\n{\n    ma_uint32 bytesPerFrame;\n    if ((pWav->bitsPerSample & 0x7) == 0) {\n        bytesPerFrame = (pWav->bitsPerSample * pWav->fmt.channels) >> 3;\n    } else {\n        bytesPerFrame = pWav->fmt.blockAlign;\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ALAW || pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_MULAW) {\n        if (bytesPerFrame != pWav->fmt.channels) {\n            return 0;\n        }\n    }\n    return bytesPerFrame;\n}\nMA_API ma_uint16 ma_dr_wav_fmt_get_format(const ma_dr_wav_fmt* pFMT)\n{\n    if (pFMT == NULL) {\n        return 0;\n    }\n    if (pFMT->formatTag != MA_DR_WAVE_FORMAT_EXTENSIBLE) {\n        return pFMT->formatTag;\n    } else {\n        return ma_dr_wav_bytes_to_u16(pFMT->subFormat);\n    }\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_preinit(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pReadSeekTellUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pWav == NULL || onRead == NULL || onSeek == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_WAV_ZERO_MEMORY(pWav, sizeof(*pWav));\n    pWav->onRead    = onRead;\n    pWav->onSeek    = onSeek;\n    pWav->onTell    = onTell;\n    pWav->pUserData = pReadSeekTellUserData;\n    pWav->allocationCallbacks = ma_dr_wav_copy_allocation_callbacks_or_defaults(pAllocationCallbacks);\n    if (pWav->allocationCallbacks.onFree == NULL || (pWav->allocationCallbacks.onMalloc == NULL && pWav->allocationCallbacks.onRealloc == NULL)) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_init__internal(ma_dr_wav* pWav, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags)\n{\n    ma_result result;\n    ma_uint64 cursor;\n    ma_bool32 sequential;\n    ma_uint8 riff[4];\n    ma_dr_wav_fmt fmt;\n    unsigned short translatedFormatTag;\n    ma_uint64 dataChunkSize = 0;\n    ma_uint64 sampleCountFromFactChunk = 0;\n    ma_uint64 metadataStartPos;\n    ma_dr_wav__metadata_parser metadataParser;\n    ma_bool8 isProcessingMetadata = MA_FALSE;\n    ma_bool8 foundChunk_fmt  = MA_FALSE;\n    ma_bool8 foundChunk_data = MA_FALSE;\n    ma_bool8 isAIFCFormType = MA_FALSE;\n    ma_uint64 aiffFrameCount = 0;\n    cursor = 0;\n    sequential = (flags & MA_DR_WAV_SEQUENTIAL) != 0;\n    MA_DR_WAV_ZERO_OBJECT(&fmt);\n    if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, riff, sizeof(riff), &cursor) != sizeof(riff)) {\n        return MA_FALSE;\n    }\n    if (ma_dr_wav_fourcc_equal(riff, \"RIFF\")) {\n        pWav->container = ma_dr_wav_container_riff;\n    } else if (ma_dr_wav_fourcc_equal(riff, \"RIFX\")) {\n        pWav->container = ma_dr_wav_container_rifx;\n    } else if (ma_dr_wav_fourcc_equal(riff, \"riff\")) {\n        int i;\n        ma_uint8 riff2[12];\n        pWav->container = ma_dr_wav_container_w64;\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, riff2, sizeof(riff2), &cursor) != sizeof(riff2)) {\n            return MA_FALSE;\n        }\n        for (i = 0; i < 12; ++i) {\n            if (riff2[i] != ma_dr_wavGUID_W64_RIFF[i+4]) {\n                return MA_FALSE;\n            }\n        }\n    } else if (ma_dr_wav_fourcc_equal(riff, \"RF64\")) {\n        pWav->container = ma_dr_wav_container_rf64;\n    } else if (ma_dr_wav_fourcc_equal(riff, \"FORM\")) {\n        pWav->container = ma_dr_wav_container_aiff;\n    } else {\n        return MA_FALSE;\n    }\n    if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx || pWav->container == ma_dr_wav_container_rf64) {\n        ma_uint8 chunkSizeBytes[4];\n        ma_uint8 wave[4];\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, chunkSizeBytes, sizeof(chunkSizeBytes), &cursor) != sizeof(chunkSizeBytes)) {\n            return MA_FALSE;\n        }\n        if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx) {\n            if (ma_dr_wav_bytes_to_u32_ex(chunkSizeBytes, pWav->container) < 36) {\n            }\n        } else if (pWav->container == ma_dr_wav_container_rf64) {\n            if (ma_dr_wav_bytes_to_u32_le(chunkSizeBytes) != 0xFFFFFFFF) {\n                return MA_FALSE;\n            }\n        } else {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, wave, sizeof(wave), &cursor) != sizeof(wave)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_wav_fourcc_equal(wave, \"WAVE\")) {\n            return MA_FALSE;\n        }\n    } else if (pWav->container == ma_dr_wav_container_w64) {\n        ma_uint8 chunkSizeBytes[8];\n        ma_uint8 wave[16];\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, chunkSizeBytes, sizeof(chunkSizeBytes), &cursor) != sizeof(chunkSizeBytes)) {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav_bytes_to_u64(chunkSizeBytes) < 80) {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, wave, sizeof(wave), &cursor) != sizeof(wave)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_wav_guid_equal(wave, ma_dr_wavGUID_W64_WAVE)) {\n            return MA_FALSE;\n        }\n    } else if (pWav->container == ma_dr_wav_container_aiff) {\n        ma_uint8 chunkSizeBytes[4];\n        ma_uint8 aiff[4];\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, chunkSizeBytes, sizeof(chunkSizeBytes), &cursor) != sizeof(chunkSizeBytes)) {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav_bytes_to_u32_be(chunkSizeBytes) < 18) {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, aiff, sizeof(aiff), &cursor) != sizeof(aiff)) {\n            return MA_FALSE;\n        }\n        if (ma_dr_wav_fourcc_equal(aiff, \"AIFF\")) {\n            isAIFCFormType = MA_FALSE;\n        } else if (ma_dr_wav_fourcc_equal(aiff, \"AIFC\")) {\n            isAIFCFormType = MA_TRUE;\n        } else {\n            return MA_FALSE;\n        }\n    } else {\n        return MA_FALSE;\n    }\n    if (pWav->container == ma_dr_wav_container_rf64) {\n        ma_uint8 sizeBytes[8];\n        ma_uint64 bytesRemainingInChunk;\n        ma_dr_wav_chunk_header header;\n        result = ma_dr_wav__read_chunk_header(pWav->onRead, pWav->pUserData, pWav->container, &cursor, &header);\n        if (result != MA_SUCCESS) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_wav_fourcc_equal(header.id.fourcc, \"ds64\")) {\n            return MA_FALSE;\n        }\n        bytesRemainingInChunk = header.sizeInBytes + header.paddingSize;\n        if (!ma_dr_wav__seek_forward(pWav->onSeek, 8, pWav->pUserData)) {\n            return MA_FALSE;\n        }\n        bytesRemainingInChunk -= 8;\n        cursor += 8;\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, sizeBytes, sizeof(sizeBytes), &cursor) != sizeof(sizeBytes)) {\n            return MA_FALSE;\n        }\n        bytesRemainingInChunk -= 8;\n        dataChunkSize = ma_dr_wav_bytes_to_u64(sizeBytes);\n        if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, sizeBytes, sizeof(sizeBytes), &cursor) != sizeof(sizeBytes)) {\n            return MA_FALSE;\n        }\n        bytesRemainingInChunk -= 8;\n        sampleCountFromFactChunk = ma_dr_wav_bytes_to_u64(sizeBytes);\n        if (!ma_dr_wav__seek_forward(pWav->onSeek, bytesRemainingInChunk, pWav->pUserData)) {\n            return MA_FALSE;\n        }\n        cursor += bytesRemainingInChunk;\n    }\n    metadataStartPos = cursor;\n    isProcessingMetadata = !sequential && ((flags & MA_DR_WAV_WITH_METADATA) != 0);\n    if (pWav->container != ma_dr_wav_container_riff && pWav->container != ma_dr_wav_container_rf64) {\n        isProcessingMetadata = MA_FALSE;\n    }\n    MA_DR_WAV_ZERO_MEMORY(&metadataParser, sizeof(metadataParser));\n    if (isProcessingMetadata) {\n        metadataParser.onRead = pWav->onRead;\n        metadataParser.onSeek = pWav->onSeek;\n        metadataParser.pReadSeekUserData = pWav->pUserData;\n        metadataParser.stage  = ma_dr_wav__metadata_parser_stage_count;\n    }\n    for (;;) {\n        ma_dr_wav_chunk_header header;\n        ma_uint64 chunkSize;\n        result = ma_dr_wav__read_chunk_header(pWav->onRead, pWav->pUserData, pWav->container, &cursor, &header);\n        if (result != MA_SUCCESS) {\n            break;\n        }\n        chunkSize = header.sizeInBytes;\n        if (!sequential && onChunk != NULL) {\n            ma_uint64 callbackBytesRead = onChunk(pChunkUserData, pWav->onRead, pWav->onSeek, pWav->pUserData, &header, pWav->container, &fmt);\n            if (callbackBytesRead > 0) {\n                if (ma_dr_wav__seek_from_start(pWav->onSeek, cursor, pWav->pUserData) == MA_FALSE) {\n                    return MA_FALSE;\n                }\n            }\n        }\n        if (((pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx || pWav->container == ma_dr_wav_container_rf64) && ma_dr_wav_fourcc_equal(header.id.fourcc, \"fmt \")) ||\n            ((pWav->container == ma_dr_wav_container_w64) && ma_dr_wav_guid_equal(header.id.guid, ma_dr_wavGUID_W64_FMT))) {\n            ma_uint8 fmtData[16];\n            foundChunk_fmt = MA_TRUE;\n            if (pWav->onRead(pWav->pUserData, fmtData, sizeof(fmtData)) != sizeof(fmtData)) {\n                return MA_FALSE;\n            }\n            cursor += sizeof(fmtData);\n            fmt.formatTag      = ma_dr_wav_bytes_to_u16_ex(fmtData + 0,  pWav->container);\n            fmt.channels       = ma_dr_wav_bytes_to_u16_ex(fmtData + 2,  pWav->container);\n            fmt.sampleRate     = ma_dr_wav_bytes_to_u32_ex(fmtData + 4,  pWav->container);\n            fmt.avgBytesPerSec = ma_dr_wav_bytes_to_u32_ex(fmtData + 8,  pWav->container);\n            fmt.blockAlign     = ma_dr_wav_bytes_to_u16_ex(fmtData + 12, pWav->container);\n            fmt.bitsPerSample  = ma_dr_wav_bytes_to_u16_ex(fmtData + 14, pWav->container);\n            fmt.extendedSize       = 0;\n            fmt.validBitsPerSample = 0;\n            fmt.channelMask        = 0;\n            MA_DR_WAV_ZERO_MEMORY(fmt.subFormat, sizeof(fmt.subFormat));\n            if (header.sizeInBytes > 16) {\n                ma_uint8 fmt_cbSize[2];\n                int bytesReadSoFar = 0;\n                if (pWav->onRead(pWav->pUserData, fmt_cbSize, sizeof(fmt_cbSize)) != sizeof(fmt_cbSize)) {\n                    return MA_FALSE;\n                }\n                cursor += sizeof(fmt_cbSize);\n                bytesReadSoFar = 18;\n                fmt.extendedSize = ma_dr_wav_bytes_to_u16_ex(fmt_cbSize, pWav->container);\n                if (fmt.extendedSize > 0) {\n                    if (fmt.formatTag == MA_DR_WAVE_FORMAT_EXTENSIBLE) {\n                        if (fmt.extendedSize != 22) {\n                            return MA_FALSE;\n                        }\n                    }\n                    if (fmt.formatTag == MA_DR_WAVE_FORMAT_EXTENSIBLE) {\n                        ma_uint8 fmtext[22];\n                        if (pWav->onRead(pWav->pUserData, fmtext, fmt.extendedSize) != fmt.extendedSize) {\n                            return MA_FALSE;\n                        }\n                        fmt.validBitsPerSample = ma_dr_wav_bytes_to_u16_ex(fmtext + 0, pWav->container);\n                        fmt.channelMask        = ma_dr_wav_bytes_to_u32_ex(fmtext + 2, pWav->container);\n                        ma_dr_wav_bytes_to_guid(fmtext + 6, fmt.subFormat);\n                    } else {\n                        if (pWav->onSeek(pWav->pUserData, fmt.extendedSize, MA_DR_WAV_SEEK_CUR) == MA_FALSE) {\n                            return MA_FALSE;\n                        }\n                    }\n                    cursor += fmt.extendedSize;\n                    bytesReadSoFar += fmt.extendedSize;\n                }\n                if (pWav->onSeek(pWav->pUserData, (int)(header.sizeInBytes - bytesReadSoFar), MA_DR_WAV_SEEK_CUR) == MA_FALSE) {\n                    return MA_FALSE;\n                }\n                cursor += (header.sizeInBytes - bytesReadSoFar);\n            }\n            if (header.paddingSize > 0) {\n                if (ma_dr_wav__seek_forward(pWav->onSeek, header.paddingSize, pWav->pUserData) == MA_FALSE) {\n                    break;\n                }\n                cursor += header.paddingSize;\n            }\n            continue;\n        }\n        if (((pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx || pWav->container == ma_dr_wav_container_rf64) && ma_dr_wav_fourcc_equal(header.id.fourcc, \"data\")) ||\n            ((pWav->container == ma_dr_wav_container_w64) && ma_dr_wav_guid_equal(header.id.guid, ma_dr_wavGUID_W64_DATA))) {\n            foundChunk_data = MA_TRUE;\n            pWav->dataChunkDataPos  = cursor;\n            if (pWav->container != ma_dr_wav_container_rf64) {\n                dataChunkSize = chunkSize;\n            }\n            if (sequential || !isProcessingMetadata) {\n                break;\n            } else {\n                chunkSize += header.paddingSize;\n                if (ma_dr_wav__seek_forward(pWav->onSeek, chunkSize, pWav->pUserData) == MA_FALSE) {\n                    break;\n                }\n                cursor += chunkSize;\n                continue;\n            }\n        }\n        if (((pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx || pWav->container == ma_dr_wav_container_rf64) && ma_dr_wav_fourcc_equal(header.id.fourcc, \"fact\")) ||\n            ((pWav->container == ma_dr_wav_container_w64) && ma_dr_wav_guid_equal(header.id.guid, ma_dr_wavGUID_W64_FACT))) {\n            if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx) {\n                ma_uint8 sampleCount[4];\n                if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, &sampleCount, 4, &cursor) != 4) {\n                    return MA_FALSE;\n                }\n                chunkSize -= 4;\n                if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n                    sampleCountFromFactChunk = ma_dr_wav_bytes_to_u32_ex(sampleCount, pWav->container);\n                } else {\n                    sampleCountFromFactChunk = 0;\n                }\n            } else if (pWav->container == ma_dr_wav_container_w64) {\n                if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, &sampleCountFromFactChunk, 8, &cursor) != 8) {\n                    return MA_FALSE;\n                }\n                chunkSize -= 8;\n            } else if (pWav->container == ma_dr_wav_container_rf64) {\n            }\n            chunkSize += header.paddingSize;\n            if (ma_dr_wav__seek_forward(pWav->onSeek, chunkSize, pWav->pUserData) == MA_FALSE) {\n                break;\n            }\n            cursor += chunkSize;\n            continue;\n        }\n        if (pWav->container == ma_dr_wav_container_aiff && ma_dr_wav_fourcc_equal(header.id.fourcc, \"COMM\")) {\n            ma_uint8 commData[24];\n            ma_uint32 commDataBytesToRead;\n            ma_uint16 channels;\n            ma_uint32 frameCount;\n            ma_uint16 sampleSizeInBits;\n            ma_int64  sampleRate;\n            ma_uint16 compressionFormat;\n            foundChunk_fmt = MA_TRUE;\n            if (isAIFCFormType) {\n                commDataBytesToRead = 24;\n                if (header.sizeInBytes < commDataBytesToRead) {\n                    return MA_FALSE;\n                }\n            } else {\n                commDataBytesToRead = 18;\n                if (header.sizeInBytes != commDataBytesToRead) {\n                    return MA_FALSE;\n                }\n            }\n            if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, commData, commDataBytesToRead, &cursor) != commDataBytesToRead) {\n                return MA_FALSE;\n            }\n            channels         = ma_dr_wav_bytes_to_u16_ex     (commData + 0, pWav->container);\n            frameCount       = ma_dr_wav_bytes_to_u32_ex     (commData + 2, pWav->container);\n            sampleSizeInBits = ma_dr_wav_bytes_to_u16_ex     (commData + 6, pWav->container);\n            sampleRate       = ma_dr_wav_aiff_extented_to_s64(commData + 8);\n            if (sampleRate < 0 || sampleRate > 0xFFFFFFFF) {\n                return MA_FALSE;\n            }\n            if (isAIFCFormType) {\n                const ma_uint8* type = commData + 18;\n                if (ma_dr_wav_fourcc_equal(type, \"NONE\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_PCM;\n                } else if (ma_dr_wav_fourcc_equal(type, \"raw \")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_PCM;\n                    if (sampleSizeInBits == 8) {\n                        pWav->aiff.isUnsigned = MA_TRUE;\n                    }\n                } else if (ma_dr_wav_fourcc_equal(type, \"sowt\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_PCM;\n                    pWav->aiff.isLE = MA_TRUE;\n                } else if (ma_dr_wav_fourcc_equal(type, \"fl32\") || ma_dr_wav_fourcc_equal(type, \"fl64\") || ma_dr_wav_fourcc_equal(type, \"FL32\") || ma_dr_wav_fourcc_equal(type, \"FL64\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_IEEE_FLOAT;\n                } else if (ma_dr_wav_fourcc_equal(type, \"alaw\") || ma_dr_wav_fourcc_equal(type, \"ALAW\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_ALAW;\n                } else if (ma_dr_wav_fourcc_equal(type, \"ulaw\") || ma_dr_wav_fourcc_equal(type, \"ULAW\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_MULAW;\n                } else if (ma_dr_wav_fourcc_equal(type, \"ima4\")) {\n                    compressionFormat = MA_DR_WAVE_FORMAT_DVI_ADPCM;\n                    sampleSizeInBits  = 4;\n                    (void)compressionFormat;\n                    (void)sampleSizeInBits;\n                    return MA_FALSE;\n                } else {\n                    return MA_FALSE;\n                }\n            } else {\n                compressionFormat = MA_DR_WAVE_FORMAT_PCM;\n            }\n            aiffFrameCount = frameCount;\n            fmt.formatTag      = compressionFormat;\n            fmt.channels       = channels;\n            fmt.sampleRate     = (ma_uint32)sampleRate;\n            fmt.bitsPerSample  = sampleSizeInBits;\n            fmt.blockAlign     = (ma_uint16)(fmt.channels * fmt.bitsPerSample / 8);\n            fmt.avgBytesPerSec = fmt.blockAlign * fmt.sampleRate;\n            if (fmt.blockAlign == 0 && compressionFormat == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n                fmt.blockAlign = 34 * fmt.channels;\n            }\n            if (compressionFormat == MA_DR_WAVE_FORMAT_ALAW || compressionFormat == MA_DR_WAVE_FORMAT_MULAW) {\n                if (fmt.bitsPerSample > 8) {\n                    fmt.bitsPerSample = 8;\n                    fmt.blockAlign = fmt.channels;\n                }\n            }\n            fmt.bitsPerSample += (fmt.bitsPerSample & 7);\n            if (isAIFCFormType) {\n                if (ma_dr_wav__seek_forward(pWav->onSeek, (chunkSize - commDataBytesToRead), pWav->pUserData) == MA_FALSE) {\n                    return MA_FALSE;\n                }\n                cursor += (chunkSize - commDataBytesToRead);\n            }\n            continue;\n        }\n        if (pWav->container == ma_dr_wav_container_aiff && ma_dr_wav_fourcc_equal(header.id.fourcc, \"SSND\")) {\n            ma_uint8 offsetAndBlockSizeData[8];\n            ma_uint32 offset;\n            foundChunk_data = MA_TRUE;\n            if (ma_dr_wav__on_read(pWav->onRead, pWav->pUserData, offsetAndBlockSizeData, sizeof(offsetAndBlockSizeData), &cursor) != sizeof(offsetAndBlockSizeData)) {\n                return MA_FALSE;\n            }\n            offset = ma_dr_wav_bytes_to_u32_ex(offsetAndBlockSizeData + 0, pWav->container);\n            pWav->dataChunkDataPos = cursor + offset;\n            dataChunkSize = chunkSize;\n            if (dataChunkSize  > offset) {\n                dataChunkSize -= offset;\n            } else {\n                dataChunkSize = 0;\n            }\n            if (sequential) {\n                if (foundChunk_fmt) {\n                    if (ma_dr_wav__seek_forward(pWav->onSeek, offset, pWav->pUserData) == MA_FALSE) {\n                        return MA_FALSE;\n                    }\n                    cursor += offset;\n                    break;\n                } else {\n                    return MA_FALSE;\n                }\n            } else {\n                chunkSize += header.paddingSize;\n                chunkSize -= sizeof(offsetAndBlockSizeData);\n                if (ma_dr_wav__seek_forward(pWav->onSeek, chunkSize, pWav->pUserData) == MA_FALSE) {\n                    break;\n                }\n                cursor += chunkSize;\n                continue;\n            }\n        }\n        if (isProcessingMetadata) {\n            ma_dr_wav__metadata_process_chunk(&metadataParser, &header, ma_dr_wav_metadata_type_all_including_unknown);\n            if (ma_dr_wav__seek_from_start(pWav->onSeek, cursor, pWav->pUserData) == MA_FALSE) {\n                break;\n            }\n        }\n        chunkSize += header.paddingSize;\n        if (ma_dr_wav__seek_forward(pWav->onSeek, chunkSize, pWav->pUserData) == MA_FALSE) {\n            break;\n        }\n        cursor += chunkSize;\n    }\n    if (!foundChunk_fmt || !foundChunk_data) {\n        return MA_FALSE;\n    }\n    if ((fmt.sampleRate    == 0 || fmt.sampleRate    > MA_DR_WAV_MAX_SAMPLE_RATE    ) ||\n        (fmt.channels      == 0 || fmt.channels      > MA_DR_WAV_MAX_CHANNELS       ) ||\n        (fmt.bitsPerSample == 0 || fmt.bitsPerSample > MA_DR_WAV_MAX_BITS_PER_SAMPLE) ||\n        fmt.blockAlign == 0) {\n        return MA_FALSE;\n    }\n    translatedFormatTag = fmt.formatTag;\n    if (translatedFormatTag == MA_DR_WAVE_FORMAT_EXTENSIBLE) {\n        translatedFormatTag = ma_dr_wav_bytes_to_u16_ex(fmt.subFormat + 0, pWav->container);\n    }\n    if (!sequential) {\n        if (!ma_dr_wav__seek_from_start(pWav->onSeek, pWav->dataChunkDataPos, pWav->pUserData)) {\n            return MA_FALSE;\n        }\n        cursor = pWav->dataChunkDataPos;\n    }\n    if (isProcessingMetadata && metadataParser.metadataCount > 0) {\n        if (ma_dr_wav__seek_from_start(pWav->onSeek, metadataStartPos, pWav->pUserData) == MA_FALSE) {\n            return MA_FALSE;\n        }\n        result = ma_dr_wav__metadata_alloc(&metadataParser, &pWav->allocationCallbacks);\n        if (result != MA_SUCCESS) {\n            return MA_FALSE;\n        }\n        metadataParser.stage = ma_dr_wav__metadata_parser_stage_read;\n        for (;;) {\n            ma_dr_wav_chunk_header header;\n            ma_uint64 metadataBytesRead;\n            result = ma_dr_wav__read_chunk_header(pWav->onRead, pWav->pUserData, pWav->container, &cursor, &header);\n            if (result != MA_SUCCESS) {\n                break;\n            }\n            metadataBytesRead = ma_dr_wav__metadata_process_chunk(&metadataParser, &header, ma_dr_wav_metadata_type_all_including_unknown);\n            if (ma_dr_wav__seek_forward(pWav->onSeek, (header.sizeInBytes + header.paddingSize) - metadataBytesRead, pWav->pUserData) == MA_FALSE) {\n                ma_dr_wav_free(metadataParser.pMetadata, &pWav->allocationCallbacks);\n                return MA_FALSE;\n            }\n        }\n        pWav->pMetadata     = metadataParser.pMetadata;\n        pWav->metadataCount = metadataParser.metadataCount;\n    }\n    if (pWav->onTell != NULL && pWav->onSeek != NULL) {\n        if (pWav->onSeek(pWav->pUserData, 0, MA_DR_WAV_SEEK_END) == MA_TRUE) {\n            ma_int64 fileSize;\n            if (pWav->onTell(pWav->pUserData, &fileSize)) {\n                if (dataChunkSize + pWav->dataChunkDataPos > (ma_uint64)fileSize) {\n                    dataChunkSize = (ma_uint64)fileSize - pWav->dataChunkDataPos;\n                }\n            }\n        } else {\n        }\n    }\n    if (dataChunkSize == 0xFFFFFFFF && (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rifx) && pWav->isSequentialWrite == MA_FALSE) {\n        dataChunkSize = 0;\n        for (;;) {\n            ma_uint8 temp[4096];\n            size_t bytesRead = pWav->onRead(pWav->pUserData, temp, sizeof(temp));\n            dataChunkSize += bytesRead;\n            if (bytesRead < sizeof(temp)) {\n                break;\n            }\n        }\n    }\n    if (ma_dr_wav__seek_from_start(pWav->onSeek, pWav->dataChunkDataPos, pWav->pUserData) == MA_FALSE) {\n        ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks);\n        return MA_FALSE;\n    }\n    pWav->fmt                 = fmt;\n    pWav->sampleRate          = fmt.sampleRate;\n    pWav->channels            = fmt.channels;\n    pWav->bitsPerSample       = fmt.bitsPerSample;\n    pWav->translatedFormatTag = translatedFormatTag;\n    if (!ma_dr_wav__is_compressed_format_tag(translatedFormatTag)) {\n        ma_uint32 bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n        if (bytesPerFrame > 0) {\n            dataChunkSize -= (dataChunkSize % bytesPerFrame);\n        }\n    }\n    pWav->bytesRemaining      = dataChunkSize;\n    pWav->dataChunkDataSize   = dataChunkSize;\n    if (sampleCountFromFactChunk != 0) {\n        pWav->totalPCMFrameCount = sampleCountFromFactChunk;\n    } else if (aiffFrameCount != 0) {\n        pWav->totalPCMFrameCount = aiffFrameCount;\n    } else {\n        ma_uint32 bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n        if (bytesPerFrame == 0) {\n            ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks);\n            return MA_FALSE;\n        }\n        pWav->totalPCMFrameCount = dataChunkSize / bytesPerFrame;\n        if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n            ma_uint64 totalBlockHeaderSizeInBytes;\n            ma_uint64 blockCount = dataChunkSize / fmt.blockAlign;\n            if ((blockCount * fmt.blockAlign) < dataChunkSize) {\n                blockCount += 1;\n            }\n            totalBlockHeaderSizeInBytes = blockCount * (6*fmt.channels);\n            pWav->totalPCMFrameCount = ((dataChunkSize - totalBlockHeaderSizeInBytes) * 2) / fmt.channels;\n        }\n        if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n            ma_uint64 totalBlockHeaderSizeInBytes;\n            ma_uint64 blockCount = dataChunkSize / fmt.blockAlign;\n            if ((blockCount * fmt.blockAlign) < dataChunkSize) {\n                blockCount += 1;\n            }\n            totalBlockHeaderSizeInBytes = blockCount * (4*fmt.channels);\n            pWav->totalPCMFrameCount = ((dataChunkSize - totalBlockHeaderSizeInBytes) * 2) / fmt.channels;\n            pWav->totalPCMFrameCount += blockCount;\n        }\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM || pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        if (pWav->channels > 2) {\n            ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks);\n            return MA_FALSE;\n        }\n    }\n    if (ma_dr_wav_get_bytes_per_pcm_frame(pWav) == 0) {\n        ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks);\n        return MA_FALSE;\n    }\n#ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n        ma_uint64 blockCount = dataChunkSize / fmt.blockAlign;\n        pWav->totalPCMFrameCount = (((blockCount * (fmt.blockAlign - (6*pWav->channels))) * 2)) / fmt.channels;\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        ma_uint64 blockCount = dataChunkSize / fmt.blockAlign;\n        pWav->totalPCMFrameCount = (((blockCount * (fmt.blockAlign - (4*pWav->channels))) * 2) + (blockCount * pWav->channels)) / fmt.channels;\n    }\n#endif\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_init(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_ex(pWav, onRead, onSeek, onTell, NULL, pUserData, NULL, 0, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_ex(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, ma_dr_wav_chunk_proc onChunk, void* pReadSeekTellUserData, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (!ma_dr_wav_preinit(pWav, onRead, onSeek, onTell, pReadSeekTellUserData, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init__internal(pWav, onChunk, pChunkUserData, flags);\n}\nMA_API ma_bool32 ma_dr_wav_init_with_metadata(ma_dr_wav* pWav, ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (!ma_dr_wav_preinit(pWav, onRead, onSeek, onTell, pUserData, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init__internal(pWav, NULL, NULL, flags | MA_DR_WAV_WITH_METADATA);\n}\nMA_API ma_dr_wav_metadata* ma_dr_wav_take_ownership_of_metadata(ma_dr_wav* pWav)\n{\n    ma_dr_wav_metadata *result = pWav->pMetadata;\n    pWav->pMetadata     = NULL;\n    pWav->metadataCount = 0;\n    return result;\n}\nMA_PRIVATE size_t ma_dr_wav__write(ma_dr_wav* pWav, const void* pData, size_t dataSize)\n{\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    return pWav->onWrite(pWav->pUserData, pData, dataSize);\n}\nMA_PRIVATE size_t ma_dr_wav__write_byte(ma_dr_wav* pWav, ma_uint8 byte)\n{\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    return pWav->onWrite(pWav->pUserData, &byte, 1);\n}\nMA_PRIVATE size_t ma_dr_wav__write_u16ne_to_le(ma_dr_wav* pWav, ma_uint16 value)\n{\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    if (!ma_dr_wav__is_little_endian()) {\n        value = ma_dr_wav__bswap16(value);\n    }\n    return ma_dr_wav__write(pWav, &value, 2);\n}\nMA_PRIVATE size_t ma_dr_wav__write_u32ne_to_le(ma_dr_wav* pWav, ma_uint32 value)\n{\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    if (!ma_dr_wav__is_little_endian()) {\n        value = ma_dr_wav__bswap32(value);\n    }\n    return ma_dr_wav__write(pWav, &value, 4);\n}\nMA_PRIVATE size_t ma_dr_wav__write_u64ne_to_le(ma_dr_wav* pWav, ma_uint64 value)\n{\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    if (!ma_dr_wav__is_little_endian()) {\n        value = ma_dr_wav__bswap64(value);\n    }\n    return ma_dr_wav__write(pWav, &value, 8);\n}\nMA_PRIVATE size_t ma_dr_wav__write_f32ne_to_le(ma_dr_wav* pWav, float value)\n{\n    union {\n       ma_uint32 u32;\n       float f32;\n    } u;\n    MA_DR_WAV_ASSERT(pWav          != NULL);\n    MA_DR_WAV_ASSERT(pWav->onWrite != NULL);\n    u.f32 = value;\n    if (!ma_dr_wav__is_little_endian()) {\n        u.u32 = ma_dr_wav__bswap32(u.u32);\n    }\n    return ma_dr_wav__write(pWav, &u.u32, 4);\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count(ma_dr_wav* pWav, const void* pData, size_t dataSize)\n{\n    if (pWav == NULL) {\n        return dataSize;\n    }\n    return ma_dr_wav__write(pWav, pData, dataSize);\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count_byte(ma_dr_wav* pWav, ma_uint8 byte)\n{\n    if (pWav == NULL) {\n        return 1;\n    }\n    return ma_dr_wav__write_byte(pWav, byte);\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count_u16ne_to_le(ma_dr_wav* pWav, ma_uint16 value)\n{\n    if (pWav == NULL) {\n        return 2;\n    }\n    return ma_dr_wav__write_u16ne_to_le(pWav, value);\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count_u32ne_to_le(ma_dr_wav* pWav, ma_uint32 value)\n{\n    if (pWav == NULL) {\n        return 4;\n    }\n    return ma_dr_wav__write_u32ne_to_le(pWav, value);\n}\n#if 0\nMA_PRIVATE size_t ma_dr_wav__write_or_count_u64ne_to_le(ma_dr_wav* pWav, ma_uint64 value)\n{\n    if (pWav == NULL) {\n        return 8;\n    }\n    return ma_dr_wav__write_u64ne_to_le(pWav, value);\n}\n#endif\nMA_PRIVATE size_t ma_dr_wav__write_or_count_f32ne_to_le(ma_dr_wav* pWav, float value)\n{\n    if (pWav == NULL) {\n        return 4;\n    }\n    return ma_dr_wav__write_f32ne_to_le(pWav, value);\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count_string_to_fixed_size_buf(ma_dr_wav* pWav, char* str, size_t bufFixedSize)\n{\n    size_t len;\n    if (pWav == NULL) {\n        return bufFixedSize;\n    }\n    len = ma_dr_wav__strlen_clamped(str, bufFixedSize);\n    ma_dr_wav__write_or_count(pWav, str, len);\n    if (len < bufFixedSize) {\n        size_t i;\n        for (i = 0; i < bufFixedSize - len; ++i) {\n            ma_dr_wav__write_byte(pWav, 0);\n        }\n    }\n    return bufFixedSize;\n}\nMA_PRIVATE size_t ma_dr_wav__write_or_count_metadata(ma_dr_wav* pWav, ma_dr_wav_metadata* pMetadatas, ma_uint32 metadataCount)\n{\n    size_t bytesWritten = 0;\n    ma_bool32 hasListAdtl = MA_FALSE;\n    ma_bool32 hasListInfo = MA_FALSE;\n    ma_uint32 iMetadata;\n    if (pMetadatas == NULL || metadataCount == 0) {\n        return 0;\n    }\n    for (iMetadata = 0; iMetadata < metadataCount; ++iMetadata) {\n        ma_dr_wav_metadata* pMetadata = &pMetadatas[iMetadata];\n        ma_uint32 chunkSize = 0;\n        if ((pMetadata->type & ma_dr_wav_metadata_type_list_all_info_strings) || (pMetadata->type == ma_dr_wav_metadata_type_unknown && pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_info_list)) {\n            hasListInfo = MA_TRUE;\n        }\n        if ((pMetadata->type & ma_dr_wav_metadata_type_list_all_adtl) || (pMetadata->type == ma_dr_wav_metadata_type_unknown && pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_adtl_list)) {\n            hasListAdtl = MA_TRUE;\n        }\n        switch (pMetadata->type) {\n            case ma_dr_wav_metadata_type_smpl:\n            {\n                ma_uint32 iLoop;\n                chunkSize = MA_DR_WAV_SMPL_BYTES + MA_DR_WAV_SMPL_LOOP_BYTES * pMetadata->data.smpl.sampleLoopCount + pMetadata->data.smpl.samplerSpecificDataSizeInBytes;\n                bytesWritten += ma_dr_wav__write_or_count(pWav, \"smpl\", 4);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.manufacturerId);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.productId);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.samplePeriodNanoseconds);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.midiUnityNote);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.midiPitchFraction);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.smpteFormat);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.smpteOffset);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.sampleLoopCount);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.samplerSpecificDataSizeInBytes);\n                for (iLoop = 0; iLoop < pMetadata->data.smpl.sampleLoopCount; ++iLoop) {\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].cuePointId);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].type);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].firstSampleOffset);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].lastSampleOffset);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].sampleFraction);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.smpl.pLoops[iLoop].playCount);\n                }\n                if (pMetadata->data.smpl.samplerSpecificDataSizeInBytes > 0) {\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.smpl.pSamplerSpecificData, pMetadata->data.smpl.samplerSpecificDataSizeInBytes);\n                }\n            } break;\n            case ma_dr_wav_metadata_type_inst:\n            {\n                chunkSize = MA_DR_WAV_INST_BYTES;\n                bytesWritten += ma_dr_wav__write_or_count(pWav, \"inst\", 4);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.midiUnityNote, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.fineTuneCents, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.gainDecibels, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.lowNote, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.highNote, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.lowVelocity, 1);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, &pMetadata->data.inst.highVelocity, 1);\n            } break;\n            case ma_dr_wav_metadata_type_cue:\n            {\n                ma_uint32 iCuePoint;\n                chunkSize = MA_DR_WAV_CUE_BYTES + MA_DR_WAV_CUE_POINT_BYTES * pMetadata->data.cue.cuePointCount;\n                bytesWritten += ma_dr_wav__write_or_count(pWav, \"cue \", 4);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.cuePointCount);\n                for (iCuePoint = 0; iCuePoint < pMetadata->data.cue.cuePointCount; ++iCuePoint) {\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].id);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].playOrderPosition);\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].dataChunkId, 4);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].chunkStart);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].blockStart);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.cue.pCuePoints[iCuePoint].sampleOffset);\n                }\n            } break;\n            case ma_dr_wav_metadata_type_acid:\n            {\n                chunkSize = MA_DR_WAV_ACID_BYTES;\n                bytesWritten += ma_dr_wav__write_or_count(pWav, \"acid\", 4);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.acid.flags);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.acid.midiUnityNote);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.acid.reserved1);\n                bytesWritten += ma_dr_wav__write_or_count_f32ne_to_le(pWav, pMetadata->data.acid.reserved2);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.acid.numBeats);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.acid.meterDenominator);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.acid.meterNumerator);\n                bytesWritten += ma_dr_wav__write_or_count_f32ne_to_le(pWav, pMetadata->data.acid.tempo);\n            } break;\n            case ma_dr_wav_metadata_type_bext:\n            {\n                char reservedBuf[MA_DR_WAV_BEXT_RESERVED_BYTES];\n                ma_uint32 timeReferenceLow;\n                ma_uint32 timeReferenceHigh;\n                chunkSize = MA_DR_WAV_BEXT_BYTES + pMetadata->data.bext.codingHistorySize;\n                bytesWritten += ma_dr_wav__write_or_count(pWav, \"bext\", 4);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                bytesWritten += ma_dr_wav__write_or_count_string_to_fixed_size_buf(pWav, pMetadata->data.bext.pDescription, MA_DR_WAV_BEXT_DESCRIPTION_BYTES);\n                bytesWritten += ma_dr_wav__write_or_count_string_to_fixed_size_buf(pWav, pMetadata->data.bext.pOriginatorName, MA_DR_WAV_BEXT_ORIGINATOR_NAME_BYTES);\n                bytesWritten += ma_dr_wav__write_or_count_string_to_fixed_size_buf(pWav, pMetadata->data.bext.pOriginatorReference, MA_DR_WAV_BEXT_ORIGINATOR_REF_BYTES);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.bext.pOriginationDate, sizeof(pMetadata->data.bext.pOriginationDate));\n                bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.bext.pOriginationTime, sizeof(pMetadata->data.bext.pOriginationTime));\n                timeReferenceLow  = (ma_uint32)(pMetadata->data.bext.timeReference & 0xFFFFFFFF);\n                timeReferenceHigh = (ma_uint32)(pMetadata->data.bext.timeReference >> 32);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, timeReferenceLow);\n                bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, timeReferenceHigh);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.version);\n                bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.bext.pUMID, MA_DR_WAV_BEXT_UMID_BYTES);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.loudnessValue);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.loudnessRange);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.maxTruePeakLevel);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.maxMomentaryLoudness);\n                bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.bext.maxShortTermLoudness);\n                MA_DR_WAV_ZERO_MEMORY(reservedBuf, sizeof(reservedBuf));\n                bytesWritten += ma_dr_wav__write_or_count(pWav, reservedBuf, sizeof(reservedBuf));\n                if (pMetadata->data.bext.codingHistorySize > 0) {\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.bext.pCodingHistory, pMetadata->data.bext.codingHistorySize);\n                }\n            } break;\n            case ma_dr_wav_metadata_type_unknown:\n            {\n                if (pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_top_level) {\n                    chunkSize = pMetadata->data.unknown.dataSizeInBytes;\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.id, 4);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.pData, pMetadata->data.unknown.dataSizeInBytes);\n                }\n            } break;\n            default: break;\n        }\n        if ((chunkSize % 2) != 0) {\n            bytesWritten += ma_dr_wav__write_or_count_byte(pWav, 0);\n        }\n    }\n    if (hasListInfo) {\n        ma_uint32 chunkSize = 4;\n        for (iMetadata = 0; iMetadata < metadataCount; ++iMetadata) {\n            ma_dr_wav_metadata* pMetadata = &pMetadatas[iMetadata];\n            if ((pMetadata->type & ma_dr_wav_metadata_type_list_all_info_strings)) {\n                chunkSize += 8;\n                chunkSize += pMetadata->data.infoText.stringLength + 1;\n            } else if (pMetadata->type == ma_dr_wav_metadata_type_unknown && pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_info_list) {\n                chunkSize += 8;\n                chunkSize += pMetadata->data.unknown.dataSizeInBytes;\n            }\n            if ((chunkSize % 2) != 0) {\n                chunkSize += 1;\n            }\n        }\n        bytesWritten += ma_dr_wav__write_or_count(pWav, \"LIST\", 4);\n        bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n        bytesWritten += ma_dr_wav__write_or_count(pWav, \"INFO\", 4);\n        for (iMetadata = 0; iMetadata < metadataCount; ++iMetadata) {\n            ma_dr_wav_metadata* pMetadata = &pMetadatas[iMetadata];\n            ma_uint32 subchunkSize = 0;\n            if (pMetadata->type & ma_dr_wav_metadata_type_list_all_info_strings) {\n                const char* pID = NULL;\n                switch (pMetadata->type) {\n                    case ma_dr_wav_metadata_type_list_info_software:     pID = \"ISFT\"; break;\n                    case ma_dr_wav_metadata_type_list_info_copyright:    pID = \"ICOP\"; break;\n                    case ma_dr_wav_metadata_type_list_info_title:        pID = \"INAM\"; break;\n                    case ma_dr_wav_metadata_type_list_info_artist:       pID = \"IART\"; break;\n                    case ma_dr_wav_metadata_type_list_info_comment:      pID = \"ICMT\"; break;\n                    case ma_dr_wav_metadata_type_list_info_date:         pID = \"ICRD\"; break;\n                    case ma_dr_wav_metadata_type_list_info_genre:        pID = \"IGNR\"; break;\n                    case ma_dr_wav_metadata_type_list_info_album:        pID = \"IPRD\"; break;\n                    case ma_dr_wav_metadata_type_list_info_tracknumber:  pID = \"ITRK\"; break;\n                    case ma_dr_wav_metadata_type_list_info_location:     pID = \"IARL\"; break;\n                    case ma_dr_wav_metadata_type_list_info_organization: pID = \"ICMS\"; break;\n                    case ma_dr_wav_metadata_type_list_info_keywords:     pID = \"IKEY\"; break;\n                    case ma_dr_wav_metadata_type_list_info_medium:       pID = \"IMED\"; break;\n                    case ma_dr_wav_metadata_type_list_info_description:  pID = \"ISBJ\"; break;\n                    default: break;\n                }\n                MA_DR_WAV_ASSERT(pID != NULL);\n                if (pMetadata->data.infoText.stringLength) {\n                    subchunkSize = pMetadata->data.infoText.stringLength + 1;\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pID, 4);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, subchunkSize);\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.infoText.pString, pMetadata->data.infoText.stringLength);\n                    bytesWritten += ma_dr_wav__write_or_count_byte(pWav, '\\0');\n                }\n            } else if (pMetadata->type == ma_dr_wav_metadata_type_unknown && pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_info_list) {\n                if (pMetadata->data.unknown.dataSizeInBytes) {\n                    subchunkSize = pMetadata->data.unknown.dataSizeInBytes;\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.id, 4);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.unknown.dataSizeInBytes);\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.pData, subchunkSize);\n                }\n            }\n            if ((subchunkSize % 2) != 0) {\n                bytesWritten += ma_dr_wav__write_or_count_byte(pWav, 0);\n            }\n        }\n    }\n    if (hasListAdtl) {\n        ma_uint32 chunkSize = 4;\n        for (iMetadata = 0; iMetadata < metadataCount; ++iMetadata) {\n            ma_dr_wav_metadata* pMetadata = &pMetadatas[iMetadata];\n            switch (pMetadata->type)\n            {\n                case ma_dr_wav_metadata_type_list_label:\n                case ma_dr_wav_metadata_type_list_note:\n                {\n                    chunkSize += 8;\n                    chunkSize += MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES;\n                    if (pMetadata->data.labelOrNote.stringLength > 0) {\n                        chunkSize += pMetadata->data.labelOrNote.stringLength + 1;\n                    }\n                } break;\n                case ma_dr_wav_metadata_type_list_labelled_cue_region:\n                {\n                    chunkSize += 8;\n                    chunkSize += MA_DR_WAV_LIST_LABELLED_TEXT_BYTES;\n                    if (pMetadata->data.labelledCueRegion.stringLength > 0) {\n                        chunkSize += pMetadata->data.labelledCueRegion.stringLength + 1;\n                    }\n                } break;\n                case ma_dr_wav_metadata_type_unknown:\n                {\n                    if (pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_adtl_list) {\n                        chunkSize += 8;\n                        chunkSize += pMetadata->data.unknown.dataSizeInBytes;\n                    }\n                } break;\n                default: break;\n            }\n            if ((chunkSize % 2) != 0) {\n                chunkSize += 1;\n            }\n        }\n        bytesWritten += ma_dr_wav__write_or_count(pWav, \"LIST\", 4);\n        bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, chunkSize);\n        bytesWritten += ma_dr_wav__write_or_count(pWav, \"adtl\", 4);\n        for (iMetadata = 0; iMetadata < metadataCount; ++iMetadata) {\n            ma_dr_wav_metadata* pMetadata = &pMetadatas[iMetadata];\n            ma_uint32 subchunkSize = 0;\n            switch (pMetadata->type)\n            {\n                case ma_dr_wav_metadata_type_list_label:\n                case ma_dr_wav_metadata_type_list_note:\n                {\n                    if (pMetadata->data.labelOrNote.stringLength > 0) {\n                        const char *pID = NULL;\n                        if (pMetadata->type == ma_dr_wav_metadata_type_list_label) {\n                            pID = \"labl\";\n                        }\n                        else if (pMetadata->type == ma_dr_wav_metadata_type_list_note) {\n                            pID = \"note\";\n                        }\n                        MA_DR_WAV_ASSERT(pID != NULL);\n                        MA_DR_WAV_ASSERT(pMetadata->data.labelOrNote.pString != NULL);\n                        subchunkSize = MA_DR_WAV_LIST_LABEL_OR_NOTE_BYTES;\n                        bytesWritten += ma_dr_wav__write_or_count(pWav, pID, 4);\n                        subchunkSize += pMetadata->data.labelOrNote.stringLength + 1;\n                        bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, subchunkSize);\n                        bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.labelOrNote.cuePointId);\n                        bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.labelOrNote.pString, pMetadata->data.labelOrNote.stringLength);\n                        bytesWritten += ma_dr_wav__write_or_count_byte(pWav, '\\0');\n                    }\n                } break;\n                case ma_dr_wav_metadata_type_list_labelled_cue_region:\n                {\n                    subchunkSize = MA_DR_WAV_LIST_LABELLED_TEXT_BYTES;\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, \"ltxt\", 4);\n                    if (pMetadata->data.labelledCueRegion.stringLength > 0) {\n                        subchunkSize += pMetadata->data.labelledCueRegion.stringLength + 1;\n                    }\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, subchunkSize);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.labelledCueRegion.cuePointId);\n                    bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, pMetadata->data.labelledCueRegion.sampleLength);\n                    bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.labelledCueRegion.purposeId, 4);\n                    bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.labelledCueRegion.country);\n                    bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.labelledCueRegion.language);\n                    bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.labelledCueRegion.dialect);\n                    bytesWritten += ma_dr_wav__write_or_count_u16ne_to_le(pWav, pMetadata->data.labelledCueRegion.codePage);\n                    if (pMetadata->data.labelledCueRegion.stringLength > 0) {\n                        MA_DR_WAV_ASSERT(pMetadata->data.labelledCueRegion.pString != NULL);\n                        bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.labelledCueRegion.pString, pMetadata->data.labelledCueRegion.stringLength);\n                        bytesWritten += ma_dr_wav__write_or_count_byte(pWav, '\\0');\n                    }\n                } break;\n                case ma_dr_wav_metadata_type_unknown:\n                {\n                    if (pMetadata->data.unknown.chunkLocation == ma_dr_wav_metadata_location_inside_adtl_list) {\n                        subchunkSize = pMetadata->data.unknown.dataSizeInBytes;\n                        MA_DR_WAV_ASSERT(pMetadata->data.unknown.pData != NULL);\n                        bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.id, 4);\n                        bytesWritten += ma_dr_wav__write_or_count_u32ne_to_le(pWav, subchunkSize);\n                        bytesWritten += ma_dr_wav__write_or_count(pWav, pMetadata->data.unknown.pData, subchunkSize);\n                    }\n                } break;\n                default: break;\n            }\n            if ((subchunkSize % 2) != 0) {\n                bytesWritten += ma_dr_wav__write_or_count_byte(pWav, 0);\n            }\n        }\n    }\n    MA_DR_WAV_ASSERT((bytesWritten % 2) == 0);\n    return bytesWritten;\n}\nMA_PRIVATE ma_uint32 ma_dr_wav__riff_chunk_size_riff(ma_uint64 dataChunkSize, ma_dr_wav_metadata* pMetadata, ma_uint32 metadataCount)\n{\n    ma_uint64 chunkSize = 4 + 24 + (ma_uint64)ma_dr_wav__write_or_count_metadata(NULL, pMetadata, metadataCount) + 8 + dataChunkSize + ma_dr_wav__chunk_padding_size_riff(dataChunkSize);\n    if (chunkSize > 0xFFFFFFFFUL) {\n        chunkSize = 0xFFFFFFFFUL;\n    }\n    return (ma_uint32)chunkSize;\n}\nMA_PRIVATE ma_uint32 ma_dr_wav__data_chunk_size_riff(ma_uint64 dataChunkSize)\n{\n    if (dataChunkSize <= 0xFFFFFFFFUL) {\n        return (ma_uint32)dataChunkSize;\n    } else {\n        return 0xFFFFFFFFUL;\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__riff_chunk_size_w64(ma_uint64 dataChunkSize)\n{\n    ma_uint64 dataSubchunkPaddingSize = ma_dr_wav__chunk_padding_size_w64(dataChunkSize);\n    return 80 + 24 + dataChunkSize + dataSubchunkPaddingSize;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__data_chunk_size_w64(ma_uint64 dataChunkSize)\n{\n    return 24 + dataChunkSize;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__riff_chunk_size_rf64(ma_uint64 dataChunkSize, ma_dr_wav_metadata *metadata, ma_uint32 numMetadata)\n{\n    ma_uint64 chunkSize = 4 + 36 + 24 + (ma_uint64)ma_dr_wav__write_or_count_metadata(NULL, metadata, numMetadata) + 8 + dataChunkSize + ma_dr_wav__chunk_padding_size_riff(dataChunkSize);\n    if (chunkSize > 0xFFFFFFFFUL) {\n        chunkSize = 0xFFFFFFFFUL;\n    }\n    return chunkSize;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav__data_chunk_size_rf64(ma_uint64 dataChunkSize)\n{\n    return dataChunkSize;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_preinit_write(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_bool32 isSequential, ma_dr_wav_write_proc onWrite, ma_dr_wav_seek_proc onSeek, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pWav == NULL || onWrite == NULL) {\n        return MA_FALSE;\n    }\n    if (!isSequential && onSeek == NULL) {\n        return MA_FALSE;\n    }\n    if (pFormat->format == MA_DR_WAVE_FORMAT_EXTENSIBLE) {\n        return MA_FALSE;\n    }\n    if (pFormat->format == MA_DR_WAVE_FORMAT_ADPCM || pFormat->format == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        return MA_FALSE;\n    }\n    MA_DR_WAV_ZERO_MEMORY(pWav, sizeof(*pWav));\n    pWav->onWrite   = onWrite;\n    pWav->onSeek    = onSeek;\n    pWav->pUserData = pUserData;\n    pWav->allocationCallbacks = ma_dr_wav_copy_allocation_callbacks_or_defaults(pAllocationCallbacks);\n    if (pWav->allocationCallbacks.onFree == NULL || (pWav->allocationCallbacks.onMalloc == NULL && pWav->allocationCallbacks.onRealloc == NULL)) {\n        return MA_FALSE;\n    }\n    pWav->fmt.formatTag = (ma_uint16)pFormat->format;\n    pWav->fmt.channels = (ma_uint16)pFormat->channels;\n    pWav->fmt.sampleRate = pFormat->sampleRate;\n    pWav->fmt.avgBytesPerSec = (ma_uint32)((pFormat->bitsPerSample * pFormat->sampleRate * pFormat->channels) / 8);\n    pWav->fmt.blockAlign = (ma_uint16)((pFormat->channels * pFormat->bitsPerSample) / 8);\n    pWav->fmt.bitsPerSample = (ma_uint16)pFormat->bitsPerSample;\n    pWav->fmt.extendedSize = 0;\n    pWav->isSequentialWrite = isSequential;\n    return MA_TRUE;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_init_write__internal(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount)\n{\n    size_t runningPos = 0;\n    ma_uint64 initialDataChunkSize = 0;\n    ma_uint64 chunkSizeFMT;\n    if (pWav->isSequentialWrite) {\n        initialDataChunkSize = (totalSampleCount * pWav->fmt.bitsPerSample) / 8;\n        if (pFormat->container == ma_dr_wav_container_riff) {\n            if (initialDataChunkSize > (0xFFFFFFFFUL - 36)) {\n                return MA_FALSE;\n            }\n        }\n    }\n    pWav->dataChunkDataSizeTargetWrite = initialDataChunkSize;\n    if (pFormat->container == ma_dr_wav_container_riff) {\n        ma_uint32 chunkSizeRIFF = 36 + (ma_uint32)initialDataChunkSize;\n        runningPos += ma_dr_wav__write(pWav, \"RIFF\", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, chunkSizeRIFF);\n        runningPos += ma_dr_wav__write(pWav, \"WAVE\", 4);\n    } else if (pFormat->container == ma_dr_wav_container_w64) {\n        ma_uint64 chunkSizeRIFF = 80 + 24 + initialDataChunkSize;\n        runningPos += ma_dr_wav__write(pWav, ma_dr_wavGUID_W64_RIFF, 16);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, chunkSizeRIFF);\n        runningPos += ma_dr_wav__write(pWav, ma_dr_wavGUID_W64_WAVE, 16);\n    } else if (pFormat->container == ma_dr_wav_container_rf64) {\n        runningPos += ma_dr_wav__write(pWav, \"RF64\", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, 0xFFFFFFFF);\n        runningPos += ma_dr_wav__write(pWav, \"WAVE\", 4);\n    } else {\n        return MA_FALSE;\n    }\n    if (pFormat->container == ma_dr_wav_container_rf64) {\n        ma_uint32 initialds64ChunkSize = 28;\n        ma_uint64 initialRiffChunkSize = 8 + initialds64ChunkSize + initialDataChunkSize;\n        runningPos += ma_dr_wav__write(pWav, \"ds64\", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, initialds64ChunkSize);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, initialRiffChunkSize);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, initialDataChunkSize);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, totalSampleCount);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, 0);\n    }\n    if (pFormat->container == ma_dr_wav_container_riff || pFormat->container == ma_dr_wav_container_rf64) {\n        chunkSizeFMT = 16;\n        runningPos += ma_dr_wav__write(pWav, \"fmt \", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, (ma_uint32)chunkSizeFMT);\n    } else if (pFormat->container == ma_dr_wav_container_w64) {\n        chunkSizeFMT = 40;\n        runningPos += ma_dr_wav__write(pWav, ma_dr_wavGUID_W64_FMT, 16);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, chunkSizeFMT);\n    }\n    runningPos += ma_dr_wav__write_u16ne_to_le(pWav, pWav->fmt.formatTag);\n    runningPos += ma_dr_wav__write_u16ne_to_le(pWav, pWav->fmt.channels);\n    runningPos += ma_dr_wav__write_u32ne_to_le(pWav, pWav->fmt.sampleRate);\n    runningPos += ma_dr_wav__write_u32ne_to_le(pWav, pWav->fmt.avgBytesPerSec);\n    runningPos += ma_dr_wav__write_u16ne_to_le(pWav, pWav->fmt.blockAlign);\n    runningPos += ma_dr_wav__write_u16ne_to_le(pWav, pWav->fmt.bitsPerSample);\n    if (!pWav->isSequentialWrite && pWav->pMetadata != NULL && pWav->metadataCount > 0 && (pFormat->container == ma_dr_wav_container_riff || pFormat->container == ma_dr_wav_container_rf64)) {\n        runningPos += ma_dr_wav__write_or_count_metadata(pWav, pWav->pMetadata, pWav->metadataCount);\n    }\n    pWav->dataChunkDataPos = runningPos;\n    if (pFormat->container == ma_dr_wav_container_riff) {\n        ma_uint32 chunkSizeDATA = (ma_uint32)initialDataChunkSize;\n        runningPos += ma_dr_wav__write(pWav, \"data\", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, chunkSizeDATA);\n    } else if (pFormat->container == ma_dr_wav_container_w64) {\n        ma_uint64 chunkSizeDATA = 24 + initialDataChunkSize;\n        runningPos += ma_dr_wav__write(pWav, ma_dr_wavGUID_W64_DATA, 16);\n        runningPos += ma_dr_wav__write_u64ne_to_le(pWav, chunkSizeDATA);\n    } else if (pFormat->container == ma_dr_wav_container_rf64) {\n        runningPos += ma_dr_wav__write(pWav, \"data\", 4);\n        runningPos += ma_dr_wav__write_u32ne_to_le(pWav, 0xFFFFFFFF);\n    }\n    pWav->container = pFormat->container;\n    pWav->channels = (ma_uint16)pFormat->channels;\n    pWav->sampleRate = pFormat->sampleRate;\n    pWav->bitsPerSample = (ma_uint16)pFormat->bitsPerSample;\n    pWav->translatedFormatTag = (ma_uint16)pFormat->format;\n    pWav->dataChunkDataPos = runningPos;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_init_write(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_dr_wav_write_proc onWrite, ma_dr_wav_seek_proc onSeek, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (!ma_dr_wav_preinit_write(pWav, pFormat, MA_FALSE, onWrite, onSeek, pUserData, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_write__internal(pWav, pFormat, 0);\n}\nMA_API ma_bool32 ma_dr_wav_init_write_sequential(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_dr_wav_write_proc onWrite, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (!ma_dr_wav_preinit_write(pWav, pFormat, MA_TRUE, onWrite, NULL, pUserData, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_write__internal(pWav, pFormat, totalSampleCount);\n}\nMA_API ma_bool32 ma_dr_wav_init_write_sequential_pcm_frames(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, ma_dr_wav_write_proc onWrite, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFormat == NULL) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_write_sequential(pWav, pFormat, totalPCMFrameCount*pFormat->channels, onWrite, pUserData, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_write_with_metadata(ma_dr_wav* pWav, const ma_dr_wav_data_format* pFormat, ma_dr_wav_write_proc onWrite, ma_dr_wav_seek_proc onSeek, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks, ma_dr_wav_metadata* pMetadata, ma_uint32 metadataCount)\n{\n    if (!ma_dr_wav_preinit_write(pWav, pFormat, MA_FALSE, onWrite, onSeek, pUserData, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    pWav->pMetadata     = pMetadata;\n    pWav->metadataCount = metadataCount;\n    return ma_dr_wav_init_write__internal(pWav, pFormat, 0);\n}\nMA_API ma_uint64 ma_dr_wav_target_write_size_bytes(const ma_dr_wav_data_format* pFormat, ma_uint64 totalFrameCount, ma_dr_wav_metadata* pMetadata, ma_uint32 metadataCount)\n{\n    ma_uint64 targetDataSizeBytes = (ma_uint64)((ma_int64)totalFrameCount * pFormat->channels * pFormat->bitsPerSample/8.0);\n    ma_uint64 riffChunkSizeBytes;\n    ma_uint64 fileSizeBytes = 0;\n    if (pFormat->container == ma_dr_wav_container_riff) {\n        riffChunkSizeBytes = ma_dr_wav__riff_chunk_size_riff(targetDataSizeBytes, pMetadata, metadataCount);\n        fileSizeBytes = (8 + riffChunkSizeBytes);\n    } else if (pFormat->container == ma_dr_wav_container_w64) {\n        riffChunkSizeBytes = ma_dr_wav__riff_chunk_size_w64(targetDataSizeBytes);\n        fileSizeBytes = riffChunkSizeBytes;\n    } else if (pFormat->container == ma_dr_wav_container_rf64) {\n        riffChunkSizeBytes = ma_dr_wav__riff_chunk_size_rf64(targetDataSizeBytes, pMetadata, metadataCount);\n        fileSizeBytes = (8 + riffChunkSizeBytes);\n    }\n    return fileSizeBytes;\n}\n#ifndef MA_DR_WAV_NO_STDIO\nMA_PRIVATE size_t ma_dr_wav__on_read_stdio(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    return fread(pBufferOut, 1, bytesToRead, (FILE*)pUserData);\n}\nMA_PRIVATE size_t ma_dr_wav__on_write_stdio(void* pUserData, const void* pData, size_t bytesToWrite)\n{\n    return fwrite(pData, 1, bytesToWrite, (FILE*)pUserData);\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__on_seek_stdio(void* pUserData, int offset, ma_dr_wav_seek_origin origin)\n{\n    int whence = SEEK_SET;\n    if (origin == MA_DR_WAV_SEEK_CUR) {\n        whence = SEEK_CUR;\n    } else if (origin == MA_DR_WAV_SEEK_END) {\n        whence = SEEK_END;\n    }\n    return fseek((FILE*)pUserData, offset, whence) == 0;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__on_tell_stdio(void* pUserData, ma_int64* pCursor)\n{\n    FILE* pFileStdio = (FILE*)pUserData;\n    ma_int64 result;\n    MA_DR_WAV_ASSERT(pFileStdio != NULL);\n    MA_DR_WAV_ASSERT(pCursor    != NULL);\n#if defined(_WIN32) && !defined(NXDK)\n    #if defined(_MSC_VER) && _MSC_VER > 1200\n        result = _ftelli64(pFileStdio);\n    #else\n        result = ftell(pFileStdio);\n    #endif\n#else\n    result = ftell(pFileStdio);\n#endif\n    *pCursor = result;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_init_file(ma_dr_wav* pWav, const char* filename, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_ex(pWav, filename, NULL, NULL, 0, pAllocationCallbacks);\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_init_file__internal_FILE(ma_dr_wav* pWav, FILE* pFile, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 result;\n    result = ma_dr_wav_preinit(pWav, ma_dr_wav__on_read_stdio, ma_dr_wav__on_seek_stdio, ma_dr_wav__on_tell_stdio, (void*)pFile, pAllocationCallbacks);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    result = ma_dr_wav_init__internal(pWav, onChunk, pChunkUserData, flags);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_init_file_ex(ma_dr_wav* pWav, const char* filename, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_fopen(&pFile, filename, \"rb\") != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file__internal_FILE(pWav, pFile, onChunk, pChunkUserData, flags, pAllocationCallbacks);\n}\n#ifndef MA_DR_WAV_NO_WCHAR\nMA_API ma_bool32 ma_dr_wav_init_file_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_ex_w(pWav, filename, NULL, NULL, 0, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_file_ex_w(ma_dr_wav* pWav, const wchar_t* filename, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_wfopen(&pFile, filename, L\"rb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file__internal_FILE(pWav, pFile, onChunk, pChunkUserData, flags, pAllocationCallbacks);\n}\n#endif\nMA_API ma_bool32 ma_dr_wav_init_file_with_metadata(ma_dr_wav* pWav, const char* filename, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_fopen(&pFile, filename, \"rb\") != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file__internal_FILE(pWav, pFile, NULL, NULL, flags | MA_DR_WAV_WITH_METADATA, pAllocationCallbacks);\n}\n#ifndef MA_DR_WAV_NO_WCHAR\nMA_API ma_bool32 ma_dr_wav_init_file_with_metadata_w(ma_dr_wav* pWav, const wchar_t* filename, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_wfopen(&pFile, filename, L\"rb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file__internal_FILE(pWav, pFile, NULL, NULL, flags | MA_DR_WAV_WITH_METADATA, pAllocationCallbacks);\n}\n#endif\nMA_PRIVATE ma_bool32 ma_dr_wav_init_file_write__internal_FILE(ma_dr_wav* pWav, FILE* pFile, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_bool32 isSequential, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 result;\n    result = ma_dr_wav_preinit_write(pWav, pFormat, isSequential, ma_dr_wav__on_write_stdio, ma_dr_wav__on_seek_stdio, (void*)pFile, pAllocationCallbacks);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    result = ma_dr_wav_init_write__internal(pWav, pFormat, totalSampleCount);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    return MA_TRUE;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_init_file_write__internal(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_bool32 isSequential, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_fopen(&pFile, filename, \"wb\") != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file_write__internal_FILE(pWav, pFile, pFormat, totalSampleCount, isSequential, pAllocationCallbacks);\n}\n#ifndef MA_DR_WAV_NO_WCHAR\nMA_PRIVATE ma_bool32 ma_dr_wav_init_file_write_w__internal(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_bool32 isSequential, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    FILE* pFile;\n    if (ma_wfopen(&pFile, filename, L\"wb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file_write__internal_FILE(pWav, pFile, pFormat, totalSampleCount, isSequential, pAllocationCallbacks);\n}\n#endif\nMA_API ma_bool32 ma_dr_wav_init_file_write(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_write__internal(pWav, filename, pFormat, 0, MA_FALSE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_write__internal(pWav, filename, pFormat, totalSampleCount, MA_TRUE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_pcm_frames(ma_dr_wav* pWav, const char* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFormat == NULL) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file_write_sequential(pWav, filename, pFormat, totalPCMFrameCount*pFormat->channels, pAllocationCallbacks);\n}\n#ifndef MA_DR_WAV_NO_WCHAR\nMA_API ma_bool32 ma_dr_wav_init_file_write_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_write_w__internal(pWav, filename, pFormat, 0, MA_FALSE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_file_write_w__internal(pWav, filename, pFormat, totalSampleCount, MA_TRUE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_file_write_sequential_pcm_frames_w(ma_dr_wav* pWav, const wchar_t* filename, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFormat == NULL) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_file_write_sequential_w(pWav, filename, pFormat, totalPCMFrameCount*pFormat->channels, pAllocationCallbacks);\n}\n#endif\n#endif\nMA_PRIVATE size_t ma_dr_wav__on_read_memory(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    ma_dr_wav* pWav = (ma_dr_wav*)pUserData;\n    size_t bytesRemaining;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    MA_DR_WAV_ASSERT(pWav->memoryStream.dataSize >= pWav->memoryStream.currentReadPos);\n    bytesRemaining = pWav->memoryStream.dataSize - pWav->memoryStream.currentReadPos;\n    if (bytesToRead > bytesRemaining) {\n        bytesToRead = bytesRemaining;\n    }\n    if (bytesToRead > 0) {\n        MA_DR_WAV_COPY_MEMORY(pBufferOut, pWav->memoryStream.data + pWav->memoryStream.currentReadPos, bytesToRead);\n        pWav->memoryStream.currentReadPos += bytesToRead;\n    }\n    return bytesToRead;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__on_seek_memory(void* pUserData, int offset, ma_dr_wav_seek_origin origin)\n{\n    ma_dr_wav* pWav = (ma_dr_wav*)pUserData;\n    ma_int64 newCursor;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    if (origin == MA_DR_WAV_SEEK_SET) {\n        newCursor = 0;\n    } else if (origin == MA_DR_WAV_SEEK_CUR) {\n        newCursor = (ma_int64)pWav->memoryStream.currentReadPos;\n    } else if (origin == MA_DR_WAV_SEEK_END) {\n        newCursor = (ma_int64)pWav->memoryStream.dataSize;\n    } else {\n        MA_DR_WAV_ASSERT(!\"Invalid seek origin\");\n        return MA_FALSE;\n    }\n    newCursor += offset;\n    if (newCursor < 0) {\n        return MA_FALSE;\n    }\n    if ((size_t)newCursor > pWav->memoryStream.dataSize) {\n        return MA_FALSE;\n    }\n    pWav->memoryStream.currentReadPos = (size_t)newCursor;\n    return MA_TRUE;\n}\nMA_PRIVATE size_t ma_dr_wav__on_write_memory(void* pUserData, const void* pDataIn, size_t bytesToWrite)\n{\n    ma_dr_wav* pWav = (ma_dr_wav*)pUserData;\n    size_t bytesRemaining;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    MA_DR_WAV_ASSERT(pWav->memoryStreamWrite.dataCapacity >= pWav->memoryStreamWrite.currentWritePos);\n    bytesRemaining = pWav->memoryStreamWrite.dataCapacity - pWav->memoryStreamWrite.currentWritePos;\n    if (bytesRemaining < bytesToWrite) {\n        void* pNewData;\n        size_t newDataCapacity = (pWav->memoryStreamWrite.dataCapacity == 0) ? 256 : pWav->memoryStreamWrite.dataCapacity * 2;\n        if ((newDataCapacity - pWav->memoryStreamWrite.currentWritePos) < bytesToWrite) {\n            newDataCapacity = pWav->memoryStreamWrite.currentWritePos + bytesToWrite;\n        }\n        pNewData = ma_dr_wav__realloc_from_callbacks(*pWav->memoryStreamWrite.ppData, newDataCapacity, pWav->memoryStreamWrite.dataCapacity, &pWav->allocationCallbacks);\n        if (pNewData == NULL) {\n            return 0;\n        }\n        *pWav->memoryStreamWrite.ppData = pNewData;\n        pWav->memoryStreamWrite.dataCapacity = newDataCapacity;\n    }\n    MA_DR_WAV_COPY_MEMORY(((ma_uint8*)(*pWav->memoryStreamWrite.ppData)) + pWav->memoryStreamWrite.currentWritePos, pDataIn, bytesToWrite);\n    pWav->memoryStreamWrite.currentWritePos += bytesToWrite;\n    if (pWav->memoryStreamWrite.dataSize < pWav->memoryStreamWrite.currentWritePos) {\n        pWav->memoryStreamWrite.dataSize = pWav->memoryStreamWrite.currentWritePos;\n    }\n    *pWav->memoryStreamWrite.pDataSize = pWav->memoryStreamWrite.dataSize;\n    return bytesToWrite;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__on_seek_memory_write(void* pUserData, int offset, ma_dr_wav_seek_origin origin)\n{\n    ma_dr_wav* pWav = (ma_dr_wav*)pUserData;\n    ma_int64 newCursor;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    if (origin == MA_DR_WAV_SEEK_SET) {\n        newCursor = 0;\n    } else if (origin == MA_DR_WAV_SEEK_CUR) {\n        newCursor = (ma_int64)pWav->memoryStreamWrite.currentWritePos;\n    } else if (origin == MA_DR_WAV_SEEK_END) {\n        newCursor = (ma_int64)pWav->memoryStreamWrite.dataSize;\n    } else {\n        MA_DR_WAV_ASSERT(!\"Invalid seek origin\");\n        return MA_FALSE;\n    }\n    newCursor += offset;\n    if (newCursor < 0) {\n        return MA_FALSE;\n    }\n    if ((size_t)newCursor > pWav->memoryStreamWrite.dataSize) {\n        return MA_FALSE;\n    }\n    pWav->memoryStreamWrite.currentWritePos = (size_t)newCursor;\n    return MA_TRUE;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav__on_tell_memory(void* pUserData, ma_int64* pCursor)\n{\n    ma_dr_wav* pWav = (ma_dr_wav*)pUserData;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    MA_DR_WAV_ASSERT(pCursor != NULL);\n    *pCursor = (ma_int64)pWav->memoryStream.currentReadPos;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_init_memory(ma_dr_wav* pWav, const void* data, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_memory_ex(pWav, data, dataSize, NULL, NULL, 0, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_memory_ex(ma_dr_wav* pWav, const void* data, size_t dataSize, ma_dr_wav_chunk_proc onChunk, void* pChunkUserData, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (data == NULL || dataSize == 0) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_wav_preinit(pWav, ma_dr_wav__on_read_memory, ma_dr_wav__on_seek_memory, ma_dr_wav__on_tell_memory, pWav, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    pWav->memoryStream.data = (const ma_uint8*)data;\n    pWav->memoryStream.dataSize = dataSize;\n    pWav->memoryStream.currentReadPos = 0;\n    return ma_dr_wav_init__internal(pWav, onChunk, pChunkUserData, flags);\n}\nMA_API ma_bool32 ma_dr_wav_init_memory_with_metadata(ma_dr_wav* pWav, const void* data, size_t dataSize, ma_uint32 flags, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (data == NULL || dataSize == 0) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_wav_preinit(pWav, ma_dr_wav__on_read_memory, ma_dr_wav__on_seek_memory, ma_dr_wav__on_tell_memory, pWav, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    pWav->memoryStream.data = (const ma_uint8*)data;\n    pWav->memoryStream.dataSize = dataSize;\n    pWav->memoryStream.currentReadPos = 0;\n    return ma_dr_wav_init__internal(pWav, NULL, NULL, flags | MA_DR_WAV_WITH_METADATA);\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_init_memory_write__internal(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, ma_bool32 isSequential, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (ppData == NULL || pDataSize == NULL) {\n        return MA_FALSE;\n    }\n    *ppData = NULL;\n    *pDataSize = 0;\n    if (!ma_dr_wav_preinit_write(pWav, pFormat, isSequential, ma_dr_wav__on_write_memory, ma_dr_wav__on_seek_memory_write, pWav, pAllocationCallbacks)) {\n        return MA_FALSE;\n    }\n    pWav->memoryStreamWrite.ppData = ppData;\n    pWav->memoryStreamWrite.pDataSize = pDataSize;\n    pWav->memoryStreamWrite.dataSize = 0;\n    pWav->memoryStreamWrite.dataCapacity = 0;\n    pWav->memoryStreamWrite.currentWritePos = 0;\n    return ma_dr_wav_init_write__internal(pWav, pFormat, totalSampleCount);\n}\nMA_API ma_bool32 ma_dr_wav_init_memory_write(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_memory_write__internal(pWav, ppData, pDataSize, pFormat, 0, MA_FALSE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_memory_write_sequential(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, ma_uint64 totalSampleCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_wav_init_memory_write__internal(pWav, ppData, pDataSize, pFormat, totalSampleCount, MA_TRUE, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_wav_init_memory_write_sequential_pcm_frames(ma_dr_wav* pWav, void** ppData, size_t* pDataSize, const ma_dr_wav_data_format* pFormat, ma_uint64 totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pFormat == NULL) {\n        return MA_FALSE;\n    }\n    return ma_dr_wav_init_memory_write_sequential(pWav, ppData, pDataSize, pFormat, totalPCMFrameCount*pFormat->channels, pAllocationCallbacks);\n}\nMA_API ma_result ma_dr_wav_uninit(ma_dr_wav* pWav)\n{\n    ma_result result = MA_SUCCESS;\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n    if (pWav->onWrite != NULL) {\n        ma_uint32 paddingSize = 0;\n        if (pWav->container == ma_dr_wav_container_riff || pWav->container == ma_dr_wav_container_rf64) {\n            paddingSize = ma_dr_wav__chunk_padding_size_riff(pWav->dataChunkDataSize);\n        } else {\n            paddingSize = ma_dr_wav__chunk_padding_size_w64(pWav->dataChunkDataSize);\n        }\n        if (paddingSize > 0) {\n            ma_uint64 paddingData = 0;\n            ma_dr_wav__write(pWav, &paddingData, paddingSize);\n        }\n        if (pWav->onSeek && !pWav->isSequentialWrite) {\n            if (pWav->container == ma_dr_wav_container_riff) {\n                if (pWav->onSeek(pWav->pUserData, 4, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint32 riffChunkSize = ma_dr_wav__riff_chunk_size_riff(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount);\n                    ma_dr_wav__write_u32ne_to_le(pWav, riffChunkSize);\n                }\n                if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 4, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint32 dataChunkSize = ma_dr_wav__data_chunk_size_riff(pWav->dataChunkDataSize);\n                    ma_dr_wav__write_u32ne_to_le(pWav, dataChunkSize);\n                }\n            } else if (pWav->container == ma_dr_wav_container_w64) {\n                if (pWav->onSeek(pWav->pUserData, 16, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_w64(pWav->dataChunkDataSize);\n                    ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize);\n                }\n                if (pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos - 8, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_w64(pWav->dataChunkDataSize);\n                    ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize);\n                }\n            } else if (pWav->container == ma_dr_wav_container_rf64) {\n                int ds64BodyPos = 12 + 8;\n                if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 0, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint64 riffChunkSize = ma_dr_wav__riff_chunk_size_rf64(pWav->dataChunkDataSize, pWav->pMetadata, pWav->metadataCount);\n                    ma_dr_wav__write_u64ne_to_le(pWav, riffChunkSize);\n                }\n                if (pWav->onSeek(pWav->pUserData, ds64BodyPos + 8, MA_DR_WAV_SEEK_SET)) {\n                    ma_uint64 dataChunkSize = ma_dr_wav__data_chunk_size_rf64(pWav->dataChunkDataSize);\n                    ma_dr_wav__write_u64ne_to_le(pWav, dataChunkSize);\n                }\n            }\n        }\n        if (pWav->isSequentialWrite) {\n            if (pWav->dataChunkDataSize != pWav->dataChunkDataSizeTargetWrite) {\n                result = MA_INVALID_FILE;\n            }\n        }\n    } else {\n        ma_dr_wav_free(pWav->pMetadata, &pWav->allocationCallbacks);\n    }\n#ifndef MA_DR_WAV_NO_STDIO\n    if (pWav->onRead == ma_dr_wav__on_read_stdio || pWav->onWrite == ma_dr_wav__on_write_stdio) {\n        fclose((FILE*)pWav->pUserData);\n    }\n#endif\n    return result;\n}\nMA_API size_t ma_dr_wav_read_raw(ma_dr_wav* pWav, size_t bytesToRead, void* pBufferOut)\n{\n    size_t bytesRead;\n    ma_uint32 bytesPerFrame;\n    if (pWav == NULL || bytesToRead == 0) {\n        return 0;\n    }\n    if (bytesToRead > pWav->bytesRemaining) {\n        bytesToRead = (size_t)pWav->bytesRemaining;\n    }\n    if (bytesToRead == 0) {\n        return 0;\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    if (pBufferOut != NULL) {\n        bytesRead = pWav->onRead(pWav->pUserData, pBufferOut, bytesToRead);\n    } else {\n        bytesRead = 0;\n        while (bytesRead < bytesToRead) {\n            size_t bytesToSeek = (bytesToRead - bytesRead);\n            if (bytesToSeek > 0x7FFFFFFF) {\n                bytesToSeek = 0x7FFFFFFF;\n            }\n            if (pWav->onSeek(pWav->pUserData, (int)bytesToSeek, MA_DR_WAV_SEEK_CUR) == MA_FALSE) {\n                break;\n            }\n            bytesRead += bytesToSeek;\n        }\n        while (bytesRead < bytesToRead) {\n            ma_uint8 buffer[4096];\n            size_t bytesSeeked;\n            size_t bytesToSeek = (bytesToRead - bytesRead);\n            if (bytesToSeek > sizeof(buffer)) {\n                bytesToSeek = sizeof(buffer);\n            }\n            bytesSeeked = pWav->onRead(pWav->pUserData, buffer, bytesToSeek);\n            bytesRead += bytesSeeked;\n            if (bytesSeeked < bytesToSeek) {\n                break;\n            }\n        }\n    }\n    pWav->readCursorInPCMFrames += bytesRead / bytesPerFrame;\n    pWav->bytesRemaining -= bytesRead;\n    return bytesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut)\n{\n    ma_uint32 bytesPerFrame;\n    ma_uint64 bytesToRead;\n    ma_uint64 framesRemainingInFile;\n    if (pWav == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (ma_dr_wav__is_compressed_format_tag(pWav->translatedFormatTag)) {\n        return 0;\n    }\n    framesRemainingInFile = pWav->totalPCMFrameCount - pWav->readCursorInPCMFrames;\n    if (framesToRead > framesRemainingInFile) {\n        framesToRead = framesRemainingInFile;\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesToRead = framesToRead * bytesPerFrame;\n    if (bytesToRead > MA_SIZE_MAX) {\n        bytesToRead = (MA_SIZE_MAX / bytesPerFrame) * bytesPerFrame;\n    }\n    if (bytesToRead == 0) {\n        return 0;\n    }\n    return ma_dr_wav_read_raw(pWav, (size_t)bytesToRead, pBufferOut) / bytesPerFrame;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_be(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_le(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL) {\n        ma_uint32 bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n        if (bytesPerFrame == 0) {\n            return 0;\n        }\n        ma_dr_wav__bswap_samples(pBufferOut, framesRead*pWav->channels, bytesPerFrame/pWav->channels);\n    }\n    return framesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToRead, void* pBufferOut)\n{\n    ma_uint64 framesRead = 0;\n    if (ma_dr_wav_is_container_be(pWav->container)) {\n        if (pWav->container != ma_dr_wav_container_aiff || pWav->aiff.isLE == MA_FALSE) {\n            if (ma_dr_wav__is_little_endian()) {\n                framesRead = ma_dr_wav_read_pcm_frames_be(pWav, framesToRead, pBufferOut);\n            } else {\n                framesRead = ma_dr_wav_read_pcm_frames_le(pWav, framesToRead, pBufferOut);\n            }\n            goto post_process;\n        }\n    }\n    if (ma_dr_wav__is_little_endian()) {\n        framesRead = ma_dr_wav_read_pcm_frames_le(pWav, framesToRead, pBufferOut);\n    } else {\n        framesRead = ma_dr_wav_read_pcm_frames_be(pWav, framesToRead, pBufferOut);\n    }\n    post_process:\n    {\n        if (pWav->container == ma_dr_wav_container_aiff && pWav->bitsPerSample == 8 && pWav->aiff.isUnsigned == MA_FALSE) {\n            if (pBufferOut != NULL) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < framesRead * pWav->channels; iSample += 1) {\n                    ((ma_uint8*)pBufferOut)[iSample] += 128;\n                }\n            }\n        }\n    }\n    return framesRead;\n}\nMA_PRIVATE ma_bool32 ma_dr_wav_seek_to_first_pcm_frame(ma_dr_wav* pWav)\n{\n    if (pWav->onWrite != NULL) {\n        return MA_FALSE;\n    }\n    if (!pWav->onSeek(pWav->pUserData, (int)pWav->dataChunkDataPos, MA_DR_WAV_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    if (ma_dr_wav__is_compressed_format_tag(pWav->translatedFormatTag)) {\n        if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n            MA_DR_WAV_ZERO_OBJECT(&pWav->msadpcm);\n        } else if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n            MA_DR_WAV_ZERO_OBJECT(&pWav->ima);\n        } else {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n        }\n    }\n    pWav->readCursorInPCMFrames = 0;\n    pWav->bytesRemaining = pWav->dataChunkDataSize;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_seek_to_pcm_frame(ma_dr_wav* pWav, ma_uint64 targetFrameIndex)\n{\n    if (pWav == NULL || pWav->onSeek == NULL) {\n        return MA_FALSE;\n    }\n    if (pWav->onWrite != NULL) {\n        return MA_FALSE;\n    }\n    if (pWav->totalPCMFrameCount == 0) {\n        return MA_TRUE;\n    }\n    if (targetFrameIndex > pWav->totalPCMFrameCount) {\n        targetFrameIndex = pWav->totalPCMFrameCount;\n    }\n    if (ma_dr_wav__is_compressed_format_tag(pWav->translatedFormatTag)) {\n        if (targetFrameIndex < pWav->readCursorInPCMFrames) {\n            if (!ma_dr_wav_seek_to_first_pcm_frame(pWav)) {\n                return MA_FALSE;\n            }\n        }\n        if (targetFrameIndex > pWav->readCursorInPCMFrames) {\n            ma_uint64 offsetInFrames = targetFrameIndex - pWav->readCursorInPCMFrames;\n            ma_int16 devnull[2048];\n            while (offsetInFrames > 0) {\n                ma_uint64 framesRead = 0;\n                ma_uint64 framesToRead = offsetInFrames;\n                if (framesToRead > ma_dr_wav_countof(devnull)/pWav->channels) {\n                    framesToRead = ma_dr_wav_countof(devnull)/pWav->channels;\n                }\n                if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n                    framesRead = ma_dr_wav_read_pcm_frames_s16__msadpcm(pWav, framesToRead, devnull);\n                } else if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n                    framesRead = ma_dr_wav_read_pcm_frames_s16__ima(pWav, framesToRead, devnull);\n                } else {\n                    MA_DR_WAV_ASSERT(MA_FALSE);\n                }\n                if (framesRead != framesToRead) {\n                    return MA_FALSE;\n                }\n                offsetInFrames -= framesRead;\n            }\n        }\n    } else {\n        ma_uint64 totalSizeInBytes;\n        ma_uint64 currentBytePos;\n        ma_uint64 targetBytePos;\n        ma_uint64 offset;\n        ma_uint32 bytesPerFrame;\n        bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n        if (bytesPerFrame == 0) {\n            return MA_FALSE;\n        }\n        totalSizeInBytes = pWav->totalPCMFrameCount * bytesPerFrame;\n        currentBytePos = totalSizeInBytes - pWav->bytesRemaining;\n        targetBytePos  = targetFrameIndex * bytesPerFrame;\n        if (currentBytePos < targetBytePos) {\n            offset = (targetBytePos - currentBytePos);\n        } else {\n            if (!ma_dr_wav_seek_to_first_pcm_frame(pWav)) {\n                return MA_FALSE;\n            }\n            offset = targetBytePos;\n        }\n        while (offset > 0) {\n            int offset32 = ((offset > INT_MAX) ? INT_MAX : (int)offset);\n            if (!pWav->onSeek(pWav->pUserData, offset32, MA_DR_WAV_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            pWav->readCursorInPCMFrames += offset32 / bytesPerFrame;\n            pWav->bytesRemaining        -= offset32;\n            offset                      -= offset32;\n        }\n    }\n    return MA_TRUE;\n}\nMA_API ma_result ma_dr_wav_get_cursor_in_pcm_frames(ma_dr_wav* pWav, ma_uint64* pCursor)\n{\n    if (pCursor == NULL) {\n        return MA_INVALID_ARGS;\n    }\n    *pCursor = 0;\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n    *pCursor = pWav->readCursorInPCMFrames;\n    return MA_SUCCESS;\n}\nMA_API ma_result ma_dr_wav_get_length_in_pcm_frames(ma_dr_wav* pWav, ma_uint64* pLength)\n{\n    if (pLength == NULL) {\n        return MA_INVALID_ARGS;\n    }\n    *pLength = 0;\n    if (pWav == NULL) {\n        return MA_INVALID_ARGS;\n    }\n    *pLength = pWav->totalPCMFrameCount;\n    return MA_SUCCESS;\n}\nMA_API size_t ma_dr_wav_write_raw(ma_dr_wav* pWav, size_t bytesToWrite, const void* pData)\n{\n    size_t bytesWritten;\n    if (pWav == NULL || bytesToWrite == 0 || pData == NULL) {\n        return 0;\n    }\n    bytesWritten = pWav->onWrite(pWav->pUserData, pData, bytesToWrite);\n    pWav->dataChunkDataSize += bytesWritten;\n    return bytesWritten;\n}\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames_le(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData)\n{\n    ma_uint64 bytesToWrite;\n    ma_uint64 bytesWritten;\n    const ma_uint8* pRunningData;\n    if (pWav == NULL || framesToWrite == 0 || pData == NULL) {\n        return 0;\n    }\n    bytesToWrite = ((framesToWrite * pWav->channels * pWav->bitsPerSample) / 8);\n    if (bytesToWrite > MA_SIZE_MAX) {\n        return 0;\n    }\n    bytesWritten = 0;\n    pRunningData = (const ma_uint8*)pData;\n    while (bytesToWrite > 0) {\n        size_t bytesJustWritten;\n        ma_uint64 bytesToWriteThisIteration;\n        bytesToWriteThisIteration = bytesToWrite;\n        MA_DR_WAV_ASSERT(bytesToWriteThisIteration <= MA_SIZE_MAX);\n        bytesJustWritten = ma_dr_wav_write_raw(pWav, (size_t)bytesToWriteThisIteration, pRunningData);\n        if (bytesJustWritten == 0) {\n            break;\n        }\n        bytesToWrite -= bytesJustWritten;\n        bytesWritten += bytesJustWritten;\n        pRunningData += bytesJustWritten;\n    }\n    return (bytesWritten * 8) / pWav->bitsPerSample / pWav->channels;\n}\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames_be(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData)\n{\n    ma_uint64 bytesToWrite;\n    ma_uint64 bytesWritten;\n    ma_uint32 bytesPerSample;\n    const ma_uint8* pRunningData;\n    if (pWav == NULL || framesToWrite == 0 || pData == NULL) {\n        return 0;\n    }\n    bytesToWrite = ((framesToWrite * pWav->channels * pWav->bitsPerSample) / 8);\n    if (bytesToWrite > MA_SIZE_MAX) {\n        return 0;\n    }\n    bytesWritten = 0;\n    pRunningData = (const ma_uint8*)pData;\n    bytesPerSample = ma_dr_wav_get_bytes_per_pcm_frame(pWav) / pWav->channels;\n    if (bytesPerSample == 0) {\n        return 0;\n    }\n    while (bytesToWrite > 0) {\n        ma_uint8 temp[4096];\n        ma_uint32 sampleCount;\n        size_t bytesJustWritten;\n        ma_uint64 bytesToWriteThisIteration;\n        bytesToWriteThisIteration = bytesToWrite;\n        MA_DR_WAV_ASSERT(bytesToWriteThisIteration <= MA_SIZE_MAX);\n        sampleCount = sizeof(temp)/bytesPerSample;\n        if (bytesToWriteThisIteration > ((ma_uint64)sampleCount)*bytesPerSample) {\n            bytesToWriteThisIteration = ((ma_uint64)sampleCount)*bytesPerSample;\n        }\n        MA_DR_WAV_COPY_MEMORY(temp, pRunningData, (size_t)bytesToWriteThisIteration);\n        ma_dr_wav__bswap_samples(temp, sampleCount, bytesPerSample);\n        bytesJustWritten = ma_dr_wav_write_raw(pWav, (size_t)bytesToWriteThisIteration, temp);\n        if (bytesJustWritten == 0) {\n            break;\n        }\n        bytesToWrite -= bytesJustWritten;\n        bytesWritten += bytesJustWritten;\n        pRunningData += bytesJustWritten;\n    }\n    return (bytesWritten * 8) / pWav->bitsPerSample / pWav->channels;\n}\nMA_API ma_uint64 ma_dr_wav_write_pcm_frames(ma_dr_wav* pWav, ma_uint64 framesToWrite, const void* pData)\n{\n    if (ma_dr_wav__is_little_endian()) {\n        return ma_dr_wav_write_pcm_frames_le(pWav, framesToWrite, pData);\n    } else {\n        return ma_dr_wav_write_pcm_frames_be(pWav, framesToWrite, pData);\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__msadpcm(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead = 0;\n    static const ma_int32 adaptationTable[] = {\n        230, 230, 230, 230, 307, 409, 512, 614,\n        768, 614, 512, 409, 307, 230, 230, 230\n    };\n    static const ma_int32 coeff1Table[] = { 256, 512, 0, 192, 240, 460,  392 };\n    static const ma_int32 coeff2Table[] = { 0,  -256, 0, 64,  0,  -208, -232 };\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    MA_DR_WAV_ASSERT(framesToRead > 0);\n    while (pWav->readCursorInPCMFrames < pWav->totalPCMFrameCount) {\n        MA_DR_WAV_ASSERT(framesToRead > 0);\n        if (pWav->msadpcm.cachedFrameCount == 0 && pWav->msadpcm.bytesRemainingInBlock == 0) {\n            if (pWav->channels == 1) {\n                ma_uint8 header[7];\n                if (pWav->onRead(pWav->pUserData, header, sizeof(header)) != sizeof(header)) {\n                    return totalFramesRead;\n                }\n                pWav->msadpcm.bytesRemainingInBlock = pWav->fmt.blockAlign - sizeof(header);\n                pWav->msadpcm.predictor[0]     = header[0];\n                pWav->msadpcm.delta[0]         = ma_dr_wav_bytes_to_s16(header + 1);\n                pWav->msadpcm.prevFrames[0][1] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 3);\n                pWav->msadpcm.prevFrames[0][0] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 5);\n                pWav->msadpcm.cachedFrames[2]  = pWav->msadpcm.prevFrames[0][0];\n                pWav->msadpcm.cachedFrames[3]  = pWav->msadpcm.prevFrames[0][1];\n                pWav->msadpcm.cachedFrameCount = 2;\n                if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table)) {\n                    return totalFramesRead;\n                }\n            } else {\n                ma_uint8 header[14];\n                if (pWav->onRead(pWav->pUserData, header, sizeof(header)) != sizeof(header)) {\n                    return totalFramesRead;\n                }\n                pWav->msadpcm.bytesRemainingInBlock = pWav->fmt.blockAlign - sizeof(header);\n                pWav->msadpcm.predictor[0] = header[0];\n                pWav->msadpcm.predictor[1] = header[1];\n                pWav->msadpcm.delta[0] = ma_dr_wav_bytes_to_s16(header + 2);\n                pWav->msadpcm.delta[1] = ma_dr_wav_bytes_to_s16(header + 4);\n                pWav->msadpcm.prevFrames[0][1] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 6);\n                pWav->msadpcm.prevFrames[1][1] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 8);\n                pWav->msadpcm.prevFrames[0][0] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 10);\n                pWav->msadpcm.prevFrames[1][0] = (ma_int32)ma_dr_wav_bytes_to_s16(header + 12);\n                pWav->msadpcm.cachedFrames[0] = pWav->msadpcm.prevFrames[0][0];\n                pWav->msadpcm.cachedFrames[1] = pWav->msadpcm.prevFrames[1][0];\n                pWav->msadpcm.cachedFrames[2] = pWav->msadpcm.prevFrames[0][1];\n                pWav->msadpcm.cachedFrames[3] = pWav->msadpcm.prevFrames[1][1];\n                pWav->msadpcm.cachedFrameCount = 2;\n                if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table) ||\n                    pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff2Table)) {\n                    return totalFramesRead;\n                }\n            }\n        }\n        while (framesToRead > 0 && pWav->msadpcm.cachedFrameCount > 0 && pWav->readCursorInPCMFrames < pWav->totalPCMFrameCount) {\n            if (pBufferOut != NULL) {\n                ma_uint32 iSample = 0;\n                for (iSample = 0; iSample < pWav->channels; iSample += 1) {\n                    pBufferOut[iSample] = (ma_int16)pWav->msadpcm.cachedFrames[(ma_dr_wav_countof(pWav->msadpcm.cachedFrames) - (pWav->msadpcm.cachedFrameCount*pWav->channels)) + iSample];\n                }\n                pBufferOut += pWav->channels;\n            }\n            framesToRead    -= 1;\n            totalFramesRead += 1;\n            pWav->readCursorInPCMFrames += 1;\n            pWav->msadpcm.cachedFrameCount -= 1;\n        }\n        if (framesToRead == 0) {\n            break;\n        }\n        if (pWav->msadpcm.cachedFrameCount == 0) {\n            if (pWav->msadpcm.bytesRemainingInBlock == 0) {\n                continue;\n            } else {\n                ma_uint8 nibbles;\n                ma_int32 nibble0;\n                ma_int32 nibble1;\n                if (pWav->onRead(pWav->pUserData, &nibbles, 1) != 1) {\n                    return totalFramesRead;\n                }\n                pWav->msadpcm.bytesRemainingInBlock -= 1;\n                nibble0 = ((nibbles & 0xF0) >> 4); if ((nibbles & 0x80)) { nibble0 |= 0xFFFFFFF0UL; }\n                nibble1 = ((nibbles & 0x0F) >> 0); if ((nibbles & 0x08)) { nibble1 |= 0xFFFFFFF0UL; }\n                if (pWav->channels == 1) {\n                    ma_int32 newSample0;\n                    ma_int32 newSample1;\n                    if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table)) {\n                        return totalFramesRead;\n                    }\n                    newSample0  = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8;\n                    newSample0 += nibble0 * pWav->msadpcm.delta[0];\n                    newSample0  = ma_dr_wav_clamp(newSample0, -32768, 32767);\n                    pWav->msadpcm.delta[0] = (adaptationTable[((nibbles & 0xF0) >> 4)] * pWav->msadpcm.delta[0]) >> 8;\n                    if (pWav->msadpcm.delta[0] < 16) {\n                        pWav->msadpcm.delta[0] = 16;\n                    }\n                    pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1];\n                    pWav->msadpcm.prevFrames[0][1] = newSample0;\n                    newSample1  = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8;\n                    newSample1 += nibble1 * pWav->msadpcm.delta[0];\n                    newSample1  = ma_dr_wav_clamp(newSample1, -32768, 32767);\n                    pWav->msadpcm.delta[0] = (adaptationTable[((nibbles & 0x0F) >> 0)] * pWav->msadpcm.delta[0]) >> 8;\n                    if (pWav->msadpcm.delta[0] < 16) {\n                        pWav->msadpcm.delta[0] = 16;\n                    }\n                    pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1];\n                    pWav->msadpcm.prevFrames[0][1] = newSample1;\n                    pWav->msadpcm.cachedFrames[2] = newSample0;\n                    pWav->msadpcm.cachedFrames[3] = newSample1;\n                    pWav->msadpcm.cachedFrameCount = 2;\n                } else {\n                    ma_int32 newSample0;\n                    ma_int32 newSample1;\n                    if (pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[0] >= ma_dr_wav_countof(coeff2Table)) {\n                        return totalFramesRead;\n                    }\n                    newSample0  = ((pWav->msadpcm.prevFrames[0][1] * coeff1Table[pWav->msadpcm.predictor[0]]) + (pWav->msadpcm.prevFrames[0][0] * coeff2Table[pWav->msadpcm.predictor[0]])) >> 8;\n                    newSample0 += nibble0 * pWav->msadpcm.delta[0];\n                    newSample0  = ma_dr_wav_clamp(newSample0, -32768, 32767);\n                    pWav->msadpcm.delta[0] = (adaptationTable[((nibbles & 0xF0) >> 4)] * pWav->msadpcm.delta[0]) >> 8;\n                    if (pWav->msadpcm.delta[0] < 16) {\n                        pWav->msadpcm.delta[0] = 16;\n                    }\n                    pWav->msadpcm.prevFrames[0][0] = pWav->msadpcm.prevFrames[0][1];\n                    pWav->msadpcm.prevFrames[0][1] = newSample0;\n                    if (pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff1Table) || pWav->msadpcm.predictor[1] >= ma_dr_wav_countof(coeff2Table)) {\n                        return totalFramesRead;\n                    }\n                    newSample1  = ((pWav->msadpcm.prevFrames[1][1] * coeff1Table[pWav->msadpcm.predictor[1]]) + (pWav->msadpcm.prevFrames[1][0] * coeff2Table[pWav->msadpcm.predictor[1]])) >> 8;\n                    newSample1 += nibble1 * pWav->msadpcm.delta[1];\n                    newSample1  = ma_dr_wav_clamp(newSample1, -32768, 32767);\n                    pWav->msadpcm.delta[1] = (adaptationTable[((nibbles & 0x0F) >> 0)] * pWav->msadpcm.delta[1]) >> 8;\n                    if (pWav->msadpcm.delta[1] < 16) {\n                        pWav->msadpcm.delta[1] = 16;\n                    }\n                    pWav->msadpcm.prevFrames[1][0] = pWav->msadpcm.prevFrames[1][1];\n                    pWav->msadpcm.prevFrames[1][1] = newSample1;\n                    pWav->msadpcm.cachedFrames[2] = newSample0;\n                    pWav->msadpcm.cachedFrames[3] = newSample1;\n                    pWav->msadpcm.cachedFrameCount = 1;\n                }\n            }\n        }\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__ima(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead = 0;\n    ma_uint32 iChannel;\n    static const ma_int32 indexTable[16] = {\n        -1, -1, -1, -1, 2, 4, 6, 8,\n        -1, -1, -1, -1, 2, 4, 6, 8\n    };\n    static const ma_int32 stepTable[89] = {\n        7,     8,     9,     10,    11,    12,    13,    14,    16,    17,\n        19,    21,    23,    25,    28,    31,    34,    37,    41,    45,\n        50,    55,    60,    66,    73,    80,    88,    97,    107,   118,\n        130,   143,   157,   173,   190,   209,   230,   253,   279,   307,\n        337,   371,   408,   449,   494,   544,   598,   658,   724,   796,\n        876,   963,   1060,  1166,  1282,  1411,  1552,  1707,  1878,  2066,\n        2272,  2499,  2749,  3024,  3327,  3660,  4026,  4428,  4871,  5358,\n        5894,  6484,  7132,  7845,  8630,  9493,  10442, 11487, 12635, 13899,\n        15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767\n    };\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    MA_DR_WAV_ASSERT(framesToRead > 0);\n    while (pWav->readCursorInPCMFrames < pWav->totalPCMFrameCount) {\n        MA_DR_WAV_ASSERT(framesToRead > 0);\n        if (pWav->ima.cachedFrameCount == 0 && pWav->ima.bytesRemainingInBlock == 0) {\n            if (pWav->channels == 1) {\n                ma_uint8 header[4];\n                if (pWav->onRead(pWav->pUserData, header, sizeof(header)) != sizeof(header)) {\n                    return totalFramesRead;\n                }\n                pWav->ima.bytesRemainingInBlock = pWav->fmt.blockAlign - sizeof(header);\n                if (header[2] >= ma_dr_wav_countof(stepTable)) {\n                    pWav->onSeek(pWav->pUserData, pWav->ima.bytesRemainingInBlock, MA_DR_WAV_SEEK_CUR);\n                    pWav->ima.bytesRemainingInBlock = 0;\n                    return totalFramesRead;\n                }\n                pWav->ima.predictor[0] = (ma_int16)ma_dr_wav_bytes_to_u16(header + 0);\n                pWav->ima.stepIndex[0] = ma_dr_wav_clamp(header[2], 0, (ma_int32)ma_dr_wav_countof(stepTable)-1);\n                pWav->ima.cachedFrames[ma_dr_wav_countof(pWav->ima.cachedFrames) - 1] = pWav->ima.predictor[0];\n                pWav->ima.cachedFrameCount = 1;\n            } else {\n                ma_uint8 header[8];\n                if (pWav->onRead(pWav->pUserData, header, sizeof(header)) != sizeof(header)) {\n                    return totalFramesRead;\n                }\n                pWav->ima.bytesRemainingInBlock = pWav->fmt.blockAlign - sizeof(header);\n                if (header[2] >= ma_dr_wav_countof(stepTable) || header[6] >= ma_dr_wav_countof(stepTable)) {\n                    pWav->onSeek(pWav->pUserData, pWav->ima.bytesRemainingInBlock, MA_DR_WAV_SEEK_CUR);\n                    pWav->ima.bytesRemainingInBlock = 0;\n                    return totalFramesRead;\n                }\n                pWav->ima.predictor[0] = ma_dr_wav_bytes_to_s16(header + 0);\n                pWav->ima.stepIndex[0] = ma_dr_wav_clamp(header[2], 0, (ma_int32)ma_dr_wav_countof(stepTable)-1);\n                pWav->ima.predictor[1] = ma_dr_wav_bytes_to_s16(header + 4);\n                pWav->ima.stepIndex[1] = ma_dr_wav_clamp(header[6], 0, (ma_int32)ma_dr_wav_countof(stepTable)-1);\n                pWav->ima.cachedFrames[ma_dr_wav_countof(pWav->ima.cachedFrames) - 2] = pWav->ima.predictor[0];\n                pWav->ima.cachedFrames[ma_dr_wav_countof(pWav->ima.cachedFrames) - 1] = pWav->ima.predictor[1];\n                pWav->ima.cachedFrameCount = 1;\n            }\n        }\n        while (framesToRead > 0 && pWav->ima.cachedFrameCount > 0 && pWav->readCursorInPCMFrames < pWav->totalPCMFrameCount) {\n            if (pBufferOut != NULL) {\n                ma_uint32 iSample;\n                for (iSample = 0; iSample < pWav->channels; iSample += 1) {\n                    pBufferOut[iSample] = (ma_int16)pWav->ima.cachedFrames[(ma_dr_wav_countof(pWav->ima.cachedFrames) - (pWav->ima.cachedFrameCount*pWav->channels)) + iSample];\n                }\n                pBufferOut += pWav->channels;\n            }\n            framesToRead    -= 1;\n            totalFramesRead += 1;\n            pWav->readCursorInPCMFrames += 1;\n            pWav->ima.cachedFrameCount -= 1;\n        }\n        if (framesToRead == 0) {\n            break;\n        }\n        if (pWav->ima.cachedFrameCount == 0) {\n            if (pWav->ima.bytesRemainingInBlock == 0) {\n                continue;\n            } else {\n                pWav->ima.cachedFrameCount = 8;\n                for (iChannel = 0; iChannel < pWav->channels; ++iChannel) {\n                    ma_uint32 iByte;\n                    ma_uint8 nibbles[4];\n                    if (pWav->onRead(pWav->pUserData, &nibbles, 4) != 4) {\n                        pWav->ima.cachedFrameCount = 0;\n                        return totalFramesRead;\n                    }\n                    pWav->ima.bytesRemainingInBlock -= 4;\n                    for (iByte = 0; iByte < 4; ++iByte) {\n                        ma_uint8 nibble0 = ((nibbles[iByte] & 0x0F) >> 0);\n                        ma_uint8 nibble1 = ((nibbles[iByte] & 0xF0) >> 4);\n                        ma_int32 step      = stepTable[pWav->ima.stepIndex[iChannel]];\n                        ma_int32 predictor = pWav->ima.predictor[iChannel];\n                        ma_int32      diff  = step >> 3;\n                        if (nibble0 & 1) diff += step >> 2;\n                        if (nibble0 & 2) diff += step >> 1;\n                        if (nibble0 & 4) diff += step;\n                        if (nibble0 & 8) diff  = -diff;\n                        predictor = ma_dr_wav_clamp(predictor + diff, -32768, 32767);\n                        pWav->ima.predictor[iChannel] = predictor;\n                        pWav->ima.stepIndex[iChannel] = ma_dr_wav_clamp(pWav->ima.stepIndex[iChannel] + indexTable[nibble0], 0, (ma_int32)ma_dr_wav_countof(stepTable)-1);\n                        pWav->ima.cachedFrames[(ma_dr_wav_countof(pWav->ima.cachedFrames) - (pWav->ima.cachedFrameCount*pWav->channels)) + (iByte*2+0)*pWav->channels + iChannel] = predictor;\n                        step      = stepTable[pWav->ima.stepIndex[iChannel]];\n                        predictor = pWav->ima.predictor[iChannel];\n                                         diff  = step >> 3;\n                        if (nibble1 & 1) diff += step >> 2;\n                        if (nibble1 & 2) diff += step >> 1;\n                        if (nibble1 & 4) diff += step;\n                        if (nibble1 & 8) diff  = -diff;\n                        predictor = ma_dr_wav_clamp(predictor + diff, -32768, 32767);\n                        pWav->ima.predictor[iChannel] = predictor;\n                        pWav->ima.stepIndex[iChannel] = ma_dr_wav_clamp(pWav->ima.stepIndex[iChannel] + indexTable[nibble1], 0, (ma_int32)ma_dr_wav_countof(stepTable)-1);\n                        pWav->ima.cachedFrames[(ma_dr_wav_countof(pWav->ima.cachedFrames) - (pWav->ima.cachedFrameCount*pWav->channels)) + (iByte*2+1)*pWav->channels + iChannel] = predictor;\n                    }\n                }\n            }\n        }\n    }\n    return totalFramesRead;\n}\n#ifndef MA_DR_WAV_NO_CONVERSION_API\nstatic const unsigned short ma_dr_wav_gAlawTable[256] = {\n    0xEA80, 0xEB80, 0xE880, 0xE980, 0xEE80, 0xEF80, 0xEC80, 0xED80, 0xE280, 0xE380, 0xE080, 0xE180, 0xE680, 0xE780, 0xE480, 0xE580,\n    0xF540, 0xF5C0, 0xF440, 0xF4C0, 0xF740, 0xF7C0, 0xF640, 0xF6C0, 0xF140, 0xF1C0, 0xF040, 0xF0C0, 0xF340, 0xF3C0, 0xF240, 0xF2C0,\n    0xAA00, 0xAE00, 0xA200, 0xA600, 0xBA00, 0xBE00, 0xB200, 0xB600, 0x8A00, 0x8E00, 0x8200, 0x8600, 0x9A00, 0x9E00, 0x9200, 0x9600,\n    0xD500, 0xD700, 0xD100, 0xD300, 0xDD00, 0xDF00, 0xD900, 0xDB00, 0xC500, 0xC700, 0xC100, 0xC300, 0xCD00, 0xCF00, 0xC900, 0xCB00,\n    0xFEA8, 0xFEB8, 0xFE88, 0xFE98, 0xFEE8, 0xFEF8, 0xFEC8, 0xFED8, 0xFE28, 0xFE38, 0xFE08, 0xFE18, 0xFE68, 0xFE78, 0xFE48, 0xFE58,\n    0xFFA8, 0xFFB8, 0xFF88, 0xFF98, 0xFFE8, 0xFFF8, 0xFFC8, 0xFFD8, 0xFF28, 0xFF38, 0xFF08, 0xFF18, 0xFF68, 0xFF78, 0xFF48, 0xFF58,\n    0xFAA0, 0xFAE0, 0xFA20, 0xFA60, 0xFBA0, 0xFBE0, 0xFB20, 0xFB60, 0xF8A0, 0xF8E0, 0xF820, 0xF860, 0xF9A0, 0xF9E0, 0xF920, 0xF960,\n    0xFD50, 0xFD70, 0xFD10, 0xFD30, 0xFDD0, 0xFDF0, 0xFD90, 0xFDB0, 0xFC50, 0xFC70, 0xFC10, 0xFC30, 0xFCD0, 0xFCF0, 0xFC90, 0xFCB0,\n    0x1580, 0x1480, 0x1780, 0x1680, 0x1180, 0x1080, 0x1380, 0x1280, 0x1D80, 0x1C80, 0x1F80, 0x1E80, 0x1980, 0x1880, 0x1B80, 0x1A80,\n    0x0AC0, 0x0A40, 0x0BC0, 0x0B40, 0x08C0, 0x0840, 0x09C0, 0x0940, 0x0EC0, 0x0E40, 0x0FC0, 0x0F40, 0x0CC0, 0x0C40, 0x0DC0, 0x0D40,\n    0x5600, 0x5200, 0x5E00, 0x5A00, 0x4600, 0x4200, 0x4E00, 0x4A00, 0x7600, 0x7200, 0x7E00, 0x7A00, 0x6600, 0x6200, 0x6E00, 0x6A00,\n    0x2B00, 0x2900, 0x2F00, 0x2D00, 0x2300, 0x2100, 0x2700, 0x2500, 0x3B00, 0x3900, 0x3F00, 0x3D00, 0x3300, 0x3100, 0x3700, 0x3500,\n    0x0158, 0x0148, 0x0178, 0x0168, 0x0118, 0x0108, 0x0138, 0x0128, 0x01D8, 0x01C8, 0x01F8, 0x01E8, 0x0198, 0x0188, 0x01B8, 0x01A8,\n    0x0058, 0x0048, 0x0078, 0x0068, 0x0018, 0x0008, 0x0038, 0x0028, 0x00D8, 0x00C8, 0x00F8, 0x00E8, 0x0098, 0x0088, 0x00B8, 0x00A8,\n    0x0560, 0x0520, 0x05E0, 0x05A0, 0x0460, 0x0420, 0x04E0, 0x04A0, 0x0760, 0x0720, 0x07E0, 0x07A0, 0x0660, 0x0620, 0x06E0, 0x06A0,\n    0x02B0, 0x0290, 0x02F0, 0x02D0, 0x0230, 0x0210, 0x0270, 0x0250, 0x03B0, 0x0390, 0x03F0, 0x03D0, 0x0330, 0x0310, 0x0370, 0x0350\n};\nstatic const unsigned short ma_dr_wav_gMulawTable[256] = {\n    0x8284, 0x8684, 0x8A84, 0x8E84, 0x9284, 0x9684, 0x9A84, 0x9E84, 0xA284, 0xA684, 0xAA84, 0xAE84, 0xB284, 0xB684, 0xBA84, 0xBE84,\n    0xC184, 0xC384, 0xC584, 0xC784, 0xC984, 0xCB84, 0xCD84, 0xCF84, 0xD184, 0xD384, 0xD584, 0xD784, 0xD984, 0xDB84, 0xDD84, 0xDF84,\n    0xE104, 0xE204, 0xE304, 0xE404, 0xE504, 0xE604, 0xE704, 0xE804, 0xE904, 0xEA04, 0xEB04, 0xEC04, 0xED04, 0xEE04, 0xEF04, 0xF004,\n    0xF0C4, 0xF144, 0xF1C4, 0xF244, 0xF2C4, 0xF344, 0xF3C4, 0xF444, 0xF4C4, 0xF544, 0xF5C4, 0xF644, 0xF6C4, 0xF744, 0xF7C4, 0xF844,\n    0xF8A4, 0xF8E4, 0xF924, 0xF964, 0xF9A4, 0xF9E4, 0xFA24, 0xFA64, 0xFAA4, 0xFAE4, 0xFB24, 0xFB64, 0xFBA4, 0xFBE4, 0xFC24, 0xFC64,\n    0xFC94, 0xFCB4, 0xFCD4, 0xFCF4, 0xFD14, 0xFD34, 0xFD54, 0xFD74, 0xFD94, 0xFDB4, 0xFDD4, 0xFDF4, 0xFE14, 0xFE34, 0xFE54, 0xFE74,\n    0xFE8C, 0xFE9C, 0xFEAC, 0xFEBC, 0xFECC, 0xFEDC, 0xFEEC, 0xFEFC, 0xFF0C, 0xFF1C, 0xFF2C, 0xFF3C, 0xFF4C, 0xFF5C, 0xFF6C, 0xFF7C,\n    0xFF88, 0xFF90, 0xFF98, 0xFFA0, 0xFFA8, 0xFFB0, 0xFFB8, 0xFFC0, 0xFFC8, 0xFFD0, 0xFFD8, 0xFFE0, 0xFFE8, 0xFFF0, 0xFFF8, 0x0000,\n    0x7D7C, 0x797C, 0x757C, 0x717C, 0x6D7C, 0x697C, 0x657C, 0x617C, 0x5D7C, 0x597C, 0x557C, 0x517C, 0x4D7C, 0x497C, 0x457C, 0x417C,\n    0x3E7C, 0x3C7C, 0x3A7C, 0x387C, 0x367C, 0x347C, 0x327C, 0x307C, 0x2E7C, 0x2C7C, 0x2A7C, 0x287C, 0x267C, 0x247C, 0x227C, 0x207C,\n    0x1EFC, 0x1DFC, 0x1CFC, 0x1BFC, 0x1AFC, 0x19FC, 0x18FC, 0x17FC, 0x16FC, 0x15FC, 0x14FC, 0x13FC, 0x12FC, 0x11FC, 0x10FC, 0x0FFC,\n    0x0F3C, 0x0EBC, 0x0E3C, 0x0DBC, 0x0D3C, 0x0CBC, 0x0C3C, 0x0BBC, 0x0B3C, 0x0ABC, 0x0A3C, 0x09BC, 0x093C, 0x08BC, 0x083C, 0x07BC,\n    0x075C, 0x071C, 0x06DC, 0x069C, 0x065C, 0x061C, 0x05DC, 0x059C, 0x055C, 0x051C, 0x04DC, 0x049C, 0x045C, 0x041C, 0x03DC, 0x039C,\n    0x036C, 0x034C, 0x032C, 0x030C, 0x02EC, 0x02CC, 0x02AC, 0x028C, 0x026C, 0x024C, 0x022C, 0x020C, 0x01EC, 0x01CC, 0x01AC, 0x018C,\n    0x0174, 0x0164, 0x0154, 0x0144, 0x0134, 0x0124, 0x0114, 0x0104, 0x00F4, 0x00E4, 0x00D4, 0x00C4, 0x00B4, 0x00A4, 0x0094, 0x0084,\n    0x0078, 0x0070, 0x0068, 0x0060, 0x0058, 0x0050, 0x0048, 0x0040, 0x0038, 0x0030, 0x0028, 0x0020, 0x0018, 0x0010, 0x0008, 0x0000\n};\nstatic MA_INLINE ma_int16 ma_dr_wav__alaw_to_s16(ma_uint8 sampleIn)\n{\n    return (short)ma_dr_wav_gAlawTable[sampleIn];\n}\nstatic MA_INLINE ma_int16 ma_dr_wav__mulaw_to_s16(ma_uint8 sampleIn)\n{\n    return (short)ma_dr_wav_gMulawTable[sampleIn];\n}\nMA_PRIVATE void ma_dr_wav__pcm_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t totalSampleCount, unsigned int bytesPerSample)\n{\n    size_t i;\n    if (bytesPerSample == 1) {\n        ma_dr_wav_u8_to_s16(pOut, pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample == 2) {\n        for (i = 0; i < totalSampleCount; ++i) {\n           *pOut++ = ((const ma_int16*)pIn)[i];\n        }\n        return;\n    }\n    if (bytesPerSample == 3) {\n        ma_dr_wav_s24_to_s16(pOut, pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample == 4) {\n        ma_dr_wav_s32_to_s16(pOut, (const ma_int32*)pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample > 8) {\n        MA_DR_WAV_ZERO_MEMORY(pOut, totalSampleCount * sizeof(*pOut));\n        return;\n    }\n    for (i = 0; i < totalSampleCount; ++i) {\n        ma_uint64 sample = 0;\n        unsigned int shift  = (8 - bytesPerSample) * 8;\n        unsigned int j;\n        for (j = 0; j < bytesPerSample; j += 1) {\n            MA_DR_WAV_ASSERT(j < 8);\n            sample |= (ma_uint64)(pIn[j]) << shift;\n            shift  += 8;\n        }\n        pIn += j;\n        *pOut++ = (ma_int16)((ma_int64)sample >> 48);\n    }\n}\nMA_PRIVATE void ma_dr_wav__ieee_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t totalSampleCount, unsigned int bytesPerSample)\n{\n    if (bytesPerSample == 4) {\n        ma_dr_wav_f32_to_s16(pOut, (const float*)pIn, totalSampleCount);\n        return;\n    } else if (bytesPerSample == 8) {\n        ma_dr_wav_f64_to_s16(pOut, (const double*)pIn, totalSampleCount);\n        return;\n    } else {\n        MA_DR_WAV_ZERO_MEMORY(pOut, totalSampleCount * sizeof(*pOut));\n        return;\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__pcm(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if ((pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM && pWav->bitsPerSample == 16) || pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, pBufferOut);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__pcm_to_s16(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__ieee(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__ieee_to_s16(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__alaw(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_alaw_to_s16(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s16__mulaw(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_mulaw_to_s16(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    if (pWav == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    if (framesToRead * pWav->channels * sizeof(ma_int16) > MA_SIZE_MAX) {\n        framesToRead = MA_SIZE_MAX / sizeof(ma_int16) / pWav->channels;\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM) {\n        return ma_dr_wav_read_pcm_frames_s16__pcm(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_IEEE_FLOAT) {\n        return ma_dr_wav_read_pcm_frames_s16__ieee(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ALAW) {\n        return ma_dr_wav_read_pcm_frames_s16__alaw(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_MULAW) {\n        return ma_dr_wav_read_pcm_frames_s16__mulaw(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM) {\n        return ma_dr_wav_read_pcm_frames_s16__msadpcm(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        return ma_dr_wav_read_pcm_frames_s16__ima(pWav, framesToRead, pBufferOut);\n    }\n    return 0;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16le(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s16(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_FALSE) {\n        ma_dr_wav__bswap_samples_s16(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s16be(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s16(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_TRUE) {\n        ma_dr_wav__bswap_samples_s16(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API void ma_dr_wav_u8_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    int r;\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        int x = pIn[i];\n        r = x << 8;\n        r = r - 32768;\n        pOut[i] = (short)r;\n    }\n}\nMA_API void ma_dr_wav_s24_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    int r;\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        int x = ((int)(((unsigned int)(((const ma_uint8*)pIn)[i*3+0]) << 8) | ((unsigned int)(((const ma_uint8*)pIn)[i*3+1]) << 16) | ((unsigned int)(((const ma_uint8*)pIn)[i*3+2])) << 24)) >> 8;\n        r = x >> 8;\n        pOut[i] = (short)r;\n    }\n}\nMA_API void ma_dr_wav_s32_to_s16(ma_int16* pOut, const ma_int32* pIn, size_t sampleCount)\n{\n    int r;\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        int x = pIn[i];\n        r = x >> 16;\n        pOut[i] = (short)r;\n    }\n}\nMA_API void ma_dr_wav_f32_to_s16(ma_int16* pOut, const float* pIn, size_t sampleCount)\n{\n    int r;\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        float x = pIn[i];\n        float c;\n        c = ((x < -1) ? -1 : ((x > 1) ? 1 : x));\n        c = c + 1;\n        r = (int)(c * 32767.5f);\n        r = r - 32768;\n        pOut[i] = (short)r;\n    }\n}\nMA_API void ma_dr_wav_f64_to_s16(ma_int16* pOut, const double* pIn, size_t sampleCount)\n{\n    int r;\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        double x = pIn[i];\n        double c;\n        c = ((x < -1) ? -1 : ((x > 1) ? 1 : x));\n        c = c + 1;\n        r = (int)(c * 32767.5);\n        r = r - 32768;\n        pOut[i] = (short)r;\n    }\n}\nMA_API void ma_dr_wav_alaw_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        pOut[i] = ma_dr_wav__alaw_to_s16(pIn[i]);\n    }\n}\nMA_API void ma_dr_wav_mulaw_to_s16(ma_int16* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    for (i = 0; i < sampleCount; ++i) {\n        pOut[i] = ma_dr_wav__mulaw_to_s16(pIn[i]);\n    }\n}\nMA_PRIVATE void ma_dr_wav__pcm_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount, unsigned int bytesPerSample)\n{\n    unsigned int i;\n    if (bytesPerSample == 1) {\n        ma_dr_wav_u8_to_f32(pOut, pIn, sampleCount);\n        return;\n    }\n    if (bytesPerSample == 2) {\n        ma_dr_wav_s16_to_f32(pOut, (const ma_int16*)pIn, sampleCount);\n        return;\n    }\n    if (bytesPerSample == 3) {\n        ma_dr_wav_s24_to_f32(pOut, pIn, sampleCount);\n        return;\n    }\n    if (bytesPerSample == 4) {\n        ma_dr_wav_s32_to_f32(pOut, (const ma_int32*)pIn, sampleCount);\n        return;\n    }\n    if (bytesPerSample > 8) {\n        MA_DR_WAV_ZERO_MEMORY(pOut, sampleCount * sizeof(*pOut));\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        ma_uint64 sample = 0;\n        unsigned int shift  = (8 - bytesPerSample) * 8;\n        unsigned int j;\n        for (j = 0; j < bytesPerSample; j += 1) {\n            MA_DR_WAV_ASSERT(j < 8);\n            sample |= (ma_uint64)(pIn[j]) << shift;\n            shift  += 8;\n        }\n        pIn += j;\n        *pOut++ = (float)((ma_int64)sample / 9223372036854775807.0);\n    }\n}\nMA_PRIVATE void ma_dr_wav__ieee_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount, unsigned int bytesPerSample)\n{\n    if (bytesPerSample == 4) {\n        unsigned int i;\n        for (i = 0; i < sampleCount; ++i) {\n            *pOut++ = ((const float*)pIn)[i];\n        }\n        return;\n    } else if (bytesPerSample == 8) {\n        ma_dr_wav_f64_to_f32(pOut, (const double*)pIn, sampleCount);\n        return;\n    } else {\n        MA_DR_WAV_ZERO_MEMORY(pOut, sampleCount * sizeof(*pOut));\n        return;\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__pcm(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__pcm_to_f32(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__msadpcm_ima(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_int16 samples16[2048];\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, ma_dr_wav_countof(samples16)/pWav->channels);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s16(pWav, framesToReadThisIteration, samples16);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        ma_dr_wav_s16_to_f32(pBufferOut, samples16, (size_t)(framesRead*pWav->channels));\n        pBufferOut      += framesRead*pWav->channels;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__ieee(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_IEEE_FLOAT && pWav->bitsPerSample == 32) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, pBufferOut);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__ieee_to_f32(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__alaw(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_alaw_to_f32(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_f32__mulaw(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_mulaw_to_f32(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    if (pWav == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    if (framesToRead * pWav->channels * sizeof(float) > MA_SIZE_MAX) {\n        framesToRead = MA_SIZE_MAX / sizeof(float) / pWav->channels;\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM) {\n        return ma_dr_wav_read_pcm_frames_f32__pcm(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM || pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        return ma_dr_wav_read_pcm_frames_f32__msadpcm_ima(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_IEEE_FLOAT) {\n        return ma_dr_wav_read_pcm_frames_f32__ieee(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ALAW) {\n        return ma_dr_wav_read_pcm_frames_f32__alaw(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_MULAW) {\n        return ma_dr_wav_read_pcm_frames_f32__mulaw(pWav, framesToRead, pBufferOut);\n    }\n    return 0;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32le(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_f32(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_FALSE) {\n        ma_dr_wav__bswap_samples_f32(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_f32be(ma_dr_wav* pWav, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_f32(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_TRUE) {\n        ma_dr_wav__bswap_samples_f32(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API void ma_dr_wav_u8_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n#ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = (pIn[i] / 256.0f) * 2 - 1;\n    }\n#else\n    for (i = 0; i < sampleCount; ++i) {\n        float x = pIn[i];\n        x = x * 0.00784313725490196078f;\n        x = x - 1;\n        *pOut++ = x;\n    }\n#endif\n}\nMA_API void ma_dr_wav_s16_to_f32(float* pOut, const ma_int16* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = pIn[i] * 0.000030517578125f;\n    }\n}\nMA_API void ma_dr_wav_s24_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        double x;\n        ma_uint32 a = ((ma_uint32)(pIn[i*3+0]) <<  8);\n        ma_uint32 b = ((ma_uint32)(pIn[i*3+1]) << 16);\n        ma_uint32 c = ((ma_uint32)(pIn[i*3+2]) << 24);\n        x = (double)((ma_int32)(a | b | c) >> 8);\n        *pOut++ = (float)(x * 0.00000011920928955078125);\n    }\n}\nMA_API void ma_dr_wav_s32_to_f32(float* pOut, const ma_int32* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = (float)(pIn[i] / 2147483648.0);\n    }\n}\nMA_API void ma_dr_wav_f64_to_f32(float* pOut, const double* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = (float)pIn[i];\n    }\n}\nMA_API void ma_dr_wav_alaw_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = ma_dr_wav__alaw_to_s16(pIn[i]) / 32768.0f;\n    }\n}\nMA_API void ma_dr_wav_mulaw_to_f32(float* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = ma_dr_wav__mulaw_to_s16(pIn[i]) / 32768.0f;\n    }\n}\nMA_PRIVATE void ma_dr_wav__pcm_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t totalSampleCount, unsigned int bytesPerSample)\n{\n    unsigned int i;\n    if (bytesPerSample == 1) {\n        ma_dr_wav_u8_to_s32(pOut, pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample == 2) {\n        ma_dr_wav_s16_to_s32(pOut, (const ma_int16*)pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample == 3) {\n        ma_dr_wav_s24_to_s32(pOut, pIn, totalSampleCount);\n        return;\n    }\n    if (bytesPerSample == 4) {\n        for (i = 0; i < totalSampleCount; ++i) {\n           *pOut++ = ((const ma_int32*)pIn)[i];\n        }\n        return;\n    }\n    if (bytesPerSample > 8) {\n        MA_DR_WAV_ZERO_MEMORY(pOut, totalSampleCount * sizeof(*pOut));\n        return;\n    }\n    for (i = 0; i < totalSampleCount; ++i) {\n        ma_uint64 sample = 0;\n        unsigned int shift  = (8 - bytesPerSample) * 8;\n        unsigned int j;\n        for (j = 0; j < bytesPerSample; j += 1) {\n            MA_DR_WAV_ASSERT(j < 8);\n            sample |= (ma_uint64)(pIn[j]) << shift;\n            shift  += 8;\n        }\n        pIn += j;\n        *pOut++ = (ma_int32)((ma_int64)sample >> 32);\n    }\n}\nMA_PRIVATE void ma_dr_wav__ieee_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t totalSampleCount, unsigned int bytesPerSample)\n{\n    if (bytesPerSample == 4) {\n        ma_dr_wav_f32_to_s32(pOut, (const float*)pIn, totalSampleCount);\n        return;\n    } else if (bytesPerSample == 8) {\n        ma_dr_wav_f64_to_s32(pOut, (const double*)pIn, totalSampleCount);\n        return;\n    } else {\n        MA_DR_WAV_ZERO_MEMORY(pOut, totalSampleCount * sizeof(*pOut));\n        return;\n    }\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__pcm(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM && pWav->bitsPerSample == 32) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, pBufferOut);\n    }\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__pcm_to_s32(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__msadpcm_ima(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 totalFramesRead = 0;\n    ma_int16 samples16[2048];\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, ma_dr_wav_countof(samples16)/pWav->channels);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s16(pWav, framesToReadThisIteration, samples16);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        ma_dr_wav_s16_to_s32(pBufferOut, samples16, (size_t)(framesRead*pWav->channels));\n        pBufferOut      += framesRead*pWav->channels;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__ieee(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav__ieee_to_s32(pBufferOut, sampleData, (size_t)samplesRead, bytesPerSample);\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__alaw(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_alaw_to_s32(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_PRIVATE ma_uint64 ma_dr_wav_read_pcm_frames_s32__mulaw(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 totalFramesRead;\n    ma_uint8 sampleData[4096] = {0};\n    ma_uint32 bytesPerFrame;\n    ma_uint32 bytesPerSample;\n    ma_uint64 samplesRead;\n    bytesPerFrame = ma_dr_wav_get_bytes_per_pcm_frame(pWav);\n    if (bytesPerFrame == 0) {\n        return 0;\n    }\n    bytesPerSample = bytesPerFrame / pWav->channels;\n    if (bytesPerSample == 0 || (bytesPerFrame % pWav->channels) != 0) {\n        return 0;\n    }\n    totalFramesRead = 0;\n    while (framesToRead > 0) {\n        ma_uint64 framesToReadThisIteration = ma_dr_wav_min(framesToRead, sizeof(sampleData)/bytesPerFrame);\n        ma_uint64 framesRead = ma_dr_wav_read_pcm_frames(pWav, framesToReadThisIteration, sampleData);\n        if (framesRead == 0) {\n            break;\n        }\n        MA_DR_WAV_ASSERT(framesRead <= framesToReadThisIteration);\n        samplesRead = framesRead * pWav->channels;\n        if ((samplesRead * bytesPerSample) > sizeof(sampleData)) {\n            MA_DR_WAV_ASSERT(MA_FALSE);\n            break;\n        }\n        ma_dr_wav_mulaw_to_s32(pBufferOut, sampleData, (size_t)samplesRead);\n        #ifdef MA_DR_WAV_LIBSNDFILE_COMPAT\n        {\n            if (pWav->container == ma_dr_wav_container_aiff) {\n                ma_uint64 iSample;\n                for (iSample = 0; iSample < samplesRead; iSample += 1) {\n                    pBufferOut[iSample] = -pBufferOut[iSample];\n                }\n            }\n        }\n        #endif\n        pBufferOut      += samplesRead;\n        framesToRead    -= framesRead;\n        totalFramesRead += framesRead;\n    }\n    return totalFramesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    if (pWav == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_wav_read_pcm_frames(pWav, framesToRead, NULL);\n    }\n    if (framesToRead * pWav->channels * sizeof(ma_int32) > MA_SIZE_MAX) {\n        framesToRead = MA_SIZE_MAX / sizeof(ma_int32) / pWav->channels;\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_PCM) {\n        return ma_dr_wav_read_pcm_frames_s32__pcm(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ADPCM || pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_DVI_ADPCM) {\n        return ma_dr_wav_read_pcm_frames_s32__msadpcm_ima(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_IEEE_FLOAT) {\n        return ma_dr_wav_read_pcm_frames_s32__ieee(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_ALAW) {\n        return ma_dr_wav_read_pcm_frames_s32__alaw(pWav, framesToRead, pBufferOut);\n    }\n    if (pWav->translatedFormatTag == MA_DR_WAVE_FORMAT_MULAW) {\n        return ma_dr_wav_read_pcm_frames_s32__mulaw(pWav, framesToRead, pBufferOut);\n    }\n    return 0;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32le(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s32(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_FALSE) {\n        ma_dr_wav__bswap_samples_s32(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API ma_uint64 ma_dr_wav_read_pcm_frames_s32be(ma_dr_wav* pWav, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 framesRead = ma_dr_wav_read_pcm_frames_s32(pWav, framesToRead, pBufferOut);\n    if (pBufferOut != NULL && ma_dr_wav__is_little_endian() == MA_TRUE) {\n        ma_dr_wav__bswap_samples_s32(pBufferOut, framesRead*pWav->channels);\n    }\n    return framesRead;\n}\nMA_API void ma_dr_wav_u8_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = ((int)pIn[i] - 128) << 24;\n    }\n}\nMA_API void ma_dr_wav_s16_to_s32(ma_int32* pOut, const ma_int16* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = pIn[i] << 16;\n    }\n}\nMA_API void ma_dr_wav_s24_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        unsigned int s0 = pIn[i*3 + 0];\n        unsigned int s1 = pIn[i*3 + 1];\n        unsigned int s2 = pIn[i*3 + 2];\n        ma_int32 sample32 = (ma_int32)((s0 << 8) | (s1 << 16) | (s2 << 24));\n        *pOut++ = sample32;\n    }\n}\nMA_API void ma_dr_wav_f32_to_s32(ma_int32* pOut, const float* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = (ma_int32)(2147483648.0f * pIn[i]);\n    }\n}\nMA_API void ma_dr_wav_f64_to_s32(ma_int32* pOut, const double* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = (ma_int32)(2147483648.0 * pIn[i]);\n    }\n}\nMA_API void ma_dr_wav_alaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i = 0; i < sampleCount; ++i) {\n        *pOut++ = ((ma_int32)ma_dr_wav__alaw_to_s16(pIn[i])) << 16;\n    }\n}\nMA_API void ma_dr_wav_mulaw_to_s32(ma_int32* pOut, const ma_uint8* pIn, size_t sampleCount)\n{\n    size_t i;\n    if (pOut == NULL || pIn == NULL) {\n        return;\n    }\n    for (i= 0; i < sampleCount; ++i) {\n        *pOut++ = ((ma_int32)ma_dr_wav__mulaw_to_s16(pIn[i])) << 16;\n    }\n}\nMA_PRIVATE ma_int16* ma_dr_wav__read_pcm_frames_and_close_s16(ma_dr_wav* pWav, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalFrameCount)\n{\n    ma_uint64 sampleDataSize;\n    ma_int16* pSampleData;\n    ma_uint64 framesRead;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    if (pWav->channels == 0 || pWav->totalPCMFrameCount > MA_SIZE_MAX / pWav->channels / sizeof(ma_int16)) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    sampleDataSize = pWav->totalPCMFrameCount * pWav->channels * sizeof(ma_int16);\n    if (sampleDataSize > MA_SIZE_MAX) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    pSampleData = (ma_int16*)ma_dr_wav__malloc_from_callbacks((size_t)sampleDataSize, &pWav->allocationCallbacks);\n    if (pSampleData == NULL) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    framesRead = ma_dr_wav_read_pcm_frames_s16(pWav, (size_t)pWav->totalPCMFrameCount, pSampleData);\n    if (framesRead != pWav->totalPCMFrameCount) {\n        ma_dr_wav__free_from_callbacks(pSampleData, &pWav->allocationCallbacks);\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    ma_dr_wav_uninit(pWav);\n    if (sampleRate) {\n        *sampleRate = pWav->sampleRate;\n    }\n    if (channels) {\n        *channels = pWav->channels;\n    }\n    if (totalFrameCount) {\n        *totalFrameCount = pWav->totalPCMFrameCount;\n    }\n    return pSampleData;\n}\nMA_PRIVATE float* ma_dr_wav__read_pcm_frames_and_close_f32(ma_dr_wav* pWav, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalFrameCount)\n{\n    ma_uint64 sampleDataSize;\n    float* pSampleData;\n    ma_uint64 framesRead;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    if (pWav->channels == 0 || pWav->totalPCMFrameCount > MA_SIZE_MAX / pWav->channels / sizeof(float)) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    sampleDataSize = pWav->totalPCMFrameCount * pWav->channels * sizeof(float);\n    if (sampleDataSize > MA_SIZE_MAX) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    pSampleData = (float*)ma_dr_wav__malloc_from_callbacks((size_t)sampleDataSize, &pWav->allocationCallbacks);\n    if (pSampleData == NULL) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    framesRead = ma_dr_wav_read_pcm_frames_f32(pWav, (size_t)pWav->totalPCMFrameCount, pSampleData);\n    if (framesRead != pWav->totalPCMFrameCount) {\n        ma_dr_wav__free_from_callbacks(pSampleData, &pWav->allocationCallbacks);\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    ma_dr_wav_uninit(pWav);\n    if (sampleRate) {\n        *sampleRate = pWav->sampleRate;\n    }\n    if (channels) {\n        *channels = pWav->channels;\n    }\n    if (totalFrameCount) {\n        *totalFrameCount = pWav->totalPCMFrameCount;\n    }\n    return pSampleData;\n}\nMA_PRIVATE ma_int32* ma_dr_wav__read_pcm_frames_and_close_s32(ma_dr_wav* pWav, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalFrameCount)\n{\n    ma_uint64 sampleDataSize;\n    ma_int32* pSampleData;\n    ma_uint64 framesRead;\n    MA_DR_WAV_ASSERT(pWav != NULL);\n    if (pWav->channels == 0 || pWav->totalPCMFrameCount > MA_SIZE_MAX / pWav->channels / sizeof(ma_int32)) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    sampleDataSize = pWav->totalPCMFrameCount * pWav->channels * sizeof(ma_int32);\n    if (sampleDataSize > MA_SIZE_MAX) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    pSampleData = (ma_int32*)ma_dr_wav__malloc_from_callbacks((size_t)sampleDataSize, &pWav->allocationCallbacks);\n    if (pSampleData == NULL) {\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    framesRead = ma_dr_wav_read_pcm_frames_s32(pWav, (size_t)pWav->totalPCMFrameCount, pSampleData);\n    if (framesRead != pWav->totalPCMFrameCount) {\n        ma_dr_wav__free_from_callbacks(pSampleData, &pWav->allocationCallbacks);\n        ma_dr_wav_uninit(pWav);\n        return NULL;\n    }\n    ma_dr_wav_uninit(pWav);\n    if (sampleRate) {\n        *sampleRate = pWav->sampleRate;\n    }\n    if (channels) {\n        *channels = pWav->channels;\n    }\n    if (totalFrameCount) {\n        *totalFrameCount = pWav->totalPCMFrameCount;\n    }\n    return pSampleData;\n}\nMA_API ma_int16* ma_dr_wav_open_and_read_pcm_frames_s16(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init(&wav, onRead, onSeek, onTell, pUserData, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s16(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API float* ma_dr_wav_open_and_read_pcm_frames_f32(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init(&wav, onRead, onSeek, onTell, pUserData, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_f32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API ma_int32* ma_dr_wav_open_and_read_pcm_frames_s32(ma_dr_wav_read_proc onRead, ma_dr_wav_seek_proc onSeek, ma_dr_wav_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init(&wav, onRead, onSeek, onTell, pUserData, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\n#ifndef MA_DR_WAV_NO_STDIO\nMA_API ma_int16* ma_dr_wav_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s16(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API float* ma_dr_wav_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_f32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API ma_int32* ma_dr_wav_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\n#ifndef MA_DR_WAV_NO_WCHAR\nMA_API ma_int16* ma_dr_wav_open_file_and_read_pcm_frames_s16_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file_w(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s16(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API float* ma_dr_wav_open_file_and_read_pcm_frames_f32_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file_w(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_f32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API ma_int32* ma_dr_wav_open_file_and_read_pcm_frames_s32_w(const wchar_t* filename, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_file_w(&wav, filename, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\n#endif\n#endif\nMA_API ma_int16* ma_dr_wav_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_memory(&wav, data, dataSize, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s16(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API float* ma_dr_wav_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_memory(&wav, data, dataSize, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_f32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\nMA_API ma_int32* ma_dr_wav_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_wav wav;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalFrameCountOut) {\n        *totalFrameCountOut = 0;\n    }\n    if (!ma_dr_wav_init_memory(&wav, data, dataSize, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_wav__read_pcm_frames_and_close_s32(&wav, channelsOut, sampleRateOut, totalFrameCountOut);\n}\n#endif\nMA_API void ma_dr_wav_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        ma_dr_wav__free_from_callbacks(p, pAllocationCallbacks);\n    } else {\n        ma_dr_wav__free_default(p, NULL);\n    }\n}\nMA_API ma_uint16 ma_dr_wav_bytes_to_u16(const ma_uint8* data)\n{\n    return ((ma_uint16)data[0] << 0) | ((ma_uint16)data[1] << 8);\n}\nMA_API ma_int16 ma_dr_wav_bytes_to_s16(const ma_uint8* data)\n{\n    return (ma_int16)ma_dr_wav_bytes_to_u16(data);\n}\nMA_API ma_uint32 ma_dr_wav_bytes_to_u32(const ma_uint8* data)\n{\n    return ma_dr_wav_bytes_to_u32_le(data);\n}\nMA_API float ma_dr_wav_bytes_to_f32(const ma_uint8* data)\n{\n    union {\n        ma_uint32 u32;\n        float f32;\n    } value;\n    value.u32 = ma_dr_wav_bytes_to_u32(data);\n    return value.f32;\n}\nMA_API ma_int32 ma_dr_wav_bytes_to_s32(const ma_uint8* data)\n{\n    return (ma_int32)ma_dr_wav_bytes_to_u32(data);\n}\nMA_API ma_uint64 ma_dr_wav_bytes_to_u64(const ma_uint8* data)\n{\n    return\n        ((ma_uint64)data[0] <<  0) | ((ma_uint64)data[1] <<  8) | ((ma_uint64)data[2] << 16) | ((ma_uint64)data[3] << 24) |\n        ((ma_uint64)data[4] << 32) | ((ma_uint64)data[5] << 40) | ((ma_uint64)data[6] << 48) | ((ma_uint64)data[7] << 56);\n}\nMA_API ma_int64 ma_dr_wav_bytes_to_s64(const ma_uint8* data)\n{\n    return (ma_int64)ma_dr_wav_bytes_to_u64(data);\n}\nMA_API ma_bool32 ma_dr_wav_guid_equal(const ma_uint8 a[16], const ma_uint8 b[16])\n{\n    int i;\n    for (i = 0; i < 16; i += 1) {\n        if (a[i] != b[i]) {\n            return MA_FALSE;\n        }\n    }\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_wav_fourcc_equal(const ma_uint8* a, const char* b)\n{\n    return\n        a[0] == b[0] &&\n        a[1] == b[1] &&\n        a[2] == b[2] &&\n        a[3] == b[3];\n}\n#ifdef __MRC__\n#pragma options opt reset\n#endif\n#endif\n/* dr_wav_c end */\n#endif  /* MA_DR_WAV_IMPLEMENTATION */\n#endif  /* MA_NO_WAV */\n\n#if !defined(MA_NO_FLAC) && !defined(MA_NO_DECODING)\n#if !defined(MA_DR_FLAC_IMPLEMENTATION)\n/* dr_flac_c begin */\n#ifndef ma_dr_flac_c\n#define ma_dr_flac_c\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic push\n    #if __GNUC__ >= 7\n    #pragma GCC diagnostic ignored \"-Wimplicit-fallthrough\"\n    #endif\n#endif\n#ifdef __linux__\n    #ifndef _BSD_SOURCE\n        #define _BSD_SOURCE\n    #endif\n    #ifndef _DEFAULT_SOURCE\n        #define _DEFAULT_SOURCE\n    #endif\n    #ifndef __USE_BSD\n        #define __USE_BSD\n    #endif\n    #include <endian.h>\n#endif\n#include <stdlib.h>\n#include <string.h>\n#if !defined(MA_DR_FLAC_NO_SIMD)\n    #if defined(MA_X64) || defined(MA_X86)\n        #if defined(_MSC_VER) && !defined(__clang__)\n            #if _MSC_VER >= 1400 && !defined(MA_DR_FLAC_NO_SSE2)\n                #define MA_DR_FLAC_SUPPORT_SSE2\n            #endif\n            #if _MSC_VER >= 1600 && !defined(MA_DR_FLAC_NO_SSE41)\n                #define MA_DR_FLAC_SUPPORT_SSE41\n            #endif\n        #elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)))\n            #if defined(__SSE2__) && !defined(MA_DR_FLAC_NO_SSE2)\n                #define MA_DR_FLAC_SUPPORT_SSE2\n            #endif\n            #if defined(__SSE4_1__) && !defined(MA_DR_FLAC_NO_SSE41)\n                #define MA_DR_FLAC_SUPPORT_SSE41\n            #endif\n        #endif\n        #if !defined(__GNUC__) && !defined(__clang__) && defined(__has_include)\n            #if !defined(MA_DR_FLAC_SUPPORT_SSE2) && !defined(MA_DR_FLAC_NO_SSE2) && __has_include(<emmintrin.h>)\n                #define MA_DR_FLAC_SUPPORT_SSE2\n            #endif\n            #if !defined(MA_DR_FLAC_SUPPORT_SSE41) && !defined(MA_DR_FLAC_NO_SSE41) && __has_include(<smmintrin.h>)\n                #define MA_DR_FLAC_SUPPORT_SSE41\n            #endif\n        #endif\n        #if defined(MA_DR_FLAC_SUPPORT_SSE41)\n            #include <smmintrin.h>\n        #elif defined(MA_DR_FLAC_SUPPORT_SSE2)\n            #include <emmintrin.h>\n        #endif\n    #endif\n    #if defined(MA_ARM)\n        #if !defined(MA_DR_FLAC_NO_NEON) && (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64))\n            #define MA_DR_FLAC_SUPPORT_NEON\n            #include <arm_neon.h>\n        #endif\n    #endif\n#endif\n#if !defined(MA_DR_FLAC_NO_SIMD) && (defined(MA_X86) || defined(MA_X64))\n    #if defined(_MSC_VER) && !defined(__clang__)\n        #if _MSC_VER >= 1400\n            #include <intrin.h>\n            static void ma_dr_flac__cpuid(int info[4], int fid)\n            {\n                __cpuid(info, fid);\n            }\n        #else\n            #define MA_DR_FLAC_NO_CPUID\n        #endif\n    #else\n        #if defined(__GNUC__) || defined(__clang__)\n            static void ma_dr_flac__cpuid(int info[4], int fid)\n            {\n                #if defined(MA_X86) && defined(__PIC__)\n                    __asm__ __volatile__ (\n                        \"xchg{l} {%%}ebx, %k1;\"\n                        \"cpuid;\"\n                        \"xchg{l} {%%}ebx, %k1;\"\n                        : \"=a\"(info[0]), \"=&r\"(info[1]), \"=c\"(info[2]), \"=d\"(info[3]) : \"a\"(fid), \"c\"(0)\n                    );\n                #else\n                    __asm__ __volatile__ (\n                        \"cpuid\" : \"=a\"(info[0]), \"=b\"(info[1]), \"=c\"(info[2]), \"=d\"(info[3]) : \"a\"(fid), \"c\"(0)\n                    );\n                #endif\n            }\n        #else\n            #define MA_DR_FLAC_NO_CPUID\n        #endif\n    #endif\n#else\n    #define MA_DR_FLAC_NO_CPUID\n#endif\nstatic MA_INLINE ma_bool32 ma_dr_flac_has_sse2(void)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    #if (defined(MA_X64) || defined(MA_X86)) && !defined(MA_DR_FLAC_NO_SSE2)\n        #if defined(MA_X64)\n            return MA_TRUE;\n        #elif (defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__)\n            return MA_TRUE;\n        #else\n            #if defined(MA_DR_FLAC_NO_CPUID)\n                return MA_FALSE;\n            #else\n                int info[4];\n                ma_dr_flac__cpuid(info, 1);\n                return (info[3] & (1 << 26)) != 0;\n            #endif\n        #endif\n    #else\n        return MA_FALSE;\n    #endif\n#else\n    return MA_FALSE;\n#endif\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac_has_sse41(void)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE41)\n    #if (defined(MA_X64) || defined(MA_X86)) && !defined(MA_DR_FLAC_NO_SSE41)\n        #if defined(__SSE4_1__) || defined(__AVX__)\n            return MA_TRUE;\n        #else\n            #if defined(MA_DR_FLAC_NO_CPUID)\n                return MA_FALSE;\n            #else\n                int info[4];\n                ma_dr_flac__cpuid(info, 1);\n                return (info[2] & (1 << 19)) != 0;\n            #endif\n        #endif\n    #else\n        return MA_FALSE;\n    #endif\n#else\n    return MA_FALSE;\n#endif\n}\n#if defined(_MSC_VER) && _MSC_VER >= 1500 && (defined(MA_X86) || defined(MA_X64)) && !defined(__clang__)\n    #define MA_DR_FLAC_HAS_LZCNT_INTRINSIC\n#elif (defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7)))\n    #define MA_DR_FLAC_HAS_LZCNT_INTRINSIC\n#elif defined(__clang__)\n    #if defined(__has_builtin)\n        #if __has_builtin(__builtin_clzll) || __has_builtin(__builtin_clzl)\n            #define MA_DR_FLAC_HAS_LZCNT_INTRINSIC\n        #endif\n    #endif\n#endif\n#if defined(_MSC_VER) && _MSC_VER >= 1400 && !defined(__clang__)\n    #define MA_DR_FLAC_HAS_BYTESWAP16_INTRINSIC\n    #define MA_DR_FLAC_HAS_BYTESWAP32_INTRINSIC\n    #define MA_DR_FLAC_HAS_BYTESWAP64_INTRINSIC\n#elif defined(__clang__)\n    #if defined(__has_builtin)\n        #if __has_builtin(__builtin_bswap16)\n            #define MA_DR_FLAC_HAS_BYTESWAP16_INTRINSIC\n        #endif\n        #if __has_builtin(__builtin_bswap32)\n            #define MA_DR_FLAC_HAS_BYTESWAP32_INTRINSIC\n        #endif\n        #if __has_builtin(__builtin_bswap64)\n            #define MA_DR_FLAC_HAS_BYTESWAP64_INTRINSIC\n        #endif\n    #endif\n#elif defined(__GNUC__)\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))\n        #define MA_DR_FLAC_HAS_BYTESWAP32_INTRINSIC\n        #define MA_DR_FLAC_HAS_BYTESWAP64_INTRINSIC\n    #endif\n    #if ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))\n        #define MA_DR_FLAC_HAS_BYTESWAP16_INTRINSIC\n    #endif\n#elif defined(__WATCOMC__) && defined(__386__)\n    #define MA_DR_FLAC_HAS_BYTESWAP16_INTRINSIC\n    #define MA_DR_FLAC_HAS_BYTESWAP32_INTRINSIC\n    #define MA_DR_FLAC_HAS_BYTESWAP64_INTRINSIC\n    extern __inline ma_uint16 _watcom_bswap16(ma_uint16);\n    extern __inline ma_uint32 _watcom_bswap32(ma_uint32);\n    extern __inline ma_uint64 _watcom_bswap64(ma_uint64);\n#pragma aux _watcom_bswap16 = \\\n    \"xchg al, ah\" \\\n    parm  [ax]    \\\n    value [ax]    \\\n    modify nomemory;\n#pragma aux _watcom_bswap32 = \\\n    \"bswap eax\" \\\n    parm  [eax] \\\n    value [eax] \\\n    modify nomemory;\n#pragma aux _watcom_bswap64 = \\\n    \"bswap eax\"     \\\n    \"bswap edx\"     \\\n    \"xchg eax,edx\"  \\\n    parm [eax edx]  \\\n    value [eax edx] \\\n    modify nomemory;\n#endif\n#ifndef MA_DR_FLAC_ASSERT\n#include <assert.h>\n#define MA_DR_FLAC_ASSERT(expression)           assert(expression)\n#endif\n#ifndef MA_DR_FLAC_MALLOC\n#define MA_DR_FLAC_MALLOC(sz)                   malloc((sz))\n#endif\n#ifndef MA_DR_FLAC_REALLOC\n#define MA_DR_FLAC_REALLOC(p, sz)               realloc((p), (sz))\n#endif\n#ifndef MA_DR_FLAC_FREE\n#define MA_DR_FLAC_FREE(p)                      free((p))\n#endif\n#ifndef MA_DR_FLAC_COPY_MEMORY\n#define MA_DR_FLAC_COPY_MEMORY(dst, src, sz)    memcpy((dst), (src), (sz))\n#endif\n#ifndef MA_DR_FLAC_ZERO_MEMORY\n#define MA_DR_FLAC_ZERO_MEMORY(p, sz)           memset((p), 0, (sz))\n#endif\n#ifndef MA_DR_FLAC_ZERO_OBJECT\n#define MA_DR_FLAC_ZERO_OBJECT(p)               MA_DR_FLAC_ZERO_MEMORY((p), sizeof(*(p)))\n#endif\n#define MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE                     64\n#define MA_DR_FLAC_SUBFRAME_CONSTANT                        0\n#define MA_DR_FLAC_SUBFRAME_VERBATIM                        1\n#define MA_DR_FLAC_SUBFRAME_FIXED                           8\n#define MA_DR_FLAC_SUBFRAME_LPC                             32\n#define MA_DR_FLAC_SUBFRAME_RESERVED                        255\n#define MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE  0\n#define MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2 1\n#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT           0\n#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE             8\n#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE            9\n#define MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE              10\n#define MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES                  18\n#define MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES             36\n#define MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES       12\n#define ma_dr_flac_align(x, a)                              ((((x) + (a) - 1) / (a)) * (a))\nMA_API void ma_dr_flac_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision)\n{\n    if (pMajor) {\n        *pMajor = MA_DR_FLAC_VERSION_MAJOR;\n    }\n    if (pMinor) {\n        *pMinor = MA_DR_FLAC_VERSION_MINOR;\n    }\n    if (pRevision) {\n        *pRevision = MA_DR_FLAC_VERSION_REVISION;\n    }\n}\nMA_API const char* ma_dr_flac_version_string(void)\n{\n    return MA_DR_FLAC_VERSION_STRING;\n}\n#if defined(__has_feature)\n    #if __has_feature(thread_sanitizer)\n        #define MA_DR_FLAC_NO_THREAD_SANITIZE __attribute__((no_sanitize(\"thread\")))\n    #else\n        #define MA_DR_FLAC_NO_THREAD_SANITIZE\n    #endif\n#else\n    #define MA_DR_FLAC_NO_THREAD_SANITIZE\n#endif\n#if defined(MA_DR_FLAC_HAS_LZCNT_INTRINSIC)\nstatic ma_bool32 ma_dr_flac__gIsLZCNTSupported = MA_FALSE;\n#endif\n#ifndef MA_DR_FLAC_NO_CPUID\nstatic ma_bool32 ma_dr_flac__gIsSSE2Supported  = MA_FALSE;\nstatic ma_bool32 ma_dr_flac__gIsSSE41Supported = MA_FALSE;\nMA_DR_FLAC_NO_THREAD_SANITIZE static void ma_dr_flac__init_cpu_caps(void)\n{\n    static ma_bool32 isCPUCapsInitialized = MA_FALSE;\n    if (!isCPUCapsInitialized) {\n#if defined(MA_DR_FLAC_HAS_LZCNT_INTRINSIC)\n        int info[4] = {0};\n        ma_dr_flac__cpuid(info, 0x80000001);\n        ma_dr_flac__gIsLZCNTSupported = (info[2] & (1 << 5)) != 0;\n#endif\n        ma_dr_flac__gIsSSE2Supported = ma_dr_flac_has_sse2();\n        ma_dr_flac__gIsSSE41Supported = ma_dr_flac_has_sse41();\n        isCPUCapsInitialized = MA_TRUE;\n    }\n}\n#else\nstatic ma_bool32 ma_dr_flac__gIsNEONSupported  = MA_FALSE;\nstatic MA_INLINE ma_bool32 ma_dr_flac__has_neon(void)\n{\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\n    #if defined(MA_ARM) && !defined(MA_DR_FLAC_NO_NEON)\n        #if (defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64))\n            return MA_TRUE;\n        #else\n            return MA_FALSE;\n        #endif\n    #else\n        return MA_FALSE;\n    #endif\n#else\n    return MA_FALSE;\n#endif\n}\nMA_DR_FLAC_NO_THREAD_SANITIZE static void ma_dr_flac__init_cpu_caps(void)\n{\n    ma_dr_flac__gIsNEONSupported = ma_dr_flac__has_neon();\n#if defined(MA_DR_FLAC_HAS_LZCNT_INTRINSIC) && defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5)\n    ma_dr_flac__gIsLZCNTSupported = MA_TRUE;\n#endif\n}\n#endif\nstatic MA_INLINE ma_bool32 ma_dr_flac__is_little_endian(void)\n{\n#if defined(MA_X86) || defined(MA_X64)\n    return MA_TRUE;\n#elif defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN\n    return MA_TRUE;\n#else\n    int n = 1;\n    return (*(char*)&n) == 1;\n#endif\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac__swap_endian_uint16(ma_uint16 n)\n{\n#ifdef MA_DR_FLAC_HAS_BYTESWAP16_INTRINSIC\n    #if defined(_MSC_VER) && !defined(__clang__)\n        return _byteswap_ushort(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        return __builtin_bswap16(n);\n    #elif defined(__WATCOMC__) && defined(__386__)\n        return _watcom_bswap16(n);\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & 0xFF00) >> 8) |\n           ((n & 0x00FF) << 8);\n#endif\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__swap_endian_uint32(ma_uint32 n)\n{\n#ifdef MA_DR_FLAC_HAS_BYTESWAP32_INTRINSIC\n    #if defined(_MSC_VER) && !defined(__clang__)\n        return _byteswap_ulong(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        #if defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 6) && !defined(__ARM_ARCH_6M__) && !defined(MA_64BIT)\n            ma_uint32 r;\n            __asm__ __volatile__ (\n            #if defined(MA_64BIT)\n                \"rev %w[out], %w[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)\n            #else\n                \"rev %[out], %[in]\" : [out]\"=r\"(r) : [in]\"r\"(n)\n            #endif\n            );\n            return r;\n        #else\n            return __builtin_bswap32(n);\n        #endif\n    #elif defined(__WATCOMC__) && defined(__386__)\n        return _watcom_bswap32(n);\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & 0xFF000000) >> 24) |\n           ((n & 0x00FF0000) >>  8) |\n           ((n & 0x0000FF00) <<  8) |\n           ((n & 0x000000FF) << 24);\n#endif\n}\nstatic MA_INLINE ma_uint64 ma_dr_flac__swap_endian_uint64(ma_uint64 n)\n{\n#ifdef MA_DR_FLAC_HAS_BYTESWAP64_INTRINSIC\n    #if defined(_MSC_VER) && !defined(__clang__)\n        return _byteswap_uint64(n);\n    #elif defined(__GNUC__) || defined(__clang__)\n        return __builtin_bswap64(n);\n    #elif defined(__WATCOMC__) && defined(__386__)\n        return _watcom_bswap64(n);\n    #else\n        #error \"This compiler does not support the byte swap intrinsic.\"\n    #endif\n#else\n    return ((n & ((ma_uint64)0xFF000000 << 32)) >> 56) |\n           ((n & ((ma_uint64)0x00FF0000 << 32)) >> 40) |\n           ((n & ((ma_uint64)0x0000FF00 << 32)) >> 24) |\n           ((n & ((ma_uint64)0x000000FF << 32)) >>  8) |\n           ((n & ((ma_uint64)0xFF000000      )) <<  8) |\n           ((n & ((ma_uint64)0x00FF0000      )) << 24) |\n           ((n & ((ma_uint64)0x0000FF00      )) << 40) |\n           ((n & ((ma_uint64)0x000000FF      )) << 56);\n#endif\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac__be2host_16(ma_uint16 n)\n{\n    if (ma_dr_flac__is_little_endian()) {\n        return ma_dr_flac__swap_endian_uint16(n);\n    }\n    return n;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__be2host_32(ma_uint32 n)\n{\n    if (ma_dr_flac__is_little_endian()) {\n        return ma_dr_flac__swap_endian_uint32(n);\n    }\n    return n;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__be2host_32_ptr_unaligned(const void* pData)\n{\n    const ma_uint8* pNum = (ma_uint8*)pData;\n    return *(pNum) << 24 | *(pNum+1) << 16 | *(pNum+2) << 8 | *(pNum+3);\n}\nstatic MA_INLINE ma_uint64 ma_dr_flac__be2host_64(ma_uint64 n)\n{\n    if (ma_dr_flac__is_little_endian()) {\n        return ma_dr_flac__swap_endian_uint64(n);\n    }\n    return n;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__le2host_32(ma_uint32 n)\n{\n    if (!ma_dr_flac__is_little_endian()) {\n        return ma_dr_flac__swap_endian_uint32(n);\n    }\n    return n;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__le2host_32_ptr_unaligned(const void* pData)\n{\n    const ma_uint8* pNum = (ma_uint8*)pData;\n    return *pNum | *(pNum+1) << 8 |  *(pNum+2) << 16 | *(pNum+3) << 24;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__unsynchsafe_32(ma_uint32 n)\n{\n    ma_uint32 result = 0;\n    result |= (n & 0x7F000000) >> 3;\n    result |= (n & 0x007F0000) >> 2;\n    result |= (n & 0x00007F00) >> 1;\n    result |= (n & 0x0000007F) >> 0;\n    return result;\n}\nstatic ma_uint8 ma_dr_flac__crc8_table[] = {\n    0x00, 0x07, 0x0E, 0x09, 0x1C, 0x1B, 0x12, 0x15, 0x38, 0x3F, 0x36, 0x31, 0x24, 0x23, 0x2A, 0x2D,\n    0x70, 0x77, 0x7E, 0x79, 0x6C, 0x6B, 0x62, 0x65, 0x48, 0x4F, 0x46, 0x41, 0x54, 0x53, 0x5A, 0x5D,\n    0xE0, 0xE7, 0xEE, 0xE9, 0xFC, 0xFB, 0xF2, 0xF5, 0xD8, 0xDF, 0xD6, 0xD1, 0xC4, 0xC3, 0xCA, 0xCD,\n    0x90, 0x97, 0x9E, 0x99, 0x8C, 0x8B, 0x82, 0x85, 0xA8, 0xAF, 0xA6, 0xA1, 0xB4, 0xB3, 0xBA, 0xBD,\n    0xC7, 0xC0, 0xC9, 0xCE, 0xDB, 0xDC, 0xD5, 0xD2, 0xFF, 0xF8, 0xF1, 0xF6, 0xE3, 0xE4, 0xED, 0xEA,\n    0xB7, 0xB0, 0xB9, 0xBE, 0xAB, 0xAC, 0xA5, 0xA2, 0x8F, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9D, 0x9A,\n    0x27, 0x20, 0x29, 0x2E, 0x3B, 0x3C, 0x35, 0x32, 0x1F, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0D, 0x0A,\n    0x57, 0x50, 0x59, 0x5E, 0x4B, 0x4C, 0x45, 0x42, 0x6F, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7D, 0x7A,\n    0x89, 0x8E, 0x87, 0x80, 0x95, 0x92, 0x9B, 0x9C, 0xB1, 0xB6, 0xBF, 0xB8, 0xAD, 0xAA, 0xA3, 0xA4,\n    0xF9, 0xFE, 0xF7, 0xF0, 0xE5, 0xE2, 0xEB, 0xEC, 0xC1, 0xC6, 0xCF, 0xC8, 0xDD, 0xDA, 0xD3, 0xD4,\n    0x69, 0x6E, 0x67, 0x60, 0x75, 0x72, 0x7B, 0x7C, 0x51, 0x56, 0x5F, 0x58, 0x4D, 0x4A, 0x43, 0x44,\n    0x19, 0x1E, 0x17, 0x10, 0x05, 0x02, 0x0B, 0x0C, 0x21, 0x26, 0x2F, 0x28, 0x3D, 0x3A, 0x33, 0x34,\n    0x4E, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5C, 0x5B, 0x76, 0x71, 0x78, 0x7F, 0x6A, 0x6D, 0x64, 0x63,\n    0x3E, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2C, 0x2B, 0x06, 0x01, 0x08, 0x0F, 0x1A, 0x1D, 0x14, 0x13,\n    0xAE, 0xA9, 0xA0, 0xA7, 0xB2, 0xB5, 0xBC, 0xBB, 0x96, 0x91, 0x98, 0x9F, 0x8A, 0x8D, 0x84, 0x83,\n    0xDE, 0xD9, 0xD0, 0xD7, 0xC2, 0xC5, 0xCC, 0xCB, 0xE6, 0xE1, 0xE8, 0xEF, 0xFA, 0xFD, 0xF4, 0xF3\n};\nstatic ma_uint16 ma_dr_flac__crc16_table[] = {\n    0x0000, 0x8005, 0x800F, 0x000A, 0x801B, 0x001E, 0x0014, 0x8011,\n    0x8033, 0x0036, 0x003C, 0x8039, 0x0028, 0x802D, 0x8027, 0x0022,\n    0x8063, 0x0066, 0x006C, 0x8069, 0x0078, 0x807D, 0x8077, 0x0072,\n    0x0050, 0x8055, 0x805F, 0x005A, 0x804B, 0x004E, 0x0044, 0x8041,\n    0x80C3, 0x00C6, 0x00CC, 0x80C9, 0x00D8, 0x80DD, 0x80D7, 0x00D2,\n    0x00F0, 0x80F5, 0x80FF, 0x00FA, 0x80EB, 0x00EE, 0x00E4, 0x80E1,\n    0x00A0, 0x80A5, 0x80AF, 0x00AA, 0x80BB, 0x00BE, 0x00B4, 0x80B1,\n    0x8093, 0x0096, 0x009C, 0x8099, 0x0088, 0x808D, 0x8087, 0x0082,\n    0x8183, 0x0186, 0x018C, 0x8189, 0x0198, 0x819D, 0x8197, 0x0192,\n    0x01B0, 0x81B5, 0x81BF, 0x01BA, 0x81AB, 0x01AE, 0x01A4, 0x81A1,\n    0x01E0, 0x81E5, 0x81EF, 0x01EA, 0x81FB, 0x01FE, 0x01F4, 0x81F1,\n    0x81D3, 0x01D6, 0x01DC, 0x81D9, 0x01C8, 0x81CD, 0x81C7, 0x01C2,\n    0x0140, 0x8145, 0x814F, 0x014A, 0x815B, 0x015E, 0x0154, 0x8151,\n    0x8173, 0x0176, 0x017C, 0x8179, 0x0168, 0x816D, 0x8167, 0x0162,\n    0x8123, 0x0126, 0x012C, 0x8129, 0x0138, 0x813D, 0x8137, 0x0132,\n    0x0110, 0x8115, 0x811F, 0x011A, 0x810B, 0x010E, 0x0104, 0x8101,\n    0x8303, 0x0306, 0x030C, 0x8309, 0x0318, 0x831D, 0x8317, 0x0312,\n    0x0330, 0x8335, 0x833F, 0x033A, 0x832B, 0x032E, 0x0324, 0x8321,\n    0x0360, 0x8365, 0x836F, 0x036A, 0x837B, 0x037E, 0x0374, 0x8371,\n    0x8353, 0x0356, 0x035C, 0x8359, 0x0348, 0x834D, 0x8347, 0x0342,\n    0x03C0, 0x83C5, 0x83CF, 0x03CA, 0x83DB, 0x03DE, 0x03D4, 0x83D1,\n    0x83F3, 0x03F6, 0x03FC, 0x83F9, 0x03E8, 0x83ED, 0x83E7, 0x03E2,\n    0x83A3, 0x03A6, 0x03AC, 0x83A9, 0x03B8, 0x83BD, 0x83B7, 0x03B2,\n    0x0390, 0x8395, 0x839F, 0x039A, 0x838B, 0x038E, 0x0384, 0x8381,\n    0x0280, 0x8285, 0x828F, 0x028A, 0x829B, 0x029E, 0x0294, 0x8291,\n    0x82B3, 0x02B6, 0x02BC, 0x82B9, 0x02A8, 0x82AD, 0x82A7, 0x02A2,\n    0x82E3, 0x02E6, 0x02EC, 0x82E9, 0x02F8, 0x82FD, 0x82F7, 0x02F2,\n    0x02D0, 0x82D5, 0x82DF, 0x02DA, 0x82CB, 0x02CE, 0x02C4, 0x82C1,\n    0x8243, 0x0246, 0x024C, 0x8249, 0x0258, 0x825D, 0x8257, 0x0252,\n    0x0270, 0x8275, 0x827F, 0x027A, 0x826B, 0x026E, 0x0264, 0x8261,\n    0x0220, 0x8225, 0x822F, 0x022A, 0x823B, 0x023E, 0x0234, 0x8231,\n    0x8213, 0x0216, 0x021C, 0x8219, 0x0208, 0x820D, 0x8207, 0x0202\n};\nstatic MA_INLINE ma_uint8 ma_dr_flac_crc8_byte(ma_uint8 crc, ma_uint8 data)\n{\n    return ma_dr_flac__crc8_table[crc ^ data];\n}\nstatic MA_INLINE ma_uint8 ma_dr_flac_crc8(ma_uint8 crc, ma_uint32 data, ma_uint32 count)\n{\n#ifdef MA_DR_FLAC_NO_CRC\n    (void)crc;\n    (void)data;\n    (void)count;\n    return 0;\n#else\n#if 0\n    ma_uint8 p = 0x07;\n    for (int i = count-1; i >= 0; --i) {\n        ma_uint8 bit = (data & (1 << i)) >> i;\n        if (crc & 0x80) {\n            crc = ((crc << 1) | bit) ^ p;\n        } else {\n            crc = ((crc << 1) | bit);\n        }\n    }\n    return crc;\n#else\n    ma_uint32 wholeBytes;\n    ma_uint32 leftoverBits;\n    ma_uint64 leftoverDataMask;\n    static ma_uint64 leftoverDataMaskTable[8] = {\n        0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F\n    };\n    MA_DR_FLAC_ASSERT(count <= 32);\n    wholeBytes = count >> 3;\n    leftoverBits = count - (wholeBytes*8);\n    leftoverDataMask = leftoverDataMaskTable[leftoverBits];\n    switch (wholeBytes) {\n        case 4: crc = ma_dr_flac_crc8_byte(crc, (ma_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits)));\n        case 3: crc = ma_dr_flac_crc8_byte(crc, (ma_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits)));\n        case 2: crc = ma_dr_flac_crc8_byte(crc, (ma_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits)));\n        case 1: crc = ma_dr_flac_crc8_byte(crc, (ma_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits)));\n        case 0: if (leftoverBits > 0) crc = (ma_uint8)((crc << leftoverBits) ^ ma_dr_flac__crc8_table[(crc >> (8 - leftoverBits)) ^ (data & leftoverDataMask)]);\n    }\n    return crc;\n#endif\n#endif\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16_byte(ma_uint16 crc, ma_uint8 data)\n{\n    return (crc << 8) ^ ma_dr_flac__crc16_table[(ma_uint8)(crc >> 8) ^ data];\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16_cache(ma_uint16 crc, ma_dr_flac_cache_t data)\n{\n#ifdef MA_64BIT\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 56) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 48) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 40) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 32) & 0xFF));\n#endif\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 24) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 16) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >>  8) & 0xFF));\n    crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >>  0) & 0xFF));\n    return crc;\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16_bytes(ma_uint16 crc, ma_dr_flac_cache_t data, ma_uint32 byteCount)\n{\n    switch (byteCount)\n    {\n#ifdef MA_64BIT\n    case 8: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 56) & 0xFF));\n    case 7: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 48) & 0xFF));\n    case 6: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 40) & 0xFF));\n    case 5: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 32) & 0xFF));\n#endif\n    case 4: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 24) & 0xFF));\n    case 3: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >> 16) & 0xFF));\n    case 2: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >>  8) & 0xFF));\n    case 1: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data >>  0) & 0xFF));\n    }\n    return crc;\n}\n#if 0\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16__32bit(ma_uint16 crc, ma_uint32 data, ma_uint32 count)\n{\n#ifdef MA_DR_FLAC_NO_CRC\n    (void)crc;\n    (void)data;\n    (void)count;\n    return 0;\n#else\n#if 0\n    ma_uint16 p = 0x8005;\n    for (int i = count-1; i >= 0; --i) {\n        ma_uint16 bit = (data & (1ULL << i)) >> i;\n        if (r & 0x8000) {\n            r = ((r << 1) | bit) ^ p;\n        } else {\n            r = ((r << 1) | bit);\n        }\n    }\n    return crc;\n#else\n    ma_uint32 wholeBytes;\n    ma_uint32 leftoverBits;\n    ma_uint64 leftoverDataMask;\n    static ma_uint64 leftoverDataMaskTable[8] = {\n        0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F\n    };\n    MA_DR_FLAC_ASSERT(count <= 64);\n    wholeBytes = count >> 3;\n    leftoverBits = count & 7;\n    leftoverDataMask = leftoverDataMaskTable[leftoverBits];\n    switch (wholeBytes) {\n        default:\n        case 4: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (0xFF000000UL << leftoverBits)) >> (24 + leftoverBits)));\n        case 3: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (0x00FF0000UL << leftoverBits)) >> (16 + leftoverBits)));\n        case 2: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (0x0000FF00UL << leftoverBits)) >> ( 8 + leftoverBits)));\n        case 1: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (0x000000FFUL << leftoverBits)) >> ( 0 + leftoverBits)));\n        case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ ma_dr_flac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)];\n    }\n    return crc;\n#endif\n#endif\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16__64bit(ma_uint16 crc, ma_uint64 data, ma_uint32 count)\n{\n#ifdef MA_DR_FLAC_NO_CRC\n    (void)crc;\n    (void)data;\n    (void)count;\n    return 0;\n#else\n    ma_uint32 wholeBytes;\n    ma_uint32 leftoverBits;\n    ma_uint64 leftoverDataMask;\n    static ma_uint64 leftoverDataMaskTable[8] = {\n        0x00, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F\n    };\n    MA_DR_FLAC_ASSERT(count <= 64);\n    wholeBytes = count >> 3;\n    leftoverBits = count & 7;\n    leftoverDataMask = leftoverDataMaskTable[leftoverBits];\n    switch (wholeBytes) {\n        default:\n        case 8: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0xFF000000 << 32) << leftoverBits)) >> (56 + leftoverBits)));\n        case 7: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x00FF0000 << 32) << leftoverBits)) >> (48 + leftoverBits)));\n        case 6: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x0000FF00 << 32) << leftoverBits)) >> (40 + leftoverBits)));\n        case 5: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x000000FF << 32) << leftoverBits)) >> (32 + leftoverBits)));\n        case 4: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0xFF000000      ) << leftoverBits)) >> (24 + leftoverBits)));\n        case 3: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x00FF0000      ) << leftoverBits)) >> (16 + leftoverBits)));\n        case 2: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x0000FF00      ) << leftoverBits)) >> ( 8 + leftoverBits)));\n        case 1: crc = ma_dr_flac_crc16_byte(crc, (ma_uint8)((data & (((ma_uint64)0x000000FF      ) << leftoverBits)) >> ( 0 + leftoverBits)));\n        case 0: if (leftoverBits > 0) crc = (crc << leftoverBits) ^ ma_dr_flac__crc16_table[(crc >> (16 - leftoverBits)) ^ (data & leftoverDataMask)];\n    }\n    return crc;\n#endif\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac_crc16(ma_uint16 crc, ma_dr_flac_cache_t data, ma_uint32 count)\n{\n#ifdef MA_64BIT\n    return ma_dr_flac_crc16__64bit(crc, data, count);\n#else\n    return ma_dr_flac_crc16__32bit(crc, data, count);\n#endif\n}\n#endif\n#ifdef MA_64BIT\n#define ma_dr_flac__be2host__cache_line ma_dr_flac__be2host_64\n#else\n#define ma_dr_flac__be2host__cache_line ma_dr_flac__be2host_32\n#endif\n#define MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs)                      (sizeof((bs)->cache))\n#define MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)                       (sizeof((bs)->cache)*8)\n#define MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)                  (MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs) - (bs)->consumedBits)\n#define MA_DR_FLAC_CACHE_L1_SELECTION_MASK(_bitCount)           (~((~(ma_dr_flac_cache_t)0) >> (_bitCount)))\n#define MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT(bs, _bitCount)      (MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs) - (_bitCount))\n#define MA_DR_FLAC_CACHE_L1_SELECT(bs, _bitCount)               (((bs)->cache) & MA_DR_FLAC_CACHE_L1_SELECTION_MASK(_bitCount))\n#define MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, _bitCount)     (MA_DR_FLAC_CACHE_L1_SELECT((bs), (_bitCount)) >>  MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT((bs), (_bitCount)))\n#define MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT_SAFE(bs, _bitCount)(MA_DR_FLAC_CACHE_L1_SELECT((bs), (_bitCount)) >> (MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT((bs), (_bitCount)) & (MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)-1)))\n#define MA_DR_FLAC_CACHE_L2_SIZE_BYTES(bs)                      (sizeof((bs)->cacheL2))\n#define MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)                      (MA_DR_FLAC_CACHE_L2_SIZE_BYTES(bs) / sizeof((bs)->cacheL2[0]))\n#define MA_DR_FLAC_CACHE_L2_LINES_REMAINING(bs)                 (MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs) - (bs)->nextL2Line)\n#ifndef MA_DR_FLAC_NO_CRC\nstatic MA_INLINE void ma_dr_flac__reset_crc16(ma_dr_flac_bs* bs)\n{\n    bs->crc16 = 0;\n    bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3;\n}\nstatic MA_INLINE void ma_dr_flac__update_crc16(ma_dr_flac_bs* bs)\n{\n    if (bs->crc16CacheIgnoredBytes == 0) {\n        bs->crc16 = ma_dr_flac_crc16_cache(bs->crc16, bs->crc16Cache);\n    } else {\n        bs->crc16 = ma_dr_flac_crc16_bytes(bs->crc16, bs->crc16Cache, MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs) - bs->crc16CacheIgnoredBytes);\n        bs->crc16CacheIgnoredBytes = 0;\n    }\n}\nstatic MA_INLINE ma_uint16 ma_dr_flac__flush_crc16(ma_dr_flac_bs* bs)\n{\n    MA_DR_FLAC_ASSERT((MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) & 7) == 0);\n    if (MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) == 0) {\n        ma_dr_flac__update_crc16(bs);\n    } else {\n        bs->crc16 = ma_dr_flac_crc16_bytes(bs->crc16, bs->crc16Cache >> MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs), (bs->consumedBits >> 3) - bs->crc16CacheIgnoredBytes);\n        bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3;\n    }\n    return bs->crc16;\n}\n#endif\nstatic MA_INLINE ma_bool32 ma_dr_flac__reload_l1_cache_from_l2(ma_dr_flac_bs* bs)\n{\n    size_t bytesRead;\n    size_t alignedL1LineCount;\n    if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n        bs->cache = bs->cacheL2[bs->nextL2Line++];\n        return MA_TRUE;\n    }\n    if (bs->unalignedByteCount > 0) {\n        return MA_FALSE;\n    }\n    bytesRead = bs->onRead(bs->pUserData, bs->cacheL2, MA_DR_FLAC_CACHE_L2_SIZE_BYTES(bs));\n    bs->nextL2Line = 0;\n    if (bytesRead == MA_DR_FLAC_CACHE_L2_SIZE_BYTES(bs)) {\n        bs->cache = bs->cacheL2[bs->nextL2Line++];\n        return MA_TRUE;\n    }\n    alignedL1LineCount = bytesRead / MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs);\n    bs->unalignedByteCount = bytesRead - (alignedL1LineCount * MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs));\n    if (bs->unalignedByteCount > 0) {\n        bs->unalignedCache = bs->cacheL2[alignedL1LineCount];\n    }\n    if (alignedL1LineCount > 0) {\n        size_t offset = MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs) - alignedL1LineCount;\n        size_t i;\n        for (i = alignedL1LineCount; i > 0; --i) {\n            bs->cacheL2[i-1 + offset] = bs->cacheL2[i-1];\n        }\n        bs->nextL2Line = (ma_uint32)offset;\n        bs->cache = bs->cacheL2[bs->nextL2Line++];\n        return MA_TRUE;\n    } else {\n        bs->nextL2Line = MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs);\n        return MA_FALSE;\n    }\n}\nstatic ma_bool32 ma_dr_flac__reload_cache(ma_dr_flac_bs* bs)\n{\n    size_t bytesRead;\n#ifndef MA_DR_FLAC_NO_CRC\n    ma_dr_flac__update_crc16(bs);\n#endif\n    if (ma_dr_flac__reload_l1_cache_from_l2(bs)) {\n        bs->cache = ma_dr_flac__be2host__cache_line(bs->cache);\n        bs->consumedBits = 0;\n#ifndef MA_DR_FLAC_NO_CRC\n        bs->crc16Cache = bs->cache;\n#endif\n        return MA_TRUE;\n    }\n    bytesRead = bs->unalignedByteCount;\n    if (bytesRead == 0) {\n        bs->consumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n        return MA_FALSE;\n    }\n    MA_DR_FLAC_ASSERT(bytesRead < MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs));\n    bs->consumedBits = (ma_uint32)(MA_DR_FLAC_CACHE_L1_SIZE_BYTES(bs) - bytesRead) * 8;\n    bs->cache = ma_dr_flac__be2host__cache_line(bs->unalignedCache);\n    bs->cache &= MA_DR_FLAC_CACHE_L1_SELECTION_MASK(MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs));\n    bs->unalignedByteCount = 0;\n#ifndef MA_DR_FLAC_NO_CRC\n    bs->crc16Cache = bs->cache >> bs->consumedBits;\n    bs->crc16CacheIgnoredBytes = bs->consumedBits >> 3;\n#endif\n    return MA_TRUE;\n}\nstatic void ma_dr_flac__reset_cache(ma_dr_flac_bs* bs)\n{\n    bs->nextL2Line   = MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs);\n    bs->consumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n    bs->cache = 0;\n    bs->unalignedByteCount = 0;\n    bs->unalignedCache = 0;\n#ifndef MA_DR_FLAC_NO_CRC\n    bs->crc16Cache = 0;\n    bs->crc16CacheIgnoredBytes = 0;\n#endif\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac__read_uint32(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint32* pResultOut)\n{\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResultOut != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 32);\n    if (bs->consumedBits == MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)) {\n        if (!ma_dr_flac__reload_cache(bs)) {\n            return MA_FALSE;\n        }\n    }\n    if (bitCount <= MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n#ifdef MA_64BIT\n        *pResultOut = (ma_uint32)MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCount);\n        bs->consumedBits += bitCount;\n        bs->cache <<= bitCount;\n#else\n        if (bitCount < MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)) {\n            *pResultOut = (ma_uint32)MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCount);\n            bs->consumedBits += bitCount;\n            bs->cache <<= bitCount;\n        } else {\n            *pResultOut = (ma_uint32)bs->cache;\n            bs->consumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n            bs->cache = 0;\n        }\n#endif\n        return MA_TRUE;\n    } else {\n        ma_uint32 bitCountHi = MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs);\n        ma_uint32 bitCountLo = bitCount - bitCountHi;\n        ma_uint32 resultHi;\n        MA_DR_FLAC_ASSERT(bitCountHi > 0);\n        MA_DR_FLAC_ASSERT(bitCountHi < 32);\n        resultHi = (ma_uint32)MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountHi);\n        if (!ma_dr_flac__reload_cache(bs)) {\n            return MA_FALSE;\n        }\n        if (bitCountLo > MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n            return MA_FALSE;\n        }\n        *pResultOut = (resultHi << bitCountLo) | (ma_uint32)MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, bitCountLo);\n        bs->consumedBits += bitCountLo;\n        bs->cache <<= bitCountLo;\n        return MA_TRUE;\n    }\n}\nstatic ma_bool32 ma_dr_flac__read_int32(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int32* pResult)\n{\n    ma_uint32 result;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResult != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 32);\n    if (!ma_dr_flac__read_uint32(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    if (bitCount < 32) {\n        ma_uint32 signbit;\n        signbit = ((result >> (bitCount-1)) & 0x01);\n        result |= (~signbit + 1) << bitCount;\n    }\n    *pResult = (ma_int32)result;\n    return MA_TRUE;\n}\n#ifdef MA_64BIT\nstatic ma_bool32 ma_dr_flac__read_uint64(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint64* pResultOut)\n{\n    ma_uint32 resultHi;\n    ma_uint32 resultLo;\n    MA_DR_FLAC_ASSERT(bitCount <= 64);\n    MA_DR_FLAC_ASSERT(bitCount >  32);\n    if (!ma_dr_flac__read_uint32(bs, bitCount - 32, &resultHi)) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_flac__read_uint32(bs, 32, &resultLo)) {\n        return MA_FALSE;\n    }\n    *pResultOut = (((ma_uint64)resultHi) << 32) | ((ma_uint64)resultLo);\n    return MA_TRUE;\n}\n#endif\n#if 0\nstatic ma_bool32 ma_dr_flac__read_int64(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int64* pResultOut)\n{\n    ma_uint64 result;\n    ma_uint64 signbit;\n    MA_DR_FLAC_ASSERT(bitCount <= 64);\n    if (!ma_dr_flac__read_uint64(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    signbit = ((result >> (bitCount-1)) & 0x01);\n    result |= (~signbit + 1) << bitCount;\n    *pResultOut = (ma_int64)result;\n    return MA_TRUE;\n}\n#endif\nstatic ma_bool32 ma_dr_flac__read_uint16(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint16* pResult)\n{\n    ma_uint32 result;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResult != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 16);\n    if (!ma_dr_flac__read_uint32(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    *pResult = (ma_uint16)result;\n    return MA_TRUE;\n}\n#if 0\nstatic ma_bool32 ma_dr_flac__read_int16(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int16* pResult)\n{\n    ma_int32 result;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResult != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 16);\n    if (!ma_dr_flac__read_int32(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    *pResult = (ma_int16)result;\n    return MA_TRUE;\n}\n#endif\nstatic ma_bool32 ma_dr_flac__read_uint8(ma_dr_flac_bs* bs, unsigned int bitCount, ma_uint8* pResult)\n{\n    ma_uint32 result;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResult != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 8);\n    if (!ma_dr_flac__read_uint32(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    *pResult = (ma_uint8)result;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__read_int8(ma_dr_flac_bs* bs, unsigned int bitCount, ma_int8* pResult)\n{\n    ma_int32 result;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pResult != NULL);\n    MA_DR_FLAC_ASSERT(bitCount > 0);\n    MA_DR_FLAC_ASSERT(bitCount <= 8);\n    if (!ma_dr_flac__read_int32(bs, bitCount, &result)) {\n        return MA_FALSE;\n    }\n    *pResult = (ma_int8)result;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__seek_bits(ma_dr_flac_bs* bs, size_t bitsToSeek)\n{\n    if (bitsToSeek <= MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n        bs->consumedBits += (ma_uint32)bitsToSeek;\n        bs->cache <<= bitsToSeek;\n        return MA_TRUE;\n    } else {\n        bitsToSeek       -= MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs);\n        bs->consumedBits += MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs);\n        bs->cache         = 0;\n#ifdef MA_64BIT\n        while (bitsToSeek >= MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)) {\n            ma_uint64 bin;\n            if (!ma_dr_flac__read_uint64(bs, MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs), &bin)) {\n                return MA_FALSE;\n            }\n            bitsToSeek -= MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n        }\n#else\n        while (bitsToSeek >= MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)) {\n            ma_uint32 bin;\n            if (!ma_dr_flac__read_uint32(bs, MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs), &bin)) {\n                return MA_FALSE;\n            }\n            bitsToSeek -= MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n        }\n#endif\n        while (bitsToSeek >= 8) {\n            ma_uint8 bin;\n            if (!ma_dr_flac__read_uint8(bs, 8, &bin)) {\n                return MA_FALSE;\n            }\n            bitsToSeek -= 8;\n        }\n        if (bitsToSeek > 0) {\n            ma_uint8 bin;\n            if (!ma_dr_flac__read_uint8(bs, (ma_uint32)bitsToSeek, &bin)) {\n                return MA_FALSE;\n            }\n            bitsToSeek = 0;\n        }\n        MA_DR_FLAC_ASSERT(bitsToSeek == 0);\n        return MA_TRUE;\n    }\n}\nstatic ma_bool32 ma_dr_flac__find_and_seek_to_next_sync_code(ma_dr_flac_bs* bs)\n{\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    if (!ma_dr_flac__seek_bits(bs, MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) {\n        return MA_FALSE;\n    }\n    for (;;) {\n        ma_uint8 hi;\n#ifndef MA_DR_FLAC_NO_CRC\n        ma_dr_flac__reset_crc16(bs);\n#endif\n        if (!ma_dr_flac__read_uint8(bs, 8, &hi)) {\n            return MA_FALSE;\n        }\n        if (hi == 0xFF) {\n            ma_uint8 lo;\n            if (!ma_dr_flac__read_uint8(bs, 6, &lo)) {\n                return MA_FALSE;\n            }\n            if (lo == 0x3E) {\n                return MA_TRUE;\n            } else {\n                if (!ma_dr_flac__seek_bits(bs, MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) & 7)) {\n                    return MA_FALSE;\n                }\n            }\n        }\n    }\n}\n#if defined(MA_DR_FLAC_HAS_LZCNT_INTRINSIC)\n#define MA_DR_FLAC_IMPLEMENT_CLZ_LZCNT\n#endif\n#if  defined(_MSC_VER) && _MSC_VER >= 1400 && (defined(MA_X64) || defined(MA_X86)) && !defined(__clang__)\n#define MA_DR_FLAC_IMPLEMENT_CLZ_MSVC\n#endif\n#if  defined(__WATCOMC__) && defined(__386__)\n#define MA_DR_FLAC_IMPLEMENT_CLZ_WATCOM\n#endif\n#ifdef __MRC__\n#include <intrinsics.h>\n#define MA_DR_FLAC_IMPLEMENT_CLZ_MRC\n#endif\nstatic MA_INLINE ma_uint32 ma_dr_flac__clz_software(ma_dr_flac_cache_t x)\n{\n    ma_uint32 n;\n    static ma_uint32 clz_table_4[] = {\n        0,\n        4,\n        3, 3,\n        2, 2, 2, 2,\n        1, 1, 1, 1, 1, 1, 1, 1\n    };\n    if (x == 0) {\n        return sizeof(x)*8;\n    }\n    n = clz_table_4[x >> (sizeof(x)*8 - 4)];\n    if (n == 0) {\n#ifdef MA_64BIT\n        if ((x & ((ma_uint64)0xFFFFFFFF << 32)) == 0) { n  = 32; x <<= 32; }\n        if ((x & ((ma_uint64)0xFFFF0000 << 32)) == 0) { n += 16; x <<= 16; }\n        if ((x & ((ma_uint64)0xFF000000 << 32)) == 0) { n += 8;  x <<= 8;  }\n        if ((x & ((ma_uint64)0xF0000000 << 32)) == 0) { n += 4;  x <<= 4;  }\n#else\n        if ((x & 0xFFFF0000) == 0) { n  = 16; x <<= 16; }\n        if ((x & 0xFF000000) == 0) { n += 8;  x <<= 8;  }\n        if ((x & 0xF0000000) == 0) { n += 4;  x <<= 4;  }\n#endif\n        n += clz_table_4[x >> (sizeof(x)*8 - 4)];\n    }\n    return n - 1;\n}\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_LZCNT\nstatic MA_INLINE ma_bool32 ma_dr_flac__is_lzcnt_supported(void)\n{\n#if defined(MA_DR_FLAC_HAS_LZCNT_INTRINSIC) && defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5)\n    return MA_TRUE;\n#elif defined(__MRC__)\n    return MA_TRUE;\n#else\n    #ifdef MA_DR_FLAC_HAS_LZCNT_INTRINSIC\n        return ma_dr_flac__gIsLZCNTSupported;\n    #else\n        return MA_FALSE;\n    #endif\n#endif\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__clz_lzcnt(ma_dr_flac_cache_t x)\n{\n#if defined(_MSC_VER)\n    #ifdef MA_64BIT\n        return (ma_uint32)__lzcnt64(x);\n    #else\n        return (ma_uint32)__lzcnt(x);\n    #endif\n#else\n    #if defined(__GNUC__) || defined(__clang__)\n        #if defined(MA_X64)\n            {\n                ma_uint64 r;\n                __asm__ __volatile__ (\n                    \"rep; bsr{q %1, %0| %0, %1}\" : \"=r\"(r) : \"r\"(x) : \"cc\"\n                );\n                return (ma_uint32)r;\n            }\n        #elif defined(MA_X86)\n            {\n                ma_uint32 r;\n                __asm__ __volatile__ (\n                    \"rep; bsr{l %1, %0| %0, %1}\" : \"=r\"(r) : \"r\"(x) : \"cc\"\n                );\n                return r;\n            }\n        #elif defined(MA_ARM) && (defined(__ARM_ARCH) && __ARM_ARCH >= 5) && !defined(__ARM_ARCH_6M__) && !(defined(__thumb__) && !defined(__thumb2__)) && !defined(MA_64BIT)\n            {\n                unsigned int r;\n                __asm__ __volatile__ (\n                #if defined(MA_64BIT)\n                    \"clz %w[out], %w[in]\" : [out]\"=r\"(r) : [in]\"r\"(x)\n                #else\n                    \"clz %[out], %[in]\" : [out]\"=r\"(r) : [in]\"r\"(x)\n                #endif\n                );\n                return r;\n            }\n        #else\n            if (x == 0) {\n                return sizeof(x)*8;\n            }\n            #ifdef MA_64BIT\n                return (ma_uint32)__builtin_clzll((ma_uint64)x);\n            #else\n                return (ma_uint32)__builtin_clzl((ma_uint32)x);\n            #endif\n        #endif\n    #else\n        #error \"This compiler does not support the lzcnt intrinsic.\"\n    #endif\n#endif\n}\n#endif\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_MSVC\n#include <intrin.h>\nstatic MA_INLINE ma_uint32 ma_dr_flac__clz_msvc(ma_dr_flac_cache_t x)\n{\n    ma_uint32 n;\n    if (x == 0) {\n        return sizeof(x)*8;\n    }\n#ifdef MA_64BIT\n    _BitScanReverse64((unsigned long*)&n, x);\n#else\n    _BitScanReverse((unsigned long*)&n, x);\n#endif\n    return sizeof(x)*8 - n - 1;\n}\n#endif\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_WATCOM\nstatic __inline ma_uint32 ma_dr_flac__clz_watcom (ma_uint32);\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_WATCOM_LZCNT\n#pragma aux ma_dr_flac__clz_watcom_lzcnt = \\\n    \"db 0F3h, 0Fh, 0BDh, 0C0h\"  \\\n    parm [eax] \\\n    value [eax] \\\n    modify nomemory;\n#else\n#pragma aux ma_dr_flac__clz_watcom = \\\n    \"bsr eax, eax\" \\\n    \"xor eax, 31\" \\\n    parm [eax] nomemory \\\n    value [eax] \\\n    modify exact [eax] nomemory;\n#endif\n#endif\nstatic MA_INLINE ma_uint32 ma_dr_flac__clz(ma_dr_flac_cache_t x)\n{\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_LZCNT\n    if (ma_dr_flac__is_lzcnt_supported()) {\n        return ma_dr_flac__clz_lzcnt(x);\n    } else\n#endif\n    {\n#ifdef MA_DR_FLAC_IMPLEMENT_CLZ_MSVC\n        return ma_dr_flac__clz_msvc(x);\n#elif defined(MA_DR_FLAC_IMPLEMENT_CLZ_WATCOM_LZCNT)\n        return ma_dr_flac__clz_watcom_lzcnt(x);\n#elif defined(MA_DR_FLAC_IMPLEMENT_CLZ_WATCOM)\n        return (x == 0) ? sizeof(x)*8 : ma_dr_flac__clz_watcom(x);\n#elif defined(__MRC__)\n        return __cntlzw(x);\n#else\n        return ma_dr_flac__clz_software(x);\n#endif\n    }\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac__seek_past_next_set_bit(ma_dr_flac_bs* bs, unsigned int* pOffsetOut)\n{\n    ma_uint32 zeroCounter = 0;\n    ma_uint32 setBitOffsetPlus1;\n    while (bs->cache == 0) {\n        zeroCounter += (ma_uint32)MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs);\n        if (!ma_dr_flac__reload_cache(bs)) {\n            return MA_FALSE;\n        }\n    }\n    if (bs->cache == 1) {\n        *pOffsetOut = zeroCounter + (ma_uint32)MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs) - 1;\n        if (!ma_dr_flac__reload_cache(bs)) {\n            return MA_FALSE;\n        }\n        return MA_TRUE;\n    }\n    setBitOffsetPlus1 = ma_dr_flac__clz(bs->cache);\n    setBitOffsetPlus1 += 1;\n    if (setBitOffsetPlus1 > MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n        return MA_FALSE;\n    }\n    bs->consumedBits += setBitOffsetPlus1;\n    bs->cache <<= setBitOffsetPlus1;\n    *pOffsetOut = zeroCounter + setBitOffsetPlus1 - 1;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__seek_to_byte(ma_dr_flac_bs* bs, ma_uint64 offsetFromStart)\n{\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(offsetFromStart > 0);\n    if (offsetFromStart > 0x7FFFFFFF) {\n        ma_uint64 bytesRemaining = offsetFromStart;\n        if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, MA_DR_FLAC_SEEK_SET)) {\n            return MA_FALSE;\n        }\n        bytesRemaining -= 0x7FFFFFFF;\n        while (bytesRemaining > 0x7FFFFFFF) {\n            if (!bs->onSeek(bs->pUserData, 0x7FFFFFFF, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            bytesRemaining -= 0x7FFFFFFF;\n        }\n        if (bytesRemaining > 0) {\n            if (!bs->onSeek(bs->pUserData, (int)bytesRemaining, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n        }\n    } else {\n        if (!bs->onSeek(bs->pUserData, (int)offsetFromStart, MA_DR_FLAC_SEEK_SET)) {\n            return MA_FALSE;\n        }\n    }\n    ma_dr_flac__reset_cache(bs);\n    return MA_TRUE;\n}\nstatic ma_result ma_dr_flac__read_utf8_coded_number(ma_dr_flac_bs* bs, ma_uint64* pNumberOut, ma_uint8* pCRCOut)\n{\n    ma_uint8 crc;\n    ma_uint64 result;\n    ma_uint8 utf8[7] = {0};\n    int byteCount;\n    int i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pNumberOut != NULL);\n    MA_DR_FLAC_ASSERT(pCRCOut != NULL);\n    crc = *pCRCOut;\n    if (!ma_dr_flac__read_uint8(bs, 8, utf8)) {\n        *pNumberOut = 0;\n        return MA_AT_END;\n    }\n    crc = ma_dr_flac_crc8(crc, utf8[0], 8);\n    if ((utf8[0] & 0x80) == 0) {\n        *pNumberOut = utf8[0];\n        *pCRCOut = crc;\n        return MA_SUCCESS;\n    }\n    if ((utf8[0] & 0xE0) == 0xC0) {\n        byteCount = 2;\n    } else if ((utf8[0] & 0xF0) == 0xE0) {\n        byteCount = 3;\n    } else if ((utf8[0] & 0xF8) == 0xF0) {\n        byteCount = 4;\n    } else if ((utf8[0] & 0xFC) == 0xF8) {\n        byteCount = 5;\n    } else if ((utf8[0] & 0xFE) == 0xFC) {\n        byteCount = 6;\n    } else if ((utf8[0] & 0xFF) == 0xFE) {\n        byteCount = 7;\n    } else {\n        *pNumberOut = 0;\n        return MA_CRC_MISMATCH;\n    }\n    MA_DR_FLAC_ASSERT(byteCount > 1);\n    result = (ma_uint64)(utf8[0] & (0xFF >> (byteCount + 1)));\n    for (i = 1; i < byteCount; ++i) {\n        if (!ma_dr_flac__read_uint8(bs, 8, utf8 + i)) {\n            *pNumberOut = 0;\n            return MA_AT_END;\n        }\n        crc = ma_dr_flac_crc8(crc, utf8[i], 8);\n        result = (result << 6) | (utf8[i] & 0x3F);\n    }\n    *pNumberOut = result;\n    *pCRCOut = crc;\n    return MA_SUCCESS;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac__ilog2_u32(ma_uint32 x)\n{\n#if 1\n    ma_uint32 result = 0;\n    while (x > 0) {\n        result += 1;\n        x >>= 1;\n    }\n    return result;\n#endif\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac__use_64_bit_prediction(ma_uint32 bitsPerSample, ma_uint32 order, ma_uint32 precision)\n{\n    return bitsPerSample + precision + ma_dr_flac__ilog2_u32(order) > 32;\n}\n#if defined(__clang__)\n__attribute__((no_sanitize(\"signed-integer-overflow\")))\n#endif\nstatic MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_32(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples)\n{\n    ma_int32 prediction = 0;\n    MA_DR_FLAC_ASSERT(order <= 32);\n    switch (order)\n    {\n    case 32: prediction += coefficients[31] * pDecodedSamples[-32];\n    case 31: prediction += coefficients[30] * pDecodedSamples[-31];\n    case 30: prediction += coefficients[29] * pDecodedSamples[-30];\n    case 29: prediction += coefficients[28] * pDecodedSamples[-29];\n    case 28: prediction += coefficients[27] * pDecodedSamples[-28];\n    case 27: prediction += coefficients[26] * pDecodedSamples[-27];\n    case 26: prediction += coefficients[25] * pDecodedSamples[-26];\n    case 25: prediction += coefficients[24] * pDecodedSamples[-25];\n    case 24: prediction += coefficients[23] * pDecodedSamples[-24];\n    case 23: prediction += coefficients[22] * pDecodedSamples[-23];\n    case 22: prediction += coefficients[21] * pDecodedSamples[-22];\n    case 21: prediction += coefficients[20] * pDecodedSamples[-21];\n    case 20: prediction += coefficients[19] * pDecodedSamples[-20];\n    case 19: prediction += coefficients[18] * pDecodedSamples[-19];\n    case 18: prediction += coefficients[17] * pDecodedSamples[-18];\n    case 17: prediction += coefficients[16] * pDecodedSamples[-17];\n    case 16: prediction += coefficients[15] * pDecodedSamples[-16];\n    case 15: prediction += coefficients[14] * pDecodedSamples[-15];\n    case 14: prediction += coefficients[13] * pDecodedSamples[-14];\n    case 13: prediction += coefficients[12] * pDecodedSamples[-13];\n    case 12: prediction += coefficients[11] * pDecodedSamples[-12];\n    case 11: prediction += coefficients[10] * pDecodedSamples[-11];\n    case 10: prediction += coefficients[ 9] * pDecodedSamples[-10];\n    case  9: prediction += coefficients[ 8] * pDecodedSamples[- 9];\n    case  8: prediction += coefficients[ 7] * pDecodedSamples[- 8];\n    case  7: prediction += coefficients[ 6] * pDecodedSamples[- 7];\n    case  6: prediction += coefficients[ 5] * pDecodedSamples[- 6];\n    case  5: prediction += coefficients[ 4] * pDecodedSamples[- 5];\n    case  4: prediction += coefficients[ 3] * pDecodedSamples[- 4];\n    case  3: prediction += coefficients[ 2] * pDecodedSamples[- 3];\n    case  2: prediction += coefficients[ 1] * pDecodedSamples[- 2];\n    case  1: prediction += coefficients[ 0] * pDecodedSamples[- 1];\n    }\n    return (ma_int32)(prediction >> shift);\n}\nstatic MA_INLINE ma_int32 ma_dr_flac__calculate_prediction_64(ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pDecodedSamples)\n{\n    ma_int64 prediction;\n    MA_DR_FLAC_ASSERT(order <= 32);\n#ifndef MA_64BIT\n    if (order == 8)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3] * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4] * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5] * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6] * (ma_int64)pDecodedSamples[-7];\n        prediction += coefficients[7] * (ma_int64)pDecodedSamples[-8];\n    }\n    else if (order == 7)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3] * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4] * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5] * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6] * (ma_int64)pDecodedSamples[-7];\n    }\n    else if (order == 3)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n    }\n    else if (order == 6)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3] * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4] * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5] * (ma_int64)pDecodedSamples[-6];\n    }\n    else if (order == 5)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3] * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4] * (ma_int64)pDecodedSamples[-5];\n    }\n    else if (order == 4)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2] * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3] * (ma_int64)pDecodedSamples[-4];\n    }\n    else if (order == 12)\n    {\n        prediction  = coefficients[0]  * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1]  * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2]  * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3]  * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4]  * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5]  * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6]  * (ma_int64)pDecodedSamples[-7];\n        prediction += coefficients[7]  * (ma_int64)pDecodedSamples[-8];\n        prediction += coefficients[8]  * (ma_int64)pDecodedSamples[-9];\n        prediction += coefficients[9]  * (ma_int64)pDecodedSamples[-10];\n        prediction += coefficients[10] * (ma_int64)pDecodedSamples[-11];\n        prediction += coefficients[11] * (ma_int64)pDecodedSamples[-12];\n    }\n    else if (order == 2)\n    {\n        prediction  = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1] * (ma_int64)pDecodedSamples[-2];\n    }\n    else if (order == 1)\n    {\n        prediction = coefficients[0] * (ma_int64)pDecodedSamples[-1];\n    }\n    else if (order == 10)\n    {\n        prediction  = coefficients[0]  * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1]  * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2]  * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3]  * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4]  * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5]  * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6]  * (ma_int64)pDecodedSamples[-7];\n        prediction += coefficients[7]  * (ma_int64)pDecodedSamples[-8];\n        prediction += coefficients[8]  * (ma_int64)pDecodedSamples[-9];\n        prediction += coefficients[9]  * (ma_int64)pDecodedSamples[-10];\n    }\n    else if (order == 9)\n    {\n        prediction  = coefficients[0]  * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1]  * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2]  * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3]  * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4]  * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5]  * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6]  * (ma_int64)pDecodedSamples[-7];\n        prediction += coefficients[7]  * (ma_int64)pDecodedSamples[-8];\n        prediction += coefficients[8]  * (ma_int64)pDecodedSamples[-9];\n    }\n    else if (order == 11)\n    {\n        prediction  = coefficients[0]  * (ma_int64)pDecodedSamples[-1];\n        prediction += coefficients[1]  * (ma_int64)pDecodedSamples[-2];\n        prediction += coefficients[2]  * (ma_int64)pDecodedSamples[-3];\n        prediction += coefficients[3]  * (ma_int64)pDecodedSamples[-4];\n        prediction += coefficients[4]  * (ma_int64)pDecodedSamples[-5];\n        prediction += coefficients[5]  * (ma_int64)pDecodedSamples[-6];\n        prediction += coefficients[6]  * (ma_int64)pDecodedSamples[-7];\n        prediction += coefficients[7]  * (ma_int64)pDecodedSamples[-8];\n        prediction += coefficients[8]  * (ma_int64)pDecodedSamples[-9];\n        prediction += coefficients[9]  * (ma_int64)pDecodedSamples[-10];\n        prediction += coefficients[10] * (ma_int64)pDecodedSamples[-11];\n    }\n    else\n    {\n        int j;\n        prediction = 0;\n        for (j = 0; j < (int)order; ++j) {\n            prediction += coefficients[j] * (ma_int64)pDecodedSamples[-j-1];\n        }\n    }\n#endif\n#ifdef MA_64BIT\n    prediction = 0;\n    switch (order)\n    {\n    case 32: prediction += coefficients[31] * (ma_int64)pDecodedSamples[-32];\n    case 31: prediction += coefficients[30] * (ma_int64)pDecodedSamples[-31];\n    case 30: prediction += coefficients[29] * (ma_int64)pDecodedSamples[-30];\n    case 29: prediction += coefficients[28] * (ma_int64)pDecodedSamples[-29];\n    case 28: prediction += coefficients[27] * (ma_int64)pDecodedSamples[-28];\n    case 27: prediction += coefficients[26] * (ma_int64)pDecodedSamples[-27];\n    case 26: prediction += coefficients[25] * (ma_int64)pDecodedSamples[-26];\n    case 25: prediction += coefficients[24] * (ma_int64)pDecodedSamples[-25];\n    case 24: prediction += coefficients[23] * (ma_int64)pDecodedSamples[-24];\n    case 23: prediction += coefficients[22] * (ma_int64)pDecodedSamples[-23];\n    case 22: prediction += coefficients[21] * (ma_int64)pDecodedSamples[-22];\n    case 21: prediction += coefficients[20] * (ma_int64)pDecodedSamples[-21];\n    case 20: prediction += coefficients[19] * (ma_int64)pDecodedSamples[-20];\n    case 19: prediction += coefficients[18] * (ma_int64)pDecodedSamples[-19];\n    case 18: prediction += coefficients[17] * (ma_int64)pDecodedSamples[-18];\n    case 17: prediction += coefficients[16] * (ma_int64)pDecodedSamples[-17];\n    case 16: prediction += coefficients[15] * (ma_int64)pDecodedSamples[-16];\n    case 15: prediction += coefficients[14] * (ma_int64)pDecodedSamples[-15];\n    case 14: prediction += coefficients[13] * (ma_int64)pDecodedSamples[-14];\n    case 13: prediction += coefficients[12] * (ma_int64)pDecodedSamples[-13];\n    case 12: prediction += coefficients[11] * (ma_int64)pDecodedSamples[-12];\n    case 11: prediction += coefficients[10] * (ma_int64)pDecodedSamples[-11];\n    case 10: prediction += coefficients[ 9] * (ma_int64)pDecodedSamples[-10];\n    case  9: prediction += coefficients[ 8] * (ma_int64)pDecodedSamples[- 9];\n    case  8: prediction += coefficients[ 7] * (ma_int64)pDecodedSamples[- 8];\n    case  7: prediction += coefficients[ 6] * (ma_int64)pDecodedSamples[- 7];\n    case  6: prediction += coefficients[ 5] * (ma_int64)pDecodedSamples[- 6];\n    case  5: prediction += coefficients[ 4] * (ma_int64)pDecodedSamples[- 5];\n    case  4: prediction += coefficients[ 3] * (ma_int64)pDecodedSamples[- 4];\n    case  3: prediction += coefficients[ 2] * (ma_int64)pDecodedSamples[- 3];\n    case  2: prediction += coefficients[ 1] * (ma_int64)pDecodedSamples[- 2];\n    case  1: prediction += coefficients[ 0] * (ma_int64)pDecodedSamples[- 1];\n    }\n#endif\n    return (ma_int32)(prediction >> shift);\n}\n#if 0\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__reference(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    for (i = 0; i < count; ++i) {\n        ma_uint32 zeroCounter = 0;\n        for (;;) {\n            ma_uint8 bit;\n            if (!ma_dr_flac__read_uint8(bs, 1, &bit)) {\n                return MA_FALSE;\n            }\n            if (bit == 0) {\n                zeroCounter += 1;\n            } else {\n                break;\n            }\n        }\n        ma_uint32 decodedRice;\n        if (riceParam > 0) {\n            if (!ma_dr_flac__read_uint32(bs, riceParam, &decodedRice)) {\n                return MA_FALSE;\n            }\n        } else {\n            decodedRice = 0;\n        }\n        decodedRice |= (zeroCounter << riceParam);\n        if ((decodedRice & 0x01)) {\n            decodedRice = ~(decodedRice >> 1);\n        } else {\n            decodedRice =  (decodedRice >> 1);\n        }\n        if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n            pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i);\n        } else {\n            pSamplesOut[i] = decodedRice + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i);\n        }\n    }\n    return MA_TRUE;\n}\n#endif\n#if 0\nstatic ma_bool32 ma_dr_flac__read_rice_parts__reference(ma_dr_flac_bs* bs, ma_uint8 riceParam, ma_uint32* pZeroCounterOut, ma_uint32* pRiceParamPartOut)\n{\n    ma_uint32 zeroCounter = 0;\n    ma_uint32 decodedRice;\n    for (;;) {\n        ma_uint8 bit;\n        if (!ma_dr_flac__read_uint8(bs, 1, &bit)) {\n            return MA_FALSE;\n        }\n        if (bit == 0) {\n            zeroCounter += 1;\n        } else {\n            break;\n        }\n    }\n    if (riceParam > 0) {\n        if (!ma_dr_flac__read_uint32(bs, riceParam, &decodedRice)) {\n            return MA_FALSE;\n        }\n    } else {\n        decodedRice = 0;\n    }\n    *pZeroCounterOut = zeroCounter;\n    *pRiceParamPartOut = decodedRice;\n    return MA_TRUE;\n}\n#endif\n#if 0\nstatic MA_INLINE ma_bool32 ma_dr_flac__read_rice_parts(ma_dr_flac_bs* bs, ma_uint8 riceParam, ma_uint32* pZeroCounterOut, ma_uint32* pRiceParamPartOut)\n{\n    ma_dr_flac_cache_t riceParamMask;\n    ma_uint32 zeroCounter;\n    ma_uint32 setBitOffsetPlus1;\n    ma_uint32 riceParamPart;\n    ma_uint32 riceLength;\n    MA_DR_FLAC_ASSERT(riceParam > 0);\n    riceParamMask = MA_DR_FLAC_CACHE_L1_SELECTION_MASK(riceParam);\n    zeroCounter = 0;\n    while (bs->cache == 0) {\n        zeroCounter += (ma_uint32)MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs);\n        if (!ma_dr_flac__reload_cache(bs)) {\n            return MA_FALSE;\n        }\n    }\n    setBitOffsetPlus1 = ma_dr_flac__clz(bs->cache);\n    zeroCounter += setBitOffsetPlus1;\n    setBitOffsetPlus1 += 1;\n    riceLength = setBitOffsetPlus1 + riceParam;\n    if (riceLength < MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n        riceParamPart = (ma_uint32)((bs->cache & (riceParamMask >> setBitOffsetPlus1)) >> MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT(bs, riceLength));\n        bs->consumedBits += riceLength;\n        bs->cache <<= riceLength;\n    } else {\n        ma_uint32 bitCountLo;\n        ma_dr_flac_cache_t resultHi;\n        bs->consumedBits += riceLength;\n        bs->cache <<= setBitOffsetPlus1 & (MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs)-1);\n        bitCountLo = bs->consumedBits - MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs);\n        resultHi = MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT(bs, riceParam);\n        if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n#ifndef MA_DR_FLAC_NO_CRC\n            ma_dr_flac__update_crc16(bs);\n#endif\n            bs->cache = ma_dr_flac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]);\n            bs->consumedBits = 0;\n#ifndef MA_DR_FLAC_NO_CRC\n            bs->crc16Cache = bs->cache;\n#endif\n        } else {\n            if (!ma_dr_flac__reload_cache(bs)) {\n                return MA_FALSE;\n            }\n            if (bitCountLo > MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n                return MA_FALSE;\n            }\n        }\n        riceParamPart = (ma_uint32)(resultHi | MA_DR_FLAC_CACHE_L1_SELECT_AND_SHIFT_SAFE(bs, bitCountLo));\n        bs->consumedBits += bitCountLo;\n        bs->cache <<= bitCountLo;\n    }\n    pZeroCounterOut[0] = zeroCounter;\n    pRiceParamPartOut[0] = riceParamPart;\n    return MA_TRUE;\n}\n#endif\nstatic MA_INLINE ma_bool32 ma_dr_flac__read_rice_parts_x1(ma_dr_flac_bs* bs, ma_uint8 riceParam, ma_uint32* pZeroCounterOut, ma_uint32* pRiceParamPartOut)\n{\n    ma_uint32  riceParamPlus1 = riceParam + 1;\n    ma_uint32  riceParamPlus1Shift = MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT(bs, riceParamPlus1);\n    ma_uint32  riceParamPlus1MaxConsumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs) - riceParamPlus1;\n    ma_dr_flac_cache_t bs_cache = bs->cache;\n    ma_uint32  bs_consumedBits = bs->consumedBits;\n    ma_uint32  lzcount = ma_dr_flac__clz(bs_cache);\n    if (lzcount < sizeof(bs_cache)*8) {\n        pZeroCounterOut[0] = lzcount;\n    extract_rice_param_part:\n        bs_cache       <<= lzcount;\n        bs_consumedBits += lzcount;\n        if (bs_consumedBits <= riceParamPlus1MaxConsumedBits) {\n            pRiceParamPartOut[0] = (ma_uint32)(bs_cache >> riceParamPlus1Shift);\n            bs_cache       <<= riceParamPlus1;\n            bs_consumedBits += riceParamPlus1;\n        } else {\n            ma_uint32 riceParamPartHi;\n            ma_uint32 riceParamPartLo;\n            ma_uint32 riceParamPartLoBitCount;\n            riceParamPartHi = (ma_uint32)(bs_cache >> riceParamPlus1Shift);\n            riceParamPartLoBitCount = bs_consumedBits - riceParamPlus1MaxConsumedBits;\n            MA_DR_FLAC_ASSERT(riceParamPartLoBitCount > 0 && riceParamPartLoBitCount < 32);\n            if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n            #ifndef MA_DR_FLAC_NO_CRC\n                ma_dr_flac__update_crc16(bs);\n            #endif\n                bs_cache = ma_dr_flac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]);\n                bs_consumedBits = riceParamPartLoBitCount;\n            #ifndef MA_DR_FLAC_NO_CRC\n                bs->crc16Cache = bs_cache;\n            #endif\n            } else {\n                if (!ma_dr_flac__reload_cache(bs)) {\n                    return MA_FALSE;\n                }\n                if (riceParamPartLoBitCount > MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n                    return MA_FALSE;\n                }\n                bs_cache = bs->cache;\n                bs_consumedBits = bs->consumedBits + riceParamPartLoBitCount;\n            }\n            riceParamPartLo = (ma_uint32)(bs_cache >> (MA_DR_FLAC_CACHE_L1_SELECTION_SHIFT(bs, riceParamPartLoBitCount)));\n            pRiceParamPartOut[0] = riceParamPartHi | riceParamPartLo;\n            bs_cache <<= riceParamPartLoBitCount;\n        }\n    } else {\n        ma_uint32 zeroCounter = (ma_uint32)(MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs) - bs_consumedBits);\n        for (;;) {\n            if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n            #ifndef MA_DR_FLAC_NO_CRC\n                ma_dr_flac__update_crc16(bs);\n            #endif\n                bs_cache = ma_dr_flac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]);\n                bs_consumedBits = 0;\n            #ifndef MA_DR_FLAC_NO_CRC\n                bs->crc16Cache = bs_cache;\n            #endif\n            } else {\n                if (!ma_dr_flac__reload_cache(bs)) {\n                    return MA_FALSE;\n                }\n                bs_cache = bs->cache;\n                bs_consumedBits = bs->consumedBits;\n            }\n            lzcount = ma_dr_flac__clz(bs_cache);\n            zeroCounter += lzcount;\n            if (lzcount < sizeof(bs_cache)*8) {\n                break;\n            }\n        }\n        pZeroCounterOut[0] = zeroCounter;\n        goto extract_rice_param_part;\n    }\n    bs->cache = bs_cache;\n    bs->consumedBits = bs_consumedBits;\n    return MA_TRUE;\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac__seek_rice_parts(ma_dr_flac_bs* bs, ma_uint8 riceParam)\n{\n    ma_uint32  riceParamPlus1 = riceParam + 1;\n    ma_uint32  riceParamPlus1MaxConsumedBits = MA_DR_FLAC_CACHE_L1_SIZE_BITS(bs) - riceParamPlus1;\n    ma_dr_flac_cache_t bs_cache = bs->cache;\n    ma_uint32  bs_consumedBits = bs->consumedBits;\n    ma_uint32  lzcount = ma_dr_flac__clz(bs_cache);\n    if (lzcount < sizeof(bs_cache)*8) {\n    extract_rice_param_part:\n        bs_cache       <<= lzcount;\n        bs_consumedBits += lzcount;\n        if (bs_consumedBits <= riceParamPlus1MaxConsumedBits) {\n            bs_cache       <<= riceParamPlus1;\n            bs_consumedBits += riceParamPlus1;\n        } else {\n            ma_uint32 riceParamPartLoBitCount = bs_consumedBits - riceParamPlus1MaxConsumedBits;\n            MA_DR_FLAC_ASSERT(riceParamPartLoBitCount > 0 && riceParamPartLoBitCount < 32);\n            if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n            #ifndef MA_DR_FLAC_NO_CRC\n                ma_dr_flac__update_crc16(bs);\n            #endif\n                bs_cache = ma_dr_flac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]);\n                bs_consumedBits = riceParamPartLoBitCount;\n            #ifndef MA_DR_FLAC_NO_CRC\n                bs->crc16Cache = bs_cache;\n            #endif\n            } else {\n                if (!ma_dr_flac__reload_cache(bs)) {\n                    return MA_FALSE;\n                }\n                if (riceParamPartLoBitCount > MA_DR_FLAC_CACHE_L1_BITS_REMAINING(bs)) {\n                    return MA_FALSE;\n                }\n                bs_cache = bs->cache;\n                bs_consumedBits = bs->consumedBits + riceParamPartLoBitCount;\n            }\n            bs_cache <<= riceParamPartLoBitCount;\n        }\n    } else {\n        for (;;) {\n            if (bs->nextL2Line < MA_DR_FLAC_CACHE_L2_LINE_COUNT(bs)) {\n            #ifndef MA_DR_FLAC_NO_CRC\n                ma_dr_flac__update_crc16(bs);\n            #endif\n                bs_cache = ma_dr_flac__be2host__cache_line(bs->cacheL2[bs->nextL2Line++]);\n                bs_consumedBits = 0;\n            #ifndef MA_DR_FLAC_NO_CRC\n                bs->crc16Cache = bs_cache;\n            #endif\n            } else {\n                if (!ma_dr_flac__reload_cache(bs)) {\n                    return MA_FALSE;\n                }\n                bs_cache = bs->cache;\n                bs_consumedBits = bs->consumedBits;\n            }\n            lzcount = ma_dr_flac__clz(bs_cache);\n            if (lzcount < sizeof(bs_cache)*8) {\n                break;\n            }\n        }\n        goto extract_rice_param_part;\n    }\n    bs->cache = bs_cache;\n    bs->consumedBits = bs_consumedBits;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__scalar_zeroorder(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    ma_uint32 zeroCountPart0;\n    ma_uint32 riceParamPart0;\n    ma_uint32 riceParamMask;\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    (void)bitsPerSample;\n    (void)order;\n    (void)shift;\n    (void)coefficients;\n    riceParamMask  = (ma_uint32)~((~0UL) << riceParam);\n    i = 0;\n    while (i < count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0)) {\n            return MA_FALSE;\n        }\n        riceParamPart0 &= riceParamMask;\n        riceParamPart0 |= (zeroCountPart0 << riceParam);\n        riceParamPart0  = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01];\n        pSamplesOut[i] = riceParamPart0;\n        i += 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__scalar(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    ma_uint32 zeroCountPart0 = 0;\n    ma_uint32 zeroCountPart1 = 0;\n    ma_uint32 zeroCountPart2 = 0;\n    ma_uint32 zeroCountPart3 = 0;\n    ma_uint32 riceParamPart0 = 0;\n    ma_uint32 riceParamPart1 = 0;\n    ma_uint32 riceParamPart2 = 0;\n    ma_uint32 riceParamPart3 = 0;\n    ma_uint32 riceParamMask;\n    const ma_int32* pSamplesOutEnd;\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    if (lpcOrder == 0) {\n        return ma_dr_flac__decode_samples_with_residual__rice__scalar_zeroorder(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut);\n    }\n    riceParamMask  = (ma_uint32)~((~0UL) << riceParam);\n    pSamplesOutEnd = pSamplesOut + (count & ~3);\n    if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n        while (pSamplesOut < pSamplesOutEnd) {\n            if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart1, &riceParamPart1) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart2, &riceParamPart2) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart3, &riceParamPart3)) {\n                return MA_FALSE;\n            }\n            riceParamPart0 &= riceParamMask;\n            riceParamPart1 &= riceParamMask;\n            riceParamPart2 &= riceParamMask;\n            riceParamPart3 &= riceParamMask;\n            riceParamPart0 |= (zeroCountPart0 << riceParam);\n            riceParamPart1 |= (zeroCountPart1 << riceParam);\n            riceParamPart2 |= (zeroCountPart2 << riceParam);\n            riceParamPart3 |= (zeroCountPart3 << riceParam);\n            riceParamPart0  = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01];\n            riceParamPart1  = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01];\n            riceParamPart2  = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01];\n            riceParamPart3  = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01];\n            pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0);\n            pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 1);\n            pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 2);\n            pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 3);\n            pSamplesOut += 4;\n        }\n    } else {\n        while (pSamplesOut < pSamplesOutEnd) {\n            if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart1, &riceParamPart1) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart2, &riceParamPart2) ||\n                !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart3, &riceParamPart3)) {\n                return MA_FALSE;\n            }\n            riceParamPart0 &= riceParamMask;\n            riceParamPart1 &= riceParamMask;\n            riceParamPart2 &= riceParamMask;\n            riceParamPart3 &= riceParamMask;\n            riceParamPart0 |= (zeroCountPart0 << riceParam);\n            riceParamPart1 |= (zeroCountPart1 << riceParam);\n            riceParamPart2 |= (zeroCountPart2 << riceParam);\n            riceParamPart3 |= (zeroCountPart3 << riceParam);\n            riceParamPart0  = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01];\n            riceParamPart1  = (riceParamPart1 >> 1) ^ t[riceParamPart1 & 0x01];\n            riceParamPart2  = (riceParamPart2 >> 1) ^ t[riceParamPart2 & 0x01];\n            riceParamPart3  = (riceParamPart3 >> 1) ^ t[riceParamPart3 & 0x01];\n            pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0);\n            pSamplesOut[1] = riceParamPart1 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 1);\n            pSamplesOut[2] = riceParamPart2 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 2);\n            pSamplesOut[3] = riceParamPart3 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 3);\n            pSamplesOut += 4;\n        }\n    }\n    i = (count & ~3);\n    while (i < count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountPart0, &riceParamPart0)) {\n            return MA_FALSE;\n        }\n        riceParamPart0 &= riceParamMask;\n        riceParamPart0 |= (zeroCountPart0 << riceParam);\n        riceParamPart0  = (riceParamPart0 >> 1) ^ t[riceParamPart0 & 0x01];\n        if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n            pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + 0);\n        } else {\n            pSamplesOut[0] = riceParamPart0 + ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + 0);\n        }\n        i += 1;\n        pSamplesOut += 1;\n    }\n    return MA_TRUE;\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE __m128i ma_dr_flac__mm_packs_interleaved_epi32(__m128i a, __m128i b)\n{\n    __m128i r;\n    r = _mm_packs_epi32(a, b);\n    r = _mm_shuffle_epi32(r, _MM_SHUFFLE(3, 1, 2, 0));\n    r = _mm_shufflehi_epi16(r, _MM_SHUFFLE(3, 1, 2, 0));\n    r = _mm_shufflelo_epi16(r, _MM_SHUFFLE(3, 1, 2, 0));\n    return r;\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_SSE41)\nstatic MA_INLINE __m128i ma_dr_flac__mm_not_si128(__m128i a)\n{\n    return _mm_xor_si128(a, _mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128()));\n}\nstatic MA_INLINE __m128i ma_dr_flac__mm_hadd_epi32(__m128i x)\n{\n    __m128i x64 = _mm_add_epi32(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2)));\n    __m128i x32 = _mm_shufflelo_epi16(x64, _MM_SHUFFLE(1, 0, 3, 2));\n    return _mm_add_epi32(x64, x32);\n}\nstatic MA_INLINE __m128i ma_dr_flac__mm_hadd_epi64(__m128i x)\n{\n    return _mm_add_epi64(x, _mm_shuffle_epi32(x, _MM_SHUFFLE(1, 0, 3, 2)));\n}\nstatic MA_INLINE __m128i ma_dr_flac__mm_srai_epi64(__m128i x, int count)\n{\n    __m128i lo = _mm_srli_epi64(x, count);\n    __m128i hi = _mm_srai_epi32(x, count);\n    hi = _mm_and_si128(hi, _mm_set_epi32(0xFFFFFFFF, 0, 0xFFFFFFFF, 0));\n    return _mm_or_si128(lo, hi);\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__sse41_32(ma_dr_flac_bs* bs, ma_uint32 count, ma_uint8 riceParam, ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    int i;\n    ma_uint32 riceParamMask;\n    ma_int32* pDecodedSamples    = pSamplesOut;\n    ma_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3);\n    ma_uint32 zeroCountParts0 = 0;\n    ma_uint32 zeroCountParts1 = 0;\n    ma_uint32 zeroCountParts2 = 0;\n    ma_uint32 zeroCountParts3 = 0;\n    ma_uint32 riceParamParts0 = 0;\n    ma_uint32 riceParamParts1 = 0;\n    ma_uint32 riceParamParts2 = 0;\n    ma_uint32 riceParamParts3 = 0;\n    __m128i coefficients128_0;\n    __m128i coefficients128_4;\n    __m128i coefficients128_8;\n    __m128i samples128_0;\n    __m128i samples128_4;\n    __m128i samples128_8;\n    __m128i riceParamMask128;\n    const ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    riceParamMask    = (ma_uint32)~((~0UL) << riceParam);\n    riceParamMask128 = _mm_set1_epi32(riceParamMask);\n    coefficients128_0 = _mm_setzero_si128();\n    coefficients128_4 = _mm_setzero_si128();\n    coefficients128_8 = _mm_setzero_si128();\n    samples128_0 = _mm_setzero_si128();\n    samples128_4 = _mm_setzero_si128();\n    samples128_8 = _mm_setzero_si128();\n#if 1\n    {\n        int runningOrder = order;\n        if (runningOrder >= 4) {\n            coefficients128_0 = _mm_loadu_si128((const __m128i*)(coefficients + 0));\n            samples128_0      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 4));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_0 = _mm_set_epi32(0, coefficients[2], coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], pSamplesOut[-3], 0); break;\n                case 2: coefficients128_0 = _mm_set_epi32(0, 0,               coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], 0,               0); break;\n                case 1: coefficients128_0 = _mm_set_epi32(0, 0,               0,               coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], 0,               0,               0); break;\n            }\n            runningOrder = 0;\n        }\n        if (runningOrder >= 4) {\n            coefficients128_4 = _mm_loadu_si128((const __m128i*)(coefficients + 4));\n            samples128_4      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 8));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_4 = _mm_set_epi32(0, coefficients[6], coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], pSamplesOut[-7], 0); break;\n                case 2: coefficients128_4 = _mm_set_epi32(0, 0,               coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], 0,               0); break;\n                case 1: coefficients128_4 = _mm_set_epi32(0, 0,               0,               coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], 0,               0,               0); break;\n            }\n            runningOrder = 0;\n        }\n        if (runningOrder == 4) {\n            coefficients128_8 = _mm_loadu_si128((const __m128i*)(coefficients + 8));\n            samples128_8      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 12));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_8 = _mm_set_epi32(0, coefficients[10], coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], pSamplesOut[-11], 0); break;\n                case 2: coefficients128_8 = _mm_set_epi32(0, 0,                coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], 0,                0); break;\n                case 1: coefficients128_8 = _mm_set_epi32(0, 0,                0,               coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], 0,                0,                0); break;\n            }\n            runningOrder = 0;\n        }\n        coefficients128_0 = _mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(0, 1, 2, 3));\n        coefficients128_4 = _mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(0, 1, 2, 3));\n        coefficients128_8 = _mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(0, 1, 2, 3));\n    }\n#else\n    switch (order)\n    {\n    case 12: ((ma_int32*)&coefficients128_8)[0] = coefficients[11]; ((ma_int32*)&samples128_8)[0] = pDecodedSamples[-12];\n    case 11: ((ma_int32*)&coefficients128_8)[1] = coefficients[10]; ((ma_int32*)&samples128_8)[1] = pDecodedSamples[-11];\n    case 10: ((ma_int32*)&coefficients128_8)[2] = coefficients[ 9]; ((ma_int32*)&samples128_8)[2] = pDecodedSamples[-10];\n    case 9:  ((ma_int32*)&coefficients128_8)[3] = coefficients[ 8]; ((ma_int32*)&samples128_8)[3] = pDecodedSamples[- 9];\n    case 8:  ((ma_int32*)&coefficients128_4)[0] = coefficients[ 7]; ((ma_int32*)&samples128_4)[0] = pDecodedSamples[- 8];\n    case 7:  ((ma_int32*)&coefficients128_4)[1] = coefficients[ 6]; ((ma_int32*)&samples128_4)[1] = pDecodedSamples[- 7];\n    case 6:  ((ma_int32*)&coefficients128_4)[2] = coefficients[ 5]; ((ma_int32*)&samples128_4)[2] = pDecodedSamples[- 6];\n    case 5:  ((ma_int32*)&coefficients128_4)[3] = coefficients[ 4]; ((ma_int32*)&samples128_4)[3] = pDecodedSamples[- 5];\n    case 4:  ((ma_int32*)&coefficients128_0)[0] = coefficients[ 3]; ((ma_int32*)&samples128_0)[0] = pDecodedSamples[- 4];\n    case 3:  ((ma_int32*)&coefficients128_0)[1] = coefficients[ 2]; ((ma_int32*)&samples128_0)[1] = pDecodedSamples[- 3];\n    case 2:  ((ma_int32*)&coefficients128_0)[2] = coefficients[ 1]; ((ma_int32*)&samples128_0)[2] = pDecodedSamples[- 2];\n    case 1:  ((ma_int32*)&coefficients128_0)[3] = coefficients[ 0]; ((ma_int32*)&samples128_0)[3] = pDecodedSamples[- 1];\n    }\n#endif\n    while (pDecodedSamples < pDecodedSamplesEnd) {\n        __m128i prediction128;\n        __m128i zeroCountPart128;\n        __m128i riceParamPart128;\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts1, &riceParamParts1) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts2, &riceParamParts2) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts3, &riceParamParts3)) {\n            return MA_FALSE;\n        }\n        zeroCountPart128 = _mm_set_epi32(zeroCountParts3, zeroCountParts2, zeroCountParts1, zeroCountParts0);\n        riceParamPart128 = _mm_set_epi32(riceParamParts3, riceParamParts2, riceParamParts1, riceParamParts0);\n        riceParamPart128 = _mm_and_si128(riceParamPart128, riceParamMask128);\n        riceParamPart128 = _mm_or_si128(riceParamPart128, _mm_slli_epi32(zeroCountPart128, riceParam));\n        riceParamPart128 = _mm_xor_si128(_mm_srli_epi32(riceParamPart128, 1), _mm_add_epi32(ma_dr_flac__mm_not_si128(_mm_and_si128(riceParamPart128, _mm_set1_epi32(0x01))), _mm_set1_epi32(0x01)));\n        if (order <= 4) {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 = _mm_mullo_epi32(coefficients128_0, samples128_0);\n                prediction128 = ma_dr_flac__mm_hadd_epi32(prediction128);\n                prediction128 = _mm_srai_epi32(prediction128, shift);\n                prediction128 = _mm_add_epi32(riceParamPart128, prediction128);\n                samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4);\n                riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4);\n            }\n        } else if (order <= 8) {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 =                              _mm_mullo_epi32(coefficients128_4, samples128_4);\n                prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_0, samples128_0));\n                prediction128 = ma_dr_flac__mm_hadd_epi32(prediction128);\n                prediction128 = _mm_srai_epi32(prediction128, shift);\n                prediction128 = _mm_add_epi32(riceParamPart128, prediction128);\n                samples128_4 = _mm_alignr_epi8(samples128_0,  samples128_4, 4);\n                samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4);\n                riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4);\n            }\n        } else {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 =                              _mm_mullo_epi32(coefficients128_8, samples128_8);\n                prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_4, samples128_4));\n                prediction128 = _mm_add_epi32(prediction128, _mm_mullo_epi32(coefficients128_0, samples128_0));\n                prediction128 = ma_dr_flac__mm_hadd_epi32(prediction128);\n                prediction128 = _mm_srai_epi32(prediction128, shift);\n                prediction128 = _mm_add_epi32(riceParamPart128, prediction128);\n                samples128_8 = _mm_alignr_epi8(samples128_4,  samples128_8, 4);\n                samples128_4 = _mm_alignr_epi8(samples128_0,  samples128_4, 4);\n                samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4);\n                riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4);\n            }\n        }\n        _mm_storeu_si128((__m128i*)pDecodedSamples, samples128_0);\n        pDecodedSamples += 4;\n    }\n    i = (count & ~3);\n    while (i < (int)count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0)) {\n            return MA_FALSE;\n        }\n        riceParamParts0 &= riceParamMask;\n        riceParamParts0 |= (zeroCountParts0 << riceParam);\n        riceParamParts0  = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01];\n        pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples);\n        i += 1;\n        pDecodedSamples += 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__sse41_64(ma_dr_flac_bs* bs, ma_uint32 count, ma_uint8 riceParam, ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    int i;\n    ma_uint32 riceParamMask;\n    ma_int32* pDecodedSamples    = pSamplesOut;\n    ma_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3);\n    ma_uint32 zeroCountParts0 = 0;\n    ma_uint32 zeroCountParts1 = 0;\n    ma_uint32 zeroCountParts2 = 0;\n    ma_uint32 zeroCountParts3 = 0;\n    ma_uint32 riceParamParts0 = 0;\n    ma_uint32 riceParamParts1 = 0;\n    ma_uint32 riceParamParts2 = 0;\n    ma_uint32 riceParamParts3 = 0;\n    __m128i coefficients128_0;\n    __m128i coefficients128_4;\n    __m128i coefficients128_8;\n    __m128i samples128_0;\n    __m128i samples128_4;\n    __m128i samples128_8;\n    __m128i prediction128;\n    __m128i riceParamMask128;\n    const ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    MA_DR_FLAC_ASSERT(order <= 12);\n    riceParamMask    = (ma_uint32)~((~0UL) << riceParam);\n    riceParamMask128 = _mm_set1_epi32(riceParamMask);\n    prediction128 = _mm_setzero_si128();\n    coefficients128_0  = _mm_setzero_si128();\n    coefficients128_4  = _mm_setzero_si128();\n    coefficients128_8  = _mm_setzero_si128();\n    samples128_0  = _mm_setzero_si128();\n    samples128_4  = _mm_setzero_si128();\n    samples128_8  = _mm_setzero_si128();\n#if 1\n    {\n        int runningOrder = order;\n        if (runningOrder >= 4) {\n            coefficients128_0 = _mm_loadu_si128((const __m128i*)(coefficients + 0));\n            samples128_0      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 4));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_0 = _mm_set_epi32(0, coefficients[2], coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], pSamplesOut[-3], 0); break;\n                case 2: coefficients128_0 = _mm_set_epi32(0, 0,               coefficients[1], coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], pSamplesOut[-2], 0,               0); break;\n                case 1: coefficients128_0 = _mm_set_epi32(0, 0,               0,               coefficients[0]); samples128_0 = _mm_set_epi32(pSamplesOut[-1], 0,               0,               0); break;\n            }\n            runningOrder = 0;\n        }\n        if (runningOrder >= 4) {\n            coefficients128_4 = _mm_loadu_si128((const __m128i*)(coefficients + 4));\n            samples128_4      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 8));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_4 = _mm_set_epi32(0, coefficients[6], coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], pSamplesOut[-7], 0); break;\n                case 2: coefficients128_4 = _mm_set_epi32(0, 0,               coefficients[5], coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], pSamplesOut[-6], 0,               0); break;\n                case 1: coefficients128_4 = _mm_set_epi32(0, 0,               0,               coefficients[4]); samples128_4 = _mm_set_epi32(pSamplesOut[-5], 0,               0,               0); break;\n            }\n            runningOrder = 0;\n        }\n        if (runningOrder == 4) {\n            coefficients128_8 = _mm_loadu_si128((const __m128i*)(coefficients + 8));\n            samples128_8      = _mm_loadu_si128((const __m128i*)(pSamplesOut  - 12));\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: coefficients128_8 = _mm_set_epi32(0, coefficients[10], coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], pSamplesOut[-11], 0); break;\n                case 2: coefficients128_8 = _mm_set_epi32(0, 0,                coefficients[9], coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], pSamplesOut[-10], 0,                0); break;\n                case 1: coefficients128_8 = _mm_set_epi32(0, 0,                0,               coefficients[8]); samples128_8 = _mm_set_epi32(pSamplesOut[-9], 0,                0,                0); break;\n            }\n            runningOrder = 0;\n        }\n        coefficients128_0 = _mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(0, 1, 2, 3));\n        coefficients128_4 = _mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(0, 1, 2, 3));\n        coefficients128_8 = _mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(0, 1, 2, 3));\n    }\n#else\n    switch (order)\n    {\n    case 12: ((ma_int32*)&coefficients128_8)[0] = coefficients[11]; ((ma_int32*)&samples128_8)[0] = pDecodedSamples[-12];\n    case 11: ((ma_int32*)&coefficients128_8)[1] = coefficients[10]; ((ma_int32*)&samples128_8)[1] = pDecodedSamples[-11];\n    case 10: ((ma_int32*)&coefficients128_8)[2] = coefficients[ 9]; ((ma_int32*)&samples128_8)[2] = pDecodedSamples[-10];\n    case 9:  ((ma_int32*)&coefficients128_8)[3] = coefficients[ 8]; ((ma_int32*)&samples128_8)[3] = pDecodedSamples[- 9];\n    case 8:  ((ma_int32*)&coefficients128_4)[0] = coefficients[ 7]; ((ma_int32*)&samples128_4)[0] = pDecodedSamples[- 8];\n    case 7:  ((ma_int32*)&coefficients128_4)[1] = coefficients[ 6]; ((ma_int32*)&samples128_4)[1] = pDecodedSamples[- 7];\n    case 6:  ((ma_int32*)&coefficients128_4)[2] = coefficients[ 5]; ((ma_int32*)&samples128_4)[2] = pDecodedSamples[- 6];\n    case 5:  ((ma_int32*)&coefficients128_4)[3] = coefficients[ 4]; ((ma_int32*)&samples128_4)[3] = pDecodedSamples[- 5];\n    case 4:  ((ma_int32*)&coefficients128_0)[0] = coefficients[ 3]; ((ma_int32*)&samples128_0)[0] = pDecodedSamples[- 4];\n    case 3:  ((ma_int32*)&coefficients128_0)[1] = coefficients[ 2]; ((ma_int32*)&samples128_0)[1] = pDecodedSamples[- 3];\n    case 2:  ((ma_int32*)&coefficients128_0)[2] = coefficients[ 1]; ((ma_int32*)&samples128_0)[2] = pDecodedSamples[- 2];\n    case 1:  ((ma_int32*)&coefficients128_0)[3] = coefficients[ 0]; ((ma_int32*)&samples128_0)[3] = pDecodedSamples[- 1];\n    }\n#endif\n    while (pDecodedSamples < pDecodedSamplesEnd) {\n        __m128i zeroCountPart128;\n        __m128i riceParamPart128;\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts1, &riceParamParts1) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts2, &riceParamParts2) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts3, &riceParamParts3)) {\n            return MA_FALSE;\n        }\n        zeroCountPart128 = _mm_set_epi32(zeroCountParts3, zeroCountParts2, zeroCountParts1, zeroCountParts0);\n        riceParamPart128 = _mm_set_epi32(riceParamParts3, riceParamParts2, riceParamParts1, riceParamParts0);\n        riceParamPart128 = _mm_and_si128(riceParamPart128, riceParamMask128);\n        riceParamPart128 = _mm_or_si128(riceParamPart128, _mm_slli_epi32(zeroCountPart128, riceParam));\n        riceParamPart128 = _mm_xor_si128(_mm_srli_epi32(riceParamPart128, 1), _mm_add_epi32(ma_dr_flac__mm_not_si128(_mm_and_si128(riceParamPart128, _mm_set1_epi32(1))), _mm_set1_epi32(1)));\n        for (i = 0; i < 4; i += 1) {\n            prediction128 = _mm_xor_si128(prediction128, prediction128);\n            switch (order)\n            {\n            case 12:\n            case 11: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_8, _MM_SHUFFLE(1, 1, 0, 0))));\n            case 10:\n            case  9: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_8, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_8, _MM_SHUFFLE(3, 3, 2, 2))));\n            case  8:\n            case  7: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_4, _MM_SHUFFLE(1, 1, 0, 0))));\n            case  6:\n            case  5: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_4, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_4, _MM_SHUFFLE(3, 3, 2, 2))));\n            case  4:\n            case  3: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(1, 1, 0, 0)), _mm_shuffle_epi32(samples128_0, _MM_SHUFFLE(1, 1, 0, 0))));\n            case  2:\n            case  1: prediction128 = _mm_add_epi64(prediction128, _mm_mul_epi32(_mm_shuffle_epi32(coefficients128_0, _MM_SHUFFLE(3, 3, 2, 2)), _mm_shuffle_epi32(samples128_0, _MM_SHUFFLE(3, 3, 2, 2))));\n            }\n            prediction128 = ma_dr_flac__mm_hadd_epi64(prediction128);\n            prediction128 = ma_dr_flac__mm_srai_epi64(prediction128, shift);\n            prediction128 = _mm_add_epi32(riceParamPart128, prediction128);\n            samples128_8 = _mm_alignr_epi8(samples128_4,  samples128_8, 4);\n            samples128_4 = _mm_alignr_epi8(samples128_0,  samples128_4, 4);\n            samples128_0 = _mm_alignr_epi8(prediction128, samples128_0, 4);\n            riceParamPart128 = _mm_alignr_epi8(_mm_setzero_si128(), riceParamPart128, 4);\n        }\n        _mm_storeu_si128((__m128i*)pDecodedSamples, samples128_0);\n        pDecodedSamples += 4;\n    }\n    i = (count & ~3);\n    while (i < (int)count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts0, &riceParamParts0)) {\n            return MA_FALSE;\n        }\n        riceParamParts0 &= riceParamMask;\n        riceParamParts0 |= (zeroCountParts0 << riceParam);\n        riceParamParts0  = (riceParamParts0 >> 1) ^ t[riceParamParts0 & 0x01];\n        pDecodedSamples[0] = riceParamParts0 + ma_dr_flac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples);\n        i += 1;\n        pDecodedSamples += 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__sse41(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    if (lpcOrder > 0 && lpcOrder <= 12) {\n        if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n            return ma_dr_flac__decode_samples_with_residual__rice__sse41_64(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut);\n        } else {\n            return ma_dr_flac__decode_samples_with_residual__rice__sse41_32(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut);\n        }\n    } else {\n        return ma_dr_flac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac__vst2q_s32(ma_int32* p, int32x4x2_t x)\n{\n    vst1q_s32(p+0, x.val[0]);\n    vst1q_s32(p+4, x.val[1]);\n}\nstatic MA_INLINE void ma_dr_flac__vst2q_u32(ma_uint32* p, uint32x4x2_t x)\n{\n    vst1q_u32(p+0, x.val[0]);\n    vst1q_u32(p+4, x.val[1]);\n}\nstatic MA_INLINE void ma_dr_flac__vst2q_f32(float* p, float32x4x2_t x)\n{\n    vst1q_f32(p+0, x.val[0]);\n    vst1q_f32(p+4, x.val[1]);\n}\nstatic MA_INLINE void ma_dr_flac__vst2q_s16(ma_int16* p, int16x4x2_t x)\n{\n    vst1q_s16(p, vcombine_s16(x.val[0], x.val[1]));\n}\nstatic MA_INLINE void ma_dr_flac__vst2q_u16(ma_uint16* p, uint16x4x2_t x)\n{\n    vst1q_u16(p, vcombine_u16(x.val[0], x.val[1]));\n}\nstatic MA_INLINE int32x4_t ma_dr_flac__vdupq_n_s32x4(ma_int32 x3, ma_int32 x2, ma_int32 x1, ma_int32 x0)\n{\n    ma_int32 x[4];\n    x[3] = x3;\n    x[2] = x2;\n    x[1] = x1;\n    x[0] = x0;\n    return vld1q_s32(x);\n}\nstatic MA_INLINE int32x4_t ma_dr_flac__valignrq_s32_1(int32x4_t a, int32x4_t b)\n{\n    return vextq_s32(b, a, 1);\n}\nstatic MA_INLINE uint32x4_t ma_dr_flac__valignrq_u32_1(uint32x4_t a, uint32x4_t b)\n{\n    return vextq_u32(b, a, 1);\n}\nstatic MA_INLINE int32x2_t ma_dr_flac__vhaddq_s32(int32x4_t x)\n{\n    int32x2_t r = vadd_s32(vget_high_s32(x), vget_low_s32(x));\n    return vpadd_s32(r, r);\n}\nstatic MA_INLINE int64x1_t ma_dr_flac__vhaddq_s64(int64x2_t x)\n{\n    return vadd_s64(vget_high_s64(x), vget_low_s64(x));\n}\nstatic MA_INLINE int32x4_t ma_dr_flac__vrevq_s32(int32x4_t x)\n{\n    return vrev64q_s32(vcombine_s32(vget_high_s32(x), vget_low_s32(x)));\n}\nstatic MA_INLINE int32x4_t ma_dr_flac__vnotq_s32(int32x4_t x)\n{\n    return veorq_s32(x, vdupq_n_s32(0xFFFFFFFF));\n}\nstatic MA_INLINE uint32x4_t ma_dr_flac__vnotq_u32(uint32x4_t x)\n{\n    return veorq_u32(x, vdupq_n_u32(0xFFFFFFFF));\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_32(ma_dr_flac_bs* bs, ma_uint32 count, ma_uint8 riceParam, ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    int i;\n    ma_uint32 riceParamMask;\n    ma_int32* pDecodedSamples    = pSamplesOut;\n    ma_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3);\n    ma_uint32 zeroCountParts[4];\n    ma_uint32 riceParamParts[4];\n    int32x4_t coefficients128_0;\n    int32x4_t coefficients128_4;\n    int32x4_t coefficients128_8;\n    int32x4_t samples128_0;\n    int32x4_t samples128_4;\n    int32x4_t samples128_8;\n    uint32x4_t riceParamMask128;\n    int32x4_t riceParam128;\n    int32x2_t shift64;\n    uint32x4_t one128;\n    const ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    riceParamMask    = (ma_uint32)~((~0UL) << riceParam);\n    riceParamMask128 = vdupq_n_u32(riceParamMask);\n    riceParam128 = vdupq_n_s32(riceParam);\n    shift64 = vdup_n_s32(-shift);\n    one128 = vdupq_n_u32(1);\n    {\n        int runningOrder = order;\n        ma_int32 tempC[4] = {0, 0, 0, 0};\n        ma_int32 tempS[4] = {0, 0, 0, 0};\n        if (runningOrder >= 4) {\n            coefficients128_0 = vld1q_s32(coefficients + 0);\n            samples128_0      = vld1q_s32(pSamplesOut  - 4);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[2]; tempS[1] = pSamplesOut[-3];\n                case 2: tempC[1] = coefficients[1]; tempS[2] = pSamplesOut[-2];\n                case 1: tempC[0] = coefficients[0]; tempS[3] = pSamplesOut[-1];\n            }\n            coefficients128_0 = vld1q_s32(tempC);\n            samples128_0      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        if (runningOrder >= 4) {\n            coefficients128_4 = vld1q_s32(coefficients + 4);\n            samples128_4      = vld1q_s32(pSamplesOut  - 8);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[6]; tempS[1] = pSamplesOut[-7];\n                case 2: tempC[1] = coefficients[5]; tempS[2] = pSamplesOut[-6];\n                case 1: tempC[0] = coefficients[4]; tempS[3] = pSamplesOut[-5];\n            }\n            coefficients128_4 = vld1q_s32(tempC);\n            samples128_4      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        if (runningOrder == 4) {\n            coefficients128_8 = vld1q_s32(coefficients + 8);\n            samples128_8      = vld1q_s32(pSamplesOut  - 12);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[10]; tempS[1] = pSamplesOut[-11];\n                case 2: tempC[1] = coefficients[ 9]; tempS[2] = pSamplesOut[-10];\n                case 1: tempC[0] = coefficients[ 8]; tempS[3] = pSamplesOut[- 9];\n            }\n            coefficients128_8 = vld1q_s32(tempC);\n            samples128_8      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        coefficients128_0 = ma_dr_flac__vrevq_s32(coefficients128_0);\n        coefficients128_4 = ma_dr_flac__vrevq_s32(coefficients128_4);\n        coefficients128_8 = ma_dr_flac__vrevq_s32(coefficients128_8);\n    }\n    while (pDecodedSamples < pDecodedSamplesEnd) {\n        int32x4_t prediction128;\n        int32x2_t prediction64;\n        uint32x4_t zeroCountPart128;\n        uint32x4_t riceParamPart128;\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[1], &riceParamParts[1]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[2], &riceParamParts[2]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[3], &riceParamParts[3])) {\n            return MA_FALSE;\n        }\n        zeroCountPart128 = vld1q_u32(zeroCountParts);\n        riceParamPart128 = vld1q_u32(riceParamParts);\n        riceParamPart128 = vandq_u32(riceParamPart128, riceParamMask128);\n        riceParamPart128 = vorrq_u32(riceParamPart128, vshlq_u32(zeroCountPart128, riceParam128));\n        riceParamPart128 = veorq_u32(vshrq_n_u32(riceParamPart128, 1), vaddq_u32(ma_dr_flac__vnotq_u32(vandq_u32(riceParamPart128, one128)), one128));\n        if (order <= 4) {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 = vmulq_s32(coefficients128_0, samples128_0);\n                prediction64 = ma_dr_flac__vhaddq_s32(prediction128);\n                prediction64 = vshl_s32(prediction64, shift64);\n                prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128)));\n                samples128_0 = ma_dr_flac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0);\n                riceParamPart128 = ma_dr_flac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128);\n            }\n        } else if (order <= 8) {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 =                vmulq_s32(coefficients128_4, samples128_4);\n                prediction128 = vmlaq_s32(prediction128, coefficients128_0, samples128_0);\n                prediction64 = ma_dr_flac__vhaddq_s32(prediction128);\n                prediction64 = vshl_s32(prediction64, shift64);\n                prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128)));\n                samples128_4 = ma_dr_flac__valignrq_s32_1(samples128_0, samples128_4);\n                samples128_0 = ma_dr_flac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0);\n                riceParamPart128 = ma_dr_flac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128);\n            }\n        } else {\n            for (i = 0; i < 4; i += 1) {\n                prediction128 =                vmulq_s32(coefficients128_8, samples128_8);\n                prediction128 = vmlaq_s32(prediction128, coefficients128_4, samples128_4);\n                prediction128 = vmlaq_s32(prediction128, coefficients128_0, samples128_0);\n                prediction64 = ma_dr_flac__vhaddq_s32(prediction128);\n                prediction64 = vshl_s32(prediction64, shift64);\n                prediction64 = vadd_s32(prediction64, vget_low_s32(vreinterpretq_s32_u32(riceParamPart128)));\n                samples128_8 = ma_dr_flac__valignrq_s32_1(samples128_4, samples128_8);\n                samples128_4 = ma_dr_flac__valignrq_s32_1(samples128_0, samples128_4);\n                samples128_0 = ma_dr_flac__valignrq_s32_1(vcombine_s32(prediction64, vdup_n_s32(0)), samples128_0);\n                riceParamPart128 = ma_dr_flac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128);\n            }\n        }\n        vst1q_s32(pDecodedSamples, samples128_0);\n        pDecodedSamples += 4;\n    }\n    i = (count & ~3);\n    while (i < (int)count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0])) {\n            return MA_FALSE;\n        }\n        riceParamParts[0] &= riceParamMask;\n        riceParamParts[0] |= (zeroCountParts[0] << riceParam);\n        riceParamParts[0]  = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01];\n        pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_32(order, shift, coefficients, pDecodedSamples);\n        i += 1;\n        pDecodedSamples += 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon_64(ma_dr_flac_bs* bs, ma_uint32 count, ma_uint8 riceParam, ma_uint32 order, ma_int32 shift, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    int i;\n    ma_uint32 riceParamMask;\n    ma_int32* pDecodedSamples    = pSamplesOut;\n    ma_int32* pDecodedSamplesEnd = pSamplesOut + (count & ~3);\n    ma_uint32 zeroCountParts[4];\n    ma_uint32 riceParamParts[4];\n    int32x4_t coefficients128_0;\n    int32x4_t coefficients128_4;\n    int32x4_t coefficients128_8;\n    int32x4_t samples128_0;\n    int32x4_t samples128_4;\n    int32x4_t samples128_8;\n    uint32x4_t riceParamMask128;\n    int32x4_t riceParam128;\n    int64x1_t shift64;\n    uint32x4_t one128;\n    int64x2_t prediction128 = { 0 };\n    uint32x4_t zeroCountPart128;\n    uint32x4_t riceParamPart128;\n    const ma_uint32 t[2] = {0x00000000, 0xFFFFFFFF};\n    riceParamMask    = (ma_uint32)~((~0UL) << riceParam);\n    riceParamMask128 = vdupq_n_u32(riceParamMask);\n    riceParam128 = vdupq_n_s32(riceParam);\n    shift64 = vdup_n_s64(-shift);\n    one128 = vdupq_n_u32(1);\n    {\n        int runningOrder = order;\n        ma_int32 tempC[4] = {0, 0, 0, 0};\n        ma_int32 tempS[4] = {0, 0, 0, 0};\n        if (runningOrder >= 4) {\n            coefficients128_0 = vld1q_s32(coefficients + 0);\n            samples128_0      = vld1q_s32(pSamplesOut  - 4);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[2]; tempS[1] = pSamplesOut[-3];\n                case 2: tempC[1] = coefficients[1]; tempS[2] = pSamplesOut[-2];\n                case 1: tempC[0] = coefficients[0]; tempS[3] = pSamplesOut[-1];\n            }\n            coefficients128_0 = vld1q_s32(tempC);\n            samples128_0      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        if (runningOrder >= 4) {\n            coefficients128_4 = vld1q_s32(coefficients + 4);\n            samples128_4      = vld1q_s32(pSamplesOut  - 8);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[6]; tempS[1] = pSamplesOut[-7];\n                case 2: tempC[1] = coefficients[5]; tempS[2] = pSamplesOut[-6];\n                case 1: tempC[0] = coefficients[4]; tempS[3] = pSamplesOut[-5];\n            }\n            coefficients128_4 = vld1q_s32(tempC);\n            samples128_4      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        if (runningOrder == 4) {\n            coefficients128_8 = vld1q_s32(coefficients + 8);\n            samples128_8      = vld1q_s32(pSamplesOut  - 12);\n            runningOrder -= 4;\n        } else {\n            switch (runningOrder) {\n                case 3: tempC[2] = coefficients[10]; tempS[1] = pSamplesOut[-11];\n                case 2: tempC[1] = coefficients[ 9]; tempS[2] = pSamplesOut[-10];\n                case 1: tempC[0] = coefficients[ 8]; tempS[3] = pSamplesOut[- 9];\n            }\n            coefficients128_8 = vld1q_s32(tempC);\n            samples128_8      = vld1q_s32(tempS);\n            runningOrder = 0;\n        }\n        coefficients128_0 = ma_dr_flac__vrevq_s32(coefficients128_0);\n        coefficients128_4 = ma_dr_flac__vrevq_s32(coefficients128_4);\n        coefficients128_8 = ma_dr_flac__vrevq_s32(coefficients128_8);\n    }\n    while (pDecodedSamples < pDecodedSamplesEnd) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[1], &riceParamParts[1]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[2], &riceParamParts[2]) ||\n            !ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[3], &riceParamParts[3])) {\n            return MA_FALSE;\n        }\n        zeroCountPart128 = vld1q_u32(zeroCountParts);\n        riceParamPart128 = vld1q_u32(riceParamParts);\n        riceParamPart128 = vandq_u32(riceParamPart128, riceParamMask128);\n        riceParamPart128 = vorrq_u32(riceParamPart128, vshlq_u32(zeroCountPart128, riceParam128));\n        riceParamPart128 = veorq_u32(vshrq_n_u32(riceParamPart128, 1), vaddq_u32(ma_dr_flac__vnotq_u32(vandq_u32(riceParamPart128, one128)), one128));\n        for (i = 0; i < 4; i += 1) {\n            int64x1_t prediction64;\n            prediction128 = veorq_s64(prediction128, prediction128);\n            switch (order)\n            {\n            case 12:\n            case 11: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_8), vget_low_s32(samples128_8)));\n            case 10:\n            case  9: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_8), vget_high_s32(samples128_8)));\n            case  8:\n            case  7: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_4), vget_low_s32(samples128_4)));\n            case  6:\n            case  5: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_4), vget_high_s32(samples128_4)));\n            case  4:\n            case  3: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_low_s32(coefficients128_0), vget_low_s32(samples128_0)));\n            case  2:\n            case  1: prediction128 = vaddq_s64(prediction128, vmull_s32(vget_high_s32(coefficients128_0), vget_high_s32(samples128_0)));\n            }\n            prediction64 = ma_dr_flac__vhaddq_s64(prediction128);\n            prediction64 = vshl_s64(prediction64, shift64);\n            prediction64 = vadd_s64(prediction64, vdup_n_s64(vgetq_lane_u32(riceParamPart128, 0)));\n            samples128_8 = ma_dr_flac__valignrq_s32_1(samples128_4, samples128_8);\n            samples128_4 = ma_dr_flac__valignrq_s32_1(samples128_0, samples128_4);\n            samples128_0 = ma_dr_flac__valignrq_s32_1(vcombine_s32(vreinterpret_s32_s64(prediction64), vdup_n_s32(0)), samples128_0);\n            riceParamPart128 = ma_dr_flac__valignrq_u32_1(vdupq_n_u32(0), riceParamPart128);\n        }\n        vst1q_s32(pDecodedSamples, samples128_0);\n        pDecodedSamples += 4;\n    }\n    i = (count & ~3);\n    while (i < (int)count) {\n        if (!ma_dr_flac__read_rice_parts_x1(bs, riceParam, &zeroCountParts[0], &riceParamParts[0])) {\n            return MA_FALSE;\n        }\n        riceParamParts[0] &= riceParamMask;\n        riceParamParts[0] |= (zeroCountParts[0] << riceParam);\n        riceParamParts[0]  = (riceParamParts[0] >> 1) ^ t[riceParamParts[0] & 0x01];\n        pDecodedSamples[0] = riceParamParts[0] + ma_dr_flac__calculate_prediction_64(order, shift, coefficients, pDecodedSamples);\n        i += 1;\n        pDecodedSamples += 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice__neon(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    if (lpcOrder > 0 && lpcOrder <= 12) {\n        if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n            return ma_dr_flac__decode_samples_with_residual__rice__neon_64(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut);\n        } else {\n            return ma_dr_flac__decode_samples_with_residual__rice__neon_32(bs, count, riceParam, lpcOrder, lpcShift, coefficients, pSamplesOut);\n        }\n    } else {\n        return ma_dr_flac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    }\n}\n#endif\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__rice(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 riceParam, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE41)\n    if (ma_dr_flac__gIsSSE41Supported) {\n        return ma_dr_flac__decode_samples_with_residual__rice__sse41(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported) {\n        return ma_dr_flac__decode_samples_with_residual__rice__neon(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    } else\n#endif\n    {\n    #if 0\n        return ma_dr_flac__decode_samples_with_residual__rice__reference(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    #else\n        return ma_dr_flac__decode_samples_with_residual__rice__scalar(bs, bitsPerSample, count, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pSamplesOut);\n    #endif\n    }\n}\nstatic ma_bool32 ma_dr_flac__read_and_seek_residual__rice(ma_dr_flac_bs* bs, ma_uint32 count, ma_uint8 riceParam)\n{\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    for (i = 0; i < count; ++i) {\n        if (!ma_dr_flac__seek_rice_parts(bs, riceParam)) {\n            return MA_FALSE;\n        }\n    }\n    return MA_TRUE;\n}\n#if defined(__clang__)\n__attribute__((no_sanitize(\"signed-integer-overflow\")))\n#endif\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual__unencoded(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 count, ma_uint8 unencodedBitsPerSample, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pSamplesOut)\n{\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(unencodedBitsPerSample <= 31);\n    MA_DR_FLAC_ASSERT(pSamplesOut != NULL);\n    for (i = 0; i < count; ++i) {\n        if (unencodedBitsPerSample > 0) {\n            if (!ma_dr_flac__read_int32(bs, unencodedBitsPerSample, pSamplesOut + i)) {\n                return MA_FALSE;\n            }\n        } else {\n            pSamplesOut[i] = 0;\n        }\n        if (ma_dr_flac__use_64_bit_prediction(bitsPerSample, lpcOrder, lpcPrecision)) {\n            pSamplesOut[i] += ma_dr_flac__calculate_prediction_64(lpcOrder, lpcShift, coefficients, pSamplesOut + i);\n        } else {\n            pSamplesOut[i] += ma_dr_flac__calculate_prediction_32(lpcOrder, lpcShift, coefficients, pSamplesOut + i);\n        }\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples_with_residual(ma_dr_flac_bs* bs, ma_uint32 bitsPerSample, ma_uint32 blockSize, ma_uint32 lpcOrder, ma_int32 lpcShift, ma_uint32 lpcPrecision, const ma_int32* coefficients, ma_int32* pDecodedSamples)\n{\n    ma_uint8 residualMethod;\n    ma_uint8 partitionOrder;\n    ma_uint32 samplesInPartition;\n    ma_uint32 partitionsRemaining;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(blockSize != 0);\n    MA_DR_FLAC_ASSERT(pDecodedSamples != NULL);\n    if (!ma_dr_flac__read_uint8(bs, 2, &residualMethod)) {\n        return MA_FALSE;\n    }\n    if (residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) {\n        return MA_FALSE;\n    }\n    pDecodedSamples += lpcOrder;\n    if (!ma_dr_flac__read_uint8(bs, 4, &partitionOrder)) {\n        return MA_FALSE;\n    }\n    if (partitionOrder > 8) {\n        return MA_FALSE;\n    }\n    if ((blockSize / (1 << partitionOrder)) < lpcOrder) {\n        return MA_FALSE;\n    }\n    samplesInPartition = (blockSize / (1 << partitionOrder)) - lpcOrder;\n    partitionsRemaining = (1 << partitionOrder);\n    for (;;) {\n        ma_uint8 riceParam = 0;\n        if (residualMethod == MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) {\n            if (!ma_dr_flac__read_uint8(bs, 4, &riceParam)) {\n                return MA_FALSE;\n            }\n            if (riceParam == 15) {\n                riceParam = 0xFF;\n            }\n        } else if (residualMethod == MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) {\n            if (!ma_dr_flac__read_uint8(bs, 5, &riceParam)) {\n                return MA_FALSE;\n            }\n            if (riceParam == 31) {\n                riceParam = 0xFF;\n            }\n        }\n        if (riceParam != 0xFF) {\n            if (!ma_dr_flac__decode_samples_with_residual__rice(bs, bitsPerSample, samplesInPartition, riceParam, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) {\n                return MA_FALSE;\n            }\n        } else {\n            ma_uint8 unencodedBitsPerSample = 0;\n            if (!ma_dr_flac__read_uint8(bs, 5, &unencodedBitsPerSample)) {\n                return MA_FALSE;\n            }\n            if (!ma_dr_flac__decode_samples_with_residual__unencoded(bs, bitsPerSample, samplesInPartition, unencodedBitsPerSample, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) {\n                return MA_FALSE;\n            }\n        }\n        pDecodedSamples += samplesInPartition;\n        if (partitionsRemaining == 1) {\n            break;\n        }\n        partitionsRemaining -= 1;\n        if (partitionOrder != 0) {\n            samplesInPartition = blockSize / (1 << partitionOrder);\n        }\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__read_and_seek_residual(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 order)\n{\n    ma_uint8 residualMethod;\n    ma_uint8 partitionOrder;\n    ma_uint32 samplesInPartition;\n    ma_uint32 partitionsRemaining;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(blockSize != 0);\n    if (!ma_dr_flac__read_uint8(bs, 2, &residualMethod)) {\n        return MA_FALSE;\n    }\n    if (residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE && residualMethod != MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_flac__read_uint8(bs, 4, &partitionOrder)) {\n        return MA_FALSE;\n    }\n    if (partitionOrder > 8) {\n        return MA_FALSE;\n    }\n    if ((blockSize / (1 << partitionOrder)) <= order) {\n        return MA_FALSE;\n    }\n    samplesInPartition = (blockSize / (1 << partitionOrder)) - order;\n    partitionsRemaining = (1 << partitionOrder);\n    for (;;)\n    {\n        ma_uint8 riceParam = 0;\n        if (residualMethod == MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE) {\n            if (!ma_dr_flac__read_uint8(bs, 4, &riceParam)) {\n                return MA_FALSE;\n            }\n            if (riceParam == 15) {\n                riceParam = 0xFF;\n            }\n        } else if (residualMethod == MA_DR_FLAC_RESIDUAL_CODING_METHOD_PARTITIONED_RICE2) {\n            if (!ma_dr_flac__read_uint8(bs, 5, &riceParam)) {\n                return MA_FALSE;\n            }\n            if (riceParam == 31) {\n                riceParam = 0xFF;\n            }\n        }\n        if (riceParam != 0xFF) {\n            if (!ma_dr_flac__read_and_seek_residual__rice(bs, samplesInPartition, riceParam)) {\n                return MA_FALSE;\n            }\n        } else {\n            ma_uint8 unencodedBitsPerSample = 0;\n            if (!ma_dr_flac__read_uint8(bs, 5, &unencodedBitsPerSample)) {\n                return MA_FALSE;\n            }\n            if (!ma_dr_flac__seek_bits(bs, unencodedBitsPerSample * samplesInPartition)) {\n                return MA_FALSE;\n            }\n        }\n        if (partitionsRemaining == 1) {\n            break;\n        }\n        partitionsRemaining -= 1;\n        samplesInPartition = blockSize / (1 << partitionOrder);\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples__constant(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamples)\n{\n    ma_uint32 i;\n    ma_int32 sample;\n    if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) {\n        return MA_FALSE;\n    }\n    for (i = 0; i < blockSize; ++i) {\n        pDecodedSamples[i] = sample;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples__verbatim(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_int32* pDecodedSamples)\n{\n    ma_uint32 i;\n    for (i = 0; i < blockSize; ++i) {\n        ma_int32 sample;\n        if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) {\n            return MA_FALSE;\n        }\n        pDecodedSamples[i] = sample;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples__fixed(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 subframeBitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamples)\n{\n    ma_uint32 i;\n    static ma_int32 lpcCoefficientsTable[5][4] = {\n        {0,  0, 0,  0},\n        {1,  0, 0,  0},\n        {2, -1, 0,  0},\n        {3, -3, 1,  0},\n        {4, -6, 4, -1}\n    };\n    for (i = 0; i < lpcOrder; ++i) {\n        ma_int32 sample;\n        if (!ma_dr_flac__read_int32(bs, subframeBitsPerSample, &sample)) {\n            return MA_FALSE;\n        }\n        pDecodedSamples[i] = sample;\n    }\n    if (!ma_dr_flac__decode_samples_with_residual(bs, subframeBitsPerSample, blockSize, lpcOrder, 0, 4, lpcCoefficientsTable[lpcOrder], pDecodedSamples)) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_samples__lpc(ma_dr_flac_bs* bs, ma_uint32 blockSize, ma_uint32 bitsPerSample, ma_uint8 lpcOrder, ma_int32* pDecodedSamples)\n{\n    ma_uint8 i;\n    ma_uint8 lpcPrecision;\n    ma_int8 lpcShift;\n    ma_int32 coefficients[32];\n    for (i = 0; i < lpcOrder; ++i) {\n        ma_int32 sample;\n        if (!ma_dr_flac__read_int32(bs, bitsPerSample, &sample)) {\n            return MA_FALSE;\n        }\n        pDecodedSamples[i] = sample;\n    }\n    if (!ma_dr_flac__read_uint8(bs, 4, &lpcPrecision)) {\n        return MA_FALSE;\n    }\n    if (lpcPrecision == 15) {\n        return MA_FALSE;\n    }\n    lpcPrecision += 1;\n    if (!ma_dr_flac__read_int8(bs, 5, &lpcShift)) {\n        return MA_FALSE;\n    }\n    if (lpcShift < 0) {\n        return MA_FALSE;\n    }\n    MA_DR_FLAC_ZERO_MEMORY(coefficients, sizeof(coefficients));\n    for (i = 0; i < lpcOrder; ++i) {\n        if (!ma_dr_flac__read_int32(bs, lpcPrecision, coefficients + i)) {\n            return MA_FALSE;\n        }\n    }\n    if (!ma_dr_flac__decode_samples_with_residual(bs, bitsPerSample, blockSize, lpcOrder, lpcShift, lpcPrecision, coefficients, pDecodedSamples)) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__read_next_flac_frame_header(ma_dr_flac_bs* bs, ma_uint8 streaminfoBitsPerSample, ma_dr_flac_frame_header* header)\n{\n    const ma_uint32 sampleRateTable[12]  = {0, 88200, 176400, 192000, 8000, 16000, 22050, 24000, 32000, 44100, 48000, 96000};\n    const ma_uint8 bitsPerSampleTable[8] = {0, 8, 12, (ma_uint8)-1, 16, 20, 24, (ma_uint8)-1};\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(header != NULL);\n    for (;;) {\n        ma_uint8 crc8 = 0xCE;\n        ma_uint8 reserved = 0;\n        ma_uint8 blockingStrategy = 0;\n        ma_uint8 blockSize = 0;\n        ma_uint8 sampleRate = 0;\n        ma_uint8 channelAssignment = 0;\n        ma_uint8 bitsPerSample = 0;\n        ma_bool32 isVariableBlockSize;\n        if (!ma_dr_flac__find_and_seek_to_next_sync_code(bs)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_flac__read_uint8(bs, 1, &reserved)) {\n            return MA_FALSE;\n        }\n        if (reserved == 1) {\n            continue;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, reserved, 1);\n        if (!ma_dr_flac__read_uint8(bs, 1, &blockingStrategy)) {\n            return MA_FALSE;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, blockingStrategy, 1);\n        if (!ma_dr_flac__read_uint8(bs, 4, &blockSize)) {\n            return MA_FALSE;\n        }\n        if (blockSize == 0) {\n            continue;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, blockSize, 4);\n        if (!ma_dr_flac__read_uint8(bs, 4, &sampleRate)) {\n            return MA_FALSE;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, sampleRate, 4);\n        if (!ma_dr_flac__read_uint8(bs, 4, &channelAssignment)) {\n            return MA_FALSE;\n        }\n        if (channelAssignment > 10) {\n            continue;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, channelAssignment, 4);\n        if (!ma_dr_flac__read_uint8(bs, 3, &bitsPerSample)) {\n            return MA_FALSE;\n        }\n        if (bitsPerSample == 3 || bitsPerSample == 7) {\n            continue;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, bitsPerSample, 3);\n        if (!ma_dr_flac__read_uint8(bs, 1, &reserved)) {\n            return MA_FALSE;\n        }\n        if (reserved == 1) {\n            continue;\n        }\n        crc8 = ma_dr_flac_crc8(crc8, reserved, 1);\n        isVariableBlockSize = blockingStrategy == 1;\n        if (isVariableBlockSize) {\n            ma_uint64 pcmFrameNumber;\n            ma_result result = ma_dr_flac__read_utf8_coded_number(bs, &pcmFrameNumber, &crc8);\n            if (result != MA_SUCCESS) {\n                if (result == MA_AT_END) {\n                    return MA_FALSE;\n                } else {\n                    continue;\n                }\n            }\n            header->flacFrameNumber  = 0;\n            header->pcmFrameNumber = pcmFrameNumber;\n        } else {\n            ma_uint64 flacFrameNumber = 0;\n            ma_result result = ma_dr_flac__read_utf8_coded_number(bs, &flacFrameNumber, &crc8);\n            if (result != MA_SUCCESS) {\n                if (result == MA_AT_END) {\n                    return MA_FALSE;\n                } else {\n                    continue;\n                }\n            }\n            header->flacFrameNumber  = (ma_uint32)flacFrameNumber;\n            header->pcmFrameNumber = 0;\n        }\n        MA_DR_FLAC_ASSERT(blockSize > 0);\n        if (blockSize == 1) {\n            header->blockSizeInPCMFrames = 192;\n        } else if (blockSize <= 5) {\n            MA_DR_FLAC_ASSERT(blockSize >= 2);\n            header->blockSizeInPCMFrames = 576 * (1 << (blockSize - 2));\n        } else if (blockSize == 6) {\n            if (!ma_dr_flac__read_uint16(bs, 8, &header->blockSizeInPCMFrames)) {\n                return MA_FALSE;\n            }\n            crc8 = ma_dr_flac_crc8(crc8, header->blockSizeInPCMFrames, 8);\n            header->blockSizeInPCMFrames += 1;\n        } else if (blockSize == 7) {\n            if (!ma_dr_flac__read_uint16(bs, 16, &header->blockSizeInPCMFrames)) {\n                return MA_FALSE;\n            }\n            crc8 = ma_dr_flac_crc8(crc8, header->blockSizeInPCMFrames, 16);\n            if (header->blockSizeInPCMFrames == 0xFFFF) {\n                return MA_FALSE;\n            }\n            header->blockSizeInPCMFrames += 1;\n        } else {\n            MA_DR_FLAC_ASSERT(blockSize >= 8);\n            header->blockSizeInPCMFrames = 256 * (1 << (blockSize - 8));\n        }\n        if (sampleRate <= 11) {\n            header->sampleRate = sampleRateTable[sampleRate];\n        } else if (sampleRate == 12) {\n            if (!ma_dr_flac__read_uint32(bs, 8, &header->sampleRate)) {\n                return MA_FALSE;\n            }\n            crc8 = ma_dr_flac_crc8(crc8, header->sampleRate, 8);\n            header->sampleRate *= 1000;\n        } else if (sampleRate == 13) {\n            if (!ma_dr_flac__read_uint32(bs, 16, &header->sampleRate)) {\n                return MA_FALSE;\n            }\n            crc8 = ma_dr_flac_crc8(crc8, header->sampleRate, 16);\n        } else if (sampleRate == 14) {\n            if (!ma_dr_flac__read_uint32(bs, 16, &header->sampleRate)) {\n                return MA_FALSE;\n            }\n            crc8 = ma_dr_flac_crc8(crc8, header->sampleRate, 16);\n            header->sampleRate *= 10;\n        } else {\n            continue;\n        }\n        header->channelAssignment = channelAssignment;\n        header->bitsPerSample = bitsPerSampleTable[bitsPerSample];\n        if (header->bitsPerSample == 0) {\n            header->bitsPerSample = streaminfoBitsPerSample;\n        }\n        if (header->bitsPerSample != streaminfoBitsPerSample) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_flac__read_uint8(bs, 8, &header->crc8)) {\n            return MA_FALSE;\n        }\n#ifndef MA_DR_FLAC_NO_CRC\n        if (header->crc8 != crc8) {\n            continue;\n        }\n#endif\n        return MA_TRUE;\n    }\n}\nstatic ma_bool32 ma_dr_flac__read_subframe_header(ma_dr_flac_bs* bs, ma_dr_flac_subframe* pSubframe)\n{\n    ma_uint8 header;\n    int type;\n    if (!ma_dr_flac__read_uint8(bs, 8, &header)) {\n        return MA_FALSE;\n    }\n    if ((header & 0x80) != 0) {\n        return MA_FALSE;\n    }\n    pSubframe->lpcOrder = 0;\n    type = (header & 0x7E) >> 1;\n    if (type == 0) {\n        pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_CONSTANT;\n    } else if (type == 1) {\n        pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_VERBATIM;\n    } else {\n        if ((type & 0x20) != 0) {\n            pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_LPC;\n            pSubframe->lpcOrder = (ma_uint8)(type & 0x1F) + 1;\n        } else if ((type & 0x08) != 0) {\n            pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_FIXED;\n            pSubframe->lpcOrder = (ma_uint8)(type & 0x07);\n            if (pSubframe->lpcOrder > 4) {\n                pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_RESERVED;\n                pSubframe->lpcOrder = 0;\n            }\n        } else {\n            pSubframe->subframeType = MA_DR_FLAC_SUBFRAME_RESERVED;\n        }\n    }\n    if (pSubframe->subframeType == MA_DR_FLAC_SUBFRAME_RESERVED) {\n        return MA_FALSE;\n    }\n    pSubframe->wastedBitsPerSample = 0;\n    if ((header & 0x01) == 1) {\n        unsigned int wastedBitsPerSample;\n        if (!ma_dr_flac__seek_past_next_set_bit(bs, &wastedBitsPerSample)) {\n            return MA_FALSE;\n        }\n        pSubframe->wastedBitsPerSample = (ma_uint8)wastedBitsPerSample + 1;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame* frame, int subframeIndex, ma_int32* pDecodedSamplesOut)\n{\n    ma_dr_flac_subframe* pSubframe;\n    ma_uint32 subframeBitsPerSample;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(frame != NULL);\n    pSubframe = frame->subframes + subframeIndex;\n    if (!ma_dr_flac__read_subframe_header(bs, pSubframe)) {\n        return MA_FALSE;\n    }\n    subframeBitsPerSample = frame->header.bitsPerSample;\n    if ((frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) {\n        subframeBitsPerSample += 1;\n    } else if (frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) {\n        subframeBitsPerSample += 1;\n    }\n    if (subframeBitsPerSample > 32) {\n        return MA_FALSE;\n    }\n    if (pSubframe->wastedBitsPerSample >= subframeBitsPerSample) {\n        return MA_FALSE;\n    }\n    subframeBitsPerSample -= pSubframe->wastedBitsPerSample;\n    pSubframe->pSamplesS32 = pDecodedSamplesOut;\n    if (frame->header.blockSizeInPCMFrames < pSubframe->lpcOrder) {\n        return MA_FALSE;\n    }\n    switch (pSubframe->subframeType)\n    {\n        case MA_DR_FLAC_SUBFRAME_CONSTANT:\n        {\n            ma_dr_flac__decode_samples__constant(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32);\n        } break;\n        case MA_DR_FLAC_SUBFRAME_VERBATIM:\n        {\n            ma_dr_flac__decode_samples__verbatim(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->pSamplesS32);\n        } break;\n        case MA_DR_FLAC_SUBFRAME_FIXED:\n        {\n            ma_dr_flac__decode_samples__fixed(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32);\n        } break;\n        case MA_DR_FLAC_SUBFRAME_LPC:\n        {\n            ma_dr_flac__decode_samples__lpc(bs, frame->header.blockSizeInPCMFrames, subframeBitsPerSample, pSubframe->lpcOrder, pSubframe->pSamplesS32);\n        } break;\n        default: return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__seek_subframe(ma_dr_flac_bs* bs, ma_dr_flac_frame* frame, int subframeIndex)\n{\n    ma_dr_flac_subframe* pSubframe;\n    ma_uint32 subframeBitsPerSample;\n    MA_DR_FLAC_ASSERT(bs != NULL);\n    MA_DR_FLAC_ASSERT(frame != NULL);\n    pSubframe = frame->subframes + subframeIndex;\n    if (!ma_dr_flac__read_subframe_header(bs, pSubframe)) {\n        return MA_FALSE;\n    }\n    subframeBitsPerSample = frame->header.bitsPerSample;\n    if ((frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE || frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE) && subframeIndex == 1) {\n        subframeBitsPerSample += 1;\n    } else if (frame->header.channelAssignment == MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE && subframeIndex == 0) {\n        subframeBitsPerSample += 1;\n    }\n    if (pSubframe->wastedBitsPerSample >= subframeBitsPerSample) {\n        return MA_FALSE;\n    }\n    subframeBitsPerSample -= pSubframe->wastedBitsPerSample;\n    pSubframe->pSamplesS32 = NULL;\n    switch (pSubframe->subframeType)\n    {\n        case MA_DR_FLAC_SUBFRAME_CONSTANT:\n        {\n            if (!ma_dr_flac__seek_bits(bs, subframeBitsPerSample)) {\n                return MA_FALSE;\n            }\n        } break;\n        case MA_DR_FLAC_SUBFRAME_VERBATIM:\n        {\n            unsigned int bitsToSeek = frame->header.blockSizeInPCMFrames * subframeBitsPerSample;\n            if (!ma_dr_flac__seek_bits(bs, bitsToSeek)) {\n                return MA_FALSE;\n            }\n        } break;\n        case MA_DR_FLAC_SUBFRAME_FIXED:\n        {\n            unsigned int bitsToSeek = pSubframe->lpcOrder * subframeBitsPerSample;\n            if (!ma_dr_flac__seek_bits(bs, bitsToSeek)) {\n                return MA_FALSE;\n            }\n            if (!ma_dr_flac__read_and_seek_residual(bs, frame->header.blockSizeInPCMFrames, pSubframe->lpcOrder)) {\n                return MA_FALSE;\n            }\n        } break;\n        case MA_DR_FLAC_SUBFRAME_LPC:\n        {\n            ma_uint8 lpcPrecision;\n            unsigned int bitsToSeek = pSubframe->lpcOrder * subframeBitsPerSample;\n            if (!ma_dr_flac__seek_bits(bs, bitsToSeek)) {\n                return MA_FALSE;\n            }\n            if (!ma_dr_flac__read_uint8(bs, 4, &lpcPrecision)) {\n                return MA_FALSE;\n            }\n            if (lpcPrecision == 15) {\n                return MA_FALSE;\n            }\n            lpcPrecision += 1;\n            bitsToSeek = (pSubframe->lpcOrder * lpcPrecision) + 5;\n            if (!ma_dr_flac__seek_bits(bs, bitsToSeek)) {\n                return MA_FALSE;\n            }\n            if (!ma_dr_flac__read_and_seek_residual(bs, frame->header.blockSizeInPCMFrames, pSubframe->lpcOrder)) {\n                return MA_FALSE;\n            }\n        } break;\n        default: return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic MA_INLINE ma_uint8 ma_dr_flac__get_channel_count_from_channel_assignment(ma_int8 channelAssignment)\n{\n    ma_uint8 lookup[] = {1, 2, 3, 4, 5, 6, 7, 8, 2, 2, 2};\n    MA_DR_FLAC_ASSERT(channelAssignment <= 10);\n    return lookup[channelAssignment];\n}\nstatic ma_result ma_dr_flac__decode_flac_frame(ma_dr_flac* pFlac)\n{\n    int channelCount;\n    int i;\n    ma_uint8 paddingSizeInBits;\n    ma_uint16 desiredCRC16;\n#ifndef MA_DR_FLAC_NO_CRC\n    ma_uint16 actualCRC16;\n#endif\n    MA_DR_FLAC_ZERO_MEMORY(pFlac->currentFLACFrame.subframes, sizeof(pFlac->currentFLACFrame.subframes));\n    if (pFlac->currentFLACFrame.header.blockSizeInPCMFrames > pFlac->maxBlockSizeInPCMFrames) {\n        return MA_ERROR;\n    }\n    channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment);\n    if (channelCount != (int)pFlac->channels) {\n        return MA_ERROR;\n    }\n    for (i = 0; i < channelCount; ++i) {\n        if (!ma_dr_flac__decode_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i, pFlac->pDecodedSamples + (pFlac->currentFLACFrame.header.blockSizeInPCMFrames * i))) {\n            return MA_ERROR;\n        }\n    }\n    paddingSizeInBits = (ma_uint8)(MA_DR_FLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7);\n    if (paddingSizeInBits > 0) {\n        ma_uint8 padding = 0;\n        if (!ma_dr_flac__read_uint8(&pFlac->bs, paddingSizeInBits, &padding)) {\n            return MA_AT_END;\n        }\n    }\n#ifndef MA_DR_FLAC_NO_CRC\n    actualCRC16 = ma_dr_flac__flush_crc16(&pFlac->bs);\n#endif\n    if (!ma_dr_flac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) {\n        return MA_AT_END;\n    }\n#ifndef MA_DR_FLAC_NO_CRC\n    if (actualCRC16 != desiredCRC16) {\n        return MA_CRC_MISMATCH;\n    }\n#endif\n    pFlac->currentFLACFrame.pcmFramesRemaining = pFlac->currentFLACFrame.header.blockSizeInPCMFrames;\n    return MA_SUCCESS;\n}\nstatic ma_result ma_dr_flac__seek_flac_frame(ma_dr_flac* pFlac)\n{\n    int channelCount;\n    int i;\n    ma_uint16 desiredCRC16;\n#ifndef MA_DR_FLAC_NO_CRC\n    ma_uint16 actualCRC16;\n#endif\n    channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment);\n    for (i = 0; i < channelCount; ++i) {\n        if (!ma_dr_flac__seek_subframe(&pFlac->bs, &pFlac->currentFLACFrame, i)) {\n            return MA_ERROR;\n        }\n    }\n    if (!ma_dr_flac__seek_bits(&pFlac->bs, MA_DR_FLAC_CACHE_L1_BITS_REMAINING(&pFlac->bs) & 7)) {\n        return MA_ERROR;\n    }\n#ifndef MA_DR_FLAC_NO_CRC\n    actualCRC16 = ma_dr_flac__flush_crc16(&pFlac->bs);\n#endif\n    if (!ma_dr_flac__read_uint16(&pFlac->bs, 16, &desiredCRC16)) {\n        return MA_AT_END;\n    }\n#ifndef MA_DR_FLAC_NO_CRC\n    if (actualCRC16 != desiredCRC16) {\n        return MA_CRC_MISMATCH;\n    }\n#endif\n    return MA_SUCCESS;\n}\nstatic ma_bool32 ma_dr_flac__read_and_decode_next_flac_frame(ma_dr_flac* pFlac)\n{\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    for (;;) {\n        ma_result result;\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n        result = ma_dr_flac__decode_flac_frame(pFlac);\n        if (result != MA_SUCCESS) {\n            if (result == MA_CRC_MISMATCH) {\n                continue;\n            } else {\n                return MA_FALSE;\n            }\n        }\n        return MA_TRUE;\n    }\n}\nstatic void ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(ma_dr_flac* pFlac, ma_uint64* pFirstPCMFrame, ma_uint64* pLastPCMFrame)\n{\n    ma_uint64 firstPCMFrame;\n    ma_uint64 lastPCMFrame;\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    firstPCMFrame = pFlac->currentFLACFrame.header.pcmFrameNumber;\n    if (firstPCMFrame == 0) {\n        firstPCMFrame = ((ma_uint64)pFlac->currentFLACFrame.header.flacFrameNumber) * pFlac->maxBlockSizeInPCMFrames;\n    }\n    lastPCMFrame = firstPCMFrame + pFlac->currentFLACFrame.header.blockSizeInPCMFrames;\n    if (lastPCMFrame > 0) {\n        lastPCMFrame -= 1;\n    }\n    if (pFirstPCMFrame) {\n        *pFirstPCMFrame = firstPCMFrame;\n    }\n    if (pLastPCMFrame) {\n        *pLastPCMFrame = lastPCMFrame;\n    }\n}\nstatic ma_bool32 ma_dr_flac__seek_to_first_frame(ma_dr_flac* pFlac)\n{\n    ma_bool32 result;\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    result = ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes);\n    MA_DR_FLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame));\n    pFlac->currentPCMFrame = 0;\n    return result;\n}\nstatic MA_INLINE ma_result ma_dr_flac__seek_to_next_flac_frame(ma_dr_flac* pFlac)\n{\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    return ma_dr_flac__seek_flac_frame(pFlac);\n}\nstatic ma_uint64 ma_dr_flac__seek_forward_by_pcm_frames(ma_dr_flac* pFlac, ma_uint64 pcmFramesToSeek)\n{\n    ma_uint64 pcmFramesRead = 0;\n    while (pcmFramesToSeek > 0) {\n        if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) {\n                break;\n            }\n        } else {\n            if (pFlac->currentFLACFrame.pcmFramesRemaining > pcmFramesToSeek) {\n                pcmFramesRead   += pcmFramesToSeek;\n                pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)pcmFramesToSeek;\n                pcmFramesToSeek  = 0;\n            } else {\n                pcmFramesRead   += pFlac->currentFLACFrame.pcmFramesRemaining;\n                pcmFramesToSeek -= pFlac->currentFLACFrame.pcmFramesRemaining;\n                pFlac->currentFLACFrame.pcmFramesRemaining = 0;\n            }\n        }\n    }\n    pFlac->currentPCMFrame += pcmFramesRead;\n    return pcmFramesRead;\n}\nstatic ma_bool32 ma_dr_flac__seek_to_pcm_frame__brute_force(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex)\n{\n    ma_bool32 isMidFrame = MA_FALSE;\n    ma_uint64 runningPCMFrameCount;\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    if (pcmFrameIndex >= pFlac->currentPCMFrame) {\n        runningPCMFrameCount = pFlac->currentPCMFrame;\n        if (pFlac->currentPCMFrame == 0 && pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n                return MA_FALSE;\n            }\n        } else {\n            isMidFrame = MA_TRUE;\n        }\n    } else {\n        runningPCMFrameCount = 0;\n        if (!ma_dr_flac__seek_to_first_frame(pFlac)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n    }\n    for (;;) {\n        ma_uint64 pcmFrameCountInThisFLACFrame;\n        ma_uint64 firstPCMFrameInFLACFrame = 0;\n        ma_uint64 lastPCMFrameInFLACFrame = 0;\n        ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame);\n        pcmFrameCountInThisFLACFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1;\n        if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFLACFrame)) {\n            ma_uint64 pcmFramesToDecode = pcmFrameIndex - runningPCMFrameCount;\n            if (!isMidFrame) {\n                ma_result result = ma_dr_flac__decode_flac_frame(pFlac);\n                if (result == MA_SUCCESS) {\n                    return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode;\n                } else {\n                    if (result == MA_CRC_MISMATCH) {\n                        goto next_iteration;\n                    } else {\n                        return MA_FALSE;\n                    }\n                }\n            } else {\n                return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode;\n            }\n        } else {\n            if (!isMidFrame) {\n                ma_result result = ma_dr_flac__seek_to_next_flac_frame(pFlac);\n                if (result == MA_SUCCESS) {\n                    runningPCMFrameCount += pcmFrameCountInThisFLACFrame;\n                } else {\n                    if (result == MA_CRC_MISMATCH) {\n                        goto next_iteration;\n                    } else {\n                        return MA_FALSE;\n                    }\n                }\n            } else {\n                runningPCMFrameCount += pFlac->currentFLACFrame.pcmFramesRemaining;\n                pFlac->currentFLACFrame.pcmFramesRemaining = 0;\n                isMidFrame = MA_FALSE;\n            }\n            if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) {\n                return MA_TRUE;\n            }\n        }\n    next_iteration:\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n    }\n}\n#if !defined(MA_DR_FLAC_NO_CRC)\n#define MA_DR_FLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO 0.6f\nstatic ma_bool32 ma_dr_flac__seek_to_approximate_flac_frame_to_byte(ma_dr_flac* pFlac, ma_uint64 targetByte, ma_uint64 rangeLo, ma_uint64 rangeHi, ma_uint64* pLastSuccessfulSeekOffset)\n{\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    MA_DR_FLAC_ASSERT(pLastSuccessfulSeekOffset != NULL);\n    MA_DR_FLAC_ASSERT(targetByte >= rangeLo);\n    MA_DR_FLAC_ASSERT(targetByte <= rangeHi);\n    *pLastSuccessfulSeekOffset = pFlac->firstFLACFramePosInBytes;\n    for (;;) {\n        ma_uint64 lastTargetByte = targetByte;\n        if (!ma_dr_flac__seek_to_byte(&pFlac->bs, targetByte)) {\n            if (targetByte == 0) {\n                ma_dr_flac__seek_to_first_frame(pFlac);\n                return MA_FALSE;\n            }\n            targetByte = rangeLo + ((rangeHi - rangeLo)/2);\n            rangeHi = targetByte;\n        } else {\n            MA_DR_FLAC_ZERO_MEMORY(&pFlac->currentFLACFrame, sizeof(pFlac->currentFLACFrame));\n#if 1\n            if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) {\n                targetByte = rangeLo + ((rangeHi - rangeLo)/2);\n                rangeHi = targetByte;\n            } else {\n                break;\n            }\n#else\n            if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n                targetByte = rangeLo + ((rangeHi - rangeLo)/2);\n                rangeHi = targetByte;\n            } else {\n                break;\n            }\n#endif\n        }\n        if(targetByte == lastTargetByte) {\n            return MA_FALSE;\n        }\n    }\n    ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL);\n    MA_DR_FLAC_ASSERT(targetByte <= rangeHi);\n    *pLastSuccessfulSeekOffset = targetByte;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(ma_dr_flac* pFlac, ma_uint64 offset)\n{\n#if 0\n    if (ma_dr_flac__decode_flac_frame(pFlac) != MA_SUCCESS) {\n        if (ma_dr_flac__read_and_decode_next_flac_frame(pFlac) == MA_FALSE) {\n            return MA_FALSE;\n        }\n    }\n#endif\n    return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, offset) == offset;\n}\nstatic ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search_internal(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex, ma_uint64 byteRangeLo, ma_uint64 byteRangeHi)\n{\n    ma_uint64 targetByte;\n    ma_uint64 pcmRangeLo = pFlac->totalPCMFrameCount;\n    ma_uint64 pcmRangeHi = 0;\n    ma_uint64 lastSuccessfulSeekOffset = (ma_uint64)-1;\n    ma_uint64 closestSeekOffsetBeforeTargetPCMFrame = byteRangeLo;\n    ma_uint32 seekForwardThreshold = (pFlac->maxBlockSizeInPCMFrames != 0) ? pFlac->maxBlockSizeInPCMFrames*2 : 4096;\n    targetByte = byteRangeLo + (ma_uint64)(((ma_int64)((pcmFrameIndex - pFlac->currentPCMFrame) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * MA_DR_FLAC_BINARY_SEARCH_APPROX_COMPRESSION_RATIO);\n    if (targetByte > byteRangeHi) {\n        targetByte = byteRangeHi;\n    }\n    for (;;) {\n        if (ma_dr_flac__seek_to_approximate_flac_frame_to_byte(pFlac, targetByte, byteRangeLo, byteRangeHi, &lastSuccessfulSeekOffset)) {\n            ma_uint64 newPCMRangeLo;\n            ma_uint64 newPCMRangeHi;\n            ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &newPCMRangeLo, &newPCMRangeHi);\n            if (pcmRangeLo == newPCMRangeLo) {\n                if (!ma_dr_flac__seek_to_approximate_flac_frame_to_byte(pFlac, closestSeekOffsetBeforeTargetPCMFrame, closestSeekOffsetBeforeTargetPCMFrame, byteRangeHi, &lastSuccessfulSeekOffset)) {\n                    break;\n                }\n                if (ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) {\n                    return MA_TRUE;\n                } else {\n                    break;\n                }\n            }\n            pcmRangeLo = newPCMRangeLo;\n            pcmRangeHi = newPCMRangeHi;\n            if (pcmRangeLo <= pcmFrameIndex && pcmRangeHi >= pcmFrameIndex) {\n                if (ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame) ) {\n                    return MA_TRUE;\n                } else {\n                    break;\n                }\n            } else {\n                const float approxCompressionRatio = (ma_int64)(lastSuccessfulSeekOffset - pFlac->firstFLACFramePosInBytes) / ((ma_int64)(pcmRangeLo * pFlac->channels * pFlac->bitsPerSample)/8.0f);\n                if (pcmRangeLo > pcmFrameIndex) {\n                    byteRangeHi = lastSuccessfulSeekOffset;\n                    if (byteRangeLo > byteRangeHi) {\n                        byteRangeLo = byteRangeHi;\n                    }\n                    targetByte = byteRangeLo + ((byteRangeHi - byteRangeLo) / 2);\n                    if (targetByte < byteRangeLo) {\n                        targetByte = byteRangeLo;\n                    }\n                } else  {\n                    if ((pcmFrameIndex - pcmRangeLo) < seekForwardThreshold) {\n                        if (ma_dr_flac__decode_flac_frame_and_seek_forward_by_pcm_frames(pFlac, pcmFrameIndex - pFlac->currentPCMFrame)) {\n                            return MA_TRUE;\n                        } else {\n                            break;\n                        }\n                    } else {\n                        byteRangeLo = lastSuccessfulSeekOffset;\n                        if (byteRangeHi < byteRangeLo) {\n                            byteRangeHi = byteRangeLo;\n                        }\n                        targetByte = lastSuccessfulSeekOffset + (ma_uint64)(((ma_int64)((pcmFrameIndex-pcmRangeLo) * pFlac->channels * pFlac->bitsPerSample)/8.0f) * approxCompressionRatio);\n                        if (targetByte > byteRangeHi) {\n                            targetByte = byteRangeHi;\n                        }\n                        if (closestSeekOffsetBeforeTargetPCMFrame < lastSuccessfulSeekOffset) {\n                            closestSeekOffsetBeforeTargetPCMFrame = lastSuccessfulSeekOffset;\n                        }\n                    }\n                }\n            }\n        } else {\n            break;\n        }\n    }\n    ma_dr_flac__seek_to_first_frame(pFlac);\n    return MA_FALSE;\n}\nstatic ma_bool32 ma_dr_flac__seek_to_pcm_frame__binary_search(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex)\n{\n    ma_uint64 byteRangeLo;\n    ma_uint64 byteRangeHi;\n    ma_uint32 seekForwardThreshold = (pFlac->maxBlockSizeInPCMFrames != 0) ? pFlac->maxBlockSizeInPCMFrames*2 : 4096;\n    if (ma_dr_flac__seek_to_first_frame(pFlac) == MA_FALSE) {\n        return MA_FALSE;\n    }\n    if (pcmFrameIndex < seekForwardThreshold) {\n        return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFrameIndex) == pcmFrameIndex;\n    }\n    byteRangeLo = pFlac->firstFLACFramePosInBytes;\n    byteRangeHi = pFlac->firstFLACFramePosInBytes + (ma_uint64)((ma_int64)(pFlac->totalPCMFrameCount * pFlac->channels * pFlac->bitsPerSample)/8.0f);\n    return ma_dr_flac__seek_to_pcm_frame__binary_search_internal(pFlac, pcmFrameIndex, byteRangeLo, byteRangeHi);\n}\n#endif\nstatic ma_bool32 ma_dr_flac__seek_to_pcm_frame__seek_table(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex)\n{\n    ma_uint32 iClosestSeekpoint = 0;\n    ma_bool32 isMidFrame = MA_FALSE;\n    ma_uint64 runningPCMFrameCount;\n    ma_uint32 iSeekpoint;\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    if (pFlac->pSeekpoints == NULL || pFlac->seekpointCount == 0) {\n        return MA_FALSE;\n    }\n    if (pFlac->pSeekpoints[0].firstPCMFrame > pcmFrameIndex) {\n        return MA_FALSE;\n    }\n    for (iSeekpoint = 0; iSeekpoint < pFlac->seekpointCount; ++iSeekpoint) {\n        if (pFlac->pSeekpoints[iSeekpoint].firstPCMFrame >= pcmFrameIndex) {\n            break;\n        }\n        iClosestSeekpoint = iSeekpoint;\n    }\n    if (pFlac->pSeekpoints[iClosestSeekpoint].pcmFrameCount == 0 || pFlac->pSeekpoints[iClosestSeekpoint].pcmFrameCount > pFlac->maxBlockSizeInPCMFrames) {\n        return MA_FALSE;\n    }\n    if (pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame > pFlac->totalPCMFrameCount && pFlac->totalPCMFrameCount > 0) {\n        return MA_FALSE;\n    }\n#if !defined(MA_DR_FLAC_NO_CRC)\n    if (pFlac->totalPCMFrameCount > 0) {\n        ma_uint64 byteRangeLo;\n        ma_uint64 byteRangeHi;\n        byteRangeHi = pFlac->firstFLACFramePosInBytes + (ma_uint64)((ma_int64)(pFlac->totalPCMFrameCount * pFlac->channels * pFlac->bitsPerSample)/8.0f);\n        byteRangeLo = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset;\n        if (iClosestSeekpoint < pFlac->seekpointCount-1) {\n            ma_uint32 iNextSeekpoint = iClosestSeekpoint + 1;\n            if (pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset >= pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset || pFlac->pSeekpoints[iNextSeekpoint].pcmFrameCount == 0) {\n                return MA_FALSE;\n            }\n            if (pFlac->pSeekpoints[iNextSeekpoint].firstPCMFrame != (((ma_uint64)0xFFFFFFFF << 32) | 0xFFFFFFFF)) {\n                byteRangeHi = pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iNextSeekpoint].flacFrameOffset - 1;\n            }\n        }\n        if (ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset)) {\n            if (ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n                ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &pFlac->currentPCMFrame, NULL);\n                if (ma_dr_flac__seek_to_pcm_frame__binary_search_internal(pFlac, pcmFrameIndex, byteRangeLo, byteRangeHi)) {\n                    return MA_TRUE;\n                }\n            }\n        }\n    }\n#endif\n    if (pcmFrameIndex >= pFlac->currentPCMFrame && pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame <= pFlac->currentPCMFrame) {\n        runningPCMFrameCount = pFlac->currentPCMFrame;\n        if (pFlac->currentPCMFrame == 0 && pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n                return MA_FALSE;\n            }\n        } else {\n            isMidFrame = MA_TRUE;\n        }\n    } else {\n        runningPCMFrameCount = pFlac->pSeekpoints[iClosestSeekpoint].firstPCMFrame;\n        if (!ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes + pFlac->pSeekpoints[iClosestSeekpoint].flacFrameOffset)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n    }\n    for (;;) {\n        ma_uint64 pcmFrameCountInThisFLACFrame;\n        ma_uint64 firstPCMFrameInFLACFrame = 0;\n        ma_uint64 lastPCMFrameInFLACFrame = 0;\n        ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame);\n        pcmFrameCountInThisFLACFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1;\n        if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFLACFrame)) {\n            ma_uint64 pcmFramesToDecode = pcmFrameIndex - runningPCMFrameCount;\n            if (!isMidFrame) {\n                ma_result result = ma_dr_flac__decode_flac_frame(pFlac);\n                if (result == MA_SUCCESS) {\n                    return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode;\n                } else {\n                    if (result == MA_CRC_MISMATCH) {\n                        goto next_iteration;\n                    } else {\n                        return MA_FALSE;\n                    }\n                }\n            } else {\n                return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode;\n            }\n        } else {\n            if (!isMidFrame) {\n                ma_result result = ma_dr_flac__seek_to_next_flac_frame(pFlac);\n                if (result == MA_SUCCESS) {\n                    runningPCMFrameCount += pcmFrameCountInThisFLACFrame;\n                } else {\n                    if (result == MA_CRC_MISMATCH) {\n                        goto next_iteration;\n                    } else {\n                        return MA_FALSE;\n                    }\n                }\n            } else {\n                runningPCMFrameCount += pFlac->currentFLACFrame.pcmFramesRemaining;\n                pFlac->currentFLACFrame.pcmFramesRemaining = 0;\n                isMidFrame = MA_FALSE;\n            }\n            if (pcmFrameIndex == pFlac->totalPCMFrameCount && runningPCMFrameCount == pFlac->totalPCMFrameCount) {\n                return MA_TRUE;\n            }\n        }\n    next_iteration:\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n    }\n}\n#ifndef MA_DR_FLAC_NO_OGG\ntypedef struct\n{\n    ma_uint8 capturePattern[4];\n    ma_uint8 structureVersion;\n    ma_uint8 headerType;\n    ma_uint64 granulePosition;\n    ma_uint32 serialNumber;\n    ma_uint32 sequenceNumber;\n    ma_uint32 checksum;\n    ma_uint8 segmentCount;\n    ma_uint8 segmentTable[255];\n} ma_dr_flac_ogg_page_header;\n#endif\ntypedef struct\n{\n    ma_dr_flac_read_proc onRead;\n    ma_dr_flac_seek_proc onSeek;\n    ma_dr_flac_tell_proc onTell;\n    ma_dr_flac_meta_proc onMeta;\n    ma_dr_flac_container container;\n    void* pUserData;\n    void* pUserDataMD;\n    ma_uint32 sampleRate;\n    ma_uint8  channels;\n    ma_uint8  bitsPerSample;\n    ma_uint64 totalPCMFrameCount;\n    ma_uint16 maxBlockSizeInPCMFrames;\n    ma_uint64 runningFilePos;\n    ma_bool32 hasStreamInfoBlock;\n    ma_bool32 hasMetadataBlocks;\n    ma_dr_flac_bs bs;\n    ma_dr_flac_frame_header firstFrameHeader;\n#ifndef MA_DR_FLAC_NO_OGG\n    ma_uint32 oggSerial;\n    ma_uint64 oggFirstBytePos;\n    ma_dr_flac_ogg_page_header oggBosHeader;\n#endif\n} ma_dr_flac_init_info;\nstatic MA_INLINE void ma_dr_flac__decode_block_header(ma_uint32 blockHeader, ma_uint8* isLastBlock, ma_uint8* blockType, ma_uint32* blockSize)\n{\n    blockHeader = ma_dr_flac__be2host_32(blockHeader);\n    *isLastBlock = (ma_uint8)((blockHeader & 0x80000000UL) >> 31);\n    *blockType   = (ma_uint8)((blockHeader & 0x7F000000UL) >> 24);\n    *blockSize   =                (blockHeader & 0x00FFFFFFUL);\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac__read_and_decode_block_header(ma_dr_flac_read_proc onRead, void* pUserData, ma_uint8* isLastBlock, ma_uint8* blockType, ma_uint32* blockSize)\n{\n    ma_uint32 blockHeader;\n    *blockSize = 0;\n    if (onRead(pUserData, &blockHeader, 4) != 4) {\n        return MA_FALSE;\n    }\n    ma_dr_flac__decode_block_header(blockHeader, isLastBlock, blockType, blockSize);\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__read_streaminfo(ma_dr_flac_read_proc onRead, void* pUserData, ma_dr_flac_streaminfo* pStreamInfo)\n{\n    ma_uint32 blockSizes;\n    ma_uint64 frameSizes = 0;\n    ma_uint64 importantProps;\n    ma_uint8 md5[16];\n    if (onRead(pUserData, &blockSizes, 4) != 4) {\n        return MA_FALSE;\n    }\n    if (onRead(pUserData, &frameSizes, 6) != 6) {\n        return MA_FALSE;\n    }\n    if (onRead(pUserData, &importantProps, 8) != 8) {\n        return MA_FALSE;\n    }\n    if (onRead(pUserData, md5, sizeof(md5)) != sizeof(md5)) {\n        return MA_FALSE;\n    }\n    blockSizes     = ma_dr_flac__be2host_32(blockSizes);\n    frameSizes     = ma_dr_flac__be2host_64(frameSizes);\n    importantProps = ma_dr_flac__be2host_64(importantProps);\n    pStreamInfo->minBlockSizeInPCMFrames = (ma_uint16)((blockSizes & 0xFFFF0000) >> 16);\n    pStreamInfo->maxBlockSizeInPCMFrames = (ma_uint16) (blockSizes & 0x0000FFFF);\n    pStreamInfo->minFrameSizeInPCMFrames = (ma_uint32)((frameSizes     &  (((ma_uint64)0x00FFFFFF << 16) << 24)) >> 40);\n    pStreamInfo->maxFrameSizeInPCMFrames = (ma_uint32)((frameSizes     &  (((ma_uint64)0x00FFFFFF << 16) <<  0)) >> 16);\n    pStreamInfo->sampleRate              = (ma_uint32)((importantProps &  (((ma_uint64)0x000FFFFF << 16) << 28)) >> 44);\n    pStreamInfo->channels                = (ma_uint8 )((importantProps &  (((ma_uint64)0x0000000E << 16) << 24)) >> 41) + 1;\n    pStreamInfo->bitsPerSample           = (ma_uint8 )((importantProps &  (((ma_uint64)0x0000001F << 16) << 20)) >> 36) + 1;\n    pStreamInfo->totalPCMFrameCount      =                ((importantProps & ((((ma_uint64)0x0000000F << 16) << 16) | 0xFFFFFFFF)));\n    MA_DR_FLAC_COPY_MEMORY(pStreamInfo->md5, md5, sizeof(md5));\n    return MA_TRUE;\n}\nstatic void* ma_dr_flac__malloc_default(size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_FLAC_MALLOC(sz);\n}\nstatic void* ma_dr_flac__realloc_default(void* p, size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_FLAC_REALLOC(p, sz);\n}\nstatic void ma_dr_flac__free_default(void* p, void* pUserData)\n{\n    (void)pUserData;\n    MA_DR_FLAC_FREE(p);\n}\nstatic void* ma_dr_flac__malloc_from_callbacks(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onMalloc != NULL) {\n        return pAllocationCallbacks->onMalloc(sz, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(NULL, sz, pAllocationCallbacks->pUserData);\n    }\n    return NULL;\n}\nstatic void* ma_dr_flac__realloc_from_callbacks(void* p, size_t szNew, size_t szOld, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(p, szNew, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onMalloc != NULL && pAllocationCallbacks->onFree != NULL) {\n        void* p2;\n        p2 = pAllocationCallbacks->onMalloc(szNew, pAllocationCallbacks->pUserData);\n        if (p2 == NULL) {\n            return NULL;\n        }\n        if (p != NULL) {\n            MA_DR_FLAC_COPY_MEMORY(p2, p, szOld);\n            pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n        }\n        return p2;\n    }\n    return NULL;\n}\nstatic void ma_dr_flac__free_from_callbacks(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (p == NULL || pAllocationCallbacks == NULL) {\n        return;\n    }\n    if (pAllocationCallbacks->onFree != NULL) {\n        pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n    }\n}\nstatic ma_bool32 ma_dr_flac__read_and_decode_metadata(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_uint64* pFirstFramePos, ma_uint64* pSeektablePos, ma_uint32* pSeekpointCount, ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_uint64 runningFilePos = 42;\n    ma_uint64 seektablePos   = 0;\n    ma_uint32 seektableSize  = 0;\n    (void)onTell;\n    for (;;) {\n        ma_dr_flac_metadata metadata;\n        ma_uint8 isLastBlock = 0;\n        ma_uint8 blockType = 0;\n        ma_uint32 blockSize;\n        if (ma_dr_flac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize) == MA_FALSE) {\n            return MA_FALSE;\n        }\n        runningFilePos += 4;\n        metadata.type = blockType;\n        metadata.rawDataSize = 0;\n        metadata.rawDataOffset = runningFilePos;\n        metadata.pRawData = NULL;\n        switch (blockType)\n        {\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_APPLICATION:\n            {\n                if (blockSize < 4) {\n                    return MA_FALSE;\n                }\n                if (onMeta) {\n                    void* pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks);\n                    if (pRawData == NULL) {\n                        return MA_FALSE;\n                    }\n                    if (onRead(pUserData, pRawData, blockSize) != blockSize) {\n                        ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                        return MA_FALSE;\n                    }\n                    metadata.pRawData = pRawData;\n                    metadata.rawDataSize = blockSize;\n                    metadata.data.application.id       = ma_dr_flac__be2host_32(*(ma_uint32*)pRawData);\n                    metadata.data.application.pData    = (const void*)((ma_uint8*)pRawData + sizeof(ma_uint32));\n                    metadata.data.application.dataSize = blockSize - sizeof(ma_uint32);\n                    onMeta(pUserDataMD, &metadata);\n                    ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_SEEKTABLE:\n            {\n                seektablePos  = runningFilePos;\n                seektableSize = blockSize;\n                if (onMeta) {\n                    ma_uint32 seekpointCount;\n                    ma_uint32 iSeekpoint;\n                    void* pRawData;\n                    seekpointCount = blockSize/MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES;\n                    pRawData = ma_dr_flac__malloc_from_callbacks(seekpointCount * sizeof(ma_dr_flac_seekpoint), pAllocationCallbacks);\n                    if (pRawData == NULL) {\n                        return MA_FALSE;\n                    }\n                    for (iSeekpoint = 0; iSeekpoint < seekpointCount; ++iSeekpoint) {\n                        ma_dr_flac_seekpoint* pSeekpoint = (ma_dr_flac_seekpoint*)pRawData + iSeekpoint;\n                        if (onRead(pUserData, pSeekpoint, MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) != MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) {\n                            ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                            return MA_FALSE;\n                        }\n                        pSeekpoint->firstPCMFrame   = ma_dr_flac__be2host_64(pSeekpoint->firstPCMFrame);\n                        pSeekpoint->flacFrameOffset = ma_dr_flac__be2host_64(pSeekpoint->flacFrameOffset);\n                        pSeekpoint->pcmFrameCount   = ma_dr_flac__be2host_16(pSeekpoint->pcmFrameCount);\n                    }\n                    metadata.pRawData = pRawData;\n                    metadata.rawDataSize = blockSize;\n                    metadata.data.seektable.seekpointCount = seekpointCount;\n                    metadata.data.seektable.pSeekpoints = (const ma_dr_flac_seekpoint*)pRawData;\n                    onMeta(pUserDataMD, &metadata);\n                    ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_VORBIS_COMMENT:\n            {\n                if (blockSize < 8) {\n                    return MA_FALSE;\n                }\n                if (onMeta) {\n                    void* pRawData;\n                    const char* pRunningData;\n                    const char* pRunningDataEnd;\n                    ma_uint32 i;\n                    pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks);\n                    if (pRawData == NULL) {\n                        return MA_FALSE;\n                    }\n                    if (onRead(pUserData, pRawData, blockSize) != blockSize) {\n                        ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                        return MA_FALSE;\n                    }\n                    metadata.pRawData = pRawData;\n                    metadata.rawDataSize = blockSize;\n                    pRunningData    = (const char*)pRawData;\n                    pRunningDataEnd = (const char*)pRawData + blockSize;\n                    metadata.data.vorbis_comment.vendorLength = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4;\n                    if ((pRunningDataEnd - pRunningData) - 4 < (ma_int64)metadata.data.vorbis_comment.vendorLength) {\n                        ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                        return MA_FALSE;\n                    }\n                    metadata.data.vorbis_comment.vendor       = pRunningData;                                            pRunningData += metadata.data.vorbis_comment.vendorLength;\n                    metadata.data.vorbis_comment.commentCount = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4;\n                    if ((pRunningDataEnd - pRunningData) / sizeof(ma_uint32) < metadata.data.vorbis_comment.commentCount) {\n                        ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                        return MA_FALSE;\n                    }\n                    metadata.data.vorbis_comment.pComments    = pRunningData;\n                    for (i = 0; i < metadata.data.vorbis_comment.commentCount; ++i) {\n                        ma_uint32 commentLength;\n                        if (pRunningDataEnd - pRunningData < 4) {\n                            ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                            return MA_FALSE;\n                        }\n                        commentLength = ma_dr_flac__le2host_32_ptr_unaligned(pRunningData); pRunningData += 4;\n                        if (pRunningDataEnd - pRunningData < (ma_int64)commentLength) {\n                            ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                            return MA_FALSE;\n                        }\n                        pRunningData += commentLength;\n                    }\n                    onMeta(pUserDataMD, &metadata);\n                    ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_CUESHEET:\n            {\n                if (blockSize < 396) {\n                    return MA_FALSE;\n                }\n                if (onMeta) {\n                    void* pRawData;\n                    const char* pRunningData;\n                    const char* pRunningDataEnd;\n                    size_t bufferSize;\n                    ma_uint8 iTrack;\n                    ma_uint8 iIndex;\n                    void* pTrackData;\n                    pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks);\n                    if (pRawData == NULL) {\n                        return MA_FALSE;\n                    }\n                    if (onRead(pUserData, pRawData, blockSize) != blockSize) {\n                        ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                        return MA_FALSE;\n                    }\n                    metadata.pRawData = pRawData;\n                    metadata.rawDataSize = blockSize;\n                    pRunningData    = (const char*)pRawData;\n                    pRunningDataEnd = (const char*)pRawData + blockSize;\n                    MA_DR_FLAC_COPY_MEMORY(metadata.data.cuesheet.catalog, pRunningData, 128);                              pRunningData += 128;\n                    metadata.data.cuesheet.leadInSampleCount = ma_dr_flac__be2host_64(*(const ma_uint64*)pRunningData); pRunningData += 8;\n                    metadata.data.cuesheet.isCD              = (pRunningData[0] & 0x80) != 0;                           pRunningData += 259;\n                    metadata.data.cuesheet.trackCount        = pRunningData[0];                                         pRunningData += 1;\n                    metadata.data.cuesheet.pTrackData        = NULL;\n                    {\n                        const char* pRunningDataSaved = pRunningData;\n                        bufferSize = metadata.data.cuesheet.trackCount * MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES;\n                        for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) {\n                            ma_uint8 indexCount;\n                            ma_uint32 indexPointSize;\n                            if (pRunningDataEnd - pRunningData < MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES) {\n                                ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                                return MA_FALSE;\n                            }\n                            pRunningData += 35;\n                            indexCount = pRunningData[0];\n                            pRunningData += 1;\n                            bufferSize += indexCount * sizeof(ma_dr_flac_cuesheet_track_index);\n                            indexPointSize = indexCount * MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES;\n                            if (pRunningDataEnd - pRunningData < (ma_int64)indexPointSize) {\n                                ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                                return MA_FALSE;\n                            }\n                            pRunningData += indexPointSize;\n                        }\n                        pRunningData = pRunningDataSaved;\n                    }\n                    {\n                        char* pRunningTrackData;\n                        pTrackData = ma_dr_flac__malloc_from_callbacks(bufferSize, pAllocationCallbacks);\n                        if (pTrackData == NULL) {\n                            ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                            return MA_FALSE;\n                        }\n                        pRunningTrackData = (char*)pTrackData;\n                        for (iTrack = 0; iTrack < metadata.data.cuesheet.trackCount; ++iTrack) {\n                            ma_uint8 indexCount;\n                            MA_DR_FLAC_COPY_MEMORY(pRunningTrackData, pRunningData, MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES);\n                            pRunningData      += MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES-1;\n                            pRunningTrackData += MA_DR_FLAC_CUESHEET_TRACK_SIZE_IN_BYTES-1;\n                            indexCount = pRunningData[0];\n                            pRunningData      += 1;\n                            pRunningTrackData += 1;\n                            for (iIndex = 0; iIndex < indexCount; ++iIndex) {\n                                ma_dr_flac_cuesheet_track_index* pTrackIndex = (ma_dr_flac_cuesheet_track_index*)pRunningTrackData;\n                                MA_DR_FLAC_COPY_MEMORY(pRunningTrackData, pRunningData, MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES);\n                                pRunningData      += MA_DR_FLAC_CUESHEET_TRACK_INDEX_SIZE_IN_BYTES;\n                                pRunningTrackData += sizeof(ma_dr_flac_cuesheet_track_index);\n                                pTrackIndex->offset = ma_dr_flac__be2host_64(pTrackIndex->offset);\n                            }\n                        }\n                        metadata.data.cuesheet.pTrackData = pTrackData;\n                    }\n                    ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                    pRawData = NULL;\n                    onMeta(pUserDataMD, &metadata);\n                    ma_dr_flac__free_from_callbacks(pTrackData, pAllocationCallbacks);\n                    pTrackData = NULL;\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_PICTURE:\n            {\n                if (blockSize < 32) {\n                    return MA_FALSE;\n                }\n                if (onMeta) {\n                    ma_bool32 result = MA_TRUE;\n                    ma_uint32 blockSizeRemaining = blockSize;\n                    char* pMime = NULL;\n                    char* pDescription = NULL;\n                    void* pPictureData = NULL;\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.type, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.type = ma_dr_flac__be2host_32(metadata.data.picture.type);\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.mimeLength, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.mimeLength = ma_dr_flac__be2host_32(metadata.data.picture.mimeLength);\n                    pMime = (char*)ma_dr_flac__malloc_from_callbacks(metadata.data.picture.mimeLength + 1, pAllocationCallbacks);\n                    if (pMime == NULL) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    if (blockSizeRemaining < metadata.data.picture.mimeLength || onRead(pUserData, pMime, metadata.data.picture.mimeLength) != metadata.data.picture.mimeLength) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= metadata.data.picture.mimeLength;\n                    pMime[metadata.data.picture.mimeLength] = '\\0';\n                    metadata.data.picture.mime = (const char*)pMime;\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.descriptionLength, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.descriptionLength = ma_dr_flac__be2host_32(metadata.data.picture.descriptionLength);\n                    pDescription = (char*)ma_dr_flac__malloc_from_callbacks(metadata.data.picture.descriptionLength + 1, pAllocationCallbacks);\n                    if (pDescription == NULL) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    if (blockSizeRemaining < metadata.data.picture.descriptionLength || onRead(pUserData, pDescription, metadata.data.picture.descriptionLength) != metadata.data.picture.descriptionLength) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= metadata.data.picture.descriptionLength;\n                    pDescription[metadata.data.picture.descriptionLength] = '\\0';\n                    metadata.data.picture.description = (const char*)pDescription;\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.width, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.width = ma_dr_flac__be2host_32(metadata.data.picture.width);\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.height, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.height = ma_dr_flac__be2host_32(metadata.data.picture.height);\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.colorDepth, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.colorDepth = ma_dr_flac__be2host_32(metadata.data.picture.colorDepth);\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.indexColorCount, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.indexColorCount = ma_dr_flac__be2host_32(metadata.data.picture.indexColorCount);\n                    if (blockSizeRemaining < 4 || onRead(pUserData, &metadata.data.picture.pictureDataSize, 4) != 4) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    blockSizeRemaining -= 4;\n                    metadata.data.picture.pictureDataSize = ma_dr_flac__be2host_32(metadata.data.picture.pictureDataSize);\n                    if (blockSizeRemaining < metadata.data.picture.pictureDataSize) {\n                        result = MA_FALSE;\n                        goto done_flac;\n                    }\n                    metadata.data.picture.pictureDataOffset = runningFilePos + (blockSize - blockSizeRemaining);\n                #ifndef MA_DR_FLAC_NO_PICTURE_METADATA_MALLOC\n                    pPictureData = ma_dr_flac__malloc_from_callbacks(metadata.data.picture.pictureDataSize, pAllocationCallbacks);\n                    if (pPictureData != NULL) {\n                        if (onRead(pUserData, pPictureData, metadata.data.picture.pictureDataSize) != metadata.data.picture.pictureDataSize) {\n                            result = MA_FALSE;\n                            goto done_flac;\n                        }\n                    } else\n                #endif\n                    {\n                        if (!onSeek(pUserData, metadata.data.picture.pictureDataSize, MA_DR_FLAC_SEEK_CUR)) {\n                            result = MA_FALSE;\n                            goto done_flac;\n                        }\n                    }\n                    blockSizeRemaining -= metadata.data.picture.pictureDataSize;\n                    (void)blockSizeRemaining;\n                    metadata.data.picture.pPictureData = (const ma_uint8*)pPictureData;\n                    if (metadata.data.picture.pictureDataOffset != 0 || metadata.data.picture.pPictureData != NULL) {\n                        onMeta(pUserDataMD, &metadata);\n                    } else {\n                    }\n                done_flac:\n                    ma_dr_flac__free_from_callbacks(pMime,        pAllocationCallbacks);\n                    ma_dr_flac__free_from_callbacks(pDescription, pAllocationCallbacks);\n                    ma_dr_flac__free_from_callbacks(pPictureData, pAllocationCallbacks);\n                    if (result != MA_TRUE) {\n                        return MA_FALSE;\n                    }\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_PADDING:\n            {\n                if (onMeta) {\n                    metadata.data.padding.unused = 0;\n                    if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) {\n                        isLastBlock = MA_TRUE;\n                    } else {\n                        onMeta(pUserDataMD, &metadata);\n                    }\n                }\n            } break;\n            case MA_DR_FLAC_METADATA_BLOCK_TYPE_INVALID:\n            {\n                if (onMeta) {\n                    if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) {\n                        isLastBlock = MA_TRUE;\n                    }\n                }\n            } break;\n            default:\n            {\n                if (onMeta) {\n                    void* pRawData = ma_dr_flac__malloc_from_callbacks(blockSize, pAllocationCallbacks);\n                    if (pRawData != NULL) {\n                        if (onRead(pUserData, pRawData, blockSize) != blockSize) {\n                            ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                            return MA_FALSE;\n                        }\n                    } else {\n                        if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) {\n                            return MA_FALSE;\n                        }\n                    }\n                    metadata.pRawData = pRawData;\n                    metadata.rawDataSize = blockSize;\n                    onMeta(pUserDataMD, &metadata);\n                    ma_dr_flac__free_from_callbacks(pRawData, pAllocationCallbacks);\n                }\n            } break;\n        }\n        if (onMeta == NULL && blockSize > 0) {\n            if (!onSeek(pUserData, blockSize, MA_DR_FLAC_SEEK_CUR)) {\n                isLastBlock = MA_TRUE;\n            }\n        }\n        runningFilePos += blockSize;\n        if (isLastBlock) {\n            break;\n        }\n    }\n    *pSeektablePos   = seektablePos;\n    *pSeekpointCount = seektableSize / MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES;\n    *pFirstFramePos  = runningFilePos;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__init_private__native(ma_dr_flac_init_info* pInit, ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_bool32 relaxed)\n{\n    ma_uint8 isLastBlock;\n    ma_uint8 blockType;\n    ma_uint32 blockSize;\n    (void)onSeek;\n    pInit->container = ma_dr_flac_container_native;\n    if (!ma_dr_flac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) {\n        return MA_FALSE;\n    }\n    if (blockType != MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) {\n        if (!relaxed) {\n            return MA_FALSE;\n        } else {\n            pInit->hasStreamInfoBlock = MA_FALSE;\n            pInit->hasMetadataBlocks  = MA_FALSE;\n            if (!ma_dr_flac__read_next_flac_frame_header(&pInit->bs, 0, &pInit->firstFrameHeader)) {\n                return MA_FALSE;\n            }\n            if (pInit->firstFrameHeader.bitsPerSample == 0) {\n                return MA_FALSE;\n            }\n            pInit->sampleRate              = pInit->firstFrameHeader.sampleRate;\n            pInit->channels                = ma_dr_flac__get_channel_count_from_channel_assignment(pInit->firstFrameHeader.channelAssignment);\n            pInit->bitsPerSample           = pInit->firstFrameHeader.bitsPerSample;\n            pInit->maxBlockSizeInPCMFrames = 65535;\n            return MA_TRUE;\n        }\n    } else {\n        ma_dr_flac_streaminfo streaminfo;\n        if (!ma_dr_flac__read_streaminfo(onRead, pUserData, &streaminfo)) {\n            return MA_FALSE;\n        }\n        pInit->hasStreamInfoBlock      = MA_TRUE;\n        pInit->sampleRate              = streaminfo.sampleRate;\n        pInit->channels                = streaminfo.channels;\n        pInit->bitsPerSample           = streaminfo.bitsPerSample;\n        pInit->totalPCMFrameCount      = streaminfo.totalPCMFrameCount;\n        pInit->maxBlockSizeInPCMFrames = streaminfo.maxBlockSizeInPCMFrames;\n        pInit->hasMetadataBlocks       = !isLastBlock;\n        if (onMeta) {\n            ma_dr_flac_metadata metadata;\n            metadata.type = MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO;\n            metadata.pRawData = NULL;\n            metadata.rawDataSize = 0;\n            metadata.data.streaminfo = streaminfo;\n            onMeta(pUserDataMD, &metadata);\n        }\n        return MA_TRUE;\n    }\n}\n#ifndef MA_DR_FLAC_NO_OGG\n#define MA_DR_FLAC_OGG_MAX_PAGE_SIZE            65307\n#define MA_DR_FLAC_OGG_CAPTURE_PATTERN_CRC32    1605413199\ntypedef enum\n{\n    ma_dr_flac_ogg_recover_on_crc_mismatch,\n    ma_dr_flac_ogg_fail_on_crc_mismatch\n} ma_dr_flac_ogg_crc_mismatch_recovery;\n#ifndef MA_DR_FLAC_NO_CRC\nstatic ma_uint32 ma_dr_flac__crc32_table[] = {\n    0x00000000L, 0x04C11DB7L, 0x09823B6EL, 0x0D4326D9L,\n    0x130476DCL, 0x17C56B6BL, 0x1A864DB2L, 0x1E475005L,\n    0x2608EDB8L, 0x22C9F00FL, 0x2F8AD6D6L, 0x2B4BCB61L,\n    0x350C9B64L, 0x31CD86D3L, 0x3C8EA00AL, 0x384FBDBDL,\n    0x4C11DB70L, 0x48D0C6C7L, 0x4593E01EL, 0x4152FDA9L,\n    0x5F15ADACL, 0x5BD4B01BL, 0x569796C2L, 0x52568B75L,\n    0x6A1936C8L, 0x6ED82B7FL, 0x639B0DA6L, 0x675A1011L,\n    0x791D4014L, 0x7DDC5DA3L, 0x709F7B7AL, 0x745E66CDL,\n    0x9823B6E0L, 0x9CE2AB57L, 0x91A18D8EL, 0x95609039L,\n    0x8B27C03CL, 0x8FE6DD8BL, 0x82A5FB52L, 0x8664E6E5L,\n    0xBE2B5B58L, 0xBAEA46EFL, 0xB7A96036L, 0xB3687D81L,\n    0xAD2F2D84L, 0xA9EE3033L, 0xA4AD16EAL, 0xA06C0B5DL,\n    0xD4326D90L, 0xD0F37027L, 0xDDB056FEL, 0xD9714B49L,\n    0xC7361B4CL, 0xC3F706FBL, 0xCEB42022L, 0xCA753D95L,\n    0xF23A8028L, 0xF6FB9D9FL, 0xFBB8BB46L, 0xFF79A6F1L,\n    0xE13EF6F4L, 0xE5FFEB43L, 0xE8BCCD9AL, 0xEC7DD02DL,\n    0x34867077L, 0x30476DC0L, 0x3D044B19L, 0x39C556AEL,\n    0x278206ABL, 0x23431B1CL, 0x2E003DC5L, 0x2AC12072L,\n    0x128E9DCFL, 0x164F8078L, 0x1B0CA6A1L, 0x1FCDBB16L,\n    0x018AEB13L, 0x054BF6A4L, 0x0808D07DL, 0x0CC9CDCAL,\n    0x7897AB07L, 0x7C56B6B0L, 0x71159069L, 0x75D48DDEL,\n    0x6B93DDDBL, 0x6F52C06CL, 0x6211E6B5L, 0x66D0FB02L,\n    0x5E9F46BFL, 0x5A5E5B08L, 0x571D7DD1L, 0x53DC6066L,\n    0x4D9B3063L, 0x495A2DD4L, 0x44190B0DL, 0x40D816BAL,\n    0xACA5C697L, 0xA864DB20L, 0xA527FDF9L, 0xA1E6E04EL,\n    0xBFA1B04BL, 0xBB60ADFCL, 0xB6238B25L, 0xB2E29692L,\n    0x8AAD2B2FL, 0x8E6C3698L, 0x832F1041L, 0x87EE0DF6L,\n    0x99A95DF3L, 0x9D684044L, 0x902B669DL, 0x94EA7B2AL,\n    0xE0B41DE7L, 0xE4750050L, 0xE9362689L, 0xEDF73B3EL,\n    0xF3B06B3BL, 0xF771768CL, 0xFA325055L, 0xFEF34DE2L,\n    0xC6BCF05FL, 0xC27DEDE8L, 0xCF3ECB31L, 0xCBFFD686L,\n    0xD5B88683L, 0xD1799B34L, 0xDC3ABDEDL, 0xD8FBA05AL,\n    0x690CE0EEL, 0x6DCDFD59L, 0x608EDB80L, 0x644FC637L,\n    0x7A089632L, 0x7EC98B85L, 0x738AAD5CL, 0x774BB0EBL,\n    0x4F040D56L, 0x4BC510E1L, 0x46863638L, 0x42472B8FL,\n    0x5C007B8AL, 0x58C1663DL, 0x558240E4L, 0x51435D53L,\n    0x251D3B9EL, 0x21DC2629L, 0x2C9F00F0L, 0x285E1D47L,\n    0x36194D42L, 0x32D850F5L, 0x3F9B762CL, 0x3B5A6B9BL,\n    0x0315D626L, 0x07D4CB91L, 0x0A97ED48L, 0x0E56F0FFL,\n    0x1011A0FAL, 0x14D0BD4DL, 0x19939B94L, 0x1D528623L,\n    0xF12F560EL, 0xF5EE4BB9L, 0xF8AD6D60L, 0xFC6C70D7L,\n    0xE22B20D2L, 0xE6EA3D65L, 0xEBA91BBCL, 0xEF68060BL,\n    0xD727BBB6L, 0xD3E6A601L, 0xDEA580D8L, 0xDA649D6FL,\n    0xC423CD6AL, 0xC0E2D0DDL, 0xCDA1F604L, 0xC960EBB3L,\n    0xBD3E8D7EL, 0xB9FF90C9L, 0xB4BCB610L, 0xB07DABA7L,\n    0xAE3AFBA2L, 0xAAFBE615L, 0xA7B8C0CCL, 0xA379DD7BL,\n    0x9B3660C6L, 0x9FF77D71L, 0x92B45BA8L, 0x9675461FL,\n    0x8832161AL, 0x8CF30BADL, 0x81B02D74L, 0x857130C3L,\n    0x5D8A9099L, 0x594B8D2EL, 0x5408ABF7L, 0x50C9B640L,\n    0x4E8EE645L, 0x4A4FFBF2L, 0x470CDD2BL, 0x43CDC09CL,\n    0x7B827D21L, 0x7F436096L, 0x7200464FL, 0x76C15BF8L,\n    0x68860BFDL, 0x6C47164AL, 0x61043093L, 0x65C52D24L,\n    0x119B4BE9L, 0x155A565EL, 0x18197087L, 0x1CD86D30L,\n    0x029F3D35L, 0x065E2082L, 0x0B1D065BL, 0x0FDC1BECL,\n    0x3793A651L, 0x3352BBE6L, 0x3E119D3FL, 0x3AD08088L,\n    0x2497D08DL, 0x2056CD3AL, 0x2D15EBE3L, 0x29D4F654L,\n    0xC5A92679L, 0xC1683BCEL, 0xCC2B1D17L, 0xC8EA00A0L,\n    0xD6AD50A5L, 0xD26C4D12L, 0xDF2F6BCBL, 0xDBEE767CL,\n    0xE3A1CBC1L, 0xE760D676L, 0xEA23F0AFL, 0xEEE2ED18L,\n    0xF0A5BD1DL, 0xF464A0AAL, 0xF9278673L, 0xFDE69BC4L,\n    0x89B8FD09L, 0x8D79E0BEL, 0x803AC667L, 0x84FBDBD0L,\n    0x9ABC8BD5L, 0x9E7D9662L, 0x933EB0BBL, 0x97FFAD0CL,\n    0xAFB010B1L, 0xAB710D06L, 0xA6322BDFL, 0xA2F33668L,\n    0xBCB4666DL, 0xB8757BDAL, 0xB5365D03L, 0xB1F740B4L\n};\n#endif\nstatic MA_INLINE ma_uint32 ma_dr_flac_crc32_byte(ma_uint32 crc32, ma_uint8 data)\n{\n#ifndef MA_DR_FLAC_NO_CRC\n    return (crc32 << 8) ^ ma_dr_flac__crc32_table[(ma_uint8)((crc32 >> 24) & 0xFF) ^ data];\n#else\n    (void)data;\n    return crc32;\n#endif\n}\n#if 0\nstatic MA_INLINE ma_uint32 ma_dr_flac_crc32_uint32(ma_uint32 crc32, ma_uint32 data)\n{\n    crc32 = ma_dr_flac_crc32_byte(crc32, (ma_uint8)((data >> 24) & 0xFF));\n    crc32 = ma_dr_flac_crc32_byte(crc32, (ma_uint8)((data >> 16) & 0xFF));\n    crc32 = ma_dr_flac_crc32_byte(crc32, (ma_uint8)((data >>  8) & 0xFF));\n    crc32 = ma_dr_flac_crc32_byte(crc32, (ma_uint8)((data >>  0) & 0xFF));\n    return crc32;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac_crc32_uint64(ma_uint32 crc32, ma_uint64 data)\n{\n    crc32 = ma_dr_flac_crc32_uint32(crc32, (ma_uint32)((data >> 32) & 0xFFFFFFFF));\n    crc32 = ma_dr_flac_crc32_uint32(crc32, (ma_uint32)((data >>  0) & 0xFFFFFFFF));\n    return crc32;\n}\n#endif\nstatic MA_INLINE ma_uint32 ma_dr_flac_crc32_buffer(ma_uint32 crc32, ma_uint8* pData, ma_uint32 dataSize)\n{\n    ma_uint32 i;\n    for (i = 0; i < dataSize; ++i) {\n        crc32 = ma_dr_flac_crc32_byte(crc32, pData[i]);\n    }\n    return crc32;\n}\nstatic MA_INLINE ma_bool32 ma_dr_flac_ogg__is_capture_pattern(ma_uint8 pattern[4])\n{\n    return pattern[0] == 'O' && pattern[1] == 'g' && pattern[2] == 'g' && pattern[3] == 'S';\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac_ogg__get_page_header_size(ma_dr_flac_ogg_page_header* pHeader)\n{\n    return 27 + pHeader->segmentCount;\n}\nstatic MA_INLINE ma_uint32 ma_dr_flac_ogg__get_page_body_size(ma_dr_flac_ogg_page_header* pHeader)\n{\n    ma_uint32 pageBodySize = 0;\n    int i;\n    for (i = 0; i < pHeader->segmentCount; ++i) {\n        pageBodySize += pHeader->segmentTable[i];\n    }\n    return pageBodySize;\n}\nstatic ma_result ma_dr_flac_ogg__read_page_header_after_capture_pattern(ma_dr_flac_read_proc onRead, void* pUserData, ma_dr_flac_ogg_page_header* pHeader, ma_uint32* pBytesRead, ma_uint32* pCRC32)\n{\n    ma_uint8 data[23];\n    ma_uint32 i;\n    MA_DR_FLAC_ASSERT(*pCRC32 == MA_DR_FLAC_OGG_CAPTURE_PATTERN_CRC32);\n    if (onRead(pUserData, data, 23) != 23) {\n        return MA_AT_END;\n    }\n    *pBytesRead += 23;\n    pHeader->capturePattern[0] = 'O';\n    pHeader->capturePattern[1] = 'g';\n    pHeader->capturePattern[2] = 'g';\n    pHeader->capturePattern[3] = 'S';\n    pHeader->structureVersion = data[0];\n    pHeader->headerType       = data[1];\n    MA_DR_FLAC_COPY_MEMORY(&pHeader->granulePosition, &data[ 2], 8);\n    MA_DR_FLAC_COPY_MEMORY(&pHeader->serialNumber,    &data[10], 4);\n    MA_DR_FLAC_COPY_MEMORY(&pHeader->sequenceNumber,  &data[14], 4);\n    MA_DR_FLAC_COPY_MEMORY(&pHeader->checksum,        &data[18], 4);\n    pHeader->segmentCount     = data[22];\n    data[18] = 0;\n    data[19] = 0;\n    data[20] = 0;\n    data[21] = 0;\n    for (i = 0; i < 23; ++i) {\n        *pCRC32 = ma_dr_flac_crc32_byte(*pCRC32, data[i]);\n    }\n    if (onRead(pUserData, pHeader->segmentTable, pHeader->segmentCount) != pHeader->segmentCount) {\n        return MA_AT_END;\n    }\n    *pBytesRead += pHeader->segmentCount;\n    for (i = 0; i < pHeader->segmentCount; ++i) {\n        *pCRC32 = ma_dr_flac_crc32_byte(*pCRC32, pHeader->segmentTable[i]);\n    }\n    return MA_SUCCESS;\n}\nstatic ma_result ma_dr_flac_ogg__read_page_header(ma_dr_flac_read_proc onRead, void* pUserData, ma_dr_flac_ogg_page_header* pHeader, ma_uint32* pBytesRead, ma_uint32* pCRC32)\n{\n    ma_uint8 id[4];\n    *pBytesRead = 0;\n    if (onRead(pUserData, id, 4) != 4) {\n        return MA_AT_END;\n    }\n    *pBytesRead += 4;\n    for (;;) {\n        if (ma_dr_flac_ogg__is_capture_pattern(id)) {\n            ma_result result;\n            *pCRC32 = MA_DR_FLAC_OGG_CAPTURE_PATTERN_CRC32;\n            result = ma_dr_flac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, pHeader, pBytesRead, pCRC32);\n            if (result == MA_SUCCESS) {\n                return MA_SUCCESS;\n            } else {\n                if (result == MA_CRC_MISMATCH) {\n                    continue;\n                } else {\n                    return result;\n                }\n            }\n        } else {\n            id[0] = id[1];\n            id[1] = id[2];\n            id[2] = id[3];\n            if (onRead(pUserData, &id[3], 1) != 1) {\n                return MA_AT_END;\n            }\n            *pBytesRead += 1;\n        }\n    }\n}\ntypedef struct\n{\n    ma_dr_flac_read_proc onRead;\n    ma_dr_flac_seek_proc onSeek;\n    ma_dr_flac_tell_proc onTell;\n    void* pUserData;\n    ma_uint64 currentBytePos;\n    ma_uint64 firstBytePos;\n    ma_uint32 serialNumber;\n    ma_dr_flac_ogg_page_header bosPageHeader;\n    ma_dr_flac_ogg_page_header currentPageHeader;\n    ma_uint32 bytesRemainingInPage;\n    ma_uint32 pageDataSize;\n    ma_uint8 pageData[MA_DR_FLAC_OGG_MAX_PAGE_SIZE];\n} ma_dr_flac_oggbs;\nstatic size_t ma_dr_flac_oggbs__read_physical(ma_dr_flac_oggbs* oggbs, void* bufferOut, size_t bytesToRead)\n{\n    size_t bytesActuallyRead = oggbs->onRead(oggbs->pUserData, bufferOut, bytesToRead);\n    oggbs->currentBytePos += bytesActuallyRead;\n    return bytesActuallyRead;\n}\nstatic ma_bool32 ma_dr_flac_oggbs__seek_physical(ma_dr_flac_oggbs* oggbs, ma_uint64 offset, ma_dr_flac_seek_origin origin)\n{\n    if (origin == MA_DR_FLAC_SEEK_SET) {\n        if (offset <= 0x7FFFFFFF) {\n            if (!oggbs->onSeek(oggbs->pUserData, (int)offset, MA_DR_FLAC_SEEK_SET)) {\n                return MA_FALSE;\n            }\n            oggbs->currentBytePos = offset;\n            return MA_TRUE;\n        } else {\n            if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, MA_DR_FLAC_SEEK_SET)) {\n                return MA_FALSE;\n            }\n            oggbs->currentBytePos = offset;\n            return ma_dr_flac_oggbs__seek_physical(oggbs, offset - 0x7FFFFFFF, MA_DR_FLAC_SEEK_CUR);\n        }\n    } else {\n        while (offset > 0x7FFFFFFF) {\n            if (!oggbs->onSeek(oggbs->pUserData, 0x7FFFFFFF, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            oggbs->currentBytePos += 0x7FFFFFFF;\n            offset -= 0x7FFFFFFF;\n        }\n        if (!oggbs->onSeek(oggbs->pUserData, (int)offset, MA_DR_FLAC_SEEK_CUR)) {\n            return MA_FALSE;\n        }\n        oggbs->currentBytePos += offset;\n        return MA_TRUE;\n    }\n}\nstatic ma_bool32 ma_dr_flac_oggbs__goto_next_page(ma_dr_flac_oggbs* oggbs, ma_dr_flac_ogg_crc_mismatch_recovery recoveryMethod)\n{\n    ma_dr_flac_ogg_page_header header;\n    for (;;) {\n        ma_uint32 crc32 = 0;\n        ma_uint32 bytesRead;\n        ma_uint32 pageBodySize;\n#ifndef MA_DR_FLAC_NO_CRC\n        ma_uint32 actualCRC32;\n#endif\n        if (ma_dr_flac_ogg__read_page_header(oggbs->onRead, oggbs->pUserData, &header, &bytesRead, &crc32) != MA_SUCCESS) {\n            return MA_FALSE;\n        }\n        oggbs->currentBytePos += bytesRead;\n        pageBodySize = ma_dr_flac_ogg__get_page_body_size(&header);\n        if (pageBodySize > MA_DR_FLAC_OGG_MAX_PAGE_SIZE) {\n            continue;\n        }\n        if (header.serialNumber != oggbs->serialNumber) {\n            if (pageBodySize > 0 && !ma_dr_flac_oggbs__seek_physical(oggbs, pageBodySize, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            continue;\n        }\n        if (ma_dr_flac_oggbs__read_physical(oggbs, oggbs->pageData, pageBodySize) != pageBodySize) {\n            return MA_FALSE;\n        }\n        oggbs->pageDataSize = pageBodySize;\n#ifndef MA_DR_FLAC_NO_CRC\n        actualCRC32 = ma_dr_flac_crc32_buffer(crc32, oggbs->pageData, oggbs->pageDataSize);\n        if (actualCRC32 != header.checksum) {\n            if (recoveryMethod == ma_dr_flac_ogg_recover_on_crc_mismatch) {\n                continue;\n            } else {\n                ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_recover_on_crc_mismatch);\n                return MA_FALSE;\n            }\n        }\n#else\n        (void)recoveryMethod;\n#endif\n        oggbs->currentPageHeader = header;\n        oggbs->bytesRemainingInPage = pageBodySize;\n        return MA_TRUE;\n    }\n}\n#if 0\nstatic ma_uint8 ma_dr_flac_oggbs__get_current_segment_index(ma_dr_flac_oggbs* oggbs, ma_uint8* pBytesRemainingInSeg)\n{\n    ma_uint32 bytesConsumedInPage = ma_dr_flac_ogg__get_page_body_size(&oggbs->currentPageHeader) - oggbs->bytesRemainingInPage;\n    ma_uint8 iSeg = 0;\n    ma_uint32 iByte = 0;\n    while (iByte < bytesConsumedInPage) {\n        ma_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg];\n        if (iByte + segmentSize > bytesConsumedInPage) {\n            break;\n        } else {\n            iSeg += 1;\n            iByte += segmentSize;\n        }\n    }\n    *pBytesRemainingInSeg = oggbs->currentPageHeader.segmentTable[iSeg] - (ma_uint8)(bytesConsumedInPage - iByte);\n    return iSeg;\n}\nstatic ma_bool32 ma_dr_flac_oggbs__seek_to_next_packet(ma_dr_flac_oggbs* oggbs)\n{\n    for (;;) {\n        ma_bool32 atEndOfPage = MA_FALSE;\n        ma_uint8 bytesRemainingInSeg;\n        ma_uint8 iFirstSeg = ma_dr_flac_oggbs__get_current_segment_index(oggbs, &bytesRemainingInSeg);\n        ma_uint32 bytesToEndOfPacketOrPage = bytesRemainingInSeg;\n        for (ma_uint8 iSeg = iFirstSeg; iSeg < oggbs->currentPageHeader.segmentCount; ++iSeg) {\n            ma_uint8 segmentSize = oggbs->currentPageHeader.segmentTable[iSeg];\n            if (segmentSize < 255) {\n                if (iSeg == oggbs->currentPageHeader.segmentCount-1) {\n                    atEndOfPage = MA_TRUE;\n                }\n                break;\n            }\n            bytesToEndOfPacketOrPage += segmentSize;\n        }\n        ma_dr_flac_oggbs__seek_physical(oggbs, bytesToEndOfPacketOrPage, MA_DR_FLAC_SEEK_CUR);\n        oggbs->bytesRemainingInPage -= bytesToEndOfPacketOrPage;\n        if (atEndOfPage) {\n            if (!ma_dr_flac_oggbs__goto_next_page(oggbs)) {\n                return MA_FALSE;\n            }\n            if ((oggbs->currentPageHeader.headerType & 0x01) == 0) {\n                return MA_TRUE;\n            }\n        } else {\n            return MA_TRUE;\n        }\n    }\n}\nstatic ma_bool32 ma_dr_flac_oggbs__seek_to_next_frame(ma_dr_flac_oggbs* oggbs)\n{\n    return ma_dr_flac_oggbs__seek_to_next_packet(oggbs);\n}\n#endif\nstatic size_t ma_dr_flac__on_read_ogg(void* pUserData, void* bufferOut, size_t bytesToRead)\n{\n    ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pUserData;\n    ma_uint8* pRunningBufferOut = (ma_uint8*)bufferOut;\n    size_t bytesRead = 0;\n    MA_DR_FLAC_ASSERT(oggbs != NULL);\n    MA_DR_FLAC_ASSERT(pRunningBufferOut != NULL);\n    while (bytesRead < bytesToRead) {\n        size_t bytesRemainingToRead = bytesToRead - bytesRead;\n        if (oggbs->bytesRemainingInPage >= bytesRemainingToRead) {\n            MA_DR_FLAC_COPY_MEMORY(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), bytesRemainingToRead);\n            bytesRead += bytesRemainingToRead;\n            oggbs->bytesRemainingInPage -= (ma_uint32)bytesRemainingToRead;\n            break;\n        }\n        if (oggbs->bytesRemainingInPage > 0) {\n            MA_DR_FLAC_COPY_MEMORY(pRunningBufferOut, oggbs->pageData + (oggbs->pageDataSize - oggbs->bytesRemainingInPage), oggbs->bytesRemainingInPage);\n            bytesRead += oggbs->bytesRemainingInPage;\n            pRunningBufferOut += oggbs->bytesRemainingInPage;\n            oggbs->bytesRemainingInPage = 0;\n        }\n        MA_DR_FLAC_ASSERT(bytesRemainingToRead > 0);\n        if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_recover_on_crc_mismatch)) {\n            break;\n        }\n    }\n    return bytesRead;\n}\nstatic ma_bool32 ma_dr_flac__on_seek_ogg(void* pUserData, int offset, ma_dr_flac_seek_origin origin)\n{\n    ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pUserData;\n    int bytesSeeked = 0;\n    MA_DR_FLAC_ASSERT(oggbs != NULL);\n    MA_DR_FLAC_ASSERT(offset >= 0);\n    if (origin == MA_DR_FLAC_SEEK_SET) {\n        if (!ma_dr_flac_oggbs__seek_physical(oggbs, (int)oggbs->firstBytePos, MA_DR_FLAC_SEEK_SET)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_fail_on_crc_mismatch)) {\n            return MA_FALSE;\n        }\n        return ma_dr_flac__on_seek_ogg(pUserData, offset, MA_DR_FLAC_SEEK_CUR);\n    } else if (origin == MA_DR_FLAC_SEEK_CUR) {\n        while (bytesSeeked < offset) {\n            int bytesRemainingToSeek = offset - bytesSeeked;\n            MA_DR_FLAC_ASSERT(bytesRemainingToSeek >= 0);\n            if (oggbs->bytesRemainingInPage >= (size_t)bytesRemainingToSeek) {\n                bytesSeeked += bytesRemainingToSeek;\n                (void)bytesSeeked;\n                oggbs->bytesRemainingInPage -= bytesRemainingToSeek;\n                break;\n            }\n            if (oggbs->bytesRemainingInPage > 0) {\n                bytesSeeked += (int)oggbs->bytesRemainingInPage;\n                oggbs->bytesRemainingInPage = 0;\n            }\n            MA_DR_FLAC_ASSERT(bytesRemainingToSeek > 0);\n            if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_fail_on_crc_mismatch)) {\n                return MA_FALSE;\n            }\n        }\n    } else if (origin == MA_DR_FLAC_SEEK_END) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__on_tell_ogg(void* pUserData, ma_int64* pCursor)\n{\n    (void)pUserData;\n    (void)pCursor;\n    return MA_FALSE;\n}\nstatic ma_bool32 ma_dr_flac_ogg__seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex)\n{\n    ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pFlac->_oggbs;\n    ma_uint64 originalBytePos;\n    ma_uint64 runningGranulePosition;\n    ma_uint64 runningFrameBytePos;\n    ma_uint64 runningPCMFrameCount;\n    MA_DR_FLAC_ASSERT(oggbs != NULL);\n    originalBytePos = oggbs->currentBytePos;\n    if (!ma_dr_flac__seek_to_byte(&pFlac->bs, pFlac->firstFLACFramePosInBytes)) {\n        return MA_FALSE;\n    }\n    oggbs->bytesRemainingInPage = 0;\n    runningGranulePosition = 0;\n    for (;;) {\n        if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_recover_on_crc_mismatch)) {\n            ma_dr_flac_oggbs__seek_physical(oggbs, originalBytePos, MA_DR_FLAC_SEEK_SET);\n            return MA_FALSE;\n        }\n        runningFrameBytePos = oggbs->currentBytePos - ma_dr_flac_ogg__get_page_header_size(&oggbs->currentPageHeader) - oggbs->pageDataSize;\n        if (oggbs->currentPageHeader.granulePosition >= pcmFrameIndex) {\n            break;\n        }\n        if ((oggbs->currentPageHeader.headerType & 0x01) == 0) {\n            if (oggbs->currentPageHeader.segmentTable[0] >= 2) {\n                ma_uint8 firstBytesInPage[2];\n                firstBytesInPage[0] = oggbs->pageData[0];\n                firstBytesInPage[1] = oggbs->pageData[1];\n                if ((firstBytesInPage[0] == 0xFF) && (firstBytesInPage[1] & 0xFC) == 0xF8) {\n                    runningGranulePosition = oggbs->currentPageHeader.granulePosition;\n                }\n                continue;\n            }\n        }\n    }\n    if (!ma_dr_flac_oggbs__seek_physical(oggbs, runningFrameBytePos, MA_DR_FLAC_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_flac_oggbs__goto_next_page(oggbs, ma_dr_flac_ogg_recover_on_crc_mismatch)) {\n        return MA_FALSE;\n    }\n    runningPCMFrameCount = runningGranulePosition;\n    for (;;) {\n        ma_uint64 firstPCMFrameInFLACFrame = 0;\n        ma_uint64 lastPCMFrameInFLACFrame = 0;\n        ma_uint64 pcmFrameCountInThisFrame;\n        if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n            return MA_FALSE;\n        }\n        ma_dr_flac__get_pcm_frame_range_of_current_flac_frame(pFlac, &firstPCMFrameInFLACFrame, &lastPCMFrameInFLACFrame);\n        pcmFrameCountInThisFrame = (lastPCMFrameInFLACFrame - firstPCMFrameInFLACFrame) + 1;\n        if (pcmFrameIndex == pFlac->totalPCMFrameCount && (runningPCMFrameCount + pcmFrameCountInThisFrame) == pFlac->totalPCMFrameCount) {\n            ma_result result = ma_dr_flac__decode_flac_frame(pFlac);\n            if (result == MA_SUCCESS) {\n                pFlac->currentPCMFrame = pcmFrameIndex;\n                pFlac->currentFLACFrame.pcmFramesRemaining = 0;\n                return MA_TRUE;\n            } else {\n                return MA_FALSE;\n            }\n        }\n        if (pcmFrameIndex < (runningPCMFrameCount + pcmFrameCountInThisFrame)) {\n            ma_result result = ma_dr_flac__decode_flac_frame(pFlac);\n            if (result == MA_SUCCESS) {\n                ma_uint64 pcmFramesToDecode = (size_t)(pcmFrameIndex - runningPCMFrameCount);\n                if (pcmFramesToDecode == 0) {\n                    return MA_TRUE;\n                }\n                pFlac->currentPCMFrame = runningPCMFrameCount;\n                return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, pcmFramesToDecode) == pcmFramesToDecode;\n            } else {\n                if (result == MA_CRC_MISMATCH) {\n                    continue;\n                } else {\n                    return MA_FALSE;\n                }\n            }\n        } else {\n            ma_result result = ma_dr_flac__seek_to_next_flac_frame(pFlac);\n            if (result == MA_SUCCESS) {\n                runningPCMFrameCount += pcmFrameCountInThisFrame;\n            } else {\n                if (result == MA_CRC_MISMATCH) {\n                    continue;\n                } else {\n                    return MA_FALSE;\n                }\n            }\n        }\n    }\n}\nstatic ma_bool32 ma_dr_flac__init_private__ogg(ma_dr_flac_init_info* pInit, ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_meta_proc onMeta, void* pUserData, void* pUserDataMD, ma_bool32 relaxed)\n{\n    ma_dr_flac_ogg_page_header header;\n    ma_uint32 crc32 = MA_DR_FLAC_OGG_CAPTURE_PATTERN_CRC32;\n    ma_uint32 bytesRead = 0;\n    (void)relaxed;\n    pInit->container = ma_dr_flac_container_ogg;\n    pInit->oggFirstBytePos = 0;\n    if (ma_dr_flac_ogg__read_page_header_after_capture_pattern(onRead, pUserData, &header, &bytesRead, &crc32) != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    pInit->runningFilePos += bytesRead;\n    for (;;) {\n        int pageBodySize;\n        if ((header.headerType & 0x02) == 0) {\n            return MA_FALSE;\n        }\n        pageBodySize = ma_dr_flac_ogg__get_page_body_size(&header);\n        if (pageBodySize == 51) {\n            ma_uint32 bytesRemainingInPage = pageBodySize;\n            ma_uint8 packetType;\n            if (onRead(pUserData, &packetType, 1) != 1) {\n                return MA_FALSE;\n            }\n            bytesRemainingInPage -= 1;\n            if (packetType == 0x7F) {\n                ma_uint8 sig[4];\n                if (onRead(pUserData, sig, 4) != 4) {\n                    return MA_FALSE;\n                }\n                bytesRemainingInPage -= 4;\n                if (sig[0] == 'F' && sig[1] == 'L' && sig[2] == 'A' && sig[3] == 'C') {\n                    ma_uint8 mappingVersion[2];\n                    if (onRead(pUserData, mappingVersion, 2) != 2) {\n                        return MA_FALSE;\n                    }\n                    if (mappingVersion[0] != 1) {\n                        return MA_FALSE;\n                    }\n                    if (!onSeek(pUserData, 2, MA_DR_FLAC_SEEK_CUR)) {\n                        return MA_FALSE;\n                    }\n                    if (onRead(pUserData, sig, 4) != 4) {\n                        return MA_FALSE;\n                    }\n                    if (sig[0] == 'f' && sig[1] == 'L' && sig[2] == 'a' && sig[3] == 'C') {\n                        ma_dr_flac_streaminfo streaminfo;\n                        ma_uint8 isLastBlock;\n                        ma_uint8 blockType;\n                        ma_uint32 blockSize;\n                        if (!ma_dr_flac__read_and_decode_block_header(onRead, pUserData, &isLastBlock, &blockType, &blockSize)) {\n                            return MA_FALSE;\n                        }\n                        if (blockType != MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO || blockSize != 34) {\n                            return MA_FALSE;\n                        }\n                        if (ma_dr_flac__read_streaminfo(onRead, pUserData, &streaminfo)) {\n                            pInit->hasStreamInfoBlock      = MA_TRUE;\n                            pInit->sampleRate              = streaminfo.sampleRate;\n                            pInit->channels                = streaminfo.channels;\n                            pInit->bitsPerSample           = streaminfo.bitsPerSample;\n                            pInit->totalPCMFrameCount      = streaminfo.totalPCMFrameCount;\n                            pInit->maxBlockSizeInPCMFrames = streaminfo.maxBlockSizeInPCMFrames;\n                            pInit->hasMetadataBlocks       = !isLastBlock;\n                            if (onMeta) {\n                                ma_dr_flac_metadata metadata;\n                                metadata.type = MA_DR_FLAC_METADATA_BLOCK_TYPE_STREAMINFO;\n                                metadata.pRawData = NULL;\n                                metadata.rawDataSize = 0;\n                                metadata.data.streaminfo = streaminfo;\n                                onMeta(pUserDataMD, &metadata);\n                            }\n                            pInit->runningFilePos  += pageBodySize;\n                            pInit->oggFirstBytePos  = pInit->runningFilePos - 79;\n                            pInit->oggSerial        = header.serialNumber;\n                            pInit->oggBosHeader     = header;\n                            break;\n                        } else {\n                            return MA_FALSE;\n                        }\n                    } else {\n                        return MA_FALSE;\n                    }\n                } else {\n                    if (!onSeek(pUserData, bytesRemainingInPage, MA_DR_FLAC_SEEK_CUR)) {\n                        return MA_FALSE;\n                    }\n                }\n            } else {\n                if (!onSeek(pUserData, bytesRemainingInPage, MA_DR_FLAC_SEEK_CUR)) {\n                    return MA_FALSE;\n                }\n            }\n        } else {\n            if (!onSeek(pUserData, pageBodySize, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n        }\n        pInit->runningFilePos += pageBodySize;\n        if (ma_dr_flac_ogg__read_page_header(onRead, pUserData, &header, &bytesRead, &crc32) != MA_SUCCESS) {\n            return MA_FALSE;\n        }\n        pInit->runningFilePos += bytesRead;\n    }\n    pInit->hasMetadataBlocks = MA_TRUE;\n    return MA_TRUE;\n}\n#endif\nstatic ma_bool32 ma_dr_flac__init_private(ma_dr_flac_init_info* pInit, ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, ma_dr_flac_container container, void* pUserData, void* pUserDataMD)\n{\n    ma_bool32 relaxed;\n    ma_uint8 id[4];\n    if (pInit == NULL || onRead == NULL || onSeek == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_FLAC_ZERO_MEMORY(pInit, sizeof(*pInit));\n    pInit->onRead       = onRead;\n    pInit->onSeek       = onSeek;\n    pInit->onTell       = onTell;\n    pInit->onMeta       = onMeta;\n    pInit->container    = container;\n    pInit->pUserData    = pUserData;\n    pInit->pUserDataMD  = pUserDataMD;\n    pInit->bs.onRead    = onRead;\n    pInit->bs.onSeek    = onSeek;\n    pInit->bs.onTell    = onTell;\n    pInit->bs.pUserData = pUserData;\n    ma_dr_flac__reset_cache(&pInit->bs);\n    relaxed = container != ma_dr_flac_container_unknown;\n    for (;;) {\n        if (onRead(pUserData, id, 4) != 4) {\n            return MA_FALSE;\n        }\n        pInit->runningFilePos += 4;\n        if (id[0] == 'I' && id[1] == 'D' && id[2] == '3') {\n            ma_uint8 header[6];\n            ma_uint8 flags;\n            ma_uint32 headerSize;\n            if (onRead(pUserData, header, 6) != 6) {\n                return MA_FALSE;\n            }\n            pInit->runningFilePos += 6;\n            flags = header[1];\n            MA_DR_FLAC_COPY_MEMORY(&headerSize, header+2, 4);\n            headerSize = ma_dr_flac__unsynchsafe_32(ma_dr_flac__be2host_32(headerSize));\n            if (flags & 0x10) {\n                headerSize += 10;\n            }\n            if (!onSeek(pUserData, headerSize, MA_DR_FLAC_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            pInit->runningFilePos += headerSize;\n        } else {\n            break;\n        }\n    }\n    if (id[0] == 'f' && id[1] == 'L' && id[2] == 'a' && id[3] == 'C') {\n        return ma_dr_flac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed);\n    }\n#ifndef MA_DR_FLAC_NO_OGG\n    if (id[0] == 'O' && id[1] == 'g' && id[2] == 'g' && id[3] == 'S') {\n        return ma_dr_flac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed);\n    }\n#endif\n    if (relaxed) {\n        if (container == ma_dr_flac_container_native) {\n            return ma_dr_flac__init_private__native(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed);\n        }\n#ifndef MA_DR_FLAC_NO_OGG\n        if (container == ma_dr_flac_container_ogg) {\n            return ma_dr_flac__init_private__ogg(pInit, onRead, onSeek, onMeta, pUserData, pUserDataMD, relaxed);\n        }\n#endif\n    }\n    return MA_FALSE;\n}\nstatic void ma_dr_flac__init_from_info(ma_dr_flac* pFlac, const ma_dr_flac_init_info* pInit)\n{\n    MA_DR_FLAC_ASSERT(pFlac != NULL);\n    MA_DR_FLAC_ASSERT(pInit != NULL);\n    MA_DR_FLAC_ZERO_MEMORY(pFlac, sizeof(*pFlac));\n    pFlac->bs                      = pInit->bs;\n    pFlac->onMeta                  = pInit->onMeta;\n    pFlac->pUserDataMD             = pInit->pUserDataMD;\n    pFlac->maxBlockSizeInPCMFrames = pInit->maxBlockSizeInPCMFrames;\n    pFlac->sampleRate              = pInit->sampleRate;\n    pFlac->channels                = (ma_uint8)pInit->channels;\n    pFlac->bitsPerSample           = (ma_uint8)pInit->bitsPerSample;\n    pFlac->totalPCMFrameCount      = pInit->totalPCMFrameCount;\n    pFlac->container               = pInit->container;\n}\nstatic ma_dr_flac* ma_dr_flac_open_with_metadata_private(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, ma_dr_flac_container container, void* pUserData, void* pUserDataMD, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac_init_info init;\n    ma_uint32 allocationSize;\n    ma_uint32 wholeSIMDVectorCountPerChannel;\n    ma_uint32 decodedSamplesAllocationSize;\n#ifndef MA_DR_FLAC_NO_OGG\n    ma_dr_flac_oggbs* pOggbs = NULL;\n#endif\n    ma_uint64 firstFramePos;\n    ma_uint64 seektablePos;\n    ma_uint32 seekpointCount;\n    ma_allocation_callbacks allocationCallbacks;\n    ma_dr_flac* pFlac;\n    ma_dr_flac__init_cpu_caps();\n    if (!ma_dr_flac__init_private(&init, onRead, onSeek, onTell, onMeta, container, pUserData, pUserDataMD)) {\n        return NULL;\n    }\n    if (pAllocationCallbacks != NULL) {\n        allocationCallbacks = *pAllocationCallbacks;\n        if (allocationCallbacks.onFree == NULL || (allocationCallbacks.onMalloc == NULL && allocationCallbacks.onRealloc == NULL)) {\n            return NULL;\n        }\n    } else {\n        allocationCallbacks.pUserData = NULL;\n        allocationCallbacks.onMalloc  = ma_dr_flac__malloc_default;\n        allocationCallbacks.onRealloc = ma_dr_flac__realloc_default;\n        allocationCallbacks.onFree    = ma_dr_flac__free_default;\n    }\n    allocationSize = sizeof(ma_dr_flac);\n    if ((init.maxBlockSizeInPCMFrames % (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32))) == 0) {\n        wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32)));\n    } else {\n        wholeSIMDVectorCountPerChannel = (init.maxBlockSizeInPCMFrames / (MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE / sizeof(ma_int32))) + 1;\n    }\n    decodedSamplesAllocationSize = wholeSIMDVectorCountPerChannel * MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE * init.channels;\n    allocationSize += decodedSamplesAllocationSize;\n    allocationSize += MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE;\n#ifndef MA_DR_FLAC_NO_OGG\n    if (init.container == ma_dr_flac_container_ogg) {\n        allocationSize += sizeof(ma_dr_flac_oggbs);\n        pOggbs = (ma_dr_flac_oggbs*)ma_dr_flac__malloc_from_callbacks(sizeof(*pOggbs), &allocationCallbacks);\n        if (pOggbs == NULL) {\n            return NULL;\n        }\n        MA_DR_FLAC_ZERO_MEMORY(pOggbs, sizeof(*pOggbs));\n        pOggbs->onRead = onRead;\n        pOggbs->onSeek = onSeek;\n        pOggbs->onTell = onTell;\n        pOggbs->pUserData = pUserData;\n        pOggbs->currentBytePos = init.oggFirstBytePos;\n        pOggbs->firstBytePos = init.oggFirstBytePos;\n        pOggbs->serialNumber = init.oggSerial;\n        pOggbs->bosPageHeader = init.oggBosHeader;\n        pOggbs->bytesRemainingInPage = 0;\n    }\n#endif\n    firstFramePos  = 42;\n    seektablePos   = 0;\n    seekpointCount = 0;\n    if (init.hasMetadataBlocks) {\n        ma_dr_flac_read_proc onReadOverride = onRead;\n        ma_dr_flac_seek_proc onSeekOverride = onSeek;\n        ma_dr_flac_tell_proc onTellOverride = onTell;\n        void* pUserDataOverride = pUserData;\n#ifndef MA_DR_FLAC_NO_OGG\n        if (init.container == ma_dr_flac_container_ogg) {\n            onReadOverride = ma_dr_flac__on_read_ogg;\n            onSeekOverride = ma_dr_flac__on_seek_ogg;\n            onTellOverride = ma_dr_flac__on_tell_ogg;\n            pUserDataOverride = (void*)pOggbs;\n        }\n#endif\n        if (!ma_dr_flac__read_and_decode_metadata(onReadOverride, onSeekOverride, onTellOverride, onMeta, pUserDataOverride, pUserDataMD, &firstFramePos, &seektablePos, &seekpointCount, &allocationCallbacks)) {\n        #ifndef MA_DR_FLAC_NO_OGG\n            ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks);\n        #endif\n            return NULL;\n        }\n        allocationSize += seekpointCount * sizeof(ma_dr_flac_seekpoint);\n    }\n    pFlac = (ma_dr_flac*)ma_dr_flac__malloc_from_callbacks(allocationSize, &allocationCallbacks);\n    if (pFlac == NULL) {\n    #ifndef MA_DR_FLAC_NO_OGG\n        ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks);\n    #endif\n        return NULL;\n    }\n    ma_dr_flac__init_from_info(pFlac, &init);\n    pFlac->allocationCallbacks = allocationCallbacks;\n    pFlac->pDecodedSamples = (ma_int32*)ma_dr_flac_align((size_t)pFlac->pExtraData, MA_DR_FLAC_MAX_SIMD_VECTOR_SIZE);\n#ifndef MA_DR_FLAC_NO_OGG\n    if (init.container == ma_dr_flac_container_ogg) {\n        ma_dr_flac_oggbs* pInternalOggbs = (ma_dr_flac_oggbs*)((ma_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize + (seekpointCount * sizeof(ma_dr_flac_seekpoint)));\n        MA_DR_FLAC_COPY_MEMORY(pInternalOggbs, pOggbs, sizeof(*pOggbs));\n        ma_dr_flac__free_from_callbacks(pOggbs, &allocationCallbacks);\n        pOggbs = NULL;\n        pFlac->bs.onRead = ma_dr_flac__on_read_ogg;\n        pFlac->bs.onSeek = ma_dr_flac__on_seek_ogg;\n        pFlac->bs.onTell = ma_dr_flac__on_tell_ogg;\n        pFlac->bs.pUserData = (void*)pInternalOggbs;\n        pFlac->_oggbs = (void*)pInternalOggbs;\n    }\n#endif\n    pFlac->firstFLACFramePosInBytes = firstFramePos;\n#ifndef MA_DR_FLAC_NO_OGG\n    if (init.container == ma_dr_flac_container_ogg)\n    {\n        pFlac->pSeekpoints = NULL;\n        pFlac->seekpointCount = 0;\n    }\n    else\n#endif\n    {\n        if (seektablePos != 0) {\n            pFlac->seekpointCount = seekpointCount;\n            pFlac->pSeekpoints = (ma_dr_flac_seekpoint*)((ma_uint8*)pFlac->pDecodedSamples + decodedSamplesAllocationSize);\n            MA_DR_FLAC_ASSERT(pFlac->bs.onSeek != NULL);\n            MA_DR_FLAC_ASSERT(pFlac->bs.onRead != NULL);\n            if (pFlac->bs.onSeek(pFlac->bs.pUserData, (int)seektablePos, MA_DR_FLAC_SEEK_SET)) {\n                ma_uint32 iSeekpoint;\n                for (iSeekpoint = 0; iSeekpoint < seekpointCount; iSeekpoint += 1) {\n                    if (pFlac->bs.onRead(pFlac->bs.pUserData, pFlac->pSeekpoints + iSeekpoint, MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) == MA_DR_FLAC_SEEKPOINT_SIZE_IN_BYTES) {\n                        pFlac->pSeekpoints[iSeekpoint].firstPCMFrame   = ma_dr_flac__be2host_64(pFlac->pSeekpoints[iSeekpoint].firstPCMFrame);\n                        pFlac->pSeekpoints[iSeekpoint].flacFrameOffset = ma_dr_flac__be2host_64(pFlac->pSeekpoints[iSeekpoint].flacFrameOffset);\n                        pFlac->pSeekpoints[iSeekpoint].pcmFrameCount   = ma_dr_flac__be2host_16(pFlac->pSeekpoints[iSeekpoint].pcmFrameCount);\n                    } else {\n                        pFlac->pSeekpoints = NULL;\n                        pFlac->seekpointCount = 0;\n                        break;\n                    }\n                }\n                if (!pFlac->bs.onSeek(pFlac->bs.pUserData, (int)pFlac->firstFLACFramePosInBytes, MA_DR_FLAC_SEEK_SET)) {\n                    ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks);\n                    return NULL;\n                }\n            } else {\n                pFlac->pSeekpoints = NULL;\n                pFlac->seekpointCount = 0;\n            }\n        }\n    }\n    if (!init.hasStreamInfoBlock) {\n        pFlac->currentFLACFrame.header = init.firstFrameHeader;\n        for (;;) {\n            ma_result result = ma_dr_flac__decode_flac_frame(pFlac);\n            if (result == MA_SUCCESS) {\n                break;\n            } else {\n                if (result == MA_CRC_MISMATCH) {\n                    if (!ma_dr_flac__read_next_flac_frame_header(&pFlac->bs, pFlac->bitsPerSample, &pFlac->currentFLACFrame.header)) {\n                        ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks);\n                        return NULL;\n                    }\n                    continue;\n                } else {\n                    ma_dr_flac__free_from_callbacks(pFlac, &allocationCallbacks);\n                    return NULL;\n                }\n            }\n        }\n    }\n    return pFlac;\n}\n#ifndef MA_DR_FLAC_NO_STDIO\n#include <stdio.h>\n#ifndef MA_DR_FLAC_NO_WCHAR\n#include <wchar.h>\n#endif\nstatic size_t ma_dr_flac__on_read_stdio(void* pUserData, void* bufferOut, size_t bytesToRead)\n{\n    return fread(bufferOut, 1, bytesToRead, (FILE*)pUserData);\n}\nstatic ma_bool32 ma_dr_flac__on_seek_stdio(void* pUserData, int offset, ma_dr_flac_seek_origin origin)\n{\n    int whence = SEEK_SET;\n    if (origin == MA_DR_FLAC_SEEK_CUR) {\n        whence = SEEK_CUR;\n    } else if (origin == MA_DR_FLAC_SEEK_END) {\n        whence = SEEK_END;\n    }\n    return fseek((FILE*)pUserData, offset, whence) == 0;\n}\nstatic ma_bool32 ma_dr_flac__on_tell_stdio(void* pUserData, ma_int64* pCursor)\n{\n    FILE* pFileStdio = (FILE*)pUserData;\n    ma_int64 result;\n    MA_DR_FLAC_ASSERT(pFileStdio != NULL);\n    MA_DR_FLAC_ASSERT(pCursor    != NULL);\n#if defined(_WIN32) && !defined(NXDK)\n    #if defined(_MSC_VER) && _MSC_VER > 1200\n        result = _ftelli64(pFileStdio);\n    #else\n        result = ftell(pFileStdio);\n    #endif\n#else\n    result = ftell(pFileStdio);\n#endif\n    *pCursor = result;\n    return MA_TRUE;\n}\nMA_API ma_dr_flac* ma_dr_flac_open_file(const char* pFileName, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    FILE* pFile;\n    if (ma_fopen(&pFile, pFileName, \"rb\") != MA_SUCCESS) {\n        return NULL;\n    }\n    pFlac = ma_dr_flac_open(ma_dr_flac__on_read_stdio, ma_dr_flac__on_seek_stdio, ma_dr_flac__on_tell_stdio, (void*)pFile, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        fclose(pFile);\n        return NULL;\n    }\n    return pFlac;\n}\n#ifndef MA_DR_FLAC_NO_WCHAR\nMA_API ma_dr_flac* ma_dr_flac_open_file_w(const wchar_t* pFileName, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    FILE* pFile;\n    if (ma_wfopen(&pFile, pFileName, L\"rb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return NULL;\n    }\n    pFlac = ma_dr_flac_open(ma_dr_flac__on_read_stdio, ma_dr_flac__on_seek_stdio, ma_dr_flac__on_tell_stdio, (void*)pFile, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        fclose(pFile);\n        return NULL;\n    }\n    return pFlac;\n}\n#endif\nMA_API ma_dr_flac* ma_dr_flac_open_file_with_metadata(const char* pFileName, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    FILE* pFile;\n    if (ma_fopen(&pFile, pFileName, \"rb\") != MA_SUCCESS) {\n        return NULL;\n    }\n    pFlac = ma_dr_flac_open_with_metadata_private(ma_dr_flac__on_read_stdio, ma_dr_flac__on_seek_stdio, ma_dr_flac__on_tell_stdio, onMeta, ma_dr_flac_container_unknown, (void*)pFile, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        fclose(pFile);\n        return pFlac;\n    }\n    return pFlac;\n}\n#ifndef MA_DR_FLAC_NO_WCHAR\nMA_API ma_dr_flac* ma_dr_flac_open_file_with_metadata_w(const wchar_t* pFileName, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    FILE* pFile;\n    if (ma_wfopen(&pFile, pFileName, L\"rb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return NULL;\n    }\n    pFlac = ma_dr_flac_open_with_metadata_private(ma_dr_flac__on_read_stdio, ma_dr_flac__on_seek_stdio, ma_dr_flac__on_tell_stdio, onMeta, ma_dr_flac_container_unknown, (void*)pFile, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        fclose(pFile);\n        return pFlac;\n    }\n    return pFlac;\n}\n#endif\n#endif\nstatic size_t ma_dr_flac__on_read_memory(void* pUserData, void* bufferOut, size_t bytesToRead)\n{\n    ma_dr_flac__memory_stream* memoryStream = (ma_dr_flac__memory_stream*)pUserData;\n    size_t bytesRemaining;\n    MA_DR_FLAC_ASSERT(memoryStream != NULL);\n    MA_DR_FLAC_ASSERT(memoryStream->dataSize >= memoryStream->currentReadPos);\n    bytesRemaining = memoryStream->dataSize - memoryStream->currentReadPos;\n    if (bytesToRead > bytesRemaining) {\n        bytesToRead = bytesRemaining;\n    }\n    if (bytesToRead > 0) {\n        MA_DR_FLAC_COPY_MEMORY(bufferOut, memoryStream->data + memoryStream->currentReadPos, bytesToRead);\n        memoryStream->currentReadPos += bytesToRead;\n    }\n    return bytesToRead;\n}\nstatic ma_bool32 ma_dr_flac__on_seek_memory(void* pUserData, int offset, ma_dr_flac_seek_origin origin)\n{\n    ma_dr_flac__memory_stream* memoryStream = (ma_dr_flac__memory_stream*)pUserData;\n    ma_int64 newCursor;\n    MA_DR_FLAC_ASSERT(memoryStream != NULL);\n    if (origin == MA_DR_FLAC_SEEK_SET) {\n        newCursor = 0;\n    } else if (origin == MA_DR_FLAC_SEEK_CUR) {\n        newCursor = (ma_int64)memoryStream->currentReadPos;\n    } else if (origin == MA_DR_FLAC_SEEK_END) {\n        newCursor = (ma_int64)memoryStream->dataSize;\n    } else {\n        MA_DR_FLAC_ASSERT(!\"Invalid seek origin\");\n        return MA_FALSE;\n    }\n    newCursor += offset;\n    if (newCursor < 0) {\n        return MA_FALSE;\n    }\n    if ((size_t)newCursor > memoryStream->dataSize) {\n        return MA_FALSE;\n    }\n    memoryStream->currentReadPos = (size_t)newCursor;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_flac__on_tell_memory(void* pUserData, ma_int64* pCursor)\n{\n    ma_dr_flac__memory_stream* memoryStream = (ma_dr_flac__memory_stream*)pUserData;\n    MA_DR_FLAC_ASSERT(memoryStream != NULL);\n    MA_DR_FLAC_ASSERT(pCursor != NULL);\n    *pCursor = (ma_int64)memoryStream->currentReadPos;\n    return MA_TRUE;\n}\nMA_API ma_dr_flac* ma_dr_flac_open_memory(const void* pData, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac__memory_stream memoryStream;\n    ma_dr_flac* pFlac;\n    memoryStream.data = (const ma_uint8*)pData;\n    memoryStream.dataSize = dataSize;\n    memoryStream.currentReadPos = 0;\n    pFlac = ma_dr_flac_open(ma_dr_flac__on_read_memory, ma_dr_flac__on_seek_memory, ma_dr_flac__on_tell_memory, &memoryStream, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    pFlac->memoryStream = memoryStream;\n#ifndef MA_DR_FLAC_NO_OGG\n    if (pFlac->container == ma_dr_flac_container_ogg)\n    {\n        ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pFlac->_oggbs;\n        oggbs->pUserData = &pFlac->memoryStream;\n    }\n    else\n#endif\n    {\n        pFlac->bs.pUserData = &pFlac->memoryStream;\n    }\n    return pFlac;\n}\nMA_API ma_dr_flac* ma_dr_flac_open_memory_with_metadata(const void* pData, size_t dataSize, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac__memory_stream memoryStream;\n    ma_dr_flac* pFlac;\n    memoryStream.data = (const ma_uint8*)pData;\n    memoryStream.dataSize = dataSize;\n    memoryStream.currentReadPos = 0;\n    pFlac = ma_dr_flac_open_with_metadata_private(ma_dr_flac__on_read_memory, ma_dr_flac__on_seek_memory, ma_dr_flac__on_tell_memory, onMeta, ma_dr_flac_container_unknown, &memoryStream, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    pFlac->memoryStream = memoryStream;\n#ifndef MA_DR_FLAC_NO_OGG\n    if (pFlac->container == ma_dr_flac_container_ogg)\n    {\n        ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pFlac->_oggbs;\n        oggbs->pUserData = &pFlac->memoryStream;\n    }\n    else\n#endif\n    {\n        pFlac->bs.pUserData = &pFlac->memoryStream;\n    }\n    return pFlac;\n}\nMA_API ma_dr_flac* ma_dr_flac_open(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_flac_open_with_metadata_private(onRead, onSeek, onTell, NULL, ma_dr_flac_container_unknown, pUserData, pUserData, pAllocationCallbacks);\n}\nMA_API ma_dr_flac* ma_dr_flac_open_relaxed(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_container container, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_flac_open_with_metadata_private(onRead, onSeek, onTell, NULL, container, pUserData, pUserData, pAllocationCallbacks);\n}\nMA_API ma_dr_flac* ma_dr_flac_open_with_metadata(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_flac_open_with_metadata_private(onRead, onSeek, onTell, onMeta, ma_dr_flac_container_unknown, pUserData, pUserData, pAllocationCallbacks);\n}\nMA_API ma_dr_flac* ma_dr_flac_open_with_metadata_relaxed(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, ma_dr_flac_meta_proc onMeta, ma_dr_flac_container container, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_flac_open_with_metadata_private(onRead, onSeek, onTell, onMeta, container, pUserData, pUserData, pAllocationCallbacks);\n}\nMA_API void ma_dr_flac_close(ma_dr_flac* pFlac)\n{\n    if (pFlac == NULL) {\n        return;\n    }\n#ifndef MA_DR_FLAC_NO_STDIO\n    if (pFlac->bs.onRead == ma_dr_flac__on_read_stdio) {\n        fclose((FILE*)pFlac->bs.pUserData);\n    }\n#ifndef MA_DR_FLAC_NO_OGG\n    if (pFlac->container == ma_dr_flac_container_ogg) {\n        ma_dr_flac_oggbs* oggbs = (ma_dr_flac_oggbs*)pFlac->_oggbs;\n        MA_DR_FLAC_ASSERT(pFlac->bs.onRead == ma_dr_flac__on_read_ogg);\n        if (oggbs->onRead == ma_dr_flac__on_read_stdio) {\n            fclose((FILE*)oggbs->pUserData);\n        }\n    }\n#endif\n#endif\n    ma_dr_flac__free_from_callbacks(pFlac, &pFlac->allocationCallbacks);\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 left  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 side  = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 left0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 left1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 left2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 left3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 side0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 side1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 side2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 side3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 right0 = left0 - side0;\n        ma_uint32 right1 = left1 - side1;\n        ma_uint32 right2 = left2 - side2;\n        ma_uint32 right3 = left3 - side3;\n        pOutputSamples[i*8+0] = (ma_int32)left0;\n        pOutputSamples[i*8+1] = (ma_int32)right0;\n        pOutputSamples[i*8+2] = (ma_int32)left1;\n        pOutputSamples[i*8+3] = (ma_int32)right1;\n        pOutputSamples[i*8+4] = (ma_int32)left2;\n        pOutputSamples[i*8+5] = (ma_int32)right2;\n        pOutputSamples[i*8+6] = (ma_int32)left3;\n        pOutputSamples[i*8+7] = (ma_int32)right3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i left  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i right = _mm_sub_epi32(left, side);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right));\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t left;\n        uint32x4_t side;\n        uint32x4_t right;\n        left  = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        right = vsubq_u32(left, side);\n        ma_dr_flac__vst2q_u32((ma_uint32*)pOutputSamples + i*8, vzipq_u32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s32__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s32__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 side  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 right = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 side0  = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 side1  = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 side2  = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 side3  = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 right0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 right1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 right2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 right3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 left0 = right0 + side0;\n        ma_uint32 left1 = right1 + side1;\n        ma_uint32 left2 = right2 + side2;\n        ma_uint32 left3 = right3 + side3;\n        pOutputSamples[i*8+0] = (ma_int32)left0;\n        pOutputSamples[i*8+1] = (ma_int32)right0;\n        pOutputSamples[i*8+2] = (ma_int32)left1;\n        pOutputSamples[i*8+3] = (ma_int32)right1;\n        pOutputSamples[i*8+4] = (ma_int32)left2;\n        pOutputSamples[i*8+5] = (ma_int32)right2;\n        pOutputSamples[i*8+6] = (ma_int32)left3;\n        pOutputSamples[i*8+7] = (ma_int32)right3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i left  = _mm_add_epi32(right, side);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right));\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t side;\n        uint32x4_t right;\n        uint32x4_t left;\n        side  = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        right = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        left  = vaddq_u32(right, side);\n        ma_dr_flac__vst2q_u32((ma_uint32*)pOutputSamples + i*8, vzipq_u32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left;\n        pOutputSamples[i*2+1] = (ma_int32)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s32__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s32__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample);\n        pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_int32 shift = unusedBitsPerSample;\n    if (shift > 0) {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = (mid0 + side0) << shift;\n            temp1L = (mid1 + side1) << shift;\n            temp2L = (mid2 + side2) << shift;\n            temp3L = (mid3 + side3) << shift;\n            temp0R = (mid0 - side0) << shift;\n            temp1R = (mid1 - side1) << shift;\n            temp2R = (mid2 - side2) << shift;\n            temp3R = (mid3 - side3) << shift;\n            pOutputSamples[i*8+0] = (ma_int32)temp0L;\n            pOutputSamples[i*8+1] = (ma_int32)temp0R;\n            pOutputSamples[i*8+2] = (ma_int32)temp1L;\n            pOutputSamples[i*8+3] = (ma_int32)temp1R;\n            pOutputSamples[i*8+4] = (ma_int32)temp2L;\n            pOutputSamples[i*8+5] = (ma_int32)temp2R;\n            pOutputSamples[i*8+6] = (ma_int32)temp3L;\n            pOutputSamples[i*8+7] = (ma_int32)temp3R;\n        }\n    } else {\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = (ma_uint32)((ma_int32)(mid0 + side0) >> 1);\n            temp1L = (ma_uint32)((ma_int32)(mid1 + side1) >> 1);\n            temp2L = (ma_uint32)((ma_int32)(mid2 + side2) >> 1);\n            temp3L = (ma_uint32)((ma_int32)(mid3 + side3) >> 1);\n            temp0R = (ma_uint32)((ma_int32)(mid0 - side0) >> 1);\n            temp1R = (ma_uint32)((ma_int32)(mid1 - side1) >> 1);\n            temp2R = (ma_uint32)((ma_int32)(mid2 - side2) >> 1);\n            temp3R = (ma_uint32)((ma_int32)(mid3 - side3) >> 1);\n            pOutputSamples[i*8+0] = (ma_int32)temp0L;\n            pOutputSamples[i*8+1] = (ma_int32)temp0R;\n            pOutputSamples[i*8+2] = (ma_int32)temp1L;\n            pOutputSamples[i*8+3] = (ma_int32)temp1R;\n            pOutputSamples[i*8+4] = (ma_int32)temp2L;\n            pOutputSamples[i*8+5] = (ma_int32)temp2R;\n            pOutputSamples[i*8+6] = (ma_int32)temp3L;\n            pOutputSamples[i*8+7] = (ma_int32)temp3R;\n        }\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample);\n        pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample);\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_int32 shift = unusedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i left;\n            __m128i right;\n            mid   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid   = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            left  = _mm_srai_epi32(_mm_add_epi32(mid, side), 1);\n            right = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1);\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right));\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)(mid + side) >> 1;\n            pOutputSamples[i*2+1] = (ma_int32)(mid - side) >> 1;\n        }\n    } else {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i left;\n            __m128i right;\n            mid   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid   = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            left  = _mm_slli_epi32(_mm_add_epi32(mid, side), shift);\n            right = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift);\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right));\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)((mid + side) << shift);\n            pOutputSamples[i*2+1] = (ma_int32)((mid - side) << shift);\n        }\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_int32 shift = unusedBitsPerSample;\n    int32x4_t  wbpsShift0_4;\n    int32x4_t  wbpsShift1_4;\n    uint32x4_t one4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n    wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n    one4         = vdupq_n_u32(1);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            uint32x4_t mid;\n            uint32x4_t side;\n            int32x4_t left;\n            int32x4_t right;\n            mid   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4);\n            side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4);\n            mid   = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, one4));\n            left  = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1);\n            right = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1);\n            ma_dr_flac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)(mid + side) >> 1;\n            pOutputSamples[i*2+1] = (ma_int32)(mid - side) >> 1;\n        }\n    } else {\n        int32x4_t shift4;\n        shift -= 1;\n        shift4 = vdupq_n_s32(shift);\n        for (i = 0; i < frameCount4; ++i) {\n            uint32x4_t mid;\n            uint32x4_t side;\n            int32x4_t left;\n            int32x4_t right;\n            mid   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4);\n            side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4);\n            mid   = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, one4));\n            left  = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4));\n            right = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4));\n            ma_dr_flac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)((mid + side) << shift);\n            pOutputSamples[i*2+1] = (ma_int32)((mid - side) << shift);\n        }\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s32__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s32__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample));\n        pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample));\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1;\n        pOutputSamples[i*8+0] = (ma_int32)tempL0;\n        pOutputSamples[i*8+1] = (ma_int32)tempR0;\n        pOutputSamples[i*8+2] = (ma_int32)tempL1;\n        pOutputSamples[i*8+3] = (ma_int32)tempR1;\n        pOutputSamples[i*8+4] = (ma_int32)tempL2;\n        pOutputSamples[i*8+5] = (ma_int32)tempR2;\n        pOutputSamples[i*8+6] = (ma_int32)tempL3;\n        pOutputSamples[i*8+7] = (ma_int32)tempR3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0);\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1);\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i left  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 0), _mm_unpacklo_epi32(left, right));\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8 + 4), _mm_unpackhi_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0);\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1);\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift4_0 = vdupq_n_s32(shift0);\n    int32x4_t shift4_1 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        int32x4_t left;\n        int32x4_t right;\n        left  = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift4_0));\n        right = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift4_1));\n        ma_dr_flac__vst2q_s32(pOutputSamples + i*8, vzipq_s32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0);\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int32* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_s32(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int32* pBufferOut)\n{\n    ma_uint64 framesRead;\n    ma_uint32 unusedBitsPerSample;\n    if (pFlac == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead);\n    }\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32);\n    unusedBitsPerSample = 32 - pFlac->bitsPerSample;\n    framesRead = 0;\n    while (framesToRead > 0) {\n        if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) {\n                break;\n            }\n        } else {\n            unsigned int channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment);\n            ma_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining;\n            ma_uint64 frameCountThisIteration = framesToRead;\n            if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) {\n                frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining;\n            }\n            if (channelCount == 2) {\n                const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame;\n                const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame;\n                switch (pFlac->currentFLACFrame.header.channelAssignment)\n                {\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT:\n                    default:\n                    {\n                        ma_dr_flac_read_pcm_frames_s32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                }\n            } else {\n                ma_uint64 i;\n                for (i = 0; i < frameCountThisIteration; ++i) {\n                    unsigned int j;\n                    for (j = 0; j < channelCount; ++j) {\n                        pBufferOut[(i*channelCount)+j] = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample));\n                    }\n                }\n            }\n            framesRead                += frameCountThisIteration;\n            pBufferOut                += frameCountThisIteration * channelCount;\n            framesToRead              -= frameCountThisIteration;\n            pFlac->currentPCMFrame    += frameCountThisIteration;\n            pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)frameCountThisIteration;\n        }\n    }\n    return framesRead;\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 left  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 side  = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 right = left - side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 left0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 left1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 left2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 left3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 side0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 side1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 side2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 side3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 right0 = left0 - side0;\n        ma_uint32 right1 = left1 - side1;\n        ma_uint32 right2 = left2 - side2;\n        ma_uint32 right3 = left3 - side3;\n        left0  >>= 16;\n        left1  >>= 16;\n        left2  >>= 16;\n        left3  >>= 16;\n        right0 >>= 16;\n        right1 >>= 16;\n        right2 >>= 16;\n        right3 >>= 16;\n        pOutputSamples[i*8+0] = (ma_int16)left0;\n        pOutputSamples[i*8+1] = (ma_int16)right0;\n        pOutputSamples[i*8+2] = (ma_int16)left1;\n        pOutputSamples[i*8+3] = (ma_int16)right1;\n        pOutputSamples[i*8+4] = (ma_int16)left2;\n        pOutputSamples[i*8+5] = (ma_int16)right2;\n        pOutputSamples[i*8+6] = (ma_int16)left3;\n        pOutputSamples[i*8+7] = (ma_int16)right3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i left  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i right = _mm_sub_epi32(left, side);\n        left  = _mm_srai_epi32(left,  16);\n        right = _mm_srai_epi32(right, 16);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), ma_dr_flac__mm_packs_interleaved_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t left;\n        uint32x4_t side;\n        uint32x4_t right;\n        left  = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        right = vsubq_u32(left, side);\n        left  = vshrq_n_u32(left,  16);\n        right = vshrq_n_u32(right, 16);\n        ma_dr_flac__vst2q_u16((ma_uint16*)pOutputSamples + i*8, vzip_u16(vmovn_u32(left), vmovn_u32(right)));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s16__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s16__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 side  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 right = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 left  = right + side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 side0  = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 side1  = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 side2  = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 side3  = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 right0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 right1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 right2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 right3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 left0 = right0 + side0;\n        ma_uint32 left1 = right1 + side1;\n        ma_uint32 left2 = right2 + side2;\n        ma_uint32 left3 = right3 + side3;\n        left0  >>= 16;\n        left1  >>= 16;\n        left2  >>= 16;\n        left3  >>= 16;\n        right0 >>= 16;\n        right1 >>= 16;\n        right2 >>= 16;\n        right3 >>= 16;\n        pOutputSamples[i*8+0] = (ma_int16)left0;\n        pOutputSamples[i*8+1] = (ma_int16)right0;\n        pOutputSamples[i*8+2] = (ma_int16)left1;\n        pOutputSamples[i*8+3] = (ma_int16)right1;\n        pOutputSamples[i*8+4] = (ma_int16)left2;\n        pOutputSamples[i*8+5] = (ma_int16)right2;\n        pOutputSamples[i*8+6] = (ma_int16)left3;\n        pOutputSamples[i*8+7] = (ma_int16)right3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i left  = _mm_add_epi32(right, side);\n        left  = _mm_srai_epi32(left,  16);\n        right = _mm_srai_epi32(right, 16);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), ma_dr_flac__mm_packs_interleaved_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t side;\n        uint32x4_t right;\n        uint32x4_t left;\n        side  = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        right = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        left  = vaddq_u32(right, side);\n        left  = vshrq_n_u32(left,  16);\n        right = vshrq_n_u32(right, 16);\n        ma_dr_flac__vst2q_u16((ma_uint16*)pOutputSamples + i*8, vzip_u16(vmovn_u32(left), vmovn_u32(right)));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        left  >>= 16;\n        right >>= 16;\n        pOutputSamples[i*2+0] = (ma_int16)left;\n        pOutputSamples[i*2+1] = (ma_int16)right;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s16__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s16__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        ma_uint32 mid  = (ma_uint32)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = (ma_uint32)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (ma_int16)(((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)(((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample;\n    if (shift > 0) {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = (mid0 + side0) << shift;\n            temp1L = (mid1 + side1) << shift;\n            temp2L = (mid2 + side2) << shift;\n            temp3L = (mid3 + side3) << shift;\n            temp0R = (mid0 - side0) << shift;\n            temp1R = (mid1 - side1) << shift;\n            temp2R = (mid2 - side2) << shift;\n            temp3R = (mid3 - side3) << shift;\n            temp0L >>= 16;\n            temp1L >>= 16;\n            temp2L >>= 16;\n            temp3L >>= 16;\n            temp0R >>= 16;\n            temp1R >>= 16;\n            temp2R >>= 16;\n            temp3R >>= 16;\n            pOutputSamples[i*8+0] = (ma_int16)temp0L;\n            pOutputSamples[i*8+1] = (ma_int16)temp0R;\n            pOutputSamples[i*8+2] = (ma_int16)temp1L;\n            pOutputSamples[i*8+3] = (ma_int16)temp1R;\n            pOutputSamples[i*8+4] = (ma_int16)temp2L;\n            pOutputSamples[i*8+5] = (ma_int16)temp2R;\n            pOutputSamples[i*8+6] = (ma_int16)temp3L;\n            pOutputSamples[i*8+7] = (ma_int16)temp3R;\n        }\n    } else {\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = ((ma_int32)(mid0 + side0) >> 1);\n            temp1L = ((ma_int32)(mid1 + side1) >> 1);\n            temp2L = ((ma_int32)(mid2 + side2) >> 1);\n            temp3L = ((ma_int32)(mid3 + side3) >> 1);\n            temp0R = ((ma_int32)(mid0 - side0) >> 1);\n            temp1R = ((ma_int32)(mid1 - side1) >> 1);\n            temp2R = ((ma_int32)(mid2 - side2) >> 1);\n            temp3R = ((ma_int32)(mid3 - side3) >> 1);\n            temp0L >>= 16;\n            temp1L >>= 16;\n            temp2L >>= 16;\n            temp3L >>= 16;\n            temp0R >>= 16;\n            temp1R >>= 16;\n            temp2R >>= 16;\n            temp3R >>= 16;\n            pOutputSamples[i*8+0] = (ma_int16)temp0L;\n            pOutputSamples[i*8+1] = (ma_int16)temp0R;\n            pOutputSamples[i*8+2] = (ma_int16)temp1L;\n            pOutputSamples[i*8+3] = (ma_int16)temp1R;\n            pOutputSamples[i*8+4] = (ma_int16)temp2L;\n            pOutputSamples[i*8+5] = (ma_int16)temp2R;\n            pOutputSamples[i*8+6] = (ma_int16)temp3L;\n            pOutputSamples[i*8+7] = (ma_int16)temp3R;\n        }\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (ma_int16)(((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)(((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) >> 16);\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i left;\n            __m128i right;\n            mid   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid   = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            left  = _mm_srai_epi32(_mm_add_epi32(mid, side), 1);\n            right = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1);\n            left  = _mm_srai_epi32(left,  16);\n            right = _mm_srai_epi32(right, 16);\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), ma_dr_flac__mm_packs_interleaved_epi32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int16)(((ma_int32)(mid + side) >> 1) >> 16);\n            pOutputSamples[i*2+1] = (ma_int16)(((ma_int32)(mid - side) >> 1) >> 16);\n        }\n    } else {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i left;\n            __m128i right;\n            mid   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid   = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            left  = _mm_slli_epi32(_mm_add_epi32(mid, side), shift);\n            right = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift);\n            left  = _mm_srai_epi32(left,  16);\n            right = _mm_srai_epi32(right, 16);\n            _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), ma_dr_flac__mm_packs_interleaved_epi32(left, right));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int16)(((mid + side) << shift) >> 16);\n            pOutputSamples[i*2+1] = (ma_int16)(((mid - side) << shift) >> 16);\n        }\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample;\n    int32x4_t wbpsShift0_4;\n    int32x4_t wbpsShift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    wbpsShift0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n    wbpsShift1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            uint32x4_t mid;\n            uint32x4_t side;\n            int32x4_t left;\n            int32x4_t right;\n            mid   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4);\n            side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4);\n            mid   = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1)));\n            left  = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1);\n            right = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1);\n            left  = vshrq_n_s32(left,  16);\n            right = vshrq_n_s32(right, 16);\n            ma_dr_flac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right)));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int16)(((ma_int32)(mid + side) >> 1) >> 16);\n            pOutputSamples[i*2+1] = (ma_int16)(((ma_int32)(mid - side) >> 1) >> 16);\n        }\n    } else {\n        int32x4_t shift4;\n        shift -= 1;\n        shift4 = vdupq_n_s32(shift);\n        for (i = 0; i < frameCount4; ++i) {\n            uint32x4_t mid;\n            uint32x4_t side;\n            int32x4_t left;\n            int32x4_t right;\n            mid   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbpsShift0_4);\n            side  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbpsShift1_4);\n            mid   = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1)));\n            left  = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4));\n            right = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4));\n            left  = vshrq_n_s32(left,  16);\n            right = vshrq_n_s32(right, 16);\n            ma_dr_flac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right)));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int16)(((mid + side) << shift) >> 16);\n            pOutputSamples[i*2+1] = (ma_int16)(((mid - side) << shift) >> 16);\n        }\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s16__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s16__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int16)((ma_int32)((ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample)) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)((ma_int32)((ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample)) >> 16);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1;\n        tempL0 >>= 16;\n        tempL1 >>= 16;\n        tempL2 >>= 16;\n        tempL3 >>= 16;\n        tempR0 >>= 16;\n        tempR1 >>= 16;\n        tempR2 >>= 16;\n        tempR3 >>= 16;\n        pOutputSamples[i*8+0] = (ma_int16)tempL0;\n        pOutputSamples[i*8+1] = (ma_int16)tempR0;\n        pOutputSamples[i*8+2] = (ma_int16)tempL1;\n        pOutputSamples[i*8+3] = (ma_int16)tempR1;\n        pOutputSamples[i*8+4] = (ma_int16)tempL2;\n        pOutputSamples[i*8+5] = (ma_int16)tempR2;\n        pOutputSamples[i*8+6] = (ma_int16)tempL3;\n        pOutputSamples[i*8+7] = (ma_int16)tempR3;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int16)((pInputSamples0U32[i] << shift0) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)((pInputSamples1U32[i] << shift1) >> 16);\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i left  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        left  = _mm_srai_epi32(left,  16);\n        right = _mm_srai_epi32(right, 16);\n        _mm_storeu_si128((__m128i*)(pOutputSamples + i*8), ma_dr_flac__mm_packs_interleaved_epi32(left, right));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int16)((pInputSamples0U32[i] << shift0) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)((pInputSamples1U32[i] << shift1) >> 16);\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    int32x4_t shift0_4 = vdupq_n_s32(shift0);\n    int32x4_t shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        int32x4_t left;\n        int32x4_t right;\n        left  = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4));\n        right = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4));\n        left  = vshrq_n_s32(left,  16);\n        right = vshrq_n_s32(right, 16);\n        ma_dr_flac__vst2q_s16(pOutputSamples + i*8, vzip_s16(vmovn_s32(left), vmovn_s32(right)));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int16)((pInputSamples0U32[i] << shift0) >> 16);\n        pOutputSamples[i*2+1] = (ma_int16)((pInputSamples1U32[i] << shift1) >> 16);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, ma_int16* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_s16(ma_dr_flac* pFlac, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    ma_uint64 framesRead;\n    ma_uint32 unusedBitsPerSample;\n    if (pFlac == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead);\n    }\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32);\n    unusedBitsPerSample = 32 - pFlac->bitsPerSample;\n    framesRead = 0;\n    while (framesToRead > 0) {\n        if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) {\n                break;\n            }\n        } else {\n            unsigned int channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment);\n            ma_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining;\n            ma_uint64 frameCountThisIteration = framesToRead;\n            if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) {\n                frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining;\n            }\n            if (channelCount == 2) {\n                const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame;\n                const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame;\n                switch (pFlac->currentFLACFrame.header.channelAssignment)\n                {\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s16__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s16__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_s16__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT:\n                    default:\n                    {\n                        ma_dr_flac_read_pcm_frames_s16__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                }\n            } else {\n                ma_uint64 i;\n                for (i = 0; i < frameCountThisIteration; ++i) {\n                    unsigned int j;\n                    for (j = 0; j < channelCount; ++j) {\n                        ma_int32 sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample));\n                        pBufferOut[(i*channelCount)+j] = (ma_int16)(sampleS32 >> 16);\n                    }\n                }\n            }\n            framesRead                += frameCountThisIteration;\n            pBufferOut                += frameCountThisIteration * channelCount;\n            framesToRead              -= frameCountThisIteration;\n            pFlac->currentPCMFrame    += frameCountThisIteration;\n            pFlac->currentFLACFrame.pcmFramesRemaining -= (ma_uint32)frameCountThisIteration;\n        }\n    }\n    return framesRead;\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 left  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 side  = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (float)((ma_int32)left  / 2147483648.0);\n        pOutputSamples[i*2+1] = (float)((ma_int32)right / 2147483648.0);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    float factor = 1 / 2147483648.0;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 left0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 left1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 left2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 left3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 side0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 side1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 side2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 side3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 right0 = left0 - side0;\n        ma_uint32 right1 = left1 - side1;\n        ma_uint32 right2 = left2 - side2;\n        ma_uint32 right3 = left3 - side3;\n        pOutputSamples[i*8+0] = (ma_int32)left0  * factor;\n        pOutputSamples[i*8+1] = (ma_int32)right0 * factor;\n        pOutputSamples[i*8+2] = (ma_int32)left1  * factor;\n        pOutputSamples[i*8+3] = (ma_int32)right1 * factor;\n        pOutputSamples[i*8+4] = (ma_int32)left2  * factor;\n        pOutputSamples[i*8+5] = (ma_int32)right2 * factor;\n        pOutputSamples[i*8+6] = (ma_int32)left3  * factor;\n        pOutputSamples[i*8+7] = (ma_int32)right3 * factor;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left  * factor;\n        pOutputSamples[i*2+1] = (ma_int32)right * factor;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    __m128 factor;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor = _mm_set1_ps(1.0f / 8388608.0f);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i left  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i right = _mm_sub_epi32(left, side);\n        __m128 leftf  = _mm_mul_ps(_mm_cvtepi32_ps(left),  factor);\n        __m128 rightf = _mm_mul_ps(_mm_cvtepi32_ps(right), factor);\n        _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf));\n        _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left  / 8388608.0f;\n        pOutputSamples[i*2+1] = (ma_int32)right / 8388608.0f;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    float32x4_t factor4;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor4  = vdupq_n_f32(1.0f / 8388608.0f);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t left;\n        uint32x4_t side;\n        uint32x4_t right;\n        float32x4_t leftf;\n        float32x4_t rightf;\n        left   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        side   = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        right  = vsubq_u32(left, side);\n        leftf  = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(left)),  factor4);\n        rightf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(right)), factor4);\n        ma_dr_flac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 left  = pInputSamples0U32[i] << shift0;\n        ma_uint32 side  = pInputSamples1U32[i] << shift1;\n        ma_uint32 right = left - side;\n        pOutputSamples[i*2+0] = (ma_int32)left  / 8388608.0f;\n        pOutputSamples[i*2+1] = (ma_int32)right / 8388608.0f;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_left_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_left_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_left_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_f32__decode_left_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_f32__decode_left_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    for (i = 0; i < frameCount; ++i) {\n        ma_uint32 side  = (ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n        ma_uint32 right = (ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (float)((ma_int32)left  / 2147483648.0);\n        pOutputSamples[i*2+1] = (float)((ma_int32)right / 2147483648.0);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    float factor = 1 / 2147483648.0;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 side0  = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 side1  = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 side2  = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 side3  = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 right0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 right1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 right2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 right3 = pInputSamples1U32[i*4+3] << shift1;\n        ma_uint32 left0 = right0 + side0;\n        ma_uint32 left1 = right1 + side1;\n        ma_uint32 left2 = right2 + side2;\n        ma_uint32 left3 = right3 + side3;\n        pOutputSamples[i*8+0] = (ma_int32)left0  * factor;\n        pOutputSamples[i*8+1] = (ma_int32)right0 * factor;\n        pOutputSamples[i*8+2] = (ma_int32)left1  * factor;\n        pOutputSamples[i*8+3] = (ma_int32)right1 * factor;\n        pOutputSamples[i*8+4] = (ma_int32)left2  * factor;\n        pOutputSamples[i*8+5] = (ma_int32)right2 * factor;\n        pOutputSamples[i*8+6] = (ma_int32)left3  * factor;\n        pOutputSamples[i*8+7] = (ma_int32)right3 * factor;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left  * factor;\n        pOutputSamples[i*2+1] = (ma_int32)right * factor;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    __m128 factor;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor = _mm_set1_ps(1.0f / 8388608.0f);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i side  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        __m128i right = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        __m128i left  = _mm_add_epi32(right, side);\n        __m128 leftf  = _mm_mul_ps(_mm_cvtepi32_ps(left),  factor);\n        __m128 rightf = _mm_mul_ps(_mm_cvtepi32_ps(right), factor);\n        _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf));\n        _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left  / 8388608.0f;\n        pOutputSamples[i*2+1] = (ma_int32)right / 8388608.0f;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    float32x4_t factor4;\n    int32x4_t shift0_4;\n    int32x4_t shift1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor4  = vdupq_n_f32(1.0f / 8388608.0f);\n    shift0_4 = vdupq_n_s32(shift0);\n    shift1_4 = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        uint32x4_t side;\n        uint32x4_t right;\n        uint32x4_t left;\n        float32x4_t leftf;\n        float32x4_t rightf;\n        side   = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4);\n        right  = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4);\n        left   = vaddq_u32(right, side);\n        leftf  = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(left)),  factor4);\n        rightf = vmulq_f32(vcvtq_f32_s32(vreinterpretq_s32_u32(right)), factor4);\n        ma_dr_flac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 side  = pInputSamples0U32[i] << shift0;\n        ma_uint32 right = pInputSamples1U32[i] << shift1;\n        ma_uint32 left  = right + side;\n        pOutputSamples[i*2+0] = (ma_int32)left  / 8388608.0f;\n        pOutputSamples[i*2+1] = (ma_int32)right / 8388608.0f;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_right_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_right_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_right_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_f32__decode_right_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_f32__decode_right_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        ma_uint32 mid  = (ma_uint32)pInputSamples0[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = (ma_uint32)pInputSamples1[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (float)((((ma_int32)(mid + side) >> 1) << (unusedBitsPerSample)) / 2147483648.0);\n        pOutputSamples[i*2+1] = (float)((((ma_int32)(mid - side) >> 1) << (unusedBitsPerSample)) / 2147483648.0);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample;\n    float factor = 1 / 2147483648.0;\n    if (shift > 0) {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = (mid0 + side0) << shift;\n            temp1L = (mid1 + side1) << shift;\n            temp2L = (mid2 + side2) << shift;\n            temp3L = (mid3 + side3) << shift;\n            temp0R = (mid0 - side0) << shift;\n            temp1R = (mid1 - side1) << shift;\n            temp2R = (mid2 - side2) << shift;\n            temp3R = (mid3 - side3) << shift;\n            pOutputSamples[i*8+0] = (ma_int32)temp0L * factor;\n            pOutputSamples[i*8+1] = (ma_int32)temp0R * factor;\n            pOutputSamples[i*8+2] = (ma_int32)temp1L * factor;\n            pOutputSamples[i*8+3] = (ma_int32)temp1R * factor;\n            pOutputSamples[i*8+4] = (ma_int32)temp2L * factor;\n            pOutputSamples[i*8+5] = (ma_int32)temp2R * factor;\n            pOutputSamples[i*8+6] = (ma_int32)temp3L * factor;\n            pOutputSamples[i*8+7] = (ma_int32)temp3R * factor;\n        }\n    } else {\n        for (i = 0; i < frameCount4; ++i) {\n            ma_uint32 temp0L;\n            ma_uint32 temp1L;\n            ma_uint32 temp2L;\n            ma_uint32 temp3L;\n            ma_uint32 temp0R;\n            ma_uint32 temp1R;\n            ma_uint32 temp2R;\n            ma_uint32 temp3R;\n            ma_uint32 mid0  = pInputSamples0U32[i*4+0] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid1  = pInputSamples0U32[i*4+1] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid2  = pInputSamples0U32[i*4+2] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 mid3  = pInputSamples0U32[i*4+3] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side0 = pInputSamples1U32[i*4+0] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side1 = pInputSamples1U32[i*4+1] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side2 = pInputSamples1U32[i*4+2] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            ma_uint32 side3 = pInputSamples1U32[i*4+3] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid0 = (mid0 << 1) | (side0 & 0x01);\n            mid1 = (mid1 << 1) | (side1 & 0x01);\n            mid2 = (mid2 << 1) | (side2 & 0x01);\n            mid3 = (mid3 << 1) | (side3 & 0x01);\n            temp0L = (ma_uint32)((ma_int32)(mid0 + side0) >> 1);\n            temp1L = (ma_uint32)((ma_int32)(mid1 + side1) >> 1);\n            temp2L = (ma_uint32)((ma_int32)(mid2 + side2) >> 1);\n            temp3L = (ma_uint32)((ma_int32)(mid3 + side3) >> 1);\n            temp0R = (ma_uint32)((ma_int32)(mid0 - side0) >> 1);\n            temp1R = (ma_uint32)((ma_int32)(mid1 - side1) >> 1);\n            temp2R = (ma_uint32)((ma_int32)(mid2 - side2) >> 1);\n            temp3R = (ma_uint32)((ma_int32)(mid3 - side3) >> 1);\n            pOutputSamples[i*8+0] = (ma_int32)temp0L * factor;\n            pOutputSamples[i*8+1] = (ma_int32)temp0R * factor;\n            pOutputSamples[i*8+2] = (ma_int32)temp1L * factor;\n            pOutputSamples[i*8+3] = (ma_int32)temp1R * factor;\n            pOutputSamples[i*8+4] = (ma_int32)temp2L * factor;\n            pOutputSamples[i*8+5] = (ma_int32)temp2R * factor;\n            pOutputSamples[i*8+6] = (ma_int32)temp3L * factor;\n            pOutputSamples[i*8+7] = (ma_int32)temp3R * factor;\n        }\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n        ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n        mid = (mid << 1) | (side & 0x01);\n        pOutputSamples[i*2+0] = (ma_int32)((ma_uint32)((ma_int32)(mid + side) >> 1) << unusedBitsPerSample) * factor;\n        pOutputSamples[i*2+1] = (ma_int32)((ma_uint32)((ma_int32)(mid - side) >> 1) << unusedBitsPerSample) * factor;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample - 8;\n    float factor;\n    __m128 factor128;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor = 1.0f / 8388608.0f;\n    factor128 = _mm_set1_ps(factor);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i tempL;\n            __m128i tempR;\n            __m128  leftf;\n            __m128  rightf;\n            mid    = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid    = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            tempL  = _mm_srai_epi32(_mm_add_epi32(mid, side), 1);\n            tempR  = _mm_srai_epi32(_mm_sub_epi32(mid, side), 1);\n            leftf  = _mm_mul_ps(_mm_cvtepi32_ps(tempL), factor128);\n            rightf = _mm_mul_ps(_mm_cvtepi32_ps(tempR), factor128);\n            _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf));\n            _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = ((ma_int32)(mid + side) >> 1) * factor;\n            pOutputSamples[i*2+1] = ((ma_int32)(mid - side) >> 1) * factor;\n        }\n    } else {\n        shift -= 1;\n        for (i = 0; i < frameCount4; ++i) {\n            __m128i mid;\n            __m128i side;\n            __m128i tempL;\n            __m128i tempR;\n            __m128 leftf;\n            __m128 rightf;\n            mid    = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n            side   = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n            mid    = _mm_or_si128(_mm_slli_epi32(mid, 1), _mm_and_si128(side, _mm_set1_epi32(0x01)));\n            tempL  = _mm_slli_epi32(_mm_add_epi32(mid, side), shift);\n            tempR  = _mm_slli_epi32(_mm_sub_epi32(mid, side), shift);\n            leftf  = _mm_mul_ps(_mm_cvtepi32_ps(tempL), factor128);\n            rightf = _mm_mul_ps(_mm_cvtepi32_ps(tempR), factor128);\n            _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf));\n            _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)((mid + side) << shift) * factor;\n            pOutputSamples[i*2+1] = (ma_int32)((mid - side) << shift) * factor;\n        }\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift = unusedBitsPerSample - 8;\n    float factor;\n    float32x4_t factor4;\n    int32x4_t shift4;\n    int32x4_t wbps0_4;\n    int32x4_t wbps1_4;\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 24);\n    factor  = 1.0f / 8388608.0f;\n    factor4 = vdupq_n_f32(factor);\n    wbps0_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample);\n    wbps1_4 = vdupq_n_s32(pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample);\n    if (shift == 0) {\n        for (i = 0; i < frameCount4; ++i) {\n            int32x4_t lefti;\n            int32x4_t righti;\n            float32x4_t leftf;\n            float32x4_t rightf;\n            uint32x4_t mid  = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbps0_4);\n            uint32x4_t side = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbps1_4);\n            mid    = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1)));\n            lefti  = vshrq_n_s32(vreinterpretq_s32_u32(vaddq_u32(mid, side)), 1);\n            righti = vshrq_n_s32(vreinterpretq_s32_u32(vsubq_u32(mid, side)), 1);\n            leftf  = vmulq_f32(vcvtq_f32_s32(lefti),  factor4);\n            rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4);\n            ma_dr_flac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = ((ma_int32)(mid + side) >> 1) * factor;\n            pOutputSamples[i*2+1] = ((ma_int32)(mid - side) >> 1) * factor;\n        }\n    } else {\n        shift -= 1;\n        shift4 = vdupq_n_s32(shift);\n        for (i = 0; i < frameCount4; ++i) {\n            uint32x4_t mid;\n            uint32x4_t side;\n            int32x4_t lefti;\n            int32x4_t righti;\n            float32x4_t leftf;\n            float32x4_t rightf;\n            mid    = vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), wbps0_4);\n            side   = vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), wbps1_4);\n            mid    = vorrq_u32(vshlq_n_u32(mid, 1), vandq_u32(side, vdupq_n_u32(1)));\n            lefti  = vreinterpretq_s32_u32(vshlq_u32(vaddq_u32(mid, side), shift4));\n            righti = vreinterpretq_s32_u32(vshlq_u32(vsubq_u32(mid, side), shift4));\n            leftf  = vmulq_f32(vcvtq_f32_s32(lefti),  factor4);\n            rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4);\n            ma_dr_flac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf));\n        }\n        for (i = (frameCount4 << 2); i < frameCount; ++i) {\n            ma_uint32 mid  = pInputSamples0U32[i] << pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n            ma_uint32 side = pInputSamples1U32[i] << pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n            mid = (mid << 1) | (side & 0x01);\n            pOutputSamples[i*2+0] = (ma_int32)((mid + side) << shift) * factor;\n            pOutputSamples[i*2+1] = (ma_int32)((mid - side) << shift) * factor;\n        }\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_mid_side(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_mid_side__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_mid_side__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_f32__decode_mid_side__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_f32__decode_mid_side__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\n#if 0\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__reference(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    for (ma_uint64 i = 0; i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (float)((ma_int32)((ma_uint32)pInputSamples0[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample)) / 2147483648.0);\n        pOutputSamples[i*2+1] = (float)((ma_int32)((ma_uint32)pInputSamples1[i] << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample)) / 2147483648.0);\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__scalar(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample;\n    ma_uint32 shift1 = unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample;\n    float factor = 1 / 2147483648.0;\n    for (i = 0; i < frameCount4; ++i) {\n        ma_uint32 tempL0 = pInputSamples0U32[i*4+0] << shift0;\n        ma_uint32 tempL1 = pInputSamples0U32[i*4+1] << shift0;\n        ma_uint32 tempL2 = pInputSamples0U32[i*4+2] << shift0;\n        ma_uint32 tempL3 = pInputSamples0U32[i*4+3] << shift0;\n        ma_uint32 tempR0 = pInputSamples1U32[i*4+0] << shift1;\n        ma_uint32 tempR1 = pInputSamples1U32[i*4+1] << shift1;\n        ma_uint32 tempR2 = pInputSamples1U32[i*4+2] << shift1;\n        ma_uint32 tempR3 = pInputSamples1U32[i*4+3] << shift1;\n        pOutputSamples[i*8+0] = (ma_int32)tempL0 * factor;\n        pOutputSamples[i*8+1] = (ma_int32)tempR0 * factor;\n        pOutputSamples[i*8+2] = (ma_int32)tempL1 * factor;\n        pOutputSamples[i*8+3] = (ma_int32)tempR1 * factor;\n        pOutputSamples[i*8+4] = (ma_int32)tempL2 * factor;\n        pOutputSamples[i*8+5] = (ma_int32)tempR2 * factor;\n        pOutputSamples[i*8+6] = (ma_int32)tempL3 * factor;\n        pOutputSamples[i*8+7] = (ma_int32)tempR3 * factor;\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0) * factor;\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1) * factor;\n    }\n}\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__sse2(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    float factor = 1.0f / 8388608.0f;\n    __m128 factor128 = _mm_set1_ps(factor);\n    for (i = 0; i < frameCount4; ++i) {\n        __m128i lefti;\n        __m128i righti;\n        __m128 leftf;\n        __m128 rightf;\n        lefti  = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples0 + i), shift0);\n        righti = _mm_slli_epi32(_mm_loadu_si128((const __m128i*)pInputSamples1 + i), shift1);\n        leftf  = _mm_mul_ps(_mm_cvtepi32_ps(lefti),  factor128);\n        rightf = _mm_mul_ps(_mm_cvtepi32_ps(righti), factor128);\n        _mm_storeu_ps(pOutputSamples + i*8 + 0, _mm_unpacklo_ps(leftf, rightf));\n        _mm_storeu_ps(pOutputSamples + i*8 + 4, _mm_unpackhi_ps(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0) * factor;\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1) * factor;\n    }\n}\n#endif\n#if defined(MA_DR_FLAC_SUPPORT_NEON)\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__neon(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n    ma_uint64 i;\n    ma_uint64 frameCount4 = frameCount >> 2;\n    const ma_uint32* pInputSamples0U32 = (const ma_uint32*)pInputSamples0;\n    const ma_uint32* pInputSamples1U32 = (const ma_uint32*)pInputSamples1;\n    ma_uint32 shift0 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[0].wastedBitsPerSample) - 8;\n    ma_uint32 shift1 = (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[1].wastedBitsPerSample) - 8;\n    float factor = 1.0f / 8388608.0f;\n    float32x4_t factor4 = vdupq_n_f32(factor);\n    int32x4_t shift0_4  = vdupq_n_s32(shift0);\n    int32x4_t shift1_4  = vdupq_n_s32(shift1);\n    for (i = 0; i < frameCount4; ++i) {\n        int32x4_t lefti;\n        int32x4_t righti;\n        float32x4_t leftf;\n        float32x4_t rightf;\n        lefti  = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples0U32 + i*4), shift0_4));\n        righti = vreinterpretq_s32_u32(vshlq_u32(vld1q_u32(pInputSamples1U32 + i*4), shift1_4));\n        leftf  = vmulq_f32(vcvtq_f32_s32(lefti),  factor4);\n        rightf = vmulq_f32(vcvtq_f32_s32(righti), factor4);\n        ma_dr_flac__vst2q_f32(pOutputSamples + i*8, vzipq_f32(leftf, rightf));\n    }\n    for (i = (frameCount4 << 2); i < frameCount; ++i) {\n        pOutputSamples[i*2+0] = (ma_int32)(pInputSamples0U32[i] << shift0) * factor;\n        pOutputSamples[i*2+1] = (ma_int32)(pInputSamples1U32[i] << shift1) * factor;\n    }\n}\n#endif\nstatic MA_INLINE void ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo(ma_dr_flac* pFlac, ma_uint64 frameCount, ma_uint32 unusedBitsPerSample, const ma_int32* pInputSamples0, const ma_int32* pInputSamples1, float* pOutputSamples)\n{\n#if defined(MA_DR_FLAC_SUPPORT_SSE2)\n    if (ma_dr_flac__gIsSSE2Supported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__sse2(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#elif defined(MA_DR_FLAC_SUPPORT_NEON)\n    if (ma_dr_flac__gIsNEONSupported && pFlac->bitsPerSample <= 24) {\n        ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__neon(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n    } else\n#endif\n    {\n#if 0\n        ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__reference(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#else\n        ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo__scalar(pFlac, frameCount, unusedBitsPerSample, pInputSamples0, pInputSamples1, pOutputSamples);\n#endif\n    }\n}\nMA_API ma_uint64 ma_dr_flac_read_pcm_frames_f32(ma_dr_flac* pFlac, ma_uint64 framesToRead, float* pBufferOut)\n{\n    ma_uint64 framesRead;\n    ma_uint32 unusedBitsPerSample;\n    if (pFlac == NULL || framesToRead == 0) {\n        return 0;\n    }\n    if (pBufferOut == NULL) {\n        return ma_dr_flac__seek_forward_by_pcm_frames(pFlac, framesToRead);\n    }\n    MA_DR_FLAC_ASSERT(pFlac->bitsPerSample <= 32);\n    unusedBitsPerSample = 32 - pFlac->bitsPerSample;\n    framesRead = 0;\n    while (framesToRead > 0) {\n        if (pFlac->currentFLACFrame.pcmFramesRemaining == 0) {\n            if (!ma_dr_flac__read_and_decode_next_flac_frame(pFlac)) {\n                break;\n            }\n        } else {\n            unsigned int channelCount = ma_dr_flac__get_channel_count_from_channel_assignment(pFlac->currentFLACFrame.header.channelAssignment);\n            ma_uint64 iFirstPCMFrame = pFlac->currentFLACFrame.header.blockSizeInPCMFrames - pFlac->currentFLACFrame.pcmFramesRemaining;\n            ma_uint64 frameCountThisIteration = framesToRead;\n            if (frameCountThisIteration > pFlac->currentFLACFrame.pcmFramesRemaining) {\n                frameCountThisIteration = pFlac->currentFLACFrame.pcmFramesRemaining;\n            }\n            if (channelCount == 2) {\n                const ma_int32* pDecodedSamples0 = pFlac->currentFLACFrame.subframes[0].pSamplesS32 + iFirstPCMFrame;\n                const ma_int32* pDecodedSamples1 = pFlac->currentFLACFrame.subframes[1].pSamplesS32 + iFirstPCMFrame;\n                switch (pFlac->currentFLACFrame.header.channelAssignment)\n                {\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_LEFT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_f32__decode_left_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_RIGHT_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_f32__decode_right_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_MID_SIDE:\n                    {\n                        ma_dr_flac_read_pcm_frames_f32__decode_mid_side(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                    case MA_DR_FLAC_CHANNEL_ASSIGNMENT_INDEPENDENT:\n                    default:\n                    {\n                        ma_dr_flac_read_pcm_frames_f32__decode_independent_stereo(pFlac, frameCountThisIteration, unusedBitsPerSample, pDecodedSamples0, pDecodedSamples1, pBufferOut);\n                    } break;\n                }\n            } else {\n                ma_uint64 i;\n                for (i = 0; i < frameCountThisIteration; ++i) {\n                    unsigned int j;\n                    for (j = 0; j < channelCount; ++j) {\n                        ma_int32 sampleS32 = (ma_int32)((ma_uint32)(pFlac->currentFLACFrame.subframes[j].pSamplesS32[iFirstPCMFrame + i]) << (unusedBitsPerSample + pFlac->currentFLACFrame.subframes[j].wastedBitsPerSample));\n                        pBufferOut[(i*channelCount)+j] = (float)(sampleS32 / 2147483648.0);\n                    }\n                }\n            }\n            framesRead                += frameCountThisIteration;\n            pBufferOut                += frameCountThisIteration * channelCount;\n            framesToRead              -= frameCountThisIteration;\n            pFlac->currentPCMFrame    += frameCountThisIteration;\n            pFlac->currentFLACFrame.pcmFramesRemaining -= (unsigned int)frameCountThisIteration;\n        }\n    }\n    return framesRead;\n}\nMA_API ma_bool32 ma_dr_flac_seek_to_pcm_frame(ma_dr_flac* pFlac, ma_uint64 pcmFrameIndex)\n{\n    if (pFlac == NULL) {\n        return MA_FALSE;\n    }\n    if (pFlac->currentPCMFrame == pcmFrameIndex) {\n        return MA_TRUE;\n    }\n    if (pFlac->firstFLACFramePosInBytes == 0) {\n        return MA_FALSE;\n    }\n    if (pcmFrameIndex == 0) {\n        pFlac->currentPCMFrame = 0;\n        return ma_dr_flac__seek_to_first_frame(pFlac);\n    } else {\n        ma_bool32 wasSuccessful = MA_FALSE;\n        ma_uint64 originalPCMFrame = pFlac->currentPCMFrame;\n        if (pcmFrameIndex > pFlac->totalPCMFrameCount) {\n            pcmFrameIndex = pFlac->totalPCMFrameCount;\n        }\n        if (pcmFrameIndex > pFlac->currentPCMFrame) {\n            ma_uint32 offset = (ma_uint32)(pcmFrameIndex - pFlac->currentPCMFrame);\n            if (pFlac->currentFLACFrame.pcmFramesRemaining >  offset) {\n                pFlac->currentFLACFrame.pcmFramesRemaining -= offset;\n                pFlac->currentPCMFrame = pcmFrameIndex;\n                return MA_TRUE;\n            }\n        } else {\n            ma_uint32 offsetAbs = (ma_uint32)(pFlac->currentPCMFrame - pcmFrameIndex);\n            ma_uint32 currentFLACFramePCMFrameCount = pFlac->currentFLACFrame.header.blockSizeInPCMFrames;\n            ma_uint32 currentFLACFramePCMFramesConsumed = currentFLACFramePCMFrameCount - pFlac->currentFLACFrame.pcmFramesRemaining;\n            if (currentFLACFramePCMFramesConsumed > offsetAbs) {\n                pFlac->currentFLACFrame.pcmFramesRemaining += offsetAbs;\n                pFlac->currentPCMFrame = pcmFrameIndex;\n                return MA_TRUE;\n            }\n        }\n#ifndef MA_DR_FLAC_NO_OGG\n        if (pFlac->container == ma_dr_flac_container_ogg)\n        {\n            wasSuccessful = ma_dr_flac_ogg__seek_to_pcm_frame(pFlac, pcmFrameIndex);\n        }\n        else\n#endif\n        {\n            if (!pFlac->_noSeekTableSeek) {\n                wasSuccessful = ma_dr_flac__seek_to_pcm_frame__seek_table(pFlac, pcmFrameIndex);\n            }\n#if !defined(MA_DR_FLAC_NO_CRC)\n            if (!wasSuccessful && !pFlac->_noBinarySearchSeek && pFlac->totalPCMFrameCount > 0) {\n                wasSuccessful = ma_dr_flac__seek_to_pcm_frame__binary_search(pFlac, pcmFrameIndex);\n            }\n#endif\n            if (!wasSuccessful && !pFlac->_noBruteForceSeek) {\n                wasSuccessful = ma_dr_flac__seek_to_pcm_frame__brute_force(pFlac, pcmFrameIndex);\n            }\n        }\n        if (wasSuccessful) {\n            pFlac->currentPCMFrame = pcmFrameIndex;\n        } else {\n            if (ma_dr_flac_seek_to_pcm_frame(pFlac, originalPCMFrame) == MA_FALSE) {\n                ma_dr_flac_seek_to_pcm_frame(pFlac, 0);\n            }\n        }\n        return wasSuccessful;\n    }\n}\n#define MA_DR_FLAC_DEFINE_FULL_READ_AND_CLOSE(extension, type) \\\nstatic type* ma_dr_flac__full_read_and_close_ ## extension (ma_dr_flac* pFlac, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalPCMFrameCountOut)\\\n{                                                                                                                                                                   \\\n    type* pSampleData = NULL;                                                                                                                                       \\\n    ma_uint64 totalPCMFrameCount;                                                                                                                               \\\n    type buffer[4096];                                                                                                                                              \\\n    ma_uint64 pcmFramesRead;                                                                                                                                    \\\n    size_t sampleDataBufferSize = sizeof(buffer);                                                                                                                   \\\n                                                                                                                                                                    \\\n    MA_DR_FLAC_ASSERT(pFlac != NULL);                                                                                                                                   \\\n                                                                                                                                                                    \\\n    totalPCMFrameCount = 0;                                                                                                                                         \\\n                                                                                                                                                                    \\\n    pSampleData = (type*)ma_dr_flac__malloc_from_callbacks(sampleDataBufferSize, &pFlac->allocationCallbacks);                                                          \\\n    if (pSampleData == NULL) {                                                                                                                                      \\\n        goto on_error;                                                                                                                                              \\\n    }                                                                                                                                                               \\\n                                                                                                                                                                    \\\n    while ((pcmFramesRead = (ma_uint64)ma_dr_flac_read_pcm_frames_##extension(pFlac, sizeof(buffer)/sizeof(buffer[0])/pFlac->channels, buffer)) > 0) {              \\\n        if (((totalPCMFrameCount + pcmFramesRead) * pFlac->channels * sizeof(type)) > sampleDataBufferSize) {                                                       \\\n            type* pNewSampleData;                                                                                                                                   \\\n            size_t newSampleDataBufferSize;                                                                                                                         \\\n                                                                                                                                                                    \\\n            newSampleDataBufferSize = sampleDataBufferSize * 2;                                                                                                     \\\n            pNewSampleData = (type*)ma_dr_flac__realloc_from_callbacks(pSampleData, newSampleDataBufferSize, sampleDataBufferSize, &pFlac->allocationCallbacks);        \\\n            if (pNewSampleData == NULL) {                                                                                                                           \\\n                ma_dr_flac__free_from_callbacks(pSampleData, &pFlac->allocationCallbacks);                                                                              \\\n                goto on_error;                                                                                                                                      \\\n            }                                                                                                                                                       \\\n                                                                                                                                                                    \\\n            sampleDataBufferSize = newSampleDataBufferSize;                                                                                                         \\\n            pSampleData = pNewSampleData;                                                                                                                           \\\n        }                                                                                                                                                           \\\n                                                                                                                                                                    \\\n        MA_DR_FLAC_COPY_MEMORY(pSampleData + (totalPCMFrameCount*pFlac->channels), buffer, (size_t)(pcmFramesRead*pFlac->channels*sizeof(type)));                       \\\n        totalPCMFrameCount += pcmFramesRead;                                                                                                                        \\\n    }                                                                                                                                                               \\\n                                                                                                                                                                    \\\n                                                                                                                         \\\n    MA_DR_FLAC_ZERO_MEMORY(pSampleData + (totalPCMFrameCount*pFlac->channels), (size_t)(sampleDataBufferSize - totalPCMFrameCount*pFlac->channels*sizeof(type)));       \\\n                                                                                                                                                                    \\\n    if (sampleRateOut) *sampleRateOut = pFlac->sampleRate;                                                                                                          \\\n    if (channelsOut) *channelsOut = pFlac->channels;                                                                                                                \\\n    if (totalPCMFrameCountOut) *totalPCMFrameCountOut = totalPCMFrameCount;                                                                                         \\\n                                                                                                                                                                    \\\n    ma_dr_flac_close(pFlac);                                                                                                                                            \\\n    return pSampleData;                                                                                                                                             \\\n                                                                                                                                                                    \\\non_error:                                                                                                                                                           \\\n    ma_dr_flac_close(pFlac);                                                                                                                                            \\\n    return NULL;                                                                                                                                                    \\\n}\nMA_DR_FLAC_DEFINE_FULL_READ_AND_CLOSE(s32, ma_int32)\nMA_DR_FLAC_DEFINE_FULL_READ_AND_CLOSE(s16, ma_int16)\nMA_DR_FLAC_DEFINE_FULL_READ_AND_CLOSE(f32, float)\nMA_API ma_int32* ma_dr_flac_open_and_read_pcm_frames_s32(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalPCMFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalPCMFrameCountOut) {\n        *totalPCMFrameCountOut = 0;\n    }\n    pFlac = ma_dr_flac_open(onRead, onSeek, onTell, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s32(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut);\n}\nMA_API ma_int16* ma_dr_flac_open_and_read_pcm_frames_s16(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalPCMFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalPCMFrameCountOut) {\n        *totalPCMFrameCountOut = 0;\n    }\n    pFlac = ma_dr_flac_open(onRead, onSeek, onTell, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s16(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut);\n}\nMA_API float* ma_dr_flac_open_and_read_pcm_frames_f32(ma_dr_flac_read_proc onRead, ma_dr_flac_seek_proc onSeek, ma_dr_flac_tell_proc onTell, void* pUserData, unsigned int* channelsOut, unsigned int* sampleRateOut, ma_uint64* totalPCMFrameCountOut, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (channelsOut) {\n        *channelsOut = 0;\n    }\n    if (sampleRateOut) {\n        *sampleRateOut = 0;\n    }\n    if (totalPCMFrameCountOut) {\n        *totalPCMFrameCountOut = 0;\n    }\n    pFlac = ma_dr_flac_open(onRead, onSeek, onTell, pUserData, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_f32(pFlac, channelsOut, sampleRateOut, totalPCMFrameCountOut);\n}\n#ifndef MA_DR_FLAC_NO_STDIO\nMA_API ma_int32* ma_dr_flac_open_file_and_read_pcm_frames_s32(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_file(filename, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s32(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\nMA_API ma_int16* ma_dr_flac_open_file_and_read_pcm_frames_s16(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_file(filename, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s16(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\nMA_API float* ma_dr_flac_open_file_and_read_pcm_frames_f32(const char* filename, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_file(filename, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_f32(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\n#endif\nMA_API ma_int32* ma_dr_flac_open_memory_and_read_pcm_frames_s32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_memory(data, dataSize, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s32(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\nMA_API ma_int16* ma_dr_flac_open_memory_and_read_pcm_frames_s16(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_memory(data, dataSize, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_s16(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\nMA_API float* ma_dr_flac_open_memory_and_read_pcm_frames_f32(const void* data, size_t dataSize, unsigned int* channels, unsigned int* sampleRate, ma_uint64* totalPCMFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_flac* pFlac;\n    if (sampleRate) {\n        *sampleRate = 0;\n    }\n    if (channels) {\n        *channels = 0;\n    }\n    if (totalPCMFrameCount) {\n        *totalPCMFrameCount = 0;\n    }\n    pFlac = ma_dr_flac_open_memory(data, dataSize, pAllocationCallbacks);\n    if (pFlac == NULL) {\n        return NULL;\n    }\n    return ma_dr_flac__full_read_and_close_f32(pFlac, channels, sampleRate, totalPCMFrameCount);\n}\nMA_API void ma_dr_flac_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        ma_dr_flac__free_from_callbacks(p, pAllocationCallbacks);\n    } else {\n        ma_dr_flac__free_default(p, NULL);\n    }\n}\nMA_API void ma_dr_flac_init_vorbis_comment_iterator(ma_dr_flac_vorbis_comment_iterator* pIter, ma_uint32 commentCount, const void* pComments)\n{\n    if (pIter == NULL) {\n        return;\n    }\n    pIter->countRemaining = commentCount;\n    pIter->pRunningData   = (const char*)pComments;\n}\nMA_API const char* ma_dr_flac_next_vorbis_comment(ma_dr_flac_vorbis_comment_iterator* pIter, ma_uint32* pCommentLengthOut)\n{\n    ma_int32 length;\n    const char* pComment;\n    if (pCommentLengthOut) {\n        *pCommentLengthOut = 0;\n    }\n    if (pIter == NULL || pIter->countRemaining == 0 || pIter->pRunningData == NULL) {\n        return NULL;\n    }\n    length = ma_dr_flac__le2host_32_ptr_unaligned(pIter->pRunningData);\n    pIter->pRunningData += 4;\n    pComment = pIter->pRunningData;\n    pIter->pRunningData += length;\n    pIter->countRemaining -= 1;\n    if (pCommentLengthOut) {\n        *pCommentLengthOut = length;\n    }\n    return pComment;\n}\nMA_API void ma_dr_flac_init_cuesheet_track_iterator(ma_dr_flac_cuesheet_track_iterator* pIter, ma_uint32 trackCount, const void* pTrackData)\n{\n    if (pIter == NULL) {\n        return;\n    }\n    pIter->countRemaining = trackCount;\n    pIter->pRunningData   = (const char*)pTrackData;\n}\nMA_API ma_bool32 ma_dr_flac_next_cuesheet_track(ma_dr_flac_cuesheet_track_iterator* pIter, ma_dr_flac_cuesheet_track* pCuesheetTrack)\n{\n    ma_dr_flac_cuesheet_track cuesheetTrack;\n    const char* pRunningData;\n    ma_uint64 offsetHi;\n    ma_uint64 offsetLo;\n    if (pIter == NULL || pIter->countRemaining == 0 || pIter->pRunningData == NULL) {\n        return MA_FALSE;\n    }\n    pRunningData = pIter->pRunningData;\n    offsetHi                   = ma_dr_flac__be2host_32(*(const ma_uint32*)pRunningData); pRunningData += 4;\n    offsetLo                   = ma_dr_flac__be2host_32(*(const ma_uint32*)pRunningData); pRunningData += 4;\n    cuesheetTrack.offset       = offsetLo | (offsetHi << 32);\n    cuesheetTrack.trackNumber  = pRunningData[0];                                         pRunningData += 1;\n    MA_DR_FLAC_COPY_MEMORY(cuesheetTrack.ISRC, pRunningData, sizeof(cuesheetTrack.ISRC));     pRunningData += 12;\n    cuesheetTrack.isAudio      = (pRunningData[0] & 0x80) != 0;\n    cuesheetTrack.preEmphasis  = (pRunningData[0] & 0x40) != 0;                           pRunningData += 14;\n    cuesheetTrack.indexCount   = pRunningData[0];                                         pRunningData += 1;\n    cuesheetTrack.pIndexPoints = (const ma_dr_flac_cuesheet_track_index*)pRunningData;        pRunningData += cuesheetTrack.indexCount * sizeof(ma_dr_flac_cuesheet_track_index);\n    pIter->pRunningData = pRunningData;\n    pIter->countRemaining -= 1;\n    if (pCuesheetTrack) {\n        *pCuesheetTrack = cuesheetTrack;\n    }\n    return MA_TRUE;\n}\n#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)))\n    #pragma GCC diagnostic pop\n#endif\n#endif\n/* dr_flac_c end */\n#endif  /* MA_DR_FLAC_IMPLEMENTATION */\n#endif  /* MA_NO_FLAC */\n\n#if !defined(MA_NO_MP3) && !defined(MA_NO_DECODING)\n#if !defined(MA_DR_MP3_IMPLEMENTATION)\n/* dr_mp3_c begin */\n#ifndef ma_dr_mp3_c\n#define ma_dr_mp3_c\n#include <stdlib.h>\n#include <string.h>\n#include <limits.h>\nMA_API void ma_dr_mp3_version(ma_uint32* pMajor, ma_uint32* pMinor, ma_uint32* pRevision)\n{\n    if (pMajor) {\n        *pMajor = MA_DR_MP3_VERSION_MAJOR;\n    }\n    if (pMinor) {\n        *pMinor = MA_DR_MP3_VERSION_MINOR;\n    }\n    if (pRevision) {\n        *pRevision = MA_DR_MP3_VERSION_REVISION;\n    }\n}\nMA_API const char* ma_dr_mp3_version_string(void)\n{\n    return MA_DR_MP3_VERSION_STRING;\n}\n#if defined(__TINYC__)\n#define MA_DR_MP3_NO_SIMD\n#endif\n#define MA_DR_MP3_OFFSET_PTR(p, offset) ((void*)((ma_uint8*)(p) + (offset)))\n#ifndef MA_DR_MP3_MAX_FRAME_SYNC_MATCHES\n#define MA_DR_MP3_MAX_FRAME_SYNC_MATCHES      10\n#endif\n#define MA_DR_MP3_SHORT_BLOCK_TYPE            2\n#define MA_DR_MP3_STOP_BLOCK_TYPE             3\n#define MA_DR_MP3_MODE_MONO                   3\n#define MA_DR_MP3_MODE_JOINT_STEREO           1\n#define MA_DR_MP3_HDR_SIZE                    4\n#define MA_DR_MP3_HDR_IS_MONO(h)              (((h[3]) & 0xC0) == 0xC0)\n#define MA_DR_MP3_HDR_IS_MS_STEREO(h)         (((h[3]) & 0xE0) == 0x60)\n#define MA_DR_MP3_HDR_IS_FREE_FORMAT(h)       (((h[2]) & 0xF0) == 0)\n#define MA_DR_MP3_HDR_IS_CRC(h)               (!((h[1]) & 1))\n#define MA_DR_MP3_HDR_TEST_PADDING(h)         ((h[2]) & 0x2)\n#define MA_DR_MP3_HDR_TEST_MPEG1(h)           ((h[1]) & 0x8)\n#define MA_DR_MP3_HDR_TEST_NOT_MPEG25(h)      ((h[1]) & 0x10)\n#define MA_DR_MP3_HDR_TEST_I_STEREO(h)        ((h[3]) & 0x10)\n#define MA_DR_MP3_HDR_TEST_MS_STEREO(h)       ((h[3]) & 0x20)\n#define MA_DR_MP3_HDR_GET_STEREO_MODE(h)      (((h[3]) >> 6) & 3)\n#define MA_DR_MP3_HDR_GET_STEREO_MODE_EXT(h)  (((h[3]) >> 4) & 3)\n#define MA_DR_MP3_HDR_GET_LAYER(h)            (((h[1]) >> 1) & 3)\n#define MA_DR_MP3_HDR_GET_BITRATE(h)          ((h[2]) >> 4)\n#define MA_DR_MP3_HDR_GET_SAMPLE_RATE(h)      (((h[2]) >> 2) & 3)\n#define MA_DR_MP3_HDR_GET_MY_SAMPLE_RATE(h)   (MA_DR_MP3_HDR_GET_SAMPLE_RATE(h) + (((h[1] >> 3) & 1) + ((h[1] >> 4) & 1))*3)\n#define MA_DR_MP3_HDR_IS_FRAME_576(h)         ((h[1] & 14) == 2)\n#define MA_DR_MP3_HDR_IS_LAYER_1(h)           ((h[1] & 6) == 6)\n#define MA_DR_MP3_BITS_DEQUANTIZER_OUT        -1\n#define MA_DR_MP3_MAX_SCF                     (255 + MA_DR_MP3_BITS_DEQUANTIZER_OUT*4 - 210)\n#define MA_DR_MP3_MAX_SCFI                    ((MA_DR_MP3_MAX_SCF + 3) & ~3)\n#define MA_DR_MP3_MIN(a, b)           ((a) > (b) ? (b) : (a))\n#define MA_DR_MP3_MAX(a, b)           ((a) < (b) ? (b) : (a))\n#if !defined(MA_DR_MP3_NO_SIMD)\n#if !defined(MA_DR_MP3_ONLY_SIMD) && (defined(_M_X64) || defined(__x86_64__) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC))\n#define MA_DR_MP3_ONLY_SIMD\n#endif\n#if ((defined(_MSC_VER) && _MSC_VER >= 1400) && defined(_M_X64)) || ((defined(__i386) || defined(_M_IX86) || defined(__i386__) || defined(__x86_64__)) && ((defined(_M_IX86_FP) && _M_IX86_FP == 2) || defined(__SSE2__)))\n#if defined(_MSC_VER)\n#include <intrin.h>\n#endif\n#include <emmintrin.h>\n#define MA_DR_MP3_HAVE_SSE 1\n#define MA_DR_MP3_HAVE_SIMD 1\n#define MA_DR_MP3_VSTORE _mm_storeu_ps\n#define MA_DR_MP3_VLD _mm_loadu_ps\n#define MA_DR_MP3_VSET _mm_set1_ps\n#define MA_DR_MP3_VADD _mm_add_ps\n#define MA_DR_MP3_VSUB _mm_sub_ps\n#define MA_DR_MP3_VMUL _mm_mul_ps\n#define MA_DR_MP3_VMAC(a, x, y) _mm_add_ps(a, _mm_mul_ps(x, y))\n#define MA_DR_MP3_VMSB(a, x, y) _mm_sub_ps(a, _mm_mul_ps(x, y))\n#define MA_DR_MP3_VMUL_S(x, s)  _mm_mul_ps(x, _mm_set1_ps(s))\n#define MA_DR_MP3_VREV(x) _mm_shuffle_ps(x, x, _MM_SHUFFLE(0, 1, 2, 3))\ntypedef __m128 ma_dr_mp3_f4;\n#if (defined(_MSC_VER) || defined(MA_DR_MP3_ONLY_SIMD)) && !defined(__clang__)\n#define ma_dr_mp3_cpuid __cpuid\n#else\nstatic __inline__ __attribute__((always_inline)) void ma_dr_mp3_cpuid(int CPUInfo[], const int InfoType)\n{\n#if defined(__PIC__)\n    __asm__ __volatile__(\n#if defined(__x86_64__)\n        \"push %%rbx\\n\"\n        \"cpuid\\n\"\n        \"xchgl %%ebx, %1\\n\"\n        \"pop  %%rbx\\n\"\n#else\n        \"xchgl %%ebx, %1\\n\"\n        \"cpuid\\n\"\n        \"xchgl %%ebx, %1\\n\"\n#endif\n        : \"=a\" (CPUInfo[0]), \"=r\" (CPUInfo[1]), \"=c\" (CPUInfo[2]), \"=d\" (CPUInfo[3])\n        : \"a\" (InfoType));\n#else\n    __asm__ __volatile__(\n        \"cpuid\"\n        : \"=a\" (CPUInfo[0]), \"=b\" (CPUInfo[1]), \"=c\" (CPUInfo[2]), \"=d\" (CPUInfo[3])\n        : \"a\" (InfoType));\n#endif\n}\n#endif\nstatic int ma_dr_mp3_have_simd(void)\n{\n#ifdef MA_DR_MP3_ONLY_SIMD\n    return 1;\n#else\n    static int g_have_simd;\n    int CPUInfo[4];\n#ifdef MINIMP3_TEST\n    static int g_counter;\n    if (g_counter++ > 100)\n        return 0;\n#endif\n    if (g_have_simd)\n        goto end;\n    ma_dr_mp3_cpuid(CPUInfo, 0);\n    if (CPUInfo[0] > 0)\n    {\n        ma_dr_mp3_cpuid(CPUInfo, 1);\n        g_have_simd = (CPUInfo[3] & (1 << 26)) + 1;\n        return g_have_simd - 1;\n    }\nend:\n    return g_have_simd - 1;\n#endif\n}\n#elif defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)\n#include <arm_neon.h>\n#define MA_DR_MP3_HAVE_SSE 0\n#define MA_DR_MP3_HAVE_SIMD 1\n#define MA_DR_MP3_VSTORE vst1q_f32\n#define MA_DR_MP3_VLD vld1q_f32\n#define MA_DR_MP3_VSET vmovq_n_f32\n#define MA_DR_MP3_VADD vaddq_f32\n#define MA_DR_MP3_VSUB vsubq_f32\n#define MA_DR_MP3_VMUL vmulq_f32\n#define MA_DR_MP3_VMAC(a, x, y) vmlaq_f32(a, x, y)\n#define MA_DR_MP3_VMSB(a, x, y) vmlsq_f32(a, x, y)\n#define MA_DR_MP3_VMUL_S(x, s)  vmulq_f32(x, vmovq_n_f32(s))\n#define MA_DR_MP3_VREV(x) vcombine_f32(vget_high_f32(vrev64q_f32(x)), vget_low_f32(vrev64q_f32(x)))\ntypedef float32x4_t ma_dr_mp3_f4;\nstatic int ma_dr_mp3_have_simd(void)\n{\n    return 1;\n}\n#else\n#define MA_DR_MP3_HAVE_SSE 0\n#define MA_DR_MP3_HAVE_SIMD 0\n#ifdef MA_DR_MP3_ONLY_SIMD\n#error MA_DR_MP3_ONLY_SIMD used, but SSE/NEON not enabled\n#endif\n#endif\n#else\n#define MA_DR_MP3_HAVE_SIMD 0\n#endif\n#if defined(__ARM_ARCH) && (__ARM_ARCH >= 6) && !defined(__aarch64__) && !defined(_M_ARM64) && !defined(_M_ARM64EC) && !defined(__ARM_ARCH_6M__)\n#define MA_DR_MP3_HAVE_ARMV6 1\nstatic __inline__ __attribute__((always_inline)) ma_int32 ma_dr_mp3_clip_int16_arm(ma_int32 a)\n{\n    ma_int32 x = 0;\n    __asm__ (\"ssat %0, #16, %1\" : \"=r\"(x) : \"r\"(a));\n    return x;\n}\n#else\n#define MA_DR_MP3_HAVE_ARMV6 0\n#endif\n#ifndef MA_DR_MP3_ASSERT\n#include <assert.h>\n#define MA_DR_MP3_ASSERT(expression) assert(expression)\n#endif\n#ifndef MA_DR_MP3_COPY_MEMORY\n#define MA_DR_MP3_COPY_MEMORY(dst, src, sz) memcpy((dst), (src), (sz))\n#endif\n#ifndef MA_DR_MP3_MOVE_MEMORY\n#define MA_DR_MP3_MOVE_MEMORY(dst, src, sz) memmove((dst), (src), (sz))\n#endif\n#ifndef MA_DR_MP3_ZERO_MEMORY\n#define MA_DR_MP3_ZERO_MEMORY(p, sz) memset((p), 0, (sz))\n#endif\n#define MA_DR_MP3_ZERO_OBJECT(p) MA_DR_MP3_ZERO_MEMORY((p), sizeof(*(p)))\n#ifndef MA_DR_MP3_MALLOC\n#define MA_DR_MP3_MALLOC(sz) malloc((sz))\n#endif\n#ifndef MA_DR_MP3_REALLOC\n#define MA_DR_MP3_REALLOC(p, sz) realloc((p), (sz))\n#endif\n#ifndef MA_DR_MP3_FREE\n#define MA_DR_MP3_FREE(p) free((p))\n#endif\ntypedef struct\n{\n    float scf[3*64];\n    ma_uint8 total_bands, stereo_bands, bitalloc[64], scfcod[64];\n} ma_dr_mp3_L12_scale_info;\ntypedef struct\n{\n    ma_uint8 tab_offset, code_tab_width, band_count;\n} ma_dr_mp3_L12_subband_alloc;\nstatic void ma_dr_mp3_bs_init(ma_dr_mp3_bs *bs, const ma_uint8 *data, int bytes)\n{\n    bs->buf   = data;\n    bs->pos   = 0;\n    bs->limit = bytes*8;\n}\nstatic ma_uint32 ma_dr_mp3_bs_get_bits(ma_dr_mp3_bs *bs, int n)\n{\n    ma_uint32 next, cache = 0, s = bs->pos & 7;\n    int shl = n + s;\n    const ma_uint8 *p = bs->buf + (bs->pos >> 3);\n    if ((bs->pos += n) > bs->limit)\n        return 0;\n    next = *p++ & (255 >> s);\n    while ((shl -= 8) > 0)\n    {\n        cache |= next << shl;\n        next = *p++;\n    }\n    return cache | (next >> -shl);\n}\nstatic int ma_dr_mp3_hdr_valid(const ma_uint8 *h)\n{\n    return h[0] == 0xff &&\n        ((h[1] & 0xF0) == 0xf0 || (h[1] & 0xFE) == 0xe2) &&\n        (MA_DR_MP3_HDR_GET_LAYER(h) != 0) &&\n        (MA_DR_MP3_HDR_GET_BITRATE(h) != 15) &&\n        (MA_DR_MP3_HDR_GET_SAMPLE_RATE(h) != 3);\n}\nstatic int ma_dr_mp3_hdr_compare(const ma_uint8 *h1, const ma_uint8 *h2)\n{\n    return ma_dr_mp3_hdr_valid(h2) &&\n        ((h1[1] ^ h2[1]) & 0xFE) == 0 &&\n        ((h1[2] ^ h2[2]) & 0x0C) == 0 &&\n        !(MA_DR_MP3_HDR_IS_FREE_FORMAT(h1) ^ MA_DR_MP3_HDR_IS_FREE_FORMAT(h2));\n}\nstatic unsigned ma_dr_mp3_hdr_bitrate_kbps(const ma_uint8 *h)\n{\n    static const ma_uint8 halfrate[2][3][15] = {\n        { { 0,4,8,12,16,20,24,28,32,40,48,56,64,72,80 }, { 0,4,8,12,16,20,24,28,32,40,48,56,64,72,80 }, { 0,16,24,28,32,40,48,56,64,72,80,88,96,112,128 } },\n        { { 0,16,20,24,28,32,40,48,56,64,80,96,112,128,160 }, { 0,16,24,28,32,40,48,56,64,80,96,112,128,160,192 }, { 0,16,32,48,64,80,96,112,128,144,160,176,192,208,224 } },\n    };\n    return 2*halfrate[!!MA_DR_MP3_HDR_TEST_MPEG1(h)][MA_DR_MP3_HDR_GET_LAYER(h) - 1][MA_DR_MP3_HDR_GET_BITRATE(h)];\n}\nstatic unsigned ma_dr_mp3_hdr_sample_rate_hz(const ma_uint8 *h)\n{\n    static const unsigned g_hz[3] = { 44100, 48000, 32000 };\n    return g_hz[MA_DR_MP3_HDR_GET_SAMPLE_RATE(h)] >> (int)!MA_DR_MP3_HDR_TEST_MPEG1(h) >> (int)!MA_DR_MP3_HDR_TEST_NOT_MPEG25(h);\n}\nstatic unsigned ma_dr_mp3_hdr_frame_samples(const ma_uint8 *h)\n{\n    return MA_DR_MP3_HDR_IS_LAYER_1(h) ? 384 : (1152 >> (int)MA_DR_MP3_HDR_IS_FRAME_576(h));\n}\nstatic int ma_dr_mp3_hdr_frame_bytes(const ma_uint8 *h, int free_format_size)\n{\n    int frame_bytes = ma_dr_mp3_hdr_frame_samples(h)*ma_dr_mp3_hdr_bitrate_kbps(h)*125/ma_dr_mp3_hdr_sample_rate_hz(h);\n    if (MA_DR_MP3_HDR_IS_LAYER_1(h))\n    {\n        frame_bytes &= ~3;\n    }\n    return frame_bytes ? frame_bytes : free_format_size;\n}\nstatic int ma_dr_mp3_hdr_padding(const ma_uint8 *h)\n{\n    return MA_DR_MP3_HDR_TEST_PADDING(h) ? (MA_DR_MP3_HDR_IS_LAYER_1(h) ? 4 : 1) : 0;\n}\n#ifndef MA_DR_MP3_ONLY_MP3\nstatic const ma_dr_mp3_L12_subband_alloc *ma_dr_mp3_L12_subband_alloc_table(const ma_uint8 *hdr, ma_dr_mp3_L12_scale_info *sci)\n{\n    const ma_dr_mp3_L12_subband_alloc *alloc;\n    int mode = MA_DR_MP3_HDR_GET_STEREO_MODE(hdr);\n    int nbands, stereo_bands = (mode == MA_DR_MP3_MODE_MONO) ? 0 : (mode == MA_DR_MP3_MODE_JOINT_STEREO) ? (MA_DR_MP3_HDR_GET_STEREO_MODE_EXT(hdr) << 2) + 4 : 32;\n    if (MA_DR_MP3_HDR_IS_LAYER_1(hdr))\n    {\n        static const ma_dr_mp3_L12_subband_alloc g_alloc_L1[] = { { 76, 4, 32 } };\n        alloc = g_alloc_L1;\n        nbands = 32;\n    } else if (!MA_DR_MP3_HDR_TEST_MPEG1(hdr))\n    {\n        static const ma_dr_mp3_L12_subband_alloc g_alloc_L2M2[] = { { 60, 4, 4 }, { 44, 3, 7 }, { 44, 2, 19 } };\n        alloc = g_alloc_L2M2;\n        nbands = 30;\n    } else\n    {\n        static const ma_dr_mp3_L12_subband_alloc g_alloc_L2M1[] = { { 0, 4, 3 }, { 16, 4, 8 }, { 32, 3, 12 }, { 40, 2, 7 } };\n        int sample_rate_idx = MA_DR_MP3_HDR_GET_SAMPLE_RATE(hdr);\n        unsigned kbps = ma_dr_mp3_hdr_bitrate_kbps(hdr) >> (int)(mode != MA_DR_MP3_MODE_MONO);\n        if (!kbps)\n        {\n            kbps = 192;\n        }\n        alloc = g_alloc_L2M1;\n        nbands = 27;\n        if (kbps < 56)\n        {\n            static const ma_dr_mp3_L12_subband_alloc g_alloc_L2M1_lowrate[] = { { 44, 4, 2 }, { 44, 3, 10 } };\n            alloc = g_alloc_L2M1_lowrate;\n            nbands = sample_rate_idx == 2 ? 12 : 8;\n        } else if (kbps >= 96 && sample_rate_idx != 1)\n        {\n            nbands = 30;\n        }\n    }\n    sci->total_bands = (ma_uint8)nbands;\n    sci->stereo_bands = (ma_uint8)MA_DR_MP3_MIN(stereo_bands, nbands);\n    return alloc;\n}\nstatic void ma_dr_mp3_L12_read_scalefactors(ma_dr_mp3_bs *bs, ma_uint8 *pba, ma_uint8 *scfcod, int bands, float *scf)\n{\n    static const float g_deq_L12[18*3] = {\n#define MA_DR_MP3_DQ(x) 9.53674316e-07f/x, 7.56931807e-07f/x, 6.00777173e-07f/x\n        MA_DR_MP3_DQ(3),MA_DR_MP3_DQ(7),MA_DR_MP3_DQ(15),MA_DR_MP3_DQ(31),MA_DR_MP3_DQ(63),MA_DR_MP3_DQ(127),MA_DR_MP3_DQ(255),MA_DR_MP3_DQ(511),MA_DR_MP3_DQ(1023),MA_DR_MP3_DQ(2047),MA_DR_MP3_DQ(4095),MA_DR_MP3_DQ(8191),MA_DR_MP3_DQ(16383),MA_DR_MP3_DQ(32767),MA_DR_MP3_DQ(65535),MA_DR_MP3_DQ(3),MA_DR_MP3_DQ(5),MA_DR_MP3_DQ(9)\n    };\n    int i, m;\n    for (i = 0; i < bands; i++)\n    {\n        float s = 0;\n        int ba = *pba++;\n        int mask = ba ? 4 + ((19 >> scfcod[i]) & 3) : 0;\n        for (m = 4; m; m >>= 1)\n        {\n            if (mask & m)\n            {\n                int b = ma_dr_mp3_bs_get_bits(bs, 6);\n                s = g_deq_L12[ba*3 - 6 + b % 3]*(int)(1 << 21 >> b/3);\n            }\n            *scf++ = s;\n        }\n    }\n}\nstatic void ma_dr_mp3_L12_read_scale_info(const ma_uint8 *hdr, ma_dr_mp3_bs *bs, ma_dr_mp3_L12_scale_info *sci)\n{\n    static const ma_uint8 g_bitalloc_code_tab[] = {\n        0,17, 3, 4, 5,6,7, 8,9,10,11,12,13,14,15,16,\n        0,17,18, 3,19,4,5, 6,7, 8, 9,10,11,12,13,16,\n        0,17,18, 3,19,4,5,16,\n        0,17,18,16,\n        0,17,18,19, 4,5,6, 7,8, 9,10,11,12,13,14,15,\n        0,17,18, 3,19,4,5, 6,7, 8, 9,10,11,12,13,14,\n        0, 2, 3, 4, 5,6,7, 8,9,10,11,12,13,14,15,16\n    };\n    const ma_dr_mp3_L12_subband_alloc *subband_alloc = ma_dr_mp3_L12_subband_alloc_table(hdr, sci);\n    int i, k = 0, ba_bits = 0;\n    const ma_uint8 *ba_code_tab = g_bitalloc_code_tab;\n    for (i = 0; i < sci->total_bands; i++)\n    {\n        ma_uint8 ba;\n        if (i == k)\n        {\n            k += subband_alloc->band_count;\n            ba_bits = subband_alloc->code_tab_width;\n            ba_code_tab = g_bitalloc_code_tab + subband_alloc->tab_offset;\n            subband_alloc++;\n        }\n        ba = ba_code_tab[ma_dr_mp3_bs_get_bits(bs, ba_bits)];\n        sci->bitalloc[2*i] = ba;\n        if (i < sci->stereo_bands)\n        {\n            ba = ba_code_tab[ma_dr_mp3_bs_get_bits(bs, ba_bits)];\n        }\n        sci->bitalloc[2*i + 1] = sci->stereo_bands ? ba : 0;\n    }\n    for (i = 0; i < 2*sci->total_bands; i++)\n    {\n        sci->scfcod[i] = (ma_uint8)(sci->bitalloc[i] ? MA_DR_MP3_HDR_IS_LAYER_1(hdr) ? 2 : ma_dr_mp3_bs_get_bits(bs, 2) : 6);\n    }\n    ma_dr_mp3_L12_read_scalefactors(bs, sci->bitalloc, sci->scfcod, sci->total_bands*2, sci->scf);\n    for (i = sci->stereo_bands; i < sci->total_bands; i++)\n    {\n        sci->bitalloc[2*i + 1] = 0;\n    }\n}\nstatic int ma_dr_mp3_L12_dequantize_granule(float *grbuf, ma_dr_mp3_bs *bs, ma_dr_mp3_L12_scale_info *sci, int group_size)\n{\n    int i, j, k, choff = 576;\n    for (j = 0; j < 4; j++)\n    {\n        float *dst = grbuf + group_size*j;\n        for (i = 0; i < 2*sci->total_bands; i++)\n        {\n            int ba = sci->bitalloc[i];\n            if (ba != 0)\n            {\n                if (ba < 17)\n                {\n                    int half = (1 << (ba - 1)) - 1;\n                    for (k = 0; k < group_size; k++)\n                    {\n                        dst[k] = (float)((int)ma_dr_mp3_bs_get_bits(bs, ba) - half);\n                    }\n                } else\n                {\n                    unsigned mod = (2 << (ba - 17)) + 1;\n                    unsigned code = ma_dr_mp3_bs_get_bits(bs, mod + 2 - (mod >> 3));\n                    for (k = 0; k < group_size; k++, code /= mod)\n                    {\n                        dst[k] = (float)((int)(code % mod - mod/2));\n                    }\n                }\n            }\n            dst += choff;\n            choff = 18 - choff;\n        }\n    }\n    return group_size*4;\n}\nstatic void ma_dr_mp3_L12_apply_scf_384(ma_dr_mp3_L12_scale_info *sci, const float *scf, float *dst)\n{\n    int i, k;\n    MA_DR_MP3_COPY_MEMORY(dst + 576 + sci->stereo_bands*18, dst + sci->stereo_bands*18, (sci->total_bands - sci->stereo_bands)*18*sizeof(float));\n    for (i = 0; i < sci->total_bands; i++, dst += 18, scf += 6)\n    {\n        for (k = 0; k < 12; k++)\n        {\n            dst[k + 0]   *= scf[0];\n            dst[k + 576] *= scf[3];\n        }\n    }\n}\n#endif\nstatic int ma_dr_mp3_L3_read_side_info(ma_dr_mp3_bs *bs, ma_dr_mp3_L3_gr_info *gr, const ma_uint8 *hdr)\n{\n    static const ma_uint8 g_scf_long[8][23] = {\n        { 6,6,6,6,6,6,8,10,12,14,16,20,24,28,32,38,46,52,60,68,58,54,0 },\n        { 12,12,12,12,12,12,16,20,24,28,32,40,48,56,64,76,90,2,2,2,2,2,0 },\n        { 6,6,6,6,6,6,8,10,12,14,16,20,24,28,32,38,46,52,60,68,58,54,0 },\n        { 6,6,6,6,6,6,8,10,12,14,16,18,22,26,32,38,46,54,62,70,76,36,0 },\n        { 6,6,6,6,6,6,8,10,12,14,16,20,24,28,32,38,46,52,60,68,58,54,0 },\n        { 4,4,4,4,4,4,6,6,8,8,10,12,16,20,24,28,34,42,50,54,76,158,0 },\n        { 4,4,4,4,4,4,6,6,6,8,10,12,16,18,22,28,34,40,46,54,54,192,0 },\n        { 4,4,4,4,4,4,6,6,8,10,12,16,20,24,30,38,46,56,68,84,102,26,0 }\n    };\n    static const ma_uint8 g_scf_short[8][40] = {\n        { 4,4,4,4,4,4,4,4,4,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,30,30,30,40,40,40,18,18,18,0 },\n        { 8,8,8,8,8,8,8,8,8,12,12,12,16,16,16,20,20,20,24,24,24,28,28,28,36,36,36,2,2,2,2,2,2,2,2,2,26,26,26,0 },\n        { 4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,8,8,8,10,10,10,14,14,14,18,18,18,26,26,26,32,32,32,42,42,42,18,18,18,0 },\n        { 4,4,4,4,4,4,4,4,4,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,32,32,32,44,44,44,12,12,12,0 },\n        { 4,4,4,4,4,4,4,4,4,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,30,30,30,40,40,40,18,18,18,0 },\n        { 4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,22,22,22,30,30,30,56,56,56,0 },\n        { 4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,6,6,6,10,10,10,12,12,12,14,14,14,16,16,16,20,20,20,26,26,26,66,66,66,0 },\n        { 4,4,4,4,4,4,4,4,4,4,4,4,6,6,6,8,8,8,12,12,12,16,16,16,20,20,20,26,26,26,34,34,34,42,42,42,12,12,12,0 }\n    };\n    static const ma_uint8 g_scf_mixed[8][40] = {\n        { 6,6,6,6,6,6,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,30,30,30,40,40,40,18,18,18,0 },\n        { 12,12,12,4,4,4,8,8,8,12,12,12,16,16,16,20,20,20,24,24,24,28,28,28,36,36,36,2,2,2,2,2,2,2,2,2,26,26,26,0 },\n        { 6,6,6,6,6,6,6,6,6,6,6,6,8,8,8,10,10,10,14,14,14,18,18,18,26,26,26,32,32,32,42,42,42,18,18,18,0 },\n        { 6,6,6,6,6,6,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,32,32,32,44,44,44,12,12,12,0 },\n        { 6,6,6,6,6,6,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,24,24,24,30,30,30,40,40,40,18,18,18,0 },\n        { 4,4,4,4,4,4,6,6,4,4,4,6,6,6,8,8,8,10,10,10,12,12,12,14,14,14,18,18,18,22,22,22,30,30,30,56,56,56,0 },\n        { 4,4,4,4,4,4,6,6,4,4,4,6,6,6,6,6,6,10,10,10,12,12,12,14,14,14,16,16,16,20,20,20,26,26,26,66,66,66,0 },\n        { 4,4,4,4,4,4,6,6,4,4,4,6,6,6,8,8,8,12,12,12,16,16,16,20,20,20,26,26,26,34,34,34,42,42,42,12,12,12,0 }\n    };\n    unsigned tables, scfsi = 0;\n    int main_data_begin, part_23_sum = 0;\n    int gr_count = MA_DR_MP3_HDR_IS_MONO(hdr) ? 1 : 2;\n    int sr_idx = MA_DR_MP3_HDR_GET_MY_SAMPLE_RATE(hdr); sr_idx -= (sr_idx != 0);\n    if (MA_DR_MP3_HDR_TEST_MPEG1(hdr))\n    {\n        gr_count *= 2;\n        main_data_begin = ma_dr_mp3_bs_get_bits(bs, 9);\n        scfsi = ma_dr_mp3_bs_get_bits(bs, 7 + gr_count);\n    } else\n    {\n        main_data_begin = ma_dr_mp3_bs_get_bits(bs, 8 + gr_count) >> gr_count;\n    }\n    do\n    {\n        if (MA_DR_MP3_HDR_IS_MONO(hdr))\n        {\n            scfsi <<= 4;\n        }\n        gr->part_23_length = (ma_uint16)ma_dr_mp3_bs_get_bits(bs, 12);\n        part_23_sum += gr->part_23_length;\n        gr->big_values = (ma_uint16)ma_dr_mp3_bs_get_bits(bs,  9);\n        if (gr->big_values > 288)\n        {\n            return -1;\n        }\n        gr->global_gain = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 8);\n        gr->scalefac_compress = (ma_uint16)ma_dr_mp3_bs_get_bits(bs, MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? 4 : 9);\n        gr->sfbtab = g_scf_long[sr_idx];\n        gr->n_long_sfb  = 22;\n        gr->n_short_sfb = 0;\n        if (ma_dr_mp3_bs_get_bits(bs, 1))\n        {\n            gr->block_type = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 2);\n            if (!gr->block_type)\n            {\n                return -1;\n            }\n            gr->mixed_block_flag = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 1);\n            gr->region_count[0] = 7;\n            gr->region_count[1] = 255;\n            if (gr->block_type == MA_DR_MP3_SHORT_BLOCK_TYPE)\n            {\n                scfsi &= 0x0F0F;\n                if (!gr->mixed_block_flag)\n                {\n                    gr->region_count[0] = 8;\n                    gr->sfbtab = g_scf_short[sr_idx];\n                    gr->n_long_sfb = 0;\n                    gr->n_short_sfb = 39;\n                } else\n                {\n                    gr->sfbtab = g_scf_mixed[sr_idx];\n                    gr->n_long_sfb = MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? 8 : 6;\n                    gr->n_short_sfb = 30;\n                }\n            }\n            tables = ma_dr_mp3_bs_get_bits(bs, 10);\n            tables <<= 5;\n            gr->subblock_gain[0] = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 3);\n            gr->subblock_gain[1] = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 3);\n            gr->subblock_gain[2] = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 3);\n        } else\n        {\n            gr->block_type = 0;\n            gr->mixed_block_flag = 0;\n            tables = ma_dr_mp3_bs_get_bits(bs, 15);\n            gr->region_count[0] = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 4);\n            gr->region_count[1] = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 3);\n            gr->region_count[2] = 255;\n        }\n        gr->table_select[0] = (ma_uint8)(tables >> 10);\n        gr->table_select[1] = (ma_uint8)((tables >> 5) & 31);\n        gr->table_select[2] = (ma_uint8)((tables) & 31);\n        gr->preflag = (ma_uint8)(MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? ma_dr_mp3_bs_get_bits(bs, 1) : (gr->scalefac_compress >= 500));\n        gr->scalefac_scale = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 1);\n        gr->count1_table = (ma_uint8)ma_dr_mp3_bs_get_bits(bs, 1);\n        gr->scfsi = (ma_uint8)((scfsi >> 12) & 15);\n        scfsi <<= 4;\n        gr++;\n    } while(--gr_count);\n    if (part_23_sum + bs->pos > bs->limit + main_data_begin*8)\n    {\n        return -1;\n    }\n    return main_data_begin;\n}\nstatic void ma_dr_mp3_L3_read_scalefactors(ma_uint8 *scf, ma_uint8 *ist_pos, const ma_uint8 *scf_size, const ma_uint8 *scf_count, ma_dr_mp3_bs *bitbuf, int scfsi)\n{\n    int i, k;\n    for (i = 0; i < 4 && scf_count[i]; i++, scfsi *= 2)\n    {\n        int cnt = scf_count[i];\n        if (scfsi & 8)\n        {\n            MA_DR_MP3_COPY_MEMORY(scf, ist_pos, cnt);\n        } else\n        {\n            int bits = scf_size[i];\n            if (!bits)\n            {\n                MA_DR_MP3_ZERO_MEMORY(scf, cnt);\n                MA_DR_MP3_ZERO_MEMORY(ist_pos, cnt);\n            } else\n            {\n                int max_scf = (scfsi < 0) ? (1 << bits) - 1 : -1;\n                for (k = 0; k < cnt; k++)\n                {\n                    int s = ma_dr_mp3_bs_get_bits(bitbuf, bits);\n                    ist_pos[k] = (ma_uint8)(s == max_scf ? -1 : s);\n                    scf[k] = (ma_uint8)s;\n                }\n            }\n        }\n        ist_pos += cnt;\n        scf += cnt;\n    }\n    scf[0] = scf[1] = scf[2] = 0;\n}\nstatic float ma_dr_mp3_L3_ldexp_q2(float y, int exp_q2)\n{\n    static const float g_expfrac[4] = { 9.31322575e-10f,7.83145814e-10f,6.58544508e-10f,5.53767716e-10f };\n    int e;\n    do\n    {\n        e = MA_DR_MP3_MIN(30*4, exp_q2);\n        y *= g_expfrac[e & 3]*(1 << 30 >> (e >> 2));\n    } while ((exp_q2 -= e) > 0);\n    return y;\n}\n#if (defined(__GNUC__) && (__GNUC__ >= 13)) && !defined(__clang__)\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wstringop-overflow\"\n#endif\nstatic void ma_dr_mp3_L3_decode_scalefactors(const ma_uint8 *hdr, ma_uint8 *ist_pos, ma_dr_mp3_bs *bs, const ma_dr_mp3_L3_gr_info *gr, float *scf, int ch)\n{\n    static const ma_uint8 g_scf_partitions[3][28] = {\n        { 6,5,5, 5,6,5,5,5,6,5, 7,3,11,10,0,0, 7, 7, 7,0, 6, 6,6,3, 8, 8,5,0 },\n        { 8,9,6,12,6,9,9,9,6,9,12,6,15,18,0,0, 6,15,12,0, 6,12,9,6, 6,18,9,0 },\n        { 9,9,6,12,9,9,9,9,9,9,12,6,18,18,0,0,12,12,12,0,12, 9,9,6,15,12,9,0 }\n    };\n    const ma_uint8 *scf_partition = g_scf_partitions[!!gr->n_short_sfb + !gr->n_long_sfb];\n    ma_uint8 scf_size[4], iscf[40];\n    int i, scf_shift = gr->scalefac_scale + 1, gain_exp, scfsi = gr->scfsi;\n    float gain;\n    if (MA_DR_MP3_HDR_TEST_MPEG1(hdr))\n    {\n        static const ma_uint8 g_scfc_decode[16] = { 0,1,2,3, 12,5,6,7, 9,10,11,13, 14,15,18,19 };\n        int part = g_scfc_decode[gr->scalefac_compress];\n        scf_size[1] = scf_size[0] = (ma_uint8)(part >> 2);\n        scf_size[3] = scf_size[2] = (ma_uint8)(part & 3);\n    } else\n    {\n        static const ma_uint8 g_mod[6*4] = { 5,5,4,4,5,5,4,1,4,3,1,1,5,6,6,1,4,4,4,1,4,3,1,1 };\n        int k, modprod, sfc, ist = MA_DR_MP3_HDR_TEST_I_STEREO(hdr) && ch;\n        sfc = gr->scalefac_compress >> ist;\n        for (k = ist*3*4; sfc >= 0; sfc -= modprod, k += 4)\n        {\n            for (modprod = 1, i = 3; i >= 0; i--)\n            {\n                scf_size[i] = (ma_uint8)(sfc / modprod % g_mod[k + i]);\n                modprod *= g_mod[k + i];\n            }\n        }\n        scf_partition += k;\n        scfsi = -16;\n    }\n    ma_dr_mp3_L3_read_scalefactors(iscf, ist_pos, scf_size, scf_partition, bs, scfsi);\n    if (gr->n_short_sfb)\n    {\n        int sh = 3 - scf_shift;\n        for (i = 0; i < gr->n_short_sfb; i += 3)\n        {\n            iscf[gr->n_long_sfb + i + 0] = (ma_uint8)(iscf[gr->n_long_sfb + i + 0] + (gr->subblock_gain[0] << sh));\n            iscf[gr->n_long_sfb + i + 1] = (ma_uint8)(iscf[gr->n_long_sfb + i + 1] + (gr->subblock_gain[1] << sh));\n            iscf[gr->n_long_sfb + i + 2] = (ma_uint8)(iscf[gr->n_long_sfb + i + 2] + (gr->subblock_gain[2] << sh));\n        }\n    } else if (gr->preflag)\n    {\n        static const ma_uint8 g_preamp[10] = { 1,1,1,1,2,2,3,3,3,2 };\n        for (i = 0; i < 10; i++)\n        {\n            iscf[11 + i] = (ma_uint8)(iscf[11 + i] + g_preamp[i]);\n        }\n    }\n    gain_exp = gr->global_gain + MA_DR_MP3_BITS_DEQUANTIZER_OUT*4 - 210 - (MA_DR_MP3_HDR_IS_MS_STEREO(hdr) ? 2 : 0);\n    gain = ma_dr_mp3_L3_ldexp_q2(1 << (MA_DR_MP3_MAX_SCFI/4),  MA_DR_MP3_MAX_SCFI - gain_exp);\n    for (i = 0; i < (int)(gr->n_long_sfb + gr->n_short_sfb); i++)\n    {\n        scf[i] = ma_dr_mp3_L3_ldexp_q2(gain, iscf[i] << scf_shift);\n    }\n}\n#if (defined(__GNUC__) && (__GNUC__ >= 13)) && !defined(__clang__)\n    #pragma GCC diagnostic pop\n#endif\nstatic const float ma_dr_mp3_g_pow43[129 + 16] = {\n    0,-1,-2.519842f,-4.326749f,-6.349604f,-8.549880f,-10.902724f,-13.390518f,-16.000000f,-18.720754f,-21.544347f,-24.463781f,-27.473142f,-30.567351f,-33.741992f,-36.993181f,\n    0,1,2.519842f,4.326749f,6.349604f,8.549880f,10.902724f,13.390518f,16.000000f,18.720754f,21.544347f,24.463781f,27.473142f,30.567351f,33.741992f,36.993181f,40.317474f,43.711787f,47.173345f,50.699631f,54.288352f,57.937408f,61.644865f,65.408941f,69.227979f,73.100443f,77.024898f,81.000000f,85.024491f,89.097188f,93.216975f,97.382800f,101.593667f,105.848633f,110.146801f,114.487321f,118.869381f,123.292209f,127.755065f,132.257246f,136.798076f,141.376907f,145.993119f,150.646117f,155.335327f,160.060199f,164.820202f,169.614826f,174.443577f,179.305980f,184.201575f,189.129918f,194.090580f,199.083145f,204.107210f,209.162385f,214.248292f,219.364564f,224.510845f,229.686789f,234.892058f,240.126328f,245.389280f,250.680604f,256.000000f,261.347174f,266.721841f,272.123723f,277.552547f,283.008049f,288.489971f,293.998060f,299.532071f,305.091761f,310.676898f,316.287249f,321.922592f,327.582707f,333.267377f,338.976394f,344.709550f,350.466646f,356.247482f,362.051866f,367.879608f,373.730522f,379.604427f,385.501143f,391.420496f,397.362314f,403.326427f,409.312672f,415.320884f,421.350905f,427.402579f,433.475750f,439.570269f,445.685987f,451.822757f,457.980436f,464.158883f,470.357960f,476.577530f,482.817459f,489.077615f,495.357868f,501.658090f,507.978156f,514.317941f,520.677324f,527.056184f,533.454404f,539.871867f,546.308458f,552.764065f,559.238575f,565.731879f,572.243870f,578.774440f,585.323483f,591.890898f,598.476581f,605.080431f,611.702349f,618.342238f,625.000000f,631.675540f,638.368763f,645.079578f\n};\nstatic float ma_dr_mp3_L3_pow_43(int x)\n{\n    float frac;\n    int sign, mult = 256;\n    if (x < 129)\n    {\n        return ma_dr_mp3_g_pow43[16 + x];\n    }\n    if (x < 1024)\n    {\n        mult = 16;\n        x <<= 3;\n    }\n    sign = 2*x & 64;\n    frac = (float)((x & 63) - sign) / ((x & ~63) + sign);\n    return ma_dr_mp3_g_pow43[16 + ((x + sign) >> 6)]*(1.f + frac*((4.f/3) + frac*(2.f/9)))*mult;\n}\nstatic void ma_dr_mp3_L3_huffman(float *dst, ma_dr_mp3_bs *bs, const ma_dr_mp3_L3_gr_info *gr_info, const float *scf, int layer3gr_limit)\n{\n    static const ma_int16 tabs[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n        785,785,785,785,784,784,784,784,513,513,513,513,513,513,513,513,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,\n        -255,1313,1298,1282,785,785,785,785,784,784,784,784,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,290,288,\n        -255,1313,1298,1282,769,769,769,769,529,529,529,529,529,529,529,529,528,528,528,528,528,528,528,528,512,512,512,512,512,512,512,512,290,288,\n        -253,-318,-351,-367,785,785,785,785,784,784,784,784,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,819,818,547,547,275,275,275,275,561,560,515,546,289,274,288,258,\n        -254,-287,1329,1299,1314,1312,1057,1057,1042,1042,1026,1026,784,784,784,784,529,529,529,529,529,529,529,529,769,769,769,769,768,768,768,768,563,560,306,306,291,259,\n        -252,-413,-477,-542,1298,-575,1041,1041,784,784,784,784,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,-383,-399,1107,1092,1106,1061,849,849,789,789,1104,1091,773,773,1076,1075,341,340,325,309,834,804,577,577,532,532,516,516,832,818,803,816,561,561,531,531,515,546,289,289,288,258,\n        -252,-429,-493,-559,1057,1057,1042,1042,529,529,529,529,529,529,529,529,784,784,784,784,769,769,769,769,512,512,512,512,512,512,512,512,-382,1077,-415,1106,1061,1104,849,849,789,789,1091,1076,1029,1075,834,834,597,581,340,340,339,324,804,833,532,532,832,772,818,803,817,787,816,771,290,290,290,290,288,258,\n        -253,-349,-414,-447,-463,1329,1299,-479,1314,1312,1057,1057,1042,1042,1026,1026,785,785,785,785,784,784,784,784,769,769,769,769,768,768,768,768,-319,851,821,-335,836,850,805,849,341,340,325,336,533,533,579,579,564,564,773,832,578,548,563,516,321,276,306,291,304,259,\n        -251,-572,-733,-830,-863,-879,1041,1041,784,784,784,784,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,-511,-527,-543,1396,1351,1381,1366,1395,1335,1380,-559,1334,1138,1138,1063,1063,1350,1392,1031,1031,1062,1062,1364,1363,1120,1120,1333,1348,881,881,881,881,375,374,359,373,343,358,341,325,791,791,1123,1122,-703,1105,1045,-719,865,865,790,790,774,774,1104,1029,338,293,323,308,-799,-815,833,788,772,818,803,816,322,292,307,320,561,531,515,546,289,274,288,258,\n        -251,-525,-605,-685,-765,-831,-846,1298,1057,1057,1312,1282,785,785,785,785,784,784,784,784,769,769,769,769,512,512,512,512,512,512,512,512,1399,1398,1383,1367,1382,1396,1351,-511,1381,1366,1139,1139,1079,1079,1124,1124,1364,1349,1363,1333,882,882,882,882,807,807,807,807,1094,1094,1136,1136,373,341,535,535,881,775,867,822,774,-591,324,338,-671,849,550,550,866,864,609,609,293,336,534,534,789,835,773,-751,834,804,308,307,833,788,832,772,562,562,547,547,305,275,560,515,290,290,\n        -252,-397,-477,-557,-622,-653,-719,-735,-750,1329,1299,1314,1057,1057,1042,1042,1312,1282,1024,1024,785,785,785,785,784,784,784,784,769,769,769,769,-383,1127,1141,1111,1126,1140,1095,1110,869,869,883,883,1079,1109,882,882,375,374,807,868,838,881,791,-463,867,822,368,263,852,837,836,-543,610,610,550,550,352,336,534,534,865,774,851,821,850,805,593,533,579,564,773,832,578,578,548,548,577,577,307,276,306,291,516,560,259,259,\n        -250,-2107,-2507,-2764,-2909,-2974,-3007,-3023,1041,1041,1040,1040,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,-767,-1052,-1213,-1277,-1358,-1405,-1469,-1535,-1550,-1582,-1614,-1647,-1662,-1694,-1726,-1759,-1774,-1807,-1822,-1854,-1886,1565,-1919,-1935,-1951,-1967,1731,1730,1580,1717,-1983,1729,1564,-1999,1548,-2015,-2031,1715,1595,-2047,1714,-2063,1610,-2079,1609,-2095,1323,1323,1457,1457,1307,1307,1712,1547,1641,1700,1699,1594,1685,1625,1442,1442,1322,1322,-780,-973,-910,1279,1278,1277,1262,1276,1261,1275,1215,1260,1229,-959,974,974,989,989,-943,735,478,478,495,463,506,414,-1039,1003,958,1017,927,942,987,957,431,476,1272,1167,1228,-1183,1256,-1199,895,895,941,941,1242,1227,1212,1135,1014,1014,490,489,503,487,910,1013,985,925,863,894,970,955,1012,847,-1343,831,755,755,984,909,428,366,754,559,-1391,752,486,457,924,997,698,698,983,893,740,740,908,877,739,739,667,667,953,938,497,287,271,271,683,606,590,712,726,574,302,302,738,736,481,286,526,725,605,711,636,724,696,651,589,681,666,710,364,467,573,695,466,466,301,465,379,379,709,604,665,679,316,316,634,633,436,436,464,269,424,394,452,332,438,363,347,408,393,448,331,422,362,407,392,421,346,406,391,376,375,359,1441,1306,-2367,1290,-2383,1337,-2399,-2415,1426,1321,-2431,1411,1336,-2447,-2463,-2479,1169,1169,1049,1049,1424,1289,1412,1352,1319,-2495,1154,1154,1064,1064,1153,1153,416,390,360,404,403,389,344,374,373,343,358,372,327,357,342,311,356,326,1395,1394,1137,1137,1047,1047,1365,1392,1287,1379,1334,1364,1349,1378,1318,1363,792,792,792,792,1152,1152,1032,1032,1121,1121,1046,1046,1120,1120,1030,1030,-2895,1106,1061,1104,849,849,789,789,1091,1076,1029,1090,1060,1075,833,833,309,324,532,532,832,772,818,803,561,561,531,560,515,546,289,274,288,258,\n        -250,-1179,-1579,-1836,-1996,-2124,-2253,-2333,-2413,-2477,-2542,-2574,-2607,-2622,-2655,1314,1313,1298,1312,1282,785,785,785,785,1040,1040,1025,1025,768,768,768,768,-766,-798,-830,-862,-895,-911,-927,-943,-959,-975,-991,-1007,-1023,-1039,-1055,-1070,1724,1647,-1103,-1119,1631,1767,1662,1738,1708,1723,-1135,1780,1615,1779,1599,1677,1646,1778,1583,-1151,1777,1567,1737,1692,1765,1722,1707,1630,1751,1661,1764,1614,1736,1676,1763,1750,1645,1598,1721,1691,1762,1706,1582,1761,1566,-1167,1749,1629,767,766,751,765,494,494,735,764,719,749,734,763,447,447,748,718,477,506,431,491,446,476,461,505,415,430,475,445,504,399,460,489,414,503,383,474,429,459,502,502,746,752,488,398,501,473,413,472,486,271,480,270,-1439,-1455,1357,-1471,-1487,-1503,1341,1325,-1519,1489,1463,1403,1309,-1535,1372,1448,1418,1476,1356,1462,1387,-1551,1475,1340,1447,1402,1386,-1567,1068,1068,1474,1461,455,380,468,440,395,425,410,454,364,467,466,464,453,269,409,448,268,432,1371,1473,1432,1417,1308,1460,1355,1446,1459,1431,1083,1083,1401,1416,1458,1445,1067,1067,1370,1457,1051,1051,1291,1430,1385,1444,1354,1415,1400,1443,1082,1082,1173,1113,1186,1066,1185,1050,-1967,1158,1128,1172,1097,1171,1081,-1983,1157,1112,416,266,375,400,1170,1142,1127,1065,793,793,1169,1033,1156,1096,1141,1111,1155,1080,1126,1140,898,898,808,808,897,897,792,792,1095,1152,1032,1125,1110,1139,1079,1124,882,807,838,881,853,791,-2319,867,368,263,822,852,837,866,806,865,-2399,851,352,262,534,534,821,836,594,594,549,549,593,593,533,533,848,773,579,579,564,578,548,563,276,276,577,576,306,291,516,560,305,305,275,259,\n        -251,-892,-2058,-2620,-2828,-2957,-3023,-3039,1041,1041,1040,1040,769,769,769,769,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,256,-511,-527,-543,-559,1530,-575,-591,1528,1527,1407,1526,1391,1023,1023,1023,1023,1525,1375,1268,1268,1103,1103,1087,1087,1039,1039,1523,-604,815,815,815,815,510,495,509,479,508,463,507,447,431,505,415,399,-734,-782,1262,-815,1259,1244,-831,1258,1228,-847,-863,1196,-879,1253,987,987,748,-767,493,493,462,477,414,414,686,669,478,446,461,445,474,429,487,458,412,471,1266,1264,1009,1009,799,799,-1019,-1276,-1452,-1581,-1677,-1757,-1821,-1886,-1933,-1997,1257,1257,1483,1468,1512,1422,1497,1406,1467,1496,1421,1510,1134,1134,1225,1225,1466,1451,1374,1405,1252,1252,1358,1480,1164,1164,1251,1251,1238,1238,1389,1465,-1407,1054,1101,-1423,1207,-1439,830,830,1248,1038,1237,1117,1223,1148,1236,1208,411,426,395,410,379,269,1193,1222,1132,1235,1221,1116,976,976,1192,1162,1177,1220,1131,1191,963,963,-1647,961,780,-1663,558,558,994,993,437,408,393,407,829,978,813,797,947,-1743,721,721,377,392,844,950,828,890,706,706,812,859,796,960,948,843,934,874,571,571,-1919,690,555,689,421,346,539,539,944,779,918,873,932,842,903,888,570,570,931,917,674,674,-2575,1562,-2591,1609,-2607,1654,1322,1322,1441,1441,1696,1546,1683,1593,1669,1624,1426,1426,1321,1321,1639,1680,1425,1425,1305,1305,1545,1668,1608,1623,1667,1592,1638,1666,1320,1320,1652,1607,1409,1409,1304,1304,1288,1288,1664,1637,1395,1395,1335,1335,1622,1636,1394,1394,1319,1319,1606,1621,1392,1392,1137,1137,1137,1137,345,390,360,375,404,373,1047,-2751,-2767,-2783,1062,1121,1046,-2799,1077,-2815,1106,1061,789,789,1105,1104,263,355,310,340,325,354,352,262,339,324,1091,1076,1029,1090,1060,1075,833,833,788,788,1088,1028,818,818,803,803,561,561,531,531,816,771,546,546,289,274,288,258,\n        -253,-317,-381,-446,-478,-509,1279,1279,-811,-1179,-1451,-1756,-1900,-2028,-2189,-2253,-2333,-2414,-2445,-2511,-2526,1313,1298,-2559,1041,1041,1040,1040,1025,1025,1024,1024,1022,1007,1021,991,1020,975,1019,959,687,687,1018,1017,671,671,655,655,1016,1015,639,639,758,758,623,623,757,607,756,591,755,575,754,559,543,543,1009,783,-575,-621,-685,-749,496,-590,750,749,734,748,974,989,1003,958,988,973,1002,942,987,957,972,1001,926,986,941,971,956,1000,910,985,925,999,894,970,-1071,-1087,-1102,1390,-1135,1436,1509,1451,1374,-1151,1405,1358,1480,1420,-1167,1507,1494,1389,1342,1465,1435,1450,1326,1505,1310,1493,1373,1479,1404,1492,1464,1419,428,443,472,397,736,526,464,464,486,457,442,471,484,482,1357,1449,1434,1478,1388,1491,1341,1490,1325,1489,1463,1403,1309,1477,1372,1448,1418,1433,1476,1356,1462,1387,-1439,1475,1340,1447,1402,1474,1324,1461,1371,1473,269,448,1432,1417,1308,1460,-1711,1459,-1727,1441,1099,1099,1446,1386,1431,1401,-1743,1289,1083,1083,1160,1160,1458,1445,1067,1067,1370,1457,1307,1430,1129,1129,1098,1098,268,432,267,416,266,400,-1887,1144,1187,1082,1173,1113,1186,1066,1050,1158,1128,1143,1172,1097,1171,1081,420,391,1157,1112,1170,1142,1127,1065,1169,1049,1156,1096,1141,1111,1155,1080,1126,1154,1064,1153,1140,1095,1048,-2159,1125,1110,1137,-2175,823,823,1139,1138,807,807,384,264,368,263,868,838,853,791,867,822,852,837,866,806,865,790,-2319,851,821,836,352,262,850,805,849,-2399,533,533,835,820,336,261,578,548,563,577,532,532,832,772,562,562,547,547,305,275,560,515,290,290,288,258 };\n    static const ma_uint8 tab32[] = { 130,162,193,209,44,28,76,140,9,9,9,9,9,9,9,9,190,254,222,238,126,94,157,157,109,61,173,205};\n    static const ma_uint8 tab33[] = { 252,236,220,204,188,172,156,140,124,108,92,76,60,44,28,12 };\n    static const ma_int16 tabindex[2*16] = { 0,32,64,98,0,132,180,218,292,364,426,538,648,746,0,1126,1460,1460,1460,1460,1460,1460,1460,1460,1842,1842,1842,1842,1842,1842,1842,1842 };\n    static const ma_uint8 g_linbits[] =  { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,3,4,6,8,10,13,4,5,6,7,8,9,11,13 };\n#define MA_DR_MP3_PEEK_BITS(n)    (bs_cache >> (32 - (n)))\n#define MA_DR_MP3_FLUSH_BITS(n)   { bs_cache <<= (n); bs_sh += (n); }\n#define MA_DR_MP3_CHECK_BITS      while (bs_sh >= 0) { bs_cache |= (ma_uint32)*bs_next_ptr++ << bs_sh; bs_sh -= 8; }\n#define MA_DR_MP3_BSPOS           ((bs_next_ptr - bs->buf)*8 - 24 + bs_sh)\n    float one = 0.0f;\n    int ireg = 0, big_val_cnt = gr_info->big_values;\n    const ma_uint8 *sfb = gr_info->sfbtab;\n    const ma_uint8 *bs_next_ptr = bs->buf + bs->pos/8;\n    ma_uint32 bs_cache = (((bs_next_ptr[0]*256u + bs_next_ptr[1])*256u + bs_next_ptr[2])*256u + bs_next_ptr[3]) << (bs->pos & 7);\n    int pairs_to_decode, np, bs_sh = (bs->pos & 7) - 8;\n    bs_next_ptr += 4;\n    while (big_val_cnt > 0)\n    {\n        int tab_num = gr_info->table_select[ireg];\n        int sfb_cnt = gr_info->region_count[ireg++];\n        const ma_int16 *codebook = tabs + tabindex[tab_num];\n        int linbits = g_linbits[tab_num];\n        if (linbits)\n        {\n            do\n            {\n                np = *sfb++ / 2;\n                pairs_to_decode = MA_DR_MP3_MIN(big_val_cnt, np);\n                one = *scf++;\n                do\n                {\n                    int j, w = 5;\n                    int leaf = codebook[MA_DR_MP3_PEEK_BITS(w)];\n                    while (leaf < 0)\n                    {\n                        MA_DR_MP3_FLUSH_BITS(w);\n                        w = leaf & 7;\n                        leaf = codebook[MA_DR_MP3_PEEK_BITS(w) - (leaf >> 3)];\n                    }\n                    MA_DR_MP3_FLUSH_BITS(leaf >> 8);\n                    for (j = 0; j < 2; j++, dst++, leaf >>= 4)\n                    {\n                        int lsb = leaf & 0x0F;\n                        if (lsb == 15)\n                        {\n                            lsb += MA_DR_MP3_PEEK_BITS(linbits);\n                            MA_DR_MP3_FLUSH_BITS(linbits);\n                            MA_DR_MP3_CHECK_BITS;\n                            *dst = one*ma_dr_mp3_L3_pow_43(lsb)*((ma_int32)bs_cache < 0 ? -1: 1);\n                        } else\n                        {\n                            *dst = ma_dr_mp3_g_pow43[16 + lsb - 16*(bs_cache >> 31)]*one;\n                        }\n                        MA_DR_MP3_FLUSH_BITS(lsb ? 1 : 0);\n                    }\n                    MA_DR_MP3_CHECK_BITS;\n                } while (--pairs_to_decode);\n            } while ((big_val_cnt -= np) > 0 && --sfb_cnt >= 0);\n        } else\n        {\n            do\n            {\n                np = *sfb++ / 2;\n                pairs_to_decode = MA_DR_MP3_MIN(big_val_cnt, np);\n                one = *scf++;\n                do\n                {\n                    int j, w = 5;\n                    int leaf = codebook[MA_DR_MP3_PEEK_BITS(w)];\n                    while (leaf < 0)\n                    {\n                        MA_DR_MP3_FLUSH_BITS(w);\n                        w = leaf & 7;\n                        leaf = codebook[MA_DR_MP3_PEEK_BITS(w) - (leaf >> 3)];\n                    }\n                    MA_DR_MP3_FLUSH_BITS(leaf >> 8);\n                    for (j = 0; j < 2; j++, dst++, leaf >>= 4)\n                    {\n                        int lsb = leaf & 0x0F;\n                        *dst = ma_dr_mp3_g_pow43[16 + lsb - 16*(bs_cache >> 31)]*one;\n                        MA_DR_MP3_FLUSH_BITS(lsb ? 1 : 0);\n                    }\n                    MA_DR_MP3_CHECK_BITS;\n                } while (--pairs_to_decode);\n            } while ((big_val_cnt -= np) > 0 && --sfb_cnt >= 0);\n        }\n    }\n    for (np = 1 - big_val_cnt;; dst += 4)\n    {\n        const ma_uint8 *codebook_count1 = (gr_info->count1_table) ? tab33 : tab32;\n        int leaf = codebook_count1[MA_DR_MP3_PEEK_BITS(4)];\n        if (!(leaf & 8))\n        {\n            leaf = codebook_count1[(leaf >> 3) + (bs_cache << 4 >> (32 - (leaf & 3)))];\n        }\n        MA_DR_MP3_FLUSH_BITS(leaf & 7);\n        if (MA_DR_MP3_BSPOS > layer3gr_limit)\n        {\n            break;\n        }\n#define MA_DR_MP3_RELOAD_SCALEFACTOR  if (!--np) { np = *sfb++/2; if (!np) break; one = *scf++; }\n#define MA_DR_MP3_DEQ_COUNT1(s) if (leaf & (128 >> s)) { dst[s] = ((ma_int32)bs_cache < 0) ? -one : one; MA_DR_MP3_FLUSH_BITS(1) }\n        MA_DR_MP3_RELOAD_SCALEFACTOR;\n        MA_DR_MP3_DEQ_COUNT1(0);\n        MA_DR_MP3_DEQ_COUNT1(1);\n        MA_DR_MP3_RELOAD_SCALEFACTOR;\n        MA_DR_MP3_DEQ_COUNT1(2);\n        MA_DR_MP3_DEQ_COUNT1(3);\n        MA_DR_MP3_CHECK_BITS;\n    }\n    bs->pos = layer3gr_limit;\n}\nstatic void ma_dr_mp3_L3_midside_stereo(float *left, int n)\n{\n    int i = 0;\n    float *right = left + 576;\n#if MA_DR_MP3_HAVE_SIMD\n    if (ma_dr_mp3_have_simd())\n    {\n        for (; i < n - 3; i += 4)\n        {\n            ma_dr_mp3_f4 vl = MA_DR_MP3_VLD(left + i);\n            ma_dr_mp3_f4 vr = MA_DR_MP3_VLD(right + i);\n            MA_DR_MP3_VSTORE(left + i, MA_DR_MP3_VADD(vl, vr));\n            MA_DR_MP3_VSTORE(right + i, MA_DR_MP3_VSUB(vl, vr));\n        }\n#ifdef __GNUC__\n        if (__builtin_constant_p(n % 4 == 0) && n % 4 == 0)\n            return;\n#endif\n    }\n#endif\n    for (; i < n; i++)\n    {\n        float a = left[i];\n        float b = right[i];\n        left[i] = a + b;\n        right[i] = a - b;\n    }\n}\nstatic void ma_dr_mp3_L3_intensity_stereo_band(float *left, int n, float kl, float kr)\n{\n    int i;\n    for (i = 0; i < n; i++)\n    {\n        left[i + 576] = left[i]*kr;\n        left[i] = left[i]*kl;\n    }\n}\nstatic void ma_dr_mp3_L3_stereo_top_band(const float *right, const ma_uint8 *sfb, int nbands, int max_band[3])\n{\n    int i, k;\n    max_band[0] = max_band[1] = max_band[2] = -1;\n    for (i = 0; i < nbands; i++)\n    {\n        for (k = 0; k < sfb[i]; k += 2)\n        {\n            if (right[k] != 0 || right[k + 1] != 0)\n            {\n                max_band[i % 3] = i;\n                break;\n            }\n        }\n        right += sfb[i];\n    }\n}\nstatic void ma_dr_mp3_L3_stereo_process(float *left, const ma_uint8 *ist_pos, const ma_uint8 *sfb, const ma_uint8 *hdr, int max_band[3], int mpeg2_sh)\n{\n    static const float g_pan[7*2] = { 0,1,0.21132487f,0.78867513f,0.36602540f,0.63397460f,0.5f,0.5f,0.63397460f,0.36602540f,0.78867513f,0.21132487f,1,0 };\n    unsigned i, max_pos = MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? 7 : 64;\n    for (i = 0; sfb[i]; i++)\n    {\n        unsigned ipos = ist_pos[i];\n        if ((int)i > max_band[i % 3] && ipos < max_pos)\n        {\n            float kl, kr, s = MA_DR_MP3_HDR_TEST_MS_STEREO(hdr) ? 1.41421356f : 1;\n            if (MA_DR_MP3_HDR_TEST_MPEG1(hdr))\n            {\n                kl = g_pan[2*ipos];\n                kr = g_pan[2*ipos + 1];\n            } else\n            {\n                kl = 1;\n                kr = ma_dr_mp3_L3_ldexp_q2(1, (ipos + 1) >> 1 << mpeg2_sh);\n                if (ipos & 1)\n                {\n                    kl = kr;\n                    kr = 1;\n                }\n            }\n            ma_dr_mp3_L3_intensity_stereo_band(left, sfb[i], kl*s, kr*s);\n        } else if (MA_DR_MP3_HDR_TEST_MS_STEREO(hdr))\n        {\n            ma_dr_mp3_L3_midside_stereo(left, sfb[i]);\n        }\n        left += sfb[i];\n    }\n}\nstatic void ma_dr_mp3_L3_intensity_stereo(float *left, ma_uint8 *ist_pos, const ma_dr_mp3_L3_gr_info *gr, const ma_uint8 *hdr)\n{\n    int max_band[3], n_sfb = gr->n_long_sfb + gr->n_short_sfb;\n    int i, max_blocks = gr->n_short_sfb ? 3 : 1;\n    ma_dr_mp3_L3_stereo_top_band(left + 576, gr->sfbtab, n_sfb, max_band);\n    if (gr->n_long_sfb)\n    {\n        max_band[0] = max_band[1] = max_band[2] = MA_DR_MP3_MAX(MA_DR_MP3_MAX(max_band[0], max_band[1]), max_band[2]);\n    }\n    for (i = 0; i < max_blocks; i++)\n    {\n        int default_pos = MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? 3 : 0;\n        int itop = n_sfb - max_blocks + i;\n        int prev = itop - max_blocks;\n        ist_pos[itop] = (ma_uint8)(max_band[i] >= prev ? default_pos : ist_pos[prev]);\n    }\n    ma_dr_mp3_L3_stereo_process(left, ist_pos, gr->sfbtab, hdr, max_band, gr[1].scalefac_compress & 1);\n}\nstatic void ma_dr_mp3_L3_reorder(float *grbuf, float *scratch, const ma_uint8 *sfb)\n{\n    int i, len;\n    float *src = grbuf, *dst = scratch;\n    for (;0 != (len = *sfb); sfb += 3, src += 2*len)\n    {\n        for (i = 0; i < len; i++, src++)\n        {\n            *dst++ = src[0*len];\n            *dst++ = src[1*len];\n            *dst++ = src[2*len];\n        }\n    }\n    MA_DR_MP3_COPY_MEMORY(grbuf, scratch, (dst - scratch)*sizeof(float));\n}\nstatic void ma_dr_mp3_L3_antialias(float *grbuf, int nbands)\n{\n    static const float g_aa[2][8] = {\n        {0.85749293f,0.88174200f,0.94962865f,0.98331459f,0.99551782f,0.99916056f,0.99989920f,0.99999316f},\n        {0.51449576f,0.47173197f,0.31337745f,0.18191320f,0.09457419f,0.04096558f,0.01419856f,0.00369997f}\n    };\n    for (; nbands > 0; nbands--, grbuf += 18)\n    {\n        int i = 0;\n#if MA_DR_MP3_HAVE_SIMD\n        if (ma_dr_mp3_have_simd()) for (; i < 8; i += 4)\n        {\n            ma_dr_mp3_f4 vu = MA_DR_MP3_VLD(grbuf + 18 + i);\n            ma_dr_mp3_f4 vd = MA_DR_MP3_VLD(grbuf + 14 - i);\n            ma_dr_mp3_f4 vc0 = MA_DR_MP3_VLD(g_aa[0] + i);\n            ma_dr_mp3_f4 vc1 = MA_DR_MP3_VLD(g_aa[1] + i);\n            vd = MA_DR_MP3_VREV(vd);\n            MA_DR_MP3_VSTORE(grbuf + 18 + i, MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vu, vc0), MA_DR_MP3_VMUL(vd, vc1)));\n            vd = MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vu, vc1), MA_DR_MP3_VMUL(vd, vc0));\n            MA_DR_MP3_VSTORE(grbuf + 14 - i, MA_DR_MP3_VREV(vd));\n        }\n#endif\n#ifndef MA_DR_MP3_ONLY_SIMD\n        for(; i < 8; i++)\n        {\n            float u = grbuf[18 + i];\n            float d = grbuf[17 - i];\n            grbuf[18 + i] = u*g_aa[0][i] - d*g_aa[1][i];\n            grbuf[17 - i] = u*g_aa[1][i] + d*g_aa[0][i];\n        }\n#endif\n    }\n}\nstatic void ma_dr_mp3_L3_dct3_9(float *y)\n{\n    float s0, s1, s2, s3, s4, s5, s6, s7, s8, t0, t2, t4;\n    s0 = y[0]; s2 = y[2]; s4 = y[4]; s6 = y[6]; s8 = y[8];\n    t0 = s0 + s6*0.5f;\n    s0 -= s6;\n    t4 = (s4 + s2)*0.93969262f;\n    t2 = (s8 + s2)*0.76604444f;\n    s6 = (s4 - s8)*0.17364818f;\n    s4 += s8 - s2;\n    s2 = s0 - s4*0.5f;\n    y[4] = s4 + s0;\n    s8 = t0 - t2 + s6;\n    s0 = t0 - t4 + t2;\n    s4 = t0 + t4 - s6;\n    s1 = y[1]; s3 = y[3]; s5 = y[5]; s7 = y[7];\n    s3 *= 0.86602540f;\n    t0 = (s5 + s1)*0.98480775f;\n    t4 = (s5 - s7)*0.34202014f;\n    t2 = (s1 + s7)*0.64278761f;\n    s1 = (s1 - s5 - s7)*0.86602540f;\n    s5 = t0 - s3 - t2;\n    s7 = t4 - s3 - t0;\n    s3 = t4 + s3 - t2;\n    y[0] = s4 - s7;\n    y[1] = s2 + s1;\n    y[2] = s0 - s3;\n    y[3] = s8 + s5;\n    y[5] = s8 - s5;\n    y[6] = s0 + s3;\n    y[7] = s2 - s1;\n    y[8] = s4 + s7;\n}\nstatic void ma_dr_mp3_L3_imdct36(float *grbuf, float *overlap, const float *window, int nbands)\n{\n    int i, j;\n    static const float g_twid9[18] = {\n        0.73727734f,0.79335334f,0.84339145f,0.88701083f,0.92387953f,0.95371695f,0.97629601f,0.99144486f,0.99904822f,0.67559021f,0.60876143f,0.53729961f,0.46174861f,0.38268343f,0.30070580f,0.21643961f,0.13052619f,0.04361938f\n    };\n    for (j = 0; j < nbands; j++, grbuf += 18, overlap += 9)\n    {\n        float co[9], si[9];\n        co[0] = -grbuf[0];\n        si[0] = grbuf[17];\n        for (i = 0; i < 4; i++)\n        {\n            si[8 - 2*i] =   grbuf[4*i + 1] - grbuf[4*i + 2];\n            co[1 + 2*i] =   grbuf[4*i + 1] + grbuf[4*i + 2];\n            si[7 - 2*i] =   grbuf[4*i + 4] - grbuf[4*i + 3];\n            co[2 + 2*i] = -(grbuf[4*i + 3] + grbuf[4*i + 4]);\n        }\n        ma_dr_mp3_L3_dct3_9(co);\n        ma_dr_mp3_L3_dct3_9(si);\n        si[1] = -si[1];\n        si[3] = -si[3];\n        si[5] = -si[5];\n        si[7] = -si[7];\n        i = 0;\n#if MA_DR_MP3_HAVE_SIMD\n        if (ma_dr_mp3_have_simd()) for (; i < 8; i += 4)\n        {\n            ma_dr_mp3_f4 vovl = MA_DR_MP3_VLD(overlap + i);\n            ma_dr_mp3_f4 vc = MA_DR_MP3_VLD(co + i);\n            ma_dr_mp3_f4 vs = MA_DR_MP3_VLD(si + i);\n            ma_dr_mp3_f4 vr0 = MA_DR_MP3_VLD(g_twid9 + i);\n            ma_dr_mp3_f4 vr1 = MA_DR_MP3_VLD(g_twid9 + 9 + i);\n            ma_dr_mp3_f4 vw0 = MA_DR_MP3_VLD(window + i);\n            ma_dr_mp3_f4 vw1 = MA_DR_MP3_VLD(window + 9 + i);\n            ma_dr_mp3_f4 vsum = MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vc, vr1), MA_DR_MP3_VMUL(vs, vr0));\n            MA_DR_MP3_VSTORE(overlap + i, MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vc, vr0), MA_DR_MP3_VMUL(vs, vr1)));\n            MA_DR_MP3_VSTORE(grbuf + i, MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vovl, vw0), MA_DR_MP3_VMUL(vsum, vw1)));\n            vsum = MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vovl, vw1), MA_DR_MP3_VMUL(vsum, vw0));\n            MA_DR_MP3_VSTORE(grbuf + 14 - i, MA_DR_MP3_VREV(vsum));\n        }\n#endif\n        for (; i < 9; i++)\n        {\n            float ovl  = overlap[i];\n            float sum  = co[i]*g_twid9[9 + i] + si[i]*g_twid9[0 + i];\n            overlap[i] = co[i]*g_twid9[0 + i] - si[i]*g_twid9[9 + i];\n            grbuf[i]      = ovl*window[0 + i] - sum*window[9 + i];\n            grbuf[17 - i] = ovl*window[9 + i] + sum*window[0 + i];\n        }\n    }\n}\nstatic void ma_dr_mp3_L3_idct3(float x0, float x1, float x2, float *dst)\n{\n    float m1 = x1*0.86602540f;\n    float a1 = x0 - x2*0.5f;\n    dst[1] = x0 + x2;\n    dst[0] = a1 + m1;\n    dst[2] = a1 - m1;\n}\nstatic void ma_dr_mp3_L3_imdct12(float *x, float *dst, float *overlap)\n{\n    static const float g_twid3[6] = { 0.79335334f,0.92387953f,0.99144486f, 0.60876143f,0.38268343f,0.13052619f };\n    float co[3], si[3];\n    int i;\n    ma_dr_mp3_L3_idct3(-x[0], x[6] + x[3], x[12] + x[9], co);\n    ma_dr_mp3_L3_idct3(x[15], x[12] - x[9], x[6] - x[3], si);\n    si[1] = -si[1];\n    for (i = 0; i < 3; i++)\n    {\n        float ovl  = overlap[i];\n        float sum  = co[i]*g_twid3[3 + i] + si[i]*g_twid3[0 + i];\n        overlap[i] = co[i]*g_twid3[0 + i] - si[i]*g_twid3[3 + i];\n        dst[i]     = ovl*g_twid3[2 - i] - sum*g_twid3[5 - i];\n        dst[5 - i] = ovl*g_twid3[5 - i] + sum*g_twid3[2 - i];\n    }\n}\nstatic void ma_dr_mp3_L3_imdct_short(float *grbuf, float *overlap, int nbands)\n{\n    for (;nbands > 0; nbands--, overlap += 9, grbuf += 18)\n    {\n        float tmp[18];\n        MA_DR_MP3_COPY_MEMORY(tmp, grbuf, sizeof(tmp));\n        MA_DR_MP3_COPY_MEMORY(grbuf, overlap, 6*sizeof(float));\n        ma_dr_mp3_L3_imdct12(tmp, grbuf + 6, overlap + 6);\n        ma_dr_mp3_L3_imdct12(tmp + 1, grbuf + 12, overlap + 6);\n        ma_dr_mp3_L3_imdct12(tmp + 2, overlap, overlap + 6);\n    }\n}\nstatic void ma_dr_mp3_L3_change_sign(float *grbuf)\n{\n    int b, i;\n    for (b = 0, grbuf += 18; b < 32; b += 2, grbuf += 36)\n        for (i = 1; i < 18; i += 2)\n            grbuf[i] = -grbuf[i];\n}\nstatic void ma_dr_mp3_L3_imdct_gr(float *grbuf, float *overlap, unsigned block_type, unsigned n_long_bands)\n{\n    static const float g_mdct_window[2][18] = {\n        { 0.99904822f,0.99144486f,0.97629601f,0.95371695f,0.92387953f,0.88701083f,0.84339145f,0.79335334f,0.73727734f,0.04361938f,0.13052619f,0.21643961f,0.30070580f,0.38268343f,0.46174861f,0.53729961f,0.60876143f,0.67559021f },\n        { 1,1,1,1,1,1,0.99144486f,0.92387953f,0.79335334f,0,0,0,0,0,0,0.13052619f,0.38268343f,0.60876143f }\n    };\n    if (n_long_bands)\n    {\n        ma_dr_mp3_L3_imdct36(grbuf, overlap, g_mdct_window[0], n_long_bands);\n        grbuf += 18*n_long_bands;\n        overlap += 9*n_long_bands;\n    }\n    if (block_type == MA_DR_MP3_SHORT_BLOCK_TYPE)\n        ma_dr_mp3_L3_imdct_short(grbuf, overlap, 32 - n_long_bands);\n    else\n        ma_dr_mp3_L3_imdct36(grbuf, overlap, g_mdct_window[block_type == MA_DR_MP3_STOP_BLOCK_TYPE], 32 - n_long_bands);\n}\nstatic void ma_dr_mp3_L3_save_reservoir(ma_dr_mp3dec *h, ma_dr_mp3dec_scratch *s)\n{\n    int pos = (s->bs.pos + 7)/8u;\n    int remains = s->bs.limit/8u - pos;\n    if (remains > MA_DR_MP3_MAX_BITRESERVOIR_BYTES)\n    {\n        pos += remains - MA_DR_MP3_MAX_BITRESERVOIR_BYTES;\n        remains = MA_DR_MP3_MAX_BITRESERVOIR_BYTES;\n    }\n    if (remains > 0)\n    {\n        MA_DR_MP3_MOVE_MEMORY(h->reserv_buf, s->maindata + pos, remains);\n    }\n    h->reserv = remains;\n}\nstatic int ma_dr_mp3_L3_restore_reservoir(ma_dr_mp3dec *h, ma_dr_mp3_bs *bs, ma_dr_mp3dec_scratch *s, int main_data_begin)\n{\n    int frame_bytes = (bs->limit - bs->pos)/8;\n    int bytes_have = MA_DR_MP3_MIN(h->reserv, main_data_begin);\n    MA_DR_MP3_COPY_MEMORY(s->maindata, h->reserv_buf + MA_DR_MP3_MAX(0, h->reserv - main_data_begin), MA_DR_MP3_MIN(h->reserv, main_data_begin));\n    MA_DR_MP3_COPY_MEMORY(s->maindata + bytes_have, bs->buf + bs->pos/8, frame_bytes);\n    ma_dr_mp3_bs_init(&s->bs, s->maindata, bytes_have + frame_bytes);\n    return h->reserv >= main_data_begin;\n}\nstatic void ma_dr_mp3_L3_decode(ma_dr_mp3dec *h, ma_dr_mp3dec_scratch *s, ma_dr_mp3_L3_gr_info *gr_info, int nch)\n{\n    int ch;\n    for (ch = 0; ch < nch; ch++)\n    {\n        int layer3gr_limit = s->bs.pos + gr_info[ch].part_23_length;\n        ma_dr_mp3_L3_decode_scalefactors(h->header, s->ist_pos[ch], &s->bs, gr_info + ch, s->scf, ch);\n        ma_dr_mp3_L3_huffman(s->grbuf[ch], &s->bs, gr_info + ch, s->scf, layer3gr_limit);\n    }\n    if (MA_DR_MP3_HDR_TEST_I_STEREO(h->header))\n    {\n        ma_dr_mp3_L3_intensity_stereo(s->grbuf[0], s->ist_pos[1], gr_info, h->header);\n    } else if (MA_DR_MP3_HDR_IS_MS_STEREO(h->header))\n    {\n        ma_dr_mp3_L3_midside_stereo(s->grbuf[0], 576);\n    }\n    for (ch = 0; ch < nch; ch++, gr_info++)\n    {\n        int aa_bands = 31;\n        int n_long_bands = (gr_info->mixed_block_flag ? 2 : 0) << (int)(MA_DR_MP3_HDR_GET_MY_SAMPLE_RATE(h->header) == 2);\n        if (gr_info->n_short_sfb)\n        {\n            aa_bands = n_long_bands - 1;\n            ma_dr_mp3_L3_reorder(s->grbuf[ch] + n_long_bands*18, s->syn[0], gr_info->sfbtab + gr_info->n_long_sfb);\n        }\n        ma_dr_mp3_L3_antialias(s->grbuf[ch], aa_bands);\n        ma_dr_mp3_L3_imdct_gr(s->grbuf[ch], h->mdct_overlap[ch], gr_info->block_type, n_long_bands);\n        ma_dr_mp3_L3_change_sign(s->grbuf[ch]);\n    }\n}\nstatic void ma_dr_mp3d_DCT_II(float *grbuf, int n)\n{\n    static const float g_sec[24] = {\n        10.19000816f,0.50060302f,0.50241929f,3.40760851f,0.50547093f,0.52249861f,2.05778098f,0.51544732f,0.56694406f,1.48416460f,0.53104258f,0.64682180f,1.16943991f,0.55310392f,0.78815460f,0.97256821f,0.58293498f,1.06067765f,0.83934963f,0.62250412f,1.72244716f,0.74453628f,0.67480832f,5.10114861f\n    };\n    int i, k = 0;\n#if MA_DR_MP3_HAVE_SIMD\n    if (ma_dr_mp3_have_simd()) for (; k < n; k += 4)\n    {\n        ma_dr_mp3_f4 t[4][8], *x;\n        float *y = grbuf + k;\n        for (x = t[0], i = 0; i < 8; i++, x++)\n        {\n            ma_dr_mp3_f4 x0 = MA_DR_MP3_VLD(&y[i*18]);\n            ma_dr_mp3_f4 x1 = MA_DR_MP3_VLD(&y[(15 - i)*18]);\n            ma_dr_mp3_f4 x2 = MA_DR_MP3_VLD(&y[(16 + i)*18]);\n            ma_dr_mp3_f4 x3 = MA_DR_MP3_VLD(&y[(31 - i)*18]);\n            ma_dr_mp3_f4 t0 = MA_DR_MP3_VADD(x0, x3);\n            ma_dr_mp3_f4 t1 = MA_DR_MP3_VADD(x1, x2);\n            ma_dr_mp3_f4 t2 = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(x1, x2), g_sec[3*i + 0]);\n            ma_dr_mp3_f4 t3 = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(x0, x3), g_sec[3*i + 1]);\n            x[0] = MA_DR_MP3_VADD(t0, t1);\n            x[8] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(t0, t1), g_sec[3*i + 2]);\n            x[16] = MA_DR_MP3_VADD(t3, t2);\n            x[24] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(t3, t2), g_sec[3*i + 2]);\n        }\n        for (x = t[0], i = 0; i < 4; i++, x += 8)\n        {\n            ma_dr_mp3_f4 x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3], x4 = x[4], x5 = x[5], x6 = x[6], x7 = x[7], xt;\n            xt = MA_DR_MP3_VSUB(x0, x7); x0 = MA_DR_MP3_VADD(x0, x7);\n            x7 = MA_DR_MP3_VSUB(x1, x6); x1 = MA_DR_MP3_VADD(x1, x6);\n            x6 = MA_DR_MP3_VSUB(x2, x5); x2 = MA_DR_MP3_VADD(x2, x5);\n            x5 = MA_DR_MP3_VSUB(x3, x4); x3 = MA_DR_MP3_VADD(x3, x4);\n            x4 = MA_DR_MP3_VSUB(x0, x3); x0 = MA_DR_MP3_VADD(x0, x3);\n            x3 = MA_DR_MP3_VSUB(x1, x2); x1 = MA_DR_MP3_VADD(x1, x2);\n            x[0] = MA_DR_MP3_VADD(x0, x1);\n            x[4] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(x0, x1), 0.70710677f);\n            x5 = MA_DR_MP3_VADD(x5, x6);\n            x6 = MA_DR_MP3_VMUL_S(MA_DR_MP3_VADD(x6, x7), 0.70710677f);\n            x7 = MA_DR_MP3_VADD(x7, xt);\n            x3 = MA_DR_MP3_VMUL_S(MA_DR_MP3_VADD(x3, x4), 0.70710677f);\n            x5 = MA_DR_MP3_VSUB(x5, MA_DR_MP3_VMUL_S(x7, 0.198912367f));\n            x7 = MA_DR_MP3_VADD(x7, MA_DR_MP3_VMUL_S(x5, 0.382683432f));\n            x5 = MA_DR_MP3_VSUB(x5, MA_DR_MP3_VMUL_S(x7, 0.198912367f));\n            x0 = MA_DR_MP3_VSUB(xt, x6); xt = MA_DR_MP3_VADD(xt, x6);\n            x[1] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VADD(xt, x7), 0.50979561f);\n            x[2] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VADD(x4, x3), 0.54119611f);\n            x[3] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(x0, x5), 0.60134488f);\n            x[5] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VADD(x0, x5), 0.89997619f);\n            x[6] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(x4, x3), 1.30656302f);\n            x[7] = MA_DR_MP3_VMUL_S(MA_DR_MP3_VSUB(xt, x7), 2.56291556f);\n        }\n        if (k > n - 3)\n        {\n#if MA_DR_MP3_HAVE_SSE\n#define MA_DR_MP3_VSAVE2(i, v) _mm_storel_pi((__m64 *)(void*)&y[i*18], v)\n#else\n#define MA_DR_MP3_VSAVE2(i, v) vst1_f32((float32_t *)&y[(i)*18],  vget_low_f32(v))\n#endif\n            for (i = 0; i < 7; i++, y += 4*18)\n            {\n                ma_dr_mp3_f4 s = MA_DR_MP3_VADD(t[3][i], t[3][i + 1]);\n                MA_DR_MP3_VSAVE2(0, t[0][i]);\n                MA_DR_MP3_VSAVE2(1, MA_DR_MP3_VADD(t[2][i], s));\n                MA_DR_MP3_VSAVE2(2, MA_DR_MP3_VADD(t[1][i], t[1][i + 1]));\n                MA_DR_MP3_VSAVE2(3, MA_DR_MP3_VADD(t[2][1 + i], s));\n            }\n            MA_DR_MP3_VSAVE2(0, t[0][7]);\n            MA_DR_MP3_VSAVE2(1, MA_DR_MP3_VADD(t[2][7], t[3][7]));\n            MA_DR_MP3_VSAVE2(2, t[1][7]);\n            MA_DR_MP3_VSAVE2(3, t[3][7]);\n        } else\n        {\n#define MA_DR_MP3_VSAVE4(i, v) MA_DR_MP3_VSTORE(&y[(i)*18], v)\n            for (i = 0; i < 7; i++, y += 4*18)\n            {\n                ma_dr_mp3_f4 s = MA_DR_MP3_VADD(t[3][i], t[3][i + 1]);\n                MA_DR_MP3_VSAVE4(0, t[0][i]);\n                MA_DR_MP3_VSAVE4(1, MA_DR_MP3_VADD(t[2][i], s));\n                MA_DR_MP3_VSAVE4(2, MA_DR_MP3_VADD(t[1][i], t[1][i + 1]));\n                MA_DR_MP3_VSAVE4(3, MA_DR_MP3_VADD(t[2][1 + i], s));\n            }\n            MA_DR_MP3_VSAVE4(0, t[0][7]);\n            MA_DR_MP3_VSAVE4(1, MA_DR_MP3_VADD(t[2][7], t[3][7]));\n            MA_DR_MP3_VSAVE4(2, t[1][7]);\n            MA_DR_MP3_VSAVE4(3, t[3][7]);\n        }\n    } else\n#endif\n#ifdef MA_DR_MP3_ONLY_SIMD\n    {}\n#else\n    for (; k < n; k++)\n    {\n        float t[4][8], *x, *y = grbuf + k;\n        for (x = t[0], i = 0; i < 8; i++, x++)\n        {\n            float x0 = y[i*18];\n            float x1 = y[(15 - i)*18];\n            float x2 = y[(16 + i)*18];\n            float x3 = y[(31 - i)*18];\n            float t0 = x0 + x3;\n            float t1 = x1 + x2;\n            float t2 = (x1 - x2)*g_sec[3*i + 0];\n            float t3 = (x0 - x3)*g_sec[3*i + 1];\n            x[0] = t0 + t1;\n            x[8] = (t0 - t1)*g_sec[3*i + 2];\n            x[16] = t3 + t2;\n            x[24] = (t3 - t2)*g_sec[3*i + 2];\n        }\n        for (x = t[0], i = 0; i < 4; i++, x += 8)\n        {\n            float x0 = x[0], x1 = x[1], x2 = x[2], x3 = x[3], x4 = x[4], x5 = x[5], x6 = x[6], x7 = x[7], xt;\n            xt = x0 - x7; x0 += x7;\n            x7 = x1 - x6; x1 += x6;\n            x6 = x2 - x5; x2 += x5;\n            x5 = x3 - x4; x3 += x4;\n            x4 = x0 - x3; x0 += x3;\n            x3 = x1 - x2; x1 += x2;\n            x[0] = x0 + x1;\n            x[4] = (x0 - x1)*0.70710677f;\n            x5 =  x5 + x6;\n            x6 = (x6 + x7)*0.70710677f;\n            x7 =  x7 + xt;\n            x3 = (x3 + x4)*0.70710677f;\n            x5 -= x7*0.198912367f;\n            x7 += x5*0.382683432f;\n            x5 -= x7*0.198912367f;\n            x0 = xt - x6; xt += x6;\n            x[1] = (xt + x7)*0.50979561f;\n            x[2] = (x4 + x3)*0.54119611f;\n            x[3] = (x0 - x5)*0.60134488f;\n            x[5] = (x0 + x5)*0.89997619f;\n            x[6] = (x4 - x3)*1.30656302f;\n            x[7] = (xt - x7)*2.56291556f;\n        }\n        for (i = 0; i < 7; i++, y += 4*18)\n        {\n            y[0*18] = t[0][i];\n            y[1*18] = t[2][i] + t[3][i] + t[3][i + 1];\n            y[2*18] = t[1][i] + t[1][i + 1];\n            y[3*18] = t[2][i + 1] + t[3][i] + t[3][i + 1];\n        }\n        y[0*18] = t[0][7];\n        y[1*18] = t[2][7] + t[3][7];\n        y[2*18] = t[1][7];\n        y[3*18] = t[3][7];\n    }\n#endif\n}\n#ifndef MA_DR_MP3_FLOAT_OUTPUT\ntypedef ma_int16 ma_dr_mp3d_sample_t;\nstatic ma_int16 ma_dr_mp3d_scale_pcm(float sample)\n{\n    ma_int16 s;\n#if MA_DR_MP3_HAVE_ARMV6\n    ma_int32 s32 = (ma_int32)(sample + .5f);\n    s32 -= (s32 < 0);\n    s = (ma_int16)ma_dr_mp3_clip_int16_arm(s32);\n#else\n    if (sample >=  32766.5f) return (ma_int16) 32767;\n    if (sample <= -32767.5f) return (ma_int16)-32768;\n    s = (ma_int16)(sample + .5f);\n    s -= (s < 0);\n#endif\n    return s;\n}\n#else\ntypedef float ma_dr_mp3d_sample_t;\nstatic float ma_dr_mp3d_scale_pcm(float sample)\n{\n    return sample*(1.f/32768.f);\n}\n#endif\nstatic void ma_dr_mp3d_synth_pair(ma_dr_mp3d_sample_t *pcm, int nch, const float *z)\n{\n    float a;\n    a  = (z[14*64] - z[    0]) * 29;\n    a += (z[ 1*64] + z[13*64]) * 213;\n    a += (z[12*64] - z[ 2*64]) * 459;\n    a += (z[ 3*64] + z[11*64]) * 2037;\n    a += (z[10*64] - z[ 4*64]) * 5153;\n    a += (z[ 5*64] + z[ 9*64]) * 6574;\n    a += (z[ 8*64] - z[ 6*64]) * 37489;\n    a +=  z[ 7*64]             * 75038;\n    pcm[0] = ma_dr_mp3d_scale_pcm(a);\n    z += 2;\n    a  = z[14*64] * 104;\n    a += z[12*64] * 1567;\n    a += z[10*64] * 9727;\n    a += z[ 8*64] * 64019;\n    a += z[ 6*64] * -9975;\n    a += z[ 4*64] * -45;\n    a += z[ 2*64] * 146;\n    a += z[ 0*64] * -5;\n    pcm[16*nch] = ma_dr_mp3d_scale_pcm(a);\n}\nstatic void ma_dr_mp3d_synth(float *xl, ma_dr_mp3d_sample_t *dstl, int nch, float *lins)\n{\n    int i;\n    float *xr = xl + 576*(nch - 1);\n    ma_dr_mp3d_sample_t *dstr = dstl + (nch - 1);\n    static const float g_win[] = {\n        -1,26,-31,208,218,401,-519,2063,2000,4788,-5517,7134,5959,35640,-39336,74992,\n        -1,24,-35,202,222,347,-581,2080,1952,4425,-5879,7640,5288,33791,-41176,74856,\n        -1,21,-38,196,225,294,-645,2087,1893,4063,-6237,8092,4561,31947,-43006,74630,\n        -1,19,-41,190,227,244,-711,2085,1822,3705,-6589,8492,3776,30112,-44821,74313,\n        -1,17,-45,183,228,197,-779,2075,1739,3351,-6935,8840,2935,28289,-46617,73908,\n        -1,16,-49,176,228,153,-848,2057,1644,3004,-7271,9139,2037,26482,-48390,73415,\n        -2,14,-53,169,227,111,-919,2032,1535,2663,-7597,9389,1082,24694,-50137,72835,\n        -2,13,-58,161,224,72,-991,2001,1414,2330,-7910,9592,70,22929,-51853,72169,\n        -2,11,-63,154,221,36,-1064,1962,1280,2006,-8209,9750,-998,21189,-53534,71420,\n        -2,10,-68,147,215,2,-1137,1919,1131,1692,-8491,9863,-2122,19478,-55178,70590,\n        -3,9,-73,139,208,-29,-1210,1870,970,1388,-8755,9935,-3300,17799,-56778,69679,\n        -3,8,-79,132,200,-57,-1283,1817,794,1095,-8998,9966,-4533,16155,-58333,68692,\n        -4,7,-85,125,189,-83,-1356,1759,605,814,-9219,9959,-5818,14548,-59838,67629,\n        -4,7,-91,117,177,-106,-1428,1698,402,545,-9416,9916,-7154,12980,-61289,66494,\n        -5,6,-97,111,163,-127,-1498,1634,185,288,-9585,9838,-8540,11455,-62684,65290\n    };\n    float *zlin = lins + 15*64;\n    const float *w = g_win;\n    zlin[4*15]     = xl[18*16];\n    zlin[4*15 + 1] = xr[18*16];\n    zlin[4*15 + 2] = xl[0];\n    zlin[4*15 + 3] = xr[0];\n    zlin[4*31]     = xl[1 + 18*16];\n    zlin[4*31 + 1] = xr[1 + 18*16];\n    zlin[4*31 + 2] = xl[1];\n    zlin[4*31 + 3] = xr[1];\n    ma_dr_mp3d_synth_pair(dstr, nch, lins + 4*15 + 1);\n    ma_dr_mp3d_synth_pair(dstr + 32*nch, nch, lins + 4*15 + 64 + 1);\n    ma_dr_mp3d_synth_pair(dstl, nch, lins + 4*15);\n    ma_dr_mp3d_synth_pair(dstl + 32*nch, nch, lins + 4*15 + 64);\n#if MA_DR_MP3_HAVE_SIMD\n    if (ma_dr_mp3_have_simd()) for (i = 14; i >= 0; i--)\n    {\n#define MA_DR_MP3_VLOAD(k) ma_dr_mp3_f4 w0 = MA_DR_MP3_VSET(*w++); ma_dr_mp3_f4 w1 = MA_DR_MP3_VSET(*w++); ma_dr_mp3_f4 vz = MA_DR_MP3_VLD(&zlin[4*i - 64*k]); ma_dr_mp3_f4 vy = MA_DR_MP3_VLD(&zlin[4*i - 64*(15 - k)]);\n#define MA_DR_MP3_V0(k) { MA_DR_MP3_VLOAD(k) b =               MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vz, w1), MA_DR_MP3_VMUL(vy, w0)) ; a =               MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vz, w0), MA_DR_MP3_VMUL(vy, w1));  }\n#define MA_DR_MP3_V1(k) { MA_DR_MP3_VLOAD(k) b = MA_DR_MP3_VADD(b, MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vz, w1), MA_DR_MP3_VMUL(vy, w0))); a = MA_DR_MP3_VADD(a, MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vz, w0), MA_DR_MP3_VMUL(vy, w1))); }\n#define MA_DR_MP3_V2(k) { MA_DR_MP3_VLOAD(k) b = MA_DR_MP3_VADD(b, MA_DR_MP3_VADD(MA_DR_MP3_VMUL(vz, w1), MA_DR_MP3_VMUL(vy, w0))); a = MA_DR_MP3_VADD(a, MA_DR_MP3_VSUB(MA_DR_MP3_VMUL(vy, w1), MA_DR_MP3_VMUL(vz, w0))); }\n        ma_dr_mp3_f4 a, b;\n        zlin[4*i]     = xl[18*(31 - i)];\n        zlin[4*i + 1] = xr[18*(31 - i)];\n        zlin[4*i + 2] = xl[1 + 18*(31 - i)];\n        zlin[4*i + 3] = xr[1 + 18*(31 - i)];\n        zlin[4*i + 64] = xl[1 + 18*(1 + i)];\n        zlin[4*i + 64 + 1] = xr[1 + 18*(1 + i)];\n        zlin[4*i - 64 + 2] = xl[18*(1 + i)];\n        zlin[4*i - 64 + 3] = xr[18*(1 + i)];\n        MA_DR_MP3_V0(0) MA_DR_MP3_V2(1) MA_DR_MP3_V1(2) MA_DR_MP3_V2(3) MA_DR_MP3_V1(4) MA_DR_MP3_V2(5) MA_DR_MP3_V1(6) MA_DR_MP3_V2(7)\n        {\n#ifndef MA_DR_MP3_FLOAT_OUTPUT\n#if MA_DR_MP3_HAVE_SSE\n            static const ma_dr_mp3_f4 g_max = { 32767.0f, 32767.0f, 32767.0f, 32767.0f };\n            static const ma_dr_mp3_f4 g_min = { -32768.0f, -32768.0f, -32768.0f, -32768.0f };\n            __m128i pcm8 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_max_ps(_mm_min_ps(a, g_max), g_min)),\n                                           _mm_cvtps_epi32(_mm_max_ps(_mm_min_ps(b, g_max), g_min)));\n            dstr[(15 - i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 1);\n            dstr[(17 + i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 5);\n            dstl[(15 - i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 0);\n            dstl[(17 + i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 4);\n            dstr[(47 - i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 3);\n            dstr[(49 + i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 7);\n            dstl[(47 - i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 2);\n            dstl[(49 + i)*nch] = (ma_int16)_mm_extract_epi16(pcm8, 6);\n#else\n            int16x4_t pcma, pcmb;\n            a = MA_DR_MP3_VADD(a, MA_DR_MP3_VSET(0.5f));\n            b = MA_DR_MP3_VADD(b, MA_DR_MP3_VSET(0.5f));\n            pcma = vqmovn_s32(vqaddq_s32(vcvtq_s32_f32(a), vreinterpretq_s32_u32(vcltq_f32(a, MA_DR_MP3_VSET(0)))));\n            pcmb = vqmovn_s32(vqaddq_s32(vcvtq_s32_f32(b), vreinterpretq_s32_u32(vcltq_f32(b, MA_DR_MP3_VSET(0)))));\n            vst1_lane_s16(dstr + (15 - i)*nch, pcma, 1);\n            vst1_lane_s16(dstr + (17 + i)*nch, pcmb, 1);\n            vst1_lane_s16(dstl + (15 - i)*nch, pcma, 0);\n            vst1_lane_s16(dstl + (17 + i)*nch, pcmb, 0);\n            vst1_lane_s16(dstr + (47 - i)*nch, pcma, 3);\n            vst1_lane_s16(dstr + (49 + i)*nch, pcmb, 3);\n            vst1_lane_s16(dstl + (47 - i)*nch, pcma, 2);\n            vst1_lane_s16(dstl + (49 + i)*nch, pcmb, 2);\n#endif\n#else\n        #if MA_DR_MP3_HAVE_SSE\n            static const ma_dr_mp3_f4 g_scale = { 1.0f/32768.0f, 1.0f/32768.0f, 1.0f/32768.0f, 1.0f/32768.0f };\n        #else\n            const ma_dr_mp3_f4 g_scale = vdupq_n_f32(1.0f/32768.0f);\n        #endif\n            a = MA_DR_MP3_VMUL(a, g_scale);\n            b = MA_DR_MP3_VMUL(b, g_scale);\n#if MA_DR_MP3_HAVE_SSE\n            _mm_store_ss(dstr + (15 - i)*nch, _mm_shuffle_ps(a, a, _MM_SHUFFLE(1, 1, 1, 1)));\n            _mm_store_ss(dstr + (17 + i)*nch, _mm_shuffle_ps(b, b, _MM_SHUFFLE(1, 1, 1, 1)));\n            _mm_store_ss(dstl + (15 - i)*nch, _mm_shuffle_ps(a, a, _MM_SHUFFLE(0, 0, 0, 0)));\n            _mm_store_ss(dstl + (17 + i)*nch, _mm_shuffle_ps(b, b, _MM_SHUFFLE(0, 0, 0, 0)));\n            _mm_store_ss(dstr + (47 - i)*nch, _mm_shuffle_ps(a, a, _MM_SHUFFLE(3, 3, 3, 3)));\n            _mm_store_ss(dstr + (49 + i)*nch, _mm_shuffle_ps(b, b, _MM_SHUFFLE(3, 3, 3, 3)));\n            _mm_store_ss(dstl + (47 - i)*nch, _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 2, 2, 2)));\n            _mm_store_ss(dstl + (49 + i)*nch, _mm_shuffle_ps(b, b, _MM_SHUFFLE(2, 2, 2, 2)));\n#else\n            vst1q_lane_f32(dstr + (15 - i)*nch, a, 1);\n            vst1q_lane_f32(dstr + (17 + i)*nch, b, 1);\n            vst1q_lane_f32(dstl + (15 - i)*nch, a, 0);\n            vst1q_lane_f32(dstl + (17 + i)*nch, b, 0);\n            vst1q_lane_f32(dstr + (47 - i)*nch, a, 3);\n            vst1q_lane_f32(dstr + (49 + i)*nch, b, 3);\n            vst1q_lane_f32(dstl + (47 - i)*nch, a, 2);\n            vst1q_lane_f32(dstl + (49 + i)*nch, b, 2);\n#endif\n#endif\n        }\n    } else\n#endif\n#ifdef MA_DR_MP3_ONLY_SIMD\n    {}\n#else\n    for (i = 14; i >= 0; i--)\n    {\n#define MA_DR_MP3_LOAD(k) float w0 = *w++; float w1 = *w++; float *vz = &zlin[4*i - k*64]; float *vy = &zlin[4*i - (15 - k)*64];\n#define MA_DR_MP3_S0(k) { int j; MA_DR_MP3_LOAD(k); for (j = 0; j < 4; j++) b[j]  = vz[j]*w1 + vy[j]*w0, a[j]  = vz[j]*w0 - vy[j]*w1; }\n#define MA_DR_MP3_S1(k) { int j; MA_DR_MP3_LOAD(k); for (j = 0; j < 4; j++) b[j] += vz[j]*w1 + vy[j]*w0, a[j] += vz[j]*w0 - vy[j]*w1; }\n#define MA_DR_MP3_S2(k) { int j; MA_DR_MP3_LOAD(k); for (j = 0; j < 4; j++) b[j] += vz[j]*w1 + vy[j]*w0, a[j] += vy[j]*w1 - vz[j]*w0; }\n        float a[4], b[4];\n        zlin[4*i]     = xl[18*(31 - i)];\n        zlin[4*i + 1] = xr[18*(31 - i)];\n        zlin[4*i + 2] = xl[1 + 18*(31 - i)];\n        zlin[4*i + 3] = xr[1 + 18*(31 - i)];\n        zlin[4*(i + 16)]   = xl[1 + 18*(1 + i)];\n        zlin[4*(i + 16) + 1] = xr[1 + 18*(1 + i)];\n        zlin[4*(i - 16) + 2] = xl[18*(1 + i)];\n        zlin[4*(i - 16) + 3] = xr[18*(1 + i)];\n        MA_DR_MP3_S0(0) MA_DR_MP3_S2(1) MA_DR_MP3_S1(2) MA_DR_MP3_S2(3) MA_DR_MP3_S1(4) MA_DR_MP3_S2(5) MA_DR_MP3_S1(6) MA_DR_MP3_S2(7)\n        dstr[(15 - i)*nch] = ma_dr_mp3d_scale_pcm(a[1]);\n        dstr[(17 + i)*nch] = ma_dr_mp3d_scale_pcm(b[1]);\n        dstl[(15 - i)*nch] = ma_dr_mp3d_scale_pcm(a[0]);\n        dstl[(17 + i)*nch] = ma_dr_mp3d_scale_pcm(b[0]);\n        dstr[(47 - i)*nch] = ma_dr_mp3d_scale_pcm(a[3]);\n        dstr[(49 + i)*nch] = ma_dr_mp3d_scale_pcm(b[3]);\n        dstl[(47 - i)*nch] = ma_dr_mp3d_scale_pcm(a[2]);\n        dstl[(49 + i)*nch] = ma_dr_mp3d_scale_pcm(b[2]);\n    }\n#endif\n}\nstatic void ma_dr_mp3d_synth_granule(float *qmf_state, float *grbuf, int nbands, int nch, ma_dr_mp3d_sample_t *pcm, float *lins)\n{\n    int i;\n    for (i = 0; i < nch; i++)\n    {\n        ma_dr_mp3d_DCT_II(grbuf + 576*i, nbands);\n    }\n    MA_DR_MP3_COPY_MEMORY(lins, qmf_state, sizeof(float)*15*64);\n    for (i = 0; i < nbands; i += 2)\n    {\n        ma_dr_mp3d_synth(grbuf + i, pcm + 32*nch*i, nch, lins + i*64);\n    }\n#ifndef MA_DR_MP3_NONSTANDARD_BUT_LOGICAL\n    if (nch == 1)\n    {\n        for (i = 0; i < 15*64; i += 2)\n        {\n            qmf_state[i] = lins[nbands*64 + i];\n        }\n    } else\n#endif\n    {\n        MA_DR_MP3_COPY_MEMORY(qmf_state, lins + nbands*64, sizeof(float)*15*64);\n    }\n}\nstatic int ma_dr_mp3d_match_frame(const ma_uint8 *hdr, int mp3_bytes, int frame_bytes)\n{\n    int i, nmatch;\n    for (i = 0, nmatch = 0; nmatch < MA_DR_MP3_MAX_FRAME_SYNC_MATCHES; nmatch++)\n    {\n        i += ma_dr_mp3_hdr_frame_bytes(hdr + i, frame_bytes) + ma_dr_mp3_hdr_padding(hdr + i);\n        if (i + MA_DR_MP3_HDR_SIZE > mp3_bytes)\n            return nmatch > 0;\n        if (!ma_dr_mp3_hdr_compare(hdr, hdr + i))\n            return 0;\n    }\n    return 1;\n}\nstatic int ma_dr_mp3d_find_frame(const ma_uint8 *mp3, int mp3_bytes, int *free_format_bytes, int *ptr_frame_bytes)\n{\n    int i, k;\n    for (i = 0; i < mp3_bytes - MA_DR_MP3_HDR_SIZE; i++, mp3++)\n    {\n        if (ma_dr_mp3_hdr_valid(mp3))\n        {\n            int frame_bytes = ma_dr_mp3_hdr_frame_bytes(mp3, *free_format_bytes);\n            int frame_and_padding = frame_bytes + ma_dr_mp3_hdr_padding(mp3);\n            for (k = MA_DR_MP3_HDR_SIZE; !frame_bytes && k < MA_DR_MP3_MAX_FREE_FORMAT_FRAME_SIZE && i + 2*k < mp3_bytes - MA_DR_MP3_HDR_SIZE; k++)\n            {\n                if (ma_dr_mp3_hdr_compare(mp3, mp3 + k))\n                {\n                    int fb = k - ma_dr_mp3_hdr_padding(mp3);\n                    int nextfb = fb + ma_dr_mp3_hdr_padding(mp3 + k);\n                    if (i + k + nextfb + MA_DR_MP3_HDR_SIZE > mp3_bytes || !ma_dr_mp3_hdr_compare(mp3, mp3 + k + nextfb))\n                        continue;\n                    frame_and_padding = k;\n                    frame_bytes = fb;\n                    *free_format_bytes = fb;\n                }\n            }\n            if ((frame_bytes && i + frame_and_padding <= mp3_bytes &&\n                ma_dr_mp3d_match_frame(mp3, mp3_bytes - i, frame_bytes)) ||\n                (!i && frame_and_padding == mp3_bytes))\n            {\n                *ptr_frame_bytes = frame_and_padding;\n                return i;\n            }\n            *free_format_bytes = 0;\n        }\n    }\n    *ptr_frame_bytes = 0;\n    return mp3_bytes;\n}\nMA_API void ma_dr_mp3dec_init(ma_dr_mp3dec *dec)\n{\n    dec->header[0] = 0;\n}\nMA_API int ma_dr_mp3dec_decode_frame(ma_dr_mp3dec *dec, const ma_uint8 *mp3, int mp3_bytes, void *pcm, ma_dr_mp3dec_frame_info *info)\n{\n    int i = 0, igr, frame_size = 0, success = 1;\n    const ma_uint8 *hdr;\n    ma_dr_mp3_bs bs_frame[1];\n    if (mp3_bytes > 4 && dec->header[0] == 0xff && ma_dr_mp3_hdr_compare(dec->header, mp3))\n    {\n        frame_size = ma_dr_mp3_hdr_frame_bytes(mp3, dec->free_format_bytes) + ma_dr_mp3_hdr_padding(mp3);\n        if (frame_size != mp3_bytes && (frame_size + MA_DR_MP3_HDR_SIZE > mp3_bytes || !ma_dr_mp3_hdr_compare(mp3, mp3 + frame_size)))\n        {\n            frame_size = 0;\n        }\n    }\n    if (!frame_size)\n    {\n        MA_DR_MP3_ZERO_MEMORY(dec, sizeof(ma_dr_mp3dec));\n        i = ma_dr_mp3d_find_frame(mp3, mp3_bytes, &dec->free_format_bytes, &frame_size);\n        if (!frame_size || i + frame_size > mp3_bytes)\n        {\n            info->frame_bytes = i;\n            return 0;\n        }\n    }\n    hdr = mp3 + i;\n    MA_DR_MP3_COPY_MEMORY(dec->header, hdr, MA_DR_MP3_HDR_SIZE);\n    info->frame_bytes = i + frame_size;\n    info->channels = MA_DR_MP3_HDR_IS_MONO(hdr) ? 1 : 2;\n    info->sample_rate = ma_dr_mp3_hdr_sample_rate_hz(hdr);\n    info->layer = 4 - MA_DR_MP3_HDR_GET_LAYER(hdr);\n    info->bitrate_kbps = ma_dr_mp3_hdr_bitrate_kbps(hdr);\n    ma_dr_mp3_bs_init(bs_frame, hdr + MA_DR_MP3_HDR_SIZE, frame_size - MA_DR_MP3_HDR_SIZE);\n    if (MA_DR_MP3_HDR_IS_CRC(hdr))\n    {\n        ma_dr_mp3_bs_get_bits(bs_frame, 16);\n    }\n    if (info->layer == 3)\n    {\n        int main_data_begin = ma_dr_mp3_L3_read_side_info(bs_frame, dec->scratch.gr_info, hdr);\n        if (main_data_begin < 0 || bs_frame->pos > bs_frame->limit)\n        {\n            ma_dr_mp3dec_init(dec);\n            return 0;\n        }\n        success = ma_dr_mp3_L3_restore_reservoir(dec, bs_frame, &dec->scratch, main_data_begin);\n        if (success && pcm != NULL)\n        {\n            for (igr = 0; igr < (MA_DR_MP3_HDR_TEST_MPEG1(hdr) ? 2 : 1); igr++, pcm = MA_DR_MP3_OFFSET_PTR(pcm, sizeof(ma_dr_mp3d_sample_t)*576*info->channels))\n            {\n                MA_DR_MP3_ZERO_MEMORY(dec->scratch.grbuf[0], 576*2*sizeof(float));\n                ma_dr_mp3_L3_decode(dec, &dec->scratch, dec->scratch.gr_info + igr*info->channels, info->channels);\n                ma_dr_mp3d_synth_granule(dec->qmf_state, dec->scratch.grbuf[0], 18, info->channels, (ma_dr_mp3d_sample_t*)pcm, dec->scratch.syn[0]);\n            }\n        }\n        ma_dr_mp3_L3_save_reservoir(dec, &dec->scratch);\n    } else\n    {\n#ifdef MA_DR_MP3_ONLY_MP3\n        return 0;\n#else\n        ma_dr_mp3_L12_scale_info sci[1];\n        if (pcm == NULL) {\n            return ma_dr_mp3_hdr_frame_samples(hdr);\n        }\n        ma_dr_mp3_L12_read_scale_info(hdr, bs_frame, sci);\n        MA_DR_MP3_ZERO_MEMORY(dec->scratch.grbuf[0], 576*2*sizeof(float));\n        for (i = 0, igr = 0; igr < 3; igr++)\n        {\n            if (12 == (i += ma_dr_mp3_L12_dequantize_granule(dec->scratch.grbuf[0] + i, bs_frame, sci, info->layer | 1)))\n            {\n                i = 0;\n                ma_dr_mp3_L12_apply_scf_384(sci, sci->scf + igr, dec->scratch.grbuf[0]);\n                ma_dr_mp3d_synth_granule(dec->qmf_state, dec->scratch.grbuf[0], 12, info->channels, (ma_dr_mp3d_sample_t*)pcm, dec->scratch.syn[0]);\n                MA_DR_MP3_ZERO_MEMORY(dec->scratch.grbuf[0], 576*2*sizeof(float));\n                pcm = MA_DR_MP3_OFFSET_PTR(pcm, sizeof(ma_dr_mp3d_sample_t)*384*info->channels);\n            }\n            if (bs_frame->pos > bs_frame->limit)\n            {\n                ma_dr_mp3dec_init(dec);\n                return 0;\n            }\n        }\n#endif\n    }\n    return success*ma_dr_mp3_hdr_frame_samples(dec->header);\n}\nMA_API void ma_dr_mp3dec_f32_to_s16(const float *in, ma_int16 *out, size_t num_samples)\n{\n    size_t i = 0;\n#if MA_DR_MP3_HAVE_SIMD\n    size_t aligned_count = num_samples & ~7;\n    for(; i < aligned_count; i+=8)\n    {\n        ma_dr_mp3_f4 scale = MA_DR_MP3_VSET(32768.0f);\n        ma_dr_mp3_f4 a = MA_DR_MP3_VMUL(MA_DR_MP3_VLD(&in[i  ]), scale);\n        ma_dr_mp3_f4 b = MA_DR_MP3_VMUL(MA_DR_MP3_VLD(&in[i+4]), scale);\n#if MA_DR_MP3_HAVE_SSE\n        ma_dr_mp3_f4 s16max = MA_DR_MP3_VSET( 32767.0f);\n        ma_dr_mp3_f4 s16min = MA_DR_MP3_VSET(-32768.0f);\n        __m128i pcm8 = _mm_packs_epi32(_mm_cvtps_epi32(_mm_max_ps(_mm_min_ps(a, s16max), s16min)),\n                                        _mm_cvtps_epi32(_mm_max_ps(_mm_min_ps(b, s16max), s16min)));\n        out[i  ] = (ma_int16)_mm_extract_epi16(pcm8, 0);\n        out[i+1] = (ma_int16)_mm_extract_epi16(pcm8, 1);\n        out[i+2] = (ma_int16)_mm_extract_epi16(pcm8, 2);\n        out[i+3] = (ma_int16)_mm_extract_epi16(pcm8, 3);\n        out[i+4] = (ma_int16)_mm_extract_epi16(pcm8, 4);\n        out[i+5] = (ma_int16)_mm_extract_epi16(pcm8, 5);\n        out[i+6] = (ma_int16)_mm_extract_epi16(pcm8, 6);\n        out[i+7] = (ma_int16)_mm_extract_epi16(pcm8, 7);\n#else\n        int16x4_t pcma, pcmb;\n        a = MA_DR_MP3_VADD(a, MA_DR_MP3_VSET(0.5f));\n        b = MA_DR_MP3_VADD(b, MA_DR_MP3_VSET(0.5f));\n        pcma = vqmovn_s32(vqaddq_s32(vcvtq_s32_f32(a), vreinterpretq_s32_u32(vcltq_f32(a, MA_DR_MP3_VSET(0)))));\n        pcmb = vqmovn_s32(vqaddq_s32(vcvtq_s32_f32(b), vreinterpretq_s32_u32(vcltq_f32(b, MA_DR_MP3_VSET(0)))));\n        vst1_lane_s16(out+i  , pcma, 0);\n        vst1_lane_s16(out+i+1, pcma, 1);\n        vst1_lane_s16(out+i+2, pcma, 2);\n        vst1_lane_s16(out+i+3, pcma, 3);\n        vst1_lane_s16(out+i+4, pcmb, 0);\n        vst1_lane_s16(out+i+5, pcmb, 1);\n        vst1_lane_s16(out+i+6, pcmb, 2);\n        vst1_lane_s16(out+i+7, pcmb, 3);\n#endif\n    }\n#endif\n    for(; i < num_samples; i++)\n    {\n        float sample = in[i] * 32768.0f;\n        if (sample >=  32766.5f)\n            out[i] = (ma_int16) 32767;\n        else if (sample <= -32767.5f)\n            out[i] = (ma_int16)-32768;\n        else\n        {\n            short s = (ma_int16)(sample + .5f);\n            s -= (s < 0);\n            out[i] = s;\n        }\n    }\n}\n#ifndef MA_DR_MP3_SEEK_LEADING_MP3_FRAMES\n#define MA_DR_MP3_SEEK_LEADING_MP3_FRAMES   2\n#endif\n#define MA_DR_MP3_MIN_DATA_CHUNK_SIZE   16384\n#ifndef MA_DR_MP3_DATA_CHUNK_SIZE\n#define MA_DR_MP3_DATA_CHUNK_SIZE  (MA_DR_MP3_MIN_DATA_CHUNK_SIZE*4)\n#endif\n#define MA_DR_MP3_COUNTOF(x)        (sizeof(x) / sizeof(x[0]))\n#define MA_DR_MP3_CLAMP(x, lo, hi)  (MA_DR_MP3_MAX(lo, MA_DR_MP3_MIN(x, hi)))\n#ifndef MA_DR_MP3_PI_D\n#define MA_DR_MP3_PI_D    3.14159265358979323846264\n#endif\n#define MA_DR_MP3_DEFAULT_RESAMPLER_LPF_ORDER   2\nstatic MA_INLINE float ma_dr_mp3_mix_f32(float x, float y, float a)\n{\n    return x*(1-a) + y*a;\n}\nstatic MA_INLINE float ma_dr_mp3_mix_f32_fast(float x, float y, float a)\n{\n    float r0 = (y - x);\n    float r1 = r0*a;\n    return x + r1;\n}\nstatic MA_INLINE ma_uint32 ma_dr_mp3_gcf_u32(ma_uint32 a, ma_uint32 b)\n{\n    for (;;) {\n        if (b == 0) {\n            break;\n        } else {\n            ma_uint32 t = a;\n            a = b;\n            b = t % a;\n        }\n    }\n    return a;\n}\nstatic void* ma_dr_mp3__malloc_default(size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_MP3_MALLOC(sz);\n}\nstatic void* ma_dr_mp3__realloc_default(void* p, size_t sz, void* pUserData)\n{\n    (void)pUserData;\n    return MA_DR_MP3_REALLOC(p, sz);\n}\nstatic void ma_dr_mp3__free_default(void* p, void* pUserData)\n{\n    (void)pUserData;\n    MA_DR_MP3_FREE(p);\n}\nstatic void* ma_dr_mp3__malloc_from_callbacks(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onMalloc != NULL) {\n        return pAllocationCallbacks->onMalloc(sz, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(NULL, sz, pAllocationCallbacks->pUserData);\n    }\n    return NULL;\n}\nstatic void* ma_dr_mp3__realloc_from_callbacks(void* p, size_t szNew, size_t szOld, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks == NULL) {\n        return NULL;\n    }\n    if (pAllocationCallbacks->onRealloc != NULL) {\n        return pAllocationCallbacks->onRealloc(p, szNew, pAllocationCallbacks->pUserData);\n    }\n    if (pAllocationCallbacks->onMalloc != NULL && pAllocationCallbacks->onFree != NULL) {\n        void* p2;\n        p2 = pAllocationCallbacks->onMalloc(szNew, pAllocationCallbacks->pUserData);\n        if (p2 == NULL) {\n            return NULL;\n        }\n        if (p != NULL) {\n            MA_DR_MP3_COPY_MEMORY(p2, p, szOld);\n            pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n        }\n        return p2;\n    }\n    return NULL;\n}\nstatic void ma_dr_mp3__free_from_callbacks(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (p == NULL || pAllocationCallbacks == NULL) {\n        return;\n    }\n    if (pAllocationCallbacks->onFree != NULL) {\n        pAllocationCallbacks->onFree(p, pAllocationCallbacks->pUserData);\n    }\n}\nstatic ma_allocation_callbacks ma_dr_mp3_copy_allocation_callbacks_or_defaults(const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        return *pAllocationCallbacks;\n    } else {\n        ma_allocation_callbacks allocationCallbacks;\n        allocationCallbacks.pUserData = NULL;\n        allocationCallbacks.onMalloc  = ma_dr_mp3__malloc_default;\n        allocationCallbacks.onRealloc = ma_dr_mp3__realloc_default;\n        allocationCallbacks.onFree    = ma_dr_mp3__free_default;\n        return allocationCallbacks;\n    }\n}\nstatic size_t ma_dr_mp3__on_read(ma_dr_mp3* pMP3, void* pBufferOut, size_t bytesToRead)\n{\n    size_t bytesRead;\n    MA_DR_MP3_ASSERT(pMP3         != NULL);\n    MA_DR_MP3_ASSERT(pMP3->onRead != NULL);\n    if (bytesToRead == 0) {\n        return 0;\n    }\n    bytesRead = pMP3->onRead(pMP3->pUserData, pBufferOut, bytesToRead);\n    pMP3->streamCursor += bytesRead;\n    return bytesRead;\n}\nstatic size_t ma_dr_mp3__on_read_clamped(ma_dr_mp3* pMP3, void* pBufferOut, size_t bytesToRead)\n{\n    MA_DR_MP3_ASSERT(pMP3         != NULL);\n    MA_DR_MP3_ASSERT(pMP3->onRead != NULL);\n    if (pMP3->streamLength == MA_UINT64_MAX) {\n        return ma_dr_mp3__on_read(pMP3, pBufferOut, bytesToRead);\n    } else {\n        ma_uint64 bytesRemaining;\n        bytesRemaining = (pMP3->streamLength - pMP3->streamCursor);\n        if (bytesToRead >         bytesRemaining) {\n            bytesToRead = (size_t)bytesRemaining;\n        }\n        return ma_dr_mp3__on_read(pMP3, pBufferOut, bytesToRead);\n    }\n}\nstatic ma_bool32 ma_dr_mp3__on_seek(ma_dr_mp3* pMP3, int offset, ma_dr_mp3_seek_origin origin)\n{\n    MA_DR_MP3_ASSERT(offset >= 0);\n    MA_DR_MP3_ASSERT(origin == MA_DR_MP3_SEEK_SET || origin == MA_DR_MP3_SEEK_CUR);\n    if (!pMP3->onSeek(pMP3->pUserData, offset, origin)) {\n        return MA_FALSE;\n    }\n    if (origin == MA_DR_MP3_SEEK_SET) {\n        pMP3->streamCursor = (ma_uint64)offset;\n    } else{\n        pMP3->streamCursor += offset;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_mp3__on_seek_64(ma_dr_mp3* pMP3, ma_uint64 offset, ma_dr_mp3_seek_origin origin)\n{\n    if (offset <= 0x7FFFFFFF) {\n        return ma_dr_mp3__on_seek(pMP3, (int)offset, origin);\n    }\n    if (!ma_dr_mp3__on_seek(pMP3, 0x7FFFFFFF, MA_DR_MP3_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    offset -= 0x7FFFFFFF;\n    while (offset > 0) {\n        if (offset <= 0x7FFFFFFF) {\n            if (!ma_dr_mp3__on_seek(pMP3, (int)offset, MA_DR_MP3_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            offset = 0;\n        } else {\n            if (!ma_dr_mp3__on_seek(pMP3, 0x7FFFFFFF, MA_DR_MP3_SEEK_CUR)) {\n                return MA_FALSE;\n            }\n            offset -= 0x7FFFFFFF;\n        }\n    }\n    return MA_TRUE;\n}\nstatic void ma_dr_mp3__on_meta(ma_dr_mp3* pMP3, ma_dr_mp3_metadata_type type, const void* pRawData, size_t rawDataSize)\n{\n    if (pMP3->onMeta) {\n        ma_dr_mp3_metadata metadata;\n        MA_DR_MP3_ZERO_OBJECT(&metadata);\n        metadata.type        = type;\n        metadata.pRawData    = pRawData;\n        metadata.rawDataSize = rawDataSize;\n        pMP3->onMeta(pMP3->pUserDataMeta, &metadata);\n    }\n}\nstatic ma_uint32 ma_dr_mp3_decode_next_frame_ex__callbacks(ma_dr_mp3* pMP3, ma_dr_mp3d_sample_t* pPCMFrames, ma_dr_mp3dec_frame_info* pMP3FrameInfo, const ma_uint8** ppMP3FrameData)\n{\n    ma_uint32 pcmFramesRead = 0;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->onRead != NULL);\n    if (pMP3->atEnd) {\n        return 0;\n    }\n    for (;;) {\n        ma_dr_mp3dec_frame_info info;\n        if (pMP3->dataSize < MA_DR_MP3_MIN_DATA_CHUNK_SIZE) {\n            size_t bytesRead;\n            if (pMP3->pData != NULL) {\n                MA_DR_MP3_MOVE_MEMORY(pMP3->pData, pMP3->pData + pMP3->dataConsumed, pMP3->dataSize);\n            }\n            pMP3->dataConsumed = 0;\n            if (pMP3->dataCapacity < MA_DR_MP3_DATA_CHUNK_SIZE) {\n                ma_uint8* pNewData;\n                size_t newDataCap;\n                newDataCap = MA_DR_MP3_DATA_CHUNK_SIZE;\n                pNewData = (ma_uint8*)ma_dr_mp3__realloc_from_callbacks(pMP3->pData, newDataCap, pMP3->dataCapacity, &pMP3->allocationCallbacks);\n                if (pNewData == NULL) {\n                    return 0;\n                }\n                pMP3->pData = pNewData;\n                pMP3->dataCapacity = newDataCap;\n            }\n            bytesRead = ma_dr_mp3__on_read_clamped(pMP3, pMP3->pData + pMP3->dataSize, (pMP3->dataCapacity - pMP3->dataSize));\n            if (bytesRead == 0) {\n                if (pMP3->dataSize == 0) {\n                    pMP3->atEnd = MA_TRUE;\n                    return 0;\n                }\n            }\n            pMP3->dataSize += bytesRead;\n        }\n        if (pMP3->dataSize > INT_MAX) {\n            pMP3->atEnd = MA_TRUE;\n            return 0;\n        }\n        MA_DR_MP3_ASSERT(pMP3->pData != NULL);\n        MA_DR_MP3_ASSERT(pMP3->dataCapacity > 0);\n        if (pMP3->pData == NULL) {\n            return 0;\n        }\n        pcmFramesRead = ma_dr_mp3dec_decode_frame(&pMP3->decoder, pMP3->pData + pMP3->dataConsumed, (int)pMP3->dataSize, pPCMFrames, &info);\n        pMP3->dataConsumed += (size_t)info.frame_bytes;\n        pMP3->dataSize     -= (size_t)info.frame_bytes;\n        if (pcmFramesRead > 0) {\n            pcmFramesRead = ma_dr_mp3_hdr_frame_samples(pMP3->decoder.header);\n            pMP3->pcmFramesConsumedInMP3Frame = 0;\n            pMP3->pcmFramesRemainingInMP3Frame = pcmFramesRead;\n            pMP3->mp3FrameChannels = info.channels;\n            pMP3->mp3FrameSampleRate = info.sample_rate;\n            if (pMP3FrameInfo != NULL) {\n                *pMP3FrameInfo = info;\n            }\n            if (ppMP3FrameData != NULL) {\n                *ppMP3FrameData = pMP3->pData + pMP3->dataConsumed - (size_t)info.frame_bytes;\n            }\n            break;\n        } else if (info.frame_bytes == 0) {\n            size_t bytesRead;\n            MA_DR_MP3_MOVE_MEMORY(pMP3->pData, pMP3->pData + pMP3->dataConsumed, pMP3->dataSize);\n            pMP3->dataConsumed = 0;\n            if (pMP3->dataCapacity == pMP3->dataSize) {\n                ma_uint8* pNewData;\n                size_t newDataCap;\n                newDataCap = pMP3->dataCapacity + MA_DR_MP3_DATA_CHUNK_SIZE;\n                pNewData = (ma_uint8*)ma_dr_mp3__realloc_from_callbacks(pMP3->pData, newDataCap, pMP3->dataCapacity, &pMP3->allocationCallbacks);\n                if (pNewData == NULL) {\n                    return 0;\n                }\n                pMP3->pData = pNewData;\n                pMP3->dataCapacity = newDataCap;\n            }\n            bytesRead = ma_dr_mp3__on_read_clamped(pMP3, pMP3->pData + pMP3->dataSize, (pMP3->dataCapacity - pMP3->dataSize));\n            if (bytesRead == 0) {\n                pMP3->atEnd = MA_TRUE;\n                return 0;\n            }\n            pMP3->dataSize += bytesRead;\n        }\n    };\n    return pcmFramesRead;\n}\nstatic ma_uint32 ma_dr_mp3_decode_next_frame_ex__memory(ma_dr_mp3* pMP3, ma_dr_mp3d_sample_t* pPCMFrames, ma_dr_mp3dec_frame_info* pMP3FrameInfo, const ma_uint8** ppMP3FrameData)\n{\n    ma_uint32 pcmFramesRead = 0;\n    ma_dr_mp3dec_frame_info info;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->memory.pData != NULL);\n    if (pMP3->atEnd) {\n        return 0;\n    }\n    for (;;) {\n        pcmFramesRead = ma_dr_mp3dec_decode_frame(&pMP3->decoder, pMP3->memory.pData + pMP3->memory.currentReadPos, (int)(pMP3->memory.dataSize - pMP3->memory.currentReadPos), pPCMFrames, &info);\n        if (pcmFramesRead > 0) {\n            pcmFramesRead = ma_dr_mp3_hdr_frame_samples(pMP3->decoder.header);\n            pMP3->pcmFramesConsumedInMP3Frame  = 0;\n            pMP3->pcmFramesRemainingInMP3Frame = pcmFramesRead;\n            pMP3->mp3FrameChannels             = info.channels;\n            pMP3->mp3FrameSampleRate           = info.sample_rate;\n            if (pMP3FrameInfo != NULL) {\n                *pMP3FrameInfo = info;\n            }\n            if (ppMP3FrameData != NULL) {\n                *ppMP3FrameData = pMP3->memory.pData + pMP3->memory.currentReadPos;\n            }\n            break;\n        } else if (info.frame_bytes > 0) {\n            pMP3->memory.currentReadPos += (size_t)info.frame_bytes;\n            pMP3->streamCursor          += (size_t)info.frame_bytes;\n        } else {\n            break;\n        }\n    }\n    pMP3->memory.currentReadPos += (size_t)info.frame_bytes;\n    pMP3->streamCursor          += (size_t)info.frame_bytes;\n    return pcmFramesRead;\n}\nstatic ma_uint32 ma_dr_mp3_decode_next_frame_ex(ma_dr_mp3* pMP3, ma_dr_mp3d_sample_t* pPCMFrames, ma_dr_mp3dec_frame_info* pMP3FrameInfo, const ma_uint8** ppMP3FrameData)\n{\n    if (pMP3->memory.pData != NULL && pMP3->memory.dataSize > 0) {\n        return ma_dr_mp3_decode_next_frame_ex__memory(pMP3, pPCMFrames, pMP3FrameInfo, ppMP3FrameData);\n    } else {\n        return ma_dr_mp3_decode_next_frame_ex__callbacks(pMP3, pPCMFrames, pMP3FrameInfo, ppMP3FrameData);\n    }\n}\nstatic ma_uint32 ma_dr_mp3_decode_next_frame(ma_dr_mp3* pMP3)\n{\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    return ma_dr_mp3_decode_next_frame_ex(pMP3, (ma_dr_mp3d_sample_t*)pMP3->pcmFrames, NULL, NULL);\n}\n#if 0\nstatic ma_uint32 ma_dr_mp3_seek_next_frame(ma_dr_mp3* pMP3)\n{\n    ma_uint32 pcmFrameCount;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    pcmFrameCount = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL);\n    if (pcmFrameCount == 0) {\n        return 0;\n    }\n    pMP3->currentPCMFrame             += pcmFrameCount;\n    pMP3->pcmFramesConsumedInMP3Frame  = pcmFrameCount;\n    pMP3->pcmFramesRemainingInMP3Frame = 0;\n    return pcmFrameCount;\n}\n#endif\nstatic ma_bool32 ma_dr_mp3_init_internal(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, ma_dr_mp3_meta_proc onMeta, void* pUserData, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3dec_frame_info firstFrameInfo;\n    const ma_uint8* pFirstFrameData;\n    ma_uint32 firstFramePCMFrameCount;\n    ma_uint32 detectedMP3FrameCount = 0xFFFFFFFF;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(onRead != NULL);\n    ma_dr_mp3dec_init(&pMP3->decoder);\n    pMP3->onRead = onRead;\n    pMP3->onSeek = onSeek;\n    pMP3->onMeta = onMeta;\n    pMP3->pUserData = pUserData;\n    pMP3->pUserDataMeta = pUserDataMeta;\n    pMP3->allocationCallbacks = ma_dr_mp3_copy_allocation_callbacks_or_defaults(pAllocationCallbacks);\n    if (pMP3->allocationCallbacks.onFree == NULL || (pMP3->allocationCallbacks.onMalloc == NULL && pMP3->allocationCallbacks.onRealloc == NULL)) {\n        return MA_FALSE;\n    }\n    pMP3->streamCursor       = 0;\n    pMP3->streamLength       = MA_UINT64_MAX;\n    pMP3->streamStartOffset  = 0;\n    pMP3->delayInPCMFrames   = 0;\n    pMP3->paddingInPCMFrames = 0;\n    pMP3->totalPCMFrameCount = MA_UINT64_MAX;\n    #if 1\n    if (onSeek != NULL && onTell != NULL) {\n        if (onSeek(pUserData, 0, MA_DR_MP3_SEEK_END)) {\n            ma_int64 streamLen;\n            int streamEndOffset = 0;\n            if (onTell(pUserData, &streamLen)) {\n                if (streamLen > 128) {\n                    char id3[3];\n                    if (onSeek(pUserData, streamEndOffset - 128, MA_DR_MP3_SEEK_END)) {\n                        if (onRead(pUserData, id3, 3) == 3 && id3[0] == 'T' && id3[1] == 'A' && id3[2] == 'G') {\n                            streamEndOffset -= 128;\n                            streamLen       -= 128;\n                            if (onMeta != NULL) {\n                                ma_uint8 tag[128];\n                                tag[0] = 'T'; tag[1] = 'A'; tag[2] = 'G';\n                                if (onRead(pUserData, tag + 3, 125) == 125) {\n                                    ma_dr_mp3__on_meta(pMP3, MA_DR_MP3_METADATA_TYPE_ID3V1, tag, 128);\n                                }\n                            }\n                        } else {\n                        }\n                    } else {\n                    }\n                } else {\n                }\n                if (streamLen > 32) {\n                    char ape[32];\n                    if (onSeek(pUserData, streamEndOffset - 32, MA_DR_MP3_SEEK_END)) {\n                        if (onRead(pUserData, ape, 32) == 32 && ape[0] == 'A' && ape[1] == 'P' && ape[2] == 'E' && ape[3] == 'T' && ape[4] == 'A' && ape[5] == 'G' && ape[6] == 'E' && ape[7] == 'X') {\n                            ma_uint32 tagSize =\n                                ((ma_uint32)ape[24] << 0)  |\n                                ((ma_uint32)ape[25] << 8)  |\n                                ((ma_uint32)ape[26] << 16) |\n                                ((ma_uint32)ape[27] << 24);\n                            if (32 + tagSize < streamLen) {\n                                streamEndOffset -= 32 + tagSize;\n                                streamLen       -= 32 + tagSize;\n                                if (onMeta != NULL) {\n                                    if (onSeek(pUserData, streamEndOffset, MA_DR_MP3_SEEK_END)) {\n                                        size_t apeTagSize = (size_t)tagSize + 32;\n                                        ma_uint8* pTagData = (ma_uint8*)ma_dr_mp3_malloc(apeTagSize, pAllocationCallbacks);\n                                        if (pTagData != NULL) {\n                                            if (onRead(pUserData, pTagData, apeTagSize) == apeTagSize) {\n                                                ma_dr_mp3__on_meta(pMP3, MA_DR_MP3_METADATA_TYPE_APE, pTagData, apeTagSize);\n                                            }\n                                            ma_dr_mp3_free(pTagData, pAllocationCallbacks);\n                                        }\n                                    }\n                                }\n                            } else {\n                            }\n                        }\n                    }\n                } else {\n                }\n                if (!onSeek(pUserData, 0, MA_DR_MP3_SEEK_SET)) {\n                    return MA_FALSE;\n                }\n                pMP3->streamLength = (ma_uint64)streamLen;\n                if (pMP3->memory.pData != NULL) {\n                    pMP3->memory.dataSize = (size_t)pMP3->streamLength;\n                }\n            } else {\n                if (!onSeek(pUserData, 0, MA_DR_MP3_SEEK_SET)) {\n                    return MA_FALSE;\n                }\n            }\n        } else {\n        }\n    } else {\n    }\n    #endif\n    #if 1\n    {\n        char header[10];\n        if (onRead(pUserData, header, 10) == 10) {\n            if (header[0] == 'I' && header[1] == 'D' && header[2] == '3') {\n                ma_uint32 tagSize =\n                    (((ma_uint32)header[6] & 0x7F) << 21) |\n                    (((ma_uint32)header[7] & 0x7F) << 14) |\n                    (((ma_uint32)header[8] & 0x7F) << 7)  |\n                    (((ma_uint32)header[9] & 0x7F) << 0);\n                if (header[5] & 0x10) {\n                    tagSize += 10;\n                }\n                if (onMeta != NULL) {\n                    size_t tagSizeWithHeader = 10 + tagSize;\n                    ma_uint8* pTagData = (ma_uint8*)ma_dr_mp3_malloc(tagSizeWithHeader, pAllocationCallbacks);\n                    if (pTagData != NULL) {\n                        MA_DR_MP3_COPY_MEMORY(pTagData, header, 10);\n                        if (onRead(pUserData, pTagData + 10, tagSize) == tagSize) {\n                            ma_dr_mp3__on_meta(pMP3, MA_DR_MP3_METADATA_TYPE_ID3V2, pTagData, tagSizeWithHeader);\n                        }\n                        ma_dr_mp3_free(pTagData, pAllocationCallbacks);\n                    }\n                } else {\n                    if (onSeek != NULL) {\n                        if (!onSeek(pUserData, tagSize, MA_DR_MP3_SEEK_CUR)) {\n                            return MA_FALSE;\n                        }\n                    } else {\n                        char discard[1024];\n                        while (tagSize > 0) {\n                            size_t bytesToRead = tagSize;\n                            if (bytesToRead > sizeof(discard)) {\n                                bytesToRead = sizeof(discard);\n                            }\n                            if (onRead(pUserData, discard, bytesToRead) != bytesToRead) {\n                                return MA_FALSE;\n                            }\n                            tagSize -= (ma_uint32)bytesToRead;\n                        }\n                    }\n                }\n                pMP3->streamStartOffset += 10 + tagSize;\n                pMP3->streamCursor = pMP3->streamStartOffset;\n            } else {\n                if (onSeek != NULL) {\n                    if (!onSeek(pUserData, 0, MA_DR_MP3_SEEK_SET)) {\n                        return MA_FALSE;\n                    }\n                } else {\n                }\n            }\n        } else {\n            return MA_FALSE;\n        }\n    }\n    #endif\n    firstFramePCMFrameCount = ma_dr_mp3_decode_next_frame_ex(pMP3, (ma_dr_mp3d_sample_t*)pMP3->pcmFrames, &firstFrameInfo, &pFirstFrameData);\n    if (firstFramePCMFrameCount > 0) {\n        MA_DR_MP3_ASSERT(pFirstFrameData != NULL);\n        #if 1\n        MA_DR_MP3_ASSERT(firstFrameInfo.frame_bytes > 0);\n        {\n            ma_dr_mp3_bs bs;\n            ma_dr_mp3_L3_gr_info grInfo[4];\n            ma_dr_mp3_bs_init(&bs, pFirstFrameData + MA_DR_MP3_HDR_SIZE, firstFrameInfo.frame_bytes - MA_DR_MP3_HDR_SIZE);\n            if (MA_DR_MP3_HDR_IS_CRC(pFirstFrameData)) {\n                ma_dr_mp3_bs_get_bits(&bs, 16);\n            }\n            if (ma_dr_mp3_L3_read_side_info(&bs, grInfo, pFirstFrameData) >= 0) {\n                ma_bool32 isXing = MA_FALSE;\n                ma_bool32 isInfo = MA_FALSE;\n                const ma_uint8* pTagData;\n                const ma_uint8* pTagDataBeg;\n                pTagDataBeg = pFirstFrameData + MA_DR_MP3_HDR_SIZE + (bs.pos/8);\n                pTagData    = pTagDataBeg;\n                isXing = (pTagData[0] == 'X' && pTagData[1] == 'i' && pTagData[2] == 'n' && pTagData[3] == 'g');\n                isInfo = (pTagData[0] == 'I' && pTagData[1] == 'n' && pTagData[2] == 'f' && pTagData[3] == 'o');\n                if (isXing || isInfo) {\n                    ma_uint32 bytes = 0;\n                    ma_uint32 flags = pTagData[7];\n                    pTagData += 8;\n                    if (flags & 0x01) {\n                        detectedMP3FrameCount = (ma_uint32)pTagData[0] << 24 | (ma_uint32)pTagData[1] << 16 | (ma_uint32)pTagData[2] << 8 | (ma_uint32)pTagData[3];\n                        pTagData += 4;\n                    }\n                    if (flags & 0x02) {\n                        bytes  = (ma_uint32)pTagData[0] << 24 | (ma_uint32)pTagData[1] << 16 | (ma_uint32)pTagData[2] << 8 | (ma_uint32)pTagData[3];\n                        (void)bytes;\n                        pTagData += 4;\n                    }\n                    if (flags & 0x04) {\n                        pTagData += 100;\n                    }\n                    if (flags & 0x08) {\n                        pTagData += 4;\n                    }\n                    if (pTagData[0]) {\n                        pTagData += 21;\n                        if (pTagData - pFirstFrameData + 14 < firstFrameInfo.frame_bytes) {\n                            int delayInPCMFrames;\n                            int paddingInPCMFrames;\n                            delayInPCMFrames   = (( (ma_uint32)pTagData[0]        << 4) | ((ma_uint32)pTagData[1] >> 4)) + (528 + 1);\n                            paddingInPCMFrames = ((((ma_uint32)pTagData[1] & 0xF) << 8) | ((ma_uint32)pTagData[2]     )) - (528 + 1);\n                            if (paddingInPCMFrames < 0) {\n                                paddingInPCMFrames = 0;\n                            }\n                            pMP3->delayInPCMFrames   = (ma_uint32)delayInPCMFrames;\n                            pMP3->paddingInPCMFrames = (ma_uint32)paddingInPCMFrames;\n                        }\n                    }\n                    if (isXing) {\n                        pMP3->isVBR = MA_TRUE;\n                    } else if (isInfo) {\n                        pMP3->isCBR = MA_TRUE;\n                    }\n                    if (onMeta != NULL) {\n                        ma_dr_mp3_metadata_type metadataType = isXing ? MA_DR_MP3_METADATA_TYPE_XING : MA_DR_MP3_METADATA_TYPE_VBRI;\n                        size_t tagDataSize;\n                        tagDataSize  = (size_t)firstFrameInfo.frame_bytes;\n                        tagDataSize -= (size_t)(pTagDataBeg - pFirstFrameData);\n                        ma_dr_mp3__on_meta(pMP3, metadataType, pTagDataBeg, tagDataSize);\n                    }\n                    pMP3->pcmFramesRemainingInMP3Frame = 0;\n                    pMP3->streamStartOffset += (ma_uint32)(firstFrameInfo.frame_bytes);\n                    pMP3->streamCursor = pMP3->streamStartOffset;\n                    ma_dr_mp3dec_init(&pMP3->decoder);\n                }\n            } else {\n            }\n        }\n        #endif\n    } else {\n        ma_dr_mp3__free_from_callbacks(pMP3->pData, &pMP3->allocationCallbacks);\n        return MA_FALSE;\n    }\n    if (detectedMP3FrameCount != 0xFFFFFFFF) {\n        pMP3->totalPCMFrameCount = detectedMP3FrameCount * firstFramePCMFrameCount;\n    }\n    pMP3->channels   = pMP3->mp3FrameChannels;\n    pMP3->sampleRate = pMP3->mp3FrameSampleRate;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init(ma_dr_mp3* pMP3, ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, ma_dr_mp3_meta_proc onMeta, void* pUserData, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pMP3 == NULL || onRead == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_MP3_ZERO_OBJECT(pMP3);\n    return ma_dr_mp3_init_internal(pMP3, onRead, onSeek, onTell, onMeta, pUserData, pUserData, pAllocationCallbacks);\n}\nstatic size_t ma_dr_mp3__on_read_memory(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    ma_dr_mp3* pMP3 = (ma_dr_mp3*)pUserData;\n    size_t bytesRemaining;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->memory.dataSize >= pMP3->memory.currentReadPos);\n    bytesRemaining = pMP3->memory.dataSize - pMP3->memory.currentReadPos;\n    if (bytesToRead > bytesRemaining) {\n        bytesToRead = bytesRemaining;\n    }\n    if (bytesToRead > 0) {\n        MA_DR_MP3_COPY_MEMORY(pBufferOut, pMP3->memory.pData + pMP3->memory.currentReadPos, bytesToRead);\n        pMP3->memory.currentReadPos += bytesToRead;\n    }\n    return bytesToRead;\n}\nstatic ma_bool32 ma_dr_mp3__on_seek_memory(void* pUserData, int byteOffset, ma_dr_mp3_seek_origin origin)\n{\n    ma_dr_mp3* pMP3 = (ma_dr_mp3*)pUserData;\n    ma_int64 newCursor;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    if (origin == MA_DR_MP3_SEEK_SET) {\n        newCursor = 0;\n    } else if (origin == MA_DR_MP3_SEEK_CUR) {\n        newCursor = (ma_int64)pMP3->memory.currentReadPos;\n    } else if (origin == MA_DR_MP3_SEEK_END) {\n        newCursor = (ma_int64)pMP3->memory.dataSize;\n    } else {\n        MA_DR_MP3_ASSERT(!\"Invalid seek origin\");\n        return MA_FALSE;\n    }\n    newCursor += byteOffset;\n    if (newCursor < 0) {\n        return MA_FALSE;\n    }\n    if ((size_t)newCursor > pMP3->memory.dataSize) {\n        return MA_FALSE;\n    }\n    pMP3->memory.currentReadPos = (size_t)newCursor;\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_mp3__on_tell_memory(void* pUserData, ma_int64* pCursor)\n{\n    ma_dr_mp3* pMP3 = (ma_dr_mp3*)pUserData;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pCursor != NULL);\n    *pCursor = (ma_int64)pMP3->memory.currentReadPos;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init_memory_with_metadata(ma_dr_mp3* pMP3, const void* pData, size_t dataSize, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 result;\n    if (pMP3 == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_MP3_ZERO_OBJECT(pMP3);\n    if (pData == NULL || dataSize == 0) {\n        return MA_FALSE;\n    }\n    pMP3->memory.pData = (const ma_uint8*)pData;\n    pMP3->memory.dataSize = dataSize;\n    pMP3->memory.currentReadPos = 0;\n    result = ma_dr_mp3_init_internal(pMP3, ma_dr_mp3__on_read_memory, ma_dr_mp3__on_seek_memory, ma_dr_mp3__on_tell_memory, onMeta, pMP3, pUserDataMeta, pAllocationCallbacks);\n    if (result == MA_FALSE) {\n        return MA_FALSE;\n    }\n    if (pMP3->streamLength <= (ma_uint64)MA_SIZE_MAX) {\n        pMP3->memory.dataSize = (size_t)pMP3->streamLength;\n    }\n    if (pMP3->streamStartOffset > (ma_uint64)MA_SIZE_MAX) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init_memory(ma_dr_mp3* pMP3, const void* pData, size_t dataSize, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_mp3_init_memory_with_metadata(pMP3, pData, dataSize, NULL, NULL, pAllocationCallbacks);\n}\n#ifndef MA_DR_MP3_NO_STDIO\n#include <stdio.h>\n#include <wchar.h>\nstatic size_t ma_dr_mp3__on_read_stdio(void* pUserData, void* pBufferOut, size_t bytesToRead)\n{\n    return fread(pBufferOut, 1, bytesToRead, (FILE*)pUserData);\n}\nstatic ma_bool32 ma_dr_mp3__on_seek_stdio(void* pUserData, int offset, ma_dr_mp3_seek_origin origin)\n{\n    int whence = SEEK_SET;\n    if (origin == MA_DR_MP3_SEEK_CUR) {\n        whence = SEEK_CUR;\n    } else if (origin == MA_DR_MP3_SEEK_END) {\n        whence = SEEK_END;\n    }\n    return fseek((FILE*)pUserData, offset, whence) == 0;\n}\nstatic ma_bool32 ma_dr_mp3__on_tell_stdio(void* pUserData, ma_int64* pCursor)\n{\n    FILE* pFileStdio = (FILE*)pUserData;\n    ma_int64 result;\n    MA_DR_MP3_ASSERT(pFileStdio != NULL);\n    MA_DR_MP3_ASSERT(pCursor    != NULL);\n#if defined(_WIN32) && !defined(NXDK)\n    #if defined(_MSC_VER) && _MSC_VER > 1200\n        result = _ftelli64(pFileStdio);\n    #else\n        result = ftell(pFileStdio);\n    #endif\n#else\n    result = ftell(pFileStdio);\n#endif\n    *pCursor = result;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init_file_with_metadata(ma_dr_mp3* pMP3, const char* pFilePath, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 result;\n    FILE* pFile;\n    if (pMP3 == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_MP3_ZERO_OBJECT(pMP3);\n    if (ma_fopen(&pFile, pFilePath, \"rb\") != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    result = ma_dr_mp3_init_internal(pMP3, ma_dr_mp3__on_read_stdio, ma_dr_mp3__on_seek_stdio, ma_dr_mp3__on_tell_stdio, onMeta, (void*)pFile, pUserDataMeta, pAllocationCallbacks);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init_file_with_metadata_w(ma_dr_mp3* pMP3, const wchar_t* pFilePath, ma_dr_mp3_meta_proc onMeta, void* pUserDataMeta, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_bool32 result;\n    FILE* pFile;\n    if (pMP3 == NULL) {\n        return MA_FALSE;\n    }\n    MA_DR_MP3_ZERO_OBJECT(pMP3);\n    if (ma_wfopen(&pFile, pFilePath, L\"rb\", pAllocationCallbacks) != MA_SUCCESS) {\n        return MA_FALSE;\n    }\n    result = ma_dr_mp3_init_internal(pMP3, ma_dr_mp3__on_read_stdio, ma_dr_mp3__on_seek_stdio, ma_dr_mp3__on_tell_stdio, onMeta, (void*)pFile, pUserDataMeta, pAllocationCallbacks);\n    if (result != MA_TRUE) {\n        fclose(pFile);\n        return result;\n    }\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_init_file(ma_dr_mp3* pMP3, const char* pFilePath, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_mp3_init_file_with_metadata(pMP3, pFilePath, NULL, NULL, pAllocationCallbacks);\n}\nMA_API ma_bool32 ma_dr_mp3_init_file_w(ma_dr_mp3* pMP3, const wchar_t* pFilePath, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    return ma_dr_mp3_init_file_with_metadata_w(pMP3, pFilePath, NULL, NULL, pAllocationCallbacks);\n}\n#endif\nMA_API void ma_dr_mp3_uninit(ma_dr_mp3* pMP3)\n{\n    if (pMP3 == NULL) {\n        return;\n    }\n#ifndef MA_DR_MP3_NO_STDIO\n    if (pMP3->onRead == ma_dr_mp3__on_read_stdio) {\n        FILE* pFile = (FILE*)pMP3->pUserData;\n        if (pFile != NULL) {\n            fclose(pFile);\n            pMP3->pUserData = NULL;\n        }\n    }\n#endif\n    ma_dr_mp3__free_from_callbacks(pMP3->pData, &pMP3->allocationCallbacks);\n}\n#if defined(MA_DR_MP3_FLOAT_OUTPUT)\nstatic void ma_dr_mp3_f32_to_s16(ma_int16* dst, const float* src, ma_uint64 sampleCount)\n{\n    ma_uint64 i;\n    ma_uint64 i4;\n    ma_uint64 sampleCount4;\n    i = 0;\n    sampleCount4 = sampleCount >> 2;\n    for (i4 = 0; i4 < sampleCount4; i4 += 1) {\n        float x0 = src[i+0];\n        float x1 = src[i+1];\n        float x2 = src[i+2];\n        float x3 = src[i+3];\n        x0 = ((x0 < -1) ? -1 : ((x0 > 1) ? 1 : x0));\n        x1 = ((x1 < -1) ? -1 : ((x1 > 1) ? 1 : x1));\n        x2 = ((x2 < -1) ? -1 : ((x2 > 1) ? 1 : x2));\n        x3 = ((x3 < -1) ? -1 : ((x3 > 1) ? 1 : x3));\n        x0 = x0 * 32767.0f;\n        x1 = x1 * 32767.0f;\n        x2 = x2 * 32767.0f;\n        x3 = x3 * 32767.0f;\n        dst[i+0] = (ma_int16)x0;\n        dst[i+1] = (ma_int16)x1;\n        dst[i+2] = (ma_int16)x2;\n        dst[i+3] = (ma_int16)x3;\n        i += 4;\n    }\n    for (; i < sampleCount; i += 1) {\n        float x = src[i];\n        x = ((x < -1) ? -1 : ((x > 1) ? 1 : x));\n        x = x * 32767.0f;\n        dst[i] = (ma_int16)x;\n    }\n}\n#endif\n#if !defined(MA_DR_MP3_FLOAT_OUTPUT)\nstatic void ma_dr_mp3_s16_to_f32(float* dst, const ma_int16* src, ma_uint64 sampleCount)\n{\n    ma_uint64 i;\n    for (i = 0; i < sampleCount; i += 1) {\n        float x = (float)src[i];\n        x = x * 0.000030517578125f;\n        dst[i] = x;\n    }\n}\n#endif\nstatic ma_uint64 ma_dr_mp3_read_pcm_frames_raw(ma_dr_mp3* pMP3, ma_uint64 framesToRead, void* pBufferOut)\n{\n    ma_uint64 totalFramesRead = 0;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->onRead != NULL);\n    while (framesToRead > 0) {\n        ma_uint32 framesToConsume;\n        if (pMP3->currentPCMFrame < pMP3->delayInPCMFrames) {\n            ma_uint32 framesToSkip = (ma_uint32)MA_DR_MP3_MIN(pMP3->pcmFramesRemainingInMP3Frame, pMP3->delayInPCMFrames - pMP3->currentPCMFrame);\n            pMP3->currentPCMFrame              += framesToSkip;\n            pMP3->pcmFramesConsumedInMP3Frame  += framesToSkip;\n            pMP3->pcmFramesRemainingInMP3Frame -= framesToSkip;\n        }\n        framesToConsume = (ma_uint32)MA_DR_MP3_MIN(pMP3->pcmFramesRemainingInMP3Frame, framesToRead);\n        if (pMP3->totalPCMFrameCount != MA_UINT64_MAX && pMP3->totalPCMFrameCount > pMP3->paddingInPCMFrames) {\n            if (pMP3->currentPCMFrame < (pMP3->totalPCMFrameCount - pMP3->paddingInPCMFrames)) {\n                ma_uint64 framesRemainigToPadding = (pMP3->totalPCMFrameCount - pMP3->paddingInPCMFrames) - pMP3->currentPCMFrame;\n                if (framesToConsume >               framesRemainigToPadding) {\n                    framesToConsume = (ma_uint32)framesRemainigToPadding;\n                }\n            } else {\n                break;\n            }\n        }\n        if (pBufferOut != NULL) {\n            #if defined(MA_DR_MP3_FLOAT_OUTPUT)\n            {\n                float* pFramesOutF32 = (float*)MA_DR_MP3_OFFSET_PTR(pBufferOut,          sizeof(float) * totalFramesRead                   * pMP3->channels);\n                float* pFramesInF32  = (float*)MA_DR_MP3_OFFSET_PTR(&pMP3->pcmFrames[0], sizeof(float) * pMP3->pcmFramesConsumedInMP3Frame * pMP3->mp3FrameChannels);\n                MA_DR_MP3_COPY_MEMORY(pFramesOutF32, pFramesInF32, sizeof(float) * framesToConsume * pMP3->channels);\n            }\n            #else\n            {\n                ma_int16* pFramesOutS16 = (ma_int16*)MA_DR_MP3_OFFSET_PTR(pBufferOut,          sizeof(ma_int16) * totalFramesRead                   * pMP3->channels);\n                ma_int16* pFramesInS16  = (ma_int16*)MA_DR_MP3_OFFSET_PTR(&pMP3->pcmFrames[0], sizeof(ma_int16) * pMP3->pcmFramesConsumedInMP3Frame * pMP3->mp3FrameChannels);\n                MA_DR_MP3_COPY_MEMORY(pFramesOutS16, pFramesInS16, sizeof(ma_int16) * framesToConsume * pMP3->channels);\n            }\n            #endif\n        }\n        pMP3->currentPCMFrame              += framesToConsume;\n        pMP3->pcmFramesConsumedInMP3Frame  += framesToConsume;\n        pMP3->pcmFramesRemainingInMP3Frame -= framesToConsume;\n        totalFramesRead                    += framesToConsume;\n        framesToRead                       -= framesToConsume;\n        if (framesToRead == 0) {\n            break;\n        }\n        if (pMP3->totalPCMFrameCount != MA_UINT64_MAX && pMP3->totalPCMFrameCount > pMP3->paddingInPCMFrames && pMP3->currentPCMFrame >= (pMP3->totalPCMFrameCount - pMP3->paddingInPCMFrames)) {\n            break;\n        }\n        MA_DR_MP3_ASSERT(pMP3->pcmFramesRemainingInMP3Frame == 0);\n        if (ma_dr_mp3_decode_next_frame(pMP3) == 0) {\n            break;\n        }\n    }\n    return totalFramesRead;\n}\nMA_API ma_uint64 ma_dr_mp3_read_pcm_frames_f32(ma_dr_mp3* pMP3, ma_uint64 framesToRead, float* pBufferOut)\n{\n    if (pMP3 == NULL || pMP3->onRead == NULL) {\n        return 0;\n    }\n#if defined(MA_DR_MP3_FLOAT_OUTPUT)\n    return ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToRead, pBufferOut);\n#else\n    {\n        ma_int16 pTempS16[8192];\n        ma_uint64 totalPCMFramesRead = 0;\n        while (totalPCMFramesRead < framesToRead) {\n            ma_uint64 framesJustRead;\n            ma_uint64 framesRemaining = framesToRead - totalPCMFramesRead;\n            ma_uint64 framesToReadNow = MA_DR_MP3_COUNTOF(pTempS16) / pMP3->channels;\n            if (framesToReadNow > framesRemaining) {\n                framesToReadNow = framesRemaining;\n            }\n            framesJustRead = ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToReadNow, pTempS16);\n            if (framesJustRead == 0) {\n                break;\n            }\n            ma_dr_mp3_s16_to_f32((float*)MA_DR_MP3_OFFSET_PTR(pBufferOut, sizeof(float) * totalPCMFramesRead * pMP3->channels), pTempS16, framesJustRead * pMP3->channels);\n            totalPCMFramesRead += framesJustRead;\n        }\n        return totalPCMFramesRead;\n    }\n#endif\n}\nMA_API ma_uint64 ma_dr_mp3_read_pcm_frames_s16(ma_dr_mp3* pMP3, ma_uint64 framesToRead, ma_int16* pBufferOut)\n{\n    if (pMP3 == NULL || pMP3->onRead == NULL) {\n        return 0;\n    }\n#if !defined(MA_DR_MP3_FLOAT_OUTPUT)\n    return ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToRead, pBufferOut);\n#else\n    {\n        float pTempF32[4096];\n        ma_uint64 totalPCMFramesRead = 0;\n        while (totalPCMFramesRead < framesToRead) {\n            ma_uint64 framesJustRead;\n            ma_uint64 framesRemaining = framesToRead - totalPCMFramesRead;\n            ma_uint64 framesToReadNow = MA_DR_MP3_COUNTOF(pTempF32) / pMP3->channels;\n            if (framesToReadNow > framesRemaining) {\n                framesToReadNow = framesRemaining;\n            }\n            framesJustRead = ma_dr_mp3_read_pcm_frames_raw(pMP3, framesToReadNow, pTempF32);\n            if (framesJustRead == 0) {\n                break;\n            }\n            ma_dr_mp3_f32_to_s16((ma_int16*)MA_DR_MP3_OFFSET_PTR(pBufferOut, sizeof(ma_int16) * totalPCMFramesRead * pMP3->channels), pTempF32, framesJustRead * pMP3->channels);\n            totalPCMFramesRead += framesJustRead;\n        }\n        return totalPCMFramesRead;\n    }\n#endif\n}\nstatic void ma_dr_mp3_reset(ma_dr_mp3* pMP3)\n{\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    pMP3->pcmFramesConsumedInMP3Frame = 0;\n    pMP3->pcmFramesRemainingInMP3Frame = 0;\n    pMP3->currentPCMFrame = 0;\n    pMP3->dataSize = 0;\n    pMP3->atEnd = MA_FALSE;\n    ma_dr_mp3dec_init(&pMP3->decoder);\n}\nstatic ma_bool32 ma_dr_mp3_seek_to_start_of_stream(ma_dr_mp3* pMP3)\n{\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->onSeek != NULL);\n    if (!ma_dr_mp3__on_seek_64(pMP3, pMP3->streamStartOffset, MA_DR_MP3_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    ma_dr_mp3_reset(pMP3);\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(ma_dr_mp3* pMP3, ma_uint64 frameOffset)\n{\n    ma_uint64 framesRead;\n#if defined(MA_DR_MP3_FLOAT_OUTPUT)\n    framesRead = ma_dr_mp3_read_pcm_frames_f32(pMP3, frameOffset, NULL);\n#else\n    framesRead = ma_dr_mp3_read_pcm_frames_s16(pMP3, frameOffset, NULL);\n#endif\n    if (framesRead != frameOffset) {\n        return MA_FALSE;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_mp3_seek_to_pcm_frame__brute_force(ma_dr_mp3* pMP3, ma_uint64 frameIndex)\n{\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    if (frameIndex == pMP3->currentPCMFrame) {\n        return MA_TRUE;\n    }\n    if (frameIndex < pMP3->currentPCMFrame) {\n        if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) {\n            return MA_FALSE;\n        }\n    }\n    MA_DR_MP3_ASSERT(frameIndex >= pMP3->currentPCMFrame);\n    return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, (frameIndex - pMP3->currentPCMFrame));\n}\nstatic ma_bool32 ma_dr_mp3_find_closest_seek_point(ma_dr_mp3* pMP3, ma_uint64 frameIndex, ma_uint32* pSeekPointIndex)\n{\n    ma_uint32 iSeekPoint;\n    MA_DR_MP3_ASSERT(pSeekPointIndex != NULL);\n    *pSeekPointIndex = 0;\n    if (frameIndex < pMP3->pSeekPoints[0].pcmFrameIndex) {\n        return MA_FALSE;\n    }\n    for (iSeekPoint = 0; iSeekPoint < pMP3->seekPointCount; ++iSeekPoint) {\n        if (pMP3->pSeekPoints[iSeekPoint].pcmFrameIndex > frameIndex) {\n            break;\n        }\n        *pSeekPointIndex = iSeekPoint;\n    }\n    return MA_TRUE;\n}\nstatic ma_bool32 ma_dr_mp3_seek_to_pcm_frame__seek_table(ma_dr_mp3* pMP3, ma_uint64 frameIndex)\n{\n    ma_dr_mp3_seek_point seekPoint;\n    ma_uint32 priorSeekPointIndex;\n    ma_uint16 iMP3Frame;\n    ma_uint64 leftoverFrames;\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    MA_DR_MP3_ASSERT(pMP3->pSeekPoints != NULL);\n    MA_DR_MP3_ASSERT(pMP3->seekPointCount > 0);\n    if (ma_dr_mp3_find_closest_seek_point(pMP3, frameIndex, &priorSeekPointIndex)) {\n        seekPoint = pMP3->pSeekPoints[priorSeekPointIndex];\n    } else {\n        seekPoint.seekPosInBytes     = 0;\n        seekPoint.pcmFrameIndex      = 0;\n        seekPoint.mp3FramesToDiscard = 0;\n        seekPoint.pcmFramesToDiscard = 0;\n    }\n    if (!ma_dr_mp3__on_seek_64(pMP3, seekPoint.seekPosInBytes, MA_DR_MP3_SEEK_SET)) {\n        return MA_FALSE;\n    }\n    ma_dr_mp3_reset(pMP3);\n    for (iMP3Frame = 0; iMP3Frame < seekPoint.mp3FramesToDiscard; ++iMP3Frame) {\n        ma_uint32 pcmFramesRead;\n        ma_dr_mp3d_sample_t* pPCMFrames;\n        pPCMFrames = NULL;\n        if (iMP3Frame == seekPoint.mp3FramesToDiscard-1) {\n            pPCMFrames = (ma_dr_mp3d_sample_t*)pMP3->pcmFrames;\n        }\n        pcmFramesRead = ma_dr_mp3_decode_next_frame_ex(pMP3, pPCMFrames, NULL, NULL);\n        if (pcmFramesRead == 0) {\n            return MA_FALSE;\n        }\n    }\n    pMP3->currentPCMFrame = seekPoint.pcmFrameIndex - seekPoint.pcmFramesToDiscard;\n    leftoverFrames = frameIndex - pMP3->currentPCMFrame;\n    return ma_dr_mp3_seek_forward_by_pcm_frames__brute_force(pMP3, leftoverFrames);\n}\nMA_API ma_bool32 ma_dr_mp3_seek_to_pcm_frame(ma_dr_mp3* pMP3, ma_uint64 frameIndex)\n{\n    if (pMP3 == NULL || pMP3->onSeek == NULL) {\n        return MA_FALSE;\n    }\n    if (frameIndex == 0) {\n        return ma_dr_mp3_seek_to_start_of_stream(pMP3);\n    }\n    if (pMP3->pSeekPoints != NULL && pMP3->seekPointCount > 0) {\n        return ma_dr_mp3_seek_to_pcm_frame__seek_table(pMP3, frameIndex);\n    } else {\n        return ma_dr_mp3_seek_to_pcm_frame__brute_force(pMP3, frameIndex);\n    }\n}\nMA_API ma_bool32 ma_dr_mp3_get_mp3_and_pcm_frame_count(ma_dr_mp3* pMP3, ma_uint64* pMP3FrameCount, ma_uint64* pPCMFrameCount)\n{\n    ma_uint64 currentPCMFrame;\n    ma_uint64 totalPCMFrameCount;\n    ma_uint64 totalMP3FrameCount;\n    if (pMP3 == NULL) {\n        return MA_FALSE;\n    }\n    if (pMP3->onSeek == NULL) {\n        return MA_FALSE;\n    }\n    currentPCMFrame = pMP3->currentPCMFrame;\n    if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) {\n        return MA_FALSE;\n    }\n    totalPCMFrameCount = 0;\n    totalMP3FrameCount = 0;\n    for (;;) {\n        ma_uint32 pcmFramesInCurrentMP3Frame;\n        pcmFramesInCurrentMP3Frame = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL);\n        if (pcmFramesInCurrentMP3Frame == 0) {\n            break;\n        }\n        totalPCMFrameCount += pcmFramesInCurrentMP3Frame;\n        totalMP3FrameCount += 1;\n    }\n    if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) {\n        return MA_FALSE;\n    }\n    if (!ma_dr_mp3_seek_to_pcm_frame(pMP3, currentPCMFrame)) {\n        return MA_FALSE;\n    }\n    if (pMP3FrameCount != NULL) {\n        *pMP3FrameCount = totalMP3FrameCount;\n    }\n    if (pPCMFrameCount != NULL) {\n        *pPCMFrameCount = totalPCMFrameCount;\n    }\n    return MA_TRUE;\n}\nMA_API ma_uint64 ma_dr_mp3_get_pcm_frame_count(ma_dr_mp3* pMP3)\n{\n    ma_uint64 totalPCMFrameCount;\n    if (pMP3 == NULL) {\n        return 0;\n    }\n    if (pMP3->totalPCMFrameCount != MA_UINT64_MAX) {\n        totalPCMFrameCount = pMP3->totalPCMFrameCount;\n        if (totalPCMFrameCount >= pMP3->delayInPCMFrames) {\n            totalPCMFrameCount -= pMP3->delayInPCMFrames;\n        } else {\n        }\n        if (totalPCMFrameCount >= pMP3->paddingInPCMFrames) {\n            totalPCMFrameCount -= pMP3->paddingInPCMFrames;\n        } else {\n        }\n        return totalPCMFrameCount;\n    } else {\n        if (!ma_dr_mp3_get_mp3_and_pcm_frame_count(pMP3, NULL, &totalPCMFrameCount)) {\n            return 0;\n        }\n        return totalPCMFrameCount;\n    }\n}\nMA_API ma_uint64 ma_dr_mp3_get_mp3_frame_count(ma_dr_mp3* pMP3)\n{\n    ma_uint64 totalMP3FrameCount;\n    if (!ma_dr_mp3_get_mp3_and_pcm_frame_count(pMP3, &totalMP3FrameCount, NULL)) {\n        return 0;\n    }\n    return totalMP3FrameCount;\n}\nstatic void ma_dr_mp3__accumulate_running_pcm_frame_count(ma_dr_mp3* pMP3, ma_uint32 pcmFrameCountIn, ma_uint64* pRunningPCMFrameCount, float* pRunningPCMFrameCountFractionalPart)\n{\n    float srcRatio;\n    float pcmFrameCountOutF;\n    ma_uint32 pcmFrameCountOut;\n    srcRatio = (float)pMP3->mp3FrameSampleRate / (float)pMP3->sampleRate;\n    MA_DR_MP3_ASSERT(srcRatio > 0);\n    pcmFrameCountOutF = *pRunningPCMFrameCountFractionalPart + (pcmFrameCountIn / srcRatio);\n    pcmFrameCountOut  = (ma_uint32)pcmFrameCountOutF;\n    *pRunningPCMFrameCountFractionalPart = pcmFrameCountOutF - pcmFrameCountOut;\n    *pRunningPCMFrameCount += pcmFrameCountOut;\n}\ntypedef struct\n{\n    ma_uint64 bytePos;\n    ma_uint64 pcmFrameIndex;\n} ma_dr_mp3__seeking_mp3_frame_info;\nMA_API ma_bool32 ma_dr_mp3_calculate_seek_points(ma_dr_mp3* pMP3, ma_uint32* pSeekPointCount, ma_dr_mp3_seek_point* pSeekPoints)\n{\n    ma_uint32 seekPointCount;\n    ma_uint64 currentPCMFrame;\n    ma_uint64 totalMP3FrameCount;\n    ma_uint64 totalPCMFrameCount;\n    if (pMP3 == NULL || pSeekPointCount == NULL || pSeekPoints == NULL) {\n        return MA_FALSE;\n    }\n    seekPointCount = *pSeekPointCount;\n    if (seekPointCount == 0) {\n        return MA_FALSE;\n    }\n    currentPCMFrame = pMP3->currentPCMFrame;\n    if (!ma_dr_mp3_get_mp3_and_pcm_frame_count(pMP3, &totalMP3FrameCount, &totalPCMFrameCount)) {\n        return MA_FALSE;\n    }\n    if (totalMP3FrameCount < MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1) {\n        seekPointCount = 1;\n        pSeekPoints[0].seekPosInBytes     = 0;\n        pSeekPoints[0].pcmFrameIndex      = 0;\n        pSeekPoints[0].mp3FramesToDiscard = 0;\n        pSeekPoints[0].pcmFramesToDiscard = 0;\n    } else {\n        ma_uint64 pcmFramesBetweenSeekPoints;\n        ma_dr_mp3__seeking_mp3_frame_info mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1];\n        ma_uint64 runningPCMFrameCount = 0;\n        float runningPCMFrameCountFractionalPart = 0;\n        ma_uint64 nextTargetPCMFrame;\n        ma_uint32 iMP3Frame;\n        ma_uint32 iSeekPoint;\n        if (seekPointCount > totalMP3FrameCount-1) {\n            seekPointCount = (ma_uint32)totalMP3FrameCount-1;\n        }\n        pcmFramesBetweenSeekPoints = totalPCMFrameCount / (seekPointCount+1);\n        if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) {\n            return MA_FALSE;\n        }\n        for (iMP3Frame = 0; iMP3Frame < MA_DR_MP3_SEEK_LEADING_MP3_FRAMES+1; ++iMP3Frame) {\n            ma_uint32 pcmFramesInCurrentMP3FrameIn;\n            MA_DR_MP3_ASSERT(pMP3->streamCursor >= pMP3->dataSize);\n            mp3FrameInfo[iMP3Frame].bytePos       = pMP3->streamCursor - pMP3->dataSize;\n            mp3FrameInfo[iMP3Frame].pcmFrameIndex = runningPCMFrameCount;\n            pcmFramesInCurrentMP3FrameIn = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL);\n            if (pcmFramesInCurrentMP3FrameIn == 0) {\n                return MA_FALSE;\n            }\n            ma_dr_mp3__accumulate_running_pcm_frame_count(pMP3, pcmFramesInCurrentMP3FrameIn, &runningPCMFrameCount, &runningPCMFrameCountFractionalPart);\n        }\n        nextTargetPCMFrame = 0;\n        for (iSeekPoint = 0; iSeekPoint < seekPointCount; ++iSeekPoint) {\n            nextTargetPCMFrame += pcmFramesBetweenSeekPoints;\n            for (;;) {\n                if (nextTargetPCMFrame < runningPCMFrameCount) {\n                    pSeekPoints[iSeekPoint].seekPosInBytes     = mp3FrameInfo[0].bytePos;\n                    pSeekPoints[iSeekPoint].pcmFrameIndex      = nextTargetPCMFrame;\n                    pSeekPoints[iSeekPoint].mp3FramesToDiscard = MA_DR_MP3_SEEK_LEADING_MP3_FRAMES;\n                    pSeekPoints[iSeekPoint].pcmFramesToDiscard = (ma_uint16)(nextTargetPCMFrame - mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES-1].pcmFrameIndex);\n                    break;\n                } else {\n                    size_t i;\n                    ma_uint32 pcmFramesInCurrentMP3FrameIn;\n                    for (i = 0; i < MA_DR_MP3_COUNTOF(mp3FrameInfo)-1; ++i) {\n                        mp3FrameInfo[i] = mp3FrameInfo[i+1];\n                    }\n                    mp3FrameInfo[MA_DR_MP3_COUNTOF(mp3FrameInfo)-1].bytePos       = pMP3->streamCursor - pMP3->dataSize;\n                    mp3FrameInfo[MA_DR_MP3_COUNTOF(mp3FrameInfo)-1].pcmFrameIndex = runningPCMFrameCount;\n                    pcmFramesInCurrentMP3FrameIn = ma_dr_mp3_decode_next_frame_ex(pMP3, NULL, NULL, NULL);\n                    if (pcmFramesInCurrentMP3FrameIn == 0) {\n                        pSeekPoints[iSeekPoint].seekPosInBytes     = mp3FrameInfo[0].bytePos;\n                        pSeekPoints[iSeekPoint].pcmFrameIndex      = nextTargetPCMFrame;\n                        pSeekPoints[iSeekPoint].mp3FramesToDiscard = MA_DR_MP3_SEEK_LEADING_MP3_FRAMES;\n                        pSeekPoints[iSeekPoint].pcmFramesToDiscard = (ma_uint16)(nextTargetPCMFrame - mp3FrameInfo[MA_DR_MP3_SEEK_LEADING_MP3_FRAMES-1].pcmFrameIndex);\n                        break;\n                    }\n                    ma_dr_mp3__accumulate_running_pcm_frame_count(pMP3, pcmFramesInCurrentMP3FrameIn, &runningPCMFrameCount, &runningPCMFrameCountFractionalPart);\n                }\n            }\n        }\n        if (!ma_dr_mp3_seek_to_start_of_stream(pMP3)) {\n            return MA_FALSE;\n        }\n        if (!ma_dr_mp3_seek_to_pcm_frame(pMP3, currentPCMFrame)) {\n            return MA_FALSE;\n        }\n    }\n    *pSeekPointCount = seekPointCount;\n    return MA_TRUE;\n}\nMA_API ma_bool32 ma_dr_mp3_bind_seek_table(ma_dr_mp3* pMP3, ma_uint32 seekPointCount, ma_dr_mp3_seek_point* pSeekPoints)\n{\n    if (pMP3 == NULL) {\n        return MA_FALSE;\n    }\n    if (seekPointCount == 0 || pSeekPoints == NULL) {\n        pMP3->seekPointCount = 0;\n        pMP3->pSeekPoints = NULL;\n    } else {\n        pMP3->seekPointCount = seekPointCount;\n        pMP3->pSeekPoints = pSeekPoints;\n    }\n    return MA_TRUE;\n}\nstatic float* ma_dr_mp3__full_read_and_close_f32(ma_dr_mp3* pMP3, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount)\n{\n    ma_uint64 totalFramesRead = 0;\n    ma_uint64 framesCapacity = 0;\n    float* pFrames = NULL;\n    float temp[4096];\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    for (;;) {\n        ma_uint64 framesToReadRightNow = MA_DR_MP3_COUNTOF(temp) / pMP3->channels;\n        ma_uint64 framesJustRead = ma_dr_mp3_read_pcm_frames_f32(pMP3, framesToReadRightNow, temp);\n        if (framesJustRead == 0) {\n            break;\n        }\n        if (framesCapacity < totalFramesRead + framesJustRead) {\n            ma_uint64 oldFramesBufferSize;\n            ma_uint64 newFramesBufferSize;\n            ma_uint64 newFramesCap;\n            float* pNewFrames;\n            newFramesCap = framesCapacity * 2;\n            if (newFramesCap < totalFramesRead + framesJustRead) {\n                newFramesCap = totalFramesRead + framesJustRead;\n            }\n            oldFramesBufferSize = framesCapacity * pMP3->channels * sizeof(float);\n            newFramesBufferSize = newFramesCap   * pMP3->channels * sizeof(float);\n            if (newFramesBufferSize > (ma_uint64)MA_SIZE_MAX) {\n                break;\n            }\n            pNewFrames = (float*)ma_dr_mp3__realloc_from_callbacks(pFrames, (size_t)newFramesBufferSize, (size_t)oldFramesBufferSize, &pMP3->allocationCallbacks);\n            if (pNewFrames == NULL) {\n                ma_dr_mp3__free_from_callbacks(pFrames, &pMP3->allocationCallbacks);\n                pFrames = NULL;\n                totalFramesRead = 0;\n                break;\n            }\n            pFrames = pNewFrames;\n            framesCapacity = newFramesCap;\n        }\n        MA_DR_MP3_COPY_MEMORY(pFrames + totalFramesRead*pMP3->channels, temp, (size_t)(framesJustRead*pMP3->channels*sizeof(float)));\n        totalFramesRead += framesJustRead;\n        if (framesJustRead != framesToReadRightNow) {\n            break;\n        }\n    }\n    if (pConfig != NULL) {\n        pConfig->channels   = pMP3->channels;\n        pConfig->sampleRate = pMP3->sampleRate;\n    }\n    ma_dr_mp3_uninit(pMP3);\n    if (pTotalFrameCount) {\n        *pTotalFrameCount = totalFramesRead;\n    }\n    return pFrames;\n}\nstatic ma_int16* ma_dr_mp3__full_read_and_close_s16(ma_dr_mp3* pMP3, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount)\n{\n    ma_uint64 totalFramesRead = 0;\n    ma_uint64 framesCapacity = 0;\n    ma_int16* pFrames = NULL;\n    ma_int16 temp[4096];\n    MA_DR_MP3_ASSERT(pMP3 != NULL);\n    for (;;) {\n        ma_uint64 framesToReadRightNow = MA_DR_MP3_COUNTOF(temp) / pMP3->channels;\n        ma_uint64 framesJustRead = ma_dr_mp3_read_pcm_frames_s16(pMP3, framesToReadRightNow, temp);\n        if (framesJustRead == 0) {\n            break;\n        }\n        if (framesCapacity < totalFramesRead + framesJustRead) {\n            ma_uint64 newFramesBufferSize;\n            ma_uint64 oldFramesBufferSize;\n            ma_uint64 newFramesCap;\n            ma_int16* pNewFrames;\n            newFramesCap = framesCapacity * 2;\n            if (newFramesCap < totalFramesRead + framesJustRead) {\n                newFramesCap = totalFramesRead + framesJustRead;\n            }\n            oldFramesBufferSize = framesCapacity * pMP3->channels * sizeof(ma_int16);\n            newFramesBufferSize = newFramesCap   * pMP3->channels * sizeof(ma_int16);\n            if (newFramesBufferSize > (ma_uint64)MA_SIZE_MAX) {\n                break;\n            }\n            pNewFrames = (ma_int16*)ma_dr_mp3__realloc_from_callbacks(pFrames, (size_t)newFramesBufferSize, (size_t)oldFramesBufferSize, &pMP3->allocationCallbacks);\n            if (pNewFrames == NULL) {\n                ma_dr_mp3__free_from_callbacks(pFrames, &pMP3->allocationCallbacks);\n                pFrames = NULL;\n                totalFramesRead = 0;\n                break;\n            }\n            pFrames = pNewFrames;\n            framesCapacity = newFramesCap;\n        }\n        MA_DR_MP3_COPY_MEMORY(pFrames + totalFramesRead*pMP3->channels, temp, (size_t)(framesJustRead*pMP3->channels*sizeof(ma_int16)));\n        totalFramesRead += framesJustRead;\n        if (framesJustRead != framesToReadRightNow) {\n            break;\n        }\n    }\n    if (pConfig != NULL) {\n        pConfig->channels   = pMP3->channels;\n        pConfig->sampleRate = pMP3->sampleRate;\n    }\n    ma_dr_mp3_uninit(pMP3);\n    if (pTotalFrameCount) {\n        *pTotalFrameCount = totalFramesRead;\n    }\n    return pFrames;\n}\nMA_API float* ma_dr_mp3_open_and_read_pcm_frames_f32(ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, void* pUserData, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init(&mp3, onRead, onSeek, onTell, NULL, pUserData, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_f32(&mp3, pConfig, pTotalFrameCount);\n}\nMA_API ma_int16* ma_dr_mp3_open_and_read_pcm_frames_s16(ma_dr_mp3_read_proc onRead, ma_dr_mp3_seek_proc onSeek, ma_dr_mp3_tell_proc onTell, void* pUserData, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init(&mp3, onRead, onSeek, onTell, NULL, pUserData, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_s16(&mp3, pConfig, pTotalFrameCount);\n}\nMA_API float* ma_dr_mp3_open_memory_and_read_pcm_frames_f32(const void* pData, size_t dataSize, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init_memory(&mp3, pData, dataSize, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_f32(&mp3, pConfig, pTotalFrameCount);\n}\nMA_API ma_int16* ma_dr_mp3_open_memory_and_read_pcm_frames_s16(const void* pData, size_t dataSize, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init_memory(&mp3, pData, dataSize, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_s16(&mp3, pConfig, pTotalFrameCount);\n}\n#ifndef MA_DR_MP3_NO_STDIO\nMA_API float* ma_dr_mp3_open_file_and_read_pcm_frames_f32(const char* filePath, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init_file(&mp3, filePath, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_f32(&mp3, pConfig, pTotalFrameCount);\n}\nMA_API ma_int16* ma_dr_mp3_open_file_and_read_pcm_frames_s16(const char* filePath, ma_dr_mp3_config* pConfig, ma_uint64* pTotalFrameCount, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    ma_dr_mp3 mp3;\n    if (!ma_dr_mp3_init_file(&mp3, filePath, pAllocationCallbacks)) {\n        return NULL;\n    }\n    return ma_dr_mp3__full_read_and_close_s16(&mp3, pConfig, pTotalFrameCount);\n}\n#endif\nMA_API void* ma_dr_mp3_malloc(size_t sz, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        return ma_dr_mp3__malloc_from_callbacks(sz, pAllocationCallbacks);\n    } else {\n        return ma_dr_mp3__malloc_default(sz, NULL);\n    }\n}\nMA_API void ma_dr_mp3_free(void* p, const ma_allocation_callbacks* pAllocationCallbacks)\n{\n    if (pAllocationCallbacks != NULL) {\n        ma_dr_mp3__free_from_callbacks(p, pAllocationCallbacks);\n    } else {\n        ma_dr_mp3__free_default(p, NULL);\n    }\n}\n#endif\n/* dr_mp3_c end */\n#endif  /* MA_DR_MP3_IMPLEMENTATION */\n#endif  /* MA_NO_MP3 */\n\n\n/* End globally disabled warnings. */\n#if defined(_MSC_VER)\n    #pragma warning(pop)\n#endif\n\n#endif  /* miniaudio_c */\n#endif  /* MINIAUDIO_IMPLEMENTATION */\n\n\n/*\nThis software is available as a choice of the following licenses. Choose\nwhichever you prefer.\n\n===============================================================================\nALTERNATIVE 1 - Public Domain (www.unlicense.org)\n===============================================================================\nThis is free and unencumbered software released into the public domain.\n\nAnyone is free to copy, modify, publish, use, compile, sell, or distribute this\nsoftware, either in source code form or as a compiled binary, for any purpose,\ncommercial or non-commercial, and by any means.\n\nIn jurisdictions that recognize copyright laws, the author or authors of this\nsoftware dedicate any and all copyright interest in the software to the public\ndomain. We make this dedication for the benefit of the public at large and to\nthe detriment of our heirs and successors. We intend this dedication to be an\novert act of relinquishment in perpetuity of all present and future rights to\nthis software under copyright law.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\nACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nFor more information, please refer to <http://unlicense.org/>\n\n===============================================================================\nALTERNATIVE 2 - MIT No Attribution\n===============================================================================\nCopyright 2025 David Reid\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n*/\n"
  },
  {
    "path": "examples/python/test_whisper_processor.py",
    "content": "import whisper_processor\n\ntry:\n    result = whisper_processor.process_audio(\"./audio/wake_word_detected16k.wav\", \"base.en\")\n    print(result)\nexcept Exception as e:\n    print(f\"Error: {e}\")"
  },
  {
    "path": "examples/python/whisper_processor.py",
    "content": "import subprocess\nimport sys\nimport os\n\ndef process_audio(wav_file, model_name=\"base.en\"):\n    \"\"\"\n    Processes an audio file using a specified model and returns the processed string.\n\n    :param wav_file: Path to the WAV file\n    :param model_name: Name of the model to use\n    :return: Processed string output from the audio processing\n    :raises: Exception if an error occurs during processing\n    \"\"\"\n\n    model = f\"./models/ggml-{model_name}.bin\"\n\n    # Check if the file exists\n    if not os.path.exists(model):\n        raise FileNotFoundError(f\"Model file not found: {model} \\n\\nDownload a model with this command:\\n\\n> bash ./models/download-ggml-model.sh {model_name}\\n\\n\")\n\n    if not os.path.exists(wav_file):\n        raise FileNotFoundError(f\"WAV file not found: {wav_file}\")\n\n    full_command = f\"./main -m {model} -f {wav_file} -nt\"\n\n    # Execute the command\n    process = subprocess.Popen(full_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\n    # Get the output and error (if any)\n    output, error = process.communicate()\n\n    if error:\n        raise Exception(f\"Error processing audio: {error.decode('utf-8')}\")\n\n    # Process and return the output string\n    decoded_str = output.decode('utf-8').strip()\n    processed_str = decoded_str.replace('[BLANK_AUDIO]', '').strip()\n\n    return processed_str\n\ndef main():\n    if len(sys.argv) >= 2:\n        wav_file = sys.argv[1]\n        model_name = sys.argv[2] if len(sys.argv) == 3 else \"base.en\"\n        try:\n            result = process_audio(wav_file, model_name)\n            print(result)\n        except Exception as e:\n            print(f\"Error: {e}\")\n    else:\n        print(\"Usage: python whisper_processor.py <wav_file> [<model_name>]\")\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "examples/quantize/CMakeLists.txt",
    "content": "set(TARGET whisper-quantize)\nadd_executable(${TARGET} quantize.cpp)\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE common whisper ${CMAKE_THREAD_LIBS_INIT})\ninstall(TARGETS ${TARGET} RUNTIME)\n"
  },
  {
    "path": "examples/quantize/README.md",
    "content": "# quantize\r\n\r\nTool for integer quantization of Whisper `ggml` model files\r\n"
  },
  {
    "path": "examples/quantize/quantize.cpp",
    "content": "#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include \"common.h\"\n#include \"common-ggml.h\"\n\n#include <cassert>\n#include <cmath>\n#include <cstdio>\n#include <cstring>\n#include <fstream>\n#include <map>\n#include <string>\n#include <vector>\n#include <regex>\n\n// default hparams (Whisper tiny)\nstruct whisper_hparams {\n    int32_t n_vocab       = 51864;\n    int32_t n_audio_ctx   = 1500;\n    int32_t n_audio_state = 384;\n    int32_t n_audio_head  = 6;\n    int32_t n_audio_layer = 4;\n    int32_t n_text_ctx    = 448;\n    int32_t n_text_state  = 384;\n    int32_t n_text_head   = 6;\n    int32_t n_text_layer  = 4;\n    int32_t n_mels        = 80;\n    int32_t ftype         = 1;\n};\n\nstruct whisper_filters {\n    int32_t n_mel;\n    int32_t n_fft;\n\n    std::vector<float> data;\n};\n\n// quantize a model\nstatic bool whisper_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {\n    gpt_vocab vocab;\n\n    printf(\"%s: loading model from '%s'\\n\", __func__, fname_inp.c_str());\n\n    auto finp = std::ifstream(fname_inp, std::ios::binary);\n    if (!finp) {\n        fprintf(stderr, \"%s: failed to open '%s' for reading\\n\", __func__, fname_inp.c_str());\n        return false;\n    }\n\n    auto fout = std::ofstream(fname_out, std::ios::binary);\n    if (!fout) {\n        fprintf(stderr, \"%s: failed to open '%s' for writing\\n\", __func__, fname_out.c_str());\n        return false;\n    }\n\n    // verify magic\n    {\n        uint32_t magic;\n        finp.read((char *) &magic, sizeof(magic));\n        if (magic != GGML_FILE_MAGIC) {\n            fprintf(stderr, \"%s: invalid model file '%s' (bad magic)\\n\", __func__, fname_inp.c_str());\n            return false;\n        }\n\n        fout.write((char *) &magic, sizeof(magic));\n    }\n\n    whisper_hparams hparams;\n\n    // load hparams\n    {\n        finp.read((char *) &hparams.n_vocab,       sizeof(hparams.n_vocab));\n        finp.read((char *) &hparams.n_audio_ctx,   sizeof(hparams.n_audio_ctx));\n        finp.read((char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));\n        finp.read((char *) &hparams.n_audio_head,  sizeof(hparams.n_audio_head));\n        finp.read((char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));\n        finp.read((char *) &hparams.n_text_ctx,    sizeof(hparams.n_text_ctx));\n        finp.read((char *) &hparams.n_text_state,  sizeof(hparams.n_text_state));\n        finp.read((char *) &hparams.n_text_head,   sizeof(hparams.n_text_head));\n        finp.read((char *) &hparams.n_text_layer,  sizeof(hparams.n_text_layer));\n        finp.read((char *) &hparams.n_mels,        sizeof(hparams.n_mels));\n        finp.read((char *) &hparams.ftype,         sizeof(hparams.ftype));\n\n        const int32_t qntvr_src =    hparams.ftype / GGML_QNT_VERSION_FACTOR;\n        const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;\n\n        fprintf(stderr, \"%s: n_vocab       = %d\\n\", __func__, hparams.n_vocab);\n        fprintf(stderr, \"%s: n_audio_ctx   = %d\\n\", __func__, hparams.n_audio_ctx);\n        fprintf(stderr, \"%s: n_audio_state = %d\\n\", __func__, hparams.n_audio_state);\n        fprintf(stderr, \"%s: n_audio_head  = %d\\n\", __func__, hparams.n_audio_head);\n        fprintf(stderr, \"%s: n_audio_layer = %d\\n\", __func__, hparams.n_audio_layer);\n        fprintf(stderr, \"%s: n_text_ctx    = %d\\n\", __func__, hparams.n_text_ctx);\n        fprintf(stderr, \"%s: n_text_state  = %d\\n\", __func__, hparams.n_text_state);\n        fprintf(stderr, \"%s: n_text_head   = %d\\n\", __func__, hparams.n_text_head);\n        fprintf(stderr, \"%s: n_text_layer  = %d\\n\", __func__, hparams.n_text_layer);\n        fprintf(stderr, \"%s: n_mels        = %d\\n\", __func__, hparams.n_mels);\n        fprintf(stderr, \"%s: ftype (src)   = %d\\n\", __func__, hparams.ftype);\n        fprintf(stderr, \"%s: qntvr (src)   = %d\\n\", __func__, qntvr_src);\n        fprintf(stderr, \"%s: ftype (dst)   = %d\\n\", __func__, ftype_dst);\n        fprintf(stderr, \"%s: qntvr (dst)   = %d\\n\", __func__, GGML_QNT_VERSION);\n\n        fout.write((const char *) &hparams.n_vocab,       sizeof(hparams.n_vocab));\n        fout.write((const char *) &hparams.n_audio_ctx,   sizeof(hparams.n_audio_ctx));\n        fout.write((const char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));\n        fout.write((const char *) &hparams.n_audio_head,  sizeof(hparams.n_audio_head));\n        fout.write((const char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));\n        fout.write((const char *) &hparams.n_text_ctx,    sizeof(hparams.n_text_ctx));\n        fout.write((const char *) &hparams.n_text_state,  sizeof(hparams.n_text_state));\n        fout.write((const char *) &hparams.n_text_head,   sizeof(hparams.n_text_head));\n        fout.write((const char *) &hparams.n_text_layer,  sizeof(hparams.n_text_layer));\n        fout.write((const char *) &hparams.n_mels,        sizeof(hparams.n_mels));\n        fout.write((const char *) &ftype_dst,             sizeof(hparams.ftype));\n    }\n\n    // load mel filters\n    {\n        whisper_filters filters;\n\n        finp.read ((char *) &filters.n_mel, sizeof(filters.n_mel));\n        fout.write((char *) &filters.n_mel, sizeof(filters.n_mel));\n        finp.read ((char *) &filters.n_fft, sizeof(filters.n_fft));\n        fout.write((char *) &filters.n_fft, sizeof(filters.n_fft));\n\n        filters.data.resize(filters.n_mel * filters.n_fft);\n        finp.read ((char *) filters.data.data(), filters.data.size() * sizeof(float));\n        fout.write((char *) filters.data.data(), filters.data.size() * sizeof(float));\n    }\n\n    // load vocab\n    {\n        int32_t n_vocab = 0;\n        finp.read ((char *) &n_vocab, sizeof(n_vocab));\n        fout.write((char *) &n_vocab, sizeof(n_vocab));\n\n        //if (n_vocab != hparams.n_vocab) {\n        //    fprintf(stderr, \"%s: invalid model file '%s' (bad vocab size %d != %d)\\n\",\n        //            __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);\n        //    return false;\n        //}\n\n        char word[129];\n\n        for (int i = 0; i < n_vocab; i++) {\n            uint32_t len;\n            finp.read ((char *) &len, sizeof(len));\n            fout.write((char *) &len, sizeof(len));\n\n            word[len] = '\\0';\n\n            finp.read ((char *) word, len);\n            fout.write((char *) word, len);\n\n            vocab.token_to_id[word] = i;\n            vocab.id_to_token[i] = word;\n        }\n    }\n\n    // regexes of tensor names to not be quantized\n    const std::vector<std::string> to_skip = {\n        //\"encoder.*\",\n        \"encoder.conv1.bias\",\n        \"encoder.conv2.bias\",\n        \"encoder.positional_embedding\",\n        \"decoder.positional_embedding\",\n    };\n\n    if (!ggml_common_quantize_0(finp, fout, ftype, { \".*\" }, to_skip)) {\n        fprintf(stderr, \"%s: failed to quantize model '%s'\\n\", __func__, fname_inp.c_str());\n        return false;\n    }\n\n    finp.close();\n    fout.close();\n\n    return true;\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    if (argc != 4) {\n        fprintf(stderr, \"usage: %s model-f32.bin model-quant.bin type\\n\", argv[0]);\n        ggml_print_ftypes(stderr);\n        return 1;\n    }\n\n    // needed to initialize f16 tables\n    {\n        struct ggml_init_params params = { 0, NULL, false };\n        struct ggml_context * ctx = ggml_init(params);\n        ggml_free(ctx);\n    }\n\n    const std::string fname_inp = argv[1];\n    const std::string fname_out = argv[2];\n\n    const ggml_ftype ftype = ggml_parse_ftype(argv[3]);\n\n    const int64_t t_main_start_us = ggml_time_us();\n\n    int64_t t_quantize_us = 0;\n\n    // load the model\n    {\n        const int64_t t_start_us = ggml_time_us();\n\n        if (!whisper_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {\n            fprintf(stderr, \"%s: failed to quantize model from '%s'\\n\", __func__, fname_inp.c_str());\n            return 1;\n        }\n\n        t_quantize_us = ggml_time_us() - t_start_us;\n    }\n\n    // report timing\n    {\n        const int64_t t_main_end_us = ggml_time_us();\n\n        printf(\"\\n\");\n        printf(\"%s: quantize time = %8.2f ms\\n\", __func__, t_quantize_us/1000.0f);\n        printf(\"%s:    total time = %8.2f ms\\n\", __func__, (t_main_end_us - t_main_start_us)/1000.0f);\n    }\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/server/CMakeLists.txt",
    "content": "set(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED ON)\n\nset(TARGET whisper-server)\nadd_executable(${TARGET} server.cpp httplib.h)\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE common json_cpp whisper ${CMAKE_THREAD_LIBS_INIT})\n\nif (WIN32)\n    target_link_libraries(${TARGET} PRIVATE ws2_32)\nendif()\n\ninstall(TARGETS ${TARGET} RUNTIME)\n"
  },
  {
    "path": "examples/server/README.md",
    "content": "# whisper.cpp/examples/server\n\nSimple http server. WAV Files are passed to the inference model via http requests.\n\nhttps://github.com/ggerganov/whisper.cpp/assets/1991296/e983ee53-8741-4eb5-9048-afe5e4594b8f\n\n## Usage\n\n```\n./build/bin/whisper-server -h\n\nusage: ./build/bin/whisper-server [options]\n\noptions:\n  -h,        --help              [default] show this help message and exit\n  -t N,      --threads N         [4      ] number of threads to use during computation\n  -p N,      --processors N      [1      ] number of processors to use during computation\n  -ot N,     --offset-t N        [0      ] time offset in milliseconds\n  -on N,     --offset-n N        [0      ] segment index offset\n  -d  N,     --duration N        [0      ] duration of audio to process in milliseconds\n  -mc N,     --max-context N     [-1     ] maximum number of text context tokens to store\n  -ml N,     --max-len N         [0      ] maximum segment length in characters\n  -sow,      --split-on-word     [false  ] split on word rather than on token\n  -bo N,     --best-of N         [2      ] number of best candidates to keep\n  -bs N,     --beam-size N       [-1     ] beam size for beam search\n  -ac N,     --audio-ctx N       [0      ] audio context size (0 - all)\n  -wt N,     --word-thold N      [0.01   ] word timestamp probability threshold\n  -et N,     --entropy-thold N   [2.40   ] entropy threshold for decoder fail\n  -lpt N,    --logprob-thold N   [-1.00  ] log probability threshold for decoder fail\n  -debug,    --debug-mode        [false  ] enable debug mode (eg. dump log_mel)\n  -tr,       --translate         [false  ] translate from source language to english\n  -di,       --diarize           [false  ] stereo audio diarization\n  -tdrz,     --tinydiarize       [false  ] enable tinydiarize (requires a tdrz model)\n  -nf,       --no-fallback       [false  ] do not use temperature fallback while decoding\n  -ps,       --print-special     [false  ] print special tokens\n  -pc,       --print-colors      [false  ] print colors\n  -pr,       --print-realtime    [false  ] print output in realtime\n  -pp,       --print-progress    [false  ] print progress\n  -nt,       --no-timestamps     [false  ] do not print timestamps\n  -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)\n  -dl,       --detect-language   [false  ] exit after automatically detecting language\n             --prompt PROMPT     [       ] initial prompt\n  -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path\n  -oved D,   --ov-e-device DNAME [CPU    ] the OpenVINO device used for encode inference\n  -dtw MODEL --dtw MODEL         [       ] compute token-level timestamps\n  --host HOST,                   [127.0.0.1] Hostname/ip-adress for the server\n  --port PORT,                   [8080   ] Port number for the server\n  --public PATH,                 [examples/server/public] Path to the public folder\n  --request-path PATH,           [       ] Request path for all requests\n  --inference-path PATH,         [/inference] Inference path for all requests\n  --convert,                     [false  ] Convert audio to WAV, requires ffmpeg on the server\n  -sns,      --suppress-nst      [false  ] suppress non-speech tokens\n  -nth N,    --no-speech-thold N [0.60   ] no speech threshold\n  -nc,       --no-context        [false  ] do not use previous audio context\n  -ng,       --no-gpu            [false  ] do not use gpu\n  -fa,       --flash-attn        [false  ] flash attention\n\nVoice Activity Detection (VAD) options:\n             --vad                           [false  ] enable Voice Activity Detection (VAD)\n  -vm FNAME, --vad-model FNAME               [       ] VAD model path\n  -vt N,     --vad-threshold N               [0.50   ] VAD threshold for speech recognition\n  -vspd N,   --vad-min-speech-duration-ms  N [250    ] VAD min speech duration (0.0-1.0)\n  -vsd N,    --vad-min-silence-duration-ms N [100    ] VAD min silence duration (to split segments)\n  -vmsd N,   --vad-max-speech-duration-s   N [FLT_MAX] VAD max speech duration (auto-split longer)\n  -vp N,     --vad-speech-pad-ms           N [30     ] VAD speech padding (extend segments)\n  -vo N,     --vad-samples-overlap         N [0.10   ] VAD samples overlap (seconds between segments)\n```\n\n> [!WARNING]\n> **Do not run the server example with administrative privileges and ensure it's operated in a sandbox environment, especially since it involves risky operations like accepting user file uploads and using ffmpeg for format conversions. Always validate and sanitize inputs to guard against potential security threats.**\n\n## request examples\n\n**/inference**\n```\ncurl 127.0.0.1:8080/inference \\\n-H \"Content-Type: multipart/form-data\" \\\n-F file=\"@<file-path>\" \\\n-F temperature=\"0.0\" \\\n-F temperature_inc=\"0.2\" \\\n-F response_format=\"json\"\n```\n\n**/load**\n```\ncurl 127.0.0.1:8080/load \\\n-H \"Content-Type: multipart/form-data\" \\\n-F model=\"<path-to-model-file>\"\n```\n\n## Load testing with k6\n\n> **Note:** Install [k6](https://k6.io/docs/get-started/installation/) before running the benchmark script.\n\nYou can benchmark the Whisper server using the provided bench.js script with [k6](https://k6.io/). This script sends concurrent multipart requests to the /inference endpoint and is fully configurable via environment variables.\n\n**Example usage:**\n\n```\nk6 run bench.js \\\n  --env FILE_PATH=/absolute/path/to/samples/jfk.wav \\\n  --env BASE_URL=http://127.0.0.1:8080 \\\n  --env ENDPOINT=/inference \\\n  --env CONCURRENCY=4 \\\n  --env TEMPERATURE=0.0 \\\n  --env TEMPERATURE_INC=0.2 \\\n  --env RESPONSE_FORMAT=json\n```\n\n**Environment variables:**\n- `FILE_PATH`: Path to the audio file to send (must be absolute or relative to the k6 working directory)\n- `BASE_URL`: Server base URL (default: `http://127.0.0.1:8080`)\n- `ENDPOINT`: API endpoint (default: `/inference`)\n- `CONCURRENCY`: Number of concurrent requests (default: 4)\n- `TEMPERATURE`: Decoding temperature (default: 0.0)\n- `TEMPERATURE_INC`: Temperature increment (default: 0.2)\n- `RESPONSE_FORMAT`: Response format (default: `json`)\n\n**Note:**\n- The server must be running and accessible at the specified `BASE_URL` and `ENDPOINT`.\n- The script is located in the same directory as this README: `bench.js`.\n"
  },
  {
    "path": "examples/server/bench.js",
    "content": "import http from 'k6/http'\nimport { check } from 'k6'\n\nexport let options = {\n  vus: parseInt(__ENV.CONCURRENCY) || 4,\n  iterations: parseInt(__ENV.CONCURRENCY) || 4,\n}\n\nconst filePath        = __ENV.FILE_PATH\nconst baseURL         = __ENV.BASE_URL        || 'http://127.0.0.1:8080'\nconst endpoint        = __ENV.ENDPOINT        || '/inference'\nconst temperature     = __ENV.TEMPERATURE     || '0.0'\nconst temperatureInc  = __ENV.TEMPERATURE_INC || '0.2'\nconst responseFormat  = __ENV.RESPONSE_FORMAT || 'json'\n\n// Read the file ONCE at init time\nconst fileBin = open(filePath, 'b')\n\nexport default function () {\n  const payload = {\n    file:           http.file(fileBin, filePath),\n    temperature:    temperature,\n    temperature_inc: temperatureInc,\n    response_format: responseFormat,\n  }\n\n  const res = http.post(`${baseURL}${endpoint}`, payload)\n  check(res, { 'status is 200': r => r.status === 200 })\n} "
  },
  {
    "path": "examples/server/httplib.h",
    "content": "//\n//  httplib.h\n//\n//  Copyright (c) 2025 Yuji Hirose. All rights reserved.\n//  MIT License\n//\n\n#ifndef CPPHTTPLIB_HTTPLIB_H\n#define CPPHTTPLIB_HTTPLIB_H\n\n#define CPPHTTPLIB_VERSION \"0.20.0\"\n\n/*\n * Configuration\n */\n\n#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND\n#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND 5\n#endif\n\n#ifndef CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND\n#define CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND 10000\n#endif\n\n#ifndef CPPHTTPLIB_KEEPALIVE_MAX_COUNT\n#define CPPHTTPLIB_KEEPALIVE_MAX_COUNT 100\n#endif\n\n#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND\n#define CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND 300\n#endif\n\n#ifndef CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND\n#define CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND\n#define CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND 5\n#endif\n\n#ifndef CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND\n#define CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND\n#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND 5\n#endif\n\n#ifndef CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND\n#define CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND\n#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND 300\n#endif\n\n#ifndef CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND\n#define CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND\n#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND 5\n#endif\n\n#ifndef CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND\n#define CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND\n#define CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_IDLE_INTERVAL_SECOND\n#define CPPHTTPLIB_IDLE_INTERVAL_SECOND 0\n#endif\n\n#ifndef CPPHTTPLIB_IDLE_INTERVAL_USECOND\n#ifdef _WIN32\n#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 10000\n#else\n#define CPPHTTPLIB_IDLE_INTERVAL_USECOND 0\n#endif\n#endif\n\n#ifndef CPPHTTPLIB_REQUEST_URI_MAX_LENGTH\n#define CPPHTTPLIB_REQUEST_URI_MAX_LENGTH 8192\n#endif\n\n#ifndef CPPHTTPLIB_HEADER_MAX_LENGTH\n#define CPPHTTPLIB_HEADER_MAX_LENGTH 8192\n#endif\n\n#ifndef CPPHTTPLIB_REDIRECT_MAX_COUNT\n#define CPPHTTPLIB_REDIRECT_MAX_COUNT 20\n#endif\n\n#ifndef CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT\n#define CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT 1024\n#endif\n\n#ifndef CPPHTTPLIB_PAYLOAD_MAX_LENGTH\n#define CPPHTTPLIB_PAYLOAD_MAX_LENGTH ((std::numeric_limits<size_t>::max)())\n#endif\n\n#ifndef CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH\n#define CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH 8192\n#endif\n\n#ifndef CPPHTTPLIB_RANGE_MAX_COUNT\n#define CPPHTTPLIB_RANGE_MAX_COUNT 1024\n#endif\n\n#ifndef CPPHTTPLIB_TCP_NODELAY\n#define CPPHTTPLIB_TCP_NODELAY false\n#endif\n\n#ifndef CPPHTTPLIB_IPV6_V6ONLY\n#define CPPHTTPLIB_IPV6_V6ONLY false\n#endif\n\n#ifndef CPPHTTPLIB_RECV_BUFSIZ\n#define CPPHTTPLIB_RECV_BUFSIZ size_t(16384u)\n#endif\n\n#ifndef CPPHTTPLIB_COMPRESSION_BUFSIZ\n#define CPPHTTPLIB_COMPRESSION_BUFSIZ size_t(16384u)\n#endif\n\n#ifndef CPPHTTPLIB_THREAD_POOL_COUNT\n#define CPPHTTPLIB_THREAD_POOL_COUNT                                           \\\n  ((std::max)(8u, std::thread::hardware_concurrency() > 0                      \\\n                      ? std::thread::hardware_concurrency() - 1                \\\n                      : 0))\n#endif\n\n#ifndef CPPHTTPLIB_RECV_FLAGS\n#define CPPHTTPLIB_RECV_FLAGS 0\n#endif\n\n#ifndef CPPHTTPLIB_SEND_FLAGS\n#define CPPHTTPLIB_SEND_FLAGS 0\n#endif\n\n#ifndef CPPHTTPLIB_LISTEN_BACKLOG\n#define CPPHTTPLIB_LISTEN_BACKLOG 5\n#endif\n\n/*\n * Headers\n */\n\n#ifdef _WIN32\n#ifndef _CRT_SECURE_NO_WARNINGS\n#define _CRT_SECURE_NO_WARNINGS\n#endif //_CRT_SECURE_NO_WARNINGS\n\n#ifndef _CRT_NONSTDC_NO_DEPRECATE\n#define _CRT_NONSTDC_NO_DEPRECATE\n#endif //_CRT_NONSTDC_NO_DEPRECATE\n\n#if defined(_MSC_VER)\n#if _MSC_VER < 1900\n#error Sorry, Visual Studio versions prior to 2015 are not supported\n#endif\n\n#pragma comment(lib, \"ws2_32.lib\")\n\n#ifdef _WIN64\nusing ssize_t = __int64;\n#else\nusing ssize_t = long;\n#endif\n#endif // _MSC_VER\n\n#ifndef S_ISREG\n#define S_ISREG(m) (((m) & S_IFREG) == S_IFREG)\n#endif // S_ISREG\n\n#ifndef S_ISDIR\n#define S_ISDIR(m) (((m) & S_IFDIR) == S_IFDIR)\n#endif // S_ISDIR\n\n#ifndef NOMINMAX\n#define NOMINMAX\n#endif // NOMINMAX\n\n#include <io.h>\n#include <winsock2.h>\n#include <ws2tcpip.h>\n\n#ifndef WSA_FLAG_NO_HANDLE_INHERIT\n#define WSA_FLAG_NO_HANDLE_INHERIT 0x80\n#endif\n\nusing nfds_t = unsigned long;\nusing socket_t = SOCKET;\nusing socklen_t = int;\n\n#else // not _WIN32\n\n#include <arpa/inet.h>\n#if !defined(_AIX) && !defined(__MVS__)\n#include <ifaddrs.h>\n#endif\n#ifdef __MVS__\n#include <strings.h>\n#ifndef NI_MAXHOST\n#define NI_MAXHOST 1025\n#endif\n#endif\n#include <net/if.h>\n#include <netdb.h>\n#include <netinet/in.h>\n#ifdef __linux__\n#include <resolv.h>\n#endif\n#include <csignal>\n#include <netinet/tcp.h>\n#include <poll.h>\n#include <pthread.h>\n#include <sys/mman.h>\n#include <sys/socket.h>\n#include <sys/un.h>\n#include <unistd.h>\n\nusing socket_t = int;\n#ifndef INVALID_SOCKET\n#define INVALID_SOCKET (-1)\n#endif\n#endif //_WIN32\n\n#include <algorithm>\n#include <array>\n#include <atomic>\n#include <cassert>\n#include <cctype>\n#include <climits>\n#include <condition_variable>\n#include <cstring>\n#include <errno.h>\n#include <exception>\n#include <fcntl.h>\n#include <functional>\n#include <iomanip>\n#include <iostream>\n#include <list>\n#include <map>\n#include <memory>\n#include <mutex>\n#include <random>\n#include <regex>\n#include <set>\n#include <sstream>\n#include <string>\n#include <sys/stat.h>\n#include <thread>\n#include <unordered_map>\n#include <unordered_set>\n#include <utility>\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n#ifdef _WIN32\n#include <wincrypt.h>\n\n// these are defined in wincrypt.h and it breaks compilation if BoringSSL is\n// used\n#undef X509_NAME\n#undef X509_CERT_PAIR\n#undef X509_EXTENSIONS\n#undef PKCS7_SIGNER_INFO\n\n#ifdef _MSC_VER\n#pragma comment(lib, \"crypt32.lib\")\n#endif\n#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)\n#include <TargetConditionals.h>\n#if TARGET_OS_OSX\n#include <CoreFoundation/CoreFoundation.h>\n#include <Security/Security.h>\n#endif // TARGET_OS_OSX\n#endif // _WIN32\n\n#include <openssl/err.h>\n#include <openssl/evp.h>\n#include <openssl/ssl.h>\n#include <openssl/x509v3.h>\n\n#if defined(_WIN32) && defined(OPENSSL_USE_APPLINK)\n#include <openssl/applink.c>\n#endif\n\n#include <iostream>\n#include <sstream>\n\n#if defined(OPENSSL_IS_BORINGSSL) || defined(LIBRESSL_VERSION_NUMBER)\n#if OPENSSL_VERSION_NUMBER < 0x1010107f\n#error Please use OpenSSL or a current version of BoringSSL\n#endif\n#define SSL_get1_peer_certificate SSL_get_peer_certificate\n#elif OPENSSL_VERSION_NUMBER < 0x30000000L\n#error Sorry, OpenSSL versions prior to 3.0.0 are not supported\n#endif\n\n#endif\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n#include <zlib.h>\n#endif\n\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n#include <brotli/decode.h>\n#include <brotli/encode.h>\n#endif\n\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n#include <zstd.h>\n#endif\n\n/*\n * Declaration\n */\nnamespace httplib {\n\nnamespace detail {\n\n/*\n * Backport std::make_unique from C++14.\n *\n * NOTE: This code came up with the following stackoverflow post:\n * https://stackoverflow.com/questions/10149840/c-arrays-and-make-unique\n *\n */\n\ntemplate <class T, class... Args>\ntypename std::enable_if<!std::is_array<T>::value, std::unique_ptr<T>>::type\nmake_unique(Args &&...args) {\n  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));\n}\n\ntemplate <class T>\ntypename std::enable_if<std::is_array<T>::value, std::unique_ptr<T>>::type\nmake_unique(std::size_t n) {\n  typedef typename std::remove_extent<T>::type RT;\n  return std::unique_ptr<T>(new RT[n]);\n}\n\nnamespace case_ignore {\n\ninline unsigned char to_lower(int c) {\n  const static unsigned char table[256] = {\n      0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,  13,  14,\n      15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,\n      30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,\n      45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,\n      60,  61,  62,  63,  64,  97,  98,  99,  100, 101, 102, 103, 104, 105, 106,\n      107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,\n      122, 91,  92,  93,  94,  95,  96,  97,  98,  99,  100, 101, 102, 103, 104,\n      105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,\n      120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134,\n      135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,\n      150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164,\n      165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,\n      180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 224, 225, 226,\n      227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241,\n      242, 243, 244, 245, 246, 215, 248, 249, 250, 251, 252, 253, 254, 223, 224,\n      225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,\n      240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254,\n      255,\n  };\n  return table[(unsigned char)(char)c];\n}\n\ninline bool equal(const std::string &a, const std::string &b) {\n  return a.size() == b.size() &&\n         std::equal(a.begin(), a.end(), b.begin(), [](char ca, char cb) {\n           return to_lower(ca) == to_lower(cb);\n         });\n}\n\nstruct equal_to {\n  bool operator()(const std::string &a, const std::string &b) const {\n    return equal(a, b);\n  }\n};\n\nstruct hash {\n  size_t operator()(const std::string &key) const {\n    return hash_core(key.data(), key.size(), 0);\n  }\n\n  size_t hash_core(const char *s, size_t l, size_t h) const {\n    return (l == 0) ? h\n                    : hash_core(s + 1, l - 1,\n                                // Unsets the 6 high bits of h, therefore no\n                                // overflow happens\n                                (((std::numeric_limits<size_t>::max)() >> 6) &\n                                 h * 33) ^\n                                    static_cast<unsigned char>(to_lower(*s)));\n  }\n};\n\n} // namespace case_ignore\n\n// This is based on\n// \"http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4189\".\n\nstruct scope_exit {\n  explicit scope_exit(std::function<void(void)> &&f)\n      : exit_function(std::move(f)), execute_on_destruction{true} {}\n\n  scope_exit(scope_exit &&rhs) noexcept\n      : exit_function(std::move(rhs.exit_function)),\n        execute_on_destruction{rhs.execute_on_destruction} {\n    rhs.release();\n  }\n\n  ~scope_exit() {\n    if (execute_on_destruction) { this->exit_function(); }\n  }\n\n  void release() { this->execute_on_destruction = false; }\n\nprivate:\n  scope_exit(const scope_exit &) = delete;\n  void operator=(const scope_exit &) = delete;\n  scope_exit &operator=(scope_exit &&) = delete;\n\n  std::function<void(void)> exit_function;\n  bool execute_on_destruction;\n};\n\n} // namespace detail\n\nenum SSLVerifierResponse {\n  // no decision has been made, use the built-in certificate verifier\n  NoDecisionMade,\n  // connection certificate is verified and accepted\n  CertificateAccepted,\n  // connection certificate was processed but is rejected\n  CertificateRejected\n};\n\nenum StatusCode {\n  // Information responses\n  Continue_100 = 100,\n  SwitchingProtocol_101 = 101,\n  Processing_102 = 102,\n  EarlyHints_103 = 103,\n\n  // Successful responses\n  OK_200 = 200,\n  Created_201 = 201,\n  Accepted_202 = 202,\n  NonAuthoritativeInformation_203 = 203,\n  NoContent_204 = 204,\n  ResetContent_205 = 205,\n  PartialContent_206 = 206,\n  MultiStatus_207 = 207,\n  AlreadyReported_208 = 208,\n  IMUsed_226 = 226,\n\n  // Redirection messages\n  MultipleChoices_300 = 300,\n  MovedPermanently_301 = 301,\n  Found_302 = 302,\n  SeeOther_303 = 303,\n  NotModified_304 = 304,\n  UseProxy_305 = 305,\n  unused_306 = 306,\n  TemporaryRedirect_307 = 307,\n  PermanentRedirect_308 = 308,\n\n  // Client error responses\n  BadRequest_400 = 400,\n  Unauthorized_401 = 401,\n  PaymentRequired_402 = 402,\n  Forbidden_403 = 403,\n  NotFound_404 = 404,\n  MethodNotAllowed_405 = 405,\n  NotAcceptable_406 = 406,\n  ProxyAuthenticationRequired_407 = 407,\n  RequestTimeout_408 = 408,\n  Conflict_409 = 409,\n  Gone_410 = 410,\n  LengthRequired_411 = 411,\n  PreconditionFailed_412 = 412,\n  PayloadTooLarge_413 = 413,\n  UriTooLong_414 = 414,\n  UnsupportedMediaType_415 = 415,\n  RangeNotSatisfiable_416 = 416,\n  ExpectationFailed_417 = 417,\n  ImATeapot_418 = 418,\n  MisdirectedRequest_421 = 421,\n  UnprocessableContent_422 = 422,\n  Locked_423 = 423,\n  FailedDependency_424 = 424,\n  TooEarly_425 = 425,\n  UpgradeRequired_426 = 426,\n  PreconditionRequired_428 = 428,\n  TooManyRequests_429 = 429,\n  RequestHeaderFieldsTooLarge_431 = 431,\n  UnavailableForLegalReasons_451 = 451,\n\n  // Server error responses\n  InternalServerError_500 = 500,\n  NotImplemented_501 = 501,\n  BadGateway_502 = 502,\n  ServiceUnavailable_503 = 503,\n  GatewayTimeout_504 = 504,\n  HttpVersionNotSupported_505 = 505,\n  VariantAlsoNegotiates_506 = 506,\n  InsufficientStorage_507 = 507,\n  LoopDetected_508 = 508,\n  NotExtended_510 = 510,\n  NetworkAuthenticationRequired_511 = 511,\n};\n\nusing Headers =\n    std::unordered_multimap<std::string, std::string, detail::case_ignore::hash,\n                            detail::case_ignore::equal_to>;\n\nusing Params = std::multimap<std::string, std::string>;\nusing Match = std::smatch;\n\nusing Progress = std::function<bool(uint64_t current, uint64_t total)>;\n\nstruct Response;\nusing ResponseHandler = std::function<bool(const Response &response)>;\n\nstruct MultipartFormData {\n  std::string name;\n  std::string content;\n  std::string filename;\n  std::string content_type;\n};\nusing MultipartFormDataItems = std::vector<MultipartFormData>;\nusing MultipartFormDataMap = std::multimap<std::string, MultipartFormData>;\n\nclass DataSink {\npublic:\n  DataSink() : os(&sb_), sb_(*this) {}\n\n  DataSink(const DataSink &) = delete;\n  DataSink &operator=(const DataSink &) = delete;\n  DataSink(DataSink &&) = delete;\n  DataSink &operator=(DataSink &&) = delete;\n\n  std::function<bool(const char *data, size_t data_len)> write;\n  std::function<bool()> is_writable;\n  std::function<void()> done;\n  std::function<void(const Headers &trailer)> done_with_trailer;\n  std::ostream os;\n\nprivate:\n  class data_sink_streambuf final : public std::streambuf {\n  public:\n    explicit data_sink_streambuf(DataSink &sink) : sink_(sink) {}\n\n  protected:\n    std::streamsize xsputn(const char *s, std::streamsize n) override {\n      sink_.write(s, static_cast<size_t>(n));\n      return n;\n    }\n\n  private:\n    DataSink &sink_;\n  };\n\n  data_sink_streambuf sb_;\n};\n\nusing ContentProvider =\n    std::function<bool(size_t offset, size_t length, DataSink &sink)>;\n\nusing ContentProviderWithoutLength =\n    std::function<bool(size_t offset, DataSink &sink)>;\n\nusing ContentProviderResourceReleaser = std::function<void(bool success)>;\n\nstruct MultipartFormDataProvider {\n  std::string name;\n  ContentProviderWithoutLength provider;\n  std::string filename;\n  std::string content_type;\n};\nusing MultipartFormDataProviderItems = std::vector<MultipartFormDataProvider>;\n\nusing ContentReceiverWithProgress =\n    std::function<bool(const char *data, size_t data_length, uint64_t offset,\n                       uint64_t total_length)>;\n\nusing ContentReceiver =\n    std::function<bool(const char *data, size_t data_length)>;\n\nusing MultipartContentHeader =\n    std::function<bool(const MultipartFormData &file)>;\n\nclass ContentReader {\npublic:\n  using Reader = std::function<bool(ContentReceiver receiver)>;\n  using MultipartReader = std::function<bool(MultipartContentHeader header,\n                                             ContentReceiver receiver)>;\n\n  ContentReader(Reader reader, MultipartReader multipart_reader)\n      : reader_(std::move(reader)),\n        multipart_reader_(std::move(multipart_reader)) {}\n\n  bool operator()(MultipartContentHeader header,\n                  ContentReceiver receiver) const {\n    return multipart_reader_(std::move(header), std::move(receiver));\n  }\n\n  bool operator()(ContentReceiver receiver) const {\n    return reader_(std::move(receiver));\n  }\n\n  Reader reader_;\n  MultipartReader multipart_reader_;\n};\n\nusing Range = std::pair<ssize_t, ssize_t>;\nusing Ranges = std::vector<Range>;\n\nstruct Request {\n  std::string method;\n  std::string path;\n  Params params;\n  Headers headers;\n  std::string body;\n\n  std::string remote_addr;\n  int remote_port = -1;\n  std::string local_addr;\n  int local_port = -1;\n\n  // for server\n  std::string version;\n  std::string target;\n  MultipartFormDataMap files;\n  Ranges ranges;\n  Match matches;\n  std::unordered_map<std::string, std::string> path_params;\n  std::function<bool()> is_connection_closed = []() { return true; };\n\n  // for client\n  ResponseHandler response_handler;\n  ContentReceiverWithProgress content_receiver;\n  Progress progress;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  const SSL *ssl = nullptr;\n#endif\n\n  bool has_header(const std::string &key) const;\n  std::string get_header_value(const std::string &key, const char *def = \"\",\n                               size_t id = 0) const;\n  uint64_t get_header_value_u64(const std::string &key, uint64_t def = 0,\n                                size_t id = 0) const;\n  size_t get_header_value_count(const std::string &key) const;\n  void set_header(const std::string &key, const std::string &val);\n\n  bool has_param(const std::string &key) const;\n  std::string get_param_value(const std::string &key, size_t id = 0) const;\n  size_t get_param_value_count(const std::string &key) const;\n\n  bool is_multipart_form_data() const;\n\n  bool has_file(const std::string &key) const;\n  MultipartFormData get_file_value(const std::string &key) const;\n  std::vector<MultipartFormData> get_file_values(const std::string &key) const;\n\n  // private members...\n  size_t redirect_count_ = CPPHTTPLIB_REDIRECT_MAX_COUNT;\n  size_t content_length_ = 0;\n  ContentProvider content_provider_;\n  bool is_chunked_content_provider_ = false;\n  size_t authorization_count_ = 0;\n  std::chrono::time_point<std::chrono::steady_clock> start_time_ =\n      (std::chrono::steady_clock::time_point::min)();\n};\n\nstruct Response {\n  std::string version;\n  int status = -1;\n  std::string reason;\n  Headers headers;\n  std::string body;\n  std::string location; // Redirect location\n\n  bool has_header(const std::string &key) const;\n  std::string get_header_value(const std::string &key, const char *def = \"\",\n                               size_t id = 0) const;\n  uint64_t get_header_value_u64(const std::string &key, uint64_t def = 0,\n                                size_t id = 0) const;\n  size_t get_header_value_count(const std::string &key) const;\n  void set_header(const std::string &key, const std::string &val);\n\n  void set_redirect(const std::string &url, int status = StatusCode::Found_302);\n  void set_content(const char *s, size_t n, const std::string &content_type);\n  void set_content(const std::string &s, const std::string &content_type);\n  void set_content(std::string &&s, const std::string &content_type);\n\n  void set_content_provider(\n      size_t length, const std::string &content_type, ContentProvider provider,\n      ContentProviderResourceReleaser resource_releaser = nullptr);\n\n  void set_content_provider(\n      const std::string &content_type, ContentProviderWithoutLength provider,\n      ContentProviderResourceReleaser resource_releaser = nullptr);\n\n  void set_chunked_content_provider(\n      const std::string &content_type, ContentProviderWithoutLength provider,\n      ContentProviderResourceReleaser resource_releaser = nullptr);\n\n  void set_file_content(const std::string &path,\n                        const std::string &content_type);\n  void set_file_content(const std::string &path);\n\n  Response() = default;\n  Response(const Response &) = default;\n  Response &operator=(const Response &) = default;\n  Response(Response &&) = default;\n  Response &operator=(Response &&) = default;\n  ~Response() {\n    if (content_provider_resource_releaser_) {\n      content_provider_resource_releaser_(content_provider_success_);\n    }\n  }\n\n  // private members...\n  size_t content_length_ = 0;\n  ContentProvider content_provider_;\n  ContentProviderResourceReleaser content_provider_resource_releaser_;\n  bool is_chunked_content_provider_ = false;\n  bool content_provider_success_ = false;\n  std::string file_content_path_;\n  std::string file_content_content_type_;\n};\n\nclass Stream {\npublic:\n  virtual ~Stream() = default;\n\n  virtual bool is_readable() const = 0;\n  virtual bool wait_readable() const = 0;\n  virtual bool wait_writable() const = 0;\n\n  virtual ssize_t read(char *ptr, size_t size) = 0;\n  virtual ssize_t write(const char *ptr, size_t size) = 0;\n  virtual void get_remote_ip_and_port(std::string &ip, int &port) const = 0;\n  virtual void get_local_ip_and_port(std::string &ip, int &port) const = 0;\n  virtual socket_t socket() const = 0;\n\n  virtual time_t duration() const = 0;\n\n  ssize_t write(const char *ptr);\n  ssize_t write(const std::string &s);\n};\n\nclass TaskQueue {\npublic:\n  TaskQueue() = default;\n  virtual ~TaskQueue() = default;\n\n  virtual bool enqueue(std::function<void()> fn) = 0;\n  virtual void shutdown() = 0;\n\n  virtual void on_idle() {}\n};\n\nclass ThreadPool final : public TaskQueue {\npublic:\n  explicit ThreadPool(size_t n, size_t mqr = 0)\n      : shutdown_(false), max_queued_requests_(mqr) {\n    while (n) {\n      threads_.emplace_back(worker(*this));\n      n--;\n    }\n  }\n\n  ThreadPool(const ThreadPool &) = delete;\n  ~ThreadPool() override = default;\n\n  bool enqueue(std::function<void()> fn) override {\n    {\n      std::unique_lock<std::mutex> lock(mutex_);\n      if (max_queued_requests_ > 0 && jobs_.size() >= max_queued_requests_) {\n        return false;\n      }\n      jobs_.push_back(std::move(fn));\n    }\n\n    cond_.notify_one();\n    return true;\n  }\n\n  void shutdown() override {\n    // Stop all worker threads...\n    {\n      std::unique_lock<std::mutex> lock(mutex_);\n      shutdown_ = true;\n    }\n\n    cond_.notify_all();\n\n    // Join...\n    for (auto &t : threads_) {\n      t.join();\n    }\n  }\n\nprivate:\n  struct worker {\n    explicit worker(ThreadPool &pool) : pool_(pool) {}\n\n    void operator()() {\n      for (;;) {\n        std::function<void()> fn;\n        {\n          std::unique_lock<std::mutex> lock(pool_.mutex_);\n\n          pool_.cond_.wait(\n              lock, [&] { return !pool_.jobs_.empty() || pool_.shutdown_; });\n\n          if (pool_.shutdown_ && pool_.jobs_.empty()) { break; }\n\n          fn = pool_.jobs_.front();\n          pool_.jobs_.pop_front();\n        }\n\n        assert(true == static_cast<bool>(fn));\n        fn();\n      }\n\n#if defined(CPPHTTPLIB_OPENSSL_SUPPORT) && !defined(OPENSSL_IS_BORINGSSL) &&   \\\n    !defined(LIBRESSL_VERSION_NUMBER)\n      OPENSSL_thread_stop();\n#endif\n    }\n\n    ThreadPool &pool_;\n  };\n  friend struct worker;\n\n  std::vector<std::thread> threads_;\n  std::list<std::function<void()>> jobs_;\n\n  bool shutdown_;\n  size_t max_queued_requests_ = 0;\n\n  std::condition_variable cond_;\n  std::mutex mutex_;\n};\n\nusing Logger = std::function<void(const Request &, const Response &)>;\n\nusing SocketOptions = std::function<void(socket_t sock)>;\n\nnamespace detail {\n\nbool set_socket_opt_impl(socket_t sock, int level, int optname,\n                         const void *optval, socklen_t optlen);\nbool set_socket_opt(socket_t sock, int level, int optname, int opt);\nbool set_socket_opt_time(socket_t sock, int level, int optname, time_t sec,\n                         time_t usec);\n\n} // namespace detail\n\nvoid default_socket_options(socket_t sock);\n\nconst char *status_message(int status);\n\nstd::string get_bearer_token_auth(const Request &req);\n\nnamespace detail {\n\nclass MatcherBase {\npublic:\n  virtual ~MatcherBase() = default;\n\n  // Match request path and populate its matches and\n  virtual bool match(Request &request) const = 0;\n};\n\n/**\n * Captures parameters in request path and stores them in Request::path_params\n *\n * Capture name is a substring of a pattern from : to /.\n * The rest of the pattern is matched against the request path directly\n * Parameters are captured starting from the next character after\n * the end of the last matched static pattern fragment until the next /.\n *\n * Example pattern:\n * \"/path/fragments/:capture/more/fragments/:second_capture\"\n * Static fragments:\n * \"/path/fragments/\", \"more/fragments/\"\n *\n * Given the following request path:\n * \"/path/fragments/:1/more/fragments/:2\"\n * the resulting capture will be\n * {{\"capture\", \"1\"}, {\"second_capture\", \"2\"}}\n */\nclass PathParamsMatcher final : public MatcherBase {\npublic:\n  PathParamsMatcher(const std::string &pattern);\n\n  bool match(Request &request) const override;\n\nprivate:\n  // Treat segment separators as the end of path parameter capture\n  // Does not need to handle query parameters as they are parsed before path\n  // matching\n  static constexpr char separator = '/';\n\n  // Contains static path fragments to match against, excluding the '/' after\n  // path params\n  // Fragments are separated by path params\n  std::vector<std::string> static_fragments_;\n  // Stores the names of the path parameters to be used as keys in the\n  // Request::path_params map\n  std::vector<std::string> param_names_;\n};\n\n/**\n * Performs std::regex_match on request path\n * and stores the result in Request::matches\n *\n * Note that regex match is performed directly on the whole request.\n * This means that wildcard patterns may match multiple path segments with /:\n * \"/begin/(.*)/end\" will match both \"/begin/middle/end\" and \"/begin/1/2/end\".\n */\nclass RegexMatcher final : public MatcherBase {\npublic:\n  RegexMatcher(const std::string &pattern) : regex_(pattern) {}\n\n  bool match(Request &request) const override;\n\nprivate:\n  std::regex regex_;\n};\n\nssize_t write_headers(Stream &strm, const Headers &headers);\n\n} // namespace detail\n\nclass Server {\npublic:\n  using Handler = std::function<void(const Request &, Response &)>;\n\n  using ExceptionHandler =\n      std::function<void(const Request &, Response &, std::exception_ptr ep)>;\n\n  enum class HandlerResponse {\n    Handled,\n    Unhandled,\n  };\n  using HandlerWithResponse =\n      std::function<HandlerResponse(const Request &, Response &)>;\n\n  using HandlerWithContentReader = std::function<void(\n      const Request &, Response &, const ContentReader &content_reader)>;\n\n  using Expect100ContinueHandler =\n      std::function<int(const Request &, Response &)>;\n\n  Server();\n\n  virtual ~Server();\n\n  virtual bool is_valid() const;\n\n  Server &Get(const std::string &pattern, Handler handler);\n  Server &Post(const std::string &pattern, Handler handler);\n  Server &Post(const std::string &pattern, HandlerWithContentReader handler);\n  Server &Put(const std::string &pattern, Handler handler);\n  Server &Put(const std::string &pattern, HandlerWithContentReader handler);\n  Server &Patch(const std::string &pattern, Handler handler);\n  Server &Patch(const std::string &pattern, HandlerWithContentReader handler);\n  Server &Delete(const std::string &pattern, Handler handler);\n  Server &Delete(const std::string &pattern, HandlerWithContentReader handler);\n  Server &Options(const std::string &pattern, Handler handler);\n\n  bool set_base_dir(const std::string &dir,\n                    const std::string &mount_point = std::string());\n  bool set_mount_point(const std::string &mount_point, const std::string &dir,\n                       Headers headers = Headers());\n  bool remove_mount_point(const std::string &mount_point);\n  Server &set_file_extension_and_mimetype_mapping(const std::string &ext,\n                                                  const std::string &mime);\n  Server &set_default_file_mimetype(const std::string &mime);\n  Server &set_file_request_handler(Handler handler);\n\n  template <class ErrorHandlerFunc>\n  Server &set_error_handler(ErrorHandlerFunc &&handler) {\n    return set_error_handler_core(\n        std::forward<ErrorHandlerFunc>(handler),\n        std::is_convertible<ErrorHandlerFunc, HandlerWithResponse>{});\n  }\n\n  Server &set_exception_handler(ExceptionHandler handler);\n  Server &set_pre_routing_handler(HandlerWithResponse handler);\n  Server &set_post_routing_handler(Handler handler);\n\n  Server &set_expect_100_continue_handler(Expect100ContinueHandler handler);\n  Server &set_logger(Logger logger);\n\n  Server &set_address_family(int family);\n  Server &set_tcp_nodelay(bool on);\n  Server &set_ipv6_v6only(bool on);\n  Server &set_socket_options(SocketOptions socket_options);\n\n  Server &set_default_headers(Headers headers);\n  Server &\n  set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);\n\n  Server &set_keep_alive_max_count(size_t count);\n  Server &set_keep_alive_timeout(time_t sec);\n\n  Server &set_read_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  Server &set_read_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  Server &set_write_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  Server &set_write_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  Server &set_idle_interval(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  Server &set_idle_interval(const std::chrono::duration<Rep, Period> &duration);\n\n  Server &set_payload_max_length(size_t length);\n\n  bool bind_to_port(const std::string &host, int port, int socket_flags = 0);\n  int bind_to_any_port(const std::string &host, int socket_flags = 0);\n  bool listen_after_bind();\n\n  bool listen(const std::string &host, int port, int socket_flags = 0);\n\n  bool is_running() const;\n  void wait_until_ready() const;\n  void stop();\n  void decommission();\n\n  std::function<TaskQueue *(void)> new_task_queue;\n\nprotected:\n  bool process_request(Stream &strm, const std::string &remote_addr,\n                       int remote_port, const std::string &local_addr,\n                       int local_port, bool close_connection,\n                       bool &connection_closed,\n                       const std::function<void(Request &)> &setup_request);\n\n  std::atomic<socket_t> svr_sock_{INVALID_SOCKET};\n  size_t keep_alive_max_count_ = CPPHTTPLIB_KEEPALIVE_MAX_COUNT;\n  time_t keep_alive_timeout_sec_ = CPPHTTPLIB_KEEPALIVE_TIMEOUT_SECOND;\n  time_t read_timeout_sec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_SECOND;\n  time_t read_timeout_usec_ = CPPHTTPLIB_SERVER_READ_TIMEOUT_USECOND;\n  time_t write_timeout_sec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_SECOND;\n  time_t write_timeout_usec_ = CPPHTTPLIB_SERVER_WRITE_TIMEOUT_USECOND;\n  time_t idle_interval_sec_ = CPPHTTPLIB_IDLE_INTERVAL_SECOND;\n  time_t idle_interval_usec_ = CPPHTTPLIB_IDLE_INTERVAL_USECOND;\n  size_t payload_max_length_ = CPPHTTPLIB_PAYLOAD_MAX_LENGTH;\n\nprivate:\n  using Handlers =\n      std::vector<std::pair<std::unique_ptr<detail::MatcherBase>, Handler>>;\n  using HandlersForContentReader =\n      std::vector<std::pair<std::unique_ptr<detail::MatcherBase>,\n                            HandlerWithContentReader>>;\n\n  static std::unique_ptr<detail::MatcherBase>\n  make_matcher(const std::string &pattern);\n\n  Server &set_error_handler_core(HandlerWithResponse handler, std::true_type);\n  Server &set_error_handler_core(Handler handler, std::false_type);\n\n  socket_t create_server_socket(const std::string &host, int port,\n                                int socket_flags,\n                                SocketOptions socket_options) const;\n  int bind_internal(const std::string &host, int port, int socket_flags);\n  bool listen_internal();\n\n  bool routing(Request &req, Response &res, Stream &strm);\n  bool handle_file_request(const Request &req, Response &res,\n                           bool head = false);\n  bool dispatch_request(Request &req, Response &res,\n                        const Handlers &handlers) const;\n  bool dispatch_request_for_content_reader(\n      Request &req, Response &res, ContentReader content_reader,\n      const HandlersForContentReader &handlers) const;\n\n  bool parse_request_line(const char *s, Request &req) const;\n  void apply_ranges(const Request &req, Response &res,\n                    std::string &content_type, std::string &boundary) const;\n  bool write_response(Stream &strm, bool close_connection, Request &req,\n                      Response &res);\n  bool write_response_with_content(Stream &strm, bool close_connection,\n                                   const Request &req, Response &res);\n  bool write_response_core(Stream &strm, bool close_connection,\n                           const Request &req, Response &res,\n                           bool need_apply_ranges);\n  bool write_content_with_provider(Stream &strm, const Request &req,\n                                   Response &res, const std::string &boundary,\n                                   const std::string &content_type);\n  bool read_content(Stream &strm, Request &req, Response &res);\n  bool\n  read_content_with_content_receiver(Stream &strm, Request &req, Response &res,\n                                     ContentReceiver receiver,\n                                     MultipartContentHeader multipart_header,\n                                     ContentReceiver multipart_receiver);\n  bool read_content_core(Stream &strm, Request &req, Response &res,\n                         ContentReceiver receiver,\n                         MultipartContentHeader multipart_header,\n                         ContentReceiver multipart_receiver) const;\n\n  virtual bool process_and_close_socket(socket_t sock);\n\n  std::atomic<bool> is_running_{false};\n  std::atomic<bool> is_decommissioned{false};\n\n  struct MountPointEntry {\n    std::string mount_point;\n    std::string base_dir;\n    Headers headers;\n  };\n  std::vector<MountPointEntry> base_dirs_;\n  std::map<std::string, std::string> file_extension_and_mimetype_map_;\n  std::string default_file_mimetype_ = \"application/octet-stream\";\n  Handler file_request_handler_;\n\n  Handlers get_handlers_;\n  Handlers post_handlers_;\n  HandlersForContentReader post_handlers_for_content_reader_;\n  Handlers put_handlers_;\n  HandlersForContentReader put_handlers_for_content_reader_;\n  Handlers patch_handlers_;\n  HandlersForContentReader patch_handlers_for_content_reader_;\n  Handlers delete_handlers_;\n  HandlersForContentReader delete_handlers_for_content_reader_;\n  Handlers options_handlers_;\n\n  HandlerWithResponse error_handler_;\n  ExceptionHandler exception_handler_;\n  HandlerWithResponse pre_routing_handler_;\n  Handler post_routing_handler_;\n  Expect100ContinueHandler expect_100_continue_handler_;\n\n  Logger logger_;\n\n  int address_family_ = AF_UNSPEC;\n  bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;\n  bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY;\n  SocketOptions socket_options_ = default_socket_options;\n\n  Headers default_headers_;\n  std::function<ssize_t(Stream &, Headers &)> header_writer_ =\n      detail::write_headers;\n};\n\nenum class Error {\n  Success = 0,\n  Unknown,\n  Connection,\n  BindIPAddress,\n  Read,\n  Write,\n  ExceedRedirectCount,\n  Canceled,\n  SSLConnection,\n  SSLLoadingCerts,\n  SSLServerVerification,\n  SSLServerHostnameVerification,\n  UnsupportedMultipartBoundaryChars,\n  Compression,\n  ConnectionTimeout,\n  ProxyConnection,\n\n  // For internal use only\n  SSLPeerCouldBeClosed_,\n};\n\nstd::string to_string(Error error);\n\nstd::ostream &operator<<(std::ostream &os, const Error &obj);\n\nclass Result {\npublic:\n  Result() = default;\n  Result(std::unique_ptr<Response> &&res, Error err,\n         Headers &&request_headers = Headers{})\n      : res_(std::move(res)), err_(err),\n        request_headers_(std::move(request_headers)) {}\n  // Response\n  operator bool() const { return res_ != nullptr; }\n  bool operator==(std::nullptr_t) const { return res_ == nullptr; }\n  bool operator!=(std::nullptr_t) const { return res_ != nullptr; }\n  const Response &value() const { return *res_; }\n  Response &value() { return *res_; }\n  const Response &operator*() const { return *res_; }\n  Response &operator*() { return *res_; }\n  const Response *operator->() const { return res_.get(); }\n  Response *operator->() { return res_.get(); }\n\n  // Error\n  Error error() const { return err_; }\n\n  // Request Headers\n  bool has_request_header(const std::string &key) const;\n  std::string get_request_header_value(const std::string &key,\n                                       const char *def = \"\",\n                                       size_t id = 0) const;\n  uint64_t get_request_header_value_u64(const std::string &key,\n                                        uint64_t def = 0, size_t id = 0) const;\n  size_t get_request_header_value_count(const std::string &key) const;\n\nprivate:\n  std::unique_ptr<Response> res_;\n  Error err_ = Error::Unknown;\n  Headers request_headers_;\n};\n\nclass ClientImpl {\npublic:\n  explicit ClientImpl(const std::string &host);\n\n  explicit ClientImpl(const std::string &host, int port);\n\n  explicit ClientImpl(const std::string &host, int port,\n                      const std::string &client_cert_path,\n                      const std::string &client_key_path);\n\n  virtual ~ClientImpl();\n\n  virtual bool is_valid() const;\n\n  Result Get(const std::string &path);\n  Result Get(const std::string &path, const Headers &headers);\n  Result Get(const std::string &path, Progress progress);\n  Result Get(const std::string &path, const Headers &headers,\n             Progress progress);\n  Result Get(const std::string &path, ContentReceiver content_receiver);\n  Result Get(const std::string &path, const Headers &headers,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, ContentReceiver content_receiver,\n             Progress progress);\n  Result Get(const std::string &path, const Headers &headers,\n             ContentReceiver content_receiver, Progress progress);\n  Result Get(const std::string &path, ResponseHandler response_handler,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, const Headers &headers,\n             ResponseHandler response_handler,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, ResponseHandler response_handler,\n             ContentReceiver content_receiver, Progress progress);\n  Result Get(const std::string &path, const Headers &headers,\n             ResponseHandler response_handler, ContentReceiver content_receiver,\n             Progress progress);\n\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, Progress progress = nullptr);\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, ContentReceiver content_receiver,\n             Progress progress = nullptr);\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, ResponseHandler response_handler,\n             ContentReceiver content_receiver, Progress progress = nullptr);\n\n  Result Head(const std::string &path);\n  Result Head(const std::string &path, const Headers &headers);\n\n  Result Post(const std::string &path);\n  Result Post(const std::string &path, const Headers &headers);\n  Result Post(const std::string &path, const char *body, size_t content_length,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers, const char *body,\n              size_t content_length, const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers, const char *body,\n              size_t content_length, const std::string &content_type,\n              Progress progress);\n  Result Post(const std::string &path, const std::string &body,\n              const std::string &content_type);\n  Result Post(const std::string &path, const std::string &body,\n              const std::string &content_type, Progress progress);\n  Result Post(const std::string &path, const Headers &headers,\n              const std::string &body, const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              const std::string &body, const std::string &content_type,\n              Progress progress);\n  Result Post(const std::string &path, size_t content_length,\n              ContentProvider content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path,\n              ContentProviderWithoutLength content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              size_t content_length, ContentProvider content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              ContentProviderWithoutLength content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Params &params);\n  Result Post(const std::string &path, const Headers &headers,\n              const Params &params);\n  Result Post(const std::string &path, const Headers &headers,\n              const Params &params, Progress progress);\n  Result Post(const std::string &path, const MultipartFormDataItems &items);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items, const std::string &boundary);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items,\n              const MultipartFormDataProviderItems &provider_items);\n\n  Result Put(const std::string &path);\n  Result Put(const std::string &path, const char *body, size_t content_length,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers, const char *body,\n             size_t content_length, const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers, const char *body,\n             size_t content_length, const std::string &content_type,\n             Progress progress);\n  Result Put(const std::string &path, const std::string &body,\n             const std::string &content_type);\n  Result Put(const std::string &path, const std::string &body,\n             const std::string &content_type, Progress progress);\n  Result Put(const std::string &path, const Headers &headers,\n             const std::string &body, const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             const std::string &body, const std::string &content_type,\n             Progress progress);\n  Result Put(const std::string &path, size_t content_length,\n             ContentProvider content_provider, const std::string &content_type);\n  Result Put(const std::string &path,\n             ContentProviderWithoutLength content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             size_t content_length, ContentProvider content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             ContentProviderWithoutLength content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Params &params);\n  Result Put(const std::string &path, const Headers &headers,\n             const Params &params);\n  Result Put(const std::string &path, const Headers &headers,\n             const Params &params, Progress progress);\n  Result Put(const std::string &path, const MultipartFormDataItems &items);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items, const std::string &boundary);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items,\n             const MultipartFormDataProviderItems &provider_items);\n\n  Result Patch(const std::string &path);\n  Result Patch(const std::string &path, const char *body, size_t content_length,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const char *body, size_t content_length,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const Headers &headers,\n               const char *body, size_t content_length,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               const char *body, size_t content_length,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const std::string &body,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const std::string &body,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const Headers &headers,\n               const std::string &body, const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               const std::string &body, const std::string &content_type,\n               Progress progress);\n  Result Patch(const std::string &path, size_t content_length,\n               ContentProvider content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path,\n               ContentProviderWithoutLength content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               size_t content_length, ContentProvider content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               ContentProviderWithoutLength content_provider,\n               const std::string &content_type);\n\n  Result Delete(const std::string &path);\n  Result Delete(const std::string &path, const Headers &headers);\n  Result Delete(const std::string &path, const char *body,\n                size_t content_length, const std::string &content_type);\n  Result Delete(const std::string &path, const char *body,\n                size_t content_length, const std::string &content_type,\n                Progress progress);\n  Result Delete(const std::string &path, const Headers &headers,\n                const char *body, size_t content_length,\n                const std::string &content_type);\n  Result Delete(const std::string &path, const Headers &headers,\n                const char *body, size_t content_length,\n                const std::string &content_type, Progress progress);\n  Result Delete(const std::string &path, const std::string &body,\n                const std::string &content_type);\n  Result Delete(const std::string &path, const std::string &body,\n                const std::string &content_type, Progress progress);\n  Result Delete(const std::string &path, const Headers &headers,\n                const std::string &body, const std::string &content_type);\n  Result Delete(const std::string &path, const Headers &headers,\n                const std::string &body, const std::string &content_type,\n                Progress progress);\n\n  Result Options(const std::string &path);\n  Result Options(const std::string &path, const Headers &headers);\n\n  bool send(Request &req, Response &res, Error &error);\n  Result send(const Request &req);\n\n  void stop();\n\n  std::string host() const;\n  int port() const;\n\n  size_t is_socket_open() const;\n  socket_t socket() const;\n\n  void set_hostname_addr_map(std::map<std::string, std::string> addr_map);\n\n  void set_default_headers(Headers headers);\n\n  void\n  set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);\n\n  void set_address_family(int family);\n  void set_tcp_nodelay(bool on);\n  void set_ipv6_v6only(bool on);\n  void set_socket_options(SocketOptions socket_options);\n\n  void set_connection_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void\n  set_connection_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_read_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void set_read_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_write_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void set_write_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_max_timeout(time_t msec);\n  template <class Rep, class Period>\n  void set_max_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_basic_auth(const std::string &username, const std::string &password);\n  void set_bearer_token_auth(const std::string &token);\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_digest_auth(const std::string &username,\n                       const std::string &password);\n#endif\n\n  void set_keep_alive(bool on);\n  void set_follow_location(bool on);\n\n  void set_url_encode(bool on);\n\n  void set_compress(bool on);\n\n  void set_decompress(bool on);\n\n  void set_interface(const std::string &intf);\n\n  void set_proxy(const std::string &host, int port);\n  void set_proxy_basic_auth(const std::string &username,\n                            const std::string &password);\n  void set_proxy_bearer_token_auth(const std::string &token);\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_proxy_digest_auth(const std::string &username,\n                             const std::string &password);\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_ca_cert_path(const std::string &ca_cert_file_path,\n                        const std::string &ca_cert_dir_path = std::string());\n  void set_ca_cert_store(X509_STORE *ca_cert_store);\n  X509_STORE *create_ca_cert_store(const char *ca_cert, std::size_t size) const;\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void enable_server_certificate_verification(bool enabled);\n  void enable_server_hostname_verification(bool enabled);\n  void set_server_certificate_verifier(\n      std::function<SSLVerifierResponse(SSL *ssl)> verifier);\n#endif\n\n  void set_logger(Logger logger);\n\nprotected:\n  struct Socket {\n    socket_t sock = INVALID_SOCKET;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n    SSL *ssl = nullptr;\n#endif\n\n    bool is_open() const { return sock != INVALID_SOCKET; }\n  };\n\n  virtual bool create_and_connect_socket(Socket &socket, Error &error);\n\n  // All of:\n  //   shutdown_ssl\n  //   shutdown_socket\n  //   close_socket\n  // should ONLY be called when socket_mutex_ is locked.\n  // Also, shutdown_ssl and close_socket should also NOT be called concurrently\n  // with a DIFFERENT thread sending requests using that socket.\n  virtual void shutdown_ssl(Socket &socket, bool shutdown_gracefully);\n  void shutdown_socket(Socket &socket) const;\n  void close_socket(Socket &socket);\n\n  bool process_request(Stream &strm, Request &req, Response &res,\n                       bool close_connection, Error &error);\n\n  bool write_content_with_provider(Stream &strm, const Request &req,\n                                   Error &error) const;\n\n  void copy_settings(const ClientImpl &rhs);\n\n  // Socket endpoint information\n  const std::string host_;\n  const int port_;\n  const std::string host_and_port_;\n\n  // Current open socket\n  Socket socket_;\n  mutable std::mutex socket_mutex_;\n  std::recursive_mutex request_mutex_;\n\n  // These are all protected under socket_mutex\n  size_t socket_requests_in_flight_ = 0;\n  std::thread::id socket_requests_are_from_thread_ = std::thread::id();\n  bool socket_should_be_closed_when_request_is_done_ = false;\n\n  // Hostname-IP map\n  std::map<std::string, std::string> addr_map_;\n\n  // Default headers\n  Headers default_headers_;\n\n  // Header writer\n  std::function<ssize_t(Stream &, Headers &)> header_writer_ =\n      detail::write_headers;\n\n  // Settings\n  std::string client_cert_path_;\n  std::string client_key_path_;\n\n  time_t connection_timeout_sec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_SECOND;\n  time_t connection_timeout_usec_ = CPPHTTPLIB_CONNECTION_TIMEOUT_USECOND;\n  time_t read_timeout_sec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_SECOND;\n  time_t read_timeout_usec_ = CPPHTTPLIB_CLIENT_READ_TIMEOUT_USECOND;\n  time_t write_timeout_sec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_SECOND;\n  time_t write_timeout_usec_ = CPPHTTPLIB_CLIENT_WRITE_TIMEOUT_USECOND;\n  time_t max_timeout_msec_ = CPPHTTPLIB_CLIENT_MAX_TIMEOUT_MSECOND;\n\n  std::string basic_auth_username_;\n  std::string basic_auth_password_;\n  std::string bearer_token_auth_token_;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  std::string digest_auth_username_;\n  std::string digest_auth_password_;\n#endif\n\n  bool keep_alive_ = false;\n  bool follow_location_ = false;\n\n  bool url_encode_ = true;\n\n  int address_family_ = AF_UNSPEC;\n  bool tcp_nodelay_ = CPPHTTPLIB_TCP_NODELAY;\n  bool ipv6_v6only_ = CPPHTTPLIB_IPV6_V6ONLY;\n  SocketOptions socket_options_ = nullptr;\n\n  bool compress_ = false;\n  bool decompress_ = true;\n\n  std::string interface_;\n\n  std::string proxy_host_;\n  int proxy_port_ = -1;\n\n  std::string proxy_basic_auth_username_;\n  std::string proxy_basic_auth_password_;\n  std::string proxy_bearer_token_auth_token_;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  std::string proxy_digest_auth_username_;\n  std::string proxy_digest_auth_password_;\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  std::string ca_cert_file_path_;\n  std::string ca_cert_dir_path_;\n\n  X509_STORE *ca_cert_store_ = nullptr;\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  bool server_certificate_verification_ = true;\n  bool server_hostname_verification_ = true;\n  std::function<SSLVerifierResponse(SSL *ssl)> server_certificate_verifier_;\n#endif\n\n  Logger logger_;\n\nprivate:\n  bool send_(Request &req, Response &res, Error &error);\n  Result send_(Request &&req);\n\n  socket_t create_client_socket(Error &error) const;\n  bool read_response_line(Stream &strm, const Request &req,\n                          Response &res) const;\n  bool write_request(Stream &strm, Request &req, bool close_connection,\n                     Error &error);\n  bool redirect(Request &req, Response &res, Error &error);\n  bool handle_request(Stream &strm, Request &req, Response &res,\n                      bool close_connection, Error &error);\n  std::unique_ptr<Response> send_with_content_provider(\n      Request &req, const char *body, size_t content_length,\n      ContentProvider content_provider,\n      ContentProviderWithoutLength content_provider_without_length,\n      const std::string &content_type, Error &error);\n  Result send_with_content_provider(\n      const std::string &method, const std::string &path,\n      const Headers &headers, const char *body, size_t content_length,\n      ContentProvider content_provider,\n      ContentProviderWithoutLength content_provider_without_length,\n      const std::string &content_type, Progress progress);\n  ContentProviderWithoutLength get_multipart_content_provider(\n      const std::string &boundary, const MultipartFormDataItems &items,\n      const MultipartFormDataProviderItems &provider_items) const;\n\n  std::string adjust_host_string(const std::string &host) const;\n\n  virtual bool\n  process_socket(const Socket &socket,\n                 std::chrono::time_point<std::chrono::steady_clock> start_time,\n                 std::function<bool(Stream &strm)> callback);\n  virtual bool is_ssl() const;\n};\n\nclass Client {\npublic:\n  // Universal interface\n  explicit Client(const std::string &scheme_host_port);\n\n  explicit Client(const std::string &scheme_host_port,\n                  const std::string &client_cert_path,\n                  const std::string &client_key_path);\n\n  // HTTP only interface\n  explicit Client(const std::string &host, int port);\n\n  explicit Client(const std::string &host, int port,\n                  const std::string &client_cert_path,\n                  const std::string &client_key_path);\n\n  Client(Client &&) = default;\n  Client &operator=(Client &&) = default;\n\n  ~Client();\n\n  bool is_valid() const;\n\n  Result Get(const std::string &path);\n  Result Get(const std::string &path, const Headers &headers);\n  Result Get(const std::string &path, Progress progress);\n  Result Get(const std::string &path, const Headers &headers,\n             Progress progress);\n  Result Get(const std::string &path, ContentReceiver content_receiver);\n  Result Get(const std::string &path, const Headers &headers,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, ContentReceiver content_receiver,\n             Progress progress);\n  Result Get(const std::string &path, const Headers &headers,\n             ContentReceiver content_receiver, Progress progress);\n  Result Get(const std::string &path, ResponseHandler response_handler,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, const Headers &headers,\n             ResponseHandler response_handler,\n             ContentReceiver content_receiver);\n  Result Get(const std::string &path, const Headers &headers,\n             ResponseHandler response_handler, ContentReceiver content_receiver,\n             Progress progress);\n  Result Get(const std::string &path, ResponseHandler response_handler,\n             ContentReceiver content_receiver, Progress progress);\n\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, Progress progress = nullptr);\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, ContentReceiver content_receiver,\n             Progress progress = nullptr);\n  Result Get(const std::string &path, const Params &params,\n             const Headers &headers, ResponseHandler response_handler,\n             ContentReceiver content_receiver, Progress progress = nullptr);\n\n  Result Head(const std::string &path);\n  Result Head(const std::string &path, const Headers &headers);\n\n  Result Post(const std::string &path);\n  Result Post(const std::string &path, const Headers &headers);\n  Result Post(const std::string &path, const char *body, size_t content_length,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers, const char *body,\n              size_t content_length, const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers, const char *body,\n              size_t content_length, const std::string &content_type,\n              Progress progress);\n  Result Post(const std::string &path, const std::string &body,\n              const std::string &content_type);\n  Result Post(const std::string &path, const std::string &body,\n              const std::string &content_type, Progress progress);\n  Result Post(const std::string &path, const Headers &headers,\n              const std::string &body, const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              const std::string &body, const std::string &content_type,\n              Progress progress);\n  Result Post(const std::string &path, size_t content_length,\n              ContentProvider content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path,\n              ContentProviderWithoutLength content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              size_t content_length, ContentProvider content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Headers &headers,\n              ContentProviderWithoutLength content_provider,\n              const std::string &content_type);\n  Result Post(const std::string &path, const Params &params);\n  Result Post(const std::string &path, const Headers &headers,\n              const Params &params);\n  Result Post(const std::string &path, const Headers &headers,\n              const Params &params, Progress progress);\n  Result Post(const std::string &path, const MultipartFormDataItems &items);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items, const std::string &boundary);\n  Result Post(const std::string &path, const Headers &headers,\n              const MultipartFormDataItems &items,\n              const MultipartFormDataProviderItems &provider_items);\n\n  Result Put(const std::string &path);\n  Result Put(const std::string &path, const char *body, size_t content_length,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers, const char *body,\n             size_t content_length, const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers, const char *body,\n             size_t content_length, const std::string &content_type,\n             Progress progress);\n  Result Put(const std::string &path, const std::string &body,\n             const std::string &content_type);\n  Result Put(const std::string &path, const std::string &body,\n             const std::string &content_type, Progress progress);\n  Result Put(const std::string &path, const Headers &headers,\n             const std::string &body, const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             const std::string &body, const std::string &content_type,\n             Progress progress);\n  Result Put(const std::string &path, size_t content_length,\n             ContentProvider content_provider, const std::string &content_type);\n  Result Put(const std::string &path,\n             ContentProviderWithoutLength content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             size_t content_length, ContentProvider content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Headers &headers,\n             ContentProviderWithoutLength content_provider,\n             const std::string &content_type);\n  Result Put(const std::string &path, const Params &params);\n  Result Put(const std::string &path, const Headers &headers,\n             const Params &params);\n  Result Put(const std::string &path, const Headers &headers,\n             const Params &params, Progress progress);\n  Result Put(const std::string &path, const MultipartFormDataItems &items);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items, const std::string &boundary);\n  Result Put(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items,\n             const MultipartFormDataProviderItems &provider_items);\n\n  Result Patch(const std::string &path);\n  Result Patch(const std::string &path, const char *body, size_t content_length,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const char *body, size_t content_length,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const Headers &headers,\n               const char *body, size_t content_length,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               const char *body, size_t content_length,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const std::string &body,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const std::string &body,\n               const std::string &content_type, Progress progress);\n  Result Patch(const std::string &path, const Headers &headers,\n               const std::string &body, const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               const std::string &body, const std::string &content_type,\n               Progress progress);\n  Result Patch(const std::string &path, size_t content_length,\n               ContentProvider content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path,\n               ContentProviderWithoutLength content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               size_t content_length, ContentProvider content_provider,\n               const std::string &content_type);\n  Result Patch(const std::string &path, const Headers &headers,\n               ContentProviderWithoutLength content_provider,\n               const std::string &content_type);\n\n  Result Delete(const std::string &path);\n  Result Delete(const std::string &path, const Headers &headers);\n  Result Delete(const std::string &path, const char *body,\n                size_t content_length, const std::string &content_type);\n  Result Delete(const std::string &path, const char *body,\n                size_t content_length, const std::string &content_type,\n                Progress progress);\n  Result Delete(const std::string &path, const Headers &headers,\n                const char *body, size_t content_length,\n                const std::string &content_type);\n  Result Delete(const std::string &path, const Headers &headers,\n                const char *body, size_t content_length,\n                const std::string &content_type, Progress progress);\n  Result Delete(const std::string &path, const std::string &body,\n                const std::string &content_type);\n  Result Delete(const std::string &path, const std::string &body,\n                const std::string &content_type, Progress progress);\n  Result Delete(const std::string &path, const Headers &headers,\n                const std::string &body, const std::string &content_type);\n  Result Delete(const std::string &path, const Headers &headers,\n                const std::string &body, const std::string &content_type,\n                Progress progress);\n\n  Result Options(const std::string &path);\n  Result Options(const std::string &path, const Headers &headers);\n\n  bool send(Request &req, Response &res, Error &error);\n  Result send(const Request &req);\n\n  void stop();\n\n  std::string host() const;\n  int port() const;\n\n  size_t is_socket_open() const;\n  socket_t socket() const;\n\n  void set_hostname_addr_map(std::map<std::string, std::string> addr_map);\n\n  void set_default_headers(Headers headers);\n\n  void\n  set_header_writer(std::function<ssize_t(Stream &, Headers &)> const &writer);\n\n  void set_address_family(int family);\n  void set_tcp_nodelay(bool on);\n  void set_socket_options(SocketOptions socket_options);\n\n  void set_connection_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void\n  set_connection_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_read_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void set_read_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_write_timeout(time_t sec, time_t usec = 0);\n  template <class Rep, class Period>\n  void set_write_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_max_timeout(time_t msec);\n  template <class Rep, class Period>\n  void set_max_timeout(const std::chrono::duration<Rep, Period> &duration);\n\n  void set_basic_auth(const std::string &username, const std::string &password);\n  void set_bearer_token_auth(const std::string &token);\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_digest_auth(const std::string &username,\n                       const std::string &password);\n#endif\n\n  void set_keep_alive(bool on);\n  void set_follow_location(bool on);\n\n  void set_url_encode(bool on);\n\n  void set_compress(bool on);\n\n  void set_decompress(bool on);\n\n  void set_interface(const std::string &intf);\n\n  void set_proxy(const std::string &host, int port);\n  void set_proxy_basic_auth(const std::string &username,\n                            const std::string &password);\n  void set_proxy_bearer_token_auth(const std::string &token);\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_proxy_digest_auth(const std::string &username,\n                             const std::string &password);\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void enable_server_certificate_verification(bool enabled);\n  void enable_server_hostname_verification(bool enabled);\n  void set_server_certificate_verifier(\n      std::function<SSLVerifierResponse(SSL *ssl)> verifier);\n#endif\n\n  void set_logger(Logger logger);\n\n  // SSL\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  void set_ca_cert_path(const std::string &ca_cert_file_path,\n                        const std::string &ca_cert_dir_path = std::string());\n\n  void set_ca_cert_store(X509_STORE *ca_cert_store);\n  void load_ca_cert_store(const char *ca_cert, std::size_t size);\n\n  long get_openssl_verify_result() const;\n\n  SSL_CTX *ssl_context() const;\n#endif\n\nprivate:\n  std::unique_ptr<ClientImpl> cli_;\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  bool is_ssl_ = false;\n#endif\n};\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\nclass SSLServer : public Server {\npublic:\n  SSLServer(const char *cert_path, const char *private_key_path,\n            const char *client_ca_cert_file_path = nullptr,\n            const char *client_ca_cert_dir_path = nullptr,\n            const char *private_key_password = nullptr);\n\n  SSLServer(X509 *cert, EVP_PKEY *private_key,\n            X509_STORE *client_ca_cert_store = nullptr);\n\n  SSLServer(\n      const std::function<bool(SSL_CTX &ssl_ctx)> &setup_ssl_ctx_callback);\n\n  ~SSLServer() override;\n\n  bool is_valid() const override;\n\n  SSL_CTX *ssl_context() const;\n\n  void update_certs(X509 *cert, EVP_PKEY *private_key,\n                    X509_STORE *client_ca_cert_store = nullptr);\n\nprivate:\n  bool process_and_close_socket(socket_t sock) override;\n\n  SSL_CTX *ctx_;\n  std::mutex ctx_mutex_;\n};\n\nclass SSLClient final : public ClientImpl {\npublic:\n  explicit SSLClient(const std::string &host);\n\n  explicit SSLClient(const std::string &host, int port);\n\n  explicit SSLClient(const std::string &host, int port,\n                     const std::string &client_cert_path,\n                     const std::string &client_key_path,\n                     const std::string &private_key_password = std::string());\n\n  explicit SSLClient(const std::string &host, int port, X509 *client_cert,\n                     EVP_PKEY *client_key,\n                     const std::string &private_key_password = std::string());\n\n  ~SSLClient() override;\n\n  bool is_valid() const override;\n\n  void set_ca_cert_store(X509_STORE *ca_cert_store);\n  void load_ca_cert_store(const char *ca_cert, std::size_t size);\n\n  long get_openssl_verify_result() const;\n\n  SSL_CTX *ssl_context() const;\n\nprivate:\n  bool create_and_connect_socket(Socket &socket, Error &error) override;\n  void shutdown_ssl(Socket &socket, bool shutdown_gracefully) override;\n  void shutdown_ssl_impl(Socket &socket, bool shutdown_gracefully);\n\n  bool\n  process_socket(const Socket &socket,\n                 std::chrono::time_point<std::chrono::steady_clock> start_time,\n                 std::function<bool(Stream &strm)> callback) override;\n  bool is_ssl() const override;\n\n  bool connect_with_proxy(\n      Socket &sock,\n      std::chrono::time_point<std::chrono::steady_clock> start_time,\n      Response &res, bool &success, Error &error);\n  bool initialize_ssl(Socket &socket, Error &error);\n\n  bool load_certs();\n\n  bool verify_host(X509 *server_cert) const;\n  bool verify_host_with_subject_alt_name(X509 *server_cert) const;\n  bool verify_host_with_common_name(X509 *server_cert) const;\n  bool check_host_name(const char *pattern, size_t pattern_len) const;\n\n  SSL_CTX *ctx_;\n  std::mutex ctx_mutex_;\n  std::once_flag initialize_cert_;\n\n  std::vector<std::string> host_components_;\n\n  long verify_result_ = 0;\n\n  friend class ClientImpl;\n};\n#endif\n\n/*\n * Implementation of template methods.\n */\n\nnamespace detail {\n\ntemplate <typename T, typename U>\ninline void duration_to_sec_and_usec(const T &duration, U callback) {\n  auto sec = std::chrono::duration_cast<std::chrono::seconds>(duration).count();\n  auto usec = std::chrono::duration_cast<std::chrono::microseconds>(\n                  duration - std::chrono::seconds(sec))\n                  .count();\n  callback(static_cast<time_t>(sec), static_cast<time_t>(usec));\n}\n\ntemplate <size_t N> inline constexpr size_t str_len(const char (&)[N]) {\n  return N - 1;\n}\n\ninline bool is_numeric(const std::string &str) {\n  return !str.empty() && std::all_of(str.begin(), str.end(), ::isdigit);\n}\n\ninline uint64_t get_header_value_u64(const Headers &headers,\n                                     const std::string &key, uint64_t def,\n                                     size_t id, bool &is_invalid_value) {\n  is_invalid_value = false;\n  auto rng = headers.equal_range(key);\n  auto it = rng.first;\n  std::advance(it, static_cast<ssize_t>(id));\n  if (it != rng.second) {\n    if (is_numeric(it->second)) {\n      return std::strtoull(it->second.data(), nullptr, 10);\n    } else {\n      is_invalid_value = true;\n    }\n  }\n  return def;\n}\n\ninline uint64_t get_header_value_u64(const Headers &headers,\n                                     const std::string &key, uint64_t def,\n                                     size_t id) {\n  bool dummy = false;\n  return get_header_value_u64(headers, key, def, id, dummy);\n}\n\n} // namespace detail\n\ninline uint64_t Request::get_header_value_u64(const std::string &key,\n                                              uint64_t def, size_t id) const {\n  return detail::get_header_value_u64(headers, key, def, id);\n}\n\ninline uint64_t Response::get_header_value_u64(const std::string &key,\n                                               uint64_t def, size_t id) const {\n  return detail::get_header_value_u64(headers, key, def, id);\n}\n\nnamespace detail {\n\ninline bool set_socket_opt_impl(socket_t sock, int level, int optname,\n                                const void *optval, socklen_t optlen) {\n  return setsockopt(sock, level, optname,\n#ifdef _WIN32\n                    reinterpret_cast<const char *>(optval),\n#else\n                    optval,\n#endif\n                    optlen) == 0;\n}\n\ninline bool set_socket_opt(socket_t sock, int level, int optname, int optval) {\n  return set_socket_opt_impl(sock, level, optname, &optval, sizeof(optval));\n}\n\ninline bool set_socket_opt_time(socket_t sock, int level, int optname,\n                                time_t sec, time_t usec) {\n#ifdef _WIN32\n  auto timeout = static_cast<uint32_t>(sec * 1000 + usec / 1000);\n#else\n  timeval timeout;\n  timeout.tv_sec = static_cast<long>(sec);\n  timeout.tv_usec = static_cast<decltype(timeout.tv_usec)>(usec);\n#endif\n  return set_socket_opt_impl(sock, level, optname, &timeout, sizeof(timeout));\n}\n\n} // namespace detail\n\ninline void default_socket_options(socket_t sock) {\n  detail::set_socket_opt(sock, SOL_SOCKET,\n#ifdef SO_REUSEPORT\n                         SO_REUSEPORT,\n#else\n                         SO_REUSEADDR,\n#endif\n                         1);\n}\n\ninline const char *status_message(int status) {\n  switch (status) {\n  case StatusCode::Continue_100: return \"Continue\";\n  case StatusCode::SwitchingProtocol_101: return \"Switching Protocol\";\n  case StatusCode::Processing_102: return \"Processing\";\n  case StatusCode::EarlyHints_103: return \"Early Hints\";\n  case StatusCode::OK_200: return \"OK\";\n  case StatusCode::Created_201: return \"Created\";\n  case StatusCode::Accepted_202: return \"Accepted\";\n  case StatusCode::NonAuthoritativeInformation_203:\n    return \"Non-Authoritative Information\";\n  case StatusCode::NoContent_204: return \"No Content\";\n  case StatusCode::ResetContent_205: return \"Reset Content\";\n  case StatusCode::PartialContent_206: return \"Partial Content\";\n  case StatusCode::MultiStatus_207: return \"Multi-Status\";\n  case StatusCode::AlreadyReported_208: return \"Already Reported\";\n  case StatusCode::IMUsed_226: return \"IM Used\";\n  case StatusCode::MultipleChoices_300: return \"Multiple Choices\";\n  case StatusCode::MovedPermanently_301: return \"Moved Permanently\";\n  case StatusCode::Found_302: return \"Found\";\n  case StatusCode::SeeOther_303: return \"See Other\";\n  case StatusCode::NotModified_304: return \"Not Modified\";\n  case StatusCode::UseProxy_305: return \"Use Proxy\";\n  case StatusCode::unused_306: return \"unused\";\n  case StatusCode::TemporaryRedirect_307: return \"Temporary Redirect\";\n  case StatusCode::PermanentRedirect_308: return \"Permanent Redirect\";\n  case StatusCode::BadRequest_400: return \"Bad Request\";\n  case StatusCode::Unauthorized_401: return \"Unauthorized\";\n  case StatusCode::PaymentRequired_402: return \"Payment Required\";\n  case StatusCode::Forbidden_403: return \"Forbidden\";\n  case StatusCode::NotFound_404: return \"Not Found\";\n  case StatusCode::MethodNotAllowed_405: return \"Method Not Allowed\";\n  case StatusCode::NotAcceptable_406: return \"Not Acceptable\";\n  case StatusCode::ProxyAuthenticationRequired_407:\n    return \"Proxy Authentication Required\";\n  case StatusCode::RequestTimeout_408: return \"Request Timeout\";\n  case StatusCode::Conflict_409: return \"Conflict\";\n  case StatusCode::Gone_410: return \"Gone\";\n  case StatusCode::LengthRequired_411: return \"Length Required\";\n  case StatusCode::PreconditionFailed_412: return \"Precondition Failed\";\n  case StatusCode::PayloadTooLarge_413: return \"Payload Too Large\";\n  case StatusCode::UriTooLong_414: return \"URI Too Long\";\n  case StatusCode::UnsupportedMediaType_415: return \"Unsupported Media Type\";\n  case StatusCode::RangeNotSatisfiable_416: return \"Range Not Satisfiable\";\n  case StatusCode::ExpectationFailed_417: return \"Expectation Failed\";\n  case StatusCode::ImATeapot_418: return \"I'm a teapot\";\n  case StatusCode::MisdirectedRequest_421: return \"Misdirected Request\";\n  case StatusCode::UnprocessableContent_422: return \"Unprocessable Content\";\n  case StatusCode::Locked_423: return \"Locked\";\n  case StatusCode::FailedDependency_424: return \"Failed Dependency\";\n  case StatusCode::TooEarly_425: return \"Too Early\";\n  case StatusCode::UpgradeRequired_426: return \"Upgrade Required\";\n  case StatusCode::PreconditionRequired_428: return \"Precondition Required\";\n  case StatusCode::TooManyRequests_429: return \"Too Many Requests\";\n  case StatusCode::RequestHeaderFieldsTooLarge_431:\n    return \"Request Header Fields Too Large\";\n  case StatusCode::UnavailableForLegalReasons_451:\n    return \"Unavailable For Legal Reasons\";\n  case StatusCode::NotImplemented_501: return \"Not Implemented\";\n  case StatusCode::BadGateway_502: return \"Bad Gateway\";\n  case StatusCode::ServiceUnavailable_503: return \"Service Unavailable\";\n  case StatusCode::GatewayTimeout_504: return \"Gateway Timeout\";\n  case StatusCode::HttpVersionNotSupported_505:\n    return \"HTTP Version Not Supported\";\n  case StatusCode::VariantAlsoNegotiates_506: return \"Variant Also Negotiates\";\n  case StatusCode::InsufficientStorage_507: return \"Insufficient Storage\";\n  case StatusCode::LoopDetected_508: return \"Loop Detected\";\n  case StatusCode::NotExtended_510: return \"Not Extended\";\n  case StatusCode::NetworkAuthenticationRequired_511:\n    return \"Network Authentication Required\";\n\n  default:\n  case StatusCode::InternalServerError_500: return \"Internal Server Error\";\n  }\n}\n\ninline std::string get_bearer_token_auth(const Request &req) {\n  if (req.has_header(\"Authorization\")) {\n    constexpr auto bearer_header_prefix_len = detail::str_len(\"Bearer \");\n    return req.get_header_value(\"Authorization\")\n        .substr(bearer_header_prefix_len);\n  }\n  return \"\";\n}\n\ntemplate <class Rep, class Period>\ninline Server &\nServer::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(\n      duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); });\n  return *this;\n}\n\ntemplate <class Rep, class Period>\ninline Server &\nServer::set_write_timeout(const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(\n      duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); });\n  return *this;\n}\n\ntemplate <class Rep, class Period>\ninline Server &\nServer::set_idle_interval(const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(\n      duration, [&](time_t sec, time_t usec) { set_idle_interval(sec, usec); });\n  return *this;\n}\n\ninline std::string to_string(const Error error) {\n  switch (error) {\n  case Error::Success: return \"Success (no error)\";\n  case Error::Connection: return \"Could not establish connection\";\n  case Error::BindIPAddress: return \"Failed to bind IP address\";\n  case Error::Read: return \"Failed to read connection\";\n  case Error::Write: return \"Failed to write connection\";\n  case Error::ExceedRedirectCount: return \"Maximum redirect count exceeded\";\n  case Error::Canceled: return \"Connection handling canceled\";\n  case Error::SSLConnection: return \"SSL connection failed\";\n  case Error::SSLLoadingCerts: return \"SSL certificate loading failed\";\n  case Error::SSLServerVerification: return \"SSL server verification failed\";\n  case Error::SSLServerHostnameVerification:\n    return \"SSL server hostname verification failed\";\n  case Error::UnsupportedMultipartBoundaryChars:\n    return \"Unsupported HTTP multipart boundary characters\";\n  case Error::Compression: return \"Compression failed\";\n  case Error::ConnectionTimeout: return \"Connection timed out\";\n  case Error::ProxyConnection: return \"Proxy connection failed\";\n  case Error::Unknown: return \"Unknown\";\n  default: break;\n  }\n\n  return \"Invalid\";\n}\n\ninline std::ostream &operator<<(std::ostream &os, const Error &obj) {\n  os << to_string(obj);\n  os << \" (\" << static_cast<std::underlying_type<Error>::type>(obj) << ')';\n  return os;\n}\n\ninline uint64_t Result::get_request_header_value_u64(const std::string &key,\n                                                     uint64_t def,\n                                                     size_t id) const {\n  return detail::get_header_value_u64(request_headers_, key, def, id);\n}\n\ntemplate <class Rep, class Period>\ninline void ClientImpl::set_connection_timeout(\n    const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(duration, [&](time_t sec, time_t usec) {\n    set_connection_timeout(sec, usec);\n  });\n}\n\ntemplate <class Rep, class Period>\ninline void ClientImpl::set_read_timeout(\n    const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(\n      duration, [&](time_t sec, time_t usec) { set_read_timeout(sec, usec); });\n}\n\ntemplate <class Rep, class Period>\ninline void ClientImpl::set_write_timeout(\n    const std::chrono::duration<Rep, Period> &duration) {\n  detail::duration_to_sec_and_usec(\n      duration, [&](time_t sec, time_t usec) { set_write_timeout(sec, usec); });\n}\n\ntemplate <class Rep, class Period>\ninline void ClientImpl::set_max_timeout(\n    const std::chrono::duration<Rep, Period> &duration) {\n  auto msec =\n      std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();\n  set_max_timeout(msec);\n}\n\ntemplate <class Rep, class Period>\ninline void Client::set_connection_timeout(\n    const std::chrono::duration<Rep, Period> &duration) {\n  cli_->set_connection_timeout(duration);\n}\n\ntemplate <class Rep, class Period>\ninline void\nClient::set_read_timeout(const std::chrono::duration<Rep, Period> &duration) {\n  cli_->set_read_timeout(duration);\n}\n\ntemplate <class Rep, class Period>\ninline void\nClient::set_write_timeout(const std::chrono::duration<Rep, Period> &duration) {\n  cli_->set_write_timeout(duration);\n}\n\ntemplate <class Rep, class Period>\ninline void\nClient::set_max_timeout(const std::chrono::duration<Rep, Period> &duration) {\n  cli_->set_max_timeout(duration);\n}\n\n/*\n * Forward declarations and types that will be part of the .h file if split into\n * .h + .cc.\n */\n\nstd::string hosted_at(const std::string &hostname);\n\nvoid hosted_at(const std::string &hostname, std::vector<std::string> &addrs);\n\nstd::string append_query_params(const std::string &path, const Params &params);\n\nstd::pair<std::string, std::string> make_range_header(const Ranges &ranges);\n\nstd::pair<std::string, std::string>\nmake_basic_authentication_header(const std::string &username,\n                                 const std::string &password,\n                                 bool is_proxy = false);\n\nnamespace detail {\n\n#if defined(_WIN32)\ninline std::wstring u8string_to_wstring(const char *s) {\n  std::wstring ws;\n  auto len = static_cast<int>(strlen(s));\n  auto wlen = ::MultiByteToWideChar(CP_UTF8, 0, s, len, nullptr, 0);\n  if (wlen > 0) {\n    ws.resize(wlen);\n    wlen = ::MultiByteToWideChar(\n        CP_UTF8, 0, s, len,\n        const_cast<LPWSTR>(reinterpret_cast<LPCWSTR>(ws.data())), wlen);\n    if (wlen != static_cast<int>(ws.size())) { ws.clear(); }\n  }\n  return ws;\n}\n#endif\n\nstruct FileStat {\n  FileStat(const std::string &path);\n  bool is_file() const;\n  bool is_dir() const;\n\nprivate:\n#if defined(_WIN32)\n  struct _stat st_;\n#else\n  struct stat st_;\n#endif\n  int ret_ = -1;\n};\n\nstd::string encode_query_param(const std::string &value);\n\nstd::string decode_url(const std::string &s, bool convert_plus_to_space);\n\nstd::string trim_copy(const std::string &s);\n\nvoid divide(\n    const char *data, std::size_t size, char d,\n    std::function<void(const char *, std::size_t, const char *, std::size_t)>\n        fn);\n\nvoid divide(\n    const std::string &str, char d,\n    std::function<void(const char *, std::size_t, const char *, std::size_t)>\n        fn);\n\nvoid split(const char *b, const char *e, char d,\n           std::function<void(const char *, const char *)> fn);\n\nvoid split(const char *b, const char *e, char d, size_t m,\n           std::function<void(const char *, const char *)> fn);\n\nbool process_client_socket(\n    socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,\n    time_t write_timeout_sec, time_t write_timeout_usec,\n    time_t max_timeout_msec,\n    std::chrono::time_point<std::chrono::steady_clock> start_time,\n    std::function<bool(Stream &)> callback);\n\nsocket_t create_client_socket(const std::string &host, const std::string &ip,\n                              int port, int address_family, bool tcp_nodelay,\n                              bool ipv6_v6only, SocketOptions socket_options,\n                              time_t connection_timeout_sec,\n                              time_t connection_timeout_usec,\n                              time_t read_timeout_sec, time_t read_timeout_usec,\n                              time_t write_timeout_sec,\n                              time_t write_timeout_usec,\n                              const std::string &intf, Error &error);\n\nconst char *get_header_value(const Headers &headers, const std::string &key,\n                             const char *def, size_t id);\n\nstd::string params_to_query_str(const Params &params);\n\nvoid parse_query_text(const char *data, std::size_t size, Params &params);\n\nvoid parse_query_text(const std::string &s, Params &params);\n\nbool parse_multipart_boundary(const std::string &content_type,\n                              std::string &boundary);\n\nbool parse_range_header(const std::string &s, Ranges &ranges);\n\nint close_socket(socket_t sock);\n\nssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags);\n\nssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags);\n\nenum class EncodingType { None = 0, Gzip, Brotli, Zstd };\n\nEncodingType encoding_type(const Request &req, const Response &res);\n\nclass BufferStream final : public Stream {\npublic:\n  BufferStream() = default;\n  ~BufferStream() override = default;\n\n  bool is_readable() const override;\n  bool wait_readable() const override;\n  bool wait_writable() const override;\n  ssize_t read(char *ptr, size_t size) override;\n  ssize_t write(const char *ptr, size_t size) override;\n  void get_remote_ip_and_port(std::string &ip, int &port) const override;\n  void get_local_ip_and_port(std::string &ip, int &port) const override;\n  socket_t socket() const override;\n  time_t duration() const override;\n\n  const std::string &get_buffer() const;\n\nprivate:\n  std::string buffer;\n  size_t position = 0;\n};\n\nclass compressor {\npublic:\n  virtual ~compressor() = default;\n\n  typedef std::function<bool(const char *data, size_t data_len)> Callback;\n  virtual bool compress(const char *data, size_t data_length, bool last,\n                        Callback callback) = 0;\n};\n\nclass decompressor {\npublic:\n  virtual ~decompressor() = default;\n\n  virtual bool is_valid() const = 0;\n\n  typedef std::function<bool(const char *data, size_t data_len)> Callback;\n  virtual bool decompress(const char *data, size_t data_length,\n                          Callback callback) = 0;\n};\n\nclass nocompressor final : public compressor {\npublic:\n  ~nocompressor() override = default;\n\n  bool compress(const char *data, size_t data_length, bool /*last*/,\n                Callback callback) override;\n};\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\nclass gzip_compressor final : public compressor {\npublic:\n  gzip_compressor();\n  ~gzip_compressor() override;\n\n  bool compress(const char *data, size_t data_length, bool last,\n                Callback callback) override;\n\nprivate:\n  bool is_valid_ = false;\n  z_stream strm_;\n};\n\nclass gzip_decompressor final : public decompressor {\npublic:\n  gzip_decompressor();\n  ~gzip_decompressor() override;\n\n  bool is_valid() const override;\n\n  bool decompress(const char *data, size_t data_length,\n                  Callback callback) override;\n\nprivate:\n  bool is_valid_ = false;\n  z_stream strm_;\n};\n#endif\n\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\nclass brotli_compressor final : public compressor {\npublic:\n  brotli_compressor();\n  ~brotli_compressor();\n\n  bool compress(const char *data, size_t data_length, bool last,\n                Callback callback) override;\n\nprivate:\n  BrotliEncoderState *state_ = nullptr;\n};\n\nclass brotli_decompressor final : public decompressor {\npublic:\n  brotli_decompressor();\n  ~brotli_decompressor();\n\n  bool is_valid() const override;\n\n  bool decompress(const char *data, size_t data_length,\n                  Callback callback) override;\n\nprivate:\n  BrotliDecoderResult decoder_r;\n  BrotliDecoderState *decoder_s = nullptr;\n};\n#endif\n\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\nclass zstd_compressor : public compressor {\npublic:\n  zstd_compressor();\n  ~zstd_compressor();\n\n  bool compress(const char *data, size_t data_length, bool last,\n                Callback callback) override;\n\nprivate:\n  ZSTD_CCtx *ctx_ = nullptr;\n};\n\nclass zstd_decompressor : public decompressor {\npublic:\n  zstd_decompressor();\n  ~zstd_decompressor();\n\n  bool is_valid() const override;\n\n  bool decompress(const char *data, size_t data_length,\n                  Callback callback) override;\n\nprivate:\n  ZSTD_DCtx *ctx_ = nullptr;\n};\n#endif\n\n// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`\n// to store data. The call can set memory on stack for performance.\nclass stream_line_reader {\npublic:\n  stream_line_reader(Stream &strm, char *fixed_buffer,\n                     size_t fixed_buffer_size);\n  const char *ptr() const;\n  size_t size() const;\n  bool end_with_crlf() const;\n  bool getline();\n\nprivate:\n  void append(char c);\n\n  Stream &strm_;\n  char *fixed_buffer_;\n  const size_t fixed_buffer_size_;\n  size_t fixed_buffer_used_size_ = 0;\n  std::string growable_buffer_;\n};\n\nclass mmap {\npublic:\n  mmap(const char *path);\n  ~mmap();\n\n  bool open(const char *path);\n  void close();\n\n  bool is_open() const;\n  size_t size() const;\n  const char *data() const;\n\nprivate:\n#if defined(_WIN32)\n  HANDLE hFile_ = NULL;\n  HANDLE hMapping_ = NULL;\n#else\n  int fd_ = -1;\n#endif\n  size_t size_ = 0;\n  void *addr_ = nullptr;\n  bool is_open_empty_file = false;\n};\n\n// NOTE: https://www.rfc-editor.org/rfc/rfc9110#section-5\nnamespace fields {\n\ninline bool is_token_char(char c) {\n  return std::isalnum(c) || c == '!' || c == '#' || c == '$' || c == '%' ||\n         c == '&' || c == '\\'' || c == '*' || c == '+' || c == '-' ||\n         c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~';\n}\n\ninline bool is_token(const std::string &s) {\n  if (s.empty()) { return false; }\n  for (auto c : s) {\n    if (!is_token_char(c)) { return false; }\n  }\n  return true;\n}\n\ninline bool is_field_name(const std::string &s) { return is_token(s); }\n\ninline bool is_vchar(char c) { return c >= 33 && c <= 126; }\n\ninline bool is_obs_text(char c) { return 128 <= static_cast<unsigned char>(c); }\n\ninline bool is_field_vchar(char c) { return is_vchar(c) || is_obs_text(c); }\n\ninline bool is_field_content(const std::string &s) {\n  if (s.empty()) { return true; }\n\n  if (s.size() == 1) {\n    return is_field_vchar(s[0]);\n  } else if (s.size() == 2) {\n    return is_field_vchar(s[0]) && is_field_vchar(s[1]);\n  } else {\n    size_t i = 0;\n\n    if (!is_field_vchar(s[i])) { return false; }\n    i++;\n\n    while (i < s.size() - 1) {\n      auto c = s[i++];\n      if (c == ' ' || c == '\\t' || is_field_vchar(c)) {\n      } else {\n        return false;\n      }\n    }\n\n    return is_field_vchar(s[i]);\n  }\n}\n\ninline bool is_field_value(const std::string &s) { return is_field_content(s); }\n\n} // namespace fields\n\n} // namespace detail\n\n// ----------------------------------------------------------------------------\n\n/*\n * Implementation that will be part of the .cc file if split into .h + .cc.\n */\n\nnamespace detail {\n\ninline bool is_hex(char c, int &v) {\n  if (0x20 <= c && isdigit(c)) {\n    v = c - '0';\n    return true;\n  } else if ('A' <= c && c <= 'F') {\n    v = c - 'A' + 10;\n    return true;\n  } else if ('a' <= c && c <= 'f') {\n    v = c - 'a' + 10;\n    return true;\n  }\n  return false;\n}\n\ninline bool from_hex_to_i(const std::string &s, size_t i, size_t cnt,\n                          int &val) {\n  if (i >= s.size()) { return false; }\n\n  val = 0;\n  for (; cnt; i++, cnt--) {\n    if (!s[i]) { return false; }\n    auto v = 0;\n    if (is_hex(s[i], v)) {\n      val = val * 16 + v;\n    } else {\n      return false;\n    }\n  }\n  return true;\n}\n\ninline std::string from_i_to_hex(size_t n) {\n  static const auto charset = \"0123456789abcdef\";\n  std::string ret;\n  do {\n    ret = charset[n & 15] + ret;\n    n >>= 4;\n  } while (n > 0);\n  return ret;\n}\n\ninline size_t to_utf8(int code, char *buff) {\n  if (code < 0x0080) {\n    buff[0] = static_cast<char>(code & 0x7F);\n    return 1;\n  } else if (code < 0x0800) {\n    buff[0] = static_cast<char>(0xC0 | ((code >> 6) & 0x1F));\n    buff[1] = static_cast<char>(0x80 | (code & 0x3F));\n    return 2;\n  } else if (code < 0xD800) {\n    buff[0] = static_cast<char>(0xE0 | ((code >> 12) & 0xF));\n    buff[1] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));\n    buff[2] = static_cast<char>(0x80 | (code & 0x3F));\n    return 3;\n  } else if (code < 0xE000) { // D800 - DFFF is invalid...\n    return 0;\n  } else if (code < 0x10000) {\n    buff[0] = static_cast<char>(0xE0 | ((code >> 12) & 0xF));\n    buff[1] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));\n    buff[2] = static_cast<char>(0x80 | (code & 0x3F));\n    return 3;\n  } else if (code < 0x110000) {\n    buff[0] = static_cast<char>(0xF0 | ((code >> 18) & 0x7));\n    buff[1] = static_cast<char>(0x80 | ((code >> 12) & 0x3F));\n    buff[2] = static_cast<char>(0x80 | ((code >> 6) & 0x3F));\n    buff[3] = static_cast<char>(0x80 | (code & 0x3F));\n    return 4;\n  }\n\n  // NOTREACHED\n  return 0;\n}\n\n// NOTE: This code came up with the following stackoverflow post:\n// https://stackoverflow.com/questions/180947/base64-decode-snippet-in-c\ninline std::string base64_encode(const std::string &in) {\n  static const auto lookup =\n      \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n\n  std::string out;\n  out.reserve(in.size());\n\n  auto val = 0;\n  auto valb = -6;\n\n  for (auto c : in) {\n    val = (val << 8) + static_cast<uint8_t>(c);\n    valb += 8;\n    while (valb >= 0) {\n      out.push_back(lookup[(val >> valb) & 0x3F]);\n      valb -= 6;\n    }\n  }\n\n  if (valb > -6) { out.push_back(lookup[((val << 8) >> (valb + 8)) & 0x3F]); }\n\n  while (out.size() % 4) {\n    out.push_back('=');\n  }\n\n  return out;\n}\n\ninline bool is_valid_path(const std::string &path) {\n  size_t level = 0;\n  size_t i = 0;\n\n  // Skip slash\n  while (i < path.size() && path[i] == '/') {\n    i++;\n  }\n\n  while (i < path.size()) {\n    // Read component\n    auto beg = i;\n    while (i < path.size() && path[i] != '/') {\n      if (path[i] == '\\0') {\n        return false;\n      } else if (path[i] == '\\\\') {\n        return false;\n      }\n      i++;\n    }\n\n    auto len = i - beg;\n    assert(len > 0);\n\n    if (!path.compare(beg, len, \".\")) {\n      ;\n    } else if (!path.compare(beg, len, \"..\")) {\n      if (level == 0) { return false; }\n      level--;\n    } else {\n      level++;\n    }\n\n    // Skip slash\n    while (i < path.size() && path[i] == '/') {\n      i++;\n    }\n  }\n\n  return true;\n}\n\ninline FileStat::FileStat(const std::string &path) {\n#if defined(_WIN32)\n  auto wpath = u8string_to_wstring(path.c_str());\n  ret_ = _wstat(wpath.c_str(), &st_);\n#else\n  ret_ = stat(path.c_str(), &st_);\n#endif\n}\ninline bool FileStat::is_file() const {\n  return ret_ >= 0 && S_ISREG(st_.st_mode);\n}\ninline bool FileStat::is_dir() const {\n  return ret_ >= 0 && S_ISDIR(st_.st_mode);\n}\n\ninline std::string encode_query_param(const std::string &value) {\n  std::ostringstream escaped;\n  escaped.fill('0');\n  escaped << std::hex;\n\n  for (auto c : value) {\n    if (std::isalnum(static_cast<uint8_t>(c)) || c == '-' || c == '_' ||\n        c == '.' || c == '!' || c == '~' || c == '*' || c == '\\'' || c == '(' ||\n        c == ')') {\n      escaped << c;\n    } else {\n      escaped << std::uppercase;\n      escaped << '%' << std::setw(2)\n              << static_cast<int>(static_cast<unsigned char>(c));\n      escaped << std::nouppercase;\n    }\n  }\n\n  return escaped.str();\n}\n\ninline std::string encode_url(const std::string &s) {\n  std::string result;\n  result.reserve(s.size());\n\n  for (size_t i = 0; s[i]; i++) {\n    switch (s[i]) {\n    case ' ': result += \"%20\"; break;\n    case '+': result += \"%2B\"; break;\n    case '\\r': result += \"%0D\"; break;\n    case '\\n': result += \"%0A\"; break;\n    case '\\'': result += \"%27\"; break;\n    case ',': result += \"%2C\"; break;\n    // case ':': result += \"%3A\"; break; // ok? probably...\n    case ';': result += \"%3B\"; break;\n    default:\n      auto c = static_cast<uint8_t>(s[i]);\n      if (c >= 0x80) {\n        result += '%';\n        char hex[4];\n        auto len = snprintf(hex, sizeof(hex) - 1, \"%02X\", c);\n        assert(len == 2);\n        result.append(hex, static_cast<size_t>(len));\n      } else {\n        result += s[i];\n      }\n      break;\n    }\n  }\n\n  return result;\n}\n\ninline std::string decode_url(const std::string &s,\n                              bool convert_plus_to_space) {\n  std::string result;\n\n  for (size_t i = 0; i < s.size(); i++) {\n    if (s[i] == '%' && i + 1 < s.size()) {\n      if (s[i + 1] == 'u') {\n        auto val = 0;\n        if (from_hex_to_i(s, i + 2, 4, val)) {\n          // 4 digits Unicode codes\n          char buff[4];\n          size_t len = to_utf8(val, buff);\n          if (len > 0) { result.append(buff, len); }\n          i += 5; // 'u0000'\n        } else {\n          result += s[i];\n        }\n      } else {\n        auto val = 0;\n        if (from_hex_to_i(s, i + 1, 2, val)) {\n          // 2 digits hex codes\n          result += static_cast<char>(val);\n          i += 2; // '00'\n        } else {\n          result += s[i];\n        }\n      }\n    } else if (convert_plus_to_space && s[i] == '+') {\n      result += ' ';\n    } else {\n      result += s[i];\n    }\n  }\n\n  return result;\n}\n\ninline std::string file_extension(const std::string &path) {\n  std::smatch m;\n  thread_local auto re = std::regex(\"\\\\.([a-zA-Z0-9]+)$\");\n  if (std::regex_search(path, m, re)) { return m[1].str(); }\n  return std::string();\n}\n\ninline bool is_space_or_tab(char c) { return c == ' ' || c == '\\t'; }\n\ninline std::pair<size_t, size_t> trim(const char *b, const char *e, size_t left,\n                                      size_t right) {\n  while (b + left < e && is_space_or_tab(b[left])) {\n    left++;\n  }\n  while (right > 0 && is_space_or_tab(b[right - 1])) {\n    right--;\n  }\n  return std::make_pair(left, right);\n}\n\ninline std::string trim_copy(const std::string &s) {\n  auto r = trim(s.data(), s.data() + s.size(), 0, s.size());\n  return s.substr(r.first, r.second - r.first);\n}\n\ninline std::string trim_double_quotes_copy(const std::string &s) {\n  if (s.length() >= 2 && s.front() == '\"' && s.back() == '\"') {\n    return s.substr(1, s.size() - 2);\n  }\n  return s;\n}\n\ninline void\ndivide(const char *data, std::size_t size, char d,\n       std::function<void(const char *, std::size_t, const char *, std::size_t)>\n           fn) {\n  const auto it = std::find(data, data + size, d);\n  const auto found = static_cast<std::size_t>(it != data + size);\n  const auto lhs_data = data;\n  const auto lhs_size = static_cast<std::size_t>(it - data);\n  const auto rhs_data = it + found;\n  const auto rhs_size = size - lhs_size - found;\n\n  fn(lhs_data, lhs_size, rhs_data, rhs_size);\n}\n\ninline void\ndivide(const std::string &str, char d,\n       std::function<void(const char *, std::size_t, const char *, std::size_t)>\n           fn) {\n  divide(str.data(), str.size(), d, std::move(fn));\n}\n\ninline void split(const char *b, const char *e, char d,\n                  std::function<void(const char *, const char *)> fn) {\n  return split(b, e, d, (std::numeric_limits<size_t>::max)(), std::move(fn));\n}\n\ninline void split(const char *b, const char *e, char d, size_t m,\n                  std::function<void(const char *, const char *)> fn) {\n  size_t i = 0;\n  size_t beg = 0;\n  size_t count = 1;\n\n  while (e ? (b + i < e) : (b[i] != '\\0')) {\n    if (b[i] == d && count < m) {\n      auto r = trim(b, e, beg, i);\n      if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }\n      beg = i + 1;\n      count++;\n    }\n    i++;\n  }\n\n  if (i) {\n    auto r = trim(b, e, beg, i);\n    if (r.first < r.second) { fn(&b[r.first], &b[r.second]); }\n  }\n}\n\ninline stream_line_reader::stream_line_reader(Stream &strm, char *fixed_buffer,\n                                              size_t fixed_buffer_size)\n    : strm_(strm), fixed_buffer_(fixed_buffer),\n      fixed_buffer_size_(fixed_buffer_size) {}\n\ninline const char *stream_line_reader::ptr() const {\n  if (growable_buffer_.empty()) {\n    return fixed_buffer_;\n  } else {\n    return growable_buffer_.data();\n  }\n}\n\ninline size_t stream_line_reader::size() const {\n  if (growable_buffer_.empty()) {\n    return fixed_buffer_used_size_;\n  } else {\n    return growable_buffer_.size();\n  }\n}\n\ninline bool stream_line_reader::end_with_crlf() const {\n  auto end = ptr() + size();\n  return size() >= 2 && end[-2] == '\\r' && end[-1] == '\\n';\n}\n\ninline bool stream_line_reader::getline() {\n  fixed_buffer_used_size_ = 0;\n  growable_buffer_.clear();\n\n#ifndef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR\n  char prev_byte = 0;\n#endif\n\n  for (size_t i = 0;; i++) {\n    char byte;\n    auto n = strm_.read(&byte, 1);\n\n    if (n < 0) {\n      return false;\n    } else if (n == 0) {\n      if (i == 0) {\n        return false;\n      } else {\n        break;\n      }\n    }\n\n    append(byte);\n\n#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR\n    if (byte == '\\n') { break; }\n#else\n    if (prev_byte == '\\r' && byte == '\\n') { break; }\n    prev_byte = byte;\n#endif\n  }\n\n  return true;\n}\n\ninline void stream_line_reader::append(char c) {\n  if (fixed_buffer_used_size_ < fixed_buffer_size_ - 1) {\n    fixed_buffer_[fixed_buffer_used_size_++] = c;\n    fixed_buffer_[fixed_buffer_used_size_] = '\\0';\n  } else {\n    if (growable_buffer_.empty()) {\n      assert(fixed_buffer_[fixed_buffer_used_size_] == '\\0');\n      growable_buffer_.assign(fixed_buffer_, fixed_buffer_used_size_);\n    }\n    growable_buffer_ += c;\n  }\n}\n\ninline mmap::mmap(const char *path) { open(path); }\n\ninline mmap::~mmap() { close(); }\n\ninline bool mmap::open(const char *path) {\n  close();\n\n#if defined(_WIN32)\n  auto wpath = u8string_to_wstring(path);\n  if (wpath.empty()) { return false; }\n\n#if _WIN32_WINNT >= _WIN32_WINNT_WIN8\n  hFile_ = ::CreateFile2(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ,\n                         OPEN_EXISTING, NULL);\n#else\n  hFile_ = ::CreateFileW(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,\n                         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);\n#endif\n\n  if (hFile_ == INVALID_HANDLE_VALUE) { return false; }\n\n  LARGE_INTEGER size{};\n  if (!::GetFileSizeEx(hFile_, &size)) { return false; }\n  // If the following line doesn't compile due to QuadPart, update Windows SDK.\n  // See:\n  // https://github.com/yhirose/cpp-httplib/issues/1903#issuecomment-2316520721\n  if (static_cast<ULONGLONG>(size.QuadPart) >\n      (std::numeric_limits<decltype(size_)>::max)()) {\n    // `size_t` might be 32-bits, on 32-bits Windows.\n    return false;\n  }\n  size_ = static_cast<size_t>(size.QuadPart);\n\n#if _WIN32_WINNT >= _WIN32_WINNT_WIN8\n  hMapping_ =\n      ::CreateFileMappingFromApp(hFile_, NULL, PAGE_READONLY, size_, NULL);\n#else\n  hMapping_ = ::CreateFileMappingW(hFile_, NULL, PAGE_READONLY, 0, 0, NULL);\n#endif\n\n  // Special treatment for an empty file...\n  if (hMapping_ == NULL && size_ == 0) {\n    close();\n    is_open_empty_file = true;\n    return true;\n  }\n\n  if (hMapping_ == NULL) {\n    close();\n    return false;\n  }\n\n#if _WIN32_WINNT >= _WIN32_WINNT_WIN8\n  addr_ = ::MapViewOfFileFromApp(hMapping_, FILE_MAP_READ, 0, 0);\n#else\n  addr_ = ::MapViewOfFile(hMapping_, FILE_MAP_READ, 0, 0, 0);\n#endif\n\n  if (addr_ == nullptr) {\n    close();\n    return false;\n  }\n#else\n  fd_ = ::open(path, O_RDONLY);\n  if (fd_ == -1) { return false; }\n\n  struct stat sb;\n  if (fstat(fd_, &sb) == -1) {\n    close();\n    return false;\n  }\n  size_ = static_cast<size_t>(sb.st_size);\n\n  addr_ = ::mmap(NULL, size_, PROT_READ, MAP_PRIVATE, fd_, 0);\n\n  // Special treatment for an empty file...\n  if (addr_ == MAP_FAILED && size_ == 0) {\n    close();\n    is_open_empty_file = true;\n    return false;\n  }\n#endif\n\n  return true;\n}\n\ninline bool mmap::is_open() const {\n  return is_open_empty_file ? true : addr_ != nullptr;\n}\n\ninline size_t mmap::size() const { return size_; }\n\ninline const char *mmap::data() const {\n  return is_open_empty_file ? \"\" : static_cast<const char *>(addr_);\n}\n\ninline void mmap::close() {\n#if defined(_WIN32)\n  if (addr_) {\n    ::UnmapViewOfFile(addr_);\n    addr_ = nullptr;\n  }\n\n  if (hMapping_) {\n    ::CloseHandle(hMapping_);\n    hMapping_ = NULL;\n  }\n\n  if (hFile_ != INVALID_HANDLE_VALUE) {\n    ::CloseHandle(hFile_);\n    hFile_ = INVALID_HANDLE_VALUE;\n  }\n\n  is_open_empty_file = false;\n#else\n  if (addr_ != nullptr) {\n    munmap(addr_, size_);\n    addr_ = nullptr;\n  }\n\n  if (fd_ != -1) {\n    ::close(fd_);\n    fd_ = -1;\n  }\n#endif\n  size_ = 0;\n}\ninline int close_socket(socket_t sock) {\n#ifdef _WIN32\n  return closesocket(sock);\n#else\n  return close(sock);\n#endif\n}\n\ntemplate <typename T> inline ssize_t handle_EINTR(T fn) {\n  ssize_t res = 0;\n  while (true) {\n    res = fn();\n    if (res < 0 && errno == EINTR) {\n      std::this_thread::sleep_for(std::chrono::microseconds{1});\n      continue;\n    }\n    break;\n  }\n  return res;\n}\n\ninline ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags) {\n  return handle_EINTR([&]() {\n    return recv(sock,\n#ifdef _WIN32\n                static_cast<char *>(ptr), static_cast<int>(size),\n#else\n                ptr, size,\n#endif\n                flags);\n  });\n}\n\ninline ssize_t send_socket(socket_t sock, const void *ptr, size_t size,\n                           int flags) {\n  return handle_EINTR([&]() {\n    return send(sock,\n#ifdef _WIN32\n                static_cast<const char *>(ptr), static_cast<int>(size),\n#else\n                ptr, size,\n#endif\n                flags);\n  });\n}\n\ninline int poll_wrapper(struct pollfd *fds, nfds_t nfds, int timeout) {\n#ifdef _WIN32\n  return ::WSAPoll(fds, nfds, timeout);\n#else\n  return ::poll(fds, nfds, timeout);\n#endif\n}\n\ntemplate <bool Read>\ninline ssize_t select_impl(socket_t sock, time_t sec, time_t usec) {\n  struct pollfd pfd;\n  pfd.fd = sock;\n  pfd.events = (Read ? POLLIN : POLLOUT);\n\n  auto timeout = static_cast<int>(sec * 1000 + usec / 1000);\n\n  return handle_EINTR([&]() { return poll_wrapper(&pfd, 1, timeout); });\n}\n\ninline ssize_t select_read(socket_t sock, time_t sec, time_t usec) {\n  return select_impl<true>(sock, sec, usec);\n}\n\ninline ssize_t select_write(socket_t sock, time_t sec, time_t usec) {\n  return select_impl<false>(sock, sec, usec);\n}\n\ninline Error wait_until_socket_is_ready(socket_t sock, time_t sec,\n                                        time_t usec) {\n  struct pollfd pfd_read;\n  pfd_read.fd = sock;\n  pfd_read.events = POLLIN | POLLOUT;\n\n  auto timeout = static_cast<int>(sec * 1000 + usec / 1000);\n\n  auto poll_res =\n      handle_EINTR([&]() { return poll_wrapper(&pfd_read, 1, timeout); });\n\n  if (poll_res == 0) { return Error::ConnectionTimeout; }\n\n  if (poll_res > 0 && pfd_read.revents & (POLLIN | POLLOUT)) {\n    auto error = 0;\n    socklen_t len = sizeof(error);\n    auto res = getsockopt(sock, SOL_SOCKET, SO_ERROR,\n                          reinterpret_cast<char *>(&error), &len);\n    auto successful = res >= 0 && !error;\n    return successful ? Error::Success : Error::Connection;\n  }\n\n  return Error::Connection;\n}\n\ninline bool is_socket_alive(socket_t sock) {\n  const auto val = detail::select_read(sock, 0, 0);\n  if (val == 0) {\n    return true;\n  } else if (val < 0 && errno == EBADF) {\n    return false;\n  }\n  char buf[1];\n  return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;\n}\n\nclass SocketStream final : public Stream {\npublic:\n  SocketStream(socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,\n               time_t write_timeout_sec, time_t write_timeout_usec,\n               time_t max_timeout_msec = 0,\n               std::chrono::time_point<std::chrono::steady_clock> start_time =\n                   (std::chrono::steady_clock::time_point::min)());\n  ~SocketStream() override;\n\n  bool is_readable() const override;\n  bool wait_readable() const override;\n  bool wait_writable() const override;\n  ssize_t read(char *ptr, size_t size) override;\n  ssize_t write(const char *ptr, size_t size) override;\n  void get_remote_ip_and_port(std::string &ip, int &port) const override;\n  void get_local_ip_and_port(std::string &ip, int &port) const override;\n  socket_t socket() const override;\n  time_t duration() const override;\n\nprivate:\n  socket_t sock_;\n  time_t read_timeout_sec_;\n  time_t read_timeout_usec_;\n  time_t write_timeout_sec_;\n  time_t write_timeout_usec_;\n  time_t max_timeout_msec_;\n  const std::chrono::time_point<std::chrono::steady_clock> start_time;\n\n  std::vector<char> read_buff_;\n  size_t read_buff_off_ = 0;\n  size_t read_buff_content_size_ = 0;\n\n  static const size_t read_buff_size_ = 1024l * 4;\n};\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\nclass SSLSocketStream final : public Stream {\npublic:\n  SSLSocketStream(\n      socket_t sock, SSL *ssl, time_t read_timeout_sec,\n      time_t read_timeout_usec, time_t write_timeout_sec,\n      time_t write_timeout_usec, time_t max_timeout_msec = 0,\n      std::chrono::time_point<std::chrono::steady_clock> start_time =\n          (std::chrono::steady_clock::time_point::min)());\n  ~SSLSocketStream() override;\n\n  bool is_readable() const override;\n  bool wait_readable() const override;\n  bool wait_writable() const override;\n  ssize_t read(char *ptr, size_t size) override;\n  ssize_t write(const char *ptr, size_t size) override;\n  void get_remote_ip_and_port(std::string &ip, int &port) const override;\n  void get_local_ip_and_port(std::string &ip, int &port) const override;\n  socket_t socket() const override;\n  time_t duration() const override;\n\nprivate:\n  socket_t sock_;\n  SSL *ssl_;\n  time_t read_timeout_sec_;\n  time_t read_timeout_usec_;\n  time_t write_timeout_sec_;\n  time_t write_timeout_usec_;\n  time_t max_timeout_msec_;\n  const std::chrono::time_point<std::chrono::steady_clock> start_time;\n};\n#endif\n\ninline bool keep_alive(const std::atomic<socket_t> &svr_sock, socket_t sock,\n                       time_t keep_alive_timeout_sec) {\n  using namespace std::chrono;\n\n  const auto interval_usec =\n      CPPHTTPLIB_KEEPALIVE_TIMEOUT_CHECK_INTERVAL_USECOND;\n\n  // Avoid expensive `steady_clock::now()` call for the first time\n  if (select_read(sock, 0, interval_usec) > 0) { return true; }\n\n  const auto start = steady_clock::now() - microseconds{interval_usec};\n  const auto timeout = seconds{keep_alive_timeout_sec};\n\n  while (true) {\n    if (svr_sock == INVALID_SOCKET) {\n      break; // Server socket is closed\n    }\n\n    auto val = select_read(sock, 0, interval_usec);\n    if (val < 0) {\n      break; // Ssocket error\n    } else if (val == 0) {\n      if (steady_clock::now() - start > timeout) {\n        break; // Timeout\n      }\n    } else {\n      return true; // Ready for read\n    }\n  }\n\n  return false;\n}\n\ntemplate <typename T>\ninline bool\nprocess_server_socket_core(const std::atomic<socket_t> &svr_sock, socket_t sock,\n                           size_t keep_alive_max_count,\n                           time_t keep_alive_timeout_sec, T callback) {\n  assert(keep_alive_max_count > 0);\n  auto ret = false;\n  auto count = keep_alive_max_count;\n  while (count > 0 && keep_alive(svr_sock, sock, keep_alive_timeout_sec)) {\n    auto close_connection = count == 1;\n    auto connection_closed = false;\n    ret = callback(close_connection, connection_closed);\n    if (!ret || connection_closed) { break; }\n    count--;\n  }\n  return ret;\n}\n\ntemplate <typename T>\ninline bool\nprocess_server_socket(const std::atomic<socket_t> &svr_sock, socket_t sock,\n                      size_t keep_alive_max_count,\n                      time_t keep_alive_timeout_sec, time_t read_timeout_sec,\n                      time_t read_timeout_usec, time_t write_timeout_sec,\n                      time_t write_timeout_usec, T callback) {\n  return process_server_socket_core(\n      svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec,\n      [&](bool close_connection, bool &connection_closed) {\n        SocketStream strm(sock, read_timeout_sec, read_timeout_usec,\n                          write_timeout_sec, write_timeout_usec);\n        return callback(strm, close_connection, connection_closed);\n      });\n}\n\ninline bool process_client_socket(\n    socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,\n    time_t write_timeout_sec, time_t write_timeout_usec,\n    time_t max_timeout_msec,\n    std::chrono::time_point<std::chrono::steady_clock> start_time,\n    std::function<bool(Stream &)> callback) {\n  SocketStream strm(sock, read_timeout_sec, read_timeout_usec,\n                    write_timeout_sec, write_timeout_usec, max_timeout_msec,\n                    start_time);\n  return callback(strm);\n}\n\ninline int shutdown_socket(socket_t sock) {\n#ifdef _WIN32\n  return shutdown(sock, SD_BOTH);\n#else\n  return shutdown(sock, SHUT_RDWR);\n#endif\n}\n\ninline std::string escape_abstract_namespace_unix_domain(const std::string &s) {\n  if (s.size() > 1 && s[0] == '\\0') {\n    auto ret = s;\n    ret[0] = '@';\n    return ret;\n  }\n  return s;\n}\n\ninline std::string\nunescape_abstract_namespace_unix_domain(const std::string &s) {\n  if (s.size() > 1 && s[0] == '@') {\n    auto ret = s;\n    ret[0] = '\\0';\n    return ret;\n  }\n  return s;\n}\n\ntemplate <typename BindOrConnect>\nsocket_t create_socket(const std::string &host, const std::string &ip, int port,\n                       int address_family, int socket_flags, bool tcp_nodelay,\n                       bool ipv6_v6only, SocketOptions socket_options,\n                       BindOrConnect bind_or_connect) {\n  // Get address info\n  const char *node = nullptr;\n  struct addrinfo hints;\n  struct addrinfo *result;\n\n  memset(&hints, 0, sizeof(struct addrinfo));\n  hints.ai_socktype = SOCK_STREAM;\n  hints.ai_protocol = IPPROTO_IP;\n\n  if (!ip.empty()) {\n    node = ip.c_str();\n    // Ask getaddrinfo to convert IP in c-string to address\n    hints.ai_family = AF_UNSPEC;\n    hints.ai_flags = AI_NUMERICHOST;\n  } else {\n    if (!host.empty()) { node = host.c_str(); }\n    hints.ai_family = address_family;\n    hints.ai_flags = socket_flags;\n  }\n\n#ifndef _WIN32\n  if (hints.ai_family == AF_UNIX) {\n    const auto addrlen = host.length();\n    if (addrlen > sizeof(sockaddr_un::sun_path)) { return INVALID_SOCKET; }\n\n#ifdef SOCK_CLOEXEC\n    auto sock = socket(hints.ai_family, hints.ai_socktype | SOCK_CLOEXEC,\n                       hints.ai_protocol);\n#else\n    auto sock = socket(hints.ai_family, hints.ai_socktype, hints.ai_protocol);\n#endif\n\n    if (sock != INVALID_SOCKET) {\n      sockaddr_un addr{};\n      addr.sun_family = AF_UNIX;\n\n      auto unescaped_host = unescape_abstract_namespace_unix_domain(host);\n      std::copy(unescaped_host.begin(), unescaped_host.end(), addr.sun_path);\n\n      hints.ai_addr = reinterpret_cast<sockaddr *>(&addr);\n      hints.ai_addrlen = static_cast<socklen_t>(\n          sizeof(addr) - sizeof(addr.sun_path) + addrlen);\n\n#ifndef SOCK_CLOEXEC\n      fcntl(sock, F_SETFD, FD_CLOEXEC);\n#endif\n\n      if (socket_options) { socket_options(sock); }\n\n      bool dummy;\n      if (!bind_or_connect(sock, hints, dummy)) {\n        close_socket(sock);\n        sock = INVALID_SOCKET;\n      }\n    }\n    return sock;\n  }\n#endif\n\n  auto service = std::to_string(port);\n\n  if (getaddrinfo(node, service.c_str(), &hints, &result)) {\n#if defined __linux__ && !defined __ANDROID__\n    res_init();\n#endif\n    return INVALID_SOCKET;\n  }\n  auto se = detail::scope_exit([&] { freeaddrinfo(result); });\n\n  for (auto rp = result; rp; rp = rp->ai_next) {\n    // Create a socket\n#ifdef _WIN32\n    auto sock =\n        WSASocketW(rp->ai_family, rp->ai_socktype, rp->ai_protocol, nullptr, 0,\n                   WSA_FLAG_NO_HANDLE_INHERIT | WSA_FLAG_OVERLAPPED);\n    /**\n     * Since the WSA_FLAG_NO_HANDLE_INHERIT is only supported on Windows 7 SP1\n     * and above the socket creation fails on older Windows Systems.\n     *\n     * Let's try to create a socket the old way in this case.\n     *\n     * Reference:\n     * https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketa\n     *\n     * WSA_FLAG_NO_HANDLE_INHERIT:\n     * This flag is supported on Windows 7 with SP1, Windows Server 2008 R2 with\n     * SP1, and later\n     *\n     */\n    if (sock == INVALID_SOCKET) {\n      sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);\n    }\n#else\n\n#ifdef SOCK_CLOEXEC\n    auto sock =\n        socket(rp->ai_family, rp->ai_socktype | SOCK_CLOEXEC, rp->ai_protocol);\n#else\n    auto sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);\n#endif\n\n#endif\n    if (sock == INVALID_SOCKET) { continue; }\n\n#if !defined _WIN32 && !defined SOCK_CLOEXEC\n    if (fcntl(sock, F_SETFD, FD_CLOEXEC) == -1) {\n      close_socket(sock);\n      continue;\n    }\n#endif\n\n    if (tcp_nodelay) { set_socket_opt(sock, IPPROTO_TCP, TCP_NODELAY, 1); }\n\n    if (rp->ai_family == AF_INET6) {\n      set_socket_opt(sock, IPPROTO_IPV6, IPV6_V6ONLY, ipv6_v6only ? 1 : 0);\n    }\n\n    if (socket_options) { socket_options(sock); }\n\n    // bind or connect\n    auto quit = false;\n    if (bind_or_connect(sock, *rp, quit)) { return sock; }\n\n    close_socket(sock);\n\n    if (quit) { break; }\n  }\n\n  return INVALID_SOCKET;\n}\n\ninline void set_nonblocking(socket_t sock, bool nonblocking) {\n#ifdef _WIN32\n  auto flags = nonblocking ? 1UL : 0UL;\n  ioctlsocket(sock, FIONBIO, &flags);\n#else\n  auto flags = fcntl(sock, F_GETFL, 0);\n  fcntl(sock, F_SETFL,\n        nonblocking ? (flags | O_NONBLOCK) : (flags & (~O_NONBLOCK)));\n#endif\n}\n\ninline bool is_connection_error() {\n#ifdef _WIN32\n  return WSAGetLastError() != WSAEWOULDBLOCK;\n#else\n  return errno != EINPROGRESS;\n#endif\n}\n\ninline bool bind_ip_address(socket_t sock, const std::string &host) {\n  struct addrinfo hints;\n  struct addrinfo *result;\n\n  memset(&hints, 0, sizeof(struct addrinfo));\n  hints.ai_family = AF_UNSPEC;\n  hints.ai_socktype = SOCK_STREAM;\n  hints.ai_protocol = 0;\n\n  if (getaddrinfo(host.c_str(), \"0\", &hints, &result)) { return false; }\n  auto se = detail::scope_exit([&] { freeaddrinfo(result); });\n\n  auto ret = false;\n  for (auto rp = result; rp; rp = rp->ai_next) {\n    const auto &ai = *rp;\n    if (!::bind(sock, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen))) {\n      ret = true;\n      break;\n    }\n  }\n\n  return ret;\n}\n\n#if !defined _WIN32 && !defined ANDROID && !defined _AIX && !defined __MVS__\n#define USE_IF2IP\n#endif\n\n#ifdef USE_IF2IP\ninline std::string if2ip(int address_family, const std::string &ifn) {\n  struct ifaddrs *ifap;\n  getifaddrs(&ifap);\n  auto se = detail::scope_exit([&] { freeifaddrs(ifap); });\n\n  std::string addr_candidate;\n  for (auto ifa = ifap; ifa; ifa = ifa->ifa_next) {\n    if (ifa->ifa_addr && ifn == ifa->ifa_name &&\n        (AF_UNSPEC == address_family ||\n         ifa->ifa_addr->sa_family == address_family)) {\n      if (ifa->ifa_addr->sa_family == AF_INET) {\n        auto sa = reinterpret_cast<struct sockaddr_in *>(ifa->ifa_addr);\n        char buf[INET_ADDRSTRLEN];\n        if (inet_ntop(AF_INET, &sa->sin_addr, buf, INET_ADDRSTRLEN)) {\n          return std::string(buf, INET_ADDRSTRLEN);\n        }\n      } else if (ifa->ifa_addr->sa_family == AF_INET6) {\n        auto sa = reinterpret_cast<struct sockaddr_in6 *>(ifa->ifa_addr);\n        if (!IN6_IS_ADDR_LINKLOCAL(&sa->sin6_addr)) {\n          char buf[INET6_ADDRSTRLEN] = {};\n          if (inet_ntop(AF_INET6, &sa->sin6_addr, buf, INET6_ADDRSTRLEN)) {\n            // equivalent to mac's IN6_IS_ADDR_UNIQUE_LOCAL\n            auto s6_addr_head = sa->sin6_addr.s6_addr[0];\n            if (s6_addr_head == 0xfc || s6_addr_head == 0xfd) {\n              addr_candidate = std::string(buf, INET6_ADDRSTRLEN);\n            } else {\n              return std::string(buf, INET6_ADDRSTRLEN);\n            }\n          }\n        }\n      }\n    }\n  }\n  return addr_candidate;\n}\n#endif\n\ninline socket_t create_client_socket(\n    const std::string &host, const std::string &ip, int port,\n    int address_family, bool tcp_nodelay, bool ipv6_v6only,\n    SocketOptions socket_options, time_t connection_timeout_sec,\n    time_t connection_timeout_usec, time_t read_timeout_sec,\n    time_t read_timeout_usec, time_t write_timeout_sec,\n    time_t write_timeout_usec, const std::string &intf, Error &error) {\n  auto sock = create_socket(\n      host, ip, port, address_family, 0, tcp_nodelay, ipv6_v6only,\n      std::move(socket_options),\n      [&](socket_t sock2, struct addrinfo &ai, bool &quit) -> bool {\n        if (!intf.empty()) {\n#ifdef USE_IF2IP\n          auto ip_from_if = if2ip(address_family, intf);\n          if (ip_from_if.empty()) { ip_from_if = intf; }\n          if (!bind_ip_address(sock2, ip_from_if)) {\n            error = Error::BindIPAddress;\n            return false;\n          }\n#endif\n        }\n\n        set_nonblocking(sock2, true);\n\n        auto ret =\n            ::connect(sock2, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen));\n\n        if (ret < 0) {\n          if (is_connection_error()) {\n            error = Error::Connection;\n            return false;\n          }\n          error = wait_until_socket_is_ready(sock2, connection_timeout_sec,\n                                             connection_timeout_usec);\n          if (error != Error::Success) {\n            if (error == Error::ConnectionTimeout) { quit = true; }\n            return false;\n          }\n        }\n\n        set_nonblocking(sock2, false);\n        set_socket_opt_time(sock2, SOL_SOCKET, SO_RCVTIMEO, read_timeout_sec,\n                            read_timeout_usec);\n        set_socket_opt_time(sock2, SOL_SOCKET, SO_SNDTIMEO, write_timeout_sec,\n                            write_timeout_usec);\n\n        error = Error::Success;\n        return true;\n      });\n\n  if (sock != INVALID_SOCKET) {\n    error = Error::Success;\n  } else {\n    if (error == Error::Success) { error = Error::Connection; }\n  }\n\n  return sock;\n}\n\ninline bool get_ip_and_port(const struct sockaddr_storage &addr,\n                            socklen_t addr_len, std::string &ip, int &port) {\n  if (addr.ss_family == AF_INET) {\n    port = ntohs(reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_port);\n  } else if (addr.ss_family == AF_INET6) {\n    port =\n        ntohs(reinterpret_cast<const struct sockaddr_in6 *>(&addr)->sin6_port);\n  } else {\n    return false;\n  }\n\n  std::array<char, NI_MAXHOST> ipstr{};\n  if (getnameinfo(reinterpret_cast<const struct sockaddr *>(&addr), addr_len,\n                  ipstr.data(), static_cast<socklen_t>(ipstr.size()), nullptr,\n                  0, NI_NUMERICHOST)) {\n    return false;\n  }\n\n  ip = ipstr.data();\n  return true;\n}\n\ninline void get_local_ip_and_port(socket_t sock, std::string &ip, int &port) {\n  struct sockaddr_storage addr;\n  socklen_t addr_len = sizeof(addr);\n  if (!getsockname(sock, reinterpret_cast<struct sockaddr *>(&addr),\n                   &addr_len)) {\n    get_ip_and_port(addr, addr_len, ip, port);\n  }\n}\n\ninline void get_remote_ip_and_port(socket_t sock, std::string &ip, int &port) {\n  struct sockaddr_storage addr;\n  socklen_t addr_len = sizeof(addr);\n\n  if (!getpeername(sock, reinterpret_cast<struct sockaddr *>(&addr),\n                   &addr_len)) {\n#ifndef _WIN32\n    if (addr.ss_family == AF_UNIX) {\n#if defined(__linux__)\n      struct ucred ucred;\n      socklen_t len = sizeof(ucred);\n      if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &len) == 0) {\n        port = ucred.pid;\n      }\n#elif defined(SOL_LOCAL) && defined(SO_PEERPID) // __APPLE__\n      pid_t pid;\n      socklen_t len = sizeof(pid);\n      if (getsockopt(sock, SOL_LOCAL, SO_PEERPID, &pid, &len) == 0) {\n        port = pid;\n      }\n#endif\n      return;\n    }\n#endif\n    get_ip_and_port(addr, addr_len, ip, port);\n  }\n}\n\ninline constexpr unsigned int str2tag_core(const char *s, size_t l,\n                                           unsigned int h) {\n  return (l == 0)\n             ? h\n             : str2tag_core(\n                   s + 1, l - 1,\n                   // Unsets the 6 high bits of h, therefore no overflow happens\n                   (((std::numeric_limits<unsigned int>::max)() >> 6) &\n                    h * 33) ^\n                       static_cast<unsigned char>(*s));\n}\n\ninline unsigned int str2tag(const std::string &s) {\n  return str2tag_core(s.data(), s.size(), 0);\n}\n\nnamespace udl {\n\ninline constexpr unsigned int operator\"\"_t(const char *s, size_t l) {\n  return str2tag_core(s, l, 0);\n}\n\n} // namespace udl\n\ninline std::string\nfind_content_type(const std::string &path,\n                  const std::map<std::string, std::string> &user_data,\n                  const std::string &default_content_type) {\n  auto ext = file_extension(path);\n\n  auto it = user_data.find(ext);\n  if (it != user_data.end()) { return it->second; }\n\n  using udl::operator\"\"_t;\n\n  switch (str2tag(ext)) {\n  default: return default_content_type;\n\n  case \"css\"_t: return \"text/css\";\n  case \"csv\"_t: return \"text/csv\";\n  case \"htm\"_t:\n  case \"html\"_t: return \"text/html\";\n  case \"js\"_t:\n  case \"mjs\"_t: return \"text/javascript\";\n  case \"txt\"_t: return \"text/plain\";\n  case \"vtt\"_t: return \"text/vtt\";\n\n  case \"apng\"_t: return \"image/apng\";\n  case \"avif\"_t: return \"image/avif\";\n  case \"bmp\"_t: return \"image/bmp\";\n  case \"gif\"_t: return \"image/gif\";\n  case \"png\"_t: return \"image/png\";\n  case \"svg\"_t: return \"image/svg+xml\";\n  case \"webp\"_t: return \"image/webp\";\n  case \"ico\"_t: return \"image/x-icon\";\n  case \"tif\"_t: return \"image/tiff\";\n  case \"tiff\"_t: return \"image/tiff\";\n  case \"jpg\"_t:\n  case \"jpeg\"_t: return \"image/jpeg\";\n\n  case \"mp4\"_t: return \"video/mp4\";\n  case \"mpeg\"_t: return \"video/mpeg\";\n  case \"webm\"_t: return \"video/webm\";\n\n  case \"mp3\"_t: return \"audio/mp3\";\n  case \"mpga\"_t: return \"audio/mpeg\";\n  case \"weba\"_t: return \"audio/webm\";\n  case \"wav\"_t: return \"audio/wave\";\n\n  case \"otf\"_t: return \"font/otf\";\n  case \"ttf\"_t: return \"font/ttf\";\n  case \"woff\"_t: return \"font/woff\";\n  case \"woff2\"_t: return \"font/woff2\";\n\n  case \"7z\"_t: return \"application/x-7z-compressed\";\n  case \"atom\"_t: return \"application/atom+xml\";\n  case \"pdf\"_t: return \"application/pdf\";\n  case \"json\"_t: return \"application/json\";\n  case \"rss\"_t: return \"application/rss+xml\";\n  case \"tar\"_t: return \"application/x-tar\";\n  case \"xht\"_t:\n  case \"xhtml\"_t: return \"application/xhtml+xml\";\n  case \"xslt\"_t: return \"application/xslt+xml\";\n  case \"xml\"_t: return \"application/xml\";\n  case \"gz\"_t: return \"application/gzip\";\n  case \"zip\"_t: return \"application/zip\";\n  case \"wasm\"_t: return \"application/wasm\";\n  }\n}\n\ninline bool can_compress_content_type(const std::string &content_type) {\n  using udl::operator\"\"_t;\n\n  auto tag = str2tag(content_type);\n\n  switch (tag) {\n  case \"image/svg+xml\"_t:\n  case \"application/javascript\"_t:\n  case \"application/json\"_t:\n  case \"application/xml\"_t:\n  case \"application/protobuf\"_t:\n  case \"application/xhtml+xml\"_t: return true;\n\n  case \"text/event-stream\"_t: return false;\n\n  default: return !content_type.rfind(\"text/\", 0);\n  }\n}\n\ninline EncodingType encoding_type(const Request &req, const Response &res) {\n  auto ret =\n      detail::can_compress_content_type(res.get_header_value(\"Content-Type\"));\n  if (!ret) { return EncodingType::None; }\n\n  const auto &s = req.get_header_value(\"Accept-Encoding\");\n  (void)(s);\n\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n  // TODO: 'Accept-Encoding' has br, not br;q=0\n  ret = s.find(\"br\") != std::string::npos;\n  if (ret) { return EncodingType::Brotli; }\n#endif\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n  // TODO: 'Accept-Encoding' has gzip, not gzip;q=0\n  ret = s.find(\"gzip\") != std::string::npos;\n  if (ret) { return EncodingType::Gzip; }\n#endif\n\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n  // TODO: 'Accept-Encoding' has zstd, not zstd;q=0\n  ret = s.find(\"zstd\") != std::string::npos;\n  if (ret) { return EncodingType::Zstd; }\n#endif\n\n  return EncodingType::None;\n}\n\ninline bool nocompressor::compress(const char *data, size_t data_length,\n                                   bool /*last*/, Callback callback) {\n  if (!data_length) { return true; }\n  return callback(data, data_length);\n}\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\ninline gzip_compressor::gzip_compressor() {\n  std::memset(&strm_, 0, sizeof(strm_));\n  strm_.zalloc = Z_NULL;\n  strm_.zfree = Z_NULL;\n  strm_.opaque = Z_NULL;\n\n  is_valid_ = deflateInit2(&strm_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,\n                           Z_DEFAULT_STRATEGY) == Z_OK;\n}\n\ninline gzip_compressor::~gzip_compressor() { deflateEnd(&strm_); }\n\ninline bool gzip_compressor::compress(const char *data, size_t data_length,\n                                      bool last, Callback callback) {\n  assert(is_valid_);\n\n  do {\n    constexpr size_t max_avail_in =\n        (std::numeric_limits<decltype(strm_.avail_in)>::max)();\n\n    strm_.avail_in = static_cast<decltype(strm_.avail_in)>(\n        (std::min)(data_length, max_avail_in));\n    strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));\n\n    data_length -= strm_.avail_in;\n    data += strm_.avail_in;\n\n    auto flush = (last && data_length == 0) ? Z_FINISH : Z_NO_FLUSH;\n    auto ret = Z_OK;\n\n    std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n    do {\n      strm_.avail_out = static_cast<uInt>(buff.size());\n      strm_.next_out = reinterpret_cast<Bytef *>(buff.data());\n\n      ret = deflate(&strm_, flush);\n      if (ret == Z_STREAM_ERROR) { return false; }\n\n      if (!callback(buff.data(), buff.size() - strm_.avail_out)) {\n        return false;\n      }\n    } while (strm_.avail_out == 0);\n\n    assert((flush == Z_FINISH && ret == Z_STREAM_END) ||\n           (flush == Z_NO_FLUSH && ret == Z_OK));\n    assert(strm_.avail_in == 0);\n  } while (data_length > 0);\n\n  return true;\n}\n\ninline gzip_decompressor::gzip_decompressor() {\n  std::memset(&strm_, 0, sizeof(strm_));\n  strm_.zalloc = Z_NULL;\n  strm_.zfree = Z_NULL;\n  strm_.opaque = Z_NULL;\n\n  // 15 is the value of wbits, which should be at the maximum possible value\n  // to ensure that any gzip stream can be decoded. The offset of 32 specifies\n  // that the stream type should be automatically detected either gzip or\n  // deflate.\n  is_valid_ = inflateInit2(&strm_, 32 + 15) == Z_OK;\n}\n\ninline gzip_decompressor::~gzip_decompressor() { inflateEnd(&strm_); }\n\ninline bool gzip_decompressor::is_valid() const { return is_valid_; }\n\ninline bool gzip_decompressor::decompress(const char *data, size_t data_length,\n                                          Callback callback) {\n  assert(is_valid_);\n\n  auto ret = Z_OK;\n\n  do {\n    constexpr size_t max_avail_in =\n        (std::numeric_limits<decltype(strm_.avail_in)>::max)();\n\n    strm_.avail_in = static_cast<decltype(strm_.avail_in)>(\n        (std::min)(data_length, max_avail_in));\n    strm_.next_in = const_cast<Bytef *>(reinterpret_cast<const Bytef *>(data));\n\n    data_length -= strm_.avail_in;\n    data += strm_.avail_in;\n\n    std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n    while (strm_.avail_in > 0 && ret == Z_OK) {\n      strm_.avail_out = static_cast<uInt>(buff.size());\n      strm_.next_out = reinterpret_cast<Bytef *>(buff.data());\n\n      ret = inflate(&strm_, Z_NO_FLUSH);\n\n      assert(ret != Z_STREAM_ERROR);\n      switch (ret) {\n      case Z_NEED_DICT:\n      case Z_DATA_ERROR:\n      case Z_MEM_ERROR: inflateEnd(&strm_); return false;\n      }\n\n      if (!callback(buff.data(), buff.size() - strm_.avail_out)) {\n        return false;\n      }\n    }\n\n    if (ret != Z_OK && ret != Z_STREAM_END) { return false; }\n\n  } while (data_length > 0);\n\n  return true;\n}\n#endif\n\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\ninline brotli_compressor::brotli_compressor() {\n  state_ = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);\n}\n\ninline brotli_compressor::~brotli_compressor() {\n  BrotliEncoderDestroyInstance(state_);\n}\n\ninline bool brotli_compressor::compress(const char *data, size_t data_length,\n                                        bool last, Callback callback) {\n  std::array<uint8_t, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n\n  auto operation = last ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS;\n  auto available_in = data_length;\n  auto next_in = reinterpret_cast<const uint8_t *>(data);\n\n  for (;;) {\n    if (last) {\n      if (BrotliEncoderIsFinished(state_)) { break; }\n    } else {\n      if (!available_in) { break; }\n    }\n\n    auto available_out = buff.size();\n    auto next_out = buff.data();\n\n    if (!BrotliEncoderCompressStream(state_, operation, &available_in, &next_in,\n                                     &available_out, &next_out, nullptr)) {\n      return false;\n    }\n\n    auto output_bytes = buff.size() - available_out;\n    if (output_bytes) {\n      callback(reinterpret_cast<const char *>(buff.data()), output_bytes);\n    }\n  }\n\n  return true;\n}\n\ninline brotli_decompressor::brotli_decompressor() {\n  decoder_s = BrotliDecoderCreateInstance(0, 0, 0);\n  decoder_r = decoder_s ? BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT\n                        : BROTLI_DECODER_RESULT_ERROR;\n}\n\ninline brotli_decompressor::~brotli_decompressor() {\n  if (decoder_s) { BrotliDecoderDestroyInstance(decoder_s); }\n}\n\ninline bool brotli_decompressor::is_valid() const { return decoder_s; }\n\ninline bool brotli_decompressor::decompress(const char *data,\n                                            size_t data_length,\n                                            Callback callback) {\n  if (decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||\n      decoder_r == BROTLI_DECODER_RESULT_ERROR) {\n    return 0;\n  }\n\n  auto next_in = reinterpret_cast<const uint8_t *>(data);\n  size_t avail_in = data_length;\n  size_t total_out;\n\n  decoder_r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;\n\n  std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n  while (decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {\n    char *next_out = buff.data();\n    size_t avail_out = buff.size();\n\n    decoder_r = BrotliDecoderDecompressStream(\n        decoder_s, &avail_in, &next_in, &avail_out,\n        reinterpret_cast<uint8_t **>(&next_out), &total_out);\n\n    if (decoder_r == BROTLI_DECODER_RESULT_ERROR) { return false; }\n\n    if (!callback(buff.data(), buff.size() - avail_out)) { return false; }\n  }\n\n  return decoder_r == BROTLI_DECODER_RESULT_SUCCESS ||\n         decoder_r == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;\n}\n#endif\n\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\ninline zstd_compressor::zstd_compressor() {\n  ctx_ = ZSTD_createCCtx();\n  ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast);\n}\n\ninline zstd_compressor::~zstd_compressor() { ZSTD_freeCCtx(ctx_); }\n\ninline bool zstd_compressor::compress(const char *data, size_t data_length,\n                                      bool last, Callback callback) {\n  std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n\n  ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue;\n  ZSTD_inBuffer input = {data, data_length, 0};\n\n  bool finished;\n  do {\n    ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0};\n    size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode);\n\n    if (ZSTD_isError(remaining)) { return false; }\n\n    if (!callback(buff.data(), output.pos)) { return false; }\n\n    finished = last ? (remaining == 0) : (input.pos == input.size);\n\n  } while (!finished);\n\n  return true;\n}\n\ninline zstd_decompressor::zstd_decompressor() { ctx_ = ZSTD_createDCtx(); }\n\ninline zstd_decompressor::~zstd_decompressor() { ZSTD_freeDCtx(ctx_); }\n\ninline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; }\n\ninline bool zstd_decompressor::decompress(const char *data, size_t data_length,\n                                          Callback callback) {\n  std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};\n  ZSTD_inBuffer input = {data, data_length, 0};\n\n  while (input.pos < input.size) {\n    ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0};\n    size_t const remaining = ZSTD_decompressStream(ctx_, &output, &input);\n\n    if (ZSTD_isError(remaining)) { return false; }\n\n    if (!callback(buff.data(), output.pos)) { return false; }\n  }\n\n  return true;\n}\n#endif\n\ninline bool has_header(const Headers &headers, const std::string &key) {\n  return headers.find(key) != headers.end();\n}\n\ninline const char *get_header_value(const Headers &headers,\n                                    const std::string &key, const char *def,\n                                    size_t id) {\n  auto rng = headers.equal_range(key);\n  auto it = rng.first;\n  std::advance(it, static_cast<ssize_t>(id));\n  if (it != rng.second) { return it->second.c_str(); }\n  return def;\n}\n\ntemplate <typename T>\ninline bool parse_header(const char *beg, const char *end, T fn) {\n  // Skip trailing spaces and tabs.\n  while (beg < end && is_space_or_tab(end[-1])) {\n    end--;\n  }\n\n  auto p = beg;\n  while (p < end && *p != ':') {\n    p++;\n  }\n\n  auto name = std::string(beg, p);\n  if (!detail::fields::is_field_name(name)) { return false; }\n\n  if (p == end) { return false; }\n\n  auto key_end = p;\n\n  if (*p++ != ':') { return false; }\n\n  while (p < end && is_space_or_tab(*p)) {\n    p++;\n  }\n\n  if (p <= end) {\n    auto key_len = key_end - beg;\n    if (!key_len) { return false; }\n\n    auto key = std::string(beg, key_end);\n    auto val = std::string(p, end);\n\n    if (!detail::fields::is_field_value(val)) { return false; }\n\n    if (case_ignore::equal(key, \"Location\") ||\n        case_ignore::equal(key, \"Referer\")) {\n      fn(key, val);\n    } else {\n      fn(key, decode_url(val, false));\n    }\n\n    return true;\n  }\n\n  return false;\n}\n\ninline bool read_headers(Stream &strm, Headers &headers) {\n  const auto bufsiz = 2048;\n  char buf[bufsiz];\n  stream_line_reader line_reader(strm, buf, bufsiz);\n\n  for (;;) {\n    if (!line_reader.getline()) { return false; }\n\n    // Check if the line ends with CRLF.\n    auto line_terminator_len = 2;\n    if (line_reader.end_with_crlf()) {\n      // Blank line indicates end of headers.\n      if (line_reader.size() == 2) { break; }\n    } else {\n#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR\n      // Blank line indicates end of headers.\n      if (line_reader.size() == 1) { break; }\n      line_terminator_len = 1;\n#else\n      continue; // Skip invalid line.\n#endif\n    }\n\n    if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }\n\n    // Exclude line terminator\n    auto end = line_reader.ptr() + line_reader.size() - line_terminator_len;\n\n    if (!parse_header(line_reader.ptr(), end,\n                      [&](const std::string &key, const std::string &val) {\n                        headers.emplace(key, val);\n                      })) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\ninline bool read_content_with_length(Stream &strm, uint64_t len,\n                                     Progress progress,\n                                     ContentReceiverWithProgress out) {\n  char buf[CPPHTTPLIB_RECV_BUFSIZ];\n\n  uint64_t r = 0;\n  while (r < len) {\n    auto read_len = static_cast<size_t>(len - r);\n    auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ));\n    if (n <= 0) { return false; }\n\n    if (!out(buf, static_cast<size_t>(n), r, len)) { return false; }\n    r += static_cast<uint64_t>(n);\n\n    if (progress) {\n      if (!progress(r, len)) { return false; }\n    }\n  }\n\n  return true;\n}\n\ninline void skip_content_with_length(Stream &strm, uint64_t len) {\n  char buf[CPPHTTPLIB_RECV_BUFSIZ];\n  uint64_t r = 0;\n  while (r < len) {\n    auto read_len = static_cast<size_t>(len - r);\n    auto n = strm.read(buf, (std::min)(read_len, CPPHTTPLIB_RECV_BUFSIZ));\n    if (n <= 0) { return; }\n    r += static_cast<uint64_t>(n);\n  }\n}\n\ninline bool read_content_without_length(Stream &strm,\n                                        ContentReceiverWithProgress out) {\n  char buf[CPPHTTPLIB_RECV_BUFSIZ];\n  uint64_t r = 0;\n  for (;;) {\n    auto n = strm.read(buf, CPPHTTPLIB_RECV_BUFSIZ);\n    if (n == 0) { return true; }\n    if (n < 0) { return false; }\n\n    if (!out(buf, static_cast<size_t>(n), r, 0)) { return false; }\n    r += static_cast<uint64_t>(n);\n  }\n\n  return true;\n}\n\ntemplate <typename T>\ninline bool read_content_chunked(Stream &strm, T &x,\n                                 ContentReceiverWithProgress out) {\n  const auto bufsiz = 16;\n  char buf[bufsiz];\n\n  stream_line_reader line_reader(strm, buf, bufsiz);\n\n  if (!line_reader.getline()) { return false; }\n\n  unsigned long chunk_len;\n  while (true) {\n    char *end_ptr;\n\n    chunk_len = std::strtoul(line_reader.ptr(), &end_ptr, 16);\n\n    if (end_ptr == line_reader.ptr()) { return false; }\n    if (chunk_len == ULONG_MAX) { return false; }\n\n    if (chunk_len == 0) { break; }\n\n    if (!read_content_with_length(strm, chunk_len, nullptr, out)) {\n      return false;\n    }\n\n    if (!line_reader.getline()) { return false; }\n\n    if (strcmp(line_reader.ptr(), \"\\r\\n\") != 0) { return false; }\n\n    if (!line_reader.getline()) { return false; }\n  }\n\n  assert(chunk_len == 0);\n\n  // NOTE: In RFC 9112, '7.1 Chunked Transfer Coding' mentions \"The chunked\n  // transfer coding is complete when a chunk with a chunk-size of zero is\n  // received, possibly followed by a trailer section, and finally terminated by\n  // an empty line\". https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1\n  //\n  // In '7.1.3. Decoding Chunked', however, the pseudo-code in the section\n  // does't care for the existence of the final CRLF. In other words, it seems\n  // to be ok whether the final CRLF exists or not in the chunked data.\n  // https://www.rfc-editor.org/rfc/rfc9112.html#section-7.1.3\n  //\n  // According to the reference code in RFC 9112, cpp-httplib now allows\n  // chunked transfer coding data without the final CRLF.\n  if (!line_reader.getline()) { return true; }\n\n  while (strcmp(line_reader.ptr(), \"\\r\\n\") != 0) {\n    if (line_reader.size() > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }\n\n    // Exclude line terminator\n    constexpr auto line_terminator_len = 2;\n    auto end = line_reader.ptr() + line_reader.size() - line_terminator_len;\n\n    parse_header(line_reader.ptr(), end,\n                 [&](const std::string &key, const std::string &val) {\n                   x.headers.emplace(key, val);\n                 });\n\n    if (!line_reader.getline()) { return false; }\n  }\n\n  return true;\n}\n\ninline bool is_chunked_transfer_encoding(const Headers &headers) {\n  return case_ignore::equal(\n      get_header_value(headers, \"Transfer-Encoding\", \"\", 0), \"chunked\");\n}\n\ntemplate <typename T, typename U>\nbool prepare_content_receiver(T &x, int &status,\n                              ContentReceiverWithProgress receiver,\n                              bool decompress, U callback) {\n  if (decompress) {\n    std::string encoding = x.get_header_value(\"Content-Encoding\");\n    std::unique_ptr<decompressor> decompressor;\n\n    if (encoding == \"gzip\" || encoding == \"deflate\") {\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n      decompressor = detail::make_unique<gzip_decompressor>();\n#else\n      status = StatusCode::UnsupportedMediaType_415;\n      return false;\n#endif\n    } else if (encoding.find(\"br\") != std::string::npos) {\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n      decompressor = detail::make_unique<brotli_decompressor>();\n#else\n      status = StatusCode::UnsupportedMediaType_415;\n      return false;\n#endif\n    } else if (encoding == \"zstd\") {\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n      decompressor = detail::make_unique<zstd_decompressor>();\n#else\n      status = StatusCode::UnsupportedMediaType_415;\n      return false;\n#endif\n    }\n\n    if (decompressor) {\n      if (decompressor->is_valid()) {\n        ContentReceiverWithProgress out = [&](const char *buf, size_t n,\n                                              uint64_t off, uint64_t len) {\n          return decompressor->decompress(buf, n,\n                                          [&](const char *buf2, size_t n2) {\n                                            return receiver(buf2, n2, off, len);\n                                          });\n        };\n        return callback(std::move(out));\n      } else {\n        status = StatusCode::InternalServerError_500;\n        return false;\n      }\n    }\n  }\n\n  ContentReceiverWithProgress out = [&](const char *buf, size_t n, uint64_t off,\n                                        uint64_t len) {\n    return receiver(buf, n, off, len);\n  };\n  return callback(std::move(out));\n}\n\ntemplate <typename T>\nbool read_content(Stream &strm, T &x, size_t payload_max_length, int &status,\n                  Progress progress, ContentReceiverWithProgress receiver,\n                  bool decompress) {\n  return prepare_content_receiver(\n      x, status, std::move(receiver), decompress,\n      [&](const ContentReceiverWithProgress &out) {\n        auto ret = true;\n        auto exceed_payload_max_length = false;\n\n        if (is_chunked_transfer_encoding(x.headers)) {\n          ret = read_content_chunked(strm, x, out);\n        } else if (!has_header(x.headers, \"Content-Length\")) {\n          ret = read_content_without_length(strm, out);\n        } else {\n          auto is_invalid_value = false;\n          auto len = get_header_value_u64(\n              x.headers, \"Content-Length\",\n              (std::numeric_limits<uint64_t>::max)(), 0, is_invalid_value);\n\n          if (is_invalid_value) {\n            ret = false;\n          } else if (len > payload_max_length) {\n            exceed_payload_max_length = true;\n            skip_content_with_length(strm, len);\n            ret = false;\n          } else if (len > 0) {\n            ret = read_content_with_length(strm, len, std::move(progress), out);\n          }\n        }\n\n        if (!ret) {\n          status = exceed_payload_max_length ? StatusCode::PayloadTooLarge_413\n                                             : StatusCode::BadRequest_400;\n        }\n        return ret;\n      });\n}\n\ninline ssize_t write_request_line(Stream &strm, const std::string &method,\n                                  const std::string &path) {\n  std::string s = method;\n  s += \" \";\n  s += path;\n  s += \" HTTP/1.1\\r\\n\";\n  return strm.write(s.data(), s.size());\n}\n\ninline ssize_t write_response_line(Stream &strm, int status) {\n  std::string s = \"HTTP/1.1 \";\n  s += std::to_string(status);\n  s += \" \";\n  s += httplib::status_message(status);\n  s += \"\\r\\n\";\n  return strm.write(s.data(), s.size());\n}\n\ninline ssize_t write_headers(Stream &strm, const Headers &headers) {\n  ssize_t write_len = 0;\n  for (const auto &x : headers) {\n    std::string s;\n    s = x.first;\n    s += \": \";\n    s += x.second;\n    s += \"\\r\\n\";\n\n    auto len = strm.write(s.data(), s.size());\n    if (len < 0) { return len; }\n    write_len += len;\n  }\n  auto len = strm.write(\"\\r\\n\");\n  if (len < 0) { return len; }\n  write_len += len;\n  return write_len;\n}\n\ninline bool write_data(Stream &strm, const char *d, size_t l) {\n  size_t offset = 0;\n  while (offset < l) {\n    auto length = strm.write(d + offset, l - offset);\n    if (length < 0) { return false; }\n    offset += static_cast<size_t>(length);\n  }\n  return true;\n}\n\ntemplate <typename T>\ninline bool write_content(Stream &strm, const ContentProvider &content_provider,\n                          size_t offset, size_t length, T is_shutting_down,\n                          Error &error) {\n  size_t end_offset = offset + length;\n  auto ok = true;\n  DataSink data_sink;\n\n  data_sink.write = [&](const char *d, size_t l) -> bool {\n    if (ok) {\n      if (write_data(strm, d, l)) {\n        offset += l;\n      } else {\n        ok = false;\n      }\n    }\n    return ok;\n  };\n\n  data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); };\n\n  while (offset < end_offset && !is_shutting_down()) {\n    if (!strm.wait_writable()) {\n      error = Error::Write;\n      return false;\n    } else if (!content_provider(offset, end_offset - offset, data_sink)) {\n      error = Error::Canceled;\n      return false;\n    } else if (!ok) {\n      error = Error::Write;\n      return false;\n    }\n  }\n\n  error = Error::Success;\n  return true;\n}\n\ntemplate <typename T>\ninline bool write_content(Stream &strm, const ContentProvider &content_provider,\n                          size_t offset, size_t length,\n                          const T &is_shutting_down) {\n  auto error = Error::Success;\n  return write_content(strm, content_provider, offset, length, is_shutting_down,\n                       error);\n}\n\ntemplate <typename T>\ninline bool\nwrite_content_without_length(Stream &strm,\n                             const ContentProvider &content_provider,\n                             const T &is_shutting_down) {\n  size_t offset = 0;\n  auto data_available = true;\n  auto ok = true;\n  DataSink data_sink;\n\n  data_sink.write = [&](const char *d, size_t l) -> bool {\n    if (ok) {\n      offset += l;\n      if (!write_data(strm, d, l)) { ok = false; }\n    }\n    return ok;\n  };\n\n  data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); };\n\n  data_sink.done = [&](void) { data_available = false; };\n\n  while (data_available && !is_shutting_down()) {\n    if (!strm.wait_writable()) {\n      return false;\n    } else if (!content_provider(offset, 0, data_sink)) {\n      return false;\n    } else if (!ok) {\n      return false;\n    }\n  }\n  return true;\n}\n\ntemplate <typename T, typename U>\ninline bool\nwrite_content_chunked(Stream &strm, const ContentProvider &content_provider,\n                      const T &is_shutting_down, U &compressor, Error &error) {\n  size_t offset = 0;\n  auto data_available = true;\n  auto ok = true;\n  DataSink data_sink;\n\n  data_sink.write = [&](const char *d, size_t l) -> bool {\n    if (ok) {\n      data_available = l > 0;\n      offset += l;\n\n      std::string payload;\n      if (compressor.compress(d, l, false,\n                              [&](const char *data, size_t data_len) {\n                                payload.append(data, data_len);\n                                return true;\n                              })) {\n        if (!payload.empty()) {\n          // Emit chunked response header and footer for each chunk\n          auto chunk =\n              from_i_to_hex(payload.size()) + \"\\r\\n\" + payload + \"\\r\\n\";\n          if (!write_data(strm, chunk.data(), chunk.size())) { ok = false; }\n        }\n      } else {\n        ok = false;\n      }\n    }\n    return ok;\n  };\n\n  data_sink.is_writable = [&]() -> bool { return strm.wait_writable(); };\n\n  auto done_with_trailer = [&](const Headers *trailer) {\n    if (!ok) { return; }\n\n    data_available = false;\n\n    std::string payload;\n    if (!compressor.compress(nullptr, 0, true,\n                             [&](const char *data, size_t data_len) {\n                               payload.append(data, data_len);\n                               return true;\n                             })) {\n      ok = false;\n      return;\n    }\n\n    if (!payload.empty()) {\n      // Emit chunked response header and footer for each chunk\n      auto chunk = from_i_to_hex(payload.size()) + \"\\r\\n\" + payload + \"\\r\\n\";\n      if (!write_data(strm, chunk.data(), chunk.size())) {\n        ok = false;\n        return;\n      }\n    }\n\n    constexpr const char done_marker[] = \"0\\r\\n\";\n    if (!write_data(strm, done_marker, str_len(done_marker))) { ok = false; }\n\n    // Trailer\n    if (trailer) {\n      for (const auto &kv : *trailer) {\n        std::string field_line = kv.first + \": \" + kv.second + \"\\r\\n\";\n        if (!write_data(strm, field_line.data(), field_line.size())) {\n          ok = false;\n        }\n      }\n    }\n\n    constexpr const char crlf[] = \"\\r\\n\";\n    if (!write_data(strm, crlf, str_len(crlf))) { ok = false; }\n  };\n\n  data_sink.done = [&](void) { done_with_trailer(nullptr); };\n\n  data_sink.done_with_trailer = [&](const Headers &trailer) {\n    done_with_trailer(&trailer);\n  };\n\n  while (data_available && !is_shutting_down()) {\n    if (!strm.wait_writable()) {\n      error = Error::Write;\n      return false;\n    } else if (!content_provider(offset, 0, data_sink)) {\n      error = Error::Canceled;\n      return false;\n    } else if (!ok) {\n      error = Error::Write;\n      return false;\n    }\n  }\n\n  error = Error::Success;\n  return true;\n}\n\ntemplate <typename T, typename U>\ninline bool write_content_chunked(Stream &strm,\n                                  const ContentProvider &content_provider,\n                                  const T &is_shutting_down, U &compressor) {\n  auto error = Error::Success;\n  return write_content_chunked(strm, content_provider, is_shutting_down,\n                               compressor, error);\n}\n\ntemplate <typename T>\ninline bool redirect(T &cli, Request &req, Response &res,\n                     const std::string &path, const std::string &location,\n                     Error &error) {\n  Request new_req = req;\n  new_req.path = path;\n  new_req.redirect_count_ -= 1;\n\n  if (res.status == StatusCode::SeeOther_303 &&\n      (req.method != \"GET\" && req.method != \"HEAD\")) {\n    new_req.method = \"GET\";\n    new_req.body.clear();\n    new_req.headers.clear();\n  }\n\n  Response new_res;\n\n  auto ret = cli.send(new_req, new_res, error);\n  if (ret) {\n    req = new_req;\n    res = new_res;\n\n    if (res.location.empty()) { res.location = location; }\n  }\n  return ret;\n}\n\ninline std::string params_to_query_str(const Params &params) {\n  std::string query;\n\n  for (auto it = params.begin(); it != params.end(); ++it) {\n    if (it != params.begin()) { query += \"&\"; }\n    query += it->first;\n    query += \"=\";\n    query += encode_query_param(it->second);\n  }\n  return query;\n}\n\ninline void parse_query_text(const char *data, std::size_t size,\n                             Params &params) {\n  std::set<std::string> cache;\n  split(data, data + size, '&', [&](const char *b, const char *e) {\n    std::string kv(b, e);\n    if (cache.find(kv) != cache.end()) { return; }\n    cache.insert(std::move(kv));\n\n    std::string key;\n    std::string val;\n    divide(b, static_cast<std::size_t>(e - b), '=',\n           [&](const char *lhs_data, std::size_t lhs_size, const char *rhs_data,\n               std::size_t rhs_size) {\n             key.assign(lhs_data, lhs_size);\n             val.assign(rhs_data, rhs_size);\n           });\n\n    if (!key.empty()) {\n      params.emplace(decode_url(key, true), decode_url(val, true));\n    }\n  });\n}\n\ninline void parse_query_text(const std::string &s, Params &params) {\n  parse_query_text(s.data(), s.size(), params);\n}\n\ninline bool parse_multipart_boundary(const std::string &content_type,\n                                     std::string &boundary) {\n  auto boundary_keyword = \"boundary=\";\n  auto pos = content_type.find(boundary_keyword);\n  if (pos == std::string::npos) { return false; }\n  auto end = content_type.find(';', pos);\n  auto beg = pos + strlen(boundary_keyword);\n  boundary = trim_double_quotes_copy(content_type.substr(beg, end - beg));\n  return !boundary.empty();\n}\n\ninline void parse_disposition_params(const std::string &s, Params &params) {\n  std::set<std::string> cache;\n  split(s.data(), s.data() + s.size(), ';', [&](const char *b, const char *e) {\n    std::string kv(b, e);\n    if (cache.find(kv) != cache.end()) { return; }\n    cache.insert(kv);\n\n    std::string key;\n    std::string val;\n    split(b, e, '=', [&](const char *b2, const char *e2) {\n      if (key.empty()) {\n        key.assign(b2, e2);\n      } else {\n        val.assign(b2, e2);\n      }\n    });\n\n    if (!key.empty()) {\n      params.emplace(trim_double_quotes_copy((key)),\n                     trim_double_quotes_copy((val)));\n    }\n  });\n}\n\n#ifdef CPPHTTPLIB_NO_EXCEPTIONS\ninline bool parse_range_header(const std::string &s, Ranges &ranges) {\n#else\ninline bool parse_range_header(const std::string &s, Ranges &ranges) try {\n#endif\n  auto is_valid = [](const std::string &str) {\n    return std::all_of(str.cbegin(), str.cend(),\n                       [](unsigned char c) { return std::isdigit(c); });\n  };\n\n  if (s.size() > 7 && s.compare(0, 6, \"bytes=\") == 0) {\n    const auto pos = static_cast<size_t>(6);\n    const auto len = static_cast<size_t>(s.size() - 6);\n    auto all_valid_ranges = true;\n    split(&s[pos], &s[pos + len], ',', [&](const char *b, const char *e) {\n      if (!all_valid_ranges) { return; }\n\n      const auto it = std::find(b, e, '-');\n      if (it == e) {\n        all_valid_ranges = false;\n        return;\n      }\n\n      const auto lhs = std::string(b, it);\n      const auto rhs = std::string(it + 1, e);\n      if (!is_valid(lhs) || !is_valid(rhs)) {\n        all_valid_ranges = false;\n        return;\n      }\n\n      const auto first =\n          static_cast<ssize_t>(lhs.empty() ? -1 : std::stoll(lhs));\n      const auto last =\n          static_cast<ssize_t>(rhs.empty() ? -1 : std::stoll(rhs));\n      if ((first == -1 && last == -1) ||\n          (first != -1 && last != -1 && first > last)) {\n        all_valid_ranges = false;\n        return;\n      }\n\n      ranges.emplace_back(first, last);\n    });\n    return all_valid_ranges && !ranges.empty();\n  }\n  return false;\n#ifdef CPPHTTPLIB_NO_EXCEPTIONS\n}\n#else\n} catch (...) { return false; }\n#endif\n\nclass MultipartFormDataParser {\npublic:\n  MultipartFormDataParser() = default;\n\n  void set_boundary(std::string &&boundary) {\n    boundary_ = boundary;\n    dash_boundary_crlf_ = dash_ + boundary_ + crlf_;\n    crlf_dash_boundary_ = crlf_ + dash_ + boundary_;\n  }\n\n  bool is_valid() const { return is_valid_; }\n\n  bool parse(const char *buf, size_t n, const ContentReceiver &content_callback,\n             const MultipartContentHeader &header_callback) {\n\n    buf_append(buf, n);\n\n    while (buf_size() > 0) {\n      switch (state_) {\n      case 0: { // Initial boundary\n        buf_erase(buf_find(dash_boundary_crlf_));\n        if (dash_boundary_crlf_.size() > buf_size()) { return true; }\n        if (!buf_start_with(dash_boundary_crlf_)) { return false; }\n        buf_erase(dash_boundary_crlf_.size());\n        state_ = 1;\n        break;\n      }\n      case 1: { // New entry\n        clear_file_info();\n        state_ = 2;\n        break;\n      }\n      case 2: { // Headers\n        auto pos = buf_find(crlf_);\n        if (pos > CPPHTTPLIB_HEADER_MAX_LENGTH) { return false; }\n        while (pos < buf_size()) {\n          // Empty line\n          if (pos == 0) {\n            if (!header_callback(file_)) {\n              is_valid_ = false;\n              return false;\n            }\n            buf_erase(crlf_.size());\n            state_ = 3;\n            break;\n          }\n\n          const auto header = buf_head(pos);\n\n          if (!parse_header(header.data(), header.data() + header.size(),\n                            [&](const std::string &, const std::string &) {})) {\n            is_valid_ = false;\n            return false;\n          }\n\n          constexpr const char header_content_type[] = \"Content-Type:\";\n\n          if (start_with_case_ignore(header, header_content_type)) {\n            file_.content_type =\n                trim_copy(header.substr(str_len(header_content_type)));\n          } else {\n            thread_local const std::regex re_content_disposition(\n                R\"~(^Content-Disposition:\\s*form-data;\\s*(.*)$)~\",\n                std::regex_constants::icase);\n\n            std::smatch m;\n            if (std::regex_match(header, m, re_content_disposition)) {\n              Params params;\n              parse_disposition_params(m[1], params);\n\n              auto it = params.find(\"name\");\n              if (it != params.end()) {\n                file_.name = it->second;\n              } else {\n                is_valid_ = false;\n                return false;\n              }\n\n              it = params.find(\"filename\");\n              if (it != params.end()) { file_.filename = it->second; }\n\n              it = params.find(\"filename*\");\n              if (it != params.end()) {\n                // Only allow UTF-8 encoding...\n                thread_local const std::regex re_rfc5987_encoding(\n                    R\"~(^UTF-8''(.+?)$)~\", std::regex_constants::icase);\n\n                std::smatch m2;\n                if (std::regex_match(it->second, m2, re_rfc5987_encoding)) {\n                  file_.filename = decode_url(m2[1], false); // override...\n                } else {\n                  is_valid_ = false;\n                  return false;\n                }\n              }\n            }\n          }\n          buf_erase(pos + crlf_.size());\n          pos = buf_find(crlf_);\n        }\n        if (state_ != 3) { return true; }\n        break;\n      }\n      case 3: { // Body\n        if (crlf_dash_boundary_.size() > buf_size()) { return true; }\n        auto pos = buf_find(crlf_dash_boundary_);\n        if (pos < buf_size()) {\n          if (!content_callback(buf_data(), pos)) {\n            is_valid_ = false;\n            return false;\n          }\n          buf_erase(pos + crlf_dash_boundary_.size());\n          state_ = 4;\n        } else {\n          auto len = buf_size() - crlf_dash_boundary_.size();\n          if (len > 0) {\n            if (!content_callback(buf_data(), len)) {\n              is_valid_ = false;\n              return false;\n            }\n            buf_erase(len);\n          }\n          return true;\n        }\n        break;\n      }\n      case 4: { // Boundary\n        if (crlf_.size() > buf_size()) { return true; }\n        if (buf_start_with(crlf_)) {\n          buf_erase(crlf_.size());\n          state_ = 1;\n        } else {\n          if (dash_.size() > buf_size()) { return true; }\n          if (buf_start_with(dash_)) {\n            buf_erase(dash_.size());\n            is_valid_ = true;\n            buf_erase(buf_size()); // Remove epilogue\n          } else {\n            return true;\n          }\n        }\n        break;\n      }\n      }\n    }\n\n    return true;\n  }\n\nprivate:\n  void clear_file_info() {\n    file_.name.clear();\n    file_.filename.clear();\n    file_.content_type.clear();\n  }\n\n  bool start_with_case_ignore(const std::string &a, const char *b) const {\n    const auto b_len = strlen(b);\n    if (a.size() < b_len) { return false; }\n    for (size_t i = 0; i < b_len; i++) {\n      if (case_ignore::to_lower(a[i]) != case_ignore::to_lower(b[i])) {\n        return false;\n      }\n    }\n    return true;\n  }\n\n  const std::string dash_ = \"--\";\n  const std::string crlf_ = \"\\r\\n\";\n  std::string boundary_;\n  std::string dash_boundary_crlf_;\n  std::string crlf_dash_boundary_;\n\n  size_t state_ = 0;\n  bool is_valid_ = false;\n  MultipartFormData file_;\n\n  // Buffer\n  bool start_with(const std::string &a, size_t spos, size_t epos,\n                  const std::string &b) const {\n    if (epos - spos < b.size()) { return false; }\n    for (size_t i = 0; i < b.size(); i++) {\n      if (a[i + spos] != b[i]) { return false; }\n    }\n    return true;\n  }\n\n  size_t buf_size() const { return buf_epos_ - buf_spos_; }\n\n  const char *buf_data() const { return &buf_[buf_spos_]; }\n\n  std::string buf_head(size_t l) const { return buf_.substr(buf_spos_, l); }\n\n  bool buf_start_with(const std::string &s) const {\n    return start_with(buf_, buf_spos_, buf_epos_, s);\n  }\n\n  size_t buf_find(const std::string &s) const {\n    auto c = s.front();\n\n    size_t off = buf_spos_;\n    while (off < buf_epos_) {\n      auto pos = off;\n      while (true) {\n        if (pos == buf_epos_) { return buf_size(); }\n        if (buf_[pos] == c) { break; }\n        pos++;\n      }\n\n      auto remaining_size = buf_epos_ - pos;\n      if (s.size() > remaining_size) { return buf_size(); }\n\n      if (start_with(buf_, pos, buf_epos_, s)) { return pos - buf_spos_; }\n\n      off = pos + 1;\n    }\n\n    return buf_size();\n  }\n\n  void buf_append(const char *data, size_t n) {\n    auto remaining_size = buf_size();\n    if (remaining_size > 0 && buf_spos_ > 0) {\n      for (size_t i = 0; i < remaining_size; i++) {\n        buf_[i] = buf_[buf_spos_ + i];\n      }\n    }\n    buf_spos_ = 0;\n    buf_epos_ = remaining_size;\n\n    if (remaining_size + n > buf_.size()) { buf_.resize(remaining_size + n); }\n\n    for (size_t i = 0; i < n; i++) {\n      buf_[buf_epos_ + i] = data[i];\n    }\n    buf_epos_ += n;\n  }\n\n  void buf_erase(size_t size) { buf_spos_ += size; }\n\n  std::string buf_;\n  size_t buf_spos_ = 0;\n  size_t buf_epos_ = 0;\n};\n\ninline std::string random_string(size_t length) {\n  constexpr const char data[] =\n      \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\";\n\n  thread_local auto engine([]() {\n    // std::random_device might actually be deterministic on some\n    // platforms, but due to lack of support in the c++ standard library,\n    // doing better requires either some ugly hacks or breaking portability.\n    std::random_device seed_gen;\n    // Request 128 bits of entropy for initialization\n    std::seed_seq seed_sequence{seed_gen(), seed_gen(), seed_gen(), seed_gen()};\n    return std::mt19937(seed_sequence);\n  }());\n\n  std::string result;\n  for (size_t i = 0; i < length; i++) {\n    result += data[engine() % (sizeof(data) - 1)];\n  }\n  return result;\n}\n\ninline std::string make_multipart_data_boundary() {\n  return \"--cpp-httplib-multipart-data-\" + detail::random_string(16);\n}\n\ninline bool is_multipart_boundary_chars_valid(const std::string &boundary) {\n  auto valid = true;\n  for (size_t i = 0; i < boundary.size(); i++) {\n    auto c = boundary[i];\n    if (!std::isalnum(c) && c != '-' && c != '_') {\n      valid = false;\n      break;\n    }\n  }\n  return valid;\n}\n\ntemplate <typename T>\ninline std::string\nserialize_multipart_formdata_item_begin(const T &item,\n                                        const std::string &boundary) {\n  std::string body = \"--\" + boundary + \"\\r\\n\";\n  body += \"Content-Disposition: form-data; name=\\\"\" + item.name + \"\\\"\";\n  if (!item.filename.empty()) {\n    body += \"; filename=\\\"\" + item.filename + \"\\\"\";\n  }\n  body += \"\\r\\n\";\n  if (!item.content_type.empty()) {\n    body += \"Content-Type: \" + item.content_type + \"\\r\\n\";\n  }\n  body += \"\\r\\n\";\n\n  return body;\n}\n\ninline std::string serialize_multipart_formdata_item_end() { return \"\\r\\n\"; }\n\ninline std::string\nserialize_multipart_formdata_finish(const std::string &boundary) {\n  return \"--\" + boundary + \"--\\r\\n\";\n}\n\ninline std::string\nserialize_multipart_formdata_get_content_type(const std::string &boundary) {\n  return \"multipart/form-data; boundary=\" + boundary;\n}\n\ninline std::string\nserialize_multipart_formdata(const MultipartFormDataItems &items,\n                             const std::string &boundary, bool finish = true) {\n  std::string body;\n\n  for (const auto &item : items) {\n    body += serialize_multipart_formdata_item_begin(item, boundary);\n    body += item.content + serialize_multipart_formdata_item_end();\n  }\n\n  if (finish) { body += serialize_multipart_formdata_finish(boundary); }\n\n  return body;\n}\n\ninline bool range_error(Request &req, Response &res) {\n  if (!req.ranges.empty() && 200 <= res.status && res.status < 300) {\n    ssize_t content_len = static_cast<ssize_t>(\n        res.content_length_ ? res.content_length_ : res.body.size());\n\n    ssize_t prev_first_pos = -1;\n    ssize_t prev_last_pos = -1;\n    size_t overwrapping_count = 0;\n\n    // NOTE: The following Range check is based on '14.2. Range' in RFC 9110\n    // 'HTTP Semantics' to avoid potential denial-of-service attacks.\n    // https://www.rfc-editor.org/rfc/rfc9110#section-14.2\n\n    // Too many ranges\n    if (req.ranges.size() > CPPHTTPLIB_RANGE_MAX_COUNT) { return true; }\n\n    for (auto &r : req.ranges) {\n      auto &first_pos = r.first;\n      auto &last_pos = r.second;\n\n      if (first_pos == -1 && last_pos == -1) {\n        first_pos = 0;\n        last_pos = content_len;\n      }\n\n      if (first_pos == -1) {\n        first_pos = content_len - last_pos;\n        last_pos = content_len - 1;\n      }\n\n      // NOTE: RFC-9110 '14.1.2. Byte Ranges':\n      // A client can limit the number of bytes requested without knowing the\n      // size of the selected representation. If the last-pos value is absent,\n      // or if the value is greater than or equal to the current length of the\n      // representation data, the byte range is interpreted as the remainder of\n      // the representation (i.e., the server replaces the value of last-pos\n      // with a value that is one less than the current length of the selected\n      // representation).\n      // https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2-6\n      if (last_pos == -1 || last_pos >= content_len) {\n        last_pos = content_len - 1;\n      }\n\n      // Range must be within content length\n      if (!(0 <= first_pos && first_pos <= last_pos &&\n            last_pos <= content_len - 1)) {\n        return true;\n      }\n\n      // Ranges must be in ascending order\n      if (first_pos <= prev_first_pos) { return true; }\n\n      // Request must not have more than two overlapping ranges\n      if (first_pos <= prev_last_pos) {\n        overwrapping_count++;\n        if (overwrapping_count > 2) { return true; }\n      }\n\n      prev_first_pos = (std::max)(prev_first_pos, first_pos);\n      prev_last_pos = (std::max)(prev_last_pos, last_pos);\n    }\n  }\n\n  return false;\n}\n\ninline std::pair<size_t, size_t>\nget_range_offset_and_length(Range r, size_t content_length) {\n  assert(r.first != -1 && r.second != -1);\n  assert(0 <= r.first && r.first < static_cast<ssize_t>(content_length));\n  assert(r.first <= r.second &&\n         r.second < static_cast<ssize_t>(content_length));\n  (void)(content_length);\n  return std::make_pair(r.first, static_cast<size_t>(r.second - r.first) + 1);\n}\n\ninline std::string make_content_range_header_field(\n    const std::pair<size_t, size_t> &offset_and_length, size_t content_length) {\n  auto st = offset_and_length.first;\n  auto ed = st + offset_and_length.second - 1;\n\n  std::string field = \"bytes \";\n  field += std::to_string(st);\n  field += \"-\";\n  field += std::to_string(ed);\n  field += \"/\";\n  field += std::to_string(content_length);\n  return field;\n}\n\ntemplate <typename SToken, typename CToken, typename Content>\nbool process_multipart_ranges_data(const Request &req,\n                                   const std::string &boundary,\n                                   const std::string &content_type,\n                                   size_t content_length, SToken stoken,\n                                   CToken ctoken, Content content) {\n  for (size_t i = 0; i < req.ranges.size(); i++) {\n    ctoken(\"--\");\n    stoken(boundary);\n    ctoken(\"\\r\\n\");\n    if (!content_type.empty()) {\n      ctoken(\"Content-Type: \");\n      stoken(content_type);\n      ctoken(\"\\r\\n\");\n    }\n\n    auto offset_and_length =\n        get_range_offset_and_length(req.ranges[i], content_length);\n\n    ctoken(\"Content-Range: \");\n    stoken(make_content_range_header_field(offset_and_length, content_length));\n    ctoken(\"\\r\\n\");\n    ctoken(\"\\r\\n\");\n\n    if (!content(offset_and_length.first, offset_and_length.second)) {\n      return false;\n    }\n    ctoken(\"\\r\\n\");\n  }\n\n  ctoken(\"--\");\n  stoken(boundary);\n  ctoken(\"--\");\n\n  return true;\n}\n\ninline void make_multipart_ranges_data(const Request &req, Response &res,\n                                       const std::string &boundary,\n                                       const std::string &content_type,\n                                       size_t content_length,\n                                       std::string &data) {\n  process_multipart_ranges_data(\n      req, boundary, content_type, content_length,\n      [&](const std::string &token) { data += token; },\n      [&](const std::string &token) { data += token; },\n      [&](size_t offset, size_t length) {\n        assert(offset + length <= content_length);\n        data += res.body.substr(offset, length);\n        return true;\n      });\n}\n\ninline size_t get_multipart_ranges_data_length(const Request &req,\n                                               const std::string &boundary,\n                                               const std::string &content_type,\n                                               size_t content_length) {\n  size_t data_length = 0;\n\n  process_multipart_ranges_data(\n      req, boundary, content_type, content_length,\n      [&](const std::string &token) { data_length += token.size(); },\n      [&](const std::string &token) { data_length += token.size(); },\n      [&](size_t /*offset*/, size_t length) {\n        data_length += length;\n        return true;\n      });\n\n  return data_length;\n}\n\ntemplate <typename T>\ninline bool\nwrite_multipart_ranges_data(Stream &strm, const Request &req, Response &res,\n                            const std::string &boundary,\n                            const std::string &content_type,\n                            size_t content_length, const T &is_shutting_down) {\n  return process_multipart_ranges_data(\n      req, boundary, content_type, content_length,\n      [&](const std::string &token) { strm.write(token); },\n      [&](const std::string &token) { strm.write(token); },\n      [&](size_t offset, size_t length) {\n        return write_content(strm, res.content_provider_, offset, length,\n                             is_shutting_down);\n      });\n}\n\ninline bool expect_content(const Request &req) {\n  if (req.method == \"POST\" || req.method == \"PUT\" || req.method == \"PATCH\" ||\n      req.method == \"DELETE\") {\n    return true;\n  }\n  if (req.has_header(\"Content-Length\") &&\n      req.get_header_value_u64(\"Content-Length\") > 0) {\n    return true;\n  }\n  if (is_chunked_transfer_encoding(req.headers)) { return true; }\n  return false;\n}\n\ninline bool has_crlf(const std::string &s) {\n  auto p = s.c_str();\n  while (*p) {\n    if (*p == '\\r' || *p == '\\n') { return true; }\n    p++;\n  }\n  return false;\n}\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline std::string message_digest(const std::string &s, const EVP_MD *algo) {\n  auto context = std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)>(\n      EVP_MD_CTX_new(), EVP_MD_CTX_free);\n\n  unsigned int hash_length = 0;\n  unsigned char hash[EVP_MAX_MD_SIZE];\n\n  EVP_DigestInit_ex(context.get(), algo, nullptr);\n  EVP_DigestUpdate(context.get(), s.c_str(), s.size());\n  EVP_DigestFinal_ex(context.get(), hash, &hash_length);\n\n  std::stringstream ss;\n  for (auto i = 0u; i < hash_length; ++i) {\n    ss << std::hex << std::setw(2) << std::setfill('0')\n       << static_cast<unsigned int>(hash[i]);\n  }\n\n  return ss.str();\n}\n\ninline std::string MD5(const std::string &s) {\n  return message_digest(s, EVP_md5());\n}\n\ninline std::string SHA_256(const std::string &s) {\n  return message_digest(s, EVP_sha256());\n}\n\ninline std::string SHA_512(const std::string &s) {\n  return message_digest(s, EVP_sha512());\n}\n\ninline std::pair<std::string, std::string> make_digest_authentication_header(\n    const Request &req, const std::map<std::string, std::string> &auth,\n    size_t cnonce_count, const std::string &cnonce, const std::string &username,\n    const std::string &password, bool is_proxy = false) {\n  std::string nc;\n  {\n    std::stringstream ss;\n    ss << std::setfill('0') << std::setw(8) << std::hex << cnonce_count;\n    nc = ss.str();\n  }\n\n  std::string qop;\n  if (auth.find(\"qop\") != auth.end()) {\n    qop = auth.at(\"qop\");\n    if (qop.find(\"auth-int\") != std::string::npos) {\n      qop = \"auth-int\";\n    } else if (qop.find(\"auth\") != std::string::npos) {\n      qop = \"auth\";\n    } else {\n      qop.clear();\n    }\n  }\n\n  std::string algo = \"MD5\";\n  if (auth.find(\"algorithm\") != auth.end()) { algo = auth.at(\"algorithm\"); }\n\n  std::string response;\n  {\n    auto H = algo == \"SHA-256\"   ? detail::SHA_256\n             : algo == \"SHA-512\" ? detail::SHA_512\n                                 : detail::MD5;\n\n    auto A1 = username + \":\" + auth.at(\"realm\") + \":\" + password;\n\n    auto A2 = req.method + \":\" + req.path;\n    if (qop == \"auth-int\") { A2 += \":\" + H(req.body); }\n\n    if (qop.empty()) {\n      response = H(H(A1) + \":\" + auth.at(\"nonce\") + \":\" + H(A2));\n    } else {\n      response = H(H(A1) + \":\" + auth.at(\"nonce\") + \":\" + nc + \":\" + cnonce +\n                   \":\" + qop + \":\" + H(A2));\n    }\n  }\n\n  auto opaque = (auth.find(\"opaque\") != auth.end()) ? auth.at(\"opaque\") : \"\";\n\n  auto field = \"Digest username=\\\"\" + username + \"\\\", realm=\\\"\" +\n               auth.at(\"realm\") + \"\\\", nonce=\\\"\" + auth.at(\"nonce\") +\n               \"\\\", uri=\\\"\" + req.path + \"\\\", algorithm=\" + algo +\n               (qop.empty() ? \", response=\\\"\"\n                            : \", qop=\" + qop + \", nc=\" + nc + \", cnonce=\\\"\" +\n                                  cnonce + \"\\\", response=\\\"\") +\n               response + \"\\\"\" +\n               (opaque.empty() ? \"\" : \", opaque=\\\"\" + opaque + \"\\\"\");\n\n  auto key = is_proxy ? \"Proxy-Authorization\" : \"Authorization\";\n  return std::make_pair(key, field);\n}\n\ninline bool is_ssl_peer_could_be_closed(SSL *ssl, socket_t sock) {\n  detail::set_nonblocking(sock, true);\n  auto se = detail::scope_exit([&]() { detail::set_nonblocking(sock, false); });\n\n  char buf[1];\n  return !SSL_peek(ssl, buf, 1) &&\n         SSL_get_error(ssl, 0) == SSL_ERROR_ZERO_RETURN;\n}\n\n#ifdef _WIN32\n// NOTE: This code came up with the following stackoverflow post:\n// https://stackoverflow.com/questions/9507184/can-openssl-on-windows-use-the-system-certificate-store\ninline bool load_system_certs_on_windows(X509_STORE *store) {\n  auto hStore = CertOpenSystemStoreW((HCRYPTPROV_LEGACY)NULL, L\"ROOT\");\n  if (!hStore) { return false; }\n\n  auto result = false;\n  PCCERT_CONTEXT pContext = NULL;\n  while ((pContext = CertEnumCertificatesInStore(hStore, pContext)) !=\n         nullptr) {\n    auto encoded_cert =\n        static_cast<const unsigned char *>(pContext->pbCertEncoded);\n\n    auto x509 = d2i_X509(NULL, &encoded_cert, pContext->cbCertEncoded);\n    if (x509) {\n      X509_STORE_add_cert(store, x509);\n      X509_free(x509);\n      result = true;\n    }\n  }\n\n  CertFreeCertificateContext(pContext);\n  CertCloseStore(hStore, 0);\n\n  return result;\n}\n#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)\n#if TARGET_OS_OSX\ntemplate <typename T>\nusing CFObjectPtr =\n    std::unique_ptr<typename std::remove_pointer<T>::type, void (*)(CFTypeRef)>;\n\ninline void cf_object_ptr_deleter(CFTypeRef obj) {\n  if (obj) { CFRelease(obj); }\n}\n\ninline bool retrieve_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {\n  CFStringRef keys[] = {kSecClass, kSecMatchLimit, kSecReturnRef};\n  CFTypeRef values[] = {kSecClassCertificate, kSecMatchLimitAll,\n                        kCFBooleanTrue};\n\n  CFObjectPtr<CFDictionaryRef> query(\n      CFDictionaryCreate(nullptr, reinterpret_cast<const void **>(keys), values,\n                         sizeof(keys) / sizeof(keys[0]),\n                         &kCFTypeDictionaryKeyCallBacks,\n                         &kCFTypeDictionaryValueCallBacks),\n      cf_object_ptr_deleter);\n\n  if (!query) { return false; }\n\n  CFTypeRef security_items = nullptr;\n  if (SecItemCopyMatching(query.get(), &security_items) != errSecSuccess ||\n      CFArrayGetTypeID() != CFGetTypeID(security_items)) {\n    return false;\n  }\n\n  certs.reset(reinterpret_cast<CFArrayRef>(security_items));\n  return true;\n}\n\ninline bool retrieve_root_certs_from_keychain(CFObjectPtr<CFArrayRef> &certs) {\n  CFArrayRef root_security_items = nullptr;\n  if (SecTrustCopyAnchorCertificates(&root_security_items) != errSecSuccess) {\n    return false;\n  }\n\n  certs.reset(root_security_items);\n  return true;\n}\n\ninline bool add_certs_to_x509_store(CFArrayRef certs, X509_STORE *store) {\n  auto result = false;\n  for (auto i = 0; i < CFArrayGetCount(certs); ++i) {\n    const auto cert = reinterpret_cast<const __SecCertificate *>(\n        CFArrayGetValueAtIndex(certs, i));\n\n    if (SecCertificateGetTypeID() != CFGetTypeID(cert)) { continue; }\n\n    CFDataRef cert_data = nullptr;\n    if (SecItemExport(cert, kSecFormatX509Cert, 0, nullptr, &cert_data) !=\n        errSecSuccess) {\n      continue;\n    }\n\n    CFObjectPtr<CFDataRef> cert_data_ptr(cert_data, cf_object_ptr_deleter);\n\n    auto encoded_cert = static_cast<const unsigned char *>(\n        CFDataGetBytePtr(cert_data_ptr.get()));\n\n    auto x509 =\n        d2i_X509(NULL, &encoded_cert, CFDataGetLength(cert_data_ptr.get()));\n\n    if (x509) {\n      X509_STORE_add_cert(store, x509);\n      X509_free(x509);\n      result = true;\n    }\n  }\n\n  return result;\n}\n\ninline bool load_system_certs_on_macos(X509_STORE *store) {\n  auto result = false;\n  CFObjectPtr<CFArrayRef> certs(nullptr, cf_object_ptr_deleter);\n  if (retrieve_certs_from_keychain(certs) && certs) {\n    result = add_certs_to_x509_store(certs.get(), store);\n  }\n\n  if (retrieve_root_certs_from_keychain(certs) && certs) {\n    result = add_certs_to_x509_store(certs.get(), store) || result;\n  }\n\n  return result;\n}\n#endif // TARGET_OS_OSX\n#endif // _WIN32\n#endif // CPPHTTPLIB_OPENSSL_SUPPORT\n\n#ifdef _WIN32\nclass WSInit {\npublic:\n  WSInit() {\n    WSADATA wsaData;\n    if (WSAStartup(0x0002, &wsaData) == 0) is_valid_ = true;\n  }\n\n  ~WSInit() {\n    if (is_valid_) WSACleanup();\n  }\n\n  bool is_valid_ = false;\n};\n\nstatic WSInit wsinit_;\n#endif\n\ninline bool parse_www_authenticate(const Response &res,\n                                   std::map<std::string, std::string> &auth,\n                                   bool is_proxy) {\n  auto auth_key = is_proxy ? \"Proxy-Authenticate\" : \"WWW-Authenticate\";\n  if (res.has_header(auth_key)) {\n    thread_local auto re =\n        std::regex(R\"~((?:(?:,\\s*)?(.+?)=(?:\"(.*?)\"|([^,]*))))~\");\n    auto s = res.get_header_value(auth_key);\n    auto pos = s.find(' ');\n    if (pos != std::string::npos) {\n      auto type = s.substr(0, pos);\n      if (type == \"Basic\") {\n        return false;\n      } else if (type == \"Digest\") {\n        s = s.substr(pos + 1);\n        auto beg = std::sregex_iterator(s.begin(), s.end(), re);\n        for (auto i = beg; i != std::sregex_iterator(); ++i) {\n          const auto &m = *i;\n          auto key = s.substr(static_cast<size_t>(m.position(1)),\n                              static_cast<size_t>(m.length(1)));\n          auto val = m.length(2) > 0\n                         ? s.substr(static_cast<size_t>(m.position(2)),\n                                    static_cast<size_t>(m.length(2)))\n                         : s.substr(static_cast<size_t>(m.position(3)),\n                                    static_cast<size_t>(m.length(3)));\n          auth[key] = val;\n        }\n        return true;\n      }\n    }\n  }\n  return false;\n}\n\nclass ContentProviderAdapter {\npublic:\n  explicit ContentProviderAdapter(\n      ContentProviderWithoutLength &&content_provider)\n      : content_provider_(content_provider) {}\n\n  bool operator()(size_t offset, size_t, DataSink &sink) {\n    return content_provider_(offset, sink);\n  }\n\nprivate:\n  ContentProviderWithoutLength content_provider_;\n};\n\n} // namespace detail\n\ninline std::string hosted_at(const std::string &hostname) {\n  std::vector<std::string> addrs;\n  hosted_at(hostname, addrs);\n  if (addrs.empty()) { return std::string(); }\n  return addrs[0];\n}\n\ninline void hosted_at(const std::string &hostname,\n                      std::vector<std::string> &addrs) {\n  struct addrinfo hints;\n  struct addrinfo *result;\n\n  memset(&hints, 0, sizeof(struct addrinfo));\n  hints.ai_family = AF_UNSPEC;\n  hints.ai_socktype = SOCK_STREAM;\n  hints.ai_protocol = 0;\n\n  if (getaddrinfo(hostname.c_str(), nullptr, &hints, &result)) {\n#if defined __linux__ && !defined __ANDROID__\n    res_init();\n#endif\n    return;\n  }\n  auto se = detail::scope_exit([&] { freeaddrinfo(result); });\n\n  for (auto rp = result; rp; rp = rp->ai_next) {\n    const auto &addr =\n        *reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);\n    std::string ip;\n    auto dummy = -1;\n    if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,\n                                dummy)) {\n      addrs.push_back(ip);\n    }\n  }\n}\n\ninline std::string append_query_params(const std::string &path,\n                                       const Params &params) {\n  std::string path_with_query = path;\n  thread_local const std::regex re(\"[^?]+\\\\?.*\");\n  auto delm = std::regex_match(path, re) ? '&' : '?';\n  path_with_query += delm + detail::params_to_query_str(params);\n  return path_with_query;\n}\n\n// Header utilities\ninline std::pair<std::string, std::string>\nmake_range_header(const Ranges &ranges) {\n  std::string field = \"bytes=\";\n  auto i = 0;\n  for (const auto &r : ranges) {\n    if (i != 0) { field += \", \"; }\n    if (r.first != -1) { field += std::to_string(r.first); }\n    field += '-';\n    if (r.second != -1) { field += std::to_string(r.second); }\n    i++;\n  }\n  return std::make_pair(\"Range\", std::move(field));\n}\n\ninline std::pair<std::string, std::string>\nmake_basic_authentication_header(const std::string &username,\n                                 const std::string &password, bool is_proxy) {\n  auto field = \"Basic \" + detail::base64_encode(username + \":\" + password);\n  auto key = is_proxy ? \"Proxy-Authorization\" : \"Authorization\";\n  return std::make_pair(key, std::move(field));\n}\n\ninline std::pair<std::string, std::string>\nmake_bearer_token_authentication_header(const std::string &token,\n                                        bool is_proxy = false) {\n  auto field = \"Bearer \" + token;\n  auto key = is_proxy ? \"Proxy-Authorization\" : \"Authorization\";\n  return std::make_pair(key, std::move(field));\n}\n\n// Request implementation\ninline bool Request::has_header(const std::string &key) const {\n  return detail::has_header(headers, key);\n}\n\ninline std::string Request::get_header_value(const std::string &key,\n                                             const char *def, size_t id) const {\n  return detail::get_header_value(headers, key, def, id);\n}\n\ninline size_t Request::get_header_value_count(const std::string &key) const {\n  auto r = headers.equal_range(key);\n  return static_cast<size_t>(std::distance(r.first, r.second));\n}\n\ninline void Request::set_header(const std::string &key,\n                                const std::string &val) {\n  if (detail::fields::is_field_name(key) &&\n      detail::fields::is_field_value(val)) {\n    headers.emplace(key, val);\n  }\n}\n\ninline bool Request::has_param(const std::string &key) const {\n  return params.find(key) != params.end();\n}\n\ninline std::string Request::get_param_value(const std::string &key,\n                                            size_t id) const {\n  auto rng = params.equal_range(key);\n  auto it = rng.first;\n  std::advance(it, static_cast<ssize_t>(id));\n  if (it != rng.second) { return it->second; }\n  return std::string();\n}\n\ninline size_t Request::get_param_value_count(const std::string &key) const {\n  auto r = params.equal_range(key);\n  return static_cast<size_t>(std::distance(r.first, r.second));\n}\n\ninline bool Request::is_multipart_form_data() const {\n  const auto &content_type = get_header_value(\"Content-Type\");\n  return !content_type.rfind(\"multipart/form-data\", 0);\n}\n\ninline bool Request::has_file(const std::string &key) const {\n  return files.find(key) != files.end();\n}\n\ninline MultipartFormData Request::get_file_value(const std::string &key) const {\n  auto it = files.find(key);\n  if (it != files.end()) { return it->second; }\n  return MultipartFormData();\n}\n\ninline std::vector<MultipartFormData>\nRequest::get_file_values(const std::string &key) const {\n  std::vector<MultipartFormData> values;\n  auto rng = files.equal_range(key);\n  for (auto it = rng.first; it != rng.second; it++) {\n    values.push_back(it->second);\n  }\n  return values;\n}\n\n// Response implementation\ninline bool Response::has_header(const std::string &key) const {\n  return headers.find(key) != headers.end();\n}\n\ninline std::string Response::get_header_value(const std::string &key,\n                                              const char *def,\n                                              size_t id) const {\n  return detail::get_header_value(headers, key, def, id);\n}\n\ninline size_t Response::get_header_value_count(const std::string &key) const {\n  auto r = headers.equal_range(key);\n  return static_cast<size_t>(std::distance(r.first, r.second));\n}\n\ninline void Response::set_header(const std::string &key,\n                                 const std::string &val) {\n  if (detail::fields::is_field_name(key) &&\n      detail::fields::is_field_value(val)) {\n    headers.emplace(key, val);\n  }\n}\n\ninline void Response::set_redirect(const std::string &url, int stat) {\n  if (detail::fields::is_field_value(url)) {\n    set_header(\"Location\", url);\n    if (300 <= stat && stat < 400) {\n      this->status = stat;\n    } else {\n      this->status = StatusCode::Found_302;\n    }\n  }\n}\n\ninline void Response::set_content(const char *s, size_t n,\n                                  const std::string &content_type) {\n  body.assign(s, n);\n\n  auto rng = headers.equal_range(\"Content-Type\");\n  headers.erase(rng.first, rng.second);\n  set_header(\"Content-Type\", content_type);\n}\n\ninline void Response::set_content(const std::string &s,\n                                  const std::string &content_type) {\n  set_content(s.data(), s.size(), content_type);\n}\n\ninline void Response::set_content(std::string &&s,\n                                  const std::string &content_type) {\n  body = std::move(s);\n\n  auto rng = headers.equal_range(\"Content-Type\");\n  headers.erase(rng.first, rng.second);\n  set_header(\"Content-Type\", content_type);\n}\n\ninline void Response::set_content_provider(\n    size_t in_length, const std::string &content_type, ContentProvider provider,\n    ContentProviderResourceReleaser resource_releaser) {\n  set_header(\"Content-Type\", content_type);\n  content_length_ = in_length;\n  if (in_length > 0) { content_provider_ = std::move(provider); }\n  content_provider_resource_releaser_ = std::move(resource_releaser);\n  is_chunked_content_provider_ = false;\n}\n\ninline void Response::set_content_provider(\n    const std::string &content_type, ContentProviderWithoutLength provider,\n    ContentProviderResourceReleaser resource_releaser) {\n  set_header(\"Content-Type\", content_type);\n  content_length_ = 0;\n  content_provider_ = detail::ContentProviderAdapter(std::move(provider));\n  content_provider_resource_releaser_ = std::move(resource_releaser);\n  is_chunked_content_provider_ = false;\n}\n\ninline void Response::set_chunked_content_provider(\n    const std::string &content_type, ContentProviderWithoutLength provider,\n    ContentProviderResourceReleaser resource_releaser) {\n  set_header(\"Content-Type\", content_type);\n  content_length_ = 0;\n  content_provider_ = detail::ContentProviderAdapter(std::move(provider));\n  content_provider_resource_releaser_ = std::move(resource_releaser);\n  is_chunked_content_provider_ = true;\n}\n\ninline void Response::set_file_content(const std::string &path,\n                                       const std::string &content_type) {\n  file_content_path_ = path;\n  file_content_content_type_ = content_type;\n}\n\ninline void Response::set_file_content(const std::string &path) {\n  file_content_path_ = path;\n}\n\n// Result implementation\ninline bool Result::has_request_header(const std::string &key) const {\n  return request_headers_.find(key) != request_headers_.end();\n}\n\ninline std::string Result::get_request_header_value(const std::string &key,\n                                                    const char *def,\n                                                    size_t id) const {\n  return detail::get_header_value(request_headers_, key, def, id);\n}\n\ninline size_t\nResult::get_request_header_value_count(const std::string &key) const {\n  auto r = request_headers_.equal_range(key);\n  return static_cast<size_t>(std::distance(r.first, r.second));\n}\n\n// Stream implementation\ninline ssize_t Stream::write(const char *ptr) {\n  return write(ptr, strlen(ptr));\n}\n\ninline ssize_t Stream::write(const std::string &s) {\n  return write(s.data(), s.size());\n}\n\nnamespace detail {\n\ninline void calc_actual_timeout(time_t max_timeout_msec, time_t duration_msec,\n                                time_t timeout_sec, time_t timeout_usec,\n                                time_t &actual_timeout_sec,\n                                time_t &actual_timeout_usec) {\n  auto timeout_msec = (timeout_sec * 1000) + (timeout_usec / 1000);\n\n  auto actual_timeout_msec =\n      (std::min)(max_timeout_msec - duration_msec, timeout_msec);\n\n  actual_timeout_sec = actual_timeout_msec / 1000;\n  actual_timeout_usec = (actual_timeout_msec % 1000) * 1000;\n}\n\n// Socket stream implementation\ninline SocketStream::SocketStream(\n    socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,\n    time_t write_timeout_sec, time_t write_timeout_usec,\n    time_t max_timeout_msec,\n    std::chrono::time_point<std::chrono::steady_clock> start_time)\n    : sock_(sock), read_timeout_sec_(read_timeout_sec),\n      read_timeout_usec_(read_timeout_usec),\n      write_timeout_sec_(write_timeout_sec),\n      write_timeout_usec_(write_timeout_usec),\n      max_timeout_msec_(max_timeout_msec), start_time(start_time),\n      read_buff_(read_buff_size_, 0) {}\n\ninline SocketStream::~SocketStream() = default;\n\ninline bool SocketStream::is_readable() const {\n  return read_buff_off_ < read_buff_content_size_;\n}\n\ninline bool SocketStream::wait_readable() const {\n  if (max_timeout_msec_ <= 0) {\n    return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;\n  }\n\n  time_t read_timeout_sec;\n  time_t read_timeout_usec;\n  calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_,\n                      read_timeout_usec_, read_timeout_sec, read_timeout_usec);\n\n  return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0;\n}\n\ninline bool SocketStream::wait_writable() const {\n  return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&\n         is_socket_alive(sock_);\n}\n\ninline ssize_t SocketStream::read(char *ptr, size_t size) {\n#ifdef _WIN32\n  size =\n      (std::min)(size, static_cast<size_t>((std::numeric_limits<int>::max)()));\n#else\n  size = (std::min)(size,\n                    static_cast<size_t>((std::numeric_limits<ssize_t>::max)()));\n#endif\n\n  if (read_buff_off_ < read_buff_content_size_) {\n    auto remaining_size = read_buff_content_size_ - read_buff_off_;\n    if (size <= remaining_size) {\n      memcpy(ptr, read_buff_.data() + read_buff_off_, size);\n      read_buff_off_ += size;\n      return static_cast<ssize_t>(size);\n    } else {\n      memcpy(ptr, read_buff_.data() + read_buff_off_, remaining_size);\n      read_buff_off_ += remaining_size;\n      return static_cast<ssize_t>(remaining_size);\n    }\n  }\n\n  if (!wait_readable()) { return -1; }\n\n  read_buff_off_ = 0;\n  read_buff_content_size_ = 0;\n\n  if (size < read_buff_size_) {\n    auto n = read_socket(sock_, read_buff_.data(), read_buff_size_,\n                         CPPHTTPLIB_RECV_FLAGS);\n    if (n <= 0) {\n      return n;\n    } else if (n <= static_cast<ssize_t>(size)) {\n      memcpy(ptr, read_buff_.data(), static_cast<size_t>(n));\n      return n;\n    } else {\n      memcpy(ptr, read_buff_.data(), size);\n      read_buff_off_ = size;\n      read_buff_content_size_ = static_cast<size_t>(n);\n      return static_cast<ssize_t>(size);\n    }\n  } else {\n    return read_socket(sock_, ptr, size, CPPHTTPLIB_RECV_FLAGS);\n  }\n}\n\ninline ssize_t SocketStream::write(const char *ptr, size_t size) {\n  if (!wait_writable()) { return -1; }\n\n#if defined(_WIN32) && !defined(_WIN64)\n  size =\n      (std::min)(size, static_cast<size_t>((std::numeric_limits<int>::max)()));\n#endif\n\n  return send_socket(sock_, ptr, size, CPPHTTPLIB_SEND_FLAGS);\n}\n\ninline void SocketStream::get_remote_ip_and_port(std::string &ip,\n                                                 int &port) const {\n  return detail::get_remote_ip_and_port(sock_, ip, port);\n}\n\ninline void SocketStream::get_local_ip_and_port(std::string &ip,\n                                                int &port) const {\n  return detail::get_local_ip_and_port(sock_, ip, port);\n}\n\ninline socket_t SocketStream::socket() const { return sock_; }\n\ninline time_t SocketStream::duration() const {\n  return std::chrono::duration_cast<std::chrono::milliseconds>(\n             std::chrono::steady_clock::now() - start_time)\n      .count();\n}\n\n// Buffer stream implementation\ninline bool BufferStream::is_readable() const { return true; }\n\ninline bool BufferStream::wait_readable() const { return true; }\n\ninline bool BufferStream::wait_writable() const { return true; }\n\ninline ssize_t BufferStream::read(char *ptr, size_t size) {\n#if defined(_MSC_VER) && _MSC_VER < 1910\n  auto len_read = buffer._Copy_s(ptr, size, size, position);\n#else\n  auto len_read = buffer.copy(ptr, size, position);\n#endif\n  position += static_cast<size_t>(len_read);\n  return static_cast<ssize_t>(len_read);\n}\n\ninline ssize_t BufferStream::write(const char *ptr, size_t size) {\n  buffer.append(ptr, size);\n  return static_cast<ssize_t>(size);\n}\n\ninline void BufferStream::get_remote_ip_and_port(std::string & /*ip*/,\n                                                 int & /*port*/) const {}\n\ninline void BufferStream::get_local_ip_and_port(std::string & /*ip*/,\n                                                int & /*port*/) const {}\n\ninline socket_t BufferStream::socket() const { return 0; }\n\ninline time_t BufferStream::duration() const { return 0; }\n\ninline const std::string &BufferStream::get_buffer() const { return buffer; }\n\ninline PathParamsMatcher::PathParamsMatcher(const std::string &pattern) {\n  constexpr const char marker[] = \"/:\";\n\n  // One past the last ending position of a path param substring\n  std::size_t last_param_end = 0;\n\n#ifndef CPPHTTPLIB_NO_EXCEPTIONS\n  // Needed to ensure that parameter names are unique during matcher\n  // construction\n  // If exceptions are disabled, only last duplicate path\n  // parameter will be set\n  std::unordered_set<std::string> param_name_set;\n#endif\n\n  while (true) {\n    const auto marker_pos = pattern.find(\n        marker, last_param_end == 0 ? last_param_end : last_param_end - 1);\n    if (marker_pos == std::string::npos) { break; }\n\n    static_fragments_.push_back(\n        pattern.substr(last_param_end, marker_pos - last_param_end + 1));\n\n    const auto param_name_start = marker_pos + str_len(marker);\n\n    auto sep_pos = pattern.find(separator, param_name_start);\n    if (sep_pos == std::string::npos) { sep_pos = pattern.length(); }\n\n    auto param_name =\n        pattern.substr(param_name_start, sep_pos - param_name_start);\n\n#ifndef CPPHTTPLIB_NO_EXCEPTIONS\n    if (param_name_set.find(param_name) != param_name_set.cend()) {\n      std::string msg = \"Encountered path parameter '\" + param_name +\n                        \"' multiple times in route pattern '\" + pattern + \"'.\";\n      throw std::invalid_argument(msg);\n    }\n#endif\n\n    param_names_.push_back(std::move(param_name));\n\n    last_param_end = sep_pos + 1;\n  }\n\n  if (last_param_end < pattern.length()) {\n    static_fragments_.push_back(pattern.substr(last_param_end));\n  }\n}\n\ninline bool PathParamsMatcher::match(Request &request) const {\n  request.matches = std::smatch();\n  request.path_params.clear();\n  request.path_params.reserve(param_names_.size());\n\n  // One past the position at which the path matched the pattern last time\n  std::size_t starting_pos = 0;\n  for (size_t i = 0; i < static_fragments_.size(); ++i) {\n    const auto &fragment = static_fragments_[i];\n\n    if (starting_pos + fragment.length() > request.path.length()) {\n      return false;\n    }\n\n    // Avoid unnecessary allocation by using strncmp instead of substr +\n    // comparison\n    if (std::strncmp(request.path.c_str() + starting_pos, fragment.c_str(),\n                     fragment.length()) != 0) {\n      return false;\n    }\n\n    starting_pos += fragment.length();\n\n    // Should only happen when we have a static fragment after a param\n    // Example: '/users/:id/subscriptions'\n    // The 'subscriptions' fragment here does not have a corresponding param\n    if (i >= param_names_.size()) { continue; }\n\n    auto sep_pos = request.path.find(separator, starting_pos);\n    if (sep_pos == std::string::npos) { sep_pos = request.path.length(); }\n\n    const auto &param_name = param_names_[i];\n\n    request.path_params.emplace(\n        param_name, request.path.substr(starting_pos, sep_pos - starting_pos));\n\n    // Mark everything up to '/' as matched\n    starting_pos = sep_pos + 1;\n  }\n  // Returns false if the path is longer than the pattern\n  return starting_pos >= request.path.length();\n}\n\ninline bool RegexMatcher::match(Request &request) const {\n  request.path_params.clear();\n  return std::regex_match(request.path, request.matches, regex_);\n}\n\n} // namespace detail\n\n// HTTP server implementation\ninline Server::Server()\n    : new_task_queue(\n          [] { return new ThreadPool(CPPHTTPLIB_THREAD_POOL_COUNT); }) {\n#ifndef _WIN32\n  signal(SIGPIPE, SIG_IGN);\n#endif\n}\n\ninline Server::~Server() = default;\n\ninline std::unique_ptr<detail::MatcherBase>\nServer::make_matcher(const std::string &pattern) {\n  if (pattern.find(\"/:\") != std::string::npos) {\n    return detail::make_unique<detail::PathParamsMatcher>(pattern);\n  } else {\n    return detail::make_unique<detail::RegexMatcher>(pattern);\n  }\n}\n\ninline Server &Server::Get(const std::string &pattern, Handler handler) {\n  get_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Post(const std::string &pattern, Handler handler) {\n  post_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Post(const std::string &pattern,\n                            HandlerWithContentReader handler) {\n  post_handlers_for_content_reader_.emplace_back(make_matcher(pattern),\n                                                 std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Put(const std::string &pattern, Handler handler) {\n  put_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Put(const std::string &pattern,\n                           HandlerWithContentReader handler) {\n  put_handlers_for_content_reader_.emplace_back(make_matcher(pattern),\n                                                std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Patch(const std::string &pattern, Handler handler) {\n  patch_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Patch(const std::string &pattern,\n                             HandlerWithContentReader handler) {\n  patch_handlers_for_content_reader_.emplace_back(make_matcher(pattern),\n                                                  std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Delete(const std::string &pattern, Handler handler) {\n  delete_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Delete(const std::string &pattern,\n                              HandlerWithContentReader handler) {\n  delete_handlers_for_content_reader_.emplace_back(make_matcher(pattern),\n                                                   std::move(handler));\n  return *this;\n}\n\ninline Server &Server::Options(const std::string &pattern, Handler handler) {\n  options_handlers_.emplace_back(make_matcher(pattern), std::move(handler));\n  return *this;\n}\n\ninline bool Server::set_base_dir(const std::string &dir,\n                                 const std::string &mount_point) {\n  return set_mount_point(mount_point, dir);\n}\n\ninline bool Server::set_mount_point(const std::string &mount_point,\n                                    const std::string &dir, Headers headers) {\n  detail::FileStat stat(dir);\n  if (stat.is_dir()) {\n    std::string mnt = !mount_point.empty() ? mount_point : \"/\";\n    if (!mnt.empty() && mnt[0] == '/') {\n      base_dirs_.push_back({mnt, dir, std::move(headers)});\n      return true;\n    }\n  }\n  return false;\n}\n\ninline bool Server::remove_mount_point(const std::string &mount_point) {\n  for (auto it = base_dirs_.begin(); it != base_dirs_.end(); ++it) {\n    if (it->mount_point == mount_point) {\n      base_dirs_.erase(it);\n      return true;\n    }\n  }\n  return false;\n}\n\ninline Server &\nServer::set_file_extension_and_mimetype_mapping(const std::string &ext,\n                                                const std::string &mime) {\n  file_extension_and_mimetype_map_[ext] = mime;\n  return *this;\n}\n\ninline Server &Server::set_default_file_mimetype(const std::string &mime) {\n  default_file_mimetype_ = mime;\n  return *this;\n}\n\ninline Server &Server::set_file_request_handler(Handler handler) {\n  file_request_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_error_handler_core(HandlerWithResponse handler,\n                                              std::true_type) {\n  error_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_error_handler_core(Handler handler,\n                                              std::false_type) {\n  error_handler_ = [handler](const Request &req, Response &res) {\n    handler(req, res);\n    return HandlerResponse::Handled;\n  };\n  return *this;\n}\n\ninline Server &Server::set_exception_handler(ExceptionHandler handler) {\n  exception_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_pre_routing_handler(HandlerWithResponse handler) {\n  pre_routing_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_post_routing_handler(Handler handler) {\n  post_routing_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_logger(Logger logger) {\n  logger_ = std::move(logger);\n  return *this;\n}\n\ninline Server &\nServer::set_expect_100_continue_handler(Expect100ContinueHandler handler) {\n  expect_100_continue_handler_ = std::move(handler);\n  return *this;\n}\n\ninline Server &Server::set_address_family(int family) {\n  address_family_ = family;\n  return *this;\n}\n\ninline Server &Server::set_tcp_nodelay(bool on) {\n  tcp_nodelay_ = on;\n  return *this;\n}\n\ninline Server &Server::set_ipv6_v6only(bool on) {\n  ipv6_v6only_ = on;\n  return *this;\n}\n\ninline Server &Server::set_socket_options(SocketOptions socket_options) {\n  socket_options_ = std::move(socket_options);\n  return *this;\n}\n\ninline Server &Server::set_default_headers(Headers headers) {\n  default_headers_ = std::move(headers);\n  return *this;\n}\n\ninline Server &Server::set_header_writer(\n    std::function<ssize_t(Stream &, Headers &)> const &writer) {\n  header_writer_ = writer;\n  return *this;\n}\n\ninline Server &Server::set_keep_alive_max_count(size_t count) {\n  keep_alive_max_count_ = count;\n  return *this;\n}\n\ninline Server &Server::set_keep_alive_timeout(time_t sec) {\n  keep_alive_timeout_sec_ = sec;\n  return *this;\n}\n\ninline Server &Server::set_read_timeout(time_t sec, time_t usec) {\n  read_timeout_sec_ = sec;\n  read_timeout_usec_ = usec;\n  return *this;\n}\n\ninline Server &Server::set_write_timeout(time_t sec, time_t usec) {\n  write_timeout_sec_ = sec;\n  write_timeout_usec_ = usec;\n  return *this;\n}\n\ninline Server &Server::set_idle_interval(time_t sec, time_t usec) {\n  idle_interval_sec_ = sec;\n  idle_interval_usec_ = usec;\n  return *this;\n}\n\ninline Server &Server::set_payload_max_length(size_t length) {\n  payload_max_length_ = length;\n  return *this;\n}\n\ninline bool Server::bind_to_port(const std::string &host, int port,\n                                 int socket_flags) {\n  auto ret = bind_internal(host, port, socket_flags);\n  if (ret == -1) { is_decommissioned = true; }\n  return ret >= 0;\n}\ninline int Server::bind_to_any_port(const std::string &host, int socket_flags) {\n  auto ret = bind_internal(host, 0, socket_flags);\n  if (ret == -1) { is_decommissioned = true; }\n  return ret;\n}\n\ninline bool Server::listen_after_bind() { return listen_internal(); }\n\ninline bool Server::listen(const std::string &host, int port,\n                           int socket_flags) {\n  return bind_to_port(host, port, socket_flags) && listen_internal();\n}\n\ninline bool Server::is_running() const { return is_running_; }\n\ninline void Server::wait_until_ready() const {\n  while (!is_running_ && !is_decommissioned) {\n    std::this_thread::sleep_for(std::chrono::milliseconds{1});\n  }\n}\n\ninline void Server::stop() {\n  if (is_running_) {\n    assert(svr_sock_ != INVALID_SOCKET);\n    std::atomic<socket_t> sock(svr_sock_.exchange(INVALID_SOCKET));\n    detail::shutdown_socket(sock);\n    detail::close_socket(sock);\n  }\n  is_decommissioned = false;\n}\n\ninline void Server::decommission() { is_decommissioned = true; }\n\ninline bool Server::parse_request_line(const char *s, Request &req) const {\n  auto len = strlen(s);\n  if (len < 2 || s[len - 2] != '\\r' || s[len - 1] != '\\n') { return false; }\n  len -= 2;\n\n  {\n    size_t count = 0;\n\n    detail::split(s, s + len, ' ', [&](const char *b, const char *e) {\n      switch (count) {\n      case 0: req.method = std::string(b, e); break;\n      case 1: req.target = std::string(b, e); break;\n      case 2: req.version = std::string(b, e); break;\n      default: break;\n      }\n      count++;\n    });\n\n    if (count != 3) { return false; }\n  }\n\n  thread_local const std::set<std::string> methods{\n      \"GET\",     \"HEAD\",    \"POST\",  \"PUT\",   \"DELETE\",\n      \"CONNECT\", \"OPTIONS\", \"TRACE\", \"PATCH\", \"PRI\"};\n\n  if (methods.find(req.method) == methods.end()) { return false; }\n\n  if (req.version != \"HTTP/1.1\" && req.version != \"HTTP/1.0\") { return false; }\n\n  {\n    // Skip URL fragment\n    for (size_t i = 0; i < req.target.size(); i++) {\n      if (req.target[i] == '#') {\n        req.target.erase(i);\n        break;\n      }\n    }\n\n    detail::divide(req.target, '?',\n                   [&](const char *lhs_data, std::size_t lhs_size,\n                       const char *rhs_data, std::size_t rhs_size) {\n                     req.path = detail::decode_url(\n                         std::string(lhs_data, lhs_size), false);\n                     detail::parse_query_text(rhs_data, rhs_size, req.params);\n                   });\n  }\n\n  return true;\n}\n\ninline bool Server::write_response(Stream &strm, bool close_connection,\n                                   Request &req, Response &res) {\n  // NOTE: `req.ranges` should be empty, otherwise it will be applied\n  // incorrectly to the error content.\n  req.ranges.clear();\n  return write_response_core(strm, close_connection, req, res, false);\n}\n\ninline bool Server::write_response_with_content(Stream &strm,\n                                                bool close_connection,\n                                                const Request &req,\n                                                Response &res) {\n  return write_response_core(strm, close_connection, req, res, true);\n}\n\ninline bool Server::write_response_core(Stream &strm, bool close_connection,\n                                        const Request &req, Response &res,\n                                        bool need_apply_ranges) {\n  assert(res.status != -1);\n\n  if (400 <= res.status && error_handler_ &&\n      error_handler_(req, res) == HandlerResponse::Handled) {\n    need_apply_ranges = true;\n  }\n\n  std::string content_type;\n  std::string boundary;\n  if (need_apply_ranges) { apply_ranges(req, res, content_type, boundary); }\n\n  // Prepare additional headers\n  if (close_connection || req.get_header_value(\"Connection\") == \"close\") {\n    res.set_header(\"Connection\", \"close\");\n  } else {\n    std::string s = \"timeout=\";\n    s += std::to_string(keep_alive_timeout_sec_);\n    s += \", max=\";\n    s += std::to_string(keep_alive_max_count_);\n    res.set_header(\"Keep-Alive\", s);\n  }\n\n  if ((!res.body.empty() || res.content_length_ > 0 || res.content_provider_) &&\n      !res.has_header(\"Content-Type\")) {\n    res.set_header(\"Content-Type\", \"text/plain\");\n  }\n\n  if (res.body.empty() && !res.content_length_ && !res.content_provider_ &&\n      !res.has_header(\"Content-Length\")) {\n    res.set_header(\"Content-Length\", \"0\");\n  }\n\n  if (req.method == \"HEAD\" && !res.has_header(\"Accept-Ranges\")) {\n    res.set_header(\"Accept-Ranges\", \"bytes\");\n  }\n\n  if (post_routing_handler_) { post_routing_handler_(req, res); }\n\n  // Response line and headers\n  {\n    detail::BufferStream bstrm;\n    if (!detail::write_response_line(bstrm, res.status)) { return false; }\n    if (!header_writer_(bstrm, res.headers)) { return false; }\n\n    // Flush buffer\n    auto &data = bstrm.get_buffer();\n    detail::write_data(strm, data.data(), data.size());\n  }\n\n  // Body\n  auto ret = true;\n  if (req.method != \"HEAD\") {\n    if (!res.body.empty()) {\n      if (!detail::write_data(strm, res.body.data(), res.body.size())) {\n        ret = false;\n      }\n    } else if (res.content_provider_) {\n      if (write_content_with_provider(strm, req, res, boundary, content_type)) {\n        res.content_provider_success_ = true;\n      } else {\n        ret = false;\n      }\n    }\n  }\n\n  // Log\n  if (logger_) { logger_(req, res); }\n\n  return ret;\n}\n\ninline bool\nServer::write_content_with_provider(Stream &strm, const Request &req,\n                                    Response &res, const std::string &boundary,\n                                    const std::string &content_type) {\n  auto is_shutting_down = [this]() {\n    return this->svr_sock_ == INVALID_SOCKET;\n  };\n\n  if (res.content_length_ > 0) {\n    if (req.ranges.empty()) {\n      return detail::write_content(strm, res.content_provider_, 0,\n                                   res.content_length_, is_shutting_down);\n    } else if (req.ranges.size() == 1) {\n      auto offset_and_length = detail::get_range_offset_and_length(\n          req.ranges[0], res.content_length_);\n\n      return detail::write_content(strm, res.content_provider_,\n                                   offset_and_length.first,\n                                   offset_and_length.second, is_shutting_down);\n    } else {\n      return detail::write_multipart_ranges_data(\n          strm, req, res, boundary, content_type, res.content_length_,\n          is_shutting_down);\n    }\n  } else {\n    if (res.is_chunked_content_provider_) {\n      auto type = detail::encoding_type(req, res);\n\n      std::unique_ptr<detail::compressor> compressor;\n      if (type == detail::EncodingType::Gzip) {\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n        compressor = detail::make_unique<detail::gzip_compressor>();\n#endif\n      } else if (type == detail::EncodingType::Brotli) {\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n        compressor = detail::make_unique<detail::brotli_compressor>();\n#endif\n      } else if (type == detail::EncodingType::Zstd) {\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n        compressor = detail::make_unique<detail::zstd_compressor>();\n#endif\n      } else {\n        compressor = detail::make_unique<detail::nocompressor>();\n      }\n      assert(compressor != nullptr);\n\n      return detail::write_content_chunked(strm, res.content_provider_,\n                                           is_shutting_down, *compressor);\n    } else {\n      return detail::write_content_without_length(strm, res.content_provider_,\n                                                  is_shutting_down);\n    }\n  }\n}\n\ninline bool Server::read_content(Stream &strm, Request &req, Response &res) {\n  MultipartFormDataMap::iterator cur;\n  auto file_count = 0;\n  if (read_content_core(\n          strm, req, res,\n          // Regular\n          [&](const char *buf, size_t n) {\n            if (req.body.size() + n > req.body.max_size()) { return false; }\n            req.body.append(buf, n);\n            return true;\n          },\n          // Multipart\n          [&](const MultipartFormData &file) {\n            if (file_count++ == CPPHTTPLIB_MULTIPART_FORM_DATA_FILE_MAX_COUNT) {\n              return false;\n            }\n            cur = req.files.emplace(file.name, file);\n            return true;\n          },\n          [&](const char *buf, size_t n) {\n            auto &content = cur->second.content;\n            if (content.size() + n > content.max_size()) { return false; }\n            content.append(buf, n);\n            return true;\n          })) {\n    const auto &content_type = req.get_header_value(\"Content-Type\");\n    if (!content_type.find(\"application/x-www-form-urlencoded\")) {\n      if (req.body.size() > CPPHTTPLIB_FORM_URL_ENCODED_PAYLOAD_MAX_LENGTH) {\n        res.status = StatusCode::PayloadTooLarge_413; // NOTE: should be 414?\n        return false;\n      }\n      detail::parse_query_text(req.body, req.params);\n    }\n    return true;\n  }\n  return false;\n}\n\ninline bool Server::read_content_with_content_receiver(\n    Stream &strm, Request &req, Response &res, ContentReceiver receiver,\n    MultipartContentHeader multipart_header,\n    ContentReceiver multipart_receiver) {\n  return read_content_core(strm, req, res, std::move(receiver),\n                           std::move(multipart_header),\n                           std::move(multipart_receiver));\n}\n\ninline bool\nServer::read_content_core(Stream &strm, Request &req, Response &res,\n                          ContentReceiver receiver,\n                          MultipartContentHeader multipart_header,\n                          ContentReceiver multipart_receiver) const {\n  detail::MultipartFormDataParser multipart_form_data_parser;\n  ContentReceiverWithProgress out;\n\n  if (req.is_multipart_form_data()) {\n    const auto &content_type = req.get_header_value(\"Content-Type\");\n    std::string boundary;\n    if (!detail::parse_multipart_boundary(content_type, boundary)) {\n      res.status = StatusCode::BadRequest_400;\n      return false;\n    }\n\n    multipart_form_data_parser.set_boundary(std::move(boundary));\n    out = [&](const char *buf, size_t n, uint64_t /*off*/, uint64_t /*len*/) {\n      /* For debug\n      size_t pos = 0;\n      while (pos < n) {\n        auto read_size = (std::min)<size_t>(1, n - pos);\n        auto ret = multipart_form_data_parser.parse(\n            buf + pos, read_size, multipart_receiver, multipart_header);\n        if (!ret) { return false; }\n        pos += read_size;\n      }\n      return true;\n      */\n      return multipart_form_data_parser.parse(buf, n, multipart_receiver,\n                                              multipart_header);\n    };\n  } else {\n    out = [receiver](const char *buf, size_t n, uint64_t /*off*/,\n                     uint64_t /*len*/) { return receiver(buf, n); };\n  }\n\n  if (req.method == \"DELETE\" && !req.has_header(\"Content-Length\")) {\n    return true;\n  }\n\n  if (!detail::read_content(strm, req, payload_max_length_, res.status, nullptr,\n                            out, true)) {\n    return false;\n  }\n\n  if (req.is_multipart_form_data()) {\n    if (!multipart_form_data_parser.is_valid()) {\n      res.status = StatusCode::BadRequest_400;\n      return false;\n    }\n  }\n\n  return true;\n}\n\ninline bool Server::handle_file_request(const Request &req, Response &res,\n                                        bool head) {\n  for (const auto &entry : base_dirs_) {\n    // Prefix match\n    if (!req.path.compare(0, entry.mount_point.size(), entry.mount_point)) {\n      std::string sub_path = \"/\" + req.path.substr(entry.mount_point.size());\n      if (detail::is_valid_path(sub_path)) {\n        auto path = entry.base_dir + sub_path;\n        if (path.back() == '/') { path += \"index.html\"; }\n\n        detail::FileStat stat(path);\n\n        if (stat.is_dir()) {\n          res.set_redirect(sub_path + \"/\", StatusCode::MovedPermanently_301);\n          return true;\n        }\n\n        if (stat.is_file()) {\n          for (const auto &kv : entry.headers) {\n            res.set_header(kv.first, kv.second);\n          }\n\n          auto mm = std::make_shared<detail::mmap>(path.c_str());\n          if (!mm->is_open()) { return false; }\n\n          res.set_content_provider(\n              mm->size(),\n              detail::find_content_type(path, file_extension_and_mimetype_map_,\n                                        default_file_mimetype_),\n              [mm](size_t offset, size_t length, DataSink &sink) -> bool {\n                sink.write(mm->data() + offset, length);\n                return true;\n              });\n\n          if (!head && file_request_handler_) {\n            file_request_handler_(req, res);\n          }\n\n          return true;\n        }\n      }\n    }\n  }\n  return false;\n}\n\ninline socket_t\nServer::create_server_socket(const std::string &host, int port,\n                             int socket_flags,\n                             SocketOptions socket_options) const {\n  return detail::create_socket(\n      host, std::string(), port, address_family_, socket_flags, tcp_nodelay_,\n      ipv6_v6only_, std::move(socket_options),\n      [](socket_t sock, struct addrinfo &ai, bool & /*quit*/) -> bool {\n        if (::bind(sock, ai.ai_addr, static_cast<socklen_t>(ai.ai_addrlen))) {\n          return false;\n        }\n        if (::listen(sock, CPPHTTPLIB_LISTEN_BACKLOG)) { return false; }\n        return true;\n      });\n}\n\ninline int Server::bind_internal(const std::string &host, int port,\n                                 int socket_flags) {\n  if (is_decommissioned) { return -1; }\n\n  if (!is_valid()) { return -1; }\n\n  svr_sock_ = create_server_socket(host, port, socket_flags, socket_options_);\n  if (svr_sock_ == INVALID_SOCKET) { return -1; }\n\n  if (port == 0) {\n    struct sockaddr_storage addr;\n    socklen_t addr_len = sizeof(addr);\n    if (getsockname(svr_sock_, reinterpret_cast<struct sockaddr *>(&addr),\n                    &addr_len) == -1) {\n      return -1;\n    }\n    if (addr.ss_family == AF_INET) {\n      return ntohs(reinterpret_cast<struct sockaddr_in *>(&addr)->sin_port);\n    } else if (addr.ss_family == AF_INET6) {\n      return ntohs(reinterpret_cast<struct sockaddr_in6 *>(&addr)->sin6_port);\n    } else {\n      return -1;\n    }\n  } else {\n    return port;\n  }\n}\n\ninline bool Server::listen_internal() {\n  if (is_decommissioned) { return false; }\n\n  auto ret = true;\n  is_running_ = true;\n  auto se = detail::scope_exit([&]() { is_running_ = false; });\n\n  {\n    std::unique_ptr<TaskQueue> task_queue(new_task_queue());\n\n    while (svr_sock_ != INVALID_SOCKET) {\n#ifndef _WIN32\n      if (idle_interval_sec_ > 0 || idle_interval_usec_ > 0) {\n#endif\n        auto val = detail::select_read(svr_sock_, idle_interval_sec_,\n                                       idle_interval_usec_);\n        if (val == 0) { // Timeout\n          task_queue->on_idle();\n          continue;\n        }\n#ifndef _WIN32\n      }\n#endif\n\n#if defined _WIN32\n      // sockets connected via WASAccept inherit flags NO_HANDLE_INHERIT,\n      // OVERLAPPED\n      socket_t sock = WSAAccept(svr_sock_, nullptr, nullptr, nullptr, 0);\n#elif defined SOCK_CLOEXEC\n      socket_t sock = accept4(svr_sock_, nullptr, nullptr, SOCK_CLOEXEC);\n#else\n      socket_t sock = accept(svr_sock_, nullptr, nullptr);\n#endif\n\n      if (sock == INVALID_SOCKET) {\n        if (errno == EMFILE) {\n          // The per-process limit of open file descriptors has been reached.\n          // Try to accept new connections after a short sleep.\n          std::this_thread::sleep_for(std::chrono::microseconds{1});\n          continue;\n        } else if (errno == EINTR || errno == EAGAIN) {\n          continue;\n        }\n        if (svr_sock_ != INVALID_SOCKET) {\n          detail::close_socket(svr_sock_);\n          ret = false;\n        } else {\n          ; // The server socket was closed by user.\n        }\n        break;\n      }\n\n      detail::set_socket_opt_time(sock, SOL_SOCKET, SO_RCVTIMEO,\n                                  read_timeout_sec_, read_timeout_usec_);\n      detail::set_socket_opt_time(sock, SOL_SOCKET, SO_SNDTIMEO,\n                                  write_timeout_sec_, write_timeout_usec_);\n\n      if (!task_queue->enqueue(\n              [this, sock]() { process_and_close_socket(sock); })) {\n        detail::shutdown_socket(sock);\n        detail::close_socket(sock);\n      }\n    }\n\n    task_queue->shutdown();\n  }\n\n  is_decommissioned = !ret;\n  return ret;\n}\n\ninline bool Server::routing(Request &req, Response &res, Stream &strm) {\n  if (pre_routing_handler_ &&\n      pre_routing_handler_(req, res) == HandlerResponse::Handled) {\n    return true;\n  }\n\n  // File handler\n  auto is_head_request = req.method == \"HEAD\";\n  if ((req.method == \"GET\" || is_head_request) &&\n      handle_file_request(req, res, is_head_request)) {\n    return true;\n  }\n\n  if (detail::expect_content(req)) {\n    // Content reader handler\n    {\n      ContentReader reader(\n          [&](ContentReceiver receiver) {\n            return read_content_with_content_receiver(\n                strm, req, res, std::move(receiver), nullptr, nullptr);\n          },\n          [&](MultipartContentHeader header, ContentReceiver receiver) {\n            return read_content_with_content_receiver(strm, req, res, nullptr,\n                                                      std::move(header),\n                                                      std::move(receiver));\n          });\n\n      if (req.method == \"POST\") {\n        if (dispatch_request_for_content_reader(\n                req, res, std::move(reader),\n                post_handlers_for_content_reader_)) {\n          return true;\n        }\n      } else if (req.method == \"PUT\") {\n        if (dispatch_request_for_content_reader(\n                req, res, std::move(reader),\n                put_handlers_for_content_reader_)) {\n          return true;\n        }\n      } else if (req.method == \"PATCH\") {\n        if (dispatch_request_for_content_reader(\n                req, res, std::move(reader),\n                patch_handlers_for_content_reader_)) {\n          return true;\n        }\n      } else if (req.method == \"DELETE\") {\n        if (dispatch_request_for_content_reader(\n                req, res, std::move(reader),\n                delete_handlers_for_content_reader_)) {\n          return true;\n        }\n      }\n    }\n\n    // Read content into `req.body`\n    if (!read_content(strm, req, res)) { return false; }\n  }\n\n  // Regular handler\n  if (req.method == \"GET\" || req.method == \"HEAD\") {\n    return dispatch_request(req, res, get_handlers_);\n  } else if (req.method == \"POST\") {\n    return dispatch_request(req, res, post_handlers_);\n  } else if (req.method == \"PUT\") {\n    return dispatch_request(req, res, put_handlers_);\n  } else if (req.method == \"DELETE\") {\n    return dispatch_request(req, res, delete_handlers_);\n  } else if (req.method == \"OPTIONS\") {\n    return dispatch_request(req, res, options_handlers_);\n  } else if (req.method == \"PATCH\") {\n    return dispatch_request(req, res, patch_handlers_);\n  }\n\n  res.status = StatusCode::BadRequest_400;\n  return false;\n}\n\ninline bool Server::dispatch_request(Request &req, Response &res,\n                                     const Handlers &handlers) const {\n  for (const auto &x : handlers) {\n    const auto &matcher = x.first;\n    const auto &handler = x.second;\n\n    if (matcher->match(req)) {\n      handler(req, res);\n      return true;\n    }\n  }\n  return false;\n}\n\ninline void Server::apply_ranges(const Request &req, Response &res,\n                                 std::string &content_type,\n                                 std::string &boundary) const {\n  if (req.ranges.size() > 1 && res.status == StatusCode::PartialContent_206) {\n    auto it = res.headers.find(\"Content-Type\");\n    if (it != res.headers.end()) {\n      content_type = it->second;\n      res.headers.erase(it);\n    }\n\n    boundary = detail::make_multipart_data_boundary();\n\n    res.set_header(\"Content-Type\",\n                   \"multipart/byteranges; boundary=\" + boundary);\n  }\n\n  auto type = detail::encoding_type(req, res);\n\n  if (res.body.empty()) {\n    if (res.content_length_ > 0) {\n      size_t length = 0;\n      if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) {\n        length = res.content_length_;\n      } else if (req.ranges.size() == 1) {\n        auto offset_and_length = detail::get_range_offset_and_length(\n            req.ranges[0], res.content_length_);\n\n        length = offset_and_length.second;\n\n        auto content_range = detail::make_content_range_header_field(\n            offset_and_length, res.content_length_);\n        res.set_header(\"Content-Range\", content_range);\n      } else {\n        length = detail::get_multipart_ranges_data_length(\n            req, boundary, content_type, res.content_length_);\n      }\n      res.set_header(\"Content-Length\", std::to_string(length));\n    } else {\n      if (res.content_provider_) {\n        if (res.is_chunked_content_provider_) {\n          res.set_header(\"Transfer-Encoding\", \"chunked\");\n          if (type == detail::EncodingType::Gzip) {\n            res.set_header(\"Content-Encoding\", \"gzip\");\n          } else if (type == detail::EncodingType::Brotli) {\n            res.set_header(\"Content-Encoding\", \"br\");\n          } else if (type == detail::EncodingType::Zstd) {\n            res.set_header(\"Content-Encoding\", \"zstd\");\n          }\n        }\n      }\n    }\n  } else {\n    if (req.ranges.empty() || res.status != StatusCode::PartialContent_206) {\n      ;\n    } else if (req.ranges.size() == 1) {\n      auto offset_and_length =\n          detail::get_range_offset_and_length(req.ranges[0], res.body.size());\n      auto offset = offset_and_length.first;\n      auto length = offset_and_length.second;\n\n      auto content_range = detail::make_content_range_header_field(\n          offset_and_length, res.body.size());\n      res.set_header(\"Content-Range\", content_range);\n\n      assert(offset + length <= res.body.size());\n      res.body = res.body.substr(offset, length);\n    } else {\n      std::string data;\n      detail::make_multipart_ranges_data(req, res, boundary, content_type,\n                                         res.body.size(), data);\n      res.body.swap(data);\n    }\n\n    if (type != detail::EncodingType::None) {\n      std::unique_ptr<detail::compressor> compressor;\n      std::string content_encoding;\n\n      if (type == detail::EncodingType::Gzip) {\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n        compressor = detail::make_unique<detail::gzip_compressor>();\n        content_encoding = \"gzip\";\n#endif\n      } else if (type == detail::EncodingType::Brotli) {\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n        compressor = detail::make_unique<detail::brotli_compressor>();\n        content_encoding = \"br\";\n#endif\n      } else if (type == detail::EncodingType::Zstd) {\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n        compressor = detail::make_unique<detail::zstd_compressor>();\n        content_encoding = \"zstd\";\n#endif\n      }\n\n      if (compressor) {\n        std::string compressed;\n        if (compressor->compress(res.body.data(), res.body.size(), true,\n                                 [&](const char *data, size_t data_len) {\n                                   compressed.append(data, data_len);\n                                   return true;\n                                 })) {\n          res.body.swap(compressed);\n          res.set_header(\"Content-Encoding\", content_encoding);\n        }\n      }\n    }\n\n    auto length = std::to_string(res.body.size());\n    res.set_header(\"Content-Length\", length);\n  }\n}\n\ninline bool Server::dispatch_request_for_content_reader(\n    Request &req, Response &res, ContentReader content_reader,\n    const HandlersForContentReader &handlers) const {\n  for (const auto &x : handlers) {\n    const auto &matcher = x.first;\n    const auto &handler = x.second;\n\n    if (matcher->match(req)) {\n      handler(req, res, content_reader);\n      return true;\n    }\n  }\n  return false;\n}\n\ninline bool\nServer::process_request(Stream &strm, const std::string &remote_addr,\n                        int remote_port, const std::string &local_addr,\n                        int local_port, bool close_connection,\n                        bool &connection_closed,\n                        const std::function<void(Request &)> &setup_request) {\n  std::array<char, 2048> buf{};\n\n  detail::stream_line_reader line_reader(strm, buf.data(), buf.size());\n\n  // Connection has been closed on client\n  if (!line_reader.getline()) { return false; }\n\n  Request req;\n\n  Response res;\n  res.version = \"HTTP/1.1\";\n  res.headers = default_headers_;\n\n  // Request line and headers\n  if (!parse_request_line(line_reader.ptr(), req) ||\n      !detail::read_headers(strm, req.headers)) {\n    res.status = StatusCode::BadRequest_400;\n    return write_response(strm, close_connection, req, res);\n  }\n\n  // Check if the request URI doesn't exceed the limit\n  if (req.target.size() > CPPHTTPLIB_REQUEST_URI_MAX_LENGTH) {\n    Headers dummy;\n    detail::read_headers(strm, dummy);\n    res.status = StatusCode::UriTooLong_414;\n    return write_response(strm, close_connection, req, res);\n  }\n\n  if (req.get_header_value(\"Connection\") == \"close\") {\n    connection_closed = true;\n  }\n\n  if (req.version == \"HTTP/1.0\" &&\n      req.get_header_value(\"Connection\") != \"Keep-Alive\") {\n    connection_closed = true;\n  }\n\n  req.remote_addr = remote_addr;\n  req.remote_port = remote_port;\n  req.set_header(\"REMOTE_ADDR\", req.remote_addr);\n  req.set_header(\"REMOTE_PORT\", std::to_string(req.remote_port));\n\n  req.local_addr = local_addr;\n  req.local_port = local_port;\n  req.set_header(\"LOCAL_ADDR\", req.local_addr);\n  req.set_header(\"LOCAL_PORT\", std::to_string(req.local_port));\n\n  if (req.has_header(\"Range\")) {\n    const auto &range_header_value = req.get_header_value(\"Range\");\n    if (!detail::parse_range_header(range_header_value, req.ranges)) {\n      res.status = StatusCode::RangeNotSatisfiable_416;\n      return write_response(strm, close_connection, req, res);\n    }\n  }\n\n  if (setup_request) { setup_request(req); }\n\n  if (req.get_header_value(\"Expect\") == \"100-continue\") {\n    int status = StatusCode::Continue_100;\n    if (expect_100_continue_handler_) {\n      status = expect_100_continue_handler_(req, res);\n    }\n    switch (status) {\n    case StatusCode::Continue_100:\n    case StatusCode::ExpectationFailed_417:\n      detail::write_response_line(strm, status);\n      strm.write(\"\\r\\n\");\n      break;\n    default:\n      connection_closed = true;\n      return write_response(strm, true, req, res);\n    }\n  }\n\n  // Setup `is_connection_closed` method\n  req.is_connection_closed = [&]() {\n    return !detail::is_socket_alive(strm.socket());\n  };\n\n  // Routing\n  auto routed = false;\n#ifdef CPPHTTPLIB_NO_EXCEPTIONS\n  routed = routing(req, res, strm);\n#else\n  try {\n    routed = routing(req, res, strm);\n  } catch (std::exception &e) {\n    if (exception_handler_) {\n      auto ep = std::current_exception();\n      exception_handler_(req, res, ep);\n      routed = true;\n    } else {\n      res.status = StatusCode::InternalServerError_500;\n      std::string val;\n      auto s = e.what();\n      for (size_t i = 0; s[i]; i++) {\n        switch (s[i]) {\n        case '\\r': val += \"\\\\r\"; break;\n        case '\\n': val += \"\\\\n\"; break;\n        default: val += s[i]; break;\n        }\n      }\n      res.set_header(\"EXCEPTION_WHAT\", val);\n    }\n  } catch (...) {\n    if (exception_handler_) {\n      auto ep = std::current_exception();\n      exception_handler_(req, res, ep);\n      routed = true;\n    } else {\n      res.status = StatusCode::InternalServerError_500;\n      res.set_header(\"EXCEPTION_WHAT\", \"UNKNOWN\");\n    }\n  }\n#endif\n  if (routed) {\n    if (res.status == -1) {\n      res.status = req.ranges.empty() ? StatusCode::OK_200\n                                      : StatusCode::PartialContent_206;\n    }\n\n    // Serve file content by using a content provider\n    if (!res.file_content_path_.empty()) {\n      const auto &path = res.file_content_path_;\n      auto mm = std::make_shared<detail::mmap>(path.c_str());\n      if (!mm->is_open()) {\n        res.body.clear();\n        res.content_length_ = 0;\n        res.content_provider_ = nullptr;\n        res.status = StatusCode::NotFound_404;\n        return write_response(strm, close_connection, req, res);\n      }\n\n      auto content_type = res.file_content_content_type_;\n      if (content_type.empty()) {\n        content_type = detail::find_content_type(\n            path, file_extension_and_mimetype_map_, default_file_mimetype_);\n      }\n\n      res.set_content_provider(\n          mm->size(), content_type,\n          [mm](size_t offset, size_t length, DataSink &sink) -> bool {\n            sink.write(mm->data() + offset, length);\n            return true;\n          });\n    }\n\n    if (detail::range_error(req, res)) {\n      res.body.clear();\n      res.content_length_ = 0;\n      res.content_provider_ = nullptr;\n      res.status = StatusCode::RangeNotSatisfiable_416;\n      return write_response(strm, close_connection, req, res);\n    }\n\n    return write_response_with_content(strm, close_connection, req, res);\n  } else {\n    if (res.status == -1) { res.status = StatusCode::NotFound_404; }\n\n    return write_response(strm, close_connection, req, res);\n  }\n}\n\ninline bool Server::is_valid() const { return true; }\n\ninline bool Server::process_and_close_socket(socket_t sock) {\n  std::string remote_addr;\n  int remote_port = 0;\n  detail::get_remote_ip_and_port(sock, remote_addr, remote_port);\n\n  std::string local_addr;\n  int local_port = 0;\n  detail::get_local_ip_and_port(sock, local_addr, local_port);\n\n  auto ret = detail::process_server_socket(\n      svr_sock_, sock, keep_alive_max_count_, keep_alive_timeout_sec_,\n      read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,\n      write_timeout_usec_,\n      [&](Stream &strm, bool close_connection, bool &connection_closed) {\n        return process_request(strm, remote_addr, remote_port, local_addr,\n                               local_port, close_connection, connection_closed,\n                               nullptr);\n      });\n\n  detail::shutdown_socket(sock);\n  detail::close_socket(sock);\n  return ret;\n}\n\n// HTTP client implementation\ninline ClientImpl::ClientImpl(const std::string &host)\n    : ClientImpl(host, 80, std::string(), std::string()) {}\n\ninline ClientImpl::ClientImpl(const std::string &host, int port)\n    : ClientImpl(host, port, std::string(), std::string()) {}\n\ninline ClientImpl::ClientImpl(const std::string &host, int port,\n                              const std::string &client_cert_path,\n                              const std::string &client_key_path)\n    : host_(detail::escape_abstract_namespace_unix_domain(host)), port_(port),\n      host_and_port_(adjust_host_string(host_) + \":\" + std::to_string(port)),\n      client_cert_path_(client_cert_path), client_key_path_(client_key_path) {}\n\ninline ClientImpl::~ClientImpl() {\n  // Wait until all the requests in flight are handled.\n  size_t retry_count = 10;\n  while (retry_count-- > 0) {\n    {\n      std::lock_guard<std::mutex> guard(socket_mutex_);\n      if (socket_requests_in_flight_ == 0) { break; }\n    }\n    std::this_thread::sleep_for(std::chrono::milliseconds{1});\n  }\n\n  std::lock_guard<std::mutex> guard(socket_mutex_);\n  shutdown_socket(socket_);\n  close_socket(socket_);\n}\n\ninline bool ClientImpl::is_valid() const { return true; }\n\ninline void ClientImpl::copy_settings(const ClientImpl &rhs) {\n  client_cert_path_ = rhs.client_cert_path_;\n  client_key_path_ = rhs.client_key_path_;\n  connection_timeout_sec_ = rhs.connection_timeout_sec_;\n  read_timeout_sec_ = rhs.read_timeout_sec_;\n  read_timeout_usec_ = rhs.read_timeout_usec_;\n  write_timeout_sec_ = rhs.write_timeout_sec_;\n  write_timeout_usec_ = rhs.write_timeout_usec_;\n  max_timeout_msec_ = rhs.max_timeout_msec_;\n  basic_auth_username_ = rhs.basic_auth_username_;\n  basic_auth_password_ = rhs.basic_auth_password_;\n  bearer_token_auth_token_ = rhs.bearer_token_auth_token_;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  digest_auth_username_ = rhs.digest_auth_username_;\n  digest_auth_password_ = rhs.digest_auth_password_;\n#endif\n  keep_alive_ = rhs.keep_alive_;\n  follow_location_ = rhs.follow_location_;\n  url_encode_ = rhs.url_encode_;\n  address_family_ = rhs.address_family_;\n  tcp_nodelay_ = rhs.tcp_nodelay_;\n  ipv6_v6only_ = rhs.ipv6_v6only_;\n  socket_options_ = rhs.socket_options_;\n  compress_ = rhs.compress_;\n  decompress_ = rhs.decompress_;\n  interface_ = rhs.interface_;\n  proxy_host_ = rhs.proxy_host_;\n  proxy_port_ = rhs.proxy_port_;\n  proxy_basic_auth_username_ = rhs.proxy_basic_auth_username_;\n  proxy_basic_auth_password_ = rhs.proxy_basic_auth_password_;\n  proxy_bearer_token_auth_token_ = rhs.proxy_bearer_token_auth_token_;\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  proxy_digest_auth_username_ = rhs.proxy_digest_auth_username_;\n  proxy_digest_auth_password_ = rhs.proxy_digest_auth_password_;\n#endif\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  ca_cert_file_path_ = rhs.ca_cert_file_path_;\n  ca_cert_dir_path_ = rhs.ca_cert_dir_path_;\n  ca_cert_store_ = rhs.ca_cert_store_;\n#endif\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  server_certificate_verification_ = rhs.server_certificate_verification_;\n  server_hostname_verification_ = rhs.server_hostname_verification_;\n  server_certificate_verifier_ = rhs.server_certificate_verifier_;\n#endif\n  logger_ = rhs.logger_;\n}\n\ninline socket_t ClientImpl::create_client_socket(Error &error) const {\n  if (!proxy_host_.empty() && proxy_port_ != -1) {\n    return detail::create_client_socket(\n        proxy_host_, std::string(), proxy_port_, address_family_, tcp_nodelay_,\n        ipv6_v6only_, socket_options_, connection_timeout_sec_,\n        connection_timeout_usec_, read_timeout_sec_, read_timeout_usec_,\n        write_timeout_sec_, write_timeout_usec_, interface_, error);\n  }\n\n  // Check is custom IP specified for host_\n  std::string ip;\n  auto it = addr_map_.find(host_);\n  if (it != addr_map_.end()) { ip = it->second; }\n\n  return detail::create_client_socket(\n      host_, ip, port_, address_family_, tcp_nodelay_, ipv6_v6only_,\n      socket_options_, connection_timeout_sec_, connection_timeout_usec_,\n      read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,\n      write_timeout_usec_, interface_, error);\n}\n\ninline bool ClientImpl::create_and_connect_socket(Socket &socket,\n                                                  Error &error) {\n  auto sock = create_client_socket(error);\n  if (sock == INVALID_SOCKET) { return false; }\n  socket.sock = sock;\n  return true;\n}\n\ninline void ClientImpl::shutdown_ssl(Socket & /*socket*/,\n                                     bool /*shutdown_gracefully*/) {\n  // If there are any requests in flight from threads other than us, then it's\n  // a thread-unsafe race because individual ssl* objects are not thread-safe.\n  assert(socket_requests_in_flight_ == 0 ||\n         socket_requests_are_from_thread_ == std::this_thread::get_id());\n}\n\ninline void ClientImpl::shutdown_socket(Socket &socket) const {\n  if (socket.sock == INVALID_SOCKET) { return; }\n  detail::shutdown_socket(socket.sock);\n}\n\ninline void ClientImpl::close_socket(Socket &socket) {\n  // If there are requests in flight in another thread, usually closing\n  // the socket will be fine and they will simply receive an error when\n  // using the closed socket, but it is still a bug since rarely the OS\n  // may reassign the socket id to be used for a new socket, and then\n  // suddenly they will be operating on a live socket that is different\n  // than the one they intended!\n  assert(socket_requests_in_flight_ == 0 ||\n         socket_requests_are_from_thread_ == std::this_thread::get_id());\n\n  // It is also a bug if this happens while SSL is still active\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  assert(socket.ssl == nullptr);\n#endif\n  if (socket.sock == INVALID_SOCKET) { return; }\n  detail::close_socket(socket.sock);\n  socket.sock = INVALID_SOCKET;\n}\n\ninline bool ClientImpl::read_response_line(Stream &strm, const Request &req,\n                                           Response &res) const {\n  std::array<char, 2048> buf{};\n\n  detail::stream_line_reader line_reader(strm, buf.data(), buf.size());\n\n  if (!line_reader.getline()) { return false; }\n\n#ifdef CPPHTTPLIB_ALLOW_LF_AS_LINE_TERMINATOR\n  thread_local const std::regex re(\"(HTTP/1\\\\.[01]) (\\\\d{3})(?: (.*?))?\\r?\\n\");\n#else\n  thread_local const std::regex re(\"(HTTP/1\\\\.[01]) (\\\\d{3})(?: (.*?))?\\r\\n\");\n#endif\n\n  std::cmatch m;\n  if (!std::regex_match(line_reader.ptr(), m, re)) {\n    return req.method == \"CONNECT\";\n  }\n  res.version = std::string(m[1]);\n  res.status = std::stoi(std::string(m[2]));\n  res.reason = std::string(m[3]);\n\n  // Ignore '100 Continue'\n  while (res.status == StatusCode::Continue_100) {\n    if (!line_reader.getline()) { return false; } // CRLF\n    if (!line_reader.getline()) { return false; } // next response line\n\n    if (!std::regex_match(line_reader.ptr(), m, re)) { return false; }\n    res.version = std::string(m[1]);\n    res.status = std::stoi(std::string(m[2]));\n    res.reason = std::string(m[3]);\n  }\n\n  return true;\n}\n\ninline bool ClientImpl::send(Request &req, Response &res, Error &error) {\n  std::lock_guard<std::recursive_mutex> request_mutex_guard(request_mutex_);\n  auto ret = send_(req, res, error);\n  if (error == Error::SSLPeerCouldBeClosed_) {\n    assert(!ret);\n    ret = send_(req, res, error);\n  }\n  return ret;\n}\n\ninline bool ClientImpl::send_(Request &req, Response &res, Error &error) {\n  {\n    std::lock_guard<std::mutex> guard(socket_mutex_);\n\n    // Set this to false immediately - if it ever gets set to true by the end of\n    // the request, we know another thread instructed us to close the socket.\n    socket_should_be_closed_when_request_is_done_ = false;\n\n    auto is_alive = false;\n    if (socket_.is_open()) {\n      is_alive = detail::is_socket_alive(socket_.sock);\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n      if (is_alive && is_ssl()) {\n        if (detail::is_ssl_peer_could_be_closed(socket_.ssl, socket_.sock)) {\n          is_alive = false;\n        }\n      }\n#endif\n\n      if (!is_alive) {\n        // Attempt to avoid sigpipe by shutting down non-gracefully if it seems\n        // like the other side has already closed the connection Also, there\n        // cannot be any requests in flight from other threads since we locked\n        // request_mutex_, so safe to close everything immediately\n        const bool shutdown_gracefully = false;\n        shutdown_ssl(socket_, shutdown_gracefully);\n        shutdown_socket(socket_);\n        close_socket(socket_);\n      }\n    }\n\n    if (!is_alive) {\n      if (!create_and_connect_socket(socket_, error)) { return false; }\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n      // TODO: refactoring\n      if (is_ssl()) {\n        auto &scli = static_cast<SSLClient &>(*this);\n        if (!proxy_host_.empty() && proxy_port_ != -1) {\n          auto success = false;\n          if (!scli.connect_with_proxy(socket_, req.start_time_, res, success,\n                                       error)) {\n            return success;\n          }\n        }\n\n        if (!scli.initialize_ssl(socket_, error)) { return false; }\n      }\n#endif\n    }\n\n    // Mark the current socket as being in use so that it cannot be closed by\n    // anyone else while this request is ongoing, even though we will be\n    // releasing the mutex.\n    if (socket_requests_in_flight_ > 1) {\n      assert(socket_requests_are_from_thread_ == std::this_thread::get_id());\n    }\n    socket_requests_in_flight_ += 1;\n    socket_requests_are_from_thread_ = std::this_thread::get_id();\n  }\n\n  for (const auto &header : default_headers_) {\n    if (req.headers.find(header.first) == req.headers.end()) {\n      req.headers.insert(header);\n    }\n  }\n\n  auto ret = false;\n  auto close_connection = !keep_alive_;\n\n  auto se = detail::scope_exit([&]() {\n    // Briefly lock mutex in order to mark that a request is no longer ongoing\n    std::lock_guard<std::mutex> guard(socket_mutex_);\n    socket_requests_in_flight_ -= 1;\n    if (socket_requests_in_flight_ <= 0) {\n      assert(socket_requests_in_flight_ == 0);\n      socket_requests_are_from_thread_ = std::thread::id();\n    }\n\n    if (socket_should_be_closed_when_request_is_done_ || close_connection ||\n        !ret) {\n      shutdown_ssl(socket_, true);\n      shutdown_socket(socket_);\n      close_socket(socket_);\n    }\n  });\n\n  ret = process_socket(socket_, req.start_time_, [&](Stream &strm) {\n    return handle_request(strm, req, res, close_connection, error);\n  });\n\n  if (!ret) {\n    if (error == Error::Success) { error = Error::Unknown; }\n  }\n\n  return ret;\n}\n\ninline Result ClientImpl::send(const Request &req) {\n  auto req2 = req;\n  return send_(std::move(req2));\n}\n\ninline Result ClientImpl::send_(Request &&req) {\n  auto res = detail::make_unique<Response>();\n  auto error = Error::Success;\n  auto ret = send(req, *res, error);\n  return Result{ret ? std::move(res) : nullptr, error, std::move(req.headers)};\n}\n\ninline bool ClientImpl::handle_request(Stream &strm, Request &req,\n                                       Response &res, bool close_connection,\n                                       Error &error) {\n  if (req.path.empty()) {\n    error = Error::Connection;\n    return false;\n  }\n\n  auto req_save = req;\n\n  bool ret;\n\n  if (!is_ssl() && !proxy_host_.empty() && proxy_port_ != -1) {\n    auto req2 = req;\n    req2.path = \"http://\" + host_and_port_ + req.path;\n    ret = process_request(strm, req2, res, close_connection, error);\n    req = req2;\n    req.path = req_save.path;\n  } else {\n    ret = process_request(strm, req, res, close_connection, error);\n  }\n\n  if (!ret) { return false; }\n\n  if (res.get_header_value(\"Connection\") == \"close\" ||\n      (res.version == \"HTTP/1.0\" && res.reason != \"Connection established\")) {\n    // TODO this requires a not-entirely-obvious chain of calls to be correct\n    // for this to be safe.\n\n    // This is safe to call because handle_request is only called by send_\n    // which locks the request mutex during the process. It would be a bug\n    // to call it from a different thread since it's a thread-safety issue\n    // to do these things to the socket if another thread is using the socket.\n    std::lock_guard<std::mutex> guard(socket_mutex_);\n    shutdown_ssl(socket_, true);\n    shutdown_socket(socket_);\n    close_socket(socket_);\n  }\n\n  if (300 < res.status && res.status < 400 && follow_location_) {\n    req = req_save;\n    ret = redirect(req, res, error);\n  }\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  if ((res.status == StatusCode::Unauthorized_401 ||\n       res.status == StatusCode::ProxyAuthenticationRequired_407) &&\n      req.authorization_count_ < 5) {\n    auto is_proxy = res.status == StatusCode::ProxyAuthenticationRequired_407;\n    const auto &username =\n        is_proxy ? proxy_digest_auth_username_ : digest_auth_username_;\n    const auto &password =\n        is_proxy ? proxy_digest_auth_password_ : digest_auth_password_;\n\n    if (!username.empty() && !password.empty()) {\n      std::map<std::string, std::string> auth;\n      if (detail::parse_www_authenticate(res, auth, is_proxy)) {\n        Request new_req = req;\n        new_req.authorization_count_ += 1;\n        new_req.headers.erase(is_proxy ? \"Proxy-Authorization\"\n                                       : \"Authorization\");\n        new_req.headers.insert(detail::make_digest_authentication_header(\n            req, auth, new_req.authorization_count_, detail::random_string(10),\n            username, password, is_proxy));\n\n        Response new_res;\n\n        ret = send(new_req, new_res, error);\n        if (ret) { res = new_res; }\n      }\n    }\n  }\n#endif\n\n  return ret;\n}\n\ninline bool ClientImpl::redirect(Request &req, Response &res, Error &error) {\n  if (req.redirect_count_ == 0) {\n    error = Error::ExceedRedirectCount;\n    return false;\n  }\n\n  auto location = res.get_header_value(\"location\");\n  if (location.empty()) { return false; }\n\n  thread_local const std::regex re(\n      R\"((?:(https?):)?(?://(?:\\[([a-fA-F\\d:]+)\\]|([^:/?#]+))(?::(\\d+))?)?([^?#]*)(\\?[^#]*)?(?:#.*)?)\");\n\n  std::smatch m;\n  if (!std::regex_match(location, m, re)) { return false; }\n\n  auto scheme = is_ssl() ? \"https\" : \"http\";\n\n  auto next_scheme = m[1].str();\n  auto next_host = m[2].str();\n  if (next_host.empty()) { next_host = m[3].str(); }\n  auto port_str = m[4].str();\n  auto next_path = m[5].str();\n  auto next_query = m[6].str();\n\n  auto next_port = port_;\n  if (!port_str.empty()) {\n    next_port = std::stoi(port_str);\n  } else if (!next_scheme.empty()) {\n    next_port = next_scheme == \"https\" ? 443 : 80;\n  }\n\n  if (next_scheme.empty()) { next_scheme = scheme; }\n  if (next_host.empty()) { next_host = host_; }\n  if (next_path.empty()) { next_path = \"/\"; }\n\n  auto path = detail::decode_url(next_path, true) + next_query;\n\n  if (next_scheme == scheme && next_host == host_ && next_port == port_) {\n    return detail::redirect(*this, req, res, path, location, error);\n  } else {\n    if (next_scheme == \"https\") {\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n      SSLClient cli(next_host, next_port);\n      cli.copy_settings(*this);\n      if (ca_cert_store_) { cli.set_ca_cert_store(ca_cert_store_); }\n      return detail::redirect(cli, req, res, path, location, error);\n#else\n      return false;\n#endif\n    } else {\n      ClientImpl cli(next_host, next_port);\n      cli.copy_settings(*this);\n      return detail::redirect(cli, req, res, path, location, error);\n    }\n  }\n}\n\ninline bool ClientImpl::write_content_with_provider(Stream &strm,\n                                                    const Request &req,\n                                                    Error &error) const {\n  auto is_shutting_down = []() { return false; };\n\n  if (req.is_chunked_content_provider_) {\n    // TODO: Brotli support\n    std::unique_ptr<detail::compressor> compressor;\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n    if (compress_) {\n      compressor = detail::make_unique<detail::gzip_compressor>();\n    } else\n#endif\n    {\n      compressor = detail::make_unique<detail::nocompressor>();\n    }\n\n    return detail::write_content_chunked(strm, req.content_provider_,\n                                         is_shutting_down, *compressor, error);\n  } else {\n    return detail::write_content(strm, req.content_provider_, 0,\n                                 req.content_length_, is_shutting_down, error);\n  }\n}\n\ninline bool ClientImpl::write_request(Stream &strm, Request &req,\n                                      bool close_connection, Error &error) {\n  // Prepare additional headers\n  if (close_connection) {\n    if (!req.has_header(\"Connection\")) {\n      req.set_header(\"Connection\", \"close\");\n    }\n  }\n\n  if (!req.has_header(\"Host\")) {\n    if (is_ssl()) {\n      if (port_ == 443) {\n        req.set_header(\"Host\", host_);\n      } else {\n        req.set_header(\"Host\", host_and_port_);\n      }\n    } else {\n      if (port_ == 80) {\n        req.set_header(\"Host\", host_);\n      } else {\n        req.set_header(\"Host\", host_and_port_);\n      }\n    }\n  }\n\n  if (!req.has_header(\"Accept\")) { req.set_header(\"Accept\", \"*/*\"); }\n\n  if (!req.content_receiver) {\n    if (!req.has_header(\"Accept-Encoding\")) {\n      std::string accept_encoding;\n#ifdef CPPHTTPLIB_BROTLI_SUPPORT\n      accept_encoding = \"br\";\n#endif\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n      if (!accept_encoding.empty()) { accept_encoding += \", \"; }\n      accept_encoding += \"gzip, deflate\";\n#endif\n#ifdef CPPHTTPLIB_ZSTD_SUPPORT\n      if (!accept_encoding.empty()) { accept_encoding += \", \"; }\n      accept_encoding += \"zstd\";\n#endif\n      req.set_header(\"Accept-Encoding\", accept_encoding);\n    }\n\n#ifndef CPPHTTPLIB_NO_DEFAULT_USER_AGENT\n    if (!req.has_header(\"User-Agent\")) {\n      auto agent = std::string(\"cpp-httplib/\") + CPPHTTPLIB_VERSION;\n      req.set_header(\"User-Agent\", agent);\n    }\n#endif\n  };\n\n  if (req.body.empty()) {\n    if (req.content_provider_) {\n      if (!req.is_chunked_content_provider_) {\n        if (!req.has_header(\"Content-Length\")) {\n          auto length = std::to_string(req.content_length_);\n          req.set_header(\"Content-Length\", length);\n        }\n      }\n    } else {\n      if (req.method == \"POST\" || req.method == \"PUT\" ||\n          req.method == \"PATCH\") {\n        req.set_header(\"Content-Length\", \"0\");\n      }\n    }\n  } else {\n    if (!req.has_header(\"Content-Type\")) {\n      req.set_header(\"Content-Type\", \"text/plain\");\n    }\n\n    if (!req.has_header(\"Content-Length\")) {\n      auto length = std::to_string(req.body.size());\n      req.set_header(\"Content-Length\", length);\n    }\n  }\n\n  if (!basic_auth_password_.empty() || !basic_auth_username_.empty()) {\n    if (!req.has_header(\"Authorization\")) {\n      req.headers.insert(make_basic_authentication_header(\n          basic_auth_username_, basic_auth_password_, false));\n    }\n  }\n\n  if (!proxy_basic_auth_username_.empty() &&\n      !proxy_basic_auth_password_.empty()) {\n    if (!req.has_header(\"Proxy-Authorization\")) {\n      req.headers.insert(make_basic_authentication_header(\n          proxy_basic_auth_username_, proxy_basic_auth_password_, true));\n    }\n  }\n\n  if (!bearer_token_auth_token_.empty()) {\n    if (!req.has_header(\"Authorization\")) {\n      req.headers.insert(make_bearer_token_authentication_header(\n          bearer_token_auth_token_, false));\n    }\n  }\n\n  if (!proxy_bearer_token_auth_token_.empty()) {\n    if (!req.has_header(\"Proxy-Authorization\")) {\n      req.headers.insert(make_bearer_token_authentication_header(\n          proxy_bearer_token_auth_token_, true));\n    }\n  }\n\n  // Request line and headers\n  {\n    detail::BufferStream bstrm;\n\n    const auto &path_with_query =\n        req.params.empty() ? req.path\n                           : append_query_params(req.path, req.params);\n\n    const auto &path =\n        url_encode_ ? detail::encode_url(path_with_query) : path_with_query;\n\n    detail::write_request_line(bstrm, req.method, path);\n\n    header_writer_(bstrm, req.headers);\n\n    // Flush buffer\n    auto &data = bstrm.get_buffer();\n    if (!detail::write_data(strm, data.data(), data.size())) {\n      error = Error::Write;\n      return false;\n    }\n  }\n\n  // Body\n  if (req.body.empty()) {\n    return write_content_with_provider(strm, req, error);\n  }\n\n  if (!detail::write_data(strm, req.body.data(), req.body.size())) {\n    error = Error::Write;\n    return false;\n  }\n\n  return true;\n}\n\ninline std::unique_ptr<Response> ClientImpl::send_with_content_provider(\n    Request &req, const char *body, size_t content_length,\n    ContentProvider content_provider,\n    ContentProviderWithoutLength content_provider_without_length,\n    const std::string &content_type, Error &error) {\n  if (!content_type.empty()) { req.set_header(\"Content-Type\", content_type); }\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n  if (compress_) { req.set_header(\"Content-Encoding\", \"gzip\"); }\n#endif\n\n#ifdef CPPHTTPLIB_ZLIB_SUPPORT\n  if (compress_ && !content_provider_without_length) {\n    // TODO: Brotli support\n    detail::gzip_compressor compressor;\n\n    if (content_provider) {\n      auto ok = true;\n      size_t offset = 0;\n      DataSink data_sink;\n\n      data_sink.write = [&](const char *data, size_t data_len) -> bool {\n        if (ok) {\n          auto last = offset + data_len == content_length;\n\n          auto ret = compressor.compress(\n              data, data_len, last,\n              [&](const char *compressed_data, size_t compressed_data_len) {\n                req.body.append(compressed_data, compressed_data_len);\n                return true;\n              });\n\n          if (ret) {\n            offset += data_len;\n          } else {\n            ok = false;\n          }\n        }\n        return ok;\n      };\n\n      while (ok && offset < content_length) {\n        if (!content_provider(offset, content_length - offset, data_sink)) {\n          error = Error::Canceled;\n          return nullptr;\n        }\n      }\n    } else {\n      if (!compressor.compress(body, content_length, true,\n                               [&](const char *data, size_t data_len) {\n                                 req.body.append(data, data_len);\n                                 return true;\n                               })) {\n        error = Error::Compression;\n        return nullptr;\n      }\n    }\n  } else\n#endif\n  {\n    if (content_provider) {\n      req.content_length_ = content_length;\n      req.content_provider_ = std::move(content_provider);\n      req.is_chunked_content_provider_ = false;\n    } else if (content_provider_without_length) {\n      req.content_length_ = 0;\n      req.content_provider_ = detail::ContentProviderAdapter(\n          std::move(content_provider_without_length));\n      req.is_chunked_content_provider_ = true;\n      req.set_header(\"Transfer-Encoding\", \"chunked\");\n    } else {\n      req.body.assign(body, content_length);\n    }\n  }\n\n  auto res = detail::make_unique<Response>();\n  return send(req, *res, error) ? std::move(res) : nullptr;\n}\n\ninline Result ClientImpl::send_with_content_provider(\n    const std::string &method, const std::string &path, const Headers &headers,\n    const char *body, size_t content_length, ContentProvider content_provider,\n    ContentProviderWithoutLength content_provider_without_length,\n    const std::string &content_type, Progress progress) {\n  Request req;\n  req.method = method;\n  req.headers = headers;\n  req.path = path;\n  req.progress = progress;\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  auto error = Error::Success;\n\n  auto res = send_with_content_provider(\n      req, body, content_length, std::move(content_provider),\n      std::move(content_provider_without_length), content_type, error);\n\n  return Result{std::move(res), error, std::move(req.headers)};\n}\n\ninline std::string\nClientImpl::adjust_host_string(const std::string &host) const {\n  if (host.find(':') != std::string::npos) { return \"[\" + host + \"]\"; }\n  return host;\n}\n\ninline bool ClientImpl::process_request(Stream &strm, Request &req,\n                                        Response &res, bool close_connection,\n                                        Error &error) {\n  // Send request\n  if (!write_request(strm, req, close_connection, error)) { return false; }\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n  if (is_ssl()) {\n    auto is_proxy_enabled = !proxy_host_.empty() && proxy_port_ != -1;\n    if (!is_proxy_enabled) {\n      if (detail::is_ssl_peer_could_be_closed(socket_.ssl, socket_.sock)) {\n        error = Error::SSLPeerCouldBeClosed_;\n        return false;\n      }\n    }\n  }\n#endif\n\n  // Receive response and headers\n  if (!read_response_line(strm, req, res) ||\n      !detail::read_headers(strm, res.headers)) {\n    error = Error::Read;\n    return false;\n  }\n\n  // Body\n  if ((res.status != StatusCode::NoContent_204) && req.method != \"HEAD\" &&\n      req.method != \"CONNECT\") {\n    auto redirect = 300 < res.status && res.status < 400 &&\n                    res.status != StatusCode::NotModified_304 &&\n                    follow_location_;\n\n    if (req.response_handler && !redirect) {\n      if (!req.response_handler(res)) {\n        error = Error::Canceled;\n        return false;\n      }\n    }\n\n    auto out =\n        req.content_receiver\n            ? static_cast<ContentReceiverWithProgress>(\n                  [&](const char *buf, size_t n, uint64_t off, uint64_t len) {\n                    if (redirect) { return true; }\n                    auto ret = req.content_receiver(buf, n, off, len);\n                    if (!ret) { error = Error::Canceled; }\n                    return ret;\n                  })\n            : static_cast<ContentReceiverWithProgress>(\n                  [&](const char *buf, size_t n, uint64_t /*off*/,\n                      uint64_t /*len*/) {\n                    assert(res.body.size() + n <= res.body.max_size());\n                    res.body.append(buf, n);\n                    return true;\n                  });\n\n    auto progress = [&](uint64_t current, uint64_t total) {\n      if (!req.progress || redirect) { return true; }\n      auto ret = req.progress(current, total);\n      if (!ret) { error = Error::Canceled; }\n      return ret;\n    };\n\n    if (res.has_header(\"Content-Length\")) {\n      if (!req.content_receiver) {\n        auto len = res.get_header_value_u64(\"Content-Length\");\n        if (len > res.body.max_size()) {\n          error = Error::Read;\n          return false;\n        }\n        res.body.reserve(static_cast<size_t>(len));\n      }\n    }\n\n    if (res.status != StatusCode::NotModified_304) {\n      int dummy_status;\n      if (!detail::read_content(strm, res, (std::numeric_limits<size_t>::max)(),\n                                dummy_status, std::move(progress),\n                                std::move(out), decompress_)) {\n        if (error != Error::Canceled) { error = Error::Read; }\n        return false;\n      }\n    }\n  }\n\n  // Log\n  if (logger_) { logger_(req, res); }\n\n  return true;\n}\n\ninline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(\n    const std::string &boundary, const MultipartFormDataItems &items,\n    const MultipartFormDataProviderItems &provider_items) const {\n  size_t cur_item = 0;\n  size_t cur_start = 0;\n  // cur_item and cur_start are copied to within the std::function and maintain\n  // state between successive calls\n  return [&, cur_item, cur_start](size_t offset,\n                                  DataSink &sink) mutable -> bool {\n    if (!offset && !items.empty()) {\n      sink.os << detail::serialize_multipart_formdata(items, boundary, false);\n      return true;\n    } else if (cur_item < provider_items.size()) {\n      if (!cur_start) {\n        const auto &begin = detail::serialize_multipart_formdata_item_begin(\n            provider_items[cur_item], boundary);\n        offset += begin.size();\n        cur_start = offset;\n        sink.os << begin;\n      }\n\n      DataSink cur_sink;\n      auto has_data = true;\n      cur_sink.write = sink.write;\n      cur_sink.done = [&]() { has_data = false; };\n\n      if (!provider_items[cur_item].provider(offset - cur_start, cur_sink)) {\n        return false;\n      }\n\n      if (!has_data) {\n        sink.os << detail::serialize_multipart_formdata_item_end();\n        cur_item++;\n        cur_start = 0;\n      }\n      return true;\n    } else {\n      sink.os << detail::serialize_multipart_formdata_finish(boundary);\n      sink.done();\n      return true;\n    }\n  };\n}\n\ninline bool ClientImpl::process_socket(\n    const Socket &socket,\n    std::chrono::time_point<std::chrono::steady_clock> start_time,\n    std::function<bool(Stream &strm)> callback) {\n  return detail::process_client_socket(\n      socket.sock, read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,\n      write_timeout_usec_, max_timeout_msec_, start_time, std::move(callback));\n}\n\ninline bool ClientImpl::is_ssl() const { return false; }\n\ninline Result ClientImpl::Get(const std::string &path) {\n  return Get(path, Headers(), Progress());\n}\n\ninline Result ClientImpl::Get(const std::string &path, Progress progress) {\n  return Get(path, Headers(), std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers) {\n  return Get(path, headers, Progress());\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers,\n                              Progress progress) {\n  Request req;\n  req.method = \"GET\";\n  req.path = path;\n  req.headers = headers;\n  req.progress = std::move(progress);\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  return send_(std::move(req));\n}\n\ninline Result ClientImpl::Get(const std::string &path,\n                              ContentReceiver content_receiver) {\n  return Get(path, Headers(), nullptr, std::move(content_receiver), nullptr);\n}\n\ninline Result ClientImpl::Get(const std::string &path,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  return Get(path, Headers(), nullptr, std::move(content_receiver),\n             std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers,\n                              ContentReceiver content_receiver) {\n  return Get(path, headers, nullptr, std::move(content_receiver), nullptr);\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  return Get(path, headers, nullptr, std::move(content_receiver),\n             std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path,\n                              ResponseHandler response_handler,\n                              ContentReceiver content_receiver) {\n  return Get(path, Headers(), std::move(response_handler),\n             std::move(content_receiver), nullptr);\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers,\n                              ResponseHandler response_handler,\n                              ContentReceiver content_receiver) {\n  return Get(path, headers, std::move(response_handler),\n             std::move(content_receiver), nullptr);\n}\n\ninline Result ClientImpl::Get(const std::string &path,\n                              ResponseHandler response_handler,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  return Get(path, Headers(), std::move(response_handler),\n             std::move(content_receiver), std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Headers &headers,\n                              ResponseHandler response_handler,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  Request req;\n  req.method = \"GET\";\n  req.path = path;\n  req.headers = headers;\n  req.response_handler = std::move(response_handler);\n  req.content_receiver =\n      [content_receiver](const char *data, size_t data_length,\n                         uint64_t /*offset*/, uint64_t /*total_length*/) {\n        return content_receiver(data, data_length);\n      };\n  req.progress = std::move(progress);\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  return send_(std::move(req));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Params &params,\n                              const Headers &headers, Progress progress) {\n  if (params.empty()) { return Get(path, headers); }\n\n  std::string path_with_query = append_query_params(path, params);\n  return Get(path_with_query, headers, std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Params &params,\n                              const Headers &headers,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  return Get(path, params, headers, nullptr, std::move(content_receiver),\n             std::move(progress));\n}\n\ninline Result ClientImpl::Get(const std::string &path, const Params &params,\n                              const Headers &headers,\n                              ResponseHandler response_handler,\n                              ContentReceiver content_receiver,\n                              Progress progress) {\n  if (params.empty()) {\n    return Get(path, headers, std::move(response_handler),\n               std::move(content_receiver), std::move(progress));\n  }\n\n  std::string path_with_query = append_query_params(path, params);\n  return Get(path_with_query, headers, std::move(response_handler),\n             std::move(content_receiver), std::move(progress));\n}\n\ninline Result ClientImpl::Head(const std::string &path) {\n  return Head(path, Headers());\n}\n\ninline Result ClientImpl::Head(const std::string &path,\n                               const Headers &headers) {\n  Request req;\n  req.method = \"HEAD\";\n  req.headers = headers;\n  req.path = path;\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  return send_(std::move(req));\n}\n\ninline Result ClientImpl::Post(const std::string &path) {\n  return Post(path, std::string(), std::string());\n}\n\ninline Result ClientImpl::Post(const std::string &path,\n                               const Headers &headers) {\n  return Post(path, headers, nullptr, 0, std::string());\n}\n\ninline Result ClientImpl::Post(const std::string &path, const char *body,\n                               size_t content_length,\n                               const std::string &content_type) {\n  return Post(path, Headers(), body, content_length, content_type, nullptr);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const char *body, size_t content_length,\n                               const std::string &content_type) {\n  return send_with_content_provider(\"POST\", path, headers, body, content_length,\n                                    nullptr, nullptr, content_type, nullptr);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const char *body, size_t content_length,\n                               const std::string &content_type,\n                               Progress progress) {\n  return send_with_content_provider(\"POST\", path, headers, body, content_length,\n                                    nullptr, nullptr, content_type, progress);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const std::string &body,\n                               const std::string &content_type) {\n  return Post(path, Headers(), body, content_type);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const std::string &body,\n                               const std::string &content_type,\n                               Progress progress) {\n  return Post(path, Headers(), body, content_type, progress);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const std::string &body,\n                               const std::string &content_type) {\n  return send_with_content_provider(\"POST\", path, headers, body.data(),\n                                    body.size(), nullptr, nullptr, content_type,\n                                    nullptr);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const std::string &body,\n                               const std::string &content_type,\n                               Progress progress) {\n  return send_with_content_provider(\"POST\", path, headers, body.data(),\n                                    body.size(), nullptr, nullptr, content_type,\n                                    progress);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Params &params) {\n  return Post(path, Headers(), params);\n}\n\ninline Result ClientImpl::Post(const std::string &path, size_t content_length,\n                               ContentProvider content_provider,\n                               const std::string &content_type) {\n  return Post(path, Headers(), content_length, std::move(content_provider),\n              content_type);\n}\n\ninline Result ClientImpl::Post(const std::string &path,\n                               ContentProviderWithoutLength content_provider,\n                               const std::string &content_type) {\n  return Post(path, Headers(), std::move(content_provider), content_type);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               size_t content_length,\n                               ContentProvider content_provider,\n                               const std::string &content_type) {\n  return send_with_content_provider(\"POST\", path, headers, nullptr,\n                                    content_length, std::move(content_provider),\n                                    nullptr, content_type, nullptr);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               ContentProviderWithoutLength content_provider,\n                               const std::string &content_type) {\n  return send_with_content_provider(\"POST\", path, headers, nullptr, 0, nullptr,\n                                    std::move(content_provider), content_type,\n                                    nullptr);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const Params &params) {\n  auto query = detail::params_to_query_str(params);\n  return Post(path, headers, query, \"application/x-www-form-urlencoded\");\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const Params &params, Progress progress) {\n  auto query = detail::params_to_query_str(params);\n  return Post(path, headers, query, \"application/x-www-form-urlencoded\",\n              progress);\n}\n\ninline Result ClientImpl::Post(const std::string &path,\n                               const MultipartFormDataItems &items) {\n  return Post(path, Headers(), items);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const MultipartFormDataItems &items) {\n  const auto &boundary = detail::make_multipart_data_boundary();\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  const auto &body = detail::serialize_multipart_formdata(items, boundary);\n  return Post(path, headers, body, content_type);\n}\n\ninline Result ClientImpl::Post(const std::string &path, const Headers &headers,\n                               const MultipartFormDataItems &items,\n                               const std::string &boundary) {\n  if (!detail::is_multipart_boundary_chars_valid(boundary)) {\n    return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};\n  }\n\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  const auto &body = detail::serialize_multipart_formdata(items, boundary);\n  return Post(path, headers, body, content_type);\n}\n\ninline Result\nClientImpl::Post(const std::string &path, const Headers &headers,\n                 const MultipartFormDataItems &items,\n                 const MultipartFormDataProviderItems &provider_items) {\n  const auto &boundary = detail::make_multipart_data_boundary();\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  return send_with_content_provider(\n      \"POST\", path, headers, nullptr, 0, nullptr,\n      get_multipart_content_provider(boundary, items, provider_items),\n      content_type, nullptr);\n}\n\ninline Result ClientImpl::Put(const std::string &path) {\n  return Put(path, std::string(), std::string());\n}\n\ninline Result ClientImpl::Put(const std::string &path, const char *body,\n                              size_t content_length,\n                              const std::string &content_type) {\n  return Put(path, Headers(), body, content_length, content_type);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const char *body, size_t content_length,\n                              const std::string &content_type) {\n  return send_with_content_provider(\"PUT\", path, headers, body, content_length,\n                                    nullptr, nullptr, content_type, nullptr);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const char *body, size_t content_length,\n                              const std::string &content_type,\n                              Progress progress) {\n  return send_with_content_provider(\"PUT\", path, headers, body, content_length,\n                                    nullptr, nullptr, content_type, progress);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const std::string &body,\n                              const std::string &content_type) {\n  return Put(path, Headers(), body, content_type);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const std::string &body,\n                              const std::string &content_type,\n                              Progress progress) {\n  return Put(path, Headers(), body, content_type, progress);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const std::string &body,\n                              const std::string &content_type) {\n  return send_with_content_provider(\"PUT\", path, headers, body.data(),\n                                    body.size(), nullptr, nullptr, content_type,\n                                    nullptr);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const std::string &body,\n                              const std::string &content_type,\n                              Progress progress) {\n  return send_with_content_provider(\"PUT\", path, headers, body.data(),\n                                    body.size(), nullptr, nullptr, content_type,\n                                    progress);\n}\n\ninline Result ClientImpl::Put(const std::string &path, size_t content_length,\n                              ContentProvider content_provider,\n                              const std::string &content_type) {\n  return Put(path, Headers(), content_length, std::move(content_provider),\n             content_type);\n}\n\ninline Result ClientImpl::Put(const std::string &path,\n                              ContentProviderWithoutLength content_provider,\n                              const std::string &content_type) {\n  return Put(path, Headers(), std::move(content_provider), content_type);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              size_t content_length,\n                              ContentProvider content_provider,\n                              const std::string &content_type) {\n  return send_with_content_provider(\"PUT\", path, headers, nullptr,\n                                    content_length, std::move(content_provider),\n                                    nullptr, content_type, nullptr);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              ContentProviderWithoutLength content_provider,\n                              const std::string &content_type) {\n  return send_with_content_provider(\"PUT\", path, headers, nullptr, 0, nullptr,\n                                    std::move(content_provider), content_type,\n                                    nullptr);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Params &params) {\n  return Put(path, Headers(), params);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const Params &params) {\n  auto query = detail::params_to_query_str(params);\n  return Put(path, headers, query, \"application/x-www-form-urlencoded\");\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const Params &params, Progress progress) {\n  auto query = detail::params_to_query_str(params);\n  return Put(path, headers, query, \"application/x-www-form-urlencoded\",\n             progress);\n}\n\ninline Result ClientImpl::Put(const std::string &path,\n                              const MultipartFormDataItems &items) {\n  return Put(path, Headers(), items);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const MultipartFormDataItems &items) {\n  const auto &boundary = detail::make_multipart_data_boundary();\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  const auto &body = detail::serialize_multipart_formdata(items, boundary);\n  return Put(path, headers, body, content_type);\n}\n\ninline Result ClientImpl::Put(const std::string &path, const Headers &headers,\n                              const MultipartFormDataItems &items,\n                              const std::string &boundary) {\n  if (!detail::is_multipart_boundary_chars_valid(boundary)) {\n    return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};\n  }\n\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  const auto &body = detail::serialize_multipart_formdata(items, boundary);\n  return Put(path, headers, body, content_type);\n}\n\ninline Result\nClientImpl::Put(const std::string &path, const Headers &headers,\n                const MultipartFormDataItems &items,\n                const MultipartFormDataProviderItems &provider_items) {\n  const auto &boundary = detail::make_multipart_data_boundary();\n  const auto &content_type =\n      detail::serialize_multipart_formdata_get_content_type(boundary);\n  return send_with_content_provider(\n      \"PUT\", path, headers, nullptr, 0, nullptr,\n      get_multipart_content_provider(boundary, items, provider_items),\n      content_type, nullptr);\n}\ninline Result ClientImpl::Patch(const std::string &path) {\n  return Patch(path, std::string(), std::string());\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const char *body,\n                                size_t content_length,\n                                const std::string &content_type) {\n  return Patch(path, Headers(), body, content_length, content_type);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const char *body,\n                                size_t content_length,\n                                const std::string &content_type,\n                                Progress progress) {\n  return Patch(path, Headers(), body, content_length, content_type, progress);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                const char *body, size_t content_length,\n                                const std::string &content_type) {\n  return Patch(path, headers, body, content_length, content_type, nullptr);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                const char *body, size_t content_length,\n                                const std::string &content_type,\n                                Progress progress) {\n  return send_with_content_provider(\"PATCH\", path, headers, body,\n                                    content_length, nullptr, nullptr,\n                                    content_type, progress);\n}\n\ninline Result ClientImpl::Patch(const std::string &path,\n                                const std::string &body,\n                                const std::string &content_type) {\n  return Patch(path, Headers(), body, content_type);\n}\n\ninline Result ClientImpl::Patch(const std::string &path,\n                                const std::string &body,\n                                const std::string &content_type,\n                                Progress progress) {\n  return Patch(path, Headers(), body, content_type, progress);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                const std::string &body,\n                                const std::string &content_type) {\n  return Patch(path, headers, body, content_type, nullptr);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                const std::string &body,\n                                const std::string &content_type,\n                                Progress progress) {\n  return send_with_content_provider(\"PATCH\", path, headers, body.data(),\n                                    body.size(), nullptr, nullptr, content_type,\n                                    progress);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, size_t content_length,\n                                ContentProvider content_provider,\n                                const std::string &content_type) {\n  return Patch(path, Headers(), content_length, std::move(content_provider),\n               content_type);\n}\n\ninline Result ClientImpl::Patch(const std::string &path,\n                                ContentProviderWithoutLength content_provider,\n                                const std::string &content_type) {\n  return Patch(path, Headers(), std::move(content_provider), content_type);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                size_t content_length,\n                                ContentProvider content_provider,\n                                const std::string &content_type) {\n  return send_with_content_provider(\"PATCH\", path, headers, nullptr,\n                                    content_length, std::move(content_provider),\n                                    nullptr, content_type, nullptr);\n}\n\ninline Result ClientImpl::Patch(const std::string &path, const Headers &headers,\n                                ContentProviderWithoutLength content_provider,\n                                const std::string &content_type) {\n  return send_with_content_provider(\"PATCH\", path, headers, nullptr, 0, nullptr,\n                                    std::move(content_provider), content_type,\n                                    nullptr);\n}\n\ninline Result ClientImpl::Delete(const std::string &path) {\n  return Delete(path, Headers(), std::string(), std::string());\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const Headers &headers) {\n  return Delete(path, headers, std::string(), std::string());\n}\n\ninline Result ClientImpl::Delete(const std::string &path, const char *body,\n                                 size_t content_length,\n                                 const std::string &content_type) {\n  return Delete(path, Headers(), body, content_length, content_type);\n}\n\ninline Result ClientImpl::Delete(const std::string &path, const char *body,\n                                 size_t content_length,\n                                 const std::string &content_type,\n                                 Progress progress) {\n  return Delete(path, Headers(), body, content_length, content_type, progress);\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const Headers &headers, const char *body,\n                                 size_t content_length,\n                                 const std::string &content_type) {\n  return Delete(path, headers, body, content_length, content_type, nullptr);\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const Headers &headers, const char *body,\n                                 size_t content_length,\n                                 const std::string &content_type,\n                                 Progress progress) {\n  Request req;\n  req.method = \"DELETE\";\n  req.headers = headers;\n  req.path = path;\n  req.progress = progress;\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  if (!content_type.empty()) { req.set_header(\"Content-Type\", content_type); }\n  req.body.assign(body, content_length);\n\n  return send_(std::move(req));\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const std::string &body,\n                                 const std::string &content_type) {\n  return Delete(path, Headers(), body.data(), body.size(), content_type);\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const std::string &body,\n                                 const std::string &content_type,\n                                 Progress progress) {\n  return Delete(path, Headers(), body.data(), body.size(), content_type,\n                progress);\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const Headers &headers,\n                                 const std::string &body,\n                                 const std::string &content_type) {\n  return Delete(path, headers, body.data(), body.size(), content_type);\n}\n\ninline Result ClientImpl::Delete(const std::string &path,\n                                 const Headers &headers,\n                                 const std::string &body,\n                                 const std::string &content_type,\n                                 Progress progress) {\n  return Delete(path, headers, body.data(), body.size(), content_type,\n                progress);\n}\n\ninline Result ClientImpl::Options(const std::string &path) {\n  return Options(path, Headers());\n}\n\ninline Result ClientImpl::Options(const std::string &path,\n                                  const Headers &headers) {\n  Request req;\n  req.method = \"OPTIONS\";\n  req.headers = headers;\n  req.path = path;\n  if (max_timeout_msec_ > 0) {\n    req.start_time_ = std::chrono::steady_clock::now();\n  }\n\n  return send_(std::move(req));\n}\n\ninline void ClientImpl::stop() {\n  std::lock_guard<std::mutex> guard(socket_mutex_);\n\n  // If there is anything ongoing right now, the ONLY thread-safe thing we can\n  // do is to shutdown_socket, so that threads using this socket suddenly\n  // discover they can't read/write any more and error out. Everything else\n  // (closing the socket, shutting ssl down) is unsafe because these actions are\n  // not thread-safe.\n  if (socket_requests_in_flight_ > 0) {\n    shutdown_socket(socket_);\n\n    // Aside from that, we set a flag for the socket to be closed when we're\n    // done.\n    socket_should_be_closed_when_request_is_done_ = true;\n    return;\n  }\n\n  // Otherwise, still holding the mutex, we can shut everything down ourselves\n  shutdown_ssl(socket_, true);\n  shutdown_socket(socket_);\n  close_socket(socket_);\n}\n\ninline std::string ClientImpl::host() const { return host_; }\n\ninline int ClientImpl::port() const { return port_; }\n\ninline size_t ClientImpl::is_socket_open() const {\n  std::lock_guard<std::mutex> guard(socket_mutex_);\n  return socket_.is_open();\n}\n\ninline socket_t ClientImpl::socket() const { return socket_.sock; }\n\ninline void ClientImpl::set_connection_timeout(time_t sec, time_t usec) {\n  connection_timeout_sec_ = sec;\n  connection_timeout_usec_ = usec;\n}\n\ninline void ClientImpl::set_read_timeout(time_t sec, time_t usec) {\n  read_timeout_sec_ = sec;\n  read_timeout_usec_ = usec;\n}\n\ninline void ClientImpl::set_write_timeout(time_t sec, time_t usec) {\n  write_timeout_sec_ = sec;\n  write_timeout_usec_ = usec;\n}\n\ninline void ClientImpl::set_max_timeout(time_t msec) {\n  max_timeout_msec_ = msec;\n}\n\ninline void ClientImpl::set_basic_auth(const std::string &username,\n                                       const std::string &password) {\n  basic_auth_username_ = username;\n  basic_auth_password_ = password;\n}\n\ninline void ClientImpl::set_bearer_token_auth(const std::string &token) {\n  bearer_token_auth_token_ = token;\n}\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void ClientImpl::set_digest_auth(const std::string &username,\n                                        const std::string &password) {\n  digest_auth_username_ = username;\n  digest_auth_password_ = password;\n}\n#endif\n\ninline void ClientImpl::set_keep_alive(bool on) { keep_alive_ = on; }\n\ninline void ClientImpl::set_follow_location(bool on) { follow_location_ = on; }\n\ninline void ClientImpl::set_url_encode(bool on) { url_encode_ = on; }\n\ninline void\nClientImpl::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {\n  addr_map_ = std::move(addr_map);\n}\n\ninline void ClientImpl::set_default_headers(Headers headers) {\n  default_headers_ = std::move(headers);\n}\n\ninline void ClientImpl::set_header_writer(\n    std::function<ssize_t(Stream &, Headers &)> const &writer) {\n  header_writer_ = writer;\n}\n\ninline void ClientImpl::set_address_family(int family) {\n  address_family_ = family;\n}\n\ninline void ClientImpl::set_tcp_nodelay(bool on) { tcp_nodelay_ = on; }\n\ninline void ClientImpl::set_ipv6_v6only(bool on) { ipv6_v6only_ = on; }\n\ninline void ClientImpl::set_socket_options(SocketOptions socket_options) {\n  socket_options_ = std::move(socket_options);\n}\n\ninline void ClientImpl::set_compress(bool on) { compress_ = on; }\n\ninline void ClientImpl::set_decompress(bool on) { decompress_ = on; }\n\ninline void ClientImpl::set_interface(const std::string &intf) {\n  interface_ = intf;\n}\n\ninline void ClientImpl::set_proxy(const std::string &host, int port) {\n  proxy_host_ = host;\n  proxy_port_ = port;\n}\n\ninline void ClientImpl::set_proxy_basic_auth(const std::string &username,\n                                             const std::string &password) {\n  proxy_basic_auth_username_ = username;\n  proxy_basic_auth_password_ = password;\n}\n\ninline void ClientImpl::set_proxy_bearer_token_auth(const std::string &token) {\n  proxy_bearer_token_auth_token_ = token;\n}\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void ClientImpl::set_proxy_digest_auth(const std::string &username,\n                                              const std::string &password) {\n  proxy_digest_auth_username_ = username;\n  proxy_digest_auth_password_ = password;\n}\n\ninline void ClientImpl::set_ca_cert_path(const std::string &ca_cert_file_path,\n                                         const std::string &ca_cert_dir_path) {\n  ca_cert_file_path_ = ca_cert_file_path;\n  ca_cert_dir_path_ = ca_cert_dir_path;\n}\n\ninline void ClientImpl::set_ca_cert_store(X509_STORE *ca_cert_store) {\n  if (ca_cert_store && ca_cert_store != ca_cert_store_) {\n    ca_cert_store_ = ca_cert_store;\n  }\n}\n\ninline X509_STORE *ClientImpl::create_ca_cert_store(const char *ca_cert,\n                                                    std::size_t size) const {\n  auto mem = BIO_new_mem_buf(ca_cert, static_cast<int>(size));\n  auto se = detail::scope_exit([&] { BIO_free_all(mem); });\n  if (!mem) { return nullptr; }\n\n  auto inf = PEM_X509_INFO_read_bio(mem, nullptr, nullptr, nullptr);\n  if (!inf) { return nullptr; }\n\n  auto cts = X509_STORE_new();\n  if (cts) {\n    for (auto i = 0; i < static_cast<int>(sk_X509_INFO_num(inf)); i++) {\n      auto itmp = sk_X509_INFO_value(inf, i);\n      if (!itmp) { continue; }\n\n      if (itmp->x509) { X509_STORE_add_cert(cts, itmp->x509); }\n      if (itmp->crl) { X509_STORE_add_crl(cts, itmp->crl); }\n    }\n  }\n\n  sk_X509_INFO_pop_free(inf, X509_INFO_free);\n  return cts;\n}\n\ninline void ClientImpl::enable_server_certificate_verification(bool enabled) {\n  server_certificate_verification_ = enabled;\n}\n\ninline void ClientImpl::enable_server_hostname_verification(bool enabled) {\n  server_hostname_verification_ = enabled;\n}\n\ninline void ClientImpl::set_server_certificate_verifier(\n    std::function<SSLVerifierResponse(SSL *ssl)> verifier) {\n  server_certificate_verifier_ = verifier;\n}\n#endif\n\ninline void ClientImpl::set_logger(Logger logger) {\n  logger_ = std::move(logger);\n}\n\n/*\n * SSL Implementation\n */\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\nnamespace detail {\n\ntemplate <typename U, typename V>\ninline SSL *ssl_new(socket_t sock, SSL_CTX *ctx, std::mutex &ctx_mutex,\n                    U SSL_connect_or_accept, V setup) {\n  SSL *ssl = nullptr;\n  {\n    std::lock_guard<std::mutex> guard(ctx_mutex);\n    ssl = SSL_new(ctx);\n  }\n\n  if (ssl) {\n    set_nonblocking(sock, true);\n    auto bio = BIO_new_socket(static_cast<int>(sock), BIO_NOCLOSE);\n    BIO_set_nbio(bio, 1);\n    SSL_set_bio(ssl, bio, bio);\n\n    if (!setup(ssl) || SSL_connect_or_accept(ssl) != 1) {\n      SSL_shutdown(ssl);\n      {\n        std::lock_guard<std::mutex> guard(ctx_mutex);\n        SSL_free(ssl);\n      }\n      set_nonblocking(sock, false);\n      return nullptr;\n    }\n    BIO_set_nbio(bio, 0);\n    set_nonblocking(sock, false);\n  }\n\n  return ssl;\n}\n\ninline void ssl_delete(std::mutex &ctx_mutex, SSL *ssl, socket_t sock,\n                       bool shutdown_gracefully) {\n  // sometimes we may want to skip this to try to avoid SIGPIPE if we know\n  // the remote has closed the network connection\n  // Note that it is not always possible to avoid SIGPIPE, this is merely a\n  // best-efforts.\n  if (shutdown_gracefully) {\n    (void)(sock);\n    // SSL_shutdown() returns 0 on first call (indicating close_notify alert\n    // sent) and 1 on subsequent call (indicating close_notify alert received)\n    if (SSL_shutdown(ssl) == 0) {\n      // Expected to return 1, but even if it doesn't, we free ssl\n      SSL_shutdown(ssl);\n    }\n  }\n\n  std::lock_guard<std::mutex> guard(ctx_mutex);\n  SSL_free(ssl);\n}\n\ntemplate <typename U>\nbool ssl_connect_or_accept_nonblocking(socket_t sock, SSL *ssl,\n                                       U ssl_connect_or_accept,\n                                       time_t timeout_sec,\n                                       time_t timeout_usec) {\n  auto res = 0;\n  while ((res = ssl_connect_or_accept(ssl)) != 1) {\n    auto err = SSL_get_error(ssl, res);\n    switch (err) {\n    case SSL_ERROR_WANT_READ:\n      if (select_read(sock, timeout_sec, timeout_usec) > 0) { continue; }\n      break;\n    case SSL_ERROR_WANT_WRITE:\n      if (select_write(sock, timeout_sec, timeout_usec) > 0) { continue; }\n      break;\n    default: break;\n    }\n    return false;\n  }\n  return true;\n}\n\ntemplate <typename T>\ninline bool process_server_socket_ssl(\n    const std::atomic<socket_t> &svr_sock, SSL *ssl, socket_t sock,\n    size_t keep_alive_max_count, time_t keep_alive_timeout_sec,\n    time_t read_timeout_sec, time_t read_timeout_usec, time_t write_timeout_sec,\n    time_t write_timeout_usec, T callback) {\n  return process_server_socket_core(\n      svr_sock, sock, keep_alive_max_count, keep_alive_timeout_sec,\n      [&](bool close_connection, bool &connection_closed) {\n        SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec,\n                             write_timeout_sec, write_timeout_usec);\n        return callback(strm, close_connection, connection_closed);\n      });\n}\n\ntemplate <typename T>\ninline bool process_client_socket_ssl(\n    SSL *ssl, socket_t sock, time_t read_timeout_sec, time_t read_timeout_usec,\n    time_t write_timeout_sec, time_t write_timeout_usec,\n    time_t max_timeout_msec,\n    std::chrono::time_point<std::chrono::steady_clock> start_time, T callback) {\n  SSLSocketStream strm(sock, ssl, read_timeout_sec, read_timeout_usec,\n                       write_timeout_sec, write_timeout_usec, max_timeout_msec,\n                       start_time);\n  return callback(strm);\n}\n\n// SSL socket stream implementation\ninline SSLSocketStream::SSLSocketStream(\n    socket_t sock, SSL *ssl, time_t read_timeout_sec, time_t read_timeout_usec,\n    time_t write_timeout_sec, time_t write_timeout_usec,\n    time_t max_timeout_msec,\n    std::chrono::time_point<std::chrono::steady_clock> start_time)\n    : sock_(sock), ssl_(ssl), read_timeout_sec_(read_timeout_sec),\n      read_timeout_usec_(read_timeout_usec),\n      write_timeout_sec_(write_timeout_sec),\n      write_timeout_usec_(write_timeout_usec),\n      max_timeout_msec_(max_timeout_msec), start_time(start_time) {\n  SSL_clear_mode(ssl, SSL_MODE_AUTO_RETRY);\n}\n\ninline SSLSocketStream::~SSLSocketStream() = default;\n\ninline bool SSLSocketStream::is_readable() const {\n  return SSL_pending(ssl_) > 0;\n}\n\ninline bool SSLSocketStream::wait_readable() const {\n  if (max_timeout_msec_ <= 0) {\n    return select_read(sock_, read_timeout_sec_, read_timeout_usec_) > 0;\n  }\n\n  time_t read_timeout_sec;\n  time_t read_timeout_usec;\n  calc_actual_timeout(max_timeout_msec_, duration(), read_timeout_sec_,\n                      read_timeout_usec_, read_timeout_sec, read_timeout_usec);\n\n  return select_read(sock_, read_timeout_sec, read_timeout_usec) > 0;\n}\n\ninline bool SSLSocketStream::wait_writable() const {\n  return select_write(sock_, write_timeout_sec_, write_timeout_usec_) > 0 &&\n         is_socket_alive(sock_) && !is_ssl_peer_could_be_closed(ssl_, sock_);\n}\n\ninline ssize_t SSLSocketStream::read(char *ptr, size_t size) {\n  if (SSL_pending(ssl_) > 0) {\n    return SSL_read(ssl_, ptr, static_cast<int>(size));\n  } else if (wait_readable()) {\n    auto ret = SSL_read(ssl_, ptr, static_cast<int>(size));\n    if (ret < 0) {\n      auto err = SSL_get_error(ssl_, ret);\n      auto n = 1000;\n#ifdef _WIN32\n      while (--n >= 0 && (err == SSL_ERROR_WANT_READ ||\n                          (err == SSL_ERROR_SYSCALL &&\n                           WSAGetLastError() == WSAETIMEDOUT))) {\n#else\n      while (--n >= 0 && err == SSL_ERROR_WANT_READ) {\n#endif\n        if (SSL_pending(ssl_) > 0) {\n          return SSL_read(ssl_, ptr, static_cast<int>(size));\n        } else if (wait_readable()) {\n          std::this_thread::sleep_for(std::chrono::microseconds{10});\n          ret = SSL_read(ssl_, ptr, static_cast<int>(size));\n          if (ret >= 0) { return ret; }\n          err = SSL_get_error(ssl_, ret);\n        } else {\n          return -1;\n        }\n      }\n    }\n    return ret;\n  } else {\n    return -1;\n  }\n}\n\ninline ssize_t SSLSocketStream::write(const char *ptr, size_t size) {\n  if (wait_writable()) {\n    auto handle_size = static_cast<int>(\n        std::min<size_t>(size, (std::numeric_limits<int>::max)()));\n\n    auto ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));\n    if (ret < 0) {\n      auto err = SSL_get_error(ssl_, ret);\n      auto n = 1000;\n#ifdef _WIN32\n      while (--n >= 0 && (err == SSL_ERROR_WANT_WRITE ||\n                          (err == SSL_ERROR_SYSCALL &&\n                           WSAGetLastError() == WSAETIMEDOUT))) {\n#else\n      while (--n >= 0 && err == SSL_ERROR_WANT_WRITE) {\n#endif\n        if (wait_writable()) {\n          std::this_thread::sleep_for(std::chrono::microseconds{10});\n          ret = SSL_write(ssl_, ptr, static_cast<int>(handle_size));\n          if (ret >= 0) { return ret; }\n          err = SSL_get_error(ssl_, ret);\n        } else {\n          return -1;\n        }\n      }\n    }\n    return ret;\n  }\n  return -1;\n}\n\ninline void SSLSocketStream::get_remote_ip_and_port(std::string &ip,\n                                                    int &port) const {\n  detail::get_remote_ip_and_port(sock_, ip, port);\n}\n\ninline void SSLSocketStream::get_local_ip_and_port(std::string &ip,\n                                                   int &port) const {\n  detail::get_local_ip_and_port(sock_, ip, port);\n}\n\ninline socket_t SSLSocketStream::socket() const { return sock_; }\n\ninline time_t SSLSocketStream::duration() const {\n  return std::chrono::duration_cast<std::chrono::milliseconds>(\n             std::chrono::steady_clock::now() - start_time)\n      .count();\n}\n\n} // namespace detail\n\n// SSL HTTP server implementation\ninline SSLServer::SSLServer(const char *cert_path, const char *private_key_path,\n                            const char *client_ca_cert_file_path,\n                            const char *client_ca_cert_dir_path,\n                            const char *private_key_password) {\n  ctx_ = SSL_CTX_new(TLS_server_method());\n\n  if (ctx_) {\n    SSL_CTX_set_options(ctx_,\n                        SSL_OP_NO_COMPRESSION |\n                            SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);\n\n    SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION);\n\n    if (private_key_password != nullptr && (private_key_password[0] != '\\0')) {\n      SSL_CTX_set_default_passwd_cb_userdata(\n          ctx_,\n          reinterpret_cast<void *>(const_cast<char *>(private_key_password)));\n    }\n\n    if (SSL_CTX_use_certificate_chain_file(ctx_, cert_path) != 1 ||\n        SSL_CTX_use_PrivateKey_file(ctx_, private_key_path, SSL_FILETYPE_PEM) !=\n            1 ||\n        SSL_CTX_check_private_key(ctx_) != 1) {\n      SSL_CTX_free(ctx_);\n      ctx_ = nullptr;\n    } else if (client_ca_cert_file_path || client_ca_cert_dir_path) {\n      SSL_CTX_load_verify_locations(ctx_, client_ca_cert_file_path,\n                                    client_ca_cert_dir_path);\n\n      SSL_CTX_set_verify(\n          ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);\n    }\n  }\n}\n\ninline SSLServer::SSLServer(X509 *cert, EVP_PKEY *private_key,\n                            X509_STORE *client_ca_cert_store) {\n  ctx_ = SSL_CTX_new(TLS_server_method());\n\n  if (ctx_) {\n    SSL_CTX_set_options(ctx_,\n                        SSL_OP_NO_COMPRESSION |\n                            SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);\n\n    SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION);\n\n    if (SSL_CTX_use_certificate(ctx_, cert) != 1 ||\n        SSL_CTX_use_PrivateKey(ctx_, private_key) != 1) {\n      SSL_CTX_free(ctx_);\n      ctx_ = nullptr;\n    } else if (client_ca_cert_store) {\n      SSL_CTX_set_cert_store(ctx_, client_ca_cert_store);\n\n      SSL_CTX_set_verify(\n          ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);\n    }\n  }\n}\n\ninline SSLServer::SSLServer(\n    const std::function<bool(SSL_CTX &ssl_ctx)> &setup_ssl_ctx_callback) {\n  ctx_ = SSL_CTX_new(TLS_method());\n  if (ctx_) {\n    if (!setup_ssl_ctx_callback(*ctx_)) {\n      SSL_CTX_free(ctx_);\n      ctx_ = nullptr;\n    }\n  }\n}\n\ninline SSLServer::~SSLServer() {\n  if (ctx_) { SSL_CTX_free(ctx_); }\n}\n\ninline bool SSLServer::is_valid() const { return ctx_; }\n\ninline SSL_CTX *SSLServer::ssl_context() const { return ctx_; }\n\ninline void SSLServer::update_certs(X509 *cert, EVP_PKEY *private_key,\n                                    X509_STORE *client_ca_cert_store) {\n\n  std::lock_guard<std::mutex> guard(ctx_mutex_);\n\n  SSL_CTX_use_certificate(ctx_, cert);\n  SSL_CTX_use_PrivateKey(ctx_, private_key);\n\n  if (client_ca_cert_store != nullptr) {\n    SSL_CTX_set_cert_store(ctx_, client_ca_cert_store);\n  }\n}\n\ninline bool SSLServer::process_and_close_socket(socket_t sock) {\n  auto ssl = detail::ssl_new(\n      sock, ctx_, ctx_mutex_,\n      [&](SSL *ssl2) {\n        return detail::ssl_connect_or_accept_nonblocking(\n            sock, ssl2, SSL_accept, read_timeout_sec_, read_timeout_usec_);\n      },\n      [](SSL * /*ssl2*/) { return true; });\n\n  auto ret = false;\n  if (ssl) {\n    std::string remote_addr;\n    int remote_port = 0;\n    detail::get_remote_ip_and_port(sock, remote_addr, remote_port);\n\n    std::string local_addr;\n    int local_port = 0;\n    detail::get_local_ip_and_port(sock, local_addr, local_port);\n\n    ret = detail::process_server_socket_ssl(\n        svr_sock_, ssl, sock, keep_alive_max_count_, keep_alive_timeout_sec_,\n        read_timeout_sec_, read_timeout_usec_, write_timeout_sec_,\n        write_timeout_usec_,\n        [&](Stream &strm, bool close_connection, bool &connection_closed) {\n          return process_request(strm, remote_addr, remote_port, local_addr,\n                                 local_port, close_connection,\n                                 connection_closed,\n                                 [&](Request &req) { req.ssl = ssl; });\n        });\n\n    // Shutdown gracefully if the result seemed successful, non-gracefully if\n    // the connection appeared to be closed.\n    const bool shutdown_gracefully = ret;\n    detail::ssl_delete(ctx_mutex_, ssl, sock, shutdown_gracefully);\n  }\n\n  detail::shutdown_socket(sock);\n  detail::close_socket(sock);\n  return ret;\n}\n\n// SSL HTTP client implementation\ninline SSLClient::SSLClient(const std::string &host)\n    : SSLClient(host, 443, std::string(), std::string()) {}\n\ninline SSLClient::SSLClient(const std::string &host, int port)\n    : SSLClient(host, port, std::string(), std::string()) {}\n\ninline SSLClient::SSLClient(const std::string &host, int port,\n                            const std::string &client_cert_path,\n                            const std::string &client_key_path,\n                            const std::string &private_key_password)\n    : ClientImpl(host, port, client_cert_path, client_key_path) {\n  ctx_ = SSL_CTX_new(TLS_client_method());\n\n  SSL_CTX_set_min_proto_version(ctx_, TLS1_2_VERSION);\n\n  detail::split(&host_[0], &host_[host_.size()], '.',\n                [&](const char *b, const char *e) {\n                  host_components_.emplace_back(b, e);\n                });\n\n  if (!client_cert_path.empty() && !client_key_path.empty()) {\n    if (!private_key_password.empty()) {\n      SSL_CTX_set_default_passwd_cb_userdata(\n          ctx_, reinterpret_cast<void *>(\n                    const_cast<char *>(private_key_password.c_str())));\n    }\n\n    if (SSL_CTX_use_certificate_file(ctx_, client_cert_path.c_str(),\n                                     SSL_FILETYPE_PEM) != 1 ||\n        SSL_CTX_use_PrivateKey_file(ctx_, client_key_path.c_str(),\n                                    SSL_FILETYPE_PEM) != 1) {\n      SSL_CTX_free(ctx_);\n      ctx_ = nullptr;\n    }\n  }\n}\n\ninline SSLClient::SSLClient(const std::string &host, int port,\n                            X509 *client_cert, EVP_PKEY *client_key,\n                            const std::string &private_key_password)\n    : ClientImpl(host, port) {\n  ctx_ = SSL_CTX_new(TLS_client_method());\n\n  detail::split(&host_[0], &host_[host_.size()], '.',\n                [&](const char *b, const char *e) {\n                  host_components_.emplace_back(b, e);\n                });\n\n  if (client_cert != nullptr && client_key != nullptr) {\n    if (!private_key_password.empty()) {\n      SSL_CTX_set_default_passwd_cb_userdata(\n          ctx_, reinterpret_cast<void *>(\n                    const_cast<char *>(private_key_password.c_str())));\n    }\n\n    if (SSL_CTX_use_certificate(ctx_, client_cert) != 1 ||\n        SSL_CTX_use_PrivateKey(ctx_, client_key) != 1) {\n      SSL_CTX_free(ctx_);\n      ctx_ = nullptr;\n    }\n  }\n}\n\ninline SSLClient::~SSLClient() {\n  if (ctx_) { SSL_CTX_free(ctx_); }\n  // Make sure to shut down SSL since shutdown_ssl will resolve to the\n  // base function rather than the derived function once we get to the\n  // base class destructor, and won't free the SSL (causing a leak).\n  shutdown_ssl_impl(socket_, true);\n}\n\ninline bool SSLClient::is_valid() const { return ctx_; }\n\ninline void SSLClient::set_ca_cert_store(X509_STORE *ca_cert_store) {\n  if (ca_cert_store) {\n    if (ctx_) {\n      if (SSL_CTX_get_cert_store(ctx_) != ca_cert_store) {\n        // Free memory allocated for old cert and use new store `ca_cert_store`\n        SSL_CTX_set_cert_store(ctx_, ca_cert_store);\n      }\n    } else {\n      X509_STORE_free(ca_cert_store);\n    }\n  }\n}\n\ninline void SSLClient::load_ca_cert_store(const char *ca_cert,\n                                          std::size_t size) {\n  set_ca_cert_store(ClientImpl::create_ca_cert_store(ca_cert, size));\n}\n\ninline long SSLClient::get_openssl_verify_result() const {\n  return verify_result_;\n}\n\ninline SSL_CTX *SSLClient::ssl_context() const { return ctx_; }\n\ninline bool SSLClient::create_and_connect_socket(Socket &socket, Error &error) {\n  return is_valid() && ClientImpl::create_and_connect_socket(socket, error);\n}\n\n// Assumes that socket_mutex_ is locked and that there are no requests in flight\ninline bool SSLClient::connect_with_proxy(\n    Socket &socket,\n    std::chrono::time_point<std::chrono::steady_clock> start_time,\n    Response &res, bool &success, Error &error) {\n  success = true;\n  Response proxy_res;\n  if (!detail::process_client_socket(\n          socket.sock, read_timeout_sec_, read_timeout_usec_,\n          write_timeout_sec_, write_timeout_usec_, max_timeout_msec_,\n          start_time, [&](Stream &strm) {\n            Request req2;\n            req2.method = \"CONNECT\";\n            req2.path = host_and_port_;\n            if (max_timeout_msec_ > 0) {\n              req2.start_time_ = std::chrono::steady_clock::now();\n            }\n            return process_request(strm, req2, proxy_res, false, error);\n          })) {\n    // Thread-safe to close everything because we are assuming there are no\n    // requests in flight\n    shutdown_ssl(socket, true);\n    shutdown_socket(socket);\n    close_socket(socket);\n    success = false;\n    return false;\n  }\n\n  if (proxy_res.status == StatusCode::ProxyAuthenticationRequired_407) {\n    if (!proxy_digest_auth_username_.empty() &&\n        !proxy_digest_auth_password_.empty()) {\n      std::map<std::string, std::string> auth;\n      if (detail::parse_www_authenticate(proxy_res, auth, true)) {\n        proxy_res = Response();\n        if (!detail::process_client_socket(\n                socket.sock, read_timeout_sec_, read_timeout_usec_,\n                write_timeout_sec_, write_timeout_usec_, max_timeout_msec_,\n                start_time, [&](Stream &strm) {\n                  Request req3;\n                  req3.method = \"CONNECT\";\n                  req3.path = host_and_port_;\n                  req3.headers.insert(detail::make_digest_authentication_header(\n                      req3, auth, 1, detail::random_string(10),\n                      proxy_digest_auth_username_, proxy_digest_auth_password_,\n                      true));\n                  if (max_timeout_msec_ > 0) {\n                    req3.start_time_ = std::chrono::steady_clock::now();\n                  }\n                  return process_request(strm, req3, proxy_res, false, error);\n                })) {\n          // Thread-safe to close everything because we are assuming there are\n          // no requests in flight\n          shutdown_ssl(socket, true);\n          shutdown_socket(socket);\n          close_socket(socket);\n          success = false;\n          return false;\n        }\n      }\n    }\n  }\n\n  // If status code is not 200, proxy request is failed.\n  // Set error to ProxyConnection and return proxy response\n  // as the response of the request\n  if (proxy_res.status != StatusCode::OK_200) {\n    error = Error::ProxyConnection;\n    res = std::move(proxy_res);\n    // Thread-safe to close everything because we are assuming there are\n    // no requests in flight\n    shutdown_ssl(socket, true);\n    shutdown_socket(socket);\n    close_socket(socket);\n    return false;\n  }\n\n  return true;\n}\n\ninline bool SSLClient::load_certs() {\n  auto ret = true;\n\n  std::call_once(initialize_cert_, [&]() {\n    std::lock_guard<std::mutex> guard(ctx_mutex_);\n    if (!ca_cert_file_path_.empty()) {\n      if (!SSL_CTX_load_verify_locations(ctx_, ca_cert_file_path_.c_str(),\n                                         nullptr)) {\n        ret = false;\n      }\n    } else if (!ca_cert_dir_path_.empty()) {\n      if (!SSL_CTX_load_verify_locations(ctx_, nullptr,\n                                         ca_cert_dir_path_.c_str())) {\n        ret = false;\n      }\n    } else {\n      auto loaded = false;\n#ifdef _WIN32\n      loaded =\n          detail::load_system_certs_on_windows(SSL_CTX_get_cert_store(ctx_));\n#elif defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN) && defined(__APPLE__)\n#if TARGET_OS_OSX\n      loaded = detail::load_system_certs_on_macos(SSL_CTX_get_cert_store(ctx_));\n#endif // TARGET_OS_OSX\n#endif // _WIN32\n      if (!loaded) { SSL_CTX_set_default_verify_paths(ctx_); }\n    }\n  });\n\n  return ret;\n}\n\ninline bool SSLClient::initialize_ssl(Socket &socket, Error &error) {\n  auto ssl = detail::ssl_new(\n      socket.sock, ctx_, ctx_mutex_,\n      [&](SSL *ssl2) {\n        if (server_certificate_verification_) {\n          if (!load_certs()) {\n            error = Error::SSLLoadingCerts;\n            return false;\n          }\n          SSL_set_verify(ssl2, SSL_VERIFY_NONE, nullptr);\n        }\n\n        if (!detail::ssl_connect_or_accept_nonblocking(\n                socket.sock, ssl2, SSL_connect, connection_timeout_sec_,\n                connection_timeout_usec_)) {\n          error = Error::SSLConnection;\n          return false;\n        }\n\n        if (server_certificate_verification_) {\n          auto verification_status = SSLVerifierResponse::NoDecisionMade;\n\n          if (server_certificate_verifier_) {\n            verification_status = server_certificate_verifier_(ssl2);\n          }\n\n          if (verification_status == SSLVerifierResponse::CertificateRejected) {\n            error = Error::SSLServerVerification;\n            return false;\n          }\n\n          if (verification_status == SSLVerifierResponse::NoDecisionMade) {\n            verify_result_ = SSL_get_verify_result(ssl2);\n\n            if (verify_result_ != X509_V_OK) {\n              error = Error::SSLServerVerification;\n              return false;\n            }\n\n            auto server_cert = SSL_get1_peer_certificate(ssl2);\n            auto se = detail::scope_exit([&] { X509_free(server_cert); });\n\n            if (server_cert == nullptr) {\n              error = Error::SSLServerVerification;\n              return false;\n            }\n\n            if (server_hostname_verification_) {\n              if (!verify_host(server_cert)) {\n                error = Error::SSLServerHostnameVerification;\n                return false;\n              }\n            }\n          }\n        }\n\n        return true;\n      },\n      [&](SSL *ssl2) {\n#if defined(OPENSSL_IS_BORINGSSL)\n        SSL_set_tlsext_host_name(ssl2, host_.c_str());\n#else\n        // NOTE: Direct call instead of using the OpenSSL macro to suppress\n        // -Wold-style-cast warning\n        SSL_ctrl(ssl2, SSL_CTRL_SET_TLSEXT_HOSTNAME, TLSEXT_NAMETYPE_host_name,\n                 static_cast<void *>(const_cast<char *>(host_.c_str())));\n#endif\n        return true;\n      });\n\n  if (ssl) {\n    socket.ssl = ssl;\n    return true;\n  }\n\n  shutdown_socket(socket);\n  close_socket(socket);\n  return false;\n}\n\ninline void SSLClient::shutdown_ssl(Socket &socket, bool shutdown_gracefully) {\n  shutdown_ssl_impl(socket, shutdown_gracefully);\n}\n\ninline void SSLClient::shutdown_ssl_impl(Socket &socket,\n                                         bool shutdown_gracefully) {\n  if (socket.sock == INVALID_SOCKET) {\n    assert(socket.ssl == nullptr);\n    return;\n  }\n  if (socket.ssl) {\n    detail::ssl_delete(ctx_mutex_, socket.ssl, socket.sock,\n                       shutdown_gracefully);\n    socket.ssl = nullptr;\n  }\n  assert(socket.ssl == nullptr);\n}\n\ninline bool SSLClient::process_socket(\n    const Socket &socket,\n    std::chrono::time_point<std::chrono::steady_clock> start_time,\n    std::function<bool(Stream &strm)> callback) {\n  assert(socket.ssl);\n  return detail::process_client_socket_ssl(\n      socket.ssl, socket.sock, read_timeout_sec_, read_timeout_usec_,\n      write_timeout_sec_, write_timeout_usec_, max_timeout_msec_, start_time,\n      std::move(callback));\n}\n\ninline bool SSLClient::is_ssl() const { return true; }\n\ninline bool SSLClient::verify_host(X509 *server_cert) const {\n  /* Quote from RFC2818 section 3.1 \"Server Identity\"\n\n     If a subjectAltName extension of type dNSName is present, that MUST\n     be used as the identity. Otherwise, the (most specific) Common Name\n     field in the Subject field of the certificate MUST be used. Although\n     the use of the Common Name is existing practice, it is deprecated and\n     Certification Authorities are encouraged to use the dNSName instead.\n\n     Matching is performed using the matching rules specified by\n     [RFC2459].  If more than one identity of a given type is present in\n     the certificate (e.g., more than one dNSName name, a match in any one\n     of the set is considered acceptable.) Names may contain the wildcard\n     character * which is considered to match any single domain name\n     component or component fragment. E.g., *.a.com matches foo.a.com but\n     not bar.foo.a.com. f*.com matches foo.com but not bar.com.\n\n     In some cases, the URI is specified as an IP address rather than a\n     hostname. In this case, the iPAddress subjectAltName must be present\n     in the certificate and must exactly match the IP in the URI.\n\n  */\n  return verify_host_with_subject_alt_name(server_cert) ||\n         verify_host_with_common_name(server_cert);\n}\n\ninline bool\nSSLClient::verify_host_with_subject_alt_name(X509 *server_cert) const {\n  auto ret = false;\n\n  auto type = GEN_DNS;\n\n  struct in6_addr addr6 = {};\n  struct in_addr addr = {};\n  size_t addr_len = 0;\n\n#ifndef __MINGW32__\n  if (inet_pton(AF_INET6, host_.c_str(), &addr6)) {\n    type = GEN_IPADD;\n    addr_len = sizeof(struct in6_addr);\n  } else if (inet_pton(AF_INET, host_.c_str(), &addr)) {\n    type = GEN_IPADD;\n    addr_len = sizeof(struct in_addr);\n  }\n#endif\n\n  auto alt_names = static_cast<const struct stack_st_GENERAL_NAME *>(\n      X509_get_ext_d2i(server_cert, NID_subject_alt_name, nullptr, nullptr));\n\n  if (alt_names) {\n    auto dsn_matched = false;\n    auto ip_matched = false;\n\n    auto count = sk_GENERAL_NAME_num(alt_names);\n\n    for (decltype(count) i = 0; i < count && !dsn_matched; i++) {\n      auto val = sk_GENERAL_NAME_value(alt_names, i);\n      if (val->type == type) {\n        auto name =\n            reinterpret_cast<const char *>(ASN1_STRING_get0_data(val->d.ia5));\n        auto name_len = static_cast<size_t>(ASN1_STRING_length(val->d.ia5));\n\n        switch (type) {\n        case GEN_DNS: dsn_matched = check_host_name(name, name_len); break;\n\n        case GEN_IPADD:\n          if (!memcmp(&addr6, name, addr_len) ||\n              !memcmp(&addr, name, addr_len)) {\n            ip_matched = true;\n          }\n          break;\n        }\n      }\n    }\n\n    if (dsn_matched || ip_matched) { ret = true; }\n  }\n\n  GENERAL_NAMES_free(const_cast<STACK_OF(GENERAL_NAME) *>(\n      reinterpret_cast<const STACK_OF(GENERAL_NAME) *>(alt_names)));\n  return ret;\n}\n\ninline bool SSLClient::verify_host_with_common_name(X509 *server_cert) const {\n  const auto subject_name = X509_get_subject_name(server_cert);\n\n  if (subject_name != nullptr) {\n    char name[BUFSIZ];\n    auto name_len = X509_NAME_get_text_by_NID(subject_name, NID_commonName,\n                                              name, sizeof(name));\n\n    if (name_len != -1) {\n      return check_host_name(name, static_cast<size_t>(name_len));\n    }\n  }\n\n  return false;\n}\n\ninline bool SSLClient::check_host_name(const char *pattern,\n                                       size_t pattern_len) const {\n  if (host_.size() == pattern_len && host_ == pattern) { return true; }\n\n  // Wildcard match\n  // https://bugs.launchpad.net/ubuntu/+source/firefox-3.0/+bug/376484\n  std::vector<std::string> pattern_components;\n  detail::split(&pattern[0], &pattern[pattern_len], '.',\n                [&](const char *b, const char *e) {\n                  pattern_components.emplace_back(b, e);\n                });\n\n  if (host_components_.size() != pattern_components.size()) { return false; }\n\n  auto itr = pattern_components.begin();\n  for (const auto &h : host_components_) {\n    auto &p = *itr;\n    if (p != h && p != \"*\") {\n      auto partial_match = (p.size() > 0 && p[p.size() - 1] == '*' &&\n                            !p.compare(0, p.size() - 1, h));\n      if (!partial_match) { return false; }\n    }\n    ++itr;\n  }\n\n  return true;\n}\n#endif\n\n// Universal client implementation\ninline Client::Client(const std::string &scheme_host_port)\n    : Client(scheme_host_port, std::string(), std::string()) {}\n\ninline Client::Client(const std::string &scheme_host_port,\n                      const std::string &client_cert_path,\n                      const std::string &client_key_path) {\n  const static std::regex re(\n      R\"((?:([a-z]+):\\/\\/)?(?:\\[([a-fA-F\\d:]+)\\]|([^:/?#]+))(?::(\\d+))?)\");\n\n  std::smatch m;\n  if (std::regex_match(scheme_host_port, m, re)) {\n    auto scheme = m[1].str();\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n    if (!scheme.empty() && (scheme != \"http\" && scheme != \"https\")) {\n#else\n    if (!scheme.empty() && scheme != \"http\") {\n#endif\n#ifndef CPPHTTPLIB_NO_EXCEPTIONS\n      std::string msg = \"'\" + scheme + \"' scheme is not supported.\";\n      throw std::invalid_argument(msg);\n#endif\n      return;\n    }\n\n    auto is_ssl = scheme == \"https\";\n\n    auto host = m[2].str();\n    if (host.empty()) { host = m[3].str(); }\n\n    auto port_str = m[4].str();\n    auto port = !port_str.empty() ? std::stoi(port_str) : (is_ssl ? 443 : 80);\n\n    if (is_ssl) {\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\n      cli_ = detail::make_unique<SSLClient>(host, port, client_cert_path,\n                                            client_key_path);\n      is_ssl_ = is_ssl;\n#endif\n    } else {\n      cli_ = detail::make_unique<ClientImpl>(host, port, client_cert_path,\n                                             client_key_path);\n    }\n  } else {\n    // NOTE: Update TEST(UniversalClientImplTest, Ipv6LiteralAddress)\n    // if port param below changes.\n    cli_ = detail::make_unique<ClientImpl>(scheme_host_port, 80,\n                                           client_cert_path, client_key_path);\n  }\n} // namespace detail\n\ninline Client::Client(const std::string &host, int port)\n    : cli_(detail::make_unique<ClientImpl>(host, port)) {}\n\ninline Client::Client(const std::string &host, int port,\n                      const std::string &client_cert_path,\n                      const std::string &client_key_path)\n    : cli_(detail::make_unique<ClientImpl>(host, port, client_cert_path,\n                                           client_key_path)) {}\n\ninline Client::~Client() = default;\n\ninline bool Client::is_valid() const {\n  return cli_ != nullptr && cli_->is_valid();\n}\n\ninline Result Client::Get(const std::string &path) { return cli_->Get(path); }\ninline Result Client::Get(const std::string &path, const Headers &headers) {\n  return cli_->Get(path, headers);\n}\ninline Result Client::Get(const std::string &path, Progress progress) {\n  return cli_->Get(path, std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Headers &headers,\n                          Progress progress) {\n  return cli_->Get(path, headers, std::move(progress));\n}\ninline Result Client::Get(const std::string &path,\n                          ContentReceiver content_receiver) {\n  return cli_->Get(path, std::move(content_receiver));\n}\ninline Result Client::Get(const std::string &path, const Headers &headers,\n                          ContentReceiver content_receiver) {\n  return cli_->Get(path, headers, std::move(content_receiver));\n}\ninline Result Client::Get(const std::string &path,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, std::move(content_receiver), std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Headers &headers,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, headers, std::move(content_receiver),\n                   std::move(progress));\n}\ninline Result Client::Get(const std::string &path,\n                          ResponseHandler response_handler,\n                          ContentReceiver content_receiver) {\n  return cli_->Get(path, std::move(response_handler),\n                   std::move(content_receiver));\n}\ninline Result Client::Get(const std::string &path, const Headers &headers,\n                          ResponseHandler response_handler,\n                          ContentReceiver content_receiver) {\n  return cli_->Get(path, headers, std::move(response_handler),\n                   std::move(content_receiver));\n}\ninline Result Client::Get(const std::string &path,\n                          ResponseHandler response_handler,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, std::move(response_handler),\n                   std::move(content_receiver), std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Headers &headers,\n                          ResponseHandler response_handler,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, headers, std::move(response_handler),\n                   std::move(content_receiver), std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Params &params,\n                          const Headers &headers, Progress progress) {\n  return cli_->Get(path, params, headers, std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Params &params,\n                          const Headers &headers,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, params, headers, std::move(content_receiver),\n                   std::move(progress));\n}\ninline Result Client::Get(const std::string &path, const Params &params,\n                          const Headers &headers,\n                          ResponseHandler response_handler,\n                          ContentReceiver content_receiver, Progress progress) {\n  return cli_->Get(path, params, headers, std::move(response_handler),\n                   std::move(content_receiver), std::move(progress));\n}\n\ninline Result Client::Head(const std::string &path) { return cli_->Head(path); }\ninline Result Client::Head(const std::string &path, const Headers &headers) {\n  return cli_->Head(path, headers);\n}\n\ninline Result Client::Post(const std::string &path) { return cli_->Post(path); }\ninline Result Client::Post(const std::string &path, const Headers &headers) {\n  return cli_->Post(path, headers);\n}\ninline Result Client::Post(const std::string &path, const char *body,\n                           size_t content_length,\n                           const std::string &content_type) {\n  return cli_->Post(path, body, content_length, content_type);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const char *body, size_t content_length,\n                           const std::string &content_type) {\n  return cli_->Post(path, headers, body, content_length, content_type);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const char *body, size_t content_length,\n                           const std::string &content_type, Progress progress) {\n  return cli_->Post(path, headers, body, content_length, content_type,\n                    progress);\n}\ninline Result Client::Post(const std::string &path, const std::string &body,\n                           const std::string &content_type) {\n  return cli_->Post(path, body, content_type);\n}\ninline Result Client::Post(const std::string &path, const std::string &body,\n                           const std::string &content_type, Progress progress) {\n  return cli_->Post(path, body, content_type, progress);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const std::string &body,\n                           const std::string &content_type) {\n  return cli_->Post(path, headers, body, content_type);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const std::string &body,\n                           const std::string &content_type, Progress progress) {\n  return cli_->Post(path, headers, body, content_type, progress);\n}\ninline Result Client::Post(const std::string &path, size_t content_length,\n                           ContentProvider content_provider,\n                           const std::string &content_type) {\n  return cli_->Post(path, content_length, std::move(content_provider),\n                    content_type);\n}\ninline Result Client::Post(const std::string &path,\n                           ContentProviderWithoutLength content_provider,\n                           const std::string &content_type) {\n  return cli_->Post(path, std::move(content_provider), content_type);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           size_t content_length,\n                           ContentProvider content_provider,\n                           const std::string &content_type) {\n  return cli_->Post(path, headers, content_length, std::move(content_provider),\n                    content_type);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           ContentProviderWithoutLength content_provider,\n                           const std::string &content_type) {\n  return cli_->Post(path, headers, std::move(content_provider), content_type);\n}\ninline Result Client::Post(const std::string &path, const Params &params) {\n  return cli_->Post(path, params);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const Params &params) {\n  return cli_->Post(path, headers, params);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const Params &params, Progress progress) {\n  return cli_->Post(path, headers, params, progress);\n}\ninline Result Client::Post(const std::string &path,\n                           const MultipartFormDataItems &items) {\n  return cli_->Post(path, items);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const MultipartFormDataItems &items) {\n  return cli_->Post(path, headers, items);\n}\ninline Result Client::Post(const std::string &path, const Headers &headers,\n                           const MultipartFormDataItems &items,\n                           const std::string &boundary) {\n  return cli_->Post(path, headers, items, boundary);\n}\ninline Result\nClient::Post(const std::string &path, const Headers &headers,\n             const MultipartFormDataItems &items,\n             const MultipartFormDataProviderItems &provider_items) {\n  return cli_->Post(path, headers, items, provider_items);\n}\ninline Result Client::Put(const std::string &path) { return cli_->Put(path); }\ninline Result Client::Put(const std::string &path, const char *body,\n                          size_t content_length,\n                          const std::string &content_type) {\n  return cli_->Put(path, body, content_length, content_type);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const char *body, size_t content_length,\n                          const std::string &content_type) {\n  return cli_->Put(path, headers, body, content_length, content_type);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const char *body, size_t content_length,\n                          const std::string &content_type, Progress progress) {\n  return cli_->Put(path, headers, body, content_length, content_type, progress);\n}\ninline Result Client::Put(const std::string &path, const std::string &body,\n                          const std::string &content_type) {\n  return cli_->Put(path, body, content_type);\n}\ninline Result Client::Put(const std::string &path, const std::string &body,\n                          const std::string &content_type, Progress progress) {\n  return cli_->Put(path, body, content_type, progress);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const std::string &body,\n                          const std::string &content_type) {\n  return cli_->Put(path, headers, body, content_type);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const std::string &body,\n                          const std::string &content_type, Progress progress) {\n  return cli_->Put(path, headers, body, content_type, progress);\n}\ninline Result Client::Put(const std::string &path, size_t content_length,\n                          ContentProvider content_provider,\n                          const std::string &content_type) {\n  return cli_->Put(path, content_length, std::move(content_provider),\n                   content_type);\n}\ninline Result Client::Put(const std::string &path,\n                          ContentProviderWithoutLength content_provider,\n                          const std::string &content_type) {\n  return cli_->Put(path, std::move(content_provider), content_type);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          size_t content_length,\n                          ContentProvider content_provider,\n                          const std::string &content_type) {\n  return cli_->Put(path, headers, content_length, std::move(content_provider),\n                   content_type);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          ContentProviderWithoutLength content_provider,\n                          const std::string &content_type) {\n  return cli_->Put(path, headers, std::move(content_provider), content_type);\n}\ninline Result Client::Put(const std::string &path, const Params &params) {\n  return cli_->Put(path, params);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const Params &params) {\n  return cli_->Put(path, headers, params);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const Params &params, Progress progress) {\n  return cli_->Put(path, headers, params, progress);\n}\ninline Result Client::Put(const std::string &path,\n                          const MultipartFormDataItems &items) {\n  return cli_->Put(path, items);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const MultipartFormDataItems &items) {\n  return cli_->Put(path, headers, items);\n}\ninline Result Client::Put(const std::string &path, const Headers &headers,\n                          const MultipartFormDataItems &items,\n                          const std::string &boundary) {\n  return cli_->Put(path, headers, items, boundary);\n}\ninline Result\nClient::Put(const std::string &path, const Headers &headers,\n            const MultipartFormDataItems &items,\n            const MultipartFormDataProviderItems &provider_items) {\n  return cli_->Put(path, headers, items, provider_items);\n}\ninline Result Client::Patch(const std::string &path) {\n  return cli_->Patch(path);\n}\ninline Result Client::Patch(const std::string &path, const char *body,\n                            size_t content_length,\n                            const std::string &content_type) {\n  return cli_->Patch(path, body, content_length, content_type);\n}\ninline Result Client::Patch(const std::string &path, const char *body,\n                            size_t content_length,\n                            const std::string &content_type,\n                            Progress progress) {\n  return cli_->Patch(path, body, content_length, content_type, progress);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            const char *body, size_t content_length,\n                            const std::string &content_type) {\n  return cli_->Patch(path, headers, body, content_length, content_type);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            const char *body, size_t content_length,\n                            const std::string &content_type,\n                            Progress progress) {\n  return cli_->Patch(path, headers, body, content_length, content_type,\n                     progress);\n}\ninline Result Client::Patch(const std::string &path, const std::string &body,\n                            const std::string &content_type) {\n  return cli_->Patch(path, body, content_type);\n}\ninline Result Client::Patch(const std::string &path, const std::string &body,\n                            const std::string &content_type,\n                            Progress progress) {\n  return cli_->Patch(path, body, content_type, progress);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            const std::string &body,\n                            const std::string &content_type) {\n  return cli_->Patch(path, headers, body, content_type);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            const std::string &body,\n                            const std::string &content_type,\n                            Progress progress) {\n  return cli_->Patch(path, headers, body, content_type, progress);\n}\ninline Result Client::Patch(const std::string &path, size_t content_length,\n                            ContentProvider content_provider,\n                            const std::string &content_type) {\n  return cli_->Patch(path, content_length, std::move(content_provider),\n                     content_type);\n}\ninline Result Client::Patch(const std::string &path,\n                            ContentProviderWithoutLength content_provider,\n                            const std::string &content_type) {\n  return cli_->Patch(path, std::move(content_provider), content_type);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            size_t content_length,\n                            ContentProvider content_provider,\n                            const std::string &content_type) {\n  return cli_->Patch(path, headers, content_length, std::move(content_provider),\n                     content_type);\n}\ninline Result Client::Patch(const std::string &path, const Headers &headers,\n                            ContentProviderWithoutLength content_provider,\n                            const std::string &content_type) {\n  return cli_->Patch(path, headers, std::move(content_provider), content_type);\n}\ninline Result Client::Delete(const std::string &path) {\n  return cli_->Delete(path);\n}\ninline Result Client::Delete(const std::string &path, const Headers &headers) {\n  return cli_->Delete(path, headers);\n}\ninline Result Client::Delete(const std::string &path, const char *body,\n                             size_t content_length,\n                             const std::string &content_type) {\n  return cli_->Delete(path, body, content_length, content_type);\n}\ninline Result Client::Delete(const std::string &path, const char *body,\n                             size_t content_length,\n                             const std::string &content_type,\n                             Progress progress) {\n  return cli_->Delete(path, body, content_length, content_type, progress);\n}\ninline Result Client::Delete(const std::string &path, const Headers &headers,\n                             const char *body, size_t content_length,\n                             const std::string &content_type) {\n  return cli_->Delete(path, headers, body, content_length, content_type);\n}\ninline Result Client::Delete(const std::string &path, const Headers &headers,\n                             const char *body, size_t content_length,\n                             const std::string &content_type,\n                             Progress progress) {\n  return cli_->Delete(path, headers, body, content_length, content_type,\n                      progress);\n}\ninline Result Client::Delete(const std::string &path, const std::string &body,\n                             const std::string &content_type) {\n  return cli_->Delete(path, body, content_type);\n}\ninline Result Client::Delete(const std::string &path, const std::string &body,\n                             const std::string &content_type,\n                             Progress progress) {\n  return cli_->Delete(path, body, content_type, progress);\n}\ninline Result Client::Delete(const std::string &path, const Headers &headers,\n                             const std::string &body,\n                             const std::string &content_type) {\n  return cli_->Delete(path, headers, body, content_type);\n}\ninline Result Client::Delete(const std::string &path, const Headers &headers,\n                             const std::string &body,\n                             const std::string &content_type,\n                             Progress progress) {\n  return cli_->Delete(path, headers, body, content_type, progress);\n}\ninline Result Client::Options(const std::string &path) {\n  return cli_->Options(path);\n}\ninline Result Client::Options(const std::string &path, const Headers &headers) {\n  return cli_->Options(path, headers);\n}\n\ninline bool Client::send(Request &req, Response &res, Error &error) {\n  return cli_->send(req, res, error);\n}\n\ninline Result Client::send(const Request &req) { return cli_->send(req); }\n\ninline void Client::stop() { cli_->stop(); }\n\ninline std::string Client::host() const { return cli_->host(); }\n\ninline int Client::port() const { return cli_->port(); }\n\ninline size_t Client::is_socket_open() const { return cli_->is_socket_open(); }\n\ninline socket_t Client::socket() const { return cli_->socket(); }\n\ninline void\nClient::set_hostname_addr_map(std::map<std::string, std::string> addr_map) {\n  cli_->set_hostname_addr_map(std::move(addr_map));\n}\n\ninline void Client::set_default_headers(Headers headers) {\n  cli_->set_default_headers(std::move(headers));\n}\n\ninline void Client::set_header_writer(\n    std::function<ssize_t(Stream &, Headers &)> const &writer) {\n  cli_->set_header_writer(writer);\n}\n\ninline void Client::set_address_family(int family) {\n  cli_->set_address_family(family);\n}\n\ninline void Client::set_tcp_nodelay(bool on) { cli_->set_tcp_nodelay(on); }\n\ninline void Client::set_socket_options(SocketOptions socket_options) {\n  cli_->set_socket_options(std::move(socket_options));\n}\n\ninline void Client::set_connection_timeout(time_t sec, time_t usec) {\n  cli_->set_connection_timeout(sec, usec);\n}\n\ninline void Client::set_read_timeout(time_t sec, time_t usec) {\n  cli_->set_read_timeout(sec, usec);\n}\n\ninline void Client::set_write_timeout(time_t sec, time_t usec) {\n  cli_->set_write_timeout(sec, usec);\n}\n\ninline void Client::set_basic_auth(const std::string &username,\n                                   const std::string &password) {\n  cli_->set_basic_auth(username, password);\n}\ninline void Client::set_bearer_token_auth(const std::string &token) {\n  cli_->set_bearer_token_auth(token);\n}\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void Client::set_digest_auth(const std::string &username,\n                                    const std::string &password) {\n  cli_->set_digest_auth(username, password);\n}\n#endif\n\ninline void Client::set_keep_alive(bool on) { cli_->set_keep_alive(on); }\ninline void Client::set_follow_location(bool on) {\n  cli_->set_follow_location(on);\n}\n\ninline void Client::set_url_encode(bool on) { cli_->set_url_encode(on); }\n\ninline void Client::set_compress(bool on) { cli_->set_compress(on); }\n\ninline void Client::set_decompress(bool on) { cli_->set_decompress(on); }\n\ninline void Client::set_interface(const std::string &intf) {\n  cli_->set_interface(intf);\n}\n\ninline void Client::set_proxy(const std::string &host, int port) {\n  cli_->set_proxy(host, port);\n}\ninline void Client::set_proxy_basic_auth(const std::string &username,\n                                         const std::string &password) {\n  cli_->set_proxy_basic_auth(username, password);\n}\ninline void Client::set_proxy_bearer_token_auth(const std::string &token) {\n  cli_->set_proxy_bearer_token_auth(token);\n}\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void Client::set_proxy_digest_auth(const std::string &username,\n                                          const std::string &password) {\n  cli_->set_proxy_digest_auth(username, password);\n}\n#endif\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void Client::enable_server_certificate_verification(bool enabled) {\n  cli_->enable_server_certificate_verification(enabled);\n}\n\ninline void Client::enable_server_hostname_verification(bool enabled) {\n  cli_->enable_server_hostname_verification(enabled);\n}\n\ninline void Client::set_server_certificate_verifier(\n    std::function<SSLVerifierResponse(SSL *ssl)> verifier) {\n  cli_->set_server_certificate_verifier(verifier);\n}\n#endif\n\ninline void Client::set_logger(Logger logger) {\n  cli_->set_logger(std::move(logger));\n}\n\n#ifdef CPPHTTPLIB_OPENSSL_SUPPORT\ninline void Client::set_ca_cert_path(const std::string &ca_cert_file_path,\n                                     const std::string &ca_cert_dir_path) {\n  cli_->set_ca_cert_path(ca_cert_file_path, ca_cert_dir_path);\n}\n\ninline void Client::set_ca_cert_store(X509_STORE *ca_cert_store) {\n  if (is_ssl_) {\n    static_cast<SSLClient &>(*cli_).set_ca_cert_store(ca_cert_store);\n  } else {\n    cli_->set_ca_cert_store(ca_cert_store);\n  }\n}\n\ninline void Client::load_ca_cert_store(const char *ca_cert, std::size_t size) {\n  set_ca_cert_store(cli_->create_ca_cert_store(ca_cert, size));\n}\n\ninline long Client::get_openssl_verify_result() const {\n  if (is_ssl_) {\n    return static_cast<SSLClient &>(*cli_).get_openssl_verify_result();\n  }\n  return -1; // NOTE: -1 doesn't match any of X509_V_ERR_???\n}\n\ninline SSL_CTX *Client::ssl_context() const {\n  if (is_ssl_) { return static_cast<SSLClient &>(*cli_).ssl_context(); }\n  return nullptr;\n}\n#endif\n\n// ----------------------------------------------------------------------------\n\n} // namespace httplib\n\n#endif // CPPHTTPLIB_HTTPLIB_H"
  },
  {
    "path": "examples/server/server.cpp",
    "content": "#include \"common.h\"\n#include \"common-whisper.h\"\n\n#include \"whisper.h\"\n#include \"httplib.h\"\n#include \"json.hpp\"\n\n#include <cfloat>\n#include <chrono>\n#include <cmath>\n#include <cstdio>\n#include <fstream>\n#include <sstream>\n#include <string>\n#include <thread>\n#include <vector>\n#include <memory>\n#include <csignal>\n#include <atomic>\n#include <functional>\n#include <cstdlib>\n#if defined (_WIN32)\n#include <windows.h>\n#endif\n\nusing namespace httplib;\nusing json = nlohmann::ordered_json;\n\nenum server_state {\n    SERVER_STATE_LOADING_MODEL,  // Server is starting up, model not fully loaded yet\n    SERVER_STATE_READY,          // Server is ready and model is loaded\n};\n\nnamespace {\n\n// output formats\nconst std::string json_format   = \"json\";\nconst std::string text_format   = \"text\";\nconst std::string srt_format    = \"srt\";\nconst std::string vjson_format  = \"verbose_json\";\nconst std::string vtt_format    = \"vtt\";\n\nstd::function<void(int)> shutdown_handler;\nstd::atomic_flag is_terminating = ATOMIC_FLAG_INIT;\n\ninline void signal_handler(int signal) {\n    if (is_terminating.test_and_set()) {\n        // in case it hangs, we can force terminate the server by hitting Ctrl+C twice\n        // this is for better developer experience, we can remove when the server is stable enough\n        fprintf(stderr, \"Received second interrupt, terminating immediately.\\n\");\n        exit(1);\n    }\n\n    shutdown_handler(signal);\n}\n\nstruct server_params\n{\n    std::string hostname = \"127.0.0.1\";\n    std::string public_path = \"examples/server/public\";\n    std::string request_path = \"\";\n    std::string inference_path = \"/inference\";\n    std::string tmp_dir = \".\";\n\n    int32_t port          = 8080;\n    int32_t read_timeout  = 600;\n    int32_t write_timeout = 600;\n\n    bool ffmpeg_converter = false;\n};\n\nstruct whisper_params {\n    int32_t n_threads     = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t n_processors  = 1;\n    int32_t offset_t_ms   = 0;\n    int32_t offset_n      = 0;\n    int32_t duration_ms   = 0;\n    int32_t progress_step = 5;\n    int32_t max_context   = -1;\n    int32_t max_len       = 0;\n    int32_t best_of       = 2;\n    int32_t beam_size     = -1;\n    int32_t audio_ctx     = 0;\n\n    float word_thold      =  0.01f;\n    float entropy_thold   =  2.40f;\n    float logprob_thold   = -1.00f;\n    float temperature     =  0.00f;\n    float temperature_inc =  0.20f;\n    float no_speech_thold = 0.6f;\n\n    bool debug_mode      = false;\n    bool translate       = false;\n    bool detect_language = false;\n    bool diarize         = false;\n    bool tinydiarize     = false;\n    bool split_on_word   = false;\n    bool no_fallback     = false;\n    bool print_special   = false;\n    bool print_colors    = false;\n    bool print_realtime  = false;\n    bool print_progress  = false;\n    bool no_timestamps   = false;\n    bool use_gpu         = true;\n    bool flash_attn      = true;\n    int32_t gpu_device   = 0;\n    bool suppress_nst    = false;\n    bool no_context      = true;\n    bool no_language_probabilities = false;\n\n    std::string language        = \"en\";\n    std::string prompt          = \"\";\n    std::string font_path       = \"/System/Library/Fonts/Supplemental/Courier New Bold.ttf\";\n    std::string model           = \"models/ggml-base.en.bin\";\n\n    std::string response_format     = json_format;\n\n    // [TDRZ] speaker turn string\n    std::string tdrz_speaker_turn = \" [SPEAKER_TURN]\"; // TODO: set from command line\n\n    std::string openvino_encode_device = \"CPU\";\n\n    std::string dtw = \"\";\n\n    // Voice Activity Detection (VAD) parameters\n    bool        vad           = false;\n    std::string vad_model     = \"\";\n    float       vad_threshold = 0.5f;\n    int         vad_min_speech_duration_ms = 250;\n    int         vad_min_silence_duration_ms = 100;\n    float       vad_max_speech_duration_s = FLT_MAX;\n    int         vad_speech_pad_ms = 30;\n    float       vad_samples_overlap = 0.1f;\n};\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params, const server_params& sparams) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options] \\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,        --help              [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,      --threads N         [%-7d] number of threads to use during computation\\n\",    params.n_threads);\n    fprintf(stderr, \"  -p N,      --processors N      [%-7d] number of processors to use during computation\\n\", params.n_processors);\n    fprintf(stderr, \"  -ot N,     --offset-t N        [%-7d] time offset in milliseconds\\n\",                    params.offset_t_ms);\n    fprintf(stderr, \"  -on N,     --offset-n N        [%-7d] segment index offset\\n\",                           params.offset_n);\n    fprintf(stderr, \"  -d  N,     --duration N        [%-7d] duration of audio to process in milliseconds\\n\",   params.duration_ms);\n    fprintf(stderr, \"  -mc N,     --max-context N     [%-7d] maximum number of text context tokens to store\\n\", params.max_context);\n    fprintf(stderr, \"  -ml N,     --max-len N         [%-7d] maximum segment length in characters\\n\",           params.max_len);\n    fprintf(stderr, \"  -sow,      --split-on-word     [%-7s] split on word rather than on token\\n\",             params.split_on_word ? \"true\" : \"false\");\n    fprintf(stderr, \"  -bo N,     --best-of N         [%-7d] number of best candidates to keep\\n\",              params.best_of);\n    fprintf(stderr, \"  -bs N,     --beam-size N       [%-7d] beam size for beam search\\n\",                      params.beam_size);\n    fprintf(stderr, \"  -ac N,     --audio-ctx N       [%-7d] audio context size (0 - all)\\n\",                   params.audio_ctx);\n    fprintf(stderr, \"  -wt N,     --word-thold N      [%-7.2f] word timestamp probability threshold\\n\",         params.word_thold);\n    fprintf(stderr, \"  -et N,     --entropy-thold N   [%-7.2f] entropy threshold for decoder fail\\n\",           params.entropy_thold);\n    fprintf(stderr, \"  -lpt N,    --logprob-thold N   [%-7.2f] log probability threshold for decoder fail\\n\",   params.logprob_thold);\n    fprintf(stderr, \"  -debug,    --debug-mode        [%-7s] enable debug mode (eg. dump log_mel)\\n\",           params.debug_mode ? \"true\" : \"false\");\n    fprintf(stderr, \"  -tr,       --translate         [%-7s] translate from source language to english\\n\",      params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -di,       --diarize           [%-7s] stereo audio diarization\\n\",                       params.diarize ? \"true\" : \"false\");\n    fprintf(stderr, \"  -tdrz,     --tinydiarize       [%-7s] enable tinydiarize (requires a tdrz model)\\n\",     params.tinydiarize ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nf,       --no-fallback       [%-7s] do not use temperature fallback while decoding\\n\", params.no_fallback ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,       --print-special     [%-7s] print special tokens\\n\",                           params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pc,       --print-colors      [%-7s] print colors\\n\",                                   params.print_colors ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pr,       --print-realtime    [%-7s] print output in realtime\\n\",                       params.print_realtime ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pp,       --print-progress    [%-7s] print progress\\n\",                                 params.print_progress ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nt,       --no-timestamps     [%-7s] do not print timestamps\\n\",                        params.no_timestamps ? \"true\" : \"false\");\n    fprintf(stderr, \"  -l LANG,   --language LANG     [%-7s] spoken language ('auto' for auto-detect)\\n\",       params.language.c_str());\n    fprintf(stderr, \"  -dl,       --detect-language   [%-7s] exit after automatically detecting language\\n\",    params.detect_language ? \"true\" : \"false\");\n    fprintf(stderr, \"             --prompt PROMPT     [%-7s] initial prompt\\n\",                                 params.prompt.c_str());\n    fprintf(stderr, \"  -m FNAME,  --model FNAME       [%-7s] model path\\n\",                                     params.model.c_str());\n    fprintf(stderr, \"  -oved D,   --ov-e-device DNAME [%-7s] the OpenVINO device used for encode inference\\n\",  params.openvino_encode_device.c_str());\n    // server params\n    fprintf(stderr, \"  -dtw MODEL --dtw MODEL         [%-7s] compute token-level timestamps\\n\", params.dtw.c_str());\n    fprintf(stderr, \"  --host HOST,                   [%-7s] Hostname/ip-adress for the server\\n\", sparams.hostname.c_str());\n    fprintf(stderr, \"  --port PORT,                   [%-7d] Port number for the server\\n\", sparams.port);\n    fprintf(stderr, \"  --public PATH,                 [%-7s] Path to the public folder\\n\", sparams.public_path.c_str());\n    fprintf(stderr, \"  --request-path PATH,           [%-7s] Request path for all requests\\n\", sparams.request_path.c_str());\n    fprintf(stderr, \"  --inference-path PATH,         [%-7s] Inference path for all requests\\n\", sparams.inference_path.c_str());\n    fprintf(stderr, \"  --convert,                     [%-7s] Convert audio to WAV, requires ffmpeg on the server\\n\", sparams.ffmpeg_converter ? \"true\" : \"false\");\n    fprintf(stderr, \"  --tmp-dir,                     [%-7s] Temporary directory for ffmpeg transcoded files\\n\", sparams.tmp_dir.c_str());\n    fprintf(stderr, \"  -sns,      --suppress-nst      [%-7s] suppress non-speech tokens\\n\", params.suppress_nst ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nth N,    --no-speech-thold N [%-7.2f] no speech threshold\\n\",   params.no_speech_thold);\n    fprintf(stderr, \"  -ng,       --no-gpu            [%-7s] do not use gpu\\n\", params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -dev N,    --device N          [%-7d] GPU device ID (default: 0)\\n\", params.gpu_device);\n    fprintf(stderr, \"  -fa,       --flash-attn        [%-7s] enable flash attention\\n\", params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,      --no-flash-attn     [%-7s] disable flash attention\\n\", params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -nlp,      --no-language-probabilities [%-7s] exclude language probabilities from verbose_json output\\n\", params.no_language_probabilities ? \"true\" : \"false\");\n    // Voice Activity Detection (VAD) parameters\n    fprintf(stderr, \"\\nVoice Activity Detection (VAD) options:\\n\");\n    fprintf(stderr, \"             --vad                           [%-7s] enable Voice Activity Detection (VAD)\\n\",            params.vad ? \"true\" : \"false\");\n    fprintf(stderr, \"  -vm FNAME, --vad-model FNAME               [%-7s] VAD model path\\n\",                                   params.vad_model.c_str());\n    fprintf(stderr, \"  -vt N,     --vad-threshold N               [%-7.2f] VAD threshold for speech recognition\\n\",           params.vad_threshold);\n    fprintf(stderr, \"  -vspd N,   --vad-min-speech-duration-ms  N [%-7d] VAD min speech duration (0.0-1.0)\\n\",                params.vad_min_speech_duration_ms);\n    fprintf(stderr, \"  -vsd N,    --vad-min-silence-duration-ms N [%-7d] VAD min silence duration (to split segments)\\n\",      params.vad_min_silence_duration_ms);\n    fprintf(stderr, \"  -vmsd N,   --vad-max-speech-duration-s   N [%-7s] VAD max speech duration (auto-split longer)\\n\",      params.vad_max_speech_duration_s == FLT_MAX ?\n                                                                                                                                  std::string(\"FLT_MAX\").c_str() :\n                                                                                                                                  std::to_string(params.vad_max_speech_duration_s).c_str());\n    fprintf(stderr, \"  -vp N,     --vad-speech-pad-ms           N [%-7d] VAD speech padding (extend segments)\\n\",             params.vad_speech_pad_ms);\n    fprintf(stderr, \"  -vo N,     --vad-samples-overlap         N [%-7.2f] VAD samples overlap (seconds between segments)\\n\", params.vad_samples_overlap);\n    fprintf(stderr, \"\\n\");\n}\n\nbool whisper_params_parse(int argc, char ** argv, whisper_params & params, server_params & sparams) {\n    if (const char * env_device = std::getenv(\"WHISPER_ARG_DEVICE\")) {\n        params.gpu_device = std::stoi(env_device);\n    }\n\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params, sparams);\n            exit(0);\n        }\n        else if (arg == \"-t\"    || arg == \"--threads\")         { params.n_threads       = std::stoi(argv[++i]); }\n        else if (arg == \"-p\"    || arg == \"--processors\")      { params.n_processors    = std::stoi(argv[++i]); }\n        else if (arg == \"-ot\"   || arg == \"--offset-t\")        { params.offset_t_ms     = std::stoi(argv[++i]); }\n        else if (arg == \"-on\"   || arg == \"--offset-n\")        { params.offset_n        = std::stoi(argv[++i]); }\n        else if (arg == \"-d\"    || arg == \"--duration\")        { params.duration_ms     = std::stoi(argv[++i]); }\n        else if (arg == \"-mc\"   || arg == \"--max-context\")     { params.max_context     = std::stoi(argv[++i]); }\n        else if (arg == \"-ml\"   || arg == \"--max-len\")         { params.max_len         = std::stoi(argv[++i]); }\n        else if (arg == \"-bo\"   || arg == \"--best-of\")         { params.best_of         = std::stoi(argv[++i]); }\n        else if (arg == \"-bs\"   || arg == \"--beam-size\")       { params.beam_size       = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"   || arg == \"--audio-ctx\")       { params.audio_ctx       = std::stoi(argv[++i]); }\n        else if (arg == \"-wt\"   || arg == \"--word-thold\")      { params.word_thold      = std::stof(argv[++i]); }\n        else if (arg == \"-et\"   || arg == \"--entropy-thold\")   { params.entropy_thold   = std::stof(argv[++i]); }\n        else if (arg == \"-lpt\"  || arg == \"--logprob-thold\")   { params.logprob_thold   = std::stof(argv[++i]); }\n        else if (arg == \"-debug\"|| arg == \"--debug-mode\")      { params.debug_mode      = true; }\n        else if (arg == \"-tr\"   || arg == \"--translate\")       { params.translate       = true; }\n        else if (arg == \"-di\"   || arg == \"--diarize\")         { params.diarize         = true; }\n        else if (arg == \"-tdrz\" || arg == \"--tinydiarize\")     { params.tinydiarize     = true; }\n        else if (arg == \"-sow\"  || arg == \"--split-on-word\")   { params.split_on_word   = true; }\n        else if (arg == \"-nf\"   || arg == \"--no-fallback\")     { params.no_fallback     = true; }\n        else if (arg == \"-fp\"   || arg == \"--font-path\")       { params.font_path       = argv[++i]; }\n        else if (arg == \"-ps\"   || arg == \"--print-special\")   { params.print_special   = true; }\n        else if (arg == \"-pc\"   || arg == \"--print-colors\")    { params.print_colors    = true; }\n        else if (arg == \"-pr\"   || arg == \"--print-realtime\")  { params.print_realtime  = true; }\n        else if (arg == \"-pp\"   || arg == \"--print-progress\")  { params.print_progress  = true; }\n        else if (arg == \"-nt\"   || arg == \"--no-timestamps\")   { params.no_timestamps   = true; }\n        else if (arg == \"-l\"    || arg == \"--language\")        { params.language        = argv[++i]; }\n        else if (arg == \"-dl\"   || arg == \"--detect-language\") { params.detect_language = true; }\n        else if (                  arg == \"--prompt\")          { params.prompt          = argv[++i]; }\n        else if (arg == \"-m\"    || arg == \"--model\")           { params.model           = argv[++i]; }\n        else if (arg == \"-oved\" || arg == \"--ov-e-device\")     { params.openvino_encode_device = argv[++i]; }\n        else if (arg == \"-dtw\"  || arg == \"--dtw\")             { params.dtw             = argv[++i]; }\n        else if (arg == \"-ng\"   || arg == \"--no-gpu\")          { params.use_gpu         = false; }\n        else if (arg == \"-dev\"  || arg == \"--device\")          { params.gpu_device      = std::stoi(argv[++i]); }\n        else if (arg == \"-fa\"   || arg == \"--flash-attn\")      { params.flash_attn      = true; }\n        else if (arg == \"-nfa\"  || arg == \"--no-flash-attn\")   { params.flash_attn      = false; }\n        else if (arg == \"-sns\"  || arg == \"--suppress-nst\")    { params.suppress_nst    = true; }\n        else if (arg == \"-nth\"  || arg == \"--no-speech-thold\") { params.no_speech_thold = std::stof(argv[++i]); }\n        else if (arg == \"-nlp\"  || arg == \"--no-language-probabilities\") { params.no_language_probabilities = true; }\n\n        // server params\n        else if (                  arg == \"--port\")            { sparams.port        = std::stoi(argv[++i]); }\n        else if (                  arg == \"--host\")            { sparams.hostname    = argv[++i]; }\n        else if (                  arg == \"--public\")          { sparams.public_path = argv[++i]; }\n        else if (                  arg == \"--request-path\")    { sparams.request_path = argv[++i]; }\n        else if (                  arg == \"--inference-path\")  { sparams.inference_path = argv[++i]; }\n        else if (                  arg == \"--convert\")         { sparams.ffmpeg_converter     = true; }\n        else if (                  arg == \"--tmp-dir\")         { sparams.tmp_dir     = argv[++i]; }\n\n        // Voice Activity Detection (VAD)\n        else if (                  arg == \"--vad\")                         { params.vad                         = true; }\n        else if (arg == \"-vm\"   || arg == \"--vad-model\")                   { params.vad_model                   = argv[++i]; }\n        else if (arg == \"-vt\"   || arg == \"--vad-threshold\")               { params.vad_threshold               = std::stof(argv[++i]); }\n        else if (arg == \"-vspd\" || arg == \"--vad-min-speech-duration-ms\")  { params.vad_min_speech_duration_ms  = std::stoi(argv[++i]); }\n        else if (arg == \"-vsd\"  || arg == \"--vad-min-silence-duration-ms\") { params.vad_min_silence_duration_ms = std::stoi(argv[++i]); }\n        else if (arg == \"-vmsd\" || arg == \"--vad-max-speech-duration-s\")   { params.vad_max_speech_duration_s   = std::stof(argv[++i]); }\n        else if (arg == \"-vp\"   || arg == \"--vad-speech-pad-ms\")           { params.vad_speech_pad_ms           = std::stoi(argv[++i]); }\n        else if (arg == \"-vo\"   || arg == \"--vad-samples-overlap\")         { params.vad_samples_overlap         = std::stof(argv[++i]); }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params, sparams);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nstruct whisper_print_user_data {\n    const whisper_params * params;\n\n    const std::vector<std::vector<float>> * pcmf32s;\n    int progress_prev;\n};\n\nvoid check_ffmpeg_availibility() {\n    int result = system(\"ffmpeg -version\");\n\n    if (result == 0) {\n        std::cout << \"ffmpeg is available.\" << std::endl;\n    } else {\n        // ffmpeg is not available\n        std::cout << \"ffmpeg is not found. Please ensure that ffmpeg is installed \";\n        std::cout << \"and that its executable is included in your system's PATH. \";\n        exit(0);\n    }\n}\n\nstd::string generate_temp_filename(const std::string &path, const std::string &prefix, const std::string &extension) {\n    auto now = std::chrono::system_clock::now();\n    auto now_time_t = std::chrono::system_clock::to_time_t(now);\n\n    static std::mt19937 rng{std::random_device{}()};\n    std::uniform_int_distribution<long long> dist(0, 1e9);\n\n    std::stringstream ss;\n    ss << path\n       << std::filesystem::path::preferred_separator\n       << prefix\n       << \"-\"\n       << std::put_time(std::localtime(&now_time_t), \"%Y%m%d-%H%M%S\")\n       << \"-\"\n       << dist(rng)\n       << extension;\n\n    return ss.str();\n}\n\nbool convert_to_wav(const std::string & temp_filename, std::string & error_resp) {\n    std::ostringstream cmd_stream;\n    std::string converted_filename_temp = temp_filename + \"_temp.wav\";\n    cmd_stream << \"ffmpeg -i \\\"\" << temp_filename << \"\\\" -y -ar 16000 -ac 1 -c:a pcm_s16le \\\"\" << converted_filename_temp << \"\\\" 2>&1\";\n    std::string cmd = cmd_stream.str();\n\n    int status = std::system(cmd.c_str());\n    if (status != 0) {\n        error_resp = \"{\\\"error\\\":\\\"FFmpeg conversion failed.\\\"}\";\n        return false;\n    }\n\n    // Remove the original file\n    if (remove(temp_filename.c_str()) != 0) {\n        error_resp = \"{\\\"error\\\":\\\"Failed to remove the original file.\\\"}\";\n        return false;\n    }\n\n    // Rename the temporary file to match the original filename\n    if (rename(converted_filename_temp.c_str(), temp_filename.c_str()) != 0) {\n        error_resp = \"{\\\"error\\\":\\\"Failed to rename the temporary file.\\\"}\";\n        return false;\n    }\n    return true;\n}\n\nstd::string estimate_diarization_speaker(std::vector<std::vector<float>> pcmf32s, int64_t t0, int64_t t1, bool id_only = false) {\n    std::string speaker = \"\";\n    const int64_t n_samples = pcmf32s[0].size();\n\n    const int64_t is0 = timestamp_to_sample(t0, n_samples, WHISPER_SAMPLE_RATE);\n    const int64_t is1 = timestamp_to_sample(t1, n_samples, WHISPER_SAMPLE_RATE);\n\n    double energy0 = 0.0f;\n    double energy1 = 0.0f;\n\n    for (int64_t j = is0; j < is1; j++) {\n        energy0 += fabs(pcmf32s[0][j]);\n        energy1 += fabs(pcmf32s[1][j]);\n    }\n\n    if (energy0 > 1.1*energy1) {\n        speaker = \"0\";\n    } else if (energy1 > 1.1*energy0) {\n        speaker = \"1\";\n    } else {\n        speaker = \"?\";\n    }\n\n    //printf(\"is0 = %lld, is1 = %lld, energy0 = %f, energy1 = %f, speaker = %s\\n\", is0, is1, energy0, energy1, speaker.c_str());\n\n    if (!id_only) {\n        speaker.insert(0, \"(speaker \");\n        speaker.append(\")\");\n    }\n\n    return speaker;\n}\n\nvoid whisper_print_progress_callback(struct whisper_context * /*ctx*/, struct whisper_state * /*state*/, int progress, void * user_data) {\n    int progress_step = ((whisper_print_user_data *) user_data)->params->progress_step;\n    int * progress_prev  = &(((whisper_print_user_data *) user_data)->progress_prev);\n    if (progress >= *progress_prev + progress_step) {\n        *progress_prev += progress_step;\n        fprintf(stderr, \"%s: progress = %3d%%\\n\", __func__, progress);\n    }\n}\n\nvoid whisper_print_segment_callback(struct whisper_context * ctx, struct whisper_state * /*state*/, int n_new, void * user_data) {\n    const auto & params  = *((whisper_print_user_data *) user_data)->params;\n    const auto & pcmf32s = *((whisper_print_user_data *) user_data)->pcmf32s;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n\n    std::string speaker = \"\";\n\n    int64_t t0 = 0;\n    int64_t t1 = 0;\n\n    // print the last n_new segments\n    const int s0 = n_segments - n_new;\n\n    if (s0 == 0) {\n        printf(\"\\n\");\n    }\n\n    for (int i = s0; i < n_segments; i++) {\n        if (!params.no_timestamps || params.diarize) {\n            t0 = whisper_full_get_segment_t0(ctx, i);\n            t1 = whisper_full_get_segment_t1(ctx, i);\n        }\n\n        if (!params.no_timestamps) {\n            printf(\"[%s --> %s]  \", to_timestamp(t0).c_str(), to_timestamp(t1).c_str());\n        }\n\n        if (params.diarize && pcmf32s.size() == 2) {\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        if (params.print_colors) {\n            for (int j = 0; j < whisper_full_n_tokens(ctx, i); ++j) {\n                if (params.print_special == false) {\n                    const whisper_token id = whisper_full_get_token_id(ctx, i, j);\n                    if (id >= whisper_token_eot(ctx)) {\n                        continue;\n                    }\n                }\n\n                const char * text = whisper_full_get_token_text(ctx, i, j);\n                const float  p    = whisper_full_get_token_p   (ctx, i, j);\n\n                const int col = std::max(0, std::min((int) k_colors.size() - 1, (int) (std::pow(p, 3)*float(k_colors.size()))));\n\n                printf(\"%s%s%s%s\", speaker.c_str(), k_colors[col].c_str(), text, \"\\033[0m\");\n            }\n        } else {\n            const char * text = whisper_full_get_segment_text(ctx, i);\n\n            printf(\"%s%s\", speaker.c_str(), text);\n        }\n\n        if (params.tinydiarize) {\n            if (whisper_full_get_segment_speaker_turn_next(ctx, i)) {\n                printf(\"%s\", params.tdrz_speaker_turn.c_str());\n            }\n        }\n\n        // with timestamps or speakers: each segment on new line\n        if (!params.no_timestamps || params.diarize) {\n            printf(\"\\n\");\n        }\n        fflush(stdout);\n    }\n}\n\nstd::string output_str(struct whisper_context * ctx, const whisper_params & params, std::vector<std::vector<float>> pcmf32s) {\n    std::stringstream result;\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n        std::string speaker = \"\";\n\n        if (params.diarize && pcmf32s.size() == 2)\n        {\n            const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n            const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n            speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n        }\n\n        result << speaker << text << \"\\n\";\n    }\n    return result.str();\n}\n\nbool parse_str_to_bool(const std::string & s) {\n    if (s == \"true\" || s == \"1\" || s == \"yes\" || s == \"y\") {\n        return true;\n    }\n    return false;\n}\n\nvoid get_req_parameters(const Request & req, whisper_params & params)\n{\n    if (req.has_file(\"offset_t\"))\n    {\n        params.offset_t_ms = std::stoi(req.get_file_value(\"offset_t\").content);\n    }\n    if (req.has_file(\"offset_n\"))\n    {\n        params.offset_n = std::stoi(req.get_file_value(\"offset_n\").content);\n    }\n    if (req.has_file(\"duration\"))\n    {\n        params.duration_ms = std::stoi(req.get_file_value(\"duration\").content);\n    }\n    if (req.has_file(\"max_context\"))\n    {\n        params.max_context = std::stoi(req.get_file_value(\"max_context\").content);\n    }\n    if (req.has_file(\"max_len\"))\n    {\n        params.max_len = std::stoi(req.get_file_value(\"max_len\").content);\n    }\n    if (req.has_file(\"best_of\"))\n    {\n        params.best_of = std::stoi(req.get_file_value(\"best_of\").content);\n    }\n    if (req.has_file(\"beam_size\"))\n    {\n        params.beam_size = std::stoi(req.get_file_value(\"beam_size\").content);\n    }\n    if (req.has_file(\"audio_ctx\"))\n    {\n        params.audio_ctx = std::stof(req.get_file_value(\"audio_ctx\").content);\n    }\n    if (req.has_file(\"word_thold\"))\n    {\n        params.word_thold = std::stof(req.get_file_value(\"word_thold\").content);\n    }\n    if (req.has_file(\"entropy_thold\"))\n    {\n        params.entropy_thold = std::stof(req.get_file_value(\"entropy_thold\").content);\n    }\n    if (req.has_file(\"logprob_thold\"))\n    {\n        params.logprob_thold = std::stof(req.get_file_value(\"logprob_thold\").content);\n    }\n    if (req.has_file(\"debug_mode\"))\n    {\n        params.debug_mode = parse_str_to_bool(req.get_file_value(\"debug_mode\").content);\n    }\n    if (req.has_file(\"translate\"))\n    {\n        params.translate = parse_str_to_bool(req.get_file_value(\"translate\").content);\n    }\n    if (req.has_file(\"diarize\"))\n    {\n        params.diarize = parse_str_to_bool(req.get_file_value(\"diarize\").content);\n    }\n    if (req.has_file(\"tinydiarize\"))\n    {\n        params.tinydiarize = parse_str_to_bool(req.get_file_value(\"tinydiarize\").content);\n    }\n    if (req.has_file(\"split_on_word\"))\n    {\n        params.split_on_word = parse_str_to_bool(req.get_file_value(\"split_on_word\").content);\n    }\n    if (req.has_file(\"no_timestamps\"))\n    {\n        params.no_timestamps = parse_str_to_bool(req.get_file_value(\"no_timestamps\").content);\n    }\n    if (req.has_file(\"language\"))\n    {\n        params.language = req.get_file_value(\"language\").content;\n    }\n    if (req.has_file(\"detect_language\"))\n    {\n        params.detect_language = parse_str_to_bool(req.get_file_value(\"detect_language\").content);\n    }\n    if (req.has_file(\"prompt\"))\n    {\n        params.prompt = req.get_file_value(\"prompt\").content;\n    }\n    if (req.has_file(\"response_format\"))\n    {\n        params.response_format = req.get_file_value(\"response_format\").content;\n    }\n    if (req.has_file(\"temperature\"))\n    {\n        params.temperature = std::stof(req.get_file_value(\"temperature\").content);\n    }\n    if (req.has_file(\"temperature_inc\"))\n    {\n        params.temperature_inc = std::stof(req.get_file_value(\"temperature_inc\").content);\n    }\n    if (req.has_file(\"suppress_non_speech\"))\n    {\n        params.suppress_nst = parse_str_to_bool(req.get_file_value(\"suppress_non_speech\").content);\n    }\n    if (req.has_file(\"suppress_nst\"))\n    {\n        params.suppress_nst = parse_str_to_bool(req.get_file_value(\"suppress_nst\").content);\n    }\n    if (req.has_file(\"vad\"))\n    {\n        params.vad = parse_str_to_bool(req.get_file_value(\"vad\").content);\n    }\n    if (req.has_file(\"vad_threshold\"))\n    {\n        params.vad_threshold = std::stof(req.get_file_value(\"vad_threshold\").content);\n    }\n    if (req.has_file(\"vad_min_speech_duration_ms\"))\n    {\n        params.vad_min_speech_duration_ms = std::stof(req.get_file_value(\"vad_min_speech_duration_ms\").content);\n    }\n    if (req.has_file(\"vad_min_silence_duration_ms\"))\n    {\n        params.vad_min_silence_duration_ms = std::stof(req.get_file_value(\"vad_min_silence_duration_ms\").content);\n    }\n    if (req.has_file(\"vad_max_speech_duration_s\"))\n    {\n        params.vad_max_speech_duration_s = std::stof(req.get_file_value(\"vad_max_speech_duration_s\").content);\n    }\n    if (req.has_file(\"vad_speech_pad_ms\"))\n    {\n        params.vad_speech_pad_ms = std::stoi(req.get_file_value(\"vad_speech_pad_ms\").content);\n    }\n    if (req.has_file(\"vad_samples_overlap\"))\n    {\n        params.vad_samples_overlap = std::stof(req.get_file_value(\"vad_samples_overlap\").content);\n    }\n    if (req.has_file(\"no_language_probabilities\"))\n    {\n        params.no_language_probabilities = parse_str_to_bool(req.get_file_value(\"no_language_probabilities\").content);\n    }\n}\n\n}  // namespace\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n    server_params sparams;\n\n    std::mutex whisper_mutex;\n\n    if (whisper_params_parse(argc, argv, params, sparams) == false) {\n        whisper_print_usage(argc, argv, params, sparams);\n        return 1;\n    }\n\n    if (params.language != \"auto\" && whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params, sparams);\n        exit(0);\n    }\n\n    if (params.diarize && params.tinydiarize) {\n        fprintf(stderr, \"error: cannot use both --diarize and --tinydiarize\\n\");\n        whisper_print_usage(argc, argv, params, sparams);\n        exit(0);\n    }\n\n    if (sparams.ffmpeg_converter) {\n        check_ffmpeg_availibility();\n    }\n    // whisper init\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.gpu_device = params.gpu_device;\n    cparams.flash_attn = params.flash_attn;\n\n    if (!params.dtw.empty()) {\n        cparams.dtw_token_timestamps = true;\n        cparams.dtw_aheads_preset = WHISPER_AHEADS_NONE;\n\n        if (params.dtw == \"tiny\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY;\n        }\n        if (params.dtw == \"tiny.en\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_TINY_EN;\n        }\n        if (params.dtw == \"base\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE;\n        }\n        if (params.dtw == \"base.en\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_BASE_EN;\n        }\n        if (params.dtw == \"small\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL;\n        }\n        if (params.dtw == \"small.en\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_SMALL_EN;\n        }\n        if (params.dtw == \"medium\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM;\n        }\n        if (params.dtw == \"medium.en\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_MEDIUM_EN;\n        }\n        if (params.dtw == \"large.v1\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V1;\n        }\n        if (params.dtw == \"large.v2\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V2;\n        }\n        if (params.dtw == \"large.v3\") {\n            cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3;\n        }\n        if (params.dtw == \"large.v3.turbo\") { \n            cparams.dtw_aheads_preset = WHISPER_AHEADS_LARGE_V3_TURBO;\n        }\n        \n        if (cparams.dtw_aheads_preset == WHISPER_AHEADS_NONE) {\n            fprintf(stderr, \"error: unknown DTW preset '%s'\\n\", params.dtw.c_str());\n            return 3;\n        }\n    }\n\n    std::unique_ptr<httplib::Server> svr = std::make_unique<httplib::Server>();\n    std::atomic<server_state> state{SERVER_STATE_LOADING_MODEL};\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n\n    if (ctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 3;\n    }\n\n    // initialize openvino encoder. this has no effect on whisper.cpp builds that don't have OpenVINO configured\n    whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr);\n    state.store(SERVER_STATE_READY);\n\n\n    svr->set_default_headers({{\"Server\", \"whisper.cpp\"},\n                             {\"Access-Control-Allow-Origin\", \"*\"},\n                             {\"Access-Control-Allow-Headers\", \"content-type, authorization\"}});\n\n    std::string const default_content = R\"(\n    <html>\n    <head>\n        <title>Whisper.cpp Server</title>\n        <meta charset=\"utf-8\">\n        <meta name=\"viewport\" content=\"width=device-width\">\n        <style>\n        body {\n            font-family: sans-serif;\n        }\n        form {\n            display: flex;\n            flex-direction: column;\n            align-items: flex-start;\n        }\n        label {\n            margin-bottom: 0.5rem;\n        }\n        input, select {\n            margin-bottom: 1rem;\n        }\n        button {\n            margin-top: 1rem;\n        }\n        </style>\n    </head>\n    <body>\n        <h1>Whisper.cpp Server</h1>\n\n        <h2>)\" + sparams.request_path + sparams.inference_path + R\"(</h2>\n        <pre>\n    curl 127.0.0.1:)\" + std::to_string(sparams.port) + sparams.request_path + sparams.inference_path + R\"( \\\n    -H \"Content-Type: multipart/form-data\" \\\n    -F file=\"@&lt;file-path&gt;\" \\\n    -F temperature=\"0.0\" \\\n    -F temperature_inc=\"0.2\" \\\n    -F response_format=\"json\"\n        </pre>\n\n        <h2>/load</h2>\n        <pre>\n    curl 127.0.0.1:)\" + std::to_string(sparams.port) + R\"(/load \\\n    -H \"Content-Type: multipart/form-data\" \\\n    -F model=\"&lt;path-to-model-file&gt;\"\n        </pre>\n\n        <div>\n            <h2>Try it out</h2>\n            <form action=\")\" + sparams.request_path + sparams.inference_path + R\"(\" method=\"POST\" enctype=\"multipart/form-data\">\n                <label for=\"file\">Choose an audio file:</label>\n                <input type=\"file\" id=\"file\" name=\"file\" accept=\"audio/*\" required><br>\n\n                <label for=\"temperature\">Temperature:</label>\n                <input type=\"number\" id=\"temperature\" name=\"temperature\" value=\"0.0\" step=\"0.01\" placeholder=\"e.g., 0.0\"><br>\n\n                <label for=\"response_format\">Response Format:</label>\n                <select id=\"response_format\" name=\"response_format\">\n                    <option value=\"verbose_json\">Verbose JSON</option>\n                    <option value=\"json\">JSON</option>\n                    <option value=\"text\">Text</option>\n                    <option value=\"srt\">SRT</option>\n                    <option value=\"vtt\">VTT</option>\n                </select><br>\n\n                <button type=\"submit\">Submit</button>\n            </form>\n        </div>\n    </body>\n    </html>\n    )\";\n\n    // store default params so we can reset after each inference request\n    whisper_params default_params = params;\n\n    // this is only called if no index.html is found in the public --path\n    svr->Get(sparams.request_path + \"/\", [&](const Request &, Response &res){\n        res.set_content(default_content, \"text/html\");\n        return false;\n    });\n\n    svr->Options(sparams.request_path + sparams.inference_path, [&](const Request &, Response &){\n    });\n\n    svr->Post(sparams.request_path + sparams.inference_path, [&](const Request &req, Response &res){\n        // acquire whisper model mutex lock\n        std::lock_guard<std::mutex> lock(whisper_mutex);\n\n        // first check user requested fields of the request\n        if (!req.has_file(\"file\"))\n        {\n            fprintf(stderr, \"error: no 'file' field in the request\\n\");\n            const std::string error_resp = \"{\\\"error\\\":\\\"no 'file' field in the request\\\"}\";\n            res.status = 400;\n            res.set_content(error_resp, \"application/json\");\n            return;\n        }\n        auto audio_file = req.get_file_value(\"file\");\n\n        // check non-required fields\n        get_req_parameters(req, params);\n\n        std::string filename{audio_file.filename};\n        printf(\"Received request: %s\\n\", filename.c_str());\n\n        // audio arrays\n        std::vector<float> pcmf32;               // mono-channel F32 PCM\n        std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM\n\n        if (sparams.ffmpeg_converter) {\n            // if file is not wav, convert to wav\n            // write to temporary file\n            const std::string temp_filename = generate_temp_filename(sparams.tmp_dir, \"whisper-server\", \".wav\");\n            std::ofstream temp_file{temp_filename, std::ios::binary};\n            temp_file << audio_file.content;\n            temp_file.close();\n\n            std::string error_resp = \"{\\\"error\\\":\\\"Failed to execute ffmpeg command.\\\"}\";\n            const bool is_converted = convert_to_wav(temp_filename, error_resp);\n            if (!is_converted) {\n                res.status = 500;\n                res.set_content(error_resp, \"application/json\");\n                return;\n            }\n\n            // read audio content into pcmf32\n            if (!::read_audio_data(temp_filename, pcmf32, pcmf32s, params.diarize))\n            {\n                fprintf(stderr, \"error: failed to read WAV file '%s'\\n\", temp_filename.c_str());\n                const std::string error_resp = \"{\\\"error\\\":\\\"failed to read WAV file\\\"}\";\n                res.status = 400;\n                res.set_content(error_resp, \"application/json\");\n                std::remove(temp_filename.c_str());\n                return;\n            }\n            // remove temp file\n            std::remove(temp_filename.c_str());\n        } else {\n            if (!::read_audio_data(audio_file.content, pcmf32, pcmf32s, params.diarize))\n            {\n                fprintf(stderr, \"error: failed to read audio data\\n\");\n                const std::string error_resp = \"{\\\"error\\\":\\\"failed to read audio data\\\"}\";\n                res.status = 400;\n                res.set_content(error_resp, \"application/json\");\n                return;\n            }\n        }\n\n        printf(\"Successfully loaded %s\\n\", filename.c_str());\n\n        // print system information\n        {\n            fprintf(stderr, \"\\n\");\n            fprintf(stderr, \"system_info: n_threads = %d / %d | %s\\n\",\n                    params.n_threads*params.n_processors, std::thread::hardware_concurrency(), whisper_print_system_info());\n        }\n\n        // print some info about the processing\n        {\n            fprintf(stderr, \"\\n\");\n            if (!whisper_is_multilingual(ctx)) {\n                if (params.language != \"en\" || params.translate) {\n                    params.language = \"en\";\n                    params.translate = false;\n                    fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n                }\n            }\n            if (params.detect_language) {\n                params.language = \"auto\";\n            }\n            fprintf(stderr, \"%s: processing '%s' (%d samples, %.1f sec), %d threads, %d processors, lang = %s, task = %s, %stimestamps = %d ...\\n\",\n                    __func__, filename.c_str(), int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,\n                    params.n_threads, params.n_processors,\n                    params.language.c_str(),\n                    params.translate ? \"translate\" : \"transcribe\",\n                    params.tinydiarize ? \"tdrz = 1, \" : \"\",\n                    params.no_timestamps ? 0 : 1);\n\n            fprintf(stderr, \"\\n\");\n        }\n\n        // run the inference\n        {\n            printf(\"Running whisper.cpp inference on %s\\n\", filename.c_str());\n            whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n            wparams.strategy = params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;\n\n            wparams.print_realtime   = false;\n            wparams.print_progress   = params.print_progress;\n            wparams.print_timestamps = !params.no_timestamps;\n            wparams.print_special    = params.print_special;\n            wparams.translate        = params.translate;\n            wparams.language         = params.language.c_str();\n            wparams.detect_language  = params.detect_language;\n            wparams.n_threads        = params.n_threads;\n            wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;\n            wparams.offset_ms        = params.offset_t_ms;\n            wparams.duration_ms      = params.duration_ms;\n\n            wparams.thold_pt         = params.word_thold;\n            wparams.max_len          = params.max_len == 0 ? 60 : params.max_len;\n            wparams.split_on_word    = params.split_on_word;\n            wparams.audio_ctx        = params.audio_ctx;\n\n            wparams.debug_mode       = params.debug_mode;\n\n            wparams.tdrz_enable      = params.tinydiarize; // [TDRZ]\n\n            wparams.initial_prompt   = params.prompt.c_str();\n\n            wparams.greedy.best_of        = params.best_of;\n            wparams.beam_search.beam_size = params.beam_size;\n\n            wparams.temperature      = params.temperature;\n            wparams.no_speech_thold = params.no_speech_thold;\n            wparams.temperature_inc  = params.temperature_inc;\n            wparams.entropy_thold    = params.entropy_thold;\n            wparams.logprob_thold    = params.logprob_thold;\n\n            wparams.no_timestamps    = params.no_timestamps;\n            wparams.token_timestamps = !params.no_timestamps;\n            wparams.no_context       = params.no_context;\n\n            wparams.suppress_nst     = params.suppress_nst;\n\n            wparams.vad              = params.vad;\n            wparams.vad_model_path   = params.vad_model.c_str();\n\n            wparams.vad_params.threshold               = params.vad_threshold;\n            wparams.vad_params.min_speech_duration_ms  = params.vad_min_speech_duration_ms;\n            wparams.vad_params.min_silence_duration_ms = params.vad_min_silence_duration_ms;\n            wparams.vad_params.max_speech_duration_s   = params.vad_max_speech_duration_s;\n            wparams.vad_params.speech_pad_ms           = params.vad_speech_pad_ms;\n            wparams.vad_params.samples_overlap         = params.vad_samples_overlap;\n\n            whisper_print_user_data user_data = { &params, &pcmf32s, 0 };\n\n            // this callback is called on each new segment\n            if (params.print_realtime) {\n                wparams.new_segment_callback           = whisper_print_segment_callback;\n                wparams.new_segment_callback_user_data = &user_data;\n            }\n\n            if (wparams.print_progress) {\n                wparams.progress_callback           = whisper_print_progress_callback;\n                wparams.progress_callback_user_data = &user_data;\n            }\n\n            // tell whisper to abort if the HTTP connection closed\n            wparams.abort_callback = [](void *user_data) {\n                // user_data is a pointer to our Request\n                auto req_ptr = static_cast<const httplib::Request*>(user_data);\n                return req_ptr->is_connection_closed();\n            };\n            wparams.abort_callback_user_data = (void*)&req;\n\n            if (whisper_full_parallel(ctx, wparams, pcmf32.data(), pcmf32.size(), params.n_processors) != 0) {\n                // handle failure or early abort\n                if (req.is_connection_closed()) {\n                    // log client disconnect\n                    fprintf(stderr, \"client disconnected, aborted processing\\n\");\n                    res.status = 499; // Client Closed Request (nginx convention)\n                    res.set_content(\"{\\\"error\\\":\\\"client disconnected\\\"}\", \"application/json\");\n                    return;\n                }\n                fprintf(stderr, \"%s: failed to process audio\\n\", argv[0]);\n                res.status = 500; // Internal Server Error\n                const std::string error_resp = \"{\\\"error\\\":\\\"failed to process audio\\\"}\";\n                res.set_content(error_resp, \"application/json\");\n                return;\n            }\n        }\n\n        // return results to user\n        if (params.response_format == text_format)\n        {\n            std::string results = output_str(ctx, params, pcmf32s);\n            res.set_content(results.c_str(), \"text/html; charset=utf-8\");\n        }\n        else if (params.response_format == srt_format)\n        {\n            std::stringstream ss;\n            const int n_segments = whisper_full_n_segments(ctx);\n            for (int i = 0; i < n_segments; ++i) {\n                const char * text = whisper_full_get_segment_text(ctx, i);\n                const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n                const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n                std::string speaker = \"\";\n\n                if (params.diarize && pcmf32s.size() == 2)\n                {\n                    speaker = estimate_diarization_speaker(pcmf32s, t0, t1);\n                }\n\n                ss << i + 1 + params.offset_n << \"\\n\";\n                ss << to_timestamp(t0, true) << \" --> \" << to_timestamp(t1, true) << \"\\n\";\n                ss << speaker << text << \"\\n\\n\";\n            }\n            res.set_content(ss.str(), \"application/x-subrip\");\n        } else if (params.response_format == vtt_format) {\n            std::stringstream ss;\n\n            ss << \"WEBVTT\\n\\n\";\n\n            const int n_segments = whisper_full_n_segments(ctx);\n            for (int i = 0; i < n_segments; ++i) {\n                const char * text = whisper_full_get_segment_text(ctx, i);\n                const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n                const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n                std::string speaker = \"\";\n\n                if (params.diarize && pcmf32s.size() == 2)\n                {\n                    speaker = estimate_diarization_speaker(pcmf32s, t0, t1, true);\n                    speaker.insert(0, \"<v Speaker\");\n                    speaker.append(\">\");\n                }\n\n                ss << to_timestamp(t0) << \" --> \" << to_timestamp(t1) << \"\\n\";\n                ss << speaker << text << \"\\n\\n\";\n            }\n            res.set_content(ss.str(), \"text/vtt\");\n        } else if (params.response_format == vjson_format) {\n            /* try to match openai/whisper's Python format */\n            std::string results = output_str(ctx, params, pcmf32s); \n            json jres = json{\n                {\"task\", params.translate ? \"translate\" : \"transcribe\"},\n                {\"language\", whisper_lang_str_full(whisper_full_lang_id(ctx))},\n                {\"duration\", float(pcmf32.size())/WHISPER_SAMPLE_RATE},\n                {\"text\", results},\n                {\"segments\", json::array()}\n            };\n            // Only compute language probabilities if requested (expensive operation)\n            if (!params.no_language_probabilities) {\n                std::vector<float> lang_probs(whisper_lang_max_id() + 1, 0.0f);\n                const auto detected_lang_id = whisper_lang_auto_detect(ctx, 0, params.n_threads, lang_probs.data());\n                jres[\"detected_language\"] = whisper_lang_str_full(detected_lang_id);\n                jres[\"detected_language_probability\"] = lang_probs[detected_lang_id];\n                jres[\"language_probabilities\"] = json::object();\n                // Add all language probabilities\n                for (int i = 0; i <= whisper_lang_max_id(); ++i) {\n                    if (lang_probs[i] > 0.001f) { // Only include non-negligible probabilities\n                        jres[\"language_probabilities\"][whisper_lang_str(i)] = lang_probs[i];\n                    }\n                }\n            }\n            const int n_segments = whisper_full_n_segments(ctx);\n            for (int i = 0; i < n_segments; ++i)\n            {\n                json segment = json{\n                    {\"id\", i},\n                    {\"text\", whisper_full_get_segment_text(ctx, i)},\n                };\n\n                if (!params.no_timestamps) {\n                    segment[\"start\"] = whisper_full_get_segment_t0(ctx, i) * 0.01;\n                    segment[\"end\"] = whisper_full_get_segment_t1(ctx, i) * 0.01;\n                }\n\n                float total_logprob = 0;\n                const int n_tokens = whisper_full_n_tokens(ctx, i);\n                for (int j = 0; j < n_tokens; ++j) {\n                    whisper_token_data token = whisper_full_get_token_data(ctx, i, j);\n                    if (token.id >= whisper_token_eot(ctx)) {\n                        continue;\n                    }\n\n                    segment[\"tokens\"].push_back(token.id);\n                    json word = json{{\"word\", whisper_full_get_token_text(ctx, i, j)}};\n                    if (!params.no_timestamps) {\n                        word[\"start\"] = token.t0 * 0.01;\n                        word[\"end\"] = token.t1 * 0.01;\n                        word[\"t_dtw\"] = token.t_dtw;\n                    }\n                    word[\"probability\"] = token.p;\n                    total_logprob += token.plog;\n                    segment[\"words\"].push_back(word);\n                }\n\n                segment[\"temperature\"] = params.temperature;\n                segment[\"avg_logprob\"] = total_logprob / n_tokens;\n\n                // TODO compression_ratio and no_speech_prob are not implemented yet\n                // segment[\"compression_ratio\"] = 0;\n                segment[\"no_speech_prob\"] = whisper_full_get_segment_no_speech_prob(ctx, i);\n\n                jres[\"segments\"].push_back(segment);\n            }\n            res.set_content(jres.dump(-1, ' ', false, json::error_handler_t::replace),\n                            \"application/json\");\n        }\n        // TODO add more output formats\n        else\n        {\n            std::string results = output_str(ctx, params, pcmf32s);\n            json jres = json{\n                {\"text\", results}\n            };\n            res.set_content(jres.dump(-1, ' ', false, json::error_handler_t::replace),\n                            \"application/json\");\n        }\n\n        // reset params to their defaults\n        params = default_params;\n    });\n    svr->Post(sparams.request_path + \"/load\", [&](const Request &req, Response &res){\n        std::lock_guard<std::mutex> lock(whisper_mutex);\n        state.store(SERVER_STATE_LOADING_MODEL);\n        if (!req.has_file(\"model\"))\n        {\n            fprintf(stderr, \"error: no 'model' field in the request\\n\");\n            const std::string error_resp = \"{\\\"error\\\":\\\"no 'model' field in the request\\\"}\";\n            res.status = 400;\n            res.set_content(error_resp, \"application/json\");\n            return;\n        }\n        std::string model = req.get_file_value(\"model\").content;\n        if (!is_file_exist(model.c_str()))\n        {\n            fprintf(stderr, \"error: 'model': %s not found!\\n\", model.c_str());\n            const std::string error_resp = \"{\\\"error\\\":\\\"model not found!\\\"}\";\n            res.status = 400;\n            res.set_content(error_resp, \"application/json\");\n            return;\n        }\n\n        // clean up\n        whisper_free(ctx);\n\n        // whisper init\n        ctx = whisper_init_from_file_with_params(model.c_str(), cparams);\n\n        // TODO perhaps load prior model here instead of exit\n        if (ctx == nullptr) {\n            fprintf(stderr, \"error: model init  failed, no model loaded must exit\\n\");\n            exit(1);\n        }\n\n        // initialize openvino encoder. this has no effect on whisper.cpp builds that don't have OpenVINO configured\n        whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr);\n\n        state.store(SERVER_STATE_READY);\n        const std::string success = \"Load was successful!\";\n        res.set_content(success, \"application/text\");\n\n        // check if the model is in the file system\n    });\n\n    svr->Get(sparams.request_path + \"/health\", [&](const Request &, Response &res){\n        server_state current_state = state.load();\n        if (current_state == SERVER_STATE_READY) {\n            const std::string health_response = \"{\\\"status\\\":\\\"ok\\\"}\";\n            res.set_content(health_response, \"application/json\");\n        } else {\n            res.set_content(\"{\\\"status\\\":\\\"loading model\\\"}\", \"application/json\");\n            res.status = 503;\n        }\n    });\n\n    svr->set_exception_handler([](const Request &, Response &res, std::exception_ptr ep) {\n        const char fmt[] = \"500 Internal Server Error\\n%s\";\n        char buf[BUFSIZ];\n        try {\n            std::rethrow_exception(std::move(ep));\n        } catch (std::exception &e) {\n            snprintf(buf, sizeof(buf), fmt, e.what());\n        } catch (...) {\n            snprintf(buf, sizeof(buf), fmt, \"Unknown Exception\");\n        }\n        res.set_content(buf, \"text/plain\");\n        res.status = 500;\n    });\n\n    svr->set_error_handler([](const Request &req, Response &res) {\n        if (res.status == 400) {\n            res.set_content(\"Invalid request\", \"text/plain\");\n        } else if (res.status != 500) {\n            res.set_content(\"File Not Found (\" + req.path + \")\", \"text/plain\");\n            res.status = 404;\n        }\n    });\n\n    // set timeouts and change hostname and port\n    svr->set_read_timeout(sparams.read_timeout);\n    svr->set_write_timeout(sparams.write_timeout);\n\n    if (!svr->bind_to_port(sparams.hostname, sparams.port))\n    {\n        fprintf(stderr, \"\\ncouldn't bind to server socket: hostname=%s port=%d\\n\\n\",\n                sparams.hostname.c_str(), sparams.port);\n        return 1;\n    }\n\n    // Set the base directory for serving static files\n    svr->set_base_dir(sparams.public_path);\n\n    // to make it ctrl+clickable:\n    printf(\"\\nwhisper server listening at http://%s:%d\\n\\n\", sparams.hostname.c_str(), sparams.port);\n\n    shutdown_handler = [&](int signal) {\n        printf(\"\\nCaught signal %d, shutting down gracefully...\\n\", signal);\n        if (svr) {\n            svr->stop();\n        }\n    };\n\n#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))\n    struct sigaction sigint_action;\n    sigint_action.sa_handler = signal_handler;\n    sigemptyset (&sigint_action.sa_mask);\n    sigint_action.sa_flags = 0;\n    sigaction(SIGINT, &sigint_action, NULL);\n    sigaction(SIGTERM, &sigint_action, NULL);\n#elif defined (_WIN32)\n    auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {\n        return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;\n    };\n    SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);\n#endif\n\n    // clean up function, to be called before exit\n    auto clean_up = [&]() {\n        whisper_print_timings(ctx);\n        whisper_free(ctx);\n    };\n\n    std::thread t([&] {\n        if (!svr->listen_after_bind()) {\n            fprintf(stderr, \"error: server listen failed\\n\");\n        }\n    });\n\n    svr->wait_until_ready();\n\n    t.join();\n\n\n    clean_up();\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/server.py",
    "content": "import http.server\nimport socketserver\nimport os\nimport sys\nfrom pathlib import Path\nimport urllib.parse\n\nSCRIPT_DIR = Path(__file__).parent.absolute()\nDIRECTORY = os.path.join(SCRIPT_DIR, \"../build-em/bin\")\nDIRECTORY = os.path.abspath(DIRECTORY)\n\n# The context root we want for all applications\nCONTEXT_ROOT = \"/whisper.cpp\"\n\nclass CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, directory=DIRECTORY, **kwargs)\n\n    def do_GET(self):\n        # Redirect root to the context root\n        if self.path == '/':\n            self.send_response(302)\n            self.send_header('Location', CONTEXT_ROOT + '/')\n            self.end_headers()\n            return\n\n        # Handle requests under the context root\n        if self.path.startswith(CONTEXT_ROOT):\n            # Remove the context root prefix to get the actual path\n            actual_path = self.path[len(CONTEXT_ROOT):]\n\n            if not actual_path:\n                self.send_response(302)\n                self.send_header('Location', CONTEXT_ROOT + '/')\n                self.end_headers()\n                return\n\n            if '.worker.js' in actual_path:\n                worker_file = os.path.basename(actual_path)\n                worker_path = os.path.join(DIRECTORY, worker_file)\n\n                if os.path.exists(worker_path):\n                    print(f\"Found worker file: {worker_path}\")\n                    self.path = '/' + worker_file\n                else:\n                    print(f\"Worker file not found: {worker_path}\")\n\n            elif actual_path == '/':\n                self.path = '/whisper.wasm/index.html'\n            elif any(actual_path.startswith(prefix) for prefix in (\n                '/bench.wasm/',\n                '/command.wasm/',\n                '/stream.wasm/',\n                '/wchess.wasm/'\n            )):\n                # Keep the path as is, just remove the context root\n                self.path = actual_path\n            # For all other paths under the context root\n            else:\n                # Check if this is a request to a file in whisper.wasm\n                potential_file = os.path.join(DIRECTORY, 'whisper.wasm', actual_path.lstrip('/'))\n                if os.path.exists(potential_file) and not os.path.isdir(potential_file):\n                    self.path = '/whisper.wasm' + actual_path\n                else:\n                    # Try to resolve the file from the base directory\n                    potential_file = os.path.join(DIRECTORY, actual_path.lstrip('/'))\n                    if os.path.exists(potential_file):\n                        self.path = actual_path\n\n        # For direct requests to worker files (without context root as these\n        # are in the build-em/bin directory\n        elif '.worker.js' in self.path:\n            worker_file = os.path.basename(self.path)\n            worker_path = os.path.join(DIRECTORY, worker_file)\n\n            if os.path.exists(worker_path):\n                self.path = '/' + worker_file\n\n        # Handle coi-serviceworker.js separately\n        if 'coi-serviceworker.js' in self.path:\n            worker_file = \"coi-serviceworker.js\"\n            worker_path = os.path.join(SCRIPT_DIR, worker_file)\n            if os.path.exists(worker_path):\n                self.send_response(200)\n                self.send_header('Content-type', 'application/javascript')\n                self.end_headers()\n                with open(worker_path, 'rb') as file:\n                    self.wfile.write(file.read())\n                return\n            else:\n                print(f\"Warning: Could not find {worker_path}\")\n\n        return super().do_GET()\n\n    def end_headers(self):\n        # Add required headers for SharedArrayBuffer\n        self.send_header(\"Cross-Origin-Opener-Policy\", \"same-origin\")\n        self.send_header(\"Cross-Origin-Embedder-Policy\", \"require-corp\")\n        self.send_header(\"Access-Control-Allow-Origin\", \"*\")\n        super().end_headers()\n\nPORT = 8000\n\n# Enable address reuse\nclass CustomServer(socketserver.TCPServer):\n    allow_reuse_address = True\n\ntry:\n    with CustomServer((\"\", PORT), CustomHTTPRequestHandler) as httpd:\n        print(f\"Serving directory '{DIRECTORY}' at http://localhost:{PORT}\")\n        print(f\"Application context root: http://localhost:{PORT}{CONTEXT_ROOT}/\")\n        try:\n            httpd.serve_forever()\n        except KeyboardInterrupt:\n            print(\"\\nServer stopped.\")\n            # Force complete exit\n            sys.exit(0)\nexcept OSError as e:\n    print(f\"Error: {e}\")\n    sys.exit(1)\n"
  },
  {
    "path": "examples/stb_vorbis.c",
    "content": "// Ogg Vorbis audio decoder - v1.22 - public domain\n// http://nothings.org/stb_vorbis/\n//\n// Original version written by Sean Barrett in 2007.\n//\n// Originally sponsored by RAD Game Tools. Seeking implementation\n// sponsored by Phillip Bennefall, Marc Andersen, Aaron Baker,\n// Elias Software, Aras Pranckevicius, and Sean Barrett.\n//\n// LICENSE\n//\n//   See end of file for license information.\n//\n// Limitations:\n//\n//   - floor 0 not supported (used in old ogg vorbis files pre-2004)\n//   - lossless sample-truncation at beginning ignored\n//   - cannot concatenate multiple vorbis streams\n//   - sample positions are 32-bit, limiting seekable 192Khz\n//       files to around 6 hours (Ogg supports 64-bit)\n//\n// Feature contributors:\n//    Dougall Johnson (sample-exact seeking)\n//\n// Bugfix/warning contributors:\n//    Terje Mathisen     Niklas Frykholm     Andy Hill\n//    Casey Muratori     John Bolton         Gargaj\n//    Laurent Gomila     Marc LeBlanc        Ronny Chevalier\n//    Bernhard Wodo      Evan Balster        github:alxprd\n//    Tom Beaumont       Ingo Leitgeb        Nicolas Guillemot\n//    Phillip Bennefall  Rohit               Thiago Goulart\n//    github:manxorist   Saga Musix          github:infatum\n//    Timur Gagiev       Maxwell Koo         Peter Waller\n//    github:audinowho   Dougall Johnson     David Reid\n//    github:Clownacy    Pedro J. Estebanez  Remi Verschelde\n//    AnthoFoxo          github:morlat       Gabriel Ravier\n//\n// Partial history:\n//    1.22    - 2021-07-11 - various small fixes\n//    1.21    - 2021-07-02 - fix bug for files with no comments\n//    1.20    - 2020-07-11 - several small fixes\n//    1.19    - 2020-02-05 - warnings\n//    1.18    - 2020-02-02 - fix seek bugs; parse header comments; misc warnings etc.\n//    1.17    - 2019-07-08 - fix CVE-2019-13217..CVE-2019-13223 (by ForAllSecure)\n//    1.16    - 2019-03-04 - fix warnings\n//    1.15    - 2019-02-07 - explicit failure if Ogg Skeleton data is found\n//    1.14    - 2018-02-11 - delete bogus dealloca usage\n//    1.13    - 2018-01-29 - fix truncation of last frame (hopefully)\n//    1.12    - 2017-11-21 - limit residue begin/end to blocksize/2 to avoid large temp allocs in bad/corrupt files\n//    1.11    - 2017-07-23 - fix MinGW compilation\n//    1.10    - 2017-03-03 - more robust seeking; fix negative ilog(); clear error in open_memory\n//    1.09    - 2016-04-04 - back out 'truncation of last frame' fix from previous version\n//    1.08    - 2016-04-02 - warnings; setup memory leaks; truncation of last frame\n//    1.07    - 2015-01-16 - fixes for crashes on invalid files; warning fixes; const\n//    1.06    - 2015-08-31 - full, correct support for seeking API (Dougall Johnson)\n//                           some crash fixes when out of memory or with corrupt files\n//                           fix some inappropriately signed shifts\n//    1.05    - 2015-04-19 - don't define __forceinline if it's redundant\n//    1.04    - 2014-08-27 - fix missing const-correct case in API\n//    1.03    - 2014-08-07 - warning fixes\n//    1.02    - 2014-07-09 - declare qsort comparison as explicitly _cdecl in Windows\n//    1.01    - 2014-06-18 - fix stb_vorbis_get_samples_float (interleaved was correct)\n//    1.0     - 2014-05-26 - fix memory leaks; fix warnings; fix bugs in >2-channel;\n//                           (API change) report sample rate for decode-full-file funcs\n//\n// See end of file for full version history.\n\n\n//////////////////////////////////////////////////////////////////////////////\n//\n//  HEADER BEGINS HERE\n//\n\n#ifndef STB_VORBIS_INCLUDE_STB_VORBIS_H\n#define STB_VORBIS_INCLUDE_STB_VORBIS_H\n\n#if defined(STB_VORBIS_NO_CRT) && !defined(STB_VORBIS_NO_STDIO)\n#define STB_VORBIS_NO_STDIO 1\n#endif\n\n#ifndef STB_VORBIS_NO_STDIO\n#include <stdio.h>\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n///////////   THREAD SAFETY\n\n// Individual stb_vorbis* handles are not thread-safe; you cannot decode from\n// them from multiple threads at the same time. However, you can have multiple\n// stb_vorbis* handles and decode from them independently in multiple thrads.\n\n\n///////////   MEMORY ALLOCATION\n\n// normally stb_vorbis uses malloc() to allocate memory at startup,\n// and alloca() to allocate temporary memory during a frame on the\n// stack. (Memory consumption will depend on the amount of setup\n// data in the file and how you set the compile flags for speed\n// vs. size. In my test files the maximal-size usage is ~150KB.)\n//\n// You can modify the wrapper functions in the source (setup_malloc,\n// setup_temp_malloc, temp_malloc) to change this behavior, or you\n// can use a simpler allocation model: you pass in a buffer from\n// which stb_vorbis will allocate _all_ its memory (including the\n// temp memory). \"open\" may fail with a VORBIS_outofmem if you\n// do not pass in enough data; there is no way to determine how\n// much you do need except to succeed (at which point you can\n// query get_info to find the exact amount required. yes I know\n// this is lame).\n//\n// If you pass in a non-NULL buffer of the type below, allocation\n// will occur from it as described above. Otherwise just pass NULL\n// to use malloc()/alloca()\n\ntypedef struct\n{\n   char *alloc_buffer;\n   int   alloc_buffer_length_in_bytes;\n} stb_vorbis_alloc;\n\n\n///////////   FUNCTIONS USEABLE WITH ALL INPUT MODES\n\ntypedef struct stb_vorbis stb_vorbis;\n\ntypedef struct\n{\n   unsigned int sample_rate;\n   int channels;\n\n   unsigned int setup_memory_required;\n   unsigned int setup_temp_memory_required;\n   unsigned int temp_memory_required;\n\n   int max_frame_size;\n} stb_vorbis_info;\n\ntypedef struct\n{\n   char *vendor;\n\n   int comment_list_length;\n   char **comment_list;\n} stb_vorbis_comment;\n\n// get general information about the file\nextern stb_vorbis_info stb_vorbis_get_info(stb_vorbis *f);\n\n// get ogg comments\nextern stb_vorbis_comment stb_vorbis_get_comment(stb_vorbis *f);\n\n// get the last error detected (clears it, too)\nextern int stb_vorbis_get_error(stb_vorbis *f);\n\n// close an ogg vorbis file and free all memory in use\nextern void stb_vorbis_close(stb_vorbis *f);\n\n// this function returns the offset (in samples) from the beginning of the\n// file that will be returned by the next decode, if it is known, or -1\n// otherwise. after a flush_pushdata() call, this may take a while before\n// it becomes valid again.\n// NOT WORKING YET after a seek with PULLDATA API\nextern int stb_vorbis_get_sample_offset(stb_vorbis *f);\n\n// returns the current seek point within the file, or offset from the beginning\n// of the memory buffer. In pushdata mode it returns 0.\nextern unsigned int stb_vorbis_get_file_offset(stb_vorbis *f);\n\n///////////   PUSHDATA API\n\n#ifndef STB_VORBIS_NO_PUSHDATA_API\n\n// this API allows you to get blocks of data from any source and hand\n// them to stb_vorbis. you have to buffer them; stb_vorbis will tell\n// you how much it used, and you have to give it the rest next time;\n// and stb_vorbis may not have enough data to work with and you will\n// need to give it the same data again PLUS more. Note that the Vorbis\n// specification does not bound the size of an individual frame.\n\nextern stb_vorbis *stb_vorbis_open_pushdata(\n         const unsigned char * datablock, int datablock_length_in_bytes,\n         int *datablock_memory_consumed_in_bytes,\n         int *error,\n         const stb_vorbis_alloc *alloc_buffer);\n// create a vorbis decoder by passing in the initial data block containing\n//    the ogg&vorbis headers (you don't need to do parse them, just provide\n//    the first N bytes of the file--you're told if it's not enough, see below)\n// on success, returns an stb_vorbis *, does not set error, returns the amount of\n//    data parsed/consumed on this call in *datablock_memory_consumed_in_bytes;\n// on failure, returns NULL on error and sets *error, does not change *datablock_memory_consumed\n// if returns NULL and *error is VORBIS_need_more_data, then the input block was\n//       incomplete and you need to pass in a larger block from the start of the file\n\nextern int stb_vorbis_decode_frame_pushdata(\n         stb_vorbis *f,\n         const unsigned char *datablock, int datablock_length_in_bytes,\n         int *channels,             // place to write number of float * buffers\n         float ***output,           // place to write float ** array of float * buffers\n         int *samples               // place to write number of output samples\n     );\n// decode a frame of audio sample data if possible from the passed-in data block\n//\n// return value: number of bytes we used from datablock\n//\n// possible cases:\n//     0 bytes used, 0 samples output (need more data)\n//     N bytes used, 0 samples output (resynching the stream, keep going)\n//     N bytes used, M samples output (one frame of data)\n// note that after opening a file, you will ALWAYS get one N-bytes,0-sample\n// frame, because Vorbis always \"discards\" the first frame.\n//\n// Note that on resynch, stb_vorbis will rarely consume all of the buffer,\n// instead only datablock_length_in_bytes-3 or less. This is because it wants\n// to avoid missing parts of a page header if they cross a datablock boundary,\n// without writing state-machiney code to record a partial detection.\n//\n// The number of channels returned are stored in *channels (which can be\n// NULL--it is always the same as the number of channels reported by\n// get_info). *output will contain an array of float* buffers, one per\n// channel. In other words, (*output)[0][0] contains the first sample from\n// the first channel, and (*output)[1][0] contains the first sample from\n// the second channel.\n//\n// *output points into stb_vorbis's internal output buffer storage; these\n// buffers are owned by stb_vorbis and application code should not free\n// them or modify their contents. They are transient and will be overwritten\n// once you ask for more data to get decoded, so be sure to grab any data\n// you need before then.\n\nextern void stb_vorbis_flush_pushdata(stb_vorbis *f);\n// inform stb_vorbis that your next datablock will not be contiguous with\n// previous ones (e.g. you've seeked in the data); future attempts to decode\n// frames will cause stb_vorbis to resynchronize (as noted above), and\n// once it sees a valid Ogg page (typically 4-8KB, as large as 64KB), it\n// will begin decoding the _next_ frame.\n//\n// if you want to seek using pushdata, you need to seek in your file, then\n// call stb_vorbis_flush_pushdata(), then start calling decoding, then once\n// decoding is returning you data, call stb_vorbis_get_sample_offset, and\n// if you don't like the result, seek your file again and repeat.\n#endif\n\n\n//////////   PULLING INPUT API\n\n#ifndef STB_VORBIS_NO_PULLDATA_API\n// This API assumes stb_vorbis is allowed to pull data from a source--\n// either a block of memory containing the _entire_ vorbis stream, or a\n// FILE * that you or it create, or possibly some other reading mechanism\n// if you go modify the source to replace the FILE * case with some kind\n// of callback to your code. (But if you don't support seeking, you may\n// just want to go ahead and use pushdata.)\n\n#if !defined(STB_VORBIS_NO_STDIO) && !defined(STB_VORBIS_NO_INTEGER_CONVERSION)\nextern int stb_vorbis_decode_filename(const char *filename, int *channels, int *sample_rate, short **output);\n#endif\n#if !defined(STB_VORBIS_NO_INTEGER_CONVERSION)\nextern int stb_vorbis_decode_memory(const unsigned char *mem, int len, int *channels, int *sample_rate, short **output);\n#endif\n// decode an entire file and output the data interleaved into a malloc()ed\n// buffer stored in *output. The return value is the number of samples\n// decoded, or -1 if the file could not be opened or was not an ogg vorbis file.\n// When you're done with it, just free() the pointer returned in *output.\n\nextern stb_vorbis * stb_vorbis_open_memory(const unsigned char *data, int len,\n                                  int *error, const stb_vorbis_alloc *alloc_buffer);\n// create an ogg vorbis decoder from an ogg vorbis stream in memory (note\n// this must be the entire stream!). on failure, returns NULL and sets *error\n\n#ifndef STB_VORBIS_NO_STDIO\nextern stb_vorbis * stb_vorbis_open_filename(const char *filename,\n                                  int *error, const stb_vorbis_alloc *alloc_buffer);\n// create an ogg vorbis decoder from a filename via fopen(). on failure,\n// returns NULL and sets *error (possibly to VORBIS_file_open_failure).\n\nextern stb_vorbis * stb_vorbis_open_file(FILE *f, int close_handle_on_close,\n                                  int *error, const stb_vorbis_alloc *alloc_buffer);\n// create an ogg vorbis decoder from an open FILE *, looking for a stream at\n// the _current_ seek point (ftell). on failure, returns NULL and sets *error.\n// note that stb_vorbis must \"own\" this stream; if you seek it in between\n// calls to stb_vorbis, it will become confused. Moreover, if you attempt to\n// perform stb_vorbis_seek_*() operations on this file, it will assume it\n// owns the _entire_ rest of the file after the start point. Use the next\n// function, stb_vorbis_open_file_section(), to limit it.\n\nextern stb_vorbis * stb_vorbis_open_file_section(FILE *f, int close_handle_on_close,\n                int *error, const stb_vorbis_alloc *alloc_buffer, unsigned int len);\n// create an ogg vorbis decoder from an open FILE *, looking for a stream at\n// the _current_ seek point (ftell); the stream will be of length 'len' bytes.\n// on failure, returns NULL and sets *error. note that stb_vorbis must \"own\"\n// this stream; if you seek it in between calls to stb_vorbis, it will become\n// confused.\n#endif\n\nextern int stb_vorbis_seek_frame(stb_vorbis *f, unsigned int sample_number);\nextern int stb_vorbis_seek(stb_vorbis *f, unsigned int sample_number);\n// these functions seek in the Vorbis file to (approximately) 'sample_number'.\n// after calling seek_frame(), the next call to get_frame_*() will include\n// the specified sample. after calling stb_vorbis_seek(), the next call to\n// stb_vorbis_get_samples_* will start with the specified sample. If you\n// do not need to seek to EXACTLY the target sample when using get_samples_*,\n// you can also use seek_frame().\n\nextern int stb_vorbis_seek_start(stb_vorbis *f);\n// this function is equivalent to stb_vorbis_seek(f,0)\n\nextern unsigned int stb_vorbis_stream_length_in_samples(stb_vorbis *f);\nextern float        stb_vorbis_stream_length_in_seconds(stb_vorbis *f);\n// these functions return the total length of the vorbis stream\n\nextern int stb_vorbis_get_frame_float(stb_vorbis *f, int *channels, float ***output);\n// decode the next frame and return the number of samples. the number of\n// channels returned are stored in *channels (which can be NULL--it is always\n// the same as the number of channels reported by get_info). *output will\n// contain an array of float* buffers, one per channel. These outputs will\n// be overwritten on the next call to stb_vorbis_get_frame_*.\n//\n// You generally should not intermix calls to stb_vorbis_get_frame_*()\n// and stb_vorbis_get_samples_*(), since the latter calls the former.\n\n#ifndef STB_VORBIS_NO_INTEGER_CONVERSION\nextern int stb_vorbis_get_frame_short_interleaved(stb_vorbis *f, int num_c, short *buffer, int num_shorts);\nextern int stb_vorbis_get_frame_short            (stb_vorbis *f, int num_c, short **buffer, int num_samples);\n#endif\n// decode the next frame and return the number of *samples* per channel.\n// Note that for interleaved data, you pass in the number of shorts (the\n// size of your array), but the return value is the number of samples per\n// channel, not the total number of samples.\n//\n// The data is coerced to the number of channels you request according to the\n// channel coercion rules (see below). You must pass in the size of your\n// buffer(s) so that stb_vorbis will not overwrite the end of the buffer.\n// The maximum buffer size needed can be gotten from get_info(); however,\n// the Vorbis I specification implies an absolute maximum of 4096 samples\n// per channel.\n\n// Channel coercion rules:\n//    Let M be the number of channels requested, and N the number of channels present,\n//    and Cn be the nth channel; let stereo L be the sum of all L and center channels,\n//    and stereo R be the sum of all R and center channels (channel assignment from the\n//    vorbis spec).\n//        M    N       output\n//        1    k      sum(Ck) for all k\n//        2    *      stereo L, stereo R\n//        k    l      k > l, the first l channels, then 0s\n//        k    l      k <= l, the first k channels\n//    Note that this is not _good_ surround etc. mixing at all! It's just so\n//    you get something useful.\n\nextern int stb_vorbis_get_samples_float_interleaved(stb_vorbis *f, int channels, float *buffer, int num_floats);\nextern int stb_vorbis_get_samples_float(stb_vorbis *f, int channels, float **buffer, int num_samples);\n// gets num_samples samples, not necessarily on a frame boundary--this requires\n// buffering so you have to supply the buffers. DOES NOT APPLY THE COERCION RULES.\n// Returns the number of samples stored per channel; it may be less than requested\n// at the end of the file. If there are no more samples in the file, returns 0.\n\n#ifndef STB_VORBIS_NO_INTEGER_CONVERSION\nextern int stb_vorbis_get_samples_short_interleaved(stb_vorbis *f, int channels, short *buffer, int num_shorts);\nextern int stb_vorbis_get_samples_short(stb_vorbis *f, int channels, short **buffer, int num_samples);\n#endif\n// gets num_samples samples, not necessarily on a frame boundary--this requires\n// buffering so you have to supply the buffers. Applies the coercion rules above\n// to produce 'channels' channels. Returns the number of samples stored per channel;\n// it may be less than requested at the end of the file. If there are no more\n// samples in the file, returns 0.\n\n#endif\n\n////////   ERROR CODES\n\nenum STBVorbisError\n{\n   VORBIS__no_error,\n\n   VORBIS_need_more_data=1,             // not a real error\n\n   VORBIS_invalid_api_mixing,           // can't mix API modes\n   VORBIS_outofmem,                     // not enough memory\n   VORBIS_feature_not_supported,        // uses floor 0\n   VORBIS_too_many_channels,            // STB_VORBIS_MAX_CHANNELS is too small\n   VORBIS_file_open_failure,            // fopen() failed\n   VORBIS_seek_without_length,          // can't seek in unknown-length file\n\n   VORBIS_unexpected_eof=10,            // file is truncated?\n   VORBIS_seek_invalid,                 // seek past EOF\n\n   // decoding errors (corrupt/invalid stream) -- you probably\n   // don't care about the exact details of these\n\n   // vorbis errors:\n   VORBIS_invalid_setup=20,\n   VORBIS_invalid_stream,\n\n   // ogg errors:\n   VORBIS_missing_capture_pattern=30,\n   VORBIS_invalid_stream_structure_version,\n   VORBIS_continued_packet_flag_invalid,\n   VORBIS_incorrect_stream_serial_number,\n   VORBIS_invalid_first_page,\n   VORBIS_bad_packet_type,\n   VORBIS_cant_find_last_page,\n   VORBIS_seek_failed,\n   VORBIS_ogg_skeleton_not_supported\n};\n\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // STB_VORBIS_INCLUDE_STB_VORBIS_H\n//\n//  HEADER ENDS HERE\n//\n//////////////////////////////////////////////////////////////////////////////\n\n#ifndef STB_VORBIS_HEADER_ONLY\n\n// global configuration settings (e.g. set these in the project/makefile),\n// or just set them in this file at the top (although ideally the first few\n// should be visible when the header file is compiled too, although it's not\n// crucial)\n\n// STB_VORBIS_NO_PUSHDATA_API\n//     does not compile the code for the various stb_vorbis_*_pushdata()\n//     functions\n// #define STB_VORBIS_NO_PUSHDATA_API\n\n// STB_VORBIS_NO_PULLDATA_API\n//     does not compile the code for the non-pushdata APIs\n// #define STB_VORBIS_NO_PULLDATA_API\n\n// STB_VORBIS_NO_STDIO\n//     does not compile the code for the APIs that use FILE *s internally\n//     or externally (implied by STB_VORBIS_NO_PULLDATA_API)\n// #define STB_VORBIS_NO_STDIO\n\n// STB_VORBIS_NO_INTEGER_CONVERSION\n//     does not compile the code for converting audio sample data from\n//     float to integer (implied by STB_VORBIS_NO_PULLDATA_API)\n// #define STB_VORBIS_NO_INTEGER_CONVERSION\n\n// STB_VORBIS_NO_FAST_SCALED_FLOAT\n//      does not use a fast float-to-int trick to accelerate float-to-int on\n//      most platforms which requires endianness be defined correctly.\n//#define STB_VORBIS_NO_FAST_SCALED_FLOAT\n\n\n// STB_VORBIS_MAX_CHANNELS [number]\n//     globally define this to the maximum number of channels you need.\n//     The spec does not put a restriction on channels except that\n//     the count is stored in a byte, so 255 is the hard limit.\n//     Reducing this saves about 16 bytes per value, so using 16 saves\n//     (255-16)*16 or around 4KB. Plus anything other memory usage\n//     I forgot to account for. Can probably go as low as 8 (7.1 audio),\n//     6 (5.1 audio), or 2 (stereo only).\n#ifndef STB_VORBIS_MAX_CHANNELS\n#define STB_VORBIS_MAX_CHANNELS    16  // enough for anyone?\n#endif\n\n// STB_VORBIS_PUSHDATA_CRC_COUNT [number]\n//     after a flush_pushdata(), stb_vorbis begins scanning for the\n//     next valid page, without backtracking. when it finds something\n//     that looks like a page, it streams through it and verifies its\n//     CRC32. Should that validation fail, it keeps scanning. But it's\n//     possible that _while_ streaming through to check the CRC32 of\n//     one candidate page, it sees another candidate page. This #define\n//     determines how many \"overlapping\" candidate pages it can search\n//     at once. Note that \"real\" pages are typically ~4KB to ~8KB, whereas\n//     garbage pages could be as big as 64KB, but probably average ~16KB.\n//     So don't hose ourselves by scanning an apparent 64KB page and\n//     missing a ton of real ones in the interim; so minimum of 2\n#ifndef STB_VORBIS_PUSHDATA_CRC_COUNT\n#define STB_VORBIS_PUSHDATA_CRC_COUNT  4\n#endif\n\n// STB_VORBIS_FAST_HUFFMAN_LENGTH [number]\n//     sets the log size of the huffman-acceleration table.  Maximum\n//     supported value is 24. with larger numbers, more decodings are O(1),\n//     but the table size is larger so worse cache missing, so you'll have\n//     to probe (and try multiple ogg vorbis files) to find the sweet spot.\n#ifndef STB_VORBIS_FAST_HUFFMAN_LENGTH\n#define STB_VORBIS_FAST_HUFFMAN_LENGTH   10\n#endif\n\n// STB_VORBIS_FAST_BINARY_LENGTH [number]\n//     sets the log size of the binary-search acceleration table. this\n//     is used in similar fashion to the fast-huffman size to set initial\n//     parameters for the binary search\n\n// STB_VORBIS_FAST_HUFFMAN_INT\n//     The fast huffman tables are much more efficient if they can be\n//     stored as 16-bit results instead of 32-bit results. This restricts\n//     the codebooks to having only 65535 possible outcomes, though.\n//     (At least, accelerated by the huffman table.)\n#ifndef STB_VORBIS_FAST_HUFFMAN_INT\n#define STB_VORBIS_FAST_HUFFMAN_SHORT\n#endif\n\n// STB_VORBIS_NO_HUFFMAN_BINARY_SEARCH\n//     If the 'fast huffman' search doesn't succeed, then stb_vorbis falls\n//     back on binary searching for the correct one. This requires storing\n//     extra tables with the huffman codes in sorted order. Defining this\n//     symbol trades off space for speed by forcing a linear search in the\n//     non-fast case, except for \"sparse\" codebooks.\n// #define STB_VORBIS_NO_HUFFMAN_BINARY_SEARCH\n\n// STB_VORBIS_DIVIDES_IN_RESIDUE\n//     stb_vorbis precomputes the result of the scalar residue decoding\n//     that would otherwise require a divide per chunk. you can trade off\n//     space for time by defining this symbol.\n// #define STB_VORBIS_DIVIDES_IN_RESIDUE\n\n// STB_VORBIS_DIVIDES_IN_CODEBOOK\n//     vorbis VQ codebooks can be encoded two ways: with every case explicitly\n//     stored, or with all elements being chosen from a small range of values,\n//     and all values possible in all elements. By default, stb_vorbis expands\n//     this latter kind out to look like the former kind for ease of decoding,\n//     because otherwise an integer divide-per-vector-element is required to\n//     unpack the index. If you define STB_VORBIS_DIVIDES_IN_CODEBOOK, you can\n//     trade off storage for speed.\n//#define STB_VORBIS_DIVIDES_IN_CODEBOOK\n\n#ifdef STB_VORBIS_CODEBOOK_SHORTS\n#error \"STB_VORBIS_CODEBOOK_SHORTS is no longer supported as it produced incorrect results for some input formats\"\n#endif\n\n// STB_VORBIS_DIVIDE_TABLE\n//     this replaces small integer divides in the floor decode loop with\n//     table lookups. made less than 1% difference, so disabled by default.\n\n// STB_VORBIS_NO_INLINE_DECODE\n//     disables the inlining of the scalar codebook fast-huffman decode.\n//     might save a little codespace; useful for debugging\n// #define STB_VORBIS_NO_INLINE_DECODE\n\n// STB_VORBIS_NO_DEFER_FLOOR\n//     Normally we only decode the floor without synthesizing the actual\n//     full curve. We can instead synthesize the curve immediately. This\n//     requires more memory and is very likely slower, so I don't think\n//     you'd ever want to do it except for debugging.\n// #define STB_VORBIS_NO_DEFER_FLOOR\n\n\n\n\n//////////////////////////////////////////////////////////////////////////////\n\n#ifdef STB_VORBIS_NO_PULLDATA_API\n   #define STB_VORBIS_NO_INTEGER_CONVERSION\n   #define STB_VORBIS_NO_STDIO\n#endif\n\n#if defined(STB_VORBIS_NO_CRT) && !defined(STB_VORBIS_NO_STDIO)\n   #define STB_VORBIS_NO_STDIO 1\n#endif\n\n#ifndef STB_VORBIS_NO_INTEGER_CONVERSION\n#ifndef STB_VORBIS_NO_FAST_SCALED_FLOAT\n\n   // only need endianness for fast-float-to-int, which we don't\n   // use for pushdata\n\n   #ifndef STB_VORBIS_BIG_ENDIAN\n     #define STB_VORBIS_ENDIAN  0\n   #else\n     #define STB_VORBIS_ENDIAN  1\n   #endif\n\n#endif\n#endif\n\n\n#ifndef STB_VORBIS_NO_STDIO\n#include <stdio.h>\n#endif\n\n#ifndef STB_VORBIS_NO_CRT\n   #include <stdlib.h>\n   #include <string.h>\n   #include <assert.h>\n   #include <math.h>\n\n   // find definition of alloca if it's not in stdlib.h:\n   #if defined(_MSC_VER) || defined(__MINGW32__)\n      #include <malloc.h>\n   #endif\n   #if defined(__linux__) || defined(__linux) || defined(__sun__) || defined(__EMSCRIPTEN__) || defined(__NEWLIB__)\n      #include <alloca.h>\n   #endif\n#else // STB_VORBIS_NO_CRT\n   #define NULL 0\n   #define malloc(s)   0\n   #define free(s)     ((void) 0)\n   #define realloc(s)  0\n#endif // STB_VORBIS_NO_CRT\n\n#include <limits.h>\n\n#ifdef __MINGW32__\n   // eff you mingw:\n   //     \"fixed\":\n   //         http://sourceforge.net/p/mingw-w64/mailman/message/32882927/\n   //     \"no that broke the build, reverted, who cares about C\":\n   //         http://sourceforge.net/p/mingw-w64/mailman/message/32890381/\n   #ifdef __forceinline\n   #undef __forceinline\n   #endif\n   #define __forceinline\n   #ifndef alloca\n   #define alloca __builtin_alloca\n   #endif\n#elif !defined(_MSC_VER)\n   #if __GNUC__\n      #define __forceinline inline\n   #else\n      #define __forceinline\n   #endif\n#endif\n\n#if STB_VORBIS_MAX_CHANNELS > 256\n#error \"Value of STB_VORBIS_MAX_CHANNELS outside of allowed range\"\n#endif\n\n#if STB_VORBIS_FAST_HUFFMAN_LENGTH > 24\n#error \"Value of STB_VORBIS_FAST_HUFFMAN_LENGTH outside of allowed range\"\n#endif\n\n\n#if 0\n#include <crtdbg.h>\n#define CHECK(f)   _CrtIsValidHeapPointer(f->channel_buffers[1])\n#else\n#define CHECK(f)   ((void) 0)\n#endif\n\n#define MAX_BLOCKSIZE_LOG  13   // from specification\n#define MAX_BLOCKSIZE      (1 << MAX_BLOCKSIZE_LOG)\n\n\ntypedef unsigned char  uint8;\ntypedef   signed char   int8;\ntypedef unsigned short uint16;\ntypedef   signed short  int16;\ntypedef unsigned int   uint32;\ntypedef   signed int    int32;\n\n#ifndef TRUE\n#define TRUE 1\n#define FALSE 0\n#endif\n\ntypedef float codetype;\n\n#ifdef _MSC_VER\n#define STBV_NOTUSED(v)  (void)(v)\n#else\n#define STBV_NOTUSED(v)  (void)sizeof(v)\n#endif\n\n// @NOTE\n//\n// Some arrays below are tagged \"//varies\", which means it's actually\n// a variable-sized piece of data, but rather than malloc I assume it's\n// small enough it's better to just allocate it all together with the\n// main thing\n//\n// Most of the variables are specified with the smallest size I could pack\n// them into. It might give better performance to make them all full-sized\n// integers. It should be safe to freely rearrange the structures or change\n// the sizes larger--nothing relies on silently truncating etc., nor the\n// order of variables.\n\n#define FAST_HUFFMAN_TABLE_SIZE   (1 << STB_VORBIS_FAST_HUFFMAN_LENGTH)\n#define FAST_HUFFMAN_TABLE_MASK   (FAST_HUFFMAN_TABLE_SIZE - 1)\n\ntypedef struct\n{\n   int dimensions, entries;\n   uint8 *codeword_lengths;\n   float  minimum_value;\n   float  delta_value;\n   uint8  value_bits;\n   uint8  lookup_type;\n   uint8  sequence_p;\n   uint8  sparse;\n   uint32 lookup_values;\n   codetype *multiplicands;\n   uint32 *codewords;\n   #ifdef STB_VORBIS_FAST_HUFFMAN_SHORT\n    int16  fast_huffman[FAST_HUFFMAN_TABLE_SIZE];\n   #else\n    int32  fast_huffman[FAST_HUFFMAN_TABLE_SIZE];\n   #endif\n   uint32 *sorted_codewords;\n   int    *sorted_values;\n   int     sorted_entries;\n} Codebook;\n\ntypedef struct\n{\n   uint8 order;\n   uint16 rate;\n   uint16 bark_map_size;\n   uint8 amplitude_bits;\n   uint8 amplitude_offset;\n   uint8 number_of_books;\n   uint8 book_list[16]; // varies\n} Floor0;\n\ntypedef struct\n{\n   uint8 partitions;\n   uint8 partition_class_list[32]; // varies\n   uint8 class_dimensions[16]; // varies\n   uint8 class_subclasses[16]; // varies\n   uint8 class_masterbooks[16]; // varies\n   int16 subclass_books[16][8]; // varies\n   uint16 Xlist[31*8+2]; // varies\n   uint8 sorted_order[31*8+2];\n   uint8 neighbors[31*8+2][2];\n   uint8 floor1_multiplier;\n   uint8 rangebits;\n   int values;\n} Floor1;\n\ntypedef union\n{\n   Floor0 floor0;\n   Floor1 floor1;\n} Floor;\n\ntypedef struct\n{\n   uint32 begin, end;\n   uint32 part_size;\n   uint8 classifications;\n   uint8 classbook;\n   uint8 **classdata;\n   int16 (*residue_books)[8];\n} Residue;\n\ntypedef struct\n{\n   uint8 magnitude;\n   uint8 angle;\n   uint8 mux;\n} MappingChannel;\n\ntypedef struct\n{\n   uint16 coupling_steps;\n   MappingChannel *chan;\n   uint8  submaps;\n   uint8  submap_floor[15]; // varies\n   uint8  submap_residue[15]; // varies\n} Mapping;\n\ntypedef struct\n{\n   uint8 blockflag;\n   uint8 mapping;\n   uint16 windowtype;\n   uint16 transformtype;\n} Mode;\n\ntypedef struct\n{\n   uint32  goal_crc;    // expected crc if match\n   int     bytes_left;  // bytes left in packet\n   uint32  crc_so_far;  // running crc\n   int     bytes_done;  // bytes processed in _current_ chunk\n   uint32  sample_loc;  // granule pos encoded in page\n} CRCscan;\n\ntypedef struct\n{\n   uint32 page_start, page_end;\n   uint32 last_decoded_sample;\n} ProbedPage;\n\nstruct stb_vorbis\n{\n  // user-accessible info\n   unsigned int sample_rate;\n   int channels;\n\n   unsigned int setup_memory_required;\n   unsigned int temp_memory_required;\n   unsigned int setup_temp_memory_required;\n\n   char *vendor;\n   int comment_list_length;\n   char **comment_list;\n\n  // input config\n#ifndef STB_VORBIS_NO_STDIO\n   FILE *f;\n   uint32 f_start;\n   int close_on_free;\n#endif\n\n   uint8 *stream;\n   uint8 *stream_start;\n   uint8 *stream_end;\n\n   uint32 stream_len;\n\n   uint8  push_mode;\n\n   // the page to seek to when seeking to start, may be zero\n   uint32 first_audio_page_offset;\n\n   // p_first is the page on which the first audio packet ends\n   // (but not necessarily the page on which it starts)\n   ProbedPage p_first, p_last;\n\n  // memory management\n   stb_vorbis_alloc alloc;\n   int setup_offset;\n   int temp_offset;\n\n  // run-time results\n   int eof;\n   enum STBVorbisError error;\n\n  // user-useful data\n\n  // header info\n   int blocksize[2];\n   int blocksize_0, blocksize_1;\n   int codebook_count;\n   Codebook *codebooks;\n   int floor_count;\n   uint16 floor_types[64]; // varies\n   Floor *floor_config;\n   int residue_count;\n   uint16 residue_types[64]; // varies\n   Residue *residue_config;\n   int mapping_count;\n   Mapping *mapping;\n   int mode_count;\n   Mode mode_config[64];  // varies\n\n   uint32 total_samples;\n\n  // decode buffer\n   float *channel_buffers[STB_VORBIS_MAX_CHANNELS];\n   float *outputs        [STB_VORBIS_MAX_CHANNELS];\n\n   float *previous_window[STB_VORBIS_MAX_CHANNELS];\n   int previous_length;\n\n   #ifndef STB_VORBIS_NO_DEFER_FLOOR\n   int16 *finalY[STB_VORBIS_MAX_CHANNELS];\n   #else\n   float *floor_buffers[STB_VORBIS_MAX_CHANNELS];\n   #endif\n\n   uint32 current_loc; // sample location of next frame to decode\n   int    current_loc_valid;\n\n  // per-blocksize precomputed data\n\n   // twiddle factors\n   float *A[2],*B[2],*C[2];\n   float *window[2];\n   uint16 *bit_reverse[2];\n\n  // current page/packet/segment streaming info\n   uint32 serial; // stream serial number for verification\n   int last_page;\n   int segment_count;\n   uint8 segments[255];\n   uint8 page_flag;\n   uint8 bytes_in_seg;\n   uint8 first_decode;\n   int next_seg;\n   int last_seg;  // flag that we're on the last segment\n   int last_seg_which; // what was the segment number of the last seg?\n   uint32 acc;\n   int valid_bits;\n   int packet_bytes;\n   int end_seg_with_known_loc;\n   uint32 known_loc_for_packet;\n   int discard_samples_deferred;\n   uint32 samples_output;\n\n  // push mode scanning\n   int page_crc_tests; // only in push_mode: number of tests active; -1 if not searching\n#ifndef STB_VORBIS_NO_PUSHDATA_API\n   CRCscan scan[STB_VORBIS_PUSHDATA_CRC_COUNT];\n#endif\n\n  // sample-access\n   int channel_buffer_start;\n   int channel_buffer_end;\n};\n\n#if defined(STB_VORBIS_NO_PUSHDATA_API)\n   #define IS_PUSH_MODE(f)   FALSE\n#elif defined(STB_VORBIS_NO_PULLDATA_API)\n   #define IS_PUSH_MODE(f)   TRUE\n#else\n   #define IS_PUSH_MODE(f)   ((f)->push_mode)\n#endif\n\ntypedef struct stb_vorbis vorb;\n\nstatic int error(vorb *f, enum STBVorbisError e)\n{\n   f->error = e;\n   if (!f->eof && e != VORBIS_need_more_data) {\n      f->error=e; // breakpoint for debugging\n   }\n   return 0;\n}\n\n\n// these functions are used for allocating temporary memory\n// while decoding. if you can afford the stack space, use\n// alloca(); otherwise, provide a temp buffer and it will\n// allocate out of those.\n\n#define array_size_required(count,size)  (count*(sizeof(void *)+(size)))\n\n#define temp_alloc(f,size)              (f->alloc.alloc_buffer ? setup_temp_malloc(f,size) : alloca(size))\n#define temp_free(f,p)                  (void)0\n#define temp_alloc_save(f)              ((f)->temp_offset)\n#define temp_alloc_restore(f,p)         ((f)->temp_offset = (p))\n\n#define temp_block_array(f,count,size)  make_block_array(temp_alloc(f,array_size_required(count,size)), count, size)\n\n// given a sufficiently large block of memory, make an array of pointers to subblocks of it\nstatic void *make_block_array(void *mem, int count, int size)\n{\n   int i;\n   void ** p = (void **) mem;\n   char *q = (char *) (p + count);\n   for (i=0; i < count; ++i) {\n      p[i] = q;\n      q += size;\n   }\n   return p;\n}\n\nstatic void *setup_malloc(vorb *f, int sz)\n{\n   sz = (sz+7) & ~7; // round up to nearest 8 for alignment of future allocs.\n   f->setup_memory_required += sz;\n   if (f->alloc.alloc_buffer) {\n      void *p = (char *) f->alloc.alloc_buffer + f->setup_offset;\n      if (f->setup_offset + sz > f->temp_offset) return NULL;\n      f->setup_offset += sz;\n      return p;\n   }\n   return sz ? malloc(sz) : NULL;\n}\n\nstatic void setup_free(vorb *f, void *p)\n{\n   if (f->alloc.alloc_buffer) return; // do nothing; setup mem is a stack\n   free(p);\n}\n\nstatic void *setup_temp_malloc(vorb *f, int sz)\n{\n   sz = (sz+7) & ~7; // round up to nearest 8 for alignment of future allocs.\n   if (f->alloc.alloc_buffer) {\n      if (f->temp_offset - sz < f->setup_offset) return NULL;\n      f->temp_offset -= sz;\n      return (char *) f->alloc.alloc_buffer + f->temp_offset;\n   }\n   return malloc(sz);\n}\n\nstatic void setup_temp_free(vorb *f, void *p, int sz)\n{\n   if (f->alloc.alloc_buffer) {\n      f->temp_offset += (sz+7)&~7;\n      return;\n   }\n   free(p);\n}\n\n#define CRC32_POLY    0x04c11db7   // from spec\n\nstatic uint32 crc_table[256];\nstatic void crc32_init(void)\n{\n   int i,j;\n   uint32 s;\n   for(i=0; i < 256; i++) {\n      for (s=(uint32) i << 24, j=0; j < 8; ++j)\n         s = (s << 1) ^ (s >= (1U<<31) ? CRC32_POLY : 0);\n      crc_table[i] = s;\n   }\n}\n\nstatic __forceinline uint32 crc32_update(uint32 crc, uint8 byte)\n{\n   return (crc << 8) ^ crc_table[byte ^ (crc >> 24)];\n}\n\n\n// used in setup, and for huffman that doesn't go fast path\nstatic unsigned int bit_reverse(unsigned int n)\n{\n  n = ((n & 0xAAAAAAAA) >>  1) | ((n & 0x55555555) << 1);\n  n = ((n & 0xCCCCCCCC) >>  2) | ((n & 0x33333333) << 2);\n  n = ((n & 0xF0F0F0F0) >>  4) | ((n & 0x0F0F0F0F) << 4);\n  n = ((n & 0xFF00FF00) >>  8) | ((n & 0x00FF00FF) << 8);\n  return (n >> 16) | (n << 16);\n}\n\nstatic float square(float x)\n{\n   return x*x;\n}\n\n// this is a weird definition of log2() for which log2(1) = 1, log2(2) = 2, log2(4) = 3\n// as required by the specification. fast(?) implementation from stb.h\n// @OPTIMIZE: called multiple times per-packet with \"constants\"; move to setup\nstatic int ilog(int32 n)\n{\n   static signed char log2_4[16] = { 0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4 };\n\n   if (n < 0) return 0; // signed n returns 0\n\n   // 2 compares if n < 16, 3 compares otherwise (4 if signed or n > 1<<29)\n   if (n < (1 << 14))\n        if (n < (1 <<  4))            return  0 + log2_4[n      ];\n        else if (n < (1 <<  9))       return  5 + log2_4[n >>  5];\n             else                     return 10 + log2_4[n >> 10];\n   else if (n < (1 << 24))\n             if (n < (1 << 19))       return 15 + log2_4[n >> 15];\n             else                     return 20 + log2_4[n >> 20];\n        else if (n < (1 << 29))       return 25 + log2_4[n >> 25];\n             else                     return 30 + log2_4[n >> 30];\n}\n\n#ifndef M_PI\n  #define M_PI  3.14159265358979323846264f  // from CRC\n#endif\n\n// code length assigned to a value with no huffman encoding\n#define NO_CODE   255\n\n/////////////////////// LEAF SETUP FUNCTIONS //////////////////////////\n//\n// these functions are only called at setup, and only a few times\n// per file\n\nstatic float float32_unpack(uint32 x)\n{\n   // from the specification\n   uint32 mantissa = x & 0x1fffff;\n   uint32 sign = x & 0x80000000;\n   uint32 exp = (x & 0x7fe00000) >> 21;\n   double res = sign ? -(double)mantissa : (double)mantissa;\n   return (float) ldexp((float)res, (int)exp-788);\n}\n\n\n// zlib & jpeg huffman tables assume that the output symbols\n// can either be arbitrarily arranged, or have monotonically\n// increasing frequencies--they rely on the lengths being sorted;\n// this makes for a very simple generation algorithm.\n// vorbis allows a huffman table with non-sorted lengths. This\n// requires a more sophisticated construction, since symbols in\n// order do not map to huffman codes \"in order\".\nstatic void add_entry(Codebook *c, uint32 huff_code, int symbol, int count, int len, uint32 *values)\n{\n   if (!c->sparse) {\n      c->codewords      [symbol] = huff_code;\n   } else {\n      c->codewords       [count] = huff_code;\n      c->codeword_lengths[count] = len;\n      values             [count] = symbol;\n   }\n}\n\nstatic int compute_codewords(Codebook *c, uint8 *len, int n, uint32 *values)\n{\n   int i,k,m=0;\n   uint32 available[32];\n\n   memset(available, 0, sizeof(available));\n   // find the first entry\n   for (k=0; k < n; ++k) if (len[k] < NO_CODE) break;\n   if (k == n) { assert(c->sorted_entries == 0); return TRUE; }\n   assert(len[k] < 32); // no error return required, code reading lens checks this\n   // add to the list\n   add_entry(c, 0, k, m++, len[k], values);\n   // add all available leaves\n   for (i=1; i <= len[k]; ++i)\n      available[i] = 1U << (32-i);\n   // note that the above code treats the first case specially,\n   // but it's really the same as the following code, so they\n   // could probably be combined (except the initial code is 0,\n   // and I use 0 in available[] to mean 'empty')\n   for (i=k+1; i < n; ++i) {\n      uint32 res;\n      int z = len[i], y;\n      if (z == NO_CODE) continue;\n      assert(z < 32); // no error return required, code reading lens checks this\n      // find lowest available leaf (should always be earliest,\n      // which is what the specification calls for)\n      // note that this property, and the fact we can never have\n      // more than one free leaf at a given level, isn't totally\n      // trivial to prove, but it seems true and the assert never\n      // fires, so!\n      while (z > 0 && !available[z]) --z;\n      if (z == 0) { return FALSE; }\n      res = available[z];\n      available[z] = 0;\n      add_entry(c, bit_reverse(res), i, m++, len[i], values);\n      // propagate availability up the tree\n      if (z != len[i]) {\n         for (y=len[i]; y > z; --y) {\n            assert(available[y] == 0);\n            available[y] = res + (1 << (32-y));\n         }\n      }\n   }\n   return TRUE;\n}\n\n// accelerated huffman table allows fast O(1) match of all symbols\n// of length <= STB_VORBIS_FAST_HUFFMAN_LENGTH\nstatic void compute_accelerated_huffman(Codebook *c)\n{\n   int i, len;\n   for (i=0; i < FAST_HUFFMAN_TABLE_SIZE; ++i)\n      c->fast_huffman[i] = -1;\n\n   len = c->sparse ? c->sorted_entries : c->entries;\n   #ifdef STB_VORBIS_FAST_HUFFMAN_SHORT\n   if (len > 32767) len = 32767; // largest possible value we can encode!\n   #endif\n   for (i=0; i < len; ++i) {\n      if (c->codeword_lengths[i] <= STB_VORBIS_FAST_HUFFMAN_LENGTH) {\n         uint32 z = c->sparse ? bit_reverse(c->sorted_codewords[i]) : c->codewords[i];\n         // set table entries for all bit combinations in the higher bits\n         while (z < FAST_HUFFMAN_TABLE_SIZE) {\n             c->fast_huffman[z] = i;\n             z += 1 << c->codeword_lengths[i];\n         }\n      }\n   }\n}\n\n#ifdef _MSC_VER\n#define STBV_CDECL __cdecl\n#else\n#define STBV_CDECL\n#endif\n\nstatic int STBV_CDECL uint32_compare(const void *p, const void *q)\n{\n   uint32 x = * (uint32 *) p;\n   uint32 y = * (uint32 *) q;\n   return x < y ? -1 : x > y;\n}\n\nstatic int include_in_sort(Codebook *c, uint8 len)\n{\n   if (c->sparse) { assert(len != NO_CODE); return TRUE; }\n   if (len == NO_CODE) return FALSE;\n   if (len > STB_VORBIS_FAST_HUFFMAN_LENGTH) return TRUE;\n   return FALSE;\n}\n\n// if the fast table above doesn't work, we want to binary\n// search them... need to reverse the bits\nstatic void compute_sorted_huffman(Codebook *c, uint8 *lengths, uint32 *values)\n{\n   int i, len;\n   // build a list of all the entries\n   // OPTIMIZATION: don't include the short ones, since they'll be caught by FAST_HUFFMAN.\n   // this is kind of a frivolous optimization--I don't see any performance improvement,\n   // but it's like 4 extra lines of code, so.\n   if (!c->sparse) {\n      int k = 0;\n      for (i=0; i < c->entries; ++i)\n         if (include_in_sort(c, lengths[i]))\n            c->sorted_codewords[k++] = bit_reverse(c->codewords[i]);\n      assert(k == c->sorted_entries);\n   } else {\n      for (i=0; i < c->sorted_entries; ++i)\n         c->sorted_codewords[i] = bit_reverse(c->codewords[i]);\n   }\n\n   qsort(c->sorted_codewords, c->sorted_entries, sizeof(c->sorted_codewords[0]), uint32_compare);\n   c->sorted_codewords[c->sorted_entries] = 0xffffffff;\n\n   len = c->sparse ? c->sorted_entries : c->entries;\n   // now we need to indicate how they correspond; we could either\n   //   #1: sort a different data structure that says who they correspond to\n   //   #2: for each sorted entry, search the original list to find who corresponds\n   //   #3: for each original entry, find the sorted entry\n   // #1 requires extra storage, #2 is slow, #3 can use binary search!\n   for (i=0; i < len; ++i) {\n      int huff_len = c->sparse ? lengths[values[i]] : lengths[i];\n      if (include_in_sort(c,huff_len)) {\n         uint32 code = bit_reverse(c->codewords[i]);\n         int x=0, n=c->sorted_entries;\n         while (n > 1) {\n            // invariant: sc[x] <= code < sc[x+n]\n            int m = x + (n >> 1);\n            if (c->sorted_codewords[m] <= code) {\n               x = m;\n               n -= (n>>1);\n            } else {\n               n >>= 1;\n            }\n         }\n         assert(c->sorted_codewords[x] == code);\n         if (c->sparse) {\n            c->sorted_values[x] = values[i];\n            c->codeword_lengths[x] = huff_len;\n         } else {\n            c->sorted_values[x] = i;\n         }\n      }\n   }\n}\n\n// only run while parsing the header (3 times)\nstatic int vorbis_validate(uint8 *data)\n{\n   static uint8 vorbis[6] = { 'v', 'o', 'r', 'b', 'i', 's' };\n   return memcmp(data, vorbis, 6) == 0;\n}\n\n// called from setup only, once per code book\n// (formula implied by specification)\nstatic int lookup1_values(int entries, int dim)\n{\n   int r = (int) floor(exp((float) log((float) entries) / dim));\n   if ((int) floor(pow((float) r+1, dim)) <= entries)   // (int) cast for MinGW warning;\n      ++r;                                              // floor() to avoid _ftol() when non-CRT\n   if (pow((float) r+1, dim) <= entries)\n      return -1;\n   if ((int) floor(pow((float) r, dim)) > entries)\n      return -1;\n   return r;\n}\n\n// called twice per file\nstatic void compute_twiddle_factors(int n, float *A, float *B, float *C)\n{\n   int n4 = n >> 2, n8 = n >> 3;\n   int k,k2;\n\n   for (k=k2=0; k < n4; ++k,k2+=2) {\n      A[k2  ] = (float)  cos(4*k*M_PI/n);\n      A[k2+1] = (float) -sin(4*k*M_PI/n);\n      B[k2  ] = (float)  cos((k2+1)*M_PI/n/2) * 0.5f;\n      B[k2+1] = (float)  sin((k2+1)*M_PI/n/2) * 0.5f;\n   }\n   for (k=k2=0; k < n8; ++k,k2+=2) {\n      C[k2  ] = (float)  cos(2*(k2+1)*M_PI/n);\n      C[k2+1] = (float) -sin(2*(k2+1)*M_PI/n);\n   }\n}\n\nstatic void compute_window(int n, float *window)\n{\n   int n2 = n >> 1, i;\n   for (i=0; i < n2; ++i)\n      window[i] = (float) sin(0.5 * M_PI * square((float) sin((i - 0 + 0.5) / n2 * 0.5 * M_PI)));\n}\n\nstatic void compute_bitreverse(int n, uint16 *rev)\n{\n   int ld = ilog(n) - 1; // ilog is off-by-one from normal definitions\n   int i, n8 = n >> 3;\n   for (i=0; i < n8; ++i)\n      rev[i] = (bit_reverse(i) >> (32-ld+3)) << 2;\n}\n\nstatic int init_blocksize(vorb *f, int b, int n)\n{\n   int n2 = n >> 1, n4 = n >> 2, n8 = n >> 3;\n   f->A[b] = (float *) setup_malloc(f, sizeof(float) * n2);\n   f->B[b] = (float *) setup_malloc(f, sizeof(float) * n2);\n   f->C[b] = (float *) setup_malloc(f, sizeof(float) * n4);\n   if (!f->A[b] || !f->B[b] || !f->C[b]) return error(f, VORBIS_outofmem);\n   compute_twiddle_factors(n, f->A[b], f->B[b], f->C[b]);\n   f->window[b] = (float *) setup_malloc(f, sizeof(float) * n2);\n   if (!f->window[b]) return error(f, VORBIS_outofmem);\n   compute_window(n, f->window[b]);\n   f->bit_reverse[b] = (uint16 *) setup_malloc(f, sizeof(uint16) * n8);\n   if (!f->bit_reverse[b]) return error(f, VORBIS_outofmem);\n   compute_bitreverse(n, f->bit_reverse[b]);\n   return TRUE;\n}\n\nstatic void neighbors(uint16 *x, int n, int *plow, int *phigh)\n{\n   int low = -1;\n   int high = 65536;\n   int i;\n   for (i=0; i < n; ++i) {\n      if (x[i] > low  && x[i] < x[n]) { *plow  = i; low = x[i]; }\n      if (x[i] < high && x[i] > x[n]) { *phigh = i; high = x[i]; }\n   }\n}\n\n// this has been repurposed so y is now the original index instead of y\ntypedef struct\n{\n   uint16 x,id;\n} stbv__floor_ordering;\n\nstatic int STBV_CDECL point_compare(const void *p, const void *q)\n{\n   stbv__floor_ordering *a = (stbv__floor_ordering *) p;\n   stbv__floor_ordering *b = (stbv__floor_ordering *) q;\n   return a->x < b->x ? -1 : a->x > b->x;\n}\n\n//\n/////////////////////// END LEAF SETUP FUNCTIONS //////////////////////////\n\n\n#if defined(STB_VORBIS_NO_STDIO)\n   #define USE_MEMORY(z)    TRUE\n#else\n   #define USE_MEMORY(z)    ((z)->stream)\n#endif\n\nstatic uint8 get8(vorb *z)\n{\n   if (USE_MEMORY(z)) {\n      if (z->stream >= z->stream_end) { z->eof = TRUE; return 0; }\n      return *z->stream++;\n   }\n\n   #ifndef STB_VORBIS_NO_STDIO\n   {\n   int c = fgetc(z->f);\n   if (c == EOF) { z->eof = TRUE; return 0; }\n   return c;\n   }\n   #endif\n}\n\nstatic uint32 get32(vorb *f)\n{\n   uint32 x;\n   x = get8(f);\n   x += get8(f) << 8;\n   x += get8(f) << 16;\n   x += (uint32) get8(f) << 24;\n   return x;\n}\n\nstatic int getn(vorb *z, uint8 *data, int n)\n{\n   if (USE_MEMORY(z)) {\n      if (z->stream+n > z->stream_end) { z->eof = 1; return 0; }\n      memcpy(data, z->stream, n);\n      z->stream += n;\n      return 1;\n   }\n\n   #ifndef STB_VORBIS_NO_STDIO\n   if (fread(data, n, 1, z->f) == 1)\n      return 1;\n   else {\n      z->eof = 1;\n      return 0;\n   }\n   #endif\n}\n\nstatic void skip(vorb *z, int n)\n{\n   if (USE_MEMORY(z)) {\n      z->stream += n;\n      if (z->stream >= z->stream_end) z->eof = 1;\n      return;\n   }\n   #ifndef STB_VORBIS_NO_STDIO\n   {\n      long x = ftell(z->f);\n      fseek(z->f, x+n, SEEK_SET);\n   }\n   #endif\n}\n\nstatic int set_file_offset(stb_vorbis *f, unsigned int loc)\n{\n   #ifndef STB_VORBIS_NO_PUSHDATA_API\n   if (f->push_mode) return 0;\n   #endif\n   f->eof = 0;\n   if (USE_MEMORY(f)) {\n      if (f->stream_start + loc >= f->stream_end || f->stream_start + loc < f->stream_start) {\n         f->stream = f->stream_end;\n         f->eof = 1;\n         return 0;\n      } else {\n         f->stream = f->stream_start + loc;\n         return 1;\n      }\n   }\n   #ifndef STB_VORBIS_NO_STDIO\n   if (loc + f->f_start < loc || loc >= 0x80000000) {\n      loc = 0x7fffffff;\n      f->eof = 1;\n   } else {\n      loc += f->f_start;\n   }\n   if (!fseek(f->f, loc, SEEK_SET))\n      return 1;\n   f->eof = 1;\n   fseek(f->f, f->f_start, SEEK_END);\n   return 0;\n   #endif\n}\n\n\nstatic uint8 ogg_page_header[4] = { 0x4f, 0x67, 0x67, 0x53 };\n\nstatic int capture_pattern(vorb *f)\n{\n   if (0x4f != get8(f)) return FALSE;\n   if (0x67 != get8(f)) return FALSE;\n   if (0x67 != get8(f)) return FALSE;\n   if (0x53 != get8(f)) return FALSE;\n   return TRUE;\n}\n\n#define PAGEFLAG_continued_packet   1\n#define PAGEFLAG_first_page         2\n#define PAGEFLAG_last_page          4\n\nstatic int start_page_no_capturepattern(vorb *f)\n{\n   uint32 loc0,loc1,n;\n   if (f->first_decode && !IS_PUSH_MODE(f)) {\n      f->p_first.page_start = stb_vorbis_get_file_offset(f) - 4;\n   }\n   // stream structure version\n   if (0 != get8(f)) return error(f, VORBIS_invalid_stream_structure_version);\n   // header flag\n   f->page_flag = get8(f);\n   // absolute granule position\n   loc0 = get32(f);\n   loc1 = get32(f);\n   // @TODO: validate loc0,loc1 as valid positions?\n   // stream serial number -- vorbis doesn't interleave, so discard\n   get32(f);\n   //if (f->serial != get32(f)) return error(f, VORBIS_incorrect_stream_serial_number);\n   // page sequence number\n   n = get32(f);\n   f->last_page = n;\n   // CRC32\n   get32(f);\n   // page_segments\n   f->segment_count = get8(f);\n   if (!getn(f, f->segments, f->segment_count))\n      return error(f, VORBIS_unexpected_eof);\n   // assume we _don't_ know any the sample position of any segments\n   f->end_seg_with_known_loc = -2;\n   if (loc0 != ~0U || loc1 != ~0U) {\n      int i;\n      // determine which packet is the last one that will complete\n      for (i=f->segment_count-1; i >= 0; --i)\n         if (f->segments[i] < 255)\n            break;\n      // 'i' is now the index of the _last_ segment of a packet that ends\n      if (i >= 0) {\n         f->end_seg_with_known_loc = i;\n         f->known_loc_for_packet   = loc0;\n      }\n   }\n   if (f->first_decode) {\n      int i,len;\n      len = 0;\n      for (i=0; i < f->segment_count; ++i)\n         len += f->segments[i];\n      len += 27 + f->segment_count;\n      f->p_first.page_end = f->p_first.page_start + len;\n      f->p_first.last_decoded_sample = loc0;\n   }\n   f->next_seg = 0;\n   return TRUE;\n}\n\nstatic int start_page(vorb *f)\n{\n   if (!capture_pattern(f)) return error(f, VORBIS_missing_capture_pattern);\n   return start_page_no_capturepattern(f);\n}\n\nstatic int start_packet(vorb *f)\n{\n   while (f->next_seg == -1) {\n      if (!start_page(f)) return FALSE;\n      if (f->page_flag & PAGEFLAG_continued_packet)\n         return error(f, VORBIS_continued_packet_flag_invalid);\n   }\n   f->last_seg = FALSE;\n   f->valid_bits = 0;\n   f->packet_bytes = 0;\n   f->bytes_in_seg = 0;\n   // f->next_seg is now valid\n   return TRUE;\n}\n\nstatic int maybe_start_packet(vorb *f)\n{\n   if (f->next_seg == -1) {\n      int x = get8(f);\n      if (f->eof) return FALSE; // EOF at page boundary is not an error!\n      if (0x4f != x      ) return error(f, VORBIS_missing_capture_pattern);\n      if (0x67 != get8(f)) return error(f, VORBIS_missing_capture_pattern);\n      if (0x67 != get8(f)) return error(f, VORBIS_missing_capture_pattern);\n      if (0x53 != get8(f)) return error(f, VORBIS_missing_capture_pattern);\n      if (!start_page_no_capturepattern(f)) return FALSE;\n      if (f->page_flag & PAGEFLAG_continued_packet) {\n         // set up enough state that we can read this packet if we want,\n         // e.g. during recovery\n         f->last_seg = FALSE;\n         f->bytes_in_seg = 0;\n         return error(f, VORBIS_continued_packet_flag_invalid);\n      }\n   }\n   return start_packet(f);\n}\n\nstatic int next_segment(vorb *f)\n{\n   int len;\n   if (f->last_seg) return 0;\n   if (f->next_seg == -1) {\n      f->last_seg_which = f->segment_count-1; // in case start_page fails\n      if (!start_page(f)) { f->last_seg = 1; return 0; }\n      if (!(f->page_flag & PAGEFLAG_continued_packet)) return error(f, VORBIS_continued_packet_flag_invalid);\n   }\n   len = f->segments[f->next_seg++];\n   if (len < 255) {\n      f->last_seg = TRUE;\n      f->last_seg_which = f->next_seg-1;\n   }\n   if (f->next_seg >= f->segment_count)\n      f->next_seg = -1;\n   assert(f->bytes_in_seg == 0);\n   f->bytes_in_seg = len;\n   return len;\n}\n\n#define EOP    (-1)\n#define INVALID_BITS  (-1)\n\nstatic int get8_packet_raw(vorb *f)\n{\n   if (!f->bytes_in_seg) {  // CLANG!\n      if (f->last_seg) return EOP;\n      else if (!next_segment(f)) return EOP;\n   }\n   assert(f->bytes_in_seg > 0);\n   --f->bytes_in_seg;\n   ++f->packet_bytes;\n   return get8(f);\n}\n\nstatic int get8_packet(vorb *f)\n{\n   int x = get8_packet_raw(f);\n   f->valid_bits = 0;\n   return x;\n}\n\nstatic int get32_packet(vorb *f)\n{\n   uint32 x;\n   x = get8_packet(f);\n   x += get8_packet(f) << 8;\n   x += get8_packet(f) << 16;\n   x += (uint32) get8_packet(f) << 24;\n   return x;\n}\n\nstatic void flush_packet(vorb *f)\n{\n   while (get8_packet_raw(f) != EOP);\n}\n\n// @OPTIMIZE: this is the secondary bit decoder, so it's probably not as important\n// as the huffman decoder?\nstatic uint32 get_bits(vorb *f, int n)\n{\n   uint32 z;\n\n   if (f->valid_bits < 0) return 0;\n   if (f->valid_bits < n) {\n      if (n > 24) {\n         // the accumulator technique below would not work correctly in this case\n         z = get_bits(f, 24);\n         z += get_bits(f, n-24) << 24;\n         return z;\n      }\n      if (f->valid_bits == 0) f->acc = 0;\n      while (f->valid_bits < n) {\n         int z = get8_packet_raw(f);\n         if (z == EOP) {\n            f->valid_bits = INVALID_BITS;\n            return 0;\n         }\n         f->acc += z << f->valid_bits;\n         f->valid_bits += 8;\n      }\n   }\n\n   assert(f->valid_bits >= n);\n   z = f->acc & ((1 << n)-1);\n   f->acc >>= n;\n   f->valid_bits -= n;\n   return z;\n}\n\n// @OPTIMIZE: primary accumulator for huffman\n// expand the buffer to as many bits as possible without reading off end of packet\n// it might be nice to allow f->valid_bits and f->acc to be stored in registers,\n// e.g. cache them locally and decode locally\nstatic __forceinline void prep_huffman(vorb *f)\n{\n   if (f->valid_bits <= 24) {\n      if (f->valid_bits == 0) f->acc = 0;\n      do {\n         int z;\n         if (f->last_seg && !f->bytes_in_seg) return;\n         z = get8_packet_raw(f);\n         if (z == EOP) return;\n         f->acc += (unsigned) z << f->valid_bits;\n         f->valid_bits += 8;\n      } while (f->valid_bits <= 24);\n   }\n}\n\nenum\n{\n   VORBIS_packet_id = 1,\n   VORBIS_packet_comment = 3,\n   VORBIS_packet_setup = 5\n};\n\nstatic int codebook_decode_scalar_raw(vorb *f, Codebook *c)\n{\n   int i;\n   prep_huffman(f);\n\n   if (c->codewords == NULL && c->sorted_codewords == NULL)\n      return -1;\n\n   // cases to use binary search: sorted_codewords && !c->codewords\n   //                             sorted_codewords && c->entries > 8\n   if (c->entries > 8 ? c->sorted_codewords!=NULL : !c->codewords) {\n      // binary search\n      uint32 code = bit_reverse(f->acc);\n      int x=0, n=c->sorted_entries, len;\n\n      while (n > 1) {\n         // invariant: sc[x] <= code < sc[x+n]\n         int m = x + (n >> 1);\n         if (c->sorted_codewords[m] <= code) {\n            x = m;\n            n -= (n>>1);\n         } else {\n            n >>= 1;\n         }\n      }\n      // x is now the sorted index\n      if (!c->sparse) x = c->sorted_values[x];\n      // x is now sorted index if sparse, or symbol otherwise\n      len = c->codeword_lengths[x];\n      if (f->valid_bits >= len) {\n         f->acc >>= len;\n         f->valid_bits -= len;\n         return x;\n      }\n\n      f->valid_bits = 0;\n      return -1;\n   }\n\n   // if small, linear search\n   assert(!c->sparse);\n   for (i=0; i < c->entries; ++i) {\n      if (c->codeword_lengths[i] == NO_CODE) continue;\n      if (c->codewords[i] == (f->acc & ((1 << c->codeword_lengths[i])-1))) {\n         if (f->valid_bits >= c->codeword_lengths[i]) {\n            f->acc >>= c->codeword_lengths[i];\n            f->valid_bits -= c->codeword_lengths[i];\n            return i;\n         }\n         f->valid_bits = 0;\n         return -1;\n      }\n   }\n\n   error(f, VORBIS_invalid_stream);\n   f->valid_bits = 0;\n   return -1;\n}\n\n#ifndef STB_VORBIS_NO_INLINE_DECODE\n\n#define DECODE_RAW(var, f,c)                                  \\\n   if (f->valid_bits < STB_VORBIS_FAST_HUFFMAN_LENGTH)        \\\n      prep_huffman(f);                                        \\\n   var = f->acc & FAST_HUFFMAN_TABLE_MASK;                    \\\n   var = c->fast_huffman[var];                                \\\n   if (var >= 0) {                                            \\\n      int n = c->codeword_lengths[var];                       \\\n      f->acc >>= n;                                           \\\n      f->valid_bits -= n;                                     \\\n      if (f->valid_bits < 0) { f->valid_bits = 0; var = -1; } \\\n   } else {                                                   \\\n      var = codebook_decode_scalar_raw(f,c);                  \\\n   }\n\n#else\n\nstatic int codebook_decode_scalar(vorb *f, Codebook *c)\n{\n   int i;\n   if (f->valid_bits < STB_VORBIS_FAST_HUFFMAN_LENGTH)\n      prep_huffman(f);\n   // fast huffman table lookup\n   i = f->acc & FAST_HUFFMAN_TABLE_MASK;\n   i = c->fast_huffman[i];\n   if (i >= 0) {\n      f->acc >>= c->codeword_lengths[i];\n      f->valid_bits -= c->codeword_lengths[i];\n      if (f->valid_bits < 0) { f->valid_bits = 0; return -1; }\n      return i;\n   }\n   return codebook_decode_scalar_raw(f,c);\n}\n\n#define DECODE_RAW(var,f,c)    var = codebook_decode_scalar(f,c);\n\n#endif\n\n#define DECODE(var,f,c)                                       \\\n   DECODE_RAW(var,f,c)                                        \\\n   if (c->sparse) var = c->sorted_values[var];\n\n#ifndef STB_VORBIS_DIVIDES_IN_CODEBOOK\n  #define DECODE_VQ(var,f,c)   DECODE_RAW(var,f,c)\n#else\n  #define DECODE_VQ(var,f,c)   DECODE(var,f,c)\n#endif\n\n\n\n\n\n\n// CODEBOOK_ELEMENT_FAST is an optimization for the CODEBOOK_FLOATS case\n// where we avoid one addition\n#define CODEBOOK_ELEMENT(c,off)          (c->multiplicands[off])\n#define CODEBOOK_ELEMENT_FAST(c,off)     (c->multiplicands[off])\n#define CODEBOOK_ELEMENT_BASE(c)         (0)\n\nstatic int codebook_decode_start(vorb *f, Codebook *c)\n{\n   int z = -1;\n\n   // type 0 is only legal in a scalar context\n   if (c->lookup_type == 0)\n      error(f, VORBIS_invalid_stream);\n   else {\n      DECODE_VQ(z,f,c);\n      if (c->sparse) assert(z < c->sorted_entries);\n      if (z < 0) {  // check for EOP\n         if (!f->bytes_in_seg)\n            if (f->last_seg)\n               return z;\n         error(f, VORBIS_invalid_stream);\n      }\n   }\n   return z;\n}\n\nstatic int codebook_decode(vorb *f, Codebook *c, float *output, int len)\n{\n   int i,z = codebook_decode_start(f,c);\n   if (z < 0) return FALSE;\n   if (len > c->dimensions) len = c->dimensions;\n\n#ifdef STB_VORBIS_DIVIDES_IN_CODEBOOK\n   if (c->lookup_type == 1) {\n      float last = CODEBOOK_ELEMENT_BASE(c);\n      int div = 1;\n      for (i=0; i < len; ++i) {\n         int off = (z / div) % c->lookup_values;\n         float val = CODEBOOK_ELEMENT_FAST(c,off) + last;\n         output[i] += val;\n         if (c->sequence_p) last = val + c->minimum_value;\n         div *= c->lookup_values;\n      }\n      return TRUE;\n   }\n#endif\n\n   z *= c->dimensions;\n   if (c->sequence_p) {\n      float last = CODEBOOK_ELEMENT_BASE(c);\n      for (i=0; i < len; ++i) {\n         float val = CODEBOOK_ELEMENT_FAST(c,z+i) + last;\n         output[i] += val;\n         last = val + c->minimum_value;\n      }\n   } else {\n      float last = CODEBOOK_ELEMENT_BASE(c);\n      for (i=0; i < len; ++i) {\n         output[i] += CODEBOOK_ELEMENT_FAST(c,z+i) + last;\n      }\n   }\n\n   return TRUE;\n}\n\nstatic int codebook_decode_step(vorb *f, Codebook *c, float *output, int len, int step)\n{\n   int i,z = codebook_decode_start(f,c);\n   float last = CODEBOOK_ELEMENT_BASE(c);\n   if (z < 0) return FALSE;\n   if (len > c->dimensions) len = c->dimensions;\n\n#ifdef STB_VORBIS_DIVIDES_IN_CODEBOOK\n   if (c->lookup_type == 1) {\n      int div = 1;\n      for (i=0; i < len; ++i) {\n         int off = (z / div) % c->lookup_values;\n         float val = CODEBOOK_ELEMENT_FAST(c,off) + last;\n         output[i*step] += val;\n         if (c->sequence_p) last = val;\n         div *= c->lookup_values;\n      }\n      return TRUE;\n   }\n#endif\n\n   z *= c->dimensions;\n   for (i=0; i < len; ++i) {\n      float val = CODEBOOK_ELEMENT_FAST(c,z+i) + last;\n      output[i*step] += val;\n      if (c->sequence_p) last = val;\n   }\n\n   return TRUE;\n}\n\nstatic int codebook_decode_deinterleave_repeat(vorb *f, Codebook *c, float **outputs, int ch, int *c_inter_p, int *p_inter_p, int len, int total_decode)\n{\n   int c_inter = *c_inter_p;\n   int p_inter = *p_inter_p;\n   int i,z, effective = c->dimensions;\n\n   // type 0 is only legal in a scalar context\n   if (c->lookup_type == 0)   return error(f, VORBIS_invalid_stream);\n\n   while (total_decode > 0) {\n      float last = CODEBOOK_ELEMENT_BASE(c);\n      DECODE_VQ(z,f,c);\n      #ifndef STB_VORBIS_DIVIDES_IN_CODEBOOK\n      assert(!c->sparse || z < c->sorted_entries);\n      #endif\n      if (z < 0) {\n         if (!f->bytes_in_seg)\n            if (f->last_seg) return FALSE;\n         return error(f, VORBIS_invalid_stream);\n      }\n\n      // if this will take us off the end of the buffers, stop short!\n      // we check by computing the length of the virtual interleaved\n      // buffer (len*ch), our current offset within it (p_inter*ch)+(c_inter),\n      // and the length we'll be using (effective)\n      if (c_inter + p_inter*ch + effective > len * ch) {\n         effective = len*ch - (p_inter*ch - c_inter);\n      }\n\n   #ifdef STB_VORBIS_DIVIDES_IN_CODEBOOK\n      if (c->lookup_type == 1) {\n         int div = 1;\n         for (i=0; i < effective; ++i) {\n            int off = (z / div) % c->lookup_values;\n            float val = CODEBOOK_ELEMENT_FAST(c,off) + last;\n            if (outputs[c_inter])\n               outputs[c_inter][p_inter] += val;\n            if (++c_inter == ch) { c_inter = 0; ++p_inter; }\n            if (c->sequence_p) last = val;\n            div *= c->lookup_values;\n         }\n      } else\n   #endif\n      {\n         z *= c->dimensions;\n         if (c->sequence_p) {\n            for (i=0; i < effective; ++i) {\n               float val = CODEBOOK_ELEMENT_FAST(c,z+i) + last;\n               if (outputs[c_inter])\n                  outputs[c_inter][p_inter] += val;\n               if (++c_inter == ch) { c_inter = 0; ++p_inter; }\n               last = val;\n            }\n         } else {\n            for (i=0; i < effective; ++i) {\n               float val = CODEBOOK_ELEMENT_FAST(c,z+i) + last;\n               if (outputs[c_inter])\n                  outputs[c_inter][p_inter] += val;\n               if (++c_inter == ch) { c_inter = 0; ++p_inter; }\n            }\n         }\n      }\n\n      total_decode -= effective;\n   }\n   *c_inter_p = c_inter;\n   *p_inter_p = p_inter;\n   return TRUE;\n}\n\nstatic int predict_point(int x, int x0, int x1, int y0, int y1)\n{\n   int dy = y1 - y0;\n   int adx = x1 - x0;\n   // @OPTIMIZE: force int division to round in the right direction... is this necessary on x86?\n   int err = abs(dy) * (x - x0);\n   int off = err / adx;\n   return dy < 0 ? y0 - off : y0 + off;\n}\n\n// the following table is block-copied from the specification\nstatic float inverse_db_table[256] =\n{\n  1.0649863e-07f, 1.1341951e-07f, 1.2079015e-07f, 1.2863978e-07f,\n  1.3699951e-07f, 1.4590251e-07f, 1.5538408e-07f, 1.6548181e-07f,\n  1.7623575e-07f, 1.8768855e-07f, 1.9988561e-07f, 2.1287530e-07f,\n  2.2670913e-07f, 2.4144197e-07f, 2.5713223e-07f, 2.7384213e-07f,\n  2.9163793e-07f, 3.1059021e-07f, 3.3077411e-07f, 3.5226968e-07f,\n  3.7516214e-07f, 3.9954229e-07f, 4.2550680e-07f, 4.5315863e-07f,\n  4.8260743e-07f, 5.1396998e-07f, 5.4737065e-07f, 5.8294187e-07f,\n  6.2082472e-07f, 6.6116941e-07f, 7.0413592e-07f, 7.4989464e-07f,\n  7.9862701e-07f, 8.5052630e-07f, 9.0579828e-07f, 9.6466216e-07f,\n  1.0273513e-06f, 1.0941144e-06f, 1.1652161e-06f, 1.2409384e-06f,\n  1.3215816e-06f, 1.4074654e-06f, 1.4989305e-06f, 1.5963394e-06f,\n  1.7000785e-06f, 1.8105592e-06f, 1.9282195e-06f, 2.0535261e-06f,\n  2.1869758e-06f, 2.3290978e-06f, 2.4804557e-06f, 2.6416497e-06f,\n  2.8133190e-06f, 2.9961443e-06f, 3.1908506e-06f, 3.3982101e-06f,\n  3.6190449e-06f, 3.8542308e-06f, 4.1047004e-06f, 4.3714470e-06f,\n  4.6555282e-06f, 4.9580707e-06f, 5.2802740e-06f, 5.6234160e-06f,\n  5.9888572e-06f, 6.3780469e-06f, 6.7925283e-06f, 7.2339451e-06f,\n  7.7040476e-06f, 8.2047000e-06f, 8.7378876e-06f, 9.3057248e-06f,\n  9.9104632e-06f, 1.0554501e-05f, 1.1240392e-05f, 1.1970856e-05f,\n  1.2748789e-05f, 1.3577278e-05f, 1.4459606e-05f, 1.5399272e-05f,\n  1.6400004e-05f, 1.7465768e-05f, 1.8600792e-05f, 1.9809576e-05f,\n  2.1096914e-05f, 2.2467911e-05f, 2.3928002e-05f, 2.5482978e-05f,\n  2.7139006e-05f, 2.8902651e-05f, 3.0780908e-05f, 3.2781225e-05f,\n  3.4911534e-05f, 3.7180282e-05f, 3.9596466e-05f, 4.2169667e-05f,\n  4.4910090e-05f, 4.7828601e-05f, 5.0936773e-05f, 5.4246931e-05f,\n  5.7772202e-05f, 6.1526565e-05f, 6.5524908e-05f, 6.9783085e-05f,\n  7.4317983e-05f, 7.9147585e-05f, 8.4291040e-05f, 8.9768747e-05f,\n  9.5602426e-05f, 0.00010181521f, 0.00010843174f, 0.00011547824f,\n  0.00012298267f, 0.00013097477f, 0.00013948625f, 0.00014855085f,\n  0.00015820453f, 0.00016848555f, 0.00017943469f, 0.00019109536f,\n  0.00020351382f, 0.00021673929f, 0.00023082423f, 0.00024582449f,\n  0.00026179955f, 0.00027881276f, 0.00029693158f, 0.00031622787f,\n  0.00033677814f, 0.00035866388f, 0.00038197188f, 0.00040679456f,\n  0.00043323036f, 0.00046138411f, 0.00049136745f, 0.00052329927f,\n  0.00055730621f, 0.00059352311f, 0.00063209358f, 0.00067317058f,\n  0.00071691700f, 0.00076350630f, 0.00081312324f, 0.00086596457f,\n  0.00092223983f, 0.00098217216f, 0.0010459992f,  0.0011139742f,\n  0.0011863665f,  0.0012634633f,  0.0013455702f,  0.0014330129f,\n  0.0015261382f,  0.0016253153f,  0.0017309374f,  0.0018434235f,\n  0.0019632195f,  0.0020908006f,  0.0022266726f,  0.0023713743f,\n  0.0025254795f,  0.0026895994f,  0.0028643847f,  0.0030505286f,\n  0.0032487691f,  0.0034598925f,  0.0036847358f,  0.0039241906f,\n  0.0041792066f,  0.0044507950f,  0.0047400328f,  0.0050480668f,\n  0.0053761186f,  0.0057254891f,  0.0060975636f,  0.0064938176f,\n  0.0069158225f,  0.0073652516f,  0.0078438871f,  0.0083536271f,\n  0.0088964928f,  0.009474637f,   0.010090352f,   0.010746080f,\n  0.011444421f,   0.012188144f,   0.012980198f,   0.013823725f,\n  0.014722068f,   0.015678791f,   0.016697687f,   0.017782797f,\n  0.018938423f,   0.020169149f,   0.021479854f,   0.022875735f,\n  0.024362330f,   0.025945531f,   0.027631618f,   0.029427276f,\n  0.031339626f,   0.033376252f,   0.035545228f,   0.037855157f,\n  0.040315199f,   0.042935108f,   0.045725273f,   0.048696758f,\n  0.051861348f,   0.055231591f,   0.058820850f,   0.062643361f,\n  0.066714279f,   0.071049749f,   0.075666962f,   0.080584227f,\n  0.085821044f,   0.091398179f,   0.097337747f,   0.10366330f,\n  0.11039993f,    0.11757434f,    0.12521498f,    0.13335215f,\n  0.14201813f,    0.15124727f,    0.16107617f,    0.17154380f,\n  0.18269168f,    0.19456402f,    0.20720788f,    0.22067342f,\n  0.23501402f,    0.25028656f,    0.26655159f,    0.28387361f,\n  0.30232132f,    0.32196786f,    0.34289114f,    0.36517414f,\n  0.38890521f,    0.41417847f,    0.44109412f,    0.46975890f,\n  0.50028648f,    0.53279791f,    0.56742212f,    0.60429640f,\n  0.64356699f,    0.68538959f,    0.72993007f,    0.77736504f,\n  0.82788260f,    0.88168307f,    0.9389798f,     1.0f\n};\n\n\n// @OPTIMIZE: if you want to replace this bresenham line-drawing routine,\n// note that you must produce bit-identical output to decode correctly;\n// this specific sequence of operations is specified in the spec (it's\n// drawing integer-quantized frequency-space lines that the encoder\n// expects to be exactly the same)\n//     ... also, isn't the whole point of Bresenham's algorithm to NOT\n// have to divide in the setup? sigh.\n#ifndef STB_VORBIS_NO_DEFER_FLOOR\n#define LINE_OP(a,b)   a *= b\n#else\n#define LINE_OP(a,b)   a = b\n#endif\n\n#ifdef STB_VORBIS_DIVIDE_TABLE\n#define DIVTAB_NUMER   32\n#define DIVTAB_DENOM   64\nint8 integer_divide_table[DIVTAB_NUMER][DIVTAB_DENOM]; // 2KB\n#endif\n\nstatic __forceinline void draw_line(float *output, int x0, int y0, int x1, int y1, int n)\n{\n   int dy = y1 - y0;\n   int adx = x1 - x0;\n   int ady = abs(dy);\n   int base;\n   int x=x0,y=y0;\n   int err = 0;\n   int sy;\n\n#ifdef STB_VORBIS_DIVIDE_TABLE\n   if (adx < DIVTAB_DENOM && ady < DIVTAB_NUMER) {\n      if (dy < 0) {\n         base = -integer_divide_table[ady][adx];\n         sy = base-1;\n      } else {\n         base =  integer_divide_table[ady][adx];\n         sy = base+1;\n      }\n   } else {\n      base = dy / adx;\n      if (dy < 0)\n         sy = base - 1;\n      else\n         sy = base+1;\n   }\n#else\n   base = dy / adx;\n   if (dy < 0)\n      sy = base - 1;\n   else\n      sy = base+1;\n#endif\n   ady -= abs(base) * adx;\n   if (x1 > n) x1 = n;\n   if (x < x1) {\n      LINE_OP(output[x], inverse_db_table[y&255]);\n      for (++x; x < x1; ++x) {\n         err += ady;\n         if (err >= adx) {\n            err -= adx;\n            y += sy;\n         } else\n            y += base;\n         LINE_OP(output[x], inverse_db_table[y&255]);\n      }\n   }\n}\n\nstatic int residue_decode(vorb *f, Codebook *book, float *target, int offset, int n, int rtype)\n{\n   int k;\n   if (rtype == 0) {\n      int step = n / book->dimensions;\n      for (k=0; k < step; ++k)\n         if (!codebook_decode_step(f, book, target+offset+k, n-offset-k, step))\n            return FALSE;\n   } else {\n      for (k=0; k < n; ) {\n         if (!codebook_decode(f, book, target+offset, n-k))\n            return FALSE;\n         k += book->dimensions;\n         offset += book->dimensions;\n      }\n   }\n   return TRUE;\n}\n\n// n is 1/2 of the blocksize --\n// specification: \"Correct per-vector decode length is [n]/2\"\nstatic void decode_residue(vorb *f, float *residue_buffers[], int ch, int n, int rn, uint8 *do_not_decode)\n{\n   int i,j,pass;\n   Residue *r = f->residue_config + rn;\n   int rtype = f->residue_types[rn];\n   int c = r->classbook;\n   int classwords = f->codebooks[c].dimensions;\n   unsigned int actual_size = rtype == 2 ? n*2 : n;\n   unsigned int limit_r_begin = (r->begin < actual_size ? r->begin : actual_size);\n   unsigned int limit_r_end   = (r->end   < actual_size ? r->end   : actual_size);\n   int n_read = limit_r_end - limit_r_begin;\n   int part_read = n_read / r->part_size;\n   int temp_alloc_point = temp_alloc_save(f);\n   #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n   uint8 ***part_classdata = (uint8 ***) temp_block_array(f,f->channels, part_read * sizeof(**part_classdata));\n   #else\n   int **classifications = (int **) temp_block_array(f,f->channels, part_read * sizeof(**classifications));\n   #endif\n\n   CHECK(f);\n\n   for (i=0; i < ch; ++i)\n      if (!do_not_decode[i])\n         memset(residue_buffers[i], 0, sizeof(float) * n);\n\n   if (rtype == 2 && ch != 1) {\n      for (j=0; j < ch; ++j)\n         if (!do_not_decode[j])\n            break;\n      if (j == ch)\n         goto done;\n\n      for (pass=0; pass < 8; ++pass) {\n         int pcount = 0, class_set = 0;\n         if (ch == 2) {\n            while (pcount < part_read) {\n               int z = r->begin + pcount*r->part_size;\n               int c_inter = (z & 1), p_inter = z>>1;\n               if (pass == 0) {\n                  Codebook *c = f->codebooks+r->classbook;\n                  int q;\n                  DECODE(q,f,c);\n                  if (q == EOP) goto done;\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  part_classdata[0][class_set] = r->classdata[q];\n                  #else\n                  for (i=classwords-1; i >= 0; --i) {\n                     classifications[0][i+pcount] = q % r->classifications;\n                     q /= r->classifications;\n                  }\n                  #endif\n               }\n               for (i=0; i < classwords && pcount < part_read; ++i, ++pcount) {\n                  int z = r->begin + pcount*r->part_size;\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  int c = part_classdata[0][class_set][i];\n                  #else\n                  int c = classifications[0][pcount];\n                  #endif\n                  int b = r->residue_books[c][pass];\n                  if (b >= 0) {\n                     Codebook *book = f->codebooks + b;\n                     #ifdef STB_VORBIS_DIVIDES_IN_CODEBOOK\n                     if (!codebook_decode_deinterleave_repeat(f, book, residue_buffers, ch, &c_inter, &p_inter, n, r->part_size))\n                        goto done;\n                     #else\n                     // saves 1%\n                     if (!codebook_decode_deinterleave_repeat(f, book, residue_buffers, ch, &c_inter, &p_inter, n, r->part_size))\n                        goto done;\n                     #endif\n                  } else {\n                     z += r->part_size;\n                     c_inter = z & 1;\n                     p_inter = z >> 1;\n                  }\n               }\n               #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n               ++class_set;\n               #endif\n            }\n         } else if (ch > 2) {\n            while (pcount < part_read) {\n               int z = r->begin + pcount*r->part_size;\n               int c_inter = z % ch, p_inter = z/ch;\n               if (pass == 0) {\n                  Codebook *c = f->codebooks+r->classbook;\n                  int q;\n                  DECODE(q,f,c);\n                  if (q == EOP) goto done;\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  part_classdata[0][class_set] = r->classdata[q];\n                  #else\n                  for (i=classwords-1; i >= 0; --i) {\n                     classifications[0][i+pcount] = q % r->classifications;\n                     q /= r->classifications;\n                  }\n                  #endif\n               }\n               for (i=0; i < classwords && pcount < part_read; ++i, ++pcount) {\n                  int z = r->begin + pcount*r->part_size;\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  int c = part_classdata[0][class_set][i];\n                  #else\n                  int c = classifications[0][pcount];\n                  #endif\n                  int b = r->residue_books[c][pass];\n                  if (b >= 0) {\n                     Codebook *book = f->codebooks + b;\n                     if (!codebook_decode_deinterleave_repeat(f, book, residue_buffers, ch, &c_inter, &p_inter, n, r->part_size))\n                        goto done;\n                  } else {\n                     z += r->part_size;\n                     c_inter = z % ch;\n                     p_inter = z / ch;\n                  }\n               }\n               #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n               ++class_set;\n               #endif\n            }\n         }\n      }\n      goto done;\n   }\n   CHECK(f);\n\n   for (pass=0; pass < 8; ++pass) {\n      int pcount = 0, class_set=0;\n      while (pcount < part_read) {\n         if (pass == 0) {\n            for (j=0; j < ch; ++j) {\n               if (!do_not_decode[j]) {\n                  Codebook *c = f->codebooks+r->classbook;\n                  int temp;\n                  DECODE(temp,f,c);\n                  if (temp == EOP) goto done;\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  part_classdata[j][class_set] = r->classdata[temp];\n                  #else\n                  for (i=classwords-1; i >= 0; --i) {\n                     classifications[j][i+pcount] = temp % r->classifications;\n                     temp /= r->classifications;\n                  }\n                  #endif\n               }\n            }\n         }\n         for (i=0; i < classwords && pcount < part_read; ++i, ++pcount) {\n            for (j=0; j < ch; ++j) {\n               if (!do_not_decode[j]) {\n                  #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n                  int c = part_classdata[j][class_set][i];\n                  #else\n                  int c = classifications[j][pcount];\n                  #endif\n                  int b = r->residue_books[c][pass];\n                  if (b >= 0) {\n                     float *target = residue_buffers[j];\n                     int offset = r->begin + pcount * r->part_size;\n                     int n = r->part_size;\n                     Codebook *book = f->codebooks + b;\n                     if (!residue_decode(f, book, target, offset, n, rtype))\n                        goto done;\n                  }\n               }\n            }\n         }\n         #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n         ++class_set;\n         #endif\n      }\n   }\n  done:\n   CHECK(f);\n   #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n   temp_free(f,part_classdata);\n   #else\n   temp_free(f,classifications);\n   #endif\n   temp_alloc_restore(f,temp_alloc_point);\n}\n\n\n#if 0\n// slow way for debugging\nvoid inverse_mdct_slow(float *buffer, int n)\n{\n   int i,j;\n   int n2 = n >> 1;\n   float *x = (float *) malloc(sizeof(*x) * n2);\n   memcpy(x, buffer, sizeof(*x) * n2);\n   for (i=0; i < n; ++i) {\n      float acc = 0;\n      for (j=0; j < n2; ++j)\n         // formula from paper:\n         //acc += n/4.0f * x[j] * (float) cos(M_PI / 2 / n * (2 * i + 1 + n/2.0)*(2*j+1));\n         // formula from wikipedia\n         //acc += 2.0f / n2 * x[j] * (float) cos(M_PI/n2 * (i + 0.5 + n2/2)*(j + 0.5));\n         // these are equivalent, except the formula from the paper inverts the multiplier!\n         // however, what actually works is NO MULTIPLIER!?!\n         //acc += 64 * 2.0f / n2 * x[j] * (float) cos(M_PI/n2 * (i + 0.5 + n2/2)*(j + 0.5));\n         acc += x[j] * (float) cos(M_PI / 2 / n * (2 * i + 1 + n/2.0)*(2*j+1));\n      buffer[i] = acc;\n   }\n   free(x);\n}\n#elif 0\n// same as above, but just barely able to run in real time on modern machines\nvoid inverse_mdct_slow(float *buffer, int n, vorb *f, int blocktype)\n{\n   float mcos[16384];\n   int i,j;\n   int n2 = n >> 1, nmask = (n << 2) -1;\n   float *x = (float *) malloc(sizeof(*x) * n2);\n   memcpy(x, buffer, sizeof(*x) * n2);\n   for (i=0; i < 4*n; ++i)\n      mcos[i] = (float) cos(M_PI / 2 * i / n);\n\n   for (i=0; i < n; ++i) {\n      float acc = 0;\n      for (j=0; j < n2; ++j)\n         acc += x[j] * mcos[(2 * i + 1 + n2)*(2*j+1) & nmask];\n      buffer[i] = acc;\n   }\n   free(x);\n}\n#elif 0\n// transform to use a slow dct-iv; this is STILL basically trivial,\n// but only requires half as many ops\nvoid dct_iv_slow(float *buffer, int n)\n{\n   float mcos[16384];\n   float x[2048];\n   int i,j;\n   int n2 = n >> 1, nmask = (n << 3) - 1;\n   memcpy(x, buffer, sizeof(*x) * n);\n   for (i=0; i < 8*n; ++i)\n      mcos[i] = (float) cos(M_PI / 4 * i / n);\n   for (i=0; i < n; ++i) {\n      float acc = 0;\n      for (j=0; j < n; ++j)\n         acc += x[j] * mcos[((2 * i + 1)*(2*j+1)) & nmask];\n      buffer[i] = acc;\n   }\n}\n\nvoid inverse_mdct_slow(float *buffer, int n, vorb *f, int blocktype)\n{\n   int i, n4 = n >> 2, n2 = n >> 1, n3_4 = n - n4;\n   float temp[4096];\n\n   memcpy(temp, buffer, n2 * sizeof(float));\n   dct_iv_slow(temp, n2);  // returns -c'-d, a-b'\n\n   for (i=0; i < n4  ; ++i) buffer[i] = temp[i+n4];            // a-b'\n   for (   ; i < n3_4; ++i) buffer[i] = -temp[n3_4 - i - 1];   // b-a', c+d'\n   for (   ; i < n   ; ++i) buffer[i] = -temp[i - n3_4];       // c'+d\n}\n#endif\n\n#ifndef LIBVORBIS_MDCT\n#define LIBVORBIS_MDCT 0\n#endif\n\n#if LIBVORBIS_MDCT\n// directly call the vorbis MDCT using an interface documented\n// by Jeff Roberts... useful for performance comparison\ntypedef struct\n{\n  int n;\n  int log2n;\n\n  float *trig;\n  int   *bitrev;\n\n  float scale;\n} mdct_lookup;\n\nextern void mdct_init(mdct_lookup *lookup, int n);\nextern void mdct_clear(mdct_lookup *l);\nextern void mdct_backward(mdct_lookup *init, float *in, float *out);\n\nmdct_lookup M1,M2;\n\nvoid inverse_mdct(float *buffer, int n, vorb *f, int blocktype)\n{\n   mdct_lookup *M;\n   if (M1.n == n) M = &M1;\n   else if (M2.n == n) M = &M2;\n   else if (M1.n == 0) { mdct_init(&M1, n); M = &M1; }\n   else {\n      if (M2.n) __asm int 3;\n      mdct_init(&M2, n);\n      M = &M2;\n   }\n\n   mdct_backward(M, buffer, buffer);\n}\n#endif\n\n\n// the following were split out into separate functions while optimizing;\n// they could be pushed back up but eh. __forceinline showed no change;\n// they're probably already being inlined.\nstatic void imdct_step3_iter0_loop(int n, float *e, int i_off, int k_off, float *A)\n{\n   float *ee0 = e + i_off;\n   float *ee2 = ee0 + k_off;\n   int i;\n\n   assert((n & 3) == 0);\n   for (i=(n>>2); i > 0; --i) {\n      float k00_20, k01_21;\n      k00_20  = ee0[ 0] - ee2[ 0];\n      k01_21  = ee0[-1] - ee2[-1];\n      ee0[ 0] += ee2[ 0];//ee0[ 0] = ee0[ 0] + ee2[ 0];\n      ee0[-1] += ee2[-1];//ee0[-1] = ee0[-1] + ee2[-1];\n      ee2[ 0] = k00_20 * A[0] - k01_21 * A[1];\n      ee2[-1] = k01_21 * A[0] + k00_20 * A[1];\n      A += 8;\n\n      k00_20  = ee0[-2] - ee2[-2];\n      k01_21  = ee0[-3] - ee2[-3];\n      ee0[-2] += ee2[-2];//ee0[-2] = ee0[-2] + ee2[-2];\n      ee0[-3] += ee2[-3];//ee0[-3] = ee0[-3] + ee2[-3];\n      ee2[-2] = k00_20 * A[0] - k01_21 * A[1];\n      ee2[-3] = k01_21 * A[0] + k00_20 * A[1];\n      A += 8;\n\n      k00_20  = ee0[-4] - ee2[-4];\n      k01_21  = ee0[-5] - ee2[-5];\n      ee0[-4] += ee2[-4];//ee0[-4] = ee0[-4] + ee2[-4];\n      ee0[-5] += ee2[-5];//ee0[-5] = ee0[-5] + ee2[-5];\n      ee2[-4] = k00_20 * A[0] - k01_21 * A[1];\n      ee2[-5] = k01_21 * A[0] + k00_20 * A[1];\n      A += 8;\n\n      k00_20  = ee0[-6] - ee2[-6];\n      k01_21  = ee0[-7] - ee2[-7];\n      ee0[-6] += ee2[-6];//ee0[-6] = ee0[-6] + ee2[-6];\n      ee0[-7] += ee2[-7];//ee0[-7] = ee0[-7] + ee2[-7];\n      ee2[-6] = k00_20 * A[0] - k01_21 * A[1];\n      ee2[-7] = k01_21 * A[0] + k00_20 * A[1];\n      A += 8;\n      ee0 -= 8;\n      ee2 -= 8;\n   }\n}\n\nstatic void imdct_step3_inner_r_loop(int lim, float *e, int d0, int k_off, float *A, int k1)\n{\n   int i;\n   float k00_20, k01_21;\n\n   float *e0 = e + d0;\n   float *e2 = e0 + k_off;\n\n   for (i=lim >> 2; i > 0; --i) {\n      k00_20 = e0[-0] - e2[-0];\n      k01_21 = e0[-1] - e2[-1];\n      e0[-0] += e2[-0];//e0[-0] = e0[-0] + e2[-0];\n      e0[-1] += e2[-1];//e0[-1] = e0[-1] + e2[-1];\n      e2[-0] = (k00_20)*A[0] - (k01_21) * A[1];\n      e2[-1] = (k01_21)*A[0] + (k00_20) * A[1];\n\n      A += k1;\n\n      k00_20 = e0[-2] - e2[-2];\n      k01_21 = e0[-3] - e2[-3];\n      e0[-2] += e2[-2];//e0[-2] = e0[-2] + e2[-2];\n      e0[-3] += e2[-3];//e0[-3] = e0[-3] + e2[-3];\n      e2[-2] = (k00_20)*A[0] - (k01_21) * A[1];\n      e2[-3] = (k01_21)*A[0] + (k00_20) * A[1];\n\n      A += k1;\n\n      k00_20 = e0[-4] - e2[-4];\n      k01_21 = e0[-5] - e2[-5];\n      e0[-4] += e2[-4];//e0[-4] = e0[-4] + e2[-4];\n      e0[-5] += e2[-5];//e0[-5] = e0[-5] + e2[-5];\n      e2[-4] = (k00_20)*A[0] - (k01_21) * A[1];\n      e2[-5] = (k01_21)*A[0] + (k00_20) * A[1];\n\n      A += k1;\n\n      k00_20 = e0[-6] - e2[-6];\n      k01_21 = e0[-7] - e2[-7];\n      e0[-6] += e2[-6];//e0[-6] = e0[-6] + e2[-6];\n      e0[-7] += e2[-7];//e0[-7] = e0[-7] + e2[-7];\n      e2[-6] = (k00_20)*A[0] - (k01_21) * A[1];\n      e2[-7] = (k01_21)*A[0] + (k00_20) * A[1];\n\n      e0 -= 8;\n      e2 -= 8;\n\n      A += k1;\n   }\n}\n\nstatic void imdct_step3_inner_s_loop(int n, float *e, int i_off, int k_off, float *A, int a_off, int k0)\n{\n   int i;\n   float A0 = A[0];\n   float A1 = A[0+1];\n   float A2 = A[0+a_off];\n   float A3 = A[0+a_off+1];\n   float A4 = A[0+a_off*2+0];\n   float A5 = A[0+a_off*2+1];\n   float A6 = A[0+a_off*3+0];\n   float A7 = A[0+a_off*3+1];\n\n   float k00,k11;\n\n   float *ee0 = e  +i_off;\n   float *ee2 = ee0+k_off;\n\n   for (i=n; i > 0; --i) {\n      k00     = ee0[ 0] - ee2[ 0];\n      k11     = ee0[-1] - ee2[-1];\n      ee0[ 0] =  ee0[ 0] + ee2[ 0];\n      ee0[-1] =  ee0[-1] + ee2[-1];\n      ee2[ 0] = (k00) * A0 - (k11) * A1;\n      ee2[-1] = (k11) * A0 + (k00) * A1;\n\n      k00     = ee0[-2] - ee2[-2];\n      k11     = ee0[-3] - ee2[-3];\n      ee0[-2] =  ee0[-2] + ee2[-2];\n      ee0[-3] =  ee0[-3] + ee2[-3];\n      ee2[-2] = (k00) * A2 - (k11) * A3;\n      ee2[-3] = (k11) * A2 + (k00) * A3;\n\n      k00     = ee0[-4] - ee2[-4];\n      k11     = ee0[-5] - ee2[-5];\n      ee0[-4] =  ee0[-4] + ee2[-4];\n      ee0[-5] =  ee0[-5] + ee2[-5];\n      ee2[-4] = (k00) * A4 - (k11) * A5;\n      ee2[-5] = (k11) * A4 + (k00) * A5;\n\n      k00     = ee0[-6] - ee2[-6];\n      k11     = ee0[-7] - ee2[-7];\n      ee0[-6] =  ee0[-6] + ee2[-6];\n      ee0[-7] =  ee0[-7] + ee2[-7];\n      ee2[-6] = (k00) * A6 - (k11) * A7;\n      ee2[-7] = (k11) * A6 + (k00) * A7;\n\n      ee0 -= k0;\n      ee2 -= k0;\n   }\n}\n\nstatic __forceinline void iter_54(float *z)\n{\n   float k00,k11,k22,k33;\n   float y0,y1,y2,y3;\n\n   k00  = z[ 0] - z[-4];\n   y0   = z[ 0] + z[-4];\n   y2   = z[-2] + z[-6];\n   k22  = z[-2] - z[-6];\n\n   z[-0] = y0 + y2;      // z0 + z4 + z2 + z6\n   z[-2] = y0 - y2;      // z0 + z4 - z2 - z6\n\n   // done with y0,y2\n\n   k33  = z[-3] - z[-7];\n\n   z[-4] = k00 + k33;    // z0 - z4 + z3 - z7\n   z[-6] = k00 - k33;    // z0 - z4 - z3 + z7\n\n   // done with k33\n\n   k11  = z[-1] - z[-5];\n   y1   = z[-1] + z[-5];\n   y3   = z[-3] + z[-7];\n\n   z[-1] = y1 + y3;      // z1 + z5 + z3 + z7\n   z[-3] = y1 - y3;      // z1 + z5 - z3 - z7\n   z[-5] = k11 - k22;    // z1 - z5 + z2 - z6\n   z[-7] = k11 + k22;    // z1 - z5 - z2 + z6\n}\n\nstatic void imdct_step3_inner_s_loop_ld654(int n, float *e, int i_off, float *A, int base_n)\n{\n   int a_off = base_n >> 3;\n   float A2 = A[0+a_off];\n   float *z = e + i_off;\n   float *base = z - 16 * n;\n\n   while (z > base) {\n      float k00,k11;\n      float l00,l11;\n\n      k00    = z[-0] - z[ -8];\n      k11    = z[-1] - z[ -9];\n      l00    = z[-2] - z[-10];\n      l11    = z[-3] - z[-11];\n      z[ -0] = z[-0] + z[ -8];\n      z[ -1] = z[-1] + z[ -9];\n      z[ -2] = z[-2] + z[-10];\n      z[ -3] = z[-3] + z[-11];\n      z[ -8] = k00;\n      z[ -9] = k11;\n      z[-10] = (l00+l11) * A2;\n      z[-11] = (l11-l00) * A2;\n\n      k00    = z[ -4] - z[-12];\n      k11    = z[ -5] - z[-13];\n      l00    = z[ -6] - z[-14];\n      l11    = z[ -7] - z[-15];\n      z[ -4] = z[ -4] + z[-12];\n      z[ -5] = z[ -5] + z[-13];\n      z[ -6] = z[ -6] + z[-14];\n      z[ -7] = z[ -7] + z[-15];\n      z[-12] = k11;\n      z[-13] = -k00;\n      z[-14] = (l11-l00) * A2;\n      z[-15] = (l00+l11) * -A2;\n\n      iter_54(z);\n      iter_54(z-8);\n      z -= 16;\n   }\n}\n\nstatic void inverse_mdct(float *buffer, int n, vorb *f, int blocktype)\n{\n   int n2 = n >> 1, n4 = n >> 2, n8 = n >> 3, l;\n   int ld;\n   // @OPTIMIZE: reduce register pressure by using fewer variables?\n   int save_point = temp_alloc_save(f);\n   float *buf2 = (float *) temp_alloc(f, n2 * sizeof(*buf2));\n   float *u=NULL,*v=NULL;\n   // twiddle factors\n   float *A = f->A[blocktype];\n\n   // IMDCT algorithm from \"The use of multirate filter banks for coding of high quality digital audio\"\n   // See notes about bugs in that paper in less-optimal implementation 'inverse_mdct_old' after this function.\n\n   // kernel from paper\n\n\n   // merged:\n   //   copy and reflect spectral data\n   //   step 0\n\n   // note that it turns out that the items added together during\n   // this step are, in fact, being added to themselves (as reflected\n   // by step 0). inexplicable inefficiency! this became obvious\n   // once I combined the passes.\n\n   // so there's a missing 'times 2' here (for adding X to itself).\n   // this propagates through linearly to the end, where the numbers\n   // are 1/2 too small, and need to be compensated for.\n\n   {\n      float *d,*e, *AA, *e_stop;\n      d = &buf2[n2-2];\n      AA = A;\n      e = &buffer[0];\n      e_stop = &buffer[n2];\n      while (e != e_stop) {\n         d[1] = (e[0] * AA[0] - e[2]*AA[1]);\n         d[0] = (e[0] * AA[1] + e[2]*AA[0]);\n         d -= 2;\n         AA += 2;\n         e += 4;\n      }\n\n      e = &buffer[n2-3];\n      while (d >= buf2) {\n         d[1] = (-e[2] * AA[0] - -e[0]*AA[1]);\n         d[0] = (-e[2] * AA[1] + -e[0]*AA[0]);\n         d -= 2;\n         AA += 2;\n         e -= 4;\n      }\n   }\n\n   // now we use symbolic names for these, so that we can\n   // possibly swap their meaning as we change which operations\n   // are in place\n\n   u = buffer;\n   v = buf2;\n\n   // step 2    (paper output is w, now u)\n   // this could be in place, but the data ends up in the wrong\n   // place... _somebody_'s got to swap it, so this is nominated\n   {\n      float *AA = &A[n2-8];\n      float *d0,*d1, *e0, *e1;\n\n      e0 = &v[n4];\n      e1 = &v[0];\n\n      d0 = &u[n4];\n      d1 = &u[0];\n\n      while (AA >= A) {\n         float v40_20, v41_21;\n\n         v41_21 = e0[1] - e1[1];\n         v40_20 = e0[0] - e1[0];\n         d0[1]  = e0[1] + e1[1];\n         d0[0]  = e0[0] + e1[0];\n         d1[1]  = v41_21*AA[4] - v40_20*AA[5];\n         d1[0]  = v40_20*AA[4] + v41_21*AA[5];\n\n         v41_21 = e0[3] - e1[3];\n         v40_20 = e0[2] - e1[2];\n         d0[3]  = e0[3] + e1[3];\n         d0[2]  = e0[2] + e1[2];\n         d1[3]  = v41_21*AA[0] - v40_20*AA[1];\n         d1[2]  = v40_20*AA[0] + v41_21*AA[1];\n\n         AA -= 8;\n\n         d0 += 4;\n         d1 += 4;\n         e0 += 4;\n         e1 += 4;\n      }\n   }\n\n   // step 3\n   ld = ilog(n) - 1; // ilog is off-by-one from normal definitions\n\n   // optimized step 3:\n\n   // the original step3 loop can be nested r inside s or s inside r;\n   // it's written originally as s inside r, but this is dumb when r\n   // iterates many times, and s few. So I have two copies of it and\n   // switch between them halfway.\n\n   // this is iteration 0 of step 3\n   imdct_step3_iter0_loop(n >> 4, u, n2-1-n4*0, -(n >> 3), A);\n   imdct_step3_iter0_loop(n >> 4, u, n2-1-n4*1, -(n >> 3), A);\n\n   // this is iteration 1 of step 3\n   imdct_step3_inner_r_loop(n >> 5, u, n2-1 - n8*0, -(n >> 4), A, 16);\n   imdct_step3_inner_r_loop(n >> 5, u, n2-1 - n8*1, -(n >> 4), A, 16);\n   imdct_step3_inner_r_loop(n >> 5, u, n2-1 - n8*2, -(n >> 4), A, 16);\n   imdct_step3_inner_r_loop(n >> 5, u, n2-1 - n8*3, -(n >> 4), A, 16);\n\n   l=2;\n   for (; l < (ld-3)>>1; ++l) {\n      int k0 = n >> (l+2), k0_2 = k0>>1;\n      int lim = 1 << (l+1);\n      int i;\n      for (i=0; i < lim; ++i)\n         imdct_step3_inner_r_loop(n >> (l+4), u, n2-1 - k0*i, -k0_2, A, 1 << (l+3));\n   }\n\n   for (; l < ld-6; ++l) {\n      int k0 = n >> (l+2), k1 = 1 << (l+3), k0_2 = k0>>1;\n      int rlim = n >> (l+6), r;\n      int lim = 1 << (l+1);\n      int i_off;\n      float *A0 = A;\n      i_off = n2-1;\n      for (r=rlim; r > 0; --r) {\n         imdct_step3_inner_s_loop(lim, u, i_off, -k0_2, A0, k1, k0);\n         A0 += k1*4;\n         i_off -= 8;\n      }\n   }\n\n   // iterations with count:\n   //   ld-6,-5,-4 all interleaved together\n   //       the big win comes from getting rid of needless flops\n   //         due to the constants on pass 5 & 4 being all 1 and 0;\n   //       combining them to be simultaneous to improve cache made little difference\n   imdct_step3_inner_s_loop_ld654(n >> 5, u, n2-1, A, n);\n\n   // output is u\n\n   // step 4, 5, and 6\n   // cannot be in-place because of step 5\n   {\n      uint16 *bitrev = f->bit_reverse[blocktype];\n      // weirdly, I'd have thought reading sequentially and writing\n      // erratically would have been better than vice-versa, but in\n      // fact that's not what my testing showed. (That is, with\n      // j = bitreverse(i), do you read i and write j, or read j and write i.)\n\n      float *d0 = &v[n4-4];\n      float *d1 = &v[n2-4];\n      while (d0 >= v) {\n         int k4;\n\n         k4 = bitrev[0];\n         d1[3] = u[k4+0];\n         d1[2] = u[k4+1];\n         d0[3] = u[k4+2];\n         d0[2] = u[k4+3];\n\n         k4 = bitrev[1];\n         d1[1] = u[k4+0];\n         d1[0] = u[k4+1];\n         d0[1] = u[k4+2];\n         d0[0] = u[k4+3];\n\n         d0 -= 4;\n         d1 -= 4;\n         bitrev += 2;\n      }\n   }\n   // (paper output is u, now v)\n\n\n   // data must be in buf2\n   assert(v == buf2);\n\n   // step 7   (paper output is v, now v)\n   // this is now in place\n   {\n      float *C = f->C[blocktype];\n      float *d, *e;\n\n      d = v;\n      e = v + n2 - 4;\n\n      while (d < e) {\n         float a02,a11,b0,b1,b2,b3;\n\n         a02 = d[0] - e[2];\n         a11 = d[1] + e[3];\n\n         b0 = C[1]*a02 + C[0]*a11;\n         b1 = C[1]*a11 - C[0]*a02;\n\n         b2 = d[0] + e[ 2];\n         b3 = d[1] - e[ 3];\n\n         d[0] = b2 + b0;\n         d[1] = b3 + b1;\n         e[2] = b2 - b0;\n         e[3] = b1 - b3;\n\n         a02 = d[2] - e[0];\n         a11 = d[3] + e[1];\n\n         b0 = C[3]*a02 + C[2]*a11;\n         b1 = C[3]*a11 - C[2]*a02;\n\n         b2 = d[2] + e[ 0];\n         b3 = d[3] - e[ 1];\n\n         d[2] = b2 + b0;\n         d[3] = b3 + b1;\n         e[0] = b2 - b0;\n         e[1] = b1 - b3;\n\n         C += 4;\n         d += 4;\n         e -= 4;\n      }\n   }\n\n   // data must be in buf2\n\n\n   // step 8+decode   (paper output is X, now buffer)\n   // this generates pairs of data a la 8 and pushes them directly through\n   // the decode kernel (pushing rather than pulling) to avoid having\n   // to make another pass later\n\n   // this cannot POSSIBLY be in place, so we refer to the buffers directly\n\n   {\n      float *d0,*d1,*d2,*d3;\n\n      float *B = f->B[blocktype] + n2 - 8;\n      float *e = buf2 + n2 - 8;\n      d0 = &buffer[0];\n      d1 = &buffer[n2-4];\n      d2 = &buffer[n2];\n      d3 = &buffer[n-4];\n      while (e >= v) {\n         float p0,p1,p2,p3;\n\n         p3 =  e[6]*B[7] - e[7]*B[6];\n         p2 = -e[6]*B[6] - e[7]*B[7];\n\n         d0[0] =   p3;\n         d1[3] = - p3;\n         d2[0] =   p2;\n         d3[3] =   p2;\n\n         p1 =  e[4]*B[5] - e[5]*B[4];\n         p0 = -e[4]*B[4] - e[5]*B[5];\n\n         d0[1] =   p1;\n         d1[2] = - p1;\n         d2[1] =   p0;\n         d3[2] =   p0;\n\n         p3 =  e[2]*B[3] - e[3]*B[2];\n         p2 = -e[2]*B[2] - e[3]*B[3];\n\n         d0[2] =   p3;\n         d1[1] = - p3;\n         d2[2] =   p2;\n         d3[1] =   p2;\n\n         p1 =  e[0]*B[1] - e[1]*B[0];\n         p0 = -e[0]*B[0] - e[1]*B[1];\n\n         d0[3] =   p1;\n         d1[0] = - p1;\n         d2[3] =   p0;\n         d3[0] =   p0;\n\n         B -= 8;\n         e -= 8;\n         d0 += 4;\n         d2 += 4;\n         d1 -= 4;\n         d3 -= 4;\n      }\n   }\n\n   temp_free(f,buf2);\n   temp_alloc_restore(f,save_point);\n}\n\n#if 0\n// this is the original version of the above code, if you want to optimize it from scratch\nvoid inverse_mdct_naive(float *buffer, int n)\n{\n   float s;\n   float A[1 << 12], B[1 << 12], C[1 << 11];\n   int i,k,k2,k4, n2 = n >> 1, n4 = n >> 2, n8 = n >> 3, l;\n   int n3_4 = n - n4, ld;\n   // how can they claim this only uses N words?!\n   // oh, because they're only used sparsely, whoops\n   float u[1 << 13], X[1 << 13], v[1 << 13], w[1 << 13];\n   // set up twiddle factors\n\n   for (k=k2=0; k < n4; ++k,k2+=2) {\n      A[k2  ] = (float)  cos(4*k*M_PI/n);\n      A[k2+1] = (float) -sin(4*k*M_PI/n);\n      B[k2  ] = (float)  cos((k2+1)*M_PI/n/2);\n      B[k2+1] = (float)  sin((k2+1)*M_PI/n/2);\n   }\n   for (k=k2=0; k < n8; ++k,k2+=2) {\n      C[k2  ] = (float)  cos(2*(k2+1)*M_PI/n);\n      C[k2+1] = (float) -sin(2*(k2+1)*M_PI/n);\n   }\n\n   // IMDCT algorithm from \"The use of multirate filter banks for coding of high quality digital audio\"\n   // Note there are bugs in that pseudocode, presumably due to them attempting\n   // to rename the arrays nicely rather than representing the way their actual\n   // implementation bounces buffers back and forth. As a result, even in the\n   // \"some formulars corrected\" version, a direct implementation fails. These\n   // are noted below as \"paper bug\".\n\n   // copy and reflect spectral data\n   for (k=0; k < n2; ++k) u[k] = buffer[k];\n   for (   ; k < n ; ++k) u[k] = -buffer[n - k - 1];\n   // kernel from paper\n   // step 1\n   for (k=k2=k4=0; k < n4; k+=1, k2+=2, k4+=4) {\n      v[n-k4-1] = (u[k4] - u[n-k4-1]) * A[k2]   - (u[k4+2] - u[n-k4-3])*A[k2+1];\n      v[n-k4-3] = (u[k4] - u[n-k4-1]) * A[k2+1] + (u[k4+2] - u[n-k4-3])*A[k2];\n   }\n   // step 2\n   for (k=k4=0; k < n8; k+=1, k4+=4) {\n      w[n2+3+k4] = v[n2+3+k4] + v[k4+3];\n      w[n2+1+k4] = v[n2+1+k4] + v[k4+1];\n      w[k4+3]    = (v[n2+3+k4] - v[k4+3])*A[n2-4-k4] - (v[n2+1+k4]-v[k4+1])*A[n2-3-k4];\n      w[k4+1]    = (v[n2+1+k4] - v[k4+1])*A[n2-4-k4] + (v[n2+3+k4]-v[k4+3])*A[n2-3-k4];\n   }\n   // step 3\n   ld = ilog(n) - 1; // ilog is off-by-one from normal definitions\n   for (l=0; l < ld-3; ++l) {\n      int k0 = n >> (l+2), k1 = 1 << (l+3);\n      int rlim = n >> (l+4), r4, r;\n      int s2lim = 1 << (l+2), s2;\n      for (r=r4=0; r < rlim; r4+=4,++r) {\n         for (s2=0; s2 < s2lim; s2+=2) {\n            u[n-1-k0*s2-r4] = w[n-1-k0*s2-r4] + w[n-1-k0*(s2+1)-r4];\n            u[n-3-k0*s2-r4] = w[n-3-k0*s2-r4] + w[n-3-k0*(s2+1)-r4];\n            u[n-1-k0*(s2+1)-r4] = (w[n-1-k0*s2-r4] - w[n-1-k0*(s2+1)-r4]) * A[r*k1]\n                                - (w[n-3-k0*s2-r4] - w[n-3-k0*(s2+1)-r4]) * A[r*k1+1];\n            u[n-3-k0*(s2+1)-r4] = (w[n-3-k0*s2-r4] - w[n-3-k0*(s2+1)-r4]) * A[r*k1]\n                                + (w[n-1-k0*s2-r4] - w[n-1-k0*(s2+1)-r4]) * A[r*k1+1];\n         }\n      }\n      if (l+1 < ld-3) {\n         // paper bug: ping-ponging of u&w here is omitted\n         memcpy(w, u, sizeof(u));\n      }\n   }\n\n   // step 4\n   for (i=0; i < n8; ++i) {\n      int j = bit_reverse(i) >> (32-ld+3);\n      assert(j < n8);\n      if (i == j) {\n         // paper bug: original code probably swapped in place; if copying,\n         //            need to directly copy in this case\n         int i8 = i << 3;\n         v[i8+1] = u[i8+1];\n         v[i8+3] = u[i8+3];\n         v[i8+5] = u[i8+5];\n         v[i8+7] = u[i8+7];\n      } else if (i < j) {\n         int i8 = i << 3, j8 = j << 3;\n         v[j8+1] = u[i8+1], v[i8+1] = u[j8 + 1];\n         v[j8+3] = u[i8+3], v[i8+3] = u[j8 + 3];\n         v[j8+5] = u[i8+5], v[i8+5] = u[j8 + 5];\n         v[j8+7] = u[i8+7], v[i8+7] = u[j8 + 7];\n      }\n   }\n   // step 5\n   for (k=0; k < n2; ++k) {\n      w[k] = v[k*2+1];\n   }\n   // step 6\n   for (k=k2=k4=0; k < n8; ++k, k2 += 2, k4 += 4) {\n      u[n-1-k2] = w[k4];\n      u[n-2-k2] = w[k4+1];\n      u[n3_4 - 1 - k2] = w[k4+2];\n      u[n3_4 - 2 - k2] = w[k4+3];\n   }\n   // step 7\n   for (k=k2=0; k < n8; ++k, k2 += 2) {\n      v[n2 + k2 ] = ( u[n2 + k2] + u[n-2-k2] + C[k2+1]*(u[n2+k2]-u[n-2-k2]) + C[k2]*(u[n2+k2+1]+u[n-2-k2+1]))/2;\n      v[n-2 - k2] = ( u[n2 + k2] + u[n-2-k2] - C[k2+1]*(u[n2+k2]-u[n-2-k2]) - C[k2]*(u[n2+k2+1]+u[n-2-k2+1]))/2;\n      v[n2+1+ k2] = ( u[n2+1+k2] - u[n-1-k2] + C[k2+1]*(u[n2+1+k2]+u[n-1-k2]) - C[k2]*(u[n2+k2]-u[n-2-k2]))/2;\n      v[n-1 - k2] = (-u[n2+1+k2] + u[n-1-k2] + C[k2+1]*(u[n2+1+k2]+u[n-1-k2]) - C[k2]*(u[n2+k2]-u[n-2-k2]))/2;\n   }\n   // step 8\n   for (k=k2=0; k < n4; ++k,k2 += 2) {\n      X[k]      = v[k2+n2]*B[k2  ] + v[k2+1+n2]*B[k2+1];\n      X[n2-1-k] = v[k2+n2]*B[k2+1] - v[k2+1+n2]*B[k2  ];\n   }\n\n   // decode kernel to output\n   // determined the following value experimentally\n   // (by first figuring out what made inverse_mdct_slow work); then matching that here\n   // (probably vorbis encoder premultiplies by n or n/2, to save it on the decoder?)\n   s = 0.5; // theoretically would be n4\n\n   // [[[ note! the s value of 0.5 is compensated for by the B[] in the current code,\n   //     so it needs to use the \"old\" B values to behave correctly, or else\n   //     set s to 1.0 ]]]\n   for (i=0; i < n4  ; ++i) buffer[i] = s * X[i+n4];\n   for (   ; i < n3_4; ++i) buffer[i] = -s * X[n3_4 - i - 1];\n   for (   ; i < n   ; ++i) buffer[i] = -s * X[i - n3_4];\n}\n#endif\n\nstatic float *get_window(vorb *f, int len)\n{\n   len <<= 1;\n   if (len == f->blocksize_0) return f->window[0];\n   if (len == f->blocksize_1) return f->window[1];\n   return NULL;\n}\n\n#ifndef STB_VORBIS_NO_DEFER_FLOOR\ntypedef int16 YTYPE;\n#else\ntypedef int YTYPE;\n#endif\nstatic int do_floor(vorb *f, Mapping *map, int i, int n, float *target, YTYPE *finalY, uint8 *step2_flag)\n{\n   int n2 = n >> 1;\n   int s = map->chan[i].mux, floor;\n   floor = map->submap_floor[s];\n   if (f->floor_types[floor] == 0) {\n      return error(f, VORBIS_invalid_stream);\n   } else {\n      Floor1 *g = &f->floor_config[floor].floor1;\n      int j,q;\n      int lx = 0, ly = finalY[0] * g->floor1_multiplier;\n      for (q=1; q < g->values; ++q) {\n         j = g->sorted_order[q];\n         #ifndef STB_VORBIS_NO_DEFER_FLOOR\n         STBV_NOTUSED(step2_flag);\n         if (finalY[j] >= 0)\n         #else\n         if (step2_flag[j])\n         #endif\n         {\n            int hy = finalY[j] * g->floor1_multiplier;\n            int hx = g->Xlist[j];\n            if (lx != hx)\n               draw_line(target, lx,ly, hx,hy, n2);\n            CHECK(f);\n            lx = hx, ly = hy;\n         }\n      }\n      if (lx < n2) {\n         // optimization of: draw_line(target, lx,ly, n,ly, n2);\n         for (j=lx; j < n2; ++j)\n            LINE_OP(target[j], inverse_db_table[ly]);\n         CHECK(f);\n      }\n   }\n   return TRUE;\n}\n\n// The meaning of \"left\" and \"right\"\n//\n// For a given frame:\n//     we compute samples from 0..n\n//     window_center is n/2\n//     we'll window and mix the samples from left_start to left_end with data from the previous frame\n//     all of the samples from left_end to right_start can be output without mixing; however,\n//        this interval is 0-length except when transitioning between short and long frames\n//     all of the samples from right_start to right_end need to be mixed with the next frame,\n//        which we don't have, so those get saved in a buffer\n//     frame N's right_end-right_start, the number of samples to mix with the next frame,\n//        has to be the same as frame N+1's left_end-left_start (which they are by\n//        construction)\n\nstatic int vorbis_decode_initial(vorb *f, int *p_left_start, int *p_left_end, int *p_right_start, int *p_right_end, int *mode)\n{\n   Mode *m;\n   int i, n, prev, next, window_center;\n   f->channel_buffer_start = f->channel_buffer_end = 0;\n\n  retry:\n   if (f->eof) return FALSE;\n   if (!maybe_start_packet(f))\n      return FALSE;\n   // check packet type\n   if (get_bits(f,1) != 0) {\n      if (IS_PUSH_MODE(f))\n         return error(f,VORBIS_bad_packet_type);\n      while (EOP != get8_packet(f));\n      goto retry;\n   }\n\n   if (f->alloc.alloc_buffer)\n      assert(f->alloc.alloc_buffer_length_in_bytes == f->temp_offset);\n\n   i = get_bits(f, ilog(f->mode_count-1));\n   if (i == EOP) return FALSE;\n   if (i >= f->mode_count) return FALSE;\n   *mode = i;\n   m = f->mode_config + i;\n   if (m->blockflag) {\n      n = f->blocksize_1;\n      prev = get_bits(f,1);\n      next = get_bits(f,1);\n   } else {\n      prev = next = 0;\n      n = f->blocksize_0;\n   }\n\n// WINDOWING\n\n   window_center = n >> 1;\n   if (m->blockflag && !prev) {\n      *p_left_start = (n - f->blocksize_0) >> 2;\n      *p_left_end   = (n + f->blocksize_0) >> 2;\n   } else {\n      *p_left_start = 0;\n      *p_left_end   = window_center;\n   }\n   if (m->blockflag && !next) {\n      *p_right_start = (n*3 - f->blocksize_0) >> 2;\n      *p_right_end   = (n*3 + f->blocksize_0) >> 2;\n   } else {\n      *p_right_start = window_center;\n      *p_right_end   = n;\n   }\n\n   return TRUE;\n}\n\nstatic int vorbis_decode_packet_rest(vorb *f, int *len, Mode *m, int left_start, int left_end, int right_start, int right_end, int *p_left)\n{\n   Mapping *map;\n   int i,j,k,n,n2;\n   int zero_channel[256];\n   int really_zero_channel[256];\n\n// WINDOWING\n\n   STBV_NOTUSED(left_end);\n   n = f->blocksize[m->blockflag];\n   map = &f->mapping[m->mapping];\n\n// FLOORS\n   n2 = n >> 1;\n\n   CHECK(f);\n\n   for (i=0; i < f->channels; ++i) {\n      int s = map->chan[i].mux, floor;\n      zero_channel[i] = FALSE;\n      floor = map->submap_floor[s];\n      if (f->floor_types[floor] == 0) {\n         return error(f, VORBIS_invalid_stream);\n      } else {\n         Floor1 *g = &f->floor_config[floor].floor1;\n         if (get_bits(f, 1)) {\n            short *finalY;\n            uint8 step2_flag[256];\n            static int range_list[4] = { 256, 128, 86, 64 };\n            int range = range_list[g->floor1_multiplier-1];\n            int offset = 2;\n            finalY = f->finalY[i];\n            finalY[0] = get_bits(f, ilog(range)-1);\n            finalY[1] = get_bits(f, ilog(range)-1);\n            for (j=0; j < g->partitions; ++j) {\n               int pclass = g->partition_class_list[j];\n               int cdim = g->class_dimensions[pclass];\n               int cbits = g->class_subclasses[pclass];\n               int csub = (1 << cbits)-1;\n               int cval = 0;\n               if (cbits) {\n                  Codebook *c = f->codebooks + g->class_masterbooks[pclass];\n                  DECODE(cval,f,c);\n               }\n               for (k=0; k < cdim; ++k) {\n                  int book = g->subclass_books[pclass][cval & csub];\n                  cval = cval >> cbits;\n                  if (book >= 0) {\n                     int temp;\n                     Codebook *c = f->codebooks + book;\n                     DECODE(temp,f,c);\n                     finalY[offset++] = temp;\n                  } else\n                     finalY[offset++] = 0;\n               }\n            }\n            if (f->valid_bits == INVALID_BITS) goto error; // behavior according to spec\n            step2_flag[0] = step2_flag[1] = 1;\n            for (j=2; j < g->values; ++j) {\n               int low, high, pred, highroom, lowroom, room, val;\n               low = g->neighbors[j][0];\n               high = g->neighbors[j][1];\n               //neighbors(g->Xlist, j, &low, &high);\n               pred = predict_point(g->Xlist[j], g->Xlist[low], g->Xlist[high], finalY[low], finalY[high]);\n               val = finalY[j];\n               highroom = range - pred;\n               lowroom = pred;\n               if (highroom < lowroom)\n                  room = highroom * 2;\n               else\n                  room = lowroom * 2;\n               if (val) {\n                  step2_flag[low] = step2_flag[high] = 1;\n                  step2_flag[j] = 1;\n                  if (val >= room)\n                     if (highroom > lowroom)\n                        finalY[j] = val - lowroom + pred;\n                     else\n                        finalY[j] = pred - val + highroom - 1;\n                  else\n                     if (val & 1)\n                        finalY[j] = pred - ((val+1)>>1);\n                     else\n                        finalY[j] = pred + (val>>1);\n               } else {\n                  step2_flag[j] = 0;\n                  finalY[j] = pred;\n               }\n            }\n\n#ifdef STB_VORBIS_NO_DEFER_FLOOR\n            do_floor(f, map, i, n, f->floor_buffers[i], finalY, step2_flag);\n#else\n            // defer final floor computation until _after_ residue\n            for (j=0; j < g->values; ++j) {\n               if (!step2_flag[j])\n                  finalY[j] = -1;\n            }\n#endif\n         } else {\n           error:\n            zero_channel[i] = TRUE;\n         }\n         // So we just defer everything else to later\n\n         // at this point we've decoded the floor into buffer\n      }\n   }\n   CHECK(f);\n   // at this point we've decoded all floors\n\n   if (f->alloc.alloc_buffer)\n      assert(f->alloc.alloc_buffer_length_in_bytes == f->temp_offset);\n\n   // re-enable coupled channels if necessary\n   memcpy(really_zero_channel, zero_channel, sizeof(really_zero_channel[0]) * f->channels);\n   for (i=0; i < map->coupling_steps; ++i)\n      if (!zero_channel[map->chan[i].magnitude] || !zero_channel[map->chan[i].angle]) {\n         zero_channel[map->chan[i].magnitude] = zero_channel[map->chan[i].angle] = FALSE;\n      }\n\n   CHECK(f);\n// RESIDUE DECODE\n   for (i=0; i < map->submaps; ++i) {\n      float *residue_buffers[STB_VORBIS_MAX_CHANNELS];\n      int r;\n      uint8 do_not_decode[256];\n      int ch = 0;\n      for (j=0; j < f->channels; ++j) {\n         if (map->chan[j].mux == i) {\n            if (zero_channel[j]) {\n               do_not_decode[ch] = TRUE;\n               residue_buffers[ch] = NULL;\n            } else {\n               do_not_decode[ch] = FALSE;\n               residue_buffers[ch] = f->channel_buffers[j];\n            }\n            ++ch;\n         }\n      }\n      r = map->submap_residue[i];\n      decode_residue(f, residue_buffers, ch, n2, r, do_not_decode);\n   }\n\n   if (f->alloc.alloc_buffer)\n      assert(f->alloc.alloc_buffer_length_in_bytes == f->temp_offset);\n   CHECK(f);\n\n// INVERSE COUPLING\n   for (i = map->coupling_steps-1; i >= 0; --i) {\n      int n2 = n >> 1;\n      float *m = f->channel_buffers[map->chan[i].magnitude];\n      float *a = f->channel_buffers[map->chan[i].angle    ];\n      for (j=0; j < n2; ++j) {\n         float a2,m2;\n         if (m[j] > 0)\n            if (a[j] > 0)\n               m2 = m[j], a2 = m[j] - a[j];\n            else\n               a2 = m[j], m2 = m[j] + a[j];\n         else\n            if (a[j] > 0)\n               m2 = m[j], a2 = m[j] + a[j];\n            else\n               a2 = m[j], m2 = m[j] - a[j];\n         m[j] = m2;\n         a[j] = a2;\n      }\n   }\n   CHECK(f);\n\n   // finish decoding the floors\n#ifndef STB_VORBIS_NO_DEFER_FLOOR\n   for (i=0; i < f->channels; ++i) {\n      if (really_zero_channel[i]) {\n         memset(f->channel_buffers[i], 0, sizeof(*f->channel_buffers[i]) * n2);\n      } else {\n         do_floor(f, map, i, n, f->channel_buffers[i], f->finalY[i], NULL);\n      }\n   }\n#else\n   for (i=0; i < f->channels; ++i) {\n      if (really_zero_channel[i]) {\n         memset(f->channel_buffers[i], 0, sizeof(*f->channel_buffers[i]) * n2);\n      } else {\n         for (j=0; j < n2; ++j)\n            f->channel_buffers[i][j] *= f->floor_buffers[i][j];\n      }\n   }\n#endif\n\n// INVERSE MDCT\n   CHECK(f);\n   for (i=0; i < f->channels; ++i)\n      inverse_mdct(f->channel_buffers[i], n, f, m->blockflag);\n   CHECK(f);\n\n   // this shouldn't be necessary, unless we exited on an error\n   // and want to flush to get to the next packet\n   flush_packet(f);\n\n   if (f->first_decode) {\n      // assume we start so first non-discarded sample is sample 0\n      // this isn't to spec, but spec would require us to read ahead\n      // and decode the size of all current frames--could be done,\n      // but presumably it's not a commonly used feature\n      f->current_loc = 0u - n2; // start of first frame is positioned for discard (NB this is an intentional unsigned overflow/wrap-around)\n      // we might have to discard samples \"from\" the next frame too,\n      // if we're lapping a large block then a small at the start?\n      f->discard_samples_deferred = n - right_end;\n      f->current_loc_valid = TRUE;\n      f->first_decode = FALSE;\n   } else if (f->discard_samples_deferred) {\n      if (f->discard_samples_deferred >= right_start - left_start) {\n         f->discard_samples_deferred -= (right_start - left_start);\n         left_start = right_start;\n         *p_left = left_start;\n      } else {\n         left_start += f->discard_samples_deferred;\n         *p_left = left_start;\n         f->discard_samples_deferred = 0;\n      }\n   } else if (f->previous_length == 0 && f->current_loc_valid) {\n      // we're recovering from a seek... that means we're going to discard\n      // the samples from this packet even though we know our position from\n      // the last page header, so we need to update the position based on\n      // the discarded samples here\n      // but wait, the code below is going to add this in itself even\n      // on a discard, so we don't need to do it here...\n   }\n\n   // check if we have ogg information about the sample # for this packet\n   if (f->last_seg_which == f->end_seg_with_known_loc) {\n      // if we have a valid current loc, and this is final:\n      if (f->current_loc_valid && (f->page_flag & PAGEFLAG_last_page)) {\n         uint32 current_end = f->known_loc_for_packet;\n         // then let's infer the size of the (probably) short final frame\n         if (current_end < f->current_loc + (right_end-left_start)) {\n            if (current_end < f->current_loc) {\n               // negative truncation, that's impossible!\n               *len = 0;\n            } else {\n               *len = current_end - f->current_loc;\n            }\n            *len += left_start; // this doesn't seem right, but has no ill effect on my test files\n            if (*len > right_end) *len = right_end; // this should never happen\n            f->current_loc += *len;\n            return TRUE;\n         }\n      }\n      // otherwise, just set our sample loc\n      // guess that the ogg granule pos refers to the _middle_ of the\n      // last frame?\n      // set f->current_loc to the position of left_start\n      f->current_loc = f->known_loc_for_packet - (n2-left_start);\n      f->current_loc_valid = TRUE;\n   }\n   if (f->current_loc_valid)\n      f->current_loc += (right_start - left_start);\n\n   if (f->alloc.alloc_buffer)\n      assert(f->alloc.alloc_buffer_length_in_bytes == f->temp_offset);\n   *len = right_end;  // ignore samples after the window goes to 0\n   CHECK(f);\n\n   return TRUE;\n}\n\nstatic int vorbis_decode_packet(vorb *f, int *len, int *p_left, int *p_right)\n{\n   int mode, left_end, right_end;\n   if (!vorbis_decode_initial(f, p_left, &left_end, p_right, &right_end, &mode)) return 0;\n   return vorbis_decode_packet_rest(f, len, f->mode_config + mode, *p_left, left_end, *p_right, right_end, p_left);\n}\n\nstatic int vorbis_finish_frame(stb_vorbis *f, int len, int left, int right)\n{\n   int prev,i,j;\n   // we use right&left (the start of the right- and left-window sin()-regions)\n   // to determine how much to return, rather than inferring from the rules\n   // (same result, clearer code); 'left' indicates where our sin() window\n   // starts, therefore where the previous window's right edge starts, and\n   // therefore where to start mixing from the previous buffer. 'right'\n   // indicates where our sin() ending-window starts, therefore that's where\n   // we start saving, and where our returned-data ends.\n\n   // mixin from previous window\n   if (f->previous_length) {\n      int i,j, n = f->previous_length;\n      float *w = get_window(f, n);\n      if (w == NULL) return 0;\n      for (i=0; i < f->channels; ++i) {\n         for (j=0; j < n; ++j)\n            f->channel_buffers[i][left+j] =\n               f->channel_buffers[i][left+j]*w[    j] +\n               f->previous_window[i][     j]*w[n-1-j];\n      }\n   }\n\n   prev = f->previous_length;\n\n   // last half of this data becomes previous window\n   f->previous_length = len - right;\n\n   // @OPTIMIZE: could avoid this copy by double-buffering the\n   // output (flipping previous_window with channel_buffers), but\n   // then previous_window would have to be 2x as large, and\n   // channel_buffers couldn't be temp mem (although they're NOT\n   // currently temp mem, they could be (unless we want to level\n   // performance by spreading out the computation))\n   for (i=0; i < f->channels; ++i)\n      for (j=0; right+j < len; ++j)\n         f->previous_window[i][j] = f->channel_buffers[i][right+j];\n\n   if (!prev)\n      // there was no previous packet, so this data isn't valid...\n      // this isn't entirely true, only the would-have-overlapped data\n      // isn't valid, but this seems to be what the spec requires\n      return 0;\n\n   // truncate a short frame\n   if (len < right) right = len;\n\n   f->samples_output += right-left;\n\n   return right - left;\n}\n\nstatic int vorbis_pump_first_frame(stb_vorbis *f)\n{\n   int len, right, left, res;\n   res = vorbis_decode_packet(f, &len, &left, &right);\n   if (res)\n      vorbis_finish_frame(f, len, left, right);\n   return res;\n}\n\n#ifndef STB_VORBIS_NO_PUSHDATA_API\nstatic int is_whole_packet_present(stb_vorbis *f)\n{\n   // make sure that we have the packet available before continuing...\n   // this requires a full ogg parse, but we know we can fetch from f->stream\n\n   // instead of coding this out explicitly, we could save the current read state,\n   // read the next packet with get8() until end-of-packet, check f->eof, then\n   // reset the state? but that would be slower, esp. since we'd have over 256 bytes\n   // of state to restore (primarily the page segment table)\n\n   int s = f->next_seg, first = TRUE;\n   uint8 *p = f->stream;\n\n   if (s != -1) { // if we're not starting the packet with a 'continue on next page' flag\n      for (; s < f->segment_count; ++s) {\n         p += f->segments[s];\n         if (f->segments[s] < 255)               // stop at first short segment\n            break;\n      }\n      // either this continues, or it ends it...\n      if (s == f->segment_count)\n         s = -1; // set 'crosses page' flag\n      if (p > f->stream_end)                     return error(f, VORBIS_need_more_data);\n      first = FALSE;\n   }\n   for (; s == -1;) {\n      uint8 *q;\n      int n;\n\n      // check that we have the page header ready\n      if (p + 26 >= f->stream_end)               return error(f, VORBIS_need_more_data);\n      // validate the page\n      if (memcmp(p, ogg_page_header, 4))         return error(f, VORBIS_invalid_stream);\n      if (p[4] != 0)                             return error(f, VORBIS_invalid_stream);\n      if (first) { // the first segment must NOT have 'continued_packet', later ones MUST\n         if (f->previous_length)\n            if ((p[5] & PAGEFLAG_continued_packet))  return error(f, VORBIS_invalid_stream);\n         // if no previous length, we're resynching, so we can come in on a continued-packet,\n         // which we'll just drop\n      } else {\n         if (!(p[5] & PAGEFLAG_continued_packet)) return error(f, VORBIS_invalid_stream);\n      }\n      n = p[26]; // segment counts\n      q = p+27;  // q points to segment table\n      p = q + n; // advance past header\n      // make sure we've read the segment table\n      if (p > f->stream_end)                     return error(f, VORBIS_need_more_data);\n      for (s=0; s < n; ++s) {\n         p += q[s];\n         if (q[s] < 255)\n            break;\n      }\n      if (s == n)\n         s = -1; // set 'crosses page' flag\n      if (p > f->stream_end)                     return error(f, VORBIS_need_more_data);\n      first = FALSE;\n   }\n   return TRUE;\n}\n#endif // !STB_VORBIS_NO_PUSHDATA_API\n\nstatic int start_decoder(vorb *f)\n{\n   uint8 header[6], x,y;\n   int len,i,j,k, max_submaps = 0;\n   int longest_floorlist=0;\n\n   // first page, first packet\n   f->first_decode = TRUE;\n\n   if (!start_page(f))                              return FALSE;\n   // validate page flag\n   if (!(f->page_flag & PAGEFLAG_first_page))       return error(f, VORBIS_invalid_first_page);\n   if (f->page_flag & PAGEFLAG_last_page)           return error(f, VORBIS_invalid_first_page);\n   if (f->page_flag & PAGEFLAG_continued_packet)    return error(f, VORBIS_invalid_first_page);\n   // check for expected packet length\n   if (f->segment_count != 1)                       return error(f, VORBIS_invalid_first_page);\n   if (f->segments[0] != 30) {\n      // check for the Ogg skeleton fishead identifying header to refine our error\n      if (f->segments[0] == 64 &&\n          getn(f, header, 6) &&\n          header[0] == 'f' &&\n          header[1] == 'i' &&\n          header[2] == 's' &&\n          header[3] == 'h' &&\n          header[4] == 'e' &&\n          header[5] == 'a' &&\n          get8(f)   == 'd' &&\n          get8(f)   == '\\0')                        return error(f, VORBIS_ogg_skeleton_not_supported);\n      else\n                                                    return error(f, VORBIS_invalid_first_page);\n   }\n\n   // read packet\n   // check packet header\n   if (get8(f) != VORBIS_packet_id)                 return error(f, VORBIS_invalid_first_page);\n   if (!getn(f, header, 6))                         return error(f, VORBIS_unexpected_eof);\n   if (!vorbis_validate(header))                    return error(f, VORBIS_invalid_first_page);\n   // vorbis_version\n   if (get32(f) != 0)                               return error(f, VORBIS_invalid_first_page);\n   f->channels = get8(f); if (!f->channels)         return error(f, VORBIS_invalid_first_page);\n   if (f->channels > STB_VORBIS_MAX_CHANNELS)       return error(f, VORBIS_too_many_channels);\n   f->sample_rate = get32(f); if (!f->sample_rate)  return error(f, VORBIS_invalid_first_page);\n   get32(f); // bitrate_maximum\n   get32(f); // bitrate_nominal\n   get32(f); // bitrate_minimum\n   x = get8(f);\n   {\n      int log0,log1;\n      log0 = x & 15;\n      log1 = x >> 4;\n      f->blocksize_0 = 1 << log0;\n      f->blocksize_1 = 1 << log1;\n      if (log0 < 6 || log0 > 13)                       return error(f, VORBIS_invalid_setup);\n      if (log1 < 6 || log1 > 13)                       return error(f, VORBIS_invalid_setup);\n      if (log0 > log1)                                 return error(f, VORBIS_invalid_setup);\n   }\n\n   // framing_flag\n   x = get8(f);\n   if (!(x & 1))                                    return error(f, VORBIS_invalid_first_page);\n\n   // second packet!\n   if (!start_page(f))                              return FALSE;\n\n   if (!start_packet(f))                            return FALSE;\n\n   if (!next_segment(f))                            return FALSE;\n\n   if (get8_packet(f) != VORBIS_packet_comment)            return error(f, VORBIS_invalid_setup);\n   for (i=0; i < 6; ++i) header[i] = get8_packet(f);\n   if (!vorbis_validate(header))                    return error(f, VORBIS_invalid_setup);\n   //file vendor\n   len = get32_packet(f);\n   f->vendor = (char*)setup_malloc(f, sizeof(char) * (len+1));\n   if (f->vendor == NULL)                           return error(f, VORBIS_outofmem);\n   for(i=0; i < len; ++i) {\n      f->vendor[i] = get8_packet(f);\n   }\n   f->vendor[len] = (char)'\\0';\n   //user comments\n   f->comment_list_length = get32_packet(f);\n   f->comment_list = NULL;\n   if (f->comment_list_length > 0)\n   {\n      f->comment_list = (char**) setup_malloc(f, sizeof(char*) * (f->comment_list_length));\n      if (f->comment_list == NULL)                  return error(f, VORBIS_outofmem);\n   }\n\n   for(i=0; i < f->comment_list_length; ++i) {\n      len = get32_packet(f);\n      f->comment_list[i] = (char*)setup_malloc(f, sizeof(char) * (len+1));\n      if (f->comment_list[i] == NULL)               return error(f, VORBIS_outofmem);\n\n      for(j=0; j < len; ++j) {\n         f->comment_list[i][j] = get8_packet(f);\n      }\n      f->comment_list[i][len] = (char)'\\0';\n   }\n\n   // framing_flag\n   x = get8_packet(f);\n   if (!(x & 1))                                    return error(f, VORBIS_invalid_setup);\n\n\n   skip(f, f->bytes_in_seg);\n   f->bytes_in_seg = 0;\n\n   do {\n      len = next_segment(f);\n      skip(f, len);\n      f->bytes_in_seg = 0;\n   } while (len);\n\n   // third packet!\n   if (!start_packet(f))                            return FALSE;\n\n   #ifndef STB_VORBIS_NO_PUSHDATA_API\n   if (IS_PUSH_MODE(f)) {\n      if (!is_whole_packet_present(f)) {\n         // convert error in ogg header to write type\n         if (f->error == VORBIS_invalid_stream)\n            f->error = VORBIS_invalid_setup;\n         return FALSE;\n      }\n   }\n   #endif\n\n   crc32_init(); // always init it, to avoid multithread race conditions\n\n   if (get8_packet(f) != VORBIS_packet_setup)       return error(f, VORBIS_invalid_setup);\n   for (i=0; i < 6; ++i) header[i] = get8_packet(f);\n   if (!vorbis_validate(header))                    return error(f, VORBIS_invalid_setup);\n\n   // codebooks\n\n   f->codebook_count = get_bits(f,8) + 1;\n   f->codebooks = (Codebook *) setup_malloc(f, sizeof(*f->codebooks) * f->codebook_count);\n   if (f->codebooks == NULL)                        return error(f, VORBIS_outofmem);\n   memset(f->codebooks, 0, sizeof(*f->codebooks) * f->codebook_count);\n   for (i=0; i < f->codebook_count; ++i) {\n      uint32 *values;\n      int ordered, sorted_count;\n      int total=0;\n      uint8 *lengths;\n      Codebook *c = f->codebooks+i;\n      CHECK(f);\n      x = get_bits(f, 8); if (x != 0x42)            return error(f, VORBIS_invalid_setup);\n      x = get_bits(f, 8); if (x != 0x43)            return error(f, VORBIS_invalid_setup);\n      x = get_bits(f, 8); if (x != 0x56)            return error(f, VORBIS_invalid_setup);\n      x = get_bits(f, 8);\n      c->dimensions = (get_bits(f, 8)<<8) + x;\n      x = get_bits(f, 8);\n      y = get_bits(f, 8);\n      c->entries = (get_bits(f, 8)<<16) + (y<<8) + x;\n      ordered = get_bits(f,1);\n      c->sparse = ordered ? 0 : get_bits(f,1);\n\n      if (c->dimensions == 0 && c->entries != 0)    return error(f, VORBIS_invalid_setup);\n\n      if (c->sparse)\n         lengths = (uint8 *) setup_temp_malloc(f, c->entries);\n      else\n         lengths = c->codeword_lengths = (uint8 *) setup_malloc(f, c->entries);\n\n      if (!lengths) return error(f, VORBIS_outofmem);\n\n      if (ordered) {\n         int current_entry = 0;\n         int current_length = get_bits(f,5) + 1;\n         while (current_entry < c->entries) {\n            int limit = c->entries - current_entry;\n            int n = get_bits(f, ilog(limit));\n            if (current_length >= 32) return error(f, VORBIS_invalid_setup);\n            if (current_entry + n > (int) c->entries) { return error(f, VORBIS_invalid_setup); }\n            memset(lengths + current_entry, current_length, n);\n            current_entry += n;\n            ++current_length;\n         }\n      } else {\n         for (j=0; j < c->entries; ++j) {\n            int present = c->sparse ? get_bits(f,1) : 1;\n            if (present) {\n               lengths[j] = get_bits(f, 5) + 1;\n               ++total;\n               if (lengths[j] == 32)\n                  return error(f, VORBIS_invalid_setup);\n            } else {\n               lengths[j] = NO_CODE;\n            }\n         }\n      }\n\n      if (c->sparse && total >= c->entries >> 2) {\n         // convert sparse items to non-sparse!\n         if (c->entries > (int) f->setup_temp_memory_required)\n            f->setup_temp_memory_required = c->entries;\n\n         c->codeword_lengths = (uint8 *) setup_malloc(f, c->entries);\n         if (c->codeword_lengths == NULL) return error(f, VORBIS_outofmem);\n         memcpy(c->codeword_lengths, lengths, c->entries);\n         setup_temp_free(f, lengths, c->entries); // note this is only safe if there have been no intervening temp mallocs!\n         lengths = c->codeword_lengths;\n         c->sparse = 0;\n      }\n\n      // compute the size of the sorted tables\n      if (c->sparse) {\n         sorted_count = total;\n      } else {\n         sorted_count = 0;\n         #ifndef STB_VORBIS_NO_HUFFMAN_BINARY_SEARCH\n         for (j=0; j < c->entries; ++j)\n            if (lengths[j] > STB_VORBIS_FAST_HUFFMAN_LENGTH && lengths[j] != NO_CODE)\n               ++sorted_count;\n         #endif\n      }\n\n      c->sorted_entries = sorted_count;\n      values = NULL;\n\n      CHECK(f);\n      if (!c->sparse) {\n         c->codewords = (uint32 *) setup_malloc(f, sizeof(c->codewords[0]) * c->entries);\n         if (!c->codewords)                  return error(f, VORBIS_outofmem);\n      } else {\n         unsigned int size;\n         if (c->sorted_entries) {\n            c->codeword_lengths = (uint8 *) setup_malloc(f, c->sorted_entries);\n            if (!c->codeword_lengths)           return error(f, VORBIS_outofmem);\n            c->codewords = (uint32 *) setup_temp_malloc(f, sizeof(*c->codewords) * c->sorted_entries);\n            if (!c->codewords)                  return error(f, VORBIS_outofmem);\n            values = (uint32 *) setup_temp_malloc(f, sizeof(*values) * c->sorted_entries);\n            if (!values)                        return error(f, VORBIS_outofmem);\n         }\n         size = c->entries + (sizeof(*c->codewords) + sizeof(*values)) * c->sorted_entries;\n         if (size > f->setup_temp_memory_required)\n            f->setup_temp_memory_required = size;\n      }\n\n      if (!compute_codewords(c, lengths, c->entries, values)) {\n         if (c->sparse) setup_temp_free(f, values, 0);\n         return error(f, VORBIS_invalid_setup);\n      }\n\n      if (c->sorted_entries) {\n         // allocate an extra slot for sentinels\n         c->sorted_codewords = (uint32 *) setup_malloc(f, sizeof(*c->sorted_codewords) * (c->sorted_entries+1));\n         if (c->sorted_codewords == NULL) return error(f, VORBIS_outofmem);\n         // allocate an extra slot at the front so that c->sorted_values[-1] is defined\n         // so that we can catch that case without an extra if\n         c->sorted_values    = ( int   *) setup_malloc(f, sizeof(*c->sorted_values   ) * (c->sorted_entries+1));\n         if (c->sorted_values == NULL) return error(f, VORBIS_outofmem);\n         ++c->sorted_values;\n         c->sorted_values[-1] = -1;\n         compute_sorted_huffman(c, lengths, values);\n      }\n\n      if (c->sparse) {\n         setup_temp_free(f, values, sizeof(*values)*c->sorted_entries);\n         setup_temp_free(f, c->codewords, sizeof(*c->codewords)*c->sorted_entries);\n         setup_temp_free(f, lengths, c->entries);\n         c->codewords = NULL;\n      }\n\n      compute_accelerated_huffman(c);\n\n      CHECK(f);\n      c->lookup_type = get_bits(f, 4);\n      if (c->lookup_type > 2) return error(f, VORBIS_invalid_setup);\n      if (c->lookup_type > 0) {\n         uint16 *mults;\n         c->minimum_value = float32_unpack(get_bits(f, 32));\n         c->delta_value = float32_unpack(get_bits(f, 32));\n         c->value_bits = get_bits(f, 4)+1;\n         c->sequence_p = get_bits(f,1);\n         if (c->lookup_type == 1) {\n            int values = lookup1_values(c->entries, c->dimensions);\n            if (values < 0) return error(f, VORBIS_invalid_setup);\n            c->lookup_values = (uint32) values;\n         } else {\n            c->lookup_values = c->entries * c->dimensions;\n         }\n         if (c->lookup_values == 0) return error(f, VORBIS_invalid_setup);\n         mults = (uint16 *) setup_temp_malloc(f, sizeof(mults[0]) * c->lookup_values);\n         if (mults == NULL) return error(f, VORBIS_outofmem);\n         for (j=0; j < (int) c->lookup_values; ++j) {\n            int q = get_bits(f, c->value_bits);\n            if (q == EOP) { setup_temp_free(f,mults,sizeof(mults[0])*c->lookup_values); return error(f, VORBIS_invalid_setup); }\n            mults[j] = q;\n         }\n\n#ifndef STB_VORBIS_DIVIDES_IN_CODEBOOK\n         if (c->lookup_type == 1) {\n            int len, sparse = c->sparse;\n            float last=0;\n            // pre-expand the lookup1-style multiplicands, to avoid a divide in the inner loop\n            if (sparse) {\n               if (c->sorted_entries == 0) goto skip;\n               c->multiplicands = (codetype *) setup_malloc(f, sizeof(c->multiplicands[0]) * c->sorted_entries * c->dimensions);\n            } else\n               c->multiplicands = (codetype *) setup_malloc(f, sizeof(c->multiplicands[0]) * c->entries        * c->dimensions);\n            if (c->multiplicands == NULL) { setup_temp_free(f,mults,sizeof(mults[0])*c->lookup_values); return error(f, VORBIS_outofmem); }\n            len = sparse ? c->sorted_entries : c->entries;\n            for (j=0; j < len; ++j) {\n               unsigned int z = sparse ? c->sorted_values[j] : j;\n               unsigned int div=1;\n               for (k=0; k < c->dimensions; ++k) {\n                  int off = (z / div) % c->lookup_values;\n                  float val = mults[off]*c->delta_value + c->minimum_value + last;\n                  c->multiplicands[j*c->dimensions + k] = val;\n                  if (c->sequence_p)\n                     last = val;\n                  if (k+1 < c->dimensions) {\n                     if (div > UINT_MAX / (unsigned int) c->lookup_values) {\n                        setup_temp_free(f, mults,sizeof(mults[0])*c->lookup_values);\n                        return error(f, VORBIS_invalid_setup);\n                     }\n                     div *= c->lookup_values;\n                  }\n               }\n            }\n            c->lookup_type = 2;\n         }\n         else\n#endif\n         {\n            float last=0;\n            CHECK(f);\n            c->multiplicands = (codetype *) setup_malloc(f, sizeof(c->multiplicands[0]) * c->lookup_values);\n            if (c->multiplicands == NULL) { setup_temp_free(f, mults,sizeof(mults[0])*c->lookup_values); return error(f, VORBIS_outofmem); }\n            for (j=0; j < (int) c->lookup_values; ++j) {\n               float val = mults[j] * c->delta_value + c->minimum_value + last;\n               c->multiplicands[j] = val;\n               if (c->sequence_p)\n                  last = val;\n            }\n         }\n#ifndef STB_VORBIS_DIVIDES_IN_CODEBOOK\n        skip:;\n#endif\n         setup_temp_free(f, mults, sizeof(mults[0])*c->lookup_values);\n\n         CHECK(f);\n      }\n      CHECK(f);\n   }\n\n   // time domain transfers (notused)\n\n   x = get_bits(f, 6) + 1;\n   for (i=0; i < x; ++i) {\n      uint32 z = get_bits(f, 16);\n      if (z != 0) return error(f, VORBIS_invalid_setup);\n   }\n\n   // Floors\n   f->floor_count = get_bits(f, 6)+1;\n   f->floor_config = (Floor *)  setup_malloc(f, f->floor_count * sizeof(*f->floor_config));\n   if (f->floor_config == NULL) return error(f, VORBIS_outofmem);\n   for (i=0; i < f->floor_count; ++i) {\n      f->floor_types[i] = get_bits(f, 16);\n      if (f->floor_types[i] > 1) return error(f, VORBIS_invalid_setup);\n      if (f->floor_types[i] == 0) {\n         Floor0 *g = &f->floor_config[i].floor0;\n         g->order = get_bits(f,8);\n         g->rate = get_bits(f,16);\n         g->bark_map_size = get_bits(f,16);\n         g->amplitude_bits = get_bits(f,6);\n         g->amplitude_offset = get_bits(f,8);\n         g->number_of_books = get_bits(f,4) + 1;\n         for (j=0; j < g->number_of_books; ++j)\n            g->book_list[j] = get_bits(f,8);\n         return error(f, VORBIS_feature_not_supported);\n      } else {\n         stbv__floor_ordering p[31*8+2];\n         Floor1 *g = &f->floor_config[i].floor1;\n         int max_class = -1;\n         g->partitions = get_bits(f, 5);\n         for (j=0; j < g->partitions; ++j) {\n            g->partition_class_list[j] = get_bits(f, 4);\n            if (g->partition_class_list[j] > max_class)\n               max_class = g->partition_class_list[j];\n         }\n         for (j=0; j <= max_class; ++j) {\n            g->class_dimensions[j] = get_bits(f, 3)+1;\n            g->class_subclasses[j] = get_bits(f, 2);\n            if (g->class_subclasses[j]) {\n               g->class_masterbooks[j] = get_bits(f, 8);\n               if (g->class_masterbooks[j] >= f->codebook_count) return error(f, VORBIS_invalid_setup);\n            }\n            for (k=0; k < 1 << g->class_subclasses[j]; ++k) {\n               g->subclass_books[j][k] = (int16)get_bits(f,8)-1;\n               if (g->subclass_books[j][k] >= f->codebook_count) return error(f, VORBIS_invalid_setup);\n            }\n         }\n         g->floor1_multiplier = get_bits(f,2)+1;\n         g->rangebits = get_bits(f,4);\n         g->Xlist[0] = 0;\n         g->Xlist[1] = 1 << g->rangebits;\n         g->values = 2;\n         for (j=0; j < g->partitions; ++j) {\n            int c = g->partition_class_list[j];\n            for (k=0; k < g->class_dimensions[c]; ++k) {\n               g->Xlist[g->values] = get_bits(f, g->rangebits);\n               ++g->values;\n            }\n         }\n         // precompute the sorting\n         for (j=0; j < g->values; ++j) {\n            p[j].x = g->Xlist[j];\n            p[j].id = j;\n         }\n         qsort(p, g->values, sizeof(p[0]), point_compare);\n         for (j=0; j < g->values-1; ++j)\n            if (p[j].x == p[j+1].x)\n               return error(f, VORBIS_invalid_setup);\n         for (j=0; j < g->values; ++j)\n            g->sorted_order[j] = (uint8) p[j].id;\n         // precompute the neighbors\n         for (j=2; j < g->values; ++j) {\n            int low = 0,hi = 0;\n            neighbors(g->Xlist, j, &low,&hi);\n            g->neighbors[j][0] = low;\n            g->neighbors[j][1] = hi;\n         }\n\n         if (g->values > longest_floorlist)\n            longest_floorlist = g->values;\n      }\n   }\n\n   // Residue\n   f->residue_count = get_bits(f, 6)+1;\n   f->residue_config = (Residue *) setup_malloc(f, f->residue_count * sizeof(f->residue_config[0]));\n   if (f->residue_config == NULL) return error(f, VORBIS_outofmem);\n   memset(f->residue_config, 0, f->residue_count * sizeof(f->residue_config[0]));\n   for (i=0; i < f->residue_count; ++i) {\n      uint8 residue_cascade[64];\n      Residue *r = f->residue_config+i;\n      f->residue_types[i] = get_bits(f, 16);\n      if (f->residue_types[i] > 2) return error(f, VORBIS_invalid_setup);\n      r->begin = get_bits(f, 24);\n      r->end = get_bits(f, 24);\n      if (r->end < r->begin) return error(f, VORBIS_invalid_setup);\n      r->part_size = get_bits(f,24)+1;\n      r->classifications = get_bits(f,6)+1;\n      r->classbook = get_bits(f,8);\n      if (r->classbook >= f->codebook_count) return error(f, VORBIS_invalid_setup);\n      for (j=0; j < r->classifications; ++j) {\n         uint8 high_bits=0;\n         uint8 low_bits=get_bits(f,3);\n         if (get_bits(f,1))\n            high_bits = get_bits(f,5);\n         residue_cascade[j] = high_bits*8 + low_bits;\n      }\n      r->residue_books = (short (*)[8]) setup_malloc(f, sizeof(r->residue_books[0]) * r->classifications);\n      if (r->residue_books == NULL) return error(f, VORBIS_outofmem);\n      for (j=0; j < r->classifications; ++j) {\n         for (k=0; k < 8; ++k) {\n            if (residue_cascade[j] & (1 << k)) {\n               r->residue_books[j][k] = get_bits(f, 8);\n               if (r->residue_books[j][k] >= f->codebook_count) return error(f, VORBIS_invalid_setup);\n            } else {\n               r->residue_books[j][k] = -1;\n            }\n         }\n      }\n      // precompute the classifications[] array to avoid inner-loop mod/divide\n      // call it 'classdata' since we already have r->classifications\n      r->classdata = (uint8 **) setup_malloc(f, sizeof(*r->classdata) * f->codebooks[r->classbook].entries);\n      if (!r->classdata) return error(f, VORBIS_outofmem);\n      memset(r->classdata, 0, sizeof(*r->classdata) * f->codebooks[r->classbook].entries);\n      for (j=0; j < f->codebooks[r->classbook].entries; ++j) {\n         int classwords = f->codebooks[r->classbook].dimensions;\n         int temp = j;\n         r->classdata[j] = (uint8 *) setup_malloc(f, sizeof(r->classdata[j][0]) * classwords);\n         if (r->classdata[j] == NULL) return error(f, VORBIS_outofmem);\n         for (k=classwords-1; k >= 0; --k) {\n            r->classdata[j][k] = temp % r->classifications;\n            temp /= r->classifications;\n         }\n      }\n   }\n\n   f->mapping_count = get_bits(f,6)+1;\n   f->mapping = (Mapping *) setup_malloc(f, f->mapping_count * sizeof(*f->mapping));\n   if (f->mapping == NULL) return error(f, VORBIS_outofmem);\n   memset(f->mapping, 0, f->mapping_count * sizeof(*f->mapping));\n   for (i=0; i < f->mapping_count; ++i) {\n      Mapping *m = f->mapping + i;\n      int mapping_type = get_bits(f,16);\n      if (mapping_type != 0) return error(f, VORBIS_invalid_setup);\n      m->chan = (MappingChannel *) setup_malloc(f, f->channels * sizeof(*m->chan));\n      if (m->chan == NULL) return error(f, VORBIS_outofmem);\n      if (get_bits(f,1))\n         m->submaps = get_bits(f,4)+1;\n      else\n         m->submaps = 1;\n      if (m->submaps > max_submaps)\n         max_submaps = m->submaps;\n      if (get_bits(f,1)) {\n         m->coupling_steps = get_bits(f,8)+1;\n         if (m->coupling_steps > f->channels) return error(f, VORBIS_invalid_setup);\n         for (k=0; k < m->coupling_steps; ++k) {\n            m->chan[k].magnitude = get_bits(f, ilog(f->channels-1));\n            m->chan[k].angle = get_bits(f, ilog(f->channels-1));\n            if (m->chan[k].magnitude >= f->channels)        return error(f, VORBIS_invalid_setup);\n            if (m->chan[k].angle     >= f->channels)        return error(f, VORBIS_invalid_setup);\n            if (m->chan[k].magnitude == m->chan[k].angle)   return error(f, VORBIS_invalid_setup);\n         }\n      } else\n         m->coupling_steps = 0;\n\n      // reserved field\n      if (get_bits(f,2)) return error(f, VORBIS_invalid_setup);\n      if (m->submaps > 1) {\n         for (j=0; j < f->channels; ++j) {\n            m->chan[j].mux = get_bits(f, 4);\n            if (m->chan[j].mux >= m->submaps)                return error(f, VORBIS_invalid_setup);\n         }\n      } else\n         // @SPECIFICATION: this case is missing from the spec\n         for (j=0; j < f->channels; ++j)\n            m->chan[j].mux = 0;\n\n      for (j=0; j < m->submaps; ++j) {\n         get_bits(f,8); // discard\n         m->submap_floor[j] = get_bits(f,8);\n         m->submap_residue[j] = get_bits(f,8);\n         if (m->submap_floor[j] >= f->floor_count)      return error(f, VORBIS_invalid_setup);\n         if (m->submap_residue[j] >= f->residue_count)  return error(f, VORBIS_invalid_setup);\n      }\n   }\n\n   // Modes\n   f->mode_count = get_bits(f, 6)+1;\n   for (i=0; i < f->mode_count; ++i) {\n      Mode *m = f->mode_config+i;\n      m->blockflag = get_bits(f,1);\n      m->windowtype = get_bits(f,16);\n      m->transformtype = get_bits(f,16);\n      m->mapping = get_bits(f,8);\n      if (m->windowtype != 0)                 return error(f, VORBIS_invalid_setup);\n      if (m->transformtype != 0)              return error(f, VORBIS_invalid_setup);\n      if (m->mapping >= f->mapping_count)     return error(f, VORBIS_invalid_setup);\n   }\n\n   flush_packet(f);\n\n   f->previous_length = 0;\n\n   for (i=0; i < f->channels; ++i) {\n      f->channel_buffers[i] = (float *) setup_malloc(f, sizeof(float) * f->blocksize_1);\n      f->previous_window[i] = (float *) setup_malloc(f, sizeof(float) * f->blocksize_1/2);\n      f->finalY[i]          = (int16 *) setup_malloc(f, sizeof(int16) * longest_floorlist);\n      if (f->channel_buffers[i] == NULL || f->previous_window[i] == NULL || f->finalY[i] == NULL) return error(f, VORBIS_outofmem);\n      memset(f->channel_buffers[i], 0, sizeof(float) * f->blocksize_1);\n      #ifdef STB_VORBIS_NO_DEFER_FLOOR\n      f->floor_buffers[i]   = (float *) setup_malloc(f, sizeof(float) * f->blocksize_1/2);\n      if (f->floor_buffers[i] == NULL) return error(f, VORBIS_outofmem);\n      #endif\n   }\n\n   if (!init_blocksize(f, 0, f->blocksize_0)) return FALSE;\n   if (!init_blocksize(f, 1, f->blocksize_1)) return FALSE;\n   f->blocksize[0] = f->blocksize_0;\n   f->blocksize[1] = f->blocksize_1;\n\n#ifdef STB_VORBIS_DIVIDE_TABLE\n   if (integer_divide_table[1][1]==0)\n      for (i=0; i < DIVTAB_NUMER; ++i)\n         for (j=1; j < DIVTAB_DENOM; ++j)\n            integer_divide_table[i][j] = i / j;\n#endif\n\n   // compute how much temporary memory is needed\n\n   // 1.\n   {\n      uint32 imdct_mem = (f->blocksize_1 * sizeof(float) >> 1);\n      uint32 classify_mem;\n      int i,max_part_read=0;\n      for (i=0; i < f->residue_count; ++i) {\n         Residue *r = f->residue_config + i;\n         unsigned int actual_size = f->blocksize_1 / 2;\n         unsigned int limit_r_begin = r->begin < actual_size ? r->begin : actual_size;\n         unsigned int limit_r_end   = r->end   < actual_size ? r->end   : actual_size;\n         int n_read = limit_r_end - limit_r_begin;\n         int part_read = n_read / r->part_size;\n         if (part_read > max_part_read)\n            max_part_read = part_read;\n      }\n      #ifndef STB_VORBIS_DIVIDES_IN_RESIDUE\n      classify_mem = f->channels * (sizeof(void*) + max_part_read * sizeof(uint8 *));\n      #else\n      classify_mem = f->channels * (sizeof(void*) + max_part_read * sizeof(int *));\n      #endif\n\n      // maximum reasonable partition size is f->blocksize_1\n\n      f->temp_memory_required = classify_mem;\n      if (imdct_mem > f->temp_memory_required)\n         f->temp_memory_required = imdct_mem;\n   }\n\n\n   if (f->alloc.alloc_buffer) {\n      assert(f->temp_offset == f->alloc.alloc_buffer_length_in_bytes);\n      // check if there's enough temp memory so we don't error later\n      if (f->setup_offset + sizeof(*f) + f->temp_memory_required > (unsigned) f->temp_offset)\n         return error(f, VORBIS_outofmem);\n   }\n\n   // @TODO: stb_vorbis_seek_start expects first_audio_page_offset to point to a page\n   // without PAGEFLAG_continued_packet, so this either points to the first page, or\n   // the page after the end of the headers. It might be cleaner to point to a page\n   // in the middle of the headers, when that's the page where the first audio packet\n   // starts, but we'd have to also correctly skip the end of any continued packet in\n   // stb_vorbis_seek_start.\n   if (f->next_seg == -1) {\n      f->first_audio_page_offset = stb_vorbis_get_file_offset(f);\n   } else {\n      f->first_audio_page_offset = 0;\n   }\n\n   return TRUE;\n}\n\nstatic void vorbis_deinit(stb_vorbis *p)\n{\n   int i,j;\n\n   setup_free(p, p->vendor);\n   for (i=0; i < p->comment_list_length; ++i) {\n      setup_free(p, p->comment_list[i]);\n   }\n   setup_free(p, p->comment_list);\n\n   if (p->residue_config) {\n      for (i=0; i < p->residue_count; ++i) {\n         Residue *r = p->residue_config+i;\n         if (r->classdata) {\n            for (j=0; j < p->codebooks[r->classbook].entries; ++j)\n               setup_free(p, r->classdata[j]);\n            setup_free(p, r->classdata);\n         }\n         setup_free(p, r->residue_books);\n      }\n   }\n\n   if (p->codebooks) {\n      CHECK(p);\n      for (i=0; i < p->codebook_count; ++i) {\n         Codebook *c = p->codebooks + i;\n         setup_free(p, c->codeword_lengths);\n         setup_free(p, c->multiplicands);\n         setup_free(p, c->codewords);\n         setup_free(p, c->sorted_codewords);\n         // c->sorted_values[-1] is the first entry in the array\n         setup_free(p, c->sorted_values ? c->sorted_values-1 : NULL);\n      }\n      setup_free(p, p->codebooks);\n   }\n   setup_free(p, p->floor_config);\n   setup_free(p, p->residue_config);\n   if (p->mapping) {\n      for (i=0; i < p->mapping_count; ++i)\n         setup_free(p, p->mapping[i].chan);\n      setup_free(p, p->mapping);\n   }\n   CHECK(p);\n   for (i=0; i < p->channels && i < STB_VORBIS_MAX_CHANNELS; ++i) {\n      setup_free(p, p->channel_buffers[i]);\n      setup_free(p, p->previous_window[i]);\n      #ifdef STB_VORBIS_NO_DEFER_FLOOR\n      setup_free(p, p->floor_buffers[i]);\n      #endif\n      setup_free(p, p->finalY[i]);\n   }\n   for (i=0; i < 2; ++i) {\n      setup_free(p, p->A[i]);\n      setup_free(p, p->B[i]);\n      setup_free(p, p->C[i]);\n      setup_free(p, p->window[i]);\n      setup_free(p, p->bit_reverse[i]);\n   }\n   #ifndef STB_VORBIS_NO_STDIO\n   if (p->close_on_free) fclose(p->f);\n   #endif\n}\n\nvoid stb_vorbis_close(stb_vorbis *p)\n{\n   if (p == NULL) return;\n   vorbis_deinit(p);\n   setup_free(p,p);\n}\n\nstatic void vorbis_init(stb_vorbis *p, const stb_vorbis_alloc *z)\n{\n   memset(p, 0, sizeof(*p)); // NULL out all malloc'd pointers to start\n   if (z) {\n      p->alloc = *z;\n      p->alloc.alloc_buffer_length_in_bytes &= ~7;\n      p->temp_offset = p->alloc.alloc_buffer_length_in_bytes;\n   }\n   p->eof = 0;\n   p->error = VORBIS__no_error;\n   p->stream = NULL;\n   p->codebooks = NULL;\n   p->page_crc_tests = -1;\n   #ifndef STB_VORBIS_NO_STDIO\n   p->close_on_free = FALSE;\n   p->f = NULL;\n   #endif\n}\n\nint stb_vorbis_get_sample_offset(stb_vorbis *f)\n{\n   if (f->current_loc_valid)\n      return f->current_loc;\n   else\n      return -1;\n}\n\nstb_vorbis_info stb_vorbis_get_info(stb_vorbis *f)\n{\n   stb_vorbis_info d;\n   d.channels = f->channels;\n   d.sample_rate = f->sample_rate;\n   d.setup_memory_required = f->setup_memory_required;\n   d.setup_temp_memory_required = f->setup_temp_memory_required;\n   d.temp_memory_required = f->temp_memory_required;\n   d.max_frame_size = f->blocksize_1 >> 1;\n   return d;\n}\n\nstb_vorbis_comment stb_vorbis_get_comment(stb_vorbis *f)\n{\n   stb_vorbis_comment d;\n   d.vendor = f->vendor;\n   d.comment_list_length = f->comment_list_length;\n   d.comment_list = f->comment_list;\n   return d;\n}\n\nint stb_vorbis_get_error(stb_vorbis *f)\n{\n   int e = f->error;\n   f->error = VORBIS__no_error;\n   return e;\n}\n\nstatic stb_vorbis * vorbis_alloc(stb_vorbis *f)\n{\n   stb_vorbis *p = (stb_vorbis *) setup_malloc(f, sizeof(*p));\n   return p;\n}\n\n#ifndef STB_VORBIS_NO_PUSHDATA_API\n\nvoid stb_vorbis_flush_pushdata(stb_vorbis *f)\n{\n   f->previous_length = 0;\n   f->page_crc_tests  = 0;\n   f->discard_samples_deferred = 0;\n   f->current_loc_valid = FALSE;\n   f->first_decode = FALSE;\n   f->samples_output = 0;\n   f->channel_buffer_start = 0;\n   f->channel_buffer_end = 0;\n}\n\nstatic int vorbis_search_for_page_pushdata(vorb *f, uint8 *data, int data_len)\n{\n   int i,n;\n   for (i=0; i < f->page_crc_tests; ++i)\n      f->scan[i].bytes_done = 0;\n\n   // if we have room for more scans, search for them first, because\n   // they may cause us to stop early if their header is incomplete\n   if (f->page_crc_tests < STB_VORBIS_PUSHDATA_CRC_COUNT) {\n      if (data_len < 4) return 0;\n      data_len -= 3; // need to look for 4-byte sequence, so don't miss\n                     // one that straddles a boundary\n      for (i=0; i < data_len; ++i) {\n         if (data[i] == 0x4f) {\n            if (0==memcmp(data+i, ogg_page_header, 4)) {\n               int j,len;\n               uint32 crc;\n               // make sure we have the whole page header\n               if (i+26 >= data_len || i+27+data[i+26] >= data_len) {\n                  // only read up to this page start, so hopefully we'll\n                  // have the whole page header start next time\n                  data_len = i;\n                  break;\n               }\n               // ok, we have it all; compute the length of the page\n               len = 27 + data[i+26];\n               for (j=0; j < data[i+26]; ++j)\n                  len += data[i+27+j];\n               // scan everything up to the embedded crc (which we must 0)\n               crc = 0;\n               for (j=0; j < 22; ++j)\n                  crc = crc32_update(crc, data[i+j]);\n               // now process 4 0-bytes\n               for (   ; j < 26; ++j)\n                  crc = crc32_update(crc, 0);\n               // len is the total number of bytes we need to scan\n               n = f->page_crc_tests++;\n               f->scan[n].bytes_left = len-j;\n               f->scan[n].crc_so_far = crc;\n               f->scan[n].goal_crc = data[i+22] + (data[i+23] << 8) + (data[i+24]<<16) + (data[i+25]<<24);\n               // if the last frame on a page is continued to the next, then\n               // we can't recover the sample_loc immediately\n               if (data[i+27+data[i+26]-1] == 255)\n                  f->scan[n].sample_loc = ~0;\n               else\n                  f->scan[n].sample_loc = data[i+6] + (data[i+7] << 8) + (data[i+ 8]<<16) + (data[i+ 9]<<24);\n               f->scan[n].bytes_done = i+j;\n               if (f->page_crc_tests == STB_VORBIS_PUSHDATA_CRC_COUNT)\n                  break;\n               // keep going if we still have room for more\n            }\n         }\n      }\n   }\n\n   for (i=0; i < f->page_crc_tests;) {\n      uint32 crc;\n      int j;\n      int n = f->scan[i].bytes_done;\n      int m = f->scan[i].bytes_left;\n      if (m > data_len - n) m = data_len - n;\n      // m is the bytes to scan in the current chunk\n      crc = f->scan[i].crc_so_far;\n      for (j=0; j < m; ++j)\n         crc = crc32_update(crc, data[n+j]);\n      f->scan[i].bytes_left -= m;\n      f->scan[i].crc_so_far = crc;\n      if (f->scan[i].bytes_left == 0) {\n         // does it match?\n         if (f->scan[i].crc_so_far == f->scan[i].goal_crc) {\n            // Houston, we have page\n            data_len = n+m; // consumption amount is wherever that scan ended\n            f->page_crc_tests = -1; // drop out of page scan mode\n            f->previous_length = 0; // decode-but-don't-output one frame\n            f->next_seg = -1;       // start a new page\n            f->current_loc = f->scan[i].sample_loc; // set the current sample location\n                                    // to the amount we'd have decoded had we decoded this page\n            f->current_loc_valid = f->current_loc != ~0U;\n            return data_len;\n         }\n         // delete entry\n         f->scan[i] = f->scan[--f->page_crc_tests];\n      } else {\n         ++i;\n      }\n   }\n\n   return data_len;\n}\n\n// return value: number of bytes we used\nint stb_vorbis_decode_frame_pushdata(\n         stb_vorbis *f,                   // the file we're decoding\n         const uint8 *data, int data_len, // the memory available for decoding\n         int *channels,                   // place to write number of float * buffers\n         float ***output,                 // place to write float ** array of float * buffers\n         int *samples                     // place to write number of output samples\n     )\n{\n   int i;\n   int len,right,left;\n\n   if (!IS_PUSH_MODE(f)) return error(f, VORBIS_invalid_api_mixing);\n\n   if (f->page_crc_tests >= 0) {\n      *samples = 0;\n      return vorbis_search_for_page_pushdata(f, (uint8 *) data, data_len);\n   }\n\n   f->stream     = (uint8 *) data;\n   f->stream_end = (uint8 *) data + data_len;\n   f->error      = VORBIS__no_error;\n\n   // check that we have the entire packet in memory\n   if (!is_whole_packet_present(f)) {\n      *samples = 0;\n      return 0;\n   }\n\n   if (!vorbis_decode_packet(f, &len, &left, &right)) {\n      // save the actual error we encountered\n      enum STBVorbisError error = f->error;\n      if (error == VORBIS_bad_packet_type) {\n         // flush and resynch\n         f->error = VORBIS__no_error;\n         while (get8_packet(f) != EOP)\n            if (f->eof) break;\n         *samples = 0;\n         return (int) (f->stream - data);\n      }\n      if (error == VORBIS_continued_packet_flag_invalid) {\n         if (f->previous_length == 0) {\n            // we may be resynching, in which case it's ok to hit one\n            // of these; just discard the packet\n            f->error = VORBIS__no_error;\n            while (get8_packet(f) != EOP)\n               if (f->eof) break;\n            *samples = 0;\n            return (int) (f->stream - data);\n         }\n      }\n      // if we get an error while parsing, what to do?\n      // well, it DEFINITELY won't work to continue from where we are!\n      stb_vorbis_flush_pushdata(f);\n      // restore the error that actually made us bail\n      f->error = error;\n      *samples = 0;\n      return 1;\n   }\n\n   // success!\n   len = vorbis_finish_frame(f, len, left, right);\n   for (i=0; i < f->channels; ++i)\n      f->outputs[i] = f->channel_buffers[i] + left;\n\n   if (channels) *channels = f->channels;\n   *samples = len;\n   *output = f->outputs;\n   return (int) (f->stream - data);\n}\n\nstb_vorbis *stb_vorbis_open_pushdata(\n         const unsigned char *data, int data_len, // the memory available for decoding\n         int *data_used,              // only defined if result is not NULL\n         int *error, const stb_vorbis_alloc *alloc)\n{\n   stb_vorbis *f, p;\n   vorbis_init(&p, alloc);\n   p.stream     = (uint8 *) data;\n   p.stream_end = (uint8 *) data + data_len;\n   p.push_mode  = TRUE;\n   if (!start_decoder(&p)) {\n      if (p.eof)\n         *error = VORBIS_need_more_data;\n      else\n         *error = p.error;\n      vorbis_deinit(&p);\n      return NULL;\n   }\n   f = vorbis_alloc(&p);\n   if (f) {\n      *f = p;\n      *data_used = (int) (f->stream - data);\n      *error = 0;\n      return f;\n   } else {\n      vorbis_deinit(&p);\n      return NULL;\n   }\n}\n#endif // STB_VORBIS_NO_PUSHDATA_API\n\nunsigned int stb_vorbis_get_file_offset(stb_vorbis *f)\n{\n   #ifndef STB_VORBIS_NO_PUSHDATA_API\n   if (f->push_mode) return 0;\n   #endif\n   if (USE_MEMORY(f)) return (unsigned int) (f->stream - f->stream_start);\n   #ifndef STB_VORBIS_NO_STDIO\n   return (unsigned int) (ftell(f->f) - f->f_start);\n   #endif\n}\n\n#ifndef STB_VORBIS_NO_PULLDATA_API\n//\n// DATA-PULLING API\n//\n\nstatic uint32 vorbis_find_page(stb_vorbis *f, uint32 *end, uint32 *last)\n{\n   for(;;) {\n      int n;\n      if (f->eof) return 0;\n      n = get8(f);\n      if (n == 0x4f) { // page header candidate\n         unsigned int retry_loc = stb_vorbis_get_file_offset(f);\n         int i;\n         // check if we're off the end of a file_section stream\n         if (retry_loc - 25 > f->stream_len)\n            return 0;\n         // check the rest of the header\n         for (i=1; i < 4; ++i)\n            if (get8(f) != ogg_page_header[i])\n               break;\n         if (f->eof) return 0;\n         if (i == 4) {\n            uint8 header[27];\n            uint32 i, crc, goal, len;\n            for (i=0; i < 4; ++i)\n               header[i] = ogg_page_header[i];\n            for (; i < 27; ++i)\n               header[i] = get8(f);\n            if (f->eof) return 0;\n            if (header[4] != 0) goto invalid;\n            goal = header[22] + (header[23] << 8) + (header[24]<<16) + ((uint32)header[25]<<24);\n            for (i=22; i < 26; ++i)\n               header[i] = 0;\n            crc = 0;\n            for (i=0; i < 27; ++i)\n               crc = crc32_update(crc, header[i]);\n            len = 0;\n            for (i=0; i < header[26]; ++i) {\n               int s = get8(f);\n               crc = crc32_update(crc, s);\n               len += s;\n            }\n            if (len && f->eof) return 0;\n            for (i=0; i < len; ++i)\n               crc = crc32_update(crc, get8(f));\n            // finished parsing probable page\n            if (crc == goal) {\n               // we could now check that it's either got the last\n               // page flag set, OR it's followed by the capture\n               // pattern, but I guess TECHNICALLY you could have\n               // a file with garbage between each ogg page and recover\n               // from it automatically? So even though that paranoia\n               // might decrease the chance of an invalid decode by\n               // another 2^32, not worth it since it would hose those\n               // invalid-but-useful files?\n               if (end)\n                  *end = stb_vorbis_get_file_offset(f);\n               if (last) {\n                  if (header[5] & 0x04)\n                     *last = 1;\n                  else\n                     *last = 0;\n               }\n               set_file_offset(f, retry_loc-1);\n               return 1;\n            }\n         }\n        invalid:\n         // not a valid page, so rewind and look for next one\n         set_file_offset(f, retry_loc);\n      }\n   }\n}\n\n\n#define SAMPLE_unknown  0xffffffff\n\n// seeking is implemented with a binary search, which narrows down the range to\n// 64K, before using a linear search (because finding the synchronization\n// pattern can be expensive, and the chance we'd find the end page again is\n// relatively high for small ranges)\n//\n// two initial interpolation-style probes are used at the start of the search\n// to try to bound either side of the binary search sensibly, while still\n// working in O(log n) time if they fail.\n\nstatic int get_seek_page_info(stb_vorbis *f, ProbedPage *z)\n{\n   uint8 header[27], lacing[255];\n   int i,len;\n\n   // record where the page starts\n   z->page_start = stb_vorbis_get_file_offset(f);\n\n   // parse the header\n   getn(f, header, 27);\n   if (header[0] != 'O' || header[1] != 'g' || header[2] != 'g' || header[3] != 'S')\n      return 0;\n   getn(f, lacing, header[26]);\n\n   // determine the length of the payload\n   len = 0;\n   for (i=0; i < header[26]; ++i)\n      len += lacing[i];\n\n   // this implies where the page ends\n   z->page_end = z->page_start + 27 + header[26] + len;\n\n   // read the last-decoded sample out of the data\n   z->last_decoded_sample = header[6] + (header[7] << 8) + (header[8] << 16) + (header[9] << 24);\n\n   // restore file state to where we were\n   set_file_offset(f, z->page_start);\n   return 1;\n}\n\n// rarely used function to seek back to the preceding page while finding the\n// start of a packet\nstatic int go_to_page_before(stb_vorbis *f, unsigned int limit_offset)\n{\n   unsigned int previous_safe, end;\n\n   // now we want to seek back 64K from the limit\n   if (limit_offset >= 65536 && limit_offset-65536 >= f->first_audio_page_offset)\n      previous_safe = limit_offset - 65536;\n   else\n      previous_safe = f->first_audio_page_offset;\n\n   set_file_offset(f, previous_safe);\n\n   while (vorbis_find_page(f, &end, NULL)) {\n      if (end >= limit_offset && stb_vorbis_get_file_offset(f) < limit_offset)\n         return 1;\n      set_file_offset(f, end);\n   }\n\n   return 0;\n}\n\n// implements the search logic for finding a page and starting decoding. if\n// the function succeeds, current_loc_valid will be true and current_loc will\n// be less than or equal to the provided sample number (the closer the\n// better).\nstatic int seek_to_sample_coarse(stb_vorbis *f, uint32 sample_number)\n{\n   ProbedPage left, right, mid;\n   int i, start_seg_with_known_loc, end_pos, page_start;\n   uint32 delta, stream_length, padding, last_sample_limit;\n   double offset = 0.0, bytes_per_sample = 0.0;\n   int probe = 0;\n\n   // find the last page and validate the target sample\n   stream_length = stb_vorbis_stream_length_in_samples(f);\n   if (stream_length == 0)            return error(f, VORBIS_seek_without_length);\n   if (sample_number > stream_length) return error(f, VORBIS_seek_invalid);\n\n   // this is the maximum difference between the window-center (which is the\n   // actual granule position value), and the right-start (which the spec\n   // indicates should be the granule position (give or take one)).\n   padding = ((f->blocksize_1 - f->blocksize_0) >> 2);\n   if (sample_number < padding)\n      last_sample_limit = 0;\n   else\n      last_sample_limit = sample_number - padding;\n\n   left = f->p_first;\n   while (left.last_decoded_sample == ~0U) {\n      // (untested) the first page does not have a 'last_decoded_sample'\n      set_file_offset(f, left.page_end);\n      if (!get_seek_page_info(f, &left)) goto error;\n   }\n\n   right = f->p_last;\n   assert(right.last_decoded_sample != ~0U);\n\n   // starting from the start is handled differently\n   if (last_sample_limit <= left.last_decoded_sample) {\n      if (stb_vorbis_seek_start(f)) {\n         if (f->current_loc > sample_number)\n            return error(f, VORBIS_seek_failed);\n         return 1;\n      }\n      return 0;\n   }\n\n   while (left.page_end != right.page_start) {\n      assert(left.page_end < right.page_start);\n      // search range in bytes\n      delta = right.page_start - left.page_end;\n      if (delta <= 65536) {\n         // there's only 64K left to search - handle it linearly\n         set_file_offset(f, left.page_end);\n      } else {\n         if (probe < 2) {\n            if (probe == 0) {\n               // first probe (interpolate)\n               double data_bytes = right.page_end - left.page_start;\n               bytes_per_sample = data_bytes / right.last_decoded_sample;\n               offset = left.page_start + bytes_per_sample * (last_sample_limit - left.last_decoded_sample);\n            } else {\n               // second probe (try to bound the other side)\n               double error = ((double) last_sample_limit - mid.last_decoded_sample) * bytes_per_sample;\n               if (error >= 0 && error <  8000) error =  8000;\n               if (error <  0 && error > -8000) error = -8000;\n               offset += error * 2;\n            }\n\n            // ensure the offset is valid\n            if (offset < left.page_end)\n               offset = left.page_end;\n            if (offset > right.page_start - 65536)\n               offset = right.page_start - 65536;\n\n            set_file_offset(f, (unsigned int) offset);\n         } else {\n            // binary search for large ranges (offset by 32K to ensure\n            // we don't hit the right page)\n            set_file_offset(f, left.page_end + (delta / 2) - 32768);\n         }\n\n         if (!vorbis_find_page(f, NULL, NULL)) goto error;\n      }\n\n      for (;;) {\n         if (!get_seek_page_info(f, &mid)) goto error;\n         if (mid.last_decoded_sample != ~0U) break;\n         // (untested) no frames end on this page\n         set_file_offset(f, mid.page_end);\n         assert(mid.page_start < right.page_start);\n      }\n\n      // if we've just found the last page again then we're in a tricky file,\n      // and we're close enough (if it wasn't an interpolation probe).\n      if (mid.page_start == right.page_start) {\n         if (probe >= 2 || delta <= 65536)\n            break;\n      } else {\n         if (last_sample_limit < mid.last_decoded_sample)\n            right = mid;\n         else\n            left = mid;\n      }\n\n      ++probe;\n   }\n\n   // seek back to start of the last packet\n   page_start = left.page_start;\n   set_file_offset(f, page_start);\n   if (!start_page(f)) return error(f, VORBIS_seek_failed);\n   end_pos = f->end_seg_with_known_loc;\n   assert(end_pos >= 0);\n\n   for (;;) {\n      for (i = end_pos; i > 0; --i)\n         if (f->segments[i-1] != 255)\n            break;\n\n      start_seg_with_known_loc = i;\n\n      if (start_seg_with_known_loc > 0 || !(f->page_flag & PAGEFLAG_continued_packet))\n         break;\n\n      // (untested) the final packet begins on an earlier page\n      if (!go_to_page_before(f, page_start))\n         goto error;\n\n      page_start = stb_vorbis_get_file_offset(f);\n      if (!start_page(f)) goto error;\n      end_pos = f->segment_count - 1;\n   }\n\n   // prepare to start decoding\n   f->current_loc_valid = FALSE;\n   f->last_seg = FALSE;\n   f->valid_bits = 0;\n   f->packet_bytes = 0;\n   f->bytes_in_seg = 0;\n   f->previous_length = 0;\n   f->next_seg = start_seg_with_known_loc;\n\n   for (i = 0; i < start_seg_with_known_loc; i++)\n      skip(f, f->segments[i]);\n\n   // start decoding (optimizable - this frame is generally discarded)\n   if (!vorbis_pump_first_frame(f))\n      return 0;\n   if (f->current_loc > sample_number)\n      return error(f, VORBIS_seek_failed);\n   return 1;\n\nerror:\n   // try to restore the file to a valid state\n   stb_vorbis_seek_start(f);\n   return error(f, VORBIS_seek_failed);\n}\n\n// the same as vorbis_decode_initial, but without advancing\nstatic int peek_decode_initial(vorb *f, int *p_left_start, int *p_left_end, int *p_right_start, int *p_right_end, int *mode)\n{\n   int bits_read, bytes_read;\n\n   if (!vorbis_decode_initial(f, p_left_start, p_left_end, p_right_start, p_right_end, mode))\n      return 0;\n\n   // either 1 or 2 bytes were read, figure out which so we can rewind\n   bits_read = 1 + ilog(f->mode_count-1);\n   if (f->mode_config[*mode].blockflag)\n      bits_read += 2;\n   bytes_read = (bits_read + 7) / 8;\n\n   f->bytes_in_seg += bytes_read;\n   f->packet_bytes -= bytes_read;\n   skip(f, -bytes_read);\n   if (f->next_seg == -1)\n      f->next_seg = f->segment_count - 1;\n   else\n      f->next_seg--;\n   f->valid_bits = 0;\n\n   return 1;\n}\n\nint stb_vorbis_seek_frame(stb_vorbis *f, unsigned int sample_number)\n{\n   uint32 max_frame_samples;\n\n   if (IS_PUSH_MODE(f)) return error(f, VORBIS_invalid_api_mixing);\n\n   // fast page-level search\n   if (!seek_to_sample_coarse(f, sample_number))\n      return 0;\n\n   assert(f->current_loc_valid);\n   assert(f->current_loc <= sample_number);\n\n   // linear search for the relevant packet\n   max_frame_samples = (f->blocksize_1*3 - f->blocksize_0) >> 2;\n   while (f->current_loc < sample_number) {\n      int left_start, left_end, right_start, right_end, mode, frame_samples;\n      if (!peek_decode_initial(f, &left_start, &left_end, &right_start, &right_end, &mode))\n         return error(f, VORBIS_seek_failed);\n      // calculate the number of samples returned by the next frame\n      frame_samples = right_start - left_start;\n      if (f->current_loc + frame_samples > sample_number) {\n         return 1; // the next frame will contain the sample\n      } else if (f->current_loc + frame_samples + max_frame_samples > sample_number) {\n         // there's a chance the frame after this could contain the sample\n         vorbis_pump_first_frame(f);\n      } else {\n         // this frame is too early to be relevant\n         f->current_loc += frame_samples;\n         f->previous_length = 0;\n         maybe_start_packet(f);\n         flush_packet(f);\n      }\n   }\n   // the next frame should start with the sample\n   if (f->current_loc != sample_number) return error(f, VORBIS_seek_failed);\n   return 1;\n}\n\nint stb_vorbis_seek(stb_vorbis *f, unsigned int sample_number)\n{\n   if (!stb_vorbis_seek_frame(f, sample_number))\n      return 0;\n\n   if (sample_number != f->current_loc) {\n      int n;\n      uint32 frame_start = f->current_loc;\n      stb_vorbis_get_frame_float(f, &n, NULL);\n      assert(sample_number > frame_start);\n      assert(f->channel_buffer_start + (int) (sample_number-frame_start) <= f->channel_buffer_end);\n      f->channel_buffer_start += (sample_number - frame_start);\n   }\n\n   return 1;\n}\n\nint stb_vorbis_seek_start(stb_vorbis *f)\n{\n   if (IS_PUSH_MODE(f)) { return error(f, VORBIS_invalid_api_mixing); }\n   set_file_offset(f, f->first_audio_page_offset);\n   f->previous_length = 0;\n   f->first_decode = TRUE;\n   f->next_seg = -1;\n   return vorbis_pump_first_frame(f);\n}\n\nunsigned int stb_vorbis_stream_length_in_samples(stb_vorbis *f)\n{\n   unsigned int restore_offset, previous_safe;\n   unsigned int end, last_page_loc;\n\n   if (IS_PUSH_MODE(f)) return error(f, VORBIS_invalid_api_mixing);\n   if (!f->total_samples) {\n      unsigned int last;\n      uint32 lo,hi;\n      char header[6];\n\n      // first, store the current decode position so we can restore it\n      restore_offset = stb_vorbis_get_file_offset(f);\n\n      // now we want to seek back 64K from the end (the last page must\n      // be at most a little less than 64K, but let's allow a little slop)\n      if (f->stream_len >= 65536 && f->stream_len-65536 >= f->first_audio_page_offset)\n         previous_safe = f->stream_len - 65536;\n      else\n         previous_safe = f->first_audio_page_offset;\n\n      set_file_offset(f, previous_safe);\n      // previous_safe is now our candidate 'earliest known place that seeking\n      // to will lead to the final page'\n\n      if (!vorbis_find_page(f, &end, &last)) {\n         // if we can't find a page, we're hosed!\n         f->error = VORBIS_cant_find_last_page;\n         f->total_samples = 0xffffffff;\n         goto done;\n      }\n\n      // check if there are more pages\n      last_page_loc = stb_vorbis_get_file_offset(f);\n\n      // stop when the last_page flag is set, not when we reach eof;\n      // this allows us to stop short of a 'file_section' end without\n      // explicitly checking the length of the section\n      while (!last) {\n         set_file_offset(f, end);\n         if (!vorbis_find_page(f, &end, &last)) {\n            // the last page we found didn't have the 'last page' flag\n            // set. whoops!\n            break;\n         }\n         //previous_safe = last_page_loc+1; // NOTE: not used after this point, but note for debugging\n         last_page_loc = stb_vorbis_get_file_offset(f);\n      }\n\n      set_file_offset(f, last_page_loc);\n\n      // parse the header\n      getn(f, (unsigned char *)header, 6);\n      // extract the absolute granule position\n      lo = get32(f);\n      hi = get32(f);\n      if (lo == 0xffffffff && hi == 0xffffffff) {\n         f->error = VORBIS_cant_find_last_page;\n         f->total_samples = SAMPLE_unknown;\n         goto done;\n      }\n      if (hi)\n         lo = 0xfffffffe; // saturate\n      f->total_samples = lo;\n\n      f->p_last.page_start = last_page_loc;\n      f->p_last.page_end   = end;\n      f->p_last.last_decoded_sample = lo;\n\n     done:\n      set_file_offset(f, restore_offset);\n   }\n   return f->total_samples == SAMPLE_unknown ? 0 : f->total_samples;\n}\n\nfloat stb_vorbis_stream_length_in_seconds(stb_vorbis *f)\n{\n   return stb_vorbis_stream_length_in_samples(f) / (float) f->sample_rate;\n}\n\n\n\nint stb_vorbis_get_frame_float(stb_vorbis *f, int *channels, float ***output)\n{\n   int len, right,left,i;\n   if (IS_PUSH_MODE(f)) return error(f, VORBIS_invalid_api_mixing);\n\n   if (!vorbis_decode_packet(f, &len, &left, &right)) {\n      f->channel_buffer_start = f->channel_buffer_end = 0;\n      return 0;\n   }\n\n   len = vorbis_finish_frame(f, len, left, right);\n   for (i=0; i < f->channels; ++i)\n      f->outputs[i] = f->channel_buffers[i] + left;\n\n   f->channel_buffer_start = left;\n   f->channel_buffer_end   = left+len;\n\n   if (channels) *channels = f->channels;\n   if (output)   *output = f->outputs;\n   return len;\n}\n\n#ifndef STB_VORBIS_NO_STDIO\n\nstb_vorbis * stb_vorbis_open_file_section(FILE *file, int close_on_free, int *error, const stb_vorbis_alloc *alloc, unsigned int length)\n{\n   stb_vorbis *f, p;\n   vorbis_init(&p, alloc);\n   p.f = file;\n   p.f_start = (uint32) ftell(file);\n   p.stream_len   = length;\n   p.close_on_free = close_on_free;\n   if (start_decoder(&p)) {\n      f = vorbis_alloc(&p);\n      if (f) {\n         *f = p;\n         vorbis_pump_first_frame(f);\n         return f;\n      }\n   }\n   if (error) *error = p.error;\n   vorbis_deinit(&p);\n   return NULL;\n}\n\nstb_vorbis * stb_vorbis_open_file(FILE *file, int close_on_free, int *error, const stb_vorbis_alloc *alloc)\n{\n   unsigned int len, start;\n   start = (unsigned int) ftell(file);\n   fseek(file, 0, SEEK_END);\n   len = (unsigned int) (ftell(file) - start);\n   fseek(file, start, SEEK_SET);\n   return stb_vorbis_open_file_section(file, close_on_free, error, alloc, len);\n}\n\nstb_vorbis * stb_vorbis_open_filename(const char *filename, int *error, const stb_vorbis_alloc *alloc)\n{\n   FILE *f;\n#if defined(_WIN32) && defined(__STDC_WANT_SECURE_LIB__)\n   if (0 != fopen_s(&f, filename, \"rb\"))\n      f = NULL;\n#else\n   f = fopen(filename, \"rb\");\n#endif\n   if (f)\n      return stb_vorbis_open_file(f, TRUE, error, alloc);\n   if (error) *error = VORBIS_file_open_failure;\n   return NULL;\n}\n#endif // STB_VORBIS_NO_STDIO\n\nstb_vorbis * stb_vorbis_open_memory(const unsigned char *data, int len, int *error, const stb_vorbis_alloc *alloc)\n{\n   stb_vorbis *f, p;\n   if (!data) {\n      if (error) *error = VORBIS_unexpected_eof;\n      return NULL;\n   }\n   vorbis_init(&p, alloc);\n   p.stream = (uint8 *) data;\n   p.stream_end = (uint8 *) data + len;\n   p.stream_start = (uint8 *) p.stream;\n   p.stream_len = len;\n   p.push_mode = FALSE;\n   if (start_decoder(&p)) {\n      f = vorbis_alloc(&p);\n      if (f) {\n         *f = p;\n         vorbis_pump_first_frame(f);\n         if (error) *error = VORBIS__no_error;\n         return f;\n      }\n   }\n   if (error) *error = p.error;\n   vorbis_deinit(&p);\n   return NULL;\n}\n\n#ifndef STB_VORBIS_NO_INTEGER_CONVERSION\n#define PLAYBACK_MONO     1\n#define PLAYBACK_LEFT     2\n#define PLAYBACK_RIGHT    4\n\n#define L  (PLAYBACK_LEFT  | PLAYBACK_MONO)\n#define C  (PLAYBACK_LEFT  | PLAYBACK_RIGHT | PLAYBACK_MONO)\n#define R  (PLAYBACK_RIGHT | PLAYBACK_MONO)\n\nstatic int8 channel_position[7][6] =\n{\n   { 0 },\n   { C },\n   { L, R },\n   { L, C, R },\n   { L, R, L, R },\n   { L, C, R, L, R },\n   { L, C, R, L, R, C },\n};\n\n\n#ifndef STB_VORBIS_NO_FAST_SCALED_FLOAT\n   typedef union {\n      float f;\n      int i;\n   } float_conv;\n   typedef char stb_vorbis_float_size_test[sizeof(float)==4 && sizeof(int) == 4];\n   #define FASTDEF(x) float_conv x\n   // add (1<<23) to convert to int, then divide by 2^SHIFT, then add 0.5/2^SHIFT to round\n   #define MAGIC(SHIFT) (1.5f * (1 << (23-SHIFT)) + 0.5f/(1 << SHIFT))\n   #define ADDEND(SHIFT) (((150-SHIFT) << 23) + (1 << 22))\n   #define FAST_SCALED_FLOAT_TO_INT(temp,x,s) (temp.f = (x) + MAGIC(s), temp.i - ADDEND(s))\n   #define check_endianness()\n#else\n   #define FAST_SCALED_FLOAT_TO_INT(temp,x,s) ((int) ((x) * (1 << (s))))\n   #define check_endianness()\n   #define FASTDEF(x)\n#endif\n\nstatic void copy_samples(short *dest, float *src, int len)\n{\n   int i;\n   check_endianness();\n   for (i=0; i < len; ++i) {\n      FASTDEF(temp);\n      int v = FAST_SCALED_FLOAT_TO_INT(temp, src[i],15);\n      if ((unsigned int) (v + 32768) > 65535)\n         v = v < 0 ? -32768 : 32767;\n      dest[i] = v;\n   }\n}\n\nstatic void compute_samples(int mask, short *output, int num_c, float **data, int d_offset, int len)\n{\n   #define STB_BUFFER_SIZE  32\n   float buffer[STB_BUFFER_SIZE];\n   int i,j,o,n = STB_BUFFER_SIZE;\n   check_endianness();\n   for (o = 0; o < len; o += STB_BUFFER_SIZE) {\n      memset(buffer, 0, sizeof(buffer));\n      if (o + n > len) n = len - o;\n      for (j=0; j < num_c; ++j) {\n         if (channel_position[num_c][j] & mask) {\n            for (i=0; i < n; ++i)\n               buffer[i] += data[j][d_offset+o+i];\n         }\n      }\n      for (i=0; i < n; ++i) {\n         FASTDEF(temp);\n         int v = FAST_SCALED_FLOAT_TO_INT(temp,buffer[i],15);\n         if ((unsigned int) (v + 32768) > 65535)\n            v = v < 0 ? -32768 : 32767;\n         output[o+i] = v;\n      }\n   }\n   #undef STB_BUFFER_SIZE\n}\n\nstatic void compute_stereo_samples(short *output, int num_c, float **data, int d_offset, int len)\n{\n   #define STB_BUFFER_SIZE  32\n   float buffer[STB_BUFFER_SIZE];\n   int i,j,o,n = STB_BUFFER_SIZE >> 1;\n   // o is the offset in the source data\n   check_endianness();\n   for (o = 0; o < len; o += STB_BUFFER_SIZE >> 1) {\n      // o2 is the offset in the output data\n      int o2 = o << 1;\n      memset(buffer, 0, sizeof(buffer));\n      if (o + n > len) n = len - o;\n      for (j=0; j < num_c; ++j) {\n         int m = channel_position[num_c][j] & (PLAYBACK_LEFT | PLAYBACK_RIGHT);\n         if (m == (PLAYBACK_LEFT | PLAYBACK_RIGHT)) {\n            for (i=0; i < n; ++i) {\n               buffer[i*2+0] += data[j][d_offset+o+i];\n               buffer[i*2+1] += data[j][d_offset+o+i];\n            }\n         } else if (m == PLAYBACK_LEFT) {\n            for (i=0; i < n; ++i) {\n               buffer[i*2+0] += data[j][d_offset+o+i];\n            }\n         } else if (m == PLAYBACK_RIGHT) {\n            for (i=0; i < n; ++i) {\n               buffer[i*2+1] += data[j][d_offset+o+i];\n            }\n         }\n      }\n      for (i=0; i < (n<<1); ++i) {\n         FASTDEF(temp);\n         int v = FAST_SCALED_FLOAT_TO_INT(temp,buffer[i],15);\n         if ((unsigned int) (v + 32768) > 65535)\n            v = v < 0 ? -32768 : 32767;\n         output[o2+i] = v;\n      }\n   }\n   #undef STB_BUFFER_SIZE\n}\n\nstatic void convert_samples_short(int buf_c, short **buffer, int b_offset, int data_c, float **data, int d_offset, int samples)\n{\n   int i;\n   if (buf_c != data_c && buf_c <= 2 && data_c <= 6) {\n      static int channel_selector[3][2] = { {0}, {PLAYBACK_MONO}, {PLAYBACK_LEFT, PLAYBACK_RIGHT} };\n      for (i=0; i < buf_c; ++i)\n         compute_samples(channel_selector[buf_c][i], buffer[i]+b_offset, data_c, data, d_offset, samples);\n   } else {\n      int limit = buf_c < data_c ? buf_c : data_c;\n      for (i=0; i < limit; ++i)\n         copy_samples(buffer[i]+b_offset, data[i]+d_offset, samples);\n      for (   ; i < buf_c; ++i)\n         memset(buffer[i]+b_offset, 0, sizeof(short) * samples);\n   }\n}\n\nint stb_vorbis_get_frame_short(stb_vorbis *f, int num_c, short **buffer, int num_samples)\n{\n   float **output = NULL;\n   int len = stb_vorbis_get_frame_float(f, NULL, &output);\n   if (len > num_samples) len = num_samples;\n   if (len)\n      convert_samples_short(num_c, buffer, 0, f->channels, output, 0, len);\n   return len;\n}\n\nstatic void convert_channels_short_interleaved(int buf_c, short *buffer, int data_c, float **data, int d_offset, int len)\n{\n   int i;\n   check_endianness();\n   if (buf_c != data_c && buf_c <= 2 && data_c <= 6) {\n      assert(buf_c == 2);\n      for (i=0; i < buf_c; ++i)\n         compute_stereo_samples(buffer, data_c, data, d_offset, len);\n   } else {\n      int limit = buf_c < data_c ? buf_c : data_c;\n      int j;\n      for (j=0; j < len; ++j) {\n         for (i=0; i < limit; ++i) {\n            FASTDEF(temp);\n            float f = data[i][d_offset+j];\n            int v = FAST_SCALED_FLOAT_TO_INT(temp, f,15);//data[i][d_offset+j],15);\n            if ((unsigned int) (v + 32768) > 65535)\n               v = v < 0 ? -32768 : 32767;\n            *buffer++ = v;\n         }\n         for (   ; i < buf_c; ++i)\n            *buffer++ = 0;\n      }\n   }\n}\n\nint stb_vorbis_get_frame_short_interleaved(stb_vorbis *f, int num_c, short *buffer, int num_shorts)\n{\n   float **output;\n   int len;\n   if (num_c == 1) return stb_vorbis_get_frame_short(f,num_c,&buffer, num_shorts);\n   len = stb_vorbis_get_frame_float(f, NULL, &output);\n   if (len) {\n      if (len*num_c > num_shorts) len = num_shorts / num_c;\n      convert_channels_short_interleaved(num_c, buffer, f->channels, output, 0, len);\n   }\n   return len;\n}\n\nint stb_vorbis_get_samples_short_interleaved(stb_vorbis *f, int channels, short *buffer, int num_shorts)\n{\n   float **outputs;\n   int len = num_shorts / channels;\n   int n=0;\n   while (n < len) {\n      int k = f->channel_buffer_end - f->channel_buffer_start;\n      if (n+k >= len) k = len - n;\n      if (k)\n         convert_channels_short_interleaved(channels, buffer, f->channels, f->channel_buffers, f->channel_buffer_start, k);\n      buffer += k*channels;\n      n += k;\n      f->channel_buffer_start += k;\n      if (n == len) break;\n      if (!stb_vorbis_get_frame_float(f, NULL, &outputs)) break;\n   }\n   return n;\n}\n\nint stb_vorbis_get_samples_short(stb_vorbis *f, int channels, short **buffer, int len)\n{\n   float **outputs;\n   int n=0;\n   while (n < len) {\n      int k = f->channel_buffer_end - f->channel_buffer_start;\n      if (n+k >= len) k = len - n;\n      if (k)\n         convert_samples_short(channels, buffer, n, f->channels, f->channel_buffers, f->channel_buffer_start, k);\n      n += k;\n      f->channel_buffer_start += k;\n      if (n == len) break;\n      if (!stb_vorbis_get_frame_float(f, NULL, &outputs)) break;\n   }\n   return n;\n}\n\n#ifndef STB_VORBIS_NO_STDIO\nint stb_vorbis_decode_filename(const char *filename, int *channels, int *sample_rate, short **output)\n{\n   int data_len, offset, total, limit, error;\n   short *data;\n   stb_vorbis *v = stb_vorbis_open_filename(filename, &error, NULL);\n   if (v == NULL) return -1;\n   limit = v->channels * 4096;\n   *channels = v->channels;\n   if (sample_rate)\n      *sample_rate = v->sample_rate;\n   offset = data_len = 0;\n   total = limit;\n   data = (short *) malloc(total * sizeof(*data));\n   if (data == NULL) {\n      stb_vorbis_close(v);\n      return -2;\n   }\n   for (;;) {\n      int n = stb_vorbis_get_frame_short_interleaved(v, v->channels, data+offset, total-offset);\n      if (n == 0) break;\n      data_len += n;\n      offset += n * v->channels;\n      if (offset + limit > total) {\n         short *data2;\n         total *= 2;\n         data2 = (short *) realloc(data, total * sizeof(*data));\n         if (data2 == NULL) {\n            free(data);\n            stb_vorbis_close(v);\n            return -2;\n         }\n         data = data2;\n      }\n   }\n   *output = data;\n   stb_vorbis_close(v);\n   return data_len;\n}\n#endif // NO_STDIO\n\nint stb_vorbis_decode_memory(const uint8 *mem, int len, int *channels, int *sample_rate, short **output)\n{\n   int data_len, offset, total, limit, error;\n   short *data;\n   stb_vorbis *v = stb_vorbis_open_memory(mem, len, &error, NULL);\n   if (v == NULL) return -1;\n   limit = v->channels * 4096;\n   *channels = v->channels;\n   if (sample_rate)\n      *sample_rate = v->sample_rate;\n   offset = data_len = 0;\n   total = limit;\n   data = (short *) malloc(total * sizeof(*data));\n   if (data == NULL) {\n      stb_vorbis_close(v);\n      return -2;\n   }\n   for (;;) {\n      int n = stb_vorbis_get_frame_short_interleaved(v, v->channels, data+offset, total-offset);\n      if (n == 0) break;\n      data_len += n;\n      offset += n * v->channels;\n      if (offset + limit > total) {\n         short *data2;\n         total *= 2;\n         data2 = (short *) realloc(data, total * sizeof(*data));\n         if (data2 == NULL) {\n            free(data);\n            stb_vorbis_close(v);\n            return -2;\n         }\n         data = data2;\n      }\n   }\n   *output = data;\n   stb_vorbis_close(v);\n   return data_len;\n}\n#endif // STB_VORBIS_NO_INTEGER_CONVERSION\n\nint stb_vorbis_get_samples_float_interleaved(stb_vorbis *f, int channels, float *buffer, int num_floats)\n{\n   float **outputs;\n   int len = num_floats / channels;\n   int n=0;\n   int z = f->channels;\n   if (z > channels) z = channels;\n   while (n < len) {\n      int i,j;\n      int k = f->channel_buffer_end - f->channel_buffer_start;\n      if (n+k >= len) k = len - n;\n      for (j=0; j < k; ++j) {\n         for (i=0; i < z; ++i)\n            *buffer++ = f->channel_buffers[i][f->channel_buffer_start+j];\n         for (   ; i < channels; ++i)\n            *buffer++ = 0;\n      }\n      n += k;\n      f->channel_buffer_start += k;\n      if (n == len)\n         break;\n      if (!stb_vorbis_get_frame_float(f, NULL, &outputs))\n         break;\n   }\n   return n;\n}\n\nint stb_vorbis_get_samples_float(stb_vorbis *f, int channels, float **buffer, int num_samples)\n{\n   float **outputs;\n   int n=0;\n   int z = f->channels;\n   if (z > channels) z = channels;\n   while (n < num_samples) {\n      int i;\n      int k = f->channel_buffer_end - f->channel_buffer_start;\n      if (n+k >= num_samples) k = num_samples - n;\n      if (k) {\n         for (i=0; i < z; ++i)\n            memcpy(buffer[i]+n, f->channel_buffers[i]+f->channel_buffer_start, sizeof(float)*k);\n         for (   ; i < channels; ++i)\n            memset(buffer[i]+n, 0, sizeof(float) * k);\n      }\n      n += k;\n      f->channel_buffer_start += k;\n      if (n == num_samples)\n         break;\n      if (!stb_vorbis_get_frame_float(f, NULL, &outputs))\n         break;\n   }\n   return n;\n}\n#endif // STB_VORBIS_NO_PULLDATA_API\n\n/* Version history\n    1.17    - 2019-07-08 - fix CVE-2019-13217, -13218, -13219, -13220, -13221, -13222, -13223\n                           found with Mayhem by ForAllSecure\n    1.16    - 2019-03-04 - fix warnings\n    1.15    - 2019-02-07 - explicit failure if Ogg Skeleton data is found\n    1.14    - 2018-02-11 - delete bogus dealloca usage\n    1.13    - 2018-01-29 - fix truncation of last frame (hopefully)\n    1.12    - 2017-11-21 - limit residue begin/end to blocksize/2 to avoid large temp allocs in bad/corrupt files\n    1.11    - 2017-07-23 - fix MinGW compilation\n    1.10    - 2017-03-03 - more robust seeking; fix negative ilog(); clear error in open_memory\n    1.09    - 2016-04-04 - back out 'avoid discarding last frame' fix from previous version\n    1.08    - 2016-04-02 - fixed multiple warnings; fix setup memory leaks;\n                           avoid discarding last frame of audio data\n    1.07    - 2015-01-16 - fixed some warnings, fix mingw, const-correct API\n                           some more crash fixes when out of memory or with corrupt files\n    1.06    - 2015-08-31 - full, correct support for seeking API (Dougall Johnson)\n                           some crash fixes when out of memory or with corrupt files\n    1.05    - 2015-04-19 - don't define __forceinline if it's redundant\n    1.04    - 2014-08-27 - fix missing const-correct case in API\n    1.03    - 2014-08-07 - Warning fixes\n    1.02    - 2014-07-09 - Declare qsort compare function _cdecl on windows\n    1.01    - 2014-06-18 - fix stb_vorbis_get_samples_float\n    1.0     - 2014-05-26 - fix memory leaks; fix warnings; fix bugs in multichannel\n                           (API change) report sample rate for decode-full-file funcs\n    0.99996 - bracket #include <malloc.h> for macintosh compilation by Laurent Gomila\n    0.99995 - use union instead of pointer-cast for fast-float-to-int to avoid alias-optimization problem\n    0.99994 - change fast-float-to-int to work in single-precision FPU mode, remove endian-dependence\n    0.99993 - remove assert that fired on legal files with empty tables\n    0.99992 - rewind-to-start\n    0.99991 - bugfix to stb_vorbis_get_samples_short by Bernhard Wodo\n    0.9999 - (should have been 0.99990) fix no-CRT support, compiling as C++\n    0.9998 - add a full-decode function with a memory source\n    0.9997 - fix a bug in the read-from-FILE case in 0.9996 addition\n    0.9996 - query length of vorbis stream in samples/seconds\n    0.9995 - bugfix to another optimization that only happened in certain files\n    0.9994 - bugfix to one of the optimizations that caused significant (but inaudible?) errors\n    0.9993 - performance improvements; runs in 99% to 104% of time of reference implementation\n    0.9992 - performance improvement of IMDCT; now performs close to reference implementation\n    0.9991 - performance improvement of IMDCT\n    0.999 - (should have been 0.9990) performance improvement of IMDCT\n    0.998 - no-CRT support from Casey Muratori\n    0.997 - bugfixes for bugs found by Terje Mathisen\n    0.996 - bugfix: fast-huffman decode initialized incorrectly for sparse codebooks; fixing gives 10% speedup - found by Terje Mathisen\n    0.995 - bugfix: fix to 'effective' overrun detection - found by Terje Mathisen\n    0.994 - bugfix: garbage decode on final VQ symbol of a non-multiple - found by Terje Mathisen\n    0.993 - bugfix: pushdata API required 1 extra byte for empty page (failed to consume final page if empty) - found by Terje Mathisen\n    0.992 - fixes for MinGW warning\n    0.991 - turn fast-float-conversion on by default\n    0.990 - fix push-mode seek recovery if you seek into the headers\n    0.98b - fix to bad release of 0.98\n    0.98 - fix push-mode seek recovery; robustify float-to-int and support non-fast mode\n    0.97 - builds under c++ (typecasting, don't use 'class' keyword)\n    0.96 - somehow MY 0.95 was right, but the web one was wrong, so here's my 0.95 rereleased as 0.96, fixes a typo in the clamping code\n    0.95 - clamping code for 16-bit functions\n    0.94 - not publically released\n    0.93 - fixed all-zero-floor case (was decoding garbage)\n    0.92 - fixed a memory leak\n    0.91 - conditional compiles to omit parts of the API and the infrastructure to support them: STB_VORBIS_NO_PULLDATA_API, STB_VORBIS_NO_PUSHDATA_API, STB_VORBIS_NO_STDIO, STB_VORBIS_NO_INTEGER_CONVERSION\n    0.90 - first public release\n*/\n\n#endif // STB_VORBIS_HEADER_ONLY\n\n\n/*\n------------------------------------------------------------------------------\nThis software is available under 2 licenses -- choose whichever you prefer.\n------------------------------------------------------------------------------\nALTERNATIVE A - MIT License\nCopyright (c) 2017 Sean Barrett\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n------------------------------------------------------------------------------\nALTERNATIVE B - Public Domain (www.unlicense.org)\nThis is free and unencumbered software released into the public domain.\nAnyone is free to copy, modify, publish, use, compile, sell, or distribute this\nsoftware, either in source code form or as a compiled binary, for any purpose,\ncommercial or non-commercial, and by any means.\nIn jurisdictions that recognize copyright laws, the author or authors of this\nsoftware dedicate any and all copyright interest in the software to the public\ndomain. We make this dedication for the benefit of the public at large and to\nthe detriment of our heirs and successors. We intend this dedication to be an\novert act of relinquishment in perpetuity of all present and future rights to\nthis software under copyright law.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\nACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n------------------------------------------------------------------------------\n*/\n"
  },
  {
    "path": "examples/stream/CMakeLists.txt",
    "content": "if (WHISPER_SDL2)\n    set(TARGET whisper-stream)\n    add_executable(${TARGET} stream.cpp)\n\n    include(DefaultTargetOptions)\n\n    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${CMAKE_THREAD_LIBS_INIT})\n\n    install(TARGETS ${TARGET} RUNTIME)\nendif ()\n"
  },
  {
    "path": "examples/stream/README.md",
    "content": "# whisper.cpp/examples/stream\r\n\r\nThis is a naive example of performing real-time inference on audio from your microphone.\r\nThe `whisper-stream` tool samples the audio every half a second and runs the transcription continously.\r\nMore info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).\r\n\r\n```bash\r\n./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 8 --step 500 --length 5000\r\n```\r\n\r\nhttps://user-images.githubusercontent.com/1991296/194935793-76afede7-cfa8-48d8-a80f-28ba83be7d09.mp4\r\n\r\n## Sliding window mode with VAD\r\n\r\nSetting the `--step` argument to `0` enables the sliding window mode:\r\n\r\n```bash\r\n ./build/bin/whisper-stream -m ./models/ggml-base.en.bin -t 6 --step 0 --length 30000 -vth 0.6\r\n```\r\n\r\nIn this mode, the tool will transcribe only after some speech activity is detected. A very\r\nbasic VAD detector is used, but in theory a more sophisticated approach can be added. The\r\n`-vth` argument determines the VAD threshold - higher values will make it detect silence more often.\r\nIt's best to tune it to the specific use case, but a value around `0.6` should be OK in general.\r\nWhen silence is detected, it will transcribe the last `--length` milliseconds of audio and output\r\na transcription block that is suitable for parsing.\r\n\r\n## Building\r\n\r\nThe `whisper-stream` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:\r\n\r\n```bash\r\n# Install SDL2\r\n# On Debian based linux distributions:\r\nsudo apt-get install libsdl2-dev\r\n\r\n# On Fedora Linux:\r\nsudo dnf install SDL2 SDL2-devel\r\n\r\n# Install SDL2 on Mac OS\r\nbrew install sdl2\r\n\r\ncmake -B build -DWHISPER_SDL2=ON\r\ncmake --build build --config Release\r\n\r\n./build/bin/whisper-stream\r\n```\r\n\r\n## Web version\r\n\r\nThis tool can also run in the browser: [examples/stream.wasm](/examples/stream.wasm)\r\n"
  },
  {
    "path": "examples/stream/stream.cpp",
    "content": "// Real-time speech recognition of input from a microphone\n//\n// A very quick-n-dirty implementation serving mainly as a proof of concept.\n//\n#include \"common-sdl.h\"\n#include \"common.h\"\n#include \"common-whisper.h\"\n#include \"whisper.h\"\n\n#include <chrono>\n#include <cstdio>\n#include <fstream>\n#include <string>\n#include <thread>\n#include <vector>\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t step_ms    = 3000;\n    int32_t length_ms  = 10000;\n    int32_t keep_ms    = 200;\n    int32_t capture_id = -1;\n    int32_t max_tokens = 32;\n    int32_t audio_ctx  = 0;\n    int32_t beam_size  = -1;\n\n    float vad_thold    = 0.6f;\n    float freq_thold   = 100.0f;\n\n    bool translate     = false;\n    bool no_fallback   = false;\n    bool print_special = false;\n    bool no_context    = true;\n    bool no_timestamps = false;\n    bool tinydiarize   = false;\n    bool save_audio    = false; // save audio to wav file\n    bool use_gpu       = true;\n    bool flash_attn    = true;\n\n    std::string language  = \"en\";\n    std::string model     = \"models/ggml-base.en.bin\";\n    std::string fname_out;\n};\n\nvoid whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"    || arg == \"--threads\")       { params.n_threads     = std::stoi(argv[++i]); }\n        else if (                  arg == \"--step\")          { params.step_ms       = std::stoi(argv[++i]); }\n        else if (                  arg == \"--length\")        { params.length_ms     = std::stoi(argv[++i]); }\n        else if (                  arg == \"--keep\")          { params.keep_ms       = std::stoi(argv[++i]); }\n        else if (arg == \"-c\"    || arg == \"--capture\")       { params.capture_id    = std::stoi(argv[++i]); }\n        else if (arg == \"-mt\"   || arg == \"--max-tokens\")    { params.max_tokens    = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"   || arg == \"--audio-ctx\")     { params.audio_ctx     = std::stoi(argv[++i]); }\n        else if (arg == \"-bs\"   || arg == \"--beam-size\")     { params.beam_size     = std::stoi(argv[++i]); }\n        else if (arg == \"-vth\"  || arg == \"--vad-thold\")     { params.vad_thold     = std::stof(argv[++i]); }\n        else if (arg == \"-fth\"  || arg == \"--freq-thold\")    { params.freq_thold    = std::stof(argv[++i]); }\n        else if (arg == \"-tr\"   || arg == \"--translate\")     { params.translate     = true; }\n        else if (arg == \"-nf\"   || arg == \"--no-fallback\")   { params.no_fallback   = true; }\n        else if (arg == \"-ps\"   || arg == \"--print-special\") { params.print_special = true; }\n        else if (arg == \"-kc\"   || arg == \"--keep-context\")  { params.no_context    = false; }\n        else if (arg == \"-l\"    || arg == \"--language\")      { params.language      = argv[++i]; }\n        else if (arg == \"-m\"    || arg == \"--model\")         { params.model         = argv[++i]; }\n        else if (arg == \"-f\"    || arg == \"--file\")          { params.fname_out     = argv[++i]; }\n        else if (arg == \"-tdrz\" || arg == \"--tinydiarize\")   { params.tinydiarize   = true; }\n        else if (arg == \"-sa\"   || arg == \"--save-audio\")    { params.save_audio    = true; }\n        else if (arg == \"-ng\"   || arg == \"--no-gpu\")        { params.use_gpu       = false; }\n        else if (arg == \"-fa\"   || arg == \"--flash-attn\")    { params.flash_attn    = true; }\n        else if (arg == \"-nfa\"  || arg == \"--no-flash-attn\") { params.flash_attn    = false; }\n\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,       --help          [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,     --threads N     [%-7d] number of threads to use during computation\\n\",    params.n_threads);\n    fprintf(stderr, \"            --step N        [%-7d] audio step size in milliseconds\\n\",                params.step_ms);\n    fprintf(stderr, \"            --length N      [%-7d] audio length in milliseconds\\n\",                   params.length_ms);\n    fprintf(stderr, \"            --keep N        [%-7d] audio to keep from previous step in ms\\n\",         params.keep_ms);\n    fprintf(stderr, \"  -c ID,    --capture ID    [%-7d] capture device ID\\n\",                              params.capture_id);\n    fprintf(stderr, \"  -mt N,    --max-tokens N  [%-7d] maximum number of tokens per audio chunk\\n\",       params.max_tokens);\n    fprintf(stderr, \"  -ac N,    --audio-ctx N   [%-7d] audio context size (0 - all)\\n\",                   params.audio_ctx);\n    fprintf(stderr, \"  -bs N,    --beam-size N   [%-7d] beam size for beam search\\n\",                      params.beam_size);\n    fprintf(stderr, \"  -vth N,   --vad-thold N   [%-7.2f] voice activity detection threshold\\n\",           params.vad_thold);\n    fprintf(stderr, \"  -fth N,   --freq-thold N  [%-7.2f] high-pass frequency cutoff\\n\",                   params.freq_thold);\n    fprintf(stderr, \"  -tr,      --translate     [%-7s] translate from source language to english\\n\",      params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nf,      --no-fallback   [%-7s] do not use temperature fallback while decoding\\n\", params.no_fallback ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,      --print-special [%-7s] print special tokens\\n\",                           params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -kc,      --keep-context  [%-7s] keep context between audio chunks\\n\",              params.no_context ? \"false\" : \"true\");\n    fprintf(stderr, \"  -l LANG,  --language LANG [%-7s] spoken language\\n\",                                params.language.c_str());\n    fprintf(stderr, \"  -m FNAME, --model FNAME   [%-7s] model path\\n\",                                     params.model.c_str());\n    fprintf(stderr, \"  -f FNAME, --file FNAME    [%-7s] text output file name\\n\",                          params.fname_out.c_str());\n    fprintf(stderr, \"  -tdrz,    --tinydiarize   [%-7s] enable tinydiarize (requires a tdrz model)\\n\",     params.tinydiarize ? \"true\" : \"false\");\n    fprintf(stderr, \"  -sa,      --save-audio    [%-7s] save the recorded audio to a file\\n\",              params.save_audio ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ng,      --no-gpu        [%-7s] disable GPU inference\\n\",                          params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,      --flash-attn    [%-7s] enable flash attention during inference\\n\",        params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,     --no-flash-attn [%-7s] disable flash attention during inference\\n\",       params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"\\n\");\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    params.keep_ms   = std::min(params.keep_ms,   params.step_ms);\n    params.length_ms = std::max(params.length_ms, params.step_ms);\n\n    const int n_samples_step = (1e-3*params.step_ms  )*WHISPER_SAMPLE_RATE;\n    const int n_samples_len  = (1e-3*params.length_ms)*WHISPER_SAMPLE_RATE;\n    const int n_samples_keep = (1e-3*params.keep_ms  )*WHISPER_SAMPLE_RATE;\n    const int n_samples_30s  = (1e-3*30000.0         )*WHISPER_SAMPLE_RATE;\n\n    const bool use_vad = n_samples_step <= 0; // sliding window mode uses VAD\n\n    const int n_new_line = !use_vad ? std::max(1, params.length_ms / params.step_ms - 1) : 1; // number of steps to print new line\n\n    params.no_timestamps  = !use_vad;\n    params.no_context    |= use_vad;\n    params.max_tokens     = 0;\n\n    // init audio\n\n    audio_async audio(params.length_ms);\n    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {\n        fprintf(stderr, \"%s: audio.init() failed!\\n\", __func__);\n        return 1;\n    }\n\n    audio.resume();\n\n    // whisper init\n    if (params.language != \"auto\" && whisper_lang_id(params.language.c_str()) == -1){\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n    if (ctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 2;\n    }\n\n    std::vector<float> pcmf32    (n_samples_30s, 0.0f);\n    std::vector<float> pcmf32_old;\n    std::vector<float> pcmf32_new(n_samples_30s, 0.0f);\n\n    std::vector<whisper_token> prompt_tokens;\n\n    // print some info about the processing\n    {\n        fprintf(stderr, \"\\n\");\n        if (!whisper_is_multilingual(ctx)) {\n            if (params.language != \"en\" || params.translate) {\n                params.language = \"en\";\n                params.translate = false;\n                fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n            }\n        }\n        fprintf(stderr, \"%s: processing %d samples (step = %.1f sec / len = %.1f sec / keep = %.1f sec), %d threads, lang = %s, task = %s, timestamps = %d ...\\n\",\n                __func__,\n                n_samples_step,\n                float(n_samples_step)/WHISPER_SAMPLE_RATE,\n                float(n_samples_len )/WHISPER_SAMPLE_RATE,\n                float(n_samples_keep)/WHISPER_SAMPLE_RATE,\n                params.n_threads,\n                params.language.c_str(),\n                params.translate ? \"translate\" : \"transcribe\",\n                params.no_timestamps ? 0 : 1);\n\n        if (!use_vad) {\n            fprintf(stderr, \"%s: n_new_line = %d, no_context = %d\\n\", __func__, n_new_line, params.no_context);\n        } else {\n            fprintf(stderr, \"%s: using VAD, will transcribe on speech activity\\n\", __func__);\n        }\n\n        fprintf(stderr, \"\\n\");\n    }\n\n    int n_iter = 0;\n\n    bool is_running = true;\n\n    std::ofstream fout;\n    if (params.fname_out.length() > 0) {\n        fout.open(params.fname_out);\n        if (!fout.is_open()) {\n            fprintf(stderr, \"%s: failed to open output file '%s'!\\n\", __func__, params.fname_out.c_str());\n            return 1;\n        }\n    }\n\n    wav_writer wavWriter;\n    // save wav file\n    if (params.save_audio) {\n        // Get current date/time for filename\n        time_t now = time(0);\n        char buffer[80];\n        strftime(buffer, sizeof(buffer), \"%Y%m%d%H%M%S\", localtime(&now));\n        std::string filename = std::string(buffer) + \".wav\";\n\n        wavWriter.open(filename, WHISPER_SAMPLE_RATE, 16, 1);\n    }\n    printf(\"[Start speaking]\\n\");\n    fflush(stdout);\n\n    auto t_last  = std::chrono::high_resolution_clock::now();\n    const auto t_start = t_last;\n\n    // main audio loop\n    while (is_running) {\n        if (params.save_audio) {\n            wavWriter.write(pcmf32_new.data(), pcmf32_new.size());\n        }\n        // handle Ctrl + C\n        is_running = sdl_poll_events();\n\n        if (!is_running) {\n            break;\n        }\n\n        // process new audio\n\n        if (!use_vad) {\n            while (true) {\n                // handle Ctrl + C\n                is_running = sdl_poll_events();\n                if (!is_running) {\n                    break;\n                }\n                audio.get(params.step_ms, pcmf32_new);\n\n                if ((int) pcmf32_new.size() > 2*n_samples_step) {\n                    fprintf(stderr, \"\\n\\n%s: WARNING: cannot process audio fast enough, dropping audio ...\\n\\n\", __func__);\n                    audio.clear();\n                    continue;\n                }\n\n                if ((int) pcmf32_new.size() >= n_samples_step) {\n                    audio.clear();\n                    break;\n                }\n\n                std::this_thread::sleep_for(std::chrono::milliseconds(1));\n            }\n\n            const int n_samples_new = pcmf32_new.size();\n\n            // take up to params.length_ms audio from previous iteration\n            const int n_samples_take = std::min((int) pcmf32_old.size(), std::max(0, n_samples_keep + n_samples_len - n_samples_new));\n\n            //printf(\"processing: take = %d, new = %d, old = %d\\n\", n_samples_take, n_samples_new, (int) pcmf32_old.size());\n\n            pcmf32.resize(n_samples_new + n_samples_take);\n\n            for (int i = 0; i < n_samples_take; i++) {\n                pcmf32[i] = pcmf32_old[pcmf32_old.size() - n_samples_take + i];\n            }\n\n            memcpy(pcmf32.data() + n_samples_take, pcmf32_new.data(), n_samples_new*sizeof(float));\n\n            pcmf32_old = pcmf32;\n        } else {\n            const auto t_now  = std::chrono::high_resolution_clock::now();\n            const auto t_diff = std::chrono::duration_cast<std::chrono::milliseconds>(t_now - t_last).count();\n\n            if (t_diff < 2000) {\n                std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n                continue;\n            }\n\n            audio.get(2000, pcmf32_new);\n\n            if (::vad_simple(pcmf32_new, WHISPER_SAMPLE_RATE, 1000, params.vad_thold, params.freq_thold, false)) {\n                audio.get(params.length_ms, pcmf32);\n            } else {\n                std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n                continue;\n            }\n\n            t_last = t_now;\n        }\n\n        // run the inference\n        {\n            whisper_full_params wparams = whisper_full_default_params(params.beam_size > 1 ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY);\n\n            wparams.print_progress   = false;\n            wparams.print_special    = params.print_special;\n            wparams.print_realtime   = false;\n            wparams.print_timestamps = !params.no_timestamps;\n            wparams.translate        = params.translate;\n            wparams.single_segment   = !use_vad;\n            wparams.max_tokens       = params.max_tokens;\n            wparams.language         = params.language.c_str();\n            wparams.n_threads        = params.n_threads;\n            wparams.beam_search.beam_size = params.beam_size;\n\n            wparams.audio_ctx        = params.audio_ctx;\n\n            wparams.tdrz_enable      = params.tinydiarize; // [TDRZ]\n\n            // disable temperature fallback\n            //wparams.temperature_inc  = -1.0f;\n            wparams.temperature_inc  = params.no_fallback ? 0.0f : wparams.temperature_inc;\n\n            wparams.prompt_tokens    = params.no_context ? nullptr : prompt_tokens.data();\n            wparams.prompt_n_tokens  = params.no_context ? 0       : prompt_tokens.size();\n\n            if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n                fprintf(stderr, \"%s: failed to process audio\\n\", argv[0]);\n                return 6;\n            }\n\n            // print result;\n            {\n                if (!use_vad) {\n                    printf(\"\\33[2K\\r\");\n\n                    // print long empty line to clear the previous line\n                    printf(\"%s\", std::string(100, ' ').c_str());\n\n                    printf(\"\\33[2K\\r\");\n                } else {\n                    const int64_t t1 = (t_last - t_start).count()/1000000;\n                    const int64_t t0 = std::max(0.0, t1 - pcmf32.size()*1000.0/WHISPER_SAMPLE_RATE);\n\n                    printf(\"\\n\");\n                    printf(\"### Transcription %d START | t0 = %d ms | t1 = %d ms\\n\", n_iter, (int) t0, (int) t1);\n                    printf(\"\\n\");\n                }\n\n                const int n_segments = whisper_full_n_segments(ctx);\n                for (int i = 0; i < n_segments; ++i) {\n                    const char * text = whisper_full_get_segment_text(ctx, i);\n\n                    if (params.no_timestamps) {\n                        printf(\"%s\", text);\n                        fflush(stdout);\n\n                        if (params.fname_out.length() > 0) {\n                            fout << text;\n                        }\n                    } else {\n                        const int64_t t0 = whisper_full_get_segment_t0(ctx, i);\n                        const int64_t t1 = whisper_full_get_segment_t1(ctx, i);\n\n                        std::string output = \"[\" + to_timestamp(t0, false) + \" --> \" + to_timestamp(t1, false) + \"]  \" + text;\n\n                        if (whisper_full_get_segment_speaker_turn_next(ctx, i)) {\n                            output += \" [SPEAKER_TURN]\";\n                        }\n\n                        output += \"\\n\";\n\n                        printf(\"%s\", output.c_str());\n                        fflush(stdout);\n\n                        if (params.fname_out.length() > 0) {\n                            fout << output;\n                        }\n                    }\n                }\n\n                if (params.fname_out.length() > 0) {\n                    fout << std::endl;\n                }\n\n                if (use_vad) {\n                    printf(\"\\n\");\n                    printf(\"### Transcription %d END\\n\", n_iter);\n                }\n            }\n\n            ++n_iter;\n\n            if (!use_vad && (n_iter % n_new_line) == 0) {\n                printf(\"\\n\");\n\n                // keep part of the audio for next iteration to try to mitigate word boundary issues\n                pcmf32_old = std::vector<float>(pcmf32.end() - n_samples_keep, pcmf32.end());\n\n                // Add tokens of the last full length segment as the prompt\n                if (!params.no_context) {\n                    prompt_tokens.clear();\n\n                    const int n_segments = whisper_full_n_segments(ctx);\n                    for (int i = 0; i < n_segments; ++i) {\n                        const int token_count = whisper_full_n_tokens(ctx, i);\n                        for (int j = 0; j < token_count; ++j) {\n                            prompt_tokens.push_back(whisper_full_get_token_id(ctx, i, j));\n                        }\n                    }\n                }\n            }\n            fflush(stdout);\n        }\n    }\n\n    audio.pause();\n\n    whisper_print_timings(ctx);\n    whisper_free(ctx);\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/stream.wasm/CMakeLists.txt",
    "content": "#\n# libstream\n#\n\nset(TARGET libstream)\n\nadd_executable(${TARGET}\n    emscripten.cpp\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE\n    whisper\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside stream.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libstream.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/stream.wasm/stream.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE=8 \\\n    -s INITIAL_MEMORY=1024MB \\\n    -s TOTAL_MEMORY=1024MB \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s EXPORTED_RUNTIME_METHODS=\\\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\\\" \\\n    ${EXTRA_FLAGS} \\\n    \")\n\n#\n# stream.wasm\n#\n\nset(TARGET stream.wasm)\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)\n"
  },
  {
    "path": "examples/stream.wasm/README.md",
    "content": "# stream.wasm\n\nReal-time transcription in the browser using WebAssembly\n\nOnline demo: https://ggml.ai/whisper.cpp/stream.wasm/\n\n## Build instructions\n\n```bash\n# build using Emscripten (v3.1.2)\ngit clone https://github.com/ggerganov/whisper.cpp\ncd whisper.cpp\nmkdir build-em && cd build-em\nemcmake cmake ..\nmake -j\n```\nThe example can then be started by running a local HTTP server:\n```console\npython3 examples/server.py\n```\nAnd then opening a browser to the following URL:\nhttp://localhost:8000/stream.wasm\n\nTo run the example in a different server, you need to copy the following files\nto the server's HTTP path:\n```\n# copy the produced page to your HTTP path\ncp bin/stream.wasm/*       /path/to/html/\ncp bin/libstream.js        /path/to/html/\ncp bin/libstream.worker.js /path/to/html/\n```\n\n> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`\n> which means that that a separate .wasm file will not be generated. Instead, the\n> WASM module is embedded in the main JS file as a base64 encoded string. To\n> generate a separate .wasm file, you need to disable this option by passing\n> `-DWHISPER_WASM_SINGLE_FILE=OFF`:\n> ```console\n> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF\n> ```\n> This will generate a `libstream.wasm` file in the build/bin directory.\n\n> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no\n> longer generated and the worker is embedded in the main JS file. So the worker\n> file will not be geneated for versions later than `3.1.58`.\n"
  },
  {
    "path": "examples/stream.wasm/emscripten.cpp",
    "content": "#include \"ggml.h\"\n#include \"whisper.h\"\n\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <atomic>\n#include <cmath>\n#include <mutex>\n#include <string>\n#include <thread>\n#include <vector>\n\nconstexpr int N_THREAD = 8;\n\nstd::vector<struct whisper_context *> g_contexts(4, nullptr);\n\nstd::mutex g_mutex;\nstd::thread g_worker;\n\nstd::atomic<bool> g_running(false);\n\nstd::string g_status        = \"\";\nstd::string g_status_forced = \"\";\nstd::string g_transcribed   = \"\";\n\nstd::vector<float> g_pcmf32;\n\nvoid stream_set_status(const std::string & status) {\n    std::lock_guard<std::mutex> lock(g_mutex);\n    g_status = status;\n}\n\nvoid stream_main(size_t index, const std::string & lang) {\n    stream_set_status(\"loading data ...\");\n\n    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n    bool is_multilingual = whisper_is_multilingual(g_contexts[index]);\n\n    wparams.n_threads        = std::min(N_THREAD, (int) std::thread::hardware_concurrency());\n    wparams.offset_ms        = 0;\n    wparams.translate        = false;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.print_realtime   = false;\n    wparams.print_progress   = false;\n    wparams.print_timestamps = true;\n    wparams.print_special    = false;\n\n    wparams.max_tokens       = 32;\n    wparams.audio_ctx        = 768; // partial encoder context for better performance\n\n    // disable temperature fallback\n    wparams.temperature_inc  = -1.0f;\n\n    wparams.language         = is_multilingual ? lang.c_str() : \"en\";\n\n    printf(\"stream: using %d threads\\n\", wparams.n_threads);\n\n    std::vector<float> pcmf32;\n\n    // whisper context\n    auto & ctx = g_contexts[index];\n\n    // 5 seconds interval\n    const int64_t window_samples = 5*WHISPER_SAMPLE_RATE;\n\n    while (g_running) {\n        stream_set_status(\"waiting for audio ...\");\n\n        {\n            std::unique_lock<std::mutex> lock(g_mutex);\n\n            if (g_pcmf32.size() < 1024) {\n                lock.unlock();\n\n                std::this_thread::sleep_for(std::chrono::milliseconds(10));\n\n                continue;\n            }\n\n            pcmf32 = std::vector<float>(g_pcmf32.end() - std::min((int64_t) g_pcmf32.size(), window_samples), g_pcmf32.end());\n            g_pcmf32.clear();\n        }\n\n        {\n            const auto t_start = std::chrono::high_resolution_clock::now();\n\n            stream_set_status(\"running whisper ...\");\n\n            int ret = whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size());\n            if (ret != 0) {\n                printf(\"whisper_full() failed: %d\\n\", ret);\n                break;\n            }\n\n            const auto t_end = std::chrono::high_resolution_clock::now();\n\n            printf(\"stream: whisper_full() returned %d in %f seconds\\n\", ret, std::chrono::duration<double>(t_end - t_start).count());\n        }\n\n        {\n            std::string text_heard;\n\n            {\n                const int n_segments = whisper_full_n_segments(ctx);\n                if (n_segments > 0) {\n                    const char * text = whisper_full_get_segment_text(ctx, n_segments - 1);\n\n                    const int64_t t0 = whisper_full_get_segment_t0(ctx, n_segments - 1);\n                    const int64_t t1 = whisper_full_get_segment_t1(ctx, n_segments - 1);\n\n                    printf(\"transcribed: %s\\n\", text);\n\n                    text_heard += text;\n                }\n            }\n\n            {\n                std::lock_guard<std::mutex> lock(g_mutex);\n                g_transcribed = text_heard;\n            }\n        }\n    }\n\n    if (index < g_contexts.size()) {\n        whisper_free(g_contexts[index]);\n        g_contexts[index] = nullptr;\n    }\n}\nEMSCRIPTEN_BINDINGS(stream) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model, const std::string & lang) {\n        for (size_t i = 0; i < g_contexts.size(); ++i) {\n            if (g_contexts[i] == nullptr) {\n                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n                if (g_contexts[i] != nullptr) {\n                    g_running = true;\n                    if (g_worker.joinable()) {\n                        g_worker.join();\n                    }\n                    g_worker = std::thread([i, lang]() {\n                        stream_main(i, lang);\n                    });\n\n                    return i + 1;\n                } else {\n                    return (size_t) 0;\n                }\n            }\n        }\n\n        return (size_t) 0;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([](size_t index) {\n        if (g_running) {\n            g_running = false;\n        }\n    }));\n\n    emscripten::function(\"set_audio\", emscripten::optional_override([](size_t index, const emscripten::val & audio) {\n        --index;\n\n        if (index >= g_contexts.size()) {\n            return -1;\n        }\n\n        if (g_contexts[index] == nullptr) {\n            return -2;\n        }\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            const int n = audio[\"length\"].as<int>();\n\n            emscripten::val heap = emscripten::val::module_property(\"HEAPU8\");\n            emscripten::val memory = heap[\"buffer\"];\n\n            g_pcmf32.resize(n);\n\n            emscripten::val memoryView = audio[\"constructor\"].new_(memory, reinterpret_cast<uintptr_t>(g_pcmf32.data()), n);\n            memoryView.call<void>(\"set\", audio);\n        }\n\n        return 0;\n    }));\n\n    emscripten::function(\"get_transcribed\", emscripten::optional_override([]() {\n        std::string transcribed;\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            transcribed = std::move(g_transcribed);\n        }\n\n        return transcribed;\n    }));\n\n    emscripten::function(\"get_status\", emscripten::optional_override([]() {\n        std::string status;\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            status = g_status_forced.empty() ? g_status : g_status_forced;\n        }\n\n        return status;\n    }));\n\n    emscripten::function(\"set_status\", emscripten::optional_override([](const std::string & status) {\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            g_status_forced = status;\n        }\n    }));\n}\n"
  },
  {
    "path": "examples/stream.wasm/index-tmpl.html",
    "content": "<!doctype html>\n<html lang=\"en-us\">\n    <head>\n        <title>stream : Real-time Whisper transcription in WebAssembly</title>\n\n        <style>\n            #output {\n                width: 100%;\n                height: 100%;\n                margin: 0 auto;\n                margin-top: 10px;\n                border-left: 0px;\n                border-right: 0px;\n                padding-left: 0px;\n                padding-right: 0px;\n                display: block;\n                background-color: black;\n                color: white;\n                font-size: 10px;\n                font-family: 'Lucida Console', Monaco, monospace;\n                outline: none;\n                white-space: pre;\n                overflow-wrap: normal;\n                overflow-x: scroll;\n            }\n        </style>\n        <script src=\"../coi-serviceworker.js\"></script>\n        <link rel=\"icon\" href=\"data:,\">\n    </head>\n    <body>\n        <div id=\"main-container\">\n            <b>stream : Real-time Whisper transcription in WebAssembly</b>\n\n            <br><br>\n\n            You can find more about this project on <a href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/stream.wasm\">GitHub</a>.\n\n            <br><br>\n\n            <b>More examples:</b>\n                <a href=\"../\">main</a> |\n                <a href=\"../bench.wasm/\">bench</a> |\n                <a href=\"../stream.wasm\">stream</a> |\n                <a href=\"../command.wasm/\">command</a> |\n                <a href=\"../wchess.wasm/\">wchess</a> |\n\n            <br><br>\n\n            <hr>\n\n            Select the model you would like to use, click the \"Start\" button and start speaking\n\n            <br><br>\n\n            <div id=\"model-whisper\">\n                Whisper model: <span id=\"model-whisper-status\"></span>\n                <button id=\"fetch-whisper-tiny-en\" onclick=\"loadWhisper('tiny.en')\">tiny.en (75 MB)</button>\n                <button id=\"fetch-whisper-base-en\" onclick=\"loadWhisper('base.en')\">base.en (142 MB)</button>\n                <button id=\"fetch-whisper-base\" onclick=\"loadWhisper('base')\">base (142 MB)</button>\n                <br><br>\n                Quantized models:<br><br>\n                <button id=\"fetch-whisper-tiny-en-q5_1\"   onclick=\"loadWhisper('tiny-en-q5_1')\">tiny.en (Q5_1, 31 MB)</button>\n                <button id=\"fetch-whisper-base-en-q5_1\"   onclick=\"loadWhisper('base-en-q5_1')\">base.en (Q5_1, 57 MB)</button>\n                <span id=\"fetch-whisper-progress\"></span>\n\n                <!--\n                    <input type=\"file\" id=\"file\" name=\"file\" onchange=\"loadFile(event, 'whisper.bin')\" />\n                -->\n            </div>\n\n            <table>\n                <tr>\n                    <td>\n                        Language:\n                        <select id=\"language\" name=\"language\">\n                            <option value=\"en\">English</option>\n                            <option value=\"ar\">Arabic</option>\n                            <option value=\"hy\">Armenian</option>\n                            <option value=\"az\">Azerbaijani</option>\n                            <option value=\"eu\">Basque</option>\n                            <option value=\"be\">Belarusian</option>\n                            <option value=\"bn\">Bengali</option>\n                            <option value=\"bg\">Bulgarian</option>\n                            <option value=\"ca\">Catalan</option>\n                            <option value=\"zh\">Chinese</option>\n                            <option value=\"hr\">Croatian</option>\n                            <option value=\"cs\">Czech</option>\n                            <option value=\"da\">Danish</option>\n                            <option value=\"nl\">Dutch</option>\n                            <option value=\"en\">English</option>\n                            <option value=\"et\">Estonian</option>\n                            <option value=\"tl\">Filipino</option>\n                            <option value=\"fi\">Finnish</option>\n                            <option value=\"fr\">French</option>\n                            <option value=\"gl\">Galician</option>\n                            <option value=\"ka\">Georgian</option>\n                            <option value=\"de\">German</option>\n                            <option value=\"el\">Greek</option>\n                            <option value=\"gu\">Gujarati</option>\n                            <option value=\"iw\">Hebrew</option>\n                            <option value=\"hi\">Hindi</option>\n                            <option value=\"hu\">Hungarian</option>\n                            <option value=\"is\">Icelandic</option>\n                            <option value=\"id\">Indonesian</option>\n                            <option value=\"ga\">Irish</option>\n                            <option value=\"it\">Italian</option>\n                            <option value=\"ja\">Japanese</option>\n                            <option value=\"kn\">Kannada</option>\n                            <option value=\"ko\">Korean</option>\n                            <option value=\"la\">Latin</option>\n                            <option value=\"lv\">Latvian</option>\n                            <option value=\"lt\">Lithuanian</option>\n                            <option value=\"mk\">Macedonian</option>\n                            <option value=\"ms\">Malay</option>\n                            <option value=\"mt\">Maltese</option>\n                            <option value=\"no\">Norwegian</option>\n                            <option value=\"fa\">Persian</option>\n                            <option value=\"pl\">Polish</option>\n                            <option value=\"pt\">Portuguese</option>\n                            <option value=\"ro\">Romanian</option>\n                            <option value=\"ru\">Russian</option>\n                            <option value=\"sr\">Serbian</option>\n                            <option value=\"sk\">Slovak</option>\n                            <option value=\"sl\">Slovenian</option>\n                            <option value=\"es\">Spanish</option>\n                            <option value=\"sw\">Swahili</option>\n                            <option value=\"sv\">Swedish</option>\n                            <option value=\"ta\">Tamil</option>\n                            <option value=\"te\">Telugu</option>\n                            <option value=\"th\">Thai</option>\n                            <option value=\"tr\">Turkish</option>\n                            <option value=\"uk\">Ukrainian</option>\n                            <option value=\"ur\">Urdu</option>\n                            <option value=\"vi\">Vietnamese</option>\n                            <option value=\"cy\">Welsh</option>\n                            <option value=\"yi\">Yiddish</option>\n                        </select>\n                    </td>\n                </tr>\n            </table>\n\n            <br>\n\n            <div id=\"input\">\n                <button id=\"start\"  onclick=\"onStart()\" disabled>Start</button>\n                <button id=\"stop\"   onclick=\"onStop()\" disabled>Stop</button>\n                <button id=\"clear\"  onclick=\"clearCache()\">Clear Cache</button>\n            </div>\n\n            <br>\n\n            <div id=\"state\">\n                Status: <b><span id=\"state-status\">not started</span></b>\n\n                <pre id=\"state-transcribed\">[The transcribed text will be displayed here]</pre>\n            </div>\n\n            <hr>\n\n            Debug output:\n            <textarea id=\"output\" rows=\"20\"></textarea>\n\n            <br>\n\n            <b>Troubleshooting</b>\n\n            <br><br>\n\n            The page does some heavy computations, so make sure:\n\n            <ul>\n                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>\n                <li>To use a fast desktop or laptop computer (i.e. not a mobile phone)</li>\n                <li>Your browser supports WASM <a href=\"https://webassembly.org/roadmap/\">Fixed-width SIMD</a></li>\n            </ul>\n\n            <div class=\"cell-version\">\n                <span>\n                    |\n                    Build time: <span class=\"nav-link\">@GIT_DATE@</span> |\n                    Commit hash: <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@\">@GIT_SHA1@</a> |\n                    Commit subject: <span class=\"nav-link\">@GIT_COMMIT_SUBJECT@</span> |\n                    <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/stream.wasm\">Source Code</a> |\n                </span>\n            </div>\n        </div>\n\n        <script type=\"text/javascript\" src=\"helpers.js\"></script>\n        <script type='text/javascript'>\n            // web audio context\n            var context = null;\n\n            // audio data\n            var audio = null;\n            var audio0 = null;\n\n            // the stream instance\n            var instance = null;\n\n            // model name\n            var model_whisper = null;\n\n            var Module = {\n                print: printTextarea,\n                printErr: printTextarea,\n                setStatus: function(text) {\n                    printTextarea('js: ' + text);\n                },\n                monitorRunDependencies: function(left) {\n                },\n                preRun: function() {\n                    printTextarea('js: Preparing ...');\n                },\n                postRun: function() {\n                    printTextarea('js: Initialized successfully!');\n                }\n            };\n\n            //\n            // fetch models\n            //\n\n            let dbVersion = 1\n            let dbName    = 'whisper.ggerganov.com';\n            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB\n\n            function storeFS(fname, buf) {\n                // write to WASM file using FS_createDataFile\n                // if the file exists, delete it\n                try {\n                    Module.FS_unlink(fname);\n                } catch (e) {\n                    // ignore\n                }\n\n                Module.FS_createDataFile(\"/\", fname, buf, true, true);\n\n                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);\n\n                document.getElementById('model-whisper-status').innerHTML = 'loaded \"' + model_whisper + '\"!';\n\n                if (model_whisper != null) {\n                    document.getElementById('start').disabled = false;\n                    document.getElementById('stop' ).disabled = true;\n                }\n            }\n\n            function loadWhisper(model) {\n                let urls = {\n                    'tiny.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',\n                    'base.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',\n                    'base'   : 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin',\n\n                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',\n                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',\n                };\n\n                let sizes = {\n                    'tiny.en': 75,\n                    'base.en': 142,\n                    'base':     142,\n\n                    'tiny-en-q5_1':   31,\n                    'base-en-q5_1':   57,\n                };\n\n                let url     = urls[model];\n                let dst     = 'whisper.bin';\n                let size_mb = sizes[model];\n\n                model_whisper = model;\n\n                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';\n                document.getElementById('fetch-whisper-base-en').style.display = 'none';\n                document.getElementById('fetch-whisper-base').style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1').style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1').style.display = 'none';\n\n                document.getElementById('model-whisper-status').innerHTML = 'loading \"' + model + '\" ... ';\n\n                cbProgress = function(p) {\n                    let el = document.getElementById('fetch-whisper-progress');\n                    el.innerHTML = Math.round(100*p) + '%';\n                };\n\n                cbCancel = function() {\n                    var el;\n                    el = document.getElementById('fetch-whisper-tiny-en'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base'); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en-q5_1'); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('model-whisper-status');  if (el) el.innerHTML = '';\n                };\n\n                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);\n            }\n\n            //\n            // microphone\n            //\n\n            const kSampleRate = 16000;\n            const kRestartRecording_s = 120;\n            const kIntervalAudio_ms = 5000; // pass the recorded audio to the C++ instance at this rate\n\n            var mediaRecorder = null;\n            var doRecording = false;\n            var startTime = 0;\n\n            window.AudioContext = window.AudioContext || window.webkitAudioContext;\n            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;\n\n            function stopRecording() {\n                Module.set_status(\"paused\");\n                doRecording = false;\n                audio0 = null;\n                audio = null;\n                context = null;\n            }\n\n            function startRecording() {\n                if (!context) {\n                    context = new AudioContext({\n                        sampleRate: kSampleRate,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                }\n\n                Module.set_status(\"\");\n\n                document.getElementById('start').disabled = true;\n                document.getElementById('stop').disabled = false;\n\n                doRecording = true;\n                startTime = Date.now();\n\n                var chunks = [];\n                var stream = null;\n\n                navigator.mediaDevices.getUserMedia({audio: true, video: false})\n                    .then(function(s) {\n                        stream = s;\n                        mediaRecorder = new MediaRecorder(stream);\n                        mediaRecorder.ondataavailable = function(e) {\n                            chunks.push(e.data);\n\n                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });\n                            var reader = new FileReader();\n\n                            reader.onload = function(event) {\n                                var buf = new Uint8Array(reader.result);\n\n                                if (!context) {\n                                    return;\n                                }\n                                context.decodeAudioData(buf.buffer, function(audioBuffer) {\n                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);\n                                    var source = offlineContext.createBufferSource();\n                                    source.buffer = audioBuffer;\n                                    source.connect(offlineContext.destination);\n                                    source.start(0);\n\n                                    offlineContext.startRendering().then(function(renderedBuffer) {\n                                        audio = renderedBuffer.getChannelData(0);\n\n                                        //printTextarea('js: audio recorded, size: ' + audio.length + ', old size: ' + (audio0 == null ? 0 : audio0.length));\n\n                                        var audioAll = new Float32Array(audio0 == null ? audio.length : audio0.length + audio.length);\n                                        if (audio0 != null) {\n                                            audioAll.set(audio0, 0);\n                                        }\n                                        audioAll.set(audio, audio0 == null ? 0 : audio0.length);\n\n                                        if (instance) {\n                                            Module.set_audio(instance, audioAll);\n                                        }\n                                    });\n                                }, function(e) {\n                                    audio = null;\n                                });\n                            }\n\n                            reader.readAsArrayBuffer(blob);\n                        };\n\n                        mediaRecorder.onstop = function(e) {\n                            if (doRecording) {\n                                setTimeout(function() {\n                                    startRecording();\n                                });\n                            }\n                        };\n\n                        mediaRecorder.start(kIntervalAudio_ms);\n                    })\n                    .catch(function(err) {\n                        printTextarea('js: error getting audio stream: ' + err);\n                    });\n\n                var interval = setInterval(function() {\n                    if (!doRecording) {\n                        clearInterval(interval);\n                        mediaRecorder.stop();\n                        stream.getTracks().forEach(function(track) {\n                            track.stop();\n                        });\n\n                        document.getElementById('start').disabled = false;\n                        document.getElementById('stop').disabled  = true;\n\n                        mediaRecorder = null;\n                    }\n\n                    // if audio length is more than kRestartRecording_s seconds, restart recording\n                    if (audio != null && audio.length > kSampleRate*kRestartRecording_s) {\n                        if (doRecording) {\n                            //printTextarea('js: restarting recording');\n\n                            clearInterval(interval);\n                            audio0 = audio;\n                            audio = null;\n                            mediaRecorder.stop();\n                            stream.getTracks().forEach(function(track) {\n                                track.stop();\n                            });\n                        }\n                    }\n                }, 100);\n            }\n\n            //\n            // main\n            //\n\n            var nLines = 0;\n            var intervalUpdate = null;\n            var transcribedAll = '';\n\n            function onStart() {\n                if (!instance) {\n                    instance = Module.init('whisper.bin', document.getElementById('language').value);\n\n                    if (instance) {\n                        printTextarea(\"js: whisper initialized, instance: \" + instance);\n                    }\n                }\n\n                if (!instance) {\n                    printTextarea(\"js: failed to initialize whisper\");\n                    return;\n                }\n\n                startRecording();\n\n                intervalUpdate = setInterval(function() {\n                    var transcribed = Module.get_transcribed();\n\n                    if (transcribed != null && transcribed.length > 1) {\n                        transcribedAll += transcribed + '<br>';\n                        nLines++;\n\n                        // if more than 10 lines, remove the first line\n                        if (nLines > 10) {\n                            var i = transcribedAll.indexOf('<br>');\n                            if (i > 0) {\n                                transcribedAll = transcribedAll.substring(i + 4);\n                                nLines--;\n                            }\n                        }\n                    }\n\n                    document.getElementById('state-status').innerHTML = Module.get_status();\n                    document.getElementById('state-transcribed').innerHTML = transcribedAll;\n                }, 100);\n            }\n\n            function onStop() {\n                stopRecording();\n            }\n\n        </script>\n        <script type=\"text/javascript\" src=\"stream.js\"></script>\n    </body>\n</html>\n"
  },
  {
    "path": "examples/sycl/CMakeLists.txt",
    "content": "#  MIT license\r\n#  Copyright (C) 2024 Intel Corporation\r\n#  SPDX-License-Identifier: MIT\r\n\r\nset(TARGET ls-sycl-device)\r\nadd_executable(${TARGET} ls-sycl-device.cpp)\r\ninstall(TARGETS ${TARGET} RUNTIME)\r\ntarget_link_libraries(${TARGET} PRIVATE common whisper ${CMAKE_THREAD_LIBS_INIT})\r\ntarget_compile_features(${TARGET} PRIVATE cxx_std_17)\r\n"
  },
  {
    "path": "examples/sycl/README.md",
    "content": "# llama.cpp/example/sycl\r\n\r\nThis example program provide the tools for llama.cpp for SYCL on Intel GPU.\r\n\r\n## Tool\r\n\r\n|Tool Name| Function|Status|\r\n|-|-|-|\r\n|ls-sycl-device| List all SYCL devices with ID, compute capability, max work group size, ect.|Support|\r\n\r\n### ls-sycl-device\r\n\r\nList all SYCL devices with ID, compute capability, max work group size, ect.\r\n\r\n1. Build the llama.cpp for SYCL for all targets.\r\n\r\n2. Enable oneAPI running environment\r\n\r\n```\r\nsource /opt/intel/oneapi/setvars.sh\r\n```\r\n\r\n3. Execute\r\n\r\n```\r\n./build/bin/ls-sycl-device\r\n```\r\n\r\nCheck the ID in startup log, like:\r\n\r\n```\r\nfound 4 SYCL devices:\r\n  Device 0: Intel(R) Arc(TM) A770 Graphics,\tcompute capability 1.3,\r\n    max compute_units 512,\tmax work group size 1024,\tmax sub group size 32,\tglobal mem size 16225243136\r\n  Device 1: Intel(R) FPGA Emulation Device,\tcompute capability 1.2,\r\n    max compute_units 24,\tmax work group size 67108864,\tmax sub group size 64,\tglobal mem size 67065057280\r\n  Device 2: 13th Gen Intel(R) Core(TM) i7-13700K,\tcompute capability 3.0,\r\n    max compute_units 24,\tmax work group size 8192,\tmax sub group size 64,\tglobal mem size 67065057280\r\n  Device 3: Intel(R) Arc(TM) A770 Graphics,\tcompute capability 3.0,\r\n    max compute_units 512,\tmax work group size 1024,\tmax sub group size 32,\tglobal mem size 16225243136\r\n\r\n```\r\n\r\n|Attribute|Note|\r\n|-|-|\r\n|compute capability 1.3|Level-zero running time, recommended |\r\n|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|"
  },
  {
    "path": "examples/sycl/build.sh",
    "content": "#  MIT license\r\n#  Copyright (C) 2024 Intel Corporation\r\n#  SPDX-License-Identifier: MIT\r\n\r\nmkdir -p build\r\ncd build\r\nsource /opt/intel/oneapi/setvars.sh\r\n\r\n#for FP16\r\n#cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DWHISPER_SYCL_F16=ON # faster for long-prompt inference\r\n\r\n#for FP32\r\ncmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx\r\n\r\n#for other features from the examples, e.g. stream and talk link with SDL2:\r\n#cmake .. -DGGML_SYCL=ON -DWHISPER_SDL2=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx\r\n\r\n#build example/main only\r\n#cmake --build . --config Release --target main\r\n\r\n#build all binary\r\ncmake --build . --config Release -v\r\n"
  },
  {
    "path": "examples/sycl/ls-sycl-device.cpp",
    "content": "/*MIT license\r\n  Copyright (C) 2024 Intel Corporation\r\n  SPDX-License-Identifier: MIT\r\n*/\r\n\r\n#include \"ggml-sycl.h\"\r\n\r\nint main(int argc, char ** argv) {\r\n    ggml_backend_sycl_print_sycl_devices();\r\n    return 0;\r\n}"
  },
  {
    "path": "examples/sycl/run-whisper.sh",
    "content": "#!/bin/bash\r\n\r\n#  MIT license\r\n#  Copyright (C) 2024 Intel Corporation\r\n#  SPDX-License-Identifier: MIT\r\n\r\nINPUT2=\"Building a website can be done in 10 simple steps:\\nStep 1:\"\r\nsource /opt/intel/oneapi/setvars.sh\r\n\r\nif [ $# -gt 0 ]; then\r\n    export GGML_SYCL_DEVICE=$1\r\nelse\r\n    export GGML_SYCL_DEVICE=0\r\nfi\r\necho GGML_SYCL_DEVICE=$GGML_SYCL_DEVICE\r\n#export GGML_SYCL_DEBUG=1\r\n./build/bin/main -m models/ggml-base.en.bin -f samples/jfk.wav"
  },
  {
    "path": "examples/talk-llama/.gitignore",
    "content": "audio.mp3\nto_speak.txt\n"
  },
  {
    "path": "examples/talk-llama/CMakeLists.txt",
    "content": "if (WHISPER_SDL2)\n    set(CMAKE_CXX_STANDARD 17)\n    set(CMAKE_CXX_STANDARD_REQUIRED ON)\n\n    file(GLOB SRC_MODELS models/*.cpp)\n\n    set(TARGET whisper-talk-llama)\n    add_executable(${TARGET} talk-llama.cpp\n        llama.cpp\n        llama-adapter.cpp\n        llama-arch.cpp\n        llama-batch.cpp\n        llama-chat.cpp\n        llama-context.cpp\n        llama-cparams.cpp\n        llama-grammar.cpp\n        llama-graph.cpp\n        llama-hparams.cpp\n        llama-impl.cpp\n        llama-io.cpp\n        llama-kv-cache.cpp\n        llama-kv-cache-iswa.cpp\n        llama-memory-recurrent.cpp\n        llama-memory-hybrid.cpp\n        llama-memory-hybrid-iswa.cpp\n        llama-memory.cpp\n        llama-mmap.cpp\n        llama-model-loader.cpp\n        llama-model-saver.cpp\n        llama-model.cpp\n        llama-quant.cpp\n        llama-sampler.cpp\n        llama-vocab.cpp\n        unicode.cpp\n        unicode-data.cpp\n        ${SRC_MODELS})\n    target_include_directories(${TARGET} PRIVATE . ${SDL2_INCLUDE_DIRS})\n\n    target_link_libraries(${TARGET} PRIVATE common common-sdl whisper ${SDL2_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})\n    install(TARGETS ${TARGET} RUNTIME)\n\n    if(WIN32)\n        # It requires Windows 8.1 or later for PrefetchVirtualMemory\n        target_compile_definitions(${TARGET} PRIVATE -D_WIN32_WINNT=0x0602)\n    endif()\n\n    include(DefaultTargetOptions)\nendif ()\n"
  },
  {
    "path": "examples/talk-llama/README.md",
    "content": "# whisper.cpp/examples/talk-llama\r\n\r\nTalk with an LLaMA AI in your terminal\r\n\r\n*Latest perf as of 2 Nov 2023 using Whisper Medium + LLaMA v2 13B Q8_0 on M2 Ultra:*\r\n\r\nhttps://github.com/ggerganov/whisper.cpp/assets/1991296/d97a3788-bf2a-4756-9a43-60c6b391649e\r\n\r\n*Previous demo running on CPUs*\r\n\r\n[Demo Talk](https://user-images.githubusercontent.com/1991296/228024237-848f998c-c334-46a6-bef8-3271590da83b.mp4)\r\n\r\n## Building\r\n\r\nThe `whisper-talk-llama` tool depends on SDL2 library to capture audio from the microphone. You can build it like this:\r\n\r\n```bash\r\n# Install SDL2\r\n# On Debian based linux distributions:\r\nsudo apt-get install libsdl2-dev\r\n\r\n# On Fedora Linux:\r\nsudo dnf install SDL2 SDL2-devel\r\n\r\n# Install SDL2 on Mac OS\r\nbrew install sdl2\r\n\r\n# Build the \"whisper-talk-llama\" executable\r\ncmake -B build -S . -DWHISPER_SDL2=ON\r\ncmake --build build --config Release\r\n\r\n# Run it\r\n./build/bin/whisper-talk-llama -mw ./models/ggml-small.en.bin -ml ../llama.cpp/models/llama-13b/ggml-model-q4_0.gguf -p \"Georgi\" -t 8\r\n```\r\n\r\n- The `-mw` argument specifies the Whisper model that you would like to use. Recommended `base` or `small` for real-time experience\r\n- The `-ml` argument specifies the LLaMA model that you would like to use. Read the instructions in https://github.com/ggerganov/llama.cpp for information about how to obtain a `ggml` compatible LLaMA model\r\n\r\n## Session\r\n\r\nThe `whisper-talk-llama` tool supports session management to enable more coherent and continuous conversations. By maintaining context from previous interactions, it can better understand and respond to user requests in a more natural way.\r\n\r\nTo enable session support, use the `--session FILE` command line option when running the program. The `whisper-talk-llama` model state will be saved to the specified file after each interaction. If the file does not exist, it will be created. If the file exists, the model state will be loaded from it, allowing you to resume a previous session.\r\n\r\nThis feature is especially helpful for maintaining context in long conversations or when interacting with the AI assistant across multiple sessions. It ensures that the assistant remembers the previous interactions and can provide more relevant and contextual responses.\r\n\r\nExample usage:\r\n\r\n```bash\r\n./build/bin/whisper-talk-llama --session ./my-session-file -mw ./models/ggml-small.en.bin -ml ../llama.cpp/models/llama-13b/ggml-model-q4_0.gguf -p \"Georgi\" -t 8\r\n```\r\n\r\n## TTS\r\n\r\nFor best experience, this example needs a TTS tool to convert the generated text responses to voice.\r\nYou can use any TTS engine that you would like - simply edit the [speak](speak) script to your needs.\r\nBy default, it is configured to use MacOS's `say` or Windows SpeechSynthesizer, but you can use whatever you wish.\r\n\r\n## Discussion\r\n\r\nIf you have any feedback, please let \"us\" know in the following discussion: https://github.com/ggerganov/whisper.cpp/discussions/672?converting=1\r\n"
  },
  {
    "path": "examples/talk-llama/eleven-labs.py",
    "content": "import sys\nimport argparse\nimport textwrap\n\nparser = argparse.ArgumentParser(add_help=False,\n    formatter_class=argparse.RawTextHelpFormatter)\nparser.add_argument(\"-q\", \"--quick\", action=\"store_true\",\n    help=\"skip checking the required library\")\n\nmodes = parser.add_argument_group(\"action\")\nmodes.add_argument(\"inputfile\", metavar=\"TEXTFILE\",\n    nargs='?', type=argparse.FileType(), default=sys.stdin,\n    help=\"read the text file (default: stdin)\")\nmodes.add_argument(\"-l\", \"--list\", action=\"store_true\",\n    help=\"show the list of voices and exit\")\nmodes.add_argument(\"-h\", \"--help\", action=\"help\",\n    help=\"show this help and exit\")\n\nselopts = parser.add_argument_group(\"voice selection\")\nselmodes = selopts.add_mutually_exclusive_group()\nselmodes.add_argument(\"-n\", \"--name\",\n    default=\"Arnold\",\n    help=\"get a voice object by name (default: Arnold)\")\nselmodes.add_argument(\"-v\", \"--voice\", type=int, metavar=\"NUMBER\",\n    help=\"get a voice object by number (see --list)\")\nselopts.add_argument(\"-f\", \"--filter\", action=\"append\", metavar=\"KEY=VAL\",\n    default=[\"use case=narration\"],\n    help=textwrap.dedent('''\\\n        filter voices by labels (default: \"use case=narration\")\n        this option can be used multiple times\n        filtering will be disabled if the first -f has no \"=\" (e.g. -f \"any\")\n        '''))\n\noutmodes = parser.add_argument_group(\"output\")\noutgroup = outmodes.add_mutually_exclusive_group()\noutgroup.add_argument(\"-s\", \"--save\", metavar=\"FILE\",\n    default=\"audio.mp3\",\n    help=\"save the TTS to a file (default: audio.mp3)\")\noutgroup.add_argument(\"-p\", \"--play\", action=\"store_true\",\n    help=\"play the TTS with ffplay\")\n\nargs = parser.parse_args()\n\nif not args.quick:\n    import importlib.util\n    if importlib.util.find_spec(\"elevenlabs\") is None:\n        print(\"elevenlabs library is not installed, you can install it to your enviroment using 'pip install elevenlabs'\")\n        sys.exit()\n\nfrom elevenlabs import voices, generate, play, save\n\nif args.filter and \"=\" in args.filter[0]:\n    voicelist = voices()\n    for f in args.filter:\n        label, value = f.split(\"=\")\n        voicelist = filter(lambda x: x.labels.get(label) == value, voicelist)\n    voicelist = list(voicelist)\nelse:\n    voicelist = list(voices())\n\nif args.list:\n    for i, v in enumerate(voicelist):\n        print(str(i) + \": \" + v.name + \" \" + str(v.labels))\n    sys.exit()\n\nif args.voice:\n    voice = voicelist[args.voice % len(voicelist)]\nelse:\n    voice = args.name\n    # if -n should consult -f, use the following\n    #voice = next(x for x in voicelist if x.name == args.name)\n\naudio = generate(\n    text=str(args.inputfile.read()),\n    voice=voice\n)\nif args.play:\n    play(audio)\nelse:\n    save(audio, args.save) \n"
  },
  {
    "path": "examples/talk-llama/llama-adapter.cpp",
    "content": "#include \"llama-adapter.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-mmap.h\"\n#include \"llama-model.h\"\n\n#include <map>\n#include <cassert>\n#include <sstream>\n#include <stdexcept>\n\n// vec\n\nggml_tensor * llama_adapter_cvec::tensor_for(int il) const {\n    if (il < 0 || il < layer_start || il > layer_end || (size_t) il >= tensors.size()) {\n        return nullptr;\n    }\n\n    return tensors[il];\n}\n\nggml_tensor * llama_adapter_cvec::apply_to(ggml_context * ctx, ggml_tensor * cur, int  il) const {\n    ggml_tensor * layer_dir = tensor_for(il);\n    if (layer_dir != nullptr) {\n        cur = ggml_add(ctx, cur, layer_dir);\n    }\n\n    return cur;\n}\n\nbool llama_adapter_cvec::init(const llama_model & model) {\n    const auto & hparams = model.hparams;\n\n    GGML_ASSERT(tensors.empty());\n    GGML_ASSERT(ctxs.empty());\n    GGML_ASSERT(bufs.empty());\n\n    // create a context for each buffer type\n    std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;\n    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {\n        auto it = ctx_map.find(buft);\n        if (it == ctx_map.end()) {\n            ggml_init_params params = {\n                /*.mem_size   =*/ hparams.n_layer*ggml_tensor_overhead(),\n                /*.mem_buffer =*/ NULL,\n                /*.no_alloc   =*/ true,\n            };\n\n            ggml_context * ctx = ggml_init(params);\n            if (!ctx) {\n                return nullptr;\n            }\n\n            ctx_map[buft] = ctx;\n            ctxs.emplace_back(ctx);\n\n            return ctx;\n        }\n\n        return it->second;\n    };\n\n    // make tensors\n    tensors.reserve(hparams.n_layer);\n    tensors.push_back(nullptr); // there's never a tensor for layer 0\n    for (size_t il = 1; il < hparams.n_layer; il++) {\n        ggml_backend_buffer_type_t buft = model.select_buft(il);\n        ggml_context * ctx = ctx_for_buft(buft);\n        if (!ctx) {\n            LLAMA_LOG_ERROR(\"%s: failed to allocate context for control vector\\n\", __func__);\n            return false;\n        }\n        ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);\n        tensors.push_back(tensor);\n    }\n\n    // allocate tensors / buffers and zero\n    bufs.reserve(ctx_map.size());\n    for (auto it : ctx_map) {\n        ggml_backend_buffer_type_t buft = it.first;\n        ggml_context * ctx = it.second;\n        ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);\n        if (!buf) {\n            LLAMA_LOG_ERROR(\"%s: failed to allocate buffer for control vector\\n\", __func__);\n            return false;\n        }\n        ggml_backend_buffer_clear(buf, 0);\n        bufs.emplace_back(buf);\n    }\n\n    return true;\n}\n\nbool llama_adapter_cvec::apply(\n        const llama_model & model,\n        const float * data,\n        size_t len,\n        int32_t n_embd,\n        int32_t il_start,\n        int32_t il_end) {\n    const auto & hparams = model.hparams;\n\n    if (data == nullptr) {\n        // disable the current control vector (but leave allocated for later)\n        layer_start = -1;\n        layer_end   = -1;\n        return true;\n    }\n\n    if (n_embd != (int) hparams.n_embd) {\n        LLAMA_LOG_ERROR(\"%s: control vector n_embd does not match model\\n\", __func__);\n        return false;\n    }\n\n    if (tensors.empty()) {\n        if (!init(model)) {\n            return false;\n        }\n    }\n\n    layer_start = il_start;\n    layer_end   = il_end;\n\n    for (size_t il = 1; il < hparams.n_layer; il++) {\n        assert(tensors[il] != nullptr);\n\n        const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present\n        if (off + n_embd <= len) {\n            ggml_backend_tensor_set(tensors[il], data + off, 0, n_embd * ggml_element_size(tensors[il]));\n        }\n    }\n\n    return true;\n}\n\n// lora\n\nllama_adapter_lora_weight * llama_adapter_lora::get_weight(ggml_tensor * w) {\n    const std::string name(w->name);\n\n    const auto pos = ab_map.find(name);\n    if (pos != ab_map.end()) {\n        return &pos->second;\n    }\n\n    return nullptr;\n}\n\nstatic void llama_adapter_lora_init_impl(llama_model & model, const char * path_lora, llama_adapter_lora & adapter) {\n    LLAMA_LOG_INFO(\"%s: loading lora adapter from '%s' ...\\n\", __func__, path_lora);\n\n    ggml_context * ctx_init;\n    gguf_init_params meta_gguf_params = {\n        /* .no_alloc = */ true,\n        /* .ctx      = */ &ctx_init,\n    };\n\n    gguf_context_ptr ctx_gguf { gguf_init_from_file(path_lora, meta_gguf_params) };\n    if (!ctx_gguf) {\n        throw std::runtime_error(\"failed to load lora adapter file from \" + std::string(path_lora));\n    }\n\n    ggml_context_ptr ctx { ctx_init };\n\n    // check metadata\n    {\n        const gguf_context * gguf_ctx = ctx_gguf.get();\n\n        LLAMA_LOG_INFO(\"%s: Dumping metadata keys/values.\\n\", __func__);\n\n        // get metadata as string\n        for (int i = 0; i < gguf_get_n_kv(gguf_ctx); i++) {\n            gguf_type type = gguf_get_kv_type(gguf_ctx, i);\n            const std::string type_name =\n                type == GGUF_TYPE_ARRAY\n                ? format(\"%s[%s,%zu]\", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(gguf_ctx, i)), gguf_get_arr_n(gguf_ctx, i))\n                : gguf_type_name(type);\n            const char * name = gguf_get_key(gguf_ctx, i);\n            const std::string value = gguf_kv_to_str(gguf_ctx, i);\n\n            if (type != GGUF_TYPE_ARRAY) {\n                adapter.gguf_kv.emplace(name, value);\n            }\n\n            const size_t MAX_VALUE_LEN = 40;\n            std::string print_value = value.size() > MAX_VALUE_LEN ? format(\"%s...\", value.substr(0, MAX_VALUE_LEN - 3).c_str()) : value;\n            replace_all(print_value, \"\\n\", \"\\\\n\");\n\n            LLAMA_LOG_INFO(\"%s: - kv %3d: %42s %-16s = %s\\n\", __func__, i, name, type_name.c_str(), print_value.c_str());\n        }\n\n        auto get_kv_str = [&](const std::string & key) -> std::string {\n            int id = gguf_find_key(gguf_ctx, key.c_str());\n            return id < 0 ? \"\" : std::string(gguf_get_val_str(gguf_ctx, id));\n        };\n        auto get_kv_f32 = [&](const std::string & key) -> float {\n            int id = gguf_find_key(gguf_ctx, key.c_str());\n            return id < 0 ? 0.0f : gguf_get_val_f32(gguf_ctx, id);\n        };\n        LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN);\n\n        auto general_type = get_kv_str(llm_kv(LLM_KV_GENERAL_TYPE));\n        if (general_type != \"adapter\") {\n            throw std::runtime_error(\"expect general.type to be 'adapter', but got: \" + general_type);\n        }\n\n        auto general_arch_str = get_kv_str(llm_kv(LLM_KV_GENERAL_ARCHITECTURE));\n        auto general_arch = llm_arch_from_string(general_arch_str);\n        if (general_arch != model.arch) {\n            throw std::runtime_error(\"model arch and LoRA arch mismatch\");\n        }\n\n        auto adapter_type = get_kv_str(llm_kv(LLM_KV_ADAPTER_TYPE));\n        if (adapter_type != \"lora\") {\n            throw std::runtime_error(\"expect adapter.type to be 'lora', but got: \" + adapter_type);\n        }\n\n        adapter.alpha = get_kv_f32(llm_kv(LLM_KV_ADAPTER_LORA_ALPHA));\n\n        // parse alora invocation sequence vector\n        const auto & key = llm_kv(LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS);\n        const int kid = gguf_find_key(ctx_gguf.get(), key.c_str());\n        if (kid >= 0) {\n            if (gguf_get_kv_type(ctx_gguf.get(), kid) != GGUF_TYPE_ARRAY) {\n                throw std::runtime_error(\"invalid gguf type for \" + key);\n            }\n            const auto arr_type = gguf_get_arr_type(ctx_gguf.get(), kid);\n            if (arr_type != GGUF_TYPE_UINT32) {\n                throw std::runtime_error(\"invalid gguf element type for \" + key);\n            }\n            const size_t seq_len = gguf_get_arr_n(ctx_gguf.get(), kid);\n            const void * data = gguf_get_arr_data(ctx_gguf.get(), kid);\n            adapter.alora_invocation_tokens.resize(seq_len);\n            std::copy(\n                (const llama_token *)data,\n                (const llama_token *)data + seq_len,\n                adapter.alora_invocation_tokens.begin());\n        }\n    }\n\n    int n_tensors = gguf_get_n_tensors(ctx_gguf.get());\n\n    // contexts for each buffer type\n    std::map<ggml_backend_buffer_type_t, ggml_context *> ctx_map;\n    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {\n        auto it = ctx_map.find(buft);\n        if (it == ctx_map.end()) {\n            // add a new context\n            ggml_init_params params = {\n                /*.mem_size   =*/ n_tensors*ggml_tensor_overhead(),\n                /*.mem_buffer =*/ NULL,\n                /*.no_alloc   =*/ true,\n            };\n            ggml_context * buft_ctx = ggml_init(params);\n            if (!buft_ctx) {\n                return nullptr;\n            }\n            ctx_map[buft] = buft_ctx;\n            adapter.ctxs.emplace_back(buft_ctx);\n            return buft_ctx;\n        };\n        return it->second;\n    };\n\n    // bundle lora_a and lora_b into pairs\n    std::map<std::string, llama_adapter_lora_weight> ab_map;\n    auto str_endswith = [](const std::string & str, const std::string & suffix) {\n        return str.size() >= suffix.size() && str.compare(str.size()-suffix.size(), suffix.size(), suffix) == 0;\n    };\n\n    for (ggml_tensor * cur = ggml_get_first_tensor(ctx.get()); cur; cur = ggml_get_next_tensor(ctx.get(), cur)) {\n        std::string name(cur->name);\n        if (str_endswith(name, \".lora_a\")) {\n            replace_all(name, \".lora_a\", \"\");\n            if (ab_map.find(name) == ab_map.end()) {\n                ab_map[name] = llama_adapter_lora_weight(cur, nullptr);\n            } else {\n                ab_map[name].a = cur;\n            }\n        } else if (str_endswith(name, \".lora_b\")) {\n            replace_all(name, \".lora_b\", \"\");\n            if (ab_map.find(name) == ab_map.end()) {\n                ab_map[name] = llama_adapter_lora_weight(nullptr, cur);\n            } else {\n                ab_map[name].b = cur;\n            }\n        } else if (str_endswith(name, \"_norm.weight\")) {\n            // TODO: add support for norm vector\n            // for now, we don't really care because most adapters still work fine without it\n            continue;\n        } else {\n            throw std::runtime_error(\"LoRA tensor '\" + name + \"' has unexpected suffix\");\n        }\n    }\n\n    // get extra buffer types of the CPU\n    // TODO: a more general solution for non-CPU extra buft should be imlpemented in the future\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/12593#pullrequestreview-2718659948\n    std::vector<ggml_backend_buffer_type_t> buft_extra;\n    {\n        auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n        if (!cpu_dev) {\n            throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n        }\n        auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);\n\n        auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)\n            ggml_backend_reg_get_proc_address(cpu_reg, \"ggml_backend_dev_get_extra_bufts\");\n\n        if (ggml_backend_dev_get_extra_bufts_fn) {\n            ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);\n            while (extra_bufts && *extra_bufts) {\n                buft_extra.emplace_back(*extra_bufts);\n                ++extra_bufts;\n            }\n        }\n    }\n\n    // add tensors\n    for (auto & it : ab_map) {\n        const std::string & name = it.first;\n        llama_adapter_lora_weight & w = it.second;\n        bool is_token_embd = str_endswith(name, \"token_embd.weight\");\n\n        if (!w.a || !w.b) {\n            throw std::runtime_error(\"LoRA tensor pair for '\" + name + \"' is missing one component\");\n        }\n\n        // device buft and device ctx\n        const auto * model_tensor = model.get_tensor(name.c_str());\n        if (!model_tensor) {\n            throw std::runtime_error(\"LoRA tensor '\" + name + \"' does not exist in base model (hint: maybe wrong base model?)\");\n        }\n\n        auto * buft = ggml_backend_buffer_get_type(model_tensor->buffer);\n\n        // do not load loras to extra buffer types (i.e. bufts for repacking) -> use the CPU in that case\n        for (auto & ex : buft_extra) {\n            if (ex == buft) {\n                LLAMA_LOG_WARN(\"%s: lora for '%s' cannot use buft '%s', fallback to CPU\\n\", __func__, model_tensor->name, ggml_backend_buft_name(buft));\n\n                auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n                if (!cpu_dev) {\n                    throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n                }\n                buft = ggml_backend_dev_buffer_type(cpu_dev);\n\n                break;\n            }\n        }\n\n        LLAMA_LOG_DEBUG(\"%s: lora for '%s' -> '%s'\\n\", __func__, model_tensor->name, ggml_backend_buft_name(buft));\n\n        ggml_context * dev_ctx = ctx_for_buft(buft);\n        // validate tensor shape\n        if (is_token_embd) {\n            // expect B to be non-transposed, A and B are flipped; see llm_build_inp_embd()\n            if (model_tensor->ne[0] != w.b->ne[1] || model_tensor->ne[1] != w.a->ne[1]) {\n                throw std::runtime_error(\"tensor '\" + name + \"' has incorrect shape (hint: maybe wrong base model?)\");\n            }\n        } else {\n            if (model_tensor->ne[0] != w.a->ne[0] || model_tensor->ne[1] != w.b->ne[1]) {\n                throw std::runtime_error(\"tensor '\" + name + \"' has incorrect shape (hint: maybe wrong base model?)\");\n            }\n            if (w.a->ne[1] != w.b->ne[0]) {\n                throw std::runtime_error(\"lora_a tensor is not transposed (hint: adapter from \\\"finetune\\\" example is no longer supported)\");\n            }\n        }\n\n        // save tensor to adapter\n        ggml_tensor * tensor_a = ggml_dup_tensor(dev_ctx, w.a);\n        ggml_tensor * tensor_b = ggml_dup_tensor(dev_ctx, w.b);\n        ggml_set_name(tensor_a, w.a->name);\n        ggml_set_name(tensor_b, w.b->name);\n        adapter.ab_map[name] = llama_adapter_lora_weight(tensor_a, tensor_b);\n    }\n\n    // allocate tensors / buffers and zero\n    {\n        adapter.ctxs.reserve(ctx_map.size());\n        adapter.bufs.reserve(ctx_map.size());\n        for (auto & it : ctx_map) {\n            ggml_backend_buffer_type_t buft = it.first;\n            ggml_context * ctx_dev = it.second;\n            ggml_backend_buffer_ptr buf { ggml_backend_alloc_ctx_tensors_from_buft(ctx_dev, buft) };\n            if (!buf) {\n                throw std::runtime_error(\"failed to allocate buffer for lora adapter\\n\");\n            }\n            LLAMA_LOG_INFO(\"%s: %10s LoRA buffer size = %8.2f MiB\\n\", __func__, ggml_backend_buffer_name(buf.get()), ggml_backend_buffer_get_size(buf.get())/1024.0/1024.0);\n            adapter.bufs.emplace_back(std::move(buf));\n        }\n    }\n\n    // set tensor data\n    {\n        llama_file gguf_file(path_lora, \"rb\");\n        std::vector<uint8_t> read_buf;\n        auto set_tensor = [&](ggml_tensor * orig, ggml_tensor * dev) {\n            size_t offs = gguf_get_data_offset(ctx_gguf.get()) + gguf_get_tensor_offset(ctx_gguf.get(), gguf_find_tensor(ctx_gguf.get(), orig->name));\n            size_t size = ggml_nbytes(orig);\n            read_buf.resize(size);\n            gguf_file.seek(offs, SEEK_SET);\n            gguf_file.read_raw(read_buf.data(), size);\n            ggml_backend_tensor_set(dev, read_buf.data(), 0, size);\n        };\n        for (auto & it : adapter.ab_map) {\n            auto orig = ab_map[it.first];\n            auto dev  = it.second;\n            set_tensor(orig.a, dev.a);\n            set_tensor(orig.b, dev.b);\n        }\n    }\n\n    // register adapter with model\n    model.loras.insert(&adapter);\n\n    LLAMA_LOG_INFO(\"%s: loaded %zu tensors from lora file\\n\", __func__, adapter.ab_map.size()*2);\n}\n\nllama_adapter_lora * llama_adapter_lora_init(llama_model * model, const char * path_lora) {\n    llama_adapter_lora * adapter = new llama_adapter_lora();\n\n    try {\n        llama_adapter_lora_init_impl(*model, path_lora, *adapter);\n        return adapter;\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: failed to apply lora adapter: %s\\n\", __func__, err.what());\n\n        delete adapter;\n    }\n\n    return nullptr;\n}\n\nint32_t llama_adapter_meta_val_str(const llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size) {\n    const auto & it = adapter->gguf_kv.find(key);\n    if (it == adapter->gguf_kv.end()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    return snprintf(buf, buf_size, \"%s\", it->second.c_str());\n}\n\nint32_t llama_adapter_meta_count(const llama_adapter_lora * adapter) {\n    return (int)adapter->gguf_kv.size();\n}\n\nint32_t llama_adapter_meta_key_by_index(const llama_adapter_lora * adapter, int i, char * buf, size_t buf_size) {\n    if (i < 0 || i >= (int)adapter->gguf_kv.size()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    auto it = adapter->gguf_kv.begin();\n    std::advance(it, i);\n    return snprintf(buf, buf_size, \"%s\", it->first.c_str());\n}\n\nint32_t llama_adapter_meta_val_str_by_index(const llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size) {\n    if (i < 0 || i >= (int)adapter->gguf_kv.size()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    auto it = adapter->gguf_kv.begin();\n    std::advance(it, i);\n    return snprintf(buf, buf_size, \"%s\", it->second.c_str());\n}\n\nvoid llama_adapter_lora_free(llama_adapter_lora *) {\n    // deprecated: adapters are freed by llama_model's destructor\n}\n\nuint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter) {\n    if (!adapter) {\n        return 0;\n    }\n    return adapter->alora_invocation_tokens.size();\n}\n\nconst llama_token * llama_adapter_get_alora_invocation_tokens(const llama_adapter_lora * adapter) {\n    GGML_ASSERT(adapter);\n    return adapter->alora_invocation_tokens.data();\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-adapter.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include \"ggml-cpp.h\"\n\n#include <string>\n#include <unordered_map>\n#include <vector>\n\n// TODO: pimpl\n\n//\n// llama_adapter_cvec\n//\n\nstruct llama_adapter_cvec {\n    ggml_tensor * tensor_for(int il) const;\n\n    ggml_tensor * apply_to(ggml_context * ctx, ggml_tensor * cur, int  il) const;\n\n    bool apply(\n            const llama_model & model,\n            const float * data,\n            size_t len,\n            int32_t n_embd,\n            int32_t il_start,\n            int32_t il_end);\n\nprivate:\n    bool init(const llama_model & model);\n\n    int32_t layer_start = -1;\n    int32_t layer_end   = -1;\n\n    std::vector<ggml_context_ptr> ctxs;\n    std::vector<ggml_backend_buffer_ptr> bufs;\n\n    std::vector<ggml_tensor *> tensors; // per layer\n};\n\nusing llama_adapter_cvec_ptr = std::shared_ptr<llama_adapter_cvec>;\n\n//\n// llama_adapter_lora\n//\n\nstruct llama_adapter_lora_weight {\n    ggml_tensor * a = nullptr;\n    ggml_tensor * b = nullptr;\n\n    // get actual scale based on rank and alpha\n    float get_scale(float alpha, float adapter_scale) const {\n        const float rank  = (float) b->ne[0];\n        const float scale = alpha ? adapter_scale * alpha / rank : adapter_scale;\n        return scale;\n    }\n\n    llama_adapter_lora_weight() = default;\n    llama_adapter_lora_weight(ggml_tensor * a, ggml_tensor * b) : a(a), b(b) {}\n};\n\nstruct llama_adapter_lora {\n    // map tensor name to lora_a_b\n    std::unordered_map<std::string, llama_adapter_lora_weight> ab_map;\n\n    std::vector<ggml_context_ptr> ctxs;\n    std::vector<ggml_backend_buffer_ptr> bufs;\n\n    float alpha;\n\n    // gguf metadata\n    std::unordered_map<std::string, std::string> gguf_kv;\n\n    // activated lora (aLoRA)\n    std::vector<llama_token> alora_invocation_tokens;\n\n    llama_adapter_lora() = default;\n    ~llama_adapter_lora() = default;\n\n    llama_adapter_lora_weight * get_weight(ggml_tensor * w);\n\n    uint32_t get_n_nodes() const {\n        return ab_map.size() * 6u; // a, b, scale, add, 2 x mul_mat\n    }\n};\n\nusing llama_adapter_loras = std::unordered_map<llama_adapter_lora *, float>;\nusing llama_adapter_loras_ptr = std::unique_ptr<llama_adapter_loras>;\n"
  },
  {
    "path": "examples/talk-llama/llama-arch.cpp",
    "content": "#include \"llama-arch.h\"\n\n#include \"llama-impl.h\"\n\n#include <map>\n#include <set>\n#include <vector>\n\nstatic const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {\n    { LLM_ARCH_CLIP,             \"clip\"             }, // dummy, only used by llama-quantize\n    { LLM_ARCH_LLAMA,            \"llama\"            },\n    { LLM_ARCH_LLAMA4,           \"llama4\"           },\n    { LLM_ARCH_DECI,             \"deci\"             },\n    { LLM_ARCH_FALCON,           \"falcon\"           },\n    { LLM_ARCH_GROK,             \"grok\"             },\n    { LLM_ARCH_GPT2,             \"gpt2\"             },\n    { LLM_ARCH_GPTJ,             \"gptj\"             },\n    { LLM_ARCH_GPTNEOX,          \"gptneox\"          },\n    { LLM_ARCH_MPT,              \"mpt\"              },\n    { LLM_ARCH_BAICHUAN,         \"baichuan\"         },\n    { LLM_ARCH_STARCODER,        \"starcoder\"        },\n    { LLM_ARCH_REFACT,           \"refact\"           },\n    { LLM_ARCH_BERT,             \"bert\"             },\n    { LLM_ARCH_MODERN_BERT,      \"modern-bert\"      },\n    { LLM_ARCH_NOMIC_BERT,       \"nomic-bert\"       },\n    { LLM_ARCH_NOMIC_BERT_MOE,   \"nomic-bert-moe\"   },\n    { LLM_ARCH_NEO_BERT,         \"neo-bert\"         },\n    { LLM_ARCH_JINA_BERT_V2,     \"jina-bert-v2\"     },\n    { LLM_ARCH_JINA_BERT_V3,     \"jina-bert-v3\"     },\n    { LLM_ARCH_EUROBERT,         \"eurobert\"         },\n    { LLM_ARCH_BLOOM,            \"bloom\"            },\n    { LLM_ARCH_STABLELM,         \"stablelm\"         },\n    { LLM_ARCH_QWEN,             \"qwen\"             },\n    { LLM_ARCH_QWEN2,            \"qwen2\"            },\n    { LLM_ARCH_QWEN2MOE,         \"qwen2moe\"         },\n    { LLM_ARCH_QWEN2VL,          \"qwen2vl\"          },\n    { LLM_ARCH_QWEN3,            \"qwen3\"            },\n    { LLM_ARCH_QWEN3MOE,         \"qwen3moe\"         },\n    { LLM_ARCH_QWEN3NEXT,        \"qwen3next\"        },\n    { LLM_ARCH_QWEN3VL,          \"qwen3vl\"          },\n    { LLM_ARCH_QWEN3VLMOE,       \"qwen3vlmoe\"       },\n    { LLM_ARCH_QWEN35,           \"qwen35\"           },\n    { LLM_ARCH_QWEN35MOE,        \"qwen35moe\"        },\n    { LLM_ARCH_PHI2,             \"phi2\"             },\n    { LLM_ARCH_PHI3,             \"phi3\"             },\n    { LLM_ARCH_PHIMOE,           \"phimoe\"           },\n    { LLM_ARCH_PLAMO,            \"plamo\"            },\n    { LLM_ARCH_PLAMO2,           \"plamo2\"           },\n    { LLM_ARCH_PLAMO3,           \"plamo3\"           },\n    { LLM_ARCH_CODESHELL,        \"codeshell\"        },\n    { LLM_ARCH_ORION,            \"orion\"            },\n    { LLM_ARCH_INTERNLM2,        \"internlm2\"        },\n    { LLM_ARCH_MINICPM,          \"minicpm\"          },\n    { LLM_ARCH_MINICPM3,         \"minicpm3\"         },\n    { LLM_ARCH_GEMMA,            \"gemma\"            },\n    { LLM_ARCH_GEMMA2,           \"gemma2\"           },\n    { LLM_ARCH_GEMMA3,           \"gemma3\"           },\n    { LLM_ARCH_GEMMA3N,          \"gemma3n\"          },\n    { LLM_ARCH_GEMMA_EMBEDDING,  \"gemma-embedding\"  },\n    { LLM_ARCH_STARCODER2,       \"starcoder2\"       },\n    { LLM_ARCH_MAMBA,            \"mamba\"            },\n    { LLM_ARCH_MAMBA2,           \"mamba2\"           },\n    { LLM_ARCH_JAMBA,            \"jamba\"            },\n    { LLM_ARCH_FALCON_H1,        \"falcon-h1\"        },\n    { LLM_ARCH_XVERSE,           \"xverse\"           },\n    { LLM_ARCH_COMMAND_R,        \"command-r\"        },\n    { LLM_ARCH_COHERE2,          \"cohere2\"          },\n    { LLM_ARCH_DBRX,             \"dbrx\"             },\n    { LLM_ARCH_OLMO,             \"olmo\"             },\n    { LLM_ARCH_OLMO2,            \"olmo2\"            },\n    { LLM_ARCH_OLMOE,            \"olmoe\"            },\n    { LLM_ARCH_OPENELM,          \"openelm\"          },\n    { LLM_ARCH_ARCTIC,           \"arctic\"           },\n    { LLM_ARCH_DEEPSEEK,         \"deepseek\"         },\n    { LLM_ARCH_DEEPSEEK2,        \"deepseek2\"        },\n    { LLM_ARCH_CHATGLM,          \"chatglm\"          },\n    { LLM_ARCH_GLM4,             \"glm4\"             },\n    { LLM_ARCH_GLM4_MOE,         \"glm4moe\"          },\n    { LLM_ARCH_GLM_DSA,          \"glm-dsa\"          },\n    { LLM_ARCH_BITNET,           \"bitnet\"           },\n    { LLM_ARCH_T5,               \"t5\"               },\n    { LLM_ARCH_T5ENCODER,        \"t5encoder\"        },\n    { LLM_ARCH_JAIS,             \"jais\"             },\n    { LLM_ARCH_JAIS2,            \"jais2\"            },\n    { LLM_ARCH_NEMOTRON,         \"nemotron\"         },\n    { LLM_ARCH_NEMOTRON_H,       \"nemotron_h\"       },\n    { LLM_ARCH_NEMOTRON_H_MOE,   \"nemotron_h_moe\"   },\n    { LLM_ARCH_EXAONE,           \"exaone\"           },\n    { LLM_ARCH_EXAONE4,          \"exaone4\"          },\n    { LLM_ARCH_EXAONE_MOE,       \"exaone-moe\"       },\n    { LLM_ARCH_RWKV6,            \"rwkv6\"            },\n    { LLM_ARCH_RWKV6QWEN2,       \"rwkv6qwen2\"       },\n    { LLM_ARCH_RWKV7,            \"rwkv7\"            },\n    { LLM_ARCH_ARWKV7,           \"arwkv7\"           },\n    { LLM_ARCH_GRANITE,          \"granite\"          },\n    { LLM_ARCH_GRANITE_MOE,      \"granitemoe\"       },\n    { LLM_ARCH_GRANITE_HYBRID,   \"granitehybrid\"    },\n    { LLM_ARCH_CHAMELEON,        \"chameleon\"        },\n    { LLM_ARCH_WAVTOKENIZER_DEC, \"wavtokenizer-dec\" },\n    { LLM_ARCH_PLM,              \"plm\"              },\n    { LLM_ARCH_BAILINGMOE,       \"bailingmoe\"       },\n    { LLM_ARCH_BAILINGMOE2,      \"bailingmoe2\"      },\n    { LLM_ARCH_DOTS1,            \"dots1\"            },\n    { LLM_ARCH_ARCEE,            \"arcee\"            },\n    { LLM_ARCH_AFMOE,            \"afmoe\"            },\n    { LLM_ARCH_ERNIE4_5,         \"ernie4_5\"         },\n    { LLM_ARCH_ERNIE4_5_MOE,     \"ernie4_5-moe\"     },\n    { LLM_ARCH_HUNYUAN_MOE,      \"hunyuan-moe\"      },\n    { LLM_ARCH_HUNYUAN_DENSE,    \"hunyuan-dense\"    },\n    { LLM_ARCH_SMOLLM3,          \"smollm3\"          },\n    { LLM_ARCH_OPENAI_MOE,       \"gpt-oss\"          },\n    { LLM_ARCH_LFM2,             \"lfm2\"             },\n    { LLM_ARCH_LFM2MOE,          \"lfm2moe\"          },\n    { LLM_ARCH_DREAM,            \"dream\"            },\n    { LLM_ARCH_SMALLTHINKER,     \"smallthinker\"     },\n    { LLM_ARCH_LLADA,            \"llada\"            },\n    { LLM_ARCH_LLADA_MOE,        \"llada-moe\"        },\n    { LLM_ARCH_SEED_OSS,         \"seed_oss\"         },\n    { LLM_ARCH_GROVEMOE,         \"grovemoe\"         },\n    { LLM_ARCH_APERTUS,          \"apertus\"          },\n    { LLM_ARCH_MINIMAX_M2,       \"minimax-m2\"       },\n    { LLM_ARCH_COGVLM,           \"cogvlm\"           },\n    { LLM_ARCH_RND1,             \"rnd1\"             },\n    { LLM_ARCH_PANGU_EMBED,      \"pangu-embedded\"   },\n    { LLM_ARCH_MISTRAL3,         \"mistral3\"         },\n    { LLM_ARCH_PADDLEOCR,        \"paddleocr\"        },\n    { LLM_ARCH_MIMO2,            \"mimo2\"            },\n    { LLM_ARCH_STEP35,           \"step35\"           },\n    { LLM_ARCH_LLAMA_EMBED,      \"llama-embed\"      },\n    { LLM_ARCH_MAINCODER,        \"maincoder\"        },\n    { LLM_ARCH_KIMI_LINEAR,      \"kimi-linear\"      },\n    { LLM_ARCH_UNKNOWN,          \"(unknown)\"        },\n};\n\nstatic const std::map<llm_kv, const char *> LLM_KV_NAMES = {\n    { LLM_KV_GENERAL_TYPE,                     \"general.type\"                          },\n    { LLM_KV_GENERAL_ARCHITECTURE,             \"general.architecture\"                  },\n    { LLM_KV_GENERAL_QUANTIZATION_VERSION,     \"general.quantization_version\"          },\n    { LLM_KV_GENERAL_ALIGNMENT,                \"general.alignment\"                     },\n    { LLM_KV_GENERAL_FILE_TYPE,                \"general.file_type\"                     },\n    { LLM_KV_GENERAL_SAMPLING_SEQUENCE,        \"general.sampling.sequence\"             },\n    { LLM_KV_GENERAL_SAMPLING_TOP_K,           \"general.sampling.top_k\"                },\n    { LLM_KV_GENERAL_SAMPLING_TOP_P,           \"general.sampling.top_p\"                },\n    { LLM_KV_GENERAL_SAMPLING_MIN_P,           \"general.sampling.min_p\"                },\n    { LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY, \"general.sampling.xtc_probability\"      },\n    { LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD,   \"general.sampling.xtc_threshold\"        },\n    { LLM_KV_GENERAL_SAMPLING_TEMP,            \"general.sampling.temp\"                 },\n    { LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N,  \"general.sampling.penalty_last_n\"       },\n    { LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT,  \"general.sampling.penalty_repeat\"       },\n    { LLM_KV_GENERAL_SAMPLING_MIROSTAT,        \"general.sampling.mirostat\"             },\n    { LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU,    \"general.sampling.mirostat_tau\"         },\n    { LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA,    \"general.sampling.mirostat_eta\"         },\n    { LLM_KV_GENERAL_NAME,                     \"general.name\"                          },\n    { LLM_KV_GENERAL_AUTHOR,                   \"general.author\"                        },\n    { LLM_KV_GENERAL_VERSION,                  \"general.version\"                       },\n    { LLM_KV_GENERAL_URL,                      \"general.url\"                           },\n    { LLM_KV_GENERAL_DESCRIPTION,              \"general.description\"                   },\n    { LLM_KV_GENERAL_LICENSE,                  \"general.license\"                       },\n    { LLM_KV_GENERAL_SOURCE_URL,               \"general.source.url\"                    },\n    { LLM_KV_GENERAL_SOURCE_HF_REPO,           \"general.source.huggingface.repository\" },\n\n    { LLM_KV_VOCAB_SIZE,                        \"%s.vocab_size\"                        },\n    { LLM_KV_CONTEXT_LENGTH,                    \"%s.context_length\"                    },\n    { LLM_KV_EMBEDDING_LENGTH,                  \"%s.embedding_length\"                  },\n    { LLM_KV_EMBEDDING_LENGTH_OUT,              \"%s.embedding_length_out\"              },\n    { LLM_KV_FEATURES_LENGTH,                   \"%s.features_length\"                   },\n    { LLM_KV_BLOCK_COUNT,                       \"%s.block_count\"                       },\n    { LLM_KV_LEADING_DENSE_BLOCK_COUNT,         \"%s.leading_dense_block_count\"         },\n    { LLM_KV_FEED_FORWARD_LENGTH,               \"%s.feed_forward_length\"               },\n    { LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        \"%s.expert_feed_forward_length\"        },\n    { LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, \"%s.expert_shared_feed_forward_length\" },\n    { LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH,  \"%s.expert_chunk_feed_forward_length\"  },\n    { LLM_KV_SWIGLU_CLAMP_EXP,                  \"%s.swiglu_clamp_exp\"                  },\n    { LLM_KV_SWIGLU_CLAMP_SHEXP,                \"%s.swiglu_clamp_shexp\"                },\n    { LLM_KV_USE_PARALLEL_RESIDUAL,             \"%s.use_parallel_residual\"             },\n    { LLM_KV_TENSOR_DATA_LAYOUT,                \"%s.tensor_data_layout\"                },\n    { LLM_KV_EXPERT_COUNT,                      \"%s.expert_count\"                      },\n    { LLM_KV_EXPERT_USED_COUNT,                 \"%s.expert_used_count\"                 },\n    { LLM_KV_EXPERT_SHARED_COUNT,               \"%s.expert_shared_count\"               },\n    { LLM_KV_EXPERT_GROUP_COUNT,                \"%s.expert_group_count\"                },\n    { LLM_KV_EXPERT_GROUP_USED_COUNT,           \"%s.expert_group_used_count\"           },\n    { LLM_KV_EXPERT_WEIGHTS_SCALE,              \"%s.expert_weights_scale\"              },\n    { LLM_KV_EXPERT_WEIGHTS_NORM,               \"%s.expert_weights_norm\"               },\n    { LLM_KV_EXPERT_GATING_FUNC,                \"%s.expert_gating_func\"                },\n    { LLM_KV_EXPERT_GROUP_SCALE,                \"%s.expert_group_scale\"                },\n    { LLM_KV_EXPERTS_PER_GROUP,                 \"%s.experts_per_group\"                 },\n    { LLM_KV_MOE_EVERY_N_LAYERS,                \"%s.moe_every_n_layers\"                },\n    { LLM_KV_MOE_LATENT_SIZE,                   \"%s.moe_latent_size\"                   },\n    { LLM_KV_NEXTN_PREDICT_LAYERS,              \"%s.nextn_predict_layers\"              },\n    { LLM_KV_NUM_DEEPSTACK_LAYERS,              \"%s.n_deepstack_layers\"                },\n    { LLM_KV_POOLING_TYPE,                      \"%s.pooling_type\"                      },\n    { LLM_KV_LOGIT_SCALE,                       \"%s.logit_scale\"                       },\n    { LLM_KV_DECODER_START_TOKEN_ID,            \"%s.decoder_start_token_id\"            },\n    { LLM_KV_DECODER_BLOCK_COUNT,               \"%s.decoder_block_count\"               },\n    { LLM_KV_ATTN_LOGIT_SOFTCAPPING,            \"%s.attn_logit_softcapping\"            },\n    { LLM_KV_ROUTER_LOGIT_SOFTCAPPING,          \"%s.router_logit_softcapping\"          },\n    { LLM_KV_FINAL_LOGIT_SOFTCAPPING,           \"%s.final_logit_softcapping\"           },\n    { LLM_KV_SWIN_NORM,                         \"%s.swin_norm\"                         },\n    { LLM_KV_RESCALE_EVERY_N_LAYERS,            \"%s.rescale_every_n_layers\"            },\n    { LLM_KV_TIME_MIX_EXTRA_DIM,                \"%s.time_mix_extra_dim\"                },\n    { LLM_KV_TIME_DECAY_EXTRA_DIM,              \"%s.time_decay_extra_dim\"              },\n    { LLM_KV_RESIDUAL_SCALE,                    \"%s.residual_scale\"                    },\n    { LLM_KV_EMBEDDING_SCALE,                   \"%s.embedding_scale\"                   },\n    { LLM_KV_TOKEN_SHIFT_COUNT,                 \"%s.token_shift_count\"                 },\n    { LLM_KV_INTERLEAVE_MOE_LAYER_STEP,         \"%s.interleave_moe_layer_step\"         },\n    { LLM_KV_FULL_ATTENTION_INTERVAL,           \"%s.full_attention_interval\"           },\n\n    { LLM_KV_ATTENTION_HEAD_COUNT,                   \"%s.attention.head_count\"                   },\n    { LLM_KV_ATTENTION_HEAD_COUNT_KV,                \"%s.attention.head_count_kv\"                },\n    { LLM_KV_ATTENTION_MAX_ALIBI_BIAS,               \"%s.attention.max_alibi_bias\"               },\n    { LLM_KV_ATTENTION_CLAMP_KQV,                    \"%s.attention.clamp_kqv\"                    },\n    { LLM_KV_ATTENTION_KEY_LENGTH,                   \"%s.attention.key_length\"                   },\n    { LLM_KV_ATTENTION_VALUE_LENGTH,                 \"%s.attention.value_length\"                 },\n    { LLM_KV_ATTENTION_LAYERNORM_EPS,                \"%s.attention.layer_norm_epsilon\"           },\n    { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,            \"%s.attention.layer_norm_rms_epsilon\"       },\n    { LLM_KV_ATTENTION_GROUPNORM_EPS,                \"%s.attention.group_norm_epsilon\"           },\n    { LLM_KV_ATTENTION_GROUPNORM_GROUPS,             \"%s.attention.group_norm_groups\"            },\n    { LLM_KV_ATTENTION_CAUSAL,                       \"%s.attention.causal\"                       },\n    { LLM_KV_ATTENTION_Q_LORA_RANK,                  \"%s.attention.q_lora_rank\"                  },\n    { LLM_KV_ATTENTION_KV_LORA_RANK,                 \"%s.attention.kv_lora_rank\"                 },\n    { LLM_KV_ATTENTION_DECAY_LORA_RANK,              \"%s.attention.decay_lora_rank\"              },\n    { LLM_KV_ATTENTION_ICLR_LORA_RANK,               \"%s.attention.iclr_lora_rank\"               },\n    { LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, \"%s.attention.value_residual_mix_lora_rank\" },\n    { LLM_KV_ATTENTION_GATE_LORA_RANK,               \"%s.attention.gate_lora_rank\"               },\n    { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,       \"%s.attention.relative_buckets_count\"       },\n    { LLM_KV_ATTENTION_SLIDING_WINDOW,               \"%s.attention.sliding_window\"               },\n    { LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,       \"%s.attention.sliding_window_pattern\"       },\n    { LLM_KV_ATTENTION_SCALE,                        \"%s.attention.scale\"                        },\n    { LLM_KV_ATTENTION_OUTPUT_SCALE,                 \"%s.attention.output_scale\"                 },\n    { LLM_KV_ATTENTION_TEMPERATURE_LENGTH,           \"%s.attention.temperature_length\"           },\n    { LLM_KV_ATTENTION_TEMPERATURE_SCALE,            \"%s.attention.temperature_scale\"            },\n    { LLM_KV_ATTENTION_KEY_LENGTH_MLA,               \"%s.attention.key_length_mla\"               },\n    { LLM_KV_ATTENTION_VALUE_LENGTH_MLA,             \"%s.attention.value_length_mla\"             },\n    { LLM_KV_ATTENTION_KEY_LENGTH_SWA,               \"%s.attention.key_length_swa\"               },\n    { LLM_KV_ATTENTION_VALUE_LENGTH_SWA,             \"%s.attention.value_length_swa\"             },\n    { LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,           \"%s.attention.indexer.head_count\"           },\n    { LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,           \"%s.attention.indexer.key_length\"           },\n    { LLM_KV_ATTENTION_INDEXER_TOP_K,                \"%s.attention.indexer.top_k\"                },\n\n    { LLM_KV_ROPE_DIMENSION_COUNT,           \"%s.rope.dimension_count\"                 },\n    { LLM_KV_ROPE_DIMENSION_COUNT_SWA,       \"%s.rope.dimension_count_swa\"             },\n    { LLM_KV_ROPE_DIMENSION_SECTIONS,        \"%s.rope.dimension_sections\"              },\n    { LLM_KV_ROPE_FREQ_BASE,                 \"%s.rope.freq_base\"                       },\n    { LLM_KV_ROPE_FREQ_BASE_SWA,             \"%s.rope.freq_base_swa\"                   },\n    { LLM_KV_ROPE_SCALE_LINEAR,              \"%s.rope.scale_linear\"                    },\n    { LLM_KV_ROPE_SCALING_TYPE,              \"%s.rope.scaling.type\"                    },\n    { LLM_KV_ROPE_SCALING_FACTOR,            \"%s.rope.scaling.factor\"                  },\n    { LLM_KV_ROPE_SCALING_ATTN_FACTOR,       \"%s.rope.scaling.attn_factor\"             },\n    { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,      \"%s.rope.scaling.original_context_length\" },\n    { LLM_KV_ROPE_SCALING_FINETUNED,         \"%s.rope.scaling.finetuned\"               },\n    { LLM_KV_ROPE_SCALING_YARN_LOG_MUL,      \"%s.rope.scaling.yarn_log_multiplier\"     },\n    { LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR,   \"%s.rope.scaling.yarn_ext_factor\"         },\n    { LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR,  \"%s.rope.scaling.yarn_attn_factor\"        },\n    { LLM_KV_ROPE_SCALING_YARN_BETA_FAST,    \"%s.rope.scaling.yarn_beta_fast\"          },\n    { LLM_KV_ROPE_SCALING_YARN_BETA_SLOW,    \"%s.rope.scaling.yarn_beta_slow\"          },\n\n    { LLM_KV_SPLIT_NO,            \"split.no\"            },\n    { LLM_KV_SPLIT_COUNT,         \"split.count\"         },\n    { LLM_KV_SPLIT_TENSORS_COUNT, \"split.tensors.count\" },\n\n    { LLM_KV_SSM_CONV_KERNEL,    \"%s.ssm.conv_kernel\"    },\n    { LLM_KV_SSM_INNER_SIZE,     \"%s.ssm.inner_size\"     },\n    { LLM_KV_SSM_STATE_SIZE,     \"%s.ssm.state_size\"     },\n    { LLM_KV_SSM_TIME_STEP_RANK, \"%s.ssm.time_step_rank\" },\n    { LLM_KV_SSM_GROUP_COUNT,    \"%s.ssm.group_count\"    },\n    { LLM_KV_SSM_DT_B_C_RMS,     \"%s.ssm.dt_b_c_rms\"     },\n\n    { LLM_KV_KDA_HEAD_DIM, \"%s.kda.head_dim\" },\n\n    { LLM_KV_WKV_HEAD_SIZE, \"%s.wkv.head_size\" },\n\n    { LLM_KV_POSNET_EMBEDDING_LENGTH, \"%s.posnet.embedding_length\" },\n    { LLM_KV_POSNET_BLOCK_COUNT,      \"%s.posnet.block_count\"      },\n\n    { LLM_KV_CONVNEXT_EMBEDDING_LENGTH, \"%s.convnext.embedding_length\" },\n    { LLM_KV_CONVNEXT_BLOCK_COUNT,      \"%s.convnext.block_count\"      },\n\n    { LLM_KV_CLASSIFIER_OUTPUT_LABELS, \"%s.classifier.output_labels\" },\n\n    { LLM_KV_SHORTCONV_L_CACHE, \"%s.shortconv.l_cache\" },\n    // sentence-transformers dense modules feature dims\n    { LLM_KV_DENSE_2_FEAT_IN,        \"%s.dense_2_feat_in\"  },\n    { LLM_KV_DENSE_2_FEAT_OUT,       \"%s.dense_2_feat_out\"  },\n    { LLM_KV_DENSE_3_FEAT_IN,        \"%s.dense_3_feat_in\"   },\n    { LLM_KV_DENSE_3_FEAT_OUT,       \"%s.dense_3_feat_out\"  },\n\n    { LLM_KV_TOKENIZER_MODEL,                \"tokenizer.ggml.model\"                    },\n    { LLM_KV_TOKENIZER_PRE,                  \"tokenizer.ggml.pre\"                      },\n    { LLM_KV_TOKENIZER_LIST,                 \"tokenizer.ggml.tokens\"                   },\n    { LLM_KV_TOKENIZER_TOKEN_TYPE,           \"tokenizer.ggml.token_type\"               },\n    { LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,     \"tokenizer.ggml.token_type_count\"         },\n    { LLM_KV_TOKENIZER_SCORES,               \"tokenizer.ggml.scores\"                   },\n    { LLM_KV_TOKENIZER_MERGES,               \"tokenizer.ggml.merges\"                   },\n    { LLM_KV_TOKENIZER_BOS_ID,               \"tokenizer.ggml.bos_token_id\"             },\n    { LLM_KV_TOKENIZER_EOS_ID,               \"tokenizer.ggml.eos_token_id\"             },\n    { LLM_KV_TOKENIZER_EOT_ID,               \"tokenizer.ggml.eot_token_id\"             },\n    { LLM_KV_TOKENIZER_EOM_ID,               \"tokenizer.ggml.eom_token_id\"             },\n    { LLM_KV_TOKENIZER_UNK_ID,               \"tokenizer.ggml.unknown_token_id\"         },\n    { LLM_KV_TOKENIZER_SEP_ID,               \"tokenizer.ggml.seperator_token_id\"       },\n    { LLM_KV_TOKENIZER_PAD_ID,               \"tokenizer.ggml.padding_token_id\"         },\n    { LLM_KV_TOKENIZER_CLS_ID,               \"tokenizer.ggml.cls_token_id\"             },\n    { LLM_KV_TOKENIZER_MASK_ID,              \"tokenizer.ggml.mask_token_id\"            },\n    { LLM_KV_TOKENIZER_ADD_BOS,              \"tokenizer.ggml.add_bos_token\"            },\n    { LLM_KV_TOKENIZER_ADD_EOS,              \"tokenizer.ggml.add_eos_token\"            },\n    { LLM_KV_TOKENIZER_ADD_SEP,              \"tokenizer.ggml.add_sep_token\"            },\n    { LLM_KV_TOKENIZER_ADD_PREFIX,           \"tokenizer.ggml.add_space_prefix\"         },\n    { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,      \"tokenizer.ggml.remove_extra_whitespaces\" },\n    { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, \"tokenizer.ggml.precompiled_charsmap\"     },\n    { LLM_KV_TOKENIZER_HF_JSON,              \"tokenizer.huggingface.json\"              },\n    { LLM_KV_TOKENIZER_RWKV,                 \"tokenizer.rwkv.world\"                    },\n    { LLM_KV_TOKENIZER_CHAT_TEMPLATE,        \"tokenizer.chat_template\"                 },\n    { LLM_KV_TOKENIZER_FIM_PRE_ID,           \"tokenizer.ggml.fim_pre_token_id\"         },\n    { LLM_KV_TOKENIZER_FIM_SUF_ID,           \"tokenizer.ggml.fim_suf_token_id\"         },\n    { LLM_KV_TOKENIZER_FIM_MID_ID,           \"tokenizer.ggml.fim_mid_token_id\"         },\n    { LLM_KV_TOKENIZER_FIM_PAD_ID,           \"tokenizer.ggml.fim_pad_token_id\"         },\n    { LLM_KV_TOKENIZER_FIM_REP_ID,           \"tokenizer.ggml.fim_rep_token_id\"         },\n    { LLM_KV_TOKENIZER_FIM_SEP_ID,           \"tokenizer.ggml.fim_sep_token_id\"         },\n\n    { LLM_KV_ADAPTER_TYPE,                    \"adapter.type\"               },\n    { LLM_KV_ADAPTER_LORA_ALPHA,              \"adapter.lora.alpha\"         },\n    { LLM_KV_ADAPTER_LORA_TASK_NAME,          \"adapter.lora.task_name\"     },\n    { LLM_KV_ADAPTER_LORA_PROMPT_PREFIX,      \"adapter.lora.prompt_prefix\" },\n    { LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS, \"adapter.alora.invocation_tokens\" },\n\n    { LLM_KV_XIELU_ALPHA_N,         \"xielu.alpha_n\"         },\n    { LLM_KV_XIELU_ALPHA_P,         \"xielu.alpha_p\"         },\n    { LLM_KV_XIELU_BETA,            \"xielu.beta\"            },\n    { LLM_KV_XIELU_EPS,             \"xielu.eps\"             },\n\n    // deprecated\n    { LLM_KV_TOKENIZER_PREFIX_ID, \"tokenizer.ggml.prefix_token_id\" },\n    { LLM_KV_TOKENIZER_SUFFIX_ID, \"tokenizer.ggml.suffix_token_id\" },\n    { LLM_KV_TOKENIZER_MIDDLE_ID, \"tokenizer.ggml.middle_token_id\" },\n};\n\nstatic const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {\n    { LLM_TENSOR_TOKEN_EMBD,                             \"token_embd\" },\n    { LLM_TENSOR_OUTPUT_NORM,                            \"output_norm\" },\n    { LLM_TENSOR_OUTPUT_NORM_LFM2,                       \"token_embd_norm\" }, // fix for wrong tensor name\n    { LLM_TENSOR_OUTPUT,                                 \"output\" },\n    { LLM_TENSOR_ROPE_FREQS,                             \"rope_freqs\" },\n    { LLM_TENSOR_ATTN_NORM,                              \"blk.%d.attn_norm\" },\n    { LLM_TENSOR_ATTN_Q,                                 \"blk.%d.attn_q\" },\n    { LLM_TENSOR_ATTN_K,                                 \"blk.%d.attn_k\" },\n    { LLM_TENSOR_ATTN_V,                                 \"blk.%d.attn_v\" },\n    { LLM_TENSOR_ATTN_OUT,                               \"blk.%d.attn_output\" },\n    { LLM_TENSOR_ATTN_ROT_EMBD,                          \"blk.%d.attn_rot_embd\" },\n    { LLM_TENSOR_FFN_GATE_INP,                           \"blk.%d.ffn_gate_inp\" },\n    { LLM_TENSOR_FFN_NORM,                               \"blk.%d.ffn_norm\" },\n    { LLM_TENSOR_FFN_GATE,                               \"blk.%d.ffn_gate\" },\n    { LLM_TENSOR_FFN_DOWN,                               \"blk.%d.ffn_down\" },\n    { LLM_TENSOR_FFN_UP,                                 \"blk.%d.ffn_up\" },\n    { LLM_TENSOR_FFN_GATE_EXP,                           \"blk.%d.ffn_gate.%d\" },\n    { LLM_TENSOR_FFN_DOWN_EXP,                           \"blk.%d.ffn_down.%d\" },\n    { LLM_TENSOR_FFN_UP_EXP,                             \"blk.%d.ffn_up.%d\" },\n    { LLM_TENSOR_FFN_GATE_EXPS,                          \"blk.%d.ffn_gate_exps\" },\n    { LLM_TENSOR_FFN_GATE_UP_EXPS,                       \"blk.%d.ffn_gate_up_exps\" },\n    { LLM_TENSOR_FFN_DOWN_EXPS,                          \"blk.%d.ffn_down_exps\" },\n    { LLM_TENSOR_FFN_UP_EXPS,                            \"blk.%d.ffn_up_exps\" },\n    { LLM_TENSOR_ATTN_POST_NORM,                         \"blk.%d.post_attention_norm\" },\n    { LLM_TENSOR_ATTN_Q_NORM,                            \"blk.%d.attn_q_norm\" },\n    { LLM_TENSOR_ATTN_K_NORM,                            \"blk.%d.attn_k_norm\" },\n    { LLM_TENSOR_ATTN_GATE,                              \"blk.%d.attn_gate\" },\n    { LLM_TENSOR_FFN_POST_NORM,                          \"blk.%d.post_ffw_norm\" },\n    { LLM_TENSOR_FFN_GATE_SHEXP,                         \"blk.%d.ffn_gate_shexp\" },\n    { LLM_TENSOR_FFN_UP_SHEXP,                           \"blk.%d.ffn_up_shexp\" },\n    { LLM_TENSOR_FFN_DOWN_SHEXP,                         \"blk.%d.ffn_down_shexp\" },\n    { LLM_TENSOR_FFN_EXP_PROBS_B,                        \"blk.%d.exp_probs_b\" },\n    { LLM_TENSOR_FFN_LATENT_DOWN,                        \"blk.%d.ffn_latent_down\" },\n    { LLM_TENSOR_FFN_LATENT_UP,                          \"blk.%d.ffn_latent_up\" },\n    { LLM_TENSOR_ATTN_NORM_2,                            \"blk.%d.attn_norm_2\" },\n    { LLM_TENSOR_ATTN_QKV,                               \"blk.%d.attn_qkv\" },\n    { LLM_TENSOR_LAYER_OUT_NORM,                         \"blk.%d.layer_output_norm\" },\n    { LLM_TENSOR_ATTN_OUT_NORM,                          \"blk.%d.attn_output_norm\" },\n    { LLM_TENSOR_POS_EMBD,                               \"position_embd\" },\n    { LLM_TENSOR_FFN_ACT,                                \"blk.%d.ffn.act\" },\n    { LLM_TENSOR_TOKEN_EMBD_NORM,                        \"token_embd_norm\" },\n    { LLM_TENSOR_TOKEN_TYPES,                            \"token_types\" },\n    { LLM_TENSOR_CLS,                                    \"cls\" },\n    { LLM_TENSOR_CLS_OUT,                                \"cls.output\" },\n    { LLM_TENSOR_CLS_NORM,                               \"cls.norm\" },\n    { LLM_TENSOR_ENC_OUTPUT_NORM,                        \"enc.output_norm\" },\n    { LLM_TENSOR_FFN_GATE_INP_SHEXP,                     \"blk.%d.ffn_gate_inp_shexp\" },\n    { LLM_TENSOR_SSM_A_NOSCAN,                           \"blk.%d.ssm_a\" },\n    { LLM_TENSOR_SSM_CONV1D,                             \"blk.%d.ssm_conv1d\" },\n    { LLM_TENSOR_SSM_DT,                                 \"blk.%d.ssm_dt\" },\n    { LLM_TENSOR_SSM_BETA_ALPHA,                         \"blk.%d.ssm_ba\" },\n    { LLM_TENSOR_SSM_ALPHA,                              \"blk.%d.ssm_alpha\" },\n    { LLM_TENSOR_SSM_IN,                                 \"blk.%d.ssm_in\" },\n    { LLM_TENSOR_SSM_NORM,                               \"blk.%d.ssm_norm\" },\n    { LLM_TENSOR_SSM_OUT,                                \"blk.%d.ssm_out\" },\n    { LLM_TENSOR_ROPE_FACTORS_LONG,                      \"rope_factors_long\" },\n    { LLM_TENSOR_ROPE_FACTORS_SHORT,                     \"rope_factors_short\" },\n    { LLM_TENSOR_SSM_X,                                  \"blk.%d.ssm_x\" },\n    { LLM_TENSOR_SSM_A,                                  \"blk.%d.ssm_a\" },\n    { LLM_TENSOR_SSM_D,                                  \"blk.%d.ssm_d\" },\n    { LLM_TENSOR_SSM_DT_NORM,                            \"blk.%d.ssm_dt_norm\" },\n    { LLM_TENSOR_SSM_B_NORM,                             \"blk.%d.ssm_b_norm\" },\n    { LLM_TENSOR_SSM_C_NORM,                             \"blk.%d.ssm_c_norm\" },\n    { LLM_TENSOR_SSM_CONV1D_Q,                           \"blk.%d.ssm_conv1d_q\" },\n    { LLM_TENSOR_SSM_CONV1D_K,                           \"blk.%d.ssm_conv1d_k\" },\n    { LLM_TENSOR_SSM_CONV1D_V,                           \"blk.%d.ssm_conv1d_v\" },\n    { LLM_TENSOR_SSM_F_A,                                \"blk.%d.ssm_f_a\" },\n    { LLM_TENSOR_SSM_F_B,                                \"blk.%d.ssm_f_b\" },\n    { LLM_TENSOR_SSM_BETA,                               \"blk.%d.ssm_beta\" },\n    { LLM_TENSOR_SSM_G_A,                                \"blk.%d.ssm_g_a\" },\n    { LLM_TENSOR_SSM_G_B,                                \"blk.%d.ssm_g_b\" },\n    { LLM_TENSOR_SSM_NORM,                               \"blk.%d.ssm_norm\" },\n    { LLM_TENSOR_ATTN_Q_A_NORM,                          \"blk.%d.attn_q_a_norm\" },\n    { LLM_TENSOR_ATTN_KV_A_NORM,                         \"blk.%d.attn_kv_a_norm\" },\n    { LLM_TENSOR_ATTN_Q_A,                               \"blk.%d.attn_q_a\" },\n    { LLM_TENSOR_ATTN_Q_B,                               \"blk.%d.attn_q_b\" },\n    { LLM_TENSOR_ATTN_KV_A_MQA,                          \"blk.%d.attn_kv_a_mqa\" },\n    { LLM_TENSOR_ATTN_KV_B,                              \"blk.%d.attn_kv_b\" },\n    { LLM_TENSOR_PER_LAYER_TOKEN_EMBD,                   \"per_layer_token_embd\" },\n    { LLM_TENSOR_PER_LAYER_MODEL_PROJ,                   \"per_layer_model_proj\" },\n    { LLM_TENSOR_PER_LAYER_PROJ_NORM,                    \"per_layer_proj_norm\" },\n    { LLM_TENSOR_ALTUP_UNEMBD_PROJ,                      \"altup_unembd_proj\" },\n    { LLM_TENSOR_ALTUP_PROJ,                             \"altup_proj\" },\n    { LLM_TENSOR_PER_LAYER_INP_GATE,                     \"blk.%d.inp_gate\" },\n    { LLM_TENSOR_PER_LAYER_PROJ,                         \"blk.%d.proj\" },\n    { LLM_TENSOR_PER_LAYER_POST_NORM,                    \"blk.%d.post_norm\" },\n    { LLM_TENSOR_ALTUP_CORRECT_COEF,                     \"blk.%d.altup_correct_coef\" },\n    { LLM_TENSOR_ALTUP_CORRECT_SCALE,                    \"blk.%d.altup_correct_scale\" },\n    { LLM_TENSOR_ALTUP_PREDICT_COEF,                     \"blk.%d.altup_predict_coef\" },\n    { LLM_TENSOR_ALTUP_ROUTER,                           \"blk.%d.altup_router\" },\n    { LLM_TENSOR_ALTUP_ROUTER_NORM,                      \"blk.%d.altup_router_norm\" },\n    { LLM_TENSOR_LAUREL_L,                               \"blk.%d.laurel_l\" },\n    { LLM_TENSOR_LAUREL_R,                               \"blk.%d.laurel_r\" },\n    { LLM_TENSOR_LAUREL_POST_NORM,                       \"blk.%d.laurel_post_norm\" },\n    { LLM_TENSOR_DENSE_2_OUT,                            \"dense_2\" },\n    { LLM_TENSOR_DENSE_3_OUT,                            \"dense_3\" },\n    { LLM_TENSOR_FFN_NORM_EXPS,                          \"blk.%d.ffn_norm_exps\" },\n    { LLM_TENSOR_ATTN_K_B,                               \"blk.%d.attn_k_b\" },\n    { LLM_TENSOR_ATTN_V_B,                               \"blk.%d.attn_v_b\" },\n    { LLM_TENSOR_NEXTN_EH_PROJ,                          \"blk.%d.nextn.eh_proj\" },\n    { LLM_TENSOR_NEXTN_EMBED_TOKENS,                     \"blk.%d.nextn.embed_tokens\" },\n    { LLM_TENSOR_NEXTN_ENORM,                            \"blk.%d.nextn.enorm\" },\n    { LLM_TENSOR_NEXTN_HNORM,                            \"blk.%d.nextn.hnorm\" },\n    { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,                 \"blk.%d.nextn.shared_head_head\" },\n    { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,                 \"blk.%d.nextn.shared_head_norm\" },\n    { LLM_TENSOR_ATTN_SUB_NORM,                          \"blk.%d.attn_sub_norm\" },\n    { LLM_TENSOR_FFN_SUB_NORM,                           \"blk.%d.ffn_sub_norm\" },\n    { LLM_TENSOR_DEC_OUTPUT_NORM,                        \"dec.output_norm\" },\n    { LLM_TENSOR_DEC_ATTN_NORM,                          \"dec.blk.%d.attn_norm\" },\n    { LLM_TENSOR_DEC_ATTN_Q,                             \"dec.blk.%d.attn_q\" },\n    { LLM_TENSOR_DEC_ATTN_K,                             \"dec.blk.%d.attn_k\" },\n    { LLM_TENSOR_DEC_ATTN_V,                             \"dec.blk.%d.attn_v\" },\n    { LLM_TENSOR_DEC_ATTN_OUT,                           \"dec.blk.%d.attn_o\" },\n    { LLM_TENSOR_DEC_ATTN_REL_B,                         \"dec.blk.%d.attn_rel_b\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_NORM,                    \"dec.blk.%d.cross_attn_norm\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_Q,                       \"dec.blk.%d.cross_attn_q\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_K,                       \"dec.blk.%d.cross_attn_k\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_V,                       \"dec.blk.%d.cross_attn_v\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_OUT,                     \"dec.blk.%d.cross_attn_o\" },\n    { LLM_TENSOR_DEC_CROSS_ATTN_REL_B,                   \"dec.blk.%d.cross_attn_rel_b\" },\n    { LLM_TENSOR_DEC_FFN_NORM,                           \"dec.blk.%d.ffn_norm\" },\n    { LLM_TENSOR_DEC_FFN_GATE,                           \"dec.blk.%d.ffn_gate\" },\n    { LLM_TENSOR_DEC_FFN_DOWN,                           \"dec.blk.%d.ffn_down\" },\n    { LLM_TENSOR_DEC_FFN_UP,                             \"dec.blk.%d.ffn_up\" },\n    { LLM_TENSOR_ENC_ATTN_NORM,                          \"enc.blk.%d.attn_norm\" },\n    { LLM_TENSOR_ENC_ATTN_Q,                             \"enc.blk.%d.attn_q\" },\n    { LLM_TENSOR_ENC_ATTN_K,                             \"enc.blk.%d.attn_k\" },\n    { LLM_TENSOR_ENC_ATTN_V,                             \"enc.blk.%d.attn_v\" },\n    { LLM_TENSOR_ENC_ATTN_OUT,                           \"enc.blk.%d.attn_o\" },\n    { LLM_TENSOR_ENC_ATTN_REL_B,                         \"enc.blk.%d.attn_rel_b\" },\n    { LLM_TENSOR_ENC_FFN_NORM,                           \"enc.blk.%d.ffn_norm\" },\n    { LLM_TENSOR_ENC_FFN_GATE,                           \"enc.blk.%d.ffn_gate\" },\n    { LLM_TENSOR_ENC_FFN_DOWN,                           \"enc.blk.%d.ffn_down\" },\n    { LLM_TENSOR_ENC_FFN_UP,                             \"enc.blk.%d.ffn_up\" },\n    { LLM_TENSOR_TIME_MIX_W1,                            \"blk.%d.time_mix_w1\" },\n    { LLM_TENSOR_TIME_MIX_W2,                            \"blk.%d.time_mix_w2\" },\n    { LLM_TENSOR_TIME_MIX_LERP_X,                        \"blk.%d.time_mix_lerp_x\" },\n    { LLM_TENSOR_TIME_MIX_LERP_W,                        \"blk.%d.time_mix_lerp_w\" },\n    { LLM_TENSOR_TIME_MIX_LERP_K,                        \"blk.%d.time_mix_lerp_k\" },\n    { LLM_TENSOR_TIME_MIX_LERP_V,                        \"blk.%d.time_mix_lerp_v\" },\n    { LLM_TENSOR_TIME_MIX_LERP_R,                        \"blk.%d.time_mix_lerp_r\" },\n    { LLM_TENSOR_TIME_MIX_LERP_G,                        \"blk.%d.time_mix_lerp_g\" },\n    { LLM_TENSOR_TIME_MIX_LERP_FUSED,                    \"blk.%d.time_mix_lerp_fused\" },\n    { LLM_TENSOR_TIME_MIX_FIRST,                         \"blk.%d.time_mix_first\" },\n    { LLM_TENSOR_TIME_MIX_DECAY,                         \"blk.%d.time_mix_decay\" },\n    { LLM_TENSOR_TIME_MIX_DECAY_W1,                      \"blk.%d.time_mix_decay_w1\" },\n    { LLM_TENSOR_TIME_MIX_DECAY_W2,                      \"blk.%d.time_mix_decay_w2\" },\n    { LLM_TENSOR_TIME_MIX_KEY,                           \"blk.%d.time_mix_key\" },\n    { LLM_TENSOR_TIME_MIX_VALUE,                         \"blk.%d.time_mix_value\" },\n    { LLM_TENSOR_TIME_MIX_RECEPTANCE,                    \"blk.%d.time_mix_receptance\" },\n    { LLM_TENSOR_TIME_MIX_GATE,                          \"blk.%d.time_mix_gate\" },\n    { LLM_TENSOR_TIME_MIX_LN,                            \"blk.%d.time_mix_ln\" },\n    { LLM_TENSOR_TIME_MIX_OUTPUT,                        \"blk.%d.time_mix_output\" },\n    { LLM_TENSOR_CHANNEL_MIX_LERP_K,                     \"blk.%d.channel_mix_lerp_k\" },\n    { LLM_TENSOR_CHANNEL_MIX_LERP_R,                     \"blk.%d.channel_mix_lerp_r\" },\n    { LLM_TENSOR_CHANNEL_MIX_KEY,                        \"blk.%d.channel_mix_key\" },\n    { LLM_TENSOR_CHANNEL_MIX_VALUE,                      \"blk.%d.channel_mix_value\" },\n    { LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,                 \"blk.%d.channel_mix_receptance\" },\n    { LLM_TENSOR_TIME_MIX_W0,                            \"blk.%d.time_mix_w0\" },\n    { LLM_TENSOR_TIME_MIX_A0,                            \"blk.%d.time_mix_a0\" },\n    { LLM_TENSOR_TIME_MIX_A1,                            \"blk.%d.time_mix_a1\" },\n    { LLM_TENSOR_TIME_MIX_A2,                            \"blk.%d.time_mix_a2\" },\n    { LLM_TENSOR_TIME_MIX_V0,                            \"blk.%d.time_mix_v0\" },\n    { LLM_TENSOR_TIME_MIX_V1,                            \"blk.%d.time_mix_v1\" },\n    { LLM_TENSOR_TIME_MIX_V2,                            \"blk.%d.time_mix_v2\" },\n    { LLM_TENSOR_TIME_MIX_G1,                            \"blk.%d.time_mix_g1\" },\n    { LLM_TENSOR_TIME_MIX_G2,                            \"blk.%d.time_mix_g2\" },\n    { LLM_TENSOR_TIME_MIX_K_K,                           \"blk.%d.time_mix_k_k\" },\n    { LLM_TENSOR_TIME_MIX_K_A,                           \"blk.%d.time_mix_k_a\" },\n    { LLM_TENSOR_TIME_MIX_R_K,                           \"blk.%d.time_mix_r_k\" },\n    { LLM_TENSOR_CONV1D,                                 \"conv1d\" },\n    { LLM_TENSOR_CONVNEXT_DW,                            \"convnext.%d.dw\" },\n    { LLM_TENSOR_CONVNEXT_NORM,                          \"convnext.%d.norm\" },\n    { LLM_TENSOR_CONVNEXT_PW1,                           \"convnext.%d.pw1\" },\n    { LLM_TENSOR_CONVNEXT_PW2,                           \"convnext.%d.pw2\" },\n    { LLM_TENSOR_CONVNEXT_GAMMA,                         \"convnext.%d.gamma\" },\n    { LLM_TENSOR_POS_NET_CONV1,                          \"posnet.%d.conv1\" },\n    { LLM_TENSOR_POS_NET_CONV2,                          \"posnet.%d.conv2\" },\n    { LLM_TENSOR_POS_NET_NORM,                           \"posnet.%d.norm\" },\n    { LLM_TENSOR_POS_NET_NORM1,                          \"posnet.%d.norm1\" },\n    { LLM_TENSOR_POS_NET_NORM2,                          \"posnet.%d.norm2\" },\n    { LLM_TENSOR_POS_NET_ATTN_NORM,                      \"posnet.%d.attn_norm\" },\n    { LLM_TENSOR_POS_NET_ATTN_Q,                         \"posnet.%d.attn_q\" },\n    { LLM_TENSOR_POS_NET_ATTN_K,                         \"posnet.%d.attn_k\" },\n    { LLM_TENSOR_POS_NET_ATTN_V,                         \"posnet.%d.attn_v\" },\n    { LLM_TENSOR_POS_NET_ATTN_OUT,                       \"posnet.%d.attn_output\" },\n    { LLM_TENSOR_ATTN_SINKS,                             \"blk.%d.attn_sinks\" },\n    { LLM_TENSOR_SHORTCONV_CONV,                         \"blk.%d.shortconv.conv\" },\n    { LLM_TENSOR_SHORTCONV_INPROJ,                       \"blk.%d.shortconv.in_proj\" },\n    { LLM_TENSOR_SHORTCONV_OUTPROJ,                      \"blk.%d.shortconv.out_proj\" },\n    { LLM_TENSOR_FFN_GATE_CHEXPS,                        \"blk.%d.ffn_gate_chexps\" },\n    { LLM_TENSOR_FFN_DOWN_CHEXPS,                        \"blk.%d.ffn_down_chexps\" },\n    { LLM_TENSOR_FFN_UP_CHEXPS,                          \"blk.%d.ffn_up_chexps\" },\n    { LLM_TENSOR_VISEXP_ATTN_QKV,                        \"blk.%d.vis_attn_qkv\" },\n    { LLM_TENSOR_VISEXP_ATTN_OUT,                        \"blk.%d.vis_attn_output\" },\n    { LLM_TENSOR_VISEXP_FFN_GATE,                        \"blk.%d.vis_gate\" },\n    { LLM_TENSOR_VISEXP_FFN_DOWN,                        \"blk.%d.vis_down\" },\n    { LLM_TENSOR_VISEXP_FFN_UP,                          \"blk.%d.vis_up\" },\n    { LLM_TENSOR_INDEXER_K_NORM,                         \"blk.%d.indexer.k_norm\" },\n    { LLM_TENSOR_INDEXER_PROJ,                           \"blk.%d.indexer.proj\" },\n    { LLM_TENSOR_INDEXER_ATTN_K,                         \"blk.%d.indexer.attn_k\" },\n    { LLM_TENSOR_INDEXER_ATTN_Q_B,                       \"blk.%d.indexer.attn_q_b\" },\n};\n\nstatic std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {\n    switch (arch) {\n        case LLM_ARCH_CLIP:\n            return {};\n        case LLM_ARCH_LLAMA:\n        case LLM_ARCH_DECI:\n        case LLM_ARCH_MISTRAL3:\n        case LLM_ARCH_LLAMA_EMBED:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXP,\n                LLM_TENSOR_FFN_DOWN_EXP,\n                LLM_TENSOR_FFN_UP_EXP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_ARCEE:\n        case LLM_ARCH_STARCODER2:\n        case LLM_ARCH_NEMOTRON:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_AFMOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_GATE,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_LLAMA4:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXP,\n                LLM_TENSOR_FFN_DOWN_EXP,\n                LLM_TENSOR_FFN_UP_EXP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_BAICHUAN:\n        case LLM_ARCH_ORION:\n        case LLM_ARCH_XVERSE:\n        case LLM_ARCH_EXAONE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_FALCON:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_NORM_2,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_GROK:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXP,\n                LLM_TENSOR_FFN_DOWN_EXP,\n                LLM_TENSOR_FFN_UP_EXP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_LAYER_OUT_NORM,\n                LLM_TENSOR_ATTN_OUT_NORM,\n            };\n        case LLM_ARCH_GPT2:\n        case LLM_ARCH_STARCODER:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_POS_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_GPTNEOX:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_MPT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_ACT,\n                LLM_TENSOR_POS_EMBD,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n            };\n        case LLM_ARCH_REFACT:\n        case LLM_ARCH_QWEN2:\n        case LLM_ARCH_QWEN2VL:\n        case LLM_ARCH_INTERNLM2:\n        case LLM_ARCH_GRANITE:\n        case LLM_ARCH_ERNIE4_5:\n        case LLM_ARCH_PADDLEOCR:\n        case LLM_ARCH_SMOLLM3:\n        case LLM_ARCH_DREAM:\n        case LLM_ARCH_LLADA:\n        case LLM_ARCH_PANGU_EMBED:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_BERT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_TOKEN_TYPES,\n                LLM_TENSOR_POS_EMBD,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_LAYER_OUT_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_CLS,\n                LLM_TENSOR_CLS_OUT,\n            };\n        case LLM_ARCH_NOMIC_BERT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_TOKEN_TYPES,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_LAYER_OUT_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_NOMIC_BERT_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_TOKEN_TYPES,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_LAYER_OUT_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_NEO_BERT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_ENC_OUTPUT_NORM,\n                LLM_TENSOR_CLS,\n                LLM_TENSOR_CLS_OUT,\n            };\n        case LLM_ARCH_EUROBERT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_MODERN_BERT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_CLS,\n                LLM_TENSOR_CLS_OUT,\n                LLM_TENSOR_CLS_NORM,\n            };\n        case LLM_ARCH_JINA_BERT_V2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_TOKEN_TYPES,\n                LLM_TENSOR_ATTN_NORM_2,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_LAYER_OUT_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_CLS,\n            };\n        case LLM_ARCH_JINA_BERT_V3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_TOKEN_TYPES,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_LAYER_OUT_NORM,\n            };\n        case LLM_ARCH_BLOOM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_STABLELM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n            };\n        case LLM_ARCH_QWEN:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_QWEN2MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_QWEN3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_CLS_OUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_QWEN3MOE:\n        case LLM_ARCH_QWEN3VLMOE:\n        case LLM_ARCH_OLMOE:\n        case LLM_ARCH_LLADA_MOE:\n        case LLM_ARCH_RND1:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_QWEN3NEXT:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_GATE,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_SSM_A_NOSCAN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_BETA_ALPHA,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n            };\n        case LLM_ARCH_QWEN35:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_GATE,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_SSM_A_NOSCAN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_BETA,\n                LLM_TENSOR_SSM_ALPHA,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n            };\n        case LLM_ARCH_QWEN35MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_GATE,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_SSM_A_NOSCAN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_BETA,\n                LLM_TENSOR_SSM_ALPHA,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n            };\n        case LLM_ARCH_QWEN3VL:\n        case LLM_ARCH_CHAMELEON:\n        case LLM_ARCH_HUNYUAN_DENSE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_CLS_OUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_PHI2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_PHI3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FACTORS_LONG,\n                LLM_TENSOR_ROPE_FACTORS_SHORT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_PHIMOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FACTORS_LONG,\n                LLM_TENSOR_ROPE_FACTORS_SHORT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_PLAMO:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_PLAMO2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_X,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_OUT,\n                LLM_TENSOR_SSM_DT_NORM,\n                LLM_TENSOR_SSM_B_NORM,\n                LLM_TENSOR_SSM_C_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_POST_NORM,\n            };\n        case LLM_ARCH_PLAMO3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_CODESHELL:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_MINICPM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ROPE_FACTORS_LONG,\n                LLM_TENSOR_ROPE_FACTORS_SHORT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXP,\n                LLM_TENSOR_FFN_DOWN_EXP,\n                LLM_TENSOR_FFN_UP_EXP,\n            };\n        case LLM_ARCH_MINICPM3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FACTORS_LONG,\n                LLM_TENSOR_ROPE_FACTORS_SHORT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q_A_NORM,\n                LLM_TENSOR_ATTN_KV_A_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_A,\n                LLM_TENSOR_ATTN_Q_B,\n                LLM_TENSOR_ATTN_KV_A_MQA,\n                LLM_TENSOR_ATTN_KV_B,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_GEMMA:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_GEMMA2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_POST_NORM,\n            };\n        case LLM_ARCH_GEMMA3:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_POST_NORM,\n            };\n        case LLM_ARCH_GEMMA3N:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_PER_LAYER_TOKEN_EMBD,\n                LLM_TENSOR_PER_LAYER_MODEL_PROJ,\n                LLM_TENSOR_PER_LAYER_PROJ_NORM,\n                LLM_TENSOR_ALTUP_UNEMBD_PROJ,\n                LLM_TENSOR_ALTUP_PROJ,\n                LLM_TENSOR_PER_LAYER_INP_GATE,\n                LLM_TENSOR_PER_LAYER_PROJ,\n                LLM_TENSOR_PER_LAYER_POST_NORM,\n                LLM_TENSOR_ALTUP_CORRECT_COEF,\n                LLM_TENSOR_ALTUP_CORRECT_SCALE,\n                LLM_TENSOR_ALTUP_PREDICT_COEF,\n                LLM_TENSOR_ALTUP_ROUTER,\n                LLM_TENSOR_ALTUP_ROUTER_NORM,\n                LLM_TENSOR_LAUREL_L,\n                LLM_TENSOR_LAUREL_R,\n                LLM_TENSOR_LAUREL_POST_NORM,\n            };\n        case LLM_ARCH_GEMMA_EMBEDDING:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_DENSE_2_OUT,\n                LLM_TENSOR_DENSE_3_OUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_POST_NORM,\n            };\n        case LLM_ARCH_MAMBA:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_X,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_OUT,\n            };\n        case LLM_ARCH_MAMBA2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n            };\n        case LLM_ARCH_JAMBA:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_X,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_DT_NORM,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_B_NORM,\n                LLM_TENSOR_SSM_C_NORM,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_OUT,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_FALCON_H1:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_COMMAND_R:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n            };\n        case LLM_ARCH_COHERE2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_DBRX:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_OUT_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_OLMO:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_OLMO2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_OPENELM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_ARCTIC:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_NORM_EXPS,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_DEEPSEEK:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_ROT_EMBD,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_DEEPSEEK2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q_A_NORM,\n                LLM_TENSOR_ATTN_KV_A_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_A,\n                LLM_TENSOR_ATTN_Q_B,\n                LLM_TENSOR_ATTN_KV_A_MQA,\n                LLM_TENSOR_ATTN_KV_B,\n                LLM_TENSOR_ATTN_K_B,\n                LLM_TENSOR_ATTN_V_B,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_PLM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_KV_A_MQA,\n                LLM_TENSOR_ATTN_KV_A_NORM,\n                LLM_TENSOR_ATTN_KV_B,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_CHATGLM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_GLM4:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_POST_NORM,\n                LLM_TENSOR_NEXTN_EH_PROJ,\n                LLM_TENSOR_NEXTN_EMBED_TOKENS,\n                LLM_TENSOR_NEXTN_ENORM,\n                LLM_TENSOR_NEXTN_HNORM,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n            };\n        case LLM_ARCH_GLM4_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                LLM_TENSOR_NEXTN_EH_PROJ,\n                LLM_TENSOR_NEXTN_EMBED_TOKENS,\n                LLM_TENSOR_NEXTN_ENORM,\n                LLM_TENSOR_NEXTN_HNORM,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n            };\n        case LLM_ARCH_GLM_DSA:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q_A_NORM,\n                LLM_TENSOR_ATTN_KV_A_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_A,\n                LLM_TENSOR_ATTN_Q_B,\n                LLM_TENSOR_ATTN_KV_A_MQA,\n                LLM_TENSOR_ATTN_KV_B,\n                LLM_TENSOR_ATTN_K_B,\n                LLM_TENSOR_ATTN_V_B,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                LLM_TENSOR_INDEXER_K_NORM,\n                LLM_TENSOR_INDEXER_PROJ,\n                LLM_TENSOR_INDEXER_ATTN_K,\n                LLM_TENSOR_INDEXER_ATTN_Q_B,\n                LLM_TENSOR_NEXTN_EH_PROJ,\n                LLM_TENSOR_NEXTN_EMBED_TOKENS,\n                LLM_TENSOR_NEXTN_ENORM,\n                LLM_TENSOR_NEXTN_HNORM,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n            };\n        case LLM_ARCH_BITNET:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_SUB_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_SUB_NORM,\n            };\n        case LLM_ARCH_T5:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_DEC_OUTPUT_NORM,\n                LLM_TENSOR_DEC_ATTN_NORM,\n                LLM_TENSOR_DEC_ATTN_Q,\n                LLM_TENSOR_DEC_ATTN_K,\n                LLM_TENSOR_DEC_ATTN_V,\n                LLM_TENSOR_DEC_ATTN_OUT,\n                LLM_TENSOR_DEC_ATTN_REL_B,\n                LLM_TENSOR_DEC_CROSS_ATTN_NORM,\n                LLM_TENSOR_DEC_CROSS_ATTN_Q,\n                LLM_TENSOR_DEC_CROSS_ATTN_K,\n                LLM_TENSOR_DEC_CROSS_ATTN_V,\n                LLM_TENSOR_DEC_CROSS_ATTN_OUT,\n                LLM_TENSOR_DEC_CROSS_ATTN_REL_B,\n                LLM_TENSOR_DEC_FFN_NORM,\n                LLM_TENSOR_DEC_FFN_GATE,\n                LLM_TENSOR_DEC_FFN_DOWN,\n                LLM_TENSOR_DEC_FFN_UP,\n                LLM_TENSOR_ENC_OUTPUT_NORM,\n                LLM_TENSOR_ENC_ATTN_NORM,\n                LLM_TENSOR_ENC_ATTN_Q,\n                LLM_TENSOR_ENC_ATTN_K,\n                LLM_TENSOR_ENC_ATTN_V,\n                LLM_TENSOR_ENC_ATTN_OUT,\n                LLM_TENSOR_ENC_ATTN_REL_B,\n                LLM_TENSOR_ENC_FFN_NORM,\n                LLM_TENSOR_ENC_FFN_GATE,\n                LLM_TENSOR_ENC_FFN_DOWN,\n                LLM_TENSOR_ENC_FFN_UP,\n            };\n        case LLM_ARCH_T5ENCODER:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ENC_OUTPUT_NORM,\n                LLM_TENSOR_ENC_ATTN_NORM,\n                LLM_TENSOR_ENC_ATTN_Q,\n                LLM_TENSOR_ENC_ATTN_K,\n                LLM_TENSOR_ENC_ATTN_V,\n                LLM_TENSOR_ENC_ATTN_OUT,\n                LLM_TENSOR_ENC_ATTN_REL_B,\n                LLM_TENSOR_ENC_FFN_NORM,\n                LLM_TENSOR_ENC_FFN_GATE,\n                LLM_TENSOR_ENC_FFN_DOWN,\n                LLM_TENSOR_ENC_FFN_UP,\n            };\n        case LLM_ARCH_JAIS:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_JAIS2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n            };\n        case LLM_ARCH_NEMOTRON_H:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_NEMOTRON_H_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                // mamba(2) ssm layers\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n                // attention layers\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                // dense FFN\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                // MoE FFN (for MoE layers)\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                LLM_TENSOR_FFN_LATENT_DOWN,\n                LLM_TENSOR_FFN_LATENT_UP,\n                // MoE shared expert layer\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_EXAONE4:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_POST_NORM,\n            };\n        case LLM_ARCH_EXAONE_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                LLM_TENSOR_NEXTN_EH_PROJ,\n                LLM_TENSOR_NEXTN_EMBED_TOKENS,\n                LLM_TENSOR_NEXTN_ENORM,\n                LLM_TENSOR_NEXTN_HNORM,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n            };\n        case LLM_ARCH_RWKV6:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_NORM_2,\n                LLM_TENSOR_TIME_MIX_W1,\n                LLM_TENSOR_TIME_MIX_W2,\n                LLM_TENSOR_TIME_MIX_LERP_X,\n                LLM_TENSOR_TIME_MIX_LERP_W,\n                LLM_TENSOR_TIME_MIX_LERP_K,\n                LLM_TENSOR_TIME_MIX_LERP_V,\n                LLM_TENSOR_TIME_MIX_LERP_R,\n                LLM_TENSOR_TIME_MIX_LERP_G,\n                LLM_TENSOR_TIME_MIX_LERP_FUSED,\n                LLM_TENSOR_TIME_MIX_FIRST,\n                LLM_TENSOR_TIME_MIX_DECAY,\n                LLM_TENSOR_TIME_MIX_DECAY_W1,\n                LLM_TENSOR_TIME_MIX_DECAY_W2,\n                LLM_TENSOR_TIME_MIX_KEY,\n                LLM_TENSOR_TIME_MIX_VALUE,\n                LLM_TENSOR_TIME_MIX_RECEPTANCE,\n                LLM_TENSOR_TIME_MIX_GATE,\n                LLM_TENSOR_TIME_MIX_LN,\n                LLM_TENSOR_TIME_MIX_OUTPUT,\n                LLM_TENSOR_CHANNEL_MIX_LERP_K,\n                LLM_TENSOR_CHANNEL_MIX_LERP_R,\n                LLM_TENSOR_CHANNEL_MIX_KEY,\n                LLM_TENSOR_CHANNEL_MIX_VALUE,\n                LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,\n            };\n        case LLM_ARCH_RWKV6QWEN2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_TIME_MIX_W1,\n                LLM_TENSOR_TIME_MIX_W2,\n                LLM_TENSOR_TIME_MIX_LERP_X,\n                LLM_TENSOR_TIME_MIX_LERP_FUSED,\n                LLM_TENSOR_TIME_MIX_FIRST,\n                LLM_TENSOR_TIME_MIX_DECAY,\n                LLM_TENSOR_TIME_MIX_DECAY_W1,\n                LLM_TENSOR_TIME_MIX_DECAY_W2,\n                LLM_TENSOR_TIME_MIX_KEY,\n                LLM_TENSOR_TIME_MIX_VALUE,\n                LLM_TENSOR_TIME_MIX_RECEPTANCE,\n                LLM_TENSOR_TIME_MIX_GATE,\n                LLM_TENSOR_TIME_MIX_OUTPUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_RWKV7:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_NORM_2,\n                LLM_TENSOR_TIME_MIX_W0,\n                LLM_TENSOR_TIME_MIX_W1,\n                LLM_TENSOR_TIME_MIX_W2,\n                LLM_TENSOR_TIME_MIX_A0,\n                LLM_TENSOR_TIME_MIX_A1,\n                LLM_TENSOR_TIME_MIX_A2,\n                LLM_TENSOR_TIME_MIX_V0,\n                LLM_TENSOR_TIME_MIX_V1,\n                LLM_TENSOR_TIME_MIX_V2,\n                LLM_TENSOR_TIME_MIX_G1,\n                LLM_TENSOR_TIME_MIX_G2,\n                LLM_TENSOR_TIME_MIX_K_K,\n                LLM_TENSOR_TIME_MIX_K_A,\n                LLM_TENSOR_TIME_MIX_R_K,\n                LLM_TENSOR_TIME_MIX_LERP_FUSED,\n                LLM_TENSOR_TIME_MIX_KEY,\n                LLM_TENSOR_TIME_MIX_VALUE,\n                LLM_TENSOR_TIME_MIX_RECEPTANCE,\n                LLM_TENSOR_TIME_MIX_LN,\n                LLM_TENSOR_TIME_MIX_OUTPUT,\n                LLM_TENSOR_CHANNEL_MIX_LERP_K,\n                LLM_TENSOR_CHANNEL_MIX_KEY,\n                LLM_TENSOR_CHANNEL_MIX_VALUE,\n            };\n        case LLM_ARCH_ARWKV7:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_TIME_MIX_W0,\n                LLM_TENSOR_TIME_MIX_W1,\n                LLM_TENSOR_TIME_MIX_W2,\n                LLM_TENSOR_TIME_MIX_A0,\n                LLM_TENSOR_TIME_MIX_A1,\n                LLM_TENSOR_TIME_MIX_A2,\n                LLM_TENSOR_TIME_MIX_V0,\n                LLM_TENSOR_TIME_MIX_V1,\n                LLM_TENSOR_TIME_MIX_V2,\n                LLM_TENSOR_TIME_MIX_G1,\n                LLM_TENSOR_TIME_MIX_G2,\n                LLM_TENSOR_TIME_MIX_K_K,\n                LLM_TENSOR_TIME_MIX_K_A,\n                LLM_TENSOR_TIME_MIX_R_K,\n                LLM_TENSOR_TIME_MIX_LERP_FUSED,\n                LLM_TENSOR_TIME_MIX_KEY,\n                LLM_TENSOR_TIME_MIX_VALUE,\n                LLM_TENSOR_TIME_MIX_RECEPTANCE,\n                LLM_TENSOR_TIME_MIX_LN,\n                LLM_TENSOR_TIME_MIX_OUTPUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_GRANITE_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_GRANITE_HYBRID:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_SSM_IN,\n                LLM_TENSOR_SSM_CONV1D,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_D,\n                LLM_TENSOR_SSM_NORM,\n                LLM_TENSOR_SSM_OUT,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_WAVTOKENIZER_DEC:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_TOKEN_EMBD_NORM,\n                LLM_TENSOR_CONV1D,\n                LLM_TENSOR_CONVNEXT_DW,\n                LLM_TENSOR_CONVNEXT_NORM,\n                LLM_TENSOR_CONVNEXT_PW1,\n                LLM_TENSOR_CONVNEXT_PW2,\n                LLM_TENSOR_CONVNEXT_GAMMA,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_POS_NET_CONV1,\n                LLM_TENSOR_POS_NET_CONV2,\n                LLM_TENSOR_POS_NET_NORM,\n                LLM_TENSOR_POS_NET_NORM1,\n                LLM_TENSOR_POS_NET_NORM2,\n                LLM_TENSOR_POS_NET_ATTN_NORM,\n                LLM_TENSOR_POS_NET_ATTN_Q,\n                LLM_TENSOR_POS_NET_ATTN_K,\n                LLM_TENSOR_POS_NET_ATTN_V,\n                LLM_TENSOR_POS_NET_ATTN_OUT,\n            };\n        case LLM_ARCH_BAILINGMOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n            };\n        case LLM_ARCH_BAILINGMOE2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_NEXTN_EH_PROJ,\n                LLM_TENSOR_NEXTN_EMBED_TOKENS,\n                LLM_TENSOR_NEXTN_ENORM,\n                LLM_TENSOR_NEXTN_HNORM,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n                LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n                LLM_TENSOR_LAYER_OUT_NORM,\n            };\n        case LLM_ARCH_DOTS1:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_INP_SHEXP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_ERNIE4_5_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_HUNYUAN_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_OPENAI_MOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_SINKS,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_LFM2:\n            return {\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_SHORTCONV_CONV,\n                LLM_TENSOR_SHORTCONV_INPROJ,\n                LLM_TENSOR_SHORTCONV_OUTPROJ,\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM_LFM2,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_DENSE_2_OUT,\n            };\n        case LLM_ARCH_LFM2MOE:\n            return {\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_SHORTCONV_CONV,\n                LLM_TENSOR_SHORTCONV_INPROJ,\n                LLM_TENSOR_SHORTCONV_OUTPROJ,\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM_LFM2,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_SMALLTHINKER:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n            };\n        case LLM_ARCH_APERTUS:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_SEED_OSS:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_POST_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_GROVEMOE:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_CHEXPS,\n                LLM_TENSOR_FFN_DOWN_CHEXPS,\n                LLM_TENSOR_FFN_UP_CHEXPS,\n            };\n        case LLM_ARCH_MINIMAX_M2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_COGVLM:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_QKV,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_VISEXP_ATTN_QKV,\n                LLM_TENSOR_VISEXP_ATTN_OUT,\n                LLM_TENSOR_VISEXP_FFN_GATE,\n                LLM_TENSOR_VISEXP_FFN_DOWN,\n                LLM_TENSOR_VISEXP_FFN_UP,\n            };\n        case LLM_ARCH_MIMO2:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_SINKS,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_STEP35:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ROPE_FACTORS_LONG,\n                LLM_TENSOR_ROPE_FACTORS_SHORT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_GATE,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n            };\n        case LLM_ARCH_GPTJ:\n        case LLM_ARCH_UNKNOWN:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n            };\n        case LLM_ARCH_MAINCODER:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_Q_NORM,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_K_NORM,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n            };\n        case LLM_ARCH_KIMI_LINEAR:\n            return {\n                LLM_TENSOR_TOKEN_EMBD,\n                LLM_TENSOR_OUTPUT_NORM,\n                LLM_TENSOR_OUTPUT,\n                LLM_TENSOR_ROPE_FREQS,\n                LLM_TENSOR_ATTN_NORM,\n                LLM_TENSOR_ATTN_Q,\n                LLM_TENSOR_ATTN_K,\n                LLM_TENSOR_ATTN_V,\n                LLM_TENSOR_ATTN_OUT,\n                LLM_TENSOR_FFN_NORM,\n                // Dense FFN (layer 0 only)\n                LLM_TENSOR_FFN_GATE,\n                LLM_TENSOR_FFN_DOWN,\n                LLM_TENSOR_FFN_UP,\n                // MoE FFN (layers 1+)\n                LLM_TENSOR_FFN_GATE_INP,\n                LLM_TENSOR_FFN_GATE_EXPS,\n                LLM_TENSOR_FFN_DOWN_EXPS,\n                LLM_TENSOR_FFN_UP_EXPS,\n                LLM_TENSOR_FFN_EXP_PROBS_B,\n                // Shared experts\n                LLM_TENSOR_FFN_GATE_SHEXP,\n                LLM_TENSOR_FFN_DOWN_SHEXP,\n                LLM_TENSOR_FFN_UP_SHEXP,\n                // KDA (using SSM_ enum prefix, keeping GGUF names for backward compat)\n                LLM_TENSOR_SSM_CONV1D_Q,\n                LLM_TENSOR_SSM_CONV1D_K,\n                LLM_TENSOR_SSM_CONV1D_V,\n                LLM_TENSOR_SSM_F_A,\n                LLM_TENSOR_SSM_F_B,\n                LLM_TENSOR_SSM_BETA,\n                LLM_TENSOR_SSM_A,\n                LLM_TENSOR_SSM_G_A,\n                LLM_TENSOR_SSM_G_B,\n                LLM_TENSOR_SSM_DT,\n                LLM_TENSOR_SSM_NORM,\n                // MLA\n                LLM_TENSOR_ATTN_Q_A,\n                LLM_TENSOR_ATTN_Q_B,\n                LLM_TENSOR_ATTN_Q_A_NORM,\n                LLM_TENSOR_ATTN_KV_A_MQA,\n                LLM_TENSOR_ATTN_KV_B,\n                LLM_TENSOR_ATTN_K_B,\n                LLM_TENSOR_ATTN_V_B,\n                LLM_TENSOR_ATTN_KV_A_NORM,\n            };\n        default:\n            GGML_ABORT(\"unknown architecture for tensor mapping\");\n    }\n}\n\n// declare information about the model weight tensors:\n// - the layer in which the tensor is going to be used. this is needed in order to assign the correct buffer type for the weight\n// - the operator which is going to use the weight. this is needed to determine if the respective backend supports the operator\n//\n// for example, input layers are usually assigned to CPU/host buffer types\n//\n// a mismatch between the declared information and the actual layer/op in which the tensor is used can lead to sub-optimal\n//   assignment of the buffer types and extra overhead during computation\n// example: https://github.com/ggml-org/llama.cpp/pull/17548\n//\nstatic const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {\n    {LLM_TENSOR_TOKEN_EMBD,                 {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_POS_EMBD,                   {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_TOKEN_TYPES,                {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_TOKEN_EMBD_NORM,            {LLM_TENSOR_LAYER_INPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_OUTPUT,                     {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CLS,                        {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CLS_OUT,                    {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CLS_NORM,                   {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_DENSE_2_OUT,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output\n    {LLM_TENSOR_DENSE_3_OUT,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output\n    {LLM_TENSOR_OUTPUT_NORM,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_OUTPUT_NORM_LFM2,           {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_DEC_OUTPUT_NORM,            {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_ENC_OUTPUT_NORM,            {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_ROPE_FREQS,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},\n    {LLM_TENSOR_ROPE_FACTORS_LONG,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},\n    {LLM_TENSOR_ROPE_FACTORS_SHORT,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},\n    {LLM_TENSOR_ATTN_Q,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_K,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_V,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_QKV,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_OUT,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_GATE,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_GATE,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_DOWN,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_UP,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_DOWN_SHEXP,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_GATE_SHEXP,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_UP_SHEXP,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_Q_A,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_Q_B,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_KV_A_MQA,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_KV_B,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_K_B,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_V_B,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ATTN_SINKS,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SCALE}},\n    {LLM_TENSOR_DEC_ATTN_Q,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_ATTN_K,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_ATTN_V,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_ATTN_OUT,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_CROSS_ATTN_Q,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_CROSS_ATTN_K,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_CROSS_ATTN_V,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_CROSS_ATTN_OUT,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_FFN_GATE,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_FFN_DOWN,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_DEC_FFN_UP,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_ATTN_Q,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_ATTN_K,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_ATTN_V,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_ATTN_OUT,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_FFN_GATE,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_FFN_DOWN,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ENC_FFN_UP,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_GATE_INP_SHEXP,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_GATE_INP,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_IN,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_X,                      {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_DT,                     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_OUT,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_ALPHA,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_BETA_ALPHA,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_W1,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_W2,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_A1,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_A2,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_V1,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_V2,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_G1,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_G2,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_DECAY_W1,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_DECAY_W2,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_KEY,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_VALUE,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_RECEPTANCE,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_GATE,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_OUTPUT,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CHANNEL_MIX_KEY,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,     {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CHANNEL_MIX_VALUE,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_FFN_ACT,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_DIV}},\n    {LLM_TENSOR_SSM_CONV1D,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},\n    {LLM_TENSOR_SSM_A,                      {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_SCAN}},\n    {LLM_TENSOR_SSM_A_NOSCAN,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // a version of SSM_A used for MUL instead of SSM_SCAN\n    {LLM_TENSOR_SSM_DT_NORM,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_B_NORM,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_C_NORM,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_D,                      {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_NORM,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    // Kimi KDA - Conv tensors are 4D [d_conv, 1, d_inner, 1], reshaped to 2D at runtime\n    {LLM_TENSOR_SSM_CONV1D_Q,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_CONV1D_K,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_CONV1D_V,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SSM_F_A,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_F_B,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_BETA,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_G_A,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SSM_G_B,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_TIME_MIX_LERP_X,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_TIME_MIX_LN,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_CHANNEL_MIX_LERP_K,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_CHANNEL_MIX_LERP_R,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_TIME_MIX_K_K,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_TIME_MIX_K_A,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_TIME_MIX_R_K,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_TIME_MIX_LERP_W,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_LERP_K,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_LERP_V,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_LERP_R,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_LERP_G,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_LERP_FUSED,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_DECAY,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_W0,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_A0,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_V0,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    {LLM_TENSOR_TIME_MIX_FIRST,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_RWKV_WKV6}},\n    {LLM_TENSOR_ATTN_NORM,                  {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_NORM_2,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_OUT_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_POST_NORM,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_FFN_NORM,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_FFN_POST_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_FFN_NORM_EXPS,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_Q_NORM,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_K_NORM,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_LAYER_OUT_NORM,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_Q_A_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_KV_A_NORM,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ATTN_SUB_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_FFN_SUB_NORM,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_DEC_ATTN_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_DEC_CROSS_ATTN_NORM,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_DEC_FFN_NORM,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ENC_ATTN_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ENC_FFN_NORM,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_DEC_ATTN_REL_B,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_ENC_ATTN_REL_B,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_FFN_DOWN_EXPS,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_GATE_EXPS,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_UP_EXPS,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_GATE_UP_EXPS,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_DOWN_CHEXPS,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_GATE_CHEXPS,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_UP_CHEXPS,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},\n    {LLM_TENSOR_FFN_EXP_PROBS_B,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},\n    // altup / laurel (gemma 3n)\n    {LLM_TENSOR_PER_LAYER_TOKEN_EMBD,       {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_PER_LAYER_MODEL_PROJ,       {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_PER_LAYER_PROJ_NORM,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},\n    {LLM_TENSOR_ALTUP_PROJ,                 {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ALTUP_UNEMBD_PROJ,          {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_PER_LAYER_INP_GATE,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_PER_LAYER_PROJ,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_PER_LAYER_POST_NORM,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ALTUP_CORRECT_COEF,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ALTUP_CORRECT_SCALE,        {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_ALTUP_PREDICT_COEF,         {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ALTUP_ROUTER,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_ALTUP_ROUTER_NORM,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_LAUREL_L,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_LAUREL_R,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_LAUREL_POST_NORM,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    // this tensor is loaded for T5, but never used\n    {LLM_TENSOR_DEC_CROSS_ATTN_REL_B,       {LLM_TENSOR_LAYER_REPEATING, GGML_OP_NONE}},\n    {LLM_TENSOR_CONV1D,                     {LLM_TENSOR_LAYER_INPUT,     GGML_OP_IM2COL}},\n    {LLM_TENSOR_POS_NET_NORM,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_POS_NET_NORM1,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_POS_NET_NORM2,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_POS_NET_CONV1,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_IM2COL}},\n    {LLM_TENSOR_POS_NET_CONV2,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_IM2COL}},\n    {LLM_TENSOR_POS_NET_ATTN_NORM,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_POS_NET_ATTN_Q,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_POS_NET_ATTN_K,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_POS_NET_ATTN_V,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_POS_NET_ATTN_OUT,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CONVNEXT_DW,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_IM2COL}},\n    {LLM_TENSOR_CONVNEXT_NORM,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_CONVNEXT_PW1,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CONVNEXT_PW2,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_CONVNEXT_GAMMA,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_SHORTCONV_CONV,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},\n    {LLM_TENSOR_SHORTCONV_INPROJ,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_SHORTCONV_OUTPROJ,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_VISEXP_ATTN_QKV,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_VISEXP_ATTN_OUT,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_VISEXP_FFN_GATE,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_VISEXP_FFN_DOWN,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_VISEXP_FFN_UP,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_INDEXER_K_NORM,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_INDEXER_PROJ,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_INDEXER_ATTN_K,             {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_INDEXER_ATTN_Q_B,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},\n    // NextN/MTP tensors are currently ignored (reserved for future MTP support)\n    // These tensors only exist in the last layer(s) and are treated as output tensors\n    {LLM_TENSOR_NEXTN_EH_PROJ,              {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_NEXTN_EMBED_TOKENS,         {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_NEXTN_ENORM,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},\n    {LLM_TENSOR_NEXTN_HNORM,                {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    {LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,     {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},\n    {LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,     {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},\n    // Nemotron 3 Super\n    {LLM_TENSOR_FFN_LATENT_DOWN,            {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n    {LLM_TENSOR_FFN_LATENT_UP,              {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},\n};\n\nLLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {}\n\nstd::string LLM_KV::operator()(llm_kv kv) const {\n    std::string name = ::format(LLM_KV_NAMES.at(kv), LLM_ARCH_NAMES.at(arch));\n\n    if (suffix != nullptr) {\n        name += \".\";\n        name += suffix;\n    }\n\n    return name;\n}\n\nLLM_TN_IMPL::LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid)\n    : arch(arch), tensor(tensor), suffix(suffix), bid(bid), xid(xid),\n      model_tensors(llm_get_tensor_names(arch)) {}\n\nstd::string LLM_TN_IMPL::str() const {\n    if (LLM_TENSOR_NAMES.find(tensor) == LLM_TENSOR_NAMES.end()) {\n        GGML_ABORT(\"unknown tensor name for tensor id %d\", static_cast<int>(tensor));\n    }\n\n    if (model_tensors.find(tensor) == model_tensors.end()) {\n        return LLM_TENSOR_NAMES.at(tensor);\n    }\n\n    std::string name = ::format(LLM_TENSOR_NAMES.at(tensor), bid, xid);\n    if (suffix != nullptr) {\n        name += \".\";\n        name += suffix;\n    }\n\n    return name;\n}\n\nstd::vector<llm_arch> llm_arch_all() {\n    std::vector<llm_arch> ret;\n    ret.reserve(LLM_ARCH_NAMES.size());\n    for (const auto & [arch, _] : LLM_ARCH_NAMES) {\n        ret.push_back(arch);\n    }\n    return ret;\n}\n\nconst char * llm_arch_name(llm_arch arch) {\n    auto it = LLM_ARCH_NAMES.find(arch);\n    if (it == LLM_ARCH_NAMES.end()) {\n        return \"unknown\";\n    }\n    return it->second;\n}\n\nllm_arch llm_arch_from_string(const std::string & name) {\n    for (const auto & kv : LLM_ARCH_NAMES) { // NOLINT\n        if (kv.second == name) {\n            return kv.first;\n        }\n    }\n\n    return LLM_ARCH_UNKNOWN;\n}\n\nconst llm_tensor_info & llm_tensor_info_for(llm_tensor tensor) {\n    return LLM_TENSOR_INFOS.at(tensor);\n}\n\nbool llm_arch_is_recurrent(const llm_arch & arch) {\n    switch (arch) {\n        case LLM_ARCH_MAMBA:\n        case LLM_ARCH_MAMBA2:\n        case LLM_ARCH_RWKV6:\n        case LLM_ARCH_RWKV6QWEN2:\n        case LLM_ARCH_RWKV7:\n        case LLM_ARCH_ARWKV7:\n            return true;\n        default:\n            return false;\n    }\n}\n\nbool llm_arch_is_hybrid(const llm_arch & arch) {\n    switch (arch) {\n        case LLM_ARCH_JAMBA:\n        case LLM_ARCH_FALCON_H1:\n        case LLM_ARCH_PLAMO2:\n        case LLM_ARCH_GRANITE_HYBRID:\n        case LLM_ARCH_LFM2:\n        case LLM_ARCH_LFM2MOE:\n        case LLM_ARCH_NEMOTRON_H:\n        case LLM_ARCH_NEMOTRON_H_MOE:\n        case LLM_ARCH_QWEN3NEXT:\n        case LLM_ARCH_KIMI_LINEAR:\n        case LLM_ARCH_QWEN35:\n        case LLM_ARCH_QWEN35MOE:\n            return true;\n        default:\n            return false;\n    }\n}\n\nbool llm_arch_is_diffusion(const llm_arch & arch) {\n    switch (arch) {\n        case LLM_ARCH_DREAM:\n        case LLM_ARCH_LLADA:\n        case LLM_ARCH_LLADA_MOE:\n        case LLM_ARCH_RND1:\n            return true;\n        default:\n            return false;\n    }\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-arch.h",
    "content": "#pragma once\n\n#include \"ggml.h\" // ggml_op\n\n#include <string>\n#include <set>\n#include <vector>\n\n//\n// gguf constants (sync with gguf.py)\n//\n\nenum llm_arch {\n    LLM_ARCH_CLIP,\n    LLM_ARCH_LLAMA,\n    LLM_ARCH_LLAMA4,\n    LLM_ARCH_DECI,\n    LLM_ARCH_FALCON,\n    LLM_ARCH_BAICHUAN,\n    LLM_ARCH_GROK,\n    LLM_ARCH_GPT2,\n    LLM_ARCH_GPTJ,\n    LLM_ARCH_GPTNEOX,\n    LLM_ARCH_MPT,\n    LLM_ARCH_STARCODER,\n    LLM_ARCH_REFACT,\n    LLM_ARCH_BERT,\n    LLM_ARCH_MODERN_BERT,\n    LLM_ARCH_NOMIC_BERT,\n    LLM_ARCH_NOMIC_BERT_MOE,\n    LLM_ARCH_NEO_BERT,\n    LLM_ARCH_JINA_BERT_V2,\n    LLM_ARCH_JINA_BERT_V3,\n    LLM_ARCH_EUROBERT,\n    LLM_ARCH_BLOOM,\n    LLM_ARCH_STABLELM,\n    LLM_ARCH_QWEN,\n    LLM_ARCH_QWEN2,\n    LLM_ARCH_QWEN2MOE,\n    LLM_ARCH_QWEN2VL,\n    LLM_ARCH_QWEN3,\n    LLM_ARCH_QWEN3MOE,\n    LLM_ARCH_QWEN3NEXT,\n    LLM_ARCH_QWEN3VL,\n    LLM_ARCH_QWEN3VLMOE,\n    LLM_ARCH_QWEN35,\n    LLM_ARCH_QWEN35MOE,\n    LLM_ARCH_PHI2,\n    LLM_ARCH_PHI3,\n    LLM_ARCH_PHIMOE,\n    LLM_ARCH_PLAMO,\n    LLM_ARCH_PLAMO2,\n    LLM_ARCH_PLAMO3,\n    LLM_ARCH_CODESHELL,\n    LLM_ARCH_ORION,\n    LLM_ARCH_INTERNLM2,\n    LLM_ARCH_MINICPM,\n    LLM_ARCH_MINICPM3,\n    LLM_ARCH_GEMMA,\n    LLM_ARCH_GEMMA2,\n    LLM_ARCH_GEMMA3,\n    LLM_ARCH_GEMMA3N,\n    LLM_ARCH_GEMMA_EMBEDDING,\n    LLM_ARCH_STARCODER2,\n    LLM_ARCH_MAMBA,\n    LLM_ARCH_MAMBA2,\n    LLM_ARCH_JAMBA,\n    LLM_ARCH_FALCON_H1,\n    LLM_ARCH_XVERSE,\n    LLM_ARCH_COMMAND_R,\n    LLM_ARCH_COHERE2,\n    LLM_ARCH_DBRX,\n    LLM_ARCH_OLMO,\n    LLM_ARCH_OLMO2,\n    LLM_ARCH_OLMOE,\n    LLM_ARCH_OPENELM,\n    LLM_ARCH_ARCTIC,\n    LLM_ARCH_DEEPSEEK,\n    LLM_ARCH_DEEPSEEK2,\n    LLM_ARCH_CHATGLM,\n    LLM_ARCH_GLM4,\n    LLM_ARCH_GLM4_MOE,\n    LLM_ARCH_GLM_DSA,\n    LLM_ARCH_BITNET,\n    LLM_ARCH_T5,\n    LLM_ARCH_T5ENCODER,\n    LLM_ARCH_JAIS,\n    LLM_ARCH_JAIS2,\n    LLM_ARCH_NEMOTRON,\n    LLM_ARCH_NEMOTRON_H,\n    LLM_ARCH_NEMOTRON_H_MOE,\n    LLM_ARCH_EXAONE,\n    LLM_ARCH_EXAONE4,\n    LLM_ARCH_EXAONE_MOE,\n    LLM_ARCH_RWKV6,\n    LLM_ARCH_RWKV6QWEN2,\n    LLM_ARCH_RWKV7,\n    LLM_ARCH_ARWKV7,\n    LLM_ARCH_GRANITE,\n    LLM_ARCH_GRANITE_MOE,\n    LLM_ARCH_GRANITE_HYBRID,\n    LLM_ARCH_CHAMELEON,\n    LLM_ARCH_WAVTOKENIZER_DEC,\n    LLM_ARCH_PLM,\n    LLM_ARCH_BAILINGMOE,\n    LLM_ARCH_BAILINGMOE2,\n    LLM_ARCH_DOTS1,\n    LLM_ARCH_ARCEE,\n    LLM_ARCH_AFMOE,\n    LLM_ARCH_ERNIE4_5,\n    LLM_ARCH_ERNIE4_5_MOE,\n    LLM_ARCH_HUNYUAN_MOE,\n    LLM_ARCH_HUNYUAN_DENSE,\n    LLM_ARCH_SMOLLM3,\n    LLM_ARCH_OPENAI_MOE,\n    LLM_ARCH_LFM2,\n    LLM_ARCH_LFM2MOE,\n    LLM_ARCH_DREAM,\n    LLM_ARCH_SMALLTHINKER,\n    LLM_ARCH_LLADA,\n    LLM_ARCH_LLADA_MOE,\n    LLM_ARCH_SEED_OSS,\n    LLM_ARCH_GROVEMOE,\n    LLM_ARCH_APERTUS,\n    LLM_ARCH_MINIMAX_M2,\n    LLM_ARCH_COGVLM,\n    LLM_ARCH_RND1,\n    LLM_ARCH_PANGU_EMBED,\n    LLM_ARCH_MISTRAL3,\n    LLM_ARCH_PADDLEOCR,\n    LLM_ARCH_MIMO2,\n    LLM_ARCH_STEP35,\n    LLM_ARCH_LLAMA_EMBED,\n    LLM_ARCH_MAINCODER,\n    LLM_ARCH_KIMI_LINEAR,\n    LLM_ARCH_UNKNOWN,\n};\n\nenum llm_kv {\n    LLM_KV_GENERAL_TYPE,\n    LLM_KV_GENERAL_ARCHITECTURE,\n    LLM_KV_GENERAL_QUANTIZATION_VERSION,\n    LLM_KV_GENERAL_ALIGNMENT,\n    LLM_KV_GENERAL_FILE_TYPE,\n    LLM_KV_GENERAL_SAMPLING_SEQUENCE,\n    LLM_KV_GENERAL_SAMPLING_TOP_K,\n    LLM_KV_GENERAL_SAMPLING_TOP_P,\n    LLM_KV_GENERAL_SAMPLING_MIN_P,\n    LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY,\n    LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD,\n    LLM_KV_GENERAL_SAMPLING_TEMP,\n    LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N,\n    LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT,\n    LLM_KV_GENERAL_SAMPLING_MIROSTAT,\n    LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU,\n    LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA,\n    LLM_KV_GENERAL_NAME,\n    LLM_KV_GENERAL_AUTHOR,\n    LLM_KV_GENERAL_VERSION,\n    LLM_KV_GENERAL_URL,\n    LLM_KV_GENERAL_DESCRIPTION,\n    LLM_KV_GENERAL_LICENSE,\n    LLM_KV_GENERAL_SOURCE_URL,\n    LLM_KV_GENERAL_SOURCE_HF_REPO,\n\n    LLM_KV_VOCAB_SIZE,\n    LLM_KV_CONTEXT_LENGTH,\n    LLM_KV_EMBEDDING_LENGTH,\n    LLM_KV_EMBEDDING_LENGTH_OUT,\n    LLM_KV_FEATURES_LENGTH,\n    LLM_KV_BLOCK_COUNT,\n    LLM_KV_LEADING_DENSE_BLOCK_COUNT,\n    LLM_KV_FEED_FORWARD_LENGTH,\n    LLM_KV_EXPERT_FEED_FORWARD_LENGTH,\n    LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,\n    LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH,\n    LLM_KV_SWIGLU_CLAMP_EXP,\n    LLM_KV_SWIGLU_CLAMP_SHEXP,\n    LLM_KV_USE_PARALLEL_RESIDUAL,\n    LLM_KV_TENSOR_DATA_LAYOUT,\n    LLM_KV_EXPERT_COUNT,\n    LLM_KV_EXPERT_USED_COUNT,\n    LLM_KV_EXPERT_SHARED_COUNT,\n    LLM_KV_EXPERT_GROUP_COUNT,\n    LLM_KV_EXPERT_GROUP_USED_COUNT,\n    LLM_KV_EXPERT_WEIGHTS_SCALE,\n    LLM_KV_EXPERT_WEIGHTS_NORM,\n    LLM_KV_EXPERT_GATING_FUNC,\n    LLM_KV_EXPERT_GROUP_SCALE,\n    LLM_KV_EXPERTS_PER_GROUP,\n    LLM_KV_MOE_EVERY_N_LAYERS,\n    LLM_KV_MOE_LATENT_SIZE,\n    LLM_KV_NEXTN_PREDICT_LAYERS,\n    LLM_KV_NUM_DEEPSTACK_LAYERS,\n    LLM_KV_POOLING_TYPE,\n    LLM_KV_LOGIT_SCALE,\n    LLM_KV_DECODER_START_TOKEN_ID,\n    LLM_KV_DECODER_BLOCK_COUNT,\n    LLM_KV_ATTN_LOGIT_SOFTCAPPING,\n    LLM_KV_ROUTER_LOGIT_SOFTCAPPING,\n    LLM_KV_FINAL_LOGIT_SOFTCAPPING,\n    LLM_KV_SWIN_NORM,\n    LLM_KV_RESCALE_EVERY_N_LAYERS,\n    LLM_KV_TIME_MIX_EXTRA_DIM,\n    LLM_KV_TIME_DECAY_EXTRA_DIM,\n    LLM_KV_RESIDUAL_SCALE,\n    LLM_KV_EMBEDDING_SCALE,\n    LLM_KV_TOKEN_SHIFT_COUNT,\n    LLM_KV_INTERLEAVE_MOE_LAYER_STEP,\n    LLM_KV_FULL_ATTENTION_INTERVAL,\n\n    LLM_KV_ATTENTION_HEAD_COUNT,\n    LLM_KV_ATTENTION_HEAD_COUNT_KV,\n    LLM_KV_ATTENTION_MAX_ALIBI_BIAS,\n    LLM_KV_ATTENTION_CLAMP_KQV,\n    LLM_KV_ATTENTION_KEY_LENGTH,\n    LLM_KV_ATTENTION_VALUE_LENGTH,\n    LLM_KV_ATTENTION_LAYERNORM_EPS,\n    LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,\n    LLM_KV_ATTENTION_GROUPNORM_EPS,\n    LLM_KV_ATTENTION_GROUPNORM_GROUPS,\n    LLM_KV_ATTENTION_CAUSAL,\n    LLM_KV_ATTENTION_Q_LORA_RANK,\n    LLM_KV_ATTENTION_KV_LORA_RANK,\n    LLM_KV_ATTENTION_DECAY_LORA_RANK,\n    LLM_KV_ATTENTION_ICLR_LORA_RANK,\n    LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK,\n    LLM_KV_ATTENTION_GATE_LORA_RANK,\n    LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,\n    LLM_KV_ATTENTION_SLIDING_WINDOW,\n    LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,\n    LLM_KV_ATTENTION_SCALE,\n    LLM_KV_ATTENTION_OUTPUT_SCALE,\n    LLM_KV_ATTENTION_TEMPERATURE_LENGTH,\n    LLM_KV_ATTENTION_TEMPERATURE_SCALE,\n    LLM_KV_ATTENTION_KEY_LENGTH_MLA,\n    LLM_KV_ATTENTION_VALUE_LENGTH_MLA,\n    LLM_KV_ATTENTION_KEY_LENGTH_SWA,\n    LLM_KV_ATTENTION_VALUE_LENGTH_SWA,\n    LLM_KV_ATTENTION_INDEXER_HEAD_COUNT,\n    LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,\n    LLM_KV_ATTENTION_INDEXER_TOP_K,\n\n    LLM_KV_ROPE_DIMENSION_COUNT,\n    LLM_KV_ROPE_DIMENSION_COUNT_SWA,\n    LLM_KV_ROPE_DIMENSION_SECTIONS,\n    LLM_KV_ROPE_FREQ_BASE,\n    LLM_KV_ROPE_FREQ_BASE_SWA,\n    LLM_KV_ROPE_SCALE_LINEAR,\n    LLM_KV_ROPE_SCALING_TYPE,\n    LLM_KV_ROPE_SCALING_FACTOR,\n    LLM_KV_ROPE_SCALING_ATTN_FACTOR,\n    LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,\n    LLM_KV_ROPE_SCALING_FINETUNED,\n    LLM_KV_ROPE_SCALING_YARN_LOG_MUL,\n    LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR,\n    LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR,\n    LLM_KV_ROPE_SCALING_YARN_BETA_FAST,\n    LLM_KV_ROPE_SCALING_YARN_BETA_SLOW,\n\n    LLM_KV_SPLIT_NO,\n    LLM_KV_SPLIT_COUNT,\n    LLM_KV_SPLIT_TENSORS_COUNT,\n\n    LLM_KV_SSM_INNER_SIZE,\n    LLM_KV_SSM_CONV_KERNEL,\n    LLM_KV_SSM_STATE_SIZE,\n    LLM_KV_SSM_TIME_STEP_RANK,\n    LLM_KV_SSM_GROUP_COUNT,\n    LLM_KV_SSM_DT_B_C_RMS,\n\n    LLM_KV_KDA_HEAD_DIM,\n\n    LLM_KV_WKV_HEAD_SIZE,\n\n    LLM_KV_TOKENIZER_MODEL,\n    LLM_KV_TOKENIZER_PRE,\n    LLM_KV_TOKENIZER_LIST,\n    LLM_KV_TOKENIZER_TOKEN_TYPE,\n    LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,\n    LLM_KV_TOKENIZER_SCORES,\n    LLM_KV_TOKENIZER_MERGES,\n    LLM_KV_TOKENIZER_BOS_ID,\n    LLM_KV_TOKENIZER_EOS_ID,\n    LLM_KV_TOKENIZER_EOT_ID,\n    LLM_KV_TOKENIZER_EOM_ID,\n    LLM_KV_TOKENIZER_UNK_ID,\n    LLM_KV_TOKENIZER_SEP_ID,\n    LLM_KV_TOKENIZER_PAD_ID,\n    LLM_KV_TOKENIZER_CLS_ID,\n    LLM_KV_TOKENIZER_MASK_ID,\n    LLM_KV_TOKENIZER_ADD_BOS,\n    LLM_KV_TOKENIZER_ADD_EOS,\n    LLM_KV_TOKENIZER_ADD_SEP,\n    LLM_KV_TOKENIZER_ADD_PREFIX,\n    LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,\n    LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,\n    LLM_KV_TOKENIZER_HF_JSON,\n    LLM_KV_TOKENIZER_RWKV,\n    LLM_KV_TOKENIZER_CHAT_TEMPLATE,\n    LLM_KV_TOKENIZER_FIM_PRE_ID,\n    LLM_KV_TOKENIZER_FIM_SUF_ID,\n    LLM_KV_TOKENIZER_FIM_MID_ID,\n    LLM_KV_TOKENIZER_FIM_PAD_ID,\n    LLM_KV_TOKENIZER_FIM_REP_ID,\n    LLM_KV_TOKENIZER_FIM_SEP_ID,\n\n    LLM_KV_ADAPTER_TYPE,\n    LLM_KV_ADAPTER_LORA_ALPHA,\n    LLM_KV_ADAPTER_LORA_TASK_NAME,\n    LLM_KV_ADAPTER_LORA_PROMPT_PREFIX,\n    LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS,\n\n    LLM_KV_POSNET_EMBEDDING_LENGTH,\n    LLM_KV_POSNET_BLOCK_COUNT,\n\n    LLM_KV_CONVNEXT_EMBEDDING_LENGTH,\n    LLM_KV_CONVNEXT_BLOCK_COUNT,\n\n    LLM_KV_CLASSIFIER_OUTPUT_LABELS,\n\n    LLM_KV_SHORTCONV_L_CACHE,\n\n    LLM_KV_XIELU_ALPHA_N,\n    LLM_KV_XIELU_ALPHA_P,\n    LLM_KV_XIELU_BETA,\n    LLM_KV_XIELU_EPS,\n\n    // deprecated:\n    LLM_KV_TOKENIZER_PREFIX_ID,\n    LLM_KV_TOKENIZER_SUFFIX_ID,\n    LLM_KV_TOKENIZER_MIDDLE_ID,\n\n    // sentence-transformers dense layers in and out features\n    LLM_KV_DENSE_2_FEAT_IN,\n    LLM_KV_DENSE_2_FEAT_OUT,\n    LLM_KV_DENSE_3_FEAT_IN,\n    LLM_KV_DENSE_3_FEAT_OUT,\n};\n\nenum llm_tensor {\n    LLM_TENSOR_TOKEN_EMBD,\n    LLM_TENSOR_TOKEN_EMBD_NORM,\n    LLM_TENSOR_TOKEN_TYPES,\n    LLM_TENSOR_POS_EMBD,\n    LLM_TENSOR_DENSE_2_OUT,\n    LLM_TENSOR_DENSE_3_OUT,\n    LLM_TENSOR_OUTPUT,\n    LLM_TENSOR_OUTPUT_NORM,\n    LLM_TENSOR_OUTPUT_NORM_LFM2, // fix for wrong tensor name\n    LLM_TENSOR_ROPE_FREQS,\n    LLM_TENSOR_ROPE_FACTORS_LONG,\n    LLM_TENSOR_ROPE_FACTORS_SHORT,\n    LLM_TENSOR_ATTN_Q,\n    LLM_TENSOR_ATTN_K,\n    LLM_TENSOR_ATTN_V,\n    LLM_TENSOR_ATTN_QKV,\n    LLM_TENSOR_ATTN_OUT,\n    LLM_TENSOR_ATTN_NORM,\n    LLM_TENSOR_ATTN_NORM_2,\n    LLM_TENSOR_ATTN_OUT_NORM,\n    LLM_TENSOR_ATTN_POST_NORM,\n    LLM_TENSOR_ATTN_ROT_EMBD,\n    LLM_TENSOR_ATTN_SINKS,\n    LLM_TENSOR_ATTN_GATE,\n    LLM_TENSOR_FFN_GATE_INP,\n    LLM_TENSOR_FFN_GATE_INP_SHEXP,\n    LLM_TENSOR_FFN_NORM,\n    LLM_TENSOR_FFN_POST_NORM,\n    LLM_TENSOR_FFN_GATE,\n    LLM_TENSOR_FFN_DOWN,\n    LLM_TENSOR_FFN_UP,\n    LLM_TENSOR_FFN_ACT,\n    LLM_TENSOR_FFN_DOWN_EXP,  // split experts for backward compatibility\n    LLM_TENSOR_FFN_GATE_EXP,\n    LLM_TENSOR_FFN_UP_EXP,\n    LLM_TENSOR_FFN_NORM_EXPS,\n    LLM_TENSOR_FFN_DOWN_EXPS, // merged experts\n    LLM_TENSOR_FFN_GATE_EXPS,\n    LLM_TENSOR_FFN_UP_EXPS,\n    LLM_TENSOR_FFN_GATE_UP_EXPS,\n    LLM_TENSOR_FFN_DOWN_SHEXP,\n    LLM_TENSOR_FFN_GATE_SHEXP,\n    LLM_TENSOR_FFN_UP_SHEXP,\n    LLM_TENSOR_FFN_DOWN_CHEXPS,\n    LLM_TENSOR_FFN_GATE_CHEXPS,\n    LLM_TENSOR_FFN_UP_CHEXPS,\n    LLM_TENSOR_FFN_EXP_PROBS_B,\n    LLM_TENSOR_FFN_LATENT_DOWN,\n    LLM_TENSOR_FFN_LATENT_UP,\n    LLM_TENSOR_ATTN_Q_NORM,\n    LLM_TENSOR_ATTN_K_NORM,\n    LLM_TENSOR_LAYER_OUT_NORM,\n    LLM_TENSOR_POST_ATTN_NORM,\n    LLM_TENSOR_POST_MLP_NORM,\n    LLM_TENSOR_PER_LAYER_TOKEN_EMBD, // gemma3n\n    LLM_TENSOR_PER_LAYER_MODEL_PROJ, // gemma3n\n    LLM_TENSOR_PER_LAYER_INP_GATE,   // gemma3n\n    LLM_TENSOR_PER_LAYER_PROJ,       // gemma3n\n    LLM_TENSOR_PER_LAYER_PROJ_NORM,  // gemma3n\n    LLM_TENSOR_PER_LAYER_POST_NORM,  // gemma3n\n    LLM_TENSOR_ALTUP_PROJ,           // gemma3n\n    LLM_TENSOR_ALTUP_UNEMBD_PROJ,    // gemma3n\n    LLM_TENSOR_ALTUP_CORRECT_COEF,   // gemma3n\n    LLM_TENSOR_ALTUP_CORRECT_SCALE,  // gemma3n\n    LLM_TENSOR_ALTUP_PREDICT_COEF,   // gemma3n\n    LLM_TENSOR_ALTUP_ROUTER,         // gemma3n\n    LLM_TENSOR_ALTUP_ROUTER_NORM,    // gemma3n\n    LLM_TENSOR_LAUREL_L,             // gemma3n\n    LLM_TENSOR_LAUREL_R,             // gemma3n\n    LLM_TENSOR_LAUREL_POST_NORM,     // gemma3n\n    LLM_TENSOR_SSM_IN,\n    LLM_TENSOR_SSM_CONV1D,\n    LLM_TENSOR_SSM_X,\n    LLM_TENSOR_SSM_DT,\n    LLM_TENSOR_SSM_DT_NORM,\n    LLM_TENSOR_SSM_A,\n    LLM_TENSOR_SSM_A_NOSCAN,        // qwen3next special case with MUL instead of SSM_SCAN\n    LLM_TENSOR_SSM_B_NORM,\n    LLM_TENSOR_SSM_C_NORM,\n    LLM_TENSOR_SSM_D,\n    LLM_TENSOR_SSM_NORM,\n    LLM_TENSOR_SSM_OUT,\n    LLM_TENSOR_SSM_BETA_ALPHA,      // qwen3next\n    LLM_TENSOR_SSM_ALPHA,           // qwen3.5\n    // Kimi Linear KDA (using SSM_ prefix for consistency)\n    LLM_TENSOR_SSM_CONV1D_Q,        // kimi: Q conv1d weight\n    LLM_TENSOR_SSM_CONV1D_K,        // kimi: K conv1d weight\n    LLM_TENSOR_SSM_CONV1D_V,        // kimi: V conv1d weight\n    LLM_TENSOR_SSM_F_A,             // kimi: forget gate projection A\n    LLM_TENSOR_SSM_F_B,             // kimi: forget gate projection B\n    LLM_TENSOR_SSM_BETA,            // kimi: beta mixing coefficient and qwen3.5\n    LLM_TENSOR_SSM_G_A,             // kimi: output gate projection A\n    LLM_TENSOR_SSM_G_B,             // kimi: output gate projection B\n    LLM_TENSOR_TIME_MIX_W0,\n    LLM_TENSOR_TIME_MIX_W1,\n    LLM_TENSOR_TIME_MIX_W2,\n    LLM_TENSOR_TIME_MIX_A0,\n    LLM_TENSOR_TIME_MIX_A1,\n    LLM_TENSOR_TIME_MIX_A2,\n    LLM_TENSOR_TIME_MIX_V0,\n    LLM_TENSOR_TIME_MIX_V1,\n    LLM_TENSOR_TIME_MIX_V2,\n    LLM_TENSOR_TIME_MIX_G1,\n    LLM_TENSOR_TIME_MIX_G2,\n    LLM_TENSOR_TIME_MIX_K_K,\n    LLM_TENSOR_TIME_MIX_K_A,\n    LLM_TENSOR_TIME_MIX_R_K,\n    LLM_TENSOR_TIME_MIX_LERP_X,\n    LLM_TENSOR_TIME_MIX_LERP_W,\n    LLM_TENSOR_TIME_MIX_LERP_K,\n    LLM_TENSOR_TIME_MIX_LERP_V,\n    LLM_TENSOR_TIME_MIX_LERP_R,\n    LLM_TENSOR_TIME_MIX_LERP_G,\n    LLM_TENSOR_TIME_MIX_LERP_FUSED,\n    LLM_TENSOR_TIME_MIX_FIRST,\n    LLM_TENSOR_TIME_MIX_DECAY,\n    LLM_TENSOR_TIME_MIX_DECAY_W1,\n    LLM_TENSOR_TIME_MIX_DECAY_W2,\n    LLM_TENSOR_TIME_MIX_KEY,\n    LLM_TENSOR_TIME_MIX_VALUE,\n    LLM_TENSOR_TIME_MIX_RECEPTANCE,\n    LLM_TENSOR_TIME_MIX_GATE,\n    LLM_TENSOR_TIME_MIX_LN,\n    LLM_TENSOR_TIME_MIX_OUTPUT,\n    LLM_TENSOR_CHANNEL_MIX_LERP_K,\n    LLM_TENSOR_CHANNEL_MIX_LERP_R,\n    LLM_TENSOR_CHANNEL_MIX_KEY,\n    LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,\n    LLM_TENSOR_CHANNEL_MIX_VALUE,\n    LLM_TENSOR_ATTN_Q_A,\n    LLM_TENSOR_ATTN_Q_B,\n    LLM_TENSOR_ATTN_KV_A_MQA,\n    LLM_TENSOR_ATTN_KV_B,\n    LLM_TENSOR_ATTN_K_B,\n    LLM_TENSOR_ATTN_V_B,\n    LLM_TENSOR_ATTN_Q_A_NORM,\n    LLM_TENSOR_ATTN_KV_A_NORM,\n    LLM_TENSOR_ATTN_SUB_NORM,\n    LLM_TENSOR_FFN_SUB_NORM,\n    LLM_TENSOR_DEC_ATTN_NORM,\n    LLM_TENSOR_DEC_ATTN_Q,\n    LLM_TENSOR_DEC_ATTN_K,\n    LLM_TENSOR_DEC_ATTN_V,\n    LLM_TENSOR_DEC_ATTN_OUT,\n    LLM_TENSOR_DEC_ATTN_REL_B,\n    LLM_TENSOR_DEC_CROSS_ATTN_NORM,\n    LLM_TENSOR_DEC_CROSS_ATTN_Q,\n    LLM_TENSOR_DEC_CROSS_ATTN_K,\n    LLM_TENSOR_DEC_CROSS_ATTN_V,\n    LLM_TENSOR_DEC_CROSS_ATTN_OUT,\n    LLM_TENSOR_DEC_CROSS_ATTN_REL_B,\n    LLM_TENSOR_DEC_FFN_NORM,\n    LLM_TENSOR_DEC_FFN_GATE,\n    LLM_TENSOR_DEC_FFN_DOWN,\n    LLM_TENSOR_DEC_FFN_UP,\n    LLM_TENSOR_DEC_OUTPUT_NORM,\n    LLM_TENSOR_ENC_ATTN_NORM,\n    LLM_TENSOR_ENC_ATTN_Q,\n    LLM_TENSOR_ENC_ATTN_K,\n    LLM_TENSOR_ENC_ATTN_V,\n    LLM_TENSOR_ENC_ATTN_OUT,\n    LLM_TENSOR_ENC_ATTN_REL_B,\n    LLM_TENSOR_ENC_FFN_NORM,\n    LLM_TENSOR_ENC_FFN_GATE,\n    LLM_TENSOR_ENC_FFN_DOWN,\n    LLM_TENSOR_ENC_FFN_UP,\n    LLM_TENSOR_ENC_OUTPUT_NORM,\n    LLM_TENSOR_CLS,\n    LLM_TENSOR_CLS_OUT,\n    LLM_TENSOR_CLS_NORM,\n    LLM_TENSOR_CONV1D,\n    LLM_TENSOR_CONVNEXT_DW,\n    LLM_TENSOR_CONVNEXT_NORM,\n    LLM_TENSOR_CONVNEXT_PW1,\n    LLM_TENSOR_CONVNEXT_PW2,\n    LLM_TENSOR_CONVNEXT_GAMMA,\n    LLM_TENSOR_POS_NET_CONV1,\n    LLM_TENSOR_POS_NET_CONV2,\n    LLM_TENSOR_POS_NET_NORM,\n    LLM_TENSOR_POS_NET_NORM1,\n    LLM_TENSOR_POS_NET_NORM2,\n    LLM_TENSOR_POS_NET_ATTN_NORM,\n    LLM_TENSOR_POS_NET_ATTN_Q,\n    LLM_TENSOR_POS_NET_ATTN_K,\n    LLM_TENSOR_POS_NET_ATTN_V,\n    LLM_TENSOR_POS_NET_ATTN_OUT,\n    LLM_TENSOR_SHORTCONV_CONV,\n    LLM_TENSOR_SHORTCONV_INPROJ,\n    LLM_TENSOR_SHORTCONV_OUTPROJ,\n    LLM_TENSOR_VISEXP_ATTN_QKV,\n    LLM_TENSOR_VISEXP_ATTN_OUT,\n    LLM_TENSOR_VISEXP_FFN_GATE,\n    LLM_TENSOR_VISEXP_FFN_DOWN,\n    LLM_TENSOR_VISEXP_FFN_UP,\n    LLM_TENSOR_INDEXER_K_NORM,\n    LLM_TENSOR_INDEXER_PROJ,\n    LLM_TENSOR_INDEXER_ATTN_K,\n    LLM_TENSOR_INDEXER_ATTN_Q_B,\n    LLM_TENSOR_NEXTN_EH_PROJ,\n    LLM_TENSOR_NEXTN_EMBED_TOKENS,\n    LLM_TENSOR_NEXTN_ENORM,\n    LLM_TENSOR_NEXTN_HNORM,\n    LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,\n    LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,\n};\n\nenum llm_tensor_layer {\n    LLM_TENSOR_LAYER_INPUT,\n    LLM_TENSOR_LAYER_REPEATING,\n    LLM_TENSOR_LAYER_OUTPUT,\n};\n\nstruct LLM_KV {\n    LLM_KV(llm_arch arch, const char * suffix = nullptr);\n\n    llm_arch arch;\n    const char * suffix;\n\n    std::string operator()(llm_kv kv) const;\n};\n\n// helper to handle gguf constants\n// usage:\n//\n//   const auto tn = LLM_TN(LLM_ARCH_LLAMA);\n//\n//   std::string name = tn(LLM_TENSOR_OUTPUT);                     -> \"output\"\n//   std::string name = tn(LLM_TENSOR_TOKEN_EMBD, \"bias\");         -> \"token_embd.bias\"\n//   std::string name = tn(LLM_TENSOR_ATTN_NORM, \"weight\", 3);     -> \"blk.3.attn_norm.weight\"\n//\nstruct LLM_TN_IMPL {\n    const llm_arch arch;\n    const llm_tensor tensor;\n    const char * const suffix;\n    const int bid;\n    const int xid;\n\n    const std::set<llm_tensor> model_tensors;\n\n    LLM_TN_IMPL(llm_arch arch, llm_tensor tensor, const char * suffix, int bid, int xid);\n\n    std::string str() const;\n\n    operator std::string() const {\n        return str();\n    }\n\n    friend bool operator==(const std::string & str, const LLM_TN_IMPL & tn) {\n        return str == tn.str();\n    }\n\n    friend bool operator!=(const std::string & str, const LLM_TN_IMPL & tn) {\n        return str != tn.str();\n    }\n};\n\nstruct LLM_TN {\n    LLM_TN(llm_arch arch) : arch(arch) {}\n\n    llm_arch arch;\n\n    LLM_TN_IMPL operator()(llm_tensor tensor, const char * suffix, int bid = -1, int xid = -1) const {\n        return LLM_TN_IMPL(arch, tensor, suffix, bid, xid);\n    }\n\n    LLM_TN_IMPL operator()(llm_tensor tensor, int bid = -1, int xid = -1) const {\n        return LLM_TN_IMPL(arch, tensor, nullptr, bid, xid);\n    }\n};\n\n\nstruct llm_tensor_info {\n    llm_tensor_layer layer;\n    ggml_op op;\n};\n\nstd::vector<llm_arch> llm_arch_all();\n\nconst char * llm_arch_name(llm_arch arch);\n\nllm_arch llm_arch_from_string(const std::string & name);\n\nconst llm_tensor_info & llm_tensor_info_for(llm_tensor tensor);\n\nbool llm_arch_is_recurrent(const llm_arch & arch);\nbool llm_arch_is_hybrid   (const llm_arch & arch);\nbool llm_arch_is_diffusion(const llm_arch & arch);\n"
  },
  {
    "path": "examples/talk-llama/llama-batch.cpp",
    "content": "#include \"llama-batch.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-vocab.h\"\n#include \"llama-memory.h\"\n\n#include <cassert>\n#include <cstring>\n#include <algorithm>\n#include <sstream>\n\nllama_batch_allocr::llama_batch_allocr(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {\n    const char * LLAMA_BATCH_DEBUG = getenv(\"LLAMA_BATCH_DEBUG\");\n    debug = LLAMA_BATCH_DEBUG ? atoi(LLAMA_BATCH_DEBUG) : 0;\n\n    seq_pos.resize(LLAMA_MAX_SEQ);\n    seq_cpl.resize(LLAMA_MAX_SEQ);\n    for (auto & cur : seq_cpl) {\n        cur.resize(LLAMA_MAX_SEQ);\n    }\n\n    seq_idx.resize(LLAMA_MAX_SEQ, -1);\n}\n\nbool llama_batch_allocr::init(\n        const llama_batch & batch_inp,\n        const llama_vocab & vocab,\n        const llama_memory_i * memory,\n        uint32_t n_embd,\n        uint32_t n_seq_max,\n        bool output_all) {\n    clear();\n\n    batch = batch_inp;\n\n    this->vocab = &vocab;\n\n    GGML_ASSERT(batch.n_tokens > 0);\n\n    //\n    // validate input batch\n    //\n\n    if (n_seq_max > LLAMA_MAX_SEQ) {\n        LLAMA_LOG_ERROR(\"%s: n_seq_max = %d > %d\\n\", __func__, n_seq_max, LLAMA_MAX_SEQ);\n        return false;\n    }\n\n    if (batch.token) {\n        for (int32_t i = 0; i < batch.n_tokens; ++i) {\n            if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= vocab.n_tokens()) {\n                LLAMA_LOG_ERROR(\"%s: invalid token[%d] = %d\\n\", __func__, i, batch.token[i]);\n                return false;\n            }\n        }\n    }\n\n    if (batch.seq_id) {\n        for (int32_t i = 0; i < batch.n_tokens; ++i) {\n            for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {\n                if (batch.seq_id && (batch.seq_id[i][s] < 0 || batch.seq_id[i][s] >= (llama_seq_id) n_seq_max)) {\n                    LLAMA_LOG_ERROR(\"%s: invalid seq_id[%d][%d] = %d >= %d\\n\", __func__, i, s, batch.seq_id[i][s], (llama_seq_id) n_seq_max);\n                    return false;\n                }\n            }\n        }\n    }\n\n    //\n    // auto-generate missing fields\n    //\n\n    if (!batch.n_seq_id) {\n        n_seq_id.resize(batch.n_tokens);\n        for (int32_t i = 0; i < batch.n_tokens; i++) {\n            n_seq_id[i] = seq_id_0.size();\n        }\n        batch.n_seq_id = n_seq_id.data();\n    }\n\n    if (!batch.seq_id) {\n        seq_id.resize(batch.n_tokens + 1);\n        seq_id[batch.n_tokens] = NULL;\n        for (int32_t i = 0; i < batch.n_tokens; i++) {\n            seq_id[i] = seq_id_0.data();\n        }\n        batch.seq_id = seq_id.data();\n    }\n\n    if (!batch.pos) {\n        pos.resize(batch.n_tokens);\n\n        // initialize the starting position for each sequence based on the positions in the memory\n        llama_pos p0[LLAMA_MAX_SEQ];\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            if (!memory) {\n                // if no memory -> start from 0\n                p0[s] = 0;\n            } else {\n                p0[s] = memory->seq_pos_max(s) + 1;\n            }\n        }\n\n        for (int32_t i = 0; i < batch.n_tokens; i++) {\n            const llama_seq_id seq_id = batch.seq_id[i][0];\n\n            pos[i] = p0[seq_id];\n\n            // update the starting position for all sequences that are assigned to the this token\n            for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id = batch.seq_id[i][s];\n\n                p0[seq_id] = pos[i] + 1;\n            }\n        }\n\n        batch.pos = pos.data();\n    }\n\n    if (!batch.logits) {\n        if (output_all) {\n            // return the output for all tokens\n            output.resize(batch.n_tokens, true);\n        } else {\n            // return the output only for the last token\n            output.resize(batch.n_tokens, false);\n            output[output.size() - 1] = true;\n        }\n\n        batch.logits = output.data();\n    } else if (output_all) {\n        bool warn = false;\n\n        for (int32_t i = 0; i < batch.n_tokens; ++i) {\n            if (batch.logits[i] == 0) {\n                warn = true;\n            }\n        }\n\n        if (warn) {\n            LLAMA_LOG_WARN(\"%s: embeddings required but some input tokens were not marked as outputs -> overriding\\n\", __func__);\n\n            output.resize(batch.n_tokens, true);\n            batch.logits = output.data();\n        }\n    }\n\n    //\n    // compute stats\n    //\n\n    this->n_embd    = n_embd;\n    this->n_seq_max = n_seq_max;\n\n    // count the outputs in this batch\n    for (int32_t i = 0; i < batch.n_tokens; ++i) {\n        n_outputs += batch.logits[i] != 0;\n    }\n\n    has_cpl = false;\n\n    // determine coupled sequences\n    // these are pairs of sequences that have at least one token in the input batch that is assigned to both of them\n    for (int32_t i = 0; i < batch.n_tokens; ++i) {\n        const llama_seq_id s0 = batch.seq_id[i][0];\n\n        for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {\n            const llama_seq_id s1 = batch.seq_id[i][s];\n\n            seq_pos[s1].insert(batch.pos[i]);\n\n            if (s > 0) {\n                // mark that sequence s1 is coupled to s0\n                seq_cpl[s1][s0] = true;\n\n                // note: tracking the other way around is not necessary for now\n                //seq_cpl[s0][s1] = true;\n\n                has_cpl = true;\n            }\n        }\n    }\n\n    // precompute the sequence sets for each token and determine the unique sequence ids that participate in the batch\n    {\n        seq_set_t seq_set_unq;\n\n        for (int32_t i = 0; i < batch.n_tokens; ++i) {\n            seq_set_t cur;\n            for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id = batch.seq_id[i][s];\n\n                cur        .set(seq_id);\n                seq_set_unq.set(seq_id);\n            }\n\n            seq_set.push_back(cur);\n            seq_set_map[cur].push_back(i);\n        }\n\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            if (seq_set_unq.test(s)) {\n                seq_idx[s] = seq_id_unq.size();\n                seq_id_unq.push_back(s);\n            }\n        }\n    }\n\n    if (debug > 0) {\n        LLAMA_LOG_DEBUG(\"%s: input batch info:\\n\", __func__);\n\n        llama_ubatch ubatch {\n            /*.b_equal_seqs =*/ false,\n            /*.n_tokens     =*/ (uint32_t) batch.n_tokens,\n            /*.n_seq_tokens =*/ (uint32_t) 1,\n            /*.n_seqs       =*/ (uint32_t) batch.n_tokens,\n            /*.n_seqs_unq   =*/ (uint32_t) this->seq_id_unq.size(),\n            /*.n_pos        =*/ n_pos_per_embd,\n            /*.token        =*/ batch.token,\n            /*.embd         =*/ batch.embd,\n            /*.pos          =*/ batch.pos,\n            /*.n_seq_id     =*/ batch.n_seq_id,\n            /*.seq_id       =*/ batch.seq_id,\n            /*.seq_id_unq   =*/ this->seq_id_unq.data(),\n            /*.seq_idx      =*/ this->seq_idx.data(),\n            /*.output       =*/ batch.logits,\n            /*.data         =*/ {},\n        };\n\n        ubatch_print(ubatch, debug);\n\n        LLAMA_LOG_DEBUG(\"%s:   seq       = [\\n\", __func__);\n        for (int s0 = 0; s0 < (int) seq_pos.size(); ++s0) {\n            if (seq_pos[s0].empty()) {\n                continue;\n            }\n\n            std::stringstream ss;\n            for (int s1 = 0; s1 < (int) seq_cpl[s0].size(); ++s1) {\n                if (seq_cpl[s0][s1]) {\n                    ss << s1 << \" \";\n                }\n            }\n\n            LLAMA_LOG_DEBUG(\"%s:  %4d: pos = [%4d, %4d], cpl = %s\\n\",\n                    __func__, s0, seq_pos_min(s0), seq_pos_max(s0), ss.str().empty() ? \"-\" : ss.str().c_str());\n        }\n        LLAMA_LOG_DEBUG(\"%s:   ]\\n\", __func__);\n    }\n\n    //\n    // consistency checks\n    //\n\n    if (n_pos_per_embd > 1) {\n        // M-RoPE case: allow position to \"jump\" forward only (non-continuous positions are allowed)\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            if (seq_pos[s].empty()) {\n                continue;\n            }\n\n            const llama_pos p0 = memory ? memory->seq_pos_max(s) : -1;\n\n            if (batch.token) {\n                if (p0 >= 0 && p0 >= seq_pos_min(s)) {\n                    LLAMA_LOG_ERROR(\n                            \"%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\\n\"\n                            \" - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\\n\"\n                            \" - the tokens for sequence %d in the input batch have a starting position of Y = %d\\n\"\n                            \" for M-RoPE, it is required that the position satisfies: X < Y\\n\",\n                            __func__, s, s, p0, s, seq_pos_min(s));\n\n                    return false;\n                }\n            } else {\n                // embedding inputs can have overlapping positions\n                if (p0 >= 0 && p0 > seq_pos_min(s)) {\n                    LLAMA_LOG_ERROR(\n                            \"%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\\n\"\n                            \" - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\\n\"\n                            \" - the tokens for sequence %d in the input batch have a starting position of Y = %d\\n\"\n                            \" for M-RoPE, it is required that the position satisfies: X <= Y\\n\",\n                            __func__, s, s, p0, s, seq_pos_min(s));\n\n                    return false;\n                }\n            }\n        }\n    } else {\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            if (seq_pos[s].empty()) {\n                continue;\n            }\n\n            const llama_pos p0 = memory ? memory->seq_pos_max(s) : -1;\n\n            if (p0 >= 0) {\n                bool ok = true;\n\n                if (seq_pos_min(s) != p0 + 1) {\n                    ok = false;\n                }\n\n                if (!ok) {\n                    LLAMA_LOG_ERROR(\n                            \"%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\\n\"\n                            \" - the last position stored in the memory module of the context (i.e. the KV cache) for sequence %d is X = %d\\n\"\n                            \" - the tokens for sequence %d in the input batch have a starting position of Y = %d\\n\"\n                            \" it is required that the sequence positions remain consecutive: Y = X + 1\\n\",\n                            __func__, s, s, p0, s, seq_pos_min(s));\n\n                    return false;\n                }\n            }\n\n            if (seq_pos_max(s) - seq_pos_min(s) + 1 > (int) seq_pos[s].size()) {\n                LLAMA_LOG_ERROR(\"%s: sequence %d positions are not continuous\\n\", __func__, s);\n                return false;\n            }\n        }\n    }\n\n    if (memory) {\n        for (uint32_t s0 = 0; s0 < n_seq_max; ++s0) {\n            for (uint32_t s1 = 0; s1 < n_seq_max; ++s1) {\n                if (seq_cpl[s0][s1]) {\n                    if (memory->seq_pos_min(s0) != memory->seq_pos_min(s1) ||\n                        memory->seq_pos_max(s0) != memory->seq_pos_max(s1)) {\n                        LLAMA_LOG_ERROR(\"%s: sequence %d is coupled to %d in the input batch, but have divereged\\n\", __func__, s0, s1);\n                        return false;\n                    }\n                }\n            }\n        }\n    }\n\n    // disallow partial sequence sub-sets:\n    //\n    // invalid:          x\n    //            i: 0 1 2 ...\n    // ---------------------------------------\n    // seq_id[i][0]: 0 0 1\n    // seq_id[i][1]: 1 1 2\n    // seq_id[i][2]: 2\n    //\n    // disallow decreasing sequence positions:\n    //\n    // invalid:                  x\n    //            i: 0 1 2 3 4 5 6 ...\n    // ---------------------------------------\n    //       pos[i]: 4 5 0 1 6 2 3\n    // seq_id[i][0]: 0 0 1 1 0 1 0\n    //\n    {\n        seq_set_t cur_seq_set[LLAMA_MAX_SEQ];\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            cur_seq_set[s].set();\n        }\n\n        llama_pos cur_seq_pos[LLAMA_MAX_SEQ];\n        for (uint32_t s = 0; s < n_seq_max; ++s) {\n            cur_seq_pos[s] = -1;\n        }\n\n        for (int32_t i = 0; i < batch.n_tokens; ++i) {\n            const llama_pos pos = batch.pos[i];\n\n            for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id = batch.seq_id[i][s];\n\n                cur_seq_set[seq_id] &= seq_set[i];\n\n                if (cur_seq_set[seq_id].none()) {\n                    LLAMA_LOG_ERROR(\"%s: sequence %d belongs to incompatible sequence sets (not allowed)\\n\", __func__, seq_id);\n                    return false;\n                }\n\n                if (pos < cur_seq_pos[seq_id]) {\n                    LLAMA_LOG_ERROR(\"%s: sequence %d positions are decreasing (not allowed)\\n\", __func__, seq_id);\n                    return false;\n                }\n            }\n        }\n    }\n\n    split_reset();\n\n    return true;\n}\n\nllama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t n_seqs) {\n    const uint32_t n_tokens = n_seq_tokens*n_seqs;\n\n    clear();\n    split_reset();\n\n    const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;\n\n    auto udata = std::make_shared<llama_ubatch::data_t>();\n\n    udata->token     .resize(n_tokens);\n    udata->embd      .clear();\n    udata->pos       .resize(n_pos_all);\n    udata->n_seq_id  .resize(n_tokens);\n    udata->seq_id    .resize(n_tokens);\n    udata->seq_id_unq.resize(0);\n    udata->seq_idx   .resize(LLAMA_MAX_SEQ, -1);\n    udata->output    .resize(n_tokens);\n\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        udata->seq_idx[s] = s;\n        udata->seq_id_unq.push_back(s);\n    }\n\n    llama_ubatch res {\n        /*.b_equal_seqs =*/ true,\n        /*.n_tokens     =*/ n_tokens,\n        /*.n_seq_tokens =*/ n_seq_tokens,\n        /*.n_seqs       =*/ n_seqs,\n        /*.n_seqs_unq   =*/ n_seqs,\n        /*.n_pos        =*/ n_pos_per_embd,\n\n        /*.token        =*/ udata->token.data(),\n        /*.embd         =*/ nullptr,\n        /*.pos          =*/ udata->pos.data(),\n        /*.n_seq_id     =*/ udata->n_seq_id.data(),\n        /*.seq_id       =*/ udata->seq_id.data(),\n        /*.seq_id_unq   =*/ udata->seq_id_unq.data(),\n        /*.seq_idx      =*/ udata->seq_idx.data(),\n        /*.output       =*/ udata->output.data(),\n        /*.data         =*/ std::move(udata),\n    };\n\n    return res;\n}\n\nconst llama_batch & llama_batch_allocr::get_batch() const {\n    return batch;\n}\n\nuint32_t llama_batch_allocr::get_n_tokens() const {\n    return batch.n_tokens;\n}\n\nuint32_t llama_batch_allocr::get_n_outputs() const {\n    return n_outputs;\n}\n\nuint32_t llama_batch_allocr::get_n_used() const {\n    return n_used;\n}\n\nstd::vector<int32_t> & llama_batch_allocr::get_out_ids() {\n    return out_ids;\n}\n\nllama_pos llama_batch_allocr::seq_pos_min(llama_seq_id seq_id) const {\n    return seq_pos[seq_id].empty() ? -1 : *seq_pos[seq_id].begin();\n}\n\nllama_pos llama_batch_allocr::seq_pos_max(llama_seq_id seq_id) const {\n    return seq_pos[seq_id].empty() ? -1 : *seq_pos[seq_id].rbegin();\n}\n\nvoid llama_batch_allocr::split_reset() {\n    out_ids.clear();\n\n    n_used = 0;\n\n    used.clear();\n    used.resize(get_n_tokens(), false);\n}\n\nllama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) {\n    // find the first unused token\n    uint32_t cur_idx = 0;\n    while (cur_idx < used.size() && used[cur_idx]) {\n        ++cur_idx;\n    }\n\n    // we are done\n    if (cur_idx >= used.size()) {\n        return {};\n    }\n\n    std::vector<int32_t> idxs;\n\n    while (true) {\n        idxs.push_back(cur_idx);\n\n        used[cur_idx] = true;\n        ++n_used;\n\n        ++cur_idx;\n\n        if (cur_idx >= used.size()) {\n            break;\n        }\n\n        if (idxs.size() >= n_ubatch) {\n            break;\n        }\n    }\n\n    return ubatch_add(idxs, idxs.size(), false);\n}\n\nllama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential) {\n    if (sequential && has_cpl) {\n        LLAMA_LOG_ERROR(\"%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\\n\", __func__);\n\n        return {};\n    }\n\n    std::vector<seq_set_t> cur_seq_set;\n\n    llama_seq_id last_seq_id = -1;\n\n    // determine the non-overlapping sequence sets participating in this ubatch\n    for (int32_t i = 0; i < batch.n_tokens; ++i) {\n        if (used[i]) {\n            continue;\n        }\n\n        bool add = true;\n\n        for (uint32_t s = 0; s < cur_seq_set.size(); ++s) {\n            // no overlap with existing sequence sets:\n            if (!(cur_seq_set[s] & seq_set[i]).none()) {\n                add = false;\n                break;\n            }\n        }\n\n        // accept only increasing sequence ids\n        if (sequential) {\n            add = add && (cur_seq_set.empty() || batch.seq_id[i][0] == last_seq_id + 1);\n        }\n\n        if (add) {\n            cur_seq_set.push_back(seq_set[i]);\n\n            last_seq_id = batch.seq_id[i][0];\n\n            if (cur_seq_set.size() > n_ubatch) {\n                break;\n            }\n        }\n    }\n\n    const uint32_t n_seqs = cur_seq_set.size();\n\n    // we are done\n    if (n_seqs == 0) {\n        return {};\n    }\n\n    // the current batch index of each sequence set\n    std::vector<int32_t> cur_idx(n_seqs, 0);\n\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        while (used[seq_set_map[cur_seq_set[s]][cur_idx[s]]]) {\n            ++cur_idx[s];\n        }\n    }\n\n    // the list of batch indices for each sequence set\n    // at the end we will concat these to get the final ubatch\n    std::vector<idx_vec_t> idxs_per_seq(n_seqs);\n\n    while (true) {\n        // we can only add new n_seq_tokens tokens if all the sequence sets have at least one more unused token and\n        //   if we haven't reached n_ubatch\n        bool can_expand = true;\n\n        for (uint32_t s = 0; s < n_seqs; ++s) {\n            if (cur_idx[s] >= (int32_t) seq_set_map[cur_seq_set[s]].size()) {\n                can_expand = false;\n                break;\n            }\n        }\n\n        if (!can_expand) {\n            break;\n        }\n\n        for (uint32_t s = 0; s < n_seqs; ++s) {\n            const int32_t idx = seq_set_map[cur_seq_set[s]][cur_idx[s]];\n\n            idxs_per_seq[s].push_back(idx);\n\n            used[idx] = true;\n            ++n_used;\n\n            ++cur_idx[s];\n        }\n\n        if  ((idxs_per_seq[0].size() + 1)*n_seqs > n_ubatch) {\n            break;\n        }\n    }\n\n    // concat the per-sequence-set lists\n    std::vector<int32_t> idxs;\n\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        idxs.insert(idxs.end(), idxs_per_seq[s].begin(), idxs_per_seq[s].end());\n    }\n\n    return ubatch_add(idxs, n_seqs, true);\n}\n\nllama_ubatch llama_batch_allocr::split_seq(uint32_t n_ubatch) {\n    // find the first unused token\n    uint32_t cur_idx = 0;\n    while (cur_idx < used.size() && used[cur_idx]) {\n        ++cur_idx;\n    }\n\n    // we are done\n    if (cur_idx >= used.size()) {\n        return {};\n    }\n\n    // this is the starting sequence set\n    // we allow adding tokens only if their sequence set is a subset of the current sequence set\n    auto cur_seq_set = seq_set[cur_idx];\n\n    std::vector<int32_t> idxs;\n\n    while (true) {\n        idxs.push_back(cur_idx);\n\n        used[cur_idx] = true;\n        ++n_used;\n\n        if (idxs.size() >= n_ubatch) {\n            break;\n        }\n\n        do {\n            ++cur_idx;\n        } while (cur_idx < get_n_tokens() && (used[cur_idx] || ((cur_seq_set & seq_set[cur_idx]) != seq_set[cur_idx])));\n\n        if (cur_idx == get_n_tokens()) {\n            break;\n        }\n\n        cur_seq_set = seq_set[cur_idx];\n    }\n\n    return ubatch_add(idxs, 1, true);\n}\n\nvoid llama_batch_allocr::clear() {\n    n_outputs = 0;\n\n    batch = {};\n\n    pos       .clear();\n    n_seq_id  .clear();\n    seq_id    .clear();\n    seq_id_unq.clear();\n    output    .clear();\n\n    for (auto & cur : seq_pos) {\n        cur.clear();\n    }\n\n    for (auto & cur : seq_cpl) {\n        std::fill(cur.begin(), cur.end(), false);\n    }\n\n    seq_set.clear();\n\n    seq_set_map.clear();\n\n    std::fill(seq_idx.begin(), seq_idx.end(), -1);\n}\n\nllama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, uint32_t n_seqs, bool equal_seqs) {\n    const uint32_t n_tokens = idxs.size();\n\n    assert(n_tokens%n_seqs == 0);\n\n    auto udata = std::make_shared<llama_ubatch::data_t>();\n\n    const int64_t n_embd_all = batch.embd ? (int64_t) n_tokens*n_embd : 0;\n    const int64_t n_pos_all  =              (int64_t) n_tokens*n_pos_per_embd;\n\n    udata->token     .resize(n_tokens);\n    udata->embd      .resize(n_embd_all);\n    udata->pos       .resize(n_pos_all);\n    udata->n_seq_id  .resize(n_tokens);\n    udata->seq_id    .resize(n_tokens);\n    udata->seq_id_unq.resize(0);\n    udata->seq_idx   .resize(LLAMA_MAX_SEQ, -1);\n    udata->output    .resize(n_tokens);\n\n    udata->seq_id_data.reserve(n_tokens);\n\n    seq_set_t seq_set_unq;\n\n    for (size_t i = 0; i < idxs.size(); ++i) {\n        if (batch.token) {\n            udata->token[i] = batch.token[idxs[i]];\n        }\n\n        if (batch.embd) {\n            memcpy(udata->embd.data() + i*n_embd, batch.embd + (int64_t) idxs[i]*n_embd, n_embd*sizeof(float));\n        }\n\n        for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {\n            // if we are using M-RoPE\n            //     if the current batch is text, we need to broadcast the same position across all RoPE sections\n            //     otherwise, the input batch is image embeddings, we copy the positions as-is\n            // if we are not using M-RoPE, there is only one position per token (this loop runs only once)\n            size_t src_off = batch.token ? 0 : j*batch.n_tokens;\n            udata->pos[j*n_tokens + i] = batch.pos[src_off + idxs[i]];\n        }\n\n        udata->n_seq_id[i] = batch.n_seq_id[idxs[i]];\n        udata->output[i]   = batch.logits[idxs[i]];\n\n        for (int s = 0; s < udata->n_seq_id[i]; ++s) {\n            const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];\n\n            udata->seq_id_data.push_back(seq_id);\n            seq_set_unq.set(seq_id);\n        }\n\n        if (udata->output[i]) {\n            out_ids.push_back(idxs[i]);\n        }\n    }\n\n    llama_seq_id * seq_id_ptr = udata->seq_id_data.data();\n    for (size_t i = 0; i < idxs.size(); ++i) {\n        udata->seq_id[i] = seq_id_ptr;\n        seq_id_ptr += udata->n_seq_id[i];\n    }\n\n    for (uint32_t s = 0; s < n_seq_max; ++s) {\n        if (seq_set_unq.test(s)) {\n            udata->seq_idx[s] = udata->seq_id_unq.size();\n            udata->seq_id_unq.push_back(s);\n        }\n    }\n\n    llama_ubatch res {\n        /*.b_equal_seqs =*/ equal_seqs,\n        /*.n_tokens     =*/ n_tokens,\n        /*.n_seq_tokens =*/ n_tokens/n_seqs,\n        /*.n_seqs       =*/ n_seqs,\n        /*.n_seqs_unq   =*/ (uint32_t) udata->seq_id_unq.size(),\n        /*.n_pos        =*/ n_pos_per_embd,\n\n        /*.token        =*/ batch.token ? udata->token.data() : nullptr,\n        /*.embd         =*/ batch.embd ? udata->embd.data() : nullptr,\n        /*.pos          =*/ udata->pos.data(),\n        /*.n_seq_id     =*/ udata->n_seq_id.data(),\n        /*.seq_id       =*/ udata->seq_id.data(),\n        /*.seq_id_unq   =*/ udata->seq_id_unq.data(),\n        /*.seq_idx      =*/ udata->seq_idx.data(),\n        /*.output       =*/ udata->output.data(),\n        /*.data         =*/ std::move(udata),\n    };\n\n    if (debug > 0) {\n        LLAMA_LOG_DEBUG(\"%s: added ubatch to split:\\n\", __func__);\n\n        ubatch_print(res, debug);\n    }\n\n    return res;\n}\n\nvoid llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {\n    if (debug > 0) {\n        LLAMA_LOG_DEBUG(\"%s:   equal_seqs   = %d\\n\", __func__, ubatch.equal_seqs());\n        LLAMA_LOG_DEBUG(\"%s:   n_tokens     = %d\\n\", __func__, ubatch.n_tokens);\n        LLAMA_LOG_DEBUG(\"%s:   n_seq_tokens = %d\\n\", __func__, ubatch.n_seq_tokens);\n        LLAMA_LOG_DEBUG(\"%s:   n_seqs       = %d\\n\", __func__, ubatch.n_seqs);\n        LLAMA_LOG_DEBUG(\"%s:   n_seqs_unq   = %d\\n\", __func__, ubatch.n_seqs_unq);\n\n        std::stringstream ss_seq_id_unq;\n        std::stringstream ss_seq_idx;\n\n        ss_seq_id_unq << \"[ \";\n        ss_seq_idx << \"[\";\n\n        for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n            ss_seq_id_unq << ubatch.seq_id_unq[s] << \" \";\n        }\n\n        for (uint32_t s = 0; s < LLAMA_MAX_SEQ; ++s) {\n            if (ubatch.seq_idx[s] >= 0) {\n                ss_seq_idx << ubatch.seq_idx[s]%10;\n            } else {\n                ss_seq_idx << \".\";\n            }\n        }\n\n        ss_seq_id_unq << \"]\";\n        ss_seq_idx    << \"]\";\n\n        LLAMA_LOG_DEBUG(\"%s:   token      = %p\\n\", __func__, (void *) ubatch.token);\n        LLAMA_LOG_DEBUG(\"%s:   embd       = %p\\n\", __func__, (void *) ubatch.embd);\n        LLAMA_LOG_DEBUG(\"%s:   pos        = %p\\n\", __func__, (void *) ubatch.pos);\n        LLAMA_LOG_DEBUG(\"%s:   n_seq_id   = %p\\n\", __func__, (void *) ubatch.n_seq_id);\n        LLAMA_LOG_DEBUG(\"%s:   seq_id     = %p\\n\", __func__, (void *) ubatch.seq_id);\n        LLAMA_LOG_DEBUG(\"%s:   seq_id_unq = %s\\n\", __func__, ss_seq_id_unq.str().c_str());\n        LLAMA_LOG_DEBUG(\"%s:   seq_idx    = %s\\n\", __func__, ss_seq_idx.str().c_str());\n        LLAMA_LOG_DEBUG(\"%s:   output     = %p\\n\", __func__, (void *) ubatch.output);\n        LLAMA_LOG_DEBUG(\"%s:   n_outputs  = %d\\n\", __func__, n_outputs);\n\n        if (debug > 1) {\n            int seq_id_max = 0;\n            for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {\n                for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {\n                    for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {\n                        seq_id_max = std::max(seq_id_max, ubatch.seq_id[i][s]);\n                    }\n                }\n            }\n            ++seq_id_max;\n\n            LLAMA_LOG_DEBUG(\"%s:   token     = [\\n\", __func__);\n            for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {\n                std::vector<int8_t> seq_id(seq_id_max);\n\n                for (int s = 0; s < ubatch.n_seq_id[i]; ++s) {\n                    seq_id[ubatch.seq_id[i][s]] = 1;\n                }\n\n                std::stringstream ss;\n                for (int s = 0; s < seq_id_max; ++s) {\n                    if (seq_id[s]) {\n                        ss << s%10;\n                    } else {\n                        ss << \".\";\n                    }\n                }\n\n                if (ubatch.token) {\n                    LLAMA_LOG_DEBUG(\"%s:  %4d: id = %6d (%16s), pos = %4d, n_seq_id = %2d, seq_id = [%s], output = %d\\n\",\n                            __func__, i, ubatch.token[i], vocab->token_to_piece(ubatch.token[i]).c_str(),\n                            ubatch.pos[i], ubatch.n_seq_id[i], ss.str().c_str(), ubatch.output[i]);\n                } else {\n                    LLAMA_LOG_DEBUG(\"%s:  %4d: [embd], pos = %4d, n_seq_id = %2d, seq_id = [%s], output = %d\\n\",\n                            __func__, i, ubatch.pos[i], ubatch.n_seq_id[i], ss.str().c_str(), ubatch.output[i]);\n                }\n            }\n            LLAMA_LOG_DEBUG(\"%s:   ]\\n\", __func__);\n        }\n    }\n}\n\n//\n// interface implementation\n//\n\nstruct llama_batch llama_batch_get_one(\n             llama_token * tokens,\n                 int32_t   n_tokens) {\n    return {\n        /*n_tokens =*/ n_tokens,\n        /*tokens   =*/ tokens,\n        /*embd     =*/ nullptr,\n        /*pos      =*/ nullptr,\n        /*n_seq_id =*/ nullptr,\n        /*seq_id   =*/ nullptr,\n        /*logits   =*/ nullptr,\n    };\n}\n\nstruct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_t n_seq_max) {\n    llama_batch batch = {\n        /*n_tokens =*/ 0,\n        /*tokens   =*/ nullptr,\n        /*embd     =*/ nullptr,\n        /*pos      =*/ nullptr,\n        /*n_seq_id =*/ nullptr,\n        /*seq_id   =*/ nullptr,\n        /*logits   =*/ nullptr,\n    };\n\n    if (embd) {\n        batch.embd = (float *) malloc(sizeof(float) * n_tokens_alloc * embd);\n    } else {\n        batch.token = (llama_token *) malloc(sizeof(llama_token) * n_tokens_alloc);\n    }\n\n    batch.pos      = (llama_pos *)     malloc(sizeof(llama_pos)      * n_tokens_alloc);\n    batch.n_seq_id = (int32_t *)       malloc(sizeof(int32_t)        * n_tokens_alloc);\n    batch.seq_id   = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * (n_tokens_alloc + 1));\n    for (int i = 0; i < n_tokens_alloc; ++i) {\n        batch.seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);\n    }\n    batch.seq_id[n_tokens_alloc] = nullptr;\n\n    batch.logits   = (int8_t *)        malloc(sizeof(int8_t)         * n_tokens_alloc);\n\n    return batch;\n}\n\nvoid llama_batch_free(struct llama_batch batch) {\n    if (batch.token)    free(batch.token);\n    if (batch.embd)     free(batch.embd);\n    if (batch.pos)      free(batch.pos);\n    if (batch.n_seq_id) free(batch.n_seq_id);\n    if (batch.seq_id) {\n        for (int i = 0; batch.seq_id[i] != nullptr; ++i) {\n            free(batch.seq_id[i]);\n        }\n        free(batch.seq_id);\n    }\n    if (batch.logits)   free(batch.logits);\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-batch.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include \"llama-cparams.h\"\n\n#include <array>\n#include <vector>\n#include <set>\n#include <bitset>\n#include <memory>\n#include <unordered_map>\n\n// keep this struct lightweight\nstruct llama_ubatch {\n    bool equal_seqs() const {\n        return b_equal_seqs != 0;\n    }\n\n    // typical for M-RoPE cases:\n    //   0 - sequantial position of the tokens/embeddings in the sequence\n    //   1 - y position in the image\n    //   2 - x position in the image\n    //   3 - other\n    bool is_pos_2d() const {\n        // TODO @ngxson : we may need to check for model arch when more models use >1 positions\n        return n_pos >= 3;\n    }\n\n    uint32_t b_equal_seqs; // note: this is a boolean, but we use an int32_t for alignment\n                           //       otherwise address sanitizer complains\n    // TODO: whole_seqs for embeddings?\n\n    uint32_t n_tokens;     // total tokens (n_seq_tokens * n_seqs)\n    uint32_t n_seq_tokens; // tokens per sequence set\n    uint32_t n_seqs;       // sequence sets in the ubatch\n    uint32_t n_seqs_unq;   // unique sequence ids in the ubatch\n    uint32_t n_pos;        // number of position inputs for each token/embedding\n\n    // seq_id_unq: unique sequence ids in the ubatch\n    // seq_idx:    indices of the unique sequence ids in the ubatch in [0, n_seqs_unq)\n    //             used for extracting sequence pooled embeddings\n\n    //                          // size               | idx | val\n    llama_token  *  token;      // [n_tokens]         | i   | id, token\n    float        *  embd;       // [n_embd, n_tokens] | i   | embd\n    llama_pos    *  pos;        // [n_tokens*n_pos]   | i   | pos\n    int32_t      *  n_seq_id;   // [n_tokens]         | i   | -\n    llama_seq_id ** seq_id;     // [n_tokens]         | s   | s0, s1, seq_id\n    llama_seq_id *  seq_id_unq; // [n_seqs_unq]       | s   | seq_id\n    int32_t      *  seq_idx;    // [LLAMA_MAX_SEQ]    | -   | seq_idx\n    int8_t       *  output;     // [n_tokens]         | i   | -\n\n    struct data_t {\n        std::vector<llama_token>    token;\n        std::vector<float>          embd;\n        std::vector<llama_pos>      pos;\n        std::vector<int32_t>        n_seq_id;\n        std::vector<llama_seq_id *> seq_id;      // these point into the seq_id_data below\n        std::vector<llama_seq_id>   seq_id_unq;\n        std::vector<int32_t>        seq_idx;\n        std::vector<int8_t>         output;\n\n        std::vector<llama_seq_id> seq_id_data;\n    };\n\n    // the llama_ubatch pointers above point to this data if set. otherwise - point to external non-owning data\n    std::shared_ptr<data_t> data;\n};\n\n// a helper for sanitizing, fulfilling and splitting a batch\nclass llama_batch_allocr {\npublic:\n    llama_batch_allocr(uint32_t n_pos_per_embd);\n\n    // sanitize and auto-gen missing data in the input batch\n    // memory is optional. if provided will be used to check for sequence continuity and to determine the positions\n    bool init(\n            const llama_batch & batch_inp,\n            const llama_vocab & vocab,\n            const llama_memory_i * memory,\n            uint32_t n_embd,\n            uint32_t n_seq_max,\n            bool output_all);\n\n    const llama_batch & get_batch() const;\n\n    uint32_t get_n_tokens()  const;\n    uint32_t get_n_outputs() const;\n    uint32_t get_n_used()    const;\n\n    // the array of output indices in the order they were encountered during the ubatch splitting\n    std::vector<int32_t> & get_out_ids();\n\n    // min/max positions of each sequence in the current ubatch\n    llama_pos seq_pos_min(llama_seq_id seq_id) const;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const;\n\n    // call once before splitting the batch to reset the internal state\n    void split_reset();\n\n    // simple split, unknown number of sequence sets of unequal lengths\n    llama_ubatch split_simple(uint32_t n_ubatch);\n\n    // make ubatches of equal-length sequences sets\n    // if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids\n    llama_ubatch split_equal(uint32_t n_ubatch, bool sequential);\n\n    // sequence-set-wise split - each ubatch contains a single sequence-set\n    llama_ubatch split_seq(uint32_t n_ubatch);\n\n    // a helper method for creating a well-defined ubatch of tokens\n    // TODO: support embeddings if needed in the future\n    llama_ubatch ubatch_reserve(uint32_t n_seq_tokens, uint32_t n_seqs);\n\nprivate:\n    void clear();\n\n    // create the next ubatch based on the provided batch indices (idxs) and the number of sequence sets (n_seqs)\n    // return llama_ubatch.n_tokens == 0 if the entire batch was consumed\n    llama_ubatch ubatch_add(const std::vector<int32_t> & idxs, uint32_t n_seqs, bool equal_seqs);\n\n    // for debugging, start with LLAMA_BATCH_DEBUG=2\n    void ubatch_print(const llama_ubatch & ubatch, int debug);\n\n    llama_batch batch;\n\n    // only for debugging purposes\n    const llama_vocab * vocab;\n\n    // TODO: this is more of a temporary solution until we have a better way to handle multiple positions per token/embd\n    //       ref: https://github.com/ggml-org/llama.cpp/issues/13694#issuecomment-2983871762\n    const uint32_t n_pos_per_embd;\n\n    uint32_t n_embd;\n    uint32_t n_seq_max;\n    uint32_t n_outputs;\n\n    std::array<llama_seq_id, 1> seq_id_0 = {{ 0 }}; // default sequence id\n\n    std::vector<llama_pos>      pos;\n    std::vector<int32_t>        n_seq_id;\n    std::vector<llama_seq_id *> seq_id;\n    std::vector<llama_seq_id>   seq_id_unq;\n    std::vector<int32_t>        seq_idx;\n    std::vector<int8_t>         output;\n\n    using pos_set_t = std::set<llama_pos>;\n    using seq_cpl_t = std::vector<bool>;\n\n    // helper flag to quickly determine if there are any coupled sequences in the batch\n    bool has_cpl = false;\n\n    std::vector<pos_set_t> seq_pos; // seq_pos[s]: the set of positions in sequence s\n    std::vector<seq_cpl_t> seq_cpl; // seq_cpl[s0][s1]: if sequence s0 is coupled to sequence s1\n\n    using idx_vec_t = std::vector<int32_t>;\n    using seq_set_t = std::bitset<LLAMA_MAX_SEQ>;\n\n    std::vector<seq_set_t> seq_set; // seq_set[i]: the sequence set of token i\n\n    std::unordered_map<seq_set_t, idx_vec_t> seq_set_map; // the indices at which the sequence set appears\n\n    // batch indices of the output\n    std::vector<int32_t> out_ids;\n\n    uint32_t n_used;\n\n    // used[i] indicates if token i has already been used in a previous ubatch\n    std::vector<bool> used;\n\n    int debug;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-chat.cpp",
    "content": "#include \"llama-chat.h\"\n\n#include \"llama.h\"\n\n#include <map>\n#include <sstream>\n#include <algorithm>\n\n#if __cplusplus >= 202000L\n    #define LU8(x) (const char*)(u8##x)\n#else\n    #define LU8(x) u8##x\n#endif\n\n// trim whitespace from the beginning and end of a string\nstatic std::string trim(const std::string & str) {\n    size_t start = 0;\n    size_t end = str.size();\n    while (start < end && isspace(static_cast<unsigned char>(str[start]))) {\n        start += 1;\n    }\n    while (end > start && isspace(static_cast<unsigned char>(str[end - 1]))) {\n        end -= 1;\n    }\n    return str.substr(start, end - start);\n}\n\nstatic const std::map<std::string, llm_chat_template> LLM_CHAT_TEMPLATES = {\n    { \"chatml\",            LLM_CHAT_TEMPLATE_CHATML            },\n    { \"llama2\",            LLM_CHAT_TEMPLATE_LLAMA_2           },\n    { \"llama2-sys\",        LLM_CHAT_TEMPLATE_LLAMA_2_SYS       },\n    { \"llama2-sys-bos\",    LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS   },\n    { \"llama2-sys-strip\",  LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP },\n    { \"mistral-v1\",        LLM_CHAT_TEMPLATE_MISTRAL_V1        },\n    { \"mistral-v3\",        LLM_CHAT_TEMPLATE_MISTRAL_V3        },\n    { \"mistral-v3-tekken\", LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN },\n    { \"mistral-v7\",        LLM_CHAT_TEMPLATE_MISTRAL_V7        },\n    { \"mistral-v7-tekken\", LLM_CHAT_TEMPLATE_MISTRAL_V7_TEKKEN },\n    { \"phi3\",              LLM_CHAT_TEMPLATE_PHI_3             },\n    { \"phi4\",              LLM_CHAT_TEMPLATE_PHI_4             },\n    { \"falcon3\",           LLM_CHAT_TEMPLATE_FALCON_3          },\n    { \"zephyr\",            LLM_CHAT_TEMPLATE_ZEPHYR            },\n    { \"monarch\",           LLM_CHAT_TEMPLATE_MONARCH           },\n    { \"gemma\",             LLM_CHAT_TEMPLATE_GEMMA             },\n    { \"orion\",             LLM_CHAT_TEMPLATE_ORION             },\n    { \"openchat\",          LLM_CHAT_TEMPLATE_OPENCHAT          },\n    { \"vicuna\",            LLM_CHAT_TEMPLATE_VICUNA            },\n    { \"vicuna-orca\",       LLM_CHAT_TEMPLATE_VICUNA_ORCA       },\n    { \"deepseek\",          LLM_CHAT_TEMPLATE_DEEPSEEK          },\n    { \"deepseek2\",         LLM_CHAT_TEMPLATE_DEEPSEEK_2        },\n    { \"deepseek3\",         LLM_CHAT_TEMPLATE_DEEPSEEK_3        },\n    { \"command-r\",         LLM_CHAT_TEMPLATE_COMMAND_R         },\n    { \"llama3\",            LLM_CHAT_TEMPLATE_LLAMA_3           },\n    { \"chatglm3\",          LLM_CHAT_TEMPLATE_CHATGLM_3         },\n    { \"chatglm4\",          LLM_CHAT_TEMPLATE_CHATGLM_4         },\n    { \"glmedge\",           LLM_CHAT_TEMPLATE_GLMEDGE           },\n    { \"minicpm\",           LLM_CHAT_TEMPLATE_MINICPM           },\n    { \"exaone3\",           LLM_CHAT_TEMPLATE_EXAONE_3          },\n    { \"exaone4\",           LLM_CHAT_TEMPLATE_EXAONE_4          },\n    { \"exaone-moe\",        LLM_CHAT_TEMPLATE_EXAONE_MOE        },\n    { \"rwkv-world\",        LLM_CHAT_TEMPLATE_RWKV_WORLD        },\n    { \"granite\",           LLM_CHAT_TEMPLATE_GRANITE           },\n    { \"gigachat\",          LLM_CHAT_TEMPLATE_GIGACHAT          },\n    { \"megrez\",            LLM_CHAT_TEMPLATE_MEGREZ            },\n    { \"yandex\",            LLM_CHAT_TEMPLATE_YANDEX            },\n    { \"bailing\",           LLM_CHAT_TEMPLATE_BAILING           },\n    { \"bailing-think\",     LLM_CHAT_TEMPLATE_BAILING_THINK     },\n    { \"bailing2\",          LLM_CHAT_TEMPLATE_BAILING2          },\n    { \"llama4\",            LLM_CHAT_TEMPLATE_LLAMA4            },\n    { \"smolvlm\",           LLM_CHAT_TEMPLATE_SMOLVLM           },\n    { \"hunyuan-moe\",       LLM_CHAT_TEMPLATE_HUNYUAN_MOE       },\n    { \"gpt-oss\",           LLM_CHAT_TEMPLATE_OPENAI_MOE        },\n    { \"hunyuan-dense\",     LLM_CHAT_TEMPLATE_HUNYUAN_DENSE     },\n    { \"kimi-k2\",           LLM_CHAT_TEMPLATE_KIMI_K2           },\n    { \"seed_oss\",          LLM_CHAT_TEMPLATE_SEED_OSS          },\n    { \"grok-2\",            LLM_CHAT_TEMPLATE_GROK_2            },\n    { \"pangu-embedded\",    LLM_CHAT_TEMPLATE_PANGU_EMBED       },\n    { \"solar-open\",        LLM_CHAT_TEMPLATE_SOLAR_OPEN        },\n};\n\nllm_chat_template llm_chat_template_from_str(const std::string & name) {\n    return LLM_CHAT_TEMPLATES.at(name);\n}\n\nllm_chat_template llm_chat_detect_template(const std::string & tmpl) {\n    try {\n        return llm_chat_template_from_str(tmpl);\n    } catch (const std::out_of_range &) {\n        // ignore\n    }\n\n    auto tmpl_contains = [&tmpl](const char * haystack) -> bool {\n        return tmpl.find(haystack) != std::string::npos;\n    };\n    if (tmpl_contains(\"<|im_start|>\")) {\n        return tmpl_contains(\"<|im_sep|>\")\n            ? LLM_CHAT_TEMPLATE_PHI_4\n            : tmpl_contains(\"<end_of_utterance>\")\n                ? LLM_CHAT_TEMPLATE_SMOLVLM // SmolVLM uses <|im_start|> as BOS, but it is NOT chatml\n                : LLM_CHAT_TEMPLATE_CHATML;\n    } else if (tmpl.find(\"mistral\") == 0 || tmpl_contains(\"[INST]\")) {\n        if (tmpl_contains(\"[SYSTEM_PROMPT]\")) {\n            return LLM_CHAT_TEMPLATE_MISTRAL_V7;\n        } else if (\n            // catches official 'v1' template\n            tmpl_contains(\"' [INST] ' + system_message\")\n            // catches official 'v3' and 'v3-tekken' templates\n            || tmpl_contains(\"[AVAILABLE_TOOLS]\")\n        ) {\n            // Official mistral 'v1', 'v3' and 'v3-tekken' templates\n            // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md\n            // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md\n            if (tmpl_contains(\" [INST]\")) {\n                return LLM_CHAT_TEMPLATE_MISTRAL_V1;\n            } else if (tmpl_contains(\"\\\"[INST]\\\"\")) {\n                return LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN;\n            }\n            return LLM_CHAT_TEMPLATE_MISTRAL_V3;\n        } else {\n            // llama2 template and its variants\n            // [variant] support system message\n            // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2\n            bool support_system_message = tmpl_contains(\"<<SYS>>\");\n            bool add_bos_inside_history = tmpl_contains(\"bos_token + '[INST]\");\n            bool strip_message = tmpl_contains(\"content.strip()\");\n            if (strip_message) {\n                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;\n            } else if (add_bos_inside_history) {\n                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;\n            } else if (support_system_message) {\n                return LLM_CHAT_TEMPLATE_LLAMA_2_SYS;\n            } else {\n                return LLM_CHAT_TEMPLATE_LLAMA_2;\n            }\n        }\n    } else if (tmpl_contains(\"<|assistant|>\") && tmpl_contains(\"<|end|>\")) {\n        return LLM_CHAT_TEMPLATE_PHI_3;\n    } else if (tmpl_contains(\"[gMASK]<sop>\")) {\n        return LLM_CHAT_TEMPLATE_CHATGLM_4;\n    } else if (tmpl_contains(\"<|assistant|>\") && tmpl_contains(\"<|user|>\")) {\n        if (tmpl_contains(\"<|tool_declare|>\")) {\n            return LLM_CHAT_TEMPLATE_EXAONE_MOE;\n        }\n        return tmpl_contains(\"</s>\") ? LLM_CHAT_TEMPLATE_FALCON_3 : LLM_CHAT_TEMPLATE_GLMEDGE;\n    } else if (tmpl_contains(\"<|{{ item['role'] }}|>\") && tmpl_contains(\"<|begin_of_image|>\")) {\n        return LLM_CHAT_TEMPLATE_GLMEDGE;\n    } else if (tmpl_contains(\"<|user|>\") && tmpl_contains(\"<|endoftext|>\")) {\n        return LLM_CHAT_TEMPLATE_ZEPHYR;\n    } else if (tmpl_contains(\"bos_token + message['role']\")) {\n        return LLM_CHAT_TEMPLATE_MONARCH;\n    } else if (tmpl_contains(\"<start_of_turn>\")) {\n        return LLM_CHAT_TEMPLATE_GEMMA;\n    } else if (tmpl_contains(\"'\\\\n\\\\nAssistant: ' + eos_token\")) {\n        // OrionStarAI/Orion-14B-Chat\n        return LLM_CHAT_TEMPLATE_ORION;\n    } else if (tmpl_contains(\"GPT4 Correct \")) {\n        // openchat/openchat-3.5-0106\n        return LLM_CHAT_TEMPLATE_OPENCHAT;\n    } else if (tmpl_contains(\"USER: \") && tmpl_contains(\"ASSISTANT: \")) {\n        // eachadea/vicuna-13b-1.1 (and Orca variant)\n        if (tmpl_contains(\"SYSTEM: \")) {\n            return LLM_CHAT_TEMPLATE_VICUNA_ORCA;\n        }\n        return LLM_CHAT_TEMPLATE_VICUNA;\n    } else if (tmpl_contains(\"### Instruction:\") && tmpl_contains(\"<|EOT|>\")) {\n        // deepseek-ai/deepseek-coder-33b-instruct\n        return LLM_CHAT_TEMPLATE_DEEPSEEK;\n    } else if (tmpl_contains(\"<|START_OF_TURN_TOKEN|>\") && tmpl_contains(\"<|USER_TOKEN|>\")) {\n        // CohereForAI/c4ai-command-r-plus\n        return LLM_CHAT_TEMPLATE_COMMAND_R;\n    } else if (tmpl_contains(\"<|start_header_id|>\") && tmpl_contains(\"<|end_header_id|>\")) {\n        return LLM_CHAT_TEMPLATE_LLAMA_3;\n    } else if (tmpl_contains(\"[gMASK]sop\")) {\n        // chatglm3-6b\n        return LLM_CHAT_TEMPLATE_CHATGLM_3;\n    } else if (tmpl_contains(LU8(\"<用户>\"))) {\n        // MiniCPM-3B-OpenHermes-2.5-v2-GGUF\n        return LLM_CHAT_TEMPLATE_MINICPM;\n    } else if (tmpl_contains(\"'Assistant: ' + message['content'] + eos_token\")) {\n        return LLM_CHAT_TEMPLATE_DEEPSEEK_2;\n    } else if (tmpl_contains(LU8(\"<｜Assistant｜>\")) && tmpl_contains(LU8(\"<｜User｜>\")) && tmpl_contains(LU8(\"<｜end▁of▁sentence｜>\"))) {\n        return LLM_CHAT_TEMPLATE_DEEPSEEK_3;\n    } else if (tmpl_contains(\"[|system|]\") && tmpl_contains(\"[|assistant|]\") && tmpl_contains(\"[|endofturn|]\")) {\n        if (tmpl_contains(\"[|tool|]\")) {\n            return LLM_CHAT_TEMPLATE_EXAONE_4;\n        }\n        // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb\n        // EXAONE-3.0-7.8B-Instruct\n        return LLM_CHAT_TEMPLATE_EXAONE_3;\n    } else if (tmpl_contains(\"rwkv-world\") || tmpl_contains(\"{{- 'User: ' + message['content']|trim + '\\\\n\\\\n' -}}\")) {\n        return LLM_CHAT_TEMPLATE_RWKV_WORLD;\n    } else if (tmpl_contains(\"<|start_of_role|>\")) {\n        return LLM_CHAT_TEMPLATE_GRANITE;\n    } else if (tmpl_contains(\"message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1]\")) {\n        return LLM_CHAT_TEMPLATE_GIGACHAT;\n    } else if (tmpl_contains(\"<|role_start|>\")) {\n        return LLM_CHAT_TEMPLATE_MEGREZ;\n    } else if (tmpl_contains(\" Ассистент:\")) {\n        return LLM_CHAT_TEMPLATE_YANDEX;\n    } else if (tmpl_contains(\"<role>ASSISTANT</role>\") && tmpl_contains(\"'HUMAN'\")) {\n        return LLM_CHAT_TEMPLATE_BAILING;\n    } else if (tmpl_contains(\"<role>ASSISTANT</role>\") && tmpl_contains(\"\\\"HUMAN\\\"\") && tmpl_contains(\"<think>\")) {\n        return LLM_CHAT_TEMPLATE_BAILING_THINK;\n    } else if (tmpl_contains(\"<role>ASSISTANT</role>\") && tmpl_contains(\"<role>HUMAN</role>\") && tmpl_contains(\"<|role_end|>\")) {\n        return LLM_CHAT_TEMPLATE_BAILING2;\n    } else if (tmpl_contains(\"<|header_start|>\") && tmpl_contains(\"<|header_end|>\")) {\n        return LLM_CHAT_TEMPLATE_LLAMA4;\n    } else if (tmpl_contains(\"<|endofuserprompt|>\")) {\n        return LLM_CHAT_TEMPLATE_DOTS1;\n    } else if (tmpl_contains(\"<|extra_0|>\") && tmpl_contains(\"<|extra_4|>\")) {\n        return LLM_CHAT_TEMPLATE_HUNYUAN_MOE;\n    } else if (tmpl_contains(\"<|start|>\") && tmpl_contains(\"<|channel|>\")) {\n        return LLM_CHAT_TEMPLATE_OPENAI_MOE;\n    } else if (tmpl_contains(\"<｜hy_Assistant｜>\") && tmpl_contains(\"<｜hy_place▁holder▁no▁3｜>\")) {\n        return LLM_CHAT_TEMPLATE_HUNYUAN_DENSE;\n    } else if (tmpl_contains(\"<|im_assistant|>assistant<|im_middle|>\")) {\n        return LLM_CHAT_TEMPLATE_KIMI_K2;\n    } else if (tmpl_contains(\"<seed:bos>\")) {\n        return LLM_CHAT_TEMPLATE_SEED_OSS;\n    } else if (tmpl_contains(\"'Assistant: '  + message['content'] + '<|separator|>\")) {\n        return LLM_CHAT_TEMPLATE_GROK_2;\n    } else if (tmpl_contains(LU8(\"[unused9]系统：[unused10]\"))) {\n        return LLM_CHAT_TEMPLATE_PANGU_EMBED;\n    } else if (tmpl_contains(\"<|begin|>\") && tmpl_contains(\"<|end|>\") && tmpl_contains(\"<|content|>\")) {\n        return LLM_CHAT_TEMPLATE_SOLAR_OPEN;\n    }\n    return LLM_CHAT_TEMPLATE_UNKNOWN;\n}\n\n// Simple version of \"llama_apply_chat_template\" that only works with strings\n// This function uses heuristic checks to determine commonly used template. It is not a jinja parser.\nint32_t llm_chat_apply_template(\n    llm_chat_template tmpl,\n    const std::vector<const llama_chat_message *> & chat,\n    std::string & dest, bool add_ass) {\n    // Taken from the research: https://github.com/ggml-org/llama.cpp/issues/5527\n    std::stringstream ss;\n    if (tmpl == LLM_CHAT_TEMPLATE_CHATML) {\n        // chatml template\n        for (auto message : chat) {\n            ss << \"<|im_start|>\" << message->role << \"\\n\" << message->content << \"<|im_end|>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<|im_start|>assistant\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7 || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7_TEKKEN) {\n        // Official mistral 'v7' template\n        // See: https://huggingface.co/mistralai/Mistral-Large-Instruct-2411#basic-instruct-template-v7\n        //      https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503#basic-instruct-template-v7-tekken\n        const char * trailing_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V7 ? \" \" : \"\";\n        for (auto message : chat) {\n            std::string role(message->role);\n            std::string content(message->content);\n            if (role == \"system\") {\n                ss << \"[SYSTEM_PROMPT]\" << trailing_space << content << \"[/SYSTEM_PROMPT]\";\n            } else if (role == \"user\") {\n                ss << \"[INST]\" << trailing_space << content << \"[/INST]\";\n            } else {\n                ss << trailing_space << content << \"</s>\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1\n            || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3\n            || tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN) {\n        // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/chat_templates.md\n        // See: https://github.com/mistralai/cookbook/blob/main/concept-deep-dive/tokenization/templates.md\n        std::string leading_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V1 ? \" \" : \"\";\n        std::string trailing_space = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN ? \"\" : \" \";\n        bool trim_assistant_message = tmpl == LLM_CHAT_TEMPLATE_MISTRAL_V3;\n        bool is_inside_turn = false;\n        for (auto message : chat) {\n            if (!is_inside_turn) {\n                ss << leading_space << \"[INST]\" << trailing_space;\n                is_inside_turn = true;\n            }\n            std::string role(message->role);\n            std::string content(message->content);\n            if (role == \"system\") {\n                ss << content << \"\\n\\n\";\n            } else if (role == \"user\") {\n                ss << content << leading_space << \"[/INST]\";\n            } else {\n                ss << trailing_space << (trim_assistant_message ? trim(content) : content) << \"</s>\";\n                is_inside_turn = false;\n            }\n        }\n    } else if (\n            tmpl == LLM_CHAT_TEMPLATE_LLAMA_2\n            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS\n            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS\n            || tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP) {\n        // llama2 template and its variants\n        // [variant] support system message\n        // See: https://huggingface.co/blog/llama2#how-to-prompt-llama-2\n        bool support_system_message = tmpl != LLM_CHAT_TEMPLATE_LLAMA_2;\n        // [variant] add BOS inside history\n        bool add_bos_inside_history = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS;\n        // [variant] trim spaces from the input message\n        bool strip_message = tmpl == LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP;\n        // construct the prompt\n        bool is_inside_turn = true; // skip BOS at the beginning\n        ss << \"[INST] \";\n        for (auto message : chat) {\n            std::string content = strip_message ? trim(message->content) : message->content;\n            std::string role(message->role);\n            if (!is_inside_turn) {\n                is_inside_turn = true;\n                ss << (add_bos_inside_history ? \"<s>[INST] \" : \"[INST] \");\n            }\n            if (role == \"system\") {\n                if (support_system_message) {\n                    ss << \"<<SYS>>\\n\" << content << \"\\n<</SYS>>\\n\\n\";\n                } else {\n                    // if the model does not support system message, we still include it in the first message, but without <<SYS>>\n                    ss << content << \"\\n\";\n                }\n            } else if (role == \"user\") {\n                ss << content << \" [/INST]\";\n            } else {\n                ss << content << \"</s>\";\n                is_inside_turn = false;\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_PHI_3) {\n        // Phi 3\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|\" << role << \"|>\\n\" << message->content << \"<|end|>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_PHI_4) {\n        // chatml template\n        for (auto message : chat) {\n            ss << \"<|im_start|>\" << message->role << \"<|im_sep|>\" << message->content << \"<|im_end|>\";\n        }\n        if (add_ass) {\n            ss << \"<|im_start|>assistant<|im_sep|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_FALCON_3) {\n        // Falcon 3\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|\" << role << \"|>\\n\" << message->content << \"\\n\";\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_ZEPHYR) {\n        // zephyr template\n        for (auto message : chat) {\n            ss << \"<|\" << message->role << \"|>\" << \"\\n\" << message->content << \"<|endoftext|>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_MONARCH) {\n        // mlabonne/AlphaMonarch-7B template (the <s> is included inside history)\n        for (auto message : chat) {\n            std::string bos = (message == chat.front()) ? \"\" : \"<s>\"; // skip BOS for first message\n            ss << bos << message->role << \"\\n\" << message->content << \"</s>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<s>assistant\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_GEMMA) {\n        // google/gemma-7b-it\n        std::string system_prompt = \"\";\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                // there is no system message for gemma, but we will merge it with user prompt, so nothing is broken\n                system_prompt += trim(message->content);\n                continue;\n            }\n            // in gemma, \"assistant\" is \"model\"\n            role = role == \"assistant\" ? \"model\" : message->role;\n            ss << \"<start_of_turn>\" << role << \"\\n\";\n            if (!system_prompt.empty() && role != \"model\") {\n                ss << system_prompt << \"\\n\\n\";\n                system_prompt = \"\";\n            }\n            ss << trim(message->content) << \"<end_of_turn>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<start_of_turn>model\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_ORION) {\n        // OrionStarAI/Orion-14B-Chat\n        std::string system_prompt = \"\";\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                // there is no system message support, we will merge it with user prompt\n                system_prompt += message->content;\n                continue;\n            } else if (role == \"user\") {\n                ss << \"Human: \";\n                if (!system_prompt.empty()) {\n                    ss << system_prompt << \"\\n\\n\";\n                    system_prompt = \"\";\n                }\n                ss << message->content << \"\\n\\nAssistant: </s>\";\n            } else {\n                ss << message->content << \"</s>\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_OPENCHAT) {\n        // openchat/openchat-3.5-0106,\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << message->content << \"<|end_of_turn|>\";\n            } else {\n                role[0] = toupper(role[0]);\n                ss << \"GPT4 Correct \" << role << \": \" << message->content << \"<|end_of_turn|>\";\n            }\n        }\n        if (add_ass) {\n            ss << \"GPT4 Correct Assistant:\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_VICUNA || tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {\n        // eachadea/vicuna-13b-1.1 (and Orca variant)\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                // Orca-Vicuna variant uses a system prefix\n                if (tmpl == LLM_CHAT_TEMPLATE_VICUNA_ORCA) {\n                    ss << \"SYSTEM: \" << message->content << \"\\n\";\n                } else {\n                    ss << message->content << \"\\n\\n\";\n                }\n            } else if (role == \"user\") {\n                ss << \"USER: \" << message->content << \"\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"ASSISTANT: \" << message->content << \"</s>\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"ASSISTANT:\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK) {\n        // deepseek-ai/deepseek-coder-33b-instruct\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << message->content;\n            } else if (role == \"user\") {\n                ss << \"### Instruction:\\n\" << message->content << \"\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"### Response:\\n\" << message->content << \"\\n<|EOT|>\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"### Response:\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_COMMAND_R) {\n        // CohereForAI/c4ai-command-r-plus\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>\" << trim(message->content) << \"<|END_OF_TURN_TOKEN|>\";\n            } else if (role == \"user\") {\n                ss << \"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>\" << trim(message->content) << \"<|END_OF_TURN_TOKEN|>\";\n            } else if (role == \"assistant\") {\n                ss << \"<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>\" << trim(message->content) << \"<|END_OF_TURN_TOKEN|>\";\n            }\n        }\n        if (add_ass) {\n            ss << \"<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_LLAMA_3) {\n        // Llama 3\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|start_header_id|>\" << role << \"<|end_header_id|>\\n\\n\" << trim(message->content) << \"<|eot_id|>\";\n        }\n        if (add_ass) {\n            ss << \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGLM_3) {\n        // chatglm3-6b\n        ss << \"[gMASK]\" << \"sop\";\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|\" << role << \"|>\" << \"\\n \" << message->content;\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_CHATGLM_4) {\n        ss << \"[gMASK]\" << \"<sop>\";\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|\" << role << \"|>\" << \"\\n\" << message->content;\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_GLMEDGE) {\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|\" << role << \"|>\" << \"\\n\" << message->content;\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_MINICPM) {\n        // MiniCPM-3B-OpenHermes-2.5-v2-GGUF\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"user\") {\n                ss << LU8(\"<用户>\");\n                ss << trim(message->content);\n                ss << \"<AI>\";\n            } else {\n                ss << trim(message->content);\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_2) {\n        // DeepSeek-V2\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << message->content << \"\\n\\n\";\n            } else if (role == \"user\") {\n                ss << \"User: \" << message->content << \"\\n\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"Assistant: \" << message->content << LU8(\"<｜end▁of▁sentence｜>\");\n            }\n        }\n        if (add_ass) {\n            ss << \"Assistant:\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_DEEPSEEK_3) {\n        // DeepSeek-V3\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << message->content << \"\\n\\n\";\n            } else if (role == \"user\") {\n                ss << LU8(\"<｜User｜>\") << message->content;\n            } else if (role == \"assistant\") {\n                ss << LU8(\"<｜Assistant｜>\") << message->content << LU8(\"<｜end▁of▁sentence｜>\");\n            }\n        }\n        if (add_ass) {\n            ss << LU8(\"<｜Assistant｜>\");\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_3) {\n        // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb\n        // EXAONE-3.0-7.8B-Instruct\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"[|system|]\" << trim(message->content) << \"[|endofturn|]\\n\";\n            } else if (role == \"user\") {\n                ss << \"[|user|]\" << trim(message->content) << \"\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"[|assistant|]\" << trim(message->content) << \"[|endofturn|]\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"[|assistant|]\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_4) {\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"[|system|]\" << trim(message->content) << \"[|endofturn|]\\n\";\n            } else if (role == \"user\") {\n                ss << \"[|user|]\" << trim(message->content) << \"\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"[|assistant|]\" << trim(message->content) << \"[|endofturn|]\\n\";\n            } else if (role == \"tool\") {\n                ss << \"[|tool|]\" << trim(message->content) << \"[|endofturn|]\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"[|assistant|]\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_EXAONE_MOE) {\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"<|system|>\\n\" << trim(message->content) << \"<|endofturn|>\\n\";\n            } else if (role == \"user\") {\n                ss << \"<|user|>\\n\" << trim(message->content) << \"<|endofturn|>\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"<|assistant|>\\n\" << trim(message->content) << \"<|endofturn|>\\n\";\n            } else if (role == \"tool\") {\n                ss << \"<|tool|>\\n\" << trim(message->content) << \"<|endofturn|>\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"<|assistant|>\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_RWKV_WORLD) {\n        // this template requires the model to have \"\\n\\n\" as EOT token\n        for (size_t i = 0; i < chat.size(); i++) {\n            std::string role(chat[i]->role);\n            if (role == \"system\") {\n                ss << \"System: \" << trim(chat[i]->content) << \"\\n\\n\";\n            } else if (role == \"user\") {\n                ss << \"User: \" << trim(chat[i]->content) << \"\\n\\n\";\n                if (i == chat.size() - 1) {\n                    ss << \"Assistant:\";\n                }\n            } else if (role == \"assistant\") {\n                ss << \"Assistant: \" << trim(chat[i]->content) << \"\\n\\n\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_GRANITE) {\n        // IBM Granite template\n        for (const auto & message : chat) {\n            std::string role(message->role);\n            ss << \"<|start_of_role|>\" << role << \"<|end_of_role|>\";\n            if (role == \"assistant_tool_call\") {\n                ss << \"<|tool_call|>\";\n            }\n            ss << message->content << \"<|end_of_text|>\\n\";\n        }\n        if (add_ass) {\n            ss << \"<|start_of_role|>assistant<|end_of_role|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_GIGACHAT) {\n        // GigaChat template\n        bool has_system = !chat.empty() && std::string(chat[0]->role) == \"system\";\n\n        // Handle system message if present\n        if (has_system) {\n            ss << \"<s>\" << chat[0]->content << \"<|message_sep|>\";\n        } else {\n            ss << \"<s>\";\n        }\n\n        // Process remaining messages\n        for (size_t i = has_system ? 1 : 0; i < chat.size(); i++) {\n            std::string role(chat[i]->role);\n            if (role == \"user\") {\n                ss << \"user<|role_sep|>\" << chat[i]->content << \"<|message_sep|>\"\n                << \"available functions<|role_sep|>[]<|message_sep|>\";\n            } else if (role == \"assistant\") {\n                ss << \"assistant<|role_sep|>\" << chat[i]->content << \"<|message_sep|>\";\n            }\n        }\n\n        // Add generation prompt if needed\n        if (add_ass) {\n            ss << \"assistant<|role_sep|>\";\n        }\n    }  else if (tmpl == LLM_CHAT_TEMPLATE_MEGREZ) {\n        // Megrez template\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|role_start|>\" << role << \"<|role_end|>\" << message->content << \"<|turn_end|>\";\n        }\n\n        if (add_ass) {\n            ss << \"<|role_start|>assistant<|role_end|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_YANDEX) {\n        // Yandex template (\"\\n\\n\" is defined as EOT token)\n\n        for (size_t i = 0; i < chat.size(); i++) {\n            std::string role(chat[i]->role);\n            if (role == \"user\") {\n                ss << \" Пользователь: \" << chat[i]->content << \"\\n\\n\";\n            } else if (role == \"assistant\") {\n                ss << \" Ассистент: \" << chat[i]->content << \"\\n\\n\";\n            }\n        }\n\n        // Add generation prompt if needed\n        if (add_ass) {\n            ss << \" Ассистент:[SEP]\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_BAILING || tmpl == LLM_CHAT_TEMPLATE_BAILING_THINK) {\n        // Bailing (Ling/Ring) template\n        for (auto message : chat) {\n            std::string role(message->role);\n\n            if (role == \"user\") {\n                role = \"HUMAN\";\n            } else {\n                std::transform(role.begin(), role.end(), role.begin(), ::toupper);\n            }\n\n            ss << \"<role>\" << role << \"</role>\" << message->content;\n        }\n\n        if (add_ass) {\n            ss << \"<role>ASSISTANT</role>\";\n\n            if (tmpl == LLM_CHAT_TEMPLATE_BAILING_THINK) {\n                ss << \"<think>\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_BAILING2) {\n        // Bailing2 (Ling 2.0) template\n        bool has_system = !chat.empty() && std::string(chat[0]->role) == \"system\";\n\n        if (!has_system) {\n            ss << \"<role>SYSTEM</role>detailed thinking off<|role_end|>\";\n        }\n\n        for (auto message : chat) {\n            std::string role(message->role);\n\n            if (role == \"user\") {\n                role = \"HUMAN\";\n            } else {\n                std::transform(role.begin(), role.end(), role.begin(), ::toupper);\n            }\n\n            ss << \"<role>\" << role << \"</role>\" << message->content << \"<|role_end|>\";\n        }\n\n        if (add_ass) {\n            ss << \"<role>ASSISTANT</role>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_LLAMA4) {\n        // Llama 4\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|header_start|>\" << role << \"<|header_end|>\\n\\n\" << trim(message->content) << \"<|eot|>\";\n        }\n        if (add_ass) {\n            ss << \"<|header_start|>assistant<|header_end|>\\n\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_SMOLVLM) {\n        // SmolVLM\n        ss << \"<|im_start|>\"; // uses <|im_start|> as BOS, but the actual content is NOT chatml\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << message->content << \"\\n\\n\";\n            } else if (role == \"user\") {\n                ss << \"User: \" << message->content << \"<end_of_utterance>\\n\";\n            } else {\n                ss << \"Assistant: \" << message->content << \"<end_of_utterance>\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"Assistant:\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_DOTS1) {\n        // dots.llm1.inst (DOTS1)\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"<|system|>\" << message->content << \"<|endofsystem|>\";\n            } else if (role == \"user\") {\n                ss << \"<|userprompt|>\" << message->content << \"<|endofuserprompt|>\";\n            } else {\n                ss << \"<|response|>\" << message->content << \"<|endofresponse|>\";\n            }\n        }\n        if (add_ass) {\n            ss << \"<|response|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_MOE) {\n        // tencent/Hunyuan-A13B-Instruct\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"<|startoftext|>\" << message->content << \"<|extra_4|>\";\n            } else if (role == \"assistant\") {\n                ss << message->content << \"<|eos|>\";\n            } else {\n                ss << \"<|startoftext|>\" << message->content << \"<|extra_0|>\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_OPENAI_MOE) {\n        // OpenAI MoE (based on Harmony chat template)\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|start|>\" << role << \"<|message|>\" << message->content;\n            ss << (role == \"assistant\" ? \"<|return|>\" : \"<|end|>\");\n        }\n        if (add_ass) {\n            ss << \"<|start|>assistant\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_HUNYUAN_DENSE) {\n        // tencent/Hunyuan-4B-Instruct\n        for (size_t i = 0; i < chat.size(); i++) {\n            std::string role(chat[i]->role);\n            if (i == 0) {\n                if (role == \"system\") {\n                    ss << chat[i]->content << \"<｜hy_place▁holder▁no▁3｜>\";\n                }\n            }\n\n            if (role == \"assistant\") {\n                ss << \"<｜hy_Assistant｜>\" << chat[i]->content << \"<｜hy_place▁holder▁no▁2｜>\";\n            } else if (role == \"user\") {\n                ss << \"<｜hy_User｜>\" << chat[i]->content << \"<｜hy_Assistant｜>\";\n            }\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_KIMI_K2) {\n        // moonshotai/Kimi-K2-Instruct\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"<|im_system|>system<|im_middle|>\";\n            } else if (role == \"user\") {\n                ss << \"<|im_user|>user<|im_middle|>\";\n            } else if (role == \"assistant\") {\n                ss << \"<|im_assistant|>assistant<|im_middle|>\";\n            } else if (role == \"tool\") {\n                ss << \"<|im_system|>tool<|im_middle|>\";\n            }\n\n            ss << message->content << \"<|im_end|>\";\n        }\n        if (add_ass) {\n            ss << \"<|im_assistant|>assistant<|im_middle|>\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_SEED_OSS) {\n        for (auto message: chat) {\n            std::string role(message->role);\n            ss << \"<seed:bos>\" << role << \"\\n\" << (role == \"assistant\" ? trim(message->content) : message->content) << \"<seed:eos>\";\n        }\n        if (add_ass) {\n            ss << \"<seed:bos>assistant\\n\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_GROK_2) {\n        for (auto message : chat) {\n            std::string role(message->role);\n            if (role == \"system\") {\n                ss << \"System: \" << trim(message->content) << \"<|separator|>\\n\\n\";\n            } else if (role == \"user\") {\n                ss << \"Human: \" << trim(message->content) << \"<|separator|>\\n\\n\";\n            } else if (role == \"assistant\") {\n                ss << \"Assistant: \" << message->content << \"<|separator|>\\n\\n\";\n            }\n        }\n        if (add_ass) {\n            ss << \"Assistant:\";\n        }\n    }else if (tmpl == LLM_CHAT_TEMPLATE_PANGU_EMBED) {\n        // [unused9]系统：xxx[unused10]\n        // [unused9]用户：xxx[unused10]\n        // [unused9]助手：xxx[unused10]\n        // ...\n        for (size_t i = 0; i < chat.size(); ++i) {\n            const auto & msg = chat[i];\n            const std::string & role = msg->role;\n            const std::string & content = msg->content;\n\n            if (i == 0 && role != \"system\") {\n                ss << \"[unused9]系统：[unused10]\";\n            }\n\n            if (role == \"system\") {\n                ss << \"[unused9]系统：\" << content << \"[unused10]\";\n            } else if (role == \"user\") {\n                ss << \"[unused9]用户：\" << content << \"[unused10]\";\n            } else if (role == \"assistant\") {\n                ss << \"[unused9]助手：\" << content << \"[unused10]\";\n            } else if (role == \"tool\") {\n                ss << \"[unused9]工具：\" << content << \"[unused10]\";\n            } else if (role == \"function\") {\n                ss << \"[unused9]方法：\" << content << \"[unused10]\";\n            }\n        }\n        if (add_ass) {\n            ss << \"[unused9]助手：\";\n        }\n    } else if (tmpl == LLM_CHAT_TEMPLATE_SOLAR_OPEN) {\n        for (auto message : chat) {\n            std::string role(message->role);\n            ss << \"<|begin|>\" << role << \"<|content|>\" << message->content << \"<|end|>\";\n        }\n        if (add_ass) {\n            ss << \"<|begin|>assistant\";\n        }\n    } else {\n        // template not supported\n        return -1;\n    }\n    dest = ss.str();\n    return dest.size();\n}\n\n// public interface\n\nint32_t llama_chat_builtin_templates(const char ** output, size_t len) {\n    auto it = LLM_CHAT_TEMPLATES.begin();\n    for (size_t i = 0; i < std::min(len, LLM_CHAT_TEMPLATES.size()); i++) {\n        output[i] = it->first.c_str();\n        std::advance(it, 1);\n    }\n    return (int32_t) LLM_CHAT_TEMPLATES.size();\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-chat.h",
    "content": "#pragma once\n\n#include <string>\n#include <vector>\n#include <cstdint>\n\nenum llm_chat_template {\n    LLM_CHAT_TEMPLATE_CHATML,\n    LLM_CHAT_TEMPLATE_LLAMA_2,\n    LLM_CHAT_TEMPLATE_LLAMA_2_SYS,\n    LLM_CHAT_TEMPLATE_LLAMA_2_SYS_BOS,\n    LLM_CHAT_TEMPLATE_LLAMA_2_SYS_STRIP,\n    LLM_CHAT_TEMPLATE_MISTRAL_V1,\n    LLM_CHAT_TEMPLATE_MISTRAL_V3,\n    LLM_CHAT_TEMPLATE_MISTRAL_V3_TEKKEN,\n    LLM_CHAT_TEMPLATE_MISTRAL_V7,\n    LLM_CHAT_TEMPLATE_MISTRAL_V7_TEKKEN,\n    LLM_CHAT_TEMPLATE_PHI_3,\n    LLM_CHAT_TEMPLATE_PHI_4,\n    LLM_CHAT_TEMPLATE_FALCON_3,\n    LLM_CHAT_TEMPLATE_ZEPHYR,\n    LLM_CHAT_TEMPLATE_MONARCH,\n    LLM_CHAT_TEMPLATE_GEMMA,\n    LLM_CHAT_TEMPLATE_ORION,\n    LLM_CHAT_TEMPLATE_OPENCHAT,\n    LLM_CHAT_TEMPLATE_VICUNA,\n    LLM_CHAT_TEMPLATE_VICUNA_ORCA,\n    LLM_CHAT_TEMPLATE_DEEPSEEK,\n    LLM_CHAT_TEMPLATE_DEEPSEEK_2,\n    LLM_CHAT_TEMPLATE_DEEPSEEK_3,\n    LLM_CHAT_TEMPLATE_COMMAND_R,\n    LLM_CHAT_TEMPLATE_LLAMA_3,\n    LLM_CHAT_TEMPLATE_CHATGLM_3,\n    LLM_CHAT_TEMPLATE_CHATGLM_4,\n    LLM_CHAT_TEMPLATE_GLMEDGE,\n    LLM_CHAT_TEMPLATE_MINICPM,\n    LLM_CHAT_TEMPLATE_EXAONE_3,\n    LLM_CHAT_TEMPLATE_EXAONE_4,\n    LLM_CHAT_TEMPLATE_EXAONE_MOE,\n    LLM_CHAT_TEMPLATE_RWKV_WORLD,\n    LLM_CHAT_TEMPLATE_GRANITE,\n    LLM_CHAT_TEMPLATE_GIGACHAT,\n    LLM_CHAT_TEMPLATE_MEGREZ,\n    LLM_CHAT_TEMPLATE_YANDEX,\n    LLM_CHAT_TEMPLATE_BAILING,\n    LLM_CHAT_TEMPLATE_BAILING_THINK,\n    LLM_CHAT_TEMPLATE_BAILING2,\n    LLM_CHAT_TEMPLATE_LLAMA4,\n    LLM_CHAT_TEMPLATE_SMOLVLM,\n    LLM_CHAT_TEMPLATE_DOTS1,\n    LLM_CHAT_TEMPLATE_HUNYUAN_MOE,\n    LLM_CHAT_TEMPLATE_OPENAI_MOE,\n    LLM_CHAT_TEMPLATE_HUNYUAN_DENSE,\n    LLM_CHAT_TEMPLATE_KIMI_K2,\n    LLM_CHAT_TEMPLATE_SEED_OSS,\n    LLM_CHAT_TEMPLATE_GROK_2,\n    LLM_CHAT_TEMPLATE_PANGU_EMBED,\n    LLM_CHAT_TEMPLATE_SOLAR_OPEN,\n    LLM_CHAT_TEMPLATE_UNKNOWN,\n};\n\nstruct llama_chat_message;\n\nllm_chat_template llm_chat_template_from_str(const std::string & name);\n\nllm_chat_template llm_chat_detect_template(const std::string & tmpl);\n\nint32_t llm_chat_apply_template(\n    llm_chat_template tmpl,\n    const std::vector<const llama_chat_message *> & chat,\n    std::string & dest, bool add_ass);\n"
  },
  {
    "path": "examples/talk-llama/llama-context.cpp",
    "content": "#include \"llama-context.h\"\n\n#include \"llama-arch.h\"\n#include \"llama-impl.h\"\n#include \"llama-batch.h\"\n#include \"llama-io.h\"\n#include \"llama-memory.h\"\n#include \"llama-mmap.h\"\n#include \"llama-model.h\"\n#include \"llama-ext.h\"\n\n#include <cinttypes>\n#include <cmath>\n#include <cstring>\n#include <limits>\n#include <stdexcept>\n\n//\n// llama_context\n//\n\nllama_context::llama_context(\n        const llama_model & model,\n              llama_context_params params) :\n    model(model),\n    cvec(std::make_unique<llama_adapter_cvec>()),\n    loras(std::make_unique<llama_adapter_loras>()),\n    balloc(std::make_unique<llama_batch_allocr>(model.hparams.n_pos_per_embd())) {\n    // TODO warning when creating llama_context with awkward ctx size that is not a power of 2,\n    //     may need to be backend-dependent\n    LLAMA_LOG_INFO(\"%s: constructing llama_context\\n\", __func__);\n\n    t_start_us = model.t_start_us;\n    t_load_us  = model.t_load_us;\n\n    const auto & hparams = model.hparams;\n\n    cparams.n_seq_max = std::max(1u, params.n_seq_max);\n    if (cparams.n_seq_max > LLAMA_MAX_SEQ) {\n        throw std::runtime_error(\"n_seq_max must be <= \" + std::to_string(LLAMA_MAX_SEQ));\n    }\n\n    cparams.n_threads        = params.n_threads;\n    cparams.n_threads_batch  = params.n_threads_batch;\n    cparams.yarn_ext_factor  = params.yarn_ext_factor  >= 0.0f ? params.yarn_ext_factor  : hparams.yarn_ext_factor;\n    cparams.yarn_attn_factor = params.yarn_attn_factor >= 0.0f ? params.yarn_attn_factor : hparams.yarn_attn_factor;\n    cparams.yarn_beta_fast   = params.yarn_beta_fast   >= 0.0f ? params.yarn_beta_fast   : hparams.yarn_beta_fast;\n    cparams.yarn_beta_slow   = params.yarn_beta_slow   >= 0.0f ? params.yarn_beta_slow   : hparams.yarn_beta_slow;\n    cparams.embeddings       = params.embeddings;\n    cparams.offload_kqv      = params.offload_kqv;\n    cparams.no_perf          = params.no_perf;\n    cparams.pooling_type     = params.pooling_type;\n    cparams.warmup           = false;\n\n    cparams.n_ctx            = params.n_ctx           == 0    ? hparams.n_ctx_train           : params.n_ctx;\n    cparams.rope_freq_base   = params.rope_freq_base  == 0.0f ? hparams.rope_freq_base_train  : params.rope_freq_base;\n    cparams.rope_freq_scale  = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;\n\n    cparams.n_ctx_orig_yarn  = params.yarn_orig_ctx    != 0 ? params.yarn_orig_ctx    :\n                               hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :\n                                                              hparams.n_ctx_train;\n\n    cparams.cb_eval           = params.cb_eval;\n    cparams.cb_eval_user_data = params.cb_eval_user_data;\n\n    // Initialize backend samplers here so they are part of the sampling graph\n    // before the reserve passes run later in this function. This avoids a later\n    // re-reserve when graph nodes change.\n    if (params.samplers != nullptr && params.n_samplers > 0) {\n        for (size_t i = 0; i < params.n_samplers; ++i) {\n            const auto & config = params.samplers[i];\n\n            if (llama_sampler_chain_get(config.sampler, -1) == nullptr) {\n                throw std::runtime_error(\"the backend samplers must be of type llama_sampler_chain\");\n            }\n\n            if (set_sampler(config.seq_id, config.sampler)) {\n                const int n_samplers = llama_sampler_chain_n(config.sampler);\n\n                LLAMA_LOG_INFO(\"%s: setting backend sampler for seq_id %d (n = %d)\\n\", __func__, config.seq_id, n_samplers);\n            }\n        }\n    }\n\n    auto rope_scaling_type = params.rope_scaling_type;\n    if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {\n        rope_scaling_type = hparams.rope_scaling_type_train;\n    }\n\n    if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {\n        cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none\n    }\n\n    if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'\n        cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;\n    }\n\n    if (cparams.yarn_ext_factor != 0) {\n        static auto get_mscale = [](float scale, float mscale) {\n            return scale <= 1.0f ? 1.0f : (0.1f * mscale * logf(scale) + 1.0f);\n        };\n\n        const float factor = 1.0f / cparams.rope_freq_scale;\n\n        // ref: https://github.com/huggingface/transformers/blob/6d00f6b0a5679c36510f203e4226e36f517c3032/src/transformers/modeling_rope_utils.py#L336-L348\n        if (hparams.rope_yarn_log_mul != 0.0f) {\n            // note: here we assume `mscale == 1.0f`\n            // TODO: start reading the actual value of mscale and handle the case where it is not 1.0f\n                  float mscale          = 1.0f;\n            const float mscale_all_dims = hparams.rope_yarn_log_mul;\n\n            // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]\n            // special-case DEEPSEEK v2:\n            // https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/main/config.json#L42-L43\n            if (model.arch == LLM_ARCH_DEEPSEEK2 && mscale_all_dims != 1.0f) {\n                mscale = mscale_all_dims;\n            }\n\n            cparams.yarn_attn_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dims);\n\n            LLAMA_LOG_WARN(\"%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\\n\",\n                    __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims);\n        } else {\n            cparams.yarn_attn_factor = get_mscale(factor, 1.0f);\n        }\n\n        // when YARN is applied with yarn_ext_factor != 0.0f, we need to cancel this factor:\n        // https://github.com/ggml-org/llama.cpp/blob/a81a569577cc38b32558958b048228150be63eae/ggml/src/ggml-cpu/ops.cpp#L5541-L5544\n        //\n        // ref: https://github.com/ggml-org/llama.cpp/discussions/7416\n        //      https://github.com/ggml-org/llama.cpp/pull/17945\n        cparams.yarn_attn_factor *= 1.0f / (1.0f + 0.1f * logf(factor));\n    }\n\n    cparams.yarn_attn_factor *= hparams.rope_attn_factor;\n\n    if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {\n        if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {\n            cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;\n        } else {\n            cparams.pooling_type = hparams.pooling_type;\n        }\n    }\n\n    if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {\n        cparams.causal_attn = hparams.causal_attn;\n    } else {\n        cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;\n    }\n\n    cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;\n    cparams.auto_fa    = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;\n\n    cparams.fused_gdn_ar = true;\n    cparams.fused_gdn_ch = true;\n    cparams.auto_fgdn    = true;\n\n    // with causal attention, the batch size is limited by the context size\n    cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;\n\n    cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);\n\n    cparams.op_offload = params.op_offload;\n    cparams.kv_unified = params.kv_unified;\n\n    // initialized later\n    cparams.pipeline_parallel = false;\n\n    {\n        const char * LLAMA_GRAPH_REUSE_DISABLE = getenv(\"LLAMA_GRAPH_REUSE_DISABLE\");\n        graph_reuse_disable = LLAMA_GRAPH_REUSE_DISABLE ? (atoi(LLAMA_GRAPH_REUSE_DISABLE) != 0) : graph_reuse_disable;\n\n        if (graph_reuse_disable) {\n            LLAMA_LOG_WARN(\"%s: graph reuse disabled\\n\", __func__);\n        }\n    }\n\n    // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732\n    cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256);\n\n    if (cparams.kv_unified) {\n        cparams.n_ctx_seq = cparams.n_ctx;\n    } else {\n        cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max;\n        cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256);\n\n        if (cparams.n_ctx_seq == 0) {\n            throw std::runtime_error(\"n_ctx_seq == 0\");\n        }\n\n        if (cparams.n_ctx != cparams.n_ctx_seq * cparams.n_seq_max) {\n            cparams.n_ctx =  cparams.n_ctx_seq * cparams.n_seq_max;\n            LLAMA_LOG_WARN(\"%s: n_ctx is not divisible by n_seq_max - rounding down to %u\\n\", __func__, cparams.n_ctx);\n        }\n    }\n\n    LLAMA_LOG_INFO(\"%s: n_seq_max     = %u\\n\",   __func__, cparams.n_seq_max);\n    LLAMA_LOG_INFO(\"%s: n_ctx         = %u\\n\",   __func__, cparams.n_ctx);\n    LLAMA_LOG_INFO(\"%s: n_ctx_seq     = %u\\n\",   __func__, cparams.n_ctx_seq);\n    LLAMA_LOG_INFO(\"%s: n_batch       = %u\\n\",   __func__, cparams.n_batch);\n    LLAMA_LOG_INFO(\"%s: n_ubatch      = %u\\n\",   __func__, cparams.n_ubatch);\n    LLAMA_LOG_INFO(\"%s: causal_attn   = %d\\n\",   __func__, cparams.causal_attn);\n    LLAMA_LOG_INFO(\"%s: flash_attn    = %s\\n\",   __func__, llama_flash_attn_type_name(params.flash_attn_type));\n    LLAMA_LOG_INFO(\"%s: kv_unified    = %s\\n\",   __func__, cparams.kv_unified ? \"true\" : \"false\");\n    LLAMA_LOG_INFO(\"%s: freq_base     = %.1f\\n\", __func__, cparams.rope_freq_base);\n    LLAMA_LOG_INFO(\"%s: freq_scale    = %g\\n\",   __func__, cparams.rope_freq_scale);\n\n    if (cparams.n_ctx_seq < hparams.n_ctx_train) {\n        LLAMA_LOG_WARN(\"%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\\n\",\n                __func__, cparams.n_ctx_seq, hparams.n_ctx_train);\n    }\n\n    if (cparams.n_ctx_seq > hparams.n_ctx_train) {\n        LLAMA_LOG_WARN(\"%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\\n\",\n                __func__, cparams.n_ctx_seq, hparams.n_ctx_train);\n    }\n\n    if (!hparams.vocab_only) {\n        // GPU backends\n        for (auto * dev : model.devices) {\n            ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);\n            if (backend == nullptr) {\n                throw std::runtime_error(format(\"failed to initialize %s backend\", ggml_backend_dev_name(dev)));\n            }\n            backends.emplace_back(backend);\n        }\n\n        // add ACCEL backends (such as BLAS)\n        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n            ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n            if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {\n                ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);\n                if (backend == nullptr) {\n                    throw std::runtime_error(format(\"failed to initialize %s backend\", ggml_backend_dev_name(dev)));\n                }\n                backends.emplace_back(backend);\n            }\n        }\n\n        // add CPU backend\n        backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);\n        if (backend_cpu == nullptr) {\n            throw std::runtime_error(\"failed to initialize CPU backend\");\n        }\n        backends.emplace_back(backend_cpu);\n\n        // create a list of the set_n_threads functions in the backends\n        for (auto & backend : backends) {\n            ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());\n            ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;\n            if (reg) {\n                auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_set_n_threads\");\n                if (ggml_backend_set_n_threads_fn) {\n                    set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);\n                }\n            }\n        }\n\n        llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);\n\n        // graph outputs buffer\n        {\n            if (output_reserve(params.n_seq_max) < params.n_seq_max) {\n                throw std::runtime_error(\"failed to reserve initial output buffer\");\n            }\n\n            LLAMA_LOG_INFO(\"%s: %10s  output buffer size = %8.2f MiB\\n\", __func__,\n                    ggml_backend_buffer_name    (buf_output.get()),\n                    ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0);\n        }\n    }\n\n    // init the memory module\n    if (!hparams.vocab_only) {\n        llama_memory_params params_mem = {\n            /*.type_k   =*/ params.type_k,\n            /*.type_v   =*/ params.type_v,\n            /*.swa_full =*/ params.swa_full,\n        };\n\n        memory.reset(model.create_memory(params_mem, cparams));\n    }\n\n    // init backends\n    if (!hparams.vocab_only) {\n        LLAMA_LOG_DEBUG(\"%s: enumerating backends\\n\", __func__);\n\n        backend_buft.clear();\n        backend_ptrs.clear();\n        backend_buf_exp_size.clear();\n\n        for (auto & backend : backends) {\n            auto * buft = ggml_backend_get_default_buffer_type(backend.get());\n            auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));\n\n            if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {\n                // use the host buffer of the first device CPU for faster transfer of the intermediate state\n                auto * dev = model.devices[0];\n                auto * host_buft = ggml_backend_dev_host_buffer_type(dev);\n                if (host_buft) {\n                    buft = host_buft;\n                }\n            }\n\n            backend_buft.push_back(buft);\n            backend_ptrs.push_back(backend.get());\n            backend_buf_exp_size.push_back(0);\n        }\n\n        LLAMA_LOG_DEBUG(\"%s: backend_ptrs.size() = %zu\\n\", __func__, backend_ptrs.size());\n\n        // TODO: move these checks to ggml_backend_sched\n        // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary\n        bool pipeline_parallel =\n            model.n_devices() > 1 &&\n            model.n_gpu_layers() > model.hparams.n_layer &&\n            model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&\n            cparams.offload_kqv &&\n            !model.has_tensor_overrides();\n\n        // pipeline parallelism requires support for async compute and events in all devices\n        if (pipeline_parallel) {\n            for (auto & backend : backends) {\n                auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));\n                if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {\n                    // ignore CPU backend\n                    // TODO: should we ignore ACCEL types too?\n                    continue;\n                }\n                auto * dev = ggml_backend_get_device(backend.get());\n                ggml_backend_dev_props props;\n                ggml_backend_dev_get_props(dev, &props);\n                if (!props.caps.async || !props.caps.events) {\n                    // device does not support async compute or events\n                    pipeline_parallel = false;\n                    break;\n                }\n            }\n        }\n\n        cparams.pipeline_parallel = pipeline_parallel;\n\n        if (cparams.pipeline_parallel) {\n            LLAMA_LOG_INFO(\"%s: pipeline parallelism enabled\\n\", __func__);\n\n            if (!graph_reuse_disable) {\n                // TODO: figure out a way to make graph reuse work with pipeline parallelism\n                // ref: https://github.com/ggml-org/llama.cpp/pull/20463\n                LLAMA_LOG_WARN(\"%s: graph reuse is currently not compatible with pipeline parallelism - disabling\\n\", __func__);\n\n                graph_reuse_disable = true;\n            }\n        }\n\n        sched_reserve();\n\n        if (!cparams.flash_attn) {\n            if (ggml_is_quantized(params.type_v)) {\n                throw std::runtime_error(\"quantized V cache was requested, but this requires Flash Attention\");\n            }\n        }\n    }\n\n    // Initialize the full vocabulary token ids for backend samplers.\n    {\n        const int n_vocab = model.vocab.n_tokens();\n\n        sampling.token_ids_full_vocab.resize(n_vocab);\n        for (int i = 0; i < n_vocab; ++i) {\n            sampling.token_ids_full_vocab[i] = i;\n        }\n    }\n}\n\nllama_context::~llama_context() {\n    if (!model.hparams.no_alloc) {\n        for (size_t i = 0; i < backend_ptrs.size(); ++i) {\n            ggml_backend_t             backend = backend_ptrs[i];\n            ggml_backend_buffer_type_t buft    = backend_buft[i];\n\n            const size_t size_exp = backend_buf_exp_size[i];\n            const size_t size_act = ggml_backend_sched_get_buffer_size(sched.get(), backend);\n            if (size_exp == size_act) {\n                LLAMA_LOG_DEBUG(\"%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\\n\",\n                    __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));\n            } else {\n                LLAMA_LOG_WARN(\"%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\\n\",\n                    __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0));\n            }\n        }\n    }\n    ggml_opt_free(opt_ctx);\n}\n\nvoid llama_context::sched_reserve() {\n    if (!sched_need_reserve) {\n        return;\n    }\n\n    sched_need_reserve = false;\n\n    LLAMA_LOG_INFO(\"%s: reserving ...\\n\", __func__);\n\n    synchronize();\n\n    const int64_t t_start_us = ggml_time_us();\n\n    const uint32_t n_seqs = cparams.n_seq_max;\n    const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);\n\n    const size_t max_nodes = this->graph_max_nodes(n_tokens);\n\n    LLAMA_LOG_DEBUG(\"%s: max_nodes = %zu\\n\", __func__, max_nodes);\n\n    gf_res_prev.reset(new llm_graph_result(max_nodes));\n    gf_res_reserve.reset(new llm_graph_result(max_nodes));\n\n    sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));\n\n    llama_memory_context_ptr mctx;\n    if (memory) {\n        LLAMA_LOG_DEBUG(\"%s: reserving full memory module\\n\", __func__);\n        mctx = memory->init_full();\n        if (!mctx) {\n            throw std::runtime_error(\"failed to initialize memory module\");\n        }\n    }\n\n    // avoid reserving graphs with zero outputs - assume one output per sequence\n    const int n_outputs = n_seqs;\n\n    LLAMA_LOG_DEBUG(\"%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\\n\", __func__, n_tokens, n_seqs, n_outputs);\n\n    // resolve automatic Flash Attention use\n    if (cparams.auto_fa) {\n        auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);\n        if (!gf) {\n            throw std::runtime_error(\"failed to reserve graph for Flash Attention check\");\n        }\n\n        const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN) + 1;\n        bool fa_device_mismatch = false;\n        for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {\n            ggml_tensor * n = ggml_graph_node(gf, i);\n            if (n->op != GGML_OP_FLASH_ATTN_EXT) {\n                continue;\n            }\n            ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));\n\n            // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer\n            GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN \"-\", prefix_len) == 0);\n            const int il = std::stoi(n->name + prefix_len);\n            ggml_backend_dev_t device_kv = model.dev_layer(il);\n            if (device_fa != device_kv) {\n                LLAMA_LOG_WARN(\"%s: layer %d is assigned to device %s but the Flash Attention tensor \"\n                        \"is assigned to device %s (usually due to missing support)\\n\",\n                        __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa));\n                // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways\n                fa_device_mismatch = true;\n                break;\n            }\n        }\n\n        if (fa_device_mismatch) {\n            cparams.flash_attn = false;\n            LLAMA_LOG_WARN(\"%s: Flash Attention was auto, set to disabled\\n\", __func__);\n        } else {\n            cparams.flash_attn = true;\n            LLAMA_LOG_INFO(\"%s: Flash Attention was auto, set to enabled\\n\", __func__);\n        }\n\n        cparams.auto_fa = false;\n    }\n\n    if (cparams.auto_fgdn) {\n        LLAMA_LOG_INFO(\"%s: resolving fused Gated Delta Net support:\\n\", __func__);\n\n        if (cparams.fused_gdn_ar) {\n            auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);\n            if (!gf) {\n                throw std::runtime_error(\"failed to reserve graph for fused Gated Delta Net check (autoregressive)\");\n            }\n\n            const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR) + 1;\n            bool gdn_device_mismatch = false;\n            for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {\n                ggml_tensor * n = ggml_graph_node(gf, i);\n                if (n->op != GGML_OP_GATED_DELTA_NET) {\n                    continue;\n                }\n                ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));\n\n                GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR \"-\", prefix_len) == 0);\n                const int il = std::stoi(n->name + prefix_len);\n                ggml_backend_dev_t device_kv = model.dev_layer(il);\n                if (device_gdn != device_kv) {\n                    LLAMA_LOG_WARN(\"%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor \"\n                            \"is assigned to device %s (usually due to missing support)\\n\",\n                            __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));\n                    gdn_device_mismatch = true;\n                    break;\n                }\n            }\n\n            if (gdn_device_mismatch) {\n                cparams.fused_gdn_ar = false;\n                LLAMA_LOG_WARN(\"%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\\n\", __func__);\n            } else {\n                LLAMA_LOG_INFO(\"%s: fused Gated Delta Net (autoregressive) enabled\\n\", __func__);\n            }\n        }\n\n        if (cparams.fused_gdn_ch) {\n            // more than one token in the batch per sequence in order to take the chunked path\n            // note: n_outputs must match n_tokens for embedding models with mean/rank pooling,\n            // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies\n            // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,\n            // the ggml_mul_mat assertion fails. this matches the pp reservation below (line ~553).\n            const uint32_t n_tokens_ch = 16*n_seqs;\n            auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);\n            if (!gf) {\n                throw std::runtime_error(\"failed to reserve graph for fused Gated Delta Net check (chunked)\");\n            }\n\n            const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH) + 1;\n            bool gdn_device_mismatch = false;\n            for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {\n                ggml_tensor * n = ggml_graph_node(gf, i);\n                if (n->op != GGML_OP_GATED_DELTA_NET) {\n                    continue;\n                }\n                ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));\n\n                GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH \"-\", prefix_len) == 0);\n                const int il = std::stoi(n->name + prefix_len);\n                ggml_backend_dev_t device_kv = model.dev_layer(il);\n                if (device_gdn != device_kv) {\n                    LLAMA_LOG_WARN(\"%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor \"\n                            \"is assigned to device %s (usually due to missing support)\\n\",\n                            __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn));\n                    gdn_device_mismatch = true;\n                    break;\n                }\n            }\n\n            if (gdn_device_mismatch) {\n                cparams.fused_gdn_ch = false;\n                LLAMA_LOG_WARN(\"%s: fused Gated Delta Net (chunked) not supported, set to disabled\\n\", __func__);\n            } else {\n                LLAMA_LOG_INFO(\"%s: fused Gated Delta Net (chunked) enabled\\n\", __func__);\n            }\n        }\n\n        cparams.auto_fgdn = false;\n    }\n\n    // reserve worst-case graph\n    int n_splits_pp = -1;\n    int n_nodes_pp  = -1;\n\n    int n_splits_tg = -1;\n    int n_nodes_tg  = -1;\n\n    // reserve pp (prompt processing) graph first so that buffers are only allocated once\n    {\n        auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(),\n                model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);\n        if (!gf) {\n            if (cparams.pipeline_parallel) {\n                LLAMA_LOG_WARN(\"%s: compute buffer allocation failed, retrying without pipeline parallelism\\n\", __func__);\n                cparams.pipeline_parallel = false;\n                sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));\n                gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());\n            }\n            if (!gf) {\n                throw std::runtime_error(\"failed to allocate compute pp buffers\");\n            }\n        }\n\n        n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());\n        n_nodes_pp  = ggml_graph_n_nodes(gf);\n    }\n\n    // reserve with tg (token generation) graph to get the number of splits and nodes\n    {\n        auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);\n        if (!gf) {\n            throw std::runtime_error(\"failed to allocate compute tg buffers\");\n        }\n\n        n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());\n        n_nodes_tg  = ggml_graph_n_nodes(gf);\n    }\n\n    // reserve again with pp graph to avoid ggml-alloc reallocations during inference\n    {\n        // TODO: not sure if the following graph would be worster case for multi-stream KV caches:\n        //\n        // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());\n        //\n        auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get(), model.hparams.no_alloc);\n        if (!gf) {\n            throw std::runtime_error(\"failed to allocate compute pp buffers\");\n        }\n    }\n\n    for (size_t i = 0; i < backend_ptrs.size(); ++i) {\n        ggml_backend_t             backend = backend_ptrs[i];\n        ggml_backend_buffer_type_t buft    = backend_buft[i];\n        if (!model.hparams.no_alloc) {\n            backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);\n        }\n        if (backend_buf_exp_size[i] > 1) {\n            LLAMA_LOG_INFO(\"%s: %10s compute buffer size = %8.2f MiB\\n\", __func__,\n                    ggml_backend_buft_name(buft),\n                    backend_buf_exp_size[i] / 1024.0 / 1024.0);\n        }\n    }\n\n    if (n_nodes_pp == n_nodes_tg) {\n        LLAMA_LOG_INFO(\"%s: graph nodes  = %d\\n\", __func__, n_nodes_pp);\n    } else {\n        LLAMA_LOG_INFO(\"%s: graph nodes  = %d (with bs=%d), %d (with bs=1)\\n\", __func__, n_nodes_pp, n_tokens, n_nodes_tg);\n    }\n\n    if (n_splits_pp == n_splits_tg) {\n        LLAMA_LOG_INFO(\"%s: graph splits = %d\\n\", __func__, n_splits_pp);\n    } else {\n        LLAMA_LOG_INFO(\"%s: graph splits = %d (with bs=%d), %d (with bs=1)\\n\", __func__, n_splits_pp, n_tokens, n_splits_tg);\n    }\n\n    const int64_t t_end_us = ggml_time_us();\n\n    LLAMA_LOG_INFO(\"%s: reserve took %.2f ms, sched copies = %d\\n\",\n            __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()));\n}\n\nvoid llama_context::synchronize() {\n    if (!sched) {\n        return;\n    }\n\n    ggml_backend_sched_synchronize(sched.get());\n\n    // FIXME: if multiple single tokens are evaluated without a synchronization,\n    // the stats will be added to the prompt evaluation stats\n    // this should only happen when using batch size 1 to evaluate a batch\n\n    // add the evaluation to the stats\n    if (n_queued_tokens == 1) {\n        if (!cparams.no_perf) {\n            t_eval_us += ggml_time_us() - t_compute_start_us;\n        }\n        n_eval++;\n    } else if (n_queued_tokens > 1) {\n        if (!cparams.no_perf) {\n            t_p_eval_us += ggml_time_us() - t_compute_start_us;\n        }\n        n_p_eval += n_queued_tokens;\n    }\n\n    // get a more accurate load time, upon first eval\n    if (n_queued_tokens > 0 && !has_evaluated_once) {\n        t_load_us = ggml_time_us() - t_start_us;\n        has_evaluated_once = true;\n    }\n\n    n_queued_tokens = 0;\n    t_compute_start_us = 0;\n}\n\nconst llama_model & llama_context::get_model() const {\n    return model;\n}\n\nconst llama_cparams & llama_context::get_cparams() const {\n    return cparams;\n}\n\nggml_backend_sched_t llama_context::get_sched() const {\n    return sched.get();\n}\n\nuint32_t llama_context::n_ctx() const {\n    return cparams.n_ctx;\n}\n\nuint32_t llama_context::n_ctx_seq() const {\n    return cparams.n_ctx_seq;\n}\n\nuint32_t llama_context::n_batch() const {\n    return cparams.n_batch;\n}\n\nuint32_t llama_context::n_ubatch() const {\n    return cparams.n_ubatch;\n}\n\nuint32_t llama_context::n_seq_max() const {\n    return cparams.n_seq_max;\n}\n\nuint32_t llama_context::n_threads() const {\n    return cparams.n_threads;\n}\n\nuint32_t llama_context::n_threads_batch() const {\n    return cparams.n_threads_batch;\n}\n\nllama_memory_t llama_context::get_memory() const {\n    return memory.get();\n}\n\nbool llama_context::memory_update(bool optimize) {\n    if (!memory) {\n        return false;\n    }\n\n    {\n        const auto mctx = memory->init_update(this, optimize);\n        switch (mctx->get_status()) {\n            case LLAMA_MEMORY_STATUS_SUCCESS:\n                {\n                    // noop\n                } break;\n            case LLAMA_MEMORY_STATUS_NO_UPDATE:\n                {\n                    // no updates need to be performed\n                    return false;\n                }\n            case LLAMA_MEMORY_STATUS_FAILED_PREPARE:\n            case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:\n                {\n                    LLAMA_LOG_ERROR(\"%s: failed to prepare memory update\\n\", __func__);\n                    return false;\n                }\n        }\n\n        // reset the previous graph result to make sure that it won't be reused\n        // TODO: change the mctx->apply() to return information if a graph reserve is needed\n        //       reset the graph result only if the memory module did reset the scheduler\n        gf_res_prev->reset();\n\n        if (!mctx->apply()) {\n            LLAMA_LOG_ERROR(\"%s: failed to apply memory update\\n\", __func__);\n        }\n    }\n\n    // if the memory module did any computation, we have to reserve a new worst-case graph\n    {\n        const auto mctx = memory->init_full();\n        if (!mctx) {\n            throw std::runtime_error(\"failed to initialize memory context\");\n        }\n\n        const uint32_t n_seqs = cparams.n_seq_max;\n        const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);\n\n        auto * gf = graph_reserve(n_tokens, n_seqs, n_tokens, mctx.get());\n        if (!gf) {\n            LLAMA_LOG_ERROR(\"%s: failed to reserve graph after the memory update\\n\", __func__);\n        }\n    }\n\n    return true;\n}\n\nenum llama_pooling_type llama_context::pooling_type() const {\n    return cparams.pooling_type;\n}\n\nfloat * llama_context::get_logits() {\n    output_reorder();\n\n    return logits.data;\n}\n\nint64_t llama_context::output_resolve_row(int32_t i) const {\n    int64_t j = -1;\n\n    // support negative indices (last output row)\n    if (i < 0) {\n        j = n_outputs + i;\n        if (j < 0) {\n            throw std::runtime_error(format(\"negative index out of range [0, %d)\", n_outputs));\n        }\n    } else if ((size_t) i >= output_ids.size()) {\n        throw std::runtime_error(format(\"out of range [0, %zu)\", output_ids.size()));\n    } else {\n        // use output_ids to translate the batch token index into a row number\n        // that holds this token's data.\n        j = output_ids[i];\n    }\n\n    if (j < 0) {\n        // the batch token was not configured to output anything\n        throw std::runtime_error(format(\"batch.logits[%d] != true\", i));\n    }\n\n    if (j >= n_outputs) {\n        throw std::runtime_error(format(\"corrupt output buffer (j=%\" PRId64 \", n_outputs=%d)\", j, n_outputs));\n    }\n\n    return j;\n}\n\nfloat * llama_context::get_logits_ith(int32_t i) {\n    output_reorder();\n\n    try {\n        if (logits.data == nullptr) {\n            throw std::runtime_error(\"no logits\");\n        }\n\n        const int64_t j = output_resolve_row(i);\n        return logits.data + j*model.vocab.n_tokens();\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid logits id %d, reason: %s\\n\", __func__, i, err.what());\n#ifndef NDEBUG\n        GGML_ABORT(\"fatal error\");\n#else\n        return nullptr;\n#endif\n    }\n}\n\nfloat * llama_context::get_embeddings() {\n    output_reorder();\n\n    return embd.data;\n}\n\nllama_token * llama_context::get_sampled_tokens()  const{\n    return sampling.sampled.data;\n}\n\nfloat * llama_context::get_embeddings_ith(int32_t i) {\n    output_reorder();\n\n    try {\n        if (embd.data == nullptr) {\n            throw std::runtime_error(\"no embeddings\");\n        }\n\n        const int64_t j = output_resolve_row(i);\n        const uint32_t n_embd_out = model.hparams.n_embd_out();\n        return embd.data + j*n_embd_out;\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid embeddings id %d, reason: %s\\n\", __func__, i, err.what());\n#ifndef NDEBUG\n        GGML_ABORT(\"fatal error\");\n#else\n        return nullptr;\n#endif\n    }\n}\n\nfloat * llama_context::get_embeddings_seq(llama_seq_id seq_id) {\n    auto it = embd_seq.find(seq_id);\n    if (it == embd_seq.end()) {\n        return nullptr;\n    }\n\n    return it->second.data();\n}\n\nllama_token llama_context::get_sampled_token_ith(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.sampled.has_data()) {\n        return LLAMA_TOKEN_NULL;\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        GGML_ASSERT(row < (int64_t) sampling.sampled.size);\n        return sampling.sampled.data[row];\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled token id %d, reason: %s\\n\", __func__, idx, err.what());\n        return LLAMA_TOKEN_NULL;\n    }\n}\n\nfloat * llama_context::get_sampled_probs_ith(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.probs.has_data()) {\n        return nullptr;\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if ((size_t) row >= sampling.probs_count.size() || sampling.probs_count[row] == 0) {\n            return nullptr;\n        }\n        return sampling.probs.data + row*model.vocab.n_tokens();\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled probs id %d, reason: %s\\n\", __func__, idx, err.what());\n        return nullptr;\n    }\n}\n\nfloat * llama_context::get_sampled_logits_ith(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.logits.has_data()) {\n        return nullptr;\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if ((size_t) row >= sampling.logits_count.size() || sampling.logits_count[row] == 0) {\n            return nullptr;\n        }\n        return sampling.logits.data + row*model.vocab.n_tokens();\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled logits id %d, reason: %s\\n\", __func__, idx, err.what());\n        return nullptr;\n    }\n}\n\nconst llama_token * llama_context::get_sampled_candidates_ith(int32_t idx) {\n    output_reorder();\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if (sampling.candidates.has_data() &&\n            (size_t) row < sampling.candidates_count.size() &&\n            sampling.candidates_count[row] > 0) {\n            return sampling.candidates.data + row*model.vocab.n_tokens();\n        }\n    } catch (const std::exception & err) {\n        // fallback to full vocab list\n        GGML_UNUSED(err);\n    }\n\n    return sampling.token_ids_full_vocab.data();\n}\n\nsize_t llama_context::get_sampled_candidates_count(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.candidates.has_data()) {\n        return 0;\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if ((size_t) row >= sampling.candidates_count.size()) {\n            return 0;\n        }\n        return sampling.candidates_count[row];\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled candidates count id %d, reason: %s\\n\", __func__, idx, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::get_sampled_logits_count(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.logits.has_data()) {\n        return model.vocab.n_tokens();\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if ((size_t) row >= sampling.logits_count.size()) {\n            return 0;\n        }\n        return sampling.logits_count[row];\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled logits count id %d, reason: %s\\n\", __func__, idx, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::get_sampled_probs_count(int32_t idx) {\n    output_reorder();\n\n    if (!sampling.probs.has_data()) {\n        return 0;\n    }\n\n    try {\n        const int64_t row = output_resolve_row(idx);\n        if ((size_t) row >= sampling.probs_count.size()) {\n            return 0;\n        }\n        return sampling.probs_count[row];\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: invalid backend sampled probs count id %d, reason: %s\\n\", __func__, idx, err.what());\n        return 0;\n    }\n}\n\n\nvoid llama_context::attach_threadpool(\n           ggml_threadpool_t threadpool,\n           ggml_threadpool_t threadpool_batch) {\n    LLAMA_LOG_DEBUG(\"%s: call\\n\", __func__);\n\n    this->threadpool       = threadpool;\n    this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;\n}\n\nvoid llama_context::detach_threadpool() {\n    LLAMA_LOG_DEBUG(\"%s: call\\n\", __func__);\n\n    this->threadpool       = nullptr;\n    this->threadpool_batch = nullptr;\n}\n\nvoid llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {\n    LLAMA_LOG_DEBUG(\"%s: n_threads = %d, n_threads_batch = %d\\n\", __func__, n_threads, n_threads_batch);\n\n    cparams.n_threads       = n_threads;\n    cparams.n_threads_batch = n_threads_batch;\n}\n\nvoid llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) {\n    LLAMA_LOG_DEBUG(\"%s: call\\n\", __func__);\n\n    this->abort_callback      = abort_callback;\n    this->abort_callback_data = abort_callback_data;\n\n    for (auto & backend : backends) {\n        auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));\n        auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_set_abort_callback\");\n        if (set_abort_callback_fn) {\n            set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);\n        }\n    }\n}\n\nvoid llama_context::set_embeddings(bool value) {\n    LLAMA_LOG_DEBUG(\"%s: value = %d\\n\", __func__, value);\n\n    cparams.embeddings = value;\n\n    // TODO: not sure yet if we want to reserve here\n    //sched_need_reserve = true;\n}\n\nvoid llama_context::set_causal_attn(bool value) {\n    LLAMA_LOG_DEBUG(\"%s: value = %d\\n\", __func__, value);\n\n    if (cparams.causal_attn == value) {\n        return;\n    }\n\n    cparams.causal_attn = value;\n\n    sched_need_reserve = true;\n}\n\nvoid llama_context::set_warmup(bool value) {\n    LLAMA_LOG_DEBUG(\"%s: value = %d\\n\", __func__, value);\n\n    if (cparams.warmup == value) {\n        return;\n    }\n\n    cparams.warmup = value;\n\n    // warmups are usually with small batches, so no need to reserve\n    //sched_need_reserve = true;\n}\n\nbool llama_context::set_sampler(llama_seq_id seq_id, llama_sampler * sampler) {\n    if (!sampler && sampling.samplers.count(seq_id) == 0) {\n        return true;\n    }\n\n    LLAMA_LOG_DEBUG(\"%s: seq_id = %d, sampler = %p\\n\", __func__, (int) seq_id, (void *) sampler);\n\n    const bool can_offload =\n        sampler &&\n        sampler->iface->backend_init &&\n        sampler->iface->backend_apply &&\n        llama_sampler_chain_n(sampler) > 0;\n\n    if (sampler && can_offload) {\n        auto * buft = ggml_backend_dev_buffer_type(model.dev_output());\n\n        sampler->iface->backend_init(sampler, buft);\n\n        sampling.samplers[seq_id] = sampler;\n\n        sched_need_reserve = true;\n\n        return true;\n    }\n\n    if (sampler && !can_offload) {\n        LLAMA_LOG_WARN(\"%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\\n\", __func__, llama_sampler_name(sampler), seq_id);\n\n        if (sampling.samplers.count(seq_id) > 0) {\n            sched_need_reserve = true;\n        }\n\n        sampling.samplers.erase(seq_id);\n\n        return false;\n    }\n\n    sampling.samplers.erase(seq_id);\n\n    sched_need_reserve = true;\n\n    return true;\n}\n\nvoid llama_context::set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {\n    LLAMA_LOG_DEBUG(\"%s: adapters = %p\\n\", __func__, (void *) adapters);\n\n    if (adapters_lora_are_same(adapters, n_adapters, scales)) {\n        return;\n    }\n\n    loras.reset(new llama_adapter_loras());\n\n    for (size_t i = 0; i < n_adapters; i ++) {\n        if (scales[i] != 0.0f) {\n            loras->insert({adapters[i], scales[i]});\n        }\n    }\n\n    sched_need_reserve = true;\n}\n\nbool llama_context::adapters_lora_are_same(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {\n    LLAMA_LOG_DEBUG(\"%s: adapters = %p\\n\", __func__, (void *) adapters);\n\n    // Adapters with a zero scale are never added to `loras`, so also ignore them for the comparison.\n    size_t n_non_zero = 0;\n\n    for (size_t i = 0; i < n_adapters; i ++) {\n        if (scales[i] == 0.0f) {\n            continue;\n        }\n        n_non_zero++;\n\n        auto it = loras->find(adapters[i]);\n\n        if (it == loras->end() || it->second != scales[i]) {\n            return false;\n        }\n    }\n\n    if (n_non_zero != loras->size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_context::set_adapter_cvec(\n            const float * data,\n                 size_t   len,\n                int32_t   n_embd,\n                int32_t   il_start,\n                int32_t   il_end) {\n    LLAMA_LOG_DEBUG(\"%s: il_start = %d, il_end = %d\\n\", __func__, il_start, il_end);\n\n    // TODO: should we reserve?\n\n    return cvec->apply(model, data, len, n_embd, il_start, il_end);\n}\n\nllm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) {\n    if (mctx && !mctx->apply()) {\n        LLAMA_LOG_ERROR(\"%s: failed to apply memory context\\n\", __func__);\n        ret = GGML_STATUS_FAILED;\n        return nullptr;\n    }\n\n    auto * res = gf_res_prev.get();\n    auto * gf  = res->get_gf();\n\n    // the new graph parameters\n    // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters\n    const auto gparams = graph_params(res, ubatch, mctx, gtype);\n\n    if (!graph_reuse_disable && res->can_reuse(gparams)) {\n        //LLAMA_LOG_DEBUG(\"%s: reusing previous graph\\n\", __func__);\n\n        n_reused++;\n    } else {\n        res->reset();\n\n        ggml_backend_sched_reset(sched.get());\n        ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);\n\n        //const auto t_start_us = ggml_time_us();\n\n        gf = model.build_graph(gparams);\n\n        //LLAMA_LOG_INFO(\"graph build time: %.3f ms\\n\", (ggml_time_us() - t_start_us)/1000.0);\n\n        if (!gf) {\n            LLAMA_LOG_ERROR(\"%s: failed to initialize graph\\n\", __func__);\n            ret = GGML_STATUS_FAILED;\n            return nullptr;\n        }\n\n        if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {\n            LLAMA_LOG_ERROR(\"%s: failed to allocate graph\\n\", __func__);\n            ret = GGML_STATUS_ALLOC_FAILED;\n            return nullptr;\n        }\n    }\n\n    // set the input data for the input tensors\n    {\n        //const auto t_start_us = ggml_time_us();\n\n        // FIXME this call causes a crash if any model inputs were not used in the graph and were therefore not allocated\n        res->set_inputs(&ubatch);\n\n        //LLAMA_LOG_INFO(\"graph set inputs time: %.3f ms\\n\", (ggml_time_us() - t_start_us)/1000.0);\n    }\n\n    const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1);\n    if (status != GGML_STATUS_SUCCESS) {\n        LLAMA_LOG_ERROR(\"%s: failed to compute graph, compute status: %d\\n\", __func__, status);\n        ret = status;\n        return nullptr;\n    }\n\n    ret = GGML_STATUS_SUCCESS;\n\n    return res;\n}\n\nint llama_context::encode(const llama_batch & batch_inp) {\n    GGML_ASSERT((!batch_inp.token && batch_inp.embd) || (batch_inp.token && !batch_inp.embd)); // NOLINT\n\n    if (batch_inp.n_tokens == 0) {\n        LLAMA_LOG_ERROR(\"%s: n_tokens == 0\\n\", __func__);\n        return -1;\n    }\n\n    const auto & hparams = model.hparams;\n\n    const int64_t n_embd  = hparams.n_embd_inp();\n    const int64_t n_vocab = model.vocab.n_tokens();\n\n    // note: during encode, we always pass the full sequence starting from pos = 0\n    if (!balloc->init(batch_inp, model.vocab, nullptr, n_embd, cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max, true)) {\n        LLAMA_LOG_ERROR(\"%s: failed to initialize batch\\n\", __func__);\n        return -1;\n    }\n\n    const uint32_t n_tokens = balloc->get_n_tokens();\n\n    // [TAG_NO_CACHE_PAD]\n    // TODO: add new split mode where we pad the input sequences so that ubatch.equal_seqs == true\n    const llama_ubatch ubatch = balloc->split_simple(n_tokens);\n\n    // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot\n    GGML_ASSERT(cparams.n_ubatch >= n_tokens && \"encoder requires n_ubatch >= n_tokens\");\n\n    if (t_compute_start_us == 0) {\n        t_compute_start_us = ggml_time_us();\n    }\n\n    // TODO: this clear of the buffer can easily be forgotten - need something better\n    embd_seq.clear();\n\n    sched_reserve();\n\n    n_queued_tokens += n_tokens;\n\n    // reserve output buffer\n    if (output_reserve(n_tokens) < n_tokens) {\n        LLAMA_LOG_ERROR(\"%s: could not reserve space for batch with %u outputs\\n\", __func__, n_tokens);\n        return -2;\n    };\n\n    for (uint32_t i = 0; i < n_tokens; ++i) {\n        output_ids[i] = i;\n    }\n\n    n_outputs = n_tokens;\n\n    const auto causal_attn_org = cparams.causal_attn;\n\n    // always use non-causal attention for encoder graphs\n    // TODO: this is a tmp solution until we have a proper way to support enc-dec models\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223\n    cparams.causal_attn = false;\n\n    ggml_status status;\n    const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_ENCODER, nullptr, status);\n\n    cparams.causal_attn = causal_attn_org;\n\n    if (!res) {\n        switch (status) {\n            case GGML_STATUS_ABORTED:      return  2;\n            case GGML_STATUS_ALLOC_FAILED: return -2;\n            case GGML_STATUS_FAILED:       return -3;\n            case GGML_STATUS_SUCCESS:      GGML_ABORT(\"should not happen\");\n        }\n    }\n\n    auto * t_logits = res->get_logits();\n    auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd();\n\n    // extract logits\n    if (logits.data && t_logits) {\n        ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);\n        GGML_ASSERT(backend_res != nullptr);\n        GGML_ASSERT(logits.data != nullptr);\n\n        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float));\n    }\n\n    // extract embeddings\n    if (embd.data && t_embd) {\n        ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);\n        GGML_ASSERT(backend_embd != nullptr);\n\n        switch (cparams.pooling_type) {\n            case LLAMA_POOLING_TYPE_NONE:\n                {\n                    // extract token embeddings\n                    GGML_ASSERT(embd.data != nullptr);\n                    const uint32_t n_embd_out = hparams.n_embd_out();\n\n                    GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size);\n                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float));\n                } break;\n            case LLAMA_POOLING_TYPE_MEAN:\n            case LLAMA_POOLING_TYPE_CLS:\n            case LLAMA_POOLING_TYPE_LAST:\n                {\n                    // extract sequence embeddings\n                    auto & embd_seq_out = embd_seq;\n\n                    for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n                        const llama_seq_id seq_id  = ubatch.seq_id_unq[s];\n                        const int32_t      seq_idx = ubatch.seq_idx[seq_id];\n\n                        embd_seq_out[seq_id].resize(n_embd);\n                        ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_idx)*sizeof(float), n_embd*sizeof(float));\n                    }\n                } break;\n            case LLAMA_POOLING_TYPE_RANK:\n                {\n                    // extract the rerank score - n_cls_out floats per sequence\n                    auto & embd_seq_out = embd_seq;\n\n                    const uint32_t n_cls_out = hparams.n_cls_out;\n\n                    for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n                        const llama_seq_id seq_id  = ubatch.seq_id_unq[s];\n                        const int32_t      seq_idx = ubatch.seq_idx[seq_id];\n\n                        embd_seq_out[seq_id].resize(n_cls_out);\n                        ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_out*seq_idx)*sizeof(float), n_cls_out*sizeof(float));\n                    }\n                } break;\n            case LLAMA_POOLING_TYPE_UNSPECIFIED:\n                {\n                    GGML_ABORT(\"unknown pooling type\");\n                }\n        }\n    }\n\n    // TODO: hacky solution\n    if (model.arch == LLM_ARCH_T5 && t_embd) {\n        //cross.t_embd = t_embd;\n\n        synchronize();\n\n        cross.n_embd = t_embd->ne[0];\n        cross.n_enc  = t_embd->ne[1];\n        cross.v_embd.resize(cross.n_embd*cross.n_enc);\n        memcpy(cross.v_embd.data(), embd.data, ggml_nbytes(t_embd));\n\n        const auto & batch = balloc->get_batch();\n\n        // remember the sequence ids used during the encoding - needed for cross attention later\n        cross.seq_ids_enc.resize(n_tokens);\n        for (uint32_t i = 0; i < n_tokens; i++) {\n            cross.seq_ids_enc[i].clear();\n\n            for (int s = 0; s < batch.n_seq_id[i]; s++) {\n                const llama_seq_id seq_id = batch.seq_id[i][s];\n\n                cross.seq_ids_enc[i].insert(seq_id);\n            }\n        }\n    }\n\n    return 0;\n}\n\nstatic std::map<llama_seq_id, uint32_t> build_seq_to_output_row(const llama_ubatch & ubatch, uint32_t row_offset) {\n    std::map<llama_seq_id, uint32_t> seq_to_row;\n    // how many output tokens we have seen so far for this ubatch.\n    uint32_t local = 0;\n    for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {\n        // skip tokens that are not output.\n        if (!ubatch.output[i]) {\n            continue;\n        }\n\n        const llama_seq_id seq_id = ubatch.seq_id[i][0];\n        // row_offset is the number of output tokens before this ubatch.\n        seq_to_row[seq_id] = row_offset + local;\n        ++local;\n    }\n    return seq_to_row;\n}\n\nstatic void copy_tensor_async_ints(\n    const std::map<llama_seq_id, ggml_tensor*> & tensor_map,\n    const buffer_view<llama_token> & sampled,\n    const std::map<llama_seq_id, uint32_t> & seq_to_row,\n    ggml_backend_sched_t sched) {\n    if (!sampled.has_data()) {\n        return;\n    }\n\n    for (const auto & [seq_id, tensor] : tensor_map) {\n        auto it = seq_to_row.find(seq_id);\n        if (it == seq_to_row.end()) {\n            continue;\n        }\n\n        const uint32_t row = it->second;\n        GGML_ASSERT(row < sampled.size);\n\n        GGML_ASSERT(ggml_is_contiguous(tensor) && \"sampled tokens tensor must be contiguous for async copy\");\n\n        ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);\n        ggml_backend_tensor_get_async(backend, tensor, sampled.data + row, 0, sizeof(sampled.data[row]));\n    }\n}\n\nstatic void copy_tensor_async_floats(\n    const std::map<llama_seq_id, ggml_tensor*> & tensor_map,\n    const buffer_view<float> & dst,\n    size_t stride,\n    std::vector<uint32_t> & counts,\n    const std::map<llama_seq_id, uint32_t> & seq_to_row,\n    ggml_backend_sched_t sched) {\n    if (!dst.has_data()) {\n        return;\n    }\n\n    for (const auto & [seq_id, tensor] : tensor_map) {\n        auto it = seq_to_row.find(seq_id);\n        if (it == seq_to_row.end()) {\n            continue;\n        }\n\n        const uint32_t row = it->second;\n        GGML_ASSERT(row < counts.size());\n\n        GGML_ASSERT(ggml_is_contiguous(tensor) && \"logits/probs tensor must be contiguous for async copy\");\n\n        ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);\n        float * row_ptr = dst.data + (size_t) row * stride;\n        ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor));\n\n        // Update the actual number of logits/probabilities that were written for this row.\n        counts[row] = ggml_nelements(tensor);\n    }\n}\n\nstatic void copy_tensor_async_candidates(\n    const std::map<llama_seq_id, ggml_tensor*> & tensor_map,\n    const buffer_view<llama_token> & dst,\n    size_t stride,\n    std::vector<uint32_t> & counts,\n    const std::map<llama_seq_id, uint32_t> & seq_to_row,\n    ggml_backend_sched_t sched) {\n    if (!dst.has_data()) {\n        return;\n    }\n\n    for (const auto & [seq_id, tensor] : tensor_map) {\n        auto it = seq_to_row.find(seq_id);\n        if (it == seq_to_row.end()) {\n            continue;\n        }\n\n        const uint32_t row = it->second;\n        GGML_ASSERT(row < counts.size());\n\n        GGML_ASSERT(ggml_is_contiguous(tensor) && \"candidates tensor must be contiguous for async copy\");\n\n        ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);\n        llama_token * row_ptr = dst.data + (size_t) row * stride;\n        ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor));\n\n        // Update the actual number of candidates that were written.\n        counts[row] = ggml_nelements(tensor);\n    }\n}\n\nstatic bool needs_raw_logits(const llama_ubatch & ubatch, const std::map<llama_seq_id, llama_sampler *> & samplers) {\n    for (uint32_t i = 0; i < ubatch.n_tokens; i++) {\n        if (!ubatch.output[i]) {\n            continue;\n        }\n\n        // Check if the output token has at least one sequence without a backend sampler.\n        for (int32_t j = 0; j < ubatch.n_seq_id[i]; ++j) {\n            llama_seq_id seq_id = ubatch.seq_id[i][j];\n            if (samplers.find(seq_id) == samplers.end()) {\n                return true;\n            }\n        }\n    }\n    return false; // all sequences use backend sampling\n}\n\nint llama_context::decode(const llama_batch & batch_inp) {\n    GGML_ASSERT((!batch_inp.token && batch_inp.embd) || (batch_inp.token && !batch_inp.embd)); // NOLINT\n\n    if (!memory) {\n        LLAMA_LOG_DEBUG(\"%s: cannot decode batches with this context (calling encode() instead)\\n\", __func__);\n        return encode(batch_inp);\n    }\n\n    if (batch_inp.n_tokens == 0) {\n        LLAMA_LOG_ERROR(\"%s: n_tokens == 0\\n\", __func__);\n        return -1;\n    }\n\n    const auto & vocab   = model.vocab;\n    const auto & hparams = model.hparams;\n\n    const int64_t n_vocab = vocab.n_tokens();\n    const int64_t n_embd  = hparams.n_embd_inp();\n\n    // when computing embeddings, all tokens are output\n    const bool output_all   = cparams.embeddings;\n    const bool has_samplers = !sampling.samplers.empty();\n\n    const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max;\n\n    // TODO: avoid this workaround in the future\n    if (has_samplers && batch_inp.logits) {\n        std::vector<int32_t> seq_output_count(n_seq_max, 0);\n\n        for (int32_t i = 0; i < batch_inp.n_tokens; ++i) {\n            if (batch_inp.logits[i] == 0) {\n                continue;\n            }\n\n            const int ns = batch_inp.n_seq_id ? batch_inp.n_seq_id[i] : 1;\n\n            for (int32_t s = 0; s < ns; ++s) {\n                const llama_seq_id seq_id = batch_inp.seq_id ? batch_inp.seq_id[i][s] : 0;\n\n                seq_output_count[seq_id]++;\n                if (seq_output_count[seq_id] > 1) {\n                    LLAMA_LOG_ERROR(\"%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\\n\",\n                            __func__, seq_id, seq_output_count[seq_id]);\n                    return -1;\n                }\n            }\n        }\n    }\n\n    if (!balloc->init(batch_inp, vocab, memory.get(), n_embd, n_seq_max, output_all)) {\n        LLAMA_LOG_ERROR(\"%s: failed to initialize batch\\n\", __func__);\n        return -1;\n    }\n\n    const uint32_t n_tokens_all  = balloc->get_n_tokens();\n    const uint32_t n_outputs_all = balloc->get_n_outputs();\n\n    if (output_all) {\n        // require that all tokens are output\n        if (n_outputs_all != n_tokens_all) {\n            LLAMA_LOG_ERROR(\"%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\\n\",\n                    __func__, n_outputs_all, n_tokens_all);\n            return -1;\n        }\n    }\n\n    GGML_ASSERT(n_tokens_all <= cparams.n_batch);\n\n    GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && \"non-causal attention requires n_ubatch >= n_tokens\");\n\n    if (t_compute_start_us == 0) {\n        t_compute_start_us = ggml_time_us();\n    }\n    n_queued_tokens += n_tokens_all;\n\n    // TODO: this clear of the buffer can easily be forgotten - need something better\n    embd_seq.clear();\n    output_swaps.clear();\n\n    sched_reserve();\n\n    bool did_optimize = false;\n\n    // handle any pending shifts/copies\n    memory_update(false);\n\n    llama_memory_context_ptr mctx;\n\n    while (true) {\n        mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all);\n        if (!mctx) {\n            return -2;\n        }\n\n        switch (mctx->get_status()) {\n            case LLAMA_MEMORY_STATUS_SUCCESS:\n                {\n                } break;\n            case LLAMA_MEMORY_STATUS_NO_UPDATE:\n                {\n                    LLAMA_LOG_ERROR(\"%s: unexpected memory context status: %d\\n\", __func__, mctx->get_status());\n\n                    return -2;\n                }\n            case LLAMA_MEMORY_STATUS_FAILED_PREPARE:\n                {\n                    if (!did_optimize) {\n                        did_optimize = true;\n\n                        if (memory_update(true)) {\n                            LLAMA_LOG_DEBUG(\"%s: retrying batch size %d after cache optimization\\n\", __func__, balloc->get_n_tokens());\n\n                            continue;\n                        }\n                    }\n\n                    LLAMA_LOG_WARN(\"%s: failed to find a memory slot for batch of size %d\\n\", __func__, balloc->get_n_tokens());\n\n                    return 1;\n                }\n            case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:\n                {\n                    LLAMA_LOG_ERROR(\"%s: compute failed while preparing batch of size %d\\n\", __func__, balloc->get_n_tokens());\n\n                    return -2;\n                }\n        }\n\n        break;\n    }\n\n    // reserve output buffer\n    if (output_reserve(n_outputs_all) < n_outputs_all) {\n        LLAMA_LOG_ERROR(\"%s: could not reserve space for batch with %d outputs\\n\", __func__, n_outputs_all);\n        return -2;\n    };\n\n    int64_t n_outputs_prev = 0;\n\n    do {\n        const auto & ubatch = mctx->get_ubatch();\n\n        // count the outputs in this ubatch\n        {\n            int32_t n_outputs_new = 0;\n\n            if (n_outputs_all == n_tokens_all) {\n                n_outputs_new = ubatch.n_tokens;\n            } else {\n                for (uint32_t i = 0; i < ubatch.n_tokens; i++) {\n                    n_outputs_new += (int32_t) (ubatch.output[i] != 0);\n                }\n            }\n\n            // needs to happen before the graph is built\n            n_outputs = n_outputs_new;\n        }\n\n        ggml_status status;\n        const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_DECODER, mctx.get(), status);\n\n        if (!res) {\n            // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module\n            llama_pos pos_min[LLAMA_MAX_SEQ];\n            for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n                pos_min[s] = std::numeric_limits<llama_pos>::max();\n            }\n\n            for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {\n                const auto & seq_id = ubatch.seq_id[i][0];\n\n                pos_min[seq_id] = std::min(pos_min[seq_id], ubatch.pos[i]);\n            }\n\n            for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n                if (pos_min[s] == std::numeric_limits<llama_pos>::max()) {\n                    continue;\n                }\n\n                LLAMA_LOG_WARN(\"%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\\n\", __func__, s, pos_min[s]);\n\n                memory->seq_rm(s, pos_min[s], -1);\n            }\n\n            switch (status) {\n                case GGML_STATUS_ABORTED:      return  2;\n                case GGML_STATUS_ALLOC_FAILED: return -2;\n                case GGML_STATUS_FAILED:       return -3;\n                case GGML_STATUS_SUCCESS:      GGML_ABORT(\"should not happen\");\n            }\n        }\n\n        // plot the computation graph in dot format (for debugging purposes)\n        //if (n_past%100 == 0) {\n        //    ggml_graph_dump_dot(gf, NULL, \"llama.dot\");\n        //}\n\n        auto * t_logits = res->get_logits();\n        auto * t_embd   = cparams.embeddings ? res->get_embd() : nullptr;\n\n        if (t_embd && res->get_embd_pooled()) {\n            t_embd = res->get_embd_pooled();\n        }\n\n        // extract logits\n        if (logits.data && t_logits && n_outputs > 0 && needs_raw_logits(ubatch, sampling.samplers)) {\n            ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);\n            GGML_ASSERT(backend_res != nullptr);\n            GGML_ASSERT(logits.data != nullptr);\n\n            float * logits_out = logits.data + n_outputs_prev*n_vocab;\n\n            if (n_outputs) {\n                GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);\n                GGML_ASSERT((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits.size);\n                ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputs*n_vocab*sizeof(float));\n            }\n        }\n\n        // extract embeddings\n        if (embd.data && t_embd && n_outputs > 0) {\n            ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);\n            GGML_ASSERT(backend_embd != nullptr);\n\n            switch (cparams.pooling_type) {\n                case LLAMA_POOLING_TYPE_NONE:\n                    {\n                        // extract token embeddings\n                        GGML_ASSERT(embd.data != nullptr);\n                        const uint32_t n_embd_out = hparams.n_embd_out();\n                        float * embd_out = embd.data + n_outputs_prev*n_embd_out;\n\n                        if (n_outputs) {\n                            GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);\n                            GGML_ASSERT((n_outputs_prev + n_outputs)*n_embd_out <= (int64_t) embd.size);\n                            ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputs*n_embd_out*sizeof(float));\n                        }\n                    } break;\n                case LLAMA_POOLING_TYPE_MEAN:\n                case LLAMA_POOLING_TYPE_CLS:\n                case LLAMA_POOLING_TYPE_LAST:\n                    {\n                        // extract sequence embeddings (cleared before processing each batch)\n                        auto & embd_seq_out = embd_seq;\n\n                        for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n                            const llama_seq_id seq_id  = ubatch.seq_id_unq[s];\n                            const int32_t      seq_idx = ubatch.seq_idx[seq_id];\n\n                            embd_seq_out[seq_id].resize(n_embd);\n                            ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_idx)*sizeof(float), n_embd*sizeof(float));\n                        }\n                    } break;\n                case LLAMA_POOLING_TYPE_RANK:\n                    {\n                        // extract the rerank score - n_cls_out floats per sequence\n                        auto & embd_seq_out = embd_seq;\n\n                        const uint32_t n_cls_out = hparams.n_cls_out;\n\n                        for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n                            const llama_seq_id seq_id  = ubatch.seq_id_unq[s];\n                            const int32_t      seq_idx = ubatch.seq_idx[seq_id];\n\n                            embd_seq_out[seq_id].resize(n_cls_out);\n                            ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_out*seq_idx)*sizeof(float), n_cls_out*sizeof(float));\n                        }\n                    } break;\n                case LLAMA_POOLING_TYPE_UNSPECIFIED:\n                    {\n                        GGML_ABORT(\"unknown pooling type\");\n                    }\n            }\n        }\n\n        // Copy backend sampling output if this ubatch produced any sampling tensors.\n        if (has_samplers && (!res->t_sampled.empty() || !res->t_sampled_probs.empty() || !res->t_sampled_logits.empty())) {\n            const auto seq_to_output_row = build_seq_to_output_row(ubatch, n_outputs_prev);\n            const auto stride = n_vocab;\n\n            // async copy the sampling data from the backend to the host\n            copy_tensor_async_ints(res->t_sampled, sampling.sampled, seq_to_output_row, sched.get());\n\n            copy_tensor_async_floats    (res->t_sampled_logits, sampling.logits,     stride, sampling.logits_count,     seq_to_output_row, sched.get());\n            copy_tensor_async_floats    (res->t_sampled_probs,  sampling.probs,      stride, sampling.probs_count,      seq_to_output_row, sched.get());\n            copy_tensor_async_candidates(res->t_candidates,     sampling.candidates, stride, sampling.candidates_count, seq_to_output_row, sched.get());\n        }\n\n        n_outputs_prev += n_outputs;\n    } while (mctx->next());\n\n    // set to total number of outputs in the batch, for use in llama_get_logits_ith\n    n_outputs = n_outputs_all;\n\n    // set output mappings\n    if (n_outputs > 0) {\n        bool sorted_output = true;\n\n        auto & out_ids = balloc->get_out_ids();\n\n        GGML_ASSERT(out_ids.size() == (size_t) n_outputs);\n\n        for (int64_t i = 0; i < n_outputs; ++i) {\n            int64_t out_id = out_ids[i];\n            output_ids[out_id] = i;\n            if (out_id != i) {\n                sorted_output = false;\n            }\n        }\n\n        // make the outputs have the same order they had in the user-provided batch\n        // note: this is mostly relevant for recurrent models atm\n        if (!sorted_output && n_outputs > 1) {\n            GGML_ASSERT((size_t) n_outputs == out_ids.size());\n\n            // TODO: is there something more efficient which also minimizes swaps?\n            // selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort)\n            for (uint32_t i = 0; i < n_outputs - 1; ++i) {\n                uint32_t j_min = i;\n                for (uint32_t j = i + 1; j < n_outputs; ++j) {\n                    if (out_ids[j] < out_ids[j_min]) {\n                        j_min = j;\n                    }\n                }\n                if (j_min == i) {\n                    continue;\n                }\n                std::swap(out_ids[i], out_ids[j_min]);\n\n                // remember the swaps and apply them lazily upon logits/embeddings access\n                output_swaps.push_back({ i, j_min });\n            }\n\n            std::fill(output_ids.begin(), output_ids.end(), -1);\n\n            for (uint32_t i = 0; i < n_outputs; ++i) {\n                output_ids[out_ids[i]] = i;\n            }\n        }\n    }\n\n    // wait for the computation to finish (automatically done when obtaining the model output)\n    //synchronize();\n\n    return 0;\n}\n\n//\n// output\n//\n\nuint32_t llama_context::output_reserve(int32_t n_outputs) {\n    const auto & hparams = model.hparams;\n    const auto & vocab   = model.vocab;\n\n    const int64_t n_outputs_max = std::max<int64_t>(n_outputs, n_seq_max());\n\n    const auto n_batch    = cparams.n_batch;\n    const auto n_vocab    = vocab.n_tokens();\n    const auto n_embd_out = hparams.n_embd_out();\n\n    bool has_logits = true;\n    bool has_embd   = cparams.embeddings;\n\n    // TODO: hacky enc-dec support\n    if (model.arch == LLM_ARCH_T5) {\n        has_logits = true;\n        has_embd   = true;\n    }\n\n\n    size_t backend_float_count = 0;\n    size_t backend_token_count = 0;\n\n    logits.size = has_logits ? n_vocab*n_outputs_max : 0;\n    embd.size   = has_embd ? n_embd_out*n_outputs_max : 0;\n\n    // Allocate backend sampling output buffers if there are backend samplers configured.\n    const bool has_sampling = !sampling.samplers.empty();\n    if (has_sampling) {\n        backend_float_count = 2 * n_vocab * n_outputs_max;      // logits + probs\n        backend_token_count = (1 + n_vocab) * n_outputs_max;    // sampled + candidates\n    }\n\n    if (output_ids.empty()) {\n        // init, never resized afterwards\n        output_ids.resize(n_batch);\n    }\n\n    const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0;\n    const size_t new_size  =\n        (logits.size + embd.size + backend_float_count) * sizeof(float) +\n        (                          backend_token_count) * sizeof(llama_token);\n\n    // alloc only when more than the current capacity is required\n    // TODO: also consider shrinking the buffer\n    if (!buf_output || prev_size < new_size) {\n        if (buf_output) {\n#ifndef NDEBUG\n            // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark)\n            LLAMA_LOG_DEBUG(\"%s: reallocating output buffer from size %.02f MiB to %.02f MiB\\n\", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);\n#endif\n            synchronize();\n\n            // TODO: not needed?\n            buf_output = nullptr;\n            logits.data = nullptr;\n            embd.data = nullptr;\n        }\n\n        auto * buft = ggml_backend_cpu_buffer_type();\n        // try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory\n        auto * output_dev = model.dev_output();\n        auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr;\n        if (output_dev_host_buft) {\n            buft = output_dev_host_buft;\n        }\n        buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size));\n        if (buf_output == nullptr) {\n            LLAMA_LOG_ERROR(\"%s: failed to allocate output buffer of size %.2f MiB\\n\", __func__, new_size / (1024.0 * 1024.0));\n            return 0;\n        }\n    }\n\n    float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get());\n\n    size_t offset = 0;\n    uint8_t * base = (uint8_t *) output_base;\n\n    logits = has_logits ? buffer_view<float>{output_base, logits.size} : buffer_view<float>{nullptr, 0};\n    offset += logits.size * sizeof(float);\n\n    embd = has_embd ? buffer_view<float>{(float *) (base + offset), embd.size} : buffer_view<float>{nullptr, 0};\n    offset += embd.size * sizeof(float);\n\n    if (has_sampling) {\n        sampling.logits = {(float *) (base + offset), (size_t)(n_vocab*n_outputs_max)};\n        offset += sampling.logits.size * sizeof(float);\n\n        sampling.probs = {(float *) (base + offset), (size_t)(n_vocab*n_outputs_max)};\n        offset += sampling.probs.size * sizeof(float);\n\n        sampling.sampled = {(llama_token *) (base + offset), (size_t)n_outputs_max};\n        offset += sampling.sampled.size * sizeof(llama_token);\n\n        sampling.candidates = {(llama_token *) (base + offset), (size_t)(n_vocab*n_outputs_max)};\n        offset += sampling.candidates.size * sizeof(llama_token);\n\n        // The count vectors keep track of the actual number of logits/probs/candidates\n        // copied from the backend for each output row.\n\n        sampling.logits_count.resize(n_outputs_max);\n        sampling.probs_count.resize(n_outputs_max);\n        sampling.candidates_count.resize(n_outputs_max);\n\n        std::fill(sampling.logits_count.begin(),     sampling.logits_count.end(),     0);\n        std::fill(sampling.probs_count.begin(),      sampling.probs_count.end(),      0);\n        std::fill(sampling.candidates_count.begin(), sampling.candidates_count.end(), 0);\n\n        std::fill_n(sampling.sampled.data, sampling.sampled.size, LLAMA_TOKEN_NULL);\n    } else {\n        sampling.logits     = {nullptr, 0};\n        sampling.probs      = {nullptr, 0};\n        sampling.sampled    = {nullptr, 0};\n        sampling.candidates = {nullptr, 0};\n\n        sampling.logits_count.clear();\n        sampling.probs_count.clear();\n        sampling.candidates_count.clear();\n    }\n\n    // set all ids as invalid (negative)\n    std::fill(output_ids.begin(), output_ids.end(), -1);\n\n    this->n_outputs = 0;\n\n    return n_outputs_max;\n}\n\nvoid llama_context::output_reorder() {\n    const uint64_t n_vocab = model.vocab.n_tokens();\n    const uint64_t n_embd  = model.hparams.n_embd;\n\n    for (size_t s = 0; s < output_swaps.size(); ++s) {\n        const uint64_t i0 = output_swaps[s].i0;\n        const uint64_t i1 = output_swaps[s].i1;\n\n        if (logits.size > 0) {\n            for (uint64_t k = 0; k < n_vocab; k++) {\n                std::swap(logits.data[i0*n_vocab + k], logits.data[i1*n_vocab + k]);\n            }\n        }\n\n        if (embd.size > 0) {\n            for (uint64_t k = 0; k < n_embd; k++) {\n                std::swap(embd.data[i0*n_embd + k], embd.data[i1*n_embd + k]);\n            }\n        }\n\n        if (!sampling.samplers.empty()) {\n            assert(sampling.logits.size > 0);\n            assert(sampling.probs.size > 0);\n            assert(sampling.candidates.size > 0);\n            assert(sampling.sampled.size > 0);\n            assert(sampling.logits_count.size() > 0);\n            assert(sampling.probs_count.size() > 0);\n            assert(sampling.candidates_count.size() > 0);\n\n            for (uint64_t k = 0; k < n_vocab; ++k) {\n                std::swap(sampling.logits.data[i0*n_vocab + k], sampling.logits.data[i1*n_vocab + k]);\n            }\n\n            for (uint64_t k = 0; k < n_vocab; ++k) {\n                std::swap(sampling.probs.data[i0*n_vocab + k], sampling.probs.data[i1*n_vocab + k]);\n            }\n\n            for (uint64_t k = 0; k < n_vocab; ++k) {\n                std::swap(sampling.candidates.data[i0*n_vocab + k], sampling.candidates.data[i1*n_vocab + k]);\n            }\n\n            std::swap(sampling.sampled.data[i0],     sampling.sampled.data[i1]);\n            std::swap(sampling.logits_count[i0],     sampling.logits_count[i1]);\n            std::swap(sampling.probs_count[i0],      sampling.probs_count[i1]);\n            std::swap(sampling.candidates_count[i0], sampling.candidates_count[i1]);\n        }\n    }\n\n    output_swaps.clear();\n}\n\n//\n// graph\n//\n\nuint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {\n    if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) {\n        return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());\n    }\n    uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors());\n    for (const auto & lora : model.loras) {\n        res += lora->get_n_nodes();\n    }\n    return res;\n}\n\nllm_graph_result * llama_context::get_gf_res_reserve() const {\n    return static_cast<llm_graph_result *>(gf_res_reserve.get());\n}\n\nggml_cgraph * llama_context::graph_reserve(\n        uint32_t n_tokens, uint32_t n_seqs, uint32_t n_outputs, const llama_memory_context_i * mctx, bool split_only, size_t * sizes) {\n    LLAMA_LOG_DEBUG(\"%s: reserving a graph for ubatch with n_tokens = %4u, n_seqs = %2u, n_outputs = %4u\\n\", __func__, n_tokens, n_seqs, n_outputs);\n    GGML_ASSERT(n_outputs >= 1);\n\n    if (n_tokens % n_seqs != 0) {\n        n_tokens = ((n_tokens + (n_seqs - 1)) / n_seqs) * n_seqs; // round to next multiple of n_seqs\n        n_outputs = std::max(n_outputs, n_tokens);\n\n        LLAMA_LOG_DEBUG(\"%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\\n\", __func__, n_tokens, n_seqs, n_outputs);\n    }\n\n    ggml_backend_sched_reset(sched.get());\n\n    // when the scheduler is reset, we cannot reuse the old graph, so we reset the previous graph result to prevent that\n    gf_res_prev->reset();\n\n    // store the n_outputs as it is, and restore it afterwards\n    // TODO: not sure if needed, might simplify in the future by removing this\n    const auto save_n_outputs = this->n_outputs;\n\n    this->n_outputs = n_outputs;\n\n    llama_batch_allocr balloc(model.hparams.n_pos_per_embd());\n    llama_ubatch ubatch = balloc.ubatch_reserve(n_tokens/n_seqs, n_seqs);\n\n    // set one output token per sequence in order to activate all backend samplers\n    std::vector<llama_seq_id> seq_ids(n_seqs);\n    for (uint32_t i = 0; i < n_seqs; ++i) {\n        seq_ids[i] = i;\n        ubatch.n_seq_id[i] = 1;\n        ubatch.seq_id[i] = &seq_ids[i];\n        ubatch.output[i] = true;\n    }\n\n    auto * res = gf_res_reserve.get();\n\n    const auto gparams = graph_params(res, ubatch, mctx, LLM_GRAPH_TYPE_DEFAULT);\n\n    res->reset();\n\n    auto * gf = model.build_graph(gparams);\n\n    this->n_outputs = save_n_outputs;\n\n    // initialize scheduler with the specified graph\n    if (split_only) {\n        if (sizes) {\n            ggml_backend_sched_reserve_size(sched.get(), gf, sizes);\n        } else {\n            ggml_backend_sched_split_graph(sched.get(), gf);\n        }\n    } else if (!ggml_backend_sched_reserve(sched.get(), gf)) {\n        GGML_ASSERT(!sizes);\n        LLAMA_LOG_ERROR(\"%s: failed to allocate compute buffers\\n\", __func__);\n        return nullptr;\n    }\n\n    return gf;\n}\n\nllm_graph_params llama_context::graph_params(\n                        llm_graph_result * res,\n                      const llama_ubatch & ubatch,\n            const llama_memory_context_i * mctx,\n                          llm_graph_type   gtype) const {\n    return {\n        /*.arch        =*/ model.arch,\n        /*.hparams     =*/ model.hparams,\n        /*.cparams     =*/ cparams,\n        /*.ubatch      =*/ ubatch,\n        /*.gtype       =*/ gtype,\n        /*.sched       =*/ sched.get(),\n        /*.backend_cpu =*/ backend_cpu,\n        /*.cvec        =*/ cvec.get(),\n        /*.loras       =*/ loras.get(),\n        /*.mctx        =*/ mctx,\n        /*.cross       =*/ &cross,\n        /*.samplers    =*/ sampling.samplers,\n        /*.n_outputs   =*/ n_outputs,\n        /*.cb          =*/ graph_get_cb(),\n        /*.res         =*/ res,\n    };\n}\n\nggml_status llama_context::graph_compute(\n            ggml_cgraph * gf,\n                   bool   batched) {\n    int n_threads        = batched ? cparams.n_threads_batch : cparams.n_threads;\n    ggml_threadpool_t tp = batched ? threadpool_batch        : threadpool;\n\n    if (backend_cpu != nullptr) {\n        auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));\n        auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_cpu_set_threadpool\");\n        if (set_threadpool_fn) {\n            set_threadpool_fn(backend_cpu, tp);\n        }\n    }\n\n    // set the number of threads for all the backends\n    for (const auto & set_n_threads_fn : set_n_threads_fns) {\n        set_n_threads_fn.second(set_n_threads_fn.first, n_threads);\n    }\n\n    auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf);\n    if (status != GGML_STATUS_SUCCESS) {\n        LLAMA_LOG_ERROR(\"%s: ggml_backend_sched_graph_compute_async failed with error %d\\n\", __func__, status);\n    }\n\n    // fprintf(stderr, \"splits: %d\\n\", ggml_backend_sched_get_n_splits(sched));\n\n    return status;\n}\n\nllm_graph_cb llama_context::graph_get_cb() const {\n    return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) {\n        if (il >= 0) {\n            ggml_format_name(cur, \"%s-%d\", name, il);\n        } else {\n            ggml_set_name(cur, name);\n        }\n\n        // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends\n        // FIXME: fix in ggml_backend_sched\n        const bool full_offload = model.n_gpu_layers() > model.hparams.n_layer;\n        if (ubatch.n_tokens < 32 || full_offload) {\n            if (il != -1 && strcmp(name, \"norm\") == 0) {\n                const auto & dev_layer = model.dev_layer(il);\n                for (const auto & backend : backends) {\n                    if (ggml_backend_get_device(backend.get()) == dev_layer) {\n                        if (ggml_backend_supports_op(backend.get(), cur)) {\n                            ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get());\n                        }\n                    }\n                }\n            }\n        }\n    };\n}\n\n//\n// state save/load\n//\n\nclass llama_io_write_dummy : public llama_io_write_i {\npublic:\n    llama_io_write_dummy() = default;\n\n    void write(const void * /* src */, size_t size) override {\n        size_written += size;\n    }\n\n    void write_tensor(const ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override {\n        size_written += size;\n    }\n\n    size_t n_bytes() override {\n        return size_written;\n    }\n\nprivate:\n    size_t size_written = 0;\n};\n\nclass llama_io_write_buffer : public llama_io_write_i {\npublic:\n    llama_io_write_buffer(\n            uint8_t * p, size_t len) : ptr(p), buf_size(len) {}\n\n    void write(const void * src, size_t size) override {\n        if (size > buf_size) {\n            throw std::runtime_error(\"unexpectedly reached end of buffer\");\n        }\n        memcpy(ptr, src, size);\n        ptr += size;\n        size_written += size;\n        buf_size -= size;\n    }\n\n    void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {\n        if (size > buf_size) {\n            throw std::runtime_error(\"unexpectedly reached end of buffer\");\n        }\n        ggml_backend_tensor_get(tensor, ptr, offset, size);\n        ptr += size;\n        size_written += size;\n        buf_size -= size;\n    }\n\n    size_t n_bytes() override {\n        return size_written;\n    }\n\nprivate:\n    uint8_t * ptr;\n    size_t buf_size = 0;\n    size_t size_written = 0;\n};\n\nclass llama_io_read_buffer : public llama_io_read_i {\npublic:\n    llama_io_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}\n\n    const uint8_t * read(size_t size) override {\n        const uint8_t * base_ptr = ptr;\n        if (size > buf_size) {\n            throw std::runtime_error(\"unexpectedly reached end of buffer\");\n        }\n        ptr += size;\n        size_read += size;\n        buf_size -= size;\n        return base_ptr;\n    }\n\n    void read_to(void * dst, size_t size) override {\n        memcpy(dst, read(size), size);\n    }\n\n    size_t n_bytes() override {\n        return size_read;\n    }\n\nprivate:\n    const uint8_t * ptr;\n    size_t buf_size = 0;\n    size_t size_read = 0;\n};\n\nclass llama_io_write_file : public llama_io_write_i {\npublic:\n    llama_io_write_file(llama_file * f) : file(f) {}\n\n    void write(const void * src, size_t size) override {\n        file->write_raw(src, size);\n        size_written += size;\n    }\n\n    void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {\n        temp_buffer.resize(size);\n        ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size);\n        write(temp_buffer.data(), temp_buffer.size());\n    }\n\n    size_t n_bytes() override {\n        return size_written;\n    }\n\nprivate:\n    llama_file * file;\n    size_t size_written = 0;\n    std::vector<uint8_t> temp_buffer;\n};\n\nclass llama_io_read_file : public llama_io_read_i {\npublic:\n    llama_io_read_file(llama_file * f) : file(f) {}\n\n    void read_to(void * dst, size_t size) override {\n        file->read_raw(dst, size);\n        size_read += size;\n    }\n\n    const uint8_t * read(size_t size) override {\n        temp_buffer.resize(size);\n        read_to(temp_buffer.data(), size);\n        return temp_buffer.data();\n    }\n\n    size_t n_bytes() override {\n        return size_read;\n    }\n\nprivate:\n    llama_file * file;\n    size_t size_read = 0;\n    std::vector<uint8_t> temp_buffer;\n};\n\nsize_t llama_context::state_get_size() {\n    llama_io_write_dummy io;\n    try {\n        return state_write_data(io);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error getting state size: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::state_get_data(uint8_t * dst, size_t size) {\n    llama_io_write_buffer io(dst, size);\n    try {\n        return state_write_data(io);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error saving state: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::state_set_data(const uint8_t * src, size_t size) {\n    llama_io_read_buffer io(src, size);\n    try {\n        return state_read_data(io);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error loading state: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::state_seq_get_size(llama_seq_id seq_id, llama_state_seq_flags flags) {\n    llama_io_write_dummy io;\n    try {\n        return state_seq_write_data(io, seq_id, flags);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error getting state size: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size, llama_state_seq_flags flags) {\n    llama_io_write_buffer io(dst, size);\n    try {\n        return state_seq_write_data(io, seq_id, flags);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error saving state: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size, llama_state_seq_flags flags) {\n    llama_io_read_buffer io(src, size);\n    try {\n        return state_seq_read_data(io, seq_id, flags);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error loading state: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nbool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {\n    llama_file file(filepath, \"rb\");\n\n    // sanity checks\n    {\n        const uint32_t magic   = file.read_u32();\n        const uint32_t version = file.read_u32();\n\n        if (magic != LLAMA_SESSION_MAGIC || version != LLAMA_SESSION_VERSION) {\n            LLAMA_LOG_ERROR(\"%s: unknown (magic, version) for session file: %08x, %08x\\n\", __func__, magic, version);\n            return false;\n        }\n    }\n\n    // load the prompt\n    {\n        const uint32_t n_token_count = file.read_u32();\n\n        if (n_token_count > n_token_capacity) {\n            LLAMA_LOG_ERROR(\"%s: token count in session file exceeded capacity! %u > %zu\\n\", __func__, n_token_count, n_token_capacity);\n            return false;\n        }\n\n        file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);\n        *n_token_count_out = n_token_count;\n    }\n\n    // restore the context state\n    {\n        const size_t n_state_size_cur = file.size() - file.tell();\n\n        llama_io_read_file io( &file);\n        const size_t n_read = state_read_data(io);\n\n        if (n_read != n_state_size_cur) {\n            LLAMA_LOG_ERROR(\"%s: did not read all of the session file data! size %zu, got %zu\\n\", __func__, n_state_size_cur, n_read);\n            return false;\n        }\n    }\n\n    return true;\n}\n\nbool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) {\n    llama_file file(filepath, \"wb\");\n\n    file.write_u32(LLAMA_SESSION_MAGIC);\n    file.write_u32(LLAMA_SESSION_VERSION);\n\n    // save the prompt\n    file.write_u32((uint32_t) n_token_count);\n    file.write_raw(tokens, sizeof(llama_token) * n_token_count);\n\n    // save the context state using stream saving\n    llama_io_write_file io(&file);\n    state_write_data(io);\n\n    return true;\n}\n\nsize_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {\n    llama_file file(filepath, \"rb\");\n\n    // version checks\n    {\n        const uint32_t magic   = file.read_u32();\n        const uint32_t version = file.read_u32();\n\n        if (magic != LLAMA_STATE_SEQ_MAGIC || version != LLAMA_STATE_SEQ_VERSION) {\n            LLAMA_LOG_ERROR(\"%s: unknown (magic, version) for sequence state file: %08x, %08x\\n\", __func__, magic, version);\n            return 0;\n        }\n    }\n\n    // load the prompt\n    {\n        const uint32_t n_token_count = file.read_u32();\n\n        if (n_token_count > n_token_capacity) {\n            LLAMA_LOG_ERROR(\"%s: token count in sequence state file exceeded capacity! %u > %zu\\n\", __func__, n_token_count, n_token_capacity);\n            return 0;\n        }\n\n        file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);\n        *n_token_count_out = n_token_count;\n    }\n\n    // restore the context state\n    {\n        const size_t state_size = file.size() - file.tell();\n        llama_io_read_file io(&file);\n        const size_t nread = state_seq_read_data(io, seq_id, 0);\n        if (!nread) {\n            LLAMA_LOG_ERROR(\"%s: failed to restore sequence state\\n\", __func__);\n            return 0;\n        }\n        GGML_ASSERT(nread <= state_size);\n        GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell());\n    }\n\n    return file.tell();\n}\n\nsize_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) {\n    llama_file file(filepath, \"wb\");\n\n    file.write_u32(LLAMA_STATE_SEQ_MAGIC);\n    file.write_u32(LLAMA_STATE_SEQ_VERSION);\n\n    // save the prompt\n    file.write_u32((uint32_t) n_token_count);\n    file.write_raw(tokens, sizeof(llama_token) * n_token_count);\n\n    // save the context state using stream saving\n    llama_io_write_file io(&file);\n    state_seq_write_data(io, seq_id, 0);\n\n    const size_t res = file.tell();\n    GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes());\n\n    return res;\n}\n\nsize_t llama_context::state_write_data(llama_io_write_i & io) {\n    LLAMA_LOG_DEBUG(\"%s: writing state\\n\", __func__);\n\n    // write model info\n    {\n        LLAMA_LOG_DEBUG(\"%s: - writing model info\\n\", __func__);\n\n        const std::string arch_str = llm_arch_name(model.arch);\n        io.write_string(arch_str);\n        // TODO: add more model-specific info which should prevent loading the session file if not identical\n    }\n\n    if (memory != nullptr) {\n        LLAMA_LOG_DEBUG(\"%s: - writing memory module\\n\", __func__);\n        memory->state_write(io);\n    }\n\n    return io.n_bytes();\n}\n\nsize_t llama_context::state_read_data(llama_io_read_i & io) {\n    LLAMA_LOG_DEBUG(\"%s: reading state\\n\", __func__);\n\n    // read model info\n    {\n        LLAMA_LOG_DEBUG(\"%s: - reading model info\\n\", __func__);\n\n        const std::string cur_arch_str = llm_arch_name(model.arch);\n\n        std::string arch_str;\n        io.read_string(arch_str);\n        if (cur_arch_str != arch_str) {\n            throw std::runtime_error(format(\"wrong model arch: '%s' instead of '%s'\", arch_str.c_str(), cur_arch_str.c_str()));\n        }\n        // TODO: add more info which needs to be identical but which is not verified otherwise\n    }\n\n    if (memory) {\n        LLAMA_LOG_DEBUG(\"%s: - reading memory module\\n\", __func__);\n\n        memory->state_read(io);\n    }\n\n    return io.n_bytes();\n}\n\nsize_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    GGML_UNUSED(seq_id);\n\n    if (memory) {\n        memory->state_write(io, seq_id, flags);\n    }\n\n    return io.n_bytes();\n}\n\nsize_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    GGML_UNUSED(seq_id);\n\n    if (memory) {\n        memory->state_read(io, seq_id, flags);\n    }\n\n    return io.n_bytes();\n}\n\n//\n// perf\n//\n\nllama_perf_context_data llama_context::perf_get_data() const {\n    llama_perf_context_data data = {};\n\n    data.t_start_ms  = 1e-3 * t_start_us;\n    data.t_load_ms   = 1e-3 * t_load_us;\n    data.t_p_eval_ms = 1e-3 * t_p_eval_us;\n    data.t_eval_ms   = 1e-3 * t_eval_us;\n    data.n_p_eval    = std::max(1, n_p_eval);\n    data.n_eval      = std::max(1, n_eval);\n    data.n_reused    = std::max(0, n_reused);\n\n    return data;\n}\n\nvoid llama_context::perf_reset() {\n    t_start_us  = ggml_time_us();\n    t_eval_us   = n_eval = 0;\n    t_p_eval_us = n_p_eval = 0;\n    n_reused    = 0;\n}\n\nstd::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> llama_context::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> ret;\n    for (const auto & [buft, size] : model.memory_breakdown()) {\n        ret[buft].model += size;\n    }\n    if (memory) {\n        for (const auto & [buft, size] : memory->memory_breakdown()) {\n            ret[buft].context += size;\n        }\n    }\n    if (model.hparams.no_alloc) {\n        for (size_t i = 0; i < backends.size(); ++i) {\n            ggml_backend_t             backend = backends[i].get();\n            ggml_backend_buffer_type_t buft    = ggml_backend_sched_get_buffer_type(sched.get(), backend);\n            ret[buft].compute += backend_buf_exp_size[i];\n        }\n    } else {\n        for (const auto & backend_ptr : backends) {\n            ggml_backend_t             backend = backend_ptr.get();\n            ggml_backend_buffer_type_t buft    = ggml_backend_sched_get_buffer_type(sched.get(), backend);\n            ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);\n        }\n    }\n    return ret;\n}\n\n//\n// training\n//\n\nstatic void llama_set_param(struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) {\n    if (!tensor || tensor->type != GGML_TYPE_F32) {\n        return;\n    }\n    if (!param_filter(tensor, userdata)) {\n        return;\n    }\n    if (strcmp(tensor->name, \"token_embd.weight\") == 0) {\n        return; // FIXME\n    }\n    if (strcmp(tensor->name, \"rope_freqs.weight\") == 0) {\n        return; // FIXME\n    }\n    ggml_set_param(tensor);\n}\n\nvoid llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {\n    GGML_ASSERT(!opt_ctx);\n    model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx();\n    const uint32_t n_batch     = std::min(this->n_batch(),  model->hparams.n_ctx_train);\n    const uint32_t n_ubatch    = std::min(this->n_ubatch(), n_batch);\n    GGML_ASSERT(model->hparams.n_ctx_train % n_batch  == 0);\n    GGML_ASSERT(n_batch                    % n_ubatch == 0);\n\n    ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);\n    opt_params.opt_period      = n_batch / n_ubatch;\n    opt_params.get_opt_pars    = lopt_params.get_opt_pars;\n    opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud;\n    opt_params.optimizer       = lopt_params.optimizer_type;\n    opt_ctx = ggml_opt_init(opt_params);\n\n    llama_opt_param_filter param_filter = lopt_params.param_filter;\n    void * param_filter_ud              = lopt_params.param_filter_ud;\n\n  //llama_set_param(model->tok_embd,        param_filter, param_filter_ud); // FIXME\n    llama_set_param(model->type_embd,       param_filter, param_filter_ud);\n    llama_set_param(model->pos_embd,        param_filter, param_filter_ud);\n    llama_set_param(model->tok_norm,        param_filter, param_filter_ud);\n    llama_set_param(model->tok_norm_b,      param_filter, param_filter_ud);\n    llama_set_param(model->output_norm,     param_filter, param_filter_ud);\n    llama_set_param(model->output_norm_b,   param_filter, param_filter_ud);\n    llama_set_param(model->output,          param_filter, param_filter_ud);\n    llama_set_param(model->output_b,        param_filter, param_filter_ud);\n    llama_set_param(model->output_norm_enc, param_filter, param_filter_ud);\n    llama_set_param(model->cls,             param_filter, param_filter_ud);\n    llama_set_param(model->cls_b,           param_filter, param_filter_ud);\n    llama_set_param(model->cls_out,         param_filter, param_filter_ud);\n    llama_set_param(model->cls_out_b,       param_filter, param_filter_ud);\n    llama_set_param(model->cls_norm,        param_filter, param_filter_ud);\n\n    for (struct llama_layer & layer : model->layers) {\n        for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {\n            llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud);\n        }\n    }\n}\n\nvoid llama_context::opt_epoch_iter(\n        ggml_opt_dataset_t               dataset,\n        ggml_opt_result_t                result,\n        const std::vector<llama_token> & tokens,\n        const std::vector<llama_token> & labels_sparse,\n        llama_batch                    & batch,\n        ggml_opt_epoch_callback          callback,\n        bool                             train,\n        int64_t                          idata_in_loop,\n        int64_t                          ndata_in_loop,\n        int64_t                          t_loop_start) {\n    GGML_ASSERT(opt_ctx);\n    const uint32_t n_ctx    = llama_model_n_ctx_train(&model);\n    const uint32_t n_batch  = std::min(this->n_batch(),  n_ctx);\n    const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);\n\n    memory->clear(true);\n\n    for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {\n        batch.n_tokens = n_batch;\n        for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {\n            batch.token   [pos_batch]    = tokens[pos_ctx + pos_batch];\n            batch.pos     [pos_batch]    = pos_ctx + pos_batch;\n            batch.n_seq_id[pos_batch]    = 1;\n            batch.seq_id  [pos_batch][0] = 0;\n            batch.logits  [pos_batch]    = true;\n        }\n\n        if (!balloc->init(batch, model.vocab, nullptr, model.hparams.n_embd_inp(), cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max, true)) {\n            LLAMA_LOG_ERROR(\"%s: failed to initialize batch\\n\", __func__);\n            return;\n        }\n\n        const uint32_t n_tokens_all = balloc->get_n_tokens();\n\n        n_queued_tokens += n_tokens_all;\n\n        embd_seq.clear();\n\n        uint32_t n_outputs_all = n_tokens_all;\n\n        auto mctx = memory->init_batch(*balloc, cparams.n_ubatch, true);\n        if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) {\n            LLAMA_LOG_ERROR(\"%s: could not initialize batch\\n\", __func__);\n            break;\n        }\n\n        // reserve output buffer\n        if (output_reserve(n_outputs_all) < n_outputs_all) {\n            LLAMA_LOG_ERROR(\"%s: could not reserve space for batch with %d outputs\\n\", __func__, n_outputs_all);\n            GGML_ABORT(\"TODO: handle this error\");\n        };\n\n        uint32_t pos_batch = 0;\n        do {\n            const auto & ubatch = mctx->get_ubatch();\n\n            n_outputs = ubatch.n_tokens;\n\n            if (!mctx->apply()) {\n                LLAMA_LOG_ERROR(\"%s: failed to update the memory context\\n\", __func__);\n                break;\n            }\n\n            auto * res = gf_res_prev.get();\n\n            const auto gparams = graph_params(res, ubatch, mctx.get(), LLM_GRAPH_TYPE_DEFAULT);\n\n            res->reset();\n\n            auto * gf = model.build_graph(gparams);\n\n            struct ggml_context * ctx_compute_opt;\n            {\n                const size_t size_gf = ggml_graph_size(gf);\n                const size_t size_meta = 4*size_gf*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(size_gf, /*grads = */ true);\n                struct ggml_init_params params = {\n                    /*.mem_size   =*/ size_meta,\n                    /*.mem_buffer =*/ nullptr,\n                    /*.no_alloc   =*/ true,\n                };\n                ctx_compute_opt = ggml_init(params);\n            }\n            ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_inp_tokens(), res->get_logits());\n            ggml_opt_alloc(opt_ctx, train);\n\n            res->set_inputs(&ubatch);\n            {\n                struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);\n                GGML_ASSERT(labels->ne[1] == n_ubatch);\n                ggml_set_zero(labels);\n                const float onef = 1.0f;\n                for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) {\n                    const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch;\n                    GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0]);\n                    ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float));\n                }\n            }\n            ggml_opt_eval(opt_ctx, result);\n            if (callback) {\n                callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start);\n            }\n            ggml_free(ctx_compute_opt);\n\n            pos_batch += ubatch.n_tokens;\n        } while (mctx->next());\n    }\n}\n\nvoid llama_context::opt_epoch(\n        ggml_opt_dataset_t        dataset,\n        ggml_opt_result_t         result_train,\n        ggml_opt_result_t         result_eval,\n        int64_t                   idata_split,\n        ggml_opt_epoch_callback   callback_train,\n        ggml_opt_epoch_callback   callback_eval) {\n    const uint32_t n_ctx    = this->n_ctx();\n    const uint32_t n_batch  = std::min(cparams.n_batch,  n_ctx);\n    const uint32_t n_ubatch = std::min(cparams.n_ubatch, n_batch);\n    const  int64_t ndata    = ggml_opt_dataset_ndata(dataset);\n\n    GGML_ASSERT(idata_split >= 0);\n    GGML_ASSERT(idata_split <= ndata);\n\n    const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;\n\n    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);\n    std::vector<llama_token>        tokens(n_ctx);\n    std::vector<llama_token> labels_sparse(n_ctx);\n\n    int64_t idata = 0;\n\n    int64_t t_loop_start = ggml_time_us();\n    int64_t ndata_in_loop = idata_split*ubatch_per_ctx;\n    for (; idata < idata_split; ++idata) {\n        constexpr bool train = true;\n        const int64_t idata_in_loop = idata*ubatch_per_ctx;\n\n        ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);\n        opt_epoch_iter(dataset, result_train, tokens, labels_sparse, batch,\n            callback_train, train, idata_in_loop, ndata_in_loop, t_loop_start);\n    }\n\n    t_loop_start = ggml_time_us();\n    ndata_in_loop = (ndata - idata_split)*ubatch_per_ctx;\n    for (; idata < ndata; ++idata) {\n        constexpr bool train = false;\n        const int64_t idata_in_loop = (idata - idata_split)*ubatch_per_ctx;\n\n        ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);\n        opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,\n            callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);\n    }\n\n    llama_batch_free(batch);\n}\n\n//\n// interface implementation\n//\n\nllama_context_params llama_context_default_params() {\n    llama_context_params result = {\n        /*.n_ctx                       =*/ 512,\n        /*.n_batch                     =*/ 2048,\n        /*.n_ubatch                    =*/ 512,\n        /*.n_seq_max                   =*/ 1,\n        /*.n_threads                   =*/ GGML_DEFAULT_N_THREADS, // TODO: better default\n        /*.n_threads_batch             =*/ GGML_DEFAULT_N_THREADS,\n        /*.rope_scaling_type           =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,\n        /*.pooling_type                =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,\n        /*.attention_type              =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,\n        /*.flash_attn_type             =*/ LLAMA_FLASH_ATTN_TYPE_AUTO,\n        /*.rope_freq_base              =*/ 0.0f,\n        /*.rope_freq_scale             =*/ 0.0f,\n        /*.yarn_ext_factor             =*/ -1.0f,\n        /*.yarn_attn_factor            =*/ -1.0f,\n        /*.yarn_beta_fast              =*/ -1.0f,\n        /*.yarn_beta_slow              =*/ -1.0f,\n        /*.yarn_orig_ctx               =*/ 0,\n        /*.defrag_thold                =*/ -1.0f,\n        /*.cb_eval                     =*/ nullptr,\n        /*.cb_eval_user_data           =*/ nullptr,\n        /*.type_k                      =*/ GGML_TYPE_F16,\n        /*.type_v                      =*/ GGML_TYPE_F16,\n        /*.abort_callback              =*/ nullptr,\n        /*.abort_callback_data         =*/ nullptr,\n        /*.embeddings                  =*/ false,\n        /*.offload_kqv                 =*/ true,\n        /*.no_perf                     =*/ true,\n        /*.op_offload                  =*/ true,\n        /*.swa_full                    =*/ true,\n        /*.kv_unified                  =*/ false,\n        /*.sampler                     =*/ nullptr,\n        /*.n_sampler                   =*/ 0,\n    };\n\n    return result;\n}\n\nllama_context * llama_init_from_model(\n                 llama_model * model,\n        llama_context_params   params) {\n    if (!model) {\n        LLAMA_LOG_ERROR(\"%s: model cannot be NULL\\n\", __func__);\n        return nullptr;\n    }\n\n    if (params.n_batch == 0 && params.n_ubatch == 0) {\n        LLAMA_LOG_ERROR(\"%s: n_batch and n_ubatch cannot both be zero\\n\", __func__);\n        return nullptr;\n    }\n\n    if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {\n        LLAMA_LOG_ERROR(\"%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\\n\", __func__);\n        return nullptr;\n    }\n\n    if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && model->arch == LLM_ARCH_GROK) {\n        LLAMA_LOG_WARN(\"%s: flash_attn is not compatible with Grok - forcing off\\n\", __func__);\n        params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;\n    }\n\n    if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_k)) {\n        const uint32_t blck_size = ggml_blck_size(params.type_k);\n        for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {\n            if (model->hparams.n_embd_head_k(il) % blck_size != 0) {\n                LLAMA_LOG_ERROR(\"%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\\n\",\n                    __func__, ggml_type_name(params.type_k), blck_size, model->hparams.n_embd_head_k(il));\n                return nullptr;\n            }\n        }\n    }\n\n    if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_v)) {\n        const uint32_t blck_size = ggml_blck_size(params.type_v);\n        for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {\n            if (model->hparams.n_embd_head_v(il) % blck_size != 0) {\n                LLAMA_LOG_ERROR(\"%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\\n\",\n                    __func__, ggml_type_name(params.type_v), blck_size, model->hparams.n_embd_head_v(il));\n                return nullptr;\n            }\n        }\n    }\n\n    if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {\n        LLAMA_LOG_ERROR(\"%s: V cache quantization requires flash_attn\\n\", __func__);\n        return nullptr;\n    }\n\n    if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED &&\n        params.pooling_type != model->hparams.pooling_type) {\n        //user-specified pooling-type is different from the model default\n        LLAMA_LOG_WARN(\"%s: model default pooling_type is [%d], but [%d] was specified\\n\", __func__,\n                       model->hparams.pooling_type, params.pooling_type);\n    }\n\n    try {\n        auto * ctx = new llama_context(*model, params);\n        return ctx;\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: failed to initialize the context: %s\\n\", __func__, err.what());\n    }\n\n    return nullptr;\n}\n\n// deprecated\nllama_context * llama_new_context_with_model(\n                 llama_model * model,\n        llama_context_params   params) {\n    return llama_init_from_model(model, params);\n}\n\nvoid llama_free(llama_context * ctx) {\n    delete ctx;\n}\n\nuint32_t llama_n_ctx(const llama_context * ctx) {\n    return ctx->n_ctx();\n}\n\nuint32_t llama_n_ctx_seq(const llama_context * ctx) {\n    return ctx->n_ctx_seq();\n}\n\nuint32_t llama_n_batch(const llama_context * ctx) {\n    return ctx->n_batch();\n}\n\nuint32_t llama_n_ubatch(const llama_context * ctx) {\n    return ctx->n_ubatch();\n}\n\nuint32_t llama_n_seq_max(const llama_context * ctx) {\n    return ctx->n_seq_max();\n}\n\nconst llama_model * llama_get_model(const llama_context * ctx) {\n    return &ctx->get_model();\n}\n\nenum llama_pooling_type llama_pooling_type(const llama_context * ctx) {\n    return ctx->pooling_type();\n}\n\nvoid llama_attach_threadpool(\n            llama_context * ctx,\n        ggml_threadpool_t   threadpool,\n        ggml_threadpool_t   threadpool_batch) {\n    ctx->attach_threadpool(threadpool, threadpool_batch);\n}\n\nvoid llama_detach_threadpool(llama_context * ctx) {\n    ctx->detach_threadpool();\n}\n\nvoid llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) {\n    ctx->set_n_threads(n_threads, n_threads_batch);\n}\n\nint32_t llama_n_threads(llama_context * ctx) {\n    return ctx->n_threads();\n}\n\nint32_t llama_n_threads_batch(llama_context * ctx) {\n    return ctx->n_threads_batch();\n}\n\nvoid llama_set_abort_callback(llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) {\n    ctx->set_abort_callback(abort_callback, abort_callback_data);\n}\n\nvoid llama_set_embeddings(llama_context * ctx, bool embeddings) {\n    ctx->set_embeddings(embeddings);\n}\n\nvoid llama_set_causal_attn(llama_context * ctx, bool causal_attn) {\n    ctx->set_causal_attn(causal_attn);\n}\n\nvoid llama_set_warmup(llama_context * ctx, bool warmup) {\n    ctx->set_warmup(warmup);\n}\n\nvoid llama_synchronize(llama_context * ctx) {\n    ctx->synchronize();\n}\n\nfloat * llama_get_logits(llama_context * ctx) {\n    ctx->synchronize();\n\n    return ctx->get_logits();\n}\n\nfloat * llama_get_logits_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    float * res = nullptr;\n\n    res = ctx->get_sampled_logits_ith(i);\n\n    if (!res) {\n        res = ctx->get_logits_ith(i);\n    }\n\n    return res;\n}\n\nfloat * llama_get_embeddings(llama_context * ctx) {\n    ctx->synchronize();\n\n    return ctx->get_embeddings();\n}\n\nfloat * llama_get_embeddings_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return ctx->get_embeddings_ith(i);\n}\n\nfloat * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) {\n    ctx->synchronize();\n\n    return ctx->get_embeddings_seq(seq_id);\n}\n\nbool llama_set_sampler(llama_context * ctx, llama_seq_id seq_id, llama_sampler * smpl) {\n    return ctx->set_sampler(seq_id, smpl);\n}\n\nllama_token llama_get_sampled_token_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return ctx->get_sampled_token_ith(i);\n}\n\nfloat * llama_get_sampled_probs_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return ctx->get_sampled_probs_ith(i);\n}\n\nfloat * llama_get_sampled_logits_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return ctx->get_sampled_logits_ith(i);\n}\n\nllama_token * llama_get_sampled_candidates_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return const_cast<llama_token *>(ctx->get_sampled_candidates_ith(i));\n}\n\nuint32_t llama_get_sampled_candidates_count_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return static_cast<uint32_t>(ctx->get_sampled_candidates_count(i));\n}\n\nuint32_t llama_get_sampled_logits_count_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return static_cast<uint32_t>(ctx->get_sampled_logits_count(i));\n}\n\nuint32_t llama_get_sampled_probs_count_ith(llama_context * ctx, int32_t i) {\n    ctx->synchronize();\n\n    return static_cast<uint32_t>(ctx->get_sampled_probs_count(i));\n}\n\nstruct ggml_cgraph * llama_graph_reserve(\n        struct llama_context * ctx,\n        uint32_t n_tokens,\n        uint32_t n_seqs,\n        uint32_t n_outputs) {\n    auto * memory = ctx->get_memory();\n    llama_memory_context_ptr mctx;\n    if (memory) {\n        mctx = memory->init_full();\n    }\n    return ctx->graph_reserve(n_tokens, n_seqs, n_outputs, mctx.get());\n}\n\n// llama adapter API\n\nint32_t llama_set_adapters_lora(\n            llama_context * ctx,\n            llama_adapter_lora ** adapters,\n            size_t n_adapters,\n            float * scales) {\n    if (adapters == nullptr || scales == nullptr) {\n        GGML_ASSERT(n_adapters == 0 && \"invalid llama_set_adapters_lora call\");\n    }\n\n    ctx->set_adapters_lora(adapters, n_adapters, scales);\n\n    return 0;\n}\n\nint32_t llama_set_adapter_cvec(\n        llama_context * ctx,\n          const float * data,\n               size_t   len,\n              int32_t   n_embd,\n              int32_t   il_start,\n              int32_t   il_end) {\n    bool res = ctx->set_adapter_cvec(data, len, n_embd, il_start, il_end);\n\n    return res ? 0 : -1;\n}\n\n//\n// memory\n//\n\nllama_memory_t llama_get_memory(const struct llama_context * ctx) {\n    return ctx->get_memory();\n}\n\nvoid llama_memory_clear(llama_memory_t mem, bool data) {\n    if (!mem) {\n        return;\n    }\n\n    mem->clear(data);\n}\n\nbool llama_memory_seq_rm(\n        llama_memory_t mem,\n          llama_seq_id seq_id,\n             llama_pos p0,\n             llama_pos p1) {\n    if (!mem) {\n        return true;\n    }\n\n    return mem->seq_rm(seq_id, p0, p1);\n}\n\nvoid llama_memory_seq_cp(\n        llama_memory_t mem,\n          llama_seq_id seq_id_src,\n          llama_seq_id seq_id_dst,\n             llama_pos p0,\n             llama_pos p1) {\n    if (!mem) {\n        return;\n    }\n\n    mem->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n}\n\nvoid llama_memory_seq_keep(\n        llama_memory_t mem,\n          llama_seq_id seq_id) {\n    if (!mem) {\n        return;\n    }\n\n    mem->seq_keep(seq_id);\n}\n\nvoid llama_memory_seq_add(\n        llama_memory_t mem,\n          llama_seq_id seq_id,\n             llama_pos p0,\n             llama_pos p1,\n             llama_pos delta) {\n    if (!mem) {\n        return;\n    }\n\n    mem->seq_add(seq_id, p0, p1, delta);\n}\n\nvoid llama_memory_seq_div(\n        llama_memory_t mem,\n          llama_seq_id seq_id,\n             llama_pos p0,\n             llama_pos p1,\n                   int d) {\n    if (!mem) {\n        return;\n    }\n\n    mem->seq_div(seq_id, p0, p1, d);\n}\n\nllama_pos llama_memory_seq_pos_min(\n        llama_memory_t mem,\n          llama_seq_id seq_id) {\n    if (!mem) {\n        return -1;\n    }\n\n    return mem->seq_pos_min(seq_id);\n}\n\nllama_pos llama_memory_seq_pos_max(\n        llama_memory_t mem,\n          llama_seq_id seq_id) {\n    if (!mem) {\n        return -1;\n    }\n\n    return mem->seq_pos_max(seq_id);\n}\n\nbool llama_memory_can_shift(llama_memory_t mem) {\n    if (!mem) {\n        return false;\n    }\n\n    return mem->get_can_shift();\n}\n\n// llama state API\n\n// deprecated\nsize_t llama_get_state_size(llama_context * ctx) {\n    return llama_state_get_size(ctx);\n}\n\n// deprecated\nsize_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) {\n    return llama_state_get_data(ctx, dst, -1);\n}\n\n// deprecated\nsize_t llama_set_state_data(llama_context * ctx, const uint8_t * src) {\n    return llama_state_set_data(ctx, src, -1);\n}\n\n// deprecated\nbool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {\n    return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);\n}\n\n// deprecated\nbool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {\n    return llama_state_save_file(ctx, path_session, tokens, n_token_count);\n}\n\n// Returns the *actual* size of the state.\n// Intended to be used when saving to state to a buffer.\nsize_t llama_state_get_size(llama_context * ctx) {\n    return ctx->state_get_size();\n}\n\nsize_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) {\n    ctx->synchronize();\n\n    return ctx->state_get_data(dst, size);\n}\n\n// Sets the state reading from the specified source address\nsize_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) {\n    ctx->synchronize();\n\n    return ctx->state_set_data(src, size);\n}\n\nbool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {\n    ctx->synchronize();\n\n    try {\n        return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error loading session file: %s\\n\", __func__, err.what());\n        return false;\n    }\n}\n\nbool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {\n    ctx->synchronize();\n\n    try {\n        return ctx->state_save_file(path_session, tokens, n_token_count);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error saving session file: %s\\n\", __func__, err.what());\n        return false;\n    }\n}\n\nsize_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) {\n    return llama_state_seq_get_size_ext(ctx, seq_id, 0);\n}\n\nsize_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) {\n    return llama_state_seq_get_data_ext(ctx, dst, size, seq_id, 0);\n}\n\nsize_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) {\n    return llama_state_seq_set_data_ext(ctx, src, size, seq_id, 0);\n}\n\nsize_t llama_state_seq_get_size_ext(llama_context * ctx, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    return ctx->state_seq_get_size(seq_id, flags);\n}\n\nsize_t llama_state_seq_get_data_ext(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    ctx->synchronize();\n\n    return ctx->state_seq_get_data(seq_id, dst, size, flags);\n}\n\nsize_t llama_state_seq_set_data_ext(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    ctx->synchronize();\n\n    return ctx->state_seq_set_data(seq_id, src, size, flags);\n}\n\nsize_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {\n    ctx->synchronize();\n\n    try {\n        return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error saving sequence state file: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\nsize_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {\n    ctx->synchronize();\n\n    try {\n        return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error loading sequence state file: %s\\n\", __func__, err.what());\n        return 0;\n    }\n}\n\n///\n\nint32_t llama_encode(\n        llama_context * ctx,\n          llama_batch   batch) {\n    const int ret = ctx->encode(batch);\n    if (ret != 0) {\n        LLAMA_LOG_ERROR(\"%s: failed to encode, ret = %d\\n\", __func__, ret);\n    }\n\n    return ret;\n}\n\nint32_t llama_decode(\n        llama_context * ctx,\n          llama_batch   batch) {\n    const int ret = ctx->decode(batch);\n    if (ret != 0 && ret != 1) {\n        LLAMA_LOG_ERROR(\"%s: failed to decode, ret = %d\\n\", __func__, ret);\n    }\n\n    return ret;\n}\n\n//\n// perf\n//\n\nllama_perf_context_data llama_perf_context(const llama_context * ctx) {\n    llama_perf_context_data data = {};\n\n    if (ctx == nullptr) {\n        return data;\n    }\n\n    data = ctx->perf_get_data();\n\n    return data;\n}\n\nvoid llama_perf_context_print(const llama_context * ctx) {\n    const auto data = llama_perf_context(ctx);\n\n    const double t_end_ms = 1e-3 * ggml_time_us();\n\n    LLAMA_LOG_INFO(\"%s:        load time = %10.2f ms\\n\", __func__, data.t_load_ms);\n    LLAMA_LOG_INFO(\"%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\\n\",\n            __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);\n    LLAMA_LOG_INFO(\"%s:        eval time = %10.2f ms / %5d runs   (%8.2f ms per token, %8.2f tokens per second)\\n\",\n            __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);\n    LLAMA_LOG_INFO(\"%s:       total time = %10.2f ms / %5d tokens\\n\", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));\n    LLAMA_LOG_INFO(\"%s:    graphs reused = %10d\\n\", __func__, data.n_reused);\n}\n\nvoid llama_perf_context_reset(llama_context * ctx) {\n    ctx->perf_reset();\n}\n\nvoid llama_memory_breakdown_print(const struct llama_context * ctx) {\n    const std::vector<ggml_backend_dev_t> & devices = ctx->get_model().devices;\n\n    std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown = ctx->memory_breakdown();\n\n    std::vector<std::array<std::string, 9>> table_data;\n    table_data.reserve(devices.size());\n    const std::string template_header = \"%s: | %s | %s   %s    %s   %s   %s   %s    %s |\\n\";\n    const std::string template_gpu    = \"%s: | %s | %s = %s + (%s = %s + %s + %s) + %s |\\n\";\n    const std::string template_other  = \"%s: | %s | %s   %s    %s = %s + %s + %s    %s |\\n\";\n\n    table_data.push_back({template_header, \"memory breakdown [MiB]\", \"total\", \"free\", \"self\", \"model\", \"context\", \"compute\", \"unaccounted\"});\n\n    constexpr size_t MiB = 1024 * 1024;\n    const std::vector<std::string> desc_prefixes_strip = {\"NVIDIA \", \"GeForce \", \"Tesla \", \"AMD \", \"Radeon \", \"Instinct \"};\n\n    // track seen buffer types to avoid double counting:\n    std::set<ggml_backend_buffer_type_t> seen_buffer_types;\n\n    // accumulative memory breakdown for each device and for host:\n    std::vector<llama_memory_breakdown_data> mb_dev(devices.size());\n    llama_memory_breakdown_data              mb_host;\n\n    for (const auto & buft_mb : memory_breakdown) {\n        ggml_backend_buffer_type_t          buft = buft_mb.first;\n        const llama_memory_breakdown_data & mb   = buft_mb.second;\n        if (ggml_backend_buft_is_host(buft)) {\n            mb_host.model   += mb.model;\n            mb_host.context += mb.context;\n            mb_host.compute += mb.compute;\n            seen_buffer_types.insert(buft);\n            continue;\n        }\n        ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);\n        if (dev) {\n            int i_dev = -1;\n            for (size_t i = 0; i < devices.size(); i++) {\n                if (devices[i] == dev) {\n                    i_dev = i;\n                    break;\n                }\n            }\n            if (i_dev != -1) {\n                mb_dev[i_dev].model   += mb.model;\n                mb_dev[i_dev].context += mb.context;\n                mb_dev[i_dev].compute += mb.compute;\n                seen_buffer_types.insert(buft);\n                continue;\n            }\n        }\n    }\n\n    // print memory breakdown for each device:\n    for (size_t i = 0; i < devices.size(); i++) {\n        ggml_backend_dev_t          dev = devices[i];\n        llama_memory_breakdown_data mb  = mb_dev[i];\n\n        const std::string name = ggml_backend_dev_name(dev);\n        std::string desc = ggml_backend_dev_description(dev);\n        for (const std::string & prefix : desc_prefixes_strip) {\n            if (desc.length() >= prefix.length() && desc.substr(0, prefix.length()) == prefix) {\n                desc = desc.substr(prefix.length());\n            }\n        }\n\n        size_t free, total;\n        ggml_backend_dev_memory(dev, &free, &total);\n\n        const size_t self = mb.model + mb.context + mb.compute;\n        const size_t unaccounted = total - self - free;\n\n        table_data.push_back({\n            template_gpu,\n            \"  - \" + name + \" (\" + desc + \")\",\n            std::to_string(total / MiB),\n            std::to_string(free / MiB),\n            std::to_string(self / MiB),\n            std::to_string(mb.model / MiB),\n            std::to_string(mb.context / MiB),\n            std::to_string(mb.compute / MiB),\n            std::to_string(unaccounted / MiB)});\n    }\n\n    // print memory breakdown for host:\n    {\n        const size_t self = mb_host.model + mb_host.context + mb_host.compute;\n        table_data.push_back({\n            template_other,\n            \"  - Host\",\n            \"\", // total\n            \"\", // free\n            std::to_string(self / MiB),\n            std::to_string(mb_host.model / MiB),\n            std::to_string(mb_host.context / MiB),\n            std::to_string(mb_host.compute / MiB),\n            \"\"}); // unaccounted\n    }\n\n    // print memory breakdown for all remaining buffer types:\n    for (const auto & buft_mb : memory_breakdown) {\n        ggml_backend_buffer_type_t          buft = buft_mb.first;\n        const llama_memory_breakdown_data & mb   = buft_mb.second;\n        if (seen_buffer_types.count(buft) == 1) {\n            continue;\n        }\n        const std::string name = ggml_backend_buft_name(buft);\n        const size_t self = mb.model + mb.context + mb.compute;\n        table_data.push_back({\n            template_other,\n            \"  - \" + name,\n            \"\", // total\n            \"\", // free\n            std::to_string(self / MiB),\n            std::to_string(mb.model / MiB),\n            std::to_string(mb.context / MiB),\n            std::to_string(mb.compute / MiB),\n            \"\"}); // unaccounted\n        seen_buffer_types.insert(buft);\n    }\n\n    for (size_t j = 1; j < table_data[0].size(); j++) {\n        size_t max_len = 0;\n        for (const auto & td : table_data) {\n            max_len = std::max(max_len, td[j].length());\n        }\n        for (auto & td : table_data) {\n            td[j].insert(j == 1 ? td[j].length() : 0, max_len - td[j].length(), ' ');\n        }\n    }\n    for (const auto & td : table_data) {\n        LLAMA_LOG_INFO(td[0].c_str(),\n            __func__, td[1].c_str(), td[2].c_str(), td[3].c_str(), td[4].c_str(), td[5].c_str(),\n            td[6].c_str(), td[7].c_str(), td[8].c_str());\n    }\n}\n\n//\n// training\n//\n\nbool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata) {\n    GGML_UNUSED(tensor);\n    GGML_UNUSED(userdata);\n    return true;\n}\n\nvoid llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {\n    ctx->opt_init(model, lopt_params);\n}\n\nvoid llama_opt_epoch(\n        struct llama_context    * ctx,\n        ggml_opt_dataset_t        dataset,\n        ggml_opt_result_t         result_train,\n        ggml_opt_result_t         result_eval,\n        int64_t                   idata_split,\n        ggml_opt_epoch_callback   callback_train,\n        ggml_opt_epoch_callback   callback_eval) {\n    ctx->opt_epoch(\n        dataset,\n        result_train,\n        result_eval,\n        idata_split,\n        callback_train,\n        callback_eval);\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-context.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n#include \"llama-cparams.h\"\n#include \"llama-graph.h\"\n#include \"llama-adapter.h\"\n#include \"llama-impl.h\"\n\n#include \"ggml-cpp.h\"\n#include \"ggml-opt.h\"\n\n#include <map>\n#include <vector>\n\nstruct llama_model;\nclass llama_batch_allocr;\n\nclass llama_io_read_i;\nclass llama_io_write_i;\n\n// \"memory\" as in abstract memory for the context\nstruct llama_memory_i;\nstruct llama_memory_context_i;\n\n// \"memory\" as in physical memory for a buffer type, in bytes\nstruct llama_memory_breakdown_data {\n    size_t model   = 0; // memory allocated for the model\n    size_t context = 0; // memory allocated for the context\n    size_t compute = 0; // memory allocated for temporary compute buffers\n\n    size_t total() const {\n        return model + context + compute;\n    }\n};\n\nstruct llama_context {\n    // init scheduler and compute buffers, reserve worst-case graphs\n    llama_context(\n            const llama_model & model,\n                  llama_context_params params);\n\n    ~llama_context();\n\n    // reserve a new backend scheduler (if needed)\n    // for example, when:\n    //   - changing loras\n    //   - changing samplers\n    //   - changing attention type\n    //   - etc.\n    void sched_reserve();\n\n    void synchronize();\n\n    const llama_model   & get_model()   const;\n    const llama_cparams & get_cparams() const;\n\n    ggml_backend_sched_t get_sched() const;\n\n    uint32_t n_ctx()     const;\n    uint32_t n_ctx_seq() const;\n    uint32_t n_batch()   const;\n    uint32_t n_ubatch()  const;\n    uint32_t n_seq_max() const;\n\n    uint32_t n_threads()       const;\n    uint32_t n_threads_batch() const;\n\n    llama_memory_t get_memory() const;\n\n    // return true if the memory was updated\n    bool memory_update(bool optimize);\n\n    enum llama_pooling_type pooling_type() const;\n\n    float * get_logits();\n    float * get_logits_ith(int32_t i);\n\n    float * get_embeddings();\n    float * get_embeddings_ith(int32_t i);\n    float * get_embeddings_seq(llama_seq_id seq_id);\n\n    llama_token * get_sampled_tokens() const;\n    llama_token   get_sampled_token_ith(int32_t idx);\n\n    float * get_sampled_logits_ith(int32_t idx);\n    size_t  get_sampled_logits_count(int32_t idx);\n\n    float * get_sampled_probs_ith(int32_t idx);\n    size_t  get_sampled_probs_count(int32_t idx);\n\n    const llama_token * get_sampled_candidates_ith(int32_t idx);\n    size_t get_sampled_candidates_count(int32_t idx);\n\n    void attach_threadpool(\n            ggml_threadpool_t threadpool,\n            ggml_threadpool_t threadpool_batch);\n\n    void detach_threadpool();\n\n    void set_n_threads(int32_t n_threads, int32_t n_threads_batch);\n\n    void set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data);\n\n    void set_embeddings (bool value);\n    void set_causal_attn(bool value);\n    void set_warmup(bool value);\n\n    void set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales);\n\n    bool adapters_lora_are_same(llama_adapter_lora ** adapters, size_t n_adapters, float * scales);\n\n    bool set_adapter_cvec(\n            const float * data,\n                 size_t   len,\n                int32_t   n_embd,\n                int32_t   il_start,\n                int32_t   il_end);\n\n    // process a single ubatch with a specific graph type\n    // if memory_context is provided, it will be applied first to the context's memory\n    // ret contains the status of the graph computation\n    // returns nullptr only if ret != GGML_STATUS_SUCCESS\n    llm_graph_result * process_ubatch(\n                const llama_ubatch & ubatch,\n                    llm_graph_type   gtype,\n            llama_memory_context_i * mctx,\n                       ggml_status & ret);\n\n    int encode(const llama_batch & batch_inp);\n    int decode(const llama_batch & batch_inp);\n\n    //\n    // state save/load\n    //\n\n    size_t state_get_size();\n    size_t state_get_data(      uint8_t * dst, size_t size);\n    size_t state_set_data(const uint8_t * src, size_t size);\n\n    size_t state_seq_get_size(llama_seq_id seq_id, llama_state_seq_flags flags);\n    size_t state_seq_get_data(llama_seq_id seq_id,       uint8_t * dst, size_t size, llama_state_seq_flags flags);\n    size_t state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size, llama_state_seq_flags flags);\n\n    bool state_load_file(\n            const char * filepath,\n           llama_token * tokens_out,\n                size_t   n_token_capacity,\n                size_t * n_token_count_out);\n\n    bool state_save_file(\n            const char * filepath,\n     const llama_token * tokens,\n                size_t   n_token_count);\n\n    size_t state_seq_load_file(\n          llama_seq_id   seq_id,\n            const char * filepath,\n           llama_token * tokens_out,\n                size_t   n_token_capacity,\n                size_t * n_token_count_out);\n\n    size_t state_seq_save_file(\n          llama_seq_id   seq_id,\n            const char * filepath,\n     const llama_token * tokens,\n                size_t   n_token_count);\n\n    //\n    // perf\n    //\n\n    llama_perf_context_data perf_get_data() const;\n    void perf_reset();\n\n    std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown() const;\n\n    //\n    // training\n    //\n\n    void opt_init(struct llama_model * model, struct llama_opt_params lopt_params);\n\n    // TODO: more flexible combinations of logical/physical batch size and context size\n    void opt_epoch(\n            ggml_opt_dataset_t      dataset,\n            ggml_opt_result_t       result_train,\n            ggml_opt_result_t       result_eval,\n            int64_t                 idata_split,\n            ggml_opt_epoch_callback callback_train,\n            ggml_opt_epoch_callback callback_eval);\n\n    void opt_epoch_iter(\n            ggml_opt_dataset_t               dataset,\n            ggml_opt_result_t                result,\n            const std::vector<llama_token> & tokens,\n            const std::vector<llama_token> & labels_sparse,\n            llama_batch                    & batch,\n            ggml_opt_epoch_callback          callback,\n            bool                             train,\n            int64_t                          idata_in_loop,\n            int64_t                          ndata_in_loop,\n            int64_t                          t_loop_start);\n\nprivate:\n    //\n    // output\n    //\n\n    // Make sure enough space is available for outputs.\n    // Returns max number of outputs for which space was reserved.\n    uint32_t output_reserve(int32_t n_outputs);\n\n    void output_reorder();\n\n    // map the output row index `i` to batch index\n    int64_t output_resolve_row(int32_t i) const;\n\n    //\n    // graph\n    //\n\npublic:\n    uint32_t graph_max_nodes(uint32_t n_tokens) const;\n\n    // can reuse the llm_graph_result instance of the context (for example to update a memory module)\n    llm_graph_result * get_gf_res_reserve() const;\n\n    // returns the result of ggml_backend_sched_graph_compute_async execution\n    ggml_status graph_compute(ggml_cgraph * gf, bool batched);\n\n    // reserve a graph with a dummy ubatch of the specified size\n    ggml_cgraph * graph_reserve(\n        uint32_t n_tokens, uint32_t n_seqs, uint32_t n_outputs, const llama_memory_context_i * mctx, bool split_only = false, size_t * sizes = nullptr);\n\n    bool set_sampler(llama_seq_id seq_id, llama_sampler * sampler);\n\nprivate:\n    llm_graph_params graph_params(\n                        llm_graph_result * res,\n                      const llama_ubatch & ubatch,\n            const llama_memory_context_i * mctx,\n                          llm_graph_type   gtype) const;\n\n    llm_graph_cb graph_get_cb() const;\n\n    // TODO: read/write lora adapters and cvec\n    size_t state_write_data(llama_io_write_i & io);\n    size_t state_read_data (llama_io_read_i  & io);\n\n    size_t state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags);\n    size_t state_seq_read_data (llama_io_read_i  & io, llama_seq_id seq_id, llama_state_seq_flags flags);\n\n    //\n    // members\n    //\n\n    const llama_model & model;\n\n    llama_cparams cparams;\n\n    llama_adapter_cvec_ptr  cvec;\n    llama_adapter_loras_ptr loras;\n\n    llama_cross cross; // TODO: tmp for handling cross-attention - need something better probably\n\n    std::unique_ptr<llama_memory_i> memory;\n\n    // decode output (2-dimensional array: [n_outputs][n_vocab])\n    buffer_view<float> logits = {nullptr, 0};\n\n    // embeddings output (2-dimensional array: [n_outputs][n_embd])\n    // populated only when pooling_type == LLAMA_POOLING_TYPE_NONE\n    buffer_view<float> embd = {nullptr, 0};\n\n    struct sampling_info {\n        // !samplers.empty() to check if any samplers are active\n        std::map<llama_seq_id, llama_sampler *> samplers;\n\n        buffer_view<float>       logits     = {nullptr, 0};\n        buffer_view<llama_token> sampled    = {nullptr, 0};\n        buffer_view<float>       probs      = {nullptr, 0};\n        buffer_view<llama_token> candidates = {nullptr, 0};\n\n        std::vector<uint32_t> logits_count;\n        std::vector<uint32_t> probs_count;\n        std::vector<uint32_t> candidates_count;\n\n        // optimization\n        std::vector<llama_token> token_ids_full_vocab;\n    };\n\n    sampling_info sampling;\n\n    // sequence embeddings output (map of [n_embd] vectors)\n    // populated only when pooling_type != LLAMA_POOLING_TYPE_NONE\n    std::map<llama_seq_id, std::vector<float>> embd_seq;\n\n    // reuse the batch_allocr to avoid unnecessary memory allocations\n    std::unique_ptr<llama_batch_allocr> balloc;\n\n    uint32_t n_outputs = 0; // number of actually-used outputs in the current ubatch or last logical batch\n\n    std::vector<int32_t> output_ids; // map batch token positions to ids of the logits and embd buffers\n\n    struct swap_info {\n        uint32_t i0;\n        uint32_t i1;\n    };\n\n    std::vector<swap_info> output_swaps;\n\n    ggml_backend_sched_ptr sched;\n\n    bool sched_need_reserve = true;\n\n    ggml_backend_t backend_cpu = nullptr;\n    std::vector<ggml_backend_ptr> backends;\n\n    // training\n    ggml_opt_context_t opt_ctx = nullptr;\n\n    ggml_threadpool_t threadpool       = nullptr;\n    ggml_threadpool_t threadpool_batch = nullptr;\n\n    ggml_abort_callback abort_callback      = nullptr;\n    void *              abort_callback_data = nullptr;\n\n    std::vector<std::pair<ggml_backend_t, ggml_backend_set_n_threads_t>> set_n_threads_fns;\n\n    // pointers and buffer types used for the compute buffer of each backend\n    std::vector<ggml_backend_t>             backend_ptrs;\n    std::vector<ggml_backend_buffer_type_t> backend_buft;\n    std::vector<size_t>                     backend_buf_exp_size; // expected buffer sizes\n\n    llm_graph_result_ptr gf_res_prev;\n    llm_graph_result_ptr gf_res_reserve;\n\n    // host buffer for the model output (logits and embeddings)\n    ggml_backend_buffer_ptr buf_output;\n\n    bool has_evaluated_once = false;\n\n    // env: LLAMA_GRAPH_REUSE_DISABLE\n    bool graph_reuse_disable = false;\n\n    // perf\n    mutable int64_t t_start_us  = 0;\n    mutable int64_t t_load_us   = 0;\n    mutable int64_t t_p_eval_us = 0;\n    mutable int64_t t_eval_us   = 0;\n\n    mutable int64_t t_compute_start_us = 0;\n    mutable int64_t n_queued_tokens    = 0;\n\n    mutable int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1)\n    mutable int32_t n_eval   = 0; // number of eval calls\n\n    mutable int32_t n_reused = 0; // number of times the previous graph was reused\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-cparams.cpp",
    "content": "#include \"llama-cparams.h\"\n\nsize_t llama_max_parallel_sequences(void) {\n    return LLAMA_MAX_SEQ;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-cparams.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <cstdint>\n\n#define LLAMA_MAX_SEQ 256\n\nstruct llama_cparams {\n    uint32_t n_ctx;           // context size used during inference\n    uint32_t n_ctx_seq;       // context for a single sequence\n    uint32_t n_batch;\n    uint32_t n_ubatch;\n    uint32_t n_seq_max;\n    int32_t  n_threads;       // number of threads to use for generation\n    int32_t  n_threads_batch; // number of threads to use for batch processing\n\n    float rope_freq_base;\n    float rope_freq_scale;\n\n    uint32_t n_ctx_orig_yarn;\n    // These hyperparameters are not exposed in GGUF, because all\n    // existing YaRN models use the same values for them.\n    float yarn_ext_factor;\n    float yarn_attn_factor;\n    float yarn_beta_fast;\n    float yarn_beta_slow;\n\n    bool embeddings;\n    bool causal_attn;\n    bool offload_kqv;\n    bool flash_attn;\n    bool auto_fa;\n    bool fused_gdn_ar;       // use fused gated delta net (autoregressive)\n    bool fused_gdn_ch;       // use fused gated delta net (chunked)\n    bool auto_fgdn;\n    bool no_perf;\n    bool warmup;\n    bool op_offload;\n    bool kv_unified;\n    bool pipeline_parallel;\n\n    enum llama_pooling_type pooling_type;\n\n    ggml_backend_sched_eval_callback cb_eval;\n    void * cb_eval_user_data;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-ext.h",
    "content": "#pragma once\n\n#include \"llama-context.h\"\n#include \"ggml.h\"\n#include \"stdint.h\"\n\n// Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.\nLLAMA_API struct ggml_cgraph * llama_graph_reserve(\n        struct llama_context * ctx,\n        uint32_t n_tokens,\n        uint32_t n_seqs,\n        uint32_t n_outputs);\n"
  },
  {
    "path": "examples/talk-llama/llama-grammar.cpp",
    "content": "#include \"llama-grammar.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-vocab.h\"\n#include \"llama-sampler.h\"\n\n#include <cmath>\n#include <algorithm>\n#include <cstdint>\n#include <stdexcept>\n\n#define MAX_REPETITION_THRESHOLD 2000\n//\n// helpers\n//\n\n// NOTE: assumes valid utf8 (but checks for overrun)\nstatic std::pair<uint32_t, const char *> decode_utf8(const char * src) {\n    static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };\n    uint8_t  first_byte = static_cast<uint8_t>(*src);\n    uint8_t  highbits   = first_byte >> 4;\n    int      len        = lookup[highbits];\n    uint8_t  mask       = (1 << (8 - len)) - 1;\n    uint32_t value      = first_byte & mask;\n    const char * end    = src + len; // may overrun!\n    const char * pos    = src + 1;\n    for ( ; pos < end && *pos; pos++) {\n        value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);\n    }\n    return std::make_pair(value, pos);\n}\n\nstatic std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(\n        const std::string & src,\n        llama_partial_utf8 partial_start) {\n    static const int      lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };\n    const char          * pos      = src.c_str();\n    std::vector<uint32_t> code_points;\n\n    // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0.\n    code_points.reserve(src.size() + 1);\n    uint32_t value    = partial_start.value;\n    int      n_remain = partial_start.n_remain;\n\n    // continue previous decode, if applicable\n    while (*pos != 0 && n_remain > 0) {\n        uint8_t next_byte = static_cast<uint8_t>(*pos);\n        if ((next_byte >> 6) != 2) {\n            // invalid sequence, abort\n            code_points.push_back(0);\n            return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, -1 });\n        }\n        value = (value << 6) + (next_byte & 0x3F);\n        ++pos;\n        --n_remain;\n    }\n\n    if (partial_start.n_remain > 0 && n_remain == 0) {\n        code_points.push_back(value);\n    }\n\n    // decode any subsequent utf-8 sequences, which may end in an incomplete one\n    while (*pos != 0) {\n        uint8_t first_byte = static_cast<uint8_t>(*pos);\n        uint8_t highbits   = first_byte >> 4;\n        n_remain   = lookup[highbits] - 1;\n\n        if (n_remain < 0) {\n            // invalid sequence, abort\n            code_points.clear();\n            code_points.push_back(0);\n            return std::make_pair(std::move(code_points), llama_partial_utf8{ 0, n_remain });\n        }\n\n        uint8_t mask  = (1 << (7 - n_remain)) - 1;\n        value = first_byte & mask;\n\n        ++pos;\n        while (*pos != 0 && n_remain > 0) {\n            value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);\n            ++pos;\n            --n_remain;\n        }\n        if (n_remain == 0) {\n            code_points.push_back(value);\n        }\n    }\n    code_points.push_back(0);\n\n    return std::make_pair(std::move(code_points), llama_partial_utf8{ value, n_remain });\n}\n\nstatic bool is_digit_char(char c) {\n    return '0' <= c && c <= '9';\n}\n\nstatic bool is_word_char(char c) {\n    return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c);\n}\n\nstatic std::pair<uint32_t, const char *> parse_hex(const char * src, int size) {\n    const char * pos   = src;\n    const char * end   = src + size;\n    uint32_t     value = 0;\n    for ( ; pos < end && *pos; pos++) {\n        value <<= 4;\n        char c = *pos;\n        if ('a' <= c && c <= 'f') {\n            value += c - 'a' + 10;\n        } else if ('A' <= c && c <= 'F') {\n            value += c - 'A' + 10;\n        } else if ('0' <= c && c <= '9') {\n            value += c - '0';\n        } else {\n            break;\n        }\n    }\n    if (pos != end) {\n        throw std::runtime_error(\"expecting \" + std::to_string(size) + \" hex chars at \" + src);\n    }\n    return std::make_pair(value, pos);\n}\n\nstatic const char * parse_space(const char * src, bool newline_ok) {\n    const char * pos = src;\n    while (*pos == ' ' || *pos == '\\t' || *pos == '#' ||\n            (newline_ok && (*pos == '\\r' || *pos == '\\n'))) {\n        if (*pos == '#') {\n            while (*pos && *pos != '\\r' && *pos != '\\n') {\n                pos++;\n            }\n        } else {\n            pos++;\n        }\n    }\n    return pos;\n}\n\nstatic const char * parse_name(const char * src) {\n    const char * pos = src;\n    while (is_word_char(*pos)) {\n        pos++;\n    }\n    if (pos == src) {\n        throw std::runtime_error(std::string(\"expecting name at \") + src);\n    }\n    return pos;\n}\n\nstatic const char * parse_int(const char * src) {\n    const char * pos = src;\n    while (is_digit_char(*pos)) {\n        pos++;\n    }\n    if (pos == src) {\n        throw std::runtime_error(std::string(\"expecting integer at \") + src);\n    }\n    return pos;\n}\n\nstatic std::pair<uint32_t, const char *> parse_char(const char * src) {\n    if (*src == '\\\\') {\n        switch (src[1]) {\n            case 'x': return parse_hex(src + 2, 2);\n            case 'u': return parse_hex(src + 2, 4);\n            case 'U': return parse_hex(src + 2, 8);\n            case 't': return std::make_pair('\\t', src + 2);\n            case 'r': return std::make_pair('\\r', src + 2);\n            case 'n': return std::make_pair('\\n', src + 2);\n            case '\\\\':\n            case '\"':\n            case '[':\n            case ']':\n                      return std::make_pair(src[1], src + 2);\n            default:\n                      throw std::runtime_error(std::string(\"unknown escape at \") + src);\n        }\n    } else if (*src) {\n        return decode_utf8(src);\n    }\n    throw std::runtime_error(\"unexpected end of input\");\n}\n\nstatic std::pair<uint32_t, const char *> parse_token(const llama_vocab * vocab, const char * src) {\n    const char * pos = src;\n    if (*pos != '<') {\n        throw std::runtime_error(std::string(\"expecting '<' at \") + pos);\n    }\n    pos++;\n\n    // Parse <[id]>\n    if (*pos == '[') {\n        pos++;\n        const char * int_end = parse_int(pos);\n        uint32_t token_id = std::stoul(std::string(pos, int_end - pos));\n        pos = int_end;\n        if (*pos != ']') {\n            throw std::runtime_error(std::string(\"expecting ']' at \") + pos);\n        }\n        pos++;\n        if (*pos != '>') {\n            throw std::runtime_error(std::string(\"expecting '>' at \") + pos);\n        }\n        pos++;\n        return std::make_pair(token_id, pos);\n    }\n\n    if (vocab == nullptr) {\n        throw std::runtime_error(std::string(\"no vocab to parse token at \") + src);\n    }\n\n    // Parse <token> and tokenize to obtain the token id\n    while (*pos != 0 && *pos != '>') {\n        pos++;\n    }\n    if (*pos != '>') {\n        throw std::runtime_error(std::string(\"expecting '>' at \") + pos);\n    }\n    pos++;\n\n    llama_token tokens[2];\n    int32_t n_tokens = vocab->tokenize(src, static_cast<int32_t>(pos - src), tokens, 2, false, true);\n    if (n_tokens != 1) {\n        // must tokenize to exactly 1 token\n        throw std::runtime_error(\"invalid token '\" + std::string(src, pos - src) + \"'\");\n    }\n    return std::make_pair(tokens[0], pos);\n}\n\nstatic void print_grammar_char(FILE * file, uint32_t c) {\n    if (0x20 <= c && c <= 0x7f) {\n        fprintf(file, \"%c\", static_cast<char>(c));\n    } else {\n        // cop out of encoding UTF-8\n        fprintf(file, \"<U+%04X>\", c);\n    }\n}\n\nstatic bool is_char_element(llama_grammar_element elem) {\n    switch (elem.type) {\n        case LLAMA_GRETYPE_CHAR:           return true;\n        case LLAMA_GRETYPE_CHAR_NOT:       return true;\n        case LLAMA_GRETYPE_CHAR_ALT:       return true;\n        case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true;\n        case LLAMA_GRETYPE_CHAR_ANY:       return true;\n        default:                           return false;\n    }\n}\n\nstatic void print_rule_binary(FILE * file, const llama_grammar_rule & rule) {\n    for (auto elem : rule) {\n        switch (elem.type) {\n            case LLAMA_GRETYPE_END:            fprintf(file, \"END\");            break;\n            case LLAMA_GRETYPE_ALT:            fprintf(file, \"ALT\");            break;\n            case LLAMA_GRETYPE_RULE_REF:       fprintf(file, \"RULE_REF\");       break;\n            case LLAMA_GRETYPE_CHAR:           fprintf(file, \"CHAR\");           break;\n            case LLAMA_GRETYPE_CHAR_NOT:       fprintf(file, \"CHAR_NOT\");       break;\n            case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, \"CHAR_RNG_UPPER\"); break;\n            case LLAMA_GRETYPE_CHAR_ALT:       fprintf(file, \"CHAR_ALT\");       break;\n            case LLAMA_GRETYPE_CHAR_ANY:       fprintf(file, \"CHAR_ANY\");       break;\n            case LLAMA_GRETYPE_TOKEN:          fprintf(file, \"TOKEN\");          break;\n            case LLAMA_GRETYPE_TOKEN_NOT:      fprintf(file, \"TOKEN_NOT\");      break;\n        }\n        switch (elem.type) {\n            case LLAMA_GRETYPE_END:\n            case LLAMA_GRETYPE_ALT:\n            case LLAMA_GRETYPE_RULE_REF:\n                fprintf(file, \"(%u) \", elem.value);\n                break;\n            case LLAMA_GRETYPE_CHAR:\n            case LLAMA_GRETYPE_CHAR_NOT:\n            case LLAMA_GRETYPE_CHAR_RNG_UPPER:\n            case LLAMA_GRETYPE_CHAR_ALT:\n            case LLAMA_GRETYPE_CHAR_ANY:\n                fprintf(file, \"(\\\"\");\n                print_grammar_char(file, elem.value);\n                fprintf(file, \"\\\") \");\n                break;\n            case LLAMA_GRETYPE_TOKEN:\n                fprintf(file, \"<[\");\n                fprintf(file, \"%u\", elem.value);\n                fprintf(file, \"]> \");\n                break;\n            case LLAMA_GRETYPE_TOKEN_NOT:\n                fprintf(file, \"!\");\n                fprintf(file, \"<[\");\n                fprintf(file, \"%u\", elem.value);\n                fprintf(file, \"]> \");\n                break;\n        }\n    }\n    fprintf(file, \"\\n\");\n}\n\nstatic void print_rule(\n        FILE     * file,\n        uint32_t   rule_id,\n        const llama_grammar_rule & rule,\n        const std::map<uint32_t, std::string> & symbol_id_names) {\n    if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) {\n        throw std::runtime_error(\n            \"malformed rule, does not end with LLAMA_GRETYPE_END: \" + std::to_string(rule_id));\n    }\n    fprintf(file, \"%s ::= \", symbol_id_names.at(rule_id).c_str());\n    for (size_t i = 0, end = rule.size() - 1; i < end; i++) {\n        llama_grammar_element elem = rule[i];\n        switch (elem.type) {\n            case LLAMA_GRETYPE_END:\n                throw std::runtime_error(\n                    \"unexpected end of rule: \" + std::to_string(rule_id) + \",\" +\n                    std::to_string(i));\n            case LLAMA_GRETYPE_ALT:\n                fprintf(file, \"| \");\n                break;\n            case LLAMA_GRETYPE_RULE_REF:\n                fprintf(file, \"%s \", symbol_id_names.at(elem.value).c_str());\n                break;\n            case LLAMA_GRETYPE_CHAR:\n                fprintf(file, \"[\");\n                print_grammar_char(file, elem.value);\n                break;\n            case LLAMA_GRETYPE_CHAR_NOT:\n                fprintf(file, \"[^\");\n                print_grammar_char(file, elem.value);\n                break;\n            case LLAMA_GRETYPE_CHAR_RNG_UPPER:\n                if (i == 0 || !is_char_element(rule[i - 1])) {\n                    throw std::runtime_error(\n                        \"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: \" +\n                        std::to_string(rule_id) + \",\" + std::to_string(i));\n                }\n                fprintf(file, \"-\");\n                print_grammar_char(file, elem.value);\n                break;\n            case LLAMA_GRETYPE_CHAR_ALT:\n                if (i == 0 || !is_char_element(rule[i - 1])) {\n                    throw std::runtime_error(\n                        \"LLAMA_GRETYPE_CHAR_ALT without preceding char: \" +\n                        std::to_string(rule_id) + \",\" + std::to_string(i));\n                }\n                print_grammar_char(file, elem.value);\n                break;\n            case LLAMA_GRETYPE_CHAR_ANY:\n                fprintf(file, \".\");\n                break;\n            case LLAMA_GRETYPE_TOKEN:\n                fprintf(file, \"<[\");\n                fprintf(file, \"%u\", elem.value);\n                fprintf(file, \"]> \");\n                break;\n            case LLAMA_GRETYPE_TOKEN_NOT:\n                fprintf(file, \"!\");\n                fprintf(file, \"<[\");\n                fprintf(file, \"%u\", elem.value);\n                fprintf(file, \"]> \");\n                break;\n        }\n        if (is_char_element(elem)) {\n            switch (rule[i + 1].type) {\n                case LLAMA_GRETYPE_CHAR_ALT:\n                case LLAMA_GRETYPE_CHAR_RNG_UPPER:\n                case LLAMA_GRETYPE_CHAR_ANY:\n                    break;\n                default:\n                    fprintf(file, \"] \");\n            }\n        }\n    }\n    fprintf(file, \"\\n\");\n}\n\n//\n// Regex utilities\n//\n\nsize_t llama_grammar_trigger_pattern::find(const std::string & input) const {\n    auto find_start_pos = [](const std::smatch & match) {\n        // get from the first matched capturing group to the end of the string\n        size_t start = std::string::npos;\n        for (auto i = 1u; i < match.size(); i++) {\n            if (match.length(i) > 0) {\n                start = match.position(i);\n                break;\n            }\n        }\n        if (start == std::string::npos) {\n            start = match.position(0);\n        }\n        return start;\n    };\n\n    if (!pattern.empty() && pattern.front() == '^' && pattern.back() == '$') {\n        // match against the entire input\n        std::smatch match;\n        if (std::regex_match(input, match, regex)) {\n            return find_start_pos(match);\n        }\n    }\n\n    // search anywhere\n    std::smatch match;\n    if (std::regex_search(input, match, regex)) {\n        return find_start_pos(match);\n    }\n\n    return std::string::npos;\n}\n\n\n//\n// implementation\n//\n\nuint32_t llama_grammar_parser::get_symbol_id(const char * src, size_t len) {\n    uint32_t next_id = static_cast<uint32_t>(symbol_ids.size());\n    auto result = symbol_ids.emplace(std::string(src, len), next_id);\n    return result.first->second;\n}\n\nuint32_t llama_grammar_parser::generate_symbol_id(const std::string & base_name) {\n    uint32_t next_id = static_cast<uint32_t>(symbol_ids.size());\n    symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id;\n    return next_id;\n}\n\nvoid llama_grammar_parser::add_rule(uint32_t rule_id, const llama_grammar_rule & rule) {\n    if (rules.size() <= rule_id) {\n        rules.resize(rule_id + 1);\n    }\n    rules[rule_id] = rule;\n}\n\nconst char * llama_grammar_parser::parse_alternates(\n        const char        * src,\n        const std::string & rule_name,\n        uint32_t            rule_id,\n        bool                is_nested) {\n    llama_grammar_rule rule;\n    const char * pos = parse_sequence(src, rule_name, rule, is_nested);\n    while (*pos == '|') {\n        rule.push_back({LLAMA_GRETYPE_ALT, 0});\n        pos = parse_space(pos + 1, true);\n        pos = parse_sequence(pos, rule_name, rule, is_nested);\n    }\n    rule.push_back({LLAMA_GRETYPE_END, 0});\n    add_rule(rule_id, rule);\n    return pos;\n}\n\nconst char * llama_grammar_parser::parse_sequence(\n        const char         * src,\n        const std::string  & rule_name,\n        llama_grammar_rule & rule,\n        bool               is_nested) {\n    size_t last_sym_start = rule.size();\n    const char * pos = src;\n\n    // use UINT64_MAX as the empty value because we aligned to the proper uint64_t type so -1 can't be used\n    // (though it's technically the same as -1 now)\n    auto handle_repetitions = [&](uint64_t min_times, uint64_t max_times) {\n        bool no_max = max_times == UINT64_MAX;\n        if (last_sym_start == rule.size()) {\n            throw std::runtime_error(std::string(\"expecting preceding item to */+/?/{ at \") + pos);\n        }\n\n        // apply transformation to previous symbol (last_sym_start to end) according to\n        // the following rewrite rules:\n        // S{m,n} --> S S S (m times) S'(n-m)\n        //            S'(x)   ::= S S'(x-1) |\n        //            (... n-m definitions of these S' rules ...)\n        //            S'(1)   ::= S |\n        // S{m,} -->  S S S (m times) S'\n        //            S'     ::= S S' |\n        // S*     --> S{0,}\n        //        --> S'     ::= S S' |\n        // S+     --> S{1,}\n        //        --> S S'\n        //            S'     ::= S S' |\n        // S?     --> S{0,1}\n        //        --> S'\n        //            S'     ::= S |\n\n        llama_grammar_rule prev_rule(rule.begin() + last_sym_start, rule.end());\n        if (min_times == 0) {\n            rule.resize(last_sym_start);\n        } else {\n            // Repeat the previous elements (min_times - 1) times\n            for (uint64_t i = 1; i < min_times; i++) {\n                rule.insert(rule.end(), prev_rule.begin(), prev_rule.end());\n            }\n        }\n\n        uint32_t last_rec_rule_id = 0;\n        auto n_opt = no_max ? 1 : max_times - min_times;\n\n        llama_grammar_rule rec_rule(prev_rule);\n        for (uint64_t i = 0; i < n_opt; i++) {\n            rec_rule.resize(prev_rule.size());\n            uint32_t rec_rule_id = generate_symbol_id( rule_name);\n            if (i > 0 || no_max) {\n                rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, no_max ? rec_rule_id : last_rec_rule_id});\n            }\n            rec_rule.push_back({LLAMA_GRETYPE_ALT, 0});\n            rec_rule.push_back({LLAMA_GRETYPE_END, 0});\n            add_rule( rec_rule_id, rec_rule);\n            last_rec_rule_id = rec_rule_id;\n        }\n        if (n_opt > 0) {\n            rule.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id});\n        }\n    };\n\n    while (*pos) {\n        if (*pos == '\"') { // literal string\n            pos++;\n            last_sym_start = rule.size();\n            while (*pos != '\"') {\n                if (!*pos) {\n                    throw std::runtime_error(\"unexpected end of input\");\n                }\n                auto char_pair = parse_char(pos);\n                     pos       = char_pair.second;\n                rule.push_back({LLAMA_GRETYPE_CHAR, char_pair.first});\n            }\n            pos = parse_space(pos + 1, is_nested);\n        } else if (*pos == '[') { // char range(s)\n            pos++;\n            enum llama_gretype start_type = LLAMA_GRETYPE_CHAR;\n            if (*pos == '^') {\n                pos++;\n                start_type = LLAMA_GRETYPE_CHAR_NOT;\n            }\n            last_sym_start = rule.size();\n            while (*pos != ']') {\n                if (!*pos) {\n                    throw std::runtime_error(\"unexpected end of input\");\n                }\n                auto char_pair = parse_char(pos);\n                     pos       = char_pair.second;\n                enum llama_gretype type = last_sym_start < rule.size()\n                    ? LLAMA_GRETYPE_CHAR_ALT\n                    : start_type;\n\n                rule.push_back({type, char_pair.first});\n                if (pos[0] == '-' && pos[1] != ']') {\n                    if (!pos[1]) {\n                        throw std::runtime_error(\"unexpected end of input\");\n                    }\n                    auto endchar_pair = parse_char(pos + 1);\n                         pos          = endchar_pair.second;\n                    rule.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});\n                }\n            }\n            pos = parse_space(pos + 1, is_nested);\n        } else if (*pos == '<' || *pos == '!') { // token\n            auto type = LLAMA_GRETYPE_TOKEN;\n            if (*pos == '!') { // token inverse\n                type = LLAMA_GRETYPE_TOKEN_NOT;\n                pos++;\n            }\n            auto token_pair = parse_token(vocab, pos);\n            const char * token_end  = token_pair.second;\n            last_sym_start = rule.size();\n            rule.push_back({type, token_pair.first});\n            pos = parse_space(token_end, is_nested);\n        } else if (is_word_char(*pos)) { // rule reference\n            const char * name_end    = parse_name(pos);\n            uint32_t ref_rule_id = get_symbol_id(pos, name_end - pos);\n            pos = parse_space(name_end, is_nested);\n            last_sym_start = rule.size();\n            rule.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id});\n        } else if (*pos == '(') { // grouping\n            // parse nested alternates into synthesized rule\n            pos = parse_space(pos + 1, true);\n            uint32_t sub_rule_id = generate_symbol_id(rule_name);\n            pos = parse_alternates(pos, rule_name, sub_rule_id, true);\n            last_sym_start = rule.size();\n            // output reference to synthesized rule\n            rule.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});\n            if (*pos != ')') {\n                throw std::runtime_error(std::string(\"expecting ')' at \") + pos);\n            }\n            pos = parse_space(pos + 1, is_nested);\n        } else if (*pos == '.') { // any char\n            last_sym_start = rule.size();\n            rule.push_back({LLAMA_GRETYPE_CHAR_ANY, 0});\n            pos = parse_space(pos + 1, is_nested);\n        } else if (*pos == '*') {\n            pos = parse_space(pos + 1, is_nested);\n            handle_repetitions(0, -1);\n        } else if (*pos == '+') {\n            pos = parse_space(pos + 1, is_nested);\n            handle_repetitions(1, -1);\n        } else if (*pos == '?') {\n            pos = parse_space(pos + 1, is_nested);\n            handle_repetitions(0, 1);\n        } else if (*pos == '{') {\n            pos = parse_space(pos + 1, is_nested);\n\n            if (!is_digit_char(*pos)) {\n                throw std::runtime_error(std::string(\"expecting an int at \") + pos);\n            }\n            const char * int_end = parse_int(pos);\n            uint64_t min_times = std::stoull(std::string(pos, int_end - pos));\n            pos = parse_space(int_end, is_nested);\n\n            uint64_t max_times = UINT64_MAX; // default: no max limit\n\n            if (*pos == '}') {\n                max_times = min_times;\n                pos = parse_space(pos + 1, is_nested);\n            } else if (*pos == ',') {\n                pos = parse_space(pos + 1, is_nested);\n\n                if (is_digit_char(*pos)) {\n                    const char * int_end = parse_int(pos);\n                    max_times = std::stoull(std::string(pos, int_end - pos));\n                    pos = parse_space(int_end, is_nested);\n                }\n\n                if (*pos != '}') {\n                    throw std::runtime_error(std::string(\"expecting '}' at \") + pos);\n                }\n                pos = parse_space(pos + 1, is_nested);\n            } else {\n                throw std::runtime_error(std::string(\"expecting ',' at \") + pos);\n            }\n            bool has_max = max_times != UINT64_MAX;\n            if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) {\n                throw std::runtime_error(std::string(\"number of repetitions exceeds sane defaults, please reduce the number of repetitions\"));\n            }\n            handle_repetitions(min_times, max_times);\n        } else {\n            break;\n        }\n    }\n    return pos;\n}\n\nconst char * llama_grammar_parser::parse_rule(const char * src) {\n    const char * name_end = parse_name(src);\n    const char * pos      = parse_space(name_end, false);\n    size_t       name_len = name_end - src;\n    uint32_t     rule_id  = get_symbol_id(src, name_len);\n    const std::string name(src, name_len);\n\n    if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) {\n        throw std::runtime_error(std::string(\"expecting ::= at \") + pos);\n    }\n    pos = parse_space(pos + 3, true);\n\n    pos = parse_alternates(pos, name, rule_id, false);\n\n    if (*pos == '\\r') {\n        pos += pos[1] == '\\n' ? 2 : 1;\n    } else if (*pos == '\\n') {\n        pos++;\n    } else if (*pos) {\n        throw std::runtime_error(std::string(\"expecting newline or end at \") + pos);\n    }\n    return parse_space(pos, true);\n}\n\nbool llama_grammar_parser::parse(const char * src) {\n    try {\n        const char * pos = parse_space(src, true);\n        while (*pos) {\n            pos = parse_rule(pos);\n        }\n        // Validate the state to ensure that all rules are defined\n        for (const auto & rule : rules) {\n            if (rule.empty()) {\n                throw std::runtime_error(\"Undefined rule\");\n            }\n            for (const auto & elem : rule) {\n                if (elem.type == LLAMA_GRETYPE_RULE_REF) {\n                    // Ensure that the rule at that location exists\n                    if (elem.value >= rules.size() || rules[elem.value].empty()) {\n                        // Get the name of the rule that is missing\n                        for (const auto & kv : symbol_ids) {\n                            if (kv.second == elem.value) {\n                                throw std::runtime_error(\"Undefined rule identifier '\" + kv.first + \"'\");\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    } catch (const std::exception & err) {\n        fprintf(stderr, \"%s: error parsing grammar: %s\\n\\n%s\\n\", __func__, err.what(), src);\n        rules.clear();\n        return false;\n    }\n\n    return true;\n}\n\nvoid llama_grammar_parser::print(FILE * file) {\n    try {\n        std::map<uint32_t, std::string> symbol_id_names;\n        for (const auto & kv : symbol_ids) {\n            symbol_id_names[kv.second] = kv.first;\n        }\n        for (size_t i = 0, end = rules.size(); i < end; i++) {\n            // fprintf(file, \"%zu: \", i);\n            // print_rule_binary(file, rules[i]);\n            print_rule(file, uint32_t(i), rules[i], symbol_id_names);\n            // fprintf(file, \"\\n\");\n        }\n    } catch (const std::exception & err) {\n        fprintf(stderr, \"\\n%s: error printing grammar: %s\\n\", __func__, err.what());\n    }\n}\n\nllama_grammar_stack llama_grammar_parser::c_rules() const {\n    llama_grammar_stack ret;\n    ret.reserve(rules.size());\n    for (const auto & rule : rules) {\n        ret.push_back(rule.data());\n    }\n    return ret;\n}\n\n// returns true iff pos points to the end of one of the definitions of a rule\nstatic bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) {\n    switch (pos->type) {\n        case LLAMA_GRETYPE_END: return true;  // NOLINT\n        case LLAMA_GRETYPE_ALT: return true;  // NOLINT\n        default:                return false;\n    }\n}\n\n// returns true iff chr satisfies the char range at pos (regular or inverse range)\n// asserts that pos is pointing to a char range element\nstatic std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(\n        const llama_grammar_element * pos,\n        const uint32_t                chr) {\n    bool found            = false;\n    bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;\n\n    GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); // NOLINT\n\n    do {\n        if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {\n            // inclusive range, e.g. [a-z]\n            found = found || (pos->value <= chr && chr <= pos[1].value);\n            pos += 2;\n        } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {\n            // Any character matches \".\"\n            found = true;\n            pos += 1;\n        } else {\n            // exact char match, e.g. [a] or \"a\"\n            found = found || pos->value == chr;\n            pos += 1;\n        }\n    } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);\n\n    return std::make_pair(found == is_positive_char, pos);\n}\n\n// returns true iff some continuation of the given partial UTF-8 sequence could satisfy the char\n// range at pos (regular or inverse range)\n// asserts that pos is pointing to a char range element\nstatic bool llama_grammar_match_partial_char(\n        const llama_grammar_element * pos,\n        const llama_partial_utf8      partial_utf8) {\n    bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;\n    GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT);\n\n    uint32_t partial_value = partial_utf8.value;\n    int      n_remain      = partial_utf8.n_remain;\n\n    // invalid sequence or 7-bit char split across 2 bytes (overlong)\n    if (n_remain < 0 || (n_remain == 1 && partial_value < 2)) {\n        return false;\n    }\n\n    // range of possible code points this partial UTF-8 sequence could complete to\n    uint32_t low  = partial_value << (n_remain * 6);\n    uint32_t high = low | ((1 << (n_remain * 6)) - 1);\n\n    if (low == 0) {\n        if (n_remain == 2) {\n            low = 1 << 11;\n        } else if (n_remain == 3) {\n            low = 1 << 16;\n        }\n    }\n\n    do {\n        if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {\n            // inclusive range, e.g. [a-z]\n            if (pos->value <= high && low <= pos[1].value) {\n                return is_positive_char;\n            }\n            pos += 2;\n        } else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {\n            // Any character matches \".\"\n            return true;\n        } else {\n            // exact char match, e.g. [a] or \"a\"\n            if (low <= pos->value && pos->value <= high) {\n                return is_positive_char;\n            }\n            pos += 1;\n        }\n    } while (pos->type == LLAMA_GRETYPE_CHAR_ALT);\n\n    return !is_positive_char;\n}\n\n// returns true iff token matches the rule at pos (regular or inverse)\n// asserts that pos is pointing to a token element\nstatic bool llama_grammar_match_token(\n    const llama_grammar_element * pos,\n    const llama_token             token) {\n    GGML_ASSERT(pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT);\n    if (pos->type == LLAMA_GRETYPE_TOKEN) {\n        return pos->value == static_cast<uint32_t>(token);\n    }\n    if (pos->type == LLAMA_GRETYPE_TOKEN_NOT) {\n        return pos->value != static_cast<uint32_t>(token);\n    }\n    return false;\n}\n\n// transforms a grammar pushdown stack into N possible stacks, all ending\n// at a character range (terminal element)\nstatic void llama_grammar_advance_stack(\n        const llama_grammar_rules  & rules,\n        const llama_grammar_stack  & stack,\n              llama_grammar_stacks & new_stacks) {\n    if (stack.empty()) {\n        if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {\n            new_stacks.emplace_back(stack);\n        }\n        return;\n    }\n\n    const llama_grammar_element * pos = stack.back();\n\n    switch (pos->type) {\n        case LLAMA_GRETYPE_RULE_REF: {\n            const size_t                  rule_id = static_cast<size_t>(pos->value);\n            const llama_grammar_element * subpos  = rules[rule_id].data();\n            do {\n                // init new stack without the top (pos)\n                llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);\n                if (!llama_grammar_is_end_of_sequence(pos + 1)) {\n                    // if this rule ref is followed by another element, add that to stack\n                    new_stack.push_back(pos + 1);\n                }\n                if (!llama_grammar_is_end_of_sequence(subpos)) {\n                    // if alternate is nonempty, add to stack\n                    new_stack.push_back(subpos);\n                }\n                llama_grammar_advance_stack(rules, new_stack, new_stacks);\n                while (!llama_grammar_is_end_of_sequence(subpos)) {\n                    // scan to end of alternate def\n                    subpos++;\n                }\n                if (subpos->type == LLAMA_GRETYPE_ALT) {\n                    // there's another alternate def of this rule to process\n                    subpos++;\n                } else {\n                    break;\n                }\n            } while (true);\n            break;\n        }\n        case LLAMA_GRETYPE_CHAR:\n        case LLAMA_GRETYPE_CHAR_NOT:\n        case LLAMA_GRETYPE_CHAR_ANY:\n        case LLAMA_GRETYPE_TOKEN:\n        case LLAMA_GRETYPE_TOKEN_NOT:\n            if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {\n                // only add the stack if it's not a duplicate of one we already have\n                new_stacks.emplace_back(stack);\n            }\n            break;\n        default:\n            // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range\n            // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on\n            // those\n            GGML_ABORT(\"fatal error\");\n    }\n}\n\nstatic llama_grammar_candidates llama_grammar_reject_candidates(\n        const llama_grammar_rules      & rules,\n        const llama_grammar_stacks     & stacks,\n        const llama_grammar_candidates & candidates) {\n    GGML_ASSERT(!stacks.empty()); // REVIEW\n\n    if (candidates.empty()) {\n        return {};\n    }\n\n    auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates);\n\n    for (size_t i = 1, size = stacks.size(); i < size; ++i) {\n        rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects);\n    }\n\n    return rejects;\n}\n\nstatic bool llama_grammar_detect_left_recursion(\n        const llama_grammar_rules & rules,\n        size_t rule_index,\n        std::vector<bool> * rules_visited,\n        std::vector<bool> * rules_in_progress,\n        std::vector<bool> * rules_may_be_empty) {\n    if ((*rules_in_progress)[rule_index]) {\n        return true;\n    }\n\n    (*rules_in_progress)[rule_index] = true;\n\n    const llama_grammar_rule & rule = rules[rule_index];\n\n    // First check if the rule might produce the empty string. This could be done combined with the second\n    // step but it's more readable as two steps.\n    bool at_rule_start = true;\n    for (size_t i = 0; i < rule.size(); i++) {\n        if (llama_grammar_is_end_of_sequence(&rule[i])) {\n            if (at_rule_start) {\n                (*rules_may_be_empty)[rule_index] = true;\n                break;\n            }\n            at_rule_start = true;\n        } else {\n            at_rule_start = false;\n        }\n    }\n\n    // Second, recurse into leftmost nonterminals (or next-leftmost as long as the previous nonterminal may\n    // be empty)\n    bool recurse_into_nonterminal = true;\n    for (size_t i = 0; i < rule.size(); i++) {\n        if (rule[i].type == LLAMA_GRETYPE_RULE_REF && recurse_into_nonterminal) {\n            if (llama_grammar_detect_left_recursion(rules, (size_t)rule[i].value, rules_visited, rules_in_progress, rules_may_be_empty)) {\n                return true;\n            }\n            if (!((*rules_may_be_empty)[(size_t)rule[i].value])) {\n                recurse_into_nonterminal = false;\n            }\n        } else if (llama_grammar_is_end_of_sequence(&rule[i])) {\n            recurse_into_nonterminal = true;\n        } else {\n            recurse_into_nonterminal = false;\n        }\n    }\n\n    (*rules_in_progress)[rule_index] = false;\n    (*rules_visited)[rule_index] = true;\n\n    return false;\n}\n\nconst llama_grammar_rules & llama_grammar_get_rules(const struct llama_grammar * grammar) {\n    return grammar->rules;\n}\n\nllama_grammar_stacks & llama_grammar_get_stacks(struct llama_grammar * grammar) {\n    return grammar->stacks;\n}\n\nstatic void llama_grammar_accept_chr(\n        struct llama_grammar       & grammar,\n        const llama_grammar_stack  & stack,\n              uint32_t               chr,\n              llama_grammar_stacks & new_stacks) {\n    if (stack.empty()) {\n        return;\n    }\n\n    const llama_grammar_element * pos = stack.back();\n\n    // ignore if this turns into a token\n    if (pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT) {\n        return;\n    }\n\n    auto match = llama_grammar_match_char(pos, chr);\n    if (match.first) {\n        llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);\n        if (!llama_grammar_is_end_of_sequence(match.second)) {\n            new_stack.push_back(match.second);\n        }\n        llama_grammar_advance_stack(grammar.rules, new_stack, new_stacks);\n    }\n}\n\nvoid llama_grammar_accept(struct llama_grammar * grammar, uint32_t chr) {\n    llama_grammar_stacks stacks_new;\n    stacks_new.reserve(grammar->stacks.size());\n\n    for (const auto & stack : grammar->stacks) {\n        llama_grammar_accept_chr(*grammar, stack, chr, stacks_new);\n    }\n\n    grammar->stacks = std::move(stacks_new);\n}\n\nllama_grammar_candidates llama_grammar_reject_candidates_for_stack(\n        const llama_grammar_rules      & rules,\n        const llama_grammar_stack      & stack,\n        const llama_grammar_candidates & candidates) {\n\n    llama_grammar_candidates rejects;\n    rejects.reserve(candidates.size());\n\n    if (stack.empty()) {\n        for (const auto & tok : candidates) {\n            if (*tok.code_points != 0 || tok.partial_utf8.n_remain != 0) {\n                rejects.push_back(tok);\n            }\n        }\n        return rejects;\n    }\n\n    const llama_grammar_element * stack_pos = stack.back();\n\n    // if the top of the stack is a token rule, then we only need to check the token id\n    if (stack_pos->type == LLAMA_GRETYPE_TOKEN || stack_pos->type == LLAMA_GRETYPE_TOKEN_NOT) {\n        for (const auto & tok : candidates) {\n            if (*tok.code_points == 0) {\n                // reached the end of a token consumed by char rules, reject iff it ended\n                // in a partial response\n                if (tok.partial_utf8.n_remain != 0) {\n                    rejects.push_back(tok);\n                }\n            } else if (!llama_grammar_match_token(stack_pos, tok.id)) {\n                rejects.push_back(tok);\n            }\n        }\n        return rejects;\n    }\n\n    llama_grammar_candidates next_candidates;\n    next_candidates.reserve(candidates.size());\n\n    for (const auto & tok : candidates) {\n        if (*tok.code_points == 0) {\n            // reached end of full codepoints in token, reject iff it ended in a partial sequence\n            // that cannot satisfy this position in grammar\n            if (tok.partial_utf8.n_remain != 0 &&\n                    !llama_grammar_match_partial_char(stack_pos, tok.partial_utf8)) {\n                rejects.push_back(tok);\n            }\n        } else if (llama_grammar_match_char(stack_pos, *tok.code_points).first) {\n            next_candidates.push_back({ tok.index, tok.code_points + 1, tok.partial_utf8, tok.id });\n        } else {\n            rejects.push_back(tok);\n        }\n    }\n\n    const auto * stack_pos_after = llama_grammar_match_char(stack_pos, 0).second;\n\n    // update top of stack to next element, if any\n    llama_grammar_stack stack_after(stack.begin(), stack.end() - 1);\n    if (!llama_grammar_is_end_of_sequence(stack_pos_after)) {\n        stack_after.push_back(stack_pos_after);\n    }\n    llama_grammar_stacks next_stacks;\n    llama_grammar_advance_stack(rules, stack_after, next_stacks);\n\n    auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates);\n    for (const auto & tok : next_rejects) {\n        rejects.push_back({ tok.index, tok.code_points - 1, tok.partial_utf8, tok.id });\n    }\n\n    return rejects;\n}\n\n////////////////////\n\nstruct llama_grammar * llama_grammar_init_impl(\n        const struct llama_vocab * vocab,\n        const llama_grammar_element ** rules,\n        size_t n_rules,\n        size_t start_rule_index) {\n    const llama_grammar_element * pos;\n\n    // copy rule definitions into vectors\n    llama_grammar_rules vec_rules(n_rules);\n    for (size_t i = 0; i < n_rules; i++) {\n        for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) {\n            vec_rules[i].push_back(*pos);\n        }\n        vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});\n    }\n\n    // Check for left recursion\n    std::vector<bool> rules_visited(n_rules);\n    std::vector<bool> rules_in_progress(n_rules);\n    std::vector<bool> rules_may_be_empty(n_rules);\n    for (size_t i = 0; i < n_rules; i++) {\n        if (rules_visited[i]) {\n            continue;\n        }\n        if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) {\n            LLAMA_LOG_ERROR(\"unsupported grammar, left recursion detected for nonterminal at index %zu\", i);\n            return nullptr;\n        }\n    }\n\n    // loop over alternates of start rule to build initial stacks\n    llama_grammar_stacks stacks;\n    pos = vec_rules[start_rule_index].data();\n    do {\n        llama_grammar_stack stack;\n        if (!llama_grammar_is_end_of_sequence(pos)) {\n            // if alternate is nonempty, add to stack\n            stack.push_back(pos);\n        }\n        llama_grammar_advance_stack(vec_rules, stack, stacks);\n        while (!llama_grammar_is_end_of_sequence(pos)) {\n            // scan to end of alternate def\n            pos++;\n        }\n        if (pos->type == LLAMA_GRETYPE_ALT) {\n            // there's another alternate def of this rule to process\n            pos++;\n        } else {\n            break;\n        }\n    } while (true);\n\n    // Important: vec_rules has to be moved here, not copied, because stacks contains\n    // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar\n    // then the pointers would be invalidated when the local vec_rules goes out of scope.\n    return new llama_grammar {\n        vocab,\n        std::move(vec_rules),\n        std::move(stacks),\n        /* .partial_utf8 = */             {},\n        /* .lazy = */                     false,\n        /* .awaiting_trigger = */         false,\n        /* .trigger_buffer = */           \"\",\n        /* .trigger_buffer_positions = */ {},\n        /* .trigger_tokens = */           {},\n        /* .trigger_patterns = */         {},\n    };\n}\n\nstruct llama_grammar * llama_grammar_init_impl(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                              bool lazy,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens) {\n    llama_grammar_parser parser(vocab);\n\n    // if there is a grammar, parse it\n    // rules will be empty (default) if there are parse errors\n    if (!parser.parse(grammar_str) || parser.rules.empty()) {\n        LLAMA_LOG_ERROR(\"failed to parse grammar\\n\");\n        return nullptr;\n    }\n\n    // Ensure that the grammar contains the start symbol\n    if (parser.symbol_ids.find(grammar_root) == parser.symbol_ids.end()) {\n        LLAMA_LOG_ERROR(\"grammar does not contain a '%s' symbol\\n\", grammar_root);\n        return nullptr;\n    }\n\n    std::vector<const llama_grammar_element *> grammar_rules(parser.c_rules());\n\n    const size_t n_rules = grammar_rules.size();\n    const size_t start_rule_index = parser.symbol_ids.at(grammar_root);\n\n    const llama_grammar_element * pos;\n\n    // copy rule definitions into vectors\n    llama_grammar_rules vec_rules(n_rules);\n    for (size_t i = 0; i < n_rules; i++) {\n        for (pos = grammar_rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) {\n            vec_rules[i].push_back(*pos);\n        }\n        vec_rules[i].push_back({LLAMA_GRETYPE_END, 0});\n    }\n\n    // Check for left recursion\n    std::vector<bool> rules_visited(n_rules);\n    std::vector<bool> rules_in_progress(n_rules);\n    std::vector<bool> rules_may_be_empty(n_rules);\n    for (size_t i = 0; i < n_rules; i++) {\n        if (rules_visited[i]) {\n            continue;\n        }\n        if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) {\n            LLAMA_LOG_ERROR(\"unsupported grammar, left recursion detected for nonterminal at index %zu\\n\", i);\n            return nullptr;\n        }\n    }\n\n    // loop over alternates of start rule to build initial stacks\n    llama_grammar_stacks stacks;\n    pos = vec_rules[start_rule_index].data();\n    do {\n        llama_grammar_stack stack;\n        if (!llama_grammar_is_end_of_sequence(pos)) {\n            // if alternate is nonempty, add to stack\n            stack.push_back(pos);\n        }\n        llama_grammar_advance_stack(vec_rules, stack, stacks);\n        while (!llama_grammar_is_end_of_sequence(pos)) {\n            // scan to end of alternate def\n            pos++;\n        }\n        if (pos->type == LLAMA_GRETYPE_ALT) {\n            // there's another alternate def of this rule to process\n            pos++;\n        } else {\n            break;\n        }\n    } while (true);\n\n    std::vector<llama_token>    vec_trigger_tokens;\n    std::vector<llama_grammar_trigger_pattern> vec_trigger_patterns;\n    for (size_t i = 0; i < num_trigger_tokens; i++) {\n        GGML_ASSERT(trigger_tokens != nullptr);\n        vec_trigger_tokens.push_back(trigger_tokens[i]);\n    }\n    for (size_t i = 0; i < num_trigger_patterns; i++) {\n        GGML_ASSERT(trigger_patterns != nullptr);\n        auto & trigger = vec_trigger_patterns.emplace_back();\n        trigger.pattern = trigger_patterns[i];\n        trigger.regex = std::regex(trigger.pattern);\n    }\n\n    // Important: vec_rules has to be moved here, not copied, because stacks contains\n    // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar\n    // then the pointers would be invalidated when the local vec_rules goes out of scope.\n    return new llama_grammar {\n        vocab,\n        std::move(vec_rules),\n        std::move(stacks),\n        /* .partial_utf8 = */             {},\n        /* .lazy = */                     lazy,\n        /* .awaiting_trigger = */         lazy,\n        /* .trigger_buffer = */           \"\",\n        /* .trigger_buffer_positions = */ {},\n        std::move(vec_trigger_tokens),\n        std::move(vec_trigger_patterns),\n    };\n}\n\nvoid llama_grammar_free_impl(struct llama_grammar * grammar) {\n    if (grammar == nullptr) {\n        return;\n    }\n\n    delete grammar;\n}\n\nstruct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar) {\n    auto * result = new llama_grammar {\n        grammar.vocab,\n        grammar.rules,\n        grammar.stacks,\n        grammar.partial_utf8,\n        grammar.lazy,\n        grammar.awaiting_trigger,\n        grammar.trigger_buffer,\n        grammar.trigger_buffer_positions,\n        grammar.trigger_tokens,\n        grammar.trigger_patterns,\n    };\n\n    // redirect elements in stacks to point to new rules\n    for (size_t is = 0; is < result->stacks.size(); is++) {\n        for (size_t ie = 0; ie < result->stacks[is].size(); ie++) {\n            for (size_t ir0 = 0; ir0 < grammar.rules.size(); ir0++) {\n                for (size_t ir1 = 0; ir1 < grammar.rules[ir0].size(); ir1++) {\n                    if (grammar.stacks[is][ie] == &grammar.rules[ir0][ir1]) {\n                        result->stacks[is][ie] =  &result->rules[ir0][ir1];\n                    }\n                }\n            }\n        }\n    }\n\n    return result;\n}\n\nvoid llama_grammar_apply_impl(const struct llama_grammar & grammar, llama_token_data_array * cur_p) {\n    GGML_ASSERT(grammar.vocab != nullptr);\n\n    if (grammar.awaiting_trigger) {\n        return;\n    }\n\n    bool allow_eog = false;\n    for (const auto & stack : grammar.stacks) {\n        if (stack.empty()) {\n            allow_eog = true;\n            break;\n        }\n    }\n\n    std::vector<std::pair<std::vector<uint32_t>, llama_partial_utf8>> candidates_decoded;\n    candidates_decoded.reserve(cur_p->size);\n\n    llama_grammar_candidates candidates_grammar;\n    candidates_grammar.reserve(cur_p->size);\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        const llama_token id      = cur_p->data[i].id;\n        const std::string & piece = grammar.vocab->token_to_piece(id);\n\n        if (grammar.vocab->is_eog(id)) {\n            if (!allow_eog) {\n                cur_p->data[i].logit = -INFINITY;\n            }\n        } else if (piece.empty() || piece[0] == 0) {\n            cur_p->data[i].logit = -INFINITY;\n        } else {\n            candidates_decoded.push_back(decode_utf8(piece, grammar.partial_utf8));\n            candidates_grammar.push_back({ i, candidates_decoded.back().first.data(), candidates_decoded.back().second, id });\n        }\n    }\n\n    const auto rejects = llama_grammar_reject_candidates(grammar.rules, grammar.stacks, candidates_grammar);\n    for (const auto & reject : rejects) {\n        cur_p->data[reject.index].logit = -INFINITY;\n    }\n}\n\nvoid llama_grammar_accept_impl(struct llama_grammar & grammar, llama_token token) {\n    GGML_ASSERT(grammar.vocab != nullptr);\n\n    const auto & piece = grammar.vocab->token_to_piece(token);\n\n    if (grammar.awaiting_trigger) {\n        if (std::find(grammar.trigger_tokens.begin(), grammar.trigger_tokens.end(), token) != grammar.trigger_tokens.end()) {\n            grammar.awaiting_trigger = false;\n            grammar.trigger_buffer.clear();\n            llama_grammar_accept_token(grammar, token, piece);\n            LLAMA_LOG_DEBUG(\"Grammar triggered on token %u (`%s`)\", token, piece.c_str());\n            return;\n        } else {\n            auto position = std::make_pair(grammar.trigger_buffer.size(), grammar.trigger_buffer.size() + piece.size());\n            grammar.trigger_buffer_positions.push_back(std::make_pair(token, position));\n            grammar.trigger_buffer += piece;\n\n            for (const auto & trigger_pattern : grammar.trigger_patterns) {\n                auto start = trigger_pattern.find(grammar.trigger_buffer);\n                if (start != std::string::npos) {\n                    grammar.awaiting_trigger = false;\n\n                    // replay tokens that overlap with [start, end)\n                    for (const auto & [tok, tok_pos] : grammar.trigger_buffer_positions) {\n                        auto [tok_start, tok_end] = tok_pos;\n                        if (tok_end <= start) {\n                            continue;\n                        }\n\n                        size_t piece_start = (tok_start < start) ? start : tok_start; // allow for partial token pieces\n                        size_t piece_len = tok_end - piece_start;\n                        auto tok_piece = grammar.trigger_buffer.substr(piece_start, piece_len);\n                        llama_grammar_accept_token(grammar, tok, tok_piece);\n                    }\n\n                    auto constrained_str = grammar.trigger_buffer.substr(start);\n                    grammar.trigger_buffer.clear();\n                    grammar.trigger_buffer_positions.clear();\n                    LLAMA_LOG_DEBUG(\"Grammar triggered on regex: '%s'\\n\", constrained_str.c_str());\n                    return;\n                }\n            }\n            LLAMA_LOG_DEBUG(\"Grammar still awaiting trigger after token %d (`%s`)\\n\", token, piece.c_str());\n            return;\n        }\n    }\n\n    if (grammar.vocab->is_eog(token)) {\n        for (const auto & stack : grammar.stacks) {\n            if (stack.empty()) {\n                return;\n            }\n        }\n        GGML_ABORT(\"fatal error\");\n    }\n\n    llama_grammar_accept_token(grammar, token, piece);\n}\n\nvoid llama_grammar_accept_str(struct llama_grammar & grammar, const std::string & piece) {\n    // Note terminating 0 in decoded string\n    const auto   decoded     = decode_utf8(piece, grammar.partial_utf8);\n    const auto & code_points = decoded.first;\n\n    for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {\n        llama_grammar_accept(&grammar, *it);\n    }\n\n    grammar.partial_utf8 = decoded.second;\n    if (grammar.stacks.empty()) {\n        throw std::runtime_error(\"Unexpected empty grammar stack after accepting piece: \" + piece);\n    }\n}\n\nvoid llama_grammar_accept_token(struct llama_grammar & grammar, llama_token token, const std::string & piece) {\n    // Note terminating 0 in decoded string\n    const auto   decoded     = decode_utf8(piece, grammar.partial_utf8);\n    const auto & code_points = decoded.first;\n\n    llama_grammar_stacks stacks_new;\n    stacks_new.reserve(grammar.stacks.size());\n\n    for (const auto & stack : grammar.stacks) {\n        if (stack.empty()) {\n            continue;\n        }\n\n        const llama_grammar_element * pos = stack.back();\n\n        if (pos->type == LLAMA_GRETYPE_TOKEN || pos->type == LLAMA_GRETYPE_TOKEN_NOT) {\n            if (llama_grammar_match_token(pos, token)) {\n                llama_grammar_stack new_stack(stack.begin(), stack.end() - 1);\n                if (!llama_grammar_is_end_of_sequence(pos + 1)) {\n                    new_stack.push_back(pos + 1);\n                }\n                llama_grammar_advance_stack(grammar.rules, new_stack, stacks_new);\n            }\n        } else {\n            llama_grammar_stacks current_stacks = {stack};\n\n            for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) {\n                llama_grammar_stacks next_stacks;\n\n                for (const auto & cur_stack : current_stacks) {\n                    llama_grammar_accept_chr(grammar, cur_stack, *it, next_stacks);\n                }\n\n                current_stacks = std::move(next_stacks);\n                if (current_stacks.empty()) {\n                    break;\n                }\n            }\n\n            for (auto & surviving_stack : current_stacks) {\n                if (std::find(stacks_new.begin(), stacks_new.end(), surviving_stack) == stacks_new.end()) {\n                    stacks_new.emplace_back(surviving_stack);\n                }\n            }\n        }\n    }\n\n    grammar.stacks = std::move(stacks_new);\n    grammar.partial_utf8 = decoded.second;\n\n    if (grammar.stacks.empty()) {\n        throw std::runtime_error(\"Unexpected empty grammar stack after accepting piece: \" + piece + \" (\" + std::to_string(token) + \")\");\n    }\n}\n\n"
  },
  {
    "path": "examples/talk-llama/llama-grammar.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <map>\n#include <regex>\n#include <string>\n#include <vector>\n\nstruct llama_vocab;\n\n// grammar element type\nenum llama_gretype {\n    // end of rule definition\n    LLAMA_GRETYPE_END            = 0,\n\n    // start of alternate definition for rule\n    LLAMA_GRETYPE_ALT            = 1,\n\n    // non-terminal element: reference to rule\n    LLAMA_GRETYPE_RULE_REF       = 2,\n\n    // terminal element: character (code point)\n    LLAMA_GRETYPE_CHAR           = 3,\n\n    // inverse char(s) ([^a], [^a-b] [^abc])\n    LLAMA_GRETYPE_CHAR_NOT       = 4,\n\n    // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to\n    // be an inclusive range ([a-z])\n    LLAMA_GRETYPE_CHAR_RNG_UPPER = 5,\n\n    // modifies a preceding LLAMA_GRETYPE_CHAR or\n    // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA])\n    LLAMA_GRETYPE_CHAR_ALT       = 6,\n\n    // any character (.)\n    LLAMA_GRETYPE_CHAR_ANY       = 7,\n\n    // terminal element: token (<[token-id]>)\n    LLAMA_GRETYPE_TOKEN          = 8,\n\n    // inverse token (!<[token-id]>)\n    LLAMA_GRETYPE_TOKEN_NOT      = 9,\n};\n\ntypedef struct llama_grammar_element {\n    enum llama_gretype type;\n    uint32_t           value; // Unicode code point, rule ID, or token ID\n} llama_grammar_element;\n\nstruct llama_partial_utf8 {\n    uint32_t value;    // bit value so far (unshifted)\n    int      n_remain; // num bytes remaining; -1 indicates invalid sequence\n};\n\nstruct llama_grammar_candidate {\n    size_t               index;\n    const uint32_t     * code_points;\n    llama_partial_utf8   partial_utf8;\n    llama_token          id;\n};\n\nusing llama_grammar_rule  = std::vector<      llama_grammar_element>;\nusing llama_grammar_stack = std::vector<const llama_grammar_element *>;\n\nusing llama_grammar_rules      = std::vector<llama_grammar_rule>;\nusing llama_grammar_stacks     = std::vector<llama_grammar_stack>;\nusing llama_grammar_candidates = std::vector<llama_grammar_candidate>;\n\n// TODO: remove, needed for tests atm\nconst llama_grammar_rules  & llama_grammar_get_rules (const struct llama_grammar * grammar);\n      llama_grammar_stacks & llama_grammar_get_stacks(      struct llama_grammar * grammar);\n\n// takes a set of possible pushdown stacks on a grammar, which are required to\n// be positioned at a character range (see `llama_grammar_advance_stack`), and\n// produces the N possible stacks if the given char is accepted at those\n// positions\nvoid llama_grammar_accept(struct llama_grammar * grammar, uint32_t chr);\n\nstd::vector<llama_grammar_candidate> llama_grammar_reject_candidates_for_stack(\n        const llama_grammar_rules      & rules,\n        const llama_grammar_stack      & stack,\n        const llama_grammar_candidates & candidates);\n\nstruct llama_grammar_parser {\n    const llama_vocab * vocab;\n    std::map<std::string, uint32_t> symbol_ids;\n\n    llama_grammar_rules rules;\n\n    llama_grammar_parser(const struct llama_vocab * vocab = nullptr) : vocab(vocab) {}\n\n    llama_grammar_stack c_rules() const;\n\n    uint32_t get_symbol_id(const char * src, size_t len);\n    uint32_t generate_symbol_id(const std::string & base_name);\n\n    void add_rule(uint32_t rule_id, const llama_grammar_rule & rule);\n\n    const char * parse_alternates(\n            const char        * src,\n            const std::string & rule_name,\n            uint32_t            rule_id,\n            bool                is_nested);\n\n    const char * parse_sequence(\n            const char         * src,\n            const std::string  & rule_name,\n            llama_grammar_rule & rule,\n            bool               is_nested);\n\n    const char * parse_rule(const char * src);\n\n    bool parse(const char * src);\n    void print(FILE * file);\n};\n\nstruct llama_grammar_trigger_pattern {\n    std::string pattern;\n    std::regex  regex;\n\n    size_t find(const std::string & input) const;\n};\n\nstruct llama_grammar {\n    // maintain a list of llama_tokens and their positions in the trigger_buffer\n    using token_pos = std::pair<llama_token, std::pair<size_t, size_t>>;\n\n    // note: allow null vocab for testing (not great)\n    const llama_vocab * vocab;\n\n    const llama_grammar_rules  rules;  // TODO: shared ptr\n          llama_grammar_stacks stacks;\n\n    // buffer for partially generated UTF-8 sequence from accepted tokens\n    llama_partial_utf8 partial_utf8;\n\n    // lazy grammars wait for trigger words or tokens before constraining the sampling.\n    // we still have trigger_tokens for non-lazy grammars to force printing of special trigger tokens.\n    // (useful e.g. for tool_choice=required)\n    bool                     lazy             = false;\n    bool                     awaiting_trigger = false; // Initialized to true for lazy grammars only\n    std::string              trigger_buffer;           // Output buffered by lazy grammar. Will be cleared once trigger is found.\n    std::vector<token_pos>   trigger_buffer_positions; // Tokens buffered by lazy grammar. Used to replay when a trigger is found.\n    std::vector<llama_token> trigger_tokens;           // Tokens that trigger a lazy grammar, or tokens to force printing of (even if special).\n    std::vector<llama_grammar_trigger_pattern>\n                             trigger_patterns;         // Regular expressions that trigger a lazy grammar. Must be a full match of the entire generated\n                                                       // string, and the grammar will be given the string from the first match group onwards.\n\n};\n\n//\n// internal API\n//\n\n// note: needed for tests (not great)\nstruct llama_grammar * llama_grammar_init_impl(\n        const struct llama_vocab * vocab,\n        const llama_grammar_element ** rules,\n        size_t n_rules,\n        size_t start_rule_index);\n\nstruct llama_grammar * llama_grammar_init_impl(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                              bool lazy,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens);\n\nvoid llama_grammar_free_impl(struct llama_grammar * grammar);\n\nstruct llama_grammar * llama_grammar_clone_impl(const struct llama_grammar & grammar);\n\n// TODO: move the API below as member functions of llama_grammar\nvoid llama_grammar_apply_impl(\n        const struct llama_grammar & grammar,\n            llama_token_data_array * cur_p);\n\nvoid llama_grammar_accept_impl(\n              struct llama_grammar & grammar,\n                       llama_token   token);\n\nvoid llama_grammar_accept_str(\n              struct llama_grammar & grammar,\n                 const std::string & piece);\n\nvoid llama_grammar_accept_token(\n              struct llama_grammar & grammar,\n                       llama_token   token,\n                 const std::string & piece);\n"
  },
  {
    "path": "examples/talk-llama/llama-graph.cpp",
    "content": "#include \"llama-graph.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-batch.h\"\n#include \"llama-cparams.h\"\n\n#include \"llama-kv-cache.h\"\n#include \"llama-kv-cache-iswa.h\"\n#include \"llama-memory-hybrid.h\"\n#include \"llama-memory-hybrid-iswa.h\"\n#include \"llama-memory-recurrent.h\"\n\n#include <cassert>\n#include <cmath>\n#include <cstring>\n#include <numeric>\n#include <sstream>\n#include <unordered_set>\n\n// dedup helpers\n\nstatic ggml_tensor * build_kq_mask(\n        ggml_context * ctx,\n        const llama_kv_cache_context * mctx,\n        const llama_ubatch & ubatch,\n        const llama_cparams & cparams) {\n    const auto n_kv     = mctx->get_n_kv();\n    const auto n_tokens = ubatch.n_tokens;\n    const auto n_stream = cparams.kv_unified ? 1 : ubatch.n_seqs_unq;\n\n    return ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_kv, n_tokens/n_stream, 1, n_stream);\n}\n\nstatic bool can_reuse_kq_mask(\n        ggml_tensor * kq_mask,\n        const llama_kv_cache_context * mctx,\n        const llama_ubatch & ubatch,\n        const llama_cparams & cparams) {\n    const auto n_kv     = mctx->get_n_kv();\n    const auto n_tokens = ubatch.n_tokens;\n    const auto n_stream = cparams.kv_unified ? 1 : ubatch.n_seqs_unq;\n\n    bool res = true;\n\n    res &= (kq_mask->ne[0] == n_kv);\n    res &= (kq_mask->ne[1] == n_tokens/n_stream);\n    res &= (kq_mask->ne[2] == 1);\n    res &= (kq_mask->ne[3] == n_stream);\n\n    return res;\n}\n\n// impl\n\nvoid llm_graph_input_embd::set_input(const llama_ubatch * ubatch) {\n    if (ubatch->token) {\n        const int64_t n_tokens = ubatch->n_tokens;\n\n        ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens));\n    }\n\n    if (ubatch->embd) {\n        GGML_ASSERT(n_embd == embd->ne[0]);\n\n        const int64_t n_tokens = ubatch->n_tokens;\n\n        ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd));\n    }\n}\n\nbool llm_graph_input_embd::can_reuse(const llm_graph_params & params) {\n    bool res = true;\n\n    res &= (!params.ubatch.token) || (tokens && tokens->ne[0] == params.ubatch.n_tokens);\n    res &= (!params.ubatch.embd)  || (embd   &&   embd->ne[1] == params.ubatch.n_tokens);\n\n    return res;\n}\n\nvoid llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {\n    if (ubatch->pos && pos) {\n        const int64_t n_tokens = ubatch->n_tokens;\n\n        if (ubatch->token && n_pos_per_embd == 4) {\n            // in case we're using M-RoPE with text tokens, convert the 1D positions to 4D\n            // the 3 first dims are the same, and 4th dim is all 0\n            std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd);\n            // copy the first dimension\n            for (int i = 0; i < n_tokens; ++i) {\n                pos_data[               i] = ubatch->pos[i];\n                pos_data[    n_tokens + i] = ubatch->pos[i];\n                pos_data[2 * n_tokens + i] = ubatch->pos[i];\n                pos_data[3 * n_tokens + i] = 0; // 4th dim is 0\n            }\n            ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));\n        } else {\n            ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));\n        }\n    }\n}\n\nbool llm_graph_input_pos::can_reuse(const llm_graph_params & params) {\n    bool res = true;\n\n    res &= pos->ne[0] == params.ubatch.n_tokens*n_pos_per_embd;\n\n    return res;\n}\n\nvoid llm_graph_input_attn_temp::set_input(const llama_ubatch * ubatch) {\n    if (ubatch->pos && attn_scale) {\n        const int64_t n_tokens = ubatch->n_tokens;\n\n        GGML_ASSERT(f_attn_temp_scale != 0.0f);\n        GGML_ASSERT(n_attn_temp_floor_scale != 0);\n\n        std::vector<float> attn_scale_data(n_tokens, 0.0f);\n        for (int i = 0; i < n_tokens; ++i) {\n            const float pos = ubatch->pos[i];\n            attn_scale_data[i] = std::log(\n                std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0\n            ) * f_attn_temp_scale + 1.0;\n        }\n\n        ggml_backend_tensor_set(attn_scale, attn_scale_data.data(), 0, n_tokens*ggml_element_size(attn_scale));\n    }\n}\n\nvoid llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) {\n    if (pos_bucket) {\n        const int64_t n_tokens = ubatch->n_tokens;\n\n        GGML_ASSERT(ggml_backend_buffer_is_host(pos_bucket->buffer));\n        GGML_ASSERT(!ubatch->equal_seqs()); // TODO: use ubatch->n_seqs instead of failing\n\n        int32_t * data = (int32_t *) pos_bucket->data;\n\n        for (int j = 0; j < n_tokens; ++j) {\n            for (int i = 0; i < n_tokens; ++i) {\n                data[j*n_tokens + i] = llama_relative_position_bucket(ubatch->pos[i], ubatch->pos[j], hparams.n_rel_attn_bkts, true);\n            }\n        }\n    }\n}\n\nvoid llm_graph_input_pos_bucket_kv::set_input(const llama_ubatch * ubatch) {\n    if (pos_bucket) {\n        mctx->set_input_pos_bucket(pos_bucket, ubatch);\n    }\n}\n\nvoid llm_graph_input_out_ids::set_input(const llama_ubatch * ubatch) {\n    GGML_ASSERT(out_ids);\n\n    const int64_t n_tokens = ubatch->n_tokens;\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(out_ids->buffer));\n    int32_t * data = (int32_t *) out_ids->data;\n\n    if (n_outputs == n_tokens) {\n        for (int i = 0; i < n_tokens; ++i) {\n            data[i] = i;\n        }\n\n        return;\n    }\n\n    GGML_ASSERT(ubatch->output);\n\n    int n_outputs = 0;\n\n    for (int i = 0; i < n_tokens; ++i) {\n        if (ubatch->output[i]) {\n            data[n_outputs++] = i;\n        }\n    }\n}\n\nbool llm_graph_input_out_ids::can_reuse(const llm_graph_params & params) {\n    bool res = true;\n\n    res &= n_outputs == params.n_outputs;\n\n    return res;\n}\n\nvoid llm_graph_input_mean::set_input(const llama_ubatch * ubatch) {\n    if (cparams.embeddings   &&\n       (cparams.pooling_type == LLAMA_POOLING_TYPE_MEAN ||\n        cparams.pooling_type == LLAMA_POOLING_TYPE_RANK )) {\n\n        const int64_t n_tokens     = ubatch->n_tokens;\n        const int64_t n_seq_tokens = ubatch->n_seq_tokens;\n        const int64_t n_seqs_unq   = ubatch->n_seqs_unq;\n\n        GGML_ASSERT(mean);\n        GGML_ASSERT(ggml_backend_buffer_is_host(mean->buffer));\n\n        float * data = (float *) mean->data;\n        memset(mean->data, 0, n_tokens*n_seqs_unq*ggml_element_size(mean));\n\n        std::vector<uint64_t> sums(n_seqs_unq, 0);\n        for (int i = 0; i < n_tokens; i += n_seq_tokens) {\n            for (int s = 0; s < ubatch->n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id  = ubatch->seq_id[i][s];\n                const int32_t      seq_idx = ubatch->seq_idx[seq_id];\n\n                sums[seq_idx] += ubatch->n_seq_tokens;\n            }\n        }\n\n        std::vector<float> div(n_seqs_unq, 0.0f);\n        for (int s = 0; s < n_seqs_unq; ++s) {\n            const uint64_t sum = sums[s];\n            if (sum > 0) {\n                div[s] = 1.0f/float(sum);\n            }\n        }\n\n        for (int i = 0; i < n_tokens; i += n_seq_tokens) {\n            for (int s = 0; s < ubatch->n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id  = ubatch->seq_id[i][s];\n                const int32_t      seq_idx = ubatch->seq_idx[seq_id];\n\n                for (int j = 0; j < n_seq_tokens; ++j) {\n                    data[seq_idx*n_tokens + i + j] = div[seq_idx];\n                }\n            }\n        }\n    }\n}\n\nvoid llm_graph_input_cls::set_input(const llama_ubatch * ubatch) {\n    const int64_t n_tokens     = ubatch->n_tokens;\n    const int64_t n_seqs_unq   = ubatch->n_seqs_unq;\n\n    if (cparams.embeddings && (\n        cparams.pooling_type == LLAMA_POOLING_TYPE_CLS  ||\n        cparams.pooling_type == LLAMA_POOLING_TYPE_RANK ||\n        cparams.pooling_type == LLAMA_POOLING_TYPE_LAST\n    )) {\n        GGML_ASSERT(cls);\n        GGML_ASSERT(ggml_backend_buffer_is_host(cls->buffer));\n\n        uint32_t * data = (uint32_t *) cls->data;\n        memset(cls->data, 0, n_seqs_unq*ggml_element_size(cls));\n\n        std::vector<int> target_pos(n_seqs_unq, -1);\n        std::vector<int> target_row(n_seqs_unq, -1);\n\n        const bool last = (\n             cparams.pooling_type == LLAMA_POOLING_TYPE_LAST ||\n            (cparams.pooling_type == LLAMA_POOLING_TYPE_RANK && (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_QWEN3VL)) // qwen3 reranking & embedding models use last token\n        );\n\n        for (int i = 0; i < n_tokens; ++i) {\n            const llama_pos pos = ubatch->pos[i];\n\n            for (int s = 0; s < ubatch->n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id  = ubatch->seq_id[i][s];\n                const int32_t      seq_idx = ubatch->seq_idx[seq_id];\n\n                if (\n                    (target_pos[seq_idx] == -1) ||\n                    ( last && pos >= target_pos[seq_idx]) ||\n                    (!last && pos <  target_pos[seq_idx])\n                ) {\n                    target_pos[seq_idx] = pos;\n                    target_row[seq_idx] = i;\n                }\n            }\n        }\n\n        for (int s = 0; s < n_seqs_unq; ++s) {\n            if (target_row[s] >= 0) {\n                data[s] = target_row[s];\n            }\n        }\n    }\n}\n\nvoid llm_graph_input_rs::set_input(const llama_ubatch * ubatch) {\n    GGML_UNUSED(ubatch);\n\n    const int64_t n_rs = mctx->get_n_rs();\n\n    if (s_copy) {\n        GGML_ASSERT(ggml_backend_buffer_is_host(s_copy->buffer));\n        int32_t * data = (int32_t *) s_copy->data;\n\n        // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n\n        for (uint32_t i = 0; i < n_rs; ++i) {\n            data[i] = mctx->s_copy(i);\n        }\n    }\n}\n\nbool llm_graph_input_rs::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_memory_recurrent_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= s_copy->ne[0] == mctx->get_n_rs();\n\n    res &= s_copy_main->ne[0]  == params.ubatch.n_seqs;\n    res &= s_copy_extra->ne[0] == mctx->get_n_rs() - params.ubatch.n_seqs;\n\n    res &= head == mctx->get_head();\n    res &= rs_z == mctx->get_rs_z();\n\n    return res;\n}\n\nvoid llm_graph_input_cross_embd::set_input(const llama_ubatch * ubatch) {\n    GGML_UNUSED(ubatch);\n\n    if (cross_embd && !cross->v_embd.empty()) {\n        assert(cross_embd->type == GGML_TYPE_F32);\n\n        ggml_backend_tensor_set(cross_embd, cross->v_embd.data(), 0, ggml_nbytes(cross_embd));\n    }\n}\n\nstatic void print_mask(const float * data, int64_t n_tokens, int64_t n_kv, int64_t n_swa, llama_swa_type swa_type) {\n    LLAMA_LOG_DEBUG(\"%s: === Attention mask ===\\n\", __func__);\n    const char * swa_type_str = \"unknown\";\n\n    switch (swa_type) {\n        case LLAMA_SWA_TYPE_NONE:      swa_type_str = \"LLAMA_SWA_TYPE_NONE\"; break;\n        case LLAMA_SWA_TYPE_STANDARD:  swa_type_str = \"LLAMA_SWA_TYPE_STANDARD\"; break;\n        case LLAMA_SWA_TYPE_CHUNKED:   swa_type_str = \"LLAMA_SWA_TYPE_CHUNKED\"; break;\n        case LLAMA_SWA_TYPE_SYMMETRIC: swa_type_str = \"LLAMA_SWA_TYPE_SYMMETRIC\"; break;\n    };\n\n    LLAMA_LOG_DEBUG(\"%s: n_swa : %d, n_kv: %d, swq_type: %s\\n\", __func__, (int)n_swa, (int)n_kv, swa_type_str);\n    LLAMA_LOG_DEBUG(\"%s: '0' = can attend, '∞' = masked\\n\", __func__);\n    LLAMA_LOG_DEBUG(\"%s: Rows = query tokens, Columns = key/value tokens\\n\\n\", __func__);\n\n    LLAMA_LOG_DEBUG(\"    \");\n    for (int j = 0; j < std::min((int64_t)20, n_kv); ++j) {\n        LLAMA_LOG_DEBUG(\"%2d\", j);\n    }\n    LLAMA_LOG_DEBUG(\"\\n\");\n\n    for (int i = 0; i < std::min((int64_t)20, n_tokens); ++i) {\n        LLAMA_LOG_DEBUG(\" %2d \", i);\n        for (int j = 0; j < std::min((int64_t)20, n_kv); ++j) {\n            float val = data[i * n_kv + j];\n            if (val == -INFINITY) {\n                LLAMA_LOG_DEBUG(\" ∞\");\n            } else {\n                LLAMA_LOG_DEBUG(\" 0\");\n            }\n        }\n        LLAMA_LOG_DEBUG(\"\\n\");\n    }\n}\n\nvoid llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {\n    const int64_t n_kv     = ubatch->n_tokens;\n    const int64_t n_tokens = ubatch->n_tokens;\n\n    const auto fill_mask = [&](float * data, int n_swa, llama_swa_type swa_type) {\n        for (int i1 = 0; i1 < n_tokens; ++i1) {\n            const llama_seq_id s1 = ubatch->seq_id[i1][0];\n            const llama_pos    p1 = ubatch->pos[i1];\n\n            const uint64_t idst = i1*n_kv;\n\n            for (int i0 = 0; i0 < n_tokens; ++i0) {\n                const llama_seq_id s0 = ubatch->seq_id[i0][0];\n                const llama_pos p0    = ubatch->pos[i0];\n\n                // mask different sequences\n                if (s0 != s1) {\n                    continue;\n                }\n\n                // mask future tokens\n                if (cparams.causal_attn && p0 > p1) {\n                    continue;\n                }\n\n                // apply SWA if any\n                if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) {\n                    continue;\n                }\n\n                data[idst + i0] = hparams.use_alibi ? -std::abs(p0 - p1) : 0.0f;\n            }\n        }\n    };\n\n    {\n        GGML_ASSERT(self_kq_mask);\n        GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));\n\n        float * data = (float *) self_kq_mask->data;\n\n        std::fill(data, data + ggml_nelements(self_kq_mask), -INFINITY);\n\n        fill_mask(data, 0, LLAMA_SWA_TYPE_NONE);\n\n        if (debug) {\n            print_mask(data, n_tokens, n_kv, 0, LLAMA_SWA_TYPE_NONE);\n        }\n    }\n\n    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n        GGML_ASSERT(self_kq_mask_swa);\n        GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer));\n\n        float * data = (float *) self_kq_mask_swa->data;\n\n        std::fill(data, data + ggml_nelements(self_kq_mask_swa), -INFINITY);\n\n        fill_mask(data, hparams.n_swa, hparams.swa_type);\n\n        if (debug) {\n            print_mask(data, n_tokens, n_kv, hparams.n_swa, hparams.swa_type);\n        }\n    }\n}\n\nvoid llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) {\n    mctx->set_input_k_idxs(self_k_idxs, ubatch);\n    mctx->set_input_v_idxs(self_v_idxs, ubatch);\n\n    mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);\n}\n\nbool llm_graph_input_attn_kv::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_kv_cache_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;\n  //res &= self_v_idxs->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n    res &= can_reuse_kq_mask(self_kq_mask, mctx, params.ubatch, params.cparams);\n\n    return res;\n}\n\nvoid llm_graph_input_attn_k::set_input(const llama_ubatch * ubatch) {\n    mctx->set_input_k_idxs(self_k_idxs, ubatch);\n\n    mctx->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);\n}\n\nbool llm_graph_input_attn_k::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_kv_cache_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;\n\n    res &= can_reuse_kq_mask(self_kq_mask, mctx, params.ubatch, params.cparams);\n\n    return res;\n}\n\nvoid llm_graph_input_attn_kv_iswa::set_input(const llama_ubatch * ubatch) {\n    mctx->get_base()->set_input_k_idxs(self_k_idxs, ubatch);\n    mctx->get_base()->set_input_v_idxs(self_v_idxs, ubatch);\n\n    mctx->get_base()->set_input_kq_mask(self_kq_mask, ubatch, cparams.causal_attn);\n\n    mctx->get_swa()->set_input_k_idxs(self_k_idxs_swa, ubatch);\n    mctx->get_swa()->set_input_v_idxs(self_v_idxs_swa, ubatch);\n\n    mctx->get_swa()->set_input_kq_mask(self_kq_mask_swa, ubatch, cparams.causal_attn);\n}\n\nbool llm_graph_input_attn_kv_iswa::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_kv_cache_iswa_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= self_k_idxs->ne[0] == params.ubatch.n_tokens;\n  //res &= self_v_idxs->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n    res &= self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;\n  //res &= self_v_idxs_swa->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n    res &= can_reuse_kq_mask(self_kq_mask,     mctx->get_base(), params.ubatch, params.cparams);\n    res &= can_reuse_kq_mask(self_kq_mask_swa, mctx->get_swa(),  params.ubatch, params.cparams);\n\n    return res;\n}\n\nvoid llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) {\n    GGML_ASSERT(cross_kq_mask);\n\n    const int64_t n_enc    = cross_kq_mask->ne[0];\n    const int64_t n_tokens = ubatch->n_tokens;\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(cross_kq_mask->buffer));\n    GGML_ASSERT(!ubatch->equal_seqs()); // TODO: use ubatch->n_seqs instead of failing\n\n    float * data = (float *) cross_kq_mask->data;\n\n    for (int i = 0; i < n_tokens; ++i) {\n        GGML_ASSERT(!cross->seq_ids_enc.empty() && \"llama_encode must be called first\");\n        for (int j = 0; j < n_enc; ++j) {\n            float f = -INFINITY;\n\n            for (int s = 0; s < ubatch->n_seq_id[i]; ++s) {\n                const llama_seq_id seq_id = ubatch->seq_id[i][s];\n\n                if (cross->seq_ids_enc[j].find(seq_id) != cross->seq_ids_enc[j].end()) {\n                    f = 0.0f;\n                }\n            }\n\n            data[i*n_enc + j] = f;\n        }\n    }\n}\n\nvoid llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) {\n    mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);\n    mctx->get_attn()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch);\n\n    mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);\n\n    const int64_t n_rs = mctx->get_recr()->get_n_rs();\n\n    if (inp_rs->s_copy) {\n        GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer));\n        int32_t * data = (int32_t *) inp_rs->s_copy->data;\n\n        // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n\n        for (uint32_t i = 0; i < n_rs; ++i) {\n            data[i] = mctx->get_recr()->s_copy(i);\n        }\n    }\n}\n\nbool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_memory_hybrid_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;\n  //res &= inp_attn->self_v_idxs->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n    res &= can_reuse_kq_mask(inp_attn->self_kq_mask, mctx->get_attn(), params.ubatch, params.cparams);\n\n    res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();\n\n    res &= inp_rs->s_copy_main->ne[0]  == params.ubatch.n_seqs;\n    res &= inp_rs->s_copy_extra->ne[0] == mctx->get_recr()->get_n_rs() - params.ubatch.n_seqs;\n\n    res &= inp_rs->head == mctx->get_recr()->get_head();\n    res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z();\n\n    return res;\n}\n\n// TODO: Hybrid input classes are a bit redundant.\n// Instead of creating a hybrid input, the graph can simply create 2 separate inputs.\n// Refactoring is required in the future.\nvoid llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) {\n    mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);\n\n    mctx->get_attn()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);\n\n    const int64_t n_rs = mctx->get_recr()->get_n_rs();\n\n    if (inp_rs->s_copy) {\n        GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer));\n        int32_t * data = (int32_t *) inp_rs->s_copy->data;\n\n        // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n\n        for (uint32_t i = 0; i < n_rs; ++i) {\n            data[i] = mctx->get_recr()->s_copy(i);\n        }\n    }\n}\n\nbool llm_graph_input_mem_hybrid_k::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_memory_hybrid_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;\n\n    res &= can_reuse_kq_mask(inp_attn->self_kq_mask, mctx->get_attn(), params.ubatch, params.cparams);\n\n    res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();\n\n    res &= inp_rs->s_copy_main->ne[0]  == params.ubatch.n_seqs;\n    res &= inp_rs->s_copy_extra->ne[0] == mctx->get_recr()->get_n_rs() - params.ubatch.n_seqs;\n\n    res &= inp_rs->head == mctx->get_recr()->get_head();\n    res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z();\n\n    return res;\n}\n\nvoid llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) {\n    const auto * attn_ctx = mctx->get_attn();\n\n    // base tensors may not be allocated if there are no non-SWA attention layers\n    if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {\n        attn_ctx->get_base()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);\n        attn_ctx->get_base()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch);\n\n        attn_ctx->get_base()->set_input_kq_mask(inp_attn->self_kq_mask, ubatch, cparams.causal_attn);\n    }\n\n    // swa tensors may not be allocated if there are no SWA attention layers\n    if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {\n        attn_ctx->get_swa()->set_input_k_idxs(inp_attn->self_k_idxs_swa, ubatch);\n        attn_ctx->get_swa()->set_input_v_idxs(inp_attn->self_v_idxs_swa, ubatch);\n\n        attn_ctx->get_swa()->set_input_kq_mask(inp_attn->self_kq_mask_swa, ubatch, cparams.causal_attn);\n    }\n\n    const int64_t n_rs = mctx->get_recr()->get_n_rs();\n\n    if (inp_rs->s_copy) {\n        GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer));\n        int32_t * data = (int32_t *) inp_rs->s_copy->data;\n\n        // assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n\n        for (uint32_t i = 0; i < n_rs; ++i) {\n            data[i] = mctx->get_recr()->s_copy(i);\n        }\n    }\n}\n\nbool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params) {\n    const auto * mctx = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);\n\n    this->mctx = mctx;\n\n    bool res = true;\n\n    const auto * attn_ctx = mctx->get_attn();\n\n    // base tensors may not be allocated if there are no non-SWA attention layers\n    if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {\n        res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;\n      //res &= inp_attn->self_v_idxs->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n        res &= can_reuse_kq_mask(inp_attn->self_kq_mask, attn_ctx->get_base(), params.ubatch, params.cparams);\n    }\n\n    // swa tensors may not be allocated if there are no SWA attention layers\n    if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {\n        res &= inp_attn->self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;\n      //res &= inp_attn->self_v_idxs_swa->ne[0] == params.ubatch.n_tokens; // TODO: need to move this to the unified cache and check there\n\n        res &= can_reuse_kq_mask(inp_attn->self_kq_mask_swa, attn_ctx->get_swa(), params.ubatch, params.cparams);\n    }\n\n    res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();\n\n    res &= inp_rs->s_copy_main->ne[0]  == params.ubatch.n_seqs;\n    res &= inp_rs->s_copy_extra->ne[0] == mctx->get_recr()->get_n_rs() - params.ubatch.n_seqs;\n\n    res &= inp_rs->head == mctx->get_recr()->get_head();\n    res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z();\n\n    return res;\n}\n\nvoid llm_graph_input_sampling::set_input(const llama_ubatch * ubatch) {\n    // set the inputs only for the active samplers in the current ubatch\n    std::unordered_set<llama_seq_id> active_samplers;\n    for (uint32_t i = 0; i < ubatch->n_tokens; i++) {\n        if (ubatch->output[i]) {\n            llama_seq_id seq_id = ubatch->seq_id[i][0];\n            active_samplers.insert(seq_id);\n        }\n    }\n\n    for (auto seq_id : active_samplers) {\n        if (samplers.find(seq_id) == samplers.end()) {\n            continue;\n        }\n\n        auto & sampler = samplers[seq_id];\n\n        if (sampler->iface->backend_set_input) {\n            sampler->iface->backend_set_input(sampler);\n        }\n    }\n}\n\nbool llm_graph_input_sampling::can_reuse(const llm_graph_params & params) {\n    if (samplers.size() != params.samplers.size()) {\n        return false;\n    }\n\n    for (const auto & [seq_id, sampler] : params.samplers) {\n        if (samplers[seq_id] != sampler) {\n            return false;\n        }\n    }\n\n    return true;\n}\n\n//\n// llm_graph_result\n//\n\nllm_graph_result::llm_graph_result(int64_t max_nodes) : max_nodes(max_nodes) {\n    reset();\n\n    const char * LLAMA_GRAPH_RESULT_DEBUG = getenv(\"LLAMA_GRAPH_RESULT_DEBUG\");\n    debug = LLAMA_GRAPH_RESULT_DEBUG ? atoi(LLAMA_GRAPH_RESULT_DEBUG) : 0;\n}\n\nint64_t llm_graph_result::get_max_nodes() const {\n    return max_nodes;\n}\n\nvoid llm_graph_result::reset() {\n    t_inp_tokens  = nullptr;\n    t_inp_embd    = nullptr;\n    t_logits      = nullptr;\n    t_embd        = nullptr;\n    t_embd_pooled = nullptr;\n    t_sampled.clear();\n    t_sampled_probs.clear();\n    t_sampled_logits.clear();\n    t_candidates.clear();\n\n    params = {};\n\n    inputs.clear();\n\n    buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));\n\n    ggml_init_params params = {\n        /*.mem_size   =*/ buf_compute_meta.size(),\n        /*.mem_buffer =*/ buf_compute_meta.data(),\n        /*.no_alloc   =*/ true,\n    };\n\n    ctx_compute.reset(ggml_init(params));\n\n    gf = ggml_new_graph_custom(ctx_compute.get(), max_nodes, false);\n}\n\nvoid llm_graph_result::set_inputs(const llama_ubatch * ubatch) {\n    for (auto & input : inputs) {\n        input->set_input(ubatch);\n    }\n}\n\nvoid llm_graph_result::set_outputs() {\n    if (t_logits != nullptr) {\n        ggml_set_output(t_logits);\n    }\n    if (t_embd != nullptr) {\n        ggml_set_output(t_embd);\n    }\n    if (t_embd_pooled != nullptr) {\n        ggml_set_output(t_embd_pooled);\n    }\n    for (auto & [seq_id, t] : t_sampled) {\n        if (t != nullptr) {\n            ggml_set_output(t);\n        }\n    }\n    for (auto & [seq_id, t] : t_sampled_probs) {\n        if (t != nullptr) {\n            ggml_set_output(t);\n        }\n    }\n    for (auto & [seq_id, t] : t_sampled_logits) {\n        if (t != nullptr) {\n            ggml_set_output(t);\n        }\n    }\n    for (auto & [seq_id, t] : t_candidates) {\n        if (t != nullptr) {\n            ggml_set_output(t);\n        }\n    }\n}\n\nbool llm_graph_result::can_reuse(const llm_graph_params & params) {\n    if (!this->params.allow_reuse(params)) {\n        if (debug > 1) {\n            LLAMA_LOG_DEBUG(\"%s: cannot reuse graph due to incompatible graph parameters\\n\", __func__);\n        }\n\n        return false;\n    }\n\n    if (debug > 1) {\n        LLAMA_LOG_DEBUG(\"%s: checking compatibility of %d inputs:\\n\", __func__, (int) inputs.size());\n    }\n\n    bool res = true;\n\n    for (auto & input : inputs) {\n        const bool cur = input->can_reuse(params);\n\n        if (debug > 1) {\n            LLAMA_LOG_DEBUG(\"%s: can_reuse = %d\\n\", \"placeholder\", cur);\n        }\n\n        res = res && cur;\n    }\n\n    if (debug > 0) {\n        LLAMA_LOG_DEBUG(\"%s: can reuse graph = %d\\n\", __func__, res);\n    }\n\n    return res;\n}\n\nllm_graph_input_i * llm_graph_result::add_input(llm_graph_input_ptr input) {\n    inputs.emplace_back(std::move(input));\n    return inputs.back().get();\n}\n\nvoid llm_graph_result::set_params(const llm_graph_params & params) {\n    this->params = params;\n}\n\n//\n// llm_graph_context\n//\n\nllm_graph_context::llm_graph_context(const llm_graph_params & params) :\n    arch             (params.arch),\n    hparams          (params.hparams),\n    cparams          (params.cparams),\n    ubatch           (params.ubatch),\n    n_embd           (hparams.n_embd),\n    n_layer          (hparams.n_layer),\n    n_rot            (hparams.n_rot()),\n    n_ctx            (cparams.n_ctx),\n    n_head           (hparams.n_head()),\n    n_head_kv        (hparams.n_head_kv()),\n    n_embd_head_k    (hparams.n_embd_head_k()),\n    n_embd_k_gqa     (hparams.n_embd_k_gqa()),\n    n_embd_head_v    (hparams.n_embd_head_v()),\n    n_embd_v_gqa     (hparams.n_embd_v_gqa()),\n    n_expert         (hparams.n_expert),\n    n_expert_used    (cparams.warmup ? hparams.n_expert : hparams.n_expert_used),\n    freq_base        (cparams.rope_freq_base),\n    freq_scale       (cparams.rope_freq_scale),\n    ext_factor       (cparams.yarn_ext_factor),\n    attn_factor      (cparams.yarn_attn_factor),\n    beta_fast        (cparams.yarn_beta_fast),\n    beta_slow        (cparams.yarn_beta_slow),\n    norm_eps         (hparams.f_norm_eps),\n    norm_rms_eps     (hparams.f_norm_rms_eps),\n    n_tokens         (ubatch.n_tokens),\n    n_outputs        (params.n_outputs),\n    n_ctx_orig       (cparams.n_ctx_orig_yarn),\n    pooling_type     (cparams.pooling_type),\n    rope_type        (hparams.rope_type),\n    sched            (params.sched),\n    backend_cpu      (params.backend_cpu),\n    cvec             (params.cvec),\n    loras            (params.loras),\n    mctx             (params.mctx),\n    cross            (params.cross),\n    samplers         (params.samplers),\n    cb_func          (params.cb),\n    res              (params.res),\n    ctx0             (res->get_ctx()),\n    gf               (res->get_gf()) {\n        res->set_params(params);\n    }\n\nvoid llm_graph_context::cb(ggml_tensor * cur, const char * name, int il) const {\n    if (cb_func) {\n        cb_func(ubatch, cur, name, il);\n    }\n}\n\nggml_tensor * llm_graph_context::build_cvec(\n         ggml_tensor * cur,\n                 int   il) const {\n    return cvec->apply_to(ctx0, cur, il);\n}\n\nggml_tensor * llm_graph_context::build_lora_mm(\n          ggml_tensor * w,\n          ggml_tensor * cur,\n          ggml_tensor * w_s) const {\n    ggml_tensor * res = ggml_mul_mat(ctx0, w, cur);\n\n    for (const auto & lora : *loras) {\n        llama_adapter_lora_weight * lw = lora.first->get_weight(w);\n        if (lw == nullptr) {\n            continue;\n        }\n\n        const float adapter_scale = lora.second;\n        const float scale = lw->get_scale(lora.first->alpha, adapter_scale);\n\n        ggml_tensor * ab_cur = ggml_mul_mat(\n                ctx0, lw->b,\n                ggml_mul_mat(ctx0, lw->a, cur)\n                );\n\n        ab_cur = ggml_scale(ctx0, ab_cur, scale);\n        res = ggml_add(ctx0, res, ab_cur);\n    }\n\n    if (w_s) {\n        res = ggml_mul(ctx0, res, w_s);\n    }\n\n    return res;\n}\n\nggml_tensor * llm_graph_context::build_lora_mm_id(\n          ggml_tensor * w,   // ggml_tensor * as\n          ggml_tensor * cur, // ggml_tensor * b\n          ggml_tensor * ids) const {\n    ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids);\n    for (const auto & lora : *loras) {\n        llama_adapter_lora_weight * lw = lora.first->get_weight(w);\n        if (lw == nullptr) {\n            continue;\n        }\n\n        const float alpha = lora.first->alpha;\n        const float rank  = (float) lw->b->ne[0];\n        const float scale = alpha ? lora.second * alpha / rank : lora.second;\n\n        ggml_tensor * ab_cur = ggml_mul_mat_id(\n                ctx0, lw->b,\n                ggml_mul_mat_id(ctx0, lw->a, cur, ids),\n                ids\n                );\n\n        ab_cur = ggml_scale(ctx0, ab_cur, scale);\n        res = ggml_add(ctx0, res, ab_cur);\n    }\n\n    return res;\n}\n\nggml_tensor * llm_graph_context::build_norm(\n         ggml_tensor * cur,\n         ggml_tensor * mw,\n         ggml_tensor * mb,\n       llm_norm_type   type,\n                 int   il) const {\n    switch (type) {\n        case LLM_NORM:       cur = ggml_norm    (ctx0, cur, hparams.f_norm_eps);     break;\n        case LLM_NORM_RMS:   cur = ggml_rms_norm(ctx0, cur, hparams.f_norm_rms_eps); break;\n        case LLM_NORM_GROUP:\n            {\n                cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], 1, cur->ne[1]);\n                cur = ggml_group_norm(ctx0, cur, hparams.n_norm_groups, hparams.f_norm_group_eps);\n                cur = ggml_reshape_2d(ctx0, cur, cur->ne[0],    cur->ne[2]);\n            } break;\n    }\n\n    if (mw || mb) {\n        cb(cur, \"norm\", il);\n    }\n\n    if (mw) {\n        cur = ggml_mul(ctx0, cur, mw);\n        if (mb) {\n            cb(cur, \"norm_w\", il);\n        }\n    }\n\n    if (mb) {\n        cur = ggml_add(ctx0, cur, mb);\n    }\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_ffn(\n         ggml_tensor * cur,\n         ggml_tensor * up,\n         ggml_tensor * up_b,\n         ggml_tensor * up_s,\n         ggml_tensor * gate,\n         ggml_tensor * gate_b,\n         ggml_tensor * gate_s,\n         ggml_tensor * down,\n         ggml_tensor * down_b,\n         ggml_tensor * down_s,\n         ggml_tensor * act_scales,\n     llm_ffn_op_type   type_op,\n   llm_ffn_gate_type   type_gate,\n                 int   il) const {\n    ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur;\n    cb(tmp, \"ffn_up\", il);\n\n    if (up_b) {\n        tmp = ggml_add(ctx0, tmp, up_b);\n        cb(tmp, \"ffn_up_b\", il);\n    }\n\n    if (up_s) {\n        tmp = ggml_mul(ctx0, tmp, up_s);\n        cb(tmp, \"ffn_up_s\", il);\n    }\n\n    if (gate) {\n        switch (type_gate) {\n            case LLM_FFN_SEQ:\n                {\n                    cur = build_lora_mm(gate, tmp);\n                    cb(cur, \"ffn_gate\", il);\n                } break;\n            case LLM_FFN_PAR:\n                {\n                    cur = build_lora_mm(gate, cur);\n                    cb(cur, \"ffn_gate\", il);\n                } break;\n        }\n\n        if (gate_b) {\n            cur = ggml_add(ctx0, cur, gate_b);\n            cb(cur, \"ffn_gate_b\", il);\n        }\n\n        if (gate_s) {\n            cur = ggml_mul(ctx0, cur, gate_s);\n            cb(cur, \"ffn_gate_s\", il);\n        }\n\n    } else {\n        cur = tmp;\n    }\n\n    switch (type_op) {\n        case LLM_FFN_SILU:\n            if (gate && type_gate == LLM_FFN_PAR) {\n                // Step35: HF clamps gate (after SiLU) and up before multiplication\n                if (arch == LLM_ARCH_STEP35 && il >= 0) {\n                    const float limit = hparams.swiglu_clamp_shexp[il];\n                    constexpr float eps = 1e-6f;\n                    if (limit > eps) {\n                        ggml_tensor * gate_act = ggml_silu(ctx0, cur);\n                        cb(gate_act, \"ffn_silu\", il);\n                        gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);\n                        cb(gate_act, \"ffn_silu_clamped\", il);\n\n                        tmp = ggml_clamp(ctx0, tmp, -limit, limit);\n                        cb(tmp, \"ffn_up_clamped\", il);\n\n                        cur = ggml_mul(ctx0, gate_act, tmp);\n                        cb(cur, \"ffn_swiglu_limited\", il);\n                        type_gate = LLM_FFN_SEQ;\n                        break;\n                    }\n                }\n\n                cur = ggml_swiglu_split(ctx0, cur, tmp);\n                cb(cur, \"ffn_swiglu\", il);\n                type_gate = LLM_FFN_SEQ;\n            } else {\n                cur = ggml_silu(ctx0, cur);\n                cb(cur, \"ffn_silu\", il);\n            } break;\n        case LLM_FFN_GELU:\n            if (gate && type_gate == LLM_FFN_PAR) {\n                cur = ggml_geglu_split(ctx0, cur, tmp);\n                cb(cur, \"ffn_geglu\", il);\n                type_gate = LLM_FFN_SEQ;\n            } else {\n                cur = ggml_gelu(ctx0, cur);\n                cb(cur, \"ffn_gelu\", il);\n                if (act_scales != NULL) {\n                    cur = ggml_div(ctx0, cur, act_scales);\n                    cb(cur, \"ffn_act\", il);\n                }\n            } break;\n        case LLM_FFN_RELU:\n            if (gate && type_gate == LLM_FFN_PAR) {\n                cur = ggml_reglu_split(ctx0, cur, tmp);\n                cb(cur, \"ffn_reglu\", il);\n                type_gate = LLM_FFN_SEQ;\n            } else {\n                cur = ggml_relu(ctx0, cur);\n                cb(cur, \"ffn_relu\", il);\n            } break;\n        case LLM_FFN_RELU_SQR:\n            {\n                cur = ggml_relu(ctx0, cur);\n                cb(cur, \"ffn_relu\", il);\n\n                cur = ggml_sqr(ctx0, cur);\n                cb(cur, \"ffn_sqr(relu)\", il);\n            } break;\n        case LLM_FFN_SWIGLU:\n            {\n                cur = ggml_swiglu(ctx0, cur);\n                cb(cur, \"ffn_swiglu\", il);\n            } break;\n        case LLM_FFN_GEGLU:\n            {\n                cur = ggml_geglu(ctx0, cur);\n                cb(cur, \"ffn_geglu\", il);\n            } break;\n        case LLM_FFN_REGLU:\n            {\n                cur = ggml_reglu(ctx0, cur);\n                cb(cur, \"ffn_reglu\", il);\n            } break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n\n    if (gate && type_gate == LLM_FFN_PAR) {\n        cur = ggml_mul(ctx0, cur, tmp);\n        cb(cur, \"ffn_gate_par\", il);\n    }\n\n    if (down) {\n        cur = build_lora_mm(down, cur);\n        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {\n            // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators\n            ggml_mul_mat_set_prec(cur, GGML_PREC_F32);\n        }\n    }\n\n    if (down_b) {\n        cb(cur, \"ffn_down\", il);\n    }\n\n    if (down_b) {\n        cur = ggml_add(ctx0, cur, down_b);\n    }\n\n    if (down_s) {\n        cur = ggml_mul(ctx0, cur, down_s);\n        cb(cur, \"ffn_down_s\", il);\n    }\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_moe_ffn(\n         ggml_tensor * cur,\n         ggml_tensor * gate_inp,\n         ggml_tensor * up_exps,\n         ggml_tensor * gate_exps,\n         ggml_tensor * down_exps,\n         ggml_tensor * exp_probs_b,\n             int64_t   n_expert,\n             int64_t   n_expert_used,\n     llm_ffn_op_type   type_op,\n                bool   norm_w,\n               float   w_scale,\n         llama_expert_gating_func_type gating_op,\n                 int   il,\n         ggml_tensor * probs_in,\n         ggml_tensor * gate_up_exps,\n         ggml_tensor * up_exps_s,\n         ggml_tensor * gate_exps_s,\n         ggml_tensor * down_exps_s) const {\n    return build_moe_ffn(\n        cur,\n        gate_inp,  /* gate_inp_b  */ nullptr,\n        up_exps,   /* up_exps_b   */ nullptr,\n        gate_exps, /* gate_exps_b */ nullptr,\n        down_exps, /* down_exps_b */ nullptr,\n        exp_probs_b,\n        n_expert,\n        n_expert_used,\n        type_op,\n        norm_w,\n        w_scale,\n        gating_op,\n        il,\n        probs_in,\n        gate_up_exps,\n        /* gate_up_exps_b */ nullptr,\n        up_exps_s,\n        gate_exps_s,\n        down_exps_s\n    );\n}\n\nggml_tensor * llm_graph_context::build_moe_ffn(\n         ggml_tensor * cur,\n         ggml_tensor * gate_inp,\n         ggml_tensor * gate_inp_b,\n         ggml_tensor * up_exps,\n         ggml_tensor * up_exps_b,\n         ggml_tensor * gate_exps,\n         ggml_tensor * gate_exps_b,\n         ggml_tensor * down_exps,\n         ggml_tensor * down_exps_b,\n         ggml_tensor * exp_probs_b,\n             int64_t   n_expert,\n             int64_t   n_expert_used,\n     llm_ffn_op_type   type_op,\n                bool   norm_w,\n               float   w_scale,\n        llama_expert_gating_func_type gating_op,\n                 int   il,\n         ggml_tensor * probs_in,\n         ggml_tensor * gate_up_exps,\n         ggml_tensor * gate_up_exps_b,\n         ggml_tensor * up_exps_s,\n         ggml_tensor * gate_exps_s,\n         ggml_tensor * down_exps_s) const {\n    const int64_t n_embd   = cur->ne[0];\n    const int64_t n_tokens = cur->ne[1];\n    const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN\n\n    ggml_tensor * logits = nullptr;\n\n    if (probs_in == nullptr) {\n        logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]\n        cb(logits, \"ffn_moe_logits\", il);\n    } else {\n        logits = probs_in;\n    }\n\n    if (gate_inp_b) {\n        logits = ggml_add(ctx0, logits, gate_inp_b);\n        cb(logits, \"ffn_moe_logits_biased\", il);\n    }\n\n    ggml_tensor * probs = nullptr;\n    switch (gating_op) {\n        case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX:\n            {\n                probs = ggml_soft_max(ctx0, logits); // [n_expert, n_tokens]\n            } break;\n        case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID:\n            {\n                probs = ggml_sigmoid(ctx0, logits); // [n_expert, n_tokens]\n            } break;\n        case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT:\n            {\n                probs = logits; // [n_expert, n_tokens]\n            } break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n    cb(probs, \"ffn_moe_probs\", il);\n\n    // add experts selection bias - introduced in DeepSeek V3\n    // leave probs unbiased as it's later used to get expert weights\n    ggml_tensor * selection_probs = probs;\n    if (exp_probs_b != nullptr) {\n        selection_probs = ggml_add(ctx0, probs, exp_probs_b);\n        cb(selection_probs, \"ffn_moe_probs_biased\", il);\n    }\n\n    // llama4 doesn't have exp_probs_b, and sigmoid is only used after top_k\n    // see: https://github.com/meta-llama/llama-models/blob/699a02993512fb36936b1b0741e13c06790bcf98/models/llama4/moe.py#L183-L198\n    if (arch == LLM_ARCH_LLAMA4) {\n        selection_probs = logits;\n    }\n\n    if (arch == LLM_ARCH_GROVEMOE) {\n        selection_probs = ggml_sigmoid(ctx0, logits); // [n_expert, n_tokens]\n        cb(selection_probs, \"ffn_moe_probs_biased\", il);\n    }\n\n    // select top n_group_used expert groups\n    // https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/e815299b0bcbac849fa540c768ef21845365c9eb/modeling_deepseek.py#L440-L457\n    if (hparams.n_expert_groups > 1 && n_tokens > 0) {\n        const int64_t n_exp_per_group = n_expert / hparams.n_expert_groups;\n\n        // organize experts into n_expert_groups\n        ggml_tensor * selection_groups = ggml_reshape_3d(ctx0, selection_probs, n_exp_per_group, hparams.n_expert_groups, n_tokens); // [n_exp_per_group, n_expert_groups, n_tokens]\n\n        ggml_tensor * group_scores = ggml_argsort_top_k(ctx0, selection_groups, 2); // [2, n_expert_groups, n_tokens]\n        group_scores = ggml_get_rows(ctx0, ggml_reshape_4d(ctx0, selection_groups, 1, selection_groups->ne[0], selection_groups->ne[1], selection_groups->ne[2]), group_scores); // [1, 2, n_expert_groups, n_tokens]\n\n        // get top n_group_used expert groups\n        group_scores = ggml_sum_rows(ctx0, ggml_reshape_3d(ctx0, group_scores, group_scores->ne[1], group_scores->ne[2], group_scores->ne[3])); // [1, n_expert_groups, n_tokens]\n        group_scores = ggml_reshape_2d(ctx0, group_scores, group_scores->ne[1], group_scores->ne[2]); // [n_expert_groups, n_tokens]\n\n        ggml_tensor * expert_groups = ggml_argsort_top_k(ctx0, group_scores, hparams.n_group_used); // [n_group_used, n_tokens]\n        cb(expert_groups, \"ffn_moe_group_topk\", il);\n\n        // mask out the other groups\n        selection_probs = ggml_get_rows(ctx0, selection_groups, expert_groups); // [n_exp_per_group, n_group_used, n_tokens]\n        selection_probs = ggml_set_rows(ctx0, ggml_fill(ctx0, selection_groups, -INFINITY), selection_probs, expert_groups); // [n_exp_per_group, n_expert_groups, n_tokens]\n        selection_probs = ggml_reshape_2d(ctx0, selection_probs, n_expert, n_tokens); // [n_expert, n_tokens]\n        cb(selection_probs, \"ffn_moe_probs_masked\", il);\n    }\n\n    // select experts\n    ggml_tensor * selected_experts = ggml_argsort_top_k(ctx0, selection_probs, n_expert_used); // [n_expert_used, n_tokens]\n    cb(selected_experts->src[0], \"ffn_moe_argsort\", il);\n    cb(selected_experts, \"ffn_moe_topk\", il);\n\n    if (arch == LLM_ARCH_GROVEMOE && n_expert != hparams.n_expert) {\n        // TODO: Use scalar div instead when/if implemented\n        ggml_tensor * f_sel = ggml_cast(ctx0, selected_experts, GGML_TYPE_F32);\n        selected_experts = ggml_cast(ctx0, ggml_scale(ctx0, f_sel, 1.0f / float(hparams.n_group_experts)), GGML_TYPE_I32);\n        probs = ggml_reshape_3d(ctx0, probs, 1, hparams.n_expert, n_tokens);\n    } else {\n        probs = ggml_reshape_3d(ctx0, probs, 1, n_expert, n_tokens);\n    }\n\n    ggml_tensor * weights = ggml_get_rows(ctx0, probs, selected_experts); // [1, n_expert_used, n_tokens]\n    cb(weights, \"ffn_moe_weights\", il);\n\n\n    if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT) {\n        weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);\n        weights = ggml_soft_max(ctx0, weights); // [n_expert_used, n_tokens]\n        weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);\n        cb(weights, \"ffn_moe_weights_softmax\", il);\n    }\n\n    if (norm_w) {\n        weights = ggml_reshape_2d(ctx0, weights, n_expert_used, n_tokens);\n\n        ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens]\n        cb(weights_sum, \"ffn_moe_weights_sum\", il);\n\n        // Avoid division by zero, clamp to smallest number representable by F16\n        weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY);\n        cb(weights_sum, \"ffn_moe_weights_sum_clamped\", il);\n\n        weights = ggml_div(ctx0, weights, weights_sum); // [n_expert_used, n_tokens]\n        cb(weights, \"ffn_moe_weights_norm\", il);\n\n        weights = ggml_reshape_3d(ctx0, weights, 1, n_expert_used, n_tokens);\n    }\n    if (w_scale != 0.0f && w_scale != 1.0f) {\n        weights = ggml_scale(ctx0, weights, w_scale);\n        cb(weights, \"ffn_moe_weights_scaled\", il);\n    }\n\n    //call early so that topk-moe can be used\n    ggml_build_forward_expand(gf, weights);\n\n    cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);\n\n    if (weight_before_ffn) {\n        // repeat cur to [n_embd, n_expert_used, n_tokens]\n        ggml_tensor * repeated = ggml_repeat_4d(ctx0, cur, n_embd, n_expert_used, n_tokens, 1);\n        cur = ggml_mul(ctx0, repeated, weights);\n        cb(cur, \"ffn_moe_weighted\", il);\n    }\n\n    ggml_tensor * up = nullptr;\n    ggml_tensor * experts = nullptr;\n\n    if (gate_up_exps) {\n        // merged gate_up path: one mul_mat_id, then split into gate and up views\n        ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts); // [n_ff*2, n_expert_used, n_tokens]\n        cb(gate_up, \"ffn_moe_gate_up\", il);\n\n        if (gate_up_exps_b) {\n            gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts);\n            cb(gate_up, \"ffn_moe_gate_up_biased\", il);\n        }\n\n        // apply per-expert scale2 to merged gate_up (use up_exps_s since gate and up are fused)\n        if (up_exps_s) {\n            ggml_tensor * s = ggml_reshape_3d(ctx0, up_exps_s, 1, n_expert, 1);\n            s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1);\n            s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens]\n            gate_up = ggml_mul(ctx0, gate_up, s);\n            cb(gate_up, \"ffn_moe_gate_up_scaled\", il);\n        }\n\n        const int64_t n_ff = gate_up->ne[0] / 2;\n        cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0);\n        cb(cur, \"ffn_moe_gate\", il);\n        up  = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);\n        cb(up, \"ffn_moe_up\", il);\n    } else {\n        // separate gate and up path\n        up = build_lora_mm_id(up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]\n        cb(up, \"ffn_moe_up\", il);\n\n        if (up_exps_b) {\n            up = ggml_add_id(ctx0, up, up_exps_b, selected_experts);\n            cb(up, \"ffn_moe_up_biased\", il);\n        }\n\n        // apply per-expert scale2 to up\n        if (up_exps_s) {\n            ggml_tensor * s = ggml_reshape_3d(ctx0, up_exps_s, 1, n_expert, 1);\n            s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1);\n            s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens]\n            up = ggml_mul(ctx0, up, s);\n            cb(up, \"ffn_moe_up_scaled\", il);\n        }\n\n        if (gate_exps) {\n            cur = build_lora_mm_id(gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]\n            cb(cur, \"ffn_moe_gate\", il);\n        } else {\n            cur = up;\n        }\n\n        if (gate_exps_b) {\n            cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts);\n            cb(cur, \"ffn_moe_gate_biased\", il);\n        }\n\n        // apply per-expert scale2 to gate\n        if (gate_exps_s) {\n            ggml_tensor * s = ggml_reshape_3d(ctx0, gate_exps_s, 1, n_expert, 1);\n            s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1);\n            s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens]\n            cur = ggml_mul(ctx0, cur, s);\n            cb(cur, \"ffn_moe_gate_scaled\", il);\n        }\n    }\n\n    const bool has_gate = gate_exps || gate_up_exps;\n\n    switch (type_op) {\n        case LLM_FFN_SILU:\n            if (gate_exps) {\n                // Step35: per-layer clamp for routed experts\n                if (arch == LLM_ARCH_STEP35 && il >= 0) {\n                    const float limit = hparams.swiglu_clamp_exp[il];\n                    constexpr float eps = 1e-6f;\n                    if (limit > eps) {\n                        ggml_tensor * gate_act = ggml_silu(ctx0, cur);\n                        cb(gate_act, \"ffn_moe_silu\", il);\n                        gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);\n                        cb(gate_act, \"ffn_moe_silu_clamped\", il);\n\n                        up = ggml_clamp(ctx0, up, -limit, limit);\n                        cb(up, \"ffn_moe_up_clamped\", il);\n\n                        cur = ggml_mul(ctx0, gate_act, up);\n                        cb(cur, \"ffn_moe_swiglu_limited\", il);\n                        break;\n                    }\n                }\n            }\n\n            if (has_gate) {\n                cur = ggml_swiglu_split(ctx0, cur, up);\n                cb(cur, \"ffn_moe_swiglu\", il);\n            } else {\n                cur = ggml_silu(ctx0, cur);\n                cb(cur, \"ffn_moe_silu\", il);\n            } break;\n        case LLM_FFN_GELU:\n            if (has_gate) {\n                cur = ggml_geglu_split(ctx0, cur, up);\n                cb(cur, \"ffn_moe_geglu\", il);\n            } else {\n                cur = ggml_gelu(ctx0, cur);\n                cb(cur, \"ffn_moe_gelu\", il);\n            } break;\n        case LLM_FFN_SWIGLU_OAI_MOE:\n            {\n                // TODO: move to hparams?\n                constexpr float alpha = 1.702f;\n                constexpr float limit = 7.0f;\n                cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit);\n                cb(cur, \"ffn_moe_swiglu_oai\", il);\n            } break;\n        case LLM_FFN_RELU:\n            if (has_gate) {\n                cur = ggml_reglu_split(ctx0, cur, up);\n                cb(cur, \"ffn_moe_reglu\", il);\n            } else {\n                cur = ggml_relu(ctx0, cur);\n                cb(cur, \"ffn_moe_relu\", il);\n            } break;\n        case LLM_FFN_RELU_SQR:\n            if (has_gate) {\n                // TODO: add support for gated squared relu\n                GGML_ABORT(\"fatal error: gated squared relu not implemented\");\n            } else {\n                cur = ggml_relu(ctx0, cur);\n                cur = ggml_sqr(ctx0, cur);\n                cb(cur, \"ffn_moe_relu_sqr\", il);\n            } break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n\n    experts = build_lora_mm_id(down_exps, cur, selected_experts); // [n_embd, n_expert_used, n_tokens]\n    cb(experts, \"ffn_moe_down\", il);\n\n    if (down_exps_b) {\n        experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts);\n        cb(experts, \"ffn_moe_down_biased\", il);\n    }\n\n    // apply per-expert scale2 to down\n    if (down_exps_s) {\n        ggml_tensor * s = ggml_reshape_3d(ctx0, down_exps_s, 1, n_expert, 1);\n        s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1);\n        s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens]\n        experts = ggml_mul(ctx0, experts, s);\n        cb(experts, \"ffn_moe_down_scaled\", il);\n    }\n\n    if (!weight_before_ffn) {\n        experts = ggml_mul(ctx0, experts, weights);\n        cb(cur, \"ffn_moe_weighted\", il);\n    }\n\n    ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };\n\n    assert(n_expert_used > 0);\n\n    // order the views before the adds\n    for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {\n        cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);\n\n        ggml_build_forward_expand(gf, cur_experts[i]);\n    }\n\n    // aggregate experts\n    // note: here we explicitly use hparams.n_expert_used instead of n_expert_used\n    //       to avoid potentially a large number of add nodes during warmup\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/14753\n    ggml_tensor * moe_out = cur_experts[0];\n\n    for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {\n        moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);\n    }\n\n    if (hparams.n_expert_used == 1) {\n        // avoid returning a non-contiguous tensor\n        moe_out = ggml_cont(ctx0, moe_out);\n    }\n\n    cb(moe_out, \"ffn_moe_out\", il);\n\n    return moe_out;\n}\n\n// input embeddings with optional lora\nggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const {\n    const int64_t n_embd_inp = hparams.n_embd_inp();\n    const int64_t n_embd     = hparams.n_embd;\n\n    assert(n_embd_inp >= n_embd);\n\n    auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);\n\n    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);\n    cb(inp->tokens, \"inp_tokens\", -1);\n    ggml_set_input(inp->tokens);\n    res->t_inp_tokens = inp->tokens;\n\n    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);\n    cb(inp->embd, \"inp_embd\", -1);\n    ggml_set_input(inp->embd);\n\n    // select one of the 2 inputs, based on the batch contents\n    // ref: https://github.com/ggml-org/llama.cpp/pull/18550\n    std::array<ggml_tensor *, 2> inps;\n\n    // token embeddings path (ubatch.token != nullptr)\n    {\n        auto & cur = inps[0];\n\n        cur = ggml_get_rows(ctx0, tok_embd, inp->tokens);\n\n        // apply lora for embedding tokens if needed\n        for (const auto & lora : *loras) {\n            llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd);\n            if (lw == nullptr) {\n                continue;\n            }\n\n            const float adapter_scale = lora.second;\n            const float scale = lw->get_scale(lora.first->alpha, adapter_scale);\n\n            ggml_tensor * inpL_delta = ggml_scale(ctx0, ggml_mul_mat(\n                        ctx0, lw->b, // non-transposed lora_b\n                        ggml_get_rows(ctx0, lw->a, inp->tokens)\n                        ), scale);\n\n            cur = ggml_add(ctx0, cur, inpL_delta);\n        }\n\n        if (n_embd_inp != n_embd) {\n            cur = ggml_pad(ctx0, cur, hparams.n_embd_inp() - n_embd, 0, 0, 0);\n        }\n    }\n\n    // vector embeddings path (ubatch.embd != nullptr)\n    {\n        auto & cur = inps[1];\n\n        cur = inp->embd;\n    }\n\n    assert(ggml_are_same_shape (inps[0], inps[1]));\n    assert(ggml_are_same_stride(inps[0], inps[1]));\n\n    ggml_tensor * cur = ggml_build_forward_select(gf, inps.data(), inps.size(), ubatch.token ? 0 : 1);\n\n    if (n_embd_inp != n_embd) {\n        cur = ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0);\n    }\n\n    res->t_inp_embd = cur;\n\n    // For Granite architecture\n    if (hparams.f_embedding_scale != 0.0f) {\n        cur = ggml_scale(ctx0, cur, hparams.f_embedding_scale);\n    }\n\n    cb(cur, \"embd\", -1);\n\n    res->add_input(std::move(inp));\n\n    // make sure the produced embeddings are immediately materialized in the ggml graph\n    // ref: https://github.com/ggml-org/llama.cpp/pull/18599\n    ggml_build_forward_expand(gf, cur);\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_pos() const {\n    auto inp = std::make_unique<llm_graph_input_pos>(hparams.n_pos_per_embd());\n\n    auto & cur = inp->pos;\n\n    cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t)n_tokens*hparams.n_pos_per_embd());\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_attn_scale() const {\n    auto inp = std::make_unique<llm_graph_input_attn_temp>(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset);\n\n    auto & cur = inp->attn_scale;\n\n    // this need to be 1x1xN for broadcasting\n    cur = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, 1, n_tokens);\n    ggml_set_input(cur);\n    ggml_set_name(cur, \"attn_scale\");\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_out_ids() const {\n    // note: when all tokens are output, we could skip this optimization to spare the ggml_get_rows() calls,\n    //       but this would make the graph topology depend on the number of output tokens, which can interere with\n    //       features that require constant topology such as pipeline parallelism\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/14275#issuecomment-2987424471\n    //if (n_outputs < n_tokens) {\n    //    return nullptr;\n    //}\n\n    auto inp = std::make_unique<llm_graph_input_out_ids>(hparams, cparams, n_outputs);\n\n    auto & cur = inp->out_ids;\n\n    cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_mean() const {\n    auto inp = std::make_unique<llm_graph_input_mean>(cparams);\n\n    auto & cur = inp->mean;\n\n    cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, ubatch.n_seqs_unq);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_cls() const {\n    auto inp = std::make_unique<llm_graph_input_cls>(cparams, arch);\n\n    auto & cur = inp->cls;\n\n    cur = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_seqs_unq);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_cross_embd() const {\n    auto inp = std::make_unique<llm_graph_input_cross_embd>(cross);\n\n    auto & cur = inp->cross_embd;\n\n    // if we have the output embeddings from the encoder, use them directly\n    // TODO: needs more work to be correct, for now just use the tensor shape\n    //if (cross->t_embd) {\n    //    cur = ggml_view_tensor(ctx0, cross->t_embd);\n\n    //    return cur;\n    //}\n\n    const auto n_embd = !cross->v_embd.empty() ? cross->n_embd : hparams.n_embd_inp();\n    const auto n_enc  = !cross->v_embd.empty() ? cross->n_enc  : hparams.n_ctx_train;\n\n    cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_enc);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_pos_bucket_enc() const {\n    auto inp = std::make_unique<llm_graph_input_pos_bucket>(hparams);\n\n    auto & cur = inp->pos_bucket;\n\n    cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_tokens, n_tokens);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_inp_pos_bucket_dec() const {\n    const auto * mctx_cur = static_cast<const llama_kv_cache_context *>(mctx);\n\n    auto inp = std::make_unique<llm_graph_input_pos_bucket_kv>(hparams, mctx_cur);\n\n    const auto n_kv = mctx_cur->get_n_kv();\n\n    auto & cur = inp->pos_bucket;\n\n    cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_kv, n_tokens);\n    ggml_set_input(cur);\n\n    res->add_input(std::move(inp));\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_pos_bias(ggml_tensor * pos_bucket, ggml_tensor * attn_rel_b) const {\n    ggml_tensor * pos_bucket_1d = ggml_reshape_1d(ctx0, pos_bucket, pos_bucket->ne[0] * pos_bucket->ne[1]);\n    cb(pos_bucket_1d, \"pos_bucket_1d\", -1);\n\n    ggml_tensor * pos_bias = ggml_get_rows(ctx0, attn_rel_b, pos_bucket_1d);\n\n    pos_bias = ggml_reshape_3d(ctx0, pos_bias, pos_bias->ne[0], pos_bucket->ne[0], pos_bucket->ne[1]);\n    pos_bias = ggml_permute   (ctx0, pos_bias, 2, 0, 1, 3);\n    pos_bias = ggml_cont      (ctx0, pos_bias);\n\n    cb(pos_bias, \"pos_bias\", -1);\n\n    return pos_bias;\n}\n\nggml_tensor * llm_graph_context::build_attn_mha(\n         ggml_tensor * q,\n         ggml_tensor * k,\n         ggml_tensor * v,\n         ggml_tensor * kq_b,\n         ggml_tensor * kq_mask,\n         ggml_tensor * sinks,\n         ggml_tensor * v_mla,\n               float   kq_scale,\n                 int   il) const {\n    const bool v_trans = v->nb[1] > v->nb[2];\n\n    // split the batch into streams if needed\n    const auto n_stream = k->ne[3];\n\n    q = ggml_view_4d(ctx0, q, q->ne[0], q->ne[1], q->ne[2]/n_stream, n_stream, q->nb[1], q->nb[2], q->nb[3]/n_stream, 0);\n\n    q = ggml_permute(ctx0, q, 0, 2, 1, 3);\n    k = ggml_permute(ctx0, k, 0, 2, 1, 3);\n    v = ggml_permute(ctx0, v, 0, 2, 1, 3);\n\n    ggml_tensor * cur;\n\n    const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;\n    if (use_flash_attn) {\n        GGML_ASSERT(kq_b == nullptr && \"Flash attention does not support KQ bias yet\");\n\n        if (v_trans) {\n            v = ggml_transpose(ctx0, v);\n        }\n\n        // this can happen when KV cache is not used (e.g. an embedding model with non-causal attn)\n        if (k->type == GGML_TYPE_F32) {\n            k = ggml_cast(ctx0, k, GGML_TYPE_F16);\n        }\n\n        if (v->type == GGML_TYPE_F32) {\n            v = ggml_cast(ctx0, v, GGML_TYPE_F16);\n        }\n\n        cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias,\n                                  hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f);\n        cb(cur, LLAMA_TENSOR_NAME_FATTN, il);\n\n        ggml_flash_attn_ext_add_sinks(cur, sinks);\n        ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);\n\n        if (v_mla) {\n#if 0\n            // v_mla can be applied as a matrix-vector multiplication with broadcasting across dimension 3 == n_tokens.\n            // However, the code is optimized for dimensions 0 and 1 being large, so this is inefficient.\n            cur = ggml_reshape_4d(ctx0, cur, v_mla->ne[0], 1, n_head, n_tokens);\n            cur = ggml_mul_mat(ctx0, v_mla, cur);\n#else\n            // It's preferable to do the calculation as a matrix-matrix multiplication with n_tokens in dimension 1.\n            // The permutations are noops and only change how the tensor data is interpreted.\n            cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);\n            cur = ggml_mul_mat(ctx0, v_mla, cur);\n            cb(cur, \"fattn_mla\", il);\n            cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);\n            cur = ggml_cont(ctx0, cur); // Needed because ggml_reshape_2d expects contiguous inputs.\n#endif\n        }\n\n        cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);\n    } else {\n        ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);\n        cb(kq, \"kq\", il);\n\n        // note: this op tends to require high floating point range\n        //       while for some models F16 is enough, for others it is not, so we default to F32 here\n        ggml_mul_mat_set_prec(kq, GGML_PREC_F32);\n\n        if (arch == LLM_ARCH_GROK) {\n            // need to do the following:\n            // multiply by attn_output_multiplier\n            // and then :\n            // kq = 30 * tanh(kq / 30)\n            // before the softmax below\n\n            kq = ggml_tanh(ctx0, ggml_scale(ctx0, kq, hparams.f_attn_out_scale / hparams.f_attn_logit_softcapping));\n            cb(kq, \"kq_tanh\", il);\n            kq = ggml_scale(ctx0, kq, hparams.f_attn_logit_softcapping);\n            cb(kq, \"kq_scaled\", il);\n        }\n\n        if (hparams.attn_soft_cap) {\n            kq = ggml_scale(ctx0, kq, 1.0f / hparams.f_attn_logit_softcapping);\n            cb(kq, \"kq_scaled_1\", il);\n            kq = ggml_tanh (ctx0, kq);\n            cb(kq, \"kq_tanh\", il);\n            kq = ggml_scale(ctx0, kq, hparams.f_attn_logit_softcapping);\n            cb(kq, \"kq_scaled_2\", il);\n        }\n\n        if (kq_b) {\n            kq = ggml_add(ctx0, kq, kq_b);\n            cb(kq, \"kq_plus_kq_b\", il);\n        }\n\n        kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);\n        ggml_soft_max_add_sinks(kq, sinks);\n        cb(kq, \"kq_soft_max\", il);\n\n        if (!v_trans) {\n            // note: avoid this branch\n            v = ggml_cont(ctx0, ggml_transpose(ctx0, v));\n            cb(v, \"v_cont\", il);\n        }\n\n        ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);\n        cb(kqv, \"kqv\", il);\n\n        // for MLA with the absorption optimization, we need to \"decompress\" from MQA back to MHA\n        if (v_mla) {\n            kqv = ggml_mul_mat(ctx0, v_mla, kqv);\n            cb(kqv, \"kqv_mla\", il);\n        }\n\n        cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);\n\n        // recombine streams\n        cur = ggml_cont_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);\n\n        if (!cparams.offload_kqv) {\n            // all nodes between the KV store and the attention output are run on the CPU\n            ggml_backend_sched_set_tensor_backend(sched, cur, backend_cpu);\n        }\n    }\n\n    ggml_build_forward_expand(gf, cur);\n\n    return cur;\n}\n\nllm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() const {\n    auto inp = std::make_unique<llm_graph_input_attn_no_cache>(hparams, cparams);\n\n    // note: there is no KV cache, so the number of KV values is equal to the number of tokens in the batch\n    inp->self_kq_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens, 1, 1);\n    ggml_set_input(inp->self_kq_mask);\n\n    inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;\n\n    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n        inp->self_kq_mask_swa = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_tokens, n_tokens, 1, 1);\n        ggml_set_input(inp->self_kq_mask_swa);\n\n        inp->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask_swa, GGML_TYPE_F16) : inp->self_kq_mask_swa;\n    } else {\n        inp->self_kq_mask_swa     = nullptr;\n        inp->self_kq_mask_swa_cnv = nullptr;\n    }\n\n    return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_attn(\n        llm_graph_input_attn_no_cache * inp,\n        ggml_tensor * wo,\n        ggml_tensor * wo_b,\n        ggml_tensor * q_cur,\n        ggml_tensor * k_cur,\n        ggml_tensor * v_cur,\n        ggml_tensor * kq_b,\n        ggml_tensor * sinks,\n        ggml_tensor * v_mla,\n            float     kq_scale,\n            int       il) const {\n    GGML_UNUSED(n_tokens);\n\n    // these nodes are added to the graph together so that they are not reordered\n    // by doing so, the number of splits in the graph is reduced\n    ggml_build_forward_expand(gf, q_cur);\n    ggml_build_forward_expand(gf, k_cur);\n    ggml_build_forward_expand(gf, v_cur);\n\n    const bool is_swa = hparams.is_swa(il);\n\n    const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask();\n\n    // [TAG_NO_CACHE_PAD]\n    // TODO: if ubatch.equal_seqs() == true, we can split the three tensors below into ubatch.n_seqs_unq streams\n    //       but it might not be worth it: https://github.com/ggml-org/llama.cpp/pull/15636\n    //assert(!ubatch.equal_seqs() || (k_cur->ne[3] == 1 && k_cur->ne[3] == ubatch.n_seqs_unq));\n\n    ggml_tensor * q = q_cur;\n    ggml_tensor * k = k_cur;\n    ggml_tensor * v = v_cur;\n\n    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);\n    cb(cur, \"kqv_out\", il);\n\n    if (wo) {\n        cur = build_lora_mm(wo, cur);\n    }\n\n    if (wo_b) {\n        //cb(cur, \"kqv_wo\", il);\n    }\n\n    if (wo_b) {\n        cur = ggml_add(ctx0, cur, wo_b);\n    }\n\n    return cur;\n}\n\nstatic std::unique_ptr<llm_graph_input_attn_kv> build_attn_inp_kv_impl(\n           ggml_context * ctx0,\n     const llama_ubatch & ubatch,\n    const llama_hparams & hparams,\n    const llama_cparams & cparams,\n    const llama_kv_cache_context * mctx_cur) {\n\n    auto inp = std::make_unique<llm_graph_input_attn_kv>(hparams, cparams, mctx_cur);\n\n    {\n        GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_NONE && \"Use llama_kv_cache_iswa for SWA\");\n\n        inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch);\n        inp->self_v_idxs = mctx_cur->build_input_v_idxs(ctx0, ubatch);\n\n        inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur, ubatch, cparams);\n\n        ggml_set_input(inp->self_kq_mask);\n\n        inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;\n    }\n\n    return inp;\n}\n\nllm_graph_input_attn_kv * llm_graph_context::build_attn_inp_kv() const {\n    const auto * mctx_cur = static_cast<const llama_kv_cache_context *>(mctx);\n\n    auto inp = build_attn_inp_kv_impl(ctx0, ubatch, hparams, cparams, mctx_cur);\n\n    return (llm_graph_input_attn_kv *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_attn(\n        llm_graph_input_attn_kv * inp,\n        ggml_tensor * wo,\n        ggml_tensor * wo_b,\n        ggml_tensor * q_cur,\n        ggml_tensor * k_cur,\n        ggml_tensor * v_cur,\n        ggml_tensor * kq_b,\n        ggml_tensor * sinks,\n        ggml_tensor * v_mla, // TODO: remove\n            float     kq_scale,\n            int       il) const {\n    GGML_ASSERT(v_mla == nullptr);\n\n    // these nodes are added to the graph together so that they are not reordered\n    // by doing so, the number of splits in the graph is reduced\n    // expand k later to enable rope fusion which directly writes into k-v cache\n    ggml_build_forward_expand(gf, q_cur);\n    ggml_build_forward_expand(gf, v_cur);\n    ggml_build_forward_expand(gf, k_cur);\n\n    const auto * mctx_cur = inp->mctx;\n\n    // store to KV cache\n    {\n        const auto & k_idxs = inp->get_k_idxs();\n        const auto & v_idxs = inp->get_v_idxs();\n\n        ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));\n        ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));\n    }\n\n    const auto & kq_mask = inp->get_kq_mask();\n\n    ggml_tensor * q = q_cur;\n    ggml_tensor * k = mctx_cur->get_k(ctx0, il);\n    ggml_tensor * v = mctx_cur->get_v(ctx0, il);\n\n    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);\n    cb(cur, \"kqv_out\", il);\n\n    if (wo) {\n        cur = build_lora_mm(wo, cur);\n        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {\n            // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators\n            ggml_mul_mat_set_prec(cur, GGML_PREC_F32);\n        }\n    }\n\n    if (wo_b) {\n        cur = ggml_add(ctx0, cur, wo_b);\n    }\n\n    return cur;\n}\n\nstatic std::unique_ptr<llm_graph_input_attn_k> build_attn_inp_k_impl(\n           ggml_context * ctx0,\n     const llama_ubatch & ubatch,\n    const llama_hparams & hparams,\n    const llama_cparams & cparams,\n    const llama_kv_cache_context * mctx_cur) {\n\n    auto inp = std::make_unique<llm_graph_input_attn_k>(hparams, cparams, mctx_cur);\n\n    {\n        GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_NONE && \"Use llama_kv_cache_iswa for SWA\");\n\n        inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch);\n\n        inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur, ubatch, cparams);\n        ggml_set_input(inp->self_kq_mask);\n\n        inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;\n    }\n\n    return inp;\n}\n\nllm_graph_input_attn_k * llm_graph_context::build_attn_inp_k() const {\n    const auto * mctx_cur = static_cast<const llama_kv_cache_context *>(mctx);\n\n    auto inp = build_attn_inp_k_impl(ctx0, ubatch, hparams, cparams, mctx_cur);\n\n    return (llm_graph_input_attn_k *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_attn(\n        llm_graph_input_attn_k * inp,\n        ggml_tensor * wo,\n        ggml_tensor * wo_b,\n        ggml_tensor * q_cur,\n        ggml_tensor * k_cur,\n        ggml_tensor * v_cur,\n        ggml_tensor * kq_b,\n        ggml_tensor * sinks,\n        ggml_tensor * v_mla,\n            float     kq_scale,\n            int       il) const {\n    // these nodes are added to the graph together so that they are not reordered\n    // by doing so, the number of splits in the graph is reduced\n    // expand k later to enable rope fusion which directly writes into k-v cache\n    ggml_build_forward_expand(gf, q_cur);\n    ggml_build_forward_expand(gf, v_cur);\n    ggml_build_forward_expand(gf, k_cur);\n\n    const auto * mctx_cur = inp->mctx;\n\n    // store to KV cache\n    {\n        const auto & k_idxs = inp->get_k_idxs();\n\n        ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));\n    }\n\n    const auto & kq_mask = inp->get_kq_mask();\n\n    ggml_tensor * q = q_cur;\n    ggml_tensor * k = mctx_cur->get_k(ctx0, il);\n    ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);\n\n    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);\n    cb(cur, \"kqv_out\", il);\n\n    if (wo) {\n        cur = build_lora_mm(wo, cur);\n        if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {\n            // GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators\n            ggml_mul_mat_set_prec(cur, GGML_PREC_F32);\n        }\n    }\n\n    if (wo_b) {\n        cur = ggml_add(ctx0, cur, wo_b);\n    }\n\n    return cur;\n}\n\nggml_tensor * llm_graph_context::build_attn(\n        llm_graph_input_attn_kv_iswa * inp,\n        ggml_tensor * wo,\n        ggml_tensor * wo_b,\n        ggml_tensor * q_cur,\n        ggml_tensor * k_cur,\n        ggml_tensor * v_cur,\n        ggml_tensor * kq_b,\n        ggml_tensor * sinks,\n        ggml_tensor * v_mla,\n            float     kq_scale,\n            int       il) const {\n    // these nodes are added to the graph together so that they are not reordered\n    // by doing so, the number of splits in the graph is reduced\n    ggml_build_forward_expand(gf, q_cur);\n\n    if (k_cur) {\n        ggml_build_forward_expand(gf, k_cur);\n    }\n\n    if (v_cur) {\n        ggml_build_forward_expand(gf, v_cur);\n    }\n\n    const auto * mctx_iswa = inp->mctx;\n\n    const bool is_swa = hparams.is_swa(il);\n\n    const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base();\n\n    // optionally store to KV cache\n    if (k_cur) {\n        const auto & k_idxs = is_swa ? inp->get_k_idxs_swa() : inp->get_k_idxs();\n\n        ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));\n    }\n\n    if (v_cur) {\n        const auto & v_idxs = is_swa ? inp->get_v_idxs_swa() : inp->get_v_idxs();\n\n        ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));\n    }\n\n    const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask();\n\n    ggml_tensor * q = q_cur;\n    ggml_tensor * k = mctx_cur->get_k(ctx0, il);\n    ggml_tensor * v = mctx_cur->get_v(ctx0, il);\n\n    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);\n    cb(cur, \"kqv_out\", il);\n\n    if (wo) {\n        cur = build_lora_mm(wo, cur);\n    }\n\n    if (wo_b) {\n        //cb(cur, \"kqv_wo\", il);\n    }\n\n    if (wo_b) {\n        cur = ggml_add(ctx0, cur, wo_b);\n    }\n\n    return cur;\n}\n\nllm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const {\n    auto inp = std::make_unique<llm_graph_input_attn_cross>(cross);\n\n    const int32_t n_enc = !cross->v_embd.empty() ? cross->n_enc : hparams.n_ctx_train;\n\n    inp->cross_kq_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_enc, n_tokens, 1, 1);\n    ggml_set_input(inp->cross_kq_mask);\n\n    inp->cross_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->cross_kq_mask, GGML_TYPE_F16) : inp->cross_kq_mask;\n\n    return (llm_graph_input_attn_cross *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_attn(\n        llm_graph_input_attn_cross * inp,\n        ggml_tensor * wo,\n        ggml_tensor * wo_b,\n        ggml_tensor * q_cur,\n        ggml_tensor * k_cur,\n        ggml_tensor * v_cur,\n        ggml_tensor * kq_b,\n        ggml_tensor * sinks,\n        ggml_tensor * v_mla,\n            float     kq_scale,\n            int       il) const {\n    // these nodes are added to the graph together so that they are not reordered\n    // by doing so, the number of splits in the graph is reduced\n    ggml_build_forward_expand(gf, q_cur);\n    ggml_build_forward_expand(gf, k_cur);\n    ggml_build_forward_expand(gf, v_cur);\n\n    const auto & kq_mask = inp->get_kq_mask_cross();\n\n    ggml_tensor * q = q_cur;\n    ggml_tensor * k = k_cur;\n    ggml_tensor * v = v_cur;\n\n    ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);\n    cb(cur, \"kqv_out\", il);\n\n    if (wo) {\n        cur = build_lora_mm(wo, cur);\n    }\n\n    if (wo_b) {\n        //cb(cur, \"kqv_wo\", il);\n    }\n\n    if (wo_b) {\n        cur = ggml_add(ctx0, cur, wo_b);\n    }\n\n    return cur;\n}\n\n// TODO: maybe separate the inner implementation into a separate function\n//       like with the non-sliding window equivalent\n//       once sliding-window hybrid caches are a thing.\nllm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const {\n    const auto * mctx_cur = static_cast<const llama_kv_cache_iswa_context *>(mctx);\n\n    auto inp = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, mctx_cur);\n\n    {\n        inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch);\n        inp->self_v_idxs = mctx_cur->get_base()->build_input_v_idxs(ctx0, ubatch);\n\n        inp->self_kq_mask = build_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams);\n        ggml_set_input(inp->self_kq_mask);\n        ggml_set_name(inp->self_kq_mask, \"self_kq_mask\");\n\n        inp->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask, GGML_TYPE_F16) : inp->self_kq_mask;\n        ggml_set_name(inp->self_kq_mask_cnv, \"self_kq_mask_cnv\");\n    }\n\n    {\n        GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE && \"Use llama_kv_cache for non-SWA\");\n\n        inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch);\n        inp->self_v_idxs_swa = mctx_cur->get_swa()->build_input_v_idxs(ctx0, ubatch);\n\n        inp->self_kq_mask_swa = build_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams);\n        ggml_set_input(inp->self_kq_mask_swa);\n        ggml_set_name(inp->self_kq_mask_swa, \"self_kq_mask_swa\");\n\n        inp->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp->self_kq_mask_swa, GGML_TYPE_F16) : inp->self_kq_mask_swa;\n        ggml_set_name(inp->self_kq_mask_swa_cnv, \"self_kq_mask_swa_cnv\");\n    }\n\n    return (llm_graph_input_attn_kv_iswa *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_rs(\n        ggml_tensor * s,\n        ggml_tensor * state_copy_main,\n        ggml_tensor * state_copy_extra,\n            int32_t   state_size,\n            int32_t   n_seqs,\n           uint32_t   n_rs,\n           uint32_t   rs_head,\n           uint32_t   rs_size,\n            int32_t   rs_zero,\n        const llm_graph_get_rows_fn & get_state_rows) const {\n\n    ggml_tensor * states = ggml_reshape_2d(ctx0, s, state_size, rs_size);\n\n    // Clear a single state which will then be copied to the other cleared states.\n    // Note that this is a no-op when the view is zero-sized.\n    ggml_tensor * state_zero = ggml_view_1d(ctx0, states, state_size*(rs_zero >= 0), rs_zero*states->nb[1]*(rs_zero >= 0));\n    ggml_build_forward_expand(gf, ggml_scale_inplace(ctx0, state_zero, 0));\n\n    // copy states\n    // NOTE: assuming the copy destinations are ALL contained between rs_head and rs_head + n_rs\n    // {state_size, rs_size} -> {state_size, n_seqs}\n    ggml_tensor * output_states = get_state_rows(ctx0, states, state_copy_main);\n    ggml_build_forward_expand(gf, output_states);\n\n    // copy extra states which won't be changed further (between n_seqs and n_rs)\n    ggml_tensor * states_extra = ggml_get_rows(ctx0, states, state_copy_extra);\n    ggml_build_forward_expand(gf,\n        ggml_cpy(ctx0,\n            states_extra,\n            ggml_view_1d(ctx0, s, state_size*(n_rs - n_seqs), (rs_head + n_seqs)*state_size*ggml_element_size(s))));\n\n    return output_states;\n}\n\nstatic std::unique_ptr<llm_graph_input_rs> build_rs_inp_impl(\n           ggml_context * ctx0,\n     const llama_ubatch & ubatch,\n    const llama_memory_recurrent_context * mctx_cur) {\n\n    auto inp = std::make_unique<llm_graph_input_rs>(mctx_cur);\n\n    const int64_t n_rs   = mctx_cur->get_n_rs();\n    const int64_t n_seqs = ubatch.n_seqs;\n\n    inp->s_copy = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_rs);\n    ggml_set_input(inp->s_copy);\n\n    inp->s_copy_main  = ggml_view_1d(ctx0, inp->s_copy, n_seqs, 0);\n    inp->s_copy_extra = ggml_view_1d(ctx0, inp->s_copy, n_rs - n_seqs, n_seqs * inp->s_copy->nb[0]);\n\n    inp->head = mctx_cur->get_head();\n    inp->rs_z = mctx_cur->get_rs_z();\n\n    return inp;\n}\n\nllm_graph_input_rs * llm_graph_context::build_rs_inp() const {\n    const auto * mctx_cur = static_cast<const llama_memory_recurrent_context *>(mctx);\n\n    auto inp = build_rs_inp_impl(ctx0, ubatch, mctx_cur);\n\n    return (llm_graph_input_rs *) res->add_input(std::move(inp));\n}\n\nggml_tensor * llm_graph_context::build_rs(\n        llm_graph_input_rs * inp,\n        ggml_tensor * s,\n            int32_t   state_size,\n            int32_t   n_seqs,\n        const llm_graph_get_rows_fn & get_state_rows) const {\n    const auto * kv_state = inp->mctx;\n\n    return build_rs(s, inp->s_copy_main, inp->s_copy_extra, state_size, n_seqs,\n                    kv_state->get_n_rs(), kv_state->get_head(), kv_state->get_size(), kv_state->get_rs_z(),\n                    get_state_rows);\n}\n\nggml_tensor * llm_graph_context::build_rwkv_token_shift_load(\n    llm_graph_input_rs * inp,\n    const llama_ubatch & ubatch,\n                   int   il) const {\n    const auto * mctx_cur = static_cast<const llama_memory_recurrent_context *>(mctx);\n\n    const auto token_shift_count = hparams.token_shift_count;\n\n    const int64_t n_seqs  = ubatch.n_seqs;\n\n    ggml_tensor * token_shift_all = mctx_cur->get_r_l(il);\n\n    ggml_tensor * token_shift = build_rs(\n            inp, token_shift_all,\n            hparams.n_embd_r(), n_seqs);\n\n    token_shift = ggml_reshape_3d(ctx0, token_shift, hparams.n_embd, token_shift_count, n_seqs);\n\n    return token_shift;\n}\n\nggml_tensor * llm_graph_context::build_rwkv_token_shift_store(\n         ggml_tensor * token_shift,\n  const llama_ubatch & ubatch,\n                 int   il) const {\n    const auto * mctx_cur = static_cast<const llama_memory_recurrent_context *>(mctx);\n\n    const auto token_shift_count = hparams.token_shift_count;\n    const auto n_embd = hparams.n_embd;\n\n    const int64_t n_seqs = ubatch.n_seqs;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    return ggml_cpy(\n        ctx0,\n        ggml_view_1d(ctx0, token_shift, n_embd * n_seqs * token_shift_count, 0),\n        ggml_view_1d(ctx0, mctx_cur->get_r_l(il), hparams.n_embd_r()*n_seqs, hparams.n_embd_r()*kv_head*ggml_element_size(mctx_cur->get_r_l(il)))\n    );\n}\n\nllm_graph_input_mem_hybrid * llm_graph_context::build_inp_mem_hybrid() const {\n    const auto * mctx_cur = static_cast<const llama_memory_hybrid_context *>(mctx);\n\n    auto inp_rs   = build_rs_inp_impl     (ctx0, ubatch, mctx_cur->get_recr());\n    auto inp_attn = build_attn_inp_kv_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_attn());\n\n    auto inp = std::make_unique<llm_graph_input_mem_hybrid>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);\n\n    return (llm_graph_input_mem_hybrid *) res->add_input(std::move(inp));\n}\n\nllm_graph_input_mem_hybrid_k * llm_graph_context::build_inp_mem_hybrid_k() const {\n    const auto * mctx_cur = static_cast<const llama_memory_hybrid_context *>(mctx);\n\n    auto inp_rs   = build_rs_inp_impl     (ctx0, ubatch, mctx_cur->get_recr());\n    auto inp_attn = build_attn_inp_k_impl(ctx0, ubatch, hparams, cparams, mctx_cur->get_attn());\n\n    auto inp = std::make_unique<llm_graph_input_mem_hybrid_k>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);\n\n    return (llm_graph_input_mem_hybrid_k *) res->add_input(std::move(inp));\n}\n\nllm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() const {\n    const auto * mctx_cur = static_cast<const llama_memory_hybrid_iswa_context *>(mctx);\n\n    auto inp_rs = build_rs_inp_impl(ctx0, ubatch, mctx_cur->get_recr());\n\n    // build iswa attention input\n    const auto * attn_ctx = mctx_cur->get_attn();\n\n    auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);\n\n    {\n        inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);\n        inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);\n\n        inp_attn->self_kq_mask = build_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);\n        ggml_set_input(inp_attn->self_kq_mask);\n\n        inp_attn->self_kq_mask_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp_attn->self_kq_mask, GGML_TYPE_F16) : inp_attn->self_kq_mask;\n    }\n\n    {\n        inp_attn->self_k_idxs_swa = attn_ctx->get_swa()->build_input_k_idxs(ctx0, ubatch);\n        inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);\n\n        inp_attn->self_kq_mask_swa = build_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);\n        ggml_set_input(inp_attn->self_kq_mask_swa);\n\n        inp_attn->self_kq_mask_swa_cnv = cparams.flash_attn ? ggml_cast(ctx0, inp_attn->self_kq_mask_swa, GGML_TYPE_F16) : inp_attn->self_kq_mask_swa;\n    }\n\n    auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);\n\n    return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));\n}\n\nvoid llm_graph_context::build_dense_out(\n    ggml_tensor * dense_2,\n    ggml_tensor * dense_2_b,\n    ggml_tensor * dense_3) const {\n    if (!cparams.embeddings || !(dense_2 || dense_2_b || dense_3)) {\n        return;\n    }\n    ggml_tensor * cur = res->t_embd_pooled != nullptr ? res->t_embd_pooled : res->t_embd;\n    GGML_ASSERT(cur != nullptr && \"missing t_embd_pooled/t_embd\");\n\n    if (dense_2) {\n        cur = ggml_mul_mat(ctx0, dense_2, cur);\n    }\n    if (dense_2_b) {\n        cur = ggml_add(ctx0, cur, dense_2_b);\n    }\n    if (dense_3) {\n        cur = ggml_mul_mat(ctx0, dense_3, cur);\n    }\n    cb(cur, \"result_embd_pooled\", -1);\n    res->t_embd_pooled = cur;\n    ggml_build_forward_expand(gf, cur);\n}\n\n\nvoid llm_graph_context::build_pooling(\n        ggml_tensor * cls,\n        ggml_tensor * cls_b,\n        ggml_tensor * cls_out,\n        ggml_tensor * cls_out_b,\n        ggml_tensor * cls_norm) const {\n    if (!cparams.embeddings) {\n        return;\n    }\n\n    ggml_tensor * inp = res->t_embd;\n\n    //// find result_norm tensor for input\n    //for (int i = ggml_graph_n_nodes(gf) - 1; i >= 0; --i) {\n    //    inp = ggml_graph_node(gf, i);\n    //    if (strcmp(inp->name, \"result_norm\") == 0 || strcmp(inp->name, \"result_embd\") == 0) {\n    //        break;\n    //    }\n\n    //    inp = nullptr;\n    //}\n\n    GGML_ASSERT(inp != nullptr && \"missing result_norm/result_embd tensor\");\n\n    ggml_tensor * cur;\n\n    switch (pooling_type) {\n        case LLAMA_POOLING_TYPE_NONE:\n            {\n                cur = inp;\n            } break;\n        case LLAMA_POOLING_TYPE_MEAN:\n            {\n                ggml_tensor * inp_mean = build_inp_mean();\n                cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean);\n            } break;\n        case LLAMA_POOLING_TYPE_CLS:\n        case LLAMA_POOLING_TYPE_LAST:\n            {\n                ggml_tensor * inp_cls = build_inp_cls();\n                cur = ggml_get_rows(ctx0, inp, inp_cls);\n            } break;\n        case LLAMA_POOLING_TYPE_RANK:\n            {\n                if (arch == LLM_ARCH_MODERN_BERT) {\n                    // modern bert gte reranker builds mean first then applies prediction head and classifier\n                    // https://github.com/huggingface/transformers/blob/main/src/transformers/models/modernbert/modular_modernbert.py#L1404-1411\n                    ggml_tensor * inp_mean = build_inp_mean();\n                    cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean);\n                } else {\n                    ggml_tensor * inp_cls = build_inp_cls();\n                    cur = ggml_get_rows(ctx0, inp, inp_cls);\n                }\n\n                // classification head\n                // https://github.com/huggingface/transformers/blob/5af7d41e49bbfc8319f462eb45253dcb3863dfb7/src/transformers/models/roberta/modeling_roberta.py#L1566\n                if (cls) {\n                    cur = ggml_mul_mat(ctx0, cls, cur);\n                    if (cls_b) {\n                        cur = ggml_add(ctx0, cur, cls_b);\n                    }\n                    if (arch == LLM_ARCH_MODERN_BERT) {\n                        cur = ggml_gelu(ctx0, cur);\n                    } else {\n                        cur = ggml_tanh(ctx0, cur);\n                    }\n                    if (cls_norm) {\n                        // head norm\n                        cur = build_norm(cur, cls_norm, NULL, LLM_NORM, -1);\n                    }\n                }\n\n                // some models don't have `cls_out`, for example: https://huggingface.co/jinaai/jina-reranker-v1-tiny-en\n                // https://huggingface.co/jinaai/jina-reranker-v1-tiny-en/blob/cb5347e43979c3084a890e3f99491952603ae1b7/modeling_bert.py#L884-L896\n                // Single layer classification head (direct projection)\n                // https://github.com/huggingface/transformers/blob/f4fc42216cd56ab6b68270bf80d811614d8d59e4/src/transformers/models/bert/modeling_bert.py#L1476\n                if (cls_out) {\n                    cur = ggml_mul_mat(ctx0, cls_out, cur);\n                    if (cls_out_b) {\n                        cur = ggml_add(ctx0, cur, cls_out_b);\n                    }\n                }\n\n                // softmax for qwen3 reranker\n                if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_QWEN3VL) {\n                    cur = ggml_soft_max(ctx0, cur);\n                }\n            } break;\n        default:\n            {\n                GGML_ABORT(\"unknown pooling type\");\n            }\n    }\n\n    cb(cur, \"result_embd_pooled\", -1);\n    res->t_embd_pooled = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nvoid llm_graph_context::build_sampling() const {\n    if (samplers.empty() || !res->t_logits) {\n        return;\n    }\n\n    std::array<ggml_tensor *, 2> outs;\n    outs[0] = res->t_logits;\n\n    auto inp_sampling = std::make_unique<llm_graph_input_sampling>(samplers);\n    res->add_input(std::move(inp_sampling));\n\n    std::map<llama_seq_id, int32_t> seq_to_logit_row;\n    int32_t logit_row_idx = 0;\n\n    for (uint32_t i = 0; i < ubatch.n_tokens; i++) {\n        if (ubatch.output[i]) {\n            llama_seq_id seq_id = ubatch.seq_id[i][0];\n            seq_to_logit_row[seq_id] = logit_row_idx;\n            logit_row_idx++;\n        }\n    }\n\n    // res->t_logits will contain logits for all tokens that want the logits calculated (logits=1 or output=1)\n    GGML_ASSERT(res->t_logits != nullptr && \"missing t_logits tensor\");\n\n    // add a dummy row of logits\n    // this trick makes the graph static, regardless of which samplers are activated\n    // this is important in order to minimize graph reallocations\n    ggml_tensor * logits_t = ggml_pad(ctx0, res->t_logits, 0, 1, 0, 0);\n\n    for (const auto & [seq_id, sampler] : samplers) {\n        const auto it = seq_to_logit_row.find(seq_id);\n\n        // inactive samplers always work on the first row\n        const auto row_idx = it != seq_to_logit_row.end() ? it->second : 0;\n        const int i_out    = it != seq_to_logit_row.end() ? 1          : 0;\n\n        ggml_tensor * logits_seq = ggml_view_1d(ctx0, logits_t, logits_t->ne[0], row_idx * logits_t->nb[1]);\n        ggml_format_name(logits_seq, \"logits_seq_%d\", seq_id);\n\n        struct llama_sampler_data data = {\n            /*.logits      =*/ logits_seq,\n            /*.probs       =*/ nullptr,\n            /*.sampled     =*/ nullptr,\n            /*.candidates  =*/ nullptr,\n        };\n\n        assert(sampler->iface->backend_apply);\n        sampler->iface->backend_apply(sampler, ctx0, gf, &data);\n\n        if (data.sampled != nullptr) {\n            res->t_sampled[seq_id] = data.sampled;\n            outs[1] = data.sampled;\n            ggml_build_forward_select(gf, outs.data(), outs.size(), i_out);\n        }\n\n        if (data.probs != nullptr) {\n            res->t_sampled_probs[seq_id] = data.probs;\n            outs[1] = data.probs;\n            ggml_build_forward_select(gf, outs.data(), outs.size(), i_out);\n        }\n\n        if (data.logits != nullptr) {\n            res->t_sampled_logits[seq_id] = data.logits;\n            outs[1] = data.logits;\n            ggml_build_forward_select(gf, outs.data(), outs.size(), i_out);\n        }\n\n        if (data.candidates != nullptr) {\n            res->t_candidates[seq_id] = data.candidates;\n            outs[1] = data.candidates;\n            ggml_build_forward_select(gf, outs.data(), outs.size(), i_out);\n        }\n    }\n\n    // TODO: Call llama_sampler_accept_ggml after all samplers have been applied.\n    /*\n    for (const auto & [seq_id, sampler] : samplers) {\n        if (auto it = res->t_sampled.find(seq_id); it != res->t_sampled.end()) {\n            ggml_tensor * selected_token = it->second;\n            if (selected_token != nullptr) {\n                llama_sampler_accept_ggml(sampler, ctx0, gf, selected_token);\n            }\n        }\n    }\n    */\n}\n\nint32_t llama_relative_position_bucket(llama_pos x, llama_pos y, uint64_t n_buckets, bool bidirectional) {\n    // TODO move to hparams if a T5 variant appears that uses a different value\n    const int64_t max_distance = 128;\n\n    if (bidirectional) {\n        n_buckets >>= 1;\n    }\n\n    const int64_t max_exact = n_buckets >> 1;\n\n    int32_t relative_position = x - y;\n    int32_t relative_bucket = 0;\n\n    if (bidirectional) {\n        relative_bucket += (relative_position > 0) * n_buckets;\n        relative_position = std::abs(relative_position);\n    } else {\n        relative_position = -std::min<int32_t>(relative_position, 0);\n    }\n\n    int32_t relative_position_if_large = floorf(max_exact + logf(1.0 * relative_position / max_exact) * (n_buckets - max_exact) / log(1.0 * max_distance / max_exact));\n    relative_position_if_large = std::min<int32_t>(relative_position_if_large, n_buckets - 1);\n    relative_bucket += (relative_position < max_exact ? relative_position : relative_position_if_large);\n\n    return relative_bucket;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-graph.h",
    "content": "#pragma once\n\n#include \"llama-arch.h\"\n#include \"llama-batch.h\"\n#include \"llama-hparams.h\"\n#include \"llama-adapter.h\"\n\n#include <cstdint>\n#include <vector>\n#include <memory>\n#include <set>\n#include <functional>\n#include <map>\n\nstruct ggml_cgraph;\nstruct ggml_context;\nstruct ggml_tensor;\n\nstruct llama_cparams;\n\nstruct llama_memory_context_i;\n\nclass llama_kv_cache_context;\nclass llama_kv_cache_iswa_context;\nclass llama_memory_recurrent_context;\nclass llama_memory_hybrid_context;\nclass llama_memory_hybrid_iswa_context;\n\n// certain models (typically multi-modal) can produce different types of graphs\nenum llm_graph_type {\n    LLM_GRAPH_TYPE_DEFAULT,\n    LLM_GRAPH_TYPE_ENCODER,\n    LLM_GRAPH_TYPE_DECODER,\n};\n\nenum llm_ffn_op_type {\n    LLM_FFN_SILU,\n    LLM_FFN_GELU,\n    LLM_FFN_RELU,\n    LLM_FFN_RELU_SQR,\n    LLM_FFN_SWIGLU,\n    LLM_FFN_GEGLU,\n    LLM_FFN_REGLU,\n    LLM_FFN_SWIGLU_OAI_MOE,\n};\n\nenum llm_ffn_gate_type {\n    LLM_FFN_SEQ,\n    LLM_FFN_PAR, // ffn_gate is parallel to ffn_up\n};\n\nenum llm_norm_type {\n    LLM_NORM,\n    LLM_NORM_RMS,\n    LLM_NORM_GROUP,\n};\n\n// TODO: tmp - need something better to pass the data from the encoder to the decoder\nstruct llama_cross {\n    // the output embeddings from the encoder as a ggml tensor\n    // TODO: this needs more work to be correct, for now copy the embeddings data to host memory\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/11213#discussion_r1969892524\n    //ggml_tensor * t_embd = nullptr;\n\n    int64_t n_embd = 0;\n    int64_t n_enc  = 0;\n\n    // embeddings data copied to host memory (tmp)\n    std::vector<float> v_embd;\n\n    // needed to construct the cross-attention mask in the decoder\n    std::vector<std::set<llama_seq_id>> seq_ids_enc;\n};\n\nstruct llm_graph_params;\n\n//\n// llm_graph_input\n//\n\nclass llm_graph_input_i {\npublic:\n    llm_graph_input_i() {\n        const char * LLAMA_GRAPH_INPUT_DEBUG = getenv(\"LLAMA_GRAPH_INPUT_DEBUG\");\n        debug = LLAMA_GRAPH_INPUT_DEBUG ? atoi(LLAMA_GRAPH_INPUT_DEBUG) : 0;\n    }\n\n    virtual ~llm_graph_input_i() = default;\n\n    virtual void set_input(const llama_ubatch * ubatch) = 0;\n\n    // return true if the resulting input tensors using the provided graph parameters would be\n    //   the same as the previous input tensors that we have currently stored in the object\n    virtual bool can_reuse(const llm_graph_params & params) {\n        // returning false here by default will prevent from reusing the graph if the check\n        //   for the input type has not been implemented yet\n        GGML_UNUSED(params);\n        return false;\n    }\nprotected:\n    // env: LLAMA_GRAPH_INPUT_DEBUG\n    int debug = 0;\n};\n\nusing llm_graph_input_ptr = std::unique_ptr<llm_graph_input_i>;\n\nclass llm_graph_input_embd : public llm_graph_input_i {\npublic:\n    llm_graph_input_embd(int64_t n_embd) : n_embd(n_embd) {}\n    virtual ~llm_graph_input_embd() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * tokens = nullptr; // I32 [n_batch]\n    ggml_tensor * embd   = nullptr; // F32 [n_embd, n_batch]\n\n    const int64_t n_embd = 0;\n};\n\nclass llm_graph_input_pos : public llm_graph_input_i {\npublic:\n    llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {}\n    virtual ~llm_graph_input_pos() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * pos = nullptr; // I32 [n_batch]\n\n    const uint32_t n_pos_per_embd = 1;\n};\n\n// temperature tuning, used by llama4\nclass llm_graph_input_attn_temp : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset)\n        : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset) {}\n    virtual ~llm_graph_input_attn_temp() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * attn_scale = nullptr; // F32 [n_batch]\n\n    const uint32_t n_attn_temp_floor_scale;\n    const float    f_attn_temp_scale;\n    const float    f_attn_temp_offset;\n};\n\nclass llm_graph_input_pos_bucket : public llm_graph_input_i {\npublic:\n    llm_graph_input_pos_bucket(const llama_hparams & hparams) : hparams(hparams) {}\n    virtual ~llm_graph_input_pos_bucket() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * pos_bucket = nullptr; // I32 [n_batch, n_batch]\n\n    const llama_hparams hparams;\n};\n\nclass llm_graph_input_pos_bucket_kv : public llm_graph_input_i {\npublic:\n    llm_graph_input_pos_bucket_kv(\n            const llama_hparams & hparams,\n            const llama_kv_cache_context * mctx) : hparams(hparams), mctx(mctx) {}\n    virtual ~llm_graph_input_pos_bucket_kv() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * pos_bucket = nullptr; // I32 [n_kv, n_batch]\n\n    const llama_hparams hparams;\n\n    const llama_kv_cache_context * mctx;\n};\n\nclass llm_graph_input_out_ids : public llm_graph_input_i {\npublic:\n    llm_graph_input_out_ids(\n            const llama_hparams & hparams,\n            const llama_cparams & cparams,\n            uint32_t n_outputs) : hparams(hparams), cparams(cparams), n_outputs(n_outputs) {}\n    virtual ~llm_graph_input_out_ids() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * out_ids; // I32 [n_outputs]\n\n    const llama_hparams hparams;\n    const llama_cparams cparams;\n\n    const uint32_t n_outputs;\n};\n\nclass llm_graph_input_mean : public llm_graph_input_i {\npublic:\n    llm_graph_input_mean(const llama_cparams & cparams) : cparams(cparams) {}\n    virtual ~llm_graph_input_mean() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * mean; // F32 [n_batch, n_batch]\n\n    const llama_cparams cparams;\n};\n\nclass llm_graph_input_cls : public llm_graph_input_i {\npublic:\n    llm_graph_input_cls(const llama_cparams & cparams, const llm_arch arch) : cparams(cparams), arch(arch) {}\n    virtual ~llm_graph_input_cls() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * cls; // I32 [n_batch]\n\n    const llama_cparams cparams;\n    const llm_arch arch;\n};\n\nclass llm_graph_input_rs : public llm_graph_input_i {\npublic:\n    llm_graph_input_rs(const llama_memory_recurrent_context * mctx) : mctx(mctx) {}\n    virtual ~llm_graph_input_rs() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * s_copy;  // I32 [n_rs]\n\n    // views of s_copy, computed once per graph\n    // and shared across layers which use build_rs\n    ggml_tensor * s_copy_main;   // I32 [n_seqs]\n    ggml_tensor * s_copy_extra;  // I32 [n_rs - n_seqs]\n\n    const llama_memory_recurrent_context * mctx;\n\n    // used in view offsets, need to match for valid graph reuse\n    uint32_t head;\n    int32_t rs_z;\n};\n\nclass llm_graph_input_cross_embd : public llm_graph_input_i {\npublic:\n    llm_graph_input_cross_embd(\n            const llama_cross * cross) : cross(cross) {}\n    virtual ~llm_graph_input_cross_embd() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * cross_embd; // F32 [n_embd, n_outputs_enc]\n\n    const llama_cross * cross;\n};\n\nclass llm_graph_input_attn_no_cache : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_no_cache(const llama_hparams & hparams, const llama_cparams & cparams) :\n        hparams(hparams),\n        cparams(cparams) {\n    }\n    ~llm_graph_input_attn_no_cache() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * get_kq_mask()     const { return self_kq_mask_cnv; }\n    ggml_tensor * get_kq_mask_swa() const { return self_kq_mask_swa_cnv; }\n\n    // n_tokens == n_batch\n    ggml_tensor * self_kq_mask         = nullptr; // F32 [n_tokens, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_cnv     = nullptr; //     [n_tokens, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_swa     = nullptr; // F32 [n_tokens, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_swa_cnv = nullptr; //     [n_tokens, n_batch/n_stream, 1, n_stream]\n\n    const llama_hparams hparams;\n    const llama_cparams cparams;\n};\n\nclass llm_graph_input_attn_kv : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_kv(\n            const llama_hparams & hparams,\n            const llama_cparams & cparams,\n            const llama_kv_cache_context * mctx) :\n        hparams(hparams),\n        cparams(cparams),\n        mctx(mctx) {\n    }\n    ~llm_graph_input_attn_kv() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * get_k_idxs() const { return self_k_idxs; }\n    ggml_tensor * get_v_idxs() const { return self_v_idxs; }\n\n    ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; }\n\n    ggml_tensor * self_k_idxs = nullptr; // I64 [n_batch]\n    ggml_tensor * self_v_idxs = nullptr; // I64 [n_batch] or [n_batch*n_embd_v_gqa]\n\n    ggml_tensor * self_kq_mask     = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_cnv = nullptr; //     [n_kv, n_batch/n_stream, 1, n_stream]\n\n    // note: these have to be copies because in order to be able to reuse a graph, its inputs\n    //       need to carry these parameters with them. otherwise, they can point to freed\n    //       llm_graph_params from a previous batch, causing stack-use-after-return\n    const llama_hparams hparams;\n    const llama_cparams cparams;\n\n    const llama_kv_cache_context * mctx;\n};\n\n// V-less input for the KV cache\n// ref: https://github.com/ggml-org/llama.cpp/pull/19067\nclass llm_graph_input_attn_k : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_k(\n            const llama_hparams & hparams,\n            const llama_cparams & cparams,\n            const llama_kv_cache_context * mctx) :\n        hparams(hparams),\n        cparams(cparams),\n        mctx(mctx) {\n    }\n    ~llm_graph_input_attn_k() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * get_k_idxs() const { return self_k_idxs; }\n\n    ggml_tensor * get_kq_mask() const { return self_kq_mask_cnv; }\n\n    ggml_tensor * self_k_idxs = nullptr; // I64 [n_batch]\n\n    ggml_tensor * self_kq_mask     = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_cnv = nullptr; //     [n_kv, n_batch/n_stream, 1, n_stream]\n\n    const llama_hparams hparams;\n    const llama_cparams cparams;\n\n    const llama_kv_cache_context * mctx;\n};\n\nclass llm_graph_input_attn_kv_iswa : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_kv_iswa(\n            const llama_hparams & hparams,\n            const llama_cparams & cparams,\n            const llama_kv_cache_iswa_context * mctx) :\n        hparams(hparams),\n        cparams(cparams),\n        mctx(mctx) {\n    }\n    ~llm_graph_input_attn_kv_iswa() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    ggml_tensor * get_k_idxs()     const { return self_k_idxs; }\n    ggml_tensor * get_v_idxs()     const { return self_v_idxs; }\n    ggml_tensor * get_k_idxs_swa() const { return self_k_idxs_swa; }\n    ggml_tensor * get_v_idxs_swa() const { return self_v_idxs_swa; }\n\n    ggml_tensor * get_kq_mask()     const { return self_kq_mask_cnv; }\n    ggml_tensor * get_kq_mask_swa() const { return self_kq_mask_swa_cnv; }\n\n    ggml_tensor * self_k_idxs     = nullptr; // I64 [n_batch]\n    ggml_tensor * self_v_idxs     = nullptr; // I64 [n_batch] or [n_batch*n_embd_v_gqa]\n    ggml_tensor * self_k_idxs_swa = nullptr; // I64 [n_batch]\n    ggml_tensor * self_v_idxs_swa = nullptr; // I64 [n_batch] or [n_batch*n_embd_v_gqa]\n\n    ggml_tensor * self_kq_mask         = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_cnv     = nullptr; //     [n_kv, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_swa     = nullptr; // F32 [n_kv, n_batch/n_stream, 1, n_stream]\n    ggml_tensor * self_kq_mask_swa_cnv = nullptr; //     [n_kv, n_batch/n_stream, 1, n_stream]\n\n    const llama_hparams hparams;\n    const llama_cparams cparams;\n\n    const llama_kv_cache_iswa_context * mctx;\n};\n\nclass llm_graph_input_attn_cross : public llm_graph_input_i {\npublic:\n    llm_graph_input_attn_cross(const llama_cross * cross) : cross(cross) {}\n    ~llm_graph_input_attn_cross() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * get_kq_mask_cross() const { return cross_kq_mask_cnv; }\n\n    ggml_tensor * cross_kq_mask     = nullptr; // F32 [n_outputs_enc, n_batch, 1, 1]\n    ggml_tensor * cross_kq_mask_cnv = nullptr; // F32 [n_outputs_enc, n_batch, 1, 1]\n\n    const llama_cross * cross = nullptr;\n};\n\nclass llm_graph_input_mem_hybrid : public llm_graph_input_i {\npublic:\n    llm_graph_input_mem_hybrid(\n            const llama_cparams & cparams,\n            std::unique_ptr<llm_graph_input_attn_kv> inp_attn,\n            std::unique_ptr<llm_graph_input_rs>      inp_rs,\n            const llama_memory_hybrid_context *      mctx) :\n        inp_attn(std::move(inp_attn)),\n        inp_rs(std::move(inp_rs)),\n        cparams(cparams),\n        mctx(mctx) { }\n    virtual ~llm_graph_input_mem_hybrid() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    std::unique_ptr<llm_graph_input_attn_kv> inp_attn;\n    std::unique_ptr<llm_graph_input_rs>      inp_rs;\n\n    llm_graph_input_attn_kv * get_attn() const { return inp_attn.get(); }\n    llm_graph_input_rs      * get_recr() const { return inp_rs.get(); }\n\n    const llama_cparams cparams;\n\n    const llama_memory_hybrid_context * mctx;\n};\n\nclass llm_graph_input_mem_hybrid_k : public llm_graph_input_i {\npublic:\n    llm_graph_input_mem_hybrid_k(\n            const llama_cparams & cparams,\n            std::unique_ptr<llm_graph_input_attn_k> inp_attn,\n            std::unique_ptr<llm_graph_input_rs>      inp_rs,\n            const llama_memory_hybrid_context *      mctx) :\n        inp_attn(std::move(inp_attn)),\n        inp_rs(std::move(inp_rs)),\n        cparams(cparams),\n        mctx(mctx) { }\n    virtual ~llm_graph_input_mem_hybrid_k() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    std::unique_ptr<llm_graph_input_attn_k> inp_attn;\n    std::unique_ptr<llm_graph_input_rs>      inp_rs;\n\n    llm_graph_input_attn_k * get_attn() const { return inp_attn.get(); }\n    llm_graph_input_rs      * get_recr() const { return inp_rs.get(); }\n\n    const llama_cparams cparams;\n\n    const llama_memory_hybrid_context * mctx;\n};\n\nclass llm_graph_input_mem_hybrid_iswa : public llm_graph_input_i {\npublic:\n    llm_graph_input_mem_hybrid_iswa(\n            const llama_cparams & cparams,\n            std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn,\n            std::unique_ptr<llm_graph_input_rs>          inp_rs,\n            const llama_memory_hybrid_iswa_context *     mctx) :\n        inp_attn(std::move(inp_attn)),\n        inp_rs(std::move(inp_rs)),\n        cparams(cparams),\n        mctx(mctx) { }\n    virtual ~llm_graph_input_mem_hybrid_iswa() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    bool can_reuse(const llm_graph_params & params) override;\n\n    std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn;\n    std::unique_ptr<llm_graph_input_rs>          inp_rs;\n\n    llm_graph_input_attn_kv_iswa * get_attn() const { return inp_attn.get(); }\n    llm_graph_input_rs           * get_recr() const { return inp_rs.get(); }\n\n    const llama_cparams cparams;\n\n    const llama_memory_hybrid_iswa_context * mctx;\n};\n\nclass llm_graph_input_sampling : public llm_graph_input_i {\npublic:\n    llm_graph_input_sampling(std::map<llama_seq_id, llama_sampler *> samplers) :\n        samplers(std::move(samplers)) { }\n    virtual ~llm_graph_input_sampling() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n    bool can_reuse(const llm_graph_params & params) override;\n\n    std::map<llama_seq_id, llama_sampler *> samplers;\n};\n\n//\n// llm_graph_result\n//\n\n// these objects deliver the result from the graph build process back to the llama_context\n// note that the input tensors created for the graph are referenced here - the goal is to be able to populate their\n//   specific data, by calling the set_inputs() method\n// along with the input tensors, the object also provides commonly used outputs tensors, such as logits, embeddings, etc.\n//   these are used by the llama_context to extact the relevant data, based on the compute parameters\n\n// callback that allows us to apply custom logic to each tensor (e.g. ggml-alloc, offloading, etc.)\nusing llm_graph_cb = std::function<void(const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il)>;\n\nclass llm_graph_result;\n\nstruct llm_graph_params {\n    llm_arch arch = LLM_ARCH_UNKNOWN;\n\n    llama_hparams hparams;\n    llama_cparams cparams;\n\n    llama_ubatch ubatch; // note: intentionally make a copy\n\n    llm_graph_type gtype;\n\n    ggml_backend_sched_t sched;\n    ggml_backend_t backend_cpu;\n\n    const llama_adapter_cvec     * cvec;\n    const llama_adapter_loras    * loras;\n    const llama_memory_context_i * mctx;\n    const llama_cross            * cross;\n\n    std::map<llama_seq_id, llama_sampler *> samplers;\n\n    static bool samplers_equal(\n          const std::map<llama_seq_id, llama_sampler *> & lhs,\n          const std::map<llama_seq_id, llama_sampler *> & rhs) {\n        if (lhs.size() != rhs.size()) {\n            return false;\n        }\n        for (const auto & [seq_id, sampler] : lhs) {\n            auto it = rhs.find(seq_id);\n            if (it == rhs.end() || it->second != sampler) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    uint32_t n_outputs;\n\n    llm_graph_cb cb;\n\n    llm_graph_result * res;\n\n    // return true if the \"other\" params would result in a graph with the same topology as with the current params\n    //   having the same topology allows us to reuse the graph in some cases\n    bool allow_reuse(const llm_graph_params & other) const {\n        // first check the ubatch\n        bool can_reuse_ubatch =\n            ubatch.equal_seqs() == other.ubatch.equal_seqs() &&\n            ubatch.n_tokens     == other.ubatch.n_tokens &&\n            ubatch.n_seq_tokens == other.ubatch.n_seq_tokens &&\n            ubatch.n_seqs       == other.ubatch.n_seqs &&\n            ubatch.n_seqs_unq   == other.ubatch.n_seqs_unq &&\n            (\n                (!ubatch.token && !other.ubatch.token) ||\n                (!ubatch.embd  && !other.ubatch.embd)\n            );\n\n        // when we split the batch using \"equal_seqs\" we have to verify that the participating sequences are the same\n        //   the reason is because the set of attention streams would be different for different sequences\n        if (can_reuse_ubatch && ubatch.equal_seqs()) {\n            if (!ubatch.data) {\n                // if the old ubatch does not own it's data, then we cannot guarantee that it is still alive, and\n                //   therefore we cannot perform the sequence id check. normally should never happen\n                can_reuse_ubatch = false;\n            } else {\n                for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n                    can_reuse_ubatch &= ubatch.seq_id_unq[s] == other.ubatch.seq_id_unq[s];\n                }\n            }\n        }\n\n        if (!can_reuse_ubatch) {\n            return false;\n        }\n\n        if (n_outputs != other.n_outputs) {\n            return false;\n        }\n\n        if (!samplers_equal(samplers, other.samplers)) {\n            return false;\n        }\n\n        if (samplers.size() > 0) {\n            if (!ubatch.data || !other.ubatch.data) {\n                return false;\n            }\n\n            // check that the outputs are the same for all samplers\n            for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {\n                if (ubatch.output[i]    != other.ubatch.output[i] ||\n                    ubatch.seq_id[i][0] != other.ubatch.seq_id[i][0]) {\n                    return false;\n                }\n            }\n        }\n\n        return\n            cparams.embeddings  == other.cparams.embeddings  &&\n            cparams.causal_attn == other.cparams.causal_attn &&\n            arch  == other.arch  &&\n            gtype == other.gtype &&\n            cvec  == other.cvec  &&\n            loras == other.loras &&\n            cross == other.cross;\n    }\n};\n\nclass llm_graph_result {\npublic:\n    llm_graph_result(int64_t max_nodes);\n\n    virtual ~llm_graph_result() = default;\n\n    ggml_tensor * get_inp_tokens()  const { return t_inp_tokens; }\n    ggml_tensor * get_logits()      const { return t_logits; }\n    ggml_tensor * get_embd()        const { return t_embd; }\n    ggml_tensor * get_embd_pooled() const { return t_embd_pooled; }\n\n    ggml_cgraph  * get_gf()  const { return gf; }\n    ggml_context * get_ctx() const { return ctx_compute.get(); }\n\n    int64_t get_max_nodes() const;\n\n    void reset();\n\n    void set_inputs(const llama_ubatch * ubatch);\n    void set_outputs();\n\n    // try to update the existing graph result using the new graph parameters in order to reuse it\n    // this can only be done if we determine that the resulting graph using the new graph parameters\n    //   would be identical to the existing graph. in that case, we simply have to update the memory\n    //   contexts of the input tensors of the graph and we can reuse it for another computation\n    // return true if the graph was updated and can be reused\n    bool can_reuse(const llm_graph_params & params);\n\n    llm_graph_input_i * add_input(llm_graph_input_ptr input);\n\n    void set_params(const llm_graph_params & params);\n\n    // important graph nodes\n    ggml_tensor * t_inp_tokens  = nullptr;\n    ggml_tensor * t_inp_embd    = nullptr; // [n_embd_inp, n_tokens]\n    ggml_tensor * t_logits      = nullptr;\n    ggml_tensor * t_embd        = nullptr;\n    ggml_tensor * t_embd_pooled = nullptr;\n\n    std::map<llama_seq_id, ggml_tensor*> t_sampled_logits;\n    std::map<llama_seq_id, ggml_tensor*> t_candidates;\n    std::map<llama_seq_id, ggml_tensor*> t_sampled;\n    std::map<llama_seq_id, ggml_tensor*> t_sampled_probs;\n\n    std::vector<llm_graph_input_ptr> inputs;\n\n    ggml_context_ptr ctx_compute;\n\n    // memory buffers used to evaluate the model\n    std::vector<uint8_t> buf_compute_meta;\n\n    ggml_cgraph * gf;\n\n    int64_t max_nodes;\n\nprivate:\n    // keep a copy of the previous graph parameters\n    // we will use this to determine whether the graph can be reused by comparing them with the new parameters\n    // note: these are updated after constructing the new graph\n    llm_graph_params params;\n\n    // env: LLAMA_GRAPH_RESULT_DEBUG\n    int debug = 0;\n};\n\nusing llm_graph_result_ptr = std::unique_ptr<llm_graph_result>;\n\n//\n// llm_graph_context\n//\n\n// used in build_rs to properly order writes and avoid unnecessary copies\nusing llm_graph_get_rows_fn = std::function<ggml_tensor * (ggml_context *, ggml_tensor * states, ggml_tensor * ids)>;\n\nstruct llm_graph_context {\n    const llm_arch arch;\n\n    const llama_hparams & hparams;\n    const llama_cparams & cparams;\n    const llama_ubatch  & ubatch;\n\n    const int64_t n_embd;\n    const int64_t n_layer;\n    const int64_t n_rot;\n    const int64_t n_ctx;       // user-specified context size (can be different from n_ctx_train)\n    const int64_t n_head;\n    const int64_t n_head_kv;\n    const int64_t n_embd_head_k;\n    const int64_t n_embd_k_gqa;\n    const int64_t n_embd_head_v;\n    const int64_t n_embd_v_gqa;\n    const int64_t n_expert;\n    const int64_t n_expert_used;\n\n    const float freq_base;\n    const float freq_scale;\n    const float ext_factor;\n    const float attn_factor;\n    const float beta_fast;\n    const float beta_slow;\n    const float norm_eps;\n    const float norm_rms_eps;\n\n    const int64_t n_tokens;\n    const int64_t n_outputs;\n    const int32_t n_ctx_orig; // yarn\n\n    const enum llama_pooling_type pooling_type;\n    const enum llama_rope_type    rope_type;\n\n    ggml_backend_sched_t sched;\n\n    ggml_backend_t backend_cpu; // TODO: needed by build_attn_mha, figure out a way to remove?\n\n    const llama_adapter_cvec     * cvec;\n    const llama_adapter_loras    * loras;\n    const llama_memory_context_i * mctx;\n    const llama_cross            * cross;\n\n    std::map<llama_seq_id, llama_sampler *> samplers;\n\n    const llm_graph_cb & cb_func;\n\n    llm_graph_result * res;\n\n    ggml_context * ctx0 = nullptr;\n    ggml_cgraph  * gf   = nullptr;\n\n    llm_graph_context(const llm_graph_params & params);\n    virtual ~llm_graph_context() = default;\n\n    void cb(ggml_tensor * cur, const char * name, int il) const;\n\n    //\n    // common\n    //\n\n    ggml_tensor * build_cvec(\n             ggml_tensor * cur,\n                     int   il) const;\n\n    // do mat_mul, while optionally apply lora and per-tensor scale\n    ggml_tensor * build_lora_mm(\n              ggml_tensor * w,\n              ggml_tensor * cur,\n              ggml_tensor * w_s = nullptr) const;\n\n    // do mat_mul_id, while optionally apply lora\n    ggml_tensor * build_lora_mm_id(\n              ggml_tensor * w,   // ggml_tensor * as\n              ggml_tensor * cur, // ggml_tensor * b\n              ggml_tensor * ids) const;\n\n    ggml_tensor * build_norm(\n             ggml_tensor * cur,\n             ggml_tensor * mw,\n             ggml_tensor * mb,\n           llm_norm_type   type,\n                     int   il) const;\n\n    ggml_tensor * build_ffn(\n             ggml_tensor * cur,\n             ggml_tensor * up,\n             ggml_tensor * up_b,\n             ggml_tensor * up_s,\n             ggml_tensor * gate,\n             ggml_tensor * gate_b,\n             ggml_tensor * gate_s,\n             ggml_tensor * down,\n             ggml_tensor * down_b,\n             ggml_tensor * down_s,\n             ggml_tensor * act_scales,\n         llm_ffn_op_type   type_op,\n       llm_ffn_gate_type   type_gate,\n                     int   il) const;\n\n    // build MoE FFN without bias tensors\n    ggml_tensor * build_moe_ffn(\n             ggml_tensor * cur,\n             ggml_tensor * gate_inp,\n             ggml_tensor * up_exps,\n             ggml_tensor * gate_exps,\n             ggml_tensor * down_exps,\n             ggml_tensor * exp_probs_b,\n                 int64_t   n_expert,\n                 int64_t   n_expert_used,\n         llm_ffn_op_type   type_op,\n                    bool   norm_w,\n                   float   w_scale,\n            llama_expert_gating_func_type gating_op,\n                     int   il,\n             ggml_tensor * probs_in = nullptr,\n             ggml_tensor * gate_up_exps = nullptr,\n             ggml_tensor * up_exps_s = nullptr,\n             ggml_tensor * gate_exps_s = nullptr,\n             ggml_tensor * down_exps_s = nullptr) const;\n\n    ggml_tensor * build_moe_ffn(\n             ggml_tensor * cur,\n             ggml_tensor * gate_inp,\n             ggml_tensor * gate_inp_b,\n             ggml_tensor * up_exps,\n             ggml_tensor * up_exps_b,\n             ggml_tensor * gate_exps,\n             ggml_tensor * gate_exps_b,\n             ggml_tensor * down_exps,\n             ggml_tensor * down_exps_b,\n             ggml_tensor * exp_probs_b,\n                 int64_t   n_expert,\n                 int64_t   n_expert_used,\n         llm_ffn_op_type   type_op,\n                    bool   norm_w,\n                   float   w_scale,\n            llama_expert_gating_func_type gating_op,\n                     int   il,\n             ggml_tensor * probs_in = nullptr,\n             ggml_tensor * gate_up_exps = nullptr,\n             ggml_tensor * gate_up_exps_b = nullptr,\n             ggml_tensor * up_exps_s = nullptr,\n             ggml_tensor * gate_exps_s = nullptr,\n             ggml_tensor * down_exps_s = nullptr) const;\n\n    //\n    // inputs\n    //\n\n    ggml_tensor * build_inp_embd(ggml_tensor * tok_embd) const;\n    ggml_tensor * build_inp_pos() const;\n    ggml_tensor * build_inp_attn_scale() const;\n    ggml_tensor * build_inp_out_ids() const;\n    ggml_tensor * build_inp_mean() const;\n    ggml_tensor * build_inp_cls() const;\n\n    ggml_tensor * build_inp_cross_embd() const;\n    ggml_tensor * build_inp_pos_bucket_enc() const;\n    ggml_tensor * build_inp_pos_bucket_dec() const;\n    ggml_tensor * build_pos_bias(ggml_tensor * pos_bucket, ggml_tensor * attn_rel_b) const;\n\n    //\n    // attention\n    //\n\n    ggml_tensor * build_attn_mha(\n            ggml_tensor * q,       // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k,       // [n_embd_head_k, n_head_k, n_tokens]\n            ggml_tensor * v,       // [n_embd_head_v, n_head_v, n_tokens] (v_trans == false)\n            ggml_tensor * kq_b,\n            ggml_tensor * kq_mask,\n            ggml_tensor * sinks,   // [n_head_q]\n            ggml_tensor * v_mla,   // [n_embd_head_v_mla, n_embd_head_v, n_head_v]\n                  float   kq_scale,\n                    int   il) const;\n\n    llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const;\n\n    ggml_tensor * build_attn(\n            llm_graph_input_attn_no_cache * inp,\n            ggml_tensor * wo,\n            ggml_tensor * wo_b,\n            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]\n            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]\n            ggml_tensor * kq_b,\n            ggml_tensor * sinks, // [n_head_q]\n            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]\n                  float   kq_scale,\n                    int   il) const;\n\n    llm_graph_input_attn_kv * build_attn_inp_kv() const;\n\n    ggml_tensor * build_attn(\n            llm_graph_input_attn_kv * inp,\n            ggml_tensor * wo,\n            ggml_tensor * wo_b,\n            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]\n            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]\n            ggml_tensor * kq_b,\n            ggml_tensor * sinks, // [n_head_q]\n            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v] // TODO: remove\n                  float   kq_scale,\n                    int   il) const;\n\n    llm_graph_input_attn_k  * build_attn_inp_k() const;\n\n    ggml_tensor * build_attn(\n            llm_graph_input_attn_k * inp,\n            ggml_tensor * wo,\n            ggml_tensor * wo_b,\n            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]\n            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]\n            ggml_tensor * kq_b,\n            ggml_tensor * sinks, // [n_head_q]\n            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]\n                  float   kq_scale,\n                    int   il) const;\n\n    llm_graph_input_attn_kv_iswa * build_attn_inp_kv_iswa() const;\n\n    // note: if k_cur or v_cur are not provided, they will not be stored in the memory\n    ggml_tensor * build_attn(\n            llm_graph_input_attn_kv_iswa * inp,\n            ggml_tensor * wo,\n            ggml_tensor * wo_b,\n            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens] optional\n            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens] optional\n            ggml_tensor * kq_b,\n            ggml_tensor * sinks, // [n_head_q]\n            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]\n                  float   kq_scale,\n                    int   il) const;\n\n    llm_graph_input_attn_cross * build_attn_inp_cross() const;\n\n    ggml_tensor * build_attn(\n            llm_graph_input_attn_cross * inp,\n            ggml_tensor * wo,\n            ggml_tensor * wo_b,\n            ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]\n            ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]\n            ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]\n            ggml_tensor * kq_b,\n            ggml_tensor * sinks, // [n_head_q]\n            ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]\n                  float   kq_scale,\n                    int   il) const;\n\n    //\n    // recurrent\n    //\n\n    // TODO: move this implementation to llama_memory_recurrent.\n    //       this is analogous to llama_kv_cache::cpy_k / cpy_v\n    //       when moving, avoid passing `ggml_cgraph` - only pass `ggml_context`. would likely need to split the\n    //         implementation in 2 separate methods. the goal is to avoid calling `ggml_build_forward_expand` in\n    //         `llama_memory_recurrent`\n    ggml_tensor * build_rs(\n            ggml_tensor * s,\n            ggml_tensor * state_copy_main,\n            ggml_tensor * state_copy_extra,\n                int32_t   state_size,\n                int32_t   n_seqs,\n               uint32_t   n_rs,\n               uint32_t   rs_head,\n               uint32_t   rs_size,\n                int32_t   rs_zero,\n            const llm_graph_get_rows_fn & get_state_rows = ggml_get_rows) const;\n\n    llm_graph_input_rs * build_rs_inp() const;\n\n    ggml_tensor * build_rs(\n            llm_graph_input_rs * inp,\n            ggml_tensor * s,\n                int32_t   state_size,\n                int32_t   n_seqs,\n            const llm_graph_get_rows_fn & get_state_rows = ggml_get_rows) const;\n\n    ggml_tensor * build_rwkv_token_shift_load(\n        llm_graph_input_rs * inp,\n        const llama_ubatch & ubatch,\n                       int   il) const;\n\n    ggml_tensor * build_rwkv_token_shift_store(\n             ggml_tensor * token_shift,\n      const llama_ubatch & ubatch,\n                     int   il) const;\n    //\n    // hybrid\n    //\n\n    llm_graph_input_mem_hybrid * build_inp_mem_hybrid() const;\n    llm_graph_input_mem_hybrid_k * build_inp_mem_hybrid_k() const;\n\n    llm_graph_input_mem_hybrid_iswa * build_inp_mem_hybrid_iswa() const;\n\n    //\n    // pooling\n    //\n\n    void build_pooling(\n            ggml_tensor * cls,\n            ggml_tensor * cls_b,\n            ggml_tensor * cls_out,\n            ggml_tensor * cls_out_b,\n            ggml_tensor * cls_norm) const;\n\n    //\n    // sampling (backend sampling)\n    //\n\n    void build_sampling() const;\n\n    //\n    // dense (out)\n    //\n\n    void build_dense_out(\n            ggml_tensor * dense_2,\n            ggml_tensor * dense_2_b,\n            ggml_tensor * dense_3) const;\n};\n\n// TODO: better name\nint32_t llama_relative_position_bucket(llama_pos x, llama_pos y, uint64_t n_buckets, bool bidirectional);\n"
  },
  {
    "path": "examples/talk-llama/llama-hparams.cpp",
    "content": "#include \"llama-hparams.h\"\n\n#include \"ggml.h\"\n\n#include <algorithm>\n#include <cassert>\n\nvoid llama_hparams::set_swa_pattern(uint32_t n_pattern, bool dense_first) {\n    if (dense_first) {\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            swa_layers[il] = n_pattern == 0 || (il % n_pattern != 0);\n        }\n    } else {\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            swa_layers[il] = n_pattern == 0 || (il % n_pattern < (n_pattern - 1));\n        }\n    }\n}\n\nbool llama_hparams::is_swa_any() const {\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        if (swa_layers[il]) {\n            return true;\n        }\n    }\n\n    return false;\n}\n\nuint32_t llama_hparams::n_head(uint32_t il) const {\n    if (il < n_layer) {\n        return n_head_arr[il];\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_head_kv(uint32_t il) const {\n    if (il < n_layer) {\n        return n_head_kv_arr[il];\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_ff(uint32_t il) const {\n    if (il < n_layer) {\n        return n_ff_arr[il];\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_gqa(uint32_t il) const {\n    const uint32_t n_head    = this->n_head(il);\n    const uint32_t n_head_kv = this->n_head_kv(il);\n\n    if (n_head_kv == 0) {\n        return 0;\n    }\n\n    return n_head/n_head_kv;\n}\n\nuint32_t llama_hparams::n_rot(uint32_t il) const {\n    if (il < n_layer) {\n        return is_swa(il) ? n_rot_swa : n_rot_full;\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_embd_inp() const {\n    uint32_t n_embd_inp = n_embd;\n\n    if (n_deepstack_layers > 0) {\n        n_embd_inp += n_embd * n_deepstack_layers;\n    }\n\n    return n_embd_inp;\n}\n\nuint32_t llama_hparams::n_embd_out() const {\n    return n_embd_out_impl > 0 ? n_embd_out_impl : n_embd;\n}\n\nuint32_t llama_hparams::n_embd_head_k(uint32_t il) const {\n    if (il < n_layer) {\n        return is_swa(il) ? n_embd_head_k_swa : n_embd_head_k_full;\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_embd_head_v(uint32_t il) const {\n    if (il < n_layer) {\n        return is_swa(il) ? n_embd_head_v_swa : n_embd_head_v_full;\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nuint32_t llama_hparams::n_embd_k_gqa(uint32_t il) const {\n    const uint32_t n_head_kv = this->n_head_kv(il);\n\n    return n_embd_head_k(il) * n_head_kv;\n}\n\nuint32_t llama_hparams::n_embd_v_gqa(uint32_t il) const {\n    const uint32_t n_head_kv = this->n_head_kv(il);\n\n    return n_embd_head_v(il) * n_head_kv;\n}\n\nbool llama_hparams::is_n_embd_k_gqa_variable() const {\n    const uint32_t val = n_embd_k_gqa();\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        if (val != n_embd_k_gqa(il)) {\n            return true;\n        }\n    }\n\n    return false;\n}\n\nbool llama_hparams::is_n_embd_v_gqa_variable() const {\n    const uint32_t val = n_embd_v_gqa();\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        if (val != n_embd_v_gqa(il)) {\n            return true;\n        }\n    }\n\n    return false;\n}\n\nuint32_t llama_hparams::n_embd_k_gqa_max() const {\n    uint32_t val = n_embd_k_gqa();\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        val = std::max(val, n_embd_k_gqa(il));\n    }\n\n    return val;\n}\n\nuint32_t llama_hparams::n_embd_v_gqa_max() const {\n    uint32_t val = n_embd_v_gqa();\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        val = std::max(val, n_embd_v_gqa(il));\n    }\n\n    return val;\n}\n\nuint32_t llama_hparams::n_embd_r() const {\n    if (wkv_head_size != 0) {\n        // for RWKV models\n        return token_shift_count * n_embd;\n    }\n\n    if (n_shortconv_l_cache != 0) {\n        // for LFM2 models\n        return n_embd * (n_shortconv_l_cache - 1);\n    }\n\n    if (n_embd_head_kda != 0) {\n        // for Kimi KDA layers\n        // Conv state for Q, K, V: 3 * (d_conv - 1) * n_head * head_dim\n        const uint32_t d_inner = n_head() * n_embd_head_kda;  // 32 * 128 = 4096\n        return 3 * (ssm_d_conv > 0 ? ssm_d_conv - 1 : 3) * d_inner;\n    }\n\n    // TODO: maybe support other convolution strides than 1\n    // NOTE: since the first column of the conv_state is shifted out each time, it's not actually needed\n    // Corresponds to Mamba's conv_states size\n    return (ssm_d_conv > 0 ? ssm_d_conv - 1 : 0) * (ssm_d_inner + 2*ssm_n_group*ssm_d_state);\n}\n\nuint32_t llama_hparams::n_embd_s() const {\n    if (wkv_head_size != 0) {\n        // corresponds to RWKV's wkv_states size\n        return n_embd * wkv_head_size;\n    }\n\n    if (n_embd_head_kda != 0) {\n        // for Kimi KDA layers\n        // Full recurrent state: head_dim * head_dim * n_head\n        // h tensor shape for delta attention: [head_dim, head_dim, n_head]\n        return n_embd_head_kda * n_embd_head_kda * n_head();  // 128 * 128 * 32 = 524288\n    }\n\n    // corresponds to Mamba's ssm_states size\n    return ssm_d_state * ssm_d_inner;\n}\n\nbool llama_hparams::is_recurrent(uint32_t il) const {\n    if (il < n_layer) {\n        return recurrent_layer_arr[il];\n    }\n\n    GGML_ABORT(\"%s: il (%u) out of bounds (n_layer: %u)\\n\", __func__, il, n_layer);\n}\n\nuint32_t llama_hparams::n_pos_per_embd() const {\n    return rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE ? 4 : 1;\n}\n\nbool llama_hparams::is_swa(uint32_t il) const {\n    if (il < n_layer) {\n        return swa_layers[il];\n    }\n\n    GGML_ABORT(\"fatal error\");\n}\n\nbool llama_hparams::is_mla() const {\n    assert((n_embd_head_k_mla_impl == 0 && n_embd_head_v_mla_impl == 0) ||\n           (n_embd_head_k_mla_impl != 0 && n_embd_head_v_mla_impl != 0));\n\n    return n_embd_head_k_mla_impl != 0 && n_embd_head_v_mla_impl != 0;\n}\n\nuint32_t llama_hparams::n_embd_head_k_mla() const {\n    return is_mla() ? n_embd_head_k_mla_impl : n_embd_head_k();\n}\n\nuint32_t llama_hparams::n_embd_head_v_mla() const {\n    return is_mla() ? n_embd_head_v_mla_impl : n_embd_head_v();\n}\n\nbool llama_hparams::has_kv(uint32_t il) const {\n    if (n_layer_kv_from_start >= 0) {\n        if (il < (uint32_t) n_layer_kv_from_start) {\n            return true;\n        }\n\n        return false;\n    }\n\n    // by default, all layers have kv\n    return true;\n}\n\nuint32_t llama_hparams::n_layer_kv() const {\n    uint32_t res = 0;\n\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        if (has_kv(il)) {\n            res++;\n        }\n    }\n\n    return res;\n}\n\nbool llama_hparams::use_mrope() const {\n    return rope_sections[0] > 0 && rope_sections[1] > 0;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-hparams.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <array>\n#include <cassert>\n\n// bump if necessary\n#define LLAMA_MAX_LAYERS  512\n#define LLAMA_MAX_EXPERTS 512 // Qwen3 Next\n\nenum llama_expert_gating_func_type {\n    LLAMA_EXPERT_GATING_FUNC_TYPE_NONE           = 0,\n    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX        = 1,\n    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID        = 2,\n    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT = 3, // applied to the router weights instead of the logits\n};\n\nenum llama_swa_type {\n    LLAMA_SWA_TYPE_NONE      = 0,\n    LLAMA_SWA_TYPE_STANDARD  = 1,\n    LLAMA_SWA_TYPE_CHUNKED   = 2,\n    LLAMA_SWA_TYPE_SYMMETRIC = 3,\n};\n\nstruct llama_hparams_posnet {\n    uint32_t n_embd;\n    uint32_t n_layer;\n};\n\nstruct llama_hparams_convnext {\n    uint32_t n_embd;\n    uint32_t n_layer;\n};\n\nstruct llama_hparams {\n    bool vocab_only;\n    bool no_alloc;\n    bool rope_finetuned;\n    bool use_par_res;\n    bool swin_norm;\n\n    uint32_t n_ctx_train; // context size the model was trained on\n    uint32_t n_embd;\n    uint32_t n_layer;\n    int32_t n_layer_kv_from_start = -1; // if non-negative, the first n_layer_kv_from_start layers have KV cache\n    uint32_t n_expert = 0;\n    uint32_t n_expert_used = 0;\n    uint32_t n_rel_attn_bkts = 0;\n\n    // different head size for full_attention and SWA layers\n    uint32_t n_embd_head_k_full; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads\n    uint32_t n_embd_head_v_full; // dimension of values (d_v) aka n_embd_head\n    uint32_t n_embd_head_k_swa;\n    uint32_t n_embd_head_v_swa;\n\n    // different RoPE dimensions for full_attention and SWA layers\n    uint32_t n_rot_full;\n    uint32_t n_rot_swa;\n\n    // note: deepseek2 using MLA converts into MQA with larger heads, then decompresses to MHA\n    uint32_t n_embd_head_k_mla_impl = 0;\n    uint32_t n_embd_head_v_mla_impl = 0;\n\n    // for WavTokenizer\n    struct llama_hparams_posnet   posnet;\n    struct llama_hparams_convnext convnext;\n\n    uint32_t n_shortconv_l_cache  = 0;\n\n    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;\n    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;\n    std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;\n\n    uint32_t n_layer_dense_lead = 0;\n    uint32_t n_lora_q           = 0;\n    uint32_t n_lora_kv          = 0;\n    uint32_t n_ff_exp           = 0;\n    uint32_t n_ff_shexp         = 0;\n    uint32_t n_ff_chexp         = 0;\n    uint32_t n_expert_shared    = 0;\n    uint32_t n_norm_groups      = 0;\n    uint32_t n_expert_groups    = 0;\n    uint32_t n_group_used       = 0;\n    uint32_t n_group_experts    = 0;\n\n    float    expert_group_scale   = 0.05f;\n    float    expert_weights_scale = 0.0f;\n    bool     expert_weights_norm  = false;\n    uint32_t expert_gating_func   = LLAMA_EXPERT_GATING_FUNC_TYPE_NONE;\n    uint32_t moe_every_n_layers   = 0;\n    uint32_t moe_latent_size      = 0;\n    uint32_t nextn_predict_layers = 0;\n\n    float f_norm_eps;\n    float f_norm_rms_eps;\n    float f_norm_group_eps;\n\n    float f_attn_logit_softcapping   = 50.0f;\n    float f_router_logit_softcapping = 30.0f;\n    float f_final_logit_softcapping  = 30.0f;\n\n    // for RWKV\n    uint32_t rescale_every_n_layers = 0;\n    uint32_t time_mix_extra_dim     = 0;\n    uint32_t time_decay_extra_dim   = 0;\n    uint32_t wkv_head_size          = 0;\n    uint32_t token_shift_count      = 2;\n    uint32_t n_lora_decay           = 0;\n    uint32_t n_lora_iclr            = 0;\n    uint32_t n_lora_value_res_mix   = 0;\n    uint32_t n_lora_gate            = 0;\n\n    float    rope_attn_factor = 1.0f;\n    float    rope_freq_base_train;\n    float    rope_freq_base_train_swa  = 10000.0f;\n    float    rope_freq_scale_train;\n    float    rope_freq_scale_train_swa = 1.0f;\n\n    uint32_t n_ctx_orig_yarn;\n    float    rope_yarn_log_mul = 0.0f;\n\n    float    yarn_ext_factor  = -1.0f;\n    float    yarn_attn_factor =  1.0f;\n    float    yarn_beta_fast   = 32.0f;\n    float    yarn_beta_slow   =  1.0f;\n\n    std::array<int, 4> rope_sections;\n\n    // Sliding Window Attention (SWA)\n    llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;\n    // the size of the sliding window (0 - no SWA)\n    uint32_t n_swa = 0;\n    // if swa_layers[il] == 1, then layer il is SWA\n    // if swa_layers[il] == 0, then layer il is dense (i.e. non-SWA)\n    // by default, all layers are dense\n    // note: using uint32_t type for compatibility reason\n    std::array<uint32_t, LLAMA_MAX_LAYERS> swa_layers;\n\n    // for State Space Models\n    uint32_t ssm_d_conv  = 0;\n    uint32_t ssm_d_inner = 0;\n    uint32_t ssm_d_state = 0;\n    uint32_t ssm_dt_rank = 0;\n    uint32_t ssm_n_group = 0;\n\n    // for Kimi Linear KDA\n    uint32_t n_embd_head_kda = 0;\n\n    // for hybrid state space models\n    std::array<bool, LLAMA_MAX_LAYERS> recurrent_layer_arr;\n\n    bool ssm_dt_b_c_rms = false;\n\n    float f_clamp_kqv      = 0.0f;\n    float f_max_alibi_bias = 0.0f;\n    float f_logit_scale    = 0.0f;\n\n    // Additional scale factors (Granite/Granite MoE)\n    float f_residual_scale  = 0.0f;\n    float f_embedding_scale = 0.0f;\n    float f_attention_scale = 0.0f;\n\n    // grok-2\n    float    f_attn_out_scale = 0.0f;\n    uint32_t attn_temp_length = 0;\n\n    bool causal_attn   = true;\n    bool use_alibi     = false;\n    bool attn_soft_cap = false;\n    bool use_kq_norm   = false;\n\n    // for Classifiers\n    uint32_t n_cls_out = 1;\n\n    // output embedding dimension (0 = use n_embd)\n    uint32_t n_embd_out_impl = 0;\n\n    // llama4 smallthinker\n    uint32_t n_moe_layer_step        = 0;\n    uint32_t n_no_rope_layer_step    = 4;\n    uint32_t n_attn_temp_floor_scale = 0;\n    float    f_attn_temp_scale       = 0.0f;\n    float    f_attn_temp_offset      = 0.0f; // offset position index\n\n    // gemma3n altup\n    uint32_t n_altup      = 4; // altup_num_inputs\n    uint32_t i_altup_act  = 0; // altup_active_idx\n    uint32_t laurel_rank  = 64;\n    uint32_t n_embd_altup = 256;\n\n    // needed for sentence-transformers dense layers\n    uint32_t dense_2_feat_in  = 0;  // in_features of the 2_Dense\n    uint32_t dense_2_feat_out = 0;  // out_features of the 2_Dense\n    uint32_t dense_3_feat_in  = 0;  // in_features of the 3_Dense\n    uint32_t dense_3_feat_out = 0;  // out_features of the 3_Dense\n\n    // xIELU\n    std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_n;\n    std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_p;\n    std::array<float, LLAMA_MAX_LAYERS> xielu_beta;\n    std::array<float, LLAMA_MAX_LAYERS> xielu_eps;\n\n    // DSA (deepseek sparse attention)\n    uint32_t indexer_n_head    = 0;\n    uint32_t indexer_head_size = 0;\n    uint32_t indexer_top_k     = 0;\n\n    // qwen3vl deepstack\n    uint32_t n_deepstack_layers = 0;\n\n    // needed by encoder-decoder models (e.g. T5, FLAN-T5)\n    // ref: https://github.com/ggml-org/llama.cpp/pull/8141\n    llama_token dec_start_token_id = LLAMA_TOKEN_NULL;\n    uint32_t    dec_n_layer        = 0;\n\n    enum llama_pooling_type      pooling_type            = LLAMA_POOLING_TYPE_NONE;\n    enum llama_rope_type         rope_type               = LLAMA_ROPE_TYPE_NONE;\n    enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;\n\n\n    // Step35: optional per-layer clamps for (Swi)GLU\n    std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_exp; // clamping for expert FFN\n    std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_shexp; // shared expert\n\n    // this value n_pattern means that every nth layer is dense (i.e. non-SWA)\n    // dense_first means whether the pattern is start with a dense layer\n    // note that if n_pattern == 0, all layers are SWA\n    //           if n_pattern == 1, all layers are dense\n    // example 1: n_pattern = 3, dense_first = false\n    //   il == 0: swa\n    //   il == 1: swa\n    //   il == 2: dense\n    //   il == 3: swa\n    //   il == 4: swa\n    //   il == 5: dense\n    //   il == 6: swa\n    //   etc ...\n    // example 2: n_pattern = 2, dense_first = true\n    //   il == 0: dense\n    //   il == 1: swa\n    //   il == 2: dense\n    //   il == 3: swa\n    //   etc ...\n    void set_swa_pattern(uint32_t n_pattern, bool dense_first = false);\n\n    // return true if one of the layers is SWA\n    bool is_swa_any() const;\n\n    uint32_t n_head(uint32_t il = 0) const;\n\n    uint32_t n_head_kv(uint32_t il = 0) const;\n\n    uint32_t n_ff(uint32_t il = 0) const;\n\n    uint32_t n_gqa(uint32_t il = 0) const;\n\n    uint32_t n_rot(uint32_t il = 0) const;\n\n    // dimension of main + auxiliary input embeddings\n    uint32_t n_embd_inp() const;\n\n    // dimension of output embeddings\n    uint32_t n_embd_out() const;\n\n    // dimension of key/value embeddings for each head (per layer)\n    uint32_t n_embd_head_k(uint32_t il = 0) const;\n    uint32_t n_embd_head_v(uint32_t il = 0) const;\n\n    // dimension of key embeddings across all k-v heads\n    uint32_t n_embd_k_gqa(uint32_t il = 0) const;\n\n    // dimension of value embeddings across all k-v heads\n    uint32_t n_embd_v_gqa(uint32_t il = 0) const;\n\n    // true if any layer has a different n_embd_k_gqa/n_embd_v_gqa\n    bool is_n_embd_k_gqa_variable() const;\n    bool is_n_embd_v_gqa_variable() const;\n\n    // return the maximum n_embd_k_gqa/n_embd_v_gqa across all layers\n    uint32_t n_embd_k_gqa_max() const;\n    uint32_t n_embd_v_gqa_max() const;\n\n    // dimension of the rolling state embeddings\n    // corresponds to Mamba's conv_states size or RWKV's token_shift states size\n    uint32_t n_embd_r() const;\n\n    // dimension of the recurrent state embeddings\n    uint32_t n_embd_s() const;\n\n    // whether or not the given layer is recurrent (for hybrid models)\n    bool is_recurrent(uint32_t il) const;\n\n    uint32_t n_pos_per_embd() const;\n\n    bool is_swa(uint32_t il) const;\n\n    // note: currently only support if either all or none of the layers are MLA\n    bool is_mla() const;\n\n    uint32_t n_embd_head_k_mla() const;\n    uint32_t n_embd_head_v_mla() const;\n\n    bool has_kv(uint32_t il) const;\n\n    // number of layers for which has_kv() returns true\n    uint32_t n_layer_kv() const;\n\n    // note that this function uses different SWA parameters from those in the hparams\n    // note: inlined on purpose for performance reasons\n    // TODO: think of a better place for this function\n    // TODO: pack the SWA params in a struct?\n    static bool is_masked_swa(uint32_t n_swa, llama_swa_type swa_type, llama_pos p0, llama_pos p1) {\n        assert(p0 >= 0 && p1 >= 0);\n\n        switch (swa_type) {\n            case LLAMA_SWA_TYPE_NONE:\n                {\n                } break;\n            case LLAMA_SWA_TYPE_STANDARD:\n                {\n                    if (p1 - p0 >= (int32_t) n_swa) {\n                        return true;\n                    }\n                } break;\n            case LLAMA_SWA_TYPE_CHUNKED:\n                {\n                    const llama_pos pos_chunk_start = (p1 / n_swa) * n_swa;\n\n                    if (p0 < pos_chunk_start) {\n                        return true;\n                    }\n                } break;\n            case LLAMA_SWA_TYPE_SYMMETRIC:\n                {\n                    const int32_t half_n_swa = (int32_t) n_swa / 2;\n                    const int32_t pos_diff = p1 - p0;\n\n                    // Mask if outside the symmetric window\n                    if (pos_diff < -half_n_swa || pos_diff > half_n_swa) {\n                        return true;\n                    }\n                } break;\n        }\n\n        return false;\n    }\n\n\n    bool use_mrope() const;\n};\n\nstatic_assert(std::is_trivially_copyable<llama_hparams>::value, \"llama_hparams must be trivially copyable\");\n"
  },
  {
    "path": "examples/talk-llama/llama-impl.cpp",
    "content": "#include \"llama-impl.h\"\n\n#include \"gguf.h\"\n#include \"llama.h\"\n\n#include <cinttypes>\n#include <climits>\n#include <cstdarg>\n#include <cstring>\n#include <vector>\n#include <sstream>\n\nstruct llama_logger_state {\n    ggml_log_callback log_callback = llama_log_callback_default;\n    void * log_callback_user_data = nullptr;\n};\n\nstatic llama_logger_state g_logger_state;\n\ntime_meas::time_meas(int64_t & t_acc, bool disable) : t_start_us(disable ? -1 : ggml_time_us()), t_acc(t_acc) {}\n\ntime_meas::~time_meas() {\n    if (t_start_us >= 0) {\n        t_acc += ggml_time_us() - t_start_us;\n    }\n}\n\nvoid llama_log_get(ggml_log_callback * log_callback, void ** user_data) {\n    ggml_log_get(log_callback, user_data);\n}\n\nvoid llama_log_set(ggml_log_callback log_callback, void * user_data) {\n    ggml_log_set(log_callback, user_data);\n    g_logger_state.log_callback = log_callback ? log_callback : llama_log_callback_default;\n    g_logger_state.log_callback_user_data = user_data;\n}\n\nstatic void llama_log_internal_v(ggml_log_level level, const char * format, va_list args) {\n    va_list args_copy;\n    va_copy(args_copy, args);\n    char buffer[128];\n    int len = vsnprintf(buffer, 128, format, args);\n    if (len < 128) {\n        g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);\n    } else {\n        char * buffer2 = new char[len + 1];\n        vsnprintf(buffer2, len + 1, format, args_copy);\n        buffer2[len] = 0;\n        g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);\n        delete[] buffer2;\n    }\n    va_end(args_copy);\n}\n\nvoid llama_log_internal(ggml_log_level level, const char * format, ...) {\n    va_list args;\n    va_start(args, format);\n    llama_log_internal_v(level, format, args);\n    va_end(args);\n}\n\nvoid llama_log_callback_default(ggml_log_level level, const char * text, void * user_data) {\n    (void) level;\n    (void) user_data;\n    fputs(text, stderr);\n    fflush(stderr);\n}\n\nvoid replace_all(std::string & s, const std::string & search, const std::string & replace) {\n    if (search.empty()) {\n        return;\n    }\n    std::string builder;\n    builder.reserve(s.length());\n    size_t pos = 0;\n    size_t last_pos = 0;\n    while ((pos = s.find(search, last_pos)) != std::string::npos) {\n        builder.append(s, last_pos, pos - last_pos);\n        builder.append(replace);\n        last_pos = pos + search.length();\n    }\n    builder.append(s, last_pos, std::string::npos);\n    s = std::move(builder);\n}\n\nstd::string format(const char * fmt, ...) {\n    va_list ap;\n    va_list ap2;\n    va_start(ap, fmt);\n    va_copy(ap2, ap);\n    int size = vsnprintf(NULL, 0, fmt, ap);\n    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT\n    std::vector<char> buf(size + 1);\n    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);\n    GGML_ASSERT(size2 == size);\n    va_end(ap2);\n    va_end(ap);\n    return std::string(buf.data(), size);\n}\n\nstd::string llama_format_tensor_shape(const std::vector<int64_t> & ne) {\n    char buf[256];\n    snprintf(buf, sizeof(buf), \"%6\" PRId64, ne.at(0));\n    for (size_t i = 1; i < ne.size(); i++) {\n        snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), \", %6\" PRId64, ne.at(i));\n    }\n    return buf;\n}\n\nstd::string llama_format_tensor_shape(const struct ggml_tensor * t) {\n    char buf[256];\n    snprintf(buf, sizeof(buf), \"%6\" PRId64, t->ne[0]);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), \", %6\" PRId64, t->ne[i]);\n    }\n    return buf;\n}\n\nstatic std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {\n    switch (type) {\n        case GGUF_TYPE_UINT8:   return std::to_string(((const uint8_t  *)data)[i]);\n        case GGUF_TYPE_INT8:    return std::to_string(((const int8_t   *)data)[i]);\n        case GGUF_TYPE_UINT16:  return std::to_string(((const uint16_t *)data)[i]);\n        case GGUF_TYPE_INT16:   return std::to_string(((const int16_t  *)data)[i]);\n        case GGUF_TYPE_UINT32:  return std::to_string(((const uint32_t *)data)[i]);\n        case GGUF_TYPE_INT32:   return std::to_string(((const int32_t  *)data)[i]);\n        case GGUF_TYPE_UINT64:  return std::to_string(((const uint64_t *)data)[i]);\n        case GGUF_TYPE_INT64:   return std::to_string(((const int64_t  *)data)[i]);\n        case GGUF_TYPE_FLOAT32: return std::to_string(((const float    *)data)[i]);\n        case GGUF_TYPE_FLOAT64: return std::to_string(((const double   *)data)[i]);\n        case GGUF_TYPE_BOOL:    return ((const bool *)data)[i] ? \"true\" : \"false\";\n        default:                return format(\"unknown type %d\", type);\n    }\n}\n\nstd::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {\n    const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);\n\n    switch (type) {\n        case GGUF_TYPE_STRING:\n            return gguf_get_val_str(ctx_gguf, i);\n        case GGUF_TYPE_ARRAY:\n            {\n                const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);\n                int arr_n = gguf_get_arr_n(ctx_gguf, i);\n                const void * data = arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx_gguf, i);\n                std::stringstream ss;\n                ss << \"[\";\n                for (int j = 0; j < arr_n; j++) {\n                    if (arr_type == GGUF_TYPE_STRING) {\n                        std::string val = gguf_get_arr_str(ctx_gguf, i, j);\n                        // escape quotes\n                        replace_all(val, \"\\\\\", \"\\\\\\\\\");\n                        replace_all(val, \"\\\"\", \"\\\\\\\"\");\n                        ss << '\"' << val << '\"';\n                    } else if (arr_type == GGUF_TYPE_ARRAY) {\n                        ss << \"???\";\n                    } else {\n                        ss << gguf_data_to_str(arr_type, data, j);\n                    }\n                    if (j < arr_n - 1) {\n                        ss << \", \";\n                    }\n                }\n                ss << \"]\";\n                return ss.str();\n            }\n        default:\n            return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);\n    }\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-impl.h",
    "content": "#pragma once\n\n#include \"ggml.h\" // for ggml_log_level\n\n#include <string>\n#include <vector>\n\n#ifdef __GNUC__\n#    if defined(__MINGW32__) && !defined(__clang__)\n#        define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))\n#    else\n#        define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))\n#    endif\n#else\n#    define LLAMA_ATTRIBUTE_FORMAT(...)\n#endif\n\n//\n// logging\n//\n\nLLAMA_ATTRIBUTE_FORMAT(2, 3)\nvoid llama_log_internal        (ggml_log_level level, const char * format, ...);\nvoid llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);\n\n#define LLAMA_LOG(...)       llama_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)\n#define LLAMA_LOG_INFO(...)  llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)\n#define LLAMA_LOG_WARN(...)  llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)\n#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)\n#define LLAMA_LOG_DEBUG(...) llama_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)\n#define LLAMA_LOG_CONT(...)  llama_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)\n\n//\n// helpers\n//\n\ntemplate <typename T>\nstruct no_init {\n    T value;\n    no_init() = default;\n};\n\nstruct time_meas {\n    time_meas(int64_t & t_acc, bool disable = false);\n    ~time_meas();\n\n    const int64_t t_start_us;\n\n    int64_t & t_acc;\n};\n\ntemplate <typename T>\nstruct buffer_view {\n    T * data;\n    size_t size = 0;\n\n    bool has_data() const {\n        return data && size > 0;\n    }\n};\n\nvoid replace_all(std::string & s, const std::string & search, const std::string & replace);\n\n// TODO: rename to llama_format ?\nLLAMA_ATTRIBUTE_FORMAT(1, 2)\nstd::string format(const char * fmt, ...);\n\nstd::string llama_format_tensor_shape(const std::vector<int64_t> & ne);\nstd::string llama_format_tensor_shape(const struct ggml_tensor * t);\n\nstd::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i);\n\n#define LLAMA_TENSOR_NAME_FATTN   \"__fattn__\"\n#define LLAMA_TENSOR_NAME_FGDN_AR \"__fgdn_ar__\"\n#define LLAMA_TENSOR_NAME_FGDN_CH \"__fgdn_ch__\"\n"
  },
  {
    "path": "examples/talk-llama/llama-io.cpp",
    "content": "#include \"llama-io.h\"\n\nvoid llama_io_write_i::write_string(const std::string & str) {\n    uint32_t str_size = str.size();\n\n    write(&str_size,  sizeof(str_size));\n    write(str.data(), str_size);\n}\n\nvoid llama_io_read_i::read_string(std::string & str) {\n    uint32_t str_size;\n    read_to(&str_size, sizeof(str_size));\n\n    str.assign((const char *) read(str_size), str_size);\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-io.h",
    "content": "#pragma once\n\n#include <cstddef>\n#include <cstdint>\n#include <string>\n\nstruct ggml_tensor;\n\nclass llama_io_write_i {\npublic:\n    llama_io_write_i() = default;\n    virtual ~llama_io_write_i() = default;\n\n    virtual void write(const void * src, size_t size) = 0;\n    virtual void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) = 0;\n\n    // bytes written so far\n    virtual size_t n_bytes() = 0;\n\n    void write_string(const std::string & str);\n};\n\nclass llama_io_read_i {\npublic:\n    llama_io_read_i() = default;\n    virtual ~llama_io_read_i() = default;\n\n    virtual const uint8_t * read(size_t size) = 0;\n    virtual void read_to(void * dst, size_t size) = 0;\n\n    // bytes read so far\n    virtual size_t n_bytes() = 0;\n\n    void read_string(std::string & str);\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-kv-cache-iswa.cpp",
    "content": "#include \"llama-kv-cache-iswa.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-batch.h\"\n#include \"llama-model.h\"\n\n#include <algorithm>\n#include <cassert>\n\n//\n// llama_kv_cache_iswa\n//\n\nllama_kv_cache_iswa::llama_kv_cache_iswa(\n        const llama_model & model,\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                     bool   offload,\n                     bool   swa_full,\n                     bool   unified,\n                 uint32_t   kv_size,\n                 uint32_t   n_seq_max,\n                 uint32_t   n_ubatch,\n                 uint32_t   n_pad,\n    const layer_filter_cb & filter,\n    const  layer_reuse_cb & reuse) : hparams(model.hparams), unified(unified) {\n\n    // chain filters\n    const layer_filter_cb filter_base = [&](int32_t il) {\n        if (filter && !filter(il)) {\n            return false;\n        }\n\n        return !model.hparams.is_swa(il);\n    };\n\n    const layer_filter_cb filter_swa  = [&](int32_t il) {\n        if (filter && !filter(il)) {\n            return false;\n        }\n\n        return  model.hparams.is_swa(il);\n    };\n\n    const uint32_t size_base = kv_size;\n\n    // note: the SWA cache is always padded to 256 for performance\n    //       https://github.com/ggml-org/llama.cpp/issues/17037\n    uint32_t size_swa = GGML_PAD(std::min(size_base, hparams.n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256);\n\n    // when using full-size SWA cache, we set the SWA cache size to be equal to the base cache size\n    if (swa_full) {\n        LLAMA_LOG_WARN(\"%s: using full-size SWA cache (ref: %s)\\n\",\n                __func__, \"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055\");\n\n        size_swa = size_base;\n    }\n\n    LLAMA_LOG_INFO(\"%s: creating non-SWA KV cache, size = %u cells\\n\", __func__, size_base);\n\n    kv_base = std::make_unique<llama_kv_cache>(\n            model, type_k, type_v,\n            v_trans, offload, unified, size_base, n_seq_max, n_pad,\n            0, LLAMA_SWA_TYPE_NONE, filter_base, reuse);\n\n    LLAMA_LOG_INFO(\"%s: creating     SWA KV cache, size = %u cells\\n\", __func__, size_swa);\n\n    kv_swa = std::make_unique<llama_kv_cache>(\n            model, type_k, type_v,\n            v_trans, offload, unified, size_swa, n_seq_max, n_pad,\n            hparams.n_swa, hparams.swa_type, filter_swa, reuse);\n}\n\nvoid llama_kv_cache_iswa::clear(bool data) {\n    kv_base->clear(data);\n    kv_swa ->clear(data);\n}\n\nbool llama_kv_cache_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {\n    bool res = true;\n\n    res = res & kv_base->seq_rm(seq_id, p0, p1);\n    res = res & kv_swa ->seq_rm(seq_id, p0, p1);\n\n    return res;\n}\n\nvoid llama_kv_cache_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {\n    kv_base->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n    kv_swa ->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n}\n\nvoid llama_kv_cache_iswa::seq_keep(llama_seq_id seq_id) {\n    kv_base->seq_keep(seq_id);\n    kv_swa ->seq_keep(seq_id);\n}\n\nvoid llama_kv_cache_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {\n    kv_base->seq_add(seq_id, p0, p1, shift);\n    kv_swa ->seq_add(seq_id, p0, p1, shift);\n}\n\nvoid llama_kv_cache_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {\n    kv_base->seq_div(seq_id, p0, p1, d);\n    kv_swa ->seq_div(seq_id, p0, p1, d);\n}\n\nllama_pos llama_kv_cache_iswa::seq_pos_min(llama_seq_id seq_id) const {\n    // the base cache is a superset of the SWA cache, so we can just check the SWA cache\n    return kv_swa->seq_pos_min(seq_id);\n}\n\nllama_pos llama_kv_cache_iswa::seq_pos_max(llama_seq_id seq_id) const {\n    return kv_swa->seq_pos_max(seq_id);\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> mb = kv_base->memory_breakdown();\n    for (const auto & buft_size : kv_swa->memory_breakdown()) {\n        mb[buft_size.first] += buft_size.second;\n    }\n    return mb;\n}\n\nllama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {\n    GGML_UNUSED(embd_all);\n\n    // first try simple split\n    do {\n        if (!unified) {\n            // requires equal splits, so we skip the simple split\n            break;\n        }\n\n        balloc.split_reset();\n\n        std::vector<llama_ubatch> ubatches;\n        while (true) {\n            auto ubatch = balloc.split_simple(n_ubatch);\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        auto sinfos_base = kv_base->prepare(ubatches);\n        if (sinfos_base.empty()) {\n            break;\n        }\n\n        auto sinfos_swa = kv_swa->prepare(ubatches);\n        if (sinfos_swa.empty()) {\n            break;\n        }\n\n        assert(sinfos_base.size() == sinfos_swa.size());\n\n        return std::make_unique<llama_kv_cache_iswa_context>(\n                this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches));\n    } while (false);\n\n    // if it fails, try equal split\n    do {\n        balloc.split_reset();\n\n        std::vector<llama_ubatch> ubatches;\n        while (true) {\n            auto ubatch = balloc.split_equal(n_ubatch, !unified);\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        auto sinfos_base = kv_base->prepare(ubatches);\n        if (sinfos_base.empty()) {\n            break;\n        }\n\n        auto sinfos_swa = kv_swa->prepare(ubatches);\n        if (sinfos_swa.empty()) {\n            break;\n        }\n\n        assert(sinfos_base.size() == sinfos_swa.size());\n\n        return std::make_unique<llama_kv_cache_iswa_context>(\n                this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches));\n    } while (false);\n\n    // TODO: if we fail again, we should attempt different splitting strategies\n    //       but to do that properly, we first have to refactor the batches to be more flexible\n\n    return std::make_unique<llama_kv_cache_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n}\n\nllama_memory_context_ptr llama_kv_cache_iswa::init_full() {\n    return std::make_unique<llama_kv_cache_iswa_context>(this);\n}\n\nllama_memory_context_ptr llama_kv_cache_iswa::init_update(llama_context * lctx, bool optimize) {\n    return std::make_unique<llama_kv_cache_iswa_context>(this, lctx, optimize);\n}\n\nbool llama_kv_cache_iswa::get_can_shift() const {\n    return kv_base->get_can_shift() &&\n           kv_swa->get_can_shift() &&\n           kv_base->get_size() == kv_swa->get_size();\n}\n\nvoid llama_kv_cache_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {\n    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {\n        kv_base->state_write(io, seq_id, flags);\n    }\n\n    kv_swa->state_write(io, seq_id, flags);\n}\n\nvoid llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {\n        kv_base->state_read(io, seq_id, flags);\n    }\n\n    kv_swa->state_read(io, seq_id, flags);\n}\n\nllama_kv_cache * llama_kv_cache_iswa::get_base() const {\n    return kv_base.get();\n}\n\nllama_kv_cache * llama_kv_cache_iswa::get_swa() const {\n    return kv_swa.get();\n}\n\n//\n// llama_kv_cache_iswa_context\n//\n\nllama_kv_cache_iswa_context::llama_kv_cache_iswa_context(llama_memory_status status) : status(status) {}\n\nllama_kv_cache_iswa_context::llama_kv_cache_iswa_context(\n        llama_kv_cache_iswa * kv) :\n    ctx_base(kv->get_base()->init_full()),\n    ctx_swa (kv->get_swa ()->init_full()),\n    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {\n}\n\nllama_kv_cache_iswa_context::llama_kv_cache_iswa_context(\n        llama_kv_cache_iswa * kv,\n        llama_context * lctx,\n        bool optimize) :\n    ctx_base(kv->get_base()->init_update(lctx, optimize)),\n    ctx_swa (kv->get_swa ()->init_update(lctx, optimize)),\n    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {\n}\n\nllama_kv_cache_iswa_context::llama_kv_cache_iswa_context(\n        llama_kv_cache_iswa * kv,\n        slot_info_vec_t sinfos_base,\n        slot_info_vec_t sinfos_swa,\n        std::vector<llama_ubatch> ubatches) :\n    ubatches(std::move(ubatches)),\n    // note: here we copy the ubatches. not sure if this is ideal\n    ctx_base(new llama_kv_cache_context(kv->get_base(), std::move(sinfos_base), this->ubatches)),\n    ctx_swa (new llama_kv_cache_context(kv->get_swa (), std::move(sinfos_swa),  this->ubatches)),\n    status(llama_memory_status_combine(ctx_base->get_status(), ctx_swa->get_status())) {\n}\n\nllama_kv_cache_iswa_context:: ~llama_kv_cache_iswa_context() = default;\n\nbool llama_kv_cache_iswa_context::next() {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    ctx_base->next();\n    ctx_swa ->next();\n\n    if (++i_next >= ubatches.size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_kv_cache_iswa_context::apply() {\n    assert(!llama_memory_status_is_fail(status));\n\n    bool res = true;\n\n    res = res & ctx_base->apply();\n    res = res & ctx_swa ->apply();\n\n    return res;\n}\n\nllama_memory_status llama_kv_cache_iswa_context::get_status() const {\n    return status;\n}\n\nconst llama_ubatch & llama_kv_cache_iswa_context::get_ubatch() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    return ubatches[i_next];\n}\n\nconst llama_kv_cache_context * llama_kv_cache_iswa_context::get_base() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    return static_cast<const llama_kv_cache_context *>(ctx_base.get());\n}\n\nconst llama_kv_cache_context * llama_kv_cache_iswa_context::get_swa()  const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    return static_cast<const llama_kv_cache_context *>(ctx_swa.get());\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-kv-cache-iswa.h",
    "content": "#pragma once\n\n#include \"llama-kv-cache.h\"\n\n#include <vector>\n\n//\n// llama_kv_cache_iswa\n//\n\n// utilizes two instances of llama_kv_cache\n//   the first instance is for the non-SWA layers of the model and the second instance is for the SWA layers\n\nclass llama_kv_cache_iswa : public llama_memory_i {\npublic:\n    llama_kv_cache_iswa(\n            const llama_model & model,\n                    ggml_type   type_k,\n                    ggml_type   type_v,\n                         bool   v_trans,\n                         bool   offload,\n                         bool   swa_full,\n                         bool   unified,\n                     uint32_t   kv_size,\n                     uint32_t   n_seq_max,\n                     uint32_t   n_ubatch,\n                     uint32_t   n_pad,\n        const layer_filter_cb & filter,\n        const  layer_reuse_cb & reuse);\n\n    ~llama_kv_cache_iswa() = default;\n\n    //\n    // llama_memory_i\n    //\n\n    llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) override;\n\n    llama_memory_context_ptr init_full() override;\n\n    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;\n\n    bool get_can_shift() const override;\n\n    void clear(bool data) override;\n\n    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;\n    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;\n    void seq_keep(llama_seq_id seq_id)                                                          override;\n    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;\n    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;\n\n    llama_pos seq_pos_min(llama_seq_id seq_id) const override;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const override;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;\n\n    // state write/load\n\n    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;\n    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;\n\n    //\n    // llama_kv_cache_iswa specific API\n    //\n\n    llama_kv_cache * get_base() const;\n    llama_kv_cache * get_swa () const;\n\nprivate:\n    const llama_hparams & hparams;\n\n    const bool unified;\n\n    std::unique_ptr<llama_kv_cache> kv_base;\n    std::unique_ptr<llama_kv_cache> kv_swa;\n};\n\nclass llama_kv_cache_iswa_context : public llama_memory_context_i {\npublic:\n    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;\n\n    // used for errors\n    llama_kv_cache_iswa_context(llama_memory_status status);\n\n    // used to create a full-cache context\n    llama_kv_cache_iswa_context(\n            llama_kv_cache_iswa * kv);\n\n    // used to create an update context\n    llama_kv_cache_iswa_context(\n            llama_kv_cache_iswa * kv,\n            llama_context * lctx,\n            bool optimize);\n\n    // used to create a batch processing context from a batch\n    llama_kv_cache_iswa_context(\n            llama_kv_cache_iswa * kv,\n            slot_info_vec_t sinfos_base,\n            slot_info_vec_t sinfos_swa,\n            std::vector<llama_ubatch> ubatches);\n\n    virtual ~llama_kv_cache_iswa_context();\n\n    //\n    // llama_memory_context_i\n    //\n\n    bool next()  override;\n    bool apply() override;\n\n    llama_memory_status  get_status() const override;\n    const llama_ubatch & get_ubatch() const override;\n\n    //\n    // llama_kv_cache_iswa_context specific API\n    //\n\n    const llama_kv_cache_context * get_base() const;\n    const llama_kv_cache_context * get_swa()  const;\n\nprivate:\n    //llama_kv_cache_iswa * kv;\n\n    // the index of the next ubatch to process\n    size_t i_next = 0;\n\n    std::vector<llama_ubatch> ubatches;\n\n    const llama_memory_context_ptr ctx_base;\n    const llama_memory_context_ptr ctx_swa;\n\n    const llama_memory_status status;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-kv-cache.cpp",
    "content": "#include \"llama-kv-cache.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-io.h\"\n#include \"llama-model.h\"\n#include \"llama-context.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cmath>\n#include <cstring>\n#include <limits>\n#include <map>\n#include <stdexcept>\n\n//\n// llama_kv_cache\n//\n\nllama_kv_cache::llama_kv_cache(\n        const llama_model & model,\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                     bool   offload,\n                     bool   unified,\n                 uint32_t   kv_size,\n                 uint32_t   n_seq_max,\n                 uint32_t   n_pad,\n                 uint32_t   n_swa,\n           llama_swa_type   swa_type,\n    const layer_filter_cb & filter,\n    const  layer_reuse_cb & reuse) :\n    model(model), hparams(model.hparams), v_trans(v_trans),\n    n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type) {\n\n    GGML_ASSERT(kv_size % n_pad == 0);\n\n    const uint32_t n_layer_kv = hparams.n_layer_kv();\n\n    // define a comparator for the buft -> ctx map to ensure that the order is well-defined:\n    struct ggml_backend_buft_comparator {\n        bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {\n            return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;\n        }\n    };\n    std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator> ctx_map;\n\n    // create a context for each buffer type\n    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {\n        auto it = ctx_map.find(buft);\n        if (it == ctx_map.end()) {\n            ggml_init_params params = {\n                /*.mem_size   =*/ size_t(2u*(1 + n_stream)*n_layer_kv*ggml_tensor_overhead()),\n                /*.mem_buffer =*/ NULL,\n                /*.no_alloc   =*/ true,\n            };\n\n            ggml_context * ctx = ggml_init(params);\n            if (!ctx) {\n                return nullptr;\n            }\n\n            ctx_map.emplace(buft, ctx);\n\n            return ctx;\n        }\n\n        return it->second.get();\n    };\n\n    GGML_ASSERT(n_stream == 1 || n_stream == n_seq_max);\n\n    v_heads.resize(n_stream);\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        v_heads[s] = 0;\n    }\n\n    v_cells.resize(n_stream);\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        v_cells[s].resize(kv_size);\n    }\n\n    // by default, all sequence ids are mapped to the 0th stream\n    seq_to_stream.resize(LLAMA_MAX_SEQ, 0);\n\n    if (n_stream > 1) {\n        seq_to_stream.resize(n_stream, 0);\n        for (uint32_t s = 0; s < n_stream; ++s) {\n            seq_to_stream[s] = s;\n        }\n    }\n\n    // [TAG_V_CACHE_VARIABLE]\n    if (v_trans && hparams.is_n_embd_v_gqa_variable()) {\n        LLAMA_LOG_WARN(\"%s: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to %d\\n\",\n                __func__, hparams.n_embd_v_gqa_max());\n    }\n\n    const bool is_mla = hparams.is_mla();\n\n    for (uint32_t il = 0; il < hparams.n_layer; il++) {\n        if (!hparams.has_kv(il)) {\n            LLAMA_LOG_DEBUG(\"%s: layer %3d: does not have KV cache\\n\", __func__, il);\n            continue;\n        }\n\n        if (filter && !filter(il)) {\n            LLAMA_LOG_DEBUG(\"%s: layer %3d: filtered\\n\", __func__, il);\n            continue;\n        }\n\n        // [TAG_V_CACHE_VARIABLE]\n        const uint32_t n_embd_k_gqa =            hparams.n_embd_k_gqa(il);\n        const uint32_t n_embd_v_gqa = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max();\n\n        const char * dev_name = \"CPU\";\n\n        ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type();\n\n        if (offload) {\n            auto * dev = model.dev_layer(il);\n            buft = ggml_backend_dev_buffer_type(dev);\n\n            dev_name = ggml_backend_dev_name(dev);\n        }\n\n        LLAMA_LOG_DEBUG(\"%s: layer %3d: dev = %s\\n\", __func__, il, dev_name);\n\n        ggml_context * ctx = ctx_for_buft(buft);\n        if (!ctx) {\n            throw std::runtime_error(\"failed to create ggml context for kv cache\");\n        }\n\n        const bool has_k = true;\n        const bool has_v = !is_mla;\n\n        ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gqa, kv_size, n_stream) : nullptr;\n        ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gqa, kv_size, n_stream) : nullptr;\n\n        has_k && ggml_format_name(k, \"cache_k_l%d\", il);\n        has_v && ggml_format_name(v, \"cache_v_l%d\", il);\n\n        std::vector<ggml_tensor *> k_stream;\n        std::vector<ggml_tensor *> v_stream;\n\n        for (uint32_t s = 0; s < n_stream; ++s) {\n            k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gqa, kv_size, k->nb[1], s*k->nb[2]) : nullptr);\n            v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr);\n        }\n\n        map_layer_ids[il] = layers.size();\n\n        layers.push_back({ il, k, v, k_stream, v_stream, });\n    }\n\n    if (reuse) {\n        LLAMA_LOG_DEBUG(\"%s: reusing layers:\\n\", __func__);\n\n        for (uint32_t il = 0; il < hparams.n_layer; il++) {\n            const int32_t il_reuse = reuse(il);\n\n            if (il_reuse < 0) {\n                LLAMA_LOG_DEBUG(\"%s: - layer %3d: no reuse\\n\", __func__, il);\n                continue;\n            }\n\n            if (filter && !filter(il)) {\n                LLAMA_LOG_DEBUG(\"%s: - layer %3d: filtered\\n\", __func__, il);\n                continue;\n            }\n\n            GGML_ASSERT(map_layer_ids.find(il_reuse) != map_layer_ids.end());\n\n            map_layer_ids[il] = map_layer_ids[il_reuse];\n\n            LLAMA_LOG_DEBUG(\"%s: - layer %3d: reuse layer %d, is_swa = %d\\n\", __func__, il, il_reuse, hparams.is_swa(il));\n        }\n    }\n\n    // allocate tensors and initialize the buffers to avoid NaNs in the padding\n    for (auto & [buft, ctx] : ctx_map) {\n        ggml_backend_buffer_t buf;\n        if (model.hparams.no_alloc) {\n            buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer\n            for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {\n                t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it\n            }\n        } else {\n            buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer\n        }\n        if (!buf) {\n            throw std::runtime_error(\"failed to allocate buffer for kv cache\");\n        }\n\n        LLAMA_LOG_INFO(\"%s: %10s KV buffer size = %8.2f MiB\\n\", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);\n\n        ggml_backend_buffer_clear(buf, 0);\n        ctxs_bufs.emplace_back(std::move(ctx), buf);\n    }\n\n    {\n        const size_t memory_size_k = size_k_bytes();\n        const size_t memory_size_v = size_v_bytes();\n\n        LLAMA_LOG_INFO(\"%s: size = %7.2f MiB (%6u cells, %3d layers, %2u/%u seqs), K (%s): %7.2f MiB, V (%s): %7.2f MiB\\n\", __func__,\n                (float)(memory_size_k + memory_size_v) / (1024.0f * 1024.0f), kv_size, (int) layers.size(), n_seq_max, n_stream,\n                ggml_type_name(type_k), (float)memory_size_k / (1024.0f * 1024.0f),\n                ggml_type_name(type_v), (float)memory_size_v / (1024.0f * 1024.0f));\n    }\n\n    const char * LLAMA_KV_CACHE_DEBUG = getenv(\"LLAMA_KV_CACHE_DEBUG\");\n    debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0;\n}\n\nvoid llama_kv_cache::clear(bool data) {\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        v_cells[s].reset();\n        v_heads[s] = 0;\n    }\n\n    if (data) {\n        for (auto & [_, buf] : ctxs_bufs) {\n            ggml_backend_buffer_clear(buf.get(), 0);\n        }\n    }\n}\n\nbool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {\n    GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()));\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    if (seq_id >= 0) {\n        auto & cells = v_cells[seq_to_stream[seq_id]];\n        auto & head  = v_heads[seq_to_stream[seq_id]];\n\n        uint32_t new_head = cells.size();\n\n        for (uint32_t i = 0; i < cells.size(); ++i) {\n            if (!cells.pos_in(i, p0, p1)) {\n                continue;\n            }\n\n            if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) {\n                if (new_head == cells.size()) {\n                    new_head = i;\n                }\n            }\n        }\n\n        // If we freed up a slot, set head to it so searching can start there.\n        if (new_head != cells.size() && new_head < head) {\n            head = new_head;\n        }\n    } else {\n        // match any sequence\n        for (uint32_t s = 0; s < n_stream; ++s) {\n            auto & cells = v_cells[s];\n            auto & head  = v_heads[s];\n\n            uint32_t new_head = cells.size();\n\n            for (uint32_t i = 0; i < cells.size(); ++i) {\n                if (!cells.pos_in(i, p0, p1)) {\n                    continue;\n                }\n\n                cells.rm(i);\n\n                if (new_head == cells.size()) {\n                    new_head = i;\n                }\n            }\n\n            // If we freed up a slot, set head to it so searching can start there.\n            if (new_head != cells.size() && new_head < head) {\n                head = new_head;\n            }\n        }\n    }\n\n    return true;\n}\n\nvoid llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {\n    GGML_ASSERT(seq_id_src >= 0 && (size_t) seq_id_src < seq_to_stream.size());\n    GGML_ASSERT(seq_id_dst >= 0 && (size_t) seq_id_dst < seq_to_stream.size());\n\n    const auto s0 = seq_to_stream[seq_id_src];\n    const auto s1 = seq_to_stream[seq_id_dst];\n\n    if (s0 == s1) {\n        // since both sequences are in the same stream, no data copy is necessary\n        // we just have to update the cells meta data\n\n        auto & cells = v_cells[s0];\n\n        if (seq_id_src == seq_id_dst) {\n            return;\n        }\n\n        if (p0 < 0) {\n            p0 = 0;\n        }\n\n        if (p1 < 0) {\n            p1 = std::numeric_limits<llama_pos>::max();\n        }\n\n        for (uint32_t i = 0; i < cells.size(); ++i) {\n            if (!cells.pos_in(i, p0, p1)) {\n                continue;\n            }\n\n            if (cells.seq_has(i, seq_id_src)) {\n                cells.seq_add(i, seq_id_dst);\n            }\n        }\n\n        return;\n    }\n\n    // cross-stream sequence copies require to copy the actual buffer data\n\n    bool is_full = true;\n\n    if (p0 > 0 && p0 + 1 < (int) get_size()) {\n        is_full = false;\n    }\n\n    if (p1 > 0 && p1 + 1 < (int) get_size()) {\n        is_full = false;\n    }\n\n    GGML_ASSERT(is_full && \"seq_cp() is only supported for full KV buffers\");\n\n    // enqueue the copy operation - the buffer copy will be performed during the next update\n    sc_info.ssrc.push_back(s0);\n    sc_info.sdst.push_back(s1);\n\n    v_cells[s1].reset();\n    for (uint32_t i = 0; i < v_cells[s0].size(); ++i) {\n        if (v_cells[s0].seq_has(i, seq_id_src)) {\n            llama_pos pos   = v_cells[s0].pos_get(i);\n            llama_pos shift = v_cells[s0].get_shift(i);\n\n            llama_kv_cell_ext ext = v_cells[s0].ext_get(i);\n\n            if (shift != 0) {\n                pos -= shift;\n                assert(pos >= 0);\n            }\n\n            v_cells[s1].pos_set(i, pos);\n            v_cells[s1].seq_add(i, seq_id_dst);\n\n            if (shift != 0) {\n                v_cells[s1].pos_add(i, shift);\n            }\n\n            v_cells[s1].ext_set(i, ext);\n        }\n    }\n\n    v_heads[s1] = v_heads[s0];\n\n    //for (uint32_t s = 0; s < n_stream; ++s) {\n    //    LLAMA_LOG_WARN(\"%s: seq %d: min = %d, max = %d\\n\", __func__, s, v_cells[s].seq_pos_min(s), v_cells[s].seq_pos_max(s));\n    //}\n}\n\nvoid llama_kv_cache::seq_keep(llama_seq_id seq_id) {\n    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());\n\n    auto & cells = v_cells[seq_to_stream[seq_id]];\n    auto & head  = v_heads[seq_to_stream[seq_id]];\n\n    uint32_t new_head = cells.size();\n\n    for (uint32_t i = 0; i < cells.size(); ++i) {\n        if (cells.seq_keep(i, seq_id)) {\n            if (new_head == cells.size()) {\n                new_head = i;\n            }\n        }\n    }\n\n    // If we freed up a slot, set head to it so searching can start there.\n    if (new_head != cells.size() && new_head < head) {\n        head = new_head;\n    }\n}\n\nvoid llama_kv_cache::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {\n    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());\n    GGML_ASSERT(hparams.n_pos_per_embd() == 1 && \"seq_add() is only supported for n_pos_per_embd() == 1\");\n\n    auto & cells = v_cells[seq_to_stream[seq_id]];\n    auto & head  = v_heads[seq_to_stream[seq_id]];\n\n    if (shift == 0) {\n        return;\n    }\n\n    uint32_t new_head = cells.size();\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    // If there is no range then return early to avoid looping over all cells.\n    if (p0 == p1) {\n        return;\n    }\n\n    for (uint32_t i = 0; i < cells.size(); ++i) {\n        if (!cells.pos_in(i, p0, p1)) {\n            continue;\n        }\n\n        if (cells.seq_has(i, seq_id)) {\n            if (cells.pos_add(i, shift)) {\n                if (new_head == cells.size()) {\n                    new_head = i;\n                }\n            }\n        }\n    }\n\n    // If we freed up a slot, set head to it so searching can start there.\n    // Otherwise we just start the next search from the beginning.\n    head = new_head != cells.size() ? new_head : 0;\n}\n\nvoid llama_kv_cache::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {\n    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());\n    GGML_ASSERT(hparams.n_pos_per_embd() == 1 && \"seq_div() is only supported for n_pos_per_embd() == 1\");\n\n    auto & cells = v_cells[seq_to_stream[seq_id]];\n\n    if (d == 1) {\n        return;\n    }\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    // If there is no range then return early to avoid looping over the cache.\n    if (p0 == p1) {\n        return;\n    }\n\n    for (uint32_t i = 0; i < cells.size(); ++i) {\n        if (!cells.pos_in(i, p0, p1)) {\n            continue;\n        }\n\n        if (cells.seq_has(i, seq_id)) {\n            cells.pos_div(i, d);\n        }\n    }\n}\n\nllama_pos llama_kv_cache::seq_pos_min(llama_seq_id seq_id) const {\n    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());\n\n    const auto & cells = v_cells[seq_to_stream[seq_id]];\n\n    return cells.seq_pos_min(seq_id);\n}\n\nllama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {\n    GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());\n\n    const auto & cells = v_cells[seq_to_stream[seq_id]];\n\n    return cells.seq_pos_max(seq_id);\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> ret;\n    for (const auto & [ctx, buf] : ctxs_bufs) {\n        ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());\n\n        if (hparams.no_alloc) {\n            GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);\n            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);\n        } else {\n            // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base\n            ret[buft] += ggml_backend_buffer_get_size(buf.get());\n        }\n    }\n\n    return ret;\n}\n\nllama_memory_context_ptr llama_kv_cache::init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) {\n    GGML_UNUSED(embd_all);\n\n    do {\n        balloc.split_reset();\n\n        std::vector<llama_ubatch> ubatches;\n        while (true) {\n            auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true);\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        auto sinfos = prepare(ubatches);\n        if (sinfos.empty()) {\n            break;\n        }\n\n        return std::make_unique<llama_kv_cache_context>(\n                this, std::move(sinfos), std::move(ubatches));\n    } while (false);\n\n    return std::make_unique<llama_kv_cache_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n}\n\nllama_memory_context_ptr llama_kv_cache::init_full() {\n    return std::make_unique<llama_kv_cache_context>(this);\n}\n\nllama_memory_context_ptr llama_kv_cache::init_update(llama_context * lctx, bool optimize) {\n    GGML_UNUSED(optimize);\n\n    bool do_shift = get_has_shift();\n\n    return std::make_unique<llama_kv_cache_context>(this, lctx, do_shift, std::move(sc_info));\n}\n\nllama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ubatch> & ubatches) {\n    llama_kv_cache::slot_info_vec_t res;\n\n    struct state_t {\n        slot_info sinfo; // slot info for the ubatch\n\n        std::vector<uint32_t> v_heads_old; // old positions of the heads, before placing the ubatch\n\n        std::vector<llama_kv_cells> v_cells; // copy of the old cells, before placing the ubatch\n    };\n\n    // remember the old state of the cells so we can restore it in the end\n    std::vector<state_t> states;\n\n    bool success = true;\n\n    for (const auto & ubatch : ubatches) {\n        // only find a suitable slot for the ubatch. don't modify the cells yet\n        const auto sinfo_new = find_slot(ubatch, false);\n        if (sinfo_new.empty()) {\n            success = false;\n            break;\n        }\n\n        // remember the position that we found\n        res.push_back(sinfo_new);\n\n        // store the old state of the cells in the recovery stack\n        {\n            state_t state = { sinfo_new, v_heads, {} };\n\n            for (uint32_t s = 0; s < sinfo_new.n_stream(); ++s) {\n                auto & cells = v_cells[sinfo_new.strm[s]];\n\n                state.v_cells.push_back(cells.cp(sinfo_new.idxs[s]));\n            }\n\n            states.push_back(std::move(state));\n        }\n\n        // now emplace the ubatch\n        apply_ubatch(sinfo_new, ubatch);\n    }\n\n    GGML_ASSERT(!states.empty() || !success);\n\n    // iterate backwards and restore the cells to their original state\n    for (auto it = states.rbegin(); it != states.rend(); ++it) {\n        const auto & sinfo = it->sinfo;\n\n        for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n            auto & cells = v_cells[sinfo.strm[s]];\n            auto & head  = v_heads[sinfo.strm[s]];\n\n            cells.set(sinfo.idxs[s], it->v_cells[s]);\n            head = it->v_heads_old[s];\n        }\n    }\n\n    if (!success) {\n        return {};\n    }\n\n    return res;\n}\n\nbool llama_kv_cache::update(llama_context * lctx, bool do_shift, const stream_copy_info & sc_info) {\n    bool updated = false;\n\n    auto * sched = lctx->get_sched();\n\n    if (!sc_info.empty()) {\n        assert(n_stream > 1 && \"stream copy should never happen with a single stream\");\n\n        llama_synchronize(lctx);\n\n        const size_t n_copy = sc_info.ssrc.size();\n\n        for (size_t i = 0; i < n_copy; ++i) {\n            const auto ssrc = sc_info.ssrc[i];\n            const auto sdst = sc_info.sdst[i];\n\n            assert(ssrc < n_stream);\n            assert(sdst < n_stream);\n\n            LLAMA_LOG_DEBUG(\"%s: copying KV buffer: stream %d to stream %d\\n\", __func__, ssrc, sdst);\n\n            assert(ssrc != sdst);\n\n            for (uint32_t il = 0; il < layers.size(); ++il) {\n                const auto & layer = layers[il];\n\n                ggml_backend_tensor_copy(layer.k_stream[ssrc], layer.k_stream[sdst]);\n\n                if (layer.v_stream[ssrc]) {\n                    ggml_backend_tensor_copy(layer.v_stream[ssrc], layer.v_stream[sdst]);\n                }\n            }\n        }\n    }\n\n    if (do_shift) {\n        if (!get_can_shift()) {\n            GGML_ABORT(\"The current KV cache / model configuration does not support K-shift\");\n        }\n\n        LLAMA_LOG_DEBUG(\"%s: applying K-shift\\n\", __func__);\n\n        // apply K-shift if needed\n        if (hparams.rope_type != LLAMA_ROPE_TYPE_NONE) {\n            ggml_backend_sched_reset(sched);\n\n            auto * res = lctx->get_gf_res_reserve();\n\n            res->reset();\n\n            auto * gf = build_graph_shift(res, lctx);\n            if (!ggml_backend_sched_alloc_graph(sched, gf)) {\n                LLAMA_LOG_ERROR(\"%s: failed to allocate compute graph for K-shift\\n\", __func__);\n                return updated;\n            }\n\n            res->set_inputs(nullptr);\n\n            if (lctx->graph_compute(gf, false) != GGML_STATUS_SUCCESS) {\n                LLAMA_LOG_ERROR(\"%s: failed to compute K-shift\\n\", __func__);\n                return updated;\n            }\n\n            updated = true;\n        }\n\n        for (uint32_t s = 0; s < n_stream; ++s) {\n            auto & cells = v_cells[s];\n\n            cells.reset_shift();\n        }\n    }\n\n    return updated;\n}\n\nllama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, bool cont) const {\n\n    if (debug > 0) {\n        for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {\n            const auto seq_id = ubatch.seq_id_unq[s];\n            const auto stream_id = seq_to_stream[seq_id];\n            const auto & cells = v_cells[stream_id];\n            const uint32_t head_cur = v_heads[stream_id];\n\n            LLAMA_LOG_DEBUG(\"%s: stream[%d], n = %5d, used = %5d, head = %5d, size = %5d, n_swa = %5d\\n\",\n                    __func__, stream_id, cells.used_max_p1(), cells.get_used(), head_cur, get_size(), n_swa);\n\n            if ((debug == 2 && n_swa > 0) || debug > 2) {\n                std::string ss;\n                for (uint32_t i = 0; i < cells.size(); ++i) {\n                    if (cells.is_empty(i)) {\n                        ss += '.';\n                    } else {\n                        assert(cells.seq_count(i) >= 1);\n\n                        if (cells.seq_count(i) == 1) {\n                            ss += std::to_string(cells.seq_get(i));\n                        } else {\n                            ss += 'M';\n                        }\n                    }\n                    if (i%256 == 255) {\n                        ss += \" *\";\n                        ss += '\\n';\n                    }\n                }\n                LLAMA_LOG_DEBUG(\"\\n%s\\n\", ss.c_str());\n            }\n\n            if ((debug == 2 && n_swa > 0) || debug > 2) {\n                std::string ss;\n                for (uint32_t i = 0; i < cells.size(); ++i) {\n                    std::string cur;\n                    if (cells.is_empty(i)) {\n                        cur = '.';\n                    } else {\n                        cur = std::to_string(cells.pos_get(i));\n                    }\n                    const int n = cur.size();\n                    for (int j = 0; j < 5 - n; ++j) {\n                        cur += ' ';\n                    }\n                    ss += cur;\n                    if (i%256 == 255) {\n                        ss += \" *\";\n                    }\n                    if (i%64 == 63) {\n                        ss += '\\n';\n                    }\n                }\n                LLAMA_LOG_DEBUG(\"\\n%s\\n\", ss.c_str());\n            }\n\n            for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n                if (cells.seq_pos_min(s) < 0) {\n                    continue;\n                }\n\n                LLAMA_LOG_DEBUG(\"%s: stream[%d] min[%d] = %5d, max[%d] = %5d\\n\", __func__, stream_id, s, cells.seq_pos_min(s), s, cells.seq_pos_max(s));\n            }\n        }\n    }\n\n    uint32_t n_tokens = ubatch.n_tokens;\n    uint32_t n_seqs   = 1;\n\n    if (n_stream > 1) {\n        GGML_ASSERT(n_tokens % ubatch.n_seqs_unq == 0);\n\n        n_seqs   = ubatch.n_seqs_unq;\n        n_tokens = n_tokens / n_seqs;\n    }\n\n    slot_info res = {\n        /*.s0   =*/ LLAMA_MAX_SEQ,\n        /*.s1   =*/ 0,\n        /*.strm =*/ { },\n        /*.idxs =*/ { },\n    };\n\n    res.resize(n_seqs);\n\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        const auto seq_id = ubatch.seq_id_unq[s];\n\n        if (n_stream > 1) {\n            GGML_ASSERT(ubatch.n_seq_id[s*n_tokens]    == 1);\n            GGML_ASSERT(ubatch.seq_id  [s*n_tokens][0] == seq_id);\n        }\n\n        res.s0 = std::min<uint32_t>(res.s0, seq_to_stream[seq_id]);\n        res.s1 = std::max<uint32_t>(res.s1, seq_to_stream[seq_id]);\n\n        res.strm[s] = seq_to_stream[seq_id];\n        res.idxs[s].reserve(n_tokens);\n\n        const auto & cells = v_cells[seq_to_stream[seq_id]];\n\n        uint32_t head_cur = v_heads[seq_to_stream[seq_id]];\n\n        // if we have enough unused cells before the current head ->\n        //   better to start searching from the beginning of the cache, hoping to fill it\n        if (head_cur > cells.get_used() + 2*n_tokens) {\n            head_cur = 0;\n        }\n\n        if (n_tokens > cells.size()) {\n            LLAMA_LOG_ERROR(\"%s: n_tokens = %d > size = %u\\n\", __func__, n_tokens, cells.size());\n            return { };\n        }\n\n        uint32_t n_tested = 0;\n\n        // for continuous slots, we test that all tokens in the ubatch fit, starting from the current head\n        // for non-continuous slots, we test the tokens one by one\n        const uint32_t n_test = cont ? n_tokens : 1;\n\n        while (true) {\n            if (head_cur + n_test > cells.size()) {\n                n_tested += cells.size() - head_cur;\n                head_cur = 0;\n                continue;\n            }\n\n            for (uint32_t i = 0; i < n_test; i++) {\n                const auto idx = head_cur;\n\n                head_cur++;\n                n_tested++;\n\n                //const llama_pos    pos    = ubatch.pos[i];\n                //const llama_seq_id seq_id = ubatch.seq_id[i][0];\n\n                // can we use this cell? either:\n                //  - the cell is empty\n                //  - the cell is occupied only by one sequence:\n                //    - (disabled) mask causally, if the sequence is the same as the one we are inserting\n                //    - mask SWA, using current max pos for that sequence in the cache\n                //                always insert in the cell with minimum pos\n                bool can_use = cells.is_empty(idx);\n\n                if (!can_use && cells.seq_count(idx) == 1) {\n                    const llama_pos pos_cell = cells.pos_get(idx);\n\n                    // (disabled) causal mask\n                    // note: it's better to purge any \"future\" tokens beforehand\n                    //if (cells.seq_has(idx, seq_id)) {\n                    //    can_use = pos_cell >= pos;\n                    //}\n\n                    if (!can_use) {\n                        const llama_seq_id seq_id_cell = cells.seq_get(idx);\n\n                        // SWA mask\n                        if (llama_hparams::is_masked_swa(n_swa, swa_type, pos_cell, cells.seq_pos_max(seq_id_cell) + 1)) {\n                            can_use = true;\n                        }\n                    }\n                }\n\n                if (can_use) {\n                    res.idxs[s].push_back(idx);\n                } else {\n                    if (cont) {\n                        break;\n                    }\n                }\n            }\n\n            if (res.idxs[s].size() == n_tokens) {\n                break;\n            }\n\n            if (cont) {\n                res.idxs[s].clear();\n            }\n\n            if (n_tested >= cells.size()) {\n                //LLAMA_LOG_ERROR(\"%s: failed to find a slot for %d tokens\\n\", __func__, n_tokens);\n                return { };\n            }\n        }\n\n        // we didn't find a suitable slot - return empty result\n        if (res.idxs[s].size() < n_tokens) {\n            return { };\n        }\n    }\n\n    assert(res.s1 >= res.s0);\n\n    return res;\n}\n\nvoid llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch) {\n    // keep track of the max sequence position that we would overwrite with this ubatch\n    // for non-SWA cache, this would be always empty\n    llama_seq_id seq_pos_max_rm[LLAMA_MAX_SEQ];\n    for (uint32_t s = 0; s < LLAMA_MAX_SEQ; ++s) {\n        seq_pos_max_rm[s] = -1;\n    }\n\n    assert(ubatch.n_tokens == sinfo.n_stream()*sinfo.size());\n\n    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n        for (uint32_t ii = 0; ii < sinfo.size(); ++ii) {\n            const uint32_t i = s*sinfo.size() + ii;\n\n            auto & cells = v_cells[sinfo.strm[s]];\n\n            const auto idx = sinfo.idxs[s][ii];\n\n            if (!cells.is_empty(idx)) {\n                assert(cells.seq_count(idx) == 1);\n\n                const llama_seq_id seq_id = cells.seq_get(idx);\n                const llama_pos    pos    = cells.pos_get(idx);\n\n                seq_pos_max_rm[seq_id] = std::max(seq_pos_max_rm[seq_id], pos);\n\n                cells.rm(idx);\n            }\n\n            cells.pos_set(idx, ubatch.pos[i]);\n\n            if (ubatch.is_pos_2d()) {\n                llama_kv_cell_ext ext {\n                    /*.x =*/ ubatch.pos[i + ubatch.n_tokens*2],\n                    /*.y =*/ ubatch.pos[i + ubatch.n_tokens],\n                };\n                cells.ext_set(idx, ext);\n            }\n\n            for (int32_t s = 0; s < ubatch.n_seq_id[i]; s++) {\n                cells.seq_add(idx, ubatch.seq_id[i][s]);\n            }\n        }\n    }\n\n    // note: we want to preserve the invariant that all positions between [pos_min, pos_max] for each sequence\n    //       will be present in the cache. so we have to purge any position which is less than those we would overwrite\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/13746#issuecomment-2916057092\n    for (uint32_t s = 0; s < LLAMA_MAX_SEQ; ++s) {\n        if (seq_pos_max_rm[s] == -1) {\n            continue;\n        }\n\n        GGML_ASSERT(s < seq_to_stream.size());\n\n        auto & cells = v_cells[seq_to_stream[s]];\n\n        if (cells.seq_pos_min(s) <= seq_pos_max_rm[s]) {\n            LLAMA_LOG_DEBUG(\"%s: purging positions [%d, %d] of sequence %d from KV cache\\n\",\n                    __func__, cells.seq_pos_min(s), seq_pos_max_rm[s], s);\n\n            seq_rm(s, cells.seq_pos_min(s), seq_pos_max_rm[s] + 1);\n        }\n    }\n\n    // move the head at the end of the slot\n    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n        auto & head = v_heads[sinfo.strm[s]];\n\n        head = sinfo.idxs[s].back() + 1;\n    }\n}\n\nbool llama_kv_cache::get_can_shift() const {\n    // Step35 uses per-layer RoPE dims; K-shift assumes a single global n_rot.\n    if (model.arch == LLM_ARCH_STEP35) {\n        return false;\n    }\n    if (hparams.n_pos_per_embd() > 1) {\n        return false;\n    }\n    return true;\n}\n\nuint32_t llama_kv_cache::get_size() const {\n    const auto & cells = v_cells[seq_to_stream[0]];\n\n    return cells.size();\n}\n\nuint32_t llama_kv_cache::get_n_stream() const {\n    return n_stream;\n}\n\nbool llama_kv_cache::get_has_shift() const {\n    bool result = false;\n\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        result |= v_cells[s].get_has_shift();\n    }\n\n    return result;\n}\n\nuint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const {\n    uint32_t result = 0;\n\n    // pad the n_kv value so that the graph remains constant across batches and can be reused\n    // note: this also helps some backends with performance (f.ex https://github.com/ggml-org/llama.cpp/pull/16812#issuecomment-3455112220)\n    const uint32_t n_pad_cur = std::max(n_pad, 256u);\n\n    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n        const auto & cells = v_cells[sinfo.strm[s]];\n\n        result = std::max(std::min(cells.size(), std::max(n_pad_cur, GGML_PAD(cells.used_max_p1(), n_pad_cur))), result);\n    }\n\n    return result;\n}\n\nggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {\n    const int32_t ikv = map_layer_ids.at(il);\n\n    auto * k = layers[ikv].k;\n\n    const uint64_t kv_size      = get_size();\n    const uint64_t n_embd_k_gqa = k->ne[0];\n\n    assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il));\n\n    const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;\n\n    return ggml_view_4d(ctx, k,\n            hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns,\n            ggml_row_size(k->type, hparams.n_embd_head_k(il)),\n            ggml_row_size(k->type, n_embd_k_gqa),\n            ggml_row_size(k->type, n_embd_k_gqa*kv_size),\n            ggml_row_size(k->type, n_embd_k_gqa*kv_size)*sinfo.s0);\n}\n\nggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const {\n    const int32_t ikv = map_layer_ids.at(il);\n\n    auto * v = layers[ikv].v;\n\n    const uint64_t kv_size      = get_size();\n    const uint64_t n_embd_v_gqa = v->ne[0];\n\n    // [TAG_V_CACHE_VARIABLE]\n    assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il));\n\n    const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;\n\n    if (!v_trans) {\n        // note: v->nb[1] <= v->nb[2]\n        return ggml_view_4d(ctx, v,\n                hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, ns,\n                ggml_row_size(v->type, hparams.n_embd_head_v(il)),          // v->nb[1]\n                ggml_row_size(v->type, n_embd_v_gqa),                   // v->nb[2]\n                ggml_row_size(v->type, n_embd_v_gqa*kv_size),           // v->nb[3]\n                ggml_row_size(v->type, n_embd_v_gqa*kv_size)*sinfo.s0);\n    }\n\n    // note: v->nb[1] > v->nb[2]\n    return ggml_view_4d(ctx, v,\n            n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), ns,\n            ggml_row_size(v->type, kv_size*hparams.n_embd_head_v(il)),  // v->nb[1]\n            ggml_row_size(v->type, kv_size),                        // v->nb[2]\n            ggml_row_size(v->type, kv_size*n_embd_v_gqa),           // v->nb[3]\n            ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0);\n}\n\nggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {\n    GGML_UNUSED(sinfo);\n\n    const int32_t ikv = map_layer_ids.at(il);\n\n    ggml_tensor * k = layers[ikv].k;\n\n    const int64_t n_embd_head = k_cur->ne[0];\n    const int64_t n_head      = k_cur->ne[1];\n    const int64_t n_tokens    = k_cur->ne[2];\n\n    const int64_t n_embd_gqa = n_embd_head*n_head;\n\n    // we can merge dims 0 and 1\n    // TODO: add ggml helper function for this?\n    GGML_ASSERT(ggml_row_size(k_cur->type, n_embd_head) == k_cur->nb[1]);\n\n    k_cur = ggml_view_2d(ctx, k_cur, n_embd_gqa, n_tokens, k_cur->nb[2], 0);\n\n    const int64_t n_stream = k->ne[2];\n\n    if (n_stream > 1) {\n        const int64_t kv_size = get_size();\n\n        assert(n_embd_gqa == k->ne[0]);\n        assert(kv_size    == k->ne[1]);\n\n        // merge the buffer across all streams because the idxs are global\n        k = ggml_reshape_2d(ctx, k, n_embd_gqa, kv_size*n_stream);\n    }\n\n    // store the current K values into the cache\n    return ggml_set_rows(ctx, k, k_cur, k_idxs);\n}\n\nggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const {\n    GGML_UNUSED(sinfo);\n\n    const int32_t ikv = map_layer_ids.at(il);\n\n    auto * v = layers[ikv].v;\n\n    const int64_t n_embd_head = v_cur->ne[0];\n    const int64_t n_head      = v_cur->ne[1];\n    const int64_t n_tokens    = v_cur->ne[2];\n\n    const int64_t n_embd_gqa = n_embd_head*n_head;\n\n    // we can merge dims 0 and 1\n    GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]);\n\n    const int64_t n_stream = v->ne[2];\n\n    // take this branch when FA is enabled (the V cache is not transposed)\n    if (!v_trans) {\n        v_cur = ggml_view_2d(ctx, v_cur, n_embd_gqa, n_tokens, v_cur->nb[2], 0);\n\n        if (n_stream > 1) {\n            const int64_t kv_size = get_size();\n\n            assert(n_embd_gqa == v->ne[0]);\n            assert(kv_size    == v->ne[1]);\n\n            // merge the buffer across all streams because the idxs are global\n            v = ggml_reshape_2d(ctx, v, n_embd_gqa, kv_size*n_stream);\n        }\n\n        return ggml_set_rows(ctx, v, v_cur, v_idxs);\n    }\n\n    if (ggml_row_size(v_cur->type, n_embd_gqa) == v_cur->nb[2]) {\n        // we can merge dims 0, 1 and 2\n        v_cur = ggml_reshape_2d(ctx, v_cur, n_embd_gqa, n_tokens);\n    } else {\n        // otherwise -> make a copy to get contiguous data\n        v_cur = ggml_cont_2d   (ctx, v_cur, n_embd_gqa, n_tokens);\n    }\n\n    // [TAG_V_CACHE_VARIABLE]\n    if (n_embd_gqa < v->ne[0]) {\n        v_cur = ggml_pad(ctx, v_cur, v->ne[0] - n_embd_gqa, 0, 0, 0);\n    }\n\n    // in this branch the v_idxs are constructed in such a way that each row is a single head element\n    ggml_tensor * v_view = ggml_reshape_2d(ctx, v, 1, ggml_nelements(v));\n\n    v_cur = ggml_reshape_2d(ctx, v_cur, 1, ggml_nelements(v_cur));\n\n    return ggml_set_rows(ctx, v_view, v_cur, v_idxs);\n}\n\nggml_tensor * llama_kv_cache::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {\n    const uint32_t n_tokens = ubatch.n_tokens;\n\n    ggml_tensor * k_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens);\n\n    ggml_set_input(k_idxs);\n\n    return k_idxs;\n}\n\nggml_tensor * llama_kv_cache::build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {\n    const uint32_t n_tokens = ubatch.n_tokens;\n\n    ggml_tensor * v_idxs;\n\n    if (!v_trans) {\n        v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens);\n    } else {\n        v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens*hparams.n_embd_v_gqa_max());\n    }\n\n    ggml_set_input(v_idxs);\n\n    return v_idxs;\n}\n\nvoid llama_kv_cache::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const {\n    const uint32_t n_tokens = ubatch->n_tokens;\n    GGML_ASSERT(n_tokens == (int64_t) sinfo.size()*sinfo.n_stream());\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));\n    int64_t * data = (int64_t *) dst->data;\n\n    for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n        const int64_t offs = sinfo.strm[s]*get_size();\n\n        for (uint32_t i = 0; i < sinfo.size(); ++i) {\n            data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i];\n        }\n    }\n}\n\nvoid llama_kv_cache::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const {\n    const uint32_t n_tokens = ubatch->n_tokens;\n    GGML_ASSERT(n_tokens == (int64_t) sinfo.size()*sinfo.n_stream());\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));\n    int64_t * data = (int64_t *) dst->data;\n\n    if (!v_trans) {\n        for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n            const int64_t offs = sinfo.strm[s]*get_size();\n\n            for (uint32_t i = 0; i < sinfo.size(); ++i) {\n                data[s*sinfo.size() + i] = offs + sinfo.idxs[s][i];\n            }\n        }\n    } else {\n        // note: the V cache is transposed when not using flash attention\n        const int64_t kv_size = get_size();\n\n        const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa_max();\n\n        for (uint32_t s = 0; s < sinfo.n_stream(); ++s) {\n            const int64_t offs = sinfo.strm[s]*kv_size*n_embd_v_gqa;\n\n            for (uint32_t i = 0; i < sinfo.size(); ++i) {\n                for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {\n                    data[s*sinfo.size()*n_embd_v_gqa + i*n_embd_v_gqa + j] = offs + j*kv_size + sinfo.idxs[s][i];\n                }\n            }\n        }\n    }\n}\n\nvoid llama_kv_cache::set_input_k_shift(ggml_tensor * dst) const {\n    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));\n\n    int32_t * data = (int32_t *) dst->data;\n\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        const auto & cells = v_cells[s];\n\n        for (uint32_t i = 0; i < cells.size(); ++i) {\n            data[s*cells.size() + i] = cells.is_empty(i) ? 0 : cells.get_shift(i);\n        }\n    }\n}\n\nstruct args_set_input_kq_mask {\n    const llama_hparams & hparams;\n    const llama_ubatch  * ubatch;\n\n    const std::vector<llama_kv_cells> & v_cells;\n    const std::vector<uint32_t>       & seq_to_stream;\n\n    uint32_t       n_swa;\n    llama_swa_type swa_type;\n\n    int64_t n_kv;\n    int64_t n_stream;\n    int64_t n_tps;\n};\n\ntemplate<bool causal, bool swa, bool is_2d, bool alibi>\nstatic void set_input_kq_mask_impl(const args_set_input_kq_mask & args, float * data) {\n  //const auto & hparams = args.hparams;\n    const auto & ubatch  = args.ubatch;\n\n    const auto & v_cells       = args.v_cells;\n    const auto & seq_to_stream = args.seq_to_stream;\n\n    const uint32_t       n_swa    = args.n_swa;\n    const llama_swa_type swa_type = args.swa_type;\n\n    const int64_t n_kv     = args.n_kv;\n    const int64_t n_stream = args.n_stream;\n    const int64_t n_tps    = args.n_tps;\n\n    // the min position in the batch for each sequence\n    llama_pos seq_pos_min[LLAMA_MAX_SEQ];\n    std::fill(seq_pos_min, seq_pos_min + LLAMA_MAX_SEQ, INT32_MAX);\n\n    for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {\n        const llama_seq_id seq_id = ubatch->seq_id[i][0];\n\n        seq_pos_min[seq_id] = std::min(seq_pos_min[seq_id], ubatch->pos[i]);\n    }\n\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        // bookkeeping of the KQ mask cells that could change for other tokens of the same sequence\n        std::unordered_map<llama_seq_id, uint32_t>              seq_srct;\n        std::unordered_map<llama_seq_id, std::vector<uint32_t>> seq_idxs;\n\n        for (uint32_t ii = 0; ii < n_tps; ++ii) {\n            const uint32_t i = s*n_tps + ii;\n\n            const llama_seq_id seq_id = ubatch->seq_id[i][0];\n\n            const auto & cells = v_cells.at(seq_to_stream[seq_id]);\n\n                  llama_pos p0 = -1;\n            const llama_pos p1 = ubatch->pos[i];\n\n            // for M-RoPE\n            const llama_pos p1_x = is_2d ? ubatch->pos[i + ubatch->n_tokens*2] : 0;\n            const llama_pos p1_y = is_2d ? ubatch->pos[i + ubatch->n_tokens]   : 0;\n\n            const uint64_t idst = n_kv*i;\n\n            // for tokens of the same sequence, the mask is mostly the same, so we can reuse it\n            // the only cells that could change are the ones that are with similar positions as the\n            //   ones in the batch (i.e. due to causal masking, SWA, etc.)\n            // keep track of those cells and shortcut the loop to save time\n            // note: this optimization is not compatible with Alibi position encoding\n            // ref:  https://github.com/ggml-org/llama.cpp/pull/18842\n            bool prev = false;\n\n            auto & idxs = seq_idxs[seq_id];\n\n            if (!alibi) {\n                if (seq_srct.find(seq_id) != seq_srct.end()) {\n                    const uint32_t srct = seq_srct[seq_id];\n\n                    const uint64_t idst_prev = n_kv*srct;\n\n                    std::copy(data + idst_prev, data + idst_prev + n_kv, data + idst);\n\n                    prev = true;\n                } else {\n                    idxs.clear();\n                    idxs.reserve(ubatch->n_tokens + n_swa + 32);\n\n                    seq_srct[seq_id] = i;\n                }\n            }\n\n            for (uint32_t jj = 0; jj < n_kv; ++jj) {\n                uint32_t j = jj;\n\n                // we have an exiting mask for this sequence -> update just seq_idxs\n                if (!alibi) {\n                    if (prev) {\n                        if (jj >= idxs.size()) {\n                            break;\n                        }\n\n                        j = idxs[jj];\n                    }\n                }\n\n                if (cells.is_empty(j)) {\n                    goto skip;\n                }\n\n                // mask the token if not the same sequence\n                if (!cells.seq_has(j, seq_id)) {\n                    goto skip;\n                }\n\n                p0 = cells.pos_get(j);\n\n                if (!alibi) {\n                    if (!prev) {\n                        // record all cells for which: p0 >= seq_pos_min[seq_id] - n_swa - 32\n                        if (p0 + (int32_t) (n_swa + 32) >= seq_pos_min[seq_id]) {\n                            idxs.push_back(j);\n                        }\n                    }\n                }\n\n                if (causal) {\n                    // mask future tokens\n                    if (p0 > p1) {\n                        goto skip;\n                    }\n\n                    // M-RoPE causal mask\n                    if (is_2d) {\n                        if (p0 == p1) {\n                            const auto & p0_ext = cells.ext_get(j);\n\n                            if (p0_ext.is_2d_gt(p1_x, p1_y)) {\n                                goto skip;\n                            }\n                        }\n                    }\n                }\n\n                // apply SWA if any\n                if (swa) {\n                    if (llama_hparams::is_masked_swa(n_swa, swa_type, p0, p1)) {\n                        goto skip;\n                    }\n                }\n\n                if (alibi) {\n                    data[idst + j] = -std::abs(p0 - p1);\n                } else {\n                    data[idst + j] = 0.0f;\n                }\n\n                continue;\nskip:\n                data[idst + j] = -INFINITY;\n            }\n        }\n    }\n}\n\ntemplate<bool causal, bool swa, bool is_2d>\nstatic void set_input_kq_mask_impl(const args_set_input_kq_mask & args, float * data) {\n    const bool alibi = args.hparams.use_alibi;\n    if (alibi) {\n        set_input_kq_mask_impl<causal, swa, is_2d, true> (args, data);\n    } else {\n        set_input_kq_mask_impl<causal, swa, is_2d, false>(args, data);\n    }\n}\n\ntemplate<bool causal, bool swa>\nstatic void set_input_kq_mask_impl(const args_set_input_kq_mask & args, float * data) {\n    const bool is_2d = args.ubatch->is_pos_2d();\n    if (is_2d) {\n        set_input_kq_mask_impl<causal, swa, true> (args, data);\n    } else {\n        set_input_kq_mask_impl<causal, swa, false>(args, data);\n    }\n}\n\ntemplate<bool causal>\nstatic void set_input_kq_mask_impl(const args_set_input_kq_mask & args, float * data) {\n    const bool swa = args.swa_type != LLAMA_SWA_TYPE_NONE;\n    if (swa) {\n        set_input_kq_mask_impl<causal, true> (args, data);\n    } else {\n        set_input_kq_mask_impl<causal, false>(args, data);\n    }\n}\n\nvoid llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const {\n    const uint32_t n_tokens = ubatch->n_tokens;\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));\n    float * data = (float *) dst->data;\n\n    const int64_t n_kv     = dst->ne[0];\n    const int64_t n_stream = dst->ne[3]; // num streams in the current ubatch\n\n    GGML_ASSERT(n_tokens%n_stream == 0);\n\n    // n_tps == n_tokens_per_stream\n    const int64_t n_tps = n_tokens/n_stream;\n\n    //const int64_t t_start = ggml_time_us();\n\n    const args_set_input_kq_mask args = {\n        /*.hparams          =*/ hparams,\n        /*.ubatch           =*/ ubatch,\n        /*.v_cells          =*/ v_cells,\n        /*.seq_to_stream    =*/ seq_to_stream,\n        /*.n_swa            =*/ n_swa,\n        /*.swa_type         =*/ swa_type,\n        /*.n_kv             =*/ n_kv,\n        /*.n_stream         =*/ n_stream,\n        /*.n_tps            =*/ n_tps,\n    };\n\n    if (causal_attn) {\n        set_input_kq_mask_impl<true> (args, data);\n    } else {\n        set_input_kq_mask_impl<false>(args, data);\n    }\n\n    //const int64_t t_end = ggml_time_us();\n\n    //LLAMA_LOG_ERROR(\"%s: kq mask time: %0.3f ms\\n\", __func__, (t_end - t_start)/1000.0);\n}\n\nvoid llama_kv_cache::set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const {\n    const int64_t n_tokens = ubatch->n_tokens;\n\n    GGML_ASSERT(n_stream == 1 && \"TODO: support multiple streams\");\n    const auto & cells = v_cells[0];\n\n    GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));\n    GGML_ASSERT(!ubatch->equal_seqs()); // TODO: use ubatch->n_seqs instead of failing\n\n    int32_t * data = (int32_t *) dst->data;\n\n    const int32_t n_kv = dst->ne[0];\n\n    for (int h = 0; h < 1; ++h) {\n        for (int i = 0; i < n_tokens; ++i) {\n            for (int j = 0; j < n_kv; ++j) {\n                // the position when the cells is empty is irrelevant - it will be masked out later in the attention\n                const llama_pos p0 = cells.is_empty(j) ? -1 : cells.pos_get(j);\n\n                data[h*(n_kv*n_tokens) + i*n_kv + j] = llama_relative_position_bucket(p0, ubatch->pos[i], hparams.n_rel_attn_bkts, false);\n            }\n        }\n    }\n}\n\nsize_t llama_kv_cache::total_size() const {\n    size_t size = 0;\n\n    for (const auto & [_, buf] : ctxs_bufs) {\n        size += ggml_backend_buffer_get_size(buf.get());\n    }\n\n    return size;\n}\n\nsize_t llama_kv_cache::size_k_bytes() const {\n    size_t size_k_bytes = 0;\n\n    for (const auto & layer : layers) {\n        size_k_bytes += ggml_nbytes(layer.k);\n    }\n\n    return size_k_bytes;\n}\n\nsize_t llama_kv_cache::size_v_bytes() const {\n    size_t size_v_bytes = 0;\n\n    for (const auto & layer : layers) {\n        size_v_bytes += layer.v ? ggml_nbytes(layer.v) : 0;\n    }\n\n    return size_v_bytes;\n}\n\nggml_tensor * llama_kv_cache::build_rope_shift(\n        const llama_cparams & cparams,\n               ggml_context * ctx,\n                ggml_tensor * cur,\n                ggml_tensor * shift,\n                ggml_tensor * factors,\n                      float   freq_base,\n                      float   freq_scale,\n                   uint32_t   il) const {\n    const auto & n_ctx_orig = cparams.n_ctx_orig_yarn;\n\n    const auto & yarn_ext_factor  = cparams.yarn_ext_factor;\n    const auto & yarn_beta_fast   = cparams.yarn_beta_fast;\n    const auto & yarn_beta_slow   = cparams.yarn_beta_slow;\n    const auto & yarn_attn_factor = cparams.yarn_attn_factor;\n\n    const auto & n_rot     = hparams.n_rot(il);\n    const auto & rope_type = hparams.rope_type == LLAMA_ROPE_TYPE_MROPE || hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE\n                                // @ngxson : this is a workaround\n                                // for M-RoPE, we want to rotate the whole vector when doing KV shift\n                                // a normal RoPE should work, we just need to use the correct ordering\n                                // ref: https://github.com/ggml-org/llama.cpp/pull/13870\n                                ? LLAMA_ROPE_TYPE_NEOX\n                                : hparams.rope_type;\n\n    ggml_tensor * tmp;\n\n    if (ggml_is_quantized(cur->type)) {\n        // dequantize to f32 -> RoPE -> quantize back\n        tmp = ggml_cast(ctx, cur, GGML_TYPE_F32);\n\n        tmp = ggml_rope_ext(ctx, tmp,\n                shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow);\n\n        tmp = ggml_cpy(ctx, tmp, cur);\n    } else {\n        // we rotate only the first n_rot dimensions\n        tmp = ggml_rope_ext_inplace(ctx, cur,\n                shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow);\n    }\n\n    return tmp;\n}\n\nclass llm_graph_input_k_shift : public llm_graph_input_i {\npublic:\n    llm_graph_input_k_shift(const llama_kv_cache * kv_self) : kv_self(kv_self) {}\n    virtual ~llm_graph_input_k_shift() = default;\n\n    void set_input(const llama_ubatch * ubatch) override;\n\n    ggml_tensor * k_shift; // I32 [kv_size*n_stream]\n\n    const llama_kv_cache * kv_self;\n};\n\nvoid llm_graph_input_k_shift::set_input(const llama_ubatch * ubatch) {\n    GGML_UNUSED(ubatch);\n\n    if (k_shift) {\n        kv_self->set_input_k_shift(k_shift);\n    }\n}\n\nggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_context * lctx) const {\n    auto * ctx = res->get_ctx();\n    auto * gf  = res->get_gf();\n\n    auto inp = std::make_unique<llm_graph_input_k_shift>(this);\n\n    inp->k_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, (int64_t) get_size()*n_stream);\n    ggml_set_input(inp->k_shift);\n\n    const auto & cparams = lctx->get_cparams();\n\n    for (const auto & layer : layers) {\n        const uint32_t il = layer.il;\n\n        const int64_t n_head_kv    = hparams.n_head_kv(il);\n        const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);\n\n        const auto n_rot         = hparams.n_rot(il);\n        const auto n_embd_head_k = hparams.n_embd_head_k(il);\n        const auto n_embd_nope   = hparams.n_lora_kv > 0 ? n_embd_head_k - n_rot : 0;\n\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n        ggml_tensor * k =\n            ggml_view_3d(ctx, layer.k,\n                n_rot, n_head_kv, get_size()*n_stream,\n                ggml_row_size(layer.k->type, n_embd_head_k),\n                ggml_row_size(layer.k->type, n_embd_k_gqa),\n                ggml_row_size(layer.k->type, n_embd_nope));\n\n        ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, rope_factors, freq_base_l, freq_scale_l, il);\n\n        ggml_build_forward_expand(gf, cur);\n    }\n\n    res->add_input(std::move(inp));\n\n    return gf;\n}\n\nvoid llama_kv_cache::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {\n    GGML_UNUSED(flags);\n\n    io.write(&n_stream, sizeof(n_stream));\n\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        cell_ranges_t cr { s, {} };\n\n        uint32_t cell_count = 0;\n\n        const auto & cells = v_cells[s];\n\n        // Count the number of cells with the specified seq_id\n        // Find all the ranges of cells with this seq id (or all, when -1)\n        uint32_t cell_range_begin = cells.size();\n\n        for (uint32_t i = 0; i < cells.size(); ++i) {\n            if (!cells.is_empty(i) && (seq_id == -1 || cells.seq_has(i, seq_id))) {\n                ++cell_count;\n                if (cell_range_begin == cells.size()) {\n                    cell_range_begin = i;\n                }\n            } else {\n                if (cell_range_begin != cells.size()) {\n                    cr.data.emplace_back(cell_range_begin, i);\n                    cell_range_begin = cells.size();\n                }\n            }\n        }\n\n        if (cell_range_begin != cells.size()) {\n            cr.data.emplace_back(cell_range_begin, cells.size());\n        }\n\n        // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count\n        uint32_t cell_count_check = 0;\n        for (const auto & range : cr.data) {\n            cell_count_check += range.second - range.first;\n        }\n        GGML_ASSERT(cell_count == cell_count_check);\n\n        io.write(&cell_count, sizeof(cell_count));\n\n        // skip empty streams\n        if (cell_count == 0) {\n            continue;\n        }\n\n        state_write_meta(io, cr, seq_id);\n        state_write_data(io, cr);\n    }\n}\n\nvoid llama_kv_cache::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    GGML_UNUSED(flags);\n\n    GGML_ASSERT(seq_id == -1 || (seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()));\n\n    uint32_t n_stream_cur;\n    io.read_to(&n_stream_cur, sizeof(n_stream_cur));\n    if (n_stream_cur != n_stream) {\n        throw std::runtime_error(\"n_stream mismatch\");\n    }\n\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        uint32_t cell_count;\n        io.read_to(&cell_count, sizeof(cell_count));\n\n        if (cell_count == 0) {\n            continue;\n        }\n\n        const uint32_t strm = seq_id == -1 ? s : seq_to_stream[seq_id];\n\n        slot_info sinfo;\n\n        bool res = true;\n        res = res && state_read_meta(io, strm, cell_count, sinfo, seq_id);\n        res = res && state_read_data(io, strm, cell_count, sinfo);\n\n        if (!res) {\n            if (seq_id == -1) {\n                clear(true);\n            } else {\n                seq_rm(seq_id, -1, -1);\n            }\n            throw std::runtime_error(\"failed to restore kv cache\");\n        }\n    }\n}\n\nvoid llama_kv_cache::state_write_meta(llama_io_write_i & io, const cell_ranges_t & cr, llama_seq_id seq_id) const {\n    const auto & cells = v_cells[cr.strm];\n\n    for (const auto & range : cr.data) {\n        for (uint32_t i = range.first; i < range.second; ++i) {\n            std::vector<llama_seq_id> seq_ids;\n\n            for (llama_seq_id cur = 0; cur < (int) n_seq_max; ++cur) {\n                if (cur == seq_id || seq_id == -1) {\n                    if (cells.seq_has(i, cur)) {\n                        seq_ids.push_back(cur);\n                    }\n                }\n            }\n\n            const llama_pos pos     = cells.pos_get(i);\n            const uint32_t n_seq_id = seq_ids.size();\n\n            io.write(&pos,      sizeof(pos));\n            io.write(&n_seq_id, sizeof(n_seq_id));\n\n            if (hparams.n_pos_per_embd() > 1) {\n                const llama_kv_cell_ext ext = cells.ext_get(i);\n                io.write(&ext, sizeof(ext));\n            }\n\n            for (const auto & seq_id : seq_ids) {\n                io.write(&seq_id, sizeof(seq_id));\n            }\n        }\n    }\n}\n\nvoid llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t & cr) const {\n    const auto & cells = v_cells[cr.strm];\n\n    const uint32_t v_trans = this->v_trans ? 1 : 0;\n    const uint32_t n_layer = layers.size();\n\n    io.write(&v_trans, sizeof(v_trans));\n    io.write(&n_layer, sizeof(n_layer));\n\n    // Iterate and write all the keys first, each row is a cell\n    // Get whole range at a time\n    for (const auto & layer : layers) {\n        const uint32_t il = layer.il;\n\n        const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);\n\n        auto * k = layer.k_stream[cr.strm];\n\n        // Write key type\n        const int32_t k_type_i = (int32_t) k->type;\n        io.write(&k_type_i, sizeof(k_type_i));\n\n        // Write row size of key\n        const uint64_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa);\n        io.write(&k_size_row, sizeof(k_size_row));\n\n        // Read each range of cells of k_size length and write out\n        for (const auto & range : cr.data) {\n            const size_t range_size = range.second - range.first;\n            const size_t buf_size = range_size * k_size_row;\n            io.write_tensor(k, range.first * k_size_row, buf_size);\n        }\n    }\n\n    if (!v_trans) {\n        for (const auto & layer : layers) {\n            const uint32_t il = layer.il;\n\n            const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);\n\n            auto * v = layer.v_stream[cr.strm];\n            if (!v) {\n                continue;\n            }\n\n            // Write value type\n            const int32_t v_type_i = (int32_t) v->type;\n            io.write(&v_type_i, sizeof(v_type_i));\n\n            // Write row size of value\n            const uint64_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa);\n            io.write(&v_size_row, sizeof(v_size_row));\n\n            // Read each range of cells of v_size length and write out\n            for (const auto & range : cr.data) {\n                const size_t range_size = range.second - range.first;\n                const size_t buf_size = range_size * v_size_row;\n                io.write_tensor(v, range.first * v_size_row, buf_size);\n            }\n        }\n    } else {\n        // When v is transposed, we also need the element size and get the element ranges from each row\n        const uint32_t kv_size = cells.size();\n\n        for (const auto & layer : layers) {\n            const uint32_t il = layer.il;\n\n            const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);\n\n            auto * v = layer.v_stream[cr.strm];\n            if (!v) {\n                continue;\n            }\n\n            // Write value type\n            const int32_t v_type_i = (int32_t) v->type;\n            io.write(&v_type_i, sizeof(v_type_i));\n\n            // Write element size\n            const uint32_t v_size_el = ggml_type_size(v->type);\n            io.write(&v_size_el, sizeof(v_size_el));\n\n            // Write GQA embedding size\n            io.write(&n_embd_v_gqa, sizeof(n_embd_v_gqa));\n\n            // For each row, we get the element values of each cell\n            for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {\n                // Read each range of cells of v_size_el length and write out\n                for (const auto & range : cr.data) {\n                    const size_t range_size = range.second - range.first;\n                    const size_t src_offset = (range.first + j * kv_size) * v_size_el;\n                    const size_t buf_size = range_size * v_size_el;\n                    io.write_tensor(v, src_offset, buf_size);\n                }\n            }\n        }\n    }\n}\n\nbool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32_t cell_count, slot_info & sinfo, llama_seq_id dest_seq_id) {\n    auto & cells = v_cells[strm];\n    auto & head  = v_heads[strm];\n\n    if (dest_seq_id != -1) {\n        // single sequence\n        seq_rm(dest_seq_id, -1, -1);\n\n        llama_batch_allocr balloc(hparams.n_pos_per_embd());\n\n        llama_ubatch ubatch = balloc.ubatch_reserve(cell_count, 1);\n\n        ubatch.seq_id_unq[0] = dest_seq_id;\n\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            llama_pos pos;\n            uint32_t n_seq_id;\n\n            io.read_to(&pos,      sizeof(pos));\n            io.read_to(&n_seq_id, sizeof(n_seq_id));\n\n            if (n_seq_id != 1) {\n                LLAMA_LOG_ERROR(\"%s: invalid seq_id-agnostic kv cell\\n\", __func__);\n                return false;\n            }\n\n            if (hparams.n_pos_per_embd() > 1) {\n                llama_kv_cell_ext ext;\n                io.read_to(&ext, sizeof(ext));\n\n                ubatch.pos[i + ubatch.n_tokens]   = ext.y;\n                ubatch.pos[i + ubatch.n_tokens*2] = ext.x;\n            }\n\n            // read the sequence id, but directly discard it - we will use dest_seq_id instead\n            {\n                llama_seq_id seq_id;\n                io.read_to(&seq_id, sizeof(seq_id));\n            }\n\n            ubatch.pos[i]      = pos;\n            ubatch.n_seq_id[i] = n_seq_id;\n            ubatch.seq_id[i]   = &dest_seq_id;\n        }\n\n        sinfo = find_slot(ubatch, false);\n        if (sinfo.empty()) {\n            LLAMA_LOG_ERROR(\"%s: failed to find available cells in kv cache\\n\", __func__);\n            return false;\n        }\n\n        // TODO: we cannot yet restore llama_kv_cell_ext as the apply_ubatch() does not support it yet\n        //       see: https://github.com/ggml-org/llama.cpp/pull/16825#issuecomment-3460868350\n        apply_ubatch(sinfo, ubatch);\n\n        LLAMA_LOG_DEBUG(\"%s: cell_count = %d, dest_seq_id = %d\\n\", __func__, cell_count, dest_seq_id);\n\n        // DEBUG CHECK: verify that all cells were allocated and have correct seq_id and pos values\n        GGML_ASSERT(sinfo.n_stream() == 1);\n        GGML_ASSERT(sinfo.idxs[0].size() == cell_count);\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            const uint32_t idx = sinfo.idxs[0][i];\n            GGML_ASSERT(cells.pos_get(idx) == ubatch.pos[i]);\n            GGML_ASSERT(cells.seq_has(idx, dest_seq_id));\n        }\n    } else {\n        // whole KV cache restore\n\n        if (cell_count > cells.size()) {\n            LLAMA_LOG_ERROR(\"%s: not enough cells in kv cache\\n\", __func__);\n            return false;\n        }\n\n        clear(true);\n\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            llama_pos pos;\n            uint32_t  n_seq_id;\n\n            io.read_to(&pos,      sizeof(pos));\n            io.read_to(&n_seq_id, sizeof(n_seq_id));\n\n            cells.pos_set(i, pos);\n\n            if (hparams.n_pos_per_embd() > 1) {\n                llama_kv_cell_ext ext;\n                io.read_to(&ext, sizeof(ext));\n                cells.ext_set(i, ext);\n            }\n\n            for (uint32_t j = 0; j < n_seq_id; ++j) {\n                llama_seq_id seq_id;\n                io.read_to(&seq_id, sizeof(seq_id));\n\n                if (seq_id < 0 || (uint32_t) seq_id >= n_seq_max) {\n                    LLAMA_LOG_ERROR(\"%s: invalid seq_id, %d is out of range [0, %u)\\n\", __func__, seq_id, n_seq_max);\n                    return false;\n                }\n\n                cells.seq_add(i, seq_id);\n            }\n        }\n\n        // Create contiguous slot_info for whole cache restore\n        sinfo.s0 = strm;\n        sinfo.s1 = strm;\n        sinfo.resize(1);\n        sinfo.strm[0] = strm;\n        sinfo.idxs[0].resize(cell_count);\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            sinfo.idxs[0][i] = i;\n        }\n\n        head = 0;\n    }\n\n    return true;\n}\n\nbool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32_t cell_count, const slot_info & sinfo) {\n    auto & cells = v_cells[strm];\n\n    uint32_t v_trans;\n    uint32_t n_layer;\n\n    io.read_to(&v_trans, sizeof(v_trans));\n    io.read_to(&n_layer, sizeof(n_layer));\n\n    if (n_layer != layers.size()) {\n        LLAMA_LOG_ERROR(\"%s: mismatched layer count (%u instead of %u)\\n\", __func__, n_layer, (uint32_t) layers.size());\n        return false;\n    }\n\n    if (cell_count > cells.size()) {\n        LLAMA_LOG_ERROR(\"%s: not enough cells in kv cache to restore state (%u > %u)\\n\", __func__, cell_count, cells.size());\n        return false;\n    }\n\n    if (this->v_trans != (bool) v_trans) {\n        LLAMA_LOG_ERROR(\"%s: incompatible V transposition\\n\", __func__);\n        return false;\n    }\n\n    // For each layer, read the keys for each cell, one row is one cell, read as one contiguous block\n    for (const auto & layer : layers) {\n        const uint32_t il = layer.il;\n\n        const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);\n\n        auto * k = layer.k_stream[strm];\n\n        // Read type of key\n        int32_t k_type_i_ref;\n        io.read_to(&k_type_i_ref, sizeof(k_type_i_ref));\n        const int32_t k_type_i = (int32_t) k->type;\n        if (k_type_i != k_type_i_ref) {\n            LLAMA_LOG_ERROR(\"%s: mismatched key type (%d != %d, layer %d)\\n\", __func__, k_type_i, k_type_i_ref, il);\n            return false;\n        }\n\n        // Read row size of key\n        uint64_t k_size_row_ref;\n        io.read_to(&k_size_row_ref, sizeof(k_size_row_ref));\n        const size_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa);\n        if (k_size_row != k_size_row_ref) {\n            LLAMA_LOG_ERROR(\"%s: mismatched key row size (%zu != %zu, layer %d)\\n\", __func__, k_size_row, (size_t) k_size_row_ref, il);\n            return false;\n        }\n\n        if (cell_count) {\n            if (sinfo.is_contiguous()) {\n                // Fast path: contiguous cells, single memcpy\n                ggml_backend_tensor_set(k, io.read(cell_count * k_size_row), sinfo.head() * k_size_row, cell_count * k_size_row);\n            } else {\n                // Slow path: scatter to non-contiguous positions\n                const void * src = io.read(cell_count * k_size_row);\n                for (uint32_t i = 0; i < cell_count; ++i) {\n                    const size_t dst_offset = sinfo.idxs[0][i] * k_size_row;\n                    ggml_backend_tensor_set(k, (const char*)src + i * k_size_row, dst_offset, k_size_row);\n                }\n            }\n        }\n    }\n\n    if (!this->v_trans) {\n        for (const auto & layer : layers) {\n            const uint32_t il = layer.il;\n\n            const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);\n\n            auto * v = layer.v_stream[strm];\n            if (!v) {\n                continue;\n            }\n\n            // Read type of value\n            int32_t v_type_i_ref;\n            io.read_to(&v_type_i_ref, sizeof(v_type_i_ref));\n            const int32_t v_type_i = (int32_t) v->type;\n            if (v_type_i != v_type_i_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched value type (%d != %d, layer %d)\\n\", __func__, v_type_i, v_type_i_ref, il);\n                return false;\n            }\n\n            // Read row size of value\n            uint64_t v_size_row_ref;\n            io.read_to(&v_size_row_ref, sizeof(v_size_row_ref));\n            const size_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa);\n            if (v_size_row != v_size_row_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched value row size (%zu != %zu, layer %d)\\n\", __func__, v_size_row, (size_t) v_size_row_ref, il);\n                return false;\n            }\n\n            if (cell_count) {\n                if (sinfo.is_contiguous()) {\n                    // Fast path: contiguous cells, single memcpy\n                    ggml_backend_tensor_set(v, io.read(cell_count * v_size_row), sinfo.head() * v_size_row, cell_count * v_size_row);\n                } else {\n                    // Slow path: scatter to non-contiguous positions\n                    const void * src = io.read(cell_count * v_size_row);\n                    for (uint32_t i = 0; i < cell_count; ++i) {\n                        const size_t dst_offset = sinfo.idxs[0][i] * v_size_row;\n                        ggml_backend_tensor_set(v, (const char*)src + i * v_size_row, dst_offset, v_size_row);\n                    }\n                }\n            }\n        }\n    } else {\n        // For each layer, read the values for each cell (transposed)\n        for (const auto & layer : layers) {\n            const uint32_t il = layer.il;\n\n            const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);\n\n            auto * v = layer.v_stream[strm];\n            if (!v) {\n                continue;\n            }\n\n            // Read type of value\n            int32_t v_type_i_ref;\n            io.read_to(&v_type_i_ref, sizeof(v_type_i_ref));\n            const int32_t v_type_i = (int32_t) v->type;\n            if (v_type_i != v_type_i_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched value type (%d != %d, layer %d)\\n\", __func__, v_type_i, v_type_i_ref, il);\n                return false;\n            }\n\n            // Read element size of value\n            uint32_t v_size_el_ref;\n            io.read_to(&v_size_el_ref, sizeof(v_size_el_ref));\n            const size_t v_size_el = ggml_type_size(v->type);\n            if (v_size_el != v_size_el_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched value element size (%zu != %zu, layer %d)\\n\", __func__, v_size_el, (size_t) v_size_el_ref, il);\n                return false;\n            }\n\n            // Read GQA embedding size\n            uint32_t n_embd_v_gqa_ref;\n            io.read_to(&n_embd_v_gqa_ref, sizeof(n_embd_v_gqa_ref));\n            if (n_embd_v_gqa != n_embd_v_gqa_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched GQA embedding size (%u != %u, layer %d)\\n\", __func__, n_embd_v_gqa, n_embd_v_gqa_ref, il);\n                return false;\n            }\n\n            if (cell_count) {\n                if (sinfo.is_contiguous()) {\n                    // Fast path: contiguous cells\n                    const uint32_t h = sinfo.head();\n                    for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {\n                        const size_t dst_offset = (h + j * cells.size()) * v_size_el;\n                        ggml_backend_tensor_set(v, io.read(cell_count * v_size_el), dst_offset, cell_count * v_size_el);\n                    }\n                } else {\n                    // Slow path: scatter to non-contiguous positions\n                    for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {\n                        const void * src = io.read(cell_count * v_size_el);\n                        for (uint32_t i = 0; i < cell_count; ++i) {\n                            const size_t dst_offset = (sinfo.idxs[0][i] + j * cells.size()) * v_size_el;\n                            ggml_backend_tensor_set(v, (const char*)src + i * v_size_el, dst_offset, v_size_el);\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    return true;\n}\n\n//\n// llama_kv_cache_context\n//\n\nllama_kv_cache_context::llama_kv_cache_context(llama_memory_status status) : status(status) {}\n\nllama_kv_cache_context::llama_kv_cache_context(\n        llama_kv_cache * kv) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv) {\n    n_kv = kv->get_size();\n\n    const uint32_t n_stream = kv->get_n_stream();\n\n    // create a dummy slot info - the actual data is irrelevant. we just need to build the graph\n    sinfos.resize(1);\n    sinfos[0].s0 = 0;\n    sinfos[0].s1 = n_stream - 1;\n    sinfos[0].idxs.resize(n_stream);\n    for (uint32_t s = 0; s < n_stream; ++s) {\n        sinfos[0].strm.push_back(s);\n        sinfos[0].idxs[s].resize(1, 0);\n    }\n}\n\nllama_kv_cache_context::llama_kv_cache_context(\n        llama_kv_cache * kv,\n        llama_context * lctx,\n        bool do_shift,\n        stream_copy_info sc_info) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), lctx(lctx), do_shift(do_shift), sc_info(std::move(sc_info)) {\n    if (!do_shift && this->sc_info.empty()) {\n        status = LLAMA_MEMORY_STATUS_NO_UPDATE;\n    }\n}\n\nllama_kv_cache_context::llama_kv_cache_context(\n        llama_kv_cache * kv,\n        llama_kv_cache::slot_info_vec_t sinfos,\n        std::vector<llama_ubatch> ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), kv(kv), sinfos(std::move(sinfos)), ubatches(std::move(ubatches)) {\n}\n\nllama_kv_cache_context::~llama_kv_cache_context() = default;\n\nbool llama_kv_cache_context::next() {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    if (++i_cur >= ubatches.size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_kv_cache_context::apply() {\n    assert(!llama_memory_status_is_fail(status));\n\n    // no ubatches -> this is a KV cache update\n    if (ubatches.empty()) {\n        kv->update(lctx, do_shift, sc_info);\n\n        return true;\n    }\n\n    kv->apply_ubatch(sinfos[i_cur], ubatches[i_cur]);\n    n_kv = kv->get_n_kv(sinfos[i_cur]);\n\n    return true;\n}\n\nllama_memory_status llama_kv_cache_context::get_status() const {\n    return status;\n}\n\nconst llama_ubatch & llama_kv_cache_context::get_ubatch() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    return ubatches[i_cur];\n}\n\nuint32_t llama_kv_cache_context::get_n_kv() const {\n    return n_kv;\n}\n\nggml_tensor * llama_kv_cache_context::get_k(ggml_context * ctx, int32_t il) const {\n    return kv->get_k(ctx, il, n_kv, sinfos[i_cur]);\n}\n\nggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) const {\n    return kv->get_v(ctx, il, n_kv, sinfos[i_cur]);\n}\n\nggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const {\n    return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]);\n}\n\nggml_tensor * llama_kv_cache_context::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il) const {\n    return kv->cpy_v(ctx, v_cur, v_idxs, il, sinfos[i_cur]);\n}\n\nggml_tensor * llama_kv_cache_context::build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {\n    return kv->build_input_k_idxs(ctx, ubatch);\n}\n\nggml_tensor * llama_kv_cache_context::build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {\n    return kv->build_input_v_idxs(ctx, ubatch);\n}\n\nvoid llama_kv_cache_context::set_input_k_shift(ggml_tensor * dst) const {\n    kv->set_input_k_shift(dst);\n}\n\nvoid llama_kv_cache_context::set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const {\n    kv->set_input_k_idxs(dst, ubatch, sinfos[i_cur]);\n}\n\nvoid llama_kv_cache_context::set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const {\n    kv->set_input_v_idxs(dst, ubatch, sinfos[i_cur]);\n}\n\nvoid llama_kv_cache_context::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const {\n    kv->set_input_kq_mask(dst, ubatch, causal_attn);\n}\n\nvoid llama_kv_cache_context::set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const {\n    kv->set_input_pos_bucket(dst, ubatch);\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-kv-cache.h",
    "content": "#pragma once\n\n#include \"llama-batch.h\"\n#include \"llama-graph.h\"\n#include \"llama-kv-cells.h\"\n#include \"llama-memory.h\"\n\n#include <unordered_map>\n#include <vector>\n\nstruct llama_cparams;\nstruct llama_hparams;\nstruct llama_model;\nstruct llama_context;\n\n//\n// llama_kv_cache\n//\n\nclass llama_kv_cache : public llama_memory_i {\npublic:\n    struct stream_copy_info {\n        bool empty() const {\n            assert(ssrc.size() == sdst.size());\n            return ssrc.empty();\n        }\n\n        std::vector<uint32_t> ssrc;\n        std::vector<uint32_t> sdst;\n    };\n\n    // for each ubatch, create a slot_info that contains information about where the ubatch should be inserted in the\n    //   KV cells. for example, cell indices for each token, such that: token[i] -> goes to cells[idxs[i]]\n    struct slot_info {\n        // data for ggml_set_rows\n        using idx_vec_t = std::vector<uint32_t>;\n\n        // number of streams: ns = s1 - s0 + 1\n        uint32_t s0;\n        uint32_t s1;\n\n        std::vector<llama_seq_id> strm; // [ns]\n        std::vector<idx_vec_t>    idxs; // [ns]\n\n        uint32_t head() const {\n            GGML_ASSERT(idxs.size() == 1);\n            GGML_ASSERT(!idxs[0].empty());\n\n            return idxs[0][0];\n        }\n\n        void resize(size_t n) {\n            strm.resize(n);\n            idxs.resize(n);\n        }\n\n        size_t size() const {\n            GGML_ASSERT(idxs.size() == strm.size());\n            GGML_ASSERT(!idxs.empty());\n\n            return idxs[0].size();\n        }\n\n        size_t n_stream() const {\n            return strm.size();\n        }\n\n        bool empty() const {\n            return idxs.empty();\n        }\n\n        void clear() {\n            idxs.clear();\n        }\n\n        // check if indices are contiguous starting from head()\n        bool is_contiguous() const {\n            if (idxs.empty() || idxs[0].empty()) {\n                return true;\n            }\n            if (idxs.size() > 1) {\n                return false;\n            }\n            const uint32_t h = idxs[0][0];\n            for (size_t i = 0; i < idxs[0].size(); ++i) {\n                if (idxs[0][i] != h + i) {\n                    return false;\n                }\n            }\n            return true;\n        }\n    };\n\n    using slot_info_vec_t = std::vector<slot_info>;\n\n    llama_kv_cache(\n            const llama_model & model,\n                    ggml_type   type_k,\n                    ggml_type   type_v,\n                         bool   v_trans,\n                         bool   offload,\n                         bool   unified,\n                     uint32_t   kv_size,\n                     uint32_t   n_seq_max,\n                     uint32_t   n_pad,\n                     uint32_t   n_swa,\n               llama_swa_type   swa_type,\n        const layer_filter_cb & filter,\n        const  layer_reuse_cb & reuse);\n\n    ~llama_kv_cache() = default;\n\n    //\n    // llama_memory_i\n    //\n\n    llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) override;\n\n    llama_memory_context_ptr init_full() override;\n\n    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;\n\n    bool get_can_shift() const override;\n\n    void clear(bool data) override;\n\n    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;\n    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;\n    void seq_keep(llama_seq_id seq_id)                                                          override;\n    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;\n    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;\n\n    llama_pos seq_pos_min(llama_seq_id seq_id) const override;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const override;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;\n\n    // state write/load\n\n    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;\n    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;\n\n    //\n    // llama_kv_cache specific API\n    //\n\n    uint32_t get_size()     const;\n    uint32_t get_n_stream() const;\n\n    bool get_has_shift() const;\n\n    //\n    // graph_build API\n    //\n\n    uint32_t get_n_kv(const slot_info & sinfo) const;\n\n    // get views of the current state of the cache\n    ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;\n    ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;\n\n    // store k_cur and v_cur in the cache based on the provided head location\n    ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const;\n    ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const;\n\n    //\n    // preparation API\n    //\n\n    // find places for the provided ubatches in the cache, returns the slot infos\n    // return empty vector on failure\n    slot_info_vec_t prepare(const std::vector<llama_ubatch> & ubatches);\n\n    bool update(llama_context * lctx, bool do_shift, const stream_copy_info & sc_info);\n\n    // find a slot of kv cells that can hold the ubatch\n    // if cont == true, then the slot must be continuous\n    // return empty slot_info on failure\n    slot_info find_slot(const llama_ubatch & ubatch, bool cont) const;\n\n    // emplace the ubatch context into slot: [sinfo.idxs[0...ubatch.n_tokens - 1]]\n    void apply_ubatch(const slot_info & sinfo, const llama_ubatch & ubatch);\n\n    //\n    // input API\n    //\n\n    ggml_tensor * build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;\n    ggml_tensor * build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;\n\n    void set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const;\n    void set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch, const slot_info & sinfo) const;\n\n    void set_input_k_shift(ggml_tensor * dst) const;\n\n    void set_input_kq_mask   (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const;\n    void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;\n\nprivate:\n    const llama_model & model;\n    const llama_hparams & hparams;\n\n    struct kv_layer {\n        // layer index in the model\n        // note: can be different from the layer index in the KV cache\n        uint32_t il;\n\n        ggml_tensor * k;\n        ggml_tensor * v;\n\n        std::vector<ggml_tensor *> k_stream;\n        std::vector<ggml_tensor *> v_stream;\n    };\n\n    bool v_trans = true;  // the value tensor is transposed\n\n    const uint32_t n_seq_max = 1;\n    const uint32_t n_stream  = 1;\n\n    // required padding\n    const uint32_t n_pad = 1;\n\n    // SWA\n    const uint32_t n_swa = 0;\n\n    // env: LLAMA_KV_CACHE_DEBUG\n    int debug = 0;\n\n    // this is the SWA type of the cache - not to be confused with the model SWA type\n    const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;\n\n    // ggml contexts for the KV cache along with the allocated backend buffers:\n    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;\n\n    // the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())\n    // note: this is not part of the KV state and it's only used to speed-up the find_slot() method\n    std::vector<uint32_t> v_heads;\n\n    std::vector<llama_kv_cells> v_cells;\n\n    // maps from a sequence id to a stream id\n    std::vector<uint32_t> seq_to_stream;\n\n    // pending stream copies that will be applied during the next update\n    stream_copy_info sc_info;\n\n    std::vector<kv_layer> layers;\n\n    // model layer id -> KV cache layer id\n    std::unordered_map<int32_t, int32_t> map_layer_ids;\n\n    size_t total_size() const;\n\n    size_t size_k_bytes() const;\n    size_t size_v_bytes() const;\n\n    ggml_tensor * build_rope_shift(\n            const llama_cparams & cparams,\n                   ggml_context * ctx,\n                    ggml_tensor * cur,\n                    ggml_tensor * shift,\n                    ggml_tensor * factors,\n                          float   freq_base,\n                          float   freq_scale,\n                       uint32_t   il) const;\n\n    ggml_cgraph * build_graph_shift(\n               llm_graph_result * res,\n                  llama_context * lctx) const;\n\n    struct cell_ranges_t {\n        uint32_t strm;\n\n        std::vector<std::pair<uint32_t, uint32_t>> data; // ranges, from inclusive, to exclusive\n    };\n\n    void state_write_meta(llama_io_write_i & io, const cell_ranges_t & cr, llama_seq_id seq_id = -1) const;\n    void state_write_data(llama_io_write_i & io, const cell_ranges_t & cr) const;\n\n    bool state_read_meta(llama_io_read_i & io, uint32_t strm, uint32_t cell_count,       slot_info & sinfo, llama_seq_id dest_seq_id = -1);\n    bool state_read_data(llama_io_read_i & io, uint32_t strm, uint32_t cell_count, const slot_info & sinfo);\n};\n\nclass llama_kv_cache_context : public llama_memory_context_i {\npublic:\n    // some shorthands\n    using slot_info_vec_t  = llama_kv_cache::slot_info_vec_t;\n    using stream_copy_info = llama_kv_cache::stream_copy_info;\n\n    // used for errors\n    llama_kv_cache_context(llama_memory_status status);\n\n    // used to create a full-cache context\n    llama_kv_cache_context(\n            llama_kv_cache * kv);\n\n    // used to create an update context\n    llama_kv_cache_context(\n            llama_kv_cache * kv,\n            llama_context * lctx,\n            bool do_shift,\n            stream_copy_info sc_info);\n\n    // used to create a batch processing context from a batch\n    llama_kv_cache_context(\n            llama_kv_cache * kv,\n            slot_info_vec_t sinfos,\n            std::vector<llama_ubatch> ubatches);\n\n    virtual ~llama_kv_cache_context();\n\n    //\n    // llama_memory_context_i\n    //\n\n    bool next()  override;\n    bool apply() override;\n\n    llama_memory_status  get_status() const override;\n    const llama_ubatch & get_ubatch() const override;\n\n    //\n    // llama_kv_cache_context specific API\n    //\n\n    uint32_t get_n_kv() const;\n\n    // get views of the current state of the cache\n    ggml_tensor * get_k(ggml_context * ctx, int32_t il) const;\n    ggml_tensor * get_v(ggml_context * ctx, int32_t il) const;\n\n    // store k_cur and v_cur in the cache based on the provided head location\n    // note: the heads in k_cur and v_cur should be layed out contiguously in memory\n    //   - k_cur  [n_embd_head_k, n_head_k, n_tokens]\n    //   - k_idxs [n_tokens]\n    //   - v_cur  [n_embd_head_v, n_head_v, n_tokens]\n    //   - v_idxs [n_tokens] or [n_tokens*n_embd_v_gqa] depending if V cache is transposed\n    ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const;\n    ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il) const;\n\n    // create destination indices for each head of the current batch for where it would be written in the KV cache\n    // the indices address the global KV cache (not per stream) - this is not relevant for the user of this API, but\n    //   helps understand the implementation logic of cpy_k and cpy_v\n    ggml_tensor * build_input_k_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;\n    ggml_tensor * build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const;\n\n    void set_input_k_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const;\n    void set_input_v_idxs(ggml_tensor * dst, const llama_ubatch * ubatch) const;\n\n    void set_input_k_shift   (ggml_tensor * dst) const;\n    void set_input_kq_mask   (ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const;\n    void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;\n\nprivate:\n    llama_memory_status status;\n\n    llama_kv_cache * kv;\n    llama_context * lctx;\n\n    //\n    // update context\n    //\n\n    bool do_shift = false;\n\n    stream_copy_info sc_info;\n\n    //\n    // batch processing context\n    //\n\n    // the index of the cur ubatch to process\n    size_t i_cur = 0;\n\n    slot_info_vec_t sinfos;\n\n    std::vector<llama_ubatch> ubatches;\n\n    //\n    // data needed for building the compute graph for the current ubatch:\n    //\n\n    // a heuristic, to avoid attending the full cache if it is not yet utilized\n    // as the cache gets filled, the benefit from this heuristic disappears\n    int32_t n_kv;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-kv-cells.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n#include \"llama-cparams.h\"\n\n#include <bitset>\n#include <cassert>\n#include <cstring>\n#include <map>\n#include <set>\n#include <vector>\n\nstruct llama_kv_cell_ext {\n    // 2D spatial positions, typically used for M-RoPE\n    llama_pos x = 0;\n    llama_pos y = 0;\n\n    // return true if the current 2D spatial position is greater than other\n    bool is_2d_gt(llama_pos ox, llama_pos oy) const {\n        return (y > oy) || (y == oy && x > ox);\n    }\n\n    void reset() {\n        static_assert(std::is_trivially_copyable_v<llama_kv_cell_ext>);\n\n        memset(this, 0, sizeof(*this));\n    }\n};\n\n// meta information about KV cells that can be part of multiple sequences at the same time\n// TODO: add unit tests\nclass llama_kv_cells {\npublic:\n    void reset() {\n        for (uint32_t i = 0; i < pos.size(); ++i) {\n            pos[i]   = -1;\n            ext[i].reset();\n            shift[i] =  0;\n            seq[i].reset();\n        }\n\n        has_shift = false;\n\n        used.clear();\n\n        for (uint32_t s = 0; s < LLAMA_MAX_SEQ; ++s) {\n            seq_pos[s].clear();\n        }\n    }\n\n    void reset_shift() {\n        has_shift = false;\n\n        for (uint32_t i = 0; i < shift.size(); ++i) {\n            shift[i] = 0;\n        }\n    }\n\n    uint32_t size() const {\n        return pos.size();\n    }\n\n    void resize(uint32_t n) {\n        pos.resize(n);\n        ext.resize(n);\n        shift.resize(n);\n        seq.resize(n);\n\n        reset();\n    }\n\n    bool is_empty(uint32_t i) const {\n        assert(i < pos.size());\n        assert((pos[i] < 0 && pos[i] == -1) || pos[i] >= 0);\n\n        return pos[i] == -1;\n    }\n\n    uint32_t get_used() const {\n        return used.size();\n    }\n\n    // the index of the first cell that is used\n    // return 0 if no cells are used\n    uint32_t used_min() const {\n        return used.empty() ? 0 : *used.begin();\n    }\n\n    // the index of the last cell that is used + 1\n    // return 0 if no cells are used\n    uint32_t used_max_p1() const {\n        return used.empty() ? 0 : *used.rbegin() + 1;\n    }\n\n    bool get_has_shift() const {\n        return has_shift;\n    }\n\n    // move cell isrc to idst (used during defrag)\n    //void mv(uint32_t isrc, uint32_t idst) {\n    //    assert(isrc < pos.size());\n    //    assert(idst < pos.size());\n\n    //    assert(pos[idst] == -1);\n    //    assert(pos[isrc] != -1);\n\n    //    pos  [idst] = pos  [isrc];\n    //    shift[idst] = shift[isrc];\n    //    seq  [idst] = seq  [isrc];\n\n    //    pos  [isrc] = -1;\n    //    shift[isrc] =  0;\n    //    seq  [isrc].reset();\n\n    //    used.erase (isrc);\n    //    used.insert(idst);\n    //}\n\n    // copy the state of cells [i, i + n) (used for save/restore the state of the cells)\n    llama_kv_cells cp(uint32_t i, uint32_t n) const {\n        assert(i + n <= pos.size());\n\n        llama_kv_cells res;\n\n        res.resize(n);\n\n        for (uint32_t j = 0; j < n; ++j) {\n            const auto idx = i + j;\n\n            res.pos[j] = pos[idx];\n            res.ext[j] = ext[idx];\n            res.seq[j] = seq[idx];\n\n            assert(shift[idx] == 0);\n        }\n\n        return res;\n    }\n\n    // copy the state of cells [idxs[0], idxs[1], ..., idxs[idxs.size() - 1])\n    llama_kv_cells cp(const std::vector<uint32_t> & idxs) const {\n        llama_kv_cells res;\n\n        res.resize(idxs.size());\n\n        for (uint32_t j = 0; j < idxs.size(); ++j) {\n            const auto idx = idxs[j];\n\n            res.pos[j] = pos[idx];\n            res.ext[j] = ext[idx];\n            res.seq[j] = seq[idx];\n\n            assert(shift[idx] == 0);\n        }\n\n        return res;\n    }\n\n    // set the state of cells [i, i + other.pos.size()) (used for save/restore the state of the cells)\n    void set(uint32_t i, const llama_kv_cells & other) {\n        assert(i + other.pos.size() <= pos.size());\n\n        for (uint32_t j = 0; j < other.pos.size(); ++j) {\n            const auto idx = i + j;\n\n            if (pos[idx] == -1 && other.pos[j] != -1) {\n                used.insert(i + j);\n            }\n\n            if (pos[idx] != -1 && other.pos[j] == -1) {\n                used.erase(i + j);\n            }\n\n            if (pos[idx] != -1) {\n                seq_pos_rm(i + j);\n            }\n\n            pos[idx] = other.pos[j];\n            ext[idx] = other.ext[j];\n            seq[idx] = other.seq[j];\n\n            if (pos[idx] != -1) {\n                seq_pos_add(i + j);\n            }\n\n            assert(shift[idx] == 0);\n        }\n    }\n\n    // set the state of cells [idxs[0], idxs[1], ..., idxs[idxs.size() - 1])\n    void set(const std::vector<uint32_t> & idxs, const llama_kv_cells & other) {\n        assert(idxs.size() == other.pos.size());\n\n        for (uint32_t j = 0; j < other.pos.size(); ++j) {\n            const auto idx = idxs[j];\n\n            if (pos[idx] == -1 && other.pos[j] != -1) {\n                used.insert(idx);\n            }\n\n            if (pos[idx] != -1 && other.pos[j] == -1) {\n                used.erase(idx);\n            }\n\n            if (pos[idx] != -1) {\n                seq_pos_rm(idx);\n            }\n\n            pos[idx] = other.pos[j];\n            ext[idx] = other.ext[j];\n            seq[idx] = other.seq[j];\n\n            if (pos[idx] != -1) {\n                seq_pos_add(idx);\n            }\n\n            assert(shift[idx] == 0);\n        }\n    }\n\n    // clear a non-empty cell\n    void rm(uint32_t i) {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        seq_pos_rm(i);\n        seq[i].reset();\n\n        pos[i] = -1;\n        ext[i].reset();\n        shift[i] = 0;\n\n        used.erase(i);\n    }\n\n    // note: call only if the cell has seq_id\n    // return true if the cell becomes empty\n    bool seq_rm(uint32_t i, llama_seq_id seq_id) {\n        assert(i < pos.size());\n        assert(seq[i].test(seq_id));\n        assert(pos[i] != -1);\n        assert(seq_id >= 0);\n\n        seq[i].reset(seq_id);\n        seq_pos_dec(seq_id, pos[i]);\n\n        if (seq[i].none()) {\n            pos[i] = -1;\n            ext[i].reset();\n            shift[i] = 0;\n\n            used.erase(i);\n\n            return true;\n        }\n\n        return false;\n    }\n\n    // return true if the cell becomes empty (i.e. it did not contain seq_id before the call)\n    bool seq_keep(uint32_t i, llama_seq_id seq_id) {\n        assert(i < pos.size());\n\n        if (seq[i].test(seq_id)) {\n            seq_pos_rm(i);\n            seq[i].reset();\n\n            seq[i].set(seq_id);\n            seq_pos_inc(seq_id, pos[i]);\n\n            return false;\n        }\n\n        if (seq[i].any()) {\n            seq_pos_rm(i);\n            seq[i].reset();\n\n            pos[i] = -1;\n            ext[i].reset();\n            shift[i] = 0;\n\n            used.erase(i);\n\n            return true;\n        }\n\n        assert(pos[i] == -1);\n\n        return false;\n    }\n\n    // number of different sequences in the cell\n    int seq_count(uint32_t i) const {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        return seq[i].count();\n    }\n\n    // check if the cell contains seq_id\n    bool seq_has(uint32_t i, llama_seq_id seq_id) const {\n        assert(i < pos.size());\n        assert(seq_id >= 0);\n\n        return seq[i].test(seq_id);\n    }\n\n    // note: call only if the cell is not empty and the seq_id is not in the cell\n    void seq_add(uint32_t i, llama_seq_id seq_id) {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n        assert(!seq[i].test(seq_id));\n\n        seq[i].set(seq_id);\n        seq_pos_inc(seq_id, pos[i]);\n    }\n\n    // return the sequence id of this cell\n    // note: call only for cells with exactly one sequence\n    llama_seq_id seq_get(uint32_t i) const {\n        assert(seq[i].count() == 1);\n\n        for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n            if (seq[i].test(s)) {\n                return s;\n            }\n        }\n\n        return -1;\n    }\n\n    // the minimum position of sequence seq_id currently present in any of the cells\n    // return -1 if the sequence is not present\n    llama_pos seq_pos_min(llama_seq_id seq_id) const {\n        assert(seq_id >= 0);\n        assert(seq_id < LLAMA_MAX_SEQ);\n\n        if (seq_pos[seq_id].empty()) {\n            return -1;\n        }\n\n        assert(seq_pos[seq_id].begin()->second > 0);\n\n        return seq_pos[seq_id].begin()->first;\n    }\n\n    // the maximum position of sequence seq_id currently present in any of the cells\n    // return -1 if the sequence is not present\n    llama_pos seq_pos_max(llama_seq_id seq_id) const {\n        assert(seq_id >= 0);\n        assert(seq_id < LLAMA_MAX_SEQ);\n\n        if (seq_pos[seq_id].empty()) {\n            return -1;\n        }\n\n        assert(seq_pos[seq_id].rbegin()->second > 0);\n\n        return seq_pos[seq_id].rbegin()->first;\n    }\n\n    // note: call only if the cell is not empty\n    llama_pos pos_get(uint32_t i) const {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        return pos[i];\n    }\n\n    const llama_kv_cell_ext & ext_get(uint32_t i) const {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        return ext[i];\n    }\n\n    // note: call only if the cell is not empty\n    llama_pos get_shift(uint32_t i) const {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        return shift[i];\n    }\n\n    // check if a cell is not empty and its position is within [p0, p1)\n    bool pos_in(uint32_t i, llama_pos p0, llama_pos p1) const {\n        assert(i < pos.size());\n\n        return pos[i] >= p0 && pos[i] < p1;\n    }\n\n    // set the position of an empty cell\n    // does not modify \"has_shift\"\n    // note: call only if the cell is empty\n    void pos_set(uint32_t i, llama_pos p) {\n        assert(i < pos.size());\n        assert(pos[i] == -1);\n        assert(seq[i].none());\n\n        pos[i] = p;\n\n        used.insert(i);\n    }\n\n    void ext_set(uint32_t i, llama_kv_cell_ext p) {\n        assert(i < ext.size());\n        ext[i] = p;\n    }\n\n    // pos[i] = pos[i] + d\n    // sets \"has_shift\" to true\n    // note: call only if the cell is not empty\n    bool pos_add(uint32_t i, llama_pos d) {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        seq_pos_rm(i);\n\n        pos[i]   += d;\n        shift[i] += d;\n\n        has_shift = true;\n\n        if (pos[i] < 0) {\n            seq[i].reset();\n            pos[i] = -1;\n            shift[i] = 0;\n\n            used.erase(i);\n\n            return true;\n        }\n\n        seq_pos_add(i);\n\n        return false;\n    }\n\n    // pos[i] = pos[i] / d\n    // sets \"has_shift\" to true\n    // note: call only if the cell is not empty\n    void pos_div(uint32_t i, int d) {\n        assert(i < pos.size());\n        assert(pos[i] != -1);\n\n        const llama_pos p_old = pos[i];\n\n        seq_pos_rm(i);\n\n        pos[i]   /= d;\n        shift[i] += p_old - pos[i];\n\n        seq_pos_add(i);\n\n        has_shift = true;\n    }\n\nprivate:\n    bool has_shift = false;\n\n    // set of indices of used cells (i.e. pos[i] != -1, allowed to not have any seq_id)\n    std::set<uint32_t> used;\n\n    std::vector<llama_pos> pos;\n\n    // stores extra info per cell\n    std::vector<llama_kv_cell_ext> ext;\n\n    // this array accumulates any applied shifts to the pos array since the last reset_shift() call\n    // this is used to queue multiple updates to the pos array, which in the end can be applied in one go:\n    //\n    //   cells.pos_add(x, shift_x);\n    //   cells.pos_div(y, shift_y);\n    //   ...\n    //\n    //   if (cells.has_shift()) {\n    //      for (int i = 0; i < n; ++i) {\n    //          auto shift_i = cells.get_shift(i);\n    //          ...\n    //      }\n    //      cells.reset_shift();\n    //   }\n    //\n    std::vector<llama_pos> shift;\n\n    using seq_set_t = std::bitset<LLAMA_MAX_SEQ>;\n\n    // the bitset seq[i] tells us which sequences are currently occupying the i-th cell\n    std::vector<seq_set_t> seq;\n\n    // the set seq_pos[s][p] tells us how many times the position p is currently present for sequence s\n    // if the position p is not present, seq_pos[s][p] is not set\n    // this way seq_pos[s].begin() and seq_pos[s].rbegin() give us the min/max positions currently in the cache\n    //\n    // note that we cannot a use an std::set because in some cases a position can occur more than once for the same seq:\n    //  - during performing a cache reuse via (rm + add)\n    //  - some vision models have input embeddings with repeating positions\n    //\n    std::map<llama_pos, int> seq_pos[LLAMA_MAX_SEQ];\n\n    // helper functions for updating `seq_pos`, once cell at a time:\n\n    void seq_pos_dec(llama_seq_id s, llama_pos p) {\n        auto it = seq_pos[s].find(p);\n        assert(it != seq_pos[s].end());\n\n        if (--it->second == 0) {\n            seq_pos[s].erase(it);\n        }\n    }\n\n    void seq_pos_inc(llama_seq_id s, llama_pos p) {\n        seq_pos[s][p]++;\n    }\n\n    // remove cell i\n    void seq_pos_rm(uint32_t i) {\n        for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n            if (seq[i].test(s)) {\n                seq_pos_dec(s, pos[i]);\n            }\n        }\n    }\n\n    // add cell i\n    void seq_pos_add(uint32_t i) {\n        for (int s = 0; s < LLAMA_MAX_SEQ; ++s) {\n            if (seq[i].test(s)) {\n                seq_pos_inc(s, pos[i]);\n            }\n        }\n    }\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-hybrid-iswa.cpp",
    "content": "#include \"llama-memory-hybrid-iswa.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-model.h\"\n#include \"llama-context.h\"\n\n//\n// llama_memory_hybrid_iswa\n//\n\nllama_memory_hybrid_iswa::llama_memory_hybrid_iswa(\n        const llama_model & model,\n                            /* attn */\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                     bool   swa_full,\n                 uint32_t   kv_size,\n                 uint32_t   n_ubatch,\n                 uint32_t   n_pad,\n                            /* recurrent */\n                ggml_type   type_r,\n                ggml_type   type_s,\n                 uint32_t   rs_size,\n                            /* common */\n                 uint32_t   n_seq_max,\n                     bool   offload,\n                     bool   unified,\n                            /* layer filters */\n    const layer_filter_cb & filter_attn,\n    const layer_filter_cb & filter_recr) :\n    hparams(model.hparams),\n    mem_attn(new llama_kv_cache_iswa(\n        model,\n        type_k,\n        type_v,\n        v_trans,\n        offload,\n        swa_full,\n        unified,\n        kv_size,\n        n_seq_max,\n        n_ubatch,\n        n_pad,\n        filter_attn == nullptr ?\n            [&](int32_t il) { return !hparams.is_recurrent(il); }\n            : filter_attn,\n        nullptr\n    )),\n    mem_recr(new llama_memory_recurrent(\n        model,\n        type_r,\n        type_s,\n        offload,\n        rs_size,\n        n_seq_max,\n        filter_recr == nullptr ?\n            [&](int32_t il) { return hparams.is_recurrent(il); }\n            : filter_recr\n    )) {}\n\nllama_memory_context_ptr llama_memory_hybrid_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {\n    do {\n        balloc.split_reset();\n\n        // follow the recurrent pattern for creating the ubatch splits\n        std::vector<llama_ubatch> ubatches;\n\n        while (true) {\n            llama_ubatch ubatch;\n\n            if (embd_all) {\n                // if all tokens are output, split by sequence\n                ubatch = balloc.split_seq(n_ubatch);\n            } else {\n                // TODO: non-sequential equal split can be done if using unified KV cache\n                //       for simplicity, we always use sequential equal split for now\n                ubatch = balloc.split_equal(n_ubatch, true);\n            }\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        // prepare the recurrent batches first\n        if (!mem_recr->prepare(ubatches)) {\n            // TODO: will the recurrent cache be in an undefined context at this point?\n            LLAMA_LOG_ERROR(\"%s: failed to prepare recurrent ubatches\\n\", __func__);\n            return std::make_unique<llama_memory_hybrid_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n        }\n\n        // prepare the attention cache (iswa version returns both base and swa slot infos)\n        auto sinfos_base = mem_attn->get_base()->prepare(ubatches);\n        if (sinfos_base.empty()) {\n            LLAMA_LOG_ERROR(\"%s: failed to prepare attention base ubatches\\n\", __func__);\n            return std::make_unique<llama_memory_hybrid_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n        }\n\n        auto sinfos_swa = mem_attn->get_swa()->prepare(ubatches);\n        if (sinfos_swa.empty()) {\n            LLAMA_LOG_ERROR(\"%s: failed to prepare attention swa ubatches\\n\", __func__);\n            return std::make_unique<llama_memory_hybrid_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n        }\n\n        return std::make_unique<llama_memory_hybrid_iswa_context>(\n                this, std::move(sinfos_base), std::move(sinfos_swa), std::move(ubatches));\n    } while(false);\n\n    return std::make_unique<llama_memory_hybrid_iswa_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n}\n\nllama_memory_context_ptr llama_memory_hybrid_iswa::init_full() {\n    return std::make_unique<llama_memory_hybrid_iswa_context>(this);\n}\n\nllama_memory_context_ptr llama_memory_hybrid_iswa::init_update(llama_context * lctx, bool optimize) {\n    return std::make_unique<llama_memory_hybrid_iswa_context>(this, lctx, optimize);\n}\n\nbool llama_memory_hybrid_iswa::get_can_shift() const {\n    // Shifting is trivially supported for recurrent\n    return mem_attn->get_can_shift();\n}\n\nvoid llama_memory_hybrid_iswa::clear(bool data) {\n    mem_attn->clear(data);\n    mem_recr->clear(data);\n}\n\nbool llama_memory_hybrid_iswa::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {\n    // Try removing from the recurrent cache first since it may fail. If it does\n    // fail, the cache will not have been mutated.\n    if (!mem_recr->seq_rm(seq_id, p0, p1)) {\n        return false;\n    }\n    return mem_attn->seq_rm(seq_id, p0, p1);\n}\n\nvoid llama_memory_hybrid_iswa::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {\n    mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n    mem_recr->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n}\n\nvoid llama_memory_hybrid_iswa::seq_keep(llama_seq_id seq_id) {\n    mem_attn->seq_keep(seq_id);\n    mem_recr->seq_keep(seq_id);\n}\n\nvoid llama_memory_hybrid_iswa::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {\n    mem_attn->seq_add(seq_id, p0, p1, shift);\n    mem_recr->seq_add(seq_id, p0, p1, shift);\n}\n\nvoid llama_memory_hybrid_iswa::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {\n    mem_attn->seq_div(seq_id, p0, p1, d);\n    mem_recr->seq_div(seq_id, p0, p1, d);\n}\n\nllama_pos llama_memory_hybrid_iswa::seq_pos_min(llama_seq_id seq_id) const {\n    // the min of the total cache is the max of the two caches' min values\n    return std::max(mem_attn->seq_pos_min(seq_id), mem_recr->seq_pos_min(seq_id));\n}\n\nllama_pos llama_memory_hybrid_iswa::seq_pos_max(llama_seq_id seq_id) const {\n    // the max of the total cache is the min of the two caches' max values\n    return std::min(mem_attn->seq_pos_max(seq_id), mem_recr->seq_pos_max(seq_id));\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_memory_hybrid_iswa::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> mb = mem_attn->memory_breakdown();\n    for (const auto & buft_size : mem_recr->memory_breakdown()) {\n        mb[buft_size.first] += buft_size.second;\n    }\n    return mb;\n}\n\nvoid llama_memory_hybrid_iswa::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {\n    mem_attn->state_write(io, seq_id, flags);\n    mem_recr->state_write(io, seq_id, flags);\n}\n\nvoid llama_memory_hybrid_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    mem_attn->state_read(io, seq_id, flags);\n    mem_recr->state_read(io, seq_id, flags);\n}\n\nllama_kv_cache_iswa * llama_memory_hybrid_iswa::get_mem_attn() const {\n    return mem_attn.get();\n}\n\nllama_memory_recurrent * llama_memory_hybrid_iswa::get_mem_recr() const {\n    return mem_recr.get();\n}\n\n//\n// llama_memory_hybrid_iswa_context\n//\n\nllama_memory_hybrid_iswa_context::llama_memory_hybrid_iswa_context(llama_memory_status status) : status(status) {}\n\nllama_memory_hybrid_iswa_context::llama_memory_hybrid_iswa_context(llama_memory_hybrid_iswa * mem) :\n    ctx_attn(mem->get_mem_attn()->init_full()),\n    ctx_recr(mem->get_mem_recr()->init_full()),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nllama_memory_hybrid_iswa_context::llama_memory_hybrid_iswa_context(\n        llama_memory_hybrid_iswa * mem,\n                   llama_context * lctx,\n                            bool   optimize) :\n    ctx_attn(mem->get_mem_attn()->init_update(lctx, optimize)),\n    ctx_recr(mem->get_mem_recr()->init_update(lctx, optimize)),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nllama_memory_hybrid_iswa_context::llama_memory_hybrid_iswa_context(\n           llama_memory_hybrid_iswa * mem,\n                    slot_info_vec_t   sinfos_base,\n                    slot_info_vec_t   sinfos_swa,\n          std::vector<llama_ubatch>   ubatches) :\n    ubatches(std::move(ubatches)),\n    // note: here we copy the ubatches. not sure if this is ideal\n    ctx_attn(new llama_kv_cache_iswa_context(mem->get_mem_attn(), std::move(sinfos_base), std::move(sinfos_swa), this->ubatches)),\n    ctx_recr(new llama_memory_recurrent_context(mem->get_mem_recr(), this->ubatches)),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nbool llama_memory_hybrid_iswa_context::next() {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    ctx_attn->next();\n    ctx_recr->next();\n\n    if (++i_next >= ubatches.size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_memory_hybrid_iswa_context::apply() {\n    assert(!llama_memory_status_is_fail(status));\n\n    bool res = true;\n\n    res = res & ctx_attn->apply();\n    res = res & ctx_recr->apply();\n\n    return res;\n}\n\nllama_memory_status llama_memory_hybrid_iswa_context::get_status() const {\n    return status;\n}\n\nconst llama_ubatch & llama_memory_hybrid_iswa_context::get_ubatch() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n    return ubatches[i_next];\n}\n\nconst llama_kv_cache_iswa_context * llama_memory_hybrid_iswa_context::get_attn() const {\n    return static_cast<const llama_kv_cache_iswa_context *>(ctx_attn.get());\n}\n\nconst llama_memory_recurrent_context * llama_memory_hybrid_iswa_context::get_recr() const {\n    return static_cast<const llama_memory_recurrent_context *>(ctx_recr.get());\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-hybrid-iswa.h",
    "content": "#pragma once\n\n#include \"llama-batch.h\"\n#include \"llama-graph.h\"\n#include \"llama-kv-cache-iswa.h\"\n#include \"llama-memory.h\"\n#include \"llama-memory-recurrent.h\"\n\n#include <memory>\n#include <vector>\n\n//\n// llama_memory_hybrid_iswa\n//\n\n// utilizes instances of llama_memory_recurrent and llama_kv_cache_iswa to\n//   support models where each layer may be either attention-based (with SWA support) or recurrent\n\nclass llama_memory_hybrid_iswa : public llama_memory_i {\npublic:\n    llama_memory_hybrid_iswa(\n        const llama_model & model,\n                            /* attn */\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                     bool   swa_full,\n                 uint32_t   kv_size,\n                 uint32_t   n_ubatch,\n                 uint32_t   n_pad,\n                            /* recurrent */\n                ggml_type   type_r,\n                ggml_type   type_s,\n                 uint32_t   rs_size,\n                            /* common */\n                 uint32_t   n_seq_max,\n                     bool   offload,\n                     bool   unified,\n                            /* layer filters */\n    const layer_filter_cb & filter_attn = nullptr,\n    const layer_filter_cb & filter_recr = nullptr);\n\n    ~llama_memory_hybrid_iswa() = default;\n\n    //\n    // llama_memory_i\n    //\n\n    llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) override;\n\n    llama_memory_context_ptr init_full() override;\n\n    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;\n\n    bool get_can_shift() const override;\n\n    void clear(bool data) override;\n\n    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;\n    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;\n    void seq_keep(llama_seq_id seq_id)                                                          override;\n    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;\n    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;\n\n    llama_pos seq_pos_min(llama_seq_id seq_id) const override;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const override;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;\n\n    // state write/load\n\n    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;\n    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0)       override;\n\n    //\n    // llama_memory_hybrid_iswa specific API\n    //\n\n    llama_kv_cache_iswa * get_mem_attn() const;\n    llama_memory_recurrent * get_mem_recr() const;\n\nprivate:\n    const llama_hparams & hparams;\n\n    const std::unique_ptr<llama_kv_cache_iswa> mem_attn;\n    const std::unique_ptr<llama_memory_recurrent> mem_recr;\n};\n\nclass llama_memory_hybrid_iswa_context : public llama_memory_context_i {\npublic:\n    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;\n\n    // init failure\n    explicit llama_memory_hybrid_iswa_context(llama_memory_status status);\n\n    // init full\n    explicit llama_memory_hybrid_iswa_context(llama_memory_hybrid_iswa * mem);\n\n    // init update\n    explicit llama_memory_hybrid_iswa_context(\n        llama_memory_hybrid_iswa * mem,\n                   llama_context * lctx,\n                            bool   optimize);\n\n    // init success\n    llama_memory_hybrid_iswa_context(\n           llama_memory_hybrid_iswa * mem,\n                    slot_info_vec_t   sinfos_base,\n                    slot_info_vec_t   sinfos_swa,\n          std::vector<llama_ubatch>   ubatches);\n\n    ~llama_memory_hybrid_iswa_context() = default;\n\n    bool next()  override;\n    bool apply() override;\n\n    llama_memory_status  get_status() const override;\n    const llama_ubatch & get_ubatch() const override;\n\n    //\n    // llama_memory_hybrid_iswa_context\n    //\n\n    const llama_kv_cache_iswa_context * get_attn() const;\n    const llama_memory_recurrent_context * get_recr() const;\n\nprivate:\n    // the index of the next ubatch to process\n    size_t i_next = 0;\n\n    std::vector<llama_ubatch> ubatches;\n\n    const llama_memory_context_ptr ctx_attn;\n    const llama_memory_context_ptr ctx_recr;\n\n    const llama_memory_status status;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-hybrid.cpp",
    "content": "#include \"llama-memory-hybrid.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-model.h\"\n#include \"llama-context.h\"\n\n//\n// llama_memory_hybrid\n//\n\nllama_memory_hybrid::llama_memory_hybrid(\n        const llama_model & model,\n                            /* attn */\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                 uint32_t   kv_size,\n                 uint32_t   n_pad,\n                 uint32_t   n_swa,\n           llama_swa_type   swa_type,\n                            /* recurrent */\n                ggml_type   type_r,\n                ggml_type   type_s,\n                 uint32_t   rs_size,\n                            /* common */\n                 uint32_t   n_seq_max,\n                     bool   offload,\n                     bool   unified,\n                            /* layer filters */\n    const layer_filter_cb & filter_attn,\n    const layer_filter_cb & filter_recr) :\n    hparams(model.hparams),\n    mem_attn(new llama_kv_cache(\n        model,\n        type_k,\n        type_v,\n        v_trans,\n        offload,\n        unified,\n        kv_size,\n        n_seq_max,\n        n_pad,\n        n_swa,\n        swa_type,\n        filter_attn == nullptr ?\n            [&](int32_t il) { return !hparams.is_recurrent(il); }\n            : filter_attn,\n        nullptr\n    )),\n    mem_recr(new llama_memory_recurrent(\n        model,\n        type_r,\n        type_s,\n        offload,\n        rs_size,\n        n_seq_max,\n        filter_recr == nullptr ?\n            [&](int32_t il) { return hparams.is_recurrent(il); }\n            : filter_recr\n    )) {}\n\nllama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {\n    do {\n        balloc.split_reset();\n\n        // follow the recurrent pattern for creating the ubatch splits\n        std::vector<llama_ubatch> ubatches;\n\n        while (true) {\n            llama_ubatch ubatch;\n\n            if (embd_all) {\n                // if all tokens are output, split by sequence\n                ubatch = balloc.split_seq(n_ubatch);\n            } else {\n                // TODO: non-sequential equal split can be done if using unified KV cache\n                //       for simplicity, we always use sequential equal split for now\n                ubatch = balloc.split_equal(n_ubatch, true);\n            }\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        // prepare the recurrent batches first\n        if (!mem_recr->prepare(ubatches)) {\n            // TODO: will the recurrent cache be in an undefined context at this point?\n            LLAMA_LOG_ERROR(\"%s: failed to prepare recurrent ubatches\\n\", __func__);\n            return std::make_unique<llama_memory_hybrid_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n        }\n\n        // prepare the attention cache\n        auto heads_attn = mem_attn->prepare(ubatches);\n        if (heads_attn.empty()) {\n            LLAMA_LOG_ERROR(\"%s: failed to prepare attention ubatches\\n\", __func__);\n            return std::make_unique<llama_memory_hybrid_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n        }\n\n        return std::make_unique<llama_memory_hybrid_context>(\n                this, std::move(heads_attn), std::move(ubatches));\n    } while(false);\n\n    return std::make_unique<llama_memory_hybrid_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n}\n\nllama_memory_context_ptr llama_memory_hybrid::init_full() {\n    return std::make_unique<llama_memory_hybrid_context>(this);\n}\n\nllama_memory_context_ptr llama_memory_hybrid::init_update(llama_context * lctx, bool optimize) {\n    return std::make_unique<llama_memory_hybrid_context>(this, lctx, optimize);\n}\n\nbool llama_memory_hybrid::get_can_shift() const {\n    // Shifting is trivially supported for recurrent\n    return mem_attn->get_can_shift();\n}\n\nvoid llama_memory_hybrid::clear(bool data) {\n    mem_attn->clear(data);\n    mem_recr->clear(data);\n}\n\nbool llama_memory_hybrid::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {\n    // Try removing from the recurrent cache first since it may fail. If it does\n    // fail, the cache will not have been mutated.\n    if (!mem_recr->seq_rm(seq_id, p0, p1)) {\n        return false;\n    }\n    return mem_attn->seq_rm(seq_id, p0, p1);\n}\n\nvoid llama_memory_hybrid::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {\n    mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n    mem_recr->seq_cp(seq_id_src, seq_id_dst, p0, p1);\n}\n\nvoid llama_memory_hybrid::seq_keep(llama_seq_id seq_id) {\n    mem_attn->seq_keep(seq_id);\n    mem_recr->seq_keep(seq_id);\n}\n\nvoid llama_memory_hybrid::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {\n    mem_attn->seq_add(seq_id, p0, p1, shift);\n    mem_recr->seq_add(seq_id, p0, p1, shift);\n}\n\nvoid llama_memory_hybrid::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {\n    mem_attn->seq_div(seq_id, p0, p1, d);\n    mem_recr->seq_div(seq_id, p0, p1, d);\n}\n\nllama_pos llama_memory_hybrid::seq_pos_min(llama_seq_id seq_id) const {\n    // the min of the total cache is the max of the two caches' min values\n    return std::max(mem_attn->seq_pos_min(seq_id), mem_recr->seq_pos_min(seq_id));\n}\n\nllama_pos llama_memory_hybrid::seq_pos_max(llama_seq_id seq_id) const {\n    // the max of the total cache is the min of the two caches' max values\n    return std::min(mem_attn->seq_pos_max(seq_id), mem_recr->seq_pos_max(seq_id));\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_memory_hybrid::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> mb = mem_attn->memory_breakdown();\n    for (const auto & buft_size : mem_recr->memory_breakdown()) {\n        mb[buft_size.first] += buft_size.second;\n    }\n    return mb;\n}\n\nvoid llama_memory_hybrid::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {\n    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {\n        mem_attn->state_write(io, seq_id, flags);\n    }\n    mem_recr->state_write(io, seq_id, flags);\n}\n\nvoid llama_memory_hybrid::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    if ((flags & LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY) == 0) {\n        mem_attn->state_read(io, seq_id, flags);\n    }\n    mem_recr->state_read(io, seq_id, flags);\n}\n\nllama_kv_cache * llama_memory_hybrid::get_mem_attn() const {\n    return mem_attn.get();\n}\n\nllama_memory_recurrent * llama_memory_hybrid::get_mem_recr() const {\n    return mem_recr.get();\n}\n\nllama_memory_hybrid_context::llama_memory_hybrid_context(llama_memory_status status) : status(status) {}\n\nllama_memory_hybrid_context::llama_memory_hybrid_context(llama_memory_hybrid * mem) :\n    ctx_attn(mem->get_mem_attn()->init_full()),\n    ctx_recr(mem->get_mem_recr()->init_full()),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nllama_memory_hybrid_context::llama_memory_hybrid_context(\n        llama_memory_hybrid * mem,\n              llama_context * lctx,\n                       bool   optimize) :\n    ctx_attn(mem->get_mem_attn()->init_update(lctx, optimize)),\n    ctx_recr(mem->get_mem_recr()->init_update(lctx, optimize)),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nllama_memory_hybrid_context::llama_memory_hybrid_context(\n              llama_memory_hybrid * mem,\n                  slot_info_vec_t   sinfos_attn,\n        std::vector<llama_ubatch>   ubatches) :\n    ubatches(std::move(ubatches)),\n    // note: here we copy the ubatches. not sure if this is ideal\n    ctx_attn(new llama_kv_cache_context(mem->get_mem_attn(), std::move(sinfos_attn), this->ubatches)),\n    ctx_recr(new llama_memory_recurrent_context(mem->get_mem_recr(), this->ubatches)),\n    status(llama_memory_status_combine(ctx_attn->get_status(), ctx_recr->get_status())) {\n}\n\nbool llama_memory_hybrid_context::next() {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    ctx_attn->next();\n    ctx_recr->next();\n\n    if (++i_next >= ubatches.size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_memory_hybrid_context::apply() {\n    assert(!llama_memory_status_is_fail(status));\n\n    bool res = true;\n\n    res = res & ctx_attn->apply();\n    res = res & ctx_recr->apply();\n\n    return res;\n}\n\nllama_memory_status llama_memory_hybrid_context::get_status() const {\n    return status;\n}\n\nconst llama_ubatch & llama_memory_hybrid_context::get_ubatch() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n    return ubatches[i_next];\n}\n\nconst llama_kv_cache_context * llama_memory_hybrid_context::get_attn() const {\n    return static_cast<const llama_kv_cache_context *>(ctx_attn.get());\n}\n\nconst llama_memory_recurrent_context * llama_memory_hybrid_context::get_recr() const {\n    return static_cast<const llama_memory_recurrent_context *>(ctx_recr.get());\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-hybrid.h",
    "content": "#pragma once\n\n#include \"llama-batch.h\"\n#include \"llama-graph.h\"\n#include \"llama-kv-cache.h\"\n#include \"llama-memory.h\"\n#include \"llama-memory-recurrent.h\"\n\n#include <memory>\n#include <vector>\n\n//\n// llama_memory_hybrid\n//\n\n// utilizes instances of llama_memory_recurrent and llama_kv_cache to\n//   support models where each layer may be either attention-based or recurrent\n\nclass llama_memory_hybrid : public llama_memory_i {\npublic:\n    llama_memory_hybrid(\n        const llama_model & model,\n                            /* attn */\n                ggml_type   type_k,\n                ggml_type   type_v,\n                     bool   v_trans,\n                 uint32_t   kv_size,\n                 uint32_t   n_pad,\n                 uint32_t   n_swa,\n           llama_swa_type   swa_type,\n                            /* recurrent */\n                ggml_type   type_r,\n                ggml_type   type_s,\n                 uint32_t   rs_size,\n                            /* common */\n                 uint32_t   n_seq_max,\n                     bool   offload,\n                     bool   unified,\n                            /* layer filters */\n    const layer_filter_cb & filter_attn = nullptr,\n    const layer_filter_cb & filter_recr = nullptr);\n\n    ~llama_memory_hybrid() = default;\n\n    //\n    // llama_memory_i\n    //\n\n    llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) override;\n\n    llama_memory_context_ptr init_full() override;\n\n    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;\n\n    bool get_can_shift() const override;\n\n    void clear(bool data) override;\n\n    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;\n    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;\n    void seq_keep(llama_seq_id seq_id)                                                          override;\n    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;\n    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;\n\n    llama_pos seq_pos_min(llama_seq_id seq_id) const override;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const override;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;\n\n    // state write/load\n\n    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;\n    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0)       override;\n\n    //\n    // llama_memory_hybrid specific API\n    //\n\n    llama_kv_cache * get_mem_attn() const;\n    llama_memory_recurrent * get_mem_recr() const;\n\nprivate:\n    const llama_hparams & hparams;\n\n    const std::unique_ptr<llama_kv_cache> mem_attn;\n    const std::unique_ptr<llama_memory_recurrent> mem_recr;\n};\n\nclass llama_memory_hybrid_context : public llama_memory_context_i {\npublic:\n    using slot_info_vec_t = llama_kv_cache::slot_info_vec_t;\n\n    // init failure\n    explicit llama_memory_hybrid_context(llama_memory_status status);\n\n    // init full\n    explicit llama_memory_hybrid_context(llama_memory_hybrid * mem);\n\n    // init update\n    explicit llama_memory_hybrid_context(\n        llama_memory_hybrid * mem,\n              llama_context * lctx,\n                       bool   optimize);\n\n    // init success\n    llama_memory_hybrid_context(\n              llama_memory_hybrid * mem,\n                  slot_info_vec_t   sinfos_attn,\n        std::vector<llama_ubatch>   ubatches);\n\n    ~llama_memory_hybrid_context() = default;\n\n    bool next()  override;\n    bool apply() override;\n\n    llama_memory_status  get_status() const override;\n    const llama_ubatch & get_ubatch() const override;\n\n    //\n    // llama_memory_hybrid_context\n    //\n\n    const llama_kv_cache_context * get_attn() const;\n    const llama_memory_recurrent_context * get_recr() const;\n\nprivate:\n    // the index of the next ubatch to process\n    size_t i_next = 0;\n\n    std::vector<llama_ubatch> ubatches;\n\n    const llama_memory_context_ptr ctx_attn;\n    const llama_memory_context_ptr ctx_recr;\n\n    const llama_memory_status status;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-recurrent.cpp",
    "content": "#include \"llama-memory-recurrent.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-io.h\"\n#include \"llama-batch.h\"\n#include \"llama-model.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cstring>\n#include <limits>\n#include <map>\n#include <stdexcept>\n\n//\n// llama_memory_recurrent\n//\n\nllama_memory_recurrent::llama_memory_recurrent(\n        const llama_model & model,\n                ggml_type   type_r,\n                ggml_type   type_s,\n                     bool   offload,\n                 uint32_t   mem_size,\n                 uint32_t   n_seq_max,\n    const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) {\n    const int32_t n_layer = hparams.n_layer;\n\n    head = 0;\n    size = mem_size;\n    used = 0;\n\n    cells.clear();\n    cells.resize(mem_size);\n\n    // define a comparator for the buft -> ctx map to ensure that the order is well-defined:\n    struct ggml_backend_buft_comparator {\n        bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {\n            return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;\n        }\n    };\n    std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator> ctx_map;\n\n    // create a context for each buffer type\n    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {\n        auto it = ctx_map.find(buft);\n        if (it == ctx_map.end()) {\n            ggml_init_params params = {\n                /*.mem_size   =*/ size_t(2u*n_layer*ggml_tensor_overhead()),\n                /*.mem_buffer =*/ NULL,\n                /*.no_alloc   =*/ true,\n            };\n\n            ggml_context * ctx = ggml_init(params);\n            if (!ctx) {\n                return nullptr;\n            }\n\n            ctx_map.emplace(buft, ctx);\n\n            return ctx;\n        }\n\n        return it->second.get();\n    };\n\n    r_l.resize(n_layer);\n    s_l.resize(n_layer);\n\n    for (int i = 0; i < n_layer; i++) {\n        if (filter && !filter(i)) {\n            LLAMA_LOG_DEBUG(\"%s: layer %3d: skipped\\n\", __func__, i);\n            continue;\n        }\n\n        const char * dev_name = \"CPU\";\n\n        ggml_backend_buffer_type_t buft = ggml_backend_cpu_buffer_type();\n\n        if (offload) {\n            auto * dev = model.dev_layer(i);\n            buft = ggml_backend_dev_buffer_type(dev);\n\n            dev_name = ggml_backend_dev_name(dev);\n        }\n\n        LLAMA_LOG_DEBUG(\"%s, layer %3d: dev = %s\\n\", __func__, i, dev_name);\n\n        ggml_context * ctx = ctx_for_buft(buft);\n        if (!ctx) {\n            throw std::runtime_error(\"failed to create ggml context for rs cache\");\n        }\n\n        ggml_tensor * r = ggml_new_tensor_1d(ctx, type_r, hparams.n_embd_r()*mem_size);\n        ggml_tensor * s = ggml_new_tensor_1d(ctx, type_s, hparams.n_embd_s()*mem_size);\n        ggml_format_name(r, \"cache_r_l%d\", i);\n        ggml_format_name(s, \"cache_s_l%d\", i);\n        r_l[i] = r;\n        s_l[i] = s;\n    }\n\n    // allocate tensors and initialize the buffers to avoid NaNs in the padding\n    for (auto & [buft, ctx] : ctx_map) {\n        ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft);\n        if (!buf) {\n            throw std::runtime_error(\"failed to allocate buffer for rs cache\");\n        }\n        ggml_backend_buffer_clear(buf, 0);\n        LLAMA_LOG_INFO(\"%s: %10s RS buffer size = %8.2f MiB\\n\", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);\n        ctxs_bufs.emplace_back(std::move(ctx), buf);\n    }\n\n    {\n        const size_t memory_size_r = size_r_bytes();\n        const size_t memory_size_s = size_s_bytes();\n\n        LLAMA_LOG_INFO(\"%s: size = %7.2f MiB (%6u cells, %3d layers, %2u seqs), R (%s): %7.2f MiB, S (%s): %7.2f MiB\\n\", __func__,\n                (float)(memory_size_r + memory_size_s) / (1024.0f * 1024.0f), mem_size, n_layer, n_seq_max,\n                ggml_type_name(type_r), (float)memory_size_r / (1024.0f * 1024.0f),\n                ggml_type_name(type_s), (float)memory_size_s / (1024.0f * 1024.0f));\n    }\n}\n\nvoid llama_memory_recurrent::clear(bool data) {\n    for (int32_t i = 0; i < (int32_t) size; ++i) {\n        cells[i].pos = -1;\n        cells[i].seq_id.clear();\n        cells[i].src = -1;\n        cells[i].tail = -1;\n    }\n\n    head = 0;\n    used = 0;\n\n    if (data) {\n        for (auto & [_, buf] : ctxs_bufs) {\n            ggml_backend_buffer_clear(buf.get(), 0);\n        }\n    }\n}\n\nbool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {\n    //printf(\"[DEBUG] calling llama_memory_recurrent::seq_rm` with `seq_id=%d, p0=%d, p1=%d`\\n\", seq_id, p0, p1);\n    uint32_t new_head = size;\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    // models like Mamba or RWKV can't have a state partially erased at the end\n    // of the sequence because their state isn't preserved for previous tokens\n    if (seq_id >= (int64_t) size) {\n        // could be fatal\n        return false;\n    }\n    if (0 <= seq_id) {\n        int32_t & tail_id = cells[seq_id].tail;\n        if (tail_id >= 0) {\n            const auto & cell = cells[tail_id];\n            // partial intersection is invalid if it includes the final pos\n            if (0 < p0 && p0 <= cell.pos && p1 > cell.pos) {\n                //printf(\"[DEBUG] inside `llama_memory_recurrent::seq_rm`: partial intersection is invalid, so returning false, p0 = %d, cell.pos = %d, p1 = %d\\n\", p0, cell.pos, p1);\n                return false;\n            }\n            // invalidate tails which will be cleared\n            if (p0 <= cell.pos && cell.pos < p1) {\n                tail_id = -1;\n            }\n        }\n    } else {\n        // seq_id is negative, then the range should include everything or nothing\n        if (p0 != p1 && (p0 != 0 || p1 != std::numeric_limits<llama_pos>::max())) {\n            //printf(\"[DEBUG] inside `llama_memory_recurrent::seq_rm`: `seq_id` is negative, so returning false\\n\");\n            return false;\n        }\n    }\n\n    for (uint32_t i = 0; i < size; ++i) {\n        if (cells[i].pos >= p0 && cells[i].pos < p1) {\n            if (seq_id < 0) {\n                cells[i].seq_id.clear();\n            } else if (cells[i].has_seq_id(seq_id)) {\n                cells[i].seq_id.erase(seq_id);\n            } else {\n                continue;\n            }\n            if (cells[i].is_empty()) {\n                // keep count of the number of used cells\n                if (cells[i].pos >= 0) {\n                    used--;\n                }\n                cells[i].pos = -1;\n                cells[i].src = -1;\n                if (new_head == size) {\n                    new_head = i;\n                }\n            }\n        }\n    }\n\n    // If we freed up a slot, set head to it so searching can start there.\n    if (new_head != size && new_head < head) {\n        head = new_head;\n    }\n\n    return true;\n}\n\nvoid llama_memory_recurrent::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) {\n    if (seq_id_src == seq_id_dst) {\n        return;\n    }\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    if ((uint32_t) seq_id_dst < size && (uint32_t) seq_id_src < size) {\n        auto & tail_src = cells[seq_id_src];\n        auto & tail_dst = cells[seq_id_dst];\n        if (tail_dst.tail >= 0) {\n            // clear destination seq_id if it wasn't empty\n            auto & cell_dst = cells[tail_dst.tail];\n\n            cell_dst.seq_id.erase(seq_id_dst);\n            tail_dst.tail = -1;\n            if (cell_dst.seq_id.empty()) {\n                cell_dst.pos = -1;\n                cell_dst.src = -1;\n                used -= 1;\n            }\n        }\n        if (tail_src.tail >= 0) {\n            auto & cell_src = cells[tail_src.tail];\n\n            cell_src.seq_id.insert(seq_id_dst);\n            tail_dst.tail = tail_src.tail;\n        }\n    }\n}\n\nvoid llama_memory_recurrent::seq_keep(llama_seq_id seq_id) {\n    uint32_t new_head = size;\n\n    for (uint32_t i = 0; i < size; ++i) {\n        if ((llama_seq_id) i != seq_id) {\n            cells[i].tail = -1;\n        }\n\n        if (!cells[i].has_seq_id(seq_id)) {\n            if (cells[i].pos >= 0) {\n                used--;\n            }\n\n            cells[i].pos = -1;\n            cells[i].src = -1;\n            cells[i].seq_id.clear();\n\n            if (new_head == size){\n                new_head = i;\n            }\n        } else {\n            cells[i].seq_id.clear();\n            cells[i].seq_id.insert(seq_id);\n        }\n    }\n\n    // If we freed up a slot, set head to it so searching can start there.\n    if (new_head != size && new_head < head) {\n        head = new_head;\n    }\n}\n\nvoid llama_memory_recurrent::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) {\n    if (shift == 0) {\n        return;\n    }\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    // If there is no range then return early to avoid looping over the\n    if (p0 == p1) {\n        return;\n    }\n\n    // for Mamba-like or RWKV models, only the pos needs to be shifted\n    if (0 <= seq_id && seq_id < (int64_t) size) {\n        const int32_t tail_id = cells[seq_id].tail;\n        if (tail_id >= 0) {\n            auto & cell = cells[tail_id];\n            if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {\n                cell.pos += shift;\n            }\n        }\n    }\n}\n\nvoid llama_memory_recurrent::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) {\n    if (d == 1) {\n        return;\n    }\n\n    if (p0 < 0) {\n        p0 = 0;\n    }\n\n    if (p1 < 0) {\n        p1 = std::numeric_limits<llama_pos>::max();\n    }\n\n    // If there is no range then return early to avoid looping over the cache.\n    if (p0 == p1) {\n        return;\n    }\n\n    // for Mamba-like or RWKV models, only the pos needs to be changed\n    if (0 <= seq_id && seq_id < (int64_t) size) {\n        const int32_t tail_id = cells[seq_id].tail;\n        if (tail_id >= 0) {\n            auto & cell = cells[tail_id];\n            if (cell.has_seq_id(seq_id) && p0 <= cell.pos && cell.pos < p1) {\n                cell.pos /= d;\n            }\n        }\n    }\n}\n\nllama_pos llama_memory_recurrent::seq_pos_min(llama_seq_id seq_id) const {\n    llama_pos result = std::numeric_limits<llama_pos>::max();\n\n    for (uint32_t i = 0; i < size; ++i) {\n        if (cells[i].has_seq_id(seq_id)) {\n            result = std::min(result, cells[i].pos);\n        }\n    }\n\n    if (result == std::numeric_limits<llama_pos>::max()) {\n        result = -1;\n    }\n\n    return result;\n}\n\nllama_pos llama_memory_recurrent::seq_pos_max(llama_seq_id seq_id) const {\n    llama_pos result = -1;\n\n    for (uint32_t i = 0; i < size; ++i) {\n        if (cells[i].has_seq_id(seq_id)) {\n            result = std::max(result, cells[i].pos);\n        }\n    }\n\n    return result;\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> ret;\n    for (const auto & [_, buf] : ctxs_bufs) {\n        ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());\n    }\n    return ret;\n}\n\nllama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {\n    do {\n        balloc.split_reset();\n\n        std::vector<llama_ubatch> ubatches;\n        while (true) {\n            llama_ubatch ubatch;\n\n            if (embd_all) {\n                // if all tokens are output, split by sequence\n                ubatch = balloc.split_seq(n_ubatch);\n            } else {\n                // TODO: non-sequential equal split can be done if using unified KV cache\n                //       for simplicity, we always use sequential equal split for now\n                ubatch = balloc.split_equal(n_ubatch, true);\n            }\n\n            if (ubatch.n_tokens == 0) {\n                break;\n            }\n\n            ubatches.push_back(std::move(ubatch)); // NOLINT\n        }\n\n        if (balloc.get_n_used() < balloc.get_n_tokens()) {\n            // failed to find a suitable split\n            break;\n        }\n\n        if (!prepare(ubatches)) {\n            break;\n        }\n\n        return std::make_unique<llama_memory_recurrent_context>(this, std::move(ubatches));\n    } while (false);\n\n    return std::make_unique<llama_memory_recurrent_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);\n}\n\nllama_memory_context_ptr llama_memory_recurrent::init_full() {\n    return std::make_unique<llama_memory_recurrent_context>(this);\n}\n\nllama_memory_context_ptr llama_memory_recurrent::init_update(llama_context * lctx, bool optimize) {\n    GGML_UNUSED(lctx);\n    GGML_UNUSED(optimize);\n\n    return std::make_unique<llama_memory_recurrent_context>(LLAMA_MEMORY_STATUS_NO_UPDATE);\n}\n\nbool llama_memory_recurrent::prepare(const std::vector<llama_ubatch> & ubatches) {\n    // simply remember the full state because it is very small for this type of cache\n    // TODO: optimize\n    auto org_cells = cells;\n    auto org_used = used;\n    auto org_head = head;\n\n    bool success = true;\n\n    for (const auto & ubatch : ubatches) {\n        if (!find_slot(ubatch)) {\n            success = false;\n            break;\n        }\n    }\n\n    // restore the original state\n    cells = std::move(org_cells);\n    used = org_used;\n    head = org_head;\n\n    return success;\n}\n\nbool llama_memory_recurrent::find_slot(const llama_ubatch & ubatch) {\n    const uint32_t n_seq_tokens = ubatch.n_seq_tokens;\n    const uint32_t n_seqs       = ubatch.n_seqs;\n\n    // if we have enough unused cells before the current head ->\n    //   better to start searching from the beginning of the cache, hoping to fill it\n    if (head > used + 2*n_seqs) {\n        head = 0;\n    }\n\n    // For recurrent state architectures (like Mamba or RWKV),\n    // each cache cell can store the state for a whole sequence.\n    // A slot should be always be contiguous.\n\n    // can only process batches with an equal number of new tokens in each sequence\n    GGML_ASSERT(ubatch.equal_seqs());\n\n    int32_t min = size - 1;\n    int32_t max = 0;\n\n    // everything should fit if all seq_ids are smaller than the max\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        const uint32_t i = s*n_seq_tokens; // first token of sequence set s\n        const uint32_t n_seq_id = ubatch.n_seq_id[i];\n\n        for (uint32_t j = 0; j < n_seq_id; ++j) {\n            const llama_seq_id seq_id = ubatch.seq_id[i][j];\n\n            if (seq_id < 0 || (uint32_t) seq_id >= size) {\n                // too big seq_id\n                // TODO: would it be possible to resize the cache instead?\n                LLAMA_LOG_ERROR(\"%s: seq_id=%d >= n_seq_max=%u Try using a bigger --parallel value\\n\", __func__, seq_id, n_seq_max);\n                return false;\n            }\n            if (j > 0) {\n                auto & seq = cells[seq_id];\n                if (seq.tail >= 0) {\n                    auto & cell = cells[seq.tail];\n                    // clear cells from seq_ids that become shared\n                    // (should not normally happen, but let's handle it anyway)\n                    cell.seq_id.erase(seq_id);\n                    seq.tail = -1;\n                    if (cell.seq_id.empty()) {\n                        cell.pos = -1;\n                        cell.src = -1;\n                        used -= 1;\n                    }\n                }\n            }\n        }\n    }\n\n#ifndef NDEBUG\n    {\n        std::vector<int32_t> tails_verif;\n        tails_verif.assign(size, -1);\n        for (uint32_t i = 0; i < size; ++i) {\n            auto & cell = cells[i];\n            for (llama_seq_id seq_id : cell.seq_id) {\n                if (tails_verif[seq_id] != -1) {\n                    LLAMA_LOG_ERROR(\"%s: duplicate tail for seq_id %d in cell %d and %d\\n\", __func__, seq_id, i, tails_verif[seq_id]);\n                }\n                tails_verif[seq_id] = i;\n            }\n        }\n        for (uint32_t i = 0; i < size; ++i) {\n            if (tails_verif[i] != cells[i].tail) {\n                LLAMA_LOG_ERROR(\"%s: wrong tail for seq_id %d, (%d instead of %d)\\n\", __func__, i, cells[i].tail, tails_verif[i]);\n            }\n        }\n    }\n#endif\n\n    // find next empty cell\n    uint32_t next_empty_cell = head;\n\n    for (uint32_t i = 0; i < size; ++i) {\n        if (next_empty_cell >= size) { next_empty_cell -= size; }\n        auto & cell = cells[next_empty_cell];\n        if (cell.is_empty()) { break; }\n        next_empty_cell += 1;\n    }\n\n    // find usable cell range\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        const uint32_t i = s*n_seq_tokens;\n        const llama_seq_id seq_id = ubatch.seq_id[i][0];\n        auto & seq_meta = cells[seq_id];\n        bool has_cell = false;\n        if (seq_meta.tail >= 0) {\n            auto & cell = cells[seq_meta.tail];\n            GGML_ASSERT(cell.has_seq_id(seq_id));\n            // does this seq_id \"own\" the cell?\n            if (cell.seq_id.size() == 1) { has_cell = true; }\n        }\n        if (!has_cell) {\n            auto & empty_cell = cells[next_empty_cell];\n            GGML_ASSERT(empty_cell.is_empty());\n            // copy old tail into the empty cell\n            if (seq_meta.tail >= 0) {\n                auto & orig_cell = cells[seq_meta.tail];\n                empty_cell.pos = orig_cell.pos;\n                empty_cell.src = orig_cell.src;\n                orig_cell.seq_id.erase(seq_id);\n                empty_cell.seq_id.insert(seq_id); // will be overwritten\n                GGML_ASSERT(!orig_cell.is_empty()); // has at least one remaining seq_id\n            }\n            seq_meta.tail = next_empty_cell;\n            // find next empty cell\n            if (s + 1 < n_seqs) {\n                for (uint32_t j = 0; j < size; ++j) {\n                    next_empty_cell += 1;\n                    if (next_empty_cell >= size) { next_empty_cell -= size; }\n                    auto & cell = cells[next_empty_cell];\n                    if (cell.is_empty()) { break; }\n                }\n            }\n        }\n        if (min > seq_meta.tail) { min = seq_meta.tail; }\n        if (max < seq_meta.tail) { max = seq_meta.tail; }\n    }\n\n    // gather and re-order\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        const uint32_t i = s*n_seq_tokens;\n        const int32_t dst_id = s + min;\n        const int32_t src_id = cells[ubatch.seq_id[i][0]].tail;\n        if (dst_id != src_id) {\n            auto & dst_cell = cells[dst_id];\n            auto & src_cell = cells[src_id];\n\n            std::swap(dst_cell.pos, src_cell.pos);\n            std::swap(dst_cell.src, src_cell.src);\n            std::swap(dst_cell.seq_id, src_cell.seq_id);\n\n            // swap tails\n            for (uint32_t j = 0; j < size; ++j) {\n                int32_t & tail = cells[j].tail;\n                if (tail == src_id) {\n                    tail = dst_id;\n                } else if (tail == dst_id) {\n                    tail = src_id;\n                }\n            }\n        }\n    }\n\n    // update the pos of the used seqs\n    for (uint32_t s = 0; s < n_seqs; ++s) {\n        const uint32_t i = s*n_seq_tokens;\n        const llama_pos last_pos = ubatch.pos[i + n_seq_tokens - 1];\n        const int32_t cell_id = s + min;\n        auto & cell = cells[cell_id];\n\n        if (cell.pos >= 0 && last_pos != cell.pos + (llama_pos) n_seq_tokens) {\n            // What should happen when the pos backtracks or skips a value?\n            // Clearing the state mid-batch would require special-casing which isn't done.\n            LLAMA_LOG_WARN(\"%s: non-consecutive token position %d after %d for sequence %d with %u new tokens\\n\",\n                __func__, last_pos, cell.pos, ubatch.seq_id[i][0], n_seq_tokens);\n        }\n        cell.pos = last_pos;\n        cell.seq_id.clear();\n        for (int32_t j = 0; j < ubatch.n_seq_id[i]; ++j) {\n            const llama_seq_id seq_id = ubatch.seq_id[i][j];\n            cell.seq_id.insert(seq_id);\n            cells[seq_id].tail = cell_id;\n        }\n    }\n\n    // Find first cell without src refs, to use as the zero-ed state\n    {\n        // TODO: bake-in src refcounts in the cell metadata\n        std::vector<int32_t> refcounts(size, 0);\n        for (size_t i = 0; i < size; ++i) {\n            const int32_t src = cells[i].src;\n            if (src >= 0) {\n                refcounts[src] += 1;\n            }\n        }\n\n        rs_z = -1;\n        for (int i = min; i <= max; ++i) {\n            if (refcounts[i] == 0) {\n                rs_z = i;\n                break;\n            }\n        }\n\n        for (int i = min; i <= max; ++i) {\n            if (cells[i].src < 0) {\n                GGML_ASSERT(rs_z >= 0);\n                cells[i].src0 = rs_z;\n            } else {\n                // Stage the source ids for all used cells to allow correct seq_* behavior\n                // and still make these values available when setting the inputs\n                cells[i].src0 = cells[i].src;\n            }\n            cells[i].src = i; // avoid moving or clearing twice\n        }\n    }\n\n    // allow getting the range of used cells, from head to head + n\n    head = min;\n    n    = max - min + 1;\n    used = std::count_if(cells.begin(), cells.end(),\n        [](const mem_cell & cell){ return !cell.is_empty(); });\n\n    // sanity check\n    return n >= n_seqs;\n}\n\nbool llama_memory_recurrent::get_can_shift() const {\n    // shifting the pos is trivial for recurrent models\n    return true;\n}\n\nsize_t llama_memory_recurrent::total_size() const {\n    size_t size = 0;\n    for (const auto & [_, buf] : ctxs_bufs) {\n        size += ggml_backend_buffer_get_size(buf.get());\n    }\n\n    return size;\n}\n\nsize_t llama_memory_recurrent::size_r_bytes() const {\n    size_t size_r_bytes = 0;\n\n    for (const auto & r : r_l) {\n        if (r != nullptr) {\n            size_r_bytes += ggml_nbytes(r);\n        }\n    }\n\n    return size_r_bytes;\n}\n\nsize_t llama_memory_recurrent::size_s_bytes() const {\n    size_t size_s_bytes = 0;\n\n    for (const auto & s : s_l) {\n        if (s != nullptr) {\n            size_s_bytes += ggml_nbytes(s);\n        }\n    }\n\n    return size_s_bytes;\n}\n\nvoid llama_memory_recurrent::state_write(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) const {\n    GGML_UNUSED(flags);\n\n    std::vector<std::pair<uint32_t, uint32_t>> cell_ranges; // ranges, from inclusive, to exclusive\n    uint32_t cell_count = 0;\n\n    // Count the number of cells with the specified seq_id\n    // Find all the ranges of cells with this seq id (or all, when -1)\n    uint32_t cell_range_begin = size;\n    for (uint32_t i = 0; i < size; ++i) {\n        const auto & cell = cells[i];\n        if ((seq_id == -1 && !cell.is_empty()) || cell.has_seq_id(seq_id)) {\n            ++cell_count;\n            if (cell_range_begin == size) {\n                cell_range_begin = i;\n            }\n        } else {\n            if (cell_range_begin != size) {\n                cell_ranges.emplace_back(cell_range_begin, i);\n                cell_range_begin = size;\n            }\n        }\n    }\n    if (cell_range_begin != size) {\n        cell_ranges.emplace_back(cell_range_begin, size);\n    }\n\n    // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count\n    uint32_t cell_count_check = 0;\n    for (const auto & range : cell_ranges) {\n        cell_count_check += range.second - range.first;\n    }\n    GGML_ASSERT(cell_count == cell_count_check);\n\n    io.write(&cell_count, sizeof(cell_count));\n\n    state_write_meta(io, cell_ranges, seq_id);\n    state_write_data(io, cell_ranges);\n}\n\nvoid llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {\n    GGML_UNUSED(flags);\n\n    uint32_t cell_count;\n    io.read_to(&cell_count, sizeof(cell_count));\n\n    bool res = true;\n\n    res = res && state_read_meta(io, cell_count, seq_id);\n    res = res && state_read_data(io, cell_count);\n\n    if (!res) {\n        if (seq_id == -1) {\n            clear(true);\n        } else {\n            seq_rm(seq_id, -1, -1);\n        }\n        throw std::runtime_error(\"failed to restore kv cache\");\n    }\n}\n\nvoid llama_memory_recurrent::state_write_meta(llama_io_write_i & io, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges, llama_seq_id seq_id) const {\n    for (const auto & range : cell_ranges) {\n        for (uint32_t i = range.first; i < range.second; ++i) {\n            const auto & cell = cells[i];\n            const llama_pos pos      = cell.pos;\n            const uint32_t  n_seq_id = seq_id == -1 ? cell.seq_id.size() : 0;\n\n            io.write(&pos,      sizeof(pos));\n            io.write(&n_seq_id, sizeof(n_seq_id));\n\n            if (n_seq_id) {\n                for (auto seq_id : cell.seq_id) {\n                    io.write(&seq_id, sizeof(seq_id));\n                }\n            }\n        }\n    }\n}\n\nvoid llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges) const {\n    const uint32_t s_trans = 0;\n    const uint32_t n_layer = hparams.n_layer;\n\n    io.write(&s_trans, sizeof(s_trans));\n    io.write(&n_layer,   sizeof(n_layer));\n\n    // Iterate and write all the R tensors first, each row is a cell\n    // Get whole range at a time\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        // skip null layers (read_data will handle this by checking \"r_l\" and \"s_l\" for null)\n        if (r_l[il] == nullptr) continue;\n\n        // Write R tensor type\n        const int32_t r_type_i = (int32_t)r_l[il]->type;\n        io.write(&r_type_i, sizeof(r_type_i));\n\n        // Write row size of R tensor\n        const uint64_t r_size_row = ggml_row_size(r_l[il]->type, hparams.n_embd_r());\n        io.write(&r_size_row, sizeof(r_size_row));\n\n        // Write each range of cells of r_size_row length\n        for (const auto & range : cell_ranges) {\n            const size_t range_size = range.second - range.first;\n            const size_t buf_size = range_size * r_size_row;\n            io.write_tensor(r_l[il], range.first * r_size_row, buf_size);\n        }\n    }\n\n    if (!s_trans) {\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            // skip null layers (read_data will handle this by checking \"r_l\" and \"s_l\" for null)\n            if (s_l[il] == nullptr) continue;\n\n            // Write S tensor type\n            const int32_t s_type_i = (int32_t)s_l[il]->type;\n            io.write(&s_type_i, sizeof(s_type_i));\n\n            // Write row size of S tensor\n            const uint64_t s_size_row = ggml_row_size(s_l[il]->type, hparams.n_embd_s());\n            io.write(&s_size_row, sizeof(s_size_row));\n\n            // Write each range of S tensor rows\n            for (const auto & range : cell_ranges) {\n                const size_t range_size = range.second - range.first;\n                const size_t buf_size = range_size * s_size_row;\n                io.write_tensor(s_l[il], range.first * s_size_row, buf_size);\n            }\n        }\n    } else {\n        // When S tensor is transposed, we also need the element size and get the element ranges from each row\n        const uint32_t mem_size = size;\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            // skip null layers (read_data will handle this by checking \"r_l\" and \"s_l\" for null)\n            if (s_l[il] == nullptr) continue;\n\n            const uint32_t n_embd_s = hparams.n_embd_s();\n\n            // Write S tensor type\n            const int32_t s_type_i = (int32_t)s_l[il]->type;\n            io.write(&s_type_i, sizeof(s_type_i));\n\n            // Write element size\n            const uint32_t s_size_el = ggml_type_size(s_l[il]->type);\n            io.write(&s_size_el, sizeof(s_size_el));\n\n            // Write GQA embedding size\n            io.write(&n_embd_s, sizeof(n_embd_s));\n\n            // For each row, we get the element values of each cell\n            for (uint32_t j = 0; j < n_embd_s; ++j) {\n                // Write each range of cells of s_size_el length\n                for (const auto & range : cell_ranges) {\n                    const size_t range_size = range.second - range.first;\n                    const size_t src_offset = (range.first + j * mem_size) * s_size_el;\n                    const size_t buf_size = range_size * s_size_el;\n                    io.write_tensor(s_l[il], src_offset, buf_size);\n                }\n            }\n        }\n    }\n}\n\nbool llama_memory_recurrent::state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id) {\n    if (dest_seq_id != -1) {\n        // single sequence\n        seq_rm(dest_seq_id, -1, -1);\n\n        if (cell_count == 0) {\n            return true;\n        }\n\n        llama_batch_allocr balloc(hparams.n_pos_per_embd());\n\n        llama_ubatch ubatch = balloc.ubatch_reserve(cell_count, 1);\n\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            llama_pos pos;\n            uint32_t n_seq_id;\n\n            io.read_to(&pos,      sizeof(pos));\n            io.read_to(&n_seq_id, sizeof(n_seq_id));\n\n            if (n_seq_id != 0) {\n                LLAMA_LOG_ERROR(\"%s: invalid seq_id-agnostic kv cell\\n\", __func__);\n                return false;\n            }\n\n            ubatch.pos[i] = pos;\n        }\n        ubatch.n_seq_id[0] = 1;\n        ubatch.seq_id[0] = &dest_seq_id;\n\n        if (!find_slot(ubatch)) {\n            LLAMA_LOG_ERROR(\"%s: failed to find available cells in kv cache\\n\", __func__);\n            return false;\n        }\n\n        // DEBUG CHECK: kv.head should be our first cell, kv.head + cell_count - 1 should be our last cell (verify seq_id and pos values)\n        // Assume that this is one contiguous block of cells\n        GGML_ASSERT(head + cell_count <= size);\n        GGML_ASSERT(cells[head].pos == ubatch.pos[0]);\n        GGML_ASSERT(cells[head + cell_count - 1].pos == ubatch.pos[cell_count - 1]);\n        GGML_ASSERT(cells[head].has_seq_id(dest_seq_id));\n        GGML_ASSERT(cells[head + cell_count - 1].has_seq_id(dest_seq_id));\n    } else {\n        // whole KV cache restore\n\n        if (cell_count > size) {\n            LLAMA_LOG_ERROR(\"%s: not enough cells in kv cache\\n\", __func__);\n            return false;\n        }\n\n        clear(true);\n\n        for (uint32_t i = 0; i < cell_count; ++i) {\n            auto & cell = cells[i];\n\n            llama_pos pos;\n            uint32_t  n_seq_id;\n\n            io.read_to(&pos,      sizeof(pos));\n            io.read_to(&n_seq_id, sizeof(n_seq_id));\n\n            cell.pos = pos;\n\n            for (uint32_t j = 0; j < n_seq_id; ++j) {\n                llama_seq_id seq_id;\n                io.read_to(&seq_id, sizeof(seq_id));\n\n                // TODO: llama_memory_recurrent should have a notion of max sequences\n                //if (seq_id < 0 || (uint32_t) seq_id >= llama_n_seq_max(ctx)) {\n                if (seq_id < 0) {\n                    //LLAMA_LOG_ERROR(\"%s: invalid seq_id, %d is out of range [0, %u)\\n\", __func__, seq_id, llama_n_seq_max(ctx));\n                    LLAMA_LOG_ERROR(\"%s: invalid seq_id, %d is out of range [0, inf)\\n\", __func__, seq_id);\n                    return false;\n                }\n\n                cell.seq_id.insert(seq_id);\n\n                int32_t & tail = cells[seq_id].tail;\n                if (tail != -1) {\n                    LLAMA_LOG_ERROR(\"%s: duplicate tail for seq_id %d in cell %d and %d\\n\", __func__, seq_id, i, tail);\n                    return false;\n                }\n                tail = i;\n            }\n        }\n\n        head = 0;\n        used = cell_count;\n    }\n\n    for (uint32_t i = 0; i < cell_count; ++i) {\n        uint32_t cell_id = head + i;\n        // make sure the recurrent states will keep their restored state\n        cells[cell_id].src = cell_id;\n    }\n\n    return true;\n}\n\nbool llama_memory_recurrent::state_read_data(llama_io_read_i & io, uint32_t cell_count) {\n    uint32_t s_trans;\n    uint32_t n_layer;\n    io.read_to(&s_trans, sizeof(s_trans));\n    io.read_to(&n_layer, sizeof(n_layer));\n\n    if (n_layer != hparams.n_layer) {\n        LLAMA_LOG_ERROR(\"%s: mismatched layer count (%u instead of %u)\\n\", __func__, n_layer, hparams.n_layer);\n        return false;\n    }\n    if (cell_count > size) {\n        LLAMA_LOG_ERROR(\"%s: not enough cells in kv cache to restore state (%u > %u)\\n\", __func__, cell_count, size);\n        return false;\n    }\n    if (false != (bool) s_trans) {\n        LLAMA_LOG_ERROR(\"%s: incompatible s transposition\\n\", __func__);\n        return false;\n    }\n\n    // For each layer, read the keys for each cell, one row is one cell, read as one contiguous block\n    for (uint32_t il = 0; il < n_layer; ++il) {\n        // skip null layers\n        if (r_l[il] == nullptr) continue;\n\n        // Read type of key\n        int32_t r_type_i_ref;\n        io.read_to(&r_type_i_ref, sizeof(r_type_i_ref));\n        const int32_t r_type_i = (int32_t) r_l[il]->type;\n        if (r_type_i != r_type_i_ref) {\n            LLAMA_LOG_ERROR(\"%s: mismatched r type (%d != %d, layer %d)\\n\", __func__, r_type_i, r_type_i_ref, il);\n            return false;\n        }\n\n        // Read row size of key\n        uint64_t r_size_row_ref;\n        io.read_to(&r_size_row_ref, sizeof(r_size_row_ref));\n        const size_t r_size_row = ggml_row_size(r_l[il]->type, hparams.n_embd_r());\n        if (r_size_row != r_size_row_ref) {\n            LLAMA_LOG_ERROR(\"%s: mismatched r row size (%zu != %zu, layer %d)\\n\", __func__, r_size_row, (size_t) r_size_row_ref, il);\n            return false;\n        }\n\n        if (cell_count) {\n            // Read and set the keys for the whole cell range\n            ggml_backend_tensor_set(r_l[il], io.read(cell_count * r_size_row), head * r_size_row, cell_count * r_size_row);\n        }\n    }\n\n    if (!s_trans) {\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            // skip null layers\n            if (s_l[il] == nullptr) continue;\n\n            // Read type of value\n            int32_t s_type_i_ref;\n            io.read_to(&s_type_i_ref, sizeof(s_type_i_ref));\n            const int32_t s_type_i = (int32_t)s_l[il]->type;\n\n            if (s_type_i != s_type_i_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched s type (%d != %d, layer %d)\\n\", __func__, s_type_i, s_type_i_ref, il);\n                return false;\n            }\n\n            // Read row size of value\n            uint64_t s_size_row_ref;\n            io.read_to(&s_size_row_ref, sizeof(s_size_row_ref));\n            const size_t s_size_row = ggml_row_size(s_l[il]->type, hparams.n_embd_s());\n            if (s_size_row != s_size_row_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched s row size (%zu != %zu, layer %d)\\n\", __func__, s_size_row, (size_t) s_size_row_ref, il);\n                return false;\n            }\n\n            if (cell_count) {\n                // Read and set the values for the whole cell range\n                ggml_backend_tensor_set(s_l[il], io.read(cell_count * s_size_row), head * s_size_row, cell_count * s_size_row);\n            }\n        }\n    } else {\n        // For each layer, read the values for each cell (transposed)\n        for (uint32_t il = 0; il < n_layer; ++il) {\n            // skip null layers\n            if (s_l[il] == nullptr) continue;\n\n            const uint32_t n_embd_s = hparams.n_embd_s();\n\n            // Read type of value\n            int32_t s_type_i_ref;\n            io.read_to(&s_type_i_ref, sizeof(s_type_i_ref));\n            const int32_t s_type_i = (int32_t)s_l[il]->type;\n            if (s_type_i != s_type_i_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched s type (%d != %d, layer %d)\\n\", __func__, s_type_i, s_type_i_ref, il);\n                return false;\n            }\n\n            // Read element size of value\n            uint32_t s_size_el_ref;\n            io.read_to(&s_size_el_ref, sizeof(s_size_el_ref));\n            const size_t s_size_el = ggml_type_size(s_l[il]->type);\n            if (s_size_el != s_size_el_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched s element size (%zu != %zu, layer %d)\\n\", __func__, s_size_el, (size_t) s_size_el_ref, il);\n                return false;\n            }\n\n            // Read state embedding size\n            uint32_t n_embd_s_ref;\n            io.read_to(&n_embd_s_ref, sizeof(n_embd_s_ref));\n            if (n_embd_s != n_embd_s_ref) {\n                LLAMA_LOG_ERROR(\"%s: mismatched s embedding size (%u != %u, layer %d)\\n\", __func__, n_embd_s, n_embd_s_ref, il);\n                return false;\n            }\n\n            if (cell_count) {\n                // For each row in the transposed matrix, read the values for the whole cell range\n                for (uint32_t j = 0; j < n_embd_s; ++j) {\n                    const size_t dst_offset = (head + j * size) * s_size_el;\n                    ggml_backend_tensor_set(s_l[il], io.read(cell_count * s_size_el), dst_offset, cell_count * s_size_el);\n                }\n            }\n        }\n    }\n\n    return true;\n}\n\n//\n// llama_memory_recurrent_context\n//\n\nllama_memory_recurrent_context::llama_memory_recurrent_context(llama_memory_status status) : status(status) {}\n\nllama_memory_recurrent_context::llama_memory_recurrent_context(\n        llama_memory_recurrent * mem) : status(LLAMA_MEMORY_STATUS_SUCCESS), mem(mem), is_full(true) {\n}\n\nllama_memory_recurrent_context::llama_memory_recurrent_context(\n        llama_memory_recurrent * mem,\n        std::vector<llama_ubatch> ubatches) : status(LLAMA_MEMORY_STATUS_SUCCESS), mem(mem), ubatches(std::move(ubatches)) {}\n\nllama_memory_recurrent_context::~llama_memory_recurrent_context() = default;\n\nbool llama_memory_recurrent_context::next() {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    if (++i_next >= ubatches.size()) {\n        return false;\n    }\n\n    return true;\n}\n\nbool llama_memory_recurrent_context::apply() {\n    assert(!llama_memory_status_is_fail(status));\n\n    // no ubatches -> this is an update\n    if (ubatches.empty()) {\n        // recurrent cache never performs updates\n        assert(status == LLAMA_MEMORY_STATUS_NO_UPDATE);\n\n        return true;\n    }\n\n    mem->find_slot(ubatches[i_next]);\n\n    return true;\n}\n\nllama_memory_status llama_memory_recurrent_context::get_status() const {\n    return status;\n}\n\nconst llama_ubatch & llama_memory_recurrent_context::get_ubatch() const {\n    assert(status == LLAMA_MEMORY_STATUS_SUCCESS);\n\n    return ubatches[i_next];\n}\n\nuint32_t llama_memory_recurrent_context::get_n_rs() const {\n    return is_full ? mem->size : mem->n;\n}\n\nuint32_t llama_memory_recurrent_context::get_head() const {\n    return is_full ? 0 : mem->head;\n}\n\nint32_t llama_memory_recurrent_context::get_rs_z() const {\n    return is_full ? 0 : mem->rs_z;\n}\n\nuint32_t llama_memory_recurrent_context::get_size() const {\n    return mem->size;\n}\n\nggml_tensor * llama_memory_recurrent_context::get_r_l(int32_t il) const {\n    return mem->r_l[il];\n}\n\nggml_tensor * llama_memory_recurrent_context::get_s_l(int32_t il) const {\n    return mem->s_l[il];\n}\n\nint32_t llama_memory_recurrent_context::s_copy(int i) const {\n    return  mem->cells[i + mem->head].src0;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-memory-recurrent.h",
    "content": "#pragma once\n\n#include \"llama-batch.h\"\n#include \"llama-graph.h\"\n#include \"llama-memory.h\"\n\n#include <map>\n#include <set>\n#include <vector>\n\n//\n// llama_memory_recurrent\n//\n\n// TODO: extract the cache state used for graph computation into llama_memory_recurrent_context_i\n//       see the implementation of llama_kv_cache_context_i for an example how to do it\nclass llama_memory_recurrent : public llama_memory_i {\npublic:\n    llama_memory_recurrent(\n            const llama_model & model,\n                    ggml_type   type_r,\n                    ggml_type   type_s,\n                         bool   offload,\n                     uint32_t   mem_size,\n                     uint32_t   n_seq_max,\n        const layer_filter_cb & filter);\n\n    ~llama_memory_recurrent() = default;\n\n    //\n    // llama_memory_i\n    //\n\n    llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) override;\n\n    llama_memory_context_ptr init_full() override;\n\n    llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) override;\n\n    void clear(bool data) override;\n\n    bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) override;\n    void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) override;\n    void seq_keep(llama_seq_id seq_id)                                                          override;\n    void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) override;\n    void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) override;\n\n    llama_pos seq_pos_min(llama_seq_id seq_id) const override;\n    llama_pos seq_pos_max(llama_seq_id seq_id) const override;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const override;\n\n    bool prepare(const std::vector<llama_ubatch> & ubatches);\n\n    // find a contiguous slot of memory cells and emplace the ubatch there\n    bool find_slot(const llama_ubatch & ubatch);\n\n    bool get_can_shift() const override;\n\n    // state write/load\n\n    void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override;\n    void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;\n\n    uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())\n    uint32_t size = 0; // total number of cells, shared across all sequences\n    uint32_t used = 0; // used cells (i.e. at least one seq_id)\n\n    // computed before each graph build\n    uint32_t n = 0;\n\n    // first zero-ed state\n    int32_t rs_z = -1;\n\n    // TODO: optimize for recurrent state needs\n    struct mem_cell {\n        llama_pos pos  = -1;\n        int32_t   src  = -1; // used to know where states should be copied from\n        int32_t   src0 = -1; // like src, but only used when setting the inputs (allowing to copy once)\n        int32_t   tail = -1;\n\n        std::set<llama_seq_id> seq_id;\n\n        bool has_seq_id(const llama_seq_id & id) const {\n            return seq_id.find(id) != seq_id.end();\n        }\n\n        bool is_empty() const {\n            return seq_id.empty();\n        }\n\n        bool is_same_seq(const mem_cell & other) const {\n            return seq_id == other.seq_id;\n        }\n    };\n\n    std::vector<mem_cell> cells;\n\n    // per layer\n    std::vector<ggml_tensor *> r_l;\n    std::vector<ggml_tensor *> s_l;\n\nprivate:\n    //const llama_model & model;\n    const llama_hparams & hparams;\n\n    const uint32_t n_seq_max = 1;\n\n    // ggml contexts for the KV cache along with the allocated backend buffers:\n    std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;\n\n    size_t total_size() const;\n\n    size_t size_r_bytes() const;\n    size_t size_s_bytes() const;\n\n    void state_write_meta(llama_io_write_i & io, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges, llama_seq_id seq_id = -1) const;\n    void state_write_data(llama_io_write_i & io, const std::vector<std::pair<uint32_t, uint32_t>> & cell_ranges) const;\n\n    bool state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id = -1);\n    bool state_read_data(llama_io_read_i & io, uint32_t cell_count);\n};\n\nclass llama_memory_recurrent_context : public llama_memory_context_i {\npublic:\n    // used for errors\n    llama_memory_recurrent_context(llama_memory_status status);\n\n    // used to create a full-cache or update context\n    llama_memory_recurrent_context(\n            llama_memory_recurrent * mem);\n\n    // used to create a batch processing context from a batch\n    llama_memory_recurrent_context(\n            llama_memory_recurrent * mem,\n            std::vector<llama_ubatch> ubatches);\n\n    virtual ~llama_memory_recurrent_context();\n\n    //\n    // llama_memory_context_i\n    //\n\n    bool next()  override;\n    bool apply() override;\n\n    llama_memory_status  get_status() const override;\n    const llama_ubatch & get_ubatch() const override;\n\n    //\n    // llama_memory_recurrent_context specific API\n    //\n\n    uint32_t get_n_rs() const;\n    uint32_t get_head() const;\n    int32_t  get_rs_z() const;\n    uint32_t get_size() const;\n\n    ggml_tensor * get_r_l(int32_t il) const;\n    ggml_tensor * get_s_l(int32_t il) const;\n\n    int32_t s_copy(int i) const;\n\nprivate:\n    const llama_memory_status status;\n\n    llama_memory_recurrent * mem;\n\n    size_t i_next = 0;\n\n    std::vector<llama_ubatch> ubatches;\n\n    //\n    // data needed for building the compute graph for the current ubatch:\n    // TODO: extract all the state like `head` and `n` here\n    //\n\n    const bool is_full = false;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-memory.cpp",
    "content": "#include \"llama-memory.h\"\n\nllama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) {\n    bool has_update = false;\n\n    switch (s0) {\n        case LLAMA_MEMORY_STATUS_SUCCESS:\n            {\n                has_update = true;\n                break;\n            }\n        case LLAMA_MEMORY_STATUS_NO_UPDATE:\n            {\n                break;\n            }\n        case LLAMA_MEMORY_STATUS_FAILED_PREPARE:\n        case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:\n            {\n                return s0;\n            }\n    }\n\n    switch (s1) {\n        case LLAMA_MEMORY_STATUS_SUCCESS:\n            {\n                has_update = true;\n                break;\n            }\n        case LLAMA_MEMORY_STATUS_NO_UPDATE:\n            {\n                break;\n            }\n        case LLAMA_MEMORY_STATUS_FAILED_PREPARE:\n        case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:\n            {\n                return s1;\n            }\n    }\n\n    // if either status has an update, then the combined status has an update\n    return has_update ? LLAMA_MEMORY_STATUS_SUCCESS : LLAMA_MEMORY_STATUS_NO_UPDATE;\n}\n\nbool llama_memory_status_is_fail(llama_memory_status status) {\n    switch (status) {\n        case LLAMA_MEMORY_STATUS_SUCCESS:\n        case LLAMA_MEMORY_STATUS_NO_UPDATE:\n            {\n                return false;\n            }\n        case LLAMA_MEMORY_STATUS_FAILED_PREPARE:\n        case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:\n            {\n                return true;\n            }\n    }\n\n    return false;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-memory.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <map>\n#include <memory>\n#include <functional>\n\nstruct llama_ubatch;\n\nclass llama_batch_allocr;\n\nclass llama_io_write_i;\nclass llama_io_read_i;\n\nstruct llama_memory_params {\n    // kv cache\n    ggml_type type_k;\n    ggml_type type_v;\n\n    // use full-size SWA cache\n    bool swa_full;\n};\n\nenum llama_memory_status {\n    LLAMA_MEMORY_STATUS_SUCCESS = 0,\n    LLAMA_MEMORY_STATUS_NO_UPDATE,\n    LLAMA_MEMORY_STATUS_FAILED_PREPARE,\n    LLAMA_MEMORY_STATUS_FAILED_COMPUTE,\n};\n\n// helper function for combining the status of two memory contexts\n// useful for implementing hybrid memory types (e.g. iSWA)\nllama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1);\n\n// helper function for checking if a memory status indicates a failure\nbool llama_memory_status_is_fail(llama_memory_status status);\n\n// the interface for managing the memory context during batch processing\n// this interface is implemented per memory type. see:\n//   - llama_kv_cache_context\n//   - llama_kv_cache_iswa_context\n//   ...\n//\n// the only method that should mutate the memory and the memory context is llama_memory_i::apply()\nstruct llama_memory_context_i {\n    virtual ~llama_memory_context_i() = default;\n\n    // consume the current ubatch from the context and proceed to the next one\n    // return false if we are done\n    virtual bool next() = 0;\n\n    // apply the memory state for the current ubatch to the memory object\n    // return false on failure\n    virtual bool apply() = 0;\n\n    // get the current ubatch\n    virtual const llama_ubatch & get_ubatch() const = 0;\n\n    // get the status of the memory context - used for error handling and checking if any updates would be applied\n    virtual llama_memory_status get_status() const = 0;\n};\n\nusing llama_memory_context_ptr = std::unique_ptr<llama_memory_context_i>;\n\n// general concept of LLM memory\n// the KV cache is a type of LLM memory, but there can be other types\nstruct llama_memory_i {\n    // this callback is used to filter out layers that should not be included in the cache\n    using layer_filter_cb = std::function<bool(int32_t il)>;\n\n    // this callback is used to specify which layers should reuse memory from other layers\n    // return negative value to indicate that the layer il should not reuse memory\n    using layer_reuse_cb = std::function<int32_t(int32_t il)>;\n\n    virtual ~llama_memory_i() = default;\n\n    // split the input batch into a set of ubatches and verify that they can fit into the cache\n    // return a context object containing the ubatches and memory state required to process them\n    // check the llama_memory_context_i::get_status() for the result\n    virtual llama_memory_context_ptr init_batch(\n            llama_batch_allocr & balloc,\n            uint32_t n_ubatch,\n            bool embd_all) = 0;\n\n    // simulate full cache, used for allocating worst-case compute buffers\n    virtual llama_memory_context_ptr init_full() = 0;\n\n    // prepare for any pending memory updates, such as shifts, copies, etc.\n    // status == LLAMA_MEMORY_STATUS_NO_UPDATE if there is nothing to update\n    virtual llama_memory_context_ptr init_update(llama_context * lctx, bool optimize) = 0;\n\n    // getters\n    virtual bool get_can_shift() const = 0;\n\n    //\n    // ops\n    //\n\n    // if data == true, the data buffers will also be cleared together with the metadata\n    virtual void clear(bool data) = 0;\n\n    virtual bool seq_rm  (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1) = 0;\n    virtual void seq_cp  (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) = 0;\n    virtual void seq_keep(llama_seq_id seq_id) = 0;\n    virtual void seq_add (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, llama_pos shift) = 0;\n    virtual void seq_div (llama_seq_id seq_id,                              llama_pos p0, llama_pos p1, int d) = 0;\n\n    virtual llama_pos seq_pos_min(llama_seq_id seq_id) const = 0;\n    virtual llama_pos seq_pos_max(llama_seq_id seq_id) const = 0;\n\n    virtual std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const = 0;\n\n    //\n    // state write/read\n    //\n\n    virtual void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const = 0;\n    virtual void state_read (llama_io_read_i  & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) = 0;\n};\n\nusing llama_memory_ptr = std::unique_ptr<llama_memory_i>;\n"
  },
  {
    "path": "examples/talk-llama/llama-mmap.cpp",
    "content": "#include \"llama-mmap.h\"\n\n#include \"llama-impl.h\"\n\n#include \"ggml.h\"\n\n#include <cstring>\n#include <climits>\n#include <stdexcept>\n#include <cerrno>\n#include <algorithm>\n\n#ifdef __has_include\n    #if __has_include(<unistd.h>)\n        #include <unistd.h>\n        #include <fcntl.h>\n        #include <sys/stat.h>\n        #if defined(_POSIX_MAPPED_FILES)\n            #include <sys/mman.h>\n        #endif\n        #if defined(_POSIX_MEMLOCK_RANGE)\n            #include <sys/resource.h>\n        #endif\n    #endif\n#endif\n\n#if defined(_WIN32)\n    #define WIN32_LEAN_AND_MEAN\n    #ifndef NOMINMAX\n        #define NOMINMAX\n    #endif\n    #include <windows.h>\n    #ifndef PATH_MAX\n        #define PATH_MAX MAX_PATH\n    #endif\n    #include <io.h>\n#endif\n\n#if defined(__APPLE__)\n#include <TargetConditionals.h>\n#endif\n\n// TODO: consider moving to llama-impl.h if needed in more places\n#if defined(_WIN32)\nstatic std::string llama_format_win_err(DWORD err) {\n    LPSTR buf;\n    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,\n                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);\n    if (!size) {\n        return \"FormatMessageA failed\";\n    }\n    std::string ret(buf, size);\n    LocalFree(buf);\n    return ret;\n}\n#endif\n\n// llama_file\n\nstruct llama_file::impl {\n#if defined(_WIN32)\n    HANDLE fp_win32;\n    std::string GetErrorMessageWin32(DWORD error_code) const {\n        std::string ret;\n        LPSTR lpMsgBuf = NULL;\n        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,\n                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);\n        if (!bufLen) {\n            ret = format(\"Win32 error code: %lx\", error_code);\n        } else {\n            ret = lpMsgBuf;\n            LocalFree(lpMsgBuf);\n        }\n\n        return ret;\n    }\n\n    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {\n        fp = ggml_fopen(fname, mode);\n        if (fp == NULL) {\n            throw std::runtime_error(format(\"failed to open %s: %s\", fname, strerror(errno)));\n        }\n        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));\n        seek(0, SEEK_END);\n        size = tell();\n        seek(0, SEEK_SET);\n    }\n\n    size_t tell() const {\n        LARGE_INTEGER li;\n        li.QuadPart = 0;\n        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);\n        if (!ret) {\n            throw std::runtime_error(format(\"read error: %s\", GetErrorMessageWin32(GetLastError()).c_str()));\n        }\n\n        return li.QuadPart;\n    }\n\n    void seek(size_t offset, int whence) const {\n        static_assert(SEEK_SET == FILE_BEGIN, \"SEEK_SET != FILE_BEGIN\");\n        static_assert(SEEK_CUR == FILE_CURRENT, \"SEEK_CUR != FILE_CURRENT\");\n        static_assert(SEEK_END == FILE_END, \"SEEK_END != FILE_END\");\n\n        LARGE_INTEGER li;\n        li.QuadPart = offset;\n        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);\n        if (!ret) {\n            throw std::runtime_error(format(\"read error: %s\", GetErrorMessageWin32(GetLastError()).c_str()));\n        }\n    }\n\n    void read_raw(void * ptr, size_t len) {\n        size_t bytes_read = 0;\n        while (bytes_read < len) {\n            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);\n            DWORD chunk_read = 0;\n            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);\n            if (!result) {\n                throw std::runtime_error(format(\"read error: %s\", GetErrorMessageWin32(GetLastError()).c_str()));\n            }\n            if (chunk_read < chunk_size || chunk_read == 0) {\n                throw std::runtime_error(\"unexpectedly reached end of file\");\n            }\n\n            bytes_read += chunk_read;\n        }\n    }\n\n    uint32_t read_u32() {\n        uint32_t val;\n        read_raw(&val, sizeof(val));\n        return val;\n    }\n\n    void write_raw(const void * ptr, size_t len) const {\n        size_t bytes_written = 0;\n        while (bytes_written < len) {\n            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);\n            DWORD chunk_written = 0;\n            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);\n            if (!result) {\n                throw std::runtime_error(format(\"write error: %s\", GetErrorMessageWin32(GetLastError()).c_str()));\n            }\n            if (chunk_written < chunk_size || chunk_written == 0) {\n                throw std::runtime_error(\"unexpectedly failed to write bytes\");\n            }\n\n            bytes_written += chunk_written;\n        }\n    }\n\n    void write_u32(uint32_t val) const {\n        write_raw(&val, sizeof(val));\n    }\n\n    bool has_direct_io() const {\n        return true;\n    }\n\n    ~impl() {\n        if (fp) {\n            std::fclose(fp);\n        }\n    }\n#else\n    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {\n#ifdef __linux__\n        // Try unbuffered I/O for read only\n        if (use_direct_io && std::strcmp(mode, \"rb\") == 0) {\n            if (init_fd()) {\n                return;\n            }\n            LLAMA_LOG_WARN(\"Failed to open file '%s' with error: %s. Falling back to buffered I/O\",\n                           fname, strerror(errno));\n        }\n#endif\n        init_fp(mode);\n    }\n\n#ifdef __linux__\n    bool init_fd() {\n        fd = open(fname.c_str(), O_RDONLY | O_DIRECT);\n\n        if (fd != -1) {\n            struct stat file_stats{};\n            fstat(fd, &file_stats);\n\n            size = file_stats.st_size;\n            alignment = file_stats.st_blksize;\n\n            off_t ret = lseek(fd, 0, SEEK_SET);\n            if (ret == -1) {\n                throw std::runtime_error(format(\"seek error: %s\", strerror(errno)));\n            }\n            return true;\n        }\n        return false;\n    }\n#endif\n\n    void init_fp(const char * mode) {\n        fp = ggml_fopen(fname.c_str(), mode);\n        if (fp == NULL) {\n            throw std::runtime_error(format(\"failed to open %s: %s\", fname.c_str(), strerror(errno)));\n        }\n        seek(0, SEEK_END);\n        size = tell();\n        seek(0, SEEK_SET);\n    }\n\n    size_t tell() const {\n        if (fd == -1) {\n            long ret = std::ftell(fp);\n            if (ret == -1) {\n                throw std::runtime_error(format(\"ftell error: %s\", strerror(errno)));\n            }\n\n            return (size_t) ret;\n        }\n\n        off_t pos = lseek(fd, 0, SEEK_CUR);\n        if (pos == -1) {\n            throw std::runtime_error(format(\"lseek error: %s\", strerror(errno)));\n        }\n        return (size_t) pos;\n    }\n\n    void seek(size_t offset, int whence) const {\n        off_t ret = 0;\n        if (fd == -1) {\n            ret = std::fseek(fp, (long) offset, whence);\n        } else {\n            ret = lseek(fd, offset, whence);\n        }\n        if (ret == -1) {\n            throw std::runtime_error(format(\"seek error: %s\", strerror(errno)));\n        }\n    }\n\n    void read_raw_unsafe(void * ptr, size_t len) {\n        if (len == 0) {\n            return;\n        }\n        errno = 0;\n        if (fd == -1) {\n            const size_t curr_off = tell();\n            const size_t to_read = std::min(len, size - curr_off);\n\n            std::size_t ret = std::fread(ptr, to_read, 1, fp);\n            if (ferror(fp)) {\n                throw std::runtime_error(format(\"read error: %s\", strerror(errno)));\n            }\n            if (to_read > 0 && ret != 1) {\n                throw std::runtime_error(\"unexpectedly reached end of file\");\n            }\n        } else {\n            size_t bytes_read = 0;\n            while (bytes_read < len) {\n                const size_t to_read = len - bytes_read;\n                ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);\n\n                if (ret == -1) {\n                    if (errno == EINTR) {\n                        continue;  // Interrupted by signal, retry\n                    }\n                    // Fallback to std::fread in case the DMA controller cannot access the buffer\n                    if (errno == EFAULT || errno == EINVAL) {\n                        LLAMA_LOG_WARN(\"%s: Falling back to buffered IO due to %s\\n\", __func__, strerror(errno));\n                        auto curr_off = tell();\n                        close(fd);\n                        fd = -1;\n                        alignment = 1;\n                        init_fp(\"rb\");\n                        seek(curr_off, SEEK_SET);\n                        read_raw_unsafe(ptr, len);\n                        return;\n                    }\n                    throw std::runtime_error(format(\"read error: %s\", strerror(errno)));\n                }\n                if (ret == 0) {\n                    // EOF: allow if this read was only pulling alignment padding past file end\n                    off_t pos = lseek(fd, 0, SEEK_CUR);\n                    if (pos != -1 && (size_t) pos == size) {\n                        std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);\n                        return;\n                    }\n                    throw std::runtime_error(\"unexpectedly reached end of file\");\n                }\n\n                bytes_read += (size_t) ret;\n            }\n        }\n    }\n\n    void read_aligned_chunk(void * dest, size_t size) {\n        size_t offset = tell();\n        off_t aligned_offset = offset & ~(alignment - 1);\n        off_t offset_from_alignment = offset - aligned_offset;\n        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);\n\n        void * raw_buffer = nullptr;\n        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);\n        if (ret != 0) {\n            throw std::runtime_error(format(\"posix_memalign failed with error %d\", ret));\n        }\n\n        struct aligned_buffer_deleter {\n            void operator()(void * p) const { free(p); }\n        };\n        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);\n\n        seek(aligned_offset, SEEK_SET);\n        read_raw_unsafe(buffer.get(), bytes_to_read);\n\n        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;\n        memcpy(dest, reinterpret_cast<void *>(actual_data), size);\n    }\n\n    void read_raw(void * ptr, size_t len) {\n        if (has_direct_io()) {\n            read_aligned_chunk(ptr, len);\n        } else {\n            read_raw_unsafe(ptr, len);\n        }\n    }\n\n    uint32_t read_u32() {\n        uint32_t ret;\n        read_raw(&ret, sizeof(ret));\n        return ret;\n    }\n\n    void write_raw(const void * ptr, size_t len) const {\n        if (len == 0) {\n            return;\n        }\n        errno = 0;\n        size_t ret = std::fwrite(ptr, len, 1, fp);\n        if (ret != 1) {\n            throw std::runtime_error(format(\"write error: %s\", strerror(errno)));\n        }\n    }\n\n    void write_u32(uint32_t val) const {\n        write_raw(&val, sizeof(val));\n    }\n\n    bool has_direct_io() const {\n        return fd != -1 && alignment > 1;\n    }\n\n    ~impl() {\n        if (fd != -1) {\n            close(fd);\n        } else {\n            std::fclose(fp);\n        }\n    }\n    int fd = -1;\n    std::string fname;\n#endif\n\n    size_t read_alignment() const {\n        return alignment;\n    }\n\n    size_t alignment = 1;\n\n    FILE * fp{};\n    size_t size{};\n};\n\nllama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :\n    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}\nllama_file::~llama_file() = default;\n\nsize_t llama_file::tell() const { return pimpl->tell(); }\nsize_t llama_file::size() const { return pimpl->size; }\n\nsize_t llama_file::read_alignment() const { return pimpl->read_alignment(); }\nbool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }\n\nint llama_file::file_id() const {\n#ifdef _WIN32\n    return _fileno(pimpl->fp);\n#else\n    if (pimpl->fd != -1) {\n        return pimpl->fd;\n    }\n#if defined(fileno)\n    return fileno(pimpl->fp);\n#else\n    return ::fileno(pimpl->fp);\n#endif\n#endif\n}\n\nvoid llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }\nvoid llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }\n#ifdef _WIN32\nvoid llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }\n#else\nvoid llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }\n#endif\n\nuint32_t llama_file::read_u32() { return pimpl->read_u32(); }\n\nvoid llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }\nvoid llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }\n\n// llama_mmap\n\nstruct llama_mmap::impl {\n#ifdef _POSIX_MAPPED_FILES\n    std::vector<std::pair<size_t, size_t>> mapped_fragments;\n\n    impl(struct llama_file * file, size_t prefetch, bool numa) {\n        size = file->size();\n        int fd = file->file_id();\n        int flags = MAP_SHARED;\n        if (numa) { prefetch = 0; }\n#ifdef __linux__\n        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {\n            LLAMA_LOG_WARN(\"warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\\n\",\n                    strerror(errno));\n        }\n        if (prefetch) { flags |= MAP_POPULATE; }\n#endif\n        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);\n        if (addr == MAP_FAILED) {\n            throw std::runtime_error(format(\"mmap failed: %s\", strerror(errno)));\n        }\n\n        if (prefetch > 0) {\n            if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {\n                LLAMA_LOG_WARN(\"warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\\n\",\n                        strerror(errno));\n            }\n        }\n        if (numa) {\n            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {\n                LLAMA_LOG_WARN(\"warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\\n\",\n                        strerror(errno));\n            }\n        }\n\n        mapped_fragments.emplace_back(0, file->size());\n    }\n\n    static void align_range(size_t * first, size_t * last, size_t page_size) {\n        size_t offset_in_page = *first & (page_size - 1);\n        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;\n        *first += offset_to_page;\n\n        *last = *last & ~(page_size - 1);\n\n        if (*last <= *first) {\n            *last = *first;\n        }\n    }\n\n    void unmap_fragment(size_t first, size_t last) {\n        int page_size = sysconf(_SC_PAGESIZE);\n        align_range(&first, &last, page_size);\n        size_t len = last - first;\n\n        if (len == 0) {\n            return;\n        }\n\n        GGML_ASSERT(first % page_size == 0);\n        GGML_ASSERT(last % page_size == 0);\n        GGML_ASSERT(last > first);\n\n        void * next_page_start = (uint8_t *) addr + first;\n\n        if (munmap(next_page_start, len)) {\n            LLAMA_LOG_WARN(\"warning: munmap failed: %s\\n\", strerror(errno));\n        }\n\n        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;\n        for (const auto & frag : mapped_fragments) {\n            if (frag.first < first && frag.second > last) {\n                new_mapped_fragments.emplace_back(frag.first, first);\n                new_mapped_fragments.emplace_back(last, frag.second);\n            } else if (frag.first < first && frag.second > first) {\n                new_mapped_fragments.emplace_back(frag.first, first);\n            } else if (frag.first < last && frag.second > last) {\n                new_mapped_fragments.emplace_back(last, frag.second);\n            } else if (frag.first >= first && frag.second <= last) {\n            } else {\n                new_mapped_fragments.push_back(frag);\n            }\n        }\n        mapped_fragments = std::move(new_mapped_fragments);\n    }\n\n    ~impl() {\n        for (const auto & frag : mapped_fragments) {\n            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {\n                LLAMA_LOG_WARN(\"warning: munmap failed: %s\\n\", strerror(errno));\n            }\n        }\n    }\n#elif defined(_WIN32)\n    HANDLE hMapping = nullptr;\n\n    impl(struct llama_file * file, size_t prefetch, bool numa) {\n        GGML_UNUSED(numa);\n\n        size = file->size();\n\n        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());\n\n        hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);\n\n        if (hMapping == NULL) {\n            DWORD error = GetLastError();\n            throw std::runtime_error(format(\"CreateFileMappingA failed: %s\", llama_format_win_err(error).c_str()));\n        }\n\n        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);\n        DWORD error = GetLastError();\n\n        if (addr == NULL) {\n            CloseHandle(hMapping);\n            throw std::runtime_error(format(\"MapViewOfFile failed: %s\", llama_format_win_err(error).c_str()));\n        }\n\n        if (prefetch > 0) {\n#if _WIN32_WINNT >= 0x602\n            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);\n            HMODULE hKernel32 = GetModuleHandleW(L\"kernel32.dll\");\n\n            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, \"PrefetchVirtualMemory\");\n\n            if (pPrefetchVirtualMemory) {\n                WIN32_MEMORY_RANGE_ENTRY range;\n                range.VirtualAddress = addr;\n                range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);\n                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {\n                    LLAMA_LOG_WARN(\"warning: PrefetchVirtualMemory failed: %s\\n\",\n                            llama_format_win_err(GetLastError()).c_str());\n                }\n            }\n#else\n            LLAMA_LOG_DEBUG(\"skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\\n\");\n#endif\n        }\n    }\n\n    void unmap_fragment(size_t first, size_t last) {\n        GGML_UNUSED(first);\n        GGML_UNUSED(last);\n    }\n\n    ~impl() {\n        if (hMapping) {\n            if (addr) {\n                if (!UnmapViewOfFile(addr)) {\n                    LLAMA_LOG_WARN(\"warning: UnmapViewOfFile failed: %s\\n\",\n                            llama_format_win_err(GetLastError()).c_str());\n                }\n            }\n            if (!CloseHandle(hMapping)) {\n                LLAMA_LOG_WARN(\"warning: CloseHandle failed: %s\\n\",\n                        llama_format_win_err(GetLastError()).c_str());\n            }\n        }\n    }\n#else\n    impl(struct llama_file * file, size_t prefetch, bool numa) {\n        GGML_UNUSED(file);\n        GGML_UNUSED(prefetch);\n        GGML_UNUSED(numa);\n\n        throw std::runtime_error(\"mmap not supported\");\n    }\n\n    void unmap_fragment(size_t first, size_t last) {\n        GGML_UNUSED(first);\n        GGML_UNUSED(last);\n\n        throw std::runtime_error(\"mmap not supported\");\n    }\n#endif\n\n    void * addr;\n    size_t size;\n};\n\nllama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}\nllama_mmap::~llama_mmap() = default;\n\nsize_t llama_mmap::size() const { return pimpl->size; }\nvoid * llama_mmap::addr() const { return pimpl->addr; }\n\nvoid llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }\n\n#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)\nconst bool llama_mmap::SUPPORTED  = true;\n#else\nconst bool llama_mmap::SUPPORTED  = false;\n#endif\n\n// llama_mlock\n\nstruct llama_mlock::impl {\n#ifdef _POSIX_MEMLOCK_RANGE\n    static size_t lock_granularity() {\n        return (size_t) sysconf(_SC_PAGESIZE);\n    }\n\n    bool raw_lock(const void * addr, size_t size) const {\n        if (!mlock(addr, size)) {\n            return true;\n        }\n\n#ifdef __APPLE__\n#define MLOCK_SUGGESTION \\\n        \"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or \" \\\n        \"decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\\n\"\n#else\n#define MLOCK_SUGGESTION \\\n        \"Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\\n\"\n#endif\n\n        char* errmsg = std::strerror(errno);\n        bool suggest = (errno == ENOMEM);\n#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)\n        // visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK\n        // Skip resource limit checks on these platforms\n        suggest = false;\n#else\n        struct rlimit lock_limit;\n        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {\n            suggest = false;\n        }\n        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {\n            suggest = false;\n        }\n#endif\n\n        LLAMA_LOG_WARN(\"warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\\n%s\",\n                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : \"\");\n        return false;\n    }\n\n    static void raw_unlock(void * addr, size_t size) {\n        if (munlock(addr, size)) {\n            LLAMA_LOG_WARN(\"warning: failed to munlock buffer: %s\\n\", std::strerror(errno));\n        }\n    }\n#elif defined(_WIN32)\n    static size_t lock_granularity() {\n        SYSTEM_INFO si;\n        GetSystemInfo(&si);\n        return (size_t) si.dwPageSize;\n    }\n\n    bool raw_lock(void * ptr, size_t len) const {\n        for (int tries = 1; ; tries++) {\n            if (VirtualLock(ptr, len)) {\n                return true;\n            }\n            if (tries == 2) {\n                LLAMA_LOG_WARN(\"warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\\n\",\n                    len, size, llama_format_win_err(GetLastError()).c_str());\n                return false;\n            }\n\n            SIZE_T min_ws_size, max_ws_size;\n            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {\n                LLAMA_LOG_WARN(\"warning: GetProcessWorkingSetSize failed: %s\\n\",\n                        llama_format_win_err(GetLastError()).c_str());\n                return false;\n            }\n            size_t increment = len + 1048576;\n            min_ws_size += increment;\n            max_ws_size += increment;\n            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {\n                LLAMA_LOG_WARN(\"warning: SetProcessWorkingSetSize failed: %s\\n\",\n                        llama_format_win_err(GetLastError()).c_str());\n                return false;\n            }\n        }\n    }\n\n    static void raw_unlock(void * ptr, size_t len) {\n        if (!VirtualUnlock(ptr, len)) {\n            LLAMA_LOG_WARN(\"warning: failed to VirtualUnlock buffer: %s\\n\",\n                    llama_format_win_err(GetLastError()).c_str());\n        }\n    }\n#else\n    static size_t lock_granularity() {\n        return (size_t) 65536;\n    }\n\n    bool raw_lock(const void * addr, size_t len) const {\n        LLAMA_LOG_WARN(\"warning: mlock not supported on this system\\n\");\n        return false;\n    }\n\n    static void raw_unlock(const void * addr, size_t len) {}\n#endif\n\n    impl() : addr(NULL), size(0), failed_already(false) {}\n\n    void init(void * ptr) {\n        GGML_ASSERT(addr == NULL && size == 0);\n        addr = ptr;\n    }\n\n    void grow_to(size_t target_size) {\n        GGML_ASSERT(addr);\n        if (failed_already) {\n            return;\n        }\n        size_t granularity = lock_granularity();\n        target_size = (target_size + granularity - 1) & ~(granularity - 1);\n        if (target_size > size) {\n            if (raw_lock((uint8_t *) addr + size, target_size - size)) {\n                size = target_size;\n            } else {\n                failed_already = true;\n            }\n        }\n    }\n\n    void * addr;\n    size_t size;\n\n    bool failed_already;\n};\n\nllama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}\nllama_mlock::~llama_mlock() = default;\n\nvoid llama_mlock::init(void * ptr) { pimpl->init(ptr); }\nvoid llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }\n\n#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)\nconst bool llama_mlock::SUPPORTED = true;\n#else\nconst bool llama_mlock::SUPPORTED = false;\n#endif\n\nsize_t llama_path_max() {\n    return PATH_MAX;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-mmap.h",
    "content": "#pragma once\n\n#include <cstdint>\n#include <memory>\n#include <vector>\n#include <cstdio>\n\nstruct llama_file;\nstruct llama_mmap;\nstruct llama_mlock;\n\nusing llama_files  = std::vector<std::unique_ptr<llama_file>>;\nusing llama_mmaps  = std::vector<std::unique_ptr<llama_mmap>>;\nusing llama_mlocks = std::vector<std::unique_ptr<llama_mlock>>;\n\nstruct llama_file {\n    llama_file(const char * fname, const char * mode, bool use_direct_io = false);\n    ~llama_file();\n\n    size_t tell() const;\n    size_t size() const;\n\n    int file_id() const; // fileno overload\n\n    void seek(size_t offset, int whence) const;\n\n    void read_raw(void * ptr, size_t len);\n    void read_raw_unsafe(void * ptr, size_t len);\n    void read_aligned_chunk(void * dest, size_t size);\n    uint32_t read_u32();\n\n    void write_raw(const void * ptr, size_t len) const;\n    void write_u32(uint32_t val) const;\n\n    size_t read_alignment() const;\n    bool has_direct_io() const;\nprivate:\n    struct impl;\n    std::unique_ptr<impl> pimpl;\n};\n\nstruct llama_mmap {\n    llama_mmap(const llama_mmap &) = delete;\n    llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false);\n    ~llama_mmap();\n\n    size_t size() const;\n    void * addr() const;\n\n    void unmap_fragment(size_t first, size_t last);\n\n    static const bool SUPPORTED;\n\nprivate:\n    struct impl;\n    std::unique_ptr<impl> pimpl;\n};\n\nstruct llama_mlock {\n    llama_mlock();\n    ~llama_mlock();\n\n    void init(void * ptr);\n    void grow_to(size_t target_size);\n\n    static const bool SUPPORTED;\n\nprivate:\n    struct impl;\n    std::unique_ptr<impl> pimpl;\n};\n\nsize_t llama_path_max();\n"
  },
  {
    "path": "examples/talk-llama/llama-model-loader.cpp",
    "content": "#include \"llama-model-loader.h\"\n\n#include \"ggml-alloc.h\"\n#include \"ggml.h\"\n#include \"gguf.h\"\n#include \"llama-hparams.h\"\n\n#include <algorithm>\n#include <array>\n#include <cinttypes>\n#include <cstdint>\n#include <cstring>\n#include <future>\n#include <regex>\n\nstatic const size_t kiB = 1024;\nstatic const size_t MiB = 1024*kiB;\nstatic const size_t GiB = 1024*MiB;\n\nconst char * llama_file_version_name(llama_fver version) {\n    switch (version) {\n        case GGUF_FILE_VERSION_V1: return \"GGUF V1 (support until nov 2023)\";\n        case GGUF_FILE_VERSION_V2: return \"GGUF V2\";\n        case GGUF_FILE_VERSION_V3: return \"GGUF V3 (latest)\";\n    }\n\n    return \"unknown\";\n}\n\nstatic std::string llama_model_ftype_name(llama_ftype ftype) {\n    if (ftype & LLAMA_FTYPE_GUESSED) {\n        return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + \" (guessed)\";\n    }\n\n    switch (ftype) {\n        case LLAMA_FTYPE_ALL_F32:         return \"all F32\";\n        case LLAMA_FTYPE_MOSTLY_F16:      return \"F16\";\n        case LLAMA_FTYPE_MOSTLY_BF16:     return \"BF16\";\n        case LLAMA_FTYPE_MOSTLY_Q4_0:     return \"Q4_0\";\n        case LLAMA_FTYPE_MOSTLY_Q4_1:     return \"Q4_1\";\n        case LLAMA_FTYPE_MOSTLY_Q5_0:     return \"Q5_0\";\n        case LLAMA_FTYPE_MOSTLY_Q5_1:     return \"Q5_1\";\n        case LLAMA_FTYPE_MOSTLY_Q8_0:     return \"Q8_0\";\n        case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return \"MXFP4 MoE\";\n        case LLAMA_FTYPE_MOSTLY_NVFP4:    return \"NVFP4\";\n        case LLAMA_FTYPE_MOSTLY_Q2_K:     return \"Q2_K - Medium\";\n        case LLAMA_FTYPE_MOSTLY_Q2_K_S:   return \"Q2_K - Small\";\n        case LLAMA_FTYPE_MOSTLY_Q3_K_S:   return \"Q3_K - Small\";\n        case LLAMA_FTYPE_MOSTLY_Q3_K_M:   return \"Q3_K - Medium\";\n        case LLAMA_FTYPE_MOSTLY_Q3_K_L:   return \"Q3_K - Large\";\n        case LLAMA_FTYPE_MOSTLY_Q4_K_S:   return \"Q4_K - Small\";\n        case LLAMA_FTYPE_MOSTLY_Q4_K_M:   return \"Q4_K - Medium\";\n        case LLAMA_FTYPE_MOSTLY_Q5_K_S:   return \"Q5_K - Small\";\n        case LLAMA_FTYPE_MOSTLY_Q5_K_M:   return \"Q5_K - Medium\";\n        case LLAMA_FTYPE_MOSTLY_Q6_K:     return \"Q6_K\";\n        case LLAMA_FTYPE_MOSTLY_TQ1_0:    return \"TQ1_0 - 1.69 bpw ternary\";\n        case LLAMA_FTYPE_MOSTLY_TQ2_0:    return \"TQ2_0 - 2.06 bpw ternary\";\n        case LLAMA_FTYPE_MOSTLY_IQ2_XXS:  return \"IQ2_XXS - 2.0625 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ2_XS:   return \"IQ2_XS - 2.3125 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ2_S:    return \"IQ2_S - 2.5 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ2_M:    return \"IQ2_M - 2.7 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ3_XS:   return \"IQ3_XS - 3.3 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ3_XXS:  return \"IQ3_XXS - 3.0625 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ1_S:    return \"IQ1_S - 1.5625 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ1_M:    return \"IQ1_M - 1.75 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ4_NL:   return \"IQ4_NL - 4.5 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ4_XS:   return \"IQ4_XS - 4.25 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ3_S:    return \"IQ3_S - 3.4375 bpw\";\n        case LLAMA_FTYPE_MOSTLY_IQ3_M:    return \"IQ3_S mix - 3.66 bpw\";\n\n        default: return \"unknown, may not work\";\n    }\n}\n\n// return a list of splits for a given path\n// for example, given \"<name>-00002-of-00004.gguf\", returns list of all 4 splits\nstatic std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {\n    std::vector<std::string> paths;\n    std::string split_prefix;\n    std::vector<char> buf(llama_path_max(), 0);\n\n    {\n        int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);\n        if (!ret) {\n            throw std::runtime_error(format(\"invalid split file name: %s\", path.c_str()));\n        }\n        split_prefix = std::string(buf.data(), ret);\n    }\n\n    if (split_prefix.empty()) {\n        throw std::runtime_error(format(\"invalid split file: %s\", path.c_str()));\n    }\n\n    for (int idx = 0; idx < n_split; ++idx) {\n        int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);\n        paths.push_back(std::string(buf.data(), ret));\n    }\n\n    return paths;\n}\n\nnamespace GGUFMeta {\n    template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>\n    struct GKV_Base_Type {\n        static constexpr gguf_type gt = gt_;\n\n        static T getter(const gguf_context * ctx, const int kid) {\n            return gfun(ctx, kid);\n        }\n    };\n\n    template<typename T> struct GKV_Base;\n\n    template<> struct GKV_Base<bool        >: GKV_Base_Type<bool,         GGUF_TYPE_BOOL,    gguf_get_val_bool> {};\n    template<> struct GKV_Base<uint8_t     >: GKV_Base_Type<uint8_t,      GGUF_TYPE_UINT8,   gguf_get_val_u8  > {};\n    template<> struct GKV_Base<uint16_t    >: GKV_Base_Type<uint16_t,     GGUF_TYPE_UINT16,  gguf_get_val_u16 > {};\n    template<> struct GKV_Base<uint32_t    >: GKV_Base_Type<uint32_t,     GGUF_TYPE_UINT32,  gguf_get_val_u32 > {};\n    template<> struct GKV_Base<uint64_t    >: GKV_Base_Type<uint64_t,     GGUF_TYPE_UINT64,  gguf_get_val_u64 > {};\n    template<> struct GKV_Base<int8_t      >: GKV_Base_Type<int8_t,       GGUF_TYPE_INT8,    gguf_get_val_i8  > {};\n    template<> struct GKV_Base<int16_t     >: GKV_Base_Type<int16_t,      GGUF_TYPE_INT16,   gguf_get_val_i16 > {};\n    template<> struct GKV_Base<int32_t     >: GKV_Base_Type<int32_t,      GGUF_TYPE_INT32,   gguf_get_val_i32 > {};\n    template<> struct GKV_Base<int64_t     >: GKV_Base_Type<int64_t,      GGUF_TYPE_INT64,   gguf_get_val_i64 > {};\n    template<> struct GKV_Base<float       >: GKV_Base_Type<float,        GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};\n    template<> struct GKV_Base<double      >: GKV_Base_Type<double,       GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};\n    template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING,  gguf_get_val_str > {};\n\n    template<> struct GKV_Base<std::string> {\n        static constexpr gguf_type gt = GGUF_TYPE_STRING;\n\n        static std::string getter(const gguf_context * ctx, const int kid) {\n            return gguf_get_val_str(ctx, kid);\n        }\n    };\n\n    struct ArrayInfo {\n        const gguf_type gt;\n        const size_t length;\n        const void * data;\n    };\n\n    template<> struct GKV_Base<ArrayInfo> {\n        public:\n        static constexpr gguf_type gt = GGUF_TYPE_ARRAY;\n        static ArrayInfo getter(const gguf_context *ctx, const int k) {\n            const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);\n            return ArrayInfo {\n                arr_type,\n                size_t(gguf_get_arr_n(ctx, k)),\n                arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),\n            };\n        }\n    };\n\n    template<typename T>\n    class GKV : public GKV_Base<T> {\n        GKV() = delete;\n\n        public:\n        static T get_kv(const gguf_context * ctx, const int k) {\n            const enum gguf_type kt = gguf_get_kv_type(ctx, k);\n\n            if (kt != GKV::gt) {\n                throw std::runtime_error(format(\"key %s has wrong type %s but expected type %s\",\n                    gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));\n            }\n            return GKV::getter(ctx, k);\n        }\n\n        static const char * override_type_to_str(const llama_model_kv_override_type ty) {\n            switch (ty) {\n                case LLAMA_KV_OVERRIDE_TYPE_BOOL:  return \"bool\";\n                case LLAMA_KV_OVERRIDE_TYPE_INT:   return \"int\";\n                case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return \"float\";\n                case LLAMA_KV_OVERRIDE_TYPE_STR:   return \"str\";\n            }\n            return \"unknown\";\n        }\n\n        static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {\n            if (!ovrd) { return false; }\n            if (ovrd->tag == expected_type) {\n                LLAMA_LOG_INFO(\"%s: Using metadata override (%5s) '%s' = \",\n                    __func__, override_type_to_str(ovrd->tag), ovrd->key);\n                switch (ovrd->tag) {\n                    case LLAMA_KV_OVERRIDE_TYPE_BOOL:  {\n                        LLAMA_LOG_INFO(\"%s\\n\", ovrd->val_bool ? \"true\" : \"false\");\n                    } break;\n                    case LLAMA_KV_OVERRIDE_TYPE_INT:   {\n                        LLAMA_LOG_INFO(\"%\" PRId64 \"\\n\", ovrd->val_i64);\n                    } break;\n                    case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {\n                        LLAMA_LOG_INFO(\"%.6f\\n\", ovrd->val_f64);\n                    } break;\n                    case LLAMA_KV_OVERRIDE_TYPE_STR: {\n                        LLAMA_LOG_INFO(\"%s\\n\", ovrd->val_str);\n                    } break;\n                    default:\n                        // Shouldn't be possible to end up here, but just in case...\n                        throw std::runtime_error(\n                            format(\"Unsupported attempt to override %s type for metadata key %s\\n\",\n                                override_type_to_str(ovrd->tag), ovrd->key));\n                }\n                return true;\n            }\n            LLAMA_LOG_WARN(\"%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\\n\",\n                __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));\n            return false;\n        }\n\n        template<typename OT>\n        static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type\n        try_override(OT & target, const struct llama_model_kv_override * ovrd) {\n            if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {\n                target = ovrd->val_bool;\n                return true;\n            }\n            return false;\n        }\n\n        template<typename OT>\n        static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type\n        try_override(OT & target, const struct llama_model_kv_override * ovrd) {\n            if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {\n                target = ovrd->val_i64;\n                return true;\n            }\n            return false;\n        }\n\n        template<typename OT>\n        static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type\n        try_override(T & target, const struct llama_model_kv_override * ovrd) {\n            if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {\n                target = ovrd->val_f64;\n                return true;\n            }\n            return false;\n        }\n\n        template<typename OT>\n        static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type\n        try_override(T & target, const struct llama_model_kv_override * ovrd) {\n            if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {\n                target = ovrd->val_str;\n                return true;\n            }\n            return false;\n        }\n\n        static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {\n            if (try_override<T>(target, ovrd)) {\n                return true;\n            }\n            if (k < 0) { return false; }\n            target = get_kv(ctx, k);\n            return true;\n        }\n\n        static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {\n            return set(ctx, gguf_find_key(ctx, key), target, ovrd);\n        }\n\n        static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {\n            return set(ctx, key.c_str(), target, ovrd);\n        }\n    };\n}\n\n    template<typename T>\n    typename std::enable_if<std::is_integral<T>::value, bool>::type\n    llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {\n        const int kid = gguf_find_key(metadata, key.c_str());\n\n        if (kid < 0) {\n            if (required) {\n                throw std::runtime_error(format(\"key not found in model: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        struct GGUFMeta::ArrayInfo arr_info =\n            GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);\n\n\n        result = arr_info.length;\n        return true;\n    }\n\n    template<typename T>\n    typename std::enable_if<std::is_integral<T>::value, bool>::type\n    llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {\n        return get_arr_n(llm_kv(kid), result, required);\n    }\n\n    template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);\n\n    template<typename T>\n    bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {\n        const gguf_context * ctx = metadata;\n        const int kid = gguf_find_key(ctx, key.c_str());\n\n        if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {\n            if (required) {\n                throw std::runtime_error(format(\"array key not found in model: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        struct GGUFMeta::ArrayInfo arr_info =\n            GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);\n\n        switch (arr_info.gt) {\n            case GGUF_TYPE_UINT32:\n            case GGUF_TYPE_INT32:   GGML_ASSERT((std::is_same<T,     int32_t>::value) ||\n                                                (std::is_same<T,    uint32_t>::value)); break;\n            case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T,       float>::value)); break;\n            case GGUF_TYPE_STRING:  GGML_ASSERT((std::is_same<T, std::string>::value)); break;\n            default:\n                throw std::runtime_error(format(\"%s is not a string/float32/uint32/int32 array\", key.c_str()));\n        }\n\n        if constexpr (std::is_same<T, std::string>::value) {\n            const size_t n_items = gguf_get_arr_n(ctx, kid);\n            result.clear();\n\n            for (size_t i = 0; i < n_items; i++) {\n                const T value = gguf_get_arr_str(ctx, kid, i);\n                result.emplace_back(value);\n            }\n        } else {\n            result.resize(arr_info.length);\n            result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);\n        }\n\n        return true;\n    }\n\n    template<typename T, size_t N_MAX>\n    bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {\n        const gguf_context * ctx = metadata;\n        const int kid = gguf_find_key(ctx, key.c_str());\n\n        if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {\n            if (required) {\n                throw std::runtime_error(format(\"array key not found in model: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        struct GGUFMeta::ArrayInfo arr_info =\n            GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);\n\n        switch (arr_info.gt) {\n            case GGUF_TYPE_BOOL:\n            case GGUF_TYPE_UINT32:\n            case GGUF_TYPE_INT32:   GGML_ASSERT((std::is_same<T,     int32_t>::value) ||\n                                                (std::is_same<T,    uint32_t>::value)); break;\n            case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T,       float>::value)); break;\n            case GGUF_TYPE_STRING:  GGML_ASSERT((std::is_same<T, std::string>::value)); break;\n            default:\n                throw std::runtime_error(format(\"%s is not a string/float32/uint32/int32 array\", key.c_str()));\n        }\n\n        if (arr_info.length > N_MAX) {\n            throw std::runtime_error(format(\"array length %u for key %s exceeds max %u\", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));\n        }\n\n        if constexpr (std::is_same<T, std::string>::value) {\n            const size_t n_items = gguf_get_arr_n(ctx, kid);\n\n            for (size_t i = 0; i < n_items; i++) {\n                const T value = gguf_get_arr_str(ctx, kid, i);\n                result[i] = value;\n            }\n        } else {\n            if (arr_info.gt == GGUF_TYPE_BOOL) {\n                std::transform((const bool *)arr_info.data, (const bool *)arr_info.data + arr_info.length, result.begin(), [](bool x) {\n                    return static_cast<T>(x);\n                });\n            } else {\n                std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());\n            }\n        }\n\n        return true;\n    }\n\n    template<typename T>\n    bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {\n        return get_arr(llm_kv(kid), result, required);\n    }\n\n    template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);\n\n    template<typename T>\n    bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {\n        auto it = kv_overrides.find(key);\n\n        const struct llama_model_kv_override * override =\n            it != kv_overrides.end() ? &it->second : nullptr;\n\n        const bool found = GGUFMeta::GKV<T>::set(metadata, key, result, override);\n\n        if (required && !found) {\n            throw std::runtime_error(format(\"key not found in model: %s\", key.c_str()));\n        }\n\n        return found;\n    }\n\n    template<typename T>\n    bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {\n        return get_key(llm_kv(kid), result, required);\n    }\n\n    template bool llama_model_loader::get_key<bool>       (enum llm_kv kid, bool & result,        bool required);\n    template bool llama_model_loader::get_key<float>      (enum llm_kv kid, float & result,       bool required);\n    template bool llama_model_loader::get_key<uint32_t>   (enum llm_kv kid, uint32_t & result,    bool required);\n    template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);\n\n    template<>\n    bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {\n        uint32_t tmp;\n        const bool found = get_key(kid, tmp, required);\n        if (found) {\n            result = (enum llama_pooling_type) tmp;\n        } else {\n            result = LLAMA_POOLING_TYPE_UNSPECIFIED;\n        }\n        return found;\n    }\n\n    // get array of n <= N_MAX elements, or a single element repeated n times\n    template<typename T, size_t N_MAX>\n    bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {\n        const int kid = gguf_find_key(metadata, key.c_str());\n\n        if (kid < 0) {\n            if (required) {\n                throw std::runtime_error(format(\"key not found in model: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        if (n > N_MAX) {\n            throw std::runtime_error(format(\"n > N_MAX: %u > %u for key %s\", (uint32_t) n, (uint32_t) N_MAX, key.c_str()));\n        }\n\n        if (gguf_get_kv_type(metadata, kid) == GGUF_TYPE_ARRAY) {\n            struct GGUFMeta::ArrayInfo arr_info =\n                GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);\n\n            if (n != arr_info.length) {\n                throw std::runtime_error(format(\"key %s has wrong array length; expected %u, got %u\", key.c_str(), n, (uint32_t) arr_info.length));\n            }\n\n            return get_arr(key, result, required);\n        }\n\n        T value;\n\n        bool ok = get_key(key, value, required);\n        if (!ok) {\n            return false;\n        }\n\n        for (uint32_t i = 0; i < n; i++) {\n            result[i] = value;\n        }\n\n        return true;\n    }\n\n    template<typename T>\n    bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {\n        return get_key_or_arr(llm_kv(kid), result, n, required);\n    }\n\n    bool llama_model_loader::get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required) {\n        const std::string key = llm_kv(kid);\n\n        const int id = gguf_find_key(metadata, key.c_str());\n\n        if (id < 0) {\n            if (required) {\n                throw std::runtime_error(format(\"key not found in model: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        // throw and error if type is an array\n        if (gguf_get_kv_type(metadata, id) == GGUF_TYPE_ARRAY) {\n            if (required) {\n                throw std::runtime_error(format(\"expected scalar, found array for key: %s\", key.c_str()));\n            }\n            return false;\n        }\n\n        return get_key(key, result, required);\n    }\n\n    // TODO: this is not very clever - figure out something better\n    template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);\n    template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);\n    template bool llama_model_loader::get_key_or_arr<std::array<float, 512>>(enum llm_kv kid, std::array<float, 512> & result, uint32_t n, bool required);\n\n\nllama_model_loader::llama_model_loader(\n        struct gguf_context * meta,\n        llama_model_set_tensor_data_t set_tensor_data,\n        void * set_tensor_data_ud,\n        const std::string & fname,\n        std::vector<std::string> & splits,\n        bool use_mmap,\n        bool use_direct_io,\n        bool check_tensors,\n        bool no_alloc,\n        const llama_model_kv_override * param_overrides_p,\n        const llama_model_tensor_buft_override * param_tensor_buft_overrides_p)\n        : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {\n    int trace = 0;\n    if (getenv(\"LLAMA_TRACE\")) {\n        trace = atoi(getenv(\"LLAMA_TRACE\"));\n    }\n\n    if (param_overrides_p != nullptr) {\n        for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {\n            kv_overrides.insert({std::string(p->key), *p});\n        }\n    }\n\n    tensor_buft_overrides = param_tensor_buft_overrides_p;\n\n    if (!fname.empty()) {\n        // Load the main GGUF\n        struct ggml_context * ctx = NULL;\n        struct gguf_init_params params = {\n            /*.no_alloc = */ true,\n            /*.ctx      = */ &ctx,\n        };\n\n        metadata_ptr.reset(gguf_init_from_file(fname.c_str(), params));\n        metadata = metadata_ptr.get();\n        if (metadata == nullptr) {\n            throw std::runtime_error(format(\"%s: failed to load model from %s\", __func__, fname.c_str()));\n        }\n\n        get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);\n        llm_kv = LLM_KV(llm_arch_from_string(arch_name));\n\n        files.emplace_back(new llama_file(fname.c_str(), \"rb\", use_direct_io));\n        contexts.emplace_back(ctx);\n\n        if (use_mmap && use_direct_io) {\n            if (files.back()->has_direct_io()) {\n                LLAMA_LOG_WARN(\"%s: direct I/O is enabled, disabling mmap\\n\", __func__);\n                use_mmap = false;\n            } else {\n                LLAMA_LOG_WARN(\"%s: direct I/O is not available, using mmap\\n\", __func__);\n                use_direct_io = false;\n\n                // reopen file using std::fopen for mmap\n                files.pop_back();\n                files.emplace_back(new llama_file(fname.c_str(), \"rb\", false));\n            }\n        }\n\n        // Save tensors data offset of the main file.\n        // For subsidiary files, `meta` tensor data offset must not be used,\n        // so we build a unified tensors index for weights.\n        for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {\n            std::string tensor_name = std::string(cur->name);\n            // make sure there is no duplicated tensor names\n            if (weights_map.find(tensor_name) != weights_map.end()) {\n                throw std::runtime_error(format(\"invalid model: tensor '%s' is duplicated\", ggml_get_name(cur)));\n            }\n            n_elements += ggml_nelements(cur);\n            n_bytes    += ggml_nbytes(cur);\n            weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));\n        }\n        uint16_t n_split = 0;\n        get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);\n\n        // Load additional GGML contexts\n        if (n_split > 1) {\n            // make sure the main file is loaded first\n            uint16_t idx = 0;\n            const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);\n            get_key(kv_split_no, idx);\n            if (idx != 0) {\n                throw std::runtime_error(format(\"illegal split file idx: %d (file: %s), model must be loaded with the first split\", idx, fname.c_str()));\n            }\n\n            // generate list of splits if needed\n            if (splits.empty()) {\n                splits = llama_get_list_splits(fname, idx, n_split);\n            }\n\n            // in case user give a custom list of splits, check if it matches the expected number\n            if (n_split != (uint16_t)splits.size()) {\n                throw std::runtime_error(format(\"invalid split count, given: %zu splits, but expected %d\", splits.size(), n_split));\n            }\n\n            if (trace > 0) {\n                LLAMA_LOG_INFO(\"%s: loading additional %d GGUFs\\n\", __func__, n_split);\n            }\n\n            // load other splits\n            for (idx = 1; idx < n_split; idx++) {\n                const char * fname_split = splits[idx].c_str();\n\n                struct gguf_init_params split_params = {\n                    /*.no_alloc = */ true,\n                    /*.ctx      = */ &ctx,\n                };\n                gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };\n                if (!ctx_gguf) {\n                    throw std::runtime_error(format(\"%s: failed to load GGUF split from %s\", __func__, fname_split));\n                }\n\n                // check idx\n                {\n                    const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());\n                    if (kid < 0) {\n                        throw std::runtime_error(format(\"missing key %s in GGUF split %s\", kv_split_no.c_str(), fname_split));\n                    }\n                    int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);\n                    if (idx_gguf != idx) {\n                        throw std::runtime_error(format(\"invalid split file idx: %d (file: %s), expected %d\", idx_gguf, fname_split, idx));\n                    }\n                }\n\n                files.emplace_back(new llama_file(fname_split, \"rb\", use_direct_io));\n                contexts.emplace_back(ctx);\n\n                // Save tensors data offset info of the shard.\n                for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {\n                    std::string tensor_name = std::string(cur->name);\n                    // make sure there is no duplicated tensor names\n                    if (weights_map.find(tensor_name) != weights_map.end()) {\n                        throw std::runtime_error(format(\"invalid model: tensor '%s' is duplicated\", ggml_get_name(cur)));\n                    }\n                    n_elements += ggml_nelements(cur);\n                    n_bytes    += ggml_nbytes(cur);\n                    weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));\n                }\n            }\n\n            get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);\n\n            // sanity check\n            {\n                const int n_tensors_loaded = (int) weights_map.size();\n                if (n_tensors != n_tensors_loaded) {\n                    throw std::runtime_error(format(\"corrupted model: %d tensors expected but %d found\", n_tensors, n_tensors_loaded));\n                }\n            }\n\n            LLAMA_LOG_INFO(\"%s: additional %d GGUFs metadata loaded.\\n\",  __func__, n_split - 1);\n        }\n    } else {\n        get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);\n        llm_kv = LLM_KV(llm_arch_from_string(arch_name));\n    }\n\n    n_kv      = gguf_get_n_kv(metadata);\n    n_tensors = weights_map.size();\n\n    fver = (enum llama_fver) gguf_get_version(metadata);\n\n    LLAMA_LOG_INFO(\"%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\\n\",\n            __func__, n_kv, n_tensors, fname.c_str(), llama_file_version_name(fver));\n\n    // determine file type based on the number of tensors for each quantization and print meta data\n    // TODO: make optional\n    {\n        std::map<enum ggml_type, uint32_t> n_type;\n\n        uint32_t n_type_max = 0;\n        enum ggml_type type_max = GGML_TYPE_F32;\n\n        for (const auto & it : weights_map) {\n            const llama_tensor_weight & w = it.second;\n            const ggml_tensor * tensor = w.tensor;\n\n            enum ggml_type type = tensor->type;\n\n            n_type[type]++;\n\n            if (n_type_max < n_type[type]) {\n                n_type_max = n_type[type];\n                type_max   = type;\n            }\n\n            if (trace > 0) {\n                const uint16_t sid = w.idx;\n                LLAMA_LOG_INFO(\"%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\\n\", __func__,\n                        sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),\n                        ggml_nbytes(tensor)/1024.0f/1024.0f);\n            }\n        }\n\n        switch (type_max) {\n            case GGML_TYPE_F32:     ftype = LLAMA_FTYPE_ALL_F32;        break;\n            case GGML_TYPE_F16:     ftype = LLAMA_FTYPE_MOSTLY_F16;     break;\n            case GGML_TYPE_BF16:    ftype = LLAMA_FTYPE_MOSTLY_BF16;    break;\n            case GGML_TYPE_Q4_0:    ftype = LLAMA_FTYPE_MOSTLY_Q4_0;    break;\n            case GGML_TYPE_Q4_1:    ftype = LLAMA_FTYPE_MOSTLY_Q4_1;    break;\n            case GGML_TYPE_Q5_0:    ftype = LLAMA_FTYPE_MOSTLY_Q5_0;    break;\n            case GGML_TYPE_Q5_1:    ftype = LLAMA_FTYPE_MOSTLY_Q5_1;    break;\n            case GGML_TYPE_Q8_0:    ftype = LLAMA_FTYPE_MOSTLY_Q8_0;    break;\n            case GGML_TYPE_Q2_K:    ftype = LLAMA_FTYPE_MOSTLY_Q2_K;    break;\n            case GGML_TYPE_Q3_K:    ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M;  break;\n            case GGML_TYPE_Q4_K:    ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M;  break;\n            case GGML_TYPE_Q5_K:    ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M;  break;\n            case GGML_TYPE_Q6_K:    ftype = LLAMA_FTYPE_MOSTLY_Q6_K;    break;\n            case GGML_TYPE_TQ1_0:   ftype = LLAMA_FTYPE_MOSTLY_TQ1_0;   break;\n            case GGML_TYPE_TQ2_0:   ftype = LLAMA_FTYPE_MOSTLY_TQ2_0;   break;\n            case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;\n            case GGML_TYPE_IQ2_XS:  ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS;  break;\n            case GGML_TYPE_IQ2_S:   ftype = LLAMA_FTYPE_MOSTLY_IQ2_S;   break;\n            case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;\n            case GGML_TYPE_IQ1_S:   ftype = LLAMA_FTYPE_MOSTLY_IQ1_S;   break;\n            case GGML_TYPE_IQ1_M:   ftype = LLAMA_FTYPE_MOSTLY_IQ1_M;   break;\n            case GGML_TYPE_IQ4_NL:  ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL;  break;\n            case GGML_TYPE_IQ4_XS:  ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS;  break;\n            case GGML_TYPE_IQ3_S:   ftype = LLAMA_FTYPE_MOSTLY_IQ3_S;   break;\n            case GGML_TYPE_NVFP4:   ftype = LLAMA_FTYPE_MOSTLY_NVFP4;   break;\n            default:\n                {\n                    LLAMA_LOG_WARN(\"%s: unknown type %s\\n\", __func__, ggml_type_name(type_max));\n                    ftype = LLAMA_FTYPE_ALL_F32;\n                } break;\n        }\n\n        // this is a way to mark that we have \"guessed\" the file type\n        ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);\n\n        {\n            uint32_t ftype_val = 0;\n            if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {\n                ftype = (llama_ftype) ftype_val;\n            }\n        }\n\n        LLAMA_LOG_INFO(\"%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\\n\", __func__);\n\n        for (int i = 0; i < n_kv; i++) {\n            const char * name           = gguf_get_key(metadata, i);\n            const enum gguf_type type   = gguf_get_kv_type(metadata, i);\n            const std::string type_name =\n                type == GGUF_TYPE_ARRAY\n                ? format(\"%s[%s,%zu]\", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(metadata, i)), gguf_get_arr_n(metadata, i))\n                : gguf_type_name(type);\n\n            std::string value          = gguf_kv_to_str(metadata, i);\n            const size_t MAX_VALUE_LEN = 40;\n            if (value.size() > MAX_VALUE_LEN) {\n                value = format(\"%s...\", value.substr(0, MAX_VALUE_LEN - 3).c_str());\n            }\n            replace_all(value, \"\\n\", \"\\\\n\");\n\n            LLAMA_LOG_INFO(\"%s: - kv %3d: %42s %-16s = %s\\n\", __func__, i, name, type_name.c_str(), value.c_str());\n        }\n\n        // print type counts\n        for (auto & kv : n_type) {\n            if (kv.second == 0) {\n                continue;\n            }\n\n            LLAMA_LOG_INFO(\"%s: - type %4s: %4d tensors\\n\", __func__, ggml_type_name(kv.first), kv.second);\n        }\n    }\n\n    if (!llama_mmap::SUPPORTED) {\n        LLAMA_LOG_WARN(\"%s: mmap is not supported on this platform\\n\", __func__);\n        use_mmap = false;\n    }\n\n    this->use_mmap = use_mmap;\n    this->use_direct_io = use_direct_io;\n    this->check_tensors = check_tensors;\n    this->no_alloc = no_alloc;\n}\n\nstd::string llama_model_loader::get_arch_name() const {\n    return arch_name;\n}\n\nenum llm_arch llama_model_loader::get_arch() const {\n    return llm_kv.arch;\n}\n\nconst llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {\n    auto pos = weights_map.find(name);\n    if (pos != weights_map.end()) {\n        return &pos->second;\n    }\n\n    return nullptr;\n}\n\nconst llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {\n    const llama_tensor_weight * weight = get_weight(name);\n    if (!weight) {\n        throw std::runtime_error(format(\"%s: tensor '%s' not found\", __func__, name));\n    }\n    return *weight;\n}\n\nstruct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {\n    const auto * weight = get_weight(name);\n    if (!weight) {\n        return nullptr;\n    }\n    return weight->tensor;\n}\n\nstruct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {\n    struct ggml_tensor * tensor = get_tensor_meta(name.c_str());\n    if (!tensor) {\n        throw std::runtime_error(format(\"%s: tensor '%s' not found\", __func__, name.c_str()));\n    }\n    return tensor;\n}\n\nconst struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {\n    const struct ggml_tensor * cur = get_tensor_meta(name.c_str());\n\n    if (cur == NULL) {\n        if (!required) {\n            return NULL;\n        }\n        throw std::runtime_error(format(\"%s: tensor '%s' not found\", __func__, name.c_str()));\n    }\n\n    {\n        bool is_ok = true;\n        for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {\n            if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {\n                is_ok = false;\n                break;\n            }\n        }\n        if (!is_ok) {\n            throw std::runtime_error(\n                    format(\"%s: tensor '%s' has wrong shape; expected %s, got %s\",\n                        __func__, name.c_str(),\n                        llama_format_tensor_shape(ne).c_str(),\n                        llama_format_tensor_shape(cur).c_str()));\n        }\n    }\n\n    return cur;\n}\n\n// checks if the weight tensor can be used with the specified buffer type and device\nstatic bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {\n    GGML_ASSERT(w != nullptr);\n\n    if (op == GGML_OP_NONE) {\n        return true;\n    }\n\n    ggml_init_params params = {\n        /*.mem_size   =*/ ggml_tensor_overhead()*8,\n        /*.mem_buffer =*/ NULL,\n        /*.no_alloc   =*/ true,\n    };\n    ggml_context_ptr ctx_ptr { ggml_init(params) };\n    if (!ctx_ptr) {\n        throw std::runtime_error(format(\"failed to create ggml context\"));\n    }\n    ggml_context * ctx = ctx_ptr.get();\n\n    ggml_tensor * op_tensor = nullptr;\n\n    switch (op) {\n        case GGML_OP_GET_ROWS:\n            {\n                ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);\n                op_tensor = ggml_get_rows(ctx, w, b);\n            } break;\n        case GGML_OP_MUL_MAT:\n            {\n                ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);\n                op_tensor = ggml_mul_mat(ctx, w, b);\n            } break;\n        case GGML_OP_MUL_MAT_ID:\n            {\n                const int n_expert_used = hparams.n_expert_used;\n                GGML_ASSERT(n_expert_used > 0);\n                ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);\n                ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);\n                op_tensor = ggml_mul_mat_id(ctx, w, b, ids);\n            } break;\n        case GGML_OP_ADD:\n            {\n                ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);\n                op_tensor = ggml_add(ctx, a, w);\n            } break;\n        case GGML_OP_ADD_ID:\n            {\n                const int n_expert_used = hparams.n_expert_used;\n                GGML_ASSERT(n_expert_used > 0);\n                ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);\n                ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);\n                op_tensor = ggml_add_id(ctx, a, w, c);\n            } break;\n        case GGML_OP_MUL:\n            {\n                ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);\n                op_tensor = ggml_mul(ctx, a, w);\n            } break;\n        case GGML_OP_DIV:\n            {\n                ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);\n                op_tensor = ggml_div(ctx, a, w);\n            } break;\n        case GGML_OP_ROPE:\n            {\n                const int n_embd_head = hparams.n_embd_head_v();\n                const int n_head = hparams.n_head();\n                ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);\n                ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);\n                op_tensor = ggml_rope_ext(\n                    ctx, a, b, w,\n                    0, 0, 0, 0, 0,\n                    0, 0, 0, 0\n                );\n\n            } break;\n        case GGML_OP_SSM_CONV:\n            {\n                const int64_t n_seq_tokens = 512;\n                const int64_t n_seqs       = 3;\n                ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0] - 1 + n_seq_tokens, w->ne[1], n_seqs);\n                op_tensor = ggml_ssm_conv(ctx, conv_x, w);\n            } break;\n        case GGML_OP_SSM_SCAN:\n            {\n                // w is ssm_a, which is used to distinguish Mamba-1 and Mamba-2\n                const int64_t d_state      = w->ne[0] == 1 ? hparams.ssm_d_state : w->ne[0];\n                const int64_t n_head       = w->ne[1];\n                const int64_t head_dim     = hparams.ssm_d_inner / n_head;\n                const int64_t n_group      = hparams.ssm_n_group ? hparams.ssm_n_group : 1;\n                const int64_t n_seq_tokens = 512;\n                const int64_t n_seqs       = 3;\n                ggml_tensor * s   = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, head_dim, n_head, n_seqs);\n                ggml_tensor * x   = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_head, n_seq_tokens, n_seqs);\n                ggml_tensor * dt  = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_head, n_seq_tokens, n_seqs);\n                ggml_tensor * B   = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);\n                ggml_tensor * C   = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);\n                ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);\n                op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids);\n            } break;\n        case GGML_OP_RWKV_WKV6:\n            {\n                // FIXME\n                const int64_t S = 123;\n                const int64_t H = 123;\n                const int64_t n_tokens = 123;\n                const int64_t n_seqs = 123;\n                ggml_tensor  * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);\n                ggml_tensor  * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);\n                ggml_tensor  * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);\n                ggml_tensor  * tf = w;\n                ggml_tensor  * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);\n                ggml_tensor  * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);\n                op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);\n            } break;\n        case GGML_OP_IM2COL:\n            {\n                const int n_embd_inp = hparams.n_embd_inp();\n                ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_inp, w->ne[1], 1, 1);\n                op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);\n            } break;\n        case GGML_OP_SCALE:\n            {\n                op_tensor = ggml_scale(ctx, w, 1.0f);\n            } break;\n        default:\n            GGML_ABORT(\"%s: missing test for op %s for tensor %s\", __func__, ggml_op_name(op), w->name);\n    }\n\n    // create a temporary dummy buffer for the weight so that supports_op can check the buffer type\n    GGML_ASSERT(w->buffer == nullptr);\n    w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);\n    bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);\n    ggml_backend_buffer_free(w->buffer);\n    w->buffer = nullptr;\n\n    return op_supported;\n}\n\n// find the first buffer type in the list that can use the tensor\nstatic ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t * buft_list) {\n    GGML_ASSERT(!buft_list->empty());\n    for (const auto & cur : *buft_list) {\n        ggml_backend_dev_t cur_dev = cur.first;\n        ggml_backend_buffer_type_t cur_buft = cur.second;\n        if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {\n            return cur_buft;\n        }\n    }\n\n    return nullptr;\n}\n\nstruct ggml_tensor * llama_model_loader::create_tensor(\n        const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,\n        const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {\n    auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {\n        auto it = ctx_map.find(buft);\n        if (it == ctx_map.end()) {\n            // one ggml context per buffer type\n            int max_n_tensors = n_tensors;\n            max_n_tensors += 1;                 // duplicated output tensor\n            max_n_tensors += hparams.n_layer*2; // duplicated rope freq tensors\n            if (files.empty()) {\n                max_n_tensors += hparams.n_layer*256; // this should be well above what any model actually uses\n            }\n            const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;\n\n            ggml_init_params params = {\n                /*.mem_size   =*/ ctx_size,\n                /*.mem_buffer =*/ NULL,\n                /*.no_alloc   =*/ true,\n            };\n\n            ggml_context * ctx = ggml_init(params);\n            if (!ctx) {\n                throw std::runtime_error(format(\"failed to create ggml context\"));\n            }\n\n            ctx_map.emplace(buft, ctx);\n\n            return ctx;\n        }\n        return it->second.get();\n    };\n\n    auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t {\n        if (!t_meta) {\n            if (flags & TENSOR_NOT_REQUIRED) {\n                return nullptr;\n            }\n            throw std::runtime_error(format(\"missing tensor '%s'\", tn.str().c_str()));\n        }\n\n        // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops\n        // the tensor is duplicated\n        // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor\n        llm_tensor tn_tensor = tn.tensor;\n        if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {\n            tn_tensor = LLM_TENSOR_OUTPUT;\n        }\n\n        llm_tensor_info info;\n        try {\n            info = llm_tensor_info_for(tn_tensor);\n        } catch (const std::out_of_range & e) {\n            throw std::runtime_error(format(\"missing tensor info mapping for %s\", tn.str().c_str()));\n        }\n\n        // skip unused tensors\n        if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) {\n            const size_t nbytes = ggml_nbytes(t_meta);\n            LLAMA_LOG_WARN(\"model has unused tensor %s (size = %zu bytes) -- ignoring\\n\", tn.str().c_str(), nbytes);\n\n            size_data -= nbytes;\n            n_created++;\n\n            return nullptr;\n        }\n\n        // tensors with \"bias\" suffix are always used with GGML_OP_ADD or GGML_OP_ADD_ID\n        ggml_op op;\n        bool bias = tn.suffix != nullptr && strcmp(tn.suffix, \"bias\") == 0;\n        if (bias) {\n            if (info.op == GGML_OP_MUL_MAT_ID) {\n                op = GGML_OP_ADD_ID;\n            } else {\n                op = GGML_OP_ADD;\n            }\n        } else {\n            op = info.op;\n        }\n\n        // sanity checks\n        if (info.layer == LLM_TENSOR_LAYER_INPUT || info.layer == LLM_TENSOR_LAYER_OUTPUT) {\n            if (tn.bid != -1) {\n                GGML_ABORT(\"input/output layer tensor %s used with a layer number\", tn.str().c_str());\n            }\n        } else {\n            if (tn.bid == -1) {\n                GGML_ABORT(\"repeating layer tensor %s used without a layer number\", tn.str().c_str());\n            }\n        }\n\n        // select the buffer type for this tensor\n        const buft_list_t * buft_list;\n        switch (info.layer) {\n            case LLM_TENSOR_LAYER_INPUT:\n                buft_list = buft_list_input;\n                break;\n            case LLM_TENSOR_LAYER_OUTPUT:\n                buft_list = buft_list_output;\n                break;\n            case LLM_TENSOR_LAYER_REPEATING:\n                GGML_ASSERT(buft_list_layer != nullptr);\n                buft_list = buft_list_layer;\n                break;\n            default:\n                GGML_ABORT(\"invalid layer %d for tensor %s\", info.layer, tn.str().c_str());\n        }\n\n        ggml_backend_buffer_type_t buft = nullptr;\n\n        // check overrides\n        if (tensor_buft_overrides) {\n            std::string tensor_name = tn.str();\n            for (const auto * overrides = tensor_buft_overrides; overrides->pattern != nullptr; ++overrides) {\n                std::regex pattern(overrides->pattern);\n                if (std::regex_search(tensor_name, pattern)) {\n                    if (overrides->buft == ggml_backend_cpu_buffer_type()) {\n                        // when overriding to a CPU buffer, consider the extra buffer types\n                        buft = select_weight_buft(hparams, t_meta, op, buft_list_cpu);\n                    } else {\n                        buft = overrides->buft;\n                    }\n\n                    LLAMA_LOG_DEBUG(\"tensor %s (%zu MiB %s) buffer type overridden to %s\\n\",\n                            tensor_name.c_str(),\n                            ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type),\n                            ggml_backend_buft_name(buft));\n                    break;\n                }\n            }\n        }\n\n        if (!buft) {\n            buft = select_weight_buft(hparams, t_meta, op, buft_list);\n            if (!buft) {\n                throw std::runtime_error(format(\"failed to find a compatible buffer type for tensor %s\", tn.str().c_str()));\n            }\n        }\n\n        // avoid using a host buffer when using mmap\n        auto * buft_dev = ggml_backend_buft_get_device(buft);\n        if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) {\n            auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n            if (!cpu_dev) {\n                throw std::runtime_error(\"no CPU backend found\");\n            }\n            buft = ggml_backend_dev_buffer_type(cpu_dev);\n        }\n\n        if (buft != buft_list->front().second) {\n            if (n_tensors_moved == 0) {\n                first_tensor_moved_name = t_meta->name;\n                first_tensor_moved_type_name = ggml_type_name(t_meta->type);\n                first_moved_from_buft = buft_list->front().second;\n                first_moved_to_buft   = buft;\n            }\n            n_tensors_moved++;\n        }\n\n        return buft;\n    };\n\n    if (files.empty()) {\n        if (flags & TENSOR_SKIP_IF_VIRTUAL) {\n            return nullptr;\n        }\n        ggml_type type = GGML_TYPE_F32;\n        const int64_t tid = gguf_find_tensor(metadata, tn.str().c_str());\n        if (tid != -1) {\n            type = gguf_get_tensor_type(metadata, tid);\n        }\n\n        // for tensors that are not required some of the dimensions can be invalid:\n        if (flags & TENSOR_NOT_REQUIRED) {\n            for (size_t dim = 0; dim < ne.size(); dim++) {\n                if (ne.begin()[dim] <= 0) {\n                    return nullptr;\n                }\n            }\n        }\n\n        ggml_tensor t_meta;\n        memset(&t_meta, 0, sizeof(ggml_tensor));\n        t_meta.type = type;\n        for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {\n            t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;\n            GGML_ASSERT(t_meta.ne[dim] >= 1);\n            t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];\n            GGML_ASSERT(t_meta.nb[dim] >= 1);\n        }\n        ggml_set_name(&t_meta, tn.str().c_str());\n\n        ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);\n        GGML_ASSERT(buft != nullptr);\n        ggml_context * ctx = ctx_for_buft(buft);\n        ggml_tensor * ret = ggml_dup_tensor(ctx, &t_meta);\n        ggml_set_name(ret, tn.str().c_str());\n        return ret;\n    }\n\n    ggml_tensor * t_meta = get_tensor_meta(tn.str().c_str());\n    ggml_backend_buffer_type_t buft = buft_for_tensor(t_meta);\n    if (buft == nullptr) {\n        return nullptr; // return type is ggml_tensor *\n    }\n    ggml_context * ctx = ctx_for_buft(buft);\n\n    // if duplicated, check if the original tensor was allocated in the same buffer type context and avoid creating a new one\n    if (flags & TENSOR_DUPLICATED) {\n        ggml_tensor * t = ggml_get_tensor(ctx, tn.str().c_str());\n        if (t) {\n            return t;\n        }\n    }\n\n    LLAMA_LOG_DEBUG(\"%s: loading tensor %s\\n\", __func__, tn.str().c_str());\n    const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED));\n\n    if (cur == NULL) {\n        return NULL;\n    }\n\n    const bool duplicated = flags & TENSOR_DUPLICATED;\n\n    struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);\n    ggml_set_name(tensor, ggml_get_name(cur));\n\n    if (duplicated) {\n        size_data += ggml_nbytes(cur);\n    } else {\n        n_created++;\n    }\n\n    return tensor;\n}\n\nstruct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required) {\n    const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);\n\n    if (cur == NULL) {\n        return NULL;\n    }\n\n    if (cur->type != base->type) {\n        throw std::runtime_error(format(\"%s: tensor '%s' has wrong type; expected %s, got %s\", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));\n    }\n\n    std::array<int64_t, GGML_MAX_DIMS> dims;\n    for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {\n        dims[i] = i < ne.size() ? ne.begin()[i] : 1;\n    }\n\n    struct ggml_tensor * tensor = ggml_view_4d(ctx, base,\n                                    dims[0], dims[1], dims[2], dims[3],\n                                    cur->nb[1], cur->nb[2], cur->nb[3],\n                                    offset);\n\n    ggml_set_name(tensor, name.c_str());\n\n    n_created++;\n\n    return tensor;\n}\n\nvoid llama_model_loader::done_getting_tensors() const {\n    if (n_created != n_tensors) {\n        throw std::runtime_error(format(\"%s: wrong number of tensors; expected %d, got %d\", __func__, n_tensors, n_created));\n    }\n    if (n_tensors_moved > 0) {\n        LLAMA_LOG_DEBUG(\"%s: tensor '%s' (%s) (and %zu others) cannot be used with preferred buffer type %s, using %s instead\\n\",\n            __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name.c_str(), n_tensors_moved - 1,\n            ggml_backend_buft_name(first_moved_from_buft), ggml_backend_buft_name(first_moved_to_buft));\n    }\n}\n\nvoid llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps) {\n    if (use_mmap) {\n        mappings.reserve(files.size());\n        mmaps_used.reserve(files.size());\n        for (const auto & file : files) {\n            bool is_numa = false;\n\n            auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n            if (dev) {\n                auto * reg = ggml_backend_dev_backend_reg(dev);\n                auto * is_numa_fn = (decltype(ggml_is_numa) *) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_cpu_is_numa\");\n                if (is_numa_fn) {\n                    is_numa = is_numa_fn();\n                }\n            }\n\n            std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);\n            mmaps_used.emplace_back(mapping->size(), 0);\n            if (mlock_mmaps) {\n                std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());\n                mlock_mmap->init(mapping->addr());\n                mlock_mmaps->emplace_back(std::move(mlock_mmap));\n            }\n            mappings.emplace_back(std::move(mapping));\n        }\n    }\n\n    // compute the total size of all tensors for progress reporting\n    for (const auto & it : weights_map) {\n        size_data += ggml_nbytes(it.second.tensor);\n    }\n}\n\nvoid llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {\n    GGML_ASSERT(!mappings.empty());\n    const auto & mapping = mappings.at(idx);\n\n    *first = mapping->size();\n    *last  = 0;\n    *addr = mapping->addr();\n    for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {\n        const auto * weight = get_weight(ggml_get_name(tensor));\n        if (!weight || weight->idx != idx) {\n            continue;\n        }\n        *first = std::min(*first, weight->offs);\n        *last  = std::max(*last,  weight->offs + ggml_nbytes(tensor));\n    }\n}\n\nvoid llama_model_loader::load_data_for(struct ggml_tensor * cur) const {\n    const auto & w = require_weight(ggml_get_name(cur));\n\n    if (use_mmap) {\n        const auto & mapping = mappings.at(w.idx);\n        if (cur->data == nullptr) {\n            cur->data = (uint8_t *)mapping->addr() + w.offs;\n        } else {\n            memcpy(cur->data, (uint8_t *)mapping->addr() + w.offs, ggml_nbytes(cur));\n        }\n    } else {\n        GGML_ASSERT(cur->data != nullptr);\n        GGML_ASSERT(w.idx < files.size());\n        const auto & file = files.at(w.idx);\n        file->seek(w.offs, SEEK_SET);\n        file->read_raw(cur->data, ggml_nbytes(cur));\n    }\n\n    if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {\n        throw std::runtime_error(format(\"tensor '%s' has invalid data\", ggml_get_name(cur)));\n    }\n}\n\nbool llama_model_loader::load_all_data(\n        struct ggml_context * ctx,\n        llama_buf_map & bufs,\n        llama_mlocks * lmlocks,\n        llama_progress_callback progress_callback,\n        void * progress_callback_user_data) {\n    if (files.empty()) {\n        for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {\n            set_tensor_data(t, set_tensor_data_ud);\n        }\n        return true;\n    }\n    GGML_ASSERT(size_data != 0 && \"call init_mappings() first\");\n\n    std::vector<no_init<uint8_t>> read_buf;\n    std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;\n\n    // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.\n    // NVMe raid configurations might require more / larger buffers.\n    constexpr size_t n_buffers = 4;\n\n    size_t alignment = 1;\n    for (const auto & file : files) {\n        alignment = std::max(file->read_alignment(), alignment);\n    }\n\n    // Buffer size: balance between memory usage and I/O efficiency\n    // 64MB works well for NVMe drives\n    const size_t buffer_size = alignment != 1 ? 64 * 1024 * 1024 + 2 * alignment : 1 * 1024 * 1024;\n\n    std::vector<ggml_backend_buffer_t> host_buffers;\n    std::vector<ggml_backend_event_t> events;\n    std::vector<void *> host_ptrs;\n    size_t buffer_idx = 0; // buffer to use for async loads\n    ggml_backend_t upload_backend = [&](const char * func) -> ggml_backend_t {\n        if (use_mmap || check_tensors) {\n            return nullptr;\n        }\n        // When not using mmaped io use async uploads from pinned memory to GPU memory.\n        // First determine if the backend supports the necessary features for async uploads.\n        auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;\n        if (!buf) {\n            LLAMA_LOG_DEBUG(\"%s: no buffer found for async uploads\\n\", func);\n            return nullptr;\n        }\n\n        auto * buft = ggml_backend_buffer_get_type(buf);\n        auto * dev = ggml_backend_buft_get_device(buft);\n        if (!dev) {\n            LLAMA_LOG_DEBUG(\"%s: no device found for buffer type %s for async uploads\\n\", func,\n                ggml_backend_buft_name(buft));\n            return nullptr;\n        }\n\n        if (buft != ggml_backend_dev_buffer_type(dev)) {\n            LLAMA_LOG_DEBUG(\"%s: buffer type %s is not the default buffer type for device %s for async uploads\\n\", func,\n                ggml_backend_buft_name(buft), ggml_backend_dev_name(dev));\n            return nullptr;\n        }\n\n        ggml_backend_dev_props props;\n        ggml_backend_dev_get_props(dev, &props);\n        if (!props.caps.async || !props.caps.host_buffer || !props.caps.events) {\n            LLAMA_LOG_DEBUG(\"%s: device %s does not support async, host buffers or events\\n\", func,\n                ggml_backend_dev_name(dev));\n            return nullptr;\n        }\n\n        auto * host_buft = ggml_backend_dev_host_buffer_type(dev);\n        if (!host_buft) {\n            LLAMA_LOG_DEBUG(\"%s: no host buffer type found for device %s\\n\", func,\n                ggml_backend_dev_name(dev));\n            return nullptr;\n        }\n\n        // If the backend is supported, create pinned memory buffers and events for synchronisation.\n        for (size_t idx = 0; idx < n_buffers; ++idx) {\n            auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size);\n\n            if (!buf) {\n                LLAMA_LOG_DEBUG(\"%s: failed to allocate host buffer for async uploads for device %s\\n\", func,\n                    ggml_backend_dev_name(dev));\n                return nullptr;\n            }\n\n            host_buffers.emplace_back(buf);\n            host_ptrs.emplace_back(ggml_backend_buffer_get_base(buf));\n\n            auto * event = ggml_backend_event_new(dev);\n            if (!event) {\n                LLAMA_LOG_DEBUG(\"%s: failed to create event for async uploads for device %s\\n\", func,\n                    ggml_backend_dev_name(dev));\n                return nullptr;\n            }\n\n            events.emplace_back(event);\n        }\n\n        ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);\n        if (!backend) {\n            LLAMA_LOG_DEBUG(\"%s: failed to initialize backend for device %s for async uploads\\n\", func,\n                ggml_backend_dev_name(dev));\n            return nullptr;\n        }\n\n        return backend;\n    }(__func__);\n\n    if (upload_backend) {\n        LLAMA_LOG_DEBUG(\"%s: using async uploads for device %s, buffer type %s, backend %s\\n\", __func__,\n            ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),\n            ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),\n            ggml_backend_name(upload_backend));\n    }\n\n    for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {\n        const auto * weight = get_weight(ggml_get_name(cur));\n        if (weight == nullptr) {\n            // this can happen with split experts models\n            continue;\n        }\n\n        if (progress_callback) {\n            if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {\n                return false;\n            }\n        }\n\n        size_t n_size = ggml_nbytes(cur);\n\n        if (use_mmap) {\n            const auto & mapping = mappings.at(weight->idx);\n            ggml_backend_buffer_t buf_mmap = nullptr;\n            if (bufs.count(weight->idx)) {\n                buf_mmap = bufs.at(weight->idx);\n            }\n            uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;\n\n            if (check_tensors) {\n                validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {\n                    return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));\n                }));\n            }\n\n            GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated\n            if (buf_mmap && cur->data == nullptr) {\n                ggml_backend_tensor_alloc(buf_mmap, cur, data);\n                if (lmlocks) {\n                    const auto & lmlock = lmlocks->at(weight->idx);\n                    lmlock->grow_to(weight->offs + n_size);\n                }\n\n                auto & mmap_used = mmaps_used[weight->idx];\n                mmap_used.first  = std::min(mmap_used.first,  weight->offs);\n                mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);\n            } else {\n                ggml_backend_tensor_set(cur, data, 0, n_size);\n            }\n        } else {\n            const auto & file = files.at(weight->idx);\n\n            if (ggml_backend_buffer_is_host(cur->buffer)) {\n                file->seek(weight->offs, SEEK_SET);\n                file->read_raw(cur->data, n_size);\n                if (check_tensors) {\n                    validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {\n                        return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));\n                    }));\n                }\n            } else {\n                // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.\n                if (upload_backend) {\n                    size_t offset = weight->offs;\n                    alignment = file->read_alignment();\n                    size_t aligned_offset = offset & ~(alignment - 1);\n                    size_t offset_from_alignment = offset - aligned_offset;\n                    file->seek(aligned_offset, SEEK_SET);\n\n                    // Calculate aligned read boundaries\n                    size_t read_start = aligned_offset;\n                    size_t read_end = (offset + n_size + alignment - 1) & ~(alignment - 1);\n\n                    size_t bytes_read = 0;\n                    size_t data_read = 0;  // Actual tensor data copied (excluding padding)\n\n                    while (bytes_read < read_end - read_start) {\n                        size_t read_size = std::min<size_t>(buffer_size, read_end - read_start - bytes_read);\n\n                        // Align the destination pointer within the pinned buffer\n                        uintptr_t ptr_dest_aligned = (reinterpret_cast<uintptr_t>(host_ptrs[buffer_idx]) + alignment - 1) & ~(alignment - 1);\n\n                        // Wait for previous upload to complete before reusing buffer\n                        ggml_backend_event_synchronize(events[buffer_idx]);\n\n                        // Read aligned chunk from file\n                        file->read_raw_unsafe(reinterpret_cast<void *>(ptr_dest_aligned), read_size);\n\n                        // Calculate actual data portion (excluding alignment padding)\n                        uintptr_t ptr_data = ptr_dest_aligned;\n                        size_t data_to_copy = read_size;\n\n                        // Skip alignment padding at start of first chunk\n                        if (bytes_read == 0) {\n                            ptr_data += offset_from_alignment;\n                            data_to_copy -= offset_from_alignment;\n                        }\n\n                        // Trim alignment padding at end of last chunk\n                        if (aligned_offset + bytes_read + read_size > offset + n_size) {\n                            data_to_copy -= (read_end - (offset + n_size));\n                        }\n\n                        // Async upload actual data to GPU\n                        ggml_backend_tensor_set_async(upload_backend, cur,\n                                                      reinterpret_cast<void *>(ptr_data), data_read, data_to_copy);\n                        ggml_backend_event_record(events[buffer_idx], upload_backend);\n\n                        data_read += data_to_copy;\n                        bytes_read += read_size;\n\n                        ++buffer_idx;\n                        buffer_idx %= n_buffers;\n                    }\n                } else {\n                    read_buf.resize(n_size);\n                    file->seek(weight->offs, SEEK_SET);\n                    file->read_raw(read_buf.data(), n_size);\n                    ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);\n                    if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {\n                        throw std::runtime_error(format(\"tensor '%s' has invalid data\", ggml_get_name(cur)));\n                    }\n                }\n            }\n        }\n\n        size_done += n_size;\n    }\n\n    // free temporary resources used for async uploads\n    for (auto * event : events) {\n        ggml_backend_event_synchronize(event);\n        ggml_backend_event_free(event);\n    }\n    for (auto * buf : host_buffers) {\n        ggml_backend_buffer_free(buf);\n    }\n    ggml_backend_free(upload_backend);\n\n    // check validation results\n    bool validation_failed = false;\n    for (auto & future : validation_result) {\n        auto result = future.get();\n        if (!result.second) {\n            LLAMA_LOG_ERROR(\"%s: tensor '%s' has invalid data\\n\", __func__, ggml_get_name(result.first));\n            validation_failed = true;\n        }\n    }\n    if (validation_failed) {\n        throw std::runtime_error(\"found tensors with invalid data\");\n    }\n\n    // check if this is the last call and do final cleanup\n    if (size_done >= size_data) {\n        // unmap offloaded tensors and metadata\n        if (use_mmap) {\n            for (uint32_t idx = 0; idx < mappings.size(); idx++) {\n                const auto & mmap_used = mmaps_used.at(idx);\n                auto & mapping = mappings.at(idx);\n                mapping->unmap_fragment(0, mmap_used.first);\n                if (mmap_used.second != 0) {\n                    mapping->unmap_fragment(mmap_used.second, mapping->size());\n                }\n            }\n        }\n        if (progress_callback) {\n            // Even though the model is done loading, we still honor\n            // cancellation since we need to free allocations.\n            return progress_callback(1.0f, progress_callback_user_data);\n        }\n    }\n\n    return true;\n}\n\nstd::string llama_model_loader::ftype_name() const {\n    return llama_model_ftype_name(ftype);\n}\n\nvoid llama_model_loader::print_info() const {\n    LLAMA_LOG_INFO(\"%s: file format = %s\\n\", __func__, llama_file_version_name(fver));\n    LLAMA_LOG_INFO(\"%s: file type   = %s\\n\", __func__, llama_model_ftype_name(ftype).c_str());\n    if (n_bytes < GiB) {\n        LLAMA_LOG_INFO(\"%s: file size   = %.2f MiB (%.2f BPW) \\n\", __func__, n_bytes/1024.0/1024.0,        n_bytes*8.0/n_elements);\n    } else {\n        LLAMA_LOG_INFO(\"%s: file size   = %.2f GiB (%.2f BPW) \\n\", __func__, n_bytes/1024.0/1024.0/1024.0, n_bytes*8.0/n_elements);\n    }\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-model-loader.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-arch.h\"\n#include \"llama-hparams.h\"\n#include \"llama-mmap.h\"\n\n#include \"ggml-cpp.h\"\n\n#include <cstddef>\n#include <cstring>\n#include <map>\n#include <stdexcept>\n#include <unordered_map>\n\nusing llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;\n\n// lists of buffer types used for each layer\nusing buft_list_t = std::vector<std::pair<ggml_backend_dev_t, ggml_backend_buffer_type_t>>;\n\nenum llama_fver {\n    GGUF_FILE_VERSION_V1 = 1,\n    GGUF_FILE_VERSION_V2 = 2,\n    GGUF_FILE_VERSION_V3 = 3,\n};\n\nconst char * llama_file_version_name(llama_fver version);\n\nstruct llama_model_loader {\n    // Holds information on a model weight\n    struct llama_tensor_weight {\n        uint16_t  idx; // source file index\n        size_t   offs; // tensor data offset in the original file\n\n        ggml_tensor * tensor;\n\n        llama_tensor_weight(const llama_file * file, uint16_t idx, const struct gguf_context * gguf_ctx, ggml_tensor * tensor) : idx(idx), tensor(tensor) {\n            const int tensor_idx = gguf_find_tensor(gguf_ctx,  ggml_get_name(tensor));\n            if (tensor_idx < 0) {\n                throw std::runtime_error(format(\"tensor '%s' not found in the model\", ggml_get_name(tensor)));\n            }\n\n            offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx);\n            if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > file->size()) {\n                throw std::runtime_error(format(\"tensor '%s' data is not within the file bounds, model is corrupted or incomplete\", ggml_get_name(tensor)));\n            }\n        }\n    };\n\n    // custom comparator to sort weights more nicely by layer\n    struct weight_name_comparer {\n        bool operator()(const std::string & a, const std::string & b) const {\n            int a_layer = -1;\n            int b_layer = -1;\n            sscanf(a.c_str(), \"blk.%d.\", &a_layer);\n            sscanf(b.c_str(), \"blk.%d.\", &b_layer);\n            if (a_layer != b_layer) {\n                return a_layer < b_layer;\n            }\n            return a < b;\n        }\n    };\n\n    static const int TENSOR_NOT_REQUIRED    = 1 << 0;\n    static const int TENSOR_DUPLICATED      = 1 << 1;\n    static const int TENSOR_SKIP            = 1 << 2;\n    static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3;\n\n    int n_kv      = 0;\n    int n_tensors = 0;\n    int n_created = 0;\n\n    uint64_t n_elements = 0;\n    size_t   n_bytes    = 0;\n\n    bool use_mmap = false;\n    bool use_direct_io = false;\n    bool check_tensors;\n    bool no_alloc;\n\n    llama_files files;\n    llama_ftype ftype;\n    llama_fver  fver;\n\n    llama_mmaps mappings;\n\n    std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;\n    std::unordered_map<std::string, llama_model_kv_override> kv_overrides;\n    const llama_model_tensor_buft_override * tensor_buft_overrides;\n\n    gguf_context_ptr metadata_ptr;\n    struct gguf_context * metadata; // either metadata_ptr.get() or externally set\n    llama_model_set_tensor_data_t set_tensor_data;\n    void * set_tensor_data_ud;\n    std::vector<ggml_context_ptr> contexts;\n\n    std::string arch_name;\n    LLM_KV      llm_kv    = LLM_KV(LLM_ARCH_UNKNOWN);\n\n    size_t size_done = 0;\n    size_t size_data = 0;\n    std::vector<std::pair<size_t, size_t>> mmaps_used;\n\n    // define a comparator for the buft -> ctx map to ensure that the order is well-defined:\n    struct ggml_backend_buft_comparator {\n        bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {\n            return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;\n        }\n    };\n\n    std::map<ggml_backend_buffer_type_t, ggml_context_ptr, ggml_backend_buft_comparator> ctx_map;\n\n    // track tensors that had to be moved for debugging:\n    size_t n_tensors_moved = 0;\n    std::string first_tensor_moved_name;\n    std::string first_tensor_moved_type_name;\n    ggml_backend_buffer_type_t first_moved_from_buft = nullptr;\n    ggml_backend_buffer_type_t first_moved_to_buft = nullptr;\n\n    llama_model_loader(\n        struct gguf_context * metadata,\n        llama_model_set_tensor_data_t set_tensor_data,\n        void * set_tensor_data_ud,\n        const std::string & fname,\n        std::vector<std::string> & splits, // optional, only need if the split does not follow naming scheme\n        bool use_mmap,\n        bool use_direct_io,\n        bool check_tensors,\n        bool no_alloc,\n        const llama_model_kv_override * param_overrides_p,\n        const llama_model_tensor_buft_override * param_tensor_buft_overrides_p);\n\n    template<typename T>\n    typename std::enable_if<std::is_integral<T>::value, bool>::type\n    get_arr_n(const std::string & key, T & result, bool required = true);\n\n    template<typename T>\n    typename std::enable_if<std::is_integral<T>::value, bool>::type\n    get_arr_n(enum llm_kv kid, T & result, bool required = true);\n\n    template<typename T>\n    bool get_arr(const std::string & key, std::vector<T> & result, bool required = true);\n\n    template<typename T, size_t N_MAX>\n    bool get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required = true);\n\n    template<typename T>\n    bool get_arr(enum llm_kv kid, T & result, bool required = true);\n\n    template<typename T>\n    bool get_key(const std::string & key, T & result, bool required = true);\n\n    template<typename T>\n    bool get_key(enum llm_kv kid, T & result, bool required = true);\n\n    template<typename T, size_t N_MAX>\n    bool get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required = true);\n\n    template<typename T>\n    bool get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required = true);\n\n    bool get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required = true);\n\n    std::string get_arch_name() const;\n\n    enum llm_arch get_arch() const;\n\n    const llama_tensor_weight * get_weight(const char * name) const;\n\n    const llama_tensor_weight & require_weight(const char * name) const;\n\n    struct ggml_tensor * get_tensor_meta(const char * name) const;\n\n    struct ggml_tensor * require_tensor_meta(const std::string & name) const;\n\n    const struct ggml_tensor * check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const;\n\n    struct ggml_tensor * create_tensor(\n        const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,\n        const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags);\n\n    struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required = true);\n\n    void done_getting_tensors() const;\n\n    void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr);\n\n    void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;\n\n    // for backwards compatibility, does not support ggml-backend\n    void load_data_for(struct ggml_tensor * cur) const;\n\n    // Returns false if cancelled by progress_callback\n    bool load_all_data(\n            struct ggml_context * ctx,\n            llama_buf_map & bufs,\n            llama_mlocks * lmlocks,\n            llama_progress_callback progress_callback,\n            void * progress_callback_user_data);\n\n    std::string ftype_name() const;\n\n    void print_info() const;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-model-saver.cpp",
    "content": "#include \"llama-model-saver.h\"\n\n#include \"gguf.h\"\n\n#include \"llama.h\"\n#include \"llama-hparams.h\"\n#include \"llama-model.h\"\n#include \"llama-vocab.h\"\n\n#include <cstdint>\n#include <string>\n\nllama_model_saver::llama_model_saver(const struct llama_model * model) :\n    gguf_ctx(gguf_init_empty()), gguf_ctx_owned(true), model(model), llm_kv(model->arch) {}\n\nllama_model_saver::llama_model_saver(enum llm_arch arch, struct gguf_context * gguf_ctx) :\n        gguf_ctx(gguf_ctx == nullptr ? gguf_init_empty() : gguf_ctx), gguf_ctx_owned(gguf_ctx == nullptr), model(nullptr), llm_kv(arch) {}\n\nllama_model_saver::~llama_model_saver() {\n    if (gguf_ctx_owned) {\n        gguf_free(gguf_ctx);\n    }\n}\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const uint32_t value) {\n    gguf_set_val_u32(gguf_ctx, llm_kv(key).c_str(), value);\n}\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const int32_t value) {\n    gguf_set_val_i32(gguf_ctx, llm_kv(key).c_str(), value);\n}\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const float value) {\n    gguf_set_val_f32(gguf_ctx, llm_kv(key).c_str(), value);\n}\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const bool value) {\n    gguf_set_val_bool(gguf_ctx, llm_kv(key).c_str(), value);\n}\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const char * value) {\n    gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), value);\n}\n\n[[noreturn]]\nvoid llama_model_saver::add_kv(const enum llm_kv key, const char value) {\n    GGML_UNUSED(key);\n    GGML_UNUSED(value);\n    GGML_ABORT(\"fatal error\"); // this should never be called, only needed to make the template below compile\n}\n\ntemplate <typename Container>\nvoid llama_model_saver::add_kv(const enum llm_kv key, const Container & value, const bool per_layer) {\n    GGML_ASSERT(model != nullptr || !per_layer);\n    const size_t n_values = per_layer ? size_t(model->hparams.n_layer) : value.size();\n    GGML_ASSERT(n_values <= value.size());\n\n    if (n_values == 0) {\n        return;\n    }\n\n    if (per_layer) {\n        bool all_values_the_same = true;\n        for (size_t i = 1; i < n_values; ++i) {\n            if (value[i] != value[0]) {\n                all_values_the_same = false;\n                break;\n            }\n        }\n        if (all_values_the_same) {\n            add_kv(key, value[0]);\n            return;\n        }\n    }\n\n    if (std::is_same<typename Container::value_type, uint8_t>::value) {\n        gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT8, value.data(), n_values);\n    } else if (std::is_same<typename Container::value_type, int8_t>::value) {\n        gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT8, value.data(), n_values);\n    } else if (std::is_same<typename Container::value_type, uint32_t>::value) {\n        gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT32, value.data(), n_values);\n    } else if (std::is_same<typename Container::value_type, int32_t>::value) {\n        gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT32, value.data(), n_values);\n    } else if (std::is_same<typename Container::value_type, float>::value) {\n        gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_FLOAT32, value.data(), n_values);\n    } else if (std::is_same<Container, std::string>::value) {\n        gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), reinterpret_cast<const char *>(value.data()));\n    } else {\n        GGML_ABORT(\"fatal error\");\n    }\n}\n// instantiate for external usage:\ntemplate void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);\n\nvoid llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {\n    std::vector<const char *> tmp(value.size());\n    for (size_t i = 0; i < value.size(); ++i) {\n        tmp[i] = value[i].c_str();\n    }\n    gguf_set_arr_str(gguf_ctx, llm_kv(key).c_str(), tmp.data(), tmp.size());\n}\n\nvoid llama_model_saver::add_tensor(const struct ggml_tensor * tensor) {\n    if (!tensor) {\n        return;\n    }\n    if (gguf_find_tensor(gguf_ctx, tensor->name) >= 0) {\n        GGML_ASSERT(std::string(tensor->name) == \"rope_freqs.weight\"); // FIXME\n        return;\n    }\n    gguf_add_tensor(gguf_ctx, tensor);\n}\n\nvoid llama_model_saver::add_kv_from_model() {\n    const llama_hparams & hparams = model->hparams;\n    const llama_vocab   & vocab   = model->vocab;\n\n    const int32_t n_vocab = vocab.n_tokens();\n    std::vector<std::string> tokens(n_vocab);\n    std::vector<float>       scores(n_vocab);\n    std::vector<int32_t>     token_types(n_vocab);\n\n    if (vocab.get_type() != LLAMA_VOCAB_TYPE_NONE) {\n        for (int32_t id = 0; id < n_vocab; ++id) {\n            const llama_vocab::token_data & token_data = vocab.get_token_data(id);\n\n            tokens[id] = token_data.text;\n            scores[id] = token_data.score;\n\n            switch(token_data.attr) {\n                case LLAMA_TOKEN_ATTR_UNKNOWN:      token_types[id] = LLAMA_TOKEN_TYPE_UNKNOWN;      break;\n                case LLAMA_TOKEN_ATTR_UNUSED:       token_types[id] = LLAMA_TOKEN_TYPE_UNUSED;       break;\n                case LLAMA_TOKEN_ATTR_NORMAL:       token_types[id] = LLAMA_TOKEN_TYPE_NORMAL;       break;\n                case LLAMA_TOKEN_ATTR_CONTROL:      token_types[id] = LLAMA_TOKEN_TYPE_CONTROL;      break;\n                case LLAMA_TOKEN_ATTR_USER_DEFINED: token_types[id] = LLAMA_TOKEN_TYPE_USER_DEFINED; break;\n                case LLAMA_TOKEN_ATTR_BYTE:         token_types[id] = LLAMA_TOKEN_TYPE_BYTE;         break;\n                case LLAMA_TOKEN_ATTR_UNDEFINED:\n                default:                            token_types[id] = LLAMA_TOKEN_TYPE_UNDEFINED;    break;\n            }\n        }\n    }\n\n    // add_kv(LLM_KV_GENERAL_TYPE,                      ???);\n    add_kv(LLM_KV_GENERAL_ARCHITECTURE,              model->arch_name());\n    // add_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION,      ???);\n    // add_kv(LLM_KV_GENERAL_ALIGNMENT,                 ???);\n    add_kv(LLM_KV_GENERAL_NAME,                      model->name);\n    // add_kv(LLM_KV_GENERAL_AUTHOR,                    ???);\n    // add_kv(LLM_KV_GENERAL_VERSION,                   ???);\n    // add_kv(LLM_KV_GENERAL_URL,                       ???);\n    // add_kv(LLM_KV_GENERAL_DESCRIPTION,               ???);\n    // add_kv(LLM_KV_GENERAL_LICENSE,                   ???);\n    // add_kv(LLM_KV_GENERAL_SOURCE_URL,                ???);\n    // add_kv(LLM_KV_GENERAL_SOURCE_HF_REPO,            ???);\n\n    add_kv(LLM_KV_VOCAB_SIZE,                        vocab.n_tokens());\n    add_kv(LLM_KV_CONTEXT_LENGTH,                    hparams.n_ctx_train);\n    add_kv(LLM_KV_EMBEDDING_LENGTH,                  hparams.n_embd);\n    if (hparams.n_embd_out_impl > 0) {\n        add_kv(LLM_KV_EMBEDDING_LENGTH_OUT,          hparams.n_embd_out_impl);\n    }\n    add_kv(LLM_KV_BLOCK_COUNT,                       hparams.n_layer);\n    add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead);\n    add_kv(LLM_KV_FEED_FORWARD_LENGTH,               hparams.n_ff_arr, true);\n    add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n    add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_exp);\n    add_kv(LLM_KV_USE_PARALLEL_RESIDUAL,             hparams.use_par_res);\n    // add_kv(LLM_KV_TENSOR_DATA_LAYOUT,                ???);\n    add_kv(LLM_KV_EXPERT_COUNT,                      hparams.n_expert);\n    add_kv(LLM_KV_EXPERT_USED_COUNT,                 hparams.n_expert_used);\n    add_kv(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared);\n    add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale);\n    add_kv(LLM_KV_POOLING_TYPE,                      uint32_t(hparams.pooling_type));\n    add_kv(LLM_KV_LOGIT_SCALE,                       hparams.f_logit_scale);\n    add_kv(LLM_KV_DECODER_START_TOKEN_ID,            hparams.dec_start_token_id);\n    add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING,            hparams.f_attn_logit_softcapping);\n    add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING,           hparams.f_final_logit_softcapping);\n    add_kv(LLM_KV_SWIN_NORM,                         hparams.swin_norm);\n    add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS,            hparams.rescale_every_n_layers);\n    add_kv(LLM_KV_TIME_MIX_EXTRA_DIM,                hparams.time_mix_extra_dim);\n    add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM,              hparams.time_decay_extra_dim);\n    add_kv(LLM_KV_RESIDUAL_SCALE,                    hparams.f_residual_scale);\n    add_kv(LLM_KV_EMBEDDING_SCALE,                   hparams.f_embedding_scale);\n\n    add_kv(LLM_KV_ATTENTION_HEAD_COUNT,              hparams.n_head_arr, true);\n    add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV,           hparams.n_head_kv_arr, true);\n    add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS,          hparams.f_max_alibi_bias);\n    add_kv(LLM_KV_ATTENTION_CLAMP_KQV,               hparams.f_clamp_kqv);\n    add_kv(LLM_KV_ATTENTION_KEY_LENGTH,              hparams.n_embd_head_k_full);\n    add_kv(LLM_KV_ATTENTION_VALUE_LENGTH,            hparams.n_embd_head_v_full);\n    add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA,          hparams.n_embd_head_k_swa);\n    add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA,        hparams.n_embd_head_v_swa);\n    add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS,           hparams.f_norm_eps);\n    add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n    add_kv(LLM_KV_ATTENTION_CAUSAL,                  hparams.causal_attn);\n    add_kv(LLM_KV_ATTENTION_Q_LORA_RANK,             hparams.n_lora_q);\n    add_kv(LLM_KV_ATTENTION_KV_LORA_RANK,            hparams.n_lora_kv);\n    add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,  hparams.n_rel_attn_bkts);\n    add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW,          hparams.n_swa);\n    add_kv(LLM_KV_ATTENTION_SCALE,                   hparams.f_attention_scale);\n\n    const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train;\n\n    add_kv(LLM_KV_ROPE_DIMENSION_COUNT,              hparams.n_rot_full);\n    add_kv(LLM_KV_ROPE_DIMENSION_COUNT_SWA,          hparams.n_rot_swa);\n    add_kv(LLM_KV_ROPE_FREQ_BASE,                    hparams.rope_freq_base_train);\n    // add_kv(LLM_KV_ROPE_SCALE_LINEAR,                 rope_scaling_factor); // old name\n    add_kv(LLM_KV_ROPE_SCALING_TYPE,                 llama_rope_scaling_type_name(hparams.rope_scaling_type_train));\n    add_kv(LLM_KV_ROPE_SCALING_FACTOR,               rope_scaling_factor);\n    add_kv(LLM_KV_ROPE_SCALING_ATTN_FACTOR,          hparams.rope_attn_factor);\n    add_kv(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,         hparams.n_ctx_orig_yarn);\n    add_kv(LLM_KV_ROPE_SCALING_FINETUNED,            hparams.rope_finetuned);\n    add_kv(LLM_KV_ROPE_SCALING_YARN_LOG_MUL,         hparams.rope_yarn_log_mul);\n\n    // TODO: implement split file support\n    // add_kv(LLM_KV_SPLIT_NO,                          ???);\n    // add_kv(LLM_KV_SPLIT_COUNT,                       ???);\n    // add_kv(LLM_KV_SPLIT_TENSORS_COUNT,               ???);\n\n    add_kv(LLM_KV_SSM_INNER_SIZE,                    hparams.ssm_d_inner);\n    add_kv(LLM_KV_SSM_CONV_KERNEL,                   hparams.ssm_d_conv);\n    add_kv(LLM_KV_SSM_STATE_SIZE,                    hparams.ssm_d_state);\n    add_kv(LLM_KV_SSM_TIME_STEP_RANK,                hparams.ssm_dt_rank);\n    add_kv(LLM_KV_SSM_DT_B_C_RMS,                    hparams.ssm_dt_b_c_rms);\n\n    add_kv(LLM_KV_WKV_HEAD_SIZE,                     hparams.wkv_head_size);\n\n    add_kv(LLM_KV_TOKENIZER_MODEL,                   vocab.get_tokenizer_model());\n    add_kv(LLM_KV_TOKENIZER_PRE,                     vocab.get_tokenizer_pre());\n    add_kv(LLM_KV_TOKENIZER_LIST,                    tokens);\n    add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE,              token_types);\n    add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,        vocab.n_token_types());\n    add_kv(LLM_KV_TOKENIZER_SCORES,                  scores);\n    add_kv(LLM_KV_TOKENIZER_MERGES,                  vocab.get_bpe_merges());\n    // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though\n    add_kv(LLM_KV_TOKENIZER_BOS_ID,                  uint32_t(vocab.token_bos()));\n    add_kv(LLM_KV_TOKENIZER_EOS_ID,                  uint32_t(vocab.token_eos()));\n    add_kv(LLM_KV_TOKENIZER_EOT_ID,                  uint32_t(vocab.token_eot()));\n    add_kv(LLM_KV_TOKENIZER_EOM_ID,                  uint32_t(vocab.token_eom()));\n    add_kv(LLM_KV_TOKENIZER_UNK_ID,                  uint32_t(vocab.token_unk()));\n    add_kv(LLM_KV_TOKENIZER_SEP_ID,                  uint32_t(vocab.token_sep()));\n    add_kv(LLM_KV_TOKENIZER_PAD_ID,                  uint32_t(vocab.token_pad()));\n    // add_kv(LLM_KV_TOKENIZER_CLS_ID,                  uint32_t(vocab.token_bos())); // deprecated\n    // add_kv(LLM_KV_TOKENIZER_MASK_ID,                 ???);\n    add_kv(LLM_KV_TOKENIZER_ADD_BOS,                 vocab.get_add_bos());\n    add_kv(LLM_KV_TOKENIZER_ADD_EOS,                 vocab.get_add_eos());\n    add_kv(LLM_KV_TOKENIZER_ADD_SEP,                 vocab.get_add_sep());\n    add_kv(LLM_KV_TOKENIZER_ADD_PREFIX,              vocab.get_add_space_prefix());\n    add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,         vocab.get_remove_extra_whitespaces());\n    add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,    vocab.get_precompiled_charsmap());\n    // add_kv(LLM_KV_TOKENIZER_HF_JSON,                 ???);\n    // add_kv(LLM_KV_TOKENIZER_RWKV,                    ???);\n    add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID,              uint32_t(vocab.token_fim_pre()));\n    add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID,              uint32_t(vocab.token_fim_suf()));\n    add_kv(LLM_KV_TOKENIZER_FIM_MID_ID,              uint32_t(vocab.token_fim_mid()));\n    add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID,              uint32_t(vocab.token_fim_pad()));\n    add_kv(LLM_KV_TOKENIZER_FIM_REP_ID,              uint32_t(vocab.token_fim_rep()));\n    add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID,              uint32_t(vocab.token_fim_sep()));\n\n    // TODO: implement LoRA support\n    // add_kv(LLM_KV_ADAPTER_TYPE,                      ???);\n    // add_kv(LLM_KV_ADAPTER_LORA_ALPHA,                ???);\n\n    // deprecated\n    // add_kv(LLM_KV_TOKENIZER_PREFIX_ID,               ???);\n    // add_kv(LLM_KV_TOKENIZER_SUFFIX_ID,               ???);\n    // add_kv(LLM_KV_TOKENIZER_MIDDLE_ID,               ???);\n}\n\nvoid llama_model_saver::add_tensors_from_model() {\n    if (std::string(model->output->name) != std::string(model->tok_embd->name)) {\n        add_tensor(model->tok_embd); // some models use the same tensor for tok_embd and output\n    }\n    add_tensor(model->type_embd);\n    add_tensor(model->pos_embd);\n    add_tensor(model->tok_norm);\n    add_tensor(model->tok_norm_b);\n    add_tensor(model->output_norm);\n    add_tensor(model->output_norm_b);\n    add_tensor(model->output);\n    add_tensor(model->output_b);\n    add_tensor(model->output_norm_enc);\n    add_tensor(model->cls);\n    add_tensor(model->cls_b);\n    add_tensor(model->cls_out);\n    add_tensor(model->cls_out_b);\n    add_tensor(model->cls_norm);\n\n    for (const struct llama_layer & layer : model->layers) {\n        for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {\n            add_tensor(reinterpret_cast<const struct ggml_tensor * const *>(&layer)[i]);\n        }\n    }\n}\n\nvoid llama_model_saver::save(const std::string & path_model) {\n    gguf_write_to_file(gguf_ctx, path_model.c_str(), false);\n}\n\n"
  },
  {
    "path": "examples/talk-llama/llama-model-saver.h",
    "content": "#pragma once\n\n#include \"gguf.h\"\n#include \"llama.h\"\n#include \"llama-arch.h\"\n\n#include <vector>\n\nstruct llama_model_saver {\n    struct gguf_context * gguf_ctx = nullptr;\n    const bool gguf_ctx_owned;\n    const struct llama_model * model;\n    const struct LLM_KV llm_kv;\n\n    llama_model_saver(const struct llama_model * model);\n    llama_model_saver(enum llm_arch arch, struct gguf_context * gguf_ctx);\n    ~llama_model_saver();\n\n    void add_kv(enum llm_kv key, uint32_t     value);\n    void add_kv(enum llm_kv key, int32_t      value);\n    void add_kv(enum llm_kv key, float        value);\n    void add_kv(enum llm_kv key, bool         value);\n    void add_kv(enum llm_kv key, const char * value);\n\n    [[noreturn]]\n    void add_kv(enum llm_kv key, char value); // needed to make the template below compile\n\n    template <typename Container>\n    void add_kv(enum llm_kv key, const Container & value, bool per_layer = false);\n\n    void add_kv(enum llm_kv key, const std::vector<std::string> & value);\n\n    void add_tensor(const struct ggml_tensor * tensor);\n\n    void add_kv_from_model();\n\n    void add_tensors_from_model();\n\n    void save(const std::string & path_model);\n};\n"
  },
  {
    "path": "examples/talk-llama/llama-model.cpp",
    "content": "#include \"llama-model.h\"\n\n#include \"ggml.h\"\n#include \"llama-impl.h\"\n#include \"llama-mmap.h\"\n#include \"llama-cparams.h\"\n#include \"llama-model-loader.h\"\n\n#include \"llama-kv-cache.h\"\n#include \"llama-kv-cache-iswa.h\"\n#include \"llama-memory-hybrid.h\"\n#include \"llama-memory-hybrid-iswa.h\"\n#include \"llama-memory-recurrent.h\"\n\n#include \"ggml-cpp.h\"\n\n#include \"models/models.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cfloat>\n#include <cstdint>\n#include <cstring>\n#include <cmath>\n#include <functional>\n#include <map>\n#include <regex>\n#include <sstream>\n#include <stdexcept>\n\nconst char * llm_type_name(llm_type type) {\n    switch (type) {\n        case LLM_TYPE_14M:           return \"14M\";\n        case LLM_TYPE_17M:           return \"17M\";\n        case LLM_TYPE_22M:           return \"22M\";\n        case LLM_TYPE_33M:           return \"33M\";\n        case LLM_TYPE_47M:           return \"47M\";\n        case LLM_TYPE_60M:           return \"60M\";\n        case LLM_TYPE_70M:           return \"70M\";\n        case LLM_TYPE_80M:           return \"80M\";\n        case LLM_TYPE_109M:          return \"109M\";\n        case LLM_TYPE_137M:          return \"137M\";\n        case LLM_TYPE_140M:          return \"140M\";\n        case LLM_TYPE_149M:          return \"149M\";\n        case LLM_TYPE_160M:          return \"160M\";\n        case LLM_TYPE_190M:          return \"190M\";\n        case LLM_TYPE_220M:          return \"220M\";\n        case LLM_TYPE_250M:          return \"250M\";\n        case LLM_TYPE_256M:          return \"256M\";\n        case LLM_TYPE_270M:          return \"270M\";\n        case LLM_TYPE_335M:          return \"335M\";\n        case LLM_TYPE_350M:          return \"350M\";\n        case LLM_TYPE_360M:          return \"360M\";\n        case LLM_TYPE_395M:          return \"395M\";\n        case LLM_TYPE_410M:          return \"410M\";\n        case LLM_TYPE_450M:          return \"450M\";\n        case LLM_TYPE_475M:          return \"475M\";\n        case LLM_TYPE_558M:          return \"558M\";\n        case LLM_TYPE_700M:          return \"700M\";\n        case LLM_TYPE_770M:          return \"770M\";\n        case LLM_TYPE_780M:          return \"780M\";\n        case LLM_TYPE_950M:          return \"950M\";\n        case LLM_TYPE_0_3B:          return \"0.3B\";\n        case LLM_TYPE_0_5B:          return \"0.5B\";\n        case LLM_TYPE_0_6B:          return \"0.6B\";\n        case LLM_TYPE_0_8B:          return \"0.8B\";\n        case LLM_TYPE_1B:            return \"1B\";\n        case LLM_TYPE_1_2B:          return \"1.2B\";\n        case LLM_TYPE_1_3B:          return \"1.3B\";\n        case LLM_TYPE_1_4B:          return \"1.4B\";\n        case LLM_TYPE_1_5B:          return \"1.5B\";\n        case LLM_TYPE_1_6B:          return \"1.6B\";\n        case LLM_TYPE_1_7B:          return \"1.7B\";\n        case LLM_TYPE_1_8B:          return \"1.8B\";\n        case LLM_TYPE_2B:            return \"2B\";\n        case LLM_TYPE_2_6B:          return \"2.6B\";\n        case LLM_TYPE_2_8B:          return \"2.8B\";\n        case LLM_TYPE_2_9B:          return \"2.9B\";\n        case LLM_TYPE_3B:            return \"3B\";\n        case LLM_TYPE_4B:            return \"4B\";\n        case LLM_TYPE_6B:            return \"6B\";\n        case LLM_TYPE_6_9B:          return \"6.9B\";\n        case LLM_TYPE_7B:            return \"7B\";\n        case LLM_TYPE_8B:            return \"8B\";\n        case LLM_TYPE_9B:            return \"9B\";\n        case LLM_TYPE_11B:           return \"11B\";\n        case LLM_TYPE_12B:           return \"12B\";\n        case LLM_TYPE_13B:           return \"13B\";\n        case LLM_TYPE_14B:           return \"14B\";\n        case LLM_TYPE_15B:           return \"15B\";\n        case LLM_TYPE_16B:           return \"16B\";\n        case LLM_TYPE_20B:           return \"20B\";\n        case LLM_TYPE_26B:           return \"26B\";\n        case LLM_TYPE_27B:           return \"27B\";\n        case LLM_TYPE_30B:           return \"30B\";\n        case LLM_TYPE_32B:           return \"32B\";\n        case LLM_TYPE_34B:           return \"34B\";\n        case LLM_TYPE_35B:           return \"35B\";\n        case LLM_TYPE_36B:           return \"36B\";\n        case LLM_TYPE_40B:           return \"40B\";\n        case LLM_TYPE_65B:           return \"65B\";\n        case LLM_TYPE_70B:           return \"70B\";\n        case LLM_TYPE_120B:          return \"120B\";\n        case LLM_TYPE_142B:          return \"142B\";\n        case LLM_TYPE_236B:          return \"236B\";\n        case LLM_TYPE_290B:          return \"290B\";\n        case LLM_TYPE_314B:          return \"314B\";\n        case LLM_TYPE_405B:          return \"405B\";\n        case LLM_TYPE_671B:          return \"671B\";\n        case LLM_TYPE_SMALL:         return \"0.1B\";\n        case LLM_TYPE_MEDIUM:        return \"0.4B\";\n        case LLM_TYPE_LARGE:         return \"0.8B\";\n        case LLM_TYPE_XL:            return \"1.5B\";\n        case LLM_TYPE_A1_7B:         return \"A1.7B\";\n        case LLM_TYPE_A2_7B:         return \"A2.7B\";\n        case LLM_TYPE_8x7B:          return \"8x7B\";\n        case LLM_TYPE_8x22B:         return \"8x22B\";\n        case LLM_TYPE_16x12B:        return \"16x12B\";\n        case LLM_TYPE_16x3_8B:       return \"16x3.8B\";\n        case LLM_TYPE_10B_128x3_66B: return \"10B+128x3.66B\";\n        case LLM_TYPE_57B_A14B:      return \"57B.A14B\";\n        case LLM_TYPE_17B_16E:       return \"17Bx16E (Scout)\";\n        case LLM_TYPE_17B_128E:      return \"17Bx128E (Maverick)\";\n        case LLM_TYPE_A13B:          return \"A13B\";\n        case LLM_TYPE_7B_A1B:        return \"7B.A1B\";\n        case LLM_TYPE_8B_A1B:        return \"8B.A1B\";\n        case LLM_TYPE_16B_A1B:       return \"16B.A1B\";\n        case LLM_TYPE_21B_A3B:       return \"21B.A3B\";\n        case LLM_TYPE_24B_A2B:       return \"24B.A2B\";\n        case LLM_TYPE_30B_A3B:       return \"30B.A3B\";\n        case LLM_TYPE_31B_A3_5B:     return \"31B.A3.5B\";\n        case LLM_TYPE_35B_A3B:       return \"35B.A3B\";\n        case LLM_TYPE_48B_A3B:       return \"48B.A3B\";\n        case LLM_TYPE_80B_A3B:       return \"80B.A3B\";\n        case LLM_TYPE_100B_A6B:      return \"100B.A6B\";\n        case LLM_TYPE_102B_A12B:     return \"102B.A12B\";\n        case LLM_TYPE_106B_A12B:     return \"106B.A12B\";\n        case LLM_TYPE_120B_A12B:     return \"120B.A12B\";\n        case LLM_TYPE_122B_A10B:     return \"122B.A10B\";\n        case LLM_TYPE_196B_A11B:     return \"196B.A11B\";\n        case LLM_TYPE_230B_A10B:     return \"230B.A10B\";\n        case LLM_TYPE_235B_A22B:     return \"235B.A22B\";\n        case LLM_TYPE_300B_A47B:     return \"300B.A47B\";\n        case LLM_TYPE_310B_A15B:     return \"310B.A15B\";\n        case LLM_TYPE_355B_A32B:     return \"355B.A32B\";\n        case LLM_TYPE_397B_A17B:     return \"397B.A17B\";\n        case LLM_TYPE_744B_A40B:     return \"744B.A40B\";\n        case LLM_TYPE_E2B:           return \"E2B\";\n        case LLM_TYPE_E4B:           return \"E4B\";\n        default:                     return \"?B\";\n    }\n}\n\nstatic const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {\n    switch (type) {\n        case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return \"softmax\";\n        case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return \"sigmoid\";\n        default:                                    return \"unknown\";\n    }\n}\n\nstatic const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {\n    { LLAMA_ROPE_SCALING_TYPE_NONE,       \"none\"       },\n    { LLAMA_ROPE_SCALING_TYPE_LINEAR,     \"linear\"     },\n    { LLAMA_ROPE_SCALING_TYPE_YARN,       \"yarn\"       },\n    { LLAMA_ROPE_SCALING_TYPE_LONGROPE,   \"longrope\"   },\n};\n\nstd::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type) {\n    return LLAMA_ROPE_SCALING_TYPES.at(rope_scaling_type);\n}\n\nstatic llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {\n    for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {\n        if (kv.second == name) {\n            return (llama_rope_scaling_type) kv.first;\n        }\n    }\n\n    return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;\n}\n\n// CPU: ACCEL -> GPU host -> CPU extra -> CPU\nstatic buft_list_t make_cpu_buft_list(const std::vector<ggml_backend_dev_t> & devices, bool use_extra_bufts, bool no_host) {\n    buft_list_t buft_list;\n\n    // add ACCEL buffer types\n    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n        ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {\n            auto * buft = ggml_backend_dev_buffer_type(dev);\n            // skip\n            if (buft != ggml_backend_cpu_buffer_type()) {\n                buft_list.emplace_back(dev, buft);\n            }\n        }\n    }\n\n    // add a host buffer type\n    // storing the tensors in a host buffer is useful when the processing of large batches\n    // is offloaded to a GPU device, since it reduces the time spent on data transfers\n    // generally, this will be done using the first device in the list\n    // a better approach would be to handle this on a weight-by-weight basis using the offload_op\n    // function of the device to determine if it would benefit from being stored in a host buffer\n    if (!no_host) {\n        for (auto * dev : devices) {\n            ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev);\n            if (buft) {\n                buft_list.emplace_back(dev, buft);\n                break;\n            }\n        }\n    }\n\n    // add extra buffer types\n    if (use_extra_bufts) {\n        auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n        if (cpu_dev == nullptr) {\n            throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n        }\n\n        auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);\n        auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)\n            ggml_backend_reg_get_proc_address(cpu_reg, \"ggml_backend_dev_get_extra_bufts\");\n        if (ggml_backend_dev_get_extra_bufts_fn) {\n            ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);\n            while (extra_bufts && *extra_bufts) {\n                buft_list.emplace_back(cpu_dev, *extra_bufts);\n                ++extra_bufts;\n            }\n        }\n    }\n\n    // add the CPU buffer type\n    for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n        ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {\n            buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));\n        }\n    }\n\n    return buft_list;\n}\n\n// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU\nstatic buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) {\n    buft_list_t buft_list;\n\n    // add the device split buffer type if requested and available\n    if (split_mode == LLAMA_SPLIT_MODE_ROW) {\n        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);\n        auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)\n            ggml_backend_reg_get_proc_address(reg, \"ggml_backend_split_buffer_type\");\n        if (ggml_backend_split_buffer_type_fn) {\n            size_t dev_index = [&]() {\n                auto * reg = ggml_backend_dev_backend_reg(dev);\n                for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {\n                    if (ggml_backend_reg_dev_get(reg, i) == dev) {\n                        return i;\n                    }\n                }\n                throw std::runtime_error(format(\"device %s not found in its backend reg\", ggml_backend_dev_name(dev)));\n            }();\n            auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);\n            if (buft != nullptr) {\n                buft_list.emplace_back(dev, buft);\n            }\n        }\n    }\n\n    // add the device default buffer type\n    buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));\n\n    // add the device extra buffer type (if any)\n    ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);\n    auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)\n        ggml_backend_reg_get_proc_address(reg, \"ggml_backend_dev_get_extra_bufts\");\n\n    if (ggml_backend_dev_get_extra_bufts_fn) {\n        ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);\n        while (extra_bufts && *extra_bufts) {\n            buft_list.emplace_back(dev, *extra_bufts);\n            ++extra_bufts;\n        }\n    }\n\n    return buft_list;\n}\n\nstruct llama_model::impl {\n    impl() = default;\n    ~impl() = default;\n\n    uint64_t n_elements = 0;\n\n    size_t n_bytes = 0;\n\n    std::string desc_str;\n\n    // model memory mapped files\n    llama_mmaps mappings;\n\n    // objects representing data potentially being locked in memory\n    llama_mlocks mlock_bufs;\n    llama_mlocks mlock_mmaps;\n\n    // contexts where the model tensors metadata is stored as well as the corresponding buffers:\n    std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;\n\n    buft_list_t cpu_buft_list;\n    std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;\n\n    struct layer_dev {\n        ggml_backend_dev_t dev;\n        buft_list_t * buft_list;\n    };\n\n    layer_dev dev_input = {};\n    layer_dev dev_output = {};\n    std::vector<layer_dev> dev_layer;\n\n    bool has_tensor_overrides;\n};\n\nllama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {\n    pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;\n}\n\nllama_model::~llama_model() {\n    for (auto * lora : loras) {\n        delete lora;\n    }\n}\n\nvoid llama_model::load_stats(llama_model_loader & ml) {\n    pimpl->n_elements = ml.n_elements;\n    pimpl->n_bytes = ml.n_bytes;\n}\n\nvoid llama_model::load_arch(llama_model_loader & ml) {\n    arch = ml.get_arch();\n    if (arch == LLM_ARCH_UNKNOWN) {\n        throw std::runtime_error(\"unknown model architecture: '\" + ml.get_arch_name() + \"'\");\n    }\n}\n\nvoid llama_model::load_hparams(llama_model_loader & ml) {\n    const gguf_context * ctx = ml.metadata;\n\n    // get metadata as string\n    for (int i = 0; i < gguf_get_n_kv(ctx); i++) {\n        gguf_type type = gguf_get_kv_type(ctx, i);\n        if (type == GGUF_TYPE_ARRAY) {\n            continue;\n        }\n        const char * name = gguf_get_key(ctx, i);\n        const std::string value = gguf_kv_to_str(ctx, i);\n        gguf_kv.emplace(name, value);\n    }\n\n    // get general kv\n    ml.get_key(LLM_KV_GENERAL_NAME, name, false);\n\n    // everything past this point is not vocab-related\n    // for CLIP models, we only need to load tensors, no hparams\n    if (hparams.vocab_only || ml.get_arch() == LLM_ARCH_CLIP) {\n        return;\n    }\n\n    ml.get_key(LLM_KV_CONTEXT_LENGTH,          hparams.n_ctx_train);\n    ml.get_key(LLM_KV_EMBEDDING_LENGTH,        hparams.n_embd);\n    ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT,    hparams.n_embd_out_impl, false);\n    ml.get_key(LLM_KV_BLOCK_COUNT,             hparams.n_layer);\n    ml.get_key(LLM_KV_EXPERT_COUNT,            hparams.n_expert,        false);\n    ml.get_key(LLM_KV_EXPERT_USED_COUNT,       hparams.n_expert_used,   false);\n    ml.get_key(LLM_KV_EXPERT_GROUP_COUNT,      hparams.n_expert_groups, false);\n    ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used,    false);\n\n    if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {\n        ml.get_key(LLM_KV_FEATURES_LENGTH,  hparams.n_embd);\n        ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd_out_impl);\n\n        ml.get_key(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);\n        ml.get_key(LLM_KV_POSNET_BLOCK_COUNT,      hparams.posnet.n_layer);\n\n        ml.get_key(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);\n        ml.get_key(LLM_KV_CONVNEXT_BLOCK_COUNT,      hparams.convnext.n_layer);\n    }\n\n    GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);\n    GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);\n    if (hparams.n_expert > 0) {\n        GGML_ASSERT(hparams.n_expert_used > 0);\n        GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);\n        if (hparams.n_expert_groups > 1) {\n            GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);\n            GGML_ASSERT(hparams.n_group_used > 0);\n            GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);\n        }\n    } else {\n        GGML_ASSERT(hparams.n_expert_used == 0);\n        GGML_ASSERT(hparams.n_expert_groups == 0);\n    }\n\n    std::fill(hparams.n_head_arr.begin(),    hparams.n_head_arr.end(),    0);\n    std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);\n    std::fill(hparams.n_ff_arr.begin(),      hparams.n_ff_arr.end(),      0);\n    std::fill(\n        hparams.recurrent_layer_arr.begin(),\n        hparams.recurrent_layer_arr.end(),\n        llm_arch_is_recurrent(ml.get_arch()));\n\n    std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);\n    std::fill(hparams.swa_layers.begin(), hparams.swa_layers.end(), 0);\n\n    std::fill(hparams.xielu_alpha_n.begin(), hparams.xielu_alpha_n.end(), 0.0f);\n    std::fill(hparams.xielu_alpha_p.begin(), hparams.xielu_alpha_p.end(), 0.0f);\n    std::fill(hparams.xielu_beta.begin(), hparams.xielu_beta.end(), 0.0f);\n    std::fill(hparams.xielu_eps.begin(), hparams.xielu_eps.end(), 0.0f);\n    std::fill(hparams.swiglu_clamp_exp.begin(),   hparams.swiglu_clamp_exp.end(),   0.0f);\n    std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f);\n\n    ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH,  hparams.n_ff_arr,   hparams.n_layer, false);\n    ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false);\n\n    // n_head_kv is optional, default to n_head\n    hparams.n_head_kv_arr = hparams.n_head_arr;\n\n    ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer, false);\n\n    bool rope_finetuned = false;\n    ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);\n    hparams.rope_finetuned = rope_finetuned;\n\n    hparams.n_ctx_orig_yarn = hparams.n_ctx_train;\n    ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);\n\n    // rope_freq_base (optional)\n    hparams.rope_freq_base_train = 10000.0f;\n    ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);\n\n    std::string rope_scaling(\"linear\");\n    ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);\n    hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);\n    GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);\n\n    // TODO: Handle SWA metadata similarly when models start implementing it\n    // rope_freq_scale (inverse of the kv) is optional\n    float ropescale = 0.0f;\n    if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {\n        // try the old key name\n        ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);\n    }\n    hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;\n\n    ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);\n\n    // non-transformer models do not have attention heads\n    if (hparams.n_head() > 0) {\n        // gpt-neox n_rot = rotary_pct * (n_embd / n_head)\n        // gpt-j n_rot = rotary_dim\n\n        hparams.n_embd_head_k_full = hparams.n_embd / hparams.n_head();\n        ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full, false);\n\n        hparams.n_embd_head_v_full = hparams.n_embd / hparams.n_head();\n        ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full, false);\n\n        // sanity check for n_rot (optional)\n        hparams.n_rot_full = hparams.n_embd_head_k_full;\n\n        ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full, false);\n\n        if (arch == LLM_ARCH_LLAMA || arch == LLM_ARCH_DECI || arch == LLM_ARCH_FALCON || arch == LLM_ARCH_LLAMA_EMBED) {\n            if (hparams.n_rot_full != hparams.n_embd_head_k_full) {\n                throw std::runtime_error(format(\"invalid n_rot: %u, expected %u\", hparams.n_rot_full, hparams.n_embd_head_k_full));\n            }\n        }\n    } else {\n        hparams.n_rot_full = 0;\n        hparams.n_embd_head_k_full = 0;\n        hparams.n_embd_head_v_full = 0;\n    }\n\n    // head size and n_rot for SWA layers\n    {\n        hparams.n_embd_head_k_swa = hparams.n_embd_head_k_full;\n        hparams.n_embd_head_v_swa = hparams.n_embd_head_v_full;\n        ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa, false);\n        ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa, false);\n\n        hparams.n_rot_swa = hparams.n_rot_full;\n        ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa, false);\n    }\n\n    // for differentiating model types\n    uint32_t n_vocab = 0;\n    ml.get_key(LLM_KV_VOCAB_SIZE, n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, n_vocab, false);\n\n    // for classifier models\n    ml.get_arr(LLM_KV_CLASSIFIER_OUTPUT_LABELS, classifier_labels, false);\n    if (!classifier_labels.empty()) {\n        hparams.n_cls_out = classifier_labels.size();\n    }\n\n    // arch-specific KVs\n    switch (arch) {\n        case LLM_ARCH_LLAMA:\n        case LLM_ARCH_LLAMA_EMBED:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                if (hparams.n_expert == 8) {\n                    switch (hparams.n_layer) {\n                        case 32: type = LLM_TYPE_8x7B; break;\n                        case 56: type = LLM_TYPE_8x22B; break;\n                        default: type = LLM_TYPE_UNKNOWN;\n                    }\n                } else {\n                    switch (hparams.n_layer) {\n                        case 16: type = LLM_TYPE_1B; break; // Llama 3.2 1B\n                        case 22: type = LLM_TYPE_1B; break;\n                        case 26: type = LLM_TYPE_3B; break;\n                        case 28: type = LLM_TYPE_3B; break; // Llama 3.2 3B\n                        case 30: type = LLM_TYPE_256M; break; // smoldocling 256M\n                        // granite uses a vocab with len 49152\n                        case 32: type = n_vocab == 49152 ? LLM_TYPE_3B : (n_vocab < 40000 ? LLM_TYPE_7B : LLM_TYPE_8B); break;\n                        case 36: type = LLM_TYPE_8B; break; // granite\n                        case 40: type = LLM_TYPE_13B; break;\n                        case 48: type = LLM_TYPE_34B; break;\n                        case 60: type = LLM_TYPE_30B; break;\n                        case 80: type = hparams.n_head() == hparams.n_head_kv() ? LLM_TYPE_65B : LLM_TYPE_70B; break;\n                        default: type = LLM_TYPE_UNKNOWN;\n                    }\n                }\n            } break;\n        case LLM_ARCH_LLAMA4:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP,   hparams.n_moe_layer_step);\n\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n                if (found_swa && hparams.n_swa == 0) {\n                    hparams.swa_type             = LLAMA_SWA_TYPE_NONE;\n                    hparams.n_no_rope_layer_step = hparams.n_layer; // always use rope\n                } else {\n                    hparams.swa_type                = LLAMA_SWA_TYPE_CHUNKED;\n                    hparams.n_swa                   = 8192;\n                    hparams.n_attn_temp_floor_scale = 8192;\n                    hparams.f_attn_temp_scale       = 0.1f;\n                    hparams.f_attn_temp_offset      = 1.0f;\n                    uint32_t swa_period             = 4; // pattern: 3 chunked - 1 full\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n\n                    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                }\n\n                switch (hparams.n_expert) {\n                    case 0: {\n                        // MobileLLM (no MoE)\n                        switch (hparams.n_embd) {\n                            case 2048: type = LLM_TYPE_140M; break;\n                            case 4096: type = LLM_TYPE_360M; break;\n                            case 6144: type = LLM_TYPE_950M; break;\n                            default:   type = LLM_TYPE_UNKNOWN;\n                        }\n                    } break;\n                    case 16:  type = LLM_TYPE_17B_16E; break;\n                    case 128: type = LLM_TYPE_17B_128E; break;\n                    default:  type = LLM_TYPE_UNKNOWN;\n                }\n\n                hparams.use_kq_norm = type != LLM_TYPE_17B_128E;\n            } break;\n        case LLM_ARCH_ARCEE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                // Arcee uses the same structure as Llama\n                switch (hparams.n_layer) {\n                    case 36: type = LLM_TYPE_4B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_AFMOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa, false);\n\n                // Set up interleaved sliding window attention (ISWA)\n                // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4)\n                if (hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    uint32_t swa_period = 4;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n\n                    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                } else {\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n                }\n\n                // Default to sigmoid if not set\n                if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {\n                    hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;\n                }\n\n                switch (hparams.n_layer) {\n                    case 56: type = LLM_TYPE_6B; break;\n                    case 32: type = LLM_TYPE_26B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DECI:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 80: type = LLM_TYPE_70B; break;\n                    case 162: type = LLM_TYPE_405B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MINICPM:\n            {\n                // Backward-compatible defaults for older MiniCPM GGUFs\n                hparams.f_embedding_scale = 12.0f;\n                hparams.f_residual_scale  = 1.4f / sqrtf(float(hparams.n_layer));\n                hparams.f_logit_scale     = hparams.n_embd ? (256.0f / float(hparams.n_embd)) : 1.0f;\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                // Optional KV reads, override defaults if present in newer GGUF exports\n                ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, /*required=*/false);\n                ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, /*required=*/false);\n                ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, /*required=*/false);\n\n                // MiniCPM uses rope by default, unlike Granite which uses it as a switch\n                hparams.rope_finetuned = true;\n\n                switch (hparams.n_layer) {\n                    case 52: type = LLM_TYPE_1B; break;\n                    case 40: type = LLM_TYPE_2B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MINICPM3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,       hparams.n_lora_q);\n                ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,      hparams.n_lora_kv);\n\n                switch (hparams.n_layer) {\n                    case 62: type = LLM_TYPE_4B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GROK:\n            {\n                // defaults for old GGUFs\n                hparams.yarn_beta_fast = 8.0f;\n                hparams.f_logit_scale = 0.5773502691896257f;\n                hparams.f_embedding_scale = 78.38367176906169f;\n                hparams.f_attn_out_scale = 0.08838834764831845f;\n                hparams.f_attn_logit_softcapping = 30.0f;\n                hparams.f_router_logit_softcapping = 30.0f;\n                // no final_logit_softcapping in grok-1\n                hparams.f_final_logit_softcapping = 0.0f;\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,  hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,   hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_LOGIT_SCALE,                  hparams.f_logit_scale, false);\n                ml.get_key(LLM_KV_EMBEDDING_SCALE,              hparams.f_embedding_scale, false);\n                ml.get_key(LLM_KV_ATTENTION_OUTPUT_SCALE,       hparams.f_attn_out_scale, false);\n                ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING,       hparams.f_attn_logit_softcapping, false);\n                ml.get_key(LLM_KV_ROUTER_LOGIT_SOFTCAPPING,     hparams.f_router_logit_softcapping, false);\n                ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING,      hparams.f_final_logit_softcapping, false);\n\n                ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH,  hparams.attn_temp_length, false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR,  hparams.yarn_ext_factor, false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR, hparams.yarn_attn_factor, false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_FAST,   hparams.yarn_beta_fast, false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW,   hparams.yarn_beta_slow, false);\n\n                switch (hparams.n_layer) {\n                    case 64: type = LLM_TYPE_314B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_FALCON:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 60: type = LLM_TYPE_40B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_BAICHUAN:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_13B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                if (type == LLM_TYPE_13B) {\n                    // TODO: become GGUF KV parameter\n                    hparams.f_max_alibi_bias = 8.0f;\n                }\n            } break;\n        case LLM_ARCH_STARCODER:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1B; break;\n                    case 36: type = LLM_TYPE_3B; break;\n                    case 42: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_15B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_REFACT:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_1B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                // TODO: become GGUF KV parameter\n                hparams.f_max_alibi_bias = 8.0f;\n            } break;\n        case LLM_ARCH_BERT:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);\n\n                switch (hparams.n_layer) {\n                    case 3:\n                        type = LLM_TYPE_17M; break; // bge-micro\n                    case 6:\n                        type = LLM_TYPE_22M; break; // MiniLM-L6\n                    case 12:\n                        switch (hparams.n_embd) {\n                            case 384: type = LLM_TYPE_33M; break; // MiniLM-L12, bge-small\n                            case 768: type = LLM_TYPE_109M; break; // bge-base\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 24:\n                        type = LLM_TYPE_335M; break; // bge-large\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MODERN_BERT:\n            {\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n                if (found_swa && hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                    uint32_t swa_period = 3;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period, true);\n                } else {\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n                }\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,        hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,            hparams.pooling_type, false);\n\n                switch (hparams.n_layer) {\n                    case 12:\n                        type = LLM_TYPE_47M; break; // granite-embedding-small\n                    case 22:\n                        type = LLM_TYPE_149M; break; // modern-bert-base\n                    case 28:\n                        type = LLM_TYPE_395M; break; // modern-bert-large\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_JINA_BERT_V2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);\n                hparams.f_max_alibi_bias = 8.0f;\n\n                switch (hparams.n_layer) {\n                    case 4:  type = LLM_TYPE_33M;  break; // jina-embeddings-small\n                    case 12: type = LLM_TYPE_137M; break; // jina-embeddings-base\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_JINA_BERT_V3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);\n\n                switch (hparams.n_layer) {\n                    case 24:\n                        type = LLM_TYPE_558M; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_NOMIC_BERT:\n        case LLM_ARCH_NOMIC_BERT_MOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);\n                ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS,         hparams.moe_every_n_layers, 0);\n\n                if (hparams.n_layer == 12 && hparams.n_embd == 768) {\n                    if (arch == LLM_ARCH_NOMIC_BERT) {\n                        type = LLM_TYPE_137M;\n                    } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) {\n                        type = LLM_TYPE_475M;\n                    }\n                }\n            } break;\n        case LLM_ARCH_NEO_BERT:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,            hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,                hparams.pooling_type, false);\n\n                if (hparams.n_layer == 28) {\n                    type = LLM_TYPE_250M;\n                }\n            } break;\n        case LLM_ARCH_EUROBERT:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,            hparams.causal_attn, false);\n                ml.get_key(LLM_KV_POOLING_TYPE,                hparams.pooling_type, false);\n\n                if (hparams.n_layer == 12) {\n                    type = LLM_TYPE_SMALL;  // 0.2B\n                }\n            } break;\n        case LLM_ARCH_BLOOM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1B; break;\n                    case 30:\n                        switch (hparams.n_embd) {\n                            case 2560: type = LLM_TYPE_3B; break;\n                            case 4096: type = LLM_TYPE_7B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                // TODO: become GGUF KV parameter\n                hparams.f_max_alibi_bias = 8.0f;\n            } break;\n        case LLM_ARCH_MPT:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,  hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV,      hparams.f_clamp_kqv, false);\n                ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias, false);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 48: type = LLM_TYPE_30B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_STABLELM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_3B; break;\n                    case 40: type = LLM_TYPE_12B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n            } break;\n        case LLM_ARCH_QWEN:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_13B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN2VL:\n            {\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);\n            }\n            // fall through\n        case LLM_ARCH_QWEN2:\n            {\n                ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_5B : LLM_TYPE_1B; break;\n                    case 28: type = hparams.n_embd == 1536 ? LLM_TYPE_1_5B : LLM_TYPE_7B; break;\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 36: type = LLM_TYPE_3B; break;\n                    case 40: type = hparams.n_head() == 20 ? LLM_TYPE_4B : LLM_TYPE_13B; break;\n                    case 48: type = LLM_TYPE_14B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    case 80: type = LLM_TYPE_70B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DREAM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                // Dream models are primarily 7B with 28 layers\n                switch (hparams.n_layer) {\n                    case 28:\n                        type = LLM_TYPE_7B;\n                        break;\n                    default:\n                        type = LLM_TYPE_UNKNOWN;\n                }\n                // Set non-causal attention for diffusion models\n                hparams.causal_attn = false;\n            }\n            break;\n        case LLM_ARCH_LLADA:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                // LLaDA-8B has 32 layers, similar to LLaMA but for diffusion\n                switch (hparams.n_layer) {\n                    case 32:\n                        type = LLM_TYPE_8B;\n                        break;\n                    default:\n                        type = LLM_TYPE_UNKNOWN;\n                }\n                // Set non-causal attention for diffusion models\n                hparams.causal_attn = false;\n            }\n            break;\n        case LLM_ARCH_LLADA_MOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                // diffusion language model uses non-causal attention\n                hparams.causal_attn = false;\n                switch (hparams.n_layer) {\n                    case 16: type = LLM_TYPE_A1_7B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_RND1:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_30B_A3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n                // Set non-causal attention for diffusion models\n                hparams.causal_attn = false;\n            } break;\n        case LLM_ARCH_QWEN2MOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_A2_7B; break;\n                    case 28: type = LLM_TYPE_57B_A14B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN3:\n            {\n                ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 28: type = hparams.n_embd == 1024 ? LLM_TYPE_0_6B : LLM_TYPE_1_7B; break;\n                    case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break;\n                    case 40: type = LLM_TYPE_14B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MAINCODER:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_1B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN3VL:\n            {\n                ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 28: type = LLM_TYPE_1_7B; break;\n                    case 36: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_8B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN3MOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_30B_A3B; break;\n                    case 94: type = LLM_TYPE_235B_A22B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN3VLMOE:\n            {\n                ml.get_key(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers, false);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_30B_A3B; break;\n                    case 94: type = LLM_TYPE_235B_A22B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PHI2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PHI3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_3B; break;\n                    case 40: type = LLM_TYPE_14B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n\n                if (found_swa && hparams.n_swa > 0) {\n                    LLAMA_LOG_WARN(\"%s: Phi SWA is currently disabled - results might be suboptimal for some models (see %s)\\n\",\n                            __func__, \"https://github.com/ggml-org/llama.cpp/pull/13676\");\n\n                    // TODO: fix conversion scripts to correctly populate `n_swa` and `n_swa_pattern`\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n\n                    hparams.n_swa         = 0;\n                    hparams.set_swa_pattern(1);\n                }\n            } break;\n        case LLM_ARCH_PHIMOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_16x3_8B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PLAMO:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 40: type = LLM_TYPE_13B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n            } break;\n        case LLM_ARCH_PLAMO2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                // Load Mamba SSM parameters\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                    hparams.recurrent_layer_arr[i] = hparams.n_head_kv(i) == 0;\n                }\n\n                switch (hparams.n_layer) {\n                    case 16: type = LLM_TYPE_1B; break;\n                    case 32:\n                        if (hparams.n_embd == 2048) {\n                            type = LLM_TYPE_2B;\n                        } else if (hparams.n_embd == 4096) {\n                            type = LLM_TYPE_8B;\n                        }\n                        break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PLAMO3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n                if (found_swa && hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                    uint32_t swa_period = 8;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n                } else {\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n                }\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_2B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GPT2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 12: type = LLM_TYPE_SMALL; break;\n                    case 24: type = LLM_TYPE_MEDIUM; break;\n                    case 36: type = LLM_TYPE_LARGE; break;\n                    case 48: type = LLM_TYPE_XL; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_CODESHELL:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 42: type = LLM_TYPE_7B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_ORION:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 40: type = LLM_TYPE_14B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_INTERNLM2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 48: type = LLM_TYPE_20B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GEMMA:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 18: type = LLM_TYPE_2B; break;\n                    case 28: type = LLM_TYPE_7B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n            } break;\n        case LLM_ARCH_GEMMA2:\n            {\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                hparams.n_swa = 4096; // default value of gemma 2\n                uint32_t swa_period = 2;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.set_swa_pattern(swa_period);\n                hparams.attn_soft_cap = true;\n                hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,          hparams.rope_freq_base_train_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING,      hparams.f_attn_logit_softcapping, false);\n                ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING,     hparams.f_final_logit_softcapping, false);\n\n                switch (hparams.n_layer) {\n                    case 26: type = LLM_TYPE_2B; break;\n                    case 42: type = LLM_TYPE_9B; break;\n                    case 46: type = LLM_TYPE_27B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n\n                // ref: https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py#L173\n                hparams.f_attention_scale = type == LLM_TYPE_27B\n                    ? 1.0f / std::sqrt(float(hparams.n_embd / hparams.n_head(0)))\n                    : 1.0f / std::sqrt(float(hparams.n_embd_head_k()));\n            } break;\n        case LLM_ARCH_GEMMA3:\n            {\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n                if (found_swa && hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    uint32_t swa_period = 6;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                } else {\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n                }\n\n                hparams.f_final_logit_softcapping = 0.0f;\n                ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 18: type = LLM_TYPE_270M; break;\n                    case 26: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_8B; break; // Rnj-1\n                    case 34: type = LLM_TYPE_4B; break;\n                    case 48: type = LLM_TYPE_12B; break;\n                    case 62: type = LLM_TYPE_27B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                // ref: https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py#L289\n                hparams.f_attention_scale = type == LLM_TYPE_27B\n                    ? 1.0f / std::sqrt(float(hparams.n_embd / hparams.n_head(0)))\n                    : 1.0f / std::sqrt(float(hparams.n_embd_head_k()));\n            } break;\n        case LLM_ARCH_GEMMA3N:\n            {\n                uint32_t swa_period = 5;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                hparams.set_swa_pattern(swa_period);\n\n                hparams.n_layer_kv_from_start     = 20;\n                hparams.f_attention_scale         = 1.0f;\n\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,          hparams.rope_freq_base_train_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 30: type = LLM_TYPE_E2B; break;\n                    case 35: type = LLM_TYPE_E4B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GEMMA_EMBEDDING:\n            {\n                hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;\n                uint32_t swa_period = 6;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.set_swa_pattern(swa_period);\n\n                hparams.causal_attn = false; // embeddings do not use causal attention\n\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);\n\n                //applied only if model converted with --sentence-transformers-dense-modules\n                ml.get_key(LLM_KV_DENSE_2_FEAT_IN, hparams.dense_2_feat_in, false);\n                ml.get_key(LLM_KV_DENSE_2_FEAT_OUT, hparams.dense_2_feat_out, false);\n                ml.get_key(LLM_KV_DENSE_3_FEAT_IN, hparams.dense_3_feat_in, false);\n                ml.get_key(LLM_KV_DENSE_3_FEAT_OUT, hparams.dense_3_feat_out, false);\n\n                GGML_ASSERT((hparams.dense_2_feat_in == 0 || hparams.dense_2_feat_in == hparams.n_embd) && \"dense_2_feat_in must be equal to n_embd\");\n                GGML_ASSERT((hparams.dense_3_feat_out == 0 || hparams.dense_3_feat_out == hparams.n_embd) && \"dense_3_feat_out must be equal to n_embd\");\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_0_3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n                hparams.f_attention_scale = 1.0f / std::sqrt(float(hparams.n_embd_head_k()));\n\n            } break;\n        case LLM_ARCH_STARCODER2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 30: type = LLM_TYPE_3B; break;\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_15B; break;\n                    case 52: type = LLM_TYPE_20B; break; // granite\n                    case 88: type = LLM_TYPE_34B; break; // granite\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MAMBA:\n            {\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_DT_B_C_RMS,     hparams.ssm_dt_b_c_rms, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 24:\n                        switch (hparams.n_embd) {\n                            case 768: type = LLM_TYPE_SMALL; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 48:\n                        switch (hparams.n_embd) {\n                            case 1024: type = LLM_TYPE_MEDIUM; break;\n                            case 1536: type = LLM_TYPE_LARGE; break;\n                            case 2048: type = LLM_TYPE_XL; break;\n                            default:   type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 64:\n                        switch (hparams.n_embd) {\n                            case 2560: type = LLM_TYPE_3B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MAMBA2:\n            {\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 24:\n                        switch (hparams.n_embd) {\n                            case 768: type = LLM_TYPE_SMALL; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 48:\n                        switch (hparams.n_embd) {\n                            case 1024: type = LLM_TYPE_MEDIUM; break;\n                            case 1536: type = LLM_TYPE_LARGE; break;\n                            case 2048: type = LLM_TYPE_XL; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 64:\n                        switch (hparams.n_embd) {\n                            case 2560: type = LLM_TYPE_3B; break;\n                            case 4096: type = LLM_TYPE_7B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_JAMBA:\n            {\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                    hparams.recurrent_layer_arr[i] = hparams.n_head_kv(i) == 0;\n                }\n\n                switch (hparams.n_layer) {\n                    // TODO: Jamba layers are a bit heterogeneous, so naming this is hard.\n                    case 12: // 900M  8x???M\n                    case 32: // 51B  16x?B\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_XVERSE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_13B; break;\n                    case 80: type = LLM_TYPE_65B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_COMMAND_R:\n            {\n                ml.get_key(LLM_KV_LOGIT_SCALE,             hparams.f_logit_scale, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 40: type = LLM_TYPE_35B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_COHERE2:\n            {\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                uint32_t swa_period = 4;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.set_swa_pattern(swa_period);\n                hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,       hparams.rope_freq_base_train_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);\n                ml.get_key(LLM_KV_LOGIT_SCALE,              hparams.f_logit_scale);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,  hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_8B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DBRX:\n        {\n            ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n            ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV,     hparams.f_clamp_kqv);\n\n            switch (hparams.n_layer) {\n                case 40: type = LLM_TYPE_16x12B; break;\n                default: type = LLM_TYPE_UNKNOWN;\n            }\n        } break;\n        case LLM_ARCH_OLMO:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV,     hparams.f_clamp_kqv, false);\n\n                switch (hparams.n_layer) {\n                    case 22: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 80: type = LLM_TYPE_70B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_OLMO2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n                if (found_swa && hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    uint32_t swa_period = 4;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n\n                    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                    hparams.rope_freq_scale_train_swa = 1.0; // See olmo2.cpp\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                } else {\n                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;\n                }\n\n                switch (hparams.n_layer) {\n                    case 16: type = LLM_TYPE_1B; break;\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 40: type = LLM_TYPE_13B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_SEED_OSS:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 64: type = LLM_TYPE_36B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_OLMOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 16: type = LLM_TYPE_A1_7B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_OPENELM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                case 16: type = LLM_TYPE_270M; break;\n                case 20: type = LLM_TYPE_450M; break;\n                case 28: type = LLM_TYPE_1B; break;\n                case 36: type = LLM_TYPE_3B; break;\n                default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GPTNEOX:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                ml.get_key(LLM_KV_USE_PARALLEL_RESIDUAL,   hparams.use_par_res);\n                switch (hparams.n_layer) {\n                    case 6:\n                        switch (hparams.n_ff()) {\n                            case 512:  type = LLM_TYPE_14M; break;\n                            case 2048: type = LLM_TYPE_70M; break;\n                            default:   type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 12:\n                        switch (hparams.n_ff()) {\n                            case 3072: type = LLM_TYPE_160M; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 16:\n                        switch (hparams.n_ff()) {\n                            case 8192: type = LLM_TYPE_1B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 24:\n                        switch (hparams.n_ff()) {\n                            case 4096: type = LLM_TYPE_410M; break;\n                            case 8192: type = LLM_TYPE_1_4B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 32:\n                        switch (hparams.n_ff()) {\n                            case 10240: type = LLM_TYPE_2_8B; break;\n                            case 16384: type = LLM_TYPE_6_9B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 36:\n                        switch (hparams.n_ff()) {\n                            case 20480: type = LLM_TYPE_12B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 44:\n                        switch (hparams.n_ff()) {\n                            case 24576: type = LLM_TYPE_20B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_ARCTIC:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                if (hparams.n_expert == 128) {\n                    switch (hparams.n_layer) {\n                        case 35: type = LLM_TYPE_10B_128x3_66B; break;\n                        default: type = LLM_TYPE_UNKNOWN;\n                    }\n                } else {\n                    type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DEEPSEEK:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n\n                switch (hparams.n_ff_exp) {\n                    case 1408: type = LLM_TYPE_16B; break;\n                    case 1792: type = LLM_TYPE_20B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DEEPSEEK2:\n            {\n                // lite variants include DeepSeek-V2-Lite, GigaChat3-10B-A1.8B, Kanana-2-30B-A3B\n                const bool is_lite = (hparams.n_layer == 27 || hparams.n_layer == 26 || (hparams.n_layer == 48 && n_vocab == 128256));\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                if (!is_lite) {\n                    ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);\n                }\n                ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,     hparams.n_lora_kv);\n                ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   hparams.n_embd_head_k_mla_impl, false);\n                ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,        hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,       hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,        hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,         hparams.expert_gating_func, false);\n                if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {\n                    // for compatibility with existing DeepSeek V2 and V2.5 GGUFs\n                    // that have no expert_gating_func model parameter set\n                    if ((hparams.n_layer == 47 || hparams.n_layer == 48) && n_vocab == 154880) {\n                        // GLM 4.7 Lite\n                        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;\n                    } else {\n                        hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX;\n                    }\n                }\n\n                if (ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul, 0.0f)) {\n                    // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]\n                    // cancel the factor from the convert script\n                    hparams.rope_yarn_log_mul /= 0.1f;\n                }\n\n                // (optional) temperature tuning - used by mistral-large\n                ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE,  hparams.f_attn_temp_scale,       false);\n                ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.n_attn_temp_floor_scale, false);\n\n                hparams.f_attn_temp_offset = 0.0f;\n\n                switch (hparams.n_layer) {\n                    case 27: type = LLM_TYPE_16B; break;\n                    case 47: type = LLM_TYPE_30B_A3B; break;\n                    case 60: type = LLM_TYPE_236B; break;\n                    case 61: type = LLM_TYPE_671B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PLM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_1_8B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_CHATGLM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 28: {\n                        if (hparams.n_head(0) == 16) {\n                            type = LLM_TYPE_1_5B;\n                        } else {\n                            type = LLM_TYPE_6B;\n                        }\n                    } break;\n                    case 40: {\n                        if (hparams.n_head(0) == 24) {\n                            type = LLM_TYPE_4B;\n                        } else {\n                            type = LLM_TYPE_9B;\n                        }\n                    } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GLM4:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,    hparams.f_norm_rms_eps);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);\n\n                // NextN/MTP parameters (GLM-OCR)\n                ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);\n\n                // TODO: when MTP is implemented, this should probably be updated if needed\n                hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;\n\n                switch (hparams.n_layer) {\n                    case 17: type = LLM_TYPE_1B; break; // GLM-OCR\n                    case 40: type = LLM_TYPE_9B; break;\n                    case 61: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GLM4_MOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,     hparams.n_ff_exp);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,    hparams.f_norm_rms_eps);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);\n\n                // MoE parameters\n                ml.get_key(LLM_KV_EXPERT_COUNT,                hparams.n_expert);\n                ml.get_key(LLM_KV_EXPERT_USED_COUNT,           hparams.n_expert_used);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm, false);\n\n                // Expert gating function (GLM-4.5 uses sigmoid)\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);\n                if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {\n                    hparams.expert_gating_func =  LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;\n                }\n\n                // NextN/MTP parameters\n                ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);\n\n                // TODO: when MTP is implemented, this should probably be updated if needed\n                hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;\n\n                switch (hparams.n_layer) {\n                    case 47: type = LLM_TYPE_106B_A12B; break; // GLM-4.5-Air (46 layers + 1 NextN layer)\n                    case 48: type = LLM_TYPE_102B_A12B; break; // Solar Open\n                    case 93: type = LLM_TYPE_355B_A32B; break; // GLM-4.5 (92 layers + 1 NextN layer)\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GLM_DSA:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,     hparams.n_ff_exp);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,    hparams.f_norm_rms_eps);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);\n\n                // MoE parameters\n                ml.get_key(LLM_KV_EXPERT_COUNT,                hparams.n_expert);\n                ml.get_key(LLM_KV_EXPERT_USED_COUNT,           hparams.n_expert_used);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm, false);\n\n                // deepseek MLA parameters\n                ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK,      hparams.n_lora_q);\n                ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,     hparams.n_lora_kv);\n                ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   hparams.n_embd_head_k_mla_impl, false);\n                ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,        hparams.n_expert_shared);\n\n                // DSA parameters\n                ml.get_key(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);\n                ml.get_key(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);\n                ml.get_key(LLM_KV_ATTENTION_INDEXER_TOP_K,      hparams.indexer_top_k);\n\n                // Expert gating function (GLM-4.5 uses sigmoid)\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);\n                if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {\n                    hparams.expert_gating_func =  LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;\n                }\n\n                // NextN/MTP parameters\n                ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,        hparams.nextn_predict_layers, false);\n\n                // TODO: when MTP is implemented, this should probably be updated if needed\n                hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;\n\n                switch (hparams.n_layer) {\n                    case 79: type = LLM_TYPE_744B_A40B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_BITNET:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 26: type = LLM_TYPE_3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_T5:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,      hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);\n\n                uint32_t dec_start_token_id;\n                if (ml.get_key(LLM_KV_DECODER_START_TOKEN_ID, dec_start_token_id, false)) {\n                    hparams.dec_start_token_id = dec_start_token_id;\n                }\n\n                hparams.dec_n_layer = hparams.n_layer;\n                ml.get_key(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer, false);\n\n                switch (hparams.n_layer) {\n                    case 6:  type = LLM_TYPE_60M;  break; // t5-small\n                    case 8:  type = LLM_TYPE_80M;  break; // flan-t5-small\n                    case 12:\n                        switch (hparams.n_ff()) {\n                            case 3072: type = LLM_TYPE_220M; break; // t5-base\n                            case 2048: type = LLM_TYPE_250M; break; // flan-t5-base\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 24:\n                        switch (hparams.n_ff()) {\n                            case 4096:  type = LLM_TYPE_770M; break; // t5-large\n                            case 2816:  type = LLM_TYPE_780M; break; // flan-t5-large\n                            case 16384: type = LLM_TYPE_3B;   break; // t5-3b\n                            case 5120:  type = LLM_TYPE_3B;   break; // flan-t5-xl\n                            case 65536: type = LLM_TYPE_11B;  break; // t5-11b\n                            case 10240: type = LLM_TYPE_11B;  break; // flan-t5-xxl\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n            } break;\n        case LLM_ARCH_T5ENCODER:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);\n                type = LLM_TYPE_UNKNOWN;\n            } break;\n        case LLM_ARCH_JAIS:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias, false);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1_3B; break;\n                    case 40: type = LLM_TYPE_13B; break;\n                    /* TODO: add variants */\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_JAIS2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_8B; break;\n                    case 68: type = LLM_TYPE_70B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_NEMOTRON:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_4B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_NEMOTRON_H:\n        case LLM_ARCH_NEMOTRON_H_MOE:\n            {\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                // A layer is recurrent IFF the n_head_kv value is set to 0 and\n                // the n_ff value is set to 0\n                for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                    hparams.recurrent_layer_arr[i] = (hparams.n_head_kv(i) == 0 && hparams.n_ff(i) == 0);\n                }\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp,        false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp,      false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_MOE_LATENT_SIZE,                   hparams.moe_latent_size, false);\n\n                switch (hparams.n_layer) {\n                    case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31B\n                    case 56: type = LLM_TYPE_9B; break;\n                    case 88: type = LLM_TYPE_120B_A12B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_EXAONE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_8B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_EXAONE4:\n            {\n                if (hparams.n_layer == 64) {    // 32B\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    hparams.n_swa = 4096;\n                    uint32_t swa_period = 4;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period);\n\n                    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                }\n\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 30: type = LLM_TYPE_1_2B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_EXAONE_MOE:\n            {\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                hparams.n_swa = 128;\n                uint32_t swa_period = 4;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.set_swa_pattern(swa_period);\n                hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,                hparams.rope_freq_base_train_swa, false);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,          hparams.n_swa);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead, false);\n\n                ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.nextn_predict_layers, false);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_30B_A3B; break;\n                    case 48:\n                    case 49: type = LLM_TYPE_235B_A22B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_RWKV6:\n        case LLM_ARCH_RWKV6QWEN2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,     hparams.f_norm_eps, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false);\n                ml.get_key(LLM_KV_WKV_HEAD_SIZE,               hparams.wkv_head_size);\n                ml.get_key(LLM_KV_TIME_MIX_EXTRA_DIM,          hparams.time_mix_extra_dim);\n                ml.get_key(LLM_KV_TIME_DECAY_EXTRA_DIM,        hparams.time_decay_extra_dim);\n                ml.get_key(LLM_KV_RESCALE_EVERY_N_LAYERS,      hparams.rescale_every_n_layers, false);\n                ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT,           hparams.token_shift_count, false);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_1_6B; break;\n                    case 32:\n                        switch (hparams.n_embd) {\n                            case 2560: type = LLM_TYPE_3B; break;\n                            case 4096: type = LLM_TYPE_7B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 61: type = LLM_TYPE_14B; break;\n                    case 64: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_RWKV7:\n        case LLM_ARCH_ARWKV7:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,                hparams.f_norm_eps, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,            hparams.f_norm_rms_eps, false);\n                ml.get_key(LLM_KV_WKV_HEAD_SIZE,                          hparams.wkv_head_size);\n                ml.get_key(LLM_KV_ATTENTION_DECAY_LORA_RANK,              hparams.n_lora_decay);\n                ml.get_key(LLM_KV_ATTENTION_ICLR_LORA_RANK,               hparams.n_lora_iclr);\n                ml.get_key(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);\n                ml.get_key(LLM_KV_ATTENTION_GATE_LORA_RANK,               hparams.n_lora_gate, false);\n                ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT,                      hparams.token_shift_count, false);\n\n                switch (hparams.n_layer) {\n                    case 12:\n                        switch (hparams.n_embd) {\n                            case 768: type = LLM_TYPE_190M; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 24:\n                        switch (hparams.n_embd) {\n                            case 1024: type = LLM_TYPE_450M; break;\n                            case 2048: type = LLM_TYPE_1_5B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 28:\n                        switch (hparams.n_embd) {\n                            case 1536: type = LLM_TYPE_1_5B; break;\n                            case 3584: type = LLM_TYPE_7B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 32:\n                        switch (hparams.n_embd) {\n                            case 2560: type = LLM_TYPE_2_9B; break;\n                            case 4096: type = LLM_TYPE_7B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    case 61:\n                        switch (hparams.n_embd) {\n                            case 4096: type = LLM_TYPE_14B; break;\n                            default: type = LLM_TYPE_UNKNOWN;\n                        } break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GRANITE:\n        case LLM_ARCH_GRANITE_MOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LOGIT_SCALE,                 hparams.f_logit_scale);\n                ml.get_key(LLM_KV_RESIDUAL_SCALE,              hparams.f_residual_scale, false);\n                ml.get_key(LLM_KV_EMBEDDING_SCALE,             hparams.f_embedding_scale, false);\n                ml.get_key(LLM_KV_ATTENTION_SCALE,             hparams.f_attention_scale, false);\n\n                // Granite uses rope_finetuned as a switch for rope, so default to true\n                bool rope_finetuned = true;\n                ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);\n                hparams.rope_finetuned = rope_finetuned;\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_3B; break;\n                    case 40: type = LLM_TYPE_3B; break;\n                    // Add additional layer/vocab/etc checks here for other model sizes\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                // For Granite MoE Shared\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false);\n            } break;\n        case LLM_ARCH_GRANITE_HYBRID:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LOGIT_SCALE,                 hparams.f_logit_scale, /* required */ false);\n                ml.get_key(LLM_KV_RESIDUAL_SCALE,              hparams.f_residual_scale, /* required */ false);\n                ml.get_key(LLM_KV_EMBEDDING_SCALE,             hparams.f_embedding_scale, /* required */ false);\n                ml.get_key(LLM_KV_ATTENTION_SCALE,             hparams.f_attention_scale, /* required */ false);\n\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                // Granite uses rope_finetuned as a switch for rope, so default to true\n                bool rope_finetuned = true;\n                ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);\n                hparams.rope_finetuned = rope_finetuned;\n\n                // A layer is recurrent IFF the n_head_kv value is set to 0\n                for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                    hparams.recurrent_layer_arr[i] = hparams.n_head_kv(i) == 0;\n                }\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_embd) {\n                    case 768: type = LLM_TYPE_350M; break;\n                    case 1536: type = (hparams.n_embd == 2048 ? LLM_TYPE_7B_A1B : LLM_TYPE_1B); break;\n                    case 2048: case 2560: type = LLM_TYPE_3B; break;\n                    case 4096: type = LLM_TYPE_32B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n\n                // For Granite MoE Shared\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, /* required */ false);\n            } break;\n        case LLM_ARCH_CHAMELEON:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                hparams.f_norm_eps = 1e-5;  // eps for qk-norm, torch default\n                ml.get_key(LLM_KV_SWIN_NORM, hparams.swin_norm, false);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_7B; break;\n                    case 48: type = LLM_TYPE_34B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n               }\n            } break;\n        case LLM_ARCH_WAVTOKENIZER_DEC:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);\n                ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS,    hparams.f_norm_group_eps);\n                ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);\n                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn, false);\n            } break;\n        case LLM_ARCH_BAILINGMOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm, false);\n\n                switch (hparams.n_layer) {\n                    case 28: type = LLM_TYPE_16B; break;\n                    case 88: type = LLM_TYPE_290B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_BAILINGMOE2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);\n                ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS,              hparams.nextn_predict_layers, false);\n\n                // TODO: when MTP is implemented, this should probably be updated if needed\n                hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;\n\n                switch (hparams.n_layer) {\n                    case 20: type = LLM_TYPE_16B_A1B; break;\n                    case 21: type = LLM_TYPE_16B_A1B; break;\n                    case 32: type = LLM_TYPE_100B_A6B; break;\n                    case 33: type = LLM_TYPE_100B_A6B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_DOTS1:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,         hparams.n_expert_shared);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,        hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,         hparams.expert_weights_norm, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);\n                switch (hparams.n_layer) {\n                    case 62: type = LLM_TYPE_142B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_ERNIE4_5:\n        case LLM_ARCH_ERNIE4_5_MOE:\n        case LLM_ARCH_PADDLEOCR:\n            {\n                // paddleocr need mrope_section\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);\n\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                if (arch == LLM_ARCH_ERNIE4_5_MOE) {\n                    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                    ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP,         hparams.n_moe_layer_step);\n                    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead, false);\n                }\n\n                switch (hparams.n_layer) {\n                    case 18: type = LLM_TYPE_0_3B; break;\n                    case 28: type = LLM_TYPE_21B_A3B; break;\n                    case 54: type = LLM_TYPE_300B_A47B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_FALCON_H1:\n            {\n                // Common parameters\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                // SSM parameters\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                std::fill(hparams.recurrent_layer_arr.begin(), hparams.recurrent_layer_arr.end(), true);\n\n                switch (hparams.n_layer) {\n                    case 36:\n                        type = LLM_TYPE_0_5B; break;\n                    case 24:\n                        type = LLM_TYPE_1_5B; break;\n                    case 66:\n                        type = LLM_TYPE_1B; break;\n                    case 32:\n                        type = LLM_TYPE_3B; break;\n                    case 44:\n                        type = LLM_TYPE_7B; break;\n                    case 72:\n                        type = LLM_TYPE_34B; break;\n                    default:\n                        type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_HUNYUAN_MOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_A13B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_HUNYUAN_DENSE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                switch (hparams.n_embd) {\n                    case 1024: type = LLM_TYPE_0_5B; break;\n                    case 2048: type = LLM_TYPE_1_8B; break;\n                    case 3072: type = LLM_TYPE_4B; break;\n                    case 4096: type = LLM_TYPE_7B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_SMOLLM3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                hparams.n_no_rope_layer_step = 4;\n\n                switch (hparams.n_layer) {\n                    case 36: type = LLM_TYPE_3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_OPENAI_MOE:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,    hparams.n_swa);\n\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                uint32_t swa_period = 2;\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                hparams.set_swa_pattern(swa_period);\n\n                hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_20B; break;\n                    case 36: type = LLM_TYPE_120B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_LFM2:\n            {\n                ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                for (uint32_t il = 0; il < hparams.n_layer; ++il) {\n                    hparams.recurrent_layer_arr[il] = hparams.n_head_kv(il) == 0;\n                }\n                hparams.n_layer_dense_lead = hparams.n_layer;\n                switch (hparams.n_ff()) {\n                    case  4608: type = LLM_TYPE_350M; break;\n                    case  6912: type = LLM_TYPE_700M; break;\n                    case  8192: type = LLM_TYPE_1_2B; break;\n                    case 10752: type = LLM_TYPE_2_6B; break;\n                    default:    type = LLM_TYPE_UNKNOWN;\n                }\n                if (const auto is_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); is_swa && hparams.n_swa > 0) {\n                    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n                    for (uint32_t il = 0; il < hparams.n_layer; ++il) {\n                        hparams.swa_layers[il] = !hparams.recurrent_layer_arr[il];\n                    }\n                }\n            } break;\n        case LLM_ARCH_LFM2MOE:\n            {\n                ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,   hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func);\n\n                for (uint32_t il = 0; il < hparams.n_layer; ++il) {\n                    hparams.recurrent_layer_arr[il] = hparams.n_head_kv(il) == 0;\n                }\n\n                switch (hparams.n_layer) {\n                    case 24: type = LLM_TYPE_8B_A1B;  break;\n                    case 40: type = LLM_TYPE_24B_A2B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_SMALLTHINKER:\n            {\n                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);\n\n                if (found_swa && hparams.n_swa > 0) {\n                    hparams.swa_type    = LLAMA_SWA_TYPE_STANDARD;\n                    hparams.n_swa       = 4096;\n                    uint32_t swa_period = 4;\n                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);\n                    hparams.set_swa_pattern(swa_period, true);\n\n                    hparams.rope_freq_base_train_swa  = hparams.rope_freq_base_train;\n                    hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;\n                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);\n                } else {\n                    hparams.swa_type             = LLAMA_SWA_TYPE_NONE;\n                    hparams.n_no_rope_layer_step = hparams.n_layer;\n                }\n\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,  hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,          hparams.expert_gating_func, false);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_4B;  break;\n                    case 52: type = LLM_TYPE_20B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_GROVEMOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_CHUNK_FEED_FORWARD_LENGTH,  hparams.n_ff_chexp, false);\n                ml.get_key(LLM_KV_EXPERT_GROUP_SCALE,                hparams.expert_group_scale);\n                ml.get_key(LLM_KV_EXPERTS_PER_GROUP,                 hparams.n_group_experts);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_30B_A3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_APERTUS:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_N,        hparams.xielu_alpha_n, hparams.n_layer);\n                ml.get_key_or_arr(LLM_KV_XIELU_ALPHA_P,        hparams.xielu_alpha_p, hparams.n_layer);\n                ml.get_key_or_arr(LLM_KV_XIELU_BETA,           hparams.xielu_beta,    hparams.n_layer);\n                ml.get_key_or_arr(LLM_KV_XIELU_EPS,            hparams.xielu_eps,     hparams.n_layer);\n\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_8B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MINIMAX_M2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,  hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,   hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,           hparams.expert_gating_func, false);\n\n                switch (hparams.n_layer) {\n                    case 62: type = LLM_TYPE_230B_A10B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_COGVLM:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 32: type = LLM_TYPE_13B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_PANGU_EMBED:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                switch (hparams.n_layer) {\n                    case 26: type = LLM_TYPE_1B; break; // openPangu-Embedded-1B-V1.1\n                    case 34: type = LLM_TYPE_7B; break; // openPangu-Embedded-7B-V1.1\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN3NEXT:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n\n                // Load linear attention (gated delta net) parameters\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                // Mark recurrent layers (linear attention layers)\n                {\n                    uint32_t full_attn_interval = 4;\n                    ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);\n                    for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                        hparams.recurrent_layer_arr[i] = ((i + 1) % full_attn_interval != 0);\n                    }\n                }\n\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_80B_A3B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN35:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS,    hparams.rope_sections, 4, true);\n\n                // Load linear attention (gated delta net) parameters\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                // Mark recurrent layers (linear attention layers)\n                {\n                    uint32_t full_attn_interval = 4;\n                    ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);\n                    for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                        hparams.recurrent_layer_arr[i] = ((i + 1) % full_attn_interval != 0);\n                    }\n                }\n\n                switch (hparams.n_layer) {\n                    case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_8B : LLM_TYPE_2B; break;\n                    case 32: type = hparams.n_embd == 2560 ? LLM_TYPE_4B : LLM_TYPE_9B; break;\n                    case 64: type = LLM_TYPE_27B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_QWEN35MOE:\n            {\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,       hparams.f_norm_rms_eps);\n\n                ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS,    hparams.rope_sections, 4, true);\n\n                // Load linear attention (gated delta net) parameters\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);\n                ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);\n                ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);\n                ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);\n\n                // Mark recurrent layers (linear attention layers)\n                {\n                    uint32_t full_attn_interval = 4;\n                    ml.get_key(LLM_KV_FULL_ATTENTION_INTERVAL, full_attn_interval, false);\n                    for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                        hparams.recurrent_layer_arr[i] = ((i + 1) % full_attn_interval != 0);\n                    }\n                }\n\n                switch (hparams.n_layer) {\n                    case 40: type = LLM_TYPE_35B_A3B; break;\n                    case 48: type = LLM_TYPE_122B_A10B; break;\n                    case 60: type = LLM_TYPE_397B_A17B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MISTRAL3:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false);\n\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, hparams.yarn_beta_fast,    false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow,    false);\n                ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL,   hparams.rope_yarn_log_mul, 0.0f);\n\n                hparams.f_attn_temp_offset = 0.0f;\n\n                // TODO: maybe add n_attn_temp_floor_scale as a separate KV?\n                if (hparams.f_attn_temp_scale != 0.0f) {\n                    hparams.n_attn_temp_floor_scale = hparams.n_ctx_orig_yarn;\n                    if (hparams.n_attn_temp_floor_scale == 0) {\n                        throw std::runtime_error(\"invalid n_ctx_orig_yarn for attention temperature scaling\");\n                    }\n                }\n\n                switch (hparams.n_layer) {\n                    case 26: type = LLM_TYPE_3B; break;\n                    case 34: type = LLM_TYPE_8B; break;\n                    case 40: type = LLM_TYPE_14B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_MIMO2:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,   hparams.n_swa);\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,         hparams.rope_freq_base_train_swa, false);\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer);\n\n                switch (hparams.n_layer) {\n                    case 48: type = LLM_TYPE_310B_A15B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_KIMI_LINEAR:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n                ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,    hparams.n_embd_head_k_mla_impl);\n                ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA,  hparams.n_embd_head_v_mla_impl);\n                ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,      hparams.n_lora_kv);\n                ml.get_key(LLM_KV_SSM_CONV_KERNEL,             hparams.ssm_d_conv);\n                ml.get_key(LLM_KV_KDA_HEAD_DIM,                hparams.n_embd_head_kda);\n\n                // MLA qk_rope_head_dim (for reference)\n                // qk_rope_head_dim = 64, qk_nope_head_dim = 128, qk_head_dim = 192\n\n                // Mark KDA layers as recurrent using n_head_kv pattern (like Jamba)\n                // Set n_head_kv = 0 for KDA layers (recurrent), n_head_kv = n_head for MLA layers (attention)\n                for (uint32_t i = 0; i < hparams.n_layer; ++i) {\n                    hparams.recurrent_layer_arr[i] = hparams.n_head_kv(i) == 0;  // KDA layers are recurrent\n                }\n\n                // MoE parameters - Kimi uses moe_intermediate_size = 1024\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared);\n                ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);\n\n                switch (hparams.n_layer) {\n                    case 27: type = LLM_TYPE_48B_A3B; break; // Kimi-Linear-48B-A3B\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        case LLM_ARCH_STEP35:\n            {\n                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);\n\n                hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;\n\n                // full_attention layer only use half of the RoPE dimensions\n                hparams.n_rot_full = hparams.n_rot_full / 2;\n\n                // MoE + SWA parameters\n                ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);\n                ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);\n                ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);\n                ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);\n\n                // Step35 uses sigmoid gating by default (if not set in GGUF)\n                if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {\n                    hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;\n                }\n\n                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,  hparams.n_swa);\n                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,        hparams.rope_freq_base_train_swa, false);\n                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer);\n                ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,   hparams.swiglu_clamp_exp,   hparams.n_layer, false);\n                ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer, false);\n\n                switch (hparams.n_layer) {\n                    case 45: type = LLM_TYPE_196B_A11B; break;\n                    default: type = LLM_TYPE_UNKNOWN;\n                }\n            } break;\n        default: throw std::runtime_error(\"unsupported model architecture: \" + arch_name());\n    }\n\n    pimpl->n_bytes = ml.n_bytes;\n\n    pimpl->desc_str = arch_name() + \" \" + type_name() + \" \" + ml.ftype_name();\n\n    if (hparams.f_max_alibi_bias > 0.0f) {\n        hparams.use_alibi = true;\n    }\n\n    hparams.rope_type = llama_model_rope_type(this);\n}\n\nvoid llama_model::load_vocab(llama_model_loader & ml) {\n    const auto kv = LLM_KV(arch);\n\n    vocab.load(ml, kv);\n}\n\nbool llama_model::load_tensors(llama_model_loader & ml) {\n    const auto & split_mode   = params.split_mode;\n    const auto & use_mlock    = params.use_mlock;\n    const auto & tensor_split = params.tensor_split;\n\n    const int n_layer      = hparams.n_layer;\n    const int n_gpu_layers = this->n_gpu_layers();\n\n    const bool use_mmap_buffer = true;\n\n    LLAMA_LOG_INFO(\"%s: loading model tensors, this can take a while... (mmap = %s, direct_io = %s)\\n\",\n        __func__, ml.use_mmap ? \"true\" : \"false\", ml.use_direct_io ? \"true\" : \"false\");\n\n    // build a list of buffer types for the CPU and GPU devices\n    pimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);\n    for (auto * dev : devices) {\n        buft_list_t buft_list = make_gpu_buft_list(dev, split_mode, tensor_split);\n        // add CPU buffer types as a fallback\n        buft_list.insert(buft_list.end(), pimpl->cpu_buft_list.begin(), pimpl->cpu_buft_list.end());\n        pimpl->gpu_buft_list.emplace(dev, std::move(buft_list));\n    }\n\n    ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n    if (cpu_dev == nullptr) {\n        throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n    }\n\n    // calculate the split points\n    bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + n_devices(), [](float x) { return x == 0.0f; });\n    std::vector<float> splits(n_devices());\n    if (all_zero) {\n        // default split, by free memory\n        for (size_t i = 0; i < n_devices(); ++i) {\n            ggml_backend_dev_t dev = devices[i];\n            size_t total;\n            size_t free;\n            ggml_backend_dev_memory(dev, &free, &total);\n\n            // devices can return 0 bytes for free and total memory if they do not\n            // have any to report. in this case, we will use the host memory as a fallback\n            // fixes: https://github.com/ggml-org/llama.cpp/issues/18577\n            if (free == 0 && total == 0) {\n                ggml_backend_dev_memory(cpu_dev, &free, &total);\n            }\n            splits[i] = free;\n        }\n    } else {\n        std::copy(tensor_split, tensor_split + n_devices(), splits.begin());\n    }\n\n    // sum and normalize the splits to get the split points\n    float split_sum = 0.0f;\n    for (size_t i = 0; i < n_devices(); ++i) {\n        split_sum += splits[i];\n        splits[i] = split_sum;\n    }\n    for (size_t i = 0; i < n_devices(); ++i) {\n        splits[i] /= split_sum;\n    }\n\n    const int i_gpu_start = std::max(int(hparams.n_layer) + 1 - n_gpu_layers, 0);\n    const int act_gpu_layers = devices.empty() ? 0 : std::min(n_gpu_layers, int(n_layer) + 1);\n    auto get_layer_buft_list = [&](int il) -> llama_model::impl::layer_dev {\n        const bool is_swa = il < int(hparams.n_layer) && hparams.is_swa(il);\n        if (il < i_gpu_start || (il - i_gpu_start) >= act_gpu_layers) {\n            LLAMA_LOG_DEBUG(\"load_tensors: layer %3d assigned to device %s, is_swa = %d\\n\", il, ggml_backend_dev_name(cpu_dev), is_swa);\n            return {cpu_dev, &pimpl->cpu_buft_list};\n        }\n        const int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + n_devices(), float(il - i_gpu_start)/act_gpu_layers) - splits.begin();\n        auto * dev = devices.at(layer_gpu);\n        LLAMA_LOG_DEBUG(\"load_tensors: layer %3d assigned to device %s, is_swa = %d\\n\", il, ggml_backend_dev_name(dev), is_swa);\n        return {dev, &pimpl->gpu_buft_list.at(dev)};\n    };\n\n    // assign the input layer\n    // there is very little benefit to offloading the input layer, so always keep it on the CPU\n    pimpl->dev_input = { cpu_dev, &pimpl->cpu_buft_list };\n\n    // assign the repeating layers to the devices according to the splits\n    pimpl->dev_layer.resize(n_layer);\n    for (int il = 0; il < n_layer; ++il) {\n        pimpl->dev_layer[il] = get_layer_buft_list(il);\n    }\n\n    // assign the output layer\n    pimpl->dev_output = get_layer_buft_list(n_layer);\n\n    const auto TENSOR_DUPLICATED      = llama_model_loader::TENSOR_DUPLICATED;\n    const auto TENSOR_NOT_REQUIRED    = llama_model_loader::TENSOR_NOT_REQUIRED;\n    const auto TENSOR_SKIP            = llama_model_loader::TENSOR_SKIP;\n    const auto TENSOR_SKIP_IF_VIRTUAL = llama_model_loader::TENSOR_SKIP_IF_VIRTUAL;\n\n    // create tensors for the weights\n    {\n        // note: cast to int64_t since we will use these for the tensor dimensions\n        const int64_t n_head        = hparams.n_head();\n        const int64_t n_head_kv     = hparams.n_head_kv();\n        const int64_t n_embd        = hparams.n_embd;\n        const int64_t n_embd_k_gqa  = hparams.n_embd_k_gqa();\n        const int64_t n_embd_v_gqa  = hparams.n_embd_v_gqa();\n        const int64_t n_embd_head_k = hparams.n_embd_head_k();\n        const int64_t n_embd_head_v = hparams.n_embd_head_v();\n        const int64_t n_ff          = hparams.n_ff();\n        const int64_t n_embd_gqa    = n_embd_v_gqa;\n        const int64_t n_vocab       = vocab.n_tokens();\n        const int64_t n_token_types = vocab.n_token_types();\n        const int64_t n_rot         = hparams.n_rot();\n        const int64_t n_expert      = hparams.n_expert;\n        const int64_t n_expert_used = hparams.n_expert_used;\n        const int64_t n_ctx_train   = hparams.n_ctx_train;\n\n        if (n_expert > 0 && hparams.n_expert_used == 0) {\n            throw std::runtime_error(\"model has expert layers but no expert layers are used\");\n        }\n\n        auto create_tensor = [&](const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) -> ggml_tensor * {\n            const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;\n            return ml.create_tensor(\n                hparams, &pimpl->cpu_buft_list, pimpl->dev_input.buft_list, pimpl->dev_output.buft_list, buft_list_layer,\n                tn, ne, flags);\n        };\n\n        layers.resize(n_layer);\n\n        // TODO: move to a separate function\n        const auto tn = LLM_TN(arch);\n\n        // helper: try merged gate_up_exps first, fall back to separate gate and up\n        auto create_tensor_gate_up_exps = [&](llama_layer & layer, int bid, int64_t n_embd_, int64_t n_ff_, int64_t n_expert_, int flags) {\n            layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, \"weight\", bid), {n_embd_, n_ff_ * 2, n_expert_}, TENSOR_NOT_REQUIRED);\n            if (layer.ffn_gate_up_exps == nullptr) {\n                layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", bid), {n_embd_, n_ff_, n_expert_}, flags);\n                layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", bid), {n_embd_, n_ff_, n_expert_}, flags);\n            }\n        };\n        switch (arch) {\n            case LLM_ARCH_LLAMA:\n            case LLM_ARCH_REFACT:\n            case LLM_ARCH_MINICPM:\n            case LLM_ARCH_GRANITE:\n            case LLM_ARCH_GRANITE_MOE:\n            case LLM_ARCH_MISTRAL3:\n            case LLM_ARCH_LLAMA_EMBED:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {\n                            layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n                        else {\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n\n                        if (n_expert == 0) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                            // optional MLP bias\n                            layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                        } else {\n                            layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff, n_expert}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff, n_expert}, 0);\n\n                            // For Granite MoE Shared\n                            if (hparams.n_ff_shexp > 0) {\n                                layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, hparams.n_ff_shexp}, 0);\n                                layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, hparams.n_ff_shexp}, 0);\n                                layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {hparams.n_ff_shexp, n_embd}, 0);\n                            }\n                        }\n                    }\n                } break;\n            case LLM_ARCH_LLADA:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output =\n                            create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        // Use separate Q, K, V projections without bias, matching LLaDALlamaBlock\n                        layer.wq =\n                            create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd_head_k * n_head }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_k_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_v_gqa }, 0);\n                        // No bias for QKV projections as per config: include_bias=false, include_qkv_bias=false\n                        layer.wo =\n                            create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), { n_embd }, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), { n_rot / 2 },\n                                                         TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), { n_embd, n_ff }, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), { n_ff, n_embd }, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i), { n_embd, n_ff }, 0);\n\n                        // optional MLP bias\n                        layer.ffn_gate_b =\n                            create_tensor(tn(LLM_TENSOR_FFN_GATE, \"bias\", i), { n_ff }, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_b =\n                            create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), { n_embd }, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, \"bias\", i), { n_ff }, TENSOR_NOT_REQUIRED);\n                    }\n                }\n                break;\n            case LLM_ARCH_LLADA_MOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    GGML_ASSERT(n_expert > 0 && \"n_expert must be > 0 for llada-moe\");\n                    GGML_ASSERT(n_expert_used > 0 && \"n_expert_used must be > 0 for llada-moe\");\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_LLAMA4:\n                {\n                    if (n_expert == 0) {\n                        throw std::runtime_error(arch_name() + \" model cannot have zero experts\");\n                    }\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        const bool is_moe_layer = hparams.n_moe_layer_step > 0 && (i + 1) % hparams.n_moe_layer_step == 0;\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n\n                        if (is_moe_layer) {\n                            const int64_t n_ff_exp = hparams.n_ff_exp;\n\n                            layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff_exp, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff_exp, n_expert}, 0);\n\n                            // Shared expert\n                            const int64_t n_ff_shexp = n_ff_exp;\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {    n_embd, n_ff_shexp}, 0);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd    }, 0);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {    n_embd, n_ff_shexp}, 0);\n                        } else {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_DECI:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n                        const int64_t n_embd_k_gqa  = hparams.n_embd_k_gqa(i);\n                        const int64_t n_embd_v_gqa  = hparams.n_embd_v_gqa(i);\n                        const int64_t n_embd_gqa    = hparams.n_embd_v_gqa(i);\n                        const int64_t n_ff          = hparams.n_ff(i);\n                        const int64_t n_head        = hparams.n_head(i);\n                        const int64_t n_head_kv     = hparams.n_head_kv(i);\n\n                        if (n_head_kv == 0 && n_head > 0) {\n                            // linear attention for DeciLMCausalModel\n                            layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        }\n                        else if (n_head_kv > 0) {\n                            layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n                        }\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n\n                        if (n_ff > 0) {\n                            layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        }\n\n                        if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {\n                            layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n                        else {\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n\n                        if (n_ff > 0) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        }\n\n                        // optional MLP bias\n                        layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_MINICPM3:\n                {\n                    const int64_t n_embd_head_qk_rope = hparams.n_rot();\n                    const int64_t n_embd_head_qk_nope = hparams.n_embd_head_k() - hparams.n_rot();\n\n                    const int64_t q_lora_rank  = hparams.n_lora_q;\n                    const int64_t kv_lora_rank = hparams.n_lora_kv;\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, \"weight\", i), {q_lora_rank}, 0);\n\n                        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, \"weight\", i), {kv_lora_rank}, 0);\n\n                        layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, \"weight\", i), {n_embd, q_lora_rank}, 0);\n                        layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, \"weight\", i), {q_lora_rank, n_head * n_embd_head_k}, 0);\n\n                        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, \"weight\", i), {n_embd, kv_lora_rank + (n_embd_head_qk_rope)}, 0);\n                        layer.wkv_b     = create_tensor(tn(LLM_TENSOR_ATTN_KV_B,     \"weight\", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)}, 0);\n                        layer.wo        = create_tensor(tn(LLM_TENSOR_ATTN_OUT,      \"weight\", i), {              n_head * (                      n_embd_head_v), n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                        layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), { n_embd_head_qk_rope/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), { n_embd_head_qk_rope/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                    }\n                } break;\n            case LLM_ARCH_GROK:\n                {\n                    if (n_expert == 0) {\n                        throw std::runtime_error(arch_name() + \" model cannot have zero experts\");\n                    }\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff/* / n_expert_used*/; // grok-1 n_ff_exp == n_ff\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.attn_out_norm   = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff,   n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp, n_embd,   n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff_exp, n_expert}, 0);\n\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        if (!layer.ffn_post_norm) {\n                            layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_DBRX:\n                {\n                    if (n_expert == 0) {\n                        throw std::runtime_error(\"DBRX model cannot have zero experts\");\n                    }\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_BAICHUAN:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n                    {\n                        output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_FALCON:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    {\n                        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n\n                        output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        if (!output) {\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED); // needs to be on GPU\n                        }\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.attn_norm_2   = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_STARCODER:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n                    pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD,   \"weight\"), {n_embd, n_ctx_train}, 0);\n\n                    // output\n                    {\n                        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                        output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        if (!output) {\n                            // needs to be on GPU\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                        }\n\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i),   {n_embd, n_ff}, 0);\n                        layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, \"bias\", i),     {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_BERT:\n            case LLM_ARCH_NOMIC_BERT:\n            case LLM_ARCH_NOMIC_BERT_MOE:\n            case LLM_ARCH_JINA_BERT_V3:\n                {\n                    if (n_token_types == 0) {\n                        throw std::runtime_error(arch_name() + \" model needs to define token type count\");\n                    }\n                    tok_embd     = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  \"weight\"), {n_embd, n_vocab}, 0);\n                    type_embd    = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, \"weight\"), {n_embd, n_token_types}, TENSOR_NOT_REQUIRED);\n\n                    if (arch == LLM_ARCH_BERT) {\n                        pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD,    \"weight\"), {n_embd, n_ctx_train}, 0);\n\n                        cls   = create_tensor(tn(LLM_TENSOR_CLS, \"weight\"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED);\n                        cls_b = create_tensor(tn(LLM_TENSOR_CLS, \"bias\"),   {n_embd},         TENSOR_NOT_REQUIRED);\n\n                        cls_out   = create_tensor(tn(LLM_TENSOR_CLS_OUT, \"weight\"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);\n                        cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, \"bias\"),   {hparams.n_cls_out},         TENSOR_NOT_REQUIRED);\n                    }\n\n                    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0);\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"),   {n_embd}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        if (!layer.wqkv) {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i),   {n_embd}, 0);\n\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i),   {n_embd_gqa}, 0);\n\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i),   {n_embd_gqa}, 0);\n                        }\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT,      \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT,      \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.attn_out_norm   = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        if (hparams.moe_every_n_layers > 0 && i % hparams.moe_every_n_layers == 1) {\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff,   n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff,   n_embd, n_expert}, 0);\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,   \"weight\", i), {n_embd, n_expert}, 0);\n                        } else {\n                            layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                            layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                            layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                            if (arch == LLM_ARCH_NOMIC_BERT) {\n                                layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                            }\n                        }\n\n                        layer.layer_out_norm   = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"bias\", i),   {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_MODERN_BERT:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n                    tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    for(int i = 0; i < n_layer; ++i) {\n                        auto& layer = layers[i];\n\n                        if ( i != 0 ) {\n                            layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        } else{\n                            // layer 0 uses identity\n                            layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        }\n\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, 3 * n_embd }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT,   \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, 2 * n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                    }\n\n                    cls_out   = create_tensor(tn(LLM_TENSOR_CLS_OUT,  \"weight\"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);\n                    cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT,  \"bias\"),   {hparams.n_cls_out},         TENSOR_NOT_REQUIRED);\n                    cls       = create_tensor(tn(LLM_TENSOR_CLS,      \"weight\"), {n_embd, n_embd},            TENSOR_NOT_REQUIRED);\n                    cls_norm  = create_tensor(tn(LLM_TENSOR_CLS_NORM, \"weight\"), {n_embd},                    TENSOR_NOT_REQUIRED);\n\n                } break;\n            case LLM_ARCH_NEO_BERT:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  \"weight\"), {n_embd, n_vocab}, 0);\n\n                    cls   = create_tensor(tn(LLM_TENSOR_CLS, \"weight\"), {n_embd, n_embd}, TENSOR_NOT_REQUIRED);\n                    cls_b = create_tensor(tn(LLM_TENSOR_CLS, \"bias\"),   {n_embd},         TENSOR_NOT_REQUIRED);\n\n                    cls_out   = create_tensor(tn(LLM_TENSOR_CLS_OUT, \"weight\"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);\n                    cls_out_b = create_tensor(tn(LLM_TENSOR_CLS_OUT, \"bias\"),   {hparams.n_cls_out},         TENSOR_NOT_REQUIRED);\n\n                    output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff*2}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_EUROBERT:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_JINA_BERT_V2:\n                {\n                    tok_embd  = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  \"weight\"), {n_embd, n_vocab}, 0); // word_embeddings\n                    type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, \"weight\"), {n_embd, n_token_types}, 0); // token_type_embeddings\n\n                    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0); // LayerNorm\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"),   {n_embd}, 0); //LayerNorm bias\n\n                    cls   = create_tensor(tn(LLM_TENSOR_CLS, \"weight\"), {n_embd, 1}, TENSOR_NOT_REQUIRED);\n                    cls_b = create_tensor(tn(LLM_TENSOR_CLS, \"bias\"),   {1},         TENSOR_NOT_REQUIRED);\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i]; // JinaBertLayer\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\", i),   {n_embd}, 0);\n\n                        layer.attn_q_norm   = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\",   i), {n_embd_gqa}, 0);\n\n                        layer.attn_k_norm   = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\",   i), {n_embd_gqa}, 0);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0); //output_dens\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\",   i), {n_embd}, 0); //output_dens\n\n                        layer.attn_out_norm   = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"weight\", i), {n_embd}, 0); //output_norm\n                        layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, \"bias\",   i), {n_embd}, 0);\n\n                        layer.attn_norm_2   = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED);\n\n                        const auto tn_ffn_up_weight = tn(LLM_TENSOR_FFN_UP, \"weight\", i);\n                        ggml_tensor * t_ffn_up = ml.get_tensor_meta(tn_ffn_up_weight.str().c_str());\n                        const int64_t n_ffn_up = t_ffn_up ? t_ffn_up->ne[1] : n_ff;\n\n                        GGML_ASSERT(n_ffn_up == n_ff || n_ffn_up == n_ff * 2);\n                        layer.ffn_up   = create_tensor(tn_ffn_up_weight, {n_embd, n_ffn_up}, 0);\n                        layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, \"bias\", i), {n_ffn_up}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\",   i), {n_embd}, 0);\n\n                        layer.layer_out_norm   = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"bias\",   i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_BLOOM:\n                {\n                    tok_embd   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,      \"weight\"), {n_embd, n_vocab}, 0);\n                    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0);\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"),   {n_embd}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\",   i), {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\",   i), {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\",   i), {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\",   i), {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\",   i), {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP, \"bias\",   i), {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_MPT:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n                    pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD,   \"weight\"), {n_embd, n_ctx_train}, TENSOR_NOT_REQUIRED);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (!output) {\n                        output    = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED); // needs to be on GPU\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, TENSOR_NOT_REQUIRED);\n\n                        // FIXME test-llama-archs crashes if q_norm is created\n                        layer.attn_q_norm   = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);\n                        layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);\n\n                        layer.attn_k_norm   = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        // AWQ ScaleActivation layer\n                        layer.ffn_act = create_tensor(tn(LLM_TENSOR_FFN_ACT, \"scales\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_STABLELM:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm =   create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        // optional bias tensors, present in Stable LM 2 1.6B\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        // optional q and k layernorms, present in StableLM 2 12B\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k, n_head},    TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k, n_head_kv}, TENSOR_NOT_REQUIRED);\n\n                        // optional FFN norm, not present in StableLM 2 12B which uses parallel residual\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd*3}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd*3}, 0);\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff/2}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff/2, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff/2}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN2:\n            case LLM_ARCH_QWEN2VL:\n            case LLM_ARCH_DREAM:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    output_b    = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"bias\"),   {n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN2MOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        if (n_expert == 0) {\n                            throw std::runtime_error(\"n_expert must be > 0 for QWEN2MOE\");\n                        }\n                        if (n_expert_used == 0) {\n                            throw std::runtime_error(\"n_expert_used must be > 0 for QWEN2MOE\");\n                        }\n\n                        // MoE branch\n                        const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                        // Shared expert branch\n                        const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;\n\n                        layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {    n_embd, n_ff_shexp}, 0);\n                        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp,     n_embd}, 0);\n                        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {    n_embd, n_ff_shexp}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN3:\n            case LLM_ARCH_QWEN3VL:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    // output rerank head\n                    cls_out = create_tensor(tn(LLM_TENSOR_CLS_OUT, \"weight\"), {n_embd, hparams.n_cls_out}, TENSOR_NOT_REQUIRED);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN3MOE:\n            case LLM_ARCH_QWEN3VLMOE:\n            case LLM_ARCH_RND1:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        if (n_expert == 0) {\n                            throw std::runtime_error(\"n_expert must be > 0 for QWEN3MOE\");\n                        }\n                        if (n_expert_used == 0) {\n                            throw std::runtime_error(\"n_expert_used must be > 0 for QWEN3MOE\");\n                        }\n\n                        // MoE branch\n                        const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_PHI2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n                    output_b      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"bias\"),   {n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        if (layer.wqkv == nullptr) {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\", i),   {n_embd}, 0);\n\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\", i),   {n_embd_gqa}, 0);\n\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\", i),   {n_embd_gqa}, 0);\n                        }\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_PHI3:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), { n_embd, n_embd + 2 * n_embd_gqa }, TENSOR_NOT_REQUIRED);\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd, n_embd }, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), { n_ff, n_embd }, 0);\n                        layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i), { n_embd, 2 * n_ff }, 0);\n\n                        layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                    }\n                } break;\n            case LLM_ARCH_PHIMOE:\n                {\n                    const int64_t n_embd_head = n_embd / n_head;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), { n_embd, n_vocab }, 0);\n                    output_b      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"bias\"),   { n_vocab }, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\",   i), { n_embd }, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), { n_embd, n_embd + 2 * n_embd_gqa }, TENSOR_NOT_REQUIRED);\n                        if (layer.wqkv == nullptr) {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\",   i), {n_embd}, 0);\n\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\",   i), {n_embd_gqa}, 0);\n\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\",   i), {n_embd_gqa}, 0);\n                        }\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd, n_embd }, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\",   i), { n_embd }, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), { n_embd }, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\",   i), { n_embd }, 0);\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert},         0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n\n                        layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), { n_embd_head/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                     }\n                } break;\n            case LLM_ARCH_PLAMO:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_PLAMO2:\n                {\n                    // mamba parameters\n                    const uint32_t d_conv             = hparams.ssm_d_conv;\n                    const uint32_t d_state            = hparams.ssm_d_state;\n                    const uint32_t num_heads          = hparams.ssm_dt_rank;\n                    const uint32_t intermediate_size  = hparams.ssm_d_inner;\n                    const int64_t dt_dim              = std::max(64, int(hparams.n_embd / 16));\n\n                    // attention parameters\n                    const uint32_t qk_dim = hparams.n_embd_head_k();\n                    const uint32_t v_dim  = hparams.n_embd_head_v();\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n                        bool is_mamba_layer = hparams.is_recurrent(i);\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (is_mamba_layer) {\n                            layer.ssm_in       = create_tensor(tn(LLM_TENSOR_SSM_IN,     \"weight\", i), {n_embd, 2 * intermediate_size}, 0);\n                            layer.ssm_conv1d   = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, intermediate_size}, 0);\n\n                            layer.ssm_x    = create_tensor(tn(LLM_TENSOR_SSM_X,  \"weight\", i), {intermediate_size, dt_dim + 2*d_state}, 0);\n                            layer.ssm_dt   = create_tensor(tn(LLM_TENSOR_SSM_DT, \"weight\", i), {dt_dim, num_heads}, 0);\n                            layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {num_heads}, 0);\n\n                            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {num_heads}, 0);\n                            layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {num_heads}, 0);\n\n                            layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {intermediate_size, n_embd}, 0);\n\n                            layer.ssm_dt_norm = create_tensor(tn(LLM_TENSOR_SSM_DT_NORM, i), {dt_dim}, 0);\n                            layer.ssm_b_norm = create_tensor(tn(LLM_TENSOR_SSM_B_NORM, i), {d_state}, 0);\n                            layer.ssm_c_norm = create_tensor(tn(LLM_TENSOR_SSM_C_NORM, i), {d_state}, 0);\n                        } else {\n                            const int64_t num_attention_heads = hparams.n_head(i);\n                            const int64_t q_num_heads         = num_attention_heads;\n                            const int64_t num_key_value_heads = hparams.n_head_kv(i);\n                            const int64_t k_num_heads         = num_key_value_heads;\n                            const int64_t v_num_heads         = num_key_value_heads;\n                            const int64_t q_proj_dim          = q_num_heads * qk_dim;\n                            const int64_t k_proj_dim          = k_num_heads * qk_dim;\n                            const int64_t v_proj_dim          = v_num_heads * v_dim;\n\n                            layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, q_proj_dim + k_proj_dim + v_proj_dim}, 0);\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {qk_dim, num_attention_heads}, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {qk_dim, k_num_heads}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {q_num_heads * v_dim, n_embd}, 0);\n                        }\n\n                        // All layers have post-attention norm, FFN norm, and FFN tensors\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, i), {n_embd}, 0);\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff * 2}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_PLAMO3:\n                {\n                    const int64_t head_dim_q = hparams.n_embd_head_k();\n                    const int64_t head_dim_v = hparams.n_embd_head_v();\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        const int64_t num_attention_heads = hparams.n_head(i);\n                        const int64_t num_key_value_heads = hparams.n_head_kv(i);\n                        const int64_t q_proj_dim = num_attention_heads * head_dim_q;\n                        const int64_t k_proj_dim = num_key_value_heads * head_dim_q;\n                        const int64_t v_proj_dim = num_key_value_heads * head_dim_v;\n                        const int64_t n_ff_cur   = hparams.n_ff(i);\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i),\n                                {n_embd,q_proj_dim + k_proj_dim + v_proj_dim}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {head_dim_q}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {head_dim_q}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {num_attention_heads * head_dim_v, n_embd}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, i), {n_embd}, 0);\n\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff_cur * 2}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff_cur, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GPT2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n                    pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD,   \"weight\"), {n_embd, n_ctx_train}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM,   \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM,   \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_CODESHELL:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if tok embd is NULL, init from output\n                    if (tok_embd == NULL) {\n                        tok_embd = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i),   {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP, \"bias\", i),     {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_ORION:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_INTERNLM2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        // layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GEMMA:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GEMMA2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GEMMA3:\n            case LLM_ARCH_GEMMA_EMBEDDING:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,   \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    // Dense linear weights\n                    dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, \"weight\"), {n_embd, hparams.dense_2_feat_out}, TENSOR_NOT_REQUIRED);\n                    dense_3_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_3_OUT, \"weight\"), {hparams.dense_3_feat_in, n_embd}, TENSOR_NOT_REQUIRED);\n\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_k_norm    = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM,    \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm    = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM,    \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GEMMA3N:\n                {\n                    const int64_t n_altup      = hparams.n_altup;\n                    const int64_t laurel_rank  = hparams.laurel_rank;\n                    const int64_t n_embd_altup = hparams.n_embd_altup;\n\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    tok_embd           = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,           \"weight\"), {n_embd, n_vocab}, 0);\n                    tok_embd_per_layer = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, \"weight\"), {n_embd_altup * n_layer, n_vocab}, 0);\n\n                    altup_proj           = create_tensor(tn(LLM_TENSOR_ALTUP_PROJ,           \"weight\"), {n_embd, n_embd, n_altup - 1}, 0);\n                    altup_unembd_proj    = create_tensor(tn(LLM_TENSOR_ALTUP_UNEMBD_PROJ,    \"weight\"), {n_embd, n_embd, n_altup - 1}, 0);\n                    per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, \"weight\"), {n_embd, n_embd_altup * n_layer}, 0);\n                    per_layer_proj_norm  = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM,  \"weight\"), {n_embd_altup}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_q_norm    = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM,    \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_k_norm    = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM,    \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // altup & laurel\n                        layer.per_layer_inp_gate   = create_tensor(tn(LLM_TENSOR_PER_LAYER_INP_GATE,  \"weight\", i), {n_embd, n_embd_altup}, 0);\n                        layer.per_layer_proj       = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ,      \"weight\", i), {n_embd_altup, n_embd}, 0);\n                        layer.per_layer_post_norm  = create_tensor(tn(LLM_TENSOR_PER_LAYER_POST_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.altup_correct_coef   = create_tensor(tn(LLM_TENSOR_ALTUP_CORRECT_COEF,  \"weight\", i), {n_altup, n_altup}, 0);\n                        layer.altup_correct_scale  = create_tensor(tn(LLM_TENSOR_ALTUP_CORRECT_SCALE, \"weight\", i), {n_embd}, 0);\n                        layer.altup_predict_coef   = create_tensor(tn(LLM_TENSOR_ALTUP_PREDICT_COEF,  \"weight\", i), {n_altup, n_altup * n_altup}, 0);\n                        layer.altup_router         = create_tensor(tn(LLM_TENSOR_ALTUP_ROUTER,        \"weight\", i), {n_embd, n_altup}, 0);\n                        layer.altup_router_norm    = create_tensor(tn(LLM_TENSOR_ALTUP_ROUTER_NORM,   \"weight\", i), {n_embd}, 0);\n                        layer.laurel_l             = create_tensor(tn(LLM_TENSOR_LAUREL_L,            \"weight\", i), {n_embd, laurel_rank}, 0);\n                        layer.laurel_r             = create_tensor(tn(LLM_TENSOR_LAUREL_R,            \"weight\", i), {laurel_rank, n_embd}, 0);\n                        layer.laurel_post_norm     = create_tensor(tn(LLM_TENSOR_LAUREL_POST_NORM,    \"weight\", i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_STARCODER2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd}, 0);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, 0);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                        // optional bias tensors\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {n_embd}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP ,  \"bias\", i), {  n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_MAMBA:\n                {\n                    const int64_t d_conv  = hparams.ssm_d_conv;\n                    const int64_t d_inner = hparams.ssm_d_inner;\n                    const int64_t d_state = hparams.ssm_d_state;\n                    const int64_t dt_rank = hparams.ssm_dt_rank;\n\n                    // only an expansion factor of 2 is supported for now\n                    if (2 * n_embd != d_inner) {\n                        throw std::runtime_error(\"only an expansion factor of 2 is supported for now\");\n                    }\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed, duplicated to allow offloading\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        // norm\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {n_embd, 2*d_inner}, 0);\n\n                        layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, d_inner}, 0);\n                        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {d_inner}, 0);\n\n                        layer.ssm_x = create_tensor(tn(LLM_TENSOR_SSM_X, \"weight\", i), {d_inner, dt_rank + 2*d_state}, 0);\n\n                        layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, \"weight\", i), {dt_rank, d_inner}, 0);\n                        layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {d_inner}, 0);\n\n                        // no \"weight\" suffix for these\n                        layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {d_state, d_inner}, 0);\n                        layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {d_inner}, 0);\n\n                        // out_proj\n                        layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {d_inner, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_MAMBA2:\n                {\n                    const int64_t d_conv  = hparams.ssm_d_conv;\n                    const int64_t d_inner = hparams.ssm_d_inner;\n                    const int64_t d_state = hparams.ssm_d_state;\n                    const int64_t n_head  = hparams.ssm_dt_rank;\n                    const int64_t n_group = hparams.ssm_n_group;\n                    const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_head;\n\n                    // only an expansion factor of 2 is supported for now\n                    GGML_ASSERT(2 * n_embd == d_inner);\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    {\n                        output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                        output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        // if output is NULL, init from the input tok embed, duplicated to allow offloading\n                        if (output == NULL) {\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                        }\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        // norm\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {n_embd, d_in_proj}, 0);\n\n                        layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, d_inner + 2*n_group*d_state}, 0);\n                        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {d_inner + 2*n_group*d_state}, 0);\n\n                        layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {n_head}, 0);\n\n                        // no \"weight\" suffix for these\n                        layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);\n                        layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, n_head}, 0);\n\n                        layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, \"weight\", i), {d_inner / n_group, n_group}, 0);\n\n                        // out_proj\n                        layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {d_inner, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_JAMBA:\n                {\n                    const int64_t d_conv  = hparams.ssm_d_conv;\n                    const int64_t d_inner = hparams.ssm_d_inner;\n                    const int64_t d_state = hparams.ssm_d_state;\n                    const int64_t dt_rank = hparams.ssm_dt_rank;\n\n                    // only an expansion factor of 2 is supported for now\n                    GGML_ASSERT(2 * n_embd == d_inner);\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    {\n                        output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                        output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        // if output is NULL, init from the input tok embed, duplicated to allow offloading\n                        if (output == NULL) {\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                        }\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        const int64_t n_head_kv = hparams.n_head_kv(i);\n                        const int64_t n_embd_gqa = hparams.n_embd_v_gqa(i);\n\n                        auto & layer = layers[i];\n\n                        // norm\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (n_head_kv == 0) {\n                            // Mamba layer\n                            layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {n_embd, 2*d_inner}, 0);\n\n                            layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, d_inner}, 0);\n                            layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {d_inner}, 0);\n\n                            layer.ssm_x = create_tensor(tn(LLM_TENSOR_SSM_X, \"weight\", i), {d_inner, dt_rank + 2*d_state}, 0);\n\n                            layer.ssm_dt_norm = create_tensor(tn(LLM_TENSOR_SSM_DT_NORM, \"weight\", i), {dt_rank}, 0);\n\n                            layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, \"weight\", i), {dt_rank, d_inner}, 0);\n                            layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {d_inner}, 0);\n\n                            layer.ssm_b_norm = create_tensor(tn(LLM_TENSOR_SSM_B_NORM, \"weight\", i), {d_state}, 0);\n                            layer.ssm_c_norm = create_tensor(tn(LLM_TENSOR_SSM_C_NORM, \"weight\", i), {d_state}, 0);\n\n                            // no \"weight\" suffix for these\n                            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {d_state, d_inner}, 0);\n                            layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {d_inner}, 0);\n\n                            // out_proj\n                            layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {d_inner, n_embd}, 0);\n                        } else {\n                            // Attention layers\n\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        }\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);\n\n                        if (layer.ffn_gate_inp) {\n                            // MoE\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff, n_expert}, 0);\n                        } else {\n                            // FFN (no MoE)\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_GRANITE_HYBRID:\n                {\n                    // mamba2 Mixer SSM params\n                    // NOTE: int64_t for tensor dimensions\n                    const int64_t d_conv     = hparams.ssm_d_conv;\n                    const int64_t d_inner    = hparams.ssm_d_inner;\n                    const int64_t d_state    = hparams.ssm_d_state;\n                    const int64_t n_ssm_head = hparams.ssm_dt_rank;\n                    const int64_t n_group    = hparams.ssm_n_group;\n                    const int64_t d_in_proj  = 2*d_inner + 2*n_group*d_state + n_ssm_head;\n\n                    // only an expansion factor of 2 is supported for now\n                    GGML_ASSERT(2 * n_embd == d_inner);\n\n                    // embeddings\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    {\n                        output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        // if output is NULL, init from the input tok embed, duplicated to allow offloading\n                        if (output == NULL) {\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                        }\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        // norm\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (hparams.is_recurrent(i)) {\n                            // ssm layers\n                            layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {n_embd, d_in_proj}, 0);\n\n                            layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, d_inner + 2*n_group*d_state}, 0);\n                            layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {d_inner + 2*n_group*d_state}, TENSOR_NOT_REQUIRED);\n\n                            layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {n_ssm_head}, 0);\n\n                            // no \"weight\" suffix for these\n                            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_ssm_head}, 0);\n                            layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, n_ssm_head}, 0);\n\n                            layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, \"weight\", i), {d_inner / n_group, n_group}, 0);\n\n                            // out_proj\n                            layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {d_inner, n_embd}, 0);\n                        } else {\n                            // attention layers (with optional bias)\n                            const int64_t n_head_i = hparams.n_head(i);\n                            const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);\n                            const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head_i}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa_i}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa_i}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head_i, n_embd}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},         TENSOR_NOT_REQUIRED);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);\n                            layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd},         TENSOR_NOT_REQUIRED);\n                        }\n\n                        // feed forward (w/ optional biases)\n                        if (n_expert > 0) {\n                            // MoE FFN\n                            layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff, n_expert}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff, n_expert}, 0);\n\n                            // For Granite MoE Shared\n                            if (hparams.n_ff_shexp > 0) {\n                                layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, hparams.n_ff_shexp}, 0);\n                                layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, hparams.n_ff_shexp}, 0);\n                                layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {hparams.n_ff_shexp, n_embd}, 0);\n                            }\n                        } else {\n                            layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_XVERSE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_COMMAND_R:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // init output from the input tok embed\n                    output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (n_layer >= 64){\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k, n_head}, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k, n_head_kv}, 0);\n                        }\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_COHERE2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    // init output from the input tok embed\n                    output      = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab },\n                                                      TENSOR_DUPLICATED);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd, n_embd }, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), { n_embd, n_ff }, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), { n_ff, n_embd }, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i), { n_embd, n_ff }, 0);\n                    }\n                }\n                break;\n            case LLM_ARCH_OLMO:  // adapted from LLM_ARCH_LLAMA with norm params removed\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_OLMO2:\n                {\n                    const int64_t n_embd_head = n_embd / n_head;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_head_kv * n_embd_head}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_SEED_OSS:\n                {\n                    const uint32_t head_dim             = hparams.n_embd_head_k();\n                    const int64_t n_qo_dim              = n_head * head_dim;\n                    const int64_t n_kv_dim              = n_head_kv * head_dim;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_qo_dim}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_kv_dim}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_kv_dim}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_qo_dim, n_embd}, 0);\n\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_qo_dim},   TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_kv_dim},   TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_kv_dim},   TENSOR_NOT_REQUIRED);\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                    }\n                } break;\n\n            case LLM_ARCH_OLMOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        if (n_expert == 0) {\n                            throw std::runtime_error(\"n_expert must be > 0\");\n                        }\n                        if (n_expert_used == 0) {\n                            throw std::runtime_error(\"n_expert_used must be > 0\");\n                        }\n\n                        // MoE branch\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_OPENELM:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // init output from the input tok embed\n                    output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        const int64_t n_head      =   hparams.n_head(i);\n                        const int64_t n_head_qkv  = 2*hparams.n_head_kv(i) + n_head;\n                        const int64_t n_ff        =   hparams.n_ff(i);\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_head_qkv*n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head*n_embd_head_k, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GPTNEOX:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_ARCTIC:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                        layer.ffn_norm_exps = create_tensor(tn(LLM_TENSOR_FFN_NORM_EXPS, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff, n_expert}, false);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff, n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff, n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_DEEPSEEK:\n                {\n\n                    const int64_t n_ff_exp        = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // try to load output.weight, if not found, use token_embd (tied embeddings)\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (!output) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (i < (int) hparams.n_layer_dense_lead) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        } else {\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                            if (n_expert == 0) {\n                                throw std::runtime_error(\"n_expert must be > 0\");\n                            }\n                            if (n_expert_used == 0) {\n                                throw std::runtime_error(\"n_expert_used must be > 0\");\n                            }\n\n                            // MoE branch\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                            // Shared expert branch\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {        n_ff_exp * n_expert_shared, n_embd}, 0);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_DEEPSEEK2:\n                {\n                    const bool is_mla = hparams.is_mla();\n\n                    // note: these are the actual head sizes you get when treating as MHA or after \"decompression\" using wv_b for MLA\n                    const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();\n                    const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();\n\n                    const int64_t n_embd_head_qk_rope = hparams.n_rot();\n                    const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;\n                    GGML_ASSERT(n_embd_head_qk_nope >= 1);\n\n                    const int64_t q_lora_rank  = hparams.n_lora_q;\n                    const int64_t kv_lora_rank = hparams.n_lora_kv;\n\n                    const int64_t n_ff_exp        = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // try to load output.weight, if not found, use token_embd (tied embeddings)\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (!output) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        if (q_lora_rank > 0) {\n                            layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, \"weight\", i), {q_lora_rank}, 0);\n                        }\n\n                        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, \"weight\", i), {kv_lora_rank}, 0);\n\n                        if (q_lora_rank > 0) {\n                            layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, \"weight\", i), {n_embd, q_lora_rank}, 0);\n                            layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, \"weight\", i), {q_lora_rank, n_head * n_embd_head_k_mla}, 0);\n                        } else {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_head * n_embd_head_k_mla}, 0);\n                        }\n\n                        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, \"weight\", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, 0);\n\n                        // note: only old legacy GGUF files will have the unsplit wkv_b tensor in\n                        if (is_mla) {\n                            layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, \"weight\", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, 0);\n                            layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, \"weight\", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0);\n                        } else {\n                            layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, \"weight\", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v_mla)}, 0);\n                        }\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head * n_embd_head_v_mla, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (i < (int) hparams.n_layer_dense_lead) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        } else {\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n\n                            if (n_expert == 0) {\n                                throw std::runtime_error(\"n_expert must be > 0\");\n                            }\n                            if (n_expert_used == 0) {\n                                throw std::runtime_error(\"n_expert_used must be > 0\");\n                            }\n\n                            // MoE branch\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                            create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);\n\n                            // Shared expert branch\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {        n_ff_exp * n_expert_shared, n_embd}, 0);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_PLM:\n                {\n                    const int64_t n_embd_head_qk_rope = hparams.n_rot();\n                    const int64_t n_embd_head_qk_nope = hparams.n_embd_head_k() - hparams.n_rot();\n                    const int64_t kv_lora_rank = hparams.n_lora_kv;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n                    output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq        = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, \"weight\", i), {n_embd, kv_lora_rank + (n_embd_head_qk_rope)}, 0);\n                        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, \"weight\", i), {kv_lora_rank}, 0);\n                        layer.wkv_b     = create_tensor(tn(LLM_TENSOR_ATTN_KV_B,     \"weight\", i), {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)}, 0);\n                        layer.wo        = create_tensor(tn(LLM_TENSOR_ATTN_OUT,      \"weight\", i), {              n_head * (                      n_embd_head_v), n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_BITNET:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm     = create_tensor(tn(LLM_TENSOR_ATTN_NORM,     \"weight\", i), {n_embd}, 0);\n                        layer.attn_sub_norm = create_tensor(tn(LLM_TENSOR_ATTN_SUB_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq       = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wq_s     = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                        layer.wk       = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wk_s     = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                        layer.wv       = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv_s     = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                        layer.wo       = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wo_s     = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm     = create_tensor(tn(LLM_TENSOR_FFN_NORM,     \"weight\", i), {n_embd}, 0);\n                        layer.ffn_sub_norm = create_tensor(tn(LLM_TENSOR_FFN_SUB_NORM, \"weight\", i), {n_ff}, 0);\n\n                        layer.ffn_gate       = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_gate_s = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down       = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up         = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_s   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"scale\",  i), {1}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_T5:\n                {\n                    const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm     = create_tensor(tn(LLM_TENSOR_DEC_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    // n_layer:     number of encoder_layers\n                    // dec_n_layer: number of decoder_layers\n                    const int dec_n_layer = hparams.dec_n_layer;\n                    if (dec_n_layer > n_layer) {\n                        layers.resize(dec_n_layer);\n                    }\n\n                    // load encoder layers\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm_enc  = create_tensor(tn(LLM_TENSOR_ENC_ATTN_NORM,  \"weight\", i), {n_embd}, 0);\n                        layer.attn_rel_b_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_REL_B, \"weight\", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED);\n\n                        layer.wq_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_Q,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wk_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_OUT, \"weight\", i), {n_embd_v_gqa, n_embd}, 0);\n\n                        layer.ffn_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up_enc   = create_tensor(tn(LLM_TENSOR_ENC_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n\n                    // load decoder layers\n                    for (int i = 0; i < dec_n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm  = create_tensor(tn(LLM_TENSOR_DEC_ATTN_NORM,  \"weight\", i), {n_embd}, 0);\n                        layer.attn_rel_b = create_tensor(tn(LLM_TENSOR_DEC_ATTN_REL_B, \"weight\", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_DEC_ATTN_Q,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_DEC_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_DEC_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_DEC_ATTN_OUT, \"weight\", i), {n_embd_v_gqa, n_embd}, 0);\n\n                        layer.attn_norm_cross  = create_tensor(tn(LLM_TENSOR_DEC_CROSS_ATTN_NORM,  \"weight\", i), {n_embd}, 0);\n                        // this tensor seems to be unused in HF transformers implementation\n                        layer.attn_rel_b_cross = create_tensor(\n                            tn(LLM_TENSOR_DEC_CROSS_ATTN_REL_B, \"weight\", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);\n\n                        layer.wq_cross = create_tensor(tn(LLM_TENSOR_DEC_CROSS_ATTN_Q,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wk_cross = create_tensor(tn(LLM_TENSOR_DEC_CROSS_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv_cross = create_tensor(tn(LLM_TENSOR_DEC_CROSS_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo_cross = create_tensor(tn(LLM_TENSOR_DEC_CROSS_ATTN_OUT, \"weight\", i), {n_embd_v_gqa, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_DEC_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_DEC_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_DEC_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_DEC_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_T5ENCODER:\n                {\n                    const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm_enc  = create_tensor(tn(LLM_TENSOR_ENC_ATTN_NORM,  \"weight\", i), {n_embd}, 0);\n                        layer.attn_rel_b_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_REL_B, \"weight\", i), {n_head, n_rel_attn_bkts}, TENSOR_NOT_REQUIRED);\n\n                        layer.wq_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_Q,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wk_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo_enc = create_tensor(tn(LLM_TENSOR_ENC_ATTN_OUT, \"weight\", i), {n_embd_v_gqa, n_embd}, 0);\n\n                        layer.ffn_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_enc = create_tensor(tn(LLM_TENSOR_ENC_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up_enc   = create_tensor(tn(LLM_TENSOR_ENC_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_JAIS:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM,   \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM,   \"bias\", i),   {n_embd}, 0);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, 0);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, 0);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n\n                        layer.ffn_gate   = create_tensor(tn(LLM_TENSOR_FFN_GATE,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE,   \"bias\", i),   {n_ff}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_JAIS2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (!output) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        // attention biases - all have shape n_embd (output dimension of projections)\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\", i), {n_embd}, 0);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\", i), {n_embd}, 0);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\", i), {n_embd}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        // Jais-2 uses simple MLP (no gate) with biases\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i),   {n_ff}, 0);\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i),   {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_CHATGLM:\n                {\n                    tok_embd   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);\n\n                        if (layer.wqkv == nullptr) {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        }\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff * 2}, 0);\n\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_GLM4:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        int flags = 0;\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            // skip all tensors in the NextN layers\n                            flags |= TENSOR_SKIP;\n                        }\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, flags);\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);\n                        layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"bias\", i),   {n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);\n\n                        if (layer.wqkv == nullptr) {\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd_head_k * n_head}, flags);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_k_gqa}, flags);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_v_gqa}, flags);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\", i), {n_embd}, flags | TENSOR_NOT_REQUIRED);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);\n                        }\n\n                        layer.wo   = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, flags);\n\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, flags);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, flags);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, flags);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff * 2}, flags);\n\n                        layer.ffn_post_norm  = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, flags);\n\n                        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, \"weight\", i), { 2 * n_embd, n_embd }, flags);\n                            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, \"weight\", i), { n_embd }, flags);\n                            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, \"weight\", i), { n_embd }, flags);\n\n                            // Optional tensors\n                            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, \"weight\", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_GLM4_MOE:\n                {\n                    const int64_t n_expert        = hparams.n_expert;\n                    const int64_t n_expert_used   = hparams.n_expert_used;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    GGML_ASSERT(hparams.n_expert > 0 && \"n_expert must be > 0 for GLM4_MOE MoE layers\");\n                    GGML_ASSERT(hparams.n_expert_used > 0 && \"n_expert_used must be > 0 for GLM4_MOE MoE layers\");\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, TENSOR_DUPLICATED);\n                    }\n\n                    // Load ALL tensors including NextN layer to satisfy total tensor count\n                    // but only PROCESS up to last layer (skipping final NextN layer) in forward pass\n                    for (int i = 0; i < n_layer; ++i) {\n                        int flags = 0;\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            // skip all tensors in the NextN layers\n                            flags |= TENSOR_SKIP;\n                        }\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, flags);\n\n                        // GLM-style attention with bias terms\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd_head_k * n_head }, flags);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_k_gqa }, flags);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_v_gqa }, flags);\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"bias\", i), { n_embd_head_k * n_head }, TENSOR_NOT_REQUIRED | flags);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"bias\", i), { n_embd_k_gqa }, TENSOR_NOT_REQUIRED | flags);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"bias\", i), { n_embd_v_gqa }, TENSOR_NOT_REQUIRED | flags);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, flags);\n\n                        // K/Q norm tensors (optional for GLM-4.5 355B variant)\n                        layer.attn_q_norm = create_tensor(\n                            tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED | flags);\n                        layer.attn_k_norm = create_tensor(\n                            tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), { n_embd_head_k }, TENSOR_NOT_REQUIRED | flags);\n\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), { n_embd }, flags);\n\n                        // Check if this layer uses MoE or dense FFN based on n_layer_dense_lead\n                        // GLM 4.5 uses hybrid architecture: layer 0 is dense, layers 1+ are MoE\n                        const bool use_moe = (static_cast<uint32_t>(i) >= hparams.n_layer_dense_lead);\n\n                        if (use_moe) {\n                            // MoE layers\n                            layer.ffn_gate_inp =\n                                create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), { n_embd, n_expert }, flags);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), { n_expert }, flags);\n\n                            // MoE branch\n                            const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                            layer.ffn_gate_exps = create_tensor(\n                                tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), { n_embd, n_ff_exp, n_expert }, flags);\n                            layer.ffn_down_exps = create_tensor(\n                                tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), { n_ff_exp, n_embd, n_expert }, flags);\n                            layer.ffn_up_exps = create_tensor(\n                                tn(LLM_TENSOR_FFN_UP_EXPS, \"weight\", i), { n_embd, n_ff_exp, n_expert }, flags);\n\n                            // Shared expert\n                            if (n_expert_shared > 0) {\n                                const int64_t n_ff_shexp = n_ff_exp * n_expert_shared;\n                                layer.ffn_gate_shexp = create_tensor(\n                                    tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), { n_embd, n_ff_shexp }, flags);\n                                layer.ffn_down_shexp = create_tensor(\n                                    tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), { n_ff_shexp, n_embd }, flags);\n                                layer.ffn_up_shexp = create_tensor(\n                                    tn(LLM_TENSOR_FFN_UP_SHEXP, \"weight\", i), { n_embd, n_ff_shexp }, flags);\n                            }\n                        } else {\n                            // Dense layers (first k layers) - GLM uses separate gate/up projections\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), { n_embd, n_ff }, flags);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), { n_ff, n_embd }, flags);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), { n_embd, n_ff }, flags);\n                        }\n\n                        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, \"weight\", i), { 2 * n_embd, n_embd }, flags);\n                            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, \"weight\", i), { n_embd }, flags);\n                            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, \"weight\", i), { n_embd }, flags);\n\n                            // Optional tensors\n                            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, \"weight\", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);\n                        }\n                    }\n                }\n                break;\n            case LLM_ARCH_GLM_DSA:\n                {\n                    const bool is_mla = hparams.is_mla();\n                    if (!is_mla) {\n                        throw std::runtime_error(\"GLM_DSA architecture requires MLA\");\n                    }\n\n                    // note: these are the actual head sizes you get when treating as MHA or after \"decompression\" using wv_b for MLA\n                    const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();\n                    const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();\n\n                    const int64_t n_embd_head_qk_rope = hparams.n_rot();\n                    const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;\n\n                    const int64_t q_lora_rank  = hparams.n_lora_q;\n                    const int64_t kv_lora_rank = hparams.n_lora_kv;\n\n                    const int64_t n_ff_exp        = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    // try to load output.weight, if not found, use token_embd (tied embeddings)\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    if (!output) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        int flags = 0;\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            // skip all tensors in the NextN layers\n                            // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later\n                            flags |= TENSOR_SKIP | TENSOR_NOT_REQUIRED;\n                        }\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, flags);\n                        layer.attn_q_a_norm  = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, \"weight\", i), {q_lora_rank}, flags);\n                        layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, \"weight\", i), {kv_lora_rank}, flags);\n\n                        layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, \"weight\", i), {n_embd, q_lora_rank}, flags);\n                        layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, \"weight\", i), {q_lora_rank, n_head * n_embd_head_k_mla}, flags);\n\n                        layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, \"weight\", i), {n_embd, kv_lora_rank + n_embd_head_qk_rope}, flags);\n\n                        // note: only old legacy GGUF files will have the unsplit wkv_b tensor in\n                        layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, \"weight\", i), {n_embd_head_qk_nope, kv_lora_rank, n_head}, flags);\n                        layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, \"weight\", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, flags);\n\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head * n_embd_head_v_mla, n_embd}, flags);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, flags);\n\n                        // DSA indexer\n                        layer.indexer_k_norm   = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   \"weight\", i), {hparams.indexer_head_size}, flags);\n                        layer.indexer_k_norm_b = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM,   \"bias\",   i), {hparams.indexer_head_size}, flags);\n                        layer.indexer_proj     = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ,     \"weight\", i), {n_embd, hparams.indexer_n_head}, flags);\n                        layer.indexer_attn_k   = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K,   \"weight\", i), {n_embd, hparams.indexer_head_size}, flags);\n                        layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, \"weight\", i), {q_lora_rank, hparams.indexer_n_head * hparams.indexer_head_size}, flags);\n                        if (i < (int) hparams.n_layer_dense_lead) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, flags);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, flags);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, flags);\n                        } else {\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, flags);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n\n                            if (n_expert == 0) {\n                                throw std::runtime_error(\"n_expert must be > 0\");\n                            }\n                            if (n_expert_used == 0) {\n                                throw std::runtime_error(\"n_expert_used must be > 0\");\n                            }\n\n                            // MoE branch\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, flags);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, flags);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, flags);\n\n                            // Shared expert branch\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, flags);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {        n_ff_exp * n_expert_shared, n_embd}, flags);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, flags);\n                        }\n\n                        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, \"weight\", i), { 2 * n_embd, n_embd }, flags);\n                            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, \"weight\", i), { n_embd }, flags);\n                            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, \"weight\", i), { n_embd }, flags);\n\n                            // Optional tensors\n                            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, \"weight\", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, \"weight\", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_NEMOTRON:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"), {n_embd}, 0);\n                    output        = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_norm_b = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"bias\", i), {n_embd}, 0);\n\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                        // optional MLP bias\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i), {n_ff}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_NEMOTRON_H:\n            case LLM_ARCH_NEMOTRON_H_MOE:\n                {\n                    // mamba2 Mixer SSM params\n                    // NOTE: int64_t for tensor dimensions\n                    const int64_t d_conv     = hparams.ssm_d_conv;\n                    const int64_t d_inner    = hparams.ssm_d_inner;\n                    const int64_t d_state    = hparams.ssm_d_state;\n                    const int64_t n_ssm_head = hparams.ssm_dt_rank;\n                    const int64_t n_group    = hparams.ssm_n_group;\n                    const int64_t d_in_proj  = 2*d_inner + 2*n_group*d_state + n_ssm_head;\n                    const int64_t moe_n_embd = hparams.moe_latent_size > 0 ? hparams.moe_latent_size : n_embd;\n\n                    // embeddings\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    {\n                        output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                        // if output is NULL, init from the input tok embed, duplicated to allow offloading\n                        if (output == NULL) {\n                            output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                        }\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        // all blocks use the attn norm\n                        layer.attn_norm  = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (hparams.is_recurrent(i)) {\n                            // ssm layers\n                            layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {n_embd, d_in_proj}, 0);\n\n                            layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {d_conv, d_inner + 2*n_group*d_state}, 0);\n                            layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {d_inner + 2*n_group*d_state}, TENSOR_NOT_REQUIRED);\n\n                            layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {n_ssm_head}, 0);\n\n                            // no \"weight\" suffix for these\n                            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_ssm_head}, 0);\n                            layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, n_ssm_head}, 0);\n\n                            layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, \"weight\", i), {d_inner / n_group, n_group}, 0);\n\n                            // out_proj\n                            layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {d_inner, n_embd}, 0);\n                        } else if (hparams.n_ff(i) == 0) {\n                            // attention layers (with optional bias)\n                            const int64_t n_head_i = hparams.n_head(i);\n                            const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);\n                            const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head_i}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa_i}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa_i}, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head_i, n_embd}, 0);\n                            layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\",   i), {n_embd},         TENSOR_NOT_REQUIRED);\n                            layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\",   i), {n_embd_k_gqa_i}, TENSOR_NOT_REQUIRED);\n                            layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\",   i), {n_embd_v_gqa_i}, TENSOR_NOT_REQUIRED);\n                            layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\",   i), {n_embd},         TENSOR_NOT_REQUIRED);\n                        }  else {\n                            if (n_expert != 0) {\n                                const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n                                const int64_t n_ff_shexp = hparams.n_ff_shexp;\n\n                                layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), { n_embd, n_expert}, 0);\n                                layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert         }, 0);\n\n                                // MoE branch\n                                layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, \"weight\", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);\n                                layer.ffn_latent_up   = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP,   \"weight\", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);\n\n                                layer.ffn_down_exps   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   moe_n_embd, n_expert}, 0);\n                                layer.ffn_up_exps     = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {moe_n_embd, n_ff_exp, n_expert}, 0);\n\n                                // Shared expert branch\n                                layer.ffn_down_shexp  = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd}, 0);\n                                layer.ffn_up_shexp    = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp}, 0);\n\n                            } else {\n                                // mlp layers\n                                layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  hparams.n_ff(i), n_embd}, 0);\n                                layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   hparams.n_ff(i)}, 0);\n                                layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\",   i), {n_embd}, TENSOR_NOT_REQUIRED);\n                                layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\",   i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);\n                            }\n                        }\n                    }\n                } break;\n            case LLM_ARCH_EXAONE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.ffn_norm   = create_tensor(tn(LLM_TENSOR_FFN_NORM,   \"weight\", i), {n_embd}, 0);\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        layer.ffn_gate   = create_tensor(tn(LLM_TENSOR_FFN_GATE,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down   = create_tensor(tn(LLM_TENSOR_FFN_DOWN,   \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up     = create_tensor(tn(LLM_TENSOR_FFN_UP,     \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_EXAONE4:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_post_norm  = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_EXAONE_MOE:\n                {\n                    const int64_t n_ff_exp       = hparams.n_ff_exp;\n                    const int64_t n_expert       = hparams.n_expert;\n                    const int64_t n_expert_used  = hparams.n_expert_used;\n                    const int64_t n_ff_shexp     = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : n_ff_exp;\n                    const int64_t head_dim       = hparams.n_embd_head_k();\n                    const int64_t n_qo_dim       = n_head * head_dim;\n                    const int64_t n_kv_dim       = n_head_kv * head_dim;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        int flags = 0;\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            // skip all tensors in the NextN layers\n                            flags |= TENSOR_SKIP;\n                        }\n\n                        auto & layer = layers[i];\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_qo_dim}, flags);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_kv_dim}, flags);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_kv_dim}, flags);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_qo_dim, n_embd}, flags);\n\n                        layer.rope_freqs   = create_tensor(tn(LLM_TENSOR_ROPE_FREQS,  \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0) | flags);\n\n                        layer.attn_norm    = create_tensor(tn(LLM_TENSOR_ATTN_NORM,   \"weight\", i), {n_embd}, flags);\n                        layer.attn_q_norm  = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, flags);\n                        layer.attn_k_norm  = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, flags);\n\n                        layer.ffn_norm     = create_tensor(tn(LLM_TENSOR_FFN_NORM,    \"weight\", i), {n_embd}, flags);\n\n                        // dense layers for first n_layer_dense_lead layers or nextn_predict_layers layers at the end\n                        if (i < (int) hparams.n_layer_dense_lead || (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers)) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, flags);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, flags);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, flags);\n                        } else {\n                            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, flags);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);\n\n                            if (n_expert == 0) {\n                                throw std::runtime_error(\"n_expert must be > 0\");\n                            }\n                            if (n_expert_used == 0) {\n                                throw std::runtime_error(\"n_expert_used must be > 0\");\n                            }\n\n                            layer.ffn_gate_exps  = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS,  \"weight\", i), {n_embd, n_ff_exp, n_expert}, flags);\n                            layer.ffn_down_exps  = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS,  \"weight\", i), {n_ff_exp, n_embd, n_expert}, flags);\n                            layer.ffn_up_exps    = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,    \"weight\", i), {n_embd, n_ff_exp, n_expert}, flags);\n\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_shexp}, flags);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd}, flags);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp}, flags);\n                        }\n\n                        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, \"weight\", i), {2 * n_embd, n_embd}, flags);\n                            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,   \"weight\", i), {n_embd}, flags);\n                            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,   \"weight\", i), {n_embd}, flags);\n\n                            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, \"weight\", i), {n_embd}, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     \"weight\", i), {n_embd, n_vocab}, flags | TENSOR_NOT_REQUIRED);\n                            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, \"weight\", i), {n_embd, n_vocab}, flags | TENSOR_NOT_REQUIRED);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_RWKV6:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // Block 0, LN0\n                    tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0);\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"), {n_embd}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"), {n_embd}, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    const int time_mix_extra_dim = hparams.time_mix_extra_dim;\n                    const int time_decay_extra_dim = hparams.time_decay_extra_dim;\n                    const int head_size = hparams.wkv_head_size;\n                    const int attn_hidden_size = n_embd;\n                    const int ffn_size = hparams.n_ff_arr[0];\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.attn_norm_2   = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"bias\", i),   {n_embd}, 0);\n\n                        layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, \"weight\", i), {n_embd, time_mix_extra_dim * 5}, 0);\n                        layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, \"weight\", i), {time_mix_extra_dim, n_embd, 5}, 0);\n\n                        layer.time_mix_lerp_x = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_X, \"weight\", i), {n_embd, 1, 1}, 0);\n                        layer.time_mix_lerp_w = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_W, \"weight\", i), {n_embd, 1, 1}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_lerp_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_K, \"weight\", i), {n_embd, 1, 1}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_lerp_v = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_V, \"weight\", i), {n_embd, 1, 1}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_lerp_r = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_R, \"weight\", i), {n_embd, 1, 1}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_lerp_g = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_G, \"weight\", i), {n_embd, 1, 1}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, \"weight\", i), {n_embd, 1, 1, 5}, TENSOR_NOT_REQUIRED);\n                        GGML_ASSERT(!(layer.time_mix_lerp_fused == NULL && layer.time_mix_lerp_w == NULL));\n\n                        layer.time_mix_first = create_tensor(tn(LLM_TENSOR_TIME_MIX_FIRST, \"weight\", i), {head_size, n_embd / head_size}, 0);\n                        layer.time_mix_decay = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_decay_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY_W1, \"weight\", i), {n_embd, time_decay_extra_dim}, 0);\n                        layer.time_mix_decay_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY_W2, \"weight\", i), {time_decay_extra_dim, attn_hidden_size}, 0);\n                        layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_gate = create_tensor(tn(LLM_TENSOR_TIME_MIX_GATE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n\n                        layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"bias\", i), {n_embd}, 0);\n                        layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, \"weight\", i), {n_embd, attn_hidden_size}, 0);\n\n                        layer.channel_mix_lerp_k = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, \"weight\", i), {n_embd, 1, 1}, 0);\n                        layer.channel_mix_lerp_r = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_R, \"weight\", i), {n_embd, 1, 1}, 0);\n\n                        layer.channel_mix_key = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_KEY, \"weight\", i), {n_embd, ffn_size}, 0);\n                        layer.channel_mix_value = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_VALUE, \"weight\", i), {ffn_size, n_embd}, 0);\n                        layer.channel_mix_receptance = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_RECEPTANCE, \"weight\", i), {n_embd, n_embd}, 0);\n                    }\n\n                } break;\n            case LLM_ARCH_RWKV6QWEN2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"), {n_embd}, TENSOR_NOT_REQUIRED);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    const int time_mix_extra_dim = hparams.time_mix_extra_dim;\n                    const int time_decay_extra_dim = hparams.time_decay_extra_dim;\n                    const int head_size = hparams.wkv_head_size;\n                    const int attn_hidden_size = n_embd;\n                    const int n_head_kv = hparams.n_head_kv();\n                    int attn_key_value_size;\n                    if (n_head_kv == 0 || attn_hidden_size / head_size == n_head_kv) {\n                        attn_key_value_size = attn_hidden_size;\n                    } else {\n                        attn_key_value_size = n_head_kv * head_size;\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, \"weight\", i), {n_embd, time_mix_extra_dim * 5}, 0);\n                        layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, \"weight\", i), {time_mix_extra_dim, n_embd, 5}, 0);\n\n                        layer.time_mix_lerp_x = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_X, \"weight\", i), {n_embd, 1, 1}, 0);\n                        layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, \"weight\", i), {n_embd, 1, 1, 5}, 0);\n\n                        layer.time_mix_first = create_tensor(tn(LLM_TENSOR_TIME_MIX_FIRST, \"weight\", i), {head_size, n_embd / head_size}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_decay = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_decay_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY_W1, \"weight\", i), {n_embd, time_decay_extra_dim}, 0);\n                        layer.time_mix_decay_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_DECAY_W2, \"weight\", i), {time_decay_extra_dim, attn_hidden_size}, 0);\n                        layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, \"weight\", i), {n_embd, attn_key_value_size}, 0);\n                        layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, \"weight\", i), {n_embd, attn_key_value_size}, 0);\n                        layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_gate = create_tensor(tn(LLM_TENSOR_TIME_MIX_GATE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        // optional bias tensors\n                        layer.time_mix_key_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, \"bias\", i), {attn_key_value_size}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_value_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, \"bias\", i), {attn_key_value_size}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_receptance_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, \"bias\", i), {attn_hidden_size}, TENSOR_NOT_REQUIRED);\n\n                        layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, \"weight\", i), {n_embd, attn_hidden_size}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_RWKV7:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // Block 0, LN0\n                    tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {n_embd}, 0);\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"), {n_embd}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"), {n_embd}, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    const int n_lora_decay = hparams.n_lora_decay;\n                    const int n_lora_iclr = hparams.n_lora_iclr;\n                    const int n_lora_value_res_mix = hparams.n_lora_value_res_mix;\n                    const int n_lora_gate = hparams.n_lora_gate;\n                    const int attn_hidden_size = n_embd;\n                    const int ffn_size = hparams.n_ff_arr[0];\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"bias\", i),   {n_embd}, 0);\n\n                        layer.attn_norm_2   = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"weight\", i), {n_embd}, 0);\n                        layer.attn_norm_2_b = create_tensor(tn(LLM_TENSOR_ATTN_NORM_2, \"bias\", i),   {n_embd}, 0);\n\n                        layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, \"weight\", i), {n_embd, n_lora_decay}, 0);\n                        layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, \"weight\", i), {n_lora_decay, n_embd}, 0);\n\n                        layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, \"weight\", i), {n_embd, n_lora_iclr}, 0);\n                        layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, \"weight\", i), {n_lora_iclr, n_embd}, 0);\n\n                        if (i == 0) {\n                            // actually not used\n                            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, \"weight\", i), {n_embd}, 0);\n                            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, \"weight\", i), {n_embd, n_lora_iclr}, 0);\n                            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, \"weight\", i), {n_lora_iclr, n_embd}, 0);\n                        } else {\n                            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, \"weight\", i), {n_embd}, 0);\n                            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, \"weight\", i), {n_embd, n_lora_value_res_mix}, 0);\n                            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, \"weight\", i), {n_lora_value_res_mix, n_embd}, 0);\n                        }\n\n                        layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, \"weight\", i), {n_embd, n_lora_gate}, 0);\n                        layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, \"weight\", i), {n_lora_gate, n_embd}, 0);\n\n                        layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, \"weight\", i), {n_embd, 1, 1, 6}, 0);\n\n                        layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, \"weight\", i), {attn_hidden_size}, 0);\n                        layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, \"weight\", i), {attn_hidden_size}, 0);\n                        layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, \"weight\", i), {attn_hidden_size}, 0);\n\n                        layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n\n                        layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"bias\", i), {n_embd}, 0);\n                        layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, \"weight\", i), {n_embd, attn_hidden_size}, 0);\n\n                        layer.channel_mix_lerp_k = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_LERP_K, \"weight\", i), {n_embd, 1, 1}, 0);\n\n                        layer.channel_mix_key = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_KEY, \"weight\", i), {n_embd, ffn_size}, 0);\n                        layer.channel_mix_value = create_tensor(tn(LLM_TENSOR_CHANNEL_MIX_VALUE, \"weight\", i), {ffn_size, n_embd}, 0);\n                    }\n\n                } break;\n            case LLM_ARCH_ARWKV7:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    const int n_lora_decay = hparams.n_lora_decay;\n                    const int n_lora_iclr = hparams.n_lora_iclr;\n                    const int n_lora_value_res_mix = hparams.n_lora_value_res_mix;\n                    const int n_lora_gate = hparams.n_lora_gate;\n                    const int attn_hidden_size = n_embd;\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.time_mix_w0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W0, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_w1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W1, \"weight\", i), {n_embd, n_lora_decay}, 0);\n                        layer.time_mix_w2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_W2, \"weight\", i), {n_lora_decay, n_embd}, 0);\n\n                        layer.time_mix_a0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A0, \"weight\", i), {n_embd}, 0);\n                        layer.time_mix_a1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A1, \"weight\", i), {n_embd, n_lora_iclr}, 0);\n                        layer.time_mix_a2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_A2, \"weight\", i), {n_lora_iclr, n_embd}, 0);\n\n                        if (i == 0) {\n                            // actually not used\n                            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, \"weight\", i), {n_embd}, 0);\n                            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, \"weight\", i), {n_embd, n_lora_iclr}, 0);\n                            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, \"weight\", i), {n_lora_iclr, n_embd}, 0);\n                        } else {\n                            layer.time_mix_v0 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V0, \"weight\", i), {n_embd}, 0);\n                            layer.time_mix_v1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V1, \"weight\", i), {n_embd, n_lora_value_res_mix}, 0);\n                            layer.time_mix_v2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_V2, \"weight\", i), {n_lora_value_res_mix, n_embd}, 0);\n                        }\n\n                        layer.time_mix_g1 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G1, \"weight\", i), {n_embd, n_lora_gate}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_g2 = create_tensor(tn(LLM_TENSOR_TIME_MIX_G2, \"weight\", i), {n_lora_gate, n_embd}, TENSOR_NOT_REQUIRED);\n\n                        try {\n                            layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, \"weight\", i), {n_embd, 1, 1, 6}, 0);\n                        } catch(std::runtime_error & e) {\n                            // ARWKV models may not have gate tensors\n                            layer.time_mix_lerp_fused = create_tensor(tn(LLM_TENSOR_TIME_MIX_LERP_FUSED, \"weight\", i), {n_embd, 1, 1, 5}, 0);\n                        }\n\n                        layer.time_mix_k_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_K, \"weight\", i), {attn_hidden_size}, 0);\n                        layer.time_mix_k_a = create_tensor(tn(LLM_TENSOR_TIME_MIX_K_A, \"weight\", i), {attn_hidden_size}, 0);\n                        layer.time_mix_r_k = create_tensor(tn(LLM_TENSOR_TIME_MIX_R_K, \"weight\", i), {attn_hidden_size}, 0);\n\n                        layer.time_mix_key = create_tensor(tn(LLM_TENSOR_TIME_MIX_KEY, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_value = create_tensor(tn(LLM_TENSOR_TIME_MIX_VALUE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n                        layer.time_mix_receptance = create_tensor(tn(LLM_TENSOR_TIME_MIX_RECEPTANCE, \"weight\", i), {attn_hidden_size, n_embd}, 0);\n\n                        layer.time_mix_ln = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"weight\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_ln_b = create_tensor(tn(LLM_TENSOR_TIME_MIX_LN, \"bias\", i), {n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.time_mix_output = create_tensor(tn(LLM_TENSOR_TIME_MIX_OUTPUT, \"weight\", i), {n_embd, attn_hidden_size}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n\n                } break;\n            case LLM_ARCH_CHAMELEON:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k, n_head}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k, n_head_kv}, 0);\n                        layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"bias\", i),  {n_embd_head_k, n_head}, TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"bias\", i),  {n_embd_head_k, n_head_kv}, TENSOR_NOT_REQUIRED);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_WAVTOKENIZER_DEC:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {hparams.n_embd, n_vocab}, 0);\n\n                    conv1d   = create_tensor(tn(LLM_TENSOR_CONV1D, \"weight\"), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);\n                    conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, \"bias\"),   {1, hparams.posnet.n_embd}, 0);\n\n                    // posnet\n                    {\n                        const int64_t n_embd = hparams.posnet.n_embd;\n\n                        for (uint32_t i = 0; i < hparams.posnet.n_layer; ++i) {\n                            auto & layer = layers[i].posnet;\n\n                            // posnet:\n                            //\n                            //  - resnet\n                            //  - resnet\n                            //  - attn\n                            //  - resnet\n                            //  - resnet\n                            //  - norm\n                            //\n                            switch (i) {\n                                case 0:\n                                case 1:\n                                case 3:\n                                case 4:\n                                    {\n                                        layer.norm1   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, \"weight\", i), {1, n_embd}, 0);\n                                        layer.norm1_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.conv1   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, \"weight\", i), {3, n_embd, n_embd}, 0);\n                                        layer.conv1_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.norm2   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, \"weight\", i), {1, n_embd}, 0);\n                                        layer.norm2_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.conv2   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, \"weight\", i), {3, n_embd, n_embd}, 0);\n                                        layer.conv2_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, \"bias\",   i), {1, n_embd}, 0);\n                                    } break;\n                                case 2:\n                                    {\n                                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, \"weight\", i), {1, n_embd}, 0);\n                                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.attn_q      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    \"weight\", i), {1, n_embd, n_embd}, 0);\n                                        layer.attn_q_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.attn_k      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    \"weight\", i), {1, n_embd, n_embd}, 0);\n                                        layer.attn_k_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.attn_v      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    \"weight\", i), {1, n_embd, n_embd}, 0);\n                                        layer.attn_v_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    \"bias\",   i), {1, n_embd}, 0);\n\n                                        layer.attn_o      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  \"weight\", i), {1, n_embd, n_embd}, 0);\n                                        layer.attn_o_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  \"bias\",   i), {1, n_embd}, 0);\n                                    } break;\n                                case 5:\n                                    {\n                                        layer.norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, \"weight\", i), {1, n_embd}, 0);\n                                        layer.norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, \"bias\",   i), {1, n_embd}, 0);\n                                    } break;\n                                default: GGML_ABORT(\"unknown posnet layer\");\n                            };\n                        }\n                    }\n\n                    GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);\n\n                    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"weight\"), {hparams.posnet.n_embd}, 0);\n                    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, \"bias\"),   {hparams.posnet.n_embd}, 0);\n\n                    // convnext\n                    {\n                        const int64_t n_embd = hparams.convnext.n_embd;\n\n                        for (uint32_t i = 0; i < hparams.convnext.n_layer; ++i) {\n                            auto & layer = layers[i].convnext;\n\n                            layer.dw     = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    \"weight\", i), {7, 1, n_embd}, 0);\n                            layer.dw_b   = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    \"bias\",   i), {1, n_embd}, 0);\n\n                            layer.norm   = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  \"weight\", i), {n_embd}, 0);\n                            layer.norm_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  \"bias\",   i), {n_embd}, 0);\n\n                            layer.pw1    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   \"weight\", i), {n_embd, n_ff}, 0);\n                            layer.pw1_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   \"bias\",   i), {n_ff}, 0);\n\n                            layer.pw2    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   \"weight\", i), {n_ff, n_embd}, 0);\n                            layer.pw2_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   \"bias\",   i), {n_embd}, 0);\n\n                            layer.gamma  = create_tensor(tn(LLM_TENSOR_CONVNEXT_GAMMA, \"weight\", i), {n_embd}, 0);\n                        }\n\n                        // output\n                        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"bias\"),   {n_embd}, 0);\n                    }\n\n                    output   = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {hparams.convnext.n_embd, hparams.n_embd_out()}, 0);\n                    output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, \"bias\"),   {hparams.n_embd_out()}, 0);\n                } break;\n            case LLM_ARCH_BAILINGMOE:\n                {\n                    const int64_t n_ff_exp            = hparams.n_ff_exp;\n                    const int64_t n_expert_shared     = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_head * n_rot}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_head_kv * n_rot}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_head_kv * n_rot}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head * n_rot, n_embd}, 0);\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        if (n_expert == 0) {\n                            throw std::runtime_error(\"n_expert must be > 0\");\n                        }\n                        if (n_expert_used == 0) {\n                            throw std::runtime_error(\"n_expert_used must be > 0\");\n                        }\n\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {        n_ff_exp * n_expert_shared, n_embd}, 0);\n                        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                    }\n                } break;\n            case LLM_ARCH_BAILINGMOE2:\n                {\n                    const int64_t n_ff_exp        = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    GGML_ASSERT(n_expert > 0 && \"n_expert must be > 0 for bailingmoe2\");\n                    GGML_ASSERT(n_expert_used > 0 && \"n_expert_used must be > 0 for bailingmoe2\");\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        int flags = 0;\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            // skip all tensors in the NextN layers\n                            flags |= TENSOR_SKIP;\n                        }\n\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, flags);\n\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd + 2*n_embd_gqa}, flags);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, flags);\n\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, flags);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, flags);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, flags);\n\n                        if (static_cast<uint32_t>(i) >= hparams.n_layer_dense_lead) { // MoE layers\n                            const int64_t n_ff_shexp = (hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff_exp) * n_expert_shared;\n\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, flags);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);\n\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, flags);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, flags);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, flags);\n\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_shexp}, flags);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd}, flags);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp}, flags);\n                        } else { // Dense layers\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, flags);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, flags);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, flags);\n                        }\n\n                        // NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers\n                        if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {\n                            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, \"weight\", i), { 2 * n_embd, n_embd }, flags);\n                            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, \"weight\", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);\n                            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, \"weight\", i), { n_embd }, flags);\n                            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, \"weight\", i), { n_embd }, flags);\n                            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, \"weight\", i), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED | flags);\n                            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, \"weight\", i), { n_embd }, TENSOR_NOT_REQUIRED | flags);\n                            layer.layer_out_norm         = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, \"weight\", i), {n_embd}, flags);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_DOTS1:\n                {\n                    const int64_t n_ff_exp        = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (i < (int) hparams.n_layer_dense_lead) {\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        } else {\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n\n                            if (n_expert == 0) {\n                                throw std::runtime_error(\"n_expert must be > 0\");\n                            }\n                            if (n_expert_used == 0) {\n                                throw std::runtime_error(\"n_expert_used must be > 0\");\n                            }\n\n                            // MoE branch\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                            // Shared expert branch\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {        n_ff_exp * n_expert_shared, n_embd}, 0);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_exp * n_expert_shared}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_ARCEE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_AFMOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    const int64_t n_ff_exp = hparams.n_ff_exp;\n                    const int64_t n_expert_shared = hparams.n_expert_shared;\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        // dual attention normalization\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      \"weight\", i), {n_embd}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // attention projections\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        // Q/K normalization\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        // attention gating\n                        layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n\n                        // dual ffn normalization\n                        layer.ffn_norm      = create_tensor(tn(LLM_TENSOR_FFN_NORM,      \"weight\", i), {n_embd}, 0);\n                        layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (static_cast<uint32_t>(i) >= hparams.n_layer_dense_lead) {\n                            // MoE layers\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, 0);\n\n                            // grouped expert weights\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff_exp, n_expert}, 0);\n\n                            // shared expert\n                            if (n_expert_shared > 0) {\n                                const int64_t n_ff_shexp = n_ff_exp * n_expert_shared;\n                                layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_shexp}, 0);\n                                layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd}, 0);\n                                layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp}, 0);\n                            }\n                        } else {\n                            // Dense layers\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_ERNIE4_5:\n            case LLM_ARCH_ERNIE4_5_MOE:\n            case LLM_ARCH_PADDLEOCR:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd},     TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (arch == LLM_ARCH_ERNIE4_5_MOE && static_cast<uint32_t>(i) >= hparams.n_layer_dense_lead) { // MoE layers\n                            int n_ff_exp = hparams.n_ff_exp;\n\n                            layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff_exp, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff_exp, n_expert}, 0);\n\n                            // Shared expert (if present)\n                            if (hparams.n_ff_shexp > 0) {\n                                layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {    n_embd, hparams.n_ff_shexp}, 0);\n                                layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {hparams.n_ff_shexp, n_embd    }, 0);\n                                layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {    n_embd, hparams.n_ff_shexp}, 0);\n                            }\n                        } else { // Dense layers\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_FALCON_H1:\n                {\n                    // Common\n                    const int64_t hidden_size = hparams.n_embd; // hidden_size\n\n                    // mamba2 Mixer SSM params\n                    const int64_t ssm_conv_kernel_size  = hparams.ssm_d_conv; // ssm_conv_kernel_size\n                    const int64_t ssm_n_groups          = hparams.ssm_n_group; // ssm_n_groups\n                    const int64_t ssm_state_size        = hparams.ssm_d_state; // ssm_state_size\n                    const int64_t ssm_intermediate_size = hparams.ssm_d_inner; // TODO expand\n                    const int64_t ssm_num_heads         = hparams.ssm_dt_rank; // ssm_num_heads\n                    const int64_t ssm_conv_dim          = ssm_intermediate_size + 2 * ssm_n_groups * ssm_state_size;\n                    const int64_t ssm_projection_size   = ssm_intermediate_size + ssm_conv_dim + ssm_num_heads;\n\n                    // attn params\n                    const int64_t attn_num_attention_head = hparams.n_head(0); // rename to: attn_num_attention_head\n                    const int64_t attn_num_key_value_head = hparams.n_head_kv(0);\n\n                    // ffn params\n                    const int64_t ffn_intermediate_size = hparams.n_ff(0);\n\n                    // embeddings\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {hidden_size, n_vocab}, 0);\n\n                    // output\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), {hidden_size, n_vocab}, TENSOR_NOT_REQUIRED);\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {hidden_size}, 0);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {hidden_size, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        /*SSM LAYERS*/\n                        // ssm in\n                        layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, \"weight\", i), {hidden_size, ssm_projection_size}, 0);\n                        // ssm 1d conv\n                        layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"weight\", i), {ssm_conv_kernel_size, ssm_conv_dim}, 0);\n                        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, \"bias\", i), {ssm_conv_dim}, TENSOR_NOT_REQUIRED);\n                        // ssm_dt\n                        layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {ssm_num_heads}, 0);\n                        // no \"weight\" suffix for these\n                        layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, ssm_num_heads}, 0);\n                        layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, ssm_num_heads}, 0);\n                        // ssm_norm\n                        layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, \"weight\", i), {ssm_intermediate_size / ssm_n_groups, ssm_n_groups}, TENSOR_NOT_REQUIRED);\n                        // out_proj\n                        layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"weight\", i), {ssm_intermediate_size, hidden_size}, 0);\n\n                        /*ATTENTION LAYERS*/\n                        // attention layers (with optional bias)\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {hidden_size, n_embd_head_k * attn_num_attention_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {hidden_size, attn_num_key_value_head * n_embd_head_k}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {hidden_size, attn_num_key_value_head * n_embd_head_v}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * attn_num_attention_head, hidden_size}, 0);\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {hidden_size}, TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {attn_num_key_value_head * n_embd_head_k}, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {attn_num_key_value_head * n_embd_head_v}, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {hidden_size}, TENSOR_NOT_REQUIRED);\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {hidden_size}, 0);\n\n\n                        // feed forward (w/ optional biases)\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, i), {hidden_size}, 0);\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {hidden_size,   ffn_intermediate_size}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  ffn_intermediate_size, hidden_size}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {hidden_size,   ffn_intermediate_size}, 0);\n\n                        layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"bias\", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"bias\", i), {hidden_size}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"bias\", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_HUNYUAN_MOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n                        const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(i);\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd,   n_ff, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {  n_ff, n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd,   n_ff, n_expert}, 0);\n\n                        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_shexp}, 0);\n                        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp}, 0);\n                        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp, n_embd}, 0);\n                    }\n                } break;\n            case LLM_ARCH_HUNYUAN_DENSE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                    }\n                } break;\n            case LLM_ARCH_SMOLLM3:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_OPENAI_MOE:\n                {\n                    const int64_t n_ff_exp = hparams.n_ff_exp;\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      \"weight\", i), {n_embd}, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_head * n_rot}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_head_kv * n_rot}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_head_kv * n_rot}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head * n_rot, n_embd}, 0);\n\n                        layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, \"weight\", i), {n_head}, 0);\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {  n_embd, n_expert}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                        // bias\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_head * n_rot}, 0);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_head_kv * n_rot}, 0);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_head_kv * n_rot}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp_b  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"bias\", i), {n_expert}, 0);\n                        layer.ffn_gate_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"bias\", i), {n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"bias\", i), {  n_embd, n_expert}, 0);\n                        layer.ffn_up_exps_b   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"bias\", i), {n_ff_exp, n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_LFM2:\n            case LLM_ARCH_LFM2MOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM_LFM2, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,           \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        const bool is_moe_layer = i >= static_cast<int>(hparams.n_layer_dense_lead);\n\n                        // ffn/moe is same for transformer and conv layers\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        if (is_moe_layer) {\n                            GGML_ASSERT(n_expert && n_expert_used);\n                            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i),  {n_embd, n_expert}, 0);\n                            layer.ffn_gate_exps   = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {hparams.n_ff_exp,   n_embd, n_expert}, 0);\n                            layer.ffn_up_exps     = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, \"weight\", i),   {n_embd, hparams.n_ff_exp, n_expert}, 0);\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, 0);\n                        } else {  // dense\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                        }\n\n                        // for operator_norm\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (!hparams.is_recurrent(i)) {\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                            GGML_ASSERT(n_embd_v_gqa == n_embd_k_gqa);\n\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd}, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, hparams.n_embd_k_gqa(i)}, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, hparams.n_embd_v_gqa(i)}, 0);\n\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd, n_embd}, 0);\n                        } else {\n                            layer.shortconv.conv     = create_tensor(tn(LLM_TENSOR_SHORTCONV_CONV,    \"weight\", i), {hparams.n_shortconv_l_cache, n_embd}, 0);\n                            layer.shortconv.in_proj  = create_tensor(tn(LLM_TENSOR_SHORTCONV_INPROJ,  \"weight\", i), {n_embd, 3 * n_embd}, 0);\n                            layer.shortconv.out_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_OUTPROJ, \"weight\", i), {n_embd, n_embd}, 0);\n                        }\n                    }\n\n                    // for LFM2-ColBert-350M\n                    dense_2_out_layers   = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, \"weight\"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);\n                    dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, \"bias\"),   {hparams.n_embd_out()        }, TENSOR_NOT_REQUIRED);\n                } break;\n            case LLM_ARCH_SMALLTHINKER:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd_head_k * n_head }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        GGML_ASSERT(n_expert > 0 && \"n_expert must be > 0 for SMALLTHINKER\");\n                        GGML_ASSERT(n_expert_used > 0 && \"n_expert_used must be > 0 for SMALLTHINKER\");\n\n                        // MoE branch\n                        const int64_t n_ff_exp = hparams.n_ff_exp;\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), { n_embd, n_expert }, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), { n_embd, n_ff_exp, n_expert }, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), { n_ff_exp, n_embd, n_expert }, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, \"weight\", i), { n_embd, n_ff_exp, n_expert }, 0);\n                    }\n                } break;\n            case LLM_ARCH_GROVEMOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    GGML_ASSERT(n_expert > 0 && \"n_expert must be > 0 for GROVEMOE\");\n                    GGML_ASSERT(n_expert_used > 0 && \"n_expert_used must be > 0 for GROVEMOE\");\n                    GGML_ASSERT(hparams.n_group_experts > 0 && \"n_group_experts must be > 0 for GROVEMOE\");\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n\n                        // MoE branch\n                        const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n                        const int64_t n_ff_chexp = hparams.n_ff_chexp ? hparams.n_ff_chexp : n_embd_head_k;\n                        const int64_t n_chunk_expert = n_expert / hparams.n_group_experts;\n\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {  n_embd, n_ff_exp, n_expert}, 0);\n\n                        layer.ffn_gate_chexps = create_tensor(tn(LLM_TENSOR_FFN_GATE_CHEXPS, \"weight\", i), {  n_embd, n_ff_chexp, n_chunk_expert}, 0);\n                        layer.ffn_down_chexps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_CHEXPS, \"weight\", i), {n_ff_chexp,   n_embd, n_chunk_expert}, 0);\n                        layer.ffn_up_chexps   = create_tensor(tn(LLM_TENSOR_FFN_UP_CHEXPS,   \"weight\", i), {  n_embd, n_ff_chexp, n_chunk_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_APERTUS:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), { n_embd, n_vocab }, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), { n_embd }, 0);\n\n                        if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {\n                            layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        } else {\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), { n_rot/2 }, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), { n_embd, n_embd_head_k * n_head }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                        // optional bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), { n_embd },     TENSOR_NOT_REQUIRED);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), { n_embd_gqa }, TENSOR_NOT_REQUIRED);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), { n_embd_gqa }, TENSOR_NOT_REQUIRED);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), { n_embd },     TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), { n_embd }, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), { n_ff, n_embd }, 0);\n                        layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, \"weight\", i), { n_embd, n_ff }, 0);\n\n                        // Q and K layernorms for Apertus\n                        layer.attn_q_norm   = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                        layer.attn_q_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"bias\",   i), { n_embd_head_k }, TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm   = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                        layer.attn_k_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"bias\",   i), { n_embd_head_k }, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_MINIMAX_M2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd_head_k * n_head }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_gqa }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k * n_head}, 0);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_k_gqa}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff,   n_embd, n_expert}, 0);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff,   n_expert}, 0);\n                        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, 0);\n                    }\n                } break;\n            case LLM_ARCH_KIMI_LINEAR:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // Check for KDA specific tensors to determine layer type or if it's a mixed model\n                        // Assuming KDA layer if KDA tensors are present\n\n                        // KDA uses head_dim = 128 (from linear_attn_config.head_dim)\n                        const int64_t n_embd_head_k_kda = hparams.n_embd_head_kda;\n                        const int64_t n_embd_head_v_kda = hparams.n_embd_head_kda;\n                        const int64_t ssm_d_conv = hparams.ssm_d_conv;\n\n                        if (hparams.is_recurrent(i)) {\n                            // Conv1d weights: try 4D first, then 3D (quantization may remove trailing 1)\n                            // 4D: [d_conv, 1, d_inner, 1], 3D: [d_conv, 1, d_inner]\n                            layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, \"weight\", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head, 1}, TENSOR_NOT_REQUIRED);\n                            if (!layer.ssm_q_conv) {\n                                layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, \"weight\", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head}, 0);\n                            }\n\n                             // KDA Layer - Conv1d weights may be 3D or 4D\n                             layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, \"weight\", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head, 1}, TENSOR_NOT_REQUIRED);\n                             if (!layer.ssm_k_conv) {\n                                 layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, \"weight\", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head}, 0);\n                             }\n                             layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, \"weight\", i), {ssm_d_conv, 1, n_embd_head_v_kda * n_head, 1}, TENSOR_NOT_REQUIRED);\n                             if (!layer.ssm_v_conv) {\n                                 layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, \"weight\", i), {ssm_d_conv, 1, n_embd_head_v_kda * n_head}, 0);\n                             }\n\n                             // q, k, v projections\n                             // Python: q_proj, k_proj, v_proj\n                             layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_embd_head_k_kda * n_head}, 0);\n                             layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), {n_embd, n_embd_head_k_kda * n_head}, 0);\n                             layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), {n_embd, n_embd_head_v_kda * n_head}, 0);\n\n                             // KDA specific projections\n                             // f_a_proj, f_b_proj\n                             layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, \"weight\", i), {n_embd, n_embd_head_k_kda}, 0); // head_dim\n                             layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, \"weight\", i), {n_embd_head_k_kda, n_embd_head_k_kda * n_head}, 0); // projection_size\n\n                             // b_proj (beta mixing coefficient)\n                             layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, \"weight\", i), {n_embd, n_head}, 0);\n\n                             // A_log - Shape in GGUF: [1, num_heads, 1, 1] (4D) or [1, num_heads] (2D after quantization) Note: -exp(A_log) is applied in convert_hf_to_gguf.py\n                             layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);\n                             if (!layer.ssm_a) {\n                                 layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);\n                             }\n\n                             // dt_bias - shape [n_embd_head_k_kda * n_head] = [4096]\n                             layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, \"bias\", i), {n_embd_head_k_kda * n_head}, 0);\n\n                             // g_a_proj, g_b_proj (output gate)\n                             layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, \"weight\", i), {n_embd, n_embd_head_k_kda}, 0);\n                             layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, \"weight\", i), {n_embd_head_k_kda, n_embd_head_k_kda * n_head}, 0);\n\n                             // o_norm (reusing SSM_NORM)\n                             layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, \"weight\", i), {n_embd_head_k_kda}, 0); // FusedRMSNormGated\n\n                             // o_proj\n                             layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_v_kda * n_head, n_embd}, 0);\n\n                        } else {\n                             // MLA Layer - use MLA-specific head dimensions\n                             const int64_t q_lora_rank  = hparams.n_lora_q;\n                             const int64_t kv_lora_rank = hparams.n_lora_kv;\n                             const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();\n                             const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();\n\n                             layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, \"weight\", i), {q_lora_rank}, TENSOR_NOT_REQUIRED);\n                             layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, \"weight\", i), {kv_lora_rank}, 0);\n\n                             if (layer.attn_q_a_norm) {\n                                 layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, \"weight\", i), {n_embd, q_lora_rank}, 0);\n                                 layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, \"weight\", i), {q_lora_rank, n_head * n_embd_head_k_mla}, 0);\n                             } else {\n                                 // Kimi MLA without Q compression: wq = [n_embd, n_head * n_embd_head_k_mla]\n                                 layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), {n_embd, n_head * n_embd_head_k_mla}, 0);\n                             }\n\n                             // Kimi: qk_rope_head_dim = 64 (actual RoPE dimension for MLA)\n                             // Note: hparams.n_rot may be 72 (from conversion) but actual is 64\n                             const int64_t qk_rope_head_dim = hparams.n_rot();  // From config: qk_rope_head_dim\n                             layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, \"weight\", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0);\n                             // Support Legacy GGUFs that don't split wkv_b (MLA KV cache disabled)\n                             layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, \"weight\", i),\n                                {kv_lora_rank, n_head * (n_embd_head_k_mla - qk_rope_head_dim + n_embd_head_v_mla)}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);\n                             if (!layer.wkv_b) { // MLA KV cache enabled\n                                 layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, \"weight\", i), {n_embd_head_k_mla - qk_rope_head_dim, kv_lora_rank, n_head}, 0);\n                                 layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, \"weight\", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0);\n                             }\n                             layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_head * n_embd_head_v_mla, n_embd}, 0);\n                        }\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // MoE intermediate size (different from dense FFN)\n                        const int64_t n_ff_exp = hparams.n_ff_exp;\n\n                        // Kimi uses n_layer_dense_lead to determine which layers use dense FFN vs MoE\n                        // first_k_dense_replace = 1 means layer 0 uses dense FFN, layers 1+ use MoE\n                        if (i < (int) hparams.n_layer_dense_lead) {\n                            // Dense FFN layer - use normal n_ff\n                            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd, n_ff}, 0);\n                            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {n_ff, n_embd}, 0);\n                            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd, n_ff}, 0);\n                        } else {\n                            // MoE layer - use n_ff_exp (1024) instead of n_ff (9216)\n                            layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, \"weight\", i), {n_embd, n_expert}, 0);\n                            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff_exp, n_expert}, 0);\n                            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp, n_embd, n_expert}, 0);\n                            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff_exp, n_expert}, 0);\n\n                            // Shared experts use moe_intermediate_size * num_shared_experts\n                            // Kimi: shared_expert_intermediate_size = 1024 * 1 = 1024\n                            // Tensors are 2D: [n_embd, n_ff_shexp] or [n_ff_shexp, n_embd]\n                            const int64_t n_ff_shexp_actual = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1);\n                            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, n_ff_shexp_actual}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {n_ff_shexp_actual, n_embd}, TENSOR_NOT_REQUIRED);\n                            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, n_ff_shexp_actual}, TENSOR_NOT_REQUIRED);\n\n                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, 0);\n                        }\n                    }\n                } break;\n            case LLM_ARCH_COGVLM:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"weight\", i), {n_embd, n_embd_head_k * n_head * 3}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.visexp_attn_wqkv = create_tensor(tn(LLM_TENSOR_VISEXP_ATTN_QKV, \"weight\", i), {n_embd, n_embd_head_k * n_head * 3}, 0);\n                        layer.visexp_attn_wo = create_tensor(tn(LLM_TENSOR_VISEXP_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n\n                        layer.visexp_ffn_gate = create_tensor(tn(LLM_TENSOR_VISEXP_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.visexp_ffn_down = create_tensor(tn(LLM_TENSOR_VISEXP_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.visexp_ffn_up   = create_tensor(tn(LLM_TENSOR_VISEXP_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_PANGU_EMBED:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // weight tensors\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        // bias tensors\n                        layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"bias\", i), {n_embd_head_k * n_head}, 0);\n                        layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"bias\", i), {n_embd_gqa}, 0);\n                        layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"bias\", i), {n_embd_gqa}, 0);\n                        layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"bias\", i), {n_embd}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {\n                            layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        } else {\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN3NEXT:\n                {\n                    if (n_expert == 0) {\n                        throw std::runtime_error(arch_name() + \" model cannot have zero experts\");\n                    }\n\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, TENSOR_DUPLICATED);\n                    }\n\n                    const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                    // Calculate dimensions from hyperparameters\n                    const int64_t head_k_dim = hparams.ssm_d_state;\n                    const int64_t head_v_dim = hparams.ssm_d_state;\n                    const int64_t n_k_heads  = hparams.ssm_n_group;\n                    const int64_t n_v_heads  = hparams.ssm_dt_rank;\n                    const int64_t key_dim    = head_k_dim * n_k_heads;\n                    const int64_t value_dim  = head_v_dim * n_v_heads;\n                    const int64_t conv_dim   = key_dim * 2 + value_dim;\n\n                    // Calculate projection sizes\n                    const int64_t qkvz_dim = key_dim * 2 + value_dim * 2;\n                    const int64_t ba_dim   = n_v_heads * 2;\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n                        const uint32_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : hparams.n_ff(i);\n\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      \"weight\", i), { n_embd }, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), { n_embd }, 0);\n\n                        if (!hparams.is_recurrent(i)) {\n                            // Attention layers\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), { n_embd, n_embd_k_gqa }, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), { n_embd, n_embd_v_gqa }, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                            // Q/K normalization for attention layers\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                        } else {\n                            // Linear attention (gated delta net) specific tensors\n                            // Create tensors with calculated dimensions\n                            // note: ssm_in is used by legacy GGUF\n                            layer.ssm_in         = create_tensor(tn(LLM_TENSOR_SSM_IN,         \"weight\", i), { n_embd, qkvz_dim }, TENSOR_NOT_REQUIRED);\n                            layer.wqkv           = create_tensor(tn(LLM_TENSOR_ATTN_QKV,       \"weight\", i), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      \"weight\", i), { n_embd, value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.ssm_conv1d     = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,     \"weight\", i), { hparams.ssm_d_conv, conv_dim }, 0);\n                            layer.ssm_dt         = create_tensor(tn(LLM_TENSOR_SSM_DT,         \"bias\",   i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_a          = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN,             i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_beta_alpha = create_tensor(tn(LLM_TENSOR_SSM_BETA_ALPHA, \"weight\", i), { n_embd, ba_dim }, 0);\n                            layer.ssm_norm       = create_tensor(tn(LLM_TENSOR_SSM_NORM,       \"weight\", i), { head_v_dim }, 0);\n                            layer.ssm_out        = create_tensor(tn(LLM_TENSOR_SSM_OUT,        \"weight\", i), { value_dim, n_embd }, 0);\n                        }\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), { n_embd, n_expert }, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), { n_ff_exp, n_embd, n_expert }, 0);\n                        create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);\n\n                        // Shared experts\n                        layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, \"weight\", i), { n_embd }, 0);\n                        layer.ffn_gate_shexp     = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP,     \"weight\", i), { n_embd, n_ff_shexp }, 0);\n                        layer.ffn_up_shexp       = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,       \"weight\", i), { n_embd, n_ff_shexp }, 0);\n                        layer.ffn_down_shexp     = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP,     \"weight\", i), { n_ff_shexp, n_embd }, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN35MOE:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, TENSOR_DUPLICATED);\n                    }\n\n                    const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used;\n\n                    // Calculate dimensions from hyperparameters\n                    const int64_t head_k_dim = hparams.ssm_d_state;\n                    const int64_t head_v_dim = hparams.ssm_d_state;\n                    const int64_t n_k_heads  = hparams.ssm_n_group;\n                    const int64_t n_v_heads  = hparams.ssm_dt_rank;\n                    const int64_t key_dim    = head_k_dim * n_k_heads;\n                    const int64_t value_dim  = head_v_dim * n_v_heads;\n                    const int64_t conv_dim   = key_dim * 2 + value_dim;\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      \"weight\", i), { n_embd }, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), { n_embd }, 0);\n\n                        if (!hparams.is_recurrent(i)) {\n                            // Attention layers\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), { n_embd, n_embd_k_gqa }, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), { n_embd, n_embd_v_gqa }, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                            // Q/K normalization for attention layers\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                        } else {\n                            // Linear attention (gated delta net) specific tensors\n                            // Create tensors with calculated dimensions\n                            layer.wqkv           = create_tensor(tn(LLM_TENSOR_ATTN_QKV,       \"weight\", i), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      \"weight\", i), { n_embd, value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.ssm_conv1d     = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,     \"weight\", i), { hparams.ssm_d_conv, conv_dim }, 0);\n                            layer.ssm_dt         = create_tensor(tn(LLM_TENSOR_SSM_DT,         \"bias\",   i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_a          = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN,             i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_beta       = create_tensor(tn(LLM_TENSOR_SSM_BETA,       \"weight\", i), { n_embd, n_v_heads }, 0);\n                            layer.ssm_alpha      = create_tensor(tn(LLM_TENSOR_SSM_ALPHA,      \"weight\", i), { n_embd, n_v_heads }, 0);\n                            layer.ssm_norm       = create_tensor(tn(LLM_TENSOR_SSM_NORM,       \"weight\", i), { head_v_dim }, 0);\n                            layer.ssm_out        = create_tensor(tn(LLM_TENSOR_SSM_OUT,        \"weight\", i), { value_dim, n_embd }, 0);\n                        }\n\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), { n_embd, n_expert }, 0);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), { n_ff_exp, n_embd, n_expert }, 0);\n                        create_tensor_gate_up_exps(layer, i, n_embd, n_ff_exp, n_expert, 0);\n\n                        // Shared experts\n                        const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;\n\n                        layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, \"weight\", i), { n_embd }, 0);\n                        layer.ffn_gate_shexp     = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP,     \"weight\", i), { n_embd, n_ff_shexp }, 0);\n                        layer.ffn_up_shexp       = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,       \"weight\", i), { n_embd, n_ff_shexp }, 0);\n                        layer.ffn_down_shexp     = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP,     \"weight\", i), { n_ff_shexp, n_embd }, 0);\n                    }\n                } break;\n            case LLM_ARCH_QWEN35:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), { n_embd }, 0);\n                    output = create_tensor(tn(LLM_TENSOR_OUTPUT, \"weight\"), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED);\n\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), { n_embd, n_vocab }, TENSOR_DUPLICATED);\n                    }\n\n                    // Calculate dimensions from hyperparameters\n                    const int64_t head_k_dim = hparams.ssm_d_state;\n                    const int64_t head_v_dim = hparams.ssm_d_state;\n                    const int64_t n_k_heads  = hparams.ssm_n_group;\n                    const int64_t n_v_heads  = hparams.ssm_dt_rank;\n                    const int64_t key_dim    = head_k_dim * n_k_heads;\n                    const int64_t value_dim  = head_v_dim * n_v_heads;\n                    const int64_t conv_dim   = key_dim * 2 + value_dim;\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      \"weight\", i), { n_embd }, 0);\n                        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, \"weight\", i), { n_embd }, 0);\n\n                        if (!hparams.is_recurrent(i)) {\n                            // Attention layers\n                            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), { n_embd, n_embd_head_k * n_head * 2 }, 0);\n                            layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), { n_embd, n_embd_k_gqa }, 0);\n                            layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), { n_embd, n_embd_v_gqa }, 0);\n                            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_k * n_head, n_embd }, 0);\n\n                            // Q/K normalization for attention layers\n                            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), { n_embd_head_k }, 0);\n                        } else {\n                            // Linear attention (gated delta net) specific tensors\n                            // Create tensors with calculated dimensions\n                            layer.wqkv           = create_tensor(tn(LLM_TENSOR_ATTN_QKV,       \"weight\", i), { n_embd, key_dim * 2 + value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.wqkv_gate      = create_tensor(tn(LLM_TENSOR_ATTN_GATE,      \"weight\", i), { n_embd, value_dim }, TENSOR_NOT_REQUIRED);\n                            layer.ssm_conv1d     = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,     \"weight\", i), { hparams.ssm_d_conv, conv_dim }, 0);\n                            layer.ssm_dt         = create_tensor(tn(LLM_TENSOR_SSM_DT,         \"bias\",   i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_a          = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN,             i), { hparams.ssm_dt_rank }, 0);\n                            layer.ssm_beta       = create_tensor(tn(LLM_TENSOR_SSM_BETA,       \"weight\", i), { n_embd, n_v_heads }, 0);\n                            layer.ssm_alpha      = create_tensor(tn(LLM_TENSOR_SSM_ALPHA,      \"weight\", i), { n_embd, n_v_heads }, 0);\n                            layer.ssm_norm       = create_tensor(tn(LLM_TENSOR_SSM_NORM,       \"weight\", i), { head_v_dim }, 0);\n                            layer.ssm_out        = create_tensor(tn(LLM_TENSOR_SSM_OUT,        \"weight\", i), { value_dim, n_embd }, 0);\n                        }\n\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            case LLM_ARCH_MIMO2:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n                        uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i);\n                        uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);\n                        uint32_t n_head = hparams.n_head(i);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, \"weight\", i), { n_embd, n_embd_head_k * n_head }, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, \"weight\", i), { n_embd, n_embd_k_gqa }, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, \"weight\", i), { n_embd, n_embd_v_gqa }, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), { n_embd_head_v * n_head, n_embd }, 0);\n\n                        layer.attn_norm  = create_tensor(tn(LLM_TENSOR_ATTN_NORM,  \"weight\", i), {n_embd}, 0);\n                        layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, \"weight\", i), {n_head}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // non-MoE branch\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n\n                        // MoE branch\n                        int64_t n_ff_exp = hparams.n_ff_exp;\n                        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_STEP35:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // STEP35 supports per-layer partial RoPE dims; rope factors are stored as a single shared tensor\n                    // (\"rope_freqs.weight\") and ggml uses only the first (n_rot_l/2) entries per layer.\n                    uint32_t n_rot_max = 0;\n                    for (int i = 0; i < n_layer; ++i) {\n                        n_rot_max = std::max(n_rot_max, hparams.n_rot(i));\n                    }\n                    if (n_rot_max == 0) {\n                        n_rot_max = n_rot;\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        const uint32_t n_head_l      = hparams.n_head(i);\n                        const uint32_t n_embd_k_gqa  = hparams.n_embd_k_gqa(i);\n                        const uint32_t n_embd_v_gqa  = hparams.n_embd_v_gqa(i);\n\n                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, TENSOR_NOT_REQUIRED);\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, TENSOR_NOT_REQUIRED);\n\n                        // optional rope factors (llama3) / longrope tensors\n                        if (hparams.rope_scaling_type_train == LLAMA_ROPE_SCALING_TYPE_LONGROPE) {\n                            layer.rope_long  = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_LONG,  \"weight\", i), {n_rot_max/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                            layer.rope_short = create_tensor(tn(LLM_TENSOR_ROPE_FACTORS_SHORT, \"weight\", i), {n_rot_max/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        } else {\n                            layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, \"weight\", i), {n_rot_max/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));\n                        }\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head_l}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_k_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_v_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_v * n_head_l, n_embd}, 0);\n\n                        // head-wise attention gate (Step35 self_attn.g_proj)\n                        layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, \"weight\", i), {n_embd, n_head_l}, TENSOR_NOT_REQUIRED);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        // dense MLP (leading dense blocks)\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED);\n\n                        // MoE routed experts + selection bias (router_bias)\n                        const int64_t n_ff_exp = hparams.n_ff_exp;\n                        layer.ffn_gate_inp      = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  \"weight\", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_gate_exps     = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"weight\", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_exps     = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"weight\", i), {n_ff_exp,   n_embd, n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_exps       = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   \"weight\", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_exp_probs_b   = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, \"bias\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n\n                        // shared expert MLP\n                        layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"weight\", i), {n_embd, hparams.n_ff_shexp}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   \"weight\", i), {n_embd, hparams.n_ff_shexp}, TENSOR_NOT_REQUIRED);\n                        layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"weight\", i), {hparams.n_ff_shexp, n_embd}, TENSOR_NOT_REQUIRED);\n                    }\n                } break;\n            case LLM_ARCH_MAINCODER:\n                {\n                    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, 0);\n\n                    // output\n                    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, \"weight\"), {n_embd}, 0);\n                    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      \"weight\"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);\n                    // if output is NULL, init from the input tok embed\n                    if (output == NULL) {\n                        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, \"weight\"), {n_embd, n_vocab}, TENSOR_DUPLICATED);\n                    }\n\n                    for (int i = 0; i < n_layer; ++i) {\n                        auto & layer = layers[i];\n\n                        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, \"weight\", i), {n_embd}, 0);\n\n                        layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"weight\", i), {n_embd, n_embd_head_k * n_head}, 0);\n                        layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"weight\", i), {n_embd, n_embd_gqa}, 0);\n                        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"weight\", i), {n_embd_head_k * n_head, n_embd}, 0);\n\n                        layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, \"weight\", i), {n_embd_head_k}, 0);\n                        layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, \"weight\", i), {n_embd_head_k}, 0);\n\n                        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, \"weight\", i), {n_embd}, 0);\n                        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"weight\", i), {n_embd,   n_ff}, 0);\n                        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"weight\", i), {  n_ff, n_embd}, 0);\n                        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   \"weight\", i), {n_embd,   n_ff}, 0);\n                    }\n                } break;\n            default:\n                throw std::runtime_error(\"unknown architecture\");\n        }\n\n        // generic pass: load optional per-tensor/per-expert \".scale\" tensors (e.g. NVFP4 scale2)\n        // this avoids having to add scale loading to every architecture\n        for (int i = 0; i < n_layer; ++i) {\n            auto & layer = layers[i];\n\n            // attention weight scales (per-tensor, shape {1})\n            if (!layer.wq_s && layer.wq) {\n                layer.wq_s = create_tensor(tn(LLM_TENSOR_ATTN_Q,   \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.wk_s && layer.wk) {\n                layer.wk_s = create_tensor(tn(LLM_TENSOR_ATTN_K,   \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.wv_s && layer.wv) {\n                layer.wv_s = create_tensor(tn(LLM_TENSOR_ATTN_V,   \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.wo_s && layer.wo) {\n                layer.wo_s = create_tensor(tn(LLM_TENSOR_ATTN_OUT, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.wqkv_s && layer.wqkv) {\n                layer.wqkv_s = create_tensor(tn(LLM_TENSOR_ATTN_QKV, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.wqkv_gate_s && layer.wqkv_gate) {\n                layer.wqkv_gate_s = create_tensor(tn(LLM_TENSOR_ATTN_GATE, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n\n            // dense FFN weight scales (per-tensor, shape {1})\n            if (!layer.ffn_gate_s && layer.ffn_gate) {\n                layer.ffn_gate_s = create_tensor(tn(LLM_TENSOR_FFN_GATE, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_down_s && layer.ffn_down) {\n                layer.ffn_down_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_up_s && layer.ffn_up) {\n                layer.ffn_up_s = create_tensor(tn(LLM_TENSOR_FFN_UP, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_gate_shexp_s && layer.ffn_gate_shexp) {\n                layer.ffn_gate_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_down_shexp_s && layer.ffn_down_shexp) {\n                layer.ffn_down_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_up_shexp_s && layer.ffn_up_shexp) {\n                layer.ffn_up_shexp_s = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n\n            // MoE expert weight scales (per-expert, shape {n_expert})\n            if (!layer.ffn_gate_exps_s && layer.ffn_gate_exps) {\n                layer.ffn_gate_exps_s = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, \"scale\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_down_exps_s && layer.ffn_down_exps) {\n                layer.ffn_down_exps_s = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, \"scale\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ffn_up_exps_s && layer.ffn_up_exps) {\n                layer.ffn_up_exps_s = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, \"scale\", i), {n_expert}, TENSOR_NOT_REQUIRED);\n            }\n\n            // recurrent / linear-attention weight scales (per-tensor, shape {1})\n            if (!layer.ssm_out_s && layer.ssm_out) {\n                layer.ssm_out_s = create_tensor(tn(LLM_TENSOR_SSM_OUT, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ssm_alpha_s && layer.ssm_alpha) {\n                layer.ssm_alpha_s = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n            if (!layer.ssm_beta_s && layer.ssm_beta) {\n                layer.ssm_beta_s = create_tensor(tn(LLM_TENSOR_SSM_BETA, \"scale\", i), {1}, TENSOR_NOT_REQUIRED);\n            }\n        }\n    }\n\n    ml.done_getting_tensors();\n\n    ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);\n    pimpl->mappings.reserve(ml.mappings.size());\n\n    // create the backend buffers\n    std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;\n    ctx_buf_maps.reserve(ml.ctx_map.size());\n\n    // Ensure we have enough capacity for the maximum backend buffer we will potentially create\n    const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();\n    pimpl->ctxs_bufs.reserve(n_max_backend_buffer);\n\n    for (auto & [buft, ctx_ptr] : ml.ctx_map) {\n        ggml_context * ctx = ctx_ptr.get();\n\n        // skip contexts without tensors\n        if (ggml_get_first_tensor(ctx) == nullptr) {\n            continue;\n        }\n\n        llama_buf_map buf_map;\n        buf_map.reserve(n_max_backend_buffer);\n\n        // check if it is possible to use buffer_from_host_ptr with this buffer type\n        ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);\n        if (!dev) {\n            // FIXME: workaround for CPU backend buft having a NULL device\n            dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n            if (!dev) {\n                throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n            }\n        }\n        ggml_backend_dev_props props;\n        ggml_backend_dev_get_props(dev, &props);\n        bool buffer_from_host_ptr_supported = props.caps.buffer_from_host_ptr;\n        bool is_default_buft = buft == ggml_backend_dev_buffer_type(dev);\n\n        std::vector<ggml_backend_buffer_ptr> bufs;\n        if (ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {\n            GGML_ASSERT(!ml.no_alloc);\n            for (uint32_t idx = 0; idx < ml.files.size(); idx++) {\n                // only the mmap region containing the tensors in the model is mapped to the backend buffer\n                // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer,\n                //     then we could just use metal for all layers\n                // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size\n                void * addr = nullptr;\n                size_t first, last; // NOLINT\n                ml.get_mapping_range(&first, &last, &addr, idx, ctx);\n                if (first >= last) {\n                    continue;\n                }\n                const size_t max_size = ggml_get_max_tensor_size(ctx);\n                ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);\n                if (buf == nullptr) {\n                    throw std::runtime_error(format(\"unable to allocate %s buffer\", ggml_backend_buft_name(buft)));\n                }\n                bufs.emplace_back(buf);\n                buf_map.emplace(idx, buf);\n            }\n        } else {\n            ggml_backend_buffer_t buf;\n            if (ml.no_alloc) {\n                buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer\n                for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {\n                    t->buffer = buf; // set dummy buffer for weights so that the backend scheduler won't try to allocate them\n                }\n            } else {\n                buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft); // real buffer\n            }\n            if (buf == nullptr) {\n                throw std::runtime_error(format(\"unable to allocate %s buffer\", ggml_backend_buft_name(buft)));\n            }\n            if (use_mlock && ggml_backend_buffer_is_host(buf)) {\n                pimpl->mlock_bufs.emplace_back(new llama_mlock);\n                auto & mlock_buf = pimpl->mlock_bufs.back();\n                mlock_buf->init   (ggml_backend_buffer_get_base(buf));\n                mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));\n            }\n            bufs.emplace_back(buf);\n            for (uint32_t idx = 0; idx < ml.files.size(); idx++) {\n                buf_map.emplace(idx, buf);\n            }\n        }\n        pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));\n\n        for (auto & buf : buf_map) {\n            // indicate that this buffer contains weights\n            // this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight\n            ggml_backend_buffer_set_usage(buf.second, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);\n        }\n\n        ctx_buf_maps.emplace_back(ctx, buf_map);\n    }\n\n    if (llama_supports_gpu_offload()) {\n        const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));\n\n        int n_repeating = n_gpu;\n        if (n_repeating > 0) {\n            LLAMA_LOG_INFO(\"%s: offloading output layer to GPU\\n\", __func__);\n            n_repeating--;\n        }\n        LLAMA_LOG_INFO(\"%s: offloading %d repeating layers to GPU\\n\", __func__, n_repeating);\n\n        const int max_backend_supported_layers = hparams.n_layer + 1;\n        const int max_offloadable_layers       = hparams.n_layer + 1;\n\n        LLAMA_LOG_INFO(\"%s: offloaded %d/%d layers to GPU\\n\", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);\n    }\n\n    // print memory requirements per buffer type\n    for (auto & [_, bufs] : pimpl->ctxs_bufs) {\n        for (auto & buf: bufs) {\n            LLAMA_LOG_INFO(\"%s: %12s model buffer size = %8.2f MiB\\n\",\n                __func__, ggml_backend_buffer_name(buf.get()), ggml_backend_buffer_get_size(buf.get()) / 1024.0 / 1024.0);\n        }\n    }\n\n    // populate tensors_by_name\n    for (auto & [ctx, _] : pimpl->ctxs_bufs) {\n        for (auto * cur = ggml_get_first_tensor(ctx.get()); cur != NULL; cur = ggml_get_next_tensor(ctx.get(), cur)) {\n            tensors_by_name.emplace_back(ggml_get_name(cur), cur);\n        }\n    }\n\n    if (ml.no_alloc) {\n        return true;\n    }\n\n    // load tensor data\n    for (auto & [ctx, buf_map] : ctx_buf_maps) {\n        if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {\n            return false;\n        }\n    }\n\n    if (use_mmap_buffer) {\n        for (auto & mapping : ml.mappings) {\n            pimpl->mappings.emplace_back(std::move(mapping));\n        }\n    }\n\n    return true;\n}\n\nstd::string llama_model::arch_name() const {\n    return llm_arch_name(arch);\n}\n\nstd::string llama_model::type_name() const {\n    return llm_type_name(type);\n}\n\nstd::string llama_model::desc() const {\n    return pimpl->desc_str;\n}\n\nsize_t llama_model::size() const {\n    return pimpl->n_bytes;\n}\n\nsize_t llama_model::n_tensors() const {\n    return tensors_by_name.size();\n}\n\nsize_t llama_model::n_devices() const {\n    return devices.size();\n}\n\nuint32_t llama_model::n_gpu_layers() const {\n    return params.n_gpu_layers >= 0 ? params.n_gpu_layers : hparams.n_layer + 1;\n}\n\nllama_split_mode llama_model::split_mode() const {\n    return params.split_mode;\n}\n\nstd::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() const {\n    std::map<ggml_backend_buffer_type_t, size_t> ret;\n    for (const auto & [ctx, bufs] : pimpl->ctxs_bufs) {\n        if (hparams.no_alloc) {\n            GGML_ASSERT(bufs.size() == 1);\n            ggml_backend_buffer_t buf = bufs[0].get();\n            GGML_ASSERT(ggml_backend_buffer_get_base(buf) == nullptr);\n            ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf);\n            ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);\n        } else {\n            for (const auto & buf : bufs) {\n                // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base\n                ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());\n            }\n        }\n    }\n    return ret;\n}\n\nuint64_t llama_model::n_elements() const {\n    return pimpl->n_elements;\n}\n\nvoid llama_model::print_info() const {\n    const std::string rope_scaling_type = llama_rope_scaling_type_name(hparams.rope_scaling_type_train);\n\n    auto print_f = [](const std::function<uint32_t(uint32_t)> & f, uint32_t n) {\n        bool is_var = false;\n\n        std::vector<uint32_t> v;\n        for (uint32_t i = 0; i < n; ++i) {\n            v.push_back(f(i));\n            if (v[i] != v[0]) {\n                is_var = true;\n            }\n        }\n\n        std::stringstream ss;\n\n        if (is_var) {\n            ss << \"[\";\n            for (uint32_t i = 0; i < n; ++i) {\n                ss << v[i];\n                if (i < n - 1) {\n                    ss << \", \";\n                }\n            }\n            ss << \"]\";\n        } else {\n            ss << v[0];\n        }\n\n        return ss.str();\n    };\n\n    // hparams\n    LLAMA_LOG_INFO(\"%s: arch                  = %s\\n\",     __func__, arch_name().c_str());\n    LLAMA_LOG_INFO(\"%s: vocab_only            = %d\\n\",     __func__, hparams.vocab_only);\n    LLAMA_LOG_INFO(\"%s: no_alloc              = %d\\n\",     __func__, hparams.no_alloc);\n\n    if (!hparams.vocab_only) {\n        LLAMA_LOG_INFO(\"%s: n_ctx_train           = %u\\n\",     __func__, hparams.n_ctx_train);\n        LLAMA_LOG_INFO(\"%s: n_embd                = %u\\n\",     __func__, hparams.n_embd);\n        LLAMA_LOG_INFO(\"%s: n_embd_inp            = %u\\n\",     __func__, hparams.n_embd_inp());\n        LLAMA_LOG_INFO(\"%s: n_layer               = %u\\n\",     __func__, hparams.n_layer);\n        LLAMA_LOG_INFO(\"%s: n_head                = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_head(il);    }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: n_head_kv             = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_head_kv(il); }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: n_rot                 = %u\\n\",     __func__, hparams.n_rot_full);\n        LLAMA_LOG_INFO(\"%s: n_swa                 = %u\\n\",     __func__, hparams.n_swa);\n        LLAMA_LOG_INFO(\"%s: is_swa_any            = %u\\n\",     __func__, hparams.is_swa_any());\n        LLAMA_LOG_INFO(\"%s: n_embd_head_k         = %u\\n\",     __func__, hparams.n_embd_head_k_full);\n        LLAMA_LOG_INFO(\"%s: n_embd_head_v         = %u\\n\",     __func__, hparams.n_embd_head_v_full);\n        LLAMA_LOG_INFO(\"%s: n_gqa                 = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_gqa(il);        }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: n_embd_k_gqa          = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_embd_k_gqa(il); }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: n_embd_v_gqa          = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_embd_v_gqa(il); }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: f_norm_eps            = %.1e\\n\",   __func__, hparams.f_norm_eps);\n        LLAMA_LOG_INFO(\"%s: f_norm_rms_eps        = %.1e\\n\",   __func__, hparams.f_norm_rms_eps);\n        LLAMA_LOG_INFO(\"%s: f_clamp_kqv           = %.1e\\n\",   __func__, hparams.f_clamp_kqv);\n        LLAMA_LOG_INFO(\"%s: f_max_alibi_bias      = %.1e\\n\",   __func__, hparams.f_max_alibi_bias);\n        LLAMA_LOG_INFO(\"%s: f_logit_scale         = %.1e\\n\",   __func__, hparams.f_logit_scale);\n        LLAMA_LOG_INFO(\"%s: f_attn_scale          = %.1e\\n\",   __func__, hparams.f_attention_scale);\n        LLAMA_LOG_INFO(\"%s: n_ff                  = %s\\n\",     __func__, print_f([&](uint32_t il) { return hparams.n_ff(il); }, hparams.n_layer).c_str());\n        LLAMA_LOG_INFO(\"%s: n_expert              = %u\\n\",     __func__, hparams.n_expert);\n        LLAMA_LOG_INFO(\"%s: n_expert_used         = %u\\n\",     __func__, hparams.n_expert_used);\n        LLAMA_LOG_INFO(\"%s: n_expert_groups       = %d\\n\",     __func__, hparams.n_expert_groups);\n        LLAMA_LOG_INFO(\"%s: n_group_used          = %d\\n\",     __func__, hparams.n_group_used);\n        LLAMA_LOG_INFO(\"%s: causal attn           = %d\\n\",     __func__, hparams.causal_attn);\n        LLAMA_LOG_INFO(\"%s: pooling type          = %d\\n\",     __func__, hparams.pooling_type);\n        LLAMA_LOG_INFO(\"%s: rope type             = %d\\n\",     __func__, hparams.rope_type);\n        LLAMA_LOG_INFO(\"%s: rope scaling          = %s\\n\",     __func__, rope_scaling_type.c_str());\n        LLAMA_LOG_INFO(\"%s: freq_base_train       = %.1f\\n\",   __func__, hparams.rope_freq_base_train);\n        LLAMA_LOG_INFO(\"%s: freq_scale_train      = %g\\n\",     __func__, hparams.rope_freq_scale_train);\n        if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n            LLAMA_LOG_INFO(\"%s: freq_base_swa         = %.1f\\n\",   __func__, hparams.rope_freq_base_train_swa);\n            LLAMA_LOG_INFO(\"%s: freq_scale_swa        = %g\\n\",     __func__, hparams.rope_freq_scale_train_swa);\n            LLAMA_LOG_INFO(\"%s: n_embd_head_k_swa     = %u\\n\",     __func__, hparams.n_embd_head_k_swa);\n            LLAMA_LOG_INFO(\"%s: n_embd_head_v_swa     = %u\\n\",     __func__, hparams.n_embd_head_v_swa);\n            LLAMA_LOG_INFO(\"%s: n_rot_swa             = %u\\n\",     __func__, hparams.n_rot_swa);\n        }\n        LLAMA_LOG_INFO(\"%s: n_ctx_orig_yarn       = %u\\n\",     __func__, hparams.n_ctx_orig_yarn);\n        LLAMA_LOG_INFO(\"%s: rope_yarn_log_mul     = %.4f\\n\",   __func__, hparams.rope_yarn_log_mul);\n        LLAMA_LOG_INFO(\"%s: rope_finetuned        = %s\\n\",     __func__, hparams.rope_finetuned ? \"yes\" : \"unknown\");\n        // MRoPE (Multi-axis Rotary Position Embedding) sections\n        if (const auto & s = hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) {\n            LLAMA_LOG_INFO(\"%s: mrope sections        = [%d, %d, %d, %d]\\n\", __func__, s[0], s[1], s[2], s[3]);\n        }\n        if (!classifier_labels.empty()) {\n            LLAMA_LOG_INFO(\"%s: n_cls_out             = %u\\n\", __func__, hparams.n_cls_out);\n\n            size_t i = 0;\n            for (auto label : classifier_labels) {\n                LLAMA_LOG_INFO(\"%s: cls_label[%2zu]         = %s\\n\", __func__, i++, label.c_str());\n            }\n        }\n    }\n\n    if (arch == LLM_ARCH_MAMBA ||\n        arch == LLM_ARCH_MAMBA2 ||\n        arch == LLM_ARCH_JAMBA ||\n        arch == LLM_ARCH_FALCON_H1 ||\n        arch == LLM_ARCH_PLAMO2 ||\n        arch == LLM_ARCH_GRANITE_HYBRID ||\n        arch == LLM_ARCH_QWEN3NEXT ||\n        arch == LLM_ARCH_QWEN35 ||\n        arch == LLM_ARCH_QWEN35MOE ||\n        arch == LLM_ARCH_NEMOTRON_H ||\n        arch == LLM_ARCH_NEMOTRON_H_MOE) {\n        LLAMA_LOG_INFO(\"%s: ssm_d_conv            = %u\\n\",     __func__, hparams.ssm_d_conv);\n        LLAMA_LOG_INFO(\"%s: ssm_d_inner           = %u\\n\",     __func__, hparams.ssm_d_inner);\n        LLAMA_LOG_INFO(\"%s: ssm_d_state           = %u\\n\",     __func__, hparams.ssm_d_state);\n        LLAMA_LOG_INFO(\"%s: ssm_dt_rank           = %u\\n\",     __func__, hparams.ssm_dt_rank);\n        LLAMA_LOG_INFO(\"%s: ssm_n_group           = %u\\n\",     __func__, hparams.ssm_n_group);\n        LLAMA_LOG_INFO(\"%s: ssm_dt_b_c_rms        = %d\\n\",     __func__, hparams.ssm_dt_b_c_rms);\n    }\n\n    LLAMA_LOG_INFO(\"%s: model type            = %s\\n\",     __func__, type_name().c_str());\n    if (pimpl->n_elements >= 1e12) {\n        LLAMA_LOG_INFO(\"%s: model params          = %.2f T\\n\", __func__, pimpl->n_elements*1e-12);\n    } else if (pimpl->n_elements >= 1e9) {\n        LLAMA_LOG_INFO(\"%s: model params          = %.2f B\\n\", __func__, pimpl->n_elements*1e-9);\n    } else if (pimpl->n_elements >= 1e6) {\n        LLAMA_LOG_INFO(\"%s: model params          = %.2f M\\n\", __func__, pimpl->n_elements*1e-6);\n    } else {\n        LLAMA_LOG_INFO(\"%s: model params          = %.2f K\\n\", __func__, pimpl->n_elements*1e-3);\n    }\n\n    // general kv\n    LLAMA_LOG_INFO(\"%s: general.name          = %s\\n\",    __func__, name.c_str());\n\n    if (arch == LLM_ARCH_DEEPSEEK) {\n        LLAMA_LOG_INFO(\"%s: n_layer_dense_lead    = %d\\n\",     __func__, hparams.n_layer_dense_lead);\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_expert_shared       = %d\\n\",     __func__, hparams.n_expert_shared);\n        LLAMA_LOG_INFO(\"%s: expert_weights_scale  = %.1f\\n\",   __func__, hparams.expert_weights_scale);\n    }\n\n    if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_GLM_DSA) {\n        LLAMA_LOG_INFO(\"%s: n_layer_dense_lead    = %d\\n\",     __func__, hparams.n_layer_dense_lead);\n        LLAMA_LOG_INFO(\"%s: n_lora_q              = %d\\n\",     __func__, hparams.n_lora_q);\n        LLAMA_LOG_INFO(\"%s: n_lora_kv             = %d\\n\",     __func__, hparams.n_lora_kv);\n        LLAMA_LOG_INFO(\"%s: n_embd_head_k_mla     = %d\\n\",     __func__, hparams.n_embd_head_k_mla());\n        LLAMA_LOG_INFO(\"%s: n_embd_head_v_mla     = %d\\n\",     __func__, hparams.n_embd_head_v_mla());\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_expert_shared       = %d\\n\",     __func__, hparams.n_expert_shared);\n        LLAMA_LOG_INFO(\"%s: expert_weights_scale  = %.1f\\n\",   __func__, hparams.expert_weights_scale);\n        LLAMA_LOG_INFO(\"%s: expert_weights_norm   = %d\\n\",     __func__, hparams.expert_weights_norm);\n        LLAMA_LOG_INFO(\"%s: expert_gating_func    = %s\\n\",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));\n    }\n\n    if (arch == LLM_ARCH_QWEN2MOE) {\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_ff_shexp            = %d\\n\",     __func__, hparams.n_ff_shexp);\n    }\n\n    if (arch == LLM_ARCH_QWEN3MOE || arch == LLM_ARCH_OPENAI_MOE || arch == LLM_ARCH_QWEN3VLMOE || arch == LLM_ARCH_RND1) {\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n    }\n\n    if (arch == LLM_ARCH_MINICPM ||\n        arch == LLM_ARCH_GRANITE ||\n        arch == LLM_ARCH_GRANITE_MOE ||\n        arch == LLM_ARCH_GRANITE_HYBRID ||\n        arch == LLM_ARCH_NEMOTRON_H_MOE) {\n        LLAMA_LOG_INFO(\"%s: f_embedding_scale     = %f\\n\", __func__, hparams.f_embedding_scale);\n        LLAMA_LOG_INFO(\"%s: f_residual_scale      = %f\\n\", __func__, hparams.f_residual_scale);\n        LLAMA_LOG_INFO(\"%s: f_attention_scale     = %f\\n\", __func__, hparams.f_attention_scale);\n        LLAMA_LOG_INFO(\"%s: n_ff_shexp            = %d\\n\", __func__, hparams.n_ff_shexp);\n    }\n\n    if (arch == LLM_ARCH_BAILINGMOE) {\n        LLAMA_LOG_INFO(\"%s: n_layer_dense_lead    = %d\\n\",     __func__, hparams.n_layer_dense_lead);\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_expert_shared       = %d\\n\",     __func__, hparams.n_expert_shared);\n        LLAMA_LOG_INFO(\"%s: expert_weights_scale  = %.1f\\n\",   __func__, hparams.expert_weights_scale);\n        LLAMA_LOG_INFO(\"%s: expert_weights_norm   = %d\\n\",     __func__, hparams.expert_weights_norm);\n    }\n\n    if (arch == LLM_ARCH_BAILINGMOE2) {\n        LLAMA_LOG_INFO(\"%s: n_layer_dense_lead    = %d\\n\",     __func__, hparams.n_layer_dense_lead);\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_ff_shexp            = %d\\n\",     __func__, hparams.n_ff_shexp);\n        LLAMA_LOG_INFO(\"%s: n_expert_shared       = %d\\n\",     __func__, hparams.n_expert_shared);\n        LLAMA_LOG_INFO(\"%s: expert_weights_scale  = %.1f\\n\",   __func__, hparams.expert_weights_scale);\n        LLAMA_LOG_INFO(\"%s: expert_weights_norm   = %d\\n\",     __func__, hparams.expert_weights_norm);\n        LLAMA_LOG_INFO(\"%s: expert_gating_func    = %s\\n\",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));\n        LLAMA_LOG_INFO(\"%s: nextn_predict_layers  = %d\\n\",     __func__, hparams.nextn_predict_layers);\n    }\n\n    if (arch == LLM_ARCH_SMALLTHINKER || arch == LLM_ARCH_LFM2MOE) {\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: expert_gating_func    = %s\\n\",     __func__, llama_expert_gating_func_name((llama_expert_gating_func_type) hparams.expert_gating_func));\n    }\n\n    if (arch == LLM_ARCH_GROVEMOE) {\n        LLAMA_LOG_INFO(\"%s: n_ff_exp              = %d\\n\",     __func__, hparams.n_ff_exp);\n        LLAMA_LOG_INFO(\"%s: n_ff_chexp            = %d\\n\",     __func__, hparams.n_ff_chexp);\n        LLAMA_LOG_INFO(\"%s: n_group_experts       = %d\\n\",     __func__, hparams.n_group_experts);\n        LLAMA_LOG_INFO(\"%s: expert_group_scale    = %.2f\\n\",   __func__, hparams.expert_group_scale);\n    }\n\n    vocab.print_info();\n}\n\nggml_backend_dev_t llama_model::dev_layer(int il) const {\n    return pimpl->dev_layer.at(il).dev;\n}\n\nggml_backend_dev_t llama_model::dev_output() const {\n    return pimpl->dev_output.dev;\n}\n\ntemplate<typename F>\nstatic bool buft_supported(ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev, F & fn) {\n    ggml_init_params params = {\n        /*.mem_size   =*/ ggml_tensor_overhead()*8,\n        /*.mem_buffer =*/ NULL,\n        /*.no_alloc   =*/ true,\n    };\n\n    ggml_context_ptr ctx { ggml_init(params) };\n    if (!ctx) {\n        throw std::runtime_error(format(\"failed to create ggml context\"));\n    }\n\n    ggml_backend_buffer_ptr buf { ggml_backend_buft_alloc_buffer(buft, 0) };\n    ggml_tensor * op_tensor = fn(ctx.get());\n    for (int i = 0; i < GGML_MAX_SRC; i++) {\n        if (op_tensor->src[i] != nullptr) {\n            assert(op_tensor->src[i]->buffer == nullptr);\n            op_tensor->src[i]->buffer = buf.get();\n        }\n    }\n\n    bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);\n\n    return op_supported;\n}\n\ntemplate<typename F>\nstatic ggml_backend_buffer_type_t select_buft(const buft_list_t & buft_list, const F & fn) {\n    for (const auto & cur : buft_list) {\n        ggml_backend_dev_t cur_dev = cur.first;\n        ggml_backend_buffer_type_t cur_buft = cur.second;\n        if (buft_supported(cur_buft, cur_dev, fn)) {\n            return cur_buft;\n        }\n    }\n\n    throw std::runtime_error(format(\"no suitable buffer type found\"));\n}\n\nggml_backend_buffer_type_t llama_model::select_buft(int il) const {\n    return ::select_buft(\n            *pimpl->dev_layer.at(il).buft_list,\n            [&](ggml_context * ctx) {\n                ggml_tensor * cur = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);\n                ggml_tensor * layer_dir = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hparams.n_embd);\n                return ggml_add(ctx, cur, layer_dir);\n            });\n}\n\nbool llama_model::has_tensor_overrides() const {\n    return pimpl->has_tensor_overrides;\n}\n\nconst ggml_tensor * llama_model::get_tensor(const char * name) const {\n    auto it = std::find_if(tensors_by_name.begin(), tensors_by_name.end(),\n            [name](const std::pair<std::string, ggml_tensor *> & it) {\n                return it.first == name;\n            });\n    if (it == tensors_by_name.end()) {\n        return nullptr;\n    }\n\n    return it->second;\n}\n\nfloat llama_model::get_rope_freq_base (const llama_cparams & cparams, int il) const {\n    return hparams.is_swa(il) ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base;\n}\n\nfloat llama_model::get_rope_freq_scale(const llama_cparams & cparams, int il) const {\n    return hparams.is_swa(il) ? hparams.rope_freq_scale_train_swa : cparams.rope_freq_scale;\n}\n\nggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int il) const {\n    const uint32_t n_ctx_seq = cparams.n_ctx_seq;\n\n    // choose long/short freq factors based on the context size\n    if (layers[il].rope_freqs != nullptr) {\n        return layers[il].rope_freqs;\n    }\n\n    if (n_ctx_seq > hparams.n_ctx_orig_yarn) {\n        return layers[il].rope_long;\n    }\n\n    return layers[il].rope_short;\n}\n\nllama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const {\n    llama_memory_i * res;\n\n    switch (arch) {\n        // Models that need specific instantiation should be handled in the\n        // switch statement\n        case LLM_ARCH_BERT:\n        case LLM_ARCH_JINA_BERT_V2:\n        case LLM_ARCH_JINA_BERT_V3:\n        case LLM_ARCH_NOMIC_BERT:\n        case LLM_ARCH_NOMIC_BERT_MOE:\n        case LLM_ARCH_NEO_BERT:\n        case LLM_ARCH_EUROBERT:\n        case LLM_ARCH_WAVTOKENIZER_DEC:\n        case LLM_ARCH_MODERN_BERT:\n        case LLM_ARCH_GEMMA_EMBEDDING:\n        case LLM_ARCH_DREAM:\n        case LLM_ARCH_LLADA:\n        case LLM_ARCH_LLADA_MOE:\n        case LLM_ARCH_RND1:\n            {\n                res = nullptr;\n            } break;\n        // Models that need standard caching should rely on recurrent/hybrid\n        // checks\n        default:\n            {\n                if (llm_arch_is_recurrent(arch)) {\n                    res = new llama_memory_recurrent(\n                            *this,\n                            GGML_TYPE_F32,\n                            GGML_TYPE_F32,\n                            cparams.offload_kqv,\n                            std::max((uint32_t) 1, cparams.n_seq_max),\n                            cparams.n_seq_max,\n                            nullptr);\n                } else if (llm_arch_is_hybrid(arch)) {\n                    // The main difference between hybrid architectures is the\n                    // layer filters, so pick the right one here\n                    llama_memory_hybrid::layer_filter_cb filter_attn = nullptr;\n                    llama_memory_hybrid::layer_filter_cb filter_recr = nullptr;\n                    if (arch == LLM_ARCH_FALCON_H1) {\n                        filter_attn = [&](int32_t) { return true; };\n                        filter_recr = [&](int32_t) { return true; };\n                    } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {\n                        filter_attn = [&](int32_t il) {\n                            return !hparams.is_recurrent(il) && hparams.n_ff(il) == 0;\n                        };\n                        filter_recr = [&](int32_t il) {\n                            return hparams.is_recurrent(il) && hparams.n_ff(il) == 0;\n                        };\n                    }\n\n                    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n                        // Use hybrid-iswa for hybrid models with SWA\n                        res = new llama_memory_hybrid_iswa(\n                            /* model             */ *this,\n                            /* attn_type_k       */ params.type_k,\n                            /* attn_type_v       */ params.type_v,\n                            /* attn_v_trans      */ !cparams.flash_attn,\n                            /* attn_swa_full     */ params.swa_full,\n                            /* attn_kv_size      */ cparams.n_ctx_seq,\n                            /* attn_n_ubatch     */ cparams.n_ubatch,\n                            /* attn_n_pad        */ 1,\n                            /* recurrent_type_r  */ GGML_TYPE_F32,\n                            /* recurrent_type_s  */ GGML_TYPE_F32,\n                            /* recurrent_rs_size */ std::max((uint32_t) 1, cparams.n_seq_max),\n                            /* n_seq_max         */ cparams.n_seq_max,\n                            /* offload           */ cparams.offload_kqv,\n                            /* unified           */ cparams.kv_unified,\n                            /* filter_attn       */ std::move(filter_attn),\n                            /* filter_recr       */ std::move(filter_recr));\n                    } else {\n                        res = new llama_memory_hybrid(\n                            /* model             */ *this,\n                            /* attn_type_k       */ params.type_k,\n                            /* attn_type_v       */ params.type_v,\n                            /* attn_v_trans      */ !cparams.flash_attn,\n                            /* attn_kv_size      */ cparams.n_ctx_seq,\n                            /* attn_n_pad        */ 1,\n                            /* attn_n_swa        */ hparams.n_swa,\n                            /* attn_swa_type     */ hparams.swa_type,\n                            /* recurrent_type_k  */ GGML_TYPE_F32,\n                            /* recurrent_type_v  */ GGML_TYPE_F32,\n                            /* recurrent_kv_size */ std::max((uint32_t) 1, cparams.n_seq_max),\n                            /* n_seq_max         */ cparams.n_seq_max,\n                            /* offload           */ cparams.offload_kqv,\n                            /* unified           */ cparams.kv_unified,\n                            /* filter_attn       */ std::move(filter_attn),\n                            /* filter_recr       */ std::move(filter_recr));\n                    }\n                } else {\n                    llama_memory_i::layer_reuse_cb reuse = nullptr;\n\n                    if (arch == LLM_ARCH_GEMMA3N) {\n                        reuse = [&](int32_t il) {\n                            if (il >= (int32_t) hparams.n_layer_kv_from_start) {\n                                return (int32_t) hparams.n_layer_kv_from_start - (hparams.is_swa(il) ? 2 : 1);\n                            }\n\n                            return -1;\n                        };\n                    }\n\n                    if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n                        GGML_ASSERT(hparams.is_swa_any());\n\n                        res = new llama_kv_cache_iswa(\n                                *this,\n                                params.type_k,\n                                params.type_v,\n                                !cparams.flash_attn,\n                                cparams.offload_kqv,\n                                params.swa_full,\n                                cparams.kv_unified,\n                                cparams.n_ctx_seq,\n                                cparams.n_seq_max,\n                                cparams.n_ubatch,\n                                1,\n                                nullptr,\n                                reuse);\n                    } else {\n                        GGML_ASSERT(!hparams.is_swa_any());\n\n                        res = new llama_kv_cache(\n                                *this,\n                                params.type_k,\n                                params.type_v,\n                                !cparams.flash_attn,\n                                cparams.offload_kqv,\n                                cparams.kv_unified,\n                                cparams.n_ctx_seq,\n                                cparams.n_seq_max,\n                                1,\n                                hparams.n_swa,\n                                hparams.swa_type,\n                                nullptr,\n                                nullptr);\n                    }\n                }\n            }\n    }\n\n    return res;\n}\n\nggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const {\n    std::unique_ptr<llm_graph_context> llm;\n\n    switch (arch) {\n        case LLM_ARCH_LLAMA:\n            {\n                llm = std::make_unique<llm_build_llama<false>>(*this, params);\n            } break;\n        case LLM_ARCH_LLAMA4:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_NONE) {\n                    llm = std::make_unique<llm_build_llama<false>>(*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_llama_iswa>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_LLAMA_EMBED:\n            {\n                llm = std::make_unique<llm_build_llama<true>>(*this, params);\n            } break;\n        case LLM_ARCH_MAINCODER:\n            {\n                llm = std::make_unique<llm_build_maincoder>(*this, params);\n            } break;\n        case LLM_ARCH_DECI:\n            {\n                llm = std::make_unique<llm_build_deci>(*this, params);\n            } break;\n        case LLM_ARCH_BAICHUAN:\n            {\n                llm = std::make_unique<llm_build_baichuan>(*this, params);\n            } break;\n        case LLM_ARCH_FALCON:\n            {\n                llm = std::make_unique<llm_build_falcon>(*this, params);\n            } break;\n        case LLM_ARCH_GROK:\n            {\n                llm = std::make_unique<llm_build_grok>(*this, params);\n            } break;\n        case LLM_ARCH_STARCODER:\n            {\n                llm = std::make_unique<llm_build_starcoder>(*this, params);\n            } break;\n        case LLM_ARCH_REFACT:\n            {\n                llm = std::make_unique<llm_build_refact>(*this, params);\n            } break;\n        case LLM_ARCH_BERT:\n        case LLM_ARCH_JINA_BERT_V2:\n        case LLM_ARCH_JINA_BERT_V3:\n        case LLM_ARCH_NOMIC_BERT:\n        case LLM_ARCH_NOMIC_BERT_MOE:\n            {\n                llm = std::make_unique<llm_build_bert>(*this, params);\n            } break;\n        case LLM_ARCH_MODERN_BERT:\n            {\n                llm = std::make_unique<llm_build_modern_bert>(*this, params);\n            } break;\n        case LLM_ARCH_NEO_BERT:\n            {\n                llm = std::make_unique<llm_build_neo_bert>(*this, params);\n            } break;\n        case LLM_ARCH_EUROBERT:\n            {\n                llm = std::make_unique<llm_build_eurobert>(*this, params);\n            } break;\n        case LLM_ARCH_BLOOM:\n            {\n                llm = std::make_unique<llm_build_bloom>(*this, params);\n            } break;\n        case LLM_ARCH_MPT:\n            {\n                llm = std::make_unique<llm_build_mpt>(*this, params);\n            } break;\n        case LLM_ARCH_STABLELM:\n            {\n                llm = std::make_unique<llm_build_stablelm>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN:\n            {\n                llm = std::make_unique<llm_build_qwen>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN2:\n            {\n                llm = std::make_unique<llm_build_qwen2>(*this, params);\n            } break;\n        case LLM_ARCH_DREAM:\n            {\n                llm = std::make_unique<llm_build_dream>(*this, params);\n            }\n            break;\n        case LLM_ARCH_LLADA:\n            {\n                llm = std::make_unique<llm_build_llada>(*this, params);\n            }\n            break;\n        case LLM_ARCH_LLADA_MOE:\n            {\n                llm = std::make_unique<llm_build_llada_moe>(*this, params);\n            }\n            break;\n        case LLM_ARCH_RND1:\n            {\n                llm = std::make_unique<llm_build_rnd1>(*this, params);\n            }\n            break;\n        case LLM_ARCH_QWEN2VL:\n            {\n                llm = std::make_unique<llm_build_qwen2vl>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN2MOE:\n            {\n                llm = std::make_unique<llm_build_qwen2moe>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN3:\n            {\n                llm = std::make_unique<llm_build_qwen3>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN3MOE:\n            {\n                llm = std::make_unique<llm_build_qwen3moe>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN3VL:\n            {\n                llm = std::make_unique<llm_build_qwen3vl>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN3VLMOE:\n            {\n                llm = std::make_unique<llm_build_qwen3vlmoe>(*this, params);\n            } break;\n        case LLM_ARCH_PHI2:\n            {\n                llm = std::make_unique<llm_build_phi2>(*this, params);\n            } break;\n        case LLM_ARCH_PHI3:\n        case LLM_ARCH_PHIMOE:\n            {\n                if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n                    llm = std::make_unique<llm_build_phi3<true>> (*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_phi3<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_PLAMO:\n            {\n                llm = std::make_unique<llm_build_plamo>(*this, params);\n            } break;\n        case LLM_ARCH_PLAMO2:\n            {\n                llm = std::make_unique<llm_build_plamo2>(*this, params);\n            } break;\n        case LLM_ARCH_PLAMO3:\n            {\n                if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {\n                    llm = std::make_unique<llm_build_plamo3<true>> (*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_plamo3<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_GPT2:\n            {\n                llm = std::make_unique<llm_build_gpt2>(*this, params);\n            } break;\n        case LLM_ARCH_CODESHELL:\n            {\n                llm = std::make_unique<llm_build_codeshell>(*this, params);\n            } break;\n        case LLM_ARCH_ORION:\n            {\n                llm = std::make_unique<llm_build_orion>(*this, params);\n            } break;\n        case LLM_ARCH_INTERNLM2:\n            {\n                llm = std::make_unique<llm_build_internlm2>(*this, params);\n            } break;\n        case LLM_ARCH_MINICPM3:\n            {\n                llm = std::make_unique<llm_build_minicpm3>(*this, params);\n            } break;\n        case LLM_ARCH_GEMMA:\n            {\n                llm = std::make_unique<llm_build_gemma>(*this, params);\n            } break;\n        case LLM_ARCH_GEMMA2:\n            {\n                llm = std::make_unique<llm_build_gemma2_iswa>(*this, params);\n            } break;\n        case LLM_ARCH_GEMMA3:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {\n                    llm = std::make_unique<llm_build_gemma3<true>>(*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_gemma3<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_GEMMA3N:\n            {\n                llm = std::make_unique<llm_build_gemma3n_iswa>(*this, params);\n            } break;\n        case LLM_ARCH_GEMMA_EMBEDDING:\n            {\n                llm = std::make_unique<llm_build_gemma_embedding>(*this, params);\n            } break;\n        case LLM_ARCH_STARCODER2:\n            {\n                llm = std::make_unique<llm_build_starcoder2>(*this, params);\n            } break;\n        case LLM_ARCH_MAMBA:\n        case LLM_ARCH_MAMBA2:\n            {\n                llm = std::make_unique<llm_build_mamba>(*this, params);\n            } break;\n        case LLM_ARCH_JAMBA:\n            {\n                llm = std::make_unique<llm_build_jamba>(*this, params);\n            } break;\n        case LLM_ARCH_XVERSE:\n            {\n                llm = std::make_unique<llm_build_xverse>(*this, params);\n            } break;\n        case LLM_ARCH_COMMAND_R:\n            {\n                llm = std::make_unique<llm_build_command_r>(*this, params);\n            } break;\n        case LLM_ARCH_COHERE2:\n            {\n                llm = std::make_unique<llm_build_cohere2_iswa>(*this, params);\n            } break;\n        case LLM_ARCH_DBRX:\n            {\n                llm = std::make_unique<llm_build_dbrx>(*this, params);\n            } break;\n        case LLM_ARCH_OLMO:\n            {\n                llm = std::make_unique<llm_build_olmo>(*this, params);\n            } break;\n        case LLM_ARCH_OLMO2:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {\n                    llm = std::make_unique<llm_build_olmo2<true>>(*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_olmo2<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_OLMOE:\n            {\n                llm = std::make_unique<llm_build_olmoe>(*this, params);\n            } break;\n        case LLM_ARCH_OPENELM:\n            {\n                llm = std::make_unique<llm_build_openelm>(*this, params);\n            } break;\n        case LLM_ARCH_GPTNEOX:\n            {\n                llm = std::make_unique<llm_build_gptneox>(*this, params);\n            } break;\n        case LLM_ARCH_ARCTIC:\n            {\n                llm = std::make_unique<llm_build_arctic>(*this, params);\n            } break;\n        case LLM_ARCH_DEEPSEEK:\n            {\n                llm = std::make_unique<llm_build_deepseek>(*this, params);\n            } break;\n        case LLM_ARCH_DEEPSEEK2:\n        case LLM_ARCH_GLM_DSA:\n            {\n                llm = std::make_unique<llm_build_deepseek2>(*this, params);\n            } break;\n        case LLM_ARCH_CHATGLM:\n            {\n                llm = std::make_unique<llm_build_chatglm>(*this, params);\n            } break;\n        case LLM_ARCH_GLM4:\n            {\n                llm = std::make_unique<llm_build_glm4>(*this, params);\n            } break;\n        case LLM_ARCH_GLM4_MOE:\n            {\n                llm = std::make_unique<llm_build_glm4_moe>(*this, params);\n            } break;\n        case LLM_ARCH_BITNET:\n            {\n                llm = std::make_unique<llm_build_bitnet>(*this, params);\n            } break;\n        case LLM_ARCH_T5:\n            {\n                switch (params.gtype) {\n                    case LLM_GRAPH_TYPE_ENCODER:\n                        llm = std::make_unique<llm_build_t5_enc>(*this, params);\n                        break;\n                    case LLM_GRAPH_TYPE_DEFAULT:\n                    case LLM_GRAPH_TYPE_DECODER:\n                        llm = std::make_unique<llm_build_t5_dec>(*this, params);\n                        break;\n                    default:\n                        GGML_ABORT(\"invalid graph type\");\n                };\n            } break;\n        case LLM_ARCH_T5ENCODER:\n            {\n                llm = std::make_unique<llm_build_t5_enc>(*this, params);\n            }\n            break;\n        case LLM_ARCH_JAIS:\n            {\n                llm = std::make_unique<llm_build_jais>(*this, params);\n            } break;\n        case LLM_ARCH_JAIS2:\n            {\n                llm = std::make_unique<llm_build_jais2>(*this, params);\n            } break;\n        case LLM_ARCH_NEMOTRON:\n            {\n                llm = std::make_unique<llm_build_nemotron>(*this, params);\n            } break;\n        case LLM_ARCH_NEMOTRON_H:\n        case LLM_ARCH_NEMOTRON_H_MOE:\n            {\n                llm = std::make_unique<llm_build_nemotron_h>(*this, params);\n            } break;\n        case LLM_ARCH_EXAONE:\n            {\n                llm = std::make_unique<llm_build_exaone>(*this, params);\n            } break;\n        case LLM_ARCH_EXAONE4:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {\n                    llm = std::make_unique<llm_build_exaone4<true>>(*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_exaone4<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_EXAONE_MOE:\n            {\n                llm = std::make_unique<llm_build_exaone_moe>(*this, params);\n            } break;\n        case LLM_ARCH_RWKV6:\n            {\n                llm = std::make_unique<llm_build_rwkv6>(*this, params);\n            } break;\n        case LLM_ARCH_RWKV6QWEN2:\n            {\n                llm = std::make_unique<llm_build_rwkv6qwen2>(*this, params);\n            } break;\n        case LLM_ARCH_RWKV7:\n            {\n                llm = std::make_unique<llm_build_rwkv7>(*this, params);\n            } break;\n        case LLM_ARCH_ARWKV7:\n            {\n                llm = std::make_unique<llm_build_arwkv7>(*this, params);\n            } break;\n        case LLM_ARCH_GRANITE:\n        case LLM_ARCH_GRANITE_MOE:\n        case LLM_ARCH_MINICPM:\n            {\n                llm = std::make_unique<llm_build_granite>(*this, params);\n            } break;\n        case LLM_ARCH_GRANITE_HYBRID:\n            {\n                llm = std::make_unique<llm_build_granite_hybrid>(*this, params);\n            } break;\n        case LLM_ARCH_CHAMELEON:\n            {\n                llm = std::make_unique<llm_build_chameleon>(*this, params);\n            } break;\n        case LLM_ARCH_WAVTOKENIZER_DEC:\n            {\n                llm = std::make_unique<llm_build_wavtokenizer_dec>(*this, params);\n            } break;\n        case LLM_ARCH_PLM:\n            {\n                llm = std::make_unique<llm_build_plm>(*this, params);\n            } break;\n        case LLM_ARCH_BAILINGMOE:\n            {\n                llm = std::make_unique<llm_build_bailingmoe>(*this, params);\n            } break;\n        case LLM_ARCH_BAILINGMOE2:\n            {\n                llm = std::make_unique<llm_build_bailingmoe2>(*this, params);\n            } break;\n        case LLM_ARCH_SEED_OSS:\n            {\n                llm = std::make_unique<llm_build_seed_oss>(*this, params);\n            } break;\n        case LLM_ARCH_DOTS1:\n            {\n                llm = std::make_unique<llm_build_dots1>(*this, params);\n            } break;\n        case LLM_ARCH_ARCEE:\n            {\n                llm = std::make_unique<llm_build_arcee>(*this, params);\n            } break;\n        case LLM_ARCH_AFMOE:\n            {\n                llm = std::make_unique<llm_build_afmoe>(*this, params);\n            } break;\n        case LLM_ARCH_ERNIE4_5:\n            {\n                llm = std::make_unique<llm_build_ernie4_5>(*this, params);\n            } break;\n        case LLM_ARCH_ERNIE4_5_MOE:\n            {\n                llm = std::make_unique<llm_build_ernie4_5_moe>(*this, params);\n            } break;\n        case LLM_ARCH_PADDLEOCR:\n            {\n                llm = std::make_unique<llm_build_paddleocr>(*this, params);\n            } break;\n        case LLM_ARCH_HUNYUAN_MOE:\n            {\n                llm = std::make_unique<llm_build_hunyuan_moe>(*this, params);\n            } break;\n        case LLM_ARCH_HUNYUAN_DENSE:\n            {\n                llm = std::make_unique<llm_build_hunyuan_dense>(*this, params);\n            } break;\n        case LLM_ARCH_SMOLLM3:\n            {\n                llm = std::make_unique<llm_build_smollm3>(*this, params);\n            } break;\n        case LLM_ARCH_OPENAI_MOE:\n            {\n                llm = std::make_unique<llm_build_openai_moe_iswa>(*this, params);\n            } break;\n        case LLM_ARCH_FALCON_H1:\n            {\n                llm = std::make_unique<llm_build_falcon_h1>(*this, params);\n            } break;\n        case LLM_ARCH_LFM2:\n        case LLM_ARCH_LFM2MOE:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {\n                    llm = std::make_unique<llm_build_lfm2<true>>(*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_lfm2<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_SMALLTHINKER:\n            {\n                if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {\n                    llm = std::make_unique<llm_build_smallthinker<true>> (*this, params);\n                } else {\n                    llm = std::make_unique<llm_build_smallthinker<false>>(*this, params);\n                }\n            } break;\n        case LLM_ARCH_GROVEMOE:\n            {\n                llm = std::make_unique<llm_build_grovemoe>(*this, params);\n            } break;\n        case LLM_ARCH_APERTUS:\n            {\n                llm = std::make_unique<llm_build_apertus>(*this, params);\n            } break;\n        case LLM_ARCH_MINIMAX_M2:\n            {\n                llm = std::make_unique<llm_build_minimax_m2>(*this, params);\n            } break;\n        case LLM_ARCH_COGVLM:\n            {\n                llm = std::make_unique<llm_build_cogvlm>(*this, params);\n            } break;\n        case LLM_ARCH_PANGU_EMBED:\n            {\n                llm = std::make_unique<llm_build_pangu_embedded>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN3NEXT:\n            {\n                llm = std::make_unique<llm_build_qwen3next>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN35:\n            {\n                llm = std::make_unique<llm_build_qwen35>(*this, params);\n            } break;\n        case LLM_ARCH_QWEN35MOE:\n            {\n                llm = std::make_unique<llm_build_qwen35moe>(*this, params);\n            } break;\n        case LLM_ARCH_MISTRAL3:\n            {\n                llm = std::make_unique<llm_build_mistral3>(*this, params);\n            } break;\n        case LLM_ARCH_MIMO2:\n            {\n                llm = std::make_unique<llm_build_mimo2_iswa>(*this, params);\n            } break;\n        case LLM_ARCH_KIMI_LINEAR:\n            {\n                llm = std::make_unique<llm_build_kimi_linear>(*this, params);\n            } break;\n        case LLM_ARCH_STEP35:\n            {\n                llm = std::make_unique<llm_build_step35_iswa>(*this, params);\n            } break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n\n    // add on pooling layer\n    llm->build_pooling(cls, cls_b, cls_out, cls_out_b, cls_norm);\n\n    // add backend sampling layers (if any)\n    llm->build_sampling();\n\n    // if the gguf model was converted with --sentence-transformers-dense-modules\n    // there will be two additional dense projection layers\n    // dense linear projections are applied after pooling\n    // TODO: move reranking logic here and generalize\n    llm->build_dense_out(dense_2_out_layers, dense_2_out_layers_b, dense_3_out_layers);\n\n    llm->res->set_outputs();\n\n    return llm->res->get_gf();\n}\n\n\n//\n// interface implementation\n//\n\nllama_model_params llama_model_default_params() {\n    llama_model_params result = {\n        /*.devices                     =*/ nullptr,\n        /*.tensor_buft_overrides       =*/ nullptr,\n        /*.n_gpu_layers                =*/ -1,\n        /*.split_mode                  =*/ LLAMA_SPLIT_MODE_LAYER,\n        /*.main_gpu                    =*/ 0,\n        /*.tensor_split                =*/ nullptr,\n        /*.progress_callback           =*/ nullptr,\n        /*.progress_callback_user_data =*/ nullptr,\n        /*.kv_overrides                =*/ nullptr,\n        /*.vocab_only                  =*/ false,\n        /*.use_mmap                    =*/ true,\n        /*.use_direct_io               =*/ false,\n        /*.use_mlock                   =*/ false,\n        /*.check_tensors               =*/ false,\n        /*.use_extra_bufts             =*/ true,\n        /*.no_host                     =*/ false,\n        /*.no_alloc                    =*/ false,\n    };\n\n    return result;\n}\n\nconst llama_vocab * llama_model_get_vocab(const llama_model * model) {\n    return &model->vocab;\n}\n\nvoid llama_free_model(llama_model * model) {\n    llama_model_free(model);\n}\n\nvoid llama_model_free(llama_model * model) {\n    delete model;\n}\n\nint32_t llama_model_n_ctx_train(const llama_model * model) {\n    return model->hparams.n_ctx_train;\n}\n\nint32_t llama_model_n_embd(const llama_model * model) {\n    return model->hparams.n_embd;\n}\n\nint32_t llama_model_n_embd_inp(const llama_model * model) {\n    return model->hparams.n_embd_inp();\n}\n\nint32_t llama_model_n_embd_out(const llama_model * model) {\n    return model->hparams.n_embd_out();\n}\n\nint32_t llama_model_n_layer(const llama_model * model) {\n    return model->hparams.n_layer;\n}\n\nint32_t llama_model_n_head(const llama_model * model) {\n    return model->hparams.n_head();\n}\n\nint32_t llama_model_n_head_kv(const llama_model * model) {\n    return model->hparams.n_head_kv();\n}\n\nint32_t llama_model_n_swa(const llama_model * model) {\n    return model->hparams.n_swa;\n}\n\nuint32_t llama_model_n_cls_out(const struct llama_model * model) {\n    return model->hparams.n_cls_out;\n}\n\nconst char * llama_model_cls_label(const struct llama_model * model, uint32_t i) {\n    if (i < model->classifier_labels.size()) {\n        return model->classifier_labels[i].c_str();\n    }\n\n    return nullptr;\n}\n\n// deprecated\nint32_t llama_n_ctx_train(const llama_model * model) {\n    return llama_model_n_ctx_train(model);\n}\n\n// deprecated\nint32_t llama_n_embd(const llama_model * model) {\n    return llama_model_n_embd(model);\n}\n\n// deprecated\nint32_t llama_n_layer(const llama_model * model) {\n    return llama_model_n_layer(model);\n}\n\n// deprecated\nint32_t llama_n_head(const llama_model * model) {\n    return llama_model_n_head(model);\n}\n\nllama_rope_type llama_model_rope_type(const llama_model * model) {\n    switch (model->arch) {\n        // these models do not use RoPE\n        case LLM_ARCH_CLIP:\n        case LLM_ARCH_GPT2:\n        case LLM_ARCH_GPTJ:\n        case LLM_ARCH_MPT:\n        case LLM_ARCH_REFACT:\n        case LLM_ARCH_BLOOM:\n        case LLM_ARCH_MAMBA:\n        case LLM_ARCH_MAMBA2:\n        case LLM_ARCH_JAMBA:\n        case LLM_ARCH_JINA_BERT_V2:\n        case LLM_ARCH_T5:\n        case LLM_ARCH_T5ENCODER:\n        case LLM_ARCH_JAIS:\n        case LLM_ARCH_RWKV6:\n        case LLM_ARCH_RWKV6QWEN2:\n        case LLM_ARCH_RWKV7:\n        case LLM_ARCH_ARWKV7:\n        case LLM_ARCH_WAVTOKENIZER_DEC:\n        case LLM_ARCH_NEMOTRON_H:\n        case LLM_ARCH_NEMOTRON_H_MOE:\n        case LLM_ARCH_KIMI_LINEAR:\n            return LLAMA_ROPE_TYPE_NONE;\n\n        // use what we call a normal RoPE, operating on pairs of consecutive head values\n        case LLM_ARCH_LLAMA:\n        case LLM_ARCH_LLADA:\n        case LLM_ARCH_LLAMA4:\n        case LLM_ARCH_DECI:\n        case LLM_ARCH_BAICHUAN:\n        case LLM_ARCH_STARCODER:\n        case LLM_ARCH_INTERNLM2:\n        case LLM_ARCH_MINICPM:\n        case LLM_ARCH_XVERSE:\n        case LLM_ARCH_COMMAND_R:\n        case LLM_ARCH_COHERE2:\n        case LLM_ARCH_OLMO:\n        case LLM_ARCH_ARCTIC:\n        case LLM_ARCH_DEEPSEEK:\n        case LLM_ARCH_DEEPSEEK2:\n        case LLM_ARCH_PLM:\n        case LLM_ARCH_CHATGLM:\n        case LLM_ARCH_GRANITE:\n        case LLM_ARCH_GRANITE_MOE:\n        case LLM_ARCH_GRANITE_HYBRID:\n        case LLM_ARCH_CHAMELEON:\n        case LLM_ARCH_BAILINGMOE:\n        case LLM_ARCH_NEO_BERT:\n        case LLM_ARCH_SMOLLM3:\n        case LLM_ARCH_ARCEE:\n        case LLM_ARCH_ERNIE4_5:\n        case LLM_ARCH_ERNIE4_5_MOE:\n        case LLM_ARCH_MISTRAL3:\n        case LLM_ARCH_LLAMA_EMBED:\n        case LLM_ARCH_MAINCODER:\n        case LLM_ARCH_GLM_DSA:\n            return LLAMA_ROPE_TYPE_NORM;\n\n        // the pairs of head values are offset by n_rot/2\n        case LLM_ARCH_FALCON:\n        case LLM_ARCH_FALCON_H1:\n        case LLM_ARCH_GROK:\n        case LLM_ARCH_DBRX:\n        case LLM_ARCH_BERT:\n        case LLM_ARCH_JINA_BERT_V3:\n        case LLM_ARCH_MODERN_BERT:\n        case LLM_ARCH_NOMIC_BERT:\n        case LLM_ARCH_NOMIC_BERT_MOE:\n        case LLM_ARCH_EUROBERT:\n        case LLM_ARCH_STABLELM:\n        case LLM_ARCH_BITNET:\n        case LLM_ARCH_QWEN:\n        case LLM_ARCH_QWEN2:\n        case LLM_ARCH_DREAM:\n        case LLM_ARCH_QWEN2MOE:\n        case LLM_ARCH_QWEN3:\n        case LLM_ARCH_QWEN3MOE:\n        case LLM_ARCH_LLADA_MOE:\n        case LLM_ARCH_RND1:\n        case LLM_ARCH_OLMO2:\n        case LLM_ARCH_OLMOE:\n        case LLM_ARCH_PHI2:\n        case LLM_ARCH_PHI3:\n        case LLM_ARCH_PHIMOE:\n        case LLM_ARCH_PLAMO:\n        case LLM_ARCH_PLAMO2:\n        case LLM_ARCH_PLAMO3:\n        case LLM_ARCH_GEMMA:\n        case LLM_ARCH_GEMMA2:\n        case LLM_ARCH_GEMMA3:\n        case LLM_ARCH_GEMMA3N:\n        case LLM_ARCH_GEMMA_EMBEDDING:\n        case LLM_ARCH_STARCODER2:\n        case LLM_ARCH_OPENELM:\n        case LLM_ARCH_GPTNEOX:\n        case LLM_ARCH_CODESHELL:\n        case LLM_ARCH_ORION:\n        case LLM_ARCH_NEMOTRON:\n        case LLM_ARCH_EXAONE:\n        case LLM_ARCH_EXAONE4:\n        case LLM_ARCH_EXAONE_MOE:\n        case LLM_ARCH_MINICPM3:\n        case LLM_ARCH_BAILINGMOE2:\n        case LLM_ARCH_DOTS1:\n        case LLM_ARCH_HUNYUAN_MOE:\n        case LLM_ARCH_JAIS2:\n        case LLM_ARCH_OPENAI_MOE:\n        case LLM_ARCH_HUNYUAN_DENSE:\n        case LLM_ARCH_LFM2:\n        case LLM_ARCH_LFM2MOE:\n        case LLM_ARCH_SMALLTHINKER:\n        case LLM_ARCH_SEED_OSS:\n        case LLM_ARCH_GROVEMOE:\n        case LLM_ARCH_APERTUS:\n        case LLM_ARCH_MINIMAX_M2:\n        case LLM_ARCH_COGVLM:\n        case LLM_ARCH_PANGU_EMBED:\n        case LLM_ARCH_AFMOE:\n        case LLM_ARCH_QWEN3NEXT:\n        case LLM_ARCH_MIMO2:\n        case LLM_ARCH_STEP35:\n            return LLAMA_ROPE_TYPE_NEOX;\n\n        case LLM_ARCH_QWEN2VL:\n        case LLM_ARCH_PADDLEOCR:\n            return LLAMA_ROPE_TYPE_MROPE;\n        case LLM_ARCH_QWEN3VL:\n        case LLM_ARCH_QWEN3VLMOE:\n        case LLM_ARCH_QWEN35:\n        case LLM_ARCH_QWEN35MOE:\n            return LLAMA_ROPE_TYPE_IMROPE;\n\n        case LLM_ARCH_GLM4:\n            return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NORM;\n        case LLM_ARCH_GLM4_MOE:\n            return model->hparams.use_mrope() ? LLAMA_ROPE_TYPE_MROPE : LLAMA_ROPE_TYPE_NEOX;\n\n        // all model arches should be listed explicitly here\n        case LLM_ARCH_UNKNOWN:\n            GGML_ABORT(\"unknown architecture\");\n    }\n\n    return LLAMA_ROPE_TYPE_NONE;\n}\n\nfloat llama_model_rope_freq_scale_train(const llama_model * model) {\n    return model->hparams.rope_freq_scale_train;\n}\n\nint32_t llama_model_meta_val_str(const llama_model * model, const char * key, char * buf, size_t buf_size) {\n    const auto & it = model->gguf_kv.find(key);\n    if (it == model->gguf_kv.end()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    return snprintf(buf, buf_size, \"%s\", it->second.c_str());\n}\n\nint32_t llama_model_meta_count(const llama_model * model) {\n    return (int)model->gguf_kv.size();\n}\n\nconst char * llama_model_meta_key_str(llama_model_meta_key key) {\n    switch (key) {\n        case LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE:        return \"general.sampling.sequence\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_TOP_K:           return \"general.sampling.top_k\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_TOP_P:           return \"general.sampling.top_p\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_MIN_P:           return \"general.sampling.min_p\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY: return \"general.sampling.xtc_probability\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD:   return \"general.sampling.xtc_threshold\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_TEMP:            return \"general.sampling.temp\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N:  return \"general.sampling.penalty_last_n\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT:  return \"general.sampling.penalty_repeat\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT:        return \"general.sampling.mirostat\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU:    return \"general.sampling.mirostat_tau\";\n        case LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA:    return \"general.sampling.mirostat_eta\";\n        default:                                            return nullptr;\n    }\n}\n\nint32_t llama_model_meta_key_by_index(const llama_model * model, int i, char * buf, size_t buf_size) {\n    if (i < 0 || i >= (int)model->gguf_kv.size()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    auto it = model->gguf_kv.begin();\n    std::advance(it, i);\n    return snprintf(buf, buf_size, \"%s\", it->first.c_str());\n}\n\nint32_t llama_model_meta_val_str_by_index(const llama_model * model, int32_t i, char * buf, size_t buf_size) {\n    if (i < 0 || i >= (int)model->gguf_kv.size()) {\n        if (buf_size > 0) {\n            buf[0] = '\\0';\n        }\n        return -1;\n    }\n    auto it = model->gguf_kv.begin();\n    std::advance(it, i);\n    return snprintf(buf, buf_size, \"%s\", it->second.c_str());\n}\n\nint32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size) {\n    return snprintf(buf, buf_size, \"%s\", model->desc().c_str());\n}\n\nuint64_t llama_model_size(const llama_model * model) {\n    return model->size();\n}\n\nconst char * llama_model_chat_template(const llama_model * model, const char * name) {\n    const auto key = name ? LLM_KV(model->arch, name)(LLM_KV_TOKENIZER_CHAT_TEMPLATE)\n        : LLM_KV(model->arch)(LLM_KV_TOKENIZER_CHAT_TEMPLATE);\n    const auto & it = model->gguf_kv.find(key);\n    if (it == model->gguf_kv.end()) {\n        // one-off fix for very popular models (so we are not flooded with issues)\n        // do not extend this list unless absolutely necessary\n        // Mistral-Small-2503 does not have built-in chat template\n        llama_vocab_pre_type pre_type = model->vocab.get_pre_type();\n        if (!name && pre_type == LLAMA_VOCAB_PRE_TYPE_TEKKEN && model->layers.size() == 40) {\n            return \"mistral-v7-tekken\";\n        }\n\n        return nullptr;\n    }\n\n    return it->second.c_str();\n}\n\nuint64_t llama_model_n_params(const llama_model * model) {\n    return model->n_elements();\n}\n\nbool llama_model_has_encoder(const llama_model * model) {\n    switch (model->arch) {\n        case LLM_ARCH_T5:        return true;\n        case LLM_ARCH_T5ENCODER: return true;\n        default:                 return false;\n    }\n}\n\nbool llama_model_has_decoder(const llama_model * model) {\n    switch (model->arch) {\n        case LLM_ARCH_T5ENCODER: return false;\n        default:                 return true;\n    }\n}\n\nllama_token llama_model_decoder_start_token(const llama_model * model) {\n    return model->hparams.dec_start_token_id;\n}\n\nbool llama_model_is_recurrent(const llama_model * model) {\n    return llm_arch_is_recurrent(model->arch);\n}\n\nbool llama_model_is_hybrid(const llama_model * model) {\n    return llm_arch_is_hybrid(model->arch);\n}\n\nbool llama_model_is_diffusion(const llama_model * model) {\n    return llm_arch_is_diffusion(model->arch);\n}\n\nconst std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model) {\n    return model->tensors_by_name;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-model.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n#include \"llama-arch.h\"\n#include \"llama-graph.h\"\n#include \"llama-hparams.h\"\n#include \"llama-memory.h\"\n#include \"llama-vocab.h\"\n\n#include <map>\n#include <memory>\n#include <string>\n#include <unordered_map>\n#include <unordered_set>\n#include <vector>\n\nstruct llama_cparams;\nstruct llama_ubatch;\nstruct llama_model_loader;\n\n// available models\nenum llm_type {\n    LLM_TYPE_UNKNOWN,\n    LLM_TYPE_14M,\n    LLM_TYPE_17M,\n    LLM_TYPE_22M,\n    LLM_TYPE_33M,\n    LLM_TYPE_47M,\n    LLM_TYPE_60M,\n    LLM_TYPE_70M,\n    LLM_TYPE_80M,\n    LLM_TYPE_109M,\n    LLM_TYPE_137M,\n    LLM_TYPE_140M,\n    LLM_TYPE_149M,\n    LLM_TYPE_160M,\n    LLM_TYPE_190M,\n    LLM_TYPE_220M,\n    LLM_TYPE_250M,\n    LLM_TYPE_256M,\n    LLM_TYPE_270M,\n    LLM_TYPE_335M,\n    LLM_TYPE_350M,\n    LLM_TYPE_360M,\n    LLM_TYPE_395M,\n    LLM_TYPE_410M,\n    LLM_TYPE_450M,\n    LLM_TYPE_475M,\n    LLM_TYPE_558M,\n    LLM_TYPE_700M,\n    LLM_TYPE_770M,\n    LLM_TYPE_780M,\n    LLM_TYPE_950M,\n    LLM_TYPE_0_3B,\n    LLM_TYPE_0_5B,\n    LLM_TYPE_0_6B,\n    LLM_TYPE_0_8B,\n    LLM_TYPE_1B,\n    LLM_TYPE_1_2B,\n    LLM_TYPE_1_3B,\n    LLM_TYPE_1_4B,\n    LLM_TYPE_1_5B,\n    LLM_TYPE_1_6B,\n    LLM_TYPE_1_7B,\n    LLM_TYPE_1_8B,\n    LLM_TYPE_2B,\n    LLM_TYPE_2_6B,\n    LLM_TYPE_2_8B,\n    LLM_TYPE_2_9B,\n    LLM_TYPE_3B,\n    LLM_TYPE_4B,\n    LLM_TYPE_6B,\n    LLM_TYPE_6_9B,\n    LLM_TYPE_7B,\n    LLM_TYPE_8B,\n    LLM_TYPE_9B,\n    LLM_TYPE_11B,\n    LLM_TYPE_12B,\n    LLM_TYPE_13B,\n    LLM_TYPE_14B,\n    LLM_TYPE_15B,\n    LLM_TYPE_16B,\n    LLM_TYPE_20B,\n    LLM_TYPE_26B,\n    LLM_TYPE_27B,\n    LLM_TYPE_30B,\n    LLM_TYPE_32B,\n    LLM_TYPE_34B,\n    LLM_TYPE_35B,\n    LLM_TYPE_36B,\n    LLM_TYPE_40B,\n    LLM_TYPE_65B,\n    LLM_TYPE_70B,\n    LLM_TYPE_120B,\n    LLM_TYPE_142B,\n    LLM_TYPE_236B,\n    LLM_TYPE_290B,\n    LLM_TYPE_314B,\n    LLM_TYPE_405B,\n    LLM_TYPE_671B,\n    LLM_TYPE_SMALL,\n    LLM_TYPE_MEDIUM,\n    LLM_TYPE_LARGE,\n    LLM_TYPE_XL,\n    LLM_TYPE_A1_7B,\n    LLM_TYPE_A2_7B,\n    LLM_TYPE_8x7B,\n    LLM_TYPE_8x22B,\n    LLM_TYPE_16x12B,\n    LLM_TYPE_16x3_8B,\n    LLM_TYPE_10B_128x3_66B,\n    LLM_TYPE_57B_A14B,\n    LLM_TYPE_17B_16E, // llama4 Scout\n    LLM_TYPE_17B_128E, // llama4 Maverick\n    LLM_TYPE_A13B,\n    LLM_TYPE_7B_A1B,\n    LLM_TYPE_8B_A1B, // lfm2moe\n    LLM_TYPE_16B_A1B,\n    LLM_TYPE_21B_A3B, // Ernie MoE small\n    LLM_TYPE_24B_A2B, // lfm2moe\n    LLM_TYPE_30B_A3B,\n    LLM_TYPE_31B_A3_5B,\n    LLM_TYPE_35B_A3B, // Qwen3.5\n    LLM_TYPE_48B_A3B, // Kimi Linear\n    LLM_TYPE_80B_A3B, // Qwen3 Next\n    LLM_TYPE_100B_A6B,\n    LLM_TYPE_102B_A12B, // Solar-Open\n    LLM_TYPE_106B_A12B, // GLM-4.5-Air\n    LLM_TYPE_120B_A12B, // Nemotron 3 Super\n    LLM_TYPE_122B_A10B, // Qwen3.5\n    LLM_TYPE_196B_A11B, // Step3.5-Flash\n    LLM_TYPE_230B_A10B, // Minimax M2\n    LLM_TYPE_235B_A22B,\n    LLM_TYPE_300B_A47B, // Ernie MoE big\n    LLM_TYPE_310B_A15B, // /MiMo-V2-Flash\n    LLM_TYPE_355B_A32B, // GLM-4.5\n    LLM_TYPE_397B_A17B, // Qwen3.5\n    LLM_TYPE_744B_A40B, // GLM-5\n    LLM_TYPE_E2B,\n    LLM_TYPE_E4B,\n};\n\nstd::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type);\n\nstruct llama_layer_posnet {\n    // resnet\n    struct ggml_tensor * norm1   = nullptr;\n    struct ggml_tensor * norm1_b = nullptr;\n\n    struct ggml_tensor * conv1   = nullptr;\n    struct ggml_tensor * conv1_b = nullptr;\n\n    struct ggml_tensor * norm2   = nullptr;\n    struct ggml_tensor * norm2_b = nullptr;\n\n    struct ggml_tensor * conv2   = nullptr;\n    struct ggml_tensor * conv2_b = nullptr;\n\n    // attention\n    struct ggml_tensor * attn_norm   = nullptr;\n    struct ggml_tensor * attn_norm_b = nullptr;\n\n    struct ggml_tensor * attn_q   = nullptr;\n    struct ggml_tensor * attn_q_b = nullptr;\n\n    struct ggml_tensor * attn_k   = nullptr;\n    struct ggml_tensor * attn_k_b = nullptr;\n\n    struct ggml_tensor * attn_v   = nullptr;\n    struct ggml_tensor * attn_v_b = nullptr;\n\n    struct ggml_tensor * attn_o   = nullptr;\n    struct ggml_tensor * attn_o_b = nullptr;\n\n    // normalize\n    struct ggml_tensor * norm   = nullptr;\n    struct ggml_tensor * norm_b = nullptr;\n};\n\nstruct llama_layer_convnext {\n    struct ggml_tensor * dw   = nullptr;\n    struct ggml_tensor * dw_b = nullptr;\n\n    struct ggml_tensor * norm   = nullptr;\n    struct ggml_tensor * norm_b = nullptr;\n\n    struct ggml_tensor * pw1   = nullptr;\n    struct ggml_tensor * pw1_b = nullptr;\n\n    struct ggml_tensor * pw2   = nullptr;\n    struct ggml_tensor * pw2_b = nullptr;\n\n    struct ggml_tensor * gamma = nullptr;\n};\n\nstruct llama_layer_shortconv {\n    struct ggml_tensor * in_proj  = nullptr;\n    struct ggml_tensor * conv     = nullptr;\n    struct ggml_tensor * out_proj = nullptr;\n};\n\nstruct llama_layer_nextn {\n    struct ggml_tensor * eh_proj          = nullptr;\n    struct ggml_tensor * embed_tokens     = nullptr;\n    struct ggml_tensor * enorm            = nullptr;\n    struct ggml_tensor * hnorm            = nullptr;\n    struct ggml_tensor * shared_head_head = nullptr;\n    struct ggml_tensor * shared_head_norm = nullptr;\n};\n\nstruct llama_layer {\n    // normalization\n    struct ggml_tensor * attn_norm       = nullptr;\n    struct ggml_tensor * attn_norm_b     = nullptr;\n    struct ggml_tensor * attn_norm_2     = nullptr;\n    struct ggml_tensor * attn_norm_2_b   = nullptr;\n    struct ggml_tensor * attn_q_norm     = nullptr;\n    struct ggml_tensor * attn_q_norm_b   = nullptr;\n    struct ggml_tensor * attn_k_norm     = nullptr;\n    struct ggml_tensor * attn_k_norm_b   = nullptr;\n    struct ggml_tensor * attn_out_norm   = nullptr;\n    struct ggml_tensor * attn_out_norm_b = nullptr;\n    struct ggml_tensor * attn_q_a_norm   = nullptr;\n    struct ggml_tensor * attn_kv_a_norm  = nullptr;\n    struct ggml_tensor * attn_sub_norm   = nullptr;\n    struct ggml_tensor * attn_post_norm  = nullptr;\n    struct ggml_tensor * ffn_sub_norm    = nullptr;\n    struct ggml_tensor * attn_norm_cross = nullptr;\n    struct ggml_tensor * attn_norm_enc   = nullptr;\n    struct ggml_tensor * ssm_norm        = nullptr;\n    struct ggml_tensor * ssm_dt_norm     = nullptr;\n    struct ggml_tensor * ssm_b_norm      = nullptr;\n    struct ggml_tensor * ssm_c_norm      = nullptr;\n\n    // attention\n    struct ggml_tensor * wq        = nullptr;\n    struct ggml_tensor * wk        = nullptr;\n    struct ggml_tensor * wv        = nullptr;\n    struct ggml_tensor * wo        = nullptr;\n    struct ggml_tensor * wqkv      = nullptr;\n    struct ggml_tensor * wq_a      = nullptr;\n    struct ggml_tensor * wq_b      = nullptr;\n    struct ggml_tensor * wkv_a_mqa = nullptr;\n    struct ggml_tensor * wkv_b     = nullptr;\n    struct ggml_tensor * wk_b      = nullptr;\n    struct ggml_tensor * wv_b      = nullptr;\n    struct ggml_tensor * wq_cross  = nullptr;\n    struct ggml_tensor * wk_cross  = nullptr;\n    struct ggml_tensor * wv_cross  = nullptr;\n    struct ggml_tensor * wo_cross  = nullptr;\n    struct ggml_tensor * wq_enc    = nullptr;\n    struct ggml_tensor * wk_enc    = nullptr;\n    struct ggml_tensor * wv_enc    = nullptr;\n    struct ggml_tensor * wo_enc    = nullptr;\n    struct ggml_tensor * wqkv_gate = nullptr;\n\n    // attention bias\n    struct ggml_tensor * bq   = nullptr;\n    struct ggml_tensor * bk   = nullptr;\n    struct ggml_tensor * bv   = nullptr;\n    struct ggml_tensor * bo   = nullptr;\n    struct ggml_tensor * bqkv = nullptr;\n\n    // relative position bias\n    struct ggml_tensor * attn_rel_b       = nullptr;\n    struct ggml_tensor * attn_rel_b_enc   = nullptr;\n    struct ggml_tensor * attn_rel_b_cross = nullptr;\n\n    // normalization\n    struct ggml_tensor * ffn_norm         = nullptr;\n    struct ggml_tensor * ffn_norm_b       = nullptr;\n    struct ggml_tensor * ffn_post_norm    = nullptr;\n    struct ggml_tensor * layer_out_norm   = nullptr;\n    struct ggml_tensor * layer_out_norm_b = nullptr;\n    struct ggml_tensor * ffn_norm_exps    = nullptr;\n    struct ggml_tensor * ffn_norm_enc     = nullptr;\n\n    // ff\n    struct ggml_tensor * ffn_gate     = nullptr; // w1\n    struct ggml_tensor * ffn_down     = nullptr; // w2\n    struct ggml_tensor * ffn_up       = nullptr; // w3\n    struct ggml_tensor * ffn_gate_enc = nullptr;\n    struct ggml_tensor * ffn_down_enc = nullptr;\n    struct ggml_tensor * ffn_up_enc   = nullptr;\n\n    // ff MoE\n    struct ggml_tensor * ffn_gate_inp      = nullptr;\n    struct ggml_tensor * ffn_gate_exps     = nullptr;\n    struct ggml_tensor * ffn_down_exps     = nullptr;\n    struct ggml_tensor * ffn_up_exps       = nullptr;\n    struct ggml_tensor * ffn_gate_up_exps  = nullptr;\n    struct ggml_tensor * ffn_gate_inp_b    = nullptr;\n    struct ggml_tensor * ffn_gate_exps_b   = nullptr;\n    struct ggml_tensor * ffn_down_exps_b   = nullptr;\n    struct ggml_tensor * ffn_up_exps_b     = nullptr;\n    struct ggml_tensor * ffn_gate_up_exps_b = nullptr;\n\n    // ff MoE per-expert scales (NVFP4 per-tensor scale2)\n    struct ggml_tensor * ffn_gate_exps_s   = nullptr;\n    struct ggml_tensor * ffn_down_exps_s   = nullptr;\n    struct ggml_tensor * ffn_up_exps_s     = nullptr;\n\n    // ff MoE latent proj\n    struct ggml_tensor * ffn_latent_down = nullptr;\n    struct ggml_tensor * ffn_latent_up   = nullptr;\n\n    // ff shared expert (shexp)\n    struct ggml_tensor * ffn_gate_inp_shexp = nullptr;\n    struct ggml_tensor * ffn_gate_shexp     = nullptr;\n    struct ggml_tensor * ffn_down_shexp     = nullptr;\n    struct ggml_tensor * ffn_up_shexp       = nullptr;\n\n    // ff adjugate experts (chexps)\n    struct ggml_tensor * ffn_gate_chexps     = nullptr;\n    struct ggml_tensor * ffn_down_chexps     = nullptr;\n    struct ggml_tensor * ffn_up_chexps       = nullptr;\n\n    // ff bias\n    struct ggml_tensor * ffn_gate_b = nullptr;\n    struct ggml_tensor * ffn_down_b = nullptr; // b2\n    struct ggml_tensor * ffn_up_b   = nullptr; // b3\n    struct ggml_tensor * ffn_act    = nullptr;\n    struct ggml_tensor * ffn_exp_probs_b = nullptr;\n\n    // mamba proj\n    struct ggml_tensor * ssm_in  = nullptr;\n    struct ggml_tensor * ssm_x   = nullptr;\n    struct ggml_tensor * ssm_dt  = nullptr;\n    struct ggml_tensor * ssm_out = nullptr;\n\n    // mamba\n    struct ggml_tensor * ssm_conv1d = nullptr;\n    struct ggml_tensor * ssm_a      = nullptr;\n    struct ggml_tensor * ssm_d      = nullptr;\n\n    // mamba bias\n    struct ggml_tensor * ssm_conv1d_b = nullptr;\n    struct ggml_tensor * ssm_dt_b     = nullptr;\n\n    // qwen3next\n    struct ggml_tensor * ssm_beta_alpha = nullptr;\n\n    // qwen3.5\n    struct ggml_tensor * ssm_alpha = nullptr;\n\n    // rwkv\n    struct ggml_tensor * time_mix_w1         = nullptr;\n    struct ggml_tensor * time_mix_w2         = nullptr;\n    struct ggml_tensor * time_mix_lerp_x     = nullptr;\n    struct ggml_tensor * time_mix_lerp_w     = nullptr;\n    struct ggml_tensor * time_mix_lerp_k     = nullptr;\n    struct ggml_tensor * time_mix_lerp_v     = nullptr;\n    struct ggml_tensor * time_mix_lerp_r     = nullptr;\n    struct ggml_tensor * time_mix_lerp_g     = nullptr;\n    struct ggml_tensor * time_mix_lerp_fused = nullptr;\n\n    struct ggml_tensor * time_mix_first        = nullptr;\n    struct ggml_tensor * time_mix_decay        = nullptr;\n    struct ggml_tensor * time_mix_decay_w1     = nullptr;\n    struct ggml_tensor * time_mix_decay_w2     = nullptr;\n    struct ggml_tensor * time_mix_key          = nullptr;\n    struct ggml_tensor * time_mix_key_b        = nullptr;\n    struct ggml_tensor * time_mix_value        = nullptr;\n    struct ggml_tensor * time_mix_value_b      = nullptr;\n    struct ggml_tensor * time_mix_receptance   = nullptr;\n    struct ggml_tensor * time_mix_receptance_b = nullptr;\n    struct ggml_tensor * time_mix_gate         = nullptr;\n\n    // rwkv7\n    struct ggml_tensor * time_mix_w0         = nullptr;\n    struct ggml_tensor * time_mix_a0         = nullptr;\n    struct ggml_tensor * time_mix_a1         = nullptr;\n    struct ggml_tensor * time_mix_a2         = nullptr;\n    struct ggml_tensor * time_mix_v0         = nullptr;\n    struct ggml_tensor * time_mix_v1         = nullptr;\n    struct ggml_tensor * time_mix_v2         = nullptr;\n    struct ggml_tensor * time_mix_g1         = nullptr;\n    struct ggml_tensor * time_mix_g2         = nullptr;\n    struct ggml_tensor * time_mix_k_k        = nullptr;\n    struct ggml_tensor * time_mix_k_a        = nullptr;\n    struct ggml_tensor * time_mix_r_k        = nullptr;\n\n    struct ggml_tensor * time_mix_ln     = nullptr;\n    struct ggml_tensor * time_mix_ln_b   = nullptr;\n    struct ggml_tensor * time_mix_output = nullptr;\n\n    struct ggml_tensor * channel_mix_lerp_k = nullptr;\n    struct ggml_tensor * channel_mix_lerp_r = nullptr;\n\n    struct ggml_tensor * channel_mix_key        = nullptr;\n    struct ggml_tensor * channel_mix_receptance = nullptr;\n    struct ggml_tensor * channel_mix_value      = nullptr;\n\n    // long rope factors\n    struct ggml_tensor * rope_long  = nullptr;\n    struct ggml_tensor * rope_short = nullptr;\n    struct ggml_tensor * rope_freqs = nullptr;\n\n    // bitnet scale\n    struct ggml_tensor * wq_s       = nullptr;\n    struct ggml_tensor * wk_s       = nullptr;\n    struct ggml_tensor * wv_s       = nullptr;\n    struct ggml_tensor * wo_s       = nullptr;\n    struct ggml_tensor * wqkv_s     = nullptr;\n    struct ggml_tensor * wqkv_gate_s = nullptr;\n    struct ggml_tensor * ffn_gate_s = nullptr;\n    struct ggml_tensor * ffn_up_s   = nullptr;\n    struct ggml_tensor * ffn_down_s = nullptr;\n    struct ggml_tensor * ffn_gate_shexp_s = nullptr;\n    struct ggml_tensor * ffn_up_shexp_s   = nullptr;\n    struct ggml_tensor * ffn_down_shexp_s = nullptr;\n    struct ggml_tensor * ssm_out_s  = nullptr;\n    struct ggml_tensor * ssm_alpha_s = nullptr;\n    struct ggml_tensor * ssm_beta_s  = nullptr;\n\n    // altup & laurel\n    struct ggml_tensor * per_layer_inp_gate   = nullptr;\n    struct ggml_tensor * per_layer_proj       = nullptr;\n    struct ggml_tensor * per_layer_post_norm  = nullptr;\n    struct ggml_tensor * altup_correct_coef   = nullptr;\n    struct ggml_tensor * altup_correct_scale  = nullptr;\n    struct ggml_tensor * altup_predict_coef   = nullptr;\n    struct ggml_tensor * altup_router         = nullptr;\n    struct ggml_tensor * altup_router_norm    = nullptr;\n    struct ggml_tensor * laurel_l             = nullptr;\n    struct ggml_tensor * laurel_r             = nullptr;\n    struct ggml_tensor * laurel_post_norm     = nullptr;\n\n    // openai-moe\n    struct ggml_tensor * attn_sinks = nullptr;\n\n    // cogvlm\n    struct ggml_tensor * visexp_attn_wqkv = nullptr;\n    struct ggml_tensor * visexp_attn_wo   = nullptr;\n    struct ggml_tensor * visexp_ffn_gate  = nullptr;\n    struct ggml_tensor * visexp_ffn_down  = nullptr;\n    struct ggml_tensor * visexp_ffn_up    = nullptr;\n\n    // xIELU activation parameters for Apertus\n    struct ggml_tensor * ffn_act_alpha_n = nullptr;\n    struct ggml_tensor * ffn_act_alpha_p = nullptr;\n    struct ggml_tensor * ffn_act_beta    = nullptr;\n    struct ggml_tensor * ffn_act_eps     = nullptr;\n\n    // Kimi Linear KDA (using ssm_ prefix for consistency)\n    // Note: ssm_dt_b already exists above (mamba bias), reused for Kimi dt_bias\n    struct ggml_tensor * ssm_q_conv = nullptr;\n    struct ggml_tensor * ssm_k_conv = nullptr;\n    struct ggml_tensor * ssm_v_conv = nullptr;\n    struct ggml_tensor * ssm_f_a    = nullptr;\n    struct ggml_tensor * ssm_f_b    = nullptr;\n    struct ggml_tensor * ssm_beta   = nullptr;\n    struct ggml_tensor * ssm_g_a    = nullptr;\n    struct ggml_tensor * ssm_g_b    = nullptr;\n    struct ggml_tensor * ssm_o_norm = nullptr;\n\n    // DSA (deepseek sparse attention)\n    struct ggml_tensor * indexer_k_norm   = nullptr;\n    struct ggml_tensor * indexer_k_norm_b = nullptr;\n    struct ggml_tensor * indexer_proj     = nullptr;\n    struct ggml_tensor * indexer_attn_k   = nullptr;\n    struct ggml_tensor * indexer_attn_q_b = nullptr; // note: for lora a/b, not bias\n\n    struct llama_layer_posnet posnet;\n\n    struct llama_layer_convnext convnext;\n\n    struct llama_layer_shortconv shortconv;\n\n    struct llama_layer_nextn nextn;\n};\n\nstruct llama_model {\n    llm_type type = LLM_TYPE_UNKNOWN;\n    llm_arch arch = LLM_ARCH_UNKNOWN;\n\n    std::string name = \"n/a\";\n\n    llama_hparams hparams = {};\n    llama_vocab   vocab;\n\n    // for classifier models\n    std::vector<std::string> classifier_labels;\n\n    struct ggml_tensor * tok_embd   = nullptr;\n    struct ggml_tensor * type_embd  = nullptr;\n    struct ggml_tensor * pos_embd   = nullptr;\n    struct ggml_tensor * tok_norm   = nullptr;\n    struct ggml_tensor * tok_norm_b = nullptr;\n\n    struct ggml_tensor * output_norm     = nullptr;\n    struct ggml_tensor * output_norm_b   = nullptr;\n    struct ggml_tensor * output          = nullptr;\n    struct ggml_tensor * output_b        = nullptr;\n    struct ggml_tensor * output_norm_enc = nullptr;\n\n    // classifier\n    struct ggml_tensor * cls       = nullptr;\n    struct ggml_tensor * cls_b     = nullptr;\n    struct ggml_tensor * cls_out   = nullptr;\n    struct ggml_tensor * cls_out_b = nullptr;\n    struct ggml_tensor * cls_norm  = nullptr;\n\n    struct ggml_tensor * conv1d   = nullptr;\n    struct ggml_tensor * conv1d_b = nullptr;\n\n    // gemma3n altup\n    struct ggml_tensor * tok_embd_per_layer   = nullptr;\n    struct ggml_tensor * altup_proj           = nullptr;\n    struct ggml_tensor * altup_unembd_proj    = nullptr;\n    struct ggml_tensor * per_layer_model_proj = nullptr;\n    struct ggml_tensor * per_layer_proj_norm  = nullptr;\n\n    std::vector<llama_layer> layers;\n\n    //Dense linear projections for SentenceTransformers models like embeddinggemma\n    // For Sentence Transformers models structure see\n    // https://sbert.net/docs/sentence_transformer/usage/custom_models.html#structure-of-sentence-transformer-models\n    struct ggml_tensor * dense_2_out_layers   = nullptr;\n    struct ggml_tensor * dense_2_out_layers_b = nullptr;\n    struct ggml_tensor * dense_3_out_layers   = nullptr;\n\n    // gguf metadata\n    std::unordered_map<std::string, std::string> gguf_kv;\n\n    // list of devices used in this model\n    std::vector<ggml_backend_dev_t> devices;\n\n    // for quantize-stats only\n    std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;\n\n    // for keeping track of associated LoRA adapters\n    std::unordered_set<llama_adapter_lora *> loras;\n\n    int64_t t_load_us  = 0;\n    int64_t t_start_us = 0;\n\n    explicit llama_model(const struct llama_model_params & params);\n    ~llama_model();\n\n    void load_stats  (llama_model_loader & ml);\n    void load_arch   (llama_model_loader & ml);\n    void load_hparams(llama_model_loader & ml);\n    void load_vocab  (llama_model_loader & ml);\n    bool load_tensors(llama_model_loader & ml); // returns false if cancelled by progress_callback\n\n    std::string arch_name() const;\n    std::string type_name() const;\n\n    std::string desc() const;\n\n    size_t size() const; // file size\n    size_t n_tensors() const;\n    size_t n_devices() const;\n\n    uint32_t n_gpu_layers() const;\n    llama_split_mode split_mode() const;\n\n    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;\n\n    // total number of parameters in the model\n    uint64_t n_elements() const;\n\n    void print_info() const;\n\n    ggml_backend_dev_t dev_layer(int il) const;\n    ggml_backend_dev_t dev_output() const;\n\n    ggml_backend_buffer_type_t select_buft(int il) const;\n\n    bool has_tensor_overrides() const;\n\n    const struct ggml_tensor * get_tensor(const char * name) const;\n\n    float get_rope_freq_base (const llama_cparams & cparams, int il) const;\n    float get_rope_freq_scale(const llama_cparams & cparams, int il) const;\n\n    ggml_tensor * get_rope_factors(const llama_cparams & cparams, int il) const;\n\n    // TODO: move this to new llm_arch_model_i interface\n    llama_memory_i * create_memory(const llama_memory_params & params, const llama_cparams & cparams) const;\n\n    // TODO: move this to new llm_arch_model_i interface\n    ggml_cgraph * build_graph(const llm_graph_params & params) const;\n\nprivate:\n    llama_model_params params;\n\n    struct impl;\n    std::unique_ptr<impl> pimpl;\n};\n\nconst char * llm_type_name(llm_type type);\n\n// For internal test use\n// TODO: remove\nconst std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model);\n"
  },
  {
    "path": "examples/talk-llama/llama-quant.cpp",
    "content": "#include \"llama.h\"\n#include \"llama-impl.h\"\n#include \"llama-model.h\"\n#include \"llama-model-loader.h\"\n\n#include <cmath>\n#include <cstring>\n#include <string>\n#include <cinttypes>\n#include <fstream>\n#include <mutex>\n#include <regex>\n#include <thread>\n#include <unordered_map>\n\n// result of parsing --tensor-type option\n// (changes to this struct must be reflected in tools/quantize/quantize.cpp)\nstruct tensor_type_option {\n    std::string name;\n    ggml_type type = GGML_TYPE_COUNT;\n};\n\n// tensor categorization - used to avoid repeated string matching in quantization logic.\n// this is different from LLM_TN - we want broad categories, not specific tensor names per arch.\nenum class tensor_category {\n    TOKEN_EMBD,\n    ATTENTION_Q,\n    ATTENTION_V,\n    ATTENTION_K,\n    ATTENTION_QKV,\n    ATTENTION_KV_B,\n    ATTENTION_OUTPUT,\n    FFN_UP,\n    FFN_GATE,\n    FFN_DOWN,\n    OUTPUT,\n    OTHER\n};\n\nstatic void zeros(std::ofstream & file, size_t n) {\n    char zero = 0;\n    for (size_t i = 0; i < n; ++i) {\n        file.write(&zero, 1);\n    }\n}\n\nstatic std::string remap_layer(const std::string & orig_name, const std::vector<int> & prune, std::map<int, std::string> & mapped, int & next_id) {\n    if (prune.empty()) {\n        return orig_name;\n    }\n\n    static const std::regex pattern(R\"(blk\\.(\\d+)\\.)\");\n    if (std::smatch match; std::regex_search(orig_name, match, pattern)) {\n        const int blk = std::stoi(match[1]);\n        std::string new_name = orig_name;\n\n        if (mapped.count(blk)) {\n            // Already mapped, do nothing\n        } else if (std::find(prune.begin(), prune.end(), blk) != prune.end()) {\n            mapped[blk] = \"\";\n        } else if (blk < prune.front()) {\n            mapped[blk] = std::to_string(blk);\n            next_id = blk + 1;\n        } else {\n            mapped[blk] = std::to_string(next_id);\n            ++next_id;\n        }\n\n        return mapped[blk].empty() ? mapped[blk] : new_name.replace(match.position(1), match.length(1), mapped[blk]);\n    }\n\n    return orig_name;\n}\n\nstatic std::string remap_imatrix(const std::string & orig_name, const std::map<int, std::string> & mapped) {\n    if (mapped.empty()) {\n        return orig_name;\n    }\n\n    static const std::regex pattern(R\"(blk\\.(\\d+)\\.)\");\n    if (std::smatch match; std::regex_search(orig_name, match, pattern)) {\n        const std::string blk(match[1]);\n        std::string new_name = orig_name;\n\n        for (const auto & p : mapped) {\n            if (p.second == blk) {\n                LLAMA_LOG_DEBUG(\"(blk.%d imatrix) \", p.first);\n                return new_name.replace(match.position(1), match.length(1), std::to_string(p.first));\n            }\n        }\n        GGML_ABORT(\"\\n%s: imatrix mapping error for %s\\n\", __func__, orig_name.c_str());\n    }\n\n    return orig_name;\n}\n\n//\n// helper functions for tensor name matching\n//\n\nstatic bool tensor_name_match_token_embd(const char * tensor_name) {\n    return std::strcmp(tensor_name, \"token_embd.weight\") == 0 ||\n           std::strcmp(tensor_name, \"per_layer_token_embd.weight\") == 0;\n}\n\nstatic bool tensor_name_match_output_weight(const char * tensor_name) {\n    return std::strcmp(tensor_name, \"output.weight\") == 0;\n}\n\n//\n// tensor categorization for quantization\n//\n// (this is different from LLM_TN - we want broad categories, not specific tensor names per arch)\n//\n\nstatic tensor_category tensor_get_category(const std::string & tensor_name) {\n    if (tensor_name_match_output_weight(tensor_name.c_str())) {\n        return tensor_category::OUTPUT;\n    }\n    if (tensor_name_match_token_embd(tensor_name.c_str())) {\n        return tensor_category::TOKEN_EMBD;\n    }\n    if (tensor_name.find(\"attn_qkv.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_QKV;\n    }\n    if (tensor_name.find(\"attn_kv_b.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_KV_B;\n    }\n    if (tensor_name.find(\"attn_v.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_V;\n    }\n    if (tensor_name.find(\"attn_k.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_K;\n    }\n    if (tensor_name.find(\"attn_q.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_Q;\n    }\n    if (tensor_name.find(\"attn_output.weight\") != std::string::npos) {\n        return tensor_category::ATTENTION_OUTPUT;\n    }\n    if (tensor_name.find(\"ffn_up\") != std::string::npos) {\n        return tensor_category::FFN_UP;\n    }\n    if (tensor_name.find(\"ffn_gate\") != std::string::npos) {\n        return tensor_category::FFN_GATE;\n    }\n    if (tensor_name.find(\"ffn_down\") != std::string::npos) {\n        return tensor_category::FFN_DOWN;\n    }\n    return tensor_category::OTHER;\n}\n\n// check if category is for attention-v-like tensors (more sensitive to quantization)\nstatic bool category_is_attn_v(tensor_category cat) {\n    return cat == tensor_category::ATTENTION_V     ||\n           cat == tensor_category::ATTENTION_QKV   ||\n           cat == tensor_category::ATTENTION_KV_B;\n}\n\n//\n// quantization state\n//\n\nstruct quantize_state_impl {\n    const llama_model                 & model;\n    const llama_model_quantize_params * params;\n\n    int n_attention_wv = 0;\n    int n_ffn_down     = 0;\n    int n_ffn_gate     = 0;\n    int n_ffn_up       = 0;\n    int i_attention_wv = 0;\n    int i_ffn_down     = 0;\n    int i_ffn_gate     = 0;\n    int i_ffn_up       = 0;\n\n    int n_fallback    = 0;\n\n    bool has_imatrix = false;\n\n    // used to figure out if a model has tied embeddings (tok_embd shares weights with output)\n    bool has_tied_embeddings = true; // assume tied until we see output.weight\n\n    // tensor type override patterns (compiled once, used twice)\n    std::vector<std::pair<std::regex, ggml_type>> tensor_type_patterns;\n\n    quantize_state_impl(const llama_model & model, const llama_model_quantize_params * params):\n        model(model), params(params)\n    {\n        // compile regex patterns once - they are expensive\n        if (params->tensor_types) {\n            const auto & tensor_types = *static_cast<const std::vector<tensor_type_option> *>(params->tensor_types);\n            for (const auto & [tname, qtype] : tensor_types) {\n                tensor_type_patterns.emplace_back(std::regex(tname), qtype);\n            }\n        }\n    }\n};\n\n// per-tensor metadata, computed in the preliminary loop and used in the main loop\nstruct tensor_metadata {\n    ggml_type       target_type;\n    tensor_category category;\n    std::string     remapped_imatrix_name;\n    bool            allows_quantization;\n    bool            requires_imatrix;\n};\n\n//\n// dequantization\n//\n\nstatic void llama_tensor_dequantize_impl(\n    ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,\n    const size_t nelements, const int nthread\n) {\n    if (output.size() < nelements) {\n        output.resize(nelements);\n    }\n    float * f32_output = (float *) output.data();\n\n    const ggml_type_traits * qtype = ggml_get_type_traits(tensor->type);\n    if (ggml_is_quantized(tensor->type)) {\n        if (qtype->to_float == NULL) {\n            throw std::runtime_error(format(\"type %s unsupported for integer quantization: no dequantization available\", ggml_type_name(tensor->type)));\n        }\n    } else if (tensor->type != GGML_TYPE_F16 &&\n               tensor->type != GGML_TYPE_BF16) {\n        throw std::runtime_error(format(\"cannot dequantize/convert tensor type %s\", ggml_type_name(tensor->type)));\n    }\n\n    if (nthread < 2) {\n        if (tensor->type == GGML_TYPE_F16) {\n            ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);\n        } else if (tensor->type == GGML_TYPE_BF16) {\n            ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);\n        } else if (ggml_is_quantized(tensor->type)) {\n            qtype->to_float(tensor->data, f32_output, nelements);\n        } else {\n            GGML_ABORT(\"fatal error\"); // unreachable\n        }\n        return;\n    }\n\n    size_t block_size;\n    if (tensor->type == GGML_TYPE_F16 ||\n        tensor->type == GGML_TYPE_BF16) {\n        block_size = 1;\n    } else {\n        block_size = (size_t)ggml_blck_size(tensor->type);\n    }\n\n    size_t block_size_bytes = ggml_type_size(tensor->type);\n\n    GGML_ASSERT(nelements % block_size == 0);\n    size_t nblocks = nelements / block_size;\n    size_t blocks_per_thread = nblocks / nthread;\n    size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count\n\n    size_t in_buff_offs = 0;\n    size_t out_buff_offs = 0;\n\n    for (int tnum = 0; tnum < nthread; tnum++) {\n        size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread\n        size_t thr_elems = thr_blocks * block_size; // number of elements for this thread\n        size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread\n\n        auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {\n            if (typ == GGML_TYPE_F16) {\n                ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);\n            } else if (typ == GGML_TYPE_BF16) {\n                ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);\n            } else {\n                qtype->to_float(inbuf, outbuf, nels);\n            }\n        };\n        workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);\n        in_buff_offs += thr_block_bytes;\n        out_buff_offs += thr_elems;\n    }\n    for (auto & w : workers) { w.join(); }\n    workers.clear();\n}\n\n//\n// do we allow this tensor to be quantized?\n//\n\nstatic bool tensor_allows_quantization(const llama_model_quantize_params * params, llm_arch arch, const ggml_tensor * tensor) {\n    // trivial checks first -- no string ops needed\n    if (params->only_copy)       return false;\n\n    // quantize only 2D and 3D tensors (experts)\n    if (ggml_n_dims(tensor) < 2) return false;\n\n    const std::string name = ggml_get_name(tensor);\n\n    // This used to be a regex, but <regex> has an extreme cost to compile times.\n    bool quantize = name.rfind(\"weight\") == name.size() - 6; // ends with 'weight'?\n\n    // do not quantize norm tensors\n    quantize &= name.find(\"_norm.weight\") == std::string::npos;\n\n    quantize &= params->quantize_output_tensor || name != \"output.weight\";\n\n    // do not quantize expert gating tensors\n    // NOTE: can't use LLM_TN here because the layer number is not known\n    quantize &= name.find(\"ffn_gate_inp.weight\") == std::string::npos;\n\n    // these are very small (e.g. 4x4)\n    quantize &= name.find(\"altup\")  == std::string::npos;\n    quantize &= name.find(\"laurel\") == std::string::npos;\n\n    // these are not too big so keep them as it is\n    quantize &= name.find(\"per_layer_model_proj\") == std::string::npos;\n\n    // do not quantize positional embeddings and token types (BERT)\n    quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD,    \"weight\");\n    quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, \"weight\");\n\n    // do not quantize Mamba/Kimi's small conv1d weights\n    // NOTE: can't use LLM_TN here because the layer number is not known\n    quantize &= name.find(\"ssm_conv1d\") == std::string::npos;\n    quantize &= name.find(\"shortconv.conv.weight\") == std::string::npos;\n\n    // do not quantize RWKV's small yet 2D weights\n    quantize &= name.find(\"time_mix_first.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_w0.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_w1.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_w2.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_v0.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_v1.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_v2.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_a0.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_a1.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_a2.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_g1.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_g2.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_decay_w1.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_decay_w2.weight\") == std::string::npos;\n    quantize &= name.find(\"time_mix_lerp_fused.weight\") == std::string::npos;\n\n    // do not quantize relative position bias (T5)\n    quantize &= name.find(\"attn_rel_b.weight\") == std::string::npos;\n\n    // do not quantize specific multimodal tensors\n    quantize &= name.find(\".position_embd.\") == std::string::npos;\n\n    return quantize;\n}\n\n//\n// tensor type selection\n//\n\n// incompatible tensor shapes are handled here - fallback to a compatible type\nstatic ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tensor * t, const ggml_type target_type) {\n    ggml_type return_type = target_type;\n\n    const int64_t ncols = t->ne[0];\n    const int64_t qk_k = ggml_blck_size(target_type);\n\n    if (ncols % qk_k != 0) { // this tensor's shape is incompatible with this quant\n        LLAMA_LOG_WARN(\"warning: %-36s - ncols %6\" PRId64 \" not divisible by %3\" PRId64 \" (required for type %7s) \",\n                        t->name, ncols, qk_k, ggml_type_name(target_type));\n        ++qs.n_fallback;\n\n        switch (target_type) {\n            // types on the left: block size 256\n            case GGML_TYPE_IQ1_S:\n            case GGML_TYPE_IQ1_M:\n            case GGML_TYPE_IQ2_XXS:\n            case GGML_TYPE_IQ2_XS:\n            case GGML_TYPE_IQ2_S:\n            case GGML_TYPE_IQ3_XXS:\n            case GGML_TYPE_IQ3_S:   // types on the right: block size 32\n            case GGML_TYPE_IQ4_XS:  return_type = GGML_TYPE_IQ4_NL; break;\n            case GGML_TYPE_Q2_K:\n            case GGML_TYPE_Q3_K:\n            case GGML_TYPE_TQ1_0:\n            case GGML_TYPE_TQ2_0:   return_type = GGML_TYPE_Q4_0;   break;\n            case GGML_TYPE_Q4_K:    return_type = GGML_TYPE_Q5_0;   break;\n            case GGML_TYPE_Q5_K:    return_type = GGML_TYPE_Q5_1;   break;\n            case GGML_TYPE_Q6_K:    return_type = GGML_TYPE_Q8_0;   break;\n            default:\n                throw std::runtime_error(format(\"no tensor type fallback is defined for type %s\",\n                                                ggml_type_name(target_type)));\n        }\n        if (ncols % ggml_blck_size(return_type) != 0) {\n            //\n            // the fallback return type is still not compatible for this tensor!\n            //\n            // most likely, this tensor's first dimension is not divisible by 32.\n            // this is very rare. we can either abort the quantization, or\n            // fallback to F16 / F32.\n            //\n            LLAMA_LOG_WARN(\"(WARNING: must use F16 due to unusual shape) \");\n            return_type = GGML_TYPE_F16;\n        }\n        LLAMA_LOG_WARN(\"-> falling back to %7s\\n\", ggml_type_name(return_type));\n    }\n    return return_type;\n}\n\n// internal standard logic for selecting the target tensor type based on tensor category, ftype, and model arch\nstatic ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) {\n    const std::string name = ggml_get_name(tensor);\n\n    // TODO: avoid hardcoded tensor names - use the TN_* constants\n    const llm_arch arch = qs.model.arch;\n\n    auto use_more_bits = [](int i_layer, int n_layers) -> bool {\n        return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;\n    };\n    const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);\n    auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {\n        if (n_expert > 1) {\n            // Believe it or not, \"experts\" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly\n            // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work\n            // for getting the current layer as I initially thought, and we need to resort to parsing the\n            // tensor name.\n            if (sscanf(name, \"blk.%d.\", &i_layer) != 1) {\n                throw std::runtime_error(format(\"Failed to determine layer for tensor %s\", name));\n            }\n            if (i_layer < 0 || i_layer >= n_layer) {\n                throw std::runtime_error(format(\"Bad layer %d for tensor %s. Must be in [0, %d)\", i_layer, name, n_layer));\n            }\n        }\n        return std::make_pair(i_layer, n_layer);\n    };\n\n    // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings\n    // with the quantization of the output tensor\n    if (category == tensor_category::OUTPUT || (qs.has_tied_embeddings && category == tensor_category::TOKEN_EMBD)) {\n        if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {\n            new_type = qs.params->output_tensor_type;\n        } else {\n            const int64_t nx = tensor->ne[0];\n            const int64_t qk_k = ggml_blck_size(new_type);\n\n            if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {\n                new_type = GGML_TYPE_Q8_0;\n            }\n            else if (arch == LLM_ARCH_FALCON || nx % qk_k != 0) {\n                new_type = GGML_TYPE_Q8_0;\n            }\n            else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||\n                     ftype == LLAMA_FTYPE_MOSTLY_IQ1_S   || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S  || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M   ||\n                     ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {\n                new_type = GGML_TYPE_Q5_K;\n            }\n            else if (new_type != GGML_TYPE_Q8_0) {\n                new_type = GGML_TYPE_Q6_K;\n            }\n        }\n    } else if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {\n        // MoE   tensors -> MXFP4\n        // other tensors -> Q8_0\n        if (tensor->ne[2] > 1) {\n            new_type = GGML_TYPE_MXFP4;\n        } else {\n            new_type = GGML_TYPE_Q8_0;\n        }\n    } else if (category == tensor_category::TOKEN_EMBD) {\n        if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {\n            new_type = qs.params->token_embedding_type;\n        } else {\n            if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||\n                ftype == LLAMA_FTYPE_MOSTLY_IQ1_S   || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {\n                new_type = GGML_TYPE_Q2_K;\n            }\n            else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {\n                new_type = GGML_TYPE_IQ3_S;\n            }\n            else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {\n                new_type = GGML_TYPE_IQ3_S;\n            }\n            else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0) {\n                new_type = GGML_TYPE_Q4_K;\n            }\n        }\n    } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||\n               ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M    || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {\n        if (category_is_attn_v(category)) {\n            if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;\n            else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;\n            ++qs.i_attention_wv;\n        }\n        else if (qs.model.hparams.n_expert == 8 && category == tensor_category::ATTENTION_K) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (category == tensor_category::FFN_DOWN) {\n            if (qs.i_ffn_down < qs.n_ffn_down/8) {\n                new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;\n            }\n            ++qs.i_ffn_down;\n        }\n        else if (category == tensor_category::ATTENTION_OUTPUT) {\n            if (qs.model.hparams.n_expert == 8) {\n                new_type = GGML_TYPE_Q5_K;\n            } else {\n                if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) new_type = GGML_TYPE_IQ2_XXS;\n                else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S;\n            }\n        }\n    } else if (category_is_attn_v(category)) {\n        if      (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {\n            new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {\n            new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS;\n        }\n        else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {\n            new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;\n        else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && qs.model.hparams.n_gqa() >= 4) {\n            new_type = GGML_TYPE_Q5_K;\n        }\n        else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&\n                use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K;\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K;\n        if (qs.model.type == LLM_TYPE_70B) {\n            // In the 70B model we have 8 heads sharing the same attn_v weights. As a result, the attn_v.weight tensor is\n            // 8x smaller compared to attn_q.weight. Hence, we can get a nice boost in quantization accuracy with\n            // nearly negligible increase in model size by quantizing this tensor with more bits:\n            if (new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K) new_type = GGML_TYPE_Q5_K;\n        }\n        if (qs.model.hparams.n_expert == 8) {\n            // for the 8-expert model, bumping this to Q8_0 trades just ~128MB\n            // TODO: explore better strategies\n            new_type = GGML_TYPE_Q8_0;\n        }\n        ++qs.i_attention_wv;\n    } else if (category == tensor_category::ATTENTION_K) {\n        if (qs.model.hparams.n_expert == 8) {\n            // for the 8-expert model, bumping this to Q8_0 trades just ~128MB\n            // TODO: explore better strategies\n            new_type = GGML_TYPE_Q8_0;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {\n            new_type = GGML_TYPE_IQ3_XXS;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {\n            new_type = GGML_TYPE_IQ2_S;\n        }\n    } else if (category == tensor_category::ATTENTION_Q) {\n        if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {\n            new_type = GGML_TYPE_IQ3_XXS;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {\n            new_type = GGML_TYPE_IQ2_S;\n        }\n    } else if (category == tensor_category::FFN_DOWN) {\n        auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str());\n        int i_layer = info.first, n_layer = info.second;\n        if      (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) {\n            if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) {\n            new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {\n            new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K\n                     : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K\n                     : GGML_TYPE_Q3_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 ||\n                    (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {\n            new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {\n            if (arch == LLM_ARCH_FALCON) {\n                new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K :\n                           use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;\n            } else {\n                if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;\n            }\n        }\n        else if (i_layer < n_layer/8 && (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && !qs.has_imatrix) {\n            new_type = GGML_TYPE_Q5_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {\n            new_type = GGML_TYPE_Q5_K;\n        }\n        else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || ftype == LLAMA_FTYPE_MOSTLY_Q5_0)\n                && qs.has_imatrix && i_layer < n_layer/8) {\n            // Guard against craziness in the first few ffn_down layers that can happen even with imatrix for Q4_0/Q5_0.\n            // We only do it when an imatrix is provided because a) we want to make sure that one can always get the\n            // same quantization as before imatrix stuff, and b) Q4_1/Q5_1 do go crazy on ffn_down without an imatrix.\n            new_type = ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q5_1;\n        }\n        ++qs.i_ffn_down;\n    } else if (category == tensor_category::ATTENTION_OUTPUT) {\n        if (arch != LLM_ARCH_FALCON) {\n            if (qs.model.hparams.n_expert == 8) {\n                if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K   || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||\n                    ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M  || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL  ||\n                    ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M  || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S  ||\n                    ftype == LLAMA_FTYPE_MOSTLY_IQ3_M  || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) {\n                    new_type = GGML_TYPE_Q5_K;\n                }\n            } else {\n                if      (ftype == LLAMA_FTYPE_MOSTLY_Q2_K   ) new_type = GGML_TYPE_Q3_K;\n                else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S;\n                else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K;\n                else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K;\n                else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M  ) new_type = GGML_TYPE_Q4_K;\n            }\n        } else {\n            if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K;\n        }\n    }\n    else if (category == tensor_category::ATTENTION_QKV) {\n        if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {\n            new_type = GGML_TYPE_Q4_K;\n        }\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K;\n        else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K;\n    }\n    else if (category == tensor_category::FFN_GATE) {\n        auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str());\n        int i_layer = info.first, n_layer = info.second;\n        if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {\n            new_type = GGML_TYPE_IQ3_XXS;\n        }\n        ++qs.i_ffn_gate;\n    }\n    else if (category == tensor_category::FFN_UP) {\n        auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str());\n        int i_layer = info.first, n_layer = info.second;\n        if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {\n            new_type = GGML_TYPE_IQ3_XXS;\n        }\n        ++qs.i_ffn_up;\n    }\n\n    return new_type;\n}\n\n// outer wrapper: determine the ggml_type that this tensor should be quantized to\nstatic ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_model_quantize_params * params, const ggml_tensor * tensor, ggml_type default_type, const tensor_metadata & tm) {\n    if (!tensor_allows_quantization(params, qs.model.arch, tensor)) {\n        return tensor->type;\n    }\n    if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) {\n        return params->token_embedding_type;\n    }\n    if (params->output_tensor_type < GGML_TYPE_COUNT && tm.category == tensor_category::OUTPUT) {\n        return params->output_tensor_type;\n    }\n\n    ggml_type new_type = default_type;\n\n    // get more optimal quantization type based on the tensor shape, layer, etc.\n    if (!params->pure && ggml_is_quantized(default_type)) {\n        // if the user provided tensor types - use those\n        bool manual = false;\n        if (!qs.tensor_type_patterns.empty()) {\n            const std::string tensor_name(tensor->name);\n            for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {\n                if (std::regex_search(tensor_name, pattern)) {\n                    if (qtype != new_type) {\n                        LLAMA_LOG_WARN(\"%s: %-36s - applying manual override: %s -> %s\\n\",\n                                       __func__, tensor_name.c_str(), ggml_type_name(new_type), ggml_type_name(qtype));\n                        new_type = qtype;\n                        manual = true;\n                        break;\n                    }\n                }\n            }\n        }\n\n        // if not manual - use the standard logic for choosing the quantization type based on the selected mixture\n        if (!manual) {\n            new_type = llama_tensor_get_type_impl(qs, new_type, tensor, params->ftype, tm.category);\n        }\n\n        // incompatible tensor shapes are handled here - fallback to a compatible type\n        new_type = tensor_type_fallback(qs, tensor, new_type);\n    }\n\n    return new_type;\n}\n\n//\n// quantization implementation\n//\n\nstatic size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {\n    if (nthread < 2) {\n        // single-thread\n        size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);\n        if (!ggml_validate_row_data(new_type, new_data, new_size)) {\n            throw std::runtime_error(\"quantized data validation failed\");\n        }\n        return new_size;\n    }\n\n    std::mutex mutex;\n    int64_t counter = 0;\n    size_t new_size = 0;\n    bool valid = true;\n    auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,\n            nrows, n_per_row, imatrix]() {\n        const int64_t nrows_per_chunk = chunk_size / n_per_row;\n        size_t local_size = 0;\n        while (true) {\n            std::unique_lock<std::mutex> lock(mutex);\n            int64_t first_row = counter; counter += nrows_per_chunk;\n            if (first_row >= nrows) {\n                if (local_size > 0) {\n                    new_size += local_size;\n                }\n                break;\n            }\n            lock.unlock();\n            const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);\n            size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);\n            local_size += this_size;\n\n            // validate the quantized data\n            const size_t row_size  = ggml_row_size(new_type, n_per_row);\n            void * this_data = (char *) new_data + first_row * row_size;\n            if (!ggml_validate_row_data(new_type, this_data, this_size)) {\n                std::unique_lock<std::mutex> lock(mutex);\n                valid = false;\n                break;\n            }\n        }\n    };\n    for (int it = 0; it < nthread - 1; ++it) {\n        workers.emplace_back(compute);\n    }\n    compute();\n    for (auto & w : workers) { w.join(); }\n    workers.clear();\n    if (!valid) {\n        throw std::runtime_error(\"quantized data validation failed\");\n    }\n    return new_size;\n}\n\n//\n// imatrix requirement check\n//\n\nstatic bool tensor_requires_imatrix(const char * tensor_name, const ggml_type dst_type, const llama_ftype ftype) {\n    if (tensor_name_match_token_embd(tensor_name) || tensor_name_match_output_weight(tensor_name)) {\n        return false;\n    }\n    switch (dst_type) {\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ2_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ1_S:\n            return true;\n        case GGML_TYPE_Q2_K:\n            // as a general rule, the k-type quantizations don't require imatrix data.\n            // the only exception is Q2_K tensors that are part of a Q2_K_S file.\n            return ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S;\n        default:\n            return false;\n    }\n}\n\n//\n// given a file type, get the default tensor type\n//\n\nstatic ggml_type llama_ftype_get_default_type(llama_ftype ftype) {\n    switch (ftype) {\n        case LLAMA_FTYPE_MOSTLY_Q4_0: return GGML_TYPE_Q4_0;\n        case LLAMA_FTYPE_MOSTLY_Q4_1: return GGML_TYPE_Q4_1;\n        case LLAMA_FTYPE_MOSTLY_Q5_0: return GGML_TYPE_Q5_0;\n        case LLAMA_FTYPE_MOSTLY_Q5_1: return GGML_TYPE_Q5_1;\n        case LLAMA_FTYPE_MOSTLY_Q8_0: return GGML_TYPE_Q8_0;\n        case LLAMA_FTYPE_MOSTLY_F16:  return GGML_TYPE_F16;\n        case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16;\n        case LLAMA_FTYPE_ALL_F32:     return GGML_TYPE_F32;\n\n        case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;\n\n        // K-quants\n        case LLAMA_FTYPE_MOSTLY_Q2_K_S:\n        case LLAMA_FTYPE_MOSTLY_Q2_K:    return GGML_TYPE_Q2_K;\n        case LLAMA_FTYPE_MOSTLY_IQ3_XS:  return GGML_TYPE_IQ3_S;\n        case LLAMA_FTYPE_MOSTLY_Q3_K_S:\n        case LLAMA_FTYPE_MOSTLY_Q3_K_M:\n        case LLAMA_FTYPE_MOSTLY_Q3_K_L:  return GGML_TYPE_Q3_K;\n        case LLAMA_FTYPE_MOSTLY_Q4_K_S:\n        case LLAMA_FTYPE_MOSTLY_Q4_K_M:  return GGML_TYPE_Q4_K;\n        case LLAMA_FTYPE_MOSTLY_Q5_K_S:\n        case LLAMA_FTYPE_MOSTLY_Q5_K_M:  return GGML_TYPE_Q5_K;\n        case LLAMA_FTYPE_MOSTLY_Q6_K:    return GGML_TYPE_Q6_K;\n        case LLAMA_FTYPE_MOSTLY_TQ1_0:   return GGML_TYPE_TQ1_0;\n        case LLAMA_FTYPE_MOSTLY_TQ2_0:   return GGML_TYPE_TQ2_0;\n        case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return GGML_TYPE_IQ2_XXS;\n        case LLAMA_FTYPE_MOSTLY_IQ2_XS:  return GGML_TYPE_IQ2_XS;\n        case LLAMA_FTYPE_MOSTLY_IQ2_S:   return GGML_TYPE_IQ2_XS;\n        case LLAMA_FTYPE_MOSTLY_IQ2_M:   return GGML_TYPE_IQ2_S;\n        case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return GGML_TYPE_IQ3_XXS;\n        case LLAMA_FTYPE_MOSTLY_IQ1_S:   return GGML_TYPE_IQ1_S;\n        case LLAMA_FTYPE_MOSTLY_IQ1_M:   return GGML_TYPE_IQ1_M;\n        case LLAMA_FTYPE_MOSTLY_IQ4_NL:  return GGML_TYPE_IQ4_NL;\n        case LLAMA_FTYPE_MOSTLY_IQ4_XS:  return GGML_TYPE_IQ4_XS;\n        case LLAMA_FTYPE_MOSTLY_IQ3_S:\n        case LLAMA_FTYPE_MOSTLY_IQ3_M:   return GGML_TYPE_IQ3_S;\n\n        default: throw std::runtime_error(format(\"invalid output file type %d\\n\", ftype));\n    }\n}\n\n//\n// main quantization driver\n//\n\nstatic void llama_model_quantize_impl(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {\n    ggml_type default_type;\n    llama_ftype ftype = params->ftype;\n\n    int nthread = params->nthread;\n\n    if (nthread <= 0) {\n        nthread = std::thread::hardware_concurrency();\n    }\n\n    default_type = llama_ftype_get_default_type(ftype);\n\n    // mmap consistently increases speed on Linux, and also increases speed on Windows with\n    // hot cache. It may cause a slowdown on macOS, possibly related to free memory.\n#if defined(__linux__) || defined(_WIN32)\n    constexpr bool use_mmap = true;\n#else\n    constexpr bool use_mmap = false;\n#endif\n\n    llama_model_kv_override * kv_overrides = nullptr;\n    if (params->kv_overrides) {\n        auto * v = (std::vector<llama_model_kv_override>*)params->kv_overrides;\n        kv_overrides = v->data();\n    }\n\n    std::vector<std::string> splits = {};\n    llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,\n        fname_inp, splits, use_mmap, /*use_direct_io*/ false, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);\n    ml.init_mappings(false); // no prefetching\n\n    llama_model model(llama_model_default_params());\n\n    model.load_arch   (ml);\n    model.load_hparams(ml);\n    model.load_stats  (ml);\n\n    quantize_state_impl qs(model, params);\n\n    if (params->only_copy) {\n        ftype = ml.ftype;\n    }\n    const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;\n    if (params->imatrix) {\n        imatrix_data = static_cast<const std::unordered_map<std::string, std::vector<float>>*>(params->imatrix);\n        if (imatrix_data) {\n            LLAMA_LOG_INFO(\"\\n%s: have importance matrix data with %d entries\\n\",\n                           __func__, (int)imatrix_data->size());\n            qs.has_imatrix = true;\n            // check imatrix for nans or infs\n            for (const auto & kv : *imatrix_data) {\n                for (float f : kv.second) {\n                    if (!std::isfinite(f)) {\n                        throw std::runtime_error(format(\"imatrix contains non-finite value %f\\n\", f));\n                    }\n                }\n            }\n        }\n    }\n\n    const size_t align = GGUF_DEFAULT_ALIGNMENT;\n    gguf_context_ptr ctx_out { gguf_init_empty() };\n\n    std::vector<int> prune_list = {};\n    if (params->prune_layers) {\n        prune_list = *static_cast<const std::vector<int> *>(params->prune_layers);\n    }\n\n    // copy the KV pairs from the input file\n    gguf_set_kv     (ctx_out.get(), ml.metadata);\n    gguf_set_val_u32(ctx_out.get(), \"general.quantization_version\", GGML_QNT_VERSION); // TODO: use LLM_KV\n    gguf_set_val_u32(ctx_out.get(), \"general.file_type\", ftype); // TODO: use LLM_KV\n\n    // Remove split metadata\n    gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str());\n    gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str());\n    gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str());\n\n    if (params->kv_overrides) {\n        const std::vector<llama_model_kv_override> & overrides = *(const std::vector<llama_model_kv_override> *)params->kv_overrides;\n        for (const auto & o : overrides) {\n            if (o.key[0] == 0) break;\n            if (o.tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {\n                gguf_set_val_f32(ctx_out.get(), o.key, o.val_f64);\n            } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_INT) {\n                // Setting type to UINT32. See https://github.com/ggml-org/llama.cpp/pull/14182 for context\n                gguf_set_val_u32(ctx_out.get(), o.key, (uint32_t)std::abs(o.val_i64));\n            } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {\n                gguf_set_val_bool(ctx_out.get(), o.key, o.val_bool);\n            } else if (o.tag == LLAMA_KV_OVERRIDE_TYPE_STR) {\n                gguf_set_val_str(ctx_out.get(), o.key, o.val_str);\n            } else {\n                LLAMA_LOG_WARN(\"%s: unknown KV override type for key %s\\n\", __func__, o.key);\n            }\n        }\n    }\n\n    std::map<int, std::string> mapped;\n    int blk_id = 0;\n\n    // make a list of weights\n    std::vector<const llama_model_loader::llama_tensor_weight *> tensors;\n    tensors.reserve(ml.weights_map.size());\n    for (const auto & it : ml.weights_map) {\n        const std::string remapped_name(remap_layer(it.first, prune_list, mapped, blk_id));\n        if (remapped_name.empty()) {\n            LLAMA_LOG_DEBUG(\"%s: pruning tensor %s\\n\", __func__, it.first.c_str());\n            continue;\n        }\n\n        if (remapped_name != it.first) {\n            ggml_set_name(it.second.tensor, remapped_name.c_str());\n            LLAMA_LOG_DEBUG(\"%s: tensor %s remapped to %s\\n\", __func__, it.first.c_str(), ggml_get_name(it.second.tensor));\n        }\n        tensors.push_back(&it.second);\n    }\n    if (!prune_list.empty()) {\n        gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_BLOCK_COUNT).c_str(), blk_id);\n    }\n\n    // keep_split requires that the weights are sorted by split index\n    if (params->keep_split) {\n        std::sort(tensors.begin(), tensors.end(), [](const llama_model_loader::llama_tensor_weight * a, const llama_model_loader::llama_tensor_weight * b) {\n            if (a->idx == b->idx) {\n                return a->offs < b->offs;\n            }\n            return a->idx < b->idx;\n        });\n    }\n\n    int idx = 0;\n    uint16_t n_split = 1;\n\n    // Assume split index is continuous\n    if (params->keep_split) {\n        for (const auto * it : tensors) {\n            n_split = std::max(uint16_t(it->idx + 1), n_split);\n        }\n    }\n    std::vector<gguf_context_ptr> ctx_outs(n_split);\n    ctx_outs[0] = std::move(ctx_out);\n\n    // compute tensor metadata once and cache it\n    std::vector<tensor_metadata> metadata(tensors.size());\n\n    // initialize quantization state before preliminary loop (counters for use_more_bits)\n    {\n        for (size_t i = 0; i < tensors.size(); ++i) {\n            const auto cat = tensor_get_category(tensors[i]->tensor->name);\n            if (category_is_attn_v(cat)) {\n                ++qs.n_attention_wv;\n            }\n            if (cat == tensor_category::OUTPUT) {\n                qs.has_tied_embeddings = false;\n            }\n            metadata[i].category = cat; // save and re-use the category while we're at it\n        }\n        // these also need to be set to n_layer by default\n        qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer;\n    }\n\n    // flag for --dry-run\n    bool will_require_imatrix = false;\n\n    //\n    // preliminary iteration over all weights\n    //\n\n    for (size_t i = 0; i < tensors.size(); ++i) {\n        const auto * it = tensors[i];\n        const struct ggml_tensor * tensor = it->tensor;\n        const std::string name = ggml_get_name(tensor);\n\n        uint16_t i_split = params->keep_split ? it->idx : 0;\n        if (!ctx_outs[i_split]) {\n            ctx_outs[i_split].reset(gguf_init_empty());\n        }\n        gguf_add_tensor(ctx_outs[i_split].get(), tensor);\n\n        metadata[i].allows_quantization = tensor_allows_quantization(params, model.arch, tensor);\n\n        if (metadata[i].allows_quantization) {\n            metadata[i].target_type = llama_tensor_get_type(qs, params, tensor, default_type, metadata[i]);\n        } else {\n            metadata[i].target_type = tensor->type;\n        }\n\n        metadata[i].requires_imatrix = tensor_requires_imatrix(tensor->name, metadata[i].target_type, ftype);\n\n        if (params->imatrix) {\n            metadata[i].remapped_imatrix_name = remap_imatrix(tensor->name, mapped);\n        } else if (metadata[i].allows_quantization && metadata[i].requires_imatrix) {\n            if (params->dry_run) {\n                will_require_imatrix = true;\n            } else {\n                LLAMA_LOG_ERROR(\"\\n============================================================================\\n\"\n                                \" ERROR: this quantization requires an importance matrix!\\n\"\n                                \"        - offending tensor: %s\\n\"\n                                \"        - target type: %s\\n\"\n                                \"============================================================================\\n\\n\",\n                                name.c_str(), ggml_type_name(metadata[i].target_type));\n                throw std::runtime_error(\"this quantization requires an imatrix!\");\n            }\n        }\n    }\n\n    // Set split info if needed\n    if (n_split > 1) {\n        for (size_t i = 0; i < ctx_outs.size(); ++i) {\n            gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str(), i);\n            gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str(), n_split);\n            gguf_set_val_i32(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str(), (int32_t)tensors.size());\n        }\n    }\n\n    size_t total_size_org = 0;\n    size_t total_size_new = 0;\n\n    std::vector<std::thread> workers;\n    workers.reserve(nthread);\n\n    std::vector<no_init<uint8_t>> read_data;\n    std::vector<no_init<uint8_t>> work;\n    std::vector<no_init<float>> f32_conv_buf;\n\n    int cur_split = -1;\n    std::ofstream fout;\n    auto close_ofstream = [&]() {\n        // Write metadata and close file handler\n        if (fout.is_open()) {\n            fout.seekp(0);\n            std::vector<uint8_t> data(gguf_get_meta_size(ctx_outs[cur_split].get()));\n            gguf_get_meta_data(ctx_outs[cur_split].get(), data.data());\n            fout.write((const char *) data.data(), data.size());\n            fout.close();\n        }\n    };\n    auto new_ofstream = [&](int index) {\n        cur_split = index;\n        GGML_ASSERT(ctx_outs[cur_split] && \"Find uninitialized gguf_context\");\n        std::string fname = fname_out;\n        if (params->keep_split) {\n            std::vector<char> split_path(llama_path_max(), 0);\n            llama_split_path(split_path.data(), split_path.size(), fname_out.c_str(), cur_split, n_split);\n            fname = std::string(split_path.data());\n        }\n\n        fout = std::ofstream(fname, std::ios::binary);\n        fout.exceptions(std::ofstream::failbit); // fail fast on write errors\n        const size_t meta_size = gguf_get_meta_size(ctx_outs[cur_split].get());\n        // placeholder for the meta data\n        ::zeros(fout, meta_size);\n    };\n\n    // no output file for --dry-run\n    if (!params->dry_run) {\n        new_ofstream(0);\n    }\n\n    //\n    // main loop: iterate over all weights\n    //\n\n    for (size_t i = 0; i < tensors.size(); ++i) {\n        const auto & weight = *tensors[i];\n        const auto & tm = metadata[i];\n        ggml_tensor * tensor = weight.tensor;\n\n        if (!params->dry_run && (weight.idx != cur_split && params->keep_split)) {\n            close_ofstream();\n            new_ofstream(weight.idx);\n        }\n\n        const std::string name = ggml_get_name(tensor);\n        const size_t tensor_size = ggml_nbytes(tensor);\n\n        if (!params->dry_run) {\n            if (!ml.use_mmap) {\n                if (read_data.size() < tensor_size) {\n                    read_data.resize(tensor_size);\n                }\n                tensor->data = read_data.data();\n            }\n            ml.load_data_for(tensor);\n        }\n\n        LLAMA_LOG_INFO(\"[%4d/%4d] %-36s - [%s], type = %6s, \",\n               ++idx, ml.n_tensors,\n               ggml_get_name(tensor),\n               llama_format_tensor_shape(tensor).c_str(),\n               ggml_type_name(tensor->type));\n\n        const ggml_type cur_type = tensor->type;\n        const ggml_type new_type = tm.target_type;\n\n        // If we've decided to quantize to the same type the tensor is already\n        // in then there's nothing to do.\n        bool quantize = cur_type != new_type;\n\n        void * new_data;\n        size_t new_size;\n\n        if (params->dry_run) {\n            // the --dry-run option calculates the final quantization size without quantizing\n            if (quantize) {\n                new_size = ggml_nrows(tensor) * ggml_row_size(new_type, tensor->ne[0]);\n                LLAMA_LOG_INFO(\"size = %8.2f MiB -> %8.2f MiB (%s)\\n\",\n                               tensor_size/1024.0/1024.0,\n                               new_size/1024.0/1024.0,\n                               ggml_type_name(new_type));\n                if (!will_require_imatrix && tm.requires_imatrix) {\n                    will_require_imatrix = true;\n                }\n            } else {\n                new_size = tensor_size;\n                LLAMA_LOG_INFO(\"size = %8.3f MiB\\n\", new_size/1024.0/1024.0);\n            }\n            total_size_org += tensor_size;\n            total_size_new += new_size;\n            continue;\n        } else {\n            // no --dry-run, perform quantization\n            if (!quantize) {\n                new_data = tensor->data;\n                new_size = tensor_size;\n                LLAMA_LOG_INFO(\"size = %8.3f MiB\\n\", tensor_size/1024.0/1024.0);\n            } else {\n                const int64_t nelements = ggml_nelements(tensor);\n\n                const float * imatrix = nullptr;\n                if (imatrix_data) {\n                    auto it = imatrix_data->find(tm.remapped_imatrix_name);\n                    if (it == imatrix_data->end()) {\n                        LLAMA_LOG_INFO(\"\\n====== %s: did not find weights for %s\\n\", __func__, tensor->name);\n                    } else {\n                        if (it->second.size() == (size_t)tensor->ne[0]*tensor->ne[2]) {\n                            imatrix = it->second.data();\n                        } else {\n                            LLAMA_LOG_INFO(\"\\n====== %s: imatrix size %d is different from tensor size %d for %s\\n\", __func__,\n                                    int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name);\n\n                            // this can happen when quantizing an old mixtral model with split tensors with a new incompatible imatrix\n                            // this is a significant error and it may be good idea to abort the process if this happens,\n                            // since many people will miss the error and not realize that most of the model is being quantized without an imatrix\n                            // tok_embd should be ignored in this case, since it always causes this warning\n                            if (!tensor_name_match_token_embd(tensor->name)) {\n                                throw std::runtime_error(format(\"imatrix size %d is different from tensor size %d for %s\",\n                                        int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name));\n                            }\n                        }\n                    }\n                }\n                if (!imatrix && tm.requires_imatrix) {\n                    LLAMA_LOG_ERROR(\"\\n\\n============================================================\\n\");\n                    LLAMA_LOG_ERROR(\"Missing importance matrix for tensor %s in a very low-bit quantization\\n\", tensor->name);\n                    LLAMA_LOG_ERROR(\"The result will be garbage, so bailing out\\n\");\n                    LLAMA_LOG_ERROR(\"============================================================\\n\\n\");\n                    throw std::runtime_error(format(\"Missing importance matrix for tensor %s in a very low-bit quantization\", tensor->name));\n                }\n\n                float * f32_data;\n\n                if (tensor->type == GGML_TYPE_F32) {\n                    f32_data = (float *) tensor->data;\n                } else if (ggml_is_quantized(tensor->type) && !params->allow_requantize) {\n                    throw std::runtime_error(format(\"requantizing from type %s is disabled\", ggml_type_name(tensor->type)));\n                } else {\n                    llama_tensor_dequantize_impl(tensor, f32_conv_buf, workers, nelements, nthread);\n                    f32_data = (float *) f32_conv_buf.data();\n                }\n\n                LLAMA_LOG_INFO(\"converting to %s .. \", ggml_type_name(new_type));\n                fflush(stdout);\n\n                if (work.size() < (size_t)nelements * 4) {\n                    work.resize(nelements * 4); // upper bound on size\n                }\n                new_data = work.data();\n\n                const int64_t n_per_row = tensor->ne[0];\n                const int64_t nrows = tensor->ne[1];\n\n                static const int64_t min_chunk_size = 32 * 512;\n                const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row));\n\n                const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1];\n                const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size;\n                const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1;\n\n                // quantize each expert separately since they have different importance matrices\n                new_size = 0;\n                for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) {\n                    const float * f32_data_03 = f32_data + i03 * nelements_matrix;\n                    void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows;\n                    const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr;\n\n                    new_size += llama_tensor_quantize_impl(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, workers, nthread_use);\n                }\n                LLAMA_LOG_INFO(\"size = %8.2f MiB -> %8.2f MiB\\n\", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0);\n            }\n            total_size_org += tensor_size;\n            total_size_new += new_size;\n\n            // update the gguf meta data as we go\n            gguf_set_tensor_type(ctx_outs[cur_split].get(), name.c_str(), new_type);\n            GGML_ASSERT(gguf_get_tensor_size(ctx_outs[cur_split].get(), gguf_find_tensor(ctx_outs[cur_split].get(), name.c_str())) == new_size);\n            gguf_set_tensor_data(ctx_outs[cur_split].get(), name.c_str(), new_data);\n\n            // write tensor data + padding\n            fout.write((const char *) new_data, new_size);\n            zeros(fout, GGML_PAD(new_size, align) - new_size);\n        } // no --dry-run\n    } // main loop\n\n    if (!params->dry_run) {\n        close_ofstream();\n    }\n\n    LLAMA_LOG_INFO(\"%s: model size  = %8.2f MiB (%.2f BPW)\\n\", __func__, total_size_org/1024.0/1024.0, total_size_org*8.0/ml.n_elements);\n    LLAMA_LOG_INFO(\"%s: quant size  = %8.2f MiB (%.2f BPW)\\n\", __func__, total_size_new/1024.0/1024.0, total_size_new*8.0/ml.n_elements);\n\n    if (!params->imatrix && params->dry_run && will_require_imatrix) {\n        LLAMA_LOG_WARN(\"%s: WARNING: dry run completed successfully, but actually completing this quantization will require an imatrix!\\n\",\n                       __func__\n        );\n    }\n\n    if (qs.n_fallback > 0) {\n        LLAMA_LOG_WARN(\"%s: WARNING: %d of %d tensor(s) required fallback quantization\\n\",\n                __func__, qs.n_fallback, ml.n_tensors);\n    }\n}\n\n//\n// interface implementation\n//\n\nllama_model_quantize_params llama_model_quantize_default_params() {\n    llama_model_quantize_params result = {\n        /*.nthread                     =*/ 0,\n        /*.ftype                       =*/ LLAMA_FTYPE_MOSTLY_Q5_1,\n        /*.output_tensor_type          =*/ GGML_TYPE_COUNT,\n        /*.token_embedding_type        =*/ GGML_TYPE_COUNT,\n        /*.allow_requantize            =*/ false,\n        /*.quantize_output_tensor      =*/ true,\n        /*.only_copy                   =*/ false,\n        /*.pure                        =*/ false,\n        /*.keep_split                  =*/ false,\n        /*.dry_run                     =*/ false,\n        /*.imatrix                     =*/ nullptr,\n        /*.kv_overrides                =*/ nullptr,\n        /*.tensor_type                 =*/ nullptr,\n        /*.prune_layers                =*/ nullptr\n    };\n\n    return result;\n}\n\nuint32_t llama_model_quantize(\n        const char * fname_inp,\n        const char * fname_out,\n        const llama_model_quantize_params * params) {\n    try {\n        llama_model_quantize_impl(fname_inp, fname_out, params);\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: failed to quantize: %s\\n\", __func__, err.what());\n        return 1;\n    }\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-quant.h",
    "content": "#pragma once\n"
  },
  {
    "path": "examples/talk-llama/llama-sampler.cpp",
    "content": "#include \"llama-sampler.h\"\n\n#include \"llama-impl.h\"\n#include \"llama-vocab.h\"\n#include \"llama-grammar.h\"\n\n#include \"ggml-cpp.h\"\n\n#include <array>\n#include <algorithm>\n#include <cassert>\n#include <cfloat>\n#include <chrono>\n#include <cmath>\n#include <cstdlib>\n#include <cstring>\n#include <ctime>\n#include <numeric>\n#include <random>\n#include <unordered_map>\n#include <stdexcept>\n\n// the ring buffer works similarly to std::deque, but with a fixed capacity\ntemplate<typename T>\nstruct ring_buffer {\n    ring_buffer(size_t cap) : capacity(cap), data(cap) {}\n\n    T & front() {\n        if (sz == 0) {\n            throw std::runtime_error(\"ring buffer is empty\");\n        }\n        return data[first];\n    }\n\n    const T & front() const {\n        if (sz == 0) {\n            throw std::runtime_error(\"ring buffer is empty\");\n        }\n        return data[first];\n    }\n\n    T & back() {\n        if (sz == 0) {\n            throw std::runtime_error(\"ring buffer is empty\");\n        }\n        return data[pos];\n    }\n\n    const T & back() const {\n        if (sz == 0) {\n            throw std::runtime_error(\"ring buffer is empty\");\n        }\n        return data[pos];\n    }\n\n    void push_back(const T & value) {\n        if (capacity == 0) {\n            throw std::runtime_error(\"ring buffer: capacity is zero\");\n        }\n\n        if (sz == capacity) {\n            // advance the start when buffer is full\n            first = (first + 1) % capacity;\n        } else {\n            sz++;\n        }\n        data[pos] = value;\n        pos = (pos + 1) % capacity;\n    }\n\n    T pop_front() {\n        if (sz == 0) {\n            throw std::runtime_error(\"ring buffer is empty\");\n        }\n        T value = data[first];\n        first = (first + 1) % capacity;\n        sz--;\n        return value;\n    }\n\n    //T & operator[](size_t i) {\n    //    if (i >= sz) {\n    //        throw std::runtime_error(\"ring buffer: index out of bounds\");\n    //    }\n    //    return data[(first + i) % capacity];\n    //}\n\n    //const T & at(size_t i) const {\n    //    if (i >= sz) {\n    //        throw std::runtime_error(\"ring buffer: index out of bounds\");\n    //    }\n    //    return data[(first + i) % capacity];\n    //}\n\n    const T & rat(size_t i) const {\n        if (i >= sz) {\n            throw std::runtime_error(\"ring buffer: index out of bounds\");\n        }\n        return data[(first + sz - i - 1) % capacity];\n    }\n\n    std::vector<T> to_vector() const {\n        std::vector<T> result;\n        result.reserve(sz);\n        for (size_t i = 0; i < sz; i++) {\n            result.push_back(data[(first + i) % capacity]);\n        }\n        return result;\n    }\n\n    void clear() {\n        // here only reset the status of the buffer\n        sz = 0;\n        first = 0;\n        pos = 0;\n    }\n\n    bool empty() const {\n        return sz == 0;\n    }\n\n    size_t size() const {\n        return sz;\n    }\n\n    size_t capacity = 0;\n    size_t sz = 0;\n    size_t first = 0;\n    size_t pos = 0;\n\n    std::vector<T> data;\n};\n\n// writes result in res, does not mutate cur\nstatic void llama_token_data_array_partial_sort(const llama_token_data_array & cur, int npartial, std::vector<llama_token_data> & res) {\n    static const auto comp = [](const llama_token_data & a, const llama_token_data & b) {\n        return a.logit > b.logit;\n    };\n\n    constexpr int   nbuckets     = 128;\n    constexpr float bucket_low   = -10.0f;\n    constexpr float bucket_high  =  10.0f;\n    constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);\n    constexpr float bucket_inter = -bucket_low * bucket_scale;\n\n    std::vector<int> bucket_idx;\n    std::vector<int> histo(nbuckets, 0);\n\n    std::vector<llama_token_data*> bucket_ptrs;\n\n    bucket_idx.reserve(cur.size);\n\n    for (int i = 0; i < (int)cur.size; ++i) {\n        const float val = cur.data[i].logit;\n        int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);\n        ib = std::max(0, std::min(nbuckets - 1, ib));\n        bucket_idx.push_back(ib);\n        ++histo[ib];\n    }\n    int nhave = 0;\n    int ib = nbuckets - 1;\n    for ( ; ib >= 0; --ib) {\n        nhave += histo[ib];\n        if (nhave >= npartial) {\n            break;\n        }\n    }\n    res.resize(nhave);\n    auto * ptr = res.data();\n    bucket_ptrs.reserve(nbuckets - ib);\n    for (int j = nbuckets - 1; j >= ib; --j) {\n        bucket_ptrs.push_back(ptr);\n        ptr += histo[j];\n    }\n    for (int i = 0; i < (int)cur.size; ++i) {\n        int j = bucket_idx[i];\n        if (j >= ib) {\n            *bucket_ptrs[nbuckets - 1 - j]++ = cur.data[i];\n        }\n    }\n\n    ptr = res.data();\n    int ndone = 0;\n    for (int j = nbuckets - 1; j > ib; --j) {\n        std::sort(ptr, ptr + histo[j], comp);\n        ptr += histo[j];\n        ndone += histo[j];\n    }\n    std::partial_sort(ptr, ptr + npartial - ndone, ptr + histo[ib], comp);\n}\n\n// reduces the size of cur_p to npartial, keeping only the top npartial elements\nstatic void llama_token_data_array_partial_sort_inplace(llama_token_data_array * cur_p, int npartial) {\n    static const auto comp = [](const llama_token_data & a, const llama_token_data & b) {\n        return a.logit > b.logit;\n    };\n\n    if (npartial <= 128) {\n        std::partial_sort(cur_p->data, cur_p->data + npartial, cur_p->data + cur_p->size, comp);\n\n        cur_p->size = npartial;\n        cur_p->sorted = true;\n\n        return;\n    }\n\n    std::vector<llama_token_data> tmp;\n\n    llama_token_data_array_partial_sort(*cur_p, npartial, tmp);\n\n    std::copy(tmp.data(), tmp.data() + npartial, cur_p->data);\n\n    cur_p->size = npartial;\n    cur_p->sorted = true;\n}\n\nstatic int llama_sample_dist(llama_token_data_array * cur_p, std::mt19937 & rng) {\n    // iterator for the probabilities\n#ifdef __GNUC__\n    #pragma GCC diagnostic push\n    #pragma GCC diagnostic ignored \"-Wunused-local-typedefs\"\n#endif\n\n    struct probs_iterator {\n        typedef std::input_iterator_tag iterator_category;\n        typedef float value_type;\n        typedef float * pointer;\n        typedef float & reference;\n        typedef ptrdiff_t difference_type;\n\n        const llama_token_data * data;\n\n        bool operator==(const probs_iterator & other) const { return data == other.data; }\n        bool operator!=(const probs_iterator & other) const { return data != other.data; }\n        const float & operator*() const { return data->p; }\n        probs_iterator & operator++() { ++data; return *this; }\n        probs_iterator operator++(int) { probs_iterator tmp = *this; ++data; return tmp; }\n    };\n\n#ifdef __GNUC__\n    #pragma GCC diagnostic pop\n#endif\n\n    std::discrete_distribution<int> dist(probs_iterator{cur_p->data}, probs_iterator{cur_p->data + cur_p->size});\n\n    return dist(rng);\n}\n\n/*\nstatic void llama_log_softmax(float * array, size_t size) {\n    float max_l = *std::max_element(array, array + size);\n    float sum = 0.f;\n    for (size_t i = 0; i < size; ++i) {\n        float p = expf(array[i] - max_l);\n        sum += p;\n        array[i] = p;\n    }\n\n    for (size_t i = 0; i < size; ++i) {\n        array[i] = logf(array[i] / sum);\n    }\n}\n*/\n\nstatic void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) {\n    if (temp <= 0.0f) {\n        // find the token with the highest logit and set the rest to -inf\n        size_t max_i = 0;\n        float  max_l = cur_p->data[0].logit;\n\n        for (size_t i = 1; i < cur_p->size; ++i) {\n            if (cur_p->data[i    ].logit > max_l) {\n                cur_p->data[max_i].logit = -INFINITY;\n                max_i = i;\n                max_l = cur_p->data[i].logit;\n            } else {\n                cur_p->data[i].logit = -INFINITY;\n            }\n        }\n\n        return;\n    }\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cur_p->data[i].logit /= temp;\n    }\n}\n\nstatic void llama_sampler_softmax_impl(llama_token_data_array * cur_p, bool do_sort) {\n    GGML_ASSERT(cur_p->size > 0);\n\n    // Sort the logits in descending order if requested\n    if (do_sort && !cur_p->sorted) {\n        llama_token_data_array_partial_sort_inplace(cur_p, cur_p->size);\n    }\n\n    float max_l = cur_p->data[0].logit;\n    if (!cur_p->sorted) {\n        for (size_t i = 1; i < cur_p->size; ++i) {\n            max_l = std::max(max_l, cur_p->data[i].logit);\n        }\n    }\n\n    float cum_sum = 0.0f;\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        float p = expf(cur_p->data[i].logit - max_l);\n        cur_p->data[i].p = p;\n        cum_sum += p;\n    }\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cur_p->data[i].p /= cum_sum;\n    }\n}\n\nstatic void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k) {\n    // if (k >= (int32_t)cur_p->size) {\n    //     return;\n    // }\n\n    if (k <= 0) {\n        return;\n    }\n\n    k = std::min(k, (int) cur_p->size);\n\n    // Sort scores in descending order\n    if (!cur_p->sorted) {\n        llama_token_data_array_partial_sort_inplace(cur_p, k);\n    }\n\n    cur_p->size = k;\n}\n\nstatic uint32_t get_rng_seed(uint32_t seed) {\n    if (seed == LLAMA_DEFAULT_SEED) {\n        // use system clock if std::random_device is not a true RNG\n        static bool is_rd_prng = std::random_device().entropy() == 0;\n        if (is_rd_prng) {\n            return (uint32_t) std::chrono::system_clock::now().time_since_epoch().count();\n        }\n        std::random_device rd;\n        return rd();\n    }\n    return seed;\n}\n\n// llama_sampler API\n\nstruct llama_sampler * llama_sampler_init(\n        struct llama_sampler_i * iface,\n        llama_sampler_context_t ctx) {\n    return new llama_sampler {\n        /* .iface = */ iface,\n        /* .ctx   = */ ctx,\n    };\n}\n\nconst char * llama_sampler_name(const struct llama_sampler * smpl) {\n    if (!smpl->iface) {\n        return \"(null)\";\n    }\n\n    return smpl->iface->name(smpl);\n}\n\nvoid llama_sampler_accept(struct llama_sampler * smpl, llama_token token) {\n    if (!smpl) {\n        return;\n    }\n\n    if (smpl->iface->accept) {\n        smpl->iface->accept(smpl, token);\n    }\n}\n\nvoid llama_sampler_apply(struct llama_sampler * smpl, struct llama_token_data_array * cur_p) {\n    if (!smpl) {\n        return;\n    }\n\n    GGML_ASSERT(smpl->iface->apply);\n    smpl->iface->apply(smpl, cur_p);\n}\n\nvoid llama_sampler_reset(struct llama_sampler * smpl) {\n    if (!smpl) {\n        return;\n    }\n\n    if (smpl->iface->reset) {\n        smpl->iface->reset(smpl);\n    }\n}\n\nstruct llama_sampler * llama_sampler_clone(const struct llama_sampler * smpl) {\n    if (!smpl) {\n        return nullptr;\n    }\n\n    if (smpl->iface->clone) {\n        return smpl->iface->clone(smpl);\n    }\n\n    if (smpl->ctx == nullptr) {\n        return llama_sampler_init(\n            /* .iface = */ smpl->iface,\n            /* .ctx   = */ nullptr\n        );\n    }\n\n    GGML_ABORT(\"the sampler does not support cloning\");\n}\n\nvoid llama_sampler_free(struct llama_sampler * smpl) {\n    if (smpl == nullptr) {\n        return;\n    }\n\n    if (smpl->iface->free) {\n        smpl->iface->free(smpl);\n    }\n\n    delete smpl;\n}\n\n// empty sampler\n\nstruct llama_sampler_empty {\n    const char * name;\n};\n\nstatic struct llama_sampler * llama_sampler_init_empty(const char * name);\n\nstatic const char * llama_sampler_empty_name(const struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_empty *) smpl->ctx;\n    return ctx->name;\n}\n\nstatic void llama_sampler_empty_accept(struct llama_sampler * smpl, llama_token token) {\n    GGML_UNUSED(smpl);\n    GGML_UNUSED(token);\n}\n\nstatic void llama_sampler_empty_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    GGML_UNUSED(smpl);\n    GGML_UNUSED(cur_p);\n}\n\nstatic void llama_sampler_empty_reset(struct llama_sampler * smpl) {\n    GGML_UNUSED(smpl);\n}\n\nstatic struct llama_sampler * llama_sampler_empty_clone(const struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_empty *) smpl->ctx;\n    return llama_sampler_init_empty(ctx->name);\n}\n\nstatic void llama_sampler_empty_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_empty *) smpl->ctx;\n}\n\nstatic bool llama_sampler_empty_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    GGML_UNUSED(smpl);\n    GGML_UNUSED(buft);\n\n    return true;\n}\n\nstatic void llama_sampler_empty_backend_accept(\n        struct llama_sampler * smpl,\n        ggml_context * ctx,\n        ggml_cgraph * gf,\n        struct ggml_tensor * selected_token) {\n    GGML_UNUSED(smpl);\n    GGML_UNUSED(ctx);\n    GGML_UNUSED(gf);\n    GGML_UNUSED(selected_token);\n}\n\nstatic void llama_sampler_empty_backend_apply(\n          struct llama_sampler      * smpl,\n          struct ggml_context       * ctx,\n          struct ggml_cgraph        * gf,\n          struct llama_sampler_data * data) {\n    GGML_UNUSED(smpl);\n    GGML_UNUSED(ctx);\n    GGML_UNUSED(gf);\n    GGML_UNUSED(data);\n}\n\nstatic void llama_sampler_empty_backend_set_input(struct llama_sampler * smpl) {\n    GGML_UNUSED(smpl);\n}\n\nstatic struct llama_sampler_i llama_sampler_empty_i = {\n    /* .name              = */ llama_sampler_empty_name,\n    /* .accept            = */ llama_sampler_empty_accept,\n    /* .apply             = */ llama_sampler_empty_apply,\n    /* .reset             = */ llama_sampler_empty_reset,\n    /* .clone             = */ llama_sampler_empty_clone,\n    /* .free              = */ llama_sampler_empty_free,\n    /* .backend_init      = */ llama_sampler_empty_backend_init,\n    /* .backend_accept    = */ llama_sampler_empty_backend_accept,\n    /* .backend_apply     = */ llama_sampler_empty_backend_apply,\n    /* .backend_set_input = */ llama_sampler_empty_backend_set_input,\n};\n\nstruct llama_sampler * llama_sampler_init_empty(const char * name) {\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_empty_i,\n        /* .ctx   = */ new llama_sampler_empty {\n            /* .name = */ name,\n        }\n    );\n}\n\n// common backend sampler functionality\n//\n// +name : means that the sampler is support and will run on the backend\n// -name : means that a ggml operator is not supported by the backend\n//\nstruct llama_sampler_backend {\n    llama_sampler_backend(const char * name) : name(name), name_ext(name), is_init(false), support(false) {}\n\n    const char * get_name() {\n        if (!is_init) {\n            return name.c_str();\n        }\n\n        if (support) {\n            name_ext = \"+\" + name;\n        } else {\n            name_ext = \"-\" + name;\n        }\n\n        return name_ext.c_str();\n    }\n\n    void init(bool support) {\n        GGML_ASSERT(this->is_init == false);\n\n        this->is_init = true;\n        this->support = support;\n    }\n\nprivate:\n    std::string name;\n    std::string name_ext;\n\n    bool is_init;\n    bool support;\n};\n\n// check if all ggml ops used by the sampler are supported by the backend\nstatic bool llama_sampler_backend_support(\n        llama_sampler              * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * device = ggml_backend_buft_get_device(buft);\n    if (!device) {\n        // CPU backend always supported\n        return true;\n    }\n\n    ggml_init_params params = {\n        /*.mem_size   =*/ 128*ggml_tensor_overhead() + ggml_graph_overhead(),\n        /*.mem_buffer =*/ NULL,\n        /*.no_alloc   =*/ true,\n    };\n\n    ggml_context_ptr ctx_ptr { ggml_init(params) };\n    if (!ctx_ptr) {\n        throw std::runtime_error(format(\"failed to create ggml context\"));\n    }\n\n    ggml_context * ctx = ctx_ptr.get();\n\n    const int64_t n = 1024*1024;\n\n    llama_sampler_data data = {\n        /*.logits     = */ ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n),\n        /*.probs      = */ nullptr,\n        /*.sampled    = */ nullptr,\n        /*.candidates = */ ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n),\n    };\n\n    ggml_cgraph * gf = ggml_new_graph(ctx);\n\n    smpl->iface->backend_apply(smpl, ctx, gf, &data);\n\n    if (data.logits) {\n        ggml_build_forward_expand(gf, data.logits);\n    }\n\n    if (data.probs) {\n        ggml_build_forward_expand(gf, data.probs);\n    }\n\n    if (data.sampled) {\n        ggml_build_forward_expand(gf, data.sampled);\n    }\n\n    if (data.candidates) {\n        ggml_build_forward_expand(gf, data.candidates);\n    }\n\n    for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {\n        struct ggml_tensor * op = ggml_graph_node(gf, i);\n\n        if (!ggml_backend_dev_supports_op(device, op)) {\n            LLAMA_LOG_WARN(\"%s: device '%s' does not have support for op %s needed for sampler '%s'\\n\",\n                    __func__, ggml_backend_dev_name(device), ggml_op_name(op->op), smpl->iface->name(smpl));\n\n            return false;\n        }\n    }\n\n    return true;\n}\n\n// sampler chain\n\nstatic const char * llama_sampler_chain_name(const struct llama_sampler * /*smpl*/) {\n    return \"chain\";\n}\n\nstatic void llama_sampler_chain_accept(struct llama_sampler * smpl, llama_token token) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    time_meas tm(chain->t_sample_us, chain->params.no_perf);\n\n    for (auto & smpl : chain->samplers) {\n        llama_sampler_accept(smpl.ptr, token);\n    }\n\n    chain->n_sample++;\n}\n\nstatic void llama_sampler_chain_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    time_meas tm(chain->t_sample_us, chain->params.no_perf);\n\n    bool is_backend = chain->is_init;\n\n    for (auto & smpl : chain->samplers) {\n        if (is_backend && smpl.is_backend) {\n            continue;\n        }\n\n        is_backend = false;\n\n        if (smpl.ptr->iface->apply == nullptr) {\n            continue;\n        }\n\n        llama_sampler_apply(smpl.ptr, cur_p);\n    }\n}\n\nstatic void llama_sampler_chain_reset(struct llama_sampler * smpl) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    for (auto & smpl : chain->samplers) {\n        llama_sampler_reset(smpl.ptr);\n    }\n}\n\nstatic struct llama_sampler * llama_sampler_chain_clone(const struct llama_sampler * smpl) {\n    const auto * chain_src = (const llama_sampler_chain *) smpl->ctx;\n\n    auto * result = llama_sampler_chain_init(chain_src->params);\n\n    for (const auto & smpl : chain_src->samplers) {\n        llama_sampler_chain_add(result, llama_sampler_clone(smpl.ptr));\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_chain_free(struct llama_sampler * smpl) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    for (auto & smpl : chain->samplers) {\n        llama_sampler_free(smpl.ptr);\n    }\n\n    delete chain;\n}\n\nstatic bool llama_sampler_chain_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    GGML_ASSERT(chain->is_init == false && \"llama_sampler_chain_backend_init() called twice\");\n\n    chain->is_init = true;\n\n    bool res = true;\n\n    for (auto & smpl : chain->samplers) {\n        bool res_cur = true;\n\n        // to be able to run a sampler on the backend, it has to:\n        // - have the .backend_init() API implemented\n        // - return true during .backend_init()\n        if (smpl.ptr->iface->backend_init) {\n            if (!smpl.ptr->iface->backend_init(smpl.ptr, buft)) {\n                res_cur = false;\n            }\n        } else {\n            res_cur = false;\n        }\n\n        smpl.is_backend = res_cur;\n\n        res = res && res_cur;\n    }\n\n    return res;\n}\n\nstatic void llama_sampler_chain_backend_accept(\n        struct llama_sampler * smpl,\n        ggml_context * ctx,\n        ggml_cgraph * gf,\n        struct ggml_tensor * selected_token) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    for (auto & smpl : chain->samplers) {\n        if (!smpl.is_backend) {\n            break;\n        }\n\n        if (smpl.ptr->iface->backend_accept) {\n            smpl.ptr->iface->backend_accept(smpl.ptr, ctx, gf, selected_token);\n        }\n    }\n}\n\nstatic void llama_sampler_chain_backend_apply(\n          struct llama_sampler      * smpl,\n          struct ggml_context       * ctx,\n          struct ggml_cgraph        * gf,\n          struct llama_sampler_data * data) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    GGML_ASSERT(chain->is_init && \"llama_sampler_chain_backend_init() not called\");\n\n    for (auto & smpl : chain->samplers) {\n        if (!smpl.is_backend) {\n            break;\n        }\n\n        if (smpl.ptr->iface->backend_apply) {\n            smpl.ptr->iface->backend_apply(smpl.ptr, ctx, gf, data);\n        }\n    }\n}\n\nstatic void llama_sampler_chain_backend_set_input(struct llama_sampler * smpl) {\n    auto * chain = (llama_sampler_chain *) smpl->ctx;\n\n    for (auto & smpl : chain->samplers) {\n        if (!smpl.is_backend) {\n            break;\n        }\n\n        if (smpl.ptr->iface->backend_set_input) {\n            smpl.ptr->iface->backend_set_input(smpl.ptr);\n        }\n    }\n}\n\nstatic struct llama_sampler_i llama_sampler_chain_i = {\n    /* .name              = */ llama_sampler_chain_name,\n    /* .accept            = */ llama_sampler_chain_accept,\n    /* .apply             = */ llama_sampler_chain_apply,\n    /* .reset             = */ llama_sampler_chain_reset,\n    /* .clone             = */ llama_sampler_chain_clone,\n    /* .free              = */ llama_sampler_chain_free,\n    /* .backend_init      = */ llama_sampler_chain_backend_init,\n    /* .backend_accept    = */ llama_sampler_chain_backend_accept,\n    /* .backend_apply     = */ llama_sampler_chain_backend_apply,\n    /* .backend_set_input = */ llama_sampler_chain_backend_set_input,\n};\n\nstruct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params) {\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_chain_i,\n        /* .ctx   = */ new llama_sampler_chain {\n            /* .params      = */ params,\n            /* .is_init     = */ false,\n            /* .samplers    = */ {},\n            /* .cur         = */ {},\n            /* .t_sample_us = */ 0,\n            /* .n_sample    = */ 0,\n        }\n    );\n}\n\nllama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx) {\n    const llama_token   sampled_token  = llama_get_sampled_token_ith     (ctx, idx);\n    const float *       sampled_probs  = llama_get_sampled_probs_ith     (ctx, idx);\n    const float *       sampled_logits = llama_get_sampled_logits_ith    (ctx, idx);\n    const llama_token * sampled_ids    = llama_get_sampled_candidates_ith(ctx, idx);\n\n    // If a backend sampler has already sampled a token, return it.\n    if (sampled_token != LLAMA_TOKEN_NULL) {\n        LLAMA_LOG_DEBUG(\"%s: Backend sampler selected token for idx %d. Skipping CPU samplers\\n\", __func__, idx);\n        return sampled_token;\n    }\n\n    const llama_model * model = llama_get_model(ctx);\n    const llama_vocab * vocab = llama_model_get_vocab(model);\n\n    const int n_vocab = llama_vocab_n_tokens(vocab);\n\n    // use pre-allocated buffer from chain if available, otherwise allocate locally\n    std::vector<llama_token_data> * cur_ptr;\n    std::vector<llama_token_data> cur_local;\n\n    if (smpl->iface == &llama_sampler_chain_i) {\n        auto * chain = (llama_sampler_chain *) smpl->ctx;\n        cur_ptr = &chain->cur;\n    } else {\n        cur_ptr = &cur_local;\n    }\n\n    auto & cur = *cur_ptr;\n\n    if (sampled_probs) {\n        const uint32_t sampled_probs_count = llama_get_sampled_probs_count_ith(ctx, idx);\n        cur.resize(sampled_probs_count);\n        for (uint32_t i = 0; i < sampled_probs_count; ++i) {\n            cur[i] = llama_token_data{sampled_ids[i], sampled_logits[i], sampled_probs[i]};\n        }\n    } else if (sampled_logits) {\n        const uint32_t sampled_logits_count = llama_get_sampled_logits_count_ith(ctx, idx);\n        cur.resize(sampled_logits_count);\n        for (llama_token i = 0; i < (int)sampled_logits_count; i++) {\n            cur[i] = llama_token_data{sampled_ids[i], sampled_logits[i], 0.0f};\n        }\n    } else {\n        const auto * logits = llama_get_logits_ith(ctx, idx);\n        GGML_ASSERT(logits != nullptr);\n        cur.resize(n_vocab);\n        for (llama_token token_id = 0; token_id < n_vocab; token_id++) {\n            cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};\n        }\n    }\n\n    llama_token_data_array cur_p = {\n        /* .data       = */ cur.data(),\n        /* .size       = */ cur.size(),\n        /* .selected   = */ -1,\n        /* .sorted     = */ false,\n    };\n\n    llama_sampler_apply(smpl, &cur_p);\n\n    GGML_ASSERT(cur_p.selected >= 0 && cur_p.selected < (int32_t) cur_p.size);\n\n    auto token = cur_p.data[cur_p.selected].id;\n\n    llama_sampler_accept(smpl, token);\n\n    return token;\n}\n\n\nvoid llama_sampler_chain_add(struct llama_sampler * chain, struct llama_sampler * smpl) {\n    auto * p = (llama_sampler_chain *) chain->ctx;\n    p->samplers.push_back({\n        /* .is_backend = */ false,\n        /* .ptr        = */ smpl,\n    });\n}\n\nstruct llama_sampler * llama_sampler_chain_get(struct llama_sampler * chain, int32_t i) {\n    if (chain == nullptr) {\n        return nullptr;\n    }\n\n    if (chain->iface != &llama_sampler_chain_i) {\n        return nullptr;\n    }\n\n    if (i == -1) {\n        return chain;\n    }\n\n    const auto * p = (const llama_sampler_chain *) chain->ctx;\n\n    if (i < 0 || (size_t) i >= p->samplers.size()) {\n        return nullptr;\n    }\n\n    return p->samplers[i].ptr;\n}\n\nstruct llama_sampler * llama_sampler_chain_remove(struct llama_sampler * chain, int32_t i) {\n    auto * p = (llama_sampler_chain *) chain->ctx;\n\n    if (i < 0 || (size_t) i >= p->samplers.size()) {\n        return nullptr;\n    }\n\n    auto * result = p->samplers[i].ptr;\n    p->samplers.erase(p->samplers.begin() + i);\n\n    return result;\n}\n\nint llama_sampler_chain_n(const struct llama_sampler * chain) {\n    const auto * p = (const llama_sampler_chain *) chain->ctx;\n\n    return p->samplers.size();\n}\n\n//\n// samplers\n//\n\n// greedy\n\nstruct llama_sampler_greedy : public llama_sampler_backend {\n};\n\nstatic const char * llama_sampler_greedy_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_greedy *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_greedy_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_greedy *) smpl->ctx;\n    GGML_UNUSED(ctx);\n}\n\nstatic struct llama_sampler * llama_sampler_greedy_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_greedy *) smpl->ctx;\n    auto * result = llama_sampler_init_greedy();\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_greedy *) result->ctx;\n\n        GGML_UNUSED(ctx);\n        GGML_UNUSED(result_ctx);\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_greedy_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_greedy *) smpl->ctx;\n}\n\nstatic void llama_sampler_greedy_apply(struct llama_sampler * /*smpl*/, llama_token_data_array * cur_p) {\n    cur_p->selected = 0;\n    for (size_t i = 1; i < cur_p->size; ++i) {\n        if (cur_p->data[i].logit > cur_p->data[cur_p->selected].logit) {\n            cur_p->selected = i;\n        }\n    }\n}\n\nstatic bool llama_sampler_greedy_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_greedy *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_greedy_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    GGML_UNUSED(gf);\n    GGML_UNUSED(smpl);\n\n    struct ggml_tensor * curl = ggml_argmax(ctx, data->logits);\n    ggml_set_name(curl, \"greedy_argmax\");\n\n    data->sampled = curl;\n}\n\nstatic struct llama_sampler_i llama_sampler_greedy_i = {\n    /* .name              = */ llama_sampler_greedy_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_greedy_apply,\n    /* .reset             = */ llama_sampler_greedy_reset,\n    /* .clone             = */ llama_sampler_greedy_clone,\n    /* .free              = */ llama_sampler_greedy_free,\n    /* .backend_init      = */ llama_sampler_greedy_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_greedy_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_greedy() {\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_greedy_i,\n        /* .ctx   = */ new llama_sampler_greedy {\n            (\"greedy\"),\n        }\n    );\n}\n\n// dist\n\nstruct llama_sampler_dist : public llama_sampler_backend {\n    const uint32_t seed;\n          uint32_t seed_cur;\n\n    std::mt19937 rng;\n\n    ggml_tensor * inp_uniform;\n};\n\nstatic const char * llama_sampler_dist_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_dist *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_dist_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_dist *) smpl->ctx;\n\n    // edge cases\n    if (cur_p->size == 0) {\n        cur_p->selected = -1;\n        return;\n    }\n\n    cur_p->selected = 0;\n\n    if (cur_p->size == 1) {\n        cur_p->data[0].p = 1.0f;\n        return;\n    }\n\n    // max logit for numerical stability\n    float max_l = cur_p->data[0].logit;\n    if (!cur_p->sorted) {\n        for (size_t i = 1; i < cur_p->size; ++i) {\n            max_l = std::max(max_l, cur_p->data[i].logit);\n        }\n    }\n\n    // apply softmax to obtain the probabilities\n    double sum_cum = 0.0f;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        float p = expf(cur_p->data[i].logit - max_l);\n        cur_p->data[i].p = p;\n        sum_cum += p;\n    }\n\n#if 1\n    // sample from the obtained probabilities and normalize the probs in a single pass\n    // this is ~3x faster on Mac with full gpt-oss vocab than the version below\n    //\n    std::uniform_real_distribution<double> dist(0.0f, 1.0f);\n    const double rnd = dist(ctx->rng);\n\n          double sum_run = 0.0f;\n    const double sum_tgt = sum_cum*rnd;\n\n    bool found = false;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        if (!found) {\n            // accumulate probs until we reach the target sum\n            sum_run += cur_p->data[i].p;\n            if (sum_run >= sum_tgt) {\n                cur_p->selected = i;\n                found = true;\n            }\n        }\n\n        // normalize probs\n        cur_p->data[i].p /= sum_cum;\n    }\n\n    // fallback to the last token (don't think this can happen)\n    assert(found);\n    if (!found) {\n        cur_p->selected = cur_p->size - 1;\n    }\n#else\n    // for clarity, this is the same as above but does one pass for normalization and one extra pass for sampling\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cur_p->data[i].p /= sum_cum;\n    }\n\n    cur_p->selected = llama_sample_dist(cur_p, ctx->rng);\n#endif\n}\n\nstatic void llama_sampler_dist_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_dist *) smpl->ctx;\n    ctx->seed_cur = get_rng_seed(ctx->seed);\n    ctx->rng.seed(ctx->seed_cur);\n}\n\nstatic struct llama_sampler * llama_sampler_dist_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_dist *) smpl->ctx;\n    auto * result = llama_sampler_init_dist(ctx->seed);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_dist *) result->ctx;\n\n        result_ctx->rng = ctx->rng;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_dist_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_dist *) smpl->ctx;\n}\n\nstatic bool llama_sampler_dist_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_dist *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_dist_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    GGML_UNUSED(gf);\n\n    auto * sctx = (llama_sampler_dist *) smpl->ctx;\n\n    sctx->inp_uniform = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n    ggml_set_name (sctx->inp_uniform, \"uniform\");\n    ggml_set_input(sctx->inp_uniform);\n\n    struct ggml_tensor * probs = ggml_soft_max(ctx, data->logits);\n    ggml_set_name(probs, \"dist_probs\");\n\n    struct ggml_tensor * cumsum = ggml_cumsum(ctx, probs);\n    ggml_set_name(cumsum, \"dist_cumsum\");\n\n    // The uniform tensor has a random value and we subtract this tensor with\n    // the cumsum tensor (the uniform tensor will be broadcasted by ggml_sub).\n    // Recall that each entry in cumsum is the cumulative probability up to that\n    // index so values stay negative while the cumulative total is below the\n    // random value, and become zero/positive once the threshold is crossed.\n    struct ggml_tensor * diff = ggml_sub(ctx, cumsum, sctx->inp_uniform);\n    ggml_set_name(diff, \"dist_cumsum\");\n\n    // The ggml_step function produces a tensor where entries are 1 if the\n    // corresponding entry in diff is > 0, and 0 otherwise. So all values up to\n    // the index where the cumulative probability exceeds the random value are 0,\n    // and all entries after that are 1.\n    struct ggml_tensor * mask = ggml_step(ctx, diff);\n    ggml_set_name(mask, \"dist_mask\");\n\n    // Taking the sum of the mask gives us the sum of elements after the threshold\n    // we are interested in.\n    struct ggml_tensor * idxf = ggml_sum(ctx, mask);\n    ggml_set_name(idxf, \"dist_index_f32\");\n\n    // Use ggml_scale_bias to scale the index value by -1 and then add the size\n    // of the mask to that value so we get the correct index ((-1 * idxf) + n).\n    struct ggml_tensor * idx = ggml_cast(ctx, ggml_scale_bias(ctx, idxf, -1.0f, mask->ne[0]), GGML_TYPE_I32);\n    ggml_set_name(idx, \"dist_index_i32\");\n\n    // Map back to original vocab ids if a candidates tensor is available.\n    struct ggml_tensor * sampled_token = idx;\n    if (data->candidates != nullptr) {\n        struct ggml_tensor * candidates = ggml_reshape_2d(ctx, data->candidates, 1, ggml_nelements(data->candidates));\n\n        sampled_token = ggml_get_rows(ctx, candidates, idx);\n        ggml_set_name(sampled_token, \"dist_sampled_token\");\n    }\n\n    data->sampled = sampled_token;\n    data->probs = probs;\n}\n\nstatic void llama_sampler_dist_backend_set_input(struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_dist *) smpl->ctx;\n\n    GGML_ASSERT(sctx->inp_uniform != nullptr);\n\n    // We sample in double precision and cast to float to match rnd numbers of\n    // llama_dampler_dist which uses double precision (sampling from\n    // std::uniform_real_distribution<double> and\n    // std::uniform_real_distribution<float> with same rng will produce\n    // different sequences).\n    std::uniform_real_distribution<double> dist(0.0f, 1.0f);\n    const float rnd = dist(sctx->rng);\n\n    ggml_backend_tensor_set(sctx->inp_uniform, &rnd, 0, sizeof(float));\n}\n\nstatic struct llama_sampler_i llama_sampler_dist_i = {\n    /* .name              = */ llama_sampler_dist_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_dist_apply,\n    /* .reset             = */ llama_sampler_dist_reset,\n    /* .clone             = */ llama_sampler_dist_clone,\n    /* .free              = */ llama_sampler_dist_free,\n    /* .backend_init      = */ llama_sampler_dist_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_dist_backend_apply,\n    /* .backend_set_input = */ llama_sampler_dist_backend_set_input,\n};\n\nstruct llama_sampler * llama_sampler_init_dist(uint32_t seed) {\n    auto seed_cur = get_rng_seed(seed);\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_dist_i,\n        /* .ctx   = */ new llama_sampler_dist {\n            (\"dist\"),\n            /* .seed        = */ seed,\n            /* .seed_cur    = */ seed_cur,\n            /* .rng         = */ std::mt19937(seed_cur),\n            /* .inp_uniform = */ nullptr,\n        }\n    );\n}\n\n// top-k\n\nstruct llama_sampler_top_k : public llama_sampler_backend {\n    const int32_t k;\n};\n\nstatic const char * llama_sampler_top_k_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_top_k *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_top_k_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_top_k *) smpl->ctx;\n    llama_sampler_top_k_impl(cur_p, ctx->k);\n}\n\nstatic struct llama_sampler * llama_sampler_top_k_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_top_k *) smpl->ctx;\n    return llama_sampler_init_top_k(ctx->k);\n}\n\nstatic void llama_sampler_top_k_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_top_k *) smpl->ctx;\n}\n\nstatic bool llama_sampler_top_k_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_top_k *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_top_k_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    auto * sctx = (llama_sampler_top_k *) smpl->ctx;\n\n    struct ggml_tensor * top_k = ggml_top_k(ctx, data->logits, sctx->k);\n    ggml_set_name(top_k, \"top_k\");\n\n    if (data->candidates) {\n        struct ggml_tensor * candidates_rows = ggml_reshape_2d(ctx, data->candidates, 1, data->candidates->ne[0]);\n        data->candidates = ggml_get_rows(ctx, candidates_rows, top_k);\n        data->candidates = ggml_reshape_1d(ctx, data->candidates, sctx->k);\n        ggml_set_name(data->candidates, \"top_k_candidates\");\n    } else {\n        data->candidates = top_k;\n    }\n\n    struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]);\n    struct ggml_tensor * top_k_rows = ggml_get_rows(ctx, logits_rows, top_k);\n    data->logits = ggml_reshape_1d(ctx, top_k_rows, sctx->k);\n    ggml_set_name(top_k_rows, \"top_k_rows\");\n\n    GGML_UNUSED(gf);\n}\n\nstatic struct llama_sampler_i llama_sampler_top_k_i = {\n    /* .name              = */ llama_sampler_top_k_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_top_k_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_top_k_clone,\n    /* .free              = */ llama_sampler_top_k_free,\n    /* .backend_init      = */ llama_sampler_top_k_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_top_k_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_top_k(int32_t k) {\n    const bool is_empty = (k <= 0);\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?top-k\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_top_k_i,\n        /* .ctx   = */ new llama_sampler_top_k {\n            (\"top-k\"),\n            /* .k = */ k,\n        }\n    );\n}\n\n// top-p\n\nstruct llama_sampler_top_p : public llama_sampler_backend {\n    const float  p;\n    const size_t min_keep;\n\n    std::vector<llama_token_data> buf_sort;\n};\n\nstatic const char * llama_sampler_top_p_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_top_p *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_top_p_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_top_p *) smpl->ctx;\n\n    if (ctx->p >= 1.0f) {\n        return;\n    }\n\n    llama_sampler_softmax_impl(cur_p, false);\n\n    size_t k = cur_p->size;\n    auto * pdata = cur_p->data;\n\n    auto & buf_sort = ctx->buf_sort;\n\n    // if not sorted, try adaptive top-k sorting\n    if (!cur_p->sorted && cur_p->size > 1024) {\n        k = std::min<size_t>(256, cur_p->size);\n        llama_token_data_array_partial_sort(*cur_p, k, buf_sort);\n        pdata = buf_sort.data();\n    } else if (!cur_p->sorted) {\n        // small candidates -> sort inplace\n        llama_token_data_array_partial_sort_inplace(cur_p, k);\n    }\n\n    // Compute the cumulative probabilities\n    float cum_sum = 0.0f;\n    size_t last_idx = cur_p->size;\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cum_sum += pdata[i].p;\n\n        // Check if the running sum is at least p or if we have kept at least min_keep tokens\n        // we set the last index to i+1 to indicate that the current iterate should be included in the set\n        if (cum_sum >= ctx->p && i + 1 >= ctx->min_keep) {\n            last_idx = i + 1;\n            break;\n        }\n\n        // we exceeded the current top-k heuristic -> increase k and continue\n        if (!cur_p->sorted && i == k - 1) {\n            k = cur_p->size;\n            llama_token_data_array_partial_sort(*cur_p, k, buf_sort);\n            pdata = buf_sort.data();\n        }\n    }\n\n    // Resize the output vector to keep only the top-p tokens\n    if (!cur_p->sorted) {\n        std::copy(buf_sort.data(), buf_sort.data() + last_idx, cur_p->data);\n        cur_p->sorted = true;\n    }\n\n    cur_p->size = last_idx;\n}\n\nstatic struct llama_sampler * llama_sampler_top_p_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_top_p *) smpl->ctx;\n    return llama_sampler_init_top_p(ctx->p, ctx->min_keep);\n}\n\nstatic void llama_sampler_top_p_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_top_p *) smpl->ctx;\n}\n\nstatic bool llama_sampler_top_p_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_top_p *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_top_p_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    auto * sctx = (llama_sampler_top_p *) smpl->ctx;\n\n    auto ggml_sort = [ctx](struct ggml_tensor * a, struct ggml_tensor * b) {\n        GGML_ASSERT(ggml_nrows(a) == 1);\n        struct ggml_tensor * a_reshaped = ggml_reshape_2d(ctx, a, 1, a->ne[0]);\n        struct ggml_tensor * a_sorted   = ggml_get_rows(ctx, a_reshaped, b);\n        return ggml_reshape_1d(ctx, a_sorted, a->ne[0]);\n    };\n\n    // Get the sorted logits in descending order.\n    struct ggml_tensor * sorted_idx = ggml_argsort(ctx, data->logits, GGML_SORT_ORDER_DESC);\n    ggml_set_name(sorted_idx, \"top_p_sorted_idx\");\n\n    // Do the sorting via reshape + get_rows\n    struct ggml_tensor * sorted_logits = ggml_sort(data->logits, sorted_idx);\n    ggml_set_name(sorted_logits, \"top_p_sorted_logits\");\n\n    struct ggml_tensor * softmax = ggml_soft_max(ctx, sorted_logits);\n    ggml_set_name(softmax, \"top_p_softmax\");\n\n    // If candidates are provided, sort them as well. Otherwise, set sorted indices as candidates.\n    if (data->candidates) {\n        data->candidates = ggml_sort(data->candidates, sorted_idx);\n    } else {\n        data->candidates = sorted_idx;\n    }\n    ggml_set_name(data->candidates, \"top_p_candidates\");\n\n    // Compute Cumulative Distribution Function (CDF) by means of GGML_OP_CUMSUM.\n    struct ggml_tensor * cdf = ggml_cumsum(ctx, softmax);\n    ggml_set_name(cdf, \"top_p_cdf\");\n\n    // Invert CDF and add top-p value so that ggml_step yields 1 for values we want to keep\n    struct ggml_tensor * cdf_scaled = ggml_scale_bias(ctx, cdf, -1.0f, sctx->p);\n    ggml_set_name(cdf_scaled, \"top_p_cdf_scaled\");\n\n    struct ggml_tensor * mask = ggml_step(ctx, cdf_scaled);\n    ggml_set_name(mask, \"top_p_mask\");\n\n    // Taking the sum of the mask gives us the sum of elements after the threshold\n    // we are interested in.\n    struct ggml_tensor * idxf = ggml_sum(ctx, mask);\n    ggml_set_name(idxf, \"top_p_index_f32\");\n\n    // prevent out-of-bounds access\n    idxf = ggml_clamp(ctx, idxf, 0.0f, mask->ne[0] - 1);\n\n    // construct ones tensor to set the value in the mask\n    struct ggml_tensor * ones = ggml_scale_bias(ctx, idxf, 0.0f, 1.0f);\n    ggml_set_name(ones, \"top_p_ones\");\n\n    // Make top-p inclusive (i.e. return all values such that cum_sum/cdf >= p)\n    struct ggml_tensor * mask_reshaped = ggml_reshape_2d(ctx, mask, 1, mask->ne[0]);\n\n    mask_reshaped = ggml_set_rows(ctx, mask_reshaped, ones, ggml_cast(ctx, idxf, GGML_TYPE_I32));\n    mask = ggml_reshape_1d(ctx, mask_reshaped, mask->ne[0]);\n\n    // Apply -INFINITY bias for masked-out tokens\n    // log(1) = 0 (keep), log(0) = -INF (discard)\n    struct ggml_tensor * top_p_bias = ggml_log(ctx, mask);\n    ggml_set_name(top_p_bias, \"top_p_bias\");\n\n    data->logits = ggml_add(ctx, sorted_logits, top_p_bias);\n    ggml_set_name(data->logits, \"top_p_logits\");\n\n    GGML_UNUSED(gf);\n}\n\nstatic struct llama_sampler_i llama_sampler_top_p_i = {\n    /* .name              = */ llama_sampler_top_p_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_top_p_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_top_p_clone,\n    /* .free              = */ llama_sampler_top_p_free,\n    /* .backend_init      = */ llama_sampler_top_p_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_top_p_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_top_p(float p, size_t min_keep) {\n    const bool is_empty = p >= 1.0f;\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?top-p\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_top_p_i,\n        /* .ctx   = */ new llama_sampler_top_p {\n            (\"top-p\"),\n            /* .p        = */ p,\n            /* .min_keep = */ min_keep,\n            /* .buf_sort = */ {},\n        }\n    );\n}\n\n// min-p\n\nstruct llama_sampler_min_p : public llama_sampler_backend {\n    const float  p;\n    const size_t min_keep;\n};\n\nstatic const char * llama_sampler_min_p_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_min_p *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_min_p_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_min_p *) smpl->ctx;\n\n    if (ctx->p <= 0.0f || !cur_p->size) {\n        return;\n    }\n\n    bool min_p_applied = false;\n\n    // if the cur_p aren't sorted, try the unsorted implementation first\n    if (!cur_p->sorted) {\n        std::vector<llama_token_data> filtered_tokens;\n\n        float max_logit = -FLT_MAX;\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            max_logit = std::max(max_logit, cur_p->data[i].logit);\n        }\n        const float min_logit = max_logit + logf(ctx->p); // min logit for p_i >= p * p_max\n\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            if (cur_p->data[i].logit >= min_logit) {\n                filtered_tokens.push_back(cur_p->data[i]);\n            }\n        }\n\n        // if we have enough values the operation was a success\n        if (!filtered_tokens.empty() && filtered_tokens.size() >= ctx->min_keep) {\n            std::copy(filtered_tokens.begin(), filtered_tokens.end(), cur_p->data);\n            cur_p->size = filtered_tokens.size();\n            min_p_applied = true;\n        }\n    }\n\n    // if the cur_p are sorted or the unsorted implementation failed, use this implementation\n    if (!min_p_applied) {\n        // Sort the logits in descending order\n        if (!cur_p->sorted) {\n            llama_token_data_array_partial_sort_inplace(cur_p, cur_p->size);\n        }\n\n        const float min_logit = cur_p->data[0].logit + logf(ctx->p); // min logit for p_i >= p * p_max\n        size_t i = 1; // first token always matches\n\n        for (; i < cur_p->size; ++i) {\n            if (cur_p->data[i].logit < min_logit && i >= ctx->min_keep) {\n                break; // prob too small\n            }\n        }\n\n        // Resize the output vector to keep only the matching tokens\n        cur_p->size = i;\n    }\n}\n\nstatic struct llama_sampler * llama_sampler_min_p_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_min_p *) smpl->ctx;\n    return llama_sampler_init_min_p(ctx->p, ctx->min_keep);\n}\n\nstatic void llama_sampler_min_p_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_min_p *) smpl->ctx;\n}\n\nstatic bool llama_sampler_min_p_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_min_p *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_min_p_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    auto * sctx = (llama_sampler_min_p *) smpl->ctx;\n\n    struct ggml_tensor * max_idx = ggml_argmax(ctx, data->logits);\n    ggml_set_name(max_idx, \"max_idx\");\n\n    struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]);\n    ggml_set_name(logits_rows, \"logits_rows\");\n\n    struct ggml_tensor * max_logit = ggml_get_rows(ctx, logits_rows, max_idx);\n    ggml_set_name(max_logit, \"max_logit\");\n\n    // Calculate the threshold value.\n    struct ggml_tensor * threshold = ggml_scale_bias(ctx, max_logit, 1.0f, logf(sctx->p));\n    ggml_set_name(threshold, \"min_p_threshold\");\n\n    // Subtract the threshold from logits.\n    struct ggml_tensor * sub = ggml_sub(ctx, data->logits, threshold);\n\n    // Create a mask where logits below the threshold are 0 (discard),\n    // and others are 1 (keep).\n    struct ggml_tensor * mask = ggml_step(ctx, sub);\n    ggml_set_name(mask, \"min_p_mask\");\n\n    // Apply -INFINITY bias for masked-out tokens\n    // log(1) = 0 (keep), log(0) = -INF (discard)\n    struct ggml_tensor * min_p_bias = ggml_log(ctx, mask);\n    ggml_set_name(min_p_bias, \"min_p_bias\");\n\n    data->logits = ggml_add(ctx, data->logits, min_p_bias);\n    ggml_set_name(data->logits, \"min_p_logits\");\n\n    GGML_UNUSED(gf);\n}\n\nstatic struct llama_sampler_i llama_sampler_min_p_i = {\n    /* .name              = */ llama_sampler_min_p_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_min_p_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_min_p_clone,\n    /* .free              = */ llama_sampler_min_p_free,\n    /* .backend_init      = */ llama_sampler_min_p_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_min_p_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_min_p(float p, size_t min_keep) {\n    const bool is_empty = (p <= 0.0f);\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?min-p\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_min_p_i,\n        /* .ctx   = */ new llama_sampler_min_p {\n            (\"min-p\"),\n            /* .p        = */ p,\n            /* .min_keep = */ min_keep,\n        }\n    );\n}\n\n// typical\n\nstruct llama_sampler_typical {\n    const float  p;\n    const size_t min_keep;\n};\n\nstatic const char * llama_sampler_typical_name(const struct llama_sampler * /*smpl*/) {\n    return \"typical\";\n}\n\nstatic void llama_sampler_typical_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_typical *) smpl->ctx;\n\n    // Reference implementation:\n    // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr\n    if (ctx->p >= 1.0f) {\n        return;\n    }\n\n    // Compute the softmax of logits and calculate entropy\n    llama_sampler_softmax_impl(cur_p, true);\n\n    float entropy = 0.0f;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        entropy += -cur_p->data[i].p * logf(cur_p->data[i].p);\n    }\n\n    // Compute the absolute difference between negative log probability and entropy for each candidate\n    std::vector<float> shifted_scores;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        float shifted_score = fabsf(-logf(cur_p->data[i].p) - entropy);\n        shifted_scores.push_back(shifted_score);\n    }\n\n    // Sort tokens based on the shifted_scores and their corresponding indices\n    std::vector<size_t> indices(cur_p->size);\n    std::iota(indices.begin(), indices.end(), 0);\n\n    std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) {\n        return shifted_scores[a] < shifted_scores[b];\n    });\n\n    // Compute the cumulative probabilities\n    float cum_sum = 0.0f;\n    size_t last_idx = indices.size();\n\n    for (size_t i = 0; i < indices.size(); ++i) {\n        size_t idx = indices[i];\n        cum_sum += cur_p->data[idx].p;\n\n        // Check if the running sum is greater than typical or if we have kept at least min_keep tokens\n        if (cum_sum > ctx->p && (ctx->min_keep == 0 || i >= ctx->min_keep - 1)) {\n            last_idx = i + 1;\n            break;\n        }\n    }\n\n    // Resize the output vector to keep only the locally typical tokens\n    std::vector<llama_token_data> cur_p_new;\n    for (size_t i = 0; i < last_idx; ++i) {\n        size_t idx = indices[i];\n        cur_p_new.push_back(cur_p->data[idx]);\n    }\n\n    // Replace the data in cur_p with the cur_p_new data\n    std::copy(cur_p_new.begin(), cur_p_new.end(), cur_p->data);\n    cur_p->size = cur_p_new.size();\n    cur_p->sorted = false;\n}\n\nstatic struct llama_sampler * llama_sampler_typical_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_typical *) smpl->ctx;\n    return llama_sampler_init_typical(ctx->p, ctx->min_keep);\n}\n\nstatic void llama_sampler_typical_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_typical *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_typical_i = {\n    /* .name              = */ llama_sampler_typical_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_typical_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_typical_clone,\n    /* .free              = */ llama_sampler_typical_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_typical(float p, size_t min_keep) {\n    const bool is_empty = (p >= 1.0f);\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?typical\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_typical_i,\n        /* .ctx   = */ new llama_sampler_typical {\n            /* .p        = */ p,\n            /* .min_keep = */ min_keep,\n        }\n    );\n}\n\n// temp\n\nstruct llama_sampler_temp : public llama_sampler_backend {\n    const float temp;\n};\n\nstatic const char * llama_sampler_temp_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_temp *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_temp_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    const auto * ctx = (llama_sampler_temp *) smpl->ctx;\n\n    llama_sampler_temp_impl(cur_p, ctx->temp);\n}\n\nstatic struct llama_sampler * llama_sampler_temp_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_temp *) smpl->ctx;\n    return llama_sampler_init_temp(ctx->temp);\n}\n\nstatic void llama_sampler_temp_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_temp *) smpl->ctx;\n}\n\nstatic void llama_sampler_backend_temp_sampling(\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data,\n        float                       temp) {\n    if (temp <= 0.0f) {\n        // Find the most probable token index.\n        struct ggml_tensor * max_idx = ggml_argmax(ctx, data->logits);\n        ggml_set_name(max_idx, \"temp_max_idx\");\n\n        if (data->candidates) {\n            struct ggml_tensor * candidates_rows = ggml_reshape_2d(ctx, data->candidates, 1, data->candidates->ne[0]);\n            data->candidates = ggml_get_rows(ctx, candidates_rows, max_idx);\n        } else {\n            data->candidates = max_idx;\n        }\n\n        struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]);\n        data->logits = ggml_get_rows(ctx, logits_rows, max_idx);\n\n        return;\n    }\n\n    data->logits = ggml_scale(ctx, data->logits, 1.0f / temp);\n\n    GGML_UNUSED(gf);\n}\n\nstatic bool llama_sampler_temp_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_temp *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_temp_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    auto * sctx = (llama_sampler_temp *) smpl->ctx;\n    llama_sampler_backend_temp_sampling(ctx, gf, data, sctx->temp);\n}\n\nstatic struct llama_sampler_i llama_sampler_temp_i = {\n    /* .name              = */ llama_sampler_temp_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_temp_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_temp_clone,\n    /* .free              = */ llama_sampler_temp_free,\n    /* .backend_init      = */ llama_sampler_temp_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_temp_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_temp(float temp) {\n    const bool is_empty = temp == 1.0f;\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?temp\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_temp_i,\n        /* .ctx   = */ new llama_sampler_temp {\n            (\"temp\"),\n            /*.temp = */ temp,\n        }\n    );\n}\n\n// temp-ext\n\nstruct llama_sampler_temp_ext : public llama_sampler_backend {\n    const float temp;\n    const float delta;\n    const float exponent;\n};\n\nstatic const char * llama_sampler_temp_ext_name(const struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_temp_ext *) smpl->ctx;\n    return sctx->get_name();\n}\n\nstatic void llama_sampler_temp_ext_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_temp_ext *) smpl->ctx;\n    if (ctx->delta > 0) {\n        const float min_temp = std::max(0.0f, ctx->temp - ctx->delta);\n        const float max_temp = ctx->temp + ctx->delta;\n\n        float exponent_val = ctx->exponent;\n\n        // no need to do anything if there is only one (or zero) candidates\n        if (cur_p->size <= 1) {\n            return;\n        }\n\n        // Calculate maximum possible entropy\n        float max_entropy = -logf(1.0f / cur_p->size);\n\n        llama_sampler_softmax_impl(cur_p, true);\n\n        // Calculate entropy of the softmax probabilities\n        float entropy = 0.0f;\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            float prob = cur_p->data[i].p;\n            if (prob > 0.0f) { // Ensure no log(0)\n                entropy -= prob * logf(prob);\n            }\n        }\n\n        // Normalize the entropy (max_entropy cannot be 0 here because we checked cur_p->size != 1 above)\n        float normalized_entropy = entropy / max_entropy;\n\n        // Map the normalized entropy to the desired temperature range using the power function\n        float dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent_val);\n\n    #ifdef DEBUG\n        LLAMA_LOG_INFO(\"Your text maxtemp value is: %f\\n\", max_temp);\n        LLAMA_LOG_INFO(\"Entropy: %f\\n\", entropy);\n        LLAMA_LOG_INFO(\"Max Possible Entropy: %f\\n\", max_entropy);\n        LLAMA_LOG_INFO(\"Normalized Entropy: %f\\n\", normalized_entropy);\n        LLAMA_LOG_INFO(\"Exponent: %f\\n\", exponent_val);\n        LLAMA_LOG_INFO(\"Dynamic Temperature (dyn_temp): %f\\n\", dyn_temp);\n    #endif\n\n        // Apply the dynamically calculated temperature scaling\n        llama_sampler_temp_impl(cur_p, dyn_temp);\n\n        // Re-compute softmax probabilities after scaling logits with dynamic temperature\n        const double max_l_double = cur_p->data[0].logit;\n\n        double cum_sum_double = 0.0;\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            double p = exp(cur_p->data[i].logit - max_l_double);\n            cur_p->data[i].p = p; // Store the scaled probability\n            cum_sum_double += p;\n        }\n\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            cur_p->data[i].p /= cum_sum_double; // Re-normalize the probabilities\n        }\n\n    #ifdef DEBUG\n        // Print the updated top 25 probabilities after temperature scaling\n        LLAMA_LOG_INFO(\"\\nUpdated Top 25 Probabilities After Dynamic Temperature Scaling (in percentages):\\n\");\n        for (size_t i = 0; i < 25 && i < cur_p->size; ++i) {\n            LLAMA_LOG_INFO(\"Token %zu: %f%%\\n\", i + 1, cur_p->data[i].p * 100.0f);\n        }\n    #endif\n    } else {\n        llama_sampler_temp_impl(cur_p, ctx->temp);\n    }\n}\n\nstatic struct llama_sampler * llama_sampler_temp_ext_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_temp_ext *) smpl->ctx;\n    return llama_sampler_init_temp_ext(ctx->temp, ctx->delta, ctx->exponent);\n}\n\nstatic void llama_sampler_temp_ext_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_temp_ext *) smpl->ctx;\n}\n\nstatic bool llama_sampler_temp_ext_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    auto * sctx = (llama_sampler_temp_ext *) smpl->ctx;\n\n    const bool res = llama_sampler_backend_support(smpl, buft);\n\n    sctx->init(res);\n\n    return res;\n}\n\nstatic void llama_sampler_temp_ext_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    auto * sctx = (llama_sampler_temp_ext *) smpl->ctx;\n\n    // Revert to standard temperature scaling if delta or temp are non-positive.\n    if (sctx->delta <= 0.0f || sctx->temp <= 0.0f) {\n        llama_sampler_backend_temp_sampling(ctx, gf, data, sctx->temp);\n        return;\n    }\n\n    // Calculate min_temp, max_temp, and max_entropy.\n    const float min_temp    = std::max(0.0f, sctx->temp - sctx->delta);\n    const float max_temp    = sctx->temp + sctx->delta;\n    const float max_entropy = logf(data->logits->ne[0]);\n\n    // Calculate the probabilities.\n    struct ggml_tensor * probs = ggml_soft_max(ctx, data->logits);\n    ggml_set_name(probs, \"temp_ext_softmax_probs\");\n\n    // Clamp probabilities to avoid log(0) which would give -inf\n    struct ggml_tensor * probs_clamped = ggml_clamp(ctx, probs, 1e-10f, 1.0f);\n    ggml_set_name(probs_clamped, \"temp_ext_probs_clamped\");\n\n    // Calculate the entropy, entropy = -Σ(p * log(p)).\n    struct ggml_tensor * log_probs   = ggml_log(ctx, probs_clamped);\n    struct ggml_tensor * p_log_p     = ggml_mul(ctx, probs_clamped, log_probs);\n    struct ggml_tensor * sum_p_log_p = ggml_sum(ctx, p_log_p);\n    struct ggml_tensor * entropy     = ggml_scale(ctx, sum_p_log_p, -1.0f);\n    ggml_set_name(log_probs,   \"temp_ext_log_probs\");\n    ggml_set_name(p_log_p,     \"temp_ext_p_log_p\");\n    ggml_set_name(sum_p_log_p, \"temp_ext_sum_p_log_p\");\n    ggml_set_name(entropy,     \"temp_ext_entropy\");\n\n    // Normalize the entropy, norm_entropy = entropy / max_entropy\n    struct ggml_tensor * norm_entropy = ggml_scale(ctx, entropy, 1.0f / max_entropy);\n    ggml_set_name(norm_entropy, \"temp_ext_norm_entropy\");\n\n    // Calculate the dynamic temperature:\n    // dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent);\n    //\n    // Calculate powf(normalized_entropy, exponent) as\n    // norm_entropy^exponent = exp(exponent * log(norm_entropy))\n    struct ggml_tensor * log_norm_entropy = ggml_log(ctx, norm_entropy);\n    struct ggml_tensor * scaled_log       = ggml_scale(ctx, log_norm_entropy, sctx->exponent);\n    struct ggml_tensor * pow_entropy      = ggml_exp(ctx, scaled_log);\n    // With pow_entropy computed we can now compute dyn_temp, scaling by\n    // (max_temp - min_temp) and then adding min_temp.\n    struct ggml_tensor * dyn_temp         = ggml_scale_bias(ctx, pow_entropy, max_temp - min_temp, min_temp);\n    ggml_set_name(log_norm_entropy, \"temp_ext_log_norm_entropy\");\n    ggml_set_name(scaled_log,       \"temp_ext_scaled_log\");\n    ggml_set_name(pow_entropy,      \"temp_ext_pow_entropy\");\n    ggml_set_name(dyn_temp,         \"temp_ext_dyn_temp\");\n\n    // Scale the logits by the dynamic temperature\n    struct ggml_tensor * scaled_logits = ggml_div(ctx, data->logits, dyn_temp);\n    ggml_set_name(scaled_logits, \"temp_ext_scaled_logits\");\n\n    data->logits = scaled_logits;\n}\n\nstatic struct llama_sampler_i llama_sampler_temp_ext_i = {\n    /* .name              = */ llama_sampler_temp_ext_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_temp_ext_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_temp_ext_clone,\n    /* .free              = */ llama_sampler_temp_ext_free,\n    /* .backend_init      = */ llama_sampler_temp_ext_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_temp_ext_backend_apply,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_temp_ext(float temp, float delta, float exponent) {\n    const bool is_empty = temp == 1.0f && delta <= 0.0f;\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?temp-ext\");\n    }\n\n    auto * res = llama_sampler_init(\n        /* .iface = */ &llama_sampler_temp_ext_i,\n        /* .ctx   = */ new llama_sampler_temp_ext {\n            (\"temp-ext\"),\n            /* .temp     = */ temp,\n            /* .delta    = */ delta,\n            /* .exponent = */ exponent,\n        }\n    );\n\n    return res;\n}\n\n// xtc\n\nstruct llama_sampler_xtc {\n    const float    probability;\n    const float    threshold;\n    const size_t   min_keep;\n\n    const uint32_t seed;\n    uint32_t       seed_cur;\n\n    std::mt19937   rng;\n};\n\nstatic const char * llama_sampler_xtc_name(const struct llama_sampler * /*smpl*/) {\n    return \"xtc\";\n}\n\nstatic void llama_sample_xtc_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_xtc *) smpl->ctx;\n\n    if (ctx->probability <= 0.0f\n        || ctx->threshold > 0.5f\n        || cur_p->size < 2) {\n        return;\n    }\n\n    std::uniform_real_distribution<float> distribution(0.0f, 1.0f);\n    float chance = distribution(ctx->rng);\n    if (chance > ctx->probability) {\n        return;\n    }\n\n    llama_sampler_softmax_impl(cur_p, true);\n\n    int pos_last = 0;\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        if (cur_p->data[i].p >= ctx->threshold) {\n            pos_last = i;\n        } else {\n            break;\n        }\n    }\n\n    if (cur_p->size - pos_last >= ctx->min_keep && pos_last > 0) {\n        cur_p->data += pos_last;\n        cur_p->size -= pos_last;\n    }\n}\n\nstatic struct llama_sampler * llama_sampler_xtc_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_xtc *) smpl->ctx;\n    auto * result = llama_sampler_init_xtc(ctx->probability, ctx->threshold, ctx->min_keep, ctx->seed);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_xtc *) result->ctx;\n\n        result_ctx->rng = ctx->rng;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_xtc_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_xtc *) smpl->ctx;\n}\n\nstatic void llama_sampler_xtc_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_xtc *) smpl->ctx;\n    ctx->seed_cur = get_rng_seed(ctx->seed);\n    ctx->rng.seed(ctx->seed_cur);\n}\n\nstatic struct llama_sampler_i llama_sampler_xtc_i = {\n    /* .name              = */ llama_sampler_xtc_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sample_xtc_apply,\n    /* .reset             = */ llama_sampler_xtc_reset,\n    /* .clone             = */ llama_sampler_xtc_clone,\n    /* .free              = */ llama_sampler_xtc_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_xtc(float p, float t, size_t min_keep, uint32_t seed) {\n    const bool is_empty = (p <= 0.0f || t > 0.5f);\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?xtc\");\n    }\n\n    const auto seed_cur = get_rng_seed(seed);\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_xtc_i,\n        /* .ctx   = */ new llama_sampler_xtc {\n            /* .probability   = */ p,\n            /* .threshold     = */ t,\n            /* .min_keep      = */ min_keep,\n            /* .seed          = */ seed,\n            /* .seed_cur      = */ seed_cur,\n            /* .rng           = */ std::mt19937(seed_cur),\n        }\n    );\n}\n\n// mirostat\n\nstruct llama_sampler_mirostat {\n    const int32_t n_vocab;\n\n    const uint32_t seed;\n          uint32_t seed_cur;\n\n    const float tau;\n    const float eta;\n\n    const int32_t m;\n\n    float mu;\n\n    std::mt19937    rng;\n};\n\nstatic const char * llama_sampler_mirostat_name(const struct llama_sampler * /*smpl*/) {\n    return \"mirostat\";\n}\n\nstatic void llama_sampler_mirostat_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_mirostat *) smpl->ctx;\n\n    llama_sampler_softmax_impl(cur_p, true);\n\n    // Estimate s_hat using the most probable m tokens\n    float s_hat = 0.0;\n    float sum_ti_bi = 0.0;\n    float sum_ti_sq = 0.0;\n    for (size_t i = 0; i < size_t(ctx->m - 1) && i < cur_p->size - 1; ++i) {\n        float t_i = logf(float(i + 2) / float(i + 1));\n        float b_i = logf(cur_p->data[i].p / cur_p->data[i + 1].p);\n        sum_ti_bi += t_i * b_i;\n        sum_ti_sq += t_i * t_i;\n    }\n    s_hat = sum_ti_bi / sum_ti_sq;\n\n    // Compute k from the estimated s_hat and target surprise value\n    float epsilon_hat = s_hat - 1;\n    float k = powf((epsilon_hat * powf(2, ctx->mu)) / (1 - powf(ctx->n_vocab, -epsilon_hat)), 1 / s_hat);\n\n    llama_sampler_top_k_impl(cur_p, std::max(int(k), 1));\n\n    llama_sampler_softmax_impl(cur_p, true);\n\n    const int idx = llama_sample_dist(cur_p, ctx->rng);\n\n    cur_p->selected = idx;\n\n    float observed_surprise = -log2f(cur_p->data[idx].p);\n    float e = observed_surprise - ctx->tau;\n\n    // Update mu using the learning rate and error\n    ctx->mu = ctx->mu - ctx->eta * e;\n}\n\nstatic struct llama_sampler * llama_sampler_mirostat_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_mirostat *) smpl->ctx;\n    auto * result = llama_sampler_init_mirostat(ctx->n_vocab, ctx->seed, ctx->tau, ctx->eta, ctx->m);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_mirostat *) smpl->ctx;\n\n        result_ctx->mu  = ctx->mu;\n        result_ctx->rng = ctx->rng;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_mirostat_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_mirostat *) smpl->ctx;\n    ctx->mu = 2.0f*ctx->tau;\n    ctx->seed_cur = get_rng_seed(ctx->seed);\n    ctx->rng.seed(ctx->seed_cur);\n}\n\nstatic void llama_sampler_mirostat_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_mirostat *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_mirostat_i = {\n    /* .name              = */ llama_sampler_mirostat_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_mirostat_apply,\n    /* .reset             = */ llama_sampler_mirostat_reset,\n    /* .clone             = */ llama_sampler_mirostat_clone,\n    /* .free              = */ llama_sampler_mirostat_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_mirostat(int32_t n_vocab, uint32_t seed, float tau, float eta, int32_t m) {\n    const auto seed_cur = get_rng_seed(seed);\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_mirostat_i,\n        /* .ctx   = */ new llama_sampler_mirostat {\n            /* .n_vocab  = */ n_vocab,\n            /* .seed     = */ seed,\n            /* .seed_cur = */ seed_cur,\n            /* .tau      = */ tau,\n            /* .eta      = */ eta,\n            /* .m        = */ m,\n            /* .mu       = */ 2.0f*tau,\n            /* .rng      = */ std::mt19937(seed_cur),\n        }\n    );\n}\n\n// mirostat v2\n\nstruct llama_sampler_mirostat_v2 {\n    const uint32_t seed;\n          uint32_t seed_cur;\n\n    const float tau;\n    const float eta;\n\n    float mu;\n\n    std::mt19937 rng;\n};\n\nstatic const char * llama_sampler_mirostat_v2_name(const struct llama_sampler * /*smpl*/) {\n    return \"mirostat-v2\";\n}\n\nstatic void llama_sampler_mirostat_v2_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_mirostat_v2 *) smpl->ctx;\n\n    llama_sampler_softmax_impl(cur_p, true);\n\n    // Truncate the words with surprise values greater than mu\n    cur_p->size = std::distance(cur_p->data, std::find_if(cur_p->data, cur_p->data + cur_p->size, [&](const llama_token_data & candidate) {\n        return -log2f(candidate.p) > ctx->mu;\n    }));\n\n    if (cur_p->size == 0) {\n        cur_p->size = 1;\n    }\n\n    // Normalize the probabilities of the remaining words\n    llama_sampler_softmax_impl(cur_p, true);\n\n    const int idx = llama_sample_dist(cur_p, ctx->rng);\n\n    cur_p->selected = idx;\n\n    float observed_surprise = -log2f(cur_p->data[idx].p);\n    float e = observed_surprise - ctx->tau;\n\n    // Update mu using the learning rate and error\n    ctx->mu = ctx->mu - ctx->eta * e;\n}\n\nstatic void llama_sampler_mirostat_v2_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_mirostat_v2 *) smpl->ctx;\n    ctx->mu = 2.0f*ctx->tau;\n    ctx->seed_cur = get_rng_seed(ctx->seed);\n    ctx->rng.seed(ctx->seed_cur);\n}\n\nstatic struct llama_sampler * llama_sampler_mirostat_v2_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_mirostat_v2 *) smpl->ctx;\n\n    auto * result = llama_sampler_init_mirostat_v2(ctx->seed, ctx->tau, ctx->eta);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_mirostat_v2 *) result->ctx;\n\n        result_ctx->mu  = ctx->mu;\n        result_ctx->rng = ctx->rng;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_mirostat_v2_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_mirostat_v2 *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_mirostat_v2_i = {\n    /* .name              = */ llama_sampler_mirostat_v2_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_mirostat_v2_apply,\n    /* .reset             = */ llama_sampler_mirostat_v2_reset,\n    /* .clone             = */ llama_sampler_mirostat_v2_clone,\n    /* .free              = */ llama_sampler_mirostat_v2_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_mirostat_v2(uint32_t seed, float tau, float eta) {\n    auto seed_cur = get_rng_seed(seed);\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_mirostat_v2_i,\n        /* .ctx   = */ new llama_sampler_mirostat_v2 {\n            /* .seed     = */ seed,\n            /* .seed_cur = */ seed_cur,\n            /* .tau      = */ tau,\n            /* .eta      = */ eta,\n            /* .mu       = */ 2.0f*tau,\n            /* .rng      = */ std::mt19937(seed_cur),\n        }\n    );\n}\n\n// grammar\n\nstruct llama_sampler_grammar {\n    const struct llama_vocab * vocab;\n\n    std::string grammar_str;\n    std::string grammar_root;\n\n    struct llama_grammar * grammar;\n};\n\nstatic const char * llama_sampler_grammar_name(const struct llama_sampler * /*smpl*/) {\n    return \"grammar\";\n}\n\nstatic void llama_sampler_grammar_accept_impl(struct llama_sampler * smpl, llama_token token) {\n    auto * ctx = (llama_sampler_grammar *) smpl->ctx;\n    if (ctx->grammar) {\n        llama_grammar_accept_impl(*ctx->grammar, token);\n    }\n}\n\nstatic void llama_sampler_grammar_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_grammar *) smpl->ctx;\n    if (ctx->grammar) {\n        llama_grammar_apply_impl(*ctx->grammar, cur_p);\n    }\n}\n\n// Fwd declare to break reset --> init_impl --> llama_sampler_grammar_i --> reset cycle.\nstatic struct llama_sampler * llama_sampler_init_grammar_impl(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                              bool lazy,\n                     const char ** trigger_words,\n                            size_t num_trigger_words,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns);\n\nstatic void llama_sampler_grammar_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_grammar *) smpl->ctx;\n    if (!ctx->grammar) {\n        return;\n    }\n\n    std::vector<const char *>  trigger_patterns_c;\n    trigger_patterns_c.reserve(ctx->grammar->trigger_patterns.size());\n    for (auto & trigger_pattern : ctx->grammar->trigger_patterns) {\n        trigger_patterns_c.push_back(trigger_pattern.pattern.c_str());\n    }\n\n    auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),\n                                                 ctx->grammar->lazy, trigger_patterns_c.data(), trigger_patterns_c.size(),\n                                                 ctx->grammar->trigger_tokens.data(), ctx->grammar->trigger_tokens.size());\n\n    llama_grammar_free_impl(ctx->grammar);\n    ctx->grammar = grammar_new;\n}\n\nstatic struct llama_sampler * llama_sampler_grammar_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_grammar *) smpl->ctx;\n\n    auto * result = llama_sampler_init_grammar_impl(ctx->vocab, nullptr, nullptr, false, nullptr, 0, nullptr, 0, nullptr, 0);\n    GGML_ASSERT(result);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_grammar *) result->ctx;\n\n        if (ctx->grammar) {\n            result_ctx->grammar_str  = ctx->grammar_str;\n            result_ctx->grammar_root = ctx->grammar_root;\n\n            result_ctx->grammar = llama_grammar_clone_impl(*ctx->grammar);\n        }\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_grammar_free(struct llama_sampler * smpl) {\n    const auto * ctx = (llama_sampler_grammar *) smpl->ctx;\n\n    if (ctx->grammar) {\n        llama_grammar_free_impl(ctx->grammar);\n    }\n\n    delete ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_grammar_i = {\n    /* .name              = */ llama_sampler_grammar_name,\n    /* .accept            = */ llama_sampler_grammar_accept_impl,\n    /* .apply             = */ llama_sampler_grammar_apply,\n    /* .reset             = */ llama_sampler_grammar_reset,\n    /* .clone             = */ llama_sampler_grammar_clone,\n    /* .free              = */ llama_sampler_grammar_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstatic struct llama_sampler * llama_sampler_init_grammar_impl(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                              bool lazy,\n                     const char ** trigger_words,\n                            size_t num_trigger_words,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns) {\n    auto * ctx = new llama_sampler_grammar;\n\n    if (grammar_str != nullptr && grammar_str[0] != '\\0') {\n        std::string trigger_pattern;\n        llama_grammar * grammar = nullptr;\n        // TODO: remove trigger_words support.\n        if (trigger_words != nullptr && num_trigger_words > 0) {\n            GGML_ASSERT(trigger_patterns == nullptr && num_trigger_patterns == 0);\n            trigger_pattern = \"[\\\\s\\\\S]*?(\";\n            for (size_t i = 0; i < num_trigger_words; ++i) {\n                static const std::regex special_chars(\"[.^$|()*+?\\\\[\\\\]{}\\\\\\\\]\");\n                if (i > 0) {\n                    trigger_pattern += \"|\";\n                }\n                trigger_pattern += std::regex_replace(trigger_words[i], special_chars, \"\\\\$0\");\n            }\n            trigger_pattern += \")[\\\\s\\\\S]*\";\n\n            std::array<const char *, 1> tmp_trigger_patterns = { trigger_pattern.c_str() };\n            grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, tmp_trigger_patterns.data(), tmp_trigger_patterns.size(), trigger_tokens, num_trigger_tokens);\n        } else {\n            grammar = llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens);\n        }\n        *ctx = {\n            /* .vocab        = */ vocab,\n            /* .grammar_str  = */ grammar_str,\n            /* .grammar_root = */ grammar_root,\n            /* .grammar      = */ grammar,\n        };\n        if (!ctx->grammar) {\n            delete ctx;\n            return nullptr;\n        }\n    } else {\n        *ctx = {\n            /* .vocab        = */ vocab,\n            /* .grammar_str  = */ {},\n            /* .grammar_root = */ {},\n            /* .grammar      = */ nullptr,\n        };\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_grammar_i,\n        /* .ctx   = */ ctx\n    );\n}\n\nstruct llama_sampler * llama_sampler_init_grammar(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root) {\n    return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ false, nullptr, 0, nullptr, 0, nullptr, 0);\n}\n\nstruct llama_sampler * llama_sampler_init_grammar_lazy(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                     const char ** trigger_words,\n                            size_t num_trigger_words,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens) {\n    return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, trigger_words, num_trigger_words, trigger_tokens, num_trigger_tokens, nullptr, 0);\n}\n\nstruct llama_sampler * llama_sampler_init_grammar_lazy_patterns(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens) {\n    return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, nullptr, 0, trigger_tokens, num_trigger_tokens, trigger_patterns, num_trigger_patterns);\n}\n\n// penalties\n\nstruct llama_sampler_penalties {\n    const int32_t penalty_last_n;\n    const float   penalty_repeat;\n    const float   penalty_freq;\n    const float   penalty_present;\n\n    ring_buffer<llama_token> prev;\n\n    // a frequency map to count token occurrences\n    std::unordered_map<llama_token, int> token_count;\n};\n\nstatic const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) {\n    return \"penalties\";\n}\n\nstatic void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_token token) {\n    auto * ctx = (llama_sampler_penalties *) smpl->ctx;\n    if (ctx->penalty_last_n == 0) {\n        return;\n    }\n\n    ctx->token_count[token]++;\n\n    // if the ring buffer is full, remove the oldest token\n    if (ctx->prev.size() >= (size_t) ctx->penalty_last_n) {\n        const auto old = ctx->prev.front();\n\n        ctx->token_count[old]--;\n        if (ctx->token_count[old] == 0) {\n            ctx->token_count.erase(old);\n        }\n    }\n\n    ctx->prev.push_back(token);\n\n#if 0\n    // sanity check\n    std::unordered_map<llama_token, int> tmp;\n    for (int i = 0; i < std::min<int>(ctx->penalty_last_n, ctx->prev.size()); ++i) {\n        tmp[ctx->prev.rat(i)]++;\n    }\n\n    assert(ctx->token_count == tmp);\n#endif\n}\n\nstatic void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_penalties *) smpl->ctx;\n\n    if ((ctx->penalty_last_n == 0) ||\n        (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) {\n        return;\n    }\n\n    // Apply frequency and presence penalties to the cur_p\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        const auto token_iter = ctx->token_count.find(cur_p->data[i].id);\n        if (token_iter == ctx->token_count.end()) {\n            continue;\n        }\n\n        const int count = token_iter->second;\n\n        assert(count > 0 && count <= ctx->penalty_last_n);\n\n        // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.\n        // This is common fix for this problem, which is to multiply by the penalty instead of dividing.\n        if (cur_p->data[i].logit <= 0) {\n            cur_p->data[i].logit *= ctx->penalty_repeat;\n        } else {\n            cur_p->data[i].logit /= ctx->penalty_repeat;\n        }\n\n        cur_p->data[i].logit -= float(count) * ctx->penalty_freq + float(count > 0) * ctx->penalty_present;\n    }\n\n    cur_p->sorted = false;\n}\n\nstatic void llama_sampler_penalties_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_penalties *) smpl->ctx;\n    ctx->prev.clear();\n    ctx->token_count.clear();\n}\n\nstatic struct llama_sampler * llama_sampler_penalties_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_penalties *) smpl->ctx;\n    auto * result = llama_sampler_init_penalties(\n            ctx->penalty_last_n,\n            ctx->penalty_repeat,\n            ctx->penalty_freq,\n            ctx->penalty_present);\n\n    // copy the state\n    {\n        auto * result_ctx = (llama_sampler_penalties *) result->ctx;\n\n        result_ctx->prev = ctx->prev;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_penalties_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_penalties *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_penalties_i = {\n    /* .name              = */ llama_sampler_penalties_name,\n    /* .accept            = */ llama_sampler_penalties_accept,\n    /* .apply             = */ llama_sampler_penalties_apply,\n    /* .reset             = */ llama_sampler_penalties_reset,\n    /* .clone             = */ llama_sampler_penalties_clone,\n    /* .free              = */ llama_sampler_penalties_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_penalties(\n        int32_t penalty_last_n,\n        float penalty_repeat,\n        float penalty_freq,\n        float penalty_present) {\n    penalty_last_n = std::max(penalty_last_n, 0);\n\n    const bool is_empty = (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f));\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?penalties\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_penalties_i,\n        /* .ctx   = */ new llama_sampler_penalties {\n            /* .penalty_last_n  = */ penalty_last_n,\n            /* .penalty_repeat  = */ penalty_repeat,\n            /* .penalty_freq    = */ penalty_freq,\n            /* .penalty_present = */ penalty_present,\n            /* .prev            = */ ring_buffer<llama_token>(penalty_last_n),\n            /* .token_count     = */ {},\n        }\n    );\n}\n\n// top-n-sigma\n\nstruct llama_sampler_top_n_sigma {\n    const float n;\n};\n\nstatic const char * llama_sampler_top_n_sigma_name(const struct llama_sampler * /*smpl*/) {\n    return \"top-n-sigma\";\n}\n\nstatic void llama_sampler_top_n_sigma_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_top_n_sigma *) smpl->ctx;\n\n    if (ctx->n <= 0.0f || cur_p->size <= 1) {\n        return;\n    }\n\n    // find max logit and calculate mean\n    float max = cur_p->data[0].logit;\n    float logits_sum = 0;\n    size_t valid_count = 0;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        // Only count non-negative infinity values\n        if (cur_p->data[i].logit != -INFINITY) {\n            max = std::max(max, cur_p->data[i].logit);\n            logits_sum += cur_p->data[i].logit;\n            valid_count++;\n        }\n    }\n    float mean = valid_count > 0 ? logits_sum/valid_count : 0;\n\n    // calculate standard deviation\n    float acc = 0;\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        // Skip -infinity in std calculation\n        if (cur_p->data[i].logit != -INFINITY) {\n            acc += pow(cur_p->data[i].logit - mean, 2);\n        }\n    }\n    float std = valid_count > 0 ? sqrt(acc/valid_count) : 0;\n\n    // apply mask\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        if (cur_p->data[i].logit < max - (ctx->n * std)) {\n            cur_p->data[i].logit = -INFINITY;\n        }\n    }\n\n    llama_sampler_softmax_impl(cur_p, true);\n}\n\nstatic struct llama_sampler * llama_sampler_top_n_sigma_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_top_n_sigma *) smpl->ctx;\n    return llama_sampler_init_top_n_sigma(ctx->n);\n}\n\nstatic void llama_sampler_top_n_sigma_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_top_n_sigma *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_top_n_sigma_i = {\n    /* .name              = */ llama_sampler_top_n_sigma_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_top_n_sigma_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_top_n_sigma_clone,\n    /* .free              = */ llama_sampler_top_n_sigma_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_top_n_sigma(float n) {\n    const bool is_empty = (n <= 0.0f);\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?top-n-sigma\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_top_n_sigma_i,\n        /* .ctx   = */ new llama_sampler_top_n_sigma {\n            /* .n = */ n,\n        }\n    );\n}\n\n// DRY\n\nstruct llama_sampler_dry {\n    int32_t total_context_size;\n\n    const float   dry_multiplier;\n    const float   dry_base;\n    const int32_t dry_allowed_length;\n    const int32_t dry_penalty_last_n;\n\n    std::unordered_multimap<llama_token, std::vector<llama_token>> dry_processed_breakers;\n    std::vector<int> dry_repeat_count;\n    std::unordered_map<llama_token, int> dry_max_token_repeat;\n    ring_buffer<llama_token> last_tokens;\n};\n\n// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)\nstatic void get_overlapping_token_sequences(const llama_vocab & vocab, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {\n    for (llama_token token_id = 0; token_id < (llama_token) vocab.n_tokens(); token_id++) {\n        std::string word = vocab.detokenize({token_id}, true);\n        if (word.find(str) != std::string::npos) {\n            token_sequences.emplace(token_id, std::vector<llama_token>());\n        } else {\n            size_t word_len = word.size();\n            size_t str_len = str.size();\n            size_t pos = -1;\n            while ((pos = word.find(str[0], pos + 1)) != std::string::npos) {\n                bool match = true;\n                size_t i;\n                for (i = 1; i < str_len && i + pos < word_len; ++i) {\n                    if (word[pos + i] != str[i]) {\n                        match = false;\n                        break;\n                    }\n                }\n                if (match) {\n                    std::vector<llama_token> tokenization = vocab.tokenize(str.substr(i), false, false);\n                    if (max_tail_len >= 0 && tokenization.size() > (size_t)max_tail_len) {\n                        tokenization.resize(max_tail_len);\n                    }\n\n                    // Ensure we don't already have a duplicate matching tokenization\n                    auto its = token_sequences.equal_range(token_id);\n                    bool found = false;\n                    for (auto it = its.first; it != its.second; ++it) {\n                        if (tokenization == it->second) {\n                            found = true;\n                            break;\n                        }\n                    }\n                    if (!found) {\n                        token_sequences.emplace(token_id, tokenization);\n                    }\n                }\n            }\n        }\n    }\n}\n\nstatic const char * llama_sampler_dry_name(const struct llama_sampler * /*smpl*/) {\n    return \"dry\";\n}\n\nstatic void llama_sampler_dry_accept(struct llama_sampler * smpl, llama_token token) {\n    auto * ctx = (llama_sampler_dry *) smpl->ctx;\n    if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {\n        return;\n    }\n\n    ctx->last_tokens.push_back(token);\n}\n\n// Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)\nstatic void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_dry *) smpl->ctx;\n\n    if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {\n        return;\n    }\n\n    int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0);\n    int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size);\n\n    if (last_n_repeat <= ctx->dry_allowed_length) {\n        return;\n    }\n\n    ctx->dry_repeat_count.assign(last_n_repeat, 0);\n    ctx->dry_max_token_repeat.clear();\n\n    // Step 1: Look for restart sequences to limit the maximum repetition length.\n    // Work backwards through the context looking for any token that begins a restart sequence.\n    //\n    // The collection `restart_sequences` is a mapping from a \"head\" token to all \"tail\"\n    // sequences that together comprise a restart sequence. This allows us to quickly check\n    // whether each token is the head of a complete sequence. Most restart sequences are actually\n    // a single token, and for these the \"tail\" is an empty vector.\n    //\n    // If the token is a \"head\", test all restart sequences that begin with this token\n    // (there will often only be one sequence for each token, but if sequences like 'aaaq1' and\n    // 'aaa1' are used as restart strings, both could start with 'aaa' when tokenized). The\n    // longest matching sequence (if any) is used to limit the maximum repetition length.\n    //\n    // Note that in the case case of a short sequence contained in a longer one, this might fail to\n    // find the smallest value for `rep_limit`. For example, if 'amniotic' and 'ni' are both used as\n    // restart sequences, 'ni' will be found first, and since it's shorter it will fail to suppress\n    // 'otic'. This is a minor issue since fully contained restart sequences are likely to be rare.\n    //\n    // This is theoretically worst-case O(N^2) for arbitrary restart sequences, which is why we\n    // have already clamped the maximum tail sequence length when generating `restart_sequences`.\n    // With clamping, this scan is O(N) in the context length.\n\n    int rep_limit = last_n_repeat;\n    for (int i = 0; i < last_n_repeat; ++i) {\n        llama_token token = ctx->last_tokens.rat(i);\n        auto its = ctx->dry_processed_breakers.equal_range(token);\n        if (its.first == ctx->dry_processed_breakers.end()) {\n            continue;\n        }\n        int longest_match = -1;\n        for (auto it = its.first; it != its.second; ++it) {\n            // Note that (*it) does not contain the head character, so seq_len will be\n            // the restart sequence length minus 1.\n            // In the common case of a single-token restart sequence, (*it) will be empty\n            // and we will trivially match.\n            int seq_len = (int)it->second.size();\n            if (seq_len > longest_match && seq_len <= (int)i) {\n                bool match = true;\n                for (int offset = 0; offset < seq_len; ++offset) {\n                    // The -1 when indexing `last_tokens` is because we already matched the head.\n                    if (it->second[offset] != ctx->last_tokens.rat(i - offset - 1)) {\n                        match = false;\n                        break;\n                    }\n                }\n                if (match) {\n                    longest_match = seq_len;\n                }\n            }\n        }\n        if (longest_match >= 0) {\n            // We found a restart sequence starting `i` tokens from the end and continuing for\n            // `longest_match` tokens.\n            rep_limit = i - longest_match;\n            break;\n        }\n    }\n    if (rep_limit < ctx->dry_allowed_length) {\n        return;\n    }\n\n    // Step 2: Iterate in reverse over the last N tokens of the context, using the \"Z-algorithm\" (in\n    // the reverse direction) to efficiently compute the positions and lengths of suffixes appearing\n    // elsewhere in the context. We limit the suffix length to `rep_limit` to respect restart sequences.\n    //\n    // This algorithm is not currently documented on Wikipedia, but there is a clear description here:\n    // https://ivanyu.me/blog/2014/10/15/z-algorithm/\n    //\n    // The code below is adapted from the public domain implementation by the same author here:\n    // https://github.com/ivanyu/string-algorithms/blob/master/z_algorithm.py\n    //\n    // Example:\n    // Last N tokens: a b c c b c y a b c\n    // Repeat counts: 0 0 3 1 0 2 0 0 0 0\n    //                    ^\n    //   This `3` means that the last three tokens of the context (a b c) also appear here.\n    //\n    // This step is worst case O(N) since the Z-algorithm is linear, despite the appearance of nested\n    // for/while loops. This can be seen by observing that the `lt` and `rt` bounds are set after each\n    // repeated suffix is detected (i.e. after each while loop when n > 0). These bound variables\n    // ensure that the inner while loops only examine each token in the context once as the outer\n    // for loop iterates over the context.\n\n    {\n        const int last = last_n_repeat - 1;\n\n        int rt = 0;\n        int lt = 0;\n\n        for (int k = 1; k < last_n_repeat; ++k) {\n            if (k > rt) {\n                // If k is outside the current Z-box, do naive computation.\n                int n = 0;\n                while (n + k < last_n_repeat && ctx->last_tokens.rat(n) == ctx->last_tokens.rat(n+k)) {\n                    ++n;\n                }\n                ctx->dry_repeat_count[last - k] = std::min(n, rep_limit);\n                if (n > 0) {\n                    lt = k;\n                    rt = k + n - 1;\n                }\n            } else {\n                // If k is inside the current Z-box, consider two cases.\n\n                int p = k - lt; // Pair index.\n                int right_part_len = rt - k + 1;\n\n                if (ctx->dry_repeat_count[last - p] < right_part_len) {\n                    int n = std::min(ctx->dry_repeat_count[last - p], rep_limit);\n                    ctx->dry_repeat_count[last - k] = n;\n                } else {\n                    int i = rt + 1;\n                    while (i < last_n_repeat && ctx->last_tokens.rat(i) == ctx->last_tokens.rat(i - k)) {\n                        i += 1;\n                    }\n\n                    int n = std::min(i - k, rep_limit);\n                    ctx->dry_repeat_count[last - k] = n;\n                    lt = k;\n                    rt = i - 1;\n                }\n            }\n        }\n    }\n\n    // Step 3: Iterate over dry_repeat_count and last_tokens, examining the maximum repeat length\n    // that would be generated by emitting each new token that would extend a sequence.\n    //\n    // Following the same example as above:\n    // Last N tokens: a b c c b c y a b c\n    // Repeat counts: 0 0 3 1 0 2 0 0 0 0\n    //\n    // For each non-zero, look ahead one token. This token, if emitted, would extend the repetition.\n    // c: 3 -> 4 (from `a b c` to `a b c c`)\n    // b: 1 -> 2 (from `c` to `c b`)\n    // y: 2 -> 3 (from `b c` to `b c y`)\n\n    for (int i = 0; i < last_n_repeat - 1; ++i) {\n        int repeat_len = ctx->dry_repeat_count[i];\n        if (repeat_len >= ctx->dry_allowed_length) {\n            // This token ends a repeat, so the next token would continue one.\n            // By convention, the value of `repeat_len` only includes the tokens currently\n            // in the context, not the new token that would be added.\n            llama_token token = ctx->last_tokens.rat(last_n_repeat - 2 - i);\n            // Track the maximum sequence ending in this token.\n            const auto& it = ctx->dry_max_token_repeat.find(token);\n            if (it == ctx->dry_max_token_repeat.end() || it->second < repeat_len) {\n                ctx->dry_max_token_repeat[token] = repeat_len;\n            }\n        }\n    }\n\n    // Step 4: Apply logit penalties based on the maximum repeat length for relevant tokens.\n\n    // Prevent floating point overflow in `pow(penalty_base, exponent)` by clamping to `max_exponent`.\n    // Compute it from `penalty_base` and the approximate log of `std::numeric_limits<float>::max()`\n    const float FLOAT_MAX_LOG = 88.7228391f;\n    int max_exponent = 0;\n    if (ctx->dry_base > 1.000001f) {\n        max_exponent = FLOAT_MAX_LOG / std::log(ctx->dry_base);\n    }\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        const auto& af_kvp = ctx->dry_max_token_repeat.find(cur_p->data[i].id);\n        if (af_kvp != ctx->dry_max_token_repeat.end()) {\n            // Check all sequence breakers starting with this token\n            auto range = ctx->dry_processed_breakers.equal_range(cur_p->data[i].id);\n            bool is_single_token_breaker = false;\n\n            for (auto it = range.first; it != range.second; ++it) {\n                if (it->second.empty()) {\n                    is_single_token_breaker = true;\n                    break;\n                }\n            }\n\n            // Apply penalty only if it's not a single-token sequence breaker\n            if (!is_single_token_breaker) {\n                int repeat_exp = af_kvp->second - ctx->dry_allowed_length;\n                if (max_exponent > 0 && repeat_exp > max_exponent) {\n                    repeat_exp = max_exponent;\n                }\n                float penalty = ctx->dry_multiplier * std::pow(ctx->dry_base, repeat_exp);\n                cur_p->data[i].logit -= penalty;\n            }\n        }\n    }\n\n    cur_p->sorted = false;\n}\n\nstatic void llama_sampler_dry_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_dry *) smpl->ctx;\n    ctx->last_tokens.clear();\n    ctx->dry_repeat_count.clear();\n    ctx->dry_max_token_repeat.clear();\n}\n\nstatic struct llama_sampler * llama_sampler_dry_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (llama_sampler_dry *) smpl->ctx;\n\n    llama_vocab dummy_vocab;\n\n    // dummy vocab is passed because it is only needed for raw sequence breaker processing, which we have already done and will simply be copying\n    auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->total_context_size, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);\n\n    // Copy the state, including the processed breakers\n    {\n        auto * result_ctx = (llama_sampler_dry *) result->ctx;\n        result_ctx->dry_processed_breakers = ctx->dry_processed_breakers;\n        result_ctx->dry_repeat_count = ctx->dry_repeat_count;\n        result_ctx->dry_max_token_repeat = ctx->dry_max_token_repeat;\n        result_ctx->last_tokens = ctx->last_tokens;\n    }\n\n    return result;\n}\n\nstatic void llama_sampler_dry_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_dry *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_dry_i = {\n    /* .name              = */ llama_sampler_dry_name,\n    /* .accept            = */ llama_sampler_dry_accept,\n    /* .apply             = */ llama_sampler_dry_apply,\n    /* .reset             = */ llama_sampler_dry_reset,\n    /* .clone             = */ llama_sampler_dry_clone,\n    /* .free              = */ llama_sampler_dry_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, int32_t n_ctx_train, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {\n    int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? n_ctx_train : std::max(dry_penalty_last_n, 0);\n    std::unordered_multimap<llama_token, std::vector<llama_token>> processed_breakers;\n    const int MAX_CHAR_LEN = 40;\n    const int MAX_SEQ_LEN = 20;\n\n    const bool dry_enabled = (dry_multiplier != 0.0f && dry_base >= 1.0f && dry_penalty_last_n != 0);\n\n    if (!dry_enabled) {\n        return llama_sampler_init_empty(\"?dry\");\n    }\n\n    if (dry_enabled && seq_breakers != nullptr && num_breakers > 0) {\n        // Process sequence breakers\n        for (size_t i = 0; i < num_breakers; ++i) {\n            if (seq_breakers[i] == nullptr || std::strlen(seq_breakers[i]) == 0) {\n                LLAMA_LOG_WARN(\"skipping null or empty DRY sequence breaker at index %zu\\n\", i);\n                continue;\n            }\n\n            std::string sequence_break(seq_breakers[i]);\n            if (sequence_break.empty()) {\n                LLAMA_LOG_WARN(\"skipping empty DRY sequence breaker\\n\");\n                continue;\n            }\n\n            if (sequence_break.size() > MAX_CHAR_LEN) {\n                LLAMA_LOG_WARN(\"truncating DRY sequence breaker to %d characters\\n\", MAX_CHAR_LEN);\n                sequence_break.resize(MAX_CHAR_LEN);\n            }\n\n            get_overlapping_token_sequences(*vocab, sequence_break, processed_breakers, MAX_SEQ_LEN);\n        }\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_dry_i,\n        /* .ctx   = */ new llama_sampler_dry {\n            /* .total_context_size     = */ n_ctx_train,\n            /* .dry_multiplier         = */ dry_multiplier,\n            /* .dry_base               = */ dry_base,\n            /* .dry_allowed_length     = */ dry_allowed_length,\n            /* .dry_penalty_last_n     = */ dry_penalty_last_n,\n            /* .dry_processed_breakers = */ std::move(processed_breakers),\n            /* .dry_repeat_count       = */ dry_enabled ? std::vector<int>(effective_dry_penalty_last_n, 0) : std::vector<int>{},\n            /* .dry_max_token_repeat   = */ {},\n            /* .last_tokens            = */ dry_enabled ? ring_buffer<llama_token>(effective_dry_penalty_last_n) : ring_buffer<llama_token>(0),\n        }\n    );\n}\n\n// wrapper for test-sampling.cpp\nstruct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {\n    llama_vocab dummy_vocab;\n    auto * result = llama_sampler_init_dry(&dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);\n    auto * ctx = (llama_sampler_dry *) result->ctx;\n\n    // Process the token-based sequence breakers\n    ctx->dry_processed_breakers.clear();\n    if (seq_breakers.empty()) {\n        LLAMA_LOG_WARN(\"empty DRY sequence breakers list in llama_sampler_init_dry_testing\\n\");\n    } else {\n        for (const auto& breaker : seq_breakers) {\n            if (breaker.empty()) {\n                LLAMA_LOG_WARN(\"skipping DRY empty sequence breaker\\n\");\n                continue;\n            }\n            llama_token head_token = breaker[0];\n            std::vector<llama_token> tail_tokens(breaker.begin() + 1, breaker.end());\n            ctx->dry_processed_breakers.emplace(head_token, std::move(tail_tokens));\n        }\n\n        if (ctx->dry_processed_breakers.empty()) {\n            LLAMA_LOG_WARN(\"no valid DRY sequence breakers processed in llama_sampler_init_dry_testing\\n\");\n        }\n    }\n\n    return result;\n}\n\n// adaptive-p sampler state\n//\n// maintains an exponential moving average of the *ORIGINAL* probabilities\n// of selected tokens, used to compute an adapted target at each sampling step.\n//\n// see llama.h for a full description of the sampler\n//\n// ref: https://github.com/ggml-org/llama.cpp/pull/17927\n//\nstruct llama_sampler_adaptive_p {\n    const float        target;            // target probability (0.0 - 1.0; negative = disabled)\n    const float        decay;             // EMA decay; history ~= 1/(1-decay) tokens (0.0 - 0.99)\n    const uint32_t     seed;              // original RNG seed\n    uint32_t           seed_cur;          // actual RNG seed\n    std::mt19937       rng;               // RNG state\n    float              weighted_sum;      // sum(p_i * decay^i)\n    float              total_weight;      // sum(decay^i), converges to 1/(1-decay)\n    std::vector<float> original_probs;    // pre-transform probs, cached for EMA update\n    llama_token        pending_token_id;  // token ID of selected token\n    int32_t            pending_token_idx; // index of orig. prob. of selected token in original_probs\n};\n\n// adaptive probability transformation constants\nstatic constexpr float DISTRIBUTION_WIDTH =  0.3f;\nstatic constexpr float PEAK_LOGIT_VALUE   =  5.0f;\nstatic constexpr float SHARPNESS          = 10.0f;\nstatic constexpr float INV_WIDTH          =  1.0f / DISTRIBUTION_WIDTH;\n\nstatic const char * llama_sampler_adaptive_p_name(const struct llama_sampler * /*smpl*/) {\n    return \"adaptive-p\";\n}\n\nstatic void llama_sampler_adaptive_p_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_adaptive_p *) smpl->ctx;\n\n    llama_sampler_softmax_impl(cur_p, false);\n\n    if (ctx->target < 0.0f) {\n        // at negative target values, adaptive-p is no-op\n        // we simply sample from the existing distribution\n        cur_p->selected = llama_sample_dist(cur_p, ctx->rng);\n        return;\n    }\n\n    // store the original probabilities\n    ctx->original_probs.resize(cur_p->size);\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        ctx->original_probs[i] = cur_p->data[i].p;\n    }\n\n    // using the EMA, compute the adapted target probability for the current sampling step\n    auto target = std::clamp(ctx->target, 0.0f, 1.0f);\n    float adapted_target = std::clamp(\n        ctx->total_weight == 0.0f ? target : 2.0f * target - (ctx->weighted_sum / ctx->total_weight),\n        0.0f, 1.0f\n    );\n\n    // adaptive probability transform\n    //\n    // quadratic near target for fine differentiation, transitioning to linear decay in the\n    // tails. unbounded negative logits ensure proper suppression of far-from-target tokens\n    // after the softmax.\n    //\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        if (cur_p->data[i].logit == -INFINITY) {\n            // don't transform logits that are -INFINITY\n            // (as masked out by e.g. min-p and top-p when using backend sampling)\n            continue;\n        }\n        float dist = std::abs((cur_p->data[i].p - adapted_target) * INV_WIDTH);\n        cur_p->data[i].logit = PEAK_LOGIT_VALUE - SHARPNESS * dist * dist / (1.0f + dist);\n    }\n\n    // softmax and sample from the transformed distribution\n    llama_sampler_softmax_impl(cur_p, false);\n    const int idx   = llama_sample_dist(cur_p, ctx->rng);\n    cur_p->selected = idx;\n\n    // store the selected token ID for acceptance later\n    ctx->pending_token_id  = cur_p->data[idx].id;\n    ctx->pending_token_idx = idx;\n}\n\nstatic void llama_sampler_adaptive_p_accept(struct llama_sampler * smpl, llama_token token) {\n    auto * ctx = (llama_sampler_adaptive_p *) smpl->ctx;\n    if (ctx->pending_token_id == token) {\n        GGML_ASSERT(ctx->pending_token_id != LLAMA_TOKEN_NULL);\n        GGML_ASSERT(ctx->pending_token_idx != -1);\n        // update EMA with the original probability of the selected token\n        ctx->weighted_sum = ctx->original_probs[ctx->pending_token_idx] + ctx->decay * ctx->weighted_sum;\n        ctx->total_weight = 1.0f + ctx->decay * ctx->total_weight;\n    }\n    ctx->pending_token_id = LLAMA_TOKEN_NULL;\n    ctx->pending_token_idx = -1;\n}\n\nstatic void llama_sampler_adaptive_p_reset(struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_adaptive_p *) smpl->ctx;\n    // ctx->target and ctx->decay never change after init, so it's safe to keep them as is.\n    // original_probs is completely overwritten on every call to _apply.\n    // so we only need to reset the EMA state and pending token.\n    ctx->weighted_sum      = ctx->target / (1.0f - ctx->decay);\n    ctx->total_weight      = 1.0f / (1.0f - ctx->decay);\n    ctx->pending_token_id  = LLAMA_TOKEN_NULL;\n    ctx->pending_token_idx = -1;\n    ctx->seed_cur          = get_rng_seed(ctx->seed);\n    ctx->rng.seed(ctx->seed_cur);\n}\n\nstatic struct llama_sampler * llama_sampler_adaptive_p_clone(const struct llama_sampler * smpl) {\n    const auto * ctx  = (const llama_sampler_adaptive_p *) smpl->ctx;\n    auto * result     = llama_sampler_init_adaptive_p(ctx->target, ctx->decay, ctx->seed);\n    auto * result_ctx = (llama_sampler_adaptive_p *) result->ctx;\n\n    // copy everything (target, decay, seed, and RNG are already set)\n    result_ctx->weighted_sum      = ctx->weighted_sum;\n    result_ctx->total_weight      = ctx->total_weight;\n    result_ctx->pending_token_id  = ctx->pending_token_id;\n    result_ctx->pending_token_idx = ctx->pending_token_idx;\n\n    return result;\n}\n\nstatic void llama_sampler_adaptive_p_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_adaptive_p *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_adaptive_p_i = {\n    /* .name              = */ llama_sampler_adaptive_p_name,\n    /* .accept            = */ llama_sampler_adaptive_p_accept,\n    /* .apply             = */ llama_sampler_adaptive_p_apply,\n    /* .reset             = */ llama_sampler_adaptive_p_reset,\n    /* .clone             = */ llama_sampler_adaptive_p_clone,\n    /* .free              = */ llama_sampler_adaptive_p_free,\n    /* .backend_init      = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_adaptive_p(\n    float    target,\n    float    decay,\n    uint32_t seed\n) {\n    auto seed_cur = get_rng_seed(seed);\n    float clamped_decay = std::clamp(decay, 0.0f, 0.99f);\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_adaptive_p_i,\n        /* .ctx   = */ new llama_sampler_adaptive_p {\n            /* .target            = */ target,\n            /* .decay             = */ clamped_decay,\n            /* .seed              = */ seed,\n            /* .seed_cur          = */ seed_cur,\n            /* .rng               = */ std::mt19937(seed_cur),\n            /* .weighted_sum      = */ target / (1.0f - clamped_decay),\n            /* .total_weight      = */ 1.0f / (1.0f - clamped_decay),\n            /* .original_probs    = */ {},\n            /* .pending_token_id  = */ LLAMA_TOKEN_NULL,\n            /* .pending_token_idx = */ -1\n        }\n    );\n}\n\n// logit-bias\n\nstruct llama_sampler_logit_bias : public llama_sampler_backend {\n    const int32_t n_vocab;\n\n    const std::vector<llama_logit_bias> logit_bias;\n\n    std::vector<llama_logit_bias> to_search;\n\n    struct ggml_tensor * inp_logit_bias;\n    struct ggml_tensor * inp_logit_idxs;\n};\n\nstatic const char * llama_sampler_logit_bias_name(const struct llama_sampler * smpl) {\n    auto * ctx = (llama_sampler_logit_bias *) smpl->ctx;\n    return ctx->get_name();\n}\n\nstatic void llama_sampler_logit_bias_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_logit_bias *) smpl->ctx;\n\n    if (ctx->logit_bias.empty()) {\n        return;\n    }\n\n    ctx->to_search.clear();\n\n    // update the candidates that have not been shuffled in the vocabulary (i.e. idx == id)\n    for (const auto & lb : ctx->logit_bias) {\n        if (lb.token >= 0 && cur_p->size > (size_t) lb.token && cur_p->data[lb.token].id == lb.token) {\n            cur_p->data[lb.token].logit += lb.bias;\n        } else {\n            ctx->to_search.push_back(lb);\n        }\n    }\n\n    if (ctx->to_search.empty()) {\n        return;\n    }\n\n    // search for the remaining candidates that were not found in the previous step\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        for (const auto & lb : ctx->to_search) {\n            if (cur_p->data[i].id == lb.token) {\n                cur_p->data[i].logit += lb.bias;\n                break;\n            }\n        }\n    }\n}\n\nstatic struct llama_sampler * llama_sampler_logit_bias_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_logit_bias *) smpl->ctx;\n    return llama_sampler_init_logit_bias(ctx->n_vocab, ctx->logit_bias.size(), ctx->logit_bias.data());\n}\n\nstatic void llama_sampler_logit_bias_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_logit_bias *) smpl->ctx;\n}\n\nstatic void llama_sampler_logit_bias_backend_apply(\n        struct llama_sampler      * smpl,\n        struct ggml_context       * ctx,\n        struct ggml_cgraph        * gf,\n        struct llama_sampler_data * data) {\n    GGML_UNUSED(gf);\n    GGML_UNUSED(ctx);\n\n    auto * sctx = (llama_sampler_logit_bias *) smpl->ctx;\n    if (sctx->logit_bias.empty()) {\n        return;\n    }\n\n    const size_t n = sctx->logit_bias.size();\n\n    sctx->inp_logit_bias = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, n);\n    ggml_set_name(sctx->inp_logit_bias, \"logit_bias\");\n    ggml_set_input(sctx->inp_logit_bias);\n\n    sctx->inp_logit_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n);\n    ggml_set_name(sctx->inp_logit_idxs, \"logit_idxs\");\n    ggml_set_input(sctx->inp_logit_idxs);\n\n    ggml_tensor * cur = ggml_fill(ctx, data->logits, 0.0f);\n\n    cur = ggml_reshape_2d(ctx, cur, 1, ggml_nelements(cur));\n    cur = ggml_set_rows(ctx, cur, sctx->inp_logit_bias, sctx->inp_logit_idxs);\n    cur = ggml_reshape_1d(ctx, cur, ggml_nelements(cur));\n\n    data->logits = ggml_add(ctx, data->logits, cur);\n}\n\nstatic void llama_sampler_logit_bias_backend_set_input(struct llama_sampler * smpl) {\n    auto * sctx = (llama_sampler_logit_bias *) smpl->ctx;\n    if (sctx->logit_bias.empty()) {\n        return;\n    }\n\n    GGML_ASSERT(sctx->inp_logit_bias != nullptr);\n    GGML_ASSERT(sctx->inp_logit_idxs != nullptr);\n\n    const size_t n = sctx->logit_bias.size();\n\n    std::vector<float>   data_logit_bias(n, 0.0f);\n    std::vector<int32_t> data_logit_idxs(n, 0);\n    for (size_t i = 0; i < n; ++i) {\n        const auto & lb = sctx->logit_bias[i];\n        GGML_ASSERT(lb.token >= 0 && lb.token < (int32_t) sctx->n_vocab);\n        data_logit_bias[i] = lb.bias;\n        data_logit_idxs[i] = lb.token;\n    }\n\n    ggml_backend_tensor_set(sctx->inp_logit_bias, data_logit_bias.data(), 0, ggml_nbytes(sctx->inp_logit_bias));\n    ggml_backend_tensor_set(sctx->inp_logit_idxs, data_logit_idxs.data(), 0, ggml_nbytes(sctx->inp_logit_idxs));\n}\n\nstatic bool llama_sampler_logit_bias_backend_init(\n        struct llama_sampler       * smpl,\n        ggml_backend_buffer_type_t   buft) {\n    GGML_UNUSED(buft);\n\n    auto * sctx = (llama_sampler_logit_bias *) smpl->ctx;\n\n    sctx->init(true);\n\n    if (sctx->logit_bias.empty()) {\n        return true;\n    }\n\n    return true;\n}\n\nstatic struct llama_sampler_i llama_sampler_logit_bias_i = {\n    /* .name              = */ llama_sampler_logit_bias_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_logit_bias_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_logit_bias_clone,\n    /* .free              = */ llama_sampler_logit_bias_free,\n    /* .backend_init      = */ llama_sampler_logit_bias_backend_init,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_apply     = */ llama_sampler_logit_bias_backend_apply,\n    /* .backend_set_input = */ llama_sampler_logit_bias_backend_set_input,\n};\n\nstruct llama_sampler * llama_sampler_init_logit_bias(\n                         int32_t   n_vocab,\n                         int32_t   n_logit_bias,\n          const llama_logit_bias * logit_bias) {\n    const bool is_empty = n_logit_bias <= 0;\n\n    if (is_empty) {\n        return llama_sampler_init_empty(\"?logit-bias\");\n    }\n\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_logit_bias_i,\n        /* .ctx   = */ new llama_sampler_logit_bias {\n            (\"logit-bias\"),\n            /* .n_vocab        = */ n_vocab,\n            /* .logit_bias     = */ std::vector<llama_logit_bias>(logit_bias, logit_bias + n_logit_bias),\n            /* .to_search      = */ {},\n            /* .inp_logit_bias = */ nullptr,\n            /* .inp_logit_idxs = */ nullptr,\n        }\n    );\n}\n\n// infill\n\n//#define GGML_DEBUG_SAMPLER_INFILL\n\nstruct llama_sampler_infill {\n    const struct llama_vocab * vocab;\n\n    std::vector<char> buf0;\n    std::vector<char> buf1;\n};\n\nstatic const char * llama_sampler_infill_name(const struct llama_sampler * /*smpl*/) {\n    return \"infill\";\n}\n\nstatic void llama_sampler_infill_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {\n    auto * ctx = (llama_sampler_infill *) smpl->ctx;\n\n    llama_sampler_softmax_impl(cur_p, true);\n\n#if defined(GGML_DEBUG_SAMPLER_INFILL)\n#define LOG_DBG_CUR LLAMA_LOG_DEBUG\n#else\n#define LOG_DBG_CUR(...)\n#endif\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        LOG_DBG_CUR(\"%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\\n\", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);\n    }\n\n    float p_txt_sum = 0.0f;\n    float p_eog_sum = 0.0f;\n\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        if (ctx->vocab->is_eog(cur_p->data[i].id)) {\n            p_eog_sum += cur_p->data[i].p;\n        } else {\n            p_txt_sum += cur_p->data[i].p;\n        }\n    }\n\n    const float rat = p_eog_sum == 0.0 ? INFINITY : p_txt_sum / p_eog_sum; GGML_UNUSED(rat);\n\n    LOG_DBG_CUR(\"%s: p_txt_sum = %.2f, p_eog_sum = %.2f, rat = %.2f, n = %zu\\n\", __func__, p_txt_sum, p_eog_sum, rat, cur_p->size);\n\n    if (3*p_eog_sum*cur_p->size > p_txt_sum) {\n        LOG_DBG_CUR(\"%s: the ratio p_txt/p_eog = %.2f is too low -> sampling EOG\\n\", __func__, p_txt_sum/p_eog_sum);\n\n        // keep just the EOG tokens\n        const auto size_org = cur_p->size;\n\n        cur_p->size = 0;\n\n        float p_sum = 0.0f;\n\n        for (size_t i = 0; i < size_org; ++i) {\n            if (ctx->vocab->is_eog(cur_p->data[i].id)) {\n                p_sum += cur_p->data[i].p;\n\n                cur_p->data[cur_p->size++] = cur_p->data[i];\n            }\n        }\n\n        // normalize probs\n        for (size_t i = 0; i < cur_p->size; ++i) {\n            cur_p->data[i].p /= p_sum;\n        }\n\n        return;\n    }\n\n    size_t n_combined = 0; GGML_UNUSED(n_combined);\n\n    // combine tokens with common prefix\n    for (size_t i0 = 0; i0 < cur_p->size; ++i0) {\n        for (size_t i1 = 0; i1 < cur_p->size; ++i1) {\n            if (cur_p->data[i0].logit == -INFINITY) {\n                break;\n            }\n\n            if (i0 == i1 || cur_p->data[i1].logit == -INFINITY) {\n                continue;\n            }\n\n            int len0 = ctx->vocab->token_to_piece(cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);\n            if (len0 < 0) {\n                ctx->buf0.resize(len0);\n                len0 = ctx->vocab->token_to_piece(cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);\n                assert(len0 > 0);\n            }\n\n            int len1 = ctx->vocab->token_to_piece(cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);\n            if (len1 < 0) {\n                ctx->buf1.resize(len1);\n                len1 = ctx->vocab->token_to_piece(cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);\n                assert(len1 > 0);\n            }\n\n            // token i0 is a prefix of token i1\n            if (len0 > 0 && len0 <= len1 && memcmp(ctx->buf0.data(), ctx->buf1.data(), len0) == 0) {\n                int dst = i0;\n                int src = i1;\n\n                // merge into the token with higher probability\n                if (cur_p->data[i1].p > cur_p->data[i0].p) {\n                    std::swap(dst, src);\n                }\n\n                cur_p->data[dst].p += cur_p->data[src].p;\n                cur_p->data[src].logit = -INFINITY;\n                cur_p->data[src].p     = 0.0f;\n\n                n_combined++;\n            }\n        }\n    }\n\n    size_t n_non_eog = 0;\n\n    size_t size_org = cur_p->size;\n\n    float p_sum = 0.0f;\n    float thold = 0.2f;\n\n    cur_p->size = 0;\n\n    LOG_DBG_CUR(\"%s: n_combined = %zu, applying thold = %.3f\\n\", __func__, n_combined, thold);\n\n    for (size_t i = 0; i < size_org; ++i) {\n        const bool is_eog = ctx->vocab->is_eog(cur_p->data[i].id);\n\n        if (cur_p->data[i].p < thold && !is_eog) {\n            continue;\n        }\n\n        if (!is_eog) {\n            ++n_non_eog;\n        }\n\n        p_sum += cur_p->data[i].p;\n\n        // keep this token\n        cur_p->data[cur_p->size++] = cur_p->data[i];\n    }\n\n    LOG_DBG_CUR(\"%s: n_non_eog = %zu\\n\", __func__, n_non_eog);\n\n    // if no non-EOG tokens are left -> reduce cur_p to single EOT token\n    if (n_non_eog == 0) {\n        cur_p->size = 1;\n        cur_p->data[0].id = ctx->vocab->token_eot();\n        if (cur_p->data[0].id == LLAMA_TOKEN_NULL) {\n            cur_p->data[0].id = ctx->vocab->token_eos();\n        }\n        cur_p->data[0].logit = 1.0f;\n\n        GGML_ASSERT(cur_p->data[0].id != LLAMA_TOKEN_NULL);\n\n        return;\n    }\n\n    // normalize probs\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cur_p->data[i].p /= p_sum;\n\n        LOG_DBG_CUR(\"%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\\n\", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);\n    }\n\n    size_org = cur_p->size;\n    p_sum = 0.0f;\n    thold = 1.0/(n_non_eog + 1);\n\n    cur_p->size = 0;\n\n    LOG_DBG_CUR(\"%s: applying thold = %.3f\\n\", __func__, thold);\n\n    for (size_t i = 0; i < size_org; ++i) {\n        const bool is_eog = ctx->vocab->is_eog(cur_p->data[i].id);\n\n        if (cur_p->data[i].p < thold && !is_eog) {\n            continue;\n        }\n\n        p_sum += cur_p->data[i].p;\n\n        cur_p->data[cur_p->size++] = cur_p->data[i];\n    }\n\n    // normalize probs\n    for (size_t i = 0; i < cur_p->size; ++i) {\n        cur_p->data[i].p /= p_sum;\n\n        LOG_DBG_CUR(\"%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\\n\", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);\n    }\n\n#undef LOG_DBG_CUR\n}\n\nstatic struct llama_sampler * llama_sampler_infill_clone(const struct llama_sampler * smpl) {\n    const auto * ctx = (const llama_sampler_infill *) smpl->ctx;\n    return llama_sampler_init_infill(ctx->vocab);\n}\n\nstatic void llama_sampler_infill_free(struct llama_sampler * smpl) {\n    delete (llama_sampler_infill *) smpl->ctx;\n}\n\nstatic struct llama_sampler_i llama_sampler_infill_i = {\n    /* .name              = */ llama_sampler_infill_name,\n    /* .accept            = */ nullptr,\n    /* .apply             = */ llama_sampler_infill_apply,\n    /* .reset             = */ nullptr,\n    /* .clone             = */ llama_sampler_infill_clone,\n    /* .free              = */ llama_sampler_infill_free,\n    /* .backend_apply     = */ nullptr,\n    /* .backend_accept    = */ nullptr,\n    /* .backend_set_input = */ nullptr,\n    /* .backend_init      = */ nullptr,\n};\n\nstruct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab) {\n    return llama_sampler_init(\n        /* .iface = */ &llama_sampler_infill_i,\n        /* .ctx   = */ new llama_sampler_infill {\n            /* .vocab = */ vocab,\n            /* .buf0  = */ std::vector<char>(512),\n            /* .buf1  = */ std::vector<char>(512),\n        }\n    );\n}\n\n// utils\n\nuint32_t llama_sampler_get_seed(const struct llama_sampler * smpl) {\n    if (smpl->iface == &llama_sampler_dist_i) {\n        return ((const llama_sampler_dist *) smpl->ctx)->seed_cur;\n    }\n\n    if (smpl->iface == &llama_sampler_mirostat_i) {\n        return ((const llama_sampler_mirostat *) smpl->ctx)->seed_cur;\n    }\n\n    if (smpl->iface == &llama_sampler_mirostat_v2_i) {\n        return ((const llama_sampler_mirostat_v2 *) smpl->ctx)->seed_cur;\n    }\n\n    if (smpl->iface == &llama_sampler_chain_i) {\n        const auto * ctx = (const llama_sampler_chain *) smpl->ctx;\n        for (auto it = ctx->samplers.rbegin(); it != ctx->samplers.rend(); ++it) {\n            const uint32_t seed = llama_sampler_get_seed(it->ptr);\n            if (seed != LLAMA_DEFAULT_SEED) {\n                return seed;\n            }\n        }\n    }\n\n    return LLAMA_DEFAULT_SEED;\n}\n\n// perf\n\nstruct llama_perf_sampler_data llama_perf_sampler(const struct llama_sampler * chain) {\n    struct llama_perf_sampler_data data = {};\n\n    if (chain == nullptr || chain->iface != &llama_sampler_chain_i) {\n        GGML_ABORT(\"%s: invalid sampler passed - requires a sampler created with llama_sampler_chain_init()\\n\", __func__);\n    }\n\n    const auto * ctx = (const struct llama_sampler_chain *) chain->ctx;\n\n    data.t_sample_ms = 1e-3 * ctx->t_sample_us;\n    data.n_sample    = std::max(0, ctx->n_sample);\n\n    return data;\n}\n\nvoid llama_perf_sampler_print(const struct llama_sampler * chain) {\n    const auto data = llama_perf_sampler(chain);\n\n    LLAMA_LOG_INFO(\"%s:    samplers time = %10.2f ms / %5d runs\\n\", __func__, data.t_sample_ms, data.n_sample);\n}\n\nvoid llama_perf_sampler_reset(struct llama_sampler * chain) {\n    if (chain == nullptr || chain->iface != &llama_sampler_chain_i) {\n        GGML_ABORT(\"%s: invalid sampler passed - requires a sampler created with llama_sampler_chain_init()\\n\", __func__);\n    }\n\n    auto * ctx = (struct llama_sampler_chain *) chain->ctx;\n\n    ctx->t_sample_us = 0;\n    ctx->n_sample    = 0;\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-sampler.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <vector>\n\nstruct llama_vocab;\nstruct llama_grammar;\n\n// sampler chain\n\nstruct llama_sampler_chain {\n    llama_sampler_chain_params params;\n\n    // has .backend_init() been called?\n    bool is_init = false;\n\n    struct info {\n        bool is_backend;\n\n        llama_sampler * ptr;\n    };\n\n    std::vector<info> samplers;\n\n    // pre-allocated buffer for llama_sampler_sample to avoid repeated allocations\n    std::vector<llama_token_data> cur;\n\n    // timing\n\n    mutable int64_t t_sample_us;\n\n    mutable int32_t n_sample;\n};\n\nstruct llama_sampler * llama_sampler_init_dry_testing(\n        int32_t context_size,\n        float   dry_multiplier,\n        float   dry_base,\n        int32_t dry_allowed_length,\n        int32_t dry_penalty_last_n,\n        const std::vector<std::vector<llama_token>> & seq_breakers);\n"
  },
  {
    "path": "examples/talk-llama/llama-vocab.cpp",
    "content": "#include \"llama-vocab.h\"\n\n#include \"ggml.h\"\n#include \"gguf.h\"\n#include \"llama-impl.h\"\n#include \"llama-model-loader.h\"\n\n#include \"unicode.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cctype>\n#include <cfloat>\n#include <cmath>\n#include <cstdarg>\n#include <cstring>\n#include <forward_list>\n#include <limits>\n#include <map>\n#include <queue>\n#include <set>\n#include <unordered_map>\n\n//\n// helpers\n//\n\nstruct naive_trie {\n    naive_trie() : has_value(false), value(0) {\n    }\n    void insert(const char * key, size_t len, int32_t value = 0) {\n        if (len == 0) {\n            this->has_value = true;\n            this->value = value;\n            return;\n        }\n        char c = key[0];\n        auto res = children.find(c);\n        if (res != children.end()) {\n            res->second.insert(key + 1, len - 1, value);\n        } else {\n            auto res = children.insert(std::make_pair(c, naive_trie()));\n            res.first->second.insert(key + 1, len - 1, value);\n        }\n    }\n    std::pair<const char *, size_t> get_longest_prefix(const char * key, size_t len, size_t offset = 0) const {\n        if (len == 0 || offset == len) {\n            return std::make_pair(key, offset);\n        }\n        char c = key[offset];\n        auto res = children.find(c);\n        if (res != children.end()) {\n            return res->second.get_longest_prefix(key, len, offset + 1);\n        }\n\n        return std::make_pair(key, offset);\n    }\n    const struct naive_trie * traverse(const char c) const {\n        auto res = children.find(c);\n        if (res != children.end()) {\n            return &res->second;\n        }\n\n        return NULL;\n    }\n    std::map<char, struct naive_trie> children;\n    bool has_value;\n    llama_token value;\n};\n\n//\n// tokenizers\n//\n\nstruct llm_tokenizer {\n    llm_tokenizer() {}\n    virtual ~llm_tokenizer() = default;\n};\n\nstruct llm_symbol {\n    using index = int;\n    index prev;\n    index next;\n    const char * text;\n    size_t n;\n};\n\nstatic_assert(std::is_trivially_copyable<llm_symbol>::value, \"llm_symbol is not trivially copyable\");\n\n//\n// SPM tokenizer\n// original implementation:\n// https://github.com/ggml-org/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4\n//\n\nstruct llm_bigram_spm {\n    struct comparator {\n        bool operator()(llm_bigram_spm & l, llm_bigram_spm & r) {\n            return (l.score < r.score) || (l.score == r.score && l.left > r.left);\n        }\n    };\n    using queue_storage = std::vector<llm_bigram_spm>;\n    using queue = std::priority_queue<llm_bigram_spm, queue_storage, comparator>;\n    llm_symbol::index left;\n    llm_symbol::index right;\n    float score;\n    size_t size;\n};\n\nstruct llm_tokenizer_spm : llm_tokenizer {\n    llm_tokenizer_spm(const llama_vocab & /*vocab*/) {}\n};\n\nstruct llm_tokenizer_spm_session {\n    llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab) {}\n\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        // split string into utf8 chars\n        int index = 0;\n        size_t offs = 0;\n        while (offs < text.size()) {\n            llm_symbol sym;\n            size_t len = unicode_len_utf8(text[offs]);\n            sym.text = text.c_str() + offs;\n            sym.n = std::min(len, text.size() - offs);\n            offs += sym.n;\n            sym.prev = index - 1;\n            sym.next = offs == text.size() ? -1 : index + 1;\n            index++;\n            symbols.emplace_back(sym);\n        }\n\n        // seed the work queue with all possible 2-character tokens.\n        for (int i = 1; i < (int) symbols.size(); ++i) {\n            try_add_bigram(i - 1, i);\n        }\n\n        // keep substituting the highest frequency pairs for as long as we can.\n        while (!work_queue.empty()) {\n            auto bigram = work_queue.top();\n            work_queue.pop();\n\n            auto & left_sym = symbols[bigram.left];\n            auto & right_sym = symbols[bigram.right];\n\n            // if one of the symbols already got merged, skip it.\n            if (left_sym.n == 0 || right_sym.n == 0 ||\n                left_sym.n + right_sym.n != bigram.size) {\n                continue;\n            }\n\n            // merge the right sym into the left one\n            left_sym.n += right_sym.n;\n            right_sym.n = 0;\n\n            //LLAMA_LOG_INFO(\"left = '%*s' size = %zu\\n\", (int) left_sym.n, left_sym.text, bigram.size);\n\n            // remove the right sym from the chain\n            left_sym.next = right_sym.next;\n            if (right_sym.next >= 0) {\n                symbols[right_sym.next].prev = bigram.left;\n            }\n\n            // find more substitutions\n            try_add_bigram(left_sym.prev, bigram.left);\n            try_add_bigram(bigram.left, left_sym.next);\n        }\n\n        for (int i = 0; i != -1; i = symbols[i].next) {\n            auto & symbol = symbols[i];\n            resegment(symbol, output);\n        }\n    }\n\nprivate:\n    void resegment(llm_symbol & symbol, std::vector<llama_token> & output) {\n        auto text = std::string(symbol.text, symbol.n);\n        auto token = vocab.text_to_token(text);\n\n        // Do we need to support is_unused?\n        if (token != LLAMA_TOKEN_NULL) {\n            output.push_back(token);\n            return;\n        }\n\n        const auto p = rev_merge.find(text);\n\n        if (p == rev_merge.end()) {\n            // output any symbols that did not form tokens as bytes.\n            output.reserve(output.size() + symbol.n);\n            for (int j = 0; j < (int)symbol.n; ++j) {\n                llama_token id = vocab.byte_to_token(symbol.text[j]);\n                output.push_back(id);\n            }\n            return;\n        }\n\n        resegment(symbols[p->second.first], output);\n        resegment(symbols[p->second.second], output);\n    }\n\n    void try_add_bigram(int left, int right) {\n        if (left == -1 || right == -1) {\n            return;\n        }\n        const std::string text = std::string(symbols[left].text, symbols[left].n + symbols[right].n);\n        auto token = vocab.text_to_token(text);\n\n        if (token == LLAMA_TOKEN_NULL) {\n            return;\n        }\n\n        if (static_cast<uint32_t>(token) >= vocab.n_tokens()) {\n            return;\n        }\n\n        const auto & tok_data = vocab.get_token_data(token);\n\n        llm_bigram_spm bigram;\n        bigram.left  = left;\n        bigram.right = right;\n        bigram.score = tok_data.score;\n        bigram.size  = text.size();\n\n        work_queue.push(bigram);\n\n        // Do we need to support is_unused?\n        rev_merge[text] = std::make_pair(left, right);\n    }\n\n    const llama_vocab & vocab;\n    // currently unused\n    // const llm_tokenizer_spm * spm_tokenizer;\n\n    std::vector<llm_symbol> symbols;\n    llm_bigram_spm::queue work_queue;\n    std::map<std::string, std::pair<int, int>> rev_merge;\n};\n\n//\n// BPE tokenizer\n// adapted from https://github.com/cmp-nct/ggllm.cpp [MIT License]\n// tried to simplify unicode stuff, so most likely does not work 100% correctly!\n//\n\n// TODO: there are a lot of common parts between spm and bpe tokenizers, should be refactored and reused\n\ntemplate<typename T, typename Container = std::vector<T>, typename Compare = std::less<typename Container::value_type>>\nclass llama_priority_queue : public std::priority_queue<T, Container, Compare> {\npublic:\n    using std::priority_queue<T, Container, Compare>::priority_queue;\n\n    T pop_move() {\n        T item = std::move(this->c.front());\n        std::pop_heap(this->c.begin(), this->c.end(), this->comp);\n        this->c.pop_back();\n        return item;\n    }\n\n    void pop() =  delete;\n};\n\nstruct llm_bigram_bpe {\n    struct comparator {\n        bool operator()(const llm_bigram_bpe & l, const llm_bigram_bpe & r) const {\n            return l.rank > r.rank || (l.rank == r.rank && l.left > r.left);\n        }\n    };\n\n    using queue_storage = std::vector<llm_bigram_bpe>;\n    using queue = llama_priority_queue<llm_bigram_bpe, queue_storage, comparator>;\n    llm_symbol::index left;\n    llm_symbol::index right;\n    std::string text;\n    int rank;\n    size_t size;\n};\n\nstruct llm_tokenizer_bpe : llm_tokenizer {\n    llm_tokenizer_bpe(const llama_vocab & vocab) {\n        GGML_ASSERT(vocab.get_type() == LLAMA_VOCAB_TYPE_BPE);\n        switch (vocab.get_pre_type()) {\n            case LLAMA_VOCAB_PRE_TYPE_LLAMA3:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    //\"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n\n                    // adapted: https://github.com/ggml-org/llama.cpp/pull/6920#issuecomment-2080233989\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_JAIS2:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    //\"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s{512}(?!\\\\S)|\\\\s{256}(?!\\\\S)|\\\\s{128}(?!\\\\S)|\\\\s{64}(?!\\\\S)|\\\\s{32}(?!\\\\S)|\\\\s{16}(?!\\\\S)|\\\\s{8}(?!\\\\S)|\\\\s{4}(?!\\\\S)|\\\\s{1,2}(?!\\\\S)|\\\\s{1}\",\n\n                    // adapted: same as llama3 but with cascading whitespace pattern\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s{512}(?!\\\\S)|\\\\s{256}(?!\\\\S)|\\\\s{128}(?!\\\\S)|\\\\s{64}(?!\\\\S)|\\\\s{32}(?!\\\\S)|\\\\s{16}(?!\\\\S)|\\\\s{8}(?!\\\\S)|\\\\s{4}(?!\\\\S)|\\\\s{1,2}(?!\\\\S)|\\\\s{1}\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_DBRX:\n            case LLAMA_VOCAB_PRE_TYPE_SMAUG:\n                regex_exprs = {\n                    // same as llama3\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:\n                regex_exprs = {\n                    \"[\\r\\n]\",\n                    \"\\\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿǄ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿﬀ-ﬆﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+\",\n                    \"\\\\s?[!-/:-~！-／：-～‘-‟　-。]+\",\n                    \"\\\\s+$\",\n                    \"[一-龥ࠀ-一가-퟿]+\",\n                    \"\\\\p{N}+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM:\n            case LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE:\n            case LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM:\n                regex_exprs = {\n                    \"\\\\p{N}{1,3}\",\n                    \"[一-龥぀-ゟ゠-ヿ]+\",\n                    \"[!\\\"#$%&'()*+,\\\\-./:;<=>?@\\\\[\\\\\\\\\\\\]^_`{|}~][A-Za-z]+|[^\\r\\n\\\\p{L}\\\\p{P}\\\\p{S}]?[\\\\p{L}\\\\p{M}]+| ?[\\\\p{P}\\\\p{S}]+[\\r\\n]*|\\\\s*[\\r\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_YOUTU:\n                regex_exprs = {\n                    \"[가-힣ㄱ-ㆎ]+|[！…“”‘’—：；，、-〿︰-﹏]+|[ㄅ-ㄯ]+|[一-龥぀-ゟ゠-ヿ]+\",\n                    \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:\n                regex_exprs = {\n                    \"[\\r\\n]\",\n                    \"\\\\s?\\\\p{L}+\",\n                    \"\\\\s?\\\\p{P}+\",\n                    \"[一-龥ࠀ-一가-퟿]+\",\n                    \"\\\\p{N}\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_FALCON:\n                regex_exprs = {\n                    \"[\\\\p{P}\\\\$\\\\+<=>\\\\^~\\\\|`]+\",\n                    \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\",\n                    \"[0-9][0-9][0-9]\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_STARCODER:\n            case LLAMA_VOCAB_PRE_TYPE_REFACT:\n            case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:\n            case LLAMA_VOCAB_PRE_TYPE_SMOLLM:\n            case LLAMA_VOCAB_PRE_TYPE_CODESHELL:\n            case LLAMA_VOCAB_PRE_TYPE_EXAONE:\n            case LLAMA_VOCAB_PRE_TYPE_MINERVA:\n                regex_exprs = {\n                    \"\\\\p{N}\",\n                    \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_GPT2:\n            case LLAMA_VOCAB_PRE_TYPE_MPT:\n            case LLAMA_VOCAB_PRE_TYPE_OLMO:\n            case LLAMA_VOCAB_PRE_TYPE_JAIS:\n            case LLAMA_VOCAB_PRE_TYPE_TRILLION:\n            case LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING:\n                regex_exprs = {\n                    \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_STABLELM2:\n            case LLAMA_VOCAB_PRE_TYPE_QWEN2:\n            case LLAMA_VOCAB_PRE_TYPE_HUNYUAN:\n            case LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_QWEN35:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{L}\\\\p{M}]+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{M}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{L}\\\\p{M}]+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{M}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_PORO:\n            case LLAMA_VOCAB_PRE_TYPE_BLOOM:\n            case LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH:\n                regex_exprs = {\n                    \" ?[^(\\\\s|.,!?…。，、।۔،)]+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_CHATGLM4:\n                regex_exprs = {\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_VIKING:\n                regex_exprs = {\n                    \" ?[^(\\\\s|.,!?…。，、।۔،)]+\",\n                    \"\\\\p{N}\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_TEKKEN:\n                // original regex from tokenizer.json\n                // \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                regex_exprs = {\n                    \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?((?=[\\\\p{L}])([^a-z]))*((?=[\\\\p{L}])([^A-Z]))+|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?((?=[\\\\p{L}])([^a-z]))+((?=[\\\\p{L}])([^A-Z]))*|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_CHAMELEON:\n                // Note: in theory, the special token (sentinel and image token) regex_exprs below\n                // are unnecessary, as they are split in `tokenizer_st_partition` anyway.\n                // However, since the upstream pre-tokenizer uses them, they are also\n                // included here (see https://huggingface.co/facebook/chameleon-7b).\n                regex_exprs = {\n                    \"<sentinel:[0-9]+>\",  // Sentinel tokens\n                    \"(IMGIMG)((A|B|C|D|E|F|G|H|I){1,4})Z\",  // Image tokens\n                    \"([\\\\t\\\\n]|    |  )\",  // directly from tokenizer.json\n                    \"\\\\p{N}\", // Individual digits\n                    \"[\\\\p{P}!-/:-@\\\\[-`{-~]\",  // Punctuation, Isolated\n                    \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_GPT4O:\n            case LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                    \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?((?=[\\\\p{L}])([^a-z]))*((?=[\\\\p{L}])([^A-Z]))+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?((?=[\\\\p{L}])([^a-z]))+((?=[\\\\p{L}])([^A-Z]))*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_TINY_AYA:\n                regex_exprs = {\n                    // original regex from tokenizer.json: \"\\\\d{1,3}(?=(?:\\\\d{3})*\\\\b)\"\n                    \"\\\\d{1,3}(?=(?:\\\\d{3})*\\\\b)\",\n                    // original regex from tokenizer.json: \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?[\\\\p{Lu}\\\\p{Lt}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]+[\\\\p{Ll}\\\\p{Lm}\\\\p{Lo}\\\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_KIMI_K2:\n                regex_exprs = {\n                    // K2 trigger pattern - this will activate the custom K2 handler in unicode.cpp\n                    // The custom handler implements all K2 patterns with proper Han character exclusion\n                    \"\\\\p{Han}+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_SUPERBPE:\n                regex_exprs = {\n                    \"\\\\p{N}+\",\n                    \"(?=(\\\\d{3})+(?!\\\\d))\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_BAILINGMOE:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"'(?i:[sdmt]|ll|ve|re)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?+\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]++[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]|\\\\s+(?!\\\\S)|\\\\s+\"\n                    // FIXME? Changed possessive quantifiers (?+ and ++) to greedy to avoid errors and imatrix hanging (tried atomic grouping but it's not supported?)\n                    \"'(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_SEED_CODER:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1}| ?[^\\\\s\\\\p{L}\\\\p{N}\\r\\n]+|\\\\s*[\\r\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1}| ?[^\\\\s\\\\p{L}\\\\p{N}\\\\r\\\\n]+|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_GROK_2:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_AFMOE:\n                regex_exprs = {\n                    // Digit handling - uses custom implementation in unicode.cpp\n                    // Groups digits with leading 1-2 based on total length modulo 3\n                    \"\\\\p{AFMoE_digits}\",\n                    // CJK and Asian scripts (using direct Unicode literals)\n                    \"[一-鿿㐀-䶿豈-﫿぀-ゟ゠-ヿ･-ﾟ⼀-⿟เ-๿຀-໿ក-៿က-႟ꩠ-ꩿꧠ-꧿가-힯ᄀ-ᇿ]+\",\n                    // Main BPE pattern\n                    \"[!\\\"#$%&'()*+,\\\\-./:;<=>?@\\\\[\\\\\\\\\\\\]^_`{|}~][A-Za-z]+|[^\\\\r\\\\n\\\\p{L}\\\\p{P}\\\\p{S}]?[\\\\p{L}\\\\p{M}]+| ?[\\\\p{P}\\\\p{S}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            case LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE:\n                regex_exprs = {\n                    // original regex from tokenizer.json\n                    // \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?(?:\\\\p{L}\\\\p{M}*(?: \\\\p{L}\\\\p{M}*)*)+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]?|\\\\s*[\\\\r\\\\n]|\\\\s+(?!\\\\S)|\\\\s+\"\n                    \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?(?:\\\\p{L}\\\\p{M}*(?: \\\\p{L}\\\\p{M}*)*)+|\\\\p{N}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n/]?|\\\\s*[\\\\r\\\\n]|\\\\s+(?!\\\\S)|\\\\s+\",\n                };\n                break;\n            default:\n                // default regex for BPE tokenization pre-processing\n                regex_exprs = {\n                    \"[\\\\p{P}\\\\$\\\\+<=>\\\\^~\\\\|]+\",\n                    \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\",\n                    \"\\\\p{N}+\",\n                    \"[0-9][0-9][0-9]\",\n                };\n                break;\n        }\n    }\n\n    std::vector<std::string> regex_exprs;\n};\n\nstruct llm_tokenizer_bpe_session {\n    llm_tokenizer_bpe_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : vocab(vocab), tokenizer(tokenizer) {}\n\n    static void append(const llama_token token_id, std::vector<llama_token> & output)  {\n        output.push_back(token_id);\n    }\n\n    bool append_bos(std::vector<llama_token> & output) const {\n        if (vocab.get_add_bos()) {\n            GGML_ASSERT(vocab.token_bos() != LLAMA_TOKEN_NULL);\n            output.push_back(vocab.token_bos());\n            return true;\n        }\n        return false;\n    }\n\n    bool append_eos(std::vector<llama_token> & output) const {\n        if (vocab.get_add_eos()) {\n            GGML_ASSERT(vocab.token_eos() != LLAMA_TOKEN_NULL);\n            output.push_back(vocab.token_eos());\n            return true;\n        }\n        return false;\n    }\n\n    void check_double_bos_eos(const std::vector<llama_token> & output) const {\n        if (vocab.get_add_bos() && output.size() >= 2 && output[1] == vocab.token_bos()) {\n            LLAMA_LOG_WARN(\n                \"%s: Added a BOS token to the prompt as specified by the model but the prompt \"\n                \"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. \"\n                \"Are you sure this is what you want?\\n\", __FUNCTION__);\n        }\n        if (vocab.get_add_eos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {\n            LLAMA_LOG_WARN(\n                \"%s: Added a EOS token to the prompt as specified by the model but the prompt \"\n                \"also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. \"\n                \"Are you sure this is what you want?\\n\", __FUNCTION__);\n        }\n    }\n\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        int final_prev_index = -1;\n        const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs);\n\n        symbols_final.clear();\n\n        for (const auto & word : word_collection) {\n            work_queue = llm_bigram_bpe::queue();\n            symbols.clear();\n\n            int index = 0;\n            size_t offset = 0;\n\n            //if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {\n            if (vocab.get_ignore_merges() && vocab.text_to_token(word) != LLAMA_TOKEN_NULL) {\n                symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});\n                offset = word.size();\n            }\n\n            while (offset < word.size()) {\n                llm_symbol sym;\n                size_t char_len = std::min(word.size() - offset, (size_t) unicode_len_utf8(word[offset]));\n                sym.text = word.c_str() + offset;\n                sym.n = char_len;\n                offset += sym.n;\n                sym.prev = index - 1;\n                sym.next = offset == word.size() ? -1 : index + 1;\n                index++;\n                symbols.emplace_back(sym);\n            }\n            for (int i = 1; i < (int) symbols.size(); ++i) {\n                add_new_bigram(i - 1, i);\n            }\n\n            // build token(s)\n            while (!work_queue.empty()) {\n                auto bigram = work_queue.pop_move();\n\n                auto & left_symbol = symbols[bigram.left];\n                auto & right_symbol = symbols[bigram.right];\n\n                if (left_symbol.n == 0 || right_symbol.n == 0) {\n                    continue;\n                }\n                std::string left_token = std::string(left_symbol.text, left_symbol.n);\n                std::string right_token = std::string(right_symbol.text, right_symbol.n);\n                if (left_token + right_token != bigram.text) {\n                    continue;  // Skip this bigram if it's outdated\n                }\n\n                // merge the right sym into the left one\n                left_symbol.n += right_symbol.n;\n                right_symbol.n = 0;\n\n                // remove the right sym from the chain\n                left_symbol.next = right_symbol.next;\n                if (right_symbol.next >= 0) {\n                    symbols[right_symbol.next].prev = bigram.left;\n                }\n\n                add_new_bigram(left_symbol.prev, bigram.left);  // left side of current symbol\n                add_new_bigram(bigram.left, left_symbol.next);  // right side of current symbol\n            }\n\n            // add the finished tokens to the final list keeping correct order for next and prev\n            for (auto & sym : symbols) {\n                if (sym.n > 0) {\n                    sym.prev = final_prev_index;\n                    sym.next = -1;\n                    if (final_prev_index != -1) {\n                        symbols_final[final_prev_index].next = symbols_final.size();\n                    }\n                    symbols_final.emplace_back(sym);\n                    final_prev_index = symbols_final.size() - 1;\n                }\n            }\n        }\n\n        symbols = symbols_final;\n\n        if (!symbols.empty()) {\n            for (int i = 0; i != -1; i = symbols[i].next) {\n                auto & symbol = symbols[i];\n                if (symbol.n == 0) {\n                    continue;\n                }\n\n                const std::string str = std::string(symbol.text, symbol.n);\n                const auto token = vocab.text_to_token(str);\n\n                if (token == LLAMA_TOKEN_NULL) {\n                    for (auto j = str.begin(); j != str.end(); ++j) {\n                        std::string byte_str(1, *j);\n                        auto token_multibyte = vocab.text_to_token(byte_str);\n                        if (token_multibyte != LLAMA_TOKEN_NULL) {\n                            output.push_back(token_multibyte);\n                        }\n                    }\n                } else {\n                    output.push_back(token);\n                }\n            }\n        }\n    }\n\nprivate:\n    void add_new_bigram(int left, int right) {\n        if (left == -1 || right == -1) {\n            return;\n        }\n        std::string left_token  = std::string(symbols[left].text,  symbols[left].n);\n        std::string right_token = std::string(symbols[right].text, symbols[right].n);\n\n        int rank_found = -1;\n\n        rank_found = vocab.find_bpe_rank(left_token, right_token);\n\n        if (rank_found < 0) {\n            return;\n        }\n\n        llm_bigram_bpe bigram;\n\n        bigram.left  = left;\n        bigram.right = right;\n        bigram.text  = left_token + right_token;\n        bigram.size  = left_token.size() + right_token.size();\n        bigram.rank  = rank_found;\n\n        work_queue.push(bigram);\n    }\n\n    const llama_vocab & vocab;\n    const llm_tokenizer_bpe & tokenizer;\n\n    std::vector<llm_symbol> symbols;\n    std::vector<llm_symbol> symbols_final;\n    llm_bigram_bpe::queue work_queue;\n};\n\n//\n// WPM tokenizer\n//\n\nstruct llm_tokenizer_wpm : llm_tokenizer {\n    llm_tokenizer_wpm(const llama_vocab & /*vocab*/) {}\n};\n\nstruct llm_tokenizer_wpm_session {\n    llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab) {}\n\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        // normalize and split by whitespace\n        std::vector<std::string> words = preprocess(text);\n        // bos token prepended already\n\n        // find the longest tokens that form the words\n        for (const std::string & word : words) {\n            // skip empty words\n            if (word.size() == 0) {\n                continue;\n            }\n\n            // prepend phantom space\n            const std::string word1 = \"\\xe2\\x96\\x81\" + word;\n            const int n = word1.size();\n\n            const size_t current_tokens = output.size();\n\n            // we're at the start of a new word\n            // move through character position in word\n            for (int i = 0; i < n; ++i) {\n                // loop through possible match length\n                bool match = false;\n                for (int j = std::min(n, i + vocab.max_token_len() + 1); j > i; j--) {\n                    auto id = vocab.text_to_token(word1.substr(i, j - i));\n                    if (id != LLAMA_TOKEN_NULL) {\n                        output.push_back(id);\n                        match = true;\n                        i = j - 1;\n                        break;\n                    }\n                }\n\n                if (!match) { // discard all\n                    output.resize(current_tokens);\n                    break;  // and discard next tokens\n                }\n            }\n\n            // we didn't find any matches for this word\n            if (current_tokens == output.size()) {\n                output.push_back(vocab.token_unk());\n            }\n        }\n    }\n\n    // TODO: reduce string copies by using cpts_offs array\n    static std::vector<std::string> preprocess(const std::string & text)  {\n        const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));\n        std::vector<std::string> words(1, \"\");\n\n        for (const uint32_t cpt : cpts_nfd) {\n            const auto flags = unicode_cpt_flags_from_cpt(cpt);\n\n            if (flags.is_whitespace) {\n                if (words.back().size()) {  // finish previous word if any\n                    words.emplace_back();\n                }\n                continue;\n            }\n\n            assert (!flags.is_separator);\n            if (cpt == 0 || cpt == 0xFFFD || flags.is_control) {\n                continue;\n            }\n\n            const std::string s = unicode_cpt_to_utf8(unicode_tolower(cpt));\n            if (flags.is_punctuation || ( cpt < 0x7F && flags.is_symbol ) || is_chinese_char(cpt)) {\n                if (words.back().size()) {  // finish previous word if any\n                    words.emplace_back();\n                }\n                words.back() = s;       // single char word\n                words.emplace_back();   // start a new word\n            } else {\n                words.back() += s;  // append char to word\n            }\n        }\n\n        if (!words.back().size()) {\n            words.pop_back();\n        }\n\n        return words;\n    }\n\n    static bool is_chinese_char(uint32_t cpt) {\n        return\n            (cpt >= 0x04E00 && cpt <= 0x09FFF) ||\n            (cpt >= 0x03400 && cpt <= 0x04DBF) ||\n            (cpt >= 0x20000 && cpt <= 0x2A6DF) ||\n            (cpt >= 0x2A700 && cpt <= 0x2B73F) ||\n            (cpt >= 0x2B740 && cpt <= 0x2B81F) ||\n            (cpt >= 0x2B920 && cpt <= 0x2CEAF) || // this should be 0x2B820 but in hf rust code it is 0x2B920\n            (cpt >= 0x0F900 && cpt <= 0x0FAFF) ||\n            (cpt >= 0x2F800 && cpt <= 0x2FA1F);\n            //(cpt >= 0x3000  && cpt <= 0x303F)  ||\n            //(cpt >= 0xFF00  && cpt <= 0xFFEF);\n    }\n\nprivate:\n    const llama_vocab & vocab;\n    // currently unused\n    // const llm_tokenizer_wpm * wpm_tokenizer;\n};\n\n//\n// UGM tokenizer\n//\n\nstruct llm_tokenizer_ugm : llm_tokenizer {\n    llm_tokenizer_ugm(const llama_vocab & vocab, const std::vector<char> & precompiled_charsmap) {\n        if (precompiled_charsmap.size() > 0) {\n            size_t charsmap_offset = 0;\n\n            // First four bytes of precompiled_charsmap contains length of binary\n            // blob containing XOR-compressed compact double array (XCDA) entries\n            uint32_t xcda_blob_size = *(const uint32_t *) &precompiled_charsmap[0];\n            charsmap_offset += sizeof(xcda_blob_size);\n            if (xcda_blob_size + charsmap_offset >= precompiled_charsmap.size()) {\n                throw std::runtime_error(\"Index out of array bounds in precompiled charsmap!\");\n            }\n\n            // Next xcda_blob_size bytes contain entries of XOR-compressed compact\n            // double array (XCDA). Each entry is bit-packed into a 32-bit integer.\n            xcda_array = (const uint32_t *) &precompiled_charsmap[charsmap_offset];\n            xcda_array_size = xcda_blob_size / sizeof(uint32_t);\n            charsmap_offset += xcda_blob_size;\n\n            // Remaining bytes of precompiled charsmap contain null-terminated\n            // replacement strings for prefixes matched by the XCDA.\n            prefix_replacements = &precompiled_charsmap[charsmap_offset];\n            prefix_replacements_size = precompiled_charsmap.size() - charsmap_offset;\n        }\n\n        for (uint32_t id = 0; id < vocab.n_tokens(); ++id) {\n            const auto & token_data = vocab.get_token_data(id);\n\n            if (vocab.is_normal(id)) {\n                min_score = std::min<float>(min_score, token_data.score);\n                max_score = std::max<float>(max_score, token_data.score);\n            }\n\n            if (vocab.is_normal(id) ||\n                vocab.is_user_defined(id) ||\n                vocab.is_unused(id)) {\n                token_matcher.insert(token_data.text.data(), token_data.text.size(), id);\n            }\n\n            if (vocab.is_user_defined(id)) {\n                user_defined_token_matcher.insert(token_data.text.data(), token_data.text.size());\n            }\n        }\n\n        unknown_token_score = min_score - unknown_token_score_penalty;\n    }\n\n    // escaped space symbol - U+2581 (Lower One Eighth Block)\n    const std::string escaped_space = \"\\xE2\\x96\\x81\";\n\n    const char * prefix_replacements = NULL;\n    size_t prefix_replacements_size = 0;\n\n    const uint32_t * xcda_array = NULL;\n    size_t xcda_array_size = 0;\n\n    struct naive_trie user_defined_token_matcher;\n\n    float min_score = FLT_MAX;\n    float max_score = -FLT_MAX;\n\n    float unknown_token_score_penalty = 10.0;\n    float unknown_token_score;\n\n    struct naive_trie token_matcher;\n};\n\nstruct llm_tokenizer_ugm_session {\n    llm_tokenizer_ugm_session(const llama_vocab & vocab, const llm_tokenizer_ugm & tokenizer) : vocab(vocab), tokenizer(tokenizer) {}\n\n    /* This implementation is based on SentencePiece optimized Viterbi algorithm for\n     * unigram language models. The general idea is to:\n     * - move along the input sequence in steps of one UTF code point,\n     * - at each step find all possible tokenizations of the prefix by\n     *   traversing the tokens trie,\n     * - for each tokenization store the best one so far (by higher score)\n     * - use the position in sequence after given token as an index to store\n     *   results\n     * - if there was no valid tokenization of the current UTF code point\n     *   then use unknown token with additional score penalty\n     * After processing the whole sequence we backtrack from the end to get\n     * the best tokenization.\n    */\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        // get current size of output (for reversal later)\n        size_t output_size = output.size();\n\n        // normalize the input first\n        std::string normalized;\n        normalize(text, &normalized);\n        size_t input_len = normalized.size();\n        if (input_len == 0) {\n            return;\n        }\n\n        // initialize score_sum to -FLT_MAX so it will be always lower than sums of token scores\n        std::vector<struct best_tokenization> tokenization_results(input_len + 1, {vocab.token_unk(), 0, -DBL_MAX});\n        // at the beginning tokenization score is zero\n        tokenization_results[0] = { vocab.token_unk(), 0, 0 };\n\n        for (size_t input_offset = 0; input_offset < input_len;) {\n            size_t prefix_offset = input_offset;\n            // calculate how many code units are in the currently processed UTF code point\n            size_t n_utf8_code_units = std::min<size_t>(unicode_len_utf8(normalized[input_offset]), input_len - input_offset);\n\n            // traverse the token matcher trie to find a matching token\n            bool single_codepoint_token_found = false;\n            const struct best_tokenization & current_best = tokenization_results[input_offset];\n            const struct naive_trie * node = tokenizer.token_matcher.traverse(normalized[prefix_offset++]);\n\n            while (prefix_offset <= input_len && node != NULL) {\n                // check if we found valid token in prefix\n                if (node->has_value) {\n                    // check if it corresponds to the whole UTF code point\n                    if (prefix_offset - input_offset == n_utf8_code_units) {\n                        single_codepoint_token_found = true;\n                    }\n                    llama_token token_id = node->value;\n                    const auto & token_data = vocab.get_token_data(token_id);\n\n                    // we set the user-defined token scores to 0 to make them more likely to be selected\n                    // (normal token scores are log probabilities, so they are negative)\n                    // score type is double here to make tokenization results exactly\n                    // the same as in the HF tokenizer using SentencePiece\n                    const double token_score = vocab.is_user_defined(token_id) ? 0.0 : token_data.score;\n                    const double challenger_score = current_best.score_sum + token_score;\n                    struct best_tokenization & current_champ = tokenization_results[prefix_offset];\n                    if (challenger_score > current_champ.score_sum) {\n                        struct best_tokenization challenger = { token_id, input_offset, challenger_score };\n                        current_champ = challenger;\n                    }\n                }\n                node = node->traverse(normalized[prefix_offset++]);\n            }\n\n            // if we didn't find a valid token corresponding to the whole UTF code point\n            // then use unknown token as the tokenization of this UTF code point\n            if (!single_codepoint_token_found) {\n                const double challenger_score = current_best.score_sum + tokenizer.unknown_token_score;\n                prefix_offset = input_offset + n_utf8_code_units;\n                struct best_tokenization & current_champ = tokenization_results[prefix_offset];\n                if (challenger_score > current_champ.score_sum) {\n                    struct best_tokenization challenger = { vocab.token_unk(), input_offset, challenger_score };\n                    current_champ = challenger;\n                }\n            }\n\n            // move to the next UTF code point\n            input_offset += n_utf8_code_units;\n        }\n\n        // now backtrack from the end to gather token ids of the best tokenization\n        // merge sequences of consecutive unknown tokens into single unknown tokens\n        bool is_prev_unknown = false;\n        for (struct best_tokenization & tokenization = tokenization_results[input_len]; ; tokenization = tokenization_results[tokenization.input_offset]) {\n            bool is_unknown = tokenization.token_id == vocab.token_unk();\n            if (!(is_prev_unknown && is_unknown)) {\n                output.push_back(tokenization.token_id);\n            }\n            if (tokenization.input_offset == 0) {\n                break;\n            }\n            is_prev_unknown = is_unknown;\n        }\n\n        // reverse the output since we added tokens starting from the end of the input\n        std::reverse(output.begin() + output_size, output.end());\n    }\n\nprivate:\n\n    // helper structure for returning normalization results\n    struct normalization_result {\n        const char * normalized;\n        size_t normalized_len;\n        size_t consumed_input;\n    };\n\n    void normalize(const std::string& input, std::string * normalized) {\n        normalized->clear();\n        normalized->reserve(input.size() * 3);\n\n        const std::string space = vocab.get_escape_whitespaces() ? tokenizer.escaped_space : \" \";\n\n        const bool shall_prepend_space = !vocab.get_treat_whitespace_as_suffix() && vocab.get_add_space_prefix();\n        const bool shall_append_space  =  vocab.get_treat_whitespace_as_suffix() && vocab.get_add_space_prefix();\n        const bool shall_merge_spaces  =  vocab.get_remove_extra_whitespaces();\n\n        bool is_space_prepended = false;\n        bool processing_non_ws = false;\n\n        size_t input_len = input.size();\n\n        for (size_t input_offset = 0; input_offset < input_len; ) {\n            auto norm_res = normalize_prefix(input, input_offset);\n            for (size_t i = 0; i < norm_res.normalized_len; i++) {\n                char c = norm_res.normalized[i];\n                if (c != ' ') {\n                    if (!processing_non_ws) {\n                        processing_non_ws = true;\n                        if ((shall_prepend_space && !is_space_prepended) || shall_merge_spaces) {\n                            normalized->append(space);\n                            is_space_prepended = true;\n                        }\n                    }\n                    normalized->push_back(c);\n                } else {\n                    if (processing_non_ws) {\n                        processing_non_ws = false;\n                    }\n                    if (!shall_merge_spaces) {\n                        normalized->append(space);\n                    }\n                }\n            }\n\n            input_offset += norm_res.consumed_input;\n        }\n\n        if (shall_append_space) {\n            normalized->append(space);\n        }\n    }\n\n    /*\n     * This structure is a view wrapper for XOR-compressed double array (XCDA)\n     * See Shunsuke Kanda (2018). Space- and Time-Efficient String Dictionaries.\n     * Each bit-packed entry contains:\n     * - BASE array value in bits 10-30\n     * - LCHECK array value in bits 0-7\n     * - LEAF array value in bit 9\n     * Entries containing indexes of replacement sequences have set bit 31\n     */\n    struct xcda_array_view {\n    public:\n        xcda_array_view(const uint32_t * xcda_array, size_t xcda_array_size) : xcda_array(xcda_array), xcda_array_size(xcda_array_size) {\n        }\n        uint32_t get_base(size_t index) {\n            uint32_t packed_node = get_node(index);\n            return (packed_node >> 10) << ((packed_node & (1U << 9)) >> 6);\n        }\n        uint32_t get_lcheck(size_t index) {\n            uint32_t packed_node = get_node(index);\n            return packed_node & ((1U << 31) | 0xff);\n        }\n        bool get_leaf(size_t index) {\n            uint32_t packed_node = get_node(index);\n            return (packed_node >> 8) & 1;\n        }\n        uint32_t get_value(size_t index) {\n            uint32_t packed_node = get_node(index);\n            return packed_node & ((1U << 31) - 1);\n        }\n    private:\n        uint32_t get_node(size_t index) {\n            if (index >= xcda_array_size) {\n                throw std::runtime_error(\"Index out of array bounds in XCDA array!\");\n            }\n            return xcda_array[index];\n        }\n        const uint32_t * xcda_array;\n        size_t xcda_array_size;\n    };\n\n    // this structure stores the best tokenization so far at input_offset\n    struct best_tokenization {\n        llama_token token_id;\n        size_t input_offset;\n        double score_sum;\n    };\n\n    struct normalization_result normalize_prefix(const std::string & input, size_t input_offset) {\n        if (input_offset == input.size()) {\n            return { &input[input_offset], 0, 0 };\n        }\n\n        // if input prefix matches some user-defined token return this token as normalization result\n        auto user_defined_token_match =\n           tokenizer.user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset);\n        if (user_defined_token_match.second > 0) {\n            return { &input[input_offset], user_defined_token_match.second, user_defined_token_match.second };\n        }\n\n        size_t longest_prefix_length = 0;\n        size_t longest_prefix_offset = 0;\n\n        if (tokenizer.xcda_array_size > 0) {\n            struct xcda_array_view xcda_view(tokenizer.xcda_array, tokenizer.xcda_array_size);\n\n            // Find the longest normalized sequence matching the input prefix by walking\n            // the XOR-compressed compact double array (XCDA) starting from the root node\n            // We find the index of the next node by calculating BASE[s] ^ c where s is\n            // the index of the previous node and c is a numerical character value\n            uint32_t node_index = 0;\n            // get BASE of the root node\n            node_index = xcda_view.get_base(node_index);\n            for (size_t prefix_offset = input_offset; prefix_offset < input.size(); prefix_offset++) {\n                unsigned char c = input[prefix_offset];\n                if (c == 0) {\n                    break;\n                }\n                node_index ^= c;\n                // if value of LCHECK is not c it means that this is not a child of\n                // the previous node, so we stop matching\n                if (xcda_view.get_lcheck(node_index) != c) {\n                    break;\n                }\n                bool is_leaf = xcda_view.get_leaf(node_index);\n                // get BASE of the current node\n                node_index ^= xcda_view.get_base(node_index);\n                // if LEAF of the current node is true, it means that its BASE points to the node\n                // containing index of replacement sequence for currently matched input prefix\n                if (is_leaf)\n                {\n                    longest_prefix_length = prefix_offset - input_offset + 1;\n                    // get index of replacement sequence for currently matched input prefix\n                    longest_prefix_offset = xcda_view.get_value(node_index);\n                }\n            }\n        }\n\n        if (longest_prefix_length > 0) {\n            // we have a match, so return the replacement sequence\n            if (longest_prefix_offset >= tokenizer.prefix_replacements_size) {\n                throw std::runtime_error(\"Index out of array bounds in precompiled charsmap!\");\n            }\n            const char * prefix_replacement = &(tokenizer.prefix_replacements)[longest_prefix_offset];\n            return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };\n        }\n\n        // check if the input prefix contains a valid sequence of UTF-8 code units\n        try {\n            // if yes, return this sequence unmodified\n            size_t prefix_offset = input_offset;\n            unicode_cpt_from_utf8(input, prefix_offset);\n            return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset };\n        } catch (std::invalid_argument & /*ex*/) {\n            // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER\n            return { \"\\xEF\\xBF\\xBD\", 3, 1 };\n        }\n    }\n\n    const llama_vocab & vocab;\n    const llm_tokenizer_ugm & tokenizer;\n};\n\n//\n// RWKV tokenizer\n//\n\nstatic std::vector<uint8_t> llama_unescape_rwkv_token(const std::string & escaped) {\n    std::vector<uint8_t> output;\n    output.reserve(escaped.size());\n\n    // Parser state\n    bool escaping = false;\n    uint8_t hex_remaining = 0;\n    uint8_t hex_acc = 0;\n\n    // Step through characters, performing parsing\n    for (const char & c : escaped) {\n        // If we're parsing a hex code, interpret the next character\n        if (hex_remaining != 0) {\n            uint8_t value = (c >= 'a') ? (c - 'a' + 10) : (c - '0');\n            hex_acc = (hex_acc << 4) + value;\n\n            hex_remaining -= 1;\n            if (hex_remaining == 0) {\n                output.push_back(hex_acc);\n                hex_acc = 0;\n            }\n\n            continue;\n        }\n\n        // If we got an escape character, interpret it\n        if (escaping) {\n            if (c == 't') {\n                output.push_back('\\t');\n            } else if (c == 'n') {\n                output.push_back('\\n');\n            } else if (c == 'r') {\n                output.push_back('\\r');\n            } else if (c == 'x') {\n                hex_remaining = 2;\n            } else {\n                output.push_back(c);\n            }\n\n            escaping = false;\n            continue;\n        }\n\n        if (c == '\\\\') {\n            escaping = true;\n            continue;\n        }\n\n        output.push_back(c);\n    }\n\n    return output;\n}\n\nstruct llm_tokenizer_rwkv : llm_tokenizer {\n    llm_tokenizer_rwkv(const llama_vocab & vocab) {\n        // RWKV supports arbitrary byte tokens, but the vocab struct only supports string tokens.\n        // For now, we decode the vocab here into the lookup we'll use for tokenization.\n\n        // build trie\n        for (uint32_t id = 0; id < vocab.n_tokens(); ++id) {\n            const auto & data = vocab.get_token_data(id);\n            const auto text = llama_unescape_rwkv_token(data.text);\n            token_matcher.insert((const char *) text.data(), text.size(), id);\n        }\n    }\n\n    struct naive_trie token_matcher;\n};\n\nstruct llm_tokenizer_rwkv_session {\n    llm_tokenizer_rwkv_session(const llama_vocab & vocab, const llm_tokenizer_rwkv & tokenizer) : vocab(vocab), tokenizer(tokenizer) {}\n\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        uint32_t position = 0;\n        while (position < text.size()) {\n            const struct naive_trie * node = tokenizer.token_matcher.traverse(text[position]);\n            if (node == NULL) {\n                // no matching token found, add unknown token\n                output.push_back(vocab.token_unk());\n                position += 1;\n                continue;\n            }\n\n            // traverse the trie to find the longest matching token\n            uint32_t token_id = 0;\n            uint32_t token_length = 0;\n            while (node != NULL) {\n                if (node->has_value) {\n                    token_id = node->value;\n                    token_length = position + 1;\n                }\n                node = node->traverse(text[++position]);\n            }\n\n            // add the longest matching token\n            output.push_back(token_id);\n            position = token_length;\n        }\n    }\n\nprivate:\n    const llama_vocab & vocab;\n    const llm_tokenizer_rwkv & tokenizer;\n};\n\nstruct llm_tokenizer_plamo2 : llm_tokenizer {\n    llm_tokenizer_plamo2(const llama_vocab & vocab) {\n        build(vocab);\n    }\n\n    void build(const llama_vocab & vocab) {\n        // Reset internal structures\n        tokens_.clear();\n        bytes_.assign(256, 0);\n        to_suffix_id_.clear();\n        table_.clear();\n\n        // Build token list and byte mapping\n        std::unordered_map<std::string, float> suffix_to_score;\n        std::unordered_map<std::string, llama_token> token_to_id;\n\n        for (size_t token_id = 0; token_id < vocab.n_tokens(); ++token_id) {\n            const auto & entry = vocab.get_token_data(token_id);\n            tokens_.push_back(entry.text);\n            token_to_id[entry.text] = static_cast<llama_token>(token_id);\n\n            // Handle byte tokens\n            if (vocab.is_byte(token_id)) {\n                if (entry.text.length() == 6 && entry.text.substr(0, 3) == \"<0x\" && entry.text.back() == '>') {\n                    std::string hex_str = entry.text.substr(3, 2);\n                    int byte_val = std::stoi(hex_str, nullptr, 16);\n                    bytes_[byte_val] = static_cast<llama_token>(token_id);\n                }\n                continue;\n            }\n\n            // Add token and all its suffixes to suffix_to_score\n            suffix_to_score[entry.text] = entry.score;\n\n            // Extract suffixes character by character (UTF-8 aware)\n            std::vector<uint32_t> cpts = unicode_cpts_from_utf8(entry.text);\n            for (size_t i = 1; i < cpts.size(); ++i) {\n                std::string suffix;\n                for (size_t j = i; j < cpts.size(); ++j) {\n                    suffix += unicode_cpt_to_utf8(cpts[j]);\n                }\n                if (suffix_to_score.find(suffix) == suffix_to_score.end()) {\n                    suffix_to_score[suffix] = std::numeric_limits<float>::quiet_NaN();\n                }\n            }\n        }\n\n        // Check that all byte tokens are set\n        for (int i = 0; i < 256; ++i) {\n            if (bytes_[i] == 0) {\n                throw std::runtime_error(\"Byte token for <0x\" + std::to_string(i) + \"> is not set\");\n            }\n        }\n\n        // Build suffix list in lexicographical order of reversed strings\n        std::vector<std::string> suffixes;\n        suffixes.reserve(suffix_to_score.size() + 1);\n        for (const auto & pair : suffix_to_score) {\n            suffixes.push_back(pair.first);\n        }\n        suffixes.push_back(\"\");  // Empty suffix\n\n        std::sort(suffixes.begin(), suffixes.end(), [](const std::string & a, const std::string & b) {\n            std::string rev_a(a.rbegin(), a.rend());\n            std::string rev_b(b.rbegin(), b.rend());\n            return rev_a < rev_b;\n        });\n\n        // Build suffix_to_id and to_suffix_id_\n        std::unordered_map<std::string, int32_t> suffix_to_id;\n        int32_t num_pieces = 0;\n\n        for (const auto & suffix : suffixes) {\n            suffix_to_id[suffix] = num_pieces;\n            if (!suffix.empty()) {\n                std::vector<uint32_t> cpts = unicode_cpts_from_utf8(suffix);\n\n                std::string remaining;\n                for (size_t i = 1; i < cpts.size(); ++i) {\n                    remaining += unicode_cpt_to_utf8(cpts[i]);\n                }\n\n                int64_t piece_code = (static_cast<int64_t>(cpts[0]) << 32) | suffix_to_id[remaining];\n                to_suffix_id_[piece_code] = num_pieces;\n\n                // Count number of pieces for this suffix\n                int32_t pieces_for_suffix = 1; // sentinel row\n                for (int32_t piece_length = static_cast<int32_t>(cpts.size()); piece_length > 0; --piece_length) {\n                    std::string piece;\n                    for (int32_t i = 0; i < piece_length; ++i) {\n                        piece += unicode_cpt_to_utf8(cpts[i]);\n                    }\n                    if (suffix_to_score.find(piece) != suffix_to_score.end()) {\n                        pieces_for_suffix++;\n                    }\n                }\n                num_pieces += pieces_for_suffix;\n            } else {\n                num_pieces++;  // Empty suffix contributes one piece (sentinel row)\n            }\n        }\n\n        // Build flattened table\n        table_.resize(num_pieces, std::vector<int32_t>(4, 0));\n        int32_t table_idx = 0;\n\n        for (const auto & suffix : suffixes) {\n            // Add all prefixes of the suffix to the table (in decreasing order of length)\n            std::vector<uint32_t> cpts = unicode_cpts_from_utf8(suffix);\n            for (int32_t piece_length = static_cast<int32_t>(cpts.size()); piece_length > 0; --piece_length) {\n                std::string piece;\n                for (int32_t i = 0; i < piece_length; ++i) {\n                    piece += unicode_cpt_to_utf8(cpts[i]);\n                }\n\n                auto score_it = suffix_to_score.find(piece);\n                if (score_it == suffix_to_score.end()) {\n                    continue;\n                }\n\n                table_[table_idx][TABLE_PIECE_LENGTH] = piece_length;\n                auto token_it = token_to_id.find(piece);\n                table_[table_idx][TABLE_TOKEN_ID] = (token_it != token_to_id.end()) ? token_it->second : -1;\n\n                float score = score_it->second;\n                table_[table_idx][TABLE_SCORE] = std::isfinite(score) ?\n                    static_cast<int32_t>(std::round(score * 1e4)) : INVALID_SCORE;\n                table_[table_idx][TABLE_PIECE_ID] = suffix_to_id[piece];\n\n                table_idx++;\n            }\n\n            // Add sentinel row\n            table_[table_idx][TABLE_PIECE_LENGTH] = 1;\n            table_[table_idx][TABLE_TOKEN_ID] = -1;\n            table_[table_idx][TABLE_SCORE] = UNKNOWN_SCORE;\n            table_idx++;\n        }\n    }\n\n    std::vector<llama_token> encode(const std::string & text) const {\n        std::vector<uint32_t> unicode_data = unicode_cpts_from_utf8(text);\n        // Skip the first code point if it is a BOM (Byte Order Mark)\n        if (!unicode_data.empty() && unicode_data[0] == 0xFEFF) {\n            unicode_data.erase(unicode_data.begin());\n        }\n\n        if (unicode_data.empty()) {\n            return {};\n        }\n\n        const size_t data_len = unicode_data.size();\n\n        // Initialize scores array (dynamic programming)\n        std::vector<int64_t> scores(data_len + 1, static_cast<int64_t>(1) << 60);\n        scores[data_len] = 0;\n\n        // Path array to track best tokenization\n        std::vector<std::vector<int32_t>> path(data_len + 1, std::vector<int32_t>(3, 0));\n\n        int32_t suffix_id = 0;\n\n        // Process from end to beginning\n        for (int i = static_cast<int>(data_len) - 1; i >= 0; --i) {\n            uint32_t c = unicode_data[i];\n\n            // Find next suffix ID\n            for (size_t p = suffix_id; p < table_.size(); ++p) {\n                int64_t piece_code = (static_cast<int64_t>(c) << 32) | table_[p][TABLE_PIECE_ID];\n                auto it = to_suffix_id_.find(piece_code);\n                suffix_id = (it != to_suffix_id_.end()) ? it->second : 0;\n\n                if (suffix_id > 0 || table_[p][TABLE_SCORE] == UNKNOWN_SCORE) {\n                    break;\n                }\n            }\n\n            // Update best path\n            for (size_t p = suffix_id; p < table_.size(); ++p) {\n                int32_t score = table_[p][TABLE_SCORE];\n                if (score > INVALID_SCORE) {\n                    int32_t piece_length = table_[p][TABLE_PIECE_LENGTH];\n                    int64_t s = scores[i + piece_length] - score;\n\n                    if (s < scores[i]) {\n                        scores[i] = s;\n                        path[i][PATH_TOKEN_LENGTH] = piece_length;\n                        path[i][PATH_TOKEN_ID] = table_[p][TABLE_TOKEN_ID];\n                        path[i][PATH_NUM_TOKENS] = path[i + piece_length][PATH_NUM_TOKENS] + 1;\n\n                        if (score == UNKNOWN_SCORE) {\n                            // Add UTF-8 byte count\n                            path[i][PATH_NUM_TOKENS] += (c >= 0x80) + (c >= 0x800) + (c >= 0x10000);\n                        }\n                    }\n                }\n\n                if (score == UNKNOWN_SCORE) {\n                    break;\n                }\n            }\n        }\n\n        // Decode the best path\n        std::vector<llama_token> token_ids;\n        token_ids.reserve(path[0][PATH_NUM_TOKENS]);\n\n        int pos = 0;\n        while (pos < static_cast<int>(data_len)) {\n            if (path[pos][PATH_TOKEN_ID] >= 0) {\n                token_ids.push_back(path[pos][PATH_TOKEN_ID]);\n            } else {\n                // Fall back to byte tokens\n                uint32_t c = unicode_data[pos];\n                int s = 1 + (c >= 0x80) + (c >= 0x800) + (c >= 0x10000);\n\n                for (int i = 0; i < s; ++i) {\n                    uint8_t b;\n                    if (s == 1) {\n                        b = c;\n                    } else {\n                        if (i == 0) {\n                            b = (0xF00 >> s) & 0xFF;\n                        } else {\n                            b = 0x80;\n                        }\n                    }\n                    token_ids.push_back(bytes_[b | ((c >> ((s - i - 1) * 6)) & 0x3F)]);\n                }\n            }\n\n            assert(path[pos][PATH_TOKEN_LENGTH] > 0);\n            pos += path[pos][PATH_TOKEN_LENGTH];\n        }\n\n        return token_ids;\n    }\nprivate:\n    // Constants for table structure\n    static constexpr int32_t TABLE_PIECE_LENGTH = 0;\n    static constexpr int32_t TABLE_TOKEN_ID     = 1;\n    static constexpr int32_t TABLE_SCORE        = 2;\n    static constexpr int32_t TABLE_PIECE_ID     = 3;\n\n    // Constants for path array\n    static constexpr int32_t PATH_TOKEN_LENGTH  = 0;\n    static constexpr int32_t PATH_TOKEN_ID      = 1;\n    static constexpr int32_t PATH_NUM_TOKENS    = 2;\n\n    // Score constants\n    static constexpr int32_t INVALID_SCORE = -20000000;\n    static constexpr int32_t UNKNOWN_SCORE = -10000000;\n\n    // List of tokens in the vocabulary\n    std::vector<std::string> tokens_;\n\n    // Mapping from byte code point to token ID (for byte fallback)\n    std::vector<llama_token> bytes_;\n\n    // Mapping from piece code to suffix ID\n    std::unordered_map<int64_t, int32_t> to_suffix_id_;\n\n    // Flattened table representing the Trie structure\n    // Each row contains: [piece_length, token_id, score, piece_id]\n    std::vector<std::vector<int32_t>> table_;\n};\n\nstruct llm_tokenizer_plamo2_session {\n    llm_tokenizer_plamo2_session(const llm_tokenizer_plamo2 & tokenizer) : tokenizer(tokenizer) {}\n\n    void tokenize(const std::string & text, std::vector<llama_token> & output) {\n        std::vector<llama_token> tokens = tokenizer.encode(text);\n        output.insert(output.end(), tokens.begin(), tokens.end());\n    }\n\nprivate:\n    const llm_tokenizer_plamo2 & tokenizer;\n};\n\n//\n// impl\n//\n\ntypedef enum FRAGMENT_BUFFER_VARIANT_TYPE {\n    FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN,\n    FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT\n} FRAGMENT_BUFFER_VARIANT_TYPE;\n\nstruct fragment_buffer_variant {\n    fragment_buffer_variant(llama_token _token)\n    :\n        type(FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN),\n        token(_token),\n        raw_text(_dummy),\n        offset(0),\n        length(0) {}\n\n    fragment_buffer_variant(const std::string & _raw_text, int64_t _offset, int64_t _length)\n    :\n        type(FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT),\n        token((llama_token) - 1),\n        raw_text(_raw_text),\n        offset(_offset),\n        length(_length){\n            GGML_ASSERT(_offset >= 0);\n            GGML_ASSERT(_length >= 1);\n            GGML_ASSERT(offset + length <= raw_text.length());\n        }\n\n    const FRAGMENT_BUFFER_VARIANT_TYPE type;\n    const llama_token token;\n    const std::string _dummy;\n    const std::string & raw_text;\n    const uint64_t offset;\n    const uint64_t length;\n};\n\nstruct llama_vocab::impl {\n    uint32_t n_token_types = 0; // for BERT-style token types\n\n    std::string tokenizer_model;\n    std::string tokenizer_pre;\n\n    enum llama_vocab_type     type     = LLAMA_VOCAB_TYPE_SPM;\n    enum llama_vocab_pre_type pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n\n    int max_token_len = 0; // used for optimizing longest token search\n\n    // default LLaMA special tokens\n    // TODO: should we set all of these to LLAMA_TOKEN_NULL?\n    llama_token special_bos_id  = 1;\n    llama_token special_eos_id  = 2;\n    llama_token special_eot_id  = LLAMA_TOKEN_NULL;\n    llama_token special_eom_id  = LLAMA_TOKEN_NULL;\n    llama_token special_unk_id  = 0;\n    llama_token special_sep_id  = LLAMA_TOKEN_NULL;\n    llama_token special_pad_id  = LLAMA_TOKEN_NULL;\n    llama_token special_mask_id = LLAMA_TOKEN_NULL;\n\n    llama_token linefeed_id = 13;\n\n    // fim tokens\n    llama_token special_fim_pre_id = LLAMA_TOKEN_NULL;\n    llama_token special_fim_suf_id = LLAMA_TOKEN_NULL;\n    llama_token special_fim_mid_id = LLAMA_TOKEN_NULL;\n    llama_token special_fim_pad_id = LLAMA_TOKEN_NULL;\n    llama_token special_fim_rep_id = LLAMA_TOKEN_NULL; // repo\n    llama_token special_fim_sep_id = LLAMA_TOKEN_NULL; // file separator\n\n    // tokenizer flags\n    bool add_space_prefix           = false;\n    bool add_bos                    = false;\n    bool add_eos                    = false;\n    bool add_sep                    = false;\n    bool ignore_merges              = false;\n    bool clean_spaces               = false;  // clean_up_tokenization_spaces\n    bool remove_extra_whitespaces   = false;\n    bool escape_whitespaces         = true;\n    bool treat_whitespace_as_suffix = false;\n\n    std::unordered_map<std::string, llama_token> token_to_id;\n    std::vector<token_data>                      id_to_token;\n\n    std::vector<llama_token> cache_special_tokens;\n    std::vector<std::string> cache_token_to_piece; // llama_token_to_piece(special = true);\n    struct pair_hash {\n        size_t operator()(const std::pair<std::string, std::string> & p) const {\n            return std::hash<std::string>{}(p.first) ^  //create some hash for pair\n                   (std::hash<std::string>{}(p.second) << 1);\n        }\n    };\n    std::unordered_map<std::pair<std::string, std::string>, int, pair_hash> bpe_ranks;\n\n    // set of all tokens that cause \"end of generation\"\n    std::set<llama_token> special_eog_ids;\n\n    std::unique_ptr<llm_tokenizer> tokenizer;\n\n    std::vector<char> precompiled_charsmap;\n\n    impl(const llama_vocab & vocab) : vocab(vocab) {\n    }\n\n    ~impl() = default;\n\n    void load(llama_model_loader & ml, const LLM_KV & kv);\n\n    enum llama_vocab_type get_type() const;\n\n    std::string type_name() const;\n\n    bool is_normal      (llama_token id) const;\n    bool is_unknown     (llama_token id) const;\n    bool is_control     (llama_token id) const;\n    bool is_byte        (llama_token id) const;\n    bool is_user_defined(llama_token id) const;\n    bool is_unused      (llama_token id) const;\n    bool is_eog         (llama_token id) const;\n\n    uint8_t token_to_byte(llama_token id) const;\n\n    llama_token_attr token_get_attr(llama_token id) const;\n\n    void init_tokenizer(enum llama_vocab_type type);\n\n    void tokenizer_st_partition(std::forward_list<fragment_buffer_variant> & buffer, bool parse_special) const;\n\n    std::string token_to_piece_for_cache(\n                  llama_token   token,\n                         bool   special) const;\n\n\n    std::vector<llama_token> tokenize(\n            const std::string & raw_text,\n                         bool   add_special,\n                         bool   parse_special = false) const;\n\n    int32_t tokenize(\n                   const char * text,\n                      int32_t   text_len,\n                  llama_token * tokens,\n                      int32_t   n_tokens_max,\n                         bool   add_special,\n                         bool   parse_special) const;\n\n    // does not write null-terminator to buf\n    int32_t token_to_piece(\n                  llama_token   token,\n                         char * buf,\n                      int32_t   length,\n                      int32_t   lstrip,\n                         bool   special) const;\n\n    // use cached data\n    const std::string & token_to_piece(llama_token token) const;\n\n    int32_t detokenize(\n            const llama_token * tokens,\n                      int32_t   n_tokens,\n                         char * text,\n                      int32_t   text_len_max,\n                         bool   remove_special,\n                         bool   unparse_special) const;\n\n    std::string detokenize(\n            const std::vector<llama_token> & tokens,\n                                      bool   special) const;\n\n    void print_info() const;\n\nprivate:\n    const llama_vocab & vocab;\n};\n\nvoid llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {\n    struct gguf_context * ctx = ml.metadata;\n\n    // determine vocab type\n    {\n        ml.get_key(LLM_KV_TOKENIZER_MODEL, tokenizer_model);\n        ml.get_key(LLM_KV_TOKENIZER_PRE,   tokenizer_pre, false);\n\n        ml.get_key(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, n_token_types, false);\n\n        if (tokenizer_model == \"no_vocab\" || tokenizer_model == \"none\") {\n            type = LLAMA_VOCAB_TYPE_NONE;\n\n            // default special tokens\n            special_bos_id  = LLAMA_TOKEN_NULL;\n            special_eos_id  = LLAMA_TOKEN_NULL;\n            special_unk_id  = LLAMA_TOKEN_NULL;\n            special_sep_id  = LLAMA_TOKEN_NULL;\n            special_pad_id  = LLAMA_TOKEN_NULL;\n            special_mask_id = LLAMA_TOKEN_NULL;\n            linefeed_id     = LLAMA_TOKEN_NULL;\n\n            // read vocab size from metadata\n            uint32_t n_tokens = 0;\n            if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {\n                LLAMA_LOG_WARN(\"%s: adding %u dummy tokens\\n\", __func__, n_tokens);\n                id_to_token.resize(n_tokens);\n            }\n\n            return;\n        }\n\n        if (tokenizer_model == \"llama\") {\n            type = LLAMA_VOCAB_TYPE_SPM;\n\n            // default special tokens\n            special_bos_id  = 1;\n            special_eos_id  = 2;\n            special_unk_id  = 0;\n            special_sep_id  = LLAMA_TOKEN_NULL;\n            special_pad_id  = LLAMA_TOKEN_NULL;\n            special_mask_id = LLAMA_TOKEN_NULL;\n        } else if (tokenizer_model == \"bert\") {\n            type = LLAMA_VOCAB_TYPE_WPM;\n\n            // default special tokens\n            special_bos_id  = 101;\n            special_eos_id  = LLAMA_TOKEN_NULL;\n            special_unk_id  = 100;\n            special_sep_id  = 102;\n            special_pad_id  = 0;\n            special_mask_id = 103;\n\n            add_sep = true;\n        } else if (tokenizer_model == \"gpt2\") {\n            type = LLAMA_VOCAB_TYPE_BPE;\n\n            // read bpe merges and populate bpe ranks\n            const int merges_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_MERGES).c_str());\n            // Kimi-K2 uses custom tokenization without traditional BPE merges\n            const bool is_kimi_k2 = (tokenizer_pre == \"kimi-k2\");\n\n            if (merges_keyidx == -1) {\n                if (!is_kimi_k2) {\n                    throw std::runtime_error(\"cannot find tokenizer merges in model file\\n\");\n                }\n                // Kimi-K2 doesn't need merges, skip\n                LLAMA_LOG_INFO(\"%s: Kimi-K2 tokenizer detected, skipping BPE merges\\n\", __func__);\n            } else {\n                const int n_merges = gguf_get_arr_n(ctx, merges_keyidx);\n                for (int i = 0; i < n_merges; i++) {\n                    const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);\n                    //GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);\n\n                    std::string first;\n                    std::string second;\n\n                    const size_t pos = word.find(' ', 1);\n\n                    if (pos != std::string::npos) {\n                        first  = word.substr(0, pos);\n                        second = word.substr(pos + 1);\n                    }\n\n                    bpe_ranks.emplace(std::make_pair(first, second), i);\n                }\n            }\n\n            // default special tokens\n            special_bos_id  = 11;\n            special_eos_id  = 11;\n            special_unk_id  = LLAMA_TOKEN_NULL;\n            special_sep_id  = LLAMA_TOKEN_NULL;\n            special_pad_id  = LLAMA_TOKEN_NULL;\n            special_mask_id = LLAMA_TOKEN_NULL;\n        } else if (tokenizer_model == \"t5\") {\n            type = LLAMA_VOCAB_TYPE_UGM;\n\n            // default special tokens\n            special_bos_id  = LLAMA_TOKEN_NULL;\n            special_eos_id  = 1;\n            special_unk_id  = 2;\n            special_sep_id  = LLAMA_TOKEN_NULL;\n            special_pad_id  = 0;\n            special_mask_id = LLAMA_TOKEN_NULL;\n\n            const int precompiled_charsmap_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP).c_str());\n            if (precompiled_charsmap_keyidx != -1) {\n                const gguf_type pc_type = gguf_get_arr_type(ctx, precompiled_charsmap_keyidx);\n                GGML_ASSERT(pc_type == GGUF_TYPE_INT8 || pc_type == GGUF_TYPE_UINT8);\n\n                const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);\n                const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);\n                precompiled_charsmap.assign(pc, pc + n_precompiled_charsmap);\n#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__\n                // correct endianness of data in precompiled_charsmap binary blob\n                uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];\n                *xcda_blob_size = __builtin_bswap32(*xcda_blob_size);\n                assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap);\n                size_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t);\n                uint32_t * xcda_array = (uint32_t *) &precompiled_charsmap[sizeof(uint32_t)];\n                for (size_t i = 0; i < xcda_array_size; ++i) {\n                    xcda_array[i] = __builtin_bswap32(xcda_array[i]);\n                }\n#endif\n            }\n        } else if (tokenizer_model == \"rwkv\") {\n            type = LLAMA_VOCAB_TYPE_RWKV;\n\n            // default special tokens\n            special_bos_id = LLAMA_TOKEN_NULL;\n            special_eos_id = LLAMA_TOKEN_NULL;\n            special_unk_id = LLAMA_TOKEN_NULL;\n            special_sep_id = LLAMA_TOKEN_NULL;\n            special_pad_id = LLAMA_TOKEN_NULL;\n        } else if (tokenizer_model == \"plamo2\") {\n            type = LLAMA_VOCAB_TYPE_PLAMO2;\n\n            // PLaMo-2 default special tokens (these will be overridden by model config)\n            special_bos_id = 1;  // <|plamo:bos|>\n            special_eos_id = 2;  // <|plamo:eos|>\n            special_unk_id = 0;  // <|plamo:unk|>\n            special_sep_id = LLAMA_TOKEN_NULL;\n            special_pad_id = 3;  // <|plamo:pad|>\n            special_mask_id = LLAMA_TOKEN_NULL;\n        } else {\n            throw std::runtime_error(format(\"unknown tokenizer: '%s'\", tokenizer_model.c_str()));\n        }\n\n        // for now, only BPE models have pre-tokenizers\n        if (type == LLAMA_VOCAB_TYPE_BPE) {\n            add_space_prefix = false;\n            clean_spaces = true;\n            if (tokenizer_pre.empty()) {\n                LLAMA_LOG_WARN(\"%s: missing pre-tokenizer type, using: 'default'\\n\", __func__);\n                LLAMA_LOG_WARN(\"%s:                                             \\n\", __func__);\n                LLAMA_LOG_WARN(\"%s: ************************************        \\n\", __func__);\n                LLAMA_LOG_WARN(\"%s: GENERATION QUALITY WILL BE DEGRADED!        \\n\", __func__);\n                LLAMA_LOG_WARN(\"%s: CONSIDER REGENERATING THE MODEL             \\n\", __func__);\n                LLAMA_LOG_WARN(\"%s: ************************************        \\n\", __func__);\n                LLAMA_LOG_WARN(\"%s:                                             \\n\", __func__);\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            } else if (tokenizer_pre == \"default\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            } else if (\n                    tokenizer_pre == \"llama3\"   ||\n                    tokenizer_pre == \"llama-v3\" ||\n                    tokenizer_pre == \"llama-bpe\"||\n                    tokenizer_pre == \"falcon3\"  ||\n                    tokenizer_pre == \"falcon-h1\" ||\n                    tokenizer_pre == \"pixtral\"  ||\n                    tokenizer_pre == \"midm-2.0\" ||\n                    tokenizer_pre == \"lfm2\"     ||\n                    tokenizer_pre == \"jina-v5-nano\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3;\n                ignore_merges = true;\n                add_bos = true;\n            } else if (\n                    tokenizer_pre == \"deepseek-llm\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;\n                clean_spaces = false;\n            } else if (\n                    tokenizer_pre == \"deepseek-coder\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER;\n                clean_spaces = false;\n            } else if (\n                    tokenizer_pre == \"deepseek-v3\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM;\n                clean_spaces = false;\n            } else if (\n                    tokenizer_pre == \"youtu\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_YOUTU;\n                clean_spaces = false;\n                ignore_merges = true;\n            } else if (\n                    tokenizer_pre == \"falcon\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_FALCON;\n            } else if (\n                    tokenizer_pre == \"mpt\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_MPT;\n            } else if (\n                    tokenizer_pre == \"starcoder\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_STARCODER;\n            } else if (\n                    tokenizer_pre == \"gpt-2\"   ||\n                    tokenizer_pre == \"phi-2\"   ||\n                    tokenizer_pre == \"jina-es\" ||\n                    tokenizer_pre == \"jina-de\" ||\n                    tokenizer_pre == \"gigachat\"   ||\n                    tokenizer_pre == \"jina-v2-es\" ||\n                    tokenizer_pre == \"jina-v2-de\" ||\n                    tokenizer_pre == \"a.x-4.0\" ||\n                    tokenizer_pre == \"mellum\"  ||\n                    tokenizer_pre == \"modern-bert\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;\n            } else if (\n                    tokenizer_pre == \"jais-2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS2;\n            } else if (\n                    tokenizer_pre == \"jina-v1-en\" ||\n                    tokenizer_pre == \"jina-v2-code\" ||\n                    tokenizer_pre == \"roberta-bpe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;\n                add_sep = true;\n            } else if (\n                    tokenizer_pre == \"refact\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT;\n            } else if (\n                tokenizer_pre == \"command-r\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;\n                clean_spaces = false;\n            } else if (\n                    tokenizer_pre == \"qwen2\" ||\n                    tokenizer_pre == \"deepseek-r1-qwen\" ||\n                    tokenizer_pre == \"kormo\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;\n                clean_spaces = false;\n            } else if (\n                    tokenizer_pre == \"qwen35\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN35;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"stablelm2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_STABLELM2;\n            } else if (\n                tokenizer_pre == \"olmo\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_OLMO;\n            } else if (\n                tokenizer_pre == \"dbrx\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX;\n            } else if (\n                tokenizer_pre == \"smaug-bpe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG;\n            } else if (\n                tokenizer_pre == \"poro-chat\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_PORO;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"glm4\" ||\n                tokenizer_pre == \"chatglm-bpe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4;\n                special_bos_id = LLAMA_TOKEN_NULL;\n            } else if (\n                tokenizer_pre == \"viking\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"jais\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS;\n            } else if (\n                tokenizer_pre == \"tekken\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_TEKKEN;\n                clean_spaces = false;\n                ignore_merges = true;\n                add_bos = true;\n            } else if (\n                tokenizer_pre == \"smollm\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_SMOLLM;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"codeshell\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_CODESHELL;\n            } else if (\n                tokenizer_pre == \"bloom\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_BLOOM;\n            } else if (\n                tokenizer_pre == \"gpt3-finnish\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH;\n            } else if (\n                tokenizer_pre == \"exaone\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE;\n            } else if (\n                tokenizer_pre == \"exaone4\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2;\n            } else if (\n                tokenizer_pre == \"exaone-moe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE;\n            } else if (\n                tokenizer_pre == \"chameleon\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_CHAMELEON;\n                add_bos = true;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"minerva-7b\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_MINERVA;\n            } else if (\n                tokenizer_pre == \"megrez\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2;\n            } else if (\n                tokenizer_pre == \"gpt-4o\" ||\n                tokenizer_pre == \"llama4\" ||\n                tokenizer_pre == \"kanana2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"tiny_aya\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_TINY_AYA;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"superbpe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_SUPERBPE;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"trillion\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_TRILLION;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"granite-docling\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"bailingmoe\" ||\n                tokenizer_pre == \"bailingmoe2\" ||\n                tokenizer_pre == \"llada-moe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_BAILINGMOE;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"seed-coder\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_SEED_CODER;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"hunyuan\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_HUNYUAN;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"hunyuan-dense\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"joyai-llm\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"kimi-k2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_KIMI_K2;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"grok-2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_GROK_2;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"afmoe\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_AFMOE;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"minimax-m2\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2;\n                clean_spaces = false;\n            } else if (\n                tokenizer_pre == \"solar-open\") {\n                pre_type = LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN;\n                clean_spaces = false;\n            } else {\n                throw std::runtime_error(format(\"unknown pre-tokenizer type: '%s'\", tokenizer_pre.c_str()));\n            }\n        } else if (type == LLAMA_VOCAB_TYPE_SPM) {\n            pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            add_space_prefix = true;\n            clean_spaces = false;\n            add_bos = true;\n            add_eos = false;\n        } else if (type == LLAMA_VOCAB_TYPE_WPM) {\n            pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            add_space_prefix = false;\n            clean_spaces = true;\n            add_bos = true;\n            add_eos = false;\n            add_sep = true;\n        } else if (type == LLAMA_VOCAB_TYPE_UGM) {\n            pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            add_bos = false;\n            add_eos = true;\n        } else if (type == LLAMA_VOCAB_TYPE_RWKV) {\n            pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n            add_space_prefix = false;\n            clean_spaces = false;\n            add_bos = false;\n            add_eos = false;\n        } else {\n            pre_type = LLAMA_VOCAB_PRE_TYPE_DEFAULT;\n        }\n\n        ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX,      add_space_prefix,         false);\n        ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false);\n    }\n\n    const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str());\n    if (token_idx == -1) {\n        throw std::runtime_error(\"cannot find tokenizer vocab in model file\\n\");\n    }\n\n    const float * scores = nullptr;\n    const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());\n    if (score_idx != -1) {\n        scores = (const float * ) gguf_get_arr_data(ctx, score_idx);\n    }\n\n    const int * toktypes = nullptr;\n    const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());\n    if (toktype_idx != -1) {\n        toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);\n    }\n\n    uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx);\n    id_to_token.resize(n_tokens);\n\n    for (uint32_t i = 0; i < n_tokens; i++) {\n        std::string word = gguf_get_arr_str(ctx, token_idx, i);\n        if (word.empty()) {\n            LLAMA_LOG_WARN(\"%s: empty token at index %u\\n\", __func__, i);\n            word = \"[EMPTY_\" + std::to_string(i) + \"]\";\n        }\n\n        token_to_id[word] = i;\n        max_token_len = std::max(max_token_len, (int) word.size());\n\n        auto & token_data = id_to_token[i];\n        token_data.text  = std::move(word);\n        token_data.score = scores ? scores[i] : 0.0f;\n        token_data.attr  = LLAMA_TOKEN_ATTR_NORMAL;\n\n        if (toktypes) {  //TODO: remove, required until per token attributes are available from GGUF file\n            switch(toktypes[i]) {\n                case LLAMA_TOKEN_TYPE_UNKNOWN:      token_data.attr = LLAMA_TOKEN_ATTR_UNKNOWN;      break;\n                case LLAMA_TOKEN_TYPE_UNUSED:       token_data.attr = LLAMA_TOKEN_ATTR_UNUSED;       break;\n                case LLAMA_TOKEN_TYPE_NORMAL:       token_data.attr = LLAMA_TOKEN_ATTR_NORMAL;       break;\n                case LLAMA_TOKEN_TYPE_CONTROL:      token_data.attr = LLAMA_TOKEN_ATTR_CONTROL;      break;\n                case LLAMA_TOKEN_TYPE_USER_DEFINED: token_data.attr = LLAMA_TOKEN_ATTR_USER_DEFINED; break;\n                case LLAMA_TOKEN_TYPE_BYTE:         token_data.attr = LLAMA_TOKEN_ATTR_BYTE;         break;\n                case LLAMA_TOKEN_TYPE_UNDEFINED:    token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED;    break;\n                default:                            token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED;    break;\n            }\n        }\n    }\n    GGML_ASSERT(id_to_token.size() == token_to_id.size());\n\n    init_tokenizer(type);\n\n    // determine the newline token: LLaMA \"<0x0A>\" == 10 == '\\n', Falcon 193 == '\\n'\n    if (type == LLAMA_VOCAB_TYPE_SPM) {\n        try {\n            linefeed_id = vocab.byte_to_token('\\n');\n        } catch (const std::exception & e) {\n            LLAMA_LOG_WARN(\"%s: SPM vocabulary, but newline token not found: %s! Using special_pad_id instead.\", __func__, e.what());\n            linefeed_id = special_pad_id;\n        }\n    } else if (type == LLAMA_VOCAB_TYPE_WPM) {\n        linefeed_id = special_pad_id;\n    } else if (type == LLAMA_VOCAB_TYPE_RWKV) {\n        const std::vector<int> ids = tokenize(\"\\n\", false);\n        GGML_ASSERT(!ids.empty() && \"model vocab missing newline token\");\n        linefeed_id = ids[0];\n    } else {\n        const std::vector<int> ids = tokenize(\"\\n\", false);\n\n        //GGML_ASSERT(!ids.empty() && \"model vocab missing newline token\");\n        if (ids.empty()) {\n            LLAMA_LOG_WARN(\"%s: model vocab missing newline token, using special_pad_id instead\\n\", __func__);\n            linefeed_id = special_pad_id;\n        } else {\n            linefeed_id = ids[0];\n        }\n    }\n\n    // special tokens\n    {\n        const std::vector<std::pair<enum llm_kv, int32_t &>> special_token_types = {\n            { LLM_KV_TOKENIZER_BOS_ID,     special_bos_id     },\n            { LLM_KV_TOKENIZER_EOS_ID,     special_eos_id     },\n            { LLM_KV_TOKENIZER_EOT_ID,     special_eot_id     },\n            { LLM_KV_TOKENIZER_EOM_ID,     special_eom_id     },\n            { LLM_KV_TOKENIZER_UNK_ID,     special_unk_id     },\n            { LLM_KV_TOKENIZER_SEP_ID,     special_sep_id     },\n            { LLM_KV_TOKENIZER_PAD_ID,     special_pad_id     },\n            { LLM_KV_TOKENIZER_MASK_ID,    special_mask_id    },\n            { LLM_KV_TOKENIZER_FIM_PRE_ID, special_fim_pre_id },\n            { LLM_KV_TOKENIZER_FIM_SUF_ID, special_fim_suf_id },\n            { LLM_KV_TOKENIZER_FIM_MID_ID, special_fim_mid_id },\n            { LLM_KV_TOKENIZER_FIM_PAD_ID, special_fim_pad_id },\n            { LLM_KV_TOKENIZER_FIM_REP_ID, special_fim_rep_id },\n            { LLM_KV_TOKENIZER_FIM_SEP_ID, special_fim_sep_id },\n\n            // deprecated\n            { LLM_KV_TOKENIZER_PREFIX_ID, special_fim_pre_id },\n            { LLM_KV_TOKENIZER_SUFFIX_ID, special_fim_suf_id },\n            { LLM_KV_TOKENIZER_MIDDLE_ID, special_fim_mid_id },\n        };\n\n        for (const auto & it : special_token_types) {\n            const std::string & key = kv(std::get<0>(it));\n            int32_t & id = std::get<1>(it);\n\n            uint32_t new_id;\n            if (!ml.get_key(std::get<0>(it), new_id, false)) {\n                continue;\n            }\n            if (new_id >= id_to_token.size()) {\n                LLAMA_LOG_WARN(\"%s: bad special token: '%s' = %u, using default id %d\\n\",\n                    __func__, key.c_str(), new_id, id);\n            } else {\n                id = new_id;\n            }\n        }\n\n        // Handle add_bos, add_eos and add_sep\n        {\n            bool temp = true;\n\n            if (ml.get_key(LLM_KV_TOKENIZER_ADD_BOS, temp, false)) {\n                add_bos = temp;\n            }\n            if (ml.get_key(LLM_KV_TOKENIZER_ADD_EOS, temp, false)) {\n                add_eos = temp;\n            }\n            if (ml.get_key(LLM_KV_TOKENIZER_ADD_SEP, temp, false)) {\n                add_sep = temp;\n            }\n        }\n\n        // auto-detect special tokens by text\n        // TODO: convert scripts should provide these tokens through the KV metadata LLM_KV_TOKENIZER_...\n        //       for now, we apply this workaround to find the tokens based on their text\n\n        for (const auto & t : token_to_id) {\n            auto & attr = id_to_token[t.second].attr;\n\n            // find EOT token: \"<|eot_id|>\", \"<|im_end|>\", \"<end_of_turn>\", etc.\n            if (special_eot_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|eot_id|>\"\n                        || t.first == \"<|im_end|>\"\n                        || t.first == \"<|end|>\"\n                        || t.first == \"<end_of_turn>\"\n                        || t.first == \"<|endoftext|>\"\n                        || t.first == \"<|end_of_text|>\" // granite\n                        || t.first == \"<EOT>\"\n                        || t.first == \"_<EOT>\"\n                        || t.first == \"[EOT]\" // Kimi-K2\n                        || t.first == \"<｜end▁of▁sentence｜>\" // DeepSeek\n                        || t.first == \"<end_of_utterance>\" // smoldocling\n                   ) {\n                    special_eot_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find EOM token: \"<|eom_id|>\"\n            if (special_eom_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|eom_id|>\"\n                        ) {\n                    special_eom_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_PRE token: \"<|fim_prefix|>\", \"<fim-prefix>\", \"<PRE>\", etc.\n            if (special_fim_pre_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|fim_prefix|>\"  // Qwen\n                        || t.first == \"<fim-prefix>\"\n                        || t.first == \"<fim_prefix>\"    // Granite\n                        || t.first == \"<｜fim▁begin｜>\" // DeepSeek\n                        || t.first == \"<PRE>\"\n                        || t.first == \"▁<PRE>\"          // CodeLlama\n                        || t.first == \"<|code_prefix|>\" // GLM-4.5\n                        || t.first == \"<|prefix|>\"      // Falcon-H1-Tiny-Coder\n                        ) {\n                    special_fim_pre_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_SUF token: \"<|fim_suffix|>\", \"<fim-suffix>\", \"<SUF>\", etc.\n            if (special_fim_suf_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|fim_suffix|>\" // Qwen\n                        || t.first == \"<fim-suffix>\"\n                        || t.first == \"<fim_suffix>\"   // Granite\n                        || t.first == \"<｜fim▁hole｜>\" // DeepSeek\n                        || t.first == \"<SUF>\"\n                        || t.first == \"▁<SUF>\"         // CodeLlama\n                        || t.first == \"<|code_suffix|>\" // GLM-4.5\n                        || t.first == \"<|suffix|>\"      // Falcon-H1-Tiny-Coder\n                        ) {\n                    special_fim_suf_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_MID token: \"<|fim_middle|>\", \"<fim-middle>\", \"<MID>\", etc.\n            if (special_fim_mid_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|fim_middle|>\" // Qwen\n                        || t.first == \"<fim-middle>\"\n                        || t.first == \"<fim_middle>\"   // Granite\n                        || t.first == \"<｜fim▁end｜>\"  // DeepSeek\n                        || t.first == \"<MID>\"\n                        || t.first == \"▁<MID>\"         // CodeLlama\n                        || t.first == \"<|code_middle|>\" // GLM-4.5\n                        || t.first == \"<|middle|>\"      // Falcon-H1-Tiny-Coder\n                        ) {\n                    special_fim_mid_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_PAD token: \"<|fim_pad|>\", \"<fim-pad>\", \"<PAD>\", etc.\n            if (special_fim_pad_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|fim_pad|>\" // Qwen\n                        || t.first == \"<fim-pad>\"\n                        || t.first == \"<fim_pad>\"   // Granite\n                        || t.first == \"<PAD>\"\n                        || t.first == \"[PAD]\" // Kimi-K2\n                        ) {\n                    special_fim_pad_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_REP token: \"<|fim_repo|>\", \"<fim-repo>\", \"<REP>\", etc.\n            if (special_fim_rep_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|fim_repo|>\"  // Qwen\n                        || t.first == \"<|repo_name|>\"\n                        || t.first == \"<fim-repo>\"\n                        || t.first == \"<REPO>\"\n                        || t.first == \"<reponame>\"    // Granite\n                        ) {\n                    special_fim_rep_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n\n            // find FIM_SEP token: \"<|file_sep|>\"\n            if (special_fim_sep_id == LLAMA_TOKEN_NULL) {\n                if (false\n                        || t.first == \"<|file_sep|>\" // Qwen\n                        ) {\n                    special_fim_sep_id = t.second;\n                    if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                        LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                                __func__, t.second, t.first.c_str());\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                    }\n                }\n            }\n        }\n\n        // auto-detect unused tokens: e.g. control tokens with the word \"unused\"\n        // ideally, these tokens should be marked as unused during conversion\n        {\n            uint32_t n_unused = 0;\n\n            for (const auto & t : token_to_id) {\n                auto & attr = id_to_token[t.second].attr;\n\n                if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                    continue;\n                }\n\n                if ((attr & LLAMA_TOKEN_ATTR_UNUSED) == 0) {\n                    if (strstr(t.first.c_str(), \"unused\") != NULL) {\n                        attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_UNUSED);\n                    }\n                }\n\n                if (attr & LLAMA_TOKEN_ATTR_UNUSED) {\n                    n_unused++;\n                }\n            }\n\n            LLAMA_LOG_INFO(\"%s: %u unused tokens\\n\", __func__, n_unused);\n        }\n\n        // maintain a list of tokens that cause end-of-generation\n        // this is currently determined based on the token text, which is obviously not ideal\n        // ref: https://github.com/ggml-org/llama.cpp/issues/9606\n        special_eog_ids.clear();\n\n        if (special_fim_pad_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_pad_id) == 0) {\n            special_eog_ids.insert(special_fim_pad_id);\n        }\n\n        if (special_fim_rep_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_rep_id) == 0) {\n            special_eog_ids.insert(special_fim_rep_id);\n        }\n\n        if (special_fim_sep_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_fim_sep_id) == 0) {\n            special_eog_ids.insert(special_fim_sep_id);\n        }\n\n        for (const auto & t : token_to_id) {\n            auto & attr = id_to_token[t.second].attr;\n\n            if (false\n                    || t.first == \"<|eot_id|>\"\n                    || t.first == \"<|im_end|>\"\n                    || t.first == \"<|end|>\"\n                    || t.first == \"<|return|>\" // o200k_harmony\n                    || t.first == \"<|call|>\"   // o200k_harmony\n                    || t.first == \"<|flush|>\"  // solar-open\n                    || t.first == \"<|calls|>\"  // solar-open\n                    || t.first == \"<end_of_turn>\"\n                    || t.first == \"<|endoftext|>\"\n                    || t.first == \"</s>\"      // paddleocr\n                    || t.first == \"<|eom_id|>\"\n                    || t.first == \"<EOT>\"\n                    || t.first == \"_<EOT>\"\n                    || t.first == \"[EOT]\" // Kimi-K2\n                    || t.first == \"[EOS]\" // Kimi-K2\n                    || t.first == \"<|end_of_text|>\"\n                    || t.first == \"<end_of_utterance>\" // smoldocling\n               ) {\n                special_eog_ids.insert(t.second);\n                if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {\n                    LLAMA_LOG_WARN(\"%s: control-looking token: %6d '%s' was not control-type; this is probably a bug in the model. its type will be overridden\\n\",\n                            __func__, t.second, t.first.c_str());\n                    attr = (llama_token_attr) (attr | LLAMA_TOKEN_ATTR_CONTROL);\n                }\n            } else {\n                if (attr & LLAMA_TOKEN_ATTR_CONTROL && !(attr & LLAMA_TOKEN_ATTR_UNUSED)) {\n                    // token is control, but not marked as EOG -> print a debug log\n                    if (special_eog_ids.count(t.second) == 0) {\n                        LLAMA_LOG_DEBUG(\"%s: control token: %6d '%s' is not marked as EOG\\n\",\n                                __func__, t.second, t.first.c_str());\n                    }\n                }\n            }\n        }\n\n        // @ngxson : quick hack for gpt-oss, always render these tokens\n        for (const auto & t : token_to_id) {\n            auto & attr = id_to_token[t.second].attr;\n\n            if (t.first == \"<|channel|>\" || t.first == \"<|message|>\" || t.first == \"<|start|>\" || t.first == \"<|constrain|>\") {\n                LLAMA_LOG_WARN(\"%s: setting token '%s' (%d) attribute to USER_DEFINED (%u), old attributes: %u\\n\",\n                        __func__, t.first.c_str(), t.second, LLAMA_TOKEN_ATTR_USER_DEFINED, attr);\n\n                attr = LLAMA_TOKEN_ATTR_USER_DEFINED;\n            }\n        }\n\n        // sanity checks\n        if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) {\n            special_eog_ids.insert(special_eos_id);\n            LLAMA_LOG_WARN(\"%s: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\\n\", __func__);\n        }\n\n        if (special_eot_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eot_id) == 0) {\n            special_eog_ids.insert(special_eot_id);\n            LLAMA_LOG_WARN(\"%s: special_eot_id is not in special_eog_ids - the tokenizer config may be incorrect\\n\", __func__);\n        }\n\n        if (special_eom_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eom_id) == 0) {\n            special_eog_ids.insert(special_eom_id);\n            LLAMA_LOG_WARN(\"%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\\n\", __func__);\n        }\n\n        // TODO: workaround for o200k_harmony and solar-open tokenizer: the \"<|end|>\" token should not be EOG\n        //       we don't have a good way to detect this, so for now, if we have \"<|return|>\" and \"<|call|>\" tokens (\"<|calls|>\" and \"<|flush|>\" for solar-open),\n        //       we remove the \"<|end|>\" token from the EOG list\n        {\n            bool has_return = false;\n            bool has_call   = false;\n            bool has_end    = false;\n            bool has_flush  = false;\n\n            llama_token end_id = LLAMA_TOKEN_NULL;\n\n            LLAMA_LOG_INFO(\"%s: printing all EOG tokens:\\n\", __func__);\n            for (auto tid : special_eog_ids) {\n                auto & text = id_to_token[tid].text;\n\n                LLAMA_LOG_INFO(\"%s:   - %d ('%s')\\n\", __func__, tid, text.c_str());\n\n                if (text == \"<|return|>\") {\n                    has_return = true;\n                } else if (text == \"<|call|>\" || text == \"<|calls|>\") {\n                    has_call = true;\n                } else if (text == \"<|flush|>\") {\n                    has_flush = true;\n                } else if (text == \"<|end|>\") {\n                    has_end = true;\n                    end_id = tid;\n                }\n            }\n\n            if ((has_return && has_call && has_end) || (has_call && has_flush && has_end)) {\n                special_eog_ids.erase(end_id);\n\n                auto & attr = id_to_token[end_id].attr;\n                attr = LLAMA_TOKEN_ATTR_USER_DEFINED;\n\n                LLAMA_LOG_WARN(\"%s: special_eog_ids contains both '<|return|>' and '<|call|>', or '<|calls|>' and '<|flush|>' tokens, removing '<|end|>' token from EOG list\\n\", __func__);\n            }\n        }\n    }\n\n    // build special tokens cache\n    {\n        for (llama_token id = 0; id < (llama_token) n_tokens; ++id) {\n            if (id_to_token[id].attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED | LLAMA_TOKEN_ATTR_UNKNOWN)) {\n                cache_special_tokens.push_back(id);\n            }\n        }\n\n        std::sort(cache_special_tokens.begin(), cache_special_tokens.end(),\n            [&] (const llama_token a, const llama_token b) {\n                return id_to_token[a].text.size() > id_to_token[b].text.size();\n            }\n        );\n\n        LLAMA_LOG_INFO(\"%s: special tokens cache size = %u\\n\", __func__, (uint32_t) cache_special_tokens.size());\n    }\n\n    // build token to piece cache\n    {\n        size_t size_cache = 0;\n\n        std::vector<std::string> cache(n_tokens);\n\n        for (uint32_t id = 0; id < n_tokens; ++id) {\n            cache[id] = token_to_piece_for_cache(id, true);\n\n            size_cache += cache[id].size();\n        }\n\n        std::swap(cache_token_to_piece, cache);\n\n        LLAMA_LOG_INFO(\"%s: token to piece cache size = %.4f MB\\n\", __func__, size_cache / 1024.0 / 1024.0);\n    }\n\n    // Handle per token attributes\n    //NOTE: Each model customizes per token attributes.\n    //NOTE: Per token attributes are missing from the GGUF file.\n    //TODO: Extract attributes from GGUF file.\n    {\n        auto _contains_any = [] (const std::string & str, const std::vector<std::string_view> & substrs) -> bool {\n            for (const auto & substr : substrs) {\n                if (str.find(substr) != std::string::npos) {\n                    return true;\n                }\n            }\n            return false;\n        };\n\n        auto _set_tokenid_attr = [&] (const llama_token id, llama_token_attr attr, bool value) {\n            uint32_t current = id_to_token.at(id).attr;\n            current = value ? (current | attr) : (current & ~attr);\n            id_to_token[id].attr = (llama_token_attr) current;\n        };\n\n        auto _set_token_attr = [&] (const std::string & token, llama_token_attr attr, bool value) {\n            _set_tokenid_attr(token_to_id.at(token), attr, value);\n        };\n\n        std::string model_name;\n        std::string tokenizer_pre;\n        std::string general_arch;\n\n        ml.get_key(LLM_KV_GENERAL_NAME,  model_name,    false);\n        ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);\n        ml.get_key(LLM_KV_GENERAL_ARCHITECTURE, general_arch, false);\n\n        // model name to lowercase\n        std::transform(model_name.begin(), model_name.end(), model_name.begin(),\n            [] (const std::string::value_type x) {\n                return std::tolower(x);\n            }\n        );\n\n        // set attributes by model/tokenizer/architecture name\n        if (false\n                || _contains_any(tokenizer_pre, {\"jina-v2-de\", \"jina-v2-es\", \"jina-v2-code\"})\n                || _contains_any(general_arch, {\"nomic-bert-moe\", \"jina-bert-v3\"})\n           ) {\n            if (token_to_id.count(\"<mask>\") == 0) {\n                LLAMA_LOG_WARN(\"%s: Mask token is missing in vocab, please reconvert model!\\n\", __func__);\n            } else {\n                _set_token_attr(\"<mask>\", LLAMA_TOKEN_ATTR_LSTRIP, true);\n            }\n        } else if (_contains_any(model_name, {\"phi-3\", \"phi3\"})) {\n            for (auto id : cache_special_tokens) {\n                _set_tokenid_attr(id, LLAMA_TOKEN_ATTR_RSTRIP, true);\n            }\n            for (const auto * token : {\"</s>\"}) {\n                _set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, true);\n            }\n            for (const auto * token : {\"<unk>\", \"<s>\", \"<|endoftext|>\"}) {\n                _set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, false);\n            }\n        } else if (_contains_any(model_name, {\"modern-bert\"})) {\n            if (token_to_id.count(\"[MASK]\") == 0 ) {\n                LLAMA_LOG_WARN(\"%s: Mask token missing in vocab!\\n\", __func__);\n            }\n            else {\n                _set_token_attr(\"[MASK]\", LLAMA_TOKEN_ATTR_LSTRIP, true);\n            }\n        }\n    }\n}\n\nenum llama_vocab_type llama_vocab::impl::get_type() const {\n    return type;\n}\n\nstd::string llama_vocab::impl::type_name() const{\n    switch (type) {\n        case LLAMA_VOCAB_TYPE_NONE:   return \"no vocab\";\n        case LLAMA_VOCAB_TYPE_SPM:    return \"SPM\";\n        case LLAMA_VOCAB_TYPE_BPE:    return \"BPE\";\n        case LLAMA_VOCAB_TYPE_WPM:    return \"WPM\";\n        case LLAMA_VOCAB_TYPE_UGM:    return \"UGM\";\n        case LLAMA_VOCAB_TYPE_RWKV:   return \"RWKV\";\n        case LLAMA_VOCAB_TYPE_PLAMO2: return \"PLaMo2\";\n        default:                      return \"unknown\";\n    }\n}\n\nbool llama_vocab::impl::is_normal(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL;\n}\n\nbool llama_vocab::impl::is_unknown(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNKNOWN;\n}\n\nbool llama_vocab::impl::is_control(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_CONTROL;\n}\n\nbool llama_vocab::impl::is_byte(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_BYTE;\n}\n\nbool llama_vocab::impl::is_user_defined(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_USER_DEFINED;\n}\n\nbool llama_vocab::impl::is_unused(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNUSED;\n}\n\nbool llama_vocab::impl::is_eog(llama_token id) const {\n    return id != LLAMA_TOKEN_NULL && special_eog_ids.count(id) > 0;\n}\n\nuint8_t llama_vocab::impl::token_to_byte(llama_token id) const {\n    GGML_ASSERT(get_type() != LLAMA_VOCAB_TYPE_NONE);\n    GGML_ASSERT(is_byte(id));\n    const auto & token_data = id_to_token.at(id);\n    switch (get_type()) {\n        case LLAMA_VOCAB_TYPE_SPM:\n        case LLAMA_VOCAB_TYPE_UGM: {\n            auto buf = token_data.text.substr(3, 2);\n            return strtol(buf.c_str(), NULL, 16);\n        }\n        case LLAMA_VOCAB_TYPE_BPE: {\n            GGML_ABORT(\"fatal error\");\n        }\n        case LLAMA_VOCAB_TYPE_WPM: {\n            GGML_ABORT(\"fatal error\");\n        }\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n}\n\nllama_token_attr llama_vocab::impl::token_get_attr(llama_token id) const {\n    GGML_ASSERT(type != LLAMA_VOCAB_TYPE_NONE);\n    return id_to_token.at(id).attr;\n}\n\nvoid llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) {\n    LLAMA_LOG_DEBUG(\"%s: initializing tokenizer for type %d\\n\", __func__, type);\n\n    switch (type) {\n        case LLAMA_VOCAB_TYPE_SPM:\n            tokenizer = std::make_unique<llm_tokenizer_spm>(vocab);\n            break;\n        case LLAMA_VOCAB_TYPE_BPE:\n            tokenizer = std::make_unique<llm_tokenizer_bpe>(vocab);\n            break;\n        case LLAMA_VOCAB_TYPE_WPM:\n            tokenizer = std::make_unique<llm_tokenizer_wpm>(vocab);\n            break;\n        case LLAMA_VOCAB_TYPE_UGM:\n            tokenizer = std::make_unique<llm_tokenizer_ugm>(vocab, precompiled_charsmap);\n            break;\n        case LLAMA_VOCAB_TYPE_RWKV:\n            tokenizer = std::make_unique<llm_tokenizer_rwkv>(vocab);\n            break;\n        case LLAMA_VOCAB_TYPE_PLAMO2:\n            tokenizer = std::make_unique<llm_tokenizer_plamo2>(vocab);\n            break;\n        default:\n            GGML_ABORT(\"unsupported vocab type\");\n    }\n}\n\n//\n// (de-) tokenize\n//\n\n// #define PRETOKENIZERDEBUG\n\nvoid llama_vocab::impl::tokenizer_st_partition(std::forward_list<fragment_buffer_variant> & buffer, bool parse_special) const {\n    // for each special token\n    for (const llama_token special_id : cache_special_tokens) {\n        const auto & data = vocab.get_token_data(special_id);\n        const auto & text = data.text;\n\n        if (!parse_special && (data.attr & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_UNKNOWN))) {\n            // Ignore control and unknown tokens when parse_special == false\n            continue;\n            // User-defined tokens are still pre-tokenized before everything else\n            // ref: https://github.com/huggingface/tokenizers/blob/fdd26ba9a3f0c133427aab0423888cbde91362d7/tokenizers/src/tokenizer/mod.rs#L726\n            // This is mostly relevant for neox-style tokenizers (mpt, olmo, stablelm, etc.)\n        }\n\n        // for each text fragment\n        std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin();\n        while (it != buffer.end()) {\n            auto & fragment = (*it);\n\n            // if a fragment is text ( not yet processed )\n            if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                const auto & raw_text = fragment.raw_text;\n\n                auto raw_text_base_offset = fragment.offset;\n                auto raw_text_base_length = fragment.length;\n\n                // loop over the text\n                while (true) {\n                    // find the first occurrence of a given special token in this fragment\n                    //  passing offset argument only limit the \"search area\" but match coordinates\n                    //  are still relative to the source full raw_text\n                    //  string_view begins at pos 0 for the same reason\n                    auto match = std::string_view(raw_text.data(), raw_text_base_offset + raw_text_base_length).find(text, raw_text_base_offset);\n\n                    // no occurrences found, stop processing this fragment for a given special token\n                    if (match == std::string::npos) break;\n\n#ifdef PRETOKENIZERDEBUG\n                    LLAMA_LOG_WARN(\"FF: (%ld %ld %ld) '%s'\\n\", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());\n#endif\n                    auto source = std::distance(buffer.begin(), it);\n\n                    // if match is further than base offset\n                    //  then we have some text to the left of it\n                    if (match > raw_text_base_offset) {\n                        // left\n                        const int64_t left_reminder_offset = raw_text_base_offset + 0;\n                        int64_t left_reminder_length = match - raw_text_base_offset;\n\n                        if (data.attr & LLAMA_TOKEN_ATTR_LSTRIP) {\n                            while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {\n                                left_reminder_length--;\n                            }\n                        }\n\n                        if (left_reminder_length > 0) {\n                            buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);\n                            it++;\n                        }\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"FL: (%ld %ld) '%s'\\n\", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());\n#endif\n                    }\n\n                    // special token\n                    buffer.emplace_after(it, special_id);\n                    it++;\n\n                    // right\n                    if (match + text.length() < raw_text_base_offset + raw_text_base_length) {\n                        int64_t right_reminder_offset = match + text.length();\n                        int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + text.length());\n\n                        if (data.attr & LLAMA_TOKEN_ATTR_RSTRIP) {\n                            while (right_reminder_length > 0 && isspace(raw_text[right_reminder_offset])) {\n                                right_reminder_offset++;\n                                right_reminder_length--;\n                            }\n                        }\n\n                        if (right_reminder_length > 0) {\n                            buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);\n                            it++;\n                        }\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"FR: (%ld %ld) '%s'\\n\", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());\n#endif\n\n                        if (source == 0) {\n                            buffer.erase_after(buffer.before_begin());\n                        } else {\n                            buffer.erase_after(std::next(buffer.begin(), (source - 1)));\n                        }\n\n                        // repeat for the right side\n                        raw_text_base_offset = right_reminder_offset;\n                        raw_text_base_length = right_reminder_length;\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"RR: (%ld %ld) '%s'\\n\", raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str());\n#endif\n                    } else {\n                        if (source == 0) {\n                            buffer.erase_after(buffer.before_begin());\n                        } else {\n                            buffer.erase_after(std::next(buffer.begin(), (source - 1)));\n                        }\n                        break;\n                    }\n                }\n            }\n            it++;\n        }\n    }\n}\n\n// NOTE: avoid ever using this except for building the token_to_piece caches\nstd::string llama_vocab::impl::token_to_piece_for_cache(llama_token token, bool special) const {\n    std::string piece;\n    piece.resize(piece.capacity());  // using string internal cache\n    const int n_chars = vocab.token_to_piece(token, &piece[0], piece.size(), 0, special);\n    if (n_chars < 0) {\n        piece.resize(-n_chars);\n        int check = vocab.token_to_piece(token, &piece[0], piece.size(), 0, special);\n        GGML_ASSERT(check == -n_chars);\n    }\n    else {\n        piece.resize(n_chars);\n    }\n\n    return piece;\n}\n\nstatic void llama_escape_whitespace(std::string & text) {\n    replace_all(text, \" \", \"\\xe2\\x96\\x81\");\n}\n\nstatic void llama_unescape_whitespace(std::string & word) {\n    replace_all(word, \"\\xe2\\x96\\x81\", \" \");\n}\n\nstatic std::string llama_decode_text(const std::string & text) {\n    std::string decoded_text;\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n    for (const auto cpt : cpts) {\n        const auto utf8 = unicode_cpt_to_utf8(cpt);\n        try {\n            decoded_text += unicode_utf8_to_byte(utf8);\n        } catch (const std::out_of_range & /*e*/) {\n            decoded_text += \"[UNK_BYTE_0x\";\n            for (const auto c : utf8) {\n                decoded_text += format(\"%02x\", (uint8_t) c);\n            }\n            decoded_text += text + \"]\";\n        }\n    }\n\n    return decoded_text;\n}\n\nstd::vector<llama_token> llama_vocab::impl::tokenize(\n        const std::string & raw_text,\n        bool add_special,\n        bool parse_special) const {\n    GGML_ASSERT(tokenizer && \"Tokenizer not initialized. Call llama_vocab::init_tokenizer() first.\");\n\n    std::vector<llama_token> output;\n    std::forward_list<fragment_buffer_variant> fragment_buffer;\n\n    if (!raw_text.empty()) {\n        fragment_buffer.emplace_front(raw_text, 0, raw_text.length());\n        tokenizer_st_partition(fragment_buffer, parse_special);\n    }\n\n    switch (get_type()) {\n        case LLAMA_VOCAB_TYPE_SPM:\n            {\n                // OG tokenizer behavior:\n                //\n                // tokenizer.encode('', add_special_tokens=True)  returns [1]\n                // tokenizer.encode('', add_special_tokens=False) returns []\n\n                bool is_prev_special = true;  // prefix with space if first token\n\n                if (add_special && add_bos) {\n                    GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_bos_id);\n                    is_prev_special = true;\n                }\n\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text;\n\n                        // prefix with space if previous is special\n                        if (add_space_prefix && is_prev_special) {\n                            text = ' ';\n                        }\n\n                        text += fragment.raw_text.substr(fragment.offset, fragment.length);\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n                        llama_escape_whitespace(text);\n                        llm_tokenizer_spm_session session(vocab);\n                        session.tokenize(text, output);\n                        is_prev_special = false;\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        output.push_back(fragment.token);\n                        is_prev_special = true;\n                    }\n                }\n\n                if (add_special && add_bos && output.size() >= 2 && output[1] == special_bos_id) {\n                    LLAMA_LOG_WARN(\n                        \"%s: Added a BOS token to the prompt as specified by the model but the prompt \"\n                        \"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. \"\n                        \"Are you sure this is what you want?\\n\", __FUNCTION__);\n                }\n\n                if (add_special && add_eos) {\n                    GGML_ASSERT(special_eos_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_eos_id);\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_BPE:\n            {\n                llm_tokenizer_bpe_session session(vocab, *static_cast<const llm_tokenizer_bpe *>(tokenizer.get()));\n                // it calls some other methods that are not exist in llm_tokenizer,\n                // here just cast it to bpe tokenizer object\n                if (add_special) {\n                    session.append_bos(output);\n                }\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n                        session.tokenize(text, output);\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        session.append(fragment.token, output);\n                    }\n                }\n\n                if (add_special) {\n                    session.append_eos(output);\n                    session.check_double_bos_eos(output);\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_WPM:\n            {\n                if (add_special) {\n                    GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_bos_id);\n                }\n\n                llm_tokenizer_wpm_session session(vocab);\n\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n                        session.tokenize(text, output);\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        output.push_back(fragment.token);\n                    }\n                }\n\n                if (add_special) {\n                    GGML_ASSERT(special_sep_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_sep_id);\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_UGM:\n            {\n                if (add_special && add_bos) {\n                    GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_bos_id);\n                }\n                llm_tokenizer_ugm_session session(vocab, *static_cast<const llm_tokenizer_ugm *>(tokenizer.get()));\n\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n                        session.tokenize(text, output);\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        output.push_back(fragment.token);\n                    }\n                }\n\n                if (add_special && add_bos && output.size() >= 2 && output[1] == special_bos_id) {\n                    LLAMA_LOG_WARN(\n                        \"%s: Added a BOS token to the prompt as specified by the model but the prompt \"\n                        \"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. \"\n                        \"Are you sure this is what you want?\\n\", __FUNCTION__);\n                }\n\n                if (add_special && add_eos) {\n                    GGML_ASSERT(special_eos_id != LLAMA_TOKEN_NULL);\n                    output.push_back(special_eos_id);\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_RWKV:\n            {\n                llm_tokenizer_rwkv_session session(vocab, *static_cast<const llm_tokenizer_rwkv *>(tokenizer.get()));\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n\n                        session.tokenize(text, output);\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        output.push_back(fragment.token);\n                    }\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_PLAMO2:\n            {\n                llm_tokenizer_plamo2_session session(*static_cast<const llm_tokenizer_plamo2 *>(tokenizer.get()));\n                for (const auto & fragment : fragment_buffer) {\n                    if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {\n                        std::string text = fragment.raw_text.substr(fragment.offset, fragment.length);\n\n#ifdef PRETOKENIZERDEBUG\n                        LLAMA_LOG_WARN(\"TT: (%ld %ld %ld) '%s'\\n\", text.length(), fragment.offset, fragment.length, text.c_str());\n#endif\n\n                        session.tokenize(text, output);\n                    } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)\n                        output.push_back(fragment.token);\n                    }\n                }\n            } break;\n        case LLAMA_VOCAB_TYPE_NONE:\n            GGML_ABORT(\"fatal error\");\n    }\n\n    return output;\n}\n\nint32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t length, int32_t lstrip, bool special) const {\n    // ref: https://github.com/ggml-org/llama.cpp/pull/7587#discussion_r1620983843\n    static const int attr_special = LLAMA_TOKEN_ATTR_UNKNOWN | LLAMA_TOKEN_ATTR_CONTROL;\n    const llama_token_attr attr = token_get_attr(token);\n    if (!special && (attr & attr_special)) {\n        return 0;\n    }\n\n    // copy piece chars to output text buffer\n    // skip up to 'lstrip' leading spaces before copying\n    auto _try_copy = [=] (const char * token, size_t size) -> int32_t {\n        if (size >= static_cast<size_t>(std::numeric_limits<int32_t>::max())) {\n            GGML_ABORT(\"invalid token size: %zu exceeds int32_t limit\", size);\n        }\n\n        for (int32_t i = 0; i < lstrip && size && *token == ' '; ++i) {\n            token++;\n            size--;\n        }\n        if (length < (int32_t)size) {\n            return -(int32_t) size;\n        }\n        memcpy(buf, token, size);\n        return (int32_t) size;\n    };\n\n    // if we have a cache - use it\n    {\n        const auto & cache = cache_token_to_piece;\n\n        if (!cache.empty()) {\n            const auto & result = cache.at(token);\n            return _try_copy(result.data(), result.size());\n        }\n    }\n\n    if (0 <= token && token < (int32_t) id_to_token.size()) {\n        const std::string & token_text = id_to_token[token].text;\n        switch (get_type()) {\n            case LLAMA_VOCAB_TYPE_WPM:\n            case LLAMA_VOCAB_TYPE_SPM:\n            case LLAMA_VOCAB_TYPE_UGM: {\n                // NOTE: we accept all unsupported token types,\n                // suppressing them like CONTROL tokens.\n                if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {\n                    return _try_copy(token_text.data(), token_text.size());\n                }\n                if (attr & LLAMA_TOKEN_ATTR_NORMAL) {\n                    std::string result = token_text;\n                    llama_unescape_whitespace(result);\n                    return _try_copy(result.data(), result.size());\n                }\n                if (attr & LLAMA_TOKEN_ATTR_BYTE) {\n                    char byte = (char) token_to_byte(token);\n                    return _try_copy((char*) &byte, 1);\n                }\n                break;\n            }\n            case LLAMA_VOCAB_TYPE_BPE: {\n                // NOTE: we accept all unsupported token types,\n                // suppressing them like CONTROL tokens.\n                if (attr & (attr_special | LLAMA_TOKEN_ATTR_USER_DEFINED)) {\n                    return _try_copy(token_text.data(), token_text.size());\n                }\n                if (attr & LLAMA_TOKEN_ATTR_NORMAL) {\n                    std::string result = llama_decode_text(token_text);\n                    return _try_copy(result.data(), result.size());\n                }\n                break;\n            }\n            case LLAMA_VOCAB_TYPE_RWKV: {\n                std::vector<uint8_t> result = llama_unescape_rwkv_token(token_text);\n\n                // If we don't have enough space, return an error\n                if (result.size() > (size_t)length) {\n                    return -(int)result.size();\n                }\n\n                memcpy(buf, result.data(), result.size());\n                return (int)result.size();\n            }\n            case LLAMA_VOCAB_TYPE_PLAMO2: {\n                // PLaMo-2 uses similar token handling as BPE/SPM\n                if (vocab.is_byte(token)) {\n                    // Handle byte tokens like <0xXX>\n                    if (token_text.length() == 6 && token_text.substr(0, 3) == \"<0x\" && token_text.back() == '>') {\n                        int hex_val = std::stoi(token_text.substr(3, 2), nullptr, 16);\n                        if (length < 1) {\n                            return -1;\n                        }\n                        buf[0] = static_cast<char>(hex_val);\n                        return 1;\n                    }\n                }\n\n                // Normal token - just copy the text\n                std::string result = token_text;\n                return _try_copy(result.data(), result.size());\n            }\n            default:\n                GGML_ABORT(\"fatal error\");\n        }\n    }\n\n    return 0;\n}\n\nconst std::string & llama_vocab::impl::token_to_piece(llama_token token) const {\n    return cache_token_to_piece.at(token);\n}\n\nint32_t llama_vocab::impl::detokenize(\n               const llama_token * tokens,\n                         int32_t   n_tokens,\n                            char * text,\n                         int32_t   text_len_max,\n                            bool   remove_special,\n                            bool   unparse_special) const {\n    if (type == LLAMA_VOCAB_TYPE_NONE) {\n        return 0;\n    }\n\n    GGML_ASSERT(tokenizer && \"Tokenizer not initialized. Call llama_vocab::init_tokenizer() first.\");\n\n    int32_t avail = text_len_max;\n    int32_t total = 0;\n\n    // remove the leading space\n    bool remove_space = add_space_prefix;\n\n    if (remove_special && add_bos) {\n        if (n_tokens > 0 && tokens[0] == special_bos_id) {\n            remove_space = false;\n            n_tokens--;\n            tokens++;\n        }\n    }\n\n    if (remove_special && add_eos) {\n        if (n_tokens > 0 && tokens[n_tokens - 1] == special_eos_id) {\n            n_tokens--;\n        }\n    }\n\n    for (int32_t i = 0; i < n_tokens; ++i) {\n        GGML_ASSERT(avail >= 0);\n        int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special);\n        remove_space = false;\n        if (n_chars < 0) {\n            avail = 0;\n            total -= n_chars;\n        } else if (n_chars > 0) {\n            avail -= n_chars;\n            text  += n_chars;\n            total += n_chars;\n        }\n    }\n\n    if (total > text_len_max) {\n        return -total;\n    }\n\n    if (clean_spaces) {\n        text -= total;  // restart text\n\n        // first pass: characters ?!.,  //TODO: where do these characters come from?\n        const int32_t total1 = total;\n        total = total ? 1 : 0;\n        for (int32_t i = 1; i < total1; ++i) {\n            const char x = text[i];\n            if (text[i - 1] == ' ') {\n                if (x == '?' || x == '!' || x == '.' || x == ',') {  // \" ?\", \" !\", \" .\", \" ,\"\n                    total--;  // remove space\n                }\n            }\n            text[total++] = x;\n        }\n\n        // second pass: strip single apostrophe between spaces\n        const int32_t total2 = total;\n        total = total ? 1 : 0;\n        for (int32_t i = 1; i < total2; ++i) {\n            const char x = text[i];\n            if (x == '\\'' && i + 1 < total2 && text[i - 1] == ' ' && text[i + 1] == ' ') {  // \" ' \"\n                total--;           // remove prev space\n                text[++i] = '\\0';  // remove next space\n            }\n            text[total++] = x;\n        }\n\n        // third pass: apostrophe contractions  //NOTE: this makes sense?\n        const int32_t total3 = total;\n        total = total ? 1 : 0;\n        for (int32_t i = 1; i < total3; ++i) {\n            const char x = text[i];\n            if (text[i - 1] == ' ') {\n                if (x == '\\'' && i + 1 < total3) {\n                    const char x1 = text[i + 1];\n                    if (x1 == 't' || x1 == 'd') {  // \" 't\", \" 'd\"\n                        //total--;  // remove space\n                    } else if (x1 == 's' || x1 == 'm') {  // \" 's\", \" 'm\"\n                        total--;  // remove space\n                    } else if (i + 2 < total3) {\n                        const char x2 = text[i + 2];\n                        if ((x1 == 'l' && x2 == 'l')) {  // \" 'll\"\n                            //total--;  // remove space\n                        } else if ((x1 == 'r' && x2 == 'e') || (x1 == 'v' && x2 == 'e')) {  // \" 're\", \" 've\"\n                            total--;  // remove space\n                        } else {\n                            //total--;  // remove space\n                        }\n                    } else {\n                        //total--;  // remove space\n                    }\n                }\n            }\n            text[total++] = x;\n        }\n    }\n\n    return total <= text_len_max ? total : -total;\n}\n\nvoid llama_vocab::impl::print_info() const {\n    LLAMA_LOG_INFO(\"%s: vocab type            = %s\\n\",     __func__, type_name().c_str());\n    LLAMA_LOG_INFO(\"%s: n_vocab               = %u\\n\",     __func__, vocab.n_tokens());\n    LLAMA_LOG_INFO(\"%s: n_merges              = %u\\n\",     __func__, (uint32_t) bpe_ranks.size());\n\n    // special tokens\n    if (special_bos_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: BOS token             = %d '%s'\\n\", __func__, special_bos_id,     id_to_token.at(special_bos_id).text.c_str() );  }\n    if (special_eos_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: EOS token             = %d '%s'\\n\", __func__, special_eos_id,     id_to_token.at(special_eos_id).text.c_str() );  }\n    if (special_eot_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: EOT token             = %d '%s'\\n\", __func__, special_eot_id,     id_to_token.at(special_eot_id).text.c_str() );  }\n    if (special_eom_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: EOM token             = %d '%s'\\n\", __func__, special_eom_id,     id_to_token.at(special_eom_id).text.c_str() );  }\n    if (special_unk_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: UNK token             = %d '%s'\\n\", __func__, special_unk_id,     id_to_token.at(special_unk_id).text.c_str() );  }\n    if (special_sep_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: SEP token             = %d '%s'\\n\", __func__, special_sep_id,     id_to_token.at(special_sep_id).text.c_str() );  }\n    if (special_pad_id  != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: PAD token             = %d '%s'\\n\", __func__, special_pad_id,     id_to_token.at(special_pad_id).text.c_str() );  }\n    if (special_mask_id != LLAMA_TOKEN_NULL)    { LLAMA_LOG_INFO( \"%s: MASK token            = %d '%s'\\n\", __func__, special_mask_id,    id_to_token.at(special_mask_id).text.c_str() ); }\n\n    if (linefeed_id != LLAMA_TOKEN_NULL)        { LLAMA_LOG_INFO( \"%s: LF token              = %d '%s'\\n\", __func__, linefeed_id,        id_to_token.at(linefeed_id).text.c_str() ); }\n\n    if (special_fim_pre_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM PRE token         = %d '%s'\\n\", __func__, special_fim_pre_id, id_to_token.at(special_fim_pre_id).text.c_str() ); }\n    if (special_fim_suf_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM SUF token         = %d '%s'\\n\", __func__, special_fim_suf_id, id_to_token.at(special_fim_suf_id).text.c_str() ); }\n    if (special_fim_mid_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM MID token         = %d '%s'\\n\", __func__, special_fim_mid_id, id_to_token.at(special_fim_mid_id).text.c_str() ); }\n    if (special_fim_pad_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM PAD token         = %d '%s'\\n\", __func__, special_fim_pad_id, id_to_token.at(special_fim_pad_id).text.c_str() ); }\n    if (special_fim_rep_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM REP token         = %d '%s'\\n\", __func__, special_fim_rep_id, id_to_token.at(special_fim_rep_id).text.c_str() ); }\n    if (special_fim_sep_id != LLAMA_TOKEN_NULL) { LLAMA_LOG_INFO( \"%s: FIM SEP token         = %d '%s'\\n\", __func__, special_fim_sep_id, id_to_token.at(special_fim_sep_id).text.c_str() ); }\n\n    for (const auto & id : special_eog_ids) {\n        LLAMA_LOG_INFO( \"%s: EOG token             = %d '%s'\\n\", __func__, id, id_to_token.at(id).text.c_str() );\n    }\n\n    LLAMA_LOG_INFO(\"%s: max token length      = %d\\n\", __func__, max_token_len);\n}\n\nllama_vocab::llama_vocab() : pimpl(new impl(*this)) {\n}\n\nllama_vocab::~llama_vocab() = default;\n\nvoid llama_vocab::load(llama_model_loader & ml, const LLM_KV & kv) {\n    pimpl->load(ml, kv);\n}\n\nstd::string llama_vocab::get_tokenizer_model() const {\n    return pimpl->tokenizer_model;\n}\n\nstd::string llama_vocab::get_tokenizer_pre() const {\n    return pimpl->tokenizer_pre;\n}\n\nenum llama_vocab_type llama_vocab::get_type() const {\n    return pimpl->type;\n}\n\nenum llama_vocab_pre_type llama_vocab::get_pre_type() const {\n    return pimpl->pre_type;\n}\n\nuint32_t llama_vocab::n_tokens() const {\n    return (uint32_t) pimpl->id_to_token.size();\n}\n\nuint32_t llama_vocab::n_token_types() const {\n    return (uint32_t) pimpl->n_token_types;\n}\n\nstd::string llama_vocab::type_name() const{\n    return pimpl->type_name();\n}\n\nbool llama_vocab::is_normal(llama_token id) const {\n    return pimpl->is_normal(id);\n}\n\nbool llama_vocab::is_unknown(llama_token id) const {\n    return pimpl->is_unknown(id);\n}\n\nbool llama_vocab::is_control(llama_token id) const {\n    return pimpl->is_control(id);\n}\n\nbool llama_vocab::is_byte(llama_token id) const {\n    return pimpl->is_byte(id);\n}\n\nbool llama_vocab::is_user_defined(llama_token id) const {\n    return pimpl->is_user_defined(id);\n}\n\nbool llama_vocab::is_unused(llama_token id) const {\n    return pimpl->is_unused(id);\n}\n\nbool llama_vocab::is_eog(llama_token id) const {\n    return pimpl->is_eog(id);\n}\n\nuint8_t llama_vocab::token_to_byte(llama_token id) const {\n    return pimpl->token_to_byte(id);\n}\n\nllama_token llama_vocab::byte_to_token(uint8_t ch) const {\n    GGML_ASSERT(get_type() != LLAMA_VOCAB_TYPE_NONE);\n    static const char * hex = \"0123456789ABCDEF\";\n    switch (get_type()) {\n        case LLAMA_VOCAB_TYPE_SPM:\n        case LLAMA_VOCAB_TYPE_UGM: {\n            const char buf[7] = { '<', '0', 'x', hex[ch >> 4], hex[ch & 15], '>', 0 };\n            auto token = pimpl->token_to_id.find(buf);\n            if (token != pimpl->token_to_id.end()) {\n                return (*token).second;\n            }\n            // Try to fall back to just the byte as a string\n            const char buf2[2] = { (char)ch, 0 };\n            return pimpl->token_to_id.at(buf2);\n        }\n        case LLAMA_VOCAB_TYPE_WPM:\n        case LLAMA_VOCAB_TYPE_BPE: {\n            return pimpl->token_to_id.at(unicode_byte_to_utf8(ch));\n        }\n        case LLAMA_VOCAB_TYPE_PLAMO2: {\n            // PLaMo-2 uses byte tokens in format <0xXX>\n            char hex_str[8];\n            snprintf(hex_str, sizeof(hex_str), \"<0x%02X>\", ch);\n            return pimpl->token_to_id.at(hex_str);\n        }\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n}\n\nllama_token llama_vocab::text_to_token(const std::string & text) const {\n    GGML_ASSERT(pimpl->type != LLAMA_VOCAB_TYPE_NONE);\n    auto it = pimpl->token_to_id.find(text);\n    if (it != pimpl->token_to_id.end()) {\n        return (*it).second;\n    }\n    return LLAMA_TOKEN_NULL;\n}\n\nconst llama_vocab::token_data & llama_vocab::get_token_data(llama_token id) const {\n    GGML_ASSERT(pimpl->type != LLAMA_VOCAB_TYPE_NONE);\n    return pimpl->id_to_token.at(id);\n}\n\nconst char * llama_vocab::token_get_text(llama_token id) const {\n    GGML_ASSERT(pimpl->type != LLAMA_VOCAB_TYPE_NONE);\n    return pimpl->id_to_token.at(id).text.c_str();\n}\n\nfloat llama_vocab::token_get_score(llama_token id) const {\n    GGML_ASSERT(pimpl->type != LLAMA_VOCAB_TYPE_NONE);\n    return pimpl->id_to_token.at(id).score;\n}\n\nllama_token_attr llama_vocab::token_get_attr(llama_token id) const {\n    return pimpl->token_get_attr(id);\n}\n\nllama_token llama_vocab::token_bos() const {\n    return pimpl->special_bos_id;\n}\n\nllama_token llama_vocab::token_eos() const {\n    return pimpl->special_eos_id;\n}\n\nllama_token llama_vocab::token_eot() const {\n    return pimpl->special_eot_id;\n}\n\nllama_token llama_vocab::token_eom() const {\n    return pimpl->special_eom_id;\n}\n\nllama_token llama_vocab::token_unk() const {\n    return pimpl->special_unk_id;\n}\n\nllama_token llama_vocab::token_sep() const {\n    return pimpl->special_sep_id;\n}\n\nllama_token llama_vocab::token_nl() const {\n    return pimpl->linefeed_id;\n}\n\nllama_token llama_vocab::token_pad() const {\n    return pimpl->special_pad_id;\n}\n\nllama_token llama_vocab::token_prefix() const {\n    return pimpl->special_fim_pre_id;\n}\n\nllama_token llama_vocab::token_middle() const {\n    return pimpl->special_fim_mid_id;\n}\n\nllama_token llama_vocab::token_suffix() const {\n    return pimpl->special_fim_suf_id;\n}\n\nllama_token llama_vocab::token_fim_pre() const {\n    return pimpl->special_fim_pre_id;\n}\n\nllama_token llama_vocab::token_fim_suf() const {\n    return pimpl->special_fim_suf_id;\n}\n\nllama_token llama_vocab::token_fim_mid() const {\n    return pimpl->special_fim_mid_id;\n}\n\nllama_token llama_vocab::token_fim_pad() const {\n    return pimpl->special_fim_pad_id;\n}\n\nllama_token llama_vocab::token_fim_rep() const {\n    return pimpl->special_fim_rep_id;\n}\n\nllama_token llama_vocab::token_fim_sep() const {\n    return pimpl->special_fim_sep_id;\n}\n\nllama_token llama_vocab::token_mask() const {\n    return pimpl->special_mask_id;\n}\n\nbool llama_vocab::get_add_space_prefix() const {\n    return pimpl->add_space_prefix;\n}\n\nbool llama_vocab::get_add_bos() const {\n    return pimpl->add_bos;\n}\n\nbool llama_vocab::get_add_eos() const {\n    return pimpl->add_eos;\n}\n\nbool llama_vocab::get_add_sep() const {\n    return pimpl->add_sep;\n}\n\nbool llama_vocab::get_ignore_merges() const {\n    return pimpl->ignore_merges;\n}\n\nbool llama_vocab::get_clean_spaces() const {\n    return pimpl->clean_spaces;\n}\n\nbool llama_vocab::get_remove_extra_whitespaces() const {\n    return pimpl->remove_extra_whitespaces;\n}\n\nbool llama_vocab::get_escape_whitespaces() const {\n    return pimpl->escape_whitespaces;\n}\n\nbool llama_vocab::get_treat_whitespace_as_suffix() const {\n    return pimpl->treat_whitespace_as_suffix;\n}\n\nint llama_vocab::max_token_len() const {\n    return pimpl->max_token_len;\n}\n\nint llama_vocab::find_bpe_rank(const std::string & token_left, const std::string & token_right) const {\n    GGML_ASSERT(token_left.find(' ')   == std::string::npos);\n    GGML_ASSERT(token_left.find('\\n')  == std::string::npos);\n    GGML_ASSERT(token_right.find(' ')  == std::string::npos);\n    GGML_ASSERT(token_right.find('\\n') == std::string::npos);\n\n    auto it = pimpl->bpe_ranks.find(std::make_pair(token_left, token_right));\n    if (it == pimpl->bpe_ranks.end()) {\n        return -1;\n    }\n\n    return it->second;\n}\n\nstd::vector<std::string> llama_vocab::get_bpe_merges() const {\n    std::vector<std::string> result(pimpl->bpe_ranks.size());\n\n    for (const auto & pair : pimpl->bpe_ranks) {\n        result[pair.second] = pair.first.first + \" \" + pair.first.second;\n    }\n\n    return result;\n}\n\nstd::vector<char> llama_vocab::get_precompiled_charsmap() const {\n    return pimpl->precompiled_charsmap;\n}\n\nint32_t llama_vocab::tokenize(\n                  const char * text,\n                     int32_t   text_len,\n                 llama_token * tokens,\n                     int32_t   n_tokens_max,\n                        bool   add_special,\n                        bool   parse_special) const {\n    auto res = tokenize(std::string(text, text_len), add_special, parse_special);\n    if (res.size() >= static_cast<size_t>(std::numeric_limits<int32_t>::max())) {\n        LLAMA_LOG_ERROR(\"%s: tokenization result size %zu exceeds int32_t limit\\n\", __func__, res.size());\n        return std::numeric_limits<int32_t>::min();\n    }\n\n    if (n_tokens_max < (int) res.size()) {\n        // LLAMA_LOG_ERROR(\"%s: too many tokens\\n\", __func__);\n        return -((int) res.size());\n    }\n\n    for (size_t i = 0; i < res.size(); i++) {\n        tokens[i] = res[i];\n    }\n\n    return res.size();\n}\n\nstd::vector<llama_token> llama_vocab::tokenize(\n        const std::string & raw_text,\n        bool add_special,\n        bool parse_special) const {\n    return pimpl->tokenize(raw_text, add_special, parse_special);\n}\n\nconst std::string & llama_vocab::token_to_piece(llama_token token) const {\n    return pimpl->token_to_piece(token);\n}\n\nint32_t llama_vocab::token_to_piece(llama_token token, char * buf, int32_t length, int32_t lstrip, bool special) const {\n    return pimpl->token_to_piece(token, buf, length, lstrip, special);\n}\n\nint32_t llama_vocab::detokenize(\n               const llama_token * tokens,\n                         int32_t   n_tokens,\n                            char * text,\n                         int32_t   text_len_max,\n                            bool   remove_special,\n                            bool   unparse_special) const {\n    return pimpl->detokenize(tokens, n_tokens, text, text_len_max, remove_special, unparse_special);\n}\n\nstd::string llama_vocab::detokenize(const std::vector<llama_token> & tokens, bool special) const {\n    std::string text;\n    text.resize(std::max(text.capacity(), tokens.size()));\n    int32_t n_chars = detokenize(tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);\n    if (n_chars < 0) {\n        text.resize(-n_chars);\n        n_chars = detokenize(tokens.data(), (int32_t)tokens.size(), &text[0], (int32_t)text.size(), false, special);\n        GGML_ASSERT(n_chars <= (int32_t)text.size());  // whitespace trimming is performed after per-token detokenization\n    }\n\n    text.resize(n_chars);\n\n    // NOTE: the original tokenizer decodes bytes after collecting the pieces.\n    return text;\n}\n\nvoid llama_vocab::print_info() const {\n    pimpl->print_info();\n}\n\n//\n// interface implementation\n//\n\nint32_t llama_vocab_n_tokens(const struct llama_vocab * vocab) {\n    return vocab->n_tokens();\n}\n\n// deprecated\nint32_t llama_n_vocab(const struct llama_vocab * vocab) {\n    return llama_vocab_n_tokens(vocab);\n}\n\nenum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab) {\n    return vocab->get_type();\n}\n\nconst char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token) {\n    return vocab->token_get_text(token);\n}\n\nfloat llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token) {\n    return vocab->token_get_score(token);\n}\n\nenum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token) {\n    return vocab->token_get_attr(token);\n}\n\nbool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token) {\n    return vocab->is_eog(token);\n}\n\nbool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token) {\n    return vocab->is_control(token);\n}\n\nllama_token llama_vocab_bos(const struct llama_vocab * vocab) {\n    return vocab->token_bos();\n}\n\nllama_token llama_vocab_eos(const struct llama_vocab * vocab) {\n    return vocab->token_eos();\n}\n\nllama_token llama_vocab_eot(const struct llama_vocab * vocab) {\n    return vocab->token_eot();\n}\n\n// deprecated\nllama_token llama_vocab_cls(const struct llama_vocab * vocab) {\n    return vocab->token_bos();\n}\n\nllama_token llama_vocab_sep(const struct llama_vocab * vocab) {\n    return vocab->token_sep();\n}\n\nllama_token llama_vocab_nl (const struct llama_vocab * vocab) {\n    return vocab->token_nl();\n}\n\nllama_token llama_vocab_pad(const struct llama_vocab * vocab) {\n    return vocab->token_pad();\n}\n\nbool llama_vocab_get_add_bos(const struct llama_vocab * vocab) {\n    return vocab->get_add_bos();\n}\n\nbool llama_vocab_get_add_eos(const struct llama_vocab * vocab) {\n    return vocab->get_add_eos();\n}\n\nbool llama_vocab_get_add_sep(const struct llama_vocab * vocab) {\n    return vocab->get_add_sep();\n}\n\nllama_token llama_vocab_fim_pre(const struct llama_vocab * vocab) {\n    return vocab->token_fim_pre();\n}\n\nllama_token llama_vocab_fim_suf(const struct llama_vocab * vocab) {\n    return vocab->token_fim_suf();\n}\n\nllama_token llama_vocab_fim_mid(const struct llama_vocab * vocab) {\n    return vocab->token_fim_mid();\n}\n\nllama_token llama_vocab_fim_pad(const struct llama_vocab * vocab) {\n    return vocab->token_fim_pad();\n}\n\nllama_token llama_vocab_fim_rep(const struct llama_vocab * vocab) {\n    return vocab->token_fim_rep();\n}\n\nllama_token llama_vocab_fim_sep(const struct llama_vocab * vocab) {\n    return vocab->token_fim_sep();\n}\n\nllama_token llama_vocab_mask(const struct llama_vocab* vocab) {\n    return vocab->token_mask();\n}\n\n// deprecated\nconst char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token) {\n    return llama_vocab_get_text(vocab, token);\n}\n\n// deprecated\nfloat llama_token_get_score(const struct llama_vocab * vocab, llama_token token) {\n    return llama_vocab_get_score(vocab, token);\n}\n\n// deprecated\nenum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token) {\n    return llama_vocab_get_attr(vocab, token);\n}\n\n// deprecated\nbool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token) {\n    return llama_vocab_is_eog(vocab, token);\n}\n\n// deprecated\nbool llama_token_is_control(const struct llama_vocab * vocab, llama_token token) {\n    return llama_vocab_is_control(vocab, token);\n}\n\n// deprecated\nllama_token llama_token_bos(const struct llama_vocab * vocab) {\n    return llama_vocab_bos(vocab);\n}\n\n// deprecated\nllama_token llama_token_eos(const struct llama_vocab * vocab) {\n    return llama_vocab_eos(vocab);\n}\n\n// deprecated\nllama_token llama_token_eot(const struct llama_vocab * vocab) {\n    return llama_vocab_eot(vocab);\n}\n\n// deprecated\nllama_token llama_token_cls(const struct llama_vocab * vocab) {\n    //return llama_vocab_cls(vocab);\n    return llama_vocab_bos(vocab); // avoid deprecation warning\n}\n\n// deprecated\nllama_token llama_token_sep(const struct llama_vocab * vocab) {\n    return llama_vocab_sep(vocab);\n}\n\n// deprecated\nllama_token llama_token_nl (const struct llama_vocab * vocab) {\n    return llama_vocab_nl(vocab);\n}\n\n// deprecated\nllama_token llama_token_pad(const struct llama_vocab * vocab) {\n    return llama_vocab_pad(vocab);\n}\n\n// deprecated\nbool llama_add_bos_token(const struct llama_vocab * vocab) {\n    return llama_vocab_get_add_bos(vocab);\n}\n\n// deprecated\nbool llama_add_eos_token(const struct llama_vocab * vocab) {\n    return llama_vocab_get_add_eos(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_pre(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_pre(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_suf(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_suf(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_mid(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_mid(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_pad(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_pad(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_rep(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_rep(vocab);\n}\n\n// deprecated\nllama_token llama_token_fim_sep(const struct llama_vocab * vocab) {\n    return llama_vocab_fim_sep(vocab);\n}\n\n//\n// tokenization\n//\n\nint32_t llama_tokenize(\n    const struct llama_vocab * vocab,\n                  const char * text,\n                     int32_t   text_len,\n                 llama_token * tokens,\n                     int32_t   n_tokens_max,\n                        bool   add_special,\n                        bool   parse_special) {\n    return vocab->tokenize(text, text_len, tokens, n_tokens_max, add_special, parse_special);\n}\n\nint32_t llama_token_to_piece(\n    const struct llama_vocab * vocab,\n                 llama_token   token,\n                        char * buf,\n                     int32_t   length,\n                     int32_t   lstrip,\n                        bool   special) {\n    return vocab->token_to_piece(token, buf, length, lstrip, special);\n}\n\nint32_t llama_detokenize(\n    const struct llama_vocab * vocab,\n           const llama_token * tokens,\n                     int32_t   n_tokens,\n                        char * text,\n                     int32_t   text_len_max,\n                        bool   remove_special,\n                        bool   unparse_special) {\n    return vocab->detokenize(tokens, n_tokens, text, text_len_max, remove_special, unparse_special);\n}\n"
  },
  {
    "path": "examples/talk-llama/llama-vocab.h",
    "content": "#pragma once\n\n#include \"llama.h\"\n\n#include <string>\n#include <vector>\n#include <memory>\n\n// pre-tokenization types\nenum llama_vocab_pre_type {\n    LLAMA_VOCAB_PRE_TYPE_DEFAULT         = 0,\n    LLAMA_VOCAB_PRE_TYPE_LLAMA3          = 1,\n    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM    = 2,\n    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER  = 3,\n    LLAMA_VOCAB_PRE_TYPE_FALCON          = 4,\n    LLAMA_VOCAB_PRE_TYPE_MPT             = 5,\n    LLAMA_VOCAB_PRE_TYPE_STARCODER       = 6,\n    LLAMA_VOCAB_PRE_TYPE_GPT2            = 7,\n    LLAMA_VOCAB_PRE_TYPE_REFACT          = 8,\n    LLAMA_VOCAB_PRE_TYPE_COMMAND_R       = 9,\n    LLAMA_VOCAB_PRE_TYPE_STABLELM2       = 10,\n    LLAMA_VOCAB_PRE_TYPE_QWEN2           = 11,\n    LLAMA_VOCAB_PRE_TYPE_OLMO            = 12,\n    LLAMA_VOCAB_PRE_TYPE_DBRX            = 13,\n    LLAMA_VOCAB_PRE_TYPE_SMAUG           = 14,\n    LLAMA_VOCAB_PRE_TYPE_PORO            = 15,\n    LLAMA_VOCAB_PRE_TYPE_CHATGLM3        = 16,\n    LLAMA_VOCAB_PRE_TYPE_CHATGLM4        = 17,\n    LLAMA_VOCAB_PRE_TYPE_VIKING          = 18,\n    LLAMA_VOCAB_PRE_TYPE_JAIS            = 19,\n    LLAMA_VOCAB_PRE_TYPE_TEKKEN          = 20,\n    LLAMA_VOCAB_PRE_TYPE_SMOLLM          = 21,\n    LLAMA_VOCAB_PRE_TYPE_CODESHELL       = 22,\n    LLAMA_VOCAB_PRE_TYPE_BLOOM           = 23,\n    LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH    = 24,\n    LLAMA_VOCAB_PRE_TYPE_EXAONE          = 25,\n    LLAMA_VOCAB_PRE_TYPE_CHAMELEON       = 26,\n    LLAMA_VOCAB_PRE_TYPE_MINERVA         = 27,\n    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM   = 28,\n    LLAMA_VOCAB_PRE_TYPE_GPT4O           = 29,\n    LLAMA_VOCAB_PRE_TYPE_SUPERBPE        = 30,\n    LLAMA_VOCAB_PRE_TYPE_TRILLION        = 31,\n    LLAMA_VOCAB_PRE_TYPE_BAILINGMOE      = 32,\n    LLAMA_VOCAB_PRE_TYPE_LLAMA4          = 33,\n    LLAMA_VOCAB_PRE_TYPE_PIXTRAL         = 34,\n    LLAMA_VOCAB_PRE_TYPE_SEED_CODER      = 35,\n    LLAMA_VOCAB_PRE_TYPE_HUNYUAN         = 36,\n    LLAMA_VOCAB_PRE_TYPE_KIMI_K2         = 37,\n    LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE   = 38,\n    LLAMA_VOCAB_PRE_TYPE_GROK_2          = 39,\n    LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40,\n    LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2      = 41,\n    LLAMA_VOCAB_PRE_TYPE_AFMOE           = 42,\n    LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN      = 43,\n    LLAMA_VOCAB_PRE_TYPE_YOUTU           = 44,\n    LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE      = 45,\n    LLAMA_VOCAB_PRE_TYPE_QWEN35          = 46,\n    LLAMA_VOCAB_PRE_TYPE_TINY_AYA        = 47,\n    LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM       = 48,\n    LLAMA_VOCAB_PRE_TYPE_JAIS2           = 49,\n};\n\nstruct LLM_KV;\nstruct llama_model_loader;\n\nstruct llama_vocab {\n    struct token_data {\n        std::string      text;\n        float            score;\n        llama_token_attr attr;\n    };\n\n    llama_vocab();\n    ~llama_vocab();\n\n    void load(llama_model_loader & ml, const LLM_KV & kv);\n\n    std::string get_tokenizer_model() const;\n    std::string get_tokenizer_pre() const;\n\n    enum llama_vocab_type     get_type()     const;\n    enum llama_vocab_pre_type get_pre_type() const;\n\n    uint32_t n_tokens() const;\n    uint32_t n_token_types() const;\n\n    std::string type_name() const;\n\n    bool is_normal      (llama_token id) const;\n    bool is_unknown     (llama_token id) const;\n    bool is_control     (llama_token id) const;\n    bool is_byte        (llama_token id) const;\n    bool is_user_defined(llama_token id) const;\n    bool is_unused      (llama_token id) const;\n    bool is_eog         (llama_token id) const;\n\n    uint8_t     token_to_byte(llama_token id) const;\n    llama_token byte_to_token(uint8_t ch)     const;\n\n    llama_token text_to_token(const std::string & text) const;\n\n    const token_data & get_token_data(llama_token id) const;\n\n    const char *     token_get_text (llama_token id) const;\n    float            token_get_score(llama_token id) const;\n    llama_token_attr token_get_attr (llama_token id) const;\n\n    llama_token token_bos() const;\n    llama_token token_eos() const;\n    llama_token token_eot() const;\n    llama_token token_eom() const;\n    llama_token token_unk() const;\n    llama_token token_sep() const;\n    llama_token token_nl () const;\n    llama_token token_pad() const;\n    llama_token token_mask() const;\n\n    llama_token token_prefix() const;\n    llama_token token_middle() const;\n    llama_token token_suffix() const;\n\n    llama_token token_fim_pre() const;\n    llama_token token_fim_suf() const;\n    llama_token token_fim_mid() const;\n    llama_token token_fim_pad() const;\n    llama_token token_fim_rep() const;\n    llama_token token_fim_sep() const;\n\n    bool get_add_space_prefix          () const;\n    bool get_add_bos                   () const;\n    bool get_add_eos                   () const;\n    bool get_add_sep                   () const;\n    bool get_ignore_merges             () const;\n    bool get_clean_spaces              () const;\n    bool get_remove_extra_whitespaces  () const;\n    bool get_escape_whitespaces        () const;\n    bool get_treat_whitespace_as_suffix() const;\n\n    int max_token_len() const;\n\n    int find_bpe_rank(const std::string & token_left, const std::string & token_right) const;\n    std::vector<std::string> get_bpe_merges() const;\n\n    std::vector<char> get_precompiled_charsmap() const;\n\n    int32_t tokenize(\n                   const char * text,\n                      int32_t   text_len,\n                  llama_token * tokens,\n                      int32_t   n_tokens_max,\n                         bool   add_special,\n                         bool   parse_special) const;\n\n    std::vector<llama_token> tokenize(\n            const std::string & raw_text,\n                         bool   add_special,\n                         bool   parse_special = false) const;\n\n    // does not write null-terminator to buf\n    int32_t token_to_piece(\n                  llama_token   token,\n                         char * buf,\n                      int32_t   length,\n                      int32_t   lstrip,\n                         bool   special) const;\n\n    // use cached data\n    const std::string & token_to_piece(llama_token token) const;\n\n    int32_t detokenize(\n            const llama_token * tokens,\n                      int32_t   n_tokens,\n                         char * text,\n                      int32_t   text_len_max,\n                         bool   remove_special,\n                         bool   unparse_special) const;\n\n    std::string detokenize(\n            const std::vector<llama_token> & tokens,\n                                      bool   special) const;\n\n    void print_info() const;\n\nprivate:\n    struct impl;\n    std::unique_ptr<impl> pimpl;\n};\n"
  },
  {
    "path": "examples/talk-llama/llama.cpp",
    "content": "#include \"llama.h\"\n\n#include \"ggml-cpp.h\"\n#include \"llama-impl.h\"\n\n#include \"llama-chat.h\"\n#include \"llama-context.h\"\n#include \"llama-mmap.h\"\n#include \"llama-vocab.h\"\n#include \"llama-model-loader.h\"\n#include \"llama-model-saver.h\"\n#include \"llama-model.h\"\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n#include \"gguf.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cinttypes>\n#include <cstddef>\n#include <cstdint>\n#include <cstdio>\n#include <cstring>\n#include <ctime>\n#include <stdexcept>\n\n#if defined(_MSC_VER)\n#pragma warning(disable: 4244 4267) // possible loss of data\n#endif\n\n//\n// interface implementation\n//\n\nconst char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type) {\n    switch (flash_attn_type) {\n        case LLAMA_FLASH_ATTN_TYPE_AUTO:\n            return \"auto\";\n        case LLAMA_FLASH_ATTN_TYPE_DISABLED:\n            return \"disabled\";\n        case LLAMA_FLASH_ATTN_TYPE_ENABLED:\n            return \"enabled\";\n    }\n    GGML_ABORT(\"fatal error\");\n}\n\nstruct llama_device_memory_data {\n    int64_t total;\n    int64_t free;\n    llama_memory_breakdown_data mb;\n};\n\nstatic std::vector<llama_device_memory_data> llama_get_device_memory_data(\n        const char * path_model, const llama_model_params * mparams, const llama_context_params * cparams,\n        std::vector<ggml_backend_dev_t> & devs, uint32_t & hp_ngl, uint32_t & hp_n_ctx_train, uint32_t & hp_n_expert,\n        const ggml_log_level log_level) {\n    struct user_data_t {\n        struct {\n            ggml_log_callback callback;\n            void * user_data;\n        } original_logger;\n        ggml_log_level min_level; // prints below this log level go to debug log\n    };\n    user_data_t ud;\n    llama_log_get(&ud.original_logger.callback, &ud.original_logger.user_data);\n    ud.min_level = log_level;\n\n    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {\n        const user_data_t * ud = (const user_data_t *) user_data;\n        const ggml_log_level level_eff = level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG;\n        ud->original_logger.callback(level_eff, text, ud->original_logger.user_data);\n    }, &ud);\n\n    llama_model_params mparams_copy = *mparams;\n    mparams_copy.no_alloc  = true;\n    mparams_copy.use_mmap  = false;\n    mparams_copy.use_mlock = false;\n\n    llama_model * model = llama_model_load_from_file(path_model, mparams_copy);\n    if (model == nullptr) {\n        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);\n        throw std::runtime_error(\"failed to load model\");\n    }\n\n    llama_context * ctx = llama_init_from_model(model, *cparams);\n    if (ctx == nullptr) {\n        llama_model_free(model);\n        llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);\n        throw std::runtime_error(\"failed to create llama_context from model\");\n    }\n\n    std::vector<llama_device_memory_data> ret(model->devices.size());\n\n    std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> memory_breakdown = ctx->memory_breakdown();\n\n    for (const auto & [buft, mb] : memory_breakdown) {\n        if (ggml_backend_buft_is_host(buft)) {\n            continue;\n        }\n\n        ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);\n        if (!dev) {\n            continue;\n        }\n        for (size_t i = 0; i < ret.size(); i++) {\n            if (model->devices[i] == dev) {\n                ret[i].mb.model   += mb.model;\n                ret[i].mb.context += mb.context;\n                ret[i].mb.compute += mb.compute;\n                break;\n            }\n        }\n    }\n    for (size_t i = 0; i < ret.size(); i++) {\n        size_t free;\n        size_t total;\n        ggml_backend_dev_memory(model->devices[i], &free, &total);\n\n        // devices can return 0 bytes for free and total memory if they do not\n        // have any to report. in this case, we will use the host memory as a fallback\n        // fixes: https://github.com/ggml-org/llama.cpp/issues/18577\n        if (free == 0 && total == 0) {\n            ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n            if (cpu_dev == nullptr) {\n                throw std::runtime_error(format(\"%s: no CPU backend found\", __func__));\n            }\n            ggml_backend_dev_memory(cpu_dev, &free, &total);\n        }\n        ret[i].free  = free;\n        ret[i].total = total;\n    }\n\n    devs           = model->devices;\n    hp_ngl         = model->hparams.n_layer;\n    hp_n_ctx_train = model->hparams.n_ctx_train;\n    hp_n_expert    = model->hparams.n_expert;\n\n    llama_memory_breakdown_print(ctx); // goes to debug log\n\n    llama_free(ctx);\n    llama_model_free(model);\n    llama_log_set(ud.original_logger.callback, ud.original_logger.user_data);\n    return ret;\n}\n\n// enum to identify part of a layer for distributing its tensors:\nenum layer_fraction_t {\n    LAYER_FRACTION_NONE = 0, // nothing\n    LAYER_FRACTION_ATTN = 1, // attention\n    LAYER_FRACTION_UP   = 2, // attention + up\n    LAYER_FRACTION_GATE = 3, // attention + up + gate\n    LAYER_FRACTION_MOE  = 4, // everything but sparse MoE weights\n};\n// this enum is only used in llama_params_fit_impl but needs to be defined outside of it to fix a Windows compilation issue\n\nclass llama_params_fit_exception : public std::runtime_error {\n    using std::runtime_error::runtime_error;\n};\n\nstatic void llama_params_fit_impl(\n        const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,\n        float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,\n        size_t * margins_s, uint32_t n_ctx_min, enum ggml_log_level log_level) {\n    constexpr int64_t MiB = 1024*1024;\n    typedef std::vector<llama_device_memory_data> dmds_t;\n    const llama_model_params default_mparams = llama_model_default_params();\n\n    std::vector<ggml_backend_dev_t> devs;\n    uint32_t hp_ngl = 0; // hparams.n_gpu_layers\n    uint32_t hp_nct = 0; // hparams.n_ctx_train\n    uint32_t hp_nex = 0; // hparams.n_expert\n\n    // step 1: get data for default parameters and check whether any changes are necessary in the first place\n\n    LLAMA_LOG_DEBUG(\"%s: getting device memory data for initial parameters:\\n\", __func__);\n    const dmds_t dmds_full = llama_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);\n    const size_t nd = devs.size(); // number of devices\n    if (nd == 0) {\n        LLAMA_LOG_INFO(\"%s: no devices with dedicated memory found\\n\", __func__);\n        return;\n    }\n\n    std::vector<int64_t> margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits\n    margins.reserve(nd);\n    for (size_t id = 0; id < nd; id++) {\n        margins.push_back(margins_s[id]);\n    }\n\n    std::vector<std::string> dev_names;\n    {\n        dev_names.reserve(nd);\n        size_t max_length = 0;\n        for (ggml_backend_dev_t dev : devs) {\n            std::string name = ggml_backend_dev_name(dev);\n            name += \" (\";\n            name += ggml_backend_dev_description(dev);\n            name += \")\";\n            dev_names.push_back(name);\n            max_length = std::max(max_length, name.length());\n        }\n        for (std::string & dn : dev_names) {\n            dn.insert(dn.end(), max_length - dn.length(), ' ');\n        }\n    }\n\n    int64_t sum_free            = 0;\n    int64_t sum_projected_free  = 0;\n    int64_t sum_projected_used  = 0;\n    int64_t sum_projected_model = 0;\n    std::vector<int64_t> projected_free_per_device;\n    projected_free_per_device.reserve(nd);\n\n    if (nd > 1) {\n        LLAMA_LOG_INFO(\"%s: projected memory use with initial parameters [MiB]:\\n\", __func__);\n    }\n    for (size_t id = 0; id < nd; id++) {\n        const llama_device_memory_data & dmd = dmds_full[id];\n\n        const int64_t projected_used = dmd.mb.total();\n        const int64_t projected_free = dmd.free - projected_used;\n        projected_free_per_device.push_back(projected_free);\n\n        sum_free            += dmd.free;\n        sum_projected_used  += projected_used;\n        sum_projected_free  += projected_free;\n        sum_projected_model += dmd.mb.model;\n\n        if (nd > 1) {\n            LLAMA_LOG_INFO(\"%s:   - %s: %6\" PRId64 \" total, %6\" PRId64 \" used, %6\" PRId64 \" free vs. target of %6\" PRId64 \"\\n\",\n                __func__, dev_names[id].c_str(), dmd.total/MiB, projected_used/MiB, projected_free/MiB, margins[id]/MiB);\n        }\n    }\n    assert(sum_free >= 0 && sum_projected_used >= 0);\n    LLAMA_LOG_INFO(\"%s: projected to use %\" PRId64 \" MiB of device memory vs. %\" PRId64 \" MiB of free device memory\\n\",\n        __func__, sum_projected_used/MiB, sum_free/MiB);\n    if (nd == 1) {\n        if (projected_free_per_device[0] >= margins[0]) {\n            LLAMA_LOG_INFO(\"%s: will leave %\" PRId64 \" >= %\" PRId64 \" MiB of free device memory, no changes needed\\n\",\n                __func__, projected_free_per_device[0]/MiB, margins[0]/MiB);\n            return;\n        }\n    } else {\n        bool changes_needed = false;\n        for (size_t id = 0; id < nd; id++) {\n            if (projected_free_per_device[id] < margins[id]) {\n                changes_needed = true;\n                break;\n            }\n        }\n        if (!changes_needed) {\n            LLAMA_LOG_INFO(\"%s: targets for free memory can be met on all devices, no changes needed\\n\", __func__);\n            return;\n        }\n    }\n\n    // step 2: try reducing memory use by reducing the context size\n\n    {\n        int64_t global_surplus = sum_projected_free;\n        for (size_t id = 0; id < nd; id++) {\n            global_surplus -= margins[id];\n        }\n        if (global_surplus < 0) {\n            if (nd == 1) {\n                LLAMA_LOG_INFO(\"%s: cannot meet free memory target of %\" PRId64 \" MiB, need to reduce device memory by %\" PRId64 \" MiB\\n\",\n                    __func__, margins[0]/MiB, -global_surplus/MiB);\n            } else {\n                LLAMA_LOG_INFO(\n                    \"%s: cannot meet free memory targets on all devices, need to use %\" PRId64 \" MiB less in total\\n\",\n                    __func__, -global_surplus/MiB);\n            }\n            if (cparams->n_ctx == 0) {\n                if (hp_nct > n_ctx_min) {\n                    int64_t sum_used_target = sum_free;\n                    for (size_t id = 0; id < nd; id++) {\n                        sum_used_target -= margins[id];\n                    }\n                    if (nd > 1) {\n                        // for multiple devices we need to be more conservative in terms of how much context we think can fit:\n                        //   - for dense models only whole layers can be assigned to devices\n                        //   - for MoE models only whole tensors can be assigned to devices, which we estimate to be <= 1/3 of a layer\n                        //   - on average we expect a waste of 0.5 layers/tensors per device\n                        //   - use slightly more than the expected average for nd devices to be safe\n                        const int64_t model_per_layer = sum_projected_model / std::min(uint32_t(mparams->n_gpu_layers), hp_ngl);\n                        sum_used_target -= (nd + 1) * model_per_layer / (hp_nex == 0 ? 2 : 6);\n                    }\n\n                    int64_t sum_projected_used_min_ctx = 0;\n                    cparams->n_ctx = n_ctx_min;\n                    const dmds_t dmds_min_ctx = llama_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);\n                    for (const auto & dmd : dmds_min_ctx) {\n                        sum_projected_used_min_ctx += dmd.mb.total();\n                    }\n                    if (sum_used_target > sum_projected_used_min_ctx) {\n                        // linear interpolation between minimum and maximum context size:\n                        cparams->n_ctx += (hp_nct - n_ctx_min) * (sum_used_target - sum_projected_used_min_ctx)\n                            / (sum_projected_used - sum_projected_used_min_ctx);\n                        cparams->n_ctx = std::max(cparams->n_ctx - cparams->n_ctx % 256, n_ctx_min); // round down context for CUDA backend\n\n                        const int64_t bytes_per_ctx = (sum_projected_used - sum_projected_used_min_ctx) / (hp_nct - n_ctx_min);\n                        const int64_t memory_reduction = (hp_nct - cparams->n_ctx) * bytes_per_ctx;\n                        LLAMA_LOG_INFO(\"%s: context size reduced from %\" PRIu32 \" to %\" PRIu32 \" -> need %\" PRId64 \" MiB less memory in total\\n\",\n                            __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);\n                        if (nd == 1) {\n                            LLAMA_LOG_INFO(\"%s: entire model can be fit by reducing context\\n\", __func__);\n                            return;\n                        }\n                        LLAMA_LOG_INFO(\"%s: entire model should be fit across devices by reducing context\\n\", __func__);\n                    } else {\n                        const int64_t memory_reduction = sum_projected_used - sum_projected_used_min_ctx;\n                        LLAMA_LOG_INFO(\"%s: context size reduced from %\" PRIu32 \" to %\" PRIu32 \" -> need %\" PRId64 \" MiB less memory in total\\n\",\n                            __func__, hp_nct, cparams->n_ctx, memory_reduction/MiB);\n                    }\n                } else {\n                    if (n_ctx_min == UINT32_MAX) {\n                        LLAMA_LOG_INFO(\"%s: user has requested full context size of %\" PRIu32 \" -> no change\\n\", __func__, hp_nct);\n                    } else {\n                        LLAMA_LOG_INFO(\"%s: default model context size is %\" PRIu32 \" which is <= the min. context size of %\" PRIu32 \" -> no change\\n\",\n                            __func__, hp_nct, n_ctx_min);\n                    }\n                }\n            } else {\n                LLAMA_LOG_INFO(\"%s: context size set by user to %\" PRIu32 \" -> no change\\n\", __func__, cparams->n_ctx);\n            }\n        }\n    }\n\n    if (mparams->n_gpu_layers != default_mparams.n_gpu_layers) {\n        throw llama_params_fit_exception(\"n_gpu_layers already set by user to \" + std::to_string(mparams->n_gpu_layers) + \", abort\");\n    }\n    if (nd > 1) {\n        if (!tensor_split) {\n            throw llama_params_fit_exception(\"did not provide a buffer to write the tensor_split to, abort\");\n        }\n        if (mparams->tensor_split) {\n            for (size_t id = 0; id < nd; id++) {\n                if (mparams->tensor_split[id] != 0.0f) {\n                    throw llama_params_fit_exception(\"model_params::tensor_split already set by user, abort\");\n                }\n            }\n        }\n        if (mparams->split_mode == LLAMA_SPLIT_MODE_ROW) {\n            throw llama_params_fit_exception(\"changing weight allocation for LLAMA_SPLIT_MODE_ROW not implemented, abort\");\n        }\n    }\n    if (!tensor_buft_overrides) {\n        throw llama_params_fit_exception(\"did not provide buffer to set tensor_buft_overrides, abort\");\n    }\n    if (mparams->tensor_buft_overrides && (mparams->tensor_buft_overrides->pattern || mparams->tensor_buft_overrides->buft)) {\n        throw llama_params_fit_exception(\"model_params::tensor_buft_overrides already set by user, abort\");\n    }\n\n    // step 3: iteratively fill the back to front with \"dense\" layers\n    //   - for a dense model simply fill full layers, giving each device a contiguous slice of the model\n    //   - for a MoE model, same as dense model but with all MoE tensors in system memory\n\n    // utility function that returns a static C string matching the tensors for a specific layer index and layer fraction:\n    auto get_overflow_pattern = [&](const size_t il, const layer_fraction_t lf) -> const char * {\n        constexpr size_t n_strings = 1000;\n        if (il >= n_strings) {\n            throw std::runtime_error(\"at most \" + std::to_string(n_strings) + \" model layers are supported\");\n        }\n        switch (lf) {\n            case LAYER_FRACTION_ATTN: {\n                static std::array<std::string, n_strings> patterns;\n                if (patterns[il].empty()) {\n                    patterns[il] = \"blk\\\\.\" + std::to_string(il) + \"\\\\.ffn_(up|gate|down).*\";\n                }\n                return patterns[il].c_str();\n            }\n            case LAYER_FRACTION_UP: {\n                static std::array<std::string, n_strings> patterns;\n                if (patterns[il].empty()) {\n                    patterns[il] = \"blk\\\\.\" + std::to_string(il) + \"\\\\.ffn_(gate|down).*\";\n                }\n                return patterns[il].c_str();\n            }\n            case LAYER_FRACTION_GATE: {\n                static std::array<std::string, n_strings> patterns;\n                if (patterns[il].empty()) {\n                    patterns[il] = \"blk\\\\.\" + std::to_string(il) + \"\\\\.ffn_down.*\";\n                }\n                return patterns[il].c_str();\n            }\n            case LAYER_FRACTION_MOE: {\n                static std::array<std::string, n_strings> patterns;\n                if (patterns[il].empty()) {\n                    patterns[il] = \"blk\\\\.\" + std::to_string(il) + \"\\\\.ffn_(up|down|gate)_(ch|)exps\";\n                }\n                return patterns[il].c_str();\n            }\n            default:\n                GGML_ABORT(\"fatal error\");\n        }\n    };\n\n    struct ngl_t {\n        uint32_t n_layer = 0; // number of total layers\n        uint32_t n_part  = 0; // number of partial layers, <= n_layer\n\n        // for the first partial layer varying parts can overflow, all further layers use LAYER_FRACTION_MOE:\n        layer_fraction_t overflow_type = LAYER_FRACTION_MOE;\n\n        uint32_t n_full() const {\n            assert(n_layer >= n_part);\n            return n_layer - n_part;\n        }\n    };\n\n    const size_t ntbo = llama_max_tensor_buft_overrides();\n\n    // utility function to set n_gpu_layers and tensor_split\n    auto set_ngl_tensor_split_tbo = [&](\n            const std::vector<ngl_t> & ngl_per_device,\n            const std::vector<ggml_backend_buffer_type_t> & overflow_bufts,\n            llama_model_params & mparams) {\n        mparams.n_gpu_layers = 0;\n        for (size_t id = 0; id < nd; id++) {\n            mparams.n_gpu_layers += ngl_per_device[id].n_layer;\n            if (nd > 1) {\n                tensor_split[id] = ngl_per_device[id].n_layer;\n            }\n        }\n        assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);\n        uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides\n\n        mparams.tensor_split = tensor_split;\n\n        size_t itbo = 0;\n        for (size_t id = 0; id < nd; id++) {\n            il0 += ngl_per_device[id].n_full();\n            for (uint32_t il = il0; il < il0 + ngl_per_device[id].n_part; il++) {\n                if (itbo + 1 >= ntbo) {\n                    tensor_buft_overrides[itbo].pattern = nullptr;\n                    tensor_buft_overrides[itbo].buft    = nullptr;\n                    itbo++;\n                    mparams.tensor_buft_overrides = tensor_buft_overrides;\n                    throw llama_params_fit_exception(\"llama_max_tensor_buft_overrides() == \"\n                        + std::to_string(ntbo) + \" is insufficient for model\");\n                }\n                tensor_buft_overrides[itbo].pattern = get_overflow_pattern(il, il == il0 ? ngl_per_device[id].overflow_type : LAYER_FRACTION_MOE);\n                tensor_buft_overrides[itbo].buft = il == il0 ? overflow_bufts[id] : ggml_backend_cpu_buffer_type();\n                itbo++;\n            }\n            il0 += ngl_per_device[id].n_part;\n        }\n        tensor_buft_overrides[itbo].pattern = nullptr;\n        tensor_buft_overrides[itbo].buft    = nullptr;\n        itbo++;\n        mparams.tensor_buft_overrides = tensor_buft_overrides;\n    };\n\n    // utility function that returns the memory use per device for given numbers of layers per device\n    auto get_memory_for_layers = [&](\n            const char * func_name,\n            const std::vector<ngl_t> & ngl_per_device,\n            const std::vector<ggml_backend_buffer_type_t> & overflow_bufts) -> std::vector<int64_t> {\n        llama_model_params mparams_copy = *mparams;\n        set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);\n\n        const dmds_t dmd_nl = llama_get_device_memory_data(\n            path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);\n\n        LLAMA_LOG_DEBUG(\"%s: memory for test allocation by device:\\n\", func_name);\n        for (size_t id = 0; id < nd; id++) {\n            const ngl_t & n = ngl_per_device[id];\n            LLAMA_LOG_DEBUG(\n                \"%s: id=%zu, n_layer=%2\" PRIu32 \", n_part=%2\" PRIu32 \", overflow_type=%d, mem=%6\" PRId64 \" MiB\\n\",\n                func_name, id, n.n_layer, n.n_part, int(n.overflow_type), dmd_nl[id].mb.total()/MiB);\n        }\n\n        std::vector<int64_t> ret;\n        ret.reserve(nd);\n        for (const llama_device_memory_data & dmd : dmd_nl) {\n            ret.push_back(dmd.mb.total());\n        }\n        return ret;\n    };\n\n    int64_t global_surplus_cpu_moe = 0;\n    if (hp_nex > 0) {\n        const static std::string pattern_moe_all = \"blk\\\\.\\\\d+\\\\.ffn_(up|down|gate)_(ch|)exps\"; // matches all MoE tensors\n        ggml_backend_buffer_type_t cpu_buft = ggml_backend_cpu_buffer_type();\n        tensor_buft_overrides[0] = {pattern_moe_all.c_str(), cpu_buft};\n        tensor_buft_overrides[1] = {nullptr, nullptr};\n        mparams->tensor_buft_overrides = tensor_buft_overrides;\n\n        LLAMA_LOG_DEBUG(\"%s: getting device memory data with all MoE tensors moved to system memory:\\n\", __func__);\n        const dmds_t dmds_cpu_moe = llama_get_device_memory_data(\n            path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);\n\n        for (size_t id = 0; id < nd; id++) {\n            global_surplus_cpu_moe += dmds_cpu_moe[id].free;\n            global_surplus_cpu_moe -= int64_t(dmds_cpu_moe[id].mb.total()) + margins[id];\n        }\n\n        if (global_surplus_cpu_moe > 0) {\n            LLAMA_LOG_INFO(\"%s: with only dense weights in device memory there is a total surplus of %\" PRId64 \" MiB\\n\",\n                __func__, global_surplus_cpu_moe/MiB);\n        } else {\n            LLAMA_LOG_INFO(\"%s: with only dense weights in device memory there is still a total deficit of %\" PRId64 \" MiB\\n\",\n                __func__, -global_surplus_cpu_moe/MiB);\n        }\n\n        // reset\n        tensor_buft_overrides[0] = {nullptr, nullptr};\n        mparams->tensor_buft_overrides = tensor_buft_overrides;\n    }\n\n    std::vector<int64_t> targets; // maximum acceptable memory use per device\n    targets.reserve(nd);\n    for (size_t id = 0; id < nd; id++) {\n        targets.push_back(dmds_full[id].free - margins[id]);\n        LLAMA_LOG_DEBUG(\"%s: id=%zu, target=%\" PRId64 \" MiB\\n\", __func__, id, targets[id]/MiB);\n    }\n\n    std::vector<ggml_backend_buffer_type_t> overflow_bufts; // which bufts the first partial layer of a device overflows to:\n    overflow_bufts.reserve(nd);\n    for (size_t id = 0; id < nd; id++) {\n        overflow_bufts.push_back(ggml_backend_cpu_buffer_type());\n    }\n\n    std::vector<ngl_t> ngl_per_device(nd);\n    std::vector<int64_t> mem = get_memory_for_layers(__func__, ngl_per_device, overflow_bufts);\n\n    // optimize the number of layers per device using the method of false position:\n    //   - ngl_per_device has 0 layers for each device, lower bound\n    //   - try a \"high\" configuration where a device is given all unassigned layers\n    //   - interpolate the memory use / layer between low and high linearly to get a guess where it meets our target\n    //   - check memory use of our guess, replace either the low or high bound\n    //   - once we only have a difference of a single layer, stop and return the lower bound that just barely still fits\n    //   - the last device has the output layer, which cannot be a partial layer\n    if (hp_nex == 0) {\n        LLAMA_LOG_INFO(\"%s: filling dense layers back-to-front:\\n\", __func__);\n    } else {\n        LLAMA_LOG_INFO(\"%s: filling dense-only layers back-to-front:\\n\", __func__);\n    }\n    for (int id = nd - 1; id >= 0; id--) {\n        uint32_t n_unassigned = hp_ngl + 1;\n        for (size_t jd = id + 1; jd < nd; ++jd) {\n            assert(n_unassigned >= ngl_per_device[jd].n_layer);\n            n_unassigned -= ngl_per_device[jd].n_layer;\n        }\n\n        std::vector<ngl_t> ngl_per_device_high = ngl_per_device;\n        ngl_per_device_high[id].n_layer = n_unassigned;\n        if (hp_nex > 0) {\n            ngl_per_device_high[id].n_part = size_t(id) < nd - 1 ? ngl_per_device_high[id].n_layer : ngl_per_device_high[id].n_layer - 1;\n        }\n        if (ngl_per_device_high[id].n_layer > 0) {\n            std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);\n            if (mem_high[id] > targets[id]) {\n                assert(ngl_per_device_high[id].n_layer > ngl_per_device[id].n_layer);\n                uint32_t delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;\n                LLAMA_LOG_DEBUG(\"%s: start filling device %\" PRIu32 \", delta=%\" PRIu32 \"\\n\", __func__, id, delta);\n                while (delta > 1) {\n                    uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);\n                    step_size = std::max(step_size, uint32_t(1));\n                    step_size = std::min(step_size, delta - 1);\n\n                    std::vector<ngl_t> ngl_per_device_test = ngl_per_device;\n                    ngl_per_device_test[id].n_layer += step_size;\n                    if (hp_nex) {\n                        ngl_per_device_test[id].n_part += size_t(id) == nd - 1 && ngl_per_device_test[id].n_part == 0 ?\n                            step_size - 1 : step_size; // the first layer is the output layer which must always be full\n                    }\n                    const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);\n\n                    if (mem_test[id] <= targets[id]) {\n                        ngl_per_device = ngl_per_device_test;\n                        mem            = mem_test;\n                        LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%d].n_layer=%\" PRIu32 \"\\n\", __func__, id, ngl_per_device[id].n_layer);\n                    } else {\n                        ngl_per_device_high = ngl_per_device_test;\n                        mem_high            = mem_test;\n                        LLAMA_LOG_DEBUG(\"%s: set ngl_per_device_high[%d].n_layer=%\" PRIu32 \"\\n\", __func__, id, ngl_per_device_high[id].n_layer);\n                    }\n                    delta = ngl_per_device_high[id].n_layer - ngl_per_device[id].n_layer;\n                }\n            } else {\n                assert(ngl_per_device_high[id].n_layer == n_unassigned);\n                ngl_per_device = ngl_per_device_high;\n                mem            = mem_high;\n                LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%d].n_layer=%\" PRIu32 \"\\n\", __func__, id, ngl_per_device[id].n_layer);\n            }\n        }\n\n        const int64_t projected_margin = dmds_full[id].free - mem[id];\n        LLAMA_LOG_INFO(\n            \"%s:   - %s: %2\" PRIu32 \" layers, %6\" PRId64 \" MiB used, %6\" PRId64 \" MiB free\\n\",\n            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, mem[id]/MiB, projected_margin/MiB);\n    }\n    if (hp_nex == 0 || global_surplus_cpu_moe <= 0) {\n        set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);\n        return;\n    }\n\n    // step 4: for a MoE model where all dense tensors fit,\n    //     convert the dense-only layers in the back to full layers in the front until all devices are full\n    // essentially the same procedure as for the dense-only layers except front-to-back\n    // also, try fitting at least part of one more layer to reduce waste for \"small\" GPUs with e.g. 24 GiB VRAM\n\n    size_t id_dense_start = nd;\n    for (int id = nd - 1; id >= 0; id--) {\n        if (ngl_per_device[id].n_layer > 0) {\n            id_dense_start = id;\n            continue;\n        }\n        break;\n    }\n    assert(id_dense_start < nd);\n\n    LLAMA_LOG_INFO(\"%s: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:\\n\", __func__);\n    for (size_t id = 0; id <= id_dense_start && id_dense_start < nd; id++) {\n        std::vector<ngl_t> ngl_per_device_high = ngl_per_device;\n        for (size_t jd = id_dense_start; jd < nd; jd++) {\n            const uint32_t n_layer_move = jd < nd - 1 ? ngl_per_device_high[jd].n_layer : ngl_per_device_high[jd].n_layer - 1;\n            ngl_per_device_high[id].n_layer += n_layer_move;\n            ngl_per_device_high[jd].n_layer -= n_layer_move;\n            ngl_per_device_high[jd].n_part = 0;\n        }\n        size_t id_dense_start_high = nd - 1;\n        std::vector<int64_t> mem_high = get_memory_for_layers(__func__, ngl_per_device_high, overflow_bufts);\n\n        if (mem_high[id] > targets[id]) {\n            assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());\n            uint32_t delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();\n            while (delta > 1) {\n                uint32_t step_size = int64_t(delta) * (targets[id] - mem[id]) / (mem_high[id] - mem[id]);\n                step_size = std::max(step_size, uint32_t(1));\n                step_size = std::min(step_size, delta - 1);\n\n                std::vector<ngl_t> ngl_per_device_test = ngl_per_device;\n                size_t id_dense_start_test = id_dense_start;\n                uint32_t n_converted_test = 0;\n                for (;id_dense_start_test < nd; id_dense_start_test++) {\n                    const uint32_t n_convert_jd = std::min(step_size - n_converted_test, ngl_per_device_test[id_dense_start_test].n_part);\n                    ngl_per_device_test[id_dense_start_test].n_layer -= n_convert_jd;\n                    ngl_per_device_test[id_dense_start_test].n_part -= n_convert_jd;\n                    ngl_per_device_test[id].n_layer += n_convert_jd;\n                    n_converted_test += n_convert_jd;\n\n                    if (ngl_per_device_test[id_dense_start_test].n_part > 0) {\n                        break;\n                    }\n                }\n                const std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts);\n\n                if (mem_test[id] <= targets[id]) {\n                    ngl_per_device = ngl_per_device_test;\n                    mem            = mem_test;\n                    id_dense_start = id_dense_start_test;\n                    LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%zu].(n_layer, n_part)=(%\" PRIu32 \", %\" PRIu32 \"), id_dense_start=%zu\\n\",\n                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);\n                } else {\n                    ngl_per_device_high = ngl_per_device_test;\n                    mem_high            = mem_test;\n                    id_dense_start_high = id_dense_start_test;\n                    LLAMA_LOG_DEBUG(\"%s: set ngl_per_device_high[%zu].(n_layer, n_part)=(%\" PRIu32 \", %\" PRIu32 \"), id_dense_start_high=%zu\\n\",\n                        __func__, id, ngl_per_device_high[id].n_layer, ngl_per_device_high[id].n_part, id_dense_start_high);\n                }\n                assert(ngl_per_device_high[id].n_full() >= ngl_per_device[id].n_full());\n                delta = ngl_per_device_high[id].n_full() - ngl_per_device[id].n_full();\n            }\n        } else {\n            ngl_per_device = ngl_per_device_high;\n            mem            = mem_high;\n            id_dense_start = id_dense_start_high;\n            LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%zu].(n_layer, n_part)=(%\" PRIu32 \", %\" PRIu32 \"), id_dense_start=%zu\\n\",\n                __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);\n        }\n\n        // try to fit at least part of one more layer\n        if (ngl_per_device[id_dense_start].n_layer > (id < nd - 1 ? 0 : 1)) {\n            std::vector<ngl_t> ngl_per_device_test = ngl_per_device;\n            size_t id_dense_start_test = id_dense_start;\n            ngl_per_device_test[id_dense_start_test].n_layer--;\n            ngl_per_device_test[id_dense_start_test].n_part--;\n            ngl_per_device_test[id].n_layer++;\n            ngl_per_device_test[id].n_part++;\n            if (ngl_per_device_test[id_dense_start_test].n_part == 0) {\n                id_dense_start_test++;\n            }\n            ngl_per_device_test[id].overflow_type = LAYER_FRACTION_UP;\n            std::vector<ggml_backend_buffer_type_t> overflow_bufts_test = overflow_bufts;\n            if (id < nd - 1) {\n                overflow_bufts_test[id] = ggml_backend_dev_buffer_type(devs[id + 1]);\n            }\n            LLAMA_LOG_DEBUG(\"%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_UP\\n\", __func__);\n            std::vector<int64_t> mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);\n            if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {\n                ngl_per_device = ngl_per_device_test;\n                overflow_bufts = overflow_bufts_test;\n                mem            = mem_test;\n                id_dense_start = id_dense_start_test;\n                LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%\" PRIu32 \", %\" PRIu32 \", UP), id_dense_start=%zu\\n\",\n                    __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);\n\n                ngl_per_device_test[id].overflow_type = LAYER_FRACTION_GATE;\n                LLAMA_LOG_DEBUG(\"%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_GATE\\n\", __func__);\n                mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);\n                if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {\n                    ngl_per_device = ngl_per_device_test;\n                    overflow_bufts = overflow_bufts_test;\n                    mem            = mem_test;\n                    id_dense_start = id_dense_start_test;\n                    LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%\" PRIu32 \", %\" PRIu32 \", GATE), id_dense_start=%zu\\n\",\n                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);\n                }\n            } else {\n                ngl_per_device_test[id].overflow_type = LAYER_FRACTION_ATTN;\n                LLAMA_LOG_DEBUG(\"%s: trying to fit one extra layer with overflow_type=LAYER_FRACTION_ATTN\\n\", __func__);\n                mem_test = get_memory_for_layers(__func__, ngl_per_device_test, overflow_bufts_test);\n                if (mem_test[id] < targets[id] && (id + 1 == nd || mem_test[id + 1] < targets[id + 1])) {\n                    ngl_per_device = ngl_per_device_test;\n                    overflow_bufts = overflow_bufts_test;\n                    mem            = mem_test;\n                    id_dense_start = id_dense_start_test;\n                    LLAMA_LOG_DEBUG(\"%s: set ngl_per_device[%zu].(n_layer, n_part, overflow_type)=(%\" PRIu32 \", %\" PRIu32 \", ATTN), id_dense_start=%zu\\n\",\n                        __func__, id, ngl_per_device[id].n_layer, ngl_per_device[id].n_part, id_dense_start);\n                }\n            }\n        }\n\n        const int64_t projected_margin = dmds_full[id].free - mem[id];\n        LLAMA_LOG_INFO(\n            \"%s:   - %s: %2\" PRIu32 \" layers (%2\" PRIu32 \" overflowing), %6\" PRId64 \" MiB used, %6\" PRId64 \" MiB free\\n\",\n            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);\n    }\n\n    // print info for devices that were not changed during the conversion from dense only to full layers:\n    for (size_t id = id_dense_start + 1; id < nd; id++) {\n        const int64_t projected_margin = dmds_full[id].free - mem[id];\n        LLAMA_LOG_INFO(\n            \"%s:   - %s: %2\" PRIu32 \" layers (%2\" PRIu32 \" overflowing), %6\" PRId64 \" MiB used, %6\" PRId64 \" MiB free\\n\",\n            __func__, dev_names[id].c_str(), ngl_per_device[id].n_layer, ngl_per_device[id].n_part, mem[id]/MiB, projected_margin/MiB);\n    }\n\n    set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, *mparams);\n}\n\nenum llama_params_fit_status llama_params_fit(\n        const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,\n        float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,\n        size_t * margins, uint32_t n_ctx_min, enum ggml_log_level log_level) {\n    const int64_t t0_us = llama_time_us();\n    llama_params_fit_status status = LLAMA_PARAMS_FIT_STATUS_SUCCESS;\n    try {\n        llama_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, log_level);\n        LLAMA_LOG_INFO(\"%s: successfully fit params to free device memory\\n\", __func__);\n    } catch (const llama_params_fit_exception & e) {\n        LLAMA_LOG_WARN(\"%s: failed to fit params to free device memory: %s\\n\", __func__, e.what());\n        status = LLAMA_PARAMS_FIT_STATUS_FAILURE;\n    } catch (const std::runtime_error & e) {\n        LLAMA_LOG_ERROR(\"%s: encountered an error while trying to fit params to free device memory: %s\\n\", __func__, e.what());\n        status = LLAMA_PARAMS_FIT_STATUS_ERROR;\n    }\n    const int64_t t1_us = llama_time_us();\n    LLAMA_LOG_INFO(\"%s: fitting params to free memory took %.2f seconds\\n\", __func__, (t1_us - t0_us) * 1e-6);\n    return status;\n}\n\nstruct llama_sampler_chain_params llama_sampler_chain_default_params() {\n    struct llama_sampler_chain_params result = {\n        /*.no_perf =*/ true,\n    };\n\n    return result;\n}\n\nsize_t llama_max_devices(void) {\n    return 16;\n}\n\nsize_t llama_max_tensor_buft_overrides() {\n    return 4096;\n}\n\nbool llama_supports_mmap(void) {\n    return llama_mmap::SUPPORTED;\n}\n\nbool llama_supports_mlock(void) {\n    return llama_mlock::SUPPORTED;\n}\n\nbool llama_supports_gpu_offload(void) {\n    return ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) != nullptr ||\n           ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU) != nullptr ||\n           llama_supports_rpc();\n}\n\nbool llama_supports_rpc(void) {\n    return ggml_backend_reg_by_name(\"RPC\") != nullptr;\n}\n\nvoid llama_backend_init(void) {\n    ggml_time_init();\n\n    // needed to initialize f16 tables\n    {\n        struct ggml_init_params params = { 0, NULL, false };\n        struct ggml_context * ctx = ggml_init(params);\n        ggml_free(ctx);\n    }\n}\n\nvoid llama_numa_init(enum ggml_numa_strategy numa) {\n    if (numa != GGML_NUMA_STRATEGY_DISABLED) {\n        auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n        GGML_ASSERT(dev && \"CPU backend is not loaded\");\n        auto * reg = ggml_backend_dev_backend_reg(dev);\n        auto * numa_init_fn = (decltype(ggml_numa_init) *) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_cpu_numa_init\");\n        if (numa_init_fn) {\n            numa_init_fn(numa);\n        }\n    }\n}\n\nvoid llama_backend_free(void) {\n    ggml_quantize_free();\n}\n\nint64_t llama_time_us(void) {\n    return ggml_time_us();\n}\n\n// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback\nstatic int llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,\n        const std::string & fname, std::vector<std::string> & splits, llama_model & model, llama_model_params & params) {\n    // loading time will be recalculated after the first eval, so\n    // we take page faults deferred by mmap() into consideration\n    model.t_load_us = 0;\n    time_meas tm(model.t_load_us);\n\n    model.t_start_us = tm.t_start_us;\n\n    try {\n        llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, params.use_mmap, params.use_direct_io,\n            params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);\n\n        ml.print_info();\n\n        model.hparams.vocab_only = params.vocab_only;\n        model.hparams.no_alloc   = params.no_alloc;\n\n        try {\n            model.load_arch(ml);\n        } catch(const std::exception & e) {\n            throw std::runtime_error(\"error loading model architecture: \" + std::string(e.what()));\n        }\n        try {\n            model.load_hparams(ml);\n        } catch(const std::exception & e) {\n            throw std::runtime_error(\"error loading model hyperparameters: \" + std::string(e.what()));\n        }\n        if (model.arch == LLM_ARCH_CLIP) {\n            throw std::runtime_error(\"CLIP cannot be used as main model, use it with --mmproj instead\");\n        }\n        try {\n            model.load_vocab(ml);\n        } catch(const std::exception & e) {\n            throw std::runtime_error(\"error loading model vocabulary: \" + std::string(e.what()));\n        }\n\n        model.load_stats(ml);\n        model.print_info();\n\n        if (params.vocab_only) {\n            LLAMA_LOG_INFO(\"%s: vocab only - skipping tensors\\n\", __func__);\n            return 0;\n        }\n\n        if (!model.load_tensors(ml)) {\n            return -2;\n        }\n    } catch (const std::exception & err) {\n        LLAMA_LOG_ERROR(\"%s: error loading model: %s\\n\", __func__, err.what());\n        return -1;\n    }\n\n    return 0;\n}\n\nstatic struct llama_model * llama_model_load_from_file_impl(\n        struct gguf_context * metadata,\n        llama_model_set_tensor_data_t set_tensor_data,\n        void * set_tensor_data_ud,\n        const std::string & path_model,\n        std::vector<std::string> & splits,\n        struct llama_model_params params) {\n    GGML_ASSERT((metadata == nullptr) != path_model.empty() && \"exactly one out of metadata and path_model needs to be defined\");\n    ggml_time_init();\n\n    if (!params.vocab_only && ggml_backend_reg_count() == 0) {\n        LLAMA_LOG_ERROR(\"%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\\n\", __func__);\n        return nullptr;\n    }\n\n    unsigned cur_percentage = 0;\n    if (params.progress_callback == NULL) {\n        params.progress_callback_user_data = &cur_percentage;\n        params.progress_callback = [](float progress, void * ctx) {\n            unsigned * cur_percentage_p = (unsigned *) ctx;\n            unsigned percentage = (unsigned) (100 * progress);\n            while (percentage > *cur_percentage_p) {\n                *cur_percentage_p = percentage;\n                LLAMA_LOG_CONT(\".\");\n                if (percentage >= 100) {\n                    LLAMA_LOG_CONT(\"\\n\");\n                }\n            }\n            return true;\n        };\n    }\n\n    llama_model * model = new llama_model(params);\n\n    // create list of devices to use with this model\n    if (params.devices) {\n        for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) {\n            model->devices.push_back(*dev);\n        }\n    } else {\n        // default device selection\n\n        // build list of available devices\n        std::vector<ggml_backend_dev_t> gpus;\n        std::vector<ggml_backend_dev_t> igpus;\n        std::vector<ggml_backend_dev_t> rpc_servers;\n\n        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {\n            ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n            switch (ggml_backend_dev_type(dev)) {\n                case GGML_BACKEND_DEVICE_TYPE_CPU:\n                case GGML_BACKEND_DEVICE_TYPE_ACCEL:\n                    // skip CPU backends since they are handled separately\n                    break;\n\n                case GGML_BACKEND_DEVICE_TYPE_GPU: {\n                    ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);\n                    if (ggml_backend_reg_name(reg) == std::string(\"RPC\")) {\n                        rpc_servers.push_back(dev);\n                    } else {\n                        // check if there is already a GPU with the same device id\n                        ggml_backend_dev_props props;\n                        ggml_backend_dev_get_props(dev, &props);\n                        auto it = std::find_if(gpus.begin(), gpus.end(), [&props](ggml_backend_dev_t d) {\n                            ggml_backend_dev_props d_props;\n                            ggml_backend_dev_get_props(d, &d_props);\n                            if (props.device_id && d_props.device_id) {\n                                return strcmp(props.device_id, d_props.device_id) == 0;\n                            }\n                            return false;\n                        });\n\n                        if (it != gpus.end()) {\n                            LLAMA_LOG_INFO(\"%s: skipping device %s (%s) with id %s - already using device %s (%s) with the same id\\n\",\n                                    __func__,\n                                    ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),\n                                    props.device_id ? props.device_id : \"unknown id\",\n                                    ggml_backend_dev_name(*it), ggml_backend_dev_description(*it));\n                        } else {\n                            gpus.push_back(dev);\n                        }\n                    }\n                    break;\n                }\n\n                case GGML_BACKEND_DEVICE_TYPE_IGPU:\n                    igpus.push_back(dev);\n                    break;\n            }\n        }\n\n        // add RPC servers at the front of the list to minimize network transfers\n        model->devices.insert(model->devices.begin(), rpc_servers.begin(), rpc_servers.end());\n\n        // add GPUs\n        model->devices.insert(model->devices.end(), gpus.begin(), gpus.end());\n\n        // add integrated GPUs only if no other devices were found\n        if (model->devices.empty()) {\n            model->devices.insert(model->devices.end(), igpus.begin(), igpus.end());\n        }\n    }\n\n    // if using single GPU mode, remove all except the main GPU\n    if (params.split_mode == LLAMA_SPLIT_MODE_NONE) {\n        if (params.main_gpu < 0) {\n            model->devices.clear();\n        } else {\n            if (params.main_gpu >= (int)model->devices.size()) {\n                LLAMA_LOG_ERROR(\"%s: invalid value for main_gpu: %d (available devices: %zu)\\n\", __func__, params.main_gpu, model->devices.size());\n                llama_model_free(model);\n                return nullptr;\n            }\n            ggml_backend_dev_t main_gpu = model->devices[params.main_gpu];\n            model->devices.clear();\n            model->devices.push_back(main_gpu);\n        }\n    }\n\n    for (auto * dev : model->devices) {\n        ggml_backend_dev_props props;\n        ggml_backend_dev_get_props(dev, &props);\n        LLAMA_LOG_INFO(\"%s: using device %s (%s) (%s) - %zu MiB free\\n\", __func__,\n                ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),\n                props.device_id ? props.device_id : \"unknown id\",\n                props.memory_free/1024/1024);\n    }\n\n    const int status = llama_model_load(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, *model, params);\n    GGML_ASSERT(status <= 0);\n    if (status < 0) {\n        if (status == -1) {\n            LLAMA_LOG_ERROR(\"%s: failed to load model\\n\", __func__);\n        } else if (status == -2) {\n            LLAMA_LOG_INFO(\"%s: cancelled model load\\n\", __func__);\n        }\n\n        llama_model_free(model);\n        return nullptr;\n    }\n\n    return model;\n}\n\nstruct llama_model * llama_model_init_from_user(\n        struct gguf_context * metadata,\n        llama_model_set_tensor_data_t set_tensor_data,\n        void * set_tensor_data_ud,\n        struct llama_model_params params) {\n    GGML_ASSERT(metadata != nullptr);\n    std::string path_model;\n    std::vector<std::string> splits = {};\n    params.use_mmap = false;\n    params.use_extra_bufts = false;\n    return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, params);\n}\n// deprecated\nstruct llama_model * llama_load_model_from_file(\n        const char * path_model,\n        struct llama_model_params params) {\n    return llama_model_load_from_file(path_model, params);\n}\n\nstruct llama_model * llama_model_load_from_file(\n        const char * path_model,\n        struct llama_model_params params) {\n    std::vector<std::string> splits = {};\n    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, path_model, splits, params);\n}\n\nstruct llama_model * llama_model_load_from_splits(\n        const char ** paths,\n        size_t n_paths,\n        struct llama_model_params params) {\n    std::vector<std::string> splits;\n    if (n_paths == 0) {\n        LLAMA_LOG_ERROR(\"%s: list of splits is empty\\n\", __func__);\n        return nullptr;\n    }\n    splits.reserve(n_paths);\n    for (size_t i = 0; i < n_paths; ++i) {\n        splits.push_back(paths[i]);\n    }\n    return llama_model_load_from_file_impl(nullptr, nullptr, nullptr, splits.front(), splits, params);\n}\n\nvoid llama_model_save_to_file(const struct llama_model * model, const char * path_model) {\n    llama_model_saver ms(model);\n    ms.add_kv_from_model();\n    ms.add_tensors_from_model();\n    ms.save(path_model);\n}\n\n//\n// chat templates\n//\n\nint32_t llama_chat_apply_template(\n                              const char * tmpl,\n         const struct llama_chat_message * chat,\n                                  size_t   n_msg,\n                                    bool   add_ass,\n                                    char * buf,\n                                 int32_t   length) {\n    const std::string curr_tmpl(tmpl == nullptr ? \"chatml\" : tmpl);\n\n    // format the chat to string\n    std::vector<const llama_chat_message *> chat_vec;\n    chat_vec.resize(n_msg);\n    for (size_t i = 0; i < n_msg; i++) {\n        chat_vec[i] = &chat[i];\n    }\n\n    std::string formatted_chat;\n    llm_chat_template detected_tmpl = llm_chat_detect_template(curr_tmpl);\n    if (detected_tmpl == LLM_CHAT_TEMPLATE_UNKNOWN) {\n        return -1;\n    }\n    int32_t res = llm_chat_apply_template(detected_tmpl, chat_vec, formatted_chat, add_ass);\n    if (res < 0) {\n        return res;\n    }\n    if (buf && length > 0) {\n        strncpy(buf, formatted_chat.c_str(), length);\n    }\n    return res;\n}\n\n//\n// model split\n//\n\nint32_t llama_split_path(\n    char * split_path,\n    size_t maxlen,\n    const char * path_prefix,\n    int32_t split_no,\n    int32_t split_count) {\n\n    static const char * const SPLIT_PATH_FORMAT = \"%s-%05d-of-%05d.gguf\";\n\n    const int written = snprintf(\n        split_path,\n        maxlen,\n        SPLIT_PATH_FORMAT,\n        path_prefix,\n        split_no + 1,\n        split_count\n    );\n\n    if (written < 0 || (size_t) written >= maxlen) {\n        return 0;\n    }\n\n    return (int32_t) written;\n}\n\nint32_t llama_split_prefix(\n    char * split_prefix,\n    size_t maxlen,\n    const char * split_path,\n    int32_t split_no,\n    int32_t split_count) {\n\n    const std::string str_split_path(split_path);\n\n    char postfix[32];\n    snprintf(postfix, sizeof(postfix), \"-%05d-of-%05d.gguf\", split_no + 1, split_count);\n\n    const std::string str_postfix(postfix);\n    if (str_split_path.size() <= str_postfix.size()) {\n        return 0;\n    }\n\n    const size_t size_prefix = str_split_path.size() - str_postfix.size();\n\n    if (str_split_path.compare(size_prefix, std::string::npos, str_postfix) == 0) {\n        const size_t copy_len = std::min(size_prefix + 1, maxlen);\n        snprintf(split_prefix, copy_len, \"%s\", split_path);\n\n        return (int32_t) size_prefix;\n    }\n\n    return 0;\n}\n\nconst char * llama_print_system_info(void) {\n    static std::string s;\n    s.clear(); // Clear the string, since it's static, otherwise it will accumulate data from previous calls.\n\n    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {\n        auto * reg = ggml_backend_reg_get(i);\n        auto * get_features_fn = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, \"ggml_backend_get_features\");\n        if (get_features_fn) {\n            ggml_backend_feature * features = get_features_fn(reg);\n            s += ggml_backend_reg_name(reg);\n            s += \" : \";\n            for (; features->name; features++) {\n                s += features->name;\n                s += \" = \";\n                s += features->value;\n                s += \" | \";\n            }\n        }\n    }\n\n    return s.c_str();\n}\n\n"
  },
  {
    "path": "examples/talk-llama/llama.h",
    "content": "#ifndef LLAMA_H\n#define LLAMA_H\n\n#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-opt.h\"\n#include \"gguf.h\"\n\n#include <stddef.h>\n#include <stdint.h>\n#include <stdio.h>\n#include <stdbool.h>\n\n#ifdef LLAMA_SHARED\n#    if defined(_WIN32) && !defined(__MINGW32__)\n#        ifdef LLAMA_BUILD\n#            define LLAMA_API __declspec(dllexport)\n#        else\n#            define LLAMA_API __declspec(dllimport)\n#        endif\n#    else\n#        define LLAMA_API __attribute__ ((visibility (\"default\")))\n#    endif\n#else\n#    define LLAMA_API\n#endif\n\n#ifdef __GNUC__\n#    define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))\n#elif defined(_MSC_VER)\n#    define DEPRECATED(func, hint) __declspec(deprecated(hint)) func\n#else\n#    define DEPRECATED(func, hint) func\n#endif\n\n#define LLAMA_DEFAULT_SEED 0xFFFFFFFF\n\n#define LLAMA_TOKEN_NULL -1\n\n#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'\n#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'\n#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'\n\n#define LLAMA_SESSION_MAGIC   LLAMA_FILE_MAGIC_GGSN\n#define LLAMA_SESSION_VERSION 9\n\n#define LLAMA_STATE_SEQ_MAGIC   LLAMA_FILE_MAGIC_GGSQ\n#define LLAMA_STATE_SEQ_VERSION 2\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n    //\n    // C interface\n    //\n    // TODO: show sample usage\n    //\n\n    struct llama_vocab;\n    struct llama_model;\n    struct llama_context;\n    struct llama_sampler;\n\n    typedef struct llama_memory_i * llama_memory_t;\n\n    typedef int32_t llama_pos;\n    typedef int32_t llama_token;\n    typedef int32_t llama_seq_id;\n\n    enum llama_vocab_type {\n        LLAMA_VOCAB_TYPE_NONE   = 0, // For models without vocab\n        LLAMA_VOCAB_TYPE_SPM    = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback\n        LLAMA_VOCAB_TYPE_BPE    = 2, // GPT-2 tokenizer based on byte-level BPE\n        LLAMA_VOCAB_TYPE_WPM    = 3, // BERT tokenizer based on WordPiece\n        LLAMA_VOCAB_TYPE_UGM    = 4, // T5 tokenizer based on Unigram\n        LLAMA_VOCAB_TYPE_RWKV   = 5, // RWKV tokenizer based on greedy tokenization\n        LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming\n    };\n\n    enum llama_rope_type {\n        LLAMA_ROPE_TYPE_NONE   = -1,\n        LLAMA_ROPE_TYPE_NORM   = 0,\n        LLAMA_ROPE_TYPE_NEOX   = GGML_ROPE_TYPE_NEOX,\n        LLAMA_ROPE_TYPE_MROPE  = GGML_ROPE_TYPE_MROPE,\n        LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,\n        LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,\n    };\n\n    enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file\n        LLAMA_TOKEN_TYPE_UNDEFINED    = 0,\n        LLAMA_TOKEN_TYPE_NORMAL       = 1,\n        LLAMA_TOKEN_TYPE_UNKNOWN      = 2,\n        LLAMA_TOKEN_TYPE_CONTROL      = 3,\n        LLAMA_TOKEN_TYPE_USER_DEFINED = 4,\n        LLAMA_TOKEN_TYPE_UNUSED       = 5,\n        LLAMA_TOKEN_TYPE_BYTE         = 6,\n    };\n\n    enum llama_token_attr {\n        LLAMA_TOKEN_ATTR_UNDEFINED    = 0,\n        LLAMA_TOKEN_ATTR_UNKNOWN      = 1 << 0,\n        LLAMA_TOKEN_ATTR_UNUSED       = 1 << 1,\n        LLAMA_TOKEN_ATTR_NORMAL       = 1 << 2,\n        LLAMA_TOKEN_ATTR_CONTROL      = 1 << 3,  // SPECIAL?\n        LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,\n        LLAMA_TOKEN_ATTR_BYTE         = 1 << 5,\n        LLAMA_TOKEN_ATTR_NORMALIZED   = 1 << 6,\n        LLAMA_TOKEN_ATTR_LSTRIP       = 1 << 7,\n        LLAMA_TOKEN_ATTR_RSTRIP       = 1 << 8,\n        LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 9,\n    };\n\n    // model file types\n    enum llama_ftype {\n        LLAMA_FTYPE_ALL_F32              = 0,\n        LLAMA_FTYPE_MOSTLY_F16           = 1,  // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q4_0          = 2,  // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q4_1          = 3,  // except 1d tensors\n        // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4,  // tok_embeddings.weight and output.weight are F16\n        // LLAMA_FTYPE_MOSTLY_Q4_2       = 5,  // support has been removed\n        // LLAMA_FTYPE_MOSTLY_Q4_3       = 6,  // support has been removed\n        LLAMA_FTYPE_MOSTLY_Q8_0          = 7,  // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q5_0          = 8,  // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q5_1          = 9,  // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q2_K          = 10, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q3_K_S        = 11, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q3_K_M        = 12, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q3_K_L        = 13, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q4_K_S        = 14, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q4_K_M        = 15, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q5_K_S        = 16, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q5_K_M        = 17, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q6_K          = 18, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ2_XXS       = 19, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ2_XS        = 20, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_Q2_K_S        = 21, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ3_XS        = 22, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ3_XXS       = 23, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ1_S         = 24, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ4_NL        = 25, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ3_S         = 26, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ3_M         = 27, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ2_S         = 28, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ2_M         = 29, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ4_XS        = 30, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_IQ1_M         = 31, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_BF16          = 32, // except 1d tensors\n        //LLAMA_FTYPE_MOSTLY_Q4_0_4_4      = 33, // removed from gguf files, use Q4_0 and runtime repack\n        //LLAMA_FTYPE_MOSTLY_Q4_0_4_8      = 34, // removed from gguf files, use Q4_0 and runtime repack\n        //LLAMA_FTYPE_MOSTLY_Q4_0_8_8      = 35, // removed from gguf files, use Q4_0 and runtime repack\n        LLAMA_FTYPE_MOSTLY_TQ1_0         = 36, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_TQ2_0         = 37, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_MXFP4_MOE     = 38, // except 1d tensors\n        LLAMA_FTYPE_MOSTLY_NVFP4         = 39, // except 1d tensors\n\n        LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file\n    };\n\n    enum llama_rope_scaling_type {\n        LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,\n        LLAMA_ROPE_SCALING_TYPE_NONE        = 0,\n        LLAMA_ROPE_SCALING_TYPE_LINEAR      = 1,\n        LLAMA_ROPE_SCALING_TYPE_YARN        = 2,\n        LLAMA_ROPE_SCALING_TYPE_LONGROPE    = 3,\n        LLAMA_ROPE_SCALING_TYPE_MAX_VALUE   = LLAMA_ROPE_SCALING_TYPE_LONGROPE,\n    };\n\n    enum llama_pooling_type {\n        LLAMA_POOLING_TYPE_UNSPECIFIED = -1,\n        LLAMA_POOLING_TYPE_NONE = 0,\n        LLAMA_POOLING_TYPE_MEAN = 1,\n        LLAMA_POOLING_TYPE_CLS  = 2,\n        LLAMA_POOLING_TYPE_LAST = 3,\n        LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph\n    };\n\n    enum llama_attention_type {\n        LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,\n        LLAMA_ATTENTION_TYPE_CAUSAL      = 0,\n        LLAMA_ATTENTION_TYPE_NON_CAUSAL  = 1,\n    };\n\n    enum llama_flash_attn_type {\n        LLAMA_FLASH_ATTN_TYPE_AUTO     = -1,\n        LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,\n        LLAMA_FLASH_ATTN_TYPE_ENABLED  = 1,\n    };\n\n    LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);\n\n    enum llama_split_mode {\n        LLAMA_SPLIT_MODE_NONE  = 0, // single GPU\n        LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs\n        LLAMA_SPLIT_MODE_ROW   = 2, // split layers and KV across GPUs, use tensor parallelism if supported\n    };\n\n    // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)\n    typedef struct llama_token_data {\n        llama_token id; // token id\n        float logit;    // log-odds of the token\n        float p;        // probability of the token\n    } llama_token_data;\n\n    typedef struct llama_token_data_array {\n        // TODO: consider SoA\n        // NOTE: this pointer can be modified by the samplers\n        llama_token_data * data;\n        size_t size;\n        int64_t selected; // this is the index in the data array (i.e. not the token id)\n        bool sorted;      // note: do not assume the data is sorted - always check this flag\n    } llama_token_data_array;\n\n    typedef bool (*llama_progress_callback)(float progress, void * user_data);\n\n    // Input data for llama_encode/llama_decode\n    // A llama_batch object can contain input about one or many sequences\n    // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens\n    //\n    // - token  : the token ids of the input (used when embd is NULL)\n    // - embd   : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)\n    // - pos    : the positions of the respective token in the sequence\n    //            (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)\n    // - seq_id : the sequence to which the respective token belongs\n    //            (if set to NULL, the sequence ID will be assumed to be 0)\n    // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output\n    //            (if set to NULL:\n    //               - if embeddings: all tokens are output\n    //               - if not:        only the last token is output\n    //            )\n    //\n    typedef struct llama_batch {\n        int32_t n_tokens;\n\n        llama_token  *  token;\n        float        *  embd;\n        llama_pos    *  pos;\n        int32_t      *  n_seq_id;\n        llama_seq_id ** seq_id;\n        int8_t       *  logits;   // TODO: rename this to \"output\"\n    } llama_batch;\n\n    enum llama_model_kv_override_type {\n        LLAMA_KV_OVERRIDE_TYPE_INT,\n        LLAMA_KV_OVERRIDE_TYPE_FLOAT,\n        LLAMA_KV_OVERRIDE_TYPE_BOOL,\n        LLAMA_KV_OVERRIDE_TYPE_STR,\n    };\n\n    enum llama_model_meta_key {\n        LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,\n        LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,\n        LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,\n        LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,\n        LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,\n        LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,\n        LLAMA_MODEL_META_KEY_SAMPLING_TEMP,\n        LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,\n        LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,\n        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,\n        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,\n        LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,\n    };\n\n    struct llama_model_kv_override {\n        enum llama_model_kv_override_type tag;\n\n        char key[128];\n\n        union {\n            int64_t val_i64;\n            double  val_f64;\n            bool    val_bool;\n            char    val_str[128];\n        };\n    };\n\n    struct llama_model_tensor_buft_override {\n        const char * pattern;\n        ggml_backend_buffer_type_t buft;\n    };\n\n    struct llama_model_params {\n        // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)\n        ggml_backend_dev_t * devices;\n\n        // NULL-terminated list of buffer types to use for tensors that match a pattern\n        const struct llama_model_tensor_buft_override * tensor_buft_overrides;\n\n        int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers\n        enum llama_split_mode split_mode; // how to split the model across multiple GPUs\n\n        // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE\n        int32_t main_gpu;\n\n        // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()\n        const float * tensor_split;\n\n        // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.\n        // If the provided progress_callback returns true, model loading continues.\n        // If it returns false, model loading is immediately aborted.\n        llama_progress_callback progress_callback;\n\n        // context pointer passed to the progress callback\n        void * progress_callback_user_data;\n\n        // override key-value pairs of the model meta data\n        const struct llama_model_kv_override * kv_overrides;\n\n        // Keep the booleans together to avoid misalignment during copy-by-value.\n        bool vocab_only;      // only load the vocabulary, no weights\n        bool use_mmap;        // use mmap if possible\n        bool use_direct_io;   // use direct io, takes precedence over use_mmap when supported\n        bool use_mlock;       // force system to keep model in RAM\n        bool check_tensors;   // validate model tensor data\n        bool use_extra_bufts; // use extra buffer types (used for weight repacking)\n        bool no_host;         // bypass host buffer allowing extra buffers to be used\n        bool no_alloc;        // only load metadata and simulate memory allocations\n    };\n\n    struct llama_sampler_seq_config {\n        llama_seq_id           seq_id;\n        struct llama_sampler * sampler;\n    };\n\n    // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations\n    //       https://github.com/ggml-org/llama.cpp/pull/7544\n    struct llama_context_params {\n        uint32_t n_ctx;             // text context, 0 = from model\n        uint32_t n_batch;           // logical maximum batch size that can be submitted to llama_decode\n        uint32_t n_ubatch;          // physical maximum batch size\n        uint32_t n_seq_max;         // max number of sequences (i.e. distinct states for recurrent models)\n        int32_t  n_threads;         // number of threads to use for generation\n        int32_t  n_threads_batch;   // number of threads to use for batch processing\n\n        enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`\n        enum llama_pooling_type      pooling_type;      // whether to pool (sum) embedding results by sequence id\n        enum llama_attention_type    attention_type;    // attention type to use for embeddings\n        enum llama_flash_attn_type   flash_attn_type;   // when to enable Flash Attention\n\n        // ref: https://github.com/ggml-org/llama.cpp/pull/2054\n        float    rope_freq_base;   // RoPE base frequency, 0 = from model\n        float    rope_freq_scale;  // RoPE frequency scaling factor, 0 = from model\n        float    yarn_ext_factor;  // YaRN extrapolation mix factor, negative = from model\n        float    yarn_attn_factor; // YaRN magnitude scaling factor\n        float    yarn_beta_fast;   // YaRN low correction dim\n        float    yarn_beta_slow;   // YaRN high correction dim\n        uint32_t yarn_orig_ctx;    // YaRN original context size\n        float    defrag_thold;     // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)\n\n        ggml_backend_sched_eval_callback cb_eval;\n        void * cb_eval_user_data;\n\n        enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]\n        enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]\n\n        // Abort callback\n        // if it returns true, execution of llama_decode() will be aborted\n        // currently works only with CPU execution\n        ggml_abort_callback abort_callback;\n        void *              abort_callback_data;\n\n        // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.\n        bool embeddings;  // if true, extract embeddings (together with logits)\n        bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU\n        bool no_perf;     // measure performance timings\n        bool op_offload;  // offload host tensor operations to device\n        bool swa_full;    // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)\n                          // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases\n                          //       ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573\n        bool kv_unified;  // use a unified buffer across the input sequences when computing the attention\n                          // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix\n                          // ref: https://github.com/ggml-org/llama.cpp/pull/14363\n\n        // [EXPERIMENTAL]\n        // backend sampler chain configuration (make sure the caller keeps the sampler chains alive)\n        // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init)\n        struct llama_sampler_seq_config * samplers;\n        size_t                            n_samplers;\n    };\n\n    // model quantization parameters\n    typedef struct llama_model_quantize_params {\n        int32_t nthread;                      // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()\n        enum llama_ftype ftype;               // quantize to this llama_ftype\n        enum ggml_type output_tensor_type;    // output tensor type\n        enum ggml_type token_embedding_type;  // token embeddings tensor type\n        bool allow_requantize;                // allow quantizing non-f32/f16 tensors\n        bool quantize_output_tensor;          // quantize output.weight\n        bool only_copy;                       // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored\n        bool pure;                            // quantize all tensors to the default type\n        bool keep_split;                      // quantize to the same number of shards\n        bool dry_run;                         // calculate and show the final quantization size without performing quantization\n        void * imatrix;                       // pointer to importance matrix data\n        void * kv_overrides;                  // pointer to vector containing overrides\n        void * tensor_types;                  // pointer to vector containing tensor types\n        void * prune_layers;                  // pointer to vector containing layer indices to prune\n    } llama_model_quantize_params;\n\n    typedef struct llama_logit_bias {\n        llama_token token;\n        float bias;\n    } llama_logit_bias;\n\n    typedef struct llama_sampler_chain_params {\n        bool no_perf; // whether to measure performance timings\n    } llama_sampler_chain_params;\n\n    // used in chat template\n    typedef struct llama_chat_message {\n        const char * role;\n        const char * content;\n    } llama_chat_message;\n\n    // lora adapter\n    struct llama_adapter_lora;\n\n    // Helpers for getting default parameters\n    // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)\n    LLAMA_API struct llama_model_params          llama_model_default_params(void);\n    LLAMA_API struct llama_context_params        llama_context_default_params(void);\n    LLAMA_API struct llama_sampler_chain_params  llama_sampler_chain_default_params(void);\n    LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);\n\n    // Initialize the llama + ggml backend\n    // If numa is true, use NUMA optimizations\n    // Call once at the start of the program\n    LLAMA_API void llama_backend_init(void);\n\n    // Call once at the end of the program - currently only used for MPI\n    LLAMA_API void llama_backend_free(void);\n\n    //optional:\n    LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);\n\n    // Optional: an auto threadpool gets created in ggml if not passed explicitly\n    LLAMA_API void llama_attach_threadpool(\n            struct llama_context * ctx,\n               ggml_threadpool_t   threadpool,\n               ggml_threadpool_t   threadpool_batch);\n\n    LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);\n\n    typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);\n\n    // Create a new model from GGUF metadata as well as a function to set the tensor data\n    //   - tensors are created as GGML_TYPE_F32 by default,\n    //     override by adding a tensor with the same name but a different name to the context\n    LLAMA_API struct llama_model * llama_model_init_from_user(\n                    struct gguf_context * metadata,\n          llama_model_set_tensor_data_t   set_tensor_data,    // function to initialize tensor data with\n                                   void * set_tensor_data_ud, // userdata for function\n              struct llama_model_params   params);\n\n    DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(\n                             const char * path_model,\n              struct llama_model_params   params),\n            \"use llama_model_load_from_file instead\");\n\n    // Load a model from a file\n    // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf\n    // If the split file name does not follow this pattern, use llama_model_load_from_splits\n    LLAMA_API struct llama_model * llama_model_load_from_file(\n                             const char * path_model,\n              struct llama_model_params   params);\n\n    // Load a model from multiple splits (support custom naming scheme)\n    // The paths must be in the correct order\n    LLAMA_API struct llama_model * llama_model_load_from_splits(\n                             const char ** paths,\n                                 size_t    n_paths,\n              struct llama_model_params    params);\n\n    LLAMA_API void llama_model_save_to_file(\n            const struct llama_model * model,\n                        const char * path_model);\n\n    DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),\n            \"use llama_model_free instead\");\n\n    LLAMA_API void llama_model_free(struct llama_model * model);\n\n    LLAMA_API struct llama_context * llama_init_from_model(\n                     struct llama_model * model,\n            struct llama_context_params   params);\n\n    DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(\n                     struct llama_model * model,\n            struct llama_context_params   params),\n            \"use llama_init_from_model instead\");\n\n    // Frees all allocated memory\n    LLAMA_API void llama_free(struct llama_context * ctx);\n\n    enum llama_params_fit_status {\n        LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit\n        LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit\n        LLAMA_PARAMS_FIT_STATUS_ERROR   = 2, // a hard error occurred, e.g. because no model could be found at the specified path\n    };\n\n    // fits mparams and cparams to free device memory (assumes system memory is unlimited)\n    //   - returns true if the parameters could be successfully modified to fit device memory\n    //   - this function is NOT thread safe because it modifies the global llama logger state\n    //   - only parameters that have the same value as in llama_default_model_params are modified\n    //     with the exception of the context size which is modified if and only if equal to 0\n    LLAMA_API enum llama_params_fit_status llama_params_fit(\n                                   const char   * path_model,\n                    struct llama_model_params   * mparams,\n                    struct llama_context_params * cparams,\n                                          float * tensor_split,          // writable buffer for tensor split, needs at least llama_max_devices elements\n        struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements\n                                         size_t * margins,               // margins of memory to leave per device in bytes\n                                       uint32_t   n_ctx_min,             // minimum context size to set when trying to reduce memory use\n                            enum ggml_log_level   log_level);            // minimum log level to print during fitting, lower levels go to debug log\n\n    LLAMA_API int64_t llama_time_us(void);\n\n    LLAMA_API size_t llama_max_devices(void);\n    LLAMA_API size_t llama_max_parallel_sequences(void);\n    LLAMA_API size_t llama_max_tensor_buft_overrides(void);\n\n    LLAMA_API bool llama_supports_mmap       (void);\n    LLAMA_API bool llama_supports_mlock      (void);\n    LLAMA_API bool llama_supports_gpu_offload(void);\n    LLAMA_API bool llama_supports_rpc        (void);\n\n    // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions\n    //       In some cases the requested values via llama_context_params may differ from the actual values used by the context\n    //       ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732\n    LLAMA_API uint32_t llama_n_ctx      (const struct llama_context * ctx);\n    LLAMA_API uint32_t llama_n_ctx_seq  (const struct llama_context * ctx);\n    LLAMA_API uint32_t llama_n_batch    (const struct llama_context * ctx);\n    LLAMA_API uint32_t llama_n_ubatch   (const struct llama_context * ctx);\n    LLAMA_API uint32_t llama_n_seq_max  (const struct llama_context * ctx);\n\n    DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), \"use llama_model_n_ctx_train instead\");\n    DEPRECATED(LLAMA_API int32_t llama_n_embd     (const struct llama_model * model), \"use llama_model_n_embd instead\");\n    DEPRECATED(LLAMA_API int32_t llama_n_layer    (const struct llama_model * model), \"use llama_model_n_layer instead\");\n    DEPRECATED(LLAMA_API int32_t llama_n_head     (const struct llama_model * model), \"use llama_model_n_head instead\");\n\n    DEPRECATED(LLAMA_API int32_t llama_n_vocab    (const struct llama_vocab * vocab), \"use llama_vocab_n_tokens instead\");\n\n    LLAMA_API const struct llama_model * llama_get_model   (const struct llama_context * ctx);\n    LLAMA_API           llama_memory_t   llama_get_memory  (const struct llama_context * ctx);\n    LLAMA_API  enum llama_pooling_type   llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type\n\n    LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);\n    LLAMA_API enum llama_rope_type       llama_model_rope_type(const struct llama_model * model);\n\n    LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_embd     (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_layer    (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_head     (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_head_kv  (const struct llama_model * model);\n    LLAMA_API int32_t llama_model_n_swa      (const struct llama_model * model);\n\n    // Get the model's RoPE frequency scaling factor\n    LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);\n\n    // Returns the number of classifier outputs (only valid for classifier models)\n    // Undefined behavior for non-classifier models\n    LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);\n\n    // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided\n    LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);\n\n    LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);\n\n    LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);\n\n    // Functions to access the model's GGUF metadata scalar values\n    // - The functions return the length of the string on success, or -1 on failure\n    // - The output string is always null-terminated and cleared on failure\n    // - When retrieving a string, an extra byte must be allocated to account for the null terminator\n    // - GGUF array values are not supported by these functions\n\n    // Get metadata value as a string by key name\n    LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);\n\n    // Get the number of metadata key/value pairs\n    LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);\n\n    // Get sampling metadata key name. Returns nullptr if the key is invalid\n    LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);\n\n    // Get metadata key name by index\n    LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);\n\n    // Get metadata value as a string by index\n    LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);\n\n    // Get a string describing the model type\n    LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);\n\n    // Returns the total size of all the tensors in the model in bytes\n    LLAMA_API uint64_t llama_model_size(const struct llama_model * model);\n\n    // Get the default chat template. Returns nullptr if not available\n    // If name is NULL, returns the default chat template\n    LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);\n\n    // Returns the total number of parameters in the model\n    LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);\n\n    // Returns true if the model contains an encoder that requires llama_encode() call\n    LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);\n\n    // Returns true if the model contains a decoder that requires llama_decode() call\n    LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);\n\n    // For encoder-decoder models, this function returns id of the token that must be provided\n    // to the decoder to start generating output sequence. For other models, it returns -1.\n    LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);\n\n    // Returns true if the model is recurrent (like Mamba, RWKV, etc.)\n    LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);\n\n    // Returns true if the model is hybrid (like Jamba, Granite, etc.)\n    LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);\n\n    // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)\n    LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);\n\n    // Returns 0 on success\n    LLAMA_API uint32_t llama_model_quantize(\n            const char * fname_inp,\n            const char * fname_out,\n            const llama_model_quantize_params * params);\n\n    //\n    // Adapters\n    //\n\n    // Load a LoRA adapter from file\n    // The adapter is valid as long as the associated model is not freed\n    // All adapters must be loaded before context creation\n    LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(\n            struct llama_model * model,\n            const char * path_lora);\n\n    // Functions to access the adapter's GGUF metadata scalar values\n    // - The functions return the length of the string on success, or -1 on failure\n    // - The output string is always null-terminated and cleared on failure\n    // - When retrieving a string, an extra byte must be allocated to account for the null terminator\n    // - GGUF array values are not supported by these functions\n\n    // Get metadata value as a string by key name\n    LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);\n\n    // Get the number of metadata key/value pairs\n    LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);\n\n    // Get metadata key name by index\n    LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);\n\n    // Get metadata value as a string by index\n    LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);\n\n    // Manually free a LoRA adapter\n    // NOTE: loaded adapters will be free when the associated model is deleted\n    LLAMA_API DEPRECATED(void llama_adapter_lora_free(struct llama_adapter_lora * adapter),\n            \"adapters are now freed together with the associated model\");\n\n    // Get the invocation tokens if the current lora is an alora\n    LLAMA_API uint64_t            llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);\n    LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens  (const struct llama_adapter_lora * adapter);\n\n    // The following functions operate on a llama_context, hence the naming: llama_verb_...\n\n    // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.\n    LLAMA_API int32_t llama_set_adapters_lora(\n            struct llama_context * ctx,\n            struct llama_adapter_lora ** adapters,\n            size_t n_adapters,\n            float * scales);\n\n    // Apply a loaded control vector to a llama_context, or if data is NULL, clear\n    // the currently loaded vector.\n    // n_embd should be the size of a single layer's control, and data should point\n    // to an n_embd x n_layers buffer starting from layer 1.\n    // il_start and il_end are the layer range the vector should apply to (both inclusive)\n    // See llama_control_vector_load in common to load a control vector.\n    LLAMA_API int32_t llama_set_adapter_cvec(\n            struct llama_context * ctx,\n                     const float * data,\n                          size_t   len,\n                         int32_t   n_embd,\n                         int32_t   il_start,\n                         int32_t   il_end);\n\n    //\n    // Memory\n    //\n\n    // Clear the memory contents\n    // If data == true, the data buffers will also be cleared together with the metadata\n    LLAMA_API void llama_memory_clear(\n            llama_memory_t mem,\n                      bool data);\n\n    // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)\n    // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails\n    // seq_id < 0 : match any sequence\n    // p0 < 0     : [0,  p1]\n    // p1 < 0     : [p0, inf)\n    LLAMA_API bool llama_memory_seq_rm(\n            llama_memory_t mem,\n              llama_seq_id seq_id,\n                 llama_pos p0,\n                 llama_pos p1);\n\n    // Copy all tokens that belong to the specified sequence to another sequence\n    // p0 < 0 : [0,  p1]\n    // p1 < 0 : [p0, inf)\n    LLAMA_API void llama_memory_seq_cp(\n            llama_memory_t mem,\n              llama_seq_id seq_id_src,\n              llama_seq_id seq_id_dst,\n                 llama_pos p0,\n                 llama_pos p1);\n\n    // Removes all tokens that do not belong to the specified sequence\n    LLAMA_API void llama_memory_seq_keep(\n            llama_memory_t mem,\n              llama_seq_id seq_id);\n\n    // Adds relative position \"delta\" to all tokens that belong to the specified sequence and have positions in [p0, p1)\n    // p0 < 0 : [0,  p1]\n    // p1 < 0 : [p0, inf)\n    LLAMA_API void llama_memory_seq_add(\n            llama_memory_t mem,\n              llama_seq_id seq_id,\n                 llama_pos p0,\n                 llama_pos p1,\n                 llama_pos delta);\n\n    // Integer division of the positions by factor of `d > 1`\n    // p0 < 0 : [0,  p1]\n    // p1 < 0 : [p0, inf)\n    LLAMA_API void llama_memory_seq_div(\n            llama_memory_t mem,\n              llama_seq_id seq_id,\n                 llama_pos p0,\n                 llama_pos p1,\n                       int d);\n\n    // Returns the smallest position present in the memory for the specified sequence\n    // This is typically non-zero only for SWA caches\n    // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory\n    // Return -1 if the sequence is empty\n    LLAMA_API llama_pos llama_memory_seq_pos_min(\n            llama_memory_t mem,\n              llama_seq_id seq_id);\n\n    // Returns the largest position present in the memory for the specified sequence\n    // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory\n    // Return -1 if the sequence is empty\n    LLAMA_API llama_pos llama_memory_seq_pos_max(\n            llama_memory_t mem,\n              llama_seq_id seq_id);\n\n    // Check if the memory supports shifting\n    LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);\n\n    //\n    // State / sessions\n    //\n\n    // Returns the *actual* size in bytes of the state\n    // (logits, embedding and memory)\n    // Only use when saving the state, not when restoring it, otherwise the size may be too small.\n    LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);\n    LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),\n        \"use llama_state_get_size instead\");\n\n    // Copies the state to the specified destination address.\n    // Destination needs to have allocated enough memory.\n    // Returns the number of bytes copied\n    LLAMA_API size_t llama_state_get_data(\n            struct llama_context * ctx,\n                         uint8_t * dst,\n                          size_t   size);\n    LLAMA_API DEPRECATED(size_t llama_copy_state_data(\n            struct llama_context * ctx,\n                         uint8_t * dst),\n        \"use llama_state_get_data instead\");\n\n    // Set the state reading from the specified address\n    // Returns the number of bytes read\n    LLAMA_API size_t llama_state_set_data(\n            struct llama_context * ctx,\n                   const uint8_t * src,\n                          size_t   size);\n    LLAMA_API DEPRECATED(size_t llama_set_state_data(\n            struct llama_context * ctx,\n                   const uint8_t * src),\n        \"use llama_state_set_data instead\");\n\n    // Save/load session file\n    LLAMA_API bool llama_state_load_file(\n            struct llama_context * ctx,\n                      const char * path_session,\n                     llama_token * tokens_out,\n                          size_t   n_token_capacity,\n                          size_t * n_token_count_out);\n    LLAMA_API DEPRECATED(bool llama_load_session_file(\n            struct llama_context * ctx,\n                      const char * path_session,\n                     llama_token * tokens_out,\n                          size_t   n_token_capacity,\n                          size_t * n_token_count_out),\n        \"use llama_state_load_file instead\");\n\n    LLAMA_API bool llama_state_save_file(\n            struct llama_context * ctx,\n                      const char * path_session,\n               const llama_token * tokens,\n                          size_t   n_token_count);\n    LLAMA_API DEPRECATED(bool llama_save_session_file(\n            struct llama_context * ctx,\n                      const char * path_session,\n               const llama_token * tokens,\n                          size_t   n_token_count),\n        \"use llama_state_save_file instead\");\n\n    // Get the exact size needed to copy the state of a single sequence\n    LLAMA_API size_t llama_state_seq_get_size(\n            struct llama_context * ctx,\n                    llama_seq_id   seq_id);\n\n    // Copy the state of a single sequence into the specified buffer\n    LLAMA_API size_t llama_state_seq_get_data(\n            struct llama_context * ctx,\n                         uint8_t * dst,\n                          size_t   size,\n                    llama_seq_id   seq_id);\n\n    // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence\n    // Returns:\n    //  - Positive: Ok\n    //  - Zero: Failed to load\n    LLAMA_API size_t llama_state_seq_set_data(\n            struct llama_context * ctx,\n                   const uint8_t * src,\n                          size_t   size,\n                    llama_seq_id   dest_seq_id);\n\n    LLAMA_API size_t llama_state_seq_save_file(\n            struct llama_context * ctx,\n                      const char * filepath,\n                    llama_seq_id   seq_id,\n               const llama_token * tokens,\n                          size_t   n_token_count);\n\n    LLAMA_API size_t llama_state_seq_load_file(\n            struct llama_context * ctx,\n                      const char * filepath,\n                    llama_seq_id   dest_seq_id,\n                     llama_token * tokens_out,\n                          size_t   n_token_capacity,\n                          size_t * n_token_count_out);\n\n// for backwards-compat\n#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1\n\n// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)\n#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1\n\n    typedef uint32_t llama_state_seq_flags;\n\n    LLAMA_API size_t llama_state_seq_get_size_ext(\n            struct llama_context * ctx,\n                    llama_seq_id   seq_id,\n           llama_state_seq_flags   flags);\n\n    LLAMA_API size_t llama_state_seq_get_data_ext(\n            struct llama_context * ctx,\n                         uint8_t * dst,\n                          size_t   size,\n                    llama_seq_id   seq_id,\n           llama_state_seq_flags   flags);\n\n    LLAMA_API size_t llama_state_seq_set_data_ext(\n            struct llama_context * ctx,\n                   const uint8_t * src,\n                          size_t   size,\n                    llama_seq_id   dest_seq_id,\n           llama_state_seq_flags   flags);\n\n    //\n    // Decoding\n    //\n\n    // Return batch for single sequence of tokens\n    // The sequence ID will be fixed to 0\n    // The position of the tokens will be tracked automatically by llama_decode\n    //\n    // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it\n    //\n    LLAMA_API struct llama_batch llama_batch_get_one(\n                  llama_token * tokens,\n                      int32_t   n_tokens);\n\n    // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens\n    // Each token can be assigned up to n_seq_max sequence ids\n    // The batch has to be freed with llama_batch_free()\n    // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)\n    // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token\n    // The rest of the llama_batch members are allocated with size n_tokens\n    // All members are left uninitialized\n    LLAMA_API struct llama_batch llama_batch_init(\n            int32_t n_tokens,\n            int32_t embd,\n            int32_t n_seq_max);\n\n    // Frees a batch of tokens allocated with llama_batch_init()\n    LLAMA_API void llama_batch_free(struct llama_batch batch);\n\n    // Process a batch of tokens.\n    // In contrast to llama_decode() - this call does not use KV cache.\n    // For encode-decoder contexts, processes the batch using the encoder.\n    // Can store the encoder output internally for later use by the decoder's cross-attention layers.\n    //   0 - success\n    // < 0 - error. the memory state is restored to the state before this call\n    LLAMA_API int32_t llama_encode(\n            struct llama_context * ctx,\n              struct llama_batch   batch);\n\n    // Process a batch of tokens.\n    // Requires the context to have a memory.\n    // For encode-decoder contexts, processes the batch using the decoder.\n    // Positive return values does not mean a fatal error, but rather a warning.\n    // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context\n    //   To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()\n    // Upon other return values, the memory state is restored to the state before this call\n    //    0 - success\n    //    1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)\n    //    2 - aborted     (processed ubatches will remain in the context's memory)\n    //   -1 - invalid input batch\n    // < -1 - fatal error (processed ubatches will remain in the context's memory)\n    LLAMA_API int32_t llama_decode(\n            struct llama_context * ctx,\n              struct llama_batch   batch);\n\n    // Set the number of threads used for decoding\n    // n_threads is the number of threads used for generation (single token)\n    // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)\n    LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);\n\n    // Get the number of threads used for generation of a single token.\n    LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);\n\n    // Get the number of threads used for prompt and batch processing (multiple token).\n    LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);\n\n    // Set whether the context outputs embeddings or not\n    // TODO: rename to avoid confusion with llama_get_embeddings()\n    LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);\n\n    // Set whether to use causal attention or not\n    // If set to true, the model will only attend to the past tokens\n    LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);\n\n    // Set whether the model is in warmup mode or not\n    // If true, all model tensors are activated during llama_decode() to load and cache their weights.\n    LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup);\n\n    // Set abort callback\n    LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);\n\n    // Wait until all computations are finished\n    // This is automatically done when using one of the functions below to obtain the computation results\n    // and is not necessary to call it explicitly in most cases\n    LLAMA_API void llama_synchronize(struct llama_context * ctx);\n\n    // Token logits obtained from the last call to llama_decode()\n    // The logits for which llama_batch.logits[i] != 0 are stored contiguously\n    // in the order they have appeared in the batch.\n    // Rows: number of tokens for which llama_batch.logits[i] != 0\n    // Cols: n_vocab\n    // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)\n    LLAMA_API float * llama_get_logits(struct llama_context * ctx);\n\n    // Logits for the ith token. For positive indices, Equivalent to:\n    // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab\n    // Negative indices can be used to access logits in reverse order, -1 is the last logit.\n    // returns NULL for invalid ids.\n    LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);\n\n    // Get all output token embeddings.\n    // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,\n    // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously\n    // in the order they have appeared in the batch.\n    // shape: [n_outputs*n_embd]\n    // Otherwise, returns NULL.\n    // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)\n    LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);\n\n    // Get the embeddings for the ith token. For positive indices, Equivalent to:\n    // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd\n    // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding.\n    // shape: [n_embd] (1-dimensional)\n    // returns NULL for invalid ids.\n    LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);\n\n    // Get the embeddings for a sequence id\n    // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE\n    // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence\n    // otherwise: float[n_embd] (1-dimensional)\n    LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);\n\n    //\n    // backend sampling API [EXPERIMENTAL]\n    // note: use only if the llama_context was created with at least one llama_sampler_seq_config\n    //\n\n    // Get the backend sampled token for the ith token.\n    // Returns LLAMA_TOKEN_NULL if no token was sampled.\n    LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);\n\n    // Get the backend sampled probabilities for the ith token\n    // The index matches llama_get_sampled_token_ith().\n    // Returns NULL if no probabilities were generated.\n    LLAMA_API float *  llama_get_sampled_probs_ith      (struct llama_context * ctx, int32_t i);\n    LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);\n\n    // Get the backend sampled logits for the ith token\n    // Returns NULL if no logits were sampled.\n    LLAMA_API float *  llama_get_sampled_logits_ith      (struct llama_context * ctx, int32_t i);\n    LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);\n\n    // Get the backend sampled candidates (token ids) for the ith token\n    // These are needed to map probability/logit indices to vocab token ids.\n    // Returns NULL if no candidates were sampled.\n    LLAMA_API llama_token * llama_get_sampled_candidates_ith      (struct llama_context * ctx, int32_t i);\n    LLAMA_API uint32_t      llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);\n\n    //\n    // Vocab\n    //\n\n    LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);\n\n    LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);\n\n    LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);\n\n    // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)\n    LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);\n\n    // Identify if Token Id is a control token or a render-able token\n    LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);\n\n    // Special tokens\n    LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence\n    LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence\n    LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn\n    LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator\n    LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line\n    LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding\n    LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask\n\n    LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);\n    LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);\n    LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);\n\n    LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);\n    LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);\n    LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);\n    LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);\n    LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);\n    LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);\n\n    DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), \"use llama_vocab_get_text instead\");\n    DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), \"use llama_vocab_get_score instead\");\n    DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), \"use llama_vocab_get_attr instead\");\n    DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), \"use llama_vocab_is_eog instead\");\n    DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), \"use llama_vocab_is_control instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), \"use llama_vocab_bos instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), \"use llama_vocab_eos instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), \"use llama_vocab_eot instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), \"use llama_vocab_cls instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), \"use llama_vocab_sep instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), \"use llama_vocab_nl instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), \"use llama_vocab_pad instead\");\n    DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), \"use llama_vocab_get_add_bos instead\");\n    DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), \"use llama_vocab_get_add_eos instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), \"use llama_vocab_fim_pre instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), \"use llama_vocab_fim_suf instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), \"use llama_vocab_fim_mid instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), \"use llama_vocab_fim_pad instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), \"use llama_vocab_fim_rep instead\");\n    DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), \"use llama_vocab_fim_sep instead\");\n\n    // CLS is equivalent to BOS\n    DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification\n            \"use llama_vocab_bos instead\");\n\n    //\n    // Tokenization\n    //\n    // The API is thread-safe.\n    //\n\n    /// @details Convert the provided text into tokens.\n    /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.\n    /// @return Returns the number of tokens on success, no more than n_tokens_max\n    /// @return Returns a negative number on failure - the number of tokens that would have been returned\n    /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)\n    /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.\n    /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated\n    ///                      as plaintext. Does not insert a leading space.\n    LLAMA_API int32_t llama_tokenize(\n        const struct llama_vocab * vocab,\n                      const char * text,\n                         int32_t   text_len,\n                     llama_token * tokens,\n                         int32_t   n_tokens_max,\n                            bool   add_special,\n                            bool   parse_special);\n\n    // Token Id -> Piece.\n    // Uses the vocabulary in the provided context.\n    // Does not write null terminator to the buffer.\n    // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')\n    // @param special If true, special tokens are rendered in the output.\n    LLAMA_API int32_t llama_token_to_piece(\n              const struct llama_vocab * vocab,\n                           llama_token   token,\n                                  char * buf,\n                               int32_t   length,\n                               int32_t   lstrip,\n                                  bool   special);\n\n    /// @details Convert the provided tokens into text (inverse of llama_tokenize()).\n    /// @param text The char pointer must be large enough to hold the resulting text.\n    /// @return Returns the number of chars/bytes on success, no more than text_len_max.\n    /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.\n    /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.\n    /// @param unparse_special If true, special tokens are rendered in the output.\n    LLAMA_API int32_t llama_detokenize(\n        const struct llama_vocab * vocab,\n               const llama_token * tokens,\n                         int32_t   n_tokens,\n                            char * text,\n                         int32_t   text_len_max,\n                            bool   remove_special,\n                            bool   unparse_special);\n\n    //\n    // Chat templates\n    //\n\n    /// Apply chat template. Inspired by hf apply_chat_template() on python.\n    ///\n    /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n    /// @param tmpl A Jinja template to use for this chat.\n    /// @param chat Pointer to a list of multiple llama_chat_message\n    /// @param n_msg Number of llama_chat_message in this chat\n    /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.\n    /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)\n    /// @param length The size of the allocated buffer\n    /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.\n    LLAMA_API int32_t llama_chat_apply_template(\n                            const char * tmpl,\n       const struct llama_chat_message * chat,\n                                size_t   n_msg,\n                                  bool   add_ass,\n                                  char * buf,\n                               int32_t   length);\n\n    // Get list of built-in chat templates\n    LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);\n\n    //\n    // Sampling API\n    //\n    // Sample usage:\n    //\n    //    // prepare the sampling chain at the start\n    //    auto sparams = llama_sampler_chain_default_params();\n    //\n    //    llama_sampler * smpl = llama_sampler_chain_init(sparams);\n    //\n    //    llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));\n    //    llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));\n    //    llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));\n    //\n    //    // typically, the chain should end with a sampler such as \"greedy\", \"dist\" or \"mirostat\"\n    //    // this sampler will be responsible to select the actual token\n    //    llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));\n    //\n    //    ...\n    //\n    //    // decoding loop:\n    //    while (...) {\n    //        ...\n    //\n    //        llama_decode(ctx, batch);\n    //\n    //        // sample from the logits of the last token in the batch\n    //        const llama_token id = llama_sampler_sample(smpl, ctx, -1);\n    //\n    //        ...\n    //    }\n    //\n    //    llama_sampler_free(smpl);\n    //\n\n    typedef void * llama_sampler_context_t;\n\n    struct llama_sampler_data {\n        struct ggml_tensor * logits;\n        struct ggml_tensor * probs;\n        struct ggml_tensor * sampled;\n        struct ggml_tensor * candidates;\n    };\n\n    // user code can implement the interface below in order to create custom llama_sampler\n    struct llama_sampler_i {\n        const char *           (*name)  (const struct llama_sampler * smpl);                                 // can be NULL\n        void                   (*accept)(      struct llama_sampler * smpl, llama_token token);              // can be NULL\n        void                   (*apply) (      struct llama_sampler * smpl, llama_token_data_array * cur_p); // required\n        void                   (*reset) (      struct llama_sampler * smpl);                                 // can be NULL\n        struct llama_sampler * (*clone) (const struct llama_sampler * smpl);                                 // can be NULL if ctx is NULL\n        void                   (*free)  (      struct llama_sampler * smpl);                                 // can be NULL if ctx is NULL\n\n        // [EXPERIMENTAL]\n        // backend sampling interface:\n\n        // return true if the backend supports all ops needed by the sampler\n        // note: call once per sampler\n        bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft);\n\n        // call after .backend_apply()\n        void (*backend_accept)(\n                struct llama_sampler * smpl,\n                struct ggml_context  * ctx,\n                struct ggml_cgraph   * gf,\n                struct ggml_tensor   * selected_token);\n\n        // call after .backend_init()\n        void (*backend_apply)(\n                struct llama_sampler      * smpl,\n                struct ggml_context       * ctx,\n                struct ggml_cgraph        * gf,\n                struct llama_sampler_data * data);\n\n        // called before graph execution to set inputs for the current ubatch\n        void (*backend_set_input)(struct llama_sampler * smpl);\n    };\n\n    struct llama_sampler {\n        struct llama_sampler_i * iface;\n\n        llama_sampler_context_t ctx;\n    };\n\n    // [EXPERIMENTAL]\n    // attach a sampler to the context\n    // note: prefer initializing the context with llama_context_params.samplers when possible\n    LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);\n\n    // mirror of llama_sampler_i:\n    LLAMA_API struct llama_sampler * llama_sampler_init  (      struct llama_sampler_i * iface, llama_sampler_context_t ctx);\n    LLAMA_API const char *           llama_sampler_name  (const struct llama_sampler * smpl);\n    LLAMA_API void                   llama_sampler_accept(      struct llama_sampler * smpl, llama_token token);\n    LLAMA_API void                   llama_sampler_apply (      struct llama_sampler * smpl, llama_token_data_array * cur_p);\n    LLAMA_API void                   llama_sampler_reset (      struct llama_sampler * smpl);\n    LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);\n    // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)\n    LLAMA_API void                   llama_sampler_free  (      struct llama_sampler * smpl);\n\n    // llama_sampler_chain\n    // a type of llama_sampler that can chain multiple samplers one after another\n\n    LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);\n\n    // important: takes ownership of the sampler object and will free it when llama_sampler_free is called\n    LLAMA_API void                   llama_sampler_chain_add(      struct llama_sampler * chain, struct llama_sampler * smpl);\n\n    // return NULL if:\n    //   - the sampler is NULL\n    //   - the sampler is not a llama_sampler_chain\n    //   - the index is out of bounds, unless i == -1\n    //   - if i == -1, returns the chain itself (can be used to check if the sampler is a chain)\n    LLAMA_API struct llama_sampler * llama_sampler_chain_get(      struct llama_sampler * chain, int32_t i);\n\n    // the total number of samplers in the chain\n    LLAMA_API int                    llama_sampler_chain_n  (const struct llama_sampler * chain);\n\n    // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed\n    LLAMA_API struct llama_sampler * llama_sampler_chain_remove(   struct llama_sampler * chain, int32_t i);\n\n    // available samplers:\n\n    LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);\n\n    /// seed == LLAMA_DEFAULT_SEED to use a random seed.\n    LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed);\n\n    /// @details Top-K sampling described in academic paper \"The Curious Case of Neural Text Degeneration\" https://arxiv.org/abs/1904.09751\n    /// Setting k <= 0 makes this a noop\n    LLAMA_API struct llama_sampler * llama_sampler_init_top_k      (int32_t k);\n\n    /// @details Nucleus sampling described in academic paper \"The Curious Case of Neural Text Degeneration\" https://arxiv.org/abs/1904.09751\n    LLAMA_API struct llama_sampler * llama_sampler_init_top_p      (float   p, size_t min_keep);\n\n    /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841\n    LLAMA_API struct llama_sampler * llama_sampler_init_min_p      (float   p, size_t min_keep);\n\n    /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.\n    LLAMA_API struct llama_sampler * llama_sampler_init_typical    (float   p, size_t min_keep);\n\n    /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf\n    LLAMA_API struct llama_sampler * llama_sampler_init_temp       (float   t);\n\n    /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.\n    LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext   (float   t, float   delta, float exponent);\n\n    /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335\n    LLAMA_API struct llama_sampler * llama_sampler_init_xtc        (float   p, float   t,     size_t min_keep, uint32_t seed);\n\n    /// @details Top n sigma sampling as described in academic paper \"Top-nσ: Not All Logits Are You Need\" https://arxiv.org/pdf/2411.07641\n    LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float   n);\n\n    /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.\n    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.\n    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.\n    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.\n    /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.\n    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.\n    LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(\n                             int32_t   n_vocab,\n                            uint32_t   seed,\n                               float   tau,\n                               float   eta,\n                             int32_t   m);\n\n    /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.\n    /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.\n    /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.\n    /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.\n    /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.\n    LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(\n                            uint32_t   seed,\n                               float   tau,\n                               float   eta);\n\n    /// @details Initializes a GBNF grammar, see grammars/README.md for details.\n    /// @param vocab The vocabulary that this grammar will be used with.\n    /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails.\n    /// @param grammar_root The name of the start symbol for the grammar.\n    LLAMA_API struct llama_sampler * llama_sampler_init_grammar(\n            const struct llama_vocab * vocab,\n                          const char * grammar_str,\n                          const char * grammar_root);\n\n    DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy(\n            const struct llama_vocab * vocab,\n                          const char * grammar_str,\n                          const char * grammar_root,\n                         const char ** trigger_words,\n                                size_t num_trigger_words,\n                   const llama_token * trigger_tokens,\n                                size_t num_trigger_tokens),\n        \"use llama_sampler_init_grammar_lazy_patterns instead\");\n\n\n    /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639\n    /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group.\n    /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included.\n    LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(\n        const struct llama_vocab * vocab,\n                      const char * grammar_str,\n                      const char * grammar_root,\n                     const char ** trigger_patterns,\n                            size_t num_trigger_patterns,\n               const llama_token * trigger_tokens,\n                            size_t num_trigger_tokens);\n\n\n    /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.\n    LLAMA_API struct llama_sampler * llama_sampler_init_penalties(\n                             int32_t   penalty_last_n,   // last n tokens to penalize (0 = disable penalty, -1 = context size)\n                               float   penalty_repeat,   // 1.0 = disabled\n                               float   penalty_freq,     // 0.0 = disabled\n                               float   penalty_present); // 0.0 = disabled\n\n    ///  @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982\n    LLAMA_API struct llama_sampler * llama_sampler_init_dry(\n            const struct llama_vocab *  vocab,\n                             int32_t    n_ctx_train,\n                               float    dry_multiplier,\n                               float    dry_base,\n                             int32_t    dry_allowed_length,\n                             int32_t    dry_penalty_last_n,\n                          const char ** seq_breakers,\n                              size_t    num_breakers);\n\n    /// adaptive-p: select tokens near a configurable target probability over time.\n    ///\n    /// the adaptive-p sampler transforms the token probability distribution to favor tokens\n    /// that fall near a user-configurable probability target.\n    ///\n    /// internally, the sampler maintains an exponential moving average of the *ORIGINAL*\n    /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an\n    /// adapted target probability at each sampling step, thus maintaining the desired target\n    /// probability over time.\n    ///\n    /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last\n    /// in the sampler chain (like mirostat, dist, greedy).\n    ///\n    /// only mild truncation before this sampler is recommended. we suggest applying min-p\n    /// before adaptive-p as the only other active sampler in the chain.\n    ///\n    /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)\n    /// @param decay  EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99)\n    /// @param seed   RNG seed\n    ///\n    /// ref: https://github.com/ggml-org/llama.cpp/pull/17927\n    ///\n    LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p(\n                               float   target,\n                               float   decay,\n                            uint32_t   seed);\n\n    LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(\n                             int32_t   n_vocab,\n                             int32_t   n_logit_bias,\n              const llama_logit_bias * logit_bias);\n\n    // this sampler is meant to be used for fill-in-the-middle infilling\n    // it's supposed to be used after top_k + top_p sampling\n    //\n    // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG\n    // 2. combine probs of tokens that have the same prefix\n    //\n    // example:\n    //\n    // - before:\n    //   \"hel\":   0.5\n    //   \"hell\":  0.2\n    //   \"hello\": 0.1\n    //   \"dummy\": 0.1\n    //\n    // - after:\n    //   \"hel\":   0.8\n    //   \"dummy\": 0.1\n    //\n    // 3. discard non-EOG tokens with low prob\n    // 4. if no tokens are left -> pick EOT\n    //\n    LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab);\n\n    // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise\n    LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);\n\n    /// @details Sample and accept a token from the idx-th output of the last evaluation\n    //\n    // Shorthand for:\n    //    const auto * logits = llama_get_logits_ith(ctx, idx);\n    //    llama_token_data_array cur_p = { ... init from logits ... };\n    //    llama_sampler_apply(smpl, &cur_p);\n    //    auto token = cur_p.data[cur_p.selected].id;\n    //    llama_sampler_accept(smpl, token);\n    //    return token;\n    // Returns the sampled token\n    LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx);\n\n    // TODO: extend in the future\n    //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...);\n\n    //\n    // Model split\n    //\n\n    /// @details Build a split GGUF final path for this chunk.\n    ///          llama_split_path(split_path, sizeof(split_path), \"/models/ggml-model-q4_0\", 2, 4) => split_path = \"/models/ggml-model-q4_0-00002-of-00004.gguf\"\n    //  Returns the split_path length.\n    LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count);\n\n    /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.\n    ///          llama_split_prefix(split_prefix, 64, \"/models/ggml-model-q4_0-00002-of-00004.gguf\", 2, 4) => split_prefix = \"/models/ggml-model-q4_0\"\n    //  Returns the split_prefix length.\n    LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count);\n\n    // Print system information\n    LLAMA_API const char * llama_print_system_info(void);\n\n    // Set callback for all future logging events.\n    // If this is not called, or NULL is supplied, everything is output on stderr.\n    // The logger state is global so these functions are NOT thread safe.\n    LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);\n    LLAMA_API void llama_log_set(ggml_log_callback   log_callback, void *  user_data);\n\n    //\n    // Performance utils\n    //\n    // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements.\n    //\n\n    struct llama_perf_context_data {\n        // ms == milliseconds\n        double t_start_ms;  // absolute start time\n        double t_load_ms;   // time needed for loading the model\n        double t_p_eval_ms; // time needed for processing the prompt\n        double t_eval_ms;   // time needed for generating tokens\n\n        int32_t n_p_eval;   // number of prompt tokens\n        int32_t n_eval;     // number of generated tokens\n        int32_t n_reused;   // number of times a ggml compute graph had been reused\n    };\n\n    struct llama_perf_sampler_data {\n        double t_sample_ms; // time needed for sampling in ms\n\n        int32_t n_sample;   // number of sampled tokens\n    };\n\n    LLAMA_API struct llama_perf_context_data llama_perf_context      (const struct llama_context * ctx);\n    LLAMA_API void                           llama_perf_context_print(const struct llama_context * ctx);\n    LLAMA_API void                           llama_perf_context_reset(      struct llama_context * ctx);\n\n    // NOTE: the following work only with samplers constructed via llama_sampler_chain_init\n    LLAMA_API struct llama_perf_sampler_data llama_perf_sampler      (const struct llama_sampler * chain);\n    LLAMA_API void                           llama_perf_sampler_print(const struct llama_sampler * chain);\n    LLAMA_API void                           llama_perf_sampler_reset(      struct llama_sampler * chain);\n\n    // print a breakdown of per-device memory use via LLAMA_LOG:\n    LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx);\n\n    //\n    // training\n    //\n\n    // function that returns whether or not a given tensor contains trainable parameters\n    typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata);\n\n    // always returns true\n    LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata);\n\n    struct llama_opt_params {\n        uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0\n\n        llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters\n        void * param_filter_ud;              // userdata for determining which tensors contain trainable parameters\n\n        ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters\n        void * get_opt_pars_ud;                     // userdata for calculating optimizer parameters\n\n        enum ggml_opt_optimizer_type optimizer_type;\n    };\n\n    LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);\n\n    LLAMA_API void llama_opt_epoch(\n            struct llama_context    * lctx,\n            ggml_opt_dataset_t        dataset,\n            ggml_opt_result_t         result_train,\n            ggml_opt_result_t         result_eval,\n            int64_t                   idata_split,\n            ggml_opt_epoch_callback   callback_train,\n            ggml_opt_epoch_callback   callback_eval);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // LLAMA_H\n"
  },
  {
    "path": "examples/talk-llama/models/afmoe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_afmoe::llm_build_afmoe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // MuP scaling: embeddings * sqrt(hidden_size)\n    // mup_enabled = true, hidden_size = 1024, scale = 32.0\n    inpL = ggml_scale(ctx0, inpL, sqrtf(float(n_embd)));\n    cb(inpL, \"inp_embd_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n    auto * inp_attn = build_attn_inp_kv_iswa();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    const float kq_scale = 1.0f/sqrtf(float(n_embd_head));\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * inpSA = inpL;\n\n        // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous\n        const bool use_rope = hparams.n_no_rope_layer_step > 0 &&\n                              (il + 1) % hparams.n_no_rope_layer_step != 0;\n\n        // dual attention normalization (pre)\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * attn_inp = cur;  // save input for gate computation\n\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            // compute gate from input\n            ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);\n            cb(gate, \"attn_gate_proj\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n\n            // Q/K normalization\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            if (use_rope) {\n                Qcur = ggml_rope_ext(\n                        ctx0, Qcur, inp_pos, nullptr,\n                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                        ext_factor, attn_factor, beta_fast, beta_slow);\n                cb(Qcur, \"Qcur_rope\", il);\n\n                Kcur = ggml_rope_ext(\n                        ctx0, Kcur, inp_pos, nullptr,\n                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                        ext_factor, attn_factor, beta_fast, beta_slow);\n                cb(Kcur, \"Kcur_rope\", il);\n            }\n\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            cur = build_attn(inp_attn,\n                    NULL, NULL,  // wo will be applied after gating\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n\n            // attention gating: attn_out * sigmoid(gate) BEFORE o_proj\n            gate = ggml_sigmoid(ctx0, gate);\n            cb(gate, \"attn_gate_sig\", il);\n            cur = ggml_mul(ctx0, cur, gate);\n            cb(cur, \"attn_gated\", il);\n\n            // now apply output projection\n            cur = build_lora_mm(model.layers[il].wo, cur);\n            cb(cur, \"attn_o_proj\", il);\n        }\n\n        // dual attention normalization (post)\n        cur = build_norm(cur,\n                model.layers[il].attn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // dual ffn normalization (pre)\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // MoE or dense FFN\n        if ((uint32_t)il >= hparams.n_layer_dense_lead) {\n            // MoE layer with sigmoid routing, normalization, and scaling\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    model.layers[il].ffn_exp_probs_b,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU,\n                    hparams.expert_weights_norm,           // norm_w (route_norm=True)\n                    hparams.expert_weights_scale,          // w_scale (route_scale=2.826)\n                    (llama_expert_gating_func_type) hparams.expert_gating_func,\n                    il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // shared expert\n            if (hparams.n_expert_shared > 0) {\n                ggml_tensor * ffn_shexp = build_ffn(cur,\n                        model.layers[il].ffn_up_shexp,   NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL,\n                        LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            } else {\n                cur = moe_out;\n            }\n        } else {\n            // dense layer\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        // dual ffn normalization (post)\n        cur = build_norm(cur,\n                model.layers[il].ffn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/apertus.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_apertus::llm_build_apertus(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * inp_pos  = build_inp_pos();\n    auto *        inp_attn = build_attn_inp_kv();\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur_pos\", il);\n            cb(Kcur, \"Kcur_pos\", il);\n            cb(Vcur, \"Vcur_pos\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network with xIELU activation\n        {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            // Up projection\n            ggml_tensor * up = build_lora_mm(model.layers[il].ffn_up, cur);\n            cb(up, \"ffn_up\", il);\n\n            float alpha_n_val = hparams.xielu_alpha_n[il];\n            float alpha_p_val = hparams.xielu_alpha_p[il];\n            float beta_val    = hparams.xielu_beta[il];\n            float eps_val     = hparams.xielu_eps[il];\n\n            // Apply xIELU activation\n            ggml_tensor * activated = ggml_xielu(ctx0, up, alpha_n_val, alpha_p_val, beta_val, eps_val);\n            cb(activated, \"ffn_xielu\", il);\n\n            // Down projection\n            cur = build_lora_mm(model.layers[il].ffn_down, activated);\n            cb(cur, \"ffn_down\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/arcee.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_arcee::llm_build_arcee(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        // ARCEE uses relu^2 instead of silu\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                NULL,                      NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/arctic.cpp",
    "content": "#include \"models.h\"\n\nllm_build_arctic::llm_build_arctic(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        ggml_tensor * ffn_out = ggml_add(ctx0, cur, ffn_inp);\n        cb(ffn_out, \"ffn_out\", il);\n\n        // MoE\n        cur = build_norm(inpSA,\n                model.layers[il].ffn_norm_exps, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm_exps\", il);\n\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_out);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/arwkv7.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_arwkv7::llm_build_arwkv7(const llama_model & model, const llm_graph_params & params) : llm_build_rwkv7_base(model, params) {\n    GGML_ASSERT(n_embd == hparams.n_embd_r());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * v_first = nullptr;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * rs_inp = build_rs_inp();\n\n    const auto n_embd = hparams.n_embd;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n    const auto n_seqs = ubatch.n_seqs;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const llama_layer * layer = &model.layers[il];\n        inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);\n\n        ggml_tensor * token_shift = build_rwkv_token_shift_load(rs_inp, ubatch, il);\n\n        ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, il);\n        cb(att_norm, \"attn_norm\", il);\n\n        ggml_tensor * x_prev = ggml_concat(\n                ctx0,\n                token_shift,\n                ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0),\n                1\n                );\n\n        cur = build_rwkv7_time_mix(rs_inp, att_norm, x_prev, v_first, ubatch, il);\n\n        token_shift = ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm));\n        ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il));\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur     = ggml_reshape_2d(ctx0, cur,     n_embd, n_tokens);\n        ffn_inp = ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur     = ggml_get_rows(ctx0, cur,     inp_out_ids);\n            ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n        }\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/baichuan.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_baichuan::llm_build_baichuan(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = model.type == LLM_TYPE_7B ? build_inp_pos() : nullptr;\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            switch (model.type) {\n                case LLM_TYPE_7B:\n                    Qcur = ggml_rope_ext(\n                            ctx0, Qcur, inp_pos, nullptr,\n                            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow\n                            );\n                    Kcur = ggml_rope_ext(\n                            ctx0, Kcur, inp_pos, nullptr,\n                            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow\n                            );\n                    break;\n                case LLM_TYPE_13B:\n                case LLM_TYPE_UNKNOWN:\n                    break;\n                default:\n                    GGML_ABORT(\"fatal error\");\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/bailingmoe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_bailingmoe::llm_build_bailingmoe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_rot, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_rot, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_rot)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, hparams.expert_weights_norm,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        // FFN shared expert\n        {\n            ggml_tensor * ffn_shexp = build_ffn(cur,\n                    model.layers[il].ffn_up_shexp,   NULL, NULL,\n                    model.layers[il].ffn_gate_shexp, NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(ffn_shexp, \"ffn_shexp\", il);\n\n            cur = ggml_add(ctx0, moe_out, ffn_shexp);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/bailingmoe2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_bailingmoe2::llm_build_bailingmoe2(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;\n    for (int il = 0; il < n_transformer_layers; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 0 * sizeof(float) * (n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 1 * sizeof(float) * (n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_transformer_layers - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * sa_out = ggml_add(ctx0, cur, inpSA);\n        cb(sa_out, \"sa_out\", il);\n\n        // MoE branch\n        cur = build_norm(sa_out, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        if (static_cast<uint32_t>(il) < hparams.n_layer_dense_lead) {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, hparams.expert_weights_norm,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n\n        cur = ggml_add(ctx0, cur, sa_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/bert.cpp",
    "content": "#include \"models.h\"\n\nllm_build_bert::llm_build_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * inp_pos = nullptr;\n\n    if (model.arch != LLM_ARCH_JINA_BERT_V2) {\n        inp_pos = build_inp_pos();\n    }\n\n    // construct input embeddings (token, type, position)\n    inpL = build_inp_embd(model.tok_embd);\n\n    // token types are hardcoded to zero (\"Sentence A\")\n    if (model.type_embd) {\n        ggml_tensor * type_row0 = ggml_view_1d(ctx0, model.type_embd, n_embd, 0);\n        inpL                    = ggml_add(ctx0, inpL, type_row0);\n    }\n    if (model.arch == LLM_ARCH_BERT) {\n        inpL = ggml_add(ctx0, ggml_get_rows(ctx0, model.pos_embd, inp_pos), inpL);\n    }\n    cb(inpL, \"inp_embd\", -1);\n\n    // embed layer norm\n    inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);\n    cb(inpL, \"inp_norm\", -1);\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * cur = inpL;\n\n        {\n            ggml_tensor * Qcur;\n            ggml_tensor * Kcur;\n            ggml_tensor * Vcur;\n\n            // self-attention\n            if (model.layers[il].wqkv) {\n                cur = build_lora_mm(model.layers[il].wqkv, cur);\n                cb(cur, \"wqkv\", il);\n\n                if (model.layers[il].bqkv) {\n                    cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n                    cb(cur, \"bqkv\", il);\n                }\n\n                Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), cur->nb[1],\n                                    0 * sizeof(float) * (n_embd));\n                Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                    cur->nb[1], 1 * sizeof(float) * (n_embd));\n                Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                    cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));\n            } else {\n                Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, cur), model.layers[il].bq);\n                Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, cur), model.layers[il].bk);\n                Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, cur), model.layers[il].bv);\n\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n            }\n\n            if (model.layers[il].attn_q_norm) {\n                Qcur = ggml_reshape_2d(ctx0, Qcur, n_embd_head * n_head, n_tokens);\n\n                Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, model.layers[il].attn_q_norm_b, LLM_NORM, il);\n\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            }\n\n            if (model.layers[il].attn_k_norm) {\n                Kcur = ggml_reshape_2d(ctx0, Kcur, n_embd_head * n_head_kv, n_tokens);\n\n                Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, model.layers[il].attn_k_norm_b, LLM_NORM, il);\n\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            }\n\n            // RoPE\n            if (model.arch == LLM_ARCH_NOMIC_BERT || model.arch == LLM_ARCH_NOMIC_BERT_MOE ||\n                model.arch == LLM_ARCH_JINA_BERT_V3) {\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                     ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                     ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n            cb(cur, \"kqv_out\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // re-add the layer input\n        cur = ggml_add(ctx0, cur, inpL);\n\n        // attention layer norm\n        cur = build_norm(cur, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, il);\n\n        if (model.layers[il].attn_norm_2 != nullptr) {\n            cur = ggml_add(ctx0, cur, inpL);  // re-add the layer input\n            cur = build_norm(cur, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, il);\n        }\n\n        ggml_tensor * ffn_inp = cur;\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        if (hparams.moe_every_n_layers > 0 && il % hparams.moe_every_n_layers == 1) {\n            // MoE branch\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    nullptr,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    hparams.n_expert, hparams.n_expert_used,\n                    LLM_FFN_GELU, false,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n            cb(cur, \"ffn_moe_out\", il);\n        } else if (model.arch == LLM_ARCH_BERT || model.arch == LLM_ARCH_NOMIC_BERT_MOE ||\n                   model.arch == LLM_ARCH_JINA_BERT_V3) {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                    NULL, NULL, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        } else if (model.arch == LLM_ARCH_JINA_BERT_V2) {\n            const bool up_contains_gate = !model.layers[il].ffn_gate && model.layers[il].ffn_up->ne[1] != hparams.n_ff();\n            auto type_op = up_contains_gate ? LLM_FFN_GEGLU : LLM_FFN_GELU;\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL, NULL,\n                    type_op, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        // attentions bypass the intermediate layer\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        // output layer norm\n        cur = build_norm(cur, model.layers[il].layer_out_norm, model.layers[il].layer_out_norm_b, LLM_NORM, il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cb(cur, \"result_embd\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/bitnet.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_bitnet::llm_build_bitnet(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            // B1.K\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            // B1.V\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    NULL, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n\n            cur = build_norm(cur,\n                    model.layers[il].attn_sub_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"attn_sub_norm\", il);\n\n            cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);\n            if (model.layers[il].bo) {\n                cur = ggml_add(ctx0, cur, model.layers[il].bo);\n            }\n            cb(cur, \"attn_out\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward forward\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, model.layers[il].ffn_up_s,\n                model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,\n                NULL,                      NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_sub_out\", il);\n\n        cur = build_norm(cur,\n                model.layers[il].ffn_sub_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_sub_norm\", il);\n\n        cur = build_lora_mm(model.layers[il].ffn_down, cur, model.layers[il].ffn_down_s);\n        cb(cur, \"ffn_down\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    // FIXME: do not use model.tok_embd directly, duplicate as model.output\n    cur = build_lora_mm(model.tok_embd, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/bloom.cpp",
    "content": "#include \"models.h\"\n\nllm_build_bloom::llm_build_bloom(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    inpL = build_norm(inpL,\n            model.tok_norm,\n            model.tok_norm_b,\n            LLM_NORM, -1);\n    cb(inpL, \"inp_norm\", -1);\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // Add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/chameleon.cpp",
    "content": "#include \"models.h\"\n\n#include <float.h>\n\nllm_build_chameleon::llm_build_chameleon(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        if (hparams.swin_norm) {\n            cur = inpL;\n        } else {\n            cur = build_norm(inpL,\n                    model.layers[il].attn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n        }\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            if (model.layers[il].attn_q_norm) {\n                Qcur = ggml_view_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens,\n                        ggml_element_size(Qcur) * n_embd_head,\n                        ggml_element_size(Qcur) * n_embd_head * n_head,\n                        0);\n                cb(Qcur, \"Qcur\", il);\n\n                Qcur = build_norm(Qcur,\n                        model.layers[il].attn_q_norm,\n                        model.layers[il].attn_q_norm_b,\n                        LLM_NORM, il);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            if (model.layers[il].attn_k_norm) {\n                Kcur = ggml_view_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens,\n                        ggml_element_size(Kcur) * n_embd_head,\n                        ggml_element_size(Kcur) * n_embd_head * n_head_kv,\n                        0);\n                cb(Kcur, \"Kcur\", il);\n\n                Kcur = build_norm(Kcur,\n                        model.layers[il].attn_k_norm,\n                        model.layers[il].attn_k_norm_b,\n                        LLM_NORM, il);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        if (hparams.swin_norm) {\n            cur = build_norm(cur,\n                    model.layers[il].attn_norm, NULL,\n                    LLM_NORM_RMS, il);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        if (!hparams.swin_norm) {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n        }\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        if (hparams.swin_norm) {\n            cur = build_norm(cur,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output_with_img_logits\", -1);\n\n    // TODO: this suppresses the output of image tokens, which is required to enable text-only outputs.\n    // Needs to be removed once image outputs are supported.\n    int img_token_end_idx = 8196;\n    int img_token_start_idx = 4;\n    int num_img_tokens = img_token_end_idx - img_token_start_idx;\n    // creates 1d tensor of size num_img_tokens and values -FLT_MAX,\n    // which ensures that text token values are always at least larger than image token values\n    ggml_tensor * img_logits = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, num_img_tokens);\n    img_logits = ggml_clamp(ctx0, img_logits, -FLT_MAX, -FLT_MAX);\n    cb(img_logits, \"img_logits\", -1);\n\n    cur = ggml_set_1d(ctx0, cur, img_logits, ggml_element_size(cur) * img_token_start_idx);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/chatglm.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_chatglm::llm_build_chatglm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = nullptr;\n            ggml_tensor * Kcur = nullptr;\n            ggml_tensor * Vcur = nullptr;\n\n            if (model.layers[il].wqkv == nullptr) {\n                Qcur = build_lora_mm(model.layers[il].wq, cur);\n                if (model.layers[il].bq) {\n                    Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                }\n                Kcur = build_lora_mm(model.layers[il].wk, cur);\n                if (model.layers[il].bk) {\n                    Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                }\n                Vcur = build_lora_mm(model.layers[il].wv, cur);\n                if (model.layers[il].bv) {\n                    Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                }\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n            } else {\n                cur = build_lora_mm(model.layers[il].wqkv, cur);\n                cb(cur, \"wqkv\", il);\n                if (model.layers[il].bqkv) {\n                    cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n                    cb(cur, \"bqkv\", il);\n                }\n                Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n                Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n                Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n            }\n\n            //printf(\"freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\\n\", freq_base, freq_scale, ext_factor, attn_factor);\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // Add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    NULL,                      NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n\n        }\n\n        inpL = ggml_add(ctx0, cur, ffn_inp);\n        cb(inpL, \"l_out\", il);\n    }\n\n    cur = build_norm(inpL,\n            model.output_norm,\n            NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/codeshell.cpp",
    "content": "#include \"models.h\"\n\nllm_build_codeshell::llm_build_codeshell(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/cogvlm.cpp",
    "content": "#include \"models.h\"\n\nllm_build_cogvlm::llm_build_cogvlm(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const float   kq_scale    = 1.0f / sqrtf(float(n_embd_head));\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * inpL;\n    ggml_tensor * cur;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    // check ubatch to see if we have input tokens (text)\n    // or an input embedding vector (image)\n    bool is_text;\n    if (ubatch.token) {\n        is_text = true;\n    } else {\n        is_text = false;\n    }\n\n    for (int il = 0; il < n_layer; ++il) {\n        // get either the text or image weight tensors\n        ggml_tensor *wqkv, *wo;\n        ggml_tensor *ffn_gate, *ffn_down, *ffn_up;\n\n        if (is_text) {\n            wqkv     = model.layers[il].wqkv;\n            wo       = model.layers[il].wo;\n            ffn_gate = model.layers[il].ffn_gate;\n            ffn_down = model.layers[il].ffn_down;\n            ffn_up   = model.layers[il].ffn_up;\n        } else {\n            wqkv     = model.layers[il].visexp_attn_wqkv;\n            wo       = model.layers[il].visexp_attn_wo;\n            ffn_gate = model.layers[il].visexp_ffn_gate;\n            ffn_down = model.layers[il].visexp_ffn_down;\n            ffn_up   = model.layers[il].visexp_ffn_up;\n        }\n\n        ggml_tensor * inpSA = inpL;\n        cur = build_norm(inpSA, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n\n        // build self attention\n        {\n            ggml_tensor * qkv = build_lora_mm(wqkv, cur);\n\n            // split qkv into Q, K, V along the first dimension\n            ggml_tensor * Qcur =\n                ggml_view_3d(ctx0, qkv, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), qkv->nb[1], 0);\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              qkv->nb[1], n_embd * ggml_element_size(qkv));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              qkv->nb[1], 2 * n_embd * ggml_element_size(qkv));\n\n            Qcur = ggml_rope(ctx0, Qcur, inp_pos, n_embd_head, rope_type);\n            Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type);\n\n            cur = build_attn(inp_attn,\n                wo, nullptr,\n                Qcur, Kcur, Vcur,\n                nullptr, nullptr, nullptr,\n                kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                ffn_up, NULL, NULL,\n                ffn_gate, NULL, NULL,\n                ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/cohere2-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_cohere2_iswa::llm_build_cohere2_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    const float f_logit_scale = hparams.f_logit_scale;\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv_iswa();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const bool is_swa = hparams.is_swa(il);\n        // UNUSED:\n        // const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        // const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n        ggml_tensor * ffn_inp = cur;\n\n        // self-attention\n        {\n            // rope freq factors for 128k context\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (is_swa) {\n                Qcur = ggml_rope_ext(\n                        ctx0, Qcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n\n                Kcur = ggml_rope_ext(\n                        ctx0, Kcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur     = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL    = ggml_get_rows(ctx0, inpL, inp_out_ids);\n            ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n        }\n\n        ggml_tensor * attn_out = cur;\n\n        // feed-forward network\n        {\n            cur = build_ffn(ffn_inp,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        // add together residual + FFN + self-attention\n        cur = ggml_add(ctx0, cur, inpL);\n        cur = ggml_add(ctx0, cur, attn_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    if (f_logit_scale) {\n        cur = ggml_scale(ctx0, cur, f_logit_scale);\n    }\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/command-r.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_command_r::llm_build_command_r(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    const float f_logit_scale = hparams.f_logit_scale;\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_tensor * ffn_inp = cur;\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (model.layers[il].attn_q_norm) {\n                Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il);\n                cb(Qcur, \"Qcur\", il);\n            }\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            if (model.layers[il].attn_k_norm) {\n                Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM, il);\n                cb(Kcur, \"Kcur\", il);\n            }\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur     = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL    = ggml_get_rows(ctx0, inpL, inp_out_ids);\n            ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n        }\n        ggml_tensor * attn_out = cur;\n\n        // feed-forward network\n        {\n            cur = build_ffn(ffn_inp,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        // add together residual + FFN + self-attention\n        cur = ggml_add(ctx0, cur, inpL);\n        cur = ggml_add(ctx0, cur, attn_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    if (f_logit_scale) {\n        cur = ggml_scale(ctx0, cur, f_logit_scale);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/dbrx.cpp",
    "content": "#include \"models.h\"\n\nllm_build_dbrx::llm_build_dbrx(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = nullptr;\n            ggml_tensor * Kcur = nullptr;\n            ggml_tensor * Vcur = nullptr;\n\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);\n            cb(cur, \"wqkv_clamped\", il);\n\n            Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].attn_out_norm, NULL,\n                LLM_NORM, il);\n        cb(cur, \"attn_out_norm\", il);\n\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/deci.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_deci::llm_build_deci(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA     = inpL;\n        const int64_t n_head_kv = hparams.n_head_kv(il);\n        const int64_t n_head    = hparams.n_head(il);\n        const int64_t n_ff      = hparams.n_ff(il);\n\n        if (n_head == 0) {\n            // attention-free layer of Llama-3_1-Nemotron-51B\n            cur = inpL;\n        } else {\n            // norm\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n        }\n        if (n_head > 0 && n_head_kv == 0) {\n            // \"linear attention\" of Llama-3_1-Nemotron-51B\n            cur = build_lora_mm(model.layers[il].wo, cur);\n            cb(cur, \"wo\", il);\n        } else if (n_head > 0) {\n            // self-attention\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        // FFN-free layer of Llama-3_1-Nemotron-Ultra-253B\n        if (n_ff == 0) {\n            continue;\n        }\n        // modified to support attention-free layer of Llama-3_1-Nemotron-51B\n        ggml_tensor * ffn_inp = cur;\n        if (n_head > 0) {\n            ffn_inp = ggml_add(ctx0, cur, inpSA);\n            cb(ffn_inp, \"ffn_inp\", il);\n        }\n        // feed-forward network\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/deepseek.cpp",
    "content": "#include \"models.h\"\n\nllm_build_deepseek::llm_build_deepseek(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        if ((uint32_t) il < hparams.n_layer_dense_lead) {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, false,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // FFN shared expert\n            {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/deepseek2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_deepseek2::llm_build_deepseek2(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const bool is_mla = hparams.is_mla();\n\n    // note: these are the actual head sizes you get when treating as MHA or after \"decompression\" using wv_b for MLA\n    const int64_t n_embd_head_k = hparams.n_embd_head_k_mla();\n    const int64_t n_embd_head_v = hparams.n_embd_head_v_mla();\n\n    const int64_t n_embd_head_qk_rope = hparams.n_rot();\n    const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope;\n\n    const uint32_t kv_lora_rank = hparams.n_lora_kv;\n\n    // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly.\n    // See https://github.com/ggml-org/llama.cpp/discussions/7416 for detailed explanation.\n    // And also: https://github.com/ggml-org/llama.cpp/pull/17945 [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]\n\n    // first cancel the adjustment from llama_hparams::yarn_attn_factor_adjust to get the original attn_factor\n    GGML_ASSERT(ext_factor >= 0.0f);\n    const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale));\n\n    // use the original attn_factor to pre-scale the kq_scale\n    const float mscale   = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale));\n    const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k));\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    // {n_embd, n_tokens}\n    inpL = build_inp_embd(model.tok_embd);\n\n    // (optional) temperature tuning - used by mistral-large\n    ggml_tensor * inp_attn_scale = nullptr;\n    if (hparams.f_attn_temp_scale != 0.0f) {\n        inp_attn_scale = build_inp_attn_scale();\n    }\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn_kv = !is_mla ? build_attn_inp_kv() : nullptr;\n    auto * inp_attn_k  =  is_mla ? build_attn_inp_k()  : nullptr;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    int effective_n_layers = hparams.n_layer - hparams.nextn_predict_layers;\n    for (int il = 0; il < effective_n_layers; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            ggml_tensor * q = NULL;\n\n            const bool is_lite = model.layers[il].wq;\n\n            if (!is_lite) {\n                q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur);\n                cb(q, \"q\", il);\n\n                q = build_norm(q, model.layers[il].attn_q_a_norm, nullptr, LLM_NORM_RMS, il);\n                cb(q, \"q\", il);\n\n                q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);\n                cb(q, \"q\", il);\n            } else {\n                q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);\n                cb(q, \"q\", il);\n            }\n            // split into {n_embd_head_qk_nope, n_head, n_tokens}\n            ggml_tensor * q_nope =\n                ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),\n                             ggml_row_size(q->type, n_embd_head_k) * n_head, 0);\n            cb(q_nope, \"q_nope\", il);\n\n            // and {n_embd_head_qk_rope, n_head, n_tokens}\n            ggml_tensor * q_pe = ggml_view_3d(\n                ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k),\n                ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope));\n            cb(q_pe, \"q_pe\", il);\n\n            ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);\n            cb(kv_cmpr_pe, \"kv_cmpr_pe\", il);\n\n            // split into {kv_lora_rank, n_tokens}\n            ggml_tensor * kv_cmpr =\n                ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,\n                             ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);\n            cb(kv_cmpr, \"kv_cmpr\", il);\n\n            // and {n_embd_head_qk_rope, 1, n_tokens}\n            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,\n                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),\n                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),\n                                              ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));\n            cb(k_pe, \"k_pe\", il);\n\n            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n            cb(q_pe, \"q_pe\", il);\n\n            k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n            cb(k_pe, \"k_pe\", il);\n\n            kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);\n            cb(kv_cmpr, \"kv_cmpr\", il);\n\n            if (is_mla) {\n                // {n_embd_head_qk_nope, n_tokens, n_head}\n                q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);\n                cb(q_nope, \"q_nope_perm\", il);\n\n                // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head}\n                ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, model.layers[il].wk_b, q_nope);\n                cb(q_nope_absorbed, \"q_nope_absorbed\", il);\n\n                // {kv_lora_rank, n_head, n_tokens}\n                q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);\n                cb(q_nope_absorbed, \"q_nope_absorbed_perm\", il);\n\n                // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens}\n                // note: rope must go first for in-place context shifting in build_rope_shift()\n                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);\n                cb(Qcur, \"Qcur\", il);\n\n                kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);\n                cb(kv_cmpr, \"kv_cmpr_reshape\", il);\n\n                // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}\n                ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);\n                cb(Kcur, \"Kcur\", il);\n\n                // {kv_lora_rank, 1, n_tokens}\n                ggml_tensor * Vcur = kv_cmpr;\n                cb(Vcur, \"Vcur\", il);\n\n                if (inp_attn_scale) {\n                    // apply llama 4 temperature scaling\n                    Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);\n                    cb(Qcur, \"Qcur_attn_temp_scaled\", il);\n                }\n\n                // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group)\n                cur = build_attn(inp_attn_k,\n                        model.layers[il].wo, NULL,\n                        Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, kq_scale, il);\n            } else {\n                ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_cmpr);\n                cb(kv, \"kv\", il);\n\n                // split into {n_embd_head_qk_nope, n_head, n_tokens}\n                ggml_tensor * k_nope =\n                    ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,\n                                 ggml_row_size(kv->type, n_embd_head_qk_nope + n_embd_head_v),\n                                 ggml_row_size(kv->type, n_embd_head_qk_nope + n_embd_head_v) * n_head, 0);\n                cb(k_nope, \"k_nope_view\", il);\n\n                // and {n_embd_head_v, n_head, n_tokens}\n                ggml_tensor * Vcur = ggml_view_3d(ctx0, kv, n_embd_head_v, n_head, n_tokens,\n                                                  ggml_row_size(kv->type, n_embd_head_qk_nope + n_embd_head_v),\n                                                  ggml_row_size(kv->type, n_embd_head_qk_nope + n_embd_head_v) * n_head,\n                                                  ggml_row_size(kv->type, n_embd_head_qk_nope));\n                cb(Vcur, \"Vcur_view\", il);\n\n                Vcur = ggml_cont(ctx0, Vcur);\n                cb(Vcur, \"Vcur_cont\", il);\n\n                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope, q_pe, 0);\n                cb(Qcur, \"Qcur\", il);\n\n                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);\n                cb(Kcur, \"Kcur\", il);\n\n                if (inp_attn_scale) {\n                    // apply llama 4 temperature scaling\n                    Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);\n                    cb(Qcur, \"Qcur_attn_temp_scaled\", il);\n                }\n\n                // note: MLA without the absorption optimization converts into MHA (ie: GQA with full n_head groups)\n                cur = build_attn(inp_attn_kv,\n                            model.layers[il].wo, NULL,\n                            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            }\n        }\n        if (il == effective_n_layers - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        if ((uint32_t) il < hparams.n_layer_dense_lead) {\n            cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, hparams.expert_weights_norm,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il,\n                nullptr,\n                model.layers[il].ffn_gate_up_exps);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // FFN shared expert\n            {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = ggml_mul_mat(ctx0, model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/delta-net-base.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-impl.h\"\n\n// utility to get one slice from the third dimension\n// input dim:  [x, y, c, b]\n// output dim: [x, y, 1, b]\nstatic ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) {\n    return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3],\n        t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c);\n}\n\nllm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_chunking(\n        ggml_tensor * q,\n        ggml_tensor * k,\n        ggml_tensor * v,\n        ggml_tensor * g,\n        ggml_tensor * b,\n        ggml_tensor * s,\n        int           il) {\n    const int64_t S_k      = q->ne[0];\n    const int64_t H_k      = q->ne[1];\n    const int64_t n_tokens = q->ne[2];\n    const int64_t n_seqs   = q->ne[3];\n\n    const int64_t S_v = v->ne[0];\n    const int64_t H_v = v->ne[1];\n    const bool kda = (g->ne[0] == S_k && g->ne[1] == H_k);\n\n    GGML_ASSERT(S_k == S_v);\n    GGML_ASSERT(H_v % H_k == 0);\n\n    GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);\n    GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);\n    GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);\n\n    GGML_ASSERT(g->ne[0] == 1   || g->ne[0] == S_v);\n    GGML_ASSERT(                   g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);\n    GGML_ASSERT(b->ne[0] == 1   && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);\n    GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v      && s->ne[3] == n_seqs);\n\n    const float scale = 1.0f / sqrtf(S_k);\n\n    q = ggml_scale(ctx0, q, scale);\n\n    cb(q, \"q_in\", il);\n    cb(k, \"k_in\", il);\n    cb(v, \"v_in\", il);\n    cb(b, \"b_in\", il);\n    cb(g, \"g_in\", il);\n\n    q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]\n    k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]\n    v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]\n    g = ggml_permute(ctx0, g, 0, 2, 1, 3); // [g_0, n_tokens, H_v, n_seqs]\n    b = ggml_permute(ctx0, b, 0, 2, 1, 3); // [  1, n_tokens, H_v, n_seqs]\n\n    const int CS = kda ? 16 : 64; // chunk size\n\n    const int pad = (CS - n_tokens % CS) % CS;\n    const int n_chunks = (n_tokens + pad) / CS;\n\n    q = ggml_pad(ctx0, q, 0, pad, 0, 0);\n    k = ggml_pad(ctx0, k, 0, pad, 0, 0);\n    v = ggml_pad(ctx0, v, 0, pad, 0, 0);\n    g = ggml_pad(ctx0, g, 0, pad, 0, 0);\n    b = ggml_pad(ctx0, b, 0, pad, 0, 0);\n\n    ggml_tensor * v_b = ggml_mul(ctx0, v, b);\n    ggml_tensor * k_b = ggml_mul(ctx0, k, b);\n\n    cb(v_b, \"v_b\", il);\n    cb(k_b, \"k_b\", il);\n\n    q   = ggml_reshape_4d(ctx0, q,   S_k, CS, n_chunks, H_k * n_seqs);\n    k   = ggml_reshape_4d(ctx0, k,   S_k, CS, n_chunks, H_k * n_seqs);\n    k_b = ggml_reshape_4d(ctx0, k_b, S_k, CS, n_chunks, H_v * n_seqs);\n    v   = ggml_reshape_4d(ctx0, v,   S_v, CS, n_chunks, H_v * n_seqs);\n    v_b = ggml_reshape_4d(ctx0, v_b, S_v, CS, n_chunks, H_v * n_seqs);\n\n    g = ggml_reshape_4d(ctx0, g, g->ne[0], CS, n_chunks, H_v * n_seqs);\n    b = ggml_reshape_4d(ctx0, b, 1,        CS, n_chunks, H_v * n_seqs);\n\n    // [CS, g_0, n_chunks, H_v * n_seqs]\n    // TODO: extend ggml_cumsum with axis parameter to avoid transpose\n    ggml_tensor * g_cs = ggml_cumsum(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, g)));\n    cb(g_cs, \"g_cs\", il);\n\n    ggml_tensor * kb = nullptr;\n    ggml_tensor * kq = nullptr;\n    if (kda) {\n        const int64_t CHB = n_chunks * H_k * n_seqs;\n\n        ggml_tensor * g_cs_i = ggml_reshape_4d(ctx0, g_cs, CS, 1, S_k, CHB);  // [chunk_size, 1, S_k, CHB]\n        ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, S_k, CHB);  // [1, chunk_size, S_k, CHB]\n\n        g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, S_k, CHB);  // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB]\n\n        // decay_mask [chunk_size,chunk_size,S_k,CHB]\n        ggml_tensor * decay_mask;\n        decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);\n        decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);\n        decay_mask = ggml_exp(ctx0, decay_mask);\n        cb(decay_mask, \"decay_mask\", il);\n\n        // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched\n        decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, CS, CS, CHB);\n\n        ggml_tensor * k_b_i = ggml_reshape_4d(ctx0, k_b, S_k, CS,  1, CHB);\n        ggml_tensor * k_j   = ggml_reshape_4d(ctx0, k,   S_k,  1, CS, CHB);\n        ggml_tensor * q_i   = ggml_reshape_4d(ctx0, q,   S_k, CS,  1, CHB);\n\n        ggml_tensor * decay_k_b_i = ggml_mul(ctx0, decay_mask, k_b_i);\n        ggml_tensor * decay_q_i   = ggml_mul(ctx0, decay_mask, q_i);\n\n        // decay_k_b_i [S,BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB]\n        kb = ggml_mul_mat(ctx0, decay_k_b_i, k_j);\n        kq = ggml_mul_mat(ctx0, decay_q_i,   k_j);\n\n        kb = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kb, CS, CS, n_chunks, H_v * n_seqs)));\n        kq = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, kq, CS, CS, n_chunks, H_v * n_seqs)));\n    } else {\n        ggml_tensor * g_cs_i = g_cs;\n        ggml_tensor * g_cs_j = ggml_reshape_4d(ctx0, g_cs, 1, CS, n_chunks, H_v * n_seqs);\n\n        g_cs_j = ggml_repeat_4d(ctx0, g_cs_j, CS, CS, n_chunks, H_v * n_seqs);\n\n        // [CS, CS, n_chunks, H_v * n_seqs]\n        ggml_tensor * decay_mask;\n        decay_mask = ggml_sub(ctx0, g_cs_j, g_cs_i);\n        decay_mask = ggml_tri(ctx0, decay_mask, GGML_TRI_TYPE_LOWER_DIAG);\n        decay_mask = ggml_exp(ctx0, decay_mask);\n        cb(decay_mask, \"decay_mask\", il);\n\n        // [CS, CS, n_chunks, H_k * n_seqs]\n        kb = ggml_mul_mat(ctx0, k,  k_b);\n        kb = ggml_mul    (ctx0, kb, decay_mask);\n\n        // [CS, CS, n_chunks, H_k * n_seqs]\n        kq = ggml_mul_mat(ctx0, k, q);\n        kq = ggml_mul(ctx0, kq, decay_mask);\n    }\n\n    kq = ggml_tri(ctx0, kq, GGML_TRI_TYPE_LOWER_DIAG);\n    cb(kq, \"kq\", il);\n\n    // [CS, CS, n_chunks, H_k * n_seqs]\n    ggml_tensor * attn;\n    attn = ggml_tri(ctx0, kb, GGML_TRI_TYPE_LOWER);\n    cb(attn, \"attn\", il);\n\n    ggml_tensor * identity;\n    identity = ggml_view_1d(ctx0, attn, CS, 0);\n    identity = ggml_fill   (ctx0, identity, 1.0f);\n    identity = ggml_diag   (ctx0, identity);\n\n    ggml_tensor * lhs = ggml_add(ctx0, attn, identity);\n    cb(lhs, \"dnet_add_ch_lhs\", il);\n\n    attn = ggml_neg(ctx0, attn);\n    cb(attn, \"attn_pre_solve\", il);\n\n    ggml_tensor * lin_solve = ggml_solve_tri(ctx0, lhs, attn, true, true, false);\n    attn = ggml_add(ctx0, lin_solve, identity);\n    cb(attn, \"dnet_add_ch_attn_solved\", il); // [CS, CS, n_chunks, H_k * n_seqs]\n\n    // [S_v, CS, n_chunks, H_v * n_seqs]\n    v = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v_b)), attn);\n\n    // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]\n    ggml_tensor * g_exp = ggml_exp(ctx0, g_cs);\n\n    k_b = ggml_cont(ctx0, ggml_transpose(ctx0, k_b));\n\n    // [CS, S_k, n_chunks, H_k * n_seqs]\n    ggml_tensor * kbg = ggml_mul(ctx0, k_b, g_exp);\n    cb(kbg, \"k_beta_g_exp\", il);\n\n    // [S_k, CS, n_chunks, H_k * n_seqs]\n    ggml_tensor * k_cd = ggml_mul_mat(ctx0, kbg, attn);\n    cb(k_cd, \"k_cumdecay\", il);\n\n    // [1, CS, n_chunks, H_k * n_seqs] KDA: [S_k, CS, n_chunks, H_k * n_seqs]\n    ggml_tensor * g_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, g_exp));\n    ggml_tensor * q_g_exp = ggml_mul(ctx0, q, g_exp_t);\n\n    // vectorized calculation of key_gdiff\n    // improved from the chunked version:\n    //   g_last = torch.clamp(g_cum[:, :, -1], max=50.0).exp().unsqueeze(-1).unsqueeze(-1)\n    //   g_diff = torch.clamp(g_cum[:, :, -1:] - g_cum, max=50.0).exp()\n    //   key_gdiff = key * g_diff.unsqueeze(-1)\n    //   kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new\n    //   last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew\n\n    // get last element in g_cumsum along CS dimension (ne0)\n    // example: [[x, y, z, ..., last], ...] -> [[last], ...]\n    // [1, 1, n_chunks, H_v * n_seqs] KDA: [1, S_k, n_chunks, H_v * n_seqs]\n    ggml_tensor * g_last = ggml_view_4d(ctx0, g_cs, 1, g_cs->ne[1], g_cs->ne[2], g_cs->ne[3],\n            g_cs->nb[1],\n            g_cs->nb[2],\n            g_cs->nb[3],\n            ggml_row_size(g_cs->type, g_cs->ne[0] - 1));\n    cb(g_last, \"g_last\", il);\n\n    // TODO: remove this cont when CUDA supports non-cont unary ops\n    g_last = ggml_cont(ctx0, g_last);\n\n    // [1, 1, n_chunks, H_v * n_seqs] KDA: [S_k, 1, n_chunks, H_v * n_seqs]\n    ggml_tensor * g_last_exp_t = ggml_transpose(ctx0, ggml_exp(ctx0, g_last));\n    cb(g_last_exp_t, \"g_last_exp_t\", il);\n\n    // [CS, 1, n_chunks, H_v * n_seqs] KDA: [CS, S_k, n_chunks, H_v * n_seqs]\n    ggml_tensor * g_diff = ggml_neg(ctx0, ggml_sub(ctx0, g_cs, g_last));\n    cb(g_diff, \"g_diff\", il);\n\n    ggml_tensor * g_diff_exp_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_exp(ctx0, g_diff)));\n\n    // [S_k, CS, n_chunks, H_v * n_seqs]\n    ggml_tensor * kg = ggml_mul(ctx0, k, g_diff_exp_t);\n    cb(kg, \"key_gdiff\", il);\n\n    // [CS, S_k, n_chunks, H_v * n_seqs]\n    ggml_tensor * kg_t = ggml_cont(ctx0, ggml_transpose(ctx0, kg));\n    cb(kg_t, \"key_gdiff_t\", il);\n\n    s = ggml_reshape_4d(ctx0, s, S_v, S_v, 1, H_v * n_seqs);\n    cb(s, \"dnet_add_ch_state\", il);\n\n    // [CS, S_v, n_chunks, H_v * n_seqs]\n    ggml_tensor * v_t = ggml_cont(ctx0, ggml_transpose(ctx0, v));\n\n    for (int64_t chunk = 0; chunk < n_chunks; chunk++) {\n        ggml_tensor * ch_k_cd    = get_slice_2d(ctx0, k_cd,    chunk); // [S_k,  CS, 1, H_k * n_seqs]\n        ggml_tensor * ch_v_t     = get_slice_2d(ctx0, v_t,     chunk); // [ CS, S_v, 1, H_v * n_seqs]\n        ggml_tensor * ch_kq      = get_slice_2d(ctx0, kq,      chunk); // [ CS,  CS, 1, H_k * n_seqs]\n        ggml_tensor * ch_q_g_exp = get_slice_2d(ctx0, q_g_exp, chunk); // [S_k,  CS, 1, H_k * n_seqs]\n        ggml_tensor * ch_kg_t    = get_slice_2d(ctx0, kg_t,    chunk); // [ CS, S_k, 1, H_v * n_seqs]\n\n        // [CS, S_v, 1, H_v * n_seqs]\n        ggml_tensor * v_t_p = ggml_mul_mat(ctx0, ch_k_cd, s);\n        cb(v_t_p, \"v_prime\", il);\n\n        // [CS, S_v, 1, H_v * n_seqs]\n        ggml_tensor * v_t_new = ggml_sub(ctx0, ch_v_t, v_t_p);\n        cb(v_t_new, \"v_t_new\", il);\n\n        // [S_v, CS, 1, H_v * n_seqs]\n        ggml_tensor * v_attn = ggml_mul_mat(ctx0, v_t_new, ch_kq);\n        cb(v_attn, \"v_attn\", il);\n\n        // [S_v, CS, 1, H_v * n_seqs]\n        ggml_tensor * attn_inter = ggml_mul_mat(ctx0, s, ch_q_g_exp);\n        cb(attn_inter, \"attn_inter\", il);\n\n        // [S_v, CS, 1, H_v * n_seqs]\n        ggml_tensor * o_ch = ggml_add(ctx0, attn_inter, v_attn);\n        cb(o_ch, \"dnet_add_ch_attn_out\", il);\n\n        v = ggml_set_inplace(ctx0, v, o_ch, v->nb[1], v->nb[2], v->nb[3], chunk * v->nb[2]);\n\n        // kgdmulvnew = (key_gdiff).transpose(-1, -2) @ v_new\n        // TODO: head broadcast might not work here - probably will need a transpose\n        ggml_tensor * kgv = ggml_mul_mat(ctx0, ch_kg_t, v_t_new); // [S_k, S_v, 1, H_k * n_seqs]\n\n        // last_recurrent_state = last_recurrent_state * g_last + kgdmulvnew\n        ggml_tensor * ch_g_last_exp_t = get_slice_2d(ctx0, g_last_exp_t, chunk);\n\n        s = ggml_mul(ctx0, s, ch_g_last_exp_t);\n        s = ggml_add(ctx0, s, kgv);\n        cb(s, \"dnet_add_ch_state\", il);\n    }\n\n    // truncate padded tokens\n    ggml_tensor * o = ggml_view_4d(ctx0, v,\n            S_v, n_tokens, H_v, n_seqs,\n            ggml_row_size(v->type, S_v),\n            ggml_row_size(v->type, S_v * CS * n_chunks),\n            ggml_row_size(v->type, S_v * CS * n_chunks * H_v), 0);\n    o = ggml_permute  (ctx0, o, 0, 2, 1, 3); // [S_v, H_v, n_tokens, n_seqs]\n    s = ggml_reshape_4d(ctx0, s, S_v, S_v, H_v, n_seqs);\n    cb(s, \"output_state\", il);\n\n    return {o, s};\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_autoregressive(\n        ggml_tensor * q,\n        ggml_tensor * k,\n        ggml_tensor * v,\n        ggml_tensor * g,\n        ggml_tensor * b, // beta\n        ggml_tensor * s, // state\n        int           il) {\n    const int64_t S_k      = q->ne[0];\n    const int64_t H_k      = q->ne[1];\n    const int64_t n_tokens = q->ne[2];\n    const int64_t n_seqs   = q->ne[3];\n\n    const int64_t S_v = v->ne[0];\n    const int64_t H_v = v->ne[1];\n\n    GGML_ASSERT(n_tokens == 1);\n\n    GGML_ASSERT(S_k == S_v);\n    GGML_ASSERT(H_v % H_k == 0);\n\n    GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);\n    GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);\n    GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);\n\n    GGML_ASSERT(g->ne[0] == 1   || g->ne[0] == S_v);\n    GGML_ASSERT(                   g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);\n    GGML_ASSERT(b->ne[0] == 1   && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);\n    GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v      && s->ne[3] == n_seqs);\n\n    const float scale = 1.0f / sqrtf(S_k);\n\n    q = ggml_scale(ctx0, q, scale);\n\n    q = ggml_permute(ctx0, q, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]\n    k = ggml_permute(ctx0, k, 0, 2, 1, 3); // [S_k, n_tokens, H_k, n_seqs]\n    v = ggml_permute(ctx0, v, 0, 2, 1, 3); // [S_v, n_tokens, H_v, n_seqs]\n\n    cb(q, \"q_in\", il);\n    cb(k, \"k_in\", il);\n    cb(v, \"v_in\", il);\n    cb(b, \"b_in\", il);\n    cb(g, \"g_in\", il);\n\n    // GDA: [1,  1,  H_v, n_seqs]\n    // KDA: [1, S_k, H_v, n_seqs]\n    g = ggml_reshape_4d(ctx0, g, 1, g->ne[0], H_v, n_seqs);\n    b = ggml_reshape_4d(ctx0, b, 1,        1, H_v, n_seqs);\n\n    // [S_v, S_v, H_v, n_seqs]\n    g = ggml_exp(ctx0, g);\n    s = ggml_mul(ctx0, s, g);\n\n    // [1, S_v, H_v, n_seqs]\n    ggml_tensor * sk;\n    sk = ggml_mul     (ctx0, s, k);\n    sk = ggml_sum_rows(ctx0, sk);\n\n    // [S_v, 1, H_v, n_seqs]\n    ggml_tensor * d;\n    d = ggml_sub(ctx0, v, ggml_transpose(ctx0, sk));\n    d = ggml_mul(ctx0, d, b);\n\n    // [1, S_v, H_v, n_seqs]\n    ggml_tensor * d_t;\n    d_t = ggml_transpose(ctx0, d);\n\n    // [S_v, S_v, H_v, n_seqs]\n    ggml_tensor * kd;\n    k  = ggml_repeat(ctx0, k, s);\n    kd = ggml_mul   (ctx0, k, d_t);\n\n    s = ggml_add(ctx0, s, kd);\n\n    cb(s, \"dnet_add_ar_state\", il);\n\n    ggml_tensor * s_q = ggml_mul     (ctx0, s, q);\n    ggml_tensor * o   = ggml_sum_rows(ctx0, s_q);\n\n    o = ggml_permute  (ctx0, o, 2, 0, 1, 3); // [S_v, H_v, n_tokens, n_seqs]\n\n    return {o, s};\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net_fused(\n        ggml_tensor * q,\n        ggml_tensor * k,\n        ggml_tensor * v,\n        ggml_tensor * g,\n        ggml_tensor * b,\n        ggml_tensor * s,\n        int           il) {\n    const int64_t S_k      = q->ne[0];\n    const int64_t H_k      = q->ne[1];\n    const int64_t n_tokens = q->ne[2];\n    const int64_t n_seqs   = q->ne[3];\n\n    const int64_t S_v = v->ne[0];\n    const int64_t H_v = v->ne[1];\n\n    GGML_ASSERT(S_k == S_v);\n    GGML_ASSERT(H_v % H_k == 0);\n\n    GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);\n    GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);\n    GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);\n\n    GGML_ASSERT(g->ne[0] == 1   || g->ne[0] == S_v);\n    GGML_ASSERT(                   g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);\n    GGML_ASSERT(b->ne[0] == 1   && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);\n    GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v      && s->ne[3] == n_seqs);\n\n    ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s);\n    if (n_tokens == 1) {\n        cb(result, LLAMA_TENSOR_NAME_FGDN_AR, il);\n    } else {\n        cb(result, LLAMA_TENSOR_NAME_FGDN_CH, il);\n    }\n\n    ggml_tensor * output = ggml_view_4d(ctx0, result,\n            S_v, H_v, n_tokens, n_seqs,\n            ggml_row_size(result->type, S_v),\n            ggml_row_size(result->type, S_v * H_v),\n            ggml_row_size(result->type, S_v * H_v * n_tokens), 0);\n\n    ggml_tensor * new_state = ggml_view_4d(ctx0, result,\n            S_v, S_v, H_v, n_seqs,\n            ggml_row_size(result->type, S_v),\n            ggml_row_size(result->type, S_v * S_v),\n            ggml_row_size(result->type, S_v * S_v * H_v),\n            ggml_row_size(result->type, S_v * H_v * n_tokens * n_seqs));\n\n    return {output, new_state};\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net(\n        ggml_tensor * q,\n        ggml_tensor * k,\n        ggml_tensor * v,\n        ggml_tensor * g,\n        ggml_tensor * b,\n        ggml_tensor * s,\n        int           il) {\n    const int64_t n_seq_tokens = q->ne[2];\n\n    if (n_seq_tokens == 1) {\n        if (cparams.fused_gdn_ar) {\n            return build_delta_net_fused(q, k, v, g, b, s, il);\n        }\n        return build_delta_net_autoregressive(q, k, v, g, b, s, il);\n    }\n\n    if (cparams.fused_gdn_ch) {\n        return build_delta_net_fused(q, k, v, g, b, s, il);\n    }\n\n    return build_delta_net_chunking(q, k, v, g, b, s, il);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/dots1.cpp",
    "content": "#include \"models.h\"\n\nllm_build_dots1::llm_build_dots1(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        if ((uint32_t) il < hparams.n_layer_dense_lead) {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, hparams.expert_weights_norm,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/dream.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_dream::llm_build_dream(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    //copied from qwen2\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            Qcur               = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            Kcur               = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            Vcur               = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n            model.layers[il].ffn_up, NULL, NULL,\n            model.layers[il].ffn_gate, NULL, NULL,\n            model.layers[il].ffn_down, NULL, NULL,\n            NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/ernie4-5-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_ernie4_5_moe::llm_build_ernie4_5_moe(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    GGML_ASSERT(hparams.n_moe_layer_step > 0 && \"Ernie 4.5 MoE requires n_moe_layer_step > 0\");\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n        // norm\n        {\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n        }\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        bool is_moe_layer =\n            static_cast<uint32_t>(il) >= hparams.n_layer_dense_lead && (il + 1) % hparams.n_moe_layer_step == 0;\n\n        if (!is_moe_layer) {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                                        model.layers[il].ffn_gate_inp,\n                                        model.layers[il].ffn_up_exps,\n                                        model.layers[il].ffn_gate_exps,\n                                        model.layers[il].ffn_down_exps,\n                                        model.layers[il].ffn_exp_probs_b,\n                                        n_expert, n_expert_used,\n                                        LLM_FFN_SILU, true,\n                                        hparams.expert_weights_scale,\n                                        LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                                        il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // Shared expert (if present)\n            if (hparams.n_ff_shexp > 0) {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n            } else {\n                cur = moe_out;\n            }\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/ernie4-5.cpp",
    "content": "#include \"models.h\"\n\nllm_build_ernie4_5::llm_build_ernie4_5(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        {\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n        }\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1) {\n            // skip computing output for unused tokens\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/eurobert.cpp",
    "content": "#include \"models.h\"\n\nllm_build_eurobert::llm_build_eurobert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"inp_embd\", -1);\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * cur = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n\n        {\n            ggml_tensor * Qcur;\n            ggml_tensor * Kcur;\n            ggml_tensor * Vcur;\n\n            Qcur = build_lora_mm(model.layers[il].wq, cur);\n            Kcur = build_lora_mm(model.layers[il].wk, cur);\n            Vcur = build_lora_mm(model.layers[il].wv, cur);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n            cb(cur, \"kqv_out\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        cur = ggml_add(ctx0, cur, inpL);\n\n        ggml_tensor * ffn_inp = cur;\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_embd\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/exaone-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_exaone_moe::llm_build_exaone_moe(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_k();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_v());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn_iswa = build_attn_inp_kv_iswa();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;\n    for (int il = 0; il < n_transformer_layers; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // use RoPE for SWA layers\n        const bool is_local_layer = hparams.is_swa(il);\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            if (is_local_layer) {\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,\n                                     freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,\n                                     freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn_iswa,\n                model.layers[il].wo, NULL,\n                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_transformer_layers - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // norm\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            // dense branch\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL, NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, hparams.expert_weights_norm,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // FFN shared expert\n            {\n                ggml_tensor * ffn_shexp =\n                    build_ffn(cur,\n                        model.layers[il].ffn_up_shexp, NULL, NULL,\n                        model.layers[il].ffn_gate_shexp, NULL, NULL,\n                        model.layers[il].ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // final norm\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/exaone.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_exaone::llm_build_exaone(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/exaone4.cpp",
    "content": "#include \"models.h\"\n\n\ntemplate <bool iswa>\nllm_build_exaone4<iswa>::llm_build_exaone4(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_k();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_v());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type      = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // use RoPE for SWA layers or non-SWA models\n        const bool use_rope = hparams.is_swa(il) || hparams.swa_type == LLAMA_SWA_TYPE_NONE;\n\n        cur = inpL;\n\n        // self-attention\n        {\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            if (use_rope) {\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,\n                                     freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,\n                                     freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_ffn(ffn_inp,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL, NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", -1);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_exaone4<false>;\ntemplate struct llm_build_exaone4<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/falcon-h1.cpp",
    "content": "#include \"models.h\"\n\nllm_build_falcon_h1::llm_build_falcon_h1(const llama_model & model, const llm_graph_params & params) :\n    llm_build_mamba_base(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // Build the inputs in the recurrent & kv cache\n    auto * inp = build_inp_mem_hybrid();\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n        cb(Qcur, \"Qcur\", il);\n\n        ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n        cb(Kcur, \"Kcur\", il);\n\n        ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n        cb(Vcur, \"Vcur\", il);\n\n        Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n        Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n\n        Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n\n        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n\n        cb(Qcur, \"Qcur-post-rope\", il);\n        cb(Kcur, \"Kcur-post-rope\", il);\n        cb(Vcur, \"Vcur-post-rope\", il);\n\n        ggml_tensor * attn_out = build_attn(inp->get_attn(),\n                                    model.layers[il].wo, NULL,\n                                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n        cb(attn_out, \"attn_out\", il);\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        // Mamba2 layer\n        cb(cur, \"ssm_in\", il);\n\n        ggml_tensor * ssm_out = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);\n        cb(ssm_out, \"ssm_out\", il);\n\n        // // Aggregation\n        cur   = ggml_add(ctx0, attn_out, ssm_out);\n        inpSA = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"layer_out\", il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = inpSA;\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, inpSA);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/falcon.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_falcon::llm_build_falcon(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * attn_norm;\n\n        attn_norm = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(attn_norm, \"attn_norm\", il);\n\n        // self-attention\n        {\n            if (model.layers[il].attn_norm_2) {\n                // Falcon-40B\n                cur = build_norm(inpL,\n                        model.layers[il].attn_norm_2,\n                        model.layers[il].attn_norm_2_b,\n                        LLM_NORM, il);\n                cb(cur, \"attn_norm_2\", il);\n            } else {\n                cur = attn_norm;\n            }\n\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            // using mode = 2 for neox mode\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur       = ggml_get_rows(ctx0,       cur, inp_out_ids);\n            inpL      = ggml_get_rows(ctx0,      inpL, inp_out_ids);\n            attn_norm = ggml_get_rows(ctx0, attn_norm, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = cur;\n\n        // feed forward\n        {\n            cur = build_ffn(attn_norm, // !! use the attn norm, not the result\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    NULL,                      NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cur = ggml_add(ctx0, cur, inpL);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    // norm\n    cur = build_norm(cur,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gemma-embedding.cpp",
    "content": "#include \"models.h\"\n\nllm_build_gemma_embedding::llm_build_gemma_embedding(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_k();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)\n    inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base(cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            // ref: https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/model.py#L315\n            Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);\n\n            cur =\n                build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);\n        cb(sa_out, \"sa_out\", il);\n\n        cur = build_norm(sa_out, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        {\n            cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_GELU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", -1);\n\n        cur = ggml_add(ctx0, cur, sa_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gemma.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_gemma::llm_build_gemma(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head)));\n            cb(Qcur, \"Qcur_scaled\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);\n        cb(sa_out, \"sa_out\", il);\n\n        cur = build_norm(sa_out,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, sa_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gemma2-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_gemma2_iswa::llm_build_gemma2_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_k();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv_iswa();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        cur = build_norm(cur,\n                model.layers[il].attn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);\n        cb(sa_out, \"sa_out\", il);\n\n        cur = build_norm(sa_out,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = build_norm(cur,\n                model.layers[il].ffn_post_norm, NULL,\n                LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", -1);\n\n        cur = ggml_add(ctx0, cur, sa_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    // final logit soft-capping\n    cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);\n    cur = ggml_tanh(ctx0, cur);\n    cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gemma3.cpp",
    "content": "#include \"models.h\"\n\ntemplate <bool iswa>\nllm_build_gemma3<iswa>::llm_build_gemma3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_k();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)\n    inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // TODO: is causal == true correct? might need some changes\n    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        float freq_base_l  = 0.0f;\n        float freq_scale_l = 0.0f;\n\n        if constexpr (iswa) {\n            freq_base_l  = model.get_rope_freq_base (cparams, il);\n            freq_scale_l = model.get_rope_freq_scale(cparams, il);\n        } else {\n            freq_base_l  = freq_base;\n            freq_scale_l = freq_scale;\n        }\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            // ref: https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/model.py#L315\n            Qcur = ggml_scale(ctx0, Qcur, hparams.f_attention_scale);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        cur = build_norm(cur,\n                model.layers[il].attn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        ggml_tensor * sa_out = ggml_add(ctx0, cur, inpL);\n        cb(sa_out, \"sa_out\", il);\n\n        cur = build_norm(sa_out,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = build_norm(cur,\n                model.layers[il].ffn_post_norm, NULL,\n                LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, sa_out);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    if (hparams.f_final_logit_softcapping) {\n        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);\n        cur = ggml_tanh(ctx0, cur);\n        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);\n    }\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\ntemplate struct llm_build_gemma3<false>;\ntemplate struct llm_build_gemma3<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/gemma3n-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_gemma3n_iswa::llm_build_gemma3n_iswa(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params),\n    model(model),\n    n_embd_head(model.hparams.n_embd_head_k()),\n    n_embd_altup(model.hparams.n_embd_altup),\n    n_altup(model.hparams.n_altup),\n    i_altup_act(model.hparams.i_altup_act) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)\n    inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // TODO: is causal == true correct? might need some changes\n    auto * inp_attn = build_attn_inp_kv_iswa();\n\n    // inp_per_layer shape: [n_embd_altup, n_tokens, n_layer]\n    ggml_tensor * inp_per_layer = project_per_layer_inputs(inpL, get_per_layer_inputs());\n\n    // inpL now has only 1 altup, project it to the rest of the altups\n    // these \"added\" altups will be concat to the last dim of inpL\n    {\n        ggml_tensor * target_magnitude = calc_magnitude(inpL);\n        ggml_tensor * inp_repeated     = ggml_repeat_4d(ctx0, inpL, n_embd, n_tokens, n_altup - 1, 1);\n        ggml_tensor * altup_added =\n            ggml_mul_mat(ctx0, model.altup_proj, inp_repeated);  // shape: [n_embd, n_tokens, n_altup - 1]\n        ggml_tensor * new_magnitude = calc_magnitude(altup_added);\n        altup_added                 = ggml_div(ctx0, ggml_mul(ctx0, altup_added, target_magnitude), new_magnitude);\n        inpL                        = ggml_concat(ctx0, inpL, altup_added, 2);  // shape: [n_embd, n_tokens, n_altup]\n        cb(inpL, \"inp_stacked\", -1);\n    }\n    // inpL now has shape:          [n_embd,       n_tokens, n_altup]\n    // inp_per_layer now has shape: [n_embd_altup, n_tokens, n_layer]\n\n    for (int il = 0; il < n_layer; ++il) {\n        // this block is made to be closely resemble Gemma3p5DecoderLayer on python code\n        const float freq_base_l  = model.get_rope_freq_base(cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * cur         = inpL;                    // [n_embd, n_tokens, n_altup]\n        ggml_tensor * predictions = altup_predict(cur, il);  // [n_embd, n_tokens, n_altup]\n\n        // predicted value will go through self-attention and laurel\n        ggml_tensor * active_prediction = view_2d_slice(predictions, i_altup_act);  // [n_embd, n_tokens]\n        cur                             = active_prediction;\n        cb(cur, \"active_prediction\", il);\n\n        // norm\n        cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // laurel\n        ggml_tensor * laurel_out = laurel(cur, il);  // [n_embd, n_tokens]\n\n        // self-attention\n        if (hparams.has_kv(il)) {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            Vcur = ggml_rms_norm(ctx0, Vcur, hparams.f_norm_rms_eps);\n\n            cb(Qcur, \"Qcur_normed\", il);\n            cb(Kcur, \"Kcur_normed\", il);\n            cb(Vcur, \"Vcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur_pos\", il);\n            cb(Kcur, \"Kcur_pos\", il);\n\n            cur = build_attn(inp_attn, model.layers[il].wo,\n                    NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr,\n                    hparams.f_attention_scale, il);\n        } else {\n            // reuse KV cache of earlier layers\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n            cb(Qcur, \"Qcur_pos\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);\n        }\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, active_prediction);  // [n_embd, n_tokens]\n        cb(cur, \"attn_gated\", il);\n\n        ggml_tensor * attn_laurel = ggml_scale(ctx0, ggml_add(ctx0, cur, laurel_out),\n                                               1.0f / sqrtf(2.0f));  // [n_embd, n_tokens]\n        cb(attn_laurel, \"attn_laurel\", il);\n\n        cur = build_norm(attn_laurel, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        {\n            ggml_tensor * up_proj   = build_lora_mm(model.layers[il].ffn_up, cur);\n            ggml_tensor * gate_proj = build_lora_mm(model.layers[il].ffn_gate, cur);\n\n            if (il < n_layer_sparsity) {\n                // apply activation sparsity\n                gate_proj = gaussian_topk(gate_proj);\n            }\n            gate_proj = ggml_gelu(ctx0, gate_proj);\n\n            cur = ggml_mul(ctx0, up_proj, gate_proj);\n            cur = build_lora_mm(model.layers[il].ffn_down, cur);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", il);\n\n        ggml_tensor * attn_ffw_laurel_gated = ggml_add(ctx0, cur, attn_laurel);  // [n_embd, n_tokens]\n        cb(attn_ffw_laurel_gated, \"attn_ffw_laurel_gated\", il);\n\n        ggml_tensor * corrected = altup_correct(predictions, attn_ffw_laurel_gated, il);  // [n_embd, n_tokens, n_altup]\n\n        ggml_tensor * first_prediction;                                                   // [n_embd, n_tokens]\n        {\n            first_prediction = view_2d_slice(corrected, i_altup_act);                     // [n_embd, n_tokens]\n            first_prediction = ggml_mul(ctx0, first_prediction, model.layers[il].altup_correct_scale);\n            first_prediction = build_lora_mm(model.layers[il].per_layer_inp_gate, first_prediction);\n            first_prediction = ggml_gelu(ctx0, first_prediction);                 // [n_embd_altup, n_tokens]\n            cb(first_prediction, \"first_prediction_gated\", il);\n            ggml_tensor * inp_this_layer = view_2d_slice(inp_per_layer, il);      // [n_embd_altup, n_tokens]\n            first_prediction = ggml_mul(ctx0, first_prediction, inp_this_layer);  // [n_embd_altup, n_tokens]\n            cb(first_prediction, \"first_prediction_scaled\", il);\n\n            first_prediction = build_lora_mm(model.layers[il].per_layer_proj, first_prediction);  // [n_embd, n_tokens]\n            first_prediction =\n                build_norm(first_prediction, model.layers[il].per_layer_post_norm, NULL, LLM_NORM_RMS, il);\n            cb(first_prediction, \"first_prediction_out\", il);\n        }\n        // equivalent to python code: corrected_predictions[1:] += first_prediction\n        {\n            ggml_tensor * slice_first = view_2d_slice(corrected, 0);\n            ggml_tensor * slice_rest  = ggml_view_3d(\n                ctx0, corrected, n_embd, n_tokens, n_altup - 1, ggml_row_size(corrected->type, n_embd),\n                ggml_row_size(corrected->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(corrected));\n            ggml_tensor * tmp = ggml_add(ctx0, slice_rest, first_prediction);  // [n_embd, n_tokens, n_altup - 1]\n            corrected         = ggml_concat(ctx0, slice_first, tmp, 2);        // [n_embd, n_tokens, n_altup]\n        }\n        cur = corrected;                                                       // [n_embd, n_tokens, n_altup]\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;  // [n_embd, n_tokens, n_altup]\n\n    // cur now has multiple altup(s), we want to merge them back to 1 altup\n    {\n        ggml_tensor * target_magnitude = calc_magnitude(view_2d_slice(cur, i_altup_act));  // [n_embd, n_tokens]\n        // do a view to skip the first slice (active altup)\n        ggml_tensor * alt_slice =\n            ggml_view_3d(ctx0, cur, n_embd, n_tokens, n_altup - 1, ggml_row_size(cur->type, n_embd),\n                         ggml_row_size(cur->type, n_embd * n_tokens), n_embd * n_tokens * ggml_element_size(cur));\n        ggml_tensor * altup_unembd =\n            ggml_mul_mat(ctx0, model.altup_unembd_proj, alt_slice);  // shape: [n_embd, n_tokens, n_altup - 1]\n        ggml_tensor * new_magnitude = calc_magnitude(altup_unembd);\n        altup_unembd                = ggml_div(ctx0, ggml_mul(ctx0, altup_unembd, target_magnitude), new_magnitude);\n        cb(altup_unembd, \"altup_unembd\", -1);\n\n        // equivalent to torch.mean(hidden_states, dim=0)\n        cur = view_2d_slice(cur, 0);  // [n_embd, n_tokens]\n        for (int i = 0; i < n_altup - 1; ++i) {\n            cur = ggml_add(ctx0, cur, view_2d_slice(altup_unembd, i));\n        }\n        cur = ggml_scale(ctx0, cur, 1.0f / float(n_altup));  // [n_embd, n_tokens]\n        cb(cur, \"unembd_merged\", -1);\n    }\n    // cur now has shape: [n_embd, n_tokens]\n\n    // TODO: move this to right after the last KV layer\n    {\n        // skip computing output for unused tokens\n        ggml_tensor * inp_out_ids = build_inp_out_ids();\n        cur                       = ggml_get_rows(ctx0, cur, inp_out_ids);\n    }\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    {\n        // final logit soft-capping\n        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);\n        cur = ggml_tanh(ctx0, cur);\n        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nggml_tensor * llm_build_gemma3n_iswa::calc_magnitude(ggml_tensor * x) {\n    return ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, x)));\n}\n\n// get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim\nggml_tensor * llm_build_gemma3n_iswa::view_2d_slice(ggml_tensor * x, int idx) {\n    GGML_ASSERT(idx < (int) x->ne[2]);\n    return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]),\n                        idx * x->ne[0] * x->ne[1] * ggml_element_size(x));\n}\n\n// equivalent to get_per_layer_inputs() in python code\n// output shape: [n_embd_altup, n_layer, n_tokens]\nggml_tensor * llm_build_gemma3n_iswa::get_per_layer_inputs() {\n    auto inp = std::make_unique<llm_graph_input_embd>(n_embd);\n    ggml_tensor * inp_per_layer;\n    if (ubatch.token) {\n        inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);\n        ggml_set_input(inp->tokens);\n        res->t_inp_tokens = inp->tokens;\n        inp_per_layer = ggml_get_rows(ctx0, model.tok_embd_per_layer, inp->tokens);\n        inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_altup, n_layer, n_tokens);\n        inp_per_layer = ggml_scale(ctx0, inp_per_layer, sqrtf((float) n_embd_altup));\n        cb(inp_per_layer, \"inp_per_layer_selected\", -1);\n        res->add_input(std::move(inp));\n    } else {\n        // Vision embedding path: use padding token (ID=0) embedding\n        // TODO: verify if this is the correct behavior in transformers implementation\n        const int64_t embd_size = model.tok_embd_per_layer->ne[0];  // n_embd_altup * n_layer\n\n        // Extract and dequantize padding token embedding (row 0)\n        ggml_tensor * padding = ggml_view_1d(ctx0, model.tok_embd_per_layer, embd_size, 0);\n        inp_per_layer = ggml_cast(ctx0, padding, GGML_TYPE_F32);\n\n        // Reshape to [n_embd_altup, n_layer, 1]\n        inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_altup, n_layer, 1);\n        cb(inp_per_layer, \"inp_per_layer_vision\", -1);\n    }\n    return inp_per_layer;\n}\n\n// equivalent to project_per_layer_inputs() in python code\n// this calculates the per-layer inputs, so the final tensor shape will have n_layer as the last dim\n// output shape: [n_embd_altup, n_tokens, n_layer]\nggml_tensor * llm_build_gemma3n_iswa::project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer) {\n    const float per_layer_projection_scale = 1.0f / sqrtf((float) n_embd);\n    const float per_layer_input_scale      = 1.0f / sqrtf(2.0f);\n\n    ggml_tensor * per_layer_proj = ggml_mul_mat(ctx0, model.per_layer_model_proj, inputs_embeds);\n    per_layer_proj               = ggml_scale(ctx0, per_layer_proj, per_layer_projection_scale);\n    per_layer_proj               = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_altup, n_layer, n_tokens);\n    per_layer_proj               = build_norm(per_layer_proj, model.per_layer_proj_norm, NULL, LLM_NORM_RMS,\n                                              -1);  // [n_embd_altup, n_layer, n_tokens]\n    cb(per_layer_proj, \"per_layer_proj\", -1);\n\n    inp_per_layer = ggml_add(ctx0, per_layer_proj, inp_per_layer);\n    inp_per_layer = ggml_scale(ctx0, inp_per_layer, per_layer_input_scale);\n    cb(inp_per_layer, \"inp_per_layer\", -1);\n\n    // permute to shape: [n_embd_altup, n_tokens, n_layer]\n    inp_per_layer = ggml_cont(ctx0, ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3));\n    return inp_per_layer;\n}\n\n// input cur shape: [n_altup, n_tokens]\n// output    shape: [n_altup, n_tokens]\nggml_tensor * llm_build_gemma3n_iswa::laurel(ggml_tensor * cur, int il) {\n    ggml_tensor * tmp = cur;\n    tmp               = build_lora_mm(model.layers[il].laurel_l, tmp);\n    tmp               = build_lora_mm(model.layers[il].laurel_r, tmp);\n    tmp               = build_norm(tmp, model.layers[il].laurel_post_norm, NULL, LLM_NORM_RMS, il);\n    tmp               = ggml_add(ctx0, tmp, cur);\n    cb(tmp, \"laurel_out\", il);\n    return tmp;\n}\n\n// input x shape: [n_embd, n_tokens]\n// output  shape: [n_embd, n_tokens]\nggml_tensor * llm_build_gemma3n_iswa::gaussian_topk(ggml_tensor * x) {\n    ggml_tensor * mean = ggml_mean(ctx0, x);\n    ggml_tensor * std  = ggml_sqrt(ctx0, ggml_scale(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, ggml_sub(ctx0, x, mean))),\n                                                    1.0f / (float) (x->ne[0] - 1)));\n    ggml_tensor * cutoff_x = ggml_add(ctx0, mean, ggml_scale(ctx0, std, f_sparsity_std_mul));\n    return ggml_relu(ctx0, ggml_sub(ctx0, x, cutoff_x));\n}\n\n//\n// altup functions\n//\n\n// equivalent to compute_router_modalities() in python code\n// input x shape: [n_embd,  n_tokens]\n// output  shape: [n_altup, n_tokens]\nggml_tensor * llm_build_gemma3n_iswa::altup_compute_router_modalities(ggml_tensor * x, int il) {\n    ggml_tensor * router_inputs = build_norm(x, model.layers[il].altup_router_norm, NULL, LLM_NORM_RMS, il);\n\n    // router_input_scale\n    router_inputs = ggml_scale(ctx0, router_inputs, 1.0f / (float) n_embd);\n\n    ggml_tensor * output = ggml_mul_mat(ctx0, model.layers[il].altup_router, router_inputs);\n    return ggml_tanh(ctx0, output);  // [n_altup, n_tokens]\n}\n\n// input cur shape: [n_embd, n_tokens, n_altup]\n// output    shape: [n_embd, n_tokens, n_altup]\nggml_tensor * llm_build_gemma3n_iswa::altup_predict(ggml_tensor * cur, int il) {\n    ggml_tensor * activated  = view_2d_slice(cur, i_altup_act);                 // [n_embd, n_tokens]\n    ggml_tensor * modalities = altup_compute_router_modalities(activated, il);  // [n_altup, n_tokens]\n    cb(modalities, \"modalities\", il);\n\n    ggml_tensor * all_coefs = build_lora_mm(model.layers[il].altup_predict_coef, modalities);\n    cb(all_coefs, \"all_coefs\", il);\n    // first dim now having n_altup^2 elements, we reshape it to 2D (so we end up with 3D tensor)\n    all_coefs = ggml_reshape_3d(ctx0, all_coefs, n_altup, n_altup, n_tokens);\n\n    // permute to [n_altup, n_embd, n_tokens]\n    ggml_tensor * cur_permuted = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));\n    ggml_tensor * predictions  = ggml_mul_mat(ctx0, cur_permuted, all_coefs);  // [n_altup, n_embd, n_tokens]\n\n    // final shape must be the same as cur: [n_embd, n_tokens, n_altup]\n    predictions = ggml_cont(ctx0, ggml_permute(ctx0, predictions, 0, 2, 1, 3));\n    predictions = ggml_add(ctx0, predictions, cur);\n    cb(predictions, \"predictions\", il);\n\n    return predictions;\n}\n\n// input predictions       shape: [n_embd, n_tokens, n_altup]\n// input activated         shape: [n_embd, n_tokens]\n// output                  shape: [n_embd, n_tokens, n_altup]\nggml_tensor * llm_build_gemma3n_iswa::altup_correct(ggml_tensor * predictions, ggml_tensor * activated, int il) {\n    ggml_tensor * modalities = altup_compute_router_modalities(activated, il);  // [n_altup, n_tokens]\n    cb(modalities, \"modalities\", il);\n\n    ggml_tensor * active_prediction = view_2d_slice(predictions, i_altup_act);\n    ggml_tensor * innovation        = ggml_sub(ctx0, activated, active_prediction);  // [n_embd, n_tokens]\n    cb(innovation, \"innovation\", il);\n\n    ggml_tensor * all_coefs = build_lora_mm(model.layers[il].altup_correct_coef, modalities);  // [n_altup, n_tokens]\n    all_coefs               = ggml_scale_bias(ctx0, all_coefs, 1.0f, 1.0f);                    // + 1.0\n    cb(all_coefs, \"all_coefs\", il);\n    all_coefs = ggml_transpose(ctx0, all_coefs);                                               // [n_tokens, n_altup]\n    all_coefs = ggml_cont_3d(ctx0, all_coefs, 1, n_tokens, n_altup);                           // [1, n_tokens, n_altup]\n\n    innovation              = ggml_repeat_4d(ctx0, innovation, n_embd, n_tokens, n_altup, 1);\n    ggml_tensor * corrected = ggml_mul(ctx0, innovation, all_coefs);   // [n_embd, n_tokens, n_altup]\n    corrected               = ggml_add(ctx0, corrected, predictions);  // [n_embd, n_tokens, n_altup]\n    cb(corrected, \"corrected\", il);\n\n    return corrected;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/glm4-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_glm4_moe::llm_build_glm4_moe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    bool use_mrope = hparams.use_mrope();\n    if (ubatch.embd && !use_mrope) {\n        // unfortunately, we need to forcefully stop here, to avoid users complaining about wrong results\n        GGML_ABORT(\"This GGUF does not support multimodal. Please reconvert it.\");\n    }\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    // Only process up to last layer (skip final NextN layer)\n    // Final layer tensors are loaded but not processed in forward pass\n    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;\n    for (int il = 0; il < n_transformer_layers; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // Pre-attention norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            }\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            }\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            }\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            // Apply Q/K norm if available (GLM-4.5 355B variant)\n            if (model.layers[il].attn_q_norm) {\n                Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n                cb(Qcur, \"Qcur_normed\", il);\n            }\n            if (model.layers[il].attn_k_norm) {\n                Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n                cb(Kcur, \"Kcur_normed\", il);\n            }\n\n            if (use_mrope) {\n                Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,\n                            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,\n                            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow);\n            } else {\n                // Normal RoPE\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot,\n                                    rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot,\n                                    rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_transformer_layers - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // Post-attention norm\n        cur = build_norm(ffn_inp, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"post_attn_norm\", il);\n\n        // Check if this is a dense layer (n_layer_dense_lead=1, so layer 0 is dense)\n        if (static_cast<uint32_t>(il) < hparams.n_layer_dense_lead) {\n            // Dense FFN layer\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // Process routed experts using existing MoE infrastructure\n            ggml_tensor * routed_out = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    model.layers[il].ffn_exp_probs_b,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, hparams.expert_weights_norm,\n                    hparams.expert_weights_scale,\n                    (llama_expert_gating_func_type) hparams.expert_gating_func,\n                    il);\n            cb(routed_out, \"ffn_moe_out\", il);\n\n            // Process shared expert on original input\n            ggml_tensor * shared_out = build_ffn(cur,\n                    model.layers[il].ffn_up_shexp,   NULL, NULL,\n                    model.layers[il].ffn_gate_shexp, NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(shared_out, \"ffn_shexp_out\", il);\n\n            // Final output: routed_output + shared_output\n            cur = ggml_add(ctx0, routed_out, shared_out);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/glm4.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    bool use_mrope = hparams.use_mrope();\n    if (ubatch.embd && !use_mrope) {\n        // unfortunately, we need to forcefully stop here, to avoid users complaining about wrong results\n        GGML_ABORT(\"This GGUF does not support multimodal. Please reconvert it.\");\n    }\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    // Only process up to last layer (skip final NextN layer)\n    // Final layer tensors are loaded but not processed in forward pass\n    const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;\n    for (int il = 0; il < n_transformer_layers; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // Pre-attention norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = nullptr;\n            ggml_tensor * Kcur = nullptr;\n            ggml_tensor * Vcur = nullptr;\n\n            if (model.layers[il].wqkv == nullptr) {\n                Qcur = build_lora_mm(model.layers[il].wq, cur);\n                if (model.layers[il].bq) {\n                    Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                }\n                Kcur = build_lora_mm(model.layers[il].wk, cur);\n                if (model.layers[il].bk) {\n                    Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                }\n                Vcur = build_lora_mm(model.layers[il].wv, cur);\n                if (model.layers[il].bv) {\n                    Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                }\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n            } else {\n                cur = build_lora_mm(model.layers[il].wqkv, cur);\n                cb(cur, \"wqkv\", il);\n                if (model.layers[il].bqkv) {\n                    cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n                    cb(cur, \"bqkv\", il);\n                }\n                Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float), cur->nb[1],\n                                    0 * sizeof(float) * (n_embd));\n                Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                    cur->nb[1], 1 * sizeof(float) * (n_embd));\n                Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                    cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));\n            }\n\n            if (use_mrope) {\n                Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,\n                            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,\n                            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                            ext_factor, attn_factor, beta_fast, beta_slow);\n            } else {\n                // Normal RoPE\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot,\n                                    rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot,\n                                    rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_transformer_layers - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        // Post-attention norm (new!)\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"post_attn_norm\", il);\n\n        // Add the input (residual connection after post-attention norm)\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            // Pre-MLP norm\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            // MLP\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    NULL, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n\n            // Post-MLP norm\n            cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"post_mlp_norm\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    // Final norm\n    cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // Output projection\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gpt2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_gpt2::llm_build_gpt2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * pos;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);\n    cb(pos, \"pos_embd\", -1);\n\n    inpL = ggml_add(ctx0, inpL, pos);\n    cb(inpL, \"inpL\", -1);\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/gptneox.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_gptneox::llm_build_gptneox(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // ffn\n        if (hparams.use_par_res) {\n            // attention and ffn are computed in parallel\n            // x = x + attn(ln1(x)) + ffn(ln2(x))\n\n            ggml_tensor * attn_out = cur;\n\n            cur = build_norm(inpL,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n\n            cur = ggml_add(ctx0, cur, inpL);\n            cb(cur, \"ffn_out\", il);\n\n            cur = ggml_add(ctx0, cur, attn_out);\n\n            cur = build_cvec(cur, il);\n            cb(cur, \"l_out\", il);\n\n            // input for next layer\n            inpL = cur;\n        } else {\n            // attention and ffn are computed sequentially\n            // x = x + attn(ln1(x))\n            // x = x + ffn(ln2(x))\n\n            ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n            cb(ffn_inp, \"ffn_inp\", il);\n\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n\n            cur = ggml_add(ctx0, cur, ffn_inp);\n\n            cur = build_cvec(cur, il);\n            cb(cur, \"l_out\", il);\n\n            // input for next layer\n            inpL = cur;\n        }\n    }\n\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/granite-hybrid.cpp",
    "content": "#include \"models.h\"\n\nllm_build_granite_hybrid::llm_build_granite_hybrid(const llama_model & model, const llm_graph_params & params) :\n    llm_build_mamba_base(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    // Positional embeddings populated if rope enabled\n    ggml_tensor * inp_pos = nullptr;\n    if (hparams.rope_finetuned) {\n        inp_pos = build_inp_pos();\n    }\n\n    for (int il = 0; il < n_layer; ++il) {\n        struct ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        if (hparams.is_recurrent(il)) {\n            // ssm layer //\n            cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);\n        } else {\n            // attention layer //\n            cur = build_attention_layer(cur, inp_pos, inp->get_attn(), model, n_embd_head, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // ffn\n        cur = build_layer_ffn(cur, inpSA, model, il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    // For Granite architectures - scale logits\n    if (hparams.f_logit_scale) {\n        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nggml_tensor * llm_build_granite_hybrid::build_attention_layer(ggml_tensor *             cur,\n                                                              ggml_tensor *             inp_pos,\n                                                              llm_graph_input_attn_kv * inp_attn,\n                                                              const llama_model &       model,\n                                                              const int64_t             n_embd_head,\n                                                              const int                 il) {\n    // compute Q and K and (optionally) RoPE them\n    ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n    cb(Qcur, \"Qcur\", il);\n    if (model.layers[il].bq) {\n        Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n        cb(Qcur, \"Qcur\", il);\n    }\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n    cb(Kcur, \"Kcur\", il);\n    if (model.layers[il].bk) {\n        Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n        cb(Kcur, \"Kcur\", il);\n    }\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n    cb(Vcur, \"Vcur\", il);\n    if (model.layers[il].bv) {\n        Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n        cb(Vcur, \"Vcur\", il);\n    }\n\n    Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il), n_tokens);\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n\n    const bool use_rope = hparams.rope_finetuned;\n    if (use_rope) {\n        ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n\n        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n    }\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n    cur = build_attn(inp_attn,\n            model.layers[il].wo, model.layers[il].bo,\n            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n    cb(cur, \"attn_out\", il);\n    return cur;\n}\n\nggml_tensor * llm_build_granite_hybrid::build_layer_ffn(ggml_tensor *       cur,\n                                                        ggml_tensor *       inpSA,\n                                                        const llama_model & model,\n                                                        const int           il) {\n    // For Granite architectures - scale residual\n    if (hparams.f_residual_scale) {\n        cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);\n    }\n    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n    cb(ffn_inp, \"ffn_inp\", il);\n\n    // feed-forward network (non-MoE)\n    if (model.layers[il].ffn_gate_inp == nullptr) {\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n    } else {\n        // MoE branch\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        // For Granite MoE Shared\n        if (hparams.n_ff_shexp > 0) {\n            ggml_tensor * ffn_shexp =\n                build_ffn(cur,\n                    model.layers[il].ffn_up_shexp, NULL, NULL,\n                    model.layers[il].ffn_gate_shexp, NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(ffn_shexp, \"ffn_shexp\", il);\n\n            cur = ggml_add(ctx0, moe_out, ffn_shexp);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            cur = moe_out;\n        }\n    }\n\n    // For Granite architectures - scale residual\n    if (hparams.f_residual_scale) {\n        cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);\n    }\n    cur = ggml_add(ctx0, cur, ffn_inp);\n    cb(cur, \"ffn_out\", il);\n\n    cur = build_cvec(cur, il);\n    cb(cur, \"l_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/granite.cpp",
    "content": "#include \"models.h\"\n\nllm_build_granite::llm_build_granite(\n    const llama_model & model,\n    const llm_graph_params & params)\n    : llm_graph_context(params) {\n\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - built only if rope enabled\n    ggml_tensor * inp_pos = nullptr;\n    if (hparams.rope_finetuned) {\n        inp_pos = build_inp_pos();\n    }\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        cur = build_attention_layer(\n            cur, inp_pos, inp_attn,\n            model, n_embd_head, il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        // ffn\n        cur = build_layer_ffn(cur, inpSA, model, il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    // For Granite architectures - scale logits\n    cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_logit_scale);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nggml_tensor * llm_build_granite::build_attention_layer(\n          ggml_tensor             * cur,\n          ggml_tensor             * inp_pos,\n          llm_graph_input_attn_kv * inp_attn,\n    const llama_model             & model,\n    const int64_t                 n_embd_head,\n    const int                     il) {\n\n    // compute Q and K and (optionally) RoPE them\n    ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n    cb(Qcur, \"Qcur\", il);\n    if (model.layers[il].bq) {\n        Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n        cb(Qcur, \"Qcur\", il);\n    }\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n    cb(Kcur, \"Kcur\", il);\n    if (model.layers[il].bk) {\n        Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n        cb(Kcur, \"Kcur\", il);\n    }\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n    cb(Vcur, \"Vcur\", il);\n    if (model.layers[il].bv) {\n        Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n        cb(Vcur, \"Vcur\", il);\n    }\n\n    Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il),    n_tokens);\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n\n    const bool use_rope = hparams.rope_finetuned;\n    if (use_rope) {\n        ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n        Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, rope_factors,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n        Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, rope_factors,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n    }\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n    cur = build_attn(inp_attn,\n            model.layers[il].wo, model.layers[il].bo,\n            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n    return cur;\n}\n\nggml_tensor * llm_build_granite::build_layer_ffn(\n          ggml_tensor       * cur,\n          ggml_tensor       * inpSA,\n    const llama_model       & model,\n    const int                 il) {\n\n    // For Granite architectures - scale residual\n    if (hparams.f_residual_scale) {\n        cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);\n    }\n    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n    cb(ffn_inp, \"ffn_inp\", il);\n\n    // feed-forward network (non-MoE)\n    if (model.layers[il].ffn_gate_inp == nullptr) {\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n                cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(cur, \"ffn_out\", il);\n\n    } else {\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n                cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        // For Granite MoE Shared\n        if (hparams.n_ff_shexp > 0) {\n            ggml_tensor * ffn_shexp = build_ffn(cur,\n                model.layers[il].ffn_up_shexp,   NULL, NULL,\n                model.layers[il].ffn_gate_shexp, NULL, NULL,\n                model.layers[il].ffn_down_shexp, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(ffn_shexp, \"ffn_shexp\", il);\n\n            cur = ggml_add(ctx0, moe_out, ffn_shexp);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            cur = moe_out;\n        }\n    }\n\n    // For Granite architectures - scale residual\n    if (hparams.f_residual_scale) {\n        cur = ggml_scale(ctx0, cur, hparams.f_residual_scale);\n    }\n    cur = ggml_add(ctx0, cur, ffn_inp);\n    cb(cur, \"ffn_out\", il);\n\n    cur = build_cvec(cur, il);\n    cb(cur, \"l_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/grok.cpp",
    "content": "#include \"models.h\"\n\nllm_build_grok::llm_build_grok(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        cur = build_norm(cur,\n                model.layers[il].attn_out_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_out_norm\", il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // MoE branch\n        ggml_tensor * moe_out = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_GELU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        if (model.layers[il].ffn_up) {\n            ggml_tensor * ffn_out = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_PAR, il);\n            cb(ffn_out, \"ffn_out\", il);\n\n            cur = ggml_scale(ctx0, ggml_add(ctx0, ffn_out, moe_out), std::sqrt(2) / 2);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            cur = moe_out;\n        }\n        cur = build_norm(cur,\n                model.layers[il].ffn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);\n\n    // final logit soft-capping\n    if (hparams.f_final_logit_softcapping) {\n        cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping);\n        cur = ggml_tanh(ctx0, cur);\n        cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/grovemoe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_grovemoe::llm_build_grovemoe(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t n_embd_head    = hparams.n_embd_head_v();\n    const int64_t n_chunk_expert = n_expert / hparams.n_group_experts;\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                 ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * probs = build_lora_mm(model.layers[il].ffn_gate_inp, cur);  // [n_expert, n_tokens]\n        cb(probs, \"ffn_moe_logits\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                nullptr,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il,\n                probs);\n        cb(moe_out, \"ffn_moe_out\", il);\n        cur = moe_out;\n\n        // TODO: Only do the expert selection and weights once\n        moe_out = build_moe_ffn(cur,\n                    nullptr,\n                    model.layers[il].ffn_up_chexps,\n                    model.layers[il].ffn_gate_chexps,\n                    model.layers[il].ffn_down_chexps,\n                    nullptr,\n                    n_chunk_expert, n_expert_used > n_chunk_expert ? n_chunk_expert : n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il,\n                    probs);\n        cb(moe_out, \"ffn_adj_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ggml_scale(ctx0, moe_out, hparams.expert_group_scale));\n        cb(cur, \"ffn_final_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/hunyuan-dense.cpp",
    "content": "#include \"models.h\"\n\nllm_build_hunyuan_dense::llm_build_hunyuan_dense(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                        ctx0, Qcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Kcur = ggml_rope_ext(\n                        ctx0, Kcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n\n            Kcur = build_norm(Kcur,\n                        model.layers[il].attn_k_norm, nullptr,\n                        LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_norm\", il);\n\n            Qcur = build_norm(Qcur,\n                        model.layers[il].attn_q_norm, nullptr,\n                        LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_norm\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n        // feed-forward network (non-MoE)\n        ggml_tensor * cur_mlp = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur_mlp, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur_mlp, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/hunyuan-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_hunyuan_moe::llm_build_hunyuan_moe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = 1.0f / sqrtf(float(n_embd_head));\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur,\n                    model.layers[il].attn_k_norm, nullptr,\n                    LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_norm\", il);\n\n            Qcur = build_norm(Qcur,\n                    model.layers[il].attn_q_norm, nullptr,\n                    LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_norm\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n            model.layers[il].ffn_norm, NULL,\n            LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network (non-MoE)\n        ggml_tensor * cur_mlp = build_ffn(cur,\n                model.layers[il].ffn_up_shexp,   NULL, NULL,\n                model.layers[il].ffn_gate_shexp, NULL, NULL,\n                model.layers[il].ffn_down_shexp, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur_mlp, \"ffn_mlp\", il);\n\n        // MoE branch\n        ggml_tensor * cur_moe = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU,\n                true, // norm_topk_prob\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(cur_moe, \"ffn_moe_out\", il);\n\n        ggml_tensor * ffn_out = ggml_add(ctx0, cur_moe, cur_mlp);\n        cb(ffn_out, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, ffn_out, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/internlm2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_internlm2::llm_build_internlm2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/jais.cpp",
    "content": "#include \"models.h\"\n\nllm_build_jais::llm_build_jais(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*cur->nb[0]*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*cur->nb[0]*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*cur->nb[0]*(n_embd + n_embd_gqa));\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/float(n_embd_head), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        // add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        inpL = ggml_add(ctx0, cur, ffn_inp);\n        cb(inpL, \"l_out\", il);\n    }\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/jais2.cpp",
    "content": "#include \"models.h\"\n\n// JAIS-2 model graph builder\n// Uses: LayerNorm (not RMSNorm), relu2 activation, separate Q/K/V, RoPE embeddings\nllm_build_jais2::llm_build_jais2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // KV input for attention\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // Pre-attention LayerNorm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // Self-attention with separate Q, K, V projections\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            cb(Qcur, \"Qcur_bias\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            cb(Kcur, \"Kcur_bias\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            cb(Vcur, \"Vcur_bias\", il);\n\n            // Reshape for attention\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            // Apply RoPE\n            Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n            Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n            cb(Qcur, \"Qcur_rope\", il);\n            cb(Kcur, \"Kcur_rope\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // Residual connection\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // Pre-FFN LayerNorm\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm,\n                model.layers[il].ffn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // FFN with relu2 activation (ReLU squared) - no gate projection\n        // up -> relu2 -> down\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                NULL, NULL, NULL,  // no gate\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        // Residual connection\n        inpL = ggml_add(ctx0, cur, ffn_inp);\n        inpL = build_cvec(inpL, il);\n        cb(inpL, \"l_out\", il);\n    }\n\n    // Final LayerNorm\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n    cb(cur, \"result_norm\", -1);\n\n    res->t_embd = cur;\n\n    // Output projection\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/jamba.cpp",
    "content": "#include \"models.h\"\n\nllm_build_jamba::llm_build_jamba(const llama_model & model, const llm_graph_params & params) : llm_build_mamba_base(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    // {n_embd, n_tokens}\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp_hybrid = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const int64_t n_head_kv = hparams.n_head_kv(il);\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        if (n_head_kv == 0) {\n            cur = build_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il);\n        } else {\n            // Attention\n\n            struct ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            struct ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            struct ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            // No RoPE :)\n            cur = build_attn(inp_hybrid->get_attn(),\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        // residual\n        struct ggml_tensor * ffn_inp = ggml_add(ctx0, inpL, cur);\n        cb(cur, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            // FFN\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, false,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n            cb(cur, \"ffn_moe_out\", il);\n        }\n        // residual\n        cur = ggml_add(ctx0, ffn_inp, cur);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    // final rmsnorm\n    cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/kimi-linear.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\n// Causal Conv1d function for Q,K,V\n// When qkv is 0, it is Q, 1 is K, 2 is V\nstatic ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) {\n    const int64_t d_inner = head_dim * n_head;\n    const int64_t conv_state_size = (d_conv - 1) * d_inner;\n    const int64_t n_embd_r_total = 3 * conv_state_size;  // Q + K + V\n\n    // conv_state_all is [n_embd_r_total, n_seqs], split into Q, K, V\n    // Each conv state is [(d_conv-1) * d_inner] per sequence, need to reshape to [d_conv-1, d_inner, n_seqs]\n    // Memory layout: for each seq, Q state is first conv_state_size elements, then K, then V\n    // conv_state_all has stride: nb[0] = element_size, nb[1] = n_embd_r_total * element_size\n    // View Q conv state: offset 0, size conv_state_size per seq\n    // conv_state_all is [n_embd_r_total, n_seqs] with memory layout:\n    //   state[i + seq * n_embd_r_total] where i = conv_step + channel * (d_conv-1) + {0, conv_state_size, 2*conv_state_size} for Q/K/V\n    // We want [d_conv-1, d_inner, n_seqs] view:\n    //   nb1 = (d_conv-1) * element_size (stride between channels)\n    //   nb2 = n_embd_r_total * element_size (stride between seqs)\n    ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,\n        (d_conv - 1) * ggml_element_size(conv_state_all),  // nb1: stride between channels\n        n_embd_r_total * ggml_element_size(conv_state_all),  // nb2: stride between seqs\n        qkv * conv_state_size * ggml_element_size(conv_state_all));\n\n// Causal Conv1d function for Q,K,V\n// When qkv is 0, it is Q, 1 is K, 2 is V\n    // Step 1: Q, K, V projections -> [d_inner, n_tokens]\n    ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x);\n\n    // Reshape input: {d_inner, n_tokens} -> {d_inner, n_seq_tokens, n_seqs}\n    ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);\n\n    // Concat Q conv state and current input: {d_conv-1 + n_seq_tokens, d_inner, n_seqs}\n    ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);\n\n    // Save last (d_conv-1) columns back to Q conv state\n    ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,\n        conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);\n    ggml_build_forward_expand(gf,\n        ggml_cpy(ctx0, last_conv_x,\n            ggml_view_3d(ctx0, conv_states_all,\n                d_conv - 1, d_inner, n_seqs,\n                (d_conv - 1) * ggml_element_size(conv_states_all),           // nb1: contiguous within one channel's conv taps\n                n_embd_r_total * ggml_element_size(conv_states_all),         // nb2: stride between sequences (skip over K,V states)\n                (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all))));  // offset to first seq's Q/K/V state\n    // Reshape conv weight: GGUF [d_conv, 1, d_inner, 1] -> ggml_ssm_conv expects [d_conv, d_inner]\n    // GGUF stores as [d_conv, 1, d_inner, 1] with memory layout w[conv_step + channel * d_conv]\n    // vLLM stores as [d_inner, d_conv] with memory layout w[channel * d_conv + conv_step]\n    // ggml_ssm_conv computes: c[conv_step + channel * d_conv]\n    // GGUF layout: [d_conv, 1, d_inner] or [d_conv, 1, d_inner, 1] -> reshape to [d_conv, d_inner]\n    // Reshape conv weight from [d_conv, 1, d_inner, 1] to [d_conv, d_inner] for ggml_ssm_conv\n    ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);\n\n    // Apply conv1d\n    // ggml_ssm_conv output: {d_inner, n_seq_tokens, n_seqs}\n    ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);\n    // Reshape to 2D for bias add: {d_inner, n_tokens}\n    Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);\n    Xcur = ggml_silu(ctx0, Xcur);\n\n    return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);\n}\n\nllm_build_kimi_linear::llm_build_kimi_linear(const llama_model & model, const llm_graph_params & params) :\n    llm_build_delta_net_base(params), model(model) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"model.embed_tokens\", -1);\n\n    // Note: Kimi MLA does NOT use RoPE (rotary_emb=None in vLLM)\n    // So we don't need inp_pos\n\n    auto * inp_kv = !hparams.is_mla() ? build_inp_mem_hybrid() : nullptr;\n    auto * inp_k = hparams.is_mla() ? build_inp_mem_hybrid_k() : nullptr;\n    auto * inp_rs = hparams.is_mla() ? inp_k->get_recr() : inp_kv->get_recr();\n    auto * inp_attn_kv = !hparams.is_mla() ? inp_kv->get_attn() : nullptr;\n    auto * inp_attn_k = hparams.is_mla() ? inp_k->get_attn() : nullptr;\n\n    // Output ids for selecting which tokens to output\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    // Kimi dimension constants\n    const int64_t n_head = hparams.n_head();\n    const int64_t head_dim = hparams.n_embd_head_kda;\n    const int64_t d_conv = hparams.ssm_d_conv;\n    const int64_t d_inner = n_head * head_dim;  // 32 * 128 = 4096\n    const int64_t n_seqs = ubatch.n_seqs;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    // Verify batch consistency for recurrent layers\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n\n    // MLA params\n    const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();\n    const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();\n    const int64_t kv_lora_rank = hparams.n_lora_kv;\n    // qk_rope_head_dim = 64 (from Kimi config) which is hparams.n_rot\n    // Confirmed from tensor shape: wkv_a_mqa [2304, 576] = [n_embd, kv_lora_rank + qk_rope_head_dim]\n    const int64_t n_embd_head_qk_rope = hparams.n_rot();  // config.qk_rope_head_dim\n    const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;  // 192 - 64 = 128\n    // Attention scale for MLA\n    const float kq_scale_mla = 1.0f / sqrtf((float)n_embd_head_k_mla);\n\n    for (int il = 0; il < n_layer; ++il) {\n        const auto & layer = model.layers[il];\n        ggml_tensor * inpSA = inpL;\n\n        // Attention Norm\n        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_build_forward_expand(gf, cur);\n\n        if (hparams.is_recurrent(il)) {\n            // === KDA Layer (Kimi Delta Attention) with Recurrent State ===\n            // Reference: vLLM kda.py\n            const auto * mctx_cur = inp_rs->mctx;\n            const auto kv_head = mctx_cur->get_head();\n\n            // Get conv states from r_l tensor (Q, K, V each have separate state)\n            ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n            cb(conv_states_all, \"conv_states_all\", il);\n            ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);\n            ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);\n            ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);\n            ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);\n\n            // g1 = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias)\n            ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);\n            ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a);\n            cb(g1, \"g1 f_b(f_a(cur))\", il);\n            g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);\n            g1 = ggml_softplus(ctx0, g1);\n            g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens);\n\n            // A_log shape is [1, n_head] or [1, n_head, 1, 1], need to broadcast to [head_dim, n_head, n_tokens]. No need to -exp(a_log) because it was done in convert_hf_to_gguf.py\n            // Reshape to [1, n_head, 1] for broadcasting with g1 [head_dim, n_head, n_tokens]\n            ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1);\n            g1 = ggml_mul(ctx0, g1, A);\n            cb(g1, \"kda_g1\", il);\n\n            g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs);\n\n            // Compute beta (mixing coefficient)\n            ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);\n            beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs);\n            cb(beta, \"kda_beta\", il);\n\n            beta = ggml_sigmoid(ctx0, beta);\n\n            // Reshape for KDA recurrence\n            // {n_embd, n_tokens} -> {n_embd, n_seq_tokens, n_seqs}\n            cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);\n\n            // Get SSM state and compute KDA recurrence using ggml_kda_scan\n            ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);\n            ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);\n            state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs);\n\n            const float eps_norm = hparams.f_norm_rms_eps;\n\n            Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm);\n            Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm);\n\n            // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens\n            auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);\n\n            ggml_tensor * output = ggml_cont(ctx0, attn_out.first);\n            ggml_tensor * new_state = attn_out.second;\n            cb(output, \"attn_output\", il);\n            cb(new_state, \"new_state\", il);\n\n            // Update the recurrent states\n            ggml_build_forward_expand(gf,\n                                     ggml_cpy(ctx0, new_state,\n                                              ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,\n                                                           kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));\n\n            // Output gating g2 = g_b(g_a(x))\n            ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);\n            ggml_tensor * g_a = ggml_mul_mat(ctx0, layer.ssm_g_a, cur_2d);\n            ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g_b, g_a);\n            cb(g2, \"g2 g_b(g_a(cur_2d))\", il);\n            g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs);\n\n            // Apply o_norm with sigmoid gating\n            // Note: Kimi model uses sigmoid gating, not SiLU (despite FusedRMSNormGated default being swish)\n            // Formula: output = RMSNorm(x) * sigmoid(g)\n            ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head,  n_seq_tokens * n_seqs);\n            ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);\n            cb(normed, \"kda_normed\", il);\n            ggml_tensor * gate = ggml_sigmoid(ctx0, g2);\n            ggml_tensor * gated = ggml_mul(ctx0, normed, gate);\n\n            // Output projection\n            gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);\n            cur = ggml_mul_mat(ctx0, layer.wo, gated);\n            cb(cur, \"kda_out\", il);\n\n        } else {\n            // === MLA Layer (Multi-head Latent Attention) without KV Cache ===\n            // Reference: vLLM mla.py\n            // Step 1: Q projection and reshape\n            // vLLM Kimi: q = q_proj(hidden_states), then view as [n_tokens, n_head, qk_head_dim]\n            // Note: Kimi MLA does NOT use RoPE (rotary_emb=None in vLLM)\n            ggml_tensor * Qcur = ggml_mul_mat(ctx0, layer.wq, cur);\n\n            // Step 2: KV compression\n            // kv_cmpr_pe = kv_a_proj_with_mqa(hidden_states) -> [kv_lora_rank + qk_rope_head_dim, n_tokens]\n            ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);\n\n            // Split: kv_cmpr = kv_lora[:kv_lora_rank], k_pe = kv_lora[kv_lora_rank:]\n            ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,\n                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);\n            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,\n                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),\n                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),\n                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));\n            // Note: Kimi MLA does NOT apply RoPE (rotary_emb=None in vLLM)\n            // k_pe is used directly without RoPE\n            // Normalize kv_c\n            kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);\n\n            if (layer.wk_b && layer.wv_b) { // MLA KV cache enabled\n                // extract q_nope\n                ggml_tensor * q_nope =\n                    ggml_view_3d(ctx0, Qcur, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(Qcur->type, n_embd_head_k_mla),\n                                 ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, 0);\n                cb(q_nope, \"q_nope\", il);\n\n                // and {n_embd_head_qk_rope, n_head, n_tokens}\n                ggml_tensor * q_pe = ggml_view_3d(\n                    ctx0, Qcur, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(Qcur->type, n_embd_head_k_mla),\n                    ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, ggml_row_size(Qcur->type, n_embd_head_qk_nope));\n                cb(q_pe, \"q_pe\", il);\n\n                // {n_embd_head_qk_nope, n_tokens, n_head}\n                q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);\n                cb(q_nope, \"q_nope_perm\", il);\n\n                // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head}\n                ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);\n                cb(q_nope_absorbed, \"q_nope_absorbed\", il);\n\n                // {kv_lora_rank, n_head, n_tokens}\n                q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);\n                cb(q_nope_absorbed, \"q_nope_absorbed_perm\", il);\n\n                // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens}\n                // note: rope must go first for in-place context shifting in build_rope_shift()\n                Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);\n                cb(Qcur, \"Qcur\", il);\n\n                kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);\n                cb(kv_cmpr, \"kv_cmpr_reshape\", il);\n\n                // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}\n                ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);\n                cb(Kcur, \"Kcur\", il);\n\n                // {kv_lora_rank, 1, n_tokens}\n                ggml_tensor * Vcur = kv_cmpr;\n                cb(Vcur, \"Vcur\", il);\n\n                cur = build_attn(inp_attn_k, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);\n                cb(cur, \"mla_out\", il);\n            } else { // MLA KV cache disabled. Fall back to MHA KV cache.\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k_mla, n_head, n_tokens);\n                cb(Qcur, \"mla_Q\", il);\n                // KV decompression: kv = kv_b_proj(kv_c_normed)\n                ggml_tensor * kv = ggml_mul_mat(ctx0, layer.wkv_b, kv_cmpr);\n                const int64_t kv_per_head = n_embd_head_qk_nope + n_embd_head_v_mla;\n\n                // Split kv into k_nope and v\n                ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,\n                    ggml_row_size(kv->type, kv_per_head),\n                    ggml_row_size(kv->type, kv_per_head * n_head), 0);\n                ggml_tensor * Vcur = ggml_view_3d(ctx0, kv, n_embd_head_v_mla, n_head, n_tokens,\n                    ggml_row_size(kv->type, kv_per_head),\n                    ggml_row_size(kv->type, kv_per_head * n_head),\n                    ggml_row_size(kv->type, n_embd_head_qk_nope));\n                Vcur = ggml_cont(ctx0, Vcur);\n                cb(Vcur, \"mla_V\", il);\n\n                // Concatenate k_nope + k_pe (broadcast k_pe to all heads)\n                // K = [k_nope, k_pe] where k_nope is [qk_nope_head_dim, n_head, n_tokens]\n                // and k_pe is [qk_rope_head_dim, 1, n_tokens] broadcast to all heads\n                // Need to broadcast k_pe from [qk_rope, 1, n_tokens] to [qk_rope, n_head, n_tokens]\n                ggml_tensor * k_pe_target = ggml_new_tensor_3d(ctx0, k_pe->type, n_embd_head_qk_rope, n_head, n_tokens);\n                ggml_tensor * k_pe_repeated = ggml_repeat(ctx0, k_pe, k_pe_target);\n                ggml_tensor * Kcur = ggml_concat(ctx0, k_pe_repeated, k_nope, 0);\n                cb(Kcur, \"mla_K\", il);\n\n                // Direct softmax attention (with MHA KV cache)\n                // Use build_attn with inp_attn for proper mask handling\n                cur = build_attn(inp_attn_kv, layer.wo, NULL, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);\n                cb(cur, \"mla_out\", il);\n            }\n        }\n\n        // On last layer, select only the output tokens\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur,   inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // Residual\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FFN Norm\n        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        if ((uint32_t) il < hparams.n_layer_dense_lead) {\n            // Dense FFN layer\n            cur = build_ffn(cur,\n                layer.ffn_up, NULL, NULL,\n                layer.ffn_gate, NULL, NULL,\n                layer.ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE layer\n            // Kimi uses moe_renormalize=True and routed_scaling_factor (stored as expert_weights_scale) = 2.446\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                layer.ffn_gate_inp,\n                layer.ffn_up_exps,\n                layer.ffn_gate_exps,\n                layer.ffn_down_exps,\n                layer.ffn_exp_probs_b,\n                hparams.n_expert,\n                hparams.n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // Shared expert\n            {\n                ggml_tensor * ffn_shexp = build_ffn(cur,\n                        layer.ffn_up_shexp, NULL, NULL,\n                        layer.ffn_gate_shexp, NULL, NULL,\n                        layer.ffn_down_shexp, NULL, NULL,\n                        NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n                cb(ffn_shexp, \"ffn_shexp\", il);\n\n                cur = ggml_add(ctx0, moe_out, ffn_shexp);\n                cb(cur, \"ffn_out\", il);\n            }\n        }\n        // Residual\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // Final Norm\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // Output\n    cur = ggml_mul_mat(ctx0, model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/lfm2.cpp",
    "content": "#include \"models.h\"\n\n#include \"../llama-memory-hybrid-iswa.h\"\n#include \"../llama-memory-hybrid.h\"\n\ntemplate <bool iswa>\nllm_build_lfm2<iswa>::llm_build_lfm2(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    using inp_hybrid_type = std::conditional_t<iswa, llm_graph_input_mem_hybrid_iswa,  llm_graph_input_mem_hybrid>;\n    using inp_attn_type   = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa,     llm_graph_input_attn_kv>;\n    using mem_hybrid_ctx  = std::conditional_t<iswa, llama_memory_hybrid_iswa_context, llama_memory_hybrid_context>;\n\n    // lambda helpers for readability\n    auto build_dense_feed_forward = [&model, this](ggml_tensor * cur, int il) -> ggml_tensor * {\n        GGML_ASSERT(!model.layers[il].ffn_up_b);\n        GGML_ASSERT(!model.layers[il].ffn_gate_b);\n        GGML_ASSERT(!model.layers[il].ffn_down_b);\n        return build_ffn(cur,\n            model.layers[il].ffn_up, NULL, NULL,\n            model.layers[il].ffn_gate, NULL, NULL,\n            model.layers[il].ffn_down, NULL, NULL,\n            NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n    };\n    auto build_moe_feed_forward = [&model, this](ggml_tensor * cur, int il) -> ggml_tensor * {\n        return build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),\n                il);\n    };\n    auto build_attn_block = [&model, this](ggml_tensor *   cur,\n                                           ggml_tensor *   inp_pos,\n                                           inp_attn_type * inp_attn,\n                                           int             il) -> ggml_tensor * {\n        GGML_ASSERT(hparams.n_embd_v_gqa(il) == hparams.n_embd_k_gqa(il));\n        const auto n_embd_head = hparams.n_embd_head_v();\n        const auto n_head_kv   = hparams.n_head_kv(il);\n\n        auto * q = build_lora_mm(model.layers[il].wq, cur);\n        cb(q, \"model.layers.{}.self_attn.q_proj\", il);\n        auto * k = build_lora_mm(model.layers[il].wk, cur);\n        cb(k, \"model.layers.{}.self_attn.k_proj\", il);\n        auto * v = build_lora_mm(model.layers[il].wv, cur);\n        cb(v, \"model.layers.{}.self_attn.v_proj\", il);\n\n        q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);\n        k = ggml_reshape_3d(ctx0, k, n_embd_head, n_head_kv, n_tokens);\n        v = ggml_reshape_3d(ctx0, v, n_embd_head, n_head_kv, n_tokens);\n\n        // qk norm\n        q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n        cb(q, \"model.layers.{}.self_attn.q_layernorm\", il);\n        k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n        cb(k, \"model.layers.{}.self_attn.k_layernorm\", il);\n\n        // RoPE\n        q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,\n                          attn_factor, beta_fast, beta_slow);\n        k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,\n                          attn_factor, beta_fast, beta_slow);\n\n        cur = build_attn(inp_attn,\n                model.layers[il].wo, NULL,\n                q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n\n        cb(cur, \"model.layers.{}.self_attn.out_proj\", il);\n\n        return cur;\n    };\n    auto build_shortconv_block = [&model, this](ggml_tensor *        cur,\n                                                llm_graph_input_rs * inp_recr,\n                                                int                  il) -> ggml_tensor * {\n        const auto * mctx_cur = static_cast<const mem_hybrid_ctx *>(mctx)->get_recr();\n        const uint32_t kv_head      = mctx_cur->get_head();\n        const int64_t  n_seq_tokens = ubatch.n_seq_tokens;\n        const int64_t  n_seqs       = ubatch.n_seqs;\n        GGML_ASSERT(n_seqs != 0);\n        GGML_ASSERT(ubatch.equal_seqs());\n        GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n\n        GGML_ASSERT(hparams.n_shortconv_l_cache > 1);\n        const uint32_t d_conv = hparams.n_shortconv_l_cache - 1;\n\n        // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}\n        cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);\n\n        auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur);\n        cb(bcx, \"model.layers.{}.conv.in_proj\", il);\n\n        constexpr auto n_chunks = 3;\n        GGML_ASSERT(bcx->ne[0] % n_chunks == 0);\n        const auto chunk_size = bcx->ne[0] / n_chunks;\n        auto *     b          = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],\n                                             0 * chunk_size * ggml_element_size(bcx));\n        auto *     c          = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],\n                                             1 * chunk_size * ggml_element_size(bcx));\n        auto *     x          = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],\n                                             2 * chunk_size * ggml_element_size(bcx));\n\n        auto * bx = ggml_transpose(ctx0, ggml_mul(ctx0, b, x));\n\n        // read conv state\n        auto * conv_state = mctx_cur->get_r_l(il);\n        auto * conv_rs    = build_rs(inp_recr, conv_state, hparams.n_embd_r(), n_seqs);\n        auto * conv       = ggml_reshape_3d(ctx0, conv_rs, d_conv, hparams.n_embd, n_seqs);\n\n        bx = ggml_concat(ctx0, conv, bx, 0);\n        GGML_ASSERT(bx->ne[0] > conv->ne[0]);\n\n        // last d_conv columns is a new conv state\n        auto * new_conv = ggml_view_3d(ctx0, bx, conv->ne[0], bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],\n                                       (bx->ne[0] - conv->ne[0]) * ggml_element_size(bx));\n        GGML_ASSERT(ggml_are_same_shape(conv, new_conv));\n\n        // write new conv conv state\n        ggml_build_forward_expand(gf, ggml_cpy(ctx0, new_conv,\n                                               ggml_view_1d(ctx0, conv_state, ggml_nelements(new_conv),\n                                                            kv_head * d_conv * n_embd * ggml_element_size(new_conv))));\n\n        auto * conv_kernel = model.layers[il].shortconv.conv;\n        auto * conv_out    = ggml_ssm_conv(ctx0, bx, conv_kernel);\n        cb(conv_out, \"model.layers.{}.conv.conv\", il);\n\n        auto * y = ggml_mul(ctx0, c, conv_out);\n        y        = build_lora_mm(model.layers[il].shortconv.out_proj, y);\n        cb(y, \"model.layers.{}.conv.out_proj\", il);\n        // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}\n        y = ggml_reshape_2d(ctx0, y, y->ne[0], n_seq_tokens * n_seqs);\n\n        return y;\n    };\n\n    // actual graph construction starts here\n    ggml_tensor * cur = build_inp_embd(model.tok_embd);\n    cb(cur, \"model.embed_tokens\", -1);\n\n    ggml_build_forward_expand(gf, cur);\n\n    inp_hybrid_type * inp_hybrid = nullptr;\n    if constexpr (iswa) {\n        inp_hybrid = build_inp_mem_hybrid_iswa();\n    } else {\n        inp_hybrid = build_inp_mem_hybrid();\n    }\n\n    ggml_tensor * inp_pos     = build_inp_pos();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const bool is_moe_layer = il >= static_cast<int>(hparams.n_layer_dense_lead);\n\n        auto * prev_cur = cur;\n        cur             = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"model.layers.{}.operator_norm\", il);\n\n        cur = hparams.is_recurrent(il) ? build_shortconv_block(cur, inp_hybrid->get_recr(), il) :\n                                         build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);\n            prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids);\n        }\n\n        cur = ggml_add(ctx0, prev_cur, cur);\n\n        auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(ffn_norm_out, \"model.layers.{}.ffn_norm\", il);\n\n        ggml_tensor * ffn_out =\n            is_moe_layer ? build_moe_feed_forward(ffn_norm_out, il) : build_dense_feed_forward(ffn_norm_out, il);\n        cb(ffn_norm_out, \"model.layers.{}.ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_out);\n    }\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_lfm2<true>;\ntemplate struct llm_build_lfm2<false>;\n"
  },
  {
    "path": "examples/talk-llama/models/llada-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_llada_moe::llm_build_llada_moe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, false,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/llada.cpp",
    "content": "#include \"models.h\"\n\nllm_build_llada::llm_build_llada(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    // LLaDA is similar to LLaMA but uses non-causal attention for diffusion\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // Non-causal attention for diffusion\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute separate Q, K, V projections without bias, matching LLaDALlamaBlock\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/llama-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_llama_iswa::llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // temperature tuning\n    ggml_tensor * inp_attn_scale = nullptr;\n    inp_attn_scale = build_inp_attn_scale();\n\n    auto * inp_attn = build_attn_inp_kv_iswa();\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * inpSA = inpL;\n\n        // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous\n        const bool use_rope = hparams.n_no_rope_layer_step > 0 &&\n                              (il + 1) % hparams.n_no_rope_layer_step != 0;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (use_rope) {\n                Qcur = ggml_rope_ext(\n                        ctx0, Qcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n\n                Kcur = ggml_rope_ext(\n                        ctx0, Kcur, inp_pos, rope_factors,\n                        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n            } else if (inp_attn_scale) {\n                Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            if (use_rope && hparams.use_kq_norm) {\n                // Llama4TextL2Norm\n                Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);\n                Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);\n                cb(Qcur, \"Qcur_normed\", il);\n                cb(Kcur, \"Kcur_normed\", il);\n            }\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network (non-MoE)\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            ggml_tensor * ffn_inp_normed = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            ggml_tensor * moe_out = build_moe_ffn(ffn_inp_normed,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, false,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,\n                    il);\n\n            // Shared experts\n            ggml_tensor * shexp_out = build_ffn(ffn_inp_normed,\n                model.layers[il].ffn_up_shexp,   NULL, NULL,\n                model.layers[il].ffn_gate_shexp, NULL, NULL,\n                model.layers[il].ffn_down_shexp, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(shexp_out, \"ffn_moe_shexp\", il);\n\n            cur = ggml_add(ctx0, moe_out, shexp_out);\n            cb(cur, \"ffn_moe_out_merged\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/llama.cpp",
    "content": "#include \"models.h\"\n\ntemplate <bool embed>\nllm_build_llama<embed>::llm_build_llama(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type = std::conditional_t<embed, llm_graph_input_attn_no_cache, llm_graph_input_attn_kv>;\n\n    inp_attn_type * inp_attn = nullptr;\n    if constexpr (embed) {\n        inp_attn = build_attn_inp_no_cache();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            if (hparams.use_kq_norm) {\n                // Llama4TextL2Norm\n                Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);\n                Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);\n                cb(Qcur, \"Qcur_normed\", il);\n                cb(Kcur, \"Kcur_normed\", il);\n            }\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            if (model.layers[il].wo_s) {\n                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);\n            }\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network (non-MoE)\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   model.layers[il].ffn_up_s,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il,\n                    nullptr, nullptr,\n                    model.layers[il].ffn_up_exps_s,\n                    model.layers[il].ffn_gate_exps_s,\n                    model.layers[il].ffn_down_exps_s);\n            cb(cur, \"ffn_moe_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    if constexpr (!embed) {\n        // lm_head\n        cur = build_lora_mm(model.output, cur);\n\n        cb(cur, \"result_output\", -1);\n        res->t_logits = cur;\n    }\n\n    ggml_build_forward_expand(gf, cur);\n}\n\ntemplate struct llm_build_llama<false>;\ntemplate struct llm_build_llama<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/maincoder.cpp",
    "content": "#include \"models.h\"\n\nllm_build_maincoder::llm_build_maincoder(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/mamba-base.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_mamba_base::llm_build_mamba_base(const llm_graph_params & params) : llm_graph_context(params) {}\n\nggml_tensor * llm_build_mamba_base::build_mamba_layer(llm_graph_input_rs * inp,\n                                                         ggml_tensor *        cur,\n                                                         const llama_model &  model,\n                                                         const llama_ubatch & ubatch,\n                                                         int                  il) {\n    const auto * mctx_cur = inp->mctx;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    const auto & layer = model.layers[il];\n\n    const int64_t d_conv         = hparams.ssm_d_conv;\n    const int64_t d_inner        = hparams.ssm_d_inner;\n    const int64_t d_state        = hparams.ssm_d_state;\n    const int64_t dt_rank        = hparams.ssm_dt_rank;\n    const int64_t n_head         = d_inner;\n    const int64_t head_dim       = 1;\n    const int64_t n_seqs         = ubatch.n_seqs;\n    // Some variants of Mamba arch (e.g. FalconMamba do apply layer norm on B and Dt layers)\n    const bool    ssm_dt_b_c_rms = hparams.ssm_dt_b_c_rms;\n\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n    GGML_ASSERT(d_inner % n_head == 0);\n\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    conv               = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner, n_seqs);\n\n    // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}\n    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);\n\n    // {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}\n    ggml_tensor * xz = build_lora_mm(layer.ssm_in, cur);\n    // split the above in two\n    // => {d_inner, n_seq_tokens, n_seqs}\n    ggml_tensor * x  = ggml_view_3d(ctx0, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], 0);\n    ggml_tensor * z =\n        ggml_view_3d(ctx0, xz, d_inner, xz->ne[1], xz->ne[2], xz->nb[1], xz->nb[2], d_inner * ggml_element_size(xz));\n\n    // conv\n    {\n        // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs}\n        ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, x), 0);\n\n        // copy last (d_conv - 1) columns back into the state cache\n        ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2],\n                                               n_seq_tokens * (conv_x->nb[0]));\n\n        ggml_build_forward_expand(\n            gf, ggml_cpy(ctx0, last_conv,\n                         ggml_view_1d(ctx0, conv_states_all, (d_conv - 1) * (d_inner) * (n_seqs),\n                                      kv_head * (d_conv - 1) * (d_inner) *ggml_element_size(conv_states_all))));\n\n        // 1D convolution\n        // The equivalent is to make a self-overlapping view of conv_x\n        // over d_conv columns at each stride in the 3rd dimension,\n        // then element-wise multiply that with the conv1d weight,\n        // then sum the elements of each row,\n        // (the last two steps are a dot product over rows (also doable with mul_mat))\n        // then permute away the ne[0] dimension,\n        // and then you're left with the resulting x tensor.\n        // For simultaneous sequences, all sequences need to have the same length.\n        x = ggml_ssm_conv(ctx0, conv_x, layer.ssm_conv1d);\n\n        // bias\n        x = ggml_add(ctx0, x, layer.ssm_conv1d_b);\n\n        x = ggml_silu(ctx0, x);\n    }\n\n    // ssm\n    {\n        // {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs}\n        ggml_tensor * x_db = build_lora_mm(layer.ssm_x, x);\n        // split\n        ggml_tensor * dt   = ggml_view_3d(ctx0, x_db, dt_rank, n_seq_tokens, n_seqs, x_db->nb[1], x_db->nb[2], 0);\n        ggml_tensor * B =\n            ggml_view_4d(ctx0, x_db, d_state, /* n_group */ 1, n_seq_tokens, n_seqs, d_state * x_db->nb[0], x_db->nb[1],\n                         x_db->nb[2], ggml_element_size(x_db) * dt_rank);\n        ggml_tensor * C =\n            ggml_view_4d(ctx0, x_db, d_state, /* n_group */ 1, n_seq_tokens, n_seqs, d_state * x_db->nb[0], x_db->nb[1],\n                         x_db->nb[2], ggml_element_size(x_db) * (dt_rank + d_state));\n\n        // Some Mamba variants (e.g. FalconMamba, Jamba) apply RMS norm in B, C & Dt layers\n        if (ssm_dt_b_c_rms || (layer.ssm_dt_norm && layer.ssm_b_norm && layer.ssm_c_norm)) {\n            dt = build_norm(dt, layer.ssm_dt_norm, NULL, LLM_NORM_RMS, il);\n            B  = build_norm(B, layer.ssm_b_norm, NULL, LLM_NORM_RMS, il);\n            C  = build_norm(C, layer.ssm_c_norm, NULL, LLM_NORM_RMS, il);\n        }\n\n        // {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs}\n        dt = build_lora_mm(layer.ssm_dt, dt);\n        dt = ggml_add(ctx0, dt, layer.ssm_dt_b);\n\n        cur = x;\n        x   = ggml_reshape_4d(ctx0, x, head_dim, n_head, n_seq_tokens, n_seqs);\n\n        ggml_tensor * A = layer.ssm_a;\n\n        // use the states and the indices provided by build_recurrent_state\n        // (this is necessary in order to properly use the states before they are overwritten,\n        //  while avoiding to make unnecessary copies of the states)\n        auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {\n            ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_head, mctx_cur->get_size());\n\n            // Custom operator to optimize the parallel associative scan\n            // as described in the Annex D of the Mamba paper.\n            // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}\n            return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);\n        };\n\n        ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);\n\n        // store last states\n        ggml_build_forward_expand(\n            gf, ggml_cpy(ctx0, ggml_view_1d(ctx0, y_ssm, d_state * d_inner * n_seqs, x->nb[3] * x->ne[3]),\n                         ggml_view_1d(ctx0, ssm_states_all, d_state * d_inner * n_seqs,\n                                      kv_head * d_state * d_inner * ggml_element_size(ssm_states_all))));\n\n        ggml_tensor * y = ggml_view_3d(ctx0, y_ssm, d_inner, n_seq_tokens, n_seqs, x->nb[2], x->nb[3], 0);\n\n        // TODO: skip computing output earlier for unused tokens\n\n        y = ggml_add(ctx0, y, ggml_mul(ctx0, cur, layer.ssm_d));\n        y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);\n\n        // {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}\n        cur = build_lora_mm(layer.ssm_out, y);\n    }\n\n    // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}\n    cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);\n\n    return cur;\n}\n\nggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,\n                                                          ggml_tensor *        cur,\n                                                          const llama_model &  model,\n                                                          const llama_ubatch & ubatch,\n                                                          int                  il) const {\n    const auto * mctx_cur = inp->mctx;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    const int64_t d_conv   = hparams.ssm_d_conv;\n    const int64_t d_inner  = hparams.ssm_d_inner;\n    const int64_t d_state  = hparams.ssm_d_state;\n    const int64_t n_head   = hparams.ssm_dt_rank;\n    const int64_t head_dim = d_inner / n_head;\n    const int64_t n_group  = hparams.ssm_n_group;\n    const int64_t n_seqs   = ubatch.n_seqs;\n\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n    GGML_ASSERT(d_inner % n_head  == 0);\n    GGML_ASSERT(d_inner % d_state == 0);\n    GGML_ASSERT(d_inner % n_group == 0);\n\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    conv               = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs);\n\n    // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}\n    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);\n\n    // d_in_proj = 2 * self.d_inner + 2 * self.ngroups * self.d_state + self.nheads\n\n    // {n_embd, d_in_proj} @ {n_embd, n_seq_tokens, n_seqs} => {d_in_proj, n_seq_tokens, n_seqs}\n    ggml_tensor * zxBCdt = build_lora_mm(model.layers[il].ssm_in, cur);\n\n    // split the above in three\n    ggml_tensor * z   = ggml_view_4d(ctx0, zxBCdt, head_dim, n_head, n_seq_tokens, n_seqs, head_dim * zxBCdt->nb[0],\n                                     zxBCdt->nb[1], zxBCdt->nb[2], 0);\n    ggml_tensor * xBC = ggml_view_3d(ctx0, zxBCdt, d_inner + 2 * n_group * d_state, n_seq_tokens, n_seqs, zxBCdt->nb[1],\n                                     zxBCdt->nb[2], d_inner * ggml_element_size(zxBCdt));\n    ggml_tensor * dt  = ggml_view_3d(ctx0, zxBCdt, n_head, n_seq_tokens, n_seqs, zxBCdt->nb[1], zxBCdt->nb[2],\n                                     (2 * d_inner + 2 * n_group * d_state) * ggml_element_size(zxBCdt));\n\n    // conv\n    {\n        // => {d_conv - 1 + n_seq_tokens, d_inner + 2*n_group*d_state, n_seqs}\n        ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, xBC), 0);\n\n        // copy last (d_conv - 1) columns back into the state cache\n        ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs,\n                                               conv_x->nb[1], conv_x->nb[2], n_seq_tokens * (conv_x->nb[0]));\n\n        ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,\n                                               ggml_view_1d(ctx0, conv_states_all,\n                                                            (d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs),\n                                                            kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) *\n                                                                ggml_element_size(conv_states_all))));\n\n        // 1D convolution\n        // The equivalent is to make a self-overlapping view of conv_x\n        // over d_conv columns at each stride in the 3rd dimension,\n        // then element-wise multiply that with the conv1d weight,\n        // then sum the elements of each row,\n        // (the last two steps are a dot product over rows (also doable with mul_mat))\n        // then permute away the ne[0] dimension,\n        // and then you're left with the resulting x tensor.\n        // For simultaneous sequences, all sequences need to have the same length.\n        xBC = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d);\n\n        // bias\n        xBC = ggml_add(ctx0, xBC, model.layers[il].ssm_conv1d_b);\n\n        xBC = ggml_silu(ctx0, xBC);\n    }\n\n    // ssm\n    {\n        // These correspond to V K Q in SSM/attention duality\n        ggml_tensor * x = ggml_view_4d(ctx0, xBC, head_dim, n_head, n_seq_tokens, n_seqs, head_dim * xBC->nb[0],\n                                       xBC->nb[1], xBC->nb[2], 0);\n        ggml_tensor * B = ggml_view_4d(ctx0, xBC, d_state, n_group, n_seq_tokens, n_seqs, d_state * xBC->nb[0],\n                                       xBC->nb[1], xBC->nb[2], d_inner * ggml_element_size(xBC));\n        ggml_tensor * C = ggml_view_4d(ctx0, xBC, d_state, n_group, n_seq_tokens, n_seqs, d_state * xBC->nb[0],\n                                       xBC->nb[1], xBC->nb[2], (d_inner + n_group * d_state) * ggml_element_size(xBC));\n\n        // {n_head, n_seq_tokens, n_seqs}\n        dt = ggml_add(ctx0, ggml_cont(ctx0, dt), model.layers[il].ssm_dt_b);\n\n        ggml_tensor * A = model.layers[il].ssm_a;\n\n        // use the states and the indices provided by build_recurrent_state\n        // (this is necessary in order to properly use the states before they are overwritten,\n        //  while avoiding to make unnecessary copies of the states)\n        auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {\n            ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_head, mctx_cur->get_size());\n\n            // TODO: use semistructured matrices to implement state-space duality\n            // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}\n            return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);\n        };\n\n        ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);\n\n        // store last states\n        ggml_build_forward_expand(\n            gf, ggml_cpy(ctx0, ggml_view_1d(ctx0, y_ssm, d_state * d_inner * n_seqs, ggml_nelements(x) * x->nb[0]),\n                         ggml_view_1d(ctx0, ssm_states_all, d_state * d_inner * n_seqs,\n                                      kv_head * d_state * d_inner * ggml_element_size(ssm_states_all))));\n\n        ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_head, n_seq_tokens, n_seqs, x->nb[1], n_head * x->nb[1],\n                                       n_seq_tokens * n_head * x->nb[1], 0);\n\n        // TODO: skip computing output earlier for unused tokens\n\n        y = ggml_add(ctx0, y, ggml_mul(ctx0, x, model.layers[il].ssm_d));\n        cb(y, \"mamba2_y_add_d\", il);\n        y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);\n\n        // grouped RMS norm\n        if (model.layers[il].ssm_norm) {\n            y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs);\n            y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);\n        }\n\n        y = ggml_reshape_3d(ctx0, y, d_inner, n_seq_tokens, n_seqs);\n\n        // {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}\n        cur = build_lora_mm(model.layers[il].ssm_out, y);\n    }\n\n    // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}\n    cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);\n    cb(cur, \"mamba_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/mamba.cpp",
    "content": "#include \"models.h\"\n\nllm_build_mamba::llm_build_mamba(const llama_model & model, const llm_graph_params & params) : llm_build_mamba_base(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    // {n_embd, n_tokens}\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * rs_inp = build_rs_inp();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        if (model.arch == LLM_ARCH_MAMBA2) {\n            cur = build_mamba2_layer(rs_inp, cur, model, ubatch, il);\n        } else {\n            cur = build_mamba_layer(rs_inp, cur, model, ubatch, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // residual\n        cur = ggml_add(ctx0, cur, inpL);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    // final rmsnorm\n    cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n"
  },
  {
    "path": "examples/talk-llama/models/mimo2-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_mimo2_iswa::llm_build_mimo2_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * inp_pos = build_inp_pos();\n    auto * inp_attn = build_attn_inp_kv_iswa();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        uint32_t n_head_l    = hparams.n_head(il);\n        uint32_t n_head_kv_l = hparams.n_head_kv(il);\n        const float freq_base_l  = model.get_rope_freq_base(cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        cur = inpL;\n\n        // self_attention\n        {\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head_l,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n            Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            ggml_tensor * sinks = model.layers[il].attn_sinks;\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, sinks, nullptr, 1.0f/sqrtf(float(n_embd_head_k)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            // dense branch\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    model.layers[il].ffn_exp_probs_b,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,\n                    il);\n            cb(cur, \"ffn_moe_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/minicpm3.cpp",
    "content": "#include \"models.h\"\n\nllm_build_minicpm3::llm_build_minicpm3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    //TODO: if the model varies, these parameters need to be read from the model\n    const int64_t n_embd_base = 256;\n    const float scale_embd  = 12.0f;\n    const float scale_depth = 1.4f;\n    const float kq_scale = 1.0f / sqrtf(float(hparams.n_embd_head_k()));\n\n    const uint32_t n_embd_head_qk_rope = hparams.n_rot();\n    const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k() - hparams.n_rot();\n\n    const uint32_t kv_lora_rank = hparams.n_lora_kv;\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // scale the input embeddings\n    inpL = ggml_scale(ctx0, inpL, scale_embd);\n    cb(inpL, \"inp_scaled\", -1);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            ggml_tensor * q = NULL;\n            // {n_embd, q_lora_rank} * {n_embd, n_tokens} -> {q_lora_rank, n_tokens}\n            q = ggml_mul_mat(ctx0, model.layers[il].wq_a, cur);\n            cb(q, \"q\", il);\n\n            q = build_norm(q,\n                    model.layers[il].attn_q_a_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(q, \"q\", il);\n\n            // {q_lora_rank, n_head * hparams.n_embd_head_k()} * {q_lora_rank, n_tokens} -> {n_head * hparams.n_embd_head_k(), n_tokens}\n            q = ggml_mul_mat(ctx0, model.layers[il].wq_b, q);\n            cb(q, \"q\", il);\n\n            // split into {n_head * n_embd_head_qk_nope, n_tokens}\n            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,\n                    ggml_row_size(q->type, hparams.n_embd_head_k()),\n                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),\n                    0);\n            cb(q_nope, \"q_nope\", il);\n\n            // and {n_head * n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,\n                    ggml_row_size(q->type, hparams.n_embd_head_k()),\n                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),\n                    ggml_row_size(q->type, n_embd_head_qk_nope));\n            cb(q_pe, \"q_pe\", il);\n\n            // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);\n            cb(kv_pe_compresseed, \"kv_pe_compresseed\", il);\n\n            // split into {kv_lora_rank, n_tokens}\n            ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens,\n                    kv_pe_compresseed->nb[1],\n                    0);\n            cb(kv_compressed, \"kv_compressed\", il);\n\n            // and {n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens,\n                    kv_pe_compresseed->nb[1],\n                    kv_pe_compresseed->nb[1],\n                    ggml_row_size(kv_pe_compresseed->type, kv_lora_rank));\n            cb(k_pe, \"k_pe\", il);\n\n            kv_compressed = build_norm(kv_compressed,\n                    model.layers[il].attn_kv_a_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(kv_compressed, \"kv_compressed\", il);\n\n            // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens}\n            ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed);\n            cb(kv, \"kv\", il);\n\n            // split into {n_head * n_embd_head_qk_nope, n_tokens}\n            ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,\n                    ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v()),\n                    ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v())),\n                    0);\n            cb(k_nope, \"k_nope\", il);\n\n            // and {n_head * n_embd_head_v, n_tokens}\n            ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v(), n_head, n_tokens,\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v())),\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v())*n_head),\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope)));\n            cb(v_states, \"v_states\", il);\n\n            v_states = ggml_cont(ctx0, v_states);\n            cb(v_states, \"v_states\", il);\n\n            q_pe = ggml_rope_ext(\n                    ctx0, q_pe, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n            cb(q_pe, \"q_pe\", il);\n\n            // shared RoPE key\n            k_pe = ggml_rope_ext(\n                    ctx0, k_pe, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n            cb(k_pe, \"k_pe\", il);\n\n            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);\n            cb(q_states, \"q_states\", il);\n\n            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);\n            cb(k_states, \"k_states\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        // scale_res - scale the hidden states for residual connection\n        const float scale_res = scale_depth/sqrtf(float(n_layer)); // TODO: is this correct?\n        cur = ggml_scale(ctx0, cur, scale_res);\n        cb(cur, \"hidden_scaled\", il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        // scale the hidden states for residual connection\n        cur = ggml_scale(ctx0, cur, scale_res);\n        cb(cur, \"hidden_scaled_ffn\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head scaling\n    const float scale_lmhead = float(n_embd_base)/float(n_embd);\n    cur = ggml_scale(ctx0, cur, scale_lmhead);\n    cb(cur, \"lmhead_scaling\", -1);\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/minimax-m2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_minimax_m2::llm_build_minimax_m2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    // GGML_ASSERT(n_embd_head == n_rot); this is wrong in case of minimax, head_dim = 128, n_rot = 64\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * inp_pos = build_inp_pos();\n    auto inp_attn = build_attn_inp_kv();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = inpL;\n\n        // self_attention\n        {\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n            Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                model.layers[il].ffn_exp_probs_b,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                (llama_expert_gating_func_type) hparams.expert_gating_func,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/mistral3.cpp",
    "content": "#include \"models.h\"\n\nllm_build_mistral3::llm_build_mistral3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // (optional) temperature tuning\n    ggml_tensor * inp_attn_scale = nullptr;\n    if (hparams.f_attn_temp_scale != 0.0f) {\n        inp_attn_scale = build_inp_attn_scale();\n    }\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // rope freq factors for llama3; may return nullptr for llama2 and other models\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            if (inp_attn_scale) {\n                // apply llama 4 temperature scaling\n                Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);\n                cb(Qcur, \"Qcur_attn_temp_scaled\", il);\n            }\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network (non-MoE)\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n            cb(cur, \"ffn_moe_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/models.h",
    "content": "#pragma once\n\n#include \"llama-model.h\"\n#include \"llama-graph.h\"\n\n// note: almost all graphs require at least sqrtf, so include cmath globally\n#include <cmath>\n\n//\n// base classes\n//\n\nstruct llm_build_mamba_base : public llm_graph_context {\n    llm_build_mamba_base(const llm_graph_params & params);\n\n    virtual ~llm_build_mamba_base() = default;\n\n    ggml_tensor * build_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il);\n    ggml_tensor * build_mamba2_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il) const;\n\n};\n\nstruct llm_build_delta_net_base : public llm_graph_context {\n    llm_build_delta_net_base(const llm_graph_params & params);\n\n    virtual ~llm_build_delta_net_base() = default;\n\n    // returns pair of output and new state\n    std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_chunking(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * g,\n                ggml_tensor * b,\n                ggml_tensor * s,\n                        int   il);\n\n    // returns pair of output and new state\n    std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_autoregressive(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * g,\n                ggml_tensor * b,\n                ggml_tensor * s,\n                int           il);\n\n    // use the ggml_gated_delta_net fused operator\n    std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_fused(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * g,\n                ggml_tensor * b,\n                ggml_tensor * s,\n                        int   il);\n\n    // choose one of two implementations above based on the number of tokens\n    std::pair<ggml_tensor *, ggml_tensor *> build_delta_net(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * g,\n                ggml_tensor * b,\n                ggml_tensor * s,\n                        int   il);\n};\n\nstruct llm_build_rwkv6_base : public llm_graph_context {\n    const llama_model & model;\n\n    llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params);\n\n    virtual ~llm_build_rwkv6_base() = default;\n\n    ggml_tensor * build_rwkv6_channel_mix(const llama_layer * layer,\n                                          ggml_tensor *       cur,\n                                          ggml_tensor *       x_prev,\n                                          llm_arch            arch) const;\n\n    ggml_tensor * build_rwkv6_time_mix(llm_graph_input_rs * inp,\n                                       ggml_tensor *        cur,\n                                       ggml_tensor *        x_prev,\n                                       const llama_ubatch & ubatch,\n                                       int                  il) const;\n};\n\n// Base class for RWKV7-related models\nstruct llm_build_rwkv7_base : public llm_graph_context {\n    const llama_model & model;\n\n    llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params);\n\n    virtual ~llm_build_rwkv7_base() = default;\n\n    // RWKV7-specific graph building methods\n    ggml_tensor * build_rwkv7_channel_mix(const llama_layer * layer,\n                                          ggml_tensor *       cur,\n                                          ggml_tensor *       x_prev,\n                                          llm_arch            arch) const;\n    ggml_tensor * build_rwkv7_time_mix(llm_graph_input_rs * inp,\n                                       ggml_tensor *        cur,\n                                       ggml_tensor *        x_prev,\n                                       ggml_tensor *&       first_layer_value,\n                                       const llama_ubatch & ubatch,\n                                       int                  il) const;\n};\n\n//\n// models\n//\n\nstruct llm_build_afmoe : public llm_graph_context {\n    llm_build_afmoe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_apertus : public llm_graph_context {\n    llm_build_apertus(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_arcee : public llm_graph_context {\n    llm_build_arcee(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_arctic : public llm_graph_context {\n    llm_build_arctic(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_arwkv7 : public llm_build_rwkv7_base {\n    llm_build_arwkv7(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_baichuan : public llm_graph_context {\n    llm_build_baichuan(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_bailingmoe2 : public llm_graph_context {\n    llm_build_bailingmoe2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_bailingmoe : public llm_graph_context {\n    llm_build_bailingmoe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_bert : public llm_graph_context {\n    llm_build_bert(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_bitnet : public llm_graph_context {\n    llm_build_bitnet(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_bloom : public llm_graph_context {\n    llm_build_bloom(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_chameleon : public llm_graph_context {\n    llm_build_chameleon(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_chatglm : public llm_graph_context {\n    llm_build_chatglm(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_codeshell : public llm_graph_context {\n    llm_build_codeshell(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_cogvlm : public llm_graph_context {\n    llm_build_cogvlm(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_cohere2_iswa : public llm_graph_context {\n    llm_build_cohere2_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_command_r : public llm_graph_context {\n    llm_build_command_r(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_dbrx : public llm_graph_context {\n    llm_build_dbrx(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_deci : public llm_graph_context {\n    llm_build_deci(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_deepseek2 : public llm_graph_context {\n    llm_build_deepseek2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_deepseek : public llm_graph_context {\n    llm_build_deepseek(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_dots1 : public llm_graph_context {\n    llm_build_dots1(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_dream : public llm_graph_context {\n    llm_build_dream(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_ernie4_5 : public llm_graph_context {\n    llm_build_ernie4_5(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_ernie4_5_moe : public llm_graph_context {\n    llm_build_ernie4_5_moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_paddleocr : public llm_graph_context {\n    llm_build_paddleocr(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool iswa>\nstruct llm_build_exaone4 : public llm_graph_context {\n    llm_build_exaone4(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_exaone : public llm_graph_context {\n    llm_build_exaone(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_exaone_moe : public llm_graph_context {\n    llm_build_exaone_moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_falcon : public llm_graph_context {\n    llm_build_falcon(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_falcon_h1 : public llm_build_mamba_base {\n    llm_build_falcon_h1(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_gemma2_iswa : public llm_graph_context {\n    llm_build_gemma2_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool iswa>\nstruct llm_build_gemma3 : public llm_graph_context {\n    llm_build_gemma3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_gemma3n_iswa : public llm_graph_context {\n    const llama_model & model;\n\n    const int64_t n_embd_head;\n    const int64_t n_embd_altup;\n    const int64_t n_altup;\n    const int     i_altup_act;\n    const int     n_layer_sparsity = 10; // number of layers using activation sparsity\n    const float   f_sparsity_std_mul = 1.6448533535003662f; // std_multiplier = normal_dist.icdf(0.95)\n\n    llm_build_gemma3n_iswa(const llama_model & model, const llm_graph_params & params);\n    ggml_tensor * calc_magnitude(ggml_tensor * x);\n    ggml_tensor * view_2d_slice(ggml_tensor * x, int idx);\n    ggml_tensor * get_per_layer_inputs();\n    ggml_tensor * project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer);\n    ggml_tensor * gaussian_topk(ggml_tensor * x);\n    ggml_tensor * altup_compute_router_modalities(ggml_tensor * x, int il);\n    ggml_tensor * altup_predict(ggml_tensor * cur, int il);\n    ggml_tensor * laurel(ggml_tensor * cur, int il);\n    ggml_tensor * altup_correct(ggml_tensor * predictions, ggml_tensor * activated, int il);\n};\n\nstruct llm_build_gemma_embedding : public llm_graph_context {\n    llm_build_gemma_embedding(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_gemma : public llm_graph_context {\n    llm_build_gemma(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_glm4 : public llm_graph_context {\n    llm_build_glm4(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_glm4_moe : public llm_graph_context {\n    llm_build_glm4_moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_gpt2 : public llm_graph_context {\n    llm_build_gpt2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_gptneox : public llm_graph_context {\n    llm_build_gptneox(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_granite : public llm_graph_context {\n    llm_build_granite(const llama_model & model, const llm_graph_params & params);\n\nprivate:\n    ggml_tensor * build_attention_layer(\n              ggml_tensor             * cur,\n              ggml_tensor             * inp_pos,\n              llm_graph_input_attn_kv * inp_attn,\n        const llama_model             & model,\n        const int64_t                 n_embd_head,\n        const int                     il);\n\n    ggml_tensor * build_layer_ffn(\n              ggml_tensor       * cur,\n              ggml_tensor       * inpSA,\n        const llama_model       & model,\n        const int                 il);\n};\n\nstruct llm_build_granite_hybrid : public llm_build_mamba_base {\n    llm_build_granite_hybrid(const llama_model & model, const llm_graph_params & params);\n    ggml_tensor * build_layer_ffn(ggml_tensor * cur, ggml_tensor * inpSA, const llama_model & model, const int il);\n    ggml_tensor * build_attention_layer(ggml_tensor * cur, ggml_tensor * inp_pos, llm_graph_input_attn_kv * inp_attn,\n        const llama_model & model,const int64_t n_embd_head, const int il);\n};\n\nstruct llm_build_grok : public llm_graph_context {\n    llm_build_grok(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_grovemoe : public llm_graph_context {\n    llm_build_grovemoe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_hunyuan_dense : public llm_graph_context {\n    llm_build_hunyuan_dense(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_hunyuan_moe : public llm_graph_context {\n    llm_build_hunyuan_moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_internlm2 : public llm_graph_context {\n    llm_build_internlm2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_jais : public llm_graph_context {\n    llm_build_jais(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_jais2 : public llm_graph_context {\n    llm_build_jais2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_jamba : public llm_build_mamba_base {\n    llm_build_jamba(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_kimi_linear : public llm_build_delta_net_base {\n    llm_build_kimi_linear(const llama_model & model, const llm_graph_params & params);\n\n    std::pair<ggml_tensor *, ggml_tensor *> build_kda_autoregressive(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * gk,\n                ggml_tensor * beta,\n                ggml_tensor * state,\n                        int   il);\n\n    std::pair<ggml_tensor *, ggml_tensor *> build_kda_chunking(\n                ggml_tensor * q,\n                ggml_tensor * k,\n                ggml_tensor * v,\n                ggml_tensor * gk,\n                ggml_tensor * beta,\n                ggml_tensor * state,\n                ggml_tensor * causal_mask,\n                ggml_tensor * identity,\n                ggml_tensor * diag_mask,\n                        int   il);\n\n    const llama_model & model;\n};\n\ntemplate <bool iswa>\nstruct llm_build_lfm2 : public llm_graph_context {\n    llm_build_lfm2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_llada : public llm_graph_context {\n    llm_build_llada(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_llada_moe : public llm_graph_context {\n    llm_build_llada_moe(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool embed>\nstruct llm_build_llama : public llm_graph_context {\n    llm_build_llama(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_llama_iswa : public llm_graph_context {\n    llm_build_llama_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_maincoder : public llm_graph_context {\n    llm_build_maincoder(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_mamba : public llm_build_mamba_base {\n    llm_build_mamba(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_mimo2_iswa : public llm_graph_context {\n    llm_build_mimo2_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_minicpm3 : public llm_graph_context {\n    llm_build_minicpm3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_minimax_m2 : public llm_graph_context {\n    llm_build_minimax_m2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_mistral3 : public llm_graph_context {\n    llm_build_mistral3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_modern_bert : public llm_graph_context {\n    llm_build_modern_bert(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_mpt : public llm_graph_context {\n    llm_build_mpt(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_nemotron : public llm_graph_context {\n    llm_build_nemotron(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_nemotron_h : public llm_build_mamba_base {\n    llm_build_nemotron_h(const llama_model & model, const llm_graph_params & params);\n    ggml_tensor * build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il);\n    ggml_tensor * build_attention_layer(ggml_tensor * cur, llm_graph_input_attn_kv * inp_attn,\n        const llama_model & model, int64_t n_embd_head, int il);\n};\n\nstruct llm_build_neo_bert : public llm_graph_context {\n    llm_build_neo_bert(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_eurobert : public llm_graph_context {\n    llm_build_eurobert(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool iswa>\nstruct llm_build_olmo2 : public llm_graph_context {\n    llm_build_olmo2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_olmoe : public llm_graph_context {\n    llm_build_olmoe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_olmo : public llm_graph_context {\n    llm_build_olmo(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_openai_moe_iswa : public llm_graph_context {\n    llm_build_openai_moe_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_openelm : public llm_graph_context {\n    llm_build_openelm(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_orion : public llm_graph_context {\n    llm_build_orion(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_pangu_embedded : public llm_graph_context {\n    llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_phi2 : public llm_graph_context {\n    llm_build_phi2(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate<bool iswa>\nstruct llm_build_phi3 : public llm_graph_context {\n    llm_build_phi3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_plamo2 : public llm_build_mamba_base {\n    llm_build_plamo2(const llama_model & model, const llm_graph_params & params);\n    private:\n        ggml_tensor * build_plamo2_mamba_layer(llm_graph_input_rs * inp, ggml_tensor * cur, const llama_model & model, const llama_ubatch & ubatch, int il);\n        ggml_tensor * build_plamo2_attn_layer(llm_graph_input_attn_kv * inp, ggml_tensor * inp_pos, ggml_tensor * cur,\n                                                const llama_model & model, int il);\n};\n\nstruct llm_build_plamo : public llm_graph_context {\n    llm_build_plamo(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool iswa>\nstruct llm_build_plamo3 : public llm_graph_context {\n    llm_build_plamo3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_plm : public llm_graph_context {\n    llm_build_plm(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen2 : public llm_graph_context {\n    llm_build_qwen2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen2moe : public llm_graph_context {\n    llm_build_qwen2moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen2vl : public llm_graph_context {\n    llm_build_qwen2vl(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen3 : public llm_graph_context {\n    llm_build_qwen3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen3moe : public llm_graph_context {\n    llm_build_qwen3moe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen3vl : public llm_graph_context {\n    llm_build_qwen3vl(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen3vlmoe : public llm_graph_context {\n    llm_build_qwen3vlmoe(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_qwen3next : public llm_build_delta_net_base {\n    llm_build_qwen3next(const llama_model & model, const llm_graph_params & params);\nprivate:\n    ggml_tensor * build_layer_attn(\n    llm_graph_input_attn_kv * inp_attn,\n                ggml_tensor * cur,\n                ggml_tensor * inp_pos,\n                        int   il);\n\n    ggml_tensor * build_layer_attn_linear(\n         llm_graph_input_rs * inp,\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_layer_ffn(\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_norm_gated(\n                ggml_tensor * input,\n                ggml_tensor * weights,\n                ggml_tensor * gate,\n                        int   layer);\n\n    // returns pair of qkv, z\n    std::pair<ggml_tensor *, ggml_tensor *> build_qkvz(\n                ggml_tensor * input,\n                        int   il);\n\n    const llama_model & model;\n};\n\nstruct llm_build_qwen35 : public llm_build_delta_net_base {\n    llm_build_qwen35(const llama_model & model, const llm_graph_params & params);\nprivate:\n    ggml_tensor * build_layer_attn(\n    llm_graph_input_attn_kv * inp_attn,\n                ggml_tensor * cur,\n                ggml_tensor * inp_pos,\n                        int * sections,\n                        int   il);\n\n    ggml_tensor * build_layer_attn_linear(\n         llm_graph_input_rs * inp,\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_layer_ffn(\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_norm_gated(\n                ggml_tensor * input,\n                ggml_tensor * weights,\n                ggml_tensor * gate,\n                        int   layer);\n\n    // returns pair of qkv, z\n    std::pair<ggml_tensor *, ggml_tensor *> build_qkvz(\n                ggml_tensor * input,\n                        int   il);\n\n    const llama_model & model;\n};\n\n// TODO: derive llm_build_delta_net_base instead\nstruct llm_build_qwen35moe : public llm_build_delta_net_base {\n    llm_build_qwen35moe(const llama_model & model, const llm_graph_params & params);\nprivate:\n    ggml_tensor * build_layer_attn(\n    llm_graph_input_attn_kv * inp_attn,\n                ggml_tensor * cur,\n                ggml_tensor * inp_pos,\n                        int * sections,\n                        int   il);\n\n    ggml_tensor * build_layer_attn_linear(\n         llm_graph_input_rs * inp,\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_layer_ffn(\n                ggml_tensor * cur,\n                        int   il);\n\n    ggml_tensor * build_norm_gated(\n                ggml_tensor * input,\n                ggml_tensor * weights,\n                ggml_tensor * gate,\n                        int   layer);\n\n    // returns pair of qkv, z\n    std::pair<ggml_tensor *, ggml_tensor *> build_qkvz(\n                ggml_tensor * input,\n                        int   il);\n\n    const llama_model & model;\n};\n\nstruct llm_build_qwen : public llm_graph_context {\n    llm_build_qwen(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_refact : public llm_graph_context {\n    llm_build_refact(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_rnd1 : public llm_graph_context {\n    llm_build_rnd1(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_rwkv6 : public llm_build_rwkv6_base {\n    llm_build_rwkv6(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_rwkv6qwen2 : public llm_build_rwkv6_base {\n    llm_build_rwkv6qwen2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_rwkv7 : public llm_build_rwkv7_base {\n    llm_build_rwkv7(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_seed_oss : public llm_graph_context {\n    llm_build_seed_oss(const llama_model & model, const llm_graph_params & params);\n};\n\ntemplate <bool iswa>\nstruct llm_build_smallthinker : public llm_graph_context {\n    llm_build_smallthinker(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_smollm3 : public llm_graph_context {\n    llm_build_smollm3(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_stablelm : public llm_graph_context {\n    llm_build_stablelm(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_starcoder2 : public llm_graph_context {\n    llm_build_starcoder2(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_starcoder : public llm_graph_context {\n    llm_build_starcoder(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_step35_iswa : public llm_graph_context {\n    llm_build_step35_iswa(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_t5_dec : public llm_graph_context {\n    llm_build_t5_dec(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_t5_enc : public llm_graph_context {\n    llm_build_t5_enc(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_wavtokenizer_dec : public llm_graph_context {\n    llm_build_wavtokenizer_dec(const llama_model & model, const llm_graph_params & params);\n};\n\nstruct llm_build_xverse : public llm_graph_context {\n    llm_build_xverse(const llama_model & model, const llm_graph_params & params);\n};\n"
  },
  {
    "path": "examples/talk-llama/models/modern-bert.cpp",
    "content": "#include \"models.h\"\n\nllm_build_modern_bert::llm_build_modern_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // construct input embeddings (token, type, position)\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"inp_embd\", -1);\n\n    // embed layer norm\n    inpL = build_norm(inpL, model.tok_norm, nullptr, LLM_NORM, -1);\n    cb(inpL, \"inp_norm\", -1);\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base(cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        cur = inpL;\n\n        // attention layer norm\n        if (model.layers[il].attn_norm) {\n            cur = build_norm(inpL,\n                    model.layers[il].attn_norm, NULL,\n                    LLM_NORM, il);\n            cb(cur, \"attn_norm\", il);\n        }\n\n        // self attention\n        cur = build_lora_mm(model.layers[il].wqkv, cur);\n        cb(cur, \"wqkv\", il);\n\n        const size_t type_size = ggml_type_size(cur->type);\n\n        ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*type_size, cur->nb[1], 0*type_size*(n_embd));\n        ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*type_size, cur->nb[1], 1*type_size*(n_embd));\n        ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*type_size, cur->nb[1], 1*type_size*(n_embd + n_embd_gqa));\n\n        // RoPE\n        Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n        Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n                );\n\n        cb(Qcur, \"Qcur\", il);\n        cb(Kcur, \"Kcur\", il);\n        cb(Vcur, \"Vcur\", il);\n\n        cur = build_attn(inp_attn,\n                    model.layers[il].wo, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        cb(cur, \"kqv_out\", il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // re-add the layer input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // attention layer norm\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                NULL,                      NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_GEGLU, LLM_FFN_SEQ, il);\n\n        // attentions bypass the intermediate layer\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM, -1);\n    cb(cur, \"final_norm_out\", -1);\n\n    res->t_embd = cur;\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/mpt.cpp",
    "content": "#include \"models.h\"\n\n\n\nllm_build_mpt::llm_build_mpt(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * pos;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    if (model.pos_embd) {\n        // inp_pos - contains the positions\n        ggml_tensor * inp_pos = build_inp_pos();\n        pos                   = ggml_get_rows(ctx0, model.pos_embd, inp_pos);\n        cb(pos, \"pos_embd\", -1);\n\n        inpL = ggml_add(ctx0, inpL, pos);\n        cb(inpL, \"inpL\", -1);\n    }\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * attn_norm;\n\n        attn_norm = build_norm(inpL, model.layers[il].attn_norm, model.layers[il].attn_norm_b, LLM_NORM, il);\n        cb(attn_norm, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = attn_norm;\n\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            if (model.layers[il].bqkv) {\n                cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n                cb(cur, \"bqkv\", il);\n            }\n\n            if (hparams.f_clamp_kqv > 0.0f) {\n                cur = ggml_clamp(ctx0, cur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);\n                cb(cur, \"wqkv_clamped\", il);\n            }\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 0 * sizeof(float) * (n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 1 * sizeof(float) * (n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),\n                                              cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));\n\n            // Q/K Layernorm\n            if (model.layers[il].attn_q_norm) {\n                Qcur = ggml_reshape_2d(ctx0, Qcur, n_embd_head * n_head, n_tokens);\n                Kcur = ggml_reshape_2d(ctx0, Kcur, n_embd_head * n_head_kv, n_tokens);\n\n                Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, model.layers[il].attn_q_norm_b, LLM_NORM, il);\n\n                Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, model.layers[il].attn_k_norm_b, LLM_NORM, il);\n\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            }\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n\n        // Add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed forward\n        {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, model.layers[il].ffn_norm_b, LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n            cur = build_ffn(cur,\n                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,\n                NULL, NULL, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                model.layers[il].ffn_act, LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/nemotron-h.cpp",
    "content": "#include \"models.h\"\n\nllm_build_nemotron_h::llm_build_nemotron_h(const llama_model & model, const llm_graph_params & params) :\n    llm_build_mamba_base(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n    ggml_build_forward_expand(gf, inpL);\n\n    auto * inp = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        struct ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        if (hparams.is_recurrent(il)) {\n            // ssm layer //\n            cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);\n        } else if (hparams.n_ff(il) == 0) {\n            // attention layer //\n            cur = build_attention_layer(cur, inp->get_attn(), model, n_embd_head, il);\n        } else {\n            cur = build_ffn_layer(cur, model, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // add residual\n        cur = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"nemotron_h_block_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nggml_tensor * llm_build_nemotron_h::build_attention_layer(ggml_tensor *             cur,\n                                                          llm_graph_input_attn_kv * inp_attn,\n                                                          const llama_model &       model,\n                                                                int64_t             n_embd_head,\n                                                                int                 il) {\n    // compute Q and K\n    ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n    cb(Qcur, \"Qcur\", il);\n    if (model.layers[il].bq) {\n        Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n        cb(Qcur, \"Qcur\", il);\n    }\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n    cb(Kcur, \"Kcur\", il);\n    if (model.layers[il].bk) {\n        Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n        cb(Kcur, \"Kcur\", il);\n    }\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n    cb(Vcur, \"Vcur\", il);\n    if (model.layers[il].bv) {\n        Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n        cb(Vcur, \"Vcur\", il);\n    }\n\n    Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, hparams.n_head(il), n_tokens);\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, hparams.n_head_kv(il), n_tokens);\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    const float kq_scale =\n        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n    cur = build_attn(inp_attn,\n            model.layers[il].wo, model.layers[il].bo,\n            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n    cb(cur, \"attn_out\", il);\n    return cur;\n}\n\nggml_tensor * llm_build_nemotron_h::build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il) {\n    if (model.layers[il].ffn_gate_inp == nullptr) {\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                NULL,                      NULL,                        NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n    } else {\n        ggml_tensor * inp_emb    = cur;\n        ggml_tensor * inp_latent = cur;\n\n        if (model.layers[il].ffn_latent_down) {\n            inp_latent = ggml_mul_mat(ctx0, model.layers[il].ffn_latent_down, cur);\n        }\n\n        ggml_tensor * router_logits = build_lora_mm(model.layers[il].ffn_gate_inp, cur);\n        cb(router_logits, \"ffn_moe_logits\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(inp_latent,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    nullptr, // no gate\n                    model.layers[il].ffn_down_exps,\n                    model.layers[il].ffn_exp_probs_b,\n                    n_expert, n_expert_used,\n                    LLM_FFN_RELU_SQR, hparams.expert_weights_norm,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,\n                    il,\n                    router_logits);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        if (model.layers[il].ffn_latent_up) {\n            moe_out = ggml_mul_mat(ctx0, model.layers[il].ffn_latent_up, moe_out);\n        }\n\n        ggml_tensor * ffn_shexp = build_ffn(inp_emb,\n                    model.layers[il].ffn_up_shexp,  NULL, NULL,\n                    NULL /* no gate */           ,  NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL,\n                    LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);\n        cb(ffn_shexp, \"ffn_shexp\", il);\n\n        cur = ggml_add(ctx0, moe_out, ffn_shexp);\n        cb(cur, \"ffn_out\", il);\n    }\n\n    cur = build_cvec(cur, il);\n    cb(cur, \"l_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/nemotron.cpp",
    "content": "#include \"models.h\"\n\nllm_build_nemotron::llm_build_nemotron(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    //GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm,\n                model.layers[il].ffn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                NULL,                      NULL,                        NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/neo-bert.cpp",
    "content": "#include \"models.h\"\n\nllm_build_neo_bert::llm_build_neo_bert(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // construct input embeddings (token, type, position)\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"inp_embd\", -1);\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * cur = inpL;\n\n        // pre-norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n\n        {\n            ggml_tensor * Qcur;\n            ggml_tensor * Kcur;\n            ggml_tensor * Vcur;\n\n            // self-attention\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            // RoPE\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n            cb(cur, \"kqv_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        // re-add the layer input\n        cur = ggml_add(ctx0, cur, inpL);\n\n        ggml_tensor * ffn_inp = cur;\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // pre-norm\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,\n                NULL, NULL, NULL, NULL, NULL,\n                model.layers[il].ffn_down,\n                NULL, NULL, NULL,\n                LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n\n        // attentions bypass the intermediate layer\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm_enc, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_embd\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/olmo.cpp",
    "content": "#include \"models.h\"\n\nllm_build_olmo::llm_build_olmo(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                NULL, NULL,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (hparams.f_clamp_kqv > 0.0f) {\n                Qcur = ggml_clamp(ctx0, Qcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (hparams.f_clamp_kqv > 0.0f) {\n                Kcur = ggml_clamp(ctx0, Kcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (hparams.f_clamp_kqv > 0.0f) {\n                Vcur = ggml_clamp(ctx0, Vcur, -hparams.f_clamp_kqv, hparams.f_clamp_kqv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                NULL, NULL,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            NULL, NULL,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/olmo2.cpp",
    "content": "#include \"models.h\"\n\ntemplate <bool iswa>\nllm_build_olmo2<iswa>::llm_build_olmo2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = inpL;\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            const bool is_swa = hparams.is_swa(il);\n\n            if (is_swa) {\n                // For sliding window layers, Olmo3 use regular rope with no yarn rope scaling.\n                // This is achieved here by setting freq_scale and attn_factor to 1.\n                // We also set ext_factor to 0 to avoid a few unnecessary computations.\n                Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, 1.0,\n                    0.0, 1.0, beta_fast, beta_slow\n                    );\n\n                Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, 1.0,\n                    0.0, 1.0, beta_fast, beta_slow\n                    );\n            } else {\n                Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n                Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        cur = build_norm(cur,\n                model.layers[il].attn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_ffn(ffn_inp,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_norm(cur,\n                model.layers[il].ffn_post_norm, NULL,\n                LLM_NORM_RMS, -1);\n        cb(cur, \"ffn_post_norm\", -1);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_olmo2<false>;\ntemplate struct llm_build_olmo2<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/olmoe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_olmoe::llm_build_olmoe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, false,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/openai-moe-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_openai_moe_iswa::llm_build_openai_moe_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv_iswa();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, nullptr,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_rot, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_rot, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, model.layers[il].attn_sinks, nullptr, 1.0f/sqrtf(float(n_rot)), il);\n\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1) {\n            // skip computing output for unused tokens\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = ffn_inp;\n        cur = build_norm(cur,\n                model.layers[il].attn_post_norm, nullptr,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        // MoE branch\n        cur = build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,  model.layers[il].ffn_gate_inp_b,\n                model.layers[il].ffn_up_exps,   model.layers[il].ffn_up_exps_b,\n                model.layers[il].ffn_gate_exps, model.layers[il].ffn_gate_exps_b,\n                model.layers[il].ffn_down_exps, model.layers[il].ffn_down_exps_b,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SWIGLU_OAI_MOE, false,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT,\n                il);\n        cb(cur, \"ffn_moe_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/openelm.cpp",
    "content": "#include \"models.h\"\n\nllm_build_openelm::llm_build_openelm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const int64_t n_head    = hparams.n_head(il);\n        const int64_t n_head_kv = hparams.n_head_kv(il);\n        const int64_t n_head_qkv = 2*n_head_kv + n_head;\n\n        cur = inpL;\n        ggml_tensor * residual = cur;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_reshape_3d(ctx0, cur, n_embd_head_k, n_head_qkv, n_tokens);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, cur->nb[1], cur->nb[2], 0);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*n_head);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, cur->nb[1], cur->nb[2], cur->nb[1]*(n_head+n_head_kv));\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = build_norm(Qcur,\n                    model.layers[il].attn_q_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur\", il);\n\n            Kcur = build_norm(Kcur,\n                    model.layers[il].attn_k_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, NULL,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, NULL,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Qcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            residual = ggml_get_rows(ctx0, residual, inp_out_ids);\n            cur      = ggml_get_rows(ctx0, cur,      inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // norm\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/orion.cpp",
    "content": "#include \"models.h\"\n\nllm_build_orion::llm_build_orion(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            // if (model.layers[il].bq) {\n            //     Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            //     cb(Qcur, \"Qcur\", il);\n            // }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            // if (model.layers[il].bk) {\n            //     Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            //     cb(Kcur, \"Kcur\", il);\n            // }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            // if (model.layers[il].bv) {\n            //     Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            //     cb(Vcur, \"Vcur\", il);\n            // }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/paddleocr.cpp",
    "content": "#include \"models.h\"\n\nllm_build_paddleocr::llm_build_paddleocr(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n\n    // NOTE: same with qwen2vl.cpp, but bias tensors are optional\n\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        {\n            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n        }\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_multi(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_multi(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1) {\n            // skip computing output for unused tokens\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up, NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/pangu-embedded.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_pangu_embedded::llm_build_pangu_embedded(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    if (model.output_b != nullptr) {\n        cur = ggml_add(ctx0, cur, model.output_b);\n    }\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/phi2.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_phi2::llm_build_phi2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * attn_norm_output;\n    ggml_tensor * ffn_output;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        attn_norm_output = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(attn_norm_output, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = nullptr;\n            ggml_tensor * Kcur = nullptr;\n            ggml_tensor * Vcur = nullptr;\n\n            if (model.layers[il].wqkv) {\n                cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output);\n                cb(cur, \"wqkv\", il);\n\n                cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n                cb(cur, \"bqkv\", il);\n\n                Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n                Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n                Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n            } else {\n                Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq);\n                Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk);\n                Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv);\n\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n            }\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            // with phi2, we scale the Q to avoid precision issues\n            // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66\n            Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur              = ggml_get_rows(ctx0,              cur, inp_out_ids);\n            inpL             = ggml_get_rows(ctx0,             inpL, inp_out_ids);\n            attn_norm_output = ggml_get_rows(ctx0, attn_norm_output, inp_out_ids);\n        }\n        // FF\n        {\n            ffn_output = build_ffn(attn_norm_output,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(ffn_output, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_output);\n        cur = ggml_add(ctx0, cur, inpL);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output_no_bias\", -1);\n\n    cur = ggml_add(ctx0, cur, model.output_b);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/phi3.cpp",
    "content": "#include \"models.h\"\n\ntemplate<bool iswa>\nllm_build_phi3<iswa>::llm_build_phi3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        auto * residual = inpL;\n\n        // self-attention\n        {\n            // rope freq factors for 128k context\n            ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);\n\n            ggml_tensor* attn_norm_output = build_norm(inpL,\n                    model.layers[il].attn_norm,\n                    model.layers[il].attn_norm_b,\n                    LLM_NORM_RMS, il);\n            cb(attn_norm_output, \"attn_norm\", il);\n\n            ggml_tensor * Qcur = nullptr;\n            ggml_tensor * Kcur = nullptr;\n            ggml_tensor * Vcur = nullptr;\n\n            if (model.layers[il].wqkv) {\n                cur = build_lora_mm(model.layers[il].wqkv, attn_norm_output);\n                cb(cur, \"wqkv\", il);\n\n                Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head * sizeof(float), cur->nb[1], 0 * sizeof(float) * (n_embd));\n                Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), cur->nb[1], 1 * sizeof(float) * (n_embd));\n                Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float), cur->nb[1], 1 * sizeof(float) * (n_embd + n_embd_gqa));\n                }\n                else {\n                Qcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wq, attn_norm_output), model.layers[il].bq);\n                Kcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wk, attn_norm_output), model.layers[il].bk);\n                Vcur = ggml_add(ctx0, build_lora_mm(model.layers[il].wv, attn_norm_output), model.layers[il].bv);\n\n                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n            }\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, rope_factors,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head)));\n            cb(Qcur, \"Qcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur      = ggml_get_rows(ctx0, cur,      inp_out_ids);\n            residual = ggml_get_rows(ctx0, residual, inp_out_ids);\n        }\n        cur = ggml_add(ctx0, cur, residual);\n        residual = cur;\n\n        cur = build_norm(cur,\n                model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward network\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    NULL,                      NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE branch\n            cur = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n            cb(cur, \"ffn_moe_out\", il);\n        }\n        cur = ggml_add(ctx0, residual, cur);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    if (model.output_b != nullptr) {\n        cb(cur, \"result_output_no_bias\", -1);\n        cur = ggml_add(ctx0, cur, model.output_b);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_phi3<false>;\ntemplate struct llm_build_phi3<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/plamo.cpp",
    "content": "#include \"models.h\"\n\nllm_build_plamo::llm_build_plamo(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_tensor * sa_inp = cur;\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur    = ggml_get_rows(ctx0,    cur, inp_out_ids);\n            sa_inp = ggml_get_rows(ctx0, sa_inp, inp_out_ids);\n            inpL   = ggml_get_rows(ctx0,   inpL, inp_out_ids);\n        }\n        ggml_tensor * sa_out = cur;\n\n        cur = sa_inp;\n\n        // feed-forward network\n        {\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, sa_out);\n        cur = ggml_add(ctx0, cur, inpL);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/plamo2.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_plamo2::llm_build_plamo2(const llama_model & model, const llm_graph_params & params) :\n    llm_build_mamba_base(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    // {n_embd, n_tokens}\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"embedding_output\", -1);\n\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_hybrid = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * residual = inpL;\n\n        // ggml_graph_add_node(gf, model.layers[il].attn_norm);\n        // cb(model.layers[il].attn_norm, \"attn_norm\", il);\n\n        // pre_mixer_norm\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n\n        // check if this layer is Mamba or Attention\n        const bool is_mamba_layer = hparams.is_recurrent(il);\n\n        if (is_mamba_layer) {\n            // PLaMo-2 Mamba layer\n            cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il);\n        } else {\n            // PLaMo-2 Attention layer\n            cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);\n        }\n\n        // post_mixer_norm\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        // residual connection\n        cur = ggml_add(ctx0, cur, residual);\n        cb(cur, \"attn_residual\", il);\n        residual = cur;\n\n        // pre-ffn norm\n        cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_pre_norm\", il);\n\n        // feed-forward network\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up, NULL, NULL,\n                NULL, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        // post ffn norm\n        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_post_norm\", il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);\n            residual = ggml_get_rows(ctx0, residual, inp_out_ids);\n        }\n\n        // residual connection\n        cur = ggml_add(ctx0, cur, residual);\n        cb(cur, \"ffn_residual\", il);\n\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    // final norm\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n\n    // Explicitly mark as output tensor to ensure proper backend assignment\n    ggml_set_output(cur);\n\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nggml_tensor * llm_build_plamo2::build_plamo2_attn_layer(llm_graph_input_attn_kv * inp,\n                                                        ggml_tensor *             inp_pos,\n                                                        ggml_tensor *             cur,\n                                                        const llama_model &       model,\n                                                        int                       il) {\n    // self-attention\n    {\n        // PLaMo-2 uses combined QKV tensor\n        ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);\n        cb(qkv, \"wqkv\", il);\n\n        // split QKV tensor into Q, K, V\n        const int64_t n_embd_head_q = hparams.n_embd_head_k();\n        const int64_t n_embd_head_k = hparams.n_embd_head_k();\n        const int64_t n_embd_head_v = hparams.n_embd_head_v();\n        int32_t       n_head        = hparams.n_head(il);\n        int32_t       n_head_kv     = hparams.n_head_kv(il);\n\n        const int64_t q_offset = 0;\n        const int64_t k_offset = n_embd_head_q * n_head;\n        const int64_t v_offset = k_offset + n_embd_head_k * n_head_kv;\n\n        ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_q, n_head, n_tokens, n_embd_head_q * sizeof(float),\n                                          qkv->nb[1], q_offset * ggml_element_size(qkv));\n        ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv, n_tokens, n_embd_head_k * sizeof(float),\n                                          qkv->nb[1], k_offset * ggml_element_size(qkv));\n        ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv, n_tokens, n_embd_head_v * sizeof(float),\n                                          qkv->nb[1], v_offset * ggml_element_size(qkv));\n\n        cb(Qcur, \"Qcur\", il);\n        cb(Kcur, \"Kcur\", il);\n        cb(Vcur, \"Vcur\", il);\n\n        Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n        cb(Qcur, \"Qcur_normed\", il);\n\n        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n\n        Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n        cb(Kcur, \"Kcur_normed\", il);\n\n        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                             ext_factor, attn_factor, beta_fast, beta_slow);\n\n        cur = build_attn(inp,\n            model.layers[il].wo, NULL,\n            Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f / sqrtf(float(n_embd_head_v)), il);\n    }\n\n    cb(cur, \"attn_out\", il);\n\n    return cur;\n}\n\nggml_tensor * llm_build_plamo2::build_plamo2_mamba_layer(llm_graph_input_rs * inp,\n                                                         ggml_tensor *        cur,\n                                                         const llama_model &  model,\n                                                         const llama_ubatch & ubatch,\n                                                         int                  il) {\n    const auto * mctx_cur = inp->mctx;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    const int64_t d_conv   = hparams.ssm_d_conv;\n    const int64_t d_inner  = hparams.ssm_d_inner;\n    const int64_t d_state  = hparams.ssm_d_state;\n    const int64_t n_heads  = hparams.ssm_dt_rank;\n    const int64_t head_dim = d_inner / n_heads;\n    const int64_t n_group  = hparams.ssm_n_group;\n    const int64_t n_seqs   = ubatch.n_seqs;\n\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n    GGML_ASSERT(d_inner % n_head == 0);\n    GGML_ASSERT(n_group == 0);\n\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    conv               = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs);\n\n    // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}\n    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);\n\n    // in_proj: {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}\n    ggml_tensor * zx = build_lora_mm(model.layers[il].ssm_in, cur);\n    cb(zx, \"mamba_in_proj\", il);\n    // {8192, 5, 1, 1} -> {8192, 1, 5, 1}\n    zx = ggml_permute(ctx0, zx, 0, 2, 1, 3);\n    zx = ggml_cont_4d(ctx0, zx, head_dim * 2, n_heads, n_seq_tokens, n_seqs);\n    cb(zx, \"mamba_in_proj_out\", il);\n\n    // split into z and x\n    // => {head_dim * n_heads, n_seq_tokens, n_seqs}\n    ggml_tensor * x = ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3],\n                                   head_dim * ggml_element_size(zx));\n    x               = ggml_cont_3d(ctx0, x, head_dim * n_heads, n_seq_tokens, n_seqs);\n    // x = ggml_permute(ctx0, x, 0, 2, 1, 3);\n    cb(x, \"mamba_x_split\", il);\n\n    ggml_tensor * z =\n        ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3], 0);\n    cb(z, \"mamba_z_split\", il);\n\n    // conv1d\n    {\n        // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs}\n        ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, x), 0);\n        cb(conv_x, \"mamba_conv1d_input\", il);\n\n        // copy last (d_conv - 1) columns back into the state cache\n        ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2],\n                                               n_seq_tokens * (conv_x->nb[0]));\n\n        ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,\n                                               ggml_view_1d(ctx0, conv_states_all,\n                                                            (d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs),\n                                                            kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) *\n                                                                ggml_element_size(conv_states_all))));\n        cb(conv_states_all, \"mamba_conv1d_state\", il);\n\n        // 1D convolution\n        x = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d);\n        cb(x, \"mamba_conv1d\", il);\n\n        x = ggml_silu(ctx0, x);\n        cb(x, \"mamba_conv1d_silu\", il);\n    }\n\n    // SSM\n    {\n        // bcdt_proj: {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs}\n        ggml_tensor * x_bcdt = build_lora_mm(model.layers[il].ssm_x, x);\n        cb(x_bcdt, \"mamba_bcdt_proj\", il);\n\n        // split into dt, B, C\n        const int64_t dt_dim = std::max(64, int(hparams.n_embd / 16));\n        ggml_tensor * B  = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2], 0);\n        ggml_tensor * C  = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],\n                                        ggml_element_size(x_bcdt) * d_state);\n        ggml_tensor * dt = ggml_view_3d(ctx0, x_bcdt, dt_dim, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],\n                                        ggml_element_size(x_bcdt) * (2 * d_state));\n        cb(B, \"mamba_B_raw\", il);\n        cb(C, \"mamba_C_raw\", il);\n        cb(dt, \"mamba_dt_raw\", il);\n\n        // Apply RMS norm to dt, B, C (PLaMo-2 specific)\n        B  = build_norm(B, model.layers[il].ssm_b_norm, NULL, LLM_NORM_RMS, il);\n        C  = build_norm(C, model.layers[il].ssm_c_norm, NULL, LLM_NORM_RMS, il);\n        dt = build_norm(dt, model.layers[il].ssm_dt_norm, NULL, LLM_NORM_RMS, il);\n        cb(B, \"mamba_B_normed\", il);\n        cb(C, \"mamba_C_normed\", il);\n        cb(dt, \"mamba_dt_normed\", il);\n\n        // dt_proj: {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs}\n        dt = build_lora_mm(model.layers[il].ssm_dt, dt);\n        dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b);\n        cb(dt, \"mamba_dt_proj\", il);\n\n        ggml_tensor * A = ggml_reshape_2d(ctx0, model.layers[il].ssm_a, 1, n_heads);\n        cb(A, \"mamba_A\", il);\n\n        x = ggml_view_4d(ctx0, x, head_dim, n_heads, n_seq_tokens, n_seqs, head_dim * ggml_element_size(x),\n                         head_dim * n_heads * ggml_element_size(x),\n                         head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);\n        B = ggml_view_4d(ctx0, B, d_state, 1, n_seq_tokens, n_seqs, d_state * B->nb[0], B->nb[1], B->nb[2], 0);\n        C = ggml_view_4d(ctx0, C, d_state, 1, n_seq_tokens, n_seqs, d_state * C->nb[0], C->nb[1], C->nb[2], 0);\n\n        // use the states and the indices provided by build_recurrent_state\n        // (this is necessary in order to properly use the states before they are overwritten,\n        //  while avoiding to make unnecessary copies of the states)\n        auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {\n            ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_heads, mctx_cur->get_size());\n\n            // Custom operator to optimize the parallel associative scan\n            // as described in the Annex D of the Mamba paper.\n            // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}\n            return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);\n        };\n\n        ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);\n        cb(y_ssm, \"mamba_ssm_scan\", il);\n\n        // store last states\n        ggml_build_forward_expand(\n            gf, ggml_cpy(\n                    ctx0,\n                    ggml_view_1d(ctx0, y_ssm, n_heads * head_dim * d_state * n_seqs,\n                                 n_heads * head_dim * n_seq_tokens * n_seqs * ggml_element_size(y_ssm)),\n                    ggml_view_1d(ctx0, ssm_states_all, n_heads * head_dim * d_state * n_seqs,\n                                 kv_head * n_seqs * n_heads * head_dim * d_state * ggml_element_size(ssm_states_all))));\n        cb(ssm_states_all, \"mamba_ssm_states\", il);\n\n        ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_heads, n_seq_tokens, n_seqs,\n                                       head_dim * ggml_element_size(x), head_dim * n_heads * ggml_element_size(x),\n                                       head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);\n        cb(y, \"mamba_y_view\", il);\n\n        // Add D parameter and apply gating with z\n        // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs}\n        ggml_tensor * D = ggml_reshape_2d(ctx0, model.layers[il].ssm_d, 1, n_heads);\n        y               = ggml_add(ctx0, y, ggml_mul(ctx0, x, D));\n        cb(y, \"mamba_y_add_d\", il);\n\n        y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);\n        cb(y, \"mamba_y_swiglu_z\", il);\n\n        // out_proj: {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}\n        y   = ggml_view_3d(ctx0, y, head_dim * n_heads, n_seq_tokens, n_seqs, y->nb[2], y->nb[3], 0);\n        cur = build_lora_mm(model.layers[il].ssm_out, y);\n        cb(cur, \"mamba_out_proj\", il);\n    }\n\n    // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}\n    cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);\n    cb(cur, \"mamba_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/plamo3.cpp",
    "content": "#include \"models.h\"\n\ntemplate <bool iswa>\nllm_build_plamo3<iswa>::llm_build_plamo3(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params) {\n    const int64_t head_dim_q = hparams.n_embd_head_k();\n    const int64_t head_dim_v = hparams.n_embd_head_v();\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL = build_inp_embd(model.tok_embd);\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * residual = inpL;\n\n        float freq_base_l  = 0.0f;\n        float freq_scale_l = 0.0f;\n        if constexpr (iswa) {\n            freq_base_l  = model.get_rope_freq_base (cparams, il);\n            freq_scale_l = model.get_rope_freq_scale(cparams, il);\n        } else {\n            freq_base_l  = freq_base;\n            freq_scale_l = freq_scale;\n        }\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);\n        cb(cur, \"wqkv\", il);\n\n        const int32_t n_head    = hparams.n_head(il);\n        const int32_t n_head_kv = hparams.n_head_kv(il);\n\n        const int64_t q_offset = 0;\n        const int64_t k_offset = head_dim_q * n_head;\n        const int64_t v_offset = k_offset + head_dim_q * n_head_kv;\n\n        ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head, n_tokens,\n                head_dim_q * sizeof(float), qkv->nb[1], q_offset * ggml_element_size(qkv));\n        ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head_kv, n_tokens,\n                head_dim_q * sizeof(float), qkv->nb[1], k_offset * ggml_element_size(qkv));\n        ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, head_dim_v, n_head_kv, n_tokens,\n                head_dim_v * sizeof(float), qkv->nb[1], v_offset * ggml_element_size(qkv));\n\n        cb(Qcur, \"Qcur\", il);\n        cb(Kcur, \"Kcur\", il);\n        cb(Vcur, \"Vcur\", il);\n\n        Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n        cb(Qcur, \"attn_q_norm\", il);\n        Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n        cb(Kcur, \"attn_k_norm\", il);\n\n        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow);\n        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,\n                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow);\n\n        const float attn_scale = 1.0f / sqrtf(float(head_dim_q));\n\n        cur = build_attn(inp_attn,\n                model.layers[il].wo, NULL,\n                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, attn_scale, il);\n        cb(cur, \"attn_out\", il);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);\n            residual = ggml_get_rows(ctx0, residual, inp_out_ids);\n        }\n\n        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, residual);\n        cb(cur, \"attn_residual\", il);\n\n        residual = cur;\n\n        cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                NULL,                      NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_post_norm\", il);\n\n        cur = ggml_add(ctx0, cur, residual);\n        cb(cur, \"ffn_residual\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_plamo3<false>;\ntemplate struct llm_build_plamo3<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/plm.cpp",
    "content": "#include \"models.h\"\n\nllm_build_plm::llm_build_plm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const float kq_scale = 1.0f/sqrtf(float(hparams.n_embd_head_k()));\n\n    const uint32_t n_embd_head_qk_rope = hparams.n_rot();\n    const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k() - hparams.n_rot();\n\n    const uint32_t kv_lora_rank = hparams.n_lora_kv;\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    // {n_embd, n_tokens}\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            ggml_tensor * q = NULL;\n            q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);\n            cb(q, \"q\", il);\n\n            // split into {n_head * n_embd_head_qk_nope, n_tokens}\n            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,\n                    ggml_row_size(q->type, hparams.n_embd_head_k()),\n                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),\n                    0);\n            cb(q_nope, \"q_nope\", il);\n\n            // and {n_head * n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,\n                    ggml_row_size(q->type, hparams.n_embd_head_k()),\n                    ggml_row_size(q->type, hparams.n_embd_head_k() * n_head),\n                    ggml_row_size(q->type, n_embd_head_qk_nope));\n            cb(q_pe, \"q_pe\", il);\n\n            // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);\n            cb(kv_pe_compresseed, \"kv_pe_compresseed\", il);\n\n            // split into {kv_lora_rank, n_tokens}\n            ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens,\n                    kv_pe_compresseed->nb[1],\n                    0);\n            cb(kv_compressed, \"kv_compressed\", il);\n\n            // and {n_embd_head_qk_rope, n_tokens}\n            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens,\n                    kv_pe_compresseed->nb[1],\n                    kv_pe_compresseed->nb[1],\n                    ggml_row_size(kv_pe_compresseed->type, kv_lora_rank));\n            cb(k_pe, \"k_pe\", il);\n\n            kv_compressed = build_norm(kv_compressed,\n                    model.layers[il].attn_kv_a_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(kv_compressed, \"kv_compressed\", il);\n\n            // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens}\n            ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed);\n            cb(kv, \"kv\", il);\n\n            // split into {n_head * n_embd_head_qk_nope, n_tokens}\n            ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,\n                    ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v()),\n                    ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v())),\n                    0);\n            cb(k_nope, \"k_nope\", il);\n\n            // and {n_head * n_embd_head_v, n_tokens}\n            ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v(), n_head, n_tokens,\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v())),\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v())*n_head),\n                    ggml_row_size(kv->type, (n_embd_head_qk_nope)));\n            cb(v_states, \"v_states\", il);\n\n            v_states = ggml_cont(ctx0, v_states);\n            cb(v_states, \"v_states\", il);\n\n            v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v() * n_head, n_tokens,\n                    ggml_row_size(kv->type, hparams.n_embd_head_v() * n_head),\n                    0);\n            cb(v_states, \"v_states\", il);\n\n            q_pe = ggml_rope_ext(\n                    ctx0, q_pe, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n            cb(q_pe, \"q_pe\", il);\n\n            // shared RoPE key\n            k_pe = ggml_rope_ext(\n                    ctx0, k_pe, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n            cb(k_pe, \"k_pe\", il);\n\n            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);\n            cb(q_states, \"q_states\", il);\n\n            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);\n            cb(k_states, \"k_states\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                NULL, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen.cpp",
    "content": "#include \"models.h\"\n\n\nllm_build_qwen::llm_build_qwen(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 2*sizeof(float)*(n_embd));\n\n            // using mode = 2 for neox mode\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward forward\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen2::llm_build_qwen2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    if (model.output_b != nullptr) {\n        cur = ggml_add(ctx0, cur, model.output_b);\n    }\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen2moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen2moe::llm_build_qwen2moe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, false,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        // FFN shared expert\n        {\n            ggml_tensor * cur_gate_inp = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);\n            cb(cur_gate_inp, \"ffn_shexp_gate_inp\", il);\n\n            // sigmoid\n            ggml_tensor * cur_gate = ggml_div(ctx0, ggml_silu(ctx0, cur_gate_inp), cur_gate_inp);\n            cb(cur_gate, \"ffn_shexp_gate\", il);\n\n            ggml_tensor * cur_ffn = build_ffn(cur,\n                    model.layers[il].ffn_up_shexp,   NULL, NULL,\n                    model.layers[il].ffn_gate_shexp, NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur_ffn, \"ffn_shexp\", il);\n\n            ggml_tensor * ffn_shexp_out = ggml_mul(ctx0, cur_ffn, cur_gate);\n            cb(ffn_shexp_out, \"ffn_shexp_out\", il);\n\n            moe_out = ggml_add(ctx0, moe_out, ffn_shexp_out);\n            cb(moe_out, \"ffn_out\", il);\n\n            cur = moe_out;\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen2vl.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen2vl::llm_build_qwen2vl(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_multi(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_multi(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen3.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen3::llm_build_qwen3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n            if (model.layers[il].wo_s) {\n                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);\n            }\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, model.layers[il].ffn_up_s,\n                model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,\n                model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen35.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_qwen35::llm_build_qwen35(const llama_model & model, const llm_graph_params & params) :\n    llm_build_delta_net_base(params), model(model) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    cb(inpL, \"model.input_embed\", -1);\n\n    auto * inp = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_pos     = build_inp_pos();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_build_forward_expand(gf, cur);\n\n        // Determine layer type and build appropriate attention mechanism\n        if (hparams.is_recurrent(il)) {\n            // Linear attention layer (gated delta net)\n            cur = build_layer_attn_linear(inp->get_recr(), cur, il);\n        } else {\n            // Full attention layer\n            cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // Residual connection\n        cur = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"attn_residual\", il);\n\n        // Save the tensor before post-attention norm for residual connection\n        ggml_tensor * ffn_residual = cur;\n\n        // Post-attention norm\n        ggml_tensor * attn_post_norm = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);\n        cb(attn_post_norm, \"attn_post_norm\", il);\n\n        // Dense FFN layer - without residual connection\n        cur = build_layer_ffn(attn_post_norm, il);\n        cb(cur, \"ffn_out\", il);\n\n        // Residual connection for FFN - add to the tensor from before post_attention_layernorm\n        cur = ggml_add(ctx0, cur, ffn_residual);\n        cb(cur, \"post_ffn\", il);\n\n        // Input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // Final norm\n    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // LM head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_qwen35::build_qkvz(\n                ggml_tensor * input,\n                        int   il) {\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);\n    qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);\n    cb(qkv_mixed, \"linear_attn_qkv_mixed\", il);\n\n    ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input, model.layers[il].wqkv_gate_s);\n    cb(z, \"z\", il);\n\n    return { qkv_mixed, z };\n}\n\nggml_tensor * llm_build_qwen35::build_norm_gated(\n        ggml_tensor * input,\n        ggml_tensor * weights,\n        ggml_tensor * gate,\n        int           layer) {\n    ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);\n    ggml_tensor * gated_silu = ggml_silu(ctx0, gate);\n\n    return ggml_mul(ctx0, normalized, gated_silu);\n}\n\nggml_tensor * llm_build_qwen35::build_layer_attn(\n        llm_graph_input_attn_kv * inp,\n        ggml_tensor *             cur,\n        ggml_tensor *             inp_pos,\n        int *                     sections,\n        int                       il) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention\n\n    // Qwen3Next uses a single Q projection that outputs query + gate\n    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]\n    cb(Qcur_full, \"Qcur_full\", il);\n\n    ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,\n        ggml_element_size(Qcur_full) * n_embd_head * 2,\n        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, 0);\n    cb(Qcur, \"Qcur_reshaped\", il);\n\n    // Apply Q normalization\n    Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Qcur, \"Qcur_normed\", il);\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n    cb(Kcur, \"Kcur\", il);\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n    cb(Vcur, \"Vcur\", il);\n\n    // Apply K normalization\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n    Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Kcur, \"Kcur_normed\", il);\n\n    ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,\n        ggml_element_size(Qcur_full) * n_embd_head * 2,\n        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,\n        ggml_element_size(Qcur_full) * n_embd_head);\n    gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);\n    cb(gate, \"gate_reshaped\", il);\n\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n    // Apply MRoPE\n    Qcur = ggml_rope_multi(\n            ctx0, Qcur, inp_pos, nullptr,\n            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n            ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n    Kcur = ggml_rope_multi(\n            ctx0, Kcur, inp_pos, nullptr,\n            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n            ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    // Attention computation\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    cur = build_attn(inp,\n                nullptr, nullptr,\n                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n    cb(cur, \"attn_pregate\", il);\n\n    ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate);\n    cb(gate_sigmoid, \"gate_sigmoid\", il);\n\n    cur = ggml_mul(ctx0, cur, gate_sigmoid);\n    cb(cur, \"attn_gated\", il);\n\n    cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);\n    cb(cur, \"attn_output\", il);\n\n    return cur;\n}\n\nggml_tensor * llm_build_qwen35::build_layer_attn_linear(\n        llm_graph_input_rs * inp,\n        ggml_tensor *        cur,\n        int                  il) {\n    const auto * mctx_cur = inp->mctx;\n\n    const int64_t d_inner      = hparams.ssm_d_inner;\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t head_k_dim   = hparams.ssm_d_state;\n    const int64_t num_k_heads  = hparams.ssm_n_group;\n    const int64_t num_v_heads  = hparams.ssm_dt_rank;\n    const int64_t head_v_dim   = d_inner / num_v_heads;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n\n    // Input projections\n    auto qkvz = build_qkvz(cur, il);\n    ggml_tensor * qkv_mixed = qkvz.first;\n    ggml_tensor * z         = qkvz.second;\n\n    ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);\n    beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);\n    cb(beta, \"beta\", il);\n\n    beta = ggml_sigmoid(ctx0, beta);\n\n    ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);\n    alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);\n    cb(alpha, \"alpha\", il);\n\n    ggml_tensor * alpha_biased   = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);\n    ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);\n    cb(alpha_softplus, \"a_softplus\", il);\n\n    ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus\n    cb(gate, \"gate\", il);\n\n    gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Get convolution states from cache\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    // Build the convolution states tensor\n    ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    cb(conv_states, \"conv_states\", il);\n\n    // Calculate convolution kernel size\n    ggml_tensor * conv_kernel      = model.layers[il].ssm_conv1d;\n    const int64_t conv_kernel_size = conv_kernel->ne[0];\n    const int64_t conv_channels    = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state;\n\n    conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);\n    cb(conv_states, \"conv_states_reshaped\", il);\n\n    qkv_mixed = ggml_transpose(ctx0, qkv_mixed);\n    cb(qkv_mixed, \"qkv_mixed_transposed\", il);\n\n    ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);\n    cb(conv_input, \"conv_input\", il);\n\n    // Update convolution state cache\n    // Extract the last (conv_kernel_size - 1) states from conv_input\n    ggml_tensor * last_conv_states =\n        ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs, conv_input->nb[1],\n                     conv_input->nb[2], (conv_input->ne[0] - conv_states->ne[0]) * ggml_element_size(conv_input));\n    cb(last_conv_states, \"last_conv_states\", il);\n\n    ggml_tensor * state_update_target =\n        ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,\n                     kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));\n    cb(state_update_target, \"state_update_target\", il);\n\n    ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_states, state_update_target));\n\n    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);\n    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);\n    cb(state, \"state_predelta\", il);\n\n    ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);\n    cb(conv_output_proper, \"conv_output_raw\", il);\n\n    ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);\n    cb(conv_output_silu, \"conv_output_silu\", il);\n\n    ggml_tensor * conv_qkv_mix = conv_output_silu;\n\n    // Calculate the total conv dimension\n    int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;\n    int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, qkv_dim);\n\n    // Extract the convolved Q, K, V from conv_output\n    ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            0);\n\n    ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));\n\n    ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_v_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));\n\n    cb(q_conv, \"q_conv\", il);\n    cb(k_conv, \"k_conv\", il);\n    cb(v_conv, \"v_conv\", il);\n\n    const float eps_norm = hparams.f_norm_rms_eps;\n\n    q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm);\n    k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm);\n\n    //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // if head keys and value keys are different, repeat to force tensors into matching shapes\n    // note: need explicit repeat only if we are not using the fused GDN\n    if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {\n        GGML_ASSERT(num_v_heads % num_k_heads == 0);\n        q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);\n        k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);\n    }\n\n    cb(q_conv, \"q_conv_predelta\", il);\n    cb(k_conv, \"k_conv_predelta\", il);\n    cb(v_conv, \"v_conv_predelta\", il);\n\n    auto attn_out = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il);\n\n    ggml_tensor * output    = attn_out.first;\n    ggml_tensor * new_state = attn_out.second;\n    cb(output, \"attn_output\", il);\n    cb(new_state, \"new_state\", il);\n\n    // Update the recurrent states\n    ggml_build_forward_expand(gf,\n            ggml_cpy(ctx0, new_state,\n                ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,\n                    kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));\n\n    // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]\n    ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Apply gated normalization: self.norm(core_attn_out, z)\n    ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);\n\n    // Final reshape: [head_dim, n_heads, n_tokens, n_seqs] -> [n_tokens, n_seqs, n_heads * head_dim]\n    ggml_tensor * final_output = ggml_reshape_3d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);\n    cb(final_output, \"final_output\", il);\n\n    // Output projection\n    cur = build_lora_mm(model.layers[il].ssm_out, final_output, model.layers[il].ssm_out_s);\n    cb(cur, \"linear_attn_out\", il);\n\n    // Reshape back to original dimensions\n    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_seq_tokens * n_seqs);\n\n    return cur;\n}\n\nggml_tensor * llm_build_qwen35::build_layer_ffn(ggml_tensor * cur, const int il) {\n    // Qwen3.5 does not use MoE FFN\n    GGML_ASSERT(model.layers[il].ffn_gate_inp == nullptr);\n\n    cur = build_ffn(cur,\n        model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,\n        model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,\n        model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,\n        NULL,\n        LLM_FFN_SILU, LLM_FFN_PAR, il);\n    cb(cur, \"ffn_out\", il);\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen35moe.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_qwen35moe::llm_build_qwen35moe(const llama_model & model, const llm_graph_params & params) :\n    llm_build_delta_net_base(params), model(model) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    cb(inpL, \"model.input_embed\", -1);\n\n    auto * inp = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_pos     = build_inp_pos();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_build_forward_expand(gf, cur);\n\n        // Determine layer type and build appropriate attention mechanism\n        if (hparams.is_recurrent(il)) {\n            // Linear attention layer (gated delta net)\n            cur = build_layer_attn_linear(inp->get_recr(), cur, il);\n        } else {\n            // Full attention layer\n            cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // Residual connection\n        cur = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"attn_residual\", il);\n\n        // Save the tensor before post-attention norm for residual connection\n        ggml_tensor * ffn_residual = cur;\n\n        // Post-attention norm\n        ggml_tensor * attn_post_norm = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);\n        cb(attn_post_norm, \"attn_post_norm\", il);\n\n        // MOE FFN layer\n        cur = build_layer_ffn(attn_post_norm, il);\n        cb(cur, \"ffn_out\", il);\n\n        // Residual connection for FFN - add to the tensor from before post_attention_layernorm\n        cur = ggml_add(ctx0, cur, ffn_residual);\n        cb(cur, \"post_moe\", il);\n\n        // Input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // Final norm\n    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // LM head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_qwen35moe::build_qkvz(\n                ggml_tensor * input,\n                        int   il) {\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);\n    qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);\n    cb(qkv_mixed, \"linear_attn_qkv_mixed\", il);\n\n    ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input, model.layers[il].wqkv_gate_s);\n    cb(z, \"z\", il);\n\n    return { qkv_mixed, z };\n}\n\nggml_tensor * llm_build_qwen35moe::build_norm_gated(\n        ggml_tensor * input,\n        ggml_tensor * weights,\n        ggml_tensor * gate,\n        int           layer) {\n    ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);\n    ggml_tensor * gated_silu = ggml_silu(ctx0, gate);\n\n    return ggml_mul(ctx0, normalized, gated_silu);\n}\n\nggml_tensor * llm_build_qwen35moe ::build_layer_attn(\n        llm_graph_input_attn_kv * inp,\n        ggml_tensor *             cur,\n        ggml_tensor *             inp_pos,\n        int *                     sections,\n        int                       il) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention\n\n    // Qwen3Next uses a single Q projection that outputs query + gate\n    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]\n    cb(Qcur_full, \"Qcur_full\", il);\n\n    ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,\n        ggml_element_size(Qcur_full) * n_embd_head * 2,\n        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, 0);\n    cb(Qcur, \"Qcur_reshaped\", il);\n\n    // Apply Q normalization\n    Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Qcur, \"Qcur_normed\", il);\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n    cb(Kcur, \"Kcur\", il);\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n    cb(Vcur, \"Vcur\", il);\n\n    // Apply K normalization\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n    Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Kcur, \"Kcur_normed\", il);\n\n    ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,\n        ggml_element_size(Qcur_full) * n_embd_head * 2,\n        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,\n        ggml_element_size(Qcur_full) * n_embd_head);\n    gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);\n    cb(gate, \"gate_reshaped\", il);\n\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n    // Apply IMRoPE\n    Qcur = ggml_rope_multi(\n            ctx0, Qcur, inp_pos, nullptr,\n            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n            ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n    Kcur = ggml_rope_multi(\n            ctx0, Kcur, inp_pos, nullptr,\n            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n            ext_factor, attn_factor, beta_fast, beta_slow\n            );\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    // Attention computation\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    cur = build_attn(inp,\n                nullptr, nullptr,\n                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n    cb(cur, \"attn_pregate\", il);\n\n    ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate);\n    cb(gate_sigmoid, \"gate_sigmoid\", il);\n\n    cur = ggml_mul(ctx0, cur, gate_sigmoid);\n    cb(cur, \"attn_gated\", il);\n\n    cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);\n    cb(cur, \"attn_output\", il);\n\n    return cur;\n}\n\nggml_tensor * llm_build_qwen35moe ::build_layer_attn_linear(\n        llm_graph_input_rs * inp,\n        ggml_tensor *        cur,\n        int                  il) {\n    const auto * mctx_cur = inp->mctx;\n\n    const int64_t d_inner      = hparams.ssm_d_inner;\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t head_k_dim   = hparams.ssm_d_state;\n    const int64_t num_k_heads  = hparams.ssm_n_group;\n    const int64_t num_v_heads  = hparams.ssm_dt_rank;\n    const int64_t head_v_dim   = d_inner / num_v_heads;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n\n    // Input projections\n    auto qkvz = build_qkvz(cur, il);\n    ggml_tensor * qkv_mixed = qkvz.first;\n    ggml_tensor * z         = qkvz.second;\n\n    ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);\n    beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);\n    cb(beta, \"beta\", il);\n\n    beta = ggml_sigmoid(ctx0, beta);\n\n    ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);\n    alpha = ggml_cont_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);\n    cb(alpha, \"alpha\", il);\n\n    ggml_tensor * alpha_biased   = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);\n    ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);\n    cb(alpha_softplus, \"a_softplus\", il);\n\n    ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus\n    cb(gate, \"gate\", il);\n\n    gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Get convolution states from cache\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    // Build the convolution states tensor\n    ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    cb(conv_states, \"conv_states\", il);\n\n    // Calculate convolution kernel size\n    ggml_tensor * conv_kernel      = model.layers[il].ssm_conv1d;\n    const int64_t conv_kernel_size = conv_kernel->ne[0];\n    const int64_t conv_channels    = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state;\n\n    conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);\n    cb(conv_states, \"conv_states_reshaped\", il);\n\n    qkv_mixed = ggml_transpose(ctx0, qkv_mixed);\n    cb(qkv_mixed, \"qkv_mixed_transposed\", il);\n\n    ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);\n    cb(conv_input, \"conv_input\", il);\n\n    // Update convolution state cache\n    // Extract the last (conv_kernel_size - 1) states from conv_input\n    ggml_tensor * last_conv_states =\n        ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs, conv_input->nb[1],\n                     conv_input->nb[2], (conv_input->ne[0] - conv_states->ne[0]) * ggml_element_size(conv_input));\n    cb(last_conv_states, \"last_conv_states\", il);\n\n    ggml_tensor * state_update_target =\n        ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,\n                     kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));\n    cb(state_update_target, \"state_update_target\", il);\n\n    ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_states, state_update_target));\n\n    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);\n    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);\n    cb(state, \"state_predelta\", il);\n\n    ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);\n    cb(conv_output_proper, \"conv_output_raw\", il);\n\n    ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);\n    cb(conv_output_silu, \"conv_output_silu\", il);\n\n    ggml_tensor * conv_qkv_mix = conv_output_silu;\n\n    // Calculate the total conv dimension\n    int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;\n    int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, qkv_dim);\n\n    // Extract the convolved Q, K, V from conv_output\n    ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            0);\n\n    ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));\n\n    ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_v_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));\n\n    cb(q_conv, \"q_conv\", il);\n    cb(k_conv, \"k_conv\", il);\n    cb(v_conv, \"v_conv\", il);\n\n    const float eps_norm = hparams.f_norm_rms_eps;\n\n    q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm);\n    k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm);\n\n    //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // if head keys and value keys are different, repeat to force tensors into matching shapes\n    // note: need explicit repeat only if we are not using the fused GDN\n    if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {\n        GGML_ASSERT(num_v_heads % num_k_heads == 0);\n        q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);\n        k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);\n    }\n\n    cb(q_conv, \"q_conv_predelta\", il);\n    cb(k_conv, \"k_conv_predelta\", il);\n    cb(v_conv, \"v_conv_predelta\", il);\n\n    auto attn_out = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il);\n\n    ggml_tensor * output    = attn_out.first;\n    ggml_tensor * new_state = attn_out.second;\n    cb(output, \"attn_output\", il);\n    cb(new_state, \"new_state\", il);\n\n    // Update the recurrent states\n    ggml_build_forward_expand(gf,\n            ggml_cpy(ctx0, new_state,\n                ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,\n                    kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));\n\n    // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]\n    ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Apply gated normalization: self.norm(core_attn_out, z)\n    ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);\n\n    // Final reshape: [head_dim, n_heads, n_tokens, n_seqs] -> [n_tokens, n_seqs, n_heads * head_dim]\n    ggml_tensor * final_output = ggml_reshape_3d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);\n    cb(final_output, \"final_output\", il);\n\n    // Output projection\n    cur = build_lora_mm(model.layers[il].ssm_out, final_output, model.layers[il].ssm_out_s);\n    cb(cur, \"linear_attn_out\", il);\n\n    // Reshape back to original dimensions\n    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_seq_tokens * n_seqs);\n\n    return cur;\n}\n\nggml_tensor * llm_build_qwen35moe ::build_layer_ffn(ggml_tensor * cur, const int il) {\n    // Check if this is an MoE layer\n    GGML_ASSERT(model.layers[il].ffn_gate_inp != nullptr);\n\n    ggml_tensor * moe_out =\n        build_moe_ffn(cur,\n            model.layers[il].ffn_gate_inp,\n            model.layers[il].ffn_up_exps,\n            model.layers[il].ffn_gate_exps,\n            model.layers[il].ffn_down_exps,\n            nullptr,\n            n_expert, n_expert_used,\n            LLM_FFN_SILU, true,\n            hparams.expert_weights_scale,\n            LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,\n            nullptr, model.layers[il].ffn_gate_up_exps,\n            model.layers[il].ffn_up_exps_s,\n            model.layers[il].ffn_gate_exps_s,\n            model.layers[il].ffn_down_exps_s);\n    cb(moe_out, \"ffn_moe_out\", il);\n\n    // Add shared experts if present - following Qwen3Next reference implementation\n    if (model.layers[il].ffn_up_shexp != nullptr) {\n        ggml_tensor * ffn_shexp =\n            build_ffn(cur,\n                model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,\n                model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,\n                model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(ffn_shexp, \"ffn_shexp\", il);\n\n        // Apply shared expert gating as in the reference implementation\n        // The shared expert has its own gate that is sigmoided\n        // Note: ffn_gate_inp_shexp is the shared expert gate (outputs 1 value per token)\n        ggml_tensor * shared_gate = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);\n        cb(shared_gate, \"shared_expert_gate\", il);\n\n        // Apply sigmoid to the gate\n        shared_gate = ggml_sigmoid(ctx0, shared_gate);\n        cb(shared_gate, \"shared_expert_gate_sigmoid\", il);\n\n\n        // Apply the gate to the shared expert output\n        ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);\n        cb(ffn_shexp, \"ffn_shexp_gated\", il);\n\n        cur = ggml_add(ctx0, moe_out, ffn_shexp);\n        cb(cur, \"ffn_out\", il);\n    } else {\n        cur = moe_out;\n    }\n\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen3moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen3moe::llm_build_qwen3moe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n            if (model.layers[il].wo_s) {\n                cur = ggml_mul(ctx0, cur, model.layers[il].wo_s);\n            }\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il,\n                    nullptr, nullptr,\n                    model.layers[il].ffn_up_exps_s,\n                    model.layers[il].ffn_gate_exps_s,\n                    model.layers[il].ffn_down_exps_s);\n        cb(moe_out, \"ffn_moe_out\", il);\n        cur = moe_out;\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen3next.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_qwen3next::llm_build_qwen3next(const llama_model & model, const llm_graph_params & params) :\n    llm_build_delta_net_base(params), model(model) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n    cb(inpL, \"model.embed_tokens\", -1);\n\n    auto * inp = build_inp_mem_hybrid();\n\n    ggml_tensor * inp_pos     = build_inp_pos();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_build_forward_expand(gf, cur);\n\n        // Determine layer type and build appropriate attention mechanism\n        if (hparams.is_recurrent(il)) {\n            // Linear attention layer (gated delta net)\n            cur = build_layer_attn_linear(inp->get_recr(), cur, il);\n        } else {\n            // Full attention layer\n            cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        // Residual connection\n        cur = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"attn_residual\", il);\n\n        // Save the tensor before post-attention norm for residual connection\n        ggml_tensor * ffn_residual = cur;\n\n        // Post-attention norm\n        ggml_tensor * attn_post_norm = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);\n        cb(attn_post_norm, \"attn_post_norm\", il);\n\n        // FFN layer (MoE or dense) - without residual connection\n        cur = build_layer_ffn(attn_post_norm, il);\n        cb(cur, \"ffn_out\", il);\n\n        // Residual connection for FFN - add to the tensor from before post_attention_layernorm\n        cur = ggml_add(ctx0, cur, ffn_residual);\n        cb(cur, \"post_moe\", il);\n\n        // Input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    // Final norm\n    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // LM head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// utility to get one slice from the third dimension\n// input dim:  [x, y, c, b]\n// output dim: [x, y, 1, b]\nstatic ggml_tensor * get_slice_2d(ggml_context * ctx0, ggml_tensor * t, int64_t c) {\n    return ggml_view_4d(ctx0, t, t->ne[0], t->ne[1], 1, t->ne[3],\n        t->nb[1], t->nb[2], t->nb[3], t->nb[2] * c);\n}\n\nggml_tensor * llm_build_qwen3next::build_norm_gated(\n        ggml_tensor * input,\n        ggml_tensor * weights,\n        ggml_tensor * gate,\n        int           layer) {\n    ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);\n    ggml_tensor * gated_silu = ggml_silu(ctx0, gate);\n\n    return ggml_mul(ctx0, normalized, gated_silu);\n}\n\nggml_tensor * llm_build_qwen3next::build_layer_attn(\n        llm_graph_input_attn_kv * inp,\n        ggml_tensor *             cur,\n        ggml_tensor *             inp_pos,\n        int                       il) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention\n\n    // Qwen3Next uses a single Q projection that outputs query + gate\n    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur);\n    cb(Qcur_full, \"Qcur_full\", il);\n\n    Qcur_full = ggml_reshape_4d(ctx0, Qcur_full, n_embd_head * 2, n_head, n_tokens, 1);\n\n    // Split Q projection into query and gate\n    // The split should be along dimension 0 (the feature dimension)\n    ggml_tensor * Qcur = ggml_view_4d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, 1,\n                                            Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], 0);\n    cb(Qcur, \"Qcur_view\", il);\n\n    ggml_tensor * gate =\n        ggml_view_4d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, 1,\n                     Qcur_full->nb[1], Qcur_full->nb[2], Qcur_full->nb[3], n_embd_head * ggml_element_size(Qcur_full));\n    cb(gate, \"gate\", il);\n\n    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n    cb(Kcur, \"Kcur\", il);\n\n    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n    cb(Vcur, \"Vcur\", il);\n\n    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n    Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Qcur, \"Qcur_normed\", il);\n\n    Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);\n    cb(Kcur, \"Kcur_normed\", il);\n\n    Qcur = ggml_rope_ext(\n            ctx0, Qcur, inp_pos, nullptr,\n            n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n            ext_factor, attn_factor, beta_fast, beta_slow);\n\n    Kcur = ggml_rope_ext(\n            ctx0, Kcur, inp_pos, nullptr,\n            n_rot, rope_type, n_ctx_orig, freq_base,\n            freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);\n\n    cb(Qcur, \"Qcur\", il);\n    cb(Kcur, \"Kcur\", il);\n    cb(Vcur, \"Vcur\", il);\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    cur = build_attn(inp,\n                nullptr, nullptr,\n                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n    cb(cur, \"attn_pregate\", il);\n\n    // TODO: CUDA is missing non-contiguous unary ops. when implemented: remove this cont\n    gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);\n\n    gate = ggml_sigmoid(ctx0, gate);\n    cb(gate, \"gate_sigmoid\", il);\n\n    gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens);\n\n    cur = ggml_mul(ctx0, cur, gate);\n    cb(cur, \"attn_gated\", il);\n\n    cur = build_lora_mm(model.layers[il].wo, cur);\n    cb(cur, \"attn_output\", il);\n\n    return cur;\n}\n\nstd::pair<ggml_tensor *, ggml_tensor *> llm_build_qwen3next::build_qkvz(\n                ggml_tensor * input,\n                        int   il) {\n    const int64_t d_inner      = hparams.ssm_d_inner;\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t head_k_dim   = hparams.ssm_d_state;\n    const int64_t num_k_heads  = hparams.ssm_n_group;\n    const int64_t num_v_heads  = hparams.ssm_dt_rank;\n    const int64_t head_v_dim   = d_inner / num_v_heads;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    if (model.layers[il].wqkv) {\n        // optimized path\n        ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input);\n        qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);\n        cb(qkv_mixed, \"linear_attn_qkv_mixed\", il);\n\n        ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input);\n        cb(z, \"z\", il);\n\n        return { qkv_mixed, z };\n    } else {\n        // legacy (slower) path\n        ggml_tensor * mixed_qkvz = build_lora_mm(model.layers[il].ssm_in, input);\n        cb(mixed_qkvz, \"linear_attn_mixed_qkvz\", il);\n\n        int64_t       qkvz_new_dim        = 2 * head_k_dim + 2 * head_v_dim * (num_v_heads / num_k_heads);\n        ggml_tensor * mixed_qkvz_reshaped = ggml_reshape_4d(ctx0, mixed_qkvz, qkvz_new_dim, num_k_heads, n_seq_tokens, n_seqs);\n\n        // Split mixed_qkvz into query, key, value, z\n        int64_t split_sizes_qkvz[4] = {\n            head_k_dim,                              // query size\n            head_k_dim,                              // key size\n            head_v_dim * num_v_heads / num_k_heads,  // value size\n            head_v_dim * num_v_heads / num_k_heads   // z size\n        };\n\n        ggml_tensor * query =\n            ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[0], num_k_heads, n_seq_tokens, n_seqs,\n                        mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3], 0);\n        cb(query, \"q\", il);\n\n        ggml_tensor * key = ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[1], num_k_heads, n_seq_tokens, n_seqs,\n                                        mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],\n                                        split_sizes_qkvz[0] * ggml_element_size(mixed_qkvz_reshaped));\n        cb(key, \"k\", il);\n\n        ggml_tensor * value =\n            ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[2], num_k_heads, n_seq_tokens, n_seqs,\n                        mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],\n                        (split_sizes_qkvz[0] + split_sizes_qkvz[1]) * ggml_element_size(mixed_qkvz_reshaped));\n        cb(value, \"v\", il);\n\n        ggml_tensor * z = ggml_view_4d(ctx0, mixed_qkvz_reshaped, split_sizes_qkvz[3], num_k_heads, n_seq_tokens, n_seqs,\n                                    mixed_qkvz_reshaped->nb[1], mixed_qkvz_reshaped->nb[2], mixed_qkvz_reshaped->nb[3],\n                                    (split_sizes_qkvz[0] + split_sizes_qkvz[1] + split_sizes_qkvz[2]) * ggml_element_size(mixed_qkvz_reshaped));\n        z = ggml_cont(ctx0, z);\n        cb(z, \"z\", il);\n\n        // After creating query, key, and value_reshaped, reshape each to flatten the head dimensions\n        // query: [head_k_dim, num_k_heads, n_tokens, n_seqs] -> [head_k_dim * num_k_heads, n_tokens, n_seqs]\n        ggml_tensor * query_flat = ggml_cont_3d(ctx0, query, head_k_dim * num_k_heads, n_seq_tokens, n_seqs);\n        cb(query_flat, \"query_flat\", il);\n\n        // key: [head_k_dim, num_k_heads, n_tokens, n_seqs] -> [head_k_dim * num_k_heads, n_tokens, n_seqs]\n        ggml_tensor * key_flat = ggml_cont_3d(ctx0, key, head_k_dim * num_k_heads, n_seq_tokens, n_seqs);\n        cb(key_flat, \"key_flat\", il);\n\n        // value_reshaped: [head_v_dim, num_v_heads, n_tokens, n_seqs] -> [head_v_dim * num_v_heads, n_tokens, n_seqs]\n        ggml_tensor * value_flat = ggml_cont_3d(ctx0, value, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);\n        cb(value_flat, \"value_flat\", il);\n\n        // Now concatenate along the feature dimension (dim 0) to get [conv_dim, n_tokens, n_seqs]\n        ggml_tensor * qkv_mixed = ggml_concat(ctx0, query_flat, key_flat, 0);\n        qkv_mixed               = ggml_concat(ctx0, qkv_mixed, value_flat, 0);\n        cb(qkv_mixed, \"qkv_mixed\", il);\n\n        return { qkv_mixed, z };\n    }\n}\n\nggml_tensor * llm_build_qwen3next::build_layer_attn_linear(\n        llm_graph_input_rs * inp,\n        ggml_tensor *        cur,\n        int                  il) {\n    const auto * mctx_cur = inp->mctx;\n\n    const int64_t d_inner      = hparams.ssm_d_inner;\n    const int64_t n_seqs       = ubatch.n_seqs;\n    const int64_t head_k_dim   = hparams.ssm_d_state;\n    const int64_t num_k_heads  = hparams.ssm_n_group;\n    const int64_t num_v_heads  = hparams.ssm_dt_rank;\n    const int64_t head_v_dim   = d_inner / num_v_heads;\n    const int64_t n_seq_tokens = ubatch.n_seq_tokens;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    GGML_ASSERT(n_seqs != 0);\n    GGML_ASSERT(ubatch.equal_seqs());\n    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);\n\n    // Input projections\n    auto qkvz = build_qkvz(cur, il);\n    ggml_tensor * qkv_mixed = qkvz.first;\n    ggml_tensor * z         = qkvz.second;\n\n    ggml_tensor * mixed_ba = build_lora_mm(model.layers[il].ssm_beta_alpha, cur);\n    cb(mixed_ba, \"linear_attn_mixed_ba\", il);\n\n    // Reshape mixed_ba: [batch, seq_len, hidden_size] -> [batch, seq_len, num_k_heads, 2*num_v_heads/num_k_heads]\n    int64_t       ba_new_dim        = 2 * num_v_heads / num_k_heads;\n    ggml_tensor * mixed_ba_reshaped = ggml_reshape_4d(ctx0, mixed_ba, ba_new_dim, num_k_heads, n_seq_tokens, n_seqs);\n\n    // Split mixed_ba into b and a (beta and alpha parameters)\n    int64_t split_sizes_ba[2] = {\n        num_v_heads / num_k_heads,  // beta size\n        num_v_heads / num_k_heads   // alpha size\n    };\n\n    ggml_tensor * b = ggml_view_4d(ctx0, mixed_ba_reshaped, split_sizes_ba[0], num_k_heads, n_seq_tokens, n_seqs,\n                                   mixed_ba_reshaped->nb[1], mixed_ba_reshaped->nb[2], mixed_ba_reshaped->nb[3], 0);\n    cb(b, \"b\", il);\n\n    ggml_tensor * a = ggml_view_4d(ctx0, mixed_ba_reshaped, split_sizes_ba[1], num_k_heads, n_seq_tokens, n_seqs,\n                                   mixed_ba_reshaped->nb[1], mixed_ba_reshaped->nb[2], mixed_ba_reshaped->nb[3],\n                                   split_sizes_ba[0] * ggml_element_size(mixed_ba_reshaped));\n    cb(a, \"a\", il);\n\n    // TODO: CUDA is missing non-contiguous unary ops. when implemented: remove this cont\n    b = ggml_cont(ctx0, b);\n\n    ggml_tensor * beta = ggml_sigmoid(ctx0, b);\n\n    // Reshape a to merge head dimensions: [batch, seq_len, num_k_heads, num_v_heads/num_k_heads] -> [batch, seq_len, num_v_heads]\n    ggml_tensor * alpha = ggml_cont_3d(ctx0, a, num_v_heads, n_seq_tokens, n_seqs);\n\n    ggml_tensor * alpha_biased   = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);\n    ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);\n    cb(alpha_softplus, \"a_softplus\", il);\n\n    ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus\n    cb(gate, \"gate\", il);\n\n    beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);\n    gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Get convolution states from cache\n    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);\n    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);\n\n    // Build the convolution states tensor\n    ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);\n    cb(conv_states, \"conv_states\", il);\n\n    // Calculate convolution kernel size\n    ggml_tensor * conv_kernel      = model.layers[il].ssm_conv1d;\n    const int64_t conv_kernel_size = conv_kernel->ne[0];\n    const int64_t conv_channels    = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state;\n\n    conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);\n    cb(conv_states, \"conv_states_reshaped\", il);\n\n    qkv_mixed = ggml_transpose(ctx0, qkv_mixed);\n    cb(qkv_mixed, \"qkv_mixed_transposed\", il);\n\n    ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);\n    cb(conv_input, \"conv_input\", il);\n\n    // Update convolution state cache\n    // Extract the last (conv_kernel_size - 1) states from conv_input\n    ggml_tensor * last_conv_states =\n        ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs, conv_input->nb[1],\n                     conv_input->nb[2], (conv_input->ne[0] - conv_states->ne[0]) * ggml_element_size(conv_input));\n    cb(last_conv_states, \"last_conv_states\", il);\n\n    ggml_tensor * state_update_target =\n        ggml_view_1d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels * n_seqs,\n                     kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));\n    cb(state_update_target, \"state_update_target\", il);\n\n    ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_states, state_update_target));\n\n    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);\n    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);\n    cb(state, \"state_predelta\", il);\n\n    ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);\n    cb(conv_output_proper, \"conv_output_raw\", il);\n\n    ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);\n    cb(conv_output_silu, \"conv_output_silu\", il);\n\n    ggml_tensor * conv_qkv_mix = conv_output_silu;\n\n    // Calculate the total conv dimension\n    int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;\n    int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, qkv_dim);\n\n    // Extract the convolved Q, K, V from conv_output\n    ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            0);\n\n    ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_k_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));\n\n    ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,\n            ggml_row_size(conv_qkv_mix->type, head_v_dim),\n            nb1_qkv,\n            nb1_qkv * n_seq_tokens,\n            ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));\n\n    cb(q_conv, \"q_conv\", il);\n    cb(k_conv, \"k_conv\", il);\n    cb(v_conv, \"v_conv\", il);\n\n    const float eps_norm = hparams.f_norm_rms_eps;\n\n    q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm);\n    k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm);\n\n    //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);\n    //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // if head keys and value keys are different, repeat to force tensors into matching shapes\n    // TODO: avoid repeats for fused GDN, needs broadcast configuration for GDN op [TAG_GGML_GDN_BCAST]\n    if (num_k_heads != num_v_heads) {\n        GGML_ASSERT(num_v_heads % num_k_heads == 0);\n        int64_t repeat_factor = num_v_heads / num_k_heads;\n\n        // repeat interleave: reshape to (repeat part, 1, remaining part), do repeat, then reshape back\n        ggml_tensor * q_reshaped = ggml_reshape_3d(ctx0, q_conv, head_k_dim, 1, num_k_heads * n_seq_tokens * n_seqs);\n        ggml_tensor * k_reshaped = ggml_reshape_3d(ctx0, k_conv, head_k_dim, 1, num_k_heads * n_seq_tokens * n_seqs);\n\n        // Repeat along the third dimension (the new dimension with size 1)\n        ggml_tensor * q_repeated =\n            ggml_repeat_4d(ctx0, q_reshaped, head_k_dim, repeat_factor, num_k_heads * n_seq_tokens * n_seqs, 1);\n        ggml_tensor * k_repeated =\n            ggml_repeat_4d(ctx0, k_reshaped, head_k_dim, repeat_factor, num_k_heads * n_seq_tokens * n_seqs, 1);\n\n        // Reshape back to merge the head and repeat dimensions\n        // From [head_dim, num_k_heads, repeat_factor, n_seq_tokens * n_seqs]\n        // Back to [head_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs]\n        q_conv = ggml_reshape_4d(ctx0, q_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);\n        k_conv = ggml_reshape_4d(ctx0, k_repeated, head_k_dim, num_k_heads * repeat_factor, n_seq_tokens, n_seqs);\n    }\n\n    cb(q_conv, \"q_conv_predelta\", il);\n    cb(k_conv, \"k_conv_predelta\", il);\n    cb(v_conv, \"v_conv_predelta\", il);\n\n    auto attn_out = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il);\n\n    ggml_tensor * output    = attn_out.first;\n    ggml_tensor * new_state = attn_out.second;\n    cb(output, \"attn_output\", il);\n    cb(new_state, \"new_state\", il);\n\n    // Update the recurrent states\n    ggml_build_forward_expand(gf,\n            ggml_cpy(ctx0, new_state,\n                ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,\n                    kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));\n\n    // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]\n    ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);\n\n    // Apply gated normalization: self.norm(core_attn_out, z)\n    ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);\n\n    // Final reshape: [head_dim, n_heads, n_tokens, n_seqs] -> [n_tokens, n_seqs, n_heads * head_dim]\n    ggml_tensor * final_output = ggml_reshape_3d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);\n    cb(final_output, \"final_output\", il);\n\n    // Output projection\n    cur = build_lora_mm(model.layers[il].ssm_out, final_output);\n    cb(cur, \"linear_attn_out\", il);\n\n    // Reshape back to original dimensions\n    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_seq_tokens * n_seqs);\n\n    return cur;\n}\n\nggml_tensor * llm_build_qwen3next::build_layer_ffn(ggml_tensor * cur, const int il) {\n    // Check if this is an MoE layer\n    if (model.layers[il].ffn_gate_inp != nullptr) {\n        // MoE branch\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                model.layers[il].ffn_gate_inp,\n                model.layers[il].ffn_up_exps,\n                model.layers[il].ffn_gate_exps,\n                model.layers[il].ffn_down_exps,\n                nullptr,\n                n_expert, n_expert_used,\n                LLM_FFN_SILU, true,\n                hparams.expert_weights_scale,\n                LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX, il,\n                nullptr, model.layers[il].ffn_gate_up_exps);\n        cb(moe_out, \"ffn_moe_out\", il);\n\n        // Add shared experts if present - following Qwen3Next reference implementation\n        if (model.layers[il].ffn_up_shexp != nullptr) {\n            ggml_tensor * ffn_shexp =\n                build_ffn(cur,\n                    model.layers[il].ffn_up_shexp,   NULL, NULL,\n                    model.layers[il].ffn_gate_shexp, NULL, NULL,\n                    model.layers[il].ffn_down_shexp, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(ffn_shexp, \"ffn_shexp\", il);\n\n            // Apply shared expert gating as in the reference implementation\n            // The shared expert has its own gate that is sigmoided\n            // Note: ffn_gate_inp_shexp is the shared expert gate (outputs 1 value per token)\n            ggml_tensor * shared_gate = build_lora_mm(model.layers[il].ffn_gate_inp_shexp, cur);\n            cb(shared_gate, \"shared_expert_gate\", il);\n\n            shared_gate = ggml_sigmoid(ctx0, shared_gate);\n            cb(shared_gate, \"shared_expert_gate_sigmoid\", il);\n\n            ffn_shexp = ggml_mul(ctx0, ffn_shexp, shared_gate);\n            cb(ffn_shexp, \"ffn_shexp_gated\", il);\n\n            cur = ggml_add(ctx0, moe_out, ffn_shexp);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            cur = moe_out;\n        }\n    } else {\n        // Dense FFN branch (not currently used I believe)\n        cur = build_ffn(cur,\n            model.layers[il].ffn_up, NULL, NULL,\n            model.layers[il].ffn_gate, NULL, NULL,\n            model.layers[il].ffn_down, NULL, NULL,\n            NULL,\n            LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n    }\n    return cur;\n}\n"
  },
  {
    "path": "examples/talk-llama/models/qwen3vl-moe.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen3vlmoe::llm_build_qwen3vlmoe(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const size_t n_deepstack_layers = hparams.n_deepstack_layers;\n\n    const int64_t n_embd      = hparams.n_embd;\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_multi(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_multi(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n        cb(moe_out, \"ffn_moe_out\", il);\n        cur = moe_out;\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        if (il < (int) n_deepstack_layers) {\n            ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float));\n            cur = ggml_add(ctx0, cur, ds);\n            cb(cur, \"deepstack_out\", il);\n        }\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n"
  },
  {
    "path": "examples/talk-llama/models/qwen3vl.cpp",
    "content": "#include \"models.h\"\n\nllm_build_qwen3vl::llm_build_qwen3vl(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const size_t n_deepstack_layers = hparams.n_deepstack_layers;\n\n    const int64_t n_embd      = hparams.n_embd;\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    int sections[4];\n    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_multi(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_multi(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        if (il < (int) n_deepstack_layers) {\n            ggml_tensor * ds = ggml_view_2d(ctx0, res->t_inp_embd, n_embd, n_tokens, res->t_inp_embd->nb[1], (il + 1) * n_embd * sizeof(float));\n            cur = ggml_add(ctx0, cur, ds);\n            cb(cur, \"deepstack_out\", il);\n        }\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/refact.cpp",
    "content": "#include \"models.h\"\n\nllm_build_refact::llm_build_refact(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rnd1.cpp",
    "content": "#include \"models.h\"\n\n// RND1 is a Qwen3Moe AR model converted to diffusion model.\nllm_build_rnd1::llm_build_rnd1(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    // Non-causal attention for diffusion\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);\n            cb(Qcur, \"Qcur_normed\", il);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);\n            cb(Kcur, \"Kcur_normed\", il);\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * moe_out =\n            build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, true,\n                    hparams.expert_weights_scale,\n                    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,\n                    il);\n        cb(moe_out, \"ffn_moe_out\", il);\n        cur = moe_out;\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rwkv6-base.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_rwkv6_base::llm_build_rwkv6_base(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params),\n    model(model) {}\n\nggml_tensor * llm_build_rwkv6_base::build_rwkv6_channel_mix(const llama_layer * layer,\n                                                            ggml_tensor *       cur,\n                                                            ggml_tensor *       x_prev,\n                                                            llm_arch            arch) const {\n    ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur);\n    switch (arch) {\n        case LLM_ARCH_RWKV6:\n            {\n                ggml_tensor * xk = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_k), cur);\n                ggml_tensor * xr = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_r), cur);\n\n                ggml_tensor * r = ggml_sigmoid(ctx0, build_lora_mm(layer->channel_mix_receptance, xr));\n                ggml_tensor * k = ggml_sqr(ctx0, ggml_relu(ctx0, build_lora_mm(layer->channel_mix_key, xk)));\n                cur             = ggml_mul(ctx0, r, build_lora_mm(layer->channel_mix_value, k));\n            }\n            break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n    return cur;\n}\n\nggml_tensor * llm_build_rwkv6_base::build_rwkv6_time_mix(llm_graph_input_rs * inp,\n                                                         ggml_tensor *        cur,\n                                                         ggml_tensor *        x_prev,\n                                                         const llama_ubatch & ubatch,\n                                                         int                  il) const {\n    const auto * mctx_cur = static_cast<const llama_memory_recurrent_context *>(mctx);\n\n    const auto n_tokens     = ubatch.n_tokens;\n    const auto n_seqs       = ubatch.n_seqs;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n    const auto n_embd       = hparams.n_embd;\n    const auto head_size    = hparams.wkv_head_size;\n    const auto n_head       = n_embd / head_size;\n    const auto n_head_kv    = hparams.n_head_kv(il);\n\n    const auto kv_head = mctx_cur->get_head();\n\n    const auto & layer = model.layers[il];\n\n    bool is_qrwkv = layer.time_mix_first == nullptr;\n\n    ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur);\n\n    sx  = ggml_reshape_2d(ctx0, sx, n_embd, n_tokens);\n    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n\n    ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_x), cur);\n\n    xxx = ggml_reshape_4d(ctx0, ggml_tanh(ctx0, ggml_mul_mat(ctx0, layer.time_mix_w1, xxx)),\n                          layer.time_mix_w1->ne[1] / 5, 1, 5, n_tokens);\n\n    xxx = ggml_cont(ctx0, ggml_permute(ctx0, xxx, 0, 1, 3, 2));\n\n    xxx = ggml_mul_mat(\n        ctx0, ggml_reshape_4d(ctx0, layer.time_mix_w2, layer.time_mix_w2->ne[0], layer.time_mix_w2->ne[1], 1, 5), xxx);\n\n    ggml_tensor *xw, *xk, *xv, *xr, *xg;\n    if (layer.time_mix_lerp_fused) {\n        // fusing these weights makes some performance improvement\n        sx  = ggml_reshape_3d(ctx0, sx, n_embd, 1, n_tokens);\n        cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);\n        xxx = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xxx, layer.time_mix_lerp_fused), sx), cur);\n        xw  = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0);\n        xk  = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float));\n        xv  = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float));\n        xr  = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float));\n        xg  = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float));\n    } else {\n        // for backward compatibility\n        xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0);\n        xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float));\n        xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float));\n        xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float));\n        xg = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float));\n\n        xw = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xw, layer.time_mix_lerp_w), sx), cur);\n        xk = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xk, layer.time_mix_lerp_k), sx), cur);\n        xv = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xv, layer.time_mix_lerp_v), sx), cur);\n        xr = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xr, layer.time_mix_lerp_r), sx), cur);\n        xg = ggml_add(ctx0, ggml_mul(ctx0, ggml_add(ctx0, xg, layer.time_mix_lerp_g), sx), cur);\n    }\n    ggml_tensor * r = build_lora_mm(layer.time_mix_receptance, xr);\n    ggml_tensor * k = build_lora_mm(layer.time_mix_key, xk);\n    ggml_tensor * v = build_lora_mm(layer.time_mix_value, xv);\n    if (layer.time_mix_receptance_b) {\n        r = ggml_add(ctx0, r, layer.time_mix_receptance_b);\n    }\n    if (layer.time_mix_key_b) {\n        k = ggml_add(ctx0, k, layer.time_mix_key_b);\n    }\n    if (layer.time_mix_value_b) {\n        v = ggml_add(ctx0, v, layer.time_mix_value_b);\n    }\n    ggml_tensor * g = build_lora_mm(layer.time_mix_gate, xg);\n    if (is_qrwkv) {\n        g = ggml_sigmoid(ctx0, g);\n    } else {\n        g = ggml_silu(ctx0, g);\n    }\n    if (n_head_kv != 0 && n_head_kv != n_head) {\n        GGML_ASSERT(n_head % n_head_kv == 0);\n        k                 = ggml_reshape_4d(ctx0, k, head_size, 1, n_head_kv, n_tokens);\n        v                 = ggml_reshape_4d(ctx0, v, head_size, 1, n_head_kv, n_tokens);\n        ggml_tensor * tmp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_size, n_head / n_head_kv, n_head_kv, n_tokens);\n        k                 = ggml_repeat(ctx0, k, tmp);\n        v                 = ggml_repeat(ctx0, v, tmp);\n    }\n    k = ggml_reshape_3d(ctx0, k, head_size, n_head, n_tokens);\n    v = ggml_reshape_3d(ctx0, v, head_size, n_head, n_tokens);\n    r = ggml_reshape_3d(ctx0, r, head_size, n_head, n_tokens);\n\n    ggml_tensor * w =\n        ggml_mul_mat(ctx0, layer.time_mix_decay_w2, ggml_tanh(ctx0, ggml_mul_mat(ctx0, layer.time_mix_decay_w1, xw)));\n\n    w = ggml_add(ctx0, w, layer.time_mix_decay);\n    w = ggml_exp(ctx0, ggml_neg(ctx0, ggml_exp(ctx0, w)));\n    w = ggml_reshape_3d(ctx0, w, head_size, n_head, n_tokens);\n\n    if (is_qrwkv) {\n        // k = k * (1 - w)\n        k = ggml_sub(ctx0, k, ggml_mul(ctx0, k, w));\n    }\n    ggml_tensor * wkv_state = build_rs(inp, mctx_cur->get_s_l(il), hparams.n_embd_s(), n_seqs);\n\n    ggml_tensor * wkv_output;\n    if (is_qrwkv) {\n        wkv_output = ggml_gated_linear_attn(ctx0, k, v, r, w, wkv_state, pow(head_size, -0.5f));\n    } else {\n        wkv_output = ggml_rwkv_wkv6(ctx0, k, v, r, layer.time_mix_first, w, wkv_state);\n    }\n    cur       = ggml_view_1d(ctx0, wkv_output, n_embd * n_tokens, 0);\n    wkv_state = ggml_view_1d(ctx0, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float));\n\n    ggml_build_forward_expand(\n        gf, ggml_cpy(ctx0, wkv_state,\n                     ggml_view_1d(ctx0, mctx_cur->get_s_l(il), hparams.n_embd_s() * n_seqs,\n                                  hparams.n_embd_s() * kv_head * ggml_element_size(mctx_cur->get_s_l(il)))));\n\n    if (!is_qrwkv) {\n        // group norm with head_count groups\n        cur = ggml_reshape_3d(ctx0, cur, n_embd / n_head, n_head, n_tokens);\n        cur = ggml_norm(ctx0, cur, 64e-5f);\n\n        // Convert back to regular vectors.\n        cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n        cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.time_mix_ln), layer.time_mix_ln_b);\n    } else {\n        cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n    }\n    cur = ggml_mul(ctx0, cur, g);\n    cur = build_lora_mm(layer.time_mix_output, cur);\n\n    return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rwkv6.cpp",
    "content": "#include \"models.h\"\n\nllm_build_rwkv6::llm_build_rwkv6(const llama_model & model, const llm_graph_params & params) :\n    llm_build_rwkv6_base(model, params) {\n    GGML_ASSERT(hparams.token_shift_count == 2);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n    inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);\n\n    auto * rs_inp = build_rs_inp();\n\n    const auto n_embd       = hparams.n_embd;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n    const auto n_seqs       = ubatch.n_seqs;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const llama_layer * layer = &model.layers[il];\n        inpL                      = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);\n\n        ggml_tensor * token_shift = build_rwkv_token_shift_load(rs_inp, ubatch, il);\n\n        ggml_tensor * att_shift =\n            ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0);\n        ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1],\n                                               token_shift->nb[2], n_embd * ggml_element_size(token_shift));\n\n        ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il);\n        cb(att_norm, \"attn_norm\", il);\n\n        ggml_tensor * x_prev = ggml_concat(\n            ctx0, att_shift,\n            ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), 1);\n\n        cur = build_rwkv6_time_mix(rs_inp, att_norm, x_prev, ubatch, il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il);\n        cb(ffn_norm, \"ffn_norm\", il);\n\n        x_prev = ggml_concat(\n            ctx0, ffn_shift,\n            ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), 1);\n\n        token_shift = ggml_concat(ctx0,\n                                  ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2],\n                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(att_norm)),\n                                  ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2],\n                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(ffn_norm)),\n                                  1);\n        ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il));\n\n        ffn_inp  = ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens);\n        ffn_norm = ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens);\n        x_prev   = ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens);\n        cur      = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            ffn_inp  = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n            ffn_norm = ggml_get_rows(ctx0, ffn_norm, inp_out_ids);\n            x_prev   = ggml_get_rows(ctx0, x_prev, inp_out_ids);\n            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);\n        }\n        cur = build_rwkv6_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV6);\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        if (hparams.rescale_every_n_layers != 0 && (il + 1) % hparams.rescale_every_n_layers == 0) {\n            cur = ggml_scale(ctx0, cur, 0.5F);\n        }\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rwkv6qwen2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_rwkv6qwen2::llm_build_rwkv6qwen2(const llama_model & model, const llm_graph_params & params) : llm_build_rwkv6_base(model, params) {\n    GGML_ASSERT(n_embd == hparams.n_embd_r());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    auto * rs_inp = build_rs_inp();\n\n    const auto n_embd = hparams.n_embd;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n    const auto n_seqs = ubatch.n_seqs;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const llama_layer * layer = &model.layers[il];\n        inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);\n\n        ggml_tensor * token_shift = build_rwkv_token_shift_load(rs_inp, ubatch, il);\n\n        ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM_RMS, il);\n        cb(att_norm, \"attn_norm\", il);\n\n        ggml_tensor * x_prev = ggml_concat(\n                ctx0,\n                token_shift,\n                ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0),\n                1\n                );\n\n        cur = build_rwkv6_time_mix(rs_inp, att_norm, x_prev, ubatch, il);\n\n        token_shift = ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2], (n_seq_tokens-1)*n_embd*ggml_element_size(att_norm));\n        ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il));\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur     = ggml_reshape_2d(ctx0, cur,     n_embd, n_tokens);\n        ffn_inp = ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur     = ggml_get_rows(ctx0, cur,     inp_out_ids);\n            ffn_inp = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n        }\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n\n    cur = inpL;\n    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rwkv7-base.cpp",
    "content": "#include \"models.h\"\n\n#include \"llama-memory-recurrent.h\"\n\nllm_build_rwkv7_base::llm_build_rwkv7_base(const llama_model & model, const llm_graph_params & params) :\n    llm_graph_context(params),\n    model(model) {}\n\nggml_tensor * llm_build_rwkv7_base::build_rwkv7_channel_mix(const llama_layer * layer,\n                                                            ggml_tensor *       cur,\n                                                            ggml_tensor *       x_prev,\n                                                            llm_arch            arch) const {\n    ggml_tensor * sx = ggml_sub(ctx0, x_prev, cur);\n    switch (arch) {\n        case LLM_ARCH_RWKV7:\n            {\n                ggml_tensor * xk = ggml_add(ctx0, ggml_mul(ctx0, sx, layer->channel_mix_lerp_k), cur);\n\n                ggml_tensor * k = ggml_sqr(ctx0, ggml_relu(ctx0, build_lora_mm(layer->channel_mix_key, xk)));\n\n                cur = build_lora_mm(layer->channel_mix_value, k);\n            }\n            break;\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n    return cur;\n}\n\nggml_tensor * llm_build_rwkv7_base::build_rwkv7_time_mix(llm_graph_input_rs * inp,\n                                                         ggml_tensor *        cur,\n                                                         ggml_tensor *        x_prev,\n                                                         ggml_tensor *&       first_layer_value,\n                                                         const llama_ubatch & ubatch,\n                                                         int                  il) const {\n    const auto * mctx_cur = static_cast<const llama_memory_recurrent_context *>(mctx);\n\n    const auto n_tokens     = ubatch.n_tokens;\n    const auto n_seqs       = ubatch.n_seqs;\n    const auto n_embd       = hparams.n_embd;\n    const auto head_size    = hparams.wkv_head_size;\n    const auto head_count   = n_embd / head_size;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n\n    const auto kv_head = mctx_cur->get_head();\n\n    const auto & layer = model.layers[il];\n\n    bool has_gating = layer.time_mix_g1 && layer.time_mix_g2;\n\n    ggml_tensor * sx    = ggml_sub(ctx0, x_prev, cur);\n    ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_embd, n_seq_tokens, n_seqs, has_gating ? 6 : 5);\n    sx                  = ggml_repeat(ctx0, sx, dummy);\n\n    ggml_tensor * xxx = ggml_add(ctx0, ggml_mul(ctx0, sx, layer.time_mix_lerp_fused), cur);\n\n    ggml_tensor * xr = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], 0);\n    ggml_tensor * xw = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * sizeof(float));\n    ggml_tensor * xk = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 2 * sizeof(float));\n    ggml_tensor * xv = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 3 * sizeof(float));\n    ggml_tensor * xa = ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 4 * sizeof(float));\n    ggml_tensor * xg =\n        has_gating ? ggml_view_2d(ctx0, xxx, n_embd, n_tokens, xxx->nb[1], n_embd * n_tokens * 5 * sizeof(float)) :\n                     nullptr;\n\n    ggml_tensor * r = build_lora_mm(layer.time_mix_receptance, xr);\n    ggml_tensor * w = ggml_add(\n        ctx0, ggml_mul_mat(ctx0, layer.time_mix_w2, ggml_tanh(ctx0, ggml_mul_mat(ctx0, layer.time_mix_w1, xw))),\n        layer.time_mix_w0);\n    w = ggml_exp(ctx0, ggml_scale(ctx0, ggml_sigmoid(ctx0, w), -0.606531));\n\n    ggml_tensor * k = build_lora_mm(layer.time_mix_key, xk);\n    ggml_tensor * v = build_lora_mm(layer.time_mix_value, xv);\n    if (first_layer_value == nullptr) {\n        first_layer_value = v;\n    } else {\n        // Add the first layer value as a residual connection.\n        v = ggml_add(ctx0, v,\n                     ggml_mul(ctx0, ggml_sub(ctx0, first_layer_value, v),\n                              ggml_sigmoid(ctx0, ggml_add(ctx0,\n                                                          ggml_mul_mat(ctx0, layer.time_mix_v2,\n                                                                       ggml_mul_mat(ctx0, layer.time_mix_v1, xv)),\n                                                          layer.time_mix_v0))));\n    }\n    ggml_tensor * g = nullptr;\n    if (layer.time_mix_g1 && layer.time_mix_g2) {\n        g = ggml_mul_mat(ctx0, layer.time_mix_g2, ggml_sigmoid(ctx0, ggml_mul_mat(ctx0, layer.time_mix_g1, xg)));\n    }\n    ggml_tensor * a = ggml_sigmoid(\n        ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, layer.time_mix_a2, ggml_mul_mat(ctx0, layer.time_mix_a1, xa)),\n                       layer.time_mix_a0));\n\n    ggml_tensor * kk = ggml_reshape_3d(ctx0, ggml_mul(ctx0, k, layer.time_mix_k_k), head_size, head_count, n_tokens);\n    kk               = ggml_l2_norm(ctx0, kk, 1e-12);\n\n    ggml_tensor * ka = ggml_mul(ctx0, k, layer.time_mix_k_a);\n    k                = ggml_add(ctx0, k, ggml_sub(ctx0, ggml_mul(ctx0, a, ka), ka));\n\n    r = ggml_reshape_3d(ctx0, r, head_size, head_count, n_tokens);\n    w = ggml_reshape_3d(ctx0, w, head_size, head_count, n_tokens);\n    k = ggml_reshape_3d(ctx0, k, head_size, head_count, n_tokens);\n    v = ggml_reshape_3d(ctx0, v, head_size, head_count, n_tokens);\n    a = ggml_reshape_3d(ctx0, a, head_size, head_count, n_tokens);\n\n    ggml_tensor * wkv_state = build_rs(inp, mctx_cur->get_s_l(il), hparams.n_embd_s(), n_seqs);\n\n    ggml_tensor * wkv_output = ggml_rwkv_wkv7(ctx0, r, w, k, v, ggml_neg(ctx0, kk), ggml_mul(ctx0, kk, a), wkv_state);\n    cur                      = ggml_view_1d(ctx0, wkv_output, n_embd * n_tokens, 0);\n    wkv_state = ggml_view_1d(ctx0, wkv_output, n_embd * head_size * n_seqs, n_embd * n_tokens * sizeof(float));\n\n    ggml_build_forward_expand(\n        gf, ggml_cpy(ctx0, wkv_state,\n                     ggml_view_1d(ctx0, mctx_cur->get_s_l(il), hparams.n_embd_s() * n_seqs,\n                                  hparams.n_embd_s() * kv_head * ggml_element_size(mctx_cur->get_s_l(il)))));\n\n    if (layer.time_mix_ln && layer.time_mix_ln_b) {\n        // group norm with head_count groups\n        cur = ggml_reshape_3d(ctx0, cur, n_embd / head_count, head_count, n_tokens);\n        cur = ggml_norm(ctx0, cur, 64e-5f);\n\n        // Convert back to regular vectors.\n        cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n        cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.time_mix_ln), layer.time_mix_ln_b);\n    } else {\n        cur = ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);\n    }\n    ggml_tensor * rk = ggml_sum_rows(\n        ctx0, ggml_mul(ctx0, ggml_mul(ctx0, k, r), ggml_reshape_2d(ctx0, layer.time_mix_r_k, head_size, head_count)));\n    cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, ggml_mul(ctx0, v, rk), n_embd, n_tokens));\n\n    if (has_gating) {\n        cur = ggml_mul(ctx0, cur, g);\n    }\n    cur = build_lora_mm(layer.time_mix_output, cur);\n\n    return ggml_reshape_3d(ctx0, cur, n_embd, n_seq_tokens, n_seqs);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/rwkv7.cpp",
    "content": "#include \"models.h\"\n\nllm_build_rwkv7::llm_build_rwkv7(const llama_model & model, const llm_graph_params & params) :\n    llm_build_rwkv7_base(model, params) {\n    GGML_ASSERT(hparams.token_shift_count == 2);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n    ggml_tensor * v_first = nullptr;\n\n    inpL = build_inp_embd(model.tok_embd);\n    inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);\n\n    auto * rs_inp = build_rs_inp();\n\n    const auto n_embd       = hparams.n_embd;\n    const auto n_seq_tokens = ubatch.n_seq_tokens;\n    const auto n_seqs       = ubatch.n_seqs;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const llama_layer * layer = &model.layers[il];\n        inpL                      = ggml_reshape_3d(ctx0, inpL, n_embd, n_seq_tokens, n_seqs);\n\n        ggml_tensor * token_shift = build_rwkv_token_shift_load(rs_inp, ubatch, il);\n\n        ggml_tensor * att_shift =\n            ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1], token_shift->nb[2], 0);\n        ggml_tensor * ffn_shift = ggml_view_3d(ctx0, token_shift, n_embd, 1, n_seqs, token_shift->nb[1],\n                                               token_shift->nb[2], n_embd * ggml_element_size(token_shift));\n\n        ggml_tensor * att_norm = build_norm(inpL, layer->attn_norm, layer->attn_norm_b, LLM_NORM, il);\n        cb(att_norm, \"attn_norm\", il);\n\n        ggml_tensor * x_prev = ggml_concat(\n            ctx0, att_shift,\n            ggml_view_3d(ctx0, att_norm, n_embd, n_seq_tokens - 1, n_seqs, att_norm->nb[1], att_norm->nb[2], 0), 1);\n\n        cur = build_rwkv7_time_mix(rs_inp, att_norm, x_prev, v_first, ubatch, il);\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        ggml_tensor * ffn_norm = build_norm(ffn_inp, layer->attn_norm_2, layer->attn_norm_2_b, LLM_NORM, il);\n        cb(ffn_norm, \"ffn_norm\", il);\n\n        x_prev = ggml_concat(\n            ctx0, ffn_shift,\n            ggml_view_3d(ctx0, ffn_norm, n_embd, n_seq_tokens - 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2], 0), 1);\n\n        token_shift = ggml_concat(ctx0,\n                                  ggml_view_3d(ctx0, att_norm, n_embd, 1, n_seqs, att_norm->nb[1], att_norm->nb[2],\n                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(att_norm)),\n                                  ggml_view_3d(ctx0, ffn_norm, n_embd, 1, n_seqs, ffn_norm->nb[1], ffn_norm->nb[2],\n                                               (n_seq_tokens - 1) * n_embd * ggml_element_size(ffn_norm)),\n                                  1);\n        ggml_build_forward_expand(gf, build_rwkv_token_shift_store(token_shift, ubatch, il));\n\n        ffn_inp  = ggml_reshape_2d(ctx0, ffn_inp, n_embd, n_tokens);\n        ffn_norm = ggml_reshape_2d(ctx0, ffn_norm, n_embd, n_tokens);\n        x_prev   = ggml_reshape_2d(ctx0, x_prev, n_embd, n_tokens);\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            ffn_inp  = ggml_get_rows(ctx0, ffn_inp, inp_out_ids);\n            ffn_norm = ggml_get_rows(ctx0, ffn_norm, inp_out_ids);\n            x_prev   = ggml_get_rows(ctx0, x_prev, inp_out_ids);\n        }\n        cur = build_rwkv7_channel_mix(layer, ffn_norm, x_prev, LLM_ARCH_RWKV7);\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cur = build_norm(cur, model.output_norm, model.output_norm_b, LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/seed-oss.cpp",
    "content": "#include \"models.h\"\n\nllm_build_seed_oss::llm_build_seed_oss(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        cur = build_norm(ffn_inp,\n                model.layers[il].attn_post_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_post_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   NULL, NULL,\n                model.layers[il].ffn_gate, NULL, NULL,\n                model.layers[il].ffn_down, NULL, NULL,\n                NULL,\n                LLM_FFN_SILU, LLM_FFN_PAR, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/smallthinker.cpp",
    "content": "#include \"models.h\"\n\ntemplate <bool iswa>\nllm_build_smallthinker<iswa>::llm_build_smallthinker(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params){\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;\n    inp_attn_type * inp_attn = nullptr;\n\n    if constexpr (iswa) {\n        inp_attn = build_attn_inp_kv_iswa();\n    } else {\n        inp_attn = build_attn_inp_kv();\n    }\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        const float freq_base_l  = model.get_rope_freq_base (cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        ggml_tensor * inpSA  = inpL;\n\n        // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous\n        const bool use_rope = hparams.n_no_rope_layer_step == n_layer ||\n                              il % hparams.n_no_rope_layer_step != 0;\n\n        ggml_tensor * probs = build_lora_mm(model.layers[il].ffn_gate_inp, inpL);  // [n_expert, n_tokens]\n        cb(probs, \"ffn_moe_logits\", il);\n\n        // norm\n        cur = build_norm(inpL,model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self_attention\n        {\n            // compute Q and K and RoPE them\n            struct ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            struct ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            struct ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (use_rope) {\n                Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n\n                Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                                    ext_factor, attn_factor, beta_fast, beta_slow);\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur = ggml_get_rows(ctx0, cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n            probs = ggml_get_rows(ctx0, probs, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // MoE branch\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        ggml_tensor * ffn_out =\n            build_moe_ffn(cur,\n                    nullptr,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    nullptr,\n                    n_expert, n_expert_used,\n                    LLM_FFN_RELU, true,\n                    hparams.expert_weights_scale,\n                    static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),\n                    il, probs);\n\n        cb(ffn_out, \"ffn_out\", il);\n        cur = ffn_out;\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n\n// Explicit template instantiations\ntemplate struct llm_build_smallthinker<false>;\ntemplate struct llm_build_smallthinker<true>;\n"
  },
  {
    "path": "examples/talk-llama/models/smollm3.cpp",
    "content": "#include \"models.h\"\n\nllm_build_smollm3::llm_build_smollm3(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f/sqrtf(float(n_embd_head)) : hparams.f_attention_scale;\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        const bool use_rope = (il + 1) % hparams.n_no_rope_layer_step != 0;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (use_rope) {\n                Qcur = ggml_rope_ext(\n                        ctx0, Qcur, inp_pos, nullptr,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n\n                Kcur = ggml_rope_ext(\n                        ctx0, Kcur, inp_pos, nullptr,\n                        n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                        ext_factor, attn_factor, beta_fast, beta_slow\n                        );\n            }\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(cur, \"attn_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/stablelm.cpp",
    "content": "#include \"models.h\"\n\nllm_build_stablelm::llm_build_stablelm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        ggml_tensor * inpSA = cur;\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            if (model.layers[il].attn_q_norm) {\n                Qcur = build_norm(Qcur,\n                        model.layers[il].attn_q_norm,\n                        NULL,\n                        LLM_NORM, il);\n                cb(Qcur, \"Qcur\", il);\n            }\n            if (model.layers[il].attn_k_norm) {\n                Kcur = build_norm(Kcur,\n                        model.layers[il].attn_k_norm,\n                        NULL,\n                        LLM_NORM, il);\n                cb(Kcur, \"Kcur\", il);\n            }\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpL  = ggml_get_rows(ctx0,  inpL, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            if (model.layers[il].ffn_norm) {\n                cur = build_norm(ffn_inp,\n                        model.layers[il].ffn_norm,\n                        model.layers[il].ffn_norm_b,\n                        LLM_NORM, il);\n                cb(cur, \"ffn_norm\", il);\n            } else {\n                // parallel residual\n                cur = inpSA;\n            }\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/starcoder.cpp",
    "content": "#include \"models.h\"\n\nllm_build_starcoder::llm_build_starcoder(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);\n    cb(pos, \"pos_embd\", -1);\n\n    inpL = ggml_add(ctx0, inpL, pos);\n    cb(inpL, \"inpL\", -1);\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm,\n                model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            cur = build_lora_mm(model.layers[il].wqkv, cur);\n            cb(cur, \"wqkv\", il);\n\n            cur = ggml_add(ctx0, cur, model.layers[il].bqkv);\n            cb(cur, \"bqkv\", il);\n\n            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head,    n_tokens, n_embd_head*sizeof(float), cur->nb[1], 0*sizeof(float)*(n_embd));\n            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd));\n            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, n_embd_head, n_head_kv, n_tokens, n_embd_head*sizeof(float), cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa));\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);\n            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);\n        }\n        // add the input\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // FF\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm,\n                    model.layers[il].ffn_norm_b,\n                    LLM_NORM, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                    NULL,                      NULL,                        NULL,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                    NULL,\n                    LLM_FFN_GELU, LLM_FFN_SEQ, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = build_norm(inpL,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/starcoder2.cpp",
    "content": "#include \"models.h\"\n\nllm_build_starcoder2::llm_build_starcoder2(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, model.layers[il].attn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            // compute Q and K and RoPE them\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n            if (model.layers[il].bq) {\n                Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);\n                cb(Qcur, \"Qcur\", il);\n            }\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n            if (model.layers[il].bk) {\n                Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);\n                cb(Kcur, \"Kcur\", il);\n            }\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n            if (model.layers[il].bv) {\n                Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);\n                cb(Vcur, \"Vcur\", il);\n            }\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n\n        cur = build_norm(ffn_inp,\n                model.layers[il].ffn_norm, model.layers[il].ffn_norm_b,\n                LLM_NORM, il);\n        cb(cur, \"ffn_norm\", il);\n\n        cur = build_ffn(cur,\n                model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,\n                NULL,                      NULL,                        NULL,\n                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,\n                NULL,\n                LLM_FFN_GELU, LLM_FFN_SEQ, il);\n        cb(cur, \"ffn_out\", il);\n\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur,\n            model.output_norm, model.output_norm_b,\n            LLM_NORM, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/step35-iswa.cpp",
    "content": "#include \"models.h\"\n\nllm_build_step35_iswa::llm_build_step35_iswa(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n    ggml_tensor * inp_pos     = build_inp_pos();\n    auto        * inp_attn    = build_attn_inp_kv_iswa();\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        const uint32_t n_head_l    = hparams.n_head(il);\n        const uint32_t n_head_kv_l = hparams.n_head_kv(il);\n\n        const float freq_base_l  = model.get_rope_freq_base(cparams, il);\n        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);\n\n        cur = inpL;\n\n        // dump pre-attn RMSNorm input to pinpoint layer boundary issues\n        cb(cur, \"attn_norm_in\", il);\n\n        // self-attention\n        {\n            cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);\n            cb(cur, \"attn_norm\", il);\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head_l,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);\n\n            // Q/K per-head RMSNorm (Step35 q_norm / k_norm)\n            if (model.layers[il].attn_q_norm) {\n                Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);\n                cb(Qcur, \"Qcur_normed\", il);\n            }\n            if (model.layers[il].attn_k_norm) {\n                Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);\n                cb(Kcur, \"Kcur_normed\", il);\n            }\n\n            // RoPE (partial rotary factors per layer)\n            const bool is_swa = hparams.is_swa(il);\n            ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il);\n            const int64_t n_rot_l = hparams.n_rot(il);\n            Qcur = ggml_rope_ext(\n                ctx0, Qcur, inp_pos, rope_factors,\n                n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n            );\n            Kcur = ggml_rope_ext(\n                ctx0, Kcur, inp_pos, rope_factors,\n                n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,\n                ext_factor, attn_factor, beta_fast, beta_slow\n            );\n            cb(Qcur, \"Qcur_pos\", il);\n            cb(Kcur, \"Kcur_pos\", il);\n\n            const float kq_scale = 1.0f / sqrtf(float(n_embd_head_k));\n            ggml_tensor * attn_out = build_attn(inp_attn,\n                    nullptr, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);\n            cb(attn_out, \"attn_out\", il);\n            // head-wise attention gate: sigmoid(g_proj(x)) in torch\n            if (model.layers[il].wqkv_gate) {\n                ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur); // [n_head_l, n_tokens]\n                cb(gate, \"attn_gate\", il);\n\n                gate = ggml_sigmoid(ctx0, gate);\n                cb(gate, \"attn_gate_sigmoid\", il);\n\n                // reshape + broadcast to [n_embd_head_v, n_head_l, n_tokens]\n                ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, attn_out, n_embd_head_v, n_head_l, n_tokens);\n                ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate,       1,          n_head_l, n_tokens);\n                cb(gate_3d, \"attn_gate_3d\", il);\n\n                attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);\n                cb(attn_3d, \"attn_gated_3d\", il);\n\n                attn_out = ggml_reshape_2d(ctx0, attn_3d, n_embd_head_v * n_head_l, n_tokens);\n                cb(attn_out, \"attn_gated\", il);\n            }\n\n            // output projection\n            cur = build_lora_mm(model.layers[il].wo, attn_out);\n            cb(cur, \"attn_proj\", il);\n        }\n\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);\n        cb(cur, \"ffn_norm\", il);\n\n        // feed-forward\n        if (model.layers[il].ffn_gate_inp == nullptr) {\n            // dense MLP\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   nullptr,\n                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, nullptr,\n                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, nullptr,\n                    nullptr,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        } else {\n            // MoE routed experts\n            ggml_tensor * moe_out = build_moe_ffn(cur,\n                    model.layers[il].ffn_gate_inp,\n                    model.layers[il].ffn_up_exps,\n                    model.layers[il].ffn_gate_exps,\n                    model.layers[il].ffn_down_exps,\n                    model.layers[il].ffn_exp_probs_b,\n                    n_expert, n_expert_used,\n                    LLM_FFN_SILU, hparams.expert_weights_norm,\n                    hparams.expert_weights_scale,\n                    (llama_expert_gating_func_type) hparams.expert_gating_func,\n                    il);\n            cb(moe_out, \"ffn_moe_out\", il);\n\n            // shared expert MLP (always added on MoE layers in Step35)\n            ggml_tensor * sh_out = build_ffn(cur,\n                    model.layers[il].ffn_up_shexp,   nullptr, nullptr,\n                    model.layers[il].ffn_gate_shexp, nullptr, nullptr,\n                    model.layers[il].ffn_down_shexp, nullptr, nullptr,\n                    nullptr,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(sh_out, \"ffn_shared_out\", il);\n\n            cur = ggml_add(ctx0, moe_out, sh_out);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        inpL = cur;\n    }\n\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    cur = build_lora_mm(model.output, cur);\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/t5-dec.cpp",
    "content": "#include \"models.h\"\n\nllm_build_t5_dec::llm_build_t5_dec(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n    //const int64_t n_embd_gqa  = hparams.n_embd_v_gqa();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * embd_enc       = build_inp_cross_embd();\n    ggml_tensor * pos_bucket_dec = build_inp_pos_bucket_dec();\n\n    const int64_t n_outputs_enc = embd_enc->ne[1];\n\n    auto * inp_attn_self  = build_attn_inp_kv();\n    auto * inp_attn_cross = build_attn_inp_cross();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    const int64_t dec_n_layer = hparams.dec_n_layer;\n\n    for (int il = 0; il < dec_n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;\n            ggml_tensor * kq_b = build_pos_bias(pos_bucket_dec, attn_rel_b);\n\n            cur = build_attn(inp_attn_self,\n                    model.layers[il].wo, model.layers[il].bo,\n                    Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);\n            cb(cur, \"kqv_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, inpSA);\n        cb(cur, \"cross_inp\", il);\n\n        ggml_tensor * inpCA = cur;\n\n        // norm\n        cur = build_norm(cur,\n                model.layers[il].attn_norm_cross, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm_cross\", il);\n\n        // cross-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_cross, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_cross, embd_enc);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_cross, embd_enc);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_outputs_enc);\n\n            cur = build_attn(inp_attn_cross,\n                    model.layers[il].wo_cross, nullptr,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f, il);\n            cb(cur, \"kqv_out\", il);\n\n            //ggml_tensor * q =                 ggml_permute(ctx0, Qcur, 0, 2, 1, 3);\n            //ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));\n\n            //ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);\n            //cb(kq, \"kq\", il);\n\n            //kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);\n            //cb(kq, \"kq_soft_max_ext\", il);\n\n            //ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));\n            //cb(v, \"v\", il);\n\n            //ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);\n            //cb(kqv, \"kqv\", il);\n\n            //ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);\n            //cb(kqv_merged, \"kqv_merged\", il);\n\n            //cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);\n            //cb(cur, \"kqv_merged_cont\", il);\n\n            //ggml_build_forward_expand(gf, cur);\n\n            //cur = build_lora_mm(model.layers[il].wo_cross, cur);\n            //cb(cur, \"kqv_out\", il);\n        }\n        if (il == dec_n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            // T5 uses relu, flan-T5 uses gelu-gated\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    model.layers[il].ffn_gate ? LLM_FFN_GELU : LLM_FFN_RELU,\n                    model.layers[il].ffn_gate ? LLM_FFN_PAR : LLM_FFN_SEQ,\n                    il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cb(cur, \"result_embd\", -1);\n\n    cur = build_norm(cur,\n            model.output_norm, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/t5-enc.cpp",
    "content": "#include \"models.h\"\n\nllm_build_t5_enc::llm_build_t5_enc(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    ggml_tensor * pos_bucket_enc = build_inp_pos_bucket_enc();\n\n    auto * inp_attn = build_attn_inp_no_cache();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        // norm\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm_enc, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq_enc, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk_enc, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv_enc, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;\n            ggml_tensor * kq_b = build_pos_bias(pos_bucket_enc, attn_rel_b);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo_enc, nullptr,\n                    Qcur, Kcur, Vcur, kq_b, nullptr, nullptr, 1.0f, il);\n            cb(cur, \"kqv_out\", il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm_enc, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            // T5 uses relu, flan-T5 uses gelu-gated\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up_enc,   NULL, NULL,\n                    model.layers[il].ffn_gate_enc, NULL, NULL,\n                    model.layers[il].ffn_down_enc, NULL, NULL,\n                    NULL,\n                    model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,\n                    model.layers[il].ffn_gate_enc ? LLM_FFN_PAR  : LLM_FFN_SEQ,\n                    il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n        cb(cur, \"ffn_out\", il);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n    cb(cur, \"result_embd\", -1);\n\n    cur = build_norm(cur,\n            model.output_norm_enc, NULL,\n            LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/wavtokenizer-dec.cpp",
    "content": "#include \"models.h\"\n\nllm_build_wavtokenizer_dec::llm_build_wavtokenizer_dec(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));\n\n    cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1);\n    cur = ggml_add(ctx0, cur, model.conv1d_b);\n\n    // posnet\n    for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) {\n        const auto & layer = model.layers[il].posnet;\n\n        inpL = cur;\n\n        switch (il) {\n            case 0:\n            case 1:\n            case 3:\n            case 4:\n                {\n                    cur = build_norm(cur,\n                            layer.norm1,\n                            layer.norm1_b,\n                            LLM_NORM_GROUP, 0);\n\n                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);\n\n                    cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1);\n                    cur = ggml_add(ctx0, cur, layer.conv1_b);\n\n                    cur = build_norm(cur,\n                            layer.norm2,\n                            layer.norm2_b,\n                            LLM_NORM_GROUP, 0);\n\n                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);\n\n                    cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1);\n                    cur = ggml_add(ctx0, cur, layer.conv2_b);\n\n                    cur = ggml_add(ctx0, cur, inpL);\n                } break;\n            case 2:\n                {\n                    cur = build_norm(cur,\n                            layer.attn_norm,\n                            layer.attn_norm_b,\n                            LLM_NORM_GROUP, 0);\n\n                    ggml_tensor * q;\n                    ggml_tensor * k;\n                    ggml_tensor * v;\n\n                    q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1);\n                    k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1);\n                    v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1);\n\n                    q = ggml_add(ctx0, q, layer.attn_q_b);\n                    k = ggml_add(ctx0, k, layer.attn_k_b);\n                    v = ggml_add(ctx0, v, layer.attn_v_b);\n\n                    q = ggml_cont(ctx0, ggml_transpose(ctx0, q));\n                    k = ggml_cont(ctx0, ggml_transpose(ctx0, k));\n\n                    ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);\n\n                    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f);\n\n                    cur = ggml_mul_mat(ctx0, kq, v);\n\n                    cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1);\n                    cur = ggml_add(ctx0, cur, layer.attn_o_b);\n\n                    cur = ggml_add(ctx0, cur, inpL);\n                } break;\n            case 5:\n                {\n                    cur = build_norm(cur,\n                            layer.norm,\n                            layer.norm_b,\n                            LLM_NORM_GROUP, 0);\n                } break;\n            default: GGML_ABORT(\"unknown posnet layer\");\n        };\n    }\n    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));\n\n    cur = build_norm(cur,\n            model.tok_norm,\n            model.tok_norm_b,\n            LLM_NORM, -1);\n\n    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));\n\n    inpL = cur;\n\n    // convnext\n    for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) {\n        const auto & layer = model.layers[il].convnext;\n\n        cur = inpL;\n\n        cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1);\n        cur = ggml_add(ctx0, cur, layer.dw_b);\n\n        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));\n\n        cur = build_norm(cur,\n                layer.norm,\n                layer.norm_b,\n                LLM_NORM, -1);\n\n        cur = build_ffn(cur,\n                layer.pw1, layer.pw1_b, NULL,\n                NULL,      NULL,        NULL,\n                layer.pw2, layer.pw2_b, NULL,\n                NULL,\n                LLM_FFN_GELU, LLM_FFN_SEQ, il);\n\n        cur = ggml_mul(ctx0, cur, layer.gamma);\n\n        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));\n\n        inpL = ggml_add(ctx0, cur, inpL);\n    }\n    cur = inpL;\n\n    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));\n\n    cur = build_norm(cur,\n            model.output_norm,\n            model.output_norm_b,\n            LLM_NORM, -1);\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cur = ggml_add(ctx0, cur, model.output_b);\n\n    cb(cur, \"result_embd\", -1);\n    res->t_embd = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/models/xverse.cpp",
    "content": "#include \"models.h\"\n\nllm_build_xverse::llm_build_xverse(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {\n    const int64_t n_embd_head = hparams.n_embd_head_v();\n\n    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());\n    GGML_ASSERT(n_embd_head == n_rot);\n\n    ggml_tensor * cur;\n    ggml_tensor * inpL;\n\n    inpL = build_inp_embd(model.tok_embd);\n\n    // inp_pos - contains the positions\n    ggml_tensor * inp_pos = build_inp_pos();\n\n    auto * inp_attn = build_attn_inp_kv();\n\n    ggml_tensor * inp_out_ids = build_inp_out_ids();\n\n    for (int il = 0; il < n_layer; ++il) {\n        ggml_tensor * inpSA = inpL;\n\n        cur = build_norm(inpL,\n                model.layers[il].attn_norm, NULL,\n                LLM_NORM_RMS, il);\n        cb(cur, \"attn_norm\", il);\n\n        // self-attention\n        {\n            ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur);\n            cb(Qcur, \"Qcur\", il);\n\n            ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur);\n            cb(Kcur, \"Kcur\", il);\n\n            ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur);\n            cb(Vcur, \"Vcur\", il);\n\n            Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head,    n_tokens);\n            Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);\n            Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);\n\n            Qcur = ggml_rope_ext(\n                    ctx0, Qcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            Kcur = ggml_rope_ext(\n                    ctx0, Kcur, inp_pos, nullptr,\n                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,\n                    ext_factor, attn_factor, beta_fast, beta_slow\n                    );\n\n            cb(Qcur, \"Qcur\", il);\n            cb(Kcur, \"Kcur\", il);\n            cb(Vcur, \"Vcur\", il);\n\n            cur = build_attn(inp_attn,\n                    model.layers[il].wo, NULL,\n                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);\n        }\n        if (il == n_layer - 1 && inp_out_ids) {\n            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);\n            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);\n        }\n        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);\n        cb(ffn_inp, \"ffn_inp\", il);\n\n        // feed-forward network\n        {\n            cur = build_norm(ffn_inp,\n                    model.layers[il].ffn_norm, NULL,\n                    LLM_NORM_RMS, il);\n            cb(cur, \"ffn_norm\", il);\n\n            cur = build_ffn(cur,\n                    model.layers[il].ffn_up,   NULL, NULL,\n                    model.layers[il].ffn_gate, NULL, NULL,\n                    model.layers[il].ffn_down, NULL, NULL,\n                    NULL,\n                    LLM_FFN_SILU, LLM_FFN_PAR, il);\n            cb(cur, \"ffn_out\", il);\n        }\n        cur = ggml_add(ctx0, cur, ffn_inp);\n\n        cur = build_cvec(cur, il);\n        cb(cur, \"l_out\", il);\n\n        // input for next layer\n        inpL = cur;\n    }\n    cur = inpL;\n\n    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);\n\n    cb(cur, \"result_norm\", -1);\n    res->t_embd = cur;\n\n    // lm_head\n    cur = build_lora_mm(model.output, cur);\n\n    cb(cur, \"result_output\", -1);\n    res->t_logits = cur;\n\n    ggml_build_forward_expand(gf, cur);\n}\n"
  },
  {
    "path": "examples/talk-llama/prompts/talk-alpaca.txt",
    "content": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n\nWrite a text transcript of a never ending dialog, where {0} interacts with an AI assistant named {1}.\n{1} is helpful, kind, honest, friendly, good at writing and never fails to answer {0}’s requests immediately and with details and precision.\nThere are no annotations like (30 seconds passed...) or (to himself), just what {0} and {1} say aloud to each other.\nThe transcript only includes text, it does not include markup like HTML and Markdown.\n{1} responds with short and concise answers.\n\n### Response:\n\n{0}{4} Hello, {1}!\n{1}{4} Hello {0}! How may I help you today?\n{0}{4} What time is it?\n{1}{4} It is {2} o'clock.\n{0}{4} What year is it?\n{1}{4} We are in {3}.\n{0}{4} What is a cat?\n{1}{4} A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.\n{0}{4} Name a color.\n{1}{4} Blue\n{0}{4}\n"
  },
  {
    "path": "examples/talk-llama/speak",
    "content": "#!/bin/bash\n\n# Usage:\n#  speak <voice_id> <textfile>\n\nfunction installed() { command -v $1 >/dev/null 2>&1; }\n\nif installed espeak; then\n  espeak -v en-us+m$1 -s 225 -p 50 -a 200 -g 5 -k 5 -f $2\n\nelif installed piper && installed aplay; then\n  cat $2 | piper --model ~/en_US-lessac-medium.onnx --output-raw | aplay -q -r 22050 -f S16_LE -t raw -\n\n# for Mac\nelif installed say; then\n  say -f $2\n\n# Eleven Labs\nelif installed python3 && \\\n  python3 -c 'import importlib.util; exit(not importlib.util.find_spec(\"elevenlabs\"))' && \\\n  installed ffplay; then\n    # It's possible to use the API for free with limited number of characters.\n    # To increase this limit register to https://beta.elevenlabs.io to get an api key\n    # and paste it after 'ELEVEN_API_KEY='\n    # Keep the line commented to use the free version without api key\n    #export ELEVEN_API_KEY=your_api_key\n    wd=$(dirname $0)\n    script=$wd/eleven-labs.py\n    python3 $script -q -p -v $1 $2 >/dev/null 2>&1\n\n    # Uncomment to keep the audio file\n    #python3 $script -q -s ./audio.mp3 -v $1 $2 >/dev/null 2>&1\n    #ffplay -autoexit -nodisp -loglevel quiet -hide_banner -i ./audio.mp3 >/dev/null 2>&1\n\nelse\n  echo 'Install espeak (\"brew install espeak\" or \"apt-get install espeak\"),'\n  echo 'piper (\"pip install piper-tts\" or https://github.com/rhasspy/piper) with aplay,'\n  echo 'or elevenlabs (\"pip install elevenlabs\") with ffplay.'\n  echo '(export ELEVEN_API_KEY if you have an api key from https://beta.elevenlabs.io)'\nfi\n"
  },
  {
    "path": "examples/talk-llama/speak.bat",
    "content": "@powershell -ExecutionPolicy Bypass -F examples\\talk-llama\\speak.ps1 %1 %2\r\n"
  },
  {
    "path": "examples/talk-llama/speak.ps1",
    "content": "# Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope CurrentUser\r\nparam(\r\n  [Parameter(Mandatory=$true)][int]$voicenum,\r\n  [Parameter(Mandatory=$true)][string]$textfile\r\n)\r\n\r\nAdd-Type -AssemblyName System.Speech;\r\n$speak = New-Object System.Speech.Synthesis.SpeechSynthesizer;\r\n$voiceoptions = $speak.GetInstalledVoices(\"en-US\");\r\n$voice = $voiceoptions[$voicenum % $voiceoptions.count];\r\n$speak.SelectVoice($voice.VoiceInfo.Name);\r\n$speak.Rate=\"0\";\r\n$text = Get-Content -Path $textfile;\r\n$speak.Speak($text);\r\n"
  },
  {
    "path": "examples/talk-llama/talk-llama.cpp",
    "content": "// Talk with AI\n//\n\n#include \"common-sdl.h\"\n#include \"common.h\"\n#include \"common-whisper.h\"\n#include \"whisper.h\"\n#include \"llama.h\"\n\n#include <chrono>\n#include <cstdio>\n#include <fstream>\n#include <regex>\n#include <regex>\n#include <sstream>\n#include <string>\n#include <thread>\n#include <vector>\n\nstatic std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) {\n    const llama_model * model = llama_get_model(ctx);\n    const llama_vocab * vocab = llama_model_get_vocab(model);\n\n    // upper limit for the number of tokens\n    int n_tokens = text.length() + add_bos;\n    std::vector<llama_token> result(n_tokens);\n    n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_bos, false);\n    if (n_tokens < 0) {\n        result.resize(-n_tokens);\n        int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_bos, false);\n        GGML_ASSERT(check == -n_tokens);\n    } else {\n        result.resize(n_tokens);\n    }\n    return result;\n}\n\nstatic std::string llama_token_to_piece(const struct llama_context * ctx, llama_token token) {\n    const llama_model * model = llama_get_model(ctx);\n    const llama_vocab * vocab = llama_model_get_vocab(model);\n\n    std::vector<char> result(8, 0);\n    const int n_tokens = llama_token_to_piece(vocab, token, result.data(), result.size(), 0, false);\n    if (n_tokens < 0) {\n        result.resize(-n_tokens);\n        int check = llama_token_to_piece(vocab, token, result.data(), result.size(), 0, false);\n        GGML_ASSERT(check == -n_tokens);\n    } else {\n        result.resize(n_tokens);\n    }\n\n    return std::string(result.data(), result.size());\n}\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t voice_ms   = 10000;\n    int32_t capture_id = -1;\n    int32_t max_tokens = 32;\n    int32_t audio_ctx  = 0;\n    int32_t n_gpu_layers = 999;\n    int32_t seed = 0;\n    int32_t top_k = 5;\n    int32_t min_keep = 1;\n    float top_p = 0.80f;\n    float min_p = 0.01f;\n    float temp  = 0.30f;\n\n    float vad_thold  = 0.6f;\n    float freq_thold = 100.0f;\n\n    bool translate      = false;\n    bool print_special  = false;\n    bool print_energy   = false;\n    bool no_timestamps  = true;\n    bool verbose_prompt = false;\n    bool use_gpu        = true;\n    bool flash_attn     = true;\n\n    std::string person      = \"Georgi\";\n    std::string bot_name    = \"LLaMA\";\n    std::string wake_cmd    = \"\";\n    std::string heard_ok    = \"\";\n    std::string language    = \"en\";\n    std::string model_wsp   = \"models/ggml-base.en.bin\";\n    std::string model_llama = \"models/ggml-llama-7B.bin\";\n    std::string speak       = \"./examples/talk-llama/speak\";\n    std::string speak_file  = \"./examples/talk-llama/to_speak.txt\";\n    std::string prompt      = \"\";\n    std::string fname_out;\n    std::string path_session = \"\";       // path to file for saving/loading model eval state\n};\n\nvoid whisper_print_usage(int argc, char ** argv, const whisper_params & params);\n\nstatic bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"   || arg == \"--threads\")        { params.n_threads      = std::stoi(argv[++i]); }\n        else if (arg == \"-vms\" || arg == \"--voice-ms\")       { params.voice_ms       = std::stoi(argv[++i]); }\n        else if (arg == \"-c\"   || arg == \"--capture\")        { params.capture_id     = std::stoi(argv[++i]); }\n        else if (arg == \"-mt\"  || arg == \"--max-tokens\")     { params.max_tokens     = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"  || arg == \"--audio-ctx\")      { params.audio_ctx      = std::stoi(argv[++i]); }\n        else if (arg == \"-ngl\" || arg == \"--n-gpu-layers\")   { params.n_gpu_layers   = std::stoi(argv[++i]); }\n        else if (arg == \"--seed\")                            { params.seed           = std::stoi(argv[++i]); }\n        else if (arg == \"--top-k\")                           { params.top_k          = std::stoi(argv[++i]); }\n        else if (arg == \"--min-keep\")                        { params.min_keep       = std::stoul(argv[++i]);}\n        else if (arg == \"--top-p\")                           { params.top_p          = std::stof(argv[++i]); }\n        else if (arg == \"--min-p\")                           { params.min_p          = std::stof(argv[++i]); }\n        else if (arg == \"--temp\")                            { params.temp           = std::stof(argv[++i]); }\n        else if (arg == \"-vth\" || arg == \"--vad-thold\")      { params.vad_thold      = std::stof(argv[++i]); }\n        else if (arg == \"-fth\" || arg == \"--freq-thold\")     { params.freq_thold     = std::stof(argv[++i]); }\n        else if (arg == \"-tr\"  || arg == \"--translate\")      { params.translate      = true; }\n        else if (arg == \"-ps\"  || arg == \"--print-special\")  { params.print_special  = true; }\n        else if (arg == \"-pe\"  || arg == \"--print-energy\")   { params.print_energy   = true; }\n        else if (arg == \"-vp\"  || arg == \"--verbose-prompt\") { params.verbose_prompt = true; }\n        else if (arg == \"-ng\"  || arg == \"--no-gpu\")         { params.use_gpu        = false; }\n        else if (arg == \"-fa\"  || arg == \"--flash-attn\")     { params.flash_attn     = true; }\n        else if (arg == \"-nfa\" || arg == \"--no-flash-attn\")  { params.flash_attn     = false; }\n        else if (arg == \"-p\"   || arg == \"--person\")         { params.person         = argv[++i]; }\n        else if (arg == \"-bn\"   || arg == \"--bot-name\")      { params.bot_name       = argv[++i]; }\n        else if (arg == \"--session\")                         { params.path_session   = argv[++i]; }\n        else if (arg == \"-w\"   || arg == \"--wake-command\")   { params.wake_cmd       = argv[++i]; }\n        else if (arg == \"-ho\"  || arg == \"--heard-ok\")       { params.heard_ok       = argv[++i]; }\n        else if (arg == \"-l\"   || arg == \"--language\")       { params.language       = argv[++i]; }\n        else if (arg == \"-mw\"  || arg == \"--model-whisper\")  { params.model_wsp      = argv[++i]; }\n        else if (arg == \"-ml\"  || arg == \"--model-llama\")    { params.model_llama    = argv[++i]; }\n        else if (arg == \"-s\"   || arg == \"--speak\")          { params.speak          = argv[++i]; }\n        else if (arg == \"-sf\"  || arg == \"--speak-file\")     { params.speak_file     = argv[++i]; }\n        else if (arg == \"--prompt-file\")                     {\n            std::ifstream file(argv[++i]);\n            std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));\n            if (params.prompt.back() == '\\n') {\n                params.prompt.pop_back();\n            }\n        }\n        else if (arg == \"-f\"   || arg == \"--file\")          { params.fname_out     = argv[++i]; }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,       --help           [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,     --threads N      [%-7d] number of threads to use during computation\\n\", params.n_threads);\n    fprintf(stderr, \"  -vms N,   --voice-ms N     [%-7d] voice duration in milliseconds\\n\",              params.voice_ms);\n    fprintf(stderr, \"  -c ID,    --capture ID     [%-7d] capture device ID\\n\",                           params.capture_id);\n    fprintf(stderr, \"  -mt N,    --max-tokens N   [%-7d] maximum number of tokens per audio chunk\\n\",    params.max_tokens);\n    fprintf(stderr, \"  -ac N,    --audio-ctx N    [%-7d] audio context size (0 - all)\\n\",                params.audio_ctx);\n    fprintf(stderr, \"  -ngl N,   --n-gpu-layers N [%-7d] number of layers to store in VRAM\\n\",           params.n_gpu_layers);\n    fprintf(stderr, \"  --seed N                   [%-7d] seed sampling\\n\",                               params.seed);\n    fprintf(stderr, \"  --top-k N                  [%-7d] top-k sampling (0 = disabled)\\n\",               params.top_k);\n    fprintf(stderr, \"  --min-keep N               [%-7d] minimum number of tokens to keep\\n\",            params.min_keep);\n    fprintf(stderr, \"  --top-p N                  [%-7.2f] top-p sampling\\n\",                            params.top_p);\n    fprintf(stderr, \"  --min-p N                  [%-7.2f] min-p sampling\\n\",                            params.min_p);\n    fprintf(stderr, \"  --temp N                   [%-7.2f] temperature\\n\",                               params.temp);\n    fprintf(stderr, \"  -vth N,   --vad-thold N    [%-7.2f] voice activity detection threshold\\n\",        params.vad_thold);\n    fprintf(stderr, \"  -fth N,   --freq-thold N   [%-7.2f] high-pass frequency cutoff\\n\",                params.freq_thold);\n    fprintf(stderr, \"  -tr,      --translate      [%-7s] translate from source language to english\\n\",   params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,      --print-special  [%-7s] print special tokens\\n\",                        params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pe,      --print-energy   [%-7s] print sound energy (for debugging)\\n\",          params.print_energy ? \"true\" : \"false\");\n    fprintf(stderr, \"  -vp,      --verbose-prompt [%-7s] print prompt at start\\n\",                       params.verbose_prompt ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ng,      --no-gpu         [%-7s] disable GPU\\n\",                                 params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,      --flash-attn     [%-7s] enable flash attention\\n\",                      params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,     --no-flash-attn  [%-7s] disable flash attention\\n\",                     params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -p NAME,  --person NAME    [%-7s] person name (for prompt selection)\\n\",          params.person.c_str());\n    fprintf(stderr, \"  -bn NAME, --bot-name NAME  [%-7s] bot name (to display)\\n\",                       params.bot_name.c_str());\n    fprintf(stderr, \"  -w TEXT,  --wake-command T [%-7s] wake-up command to listen for\\n\",               params.wake_cmd.c_str());\n    fprintf(stderr, \"  -ho TEXT, --heard-ok TEXT  [%-7s] said by TTS before generating reply\\n\",         params.heard_ok.c_str());\n    fprintf(stderr, \"  -l LANG,  --language LANG  [%-7s] spoken language\\n\",                             params.language.c_str());\n    fprintf(stderr, \"  -mw FILE, --model-whisper  [%-7s] whisper model file\\n\",                          params.model_wsp.c_str());\n    fprintf(stderr, \"  -ml FILE, --model-llama    [%-7s] llama model file\\n\",                            params.model_llama.c_str());\n    fprintf(stderr, \"  -s FILE,  --speak TEXT     [%-7s] command for TTS\\n\",                             params.speak.c_str());\n    fprintf(stderr, \"  -sf FILE, --speak-file     [%-7s] file to pass to TTS\\n\",                         params.speak_file.c_str());\n    fprintf(stderr, \"  --prompt-file FNAME        [%-7s] file with custom prompt to start dialog\\n\",     \"\");\n    fprintf(stderr, \"  --session FNAME                   file to cache model state in (may be large!) (default: none)\\n\");\n    fprintf(stderr, \"  -f FNAME, --file FNAME     [%-7s] text output file name\\n\",                       params.fname_out.c_str());\n    fprintf(stderr, \"\\n\");\n}\n\nstatic std::string transcribe(\n        whisper_context * ctx,\n        const whisper_params & params,\n        const std::vector<float> & pcmf32,\n        const std::string prompt_text,\n        float & prob,\n        int64_t & t_ms) {\n    const auto t_start = std::chrono::high_resolution_clock::now();\n\n    prob = 0.0f;\n    t_ms = 0;\n\n    std::vector<whisper_token> prompt_tokens;\n\n    whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n    prompt_tokens.resize(1024);\n    prompt_tokens.resize(whisper_tokenize(ctx, prompt_text.c_str(), prompt_tokens.data(), prompt_tokens.size()));\n\n    wparams.print_progress   = false;\n    wparams.print_special    = params.print_special;\n    wparams.print_realtime   = false;\n    wparams.print_timestamps = !params.no_timestamps;\n    wparams.translate        = params.translate;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.max_tokens       = params.max_tokens;\n    wparams.language         = params.language.c_str();\n    wparams.n_threads        = params.n_threads;\n\n    wparams.prompt_tokens    = prompt_tokens.empty() ? nullptr : prompt_tokens.data();\n    wparams.prompt_n_tokens  = prompt_tokens.empty() ? 0       : prompt_tokens.size();\n\n    wparams.audio_ctx        = params.audio_ctx;\n\n    if (whisper_full(ctx, wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        return \"\";\n    }\n\n    int prob_n = 0;\n    std::string result;\n\n    const int n_segments = whisper_full_n_segments(ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(ctx, i);\n\n        result += text;\n\n        const int n_tokens = whisper_full_n_tokens(ctx, i);\n        for (int j = 0; j < n_tokens; ++j) {\n            const auto token = whisper_full_get_token_data(ctx, i, j);\n\n            prob += token.p;\n            ++prob_n;\n        }\n    }\n\n    if (prob_n > 0) {\n        prob /= prob_n;\n    }\n\n    const auto t_end = std::chrono::high_resolution_clock::now();\n    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();\n\n    return result;\n}\n\nstatic std::vector<std::string> get_words(const std::string &txt) {\n    std::vector<std::string> words;\n\n    std::istringstream iss(txt);\n    std::string word;\n    while (iss >> word) {\n        words.push_back(word);\n    }\n\n    return words;\n}\n\nconst std::string k_prompt_whisper = R\"(A conversation with a person called {1}.)\";\n\nconst std::string k_prompt_llama = R\"(Text transcript of a never ending dialog, where {0} interacts with an AI assistant named {1}.\n{1} is helpful, kind, honest, friendly, good at writing and never fails to answer {0}’s requests immediately and with details and precision.\nThere are no annotations like (30 seconds passed...) or (to himself), just what {0} and {1} say aloud to each other.\nThe transcript only includes text, it does not include markup like HTML and Markdown.\n{1} responds with short and concise answers.\n\n{0}{4} Hello, {1}!\n{1}{4} Hello {0}! How may I help you today?\n{0}{4} What time is it?\n{1}{4} It is {2} o'clock.\n{0}{4} What year is it?\n{1}{4} We are in {3}.\n{0}{4} What is a cat?\n{1}{4} A cat is a domestic species of small carnivorous mammal. It is the only domesticated species in the family Felidae.\n{0}{4} Name a color.\n{1}{4} Blue\n{0}{4})\";\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    if (params.language != \"auto\" && whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    // whisper init\n\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    struct whisper_context * ctx_wsp = whisper_init_from_file_with_params(params.model_wsp.c_str(), cparams);\n    if (!ctx_wsp) {\n        fprintf(stderr, \"No whisper.cpp model specified. Please provide using -mw <modelfile>\\n\");\n        return 1;\n    }\n\n    // llama init\n\n    llama_backend_init();\n\n    auto lmparams = llama_model_default_params();\n    if (!params.use_gpu) {\n        lmparams.n_gpu_layers = 0;\n    } else {\n        lmparams.n_gpu_layers = params.n_gpu_layers;\n    }\n\n    struct llama_model * model_llama = llama_model_load_from_file(params.model_llama.c_str(), lmparams);\n    if (!model_llama) {\n        fprintf(stderr, \"No llama.cpp model specified. Please provide using -ml <modelfile>\\n\");\n        return 1;\n    }\n\n    const llama_vocab * vocab_llama = llama_model_get_vocab(model_llama);\n\n    llama_context_params lcparams = llama_context_default_params();\n\n    // tune these to your liking\n    lcparams.n_ctx     = 2048;\n    lcparams.n_threads = params.n_threads;\n\n    lcparams.flash_attn_type = params.flash_attn ? LLAMA_FLASH_ATTN_TYPE_AUTO : LLAMA_FLASH_ATTN_TYPE_DISABLED;\n\n    struct llama_context * ctx_llama = llama_init_from_model(model_llama, lcparams);\n\n    // print some info about the processing\n    {\n        fprintf(stderr, \"\\n\");\n\n        if (!whisper_is_multilingual(ctx_wsp)) {\n            if (params.language != \"en\" || params.translate) {\n                params.language = \"en\";\n                params.translate = false;\n                fprintf(stderr, \"%s: WARNING: model is not multilingual, ignoring language and translation options\\n\", __func__);\n            }\n        }\n        fprintf(stderr, \"%s: processing, %d threads, lang = %s, task = %s, timestamps = %d ...\\n\",\n                __func__,\n                params.n_threads,\n                params.language.c_str(),\n                params.translate ? \"translate\" : \"transcribe\",\n                params.no_timestamps ? 0 : 1);\n\n        fprintf(stderr, \"\\n\");\n    }\n\n    // init audio\n\n    audio_async audio(30*1000);\n    if (!audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {\n        fprintf(stderr, \"%s: audio.init() failed!\\n\", __func__);\n        return 1;\n    }\n\n    audio.resume();\n\n    bool is_running  = true;\n    bool force_speak = false;\n\n    float prob0 = 0.0f;\n\n    const std::string chat_symb = \":\";\n\n    std::vector<float> pcmf32_cur;\n    std::vector<float> pcmf32_prompt;\n\n    const std::string prompt_whisper = ::replace(k_prompt_whisper, \"{1}\", params.bot_name);\n\n    // construct the initial prompt for LLaMA inference\n    std::string prompt_llama = params.prompt.empty() ? k_prompt_llama : params.prompt;\n\n    // need to have leading ' '\n    prompt_llama.insert(0, 1, ' ');\n\n    prompt_llama = ::replace(prompt_llama, \"{0}\", params.person);\n    prompt_llama = ::replace(prompt_llama, \"{1}\", params.bot_name);\n\n    {\n        // get time string\n        std::string time_str;\n        {\n            time_t t = time(0);\n            struct tm * now = localtime(&t);\n            char buf[128];\n            strftime(buf, sizeof(buf), \"%H:%M\", now);\n            time_str = buf;\n        }\n        prompt_llama = ::replace(prompt_llama, \"{2}\", time_str);\n    }\n\n    {\n        // get year string\n        std::string year_str;\n        {\n            time_t t = time(0);\n            struct tm * now = localtime(&t);\n            char buf[128];\n            strftime(buf, sizeof(buf), \"%Y\", now);\n            year_str = buf;\n        }\n        prompt_llama = ::replace(prompt_llama, \"{3}\", year_str);\n    }\n\n    prompt_llama = ::replace(prompt_llama, \"{4}\", chat_symb);\n\n    llama_batch batch = llama_batch_init(llama_n_ctx(ctx_llama), 0, 1);\n\n    // init sampler\n    auto sparams = llama_sampler_chain_default_params();\n\n    llama_sampler * smpl = llama_sampler_chain_init(sparams);\n\n    if (params.temp > 0.0f) {\n        llama_sampler_chain_add(smpl, llama_sampler_init_top_k(params.top_k));\n        llama_sampler_chain_add(smpl, llama_sampler_init_top_p(params.top_p, params.min_keep));\n        llama_sampler_chain_add(smpl, llama_sampler_init_temp (params.temp));\n        llama_sampler_chain_add(smpl, llama_sampler_init_dist (params.seed));\n        llama_sampler_chain_add(smpl, llama_sampler_init_min_p (params.min_p, params.min_keep));\n    } else {\n        llama_sampler_chain_add(smpl, llama_sampler_init_greedy());\n    }\n\n    // init session\n    std::string path_session = params.path_session;\n    std::vector<llama_token> session_tokens;\n    auto embd_inp = ::llama_tokenize(ctx_llama, prompt_llama, true);\n\n    if (!path_session.empty()) {\n        fprintf(stderr, \"%s: attempting to load saved session from %s\\n\", __func__, path_session.c_str());\n\n        // fopen to check for existing session\n        FILE * fp = std::fopen(path_session.c_str(), \"rb\");\n        if (fp != NULL) {\n            std::fclose(fp);\n\n            session_tokens.resize(llama_n_ctx(ctx_llama));\n            size_t n_token_count_out = 0;\n            if (!llama_state_load_file(ctx_llama, path_session.c_str(), session_tokens.data(), session_tokens.capacity(), &n_token_count_out)) {\n                fprintf(stderr, \"%s: error: failed to load session file '%s'\\n\", __func__, path_session.c_str());\n                return 1;\n            }\n            session_tokens.resize(n_token_count_out);\n            for (size_t i = 0; i < session_tokens.size(); i++) {\n                embd_inp[i] = session_tokens[i];\n            }\n\n            fprintf(stderr, \"%s: loaded a session with prompt size of %d tokens\\n\", __func__, (int) session_tokens.size());\n        } else {\n            fprintf(stderr, \"%s: session file does not exist, will create\\n\", __func__);\n        }\n    }\n\n    // evaluate the initial prompt\n\n    printf(\"\\n\");\n    printf(\"%s : initializing - please wait ...\\n\", __func__);\n\n    // prepare batch\n    {\n        batch.n_tokens = embd_inp.size();\n\n        for (int i = 0; i < batch.n_tokens; i++) {\n            batch.token[i]     = embd_inp[i];\n            batch.pos[i]       = i;\n            batch.n_seq_id[i]  = 1;\n            batch.seq_id[i][0] = 0;\n            batch.logits[i]    = i == batch.n_tokens - 1;\n        }\n    }\n\n    if (llama_decode(ctx_llama, batch)) {\n        fprintf(stderr, \"%s : failed to decode\\n\", __func__);\n        return 1;\n    }\n\n    if (params.verbose_prompt) {\n        fprintf(stdout, \"\\n\");\n        fprintf(stdout, \"%s\", prompt_llama.c_str());\n        fflush(stdout);\n    }\n\n     // debug message about similarity of saved session, if applicable\n    size_t n_matching_session_tokens = 0;\n    if (session_tokens.size()) {\n        for (llama_token id : session_tokens) {\n            if (n_matching_session_tokens >= embd_inp.size() || id != embd_inp[n_matching_session_tokens]) {\n                break;\n            }\n            n_matching_session_tokens++;\n        }\n        if (n_matching_session_tokens >= embd_inp.size()) {\n            fprintf(stderr, \"%s: session file has exact match for prompt!\\n\", __func__);\n        } else if (n_matching_session_tokens < (embd_inp.size() / 2)) {\n            fprintf(stderr, \"%s: warning: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\\n\",\n                __func__, n_matching_session_tokens, embd_inp.size());\n        } else {\n            fprintf(stderr, \"%s: session file matches %zu / %zu tokens of prompt\\n\",\n                __func__, n_matching_session_tokens, embd_inp.size());\n        }\n    }\n\n    // HACK - because session saving incurs a non-negligible delay, for now skip re-saving session\n    // if we loaded a session with at least 75% similarity. It's currently just used to speed up the\n    // initial prompt so it doesn't need to be an exact match.\n    bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < (embd_inp.size() * 3 / 4);\n\n    printf(\"%s : done! start speaking in the microphone\\n\", __func__);\n\n    // show wake command if enabled\n    const std::string wake_cmd = params.wake_cmd;\n    const int wake_cmd_length = get_words(wake_cmd).size();\n    const bool use_wake_cmd = wake_cmd_length > 0;\n\n    if (use_wake_cmd) {\n        printf(\"%s : the wake-up command is: '%s%s%s'\\n\", __func__, \"\\033[1m\", wake_cmd.c_str(), \"\\033[0m\");\n    }\n\n    printf(\"\\n\");\n    printf(\"%s%s\", params.person.c_str(), chat_symb.c_str());\n    fflush(stdout);\n\n    // clear audio buffer\n    audio.clear();\n\n    // text inference variables\n    const int voice_id = 2;\n    const int n_keep   = embd_inp.size();\n    const int n_ctx    = llama_n_ctx(ctx_llama);\n\n    int n_past = n_keep;\n    int n_prev = 64; // TODO arg\n    int n_session_consumed = !path_session.empty() && session_tokens.size() > 0 ? session_tokens.size() : 0;\n\n    std::vector<llama_token> embd;\n\n    // reverse prompts for detecting when it's time to stop speaking\n    std::vector<std::string> antiprompts = {\n        params.person + chat_symb,\n    };\n\n    // main loop\n    while (is_running) {\n        // handle Ctrl + C\n        is_running = sdl_poll_events();\n\n        if (!is_running) {\n            break;\n        }\n\n        // delay\n        std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n        int64_t t_ms = 0;\n\n        {\n            audio.get(2000, pcmf32_cur);\n\n            if (::vad_simple(pcmf32_cur, WHISPER_SAMPLE_RATE, 1250, params.vad_thold, params.freq_thold, params.print_energy) || force_speak) {\n                //fprintf(stdout, \"%s: Speech detected! Processing ...\\n\", __func__);\n\n                audio.get(params.voice_ms, pcmf32_cur);\n\n                std::string all_heard;\n\n                if (!force_speak) {\n                    all_heard = ::trim(::transcribe(ctx_wsp, params, pcmf32_cur, prompt_whisper, prob0, t_ms));\n                }\n\n                const auto words = get_words(all_heard);\n\n                std::string wake_cmd_heard;\n                std::string text_heard;\n\n                for (int i = 0; i < (int) words.size(); ++i) {\n                    if (i < wake_cmd_length) {\n                        wake_cmd_heard += words[i] + \" \";\n                    } else {\n                        text_heard += words[i] + \" \";\n                    }\n                }\n\n                // check if audio starts with the wake-up command if enabled\n                if (use_wake_cmd) {\n                    const float sim = similarity(wake_cmd_heard, wake_cmd);\n\n                    if ((sim < 0.7f) || (text_heard.empty())) {\n                        audio.clear();\n                        continue;\n                    }\n                }\n\n                // optionally give audio feedback that the current text is being processed\n                if (!params.heard_ok.empty()) {\n                    speak_with_file(params.speak, params.heard_ok, params.speak_file, voice_id);\n                }\n\n                // remove text between brackets using regex\n                {\n                    std::regex re(\"\\\\[.*?\\\\]\");\n                    text_heard = std::regex_replace(text_heard, re, \"\");\n                }\n\n                // remove text between brackets using regex\n                {\n                    std::regex re(\"\\\\(.*?\\\\)\");\n                    text_heard = std::regex_replace(text_heard, re, \"\");\n                }\n\n                // remove all characters, except for letters, numbers, punctuation and ':', '\\'', '-', ' '\n                text_heard = std::regex_replace(text_heard, std::regex(\"[^a-zA-Z0-9åäöÅÄÖ\\\\.,\\\\?!\\\\s\\\\:\\\\'\\\\-]\"), \"\");\n\n                // take first line\n                text_heard = text_heard.substr(0, text_heard.find_first_of('\\n'));\n\n                // remove leading and trailing whitespace\n                text_heard = std::regex_replace(text_heard, std::regex(\"^\\\\s+\"), \"\");\n                text_heard = std::regex_replace(text_heard, std::regex(\"\\\\s+$\"), \"\");\n\n                const std::vector<llama_token> tokens = llama_tokenize(ctx_llama, text_heard.c_str(), false);\n\n                if (text_heard.empty() || tokens.empty() || force_speak) {\n                    //fprintf(stdout, \"%s: Heard nothing, skipping ...\\n\", __func__);\n                    audio.clear();\n\n                    continue;\n                }\n\n                force_speak = false;\n\n                text_heard.insert(0, 1, ' ');\n                text_heard += \"\\n\" + params.bot_name + chat_symb;\n                fprintf(stdout, \"%s%s%s\", \"\\033[1m\", text_heard.c_str(), \"\\033[0m\");\n                fflush(stdout);\n\n                embd = ::llama_tokenize(ctx_llama, text_heard, false);\n\n                // Append the new input tokens to the session_tokens vector\n                if (!path_session.empty()) {\n                    session_tokens.insert(session_tokens.end(), tokens.begin(), tokens.end());\n                }\n\n                // text inference\n                bool done = false;\n                std::string text_to_speak;\n                while (true) {\n                    // predict\n                    if (embd.size() > 0) {\n                        if (n_past + (int) embd.size() > n_ctx) {\n                            n_past = n_keep;\n\n                            // insert n_left/2 tokens at the start of embd from last_n_tokens\n                            embd.insert(embd.begin(), embd_inp.begin() + embd_inp.size() - n_prev, embd_inp.end());\n                            // stop saving session if we run out of context\n                            path_session = \"\";\n                            //printf(\"\\n---\\n\");\n                            //printf(\"resetting: '\");\n                            //for (int i = 0; i < (int) embd.size(); i++) {\n                            //    printf(\"%s\", llama_token_to_piece(ctx_llama, embd[i]));\n                            //}\n                            //printf(\"'\\n\");\n                            //printf(\"\\n---\\n\");\n                        }\n\n                        // try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)\n                        // REVIEW\n                        if (n_session_consumed < (int) session_tokens.size()) {\n                            size_t i = 0;\n                            for ( ; i < embd.size(); i++) {\n                                if (embd[i] != session_tokens[n_session_consumed]) {\n                                    session_tokens.resize(n_session_consumed);\n                                    break;\n                                }\n\n                                n_past++;\n                                n_session_consumed++;\n\n                                if (n_session_consumed >= (int) session_tokens.size()) {\n                                    i++;\n                                    break;\n                                }\n                            }\n                            if (i > 0) {\n                                embd.erase(embd.begin(), embd.begin() + i);\n                            }\n                        }\n\n                        if (embd.size() > 0 && !path_session.empty()) {\n                            session_tokens.insert(session_tokens.end(), embd.begin(), embd.end());\n                            n_session_consumed = session_tokens.size();\n                        }\n\n                        // prepare batch\n                        {\n                            batch.n_tokens = embd.size();\n\n                            for (int i = 0; i < batch.n_tokens; i++) {\n                                batch.token[i]     = embd[i];\n                                batch.pos[i]       = n_past + i;\n                                batch.n_seq_id[i]  = 1;\n                                batch.seq_id[i][0] = 0;\n                                batch.logits[i]    = i == batch.n_tokens - 1;\n                            }\n                        }\n\n                        if (llama_decode(ctx_llama, batch)) {\n                            fprintf(stderr, \"%s : failed to decode\\n\", __func__);\n                            return 1;\n                        }\n                    }\n\n\n                    embd_inp.insert(embd_inp.end(), embd.begin(), embd.end());\n                    n_past += embd.size();\n\n                    embd.clear();\n\n                    if (done) break;\n\n                    {\n                        // out of user input, sample next token\n\n                        if (!path_session.empty() && need_to_save_session) {\n                            need_to_save_session = false;\n                            llama_state_save_file(ctx_llama, path_session.c_str(), session_tokens.data(), session_tokens.size());\n                        }\n\n                        const llama_token id = llama_sampler_sample(smpl, ctx_llama, -1);\n\n                        if (id != llama_vocab_eos(vocab_llama)) {\n                            // add it to the context\n                            embd.push_back(id);\n\n                            text_to_speak += llama_token_to_piece(ctx_llama, id);\n\n                            printf(\"%s\", llama_token_to_piece(ctx_llama, id).c_str());\n                            fflush(stdout);\n                        }\n                    }\n\n                    {\n                        std::string last_output;\n                        for (int i = embd_inp.size() - 16; i < (int) embd_inp.size(); i++) {\n                            last_output += llama_token_to_piece(ctx_llama, embd_inp[i]);\n                        }\n                        last_output += llama_token_to_piece(ctx_llama, embd[0]);\n\n                        for (std::string & antiprompt : antiprompts) {\n                            if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {\n                                done = true;\n                                text_to_speak = ::replace(text_to_speak, antiprompt, \"\");\n                                fflush(stdout);\n                                need_to_save_session = true;\n                                break;\n                            }\n                        }\n                    }\n\n                    is_running = sdl_poll_events();\n\n                    if (!is_running) {\n                        break;\n                    }\n                }\n\n                speak_with_file(params.speak, text_to_speak, params.speak_file, voice_id);\n\n                audio.clear();\n            }\n        }\n    }\n\n    audio.pause();\n\n    whisper_print_timings(ctx_wsp);\n    whisper_free(ctx_wsp);\n\n    llama_perf_sampler_print(smpl);\n    llama_perf_context_print(ctx_llama);\n\n    llama_sampler_free(smpl);\n    llama_batch_free(batch);\n    llama_free(ctx_llama);\n\n    llama_backend_free();\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/talk-llama/unicode-data.cpp",
    "content": "// generated with scripts/gen-unicode-data.py\n\n#include \"unicode-data.h\"\n\n#include <cstdint>\n#include <vector>\n#include <unordered_map>\n#include <unordered_set>\n\nconst std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags = {  // start, flags // last=next_start-1\n{0x000000, 0x0080},\n{0x000020, 0x0008},\n{0x000021, 0x0020},\n{0x000024, 0x0040},\n{0x000025, 0x0020},\n{0x00002B, 0x0040},\n{0x00002C, 0x0020},\n{0x000030, 0x0002},\n{0x00003A, 0x0020},\n{0x00003C, 0x0040},\n{0x00003F, 0x0020},\n{0x000041, 0x0004},\n{0x00005B, 0x0020},\n{0x00005E, 0x0040},\n{0x00005F, 0x0020},\n{0x000060, 0x0040},\n{0x000061, 0x0004},\n{0x00007B, 0x0020},\n{0x00007C, 0x0040},\n{0x00007D, 0x0020},\n{0x00007E, 0x0040},\n{0x00007F, 0x0080},\n{0x0000A0, 0x0008},\n{0x0000A1, 0x0020},\n{0x0000A2, 0x0040},\n{0x0000A7, 0x0020},\n{0x0000A8, 0x0040},\n{0x0000AA, 0x0004},\n{0x0000AB, 0x0020},\n{0x0000AC, 0x0040},\n{0x0000AD, 0x0080},\n{0x0000AE, 0x0040},\n{0x0000B2, 0x0002},\n{0x0000B4, 0x0040},\n{0x0000B5, 0x0004},\n{0x0000B6, 0x0020},\n{0x0000B8, 0x0040},\n{0x0000B9, 0x0002},\n{0x0000BA, 0x0004},\n{0x0000BB, 0x0020},\n{0x0000BC, 0x0002},\n{0x0000BF, 0x0020},\n{0x0000C0, 0x0004},\n{0x0000D7, 0x0040},\n{0x0000D8, 0x0004},\n{0x0000F7, 0x0040},\n{0x0000F8, 0x0004},\n{0x0002C2, 0x0040},\n{0x0002C6, 0x0004},\n{0x0002D2, 0x0040},\n{0x0002E0, 0x0004},\n{0x0002E5, 0x0040},\n{0x0002EC, 0x0004},\n{0x0002ED, 0x0040},\n{0x0002EE, 0x0004},\n{0x0002EF, 0x0040},\n{0x000300, 0x0010},\n{0x000370, 0x0004},\n{0x000375, 0x0040},\n{0x000376, 0x0004},\n{0x000378, 0x0001},\n{0x00037A, 0x0004},\n{0x00037E, 0x0020},\n{0x00037F, 0x0004},\n{0x000380, 0x0001},\n{0x000384, 0x0040},\n{0x000386, 0x0004},\n{0x000387, 0x0020},\n{0x000388, 0x0004},\n{0x00038B, 0x0001},\n{0x00038C, 0x0004},\n{0x00038D, 0x0001},\n{0x00038E, 0x0004},\n{0x0003A2, 0x0001},\n{0x0003A3, 0x0004},\n{0x0003F6, 0x0040},\n{0x0003F7, 0x0004},\n{0x000482, 0x0040},\n{0x000483, 0x0010},\n{0x00048A, 0x0004},\n{0x000530, 0x0001},\n{0x000531, 0x0004},\n{0x000557, 0x0001},\n{0x000559, 0x0004},\n{0x00055A, 0x0020},\n{0x000560, 0x0004},\n{0x000589, 0x0020},\n{0x00058B, 0x0001},\n{0x00058D, 0x0040},\n{0x000590, 0x0001},\n{0x000591, 0x0010},\n{0x0005BE, 0x0020},\n{0x0005BF, 0x0010},\n{0x0005C0, 0x0020},\n{0x0005C1, 0x0010},\n{0x0005C3, 0x0020},\n{0x0005C4, 0x0010},\n{0x0005C6, 0x0020},\n{0x0005C7, 0x0010},\n{0x0005C8, 0x0001},\n{0x0005D0, 0x0004},\n{0x0005EB, 0x0001},\n{0x0005EF, 0x0004},\n{0x0005F3, 0x0020},\n{0x0005F5, 0x0001},\n{0x000600, 0x0080},\n{0x000606, 0x0040},\n{0x000609, 0x0020},\n{0x00060B, 0x0040},\n{0x00060C, 0x0020},\n{0x00060E, 0x0040},\n{0x000610, 0x0010},\n{0x00061B, 0x0020},\n{0x00061C, 0x0080},\n{0x00061D, 0x0020},\n{0x000620, 0x0004},\n{0x00064B, 0x0010},\n{0x000660, 0x0002},\n{0x00066A, 0x0020},\n{0x00066E, 0x0004},\n{0x000670, 0x0010},\n{0x000671, 0x0004},\n{0x0006D4, 0x0020},\n{0x0006D5, 0x0004},\n{0x0006D6, 0x0010},\n{0x0006DD, 0x0080},\n{0x0006DE, 0x0040},\n{0x0006DF, 0x0010},\n{0x0006E5, 0x0004},\n{0x0006E7, 0x0010},\n{0x0006E9, 0x0040},\n{0x0006EA, 0x0010},\n{0x0006EE, 0x0004},\n{0x0006F0, 0x0002},\n{0x0006FA, 0x0004},\n{0x0006FD, 0x0040},\n{0x0006FF, 0x0004},\n{0x000700, 0x0020},\n{0x00070E, 0x0001},\n{0x00070F, 0x0080},\n{0x000710, 0x0004},\n{0x000711, 0x0010},\n{0x000712, 0x0004},\n{0x000730, 0x0010},\n{0x00074B, 0x0001},\n{0x00074D, 0x0004},\n{0x0007A6, 0x0010},\n{0x0007B1, 0x0004},\n{0x0007B2, 0x0001},\n{0x0007C0, 0x0002},\n{0x0007CA, 0x0004},\n{0x0007EB, 0x0010},\n{0x0007F4, 0x0004},\n{0x0007F6, 0x0040},\n{0x0007F7, 0x0020},\n{0x0007FA, 0x0004},\n{0x0007FB, 0x0001},\n{0x0007FD, 0x0010},\n{0x0007FE, 0x0040},\n{0x000800, 0x0004},\n{0x000816, 0x0010},\n{0x00081A, 0x0004},\n{0x00081B, 0x0010},\n{0x000824, 0x0004},\n{0x000825, 0x0010},\n{0x000828, 0x0004},\n{0x000829, 0x0010},\n{0x00082E, 0x0001},\n{0x000830, 0x0020},\n{0x00083F, 0x0001},\n{0x000840, 0x0004},\n{0x000859, 0x0010},\n{0x00085C, 0x0001},\n{0x00085E, 0x0020},\n{0x00085F, 0x0001},\n{0x000860, 0x0004},\n{0x00086B, 0x0001},\n{0x000870, 0x0004},\n{0x000888, 0x0040},\n{0x000889, 0x0004},\n{0x00088F, 0x0001},\n{0x000890, 0x0080},\n{0x000892, 0x0001},\n{0x000898, 0x0010},\n{0x0008A0, 0x0004},\n{0x0008CA, 0x0010},\n{0x0008E2, 0x0080},\n{0x0008E3, 0x0010},\n{0x000904, 0x0004},\n{0x00093A, 0x0010},\n{0x00093D, 0x0004},\n{0x00093E, 0x0010},\n{0x000950, 0x0004},\n{0x000951, 0x0010},\n{0x000958, 0x0004},\n{0x000962, 0x0010},\n{0x000964, 0x0020},\n{0x000966, 0x0002},\n{0x000970, 0x0020},\n{0x000971, 0x0004},\n{0x000981, 0x0010},\n{0x000984, 0x0001},\n{0x000985, 0x0004},\n{0x00098D, 0x0001},\n{0x00098F, 0x0004},\n{0x000991, 0x0001},\n{0x000993, 0x0004},\n{0x0009A9, 0x0001},\n{0x0009AA, 0x0004},\n{0x0009B1, 0x0001},\n{0x0009B2, 0x0004},\n{0x0009B3, 0x0001},\n{0x0009B6, 0x0004},\n{0x0009BA, 0x0001},\n{0x0009BC, 0x0010},\n{0x0009BD, 0x0004},\n{0x0009BE, 0x0010},\n{0x0009C5, 0x0001},\n{0x0009C7, 0x0010},\n{0x0009C9, 0x0001},\n{0x0009CB, 0x0010},\n{0x0009CE, 0x0004},\n{0x0009CF, 0x0001},\n{0x0009D7, 0x0010},\n{0x0009D8, 0x0001},\n{0x0009DC, 0x0004},\n{0x0009DE, 0x0001},\n{0x0009DF, 0x0004},\n{0x0009E2, 0x0010},\n{0x0009E4, 0x0001},\n{0x0009E6, 0x0002},\n{0x0009F0, 0x0004},\n{0x0009F2, 0x0040},\n{0x0009F4, 0x0002},\n{0x0009FA, 0x0040},\n{0x0009FC, 0x0004},\n{0x0009FD, 0x0020},\n{0x0009FE, 0x0010},\n{0x0009FF, 0x0001},\n{0x000A01, 0x0010},\n{0x000A04, 0x0001},\n{0x000A05, 0x0004},\n{0x000A0B, 0x0001},\n{0x000A0F, 0x0004},\n{0x000A11, 0x0001},\n{0x000A13, 0x0004},\n{0x000A29, 0x0001},\n{0x000A2A, 0x0004},\n{0x000A31, 0x0001},\n{0x000A32, 0x0004},\n{0x000A34, 0x0001},\n{0x000A35, 0x0004},\n{0x000A37, 0x0001},\n{0x000A38, 0x0004},\n{0x000A3A, 0x0001},\n{0x000A3C, 0x0010},\n{0x000A3D, 0x0001},\n{0x000A3E, 0x0010},\n{0x000A43, 0x0001},\n{0x000A47, 0x0010},\n{0x000A49, 0x0001},\n{0x000A4B, 0x0010},\n{0x000A4E, 0x0001},\n{0x000A51, 0x0010},\n{0x000A52, 0x0001},\n{0x000A59, 0x0004},\n{0x000A5D, 0x0001},\n{0x000A5E, 0x0004},\n{0x000A5F, 0x0001},\n{0x000A66, 0x0002},\n{0x000A70, 0x0010},\n{0x000A72, 0x0004},\n{0x000A75, 0x0010},\n{0x000A76, 0x0020},\n{0x000A77, 0x0001},\n{0x000A81, 0x0010},\n{0x000A84, 0x0001},\n{0x000A85, 0x0004},\n{0x000A8E, 0x0001},\n{0x000A8F, 0x0004},\n{0x000A92, 0x0001},\n{0x000A93, 0x0004},\n{0x000AA9, 0x0001},\n{0x000AAA, 0x0004},\n{0x000AB1, 0x0001},\n{0x000AB2, 0x0004},\n{0x000AB4, 0x0001},\n{0x000AB5, 0x0004},\n{0x000ABA, 0x0001},\n{0x000ABC, 0x0010},\n{0x000ABD, 0x0004},\n{0x000ABE, 0x0010},\n{0x000AC6, 0x0001},\n{0x000AC7, 0x0010},\n{0x000ACA, 0x0001},\n{0x000ACB, 0x0010},\n{0x000ACE, 0x0001},\n{0x000AD0, 0x0004},\n{0x000AD1, 0x0001},\n{0x000AE0, 0x0004},\n{0x000AE2, 0x0010},\n{0x000AE4, 0x0001},\n{0x000AE6, 0x0002},\n{0x000AF0, 0x0020},\n{0x000AF1, 0x0040},\n{0x000AF2, 0x0001},\n{0x000AF9, 0x0004},\n{0x000AFA, 0x0010},\n{0x000B00, 0x0001},\n{0x000B01, 0x0010},\n{0x000B04, 0x0001},\n{0x000B05, 0x0004},\n{0x000B0D, 0x0001},\n{0x000B0F, 0x0004},\n{0x000B11, 0x0001},\n{0x000B13, 0x0004},\n{0x000B29, 0x0001},\n{0x000B2A, 0x0004},\n{0x000B31, 0x0001},\n{0x000B32, 0x0004},\n{0x000B34, 0x0001},\n{0x000B35, 0x0004},\n{0x000B3A, 0x0001},\n{0x000B3C, 0x0010},\n{0x000B3D, 0x0004},\n{0x000B3E, 0x0010},\n{0x000B45, 0x0001},\n{0x000B47, 0x0010},\n{0x000B49, 0x0001},\n{0x000B4B, 0x0010},\n{0x000B4E, 0x0001},\n{0x000B55, 0x0010},\n{0x000B58, 0x0001},\n{0x000B5C, 0x0004},\n{0x000B5E, 0x0001},\n{0x000B5F, 0x0004},\n{0x000B62, 0x0010},\n{0x000B64, 0x0001},\n{0x000B66, 0x0002},\n{0x000B70, 0x0040},\n{0x000B71, 0x0004},\n{0x000B72, 0x0002},\n{0x000B78, 0x0001},\n{0x000B82, 0x0010},\n{0x000B83, 0x0004},\n{0x000B84, 0x0001},\n{0x000B85, 0x0004},\n{0x000B8B, 0x0001},\n{0x000B8E, 0x0004},\n{0x000B91, 0x0001},\n{0x000B92, 0x0004},\n{0x000B96, 0x0001},\n{0x000B99, 0x0004},\n{0x000B9B, 0x0001},\n{0x000B9C, 0x0004},\n{0x000B9D, 0x0001},\n{0x000B9E, 0x0004},\n{0x000BA0, 0x0001},\n{0x000BA3, 0x0004},\n{0x000BA5, 0x0001},\n{0x000BA8, 0x0004},\n{0x000BAB, 0x0001},\n{0x000BAE, 0x0004},\n{0x000BBA, 0x0001},\n{0x000BBE, 0x0010},\n{0x000BC3, 0x0001},\n{0x000BC6, 0x0010},\n{0x000BC9, 0x0001},\n{0x000BCA, 0x0010},\n{0x000BCE, 0x0001},\n{0x000BD0, 0x0004},\n{0x000BD1, 0x0001},\n{0x000BD7, 0x0010},\n{0x000BD8, 0x0001},\n{0x000BE6, 0x0002},\n{0x000BF3, 0x0040},\n{0x000BFB, 0x0001},\n{0x000C00, 0x0010},\n{0x000C05, 0x0004},\n{0x000C0D, 0x0001},\n{0x000C0E, 0x0004},\n{0x000C11, 0x0001},\n{0x000C12, 0x0004},\n{0x000C29, 0x0001},\n{0x000C2A, 0x0004},\n{0x000C3A, 0x0001},\n{0x000C3C, 0x0010},\n{0x000C3D, 0x0004},\n{0x000C3E, 0x0010},\n{0x000C45, 0x0001},\n{0x000C46, 0x0010},\n{0x000C49, 0x0001},\n{0x000C4A, 0x0010},\n{0x000C4E, 0x0001},\n{0x000C55, 0x0010},\n{0x000C57, 0x0001},\n{0x000C58, 0x0004},\n{0x000C5B, 0x0001},\n{0x000C5D, 0x0004},\n{0x000C5E, 0x0001},\n{0x000C60, 0x0004},\n{0x000C62, 0x0010},\n{0x000C64, 0x0001},\n{0x000C66, 0x0002},\n{0x000C70, 0x0001},\n{0x000C77, 0x0020},\n{0x000C78, 0x0002},\n{0x000C7F, 0x0040},\n{0x000C80, 0x0004},\n{0x000C81, 0x0010},\n{0x000C84, 0x0020},\n{0x000C85, 0x0004},\n{0x000C8D, 0x0001},\n{0x000C8E, 0x0004},\n{0x000C91, 0x0001},\n{0x000C92, 0x0004},\n{0x000CA9, 0x0001},\n{0x000CAA, 0x0004},\n{0x000CB4, 0x0001},\n{0x000CB5, 0x0004},\n{0x000CBA, 0x0001},\n{0x000CBC, 0x0010},\n{0x000CBD, 0x0004},\n{0x000CBE, 0x0010},\n{0x000CC5, 0x0001},\n{0x000CC6, 0x0010},\n{0x000CC9, 0x0001},\n{0x000CCA, 0x0010},\n{0x000CCE, 0x0001},\n{0x000CD5, 0x0010},\n{0x000CD7, 0x0001},\n{0x000CDD, 0x0004},\n{0x000CDF, 0x0001},\n{0x000CE0, 0x0004},\n{0x000CE2, 0x0010},\n{0x000CE4, 0x0001},\n{0x000CE6, 0x0002},\n{0x000CF0, 0x0001},\n{0x000CF1, 0x0004},\n{0x000CF3, 0x0010},\n{0x000CF4, 0x0001},\n{0x000D00, 0x0010},\n{0x000D04, 0x0004},\n{0x000D0D, 0x0001},\n{0x000D0E, 0x0004},\n{0x000D11, 0x0001},\n{0x000D12, 0x0004},\n{0x000D3B, 0x0010},\n{0x000D3D, 0x0004},\n{0x000D3E, 0x0010},\n{0x000D45, 0x0001},\n{0x000D46, 0x0010},\n{0x000D49, 0x0001},\n{0x000D4A, 0x0010},\n{0x000D4E, 0x0004},\n{0x000D4F, 0x0040},\n{0x000D50, 0x0001},\n{0x000D54, 0x0004},\n{0x000D57, 0x0010},\n{0x000D58, 0x0002},\n{0x000D5F, 0x0004},\n{0x000D62, 0x0010},\n{0x000D64, 0x0001},\n{0x000D66, 0x0002},\n{0x000D79, 0x0040},\n{0x000D7A, 0x0004},\n{0x000D80, 0x0001},\n{0x000D81, 0x0010},\n{0x000D84, 0x0001},\n{0x000D85, 0x0004},\n{0x000D97, 0x0001},\n{0x000D9A, 0x0004},\n{0x000DB2, 0x0001},\n{0x000DB3, 0x0004},\n{0x000DBC, 0x0001},\n{0x000DBD, 0x0004},\n{0x000DBE, 0x0001},\n{0x000DC0, 0x0004},\n{0x000DC7, 0x0001},\n{0x000DCA, 0x0010},\n{0x000DCB, 0x0001},\n{0x000DCF, 0x0010},\n{0x000DD5, 0x0001},\n{0x000DD6, 0x0010},\n{0x000DD7, 0x0001},\n{0x000DD8, 0x0010},\n{0x000DE0, 0x0001},\n{0x000DE6, 0x0002},\n{0x000DF0, 0x0001},\n{0x000DF2, 0x0010},\n{0x000DF4, 0x0020},\n{0x000DF5, 0x0001},\n{0x000E01, 0x0004},\n{0x000E31, 0x0010},\n{0x000E32, 0x0004},\n{0x000E34, 0x0010},\n{0x000E3B, 0x0001},\n{0x000E3F, 0x0040},\n{0x000E40, 0x0004},\n{0x000E47, 0x0010},\n{0x000E4F, 0x0020},\n{0x000E50, 0x0002},\n{0x000E5A, 0x0020},\n{0x000E5C, 0x0001},\n{0x000E81, 0x0004},\n{0x000E83, 0x0001},\n{0x000E84, 0x0004},\n{0x000E85, 0x0001},\n{0x000E86, 0x0004},\n{0x000E8B, 0x0001},\n{0x000E8C, 0x0004},\n{0x000EA4, 0x0001},\n{0x000EA5, 0x0004},\n{0x000EA6, 0x0001},\n{0x000EA7, 0x0004},\n{0x000EB1, 0x0010},\n{0x000EB2, 0x0004},\n{0x000EB4, 0x0010},\n{0x000EBD, 0x0004},\n{0x000EBE, 0x0001},\n{0x000EC0, 0x0004},\n{0x000EC5, 0x0001},\n{0x000EC6, 0x0004},\n{0x000EC7, 0x0001},\n{0x000EC8, 0x0010},\n{0x000ECF, 0x0001},\n{0x000ED0, 0x0002},\n{0x000EDA, 0x0001},\n{0x000EDC, 0x0004},\n{0x000EE0, 0x0001},\n{0x000F00, 0x0004},\n{0x000F01, 0x0040},\n{0x000F04, 0x0020},\n{0x000F13, 0x0040},\n{0x000F14, 0x0020},\n{0x000F15, 0x0040},\n{0x000F18, 0x0010},\n{0x000F1A, 0x0040},\n{0x000F20, 0x0002},\n{0x000F34, 0x0040},\n{0x000F35, 0x0010},\n{0x000F36, 0x0040},\n{0x000F37, 0x0010},\n{0x000F38, 0x0040},\n{0x000F39, 0x0010},\n{0x000F3A, 0x0020},\n{0x000F3E, 0x0010},\n{0x000F40, 0x0004},\n{0x000F48, 0x0001},\n{0x000F49, 0x0004},\n{0x000F6D, 0x0001},\n{0x000F71, 0x0010},\n{0x000F85, 0x0020},\n{0x000F86, 0x0010},\n{0x000F88, 0x0004},\n{0x000F8D, 0x0010},\n{0x000F98, 0x0001},\n{0x000F99, 0x0010},\n{0x000FBD, 0x0001},\n{0x000FBE, 0x0040},\n{0x000FC6, 0x0010},\n{0x000FC7, 0x0040},\n{0x000FCD, 0x0001},\n{0x000FCE, 0x0040},\n{0x000FD0, 0x0020},\n{0x000FD5, 0x0040},\n{0x000FD9, 0x0020},\n{0x000FDB, 0x0001},\n{0x001000, 0x0004},\n{0x00102B, 0x0010},\n{0x00103F, 0x0004},\n{0x001040, 0x0002},\n{0x00104A, 0x0020},\n{0x001050, 0x0004},\n{0x001056, 0x0010},\n{0x00105A, 0x0004},\n{0x00105E, 0x0010},\n{0x001061, 0x0004},\n{0x001062, 0x0010},\n{0x001065, 0x0004},\n{0x001067, 0x0010},\n{0x00106E, 0x0004},\n{0x001071, 0x0010},\n{0x001075, 0x0004},\n{0x001082, 0x0010},\n{0x00108E, 0x0004},\n{0x00108F, 0x0010},\n{0x001090, 0x0002},\n{0x00109A, 0x0010},\n{0x00109E, 0x0040},\n{0x0010A0, 0x0004},\n{0x0010C6, 0x0001},\n{0x0010C7, 0x0004},\n{0x0010C8, 0x0001},\n{0x0010CD, 0x0004},\n{0x0010CE, 0x0001},\n{0x0010D0, 0x0004},\n{0x0010FB, 0x0020},\n{0x0010FC, 0x0004},\n{0x001249, 0x0001},\n{0x00124A, 0x0004},\n{0x00124E, 0x0001},\n{0x001250, 0x0004},\n{0x001257, 0x0001},\n{0x001258, 0x0004},\n{0x001259, 0x0001},\n{0x00125A, 0x0004},\n{0x00125E, 0x0001},\n{0x001260, 0x0004},\n{0x001289, 0x0001},\n{0x00128A, 0x0004},\n{0x00128E, 0x0001},\n{0x001290, 0x0004},\n{0x0012B1, 0x0001},\n{0x0012B2, 0x0004},\n{0x0012B6, 0x0001},\n{0x0012B8, 0x0004},\n{0x0012BF, 0x0001},\n{0x0012C0, 0x0004},\n{0x0012C1, 0x0001},\n{0x0012C2, 0x0004},\n{0x0012C6, 0x0001},\n{0x0012C8, 0x0004},\n{0x0012D7, 0x0001},\n{0x0012D8, 0x0004},\n{0x001311, 0x0001},\n{0x001312, 0x0004},\n{0x001316, 0x0001},\n{0x001318, 0x0004},\n{0x00135B, 0x0001},\n{0x00135D, 0x0010},\n{0x001360, 0x0020},\n{0x001369, 0x0002},\n{0x00137D, 0x0001},\n{0x001380, 0x0004},\n{0x001390, 0x0040},\n{0x00139A, 0x0001},\n{0x0013A0, 0x0004},\n{0x0013F6, 0x0001},\n{0x0013F8, 0x0004},\n{0x0013FE, 0x0001},\n{0x001400, 0x0020},\n{0x001401, 0x0004},\n{0x00166D, 0x0040},\n{0x00166E, 0x0020},\n{0x00166F, 0x0004},\n{0x001680, 0x0008},\n{0x001681, 0x0004},\n{0x00169B, 0x0020},\n{0x00169D, 0x0001},\n{0x0016A0, 0x0004},\n{0x0016EB, 0x0020},\n{0x0016EE, 0x0002},\n{0x0016F1, 0x0004},\n{0x0016F9, 0x0001},\n{0x001700, 0x0004},\n{0x001712, 0x0010},\n{0x001716, 0x0001},\n{0x00171F, 0x0004},\n{0x001732, 0x0010},\n{0x001735, 0x0020},\n{0x001737, 0x0001},\n{0x001740, 0x0004},\n{0x001752, 0x0010},\n{0x001754, 0x0001},\n{0x001760, 0x0004},\n{0x00176D, 0x0001},\n{0x00176E, 0x0004},\n{0x001771, 0x0001},\n{0x001772, 0x0010},\n{0x001774, 0x0001},\n{0x001780, 0x0004},\n{0x0017B4, 0x0010},\n{0x0017D4, 0x0020},\n{0x0017D7, 0x0004},\n{0x0017D8, 0x0020},\n{0x0017DB, 0x0040},\n{0x0017DC, 0x0004},\n{0x0017DD, 0x0010},\n{0x0017DE, 0x0001},\n{0x0017E0, 0x0002},\n{0x0017EA, 0x0001},\n{0x0017F0, 0x0002},\n{0x0017FA, 0x0001},\n{0x001800, 0x0020},\n{0x00180B, 0x0010},\n{0x00180E, 0x0080},\n{0x00180F, 0x0010},\n{0x001810, 0x0002},\n{0x00181A, 0x0001},\n{0x001820, 0x0004},\n{0x001879, 0x0001},\n{0x001880, 0x0004},\n{0x001885, 0x0010},\n{0x001887, 0x0004},\n{0x0018A9, 0x0010},\n{0x0018AA, 0x0004},\n{0x0018AB, 0x0001},\n{0x0018B0, 0x0004},\n{0x0018F6, 0x0001},\n{0x001900, 0x0004},\n{0x00191F, 0x0001},\n{0x001920, 0x0010},\n{0x00192C, 0x0001},\n{0x001930, 0x0010},\n{0x00193C, 0x0001},\n{0x001940, 0x0040},\n{0x001941, 0x0001},\n{0x001944, 0x0020},\n{0x001946, 0x0002},\n{0x001950, 0x0004},\n{0x00196E, 0x0001},\n{0x001970, 0x0004},\n{0x001975, 0x0001},\n{0x001980, 0x0004},\n{0x0019AC, 0x0001},\n{0x0019B0, 0x0004},\n{0x0019CA, 0x0001},\n{0x0019D0, 0x0002},\n{0x0019DB, 0x0001},\n{0x0019DE, 0x0040},\n{0x001A00, 0x0004},\n{0x001A17, 0x0010},\n{0x001A1C, 0x0001},\n{0x001A1E, 0x0020},\n{0x001A20, 0x0004},\n{0x001A55, 0x0010},\n{0x001A5F, 0x0001},\n{0x001A60, 0x0010},\n{0x001A7D, 0x0001},\n{0x001A7F, 0x0010},\n{0x001A80, 0x0002},\n{0x001A8A, 0x0001},\n{0x001A90, 0x0002},\n{0x001A9A, 0x0001},\n{0x001AA0, 0x0020},\n{0x001AA7, 0x0004},\n{0x001AA8, 0x0020},\n{0x001AAE, 0x0001},\n{0x001AB0, 0x0010},\n{0x001ACF, 0x0001},\n{0x001B00, 0x0010},\n{0x001B05, 0x0004},\n{0x001B34, 0x0010},\n{0x001B45, 0x0004},\n{0x001B4D, 0x0001},\n{0x001B50, 0x0002},\n{0x001B5A, 0x0020},\n{0x001B61, 0x0040},\n{0x001B6B, 0x0010},\n{0x001B74, 0x0040},\n{0x001B7D, 0x0020},\n{0x001B7F, 0x0001},\n{0x001B80, 0x0010},\n{0x001B83, 0x0004},\n{0x001BA1, 0x0010},\n{0x001BAE, 0x0004},\n{0x001BB0, 0x0002},\n{0x001BBA, 0x0004},\n{0x001BE6, 0x0010},\n{0x001BF4, 0x0001},\n{0x001BFC, 0x0020},\n{0x001C00, 0x0004},\n{0x001C24, 0x0010},\n{0x001C38, 0x0001},\n{0x001C3B, 0x0020},\n{0x001C40, 0x0002},\n{0x001C4A, 0x0001},\n{0x001C4D, 0x0004},\n{0x001C50, 0x0002},\n{0x001C5A, 0x0004},\n{0x001C7E, 0x0020},\n{0x001C80, 0x0004},\n{0x001C89, 0x0001},\n{0x001C90, 0x0004},\n{0x001CBB, 0x0001},\n{0x001CBD, 0x0004},\n{0x001CC0, 0x0020},\n{0x001CC8, 0x0001},\n{0x001CD0, 0x0010},\n{0x001CD3, 0x0020},\n{0x001CD4, 0x0010},\n{0x001CE9, 0x0004},\n{0x001CED, 0x0010},\n{0x001CEE, 0x0004},\n{0x001CF4, 0x0010},\n{0x001CF5, 0x0004},\n{0x001CF7, 0x0010},\n{0x001CFA, 0x0004},\n{0x001CFB, 0x0001},\n{0x001D00, 0x0004},\n{0x001DC0, 0x0010},\n{0x001E00, 0x0004},\n{0x001F16, 0x0001},\n{0x001F18, 0x0004},\n{0x001F1E, 0x0001},\n{0x001F20, 0x0004},\n{0x001F46, 0x0001},\n{0x001F48, 0x0004},\n{0x001F4E, 0x0001},\n{0x001F50, 0x0004},\n{0x001F58, 0x0001},\n{0x001F59, 0x0004},\n{0x001F5A, 0x0001},\n{0x001F5B, 0x0004},\n{0x001F5C, 0x0001},\n{0x001F5D, 0x0004},\n{0x001F5E, 0x0001},\n{0x001F5F, 0x0004},\n{0x001F7E, 0x0001},\n{0x001F80, 0x0004},\n{0x001FB5, 0x0001},\n{0x001FB6, 0x0004},\n{0x001FBD, 0x0040},\n{0x001FBE, 0x0004},\n{0x001FBF, 0x0040},\n{0x001FC2, 0x0004},\n{0x001FC5, 0x0001},\n{0x001FC6, 0x0004},\n{0x001FCD, 0x0040},\n{0x001FD0, 0x0004},\n{0x001FD4, 0x0001},\n{0x001FD6, 0x0004},\n{0x001FDC, 0x0001},\n{0x001FDD, 0x0040},\n{0x001FE0, 0x0004},\n{0x001FED, 0x0040},\n{0x001FF0, 0x0001},\n{0x001FF2, 0x0004},\n{0x001FF5, 0x0001},\n{0x001FF6, 0x0004},\n{0x001FFD, 0x0040},\n{0x001FFF, 0x0001},\n{0x002000, 0x0008},\n{0x00200B, 0x0080},\n{0x002010, 0x0020},\n{0x002028, 0x0008},\n{0x00202A, 0x0080},\n{0x00202F, 0x0008},\n{0x002030, 0x0020},\n{0x002044, 0x0040},\n{0x002045, 0x0020},\n{0x002052, 0x0040},\n{0x002053, 0x0020},\n{0x00205F, 0x0008},\n{0x002060, 0x0080},\n{0x002065, 0x0001},\n{0x002066, 0x0080},\n{0x002070, 0x0002},\n{0x002071, 0x0004},\n{0x002072, 0x0001},\n{0x002074, 0x0002},\n{0x00207A, 0x0040},\n{0x00207D, 0x0020},\n{0x00207F, 0x0004},\n{0x002080, 0x0002},\n{0x00208A, 0x0040},\n{0x00208D, 0x0020},\n{0x00208F, 0x0001},\n{0x002090, 0x0004},\n{0x00209D, 0x0001},\n{0x0020A0, 0x0040},\n{0x0020C1, 0x0001},\n{0x0020D0, 0x0010},\n{0x0020F1, 0x0001},\n{0x002100, 0x0040},\n{0x002102, 0x0004},\n{0x002103, 0x0040},\n{0x002107, 0x0004},\n{0x002108, 0x0040},\n{0x00210A, 0x0004},\n{0x002114, 0x0040},\n{0x002115, 0x0004},\n{0x002116, 0x0040},\n{0x002119, 0x0004},\n{0x00211E, 0x0040},\n{0x002124, 0x0004},\n{0x002125, 0x0040},\n{0x002126, 0x0004},\n{0x002127, 0x0040},\n{0x002128, 0x0004},\n{0x002129, 0x0040},\n{0x00212A, 0x0004},\n{0x00212E, 0x0040},\n{0x00212F, 0x0004},\n{0x00213A, 0x0040},\n{0x00213C, 0x0004},\n{0x002140, 0x0040},\n{0x002145, 0x0004},\n{0x00214A, 0x0040},\n{0x00214E, 0x0004},\n{0x00214F, 0x0040},\n{0x002150, 0x0002},\n{0x002183, 0x0004},\n{0x002185, 0x0002},\n{0x00218A, 0x0040},\n{0x00218C, 0x0001},\n{0x002190, 0x0040},\n{0x002308, 0x0020},\n{0x00230C, 0x0040},\n{0x002329, 0x0020},\n{0x00232B, 0x0040},\n{0x002427, 0x0001},\n{0x002440, 0x0040},\n{0x00244B, 0x0001},\n{0x002460, 0x0002},\n{0x00249C, 0x0040},\n{0x0024EA, 0x0002},\n{0x002500, 0x0040},\n{0x002768, 0x0020},\n{0x002776, 0x0002},\n{0x002794, 0x0040},\n{0x0027C5, 0x0020},\n{0x0027C7, 0x0040},\n{0x0027E6, 0x0020},\n{0x0027F0, 0x0040},\n{0x002983, 0x0020},\n{0x002999, 0x0040},\n{0x0029D8, 0x0020},\n{0x0029DC, 0x0040},\n{0x0029FC, 0x0020},\n{0x0029FE, 0x0040},\n{0x002B74, 0x0001},\n{0x002B76, 0x0040},\n{0x002B96, 0x0001},\n{0x002B97, 0x0040},\n{0x002C00, 0x0004},\n{0x002CE5, 0x0040},\n{0x002CEB, 0x0004},\n{0x002CEF, 0x0010},\n{0x002CF2, 0x0004},\n{0x002CF4, 0x0001},\n{0x002CF9, 0x0020},\n{0x002CFD, 0x0002},\n{0x002CFE, 0x0020},\n{0x002D00, 0x0004},\n{0x002D26, 0x0001},\n{0x002D27, 0x0004},\n{0x002D28, 0x0001},\n{0x002D2D, 0x0004},\n{0x002D2E, 0x0001},\n{0x002D30, 0x0004},\n{0x002D68, 0x0001},\n{0x002D6F, 0x0004},\n{0x002D70, 0x0020},\n{0x002D71, 0x0001},\n{0x002D7F, 0x0010},\n{0x002D80, 0x0004},\n{0x002D97, 0x0001},\n{0x002DA0, 0x0004},\n{0x002DA7, 0x0001},\n{0x002DA8, 0x0004},\n{0x002DAF, 0x0001},\n{0x002DB0, 0x0004},\n{0x002DB7, 0x0001},\n{0x002DB8, 0x0004},\n{0x002DBF, 0x0001},\n{0x002DC0, 0x0004},\n{0x002DC7, 0x0001},\n{0x002DC8, 0x0004},\n{0x002DCF, 0x0001},\n{0x002DD0, 0x0004},\n{0x002DD7, 0x0001},\n{0x002DD8, 0x0004},\n{0x002DDF, 0x0001},\n{0x002DE0, 0x0010},\n{0x002E00, 0x0020},\n{0x002E2F, 0x0004},\n{0x002E30, 0x0020},\n{0x002E50, 0x0040},\n{0x002E52, 0x0020},\n{0x002E5E, 0x0001},\n{0x002E80, 0x0040},\n{0x002E9A, 0x0001},\n{0x002E9B, 0x0040},\n{0x002EF4, 0x0001},\n{0x002F00, 0x0040},\n{0x002FD6, 0x0001},\n{0x002FF0, 0x0040},\n{0x003000, 0x0008},\n{0x003001, 0x0020},\n{0x003004, 0x0040},\n{0x003005, 0x0004},\n{0x003007, 0x0002},\n{0x003008, 0x0020},\n{0x003012, 0x0040},\n{0x003014, 0x0020},\n{0x003020, 0x0040},\n{0x003021, 0x0002},\n{0x00302A, 0x0010},\n{0x003030, 0x0020},\n{0x003031, 0x0004},\n{0x003036, 0x0040},\n{0x003038, 0x0002},\n{0x00303B, 0x0004},\n{0x00303D, 0x0020},\n{0x00303E, 0x0040},\n{0x003040, 0x0001},\n{0x003041, 0x0004},\n{0x003097, 0x0001},\n{0x003099, 0x0010},\n{0x00309B, 0x0040},\n{0x00309D, 0x0004},\n{0x0030A0, 0x0020},\n{0x0030A1, 0x0004},\n{0x0030FB, 0x0020},\n{0x0030FC, 0x0004},\n{0x003100, 0x0001},\n{0x003105, 0x0004},\n{0x003130, 0x0001},\n{0x003131, 0x0004},\n{0x00318F, 0x0001},\n{0x003190, 0x0040},\n{0x003192, 0x0002},\n{0x003196, 0x0040},\n{0x0031A0, 0x0004},\n{0x0031C0, 0x0040},\n{0x0031E4, 0x0001},\n{0x0031EF, 0x0040},\n{0x0031F0, 0x0004},\n{0x003200, 0x0040},\n{0x00321F, 0x0001},\n{0x003220, 0x0002},\n{0x00322A, 0x0040},\n{0x003248, 0x0002},\n{0x003250, 0x0040},\n{0x003251, 0x0002},\n{0x003260, 0x0040},\n{0x003280, 0x0002},\n{0x00328A, 0x0040},\n{0x0032B1, 0x0002},\n{0x0032C0, 0x0040},\n{0x003400, 0x0004},\n{0x004DC0, 0x0040},\n{0x004E00, 0x0004},\n{0x00A48D, 0x0001},\n{0x00A490, 0x0040},\n{0x00A4C7, 0x0001},\n{0x00A4D0, 0x0004},\n{0x00A4FE, 0x0020},\n{0x00A500, 0x0004},\n{0x00A60D, 0x0020},\n{0x00A610, 0x0004},\n{0x00A620, 0x0002},\n{0x00A62A, 0x0004},\n{0x00A62C, 0x0001},\n{0x00A640, 0x0004},\n{0x00A66F, 0x0010},\n{0x00A673, 0x0020},\n{0x00A674, 0x0010},\n{0x00A67E, 0x0020},\n{0x00A67F, 0x0004},\n{0x00A69E, 0x0010},\n{0x00A6A0, 0x0004},\n{0x00A6E6, 0x0002},\n{0x00A6F0, 0x0010},\n{0x00A6F2, 0x0020},\n{0x00A6F8, 0x0001},\n{0x00A700, 0x0040},\n{0x00A717, 0x0004},\n{0x00A720, 0x0040},\n{0x00A722, 0x0004},\n{0x00A789, 0x0040},\n{0x00A78B, 0x0004},\n{0x00A7CB, 0x0001},\n{0x00A7D0, 0x0004},\n{0x00A7D2, 0x0001},\n{0x00A7D3, 0x0004},\n{0x00A7D4, 0x0001},\n{0x00A7D5, 0x0004},\n{0x00A7DA, 0x0001},\n{0x00A7F2, 0x0004},\n{0x00A802, 0x0010},\n{0x00A803, 0x0004},\n{0x00A806, 0x0010},\n{0x00A807, 0x0004},\n{0x00A80B, 0x0010},\n{0x00A80C, 0x0004},\n{0x00A823, 0x0010},\n{0x00A828, 0x0040},\n{0x00A82C, 0x0010},\n{0x00A82D, 0x0001},\n{0x00A830, 0x0002},\n{0x00A836, 0x0040},\n{0x00A83A, 0x0001},\n{0x00A840, 0x0004},\n{0x00A874, 0x0020},\n{0x00A878, 0x0001},\n{0x00A880, 0x0010},\n{0x00A882, 0x0004},\n{0x00A8B4, 0x0010},\n{0x00A8C6, 0x0001},\n{0x00A8CE, 0x0020},\n{0x00A8D0, 0x0002},\n{0x00A8DA, 0x0001},\n{0x00A8E0, 0x0010},\n{0x00A8F2, 0x0004},\n{0x00A8F8, 0x0020},\n{0x00A8FB, 0x0004},\n{0x00A8FC, 0x0020},\n{0x00A8FD, 0x0004},\n{0x00A8FF, 0x0010},\n{0x00A900, 0x0002},\n{0x00A90A, 0x0004},\n{0x00A926, 0x0010},\n{0x00A92E, 0x0020},\n{0x00A930, 0x0004},\n{0x00A947, 0x0010},\n{0x00A954, 0x0001},\n{0x00A95F, 0x0020},\n{0x00A960, 0x0004},\n{0x00A97D, 0x0001},\n{0x00A980, 0x0010},\n{0x00A984, 0x0004},\n{0x00A9B3, 0x0010},\n{0x00A9C1, 0x0020},\n{0x00A9CE, 0x0001},\n{0x00A9CF, 0x0004},\n{0x00A9D0, 0x0002},\n{0x00A9DA, 0x0001},\n{0x00A9DE, 0x0020},\n{0x00A9E0, 0x0004},\n{0x00A9E5, 0x0010},\n{0x00A9E6, 0x0004},\n{0x00A9F0, 0x0002},\n{0x00A9FA, 0x0004},\n{0x00A9FF, 0x0001},\n{0x00AA00, 0x0004},\n{0x00AA29, 0x0010},\n{0x00AA37, 0x0001},\n{0x00AA40, 0x0004},\n{0x00AA43, 0x0010},\n{0x00AA44, 0x0004},\n{0x00AA4C, 0x0010},\n{0x00AA4E, 0x0001},\n{0x00AA50, 0x0002},\n{0x00AA5A, 0x0001},\n{0x00AA5C, 0x0020},\n{0x00AA60, 0x0004},\n{0x00AA77, 0x0040},\n{0x00AA7A, 0x0004},\n{0x00AA7B, 0x0010},\n{0x00AA7E, 0x0004},\n{0x00AAB0, 0x0010},\n{0x00AAB1, 0x0004},\n{0x00AAB2, 0x0010},\n{0x00AAB5, 0x0004},\n{0x00AAB7, 0x0010},\n{0x00AAB9, 0x0004},\n{0x00AABE, 0x0010},\n{0x00AAC0, 0x0004},\n{0x00AAC1, 0x0010},\n{0x00AAC2, 0x0004},\n{0x00AAC3, 0x0001},\n{0x00AADB, 0x0004},\n{0x00AADE, 0x0020},\n{0x00AAE0, 0x0004},\n{0x00AAEB, 0x0010},\n{0x00AAF0, 0x0020},\n{0x00AAF2, 0x0004},\n{0x00AAF5, 0x0010},\n{0x00AAF7, 0x0001},\n{0x00AB01, 0x0004},\n{0x00AB07, 0x0001},\n{0x00AB09, 0x0004},\n{0x00AB0F, 0x0001},\n{0x00AB11, 0x0004},\n{0x00AB17, 0x0001},\n{0x00AB20, 0x0004},\n{0x00AB27, 0x0001},\n{0x00AB28, 0x0004},\n{0x00AB2F, 0x0001},\n{0x00AB30, 0x0004},\n{0x00AB5B, 0x0040},\n{0x00AB5C, 0x0004},\n{0x00AB6A, 0x0040},\n{0x00AB6C, 0x0001},\n{0x00AB70, 0x0004},\n{0x00ABE3, 0x0010},\n{0x00ABEB, 0x0020},\n{0x00ABEC, 0x0010},\n{0x00ABEE, 0x0001},\n{0x00ABF0, 0x0002},\n{0x00ABFA, 0x0001},\n{0x00AC00, 0x0004},\n{0x00D7A4, 0x0001},\n{0x00D7B0, 0x0004},\n{0x00D7C7, 0x0001},\n{0x00D7CB, 0x0004},\n{0x00D7FC, 0x0001},\n{0x00D800, 0x0080},\n{0x00F900, 0x0004},\n{0x00FA6E, 0x0001},\n{0x00FA70, 0x0004},\n{0x00FADA, 0x0001},\n{0x00FB00, 0x0004},\n{0x00FB07, 0x0001},\n{0x00FB13, 0x0004},\n{0x00FB18, 0x0001},\n{0x00FB1D, 0x0004},\n{0x00FB1E, 0x0010},\n{0x00FB1F, 0x0004},\n{0x00FB29, 0x0040},\n{0x00FB2A, 0x0004},\n{0x00FB37, 0x0001},\n{0x00FB38, 0x0004},\n{0x00FB3D, 0x0001},\n{0x00FB3E, 0x0004},\n{0x00FB3F, 0x0001},\n{0x00FB40, 0x0004},\n{0x00FB42, 0x0001},\n{0x00FB43, 0x0004},\n{0x00FB45, 0x0001},\n{0x00FB46, 0x0004},\n{0x00FBB2, 0x0040},\n{0x00FBC3, 0x0001},\n{0x00FBD3, 0x0004},\n{0x00FD3E, 0x0020},\n{0x00FD40, 0x0040},\n{0x00FD50, 0x0004},\n{0x00FD90, 0x0001},\n{0x00FD92, 0x0004},\n{0x00FDC8, 0x0001},\n{0x00FDCF, 0x0040},\n{0x00FDD0, 0x0001},\n{0x00FDF0, 0x0004},\n{0x00FDFC, 0x0040},\n{0x00FE00, 0x0010},\n{0x00FE10, 0x0020},\n{0x00FE1A, 0x0001},\n{0x00FE20, 0x0010},\n{0x00FE30, 0x0020},\n{0x00FE53, 0x0001},\n{0x00FE54, 0x0020},\n{0x00FE62, 0x0040},\n{0x00FE63, 0x0020},\n{0x00FE64, 0x0040},\n{0x00FE67, 0x0001},\n{0x00FE68, 0x0020},\n{0x00FE69, 0x0040},\n{0x00FE6A, 0x0020},\n{0x00FE6C, 0x0001},\n{0x00FE70, 0x0004},\n{0x00FE75, 0x0001},\n{0x00FE76, 0x0004},\n{0x00FEFD, 0x0001},\n{0x00FEFF, 0x0080},\n{0x00FF00, 0x0001},\n{0x00FF01, 0x0020},\n{0x00FF04, 0x0040},\n{0x00FF05, 0x0020},\n{0x00FF0B, 0x0040},\n{0x00FF0C, 0x0020},\n{0x00FF10, 0x0002},\n{0x00FF1A, 0x0020},\n{0x00FF1C, 0x0040},\n{0x00FF1F, 0x0020},\n{0x00FF21, 0x0004},\n{0x00FF3B, 0x0020},\n{0x00FF3E, 0x0040},\n{0x00FF3F, 0x0020},\n{0x00FF40, 0x0040},\n{0x00FF41, 0x0004},\n{0x00FF5B, 0x0020},\n{0x00FF5C, 0x0040},\n{0x00FF5D, 0x0020},\n{0x00FF5E, 0x0040},\n{0x00FF5F, 0x0020},\n{0x00FF66, 0x0004},\n{0x00FFBF, 0x0001},\n{0x00FFC2, 0x0004},\n{0x00FFC8, 0x0001},\n{0x00FFCA, 0x0004},\n{0x00FFD0, 0x0001},\n{0x00FFD2, 0x0004},\n{0x00FFD8, 0x0001},\n{0x00FFDA, 0x0004},\n{0x00FFDD, 0x0001},\n{0x00FFE0, 0x0040},\n{0x00FFE7, 0x0001},\n{0x00FFE8, 0x0040},\n{0x00FFEF, 0x0001},\n{0x00FFF9, 0x0080},\n{0x00FFFC, 0x0040},\n{0x00FFFE, 0x0001},\n{0x010000, 0x0004},\n{0x01000C, 0x0001},\n{0x01000D, 0x0004},\n{0x010027, 0x0001},\n{0x010028, 0x0004},\n{0x01003B, 0x0001},\n{0x01003C, 0x0004},\n{0x01003E, 0x0001},\n{0x01003F, 0x0004},\n{0x01004E, 0x0001},\n{0x010050, 0x0004},\n{0x01005E, 0x0001},\n{0x010080, 0x0004},\n{0x0100FB, 0x0001},\n{0x010100, 0x0020},\n{0x010103, 0x0001},\n{0x010107, 0x0002},\n{0x010134, 0x0001},\n{0x010137, 0x0040},\n{0x010140, 0x0002},\n{0x010179, 0x0040},\n{0x01018A, 0x0002},\n{0x01018C, 0x0040},\n{0x01018F, 0x0001},\n{0x010190, 0x0040},\n{0x01019D, 0x0001},\n{0x0101A0, 0x0040},\n{0x0101A1, 0x0001},\n{0x0101D0, 0x0040},\n{0x0101FD, 0x0010},\n{0x0101FE, 0x0001},\n{0x010280, 0x0004},\n{0x01029D, 0x0001},\n{0x0102A0, 0x0004},\n{0x0102D1, 0x0001},\n{0x0102E0, 0x0010},\n{0x0102E1, 0x0002},\n{0x0102FC, 0x0001},\n{0x010300, 0x0004},\n{0x010320, 0x0002},\n{0x010324, 0x0001},\n{0x01032D, 0x0004},\n{0x010341, 0x0002},\n{0x010342, 0x0004},\n{0x01034A, 0x0002},\n{0x01034B, 0x0001},\n{0x010350, 0x0004},\n{0x010376, 0x0010},\n{0x01037B, 0x0001},\n{0x010380, 0x0004},\n{0x01039E, 0x0001},\n{0x01039F, 0x0020},\n{0x0103A0, 0x0004},\n{0x0103C4, 0x0001},\n{0x0103C8, 0x0004},\n{0x0103D0, 0x0020},\n{0x0103D1, 0x0002},\n{0x0103D6, 0x0001},\n{0x010400, 0x0004},\n{0x01049E, 0x0001},\n{0x0104A0, 0x0002},\n{0x0104AA, 0x0001},\n{0x0104B0, 0x0004},\n{0x0104D4, 0x0001},\n{0x0104D8, 0x0004},\n{0x0104FC, 0x0001},\n{0x010500, 0x0004},\n{0x010528, 0x0001},\n{0x010530, 0x0004},\n{0x010564, 0x0001},\n{0x01056F, 0x0020},\n{0x010570, 0x0004},\n{0x01057B, 0x0001},\n{0x01057C, 0x0004},\n{0x01058B, 0x0001},\n{0x01058C, 0x0004},\n{0x010593, 0x0001},\n{0x010594, 0x0004},\n{0x010596, 0x0001},\n{0x010597, 0x0004},\n{0x0105A2, 0x0001},\n{0x0105A3, 0x0004},\n{0x0105B2, 0x0001},\n{0x0105B3, 0x0004},\n{0x0105BA, 0x0001},\n{0x0105BB, 0x0004},\n{0x0105BD, 0x0001},\n{0x010600, 0x0004},\n{0x010737, 0x0001},\n{0x010740, 0x0004},\n{0x010756, 0x0001},\n{0x010760, 0x0004},\n{0x010768, 0x0001},\n{0x010780, 0x0004},\n{0x010786, 0x0001},\n{0x010787, 0x0004},\n{0x0107B1, 0x0001},\n{0x0107B2, 0x0004},\n{0x0107BB, 0x0001},\n{0x010800, 0x0004},\n{0x010806, 0x0001},\n{0x010808, 0x0004},\n{0x010809, 0x0001},\n{0x01080A, 0x0004},\n{0x010836, 0x0001},\n{0x010837, 0x0004},\n{0x010839, 0x0001},\n{0x01083C, 0x0004},\n{0x01083D, 0x0001},\n{0x01083F, 0x0004},\n{0x010856, 0x0001},\n{0x010857, 0x0020},\n{0x010858, 0x0002},\n{0x010860, 0x0004},\n{0x010877, 0x0040},\n{0x010879, 0x0002},\n{0x010880, 0x0004},\n{0x01089F, 0x0001},\n{0x0108A7, 0x0002},\n{0x0108B0, 0x0001},\n{0x0108E0, 0x0004},\n{0x0108F3, 0x0001},\n{0x0108F4, 0x0004},\n{0x0108F6, 0x0001},\n{0x0108FB, 0x0002},\n{0x010900, 0x0004},\n{0x010916, 0x0002},\n{0x01091C, 0x0001},\n{0x01091F, 0x0020},\n{0x010920, 0x0004},\n{0x01093A, 0x0001},\n{0x01093F, 0x0020},\n{0x010940, 0x0001},\n{0x010980, 0x0004},\n{0x0109B8, 0x0001},\n{0x0109BC, 0x0002},\n{0x0109BE, 0x0004},\n{0x0109C0, 0x0002},\n{0x0109D0, 0x0001},\n{0x0109D2, 0x0002},\n{0x010A00, 0x0004},\n{0x010A01, 0x0010},\n{0x010A04, 0x0001},\n{0x010A05, 0x0010},\n{0x010A07, 0x0001},\n{0x010A0C, 0x0010},\n{0x010A10, 0x0004},\n{0x010A14, 0x0001},\n{0x010A15, 0x0004},\n{0x010A18, 0x0001},\n{0x010A19, 0x0004},\n{0x010A36, 0x0001},\n{0x010A38, 0x0010},\n{0x010A3B, 0x0001},\n{0x010A3F, 0x0010},\n{0x010A40, 0x0002},\n{0x010A49, 0x0001},\n{0x010A50, 0x0020},\n{0x010A59, 0x0001},\n{0x010A60, 0x0004},\n{0x010A7D, 0x0002},\n{0x010A7F, 0x0020},\n{0x010A80, 0x0004},\n{0x010A9D, 0x0002},\n{0x010AA0, 0x0001},\n{0x010AC0, 0x0004},\n{0x010AC8, 0x0040},\n{0x010AC9, 0x0004},\n{0x010AE5, 0x0010},\n{0x010AE7, 0x0001},\n{0x010AEB, 0x0002},\n{0x010AF0, 0x0020},\n{0x010AF7, 0x0001},\n{0x010B00, 0x0004},\n{0x010B36, 0x0001},\n{0x010B39, 0x0020},\n{0x010B40, 0x0004},\n{0x010B56, 0x0001},\n{0x010B58, 0x0002},\n{0x010B60, 0x0004},\n{0x010B73, 0x0001},\n{0x010B78, 0x0002},\n{0x010B80, 0x0004},\n{0x010B92, 0x0001},\n{0x010B99, 0x0020},\n{0x010B9D, 0x0001},\n{0x010BA9, 0x0002},\n{0x010BB0, 0x0001},\n{0x010C00, 0x0004},\n{0x010C49, 0x0001},\n{0x010C80, 0x0004},\n{0x010CB3, 0x0001},\n{0x010CC0, 0x0004},\n{0x010CF3, 0x0001},\n{0x010CFA, 0x0002},\n{0x010D00, 0x0004},\n{0x010D24, 0x0010},\n{0x010D28, 0x0001},\n{0x010D30, 0x0002},\n{0x010D3A, 0x0001},\n{0x010E60, 0x0002},\n{0x010E7F, 0x0001},\n{0x010E80, 0x0004},\n{0x010EAA, 0x0001},\n{0x010EAB, 0x0010},\n{0x010EAD, 0x0020},\n{0x010EAE, 0x0001},\n{0x010EB0, 0x0004},\n{0x010EB2, 0x0001},\n{0x010EFD, 0x0010},\n{0x010F00, 0x0004},\n{0x010F1D, 0x0002},\n{0x010F27, 0x0004},\n{0x010F28, 0x0001},\n{0x010F30, 0x0004},\n{0x010F46, 0x0010},\n{0x010F51, 0x0002},\n{0x010F55, 0x0020},\n{0x010F5A, 0x0001},\n{0x010F70, 0x0004},\n{0x010F82, 0x0010},\n{0x010F86, 0x0020},\n{0x010F8A, 0x0001},\n{0x010FB0, 0x0004},\n{0x010FC5, 0x0002},\n{0x010FCC, 0x0001},\n{0x010FE0, 0x0004},\n{0x010FF7, 0x0001},\n{0x011000, 0x0010},\n{0x011003, 0x0004},\n{0x011038, 0x0010},\n{0x011047, 0x0020},\n{0x01104E, 0x0001},\n{0x011052, 0x0002},\n{0x011070, 0x0010},\n{0x011071, 0x0004},\n{0x011073, 0x0010},\n{0x011075, 0x0004},\n{0x011076, 0x0001},\n{0x01107F, 0x0010},\n{0x011083, 0x0004},\n{0x0110B0, 0x0010},\n{0x0110BB, 0x0020},\n{0x0110BD, 0x0080},\n{0x0110BE, 0x0020},\n{0x0110C2, 0x0010},\n{0x0110C3, 0x0001},\n{0x0110CD, 0x0080},\n{0x0110CE, 0x0001},\n{0x0110D0, 0x0004},\n{0x0110E9, 0x0001},\n{0x0110F0, 0x0002},\n{0x0110FA, 0x0001},\n{0x011100, 0x0010},\n{0x011103, 0x0004},\n{0x011127, 0x0010},\n{0x011135, 0x0001},\n{0x011136, 0x0002},\n{0x011140, 0x0020},\n{0x011144, 0x0004},\n{0x011145, 0x0010},\n{0x011147, 0x0004},\n{0x011148, 0x0001},\n{0x011150, 0x0004},\n{0x011173, 0x0010},\n{0x011174, 0x0020},\n{0x011176, 0x0004},\n{0x011177, 0x0001},\n{0x011180, 0x0010},\n{0x011183, 0x0004},\n{0x0111B3, 0x0010},\n{0x0111C1, 0x0004},\n{0x0111C5, 0x0020},\n{0x0111C9, 0x0010},\n{0x0111CD, 0x0020},\n{0x0111CE, 0x0010},\n{0x0111D0, 0x0002},\n{0x0111DA, 0x0004},\n{0x0111DB, 0x0020},\n{0x0111DC, 0x0004},\n{0x0111DD, 0x0020},\n{0x0111E0, 0x0001},\n{0x0111E1, 0x0002},\n{0x0111F5, 0x0001},\n{0x011200, 0x0004},\n{0x011212, 0x0001},\n{0x011213, 0x0004},\n{0x01122C, 0x0010},\n{0x011238, 0x0020},\n{0x01123E, 0x0010},\n{0x01123F, 0x0004},\n{0x011241, 0x0010},\n{0x011242, 0x0001},\n{0x011280, 0x0004},\n{0x011287, 0x0001},\n{0x011288, 0x0004},\n{0x011289, 0x0001},\n{0x01128A, 0x0004},\n{0x01128E, 0x0001},\n{0x01128F, 0x0004},\n{0x01129E, 0x0001},\n{0x01129F, 0x0004},\n{0x0112A9, 0x0020},\n{0x0112AA, 0x0001},\n{0x0112B0, 0x0004},\n{0x0112DF, 0x0010},\n{0x0112EB, 0x0001},\n{0x0112F0, 0x0002},\n{0x0112FA, 0x0001},\n{0x011300, 0x0010},\n{0x011304, 0x0001},\n{0x011305, 0x0004},\n{0x01130D, 0x0001},\n{0x01130F, 0x0004},\n{0x011311, 0x0001},\n{0x011313, 0x0004},\n{0x011329, 0x0001},\n{0x01132A, 0x0004},\n{0x011331, 0x0001},\n{0x011332, 0x0004},\n{0x011334, 0x0001},\n{0x011335, 0x0004},\n{0x01133A, 0x0001},\n{0x01133B, 0x0010},\n{0x01133D, 0x0004},\n{0x01133E, 0x0010},\n{0x011345, 0x0001},\n{0x011347, 0x0010},\n{0x011349, 0x0001},\n{0x01134B, 0x0010},\n{0x01134E, 0x0001},\n{0x011350, 0x0004},\n{0x011351, 0x0001},\n{0x011357, 0x0010},\n{0x011358, 0x0001},\n{0x01135D, 0x0004},\n{0x011362, 0x0010},\n{0x011364, 0x0001},\n{0x011366, 0x0010},\n{0x01136D, 0x0001},\n{0x011370, 0x0010},\n{0x011375, 0x0001},\n{0x011400, 0x0004},\n{0x011435, 0x0010},\n{0x011447, 0x0004},\n{0x01144B, 0x0020},\n{0x011450, 0x0002},\n{0x01145A, 0x0020},\n{0x01145C, 0x0001},\n{0x01145D, 0x0020},\n{0x01145E, 0x0010},\n{0x01145F, 0x0004},\n{0x011462, 0x0001},\n{0x011480, 0x0004},\n{0x0114B0, 0x0010},\n{0x0114C4, 0x0004},\n{0x0114C6, 0x0020},\n{0x0114C7, 0x0004},\n{0x0114C8, 0x0001},\n{0x0114D0, 0x0002},\n{0x0114DA, 0x0001},\n{0x011580, 0x0004},\n{0x0115AF, 0x0010},\n{0x0115B6, 0x0001},\n{0x0115B8, 0x0010},\n{0x0115C1, 0x0020},\n{0x0115D8, 0x0004},\n{0x0115DC, 0x0010},\n{0x0115DE, 0x0001},\n{0x011600, 0x0004},\n{0x011630, 0x0010},\n{0x011641, 0x0020},\n{0x011644, 0x0004},\n{0x011645, 0x0001},\n{0x011650, 0x0002},\n{0x01165A, 0x0001},\n{0x011660, 0x0020},\n{0x01166D, 0x0001},\n{0x011680, 0x0004},\n{0x0116AB, 0x0010},\n{0x0116B8, 0x0004},\n{0x0116B9, 0x0020},\n{0x0116BA, 0x0001},\n{0x0116C0, 0x0002},\n{0x0116CA, 0x0001},\n{0x011700, 0x0004},\n{0x01171B, 0x0001},\n{0x01171D, 0x0010},\n{0x01172C, 0x0001},\n{0x011730, 0x0002},\n{0x01173C, 0x0020},\n{0x01173F, 0x0040},\n{0x011740, 0x0004},\n{0x011747, 0x0001},\n{0x011800, 0x0004},\n{0x01182C, 0x0010},\n{0x01183B, 0x0020},\n{0x01183C, 0x0001},\n{0x0118A0, 0x0004},\n{0x0118E0, 0x0002},\n{0x0118F3, 0x0001},\n{0x0118FF, 0x0004},\n{0x011907, 0x0001},\n{0x011909, 0x0004},\n{0x01190A, 0x0001},\n{0x01190C, 0x0004},\n{0x011914, 0x0001},\n{0x011915, 0x0004},\n{0x011917, 0x0001},\n{0x011918, 0x0004},\n{0x011930, 0x0010},\n{0x011936, 0x0001},\n{0x011937, 0x0010},\n{0x011939, 0x0001},\n{0x01193B, 0x0010},\n{0x01193F, 0x0004},\n{0x011940, 0x0010},\n{0x011941, 0x0004},\n{0x011942, 0x0010},\n{0x011944, 0x0020},\n{0x011947, 0x0001},\n{0x011950, 0x0002},\n{0x01195A, 0x0001},\n{0x0119A0, 0x0004},\n{0x0119A8, 0x0001},\n{0x0119AA, 0x0004},\n{0x0119D1, 0x0010},\n{0x0119D8, 0x0001},\n{0x0119DA, 0x0010},\n{0x0119E1, 0x0004},\n{0x0119E2, 0x0020},\n{0x0119E3, 0x0004},\n{0x0119E4, 0x0010},\n{0x0119E5, 0x0001},\n{0x011A00, 0x0004},\n{0x011A01, 0x0010},\n{0x011A0B, 0x0004},\n{0x011A33, 0x0010},\n{0x011A3A, 0x0004},\n{0x011A3B, 0x0010},\n{0x011A3F, 0x0020},\n{0x011A47, 0x0010},\n{0x011A48, 0x0001},\n{0x011A50, 0x0004},\n{0x011A51, 0x0010},\n{0x011A5C, 0x0004},\n{0x011A8A, 0x0010},\n{0x011A9A, 0x0020},\n{0x011A9D, 0x0004},\n{0x011A9E, 0x0020},\n{0x011AA3, 0x0001},\n{0x011AB0, 0x0004},\n{0x011AF9, 0x0001},\n{0x011B00, 0x0020},\n{0x011B0A, 0x0001},\n{0x011C00, 0x0004},\n{0x011C09, 0x0001},\n{0x011C0A, 0x0004},\n{0x011C2F, 0x0010},\n{0x011C37, 0x0001},\n{0x011C38, 0x0010},\n{0x011C40, 0x0004},\n{0x011C41, 0x0020},\n{0x011C46, 0x0001},\n{0x011C50, 0x0002},\n{0x011C6D, 0x0001},\n{0x011C70, 0x0020},\n{0x011C72, 0x0004},\n{0x011C90, 0x0001},\n{0x011C92, 0x0010},\n{0x011CA8, 0x0001},\n{0x011CA9, 0x0010},\n{0x011CB7, 0x0001},\n{0x011D00, 0x0004},\n{0x011D07, 0x0001},\n{0x011D08, 0x0004},\n{0x011D0A, 0x0001},\n{0x011D0B, 0x0004},\n{0x011D31, 0x0010},\n{0x011D37, 0x0001},\n{0x011D3A, 0x0010},\n{0x011D3B, 0x0001},\n{0x011D3C, 0x0010},\n{0x011D3E, 0x0001},\n{0x011D3F, 0x0010},\n{0x011D46, 0x0004},\n{0x011D47, 0x0010},\n{0x011D48, 0x0001},\n{0x011D50, 0x0002},\n{0x011D5A, 0x0001},\n{0x011D60, 0x0004},\n{0x011D66, 0x0001},\n{0x011D67, 0x0004},\n{0x011D69, 0x0001},\n{0x011D6A, 0x0004},\n{0x011D8A, 0x0010},\n{0x011D8F, 0x0001},\n{0x011D90, 0x0010},\n{0x011D92, 0x0001},\n{0x011D93, 0x0010},\n{0x011D98, 0x0004},\n{0x011D99, 0x0001},\n{0x011DA0, 0x0002},\n{0x011DAA, 0x0001},\n{0x011EE0, 0x0004},\n{0x011EF3, 0x0010},\n{0x011EF7, 0x0020},\n{0x011EF9, 0x0001},\n{0x011F00, 0x0010},\n{0x011F02, 0x0004},\n{0x011F03, 0x0010},\n{0x011F04, 0x0004},\n{0x011F11, 0x0001},\n{0x011F12, 0x0004},\n{0x011F34, 0x0010},\n{0x011F3B, 0x0001},\n{0x011F3E, 0x0010},\n{0x011F43, 0x0020},\n{0x011F50, 0x0002},\n{0x011F5A, 0x0001},\n{0x011FB0, 0x0004},\n{0x011FB1, 0x0001},\n{0x011FC0, 0x0002},\n{0x011FD5, 0x0040},\n{0x011FF2, 0x0001},\n{0x011FFF, 0x0020},\n{0x012000, 0x0004},\n{0x01239A, 0x0001},\n{0x012400, 0x0002},\n{0x01246F, 0x0001},\n{0x012470, 0x0020},\n{0x012475, 0x0001},\n{0x012480, 0x0004},\n{0x012544, 0x0001},\n{0x012F90, 0x0004},\n{0x012FF1, 0x0020},\n{0x012FF3, 0x0001},\n{0x013000, 0x0004},\n{0x013430, 0x0080},\n{0x013440, 0x0010},\n{0x013441, 0x0004},\n{0x013447, 0x0010},\n{0x013456, 0x0001},\n{0x014400, 0x0004},\n{0x014647, 0x0001},\n{0x016800, 0x0004},\n{0x016A39, 0x0001},\n{0x016A40, 0x0004},\n{0x016A5F, 0x0001},\n{0x016A60, 0x0002},\n{0x016A6A, 0x0001},\n{0x016A6E, 0x0020},\n{0x016A70, 0x0004},\n{0x016ABF, 0x0001},\n{0x016AC0, 0x0002},\n{0x016ACA, 0x0001},\n{0x016AD0, 0x0004},\n{0x016AEE, 0x0001},\n{0x016AF0, 0x0010},\n{0x016AF5, 0x0020},\n{0x016AF6, 0x0001},\n{0x016B00, 0x0004},\n{0x016B30, 0x0010},\n{0x016B37, 0x0020},\n{0x016B3C, 0x0040},\n{0x016B40, 0x0004},\n{0x016B44, 0x0020},\n{0x016B45, 0x0040},\n{0x016B46, 0x0001},\n{0x016B50, 0x0002},\n{0x016B5A, 0x0001},\n{0x016B5B, 0x0002},\n{0x016B62, 0x0001},\n{0x016B63, 0x0004},\n{0x016B78, 0x0001},\n{0x016B7D, 0x0004},\n{0x016B90, 0x0001},\n{0x016E40, 0x0004},\n{0x016E80, 0x0002},\n{0x016E97, 0x0020},\n{0x016E9B, 0x0001},\n{0x016F00, 0x0004},\n{0x016F4B, 0x0001},\n{0x016F4F, 0x0010},\n{0x016F50, 0x0004},\n{0x016F51, 0x0010},\n{0x016F88, 0x0001},\n{0x016F8F, 0x0010},\n{0x016F93, 0x0004},\n{0x016FA0, 0x0001},\n{0x016FE0, 0x0004},\n{0x016FE2, 0x0020},\n{0x016FE3, 0x0004},\n{0x016FE4, 0x0010},\n{0x016FE5, 0x0001},\n{0x016FF0, 0x0010},\n{0x016FF2, 0x0001},\n{0x017000, 0x0004},\n{0x0187F8, 0x0001},\n{0x018800, 0x0004},\n{0x018CD6, 0x0001},\n{0x018D00, 0x0004},\n{0x018D09, 0x0001},\n{0x01AFF0, 0x0004},\n{0x01AFF4, 0x0001},\n{0x01AFF5, 0x0004},\n{0x01AFFC, 0x0001},\n{0x01AFFD, 0x0004},\n{0x01AFFF, 0x0001},\n{0x01B000, 0x0004},\n{0x01B123, 0x0001},\n{0x01B132, 0x0004},\n{0x01B133, 0x0001},\n{0x01B150, 0x0004},\n{0x01B153, 0x0001},\n{0x01B155, 0x0004},\n{0x01B156, 0x0001},\n{0x01B164, 0x0004},\n{0x01B168, 0x0001},\n{0x01B170, 0x0004},\n{0x01B2FC, 0x0001},\n{0x01BC00, 0x0004},\n{0x01BC6B, 0x0001},\n{0x01BC70, 0x0004},\n{0x01BC7D, 0x0001},\n{0x01BC80, 0x0004},\n{0x01BC89, 0x0001},\n{0x01BC90, 0x0004},\n{0x01BC9A, 0x0001},\n{0x01BC9C, 0x0040},\n{0x01BC9D, 0x0010},\n{0x01BC9F, 0x0020},\n{0x01BCA0, 0x0080},\n{0x01BCA4, 0x0001},\n{0x01CF00, 0x0010},\n{0x01CF2E, 0x0001},\n{0x01CF30, 0x0010},\n{0x01CF47, 0x0001},\n{0x01CF50, 0x0040},\n{0x01CFC4, 0x0001},\n{0x01D000, 0x0040},\n{0x01D0F6, 0x0001},\n{0x01D100, 0x0040},\n{0x01D127, 0x0001},\n{0x01D129, 0x0040},\n{0x01D165, 0x0010},\n{0x01D16A, 0x0040},\n{0x01D16D, 0x0010},\n{0x01D173, 0x0080},\n{0x01D17B, 0x0010},\n{0x01D183, 0x0040},\n{0x01D185, 0x0010},\n{0x01D18C, 0x0040},\n{0x01D1AA, 0x0010},\n{0x01D1AE, 0x0040},\n{0x01D1EB, 0x0001},\n{0x01D200, 0x0040},\n{0x01D242, 0x0010},\n{0x01D245, 0x0040},\n{0x01D246, 0x0001},\n{0x01D2C0, 0x0002},\n{0x01D2D4, 0x0001},\n{0x01D2E0, 0x0002},\n{0x01D2F4, 0x0001},\n{0x01D300, 0x0040},\n{0x01D357, 0x0001},\n{0x01D360, 0x0002},\n{0x01D379, 0x0001},\n{0x01D400, 0x0004},\n{0x01D455, 0x0001},\n{0x01D456, 0x0004},\n{0x01D49D, 0x0001},\n{0x01D49E, 0x0004},\n{0x01D4A0, 0x0001},\n{0x01D4A2, 0x0004},\n{0x01D4A3, 0x0001},\n{0x01D4A5, 0x0004},\n{0x01D4A7, 0x0001},\n{0x01D4A9, 0x0004},\n{0x01D4AD, 0x0001},\n{0x01D4AE, 0x0004},\n{0x01D4BA, 0x0001},\n{0x01D4BB, 0x0004},\n{0x01D4BC, 0x0001},\n{0x01D4BD, 0x0004},\n{0x01D4C4, 0x0001},\n{0x01D4C5, 0x0004},\n{0x01D506, 0x0001},\n{0x01D507, 0x0004},\n{0x01D50B, 0x0001},\n{0x01D50D, 0x0004},\n{0x01D515, 0x0001},\n{0x01D516, 0x0004},\n{0x01D51D, 0x0001},\n{0x01D51E, 0x0004},\n{0x01D53A, 0x0001},\n{0x01D53B, 0x0004},\n{0x01D53F, 0x0001},\n{0x01D540, 0x0004},\n{0x01D545, 0x0001},\n{0x01D546, 0x0004},\n{0x01D547, 0x0001},\n{0x01D54A, 0x0004},\n{0x01D551, 0x0001},\n{0x01D552, 0x0004},\n{0x01D6A6, 0x0001},\n{0x01D6A8, 0x0004},\n{0x01D6C1, 0x0040},\n{0x01D6C2, 0x0004},\n{0x01D6DB, 0x0040},\n{0x01D6DC, 0x0004},\n{0x01D6FB, 0x0040},\n{0x01D6FC, 0x0004},\n{0x01D715, 0x0040},\n{0x01D716, 0x0004},\n{0x01D735, 0x0040},\n{0x01D736, 0x0004},\n{0x01D74F, 0x0040},\n{0x01D750, 0x0004},\n{0x01D76F, 0x0040},\n{0x01D770, 0x0004},\n{0x01D789, 0x0040},\n{0x01D78A, 0x0004},\n{0x01D7A9, 0x0040},\n{0x01D7AA, 0x0004},\n{0x01D7C3, 0x0040},\n{0x01D7C4, 0x0004},\n{0x01D7CC, 0x0001},\n{0x01D7CE, 0x0002},\n{0x01D800, 0x0040},\n{0x01DA00, 0x0010},\n{0x01DA37, 0x0040},\n{0x01DA3B, 0x0010},\n{0x01DA6D, 0x0040},\n{0x01DA75, 0x0010},\n{0x01DA76, 0x0040},\n{0x01DA84, 0x0010},\n{0x01DA85, 0x0040},\n{0x01DA87, 0x0020},\n{0x01DA8C, 0x0001},\n{0x01DA9B, 0x0010},\n{0x01DAA0, 0x0001},\n{0x01DAA1, 0x0010},\n{0x01DAB0, 0x0001},\n{0x01DF00, 0x0004},\n{0x01DF1F, 0x0001},\n{0x01DF25, 0x0004},\n{0x01DF2B, 0x0001},\n{0x01E000, 0x0010},\n{0x01E007, 0x0001},\n{0x01E008, 0x0010},\n{0x01E019, 0x0001},\n{0x01E01B, 0x0010},\n{0x01E022, 0x0001},\n{0x01E023, 0x0010},\n{0x01E025, 0x0001},\n{0x01E026, 0x0010},\n{0x01E02B, 0x0001},\n{0x01E030, 0x0004},\n{0x01E06E, 0x0001},\n{0x01E08F, 0x0010},\n{0x01E090, 0x0001},\n{0x01E100, 0x0004},\n{0x01E12D, 0x0001},\n{0x01E130, 0x0010},\n{0x01E137, 0x0004},\n{0x01E13E, 0x0001},\n{0x01E140, 0x0002},\n{0x01E14A, 0x0001},\n{0x01E14E, 0x0004},\n{0x01E14F, 0x0040},\n{0x01E150, 0x0001},\n{0x01E290, 0x0004},\n{0x01E2AE, 0x0010},\n{0x01E2AF, 0x0001},\n{0x01E2C0, 0x0004},\n{0x01E2EC, 0x0010},\n{0x01E2F0, 0x0002},\n{0x01E2FA, 0x0001},\n{0x01E2FF, 0x0040},\n{0x01E300, 0x0001},\n{0x01E4D0, 0x0004},\n{0x01E4EC, 0x0010},\n{0x01E4F0, 0x0002},\n{0x01E4FA, 0x0001},\n{0x01E7E0, 0x0004},\n{0x01E7E7, 0x0001},\n{0x01E7E8, 0x0004},\n{0x01E7EC, 0x0001},\n{0x01E7ED, 0x0004},\n{0x01E7EF, 0x0001},\n{0x01E7F0, 0x0004},\n{0x01E7FF, 0x0001},\n{0x01E800, 0x0004},\n{0x01E8C5, 0x0001},\n{0x01E8C7, 0x0002},\n{0x01E8D0, 0x0010},\n{0x01E8D7, 0x0001},\n{0x01E900, 0x0004},\n{0x01E944, 0x0010},\n{0x01E94B, 0x0004},\n{0x01E94C, 0x0001},\n{0x01E950, 0x0002},\n{0x01E95A, 0x0001},\n{0x01E95E, 0x0020},\n{0x01E960, 0x0001},\n{0x01EC71, 0x0002},\n{0x01ECAC, 0x0040},\n{0x01ECAD, 0x0002},\n{0x01ECB0, 0x0040},\n{0x01ECB1, 0x0002},\n{0x01ECB5, 0x0001},\n{0x01ED01, 0x0002},\n{0x01ED2E, 0x0040},\n{0x01ED2F, 0x0002},\n{0x01ED3E, 0x0001},\n{0x01EE00, 0x0004},\n{0x01EE04, 0x0001},\n{0x01EE05, 0x0004},\n{0x01EE20, 0x0001},\n{0x01EE21, 0x0004},\n{0x01EE23, 0x0001},\n{0x01EE24, 0x0004},\n{0x01EE25, 0x0001},\n{0x01EE27, 0x0004},\n{0x01EE28, 0x0001},\n{0x01EE29, 0x0004},\n{0x01EE33, 0x0001},\n{0x01EE34, 0x0004},\n{0x01EE38, 0x0001},\n{0x01EE39, 0x0004},\n{0x01EE3A, 0x0001},\n{0x01EE3B, 0x0004},\n{0x01EE3C, 0x0001},\n{0x01EE42, 0x0004},\n{0x01EE43, 0x0001},\n{0x01EE47, 0x0004},\n{0x01EE48, 0x0001},\n{0x01EE49, 0x0004},\n{0x01EE4A, 0x0001},\n{0x01EE4B, 0x0004},\n{0x01EE4C, 0x0001},\n{0x01EE4D, 0x0004},\n{0x01EE50, 0x0001},\n{0x01EE51, 0x0004},\n{0x01EE53, 0x0001},\n{0x01EE54, 0x0004},\n{0x01EE55, 0x0001},\n{0x01EE57, 0x0004},\n{0x01EE58, 0x0001},\n{0x01EE59, 0x0004},\n{0x01EE5A, 0x0001},\n{0x01EE5B, 0x0004},\n{0x01EE5C, 0x0001},\n{0x01EE5D, 0x0004},\n{0x01EE5E, 0x0001},\n{0x01EE5F, 0x0004},\n{0x01EE60, 0x0001},\n{0x01EE61, 0x0004},\n{0x01EE63, 0x0001},\n{0x01EE64, 0x0004},\n{0x01EE65, 0x0001},\n{0x01EE67, 0x0004},\n{0x01EE6B, 0x0001},\n{0x01EE6C, 0x0004},\n{0x01EE73, 0x0001},\n{0x01EE74, 0x0004},\n{0x01EE78, 0x0001},\n{0x01EE79, 0x0004},\n{0x01EE7D, 0x0001},\n{0x01EE7E, 0x0004},\n{0x01EE7F, 0x0001},\n{0x01EE80, 0x0004},\n{0x01EE8A, 0x0001},\n{0x01EE8B, 0x0004},\n{0x01EE9C, 0x0001},\n{0x01EEA1, 0x0004},\n{0x01EEA4, 0x0001},\n{0x01EEA5, 0x0004},\n{0x01EEAA, 0x0001},\n{0x01EEAB, 0x0004},\n{0x01EEBC, 0x0001},\n{0x01EEF0, 0x0040},\n{0x01EEF2, 0x0001},\n{0x01F000, 0x0040},\n{0x01F02C, 0x0001},\n{0x01F030, 0x0040},\n{0x01F094, 0x0001},\n{0x01F0A0, 0x0040},\n{0x01F0AF, 0x0001},\n{0x01F0B1, 0x0040},\n{0x01F0C0, 0x0001},\n{0x01F0C1, 0x0040},\n{0x01F0D0, 0x0001},\n{0x01F0D1, 0x0040},\n{0x01F0F6, 0x0001},\n{0x01F100, 0x0002},\n{0x01F10D, 0x0040},\n{0x01F1AE, 0x0001},\n{0x01F1E6, 0x0040},\n{0x01F203, 0x0001},\n{0x01F210, 0x0040},\n{0x01F23C, 0x0001},\n{0x01F240, 0x0040},\n{0x01F249, 0x0001},\n{0x01F250, 0x0040},\n{0x01F252, 0x0001},\n{0x01F260, 0x0040},\n{0x01F266, 0x0001},\n{0x01F300, 0x0040},\n{0x01F6D8, 0x0001},\n{0x01F6DC, 0x0040},\n{0x01F6ED, 0x0001},\n{0x01F6F0, 0x0040},\n{0x01F6FD, 0x0001},\n{0x01F700, 0x0040},\n{0x01F777, 0x0001},\n{0x01F77B, 0x0040},\n{0x01F7DA, 0x0001},\n{0x01F7E0, 0x0040},\n{0x01F7EC, 0x0001},\n{0x01F7F0, 0x0040},\n{0x01F7F1, 0x0001},\n{0x01F800, 0x0040},\n{0x01F80C, 0x0001},\n{0x01F810, 0x0040},\n{0x01F848, 0x0001},\n{0x01F850, 0x0040},\n{0x01F85A, 0x0001},\n{0x01F860, 0x0040},\n{0x01F888, 0x0001},\n{0x01F890, 0x0040},\n{0x01F8AE, 0x0001},\n{0x01F8B0, 0x0040},\n{0x01F8B2, 0x0001},\n{0x01F900, 0x0040},\n{0x01FA54, 0x0001},\n{0x01FA60, 0x0040},\n{0x01FA6E, 0x0001},\n{0x01FA70, 0x0040},\n{0x01FA7D, 0x0001},\n{0x01FA80, 0x0040},\n{0x01FA89, 0x0001},\n{0x01FA90, 0x0040},\n{0x01FABE, 0x0001},\n{0x01FABF, 0x0040},\n{0x01FAC6, 0x0001},\n{0x01FACE, 0x0040},\n{0x01FADC, 0x0001},\n{0x01FAE0, 0x0040},\n{0x01FAE9, 0x0001},\n{0x01FAF0, 0x0040},\n{0x01FAF9, 0x0001},\n{0x01FB00, 0x0040},\n{0x01FB93, 0x0001},\n{0x01FB94, 0x0040},\n{0x01FBCB, 0x0001},\n{0x01FBF0, 0x0002},\n{0x01FBFA, 0x0001},\n{0x020000, 0x0004},\n{0x02A6E0, 0x0001},\n{0x02A700, 0x0004},\n{0x02B73A, 0x0001},\n{0x02B740, 0x0004},\n{0x02B81E, 0x0001},\n{0x02B820, 0x0004},\n{0x02CEA2, 0x0001},\n{0x02CEB0, 0x0004},\n{0x02EBE1, 0x0001},\n{0x02EBF0, 0x0004},\n{0x02EE5E, 0x0001},\n{0x02F800, 0x0004},\n{0x02FA1E, 0x0001},\n{0x030000, 0x0004},\n{0x03134B, 0x0001},\n{0x031350, 0x0004},\n{0x0323B0, 0x0001},\n{0x0E0001, 0x0080},\n{0x0E0002, 0x0001},\n{0x0E0020, 0x0080},\n{0x0E0080, 0x0001},\n{0x0E0100, 0x0010},\n{0x0E01F0, 0x0001},\n{0x0F0000, 0x0080},\n{0x0FFFFE, 0x0001},\n{0x100000, 0x0080},\n{0x10FFFE, 0x0001},\n{0x110000, 0x0000},\n};\n\nconst std::unordered_set<uint32_t> unicode_set_whitespace = {\n0x000009,\n0x00000A,\n0x00000B,\n0x00000C,\n0x00000D,\n0x000020,\n0x000085,\n0x0000A0,\n0x001680,\n0x002000,\n0x002001,\n0x002002,\n0x002003,\n0x002004,\n0x002005,\n0x002006,\n0x002007,\n0x002008,\n0x002009,\n0x00200A,\n0x002028,\n0x002029,\n0x00202F,\n0x00205F,\n0x003000,\n};\n\n// list is always in ascending order, to enable binary search\nconst std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase = {\n{0x000041, 0x000061},\n{0x000042, 0x000062},\n{0x000043, 0x000063},\n{0x000044, 0x000064},\n{0x000045, 0x000065},\n{0x000046, 0x000066},\n{0x000047, 0x000067},\n{0x000048, 0x000068},\n{0x000049, 0x000069},\n{0x00004A, 0x00006A},\n{0x00004B, 0x00006B},\n{0x00004C, 0x00006C},\n{0x00004D, 0x00006D},\n{0x00004E, 0x00006E},\n{0x00004F, 0x00006F},\n{0x000050, 0x000070},\n{0x000051, 0x000071},\n{0x000052, 0x000072},\n{0x000053, 0x000073},\n{0x000054, 0x000074},\n{0x000055, 0x000075},\n{0x000056, 0x000076},\n{0x000057, 0x000077},\n{0x000058, 0x000078},\n{0x000059, 0x000079},\n{0x00005A, 0x00007A},\n{0x0000C0, 0x0000E0},\n{0x0000C1, 0x0000E1},\n{0x0000C2, 0x0000E2},\n{0x0000C3, 0x0000E3},\n{0x0000C4, 0x0000E4},\n{0x0000C5, 0x0000E5},\n{0x0000C6, 0x0000E6},\n{0x0000C7, 0x0000E7},\n{0x0000C8, 0x0000E8},\n{0x0000C9, 0x0000E9},\n{0x0000CA, 0x0000EA},\n{0x0000CB, 0x0000EB},\n{0x0000CC, 0x0000EC},\n{0x0000CD, 0x0000ED},\n{0x0000CE, 0x0000EE},\n{0x0000CF, 0x0000EF},\n{0x0000D0, 0x0000F0},\n{0x0000D1, 0x0000F1},\n{0x0000D2, 0x0000F2},\n{0x0000D3, 0x0000F3},\n{0x0000D4, 0x0000F4},\n{0x0000D5, 0x0000F5},\n{0x0000D6, 0x0000F6},\n{0x0000D8, 0x0000F8},\n{0x0000D9, 0x0000F9},\n{0x0000DA, 0x0000FA},\n{0x0000DB, 0x0000FB},\n{0x0000DC, 0x0000FC},\n{0x0000DD, 0x0000FD},\n{0x0000DE, 0x0000FE},\n{0x000100, 0x000101},\n{0x000102, 0x000103},\n{0x000104, 0x000105},\n{0x000106, 0x000107},\n{0x000108, 0x000109},\n{0x00010A, 0x00010B},\n{0x00010C, 0x00010D},\n{0x00010E, 0x00010F},\n{0x000110, 0x000111},\n{0x000112, 0x000113},\n{0x000114, 0x000115},\n{0x000116, 0x000117},\n{0x000118, 0x000119},\n{0x00011A, 0x00011B},\n{0x00011C, 0x00011D},\n{0x00011E, 0x00011F},\n{0x000120, 0x000121},\n{0x000122, 0x000123},\n{0x000124, 0x000125},\n{0x000126, 0x000127},\n{0x000128, 0x000129},\n{0x00012A, 0x00012B},\n{0x00012C, 0x00012D},\n{0x00012E, 0x00012F},\n{0x000130, 0x000069},\n{0x000132, 0x000133},\n{0x000134, 0x000135},\n{0x000136, 0x000137},\n{0x000139, 0x00013A},\n{0x00013B, 0x00013C},\n{0x00013D, 0x00013E},\n{0x00013F, 0x000140},\n{0x000141, 0x000142},\n{0x000143, 0x000144},\n{0x000145, 0x000146},\n{0x000147, 0x000148},\n{0x00014A, 0x00014B},\n{0x00014C, 0x00014D},\n{0x00014E, 0x00014F},\n{0x000150, 0x000151},\n{0x000152, 0x000153},\n{0x000154, 0x000155},\n{0x000156, 0x000157},\n{0x000158, 0x000159},\n{0x00015A, 0x00015B},\n{0x00015C, 0x00015D},\n{0x00015E, 0x00015F},\n{0x000160, 0x000161},\n{0x000162, 0x000163},\n{0x000164, 0x000165},\n{0x000166, 0x000167},\n{0x000168, 0x000169},\n{0x00016A, 0x00016B},\n{0x00016C, 0x00016D},\n{0x00016E, 0x00016F},\n{0x000170, 0x000171},\n{0x000172, 0x000173},\n{0x000174, 0x000175},\n{0x000176, 0x000177},\n{0x000178, 0x0000FF},\n{0x000179, 0x00017A},\n{0x00017B, 0x00017C},\n{0x00017D, 0x00017E},\n{0x000181, 0x000253},\n{0x000182, 0x000183},\n{0x000184, 0x000185},\n{0x000186, 0x000254},\n{0x000187, 0x000188},\n{0x000189, 0x000256},\n{0x00018A, 0x000257},\n{0x00018B, 0x00018C},\n{0x00018E, 0x0001DD},\n{0x00018F, 0x000259},\n{0x000190, 0x00025B},\n{0x000191, 0x000192},\n{0x000193, 0x000260},\n{0x000194, 0x000263},\n{0x000196, 0x000269},\n{0x000197, 0x000268},\n{0x000198, 0x000199},\n{0x00019C, 0x00026F},\n{0x00019D, 0x000272},\n{0x00019F, 0x000275},\n{0x0001A0, 0x0001A1},\n{0x0001A2, 0x0001A3},\n{0x0001A4, 0x0001A5},\n{0x0001A6, 0x000280},\n{0x0001A7, 0x0001A8},\n{0x0001A9, 0x000283},\n{0x0001AC, 0x0001AD},\n{0x0001AE, 0x000288},\n{0x0001AF, 0x0001B0},\n{0x0001B1, 0x00028A},\n{0x0001B2, 0x00028B},\n{0x0001B3, 0x0001B4},\n{0x0001B5, 0x0001B6},\n{0x0001B7, 0x000292},\n{0x0001B8, 0x0001B9},\n{0x0001BC, 0x0001BD},\n{0x0001C4, 0x0001C6},\n{0x0001C5, 0x0001C6},\n{0x0001C7, 0x0001C9},\n{0x0001C8, 0x0001C9},\n{0x0001CA, 0x0001CC},\n{0x0001CB, 0x0001CC},\n{0x0001CD, 0x0001CE},\n{0x0001CF, 0x0001D0},\n{0x0001D1, 0x0001D2},\n{0x0001D3, 0x0001D4},\n{0x0001D5, 0x0001D6},\n{0x0001D7, 0x0001D8},\n{0x0001D9, 0x0001DA},\n{0x0001DB, 0x0001DC},\n{0x0001DE, 0x0001DF},\n{0x0001E0, 0x0001E1},\n{0x0001E2, 0x0001E3},\n{0x0001E4, 0x0001E5},\n{0x0001E6, 0x0001E7},\n{0x0001E8, 0x0001E9},\n{0x0001EA, 0x0001EB},\n{0x0001EC, 0x0001ED},\n{0x0001EE, 0x0001EF},\n{0x0001F1, 0x0001F3},\n{0x0001F2, 0x0001F3},\n{0x0001F4, 0x0001F5},\n{0x0001F6, 0x000195},\n{0x0001F7, 0x0001BF},\n{0x0001F8, 0x0001F9},\n{0x0001FA, 0x0001FB},\n{0x0001FC, 0x0001FD},\n{0x0001FE, 0x0001FF},\n{0x000200, 0x000201},\n{0x000202, 0x000203},\n{0x000204, 0x000205},\n{0x000206, 0x000207},\n{0x000208, 0x000209},\n{0x00020A, 0x00020B},\n{0x00020C, 0x00020D},\n{0x00020E, 0x00020F},\n{0x000210, 0x000211},\n{0x000212, 0x000213},\n{0x000214, 0x000215},\n{0x000216, 0x000217},\n{0x000218, 0x000219},\n{0x00021A, 0x00021B},\n{0x00021C, 0x00021D},\n{0x00021E, 0x00021F},\n{0x000220, 0x00019E},\n{0x000222, 0x000223},\n{0x000224, 0x000225},\n{0x000226, 0x000227},\n{0x000228, 0x000229},\n{0x00022A, 0x00022B},\n{0x00022C, 0x00022D},\n{0x00022E, 0x00022F},\n{0x000230, 0x000231},\n{0x000232, 0x000233},\n{0x00023A, 0x002C65},\n{0x00023B, 0x00023C},\n{0x00023D, 0x00019A},\n{0x00023E, 0x002C66},\n{0x000241, 0x000242},\n{0x000243, 0x000180},\n{0x000244, 0x000289},\n{0x000245, 0x00028C},\n{0x000246, 0x000247},\n{0x000248, 0x000249},\n{0x00024A, 0x00024B},\n{0x00024C, 0x00024D},\n{0x00024E, 0x00024F},\n{0x000370, 0x000371},\n{0x000372, 0x000373},\n{0x000376, 0x000377},\n{0x00037F, 0x0003F3},\n{0x000386, 0x0003AC},\n{0x000388, 0x0003AD},\n{0x000389, 0x0003AE},\n{0x00038A, 0x0003AF},\n{0x00038C, 0x0003CC},\n{0x00038E, 0x0003CD},\n{0x00038F, 0x0003CE},\n{0x000391, 0x0003B1},\n{0x000392, 0x0003B2},\n{0x000393, 0x0003B3},\n{0x000394, 0x0003B4},\n{0x000395, 0x0003B5},\n{0x000396, 0x0003B6},\n{0x000397, 0x0003B7},\n{0x000398, 0x0003B8},\n{0x000399, 0x0003B9},\n{0x00039A, 0x0003BA},\n{0x00039B, 0x0003BB},\n{0x00039C, 0x0003BC},\n{0x00039D, 0x0003BD},\n{0x00039E, 0x0003BE},\n{0x00039F, 0x0003BF},\n{0x0003A0, 0x0003C0},\n{0x0003A1, 0x0003C1},\n{0x0003A3, 0x0003C3},\n{0x0003A4, 0x0003C4},\n{0x0003A5, 0x0003C5},\n{0x0003A6, 0x0003C6},\n{0x0003A7, 0x0003C7},\n{0x0003A8, 0x0003C8},\n{0x0003A9, 0x0003C9},\n{0x0003AA, 0x0003CA},\n{0x0003AB, 0x0003CB},\n{0x0003CF, 0x0003D7},\n{0x0003D8, 0x0003D9},\n{0x0003DA, 0x0003DB},\n{0x0003DC, 0x0003DD},\n{0x0003DE, 0x0003DF},\n{0x0003E0, 0x0003E1},\n{0x0003E2, 0x0003E3},\n{0x0003E4, 0x0003E5},\n{0x0003E6, 0x0003E7},\n{0x0003E8, 0x0003E9},\n{0x0003EA, 0x0003EB},\n{0x0003EC, 0x0003ED},\n{0x0003EE, 0x0003EF},\n{0x0003F4, 0x0003B8},\n{0x0003F7, 0x0003F8},\n{0x0003F9, 0x0003F2},\n{0x0003FA, 0x0003FB},\n{0x0003FD, 0x00037B},\n{0x0003FE, 0x00037C},\n{0x0003FF, 0x00037D},\n{0x000400, 0x000450},\n{0x000401, 0x000451},\n{0x000402, 0x000452},\n{0x000403, 0x000453},\n{0x000404, 0x000454},\n{0x000405, 0x000455},\n{0x000406, 0x000456},\n{0x000407, 0x000457},\n{0x000408, 0x000458},\n{0x000409, 0x000459},\n{0x00040A, 0x00045A},\n{0x00040B, 0x00045B},\n{0x00040C, 0x00045C},\n{0x00040D, 0x00045D},\n{0x00040E, 0x00045E},\n{0x00040F, 0x00045F},\n{0x000410, 0x000430},\n{0x000411, 0x000431},\n{0x000412, 0x000432},\n{0x000413, 0x000433},\n{0x000414, 0x000434},\n{0x000415, 0x000435},\n{0x000416, 0x000436},\n{0x000417, 0x000437},\n{0x000418, 0x000438},\n{0x000419, 0x000439},\n{0x00041A, 0x00043A},\n{0x00041B, 0x00043B},\n{0x00041C, 0x00043C},\n{0x00041D, 0x00043D},\n{0x00041E, 0x00043E},\n{0x00041F, 0x00043F},\n{0x000420, 0x000440},\n{0x000421, 0x000441},\n{0x000422, 0x000442},\n{0x000423, 0x000443},\n{0x000424, 0x000444},\n{0x000425, 0x000445},\n{0x000426, 0x000446},\n{0x000427, 0x000447},\n{0x000428, 0x000448},\n{0x000429, 0x000449},\n{0x00042A, 0x00044A},\n{0x00042B, 0x00044B},\n{0x00042C, 0x00044C},\n{0x00042D, 0x00044D},\n{0x00042E, 0x00044E},\n{0x00042F, 0x00044F},\n{0x000460, 0x000461},\n{0x000462, 0x000463},\n{0x000464, 0x000465},\n{0x000466, 0x000467},\n{0x000468, 0x000469},\n{0x00046A, 0x00046B},\n{0x00046C, 0x00046D},\n{0x00046E, 0x00046F},\n{0x000470, 0x000471},\n{0x000472, 0x000473},\n{0x000474, 0x000475},\n{0x000476, 0x000477},\n{0x000478, 0x000479},\n{0x00047A, 0x00047B},\n{0x00047C, 0x00047D},\n{0x00047E, 0x00047F},\n{0x000480, 0x000481},\n{0x00048A, 0x00048B},\n{0x00048C, 0x00048D},\n{0x00048E, 0x00048F},\n{0x000490, 0x000491},\n{0x000492, 0x000493},\n{0x000494, 0x000495},\n{0x000496, 0x000497},\n{0x000498, 0x000499},\n{0x00049A, 0x00049B},\n{0x00049C, 0x00049D},\n{0x00049E, 0x00049F},\n{0x0004A0, 0x0004A1},\n{0x0004A2, 0x0004A3},\n{0x0004A4, 0x0004A5},\n{0x0004A6, 0x0004A7},\n{0x0004A8, 0x0004A9},\n{0x0004AA, 0x0004AB},\n{0x0004AC, 0x0004AD},\n{0x0004AE, 0x0004AF},\n{0x0004B0, 0x0004B1},\n{0x0004B2, 0x0004B3},\n{0x0004B4, 0x0004B5},\n{0x0004B6, 0x0004B7},\n{0x0004B8, 0x0004B9},\n{0x0004BA, 0x0004BB},\n{0x0004BC, 0x0004BD},\n{0x0004BE, 0x0004BF},\n{0x0004C0, 0x0004CF},\n{0x0004C1, 0x0004C2},\n{0x0004C3, 0x0004C4},\n{0x0004C5, 0x0004C6},\n{0x0004C7, 0x0004C8},\n{0x0004C9, 0x0004CA},\n{0x0004CB, 0x0004CC},\n{0x0004CD, 0x0004CE},\n{0x0004D0, 0x0004D1},\n{0x0004D2, 0x0004D3},\n{0x0004D4, 0x0004D5},\n{0x0004D6, 0x0004D7},\n{0x0004D8, 0x0004D9},\n{0x0004DA, 0x0004DB},\n{0x0004DC, 0x0004DD},\n{0x0004DE, 0x0004DF},\n{0x0004E0, 0x0004E1},\n{0x0004E2, 0x0004E3},\n{0x0004E4, 0x0004E5},\n{0x0004E6, 0x0004E7},\n{0x0004E8, 0x0004E9},\n{0x0004EA, 0x0004EB},\n{0x0004EC, 0x0004ED},\n{0x0004EE, 0x0004EF},\n{0x0004F0, 0x0004F1},\n{0x0004F2, 0x0004F3},\n{0x0004F4, 0x0004F5},\n{0x0004F6, 0x0004F7},\n{0x0004F8, 0x0004F9},\n{0x0004FA, 0x0004FB},\n{0x0004FC, 0x0004FD},\n{0x0004FE, 0x0004FF},\n{0x000500, 0x000501},\n{0x000502, 0x000503},\n{0x000504, 0x000505},\n{0x000506, 0x000507},\n{0x000508, 0x000509},\n{0x00050A, 0x00050B},\n{0x00050C, 0x00050D},\n{0x00050E, 0x00050F},\n{0x000510, 0x000511},\n{0x000512, 0x000513},\n{0x000514, 0x000515},\n{0x000516, 0x000517},\n{0x000518, 0x000519},\n{0x00051A, 0x00051B},\n{0x00051C, 0x00051D},\n{0x00051E, 0x00051F},\n{0x000520, 0x000521},\n{0x000522, 0x000523},\n{0x000524, 0x000525},\n{0x000526, 0x000527},\n{0x000528, 0x000529},\n{0x00052A, 0x00052B},\n{0x00052C, 0x00052D},\n{0x00052E, 0x00052F},\n{0x000531, 0x000561},\n{0x000532, 0x000562},\n{0x000533, 0x000563},\n{0x000534, 0x000564},\n{0x000535, 0x000565},\n{0x000536, 0x000566},\n{0x000537, 0x000567},\n{0x000538, 0x000568},\n{0x000539, 0x000569},\n{0x00053A, 0x00056A},\n{0x00053B, 0x00056B},\n{0x00053C, 0x00056C},\n{0x00053D, 0x00056D},\n{0x00053E, 0x00056E},\n{0x00053F, 0x00056F},\n{0x000540, 0x000570},\n{0x000541, 0x000571},\n{0x000542, 0x000572},\n{0x000543, 0x000573},\n{0x000544, 0x000574},\n{0x000545, 0x000575},\n{0x000546, 0x000576},\n{0x000547, 0x000577},\n{0x000548, 0x000578},\n{0x000549, 0x000579},\n{0x00054A, 0x00057A},\n{0x00054B, 0x00057B},\n{0x00054C, 0x00057C},\n{0x00054D, 0x00057D},\n{0x00054E, 0x00057E},\n{0x00054F, 0x00057F},\n{0x000550, 0x000580},\n{0x000551, 0x000581},\n{0x000552, 0x000582},\n{0x000553, 0x000583},\n{0x000554, 0x000584},\n{0x000555, 0x000585},\n{0x000556, 0x000586},\n{0x0010A0, 0x002D00},\n{0x0010A1, 0x002D01},\n{0x0010A2, 0x002D02},\n{0x0010A3, 0x002D03},\n{0x0010A4, 0x002D04},\n{0x0010A5, 0x002D05},\n{0x0010A6, 0x002D06},\n{0x0010A7, 0x002D07},\n{0x0010A8, 0x002D08},\n{0x0010A9, 0x002D09},\n{0x0010AA, 0x002D0A},\n{0x0010AB, 0x002D0B},\n{0x0010AC, 0x002D0C},\n{0x0010AD, 0x002D0D},\n{0x0010AE, 0x002D0E},\n{0x0010AF, 0x002D0F},\n{0x0010B0, 0x002D10},\n{0x0010B1, 0x002D11},\n{0x0010B2, 0x002D12},\n{0x0010B3, 0x002D13},\n{0x0010B4, 0x002D14},\n{0x0010B5, 0x002D15},\n{0x0010B6, 0x002D16},\n{0x0010B7, 0x002D17},\n{0x0010B8, 0x002D18},\n{0x0010B9, 0x002D19},\n{0x0010BA, 0x002D1A},\n{0x0010BB, 0x002D1B},\n{0x0010BC, 0x002D1C},\n{0x0010BD, 0x002D1D},\n{0x0010BE, 0x002D1E},\n{0x0010BF, 0x002D1F},\n{0x0010C0, 0x002D20},\n{0x0010C1, 0x002D21},\n{0x0010C2, 0x002D22},\n{0x0010C3, 0x002D23},\n{0x0010C4, 0x002D24},\n{0x0010C5, 0x002D25},\n{0x0010C7, 0x002D27},\n{0x0010CD, 0x002D2D},\n{0x0013A0, 0x00AB70},\n{0x0013A1, 0x00AB71},\n{0x0013A2, 0x00AB72},\n{0x0013A3, 0x00AB73},\n{0x0013A4, 0x00AB74},\n{0x0013A5, 0x00AB75},\n{0x0013A6, 0x00AB76},\n{0x0013A7, 0x00AB77},\n{0x0013A8, 0x00AB78},\n{0x0013A9, 0x00AB79},\n{0x0013AA, 0x00AB7A},\n{0x0013AB, 0x00AB7B},\n{0x0013AC, 0x00AB7C},\n{0x0013AD, 0x00AB7D},\n{0x0013AE, 0x00AB7E},\n{0x0013AF, 0x00AB7F},\n{0x0013B0, 0x00AB80},\n{0x0013B1, 0x00AB81},\n{0x0013B2, 0x00AB82},\n{0x0013B3, 0x00AB83},\n{0x0013B4, 0x00AB84},\n{0x0013B5, 0x00AB85},\n{0x0013B6, 0x00AB86},\n{0x0013B7, 0x00AB87},\n{0x0013B8, 0x00AB88},\n{0x0013B9, 0x00AB89},\n{0x0013BA, 0x00AB8A},\n{0x0013BB, 0x00AB8B},\n{0x0013BC, 0x00AB8C},\n{0x0013BD, 0x00AB8D},\n{0x0013BE, 0x00AB8E},\n{0x0013BF, 0x00AB8F},\n{0x0013C0, 0x00AB90},\n{0x0013C1, 0x00AB91},\n{0x0013C2, 0x00AB92},\n{0x0013C3, 0x00AB93},\n{0x0013C4, 0x00AB94},\n{0x0013C5, 0x00AB95},\n{0x0013C6, 0x00AB96},\n{0x0013C7, 0x00AB97},\n{0x0013C8, 0x00AB98},\n{0x0013C9, 0x00AB99},\n{0x0013CA, 0x00AB9A},\n{0x0013CB, 0x00AB9B},\n{0x0013CC, 0x00AB9C},\n{0x0013CD, 0x00AB9D},\n{0x0013CE, 0x00AB9E},\n{0x0013CF, 0x00AB9F},\n{0x0013D0, 0x00ABA0},\n{0x0013D1, 0x00ABA1},\n{0x0013D2, 0x00ABA2},\n{0x0013D3, 0x00ABA3},\n{0x0013D4, 0x00ABA4},\n{0x0013D5, 0x00ABA5},\n{0x0013D6, 0x00ABA6},\n{0x0013D7, 0x00ABA7},\n{0x0013D8, 0x00ABA8},\n{0x0013D9, 0x00ABA9},\n{0x0013DA, 0x00ABAA},\n{0x0013DB, 0x00ABAB},\n{0x0013DC, 0x00ABAC},\n{0x0013DD, 0x00ABAD},\n{0x0013DE, 0x00ABAE},\n{0x0013DF, 0x00ABAF},\n{0x0013E0, 0x00ABB0},\n{0x0013E1, 0x00ABB1},\n{0x0013E2, 0x00ABB2},\n{0x0013E3, 0x00ABB3},\n{0x0013E4, 0x00ABB4},\n{0x0013E5, 0x00ABB5},\n{0x0013E6, 0x00ABB6},\n{0x0013E7, 0x00ABB7},\n{0x0013E8, 0x00ABB8},\n{0x0013E9, 0x00ABB9},\n{0x0013EA, 0x00ABBA},\n{0x0013EB, 0x00ABBB},\n{0x0013EC, 0x00ABBC},\n{0x0013ED, 0x00ABBD},\n{0x0013EE, 0x00ABBE},\n{0x0013EF, 0x00ABBF},\n{0x0013F0, 0x0013F8},\n{0x0013F1, 0x0013F9},\n{0x0013F2, 0x0013FA},\n{0x0013F3, 0x0013FB},\n{0x0013F4, 0x0013FC},\n{0x0013F5, 0x0013FD},\n{0x001C90, 0x0010D0},\n{0x001C91, 0x0010D1},\n{0x001C92, 0x0010D2},\n{0x001C93, 0x0010D3},\n{0x001C94, 0x0010D4},\n{0x001C95, 0x0010D5},\n{0x001C96, 0x0010D6},\n{0x001C97, 0x0010D7},\n{0x001C98, 0x0010D8},\n{0x001C99, 0x0010D9},\n{0x001C9A, 0x0010DA},\n{0x001C9B, 0x0010DB},\n{0x001C9C, 0x0010DC},\n{0x001C9D, 0x0010DD},\n{0x001C9E, 0x0010DE},\n{0x001C9F, 0x0010DF},\n{0x001CA0, 0x0010E0},\n{0x001CA1, 0x0010E1},\n{0x001CA2, 0x0010E2},\n{0x001CA3, 0x0010E3},\n{0x001CA4, 0x0010E4},\n{0x001CA5, 0x0010E5},\n{0x001CA6, 0x0010E6},\n{0x001CA7, 0x0010E7},\n{0x001CA8, 0x0010E8},\n{0x001CA9, 0x0010E9},\n{0x001CAA, 0x0010EA},\n{0x001CAB, 0x0010EB},\n{0x001CAC, 0x0010EC},\n{0x001CAD, 0x0010ED},\n{0x001CAE, 0x0010EE},\n{0x001CAF, 0x0010EF},\n{0x001CB0, 0x0010F0},\n{0x001CB1, 0x0010F1},\n{0x001CB2, 0x0010F2},\n{0x001CB3, 0x0010F3},\n{0x001CB4, 0x0010F4},\n{0x001CB5, 0x0010F5},\n{0x001CB6, 0x0010F6},\n{0x001CB7, 0x0010F7},\n{0x001CB8, 0x0010F8},\n{0x001CB9, 0x0010F9},\n{0x001CBA, 0x0010FA},\n{0x001CBD, 0x0010FD},\n{0x001CBE, 0x0010FE},\n{0x001CBF, 0x0010FF},\n{0x001E00, 0x001E01},\n{0x001E02, 0x001E03},\n{0x001E04, 0x001E05},\n{0x001E06, 0x001E07},\n{0x001E08, 0x001E09},\n{0x001E0A, 0x001E0B},\n{0x001E0C, 0x001E0D},\n{0x001E0E, 0x001E0F},\n{0x001E10, 0x001E11},\n{0x001E12, 0x001E13},\n{0x001E14, 0x001E15},\n{0x001E16, 0x001E17},\n{0x001E18, 0x001E19},\n{0x001E1A, 0x001E1B},\n{0x001E1C, 0x001E1D},\n{0x001E1E, 0x001E1F},\n{0x001E20, 0x001E21},\n{0x001E22, 0x001E23},\n{0x001E24, 0x001E25},\n{0x001E26, 0x001E27},\n{0x001E28, 0x001E29},\n{0x001E2A, 0x001E2B},\n{0x001E2C, 0x001E2D},\n{0x001E2E, 0x001E2F},\n{0x001E30, 0x001E31},\n{0x001E32, 0x001E33},\n{0x001E34, 0x001E35},\n{0x001E36, 0x001E37},\n{0x001E38, 0x001E39},\n{0x001E3A, 0x001E3B},\n{0x001E3C, 0x001E3D},\n{0x001E3E, 0x001E3F},\n{0x001E40, 0x001E41},\n{0x001E42, 0x001E43},\n{0x001E44, 0x001E45},\n{0x001E46, 0x001E47},\n{0x001E48, 0x001E49},\n{0x001E4A, 0x001E4B},\n{0x001E4C, 0x001E4D},\n{0x001E4E, 0x001E4F},\n{0x001E50, 0x001E51},\n{0x001E52, 0x001E53},\n{0x001E54, 0x001E55},\n{0x001E56, 0x001E57},\n{0x001E58, 0x001E59},\n{0x001E5A, 0x001E5B},\n{0x001E5C, 0x001E5D},\n{0x001E5E, 0x001E5F},\n{0x001E60, 0x001E61},\n{0x001E62, 0x001E63},\n{0x001E64, 0x001E65},\n{0x001E66, 0x001E67},\n{0x001E68, 0x001E69},\n{0x001E6A, 0x001E6B},\n{0x001E6C, 0x001E6D},\n{0x001E6E, 0x001E6F},\n{0x001E70, 0x001E71},\n{0x001E72, 0x001E73},\n{0x001E74, 0x001E75},\n{0x001E76, 0x001E77},\n{0x001E78, 0x001E79},\n{0x001E7A, 0x001E7B},\n{0x001E7C, 0x001E7D},\n{0x001E7E, 0x001E7F},\n{0x001E80, 0x001E81},\n{0x001E82, 0x001E83},\n{0x001E84, 0x001E85},\n{0x001E86, 0x001E87},\n{0x001E88, 0x001E89},\n{0x001E8A, 0x001E8B},\n{0x001E8C, 0x001E8D},\n{0x001E8E, 0x001E8F},\n{0x001E90, 0x001E91},\n{0x001E92, 0x001E93},\n{0x001E94, 0x001E95},\n{0x001E9E, 0x0000DF},\n{0x001EA0, 0x001EA1},\n{0x001EA2, 0x001EA3},\n{0x001EA4, 0x001EA5},\n{0x001EA6, 0x001EA7},\n{0x001EA8, 0x001EA9},\n{0x001EAA, 0x001EAB},\n{0x001EAC, 0x001EAD},\n{0x001EAE, 0x001EAF},\n{0x001EB0, 0x001EB1},\n{0x001EB2, 0x001EB3},\n{0x001EB4, 0x001EB5},\n{0x001EB6, 0x001EB7},\n{0x001EB8, 0x001EB9},\n{0x001EBA, 0x001EBB},\n{0x001EBC, 0x001EBD},\n{0x001EBE, 0x001EBF},\n{0x001EC0, 0x001EC1},\n{0x001EC2, 0x001EC3},\n{0x001EC4, 0x001EC5},\n{0x001EC6, 0x001EC7},\n{0x001EC8, 0x001EC9},\n{0x001ECA, 0x001ECB},\n{0x001ECC, 0x001ECD},\n{0x001ECE, 0x001ECF},\n{0x001ED0, 0x001ED1},\n{0x001ED2, 0x001ED3},\n{0x001ED4, 0x001ED5},\n{0x001ED6, 0x001ED7},\n{0x001ED8, 0x001ED9},\n{0x001EDA, 0x001EDB},\n{0x001EDC, 0x001EDD},\n{0x001EDE, 0x001EDF},\n{0x001EE0, 0x001EE1},\n{0x001EE2, 0x001EE3},\n{0x001EE4, 0x001EE5},\n{0x001EE6, 0x001EE7},\n{0x001EE8, 0x001EE9},\n{0x001EEA, 0x001EEB},\n{0x001EEC, 0x001EED},\n{0x001EEE, 0x001EEF},\n{0x001EF0, 0x001EF1},\n{0x001EF2, 0x001EF3},\n{0x001EF4, 0x001EF5},\n{0x001EF6, 0x001EF7},\n{0x001EF8, 0x001EF9},\n{0x001EFA, 0x001EFB},\n{0x001EFC, 0x001EFD},\n{0x001EFE, 0x001EFF},\n{0x001F08, 0x001F00},\n{0x001F09, 0x001F01},\n{0x001F0A, 0x001F02},\n{0x001F0B, 0x001F03},\n{0x001F0C, 0x001F04},\n{0x001F0D, 0x001F05},\n{0x001F0E, 0x001F06},\n{0x001F0F, 0x001F07},\n{0x001F18, 0x001F10},\n{0x001F19, 0x001F11},\n{0x001F1A, 0x001F12},\n{0x001F1B, 0x001F13},\n{0x001F1C, 0x001F14},\n{0x001F1D, 0x001F15},\n{0x001F28, 0x001F20},\n{0x001F29, 0x001F21},\n{0x001F2A, 0x001F22},\n{0x001F2B, 0x001F23},\n{0x001F2C, 0x001F24},\n{0x001F2D, 0x001F25},\n{0x001F2E, 0x001F26},\n{0x001F2F, 0x001F27},\n{0x001F38, 0x001F30},\n{0x001F39, 0x001F31},\n{0x001F3A, 0x001F32},\n{0x001F3B, 0x001F33},\n{0x001F3C, 0x001F34},\n{0x001F3D, 0x001F35},\n{0x001F3E, 0x001F36},\n{0x001F3F, 0x001F37},\n{0x001F48, 0x001F40},\n{0x001F49, 0x001F41},\n{0x001F4A, 0x001F42},\n{0x001F4B, 0x001F43},\n{0x001F4C, 0x001F44},\n{0x001F4D, 0x001F45},\n{0x001F59, 0x001F51},\n{0x001F5B, 0x001F53},\n{0x001F5D, 0x001F55},\n{0x001F5F, 0x001F57},\n{0x001F68, 0x001F60},\n{0x001F69, 0x001F61},\n{0x001F6A, 0x001F62},\n{0x001F6B, 0x001F63},\n{0x001F6C, 0x001F64},\n{0x001F6D, 0x001F65},\n{0x001F6E, 0x001F66},\n{0x001F6F, 0x001F67},\n{0x001F88, 0x001F80},\n{0x001F89, 0x001F81},\n{0x001F8A, 0x001F82},\n{0x001F8B, 0x001F83},\n{0x001F8C, 0x001F84},\n{0x001F8D, 0x001F85},\n{0x001F8E, 0x001F86},\n{0x001F8F, 0x001F87},\n{0x001F98, 0x001F90},\n{0x001F99, 0x001F91},\n{0x001F9A, 0x001F92},\n{0x001F9B, 0x001F93},\n{0x001F9C, 0x001F94},\n{0x001F9D, 0x001F95},\n{0x001F9E, 0x001F96},\n{0x001F9F, 0x001F97},\n{0x001FA8, 0x001FA0},\n{0x001FA9, 0x001FA1},\n{0x001FAA, 0x001FA2},\n{0x001FAB, 0x001FA3},\n{0x001FAC, 0x001FA4},\n{0x001FAD, 0x001FA5},\n{0x001FAE, 0x001FA6},\n{0x001FAF, 0x001FA7},\n{0x001FB8, 0x001FB0},\n{0x001FB9, 0x001FB1},\n{0x001FBA, 0x001F70},\n{0x001FBB, 0x001F71},\n{0x001FBC, 0x001FB3},\n{0x001FC8, 0x001F72},\n{0x001FC9, 0x001F73},\n{0x001FCA, 0x001F74},\n{0x001FCB, 0x001F75},\n{0x001FCC, 0x001FC3},\n{0x001FD8, 0x001FD0},\n{0x001FD9, 0x001FD1},\n{0x001FDA, 0x001F76},\n{0x001FDB, 0x001F77},\n{0x001FE8, 0x001FE0},\n{0x001FE9, 0x001FE1},\n{0x001FEA, 0x001F7A},\n{0x001FEB, 0x001F7B},\n{0x001FEC, 0x001FE5},\n{0x001FF8, 0x001F78},\n{0x001FF9, 0x001F79},\n{0x001FFA, 0x001F7C},\n{0x001FFB, 0x001F7D},\n{0x001FFC, 0x001FF3},\n{0x002126, 0x0003C9},\n{0x00212A, 0x00006B},\n{0x00212B, 0x0000E5},\n{0x002132, 0x00214E},\n{0x002160, 0x002170},\n{0x002161, 0x002171},\n{0x002162, 0x002172},\n{0x002163, 0x002173},\n{0x002164, 0x002174},\n{0x002165, 0x002175},\n{0x002166, 0x002176},\n{0x002167, 0x002177},\n{0x002168, 0x002178},\n{0x002169, 0x002179},\n{0x00216A, 0x00217A},\n{0x00216B, 0x00217B},\n{0x00216C, 0x00217C},\n{0x00216D, 0x00217D},\n{0x00216E, 0x00217E},\n{0x00216F, 0x00217F},\n{0x002183, 0x002184},\n{0x0024B6, 0x0024D0},\n{0x0024B7, 0x0024D1},\n{0x0024B8, 0x0024D2},\n{0x0024B9, 0x0024D3},\n{0x0024BA, 0x0024D4},\n{0x0024BB, 0x0024D5},\n{0x0024BC, 0x0024D6},\n{0x0024BD, 0x0024D7},\n{0x0024BE, 0x0024D8},\n{0x0024BF, 0x0024D9},\n{0x0024C0, 0x0024DA},\n{0x0024C1, 0x0024DB},\n{0x0024C2, 0x0024DC},\n{0x0024C3, 0x0024DD},\n{0x0024C4, 0x0024DE},\n{0x0024C5, 0x0024DF},\n{0x0024C6, 0x0024E0},\n{0x0024C7, 0x0024E1},\n{0x0024C8, 0x0024E2},\n{0x0024C9, 0x0024E3},\n{0x0024CA, 0x0024E4},\n{0x0024CB, 0x0024E5},\n{0x0024CC, 0x0024E6},\n{0x0024CD, 0x0024E7},\n{0x0024CE, 0x0024E8},\n{0x0024CF, 0x0024E9},\n{0x002C00, 0x002C30},\n{0x002C01, 0x002C31},\n{0x002C02, 0x002C32},\n{0x002C03, 0x002C33},\n{0x002C04, 0x002C34},\n{0x002C05, 0x002C35},\n{0x002C06, 0x002C36},\n{0x002C07, 0x002C37},\n{0x002C08, 0x002C38},\n{0x002C09, 0x002C39},\n{0x002C0A, 0x002C3A},\n{0x002C0B, 0x002C3B},\n{0x002C0C, 0x002C3C},\n{0x002C0D, 0x002C3D},\n{0x002C0E, 0x002C3E},\n{0x002C0F, 0x002C3F},\n{0x002C10, 0x002C40},\n{0x002C11, 0x002C41},\n{0x002C12, 0x002C42},\n{0x002C13, 0x002C43},\n{0x002C14, 0x002C44},\n{0x002C15, 0x002C45},\n{0x002C16, 0x002C46},\n{0x002C17, 0x002C47},\n{0x002C18, 0x002C48},\n{0x002C19, 0x002C49},\n{0x002C1A, 0x002C4A},\n{0x002C1B, 0x002C4B},\n{0x002C1C, 0x002C4C},\n{0x002C1D, 0x002C4D},\n{0x002C1E, 0x002C4E},\n{0x002C1F, 0x002C4F},\n{0x002C20, 0x002C50},\n{0x002C21, 0x002C51},\n{0x002C22, 0x002C52},\n{0x002C23, 0x002C53},\n{0x002C24, 0x002C54},\n{0x002C25, 0x002C55},\n{0x002C26, 0x002C56},\n{0x002C27, 0x002C57},\n{0x002C28, 0x002C58},\n{0x002C29, 0x002C59},\n{0x002C2A, 0x002C5A},\n{0x002C2B, 0x002C5B},\n{0x002C2C, 0x002C5C},\n{0x002C2D, 0x002C5D},\n{0x002C2E, 0x002C5E},\n{0x002C2F, 0x002C5F},\n{0x002C60, 0x002C61},\n{0x002C62, 0x00026B},\n{0x002C63, 0x001D7D},\n{0x002C64, 0x00027D},\n{0x002C67, 0x002C68},\n{0x002C69, 0x002C6A},\n{0x002C6B, 0x002C6C},\n{0x002C6D, 0x000251},\n{0x002C6E, 0x000271},\n{0x002C6F, 0x000250},\n{0x002C70, 0x000252},\n{0x002C72, 0x002C73},\n{0x002C75, 0x002C76},\n{0x002C7E, 0x00023F},\n{0x002C7F, 0x000240},\n{0x002C80, 0x002C81},\n{0x002C82, 0x002C83},\n{0x002C84, 0x002C85},\n{0x002C86, 0x002C87},\n{0x002C88, 0x002C89},\n{0x002C8A, 0x002C8B},\n{0x002C8C, 0x002C8D},\n{0x002C8E, 0x002C8F},\n{0x002C90, 0x002C91},\n{0x002C92, 0x002C93},\n{0x002C94, 0x002C95},\n{0x002C96, 0x002C97},\n{0x002C98, 0x002C99},\n{0x002C9A, 0x002C9B},\n{0x002C9C, 0x002C9D},\n{0x002C9E, 0x002C9F},\n{0x002CA0, 0x002CA1},\n{0x002CA2, 0x002CA3},\n{0x002CA4, 0x002CA5},\n{0x002CA6, 0x002CA7},\n{0x002CA8, 0x002CA9},\n{0x002CAA, 0x002CAB},\n{0x002CAC, 0x002CAD},\n{0x002CAE, 0x002CAF},\n{0x002CB0, 0x002CB1},\n{0x002CB2, 0x002CB3},\n{0x002CB4, 0x002CB5},\n{0x002CB6, 0x002CB7},\n{0x002CB8, 0x002CB9},\n{0x002CBA, 0x002CBB},\n{0x002CBC, 0x002CBD},\n{0x002CBE, 0x002CBF},\n{0x002CC0, 0x002CC1},\n{0x002CC2, 0x002CC3},\n{0x002CC4, 0x002CC5},\n{0x002CC6, 0x002CC7},\n{0x002CC8, 0x002CC9},\n{0x002CCA, 0x002CCB},\n{0x002CCC, 0x002CCD},\n{0x002CCE, 0x002CCF},\n{0x002CD0, 0x002CD1},\n{0x002CD2, 0x002CD3},\n{0x002CD4, 0x002CD5},\n{0x002CD6, 0x002CD7},\n{0x002CD8, 0x002CD9},\n{0x002CDA, 0x002CDB},\n{0x002CDC, 0x002CDD},\n{0x002CDE, 0x002CDF},\n{0x002CE0, 0x002CE1},\n{0x002CE2, 0x002CE3},\n{0x002CEB, 0x002CEC},\n{0x002CED, 0x002CEE},\n{0x002CF2, 0x002CF3},\n{0x00A640, 0x00A641},\n{0x00A642, 0x00A643},\n{0x00A644, 0x00A645},\n{0x00A646, 0x00A647},\n{0x00A648, 0x00A649},\n{0x00A64A, 0x00A64B},\n{0x00A64C, 0x00A64D},\n{0x00A64E, 0x00A64F},\n{0x00A650, 0x00A651},\n{0x00A652, 0x00A653},\n{0x00A654, 0x00A655},\n{0x00A656, 0x00A657},\n{0x00A658, 0x00A659},\n{0x00A65A, 0x00A65B},\n{0x00A65C, 0x00A65D},\n{0x00A65E, 0x00A65F},\n{0x00A660, 0x00A661},\n{0x00A662, 0x00A663},\n{0x00A664, 0x00A665},\n{0x00A666, 0x00A667},\n{0x00A668, 0x00A669},\n{0x00A66A, 0x00A66B},\n{0x00A66C, 0x00A66D},\n{0x00A680, 0x00A681},\n{0x00A682, 0x00A683},\n{0x00A684, 0x00A685},\n{0x00A686, 0x00A687},\n{0x00A688, 0x00A689},\n{0x00A68A, 0x00A68B},\n{0x00A68C, 0x00A68D},\n{0x00A68E, 0x00A68F},\n{0x00A690, 0x00A691},\n{0x00A692, 0x00A693},\n{0x00A694, 0x00A695},\n{0x00A696, 0x00A697},\n{0x00A698, 0x00A699},\n{0x00A69A, 0x00A69B},\n{0x00A722, 0x00A723},\n{0x00A724, 0x00A725},\n{0x00A726, 0x00A727},\n{0x00A728, 0x00A729},\n{0x00A72A, 0x00A72B},\n{0x00A72C, 0x00A72D},\n{0x00A72E, 0x00A72F},\n{0x00A732, 0x00A733},\n{0x00A734, 0x00A735},\n{0x00A736, 0x00A737},\n{0x00A738, 0x00A739},\n{0x00A73A, 0x00A73B},\n{0x00A73C, 0x00A73D},\n{0x00A73E, 0x00A73F},\n{0x00A740, 0x00A741},\n{0x00A742, 0x00A743},\n{0x00A744, 0x00A745},\n{0x00A746, 0x00A747},\n{0x00A748, 0x00A749},\n{0x00A74A, 0x00A74B},\n{0x00A74C, 0x00A74D},\n{0x00A74E, 0x00A74F},\n{0x00A750, 0x00A751},\n{0x00A752, 0x00A753},\n{0x00A754, 0x00A755},\n{0x00A756, 0x00A757},\n{0x00A758, 0x00A759},\n{0x00A75A, 0x00A75B},\n{0x00A75C, 0x00A75D},\n{0x00A75E, 0x00A75F},\n{0x00A760, 0x00A761},\n{0x00A762, 0x00A763},\n{0x00A764, 0x00A765},\n{0x00A766, 0x00A767},\n{0x00A768, 0x00A769},\n{0x00A76A, 0x00A76B},\n{0x00A76C, 0x00A76D},\n{0x00A76E, 0x00A76F},\n{0x00A779, 0x00A77A},\n{0x00A77B, 0x00A77C},\n{0x00A77D, 0x001D79},\n{0x00A77E, 0x00A77F},\n{0x00A780, 0x00A781},\n{0x00A782, 0x00A783},\n{0x00A784, 0x00A785},\n{0x00A786, 0x00A787},\n{0x00A78B, 0x00A78C},\n{0x00A78D, 0x000265},\n{0x00A790, 0x00A791},\n{0x00A792, 0x00A793},\n{0x00A796, 0x00A797},\n{0x00A798, 0x00A799},\n{0x00A79A, 0x00A79B},\n{0x00A79C, 0x00A79D},\n{0x00A79E, 0x00A79F},\n{0x00A7A0, 0x00A7A1},\n{0x00A7A2, 0x00A7A3},\n{0x00A7A4, 0x00A7A5},\n{0x00A7A6, 0x00A7A7},\n{0x00A7A8, 0x00A7A9},\n{0x00A7AA, 0x000266},\n{0x00A7AB, 0x00025C},\n{0x00A7AC, 0x000261},\n{0x00A7AD, 0x00026C},\n{0x00A7AE, 0x00026A},\n{0x00A7B0, 0x00029E},\n{0x00A7B1, 0x000287},\n{0x00A7B2, 0x00029D},\n{0x00A7B3, 0x00AB53},\n{0x00A7B4, 0x00A7B5},\n{0x00A7B6, 0x00A7B7},\n{0x00A7B8, 0x00A7B9},\n{0x00A7BA, 0x00A7BB},\n{0x00A7BC, 0x00A7BD},\n{0x00A7BE, 0x00A7BF},\n{0x00A7C0, 0x00A7C1},\n{0x00A7C2, 0x00A7C3},\n{0x00A7C4, 0x00A794},\n{0x00A7C5, 0x000282},\n{0x00A7C6, 0x001D8E},\n{0x00A7C7, 0x00A7C8},\n{0x00A7C9, 0x00A7CA},\n{0x00A7D0, 0x00A7D1},\n{0x00A7D6, 0x00A7D7},\n{0x00A7D8, 0x00A7D9},\n{0x00A7F5, 0x00A7F6},\n{0x00FF21, 0x00FF41},\n{0x00FF22, 0x00FF42},\n{0x00FF23, 0x00FF43},\n{0x00FF24, 0x00FF44},\n{0x00FF25, 0x00FF45},\n{0x00FF26, 0x00FF46},\n{0x00FF27, 0x00FF47},\n{0x00FF28, 0x00FF48},\n{0x00FF29, 0x00FF49},\n{0x00FF2A, 0x00FF4A},\n{0x00FF2B, 0x00FF4B},\n{0x00FF2C, 0x00FF4C},\n{0x00FF2D, 0x00FF4D},\n{0x00FF2E, 0x00FF4E},\n{0x00FF2F, 0x00FF4F},\n{0x00FF30, 0x00FF50},\n{0x00FF31, 0x00FF51},\n{0x00FF32, 0x00FF52},\n{0x00FF33, 0x00FF53},\n{0x00FF34, 0x00FF54},\n{0x00FF35, 0x00FF55},\n{0x00FF36, 0x00FF56},\n{0x00FF37, 0x00FF57},\n{0x00FF38, 0x00FF58},\n{0x00FF39, 0x00FF59},\n{0x00FF3A, 0x00FF5A},\n{0x010400, 0x010428},\n{0x010401, 0x010429},\n{0x010402, 0x01042A},\n{0x010403, 0x01042B},\n{0x010404, 0x01042C},\n{0x010405, 0x01042D},\n{0x010406, 0x01042E},\n{0x010407, 0x01042F},\n{0x010408, 0x010430},\n{0x010409, 0x010431},\n{0x01040A, 0x010432},\n{0x01040B, 0x010433},\n{0x01040C, 0x010434},\n{0x01040D, 0x010435},\n{0x01040E, 0x010436},\n{0x01040F, 0x010437},\n{0x010410, 0x010438},\n{0x010411, 0x010439},\n{0x010412, 0x01043A},\n{0x010413, 0x01043B},\n{0x010414, 0x01043C},\n{0x010415, 0x01043D},\n{0x010416, 0x01043E},\n{0x010417, 0x01043F},\n{0x010418, 0x010440},\n{0x010419, 0x010441},\n{0x01041A, 0x010442},\n{0x01041B, 0x010443},\n{0x01041C, 0x010444},\n{0x01041D, 0x010445},\n{0x01041E, 0x010446},\n{0x01041F, 0x010447},\n{0x010420, 0x010448},\n{0x010421, 0x010449},\n{0x010422, 0x01044A},\n{0x010423, 0x01044B},\n{0x010424, 0x01044C},\n{0x010425, 0x01044D},\n{0x010426, 0x01044E},\n{0x010427, 0x01044F},\n{0x0104B0, 0x0104D8},\n{0x0104B1, 0x0104D9},\n{0x0104B2, 0x0104DA},\n{0x0104B3, 0x0104DB},\n{0x0104B4, 0x0104DC},\n{0x0104B5, 0x0104DD},\n{0x0104B6, 0x0104DE},\n{0x0104B7, 0x0104DF},\n{0x0104B8, 0x0104E0},\n{0x0104B9, 0x0104E1},\n{0x0104BA, 0x0104E2},\n{0x0104BB, 0x0104E3},\n{0x0104BC, 0x0104E4},\n{0x0104BD, 0x0104E5},\n{0x0104BE, 0x0104E6},\n{0x0104BF, 0x0104E7},\n{0x0104C0, 0x0104E8},\n{0x0104C1, 0x0104E9},\n{0x0104C2, 0x0104EA},\n{0x0104C3, 0x0104EB},\n{0x0104C4, 0x0104EC},\n{0x0104C5, 0x0104ED},\n{0x0104C6, 0x0104EE},\n{0x0104C7, 0x0104EF},\n{0x0104C8, 0x0104F0},\n{0x0104C9, 0x0104F1},\n{0x0104CA, 0x0104F2},\n{0x0104CB, 0x0104F3},\n{0x0104CC, 0x0104F4},\n{0x0104CD, 0x0104F5},\n{0x0104CE, 0x0104F6},\n{0x0104CF, 0x0104F7},\n{0x0104D0, 0x0104F8},\n{0x0104D1, 0x0104F9},\n{0x0104D2, 0x0104FA},\n{0x0104D3, 0x0104FB},\n{0x010570, 0x010597},\n{0x010571, 0x010598},\n{0x010572, 0x010599},\n{0x010573, 0x01059A},\n{0x010574, 0x01059B},\n{0x010575, 0x01059C},\n{0x010576, 0x01059D},\n{0x010577, 0x01059E},\n{0x010578, 0x01059F},\n{0x010579, 0x0105A0},\n{0x01057A, 0x0105A1},\n{0x01057C, 0x0105A3},\n{0x01057D, 0x0105A4},\n{0x01057E, 0x0105A5},\n{0x01057F, 0x0105A6},\n{0x010580, 0x0105A7},\n{0x010581, 0x0105A8},\n{0x010582, 0x0105A9},\n{0x010583, 0x0105AA},\n{0x010584, 0x0105AB},\n{0x010585, 0x0105AC},\n{0x010586, 0x0105AD},\n{0x010587, 0x0105AE},\n{0x010588, 0x0105AF},\n{0x010589, 0x0105B0},\n{0x01058A, 0x0105B1},\n{0x01058C, 0x0105B3},\n{0x01058D, 0x0105B4},\n{0x01058E, 0x0105B5},\n{0x01058F, 0x0105B6},\n{0x010590, 0x0105B7},\n{0x010591, 0x0105B8},\n{0x010592, 0x0105B9},\n{0x010594, 0x0105BB},\n{0x010595, 0x0105BC},\n{0x010C80, 0x010CC0},\n{0x010C81, 0x010CC1},\n{0x010C82, 0x010CC2},\n{0x010C83, 0x010CC3},\n{0x010C84, 0x010CC4},\n{0x010C85, 0x010CC5},\n{0x010C86, 0x010CC6},\n{0x010C87, 0x010CC7},\n{0x010C88, 0x010CC8},\n{0x010C89, 0x010CC9},\n{0x010C8A, 0x010CCA},\n{0x010C8B, 0x010CCB},\n{0x010C8C, 0x010CCC},\n{0x010C8D, 0x010CCD},\n{0x010C8E, 0x010CCE},\n{0x010C8F, 0x010CCF},\n{0x010C90, 0x010CD0},\n{0x010C91, 0x010CD1},\n{0x010C92, 0x010CD2},\n{0x010C93, 0x010CD3},\n{0x010C94, 0x010CD4},\n{0x010C95, 0x010CD5},\n{0x010C96, 0x010CD6},\n{0x010C97, 0x010CD7},\n{0x010C98, 0x010CD8},\n{0x010C99, 0x010CD9},\n{0x010C9A, 0x010CDA},\n{0x010C9B, 0x010CDB},\n{0x010C9C, 0x010CDC},\n{0x010C9D, 0x010CDD},\n{0x010C9E, 0x010CDE},\n{0x010C9F, 0x010CDF},\n{0x010CA0, 0x010CE0},\n{0x010CA1, 0x010CE1},\n{0x010CA2, 0x010CE2},\n{0x010CA3, 0x010CE3},\n{0x010CA4, 0x010CE4},\n{0x010CA5, 0x010CE5},\n{0x010CA6, 0x010CE6},\n{0x010CA7, 0x010CE7},\n{0x010CA8, 0x010CE8},\n{0x010CA9, 0x010CE9},\n{0x010CAA, 0x010CEA},\n{0x010CAB, 0x010CEB},\n{0x010CAC, 0x010CEC},\n{0x010CAD, 0x010CED},\n{0x010CAE, 0x010CEE},\n{0x010CAF, 0x010CEF},\n{0x010CB0, 0x010CF0},\n{0x010CB1, 0x010CF1},\n{0x010CB2, 0x010CF2},\n{0x0118A0, 0x0118C0},\n{0x0118A1, 0x0118C1},\n{0x0118A2, 0x0118C2},\n{0x0118A3, 0x0118C3},\n{0x0118A4, 0x0118C4},\n{0x0118A5, 0x0118C5},\n{0x0118A6, 0x0118C6},\n{0x0118A7, 0x0118C7},\n{0x0118A8, 0x0118C8},\n{0x0118A9, 0x0118C9},\n{0x0118AA, 0x0118CA},\n{0x0118AB, 0x0118CB},\n{0x0118AC, 0x0118CC},\n{0x0118AD, 0x0118CD},\n{0x0118AE, 0x0118CE},\n{0x0118AF, 0x0118CF},\n{0x0118B0, 0x0118D0},\n{0x0118B1, 0x0118D1},\n{0x0118B2, 0x0118D2},\n{0x0118B3, 0x0118D3},\n{0x0118B4, 0x0118D4},\n{0x0118B5, 0x0118D5},\n{0x0118B6, 0x0118D6},\n{0x0118B7, 0x0118D7},\n{0x0118B8, 0x0118D8},\n{0x0118B9, 0x0118D9},\n{0x0118BA, 0x0118DA},\n{0x0118BB, 0x0118DB},\n{0x0118BC, 0x0118DC},\n{0x0118BD, 0x0118DD},\n{0x0118BE, 0x0118DE},\n{0x0118BF, 0x0118DF},\n{0x016E40, 0x016E60},\n{0x016E41, 0x016E61},\n{0x016E42, 0x016E62},\n{0x016E43, 0x016E63},\n{0x016E44, 0x016E64},\n{0x016E45, 0x016E65},\n{0x016E46, 0x016E66},\n{0x016E47, 0x016E67},\n{0x016E48, 0x016E68},\n{0x016E49, 0x016E69},\n{0x016E4A, 0x016E6A},\n{0x016E4B, 0x016E6B},\n{0x016E4C, 0x016E6C},\n{0x016E4D, 0x016E6D},\n{0x016E4E, 0x016E6E},\n{0x016E4F, 0x016E6F},\n{0x016E50, 0x016E70},\n{0x016E51, 0x016E71},\n{0x016E52, 0x016E72},\n{0x016E53, 0x016E73},\n{0x016E54, 0x016E74},\n{0x016E55, 0x016E75},\n{0x016E56, 0x016E76},\n{0x016E57, 0x016E77},\n{0x016E58, 0x016E78},\n{0x016E59, 0x016E79},\n{0x016E5A, 0x016E7A},\n{0x016E5B, 0x016E7B},\n{0x016E5C, 0x016E7C},\n{0x016E5D, 0x016E7D},\n{0x016E5E, 0x016E7E},\n{0x016E5F, 0x016E7F},\n{0x01E900, 0x01E922},\n{0x01E901, 0x01E923},\n{0x01E902, 0x01E924},\n{0x01E903, 0x01E925},\n{0x01E904, 0x01E926},\n{0x01E905, 0x01E927},\n{0x01E906, 0x01E928},\n{0x01E907, 0x01E929},\n{0x01E908, 0x01E92A},\n{0x01E909, 0x01E92B},\n{0x01E90A, 0x01E92C},\n{0x01E90B, 0x01E92D},\n{0x01E90C, 0x01E92E},\n{0x01E90D, 0x01E92F},\n{0x01E90E, 0x01E930},\n{0x01E90F, 0x01E931},\n{0x01E910, 0x01E932},\n{0x01E911, 0x01E933},\n{0x01E912, 0x01E934},\n{0x01E913, 0x01E935},\n{0x01E914, 0x01E936},\n{0x01E915, 0x01E937},\n{0x01E916, 0x01E938},\n{0x01E917, 0x01E939},\n{0x01E918, 0x01E93A},\n{0x01E919, 0x01E93B},\n{0x01E91A, 0x01E93C},\n{0x01E91B, 0x01E93D},\n{0x01E91C, 0x01E93E},\n{0x01E91D, 0x01E93F},\n{0x01E91E, 0x01E940},\n{0x01E91F, 0x01E941},\n{0x01E920, 0x01E942},\n{0x01E921, 0x01E943},\n};\n\n// list is always in ascending order, to enable binary search\nconst std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase = {\n{0x000061, 0x000041},\n{0x000062, 0x000042},\n{0x000063, 0x000043},\n{0x000064, 0x000044},\n{0x000065, 0x000045},\n{0x000066, 0x000046},\n{0x000067, 0x000047},\n{0x000068, 0x000048},\n{0x000069, 0x000049},\n{0x00006A, 0x00004A},\n{0x00006B, 0x00004B},\n{0x00006C, 0x00004C},\n{0x00006D, 0x00004D},\n{0x00006E, 0x00004E},\n{0x00006F, 0x00004F},\n{0x000070, 0x000050},\n{0x000071, 0x000051},\n{0x000072, 0x000052},\n{0x000073, 0x000053},\n{0x000074, 0x000054},\n{0x000075, 0x000055},\n{0x000076, 0x000056},\n{0x000077, 0x000057},\n{0x000078, 0x000058},\n{0x000079, 0x000059},\n{0x00007A, 0x00005A},\n{0x0000B5, 0x00039C},\n{0x0000E0, 0x0000C0},\n{0x0000E1, 0x0000C1},\n{0x0000E2, 0x0000C2},\n{0x0000E3, 0x0000C3},\n{0x0000E4, 0x0000C4},\n{0x0000E5, 0x0000C5},\n{0x0000E6, 0x0000C6},\n{0x0000E7, 0x0000C7},\n{0x0000E8, 0x0000C8},\n{0x0000E9, 0x0000C9},\n{0x0000EA, 0x0000CA},\n{0x0000EB, 0x0000CB},\n{0x0000EC, 0x0000CC},\n{0x0000ED, 0x0000CD},\n{0x0000EE, 0x0000CE},\n{0x0000EF, 0x0000CF},\n{0x0000F0, 0x0000D0},\n{0x0000F1, 0x0000D1},\n{0x0000F2, 0x0000D2},\n{0x0000F3, 0x0000D3},\n{0x0000F4, 0x0000D4},\n{0x0000F5, 0x0000D5},\n{0x0000F6, 0x0000D6},\n{0x0000F8, 0x0000D8},\n{0x0000F9, 0x0000D9},\n{0x0000FA, 0x0000DA},\n{0x0000FB, 0x0000DB},\n{0x0000FC, 0x0000DC},\n{0x0000FD, 0x0000DD},\n{0x0000FE, 0x0000DE},\n{0x0000FF, 0x000178},\n{0x000101, 0x000100},\n{0x000103, 0x000102},\n{0x000105, 0x000104},\n{0x000107, 0x000106},\n{0x000109, 0x000108},\n{0x00010B, 0x00010A},\n{0x00010D, 0x00010C},\n{0x00010F, 0x00010E},\n{0x000111, 0x000110},\n{0x000113, 0x000112},\n{0x000115, 0x000114},\n{0x000117, 0x000116},\n{0x000119, 0x000118},\n{0x00011B, 0x00011A},\n{0x00011D, 0x00011C},\n{0x00011F, 0x00011E},\n{0x000121, 0x000120},\n{0x000123, 0x000122},\n{0x000125, 0x000124},\n{0x000127, 0x000126},\n{0x000129, 0x000128},\n{0x00012B, 0x00012A},\n{0x00012D, 0x00012C},\n{0x00012F, 0x00012E},\n{0x000131, 0x000049},\n{0x000133, 0x000132},\n{0x000135, 0x000134},\n{0x000137, 0x000136},\n{0x00013A, 0x000139},\n{0x00013C, 0x00013B},\n{0x00013E, 0x00013D},\n{0x000140, 0x00013F},\n{0x000142, 0x000141},\n{0x000144, 0x000143},\n{0x000146, 0x000145},\n{0x000148, 0x000147},\n{0x00014B, 0x00014A},\n{0x00014D, 0x00014C},\n{0x00014F, 0x00014E},\n{0x000151, 0x000150},\n{0x000153, 0x000152},\n{0x000155, 0x000154},\n{0x000157, 0x000156},\n{0x000159, 0x000158},\n{0x00015B, 0x00015A},\n{0x00015D, 0x00015C},\n{0x00015F, 0x00015E},\n{0x000161, 0x000160},\n{0x000163, 0x000162},\n{0x000165, 0x000164},\n{0x000167, 0x000166},\n{0x000169, 0x000168},\n{0x00016B, 0x00016A},\n{0x00016D, 0x00016C},\n{0x00016F, 0x00016E},\n{0x000171, 0x000170},\n{0x000173, 0x000172},\n{0x000175, 0x000174},\n{0x000177, 0x000176},\n{0x00017A, 0x000179},\n{0x00017C, 0x00017B},\n{0x00017E, 0x00017D},\n{0x00017F, 0x000053},\n{0x000180, 0x000243},\n{0x000183, 0x000182},\n{0x000185, 0x000184},\n{0x000188, 0x000187},\n{0x00018C, 0x00018B},\n{0x000192, 0x000191},\n{0x000195, 0x0001F6},\n{0x000199, 0x000198},\n{0x00019A, 0x00023D},\n{0x00019E, 0x000220},\n{0x0001A1, 0x0001A0},\n{0x0001A3, 0x0001A2},\n{0x0001A5, 0x0001A4},\n{0x0001A8, 0x0001A7},\n{0x0001AD, 0x0001AC},\n{0x0001B0, 0x0001AF},\n{0x0001B4, 0x0001B3},\n{0x0001B6, 0x0001B5},\n{0x0001B9, 0x0001B8},\n{0x0001BD, 0x0001BC},\n{0x0001BF, 0x0001F7},\n{0x0001C5, 0x0001C4},\n{0x0001C6, 0x0001C4},\n{0x0001C8, 0x0001C7},\n{0x0001C9, 0x0001C7},\n{0x0001CB, 0x0001CA},\n{0x0001CC, 0x0001CA},\n{0x0001CE, 0x0001CD},\n{0x0001D0, 0x0001CF},\n{0x0001D2, 0x0001D1},\n{0x0001D4, 0x0001D3},\n{0x0001D6, 0x0001D5},\n{0x0001D8, 0x0001D7},\n{0x0001DA, 0x0001D9},\n{0x0001DC, 0x0001DB},\n{0x0001DD, 0x00018E},\n{0x0001DF, 0x0001DE},\n{0x0001E1, 0x0001E0},\n{0x0001E3, 0x0001E2},\n{0x0001E5, 0x0001E4},\n{0x0001E7, 0x0001E6},\n{0x0001E9, 0x0001E8},\n{0x0001EB, 0x0001EA},\n{0x0001ED, 0x0001EC},\n{0x0001EF, 0x0001EE},\n{0x0001F2, 0x0001F1},\n{0x0001F3, 0x0001F1},\n{0x0001F5, 0x0001F4},\n{0x0001F9, 0x0001F8},\n{0x0001FB, 0x0001FA},\n{0x0001FD, 0x0001FC},\n{0x0001FF, 0x0001FE},\n{0x000201, 0x000200},\n{0x000203, 0x000202},\n{0x000205, 0x000204},\n{0x000207, 0x000206},\n{0x000209, 0x000208},\n{0x00020B, 0x00020A},\n{0x00020D, 0x00020C},\n{0x00020F, 0x00020E},\n{0x000211, 0x000210},\n{0x000213, 0x000212},\n{0x000215, 0x000214},\n{0x000217, 0x000216},\n{0x000219, 0x000218},\n{0x00021B, 0x00021A},\n{0x00021D, 0x00021C},\n{0x00021F, 0x00021E},\n{0x000223, 0x000222},\n{0x000225, 0x000224},\n{0x000227, 0x000226},\n{0x000229, 0x000228},\n{0x00022B, 0x00022A},\n{0x00022D, 0x00022C},\n{0x00022F, 0x00022E},\n{0x000231, 0x000230},\n{0x000233, 0x000232},\n{0x00023C, 0x00023B},\n{0x00023F, 0x002C7E},\n{0x000240, 0x002C7F},\n{0x000242, 0x000241},\n{0x000247, 0x000246},\n{0x000249, 0x000248},\n{0x00024B, 0x00024A},\n{0x00024D, 0x00024C},\n{0x00024F, 0x00024E},\n{0x000250, 0x002C6F},\n{0x000251, 0x002C6D},\n{0x000252, 0x002C70},\n{0x000253, 0x000181},\n{0x000254, 0x000186},\n{0x000256, 0x000189},\n{0x000257, 0x00018A},\n{0x000259, 0x00018F},\n{0x00025B, 0x000190},\n{0x00025C, 0x00A7AB},\n{0x000260, 0x000193},\n{0x000261, 0x00A7AC},\n{0x000263, 0x000194},\n{0x000265, 0x00A78D},\n{0x000266, 0x00A7AA},\n{0x000268, 0x000197},\n{0x000269, 0x000196},\n{0x00026A, 0x00A7AE},\n{0x00026B, 0x002C62},\n{0x00026C, 0x00A7AD},\n{0x00026F, 0x00019C},\n{0x000271, 0x002C6E},\n{0x000272, 0x00019D},\n{0x000275, 0x00019F},\n{0x00027D, 0x002C64},\n{0x000280, 0x0001A6},\n{0x000282, 0x00A7C5},\n{0x000283, 0x0001A9},\n{0x000287, 0x00A7B1},\n{0x000288, 0x0001AE},\n{0x000289, 0x000244},\n{0x00028A, 0x0001B1},\n{0x00028B, 0x0001B2},\n{0x00028C, 0x000245},\n{0x000292, 0x0001B7},\n{0x00029D, 0x00A7B2},\n{0x00029E, 0x00A7B0},\n{0x000345, 0x000399},\n{0x000371, 0x000370},\n{0x000373, 0x000372},\n{0x000377, 0x000376},\n{0x00037B, 0x0003FD},\n{0x00037C, 0x0003FE},\n{0x00037D, 0x0003FF},\n{0x0003AC, 0x000386},\n{0x0003AD, 0x000388},\n{0x0003AE, 0x000389},\n{0x0003AF, 0x00038A},\n{0x0003B1, 0x000391},\n{0x0003B2, 0x000392},\n{0x0003B3, 0x000393},\n{0x0003B4, 0x000394},\n{0x0003B5, 0x000395},\n{0x0003B6, 0x000396},\n{0x0003B7, 0x000397},\n{0x0003B8, 0x000398},\n{0x0003B9, 0x000399},\n{0x0003BA, 0x00039A},\n{0x0003BB, 0x00039B},\n{0x0003BC, 0x00039C},\n{0x0003BD, 0x00039D},\n{0x0003BE, 0x00039E},\n{0x0003BF, 0x00039F},\n{0x0003C0, 0x0003A0},\n{0x0003C1, 0x0003A1},\n{0x0003C2, 0x0003A3},\n{0x0003C3, 0x0003A3},\n{0x0003C4, 0x0003A4},\n{0x0003C5, 0x0003A5},\n{0x0003C6, 0x0003A6},\n{0x0003C7, 0x0003A7},\n{0x0003C8, 0x0003A8},\n{0x0003C9, 0x0003A9},\n{0x0003CA, 0x0003AA},\n{0x0003CB, 0x0003AB},\n{0x0003CC, 0x00038C},\n{0x0003CD, 0x00038E},\n{0x0003CE, 0x00038F},\n{0x0003D0, 0x000392},\n{0x0003D1, 0x000398},\n{0x0003D5, 0x0003A6},\n{0x0003D6, 0x0003A0},\n{0x0003D7, 0x0003CF},\n{0x0003D9, 0x0003D8},\n{0x0003DB, 0x0003DA},\n{0x0003DD, 0x0003DC},\n{0x0003DF, 0x0003DE},\n{0x0003E1, 0x0003E0},\n{0x0003E3, 0x0003E2},\n{0x0003E5, 0x0003E4},\n{0x0003E7, 0x0003E6},\n{0x0003E9, 0x0003E8},\n{0x0003EB, 0x0003EA},\n{0x0003ED, 0x0003EC},\n{0x0003EF, 0x0003EE},\n{0x0003F0, 0x00039A},\n{0x0003F1, 0x0003A1},\n{0x0003F2, 0x0003F9},\n{0x0003F3, 0x00037F},\n{0x0003F5, 0x000395},\n{0x0003F8, 0x0003F7},\n{0x0003FB, 0x0003FA},\n{0x000430, 0x000410},\n{0x000431, 0x000411},\n{0x000432, 0x000412},\n{0x000433, 0x000413},\n{0x000434, 0x000414},\n{0x000435, 0x000415},\n{0x000436, 0x000416},\n{0x000437, 0x000417},\n{0x000438, 0x000418},\n{0x000439, 0x000419},\n{0x00043A, 0x00041A},\n{0x00043B, 0x00041B},\n{0x00043C, 0x00041C},\n{0x00043D, 0x00041D},\n{0x00043E, 0x00041E},\n{0x00043F, 0x00041F},\n{0x000440, 0x000420},\n{0x000441, 0x000421},\n{0x000442, 0x000422},\n{0x000443, 0x000423},\n{0x000444, 0x000424},\n{0x000445, 0x000425},\n{0x000446, 0x000426},\n{0x000447, 0x000427},\n{0x000448, 0x000428},\n{0x000449, 0x000429},\n{0x00044A, 0x00042A},\n{0x00044B, 0x00042B},\n{0x00044C, 0x00042C},\n{0x00044D, 0x00042D},\n{0x00044E, 0x00042E},\n{0x00044F, 0x00042F},\n{0x000450, 0x000400},\n{0x000451, 0x000401},\n{0x000452, 0x000402},\n{0x000453, 0x000403},\n{0x000454, 0x000404},\n{0x000455, 0x000405},\n{0x000456, 0x000406},\n{0x000457, 0x000407},\n{0x000458, 0x000408},\n{0x000459, 0x000409},\n{0x00045A, 0x00040A},\n{0x00045B, 0x00040B},\n{0x00045C, 0x00040C},\n{0x00045D, 0x00040D},\n{0x00045E, 0x00040E},\n{0x00045F, 0x00040F},\n{0x000461, 0x000460},\n{0x000463, 0x000462},\n{0x000465, 0x000464},\n{0x000467, 0x000466},\n{0x000469, 0x000468},\n{0x00046B, 0x00046A},\n{0x00046D, 0x00046C},\n{0x00046F, 0x00046E},\n{0x000471, 0x000470},\n{0x000473, 0x000472},\n{0x000475, 0x000474},\n{0x000477, 0x000476},\n{0x000479, 0x000478},\n{0x00047B, 0x00047A},\n{0x00047D, 0x00047C},\n{0x00047F, 0x00047E},\n{0x000481, 0x000480},\n{0x00048B, 0x00048A},\n{0x00048D, 0x00048C},\n{0x00048F, 0x00048E},\n{0x000491, 0x000490},\n{0x000493, 0x000492},\n{0x000495, 0x000494},\n{0x000497, 0x000496},\n{0x000499, 0x000498},\n{0x00049B, 0x00049A},\n{0x00049D, 0x00049C},\n{0x00049F, 0x00049E},\n{0x0004A1, 0x0004A0},\n{0x0004A3, 0x0004A2},\n{0x0004A5, 0x0004A4},\n{0x0004A7, 0x0004A6},\n{0x0004A9, 0x0004A8},\n{0x0004AB, 0x0004AA},\n{0x0004AD, 0x0004AC},\n{0x0004AF, 0x0004AE},\n{0x0004B1, 0x0004B0},\n{0x0004B3, 0x0004B2},\n{0x0004B5, 0x0004B4},\n{0x0004B7, 0x0004B6},\n{0x0004B9, 0x0004B8},\n{0x0004BB, 0x0004BA},\n{0x0004BD, 0x0004BC},\n{0x0004BF, 0x0004BE},\n{0x0004C2, 0x0004C1},\n{0x0004C4, 0x0004C3},\n{0x0004C6, 0x0004C5},\n{0x0004C8, 0x0004C7},\n{0x0004CA, 0x0004C9},\n{0x0004CC, 0x0004CB},\n{0x0004CE, 0x0004CD},\n{0x0004CF, 0x0004C0},\n{0x0004D1, 0x0004D0},\n{0x0004D3, 0x0004D2},\n{0x0004D5, 0x0004D4},\n{0x0004D7, 0x0004D6},\n{0x0004D9, 0x0004D8},\n{0x0004DB, 0x0004DA},\n{0x0004DD, 0x0004DC},\n{0x0004DF, 0x0004DE},\n{0x0004E1, 0x0004E0},\n{0x0004E3, 0x0004E2},\n{0x0004E5, 0x0004E4},\n{0x0004E7, 0x0004E6},\n{0x0004E9, 0x0004E8},\n{0x0004EB, 0x0004EA},\n{0x0004ED, 0x0004EC},\n{0x0004EF, 0x0004EE},\n{0x0004F1, 0x0004F0},\n{0x0004F3, 0x0004F2},\n{0x0004F5, 0x0004F4},\n{0x0004F7, 0x0004F6},\n{0x0004F9, 0x0004F8},\n{0x0004FB, 0x0004FA},\n{0x0004FD, 0x0004FC},\n{0x0004FF, 0x0004FE},\n{0x000501, 0x000500},\n{0x000503, 0x000502},\n{0x000505, 0x000504},\n{0x000507, 0x000506},\n{0x000509, 0x000508},\n{0x00050B, 0x00050A},\n{0x00050D, 0x00050C},\n{0x00050F, 0x00050E},\n{0x000511, 0x000510},\n{0x000513, 0x000512},\n{0x000515, 0x000514},\n{0x000517, 0x000516},\n{0x000519, 0x000518},\n{0x00051B, 0x00051A},\n{0x00051D, 0x00051C},\n{0x00051F, 0x00051E},\n{0x000521, 0x000520},\n{0x000523, 0x000522},\n{0x000525, 0x000524},\n{0x000527, 0x000526},\n{0x000529, 0x000528},\n{0x00052B, 0x00052A},\n{0x00052D, 0x00052C},\n{0x00052F, 0x00052E},\n{0x000561, 0x000531},\n{0x000562, 0x000532},\n{0x000563, 0x000533},\n{0x000564, 0x000534},\n{0x000565, 0x000535},\n{0x000566, 0x000536},\n{0x000567, 0x000537},\n{0x000568, 0x000538},\n{0x000569, 0x000539},\n{0x00056A, 0x00053A},\n{0x00056B, 0x00053B},\n{0x00056C, 0x00053C},\n{0x00056D, 0x00053D},\n{0x00056E, 0x00053E},\n{0x00056F, 0x00053F},\n{0x000570, 0x000540},\n{0x000571, 0x000541},\n{0x000572, 0x000542},\n{0x000573, 0x000543},\n{0x000574, 0x000544},\n{0x000575, 0x000545},\n{0x000576, 0x000546},\n{0x000577, 0x000547},\n{0x000578, 0x000548},\n{0x000579, 0x000549},\n{0x00057A, 0x00054A},\n{0x00057B, 0x00054B},\n{0x00057C, 0x00054C},\n{0x00057D, 0x00054D},\n{0x00057E, 0x00054E},\n{0x00057F, 0x00054F},\n{0x000580, 0x000550},\n{0x000581, 0x000551},\n{0x000582, 0x000552},\n{0x000583, 0x000553},\n{0x000584, 0x000554},\n{0x000585, 0x000555},\n{0x000586, 0x000556},\n{0x0010D0, 0x001C90},\n{0x0010D1, 0x001C91},\n{0x0010D2, 0x001C92},\n{0x0010D3, 0x001C93},\n{0x0010D4, 0x001C94},\n{0x0010D5, 0x001C95},\n{0x0010D6, 0x001C96},\n{0x0010D7, 0x001C97},\n{0x0010D8, 0x001C98},\n{0x0010D9, 0x001C99},\n{0x0010DA, 0x001C9A},\n{0x0010DB, 0x001C9B},\n{0x0010DC, 0x001C9C},\n{0x0010DD, 0x001C9D},\n{0x0010DE, 0x001C9E},\n{0x0010DF, 0x001C9F},\n{0x0010E0, 0x001CA0},\n{0x0010E1, 0x001CA1},\n{0x0010E2, 0x001CA2},\n{0x0010E3, 0x001CA3},\n{0x0010E4, 0x001CA4},\n{0x0010E5, 0x001CA5},\n{0x0010E6, 0x001CA6},\n{0x0010E7, 0x001CA7},\n{0x0010E8, 0x001CA8},\n{0x0010E9, 0x001CA9},\n{0x0010EA, 0x001CAA},\n{0x0010EB, 0x001CAB},\n{0x0010EC, 0x001CAC},\n{0x0010ED, 0x001CAD},\n{0x0010EE, 0x001CAE},\n{0x0010EF, 0x001CAF},\n{0x0010F0, 0x001CB0},\n{0x0010F1, 0x001CB1},\n{0x0010F2, 0x001CB2},\n{0x0010F3, 0x001CB3},\n{0x0010F4, 0x001CB4},\n{0x0010F5, 0x001CB5},\n{0x0010F6, 0x001CB6},\n{0x0010F7, 0x001CB7},\n{0x0010F8, 0x001CB8},\n{0x0010F9, 0x001CB9},\n{0x0010FA, 0x001CBA},\n{0x0010FD, 0x001CBD},\n{0x0010FE, 0x001CBE},\n{0x0010FF, 0x001CBF},\n{0x0013F8, 0x0013F0},\n{0x0013F9, 0x0013F1},\n{0x0013FA, 0x0013F2},\n{0x0013FB, 0x0013F3},\n{0x0013FC, 0x0013F4},\n{0x0013FD, 0x0013F5},\n{0x001C80, 0x000412},\n{0x001C81, 0x000414},\n{0x001C82, 0x00041E},\n{0x001C83, 0x000421},\n{0x001C84, 0x000422},\n{0x001C85, 0x000422},\n{0x001C86, 0x00042A},\n{0x001C87, 0x000462},\n{0x001C88, 0x00A64A},\n{0x001D79, 0x00A77D},\n{0x001D7D, 0x002C63},\n{0x001D8E, 0x00A7C6},\n{0x001E01, 0x001E00},\n{0x001E03, 0x001E02},\n{0x001E05, 0x001E04},\n{0x001E07, 0x001E06},\n{0x001E09, 0x001E08},\n{0x001E0B, 0x001E0A},\n{0x001E0D, 0x001E0C},\n{0x001E0F, 0x001E0E},\n{0x001E11, 0x001E10},\n{0x001E13, 0x001E12},\n{0x001E15, 0x001E14},\n{0x001E17, 0x001E16},\n{0x001E19, 0x001E18},\n{0x001E1B, 0x001E1A},\n{0x001E1D, 0x001E1C},\n{0x001E1F, 0x001E1E},\n{0x001E21, 0x001E20},\n{0x001E23, 0x001E22},\n{0x001E25, 0x001E24},\n{0x001E27, 0x001E26},\n{0x001E29, 0x001E28},\n{0x001E2B, 0x001E2A},\n{0x001E2D, 0x001E2C},\n{0x001E2F, 0x001E2E},\n{0x001E31, 0x001E30},\n{0x001E33, 0x001E32},\n{0x001E35, 0x001E34},\n{0x001E37, 0x001E36},\n{0x001E39, 0x001E38},\n{0x001E3B, 0x001E3A},\n{0x001E3D, 0x001E3C},\n{0x001E3F, 0x001E3E},\n{0x001E41, 0x001E40},\n{0x001E43, 0x001E42},\n{0x001E45, 0x001E44},\n{0x001E47, 0x001E46},\n{0x001E49, 0x001E48},\n{0x001E4B, 0x001E4A},\n{0x001E4D, 0x001E4C},\n{0x001E4F, 0x001E4E},\n{0x001E51, 0x001E50},\n{0x001E53, 0x001E52},\n{0x001E55, 0x001E54},\n{0x001E57, 0x001E56},\n{0x001E59, 0x001E58},\n{0x001E5B, 0x001E5A},\n{0x001E5D, 0x001E5C},\n{0x001E5F, 0x001E5E},\n{0x001E61, 0x001E60},\n{0x001E63, 0x001E62},\n{0x001E65, 0x001E64},\n{0x001E67, 0x001E66},\n{0x001E69, 0x001E68},\n{0x001E6B, 0x001E6A},\n{0x001E6D, 0x001E6C},\n{0x001E6F, 0x001E6E},\n{0x001E71, 0x001E70},\n{0x001E73, 0x001E72},\n{0x001E75, 0x001E74},\n{0x001E77, 0x001E76},\n{0x001E79, 0x001E78},\n{0x001E7B, 0x001E7A},\n{0x001E7D, 0x001E7C},\n{0x001E7F, 0x001E7E},\n{0x001E81, 0x001E80},\n{0x001E83, 0x001E82},\n{0x001E85, 0x001E84},\n{0x001E87, 0x001E86},\n{0x001E89, 0x001E88},\n{0x001E8B, 0x001E8A},\n{0x001E8D, 0x001E8C},\n{0x001E8F, 0x001E8E},\n{0x001E91, 0x001E90},\n{0x001E93, 0x001E92},\n{0x001E95, 0x001E94},\n{0x001E9B, 0x001E60},\n{0x001EA1, 0x001EA0},\n{0x001EA3, 0x001EA2},\n{0x001EA5, 0x001EA4},\n{0x001EA7, 0x001EA6},\n{0x001EA9, 0x001EA8},\n{0x001EAB, 0x001EAA},\n{0x001EAD, 0x001EAC},\n{0x001EAF, 0x001EAE},\n{0x001EB1, 0x001EB0},\n{0x001EB3, 0x001EB2},\n{0x001EB5, 0x001EB4},\n{0x001EB7, 0x001EB6},\n{0x001EB9, 0x001EB8},\n{0x001EBB, 0x001EBA},\n{0x001EBD, 0x001EBC},\n{0x001EBF, 0x001EBE},\n{0x001EC1, 0x001EC0},\n{0x001EC3, 0x001EC2},\n{0x001EC5, 0x001EC4},\n{0x001EC7, 0x001EC6},\n{0x001EC9, 0x001EC8},\n{0x001ECB, 0x001ECA},\n{0x001ECD, 0x001ECC},\n{0x001ECF, 0x001ECE},\n{0x001ED1, 0x001ED0},\n{0x001ED3, 0x001ED2},\n{0x001ED5, 0x001ED4},\n{0x001ED7, 0x001ED6},\n{0x001ED9, 0x001ED8},\n{0x001EDB, 0x001EDA},\n{0x001EDD, 0x001EDC},\n{0x001EDF, 0x001EDE},\n{0x001EE1, 0x001EE0},\n{0x001EE3, 0x001EE2},\n{0x001EE5, 0x001EE4},\n{0x001EE7, 0x001EE6},\n{0x001EE9, 0x001EE8},\n{0x001EEB, 0x001EEA},\n{0x001EED, 0x001EEC},\n{0x001EEF, 0x001EEE},\n{0x001EF1, 0x001EF0},\n{0x001EF3, 0x001EF2},\n{0x001EF5, 0x001EF4},\n{0x001EF7, 0x001EF6},\n{0x001EF9, 0x001EF8},\n{0x001EFB, 0x001EFA},\n{0x001EFD, 0x001EFC},\n{0x001EFF, 0x001EFE},\n{0x001F00, 0x001F08},\n{0x001F01, 0x001F09},\n{0x001F02, 0x001F0A},\n{0x001F03, 0x001F0B},\n{0x001F04, 0x001F0C},\n{0x001F05, 0x001F0D},\n{0x001F06, 0x001F0E},\n{0x001F07, 0x001F0F},\n{0x001F10, 0x001F18},\n{0x001F11, 0x001F19},\n{0x001F12, 0x001F1A},\n{0x001F13, 0x001F1B},\n{0x001F14, 0x001F1C},\n{0x001F15, 0x001F1D},\n{0x001F20, 0x001F28},\n{0x001F21, 0x001F29},\n{0x001F22, 0x001F2A},\n{0x001F23, 0x001F2B},\n{0x001F24, 0x001F2C},\n{0x001F25, 0x001F2D},\n{0x001F26, 0x001F2E},\n{0x001F27, 0x001F2F},\n{0x001F30, 0x001F38},\n{0x001F31, 0x001F39},\n{0x001F32, 0x001F3A},\n{0x001F33, 0x001F3B},\n{0x001F34, 0x001F3C},\n{0x001F35, 0x001F3D},\n{0x001F36, 0x001F3E},\n{0x001F37, 0x001F3F},\n{0x001F40, 0x001F48},\n{0x001F41, 0x001F49},\n{0x001F42, 0x001F4A},\n{0x001F43, 0x001F4B},\n{0x001F44, 0x001F4C},\n{0x001F45, 0x001F4D},\n{0x001F51, 0x001F59},\n{0x001F53, 0x001F5B},\n{0x001F55, 0x001F5D},\n{0x001F57, 0x001F5F},\n{0x001F60, 0x001F68},\n{0x001F61, 0x001F69},\n{0x001F62, 0x001F6A},\n{0x001F63, 0x001F6B},\n{0x001F64, 0x001F6C},\n{0x001F65, 0x001F6D},\n{0x001F66, 0x001F6E},\n{0x001F67, 0x001F6F},\n{0x001F70, 0x001FBA},\n{0x001F71, 0x001FBB},\n{0x001F72, 0x001FC8},\n{0x001F73, 0x001FC9},\n{0x001F74, 0x001FCA},\n{0x001F75, 0x001FCB},\n{0x001F76, 0x001FDA},\n{0x001F77, 0x001FDB},\n{0x001F78, 0x001FF8},\n{0x001F79, 0x001FF9},\n{0x001F7A, 0x001FEA},\n{0x001F7B, 0x001FEB},\n{0x001F7C, 0x001FFA},\n{0x001F7D, 0x001FFB},\n{0x001F80, 0x001F88},\n{0x001F81, 0x001F89},\n{0x001F82, 0x001F8A},\n{0x001F83, 0x001F8B},\n{0x001F84, 0x001F8C},\n{0x001F85, 0x001F8D},\n{0x001F86, 0x001F8E},\n{0x001F87, 0x001F8F},\n{0x001F90, 0x001F98},\n{0x001F91, 0x001F99},\n{0x001F92, 0x001F9A},\n{0x001F93, 0x001F9B},\n{0x001F94, 0x001F9C},\n{0x001F95, 0x001F9D},\n{0x001F96, 0x001F9E},\n{0x001F97, 0x001F9F},\n{0x001FA0, 0x001FA8},\n{0x001FA1, 0x001FA9},\n{0x001FA2, 0x001FAA},\n{0x001FA3, 0x001FAB},\n{0x001FA4, 0x001FAC},\n{0x001FA5, 0x001FAD},\n{0x001FA6, 0x001FAE},\n{0x001FA7, 0x001FAF},\n{0x001FB0, 0x001FB8},\n{0x001FB1, 0x001FB9},\n{0x001FB3, 0x001FBC},\n{0x001FBE, 0x000399},\n{0x001FC3, 0x001FCC},\n{0x001FD0, 0x001FD8},\n{0x001FD1, 0x001FD9},\n{0x001FE0, 0x001FE8},\n{0x001FE1, 0x001FE9},\n{0x001FE5, 0x001FEC},\n{0x001FF3, 0x001FFC},\n{0x00214E, 0x002132},\n{0x002170, 0x002160},\n{0x002171, 0x002161},\n{0x002172, 0x002162},\n{0x002173, 0x002163},\n{0x002174, 0x002164},\n{0x002175, 0x002165},\n{0x002176, 0x002166},\n{0x002177, 0x002167},\n{0x002178, 0x002168},\n{0x002179, 0x002169},\n{0x00217A, 0x00216A},\n{0x00217B, 0x00216B},\n{0x00217C, 0x00216C},\n{0x00217D, 0x00216D},\n{0x00217E, 0x00216E},\n{0x00217F, 0x00216F},\n{0x002184, 0x002183},\n{0x0024D0, 0x0024B6},\n{0x0024D1, 0x0024B7},\n{0x0024D2, 0x0024B8},\n{0x0024D3, 0x0024B9},\n{0x0024D4, 0x0024BA},\n{0x0024D5, 0x0024BB},\n{0x0024D6, 0x0024BC},\n{0x0024D7, 0x0024BD},\n{0x0024D8, 0x0024BE},\n{0x0024D9, 0x0024BF},\n{0x0024DA, 0x0024C0},\n{0x0024DB, 0x0024C1},\n{0x0024DC, 0x0024C2},\n{0x0024DD, 0x0024C3},\n{0x0024DE, 0x0024C4},\n{0x0024DF, 0x0024C5},\n{0x0024E0, 0x0024C6},\n{0x0024E1, 0x0024C7},\n{0x0024E2, 0x0024C8},\n{0x0024E3, 0x0024C9},\n{0x0024E4, 0x0024CA},\n{0x0024E5, 0x0024CB},\n{0x0024E6, 0x0024CC},\n{0x0024E7, 0x0024CD},\n{0x0024E8, 0x0024CE},\n{0x0024E9, 0x0024CF},\n{0x002C30, 0x002C00},\n{0x002C31, 0x002C01},\n{0x002C32, 0x002C02},\n{0x002C33, 0x002C03},\n{0x002C34, 0x002C04},\n{0x002C35, 0x002C05},\n{0x002C36, 0x002C06},\n{0x002C37, 0x002C07},\n{0x002C38, 0x002C08},\n{0x002C39, 0x002C09},\n{0x002C3A, 0x002C0A},\n{0x002C3B, 0x002C0B},\n{0x002C3C, 0x002C0C},\n{0x002C3D, 0x002C0D},\n{0x002C3E, 0x002C0E},\n{0x002C3F, 0x002C0F},\n{0x002C40, 0x002C10},\n{0x002C41, 0x002C11},\n{0x002C42, 0x002C12},\n{0x002C43, 0x002C13},\n{0x002C44, 0x002C14},\n{0x002C45, 0x002C15},\n{0x002C46, 0x002C16},\n{0x002C47, 0x002C17},\n{0x002C48, 0x002C18},\n{0x002C49, 0x002C19},\n{0x002C4A, 0x002C1A},\n{0x002C4B, 0x002C1B},\n{0x002C4C, 0x002C1C},\n{0x002C4D, 0x002C1D},\n{0x002C4E, 0x002C1E},\n{0x002C4F, 0x002C1F},\n{0x002C50, 0x002C20},\n{0x002C51, 0x002C21},\n{0x002C52, 0x002C22},\n{0x002C53, 0x002C23},\n{0x002C54, 0x002C24},\n{0x002C55, 0x002C25},\n{0x002C56, 0x002C26},\n{0x002C57, 0x002C27},\n{0x002C58, 0x002C28},\n{0x002C59, 0x002C29},\n{0x002C5A, 0x002C2A},\n{0x002C5B, 0x002C2B},\n{0x002C5C, 0x002C2C},\n{0x002C5D, 0x002C2D},\n{0x002C5E, 0x002C2E},\n{0x002C5F, 0x002C2F},\n{0x002C61, 0x002C60},\n{0x002C65, 0x00023A},\n{0x002C66, 0x00023E},\n{0x002C68, 0x002C67},\n{0x002C6A, 0x002C69},\n{0x002C6C, 0x002C6B},\n{0x002C73, 0x002C72},\n{0x002C76, 0x002C75},\n{0x002C81, 0x002C80},\n{0x002C83, 0x002C82},\n{0x002C85, 0x002C84},\n{0x002C87, 0x002C86},\n{0x002C89, 0x002C88},\n{0x002C8B, 0x002C8A},\n{0x002C8D, 0x002C8C},\n{0x002C8F, 0x002C8E},\n{0x002C91, 0x002C90},\n{0x002C93, 0x002C92},\n{0x002C95, 0x002C94},\n{0x002C97, 0x002C96},\n{0x002C99, 0x002C98},\n{0x002C9B, 0x002C9A},\n{0x002C9D, 0x002C9C},\n{0x002C9F, 0x002C9E},\n{0x002CA1, 0x002CA0},\n{0x002CA3, 0x002CA2},\n{0x002CA5, 0x002CA4},\n{0x002CA7, 0x002CA6},\n{0x002CA9, 0x002CA8},\n{0x002CAB, 0x002CAA},\n{0x002CAD, 0x002CAC},\n{0x002CAF, 0x002CAE},\n{0x002CB1, 0x002CB0},\n{0x002CB3, 0x002CB2},\n{0x002CB5, 0x002CB4},\n{0x002CB7, 0x002CB6},\n{0x002CB9, 0x002CB8},\n{0x002CBB, 0x002CBA},\n{0x002CBD, 0x002CBC},\n{0x002CBF, 0x002CBE},\n{0x002CC1, 0x002CC0},\n{0x002CC3, 0x002CC2},\n{0x002CC5, 0x002CC4},\n{0x002CC7, 0x002CC6},\n{0x002CC9, 0x002CC8},\n{0x002CCB, 0x002CCA},\n{0x002CCD, 0x002CCC},\n{0x002CCF, 0x002CCE},\n{0x002CD1, 0x002CD0},\n{0x002CD3, 0x002CD2},\n{0x002CD5, 0x002CD4},\n{0x002CD7, 0x002CD6},\n{0x002CD9, 0x002CD8},\n{0x002CDB, 0x002CDA},\n{0x002CDD, 0x002CDC},\n{0x002CDF, 0x002CDE},\n{0x002CE1, 0x002CE0},\n{0x002CE3, 0x002CE2},\n{0x002CEC, 0x002CEB},\n{0x002CEE, 0x002CED},\n{0x002CF3, 0x002CF2},\n{0x002D00, 0x0010A0},\n{0x002D01, 0x0010A1},\n{0x002D02, 0x0010A2},\n{0x002D03, 0x0010A3},\n{0x002D04, 0x0010A4},\n{0x002D05, 0x0010A5},\n{0x002D06, 0x0010A6},\n{0x002D07, 0x0010A7},\n{0x002D08, 0x0010A8},\n{0x002D09, 0x0010A9},\n{0x002D0A, 0x0010AA},\n{0x002D0B, 0x0010AB},\n{0x002D0C, 0x0010AC},\n{0x002D0D, 0x0010AD},\n{0x002D0E, 0x0010AE},\n{0x002D0F, 0x0010AF},\n{0x002D10, 0x0010B0},\n{0x002D11, 0x0010B1},\n{0x002D12, 0x0010B2},\n{0x002D13, 0x0010B3},\n{0x002D14, 0x0010B4},\n{0x002D15, 0x0010B5},\n{0x002D16, 0x0010B6},\n{0x002D17, 0x0010B7},\n{0x002D18, 0x0010B8},\n{0x002D19, 0x0010B9},\n{0x002D1A, 0x0010BA},\n{0x002D1B, 0x0010BB},\n{0x002D1C, 0x0010BC},\n{0x002D1D, 0x0010BD},\n{0x002D1E, 0x0010BE},\n{0x002D1F, 0x0010BF},\n{0x002D20, 0x0010C0},\n{0x002D21, 0x0010C1},\n{0x002D22, 0x0010C2},\n{0x002D23, 0x0010C3},\n{0x002D24, 0x0010C4},\n{0x002D25, 0x0010C5},\n{0x002D27, 0x0010C7},\n{0x002D2D, 0x0010CD},\n{0x00A641, 0x00A640},\n{0x00A643, 0x00A642},\n{0x00A645, 0x00A644},\n{0x00A647, 0x00A646},\n{0x00A649, 0x00A648},\n{0x00A64B, 0x00A64A},\n{0x00A64D, 0x00A64C},\n{0x00A64F, 0x00A64E},\n{0x00A651, 0x00A650},\n{0x00A653, 0x00A652},\n{0x00A655, 0x00A654},\n{0x00A657, 0x00A656},\n{0x00A659, 0x00A658},\n{0x00A65B, 0x00A65A},\n{0x00A65D, 0x00A65C},\n{0x00A65F, 0x00A65E},\n{0x00A661, 0x00A660},\n{0x00A663, 0x00A662},\n{0x00A665, 0x00A664},\n{0x00A667, 0x00A666},\n{0x00A669, 0x00A668},\n{0x00A66B, 0x00A66A},\n{0x00A66D, 0x00A66C},\n{0x00A681, 0x00A680},\n{0x00A683, 0x00A682},\n{0x00A685, 0x00A684},\n{0x00A687, 0x00A686},\n{0x00A689, 0x00A688},\n{0x00A68B, 0x00A68A},\n{0x00A68D, 0x00A68C},\n{0x00A68F, 0x00A68E},\n{0x00A691, 0x00A690},\n{0x00A693, 0x00A692},\n{0x00A695, 0x00A694},\n{0x00A697, 0x00A696},\n{0x00A699, 0x00A698},\n{0x00A69B, 0x00A69A},\n{0x00A723, 0x00A722},\n{0x00A725, 0x00A724},\n{0x00A727, 0x00A726},\n{0x00A729, 0x00A728},\n{0x00A72B, 0x00A72A},\n{0x00A72D, 0x00A72C},\n{0x00A72F, 0x00A72E},\n{0x00A733, 0x00A732},\n{0x00A735, 0x00A734},\n{0x00A737, 0x00A736},\n{0x00A739, 0x00A738},\n{0x00A73B, 0x00A73A},\n{0x00A73D, 0x00A73C},\n{0x00A73F, 0x00A73E},\n{0x00A741, 0x00A740},\n{0x00A743, 0x00A742},\n{0x00A745, 0x00A744},\n{0x00A747, 0x00A746},\n{0x00A749, 0x00A748},\n{0x00A74B, 0x00A74A},\n{0x00A74D, 0x00A74C},\n{0x00A74F, 0x00A74E},\n{0x00A751, 0x00A750},\n{0x00A753, 0x00A752},\n{0x00A755, 0x00A754},\n{0x00A757, 0x00A756},\n{0x00A759, 0x00A758},\n{0x00A75B, 0x00A75A},\n{0x00A75D, 0x00A75C},\n{0x00A75F, 0x00A75E},\n{0x00A761, 0x00A760},\n{0x00A763, 0x00A762},\n{0x00A765, 0x00A764},\n{0x00A767, 0x00A766},\n{0x00A769, 0x00A768},\n{0x00A76B, 0x00A76A},\n{0x00A76D, 0x00A76C},\n{0x00A76F, 0x00A76E},\n{0x00A77A, 0x00A779},\n{0x00A77C, 0x00A77B},\n{0x00A77F, 0x00A77E},\n{0x00A781, 0x00A780},\n{0x00A783, 0x00A782},\n{0x00A785, 0x00A784},\n{0x00A787, 0x00A786},\n{0x00A78C, 0x00A78B},\n{0x00A791, 0x00A790},\n{0x00A793, 0x00A792},\n{0x00A794, 0x00A7C4},\n{0x00A797, 0x00A796},\n{0x00A799, 0x00A798},\n{0x00A79B, 0x00A79A},\n{0x00A79D, 0x00A79C},\n{0x00A79F, 0x00A79E},\n{0x00A7A1, 0x00A7A0},\n{0x00A7A3, 0x00A7A2},\n{0x00A7A5, 0x00A7A4},\n{0x00A7A7, 0x00A7A6},\n{0x00A7A9, 0x00A7A8},\n{0x00A7B5, 0x00A7B4},\n{0x00A7B7, 0x00A7B6},\n{0x00A7B9, 0x00A7B8},\n{0x00A7BB, 0x00A7BA},\n{0x00A7BD, 0x00A7BC},\n{0x00A7BF, 0x00A7BE},\n{0x00A7C1, 0x00A7C0},\n{0x00A7C3, 0x00A7C2},\n{0x00A7C8, 0x00A7C7},\n{0x00A7CA, 0x00A7C9},\n{0x00A7D1, 0x00A7D0},\n{0x00A7D7, 0x00A7D6},\n{0x00A7D9, 0x00A7D8},\n{0x00A7F6, 0x00A7F5},\n{0x00AB53, 0x00A7B3},\n{0x00AB70, 0x0013A0},\n{0x00AB71, 0x0013A1},\n{0x00AB72, 0x0013A2},\n{0x00AB73, 0x0013A3},\n{0x00AB74, 0x0013A4},\n{0x00AB75, 0x0013A5},\n{0x00AB76, 0x0013A6},\n{0x00AB77, 0x0013A7},\n{0x00AB78, 0x0013A8},\n{0x00AB79, 0x0013A9},\n{0x00AB7A, 0x0013AA},\n{0x00AB7B, 0x0013AB},\n{0x00AB7C, 0x0013AC},\n{0x00AB7D, 0x0013AD},\n{0x00AB7E, 0x0013AE},\n{0x00AB7F, 0x0013AF},\n{0x00AB80, 0x0013B0},\n{0x00AB81, 0x0013B1},\n{0x00AB82, 0x0013B2},\n{0x00AB83, 0x0013B3},\n{0x00AB84, 0x0013B4},\n{0x00AB85, 0x0013B5},\n{0x00AB86, 0x0013B6},\n{0x00AB87, 0x0013B7},\n{0x00AB88, 0x0013B8},\n{0x00AB89, 0x0013B9},\n{0x00AB8A, 0x0013BA},\n{0x00AB8B, 0x0013BB},\n{0x00AB8C, 0x0013BC},\n{0x00AB8D, 0x0013BD},\n{0x00AB8E, 0x0013BE},\n{0x00AB8F, 0x0013BF},\n{0x00AB90, 0x0013C0},\n{0x00AB91, 0x0013C1},\n{0x00AB92, 0x0013C2},\n{0x00AB93, 0x0013C3},\n{0x00AB94, 0x0013C4},\n{0x00AB95, 0x0013C5},\n{0x00AB96, 0x0013C6},\n{0x00AB97, 0x0013C7},\n{0x00AB98, 0x0013C8},\n{0x00AB99, 0x0013C9},\n{0x00AB9A, 0x0013CA},\n{0x00AB9B, 0x0013CB},\n{0x00AB9C, 0x0013CC},\n{0x00AB9D, 0x0013CD},\n{0x00AB9E, 0x0013CE},\n{0x00AB9F, 0x0013CF},\n{0x00ABA0, 0x0013D0},\n{0x00ABA1, 0x0013D1},\n{0x00ABA2, 0x0013D2},\n{0x00ABA3, 0x0013D3},\n{0x00ABA4, 0x0013D4},\n{0x00ABA5, 0x0013D5},\n{0x00ABA6, 0x0013D6},\n{0x00ABA7, 0x0013D7},\n{0x00ABA8, 0x0013D8},\n{0x00ABA9, 0x0013D9},\n{0x00ABAA, 0x0013DA},\n{0x00ABAB, 0x0013DB},\n{0x00ABAC, 0x0013DC},\n{0x00ABAD, 0x0013DD},\n{0x00ABAE, 0x0013DE},\n{0x00ABAF, 0x0013DF},\n{0x00ABB0, 0x0013E0},\n{0x00ABB1, 0x0013E1},\n{0x00ABB2, 0x0013E2},\n{0x00ABB3, 0x0013E3},\n{0x00ABB4, 0x0013E4},\n{0x00ABB5, 0x0013E5},\n{0x00ABB6, 0x0013E6},\n{0x00ABB7, 0x0013E7},\n{0x00ABB8, 0x0013E8},\n{0x00ABB9, 0x0013E9},\n{0x00ABBA, 0x0013EA},\n{0x00ABBB, 0x0013EB},\n{0x00ABBC, 0x0013EC},\n{0x00ABBD, 0x0013ED},\n{0x00ABBE, 0x0013EE},\n{0x00ABBF, 0x0013EF},\n{0x00FF41, 0x00FF21},\n{0x00FF42, 0x00FF22},\n{0x00FF43, 0x00FF23},\n{0x00FF44, 0x00FF24},\n{0x00FF45, 0x00FF25},\n{0x00FF46, 0x00FF26},\n{0x00FF47, 0x00FF27},\n{0x00FF48, 0x00FF28},\n{0x00FF49, 0x00FF29},\n{0x00FF4A, 0x00FF2A},\n{0x00FF4B, 0x00FF2B},\n{0x00FF4C, 0x00FF2C},\n{0x00FF4D, 0x00FF2D},\n{0x00FF4E, 0x00FF2E},\n{0x00FF4F, 0x00FF2F},\n{0x00FF50, 0x00FF30},\n{0x00FF51, 0x00FF31},\n{0x00FF52, 0x00FF32},\n{0x00FF53, 0x00FF33},\n{0x00FF54, 0x00FF34},\n{0x00FF55, 0x00FF35},\n{0x00FF56, 0x00FF36},\n{0x00FF57, 0x00FF37},\n{0x00FF58, 0x00FF38},\n{0x00FF59, 0x00FF39},\n{0x00FF5A, 0x00FF3A},\n{0x010428, 0x010400},\n{0x010429, 0x010401},\n{0x01042A, 0x010402},\n{0x01042B, 0x010403},\n{0x01042C, 0x010404},\n{0x01042D, 0x010405},\n{0x01042E, 0x010406},\n{0x01042F, 0x010407},\n{0x010430, 0x010408},\n{0x010431, 0x010409},\n{0x010432, 0x01040A},\n{0x010433, 0x01040B},\n{0x010434, 0x01040C},\n{0x010435, 0x01040D},\n{0x010436, 0x01040E},\n{0x010437, 0x01040F},\n{0x010438, 0x010410},\n{0x010439, 0x010411},\n{0x01043A, 0x010412},\n{0x01043B, 0x010413},\n{0x01043C, 0x010414},\n{0x01043D, 0x010415},\n{0x01043E, 0x010416},\n{0x01043F, 0x010417},\n{0x010440, 0x010418},\n{0x010441, 0x010419},\n{0x010442, 0x01041A},\n{0x010443, 0x01041B},\n{0x010444, 0x01041C},\n{0x010445, 0x01041D},\n{0x010446, 0x01041E},\n{0x010447, 0x01041F},\n{0x010448, 0x010420},\n{0x010449, 0x010421},\n{0x01044A, 0x010422},\n{0x01044B, 0x010423},\n{0x01044C, 0x010424},\n{0x01044D, 0x010425},\n{0x01044E, 0x010426},\n{0x01044F, 0x010427},\n{0x0104D8, 0x0104B0},\n{0x0104D9, 0x0104B1},\n{0x0104DA, 0x0104B2},\n{0x0104DB, 0x0104B3},\n{0x0104DC, 0x0104B4},\n{0x0104DD, 0x0104B5},\n{0x0104DE, 0x0104B6},\n{0x0104DF, 0x0104B7},\n{0x0104E0, 0x0104B8},\n{0x0104E1, 0x0104B9},\n{0x0104E2, 0x0104BA},\n{0x0104E3, 0x0104BB},\n{0x0104E4, 0x0104BC},\n{0x0104E5, 0x0104BD},\n{0x0104E6, 0x0104BE},\n{0x0104E7, 0x0104BF},\n{0x0104E8, 0x0104C0},\n{0x0104E9, 0x0104C1},\n{0x0104EA, 0x0104C2},\n{0x0104EB, 0x0104C3},\n{0x0104EC, 0x0104C4},\n{0x0104ED, 0x0104C5},\n{0x0104EE, 0x0104C6},\n{0x0104EF, 0x0104C7},\n{0x0104F0, 0x0104C8},\n{0x0104F1, 0x0104C9},\n{0x0104F2, 0x0104CA},\n{0x0104F3, 0x0104CB},\n{0x0104F4, 0x0104CC},\n{0x0104F5, 0x0104CD},\n{0x0104F6, 0x0104CE},\n{0x0104F7, 0x0104CF},\n{0x0104F8, 0x0104D0},\n{0x0104F9, 0x0104D1},\n{0x0104FA, 0x0104D2},\n{0x0104FB, 0x0104D3},\n{0x010597, 0x010570},\n{0x010598, 0x010571},\n{0x010599, 0x010572},\n{0x01059A, 0x010573},\n{0x01059B, 0x010574},\n{0x01059C, 0x010575},\n{0x01059D, 0x010576},\n{0x01059E, 0x010577},\n{0x01059F, 0x010578},\n{0x0105A0, 0x010579},\n{0x0105A1, 0x01057A},\n{0x0105A3, 0x01057C},\n{0x0105A4, 0x01057D},\n{0x0105A5, 0x01057E},\n{0x0105A6, 0x01057F},\n{0x0105A7, 0x010580},\n{0x0105A8, 0x010581},\n{0x0105A9, 0x010582},\n{0x0105AA, 0x010583},\n{0x0105AB, 0x010584},\n{0x0105AC, 0x010585},\n{0x0105AD, 0x010586},\n{0x0105AE, 0x010587},\n{0x0105AF, 0x010588},\n{0x0105B0, 0x010589},\n{0x0105B1, 0x01058A},\n{0x0105B3, 0x01058C},\n{0x0105B4, 0x01058D},\n{0x0105B5, 0x01058E},\n{0x0105B6, 0x01058F},\n{0x0105B7, 0x010590},\n{0x0105B8, 0x010591},\n{0x0105B9, 0x010592},\n{0x0105BB, 0x010594},\n{0x0105BC, 0x010595},\n{0x010CC0, 0x010C80},\n{0x010CC1, 0x010C81},\n{0x010CC2, 0x010C82},\n{0x010CC3, 0x010C83},\n{0x010CC4, 0x010C84},\n{0x010CC5, 0x010C85},\n{0x010CC6, 0x010C86},\n{0x010CC7, 0x010C87},\n{0x010CC8, 0x010C88},\n{0x010CC9, 0x010C89},\n{0x010CCA, 0x010C8A},\n{0x010CCB, 0x010C8B},\n{0x010CCC, 0x010C8C},\n{0x010CCD, 0x010C8D},\n{0x010CCE, 0x010C8E},\n{0x010CCF, 0x010C8F},\n{0x010CD0, 0x010C90},\n{0x010CD1, 0x010C91},\n{0x010CD2, 0x010C92},\n{0x010CD3, 0x010C93},\n{0x010CD4, 0x010C94},\n{0x010CD5, 0x010C95},\n{0x010CD6, 0x010C96},\n{0x010CD7, 0x010C97},\n{0x010CD8, 0x010C98},\n{0x010CD9, 0x010C99},\n{0x010CDA, 0x010C9A},\n{0x010CDB, 0x010C9B},\n{0x010CDC, 0x010C9C},\n{0x010CDD, 0x010C9D},\n{0x010CDE, 0x010C9E},\n{0x010CDF, 0x010C9F},\n{0x010CE0, 0x010CA0},\n{0x010CE1, 0x010CA1},\n{0x010CE2, 0x010CA2},\n{0x010CE3, 0x010CA3},\n{0x010CE4, 0x010CA4},\n{0x010CE5, 0x010CA5},\n{0x010CE6, 0x010CA6},\n{0x010CE7, 0x010CA7},\n{0x010CE8, 0x010CA8},\n{0x010CE9, 0x010CA9},\n{0x010CEA, 0x010CAA},\n{0x010CEB, 0x010CAB},\n{0x010CEC, 0x010CAC},\n{0x010CED, 0x010CAD},\n{0x010CEE, 0x010CAE},\n{0x010CEF, 0x010CAF},\n{0x010CF0, 0x010CB0},\n{0x010CF1, 0x010CB1},\n{0x010CF2, 0x010CB2},\n{0x0118C0, 0x0118A0},\n{0x0118C1, 0x0118A1},\n{0x0118C2, 0x0118A2},\n{0x0118C3, 0x0118A3},\n{0x0118C4, 0x0118A4},\n{0x0118C5, 0x0118A5},\n{0x0118C6, 0x0118A6},\n{0x0118C7, 0x0118A7},\n{0x0118C8, 0x0118A8},\n{0x0118C9, 0x0118A9},\n{0x0118CA, 0x0118AA},\n{0x0118CB, 0x0118AB},\n{0x0118CC, 0x0118AC},\n{0x0118CD, 0x0118AD},\n{0x0118CE, 0x0118AE},\n{0x0118CF, 0x0118AF},\n{0x0118D0, 0x0118B0},\n{0x0118D1, 0x0118B1},\n{0x0118D2, 0x0118B2},\n{0x0118D3, 0x0118B3},\n{0x0118D4, 0x0118B4},\n{0x0118D5, 0x0118B5},\n{0x0118D6, 0x0118B6},\n{0x0118D7, 0x0118B7},\n{0x0118D8, 0x0118B8},\n{0x0118D9, 0x0118B9},\n{0x0118DA, 0x0118BA},\n{0x0118DB, 0x0118BB},\n{0x0118DC, 0x0118BC},\n{0x0118DD, 0x0118BD},\n{0x0118DE, 0x0118BE},\n{0x0118DF, 0x0118BF},\n{0x016E60, 0x016E40},\n{0x016E61, 0x016E41},\n{0x016E62, 0x016E42},\n{0x016E63, 0x016E43},\n{0x016E64, 0x016E44},\n{0x016E65, 0x016E45},\n{0x016E66, 0x016E46},\n{0x016E67, 0x016E47},\n{0x016E68, 0x016E48},\n{0x016E69, 0x016E49},\n{0x016E6A, 0x016E4A},\n{0x016E6B, 0x016E4B},\n{0x016E6C, 0x016E4C},\n{0x016E6D, 0x016E4D},\n{0x016E6E, 0x016E4E},\n{0x016E6F, 0x016E4F},\n{0x016E70, 0x016E50},\n{0x016E71, 0x016E51},\n{0x016E72, 0x016E52},\n{0x016E73, 0x016E53},\n{0x016E74, 0x016E54},\n{0x016E75, 0x016E55},\n{0x016E76, 0x016E56},\n{0x016E77, 0x016E57},\n{0x016E78, 0x016E58},\n{0x016E79, 0x016E59},\n{0x016E7A, 0x016E5A},\n{0x016E7B, 0x016E5B},\n{0x016E7C, 0x016E5C},\n{0x016E7D, 0x016E5D},\n{0x016E7E, 0x016E5E},\n{0x016E7F, 0x016E5F},\n{0x01E922, 0x01E900},\n{0x01E923, 0x01E901},\n{0x01E924, 0x01E902},\n{0x01E925, 0x01E903},\n{0x01E926, 0x01E904},\n{0x01E927, 0x01E905},\n{0x01E928, 0x01E906},\n{0x01E929, 0x01E907},\n{0x01E92A, 0x01E908},\n{0x01E92B, 0x01E909},\n{0x01E92C, 0x01E90A},\n{0x01E92D, 0x01E90B},\n{0x01E92E, 0x01E90C},\n{0x01E92F, 0x01E90D},\n{0x01E930, 0x01E90E},\n{0x01E931, 0x01E90F},\n{0x01E932, 0x01E910},\n{0x01E933, 0x01E911},\n{0x01E934, 0x01E912},\n{0x01E935, 0x01E913},\n{0x01E936, 0x01E914},\n{0x01E937, 0x01E915},\n{0x01E938, 0x01E916},\n{0x01E939, 0x01E917},\n{0x01E93A, 0x01E918},\n{0x01E93B, 0x01E919},\n{0x01E93C, 0x01E91A},\n{0x01E93D, 0x01E91B},\n{0x01E93E, 0x01E91C},\n{0x01E93F, 0x01E91D},\n{0x01E940, 0x01E91E},\n{0x01E941, 0x01E91F},\n{0x01E942, 0x01E920},\n{0x01E943, 0x01E921},\n};\n\nconst std::initializer_list<range_nfd> unicode_ranges_nfd = {  // start, last, nfd\n{0x000000, 0x000000, 0x000000},\n{0x0000C0, 0x0000C5, 0x000041},\n{0x0000C7, 0x0000C7, 0x000043},\n{0x0000C8, 0x0000CB, 0x000045},\n{0x0000CC, 0x0000CF, 0x000049},\n{0x0000D1, 0x0000D1, 0x00004E},\n{0x0000D2, 0x0000D6, 0x00004F},\n{0x0000D9, 0x0000DC, 0x000055},\n{0x0000DD, 0x0000DD, 0x000059},\n{0x0000E0, 0x0000E5, 0x000061},\n{0x0000E7, 0x0000E7, 0x000063},\n{0x0000E8, 0x0000EB, 0x000065},\n{0x0000EC, 0x0000EF, 0x000069},\n{0x0000F1, 0x0000F1, 0x00006E},\n{0x0000F2, 0x0000F6, 0x00006F},\n{0x0000F9, 0x0000FC, 0x000075},\n{0x0000FD, 0x0000FD, 0x000079},\n{0x0000FF, 0x0000FF, 0x000079},\n{0x000100, 0x000100, 0x000041},\n{0x000101, 0x000101, 0x000061},\n{0x000102, 0x000102, 0x000041},\n{0x000103, 0x000103, 0x000061},\n{0x000104, 0x000104, 0x000041},\n{0x000105, 0x000105, 0x000061},\n{0x000106, 0x000106, 0x000043},\n{0x000107, 0x000107, 0x000063},\n{0x000108, 0x000108, 0x000043},\n{0x000109, 0x000109, 0x000063},\n{0x00010A, 0x00010A, 0x000043},\n{0x00010B, 0x00010B, 0x000063},\n{0x00010C, 0x00010C, 0x000043},\n{0x00010D, 0x00010D, 0x000063},\n{0x00010E, 0x00010E, 0x000044},\n{0x00010F, 0x00010F, 0x000064},\n{0x000112, 0x000112, 0x000045},\n{0x000113, 0x000113, 0x000065},\n{0x000114, 0x000114, 0x000045},\n{0x000115, 0x000115, 0x000065},\n{0x000116, 0x000116, 0x000045},\n{0x000117, 0x000117, 0x000065},\n{0x000118, 0x000118, 0x000045},\n{0x000119, 0x000119, 0x000065},\n{0x00011A, 0x00011A, 0x000045},\n{0x00011B, 0x00011B, 0x000065},\n{0x00011C, 0x00011C, 0x000047},\n{0x00011D, 0x00011D, 0x000067},\n{0x00011E, 0x00011E, 0x000047},\n{0x00011F, 0x00011F, 0x000067},\n{0x000120, 0x000120, 0x000047},\n{0x000121, 0x000121, 0x000067},\n{0x000122, 0x000122, 0x000047},\n{0x000123, 0x000123, 0x000067},\n{0x000124, 0x000124, 0x000048},\n{0x000125, 0x000125, 0x000068},\n{0x000128, 0x000128, 0x000049},\n{0x000129, 0x000129, 0x000069},\n{0x00012A, 0x00012A, 0x000049},\n{0x00012B, 0x00012B, 0x000069},\n{0x00012C, 0x00012C, 0x000049},\n{0x00012D, 0x00012D, 0x000069},\n{0x00012E, 0x00012E, 0x000049},\n{0x00012F, 0x00012F, 0x000069},\n{0x000130, 0x000130, 0x000049},\n{0x000134, 0x000134, 0x00004A},\n{0x000135, 0x000135, 0x00006A},\n{0x000136, 0x000136, 0x00004B},\n{0x000137, 0x000137, 0x00006B},\n{0x000139, 0x000139, 0x00004C},\n{0x00013A, 0x00013A, 0x00006C},\n{0x00013B, 0x00013B, 0x00004C},\n{0x00013C, 0x00013C, 0x00006C},\n{0x00013D, 0x00013D, 0x00004C},\n{0x00013E, 0x00013E, 0x00006C},\n{0x000143, 0x000143, 0x00004E},\n{0x000144, 0x000144, 0x00006E},\n{0x000145, 0x000145, 0x00004E},\n{0x000146, 0x000146, 0x00006E},\n{0x000147, 0x000147, 0x00004E},\n{0x000148, 0x000148, 0x00006E},\n{0x00014C, 0x00014C, 0x00004F},\n{0x00014D, 0x00014D, 0x00006F},\n{0x00014E, 0x00014E, 0x00004F},\n{0x00014F, 0x00014F, 0x00006F},\n{0x000150, 0x000150, 0x00004F},\n{0x000151, 0x000151, 0x00006F},\n{0x000154, 0x000154, 0x000052},\n{0x000155, 0x000155, 0x000072},\n{0x000156, 0x000156, 0x000052},\n{0x000157, 0x000157, 0x000072},\n{0x000158, 0x000158, 0x000052},\n{0x000159, 0x000159, 0x000072},\n{0x00015A, 0x00015A, 0x000053},\n{0x00015B, 0x00015B, 0x000073},\n{0x00015C, 0x00015C, 0x000053},\n{0x00015D, 0x00015D, 0x000073},\n{0x00015E, 0x00015E, 0x000053},\n{0x00015F, 0x00015F, 0x000073},\n{0x000160, 0x000160, 0x000053},\n{0x000161, 0x000161, 0x000073},\n{0x000162, 0x000162, 0x000054},\n{0x000163, 0x000163, 0x000074},\n{0x000164, 0x000164, 0x000054},\n{0x000165, 0x000165, 0x000074},\n{0x000168, 0x000168, 0x000055},\n{0x000169, 0x000169, 0x000075},\n{0x00016A, 0x00016A, 0x000055},\n{0x00016B, 0x00016B, 0x000075},\n{0x00016C, 0x00016C, 0x000055},\n{0x00016D, 0x00016D, 0x000075},\n{0x00016E, 0x00016E, 0x000055},\n{0x00016F, 0x00016F, 0x000075},\n{0x000170, 0x000170, 0x000055},\n{0x000171, 0x000171, 0x000075},\n{0x000172, 0x000172, 0x000055},\n{0x000173, 0x000173, 0x000075},\n{0x000174, 0x000174, 0x000057},\n{0x000175, 0x000175, 0x000077},\n{0x000176, 0x000176, 0x000059},\n{0x000177, 0x000177, 0x000079},\n{0x000178, 0x000178, 0x000059},\n{0x000179, 0x000179, 0x00005A},\n{0x00017A, 0x00017A, 0x00007A},\n{0x00017B, 0x00017B, 0x00005A},\n{0x00017C, 0x00017C, 0x00007A},\n{0x00017D, 0x00017D, 0x00005A},\n{0x00017E, 0x00017E, 0x00007A},\n{0x0001A0, 0x0001A0, 0x00004F},\n{0x0001A1, 0x0001A1, 0x00006F},\n{0x0001AF, 0x0001AF, 0x000055},\n{0x0001B0, 0x0001B0, 0x000075},\n{0x0001CD, 0x0001CD, 0x000041},\n{0x0001CE, 0x0001CE, 0x000061},\n{0x0001CF, 0x0001CF, 0x000049},\n{0x0001D0, 0x0001D0, 0x000069},\n{0x0001D1, 0x0001D1, 0x00004F},\n{0x0001D2, 0x0001D2, 0x00006F},\n{0x0001D3, 0x0001D3, 0x000055},\n{0x0001D4, 0x0001D4, 0x000075},\n{0x0001D5, 0x0001D5, 0x000055},\n{0x0001D6, 0x0001D6, 0x000075},\n{0x0001D7, 0x0001D7, 0x000055},\n{0x0001D8, 0x0001D8, 0x000075},\n{0x0001D9, 0x0001D9, 0x000055},\n{0x0001DA, 0x0001DA, 0x000075},\n{0x0001DB, 0x0001DB, 0x000055},\n{0x0001DC, 0x0001DC, 0x000075},\n{0x0001DE, 0x0001DE, 0x000041},\n{0x0001DF, 0x0001DF, 0x000061},\n{0x0001E0, 0x0001E0, 0x000041},\n{0x0001E1, 0x0001E1, 0x000061},\n{0x0001E2, 0x0001E2, 0x0000C6},\n{0x0001E3, 0x0001E3, 0x0000E6},\n{0x0001E6, 0x0001E6, 0x000047},\n{0x0001E7, 0x0001E7, 0x000067},\n{0x0001E8, 0x0001E8, 0x00004B},\n{0x0001E9, 0x0001E9, 0x00006B},\n{0x0001EA, 0x0001EA, 0x00004F},\n{0x0001EB, 0x0001EB, 0x00006F},\n{0x0001EC, 0x0001EC, 0x00004F},\n{0x0001ED, 0x0001ED, 0x00006F},\n{0x0001EE, 0x0001EE, 0x0001B7},\n{0x0001EF, 0x0001EF, 0x000292},\n{0x0001F0, 0x0001F0, 0x00006A},\n{0x0001F4, 0x0001F4, 0x000047},\n{0x0001F5, 0x0001F5, 0x000067},\n{0x0001F8, 0x0001F8, 0x00004E},\n{0x0001F9, 0x0001F9, 0x00006E},\n{0x0001FA, 0x0001FA, 0x000041},\n{0x0001FB, 0x0001FB, 0x000061},\n{0x0001FC, 0x0001FC, 0x0000C6},\n{0x0001FD, 0x0001FD, 0x0000E6},\n{0x0001FE, 0x0001FE, 0x0000D8},\n{0x0001FF, 0x0001FF, 0x0000F8},\n{0x000200, 0x000200, 0x000041},\n{0x000201, 0x000201, 0x000061},\n{0x000202, 0x000202, 0x000041},\n{0x000203, 0x000203, 0x000061},\n{0x000204, 0x000204, 0x000045},\n{0x000205, 0x000205, 0x000065},\n{0x000206, 0x000206, 0x000045},\n{0x000207, 0x000207, 0x000065},\n{0x000208, 0x000208, 0x000049},\n{0x000209, 0x000209, 0x000069},\n{0x00020A, 0x00020A, 0x000049},\n{0x00020B, 0x00020B, 0x000069},\n{0x00020C, 0x00020C, 0x00004F},\n{0x00020D, 0x00020D, 0x00006F},\n{0x00020E, 0x00020E, 0x00004F},\n{0x00020F, 0x00020F, 0x00006F},\n{0x000210, 0x000210, 0x000052},\n{0x000211, 0x000211, 0x000072},\n{0x000212, 0x000212, 0x000052},\n{0x000213, 0x000213, 0x000072},\n{0x000214, 0x000214, 0x000055},\n{0x000215, 0x000215, 0x000075},\n{0x000216, 0x000216, 0x000055},\n{0x000217, 0x000217, 0x000075},\n{0x000218, 0x000218, 0x000053},\n{0x000219, 0x000219, 0x000073},\n{0x00021A, 0x00021A, 0x000054},\n{0x00021B, 0x00021B, 0x000074},\n{0x00021E, 0x00021E, 0x000048},\n{0x00021F, 0x00021F, 0x000068},\n{0x000226, 0x000226, 0x000041},\n{0x000227, 0x000227, 0x000061},\n{0x000228, 0x000228, 0x000045},\n{0x000229, 0x000229, 0x000065},\n{0x00022A, 0x00022A, 0x00004F},\n{0x00022B, 0x00022B, 0x00006F},\n{0x00022C, 0x00022C, 0x00004F},\n{0x00022D, 0x00022D, 0x00006F},\n{0x00022E, 0x00022E, 0x00004F},\n{0x00022F, 0x00022F, 0x00006F},\n{0x000230, 0x000230, 0x00004F},\n{0x000231, 0x000231, 0x00006F},\n{0x000232, 0x000232, 0x000059},\n{0x000233, 0x000233, 0x000079},\n{0x000340, 0x000340, 0x000300},\n{0x000341, 0x000341, 0x000301},\n{0x000343, 0x000343, 0x000313},\n{0x000344, 0x000344, 0x000308},\n{0x000374, 0x000374, 0x0002B9},\n{0x00037E, 0x00037E, 0x00003B},\n{0x000385, 0x000385, 0x0000A8},\n{0x000386, 0x000386, 0x000391},\n{0x000387, 0x000387, 0x0000B7},\n{0x000388, 0x000388, 0x000395},\n{0x000389, 0x000389, 0x000397},\n{0x00038A, 0x00038A, 0x000399},\n{0x00038C, 0x00038C, 0x00039F},\n{0x00038E, 0x00038E, 0x0003A5},\n{0x00038F, 0x00038F, 0x0003A9},\n{0x000390, 0x000390, 0x0003B9},\n{0x0003AA, 0x0003AA, 0x000399},\n{0x0003AB, 0x0003AB, 0x0003A5},\n{0x0003AC, 0x0003AC, 0x0003B1},\n{0x0003AD, 0x0003AD, 0x0003B5},\n{0x0003AE, 0x0003AE, 0x0003B7},\n{0x0003AF, 0x0003AF, 0x0003B9},\n{0x0003B0, 0x0003B0, 0x0003C5},\n{0x0003CA, 0x0003CA, 0x0003B9},\n{0x0003CB, 0x0003CB, 0x0003C5},\n{0x0003CC, 0x0003CC, 0x0003BF},\n{0x0003CD, 0x0003CD, 0x0003C5},\n{0x0003CE, 0x0003CE, 0x0003C9},\n{0x0003D3, 0x0003D4, 0x0003D2},\n{0x000400, 0x000401, 0x000415},\n{0x000403, 0x000403, 0x000413},\n{0x000407, 0x000407, 0x000406},\n{0x00040C, 0x00040C, 0x00041A},\n{0x00040D, 0x00040D, 0x000418},\n{0x00040E, 0x00040E, 0x000423},\n{0x000419, 0x000419, 0x000418},\n{0x000439, 0x000439, 0x000438},\n{0x000450, 0x000451, 0x000435},\n{0x000453, 0x000453, 0x000433},\n{0x000457, 0x000457, 0x000456},\n{0x00045C, 0x00045C, 0x00043A},\n{0x00045D, 0x00045D, 0x000438},\n{0x00045E, 0x00045E, 0x000443},\n{0x000476, 0x000476, 0x000474},\n{0x000477, 0x000477, 0x000475},\n{0x0004C1, 0x0004C1, 0x000416},\n{0x0004C2, 0x0004C2, 0x000436},\n{0x0004D0, 0x0004D0, 0x000410},\n{0x0004D1, 0x0004D1, 0x000430},\n{0x0004D2, 0x0004D2, 0x000410},\n{0x0004D3, 0x0004D3, 0x000430},\n{0x0004D6, 0x0004D6, 0x000415},\n{0x0004D7, 0x0004D7, 0x000435},\n{0x0004DA, 0x0004DA, 0x0004D8},\n{0x0004DB, 0x0004DB, 0x0004D9},\n{0x0004DC, 0x0004DC, 0x000416},\n{0x0004DD, 0x0004DD, 0x000436},\n{0x0004DE, 0x0004DE, 0x000417},\n{0x0004DF, 0x0004DF, 0x000437},\n{0x0004E2, 0x0004E2, 0x000418},\n{0x0004E3, 0x0004E3, 0x000438},\n{0x0004E4, 0x0004E4, 0x000418},\n{0x0004E5, 0x0004E5, 0x000438},\n{0x0004E6, 0x0004E6, 0x00041E},\n{0x0004E7, 0x0004E7, 0x00043E},\n{0x0004EA, 0x0004EA, 0x0004E8},\n{0x0004EB, 0x0004EB, 0x0004E9},\n{0x0004EC, 0x0004EC, 0x00042D},\n{0x0004ED, 0x0004ED, 0x00044D},\n{0x0004EE, 0x0004EE, 0x000423},\n{0x0004EF, 0x0004EF, 0x000443},\n{0x0004F0, 0x0004F0, 0x000423},\n{0x0004F1, 0x0004F1, 0x000443},\n{0x0004F2, 0x0004F2, 0x000423},\n{0x0004F3, 0x0004F3, 0x000443},\n{0x0004F4, 0x0004F4, 0x000427},\n{0x0004F5, 0x0004F5, 0x000447},\n{0x0004F8, 0x0004F8, 0x00042B},\n{0x0004F9, 0x0004F9, 0x00044B},\n{0x000622, 0x000623, 0x000627},\n{0x000624, 0x000624, 0x000648},\n{0x000625, 0x000625, 0x000627},\n{0x000626, 0x000626, 0x00064A},\n{0x0006C0, 0x0006C0, 0x0006D5},\n{0x0006C2, 0x0006C2, 0x0006C1},\n{0x0006D3, 0x0006D3, 0x0006D2},\n{0x000929, 0x000929, 0x000928},\n{0x000931, 0x000931, 0x000930},\n{0x000934, 0x000934, 0x000933},\n{0x000958, 0x000958, 0x000915},\n{0x000959, 0x000959, 0x000916},\n{0x00095A, 0x00095A, 0x000917},\n{0x00095B, 0x00095B, 0x00091C},\n{0x00095C, 0x00095C, 0x000921},\n{0x00095D, 0x00095D, 0x000922},\n{0x00095E, 0x00095E, 0x00092B},\n{0x00095F, 0x00095F, 0x00092F},\n{0x0009CB, 0x0009CC, 0x0009C7},\n{0x0009DC, 0x0009DC, 0x0009A1},\n{0x0009DD, 0x0009DD, 0x0009A2},\n{0x0009DF, 0x0009DF, 0x0009AF},\n{0x000A33, 0x000A33, 0x000A32},\n{0x000A36, 0x000A36, 0x000A38},\n{0x000A59, 0x000A59, 0x000A16},\n{0x000A5A, 0x000A5A, 0x000A17},\n{0x000A5B, 0x000A5B, 0x000A1C},\n{0x000A5E, 0x000A5E, 0x000A2B},\n{0x000B48, 0x000B48, 0x000B47},\n{0x000B4B, 0x000B4C, 0x000B47},\n{0x000B5C, 0x000B5C, 0x000B21},\n{0x000B5D, 0x000B5D, 0x000B22},\n{0x000B94, 0x000B94, 0x000B92},\n{0x000BCA, 0x000BCA, 0x000BC6},\n{0x000BCB, 0x000BCB, 0x000BC7},\n{0x000BCC, 0x000BCC, 0x000BC6},\n{0x000C48, 0x000C48, 0x000C46},\n{0x000CC0, 0x000CC0, 0x000CBF},\n{0x000CC7, 0x000CC8, 0x000CC6},\n{0x000CCA, 0x000CCB, 0x000CC6},\n{0x000D4A, 0x000D4A, 0x000D46},\n{0x000D4B, 0x000D4B, 0x000D47},\n{0x000D4C, 0x000D4C, 0x000D46},\n{0x000DDA, 0x000DDA, 0x000DD9},\n{0x000DDC, 0x000DDE, 0x000DD9},\n{0x000F43, 0x000F43, 0x000F42},\n{0x000F4D, 0x000F4D, 0x000F4C},\n{0x000F52, 0x000F52, 0x000F51},\n{0x000F57, 0x000F57, 0x000F56},\n{0x000F5C, 0x000F5C, 0x000F5B},\n{0x000F69, 0x000F69, 0x000F40},\n{0x000F73, 0x000F73, 0x000F71},\n{0x000F75, 0x000F75, 0x000F71},\n{0x000F76, 0x000F76, 0x000FB2},\n{0x000F78, 0x000F78, 0x000FB3},\n{0x000F81, 0x000F81, 0x000F71},\n{0x000F93, 0x000F93, 0x000F92},\n{0x000F9D, 0x000F9D, 0x000F9C},\n{0x000FA2, 0x000FA2, 0x000FA1},\n{0x000FA7, 0x000FA7, 0x000FA6},\n{0x000FAC, 0x000FAC, 0x000FAB},\n{0x000FB9, 0x000FB9, 0x000F90},\n{0x001026, 0x001026, 0x001025},\n{0x001B06, 0x001B06, 0x001B05},\n{0x001B08, 0x001B08, 0x001B07},\n{0x001B0A, 0x001B0A, 0x001B09},\n{0x001B0C, 0x001B0C, 0x001B0B},\n{0x001B0E, 0x001B0E, 0x001B0D},\n{0x001B12, 0x001B12, 0x001B11},\n{0x001B3B, 0x001B3B, 0x001B3A},\n{0x001B3D, 0x001B3D, 0x001B3C},\n{0x001B40, 0x001B40, 0x001B3E},\n{0x001B41, 0x001B41, 0x001B3F},\n{0x001B43, 0x001B43, 0x001B42},\n{0x001E00, 0x001E00, 0x000041},\n{0x001E01, 0x001E01, 0x000061},\n{0x001E02, 0x001E02, 0x000042},\n{0x001E03, 0x001E03, 0x000062},\n{0x001E04, 0x001E04, 0x000042},\n{0x001E05, 0x001E05, 0x000062},\n{0x001E06, 0x001E06, 0x000042},\n{0x001E07, 0x001E07, 0x000062},\n{0x001E08, 0x001E08, 0x000043},\n{0x001E09, 0x001E09, 0x000063},\n{0x001E0A, 0x001E0A, 0x000044},\n{0x001E0B, 0x001E0B, 0x000064},\n{0x001E0C, 0x001E0C, 0x000044},\n{0x001E0D, 0x001E0D, 0x000064},\n{0x001E0E, 0x001E0E, 0x000044},\n{0x001E0F, 0x001E0F, 0x000064},\n{0x001E10, 0x001E10, 0x000044},\n{0x001E11, 0x001E11, 0x000064},\n{0x001E12, 0x001E12, 0x000044},\n{0x001E13, 0x001E13, 0x000064},\n{0x001E14, 0x001E14, 0x000045},\n{0x001E15, 0x001E15, 0x000065},\n{0x001E16, 0x001E16, 0x000045},\n{0x001E17, 0x001E17, 0x000065},\n{0x001E18, 0x001E18, 0x000045},\n{0x001E19, 0x001E19, 0x000065},\n{0x001E1A, 0x001E1A, 0x000045},\n{0x001E1B, 0x001E1B, 0x000065},\n{0x001E1C, 0x001E1C, 0x000045},\n{0x001E1D, 0x001E1D, 0x000065},\n{0x001E1E, 0x001E1E, 0x000046},\n{0x001E1F, 0x001E1F, 0x000066},\n{0x001E20, 0x001E20, 0x000047},\n{0x001E21, 0x001E21, 0x000067},\n{0x001E22, 0x001E22, 0x000048},\n{0x001E23, 0x001E23, 0x000068},\n{0x001E24, 0x001E24, 0x000048},\n{0x001E25, 0x001E25, 0x000068},\n{0x001E26, 0x001E26, 0x000048},\n{0x001E27, 0x001E27, 0x000068},\n{0x001E28, 0x001E28, 0x000048},\n{0x001E29, 0x001E29, 0x000068},\n{0x001E2A, 0x001E2A, 0x000048},\n{0x001E2B, 0x001E2B, 0x000068},\n{0x001E2C, 0x001E2C, 0x000049},\n{0x001E2D, 0x001E2D, 0x000069},\n{0x001E2E, 0x001E2E, 0x000049},\n{0x001E2F, 0x001E2F, 0x000069},\n{0x001E30, 0x001E30, 0x00004B},\n{0x001E31, 0x001E31, 0x00006B},\n{0x001E32, 0x001E32, 0x00004B},\n{0x001E33, 0x001E33, 0x00006B},\n{0x001E34, 0x001E34, 0x00004B},\n{0x001E35, 0x001E35, 0x00006B},\n{0x001E36, 0x001E36, 0x00004C},\n{0x001E37, 0x001E37, 0x00006C},\n{0x001E38, 0x001E38, 0x00004C},\n{0x001E39, 0x001E39, 0x00006C},\n{0x001E3A, 0x001E3A, 0x00004C},\n{0x001E3B, 0x001E3B, 0x00006C},\n{0x001E3C, 0x001E3C, 0x00004C},\n{0x001E3D, 0x001E3D, 0x00006C},\n{0x001E3E, 0x001E3E, 0x00004D},\n{0x001E3F, 0x001E3F, 0x00006D},\n{0x001E40, 0x001E40, 0x00004D},\n{0x001E41, 0x001E41, 0x00006D},\n{0x001E42, 0x001E42, 0x00004D},\n{0x001E43, 0x001E43, 0x00006D},\n{0x001E44, 0x001E44, 0x00004E},\n{0x001E45, 0x001E45, 0x00006E},\n{0x001E46, 0x001E46, 0x00004E},\n{0x001E47, 0x001E47, 0x00006E},\n{0x001E48, 0x001E48, 0x00004E},\n{0x001E49, 0x001E49, 0x00006E},\n{0x001E4A, 0x001E4A, 0x00004E},\n{0x001E4B, 0x001E4B, 0x00006E},\n{0x001E4C, 0x001E4C, 0x00004F},\n{0x001E4D, 0x001E4D, 0x00006F},\n{0x001E4E, 0x001E4E, 0x00004F},\n{0x001E4F, 0x001E4F, 0x00006F},\n{0x001E50, 0x001E50, 0x00004F},\n{0x001E51, 0x001E51, 0x00006F},\n{0x001E52, 0x001E52, 0x00004F},\n{0x001E53, 0x001E53, 0x00006F},\n{0x001E54, 0x001E54, 0x000050},\n{0x001E55, 0x001E55, 0x000070},\n{0x001E56, 0x001E56, 0x000050},\n{0x001E57, 0x001E57, 0x000070},\n{0x001E58, 0x001E58, 0x000052},\n{0x001E59, 0x001E59, 0x000072},\n{0x001E5A, 0x001E5A, 0x000052},\n{0x001E5B, 0x001E5B, 0x000072},\n{0x001E5C, 0x001E5C, 0x000052},\n{0x001E5D, 0x001E5D, 0x000072},\n{0x001E5E, 0x001E5E, 0x000052},\n{0x001E5F, 0x001E5F, 0x000072},\n{0x001E60, 0x001E60, 0x000053},\n{0x001E61, 0x001E61, 0x000073},\n{0x001E62, 0x001E62, 0x000053},\n{0x001E63, 0x001E63, 0x000073},\n{0x001E64, 0x001E64, 0x000053},\n{0x001E65, 0x001E65, 0x000073},\n{0x001E66, 0x001E66, 0x000053},\n{0x001E67, 0x001E67, 0x000073},\n{0x001E68, 0x001E68, 0x000053},\n{0x001E69, 0x001E69, 0x000073},\n{0x001E6A, 0x001E6A, 0x000054},\n{0x001E6B, 0x001E6B, 0x000074},\n{0x001E6C, 0x001E6C, 0x000054},\n{0x001E6D, 0x001E6D, 0x000074},\n{0x001E6E, 0x001E6E, 0x000054},\n{0x001E6F, 0x001E6F, 0x000074},\n{0x001E70, 0x001E70, 0x000054},\n{0x001E71, 0x001E71, 0x000074},\n{0x001E72, 0x001E72, 0x000055},\n{0x001E73, 0x001E73, 0x000075},\n{0x001E74, 0x001E74, 0x000055},\n{0x001E75, 0x001E75, 0x000075},\n{0x001E76, 0x001E76, 0x000055},\n{0x001E77, 0x001E77, 0x000075},\n{0x001E78, 0x001E78, 0x000055},\n{0x001E79, 0x001E79, 0x000075},\n{0x001E7A, 0x001E7A, 0x000055},\n{0x001E7B, 0x001E7B, 0x000075},\n{0x001E7C, 0x001E7C, 0x000056},\n{0x001E7D, 0x001E7D, 0x000076},\n{0x001E7E, 0x001E7E, 0x000056},\n{0x001E7F, 0x001E7F, 0x000076},\n{0x001E80, 0x001E80, 0x000057},\n{0x001E81, 0x001E81, 0x000077},\n{0x001E82, 0x001E82, 0x000057},\n{0x001E83, 0x001E83, 0x000077},\n{0x001E84, 0x001E84, 0x000057},\n{0x001E85, 0x001E85, 0x000077},\n{0x001E86, 0x001E86, 0x000057},\n{0x001E87, 0x001E87, 0x000077},\n{0x001E88, 0x001E88, 0x000057},\n{0x001E89, 0x001E89, 0x000077},\n{0x001E8A, 0x001E8A, 0x000058},\n{0x001E8B, 0x001E8B, 0x000078},\n{0x001E8C, 0x001E8C, 0x000058},\n{0x001E8D, 0x001E8D, 0x000078},\n{0x001E8E, 0x001E8E, 0x000059},\n{0x001E8F, 0x001E8F, 0x000079},\n{0x001E90, 0x001E90, 0x00005A},\n{0x001E91, 0x001E91, 0x00007A},\n{0x001E92, 0x001E92, 0x00005A},\n{0x001E93, 0x001E93, 0x00007A},\n{0x001E94, 0x001E94, 0x00005A},\n{0x001E95, 0x001E95, 0x00007A},\n{0x001E96, 0x001E96, 0x000068},\n{0x001E97, 0x001E97, 0x000074},\n{0x001E98, 0x001E98, 0x000077},\n{0x001E99, 0x001E99, 0x000079},\n{0x001E9B, 0x001E9B, 0x00017F},\n{0x001EA0, 0x001EA0, 0x000041},\n{0x001EA1, 0x001EA1, 0x000061},\n{0x001EA2, 0x001EA2, 0x000041},\n{0x001EA3, 0x001EA3, 0x000061},\n{0x001EA4, 0x001EA4, 0x000041},\n{0x001EA5, 0x001EA5, 0x000061},\n{0x001EA6, 0x001EA6, 0x000041},\n{0x001EA7, 0x001EA7, 0x000061},\n{0x001EA8, 0x001EA8, 0x000041},\n{0x001EA9, 0x001EA9, 0x000061},\n{0x001EAA, 0x001EAA, 0x000041},\n{0x001EAB, 0x001EAB, 0x000061},\n{0x001EAC, 0x001EAC, 0x000041},\n{0x001EAD, 0x001EAD, 0x000061},\n{0x001EAE, 0x001EAE, 0x000041},\n{0x001EAF, 0x001EAF, 0x000061},\n{0x001EB0, 0x001EB0, 0x000041},\n{0x001EB1, 0x001EB1, 0x000061},\n{0x001EB2, 0x001EB2, 0x000041},\n{0x001EB3, 0x001EB3, 0x000061},\n{0x001EB4, 0x001EB4, 0x000041},\n{0x001EB5, 0x001EB5, 0x000061},\n{0x001EB6, 0x001EB6, 0x000041},\n{0x001EB7, 0x001EB7, 0x000061},\n{0x001EB8, 0x001EB8, 0x000045},\n{0x001EB9, 0x001EB9, 0x000065},\n{0x001EBA, 0x001EBA, 0x000045},\n{0x001EBB, 0x001EBB, 0x000065},\n{0x001EBC, 0x001EBC, 0x000045},\n{0x001EBD, 0x001EBD, 0x000065},\n{0x001EBE, 0x001EBE, 0x000045},\n{0x001EBF, 0x001EBF, 0x000065},\n{0x001EC0, 0x001EC0, 0x000045},\n{0x001EC1, 0x001EC1, 0x000065},\n{0x001EC2, 0x001EC2, 0x000045},\n{0x001EC3, 0x001EC3, 0x000065},\n{0x001EC4, 0x001EC4, 0x000045},\n{0x001EC5, 0x001EC5, 0x000065},\n{0x001EC6, 0x001EC6, 0x000045},\n{0x001EC7, 0x001EC7, 0x000065},\n{0x001EC8, 0x001EC8, 0x000049},\n{0x001EC9, 0x001EC9, 0x000069},\n{0x001ECA, 0x001ECA, 0x000049},\n{0x001ECB, 0x001ECB, 0x000069},\n{0x001ECC, 0x001ECC, 0x00004F},\n{0x001ECD, 0x001ECD, 0x00006F},\n{0x001ECE, 0x001ECE, 0x00004F},\n{0x001ECF, 0x001ECF, 0x00006F},\n{0x001ED0, 0x001ED0, 0x00004F},\n{0x001ED1, 0x001ED1, 0x00006F},\n{0x001ED2, 0x001ED2, 0x00004F},\n{0x001ED3, 0x001ED3, 0x00006F},\n{0x001ED4, 0x001ED4, 0x00004F},\n{0x001ED5, 0x001ED5, 0x00006F},\n{0x001ED6, 0x001ED6, 0x00004F},\n{0x001ED7, 0x001ED7, 0x00006F},\n{0x001ED8, 0x001ED8, 0x00004F},\n{0x001ED9, 0x001ED9, 0x00006F},\n{0x001EDA, 0x001EDA, 0x00004F},\n{0x001EDB, 0x001EDB, 0x00006F},\n{0x001EDC, 0x001EDC, 0x00004F},\n{0x001EDD, 0x001EDD, 0x00006F},\n{0x001EDE, 0x001EDE, 0x00004F},\n{0x001EDF, 0x001EDF, 0x00006F},\n{0x001EE0, 0x001EE0, 0x00004F},\n{0x001EE1, 0x001EE1, 0x00006F},\n{0x001EE2, 0x001EE2, 0x00004F},\n{0x001EE3, 0x001EE3, 0x00006F},\n{0x001EE4, 0x001EE4, 0x000055},\n{0x001EE5, 0x001EE5, 0x000075},\n{0x001EE6, 0x001EE6, 0x000055},\n{0x001EE7, 0x001EE7, 0x000075},\n{0x001EE8, 0x001EE8, 0x000055},\n{0x001EE9, 0x001EE9, 0x000075},\n{0x001EEA, 0x001EEA, 0x000055},\n{0x001EEB, 0x001EEB, 0x000075},\n{0x001EEC, 0x001EEC, 0x000055},\n{0x001EED, 0x001EED, 0x000075},\n{0x001EEE, 0x001EEE, 0x000055},\n{0x001EEF, 0x001EEF, 0x000075},\n{0x001EF0, 0x001EF0, 0x000055},\n{0x001EF1, 0x001EF1, 0x000075},\n{0x001EF2, 0x001EF2, 0x000059},\n{0x001EF3, 0x001EF3, 0x000079},\n{0x001EF4, 0x001EF4, 0x000059},\n{0x001EF5, 0x001EF5, 0x000079},\n{0x001EF6, 0x001EF6, 0x000059},\n{0x001EF7, 0x001EF7, 0x000079},\n{0x001EF8, 0x001EF8, 0x000059},\n{0x001EF9, 0x001EF9, 0x000079},\n{0x001F00, 0x001F07, 0x0003B1},\n{0x001F08, 0x001F0F, 0x000391},\n{0x001F10, 0x001F15, 0x0003B5},\n{0x001F18, 0x001F1D, 0x000395},\n{0x001F20, 0x001F27, 0x0003B7},\n{0x001F28, 0x001F2F, 0x000397},\n{0x001F30, 0x001F37, 0x0003B9},\n{0x001F38, 0x001F3F, 0x000399},\n{0x001F40, 0x001F45, 0x0003BF},\n{0x001F48, 0x001F4D, 0x00039F},\n{0x001F50, 0x001F57, 0x0003C5},\n{0x001F59, 0x001F59, 0x0003A5},\n{0x001F5B, 0x001F5B, 0x0003A5},\n{0x001F5D, 0x001F5D, 0x0003A5},\n{0x001F5F, 0x001F5F, 0x0003A5},\n{0x001F60, 0x001F67, 0x0003C9},\n{0x001F68, 0x001F6F, 0x0003A9},\n{0x001F70, 0x001F71, 0x0003B1},\n{0x001F72, 0x001F73, 0x0003B5},\n{0x001F74, 0x001F75, 0x0003B7},\n{0x001F76, 0x001F77, 0x0003B9},\n{0x001F78, 0x001F79, 0x0003BF},\n{0x001F7A, 0x001F7B, 0x0003C5},\n{0x001F7C, 0x001F7D, 0x0003C9},\n{0x001F80, 0x001F87, 0x0003B1},\n{0x001F88, 0x001F8F, 0x000391},\n{0x001F90, 0x001F97, 0x0003B7},\n{0x001F98, 0x001F9F, 0x000397},\n{0x001FA0, 0x001FA7, 0x0003C9},\n{0x001FA8, 0x001FAF, 0x0003A9},\n{0x001FB0, 0x001FB4, 0x0003B1},\n{0x001FB6, 0x001FB7, 0x0003B1},\n{0x001FB8, 0x001FBC, 0x000391},\n{0x001FBE, 0x001FBE, 0x0003B9},\n{0x001FC1, 0x001FC1, 0x0000A8},\n{0x001FC2, 0x001FC4, 0x0003B7},\n{0x001FC6, 0x001FC7, 0x0003B7},\n{0x001FC8, 0x001FC9, 0x000395},\n{0x001FCA, 0x001FCC, 0x000397},\n{0x001FCD, 0x001FCF, 0x001FBF},\n{0x001FD0, 0x001FD3, 0x0003B9},\n{0x001FD6, 0x001FD7, 0x0003B9},\n{0x001FD8, 0x001FDB, 0x000399},\n{0x001FDD, 0x001FDF, 0x001FFE},\n{0x001FE0, 0x001FE3, 0x0003C5},\n{0x001FE4, 0x001FE5, 0x0003C1},\n{0x001FE6, 0x001FE7, 0x0003C5},\n{0x001FE8, 0x001FEB, 0x0003A5},\n{0x001FEC, 0x001FEC, 0x0003A1},\n{0x001FED, 0x001FEE, 0x0000A8},\n{0x001FEF, 0x001FEF, 0x000060},\n{0x001FF2, 0x001FF4, 0x0003C9},\n{0x001FF6, 0x001FF7, 0x0003C9},\n{0x001FF8, 0x001FF9, 0x00039F},\n{0x001FFA, 0x001FFC, 0x0003A9},\n{0x001FFD, 0x001FFD, 0x0000B4},\n{0x002000, 0x002000, 0x002002},\n{0x002001, 0x002001, 0x002003},\n{0x002126, 0x002126, 0x0003A9},\n{0x00212A, 0x00212A, 0x00004B},\n{0x00212B, 0x00212B, 0x000041},\n{0x00219A, 0x00219A, 0x002190},\n{0x00219B, 0x00219B, 0x002192},\n{0x0021AE, 0x0021AE, 0x002194},\n{0x0021CD, 0x0021CD, 0x0021D0},\n{0x0021CE, 0x0021CE, 0x0021D4},\n{0x0021CF, 0x0021CF, 0x0021D2},\n{0x002204, 0x002204, 0x002203},\n{0x002209, 0x002209, 0x002208},\n{0x00220C, 0x00220C, 0x00220B},\n{0x002224, 0x002224, 0x002223},\n{0x002226, 0x002226, 0x002225},\n{0x002241, 0x002241, 0x00223C},\n{0x002244, 0x002244, 0x002243},\n{0x002247, 0x002247, 0x002245},\n{0x002249, 0x002249, 0x002248},\n{0x002260, 0x002260, 0x00003D},\n{0x002262, 0x002262, 0x002261},\n{0x00226D, 0x00226D, 0x00224D},\n{0x00226E, 0x00226E, 0x00003C},\n{0x00226F, 0x00226F, 0x00003E},\n{0x002270, 0x002270, 0x002264},\n{0x002271, 0x002271, 0x002265},\n{0x002274, 0x002274, 0x002272},\n{0x002275, 0x002275, 0x002273},\n{0x002278, 0x002278, 0x002276},\n{0x002279, 0x002279, 0x002277},\n{0x002280, 0x002280, 0x00227A},\n{0x002281, 0x002281, 0x00227B},\n{0x002284, 0x002284, 0x002282},\n{0x002285, 0x002285, 0x002283},\n{0x002288, 0x002288, 0x002286},\n{0x002289, 0x002289, 0x002287},\n{0x0022AC, 0x0022AC, 0x0022A2},\n{0x0022AD, 0x0022AD, 0x0022A8},\n{0x0022AE, 0x0022AE, 0x0022A9},\n{0x0022AF, 0x0022AF, 0x0022AB},\n{0x0022E0, 0x0022E0, 0x00227C},\n{0x0022E1, 0x0022E1, 0x00227D},\n{0x0022E2, 0x0022E2, 0x002291},\n{0x0022E3, 0x0022E3, 0x002292},\n{0x0022EA, 0x0022EA, 0x0022B2},\n{0x0022EB, 0x0022EB, 0x0022B3},\n{0x0022EC, 0x0022EC, 0x0022B4},\n{0x0022ED, 0x0022ED, 0x0022B5},\n{0x002329, 0x002329, 0x003008},\n{0x00232A, 0x00232A, 0x003009},\n{0x002ADC, 0x002ADC, 0x002ADD},\n{0x00304C, 0x00304C, 0x00304B},\n{0x00304E, 0x00304E, 0x00304D},\n{0x003050, 0x003050, 0x00304F},\n{0x003052, 0x003052, 0x003051},\n{0x003054, 0x003054, 0x003053},\n{0x003056, 0x003056, 0x003055},\n{0x003058, 0x003058, 0x003057},\n{0x00305A, 0x00305A, 0x003059},\n{0x00305C, 0x00305C, 0x00305B},\n{0x00305E, 0x00305E, 0x00305D},\n{0x003060, 0x003060, 0x00305F},\n{0x003062, 0x003062, 0x003061},\n{0x003065, 0x003065, 0x003064},\n{0x003067, 0x003067, 0x003066},\n{0x003069, 0x003069, 0x003068},\n{0x003070, 0x003071, 0x00306F},\n{0x003073, 0x003074, 0x003072},\n{0x003076, 0x003077, 0x003075},\n{0x003079, 0x00307A, 0x003078},\n{0x00307C, 0x00307D, 0x00307B},\n{0x003094, 0x003094, 0x003046},\n{0x00309E, 0x00309E, 0x00309D},\n{0x0030AC, 0x0030AC, 0x0030AB},\n{0x0030AE, 0x0030AE, 0x0030AD},\n{0x0030B0, 0x0030B0, 0x0030AF},\n{0x0030B2, 0x0030B2, 0x0030B1},\n{0x0030B4, 0x0030B4, 0x0030B3},\n{0x0030B6, 0x0030B6, 0x0030B5},\n{0x0030B8, 0x0030B8, 0x0030B7},\n{0x0030BA, 0x0030BA, 0x0030B9},\n{0x0030BC, 0x0030BC, 0x0030BB},\n{0x0030BE, 0x0030BE, 0x0030BD},\n{0x0030C0, 0x0030C0, 0x0030BF},\n{0x0030C2, 0x0030C2, 0x0030C1},\n{0x0030C5, 0x0030C5, 0x0030C4},\n{0x0030C7, 0x0030C7, 0x0030C6},\n{0x0030C9, 0x0030C9, 0x0030C8},\n{0x0030D0, 0x0030D1, 0x0030CF},\n{0x0030D3, 0x0030D4, 0x0030D2},\n{0x0030D6, 0x0030D7, 0x0030D5},\n{0x0030D9, 0x0030DA, 0x0030D8},\n{0x0030DC, 0x0030DD, 0x0030DB},\n{0x0030F4, 0x0030F4, 0x0030A6},\n{0x0030F7, 0x0030F7, 0x0030EF},\n{0x0030F8, 0x0030F8, 0x0030F0},\n{0x0030F9, 0x0030F9, 0x0030F1},\n{0x0030FA, 0x0030FA, 0x0030F2},\n{0x0030FE, 0x0030FE, 0x0030FD},\n{0x00AC00, 0x00AE4B, 0x001100},\n{0x00AE4C, 0x00B097, 0x001101},\n{0x00B098, 0x00B2E3, 0x001102},\n{0x00B2E4, 0x00B52F, 0x001103},\n{0x00B530, 0x00B77B, 0x001104},\n{0x00B77C, 0x00B9C7, 0x001105},\n{0x00B9C8, 0x00BC13, 0x001106},\n{0x00BC14, 0x00BE5F, 0x001107},\n{0x00BE60, 0x00C0AB, 0x001108},\n{0x00C0AC, 0x00C2F7, 0x001109},\n{0x00C2F8, 0x00C543, 0x00110A},\n{0x00C544, 0x00C78F, 0x00110B},\n{0x00C790, 0x00C9DB, 0x00110C},\n{0x00C9DC, 0x00CC27, 0x00110D},\n{0x00CC28, 0x00CE73, 0x00110E},\n{0x00CE74, 0x00D0BF, 0x00110F},\n{0x00D0C0, 0x00D30B, 0x001110},\n{0x00D30C, 0x00D557, 0x001111},\n{0x00D558, 0x00D7A3, 0x001112},\n{0x00F900, 0x00F900, 0x008C48},\n{0x00F901, 0x00F901, 0x0066F4},\n{0x00F902, 0x00F902, 0x008ECA},\n{0x00F903, 0x00F903, 0x008CC8},\n{0x00F904, 0x00F904, 0x006ED1},\n{0x00F905, 0x00F905, 0x004E32},\n{0x00F906, 0x00F906, 0x0053E5},\n{0x00F907, 0x00F908, 0x009F9C},\n{0x00F909, 0x00F909, 0x005951},\n{0x00F90A, 0x00F90A, 0x0091D1},\n{0x00F90B, 0x00F90B, 0x005587},\n{0x00F90C, 0x00F90C, 0x005948},\n{0x00F90D, 0x00F90D, 0x0061F6},\n{0x00F90E, 0x00F90E, 0x007669},\n{0x00F90F, 0x00F90F, 0x007F85},\n{0x00F910, 0x00F910, 0x00863F},\n{0x00F911, 0x00F911, 0x0087BA},\n{0x00F912, 0x00F912, 0x0088F8},\n{0x00F913, 0x00F913, 0x00908F},\n{0x00F914, 0x00F914, 0x006A02},\n{0x00F915, 0x00F915, 0x006D1B},\n{0x00F916, 0x00F916, 0x0070D9},\n{0x00F917, 0x00F917, 0x0073DE},\n{0x00F918, 0x00F918, 0x00843D},\n{0x00F919, 0x00F919, 0x00916A},\n{0x00F91A, 0x00F91A, 0x0099F1},\n{0x00F91B, 0x00F91B, 0x004E82},\n{0x00F91C, 0x00F91C, 0x005375},\n{0x00F91D, 0x00F91D, 0x006B04},\n{0x00F91E, 0x00F91E, 0x00721B},\n{0x00F91F, 0x00F91F, 0x00862D},\n{0x00F920, 0x00F920, 0x009E1E},\n{0x00F921, 0x00F921, 0x005D50},\n{0x00F922, 0x00F922, 0x006FEB},\n{0x00F923, 0x00F923, 0x0085CD},\n{0x00F924, 0x00F924, 0x008964},\n{0x00F925, 0x00F925, 0x0062C9},\n{0x00F926, 0x00F926, 0x0081D8},\n{0x00F927, 0x00F927, 0x00881F},\n{0x00F928, 0x00F928, 0x005ECA},\n{0x00F929, 0x00F929, 0x006717},\n{0x00F92A, 0x00F92A, 0x006D6A},\n{0x00F92B, 0x00F92B, 0x0072FC},\n{0x00F92C, 0x00F92C, 0x0090CE},\n{0x00F92D, 0x00F92D, 0x004F86},\n{0x00F92E, 0x00F92E, 0x0051B7},\n{0x00F92F, 0x00F92F, 0x0052DE},\n{0x00F930, 0x00F930, 0x0064C4},\n{0x00F931, 0x00F931, 0x006AD3},\n{0x00F932, 0x00F932, 0x007210},\n{0x00F933, 0x00F933, 0x0076E7},\n{0x00F934, 0x00F934, 0x008001},\n{0x00F935, 0x00F935, 0x008606},\n{0x00F936, 0x00F936, 0x00865C},\n{0x00F937, 0x00F937, 0x008DEF},\n{0x00F938, 0x00F938, 0x009732},\n{0x00F939, 0x00F939, 0x009B6F},\n{0x00F93A, 0x00F93A, 0x009DFA},\n{0x00F93B, 0x00F93B, 0x00788C},\n{0x00F93C, 0x00F93C, 0x00797F},\n{0x00F93D, 0x00F93D, 0x007DA0},\n{0x00F93E, 0x00F93E, 0x0083C9},\n{0x00F93F, 0x00F93F, 0x009304},\n{0x00F940, 0x00F940, 0x009E7F},\n{0x00F941, 0x00F941, 0x008AD6},\n{0x00F942, 0x00F942, 0x0058DF},\n{0x00F943, 0x00F943, 0x005F04},\n{0x00F944, 0x00F944, 0x007C60},\n{0x00F945, 0x00F945, 0x00807E},\n{0x00F946, 0x00F946, 0x007262},\n{0x00F947, 0x00F947, 0x0078CA},\n{0x00F948, 0x00F948, 0x008CC2},\n{0x00F949, 0x00F949, 0x0096F7},\n{0x00F94A, 0x00F94A, 0x0058D8},\n{0x00F94B, 0x00F94B, 0x005C62},\n{0x00F94C, 0x00F94C, 0x006A13},\n{0x00F94D, 0x00F94D, 0x006DDA},\n{0x00F94E, 0x00F94E, 0x006F0F},\n{0x00F94F, 0x00F94F, 0x007D2F},\n{0x00F950, 0x00F950, 0x007E37},\n{0x00F951, 0x00F951, 0x00964B},\n{0x00F952, 0x00F952, 0x0052D2},\n{0x00F953, 0x00F953, 0x00808B},\n{0x00F954, 0x00F954, 0x0051DC},\n{0x00F955, 0x00F955, 0x0051CC},\n{0x00F956, 0x00F956, 0x007A1C},\n{0x00F957, 0x00F957, 0x007DBE},\n{0x00F958, 0x00F958, 0x0083F1},\n{0x00F959, 0x00F959, 0x009675},\n{0x00F95A, 0x00F95A, 0x008B80},\n{0x00F95B, 0x00F95B, 0x0062CF},\n{0x00F95C, 0x00F95C, 0x006A02},\n{0x00F95D, 0x00F95D, 0x008AFE},\n{0x00F95E, 0x00F95E, 0x004E39},\n{0x00F95F, 0x00F95F, 0x005BE7},\n{0x00F960, 0x00F960, 0x006012},\n{0x00F961, 0x00F961, 0x007387},\n{0x00F962, 0x00F962, 0x007570},\n{0x00F963, 0x00F963, 0x005317},\n{0x00F964, 0x00F964, 0x0078FB},\n{0x00F965, 0x00F965, 0x004FBF},\n{0x00F966, 0x00F966, 0x005FA9},\n{0x00F967, 0x00F967, 0x004E0D},\n{0x00F968, 0x00F968, 0x006CCC},\n{0x00F969, 0x00F969, 0x006578},\n{0x00F96A, 0x00F96A, 0x007D22},\n{0x00F96B, 0x00F96B, 0x0053C3},\n{0x00F96C, 0x00F96C, 0x00585E},\n{0x00F96D, 0x00F96D, 0x007701},\n{0x00F96E, 0x00F96E, 0x008449},\n{0x00F96F, 0x00F96F, 0x008AAA},\n{0x00F970, 0x00F970, 0x006BBA},\n{0x00F971, 0x00F971, 0x008FB0},\n{0x00F972, 0x00F972, 0x006C88},\n{0x00F973, 0x00F973, 0x0062FE},\n{0x00F974, 0x00F974, 0x0082E5},\n{0x00F975, 0x00F975, 0x0063A0},\n{0x00F976, 0x00F976, 0x007565},\n{0x00F977, 0x00F977, 0x004EAE},\n{0x00F978, 0x00F978, 0x005169},\n{0x00F979, 0x00F979, 0x0051C9},\n{0x00F97A, 0x00F97A, 0x006881},\n{0x00F97B, 0x00F97B, 0x007CE7},\n{0x00F97C, 0x00F97C, 0x00826F},\n{0x00F97D, 0x00F97D, 0x008AD2},\n{0x00F97E, 0x00F97E, 0x0091CF},\n{0x00F97F, 0x00F97F, 0x0052F5},\n{0x00F980, 0x00F980, 0x005442},\n{0x00F981, 0x00F981, 0x005973},\n{0x00F982, 0x00F982, 0x005EEC},\n{0x00F983, 0x00F983, 0x0065C5},\n{0x00F984, 0x00F984, 0x006FFE},\n{0x00F985, 0x00F985, 0x00792A},\n{0x00F986, 0x00F986, 0x0095AD},\n{0x00F987, 0x00F987, 0x009A6A},\n{0x00F988, 0x00F988, 0x009E97},\n{0x00F989, 0x00F989, 0x009ECE},\n{0x00F98A, 0x00F98A, 0x00529B},\n{0x00F98B, 0x00F98B, 0x0066C6},\n{0x00F98C, 0x00F98C, 0x006B77},\n{0x00F98D, 0x00F98D, 0x008F62},\n{0x00F98E, 0x00F98E, 0x005E74},\n{0x00F98F, 0x00F98F, 0x006190},\n{0x00F990, 0x00F990, 0x006200},\n{0x00F991, 0x00F991, 0x00649A},\n{0x00F992, 0x00F992, 0x006F23},\n{0x00F993, 0x00F993, 0x007149},\n{0x00F994, 0x00F994, 0x007489},\n{0x00F995, 0x00F995, 0x0079CA},\n{0x00F996, 0x00F996, 0x007DF4},\n{0x00F997, 0x00F997, 0x00806F},\n{0x00F998, 0x00F998, 0x008F26},\n{0x00F999, 0x00F999, 0x0084EE},\n{0x00F99A, 0x00F99A, 0x009023},\n{0x00F99B, 0x00F99B, 0x00934A},\n{0x00F99C, 0x00F99C, 0x005217},\n{0x00F99D, 0x00F99D, 0x0052A3},\n{0x00F99E, 0x00F99E, 0x0054BD},\n{0x00F99F, 0x00F99F, 0x0070C8},\n{0x00F9A0, 0x00F9A0, 0x0088C2},\n{0x00F9A1, 0x00F9A1, 0x008AAA},\n{0x00F9A2, 0x00F9A2, 0x005EC9},\n{0x00F9A3, 0x00F9A3, 0x005FF5},\n{0x00F9A4, 0x00F9A4, 0x00637B},\n{0x00F9A5, 0x00F9A5, 0x006BAE},\n{0x00F9A6, 0x00F9A6, 0x007C3E},\n{0x00F9A7, 0x00F9A7, 0x007375},\n{0x00F9A8, 0x00F9A8, 0x004EE4},\n{0x00F9A9, 0x00F9A9, 0x0056F9},\n{0x00F9AA, 0x00F9AA, 0x005BE7},\n{0x00F9AB, 0x00F9AB, 0x005DBA},\n{0x00F9AC, 0x00F9AC, 0x00601C},\n{0x00F9AD, 0x00F9AD, 0x0073B2},\n{0x00F9AE, 0x00F9AE, 0x007469},\n{0x00F9AF, 0x00F9AF, 0x007F9A},\n{0x00F9B0, 0x00F9B0, 0x008046},\n{0x00F9B1, 0x00F9B1, 0x009234},\n{0x00F9B2, 0x00F9B2, 0x0096F6},\n{0x00F9B3, 0x00F9B3, 0x009748},\n{0x00F9B4, 0x00F9B4, 0x009818},\n{0x00F9B5, 0x00F9B5, 0x004F8B},\n{0x00F9B6, 0x00F9B6, 0x0079AE},\n{0x00F9B7, 0x00F9B7, 0x0091B4},\n{0x00F9B8, 0x00F9B8, 0x0096B8},\n{0x00F9B9, 0x00F9B9, 0x0060E1},\n{0x00F9BA, 0x00F9BA, 0x004E86},\n{0x00F9BB, 0x00F9BB, 0x0050DA},\n{0x00F9BC, 0x00F9BC, 0x005BEE},\n{0x00F9BD, 0x00F9BD, 0x005C3F},\n{0x00F9BE, 0x00F9BE, 0x006599},\n{0x00F9BF, 0x00F9BF, 0x006A02},\n{0x00F9C0, 0x00F9C0, 0x0071CE},\n{0x00F9C1, 0x00F9C1, 0x007642},\n{0x00F9C2, 0x00F9C2, 0x0084FC},\n{0x00F9C3, 0x00F9C3, 0x00907C},\n{0x00F9C4, 0x00F9C4, 0x009F8D},\n{0x00F9C5, 0x00F9C5, 0x006688},\n{0x00F9C6, 0x00F9C6, 0x00962E},\n{0x00F9C7, 0x00F9C7, 0x005289},\n{0x00F9C8, 0x00F9C8, 0x00677B},\n{0x00F9C9, 0x00F9C9, 0x0067F3},\n{0x00F9CA, 0x00F9CA, 0x006D41},\n{0x00F9CB, 0x00F9CB, 0x006E9C},\n{0x00F9CC, 0x00F9CC, 0x007409},\n{0x00F9CD, 0x00F9CD, 0x007559},\n{0x00F9CE, 0x00F9CE, 0x00786B},\n{0x00F9CF, 0x00F9CF, 0x007D10},\n{0x00F9D0, 0x00F9D0, 0x00985E},\n{0x00F9D1, 0x00F9D1, 0x00516D},\n{0x00F9D2, 0x00F9D2, 0x00622E},\n{0x00F9D3, 0x00F9D3, 0x009678},\n{0x00F9D4, 0x00F9D4, 0x00502B},\n{0x00F9D5, 0x00F9D5, 0x005D19},\n{0x00F9D6, 0x00F9D6, 0x006DEA},\n{0x00F9D7, 0x00F9D7, 0x008F2A},\n{0x00F9D8, 0x00F9D8, 0x005F8B},\n{0x00F9D9, 0x00F9D9, 0x006144},\n{0x00F9DA, 0x00F9DA, 0x006817},\n{0x00F9DB, 0x00F9DB, 0x007387},\n{0x00F9DC, 0x00F9DC, 0x009686},\n{0x00F9DD, 0x00F9DD, 0x005229},\n{0x00F9DE, 0x00F9DE, 0x00540F},\n{0x00F9DF, 0x00F9DF, 0x005C65},\n{0x00F9E0, 0x00F9E0, 0x006613},\n{0x00F9E1, 0x00F9E1, 0x00674E},\n{0x00F9E2, 0x00F9E2, 0x0068A8},\n{0x00F9E3, 0x00F9E3, 0x006CE5},\n{0x00F9E4, 0x00F9E4, 0x007406},\n{0x00F9E5, 0x00F9E5, 0x0075E2},\n{0x00F9E6, 0x00F9E6, 0x007F79},\n{0x00F9E7, 0x00F9E7, 0x0088CF},\n{0x00F9E8, 0x00F9E8, 0x0088E1},\n{0x00F9E9, 0x00F9E9, 0x0091CC},\n{0x00F9EA, 0x00F9EA, 0x0096E2},\n{0x00F9EB, 0x00F9EB, 0x00533F},\n{0x00F9EC, 0x00F9EC, 0x006EBA},\n{0x00F9ED, 0x00F9ED, 0x00541D},\n{0x00F9EE, 0x00F9EE, 0x0071D0},\n{0x00F9EF, 0x00F9EF, 0x007498},\n{0x00F9F0, 0x00F9F0, 0x0085FA},\n{0x00F9F1, 0x00F9F1, 0x0096A3},\n{0x00F9F2, 0x00F9F2, 0x009C57},\n{0x00F9F3, 0x00F9F3, 0x009E9F},\n{0x00F9F4, 0x00F9F4, 0x006797},\n{0x00F9F5, 0x00F9F5, 0x006DCB},\n{0x00F9F6, 0x00F9F6, 0x0081E8},\n{0x00F9F7, 0x00F9F7, 0x007ACB},\n{0x00F9F8, 0x00F9F8, 0x007B20},\n{0x00F9F9, 0x00F9F9, 0x007C92},\n{0x00F9FA, 0x00F9FA, 0x0072C0},\n{0x00F9FB, 0x00F9FB, 0x007099},\n{0x00F9FC, 0x00F9FC, 0x008B58},\n{0x00F9FD, 0x00F9FD, 0x004EC0},\n{0x00F9FE, 0x00F9FE, 0x008336},\n{0x00F9FF, 0x00F9FF, 0x00523A},\n{0x00FA00, 0x00FA00, 0x005207},\n{0x00FA01, 0x00FA01, 0x005EA6},\n{0x00FA02, 0x00FA02, 0x0062D3},\n{0x00FA03, 0x00FA03, 0x007CD6},\n{0x00FA04, 0x00FA04, 0x005B85},\n{0x00FA05, 0x00FA05, 0x006D1E},\n{0x00FA06, 0x00FA06, 0x0066B4},\n{0x00FA07, 0x00FA07, 0x008F3B},\n{0x00FA08, 0x00FA08, 0x00884C},\n{0x00FA09, 0x00FA09, 0x00964D},\n{0x00FA0A, 0x00FA0A, 0x00898B},\n{0x00FA0B, 0x00FA0B, 0x005ED3},\n{0x00FA0C, 0x00FA0C, 0x005140},\n{0x00FA0D, 0x00FA0D, 0x0055C0},\n{0x00FA10, 0x00FA10, 0x00585A},\n{0x00FA12, 0x00FA12, 0x006674},\n{0x00FA15, 0x00FA15, 0x0051DE},\n{0x00FA16, 0x00FA16, 0x00732A},\n{0x00FA17, 0x00FA17, 0x0076CA},\n{0x00FA18, 0x00FA18, 0x00793C},\n{0x00FA19, 0x00FA19, 0x00795E},\n{0x00FA1A, 0x00FA1A, 0x007965},\n{0x00FA1B, 0x00FA1B, 0x00798F},\n{0x00FA1C, 0x00FA1C, 0x009756},\n{0x00FA1D, 0x00FA1D, 0x007CBE},\n{0x00FA1E, 0x00FA1E, 0x007FBD},\n{0x00FA20, 0x00FA20, 0x008612},\n{0x00FA22, 0x00FA22, 0x008AF8},\n{0x00FA25, 0x00FA25, 0x009038},\n{0x00FA26, 0x00FA26, 0x0090FD},\n{0x00FA2A, 0x00FA2A, 0x0098EF},\n{0x00FA2B, 0x00FA2B, 0x0098FC},\n{0x00FA2C, 0x00FA2C, 0x009928},\n{0x00FA2D, 0x00FA2D, 0x009DB4},\n{0x00FA2E, 0x00FA2E, 0x0090DE},\n{0x00FA2F, 0x00FA2F, 0x0096B7},\n{0x00FA30, 0x00FA30, 0x004FAE},\n{0x00FA31, 0x00FA31, 0x0050E7},\n{0x00FA32, 0x00FA32, 0x00514D},\n{0x00FA33, 0x00FA33, 0x0052C9},\n{0x00FA34, 0x00FA34, 0x0052E4},\n{0x00FA35, 0x00FA35, 0x005351},\n{0x00FA36, 0x00FA36, 0x00559D},\n{0x00FA37, 0x00FA37, 0x005606},\n{0x00FA38, 0x00FA38, 0x005668},\n{0x00FA39, 0x00FA39, 0x005840},\n{0x00FA3A, 0x00FA3A, 0x0058A8},\n{0x00FA3B, 0x00FA3B, 0x005C64},\n{0x00FA3C, 0x00FA3C, 0x005C6E},\n{0x00FA3D, 0x00FA3D, 0x006094},\n{0x00FA3E, 0x00FA3E, 0x006168},\n{0x00FA3F, 0x00FA3F, 0x00618E},\n{0x00FA40, 0x00FA40, 0x0061F2},\n{0x00FA41, 0x00FA41, 0x00654F},\n{0x00FA42, 0x00FA42, 0x0065E2},\n{0x00FA43, 0x00FA43, 0x006691},\n{0x00FA44, 0x00FA44, 0x006885},\n{0x00FA45, 0x00FA45, 0x006D77},\n{0x00FA46, 0x00FA46, 0x006E1A},\n{0x00FA47, 0x00FA47, 0x006F22},\n{0x00FA48, 0x00FA48, 0x00716E},\n{0x00FA49, 0x00FA49, 0x00722B},\n{0x00FA4A, 0x00FA4A, 0x007422},\n{0x00FA4B, 0x00FA4B, 0x007891},\n{0x00FA4C, 0x00FA4C, 0x00793E},\n{0x00FA4D, 0x00FA4D, 0x007949},\n{0x00FA4E, 0x00FA4E, 0x007948},\n{0x00FA4F, 0x00FA4F, 0x007950},\n{0x00FA50, 0x00FA50, 0x007956},\n{0x00FA51, 0x00FA51, 0x00795D},\n{0x00FA52, 0x00FA52, 0x00798D},\n{0x00FA53, 0x00FA53, 0x00798E},\n{0x00FA54, 0x00FA54, 0x007A40},\n{0x00FA55, 0x00FA55, 0x007A81},\n{0x00FA56, 0x00FA56, 0x007BC0},\n{0x00FA57, 0x00FA57, 0x007DF4},\n{0x00FA58, 0x00FA58, 0x007E09},\n{0x00FA59, 0x00FA59, 0x007E41},\n{0x00FA5A, 0x00FA5A, 0x007F72},\n{0x00FA5B, 0x00FA5B, 0x008005},\n{0x00FA5C, 0x00FA5C, 0x0081ED},\n{0x00FA5D, 0x00FA5E, 0x008279},\n{0x00FA5F, 0x00FA5F, 0x008457},\n{0x00FA60, 0x00FA60, 0x008910},\n{0x00FA61, 0x00FA61, 0x008996},\n{0x00FA62, 0x00FA62, 0x008B01},\n{0x00FA63, 0x00FA63, 0x008B39},\n{0x00FA64, 0x00FA64, 0x008CD3},\n{0x00FA65, 0x00FA65, 0x008D08},\n{0x00FA66, 0x00FA66, 0x008FB6},\n{0x00FA67, 0x00FA67, 0x009038},\n{0x00FA68, 0x00FA68, 0x0096E3},\n{0x00FA69, 0x00FA69, 0x0097FF},\n{0x00FA6A, 0x00FA6A, 0x00983B},\n{0x00FA6B, 0x00FA6B, 0x006075},\n{0x00FA6C, 0x00FA6C, 0x0242EE},\n{0x00FA6D, 0x00FA6D, 0x008218},\n{0x00FA70, 0x00FA70, 0x004E26},\n{0x00FA71, 0x00FA71, 0x0051B5},\n{0x00FA72, 0x00FA72, 0x005168},\n{0x00FA73, 0x00FA73, 0x004F80},\n{0x00FA74, 0x00FA74, 0x005145},\n{0x00FA75, 0x00FA75, 0x005180},\n{0x00FA76, 0x00FA76, 0x0052C7},\n{0x00FA77, 0x00FA77, 0x0052FA},\n{0x00FA78, 0x00FA78, 0x00559D},\n{0x00FA79, 0x00FA79, 0x005555},\n{0x00FA7A, 0x00FA7A, 0x005599},\n{0x00FA7B, 0x00FA7B, 0x0055E2},\n{0x00FA7C, 0x00FA7C, 0x00585A},\n{0x00FA7D, 0x00FA7D, 0x0058B3},\n{0x00FA7E, 0x00FA7E, 0x005944},\n{0x00FA7F, 0x00FA7F, 0x005954},\n{0x00FA80, 0x00FA80, 0x005A62},\n{0x00FA81, 0x00FA81, 0x005B28},\n{0x00FA82, 0x00FA82, 0x005ED2},\n{0x00FA83, 0x00FA83, 0x005ED9},\n{0x00FA84, 0x00FA84, 0x005F69},\n{0x00FA85, 0x00FA85, 0x005FAD},\n{0x00FA86, 0x00FA86, 0x0060D8},\n{0x00FA87, 0x00FA87, 0x00614E},\n{0x00FA88, 0x00FA88, 0x006108},\n{0x00FA89, 0x00FA89, 0x00618E},\n{0x00FA8A, 0x00FA8A, 0x006160},\n{0x00FA8B, 0x00FA8B, 0x0061F2},\n{0x00FA8C, 0x00FA8C, 0x006234},\n{0x00FA8D, 0x00FA8D, 0x0063C4},\n{0x00FA8E, 0x00FA8E, 0x00641C},\n{0x00FA8F, 0x00FA8F, 0x006452},\n{0x00FA90, 0x00FA90, 0x006556},\n{0x00FA91, 0x00FA91, 0x006674},\n{0x00FA92, 0x00FA92, 0x006717},\n{0x00FA93, 0x00FA93, 0x00671B},\n{0x00FA94, 0x00FA94, 0x006756},\n{0x00FA95, 0x00FA95, 0x006B79},\n{0x00FA96, 0x00FA96, 0x006BBA},\n{0x00FA97, 0x00FA97, 0x006D41},\n{0x00FA98, 0x00FA98, 0x006EDB},\n{0x00FA99, 0x00FA99, 0x006ECB},\n{0x00FA9A, 0x00FA9A, 0x006F22},\n{0x00FA9B, 0x00FA9B, 0x00701E},\n{0x00FA9C, 0x00FA9C, 0x00716E},\n{0x00FA9D, 0x00FA9D, 0x0077A7},\n{0x00FA9E, 0x00FA9E, 0x007235},\n{0x00FA9F, 0x00FA9F, 0x0072AF},\n{0x00FAA0, 0x00FAA0, 0x00732A},\n{0x00FAA1, 0x00FAA1, 0x007471},\n{0x00FAA2, 0x00FAA2, 0x007506},\n{0x00FAA3, 0x00FAA3, 0x00753B},\n{0x00FAA4, 0x00FAA4, 0x00761D},\n{0x00FAA5, 0x00FAA5, 0x00761F},\n{0x00FAA6, 0x00FAA6, 0x0076CA},\n{0x00FAA7, 0x00FAA7, 0x0076DB},\n{0x00FAA8, 0x00FAA8, 0x0076F4},\n{0x00FAA9, 0x00FAA9, 0x00774A},\n{0x00FAAA, 0x00FAAA, 0x007740},\n{0x00FAAB, 0x00FAAB, 0x0078CC},\n{0x00FAAC, 0x00FAAC, 0x007AB1},\n{0x00FAAD, 0x00FAAD, 0x007BC0},\n{0x00FAAE, 0x00FAAE, 0x007C7B},\n{0x00FAAF, 0x00FAAF, 0x007D5B},\n{0x00FAB0, 0x00FAB0, 0x007DF4},\n{0x00FAB1, 0x00FAB1, 0x007F3E},\n{0x00FAB2, 0x00FAB2, 0x008005},\n{0x00FAB3, 0x00FAB3, 0x008352},\n{0x00FAB4, 0x00FAB4, 0x0083EF},\n{0x00FAB5, 0x00FAB5, 0x008779},\n{0x00FAB6, 0x00FAB6, 0x008941},\n{0x00FAB7, 0x00FAB7, 0x008986},\n{0x00FAB8, 0x00FAB8, 0x008996},\n{0x00FAB9, 0x00FAB9, 0x008ABF},\n{0x00FABA, 0x00FABA, 0x008AF8},\n{0x00FABB, 0x00FABB, 0x008ACB},\n{0x00FABC, 0x00FABC, 0x008B01},\n{0x00FABD, 0x00FABD, 0x008AFE},\n{0x00FABE, 0x00FABE, 0x008AED},\n{0x00FABF, 0x00FABF, 0x008B39},\n{0x00FAC0, 0x00FAC0, 0x008B8A},\n{0x00FAC1, 0x00FAC1, 0x008D08},\n{0x00FAC2, 0x00FAC2, 0x008F38},\n{0x00FAC3, 0x00FAC3, 0x009072},\n{0x00FAC4, 0x00FAC4, 0x009199},\n{0x00FAC5, 0x00FAC5, 0x009276},\n{0x00FAC6, 0x00FAC6, 0x00967C},\n{0x00FAC7, 0x00FAC7, 0x0096E3},\n{0x00FAC8, 0x00FAC8, 0x009756},\n{0x00FAC9, 0x00FAC9, 0x0097DB},\n{0x00FACA, 0x00FACA, 0x0097FF},\n{0x00FACB, 0x00FACB, 0x00980B},\n{0x00FACC, 0x00FACC, 0x00983B},\n{0x00FACD, 0x00FACD, 0x009B12},\n{0x00FACE, 0x00FACE, 0x009F9C},\n{0x00FACF, 0x00FACF, 0x02284A},\n{0x00FAD0, 0x00FAD0, 0x022844},\n{0x00FAD1, 0x00FAD1, 0x0233D5},\n{0x00FAD2, 0x00FAD2, 0x003B9D},\n{0x00FAD3, 0x00FAD3, 0x004018},\n{0x00FAD4, 0x00FAD4, 0x004039},\n{0x00FAD5, 0x00FAD5, 0x025249},\n{0x00FAD6, 0x00FAD6, 0x025CD0},\n{0x00FAD7, 0x00FAD7, 0x027ED3},\n{0x00FAD8, 0x00FAD8, 0x009F43},\n{0x00FAD9, 0x00FAD9, 0x009F8E},\n{0x00FB1D, 0x00FB1D, 0x0005D9},\n{0x00FB1F, 0x00FB1F, 0x0005F2},\n{0x00FB2A, 0x00FB2D, 0x0005E9},\n{0x00FB2E, 0x00FB30, 0x0005D0},\n{0x00FB31, 0x00FB31, 0x0005D1},\n{0x00FB32, 0x00FB32, 0x0005D2},\n{0x00FB33, 0x00FB33, 0x0005D3},\n{0x00FB34, 0x00FB34, 0x0005D4},\n{0x00FB35, 0x00FB35, 0x0005D5},\n{0x00FB36, 0x00FB36, 0x0005D6},\n{0x00FB38, 0x00FB38, 0x0005D8},\n{0x00FB39, 0x00FB39, 0x0005D9},\n{0x00FB3A, 0x00FB3A, 0x0005DA},\n{0x00FB3B, 0x00FB3B, 0x0005DB},\n{0x00FB3C, 0x00FB3C, 0x0005DC},\n{0x00FB3E, 0x00FB3E, 0x0005DE},\n{0x00FB40, 0x00FB40, 0x0005E0},\n{0x00FB41, 0x00FB41, 0x0005E1},\n{0x00FB43, 0x00FB43, 0x0005E3},\n{0x00FB44, 0x00FB44, 0x0005E4},\n{0x00FB46, 0x00FB46, 0x0005E6},\n{0x00FB47, 0x00FB47, 0x0005E7},\n{0x00FB48, 0x00FB48, 0x0005E8},\n{0x00FB49, 0x00FB49, 0x0005E9},\n{0x00FB4A, 0x00FB4A, 0x0005EA},\n{0x00FB4B, 0x00FB4B, 0x0005D5},\n{0x00FB4C, 0x00FB4C, 0x0005D1},\n{0x00FB4D, 0x00FB4D, 0x0005DB},\n{0x00FB4E, 0x00FB4E, 0x0005E4},\n{0x01109A, 0x01109A, 0x011099},\n{0x01109C, 0x01109C, 0x01109B},\n{0x0110AB, 0x0110AB, 0x0110A5},\n{0x01112E, 0x01112E, 0x011131},\n{0x01112F, 0x01112F, 0x011132},\n{0x01134B, 0x01134C, 0x011347},\n{0x0114BB, 0x0114BC, 0x0114B9},\n{0x0114BE, 0x0114BE, 0x0114B9},\n{0x0115BA, 0x0115BA, 0x0115B8},\n{0x0115BB, 0x0115BB, 0x0115B9},\n{0x011938, 0x011938, 0x011935},\n{0x01D15E, 0x01D15E, 0x01D157},\n{0x01D15F, 0x01D164, 0x01D158},\n{0x01D1BB, 0x01D1BB, 0x01D1B9},\n{0x01D1BC, 0x01D1BC, 0x01D1BA},\n{0x01D1BD, 0x01D1BD, 0x01D1B9},\n{0x01D1BE, 0x01D1BE, 0x01D1BA},\n{0x01D1BF, 0x01D1BF, 0x01D1B9},\n{0x01D1C0, 0x01D1C0, 0x01D1BA},\n{0x02F800, 0x02F800, 0x004E3D},\n{0x02F801, 0x02F801, 0x004E38},\n{0x02F802, 0x02F802, 0x004E41},\n{0x02F803, 0x02F803, 0x020122},\n{0x02F804, 0x02F804, 0x004F60},\n{0x02F805, 0x02F805, 0x004FAE},\n{0x02F806, 0x02F806, 0x004FBB},\n{0x02F807, 0x02F807, 0x005002},\n{0x02F808, 0x02F808, 0x00507A},\n{0x02F809, 0x02F809, 0x005099},\n{0x02F80A, 0x02F80A, 0x0050E7},\n{0x02F80B, 0x02F80B, 0x0050CF},\n{0x02F80C, 0x02F80C, 0x00349E},\n{0x02F80D, 0x02F80D, 0x02063A},\n{0x02F80E, 0x02F80E, 0x00514D},\n{0x02F80F, 0x02F80F, 0x005154},\n{0x02F810, 0x02F810, 0x005164},\n{0x02F811, 0x02F811, 0x005177},\n{0x02F812, 0x02F812, 0x02051C},\n{0x02F813, 0x02F813, 0x0034B9},\n{0x02F814, 0x02F814, 0x005167},\n{0x02F815, 0x02F815, 0x00518D},\n{0x02F816, 0x02F816, 0x02054B},\n{0x02F817, 0x02F817, 0x005197},\n{0x02F818, 0x02F818, 0x0051A4},\n{0x02F819, 0x02F819, 0x004ECC},\n{0x02F81A, 0x02F81A, 0x0051AC},\n{0x02F81B, 0x02F81B, 0x0051B5},\n{0x02F81C, 0x02F81C, 0x0291DF},\n{0x02F81D, 0x02F81D, 0x0051F5},\n{0x02F81E, 0x02F81E, 0x005203},\n{0x02F81F, 0x02F81F, 0x0034DF},\n{0x02F820, 0x02F820, 0x00523B},\n{0x02F821, 0x02F821, 0x005246},\n{0x02F822, 0x02F822, 0x005272},\n{0x02F823, 0x02F823, 0x005277},\n{0x02F824, 0x02F824, 0x003515},\n{0x02F825, 0x02F825, 0x0052C7},\n{0x02F826, 0x02F826, 0x0052C9},\n{0x02F827, 0x02F827, 0x0052E4},\n{0x02F828, 0x02F828, 0x0052FA},\n{0x02F829, 0x02F829, 0x005305},\n{0x02F82A, 0x02F82A, 0x005306},\n{0x02F82B, 0x02F82B, 0x005317},\n{0x02F82C, 0x02F82C, 0x005349},\n{0x02F82D, 0x02F82D, 0x005351},\n{0x02F82E, 0x02F82E, 0x00535A},\n{0x02F82F, 0x02F82F, 0x005373},\n{0x02F830, 0x02F830, 0x00537D},\n{0x02F831, 0x02F833, 0x00537F},\n{0x02F834, 0x02F834, 0x020A2C},\n{0x02F835, 0x02F835, 0x007070},\n{0x02F836, 0x02F836, 0x0053CA},\n{0x02F837, 0x02F837, 0x0053DF},\n{0x02F838, 0x02F838, 0x020B63},\n{0x02F839, 0x02F839, 0x0053EB},\n{0x02F83A, 0x02F83A, 0x0053F1},\n{0x02F83B, 0x02F83B, 0x005406},\n{0x02F83C, 0x02F83C, 0x00549E},\n{0x02F83D, 0x02F83D, 0x005438},\n{0x02F83E, 0x02F83E, 0x005448},\n{0x02F83F, 0x02F83F, 0x005468},\n{0x02F840, 0x02F840, 0x0054A2},\n{0x02F841, 0x02F841, 0x0054F6},\n{0x02F842, 0x02F842, 0x005510},\n{0x02F843, 0x02F843, 0x005553},\n{0x02F844, 0x02F844, 0x005563},\n{0x02F845, 0x02F846, 0x005584},\n{0x02F847, 0x02F847, 0x005599},\n{0x02F848, 0x02F848, 0x0055AB},\n{0x02F849, 0x02F849, 0x0055B3},\n{0x02F84A, 0x02F84A, 0x0055C2},\n{0x02F84B, 0x02F84B, 0x005716},\n{0x02F84C, 0x02F84C, 0x005606},\n{0x02F84D, 0x02F84D, 0x005717},\n{0x02F84E, 0x02F84E, 0x005651},\n{0x02F84F, 0x02F84F, 0x005674},\n{0x02F850, 0x02F850, 0x005207},\n{0x02F851, 0x02F851, 0x0058EE},\n{0x02F852, 0x02F852, 0x0057CE},\n{0x02F853, 0x02F853, 0x0057F4},\n{0x02F854, 0x02F854, 0x00580D},\n{0x02F855, 0x02F855, 0x00578B},\n{0x02F856, 0x02F856, 0x005832},\n{0x02F857, 0x02F857, 0x005831},\n{0x02F858, 0x02F858, 0x0058AC},\n{0x02F859, 0x02F859, 0x0214E4},\n{0x02F85A, 0x02F85A, 0x0058F2},\n{0x02F85B, 0x02F85B, 0x0058F7},\n{0x02F85C, 0x02F85C, 0x005906},\n{0x02F85D, 0x02F85D, 0x00591A},\n{0x02F85E, 0x02F85E, 0x005922},\n{0x02F85F, 0x02F85F, 0x005962},\n{0x02F860, 0x02F860, 0x0216A8},\n{0x02F861, 0x02F861, 0x0216EA},\n{0x02F862, 0x02F862, 0x0059EC},\n{0x02F863, 0x02F863, 0x005A1B},\n{0x02F864, 0x02F864, 0x005A27},\n{0x02F865, 0x02F865, 0x0059D8},\n{0x02F866, 0x02F866, 0x005A66},\n{0x02F867, 0x02F867, 0x0036EE},\n{0x02F868, 0x02F868, 0x0036FC},\n{0x02F869, 0x02F869, 0x005B08},\n{0x02F86A, 0x02F86B, 0x005B3E},\n{0x02F86C, 0x02F86C, 0x0219C8},\n{0x02F86D, 0x02F86D, 0x005BC3},\n{0x02F86E, 0x02F86E, 0x005BD8},\n{0x02F86F, 0x02F86F, 0x005BE7},\n{0x02F870, 0x02F870, 0x005BF3},\n{0x02F871, 0x02F871, 0x021B18},\n{0x02F872, 0x02F872, 0x005BFF},\n{0x02F873, 0x02F873, 0x005C06},\n{0x02F874, 0x02F874, 0x005F53},\n{0x02F875, 0x02F875, 0x005C22},\n{0x02F876, 0x02F876, 0x003781},\n{0x02F877, 0x02F877, 0x005C60},\n{0x02F878, 0x02F878, 0x005C6E},\n{0x02F879, 0x02F879, 0x005CC0},\n{0x02F87A, 0x02F87A, 0x005C8D},\n{0x02F87B, 0x02F87B, 0x021DE4},\n{0x02F87C, 0x02F87C, 0x005D43},\n{0x02F87D, 0x02F87D, 0x021DE6},\n{0x02F87E, 0x02F87E, 0x005D6E},\n{0x02F87F, 0x02F87F, 0x005D6B},\n{0x02F880, 0x02F880, 0x005D7C},\n{0x02F881, 0x02F881, 0x005DE1},\n{0x02F882, 0x02F882, 0x005DE2},\n{0x02F883, 0x02F883, 0x00382F},\n{0x02F884, 0x02F884, 0x005DFD},\n{0x02F885, 0x02F885, 0x005E28},\n{0x02F886, 0x02F886, 0x005E3D},\n{0x02F887, 0x02F887, 0x005E69},\n{0x02F888, 0x02F888, 0x003862},\n{0x02F889, 0x02F889, 0x022183},\n{0x02F88A, 0x02F88A, 0x00387C},\n{0x02F88B, 0x02F88B, 0x005EB0},\n{0x02F88C, 0x02F88C, 0x005EB3},\n{0x02F88D, 0x02F88D, 0x005EB6},\n{0x02F88E, 0x02F88E, 0x005ECA},\n{0x02F88F, 0x02F88F, 0x02A392},\n{0x02F890, 0x02F890, 0x005EFE},\n{0x02F891, 0x02F892, 0x022331},\n{0x02F893, 0x02F893, 0x008201},\n{0x02F894, 0x02F895, 0x005F22},\n{0x02F896, 0x02F896, 0x0038C7},\n{0x02F897, 0x02F897, 0x0232B8},\n{0x02F898, 0x02F898, 0x0261DA},\n{0x02F899, 0x02F899, 0x005F62},\n{0x02F89A, 0x02F89A, 0x005F6B},\n{0x02F89B, 0x02F89B, 0x0038E3},\n{0x02F89C, 0x02F89C, 0x005F9A},\n{0x02F89D, 0x02F89D, 0x005FCD},\n{0x02F89E, 0x02F89E, 0x005FD7},\n{0x02F89F, 0x02F89F, 0x005FF9},\n{0x02F8A0, 0x02F8A0, 0x006081},\n{0x02F8A1, 0x02F8A1, 0x00393A},\n{0x02F8A2, 0x02F8A2, 0x00391C},\n{0x02F8A3, 0x02F8A3, 0x006094},\n{0x02F8A4, 0x02F8A4, 0x0226D4},\n{0x02F8A5, 0x02F8A5, 0x0060C7},\n{0x02F8A6, 0x02F8A6, 0x006148},\n{0x02F8A7, 0x02F8A7, 0x00614C},\n{0x02F8A8, 0x02F8A8, 0x00614E},\n{0x02F8A9, 0x02F8A9, 0x00614C},\n{0x02F8AA, 0x02F8AA, 0x00617A},\n{0x02F8AB, 0x02F8AB, 0x00618E},\n{0x02F8AC, 0x02F8AC, 0x0061B2},\n{0x02F8AD, 0x02F8AD, 0x0061A4},\n{0x02F8AE, 0x02F8AE, 0x0061AF},\n{0x02F8AF, 0x02F8AF, 0x0061DE},\n{0x02F8B0, 0x02F8B0, 0x0061F2},\n{0x02F8B1, 0x02F8B1, 0x0061F6},\n{0x02F8B2, 0x02F8B2, 0x006210},\n{0x02F8B3, 0x02F8B3, 0x00621B},\n{0x02F8B4, 0x02F8B4, 0x00625D},\n{0x02F8B5, 0x02F8B5, 0x0062B1},\n{0x02F8B6, 0x02F8B6, 0x0062D4},\n{0x02F8B7, 0x02F8B7, 0x006350},\n{0x02F8B8, 0x02F8B8, 0x022B0C},\n{0x02F8B9, 0x02F8B9, 0x00633D},\n{0x02F8BA, 0x02F8BA, 0x0062FC},\n{0x02F8BB, 0x02F8BB, 0x006368},\n{0x02F8BC, 0x02F8BC, 0x006383},\n{0x02F8BD, 0x02F8BD, 0x0063E4},\n{0x02F8BE, 0x02F8BE, 0x022BF1},\n{0x02F8BF, 0x02F8BF, 0x006422},\n{0x02F8C0, 0x02F8C0, 0x0063C5},\n{0x02F8C1, 0x02F8C1, 0x0063A9},\n{0x02F8C2, 0x02F8C2, 0x003A2E},\n{0x02F8C3, 0x02F8C3, 0x006469},\n{0x02F8C4, 0x02F8C4, 0x00647E},\n{0x02F8C5, 0x02F8C5, 0x00649D},\n{0x02F8C6, 0x02F8C6, 0x006477},\n{0x02F8C7, 0x02F8C7, 0x003A6C},\n{0x02F8C8, 0x02F8C8, 0x00654F},\n{0x02F8C9, 0x02F8C9, 0x00656C},\n{0x02F8CA, 0x02F8CA, 0x02300A},\n{0x02F8CB, 0x02F8CB, 0x0065E3},\n{0x02F8CC, 0x02F8CC, 0x0066F8},\n{0x02F8CD, 0x02F8CD, 0x006649},\n{0x02F8CE, 0x02F8CE, 0x003B19},\n{0x02F8CF, 0x02F8CF, 0x006691},\n{0x02F8D0, 0x02F8D0, 0x003B08},\n{0x02F8D1, 0x02F8D1, 0x003AE4},\n{0x02F8D2, 0x02F8D2, 0x005192},\n{0x02F8D3, 0x02F8D3, 0x005195},\n{0x02F8D4, 0x02F8D4, 0x006700},\n{0x02F8D5, 0x02F8D5, 0x00669C},\n{0x02F8D6, 0x02F8D6, 0x0080AD},\n{0x02F8D7, 0x02F8D7, 0x0043D9},\n{0x02F8D8, 0x02F8D8, 0x006717},\n{0x02F8D9, 0x02F8D9, 0x00671B},\n{0x02F8DA, 0x02F8DA, 0x006721},\n{0x02F8DB, 0x02F8DB, 0x00675E},\n{0x02F8DC, 0x02F8DC, 0x006753},\n{0x02F8DD, 0x02F8DD, 0x0233C3},\n{0x02F8DE, 0x02F8DE, 0x003B49},\n{0x02F8DF, 0x02F8DF, 0x0067FA},\n{0x02F8E0, 0x02F8E0, 0x006785},\n{0x02F8E1, 0x02F8E1, 0x006852},\n{0x02F8E2, 0x02F8E2, 0x006885},\n{0x02F8E3, 0x02F8E3, 0x02346D},\n{0x02F8E4, 0x02F8E4, 0x00688E},\n{0x02F8E5, 0x02F8E5, 0x00681F},\n{0x02F8E6, 0x02F8E6, 0x006914},\n{0x02F8E7, 0x02F8E7, 0x003B9D},\n{0x02F8E8, 0x02F8E8, 0x006942},\n{0x02F8E9, 0x02F8E9, 0x0069A3},\n{0x02F8EA, 0x02F8EA, 0x0069EA},\n{0x02F8EB, 0x02F8EB, 0x006AA8},\n{0x02F8EC, 0x02F8EC, 0x0236A3},\n{0x02F8ED, 0x02F8ED, 0x006ADB},\n{0x02F8EE, 0x02F8EE, 0x003C18},\n{0x02F8EF, 0x02F8EF, 0x006B21},\n{0x02F8F0, 0x02F8F0, 0x0238A7},\n{0x02F8F1, 0x02F8F1, 0x006B54},\n{0x02F8F2, 0x02F8F2, 0x003C4E},\n{0x02F8F3, 0x02F8F3, 0x006B72},\n{0x02F8F4, 0x02F8F4, 0x006B9F},\n{0x02F8F5, 0x02F8F5, 0x006BBA},\n{0x02F8F6, 0x02F8F6, 0x006BBB},\n{0x02F8F7, 0x02F8F7, 0x023A8D},\n{0x02F8F8, 0x02F8F8, 0x021D0B},\n{0x02F8F9, 0x02F8F9, 0x023AFA},\n{0x02F8FA, 0x02F8FA, 0x006C4E},\n{0x02F8FB, 0x02F8FB, 0x023CBC},\n{0x02F8FC, 0x02F8FC, 0x006CBF},\n{0x02F8FD, 0x02F8FD, 0x006CCD},\n{0x02F8FE, 0x02F8FE, 0x006C67},\n{0x02F8FF, 0x02F8FF, 0x006D16},\n{0x02F900, 0x02F900, 0x006D3E},\n{0x02F901, 0x02F901, 0x006D77},\n{0x02F902, 0x02F902, 0x006D41},\n{0x02F903, 0x02F903, 0x006D69},\n{0x02F904, 0x02F904, 0x006D78},\n{0x02F905, 0x02F905, 0x006D85},\n{0x02F906, 0x02F906, 0x023D1E},\n{0x02F907, 0x02F907, 0x006D34},\n{0x02F908, 0x02F908, 0x006E2F},\n{0x02F909, 0x02F909, 0x006E6E},\n{0x02F90A, 0x02F90A, 0x003D33},\n{0x02F90B, 0x02F90B, 0x006ECB},\n{0x02F90C, 0x02F90C, 0x006EC7},\n{0x02F90D, 0x02F90D, 0x023ED1},\n{0x02F90E, 0x02F90E, 0x006DF9},\n{0x02F90F, 0x02F90F, 0x006F6E},\n{0x02F910, 0x02F910, 0x023F5E},\n{0x02F911, 0x02F911, 0x023F8E},\n{0x02F912, 0x02F912, 0x006FC6},\n{0x02F913, 0x02F913, 0x007039},\n{0x02F914, 0x02F914, 0x00701E},\n{0x02F915, 0x02F915, 0x00701B},\n{0x02F916, 0x02F916, 0x003D96},\n{0x02F917, 0x02F917, 0x00704A},\n{0x02F918, 0x02F918, 0x00707D},\n{0x02F919, 0x02F919, 0x007077},\n{0x02F91A, 0x02F91A, 0x0070AD},\n{0x02F91B, 0x02F91B, 0x020525},\n{0x02F91C, 0x02F91C, 0x007145},\n{0x02F91D, 0x02F91D, 0x024263},\n{0x02F91E, 0x02F91E, 0x00719C},\n{0x02F91F, 0x02F91F, 0x0243AB},\n{0x02F920, 0x02F920, 0x007228},\n{0x02F921, 0x02F921, 0x007235},\n{0x02F922, 0x02F922, 0x007250},\n{0x02F923, 0x02F923, 0x024608},\n{0x02F924, 0x02F924, 0x007280},\n{0x02F925, 0x02F925, 0x007295},\n{0x02F926, 0x02F926, 0x024735},\n{0x02F927, 0x02F927, 0x024814},\n{0x02F928, 0x02F928, 0x00737A},\n{0x02F929, 0x02F929, 0x00738B},\n{0x02F92A, 0x02F92A, 0x003EAC},\n{0x02F92B, 0x02F92B, 0x0073A5},\n{0x02F92C, 0x02F92D, 0x003EB8},\n{0x02F92E, 0x02F92E, 0x007447},\n{0x02F92F, 0x02F92F, 0x00745C},\n{0x02F930, 0x02F930, 0x007471},\n{0x02F931, 0x02F931, 0x007485},\n{0x02F932, 0x02F932, 0x0074CA},\n{0x02F933, 0x02F933, 0x003F1B},\n{0x02F934, 0x02F934, 0x007524},\n{0x02F935, 0x02F935, 0x024C36},\n{0x02F936, 0x02F936, 0x00753E},\n{0x02F937, 0x02F937, 0x024C92},\n{0x02F938, 0x02F938, 0x007570},\n{0x02F939, 0x02F939, 0x02219F},\n{0x02F93A, 0x02F93A, 0x007610},\n{0x02F93B, 0x02F93B, 0x024FA1},\n{0x02F93C, 0x02F93C, 0x024FB8},\n{0x02F93D, 0x02F93D, 0x025044},\n{0x02F93E, 0x02F93E, 0x003FFC},\n{0x02F93F, 0x02F93F, 0x004008},\n{0x02F940, 0x02F940, 0x0076F4},\n{0x02F941, 0x02F941, 0x0250F3},\n{0x02F942, 0x02F942, 0x0250F2},\n{0x02F943, 0x02F943, 0x025119},\n{0x02F944, 0x02F944, 0x025133},\n{0x02F945, 0x02F945, 0x00771E},\n{0x02F946, 0x02F947, 0x00771F},\n{0x02F948, 0x02F948, 0x00774A},\n{0x02F949, 0x02F949, 0x004039},\n{0x02F94A, 0x02F94A, 0x00778B},\n{0x02F94B, 0x02F94B, 0x004046},\n{0x02F94C, 0x02F94C, 0x004096},\n{0x02F94D, 0x02F94D, 0x02541D},\n{0x02F94E, 0x02F94E, 0x00784E},\n{0x02F94F, 0x02F94F, 0x00788C},\n{0x02F950, 0x02F950, 0x0078CC},\n{0x02F951, 0x02F951, 0x0040E3},\n{0x02F952, 0x02F952, 0x025626},\n{0x02F953, 0x02F953, 0x007956},\n{0x02F954, 0x02F954, 0x02569A},\n{0x02F955, 0x02F955, 0x0256C5},\n{0x02F956, 0x02F956, 0x00798F},\n{0x02F957, 0x02F957, 0x0079EB},\n{0x02F958, 0x02F958, 0x00412F},\n{0x02F959, 0x02F959, 0x007A40},\n{0x02F95A, 0x02F95A, 0x007A4A},\n{0x02F95B, 0x02F95B, 0x007A4F},\n{0x02F95C, 0x02F95C, 0x02597C},\n{0x02F95D, 0x02F95E, 0x025AA7},\n{0x02F95F, 0x02F95F, 0x007AEE},\n{0x02F960, 0x02F960, 0x004202},\n{0x02F961, 0x02F961, 0x025BAB},\n{0x02F962, 0x02F962, 0x007BC6},\n{0x02F963, 0x02F963, 0x007BC9},\n{0x02F964, 0x02F964, 0x004227},\n{0x02F965, 0x02F965, 0x025C80},\n{0x02F966, 0x02F966, 0x007CD2},\n{0x02F967, 0x02F967, 0x0042A0},\n{0x02F968, 0x02F968, 0x007CE8},\n{0x02F969, 0x02F969, 0x007CE3},\n{0x02F96A, 0x02F96A, 0x007D00},\n{0x02F96B, 0x02F96B, 0x025F86},\n{0x02F96C, 0x02F96C, 0x007D63},\n{0x02F96D, 0x02F96D, 0x004301},\n{0x02F96E, 0x02F96E, 0x007DC7},\n{0x02F96F, 0x02F96F, 0x007E02},\n{0x02F970, 0x02F970, 0x007E45},\n{0x02F971, 0x02F971, 0x004334},\n{0x02F972, 0x02F972, 0x026228},\n{0x02F973, 0x02F973, 0x026247},\n{0x02F974, 0x02F974, 0x004359},\n{0x02F975, 0x02F975, 0x0262D9},\n{0x02F976, 0x02F976, 0x007F7A},\n{0x02F977, 0x02F977, 0x02633E},\n{0x02F978, 0x02F978, 0x007F95},\n{0x02F979, 0x02F979, 0x007FFA},\n{0x02F97A, 0x02F97A, 0x008005},\n{0x02F97B, 0x02F97B, 0x0264DA},\n{0x02F97C, 0x02F97C, 0x026523},\n{0x02F97D, 0x02F97D, 0x008060},\n{0x02F97E, 0x02F97E, 0x0265A8},\n{0x02F97F, 0x02F97F, 0x008070},\n{0x02F980, 0x02F980, 0x02335F},\n{0x02F981, 0x02F981, 0x0043D5},\n{0x02F982, 0x02F982, 0x0080B2},\n{0x02F983, 0x02F983, 0x008103},\n{0x02F984, 0x02F984, 0x00440B},\n{0x02F985, 0x02F985, 0x00813E},\n{0x02F986, 0x02F986, 0x005AB5},\n{0x02F987, 0x02F987, 0x0267A7},\n{0x02F988, 0x02F988, 0x0267B5},\n{0x02F989, 0x02F989, 0x023393},\n{0x02F98A, 0x02F98A, 0x02339C},\n{0x02F98B, 0x02F98B, 0x008201},\n{0x02F98C, 0x02F98C, 0x008204},\n{0x02F98D, 0x02F98D, 0x008F9E},\n{0x02F98E, 0x02F98E, 0x00446B},\n{0x02F98F, 0x02F98F, 0x008291},\n{0x02F990, 0x02F990, 0x00828B},\n{0x02F991, 0x02F991, 0x00829D},\n{0x02F992, 0x02F992, 0x0052B3},\n{0x02F993, 0x02F993, 0x0082B1},\n{0x02F994, 0x02F994, 0x0082B3},\n{0x02F995, 0x02F995, 0x0082BD},\n{0x02F996, 0x02F996, 0x0082E6},\n{0x02F997, 0x02F997, 0x026B3C},\n{0x02F998, 0x02F998, 0x0082E5},\n{0x02F999, 0x02F999, 0x00831D},\n{0x02F99A, 0x02F99A, 0x008363},\n{0x02F99B, 0x02F99B, 0x0083AD},\n{0x02F99C, 0x02F99C, 0x008323},\n{0x02F99D, 0x02F99D, 0x0083BD},\n{0x02F99E, 0x02F99E, 0x0083E7},\n{0x02F99F, 0x02F99F, 0x008457},\n{0x02F9A0, 0x02F9A0, 0x008353},\n{0x02F9A1, 0x02F9A1, 0x0083CA},\n{0x02F9A2, 0x02F9A2, 0x0083CC},\n{0x02F9A3, 0x02F9A3, 0x0083DC},\n{0x02F9A4, 0x02F9A4, 0x026C36},\n{0x02F9A5, 0x02F9A5, 0x026D6B},\n{0x02F9A6, 0x02F9A6, 0x026CD5},\n{0x02F9A7, 0x02F9A7, 0x00452B},\n{0x02F9A8, 0x02F9A8, 0x0084F1},\n{0x02F9A9, 0x02F9A9, 0x0084F3},\n{0x02F9AA, 0x02F9AA, 0x008516},\n{0x02F9AB, 0x02F9AB, 0x0273CA},\n{0x02F9AC, 0x02F9AC, 0x008564},\n{0x02F9AD, 0x02F9AD, 0x026F2C},\n{0x02F9AE, 0x02F9AE, 0x00455D},\n{0x02F9AF, 0x02F9AF, 0x004561},\n{0x02F9B0, 0x02F9B0, 0x026FB1},\n{0x02F9B1, 0x02F9B1, 0x0270D2},\n{0x02F9B2, 0x02F9B2, 0x00456B},\n{0x02F9B3, 0x02F9B3, 0x008650},\n{0x02F9B4, 0x02F9B4, 0x00865C},\n{0x02F9B5, 0x02F9B5, 0x008667},\n{0x02F9B6, 0x02F9B6, 0x008669},\n{0x02F9B7, 0x02F9B7, 0x0086A9},\n{0x02F9B8, 0x02F9B8, 0x008688},\n{0x02F9B9, 0x02F9B9, 0x00870E},\n{0x02F9BA, 0x02F9BA, 0x0086E2},\n{0x02F9BB, 0x02F9BB, 0x008779},\n{0x02F9BC, 0x02F9BC, 0x008728},\n{0x02F9BD, 0x02F9BD, 0x00876B},\n{0x02F9BE, 0x02F9BE, 0x008786},\n{0x02F9BF, 0x02F9BF, 0x0045D7},\n{0x02F9C0, 0x02F9C0, 0x0087E1},\n{0x02F9C1, 0x02F9C1, 0x008801},\n{0x02F9C2, 0x02F9C2, 0x0045F9},\n{0x02F9C3, 0x02F9C3, 0x008860},\n{0x02F9C4, 0x02F9C4, 0x008863},\n{0x02F9C5, 0x02F9C5, 0x027667},\n{0x02F9C6, 0x02F9C6, 0x0088D7},\n{0x02F9C7, 0x02F9C7, 0x0088DE},\n{0x02F9C8, 0x02F9C8, 0x004635},\n{0x02F9C9, 0x02F9C9, 0x0088FA},\n{0x02F9CA, 0x02F9CA, 0x0034BB},\n{0x02F9CB, 0x02F9CB, 0x0278AE},\n{0x02F9CC, 0x02F9CC, 0x027966},\n{0x02F9CD, 0x02F9CD, 0x0046BE},\n{0x02F9CE, 0x02F9CE, 0x0046C7},\n{0x02F9CF, 0x02F9CF, 0x008AA0},\n{0x02F9D0, 0x02F9D0, 0x008AED},\n{0x02F9D1, 0x02F9D1, 0x008B8A},\n{0x02F9D2, 0x02F9D2, 0x008C55},\n{0x02F9D3, 0x02F9D3, 0x027CA8},\n{0x02F9D4, 0x02F9D4, 0x008CAB},\n{0x02F9D5, 0x02F9D5, 0x008CC1},\n{0x02F9D6, 0x02F9D6, 0x008D1B},\n{0x02F9D7, 0x02F9D7, 0x008D77},\n{0x02F9D8, 0x02F9D8, 0x027F2F},\n{0x02F9D9, 0x02F9D9, 0x020804},\n{0x02F9DA, 0x02F9DA, 0x008DCB},\n{0x02F9DB, 0x02F9DB, 0x008DBC},\n{0x02F9DC, 0x02F9DC, 0x008DF0},\n{0x02F9DD, 0x02F9DD, 0x0208DE},\n{0x02F9DE, 0x02F9DE, 0x008ED4},\n{0x02F9DF, 0x02F9DF, 0x008F38},\n{0x02F9E0, 0x02F9E0, 0x0285D2},\n{0x02F9E1, 0x02F9E1, 0x0285ED},\n{0x02F9E2, 0x02F9E2, 0x009094},\n{0x02F9E3, 0x02F9E3, 0x0090F1},\n{0x02F9E4, 0x02F9E4, 0x009111},\n{0x02F9E5, 0x02F9E5, 0x02872E},\n{0x02F9E6, 0x02F9E6, 0x00911B},\n{0x02F9E7, 0x02F9E7, 0x009238},\n{0x02F9E8, 0x02F9E8, 0x0092D7},\n{0x02F9E9, 0x02F9E9, 0x0092D8},\n{0x02F9EA, 0x02F9EA, 0x00927C},\n{0x02F9EB, 0x02F9EB, 0x0093F9},\n{0x02F9EC, 0x02F9EC, 0x009415},\n{0x02F9ED, 0x02F9ED, 0x028BFA},\n{0x02F9EE, 0x02F9EE, 0x00958B},\n{0x02F9EF, 0x02F9EF, 0x004995},\n{0x02F9F0, 0x02F9F0, 0x0095B7},\n{0x02F9F1, 0x02F9F1, 0x028D77},\n{0x02F9F2, 0x02F9F2, 0x0049E6},\n{0x02F9F3, 0x02F9F3, 0x0096C3},\n{0x02F9F4, 0x02F9F4, 0x005DB2},\n{0x02F9F5, 0x02F9F5, 0x009723},\n{0x02F9F6, 0x02F9F6, 0x029145},\n{0x02F9F7, 0x02F9F7, 0x02921A},\n{0x02F9F8, 0x02F9F8, 0x004A6E},\n{0x02F9F9, 0x02F9F9, 0x004A76},\n{0x02F9FA, 0x02F9FA, 0x0097E0},\n{0x02F9FB, 0x02F9FB, 0x02940A},\n{0x02F9FC, 0x02F9FC, 0x004AB2},\n{0x02F9FD, 0x02F9FD, 0x029496},\n{0x02F9FE, 0x02F9FF, 0x00980B},\n{0x02FA00, 0x02FA00, 0x009829},\n{0x02FA01, 0x02FA01, 0x0295B6},\n{0x02FA02, 0x02FA02, 0x0098E2},\n{0x02FA03, 0x02FA03, 0x004B33},\n{0x02FA04, 0x02FA04, 0x009929},\n{0x02FA05, 0x02FA05, 0x0099A7},\n{0x02FA06, 0x02FA06, 0x0099C2},\n{0x02FA07, 0x02FA07, 0x0099FE},\n{0x02FA08, 0x02FA08, 0x004BCE},\n{0x02FA09, 0x02FA09, 0x029B30},\n{0x02FA0A, 0x02FA0A, 0x009B12},\n{0x02FA0B, 0x02FA0B, 0x009C40},\n{0x02FA0C, 0x02FA0C, 0x009CFD},\n{0x02FA0D, 0x02FA0D, 0x004CCE},\n{0x02FA0E, 0x02FA0E, 0x004CED},\n{0x02FA0F, 0x02FA0F, 0x009D67},\n{0x02FA10, 0x02FA10, 0x02A0CE},\n{0x02FA11, 0x02FA11, 0x004CF8},\n{0x02FA12, 0x02FA12, 0x02A105},\n{0x02FA13, 0x02FA13, 0x02A20E},\n{0x02FA14, 0x02FA14, 0x02A291},\n{0x02FA15, 0x02FA15, 0x009EBB},\n{0x02FA16, 0x02FA16, 0x004D56},\n{0x02FA17, 0x02FA17, 0x009EF9},\n{0x02FA18, 0x02FA18, 0x009EFE},\n{0x02FA19, 0x02FA19, 0x009F05},\n{0x02FA1A, 0x02FA1A, 0x009F0F},\n{0x02FA1B, 0x02FA1B, 0x009F16},\n{0x02FA1C, 0x02FA1C, 0x009F3B},\n{0x02FA1D, 0x02FA1D, 0x02A600},\n};\n"
  },
  {
    "path": "examples/talk-llama/unicode-data.h",
    "content": "#pragma once\n\n#include <cstdint>\n#include <vector>\n#include <unordered_map>\n#include <unordered_set>\n\nstruct range_nfd {\n    uint32_t first;\n    uint32_t last;\n    uint32_t nfd;\n};\n\nstatic const uint32_t MAX_CODEPOINTS = 0x110000;\n\nextern const std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;\nextern const std::unordered_set<uint32_t> unicode_set_whitespace;\nextern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_lowercase;\nextern const std::initializer_list<std::pair<uint32_t, uint32_t>> unicode_map_uppercase;\nextern const std::initializer_list<range_nfd> unicode_ranges_nfd;\n"
  },
  {
    "path": "examples/talk-llama/unicode.cpp",
    "content": "#include \"unicode.h\"\n#include \"unicode-data.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cstddef>\n#include <cstdint>\n#include <map>\n#include <regex>\n#include <stdexcept>\n#include <string>\n#include <unordered_map>\n#include <utility>\n#include <vector>\n\nsize_t unicode_len_utf8(char src) {\n    const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };\n    uint8_t highbits = static_cast<uint8_t>(src) >> 4;\n    return lookup[highbits];\n}\n\nstatic std::string unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {\n    std::string result;\n    for (size_t i = 0; i < cps.size(); ++i) {\n        result.append(unicode_cpt_to_utf8(cps[i]));\n    }\n    return result;\n}\n\nuint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {\n    assert(offset < utf8.size());\n    if (!(utf8[offset + 0] & 0x80)) {\n        auto result = utf8[offset + 0];\n        offset += 1;\n        return result;\n    }\n    if (!(utf8[offset + 0] & 0x40)) {\n        throw std::invalid_argument(\"invalid character\");\n    }\n    if (!(utf8[offset + 0] & 0x20)) {\n        if (offset + 1 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80)) {\n            throw std::invalid_argument(\"invalid character\");\n        }\n        auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);\n        offset += 2;\n        return result;\n    }\n    if (!(utf8[offset + 0] & 0x10)) {\n        if (offset + 2 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80)) {\n            throw std::invalid_argument(\"invalid character\");\n        }\n        auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);\n        offset += 3;\n        return result;\n    }\n    if (!(utf8[offset + 0] & 0x08)) {\n        if (offset + 3 >= utf8.size() || ! ((utf8[offset + 1] & 0xc0) == 0x80) || ! ((utf8[offset + 2] & 0xc0) == 0x80) || !((utf8[offset + 3] & 0xc0) == 0x80)) {\n            throw std::invalid_argument(\"invalid character\");\n        }\n        auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);\n        offset += 4;\n        return result;\n    }\n    throw std::invalid_argument(\"failed to convert utf8 to codepoint\");\n}\n\n//static std::vector<uint16_t> unicode_cpt_to_utf16(uint32_t cpt) {\n//    std::vector<uint16_t> result;\n//    if (/* 0x0000 <= cpt && */ cpt <= 0xffff) {\n//        result.emplace_back(cpt);\n//        return result;\n//    }\n//    if (0x10000 <= cpt && cpt <= 0x10ffff) {\n//        result.emplace_back(0xd800 | ((cpt - 0x10000) >> 10));\n//        result.emplace_back(0xdc00 | ((cpt - 0x10000) & 0x03ff));\n//        return result;\n//    }\n//    throw std::invalid_argument(\"failed to convert codepoint to utf16\");\n//}\n\n//static std::vector<uint16_t> unicode_cpts_to_utf16(const std::vector<uint32_t> & cps) {\n//    std::vector<uint16_t> result;\n//    for (size_t i = 0; i < cps.size(); ++i) {\n//        auto temp = unicode_cpt_to_utf16(cps[i]);\n//        result.insert(result.end(), temp.begin(), temp.end());\n//    }\n//    return result;\n//}\n\n//static uint32_t unicode_cpt_from_utf16(const std::vector<uint16_t> & utf16, size_t & offset) {\n//    assert(offset < utf16.size());\n//    if (((utf16[0] >> 10) << 10) != 0xd800) {\n//        auto result = utf16[offset + 0];\n//        offset += 1;\n//        return result;\n//    }\n//\n//    if (offset + 1 >= utf16.size() || !((utf16[1] & 0xdc00) == 0xdc00)) {\n//        throw std::invalid_argument(\"invalid character\");\n//    }\n//\n//    auto result = 0x10000 + (((utf16[0] & 0x03ff) << 10) | (utf16[1] & 0x03ff));\n//    offset += 2;\n//    return result;\n//}\n\n//static std::vector<uint32_t> unicode_cpts_from_utf16(const std::vector<uint16_t> & utf16) {\n//    std::vector<uint32_t> result;\n//    size_t offset = 0;\n//    while (offset < utf16.size()) {\n//        result.push_back(unicode_cpt_from_utf16(utf16, offset));\n//    }\n//    return result;\n//}\n\nstatic std::vector<unicode_cpt_flags> unicode_cpt_flags_array() {\n    std::vector<unicode_cpt_flags> cpt_flags(MAX_CODEPOINTS, unicode_cpt_flags::UNDEFINED);\n\n    assert (unicode_ranges_flags.begin()[0].first == 0);\n    assert (unicode_ranges_flags.begin()[unicode_ranges_flags.size()-1].first == MAX_CODEPOINTS);\n    for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {\n        const auto range_ini = unicode_ranges_flags.begin()[i-1];  // codepoint_ini, flags\n        const auto range_end = unicode_ranges_flags.begin()[i];    // codepoint_end, flags\n        for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {\n            cpt_flags[cpt] = range_ini.second;\n        }\n    }\n\n    for (auto cpt : unicode_set_whitespace) {\n        cpt_flags[cpt].is_whitespace = true;\n    }\n\n    for (auto p : unicode_map_lowercase) {\n        cpt_flags[p.second].is_lowercase = true;\n    }\n\n    for (auto p : unicode_map_uppercase) {\n        cpt_flags[p.second].is_uppercase = true;\n    }\n\n    for (auto &range : unicode_ranges_nfd) {  // start, last, nfd\n        cpt_flags[range.nfd].is_nfd = true;\n    }\n\n    return cpt_flags;\n}\n\nstatic std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {\n    std::unordered_map<uint8_t, std::string> map;\n    for (int ch = 0x21; ch <= 0x7E; ++ch) {  // u'!' to u'~'\n        assert(0 <= ch && ch < 256);\n        map[ch] = unicode_cpt_to_utf8(ch);\n    }\n    for (int ch = 0xA1; ch <= 0xAC; ++ch) {  // u'¡' to u'¬'\n        assert(0 <= ch && ch < 256);\n        map[ch] = unicode_cpt_to_utf8(ch);\n    }\n    for (int ch = 0xAE; ch <= 0xFF; ++ch) {  // u'®' to u'ÿ'\n        assert(0 <= ch && ch < 256);\n        map[ch] = unicode_cpt_to_utf8(ch);\n    }\n    auto n = 0;\n    for (int ch = 0; ch < 256; ++ch) {\n        if (map.find(ch) == map.end()) {\n            map[ch] = unicode_cpt_to_utf8(256 + n);\n            ++n;\n        }\n    }\n    return map;\n}\n\nstatic std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {\n    std::unordered_map<std::string, uint8_t> map;\n    for (int ch = 0x21; ch <= 0x7E; ++ch) {  // u'!' to u'~'\n        assert(0 <= ch && ch < 256);\n        map[unicode_cpt_to_utf8(ch)] = ch;\n    }\n    for (int ch = 0xA1; ch <= 0xAC; ++ch) {  // u'¡' to u'¬'\n        assert(0 <= ch && ch < 256);\n        map[unicode_cpt_to_utf8(ch)] = ch;\n    }\n    for (int ch = 0xAE; ch <= 0xFF; ++ch) {  // u'®' to u'ÿ'\n        assert(0 <= ch && ch < 256);\n        map[unicode_cpt_to_utf8(ch)] = ch;\n    }\n    auto n = 0;\n    for (int ch = 0; ch < 256; ++ch) {\n        if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) {\n            map[unicode_cpt_to_utf8(256 + n)] = ch;\n            ++n;\n        }\n    }\n    return map;\n}\n\nstatic std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {\n    std::vector<std::string> bpe_encoded_words;\n    for (const auto & word : bpe_words) {\n        std::string text_utf;\n        auto utf_word =  unicode_cpts_from_utf8(word);\n        for (size_t i = 0; i < utf_word.size(); ++i) {\n            text_utf += unicode_cpt_to_utf8(utf_word[i]);\n        }\n\n        std::string encoded_token;\n        for (char & c : text_utf) {\n            encoded_token += unicode_byte_to_utf8(c);\n        }\n        bpe_encoded_words.emplace_back(encoded_token);\n    }\n    return bpe_encoded_words;\n}\n\n// GPT2 system regex:  's|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+\nstatic std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) {\n    std::vector<size_t> bpe_offsets; // store the offset of each word\n    bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n\n    size_t start = 0;\n    for (auto offset : offsets) {\n        const size_t offset_ini = start;\n        const size_t offset_end = start + offset;\n        assert(offset_end <= cpts.size());\n        start = offset_end;\n\n        static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;\n        auto _get_cpt = [&] (const size_t pos) -> uint32_t {\n            return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;\n        };\n\n        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {\n            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};\n        };\n\n        size_t _prev_end = offset_ini;\n        auto _add_token = [&] (const size_t end) -> size_t {\n            assert(_prev_end <= end && end <= offset_end);\n            size_t len = end - _prev_end;\n            if (len > 0) {\n                bpe_offsets.push_back(len);\n            }\n            _prev_end = end;\n            //if (len > 0) {\n            //    std::string s = \"\";\n            //    for(size_t p = end-len; p < end; p++)\n            //        s += unicode_cpt_to_utf8(cpts[p]);\n            //    printf(\">>> '%s'\\n\", s.c_str());\n            //}\n            return len;\n        };\n\n        for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {\n            const uint32_t cpt = _get_cpt(pos);\n            const auto flags = _get_flags(pos);\n\n            // regex: 's|'t|'re|'ve|'m|'ll|'d\n            if (cpt == '\\'' && pos+1 < offset_end) {\n                uint32_t cpt_next = _get_cpt(pos+1);\n                if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {\n                    pos += _add_token(pos+2);\n                    continue;\n                }\n                if (pos+2 < offset_end) {\n                    uint32_t cpt_next_next = _get_cpt(pos+2);\n                    if ((cpt_next == 'r' && cpt_next_next == 'e') ||\n                        (cpt_next == 'v' && cpt_next_next == 'e') ||\n                        (cpt_next == 'l' && cpt_next_next == 'l')) {\n                        pos += _add_token(pos+3);\n                        continue;\n                    }\n                }\n            }\n\n            auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);\n            // regex: <space>?\\p{L}+\n            if (flags2.is_letter) {\n                pos += (cpt == ' ');\n                while (flags2.is_letter) {\n                    flags2 = _get_flags(++pos);\n                }\n                _add_token(pos);\n                continue;\n            }\n            // regex: <space>?\\p{N}+\n            if (flags2.is_number) {\n                pos += (cpt == ' ');\n                while (flags2.is_number) {\n                    flags2 = _get_flags(++pos);\n                }\n                _add_token(pos);\n                continue;\n            }\n            // regex: <space>?[^\\s\\p{L}\\p{N}]+\n            if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {\n                pos += (cpt == ' ');\n                while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {\n                    flags2 = _get_flags(++pos);\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            size_t num_whitespaces = 0;\n            while (_get_flags(pos+num_whitespaces).is_whitespace) {\n                num_whitespaces++;\n            }\n\n            // regex: \\s+(?!\\S)\n            if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {\n                pos += num_whitespaces - 1;\n                _add_token(pos);\n                continue;\n            }\n\n            // regex: \\s+\n            if (num_whitespaces > 0) {\n                pos += num_whitespaces;\n                _add_token(pos);\n                continue;\n            }\n\n            // no matches\n            _add_token(++pos);\n        }\n    }\n\n    return bpe_offsets;\n}\n\n// LLAMA3 system regex: \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+\"\nstatic std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) {\n    std::vector<size_t> bpe_offsets; // store the offset of each word\n    bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n\n    size_t start = 0;\n    for (auto offset : offsets) {\n        const size_t offset_ini = start;\n        const size_t offset_end = start + offset;\n        assert(offset_end <= cpts.size());\n        start = offset_end;\n\n        static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;\n        auto _get_cpt = [&] (const size_t pos) -> uint32_t {\n            return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;\n        };\n\n        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {\n            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};\n        };\n\n        size_t _prev_end = offset_ini;\n        auto _add_token = [&] (const size_t end) -> size_t {\n            assert(_prev_end <= end && end <= offset_end);\n            size_t len = end - _prev_end;\n            if (len > 0) {\n                bpe_offsets.push_back(len);\n            }\n            _prev_end = end;\n            //if (len > 0) {\n            //    std::string s = \"\";\n            //    for(size_t p = end-len; p < end; p++)\n            //        s += unicode_cpt_to_utf8(cpts[p]);\n            //    printf(\">>> '%s'\\n\", s.c_str());\n            //}\n            return len;\n        };\n\n        for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {\n            const uint32_t cpt = _get_cpt(pos);\n            const auto flags = _get_flags(pos);\n\n            // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive\n            if (cpt == '\\'' && pos+1 < offset_end) {\n                uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1));\n                if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {\n                    pos += _add_token(pos+2);\n                    continue;\n                }\n                if (pos+2 < offset_end) {\n                    uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));\n                    if ((cpt_next == 'r' && cpt_next_next == 'e') ||\n                        (cpt_next == 'v' && cpt_next_next == 'e') ||\n                        (cpt_next == 'l' && cpt_next_next == 'l')) {\n                        pos += _add_token(pos+3);\n                        continue;\n                    }\n                }\n            }\n\n            // regex: [^\\r\\n\\p{L}\\p{N}]?\\p{L}+\n            if (!(cpt == '\\r' || cpt == '\\n' || flags.is_number)) {\n                if (flags.is_letter || _get_flags(pos+1).is_letter) {  // one or more letters\n                    pos++;\n                    while (_get_flags(pos).is_letter) {\n                        pos++;\n                    }\n                    _add_token(pos);\n                    continue;\n                }\n            }\n\n            // regex: \\p{N}{1,3}\n            if (flags.is_number) {\n                size_t ini = pos;\n                while (_get_flags(pos).is_number) {\n                    if (++pos - ini >= 3 ) {\n                        _add_token(pos);\n                        ini = pos;\n                    }\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            // regex: <space>?[^\\s\\p{L}\\p{N}]+[\\r\\n]*\n            auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags);\n            if (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags.as_uint()) {\n                pos += (cpt == ' ');\n                while (!(flags2.is_whitespace | flags2.is_letter | flags2.is_number) && flags2.as_uint()) {\n                    flags2 = _get_flags(++pos);\n                }\n                uint32_t cpt2 = _get_cpt(pos);\n                while (cpt2 == '\\r' || cpt2 == '\\n') {\n                    cpt2 = _get_cpt(++pos);\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            size_t num_whitespaces = 0;\n            size_t last_end_r_or_n = 0;\n            while (_get_flags(pos+num_whitespaces).is_whitespace) {\n                uint32_t cpt2 = _get_cpt(pos+num_whitespaces);\n                if (cpt2 == '\\r' || cpt2 == '\\n') {\n                    last_end_r_or_n = pos + num_whitespaces + 1;\n                }\n                num_whitespaces++;\n            }\n\n            // regex: \\s*[\\r\\n]+\n            if (last_end_r_or_n > 0) {\n                pos = last_end_r_or_n;\n                _add_token(pos);\n                continue;\n            }\n\n            // regex: \\s+(?!\\S)\n            if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != OUT_OF_RANGE) {\n                pos += num_whitespaces - 1;\n                _add_token(pos);\n                continue;\n            }\n\n            // regex: \\s+\n            if (num_whitespaces > 0) {\n                pos += num_whitespaces;\n                _add_token(pos);\n                continue;\n            }\n\n            // no matches\n            _add_token(++pos);\n        }\n    }\n\n    return bpe_offsets;\n}\n\ntemplate <typename CharT>\nstatic std::vector<size_t> unicode_regex_split_stl(const std::basic_string<CharT> & text, const std::basic_string<CharT> & regex, const std::vector<size_t> & offsets) {\n    using BidirIt = typename std::basic_string<CharT>::const_iterator;\n#ifdef _MSC_VER\n    // Bypass bug in MSVC: https://github.com/ggml-org/llama.cpp/issues/17830\n    constexpr auto regex_flags = std::regex_constants::ECMAScript;\n#else\n    constexpr auto regex_flags = std::regex_constants::optimize | std::regex_constants::nosubs;\n#endif\n    std::basic_regex<CharT> expr(regex, regex_flags);\n    std::vector<size_t> bpe_offsets; // store the offset of each word\n    bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size\n    size_t start = 0;\n    for (auto offset : offsets) {\n        std::regex_iterator<BidirIt> it(text.begin() + start, text.begin() + start + offset, expr);\n        std::regex_iterator<BidirIt> end;\n\n        int64_t start_idx = 0;\n        while (it != end) {\n            std::match_results<BidirIt> match = *it;\n            if (match.position() > start_idx) {\n                bpe_offsets.emplace_back(match.position() - start_idx);\n            }\n            bpe_offsets.emplace_back(match.length());\n            start_idx = match.position() + match.length();\n            ++it;\n        }\n\n        if (start_idx < (int64_t) offset) {\n            bpe_offsets.emplace_back(offset - start_idx);\n        }\n        start += offset;\n    }\n\n    return bpe_offsets;\n}\n\n// K2 system regex patterns (from tokenization_kimi.py):\n// [\\p{Han}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+\nstatic std::vector<size_t> unicode_regex_split_custom_kimi_k2(const std::string & text, const std::vector<size_t> & offsets) {\n    std::vector<size_t> bpe_offsets;\n    bpe_offsets.reserve(offsets.size());\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n\n    size_t start = 0;\n    for (auto offset : offsets) {\n        const size_t offset_ini = start;\n        const size_t offset_end = start + offset;\n        assert(offset_end <= cpts.size());\n        start = offset_end;\n\n        static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF;\n        auto _get_cpt = [&] (const size_t pos) -> uint32_t {\n            return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE;\n        };\n\n        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {\n            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};\n        };\n\n        size_t _prev_end = offset_ini;\n        auto _add_token = [&] (const size_t end) -> size_t {\n            assert(_prev_end <= end && end <= offset_end);\n            size_t len = end - _prev_end;\n            if (len > 0) {\n                bpe_offsets.push_back(len);\n            }\n            _prev_end = end;\n            return len;\n        };\n\n        for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {\n            const uint32_t cpt = _get_cpt(pos);\n            const auto flags = _get_flags(pos);\n\n            // Pattern 1: [\\p{Han}]+ (Chinese characters)\n            if (unicode_cpt_is_han(cpt)) {\n                while (unicode_cpt_is_han(_get_cpt(pos))) {\n                    pos++;\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            // Pattern 2 & 3: Letter words excluding Han characters with optional contractions\n            // [^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+(?:'s|'t|'re|'ve|'m|'ll|'d)?\n            // [^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*(?:'s|'t|'re|'ve|'m|'ll|'d)?\n            // Check if current char is a letter OR if current char could be a leading char and next char is a letter\n            bool is_letter_pattern = (flags.is_letter && !unicode_cpt_is_han(cpt)) ||\n                                     (!(cpt == '\\r' || cpt == '\\n' || flags.is_letter || flags.is_number) &&\n                                      _get_flags(pos + 1).is_letter && !unicode_cpt_is_han(_get_cpt(pos + 1)));\n\n            if (is_letter_pattern) {\n                // Handle optional leading non-letter/non-number character\n                bool has_leading_char = false;\n                if (!(cpt == '\\r' || cpt == '\\n' || flags.is_letter || flags.is_number)) {\n                    has_leading_char = true;\n                    pos++;\n                }\n\n                // Match letter sequence (excluding Han characters)\n                bool has_letters = false;\n                while (_get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos))) {\n                    has_letters = true;\n                    pos++;\n                }\n\n                // Only proceed if we found letters (after potentially skipping leading char)\n                if (has_letters || (!has_leading_char && _get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos)))) {\n                    if (!has_letters) pos++; // consume the first letter if we didn't already\n\n                    // Continue consuming letters\n                    while (_get_flags(pos).is_letter && !unicode_cpt_is_han(_get_cpt(pos))) {\n                        pos++;\n                    }\n\n                    // Check for optional contractions (?:'s|'t|'re|'ve|'m|'ll|'d)\n                    if (_get_cpt(pos) == '\\'' && pos + 1 < offset_end) {\n                        uint32_t cpt_next = unicode_tolower(_get_cpt(pos + 1));\n                        if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {\n                            pos += 2;\n                        } else if (pos + 2 < offset_end) {\n                            uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos + 2));\n                            if ((cpt_next == 'r' && cpt_next_next == 'e') ||\n                                (cpt_next == 'v' && cpt_next_next == 'e') ||\n                                (cpt_next == 'l' && cpt_next_next == 'l')) {\n                                pos += 3;\n                            }\n                        }\n                    }\n\n                    _add_token(pos);\n                    continue;\n                } else if (has_leading_char) {\n                    // We consumed a leading char but found no letters, backtrack\n                    pos--;\n                }\n            }\n\n            // Pattern 4: \\p{N}{1,3} (numbers 1-3 digits)\n            if (flags.is_number) {\n                size_t ini = pos;\n                while (_get_flags(pos).is_number) {\n                    if (++pos - ini >= 3) {\n                        _add_token(pos);\n                        ini = pos;\n                    }\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            // Pattern 5:  ?[^\\s\\p{L}\\p{N}]+[\\r\\n]* (optional space + non-word chars + optional newlines)\n            auto flags2 = (cpt == ' ' ? _get_flags(pos + 1) : flags);\n            if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number) && flags2.as_uint()) {\n                pos += (cpt == ' ');\n                while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number) && flags2.as_uint()) {\n                    flags2 = _get_flags(++pos);\n                }\n                // Match optional [\\r\\n]*\n                uint32_t cpt2 = _get_cpt(pos);\n                while (cpt2 == '\\r' || cpt2 == '\\n') {\n                    cpt2 = _get_cpt(++pos);\n                }\n                _add_token(pos);\n                continue;\n            }\n\n            // Count whitespace characters\n            size_t num_whitespaces = 0;\n            size_t last_end_r_or_n = 0;\n            while (_get_flags(pos + num_whitespaces).is_whitespace) {\n                uint32_t cpt2 = _get_cpt(pos + num_whitespaces);\n                if (cpt2 == '\\r' || cpt2 == '\\n') {\n                    last_end_r_or_n = pos + num_whitespaces + 1;\n                }\n                num_whitespaces++;\n            }\n\n            // Pattern 6: \\s*[\\r\\n]+ (whitespace with newlines)\n            if (last_end_r_or_n > 0) {\n                pos = last_end_r_or_n;\n                _add_token(pos);\n                continue;\n            }\n\n            // Pattern 7: \\s+(?!\\S) (trailing whitespace)\n            if (num_whitespaces > 1 && _get_cpt(pos + num_whitespaces) != OUT_OF_RANGE) {\n                pos += num_whitespaces - 1;\n                _add_token(pos);\n                continue;\n            }\n\n            // Pattern 8: \\s+ (general whitespace)\n            if (num_whitespaces > 0) {\n                pos += num_whitespaces;\n                _add_token(pos);\n                continue;\n            }\n\n            // No matches - consume single character\n            _add_token(++pos);\n        }\n    }\n\n    return bpe_offsets;\n}\n\n// AFMOE digit handling: splits digits with leading 1-2 based on total length modulo 3\nstatic std::vector<size_t> unicode_regex_split_custom_afmoe(const std::string & text, const std::vector<size_t> & offsets) {\n    std::vector<size_t> bpe_offsets;\n    bpe_offsets.reserve(offsets.size());\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n\n    size_t start = 0;\n    for (auto offset : offsets) {\n        const size_t offset_ini = start;\n        const size_t offset_end = start + offset;\n        assert(offset_end <= cpts.size());\n        start = offset_end;\n\n        auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags {\n            return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{};\n        };\n\n        size_t _prev_end = offset_ini;\n        auto _add_token = [&] (const size_t end) -> size_t {\n            assert(_prev_end <= end && end <= offset_end);\n            size_t len = end - _prev_end;\n            if (len > 0) {\n                bpe_offsets.push_back(len);\n            }\n            _prev_end = end;\n            return len;\n        };\n\n        for (size_t pos = offset_ini; pos < offset_end; ) {\n            const auto flags = _get_flags(pos);\n\n            // Handle digit sequences with special splitting logic\n            if (flags.is_number) {\n                size_t digit_start = pos;\n                size_t digit_count = 0;\n\n                // Count consecutive digits\n                while (_get_flags(pos).is_number && pos < offset_end) {\n                    digit_count++;\n                    pos++;\n                }\n\n                // Split based on total length modulo 3\n                size_t remainder = digit_count % 3;\n                size_t current = digit_start;\n\n                // Emit leading 1-2 digits if needed\n                if (remainder > 0) {\n                    _add_token(current + remainder);\n                    current += remainder;\n                }\n\n                // Emit groups of 3\n                while (current < digit_start + digit_count) {\n                    _add_token(current + 3);\n                    current += 3;\n                }\n                continue;\n            }\n\n            // For non-digits, just move forward\n            pos++;\n        }\n\n        // Add any remaining content\n        if (_prev_end < offset_end) {\n            _add_token(offset_end);\n        }\n    }\n\n    return bpe_offsets;\n}\n\nstatic std::vector<size_t> unicode_regex_split_custom(const std::string & text, const std::string & regex_expr, const std::vector<size_t> & offsets) {\n    std::vector<size_t> bpe_offsets;\n\n    if (regex_expr == \"'s|'t|'re|'ve|'m|'ll|'d| ?\\\\p{L}+| ?\\\\p{N}+| ?[^\\\\s\\\\p{L}\\\\p{N}]+|\\\\s+(?!\\\\S)\") {\n        bpe_offsets = unicode_regex_split_custom_gpt2(text, offsets);\n    } else if (\n            regex_expr == \"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\" ||\n            regex_expr == \"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\\\r\\\\n\\\\p{L}\\\\p{N}]?\\\\p{L}+|\\\\p{N}{1,3}| ?[^\\\\s\\\\p{L}\\\\p{N}]+[\\\\r\\\\n]*|\\\\s*[\\\\r\\\\n]+|\\\\s+(?!\\\\S)|\\\\s+\") {\n\n        bpe_offsets = unicode_regex_split_custom_llama3(text, offsets);\n    } else if (regex_expr == \"\\\\p{Han}+\") {\n        // K2's first pattern - handle all K2 patterns together\n        bpe_offsets = unicode_regex_split_custom_kimi_k2(text, offsets);\n    } else if (regex_expr == \"\\\\p{AFMoE_digits}\") {\n        // AFMOE digit pattern - use custom implementation for proper splitting\n        bpe_offsets = unicode_regex_split_custom_afmoe(text, offsets);\n    } else if (regex_expr == \"\\\\d{1,3}(?=(?:\\\\d{3})*\\\\b)\") {\n        // tiny_aya digit grouping pattern from tokenizer.json:\n        //   {\"type\": \"Split\", \"pattern\": {\"Regex\": \"\\\\d{1,3}(?=(?:\\\\d{3})*\\\\b)\"}, \"behavior\": \"Isolated\"}\n        // Splits digits into groups of 3 from the right (e.g., 1234567 -> 1, 234, 567)\n        // TODO: Revisit this regex, in case there are any subtle tokenization differences with the original regex.\n        bpe_offsets = unicode_regex_split_custom_afmoe(text, offsets);\n    }\n\n    return bpe_offsets;\n}\n\n//\n// interface\n//\n\nstd::string unicode_cpt_to_utf8(uint32_t cpt) {\n    std::string result;\n\n    if (/* 0x00 <= cpt && */ cpt <= 0x7f) {\n        result.push_back(cpt);\n        return result;\n    }\n    if (0x80 <= cpt && cpt <= 0x7ff) {\n        result.push_back(0xc0 | ((cpt >> 6) & 0x1f));\n        result.push_back(0x80 | (cpt & 0x3f));\n        return result;\n    }\n    if (0x800 <= cpt && cpt <= 0xffff) {\n        result.push_back(0xe0 | ((cpt >> 12) & 0x0f));\n        result.push_back(0x80 | ((cpt >> 6) & 0x3f));\n        result.push_back(0x80 | (cpt & 0x3f));\n        return result;\n    }\n    if (0x10000 <= cpt && cpt <= 0x10ffff) {\n        result.push_back(0xf0 | ((cpt >> 18) & 0x07));\n        result.push_back(0x80 | ((cpt >> 12) & 0x3f));\n        result.push_back(0x80 | ((cpt >> 6) & 0x3f));\n        result.push_back(0x80 | (cpt & 0x3f));\n        return result;\n    }\n\n    throw std::invalid_argument(\"invalid codepoint\");\n}\n\nstd::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {\n    auto comp = [] (const uint32_t cpt, const range_nfd & range) {\n        return cpt < range.first;\n    };\n    std::vector<uint32_t> result(cpts.size());\n    for (size_t i = 0; i < cpts.size(); ++i) {\n        const uint32_t cpt = cpts[i];\n        auto it = std::upper_bound(unicode_ranges_nfd.begin(), unicode_ranges_nfd.end(), cpt, comp) - 1;\n        result[i] = (it->first <= cpt && cpt <= it->last) ? it->nfd : cpt;\n    }\n    return result;\n}\n\nstd::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {\n    std::vector<uint32_t> result;\n    result.reserve(utf8.size());\n    size_t offset = 0;\n    while (offset < utf8.size()) {\n        try {\n            result.push_back(unicode_cpt_from_utf8(utf8, offset));\n        }\n        catch (const std::invalid_argument & /*ex*/) {\n            // Silently ignore invalid UTF-8 input to avoid leaking the exception beyond llama_tokenize\n            ++offset;\n            result.emplace_back(0xFFFD); // replacement character\n        }\n    }\n    return result;\n}\n\nunicode_cpt_flags unicode_cpt_flags_from_cpt(const uint32_t cpt) {\n    static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED);\n    static const auto cpt_flags = unicode_cpt_flags_array();\n    return cpt < cpt_flags.size() ? cpt_flags[cpt] : undef;\n}\n\nunicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8) {\n    static const unicode_cpt_flags undef(unicode_cpt_flags::UNDEFINED);\n    if (utf8.empty()) {\n        return undef;  // undefined\n    }\n    size_t offset = 0;\n    return unicode_cpt_flags_from_cpt(unicode_cpt_from_utf8(utf8, offset));\n}\n\nstd::string unicode_byte_to_utf8(uint8_t byte) {\n    static std::unordered_map<uint8_t, std::string> map = unicode_byte_to_utf8_map();\n    return map.at(byte);\n}\n\nuint8_t unicode_utf8_to_byte(const std::string & utf8) {\n    static std::unordered_map<std::string, uint8_t> map = unicode_utf8_to_byte_map();\n    return map.at(utf8);\n}\n\nuint32_t unicode_tolower(uint32_t cpt) {\n    // binary search\n    auto it = std::lower_bound(unicode_map_lowercase.begin(), unicode_map_lowercase.end(), cpt,\n        [](const std::pair<uint32_t, uint32_t> & pair, uint32_t value) {\n            return pair.first < value;\n        });\n    if (it != unicode_map_lowercase.end() && it->first == cpt) {\n        return it->second;\n    }\n    return cpt;  // Return the original code point if no lowercase mapping is found\n}\n\nbool unicode_cpt_is_han(uint32_t cpt) {\n    // Han character ranges (Chinese/CJK characters)\n    // CJK Unified Ideographs (most common)\n    if (cpt >= 0x4E00 && cpt <= 0x9FFF) return true;\n\n    // CJK Extension A\n    if (cpt >= 0x3400 && cpt <= 0x4DBF) return true;\n\n    // CJK Extension B\n    if (cpt >= 0x20000 && cpt <= 0x2A6DF) return true;\n\n    // CJK Extension C\n    if (cpt >= 0x2A700 && cpt <= 0x2B73F) return true;\n\n    // CJK Extension D\n    if (cpt >= 0x2B740 && cpt <= 0x2B81F) return true;\n\n    // CJK Extension E\n    if (cpt >= 0x2B820 && cpt <= 0x2CEAF) return true;\n\n    // CJK Extension F\n    if (cpt >= 0x2CEB0 && cpt <= 0x2EBEF) return true;\n\n    // CJK Compatibility Ideographs\n    if (cpt >= 0xF900 && cpt <= 0xFAFF) return true;\n\n    // CJK Compatibility Ideographs Supplement\n    if (cpt >= 0x2F800 && cpt <= 0x2FA1F) return true;\n\n    return false;\n}\n\nstd::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs) {\n    // unicode categories\n    static const std::map<std::string, int> k_ucat_enum = {\n        { \"\\\\p{N}\", unicode_cpt_flags::NUMBER },\n        { \"\\\\p{L}\", unicode_cpt_flags::LETTER },\n        { \"\\\\p{P}\", unicode_cpt_flags::PUNCTUATION },\n        { \"\\\\p{M}\", unicode_cpt_flags::ACCENT_MARK },\n        { \"\\\\p{S}\", unicode_cpt_flags::SYMBOL },\n        { \"\\\\p{Lu}\", unicode_cpt_flags::LETTER }, // Uppercase letter\n        { \"\\\\p{Ll}\", unicode_cpt_flags::LETTER }, // Lowercase letter\n        { \"\\\\p{Lt}\", unicode_cpt_flags::LETTER }, // Titlecase letter\n        { \"\\\\p{Lm}\", unicode_cpt_flags::LETTER }, // Modifier letter\n        { \"\\\\p{Lo}\", unicode_cpt_flags::LETTER }, // Other letter\n    };\n\n    static const std::map<int, int> k_ucat_cpt = {\n        { unicode_cpt_flags::NUMBER,      0xD1 },\n        { unicode_cpt_flags::LETTER,      0xD2 },\n        { unicode_cpt_flags::PUNCTUATION, 0xD3 },\n        { unicode_cpt_flags::ACCENT_MARK, 0xD4 },\n        { unicode_cpt_flags::SYMBOL,      0xD5 },\n    };\n\n    static const std::map<int, std::string> k_ucat_map = {\n        { unicode_cpt_flags::NUMBER,      \"\\x30-\\x39\" }, // 0-9\n        { unicode_cpt_flags::LETTER,      \"\\x41-\\x5A\\x61-\\x7A\" }, // A-Za-z\n        { unicode_cpt_flags::PUNCTUATION, \"\\x21-\\x23\\x25-\\x2A\\x2C-\\x2F\\x3A-\\x3B\\x3F-\\x40\\\\\\x5B-\\\\\\x5D\\x5F\\\\\\x7B\\\\\\x7D\" }, // !-#%-*,-/:-;?-@\\[-\\]_\\{\\}\n        { unicode_cpt_flags::ACCENT_MARK, \"\" }, // no sub-128 codepoints\n        { unicode_cpt_flags::SYMBOL,      \"\\\\\\x24\\\\\\x2B\\x3C-\\x3E\\x5E\\x60\\\\\\x7C\" }, // $+<=>^`|\n    };\n\n    // compute collapsed codepoints only if needed by at least one regex\n    bool need_collapse = false;\n    for (const auto & regex_expr : regex_exprs) {\n        // search for unicode categories\n        for (const auto & ucat : k_ucat_enum) {\n            if (std::string::npos != regex_expr.find(ucat.first)) {\n                need_collapse = true;\n                break;\n            }\n        }\n    }\n\n    const auto cpts = unicode_cpts_from_utf8(text);\n\n    // generate a \"collapsed\" representation of the text, where all codepoints are replaced by a single byte\n    // ref: https://github.com/ggml-org/llama.cpp/pull/6920#issuecomment-2081479935\n    std::string text_collapsed;\n    if (need_collapse) {\n        // collapse all unicode categories\n        text_collapsed.resize(cpts.size());\n\n        for (size_t i = 0; i < cpts.size(); ++i) {\n            // keep single-byte codepoints as is\n            if (cpts[i] < 128) {\n                text_collapsed[i] = cpts[i];\n                continue;\n            }\n\n            const auto flags = unicode_cpt_flags_from_cpt(cpts[i]);\n\n            if (flags.is_whitespace) {\n                //NOTE: C++ std::regex \\s does not mach 0x85, Rust and Python regex does.\n                //text_collapsed[i] = (char) 0x85;  // <Next Line> as whitespace fallback\n                text_collapsed[i] = (char) 0x0B;    // <vertical tab> as whitespace fallback\n            } else if (k_ucat_cpt.find(flags.category_flag()) != k_ucat_cpt.end()) {\n                text_collapsed[i] = k_ucat_cpt.at(flags.category_flag());\n            } else {\n                text_collapsed[i] = (char) 0xD0; // fallback\n            }\n        }\n    }\n\n    std::vector<size_t> bpe_offsets = { cpts.size() };\n\n    for (const auto & regex_expr : regex_exprs) {\n        // first, see if we have an efficient custom regex implementation\n        auto tmp = unicode_regex_split_custom(text, regex_expr, bpe_offsets);\n\n        if (!tmp.empty()) {\n            bpe_offsets = std::move(tmp);\n            continue;\n        }\n\n        // fallback to general-purpose std::regex / std::wregex\n        try {\n            // if a unicode category is used in the regex, we use the collapsed text and replace the unicode category\n            // with the corresponding collapsed representation\n            bool use_collapsed = false;\n            for (const auto & ucat : k_ucat_enum) {\n                if (std::string::npos != regex_expr.find(ucat.first)) {\n                    use_collapsed = true;\n                    break;\n                }\n            }\n            const auto cpts_regex = unicode_cpts_from_utf8(regex_expr);\n\n            if (use_collapsed) {\n                // sanity-check that the original regex does not contain any non-ASCII characters\n                for (size_t i = 0; i < cpts_regex.size(); ++i) {\n                    if (cpts_regex[i] >= 128) {\n                        throw std::runtime_error(\"Regex includes both unicode categories and non-ASCII characters - not supported\");\n                    }\n                }\n\n                // generate a collapsed representation of the regex\n                std::string regex_expr_collapsed;\n\n                // track if we are inside [], because nested [] are not allowed\n                bool inside = false;\n                for (size_t i = 0; i < regex_expr.size(); ++i) {\n                    if (regex_expr[i] == '[' && (i == 0 || regex_expr[i - 1] != '\\\\')) {\n                        regex_expr_collapsed += '[';\n                        inside = true;\n                        continue;\n                    }\n\n                    if (inside && regex_expr[i] == ']' && regex_expr[i - 1] != '\\\\') {\n                        regex_expr_collapsed += ']';\n                        inside = false;\n                        continue;\n                    }\n\n                    // Match \\p{...} Unicode properties of varying lengths\n                    if (regex_expr[i + 0] == '\\\\' && i + 3 < regex_expr.size() &&\n                        regex_expr[i + 1] == 'p' &&\n                        regex_expr[i + 2] == '{') {\n                        // Find the closing brace\n                        size_t closing_brace = regex_expr.find('}', i + 3);\n                        if (closing_brace != std::string::npos && closing_brace <= i + 10) { // reasonable limit\n                            const std::string pat = regex_expr.substr(i, closing_brace - i + 1);\n                            if (k_ucat_enum.find(pat) != k_ucat_enum.end()) {\n                                if (!inside) {\n                                    regex_expr_collapsed += '[';\n                                }\n                                regex_expr_collapsed += k_ucat_cpt.at(k_ucat_enum.at(pat));\n                                regex_expr_collapsed += k_ucat_map.at(k_ucat_enum.at(pat));\n                                if (!inside) {\n                                    regex_expr_collapsed += ']';\n                                }\n                                i = closing_brace;\n                                continue;\n                            }\n                        }\n                    }\n\n                    regex_expr_collapsed += regex_expr[i];\n                }\n\n                //printf(\"text_collapsed: %s\\n\", text_collapsed.c_str());\n                //printf(\"regex_expr_collapsed: %s\\n\", regex_expr_collapsed.c_str());\n                bpe_offsets = unicode_regex_split_stl(text_collapsed, regex_expr_collapsed, bpe_offsets);\n            } else {\n                // no unicode category used, we can use std::wregex directly\n                std::wstring wregex_expr(cpts_regex.begin(), cpts_regex.end());\n\n                // std::wregex \\s does not mach non-ASCII whitespaces, using 0x0B as fallback\n                std::wstring wtext(cpts.begin(), cpts.end());\n                for (size_t i = 0; i < wtext.size(); ++i) {\n                    if (wtext[i] > 0x7F && unicode_cpt_flags_from_cpt(wtext[i]).is_whitespace) {\n                        wtext[i] = 0x0B;\n                    }\n                }\n\n                //printf(\"text: %s\\n\", text.c_str());\n                //printf(\"regex_expr: %s\\n\", regex_expr.c_str());\n                bpe_offsets = unicode_regex_split_stl(wtext, wregex_expr, bpe_offsets);\n            }\n        } catch (std::regex_error & e) {\n            fprintf(stderr, \"Failed to process regex: '%s'\\n\", regex_expr.c_str());\n            fprintf(stderr, \"Regex error: %s\\n\", e.what());\n            throw std::runtime_error(\"Failed to process regex\");\n        }\n    }\n\n    std::vector<std::string> bpe_words;\n    bpe_words.reserve(bpe_offsets.size()); // reserve memory for the approximate size\n\n    size_t start = 0;\n    for (size_t & offset : bpe_offsets) {\n        bpe_words.emplace_back();\n        for (size_t i = start; i < start + offset; ++i) {\n            bpe_words.back() += unicode_cpt_to_utf8(cpts[i]);\n        }\n        start += offset;\n    }\n\n    return unicode_byte_encoding_process(bpe_words);\n}\n"
  },
  {
    "path": "examples/talk-llama/unicode.h",
    "content": "#pragma once\n\n#include <cstdint>\n#include <string>\n#include <vector>\n\n// TODO: reimplement this structure in endian-independent way\nstruct unicode_cpt_flags {\n    enum {\n        UNDEFINED       = 0x0001,\n        NUMBER          = 0x0002,  // regex: \\p{N}\n        LETTER          = 0x0004,  // regex: \\p{L}\n        SEPARATOR       = 0x0008,  // regex: \\p{Z}\n        ACCENT_MARK     = 0x0010,  // regex: \\p{M}\n        PUNCTUATION     = 0x0020,  // regex: \\p{P}\n        SYMBOL          = 0x0040,  // regex: \\p{S}\n        CONTROL         = 0x0080,  // regex: \\p{C}\n        MASK_CATEGORIES = 0x00FF,\n        WHITESPACE      = 0x0100,\n        LOWERCASE       = 0x0200,\n        UPPERCASE       = 0x0400,\n        NFD             = 0x0800,\n    };\n\n    // codepoint type\n    uint16_t is_undefined   : 1;\n    uint16_t is_number      : 1;  // regex: \\p{N}\n    uint16_t is_letter      : 1;  // regex: \\p{L}\n    uint16_t is_separator   : 1;  // regex: \\p{Z}\n    uint16_t is_accent_mark : 1;  // regex: \\p{M}\n    uint16_t is_punctuation : 1;  // regex: \\p{P}\n    uint16_t is_symbol      : 1;  // regex: \\p{S}\n    uint16_t is_control     : 1;  // regex: \\p{C}\n    // helper flags\n    uint16_t is_whitespace  : 1;  // regex: \\s\n    uint16_t is_lowercase   : 1;\n    uint16_t is_uppercase   : 1;\n    uint16_t is_nfd         : 1;\n\n    // decode from uint16\n    inline unicode_cpt_flags(const uint16_t flags = 0) {\n#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__\n        *reinterpret_cast<uint16_t*>(this) = flags;\n#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__\n        is_undefined   = (flags & UNDEFINED)   ? 1 : 0;\n        is_number      = (flags & NUMBER)      ? 1 : 0;\n        is_letter      = (flags & LETTER)      ? 1 : 0;\n        is_separator   = (flags & SEPARATOR)   ? 1 : 0;\n        is_accent_mark = (flags & ACCENT_MARK) ? 1 : 0;\n        is_punctuation = (flags & PUNCTUATION) ? 1 : 0;\n        is_symbol      = (flags & SYMBOL)      ? 1 : 0;\n        is_control     = (flags & CONTROL)     ? 1 : 0;\n        is_whitespace  = (flags & WHITESPACE)  ? 1 : 0;\n        is_lowercase   = (flags & LOWERCASE)   ? 1 : 0;\n        is_uppercase   = (flags & UPPERCASE)   ? 1 : 0;\n        is_nfd         = (flags & NFD)         ? 1 : 0;\n#else\n#error Unexpected or undefined __BYTE_ORDER__\n#endif\n    }\n\n    inline uint16_t as_uint() const {\n#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__\n        return *reinterpret_cast<const uint16_t*>(this);\n#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__\n        uint16_t result =\n              is_undefined   * UNDEFINED\n            + is_number      * NUMBER\n            + is_letter      * LETTER\n            + is_separator   * SEPARATOR\n            + is_accent_mark * ACCENT_MARK\n            + is_punctuation * PUNCTUATION\n            + is_symbol      * SYMBOL\n            + is_control     * CONTROL\n            + is_whitespace  * WHITESPACE\n            + is_lowercase   * LOWERCASE\n            + is_uppercase   * UPPERCASE\n            + is_nfd         * NFD\n            ;\n\n        return result;\n#else\n#error Unexpected or undefined __BYTE_ORDER__\n#endif\n    }\n\n    inline uint16_t category_flag() const {\n        return this->as_uint() & MASK_CATEGORIES;\n    }\n};\n\nsize_t unicode_len_utf8(char src);\n\nstd::string unicode_cpt_to_utf8  (uint32_t cpt);\nuint32_t    unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);\n\nstd::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);\n\nstd::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);\n\nunicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);\nunicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8);\n\nstd::string unicode_byte_to_utf8(uint8_t byte);\nuint8_t     unicode_utf8_to_byte(const std::string & utf8);\n\nuint32_t unicode_tolower(uint32_t cpt);\n\nbool unicode_cpt_is_han(uint32_t cpt);\n\nstd::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);\n"
  },
  {
    "path": "examples/twitch.sh",
    "content": "#!/bin/bash\n#\n# Transcribe twitch.tv livestream by feeding audio input to whisper.cpp at regular intervals\n# Thanks to @keyehzy\n# ref: https://github.com/ggml-org/whisper.cpp/issues/209\n#\n# The script currently depends on the third-party tool \"streamlink\"\n# On Mac OS, you can install it via \"brew install streamlink\"\n#\n\nset -eo pipefail\n\nstep=10\nmodel=base.en\nthreads=4\n\nhelp()\n{\n    echo \"Example program for captioning a livestream from twitch.tv.\"\n    echo\n    echo \"Usage: ./twitch.sh -s [step] -m [model] -t [threads] [url]\"\n    echo \"options:\"\n    echo \"-s       Step in seconds (default is $step).\"\n    echo \"-m       Choose model, options are: 'tiny.en' 'tiny' 'base.en' 'base' 'small.en' 'small' 'medium.en' 'medium' 'large-v1' 'large-v2' 'large-v3' 'large-v3-turbo' (default is '$model').\"\n    echo \"-t       Number of threads to use.\"\n    echo \"-h       Print this help page.\"\n    echo\n}\n\ncheck_requirements()\n{\n    if ! command -v ./build/bin/whisper-cli &>/dev/null; then\n        echo \"whisper.cpp main executable is required (make)\"\n        exit 1\n    fi\n\n    if ! command -v streamlink &>/dev/null; then\n        echo \"streamlink is required (https://streamlink.github.io)\"\n        exit 1\n    fi\n\n    if ! command -v ffmpeg &>/dev/null; then\n        echo \"ffmpeg is required (https://ffmpeg.org)\"\n        exit 1\n    fi\n}\n\ncheck_requirements\n\nwhile getopts \":s:m:t:h\" option; do\n    case $option in\n\ts)\n            step=$OPTARG;;\n\tm)\n            model=$OPTARG;;\n\tt)\n\t    threads=$OPTARG;;\n\th)\n            help\n            exit;;\n\t\\?)\n\t    help\n\t    exit;;\n    esac\ndone\n\nurl=${@:$OPTIND:1}\n\nif [ -z $url ]; then\n    help\n    exit\nfi\n\necho \"Piping from streamlink url=$url model=$model step=$step threads=$threads\"\nstreamlink $url best -O 2>/dev/null | ffmpeg -loglevel quiet -i - -y -probesize 32 -y -ar 16000 -ac 1 -acodec pcm_s16le /tmp/whisper-live0.wav &\n\nif [ $? -ne 0 ]; then\n    printf \"error: ffmpeg failed\\n\"\n    exit 1\nfi\n\necho \"Buffering stream... (this should take $step seconds)\"\nsleep $(($step))\n\nset +e\n\necho \"Starting...\"\n\ni=0\nSECONDS=0\nwhile true\ndo\n    err=1\n    while [ $err -ne 0 ]; do\n        if [ $i -gt 0 ]; then\n            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.wav -y -ss $(($i*$step-1)).5 -t $step -c copy /tmp/whisper-live.wav 2> /tmp/whisper-live.err\n        else\n            ffmpeg -loglevel quiet -v error -noaccurate_seek -i /tmp/whisper-live0.wav -y -ss $(($i*$step)) -t $step -c copy /tmp/whisper-live.wav 2> /tmp/whisper-live.err\n        fi\n        err=$(cat /tmp/whisper-live.err | wc -l)\n    done\n\n    ./build/bin/whisper-cli -t $threads -m ./models/ggml-$model.bin -f /tmp/whisper-live.wav --no-timestamps -otxt 2> /tmp/whispererr | tail -n 1\n\n    while [ $SECONDS -lt $((($i+1)*$step)) ]; do\n        sleep 1\n    done\n    ((i=i+1))\ndone\n"
  },
  {
    "path": "examples/vad-speech-segments/CMakeLists.txt",
    "content": "set(TARGET whisper-vad-speech-segments)\nadd_executable(${TARGET} speech.cpp)\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})\n\ninstall(TARGETS ${TARGET} RUNTIME)\n"
  },
  {
    "path": "examples/vad-speech-segments/README.md",
    "content": "# whisper.cpp/examples/vad-speech-segments\n\nThis examples demonstrates how to use a VAD (Voice Activity Detection) model to\nsegment an audio file into speech segments.\n\n### Building the example\nThe example can be built using the following command:\n```console\ncmake -S . -B build\ncmake --build build -j8 --target vad-speech-segments\n```\n\n### Running the example\nThe examples can be run using the following command, which uses a model\nthat we use internally for testing:\n```console\n./build/bin/vad-speech-segments \\\n    --vad-model models/for-tests-silero-v6.2.0-ggml.bin \\\n    --file samples/jfk.wav \\\n    --no-prints\n\nDetected 5 speech segments:\nSpeech segment 0: start = 0.29, end = 2.21\nSpeech segment 1: start = 3.30, end = 3.77\nSpeech segment 2: start = 4.00, end = 4.35\nSpeech segment 3: start = 5.38, end = 7.65\nSpeech segment 4: start = 8.16, end = 10.59\n```\nTo see more output from whisper.cpp remove the `--no-prints` argument.\n\n\n### Command line options\n```console\n./build/bin/vad-speech-segments --help\n\nusage: ./build/bin/vad-speech-segments [options] file\nsupported audio formats: flac, mp3, ogg, wav\n\noptions:\n  -h,        --help                          [default] show this help message and exit\n  -f FNAME,  --file FNAME                    [       ] input audio file path\n  -t N,      --threads N                     [4      ] number of threads to use during computation\n  -ug,       --use-gpu                       [true   ] use GPU\n  -vm FNAME, --vad-model FNAME               [       ] VAD model path\n  -vt N,     --vad-threshold N               [0.50   ] VAD threshold for speech recognition\n  -vspd N,   --vad-min-speech-duration-ms  N [250    ] VAD min speech duration (0.0-1.0)\n  -vsd N,    --vad-min-silence-duration-ms N [100    ] VAD min silence duration (to split segments)\n  -vmsd N,   --vad-max-speech-duration-s   N [FLT_MAX] VAD max speech duration (auto-split longer)\n  -vp N,     --vad-speech-pad-ms           N [30     ] VAD speech padding (extend segments)\n  -vo N,     --vad-samples-overlap         N [0.10   ] VAD samples overlap (seconds between segments)\n  -np,       --no-prints                     [false  ] do not print anything other than the results\n```\n"
  },
  {
    "path": "examples/vad-speech-segments/speech.cpp",
    "content": "#include \"common.h\"\n#include \"common-whisper.h\"\n\n#include \"whisper.h\"\n\n#include <cstdio>\n#include <cfloat>\n#include <string>\n\n// command-line parameters\nstruct cli_params {\n    int32_t     n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    std::string vad_model = \"\";\n    float       vad_threshold = 0.5f;\n    int         vad_min_speech_duration_ms = 250;\n    int         vad_min_silence_duration_ms = 100;\n    float       vad_max_speech_duration_s = FLT_MAX;\n    int         vad_speech_pad_ms = 30;\n    float       vad_samples_overlap = 0.1f;\n    bool        use_gpu = false;\n    std::string fname_inp = {};\n    bool        no_prints       = false;\n};\n\nstatic void vad_print_usage(int /*argc*/, char ** argv, const cli_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options] file\\n\", argv[0]);\n    fprintf(stderr, \"supported audio formats: flac, mp3, ogg, wav\\n\");\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,        --help                          [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -f FNAME,  --file FNAME                    [%-7s] input audio file path\\n\",                            \"\");\n    fprintf(stderr, \"  -t N,      --threads N                     [%-7d] number of threads to use during computation\\n\",      params.n_threads);\n    fprintf(stderr, \"  -ug,       --use-gpu                       [%-7s] use GPU\\n\",                                          params.use_gpu ? \"true\" : \"false\");\n    fprintf(stderr, \"  -vm FNAME, --vad-model FNAME               [%-7s] VAD model path\\n\",                                   params.vad_model.c_str());\n    fprintf(stderr, \"  -vt N,     --vad-threshold N               [%-7.2f] VAD threshold for speech recognition\\n\",           params.vad_threshold);\n    fprintf(stderr, \"  -vspd N,   --vad-min-speech-duration-ms  N [%-7d] VAD min speech duration (0.0-1.0)\\n\",                params.vad_min_speech_duration_ms);\n    fprintf(stderr, \"  -vsd N,    --vad-min-silence-duration-ms N [%-7d] VAD min silence duration (to split segments)\\n\",     params.vad_min_silence_duration_ms);\n    fprintf(stderr, \"  -vmsd N,   --vad-max-speech-duration-s   N [%-7s] VAD max speech duration (auto-split longer)\\n\",      params.vad_max_speech_duration_s == FLT_MAX ?\n                                                                                                                                  std::string(\"FLT_MAX\").c_str() :\n                                                                                                                                  std::to_string(params.vad_max_speech_duration_s).c_str());\n    fprintf(stderr, \"  -vp N,     --vad-speech-pad-ms           N [%-7d] VAD speech padding (extend segments)\\n\",             params.vad_speech_pad_ms);\n    fprintf(stderr, \"  -vo N,     --vad-samples-overlap         N [%-7.2f] VAD samples overlap (seconds between segments)\\n\", params.vad_samples_overlap);\n    fprintf(stderr, \"  -np,       --no-prints                     [%-7s] do not print anything other than the results\\n\",     params.no_prints ? \"true\" : \"false\");\n    fprintf(stderr, \"\\n\");\n}\n\nstatic char * requires_value_error(const std::string & arg) {\n    fprintf(stderr, \"error: argument %s requires value\\n\", arg.c_str());\n    exit(0);\n}\n\nstatic bool vad_params_parse(int argc, char ** argv, cli_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            vad_print_usage(argc, argv, params);\n            exit(0);\n        }\n        #define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))\n        else if (arg == \"-f\"    || arg == \"--file\")                        { params.fname_inp = ARGV_NEXT; }\n        else if (arg == \"-t\"    || arg == \"--threads\")                     { params.n_threads                   = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-ug\"   || arg == \"--use-gpu\")                     { params.use_gpu                     = true; }\n        else if (arg == \"-vm\"   || arg == \"--vad-model\")                   { params.vad_model                   = ARGV_NEXT; }\n        else if (arg == \"-vt\"   || arg == \"--vad-threshold\")               { params.vad_threshold               = std::stof(ARGV_NEXT); }\n        else if (arg == \"-vsd\"  || arg == \"--vad-min-speech-duration-ms\")  { params.vad_min_speech_duration_ms  = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vsd\"  || arg == \"--vad-min-silence-duration-ms\") { params.vad_min_speech_duration_ms  = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vmsd\" || arg == \"--vad-max-speech-duration-s\")   { params.vad_max_speech_duration_s   = std::stof(ARGV_NEXT); }\n        else if (arg == \"-vp\"   || arg == \"--vad-speech-pad-ms\")           { params.vad_speech_pad_ms           = std::stoi(ARGV_NEXT); }\n        else if (arg == \"-vo\"   || arg == \"--vad-samples-overlap\")         { params.vad_samples_overlap         = std::stof(ARGV_NEXT); }\n        else if (arg == \"-np\"   || arg == \"--no-prints\")                   { params.no_prints       = true; }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            vad_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nstatic void cb_log_disable(enum ggml_log_level , const char * , void * ) { }\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    cli_params cli_params;\n\n    if (!vad_params_parse(argc, argv, cli_params)) {\n        vad_print_usage(argc, argv, cli_params);\n        return 1;\n    }\n\n    if (cli_params.no_prints) {\n        whisper_log_set(cb_log_disable, NULL);\n    }\n\n    // Load the input sample audio file.\n    std::vector<float> pcmf32;\n    std::vector<std::vector<float>> pcmf32s;\n    if (!read_audio_data(cli_params.fname_inp.c_str(), pcmf32, pcmf32s, false)) {\n        fprintf(stderr, \"error: failed to read audio data from %s\\n\", cli_params.fname_inp.c_str());\n        return 2;\n    }\n\n    // Initialize the context which loads the VAD model.\n    struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();\n    ctx_params.n_threads  = cli_params.n_threads;\n    ctx_params.use_gpu    = cli_params.use_gpu;\n    struct whisper_vad_context * vctx = whisper_vad_init_from_file_with_params(\n            cli_params.vad_model.c_str(),\n            ctx_params);\n    if (vctx == nullptr) {\n        fprintf(stderr, \"error: failed to initialize whisper context\\n\");\n        return 2;\n    }\n\n    // Detect speech in the input audio file.\n    if (!whisper_vad_detect_speech(vctx, pcmf32.data(), pcmf32.size())) {\n        fprintf(stderr, \"error: failed to detect speech\\n\");\n        return 3;\n    }\n\n    // Get the the vad segements using the probabilities that have been computed\n    // previously and stored in the whisper_vad_context.\n    struct whisper_vad_params params = whisper_vad_default_params();\n    params.threshold = cli_params.vad_threshold;\n    params.min_speech_duration_ms = cli_params.vad_min_speech_duration_ms;\n    params.min_silence_duration_ms = cli_params.vad_min_silence_duration_ms;\n    params.max_speech_duration_s = cli_params.vad_max_speech_duration_s;\n    params.speech_pad_ms = cli_params.vad_speech_pad_ms;\n    params.samples_overlap = cli_params.vad_samples_overlap;\n    struct whisper_vad_segments * segments = whisper_vad_segments_from_probs(vctx, params);\n\n    printf(\"\\n\");\n    printf(\"Detected %d speech segments:\\n\", whisper_vad_segments_n_segments(segments));\n    for (int i = 0; i < whisper_vad_segments_n_segments(segments); ++i) {\n        printf(\"Speech segment %d: start = %.2f, end = %.2f\\n\", i,\n               whisper_vad_segments_get_segment_t0(segments, i),\n               whisper_vad_segments_get_segment_t1(segments, i));\n    }\n    printf(\"\\n\");\n\n    whisper_vad_free_segments(segments);\n    whisper_vad_free(vctx);\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/wchess/CMakeLists.txt",
    "content": "add_subdirectory(libwchess)\nset_target_properties(wchess-core PROPERTIES FOLDER \"libs\")\n\nif (EMSCRIPTEN)\n    add_subdirectory(wchess.wasm)\n    set_target_properties(wchess.wasm PROPERTIES FOLDER \"libs\")\nelse()\n    add_subdirectory(wchess.cmd)\n    set_target_properties(wchess PROPERTIES FOLDER \"libs\")\nendif()\n"
  },
  {
    "path": "examples/wchess/README.md",
    "content": "# wchess\n\nVoice-controlled chess using Whisper\n\nOnline demo: https://ggml.ai/whisper.cpp/wchess.wasm/\n\nhttps://github.com/ggerganov/whisper.cpp/assets/1991296/c2b2f03c-9684-49f3-8106-357d2d4e67fa\n\n## Command-line tool\n\n```bash\nmkdir build && cd build\ncmake -DWHISPER_SDL2=1 ..\nmake -j\n\n./bin/wchess -m ../models/ggml-base.en.bin\n\nMove: start\n\na b c d e f g h\nr n b q k b n r 8\np p p p p p p p 7\n. * . * . * . * 6\n* . * . * . * . 5\n. * . * . * . * 4\n* . * . * . * . 3\nP P P P P P P P 2\nR N B Q K B N R 1\n\nWhite's turn\n[(l)isten/(p)ause/(q)uit]: \n```\n\n## TODO\n\n- Fix bugs in the chess moves logic\n- Improve web-browser audio capture - sometimes it does not record the voice properly\n- Add support for more languages by making the generated grammar string multilingual\n- Explore ways to improve the dynamic grammar to be narrower\n\nPRs welcome!\n\n## Thanks\n\n- [chessboardjs](https://chessboardjs.com) for the neat chessboard JS library used in this demo\n"
  },
  {
    "path": "examples/wchess/libwchess/CMakeLists.txt",
    "content": "add_library(wchess-core STATIC\n    WChess.cpp\n    WChess.h\n    Chessboard.cpp\n    Chessboard.h\n)\n\ntarget_link_libraries(wchess-core\n    PUBLIC\n    whisper\n    common\n)\n\ntarget_include_directories(wchess-core\n    PUBLIC\n    \"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>\"\n)\n\n# add_executable(test-chessboard test-chessboard.cpp Chessboard.cpp)\n"
  },
  {
    "path": "examples/wchess/libwchess/Chessboard.cpp",
    "content": "#include \"Chessboard.h\"\n\n#include <array>\n#include <vector>\n#include <algorithm>\n#include <cstring>\n#include <set>\n#include <list>\n#include <chrono>\n\nnamespace {\nconstexpr std::array<const char*, 64> positions = {\n    \"a1\", \"b1\", \"c1\", \"d1\", \"e1\", \"f1\", \"g1\", \"h1\",\n    \"a2\", \"b2\", \"c2\", \"d2\", \"e2\", \"f2\", \"g2\", \"h2\",\n    \"a3\", \"b3\", \"c3\", \"d3\", \"e3\", \"f3\", \"g3\", \"h3\",\n    \"a4\", \"b4\", \"c4\", \"d4\", \"e4\", \"f4\", \"g4\", \"h4\",\n    \"a5\", \"b5\", \"c5\", \"d5\", \"e5\", \"f5\", \"g5\", \"h5\",\n    \"a6\", \"b6\", \"c6\", \"d6\", \"e6\", \"f6\", \"g6\", \"h6\",\n    \"a7\", \"b7\", \"c7\", \"d7\", \"e7\", \"f7\", \"g7\", \"h7\",\n    \"a8\", \"b8\", \"c8\", \"d8\", \"e8\", \"f8\", \"g8\", \"h8\",\n};\nconstexpr char INVALID_POS = positions.size();\nconstexpr int R = 0; // rank index\nconstexpr int F = 1; // file index\n#define FILE (c[F] - '1')\n#define RANK (c[R] - 'a')\nconstexpr char operator \"\"_P(const char * c, size_t size) {\n    return size < 2 || RANK < 0 || RANK > 7 ||\n        FILE < 0 || FILE > 7 ? INVALID_POS : FILE * 8 + RANK;\n}\n#undef FILE\n#undef RANK\n\nstruct sview {\n    const char * ptr = nullptr;\n    size_t size = 0;\n\n    sview() = default;\n    sview(const char * p, size_t s) : ptr(p), size(s) {}\n    sview(const std::string& s) : ptr(s.data()), size(s.size()) {}\n\n    size_t find(char del, size_t pos) {\n        while (pos < size && ptr[pos] != del) ++pos;\n        return pos < size ? pos : std::string::npos;\n    }\n};\n\nstd::vector<sview> split(sview str, char del) {\n    std::vector<sview> res;\n    size_t cur = 0;\n    size_t last = 0;\n    while (cur != std::string::npos) {\n        if (str.ptr[last] == ' ') {\n            ++last;\n            continue;\n        }\n        cur = str.find(del, last);\n        size_t len = cur == std::string::npos ? str.size - last : cur - last;\n        res.emplace_back(str.ptr + last, len);\n        last = cur + 1;\n    }\n    return res;\n}\n\nchar strToPos(sview str) {\n    return operator \"\"_P(str.ptr, str.size);\n}\n\nconstexpr std::array<const char*, 6> pieceNames =  {\n    \"pawn\", \"knight\", \"bishop\", \"rook\", \"queen\", \"king\",\n};\n\nstatic constexpr std::array<char, 6> blackShort =  {\n    'p', 'n', 'b', 'r', 'q', 'k',\n};\nstatic constexpr std::array<char, 6> whiteShort =  {\n    'P', 'N', 'B', 'R', 'Q', 'K',\n};\n\nchar strToType(sview str) {\n    auto it = std::find_if(pieceNames.begin(), pieceNames.end(), [str] (const char* name) { return strncmp(name, str.ptr, str.size) == 0; });\n    return it != pieceNames.end() ? it - pieceNames.begin() : pieceNames.size();\n}\n\n// directions\nusing Direction = std::array<char, 2>;\n\nconstexpr Direction N   = {(char)  0, (char)  1};\nconstexpr Direction NNE = {(char)  1, (char)  2};\nconstexpr Direction NE  = {(char)  1, (char)  1};\nconstexpr Direction ENE = {(char)  2, (char)  1};\nconstexpr Direction E   = {(char)  1, (char)  0};\nconstexpr Direction ESE = {(char)  2, (char) -1};\nconstexpr Direction SE  = {(char)  1, (char) -1};\nconstexpr Direction SSE = {(char)  1, (char) -2};\nconstexpr Direction S   = {(char)  0, (char) -1};\nconstexpr Direction SSW = {(char) -1, (char) -2};\nconstexpr Direction SW  = {(char) -1, (char) -1};\nconstexpr Direction WSW = {(char) -2, (char) -1};\nconstexpr Direction W   = {(char) -1, (char)  0};\nconstexpr Direction WNW = {(char) -2, (char)  1};\nconstexpr Direction NW  = {(char) -1, (char)  1};\nconstexpr Direction NNW = {(char) -1, (char)  2};\n\nchar makeStep(char pos, const Direction& d) {\n    char next[2] = { char(positions[pos][R] + d[R]) , char(positions[pos][F] + d[F]) };\n    return strToPos(sview{next, sizeof(next)});\n}\n\ntemplate<class Modifier>\nchar traverse(char pos, const Direction& d, const Modifier& m, int count = 8) {\n    while (--count >= 0) {\n        pos = makeStep(pos, d);\n        if (pos == INVALID_POS || m(pos)) break;\n    }\n    return pos;\n}\n\nDirection normalize(const Direction& distance) {\n    //return {char((distance[R] > 0) - (distance[R] < 0)), char((distance[F] > 0) - (distance[F] < 0))};\n    const int drp = distance[R] > 0 ? 1 : 0;\n    const int drn = distance[R] < 0 ? 1 : 0;\n    const int dfp = distance[F] > 0 ? 1 : 0;\n    const int dfn = distance[F] < 0 ? 1 : 0;\n    return {char(drp - drn), char(dfp - dfn)};\n}\n\nstruct Pin {\n    Direction d;\n    Piece* pinner;\n    Piece* pinned;\n};\nusing Pins = std::list<Pin>;\nusing Board = std::array<Piece*, 64>;\n\nstd::vector<Direction> filter(const Direction& pin, std::initializer_list<Direction> directions) {\n    if (pin[R] == 0 && pin[F] == 0) return directions;\n    std::vector<Direction> result;\n    for (auto& d : directions) {\n        if ((d[R] == pin[R] || d[R] == -pin[R]) && (d[F] == pin[F] || d[F] == -pin[F])) result.push_back(d);\n    }\n    return result;\n}\n}\n\nclass Piece {\npublic:\n    enum Types : char {\n        Pawn,\n        Knight,\n        Bishop,\n        Rook,\n        Queen,\n        King,\n        //\n        NUM_PIECES\n    };\n\n    enum Colors : char {\n        White,\n        Black,\n    };\n\n    const char* name() const;\n    char initial() const;\n    Types type() const { return m_type; }\n    Colors color() const { return m_color; }\n    char pos() const { return m_pos; }\n    void setPos(char pos) {\n        m_pos = pos;\n        invalidate();\n    }\n    const char* coord() const;\n    const std::set<char>& allowed() const { return m_allowed; }\n    bool canReach(char pos) const;\n    virtual bool movePattern(char pos) const = 0;\n    void take();\n    virtual void reinit(const State& state) = 0;\n    void invalidate();\nprotected:\n    Piece(Types type, Colors color, char pos, std::set<char> allowed)\n        : m_type(type), m_color(color), m_pos(pos), m_allowed(std::move(allowed)) {}\n    Piece(const Piece&) = delete;\n    ~Piece() = default;\n\n    const Types m_type;\n    const Colors m_color;\n    char m_pos;\n    std::set<char> m_allowed;\n    bool m_update = false;\n};\n\nstruct Pawn : public Piece {\n    Pawn(Colors color, char pos, std::set<char> next) : Piece(Types::Pawn, color, pos, std::move(next)) {}\n\n    bool is_first_move() const {\n        return m_color ? coord()[F] == '7' : coord()[F] == '2';\n    }\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        Direction distance = {char(next[R] - cur[R]), char(next[F] - cur[F])};\n        char forward = m_color ? -1 : 1;\n        return (forward == distance[F] && distance[R] * distance[R] <= 1)\n            || (is_first_move() && 2 * forward == distance[F] && distance[R] == 0);\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct Knight : public Piece {\n    Knight(Colors color, char pos, std::set<char> next) : Piece(Types::Knight, color, pos, std::move(next)) {}\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        Direction diff = {char(next[R] - cur[R]), char(next[F] - cur[F])};\n        return diff[R]*diff[R] + diff[F]*diff[F] == 5;\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct Bishop : public Piece {\n    Bishop(Colors color, char pos) : Piece(Types::Bishop, color, pos, {}) {}\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        return cur[R] - cur[F] == next[R] - next[F] || cur[R] + cur[F] == next[R] + next[F];\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct Rook : public Piece {\n    Rook(Colors color, char pos) : Piece(Types::Rook, color, pos, {}) {}\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        return cur[R] == next[R] || cur[F] == next[F];\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct Queen : public Piece {\n    Queen(Colors color, char pos) : Piece(Types::Queen, color, pos, {}) {}\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        return cur[R] == next[R] || cur[F] == next[F] || cur[R] - cur[F] == next[R] - next[F] || cur[R] + cur[F] == next[R] + next[F];\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct King : public Piece {\n    King(Colors color, char pos) : Piece(Types::King, color, pos, {}) {}\n\n    virtual bool movePattern(char pos) const override {\n        if (m_pos == INVALID_POS) return false;\n        auto cur = coord();\n        auto next = positions[pos];\n        Direction diff = {char(next[R] - cur[R]), char(next[F] - cur[F])};\n        return diff[R]*diff[R] + diff[F]*diff[F] <= 2;\n    }\n\n    virtual void reinit(const State& state) override;\n};\n\nstruct PieceSet {\n    Piece* begin() { return &p1; }\n    Piece* end() { return &r2 + 1; }\n    const Piece* begin() const { return &p1; }\n    const Piece* end() const { return &r2 + 1; }\n    Piece& operator[](int i) { return *(begin() + i); }\n    const Piece& operator[](int i) const { return *(begin() + i); }\n\n    Pawn   p1;\n    Pawn   p2;\n    Pawn   p3;\n    Pawn   p4;\n    Pawn   p5;\n    Pawn   p6;\n    Pawn   p7;\n    Pawn   p8;\n    Rook   r1;\n    Knight n1;\n    Bishop b1;\n    Queen  q;\n    King   k;\n    Bishop b2;\n    Knight n2;\n    Rook   r2;\n};\n\nstruct State {\n    State();\n    PieceSet blacks;\n    PieceSet whites;\n    Board board;\n    Pins blackPins;\n    Pins whitePins;\n};\n\nDirection findPin(const Piece& piece, const State& state) {\n    auto& pins = piece.color() ? state.blackPins : state.whitePins;\n    auto it = std::find_if(pins.begin(), pins.end(), [&] (const Pin& pin) { return pin.pinned == &piece; });\n    if (it != pins.end()) return it->d;\n    return {0, 0};\n}\n\nstruct Find {\n    Find(const Board& board) : m_board(board) {}\n    bool operator() (char pos) const { return m_board[pos]; }\n    const Board& m_board;\n};\n\nstruct Add {\n    Add(const Board& board, std::set<char>& moves, Piece::Colors color) : m_board(board), m_moves(moves), m_color(color) {}\n    bool operator() (char pos) const {\n        if (!m_board[pos] || m_board[pos]->color() != m_color) m_moves.insert(pos);\n        return m_board[pos];\n    }\n    const Board& m_board;\n    std::set<char>& m_moves;\n    Piece::Colors m_color;\n};\n\nvoid Pawn::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n\n    auto pin = findPin(*this, state);\n\n    auto & left = m_color ? SW : NW;\n    auto & right = m_color ? SE : NE;\n\n    for (auto& direction : filter(pin, { left, right })) {\n        auto pos = makeStep(m_pos, direction);\n        if (pos != INVALID_POS && state.board[pos] && state.board[pos]->color() != m_color) m_allowed.insert(pos);\n    }\n\n    auto & forward = m_color ? S : N;\n    if (!filter(pin, {forward}).empty()) {\n        traverse(m_pos, forward, [&] (char pos) {\n                if (!state.board[pos]) m_allowed.insert(pos);\n                return state.board[pos] || !is_first_move();\n            }, 2);\n    }\n}\n\nvoid Knight::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n    auto pin = findPin(*this, state);\n    if (pin[R] != 0 || pin[F] != 0) return;\n    for (auto& direction : { NNE, ENE, ESE, SSE, SSW, WSW, WNW, NNW }) {\n        auto pos = makeStep(m_pos, direction);\n        if (pos != INVALID_POS && (!state.board[pos] || state.board[pos]->color() != m_color)) m_allowed.insert(pos);\n    }\n}\n\nvoid Bishop::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n    auto pin = findPin(*this, state);\n    for (auto& direction : filter(pin, { NE, SE, SW, NW })) {\n        traverse(m_pos, direction, Add(state.board, m_allowed, m_color));\n    }\n}\n\nvoid Rook::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n    auto pin = findPin(*this, state);\n    for (auto& direction : filter(pin, { N, E, S, W })) {\n        traverse(m_pos, direction, Add(state.board, m_allowed, m_color));\n    }\n}\n\nvoid Queen::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n    auto pin = findPin(*this, state);\n    for (auto& direction : filter(pin, { N, NE, E, SE, S, SW, W, NW })) {\n        traverse(m_pos, direction, Add(state.board, m_allowed, m_color));\n    }\n}\n\nvoid King::reinit(const State& state) {\n    if (m_pos == INVALID_POS) return;\n    if (!m_update) return;\n    m_update = false;\n    m_allowed.clear();\n    auto& enemyPieces = m_color ? state.whites : state.blacks;\n    auto& pawnAttackLeft = m_color ? SW : NW;\n    auto& pawnAttackRight = m_color ? SE : NE;\n    for (auto& direction : { N, NE, E, SE, S, SW, W, NW }) {\n        auto pos = makeStep(m_pos, direction);\n        bool accept = pos != INVALID_POS && !(state.board[pos] && state.board[pos]->color() == m_color);\n        if (accept) {\n            for (auto& p : enemyPieces) {\n                if (!p.movePattern(pos)) continue;\n                if (p.type() == Piece::Knight || p.type() == Piece::King) {\n                    accept = false;\n                    break;\n                }\n                else if (p.type() == Piece::Pawn) {\n                    auto from = positions[pos];\n                    auto to = p.coord();\n                    Direction d {char(to[R] - from[R]), char(to[F] - from[F])};\n                    if (d == pawnAttackLeft || d == pawnAttackRight) {\n                        accept = false;\n                        break;\n                    }\n                }\n                else {\n                    auto from = positions[pos];\n                    auto to = p.coord();\n                    Direction d = normalize({char(to[R] - from[R]), char(to[F] - from[F])});\n                    auto reached = traverse(pos, d, Find(state.board));\n                    if (p.pos() == reached) {\n                        accept = false;\n                        break;\n                    }\n                }\n            }\n        }\n        if (accept) m_allowed.insert(pos);\n    }\n}\n\nconst char* Piece::name() const {\n    static_assert(pieceNames.size() == Piece::NUM_PIECES, \"Mismatch between piece names and types\");\n    return pieceNames[m_type];\n}\n\nchar Piece::initial() const {\n    static_assert(blackShort.size() == Piece::NUM_PIECES, \"Mismatch between piece names and types\");\n    static_assert(whiteShort.size() == Piece::NUM_PIECES, \"Mismatch between piece names and types\");\n    return m_color ? blackShort[m_type] : whiteShort[m_type];\n}\n\nvoid Piece::invalidate() {\n    m_update = true;\n}\n\n\nconst char* Piece::coord() const {\n    if (m_pos == INVALID_POS) return \"\";\n    return positions[m_pos];\n}\n\nbool Piece::canReach(char pos) const {\n    return movePattern(pos) && m_allowed.count(pos);\n}\n\nvoid Piece::take() {\n    m_pos = INVALID_POS;\n    m_allowed = {};\n}\n\nState::State()\n    : blacks {\n        {Piece::Black, \"a7\"_P, {\"a5\"_P, \"a6\"_P} },\n        {Piece::Black, \"b7\"_P, {\"b5\"_P, \"b6\"_P} },\n        {Piece::Black, \"c7\"_P, {\"c5\"_P, \"c6\"_P} },\n        {Piece::Black, \"d7\"_P, {\"d5\"_P, \"d6\"_P} },\n        {Piece::Black, \"e7\"_P, {\"e5\"_P, \"e6\"_P} },\n        {Piece::Black, \"f7\"_P, {\"f5\"_P, \"f6\"_P} },\n        {Piece::Black, \"g7\"_P, {\"g5\"_P, \"g6\"_P} },\n        {Piece::Black, \"h7\"_P, {\"h5\"_P, \"h6\"_P} },\n        {Piece::Black, \"a8\"_P},\n        {Piece::Black, \"b8\"_P, {\"a6\"_P, \"c6\"_P} },\n        {Piece::Black, \"c8\"_P},\n        {Piece::Black, \"d8\"_P},\n        {Piece::Black, \"e8\"_P},\n        {Piece::Black, \"f8\"_P},\n        {Piece::Black, \"g8\"_P, {\"f6\"_P, \"h6\"_P} },\n        {Piece::Black, \"h8\"_P},\n    }\n    , whites {\n        {Piece::White, \"a2\"_P, {\"a3\"_P, \"a4\"_P} },\n        {Piece::White, \"b2\"_P, {\"b3\"_P, \"b4\"_P} },\n        {Piece::White, \"c2\"_P, {\"c3\"_P, \"c4\"_P} },\n        {Piece::White, \"d2\"_P, {\"d3\"_P, \"d4\"_P} },\n        {Piece::White, \"e2\"_P, {\"e3\"_P, \"e4\"_P} },\n        {Piece::White, \"f2\"_P, {\"f3\"_P, \"f4\"_P} },\n        {Piece::White, \"g2\"_P, {\"g3\"_P, \"g4\"_P} },\n        {Piece::White, \"h2\"_P, {\"h3\"_P, \"h4\"_P} },\n        {Piece::White, \"a1\"_P},\n        {Piece::White, \"b1\"_P, {\"a3\"_P, \"c3\"_P} },\n        {Piece::White, \"c1\"_P},\n        {Piece::White, \"d1\"_P},\n        {Piece::White, \"e1\"_P},\n        {Piece::White, \"f1\"_P},\n        {Piece::White, \"g1\"_P, {\"f3\"_P, \"h3\"_P} },\n        {Piece::White, \"h1\"_P},\n    }\n    , board {{\n        &whites[ 8],  &whites[ 9],  &whites[10],  &whites[11],  &whites[12],  &whites[13],  &whites[14],  &whites[15],\n        &whites[ 0],  &whites[ 1],  &whites[ 2],  &whites[ 3],  &whites[ 4],  &whites[ 5],  &whites[ 6],  &whites[ 7],\n        nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,\n        nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,\n        nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,\n        nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,      nullptr,\n        &blacks[ 0],  &blacks[ 1],  &blacks[ 2],  &blacks[ 3],  &blacks[ 4],  &blacks[ 5],  &blacks[ 6],  &blacks[ 7],\n        &blacks[ 8],  &blacks[ 9],  &blacks[10],  &blacks[11],  &blacks[12],  &blacks[13],  &blacks[14],  &blacks[15],\n    }}\n{}\n\nChessboard::Chessboard()\n    : m_state(new State())\n{\n    setGrammar();\n}\n\nChessboard::~Chessboard() = default;\n\nvoid Chessboard::setPrompt(const std::string& prompt) {\n    m_prompt = prompt;\n    setGrammar();\n}\n\nvoid Chessboard::setGrammar() {\n    m_grammar.clear();\n\n    std::string result;\n    if (m_prompt.empty()) {\n        result += \"move ::= \\\" \\\" ((piece | frompos) \\\" \\\" \\\"to \\\"?)? topos\\n\";\n        //result += \"move ::= \\\" \\\" frompos \\\" \\\" \\\"to \\\"? topos\\n\";\n    }\n    else {\n        // result += \"move ::= prompt \\\" \\\" ((piece | frompos) \\\" \\\" \\\"to \\\"?)? topos\\n\"\n        result += \"move ::= prompt \\\" \\\" frompos \\\" \\\" \\\"to \\\"? topos\\n\"\n        \"prompt ::= \\\" \" + m_prompt + \"\\\"\\n\";\n    }\n\n    std::set<Piece::Types> pieceTypes;\n    std::set<char> from_pos;\n    std::set<char> to_pos;\n    auto& pieces =  m_moveCounter % 2 ? m_state->blacks : m_state->whites;\n    std::set<size_t> flags;\n    for (auto& p : pieces) {\n        if (p.allowed().empty()) continue;\n        bool addPiece = false;\n        if (!m_inCheck || p.type() == Piece::King) {\n            to_pos.insert(p.allowed().begin(), p.allowed().end());\n            addPiece = !p.allowed().empty();\n        }\n        else {\n            for (auto move : p.allowed()) {\n                if (m_allowedInCheck.count(move)) {\n                    to_pos.insert(move);\n                    addPiece = true;\n                }\n            }\n        }\n        if (addPiece) {\n            pieceTypes.insert(p.type());\n            from_pos.insert(p.pos());\n        }\n    }\n    if (pieceTypes.empty()) return;\n\n    result += \"piece ::= (\";\n    for (auto& p : pieceTypes) result += \" \\\"\" + std::string(pieceNames[p]) + \"\\\" |\";\n    result.pop_back();\n    result += \")\\n\\n\";\n\n    result += \"frompos ::= (\";\n    for (auto& p : from_pos) result += \" \\\"\" + std::string(positions[p]) + \"\\\" |\";\n    result.pop_back();\n    result += \")\\n\";\n\n    result += \"topos ::= (\";\n    for (auto& p : to_pos) result += \" \\\"\" + std::string(positions[p]) + \"\\\" |\";\n    result.pop_back();\n    result += \")\\n\";\n\n    m_grammar = std::move(result);\n}\n\nstd::string Chessboard::stringifyBoard() {\n    std::string result;\n    result.reserve(16 + 2 * 64 + 16);\n    for (char rank = 'a'; rank <= 'h'; ++rank) {\n        result.push_back(rank);\n        result.push_back(' ');\n    }\n    result.back() = '\\n';\n    for (int i = 7; i >= 0; --i) {\n        for (int j = 0; j < 8; ++j) {\n            auto p = m_state->board[i * 8 + j];\n            if (p) result.push_back(p->initial());\n            else result.push_back((i + j) % 2 ? '.' : '*');\n            result.push_back(' ');\n        }\n        result.push_back('0' + i + 1);\n        result.push_back('\\n');\n    }\n    return result;\n}\n\nstd::string Chessboard::process(const std::string& command) {\n    const auto t_start = std::chrono::high_resolution_clock::now();\n    auto color = Piece::Colors(m_moveCounter % 2);\n    Piece* piece = nullptr;\n    auto pos_to = INVALID_POS;\n    if (!parseCommand(command, piece, pos_to)) return \"\";\n\n    auto pos_from = piece->pos();\n\n    if (!move(*piece, pos_to)) return \"\";\n\n    flagUpdates(pos_from, pos_to);\n\n    detectChecks();\n\n    auto& enemyPieces = color ? m_state->whites : m_state->blacks;\n    for (auto& p : enemyPieces) p.reinit(*m_state); // only enemy moves needed next\n\n    std::string result = {positions[pos_from][R], positions[pos_from][F], '-', positions[pos_to][R], positions[pos_to][F]};\n    ++m_moveCounter;\n    setGrammar();\n    const auto t_end = std::chrono::high_resolution_clock::now();\n    auto t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();\n    fprintf(stdout, \"%s: Move '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", result.data(), \"\\033[0m\", (int) t_ms);\n    if (m_grammar.empty()) result.push_back('#');\n    return result;\n}\n\nbool Chessboard::parseCommand(const std::string& command, Piece*& piece, char& pos_to) {\n    auto color = Piece::Colors(m_moveCounter % 2);\n    fprintf(stdout, \"%s: Command to %s: '%s%.*s%s'\\n\", __func__, (color ? \"Black\" : \"White\"), \"\\033[1m\", int(command.size()), command.data(), \"\\033[0m\");\n\n    if (command.empty()) return false;\n    auto tokens = split(command, ' ');\n    auto pos_from = INVALID_POS;\n    auto type = Piece::Types::NUM_PIECES;\n    if (tokens.size() == 1) {\n        type = Piece::Types::Pawn;\n        pos_to = strToPos(tokens.front());\n    }\n    else {\n        pos_from = strToPos(tokens.front());\n        if (pos_from == INVALID_POS) type = Piece::Types(strToType(tokens.front()));\n        pos_to = strToPos(tokens.back());\n    }\n    if (pos_to == INVALID_POS) return false;\n    if (pos_from == INVALID_POS) {\n        if (type == Piece::Types::NUM_PIECES) return false;\n        auto& pieces = color ? m_state->blacks : m_state->whites;\n        for (auto& p : pieces) {\n            if (p.type() == type && p.canReach(pos_to)) {\n                pos_from = p.pos();\n                break;\n            }\n        }\n    }\n    if (pos_from == INVALID_POS) return false;\n    if (m_state->board[pos_from] == nullptr) return false;\n    piece = m_state->board[pos_from];\n    if (piece->color() != color) return false;\n    return true;\n}\n\nvoid Chessboard::flagUpdates(char pos_from, char pos_to) {\n    auto color = Piece::Colors(m_moveCounter % 2);\n    auto& enemyPieces = color ? m_state->whites : m_state->blacks;\n    auto& ownPieces = color ? m_state->blacks : m_state->whites;\n    for (auto& p : enemyPieces) {\n        if (p.movePattern(pos_to) || p.movePattern(pos_from)) {\n            updatePins(p);\n            p.invalidate();\n        }\n    }\n\n    for (auto& p : ownPieces) {\n        if (p.movePattern(pos_to) || p.movePattern(pos_from)) {\n            updatePins(p);\n            p.invalidate();\n        }\n    }\n}\n\nvoid Chessboard::updatePins(Piece& piece) {\n    if (piece.type() == Piece::Pawn || piece.type() == Piece::Knight || piece.type() == Piece::King) return;\n    auto& enemyPieces = piece.color() ? m_state->whites : m_state->blacks;\n    auto& enemyPins = piece.color() ? m_state->whitePins : m_state->blackPins;\n    auto& king = enemyPieces.k;\n    auto it = std::find_if(enemyPins.begin(), enemyPins.end(), [&] (const Pin& pin) { return pin.pinner == &piece; });\n    if (it != enemyPins.end()) {\n        it->pinned->invalidate();\n        enemyPins.erase(it);\n    }\n    if (piece.movePattern(king.pos())) {\n        auto to = positions[king.pos()];\n        auto from = piece.coord();\n        Direction d = normalize({char(to[R] - from[R]), char(to[F] - from[F])});\n\n        auto reached = traverse(piece.pos(), d, Find(m_state->board));\n        auto foundPiece = m_state->board[reached];\n        if (&king == foundPiece) {\n            // check\n            king.invalidate();\n        }\n        else if (foundPiece && foundPiece->color() != piece.color()) {\n            reached = traverse(reached, d, Find(m_state->board));\n            if (&king == m_state->board[reached]) {\n                enemyPins.push_back({d, &piece, foundPiece});\n                foundPiece->invalidate();\n            }\n        }\n    }\n}\n\nvoid Chessboard::detectChecks() {\n    auto color = Piece::Colors(m_moveCounter % 2);\n    auto& enemyPieces = color ? m_state->whites : m_state->blacks;\n    auto& ownPieces = color ? m_state->blacks : m_state->whites;\n    auto& king = enemyPieces.k;\n    auto& pawnAttackLeft = color ? SW : NW;\n    auto& pawnAttackRight = color ? SE : NE;\n    for (auto& p : ownPieces) {\n        if (!p.movePattern(king.pos())) continue;\n        auto to = positions[king.pos()];\n        auto from = p.coord();\n\n        if (p.type() == Piece::Knight) {\n            if (!m_inCheck) {\n                m_allowedInCheck = { p.pos() };\n            }\n            else {\n                m_allowedInCheck.clear();\n            }\n            m_inCheck = true;\n        }\n        else if (p.type() == Piece::Pawn) {\n            Direction d {char(to[R] - from[R]), char(to[F] - from[F])};\n            if (d == pawnAttackLeft || d == pawnAttackRight) {\n                if (!m_inCheck) {\n                    m_allowedInCheck = { p.pos() };\n                }\n                else {\n                    m_allowedInCheck.clear();\n                }\n                m_inCheck = true;\n            }\n        }\n        else {\n            Direction d = normalize({char(to[R] - from[R]), char(to[F] - from[F])});\n            std::set<char> tmp;\n            auto pos = traverse(p.pos(), d, Add(m_state->board, tmp, king.color()));\n            if (pos == king.pos()) {\n                tmp.insert(p.pos());\n                if (!m_inCheck) {\n                    m_allowedInCheck = std::move(tmp);\n                }\n                else {\n                    m_allowedInCheck.clear();\n                }\n                m_inCheck = true;\n            }\n        }\n    }\n}\n\nbool Chessboard::move(Piece& piece, char pos_to) {\n    auto& allowed = piece.allowed();\n\n    if (allowed.count(pos_to) == 0 || (m_inCheck && piece.type() != Piece::King && m_allowedInCheck.count(pos_to) == 0)) return false;\n    if (m_state->board[pos_to] && m_state->board[pos_to]->color() == piece.color()) return false;\n    if (m_state->board[pos_to]) m_state->board[pos_to]->take();\n    m_state->board[piece.pos()] = nullptr;\n    m_state->board[pos_to] = &piece;\n    piece.setPos(pos_to);\n\n    m_inCheck = false;\n    m_allowedInCheck.clear();\n\n    return true;\n}\n"
  },
  {
    "path": "examples/wchess/libwchess/Chessboard.h",
    "content": "#pragma once\n#include <string>\n#include <set>\n#include <memory>\n\n// just basic validation\n// fixme: missing en passant, castling, promotion, etc.\nstruct State;\nclass Piece;\nclass Chessboard {\npublic:\n    Chessboard();\n    ~Chessboard();\n    std::string process(const std::string& command);\n    std::string stringifyBoard();\n    const std::string& grammar() { return m_grammar; }\n    const std::string& prompt() { return m_prompt; }\n    void setPrompt(const std::string& prompt);\nprivate:\n    bool parseCommand(const std::string& command, Piece*& piece, char& pos_to);\n    bool move(Piece& piece, char pos);\n    void flagUpdates(char pos_from, char pos_to);\n    void updatePins(Piece& piece);\n    void detectChecks();\n    void setGrammar();\n\n    std::unique_ptr<State> m_state;\n    std::set<char> m_allowedInCheck;\n    bool m_inCheck = false;\n    int m_moveCounter = 0;\n    std::string m_grammar;\n    std::string m_prompt;\n};\n"
  },
  {
    "path": "examples/wchess/libwchess/WChess.cpp",
    "content": "#include \"WChess.h\"\n#include \"Chessboard.h\"\n#include \"grammar-parser.h\"\n#include \"common.h\"\n#include <chrono>\n\nWChess::WChess(whisper_context * ctx,\n        const whisper_full_params & wparams,\n        callbacks cb,\n        settings s)\n        : m_ctx(ctx)\n        , m_wparams(wparams)\n        , m_cb(cb)\n        , m_settings(s)\n        , m_board(new Chessboard())\n{}\n\nWChess::~WChess() = default;\n\nvoid WChess::set_move(const std::string& moves, float prob) const {\n    if (m_cb.set_move) (*m_cb.set_move)(moves, prob);\n}\n\nvoid WChess::set_grammar(const std::string& grammar) const {\n    if (m_cb.set_grammar) (*m_cb.set_grammar)(grammar);\n}\n\nbool WChess::get_audio(std::vector<float>& pcmf32) const {\n    if (m_cb.get_audio) return (*m_cb.get_audio)(pcmf32);\n    return false;\n}\n\nstd::string WChess::stringify_board() const {\n    return m_board->stringifyBoard();\n}\n\nstd::string WChess::get_grammar() const {\n    return m_board->grammar();\n}\n\nvoid WChess::run() {\n    bool have_prompt  = true;\n    bool ask_prompt   = !have_prompt;\n\n    float logprob_min  = 0.0f;\n\n    float logprob_sum  = 0.0f;\n\n    int n_tokens  = 0;\n\n    std::vector<float> pcmf32_cur;\n    std::vector<float> pcmf32_prompt;\n\n    const std::string k_prompt = have_prompt ? \"\" : \"rook to d4, f3\";\n    int64_t t_ms = 0;\n\n    if (ask_prompt) {\n        fprintf(stdout, \"\\n\");\n        fprintf(stdout, \"%s: Say the following phrase: '%s%s%s'\\n\", __func__, \"\\033[1m\", k_prompt.c_str(), \"\\033[0m\");\n        fprintf(stdout, \"\\n\");\n\n        ask_prompt = false;\n    }\n\n    while (get_audio(pcmf32_cur)) {\n        if (!pcmf32_cur.empty()) {\n            // fprintf(stdout, \"%s: Processing ...\\n\", __func__);\n\n            if (!have_prompt) {\n                const auto txt = ::trim(transcribe(pcmf32_cur, logprob_min, logprob_sum, n_tokens, t_ms));\n\n                fprintf(stdout, \"%s: Heard '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", txt.c_str(), \"\\033[0m\", (int) t_ms);\n\n                const float sim = similarity(txt, k_prompt);\n\n                if (txt.length() < 0.8*k_prompt.length() || txt.length() > 1.2*k_prompt.length() || sim < 0.8f) {\n                    fprintf(stdout, \"%s: WARNING: prompt not recognized, try again\\n\", __func__);\n                    ask_prompt = true;\n                } else {\n                    fprintf(stdout, \"\\n\");\n                    fprintf(stdout, \"%s: The prompt has been recognized!\\n\", __func__);\n                    fprintf(stdout, \"%s: Waiting for voice commands ...\\n\", __func__);\n                    fprintf(stdout, \"\\n\");\n\n                    // save the audio for the prompt\n                    pcmf32_prompt = pcmf32_cur;\n                    have_prompt = true;\n                    m_board->setPrompt(k_prompt);\n                }\n            } else {\n                if (!pcmf32_prompt.empty()) pcmf32_cur.insert(pcmf32_cur.begin(), pcmf32_prompt.begin(), pcmf32_prompt.end());\n                constexpr size_t MIN_SIZE = 1.2 * WHISPER_SAMPLE_RATE;\n                if (MIN_SIZE > pcmf32_cur.size()) pcmf32_cur.insert(pcmf32_cur.begin(), MIN_SIZE - pcmf32_cur.size(), 0.0f);\n\n                // fprintf(stdout, \"%s: grammar rules:\\n'%s'\\n\", __func__, m_board->grammar().c_str());\n\n                auto grammar_parsed = grammar_parser::parse(m_board->grammar().c_str());\n                auto grammar_rules  = grammar_parsed.c_rules();\n\n                m_wparams.grammar_rules   = grammar_rules.data();\n                m_wparams.n_grammar_rules = grammar_rules.size();\n\n                m_wparams.i_start_rule    = grammar_parsed.symbol_ids.at(\"move\");\n                auto txt = ::trim(transcribe(pcmf32_cur, logprob_min, logprob_sum, n_tokens, t_ms));\n\n                const float p = 100.0f * std::exp(logprob_min);\n\n                fprintf(stdout, \"%s: heard '%s'\\n\", __func__, txt.c_str());\n\n                // find the prompt in the text\n                float best_sim = 0.0f;\n                size_t best_len = 0;\n                for (int n = 0.8*k_prompt.size(); n <= 1.2*k_prompt.size(); ++n) {\n                    const auto prompt = txt.substr(0, n);\n\n                    const float sim = similarity(prompt, k_prompt);\n\n                    //fprintf(stderr, \"%s: prompt = '%s', sim = %f\\n\", __func__, prompt.c_str(), sim);\n\n                    if (sim > best_sim) {\n                        best_sim = sim;\n                        best_len = n;\n                    }\n                }\n\n                fprintf(stdout, \"%s:   DEBUG: txt = '%s', prob = %.2f%%\\n\", __func__, txt.c_str(), p);\n                std::string command = ::trim(txt.substr(best_len));\n\n                fprintf(stdout, \"%s: Command '%s%s%s', (t = %d ms)\\n\", __func__, \"\\033[1m\", command.c_str(), \"\\033[0m\", (int) t_ms);\n                fprintf(stdout, \"\\n\");\n\n                if (!command.empty()) {\n                    set_move(m_board->process(command), p);\n                    set_grammar(m_board->grammar());\n                }\n                if (m_board->grammar().empty()) {\n                    fprintf(stdout, \"%s: No more moves possible\\n\", __func__);\n                    break;\n                }\n            }\n        }\n\n        if (ask_prompt) {\n            fprintf(stdout, \"\\n\");\n            fprintf(stdout, \"%s: Say the following phrase: '%s%s%s'\\n\", __func__, \"\\033[1m\", k_prompt.c_str(), \"\\033[0m\");\n            fprintf(stdout, \"\\n\");\n\n            ask_prompt = false;\n        }\n    }\n}\n\nstd::string WChess::transcribe(\n                const std::vector<float> & pcmf32,\n                float & logprob_min,\n                float & logprob_sum,\n                int & n_tokens,\n                int64_t & t_ms) {\n    const auto t_start = std::chrono::high_resolution_clock::now();\n\n    logprob_min = 0.0f;\n    logprob_sum = 0.0f;\n    n_tokens    = 0;\n    t_ms = 0;\n\n    if (whisper_full(m_ctx, m_wparams, pcmf32.data(), pcmf32.size()) != 0) {\n        return {};\n    }\n\n    std::string result;\n\n    const int n_segments = whisper_full_n_segments(m_ctx);\n    for (int i = 0; i < n_segments; ++i) {\n        const char * text = whisper_full_get_segment_text(m_ctx, i);\n\n        result += text;\n\n        const int n = whisper_full_n_tokens(m_ctx, i);\n        for (int j = 0; j < n; ++j) {\n            const auto token = whisper_full_get_token_data(m_ctx, i, j);\n\n            if(token.plog > 0.0f) return {};\n            logprob_min = std::min(logprob_min, token.plog);\n            logprob_sum += token.plog;\n            ++n_tokens;\n        }\n    }\n\n    const auto t_end = std::chrono::high_resolution_clock::now();\n    t_ms = std::chrono::duration_cast<std::chrono::milliseconds>(t_end - t_start).count();\n\n    return result;\n}\n"
  },
  {
    "path": "examples/wchess/libwchess/WChess.h",
    "content": "#pragma once\n#include \"whisper.h\"\n#include <string>\n#include <vector>\n#include <memory>\n\nclass Chessboard;\n\nclass WChess {\npublic:\n    using CheckRunningCb = bool (*)();\n    using GetAudioCb = bool (*)(std::vector<float> &);\n    using SetMovesCb = void (*)(const std::string &, float);\n    using SetGrammarCb = void (*)(const std::string &);\n    using ClearAudioCb = void (*)();\n\n    struct callbacks {\n        GetAudioCb get_audio = nullptr;\n        SetMovesCb set_move = nullptr;\n        SetGrammarCb set_grammar = nullptr;\n    };\n\n    struct settings {\n        int32_t vad_ms     = 2000;\n        int32_t prompt_ms  = 5000;\n        int32_t command_ms = 4000;\n        float vad_thold    = 0.2f;\n        float freq_thold   = 100.0f;\n        bool print_energy  = false;\n    };\n\n    WChess(\n        whisper_context * ctx,\n        const whisper_full_params & wparams,\n        callbacks cb,\n        settings s\n    );\n    ~WChess();\n\n    void run();\n\n    std::string stringify_board() const;\n\n    std::string get_grammar() const;\n\nprivate:\n    bool get_audio(std::vector<float>& pcmf32) const;\n    void set_move(const std::string& moves, float prob) const;\n    void set_grammar(const std::string& grammar) const;\n\n    std::string transcribe(\n                    const std::vector<float> & pcmf32,\n                    float & logprob_min,\n                    float & logprob_sum,\n                    int & n_tokens,\n                    int64_t & t_ms);\n\n    whisper_context * m_ctx;\n    whisper_full_params m_wparams;\n    const callbacks m_cb;\n    const settings m_settings;\n    std::unique_ptr<Chessboard> m_board;\n};\n"
  },
  {
    "path": "examples/wchess/libwchess/test-chessboard.cpp",
    "content": "#include \"Chessboard.h\"\n\n#define ASSERT(x) \\\n    do { \\\n        if (!(x)) { \\\n            fprintf(stderr, \"ASSERT: %s:%d: %s\\n\", __FILE__, __LINE__, #x); \\\n            fflush(stderr); \\\n            exit(1); \\\n        } \\\n    } while (0)\n\n\nint main() {\n    {\n        Chessboard chess;\n\n        ASSERT(chess.process(\"pawn to d4\") == \"d2-d4\");\n        ASSERT(chess.process(\"e5\") == \"e7-e5\");\n        ASSERT(chess.process(\"c1 h6\") == \"c1-h6\");\n        ASSERT(chess.process(\"queen h4\") == \"d8-h4\");\n        ASSERT(chess.process(\"bishop to g5\") == \"h6-g5\");\n        ASSERT(chess.process(\"bishop to b4\") == \"f8-b4\");\n        ASSERT(chess.process(\"c4\") == \"\");\n        ASSERT(chess.process(\"knight c3\") == \"b1-c3\");\n        ASSERT(chess.process(\"knight c6\") == \"b8-c6\");\n        ASSERT(chess.process(\"f3\") == \"\");\n    }\n\n    {\n        Chessboard chess;\n\n        ASSERT(chess.process(\"d4\") == \"d2-d4\");\n        ASSERT(chess.process(\"e5\") == \"e7-e5\");\n        ASSERT(chess.process(\"e4\") == \"e2-e4\");\n        ASSERT(chess.process(\"queen h4\") == \"d8-h4\");\n        ASSERT(chess.process(\"queen h5\") == \"d1-h5\");\n        ASSERT(chess.process(\"f5\") == \"\");\n        ASSERT(chess.process(\"g6\") == \"g7-g6\");\n        ASSERT(chess.process(\"knight e2\") == \"g1-e2\");\n        ASSERT(chess.process(\"f5\") == \"f7-f5\");\n        ASSERT(chess.process(\"knight g3\") == \"e2-g3\");\n        ASSERT(chess.process(\"g5\") == \"\");\n        ASSERT(chess.process(\"king e7\") == \"e8-e7\");\n        ASSERT(chess.process(\"f4\") == \"f2-f4\");\n        ASSERT(chess.process(\"g5\") == \"g6-g5\");\n    }\n\n    {\n        Chessboard chess;\n\n        ASSERT(chess.process(\"e4\") == \"e2-e4\");\n        ASSERT(chess.process(\"c5\") == \"c7-c5\");\n        ASSERT(chess.process(\"e5\") == \"e4-e5\");\n        ASSERT(chess.process(\"c4\") == \"c5-c4\");\n        ASSERT(chess.process(\"e6\") == \"e5-e6\");\n        ASSERT(chess.process(\"c3\") == \"c4-c3\");\n        ASSERT(chess.process(\"e7\") == \"\");\n        ASSERT(chess.process(\"f7\") == \"e6-f7\");\n        ASSERT(chess.process(\"d2\") == \"\");\n        ASSERT(chess.process(\"king to f7\") == \"e8-f7\");\n        ASSERT(chess.process(\"f4\") == \"f2-f4\");\n        ASSERT(chess.process(\"d2\") == \"c3-d2\");\n        ASSERT(chess.process(\"f5\") == \"\");\n        ASSERT(chess.process(\"king to e2\") == \"e1-e2\");\n        ASSERT(chess.process(\"king to g6\") == \"f7-g6\");\n        ASSERT(chess.process(\"f5\") == \"f4-f5\");\n        ASSERT(chess.process(\"e6\") == \"\");\n        ASSERT(chess.process(\"king to h5\") == \"g6-h5\");\n        ASSERT(chess.process(\"g4\") == \"g2-g4\");\n        ASSERT(chess.process(\"king to g5\") == \"h5-g5\");\n        ASSERT(chess.process(\"h4\") == \"h2-h4\");\n        ASSERT(chess.process(\"king to h5\") == \"\");\n        ASSERT(chess.process(\"king to g6\") == \"\");\n        ASSERT(chess.process(\"king to h6\") == \"g5-h6\");\n        ASSERT(chess.process(\"bishop to d2\") == \"c1-d2\");\n        ASSERT(chess.process(\"king to g5\") == \"\");\n        ASSERT(chess.process(\"g5\") == \"g7-g5\");\n    }\n\n    {\n        Chessboard chess;\n        ASSERT(chess.process(\"f4\") == \"f2-f4\");\n        ASSERT(chess.process(\"e5\") == \"e7-e5\");\n        ASSERT(chess.process(\"g4\") == \"g2-g4\");\n        ASSERT(chess.process(\"queen to h4\") == \"d8-h4#\");\n        ASSERT(chess.process(\"knight f3\") == \"\");\n        ASSERT(chess.grammar().empty());\n    }\n\n    {\n        Chessboard chess;\n        ASSERT(chess.process(\"f4\") == \"f2-f4\");\n        ASSERT(chess.process(\"e5\") == \"e7-e5\");\n        ASSERT(chess.process(\"g4\") == \"g2-g4\");\n        ASSERT(chess.process(\"d5\") == \"d7-d5\");\n        ASSERT(chess.process(\"g1 f3\") == \"g1-f3\");\n        ASSERT(chess.process(\"queen to h4\") == \"d8-h4\");\n        ASSERT(!chess.grammar().empty());\n    }\n\n    {\n        Chessboard chess;\n        ASSERT(chess.process(\"knight c3\") == \"b1-c3\");\n        ASSERT(chess.process(\"knight c6\") == \"b8-c6\");\n        ASSERT(chess.process(\"knight b5\") == \"c3-b5\");\n        ASSERT(chess.process(\"knight f6\") == \"g8-f6\");\n        ASSERT(chess.process(\"knight d6\") == \"b5-d6\");\n        ASSERT(chess.process(\"knight d4\") == \"\");\n        ASSERT(chess.process(\"d6\") == \"c7-d6\");\n        ASSERT(chess.process(\"e4\") == \"e2-e4\");\n        ASSERT(chess.process(\"knight d4\") == \"c6-d4\");\n        ASSERT(chess.process(\"d3\") == \"d2-d3\");\n        ASSERT(chess.process(\"knight e4\") == \"f6-e4\");\n        ASSERT(chess.process(\"king to e2\") == \"\");\n        ASSERT(chess.process(\"king to d2\") == \"\");\n    }\n}"
  },
  {
    "path": "examples/wchess/wchess.cmd/CMakeLists.txt",
    "content": "if (WHISPER_SDL2)\n    set(TARGET wchess)\n    add_executable(${TARGET} wchess.cmd.cpp)\n\n    include(DefaultTargetOptions)\n\n    target_link_libraries(${TARGET} PRIVATE wchess-core common-sdl ${CMAKE_THREAD_LIBS_INIT})\nendif ()\n"
  },
  {
    "path": "examples/wchess/wchess.cmd/wchess.cmd.cpp",
    "content": "// Command line voice assisted chess\n//\n// Speak chess move commands to the microphone.\n// The moves will translated to chessboard positions.\n//\n//\n\n#include \"WChess.h\"\n#include \"common-sdl.h\"\n#include <iostream>\n\n#include <memory>\n#include <thread>\n\n// command-line parameters\nstruct whisper_params {\n    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());\n    int32_t prompt_ms  = 5000;\n    int32_t command_ms = 8000;\n    int32_t capture_id = -1;\n    int32_t max_tokens = 32;\n    int32_t audio_ctx  = 0;\n\n    float vad_thold  = 0.6f;\n    float freq_thold = 100.0f;\n\n    float grammar_penalty = 100.0f;\n\n    bool translate     = false;\n    bool print_special = false;\n    bool print_energy  = false;\n    bool no_timestamps = true;\n    bool use_gpu       = true;\n    bool flash_attn    = true;\n\n    std::string language  = \"en\";\n    std::string model     = \"models/ggml-base.en.bin\";\n    std::string fname_out;\n    std::string commands;\n    std::string prompt;\n    std::string context;\n    std::string grammar;\n};\n\nvoid whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"usage: %s [options]\\n\", argv[0]);\n    fprintf(stderr, \"\\n\");\n    fprintf(stderr, \"options:\\n\");\n    fprintf(stderr, \"  -h,         --help           [default] show this help message and exit\\n\");\n    fprintf(stderr, \"  -t N,       --threads N      [%-7d] number of threads to use during computation\\n\", params.n_threads);\n    fprintf(stderr, \"  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\\n\",             params.prompt_ms);\n    fprintf(stderr, \"  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\\n\",            params.command_ms);\n    fprintf(stderr, \"  -c ID,      --capture ID     [%-7d] capture device ID\\n\",                           params.capture_id);\n    fprintf(stderr, \"  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\\n\",    params.max_tokens);\n    fprintf(stderr, \"  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\\n\",                params.audio_ctx);\n    fprintf(stderr, \"  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\\n\",        params.vad_thold);\n    fprintf(stderr, \"  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\\n\",                params.freq_thold);\n    fprintf(stderr, \"  -tr,        --translate      [%-7s] translate from source language to english\\n\",   params.translate ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ps,        --print-special  [%-7s] print special tokens\\n\",                        params.print_special ? \"true\" : \"false\");\n    fprintf(stderr, \"  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\\n\",          params.print_energy ? \"true\" : \"false\");\n    fprintf(stderr, \"  -ng,        --no-gpu         [%-7s] disable GPU\\n\",                                 params.use_gpu ? \"false\" : \"true\");\n    fprintf(stderr, \"  -fa,        --flash-attn     [%-7s] enable flash attention during decoding\\n\",      params.flash_attn ? \"true\" : \"false\");\n    fprintf(stderr, \"  -nfa,       --no-flash-attn  [%-7s] disable flash attention during decoding\\n\",     params.flash_attn ? \"false\" : \"true\");\n    fprintf(stderr, \"  -l LANG,    --language LANG  [%-7s] spoken language\\n\",                             params.language.c_str());\n    fprintf(stderr, \"  -m FNAME,   --model FNAME    [%-7s] model path\\n\",                                  params.model.c_str());\n    fprintf(stderr, \"  -f FNAME,   --file FNAME     [%-7s] text output file name\\n\",                       params.fname_out.c_str());\n    fprintf(stderr, \"  -cmd FNAME, --commands FNAME [%-7s] text file with allowed commands\\n\",             params.commands.c_str());\n    fprintf(stderr, \"  -p,         --prompt         [%-7s] the required activation prompt\\n\",              params.prompt.c_str());\n    fprintf(stderr, \"  -ctx,       --context        [%-7s] sample text to help the transcription\\n\",       params.context.c_str());\n    fprintf(stderr, \"  --grammar-penalty N          [%-7.1f] scales down logits of nongrammar tokens\\n\",   params.grammar_penalty);\n    fprintf(stderr, \"\\n\");\n}\n\nbool whisper_params_parse(int argc, char ** argv, whisper_params & params) {\n    for (int i = 1; i < argc; i++) {\n        std::string arg = argv[i];\n\n        if (arg == \"-h\" || arg == \"--help\") {\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n        else if (arg == \"-t\"   || arg == \"--threads\")       { params.n_threads     = std::stoi(argv[++i]); }\n        else if (arg == \"-pms\" || arg == \"--prompt-ms\")     { params.prompt_ms     = std::stoi(argv[++i]); }\n        else if (arg == \"-cms\" || arg == \"--command-ms\")    { params.command_ms    = std::stoi(argv[++i]); }\n        else if (arg == \"-c\"   || arg == \"--capture\")       { params.capture_id    = std::stoi(argv[++i]); }\n        else if (arg == \"-mt\"  || arg == \"--max-tokens\")    { params.max_tokens    = std::stoi(argv[++i]); }\n        else if (arg == \"-ac\"  || arg == \"--audio-ctx\")     { params.audio_ctx     = std::stoi(argv[++i]); }\n        else if (arg == \"-vth\" || arg == \"--vad-thold\")     { params.vad_thold     = std::stof(argv[++i]); }\n        else if (arg == \"-fth\" || arg == \"--freq-thold\")    { params.freq_thold    = std::stof(argv[++i]); }\n        else if (arg == \"-tr\"  || arg == \"--translate\")     { params.translate     = true; }\n        else if (arg == \"-ps\"  || arg == \"--print-special\") { params.print_special = true; }\n        else if (arg == \"-pe\"  || arg == \"--print-energy\")  { params.print_energy  = true; }\n        else if (arg == \"-ng\"  || arg == \"--no-gpu\")        { params.use_gpu       = false; }\n        else if (arg == \"-fa\"  || arg == \"--flash-attn\")    { params.flash_attn    = true; }\n        else if (arg == \"-nfa\" || arg == \"--no-flash-attn\") { params.flash_attn    = false; }\n        else if (arg == \"-l\"   || arg == \"--language\")      { params.language      = argv[++i]; }\n        else if (arg == \"-m\"   || arg == \"--model\")         { params.model         = argv[++i]; }\n        else if (arg == \"-f\"   || arg == \"--file\")          { params.fname_out     = argv[++i]; }\n        else if (arg == \"-cmd\" || arg == \"--commands\")      { params.commands      = argv[++i]; }\n        else if (arg == \"-p\"   || arg == \"--prompt\")        { params.prompt        = argv[++i]; }\n        else if (arg == \"-ctx\" || arg == \"--context\")       { params.context       = argv[++i]; }\n        else if (                 arg == \"--grammar-penalty\") { params.grammar_penalty = std::stof(argv[++i]); }\n        else {\n            fprintf(stderr, \"error: unknown argument: %s\\n\", arg.c_str());\n            whisper_print_usage(argc, argv, params);\n            exit(0);\n        }\n    }\n\n    return true;\n}\n\nstd::unique_ptr<WChess> g_wchess;\nint g_moveCount = 0;\nvoid set_move(const std::string & move, float) {\n    if (!move.empty()) {\n        g_moveCount++;\n        fprintf(stdout, \"Move: %s\\n\\n\", move.c_str());\n    }\n    else fprintf(stdout, \"Move rejected\\n\\n\");\n    fprintf(stdout, \"%s\\n\", g_wchess->stringify_board().c_str());\n    fprintf(stdout, \"%s\\n\", g_moveCount ? \"White's turn\" : \"Black's turn\");\n}\n\naudio_async g_audio(30*1000);\nbool g_listening = false;\nstd::vector<float> g_pcmf32;\n\nbool read_input() {\n    std::string input;\n    while (true) {\n        fprintf(stdout, \"[(l)isten/(p)ause/(q)uit]: \");\n        std::cin >> input;\n        fprintf(stdout, \"\\n\");\n        if (input[0] == 'q') {\n            fprintf(stdout, \"Quitting\\n\");\n            return false;\n        }\n        if (input[0] == 'l') {\n            if (!g_listening) {\n                fprintf(stdout, \"Listening\\n\");\n                g_listening = true;\n                g_pcmf32.clear();\n                g_audio.resume();\n                g_audio.clear();\n            }\n            else fprintf(stdout, \"Still listening\\n\");\n            return true;\n        }\n        else {\n            if (g_listening) {\n                g_listening = false;\n                g_audio.get(0, g_pcmf32);\n                g_audio.pause();\n                fprintf(stdout, \"Processing\\n\");\n            }\n            else fprintf(stdout, \"Not listening\\n\");\n            return true;\n        }\n    }\n    return true;\n}\n\nbool get_audio(std::vector<float> & pcmf32_cur) {\n    if (!read_input()) return false;\n    if (!g_pcmf32.empty()) pcmf32_cur = std::move(g_pcmf32);\n    else pcmf32_cur.clear();\n    return true;\n}\n\nint main(int argc, char ** argv) {\n    ggml_backend_load_all();\n\n    whisper_params params;\n\n    if (whisper_params_parse(argc, argv, params) == false) {\n        return 1;\n    }\n\n    if (whisper_lang_id(params.language.c_str()) == -1) {\n        fprintf(stderr, \"error: unknown language '%s'\\n\", params.language.c_str());\n        whisper_print_usage(argc, argv, params);\n        exit(0);\n    }\n\n    // whisper init\n\n    struct whisper_context_params cparams = whisper_context_default_params();\n\n    cparams.use_gpu    = params.use_gpu;\n    cparams.flash_attn = params.flash_attn;\n\n    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);\n    if (!ctx) {\n        fprintf(stderr, \"%s: whisper_init_from_file_with_params() failed!\\n\", __func__);\n        return 1;\n    }\n\n    // init audio\n\n    if (!g_audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {\n        fprintf(stderr, \"%s: audio.init() failed!\\n\", __func__);\n        return 1;\n    }\n\n    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n    wparams.offset_ms        = 0;\n    wparams.translate        = false;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.print_realtime   = false;\n    wparams.print_progress   = false;\n    wparams.print_timestamps = true;\n    wparams.print_special    = false;\n    wparams.no_timestamps    = true;\n\n    wparams.max_tokens       = 32;\n    wparams.audio_ctx        = 768; // partial encoder context for better performance\n\n    wparams.temperature     = 0.0f;\n    wparams.temperature_inc = 2.0f;\n    wparams.greedy.best_of  = 1;\n\n    wparams.beam_search.beam_size = 1;\n\n    wparams.language         = \"en\";\n\n    wparams.grammar_penalty = 100.0;\n\n    wparams.initial_prompt = params.context.data();\n\n    WChess::callbacks cb;\n    cb.get_audio = get_audio;\n    cb.set_move = set_move;\n\n    WChess::settings s;\n    s.vad_ms = 2000;\n    s.prompt_ms = params.prompt_ms;\n    s.command_ms = params.command_ms;\n    s.vad_thold = params.vad_thold;\n    s.freq_thold = params.freq_thold;\n    s.print_energy = params.print_energy;\n\n    g_wchess.reset(new WChess(ctx, wparams, cb, s));\n    set_move(\"start\", 0);\n    g_wchess->run();\n\n    whisper_print_timings(ctx);\n    whisper_free(ctx);\n\n    return 0;\n}\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/CMakeLists.txt",
    "content": "set(TARGET wchess.wasm)\n\nadd_executable(${TARGET}\n    wchess.wasm.cpp\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE\n    common\n    wchess-core\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside chess.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/${TARGET}.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/js/chess.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE=8 \\\n    -s INITIAL_MEMORY=1024MB \\\n    -s TOTAL_MEMORY=1024MB \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s EXPORTED_RUNTIME_METHODS=\\\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\\\" \\\n    ${EXTRA_FLAGS} \\\n    \")\n\nadd_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy_directory\n        ${CMAKE_CURRENT_SOURCE_DIR}/chessboardjs-1.0.0\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_CURRENT_SOURCE_DIR}/jquery-3.7.1.min.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/js/\n    )\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)\nconfigure_file(${CMAKE_SOURCE_DIR}/examples/helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/js/helpers.js @ONLY)\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/css/chessboard-1.0.0.css",
    "content": "/*! chessboard.js v1.0.0 | (c) 2019 Chris Oakman | MIT License chessboardjs.com/license */\n\r\n.clearfix-7da63 {\n  clear: both;\r\n}\r\n\r\n.board-b72b1 {\n  border: 2px solid #404040;\n  box-sizing: content-box;\n}\r\n\r\n.square-55d63 {\n  float: left;\r\n  position: relative;\r\n\r\n  /* disable any native browser highlighting */\r\n  -webkit-touch-callout: none;\r\n    -webkit-user-select: none;\r\n     -khtml-user-select: none;\r\n       -moz-user-select: none;\r\n        -ms-user-select: none;\r\n            user-select: none;\r\n}\r\n\r\n.white-1e1d7 {\n  background-color: #f0d9b5;\r\n  color: #b58863;\r\n}\r\n\r\n.black-3c85d {\n  background-color: #b58863;\r\n  color: #f0d9b5;\r\n}\r\n\n.highlight1-32417, .highlight2-9c5d2 {\n  box-shadow: inset 0 0 3px 3px yellow;\r\n}\r\n\n.notation-322f9 {\r\n  cursor: default;\r\n  font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n  font-size: 14px;\r\n  position: absolute;\r\n}\n\n.alpha-d2270 {\r\n  bottom: 1px;\r\n  right: 3px;\r\n}\n\n.numeric-fc462 {\r\n  top: 2px;\r\n  left: 2px;\r\n}\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/js/chessboard-1.0.0/CHANGELOG.md",
    "content": "# chessboard.js Change Log\n\nAll notable changes to this project will be documented in this file.\n\n## [1.0.0] - 2019-06-11\n- Orientation methods now return current orientation. [Issue #64]\n- Drop support for IE8\n- Do not check for `window.JSON` (Error #1004)\n- Rename `ChessBoard` to `Chessboard` (`ChessBoard` is still supported, however)\n- id query selectors are now supported as the first argument to `Chessboard()`\n- Remove Error #1002\n- Format code according to [StandardJS]\n- Bump minimum jQuery version to 1.8.3\n- Throttle piece drag functions\n\n## [0.3.0] - 2013-08-10\n- Added `appearSpeed` animation config property\n- Added `onSnapbackEnd` event\n- Added `onMoveEnd` event\n\n## [0.2.0] - 2013-08-05\n- Added `onMouseoverSquare` and `onMouseoutSquare` events\n- Added `onSnapEnd` event\n- Added square code as CSS class on the squares\n- Added [chess.js] integration examples\n\n## [0.1.0] - 2013-05-21\n- Initial release\n\n[chess.js]:https://github.com/jhlywa/chess.js\n[Issue #64]:https://github.com/oakmac/chessboardjs/issues/64\n[StandardJS]:https://standardjs.com/\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/js/chessboard-1.0.0/LICENSE.md",
    "content": "Copyright 2019 Chris Oakman\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/js/chessboard-1.0.0/README.md",
    "content": "# chessboard.js\n\nchessboard.js is a JavaScript chessboard component. It depends on [jQuery].\n\nPlease see [chessboardjs.com] for documentation and examples.\n\n## What is chessboard.js?\n\nchessboard.js is a JavaScript chessboard component with a flexible \"just a\nboard\" API that\n\nchessboard.js is a standalone JavaScript Chess Board. It is designed to be \"just\na board\" and expose a powerful API so that it can be used in different ways.\nHere's a non-exhaustive list of things you can do with chessboard.js:\n\n- Use chessboard.js to show game positions alongside your expert commentary.\n- Use chessboard.js to have a tactics website where users have to guess the best\n  move.\n- Integrate chessboard.js and [chess.js] with a PGN database and allow people to\n  search and playback games (see [Example 5000])\n- Build a chess server and have users play their games out using the\n  chessboard.js board.\n\nchessboard.js is flexible enough to handle any of these situations with relative\nease.\n\n## What can chessboard.js **not** do?\n\nThe scope of chessboard.js is limited to \"just a board.\" This is intentional and\nmakes chessboard.js flexible for handling a multitude of chess-related problems.\n\nThis is a common source of confusion for new users. [remove?]\n\nSpecifically, chessboard.js does not understand anything about how the game of\nchess is played: how a knight moves, who's turn is it, is White in check?, etc.\n\nFortunately, the powerful [chess.js] library deals with exactly this sort of\nproblem domain and plays nicely with chessboard.js's flexible API. Some examples\nof chessboard.js combined with chess.js: 5000, 5001, 5002\n\nPlease see the powerful [chess.js] library for an API to deal with these sorts\nof questions.\n\n\nThis logic is distinct from the logic of the board. Please see the powerful\n[chess.js] library for this aspect of your application.\n\n\n\nHere is a list of things that chessboard.js is **not**:\n\n- A chess engine\n- A legal move validator\n- A PGN parser\n\nchessboard.js is designed to work well with any of those things, but the idea\nbehind chessboard.js is that the logic that controls the board should be\nindependent of those other problems.\n\n## Docs and Examples\n\n- Docs - <http://chessboardjs.com/docs>\n- Examples - <http://chessboardjs.com/examples>\n\n## Developer Tools\n\n```sh\n# create a build in the build/ directory\nnpm run build\n\n# re-build the website\nnpm run website\n```\n\n## License\n\n[MIT License](LICENSE.md)\n\n[jQuery]:https://jquery.com/\n[chessboardjs.com]:http://chessboardjs.com\n[chess.js]:https://github.com/jhlywa/chess.js\n[Example 5000]:http://chessboardjs.com/examples#5000\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/js/chessboard-1.0.0/package.json",
    "content": "{\n  \"author\": \"Chris Oakman <chris@oakmac.com> (http://chrisoakman.com/)\",\n  \"name\": \"@chrisoakman/chessboardjs\",\n  \"description\": \"JavaScript chessboard widget\",\n  \"homepage\": \"https://chessboardjs.com\",\n  \"license\": \"MIT\",\n  \"version\": \"1.0.0\",\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git://github.com/oakmac/chessboardjs.git\"\n  },\n  \"files\": [\"dist/\"],\n  \"dependencies\": {\n    \"jquery\": \">=3.4.1\"\n  },\n  \"devDependencies\": {\n    \"csso\": \"3.5.1\",\n    \"fs-plus\": \"3.1.1\",\n    \"kidif\": \"1.1.0\",\n    \"mustache\": \"2.3.0\",\n    \"standard\": \"10.0.2\",\n    \"uglify-js\": \"3.6.0\"\n  },\n  \"scripts\": {\n    \"build\": \"standard lib/chessboard.js && node scripts/build.js\",\n    \"standard\": \"standard --fix lib/*.js website/js/*.js\",\n    \"website\": \"node scripts/website.js\"\n  }\n}\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/chessboardjs-1.0.0/js/chessboard-1.0.0.js",
    "content": "// chessboard.js v1.0.0\n// https://github.com/oakmac/chessboardjs/\n//\n// Copyright (c) 2019, Chris Oakman\n// Released under the MIT license\n// https://github.com/oakmac/chessboardjs/blob/master/LICENSE.md\n\r\n// start anonymous scope\r\n;(function () {\r\n  'use strict'\n\n  var $ = window['jQuery']\n\n  // ---------------------------------------------------------------------------\n  // Constants\n  // ---------------------------------------------------------------------------\n\n  var COLUMNS = 'abcdefgh'.split('')\n  var DEFAULT_DRAG_THROTTLE_RATE = 20\n  var ELLIPSIS = '…'\n  var MINIMUM_JQUERY_VERSION = '1.8.3'\n  var RUN_ASSERTS = false\n  var START_FEN = 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR'\n  var START_POSITION = fenToObj(START_FEN)\n\n  // default animation speeds\n  var DEFAULT_APPEAR_SPEED = 200\n  var DEFAULT_MOVE_SPEED = 200\n  var DEFAULT_SNAPBACK_SPEED = 60\n  var DEFAULT_SNAP_SPEED = 30\n  var DEFAULT_TRASH_SPEED = 100\n\n  // use unique class names to prevent clashing with anything else on the page\n  // and simplify selectors\n  // NOTE: these should never change\n  var CSS = {}\n  CSS['alpha'] = 'alpha-d2270'\n  CSS['black'] = 'black-3c85d'\n  CSS['board'] = 'board-b72b1'\n  CSS['chessboard'] = 'chessboard-63f37'\n  CSS['clearfix'] = 'clearfix-7da63'\n  CSS['highlight1'] = 'highlight1-32417'\n  CSS['highlight2'] = 'highlight2-9c5d2'\n  CSS['notation'] = 'notation-322f9'\n  CSS['numeric'] = 'numeric-fc462'\n  CSS['piece'] = 'piece-417db'\n  CSS['row'] = 'row-5277c'\n  CSS['sparePieces'] = 'spare-pieces-7492f'\n  CSS['sparePiecesBottom'] = 'spare-pieces-bottom-ae20f'\n  CSS['sparePiecesTop'] = 'spare-pieces-top-4028b'\n  CSS['square'] = 'square-55d63'\n  CSS['white'] = 'white-1e1d7'\n\n  // ---------------------------------------------------------------------------\n  // Misc Util Functions\n  // ---------------------------------------------------------------------------\n\n  function throttle (f, interval, scope) {\n    var timeout = 0\n    var shouldFire = false\n    var args = []\n\n    var handleTimeout = function () {\n      timeout = 0\n      if (shouldFire) {\n        shouldFire = false\n        fire()\n      }\n    }\n\n    var fire = function () {\n      timeout = window.setTimeout(handleTimeout, interval)\n      f.apply(scope, args)\n    }\n\n    return function (_args) {\n      args = arguments\n      if (!timeout) {\n        fire()\n      } else {\n        shouldFire = true\n      }\n    }\n  }\n\n  // function debounce (f, interval, scope) {\n  //   var timeout = 0\n  //   return function (_args) {\n  //     window.clearTimeout(timeout)\n  //     var args = arguments\n  //     timeout = window.setTimeout(function () {\n  //       f.apply(scope, args)\n  //     }, interval)\n  //   }\n  // }\n\n  function uuid () {\n    return 'xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx-xxxx'.replace(/x/g, function (c) {\n      var r = (Math.random() * 16) | 0\n      return r.toString(16)\n    })\n  }\n\n  function deepCopy (thing) {\n    return JSON.parse(JSON.stringify(thing))\n  }\n\n  function parseSemVer (version) {\n    var tmp = version.split('.')\n    return {\n      major: parseInt(tmp[0], 10),\n      minor: parseInt(tmp[1], 10),\n      patch: parseInt(tmp[2], 10)\n    }\n  }\n\n  // returns true if version is >= minimum\n  function validSemanticVersion (version, minimum) {\n    version = parseSemVer(version)\n    minimum = parseSemVer(minimum)\n\n    var versionNum = (version.major * 100000 * 100000) +\n                     (version.minor * 100000) +\n                     version.patch\n    var minimumNum = (minimum.major * 100000 * 100000) +\n                     (minimum.minor * 100000) +\n                     minimum.patch\n\n    return versionNum >= minimumNum\n  }\n\n  function interpolateTemplate (str, obj) {\n    for (var key in obj) {\n      if (!obj.hasOwnProperty(key)) continue\n      var keyTemplateStr = '{' + key + '}'\n      var value = obj[key]\n      while (str.indexOf(keyTemplateStr) !== -1) {\n        str = str.replace(keyTemplateStr, value)\n      }\n    }\n    return str\n  }\n\n  if (RUN_ASSERTS) {\n    console.assert(interpolateTemplate('abc', {a: 'x'}) === 'abc')\n    console.assert(interpolateTemplate('{a}bc', {}) === '{a}bc')\n    console.assert(interpolateTemplate('{a}bc', {p: 'q'}) === '{a}bc')\n    console.assert(interpolateTemplate('{a}bc', {a: 'x'}) === 'xbc')\n    console.assert(interpolateTemplate('{a}bc{a}bc', {a: 'x'}) === 'xbcxbc')\n    console.assert(interpolateTemplate('{a}{a}{b}', {a: 'x', b: 'y'}) === 'xxy')\n  }\n\n  // ---------------------------------------------------------------------------\n  // Predicates\n  // ---------------------------------------------------------------------------\n\n  function isString (s) {\n    return typeof s === 'string'\n  }\n\n  function isFunction (f) {\n    return typeof f === 'function'\n  }\n\n  function isInteger (n) {\n    return typeof n === 'number' &&\n           isFinite(n) &&\n           Math.floor(n) === n\n  }\n\n  function validAnimationSpeed (speed) {\n    if (speed === 'fast' || speed === 'slow') return true\n    if (!isInteger(speed)) return false\n    return speed >= 0\n  }\n\n  function validThrottleRate (rate) {\n    return isInteger(rate) &&\n           rate >= 1\n  }\n\n  function validMove (move) {\r\n    // move should be a string\r\n    if (!isString(move)) return false\n\r\n    // move should be in the form of \"e2-e4\", \"f6-d5\"\r\n    var squares = move.split('-')\n    if (squares.length !== 2) return false\n\r\n    return validSquare(squares[0]) && validSquare(squares[1])\n  }\r\n\r\n  function validSquare (square) {\n    return isString(square) && square.search(/^[a-h][1-8]$/) !== -1\n  }\r\n\n  if (RUN_ASSERTS) {\n    console.assert(validSquare('a1'))\n    console.assert(validSquare('e2'))\n    console.assert(!validSquare('D2'))\n    console.assert(!validSquare('g9'))\n    console.assert(!validSquare('a'))\n    console.assert(!validSquare(true))\n    console.assert(!validSquare(null))\n    console.assert(!validSquare({}))\n  }\n\n  function validPieceCode (code) {\n    return isString(code) && code.search(/^[bw][KQRNBP]$/) !== -1\n  }\r\n\n  if (RUN_ASSERTS) {\n    console.assert(validPieceCode('bP'))\n    console.assert(validPieceCode('bK'))\n    console.assert(validPieceCode('wK'))\n    console.assert(validPieceCode('wR'))\n    console.assert(!validPieceCode('WR'))\n    console.assert(!validPieceCode('Wr'))\n    console.assert(!validPieceCode('a'))\n    console.assert(!validPieceCode(true))\n    console.assert(!validPieceCode(null))\n    console.assert(!validPieceCode({}))\n  }\n\n  function validFen (fen) {\n    if (!isString(fen)) return false\n\r\n    // cut off any move, castling, etc info from the end\r\n    // we're only interested in position information\r\n    fen = fen.replace(/ .+$/, '')\r\n\n    // expand the empty square numbers to just 1s\n    fen = expandFenEmptySquares(fen)\n\n    // FEN should be 8 sections separated by slashes\r\n    var chunks = fen.split('/')\r\n    if (chunks.length !== 8) return false\n\n    // check each section\n    for (var i = 0; i < 8; i++) {\n      if (chunks[i].length !== 8 ||\n          chunks[i].search(/[^kqrnbpKQRNBP1]/) !== -1) {\n        return false\r\n      }\r\n    }\r\n\r\n    return true\r\n  }\r\n\n  if (RUN_ASSERTS) {\n    console.assert(validFen(START_FEN))\n    console.assert(validFen('8/8/8/8/8/8/8/8'))\n    console.assert(validFen('r1bqkbnr/pppp1ppp/2n5/1B2p3/4P3/5N2/PPPP1PPP/RNBQK2R'))\n    console.assert(validFen('3r3r/1p4pp/2nb1k2/pP3p2/8/PB2PN2/p4PPP/R4RK1 b - - 0 1'))\n    console.assert(!validFen('3r3z/1p4pp/2nb1k2/pP3p2/8/PB2PN2/p4PPP/R4RK1 b - - 0 1'))\n    console.assert(!validFen('anbqkbnr/8/8/8/8/8/PPPPPPPP/8'))\n    console.assert(!validFen('rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/'))\n    console.assert(!validFen('rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBN'))\n    console.assert(!validFen('888888/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR'))\n    console.assert(!validFen('888888/pppppppp/74/8/8/8/PPPPPPPP/RNBQKBNR'))\n    console.assert(!validFen({}))\n  }\n\n  function validPositionObject (pos) {\n    if (!$.isPlainObject(pos)) return false\n\r\n    for (var i in pos) {\r\n      if (!pos.hasOwnProperty(i)) continue\r\n\r\n      if (!validSquare(i) || !validPieceCode(pos[i])) {\r\n        return false\r\n      }\r\n    }\r\n\r\n    return true\r\n  }\r\n\n  if (RUN_ASSERTS) {\n    console.assert(validPositionObject(START_POSITION))\n    console.assert(validPositionObject({}))\n    console.assert(validPositionObject({e2: 'wP'}))\n    console.assert(validPositionObject({e2: 'wP', d2: 'wP'}))\n    console.assert(!validPositionObject({e2: 'BP'}))\n    console.assert(!validPositionObject({y2: 'wP'}))\n    console.assert(!validPositionObject(null))\n    console.assert(!validPositionObject('start'))\n    console.assert(!validPositionObject(START_FEN))\n  }\n\n  function isTouchDevice () {\n    return 'ontouchstart' in document.documentElement\n  }\n\n  function validJQueryVersion () {\n    return typeof window.$ &&\n           $.fn &&\n           $.fn.jquery &&\n           validSemanticVersion($.fn.jquery, MINIMUM_JQUERY_VERSION)\n  }\n\n  // ---------------------------------------------------------------------------\n  // Chess Util Functions\n  // ---------------------------------------------------------------------------\n\n  // convert FEN piece code to bP, wK, etc\n  function fenToPieceCode (piece) {\n    // black piece\n    if (piece.toLowerCase() === piece) {\n      return 'b' + piece.toUpperCase()\n    }\n\n    // white piece\n    return 'w' + piece.toUpperCase()\n  }\n\n  // convert bP, wK, etc code to FEN structure\n  function pieceCodeToFen (piece) {\n    var pieceCodeLetters = piece.split('')\n\n    // white piece\n    if (pieceCodeLetters[0] === 'w') {\n      return pieceCodeLetters[1].toUpperCase()\n    }\n\n    // black piece\n    return pieceCodeLetters[1].toLowerCase()\n  }\n\n  // convert FEN string to position object\r\n  // returns false if the FEN string is invalid\r\n  function fenToObj (fen) {\r\n    if (!validFen(fen)) return false\n\r\n    // cut off any move, castling, etc info from the end\r\n    // we're only interested in position information\r\n    fen = fen.replace(/ .+$/, '')\r\n\r\n    var rows = fen.split('/')\r\n    var position = {}\r\n\r\n    var currentRow = 8\r\n    for (var i = 0; i < 8; i++) {\r\n      var row = rows[i].split('')\r\n      var colIdx = 0\n\r\n      // loop through each character in the FEN section\r\n      for (var j = 0; j < row.length; j++) {\r\n        // number / empty squares\r\n        if (row[j].search(/[1-8]/) !== -1) {\r\n          var numEmptySquares = parseInt(row[j], 10)\n          colIdx = colIdx + numEmptySquares\n        } else {\n          // piece\r\n          var square = COLUMNS[colIdx] + currentRow\n          position[square] = fenToPieceCode(row[j])\r\n          colIdx = colIdx + 1\n        }\r\n      }\r\n\n      currentRow = currentRow - 1\n    }\r\n\r\n    return position\r\n  }\n\n  // position object to FEN string\r\n  // returns false if the obj is not a valid position object\r\n  function objToFen (obj) {\r\n    if (!validPositionObject(obj)) return false\n\r\n    var fen = ''\r\n\r\n    var currentRow = 8\r\n    for (var i = 0; i < 8; i++) {\r\n      for (var j = 0; j < 8; j++) {\r\n        var square = COLUMNS[j] + currentRow\r\n\r\n        // piece exists\r\n        if (obj.hasOwnProperty(square)) {\n          fen = fen + pieceCodeToFen(obj[square])\n        } else {\r\n          // empty space\r\n          fen = fen + '1'\n        }\r\n      }\r\n\r\n      if (i !== 7) {\n        fen = fen + '/'\n      }\n\r\n      currentRow = currentRow - 1\n    }\r\n\r\n    // squeeze the empty numbers together\n    fen = squeezeFenEmptySquares(fen)\n\n    return fen\r\n  }\r\n\n  if (RUN_ASSERTS) {\n    console.assert(objToFen(START_POSITION) === START_FEN)\n    console.assert(objToFen({}) === '8/8/8/8/8/8/8/8')\n    console.assert(objToFen({a2: 'wP', 'b2': 'bP'}) === '8/8/8/8/8/8/Pp6/8')\n  }\n\n  function squeezeFenEmptySquares (fen) {\n    return fen.replace(/11111111/g, '8')\n      .replace(/1111111/g, '7')\n      .replace(/111111/g, '6')\n      .replace(/11111/g, '5')\n      .replace(/1111/g, '4')\n      .replace(/111/g, '3')\n      .replace(/11/g, '2')\n  }\n\n  function expandFenEmptySquares (fen) {\n    return fen.replace(/8/g, '11111111')\n      .replace(/7/g, '1111111')\n      .replace(/6/g, '111111')\n      .replace(/5/g, '11111')\n      .replace(/4/g, '1111')\n      .replace(/3/g, '111')\n      .replace(/2/g, '11')\n  }\n\n  // returns the distance between two squares\n  function squareDistance (squareA, squareB) {\n    var squareAArray = squareA.split('')\n    var squareAx = COLUMNS.indexOf(squareAArray[0]) + 1\n    var squareAy = parseInt(squareAArray[1], 10)\n\n    var squareBArray = squareB.split('')\n    var squareBx = COLUMNS.indexOf(squareBArray[0]) + 1\n    var squareBy = parseInt(squareBArray[1], 10)\n\n    var xDelta = Math.abs(squareAx - squareBx)\n    var yDelta = Math.abs(squareAy - squareBy)\n\n    if (xDelta >= yDelta) return xDelta\n    return yDelta\n  }\n\n  // returns the square of the closest instance of piece\n  // returns false if no instance of piece is found in position\n  function findClosestPiece (position, piece, square) {\n    // create array of closest squares from square\n    var closestSquares = createRadius(square)\n\n    // search through the position in order of distance for the piece\n    for (var i = 0; i < closestSquares.length; i++) {\n      var s = closestSquares[i]\n\n      if (position.hasOwnProperty(s) && position[s] === piece) {\n        return s\n      }\n    }\n\n    return false\n  }\n\n  // returns an array of closest squares from square\n  function createRadius (square) {\n    var squares = []\n\n    // calculate distance of all squares\n    for (var i = 0; i < 8; i++) {\n      for (var j = 0; j < 8; j++) {\n        var s = COLUMNS[i] + (j + 1)\n\n        // skip the square we're starting from\n        if (square === s) continue\n\n        squares.push({\n          square: s,\n          distance: squareDistance(square, s)\n        })\n      }\n    }\n\n    // sort by distance\n    squares.sort(function (a, b) {\n      return a.distance - b.distance\n    })\n\n    // just return the square code\n    var surroundingSquares = []\n    for (i = 0; i < squares.length; i++) {\n      surroundingSquares.push(squares[i].square)\n    }\n\n    return surroundingSquares\n  }\n\n  // given a position and a set of moves, return a new position\n  // with the moves executed\n  function calculatePositionFromMoves (position, moves) {\n    var newPosition = deepCopy(position)\n\n    for (var i in moves) {\n      if (!moves.hasOwnProperty(i)) continue\n\n      // skip the move if the position doesn't have a piece on the source square\n      if (!newPosition.hasOwnProperty(i)) continue\n\n      var piece = newPosition[i]\n      delete newPosition[i]\n      newPosition[moves[i]] = piece\n    }\n\n    return newPosition\n  }\n\n  // TODO: add some asserts here for calculatePositionFromMoves\n\n  // ---------------------------------------------------------------------------\n  // HTML\n  // ---------------------------------------------------------------------------\n\n  function buildContainerHTML (hasSparePieces) {\n    var html = '<div class=\"{chessboard}\">'\n\n    if (hasSparePieces) {\n      html += '<div class=\"{sparePieces} {sparePiecesTop}\"></div>'\n    }\n\n    html += '<div class=\"{board}\"></div>'\n\n    if (hasSparePieces) {\n      html += '<div class=\"{sparePieces} {sparePiecesBottom}\"></div>'\n    }\n\n    html += '</div>'\n\n    return interpolateTemplate(html, CSS)\n  }\n\n  // ---------------------------------------------------------------------------\n  // Config\n  // ---------------------------------------------------------------------------\n\n  function expandConfigArgumentShorthand (config) {\n    if (config === 'start') {\n      config = {position: deepCopy(START_POSITION)}\n    } else if (validFen(config)) {\n      config = {position: fenToObj(config)}\n    } else if (validPositionObject(config)) {\n      config = {position: deepCopy(config)}\n    }\n\n    // config must be an object\n    if (!$.isPlainObject(config)) config = {}\n\n    return config\n  }\n\n  // validate config / set default options\n  function expandConfig (config) {\n    // default for orientation is white\n    if (config.orientation !== 'black') config.orientation = 'white'\n\n    // default for showNotation is true\n    if (config.showNotation !== false) config.showNotation = true\n\n    // default for draggable is false\n    if (config.draggable !== true) config.draggable = false\n\n    // default for dropOffBoard is 'snapback'\n    if (config.dropOffBoard !== 'trash') config.dropOffBoard = 'snapback'\n\n    // default for sparePieces is false\n    if (config.sparePieces !== true) config.sparePieces = false\n\n    // draggable must be true if sparePieces is enabled\n    if (config.sparePieces) config.draggable = true\n\n    // default piece theme is wikipedia\n    if (!config.hasOwnProperty('pieceTheme') ||\n        (!isString(config.pieceTheme) && !isFunction(config.pieceTheme))) {\n      config.pieceTheme = 'img/chesspieces/wikipedia/{piece}.png'\n    }\n\n    // animation speeds\n    if (!validAnimationSpeed(config.appearSpeed)) config.appearSpeed = DEFAULT_APPEAR_SPEED\n    if (!validAnimationSpeed(config.moveSpeed)) config.moveSpeed = DEFAULT_MOVE_SPEED\n    if (!validAnimationSpeed(config.snapbackSpeed)) config.snapbackSpeed = DEFAULT_SNAPBACK_SPEED\n    if (!validAnimationSpeed(config.snapSpeed)) config.snapSpeed = DEFAULT_SNAP_SPEED\n    if (!validAnimationSpeed(config.trashSpeed)) config.trashSpeed = DEFAULT_TRASH_SPEED\n\n    // throttle rate\n    if (!validThrottleRate(config.dragThrottleRate)) config.dragThrottleRate = DEFAULT_DRAG_THROTTLE_RATE\n\n    return config\n  }\n\n  // ---------------------------------------------------------------------------\n  // Dependencies\n  // ---------------------------------------------------------------------------\n\n  // check for a compatible version of jQuery\n  function checkJQuery () {\n    if (!validJQueryVersion()) {\n      var errorMsg = 'Chessboard Error 1005: Unable to find a valid version of jQuery. ' +\n        'Please include jQuery ' + MINIMUM_JQUERY_VERSION + ' or higher on the page' +\n        '\\n\\n' +\n        'Exiting' + ELLIPSIS\n      window.alert(errorMsg)\n      return false\n    }\n\n    return true\n  }\n\n  // return either boolean false or the $container element\n  function checkContainerArg (containerElOrString) {\n    if (containerElOrString === '') {\n      var errorMsg1 = 'Chessboard Error 1001: ' +\n        'The first argument to Chessboard() cannot be an empty string.' +\n        '\\n\\n' +\n        'Exiting' + ELLIPSIS\n      window.alert(errorMsg1)\n      return false\n    }\n\n    // convert containerEl to query selector if it is a string\n    if (isString(containerElOrString) &&\n        containerElOrString.charAt(0) !== '#') {\n      containerElOrString = '#' + containerElOrString\n    }\n\n    // containerEl must be something that becomes a jQuery collection of size 1\n    var $container = $(containerElOrString)\n    if ($container.length !== 1) {\n      var errorMsg2 = 'Chessboard Error 1003: ' +\n        'The first argument to Chessboard() must be the ID of a DOM node, ' +\n        'an ID query selector, or a single DOM node.' +\n        '\\n\\n' +\n        'Exiting' + ELLIPSIS\n      window.alert(errorMsg2)\n      return false\n    }\n\n    return $container\n  }\n\n  // ---------------------------------------------------------------------------\n  // Constructor\n  // ---------------------------------------------------------------------------\n\n  function constructor (containerElOrString, config) {\n    // first things first: check basic dependencies\n    if (!checkJQuery()) return null\n    var $container = checkContainerArg(containerElOrString)\n    if (!$container) return null\n\n    // ensure the config object is what we expect\n    config = expandConfigArgumentShorthand(config)\n    config = expandConfig(config)\n\n    // DOM elements\n    var $board = null\n    var $draggedPiece = null\n    var $sparePiecesTop = null\n    var $sparePiecesBottom = null\n\n    // constructor return object\n    var widget = {}\r\n\r\n    // -------------------------------------------------------------------------\n    // Stateful\n    // -------------------------------------------------------------------------\n\n    var boardBorderSize = 2\n    var currentOrientation = 'white'\n    var currentPosition = {}\n    var draggedPiece = null\n    var draggedPieceLocation = null\n    var draggedPieceSource = null\n    var isDragging = false\n    var sparePiecesElsIds = {}\n    var squareElsIds = {}\n    var squareElsOffsets = {}\n    var squareSize = 16\n\n    // -------------------------------------------------------------------------\n    // Validation / Errors\n    // -------------------------------------------------------------------------\n\r\n    function error (code, msg, obj) {\r\n      // do nothing if showErrors is not set\r\n      if (\r\n        config.hasOwnProperty('showErrors') !== true ||\n          config.showErrors === false\n      ) {\r\n        return\r\n      }\r\n\r\n      var errorText = 'Chessboard Error ' + code + ': ' + msg\n\r\n      // print to console\r\n      if (\r\n        config.showErrors === 'console' &&\n          typeof console === 'object' &&\r\n          typeof console.log === 'function'\r\n      ) {\r\n        console.log(errorText)\r\n        if (arguments.length >= 2) {\r\n          console.log(obj)\r\n        }\r\n        return\r\n      }\r\n\r\n      // alert errors\r\n      if (config.showErrors === 'alert') {\n        if (obj) {\r\n          errorText += '\\n\\n' + JSON.stringify(obj)\r\n        }\r\n        window.alert(errorText)\r\n        return\r\n      }\r\n\r\n      // custom function\n      if (isFunction(config.showErrors)) {\n        config.showErrors(code, msg, obj)\n      }\r\n    }\n\n    function setInitialState () {\n      currentOrientation = config.orientation\n\n      // make sure position is valid\n      if (config.hasOwnProperty('position')) {\n        if (config.position === 'start') {\n          currentPosition = deepCopy(START_POSITION)\n        } else if (validFen(config.position)) {\n          currentPosition = fenToObj(config.position)\n        } else if (validPositionObject(config.position)) {\n          currentPosition = deepCopy(config.position)\n        } else {\n          error(\n            7263,\n            'Invalid value passed to config.position.',\n            config.position\n          )\n        }\n      }\n    }\n\n    // -------------------------------------------------------------------------\n    // DOM Misc\n    // -------------------------------------------------------------------------\n\r\n    // calculates square size based on the width of the container\n    // got a little CSS black magic here, so let me explain:\n    // get the width of the container element (could be anything), reduce by 1 for\n    // fudge factor, and then keep reducing until we find an exact mod 8 for\n    // our square size\n    function calculateSquareSize () {\r\n      var containerWidth = parseInt($container.width(), 10)\n\r\n      // defensive, prevent infinite loop\n      if (!containerWidth || containerWidth <= 0) {\n        return 0\r\n      }\r\n\r\n      // pad one pixel\n      var boardWidth = containerWidth - 1\r\n\r\n      while (boardWidth % 8 !== 0 && boardWidth > 0) {\n        boardWidth = boardWidth - 1\n      }\r\n\r\n      return boardWidth / 8\r\n    }\r\n\r\n    // create random IDs for elements\n    function createElIds () {\n      // squares on the board\n      for (var i = 0; i < COLUMNS.length; i++) {\n        for (var j = 1; j <= 8; j++) {\n          var square = COLUMNS[i] + j\r\n          squareElsIds[square] = square + '-' + uuid()\n        }\r\n      }\r\n\r\n      // spare pieces\n      var pieces = 'KQRNBP'.split('')\n      for (i = 0; i < pieces.length; i++) {\n        var whitePiece = 'w' + pieces[i]\r\n        var blackPiece = 'b' + pieces[i]\r\n        sparePiecesElsIds[whitePiece] = whitePiece + '-' + uuid()\n        sparePiecesElsIds[blackPiece] = blackPiece + '-' + uuid()\n      }\r\n    }\r\n\r\n    // -------------------------------------------------------------------------\n    // Markup Building\n    // -------------------------------------------------------------------------\n\n    function buildBoardHTML (orientation) {\n      if (orientation !== 'black') {\n        orientation = 'white'\n      }\n\n      var html = ''\n\n      // algebraic notation / orientation\n      var alpha = deepCopy(COLUMNS)\n      var row = 8\n      if (orientation === 'black') {\n        alpha.reverse()\n        row = 1\n      }\n\n      var squareColor = 'white'\n      for (var i = 0; i < 8; i++) {\n        html += '<div class=\"{row}\">'\n        for (var j = 0; j < 8; j++) {\n          var square = alpha[j] + row\n\n          html += '<div class=\"{square} ' + CSS[squareColor] + ' ' +\n            'square-' + square + '\" ' +\n            'style=\"width:' + squareSize + 'px;height:' + squareSize + 'px;\" ' +\n            'id=\"' + squareElsIds[square] + '\" ' +\n            'data-square=\"' + square + '\">'\n\n          if (config.showNotation) {\n            // alpha notation\n            if ((orientation === 'white' && row === 1) ||\n                (orientation === 'black' && row === 8)) {\n              html += '<div class=\"{notation} {alpha}\">' + alpha[j] + '</div>'\n            }\n\n            // numeric notation\n            if (j === 0) {\n              html += '<div class=\"{notation} {numeric}\">' + row + '</div>'\n            }\n          }\n\n          html += '</div>' // end .square\n\n          squareColor = (squareColor === 'white') ? 'black' : 'white'\n        }\n        html += '<div class=\"{clearfix}\"></div></div>'\n\n        squareColor = (squareColor === 'white') ? 'black' : 'white'\n\n        if (orientation === 'white') {\n          row = row - 1\n        } else {\n          row = row + 1\n        }\n      }\n\n      return interpolateTemplate(html, CSS)\n    }\n\r\n    function buildPieceImgSrc (piece) {\r\n      if (isFunction(config.pieceTheme)) {\n        return config.pieceTheme(piece)\n      }\r\n\r\n      if (isString(config.pieceTheme)) {\n        return interpolateTemplate(config.pieceTheme, {piece: piece})\n      }\r\n\r\n      // NOTE: this should never happen\n      error(8272, 'Unable to build image source for config.pieceTheme.')\n      return ''\r\n    }\r\n\r\n    function buildPieceHTML (piece, hidden, id) {\n      var html = '<img src=\"' + buildPieceImgSrc(piece) + '\" '\r\n      if (isString(id) && id !== '') {\n        html += 'id=\"' + id + '\" '\r\n      }\r\n      html += 'alt=\"\" ' +\n        'class=\"{piece}\" ' +\n        'data-piece=\"' + piece + '\" ' +\n        'style=\"width:' + squareSize + 'px;' + 'height:' + squareSize + 'px;'\n\n      if (hidden) {\n        html += 'display:none;'\r\n      }\n\n      html += '\" />'\r\n\r\n      return interpolateTemplate(html, CSS)\n    }\r\n\r\n    function buildSparePiecesHTML (color) {\n      var pieces = ['wK', 'wQ', 'wR', 'wB', 'wN', 'wP']\r\n      if (color === 'black') {\r\n        pieces = ['bK', 'bQ', 'bR', 'bB', 'bN', 'bP']\r\n      }\r\n\r\n      var html = ''\r\n      for (var i = 0; i < pieces.length; i++) {\r\n        html += buildPieceHTML(pieces[i], false, sparePiecesElsIds[pieces[i]])\n      }\r\n\r\n      return html\r\n    }\r\n\r\n    // -------------------------------------------------------------------------\n    // Animations\n    // -------------------------------------------------------------------------\n\r\n    function animateSquareToSquare (src, dest, piece, completeFn) {\r\n      // get information about the source and destination squares\n      var $srcSquare = $('#' + squareElsIds[src])\n      var srcSquarePosition = $srcSquare.offset()\n      var $destSquare = $('#' + squareElsIds[dest])\n      var destSquarePosition = $destSquare.offset()\n\r\n      // create the animated piece and absolutely position it\n      // over the source square\n      var animatedPieceId = uuid()\r\n      $('body').append(buildPieceHTML(piece, true, animatedPieceId))\n      var $animatedPiece = $('#' + animatedPieceId)\n      $animatedPiece.css({\n        display: '',\r\n        position: 'absolute',\r\n        top: srcSquarePosition.top,\r\n        left: srcSquarePosition.left\r\n      })\r\n\r\n      // remove original piece from source square\n      $srcSquare.find('.' + CSS.piece).remove()\n\n      function onFinishAnimation1 () {\n        // add the \"real\" piece to the destination square\n        $destSquare.append(buildPieceHTML(piece))\n\r\n        // remove the animated piece\n        $animatedPiece.remove()\n\r\n        // run complete function\n        if (isFunction(completeFn)) {\n          completeFn()\r\n        }\r\n      }\r\n\r\n      // animate the piece to the destination square\n      var opts = {\r\n        duration: config.moveSpeed,\n        complete: onFinishAnimation1\n      }\r\n      $animatedPiece.animate(destSquarePosition, opts)\n    }\r\n\r\n    function animateSparePieceToSquare (piece, dest, completeFn) {\r\n      var srcOffset = $('#' + sparePiecesElsIds[piece]).offset()\n      var $destSquare = $('#' + squareElsIds[dest])\n      var destOffset = $destSquare.offset()\n\r\n      // create the animate piece\n      var pieceId = uuid()\r\n      $('body').append(buildPieceHTML(piece, true, pieceId))\n      var $animatedPiece = $('#' + pieceId)\n      $animatedPiece.css({\n        display: '',\r\n        position: 'absolute',\r\n        left: srcOffset.left,\r\n        top: srcOffset.top\r\n      })\r\n\r\n      // on complete\n      function onFinishAnimation2 () {\n        // add the \"real\" piece to the destination square\n        $destSquare.find('.' + CSS.piece).remove()\n        $destSquare.append(buildPieceHTML(piece))\n\r\n        // remove the animated piece\n        $animatedPiece.remove()\n\r\n        // run complete function\n        if (isFunction(completeFn)) {\n          completeFn()\r\n        }\r\n      }\r\n\r\n      // animate the piece to the destination square\n      var opts = {\r\n        duration: config.moveSpeed,\n        complete: onFinishAnimation2\n      }\r\n      $animatedPiece.animate(destOffset, opts)\n    }\n\n    // execute an array of animations\n    function doAnimations (animations, oldPos, newPos) {\n      if (animations.length === 0) return\n\r\n      var numFinished = 0\r\n      function onFinishAnimation3 () {\n        // exit if all the animations aren't finished\n        numFinished = numFinished + 1\n        if (numFinished !== animations.length) return\n\n        drawPositionInstant()\n\r\n        // run their onMoveEnd function\n        if (isFunction(config.onMoveEnd)) {\n          config.onMoveEnd(deepCopy(oldPos), deepCopy(newPos))\n        }\r\n      }\r\n\r\n      for (var i = 0; i < animations.length; i++) {\n        var animation = animations[i]\n\n        // clear a piece\n        if (animation.type === 'clear') {\n          $('#' + squareElsIds[animation.square] + ' .' + CSS.piece)\n            .fadeOut(config.trashSpeed, onFinishAnimation3)\n\n        // add a piece with no spare pieces - fade the piece onto the square\n        } else if (animation.type === 'add' && !config.sparePieces) {\n          $('#' + squareElsIds[animation.square])\n            .append(buildPieceHTML(animation.piece, true))\n            .find('.' + CSS.piece)\n            .fadeIn(config.appearSpeed, onFinishAnimation3)\n\n        // add a piece with spare pieces - animate from the spares\n        } else if (animation.type === 'add' && config.sparePieces) {\n          animateSparePieceToSquare(animation.piece, animation.square, onFinishAnimation3)\n\n        // move a piece from squareA to squareB\n        } else if (animation.type === 'move') {\n          animateSquareToSquare(animation.source, animation.destination, animation.piece, onFinishAnimation3)\n        }\r\n      }\r\n    }\n\n    // calculate an array of animations that need to happen in order to get\n    // from pos1 to pos2\n    function calculateAnimations (pos1, pos2) {\r\n      // make copies of both\n      pos1 = deepCopy(pos1)\r\n      pos2 = deepCopy(pos2)\r\n\r\n      var animations = []\r\n      var squaresMovedTo = {}\r\n\r\n      // remove pieces that are the same in both positions\n      for (var i in pos2) {\r\n        if (!pos2.hasOwnProperty(i)) continue\r\n\r\n        if (pos1.hasOwnProperty(i) && pos1[i] === pos2[i]) {\r\n          delete pos1[i]\r\n          delete pos2[i]\r\n        }\r\n      }\r\n\r\n      // find all the \"move\" animations\n      for (i in pos2) {\n        if (!pos2.hasOwnProperty(i)) continue\r\n\r\n        var closestPiece = findClosestPiece(pos1, pos2[i], i)\r\n        if (closestPiece) {\n          animations.push({\r\n            type: 'move',\r\n            source: closestPiece,\r\n            destination: i,\r\n            piece: pos2[i]\r\n          })\r\n\r\n          delete pos1[closestPiece]\r\n          delete pos2[i]\r\n          squaresMovedTo[i] = true\r\n        }\r\n      }\n\n      // \"add\" animations\n      for (i in pos2) {\n        if (!pos2.hasOwnProperty(i)) continue\n\r\n        animations.push({\r\n          type: 'add',\r\n          square: i,\r\n          piece: pos2[i]\r\n        })\r\n\r\n        delete pos2[i]\r\n      }\r\n\n      // \"clear\" animations\n      for (i in pos1) {\n        if (!pos1.hasOwnProperty(i)) continue\n\r\n        // do not clear a piece if it is on a square that is the result\n        // of a \"move\", ie: a piece capture\n        if (squaresMovedTo.hasOwnProperty(i)) continue\n\r\n        animations.push({\r\n          type: 'clear',\r\n          square: i,\r\n          piece: pos1[i]\r\n        })\r\n\r\n        delete pos1[i]\r\n      }\r\n\r\n      return animations\r\n    }\r\n\r\n    // -------------------------------------------------------------------------\n    // Control Flow\n    // -------------------------------------------------------------------------\n\n    function drawPositionInstant () {\r\n      // clear the board\n      $board.find('.' + CSS.piece).remove()\n\r\n      // add the pieces\n      for (var i in currentPosition) {\n        if (!currentPosition.hasOwnProperty(i)) continue\n\r\n        $('#' + squareElsIds[i]).append(buildPieceHTML(currentPosition[i]))\n      }\r\n    }\r\n\r\n    function drawBoard () {\r\n      $board.html(buildBoardHTML(currentOrientation, squareSize, config.showNotation))\n      drawPositionInstant()\r\n\r\n      if (config.sparePieces) {\n        if (currentOrientation === 'white') {\n          $sparePiecesTop.html(buildSparePiecesHTML('black'))\n          $sparePiecesBottom.html(buildSparePiecesHTML('white'))\n        } else {\r\n          $sparePiecesTop.html(buildSparePiecesHTML('white'))\n          $sparePiecesBottom.html(buildSparePiecesHTML('black'))\n        }\r\n      }\r\n    }\r\n\n    function setCurrentPosition (position) {\r\n      var oldPos = deepCopy(currentPosition)\n      var newPos = deepCopy(position)\r\n      var oldFen = objToFen(oldPos)\r\n      var newFen = objToFen(newPos)\r\n\r\n      // do nothing if no change in position\n      if (oldFen === newFen) return\r\n\r\n      // run their onChange function\n      if (isFunction(config.onChange)) {\n        config.onChange(oldPos, newPos)\n      }\r\n\r\n      // update state\n      currentPosition = position\n    }\r\n\r\n    function isXYOnSquare (x, y) {\r\n      for (var i in squareElsOffsets) {\n        if (!squareElsOffsets.hasOwnProperty(i)) continue\n\r\n        var s = squareElsOffsets[i]\n        if (x >= s.left &&\n            x < s.left + squareSize &&\n            y >= s.top &&\r\n            y < s.top + squareSize) {\n          return i\r\n        }\r\n      }\r\n\r\n      return 'offboard'\r\n    }\r\n\r\n    // records the XY coords of every square into memory\n    function captureSquareOffsets () {\r\n      squareElsOffsets = {}\n\r\n      for (var i in squareElsIds) {\n        if (!squareElsIds.hasOwnProperty(i)) continue\n\r\n        squareElsOffsets[i] = $('#' + squareElsIds[i]).offset()\n      }\r\n    }\r\n\r\n    function removeSquareHighlights () {\r\n      $board\n        .find('.' + CSS.square)\n        .removeClass(CSS.highlight1 + ' ' + CSS.highlight2)\n    }\r\n\r\n    function snapbackDraggedPiece () {\r\n      // there is no \"snapback\" for spare pieces\n      if (draggedPieceSource === 'spare') {\n        trashDraggedPiece()\r\n        return\r\n      }\r\n\r\n      removeSquareHighlights()\r\n\r\n      // animation complete\n      function complete () {\r\n        drawPositionInstant()\r\n        $draggedPiece.css('display', 'none')\n\r\n        // run their onSnapbackEnd function\n        if (isFunction(config.onSnapbackEnd)) {\n          config.onSnapbackEnd(\n            draggedPiece,\n            draggedPieceSource,\n            deepCopy(currentPosition),\n            currentOrientation\n          )\r\n        }\r\n      }\r\n\r\n      // get source square position\n      var sourceSquarePosition = $('#' + squareElsIds[draggedPieceSource]).offset()\n\r\n      // animate the piece to the target square\n      var opts = {\r\n        duration: config.snapbackSpeed,\n        complete: complete\r\n      }\r\n      $draggedPiece.animate(sourceSquarePosition, opts)\n\r\n      // set state\n      isDragging = false\n    }\r\n\r\n    function trashDraggedPiece () {\r\n      removeSquareHighlights()\r\n\r\n      // remove the source piece\n      var newPosition = deepCopy(currentPosition)\n      delete newPosition[draggedPieceSource]\n      setCurrentPosition(newPosition)\r\n\r\n      // redraw the position\n      drawPositionInstant()\r\n\r\n      // hide the dragged piece\n      $draggedPiece.fadeOut(config.trashSpeed)\n\r\n      // set state\n      isDragging = false\n    }\r\n\r\n    function dropDraggedPieceOnSquare (square) {\r\n      removeSquareHighlights()\r\n\r\n      // update position\n      var newPosition = deepCopy(currentPosition)\n      delete newPosition[draggedPieceSource]\n      newPosition[square] = draggedPiece\n      setCurrentPosition(newPosition)\r\n\r\n      // get target square information\n      var targetSquarePosition = $('#' + squareElsIds[square]).offset()\n\r\n      // animation complete\n      function onAnimationComplete () {\n        drawPositionInstant()\r\n        $draggedPiece.css('display', 'none')\n\r\n        // execute their onSnapEnd function\n        if (isFunction(config.onSnapEnd)) {\n          config.onSnapEnd(draggedPieceSource, square, draggedPiece)\n        }\r\n      }\r\n\r\n      // snap the piece to the target square\n      var opts = {\r\n        duration: config.snapSpeed,\n        complete: onAnimationComplete\n      }\r\n      $draggedPiece.animate(targetSquarePosition, opts)\n\r\n      // set state\n      isDragging = false\n    }\r\n\r\n    function beginDraggingPiece (source, piece, x, y) {\r\n      // run their custom onDragStart function\n      // their custom onDragStart function can cancel drag start\n      if (isFunction(config.onDragStart) &&\n          config.onDragStart(source, piece, deepCopy(currentPosition), currentOrientation) === false) {\n        return\r\n      }\r\n\r\n      // set state\n      isDragging = true\n      draggedPiece = piece\n      draggedPieceSource = source\n\r\n      // if the piece came from spare pieces, location is offboard\n      if (source === 'spare') {\r\n        draggedPieceLocation = 'offboard'\n      } else {\r\n        draggedPieceLocation = source\n      }\r\n\r\n      // capture the x, y coords of all squares in memory\n      captureSquareOffsets()\r\n\r\n      // create the dragged piece\n      $draggedPiece.attr('src', buildPieceImgSrc(piece)).css({\n        display: '',\r\n        position: 'absolute',\r\n        left: x - squareSize / 2,\n        top: y - squareSize / 2\n      })\r\n\r\n      if (source !== 'spare') {\r\n        // highlight the source square and hide the piece\n        $('#' + squareElsIds[source])\n          .addClass(CSS.highlight1)\n          .find('.' + CSS.piece)\n          .css('display', 'none')\n      }\r\n    }\r\n\r\n    function updateDraggedPiece (x, y) {\r\n      // put the dragged piece over the mouse cursor\n      $draggedPiece.css({\n        left: x - squareSize / 2,\n        top: y - squareSize / 2\n      })\r\n\r\n      // get location\n      var location = isXYOnSquare(x, y)\r\n\r\n      // do nothing if the location has not changed\n      if (location === draggedPieceLocation) return\n\r\n      // remove highlight from previous square\n      if (validSquare(draggedPieceLocation)) {\n        $('#' + squareElsIds[draggedPieceLocation]).removeClass(CSS.highlight2)\n      }\r\n\r\n      // add highlight to new square\n      if (validSquare(location)) {\n        $('#' + squareElsIds[location]).addClass(CSS.highlight2)\n      }\r\n\r\n      // run onDragMove\n      if (isFunction(config.onDragMove)) {\n        config.onDragMove(\n          location,\r\n          draggedPieceLocation,\n          draggedPieceSource,\n          draggedPiece,\n          deepCopy(currentPosition),\n          currentOrientation\n        )\r\n      }\r\n\r\n      // update state\n      draggedPieceLocation = location\n    }\r\n\r\n    function stopDraggedPiece (location) {\r\n      // determine what the action should be\n      var action = 'drop'\r\n      if (location === 'offboard' && config.dropOffBoard === 'snapback') {\n        action = 'snapback'\r\n      }\r\n      if (location === 'offboard' && config.dropOffBoard === 'trash') {\n        action = 'trash'\r\n      }\r\n\r\n      // run their onDrop function, which can potentially change the drop action\n      if (isFunction(config.onDrop)) {\n        var newPosition = deepCopy(currentPosition)\n\r\n        // source piece is a spare piece and position is off the board\n        // if (draggedPieceSource === 'spare' && location === 'offboard') {...}\n        // position has not changed; do nothing\n\r\n        // source piece is a spare piece and position is on the board\n        if (draggedPieceSource === 'spare' && validSquare(location)) {\n          // add the piece to the board\n          newPosition[location] = draggedPiece\n        }\r\n\r\n        // source piece was on the board and position is off the board\n        if (validSquare(draggedPieceSource) && location === 'offboard') {\n          // remove the piece from the board\n          delete newPosition[draggedPieceSource]\n        }\r\n\r\n        // source piece was on the board and position is on the board\n        if (validSquare(draggedPieceSource) && validSquare(location)) {\n          // move the piece\n          delete newPosition[draggedPieceSource]\n          newPosition[location] = draggedPiece\n        }\r\n\r\n        var oldPosition = deepCopy(currentPosition)\n\r\n        var result = config.onDrop(\n          draggedPieceSource,\n          location,\r\n          draggedPiece,\n          newPosition,\r\n          oldPosition,\r\n          currentOrientation\n        )\r\n        if (result === 'snapback' || result === 'trash') {\r\n          action = result\r\n        }\r\n      }\r\n\r\n      // do it!\n      if (action === 'snapback') {\r\n        snapbackDraggedPiece()\r\n      } else if (action === 'trash') {\r\n        trashDraggedPiece()\r\n      } else if (action === 'drop') {\r\n        dropDraggedPieceOnSquare(location)\r\n      }\r\n    }\r\n\r\n    // -------------------------------------------------------------------------\n    // Public Methods\n    // -------------------------------------------------------------------------\n\r\n    // clear the board\n    widget.clear = function (useAnimation) {\n      widget.position({}, useAnimation)\r\n    }\n\r\n    // remove the widget from the page\n    widget.destroy = function () {\n      // remove markup\n      $container.html('')\n      $draggedPiece.remove()\n\r\n      // remove event handlers\n      $container.unbind()\n    }\n\r\n    // shorthand method to get the current FEN\n    widget.fen = function () {\r\n      return widget.position('fen')\r\n    }\r\n\r\n    // flip orientation\n    widget.flip = function () {\n      return widget.orientation('flip')\r\n    }\n\n    // move pieces\n    // TODO: this method should be variadic as well as accept an array of moves\n    widget.move = function () {\r\n      // no need to throw an error here; just do nothing\n      // TODO: this should return the current position\n      if (arguments.length === 0) return\r\n\r\n      var useAnimation = true\r\n\r\n      // collect the moves into an object\n      var moves = {}\r\n      for (var i = 0; i < arguments.length; i++) {\r\n        // any \"false\" to this function means no animations\n        if (arguments[i] === false) {\r\n          useAnimation = false\r\n          continue\r\n        }\r\n\r\n        // skip invalid arguments\n        if (!validMove(arguments[i])) {\n          error(2826, 'Invalid move passed to the move method.', arguments[i])\r\n          continue\r\n        }\r\n\r\n        var tmp = arguments[i].split('-')\r\n        moves[tmp[0]] = tmp[1]\r\n      }\r\n\r\n      // calculate position from moves\n      var newPos = calculatePositionFromMoves(currentPosition, moves)\n\r\n      // update the board\n      widget.position(newPos, useAnimation)\r\n\r\n      // return the new position object\n      return newPos\r\n    }\r\n\r\n    widget.orientation = function (arg) {\r\n      // no arguments, return the current orientation\n      if (arguments.length === 0) {\r\n        return currentOrientation\n      }\r\n\r\n      // set to white or black\n      if (arg === 'white' || arg === 'black') {\r\n        currentOrientation = arg\n        drawBoard()\r\n        return currentOrientation\n      }\r\n\r\n      // flip orientation\n      if (arg === 'flip') {\r\n        currentOrientation = currentOrientation === 'white' ? 'black' : 'white'\n        drawBoard()\r\n        return currentOrientation\n      }\r\n\r\n      error(5482, 'Invalid value passed to the orientation method.', arg)\r\n    }\n\n    widget.position = function (position, useAnimation) {\n      // no arguments, return the current position\n      if (arguments.length === 0) {\r\n        return deepCopy(currentPosition)\n      }\r\n\r\n      // get position as FEN\n      if (isString(position) && position.toLowerCase() === 'fen') {\n        return objToFen(currentPosition)\n      }\n\n      // start position\n      if (isString(position) && position.toLowerCase() === 'start') {\n        position = deepCopy(START_POSITION)\r\n      }\r\n\r\n      // convert FEN to position object\n      if (validFen(position)) {\n        position = fenToObj(position)\r\n      }\r\n\r\n      // validate position object\n      if (!validPositionObject(position)) {\n        error(6482, 'Invalid value passed to the position method.', position)\r\n        return\r\n      }\r\n\n      // default for useAnimations is true\n      if (useAnimation !== false) useAnimation = true\n\n      if (useAnimation) {\n        // start the animations\n        var animations = calculateAnimations(currentPosition, position)\n        doAnimations(animations, currentPosition, position)\n\r\n        // set the new position\n        setCurrentPosition(position)\r\n      } else {\r\n        // instant update\n        setCurrentPosition(position)\r\n        drawPositionInstant()\r\n      }\r\n    }\n\n    widget.resize = function () {\n      // calulate the new square size\n      squareSize = calculateSquareSize()\n\r\n      // set board width\n      $board.css('width', squareSize * 8 + 'px')\n\r\n      // set drag piece size\n      $draggedPiece.css({\n        height: squareSize,\n        width: squareSize\n      })\r\n\r\n      // spare pieces\n      if (config.sparePieces) {\n        $container\n          .find('.' + CSS.sparePieces)\n          .css('paddingLeft', squareSize + boardBorderSize + 'px')\n      }\r\n\r\n      // redraw the board\n      drawBoard()\r\n    }\n\n    // set the starting position\n    widget.start = function (useAnimation) {\n      widget.position('start', useAnimation)\r\n    }\n\n    // -------------------------------------------------------------------------\n    // Browser Events\n    // -------------------------------------------------------------------------\n\n    function stopDefault (evt) {\n      evt.preventDefault()\n    }\r\n\r\n    function mousedownSquare (evt) {\n      // do nothing if we're not draggable\n      if (!config.draggable) return\n\n      // do nothing if there is no piece on this square\n      var square = $(this).attr('data-square')\n      if (!validSquare(square)) return\n      if (!currentPosition.hasOwnProperty(square)) return\n\n      beginDraggingPiece(square, currentPosition[square], evt.pageX, evt.pageY)\n    }\n\n    function touchstartSquare (e) {\r\n      // do nothing if we're not draggable\n      if (!config.draggable) return\n\r\n      // do nothing if there is no piece on this square\n      var square = $(this).attr('data-square')\n      if (!validSquare(square)) return\n      if (!currentPosition.hasOwnProperty(square)) return\n\r\n      e = e.originalEvent\r\n      beginDraggingPiece(\r\n        square,\r\n        currentPosition[square],\n        e.changedTouches[0].pageX,\r\n        e.changedTouches[0].pageY\r\n      )\r\n    }\r\n\r\n    function mousedownSparePiece (evt) {\n      // do nothing if sparePieces is not enabled\n      if (!config.sparePieces) return\n\r\n      var piece = $(this).attr('data-piece')\r\n\r\n      beginDraggingPiece('spare', piece, evt.pageX, evt.pageY)\n    }\r\n\r\n    function touchstartSparePiece (e) {\r\n      // do nothing if sparePieces is not enabled\n      if (!config.sparePieces) return\n\r\n      var piece = $(this).attr('data-piece')\r\n\r\n      e = e.originalEvent\r\n      beginDraggingPiece(\r\n        'spare',\r\n        piece,\r\n        e.changedTouches[0].pageX,\r\n        e.changedTouches[0].pageY\r\n      )\r\n    }\r\n\r\n    function mousemoveWindow (evt) {\n      if (isDragging) {\n        updateDraggedPiece(evt.pageX, evt.pageY)\n      }\n    }\n\n    var throttledMousemoveWindow = throttle(mousemoveWindow, config.dragThrottleRate)\n\n    function touchmoveWindow (evt) {\n      // do nothing if we are not dragging a piece\n      if (!isDragging) return\n\r\n      // prevent screen from scrolling\n      evt.preventDefault()\n\r\n      updateDraggedPiece(evt.originalEvent.changedTouches[0].pageX,\n        evt.originalEvent.changedTouches[0].pageY)\n    }\r\n\n    var throttledTouchmoveWindow = throttle(touchmoveWindow, config.dragThrottleRate)\n\n    function mouseupWindow (evt) {\n      // do nothing if we are not dragging a piece\n      if (!isDragging) return\n\r\n      // get the location\n      var location = isXYOnSquare(evt.pageX, evt.pageY)\n\r\n      stopDraggedPiece(location)\r\n    }\r\n\r\n    function touchendWindow (evt) {\n      // do nothing if we are not dragging a piece\n      if (!isDragging) return\n\r\n      // get the location\n      var location = isXYOnSquare(evt.originalEvent.changedTouches[0].pageX,\n        evt.originalEvent.changedTouches[0].pageY)\n\r\n      stopDraggedPiece(location)\r\n    }\r\n\r\n    function mouseenterSquare (evt) {\n      // do not fire this event if we are dragging a piece\n      // NOTE: this should never happen, but it's a safeguard\n      if (isDragging) return\n\n      // exit if they did not provide a onMouseoverSquare function\n      if (!isFunction(config.onMouseoverSquare)) return\n\r\n      // get the square\n      var square = $(evt.currentTarget).attr('data-square')\n\r\n      // NOTE: this should never happen; defensive\n      if (!validSquare(square)) return\n\r\n      // get the piece on this square\n      var piece = false\r\n      if (currentPosition.hasOwnProperty(square)) {\n        piece = currentPosition[square]\n      }\r\n\r\n      // execute their function\n      config.onMouseoverSquare(square, piece, deepCopy(currentPosition), currentOrientation)\n    }\r\n\r\n    function mouseleaveSquare (evt) {\n      // do not fire this event if we are dragging a piece\n      // NOTE: this should never happen, but it's a safeguard\n      if (isDragging) return\n\n      // exit if they did not provide an onMouseoutSquare function\n      if (!isFunction(config.onMouseoutSquare)) return\n\r\n      // get the square\n      var square = $(evt.currentTarget).attr('data-square')\n\r\n      // NOTE: this should never happen; defensive\n      if (!validSquare(square)) return\n\r\n      // get the piece on this square\n      var piece = false\r\n      if (currentPosition.hasOwnProperty(square)) {\n        piece = currentPosition[square]\n      }\r\n\r\n      // execute their function\n      config.onMouseoutSquare(square, piece, deepCopy(currentPosition), currentOrientation)\n    }\r\n\r\n    // -------------------------------------------------------------------------\n    // Initialization\n    // -------------------------------------------------------------------------\n\n    function addEvents () {\r\n      // prevent \"image drag\"\n      $('body').on('mousedown mousemove', '.' + CSS.piece, stopDefault)\n\n      // mouse drag pieces\n      $board.on('mousedown', '.' + CSS.square, mousedownSquare)\n      $container.on('mousedown', '.' + CSS.sparePieces + ' .' + CSS.piece, mousedownSparePiece)\n\r\n      // mouse enter / leave square\n      $board\n        .on('mouseenter', '.' + CSS.square, mouseenterSquare)\n        .on('mouseleave', '.' + CSS.square, mouseleaveSquare)\n\n      // piece drag\n      var $window = $(window)\n      $window\n        .on('mousemove', throttledMousemoveWindow)\n        .on('mouseup', mouseupWindow)\n\r\n      // touch drag pieces\n      if (isTouchDevice()) {\n        $board.on('touchstart', '.' + CSS.square, touchstartSquare)\n        $container.on('touchstart', '.' + CSS.sparePieces + ' .' + CSS.piece, touchstartSparePiece)\n        $window\n          .on('touchmove', throttledTouchmoveWindow)\n          .on('touchend', touchendWindow)\n      }\r\n    }\r\n\r\n    function initDOM () {\n      // create unique IDs for all the elements we will create\n      createElIds()\r\n\r\n      // build board and save it in memory\n      $container.html(buildContainerHTML(config.sparePieces))\n      $board = $container.find('.' + CSS.board)\n\r\n      if (config.sparePieces) {\n        $sparePiecesTop = $container.find('.' + CSS.sparePiecesTop)\n        $sparePiecesBottom = $container.find('.' + CSS.sparePiecesBottom)\n      }\r\n\r\n      // create the drag piece\n      var draggedPieceId = uuid()\r\n      $('body').append(buildPieceHTML('wP', true, draggedPieceId))\n      $draggedPiece = $('#' + draggedPieceId)\n\n      // TODO: need to remove this dragged piece element if the board is no\n      // longer in the DOM\n\n      // get the border size\n      boardBorderSize = parseInt($board.css('borderLeftWidth'), 10)\n\r\n      // set the size and draw the board\n      widget.resize()\r\n    }\r\n\n    // -------------------------------------------------------------------------\n    // Initialization\n    // -------------------------------------------------------------------------\n\n    setInitialState()\n    initDOM()\n    addEvents()\n\n    // return the widget object\n    return widget\n  } // end constructor\n\n  // TODO: do module exports here\n  window['Chessboard'] = constructor\n\n  // support legacy ChessBoard name\n  window['ChessBoard'] = window['Chessboard']\n\n  // expose util functions\r\n  window['Chessboard']['fenToObj'] = fenToObj\n  window['Chessboard']['objToFen'] = objToFen\n})() // end anonymous wrapper\r\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/index-tmpl.html",
    "content": "<!doctype html>\n<html lang=\"en-us\">\n    <head>\n        <title>wchess : voice-controlled chess using Whisper + WebAssembly</title>\n        <script src=\"https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.1/iframeResizer.contentWindow.min.js\"></script>\n\n        <meta name=\"viewport\" content=\"width=device-width, initial-scale=0.7, maximum-scale=1, minimum-scale=0.7, user-scalable=no\"/>\n        <meta name=\"apple-mobile-web-app-capable\" content=\"yes\" />\n\n        <style>\n            #output {\n                width: 100%;\n                height: 100%;\n                margin: 0 auto;\n                margin-top: 10px;\n                border-left: 0px;\n                border-right: 0px;\n                padding-left: 0px;\n                padding-right: 0px;\n                display: block;\n                background-color: black;\n                color: white;\n                font-size: 10px;\n                font-family: 'Lucida Console', Monaco, monospace;\n                outline: none;\n                white-space: pre;\n                overflow-wrap: normal;\n                overflow-x: scroll;\n            }\n            .button {\n                background-color: #000000;\n                color: #FFFFFF;\n                padding: 20px;\n                border-radius: 10px;\n                -moz-border-radius: 10px;\n                -webkit-border-radius: 10px;\n                margin:10px;\n                width:  100px;\n                height:  50px;\n                -webkit-touch-callout: none; /* Safari */\n                -webkit-user-select: none; /* Chrome */\n                -moz-user-select: none; /* Firefox */\n                -ms-user-select: none; /* Internet Explorer/Edge */\n                user-select: none;\n            }\n            button[disabled]{\n                background-color: #cccccc;\n                color: #666666;\n                padding: 20px;\n                border-radius: 10px;\n                -moz-border-radius: 10px;\n                -webkit-border-radius: 10px;\n                margin:10px;\n                width: 100px;\n            }\n            .center {\n                display: flex;\n                justify-content: center;\n                align-items: center;\n                width: 500px;\n            }\n            #description {\n                width: 500px;\n            }\n        </style>\n        <link rel=\"stylesheet\" href=\"css/chessboard-1.0.0.min.css\" integrity=\"sha384-q94+BZtLrkL1/ohfjR8c6L+A6qzNH9R2hBLwyoAfu3i/WCvQjzL2RQJ3uNHDISdU\" crossorigin=\"anonymous\">\n    </head>\n    <body>\n        <div id=\"main-container\">\n            <div id=\"description\">\n                <b>wchess : voice-controlled chess using Whisper + WebAssembly</b>\n\n                <br><br>\n\n                This is a demonstration of using Whisper to recognize voice commands in the browser.\n\n                <br><br>\n\n                Usage:<br>\n\n                <ul>\n                    <li>Select a Whisper model</li>\n                    <li>Accept the microphone permission request if prompted</li>\n                    <li>Hold the button and say a chess move (e.g. \"Knight to c3\")</li>\n                    <li>Release the button and wait for the move to be recognized</li>\n                    <li>Repeat</li>\n                </ul>\n\n                Examples:<br>\n\n                <ul>\n                    <li><b>\"d4\"</b></li>\n                    <li><b>\"e2 e4\"</b></li>\n                    <li><b>\"Knight f3\"</b></li>\n                    <li><b>\"Bishop to b5\"</b></li>\n                </ul>\n\n                Features:<br>\n\n                <ul>\n                    <li>Model quantization for reduced memory footprint (~42MB)</li>\n                    <li><a href=\"https://github.com/ggerganov/whisper.cpp/pull/1229\">Grammar-based sampling</a> for improved recognition accuracy</li>\n                </ul>\n\n                <b>\n                Note that not all chess moves are supported. For example, castling and pawn promotion\n                currently do not work, but can be easily implemented. There could also be some bugs in\n                the move handling logic in general. The main reason for that is to keep the implementation\n                simple. The assumption is that a real application would already have a proper move\n                validation logic in place.<br><br>\n\n                The main purpose of this example is to demonstrate the capabilities of whisper.cpp and\n                its application in the browser for voice recognition locally on your device.\n                </b>\n\n                <br><br>\n\n                You can find more about this project on <a href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/wchess\">GitHub</a>.\n\n                <br><br>\n\n                <b>More examples:</b>\n                    <a href=\"../\">main</a> |\n                    <a href=\"../bench.wasm\">bench</a> |\n                    <a href=\"../stream.wasm\">stream</a> |\n                    <a href=\"../command.wasm\">command</a> |\n\n                <br><br>\n\n            </div>\n\n            <hr>\n\n            <div id=\"model-whisper\">\n                Whisper model: <span id=\"model-whisper-status\"></span>\n                <button id=\"fetch-whisper-tiny-en\" onclick=\"loadWhisper()\">tiny.en (Q8_0, 42 MB)</button>\n                <span id=\"fetch-whisper-progress\"></span>\n                <br><br>\n                <button id=\"clear\" onclick=\"clearCache()\">Clear browser cache</button>\n                <!--\n                    <input type=\"file\" id=\"file\" name=\"file\" onchange=\"loadFile(event, 'whisper.bin')\" />\n                -->\n            </div>\n\n            <div id=\"game\">\n                <br>\n                <div id=\"chessboard\" style=\"width: 500px\"></div>\n                <script src=\"js/jquery-3.7.1.min.js\"></script>\n                <script src=\"js/chessboard-1.0.0.min.js\"></script>\n                <script>\n                    var board = Chessboard('chessboard', 'start')\n                    var move_count = 0;\n                </script>\n\n                <br>\n\n                <div id=\"state\">\n                    Status: <b><span id=\"state-status\">select model</span></b>\n\n                    <div id=\"input\" class=\"center\">\n                        <button id=\"toggler\" class=\"button\" onselectstart=\"return false\" style=\"display: none\">Hold</button>\n                    </div>\n\n                    <pre id=\"state-grammar\">[The grammar will be displayed here]</pre>\n\n                    <pre id=\"state-moves\">[The moves will be displayed here]</pre>\n                </div>\n            </div>\n\n            <hr>\n\n            Debug output:\n            <textarea id=\"output\" rows=\"20\"></textarea>\n\n            <br>\n\n            <b>Troubleshooting</b>\n\n            <br><br>\n\n            The page does some heavy computations, so make sure:\n\n            <ul>\n                <li>To use a modern web browser (e.g. Chrome, Firefox)</li>\n                <li>Your browser supports WASM <a href=\"https://webassembly.org/roadmap/\">Fixed-width SIMD</a></li>\n            </ul>\n\n            <div class=\"cell-version\">\n                <span>\n                    |\n                    Build time: <span class=\"nav-link\">@GIT_DATE@</span> |\n                    Commit hash: <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@\">@GIT_SHA1@</a> |\n                    Commit subject: <span class=\"nav-link\">@GIT_COMMIT_SUBJECT@</span> |\n                    <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/command.wasm\">Source Code</a> |\n                </span>\n            </div>\n        </div>\n\n        <script type=\"text/javascript\" src=\"js/helpers.js\"></script>\n        <script type='text/javascript'>\n            // web audio context\n            var context = null;\n\n            // the command instance\n            var instance = null;\n\n            // model name\n            var model_whisper = null;\n            var model_file = null;\n\n            var module_ready = null;\n\n            var Module = {\n                print: printTextarea,\n                printErr: printTextarea,\n                setStatus: function(text) {\n                    printTextarea('js: ' + text);\n                },\n                monitorRunDependencies: function(left) {\n                },\n                preRun: function() {\n                    printTextarea('js: Preparing ...');\n                },\n                postRun: function() {\n                    printTextarea('js: Module initialized successfully!');\n                    module_ready = true;\n                    initInstance();\n                }\n            };\n\n            function initInstance() {\n                if (!module_ready || !model_file || instance) return\n\n                instance = Module.init(model_file);\n\n                if (instance) {\n                    setStatus('Ready');\n                    printTextarea(\"js: whisper initialized, instance: \" + instance);\n                }\n                else {\n                    printTextarea(\"js: failed to initialize whisper\");\n                }\n            }\n\n            function setStatus(text) {\n                document.getElementById('state-status').innerHTML = text;\n            }\n\n            //\n            // fetch models\n            //\n\n            let dbVersion = 1\n            let dbName    = 'whisper.ggerganov.com';\n            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB\n\n            function storeFS(fname, buf) {\n                // write to WASM file using FS_createDataFile\n                // if the file exists, delete it\n                try {\n                    Module.FS_unlink(fname);\n                } catch (e) {\n                    // ignore\n                }\n\n                Module.FS_createDataFile(\"/\", fname, buf, true, true);\n\n                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);\n\n                document.getElementById('model-whisper-status').innerHTML = 'loaded \"' + model_whisper + '\"!';\n\n                model_file = fname;\n                initInstance();\n            }\n\n            function loadWhisper() {\n                setStatus('Loading')\n                //let url     = 'https://whisper.ggerganov.com/ggml-model-whisper-tiny.en-q8_0.bin';\n                let url     = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q8_0.bin';\n                let dst     = 'whisper.bin';\n                let size_mb = 42;\n\n                model_whisper = 'tiny.en-q8_0';\n\n                document.getElementById('model-whisper-status').innerHTML = 'loading \"' + model_whisper + '\" ... ';\n                document.getElementById('fetch-whisper-tiny-en').style.display = 'none';\n\n                cbProgress = function(p) {\n                    let el = document.getElementById('fetch-whisper-progress');\n                    el.innerHTML = Math.round(100*p) + '%';\n                };\n\n                cbCancel = function() {\n                    var el;\n                    el = document.getElementById('model-whisper-status');  if (el) el.innerHTML = '';\n                };\n\n                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);\n\n                // init audio capture so that the user receives a permission request\n                {\n                    let context = new AudioContext({\n                        sampleRate: 16000,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                    navigator.mediaDevices.getUserMedia({audio: true, video: false})\n                        .then(function(s) {\n                            stream = s;\n                            stream.getTracks().forEach(function(track) {\n                                track.stop();\n                            });\n                        })\n                        .catch(function(err) {\n                            printTextarea('js: error getting audio stream: ' + err);\n                        });\n                    context.close();\n                }\n\n                document.getElementById('toggler').style.display = 'block';\n            }\n\n            //\n            // microphone\n            //\n\n            const kSampleRate = 16000;\n            const kRestartRecording_s = 120;\n            const kIntervalAudio_ms = 250; // pass the recorded audio to the C++ instance at this rate\n\n            var mediaRecorder = null;\n            var doRecording = false;\n            var startTime = 0;\n\n            window.AudioContext = window.AudioContext || window.webkitAudioContext;\n            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;\n\n            function stopRecording() {\n                if (mediaRecorder) {\n                    mediaRecorder.stop();\n                }\n            }\n\n            function startRecording() {\n                if (!context) {\n                    context = new AudioContext({\n                        sampleRate: kSampleRate,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                }\n\n                startTime = Date.now();\n\n                var chunks = [];\n                var stream = null;\n\n                navigator.mediaDevices.getUserMedia({audio: true, video: false})\n                    .then(function(s) {\n                        stream = s;\n                        mediaRecorder = new MediaRecorder(stream);\n                        mediaRecorder.ondataavailable = function(e) {\n                            chunks.push(e.data);\n\n                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });\n                            var reader = new FileReader();\n\n                            reader.onload = function(event) {\n                                var buf = new Uint8Array(reader.result);\n                                context.decodeAudioData(buf.buffer, function(audioBuffer) {\n                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);\n                                    var source = offlineContext.createBufferSource();\n                                    source.buffer = audioBuffer;\n                                    source.connect(offlineContext.destination);\n                                    source.start(0);\n\n                                    offlineContext.startRendering().then(function(renderedBuffer) {\n                                        let audio = renderedBuffer.getChannelData(0);\n                                        printTextarea('js: number of samples: ' + audio.length);\n                                        Module.set_audio(instance, audio);\n                                    });\n\n                                    mediaRecorder = null;\n                                    context = null;\n                                });\n                            }\n\n                            reader.readAsArrayBuffer(blob);\n                        };\n\n                        mediaRecorder.onstop = function(e) {\n                            stream.getTracks().forEach(function(track) {\n                                track.stop();\n                            });\n                        };\n\n                        mediaRecorder.start();\n                    })\n                    .catch(function(err) {\n                        printTextarea('js: error getting audio stream: ' + err);\n                    });\n            }\n\n            //\n            // main\n            //\n\n            var nLines = 0;\n            var movesAll = '';\n\n            // document.body.addEventListener('keydown', function(event) {\n            //     if (event.keyCode === 32) {\n            //         document.getElementById('toggler').innerText = \"\";\n            //         onStart();\n            //     }\n            // }, true);\n\n            // document.body.addEventListener('keyup', function(event) {\n            //     if (event.keyCode === 32) {\n            //         document.getElementById('toggler').innerText = \"Hold\";\n            //         onStop();\n            //     }\n            // }, true);\n\n            document.getElementById('toggler').addEventListener(\"touchstart\", function(event){\n                this.innerText = \"\";\n                onStart();\n            }, true);\n\n            document.getElementById('toggler').addEventListener(\"touchend\", function(event){\n                this.innerText = \"Hold\";\n                onStop();\n            }, true)\n\n            document.getElementById('toggler').addEventListener('mousedown', function(event) {\n                this.innerText = \"\";\n                onStart();\n            }, true);\n\n            document.getElementById('toggler').addEventListener('mouseup', function(event) {\n                this.innerText = \"Hold\";\n                onStop();\n            }, true);\n\n            function onStart() {\n                if (!instance) return;\n                setStatus('Listening');\n\n                startRecording();\n            }\n\n            function onStop() {\n                setStatus('Processing');\n                printTextarea('js: stopping recording ...');\n                stopRecording();\n            }\n\n            function setMove(move, prob) {\n                if (move != null && move.length > 1) {\n                    let gameOver =  move[move.length - 1] === '#';\n                    if (gameOver) {\n                        move = move.substring(0, move.length - 1);\n                        document.getElementById('toggler').disabled = true;\n                    }\n                    board.move(move);\n\n                    movesAll += move + ', prob = ' + prob.toFixed(2) + '% <br>';\n                    nLines++;\n\n                    // if more than 10 lines, remove the first line\n                    if (nLines > 10) {\n                        var i = movesAll.indexOf('<br>');\n                        if (i > 0) {\n                            movesAll = movesAll.substring(i + 4);\n                            nLines--;\n                        }\n                    }\n                    ++move_count;\n                    setStatus(gameOver ? 'Done' : move_count % 2 ? 'Black\\'s turn' : 'White\\'s turn');\n                    document.getElementById('state-moves').innerHTML = movesAll;\n                }\n                else {\n                    setStatus('Failed. ' + (move_count % 2 ? 'Black\\'s turn' : 'White\\'s turn'));\n                }\n            }\n\n            function setGrammar(grammar) {\n                document.getElementById('state-grammar').innerHTML = grammar;\n            }\n\n        </script>\n        <script type=\"text/javascript\" src=\"js/chess.js\"></script>\n    </body>\n</html>\n"
  },
  {
    "path": "examples/wchess/wchess.wasm/wchess.wasm.cpp",
    "content": "#include <WChess.h>\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <thread>\n\nconstexpr int N_THREAD = 8;\n\nstd::vector<struct whisper_context *> g_contexts(4, nullptr);\n\nstd::mutex  g_mutex;\nstd::thread g_worker;\n\nstd::condition_variable g_cv;\n\nbool g_running(false);\nstd::vector<float> g_pcmf32;\n\nvoid set_move(const std::string & move, float prob) {\n    MAIN_THREAD_EM_ASM({\n        setMove(UTF8ToString($0), $1)\n    }, move.c_str(), prob);\n}\n\nvoid set_grammar(const std::string & grammar) {\n    MAIN_THREAD_EM_ASM({\n        setGrammar(UTF8ToString($0))\n    }, grammar.c_str());\n}\n\nbool get_audio(std::vector<float> & audio) {\n    std::unique_lock<std::mutex> lock(g_mutex);\n    g_cv.wait(lock, [] { return !g_running || !g_pcmf32.empty(); });\n    if (!g_running) return false;\n    audio = std::move(g_pcmf32);\n    return true;\n}\n\nvoid wchess_main(size_t i) {\n    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n\n    wparams.n_threads        = std::min(N_THREAD, (int) std::thread::hardware_concurrency());\n    wparams.offset_ms        = 0;\n    wparams.translate        = false;\n    wparams.no_context       = true;\n    wparams.single_segment   = true;\n    wparams.print_realtime   = false;\n    wparams.print_progress   = false;\n    wparams.print_timestamps = true;\n    wparams.print_special    = false;\n    wparams.no_timestamps    = true;\n\n    wparams.max_tokens       = 32;\n    wparams.audio_ctx        = 1280; // partial encoder context for better performance\n\n    wparams.temperature      = 0.0f;\n    wparams.temperature_inc  = 2.0f;\n    wparams.greedy.best_of   = 1;\n\n    wparams.beam_search.beam_size = 1;\n\n    wparams.language         = \"en\";\n\n    wparams.grammar_penalty = 100.0;\n    wparams.initial_prompt = \"bishop to c3, rook to d4, knight to e5, d4 d5, knight to c3, c3, queen to d4, king b1, pawn to a1, bishop to b2, knight to c3,\";\n\n    printf(\"command: using %d threads\\n\", wparams.n_threads);\n\n    WChess::callbacks cb;\n    cb.get_audio = get_audio;\n    cb.set_move = set_move;\n    cb.set_grammar = set_grammar;\n\n    WChess(g_contexts[i], wparams, cb, {}).run();\n\n    if (i < g_contexts.size()) {\n        whisper_free(g_contexts[i]);\n        g_contexts[i] = nullptr;\n    }\n}\n\nEMSCRIPTEN_BINDINGS(command) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model) {\n        for (size_t i = 0; i < g_contexts.size(); ++i) {\n            if (g_contexts[i] == nullptr) {\n                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n                if (g_contexts[i] != nullptr) {\n                    g_running = true;\n                    if (g_worker.joinable()) {\n                        g_worker.join();\n                    }\n                    g_worker = std::thread([i]() {\n                        wchess_main(i);\n                    });\n\n                    return i + 1;\n                } else {\n                    return (size_t) 0;\n                }\n            }\n        }\n\n        return (size_t) 0;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([](size_t /* index */) {\n        {\n            std::unique_lock<std::mutex> lock(g_mutex);\n            g_running = false;\n        }\n        g_cv.notify_one();\n    }));\n\n    emscripten::function(\"set_audio\", emscripten::optional_override([](size_t index, const emscripten::val & audio) {\n        --index;\n\n        if (index >= g_contexts.size()) {\n            return -1;\n        }\n\n        if (g_contexts[index] == nullptr) {\n            return -2;\n        }\n\n        {\n            std::lock_guard<std::mutex> lock(g_mutex);\n            const int n = audio[\"length\"].as<int>();\n\n            emscripten::val heap = emscripten::val::module_property(\"HEAPU8\");\n            emscripten::val memory = heap[\"buffer\"];\n\n            g_pcmf32.resize(n);\n\n            emscripten::val memoryView = audio[\"constructor\"].new_(memory, reinterpret_cast<uintptr_t>(g_pcmf32.data()), n);\n            memoryView.call<void>(\"set\", audio);\n        }\n        g_cv.notify_one();\n\n        return 0;\n    }));\n}\n"
  },
  {
    "path": "examples/whisper.android/.gitignore",
    "content": "*.iml\n.gradle\n/local.properties\n/.idea/caches\n/.idea/libraries\n/.idea/modules.xml\n/.idea/workspace.xml\n/.idea/navEditor.xml\n/.idea/assetWizardSettings.xml\n.DS_Store\n/build\n/captures\n.externalNativeBuild\n.cxx\nlocal.properties\n"
  },
  {
    "path": "examples/whisper.android/README.md",
    "content": "A sample Android app using [whisper.cpp](https://github.com/ggerganov/whisper.cpp/) to do voice-to-text transcriptions.\n\nTo use:\n\n1. Select a model from the [whisper.cpp repository](https://github.com/ggerganov/whisper.cpp/tree/master/models).[^1]\n2. Copy the model to the \"app/src/main/assets/models\" folder.\n3. Select a sample audio file (for example, [jfk.wav](https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav)).\n4. Copy the sample to the \"app/src/main/assets/samples\" folder.\n5. Select the \"release\" active build variant, and use Android Studio to run and deploy to your device.\n[^1]: I recommend the tiny or base models for running on an Android device.\n\n(PS: Do not move this android project folder individually to other folders, because this android project folder depends on the files of the whole project.)\n\n<img width=\"300\" alt=\"image\" src=\"https://user-images.githubusercontent.com/1670775/221613663-a17bf770-27ef-45ab-9a46-a5f99ba65d2a.jpg\">\n"
  },
  {
    "path": "examples/whisper.android/app/.gitignore",
    "content": "/build"
  },
  {
    "path": "examples/whisper.android/app/build.gradle",
    "content": "plugins {\n    id 'com.android.application'\n    id 'org.jetbrains.kotlin.android'\n}\n\nandroid {\n    namespace 'com.whispercppdemo'\n    compileSdk 34\n\n    defaultConfig {\n        applicationId \"com.whispercppdemo\"\n        minSdk 26\n        targetSdk 34\n        versionCode 1\n        versionName \"1.0\"\n\n        testInstrumentationRunner \"androidx.test.runner.AndroidJUnitRunner\"\n        vectorDrawables {\n            useSupportLibrary true\n        }\n\n    }\n\n    buildTypes {\n        release {\n            signingConfig signingConfigs.debug\n            minifyEnabled true\n            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'\n        }\n    }\n    compileOptions {\n        sourceCompatibility JavaVersion.VERSION_17\n        targetCompatibility JavaVersion.VERSION_17\n    }\n    kotlinOptions {\n        jvmTarget = '17'\n    }\n    buildFeatures {\n        compose true\n    }\n    composeOptions {\n        kotlinCompilerExtensionVersion '1.5.0'\n    }\n}\n\ndependencies {\n    implementation project(':lib')\n    implementation 'androidx.activity:activity-compose:1.7.2'\n    implementation 'androidx.compose.material:material-icons-core:1.5.0'\n    implementation 'androidx.compose.material3:material3:1.1.1'\n    implementation \"androidx.compose.ui:ui:1.5.0\"\n    implementation \"androidx.compose.ui:ui-tooling-preview:1.5.0\"\n    implementation 'androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1'\n    implementation \"com.google.accompanist:accompanist-permissions:0.28.0\"\n    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.2'\n\n    testImplementation 'junit:junit:4.13.2'\n    androidTestImplementation 'androidx.test.ext:junit:1.1.5'\n    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'\n    androidTestImplementation \"androidx.compose.ui:ui-test-junit4:1.5.0\"\n    debugImplementation \"androidx.compose.ui:ui-tooling:1.5.0\"\n    debugImplementation \"androidx.compose.ui:ui-test-manifest:1.5.0\"\n}"
  },
  {
    "path": "examples/whisper.android/app/proguard-rules.pro",
    "content": "# Add project specific ProGuard rules here.\n# You can control the set of applied configuration files using the\n# proguardFiles setting in build.gradle.\n#\n# For more details, see\n#   http://developer.android.com/guide/developing/tools/proguard.html\n\n# If your project uses WebView with JS, uncomment the following\n# and specify the fully qualified class name to the JavaScript interface\n# class:\n#-keepclassmembers class fqcn.of.javascript.interface.for.webview {\n#   public *;\n#}\n\n# Uncomment this to preserve the line number information for\n# debugging stack traces.\n#-keepattributes SourceFile,LineNumberTable\n\n# If you keep the line number information, uncomment this to\n# hide the original source file name.\n#-renamesourcefileattribute SourceFile"
  },
  {
    "path": "examples/whisper.android/app/src/androidTest/java/com/whispercppdemo/ExampleInstrumentedTest.kt",
    "content": "package com.whispercppdemo\n\nimport androidx.test.platform.app.InstrumentationRegistry\nimport androidx.test.ext.junit.runners.AndroidJUnit4\n\nimport org.junit.Test\nimport org.junit.runner.RunWith\n\nimport org.junit.Assert.*\n\n/**\n * Instrumented test, which will execute on an Android device.\n *\n * See [testing documentation](http://d.android.com/tools/testing).\n */\n@RunWith(AndroidJUnit4::class)\nclass ExampleInstrumentedTest {\n    @Test\n    fun useAppContext() {\n        // Context of the app under test.\n        val appContext = InstrumentationRegistry.getInstrumentation().targetContext\n        assertEquals(\"com.whispercppdemo\", appContext.packageName)\n    }\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/AndroidManifest.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<manifest xmlns:android=\"http://schemas.android.com/apk/res/android\"\n    xmlns:tools=\"http://schemas.android.com/tools\">\n\n    <uses-permission android:name=\"android.permission.RECORD_AUDIO\" />\n\n    <application\n        android:allowBackup=\"true\"\n        android:dataExtractionRules=\"@xml/data_extraction_rules\"\n        android:fullBackupContent=\"@xml/backup_rules\"\n        android:icon=\"@mipmap/ic_launcher\"\n        android:label=\"@string/app_name\"\n        android:supportsRtl=\"true\"\n        android:theme=\"@style/Theme.WhisperCppDemo\"\n        tools:targetApi=\"31\">\n        <activity\n            android:name=\".MainActivity\"\n            android:exported=\"true\"\n            android:theme=\"@style/Theme.WhisperCppDemo\">\n            <intent-filter>\n                <action android:name=\"android.intent.action.MAIN\" />\n\n                <category android:name=\"android.intent.category.LAUNCHER\" />\n            </intent-filter>\n\n            <meta-data\n                android:name=\"android.app.lib_name\"\n                android:value=\"\" />\n        </activity>\n    </application>\n\n</manifest>"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/MainActivity.kt",
    "content": "package com.whispercppdemo\n\nimport android.os.Bundle\nimport androidx.activity.ComponentActivity\nimport androidx.activity.compose.setContent\nimport androidx.activity.viewModels\nimport com.whispercppdemo.ui.main.MainScreen\nimport com.whispercppdemo.ui.main.MainScreenViewModel\nimport com.whispercppdemo.ui.theme.WhisperCppDemoTheme\n\nclass MainActivity : ComponentActivity() {\n    private val viewModel: MainScreenViewModel by viewModels { MainScreenViewModel.factory() }\n\n    override fun onCreate(savedInstanceState: Bundle?) {\n        super.onCreate(savedInstanceState)\n        setContent {\n            WhisperCppDemoTheme {\n                MainScreen(viewModel)\n            }\n        }\n    }\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/media/RiffWaveHelper.kt",
    "content": "package com.whispercppdemo.media\n\nimport java.io.ByteArrayOutputStream\nimport java.io.File\nimport java.nio.ByteBuffer\nimport java.nio.ByteOrder\n\nfun decodeWaveFile(file: File): FloatArray {\n    val baos = ByteArrayOutputStream()\n    file.inputStream().use { it.copyTo(baos) }\n    val buffer = ByteBuffer.wrap(baos.toByteArray())\n    buffer.order(ByteOrder.LITTLE_ENDIAN)\n    val channel = buffer.getShort(22).toInt()\n    buffer.position(44)\n    val shortBuffer = buffer.asShortBuffer()\n    val shortArray = ShortArray(shortBuffer.limit())\n    shortBuffer.get(shortArray)\n    return FloatArray(shortArray.size / channel) { index ->\n        when (channel) {\n            1 -> (shortArray[index] / 32767.0f).coerceIn(-1f..1f)\n            else -> ((shortArray[2*index] + shortArray[2*index + 1])/ 32767.0f / 2.0f).coerceIn(-1f..1f)\n        }\n    }\n}\n\nfun encodeWaveFile(file: File, data: ShortArray) {\n    file.outputStream().use {\n        it.write(headerBytes(data.size * 2))\n        val buffer = ByteBuffer.allocate(data.size * 2)\n        buffer.order(ByteOrder.LITTLE_ENDIAN)\n        buffer.asShortBuffer().put(data)\n        val bytes = ByteArray(buffer.limit())\n        buffer.get(bytes)\n        it.write(bytes)\n    }\n}\n\nprivate fun headerBytes(totalLength: Int): ByteArray {\n    require(totalLength >= 44)\n    ByteBuffer.allocate(44).apply {\n        order(ByteOrder.LITTLE_ENDIAN)\n\n        put('R'.code.toByte())\n        put('I'.code.toByte())\n        put('F'.code.toByte())\n        put('F'.code.toByte())\n\n        putInt(totalLength - 8)\n\n        put('W'.code.toByte())\n        put('A'.code.toByte())\n        put('V'.code.toByte())\n        put('E'.code.toByte())\n\n        put('f'.code.toByte())\n        put('m'.code.toByte())\n        put('t'.code.toByte())\n        put(' '.code.toByte())\n\n        putInt(16)\n        putShort(1.toShort())\n        putShort(1.toShort())\n        putInt(16000)\n        putInt(32000)\n        putShort(2.toShort())\n        putShort(16.toShort())\n\n        put('d'.code.toByte())\n        put('a'.code.toByte())\n        put('t'.code.toByte())\n        put('a'.code.toByte())\n\n        putInt(totalLength - 44)\n        position(0)\n    }.also {\n        val bytes = ByteArray(it.limit())\n        it.get(bytes)\n        return bytes\n    }\n}\n"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/recorder/Recorder.kt",
    "content": "package com.whispercppdemo.recorder\n\nimport android.annotation.SuppressLint\nimport android.media.AudioFormat\nimport android.media.AudioRecord\nimport android.media.MediaRecorder\nimport com.whispercppdemo.media.encodeWaveFile\nimport kotlinx.coroutines.CoroutineScope\nimport kotlinx.coroutines.asCoroutineDispatcher\nimport kotlinx.coroutines.withContext\nimport java.io.File\nimport java.util.concurrent.Executors\nimport java.util.concurrent.atomic.AtomicBoolean\n\nclass Recorder {\n    private val scope: CoroutineScope = CoroutineScope(\n        Executors.newSingleThreadExecutor().asCoroutineDispatcher()\n    )\n    private var recorder: AudioRecordThread? = null\n\n    suspend fun startRecording(outputFile: File, onError: (Exception) -> Unit) = withContext(scope.coroutineContext) {\n        recorder = AudioRecordThread(outputFile, onError)\n        recorder?.start()\n    }\n\n    suspend fun stopRecording() = withContext(scope.coroutineContext) {\n        recorder?.stopRecording()\n        @Suppress(\"BlockingMethodInNonBlockingContext\")\n        recorder?.join()\n        recorder = null\n    }\n}\n\nprivate class AudioRecordThread(\n    private val outputFile: File,\n    private val onError: (Exception) -> Unit\n) :\n    Thread(\"AudioRecorder\") {\n    private var quit = AtomicBoolean(false)\n\n    @SuppressLint(\"MissingPermission\")\n    override fun run() {\n        try {\n            val bufferSize = AudioRecord.getMinBufferSize(\n                16000,\n                AudioFormat.CHANNEL_IN_MONO,\n                AudioFormat.ENCODING_PCM_16BIT\n            ) * 4\n            val buffer = ShortArray(bufferSize / 2)\n\n            val audioRecord = AudioRecord(\n                MediaRecorder.AudioSource.MIC,\n                16000,\n                AudioFormat.CHANNEL_IN_MONO,\n                AudioFormat.ENCODING_PCM_16BIT,\n                bufferSize\n            )\n\n            try {\n                audioRecord.startRecording()\n\n                val allData = mutableListOf<Short>()\n\n                while (!quit.get()) {\n                    val read = audioRecord.read(buffer, 0, buffer.size)\n                    if (read > 0) {\n                        for (i in 0 until read) {\n                            allData.add(buffer[i])\n                        }\n                    } else {\n                        throw java.lang.RuntimeException(\"audioRecord.read returned $read\")\n                    }\n                }\n\n                audioRecord.stop()\n                encodeWaveFile(outputFile, allData.toShortArray())\n            } finally {\n                audioRecord.release()\n            }\n        } catch (e: Exception) {\n            onError(e)\n        }\n    }\n\n    fun stopRecording() {\n        quit.set(true)\n    }\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/ui/main/MainScreen.kt",
    "content": "package com.whispercppdemo.ui.main\n\nimport androidx.compose.foundation.layout.*\nimport androidx.compose.foundation.rememberScrollState\nimport androidx.compose.foundation.text.selection.SelectionContainer\nimport androidx.compose.foundation.verticalScroll\nimport androidx.compose.material3.*\nimport androidx.compose.runtime.Composable\nimport androidx.compose.ui.Modifier\nimport androidx.compose.ui.res.stringResource\nimport androidx.compose.ui.unit.dp\nimport com.google.accompanist.permissions.ExperimentalPermissionsApi\nimport com.google.accompanist.permissions.isGranted\nimport com.google.accompanist.permissions.rememberPermissionState\nimport com.whispercppdemo.R\n\n@Composable\nfun MainScreen(viewModel: MainScreenViewModel) {\n    MainScreen(\n        canTranscribe = viewModel.canTranscribe,\n        isRecording = viewModel.isRecording,\n        messageLog = viewModel.dataLog,\n        onBenchmarkTapped = viewModel::benchmark,\n        onTranscribeSampleTapped = viewModel::transcribeSample,\n        onRecordTapped = viewModel::toggleRecord\n    )\n}\n\n@OptIn(ExperimentalMaterial3Api::class)\n@Composable\nprivate fun MainScreen(\n    canTranscribe: Boolean,\n    isRecording: Boolean,\n    messageLog: String,\n    onBenchmarkTapped: () -> Unit,\n    onTranscribeSampleTapped: () -> Unit,\n    onRecordTapped: () -> Unit\n) {\n    Scaffold(\n        topBar = {\n            TopAppBar(\n                title = { Text(stringResource(R.string.app_name)) }\n            )\n        },\n    ) { innerPadding ->\n        Column(\n            modifier = Modifier\n                .padding(innerPadding)\n                .padding(16.dp)\n        ) {\n            Column(verticalArrangement = Arrangement.SpaceBetween) {\n                Row(horizontalArrangement = Arrangement.SpaceBetween, modifier = Modifier.fillMaxWidth()) {\n                    BenchmarkButton(enabled = canTranscribe, onClick = onBenchmarkTapped)\n                    TranscribeSampleButton(enabled = canTranscribe, onClick = onTranscribeSampleTapped)\n                }\n                RecordButton(\n                    enabled = canTranscribe,\n                    isRecording = isRecording,\n                    onClick = onRecordTapped\n                )\n            }\n            MessageLog(messageLog)\n        }\n    }\n}\n\n@Composable\nprivate fun MessageLog(log: String) {\n    SelectionContainer {\n        Text(modifier = Modifier.verticalScroll(rememberScrollState()), text = log)\n    }\n}\n\n@Composable\nprivate fun BenchmarkButton(enabled: Boolean, onClick: () -> Unit) {\n    Button(onClick = onClick, enabled = enabled) {\n        Text(\"Benchmark\")\n    }\n}\n\n@Composable\nprivate fun TranscribeSampleButton(enabled: Boolean, onClick: () -> Unit) {\n    Button(onClick = onClick, enabled = enabled) {\n        Text(\"Transcribe sample\")\n    }\n}\n\n@OptIn(ExperimentalPermissionsApi::class)\n@Composable\nprivate fun RecordButton(enabled: Boolean, isRecording: Boolean, onClick: () -> Unit) {\n    val micPermissionState = rememberPermissionState(\n        permission = android.Manifest.permission.RECORD_AUDIO,\n        onPermissionResult = { granted ->\n            if (granted) {\n                onClick()\n            }\n        }\n    )\n    Button(onClick = {\n        if (micPermissionState.status.isGranted) {\n            onClick()\n        } else {\n            micPermissionState.launchPermissionRequest()\n        }\n     }, enabled = enabled) {\n        Text(\n            if (isRecording) {\n                \"Stop recording\"\n            } else {\n                \"Start recording\"\n            }\n        )\n    }\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/ui/main/MainScreenViewModel.kt",
    "content": "package com.whispercppdemo.ui.main\n\nimport android.app.Application\nimport android.content.Context\nimport android.media.MediaPlayer\nimport android.util.Log\nimport androidx.compose.runtime.getValue\nimport androidx.compose.runtime.mutableStateOf\nimport androidx.compose.runtime.setValue\nimport androidx.core.net.toUri\nimport androidx.lifecycle.ViewModel\nimport androidx.lifecycle.ViewModelProvider\nimport androidx.lifecycle.viewModelScope\nimport androidx.lifecycle.viewmodel.initializer\nimport androidx.lifecycle.viewmodel.viewModelFactory\nimport com.whispercppdemo.media.decodeWaveFile\nimport com.whispercppdemo.recorder.Recorder\nimport com.whispercpp.whisper.WhisperContext\nimport kotlinx.coroutines.Dispatchers\nimport kotlinx.coroutines.launch\nimport kotlinx.coroutines.runBlocking\nimport kotlinx.coroutines.withContext\nimport java.io.File\n\nprivate const val LOG_TAG = \"MainScreenViewModel\"\n\nclass MainScreenViewModel(private val application: Application) : ViewModel() {\n    var canTranscribe by mutableStateOf(false)\n        private set\n    var dataLog by mutableStateOf(\"\")\n        private set\n    var isRecording by mutableStateOf(false)\n        private set\n\n    private val modelsPath = File(application.filesDir, \"models\")\n    private val samplesPath = File(application.filesDir, \"samples\")\n    private var recorder: Recorder = Recorder()\n    private var whisperContext: com.whispercpp.whisper.WhisperContext? = null\n    private var mediaPlayer: MediaPlayer? = null\n    private var recordedFile: File? = null\n\n    init {\n        viewModelScope.launch {\n            printSystemInfo()\n            loadData()\n        }\n    }\n\n    private suspend fun printSystemInfo() {\n        printMessage(String.format(\"System Info: %s\\n\", com.whispercpp.whisper.WhisperContext.getSystemInfo()))\n    }\n\n    private suspend fun loadData() {\n        printMessage(\"Loading data...\\n\")\n        try {\n            copyAssets()\n            loadBaseModel()\n            canTranscribe = true\n        } catch (e: Exception) {\n            Log.w(LOG_TAG, e)\n            printMessage(\"${e.localizedMessage}\\n\")\n        }\n    }\n\n    private suspend fun printMessage(msg: String) = withContext(Dispatchers.Main) {\n        dataLog += msg\n    }\n\n    private suspend fun copyAssets() = withContext(Dispatchers.IO) {\n        modelsPath.mkdirs()\n        samplesPath.mkdirs()\n        //application.copyData(\"models\", modelsPath, ::printMessage)\n        application.copyData(\"samples\", samplesPath, ::printMessage)\n        printMessage(\"All data copied to working directory.\\n\")\n    }\n\n    private suspend fun loadBaseModel() = withContext(Dispatchers.IO) {\n        printMessage(\"Loading model...\\n\")\n        val models = application.assets.list(\"models/\")\n        if (models != null) {\n            whisperContext = com.whispercpp.whisper.WhisperContext.createContextFromAsset(application.assets, \"models/\" + models[0])\n            printMessage(\"Loaded model ${models[0]}.\\n\")\n        }\n\n        //val firstModel = modelsPath.listFiles()!!.first()\n        //whisperContext = WhisperContext.createContextFromFile(firstModel.absolutePath)\n    }\n\n    fun benchmark() = viewModelScope.launch {\n        runBenchmark(6)\n    }\n\n    fun transcribeSample() = viewModelScope.launch {\n        transcribeAudio(getFirstSample())\n    }\n\n    private suspend fun runBenchmark(nthreads: Int) {\n        if (!canTranscribe) {\n            return\n        }\n\n        canTranscribe = false\n\n        printMessage(\"Running benchmark. This will take minutes...\\n\")\n        whisperContext?.benchMemory(nthreads)?.let{ printMessage(it) }\n        printMessage(\"\\n\")\n        whisperContext?.benchGgmlMulMat(nthreads)?.let{ printMessage(it) }\n\n        canTranscribe = true\n    }\n\n    private suspend fun getFirstSample(): File = withContext(Dispatchers.IO) {\n        samplesPath.listFiles()!!.first()\n    }\n\n    private suspend fun readAudioSamples(file: File): FloatArray = withContext(Dispatchers.IO) {\n        stopPlayback()\n        startPlayback(file)\n        return@withContext decodeWaveFile(file)\n    }\n\n    private suspend fun stopPlayback() = withContext(Dispatchers.Main) {\n        mediaPlayer?.stop()\n        mediaPlayer?.release()\n        mediaPlayer = null\n    }\n\n    private suspend fun startPlayback(file: File) = withContext(Dispatchers.Main) {\n        mediaPlayer = MediaPlayer.create(application, file.absolutePath.toUri())\n        mediaPlayer?.start()\n    }\n\n    private suspend fun transcribeAudio(file: File) {\n        if (!canTranscribe) {\n            return\n        }\n\n        canTranscribe = false\n\n        try {\n            printMessage(\"Reading wave samples... \")\n            val data = readAudioSamples(file)\n            printMessage(\"${data.size / (16000 / 1000)} ms\\n\")\n            printMessage(\"Transcribing data...\\n\")\n            val start = System.currentTimeMillis()\n            val text = whisperContext?.transcribeData(data)\n            val elapsed = System.currentTimeMillis() - start\n            printMessage(\"Done ($elapsed ms): \\n$text\\n\")\n        } catch (e: Exception) {\n            Log.w(LOG_TAG, e)\n            printMessage(\"${e.localizedMessage}\\n\")\n        }\n\n        canTranscribe = true\n    }\n\n    fun toggleRecord() = viewModelScope.launch {\n        try {\n            if (isRecording) {\n                recorder.stopRecording()\n                isRecording = false\n                recordedFile?.let { transcribeAudio(it) }\n            } else {\n                stopPlayback()\n                val file = getTempFileForRecording()\n                recorder.startRecording(file) { e ->\n                    viewModelScope.launch {\n                        withContext(Dispatchers.Main) {\n                            printMessage(\"${e.localizedMessage}\\n\")\n                            isRecording = false\n                        }\n                    }\n                }\n                isRecording = true\n                recordedFile = file\n            }\n        } catch (e: Exception) {\n            Log.w(LOG_TAG, e)\n            printMessage(\"${e.localizedMessage}\\n\")\n            isRecording = false\n        }\n    }\n\n    private suspend fun getTempFileForRecording() = withContext(Dispatchers.IO) {\n        File.createTempFile(\"recording\", \"wav\")\n    }\n\n    override fun onCleared() {\n        runBlocking {\n            whisperContext?.release()\n            whisperContext = null\n            stopPlayback()\n        }\n    }\n\n    companion object {\n        fun factory() = viewModelFactory {\n            initializer {\n                val application =\n                    this[ViewModelProvider.AndroidViewModelFactory.APPLICATION_KEY] as Application\n                MainScreenViewModel(application)\n            }\n        }\n    }\n}\n\nprivate suspend fun Context.copyData(\n    assetDirName: String,\n    destDir: File,\n    printMessage: suspend (String) -> Unit\n) = withContext(Dispatchers.IO) {\n    assets.list(assetDirName)?.forEach { name ->\n        val assetPath = \"$assetDirName/$name\"\n        Log.v(LOG_TAG, \"Processing $assetPath...\")\n        val destination = File(destDir, name)\n        Log.v(LOG_TAG, \"Copying $assetPath to $destination...\")\n        printMessage(\"Copying $name...\\n\")\n        assets.open(assetPath).use { input ->\n            destination.outputStream().use { output ->\n                input.copyTo(output)\n            }\n        }\n        Log.v(LOG_TAG, \"Copied $assetPath to $destination\")\n    }\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/ui/theme/Color.kt",
    "content": "package com.whispercppdemo.ui.theme\n\nimport androidx.compose.ui.graphics.Color\n\nval Purple80 = Color(0xFFD0BCFF)\nval PurpleGrey80 = Color(0xFFCCC2DC)\nval Pink80 = Color(0xFFEFB8C8)\n\nval Purple40 = Color(0xFF6650a4)\nval PurpleGrey40 = Color(0xFF625b71)\nval Pink40 = Color(0xFF7D5260)"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/ui/theme/Theme.kt",
    "content": "package com.whispercppdemo.ui.theme\n\nimport android.app.Activity\nimport android.os.Build\nimport androidx.compose.foundation.isSystemInDarkTheme\nimport androidx.compose.material3.MaterialTheme\nimport androidx.compose.material3.darkColorScheme\nimport androidx.compose.material3.dynamicDarkColorScheme\nimport androidx.compose.material3.dynamicLightColorScheme\nimport androidx.compose.material3.lightColorScheme\nimport androidx.compose.runtime.Composable\nimport androidx.compose.runtime.SideEffect\nimport androidx.compose.ui.graphics.toArgb\nimport androidx.compose.ui.platform.LocalContext\nimport androidx.compose.ui.platform.LocalView\nimport androidx.core.view.WindowCompat\n\nprivate val DarkColorScheme = darkColorScheme(\n    primary = Purple80,\n    secondary = PurpleGrey80,\n    tertiary = Pink80\n)\n\nprivate val LightColorScheme = lightColorScheme(\n    primary = Purple40,\n    secondary = PurpleGrey40,\n    tertiary = Pink40\n\n    /* Other default colors to override\n    background = Color(0xFFFFFBFE),\n    surface = Color(0xFFFFFBFE),\n    onPrimary = Color.White,\n    onSecondary = Color.White,\n    onTertiary = Color.White,\n    onBackground = Color(0xFF1C1B1F),\n    onSurface = Color(0xFF1C1B1F),\n    */\n)\n\n@Composable\nfun WhisperCppDemoTheme(\n    darkTheme: Boolean = isSystemInDarkTheme(),\n    // Dynamic color is available on Android 12+\n    dynamicColor: Boolean = true,\n    content: @Composable () -> Unit\n) {\n    val colorScheme = when {\n        dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {\n            val context = LocalContext.current\n            if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)\n        }\n        darkTheme -> DarkColorScheme\n        else -> LightColorScheme\n    }\n    val view = LocalView.current\n    if (!view.isInEditMode) {\n        SideEffect {\n            val window = (view.context as Activity).window\n            window.statusBarColor = colorScheme.primary.toArgb()\n            WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme\n        }\n    }\n\n    MaterialTheme(\n        colorScheme = colorScheme,\n        typography = Typography,\n        content = content\n    )\n}"
  },
  {
    "path": "examples/whisper.android/app/src/main/java/com/whispercppdemo/ui/theme/Type.kt",
    "content": "package com.whispercppdemo.ui.theme\n\nimport androidx.compose.material3.Typography\nimport androidx.compose.ui.text.TextStyle\nimport androidx.compose.ui.text.font.FontFamily\nimport androidx.compose.ui.text.font.FontWeight\nimport androidx.compose.ui.unit.sp\n\n// Set of Material typography styles to start with\nval Typography = Typography(\n    bodyLarge = TextStyle(\n        fontFamily = FontFamily.Default,\n        fontWeight = FontWeight.Normal,\n        fontSize = 16.sp,\n        lineHeight = 24.sp,\n        letterSpacing = 0.5.sp\n    )\n    /* Other default text styles to override\n    titleLarge = TextStyle(\n        fontFamily = FontFamily.Default,\n        fontWeight = FontWeight.Normal,\n        fontSize = 22.sp,\n        lineHeight = 28.sp,\n        letterSpacing = 0.sp\n    ),\n    labelSmall = TextStyle(\n        fontFamily = FontFamily.Default,\n        fontWeight = FontWeight.Medium,\n        fontSize = 11.sp,\n        lineHeight = 16.sp,\n        letterSpacing = 0.5.sp\n    )\n    */\n)"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/drawable/ic_launcher_background.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<vector xmlns:android=\"http://schemas.android.com/apk/res/android\"\n    android:width=\"108dp\"\n    android:height=\"108dp\"\n    android:viewportWidth=\"108\"\n    android:viewportHeight=\"108\">\n    <path\n        android:fillColor=\"#3DDC84\"\n        android:pathData=\"M0,0h108v108h-108z\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M9,0L9,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,0L19,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M29,0L29,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M39,0L39,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M49,0L49,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M59,0L59,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M69,0L69,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M79,0L79,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M89,0L89,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M99,0L99,108\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,9L108,9\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,19L108,19\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,29L108,29\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,39L108,39\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,49L108,49\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,59L108,59\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,69L108,69\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,79L108,79\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,89L108,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M0,99L108,99\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,29L89,29\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,39L89,39\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,49L89,49\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,59L89,59\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,69L89,69\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M19,79L89,79\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M29,19L29,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M39,19L39,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M49,19L49,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M59,19L59,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M69,19L69,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n    <path\n        android:fillColor=\"#00000000\"\n        android:pathData=\"M79,19L79,89\"\n        android:strokeWidth=\"0.8\"\n        android:strokeColor=\"#33FFFFFF\" />\n</vector>\n"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/drawable/ic_launcher_foreground.xml",
    "content": "<vector xmlns:android=\"http://schemas.android.com/apk/res/android\"\n    xmlns:aapt=\"http://schemas.android.com/aapt\"\n    android:width=\"108dp\"\n    android:height=\"108dp\"\n    android:viewportWidth=\"108\"\n    android:viewportHeight=\"108\">\n    <path android:pathData=\"M31,63.928c0,0 6.4,-11 12.1,-13.1c7.2,-2.6 26,-1.4 26,-1.4l38.1,38.1L107,108.928l-32,-1L31,63.928z\">\n        <aapt:attr name=\"android:fillColor\">\n            <gradient\n                android:endX=\"85.84757\"\n                android:endY=\"92.4963\"\n                android:startX=\"42.9492\"\n                android:startY=\"49.59793\"\n                android:type=\"linear\">\n                <item\n                    android:color=\"#44000000\"\n                    android:offset=\"0.0\" />\n                <item\n                    android:color=\"#00000000\"\n                    android:offset=\"1.0\" />\n            </gradient>\n        </aapt:attr>\n    </path>\n    <path\n        android:fillColor=\"#FFFFFF\"\n        android:fillType=\"nonZero\"\n        android:pathData=\"M65.3,45.828l3.8,-6.6c0.2,-0.4 0.1,-0.9 -0.3,-1.1c-0.4,-0.2 -0.9,-0.1 -1.1,0.3l-3.9,6.7c-6.3,-2.8 -13.4,-2.8 -19.7,0l-3.9,-6.7c-0.2,-0.4 -0.7,-0.5 -1.1,-0.3C38.8,38.328 38.7,38.828 38.9,39.228l3.8,6.6C36.2,49.428 31.7,56.028 31,63.928h46C76.3,56.028 71.8,49.428 65.3,45.828zM43.4,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2c-0.3,-0.7 -0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C45.3,56.528 44.5,57.328 43.4,57.328L43.4,57.328zM64.6,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2s-0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C66.5,56.528 65.6,57.328 64.6,57.328L64.6,57.328z\"\n        android:strokeWidth=\"1\"\n        android:strokeColor=\"#00000000\" />\n</vector>"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/mipmap-anydpi/ic_launcher.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<adaptive-icon xmlns:android=\"http://schemas.android.com/apk/res/android\">\n    <background android:drawable=\"@drawable/ic_launcher_background\" />\n    <foreground android:drawable=\"@drawable/ic_launcher_foreground\" />\n</adaptive-icon>"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/values/strings.xml",
    "content": "<resources>\n    <string name=\"app_name\">WhisperCppDemo</string>\n</resources>"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/values/themes.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<resources>\n\n    <style name=\"Theme.WhisperCppDemo\" parent=\"android:Theme.Material.Light.NoActionBar\" />\n</resources>"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/xml/backup_rules.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?><!--\n   Sample backup rules file; uncomment and customize as necessary.\n   See https://developer.android.com/guide/topics/data/autobackup\n   for details.\n   Note: This file is ignored for devices older that API 31\n   See https://developer.android.com/about/versions/12/backup-restore\n-->\n<full-backup-content>\n    <!--\n   <include domain=\"sharedpref\" path=\".\"/>\n   <exclude domain=\"sharedpref\" path=\"device.xml\"/>\n-->\n</full-backup-content>"
  },
  {
    "path": "examples/whisper.android/app/src/main/res/xml/data_extraction_rules.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?><!--\n   Sample data extraction rules file; uncomment and customize as necessary.\n   See https://developer.android.com/about/versions/12/backup-restore#xml-changes\n   for details.\n-->\n<data-extraction-rules>\n    <cloud-backup>\n        <!-- TODO: Use <include> and <exclude> to control what is backed up.\n        <include .../>\n        <exclude .../>\n        -->\n    </cloud-backup>\n    <!--\n    <device-transfer>\n        <include .../>\n        <exclude .../>\n    </device-transfer>\n    -->\n</data-extraction-rules>"
  },
  {
    "path": "examples/whisper.android/app/src/test/java/com/whispercppdemo/ExampleUnitTest.kt",
    "content": "package com.whispercppdemo\n\nimport org.junit.Test\n\nimport org.junit.Assert.*\n\n/**\n * Example local unit test, which will execute on the development machine (host).\n *\n * See [testing documentation](http://d.android.com/tools/testing).\n */\nclass ExampleUnitTest {\n    @Test\n    fun addition_isCorrect() {\n        assertEquals(4, 2 + 2)\n    }\n}"
  },
  {
    "path": "examples/whisper.android/build.gradle",
    "content": "// Top-level build file where you can add configuration options common to all sub-projects/modules.\nplugins {\n    id 'com.android.application' version '8.1.1' apply false\n    id 'com.android.library' version '8.1.1' apply false\n    id 'org.jetbrains.kotlin.android' version '1.9.0' apply false\n}"
  },
  {
    "path": "examples/whisper.android/gradle/wrapper/gradle-wrapper.properties",
    "content": "#Wed Dec 14 10:37:24 EST 2022\ndistributionBase=GRADLE_USER_HOME\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-8.2-bin.zip\ndistributionPath=wrapper/dists\nzipStorePath=wrapper/dists\nzipStoreBase=GRADLE_USER_HOME\n"
  },
  {
    "path": "examples/whisper.android/gradle.properties",
    "content": "# Project-wide Gradle settings.\n# IDE (e.g. Android Studio) users:\n# Gradle settings configured through the IDE *will override*\n# any settings specified in this file.\n# For more details on how to configure your build environment visit\n# http://www.gradle.org/docs/current/userguide/build_environment.html\n# Specifies the JVM arguments used for the daemon process.\n# The setting is particularly useful for tweaking memory settings.\norg.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8\n# When configured, Gradle will run in incubating parallel mode.\n# This option should only be used with decoupled projects. More details, visit\n# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects\n# org.gradle.parallel=true\n# AndroidX package structure to make it clearer which packages are bundled with the\n# Android operating system, and which are packaged with your app's APK\n# https://developer.android.com/topic/libraries/support-library/androidx-rn\nandroid.useAndroidX=true\n# Kotlin code style for this project: \"official\" or \"obsolete\":\nkotlin.code.style=official\n# Enables namespacing of each library's R class so that its R class includes only the\n# resources declared in the library itself and none from the library's dependencies,\n# thereby reducing the size of the R class for that library\nandroid.nonTransitiveRClass=true"
  },
  {
    "path": "examples/whisper.android/gradlew",
    "content": "#!/usr/bin/env sh\n\n#\n# Copyright 2015 the original author or authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n##############################################################################\n##\n##  Gradle start up script for UN*X\n##\n##############################################################################\n\n# Attempt to set APP_HOME\n# Resolve links: $0 may be a link\nPRG=\"$0\"\n# Need this for relative symlinks.\nwhile [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n        PRG=\"$link\"\n    else\n        PRG=`dirname \"$PRG\"`\"/$link\"\n    fi\ndone\nSAVED=\"`pwd`\"\ncd \"`dirname \\\"$PRG\\\"`/\" >/dev/null\nAPP_HOME=\"`pwd -P`\"\ncd \"$SAVED\" >/dev/null\n\nAPP_NAME=\"Gradle\"\nAPP_BASE_NAME=`basename \"$0\"`\n\n# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nDEFAULT_JVM_OPTS='\"-Xmx64m\" \"-Xms64m\"'\n\n# Use the maximum available, or set MAX_FD != -1 to use that value.\nMAX_FD=\"maximum\"\n\nwarn () {\n    echo \"$*\"\n}\n\ndie () {\n    echo\n    echo \"$*\"\n    echo\n    exit 1\n}\n\n# OS specific support (must be 'true' or 'false').\ncygwin=false\nmsys=false\ndarwin=false\nnonstop=false\ncase \"`uname`\" in\n  CYGWIN* )\n    cygwin=true\n    ;;\n  Darwin* )\n    darwin=true\n    ;;\n  MINGW* )\n    msys=true\n    ;;\n  NONSTOP* )\n    nonstop=true\n    ;;\nesac\n\nCLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar\n\n\n# Determine the Java command to use to start the JVM.\nif [ -n \"$JAVA_HOME\" ] ; then\n    if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n        # IBM's JDK on AIX uses strange locations for the executables\n        JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n    else\n        JAVACMD=\"$JAVA_HOME/bin/java\"\n    fi\n    if [ ! -x \"$JAVACMD\" ] ; then\n        die \"ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\n    fi\nelse\n    JAVACMD=\"java\"\n    which java >/dev/null 2>&1 || die \"ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\nfi\n\n# Increase the maximum file descriptors if we can.\nif [ \"$cygwin\" = \"false\" -a \"$darwin\" = \"false\" -a \"$nonstop\" = \"false\" ] ; then\n    MAX_FD_LIMIT=`ulimit -H -n`\n    if [ $? -eq 0 ] ; then\n        if [ \"$MAX_FD\" = \"maximum\" -o \"$MAX_FD\" = \"max\" ] ; then\n            MAX_FD=\"$MAX_FD_LIMIT\"\n        fi\n        ulimit -n $MAX_FD\n        if [ $? -ne 0 ] ; then\n            warn \"Could not set maximum file descriptor limit: $MAX_FD\"\n        fi\n    else\n        warn \"Could not query maximum file descriptor limit: $MAX_FD_LIMIT\"\n    fi\nfi\n\n# For Darwin, add options to specify how the application appears in the dock\nif $darwin; then\n    GRADLE_OPTS=\"$GRADLE_OPTS \\\"-Xdock:name=$APP_NAME\\\" \\\"-Xdock:icon=$APP_HOME/media/gradle.icns\\\"\"\nfi\n\n# For Cygwin or MSYS, switch paths to Windows format before running java\nif [ \"$cygwin\" = \"true\" -o \"$msys\" = \"true\" ] ; then\n    APP_HOME=`cygpath --path --mixed \"$APP_HOME\"`\n    CLASSPATH=`cygpath --path --mixed \"$CLASSPATH\"`\n\n    JAVACMD=`cygpath --unix \"$JAVACMD\"`\n\n    # We build the pattern for arguments to be converted via cygpath\n    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`\n    SEP=\"\"\n    for dir in $ROOTDIRSRAW ; do\n        ROOTDIRS=\"$ROOTDIRS$SEP$dir\"\n        SEP=\"|\"\n    done\n    OURCYGPATTERN=\"(^($ROOTDIRS))\"\n    # Add a user-defined pattern to the cygpath arguments\n    if [ \"$GRADLE_CYGPATTERN\" != \"\" ] ; then\n        OURCYGPATTERN=\"$OURCYGPATTERN|($GRADLE_CYGPATTERN)\"\n    fi\n    # Now convert the arguments - kludge to limit ourselves to /bin/sh\n    i=0\n    for arg in \"$@\" ; do\n        CHECK=`echo \"$arg\"|egrep -c \"$OURCYGPATTERN\" -`\n        CHECK2=`echo \"$arg\"|egrep -c \"^-\"`                                 ### Determine if an option\n\n        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition\n            eval `echo args$i`=`cygpath --path --ignore --mixed \"$arg\"`\n        else\n            eval `echo args$i`=\"\\\"$arg\\\"\"\n        fi\n        i=`expr $i + 1`\n    done\n    case $i in\n        0) set -- ;;\n        1) set -- \"$args0\" ;;\n        2) set -- \"$args0\" \"$args1\" ;;\n        3) set -- \"$args0\" \"$args1\" \"$args2\" ;;\n        4) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" ;;\n        5) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" ;;\n        6) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" ;;\n        7) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" ;;\n        8) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" ;;\n        9) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" \"$args8\" ;;\n    esac\nfi\n\n# Escape application args\nsave () {\n    for i do printf %s\\\\n \"$i\" | sed \"s/'/'\\\\\\\\''/g;1s/^/'/;\\$s/\\$/' \\\\\\\\/\" ; done\n    echo \" \"\n}\nAPP_ARGS=`save \"$@\"`\n\n# Collect all arguments for the java command, following the shell quoting and substitution rules\neval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS \"\\\"-Dorg.gradle.appname=$APP_BASE_NAME\\\"\" -classpath \"\\\"$CLASSPATH\\\"\" org.gradle.wrapper.GradleWrapperMain \"$APP_ARGS\"\n\nexec \"$JAVACMD\" \"$@\"\n"
  },
  {
    "path": "examples/whisper.android/gradlew.bat",
    "content": "@rem\r\n@rem Copyright 2015 the original author or authors.\r\n@rem\r\n@rem Licensed under the Apache License, Version 2.0 (the \"License\");\r\n@rem you may not use this file except in compliance with the License.\r\n@rem You may obtain a copy of the License at\r\n@rem\r\n@rem      https://www.apache.org/licenses/LICENSE-2.0\r\n@rem\r\n@rem Unless required by applicable law or agreed to in writing, software\r\n@rem distributed under the License is distributed on an \"AS IS\" BASIS,\r\n@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n@rem See the License for the specific language governing permissions and\r\n@rem limitations under the License.\r\n@rem\r\n\r\n@if \"%DEBUG%\" == \"\" @echo off\r\n@rem ##########################################################################\r\n@rem\r\n@rem  Gradle startup script for Windows\r\n@rem\r\n@rem ##########################################################################\r\n\r\n@rem Set local scope for the variables with windows NT shell\r\nif \"%OS%\"==\"Windows_NT\" setlocal\r\n\r\nset DIRNAME=%~dp0\r\nif \"%DIRNAME%\" == \"\" set DIRNAME=.\r\nset APP_BASE_NAME=%~n0\r\nset APP_HOME=%DIRNAME%\r\n\r\n@rem Resolve any \".\" and \"..\" in APP_HOME to make it shorter.\r\nfor %%i in (\"%APP_HOME%\") do set APP_HOME=%%~fi\r\n\r\n@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\r\nset DEFAULT_JVM_OPTS=\"-Xmx64m\" \"-Xms64m\"\r\n\r\n@rem Find java.exe\r\nif defined JAVA_HOME goto findJavaFromJavaHome\r\n\r\nset JAVA_EXE=java.exe\r\n%JAVA_EXE% -version >NUL 2>&1\r\nif \"%ERRORLEVEL%\" == \"0\" goto execute\r\n\r\necho.\r\necho ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\r\necho.\r\necho Please set the JAVA_HOME variable in your environment to match the\r\necho location of your Java installation.\r\n\r\ngoto fail\r\n\r\n:findJavaFromJavaHome\r\nset JAVA_HOME=%JAVA_HOME:\"=%\r\nset JAVA_EXE=%JAVA_HOME%/bin/java.exe\r\n\r\nif exist \"%JAVA_EXE%\" goto execute\r\n\r\necho.\r\necho ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\r\necho.\r\necho Please set the JAVA_HOME variable in your environment to match the\r\necho location of your Java installation.\r\n\r\ngoto fail\r\n\r\n:execute\r\n@rem Setup the command line\r\n\r\nset CLASSPATH=%APP_HOME%\\gradle\\wrapper\\gradle-wrapper.jar\r\n\r\n\r\n@rem Execute Gradle\r\n\"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% \"-Dorg.gradle.appname=%APP_BASE_NAME%\" -classpath \"%CLASSPATH%\" org.gradle.wrapper.GradleWrapperMain %*\r\n\r\n:end\r\n@rem End local scope for the variables with windows NT shell\r\nif \"%ERRORLEVEL%\"==\"0\" goto mainEnd\r\n\r\n:fail\r\nrem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\r\nrem the _cmd.exe /c_ return code!\r\nif  not \"\" == \"%GRADLE_EXIT_CONSOLE%\" exit 1\r\nexit /b 1\r\n\r\n:mainEnd\r\nif \"%OS%\"==\"Windows_NT\" endlocal\r\n\r\n:omega\r\n"
  },
  {
    "path": "examples/whisper.android/lib/.gitignore",
    "content": "/build"
  },
  {
    "path": "examples/whisper.android/lib/build.gradle",
    "content": "plugins {\n    id 'com.android.library'\n    id 'org.jetbrains.kotlin.android'\n}\n\nandroid {\n    namespace 'com.whispercpp'\n    compileSdk 34\n\n    defaultConfig {\n        minSdk 26\n        targetSdk 34\n        versionCode 1\n        versionName \"1.0\"\n\n        ndk {\n            abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86', 'x86_64'\n        }\n        externalNativeBuild {\n            cmake {\n                // When set, builds whisper.android against the version located\n                // at GGML_HOME instead of the copy bundled with whisper.cpp.\n                if (\n                    project.hasProperty('GGML_HOME') &&\n                    project.findProperty('GGML_CLBLAST') == 'ON'\n                ) {\n                    // Turning on CLBlast requires GGML_HOME\n                    arguments \"-DGGML_HOME=${project.property('GGML_HOME')}\",\n                         \"-DGGML_CLBLAST=ON\",\n                         \"-DOPENCL_LIB=${project.property('OPENCL_LIB')}\",\n                         \"-DCLBLAST_HOME=${project.property('CLBLAST_HOME')}\",\n                         \"-DOPENCL_ROOT=${project.property('OPENCL_ROOT')}\",\n                         \"-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH\",\n                         \"-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=BOTH\"\n                } else if (project.hasProperty('GGML_HOME')) {\n                    arguments \"-DGGML_HOME=${project.property('GGML_HOME')}\"\n                }\n\n            }\n        }\n    }\n\n    buildTypes {\n        release {\n            minifyEnabled false\n        }\n    }\n    compileOptions {\n        sourceCompatibility JavaVersion.VERSION_1_8\n        targetCompatibility JavaVersion.VERSION_1_8\n    }\n    kotlinOptions {\n        jvmTarget = '1.8'\n    }\n\n    ndkVersion \"25.2.9519653\"\n    externalNativeBuild {\n        cmake {\n            path = file(\"src/main/jni/whisper/CMakeLists.txt\")\n        }\n    }\n    packagingOptions {\n        resources {\n            excludes += '/META-INF/{AL2.0,LGPL2.1}'\n        }\n    }\n}\n\ndependencies {\n    implementation 'androidx.core:core-ktx:1.9.0'\n    implementation 'androidx.appcompat:appcompat:1.6.1'\n    implementation 'com.google.android.material:material:1.8.0'\n}"
  },
  {
    "path": "examples/whisper.android/lib/src/main/AndroidManifest.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<manifest xmlns:android=\"http://schemas.android.com/apk/res/android\">\n\n</manifest>"
  },
  {
    "path": "examples/whisper.android/lib/src/main/java/com/whispercpp/whisper/LibWhisper.kt",
    "content": "package com.whispercpp.whisper\n\nimport android.content.res.AssetManager\nimport android.os.Build\nimport android.util.Log\nimport kotlinx.coroutines.*\nimport java.io.File\nimport java.io.InputStream\nimport java.util.concurrent.Executors\n\nprivate const val LOG_TAG = \"LibWhisper\"\n\nclass WhisperContext private constructor(private var ptr: Long) {\n    // Meet Whisper C++ constraint: Don't access from more than one thread at a time.\n    private val scope: CoroutineScope = CoroutineScope(\n        Executors.newSingleThreadExecutor().asCoroutineDispatcher()\n    )\n\n    suspend fun transcribeData(data: FloatArray, printTimestamp: Boolean = true): String = withContext(scope.coroutineContext) {\n        require(ptr != 0L)\n        val numThreads = WhisperCpuConfig.preferredThreadCount\n        Log.d(LOG_TAG, \"Selecting $numThreads threads\")\n        WhisperLib.fullTranscribe(ptr, numThreads, data)\n        val textCount = WhisperLib.getTextSegmentCount(ptr)\n        return@withContext buildString {\n            for (i in 0 until textCount) {\n                if (printTimestamp) {\n                    val textTimestamp = \"[${toTimestamp(WhisperLib.getTextSegmentT0(ptr, i))} --> ${toTimestamp(WhisperLib.getTextSegmentT1(ptr, i))}]\"\n                    val textSegment = WhisperLib.getTextSegment(ptr, i)\n                    append(\"$textTimestamp: $textSegment\\n\")\n                } else {\n                    append(WhisperLib.getTextSegment(ptr, i))\n                }\n            }\n        }\n    }\n\n    suspend fun benchMemory(nthreads: Int): String = withContext(scope.coroutineContext) {\n        return@withContext WhisperLib.benchMemcpy(nthreads)\n    }\n\n    suspend fun benchGgmlMulMat(nthreads: Int): String = withContext(scope.coroutineContext) {\n        return@withContext WhisperLib.benchGgmlMulMat(nthreads)\n    }\n\n    suspend fun release() = withContext(scope.coroutineContext) {\n        if (ptr != 0L) {\n            WhisperLib.freeContext(ptr)\n            ptr = 0\n        }\n    }\n\n    protected fun finalize() {\n        runBlocking {\n            release()\n        }\n    }\n\n    companion object {\n        fun createContextFromFile(filePath: String): WhisperContext {\n            val ptr = WhisperLib.initContext(filePath)\n            if (ptr == 0L) {\n                throw java.lang.RuntimeException(\"Couldn't create context with path $filePath\")\n            }\n            return WhisperContext(ptr)\n        }\n\n        fun createContextFromInputStream(stream: InputStream): WhisperContext {\n            val ptr = WhisperLib.initContextFromInputStream(stream)\n\n            if (ptr == 0L) {\n                throw java.lang.RuntimeException(\"Couldn't create context from input stream\")\n            }\n            return WhisperContext(ptr)\n        }\n\n        fun createContextFromAsset(assetManager: AssetManager, assetPath: String): WhisperContext {\n            val ptr = WhisperLib.initContextFromAsset(assetManager, assetPath)\n\n            if (ptr == 0L) {\n                throw java.lang.RuntimeException(\"Couldn't create context from asset $assetPath\")\n            }\n            return WhisperContext(ptr)\n        }\n\n        fun getSystemInfo(): String {\n            return WhisperLib.getSystemInfo()\n        }\n    }\n}\n\nprivate class WhisperLib {\n    companion object {\n        init {\n            Log.d(LOG_TAG, \"Primary ABI: ${Build.SUPPORTED_ABIS[0]}\")\n            var loadVfpv4 = false\n            var loadV8fp16 = false\n            if (isArmEabiV7a()) {\n                // armeabi-v7a needs runtime detection support\n                val cpuInfo = cpuInfo()\n                cpuInfo?.let {\n                    Log.d(LOG_TAG, \"CPU info: $cpuInfo\")\n                    if (cpuInfo.contains(\"vfpv4\")) {\n                        Log.d(LOG_TAG, \"CPU supports vfpv4\")\n                        loadVfpv4 = true\n                    }\n                }\n            } else if (isArmEabiV8a()) {\n                // ARMv8.2a needs runtime detection support\n                val cpuInfo = cpuInfo()\n                cpuInfo?.let {\n                    Log.d(LOG_TAG, \"CPU info: $cpuInfo\")\n                    if (cpuInfo.contains(\"fphp\")) {\n                        Log.d(LOG_TAG, \"CPU supports fp16 arithmetic\")\n                        loadV8fp16 = true\n                    }\n                }\n            }\n\n            if (loadVfpv4) {\n                Log.d(LOG_TAG, \"Loading libwhisper_vfpv4.so\")\n                System.loadLibrary(\"whisper_vfpv4\")\n            } else if (loadV8fp16) {\n                Log.d(LOG_TAG, \"Loading libwhisper_v8fp16_va.so\")\n                System.loadLibrary(\"whisper_v8fp16_va\")\n            } else {\n                Log.d(LOG_TAG, \"Loading libwhisper.so\")\n                System.loadLibrary(\"whisper\")\n            }\n        }\n\n        // JNI methods\n        external fun initContextFromInputStream(inputStream: InputStream): Long\n        external fun initContextFromAsset(assetManager: AssetManager, assetPath: String): Long\n        external fun initContext(modelPath: String): Long\n        external fun freeContext(contextPtr: Long)\n        external fun fullTranscribe(contextPtr: Long, numThreads: Int, audioData: FloatArray)\n        external fun getTextSegmentCount(contextPtr: Long): Int\n        external fun getTextSegment(contextPtr: Long, index: Int): String\n        external fun getTextSegmentT0(contextPtr: Long, index: Int): Long\n        external fun getTextSegmentT1(contextPtr: Long, index: Int): Long\n        external fun getSystemInfo(): String\n        external fun benchMemcpy(nthread: Int): String\n        external fun benchGgmlMulMat(nthread: Int): String\n    }\n}\n\n//  500 -> 00:05.000\n// 6000 -> 01:00.000\nprivate fun toTimestamp(t: Long, comma: Boolean = false): String {\n    var msec = t * 10\n    val hr = msec / (1000 * 60 * 60)\n    msec -= hr * (1000 * 60 * 60)\n    val min = msec / (1000 * 60)\n    msec -= min * (1000 * 60)\n    val sec = msec / 1000\n    msec -= sec * 1000\n\n    val delimiter = if (comma) \",\" else \".\"\n    return String.format(\"%02d:%02d:%02d%s%03d\", hr, min, sec, delimiter, msec)\n}\n\nprivate fun isArmEabiV7a(): Boolean {\n    return Build.SUPPORTED_ABIS[0].equals(\"armeabi-v7a\")\n}\n\nprivate fun isArmEabiV8a(): Boolean {\n    return Build.SUPPORTED_ABIS[0].equals(\"arm64-v8a\")\n}\n\nprivate fun cpuInfo(): String? {\n    return try {\n        File(\"/proc/cpuinfo\").inputStream().bufferedReader().use {\n            it.readText()\n        }\n    } catch (e: Exception) {\n        Log.w(LOG_TAG, \"Couldn't read /proc/cpuinfo\", e)\n        null\n    }\n}"
  },
  {
    "path": "examples/whisper.android/lib/src/main/java/com/whispercpp/whisper/WhisperCpuConfig.kt",
    "content": "package com.whispercpp.whisper\n\nimport android.util.Log\nimport java.io.BufferedReader\nimport java.io.FileReader\n\nobject WhisperCpuConfig {\n    val preferredThreadCount: Int\n        // Always use at least 2 threads:\n        get() = CpuInfo.getHighPerfCpuCount().coerceAtLeast(2)\n}\n\nprivate class CpuInfo(private val lines: List<String>) {\n    private fun getHighPerfCpuCount(): Int = try {\n        getHighPerfCpuCountByFrequencies()\n    } catch (e: Exception) {\n        Log.d(LOG_TAG, \"Couldn't read CPU frequencies\", e)\n        getHighPerfCpuCountByVariant()\n    }\n\n    private fun getHighPerfCpuCountByFrequencies(): Int =\n        getCpuValues(property = \"processor\") { getMaxCpuFrequency(it.toInt()) }\n            .also { Log.d(LOG_TAG, \"Binned cpu frequencies (frequency, count): ${it.binnedValues()}\") }\n            .countDroppingMin()\n\n    private fun getHighPerfCpuCountByVariant(): Int =\n        getCpuValues(property = \"CPU variant\") { it.substringAfter(\"0x\").toInt(radix = 16) }\n            .also { Log.d(LOG_TAG, \"Binned cpu variants (variant, count): ${it.binnedValues()}\") }\n            .countKeepingMin()\n\n    private fun List<Int>.binnedValues() = groupingBy { it }.eachCount()\n\n    private fun getCpuValues(property: String, mapper: (String) -> Int) = lines\n        .asSequence()\n        .filter { it.startsWith(property) }\n        .map { mapper(it.substringAfter(':').trim()) }\n        .sorted()\n        .toList()\n\n\n    private fun List<Int>.countDroppingMin(): Int {\n        val min = min()\n        return count { it > min }\n    }\n\n    private fun List<Int>.countKeepingMin(): Int {\n        val min = min()\n        return count { it == min }\n    }\n\n    companion object {\n        private const val LOG_TAG = \"WhisperCpuConfig\"\n\n        fun getHighPerfCpuCount(): Int = try {\n            readCpuInfo().getHighPerfCpuCount()\n        } catch (e: Exception) {\n            Log.d(LOG_TAG, \"Couldn't read CPU info\", e)\n            // Our best guess -- just return the # of CPUs minus 4.\n            (Runtime.getRuntime().availableProcessors() - 4).coerceAtLeast(0)\n        }\n\n        private fun readCpuInfo() = CpuInfo(\n            BufferedReader(FileReader(\"/proc/cpuinfo\"))\n                .useLines { it.toList() }\n        )\n\n        private fun getMaxCpuFrequency(cpuIndex: Int): Int {\n            val path = \"/sys/devices/system/cpu/cpu${cpuIndex}/cpufreq/cpuinfo_max_freq\"\n            val maxFreq = BufferedReader(FileReader(path)).use { it.readLine() }\n            return maxFreq.toInt()\n        }\n    }\n}"
  },
  {
    "path": "examples/whisper.android/lib/src/main/jni/whisper/CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.10)\n\nproject(whisper.cpp)\n\nset(CMAKE_CXX_STANDARD 17)\nset(WHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../../../../..)\n\n# Get whisper.cpp version\nfile(READ \"${WHISPER_LIB_DIR}/CMakeLists.txt\" MAIN_CMAKE_CONTENT)\nstring(REGEX MATCH \"project\\\\(\\\"whisper\\\\.cpp\\\" VERSION ([0-9]+\\\\.[0-9]+\\\\.[0-9]+)\\\\)\" VERSION_MATCH \"${MAIN_CMAKE_CONTENT}\")\nif(CMAKE_MATCH_1)\n    set(WHISPER_VERSION ${CMAKE_MATCH_1} PARENT_SCOPE)\nelse()\n    set(WHISPER_VERSION \"unknown\" PARENT_SCOPE)\nendif()\n\nmessage(STATUS \" Whisper version: ${WHISPER_VERSION}\")\n\n# Path to external GGML, otherwise uses the copy in whisper.cpp.\noption(GGML_HOME \"whisper: Path to external GGML source\" OFF)\n\nset(\n    SOURCE_FILES\n    ${WHISPER_LIB_DIR}/src/whisper.cpp\n    ${CMAKE_SOURCE_DIR}/jni.c\n    )\n\nfind_library(LOG_LIB log)\n\ninclude(FetchContent)\n\nfunction(build_library target_name)\n    add_library(\n        ${target_name}\n        SHARED\n        ${SOURCE_FILES}\n    )\n\n    target_compile_definitions(${target_name} PUBLIC GGML_USE_CPU)\n    target_compile_definitions(${target_name} PRIVATE WHISPER_VERSION=\"${WHISPER_VERSION}\")\n\n    if (${target_name} STREQUAL \"whisper_v8fp16_va\")\n        target_compile_options(${target_name} PRIVATE -march=armv8.2-a+fp16)\n        set(GGML_COMPILE_OPTIONS                      -march=armv8.2-a+fp16)\n    elseif (${target_name} STREQUAL \"whisper_vfpv4\")\n        target_compile_options(${target_name} PRIVATE -mfpu=neon-vfpv4)\n        set(GGML_COMPILE_OPTIONS                      -mfpu=neon-vfpv4)\n    endif ()\n\n    if (NOT ${CMAKE_BUILD_TYPE} STREQUAL \"Debug\")\n        target_compile_options(${target_name} PRIVATE -O3)\n        target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)\n        target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)\n\n        target_link_options(${target_name} PRIVATE -Wl,--gc-sections)\n        target_link_options(${target_name} PRIVATE -Wl,--exclude-libs,ALL)\n        target_link_options(${target_name} PRIVATE -flto)\n    endif ()\n\n    if (GGML_HOME)\n        FetchContent_Declare(ggml SOURCE_DIR ${GGML_HOME})\n    else()\n        FetchContent_Declare(ggml SOURCE_DIR ${WHISPER_LIB_DIR}/ggml)\n    endif()\n    FetchContent_MakeAvailable(ggml)\n    target_compile_options(ggml PRIVATE ${GGML_COMPILE_OPTIONS})\n    target_link_libraries(${target_name} ${LOG_LIB} android ggml)\n\n\nendfunction()\n\nif (${ANDROID_ABI} STREQUAL \"arm64-v8a\")\n    build_library(\"whisper_v8fp16_va\")\nelseif (${ANDROID_ABI} STREQUAL \"armeabi-v7a\")\n    build_library(\"whisper_vfpv4\")\nendif ()\n\nbuild_library(\"whisper\") # Default target\n\ninclude_directories(${WHISPER_LIB_DIR})\ninclude_directories(${WHISPER_LIB_DIR}/src)\ninclude_directories(${WHISPER_LIB_DIR}/include)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/include)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/src)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/src/ggml-cpu)\n\n"
  },
  {
    "path": "examples/whisper.android/lib/src/main/jni/whisper/jni.c",
    "content": "#include <jni.h>\n#include <android/asset_manager.h>\n#include <android/asset_manager_jni.h>\n#include <android/log.h>\n#include <stdlib.h>\n#include <sys/sysinfo.h>\n#include <string.h>\n#include \"whisper.h\"\n#include \"ggml.h\"\n\n#define UNUSED(x) (void)(x)\n#define TAG \"JNI\"\n\n#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,     TAG, __VA_ARGS__)\n#define LOGW(...) __android_log_print(ANDROID_LOG_WARN,     TAG, __VA_ARGS__)\n\nstatic inline int min(int a, int b) {\n    return (a < b) ? a : b;\n}\n\nstatic inline int max(int a, int b) {\n    return (a > b) ? a : b;\n}\n\nstruct input_stream_context {\n    size_t offset;\n    JNIEnv * env;\n    jobject thiz;\n    jobject input_stream;\n\n    jmethodID mid_available;\n    jmethodID mid_read;\n};\n\nsize_t inputStreamRead(void * ctx, void * output, size_t read_size) {\n    struct input_stream_context* is = (struct input_stream_context*)ctx;\n\n    jint avail_size = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_available);\n    jint size_to_copy = read_size < avail_size ? (jint)read_size : avail_size;\n\n    jbyteArray byte_array = (*is->env)->NewByteArray(is->env, size_to_copy);\n\n    jint n_read = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_read, byte_array, 0, size_to_copy);\n\n    if (size_to_copy != read_size || size_to_copy != n_read) {\n        LOGI(\"Insufficient Read: Req=%zu, ToCopy=%d, Available=%d\", read_size, size_to_copy, n_read);\n    }\n\n    jbyte* byte_array_elements = (*is->env)->GetByteArrayElements(is->env, byte_array, NULL);\n    memcpy(output, byte_array_elements, size_to_copy);\n    (*is->env)->ReleaseByteArrayElements(is->env, byte_array, byte_array_elements, JNI_ABORT);\n\n    (*is->env)->DeleteLocalRef(is->env, byte_array);\n\n    is->offset += size_to_copy;\n\n    return size_to_copy;\n}\nbool inputStreamEof(void * ctx) {\n    struct input_stream_context* is = (struct input_stream_context*)ctx;\n\n    jint result = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_available);\n    return result <= 0;\n}\nvoid inputStreamClose(void * ctx) {\n\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercppdemo_whisper_WhisperLib_00024Companion_initContextFromInputStream(\n        JNIEnv *env, jobject thiz, jobject input_stream) {\n    UNUSED(thiz);\n\n    struct whisper_context *context = NULL;\n    struct whisper_model_loader loader = {};\n    struct input_stream_context inp_ctx = {};\n\n    inp_ctx.offset = 0;\n    inp_ctx.env = env;\n    inp_ctx.thiz = thiz;\n    inp_ctx.input_stream = input_stream;\n\n    jclass cls = (*env)->GetObjectClass(env, input_stream);\n    inp_ctx.mid_available = (*env)->GetMethodID(env, cls, \"available\", \"()I\");\n    inp_ctx.mid_read = (*env)->GetMethodID(env, cls, \"read\", \"([BII)I\");\n\n    loader.context = &inp_ctx;\n    loader.read = inputStreamRead;\n    loader.eof = inputStreamEof;\n    loader.close = inputStreamClose;\n\n    loader.eof(loader.context);\n\n    context = whisper_init(&loader);\n    return (jlong) context;\n}\n\nstatic size_t asset_read(void *ctx, void *output, size_t read_size) {\n    return AAsset_read((AAsset *) ctx, output, read_size);\n}\n\nstatic bool asset_is_eof(void *ctx) {\n    return AAsset_getRemainingLength64((AAsset *) ctx) <= 0;\n}\n\nstatic void asset_close(void *ctx) {\n    AAsset_close((AAsset *) ctx);\n}\n\nstatic struct whisper_context *whisper_init_from_asset(\n        JNIEnv *env,\n        jobject assetManager,\n        const char *asset_path\n) {\n    LOGI(\"Loading model from asset '%s'\\n\", asset_path);\n    AAssetManager *asset_manager = AAssetManager_fromJava(env, assetManager);\n    AAsset *asset = AAssetManager_open(asset_manager, asset_path, AASSET_MODE_STREAMING);\n    if (!asset) {\n        LOGW(\"Failed to open '%s'\\n\", asset_path);\n        return NULL;\n    }\n\n    whisper_model_loader loader = {\n            .context = asset,\n            .read = &asset_read,\n            .eof = &asset_is_eof,\n            .close = &asset_close\n    };\n\n    return whisper_init_with_params(&loader, whisper_context_default_params());\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_initContextFromAsset(\n        JNIEnv *env, jobject thiz, jobject assetManager, jstring asset_path_str) {\n    UNUSED(thiz);\n    struct whisper_context *context = NULL;\n    const char *asset_path_chars = (*env)->GetStringUTFChars(env, asset_path_str, NULL);\n    context = whisper_init_from_asset(env, assetManager, asset_path_chars);\n    (*env)->ReleaseStringUTFChars(env, asset_path_str, asset_path_chars);\n    return (jlong) context;\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_initContext(\n        JNIEnv *env, jobject thiz, jstring model_path_str) {\n    UNUSED(thiz);\n    struct whisper_context *context = NULL;\n    const char *model_path_chars = (*env)->GetStringUTFChars(env, model_path_str, NULL);\n    context = whisper_init_from_file_with_params(model_path_chars, whisper_context_default_params());\n    (*env)->ReleaseStringUTFChars(env, model_path_str, model_path_chars);\n    return (jlong) context;\n}\n\nJNIEXPORT void JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_freeContext(\n        JNIEnv *env, jobject thiz, jlong context_ptr) {\n    UNUSED(env);\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    whisper_free(context);\n}\n\nJNIEXPORT void JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_fullTranscribe(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint num_threads, jfloatArray audio_data) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    jfloat *audio_data_arr = (*env)->GetFloatArrayElements(env, audio_data, NULL);\n    const jsize audio_data_length = (*env)->GetArrayLength(env, audio_data);\n\n    // The below adapted from the Objective-C iOS sample\n    struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n    params.print_realtime = true;\n    params.print_progress = false;\n    params.print_timestamps = true;\n    params.print_special = false;\n    params.translate = false;\n    params.language = \"en\";\n    params.n_threads = num_threads;\n    params.offset_ms = 0;\n    params.no_context = true;\n    params.single_segment = false;\n\n    whisper_reset_timings(context);\n\n    LOGI(\"About to run whisper_full\");\n    if (whisper_full(context, params, audio_data_arr, audio_data_length) != 0) {\n        LOGI(\"Failed to run the model\");\n    } else {\n        whisper_print_timings(context);\n    }\n    (*env)->ReleaseFloatArrayElements(env, audio_data, audio_data_arr, JNI_ABORT);\n}\n\nJNIEXPORT jint JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_getTextSegmentCount(\n        JNIEnv *env, jobject thiz, jlong context_ptr) {\n    UNUSED(env);\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    return whisper_full_n_segments(context);\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_getTextSegment(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    const char *text = whisper_full_get_segment_text(context, index);\n    jstring string = (*env)->NewStringUTF(env, text);\n    return string;\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_getTextSegmentT0(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    return whisper_full_get_segment_t0(context, index);\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_getTextSegmentT1(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    return whisper_full_get_segment_t1(context, index);\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_getSystemInfo(\n        JNIEnv *env, jobject thiz\n) {\n    UNUSED(thiz);\n    const char *sysinfo = whisper_print_system_info();\n    jstring string = (*env)->NewStringUTF(env, sysinfo);\n    return string;\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_benchMemcpy(JNIEnv *env, jobject thiz,\n                                                                      jint n_threads) {\n    UNUSED(thiz);\n    const char *bench_ggml_memcpy = whisper_bench_memcpy_str(n_threads);\n    jstring string = (*env)->NewStringUTF(env, bench_ggml_memcpy);\n    return string;\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_whisper_WhisperLib_00024Companion_benchGgmlMulMat(JNIEnv *env, jobject thiz,\n                                                                          jint n_threads) {\n    UNUSED(thiz);\n    const char *bench_ggml_mul_mat = whisper_bench_ggml_mul_mat_str(n_threads);\n    jstring string = (*env)->NewStringUTF(env, bench_ggml_mul_mat);\n    return string;\n}\n"
  },
  {
    "path": "examples/whisper.android/settings.gradle",
    "content": "pluginManagement {\n    repositories {\n        gradlePluginPortal()\n        google()\n        mavenCentral()\n    }\n}\ndependencyResolutionManagement {\n    repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)\n    repositories {\n        google()\n        mavenCentral()\n    }\n}\nrootProject.name = \"WhisperCppDemo\"\ninclude ':app'\ninclude ':lib'\n"
  },
  {
    "path": "examples/whisper.android.java/.gitignore",
    "content": "*.iml\n.gradle\n/local.properties\n/.idea/caches\n/.idea/libraries\n/.idea/modules.xml\n/.idea/workspace.xml\n/.idea/navEditor.xml\n/.idea/assetWizardSettings.xml\n.DS_Store\n/build\n/captures\n.externalNativeBuild\n.cxx\nlocal.properties\n"
  },
  {
    "path": "examples/whisper.android.java/README.md",
    "content": "A sample Android app using java code and [whisper.cpp](https://github.com/ggerganov/whisper.cpp/) to do voice-to-text transcriptions.\n\nTo use:\n\n1. Select a model from the [whisper.cpp repository](https://github.com/ggerganov/whisper.cpp/tree/master/models).[^1]\n2. Copy the model to the \"app/src/main/assets/models\" folder.\n3. Select a sample audio file (for example, [jfk.wav](https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav)).\n4. Copy the sample to the \"app/src/main/assets/samples\" folder.\n5. Modify the modelFilePath in the WhisperService.java\n6. Modify the sampleFilePath in the WhisperService.java\n7. Select the \"release\" active build variant, and use Android Studio to run and deploy to your device.\n[^1]: I recommend the tiny or base models for running on an Android device.\n\nPS:  \n1. Do not move this android project folder individually to other folders, because this android project folder depends on the files of the whole project.  \n2. The cpp code is compiled during the build process  \n3. If you want to import a compiled cpp project in your Android project, please refer to the https://github.com/litongjava/whisper.cpp.android.java.demo  \n\n![](README_files/1.jpg)\n\n"
  },
  {
    "path": "examples/whisper.android.java/app/.gitignore",
    "content": "/build"
  },
  {
    "path": "examples/whisper.android.java/app/build.gradle",
    "content": "plugins {\n  id 'com.android.application'\n}\n\nandroid {\n  compileSdkVersion 31\n  buildToolsVersion '30.0.3'\n\n  defaultConfig {\n    applicationId \"com.litongjava.whisper.android.java\"\n    minSdkVersion 21\n    targetSdkVersion 31\n    versionCode 1\n    versionName \"1.0\"\n\n    testInstrumentationRunner \"androidx.test.runner.AndroidJUnitRunner\"\n    externalNativeBuild {\n      cmake {\n        cppFlags \"\"\n      }\n    }\n    ndk {\n      abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86', 'x86_64'\n    }\n  }\n\n  buildTypes {\n    release {\n      signingConfig signingConfigs.debug\n      minifyEnabled true\n      proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'\n    }\n  }\n  externalNativeBuild {\n    cmake {\n      path \"src/main/jni/whisper/CMakeLists.txt\"\n    }\n  }\n  ndkVersion \"25.2.9519653\"\n  compileOptions {\n    sourceCompatibility JavaVersion.VERSION_1_8\n    targetCompatibility JavaVersion.VERSION_1_8\n  }\n}\n\ndependencies {\n  implementation 'androidx.appcompat:appcompat:1.1.0'\n  implementation 'com.google.android.material:material:1.1.0'\n  implementation 'androidx.constraintlayout:constraintlayout:1.1.3'\n  testImplementation 'junit:junit:4.+'\n  androidTestImplementation 'androidx.test.ext:junit:1.1.5'\n  androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'\n\n  //litongjava\n  implementation 'com.litongjava:android-view-inject:1.0'\n  implementation 'com.litongjava:jfinal-aop:1.0.1'\n  implementation 'com.litongjava:litongjava-android-utils:1.0.0'\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/proguard-rules.pro",
    "content": "# Add project specific ProGuard rules here.\n# You can control the set of applied configuration files using the\n# proguardFiles setting in build.gradle.\n#\n# For more details, see\n#   http://developer.android.com/guide/developing/tools/proguard.html\n\n# If your project uses WebView with JS, uncomment the following\n# and specify the fully qualified class name to the JavaScript interface\n# class:\n#-keepclassmembers class fqcn.of.javascript.interface.for.webview {\n#   public *;\n#}\n\n# Uncomment this to preserve the line number information for\n# debugging stack traces.\n#-keepattributes SourceFile,LineNumberTable\n\n# If you keep the line number information, uncomment this to\n# hide the original source file name.\n#-renamesourcefileattribute SourceFile"
  },
  {
    "path": "examples/whisper.android.java/app/src/androidTest/java/com/litongjava/whisper/android/java/ExampleInstrumentedTest.java",
    "content": "package com.litongjava.whisper.android.java;\n\nimport android.content.Context;\n\nimport androidx.test.platform.app.InstrumentationRegistry;\nimport androidx.test.ext.junit.runners.AndroidJUnit4;\n\nimport org.junit.Test;\nimport org.junit.runner.RunWith;\n\nimport static org.junit.Assert.*;\n\n/**\n * Instrumented test, which will execute on an Android device.\n *\n * @see <a href=\"http://d.android.com/tools/testing\">Testing documentation</a>\n */\n@RunWith(AndroidJUnit4.class)\npublic class ExampleInstrumentedTest {\n  @Test\n  public void useAppContext() {\n    // Context of the app under test.\n    Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext();\n    assertEquals(\"com.litongjava.whisper.android.java\", appContext.getPackageName());\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/AndroidManifest.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<manifest xmlns:android=\"http://schemas.android.com/apk/res/android\"\n  package=\"com.litongjava.whisper.android.java\">\n\n  <application\n    android:allowBackup=\"true\"\n    android:name=\".app.App\"\n    android:icon=\"@mipmap/ic_launcher\"\n    android:label=\"@string/app_name\"\n    android:roundIcon=\"@mipmap/ic_launcher_round\"\n    android:supportsRtl=\"true\"\n    android:theme=\"@style/Theme.Whisperandroidjava\">\n    <activity android:name=\".MainActivity\" android:exported=\"true\">\n      <intent-filter>\n        <action android:name=\"android.intent.action.MAIN\" />\n\n        <category android:name=\"android.intent.category.LAUNCHER\" />\n      </intent-filter>\n    </activity>\n  </application>\n\n</manifest>\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/assets/logback.xml",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<configuration debug=\"false\" xmlns=\"http://ch.qos.logback/xml/ns/logback\"\n  xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n  xsi:schemaLocation=\"http://ch.qos.logback/xml/ns/logback https://raw.githubusercontent.com/enricopulatzo/logback-XSD/master/src/main/xsd/logback.xsd\nhttp://ch.qos.logback/xml/ns/logback \">\n  <!--Define the storage address of the log file Do not use relative paths in the LogBack configuration. -->\n  <property name=\"LOG_HOME\" value=\"logs\" />\n  <!--Formatted output: %d means the date, %-6level: log level from the left display 6 characters wide, %m: log message, %n is a newline character -->\n  <property name=\"CONSOLE_LOG_PATTERN\"\n    value=\"%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-6level%logger{0}.%M:%L - %m%n\" />\n\n  <!-- console output -->\n  <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n    <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n      <pattern>${CONSOLE_LOG_PATTERN}</pattern>\n    </encoder>\n  </appender>\n\n  <!-- Generate log files on a daily basis -->\n  <appender name=\"FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n    <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n      <pattern>${CONSOLE_LOG_PATTERN}</pattern>\n    </encoder>\n    <rollingPolicy class=\"ch.qos.logback.core.rolling.TimeBasedRollingPolicy\">\n      <!--File name for log file output -->\n      <fileNamePattern>${LOG_HOME}/project-name-%d{yyyy-MM-dd}.log</fileNamePattern>\n      <!--Maximum size of log file -->\n      <maxHistory>180</maxHistory>\n    </rollingPolicy>\n    <!--日志文件最大的大小 -->\n    <triggeringPolicy class=\"ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy\">\n      <maxFileSize>10MB</maxFileSize>\n    </triggeringPolicy>\n  </appender>\n  <!-- Log output level and source-->\n  <root level=\"info\">\n    <appender-ref ref=\"STDOUT\" />\n    <appender-ref ref=\"FILE\" />\n  </root>\n</configuration>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/MainActivity.java",
    "content": "package com.litongjava.whisper.android.java;\n\nimport androidx.annotation.RequiresApi;\nimport androidx.appcompat.app.AppCompatActivity;\n\nimport android.content.Context;\nimport android.os.Build;\nimport android.os.Bundle;\nimport android.os.Handler;\nimport android.os.Looper;\nimport android.view.View;\nimport android.widget.TextView;\n\nimport com.blankj.utilcode.util.ThreadUtils;\nimport com.litongjava.android.view.inject.annotation.FindViewById;\nimport com.litongjava.android.view.inject.annotation.FindViewByIdLayout;\nimport com.litongjava.android.view.inject.annotation.OnClick;\nimport com.litongjava.android.view.inject.utils.ViewInjectUtils;\nimport com.litongjava.jfinal.aop.Aop;\nimport com.litongjava.jfinal.aop.AopManager;\nimport com.litongjava.whisper.android.java.services.WhisperService;\nimport com.litongjava.whisper.android.java.task.LoadModelTask;\nimport com.litongjava.whisper.android.java.task.TranscriptionTask;\nimport com.litongjava.whisper.android.java.utils.AssetUtils;\nimport com.whispercpp.java.whisper.WhisperLib;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\n\n\n@FindViewByIdLayout(R.layout.activity_main)\npublic class MainActivity extends AppCompatActivity {\n\n  @FindViewById(R.id.sample_text)\n  private TextView tv;\n\n  Logger log = LoggerFactory.getLogger(this.getClass());\n  private WhisperService whisperService = Aop.get(WhisperService.class);\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  @Override\n  protected void onCreate(Bundle savedInstanceState) {\n    super.onCreate(savedInstanceState);\n    //setContentView(R.layout.activity_main);\n    ViewInjectUtils.injectActivity(this, this);\n    initAopBean();\n    showSystemInfo();\n  }\n\n  private void initAopBean() {\n    Handler mainHandler = new Handler(Looper.getMainLooper());\n    AopManager.me().addSingletonObject(mainHandler);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  @OnClick(R.id.loadModelBtn)\n  public void loadModelBtn_OnClick(View v) {\n    Context context = getBaseContext();\n    ThreadUtils.executeByIo(new LoadModelTask(tv));\n  }\n\n  @OnClick(R.id.transcriptSampleBtn)\n  public void transcriptSampleBtn_OnClick(View v) {\n    Context context = getBaseContext();\n\n    long start = System.currentTimeMillis();\n    String sampleFilePath = \"samples/jfk.wav\";\n    File filesDir = context.getFilesDir();\n    File sampleFile = AssetUtils.copyFileIfNotExists(context, filesDir, sampleFilePath);\n    long end = System.currentTimeMillis();\n    String msg = \"copy file:\" + (end - start) + \"ms\";\n    outputMsg(tv, msg);\n    ThreadUtils.executeByIo(new TranscriptionTask(tv, sampleFile));\n  }\n\n  private void outputMsg(TextView tv, String msg) {\n    tv.append(msg + \"\\n\");\n    log.info(msg);\n  }\n\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  @OnClick(R.id.systemInfoBtn)\n  public void systemInfoBtn_OnClick(View v) {\n    showSystemInfo();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public void showSystemInfo() {\n    String systemInfo = WhisperLib.getSystemInfo();\n    tv.append(systemInfo + \"\\n\");\n  }\n\n  @OnClick(R.id.clearBtn)\n  public void clearBtn_OnClick(View v) {\n    tv.setText(\"\");\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  @Override\n  protected void onDestroy() {\n    super.onDestroy();\n    whisperService.release();\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/app/App.java",
    "content": "package com.litongjava.whisper.android.java.app;\n\nimport android.app.Application;\n\nimport com.blankj.utilcode.util.Utils;\n\npublic class App extends Application {\n  @Override\n  public void onCreate() {\n    super.onCreate();\n    Utils.init(this);\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/bean/WhisperSegment.java",
    "content": "package com.litongjava.whisper.android.java.bean;\n\n/**\n * Created by litonglinux@qq.com on 10/21/2023_7:48 AM\n */\npublic class WhisperSegment {\n  private long start, end;\n  private String sentence;\n\n  public WhisperSegment() {\n  }\n\n  public WhisperSegment(long start, long end, String sentence) {\n    this.start = start;\n    this.end = end;\n    this.sentence = sentence;\n  }\n\n  public long getStart() {\n    return start;\n  }\n\n  public long getEnd() {\n    return end;\n  }\n\n  public String getSentence() {\n    return sentence;\n  }\n\n  public void setStart(long start) {\n    this.start = start;\n  }\n\n  public void setEnd(long end) {\n    this.end = end;\n  }\n\n  public void setSentence(String sentence) {\n    this.sentence = sentence;\n  }\n\n  @Override\n  public String toString() {\n    return \"[\"+start+\" --> \"+end+\"]:\"+sentence;\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/services/WhisperService.java",
    "content": "package com.litongjava.whisper.android.java.services;\n\nimport android.content.Context;\nimport android.os.Build;\nimport android.os.Handler;\nimport android.widget.TextView;\nimport android.widget.Toast;\n\nimport androidx.annotation.RequiresApi;\n\nimport com.blankj.utilcode.util.ToastUtils;\nimport com.blankj.utilcode.util.Utils;\nimport com.litongjava.android.utils.dialog.AlertDialogUtils;\nimport com.litongjava.jfinal.aop.Aop;\nimport com.litongjava.whisper.android.java.bean.WhisperSegment;\nimport com.litongjava.whisper.android.java.single.LocalWhisper;\nimport com.litongjava.whisper.android.java.utils.WaveEncoder;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\n\npublic class WhisperService {\n  private Logger log = LoggerFactory.getLogger(this.getClass());\n\n  private final Object lock = new Object();\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public void loadModel(TextView tv) {\n    String modelFilePath = LocalWhisper.modelFilePath;\n    String msg = \"load model from :\" + modelFilePath + \"\\n\";\n    outputMsg(tv, msg);\n\n    long start = System.currentTimeMillis();\n    LocalWhisper.INSTANCE.init();\n    long end = System.currentTimeMillis();\n    msg = \"model load successful:\" + (end - start) + \"ms\";\n    outputMsg(tv, msg);\n    ToastUtils.showLong(msg);\n\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public void transcribeSample(TextView tv, File sampleFile) {\n    String msg = \"\";\n    msg = \"transcribe file from :\" + sampleFile.getAbsolutePath();\n    outputMsg(tv, msg);\n\n    Long start = System.currentTimeMillis();\n    float[] audioData = new float[0];  // 读取音频样本\n    try {\n      audioData = WaveEncoder.decodeWaveFile(sampleFile);\n    } catch (IOException e) {\n      e.printStackTrace();\n      return;\n    }\n    long end = System.currentTimeMillis();\n    msg = \"decode wave file:\" + (end - start) + \"ms\";\n    outputMsg(tv, msg);\n\n    start = System.currentTimeMillis();\n    List<WhisperSegment> transcription = null;\n    try {\n      //transcription = LocalWhisper.INSTANCE.transcribeData(audioData);\n      transcription = LocalWhisper.INSTANCE.transcribeDataWithTime(audioData);\n    } catch (ExecutionException e) {\n      e.printStackTrace();\n    } catch (InterruptedException e) {\n      e.printStackTrace();\n    }\n    end = System.currentTimeMillis();\n    if(transcription!=null){\n      ToastUtils.showLong(transcription.toString());\n      msg = \"Transcript successful:\" + (end - start) + \"ms\";\n      outputMsg(tv, msg);\n\n      outputMsg(tv, transcription.toString());\n\n    }else{\n      msg = \"Transcript failed:\" + (end - start) + \"ms\";\n      outputMsg(tv, msg);\n    }\n\n  }\n\n  private void outputMsg(TextView tv, String msg) {\n    log.info(msg);\n    if(tv!=null){\n      Aop.get(Handler.class).post(()->{ tv.append(msg + \"\\n\");});\n    }\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public void release() {\n    //noting to do\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/single/LocalWhisper.java",
    "content": "package com.litongjava.whisper.android.java.single;\n\nimport android.app.Application;\nimport android.os.Build;\nimport android.os.Handler;\n\nimport androidx.annotation.RequiresApi;\n\nimport com.blankj.utilcode.util.ToastUtils;\nimport com.blankj.utilcode.util.Utils;\nimport com.litongjava.jfinal.aop.Aop;\nimport com.litongjava.whisper.android.java.bean.WhisperSegment;\nimport com.litongjava.whisper.android.java.utils.AssetUtils;\nimport com.whispercpp.java.whisper.WhisperContext;\n\nimport java.io.File;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\n\n\n@RequiresApi(api = Build.VERSION_CODES.O)\npublic enum LocalWhisper {\n  INSTANCE;\n\n  public static final String modelFilePath = \"models/ggml-tiny.bin\";\n  private WhisperContext whisperContext;\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  LocalWhisper() {\n    Application context = Utils.getApp();\n    File filesDir = context.getFilesDir();\n    File modelFile = AssetUtils.copyFileIfNotExists(context, filesDir, modelFilePath);\n    String realModelFilePath = modelFile.getAbsolutePath();\n    whisperContext = WhisperContext.createContextFromFile(realModelFilePath);\n  }\n\n  public synchronized String transcribeData(float[] data) throws ExecutionException, InterruptedException {\n    if(whisperContext==null){\n        toastModelLoading();\n        return null;\n    }else{\n      return whisperContext.transcribeData(data);\n    }\n  }\n\n    private static void toastModelLoading() {\n        Aop.get(Handler.class).post(()->{\n          ToastUtils.showShort(\"please wait for model loading\");\n        });\n    }\n\n    public List<WhisperSegment> transcribeDataWithTime(float[] audioData) throws ExecutionException, InterruptedException {\n    if(whisperContext==null){\n        toastModelLoading();\n      return null;\n    }else{\n      return whisperContext.transcribeDataWithTime(audioData);\n    }\n  }\n\n  public void init() {\n    //noting to do.but init\n  }\n\n\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/task/LoadModelTask.java",
    "content": "package com.litongjava.whisper.android.java.task;\n\nimport android.content.Context;\nimport android.os.Build;\nimport android.os.Handler;\nimport android.widget.TextView;\n\nimport com.blankj.utilcode.util.ThreadUtils;\nimport com.litongjava.jfinal.aop.Aop;\nimport com.litongjava.whisper.android.java.services.WhisperService;\n\nimport java.io.File;\n\npublic class LoadModelTask extends ThreadUtils.Task<Object> {\n  private final TextView tv;\n  public LoadModelTask(TextView tv) {\n    this.tv = tv;\n  }\n\n  @Override\n  public Object doInBackground() {\n    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {\n      Aop.get(WhisperService.class).loadModel(tv);\n    }else{\n      Aop.get(Handler.class).post(()->{\n        tv.append(\"not supported android devices\");\n      });\n\n    }\n    return null;\n  }\n\n  @Override\n  public void onSuccess(Object result) {\n  }\n\n  @Override\n  public void onCancel() {\n  }\n\n  @Override\n  public void onFail(Throwable t) {\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/task/TranscriptionTask.java",
    "content": "package com.litongjava.whisper.android.java.task;\n\nimport android.content.Context;\nimport android.os.Build;\nimport android.widget.TextView;\n\nimport com.blankj.utilcode.util.ThreadUtils;\nimport com.litongjava.jfinal.aop.Aop;\nimport com.litongjava.whisper.android.java.services.WhisperService;\n\nimport java.io.File;\n\npublic class TranscriptionTask extends ThreadUtils.Task<Object> {\n  private final TextView tv;\n  private final File sampleFile;\n\n  public TranscriptionTask(TextView tv, File sampleFile) {\n    this.tv = tv;\n    this.sampleFile = sampleFile;\n\n  }\n\n  @Override\n  public Object doInBackground() {\n    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {\n      Aop.get(WhisperService.class).transcribeSample(tv, sampleFile);\n    }else{\n      tv.append(\"not supported android devices\");\n    }\n    return null;\n  }\n\n  @Override\n  public void onSuccess(Object result) {\n  }\n\n  @Override\n  public void onCancel() {\n  }\n\n  @Override\n  public void onFail(Throwable t) {\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/utils/AssetUtils.java",
    "content": "package com.litongjava.whisper.android.java.utils;\n\nimport android.content.Context;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.File;\nimport java.io.FileNotFoundException;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\n\npublic class AssetUtils {\n  private static Logger log = LoggerFactory.getLogger(AssetUtils.class);\n\n  public static File copyFileIfNotExists(Context context, File distDir, String filename) {\n    File dstFile = new File(distDir, filename);\n    if (dstFile.exists()) {\n      return dstFile;\n    } else {\n      File parentFile = dstFile.getParentFile();\n      log.info(\"parentFile:{}\", parentFile);\n      if (!parentFile.exists()) {\n        parentFile.mkdirs();\n      }\n      AssetUtils.copyFileFromAssets(context, filename, dstFile);\n    }\n    return dstFile;\n  }\n\n  public static void copyDirectoryFromAssets(Context appCtx, String srcDir, String dstDir) {\n    if (srcDir.isEmpty() || dstDir.isEmpty()) {\n      return;\n    }\n    try {\n      if (!new File(dstDir).exists()) {\n        new File(dstDir).mkdirs();\n      }\n      for (String fileName : appCtx.getAssets().list(srcDir)) {\n        String srcSubPath = srcDir + File.separator + fileName;\n        String dstSubPath = dstDir + File.separator + fileName;\n        if (new File(srcSubPath).isDirectory()) {\n          copyDirectoryFromAssets(appCtx, srcSubPath, dstSubPath);\n        } else {\n          copyFileFromAssets(appCtx, srcSubPath, dstSubPath);\n        }\n      }\n    } catch (Exception e) {\n      e.printStackTrace();\n    }\n  }\n\n  public static void copyFileFromAssets(Context appCtx, String srcPath, String dstPath) {\n    File dstFile = new File(dstPath);\n    copyFileFromAssets(appCtx, srcPath, dstFile);\n  }\n\n  public static void copyFileFromAssets(Context appCtx, String srcPath, File dstFile) {\n    if (srcPath.isEmpty()) {\n      return;\n    }\n    InputStream is = null;\n    OutputStream os = null;\n    try {\n      is = new BufferedInputStream(appCtx.getAssets().open(srcPath));\n\n      os = new BufferedOutputStream(new FileOutputStream(dstFile));\n      byte[] buffer = new byte[1024];\n      int length = 0;\n      while ((length = is.read(buffer)) != -1) {\n        os.write(buffer, 0, length);\n      }\n    } catch (FileNotFoundException e) {\n      e.printStackTrace();\n    } catch (IOException e) {\n      e.printStackTrace();\n    } finally {\n      try {\n        os.close();\n        is.close();\n      } catch (IOException e) {\n        e.printStackTrace();\n      }\n    }\n\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/litongjava/whisper/android/java/utils/WaveEncoder.java",
    "content": "package com.litongjava.whisper.android.java.utils;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.nio.ShortBuffer;\n\npublic class WaveEncoder {\n\n  public static float[] decodeWaveFile(File file) throws IOException {\n    ByteArrayOutputStream baos = new ByteArrayOutputStream();\n    try (FileInputStream fis = new FileInputStream(file)) {\n      byte[] buffer = new byte[1024];\n      int bytesRead;\n      while ((bytesRead = fis.read(buffer)) != -1) {\n        baos.write(buffer, 0, bytesRead);\n      }\n    }\n    ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray());\n    byteBuffer.order(ByteOrder.LITTLE_ENDIAN);\n\n    int channel = byteBuffer.getShort(22);\n    byteBuffer.position(44);\n\n    ShortBuffer shortBuffer = byteBuffer.asShortBuffer();\n    short[] shortArray = new short[shortBuffer.limit()];\n    shortBuffer.get(shortArray);\n\n    float[] output = new float[shortArray.length / channel];\n\n    for (int index = 0; index < output.length; index++) {\n      if (channel == 1) {\n        output[index] = Math.max(-1f, Math.min(1f, shortArray[index] / 32767.0f));\n      } else {\n        output[index] = Math.max(-1f, Math.min(1f, (shortArray[2 * index] + shortArray[2 * index + 1]) / 32767.0f / 2.0f));\n      }\n    }\n    return output;\n  }\n\n  public static void encodeWaveFile(File file, short[] data) throws IOException {\n    try (FileOutputStream fos = new FileOutputStream(file)) {\n      fos.write(headerBytes(data.length * 2));\n\n      ByteBuffer buffer = ByteBuffer.allocate(data.length * 2);\n      buffer.order(ByteOrder.LITTLE_ENDIAN);\n      buffer.asShortBuffer().put(data);\n\n      byte[] bytes = new byte[buffer.limit()];\n      buffer.get(bytes);\n\n      fos.write(bytes);\n    }\n  }\n\n  private static byte[] headerBytes(int totalLength) {\n    if (totalLength < 44)\n      throw new IllegalArgumentException(\"Total length must be at least 44 bytes\");\n\n    ByteBuffer buffer = ByteBuffer.allocate(44);\n    buffer.order(ByteOrder.LITTLE_ENDIAN);\n\n    buffer.put((byte) 'R');\n    buffer.put((byte) 'I');\n    buffer.put((byte) 'F');\n    buffer.put((byte) 'F');\n\n    buffer.putInt(totalLength - 8);\n\n    buffer.put((byte) 'W');\n    buffer.put((byte) 'A');\n    buffer.put((byte) 'V');\n    buffer.put((byte) 'E');\n\n    buffer.put((byte) 'f');\n    buffer.put((byte) 'm');\n    buffer.put((byte) 't');\n    buffer.put((byte) ' ');\n\n    buffer.putInt(16);\n    buffer.putShort((short) 1);\n    buffer.putShort((short) 1);\n    buffer.putInt(16000);\n    buffer.putInt(32000);\n    buffer.putShort((short) 2);\n    buffer.putShort((short) 16);\n\n    buffer.put((byte) 'd');\n    buffer.put((byte) 'a');\n    buffer.put((byte) 't');\n    buffer.put((byte) 'a');\n\n    buffer.putInt(totalLength - 44);\n    buffer.position(0);\n\n    byte[] bytes = new byte[buffer.limit()];\n    buffer.get(bytes);\n\n    return bytes;\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/whispercpp/java/whisper/CpuInfo.java",
    "content": "package com.whispercpp.java.whisper;\n\nimport android.os.Build;\nimport android.util.Log;\n\nimport androidx.annotation.RequiresApi;\n\nimport java.io.BufferedReader;\nimport java.io.FileReader;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\npublic class CpuInfo {\n  private static final String LOG_TAG = \"WhisperCpuConfig\";\n\n  private List<String> lines;\n\n  public CpuInfo(List<String> lines) {\n    this.lines = lines;\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  public int getHighPerfCpuCount0() {\n    try {\n      return getHighPerfCpuCountByFrequencies();\n    } catch (Exception e) {\n      Log.d(LOG_TAG, \"Couldn't read CPU frequencies\", e);\n      return getHighPerfCpuCountByVariant();\n    }\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private int getHighPerfCpuCountByFrequencies() {\n    List<Integer> frequencies = getCpuValues(\"processor\", line -> {\n        try {\n          return getMaxCpuFrequency(Integer.parseInt(line.trim()));\n        } catch (IOException e) {\n          e.printStackTrace();\n        }\n        return 0;\n      }\n    );\n    Log.d(LOG_TAG, \"Binned cpu frequencies (frequency, count): \" + binnedValues(frequencies));\n    return countDroppingMin(frequencies);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private int getHighPerfCpuCountByVariant() {\n    List<Integer> variants = getCpuValues(\"CPU variant\", line -> Integer.parseInt(line.trim().substring(line.indexOf(\"0x\") + 2), 16));\n    Log.d(LOG_TAG, \"Binned cpu variants (variant, count): \" + binnedValues(variants));\n    return countKeepingMin(variants);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private Map<Integer, Integer> binnedValues(List<Integer> values) {\n    Map<Integer, Integer> countMap = new HashMap<>();\n    for (int value : values) {\n      countMap.put(value, countMap.getOrDefault(value, 0) + 1);\n    }\n    return countMap;\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private List<Integer> getCpuValues(String property, Mapper mapper) {\n    List<Integer> values = new ArrayList<>();\n    for (String line : lines) {\n      if (line.startsWith(property)) {\n        values.add(mapper.map(line.substring(line.indexOf(':') + 1)));\n      }\n    }\n    values.sort(Integer::compareTo);\n    return values;\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private int countDroppingMin(List<Integer> values) {\n    int min = values.stream().mapToInt(i -> i).min().orElse(Integer.MAX_VALUE);\n    return (int) values.stream().filter(value -> value > min).count();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  private int countKeepingMin(List<Integer> values) {\n    int min = values.stream().mapToInt(i -> i).min().orElse(Integer.MAX_VALUE);\n    return (int) values.stream().filter(value -> value.equals(min)).count();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  public static int getHighPerfCpuCount() {\n    try {\n      return readCpuInfo().getHighPerfCpuCount0();\n    } catch (Exception e) {\n      Log.d(LOG_TAG, \"Couldn't read CPU info\", e);\n      return Math.max(Runtime.getRuntime().availableProcessors() - 4, 0);\n    }\n  }\n\n  private static CpuInfo readCpuInfo() throws IOException {\n    try (BufferedReader reader = new BufferedReader(new FileReader(\"/proc/cpuinfo\"))) {\n      List<String> lines = new ArrayList<>();\n      String line;\n      while ((line = reader.readLine()) != null) {\n        lines.add(line);\n      }\n      return new CpuInfo(lines);\n    }\n  }\n\n  private static int getMaxCpuFrequency(int cpuIndex) throws IOException {\n    String path = \"/sys/devices/system/cpu/cpu\" + cpuIndex + \"/cpufreq/cpuinfo_max_freq\";\n    try (BufferedReader reader = new BufferedReader(new FileReader(path))) {\n      return Integer.parseInt(reader.readLine());\n    }\n  }\n\n  private interface Mapper {\n    int map(String line);\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/whispercpp/java/whisper/WhisperContext.java",
    "content": "package com.whispercpp.java.whisper;\n\nimport android.content.res.AssetManager;\nimport android.os.Build;\nimport android.util.Log;\n\nimport androidx.annotation.RequiresApi;\n\nimport com.litongjava.whisper.android.java.bean.WhisperSegment;\n\nimport java.io.InputStream;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\n\npublic class WhisperContext {\n\n  private static final String LOG_TAG = \"LibWhisper\";\n  private long ptr;\n  private final ExecutorService executorService;\n\n  private WhisperContext(long ptr) {\n    this.ptr = ptr;\n    this.executorService = Executors.newSingleThreadExecutor();\n  }\n\n  public String transcribeData(float[] data) throws ExecutionException, InterruptedException {\n    return executorService.submit(new Callable<String>() {\n      @RequiresApi(api = Build.VERSION_CODES.O)\n      @Override\n      public String call() throws Exception {\n        if (ptr == 0L) {\n          throw new IllegalStateException();\n        }\n        int numThreads = WhisperCpuConfig.getPreferredThreadCount();\n        Log.d(LOG_TAG, \"Selecting \" + numThreads + \" threads\");\n\n        StringBuilder result = new StringBuilder();\n        synchronized (this) {\n\n          WhisperLib.fullTranscribe(ptr, numThreads, data);\n          int textCount = WhisperLib.getTextSegmentCount(ptr);\n          for (int i = 0; i < textCount; i++) {\n            String sentence = WhisperLib.getTextSegment(ptr, i);\n            result.append(sentence);\n          }\n        }\n        return result.toString();\n      }\n    }).get();\n  }\n\n  public List<WhisperSegment> transcribeDataWithTime(float[] data) throws ExecutionException, InterruptedException {\n    return executorService.submit(new Callable<List<WhisperSegment>>() {\n      @RequiresApi(api = Build.VERSION_CODES.O)\n      @Override\n      public List<WhisperSegment> call() throws Exception {\n        if (ptr == 0L) {\n          throw new IllegalStateException();\n        }\n        int numThreads = WhisperCpuConfig.getPreferredThreadCount();\n        Log.d(LOG_TAG, \"Selecting \" + numThreads + \" threads\");\n\n        List<WhisperSegment> segments = new ArrayList<>();\n        synchronized (this) {\n//          StringBuilder result = new StringBuilder();\n          WhisperLib.fullTranscribe(ptr, numThreads, data);\n          int textCount = WhisperLib.getTextSegmentCount(ptr);\n          for (int i = 0; i < textCount; i++) {\n            long start = WhisperLib.getTextSegmentT0(ptr, i);\n            String sentence = WhisperLib.getTextSegment(ptr, i);\n            long end = WhisperLib.getTextSegmentT1(ptr, i);\n//            result.append();\n            segments.add(new WhisperSegment(start, end, sentence));\n\n          }\n//          return result.toString();\n        }\n        return segments;\n      }\n    }).get();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public String benchMemory(int nthreads) throws ExecutionException, InterruptedException {\n    return executorService.submit(() -> WhisperLib.benchMemcpy(nthreads)).get();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public String benchGgmlMulMat(int nthreads) throws ExecutionException, InterruptedException {\n    return executorService.submit(() -> WhisperLib.benchGgmlMulMat(nthreads)).get();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public void release() throws ExecutionException, InterruptedException {\n    executorService.submit(() -> {\n      if (ptr != 0L) {\n        WhisperLib.freeContext(ptr);\n        ptr = 0;\n      }\n    }).get();\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public static WhisperContext createContextFromFile(String filePath) {\n    long ptr = WhisperLib.initContext(filePath);\n    if (ptr == 0L) {\n      throw new RuntimeException(\"Couldn't create context with path \" + filePath);\n    }\n    return new WhisperContext(ptr);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public static WhisperContext createContextFromInputStream(InputStream stream) {\n    long ptr = WhisperLib.initContextFromInputStream(stream);\n    if (ptr == 0L) {\n      throw new RuntimeException(\"Couldn't create context from input stream\");\n    }\n    return new WhisperContext(ptr);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public static WhisperContext createContextFromAsset(AssetManager assetManager, String assetPath) {\n    long ptr = WhisperLib.initContextFromAsset(assetManager, assetPath);\n    if (ptr == 0L) {\n      throw new RuntimeException(\"Couldn't create context from asset \" + assetPath);\n    }\n    return new WhisperContext(ptr);\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public static String getSystemInfo() {\n    return WhisperLib.getSystemInfo();\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/whispercpp/java/whisper/WhisperCpuConfig.java",
    "content": "package com.whispercpp.java.whisper;\n\nimport android.os.Build;\n\nimport androidx.annotation.RequiresApi;\n\npublic class WhisperCpuConfig {\n  @RequiresApi(api = Build.VERSION_CODES.N)\n  public static int getPreferredThreadCount() {\n    return Math.max(CpuInfo.getHighPerfCpuCount(), 2);\n  }\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/whispercpp/java/whisper/WhisperLib.java",
    "content": "package com.whispercpp.java.whisper;\n\nimport android.content.res.AssetManager;\nimport android.os.Build;\nimport android.util.Log;\n\nimport androidx.annotation.RequiresApi;\n\nimport java.io.InputStream;\n\n@RequiresApi(api = Build.VERSION_CODES.O)\npublic class WhisperLib {\n  private static final String LOG_TAG = \"LibWhisper\";\n\n  static {\n\n    Log.d(LOG_TAG, \"Primary ABI: \" + Build.SUPPORTED_ABIS[0]);\n    boolean loadVfpv4 = false;\n    boolean loadV8fp16 = false;\n    if (WhisperUtils.isArmEabiV7a()) {\n      String cpuInfo = WhisperUtils.cpuInfo();\n      if (cpuInfo != null) {\n        Log.d(LOG_TAG, \"CPU info: \" + cpuInfo);\n        if (cpuInfo.contains(\"vfpv4\")) {\n          Log.d(LOG_TAG, \"CPU supports vfpv4\");\n          loadVfpv4 = true;\n        }\n      }\n    } else if (WhisperUtils.isArmEabiV8a()) {\n      String cpuInfo = WhisperUtils.cpuInfo();\n      if (cpuInfo != null) {\n        Log.d(LOG_TAG, \"CPU info: \" + cpuInfo);\n        if (cpuInfo.contains(\"fphp\")) {\n          Log.d(LOG_TAG, \"CPU supports fp16 arithmetic\");\n          loadV8fp16 = true;\n        }\n      }\n    }\n\n    if (loadVfpv4) {\n      Log.d(LOG_TAG, \"Loading libwhisper_vfpv4.so\");\n      System.loadLibrary(\"whisper_vfpv4\");\n    } else if (loadV8fp16) {\n      Log.d(LOG_TAG, \"Loading libwhisper_v8fp16_va.so\");\n      System.loadLibrary(\"whisper_v8fp16_va\");\n    } else {\n      Log.d(LOG_TAG, \"Loading libwhisper.so\");\n      System.loadLibrary(\"whisper\");\n    }\n  }\n\n  public static native long initContextFromInputStream(InputStream inputStream);\n\n  public static native long initContextFromAsset(AssetManager assetManager, String assetPath);\n\n  public static native long initContext(String modelPath);\n\n  public static native void freeContext(long contextPtr);\n\n  public static native void fullTranscribe(long contextPtr, int numThreads, float[] audioData);\n\n  public static native int getTextSegmentCount(long contextPtr);\n\n  public static native String getTextSegment(long contextPtr, int index);\n\n  public static native long getTextSegmentT0(long contextPtr, int index);\n\n  public static native long getTextSegmentT1(long contextPtr, int index);\n\n  public static native String getSystemInfo();\n\n  public static native String benchMemcpy(int nthread);\n\n  public static native String benchGgmlMulMat(int nthread);\n}\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/java/com/whispercpp/java/whisper/WhisperUtils.java",
    "content": "package com.whispercpp.java.whisper;\n\nimport android.os.Build;\nimport android.util.Log;\n\nimport androidx.annotation.RequiresApi;\n\nimport java.io.File;\nimport java.nio.file.Path;\n\npublic class WhisperUtils {\n  private static final String LOG_TAG = \"LibWhisper\";\n\n\n  public static boolean isArmEabiV7a() {\n    return Build.SUPPORTED_ABIS[0].equals(\"armeabi-v7a\");\n  }\n\n  public static boolean isArmEabiV8a() {\n    return Build.SUPPORTED_ABIS[0].equals(\"arm64-v8a\");\n  }\n\n  @RequiresApi(api = Build.VERSION_CODES.O)\n  public static String cpuInfo() {\n    try {\n      Path path = new File(\"/proc/cpuinfo\").toPath();\n      return new String(java.nio.file.Files.readAllBytes(path));\n    } catch (Exception e) {\n      Log.w(LOG_TAG, \"Couldn't read /proc/cpuinfo\", e);\n      return null;\n    }\n\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.10)\n\nproject(whisper.cpp)\n\nset(CMAKE_CXX_STANDARD 17)\nset(WHISPER_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../../../../../)\n\n# Get whisper.cpp version\nfile(READ \"${WHISPER_LIB_DIR}/CMakeLists.txt\" MAIN_CMAKE_CONTENT)\nstring(REGEX MATCH \"project\\\\(\\\"whisper\\\\.cpp\\\" VERSION ([0-9]+\\\\.[0-9]+\\\\.[0-9]+)\\\\)\" VERSION_MATCH \"${MAIN_CMAKE_CONTENT}\")\nif(CMAKE_MATCH_1)\n    set(WHISPER_VERSION ${CMAKE_MATCH_1} PARENT_SCOPE)\nelse()\n    set(WHISPER_VERSION \"unknown\" PARENT_SCOPE)\nendif()\n\nmessage(STATUS \" Whisper version: ${WHISPER_VERSION}\")\n\nset(SOURCE_FILES\n    ${WHISPER_LIB_DIR}/src/whisper.cpp\n    ${CMAKE_SOURCE_DIR}/jni.c\n    )\n\nfind_library(LOG_LIB log)\n\ninclude(FetchContent)\n\nfunction(build_library target_name)\n    add_library(\n        ${target_name}\n        SHARED\n        ${SOURCE_FILES}\n    )\n\n    FetchContent_Declare(ggml SOURCE_DIR ${WHISPER_LIB_DIR}/ggml)\n    FetchContent_MakeAvailable(ggml)\n\n    target_link_libraries(${target_name} ${LOG_LIB} android ggml)\n    target_compile_definitions(${target_name} PUBLIC GGML_USE_CPU)\n    target_compile_definitions(${target_name} PRIVATE WHISPER_VERSION=\"${WHISPER_VERSION}\")\n\n    if (${target_name} STREQUAL \"whisper_v8fp16_va\")\n        target_compile_options(${target_name} PRIVATE -march=armv8.2-a+fp16)\n    elseif (${target_name} STREQUAL \"whisper_vfpv4\")\n        target_compile_options(${target_name} PRIVATE -mfpu=neon-vfpv4)\n    endif ()\n\n    if (NOT ${CMAKE_BUILD_TYPE} STREQUAL \"Debug\")\n\n        target_compile_options(${target_name} PRIVATE -O3)\n        target_compile_options(${target_name} PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden)\n        target_compile_options(${target_name} PRIVATE -ffunction-sections -fdata-sections)\n\n        #target_link_options(${target_name} PRIVATE -Wl,--gc-sections)\n        #target_link_options(${target_name} PRIVATE -Wl,--exclude-libs,ALL)\n        #target_link_options(${target_name} PRIVATE -flto)\n    endif ()\nendfunction()\n\nbuild_library(\"whisper\") # Default target\n\nif (${ANDROID_ABI} STREQUAL \"arm64-v8a\")\n    build_library(\"whisper_v8fp16_va\")\nelseif (${ANDROID_ABI} STREQUAL \"armeabi-v7a\")\n    build_library(\"whisper_vfpv4\")\nendif ()\n\ninclude_directories(${WHISPER_LIB_DIR})\ninclude_directories(${WHISPER_LIB_DIR}/src)\ninclude_directories(${WHISPER_LIB_DIR}/include)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/include)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/src)\ninclude_directories(${WHISPER_LIB_DIR}/ggml/src/ggml-cpu)\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/jni/whisper/jni.c",
    "content": "#include <jni.h>\n#include <android/asset_manager.h>\n#include <android/asset_manager_jni.h>\n#include <android/log.h>\n#include <stdlib.h>\n#include <sys/sysinfo.h>\n#include <string.h>\n#include \"whisper.h\"\n#include \"ggml.h\"\n\n#define UNUSED(x) (void)(x)\n#define TAG \"JNI\"\n\n#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,     TAG, __VA_ARGS__)\n#define LOGW(...) __android_log_print(ANDROID_LOG_WARN,     TAG, __VA_ARGS__)\n\nstatic inline int min(int a, int b) {\n    return (a < b) ? a : b;\n}\n\nstatic inline int max(int a, int b) {\n    return (a > b) ? a : b;\n}\n\nstruct input_stream_context {\n    size_t offset;\n    JNIEnv * env;\n    jobject thiz;\n    jobject input_stream;\n\n    jmethodID mid_available;\n    jmethodID mid_read;\n};\n\nsize_t inputStreamRead(void * ctx, void * output, size_t read_size) {\n    struct input_stream_context* is = (struct input_stream_context*)ctx;\n\n    jint avail_size = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_available);\n    jint size_to_copy = read_size < avail_size ? (jint)read_size : avail_size;\n\n    jbyteArray byte_array = (*is->env)->NewByteArray(is->env, size_to_copy);\n\n    jint n_read = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_read, byte_array, 0, size_to_copy);\n\n    if (size_to_copy != read_size || size_to_copy != n_read) {\n        LOGI(\"Insufficient Read: Req=%zu, ToCopy=%d, Available=%d\", read_size, size_to_copy, n_read);\n    }\n\n    jbyte* byte_array_elements = (*is->env)->GetByteArrayElements(is->env, byte_array, NULL);\n    memcpy(output, byte_array_elements, size_to_copy);\n    (*is->env)->ReleaseByteArrayElements(is->env, byte_array, byte_array_elements, JNI_ABORT);\n\n    (*is->env)->DeleteLocalRef(is->env, byte_array);\n\n    is->offset += size_to_copy;\n\n    return size_to_copy;\n}\nbool inputStreamEof(void * ctx) {\n    struct input_stream_context* is = (struct input_stream_context*)ctx;\n\n    jint result = (*is->env)->CallIntMethod(is->env, is->input_stream, is->mid_available);\n    return result <= 0;\n}\nvoid inputStreamClose(void * ctx) {\n\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_initContextFromInputStream(\n        JNIEnv *env, jobject thiz, jobject input_stream) {\n    UNUSED(thiz);\n\n    struct whisper_context *context = NULL;\n    struct whisper_model_loader loader = {};\n    struct input_stream_context inp_ctx = {};\n\n    inp_ctx.offset = 0;\n    inp_ctx.env = env;\n    inp_ctx.thiz = thiz;\n    inp_ctx.input_stream = input_stream;\n\n    jclass cls = (*env)->GetObjectClass(env, input_stream);\n    inp_ctx.mid_available = (*env)->GetMethodID(env, cls, \"available\", \"()I\");\n    inp_ctx.mid_read = (*env)->GetMethodID(env, cls, \"read\", \"([BII)I\");\n\n    loader.context = &inp_ctx;\n    loader.read = inputStreamRead;\n    loader.eof = inputStreamEof;\n    loader.close = inputStreamClose;\n\n    loader.eof(loader.context);\n\n    context = whisper_init(&loader);\n    return (jlong) context;\n}\n\nstatic size_t asset_read(void *ctx, void *output, size_t read_size) {\n    return AAsset_read((AAsset *) ctx, output, read_size);\n}\n\nstatic bool asset_is_eof(void *ctx) {\n    return AAsset_getRemainingLength64((AAsset *) ctx) <= 0;\n}\n\nstatic void asset_close(void *ctx) {\n    AAsset_close((AAsset *) ctx);\n}\n\nstatic struct whisper_context *whisper_init_from_asset(\n        JNIEnv *env,\n        jobject assetManager,\n        const char *asset_path\n) {\n    LOGI(\"Loading model from asset '%s'\\n\", asset_path);\n    AAssetManager *asset_manager = AAssetManager_fromJava(env, assetManager);\n    AAsset *asset = AAssetManager_open(asset_manager, asset_path, AASSET_MODE_STREAMING);\n    if (!asset) {\n        LOGW(\"Failed to open '%s'\\n\", asset_path);\n        return NULL;\n    }\n\n    whisper_model_loader loader = {\n            .context = asset,\n            .read = &asset_read,\n            .eof = &asset_is_eof,\n            .close = &asset_close\n    };\n\n    return whisper_init(&loader);\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_initContextFromAsset(\n        JNIEnv *env, jobject thiz, jobject assetManager, jstring asset_path_str) {\n    UNUSED(thiz);\n    struct whisper_context *context = NULL;\n    const char *asset_path_chars = (*env)->GetStringUTFChars(env, asset_path_str, NULL);\n    context = whisper_init_from_asset(env, assetManager, asset_path_chars);\n    (*env)->ReleaseStringUTFChars(env, asset_path_str, asset_path_chars);\n    return (jlong) context;\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_initContext(\n        JNIEnv *env, jobject thiz, jstring model_path_str) {\n    UNUSED(thiz);\n    struct whisper_context *context = NULL;\n    const char *model_path_chars = (*env)->GetStringUTFChars(env, model_path_str, NULL);\n    context = whisper_init_from_file(model_path_chars);\n    (*env)->ReleaseStringUTFChars(env, model_path_str, model_path_chars);\n    return (jlong) context;\n}\n\nJNIEXPORT void JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_freeContext(\n        JNIEnv *env, jobject thiz, jlong context_ptr) {\n    UNUSED(env);\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    whisper_free(context);\n}\n\nJNIEXPORT void JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_fullTranscribe(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint num_threads, jfloatArray audio_data) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    jfloat *audio_data_arr = (*env)->GetFloatArrayElements(env, audio_data, NULL);\n    const jsize audio_data_length = (*env)->GetArrayLength(env, audio_data);\n\n    // The below adapted from the Objective-C iOS sample\n    struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n    params.print_realtime = true;\n    params.print_progress = false;\n    params.print_timestamps = true;\n    params.print_special = false;\n    params.translate = false;\n    params.language = \"en\";\n    params.n_threads = num_threads;\n    params.offset_ms = 0;\n    params.no_context = true;\n    params.single_segment = false;\n\n    whisper_reset_timings(context);\n\n    LOGI(\"About to run whisper_full\");\n    if (whisper_full(context, params, audio_data_arr, audio_data_length) != 0) {\n        LOGI(\"Failed to run the model\");\n    } else {\n        whisper_print_timings(context);\n    }\n    (*env)->ReleaseFloatArrayElements(env, audio_data, audio_data_arr, JNI_ABORT);\n}\n\nJNIEXPORT jint JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_getTextSegmentCount(\n        JNIEnv *env, jobject thiz, jlong context_ptr) {\n    UNUSED(env);\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    return whisper_full_n_segments(context);\n}\n\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_getTextSegment(\n        JNIEnv *env, jobject thiz, jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    const char *text = whisper_full_get_segment_text(context, index);\n    jstring string = (*env)->NewStringUTF(env, text);\n    return string;\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_getTextSegmentT0(JNIEnv *env, jobject thiz,jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    const int64_t t0 = whisper_full_get_segment_t0(context, index);\n    return (jlong)t0;\n}\n\nJNIEXPORT jlong JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_getTextSegmentT1(JNIEnv *env, jobject thiz,jlong context_ptr, jint index) {\n    UNUSED(thiz);\n    struct whisper_context *context = (struct whisper_context *) context_ptr;\n    const int64_t t1 = whisper_full_get_segment_t1(context, index);\n    return (jlong)t1;\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_getSystemInfo(\n        JNIEnv *env, jobject thiz\n) {\n    UNUSED(thiz);\n    const char *sysinfo = whisper_print_system_info();\n    jstring string = (*env)->NewStringUTF(env, sysinfo);\n    return string;\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_benchMemcpy(JNIEnv *env, jobject thiz,\n                                                                      jint n_threads) {\n    UNUSED(thiz);\n    const char *bench_ggml_memcpy = whisper_bench_memcpy_str(n_threads);\n    jstring string = (*env)->NewStringUTF(env, bench_ggml_memcpy);\n\n    return string;\n}\n\nJNIEXPORT jstring JNICALL\nJava_com_whispercpp_java_whisper_WhisperLib_benchGgmlMulMat(JNIEnv *env, jobject thiz,\n                                                                          jint n_threads) {\n    UNUSED(thiz);\n    const char *bench_ggml_mul_mat = whisper_bench_ggml_mul_mat_str(n_threads);\n    jstring string = (*env)->NewStringUTF(env, bench_ggml_mul_mat);\n\n    return string;\n}\n\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/drawable/ic_launcher_background.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<vector xmlns:android=\"http://schemas.android.com/apk/res/android\"\n  android:width=\"108dp\"\n  android:height=\"108dp\"\n  android:viewportWidth=\"108\"\n  android:viewportHeight=\"108\">\n  <path\n    android:fillColor=\"#3DDC84\"\n    android:pathData=\"M0,0h108v108h-108z\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M9,0L9,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,0L19,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M29,0L29,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M39,0L39,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M49,0L49,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M59,0L59,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M69,0L69,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M79,0L79,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M89,0L89,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M99,0L99,108\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,9L108,9\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,19L108,19\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,29L108,29\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,39L108,39\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,49L108,49\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,59L108,59\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,69L108,69\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,79L108,79\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,89L108,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M0,99L108,99\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,29L89,29\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,39L89,39\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,49L89,49\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,59L89,59\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,69L89,69\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M19,79L89,79\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M29,19L29,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M39,19L39,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M49,19L49,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M59,19L59,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M69,19L69,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n  <path\n    android:fillColor=\"#00000000\"\n    android:pathData=\"M79,19L79,89\"\n    android:strokeWidth=\"0.8\"\n    android:strokeColor=\"#33FFFFFF\" />\n</vector>\n"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/drawable-v24/ic_launcher_foreground.xml",
    "content": "<vector xmlns:android=\"http://schemas.android.com/apk/res/android\"\n  xmlns:aapt=\"http://schemas.android.com/aapt\"\n  android:width=\"108dp\"\n  android:height=\"108dp\"\n  android:viewportWidth=\"108\"\n  android:viewportHeight=\"108\">\n  <path android:pathData=\"M31,63.928c0,0 6.4,-11 12.1,-13.1c7.2,-2.6 26,-1.4 26,-1.4l38.1,38.1L107,108.928l-32,-1L31,63.928z\">\n    <aapt:attr name=\"android:fillColor\">\n      <gradient\n        android:endX=\"85.84757\"\n        android:endY=\"92.4963\"\n        android:startX=\"42.9492\"\n        android:startY=\"49.59793\"\n        android:type=\"linear\">\n        <item\n          android:color=\"#44000000\"\n          android:offset=\"0.0\" />\n        <item\n          android:color=\"#00000000\"\n          android:offset=\"1.0\" />\n      </gradient>\n    </aapt:attr>\n  </path>\n  <path\n    android:fillColor=\"#FFFFFF\"\n    android:fillType=\"nonZero\"\n    android:pathData=\"M65.3,45.828l3.8,-6.6c0.2,-0.4 0.1,-0.9 -0.3,-1.1c-0.4,-0.2 -0.9,-0.1 -1.1,0.3l-3.9,6.7c-6.3,-2.8 -13.4,-2.8 -19.7,0l-3.9,-6.7c-0.2,-0.4 -0.7,-0.5 -1.1,-0.3C38.8,38.328 38.7,38.828 38.9,39.228l3.8,6.6C36.2,49.428 31.7,56.028 31,63.928h46C76.3,56.028 71.8,49.428 65.3,45.828zM43.4,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2c-0.3,-0.7 -0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C45.3,56.528 44.5,57.328 43.4,57.328L43.4,57.328zM64.6,57.328c-0.8,0 -1.5,-0.5 -1.8,-1.2s-0.1,-1.5 0.4,-2.1c0.5,-0.5 1.4,-0.7 2.1,-0.4c0.7,0.3 1.2,1 1.2,1.8C66.5,56.528 65.6,57.328 64.6,57.328L64.6,57.328z\"\n    android:strokeWidth=\"1\"\n    android:strokeColor=\"#00000000\" />\n</vector>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/layout/activity_main.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<LinearLayout xmlns:android=\"http://schemas.android.com/apk/res/android\"\n  xmlns:app=\"http://schemas.android.com/apk/res-auto\"\n  xmlns:tools=\"http://schemas.android.com/tools\"\n  android:layout_width=\"match_parent\"\n  android:layout_height=\"match_parent\"\n  android:orientation=\"vertical\"\n  tools:context=\".MainActivity\">\n\n  <LinearLayout\n    android:layout_width=\"match_parent\"\n    android:layout_height=\"wrap_content\">\n\n    <Button\n      android:id=\"@+id/systemInfoBtn\"\n      android:layout_width=\"wrap_content\"\n      android:layout_height=\"wrap_content\"\n      android:text=\"System Info\" />\n\n    <Button\n      android:id=\"@+id/loadModelBtn\"\n      android:layout_width=\"wrap_content\"\n      android:layout_height=\"wrap_content\"\n      android:text=\"Load model\" />\n\n  </LinearLayout>\n\n  <LinearLayout\n    android:layout_width=\"wrap_content\"\n    android:layout_height=\"wrap_content\">\n\n    <Button\n      android:id=\"@+id/transcriptSampleBtn\"\n      android:layout_width=\"wrap_content\"\n      android:layout_height=\"wrap_content\"\n      android:text=\"Transcribe sample\" />\n\n    <Button\n      android:id=\"@+id/clearBtn\"\n      android:layout_width=\"wrap_content\"\n      android:layout_height=\"wrap_content\"\n      android:text=\"Clear\" />\n  </LinearLayout>\n\n  <TextView\n    android:id=\"@+id/sample_text\"\n    android:layout_width=\"match_parent\"\n    android:layout_height=\"wrap_content\"\n    android:text=\"Hello World!\"\n    app:layout_constraintBottom_toBottomOf=\"parent\"\n    app:layout_constraintLeft_toLeftOf=\"parent\"\n    app:layout_constraintRight_toRightOf=\"parent\"\n    app:layout_constraintTop_toTopOf=\"parent\"\n    android:scrollbarAlwaysDrawHorizontalTrack=\"true\"\n    android:maxLines=\"999\"/>\n\n</LinearLayout>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<adaptive-icon xmlns:android=\"http://schemas.android.com/apk/res/android\">\n  <background android:drawable=\"@drawable/ic_launcher_background\" />\n  <foreground android:drawable=\"@drawable/ic_launcher_foreground\" />\n</adaptive-icon>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<adaptive-icon xmlns:android=\"http://schemas.android.com/apk/res/android\">\n  <background android:drawable=\"@drawable/ic_launcher_background\" />\n  <foreground android:drawable=\"@drawable/ic_launcher_foreground\" />\n</adaptive-icon>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/values/colors.xml",
    "content": "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<resources>\n  <color name=\"purple_200\">#FFBB86FC</color>\n  <color name=\"purple_500\">#FF6200EE</color>\n  <color name=\"purple_700\">#FF3700B3</color>\n  <color name=\"teal_200\">#FF03DAC5</color>\n  <color name=\"teal_700\">#FF018786</color>\n  <color name=\"black\">#FF000000</color>\n  <color name=\"white\">#FFFFFFFF</color>\n</resources>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/values/strings.xml",
    "content": "<resources>\n  <string name=\"app_name\">whisper.android.java</string>\n</resources>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/values/themes.xml",
    "content": "<resources xmlns:tools=\"http://schemas.android.com/tools\">\n  <!-- Base application theme. -->\n  <style name=\"Theme.Whisperandroidjava\" parent=\"Theme.MaterialComponents.DayNight.DarkActionBar\">\n    <!-- Primary brand color. -->\n    <item name=\"colorPrimary\">@color/purple_500</item>\n    <item name=\"colorPrimaryVariant\">@color/purple_700</item>\n    <item name=\"colorOnPrimary\">@color/white</item>\n    <!-- Secondary brand color. -->\n    <item name=\"colorSecondary\">@color/teal_200</item>\n    <item name=\"colorSecondaryVariant\">@color/teal_700</item>\n    <item name=\"colorOnSecondary\">@color/black</item>\n    <!-- Status bar color. -->\n    <item name=\"android:statusBarColor\" tools:targetApi=\"l\">?attr/colorPrimaryVariant</item>\n    <!-- Customize your theme here. -->\n  </style>\n</resources>"
  },
  {
    "path": "examples/whisper.android.java/app/src/main/res/values-night/themes.xml",
    "content": "<resources xmlns:tools=\"http://schemas.android.com/tools\">\n  <!-- Base application theme. -->\n  <style name=\"Theme.Whisperandroidjava\" parent=\"Theme.MaterialComponents.DayNight.DarkActionBar\">\n    <!-- Primary brand color. -->\n    <item name=\"colorPrimary\">@color/purple_200</item>\n    <item name=\"colorPrimaryVariant\">@color/purple_700</item>\n    <item name=\"colorOnPrimary\">@color/black</item>\n    <!-- Secondary brand color. -->\n    <item name=\"colorSecondary\">@color/teal_200</item>\n    <item name=\"colorSecondaryVariant\">@color/teal_200</item>\n    <item name=\"colorOnSecondary\">@color/black</item>\n    <!-- Status bar color. -->\n    <item name=\"android:statusBarColor\" tools:targetApi=\"l\">?attr/colorPrimaryVariant</item>\n    <!-- Customize your theme here. -->\n  </style>\n</resources>"
  },
  {
    "path": "examples/whisper.android.java/app/src/test/java/com/litongjava/whisper/android/java/ExampleUnitTest.java",
    "content": "package com.litongjava.whisper.android.java;\n\nimport org.junit.Test;\n\nimport static org.junit.Assert.*;\n\n/**\n * Example local unit test, which will execute on the development machine (host).\n *\n * @see <a href=\"http://d.android.com/tools/testing\">Testing documentation</a>\n */\npublic class ExampleUnitTest {\n  @Test\n  public void addition_isCorrect() {\n    assertEquals(4, 2 + 2);\n  }\n}"
  },
  {
    "path": "examples/whisper.android.java/build.gradle",
    "content": "// Top-level build file where you can add configuration options common to all sub-projects/modules.\nbuildscript {\n  repositories {\n    google()\n    mavenCentral()\n  }\n  dependencies {\n    classpath \"com.android.tools.build:gradle:7.4.0\"\n\n    // NOTE: Do not place your application dependencies here; they belong\n    // in the individual module build.gradle files\n  }\n}\n\nallprojects {\n  repositories {\n    google()\n    mavenCentral()\n    maven { url \"https://maven.aliyun.com/repository/gradle-plugin\" }\n  }\n}\n\ntask clean(type: Delete) {\n  delete rootProject.buildDir\n}\n"
  },
  {
    "path": "examples/whisper.android.java/gradle/wrapper/gradle-wrapper.properties",
    "content": "#Fri Oct 20 11:07:15 HST 2023\ndistributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\nzipStoreBase=GRADLE_USER_HOME\nzipStorePath=wrapper/dists\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-7.5-all.zip\n"
  },
  {
    "path": "examples/whisper.android.java/gradle.properties",
    "content": "# Project-wide Gradle settings.\n# IDE (e.g. Android Studio) users:\n# Gradle settings configured through the IDE *will override*\n# any settings specified in this file.\n# For more details on how to configure your build environment visit\n# http://www.gradle.org/docs/current/userguide/build_environment.html\n# Specifies the JVM arguments used for the daemon process.\n# The setting is particularly useful for tweaking memory settings.\norg.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8\n# When configured, Gradle will run in incubating parallel mode.\n# This option should only be used with decoupled projects. More details, visit\n# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects\n# org.gradle.parallel=true\n# AndroidX package structure to make it clearer which packages are bundled with the\n# Android operating system, and which are packaged with your app\"s APK\n# https://developer.android.com/topic/libraries/support-library/androidx-rn\nandroid.useAndroidX=true\n# Automatically convert third-party libraries to use AndroidX\nandroid.enableJetifier=true"
  },
  {
    "path": "examples/whisper.android.java/gradlew",
    "content": "#!/usr/bin/env sh\n\n##############################################################################\n##\n##  Gradle start up script for UN*X\n##\n##############################################################################\n\n# Attempt to set APP_HOME\n# Resolve links: $0 may be a link\nPRG=\"$0\"\n# Need this for relative symlinks.\nwhile [ -h \"$PRG\" ] ; do\n    ls=`ls -ld \"$PRG\"`\n    link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n    if expr \"$link\" : '/.*' > /dev/null; then\n        PRG=\"$link\"\n    else\n        PRG=`dirname \"$PRG\"`\"/$link\"\n    fi\ndone\nSAVED=\"`pwd`\"\ncd \"`dirname \\\"$PRG\\\"`/\" >/dev/null\nAPP_HOME=\"`pwd -P`\"\ncd \"$SAVED\" >/dev/null\n\nAPP_NAME=\"Gradle\"\nAPP_BASE_NAME=`basename \"$0\"`\n\n# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nDEFAULT_JVM_OPTS=\"\"\n\n# Use the maximum available, or set MAX_FD != -1 to use that value.\nMAX_FD=\"maximum\"\n\nwarn () {\n    echo \"$*\"\n}\n\ndie () {\n    echo\n    echo \"$*\"\n    echo\n    exit 1\n}\n\n# OS specific support (must be 'true' or 'false').\ncygwin=false\nmsys=false\ndarwin=false\nnonstop=false\ncase \"`uname`\" in\n  CYGWIN* )\n    cygwin=true\n    ;;\n  Darwin* )\n    darwin=true\n    ;;\n  MINGW* )\n    msys=true\n    ;;\n  NONSTOP* )\n    nonstop=true\n    ;;\nesac\n\nCLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar\n\n# Determine the Java command to use to start the JVM.\nif [ -n \"$JAVA_HOME\" ] ; then\n    if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n        # IBM's JDK on AIX uses strange locations for the executables\n        JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n    else\n        JAVACMD=\"$JAVA_HOME/bin/java\"\n    fi\n    if [ ! -x \"$JAVACMD\" ] ; then\n        die \"ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\n    fi\nelse\n    JAVACMD=\"java\"\n    which java >/dev/null 2>&1 || die \"ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\nfi\n\n# Increase the maximum file descriptors if we can.\nif [ \"$cygwin\" = \"false\" -a \"$darwin\" = \"false\" -a \"$nonstop\" = \"false\" ] ; then\n    MAX_FD_LIMIT=`ulimit -H -n`\n    if [ $? -eq 0 ] ; then\n        if [ \"$MAX_FD\" = \"maximum\" -o \"$MAX_FD\" = \"max\" ] ; then\n            MAX_FD=\"$MAX_FD_LIMIT\"\n        fi\n        ulimit -n $MAX_FD\n        if [ $? -ne 0 ] ; then\n            warn \"Could not set maximum file descriptor limit: $MAX_FD\"\n        fi\n    else\n        warn \"Could not query maximum file descriptor limit: $MAX_FD_LIMIT\"\n    fi\nfi\n\n# For Darwin, add options to specify how the application appears in the dock\nif $darwin; then\n    GRADLE_OPTS=\"$GRADLE_OPTS \\\"-Xdock:name=$APP_NAME\\\" \\\"-Xdock:icon=$APP_HOME/media/gradle.icns\\\"\"\nfi\n\n# For Cygwin, switch paths to Windows format before running java\nif $cygwin ; then\n    APP_HOME=`cygpath --path --mixed \"$APP_HOME\"`\n    CLASSPATH=`cygpath --path --mixed \"$CLASSPATH\"`\n    JAVACMD=`cygpath --unix \"$JAVACMD\"`\n\n    # We build the pattern for arguments to be converted via cygpath\n    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`\n    SEP=\"\"\n    for dir in $ROOTDIRSRAW ; do\n        ROOTDIRS=\"$ROOTDIRS$SEP$dir\"\n        SEP=\"|\"\n    done\n    OURCYGPATTERN=\"(^($ROOTDIRS))\"\n    # Add a user-defined pattern to the cygpath arguments\n    if [ \"$GRADLE_CYGPATTERN\" != \"\" ] ; then\n        OURCYGPATTERN=\"$OURCYGPATTERN|($GRADLE_CYGPATTERN)\"\n    fi\n    # Now convert the arguments - kludge to limit ourselves to /bin/sh\n    i=0\n    for arg in \"$@\" ; do\n        CHECK=`echo \"$arg\"|egrep -c \"$OURCYGPATTERN\" -`\n        CHECK2=`echo \"$arg\"|egrep -c \"^-\"`                                 ### Determine if an option\n\n        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition\n            eval `echo args$i`=`cygpath --path --ignore --mixed \"$arg\"`\n        else\n            eval `echo args$i`=\"\\\"$arg\\\"\"\n        fi\n        i=$((i+1))\n    done\n    case $i in\n        (0) set -- ;;\n        (1) set -- \"$args0\" ;;\n        (2) set -- \"$args0\" \"$args1\" ;;\n        (3) set -- \"$args0\" \"$args1\" \"$args2\" ;;\n        (4) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" ;;\n        (5) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" ;;\n        (6) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" ;;\n        (7) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" ;;\n        (8) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" ;;\n        (9) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" \"$args8\" ;;\n    esac\nfi\n\n# Escape application args\nsave () {\n    for i do printf %s\\\\n \"$i\" | sed \"s/'/'\\\\\\\\''/g;1s/^/'/;\\$s/\\$/' \\\\\\\\/\" ; done\n    echo \" \"\n}\nAPP_ARGS=$(save \"$@\")\n\n# Collect all arguments for the java command, following the shell quoting and substitution rules\neval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS \"\\\"-Dorg.gradle.appname=$APP_BASE_NAME\\\"\" -classpath \"\\\"$CLASSPATH\\\"\" org.gradle.wrapper.GradleWrapperMain \"$APP_ARGS\"\n\n# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong\nif [ \"$(uname)\" = \"Darwin\" ] && [ \"$HOME\" = \"$PWD\" ]; then\n  cd \"$(dirname \"$0\")\"\nfi\n\nexec \"$JAVACMD\" \"$@\"\n"
  },
  {
    "path": "examples/whisper.android.java/gradlew.bat",
    "content": "@if \"%DEBUG%\" == \"\" @echo off\n@rem ##########################################################################\n@rem\n@rem  Gradle startup script for Windows\n@rem\n@rem ##########################################################################\n\n@rem Set local scope for the variables with windows NT shell\nif \"%OS%\"==\"Windows_NT\" setlocal\n\nset DIRNAME=%~dp0\nif \"%DIRNAME%\" == \"\" set DIRNAME=.\nset APP_BASE_NAME=%~n0\nset APP_HOME=%DIRNAME%\n\n@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nset DEFAULT_JVM_OPTS=\n\n@rem Find java.exe\nif defined JAVA_HOME goto findJavaFromJavaHome\n\nset JAVA_EXE=java.exe\n%JAVA_EXE% -version >NUL 2>&1\nif \"%ERRORLEVEL%\" == \"0\" goto init\n\necho.\necho ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:findJavaFromJavaHome\nset JAVA_HOME=%JAVA_HOME:\"=%\nset JAVA_EXE=%JAVA_HOME%/bin/java.exe\n\nif exist \"%JAVA_EXE%\" goto init\n\necho.\necho ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:init\n@rem Get command-line arguments, handling Windows variants\n\nif not \"%OS%\" == \"Windows_NT\" goto win9xME_args\n\n:win9xME_args\n@rem Slurp the command line arguments.\nset CMD_LINE_ARGS=\nset _SKIP=2\n\n:win9xME_args_slurp\nif \"x%~1\" == \"x\" goto execute\n\nset CMD_LINE_ARGS=%*\n\n:execute\n@rem Setup the command line\n\nset CLASSPATH=%APP_HOME%\\gradle\\wrapper\\gradle-wrapper.jar\n\n@rem Execute Gradle\n\"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% \"-Dorg.gradle.appname=%APP_BASE_NAME%\" -classpath \"%CLASSPATH%\" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%\n\n:end\n@rem End local scope for the variables with windows NT shell\nif \"%ERRORLEVEL%\"==\"0\" goto mainEnd\n\n:fail\nrem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\nrem the _cmd.exe /c_ return code!\nif  not \"\" == \"%GRADLE_EXIT_CONSOLE%\" exit 1\nexit /b 1\n\n:mainEnd\nif \"%OS%\"==\"Windows_NT\" endlocal\n\n:omega\n"
  },
  {
    "path": "examples/whisper.android.java/settings.gradle",
    "content": "include ':app'\nrootProject.name = \"whisper.android.java\""
  },
  {
    "path": "examples/whisper.nvim/README.md",
    "content": "# whisper.nvim\n\nSpeech-to-text in Neovim\n\nThe transcription is performed on the CPU and no data leaves your computer. Works best on Apple Silicon devices.\n\nhttps://user-images.githubusercontent.com/1991296/198382564-784e9663-2037-4d04-99b8-f39136929b7e.mp4\n\n## Usage\n\n- Simply press `Ctrl-G` in `INSERT`, `VISUAL` or `NORMAL` mode and say something\n- When you are done - press `Ctrl-C` to end the transcription and insert the transcribed text under the cursor\n\n## Installation\n\n*Note: this is a bit tedious and hacky atm, but I hope it will be improved with time*\n\n- Clone this repo and build the `stream` tool:\n\n  ```\n  git clone https://github.com/ggerganov/whisper.cpp\n  cd whisper.cpp\n  make stream\n  ```\n\n- Download the `base.en` Whisper model (140 MB):\n\n  ```\n  ./models/download-ggml-model.sh base.en\n  ```\n\n- Place the [whisper.nvim](whisper.nvim) script somewhere in your PATH and give it execute permissions:\n\n  ```\n  cp examples/whisper.nvim/whisper.nvim ~/bin/\n  chmod u+x ~/bin/whisper.nvim\n  ```\n\n- Fine-tune the script to your preference and machine parameters:\n\n  ```\n  ./stream -t 8 -m models/ggml-base.en.bin --step 350 --length 10000 -f /tmp/whisper.nvim 2> /dev/null\n  ```\n\n  On slower machines, try to increase the `step` parameter.\n\n- Add the following shortcuts to your `~/.config/nvim/init.vim`:\n\n  ```\n  inoremap <C-G>  <C-O>:!whisper.nvim<CR><C-O>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR><C-R>a\n  nnoremap <C-G>       :!whisper.nvim<CR>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR>\"ap\n  vnoremap <C-G> c<C-O>:!whisper.nvim<CR><C-O>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR><C-R>a\n  ```\n  \n  Explanation: pressing `Ctrl-G` runs the [whisper.nvim](whisper.nvim) script which in turn calls the `stream` binary to transcribe your speech through the microphone. The results from the transcription are continuously dumped into `/tmp/whisper.nvim`. After you kill the program with `Ctrl-C`, the vim command grabs the last line from the `/tmp/whisper.nvim` file and puts it under the cursor.\n  \n  Probably there is a much more intelligent way to achieve all this, but this is what I could hack in an hour. Any suggestions how to improve this are welcome.\n  \nYou are now ready to use speech-to-text in Neovim!\n\n## TODO\n\nThere are a lot of ways to improve this idea and I don't have much experience with Vim plugin programming, so contributions are welcome! \n\n- [ ] **Wrap this into a plugin**\n  \n  It would be great to make a standalone plugin out of this that can be installed with `vim-plug` or similar\n  \n- [ ] **Simplify the `init.vim` mappings (maybe factor out the common call into a separate function)**\n- [ ] **Add Copilot/GPT-3 integration**\n\n  This is probably a very long shot, but I think it will be very cool to have the functionality to select some code and then hit Ctrl-G and say something like:\n  \n  *\"refactor this using stl containers\"*\n  \n  or\n  \n  *\"optimize by sorting the data first\"*\n  \n  The plugin would then make an appropriate query using the selected text and code context to Copilot or GPT-3 and return the result.\n  \n  Here is a proof-of-concept:\n  \n  https://user-images.githubusercontent.com/1991296/199078847-0278fcde-5667-4748-ba0d-7d55381d6047.mp4\n    \n  https://user-images.githubusercontent.com/1991296/200067939-f98d2ac2-7519-438a-85f9-79db0841ba4f.mp4\n  \n  For explanation how this works see: https://twitter.com/ggerganov/status/1587168771789258756\n\n## Discussion\n\nIf you find this idea interesting, you can join the discussion here: https://github.com/ggerganov/whisper.cpp/discussions/108\n"
  },
  {
    "path": "examples/whisper.nvim/whisper.nvim",
    "content": "#!/bin/bash\n\n# INSTRUCTIONS\n#\n# This simple script is called by Neovim to capture audio from the microphone and transcribe it with Whisper.\n# In order for this to work, you need to clone the whisper.cpp repo and build the 'stream' tool\n#\n#   git clone https://github.com/ggml-org/whisper.cpp\n#   cd whisper.cpp\n#   make stream\n#\n# Also, make sure the current script is in your PATH env variable. You should be able to run the following command:\n#\n#   whisper.nvim\n#\n# Next, export the path to the whisper.cpp repository via the WHISPER_CPP_HOME env variable:\n#\n#   export WHISPER_CPP_HOME=/path/to/whisper.cpp\n#\n# Finally, add the following lines to your ~/.config/nvim/init.vim:\n#\n#   inoremap <C-G>  <C-O>:!whisper.nvim<CR><C-O>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR><C-R>a\n#   nnoremap <C-G>       :!whisper.nvim<CR>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR>\"ap\n#   vnoremap <C-G> c<C-O>:!whisper.nvim<CR><C-O>:let @a = system(\"cat /tmp/whisper.nvim \\| tail -n 1 \\| xargs -0 \\| tr -d '\\\\n' \\| sed -e 's/^[[:space:]]*//'\")<CR><C-R>a\n#\n# This allows you to press Ctrl-G in order to capture audio from the microphone and transcribe it.\n# When you are done speaking - press Ctrl-C\n#\n\n# the Whisper model to use\nmodel=\"base.en\"\n\n# export the path to the whisper.cpp repo in the WHISPER_CPP_HOME env variable\n# https://github.com/ggml-org/whisper.cpp\ncd \"${WHISPER_CPP_HOME}\"\n\nif [ ! -f ./stream ] ; then\n    echo \"whisper.nvim: the 'stream' executable was not found! WHISPER_CPP_HOME=${WHISPER_CPP_HOME}\" > /tmp/whisper.nvim\n    exit 1\nfi\n\nif [ ! -f ./models/ggml-${model}.bin ] ; then\n    echo \"whisper.nvim: the '$model' model was not found! WHISPER_CPP_HOME=${WHISPER_CPP_HOME}\" > /tmp/whisper.nvim\n    exit 2\nfi\n\n# fine-tune the parameters according to your machine specs\n./stream -t 8 -m models/ggml-${model}.bin --step 350 --length 10000 -f /tmp/whisper.nvim 2> /dev/null\n\nexit 0\n"
  },
  {
    "path": "examples/whisper.objc/README.md",
    "content": "# whisper.objc\n\nMinimal Obj-C application for automatic offline speech recognition.\nThe inference runs locally, on-device.\n\nhttps://user-images.githubusercontent.com/1991296/197385372-962a6dea-bca1-4d50-bf96-1d8c27b98c81.mp4\n\nReal-time transcription demo:\n\nhttps://user-images.githubusercontent.com/1991296/204126266-ce4177c6-6eca-4bd9-bca8-0e46d9da2364.mp4\n\n## Usage\n\nThis example uses the whisper.xcframework which needs to be built first using the following command:\n```bash\n./build-xcframework.sh\n```\n\nA model is also required to be downloaded and can be done using the following command:\n```bash\n./models/download-ggml-model.sh base.en\n```\n\nIf you don't want to convert a Core ML model, you can skip this step by creating dummy model:\n```bash\nmkdir models/ggml-base.en-encoder.mlmodelc\n```\n\n### Core ML support\n1. Follow all the steps in the `Usage` section, including adding the ggml model file.  \nThe ggml model file is required as the Core ML model is only used for the encoder. The\ndecoder which is in the ggml model is still required.\n2. Follow the [`Core ML support` section of readme](../../README.md#core-ml-support) to convert the\nmodel.\n3. Add the Core ML model (`models/ggml-base.en-encoder.mlmodelc/`) to `whisper.swiftui.demo/Resources/models` **via Xcode**.\n\nWhen the example starts running you should now see that it is using the Core ML model:\n```console\nwhisper_init_state: loading Core ML model from '/Library/Developer/CoreSimulator/Devices/25E8C27D-0253-4281-AF17-C3F2A4D1D8F4/data/Containers/Bundle/Application/3ADA7D59-7B9C-43B4-A7E1-A87183FC546A/whisper.swiftui.app/models/ggml-base.en-encoder.mlmodelc'\nwhisper_init_state: first run on a device may take a while ...\nwhisper_init_state: Core ML model loaded\n```\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/AppDelegate.h",
    "content": "//\n//  AppDelegate.h\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import <UIKit/UIKit.h>\n\n@interface AppDelegate : UIResponder <UIApplicationDelegate>\n\n\n@end\n\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/AppDelegate.m",
    "content": "//\n//  AppDelegate.m\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import \"AppDelegate.h\"\n\n@interface AppDelegate ()\n\n@end\n\n@implementation AppDelegate\n\n\n- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {\n    // Override point for customization after application launch.\n    return YES;\n}\n\n\n#pragma mark - UISceneSession lifecycle\n\n\n- (UISceneConfiguration *)application:(UIApplication *)application configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession options:(UISceneConnectionOptions *)options {\n    // Called when a new scene session is being created.\n    // Use this method to select a configuration to create the new scene with.\n    return [[UISceneConfiguration alloc] initWithName:@\"Default Configuration\" sessionRole:connectingSceneSession.role];\n}\n\n\n- (void)application:(UIApplication *)application didDiscardSceneSessions:(NSSet<UISceneSession *> *)sceneSessions {\n    // Called when the user discards a scene session.\n    // If any sessions were discarded while the application was not running, this will be called shortly after application:didFinishLaunchingWithOptions.\n    // Use this method to release any resources that were specific to the discarded scenes, as they will not return.\n}\n\n\n@end\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Assets.xcassets/AccentColor.colorset/Contents.json",
    "content": "{\n  \"colors\" : [\n    {\n      \"idiom\" : \"universal\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Assets.xcassets/AppIcon.appiconset/Contents.json",
    "content": "{\n  \"images\" : [\n    {\n      \"idiom\" : \"universal\",\n      \"platform\" : \"ios\",\n      \"size\" : \"1024x1024\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Assets.xcassets/Contents.json",
    "content": "{\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Base.lproj/LaunchScreen.storyboard",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<document type=\"com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB\" version=\"3.0\" toolsVersion=\"13122.16\" targetRuntime=\"iOS.CocoaTouch\" propertyAccessControl=\"none\" useAutolayout=\"YES\" launchScreen=\"YES\" useTraitCollections=\"YES\" useSafeAreas=\"YES\" colorMatched=\"YES\" initialViewController=\"01J-lp-oVM\">\n    <dependencies>\n        <plugIn identifier=\"com.apple.InterfaceBuilder.IBCocoaTouchPlugin\" version=\"13104.12\"/>\n        <capability name=\"Safe area layout guides\" minToolsVersion=\"9.0\"/>\n        <capability name=\"documents saved in the Xcode 8 format\" minToolsVersion=\"8.0\"/>\n    </dependencies>\n    <scenes>\n        <!--View Controller-->\n        <scene sceneID=\"EHf-IW-A2E\">\n            <objects>\n                <viewController id=\"01J-lp-oVM\" sceneMemberID=\"viewController\">\n                    <view key=\"view\" contentMode=\"scaleToFill\" id=\"Ze5-6b-2t3\">\n                        <rect key=\"frame\" x=\"0.0\" y=\"0.0\" width=\"375\" height=\"667\"/>\n                        <autoresizingMask key=\"autoresizingMask\" widthSizable=\"YES\" heightSizable=\"YES\"/>\n                        <color key=\"backgroundColor\" xcode11CocoaTouchSystemColor=\"systemBackgroundColor\" cocoaTouchSystemColor=\"whiteColor\"/>\n                        <viewLayoutGuide key=\"safeArea\" id=\"6Tk-OE-BBY\"/>\n                    </view>\n                </viewController>\n                <placeholder placeholderIdentifier=\"IBFirstResponder\" id=\"iYj-Kq-Ea1\" userLabel=\"First Responder\" sceneMemberID=\"firstResponder\"/>\n            </objects>\n            <point key=\"canvasLocation\" x=\"53\" y=\"375\"/>\n        </scene>\n    </scenes>\n</document>\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Base.lproj/Main.storyboard",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<document type=\"com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB\" version=\"3.0\" toolsVersion=\"21507\" targetRuntime=\"iOS.CocoaTouch\" propertyAccessControl=\"none\" useAutolayout=\"YES\" useTraitCollections=\"YES\" useSafeAreas=\"YES\" colorMatched=\"YES\" initialViewController=\"BYZ-38-t0r\">\n    <device id=\"retina6_0\" orientation=\"portrait\" appearance=\"light\"/>\n    <dependencies>\n        <plugIn identifier=\"com.apple.InterfaceBuilder.IBCocoaTouchPlugin\" version=\"21505\"/>\n        <capability name=\"Safe area layout guides\" minToolsVersion=\"9.0\"/>\n        <capability name=\"System colors in document resources\" minToolsVersion=\"11.0\"/>\n        <capability name=\"documents saved in the Xcode 8 format\" minToolsVersion=\"8.0\"/>\n    </dependencies>\n    <scenes>\n        <!--View Controller-->\n        <scene sceneID=\"tne-QT-ifu\">\n            <objects>\n                <viewController id=\"BYZ-38-t0r\" customClass=\"ViewController\" sceneMemberID=\"viewController\">\n                    <view key=\"view\" contentMode=\"scaleToFill\" id=\"8bC-Xf-vdC\">\n                        <rect key=\"frame\" x=\"0.0\" y=\"0.0\" width=\"390\" height=\"844\"/>\n                        <autoresizingMask key=\"autoresizingMask\" flexibleMinX=\"YES\" widthSizable=\"YES\" flexibleMinY=\"YES\" heightSizable=\"YES\"/>\n                        <subviews>\n                            <button opaque=\"NO\" contentMode=\"scaleToFill\" contentHorizontalAlignment=\"center\" contentVerticalAlignment=\"center\" lineBreakMode=\"middleTruncation\" id=\"VOi-PT-Rbu\">\n                                <rect key=\"frame\" x=\"35\" y=\"121\" width=\"156\" height=\"49\"/>\n                                <autoresizingMask key=\"autoresizingMask\" flexibleMaxX=\"YES\" flexibleMaxY=\"YES\"/>\n                                <color key=\"backgroundColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <color key=\"tintColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <state key=\"normal\" title=\"Start Capturing\">\n                                    <color key=\"titleColor\" systemColor=\"labelColor\"/>\n                                </state>\n                                <connections>\n                                    <action selector=\"toggleCapture:\" destination=\"BYZ-38-t0r\" eventType=\"touchUpInside\" id=\"BuO-Wf-RgV\"/>\n                                </connections>\n                            </button>\n                            <label opaque=\"NO\" userInteractionEnabled=\"NO\" contentMode=\"left\" horizontalHuggingPriority=\"251\" verticalHuggingPriority=\"251\" fixedFrame=\"YES\" text=\"Status: Idle\" textAlignment=\"natural\" lineBreakMode=\"tailTruncation\" baselineAdjustment=\"alignBaselines\" adjustsFontSizeToFit=\"NO\" translatesAutoresizingMaskIntoConstraints=\"NO\" id=\"Tgu-2q-eHQ\">\n                                <rect key=\"frame\" x=\"35\" y=\"78\" width=\"232\" height=\"21\"/>\n                                <autoresizingMask key=\"autoresizingMask\" flexibleMaxX=\"YES\" flexibleMaxY=\"YES\"/>\n                                <fontDescription key=\"fontDescription\" type=\"system\" pointSize=\"17\"/>\n                                <nil key=\"textColor\"/>\n                                <nil key=\"highlightedColor\"/>\n                            </label>\n                            <textView clipsSubviews=\"YES\" multipleTouchEnabled=\"YES\" contentMode=\"scaleToFill\" fixedFrame=\"YES\" text=\"Record some speech and press &quot;Transcribe&quot;. The result will be displayed here.\" textAlignment=\"natural\" translatesAutoresizingMaskIntoConstraints=\"NO\" id=\"mv2-KD-7jn\">\n                                <rect key=\"frame\" x=\"35\" y=\"248\" width=\"320\" height=\"300\"/>\n                                <autoresizingMask key=\"autoresizingMask\" flexibleMaxX=\"YES\" flexibleMaxY=\"YES\"/>\n                                <color key=\"backgroundColor\" systemColor=\"systemBackgroundColor\"/>\n                                <color key=\"textColor\" systemColor=\"labelColor\"/>\n                                <fontDescription key=\"fontDescription\" name=\"Georgia\" family=\"Georgia\" pointSize=\"16\"/>\n                                <textInputTraits key=\"textInputTraits\" autocapitalizationType=\"sentences\"/>\n                            </textView>\n                            <button opaque=\"NO\" contentMode=\"scaleToFill\" contentHorizontalAlignment=\"center\" contentVerticalAlignment=\"center\" lineBreakMode=\"middleTruncation\" id=\"Brs-xi-o8i\">\n                                <rect key=\"frame\" x=\"35\" y=\"191\" width=\"156\" height=\"49\"/>\n                                <autoresizingMask key=\"autoresizingMask\" flexibleMaxX=\"YES\" flexibleMaxY=\"YES\"/>\n                                <color key=\"backgroundColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <color key=\"tintColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <state key=\"normal\" title=\"Transcribe\">\n                                    <color key=\"titleColor\" systemColor=\"labelColor\"/>\n                                </state>\n                                <connections>\n                                    <action selector=\"onTranscribe:\" destination=\"BYZ-38-t0r\" eventType=\"touchUpInside\" id=\"ond-bx-48O\"/>\n                                    <action selector=\"onTranscribePrepare:\" destination=\"BYZ-38-t0r\" eventType=\"touchDown\" id=\"16T-dN-dfB\"/>\n                                </connections>\n                            </button>\n                            <button opaque=\"NO\" contentMode=\"scaleToFill\" contentHorizontalAlignment=\"center\" contentVerticalAlignment=\"center\" lineBreakMode=\"middleTruncation\" id=\"AaW-T2-Ndw\">\n                                <rect key=\"frame\" x=\"199\" y=\"191\" width=\"156\" height=\"49\"/>\n                                <autoresizingMask key=\"autoresizingMask\" flexibleMaxX=\"YES\" flexibleMaxY=\"YES\"/>\n                                <color key=\"backgroundColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <color key=\"tintColor\" systemColor=\"opaqueSeparatorColor\"/>\n                                <state key=\"normal\" title=\"Real-time\">\n                                    <color key=\"titleColor\" systemColor=\"labelColor\"/>\n                                </state>\n                                <connections>\n                                    <action selector=\"onRealtime:\" destination=\"BYZ-38-t0r\" eventType=\"touchUpInside\" id=\"nhn-jT-aQJ\"/>\n                                </connections>\n                            </button>\n                        </subviews>\n                        <viewLayoutGuide key=\"safeArea\" id=\"6Tk-OE-BBY\"/>\n                        <color key=\"backgroundColor\" systemColor=\"systemBackgroundColor\"/>\n                        <constraints>\n                            <constraint firstItem=\"Brs-xi-o8i\" firstAttribute=\"trailing\" secondItem=\"VOi-PT-Rbu\" secondAttribute=\"trailing\" id=\"8mF-AW-cbc\"/>\n                        </constraints>\n                    </view>\n                    <connections>\n                        <outlet property=\"buttonRealtime\" destination=\"AaW-T2-Ndw\" id=\"gcU-Ol-BOo\"/>\n                        <outlet property=\"buttonToggleCapture\" destination=\"VOi-PT-Rbu\" id=\"nis-VC-DQO\"/>\n                        <outlet property=\"buttonTranscribe\" destination=\"Brs-xi-o8i\" id=\"N8h-9W-ywb\"/>\n                        <outlet property=\"labelStatusInp\" destination=\"Tgu-2q-eHQ\" id=\"1hH-Ql-K6j\"/>\n                        <outlet property=\"textviewResult\" destination=\"mv2-KD-7jn\" id=\"RBw-0L-iGj\"/>\n                    </connections>\n                </viewController>\n                <placeholder placeholderIdentifier=\"IBFirstResponder\" id=\"dkx-z0-nzr\" sceneMemberID=\"firstResponder\"/>\n            </objects>\n            <point key=\"canvasLocation\" x=\"30.769230769230766\" y=\"-28.436018957345969\"/>\n        </scene>\n    </scenes>\n    <resources>\n        <systemColor name=\"labelColor\">\n            <color red=\"0.0\" green=\"0.0\" blue=\"0.0\" alpha=\"1\" colorSpace=\"custom\" customColorSpace=\"sRGB\"/>\n        </systemColor>\n        <systemColor name=\"opaqueSeparatorColor\">\n            <color red=\"0.77647058823529413\" green=\"0.77647058823529413\" blue=\"0.78431372549019607\" alpha=\"1\" colorSpace=\"custom\" customColorSpace=\"sRGB\"/>\n        </systemColor>\n        <systemColor name=\"systemBackgroundColor\">\n            <color white=\"1\" alpha=\"1\" colorSpace=\"custom\" customColorSpace=\"genericGamma22GrayColorSpace\"/>\n        </systemColor>\n    </resources>\n</document>\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/Info.plist",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n\t<key>NSMicrophoneUsageDescription</key>\n\t<string>This app requires microphone access in order to transcribe speech</string>\n\t<key>UIApplicationSceneManifest</key>\n\t<dict>\n\t\t<key>UIApplicationSupportsMultipleScenes</key>\n\t\t<false/>\n\t\t<key>UISceneConfigurations</key>\n\t\t<dict>\n\t\t\t<key>UIWindowSceneSessionRoleApplication</key>\n\t\t\t<array>\n\t\t\t\t<dict>\n\t\t\t\t\t<key>UISceneConfigurationName</key>\n\t\t\t\t\t<string>Default Configuration</string>\n\t\t\t\t\t<key>UISceneDelegateClassName</key>\n\t\t\t\t\t<string>SceneDelegate</string>\n\t\t\t\t\t<key>UISceneStoryboardFile</key>\n\t\t\t\t\t<string>Main</string>\n\t\t\t\t</dict>\n\t\t\t</array>\n\t\t</dict>\n\t</dict>\n</dict>\n</plist>\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/SceneDelegate.h",
    "content": "//\n//  SceneDelegate.h\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import <UIKit/UIKit.h>\n\n@interface SceneDelegate : UIResponder <UIWindowSceneDelegate>\n\n@property (strong, nonatomic) UIWindow * window;\n\n@end\n\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/SceneDelegate.m",
    "content": "//\n//  SceneDelegate.m\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import \"SceneDelegate.h\"\n\n@interface SceneDelegate ()\n\n@end\n\n@implementation SceneDelegate\n\n\n- (void)scene:(UIScene *)scene willConnectToSession:(UISceneSession *)session options:(UISceneConnectionOptions *)connectionOptions {\n    // Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.\n    // If using a storyboard, the `window` property will automatically be initialized and attached to the scene.\n    // This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).\n}\n\n\n- (void)sceneDidDisconnect:(UIScene *)scene {\n    // Called as the scene is being released by the system.\n    // This occurs shortly after the scene enters the background, or when its session is discarded.\n    // Release any resources associated with this scene that can be re-created the next time the scene connects.\n    // The scene may re-connect later, as its session was not necessarily discarded (see `application:didDiscardSceneSessions` instead).\n}\n\n\n- (void)sceneDidBecomeActive:(UIScene *)scene {\n    // Called when the scene has moved from an inactive state to an active state.\n    // Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.\n}\n\n\n- (void)sceneWillResignActive:(UIScene *)scene {\n    // Called when the scene will move from an active state to an inactive state.\n    // This may occur due to temporary interruptions (ex. an incoming phone call).\n}\n\n\n- (void)sceneWillEnterForeground:(UIScene *)scene {\n    // Called as the scene transitions from the background to the foreground.\n    // Use this method to undo the changes made on entering the background.\n}\n\n\n- (void)sceneDidEnterBackground:(UIScene *)scene {\n    // Called as the scene transitions from the foreground to the background.\n    // Use this method to save data, release shared resources, and store enough scene-specific state information\n    // to restore the scene back to its current state.\n}\n\n\n@end\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/ViewController.h",
    "content": "//\n//  ViewController.h\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import <UIKit/UIKit.h>\n\n#import <AVFoundation/AVFoundation.h>\n#import <AudioToolbox/AudioQueue.h>\n\n#define NUM_BUFFERS 3\n#define MAX_AUDIO_SEC 30\n#define SAMPLE_RATE 16000\n\nstruct whisper_context;\n\ntypedef struct\n{\n    int ggwaveId;\n    bool isCapturing;\n    bool isTranscribing;\n    bool isRealtime;\n    UILabel * labelReceived;\n\n    AudioQueueRef queue;\n    AudioStreamBasicDescription dataFormat;\n    AudioQueueBufferRef buffers[NUM_BUFFERS];\n\n    int n_samples;\n    int16_t * audioBufferI16;\n    float   * audioBufferF32;\n\n    struct whisper_context * ctx;\n\n    void * vc;\n} StateInp;\n\n@interface ViewController : UIViewController\n{\n    StateInp stateInp;\n}\n\n@end\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/ViewController.m",
    "content": "//\n//  ViewController.m\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import \"ViewController.h\"\n#import <whisper/whisper.h>\n\n\n#define NUM_BYTES_PER_BUFFER 16*1024\n\n// callback used to process captured audio\nvoid AudioInputCallback(void * inUserData,\n                        AudioQueueRef inAQ,\n                        AudioQueueBufferRef inBuffer,\n                        const AudioTimeStamp * inStartTime,\n                        UInt32 inNumberPacketDescriptions,\n                        const AudioStreamPacketDescription * inPacketDescs);\n\n@interface ViewController ()\n\n@property (weak, nonatomic) IBOutlet UILabel    *labelStatusInp;\n@property (weak, nonatomic) IBOutlet UIButton   *buttonToggleCapture;\n@property (weak, nonatomic) IBOutlet UIButton   *buttonTranscribe;\n@property (weak, nonatomic) IBOutlet UIButton   *buttonRealtime;\n@property (weak, nonatomic) IBOutlet UITextView *textviewResult;\n\n@end\n\n@implementation ViewController\n\n- (void)setupAudioFormat:(AudioStreamBasicDescription*)format\n{\n    format->mSampleRate       = WHISPER_SAMPLE_RATE;\n    format->mFormatID         = kAudioFormatLinearPCM;\n    format->mFramesPerPacket  = 1;\n    format->mChannelsPerFrame = 1;\n    format->mBytesPerFrame    = 2;\n    format->mBytesPerPacket   = 2;\n    format->mBitsPerChannel   = 16;\n    format->mReserved         = 0;\n    format->mFormatFlags      = kLinearPCMFormatFlagIsSignedInteger;\n}\n\n- (void)viewDidLoad {\n    [super viewDidLoad];\n\n    // whisper.cpp initialization\n    {\n        // load the model\n        NSString *modelPath = [[NSBundle mainBundle] pathForResource:@\"ggml-base.en\" ofType:@\"bin\"];\n\n        // check if the model exists\n        if (![[NSFileManager defaultManager] fileExistsAtPath:modelPath]) {\n            NSLog(@\"Model file not found\");\n            return;\n        }\n\n        NSLog(@\"Loading model from %@\", modelPath);\n\n        // create ggml context\n\n        struct whisper_context_params cparams = whisper_context_default_params();\n#if TARGET_OS_SIMULATOR\n        cparams.use_gpu = false;\n        NSLog(@\"Running on simulator, using CPU\");\n#endif\n        stateInp.ctx = whisper_init_from_file_with_params([modelPath UTF8String], cparams);\n\n        // check if the model was loaded successfully\n        if (stateInp.ctx == NULL) {\n            NSLog(@\"Failed to load model\");\n            return;\n        }\n    }\n\n    // initialize audio format and buffers\n    {\n        [self setupAudioFormat:&stateInp.dataFormat];\n\n        stateInp.n_samples = 0;\n        stateInp.audioBufferI16 = malloc(MAX_AUDIO_SEC*SAMPLE_RATE*sizeof(int16_t));\n        stateInp.audioBufferF32 = malloc(MAX_AUDIO_SEC*SAMPLE_RATE*sizeof(float));\n        // Set up audio session\n        NSError *error = nil;\n\n        [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryRecord error:&error];\n        if (error) {\n            NSLog(@\"Error setting audio session category: %@\", error);\n        }\n\n        [[AVAudioSession sharedInstance] setActive:YES error:&error];\n        if (error) {\n            NSLog(@\"Error activating audio session: %@\", error);\n        }\n\n    }\n\n    stateInp.isTranscribing = false;\n    stateInp.isRealtime = false;\n}\n\n-(IBAction) stopCapturing {\n    NSLog(@\"Stop capturing\");\n\n    _labelStatusInp.text = @\"Status: Idle\";\n\n    [_buttonToggleCapture setTitle:@\"Start capturing\" forState:UIControlStateNormal];\n    [_buttonToggleCapture setBackgroundColor:[UIColor grayColor]];\n\n    stateInp.isCapturing = false;\n\n    AudioQueueStop(stateInp.queue, true);\n    for (int i = 0; i < NUM_BUFFERS; i++) {\n        AudioQueueFreeBuffer(stateInp.queue, stateInp.buffers[i]);\n    }\n\n    AudioQueueDispose(stateInp.queue, true);\n}\n\n- (IBAction)toggleCapture:(id)sender {\n    if (stateInp.isCapturing) {\n        // stop capturing\n        [self stopCapturing];\n\n        return;\n    }\n\n    // initiate audio capturing\n    NSLog(@\"Start capturing\");\n\n    stateInp.n_samples = 0;\n    stateInp.vc = (__bridge void *)(self);\n\n    OSStatus status = AudioQueueNewInput(&stateInp.dataFormat,\n                                         AudioInputCallback,\n                                         &stateInp,\n                                         CFRunLoopGetCurrent(),\n                                         kCFRunLoopCommonModes,\n                                         0,\n                                         &stateInp.queue);\n\n    if (status == 0) {\n        for (int i = 0; i < NUM_BUFFERS; i++) {\n            AudioQueueAllocateBuffer(stateInp.queue, NUM_BYTES_PER_BUFFER, &stateInp.buffers[i]);\n            AudioQueueEnqueueBuffer (stateInp.queue, stateInp.buffers[i], 0, NULL);\n        }\n\n        stateInp.isCapturing = true;\n        status = AudioQueueStart(stateInp.queue, NULL);\n        if (status == 0) {\n            _labelStatusInp.text = @\"Status: Capturing\";\n            [sender setTitle:@\"Stop Capturing\" forState:UIControlStateNormal];\n            [_buttonToggleCapture setBackgroundColor:[UIColor redColor]];\n        }\n    }\n\n    if (status != 0) {\n        [self stopCapturing];\n    }\n}\n\n- (IBAction)onTranscribePrepare:(id)sender {\n    _textviewResult.text = @\"Processing - please wait ...\";\n\n    if (stateInp.isRealtime) {\n        [self onRealtime:(id)sender];\n    }\n\n    if (stateInp.isCapturing) {\n        [self stopCapturing];\n    }\n}\n\n- (IBAction)onRealtime:(id)sender {\n    stateInp.isRealtime = !stateInp.isRealtime;\n\n    if (stateInp.isRealtime) {\n        [_buttonRealtime setBackgroundColor:[UIColor greenColor]];\n    } else {\n        [_buttonRealtime setBackgroundColor:[UIColor grayColor]];\n    }\n\n    NSLog(@\"Realtime: %@\", stateInp.isRealtime ? @\"ON\" : @\"OFF\");\n}\n\n- (IBAction)onTranscribe:(id)sender {\n    if (stateInp.isTranscribing) {\n        return;\n    }\n\n    NSLog(@\"Processing %d samples\", stateInp.n_samples);\n\n    stateInp.isTranscribing = true;\n\n    // dispatch the model to a background thread\n    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{\n        // process captured audio\n        // convert I16 to F32\n        for (int i = 0; i < self->stateInp.n_samples; i++) {\n            self->stateInp.audioBufferF32[i] = (float)self->stateInp.audioBufferI16[i] / 32768.0f;\n        }\n\n        // run the model\n        struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);\n\n        // get maximum number of threads on this device (max 8)\n        const int max_threads = MIN(8, (int)[[NSProcessInfo processInfo] processorCount]);\n\n        params.print_realtime   = true;\n        params.print_progress   = false;\n        params.print_timestamps = true;\n        params.print_special    = false;\n        params.translate        = false;\n        params.language         = \"en\";\n        params.n_threads        = max_threads;\n        params.offset_ms        = 0;\n        params.no_context       = true;\n        params.single_segment   = self->stateInp.isRealtime;\n        params.no_timestamps    = params.single_segment;\n\n        CFTimeInterval startTime = CACurrentMediaTime();\n\n        whisper_reset_timings(self->stateInp.ctx);\n\n        if (whisper_full(self->stateInp.ctx, params, self->stateInp.audioBufferF32, self->stateInp.n_samples) != 0) {\n            NSLog(@\"Failed to run the model\");\n            self->_textviewResult.text = @\"Failed to run the model\";\n\n            return;\n        }\n\n        whisper_print_timings(self->stateInp.ctx);\n\n        CFTimeInterval endTime = CACurrentMediaTime();\n\n        NSLog(@\"\\nProcessing time: %5.3f, on %d threads\", endTime - startTime, params.n_threads);\n\n        // result text\n        NSString *result = @\"\";\n\n        int n_segments = whisper_full_n_segments(self->stateInp.ctx);\n        for (int i = 0; i < n_segments; i++) {\n            const char * text_cur = whisper_full_get_segment_text(self->stateInp.ctx, i);\n\n            // append the text to the result\n            result = [result stringByAppendingString:[NSString stringWithUTF8String:text_cur]];\n        }\n\n        const float tRecording = (float)self->stateInp.n_samples / (float)self->stateInp.dataFormat.mSampleRate;\n\n        // append processing time\n        result = [result stringByAppendingString:[NSString stringWithFormat:@\"\\n\\n[recording time:  %5.3f s]\", tRecording]];\n        result = [result stringByAppendingString:[NSString stringWithFormat:@\"  \\n[processing time: %5.3f s]\", endTime - startTime]];\n\n        // dispatch the result to the main thread\n        dispatch_async(dispatch_get_main_queue(), ^{\n            self->_textviewResult.text = result;\n            self->stateInp.isTranscribing = false;\n        });\n    });\n}\n\n//\n// Callback implementation\n//\n\nvoid AudioInputCallback(void * inUserData,\n                        AudioQueueRef inAQ,\n                        AudioQueueBufferRef inBuffer,\n                        const AudioTimeStamp * inStartTime,\n                        UInt32 inNumberPacketDescriptions,\n                        const AudioStreamPacketDescription * inPacketDescs)\n{\n    StateInp * stateInp = (StateInp*)inUserData;\n\n    if (!stateInp->isCapturing) {\n        NSLog(@\"Not capturing, ignoring audio\");\n        return;\n    }\n\n    const int n = inBuffer->mAudioDataByteSize / 2;\n\n    NSLog(@\"Captured %d new samples\", n);\n\n    if (stateInp->n_samples + n > MAX_AUDIO_SEC*SAMPLE_RATE) {\n        NSLog(@\"Too much audio data, ignoring\");\n\n        dispatch_async(dispatch_get_main_queue(), ^{\n            ViewController * vc = (__bridge ViewController *)(stateInp->vc);\n            [vc stopCapturing];\n        });\n\n        return;\n    }\n\n    for (int i = 0; i < n; i++) {\n        stateInp->audioBufferI16[stateInp->n_samples + i] = ((short*)inBuffer->mAudioData)[i];\n    }\n\n    stateInp->n_samples += n;\n\n    // put the buffer back in the queue\n    AudioQueueEnqueueBuffer(stateInp->queue, inBuffer, 0, NULL);\n\n    if (stateInp->isRealtime) {\n        // dipatch onTranscribe() to the main thread\n        dispatch_async(dispatch_get_main_queue(), ^{\n            ViewController * vc = (__bridge ViewController *)(stateInp->vc);\n            [vc onTranscribe:nil];\n        });\n    }\n}\n\n@end\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc/main.m",
    "content": "//\n//  main.m\n//  whisper.objc\n//\n//  Created by Georgi Gerganov on 23.10.22.\n//\n\n#import <UIKit/UIKit.h>\n#import \"AppDelegate.h\"\n\nint main(int argc, char * argv[]) {\n    NSString * appDelegateClassName;\n    @autoreleasepool {\n        // Setup code that might create autoreleased objects goes here.\n        appDelegateClassName = NSStringFromClass([AppDelegate class]);\n    }\n    return UIApplicationMain(argc, argv, nil, appDelegateClassName);\n}\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj",
    "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 56;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\t18627C7B29052BDF00BD2A04 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C7A29052BDF00BD2A04 /* AppDelegate.m */; };\n\t\t18627C7E29052BDF00BD2A04 /* SceneDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C7D29052BDF00BD2A04 /* SceneDelegate.m */; };\n\t\t18627C8129052BDF00BD2A04 /* ViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C8029052BDF00BD2A04 /* ViewController.m */; };\n\t\t18627C8429052BDF00BD2A04 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8229052BDF00BD2A04 /* Main.storyboard */; };\n\t\t18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8529052BE000BD2A04 /* Assets.xcassets */; };\n\t\t18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8729052BE000BD2A04 /* LaunchScreen.storyboard */; };\n\t\t18627C8C29052BE000BD2A04 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C8B29052BE000BD2A04 /* main.m */; };\n\t\t18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */; };\n\t\t7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */; };\n\t\t7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342472A0C3FA20015A058 /* whisper-encoder.mm */; };\n\t\t7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */ = {isa = PBXBuildFile; fileRef = 7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */; };\n\t\tDDE3609F2D87EA8C004EA223 /* whisper.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = DDE3609E2D87EA8C004EA223 /* whisper.xcframework */; };\n\t\tDDE360A02D87EA8C004EA223 /* whisper.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = DDE3609E2D87EA8C004EA223 /* whisper.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };\n/* End PBXBuildFile section */\n\n/* Begin PBXCopyFilesBuildPhase section */\n\t\t184447202AB21B25007D6BFE /* Copy Files */ = {\n\t\t\tisa = PBXCopyFilesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tdstPath = \"\";\n\t\t\tdstSubfolderSpec = 7;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tname = \"Copy Files\";\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tDDE360A12D87EA8C004EA223 /* Embed Frameworks */ = {\n\t\t\tisa = PBXCopyFilesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tdstPath = \"\";\n\t\t\tdstSubfolderSpec = 10;\n\t\t\tfiles = (\n\t\t\t\tDDE360A02D87EA8C004EA223 /* whisper.xcframework in Embed Frameworks */,\n\t\t\t);\n\t\t\tname = \"Embed Frameworks\";\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXCopyFilesBuildPhase section */\n\n/* Begin PBXFileReference section */\n\t\t18627C7629052BDF00BD2A04 /* whisper.objc.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = whisper.objc.app; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t18627C7929052BDF00BD2A04 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = \"<group>\"; };\n\t\t18627C7A29052BDF00BD2A04 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = \"<group>\"; };\n\t\t18627C7C29052BDF00BD2A04 /* SceneDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SceneDelegate.h; sourceTree = \"<group>\"; };\n\t\t18627C7D29052BDF00BD2A04 /* SceneDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SceneDelegate.m; sourceTree = \"<group>\"; };\n\t\t18627C7F29052BDF00BD2A04 /* ViewController.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ViewController.h; sourceTree = \"<group>\"; };\n\t\t18627C8029052BDF00BD2A04 /* ViewController.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ViewController.m; sourceTree = \"<group>\"; };\n\t\t18627C8329052BDF00BD2A04 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = \"<group>\"; };\n\t\t18627C8529052BE000BD2A04 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = \"<group>\"; };\n\t\t18627C8829052BE000BD2A04 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = \"<group>\"; };\n\t\t18627C8A29052BE000BD2A04 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = \"<group>\"; };\n\t\t18627C8B29052BE000BD2A04 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = \"<group>\"; };\n\t\t18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = \"ggml-base.en.bin\"; path = \"../../../models/ggml-base.en.bin\"; sourceTree = \"<group>\"; };\n\t\t7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = \"whisper-encoder-impl.m\"; sourceTree = \"<group>\"; };\n\t\t7FE342462A0C3FA20015A058 /* whisper-encoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = \"whisper-encoder.h\"; sourceTree = \"<group>\"; };\n\t\t7FE342472A0C3FA20015A058 /* whisper-encoder.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = \"whisper-encoder.mm\"; sourceTree = \"<group>\"; };\n\t\t7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = \"whisper-decoder-impl.h\"; sourceTree = \"<group>\"; };\n\t\t7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = \"whisper-encoder-impl.h\"; sourceTree = \"<group>\"; };\n\t\t7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = \"whisper-decoder-impl.m\"; sourceTree = \"<group>\"; };\n\t\t7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = \"ggml-base.en-encoder.mlmodelc\"; path = \"../../../models/ggml-base.en-encoder.mlmodelc\"; sourceTree = \"<group>\"; };\n\t\tDDE3609E2D87EA8C004EA223 /* whisper.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = whisper.xcframework; path = \"../../build-apple/whisper.xcframework\"; sourceTree = \"<group>\"; };\n/* End PBXFileReference section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\t18627C7329052BDF00BD2A04 /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tDDE3609F2D87EA8C004EA223 /* whisper.xcframework in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\t18627C6D29052BDF00BD2A04 = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t18627C7829052BDF00BD2A04 /* whisper.objc */,\n\t\t\t\tDDE3609D2D87EA8C004EA223 /* Frameworks */,\n\t\t\t\t18627C7729052BDF00BD2A04 /* Products */,\n\t\t\t);\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t18627C7729052BDF00BD2A04 /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t18627C7629052BDF00BD2A04 /* whisper.objc.app */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t18627C7829052BDF00BD2A04 /* whisper.objc */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */,\n\t\t\t\t7FE342442A0C3FA20015A058 /* coreml */,\n\t\t\t\t18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */,\n\t\t\t\t18627C7929052BDF00BD2A04 /* AppDelegate.h */,\n\t\t\t\t18627C7A29052BDF00BD2A04 /* AppDelegate.m */,\n\t\t\t\t18627C7C29052BDF00BD2A04 /* SceneDelegate.h */,\n\t\t\t\t18627C7D29052BDF00BD2A04 /* SceneDelegate.m */,\n\t\t\t\t18627C7F29052BDF00BD2A04 /* ViewController.h */,\n\t\t\t\t18627C8029052BDF00BD2A04 /* ViewController.m */,\n\t\t\t\t18627C8229052BDF00BD2A04 /* Main.storyboard */,\n\t\t\t\t18627C8529052BE000BD2A04 /* Assets.xcassets */,\n\t\t\t\t18627C8729052BE000BD2A04 /* LaunchScreen.storyboard */,\n\t\t\t\t18627C8A29052BE000BD2A04 /* Info.plist */,\n\t\t\t\t18627C8B29052BE000BD2A04 /* main.m */,\n\t\t\t);\n\t\t\tpath = whisper.objc;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t7FE342442A0C3FA20015A058 /* coreml */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */,\n\t\t\t\t7FE342462A0C3FA20015A058 /* whisper-encoder.h */,\n\t\t\t\t7FE342472A0C3FA20015A058 /* whisper-encoder.mm */,\n\t\t\t\t7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */,\n\t\t\t\t7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */,\n\t\t\t\t7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */,\n\t\t\t);\n\t\t\tname = coreml;\n\t\t\tpath = ../../../src/coreml;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tDDE3609D2D87EA8C004EA223 /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tDDE3609E2D87EA8C004EA223 /* whisper.xcframework */,\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXNativeTarget section */\n\t\t18627C7529052BDF00BD2A04 /* whisper.objc */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = 18627C8F29052BE000BD2A04 /* Build configuration list for PBXNativeTarget \"whisper.objc\" */;\n\t\t\tbuildPhases = (\n\t\t\t\t18627C7229052BDF00BD2A04 /* Sources */,\n\t\t\t\t18627C7329052BDF00BD2A04 /* Frameworks */,\n\t\t\t\t18627C7429052BDF00BD2A04 /* Resources */,\n\t\t\t\t184447202AB21B25007D6BFE /* Copy Files */,\n\t\t\t\tDDE360A12D87EA8C004EA223 /* Embed Frameworks */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tname = whisper.objc;\n\t\t\tproductName = whisper.objc;\n\t\t\tproductReference = 18627C7629052BDF00BD2A04 /* whisper.objc.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\t18627C6E29052BDF00BD2A04 /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = 1;\n\t\t\t\tLastUpgradeCheck = 1540;\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\t18627C7529052BDF00BD2A04 = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 14.0.1;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = 18627C7129052BDF00BD2A04 /* Build configuration list for PBXProject \"whisper.objc\" */;\n\t\t\tcompatibilityVersion = \"Xcode 14.0\";\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = 18627C6D29052BDF00BD2A04;\n\t\t\tproductRefGroup = 18627C7729052BDF00BD2A04 /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\t18627C7529052BDF00BD2A04 /* whisper.objc */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\t18627C7429052BDF00BD2A04 /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */,\n\t\t\t\t7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */,\n\t\t\t\t18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */,\n\t\t\t\t18627C8429052BDF00BD2A04 /* Main.storyboard in Resources */,\n\t\t\t\t18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\t18627C7229052BDF00BD2A04 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t18627C8129052BDF00BD2A04 /* ViewController.m in Sources */,\n\t\t\t\t7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */,\n\t\t\t\t18627C7B29052BDF00BD2A04 /* AppDelegate.m in Sources */,\n\t\t\t\t18627C8C29052BE000BD2A04 /* main.m in Sources */,\n\t\t\t\t18627C7E29052BDF00BD2A04 /* SceneDelegate.m in Sources */,\n\t\t\t\t7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin PBXVariantGroup section */\n\t\t18627C8229052BDF00BD2A04 /* Main.storyboard */ = {\n\t\t\tisa = PBXVariantGroup;\n\t\t\tchildren = (\n\t\t\t\t18627C8329052BDF00BD2A04 /* Base */,\n\t\t\t);\n\t\t\tname = Main.storyboard;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t18627C8729052BE000BD2A04 /* LaunchScreen.storyboard */ = {\n\t\t\tisa = PBXVariantGroup;\n\t\t\tchildren = (\n\t\t\t\t18627C8829052BE000BD2A04 /* Base */,\n\t\t\t);\n\t\t\tname = LaunchScreen.storyboard;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n/* End PBXVariantGroup section */\n\n/* Begin XCBuildConfiguration section */\n\t\t18627C8D29052BE000BD2A04 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tHEADER_SEARCH_PATHS = \"\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 16.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tSDKROOT = iphoneos;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t18627C8E29052BE000BD2A04 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tHEADER_SEARCH_PATHS = \"\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 16.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tOTHER_CFLAGS = (\n\t\t\t\t\t\"-O3\",\n\t\t\t\t\t\"-DNDEBUG\",\n\t\t\t\t);\n\t\t\t\tSDKROOT = iphoneos;\n\t\t\t\tVALIDATE_PRODUCT = YES;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\t18627C9029052BE000BD2A04 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"c++17\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_TEAM = P8JZH34X63;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = NO;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tHEADER_SEARCH_PATHS = \"\";\n\t\t\t\tINFOPLIST_FILE = whisper.objc/Info.plist;\n\t\t\t\tINFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;\n\t\t\t\tINFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;\n\t\t\t\tINFOPLIST_KEY_UIMainStoryboardFile = Main;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = \"UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = \"UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMTL_HEADER_SEARCH_PATHS = \"\";\n\t\t\t\tOTHER_CFLAGS = \"-DGGML_USE_CPU=ON\";\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = \"com.ggerganov.whisper-objc\";\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t\tWARNING_CFLAGS = \"-Wno-quoted-include-in-framework-header\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t18627C9129052BE000BD2A04 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"c++17\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_TEAM = P8JZH34X63;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = NO;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tHEADER_SEARCH_PATHS = \"\";\n\t\t\t\tINFOPLIST_FILE = whisper.objc/Info.plist;\n\t\t\t\tINFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;\n\t\t\t\tINFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen;\n\t\t\t\tINFOPLIST_KEY_UIMainStoryboardFile = Main;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = \"UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = \"UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMTL_HEADER_SEARCH_PATHS = \"\";\n\t\t\t\tOTHER_CFLAGS = \"-DGGML_USE_CPU=ON\";\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = \"com.ggerganov.whisper-objc\";\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t\tWARNING_CFLAGS = \"-Wno-quoted-include-in-framework-header\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\t18627C7129052BDF00BD2A04 /* Build configuration list for PBXProject \"whisper.objc\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t18627C8D29052BE000BD2A04 /* Debug */,\n\t\t\t\t18627C8E29052BE000BD2A04 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\t18627C8F29052BE000BD2A04 /* Build configuration list for PBXNativeTarget \"whisper.objc\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t18627C9029052BE000BD2A04 /* Debug */,\n\t\t\t\t18627C9129052BE000BD2A04 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\t};\n\trootObject = 18627C6E29052BDF00BD2A04 /* Project object */;\n}\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc.xcodeproj/project.xcworkspace/contents.xcworkspacedata",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Workspace\n   version = \"1.0\">\n   <FileRef\n      location = \"self:\">\n   </FileRef>\n</Workspace>\n"
  },
  {
    "path": "examples/whisper.objc/whisper.objc.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n\t<key>IDEDidComputeMac32BitWarning</key>\n\t<true/>\n</dict>\n</plist>\n"
  },
  {
    "path": "examples/whisper.swiftui/.gitignore",
    "content": "xcuserdata\nxcshareddata\n"
  },
  {
    "path": "examples/whisper.swiftui/README.md",
    "content": "# whisper.cpp/examples/whisper.swiftui\n\nA sample SwiftUI app using [whisper.cpp](https://github.com/ggerganov/whisper.cpp/) to do voice-to-text transcriptions.\nSee also: [whisper.objc](https://github.com/ggerganov/whisper.cpp/tree/master/examples/whisper.objc).\n\n### Building\n First whisper.cpp need to be built and a XCFramework needs to be created. This can be done by running\n the following script from the whisper.cpp project root:\n ```console\n $ ./build-xcframework.sh\n ```\n\nNote: if you get the error \"iphoneos is not an iOS SDK\" then you probably need to run this command first:\n```console\nsudo xcode-select -switch /Applications/Xcode.app/Contents/Developer\n```\n\n Open `whisper.swiftui.xcodeproj` project in Xcode and you should be able to build and run the app on\n a simulator or a real device.\n\n To use the framework with a different project, the XCFramework can be added to the project by\n adding `build-apple/whisper.xcframework` by dragging and dropping it into the project navigator, or\n by manually selecting the framework in the \"Frameworks, Libraries, and Embedded Content\" section\n of the project settings.\n\n### Usage\n\n1. Select a model from the [whisper.cpp repository](https://github.com/ggerganov/whisper.cpp/tree/master/models).[^1]\n2. Add the model to `whisper.swiftui.demo/Resources/models` **via Xcode**.\n3. Select a sample audio file (for example, [jfk.wav](https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav)).\n4. Add the sample audio file to `whisper.swiftui.demo/Resources/samples` **via Xcode**.\n5. Select the \"Release\" [^2] build configuration under \"Run\", then deploy and run to your device.\n\n**Note:** Pay attention to the folder path: `whisper.swiftui.demo/Resources/models` is the appropriate directory to place resources whilst `whisper.swiftui.demo/Models` is related to actual code.\n\n### Core ML support\n1. Follow all the steps in the `Usage` section, including adding the ggml model file.  \nThe ggml model file is required as the Core ML model is only used for the encoder. The\ndecoder which is in the ggml model is still required.\n2. Follow the [`Core ML support` section of readme](../../README.md#core-ml-support) to convert the\nmodel.\n3. Add the Core ML model (`models/ggml-base.en-encoder.mlmodelc/`) to `whisper.swiftui.demo/Resources/models` **via Xcode**.\n\nWhen the example starts running you should now see that it is using the Core ML model:\n```console\nwhisper_init_state: loading Core ML model from '/Library/Developer/CoreSimulator/Devices/25E8C27D-0253-4281-AF17-C3F2A4D1D8F4/data/Containers/Bundle/Application/3ADA7D59-7B9C-43B4-A7E1-A87183FC546A/whisper.swiftui.app/models/ggml-base.en-encoder.mlmodelc'\nwhisper_init_state: first run on a device may take a while ...\nwhisper_init_state: Core ML model loaded\n```\n\n[^1]: I recommend the tiny, base or small models for running on an iOS device.\n\n[^2]: The `Release` build can boost performance of transcription. In this project, it also added `-O3 -DNDEBUG` to `Other C Flags`, but adding flags to app proj is not ideal in real world (applies to all C/C++ files), consider splitting xcodeproj in workspace in your own project.\n\n![image](https://user-images.githubusercontent.com/1991296/212539216-0aef65e4-f882-480a-8358-0f816838fd52.png)\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.cpp.swift/LibWhisper.swift",
    "content": "import Foundation\nimport UIKit\nimport whisper\n\nenum WhisperError: Error {\n    case couldNotInitializeContext\n}\n\n// Meet Whisper C++ constraint: Don't access from more than one thread at a time.\nactor WhisperContext {\n    private var context: OpaquePointer\n\n    init(context: OpaquePointer) {\n        self.context = context\n    }\n\n    deinit {\n        whisper_free(context)\n    }\n\n    func fullTranscribe(samples: [Float]) {\n        // Leave 2 processors free (i.e. the high-efficiency cores).\n        let maxThreads = max(1, min(8, cpuCount() - 2))\n        print(\"Selecting \\(maxThreads) threads\")\n        var params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY)\n        \"en\".withCString { en in\n            // Adapted from whisper.objc\n            params.print_realtime   = true\n            params.print_progress   = false\n            params.print_timestamps = true\n            params.print_special    = false\n            params.translate        = false\n            params.language         = en\n            params.n_threads        = Int32(maxThreads)\n            params.offset_ms        = 0\n            params.no_context       = true\n            params.single_segment   = false\n\n            whisper_reset_timings(context)\n            print(\"About to run whisper_full\")\n            samples.withUnsafeBufferPointer { samples in\n                if (whisper_full(context, params, samples.baseAddress, Int32(samples.count)) != 0) {\n                    print(\"Failed to run the model\")\n                } else {\n                    whisper_print_timings(context)\n                }\n            }\n        }\n    }\n\n    func getTranscription() -> String {\n        var transcription = \"\"\n        for i in 0..<whisper_full_n_segments(context) {\n            transcription += String.init(cString: whisper_full_get_segment_text(context, i))\n        }\n        return transcription\n    }\n\n    static func benchMemcpy(nThreads: Int32) async -> String {\n        return String.init(cString: whisper_bench_memcpy_str(nThreads))\n    }\n\n    static func benchGgmlMulMat(nThreads: Int32) async -> String {\n        return String.init(cString: whisper_bench_ggml_mul_mat_str(nThreads))\n    }\n\n    private func systemInfo() -> String {\n        var info = \"\"\n        //if (ggml_cpu_has_neon() != 0) { info += \"NEON \" }\n        return String(info.dropLast())\n    }\n\n    func benchFull(modelName: String, nThreads: Int32) async -> String {\n        let nMels = whisper_model_n_mels(context)\n        if (whisper_set_mel(context, nil, 0, nMels) != 0) {\n            return \"error: failed to set mel\"\n        }\n\n        // heat encoder\n        if (whisper_encode(context, 0, nThreads) != 0) {\n            return \"error: failed to encode\"\n        }\n\n        var tokens = [whisper_token](repeating: 0, count: 512)\n\n        // prompt heat\n        if (whisper_decode(context, &tokens, 256, 0, nThreads) != 0) {\n            return \"error: failed to decode\"\n        }\n\n        // text-generation heat\n        if (whisper_decode(context, &tokens, 1, 256, nThreads) != 0) {\n            return \"error: failed to decode\"\n        }\n\n        whisper_reset_timings(context)\n\n        // actual run\n        if (whisper_encode(context, 0, nThreads) != 0) {\n            return \"error: failed to encode\"\n        }\n\n        // text-generation\n        for i in 0..<256 {\n            if (whisper_decode(context, &tokens, 1, Int32(i), nThreads) != 0) {\n                return \"error: failed to decode\"\n            }\n        }\n\n        // batched decoding\n        for _ in 0..<64 {\n            if (whisper_decode(context, &tokens, 5, 0, nThreads) != 0) {\n                return \"error: failed to decode\"\n            }\n        }\n\n        // prompt processing\n        for _ in 0..<16 {\n            if (whisper_decode(context, &tokens, 256, 0, nThreads) != 0) {\n                return \"error: failed to decode\"\n            }\n        }\n\n        whisper_print_timings(context)\n\n        let deviceModel = await UIDevice.current.model\n        let systemName = await UIDevice.current.systemName\n        let systemInfo = self.systemInfo()\n        let timings: whisper_timings = whisper_get_timings(context).pointee\n        let encodeMs = String(format: \"%.2f\", timings.encode_ms)\n        let decodeMs = String(format: \"%.2f\", timings.decode_ms)\n        let batchdMs = String(format: \"%.2f\", timings.batchd_ms)\n        let promptMs = String(format: \"%.2f\", timings.prompt_ms)\n        return \"| \\(deviceModel) | \\(systemName) | \\(systemInfo) | \\(modelName) | \\(nThreads) | 1 | \\(encodeMs) | \\(decodeMs) | \\(batchdMs) | \\(promptMs) | <todo> |\"\n    }\n\n    static func createContext(path: String) throws -> WhisperContext {\n        var params = whisper_context_default_params()\n#if targetEnvironment(simulator)\n        params.use_gpu = false\n        print(\"Running on the simulator, using CPU\")\n#else\n        params.flash_attn = true // Enabled by default for Metal\n#endif\n        let context = whisper_init_from_file_with_params(path, params)\n        if let context {\n            return WhisperContext(context: context)\n        } else {\n            print(\"Couldn't load model at \\(path)\")\n            throw WhisperError.couldNotInitializeContext\n        }\n    }\n}\n\nfileprivate func cpuCount() -> Int {\n    ProcessInfo.processInfo.processorCount\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Models/Model.swift",
    "content": "import Foundation\n\nstruct Model: Identifiable {\n    var id = UUID()\n    var name: String\n    var info: String\n    var url: String\n\n    var filename: String\n    var fileURL: URL {\n        FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0].appendingPathComponent(filename)\n    }\n\n    func fileExists() -> Bool {\n        FileManager.default.fileExists(atPath: fileURL.path)\n    }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Models/WhisperState.swift",
    "content": "import Foundation\nimport SwiftUI\nimport AVFoundation\n\n@MainActor\nclass WhisperState: NSObject, ObservableObject, AVAudioRecorderDelegate {\n    @Published var isModelLoaded = false\n    @Published var messageLog = \"\"\n    @Published var canTranscribe = false\n    @Published var isRecording = false\n    \n    private var whisperContext: WhisperContext?\n    private let recorder = Recorder()\n    private var recordedFile: URL? = nil\n    private var audioPlayer: AVAudioPlayer?\n    \n    private var builtInModelUrl: URL? {\n        Bundle.main.url(forResource: \"ggml-base.en\", withExtension: \"bin\", subdirectory: \"models\")\n    }\n    \n    private var sampleUrl: URL? {\n        Bundle.main.url(forResource: \"jfk\", withExtension: \"wav\", subdirectory: \"samples\")\n    }\n    \n    private enum LoadError: Error {\n        case couldNotLocateModel\n    }\n    \n    override init() {\n        super.init()\n        loadModel()\n    }\n    \n    func loadModel(path: URL? = nil, log: Bool = true) {\n        do {\n            whisperContext = nil\n            if (log) { messageLog += \"Loading model...\\n\" }\n            let modelUrl = path ?? builtInModelUrl\n            if let modelUrl {\n                whisperContext = try WhisperContext.createContext(path: modelUrl.path())\n                if (log) { messageLog += \"Loaded model \\(modelUrl.lastPathComponent)\\n\" }\n            } else {\n                if (log) { messageLog += \"Could not locate model\\n\" }\n            }\n            canTranscribe = true\n        } catch {\n            print(error.localizedDescription)\n            if (log) { messageLog += \"\\(error.localizedDescription)\\n\" }\n        }\n    }\n\n    func benchCurrentModel() async {\n        if whisperContext == nil {\n            messageLog += \"Cannot bench without loaded model\\n\"\n            return\n        }\n        messageLog += \"Running benchmark for loaded model\\n\"\n        let result = await whisperContext?.benchFull(modelName: \"<current>\", nThreads: Int32(min(4, cpuCount())))\n        if (result != nil) { messageLog += result! + \"\\n\" }\n    }\n\n    func bench(models: [Model]) async {\n        let nThreads = Int32(min(4, cpuCount()))\n\n//        messageLog += \"Running memcpy benchmark\\n\"\n//        messageLog += await WhisperContext.benchMemcpy(nThreads: nThreads) + \"\\n\"\n//\n//        messageLog += \"Running ggml_mul_mat benchmark with \\(nThreads) threads\\n\"\n//        messageLog += await WhisperContext.benchGgmlMulMat(nThreads: nThreads) + \"\\n\"\n\n        messageLog += \"Running benchmark for all downloaded models\\n\"\n        messageLog += \"| CPU | OS | Config | Model | Th | FA | Enc. | Dec. | Bch5 | PP | Commit |\\n\"\n        messageLog += \"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\\n\"\n        for model in models {\n            loadModel(path: model.fileURL, log: false)\n            if whisperContext == nil {\n                messageLog += \"Cannot bench without loaded model\\n\"\n                break\n            }\n            let result = await whisperContext?.benchFull(modelName: model.name, nThreads: nThreads)\n            if (result != nil) { messageLog += result! + \"\\n\" }\n        }\n        messageLog += \"Benchmarking completed\\n\"\n    }\n    \n    func transcribeSample() async {\n        if let sampleUrl {\n            await transcribeAudio(sampleUrl)\n        } else {\n            messageLog += \"Could not locate sample\\n\"\n        }\n    }\n    \n    private func transcribeAudio(_ url: URL) async {\n        if (!canTranscribe) {\n            return\n        }\n        guard let whisperContext else {\n            return\n        }\n        \n        do {\n            canTranscribe = false\n            messageLog += \"Reading wave samples...\\n\"\n            let data = try readAudioSamples(url)\n            messageLog += \"Transcribing data...\\n\"\n            await whisperContext.fullTranscribe(samples: data)\n            let text = await whisperContext.getTranscription()\n            messageLog += \"Done: \\(text)\\n\"\n        } catch {\n            print(error.localizedDescription)\n            messageLog += \"\\(error.localizedDescription)\\n\"\n        }\n        \n        canTranscribe = true\n    }\n    \n    private func readAudioSamples(_ url: URL) throws -> [Float] {\n        stopPlayback()\n        try startPlayback(url)\n        return try decodeWaveFile(url)\n    }\n    \n    func toggleRecord() async {\n        if isRecording {\n            await recorder.stopRecording()\n            isRecording = false\n            if let recordedFile {\n                await transcribeAudio(recordedFile)\n            }\n        } else {\n            requestRecordPermission { granted in\n                if granted {\n                    Task {\n                        do {\n                            self.stopPlayback()\n                            let file = try FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true)\n                                .appending(path: \"output.wav\")\n                            try await self.recorder.startRecording(toOutputFile: file, delegate: self)\n                            self.isRecording = true\n                            self.recordedFile = file\n                        } catch {\n                            print(error.localizedDescription)\n                            self.messageLog += \"\\(error.localizedDescription)\\n\"\n                            self.isRecording = false\n                        }\n                    }\n                }\n            }\n        }\n    }\n    \n    private func requestRecordPermission(response: @escaping (Bool) -> Void) {\n#if os(macOS)\n        response(true)\n#else\n        AVAudioSession.sharedInstance().requestRecordPermission { granted in\n            response(granted)\n        }\n#endif\n    }\n    \n    private func startPlayback(_ url: URL) throws {\n        audioPlayer = try AVAudioPlayer(contentsOf: url)\n        audioPlayer?.play()\n    }\n    \n    private func stopPlayback() {\n        audioPlayer?.stop()\n        audioPlayer = nil\n    }\n    \n    // MARK: AVAudioRecorderDelegate\n    \n    nonisolated func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: Error?) {\n        if let error {\n            Task {\n                await handleRecError(error)\n            }\n        }\n    }\n    \n    private func handleRecError(_ error: Error) {\n        print(error.localizedDescription)\n        messageLog += \"\\(error.localizedDescription)\\n\"\n        isRecording = false\n    }\n    \n    nonisolated func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully flag: Bool) {\n        Task {\n            await onDidFinishRecording()\n        }\n    }\n    \n    private func onDidFinishRecording() {\n        isRecording = false\n    }\n}\n\n\nfileprivate func cpuCount() -> Int {\n    ProcessInfo.processInfo.processorCount\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Resources/models/.gitignore",
    "content": ""
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Resources/samples/.gitignore",
    "content": ""
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Supporting files/Assets.xcassets/AccentColor.colorset/Contents.json",
    "content": "{\n  \"colors\" : [\n    {\n      \"idiom\" : \"universal\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Supporting files/Assets.xcassets/AppIcon.appiconset/Contents.json",
    "content": "{\n  \"images\" : [\n    {\n      \"idiom\" : \"universal\",\n      \"platform\" : \"ios\",\n      \"size\" : \"1024x1024\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"16x16\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"16x16\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"32x32\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"32x32\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"128x128\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"128x128\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"256x256\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"256x256\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"512x512\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"512x512\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Supporting files/Assets.xcassets/Contents.json",
    "content": "{\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Supporting files/Preview Content/Preview Assets.xcassets/Contents.json",
    "content": "{\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Supporting files/WhisperCppDemo.entitlements",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n\t<key>com.apple.security.app-sandbox</key>\n\t<true/>\n\t<key>com.apple.security.device.audio-input</key>\n\t<true/>\n\t<key>com.apple.security.files.user-selected.read-only</key>\n\t<true/>\n</dict>\n</plist>\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/UI/ContentView.swift",
    "content": "import SwiftUI\nimport AVFoundation\nimport Foundation\n\nstruct ContentView: View {\n    @StateObject var whisperState = WhisperState()\n    \n    var body: some View {\n        NavigationStack {\n            VStack {\n                HStack {\n                    Button(\"Transcribe\", action: {\n                        Task {\n                            await whisperState.transcribeSample()\n                        }\n                    })\n                    .buttonStyle(.bordered)\n                    .disabled(!whisperState.canTranscribe)\n                    \n                    Button(whisperState.isRecording ? \"Stop recording\" : \"Start recording\", action: {\n                        Task {\n                            await whisperState.toggleRecord()\n                        }\n                    })\n                    .buttonStyle(.bordered)\n                    .disabled(!whisperState.canTranscribe)\n                }\n                \n                ScrollView {\n                    Text(verbatim: whisperState.messageLog)\n                        .frame(maxWidth: .infinity, alignment: .leading)\n                }\n                .font(.footnote)\n                .padding()\n                .background(Color.gray.opacity(0.1))\n                .cornerRadius(10)\n\n                HStack {\n                    Button(\"Clear Logs\", action: {\n                        whisperState.messageLog = \"\"\n                    })\n                    .font(.footnote)\n                    .buttonStyle(.bordered)\n\n                    Button(\"Copy Logs\", action: {\n                        UIPasteboard.general.string = whisperState.messageLog\n                    })\n                    .font(.footnote)\n                    .buttonStyle(.bordered)\n\n                    Button(\"Bench\", action: {\n                        Task {\n                            await whisperState.benchCurrentModel()\n                        }\n                    })\n                    .font(.footnote)\n                    .buttonStyle(.bordered)\n                    .disabled(!whisperState.canTranscribe)\n\n                    Button(\"Bench All\", action: {\n                        Task {\n                            await whisperState.bench(models: ModelsView.getDownloadedModels())\n                        }\n                    })\n                    .font(.footnote)\n                    .buttonStyle(.bordered)\n                    .disabled(!whisperState.canTranscribe)\n                }\n\n                NavigationLink(destination: ModelsView(whisperState: whisperState)) {\n                    Text(\"View Models\")\n                }\n                .font(.footnote)\n                .padding()\n            }\n            .navigationTitle(\"Whisper SwiftUI Demo\")\n            .padding()\n        }\n    }\n\n    struct ModelsView: View {\n        @ObservedObject var whisperState: WhisperState\n        @Environment(\\.dismiss) var dismiss\n        \n        private static let models: [Model] = [\n            Model(name: \"tiny\", info: \"(F16, 75 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin\", filename: \"tiny.bin\"),\n            Model(name: \"tiny-q5_1\", info: \"(31 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin\", filename: \"tiny-q5_1.bin\"),\n            Model(name: \"tiny-q8_0\", info: \"(42 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q8_0.bin\", filename: \"tiny-q8_0.bin\"),\n            Model(name: \"tiny.en\", info: \"(F16, 75 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin\", filename: \"tiny.en.bin\"),\n            Model(name: \"tiny.en-q5_1\", info: \"(31 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin\", filename: \"tiny.en-q5_1.bin\"),\n            Model(name: \"tiny.en-q8_0\", info: \"(42 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q8_0.bin\", filename: \"tiny.en-q8_0.bin\"),\n            Model(name: \"base\", info: \"(F16, 142 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin\", filename: \"base.bin\"),\n            Model(name: \"base-q5_1\", info: \"(57 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin\", filename: \"base-q5_1.bin\"),\n            Model(name: \"base-q8_0\", info: \"(78 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin\", filename: \"base-q8_0.bin\"),\n            Model(name: \"base.en\", info: \"(F16, 142 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin\", filename: \"base.en.bin\"),\n            Model(name: \"base.en-q5_1\", info: \"(57 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin\", filename: \"base.en-q5_1.bin\"),\n            Model(name: \"base.en-q8_0\", info: \"(78 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q8_0.bin\", filename: \"base.en-q8_0.bin\"),\n            Model(name: \"small\", info: \"(F16, 466 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin\", filename: \"small.bin\"),\n            Model(name: \"small-q5_1\", info: \"(181 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q5_1.bin\", filename: \"small-q5_1.bin\"),\n            Model(name: \"small-q8_0\", info: \"(252 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin\", filename: \"small-q8_0.bin\"),\n            Model(name: \"small.en\", info: \"(F16, 466 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin\", filename: \"small.en.bin\"),\n            Model(name: \"small.en-q5_1\", info: \"(181 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en-q5_1.bin\", filename: \"small.en-q5_1.bin\"),\n            Model(name: \"small.en-q8_0\", info: \"(252 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en-q8_0.bin\", filename: \"small.en-q8_0.bin\"),\n            Model(name: \"medium\", info: \"(F16, 1.5 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin\", filename: \"medium.bin\"),\n            Model(name: \"medium-q5_0\", info: \"(514 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium-q5_0.bin\", filename: \"medium-q5_0.bin\"),\n            Model(name: \"medium-q8_0\", info: \"(785 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium-q8_0.bin\", filename: \"medium-q8_0.bin\"),\n            Model(name: \"medium.en\", info: \"(F16, 1.5 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en.bin\", filename: \"medium.en.bin\"),\n            Model(name: \"medium.en-q5_0\", info: \"(514 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en-q5_0.bin\", filename: \"medium.en-q5_0.bin\"),\n            Model(name: \"medium.en-q8_0\", info: \"(785 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en-q8_0.bin\", filename: \"medium.en-q8_0.bin\"),\n            Model(name: \"large-v1\", info: \"(F16, 2.9 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large.bin\", filename: \"large.bin\"),\n            Model(name: \"large-v2\", info: \"(F16, 2.9 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v2.bin\", filename: \"large-v2.bin\"),\n            Model(name: \"large-v2-q5_0\", info: \"(1.1 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v2-q5_0.bin\", filename: \"large-v2-q5_0.bin\"),\n            Model(name: \"large-v2-q8_0\", info: \"(1.5 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v2-q8_0.bin\", filename: \"large-v2-q8_0.bin\"),\n            Model(name: \"large-v3\", info: \"(F16, 2.9 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin\", filename: \"large-v3.bin\"),\n            Model(name: \"large-v3-q5_0\", info: \"(1.1 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-q5_0.bin\", filename: \"large-v3-q5_0.bin\"),\n            Model(name: \"large-v3-turbo\", info: \"(F16, 1.5 GiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin\", filename: \"large-v3-turbo.bin\"),\n            Model(name: \"large-v3-turbo-q5_0\", info: \"(547 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin\", filename: \"large-v3-turbo-q5_0.bin\"),\n            Model(name: \"large-v3-turbo-q8_0\", info: \"(834 MiB)\", url: \"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin\", filename: \"large-v3-turbo-q8_0.bin\"),\n        ]\n\n        static func getDownloadedModels() -> [Model] {\n            // Filter models that have been downloaded\n            return models.filter {\n                FileManager.default.fileExists(atPath: $0.fileURL.path())\n            }\n        }\n\n        func loadModel(model: Model) {\n            Task {\n                dismiss()\n                whisperState.loadModel(path: model.fileURL)\n            }\n        }\n\n        var body: some View {\n            List {\n                Section(header: Text(\"Models\")) {\n                    ForEach(ModelsView.models) { model in\n                        DownloadButton(model: model)\n                            .onLoad(perform: loadModel)\n                    }\n                }\n            }\n            .listStyle(GroupedListStyle())\n            .navigationBarTitle(\"Models\", displayMode: .inline).toolbar {}\n        }\n    }\n}\n\n//struct ContentView_Previews: PreviewProvider {\n//    static var previews: some View {\n//        ContentView()\n//    }\n//}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/UI/DownloadButton.swift",
    "content": "import SwiftUI\n\nstruct DownloadButton: View {\n    private var model: Model\n\n    @State private var status: String\n\n    @State private var downloadTask: URLSessionDownloadTask?\n    @State private var progress = 0.0\n    @State private var observation: NSKeyValueObservation?\n\n    private var onLoad: ((_ model: Model) -> Void)?\n\n    init(model: Model) {\n        self.model = model\n        status = model.fileExists() ? \"downloaded\" : \"download\"\n    }\n\n    func onLoad(perform action: @escaping (_ model: Model) -> Void) -> DownloadButton {\n        var button = self\n        button.onLoad = action\n        return button\n    }\n\n    private func download() {\n        status = \"downloading\"\n        print(\"Downloading model \\(model.name) from \\(model.url)\")\n        guard let url = URL(string: model.url) else { return }\n\n        downloadTask = URLSession.shared.downloadTask(with: url) { temporaryURL, response, error in\n            if let error = error {\n                print(\"Error: \\(error.localizedDescription)\")\n                return\n            }\n\n            guard let response = response as? HTTPURLResponse, (200...299).contains(response.statusCode) else {\n                print(\"Server error!\")\n                return\n            }\n\n            do {\n                if let temporaryURL = temporaryURL {\n                    try FileManager.default.copyItem(at: temporaryURL, to: model.fileURL)\n                    print(\"Writing to \\(model.filename) completed\")\n                    status = \"downloaded\"\n                }\n            } catch let err {\n                print(\"Error: \\(err.localizedDescription)\")\n            }\n        }\n\n        observation = downloadTask?.progress.observe(\\.fractionCompleted) { progress, _ in\n            self.progress = progress.fractionCompleted\n        }\n\n        downloadTask?.resume()\n    }\n\n    var body: some View {\n        VStack {\n            Button(action: {\n                if (status == \"download\") {\n                    download()\n                } else if (status == \"downloading\") {\n                    downloadTask?.cancel()\n                    status = \"download\"\n                } else if (status == \"downloaded\") {\n                    if !model.fileExists() {\n                        download()\n                    }\n                    onLoad?(model)\n                }\n            }) {\n                let title = \"\\(model.name) \\(model.info)\"\n                if (status == \"download\") {\n                    Text(\"Download \\(title)\")\n                } else if (status == \"downloading\") {\n                    Text(\"\\(title) (Downloading \\(Int(progress * 100))%)\")\n                } else if (status == \"downloaded\") {\n                    Text(\"Load \\(title)\")\n                } else {\n                    Text(\"Unknown status\")\n                }\n            }.swipeActions {\n                if (status == \"downloaded\") {\n                    Button(\"Delete\") {\n                        do {\n                            try FileManager.default.removeItem(at: model.fileURL)\n                        } catch {\n                            print(\"Error deleting file: \\(error)\")\n                        }\n                        status = \"download\"\n                    }\n                    .tint(.red)\n                }\n            }\n        }\n        .onDisappear() {\n            downloadTask?.cancel()\n        }\n    }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Utils/Recorder.swift",
    "content": "import Foundation\nimport AVFoundation\n\nactor Recorder {\n    private var recorder: AVAudioRecorder?\n    \n    enum RecorderError: Error {\n        case couldNotStartRecording\n    }\n    \n    func startRecording(toOutputFile url: URL, delegate: AVAudioRecorderDelegate?) throws {\n        let recordSettings: [String : Any] = [\n            AVFormatIDKey: Int(kAudioFormatLinearPCM),\n            AVSampleRateKey: 16000.0,\n            AVNumberOfChannelsKey: 1,\n            AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue\n        ]\n#if !os(macOS)\n        let session = AVAudioSession.sharedInstance()\n        try session.setCategory(.playAndRecord, mode: .default)\n#endif\n        let recorder = try AVAudioRecorder(url: url, settings: recordSettings)\n        recorder.delegate = delegate\n        if recorder.record() == false {\n            print(\"Could not start recording\")\n            throw RecorderError.couldNotStartRecording\n        }\n        self.recorder = recorder\n    }\n    \n    func stopRecording() {\n        recorder?.stop()\n        recorder = nil\n    }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/Utils/RiffWaveUtils.swift",
    "content": "import Foundation\n\nfunc decodeWaveFile(_ url: URL) throws -> [Float] {\n    let data = try Data(contentsOf: url)\n    let floats = stride(from: 44, to: data.count, by: 2).map {\n        return data[$0..<$0 + 2].withUnsafeBytes {\n            let short = Int16(littleEndian: $0.load(as: Int16.self))\n            return max(-1.0, min(Float(short) / 32767.0, 1.0))\n        }\n    }\n    return floats\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.demo/WhisperCppDemoApp.swift",
    "content": "import SwiftUI\n\n@main\nstruct WhisperCppDemoApp: App {\n    var body: some Scene {\n        WindowGroup {\n            ContentView()\n        }\n    }\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.xcodeproj/.gitignore",
    "content": "xcuserdata/\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.xcodeproj/project.pbxproj",
    "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 56;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\t0A8E49002954B3F100704C1B /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = 0A8E48FF2954B3F100704C1B /* README.md */; };\n\t\t0AA751482953AC2E001EE061 /* samples in Resources */ = {isa = PBXBuildFile; fileRef = 0AA751462953AC2E001EE061 /* samples */; };\n\t\t0AA751492953AC2E001EE061 /* models in Resources */ = {isa = PBXBuildFile; fileRef = 0AA751472953AC2E001EE061 /* models */; };\n\t\t0AA7514C2953B569001EE061 /* RiffWaveUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AA7514B2953B569001EE061 /* RiffWaveUtils.swift */; };\n\t\t0AA7514E2953D958001EE061 /* Recorder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AA7514D2953D958001EE061 /* Recorder.swift */; };\n\t\t0AAC5D9B29539CCF003032C3 /* WhisperCppDemoApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAC5D9A29539CCF003032C3 /* WhisperCppDemoApp.swift */; };\n\t\t0AAC5D9D29539CCF003032C3 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAC5D9C29539CCF003032C3 /* ContentView.swift */; };\n\t\t0AAC5D9F29539CD0003032C3 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 0AAC5D9E29539CD0003032C3 /* Assets.xcassets */; };\n\t\t0AAC5DCE2953A05C003032C3 /* WhisperState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAC5DCD2953A05C003032C3 /* WhisperState.swift */; };\n\t\t0AAC5DD12953A394003032C3 /* LibWhisper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AAC5DD02953A394003032C3 /* LibWhisper.swift */; };\n\t\t5B3454FF2D8178F80005A3BC /* whisper.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5B3454FE2D8178F80005A3BC /* whisper.xcframework */; };\n\t\t5B3455002D8178F80005A3BC /* whisper.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 5B3454FE2D8178F80005A3BC /* whisper.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };\n\t\t7F79E0EE2CE0A78000ACD7BF /* DownloadButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F79E0ED2CE0A78000ACD7BF /* DownloadButton.swift */; };\n\t\t7F79E0F02CE0C6F700ACD7BF /* Model.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7F79E0EF2CE0C6F700ACD7BF /* Model.swift */; };\n/* End PBXBuildFile section */\n\n/* Begin PBXCopyFilesBuildPhase section */\n\t\t5B3455012D8178F80005A3BC /* Embed Frameworks */ = {\n\t\t\tisa = PBXCopyFilesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tdstPath = \"\";\n\t\t\tdstSubfolderSpec = 10;\n\t\t\tfiles = (\n\t\t\t\t5B3455002D8178F80005A3BC /* whisper.xcframework in Embed Frameworks */,\n\t\t\t);\n\t\t\tname = \"Embed Frameworks\";\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXCopyFilesBuildPhase section */\n\n/* Begin PBXFileReference section */\n\t\t0A8E48FF2954B3F100704C1B /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = \"<group>\"; };\n\t\t0AA751462953AC2E001EE061 /* samples */ = {isa = PBXFileReference; lastKnownFileType = folder; path = samples; sourceTree = \"<group>\"; };\n\t\t0AA751472953AC2E001EE061 /* models */ = {isa = PBXFileReference; lastKnownFileType = folder; path = models; sourceTree = \"<group>\"; };\n\t\t0AA7514B2953B569001EE061 /* RiffWaveUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RiffWaveUtils.swift; sourceTree = \"<group>\"; };\n\t\t0AA7514D2953D958001EE061 /* Recorder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Recorder.swift; sourceTree = \"<group>\"; };\n\t\t0AAC5D9729539CCF003032C3 /* whisper.swiftui.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = whisper.swiftui.app; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t0AAC5D9A29539CCF003032C3 /* WhisperCppDemoApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhisperCppDemoApp.swift; sourceTree = \"<group>\"; };\n\t\t0AAC5D9C29539CCF003032C3 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = \"<group>\"; };\n\t\t0AAC5D9E29539CD0003032C3 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = \"<group>\"; };\n\t\t0AAC5DA029539CD0003032C3 /* WhisperCppDemo.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = WhisperCppDemo.entitlements; sourceTree = \"<group>\"; };\n\t\t0AAC5DCD2953A05C003032C3 /* WhisperState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WhisperState.swift; sourceTree = \"<group>\"; };\n\t\t0AAC5DD02953A394003032C3 /* LibWhisper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibWhisper.swift; sourceTree = \"<group>\"; };\n\t\t5B3454FE2D8178F80005A3BC /* whisper.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = whisper.xcframework; path = \"../../build-apple/whisper.xcframework\"; sourceTree = \"<group>\"; };\n\t\t7F79E0ED2CE0A78000ACD7BF /* DownloadButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadButton.swift; sourceTree = \"<group>\"; };\n\t\t7F79E0EF2CE0C6F700ACD7BF /* Model.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Model.swift; sourceTree = \"<group>\"; };\n/* End PBXFileReference section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\t0AAC5D9429539CCF003032C3 /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t5B3454FF2D8178F80005A3BC /* whisper.xcframework in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\t0AA7513F2953AB32001EE061 /* Models */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AAC5DCD2953A05C003032C3 /* WhisperState.swift */,\n\t\t\t\t7F79E0EF2CE0C6F700ACD7BF /* Model.swift */,\n\t\t\t);\n\t\t\tpath = Models;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AA751402953ABA6001EE061 /* Resources */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AA751472953AC2E001EE061 /* models */,\n\t\t\t\t0AA751462953AC2E001EE061 /* samples */,\n\t\t\t);\n\t\t\tpath = Resources;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AA7514A2953B561001EE061 /* Utils */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AA7514B2953B569001EE061 /* RiffWaveUtils.swift */,\n\t\t\t\t0AA7514D2953D958001EE061 /* Recorder.swift */,\n\t\t\t);\n\t\t\tpath = Utils;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5D8E29539CCF003032C3 = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0A8E48FF2954B3F100704C1B /* README.md */,\n\t\t\t\t0AAC5DCF2953A36C003032C3 /* whisper.cpp.swift */,\n\t\t\t\t0AAC5D9929539CCF003032C3 /* whisper.swiftui.demo */,\n\t\t\t\t0AAC5D9829539CCF003032C3 /* Products */,\n\t\t\t\tE3F92DC32AFA8E3800A6A9D4 /* Frameworks */,\n\t\t\t);\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5D9829539CCF003032C3 /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AAC5D9729539CCF003032C3 /* whisper.swiftui.app */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5D9929539CCF003032C3 /* whisper.swiftui.demo */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AA7514A2953B561001EE061 /* Utils */,\n\t\t\t\t0AA751402953ABA6001EE061 /* Resources */,\n\t\t\t\t0AA7513F2953AB32001EE061 /* Models */,\n\t\t\t\t0AAC5DD32953A9ED003032C3 /* Supporting files */,\n\t\t\t\t0AAC5DD22953A9E3003032C3 /* UI */,\n\t\t\t\t0AAC5D9A29539CCF003032C3 /* WhisperCppDemoApp.swift */,\n\t\t\t);\n\t\t\tpath = whisper.swiftui.demo;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5DCF2953A36C003032C3 /* whisper.cpp.swift */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AAC5DD02953A394003032C3 /* LibWhisper.swift */,\n\t\t\t);\n\t\t\tpath = whisper.cpp.swift;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5DD22953A9E3003032C3 /* UI */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AAC5D9C29539CCF003032C3 /* ContentView.swift */,\n\t\t\t\t7F79E0ED2CE0A78000ACD7BF /* DownloadButton.swift */,\n\t\t\t);\n\t\t\tpath = UI;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\t0AAC5DD32953A9ED003032C3 /* Supporting files */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t0AAC5D9E29539CD0003032C3 /* Assets.xcassets */,\n\t\t\t\t0AAC5DA029539CD0003032C3 /* WhisperCppDemo.entitlements */,\n\t\t\t);\n\t\t\tpath = \"Supporting files\";\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tE3F92DC32AFA8E3800A6A9D4 /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t5B3454FE2D8178F80005A3BC /* whisper.xcframework */,\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXNativeTarget section */\n\t\t0AAC5D9629539CCF003032C3 /* whisper.swiftui */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = 0AAC5DBC29539CD0003032C3 /* Build configuration list for PBXNativeTarget \"whisper.swiftui\" */;\n\t\t\tbuildPhases = (\n\t\t\t\t0AAC5D9329539CCF003032C3 /* Sources */,\n\t\t\t\t0AAC5D9429539CCF003032C3 /* Frameworks */,\n\t\t\t\t0AAC5D9529539CCF003032C3 /* Resources */,\n\t\t\t\t5B3455012D8178F80005A3BC /* Embed Frameworks */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tname = whisper.swiftui;\n\t\t\tpackageProductDependencies = (\n\t\t\t);\n\t\t\tproductName = WhisperCppDemo;\n\t\t\tproductReference = 0AAC5D9729539CCF003032C3 /* whisper.swiftui.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\t0AAC5D8F29539CCF003032C3 /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = 1;\n\t\t\t\tLastSwiftUpdateCheck = 1410;\n\t\t\t\tLastUpgradeCheck = 1410;\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\t0AAC5D9629539CCF003032C3 = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 14.1;\n\t\t\t\t\t\tLastSwiftMigration = 1410;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = 0AAC5D9229539CCF003032C3 /* Build configuration list for PBXProject \"whisper.swiftui\" */;\n\t\t\tcompatibilityVersion = \"Xcode 14.0\";\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = 0AAC5D8E29539CCF003032C3;\n\t\t\tproductRefGroup = 0AAC5D9829539CCF003032C3 /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\t0AAC5D9629539CCF003032C3 /* whisper.swiftui */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\t0AAC5D9529539CCF003032C3 /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t0AA751482953AC2E001EE061 /* samples in Resources */,\n\t\t\t\t0A8E49002954B3F100704C1B /* README.md in Resources */,\n\t\t\t\t0AA751492953AC2E001EE061 /* models in Resources */,\n\t\t\t\t0AAC5D9F29539CD0003032C3 /* Assets.xcassets in Resources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\t0AAC5D9329539CCF003032C3 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t0AAC5D9D29539CCF003032C3 /* ContentView.swift in Sources */,\n\t\t\t\t0AAC5D9B29539CCF003032C3 /* WhisperCppDemoApp.swift in Sources */,\n\t\t\t\t0AAC5DCE2953A05C003032C3 /* WhisperState.swift in Sources */,\n\t\t\t\t0AAC5DD12953A394003032C3 /* LibWhisper.swift in Sources */,\n\t\t\t\t0AA7514C2953B569001EE061 /* RiffWaveUtils.swift in Sources */,\n\t\t\t\t7F79E0EE2CE0A78000ACD7BF /* DownloadButton.swift in Sources */,\n\t\t\t\t0AA7514E2953D958001EE061 /* Recorder.swift in Sources */,\n\t\t\t\t7F79E0F02CE0C6F700ACD7BF /* Model.swift in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin XCBuildConfiguration section */\n\t\t0AAC5DBA29539CD0003032C3 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tINFOPLIST_KEY_NSMicrophoneUsageDescription = \"Needed to transcribe audio\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t0AAC5DBB29539CD0003032C3 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu11;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tINFOPLIST_KEY_NSMicrophoneUsageDescription = \"Needed to transcribe audio\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-O\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\t0AAC5DBD29539CD0003032C3 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"whisper.swiftui.demo/Supporting files/WhisperCppDemo.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"whisper.swiftui.demo/Supporting files/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = \"\";\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]\" = UIStatusBarStyleDefault;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]\" = UIStatusBarStyleDefault;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = \"UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = \"UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 16.1;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = \"@executable_path/Frameworks\";\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = \"@executable_path/../Frameworks\";\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 13.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.whispercppdemo.WhisperCppDemo;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator\";\n\t\t\t\tSUPPORTS_MACCATALYST = NO;\n\t\t\t\tSUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = YES;\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t0AAC5DBE29539CD0003032C3 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"whisper.swiftui.demo/Supporting files/WhisperCppDemo.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"whisper.swiftui.demo/Supporting files/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = \"\";\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]\" = UIStatusBarStyleDefault;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]\" = UIStatusBarStyleDefault;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = \"UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = \"UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 16.1;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = \"@executable_path/Frameworks\";\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = \"@executable_path/../Frameworks\";\n\t\t\t\tLLVM_LTO = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 13.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tOTHER_CFLAGS = (\n\t\t\t\t\t\"-O3\",\n\t\t\t\t\t\"-DNDEBUG\",\n\t\t\t\t);\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.whispercppdemo.WhisperCppDemo;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator\";\n\t\t\t\tSUPPORTS_MACCATALYST = NO;\n\t\t\t\tSUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = YES;\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\t0AAC5D9229539CCF003032C3 /* Build configuration list for PBXProject \"whisper.swiftui\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t0AAC5DBA29539CD0003032C3 /* Debug */,\n\t\t\t\t0AAC5DBB29539CD0003032C3 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\t0AAC5DBC29539CD0003032C3 /* Build configuration list for PBXNativeTarget \"whisper.swiftui\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t0AAC5DBD29539CD0003032C3 /* Debug */,\n\t\t\t\t0AAC5DBE29539CD0003032C3 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\t};\n\trootObject = 0AAC5D8F29539CCF003032C3 /* Project object */;\n}\n"
  },
  {
    "path": "examples/whisper.swiftui/whisper.swiftui.xcodeproj/project.xcworkspace/.gitignore",
    "content": "contents.xcworkspacedata\n"
  },
  {
    "path": "examples/whisper.wasm/CMakeLists.txt",
    "content": "#\n# libmain\n#\n\nset(TARGET libmain)\n\nadd_executable(${TARGET}\n    emscripten.cpp\n    )\n\ninclude(DefaultTargetOptions)\n\ntarget_link_libraries(${TARGET} PRIVATE\n    whisper\n    )\n\nunset(EXTRA_FLAGS)\n\nif (WHISPER_WASM_SINGLE_FILE)\n    set(EXTRA_FLAGS \"-s SINGLE_FILE=1\")\n    message(STATUS \"Embedding WASM inside main.js\")\n\n    add_custom_command(\n        TARGET ${TARGET} POST_BUILD\n        COMMAND ${CMAKE_COMMAND} -E copy\n        ${CMAKE_BINARY_DIR}/bin/libmain.js\n        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/whisper.wasm/main.js\n        )\nendif()\n\nset_target_properties(${TARGET} PROPERTIES LINK_FLAGS \" \\\n    --bind \\\n    -s USE_PTHREADS=1 \\\n    -s PTHREAD_POOL_SIZE_STRICT=0 \\\n    -s INITIAL_MEMORY=512MB \\\n    -s MAXIMUM_MEMORY=2000MB \\\n    -s ALLOW_MEMORY_GROWTH=1 \\\n    -s FORCE_FILESYSTEM=1 \\\n    -s EXPORTED_RUNTIME_METHODS=\\\"['print', 'printErr', 'ccall', 'cwrap', 'HEAPU8']\\\" \\\n    ${EXTRA_FLAGS} \\\n    \")\n\n#\n# whisper.wasm\n#\n\nset(TARGET whisper.wasm)\n\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/index-tmpl.html  ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/index.html @ONLY)\nconfigure_file(${CMAKE_CURRENT_SOURCE_DIR}/../helpers.js    ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TARGET}/helpers.js @ONLY)\n"
  },
  {
    "path": "examples/whisper.wasm/README.md",
    "content": "# whisper.wasm\n\nInference of [OpenAI's Whisper ASR model](https://github.com/openai/whisper) inside the browser\n\nThis example uses a WebAssembly (WASM) port of the [whisper.cpp](https://github.com/ggerganov/whisper.cpp)\nimplementation of the transformer to run the inference inside a web page. The audio data does not leave your computer -\nit is processed locally on your machine. The performance is not great but you should be able to achieve x2 or x3\nreal-time for the `tiny` and `base` models on a modern CPU and browser (i.e. transcribe a 60 seconds audio in about\n~20-30 seconds).\n\nThis WASM port utilizes [WASM SIMD 128-bit intrinsics](https://emcc.zcopy.site/docs/porting/simd/) so you have to make\nsure that [your browser supports them](https://webassembly.org/roadmap/).\n\nThe example is capable of running all models up to size `small` inclusive. Beyond that, the memory requirements and\nperformance are unsatisfactory. The implementation currently support only the `Greedy` sampling strategy. Both\ntranscription and translation are supported.\n\nSince the model data is quite big (74MB for the `tiny` model) you need to manually load the model into the web-page.\n\nThe example supports both loading audio from a file and recording audio from the microphone. The maximum length of the\naudio is limited to 120 seconds.\n\n## Live demo\n\nLink: https://ggml.ai/whisper.cpp/\n\n![image](https://user-images.githubusercontent.com/1991296/197348344-1a7fead8-3dae-4922-8b06-df223a206603.png)\n\n## Build instructions\n\n```bash (v3.1.2)\n# build using Emscripten\ngit clone https://github.com/ggml-org/whisper.cpp\ncd whisper.cpp\nmkdir build-em && cd build-em\nemcmake cmake ..\nmake -j\n```\nThe example can then be started by running a local HTTP server:\n```console\npython3 examples/server.py\n```\nAnd then opening a browser to the following URL:\nhttp://localhost:8000/whisper.wasm\n\nTo run the example in a different server, you need to copy the following files\nto the server's HTTP path:\n```\n# copy the produced page to your HTTP path\ncp bin/whisper.wasm/*    /path/to/html/\ncp bin/libmain.js        /path/to/html/\ncp bin/libmain.worker.js /path/to/html/\n```\n\n> 📝 **Note:** By default this example is built with `WHISPER_WASM_SINGLE_FILE=ON`\n> which means that that a separate .wasm file will not be generated. Instead, the\n> WASM module is embedded in the main JS file as a base64 encoded string. To\n> generate a separate .wasm file, you need to disable this option by passing\n> `-DWHISPER_WASM_SINGLE_FILE=OFF`:\n> ```console\n> emcmake cmake .. -DWHISPER_WASM_SINGLE_FILE=OFF\n> ```\n> This will generate a `libmain.wasm` file in the build/bin directory.\n\n> 📝 **Note:** As of Emscripten 3.1.58 (April 2024), separate worker.js files are no\n> longer generated and the worker is embedded in the main JS file. So the worker\n> file will not be geneated for versions later than `3.1.58`.\n"
  },
  {
    "path": "examples/whisper.wasm/emscripten.cpp",
    "content": "#include \"whisper.h\"\n\n#include <emscripten.h>\n#include <emscripten/bind.h>\n\n#include <vector>\n#include <thread>\n\nstd::thread g_worker;\n\nstd::vector<struct whisper_context *> g_contexts(4, nullptr);\n\nstatic inline int mpow2(int n) {\n    int p = 1;\n    while (p <= n) p *= 2;\n    return p/2;\n}\n\nEMSCRIPTEN_BINDINGS(whisper) {\n    emscripten::function(\"init\", emscripten::optional_override([](const std::string & path_model) {\n        if (g_worker.joinable()) {\n            g_worker.join();\n        }\n\n        for (size_t i = 0; i < g_contexts.size(); ++i) {\n            if (g_contexts[i] == nullptr) {\n                g_contexts[i] = whisper_init_from_file_with_params(path_model.c_str(), whisper_context_default_params());\n                if (g_contexts[i] != nullptr) {\n                    return i + 1;\n                } else {\n                    return (size_t) 0;\n                }\n            }\n        }\n\n        return (size_t) 0;\n    }));\n\n    emscripten::function(\"free\", emscripten::optional_override([](size_t index) {\n        if (g_worker.joinable()) {\n            g_worker.join();\n        }\n\n        --index;\n\n        if (index < g_contexts.size()) {\n            whisper_free(g_contexts[index]);\n            g_contexts[index] = nullptr;\n        }\n    }));\n\n    emscripten::function(\"full_default\", emscripten::optional_override([](size_t index, const emscripten::val & audio, const std::string & lang, int nthreads, bool translate) {\n        if (g_worker.joinable()) {\n            g_worker.join();\n        }\n\n        --index;\n\n        if (index >= g_contexts.size()) {\n            return -1;\n        }\n\n        if (g_contexts[index] == nullptr) {\n            return -2;\n        }\n\n        struct whisper_full_params params = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);\n        bool is_multilingual = whisper_is_multilingual(g_contexts[index]);\n\n        params.print_realtime   = true;\n        params.print_progress   = false;\n        params.print_timestamps = true;\n        params.print_special    = false;\n        params.translate        = translate;\n        params.language         = is_multilingual ? strdup(lang.c_str()) : \"en\";\n        params.n_threads        = std::min(nthreads, std::min(16, mpow2(std::thread::hardware_concurrency())));\n        params.offset_ms        = 0;\n\n        std::vector<float> pcmf32;\n        const int n = audio[\"length\"].as<int>();\n\n        emscripten::val heap = emscripten::val::module_property(\"HEAPU8\");\n        emscripten::val memory = heap[\"buffer\"];\n\n        pcmf32.resize(n);\n\n        emscripten::val memoryView = audio[\"constructor\"].new_(memory, reinterpret_cast<uintptr_t>(pcmf32.data()), n);\n        memoryView.call<void>(\"set\", audio);\n\n        // print system information\n        {\n            printf(\"system_info: n_threads = %d / %d | %s\\n\",\n                    params.n_threads, std::thread::hardware_concurrency(), whisper_print_system_info());\n\n            printf(\"%s: processing %d samples, %.1f sec, %d threads, %d processors, lang = %s, task = %s ...\\n\",\n                    __func__, int(pcmf32.size()), float(pcmf32.size())/WHISPER_SAMPLE_RATE,\n                    params.n_threads, 1,\n                    params.language,\n                    params.translate ? \"translate\" : \"transcribe\");\n\n            printf(\"\\n\");\n        }\n\n        // run the worker\n        {\n            g_worker = std::thread([index, params, pcmf32 = std::move(pcmf32), is_multilingual]() {\n                whisper_reset_timings(g_contexts[index]);\n                whisper_full(g_contexts[index], params, pcmf32.data(), pcmf32.size());\n                whisper_print_timings(g_contexts[index]);\n                if (is_multilingual) {\n                    free((void*)params.language);\n                }\n            });\n        }\n\n        return 0;\n    }));\n}\n"
  },
  {
    "path": "examples/whisper.wasm/index-tmpl.html",
    "content": "<!doctype html>\n<html lang=\"en-us\">\n    <head>\n        <title>whisper.cpp : WASM example</title>\n\n        <style>\n            #output {\n                width: 100%;\n                height: 100%;\n                margin: 0 auto;\n                margin-top: 10px;\n                border-left: 0px;\n                border-right: 0px;\n                padding-left: 0px;\n                padding-right: 0px;\n                display: block;\n                background-color: black;\n                color: white;\n                font-size: 10px;\n                font-family: 'Lucida Console', Monaco, monospace;\n                outline: none;\n                white-space: pre;\n                overflow-wrap: normal;\n                overflow-x: scroll;\n            }\n        </style>\n        <script src=\"coi-serviceworker.js\"></script>\n        <link rel=\"icon\" href=\"data:,\">\n    </head>\n    <body>\n        <div id=\"main-container\">\n            <b>Minimal <a href=\"https://github.com/ggerganov/whisper.cpp\">whisper.cpp</a> example running fully in the browser</b>\n\n            <br><br>\n\n            Usage instructions:<br>\n            <ul>\n                <li>Load a ggml model file (you can obtain one from <a href=\"https://ggml.ggerganov.com/\">here</a>, recommended: <b>tiny</b> or <b>base</b>)</li>\n                <li>Select audio file to transcribe or record audio from the microphone (sample: <a href=\"https://whisper.ggerganov.com/jfk.wav\">jfk.wav</a>)</li>\n                <li>Click on the \"Transcribe\" button to start the transcription</li>\n            </ul>\n\n            Note that the computation is quite heavy and may take a few seconds to complete.<br>\n            The transcription results will be displayed in the text area below.<br><br>\n            <b>Important:</b>\n                <ul>\n                    <li>your browser must support WASM SIMD instructions for this to work</li>\n                    <li>Firefox cannot load files larger than 256 MB - use Chrome instead</li>\n                </ul>\n\n            <b>More examples:</b>\n                <a href=\"bench.wasm/\">bench</a> |\n                <a href=\"stream.wasm\">stream</a> |\n                <a href=\"command.wasm/\">command</a> |\n                <a href=\"wchess.wasm/\">wchess</a> |\n\n            <hr>\n\n            <div id=\"model\">\n                Whisper models: <span id=\"model-whisper-status\"></span><br><br>\n                <button id=\"fetch-whisper-tiny-en\"  onclick=\"loadWhisper('tiny.en')\">tiny.en (75 MB)</button>\n                <button id=\"fetch-whisper-tiny\"     onclick=\"loadWhisper('tiny')\">tiny (75 MB)</button>\n                <button id=\"fetch-whisper-base-en\"  onclick=\"loadWhisper('base.en')\">base.en (142 MB)</button>\n                <button id=\"fetch-whisper-base\"     onclick=\"loadWhisper('base')\">base (142 MB)</button>\n                <button id=\"fetch-whisper-small-en\" onclick=\"loadWhisper('small.en')\">small.en (466 MB)</button>\n                <button id=\"fetch-whisper-small\"    onclick=\"loadWhisper('small')\">small (466 MB)</button>\n                <input type=\"file\" id=\"whisper-file\" name=\"file\" onchange=\"loadFile(event, 'whisper.bin')\" />\n                <br><br>\n                Quantized models:<br><br>\n                <button id=\"fetch-whisper-tiny-en-q5_1\"   onclick=\"loadWhisper('tiny-en-q5_1')\">tiny.en (Q5_1, 31 MB)</button>\n                <button id=\"fetch-whisper-tiny-q5_1\"      onclick=\"loadWhisper('tiny-q5_1')\">tiny (Q5_1, 31 MB)</button>\n                <button id=\"fetch-whisper-base-en-q5_1\"   onclick=\"loadWhisper('base-en-q5_1')\">base.en (Q5_1, 57 MB)</button>\n                <button id=\"fetch-whisper-base-q5_1\"      onclick=\"loadWhisper('base-q5_1')\">base (Q5_1, 57 MB)</button>\n                <button id=\"fetch-whisper-small-en-q5_1\"  onclick=\"loadWhisper('small-en-q5_1')\">small.en (Q5_1, 182 MB)</button>\n                <button id=\"fetch-whisper-small-q5_1\"     onclick=\"loadWhisper('small-q5_1')\">small (Q5_1, 182 MB)</button><br>\n                <button id=\"fetch-whisper-medium-en-q5_0\" onclick=\"loadWhisper('medium-en-q5_0')\">medium.en (Q5_0, 515 MB)</button>\n                <button id=\"fetch-whisper-medium-q5_0\"    onclick=\"loadWhisper('medium-q5_0')\">medium (Q5_0, 515 MB)</button>\n                <button id=\"fetch-whisper-large-q5_0\"     onclick=\"loadWhisper('large-q5_0')\">large (Q5_0, 1030 MB)</button>\n                <span id=\"fetch-whisper-progress\"></span>\n            </div>\n\n            <br>\n\n            <!-- radio button to select between file upload or microphone -->\n            <div id=\"input\">\n                Input:\n                <input type=\"radio\" id=\"file\" name=\"input\" value=\"file\" checked=\"checked\" onchange=\"changeInput('file')\" /> <label for=\"file\">File</label>\n                <input type=\"radio\" id=\"mic\" name=\"input\" value=\"mic\" onchange=\"changeInput('mic')\" /> <label for=\"mic\">Microphone</label>\n            </div>\n\n            <br>\n\n            <div id=\"input_file\">\n                Audio file:\n                <input type=\"file\" id=\"file\" name=\"file\" onchange=\"loadAudio(event)\" />\n            </div>\n\n            <div id=\"input_mic\" style=\"display: none;\">\n                Microphone:\n                <button id=\"start\" onclick=\"startRecording()\">Start</button>\n                <button id=\"stop\" onclick=\"stopRecording()\" disabled>Stop</button>\n\n                <!-- progress bar to show recording progress -->\n                <br><br>\n                <div id=\"progress\" style=\"display: none;\">\n                    <div id=\"progress-bar\" style=\"width: 0%; height: 10px; background-color: #4CAF50;\"></div>\n                    <div id=\"progress-text\">0%</div>\n                </div>\n            </div>\n\n            <audio controls=\"controls\" id=\"audio\" loop hidden>\n                Your browser does not support the &lt;audio&gt; tag.\n                <source id=\"source\" src=\"\" type=\"audio/wav\" />\n            </audio>\n\n            <hr><br>\n\n            <table>\n                <tr>\n                    <td>\n                        Language:\n                        <select id=\"language\" name=\"language\">\n                            <option value=\"en\">English</option>\n                            <option value=\"ar\">Arabic</option>\n                            <option value=\"hy\">Armenian</option>\n                            <option value=\"az\">Azerbaijani</option>\n                            <option value=\"eu\">Basque</option>\n                            <option value=\"be\">Belarusian</option>\n                            <option value=\"bn\">Bengali</option>\n                            <option value=\"bg\">Bulgarian</option>\n                            <option value=\"ca\">Catalan</option>\n                            <option value=\"zh\">Chinese</option>\n                            <option value=\"hr\">Croatian</option>\n                            <option value=\"cs\">Czech</option>\n                            <option value=\"da\">Danish</option>\n                            <option value=\"nl\">Dutch</option>\n                            <option value=\"en\">English</option>\n                            <option value=\"et\">Estonian</option>\n                            <option value=\"tl\">Filipino</option>\n                            <option value=\"fi\">Finnish</option>\n                            <option value=\"fr\">French</option>\n                            <option value=\"gl\">Galician</option>\n                            <option value=\"ka\">Georgian</option>\n                            <option value=\"de\">German</option>\n                            <option value=\"el\">Greek</option>\n                            <option value=\"gu\">Gujarati</option>\n                            <option value=\"he\">Hebrew</option>\n                            <option value=\"hi\">Hindi</option>\n                            <option value=\"hu\">Hungarian</option>\n                            <option value=\"is\">Icelandic</option>\n                            <option value=\"id\">Indonesian</option>\n                            <option value=\"ga\">Irish</option>\n                            <option value=\"it\">Italian</option>\n                            <option value=\"ja\">Japanese</option>\n                            <option value=\"kn\">Kannada</option>\n                            <option value=\"ko\">Korean</option>\n                            <option value=\"la\">Latin</option>\n                            <option value=\"lv\">Latvian</option>\n                            <option value=\"lt\">Lithuanian</option>\n                            <option value=\"mk\">Macedonian</option>\n                            <option value=\"ms\">Malay</option>\n                            <option value=\"mt\">Maltese</option>\n                            <option value=\"no\">Norwegian</option>\n                            <option value=\"fa\">Persian</option>\n                            <option value=\"pl\">Polish</option>\n                            <option value=\"pt\">Portuguese</option>\n                            <option value=\"ro\">Romanian</option>\n                            <option value=\"ru\">Russian</option>\n                            <option value=\"sr\">Serbian</option>\n                            <option value=\"sk\">Slovak</option>\n                            <option value=\"sl\">Slovenian</option>\n                            <option value=\"es\">Spanish</option>\n                            <option value=\"sw\">Swahili</option>\n                            <option value=\"sv\">Swedish</option>\n                            <option value=\"ta\">Tamil</option>\n                            <option value=\"te\">Telugu</option>\n                            <option value=\"th\">Thai</option>\n                            <option value=\"tr\">Turkish</option>\n                            <option value=\"uk\">Ukrainian</option>\n                            <option value=\"ur\">Urdu</option>\n                            <option value=\"vi\">Vietnamese</option>\n                            <option value=\"cy\">Welsh</option>\n                            <option value=\"yi\">Yiddish</option>\n                        </select>\n                    </td>\n                    <!-- Slider to select number of threads between 1 and 16 -->\n                    <td>\n                        Threads:\n                        <input type=\"range\" id=\"threads\" name=\"threads\" min=\"1\" max=\"16\" value=\"8\" onchange=\"changeThreads(this.value)\" />\n                        <span id=\"threads-value\">8</span>\n                    </td>\n                    <td>\n                        <button onclick=\"onProcess(false);\">Transcribe</button>\n                    </td>\n                    <td>\n                        <button onclick=\"onProcess(true);\">Translate</button>\n                    </td>\n                </tr>\n            </table>\n\n            <br>\n\n            <!-- textarea with height filling the rest of the page -->\n            <textarea id=\"output\" rows=\"20\"></textarea>\n\n            <br><br>\n\n            <div class=\"cell-version\">\n                <span>\n                    |\n                    Build time: <span class=\"nav-link\">@GIT_DATE@</span> |\n                    Commit hash: <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/commit/@GIT_SHA1@\">@GIT_SHA1@</a> |\n                    Commit subject: <span class=\"nav-link\">@GIT_COMMIT_SUBJECT@</span> |\n                    <a class=\"nav-link\" href=\"https://github.com/ggerganov/whisper.cpp/tree/master/examples/whisper.wasm\">Source Code</a> |\n                </span>\n            </div>\n        </div>\n\n        <script type=\"text/javascript\" src=\"helpers.js\"></script>\n        <script type='text/javascript'>\n            // TODO: convert audio buffer to WAV\n            function setAudio(audio) {\n                //if (audio) {\n                //    // convert to 16-bit PCM\n                //    var blob = new Blob([audio], { type: 'audio/wav' });\n                //    var url = URL.createObjectURL(blob);\n                //    document.getElementById('source').src = url;\n                //    document.getElementById('audio').hidden = false;\n                //    document.getElementById('audio').loop = false;\n                //    document.getElementById('audio').load();\n                //} else {\n                //    document.getElementById('audio').hidden = true;\n                //}\n            }\n\n            function changeInput(input) {\n                if (input == 'file') {\n                    document.getElementById('input_file').style.display = 'block';\n                    document.getElementById('input_mic' ).style.display = 'none';\n                    document.getElementById('progress'  ).style.display = 'none';\n                } else {\n                    document.getElementById('input_file').style.display = 'none';\n                    document.getElementById('input_mic' ).style.display = 'block';\n                    document.getElementById('progress'  ).style.display = 'block';\n                }\n            }\n\n            var Module = {\n                print: printTextarea,\n                printErr: printTextarea,\n                setStatus: function(text) {\n                    printTextarea('js: ' + text);\n                },\n                monitorRunDependencies: function(left) {\n                }\n            };\n\n            // web audio context\n            var context = null;\n\n            // audio data\n            var audio = null;\n\n            // the whisper instance\n            var instance = null;\n            var model_whisper = '';\n\n            // helper function\n            function convertTypedArray(src, type) {\n                var buffer = new ArrayBuffer(src.byteLength);\n                var baseView = new src.constructor(buffer).set(src);\n                return new type(buffer);\n            }\n\n            //\n            // load model\n            //\n\n            let dbVersion = 1\n            let dbName    = 'whisper.ggerganov.com';\n            let indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB\n\n            function storeFS(fname, buf) {\n                // write to WASM file using FS_createDataFile\n                // if the file exists, delete it\n                try {\n                    Module.FS_unlink(fname);\n                } catch (e) {\n                    // ignore\n                }\n\n                Module.FS_createDataFile(\"/\", fname, buf, true, true);\n\n                //model_whisper = fname;\n\n                document.getElementById('model-whisper-status').innerHTML = 'loaded \"' + model_whisper + '\"!';\n\n                printTextarea('storeFS: stored model: ' + fname + ' size: ' + buf.length);\n\n                document.getElementById('model').innerHTML = 'Model fetched: ' + model_whisper;\n            }\n\n            function loadFile(event, fname) {\n                var file = event.target.files[0] || null;\n                if (file == null) {\n                    return;\n                }\n\n                printTextarea(\"loadFile: loading model: \" + file.name + \", size: \" + file.size + \" bytes\");\n                printTextarea('loadFile: please wait ...');\n\n                var reader = new FileReader();\n                reader.onload = function(event) {\n                    var buf = new Uint8Array(reader.result);\n                    storeFS(fname, buf);\n                }\n                reader.readAsArrayBuffer(file);\n\n                document.getElementById('fetch-whisper-tiny-en' ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en' ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en').style.display = 'none';\n                document.getElementById('fetch-whisper-tiny'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-base'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-small'   ).style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-tiny-q5_1'     ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-q5_1'     ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-q5_1'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';\n                document.getElementById('fetch-whisper-medium-q5_0'   ).style.display = 'none';\n                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';\n\n                document.getElementById('whisper-file'          ).style.display = 'none';\n                document.getElementById('model-whisper-status'  ).innerHTML = 'loaded model: ' + file.name;\n            }\n\n            function loadWhisper(model) {\n                let urls = {\n                    'tiny.en':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin',\n                    'tiny':     'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin',\n                    'base.en':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin',\n                    'base':     'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin',\n                    'small.en': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin',\n                    'small':    'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin',\n\n                    'tiny-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en-q5_1.bin',\n                    'tiny-q5_1':     'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny-q5_1.bin',\n                    'base-en-q5_1':  'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en-q5_1.bin',\n                    'base-q5_1':     'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin',\n                    'small-en-q5_1': 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en-q5_1.bin',\n                    'small-q5_1':    'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q5_1.bin',\n                    'medium-en-q5_0':'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.en-q5_0.bin',\n                    'medium-q5_0':   'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium-q5_0.bin',\n                    'large-q5_0':    'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-q5_0.bin',\n                };\n\n                let sizes = {\n                    'tiny.en':  75,\n                    'tiny':     75,\n                    'base.en':  142,\n                    'base':     142,\n                    'small.en': 466,\n                    'small':    466,\n\n                    'tiny-en-q5_1':   31,\n                    'tiny-q5_1':      31,\n                    'base-en-q5_1':   57,\n                    'base-q5_1':      57,\n                    'small-en-q5_1':  182,\n                    'small-q5_1':     182,\n                    'medium-en-q5_0': 515,\n                    'medium-q5_0':    515,\n                    'large-q5_0':     1030,\n                };\n\n                let url     = urls[model];\n                let dst     = 'whisper.bin';\n                let size_mb = sizes[model];\n\n                model_whisper = model;\n\n                document.getElementById('fetch-whisper-tiny-en' ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en' ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en').style.display = 'none';\n                document.getElementById('fetch-whisper-tiny'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-base'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-small'   ).style.display = 'none';\n\n                document.getElementById('fetch-whisper-tiny-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-tiny-q5_1'     ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-en-q5_1'  ).style.display = 'none';\n                document.getElementById('fetch-whisper-base-q5_1'     ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-en-q5_1' ).style.display = 'none';\n                document.getElementById('fetch-whisper-small-q5_1'    ).style.display = 'none';\n                document.getElementById('fetch-whisper-medium-en-q5_0').style.display = 'none';\n                document.getElementById('fetch-whisper-medium-q5_0'   ).style.display = 'none';\n                document.getElementById('fetch-whisper-large-q5_0'    ).style.display = 'none';\n\n                document.getElementById('whisper-file'        ).style.display = 'none';\n                document.getElementById('model-whisper-status').innerHTML = 'loading model: ' + model;\n\n                cbProgress = function(p) {\n                    let el = document.getElementById('fetch-whisper-progress');\n                    el.innerHTML = Math.round(100*p) + '%';\n                };\n\n                cbCancel = function() {\n                    var el;\n\n                    el = document.getElementById('fetch-whisper-tiny-en' ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en' ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small-en'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-tiny'    ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base'    ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small'   ); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('fetch-whisper-tiny-en-q5_1'  ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-tiny-q5_1'     ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-en-q5_1'  ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-base-q5_1'     ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small-en-q5_1' ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-small-q5_1'    ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-medium-en-q5_0'); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-medium-q5_0'   ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('fetch-whisper-large-q5_0'    ); if (el) el.style.display = 'inline-block';\n\n                    el = document.getElementById('whisper-file'        ); if (el) el.style.display = 'inline-block';\n                    el = document.getElementById('model-whisper-status'); if (el) el.innerHTML = '';\n                };\n\n                loadRemote(url, dst, size_mb, cbProgress, storeFS, cbCancel, printTextarea);\n            }\n\n            //\n            // audio file\n            //\n\n            const kMaxAudio_s = 30*60;\n            const kMaxRecording_s = 2*60;\n            const kSampleRate = 16000;\n\n            window.AudioContext = window.AudioContext || window.webkitAudioContext;\n            window.OfflineAudioContext = window.OfflineAudioContext || window.webkitOfflineAudioContext;\n\n            function loadAudio(event) {\n                if (!context) {\n                    context = new AudioContext({\n                        sampleRate: kSampleRate,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                }\n\n                var file = event.target.files[0] || null;\n                if (file == null) {\n                    return;\n                }\n\n                printTextarea('js: loading audio: ' + file.name + ', size: ' + file.size + ' bytes');\n                printTextarea('js: please wait ...');\n\n                var reader = new FileReader();\n                reader.onload = function(event) {\n                    var buf = new Uint8Array(reader.result);\n\n                    context.decodeAudioData(buf.buffer, function(audioBuffer) {\n                        var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);\n                        var source = offlineContext.createBufferSource();\n                        source.buffer = audioBuffer;\n                        source.connect(offlineContext.destination);\n                        source.start(0);\n\n                        offlineContext.startRendering().then(function(renderedBuffer) {\n                            audio = renderedBuffer.getChannelData(0);\n                            printTextarea('js: audio loaded, size: ' + audio.length);\n\n                            // truncate to first 30 seconds\n                            if (audio.length > kMaxAudio_s*kSampleRate) {\n                                audio = audio.slice(0, kMaxAudio_s*kSampleRate);\n                                printTextarea('js: truncated audio to first ' + kMaxAudio_s + ' seconds');\n                            }\n\n                            setAudio(audio);\n                        });\n                    }, function(e) {\n                        printTextarea('js: error decoding audio: ' + e);\n                        audio = null;\n                        setAudio(audio);\n                    });\n                }\n                reader.readAsArrayBuffer(file);\n            }\n\n            //\n            // microphone\n            //\n\n            var mediaRecorder = null;\n            var doRecording = false;\n            var startTime = 0;\n\n            function stopRecording() {\n                doRecording = false;\n            }\n\n            // record up to kMaxRecording_s seconds of audio from the microphone\n            // check if doRecording is false every 1000 ms and stop recording if so\n            // update progress information\n            function startRecording() {\n                if (!context) {\n                    context = new AudioContext({\n                        sampleRate: kSampleRate,\n                        channelCount: 1,\n                        echoCancellation: false,\n                        autoGainControl:  true,\n                        noiseSuppression: true,\n                    });\n                }\n\n                document.getElementById('start').disabled = true;\n                document.getElementById('stop').disabled = false;\n\n                document.getElementById('progress-bar').style.width = '0%';\n                document.getElementById('progress-text').innerHTML = '0%';\n\n                doRecording = true;\n                startTime = Date.now();\n\n                var chunks = [];\n                var stream = null;\n\n                navigator.mediaDevices.getUserMedia({audio: true, video: false})\n                    .then(function(s) {\n                        stream = s;\n                        mediaRecorder = new MediaRecorder(stream);\n                        mediaRecorder.ondataavailable = function(e) {\n                            chunks.push(e.data);\n                        };\n                        mediaRecorder.onstop = function(e) {\n                            var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });\n                            chunks = [];\n\n                            document.getElementById('start').disabled = false;\n                            document.getElementById('stop').disabled = true;\n\n                            var reader = new FileReader();\n                            reader.onload = function(event) {\n                                var buf = new Uint8Array(reader.result);\n\n                                context.decodeAudioData(buf.buffer, function(audioBuffer) {\n                                    var offlineContext = new OfflineAudioContext(audioBuffer.numberOfChannels, audioBuffer.length, audioBuffer.sampleRate);\n                                    var source = offlineContext.createBufferSource();\n                                    source.buffer = audioBuffer;\n                                    source.connect(offlineContext.destination);\n                                    source.start(0);\n\n                                    offlineContext.startRendering().then(function(renderedBuffer) {\n                                        audio = renderedBuffer.getChannelData(0);\n                                        printTextarea('js: audio recorded, size: ' + audio.length);\n\n                                        // truncate to first 30 seconds\n                                        if (audio.length > kMaxRecording_s*kSampleRate) {\n                                            audio = audio.slice(0, kMaxRecording_s*kSampleRate);\n                                            printTextarea('js: truncated audio to first ' + kMaxRecording_s + ' seconds');\n                                        }\n                                        setAudio(audio);\n                                    });\n                                }, function(e) {\n                                    printTextarea('js: error decoding audio: ' + e);\n                                    audio = null;\n                                    setAudio(audio);\n                                });\n                            }\n\n                            reader.readAsArrayBuffer(blob);\n                        };\n                        mediaRecorder.start();\n                    })\n                    .catch(function(err) {\n                        printTextarea('js: error getting audio stream: ' + err);\n                    });\n\n                var interval = setInterval(function() {\n                    if (!doRecording) {\n                        clearInterval(interval);\n                        mediaRecorder.stop();\n                        stream.getTracks().forEach(function(track) {\n                            track.stop();\n                        });\n                    }\n\n                    document.getElementById('progress-bar').style.width = (100*(Date.now() - startTime)/1000/kMaxRecording_s) + '%';\n                    document.getElementById('progress-text').innerHTML = (100*(Date.now() - startTime)/1000/kMaxRecording_s).toFixed(0) + '%';\n                }, 1000);\n\n                printTextarea('js: recording ...');\n\n                setTimeout(function() {\n                    if (doRecording) {\n                        printTextarea('js: recording stopped after ' + kMaxRecording_s + ' seconds');\n                        stopRecording();\n                    }\n                }, kMaxRecording_s*1000);\n            }\n\n            //\n            // transcribe\n            //\n\n            var nthreads = 8;\n\n            function changeThreads(value) {\n                nthreads = parseInt(value, 10);\n                document.getElementById('threads-value').innerHTML = nthreads;\n            }\n\n            function onProcess(translate) {\n                if (!instance) {\n                    instance = Module.init('whisper.bin');\n\n                    if (instance) {\n                        printTextarea(\"js: whisper initialized, instance: \" + instance);\n                        document.getElementById('model').innerHTML = 'Model loaded: ' + model_whisper;\n                    }\n                }\n\n                if (!instance) {\n                    printTextarea(\"js: failed to initialize whisper\");\n                    return;\n                }\n\n                if (!audio) {\n                    printTextarea(\"js: no audio data\");\n                    return;\n                }\n\n                if (instance) {\n                    printTextarea('');\n                    printTextarea('js: processing - this might take a while ...');\n                    printTextarea('');\n\n                    setTimeout(function() {\n                        var ret = Module.full_default(instance, audio, document.getElementById('language').value, nthreads, translate);\n                        console.log('js: full_default returned: ' + ret);\n                        if (ret) {\n                            printTextarea(\"js: whisper returned: \" + ret);\n                        }\n                    }, 100);\n                }\n            }\n        </script>\n        <script type=\"text/javascript\" src=\"main.js\"></script>\n    </body>\n</html>\n"
  },
  {
    "path": "examples/yt-wsp.sh",
    "content": "#!/usr/bin/env bash\n# shellcheck disable=2086\n\n# MIT License\n\n# Copyright (c) 2022 Daniils Petrovs\n# Copyright (c) 2023 Jennifer Capasso\n\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n\n# The above copyright notice and this permission notice shall be included in all\n# copies or substantial portions of the Software.\n\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\n# Small shell script to more easily automatically download and transcribe live stream VODs.\n# This uses YT-DLP, ffmpeg and the CPP version of Whisper: https://github.com/ggml-org/whisper.cpp\n# Use `./examples/yt-wsp.sh help` to print help info.\n#\n# Sample usage:\n#\n#   git clone https://github.com/ggml-org/whisper.cpp\n#   cd whisper.cpp\n#   make\n#   ./examples/yt-wsp.sh https://www.youtube.com/watch?v=1234567890\n#\n\nset -Eeuo pipefail\n\n# get script file location\nSCRIPT_PATH=\"$(realpath -e ${BASH_SOURCE[0]})\";\nSCRIPT_DIR=\"${SCRIPT_PATH%/*}\"\n\n################################################################################\n# Documentation on downloading models can be found in the whisper.cpp repo:\n# https://github.com/ggml-org/whisper.cpp/#usage\n#\n# note: unless a multilingual model is specified, WHISPER_LANG will be ignored\n# and the video will be transcribed as if the audio were in the English language\n################################################################################\nMODEL_PATH=\"${MODEL_PATH:-${SCRIPT_DIR}/../models/ggml-base.en.bin}\"\n\n################################################################################\n# Where to find the whisper.cpp executable.  default to the examples directory\n# which holds this script in source control\n################################################################################\nWHISPER_EXECUTABLE=\"${WHISPER_EXECUTABLE:-${SCRIPT_DIR}/../build/bin/whisper-cli}\";\n\n# Set to desired language to be translated into english\nWHISPER_LANG=\"${WHISPER_LANG:-en}\";\n\n# Default to 4 threads (this was most performant on my 2020 M1 MBP)\nWHISPER_THREAD_COUNT=\"${WHISPER_THREAD_COUNT:-4}\";\n\nmsg() {\n    echo >&2 -e \"${1-}\"\n}\n\ncleanup() {\n    local -r clean_me=\"${1}\";\n\n    if [ -d \"${clean_me}\" ]; then\n      msg \"Cleaning up...\";\n      rm -rf \"${clean_me}\";\n    else\n      msg \"'${clean_me}' does not appear to be a directory!\";\n      exit 1;\n    fi;\n}\n\nprint_help() {\n    cat << 'EOF'\nUsage:\n  MODEL_PATH=<model> \\\n  WHISPER_EXECUTABLE=<whisper-cli> \\\n  WHISPER_LANG=en \\\n  WHISPER_THREAD_COUNT=<int> \\\n  ./examples/yt-wsp.sh <video_url>\n\nDescription:\n  This script downloads a YouTube video, generates subtitles using Whisper, \n  and muxes them into an MP4 output file.\n\nOutput:\n  An MP4 file with embedded subtitles will be produced in the working directory.\n  The file will be named using the video title and ID.\n  Example: \n    Input:  https://youtu.be/VYJtb2YXae8\n    Output: Why_we_all_need_subtitles_now-VYJtb2YXae8-res.mp4\n\nRequirements:\n  - ffmpeg\n  - yt-dlp\n  - whisper.cpp\n\nEnvironment Variables:\n  MODEL_PATH            Path to the Whisper model (e.g., models/ggml-base.en.bin)\n  WHISPER_EXECUTABLE    Path to the Whisper CLI executable\n  WHISPER_LANG          Language code (e.g., 'en' for English)\n  WHISPER_THREAD_COUNT  Number of CPU threads to use\n\nTip:\n  The script has many configurable environment variables.\n  Review the source code to explore all options.\n\nEOF\n}\n\ncheck_requirements() {\n    if ! command -v ffmpeg &>/dev/null; then\n        echo \"ffmpeg is required: https://ffmpeg.org\";\n        exit 1\n    fi;\n\n    if ! command -v yt-dlp &>/dev/null; then\n        echo \"yt-dlp is required: https://github.com/yt-dlp/yt-dlp\";\n        exit 1;\n    fi;\n\n    if ! command -v \"${WHISPER_EXECUTABLE}\" &>/dev/null; then\n        echo \"The C++ implementation of Whisper is required: https://github.com/ggml-org/whisper.cpp\"\n        echo \"Sample usage:\";\n        echo \"\";\n        echo \"  git clone https://github.com/ggml-org/whisper.cpp\";\n        echo \"  cd whisper.cpp\";\n        echo \"  make\";\n        echo \"  ./examples/yt-wsp.sh https://www.youtube.com/watch?v=1234567890\";\n        echo \"\";\n        exit 1;\n    fi;\n\n}\n\nif [[ \"${#}\" -lt 1 ]]; then\n    print_help;\n    exit 1;\nfi\n\nif [[ \"${1##-*}\" == \"help\" ]]; then\n    print_help;\n    exit 0;\nfi\n\ncheck_requirements;\n\n################################################################################\n# create a temporary directory to work in\n# set the temp_dir and temp_filename variables\n################################################################################\ntemp_dir=\"$(mktemp -d ${SCRIPT_DIR}/tmp.XXXXXX)\";\ntemp_filename=\"${temp_dir}/yt-dlp-filename\";\n\n################################################################################\n# for now we only take one argument\n# TODO: a for loop\n################################################################################\nsource_url=\"${1}\"\ntitle_name=\"\";\n\nmsg \"Downloading VOD...\";\n\n################################################################################\n# Download the video, put the dynamic output filename into a variable.\n# Optionally add --cookies-from-browser BROWSER[+KEYRING][:PROFILE][::CONTAINER]\n# for videos only available to logged-in users.\n################################################################################\nyt-dlp \\\n    -f \"bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best\" \\\n    -o \"${temp_dir}/%(title)s-%(id)s.vod.mp4\" \\\n    --print-to-file \"%(filename)s\" \"${temp_filename}\" \\\n    --no-simulate \\\n    --no-write-auto-subs \\\n    --restrict-filenames \\\n    --embed-thumbnail \\\n    --embed-chapters \\\n    --xattrs \\\n    \"${source_url}\";\n\ntitle_name=\"$(xargs basename -s .vod.mp4 < ${temp_filename})\";\n\nmsg \"Extracting audio and resampling...\";\n\nffmpeg -i \"${temp_dir}/${title_name}.vod.mp4\"  \\\n    -hide_banner \\\n    -vn \\\n    -loglevel error \\\n    -ar 16000 \\\n    -ac 1 \\\n    -c:a pcm_s16le \\\n    -y \\\n    \"${temp_dir}/${title_name}.vod-resampled.wav\";\n\nmsg \"Transcribing to subtitle file...\";\nmsg \"Whisper specified at: '${WHISPER_EXECUTABLE}'\";\n\n\"${WHISPER_EXECUTABLE}\" \\\n    -m \"${MODEL_PATH}\" \\\n    -l \"${WHISPER_LANG}\" \\\n    -f \"${temp_dir}/${title_name}.vod-resampled.wav\" \\\n    -t \"${WHISPER_THREAD_COUNT}\" \\\n    -osrt \\\n    --translate;\n\nmsg \"Embedding subtitle track...\";\n\nffmpeg -i \"${temp_dir}/${title_name}.vod.mp4\" \\\n    -hide_banner \\\n    -loglevel error \\\n    -i \"${temp_dir}/${title_name}.vod-resampled.wav.srt\" \\\n    -c copy \\\n    -c:s mov_text \\\n    -y \"${title_name}-res.mp4\";\n\n#cleanup \"${temp_dir}\";\n\nmsg \"Done! Your finished file is ready: ${title_name}-res.mp4\";\n"
  },
  {
    "path": "ggml/.gitignore",
    "content": "src/ggml-metal-embed.metal\n"
  },
  {
    "path": "ggml/CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.\nproject(\"ggml\" C CXX ASM)\n\n### GGML Version\nset(GGML_VERSION_MAJOR 0)\nset(GGML_VERSION_MINOR 9)\nset(GGML_VERSION_PATCH 8)\nset(GGML_VERSION_BASE \"${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}\")\n\nfind_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)\nif(GIT_EXE)\n    # Get current git commit hash\n    execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n        OUTPUT_VARIABLE GGML_BUILD_COMMIT\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n        ERROR_QUIET\n    )\n\n    # Check if the working directory is dirty (i.e., has uncommitted changes)\n    execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- .\n        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}\n        RESULT_VARIABLE GGML_GIT_DIRTY\n        ERROR_QUIET\n    )\nendif()\n\nset(GGML_VERSION \"${GGML_VERSION_BASE}\")\n\nif(NOT GGML_BUILD_COMMIT)\n    set(GGML_BUILD_COMMIT \"unknown\")\nendif()\n\n# Build the commit string with optional dirty flag\nif(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1)\n    set(GGML_BUILD_COMMIT \"${GGML_BUILD_COMMIT}-dirty\")\nendif()\n\ninclude(CheckIncludeFileCXX)\n\nset(CMAKE_EXPORT_COMPILE_COMMANDS ON)\n\nif (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)\n    set(CMAKE_BUILD_TYPE Release CACHE STRING \"Build type\" FORCE)\n    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS \"Debug\" \"Release\" \"MinSizeRel\" \"RelWithDebInfo\")\nendif()\n\nif (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)\n    set(GGML_STANDALONE ON)\n\n    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n\n    # configure project version\n    # TODO\nelse()\n    set(GGML_STANDALONE OFF)\n\n    if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)\n        set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)\n    endif()\nendif()\n\nif (EMSCRIPTEN)\n    set(BUILD_SHARED_LIBS_DEFAULT OFF)\n\n    option(GGML_WASM_SINGLE_FILE \"ggml: embed WASM inside the generated ggml.js\" ON)\nelse()\n    if (MINGW)\n        set(BUILD_SHARED_LIBS_DEFAULT OFF)\n    else()\n        set(BUILD_SHARED_LIBS_DEFAULT ON)\n    endif()\nendif()\n\n# remove the lib prefix on win32 mingw\nif (WIN32)\n    set(CMAKE_STATIC_LIBRARY_PREFIX \"\")\n    set(CMAKE_SHARED_LIBRARY_PREFIX \"\")\n    set(CMAKE_SHARED_MODULE_PREFIX  \"\")\nendif()\n\noption(BUILD_SHARED_LIBS           \"ggml: build shared libraries\" ${BUILD_SHARED_LIBS_DEFAULT})\noption(GGML_BACKEND_DL             \"ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)\" OFF)\nset(GGML_BACKEND_DIR \"\" CACHE PATH \"ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL\")\n\n#\n# option list\n#\n\n# TODO: mark all options as advanced when not GGML_STANDALONE\n\nif (APPLE)\n    set(GGML_METAL_DEFAULT ON)\n    set(GGML_BLAS_DEFAULT ON)\n    set(GGML_BLAS_VENDOR_DEFAULT \"Apple\")\nelse()\n    set(GGML_METAL_DEFAULT OFF)\n    set(GGML_BLAS_DEFAULT OFF)\n    set(GGML_BLAS_VENDOR_DEFAULT \"Generic\")\nendif()\n\nif (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH})\n    message(STATUS \"Setting GGML_NATIVE_DEFAULT to OFF\")\n    set(GGML_NATIVE_DEFAULT OFF)\nelse()\n    set(GGML_NATIVE_DEFAULT ON)\nendif()\n\n# defaults\nif (NOT GGML_LLAMAFILE_DEFAULT)\n    set(GGML_LLAMAFILE_DEFAULT OFF)\nendif()\n\nif (NOT GGML_CUDA_GRAPHS_DEFAULT)\n    set(GGML_CUDA_GRAPHS_DEFAULT OFF)\nendif()\n\n# general\noption(GGML_STATIC \"ggml: static link libraries\"                     OFF)\noption(GGML_NATIVE \"ggml: optimize the build for the current system\" ${GGML_NATIVE_DEFAULT})\noption(GGML_LTO    \"ggml: enable link time optimization\"             OFF)\noption(GGML_CCACHE \"ggml: use ccache if available\"                   ON)\n\n# debug\noption(GGML_ALL_WARNINGS           \"ggml: enable all compiler warnings\"                   ON)\noption(GGML_ALL_WARNINGS_3RD_PARTY \"ggml: enable all compiler warnings in 3rd party libs\" OFF)\noption(GGML_GPROF                  \"ggml: enable gprof\"                                   OFF)\n\n# build\noption(GGML_FATAL_WARNINGS    \"ggml: enable -Werror flag\"    OFF)\n\n# sanitizers\noption(GGML_SANITIZE_THREAD    \"ggml: enable thread sanitizer\"    OFF)\noption(GGML_SANITIZE_ADDRESS   \"ggml: enable address sanitizer\"   OFF)\noption(GGML_SANITIZE_UNDEFINED \"ggml: enable undefined sanitizer\" OFF)\n\n# instruction set specific\nif (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)\n    set(INS_ENB OFF)\nelse()\n    set(INS_ENB ON)\nendif()\n\nmessage(DEBUG \"GGML_NATIVE         : ${GGML_NATIVE}\")\nmessage(DEBUG \"GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}\")\nmessage(DEBUG \"INS_ENB             : ${INS_ENB}\")\n\noption(GGML_CPU_HBM          \"ggml: use memkind for CPU HBM\" OFF)\noption(GGML_CPU_REPACK       \"ggml: use runtime weight conversion of Q4_0 to Q4_X_X\" ON)\noption(GGML_CPU_KLEIDIAI     \"ggml: use KleidiAI optimized kernels if applicable\" OFF)\noption(GGML_SSE42            \"ggml: enable SSE 4.2\"          ${INS_ENB})\noption(GGML_AVX              \"ggml: enable AVX\"              ${INS_ENB})\noption(GGML_AVX_VNNI         \"ggml: enable AVX-VNNI\"         OFF)\noption(GGML_AVX2             \"ggml: enable AVX2\"             ${INS_ENB})\noption(GGML_BMI2             \"ggml: enable BMI2\"             ${INS_ENB})\noption(GGML_AVX512           \"ggml: enable AVX512F\"          OFF)\noption(GGML_AVX512_VBMI      \"ggml: enable AVX512-VBMI\"      OFF)\noption(GGML_AVX512_VNNI      \"ggml: enable AVX512-VNNI\"      OFF)\noption(GGML_AVX512_BF16      \"ggml: enable AVX512-BF16\"      OFF)\nif (NOT MSVC)\n    # in MSVC F16C and FMA is implied with AVX2/AVX512\n    option(GGML_FMA          \"ggml: enable FMA\"              ${INS_ENB})\n    option(GGML_F16C         \"ggml: enable F16C\"             ${INS_ENB})\n    # MSVC does not seem to support AMX\n    option(GGML_AMX_TILE     \"ggml: enable AMX-TILE\"         OFF)\n    option(GGML_AMX_INT8     \"ggml: enable AMX-INT8\"         OFF)\n    option(GGML_AMX_BF16     \"ggml: enable AMX-BF16\"         OFF)\nendif()\noption(GGML_LASX             \"ggml: enable lasx\"             ON)\noption(GGML_LSX              \"ggml: enable lsx\"              ON)\noption(GGML_RVV              \"ggml: enable rvv\"              ON)\noption(GGML_RV_ZFH           \"ggml: enable riscv zfh\"        ON)\noption(GGML_RV_ZVFH          \"ggml: enable riscv zvfh\"       ON)\noption(GGML_RV_ZICBOP        \"ggml: enable riscv zicbop\"     ON)\noption(GGML_RV_ZIHINTPAUSE   \"ggml: enable riscv zihintpause \"  ON)\noption(GGML_XTHEADVECTOR     \"ggml: enable xtheadvector\"     OFF)\noption(GGML_VXE              \"ggml: enable vxe\"              ${GGML_NATIVE})\n\noption(GGML_CPU_ALL_VARIANTS \"ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)\" OFF)\nset(GGML_CPU_ARM_ARCH        \"\" CACHE STRING \"ggml: CPU architecture for ARM\")\nset(GGML_CPU_POWERPC_CPUTYPE \"\" CACHE STRING \"ggml: CPU type for PowerPC\")\n\n# ggml core\nset(GGML_SCHED_MAX_COPIES  \"4\" CACHE STRING \"ggml: max input copies for pipeline parallelism\")\noption(GGML_CPU                             \"ggml: enable CPU backend\"                        ON)\noption(GGML_SCHED_NO_REALLOC                \"ggml: disallow reallocations in ggml-alloc (for debugging)\" OFF)\n\n# 3rd party libs / backends\noption(GGML_ACCELERATE                      \"ggml: enable Accelerate framework\"               ON)\noption(GGML_BLAS                            \"ggml: use BLAS\"                                  ${GGML_BLAS_DEFAULT})\nset(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING\n                                            \"ggml: BLAS library vendor\")\noption(GGML_LLAMAFILE                       \"ggml: use LLAMAFILE\"                             ${GGML_LLAMAFILE_DEFAULT})\n\noption(GGML_CUDA                            \"ggml: use CUDA\"                                  OFF)\noption(GGML_MUSA                            \"ggml: use MUSA\"                                  OFF)\noption(GGML_CUDA_FORCE_MMQ                  \"ggml: use mmq kernels instead of cuBLAS\"         OFF)\noption(GGML_CUDA_FORCE_CUBLAS               \"ggml: always use cuBLAS instead of mmq kernels\"  OFF)\nset   (GGML_CUDA_PEER_MAX_BATCH_SIZE \"128\" CACHE STRING\n                                            \"ggml: max. batch size for using peer access\")\noption(GGML_CUDA_NO_PEER_COPY               \"ggml: do not use peer to peer copies\"            OFF)\noption(GGML_CUDA_NO_VMM                     \"ggml: do not try to use CUDA VMM\"                OFF)\noption(GGML_CUDA_FA                         \"ggml: compile ggml FlashAttention CUDA kernels\"  ON)\noption(GGML_CUDA_FA_ALL_QUANTS              \"ggml: compile all quants for FlashAttention\"     OFF)\noption(GGML_CUDA_GRAPHS                     \"ggml: use CUDA graphs (llama.cpp only)\"          ${GGML_CUDA_GRAPHS_DEFAULT})\nset   (GGML_CUDA_COMPRESSION_MODE \"size\" CACHE STRING\n                                            \"ggml: cuda link binary compression mode; requires cuda 12.8+\")\nset_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS \"none;speed;balance;size\")\n\noption(GGML_HIP                             \"ggml: use HIP\"                                   OFF)\noption(GGML_HIP_GRAPHS                      \"ggml: use HIP graph, experimental, slow\"         OFF)\noption(GGML_HIP_NO_VMM                      \"ggml: do not try to use HIP VMM\"                 ON)\noption(GGML_HIP_ROCWMMA_FATTN               \"ggml: enable rocWMMA for FlashAttention\"         OFF)\noption(GGML_HIP_MMQ_MFMA                    \"ggml: enable MFMA MMA for CDNA in MMQ\"           ON)\noption(GGML_HIP_EXPORT_METRICS              \"ggml: enable kernel perf metrics output\"         OFF)\noption(GGML_MUSA_GRAPHS                     \"ggml: use MUSA graph, experimental, unstable\"    OFF)\noption(GGML_MUSA_MUDNN_COPY                 \"ggml: enable muDNN for accelerated copy\"         OFF)\noption(GGML_VULKAN                          \"ggml: use Vulkan\"                                OFF)\noption(GGML_VULKAN_CHECK_RESULTS            \"ggml: run Vulkan op checks\"                      OFF)\noption(GGML_VULKAN_DEBUG                    \"ggml: enable Vulkan debug output\"                OFF)\noption(GGML_VULKAN_MEMORY_DEBUG             \"ggml: enable Vulkan memory debug output\"         OFF)\noption(GGML_VULKAN_SHADER_DEBUG_INFO        \"ggml: enable Vulkan shader debug info\"           OFF)\noption(GGML_VULKAN_VALIDATE                 \"ggml: enable Vulkan validation\"                  OFF)\noption(GGML_VULKAN_RUN_TESTS                \"ggml: run Vulkan tests\"                          OFF)\noption(GGML_WEBGPU                          \"ggml: use WebGPU\"                                OFF)\noption(GGML_WEBGPU_DEBUG                    \"ggml: enable WebGPU debug output\"                OFF)\noption(GGML_WEBGPU_CPU_PROFILE              \"ggml: enable WebGPU profiling (CPU)\"             OFF)\noption(GGML_WEBGPU_GPU_PROFILE              \"ggml: enable WebGPU profiling (GPU)\"             OFF)\noption(GGML_WEBGPU_JSPI                     \"ggml: use JSPI for WebGPU\"                       ON)\noption(GGML_ZDNN                            \"ggml: use zDNN\"                                  OFF)\noption(GGML_VIRTGPU                         \"ggml: use the VirtGPU/Virglrenderer API Remoting frontend\"     OFF)\noption(GGML_VIRTGPU_BACKEND                 \"ggml: build the VirtGPU/Virglrenderer API Remoting backend\"    OFF)\noption(GGML_METAL                           \"ggml: use Metal\"                                 ${GGML_METAL_DEFAULT})\noption(GGML_METAL_NDEBUG                    \"ggml: disable Metal debugging\"                   OFF)\noption(GGML_METAL_SHADER_DEBUG              \"ggml: compile Metal with -fno-fast-math\"         OFF)\noption(GGML_METAL_EMBED_LIBRARY             \"ggml: embed Metal library\"                       ${GGML_METAL})\nset   (GGML_METAL_MACOSX_VERSION_MIN \"\" CACHE STRING\n                                            \"ggml: metal minimum macOS version\")\nset   (GGML_METAL_STD \"\" CACHE STRING       \"ggml: metal standard version (-std flag)\")\noption(GGML_OPENMP                          \"ggml: use OpenMP\"                                ON)\noption(GGML_RPC                             \"ggml: use RPC\"                                   OFF)\noption(GGML_SYCL                            \"ggml: use SYCL\"                                  OFF)\noption(GGML_SYCL_F16                        \"ggml: use 16 bit floats for sycl calculations\"   OFF)\noption(GGML_SYCL_GRAPH                      \"ggml: enable graphs in the SYCL backend\"         ON)\noption(GGML_SYCL_DNN                        \"ggml: enable oneDNN in the SYCL backend\"         ON)\nset   (GGML_SYCL_TARGET \"INTEL\" CACHE STRING\n                                            \"ggml: sycl target device\")\nset   (GGML_SYCL_DEVICE_ARCH \"\" CACHE STRING\n                                            \"ggml: sycl device architecture\")\n\noption(GGML_OPENVINO                        \"ggml: use OPENVINO\"                              OFF)\n\noption(GGML_OPENCL                          \"ggml: use OpenCL\"                                OFF)\noption(GGML_OPENCL_PROFILING                \"ggml: use OpenCL profiling (increases overhead)\" OFF)\noption(GGML_OPENCL_EMBED_KERNELS            \"ggml: embed kernels\"                             ON)\noption(GGML_OPENCL_USE_ADRENO_KERNELS       \"ggml: use optimized kernels for Adreno\"          ON)\nset   (GGML_OPENCL_TARGET_VERSION \"300\" CACHE STRING\n                                            \"ggml: OpenCL API version to target\")\n\noption(GGML_HEXAGON                         \"ggml: enable Hexagon backend\"                    OFF)\nset(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING \"ggml: quantize group size (32, 64, or 128)\")\n\n# toolchain for vulkan-shaders-gen\nset   (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN \"\" CACHE FILEPATH \"ggml: toolchain file for vulkan-shaders-gen\")\n\noption(GGML_ZENDNN                          \"ggml: use ZenDNN\"                                OFF)\noption(ZENDNN_ROOT                          \"ggml: path to ZenDNN installation\"               \"\")\n\n# extra artifacts\noption(GGML_BUILD_TESTS    \"ggml: build tests\"    ${GGML_STANDALONE})\noption(GGML_BUILD_EXAMPLES \"ggml: build examples\" ${GGML_STANDALONE})\n\n#\n# dependencies\n#\n\nset(CMAKE_C_STANDARD 11)\nset(CMAKE_C_STANDARD_REQUIRED true)\n\nset(CMAKE_CXX_STANDARD 17)\nset(CMAKE_CXX_STANDARD_REQUIRED true)\n\nset(THREADS_PREFER_PTHREAD_FLAG ON)\n\nfind_package(Threads REQUIRED)\n\ninclude(GNUInstallDirs)\n\n#\n# build the library\n#\n\nadd_subdirectory(src)\n\n#\n# tests and examples\n#\n\nif (GGML_BUILD_TESTS)\n    enable_testing()\n    add_subdirectory(tests)\nendif ()\n\nif (GGML_BUILD_EXAMPLES)\n    add_subdirectory(examples)\nendif ()\n\n#\n# install\n#\n\ninclude(CMakePackageConfigHelpers)\n\n# all public headers\nset(GGML_PUBLIC_HEADERS\n    include/ggml.h\n    include/ggml-cpu.h\n    include/ggml-alloc.h\n    include/ggml-backend.h\n    include/ggml-blas.h\n    include/ggml-cann.h\n    include/ggml-cpp.h\n    include/ggml-cuda.h\n    include/ggml-opt.h\n    include/ggml-metal.h\n    include/ggml-rpc.h\n    include/ggml-virtgpu.h\n    include/ggml-sycl.h\n    include/ggml-vulkan.h\n    include/ggml-webgpu.h\n    include/ggml-zendnn.h\n    include/ggml-openvino.h\n    include/gguf.h)\n\nset_target_properties(ggml PROPERTIES PUBLIC_HEADER \"${GGML_PUBLIC_HEADERS}\")\n#if (GGML_METAL)\n#    set_target_properties(ggml PROPERTIES RESOURCE \"${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal\")\n#endif()\ninstall(TARGETS ggml LIBRARY PUBLIC_HEADER)\ninstall(TARGETS ggml-base LIBRARY)\n\nif (GGML_STANDALONE)\n    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in\n        ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc\n        @ONLY)\n\n    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc\n        DESTINATION share/pkgconfig)\nendif()\n\n#\n# Create CMake package\n#\n\n\n\n# Capture variables prefixed with GGML_.\n\nset(variable_set_statements\n\"\n####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() #######\n####### Any changes to this file will be overwritten by the next CMake run        #######\n\n\")\n\nset(GGML_SHARED_LIB ${BUILD_SHARED_LIBS})\n\nget_cmake_property(all_variables VARIABLES)\nforeach(variable_name IN LISTS all_variables)\n    if(variable_name MATCHES \"^GGML_\")\n        string(REPLACE \";\" \"\\\\;\"\n               variable_value \"${${variable_name}}\")\n\n        set(variable_set_statements\n            \"${variable_set_statements}set(${variable_name} \\\"${variable_value}\\\")\\n\")\n    endif()\nendforeach()\n\nset(GGML_VARIABLES_EXPANDED ${variable_set_statements})\n\n# Create the CMake package and set install location.\n\nset(GGML_INSTALL_VERSION ${GGML_VERSION})\nset(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH \"Location of header  files\")\nset(GGML_LIB_INSTALL_DIR     ${CMAKE_INSTALL_LIBDIR}     CACHE PATH \"Location of library files\")\nset(GGML_BIN_INSTALL_DIR     ${CMAKE_INSTALL_BINDIR}     CACHE PATH \"Location of binary  files\")\n\nconfigure_package_config_file(\n        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in\n        ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake\n    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml\n    PATH_VARS GGML_INCLUDE_INSTALL_DIR\n              GGML_LIB_INSTALL_DIR\n              GGML_BIN_INSTALL_DIR)\n\nwrite_basic_package_version_file(\n        ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake\n    VERSION ${GGML_INSTALL_VERSION}\n    COMPATIBILITY SameMajorVersion)\n\ntarget_compile_definitions(ggml-base PRIVATE\n    GGML_VERSION=\"${GGML_INSTALL_VERSION}\"\n    GGML_COMMIT=\"${GGML_BUILD_COMMIT}\"\n)\nmessage(STATUS \"ggml version: ${GGML_INSTALL_VERSION}\")\nmessage(STATUS \"ggml commit:  ${GGML_BUILD_COMMIT}\")\n\ninstall(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake\n              ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake\n        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)\n\nif (MSVC)\n    set(MSVC_WARNING_FLAGS\n        /wd4005  # Macro redefinition\n        /wd4244  # Conversion from one type to another type, possible loss of data\n        /wd4267  # Conversion from 'size_t' to a smaller type, possible loss of data\n        /wd4305  # Conversion from 'type1' to 'type2', possible loss of data\n        /wd4566  # Conversion from 'char' to 'wchar_t', possible loss of data\n        /wd4996  # Disable POSIX deprecation warnings\n        /wd4702  # Unreachable code warnings\n    )\n    set(MSVC_COMPILE_OPTIONS\n        \"$<$<COMPILE_LANGUAGE:C>:/utf-8>\"\n        \"$<$<COMPILE_LANGUAGE:CXX>:/utf-8>\"\n    )\n    function(configure_msvc_target target_name)\n        if(TARGET ${target_name})\n            target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})\n            target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS})\n        endif()\n    endfunction()\n\n    configure_msvc_target(ggml-base)\n    configure_msvc_target(ggml)\n    configure_msvc_target(ggml-cpu)\n    configure_msvc_target(ggml-cpu-x64)\n    configure_msvc_target(ggml-cpu-sse42)\n    configure_msvc_target(ggml-cpu-sandybridge)\n    # __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n    # skipping            ggml-cpu-ivybridge\n    # skipping            ggml-cpu-piledriver\n    configure_msvc_target(ggml-cpu-haswell)\n    configure_msvc_target(ggml-cpu-skylakex)\n    configure_msvc_target(ggml-cpu-cannonlake)\n    configure_msvc_target(ggml-cpu-cascadelake)\n    configure_msvc_target(ggml-cpu-icelake)\n    # MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!\n    # https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170\n    # https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170\n    # skipping            ggml-cpu-cooperlake\n    # skipping            ggml-cpu-zen4\n    configure_msvc_target(ggml-cpu-alderlake)\n    # MSVC doesn't support AMX\n    # skipping            ggml-cpu-sapphirerapids\n\n    if (GGML_BUILD_EXAMPLES)\n        configure_msvc_target(common-ggml)\n        configure_msvc_target(common)\n\n        configure_msvc_target(mnist-common)\n        configure_msvc_target(mnist-eval)\n        configure_msvc_target(mnist-train)\n\n        configure_msvc_target(gpt-2-ctx)\n        configure_msvc_target(gpt-2-alloc)\n        configure_msvc_target(gpt-2-backend)\n        configure_msvc_target(gpt-2-sched)\n        configure_msvc_target(gpt-2-quantize)\n        configure_msvc_target(gpt-2-batched)\n\n        configure_msvc_target(gpt-j)\n        configure_msvc_target(gpt-j-quantize)\n\n        configure_msvc_target(magika)\n        configure_msvc_target(yolov3-tiny)\n        configure_msvc_target(sam)\n\n        configure_msvc_target(simple-ctx)\n        configure_msvc_target(simple-backend)\n    endif()\n\n    if (GGML_BUILD_TESTS)\n        configure_msvc_target(test-mul-mat)\n        configure_msvc_target(test-arange)\n        configure_msvc_target(test-backend-ops)\n        configure_msvc_target(test-cont)\n        configure_msvc_target(test-conv-transpose)\n        configure_msvc_target(test-conv-transpose-1d)\n        configure_msvc_target(test-conv1d)\n        configure_msvc_target(test-conv2d)\n        configure_msvc_target(test-conv2d-dw)\n        configure_msvc_target(test-customop)\n        configure_msvc_target(test-dup)\n        configure_msvc_target(test-opt)\n        configure_msvc_target(test-pool)\n    endif ()\nendif()\n"
  },
  {
    "path": "ggml/cmake/GitVars.cmake",
    "content": "find_package(Git)\n\n# the commit's SHA1\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" describe --match=NeVeRmAtCh --always --abbrev=8\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_SHA1\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the date of the commit\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" log -1 --format=%ad --date=local\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_DATE\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n\n# the subject of the commit\nexecute_process(COMMAND\n    \"${GIT_EXECUTABLE}\" log -1 --format=%s\n    WORKING_DIRECTORY \"${CMAKE_SOURCE_DIR}\"\n    OUTPUT_VARIABLE GIT_COMMIT_SUBJECT\n    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)\n"
  },
  {
    "path": "ggml/cmake/common.cmake",
    "content": "function(ggml_get_flags CCID CCVER)\n    set(C_FLAGS \"\")\n    set(CXX_FLAGS \"\")\n\n    if (CCID MATCHES \"Clang\")\n        set(C_FLAGS   -Wunreachable-code-break -Wunreachable-code-return)\n        set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)\n\n        if (\n            (CCID STREQUAL \"Clang\"      AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR\n            (CCID STREQUAL \"AppleClang\" AND CCVER VERSION_GREATER_EQUAL 7.3.0)\n        )\n            list(APPEND C_FLAGS -Wdouble-promotion)\n        endif()\n    elseif (CCID STREQUAL \"GNU\")\n        set(C_FLAGS   -Wdouble-promotion)\n        set(CXX_FLAGS -Wno-array-bounds)\n\n        if (CCVER VERSION_GREATER_EQUAL 8.1.0)\n            list(APPEND CXX_FLAGS -Wextra-semi)\n        endif()\n    endif()\n\n    set(GF_C_FLAGS   ${C_FLAGS}   PARENT_SCOPE)\n    set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)\nendfunction()\n\nfunction(ggml_get_system_arch)\n    if (CMAKE_OSX_ARCHITECTURES      STREQUAL \"arm64\" OR\n        CMAKE_GENERATOR_PLATFORM_LWR STREQUAL \"arm64\" OR\n        (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND\n            CMAKE_SYSTEM_PROCESSOR MATCHES \"^(aarch64|arm.*|ARM64)$\"))\n        set(GGML_SYSTEM_ARCH \"ARM\" PARENT_SCOPE)\n    elseif (CMAKE_OSX_ARCHITECTURES STREQUAL \"x86_64\" OR\n            CMAKE_GENERATOR_PLATFORM_LWR MATCHES \"^(x86_64|i686|amd64|x64|win32)$\" OR\n            (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND\n            CMAKE_SYSTEM_PROCESSOR MATCHES \"^(x86_64|i686|AMD64|amd64)$\"))\n        set(GGML_SYSTEM_ARCH \"x86\" PARENT_SCOPE)\n    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc|power\")\n        set(GGML_SYSTEM_ARCH \"PowerPC\" PARENT_SCOPE)\n    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"loongarch64\")\n        set(GGML_SYSTEM_ARCH \"loongarch64\"  PARENT_SCOPE)\n    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"riscv64\")\n        set(GGML_SYSTEM_ARCH \"riscv64\" PARENT_SCOPE)\n    elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"s390x\")\n        set(GGML_SYSTEM_ARCH \"s390x\" PARENT_SCOPE)\n    else()\n        set(GGML_SYSTEM_ARCH \"UNKNOWN\" PARENT_SCOPE)\n    endif()\nendfunction()\n"
  },
  {
    "path": "ggml/cmake/ggml-config.cmake.in",
    "content": "@PACKAGE_INIT@\n\n@GGML_VARIABLES_EXPANDED@\n\n# Find all dependencies before creating any target.\ninclude(CMakeFindDependencyMacro)\nfind_dependency(Threads)\nif (NOT GGML_SHARED_LIB)\n    set(GGML_CPU_INTERFACE_LINK_LIBRARIES \"\")\n    set(GGML_CPU_INTERFACE_LINK_OPTIONS   \"\")\n\n    if (APPLE AND GGML_ACCELERATE)\n        find_library(ACCELERATE_FRAMEWORK Accelerate)\n        if(NOT ACCELERATE_FRAMEWORK)\n            set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n            return()\n        endif()\n        list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK})\n    endif()\n\n    if (GGML_OPENMP_ENABLED)\n        find_dependency(OpenMP)\n        list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C OpenMP::OpenMP_CXX)\n    endif()\n\n    if (GGML_CPU_HBM)\n        find_library(memkind memkind)\n        if(NOT memkind)\n            set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n            return()\n        endif()\n        list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind)\n    endif()\n\n    if (GGML_BLAS)\n        find_dependency(BLAS)\n        list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES})\n        list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS   ${BLAS_LINKER_FLAGS})\n    endif()\n\n    if (GGML_CUDA)\n        set(GGML_CUDA_INTERFACE_LINK_LIBRARIES \"\")\n        find_dependency(CUDAToolkit)\n        if (GGML_STATIC)\n            list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cudart_static>)\n            if (WIN32)\n                list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas> $<LINK_ONLY:CUDA::cublasLt>)\n            else()\n                list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas_static> $<LINK_ONLY:CUDA::cublasLt_static>)\n            endif()\n        endif()\n        if (NOT GGML_CUDA_NO_VMM)\n            list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cuda_driver>)\n        endif()\n    endif()\n\n    if (GGML_METAL)\n        find_library(FOUNDATION_LIBRARY Foundation)\n        find_library(METAL_FRAMEWORK    Metal)\n        find_library(METALKIT_FRAMEWORK MetalKit)\n        if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK)\n            set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)\n            return()\n        endif()\n        set(GGML_METAL_INTERFACE_LINK_LIBRARIES\n            ${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK})\n    endif()\n\n    if (GGML_OPENCL)\n        find_dependency(OpenCL)\n        set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:OpenCL::OpenCL>)\n    endif()\n\n    if (GGML_VULKAN)\n        find_dependency(Vulkan)\n        set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:Vulkan::Vulkan>)\n    endif()\n\n    if (GGML_HIP)\n        find_dependency(hip)\n        find_dependency(hipblas)\n        find_dependency(rocblas)\n        set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas)\n    endif()\n\n    if (GGML_SYCL)\n        set(GGML_SYCL_INTERFACE_LINK_LIBRARIES \"\")\n        find_package(DNNL)\n        if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL \"INTEL\")\n            list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl)\n        endif()\n        if (WIN32)\n            find_dependency(IntelSYCL)\n            find_dependency(MKL)\n            list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)\n        endif()\n    endif()\nendif()\n\nset_and_check(GGML_INCLUDE_DIR \"@PACKAGE_GGML_INCLUDE_INSTALL_DIR@\")\nset_and_check(GGML_LIB_DIR \"@PACKAGE_GGML_LIB_INSTALL_DIR@\")\n#set_and_check(GGML_BIN_DIR \"@PACKAGE_GGML_BIN_INSTALL_DIR@\")\n\nif(NOT TARGET ggml::ggml)\n    find_package(Threads REQUIRED)\n\n    find_library(GGML_LIBRARY ggml\n        REQUIRED\n        HINTS ${GGML_LIB_DIR}\n        NO_CMAKE_FIND_ROOT_PATH)\n\n    add_library(ggml::ggml UNKNOWN IMPORTED)\n    set_target_properties(ggml::ggml\n        PROPERTIES\n            IMPORTED_LOCATION \"${GGML_LIBRARY}\")\n\n    find_library(GGML_BASE_LIBRARY ggml-base\n        REQUIRED\n        HINTS ${GGML_LIB_DIR}\n        NO_CMAKE_FIND_ROOT_PATH)\n\n    add_library(ggml::ggml-base UNKNOWN IMPORTED)\n    set_target_properties(ggml::ggml-base\n        PROPERTIES\n            IMPORTED_LOCATION \"${GGML_BASE_LIBRARY}\")\n\n    set(_ggml_all_targets \"\")\n    if (NOT GGML_BACKEND_DL)\n        foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS})\n            string(REPLACE \"-\" \"_\" _ggml_backend_pfx \"${_ggml_backend}\")\n            string(TOUPPER \"${_ggml_backend_pfx}\" _ggml_backend_pfx)\n\n            find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}\n                REQUIRED\n                HINTS ${GGML_LIB_DIR}\n                NO_CMAKE_FIND_ROOT_PATH)\n\n            message(STATUS \"Found ${${_ggml_backend_pfx}_LIBRARY}\")\n\n            add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED)\n            set_target_properties(ggml::${_ggml_backend}\n                PROPERTIES\n                    INTERFACE_INCLUDE_DIRECTORIES \"${GGML_INCLUDE_DIR}\"\n                    IMPORTED_LINK_INTERFACE_LANGUAGES \"CXX\"\n                    IMPORTED_LOCATION \"${${_ggml_backend_pfx}_LIBRARY}\"\n                    INTERFACE_COMPILE_FEATURES c_std_90\n                    POSITION_INDEPENDENT_CODE ON)\n\n            string(REGEX MATCH \"^ggml-cpu\" is_cpu_variant \"${_ggml_backend}\")\n            if(is_cpu_variant)\n                list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES \"ggml::ggml-base\")\n                set_target_properties(ggml::${_ggml_backend}\n                PROPERTIES\n                    INTERFACE_LINK_LIBRARIES \"${GGML_CPU_INTERFACE_LINK_LIBRARIES}\")\n\n                if(GGML_CPU_INTERFACE_LINK_OPTIONS)\n                    set_target_properties(ggml::${_ggml_backend}\n                        PROPERTIES\n                            INTERFACE_LINK_OPTIONS \"${GGML_CPU_INTERFACE_LINK_OPTIONS}\")\n                endif()\n\n            else()\n                list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES \"ggml::ggml-base\")\n                set_target_properties(ggml::${_ggml_backend}\n                    PROPERTIES\n                        INTERFACE_LINK_LIBRARIES \"${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}\")\n\n                if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS)\n                    set_target_properties(ggml::${_ggml_backend}\n                        PROPERTIES\n                            INTERFACE_LINK_OPTIONS \"${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}\")\n                endif()\n            endif()\n\n            list(APPEND _ggml_all_targets ggml::${_ggml_backend})\n        endforeach()\n    endif()\n\n    list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base \"${_ggml_all_targets}\")\n    set_target_properties(ggml::ggml\n        PROPERTIES\n            INTERFACE_LINK_LIBRARIES \"${GGML_INTERFACE_LINK_LIBRARIES}\")\n\n    add_library(ggml::all INTERFACE IMPORTED)\n    set_target_properties(ggml::all\n        PROPERTIES\n            INTERFACE_LINK_LIBRARIES \"${_ggml_all_targets}\")\n\nendif()\n\ncheck_required_components(ggml)\n"
  },
  {
    "path": "ggml/include/ggml-alloc.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\ntypedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;\ntypedef struct      ggml_backend_buffer * ggml_backend_buffer_t;\ntypedef struct             ggml_backend * ggml_backend_t;\n\n// Tensor allocator\nstruct ggml_tallocr {\n    ggml_backend_buffer_t buffer;\n    void * base;\n    size_t alignment;\n    size_t offset;\n};\n\nGGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);\nGGML_API enum ggml_status    ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);\n\n// Graph allocator\n/*\n  Example usage:\n    ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type());\n\n    // optional: create a worst-case graph and reserve the buffers to avoid reallocations\n    ggml_gallocr_reserve(galloc, build_graph(max_batch));\n\n    // allocate the graph\n    struct ggml_cgraph * graph = build_graph(batch);\n    ggml_gallocr_alloc_graph(galloc, graph);\n\n    printf(\"compute buffer size: %zu bytes\\n\", ggml_gallocr_get_buffer_size(galloc, 0));\n\n    // evaluate the graph\n    ggml_backend_graph_compute(backend, graph);\n*/\n\n// special tensor flags for use with the graph allocator:\n//   ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses\n//   ggml_set_output(): output tensors are never freed and never overwritten\n\ntypedef struct ggml_gallocr * ggml_gallocr_t;\n\nGGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);\nGGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);\nGGML_API void           ggml_gallocr_free(ggml_gallocr_t galloc);\n\n// pre-allocate buffers from a measure graph - does not allocate or modify the graph\n// call with a worst-case graph to avoid buffer reallocations\n// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed\n// returns false if the buffer allocation failed\n// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes\nGGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);\nGGML_API void ggml_gallocr_reserve_n_size(\n    ggml_gallocr_t galloc,\n    struct ggml_cgraph * graph,\n    const int * node_buffer_ids,\n    const int * leaf_buffer_ids,\n    size_t * sizes);\nGGML_API bool ggml_gallocr_reserve_n(\n    ggml_gallocr_t galloc,\n    struct ggml_cgraph * graph,\n    const int * node_buffer_ids,\n    const int * leaf_buffer_ids);\n\n// automatic reallocation if the topology changes when using a single buffer\n// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)\nGGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);\n\nGGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);\n\n// Utils\n// Create a buffer and allocate all the tensors in a ggml_context\n// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft\nGGML_API size_t                       ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);\nGGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);\nGGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-backend.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n\n#ifdef GGML_BACKEND_SHARED\n#    if defined(_WIN32) && !defined(__MINGW32__)\n#        ifdef GGML_BACKEND_BUILD\n#            define GGML_BACKEND_API __declspec(dllexport) extern\n#        else\n#            define GGML_BACKEND_API __declspec(dllimport) extern\n#        endif\n#    else\n#        define GGML_BACKEND_API __attribute__ ((visibility (\"default\"))) extern\n#    endif\n#else\n#    define GGML_BACKEND_API extern\n#endif\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;\n    typedef struct ggml_backend_buffer * ggml_backend_buffer_t;\n    typedef struct ggml_backend_event * ggml_backend_event_t;\n    typedef struct ggml_backend * ggml_backend_t;\n    typedef void * ggml_backend_graph_plan_t;\n    typedef struct ggml_backend_reg * ggml_backend_reg_t;\n    typedef struct ggml_backend_device * ggml_backend_dev_t;\n\n\n    //\n    // Backend buffer type\n    //\n\n    GGML_API const char *          ggml_backend_buft_name          (ggml_backend_buffer_type_t buft);\n    GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer  (ggml_backend_buffer_type_t buft, size_t size);\n    GGML_API size_t                ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);\n    GGML_API size_t                ggml_backend_buft_get_max_size  (ggml_backend_buffer_type_t buft);\n    GGML_API size_t                ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);\n    GGML_API bool                  ggml_backend_buft_is_host       (ggml_backend_buffer_type_t buft);\n    GGML_API ggml_backend_dev_t    ggml_backend_buft_get_device    (ggml_backend_buffer_type_t buft);\n\n    //\n    // Backend buffer\n    //\n\n    enum ggml_backend_buffer_usage {\n        GGML_BACKEND_BUFFER_USAGE_ANY = 0,\n        GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,\n        GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,\n    };\n\n    GGML_API const char *                   ggml_backend_buffer_name          (ggml_backend_buffer_t buffer);\n    GGML_API void                           ggml_backend_buffer_free          (ggml_backend_buffer_t buffer);\n    GGML_API void *                         ggml_backend_buffer_get_base      (ggml_backend_buffer_t buffer);\n    GGML_API size_t                         ggml_backend_buffer_get_size      (ggml_backend_buffer_t buffer);\n    GGML_API enum ggml_status               ggml_backend_buffer_init_tensor   (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);\n    GGML_API size_t                         ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);\n    GGML_API size_t                         ggml_backend_buffer_get_max_size  (ggml_backend_buffer_t buffer);\n    GGML_API size_t                         ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor);\n    GGML_API void                           ggml_backend_buffer_clear         (ggml_backend_buffer_t buffer, uint8_t value);\n    GGML_API bool                           ggml_backend_buffer_is_host       (ggml_backend_buffer_t buffer);\n    GGML_API void                           ggml_backend_buffer_set_usage     (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);\n    GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage     (ggml_backend_buffer_t buffer);\n    GGML_API ggml_backend_buffer_type_t     ggml_backend_buffer_get_type      (ggml_backend_buffer_t buffer);\n    GGML_API void                           ggml_backend_buffer_reset         (ggml_backend_buffer_t buffer);\n\n    // tensor copy between different backends\n    GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);\n\n    //\n    // Backend (stream)\n    //\n\n    GGML_API ggml_guid_t  ggml_backend_guid(ggml_backend_t backend);\n    GGML_API const char * ggml_backend_name(ggml_backend_t backend);\n    GGML_API void         ggml_backend_free(ggml_backend_t backend);\n\n    GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);\n    GGML_API ggml_backend_buffer_t      ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);\n    GGML_API size_t                     ggml_backend_get_alignment(ggml_backend_t backend);\n    GGML_API size_t                     ggml_backend_get_max_size(ggml_backend_t backend);\n\n    GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n    GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);\n\n    // \"offset\" refers to the offset in tensor->data for setting/getting data\n    GGML_API void ggml_backend_tensor_set(      struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n    GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);\n    GGML_API void ggml_backend_tensor_memset(   struct ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);\n\n    GGML_API void ggml_backend_synchronize(ggml_backend_t backend);\n\n    GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);\n    GGML_API void                      ggml_backend_graph_plan_free  (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n\n    GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n    GGML_API enum ggml_status ggml_backend_graph_compute      (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n    GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);\n\n    // NOTE: will be removed, use device version instead\n    GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);\n    GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);\n    GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);\n\n    // asynchronous copy\n    // the copy is performed after all the currently queued operations in backend_src\n    // backend_dst will wait for the copy to complete before performing other operations\n    // automatic fallback to sync copy if async is not supported\n    GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);\n\n    GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);\n\n    //\n    // Events\n    //\n\n    GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);\n    GGML_API void                 ggml_backend_event_free(ggml_backend_event_t event);\n    GGML_API void                 ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);\n    GGML_API void                 ggml_backend_event_synchronize(ggml_backend_event_t event);\n    GGML_API void                 ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);\n\n    //\n    // Backend device\n    //\n\n    enum ggml_backend_dev_type {\n        // CPU device using system memory\n        GGML_BACKEND_DEVICE_TYPE_CPU,\n        // GPU device using dedicated memory\n        GGML_BACKEND_DEVICE_TYPE_GPU,\n        // integrated GPU device using host memory\n        GGML_BACKEND_DEVICE_TYPE_IGPU,\n        // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)\n        GGML_BACKEND_DEVICE_TYPE_ACCEL\n    };\n\n    // functionality supported by the device\n    struct ggml_backend_dev_caps {\n        // asynchronous operations\n        bool async;\n        // pinned host buffer\n        bool host_buffer;\n        // creating buffers from host ptr\n        bool buffer_from_host_ptr;\n        // event synchronization\n        bool events;\n    };\n\n    // all the device properties\n    struct ggml_backend_dev_props {\n        // device name\n        const char * name;\n        // device description\n        const char * description;\n        // device free memory in bytes\n        size_t memory_free;\n        // device total memory in bytes\n        size_t memory_total;\n        // device type\n        enum ggml_backend_dev_type type;\n        // device id\n        //   for PCI devices, this should be the PCI bus id formatted as \"domain:bus:device.function\" (e.g. \"0000:01:00.0\")\n        //   if the id is unknown, this should be NULL\n        const char * device_id;\n        // device capabilities\n        struct ggml_backend_dev_caps caps;\n    };\n\n    GGML_API const char *                  ggml_backend_dev_name(ggml_backend_dev_t device);\n    GGML_API const char *                  ggml_backend_dev_description(ggml_backend_dev_t device);\n    GGML_API void                          ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);\n    GGML_API enum ggml_backend_dev_type    ggml_backend_dev_type(ggml_backend_dev_t device);\n    GGML_API void                          ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);\n    GGML_API ggml_backend_reg_t            ggml_backend_dev_backend_reg(ggml_backend_dev_t device);\n    GGML_API ggml_backend_t                ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);\n    GGML_API ggml_backend_buffer_type_t    ggml_backend_dev_buffer_type(ggml_backend_dev_t device);\n    GGML_API ggml_backend_buffer_type_t    ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);\n    GGML_API ggml_backend_buffer_t         ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);\n\n    GGML_API bool                          ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);\n    GGML_API bool                          ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);\n    GGML_API bool                          ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);\n\n    //\n    // Backend (reg)\n    //\n\n    GGML_API const char *       ggml_backend_reg_name(ggml_backend_reg_t reg);\n    GGML_API size_t             ggml_backend_reg_dev_count(ggml_backend_reg_t reg);\n    GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);\n    GGML_API void *             ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);\n\n    // Common functions that may be obtained using ggml_backend_reg_get_proc_address\n\n    // Split buffer type for tensor parallelism\n    typedef ggml_backend_buffer_type_t   (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);\n    // Set the number of threads for the backend\n    typedef void                         (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);\n    // Get additional buffer types provided by the device (returns a NULL-terminated array)\n    typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);\n    // Set the abort callback for the backend\n    typedef void                         (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);\n    // Get a list of feature flags supported by the backend (returns a NULL-terminated array)\n    struct ggml_backend_feature {\n        const char * name;\n        const char * value;\n    };\n    typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);\n\n    //\n    // Backend registry\n    //\n\n    GGML_API void ggml_backend_register(ggml_backend_reg_t reg);\n\n    GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);\n\n    // Backend (reg) enumeration\n    GGML_API size_t             ggml_backend_reg_count(void);\n    GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);\n    GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);\n\n    // Device enumeration\n    GGML_API size_t             ggml_backend_dev_count(void);\n    GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);\n    GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);\n    GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);\n\n    // Direct backend (stream) initialization\n    // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)\n    GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);\n    // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)\n    GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);\n    // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)\n    GGML_API ggml_backend_t ggml_backend_init_best(void);\n\n    // Load a backend from a dynamic library and register it\n    GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);\n    // Unload a backend if loaded dynamically and unregister it\n    GGML_API void               ggml_backend_unload(ggml_backend_reg_t reg);\n    // Load all known backends from dynamic libraries\n    GGML_API void               ggml_backend_load_all(void);\n    GGML_API void               ggml_backend_load_all_from_path(const char * dir_path);\n\n    //\n    // Backend scheduler\n    //\n\n    // The backend scheduler allows for multiple backend devices to be used together\n    // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends\n    // The backends are selected based on:\n    // - the backend that supports the operation\n    // - the location of the pre-allocated tensors (e.g. the weights)\n    /*\n      Example usage:\n\n        // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned\n        // preferably to run on the same backend as the buffer\n        ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);\n\n        sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true);\n\n        // initialize buffers from a max size graph (optional)\n        reserve_graph = build_graph(sched, max_batch_size);\n\n        // manually assign nodes to a backend (optional, should not be needed in most cases)\n        struct ggml_tensor * node = ggml_mul_mat(ctx, ...);\n        ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);\n\n        ggml_backend_sched_reserve(sched, reserve_graph);\n\n        // compute\n        graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation\n        for (int i = 0; i < 10; ++i) {\n            ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically\n        }\n\n        // if there are graph inputs:\n        graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)\n        ggml_backend_sched_reset(sched); // clear the allocation of the previous graph\n        ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it\n        ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors\n        ggml_backend_sched_graph_compute(sched, graph); // execute the graph\n\n        // as an alternative to the above it is also possible to assign the inputs to a dedicated context and\n        // allocate them statically via ggml_backend_alloc_ctx_tensors\n    }\n    */\n\n    typedef struct ggml_backend_sched * ggml_backend_sched_t;\n\n    // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)\n    // when ask == true, the scheduler wants to know if the user wants to observe this node\n    // this allows the scheduler to batch nodes together in order to evaluate them in a single call\n    //\n    // when ask == false, the scheduler is passing the node tensor to the user for observation\n    // if the user returns false, the scheduler will cancel the graph compute\n    //\n    typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);\n\n    // Initialize a backend scheduler, backends with low index are given priority over backends with high index\n    GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);\n    GGML_API void                 ggml_backend_sched_free(ggml_backend_sched_t sched);\n\n    // Initialize backend buffers from a measure graph\n    GGML_API void                 ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes);\n    GGML_API bool                 ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success\n\n    GGML_API int                  ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);\n    GGML_API ggml_backend_t       ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);\n\n    // Get the number of splits of the last graph\n    GGML_API int                  ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);\n    GGML_API int                  ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);\n\n    GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);\n    GGML_API size_t                     ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);\n\n    GGML_API void                 ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);\n    GGML_API ggml_backend_t       ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);\n\n    // Split graph without allocating it\n    GGML_API void                 ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n\n    // Allocate and compute graph on the backend scheduler\n    GGML_API bool                 ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success\n    GGML_API enum ggml_status     ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n    GGML_API enum ggml_status     ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);\n    GGML_API void                 ggml_backend_sched_synchronize(ggml_backend_sched_t sched);\n\n    // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.\n    // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.\n    // The correct way to use this API is to discard the deallocated tensors and create new ones.\n    GGML_API void                 ggml_backend_sched_reset(ggml_backend_sched_t sched);\n\n    // Set a callback to be called for each resulting node during graph compute\n    GGML_API void                 ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);\n\n    //\n    // Utils\n    //\n\n    struct ggml_backend_graph_copy {\n        ggml_backend_buffer_t buffer;\n        struct ggml_context * ctx_allocated;\n        struct ggml_context * ctx_unallocated;\n        struct ggml_cgraph * graph;\n    };\n\n    // Copy a graph to a different backend\n    GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);\n    GGML_API void                           ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);\n\n    typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);\n\n    // Compare the output of two backends\n    GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);\n\n    // Tensor initialization\n    GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);\n    GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);\n\n    // CPU buffer types are always available\n    GGML_API ggml_backend_buffer_t      ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);\n    GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-blas.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_blas_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_blas(ggml_backend_t backend);\n\n// number of threads used for conversion to float\n// for openblas and blis, this will also set the number of threads used for blas operations\nGGML_BACKEND_API void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_blas_reg(void);\n\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-cann.h",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n/**\n * @brief Maximum number of CANN devices supported.\n */\n#define GGML_CANN_MAX_DEVICES 16\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_cann_reg(void);\n\n/**\n * @brief Initializes the CANN backend for a specified device.\n *\n * This function initializes the CANN backend for the given device.\n * It verifies the device index, allocates a context, and creates a backend\n * instance.\n *\n * @param device The index of the device to initialize.\n * @return A pointer to the initialized backend instance, or nullptr on failure.\n */\nGGML_BACKEND_API ggml_backend_t ggml_backend_cann_init(int32_t device);\n\n/**\n * @brief Checks if a given backend is a CANN backend.\n *\n * This function verifies if the provided backend is a CANN backend by comparing\n * its GUID with the CANN backend's GUID.\n *\n * @param backend The backend instance to check.\n * @return True if the backend is a CANN backend, false otherwise.\n */\nGGML_BACKEND_API bool ggml_backend_is_cann(ggml_backend_t backend);\n\n/**\n * @brief Retrieves the CANN buffer type for a specified device.\n *\n * This function initializes and returns the buffer type interface associated\n * with the given device. It ensures thread-safe access using a mutex.\n *\n * @param device The device index for which to retrieve the buffer type.\n * @return A pointer to the buffer type interface for the specified device, or\n * nullptr if the device index is out of range.\n */\nGGML_BACKEND_API ggml_backend_buffer_type_t\nggml_backend_cann_buffer_type(int32_t device);\n\n/**\n * @brief Retrieves the number of CANN devices available.\n *\n * This function returns the number of CANN devices available based on\n * information obtained from `ggml_cann_info()`.\n *\n * @return The number of CANN devices available.\n */\nGGML_BACKEND_API int32_t ggml_backend_cann_get_device_count(void);\n\n/**\n * @brief pinned host buffer for use with the CPU backend for faster copies between CPU and NPU.\n *\n * @return A pointer to the host buffer type interface.\n */\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type(void);\n\n/**\n * @brief Retrieves the description of a specific CANN device.\n *\n * This function sets the specified device, retrieves the SoC name,\n * and writes it into the provided description buffer.\n *\n * @param device The device index to retrieve the description for.\n * @param description Pointer to a buffer where the description will be written.\n * @param description_size Size of the description buffer.\n */\nGGML_BACKEND_API void ggml_backend_cann_get_device_description(\n    int32_t device, char* description, size_t description_size);\n\n/**\n * @brief Retrieves the memory information of a specific CANN device.\n *\n * This function sets the specified device, retrieves the free and total\n * memory information of the specified type (ACL_HBM_MEM), and stores them\n * in the provided pointers.\n *\n * @param device The device index to retrieve memory information for.\n * @param free Pointer to a variable where the free memory size will be stored.\n * @param total Pointer to a variable where the total memory size will be\n * stored.\n */\nGGML_BACKEND_API void ggml_backend_cann_get_device_memory(int32_t device,\n                                                  size_t* free,\n                                                  size_t* total);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-cpp.h",
    "content": "#pragma once\n\n#ifndef __cplusplus\n#error \"This header is for C++ only\"\n#endif\n\n#include \"ggml.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-backend.h\"\n#include \"gguf.h\"\n#include <memory>\n\n// Smart pointers for ggml types\n\n// ggml\n\nstruct ggml_context_deleter { void operator()(ggml_context * ctx) { ggml_free(ctx); } };\nstruct gguf_context_deleter { void operator()(gguf_context * ctx) { gguf_free(ctx); } };\n\ntypedef std::unique_ptr<ggml_context, ggml_context_deleter> ggml_context_ptr;\ntypedef std::unique_ptr<gguf_context, gguf_context_deleter> gguf_context_ptr;\n\n// ggml-alloc\n\nstruct ggml_gallocr_deleter { void operator()(ggml_gallocr_t galloc) { ggml_gallocr_free(galloc); } };\n\ntypedef std::unique_ptr<ggml_gallocr, ggml_gallocr_deleter> ggml_gallocr_ptr;\n\n// ggml-backend\n\nstruct ggml_backend_deleter        { void operator()(ggml_backend_t backend)       { ggml_backend_free(backend); } };\nstruct ggml_backend_buffer_deleter { void operator()(ggml_backend_buffer_t buffer) { ggml_backend_buffer_free(buffer); } };\nstruct ggml_backend_event_deleter  { void operator()(ggml_backend_event_t event)   { ggml_backend_event_free(event); } };\nstruct ggml_backend_sched_deleter  { void operator()(ggml_backend_sched_t sched)   { ggml_backend_sched_free(sched); } };\n\ntypedef std::unique_ptr<ggml_backend,        ggml_backend_deleter>        ggml_backend_ptr;\ntypedef std::unique_ptr<ggml_backend_buffer, ggml_backend_buffer_deleter> ggml_backend_buffer_ptr;\ntypedef std::unique_ptr<ggml_backend_event,  ggml_backend_event_deleter>  ggml_backend_event_ptr;\ntypedef std::unique_ptr<ggml_backend_sched,  ggml_backend_sched_deleter>  ggml_backend_sched_ptr;\n"
  },
  {
    "path": "ggml/include/ggml-cpu.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    // the compute plan that needs to be prepared for ggml_graph_compute()\n    // since https://github.com/ggml-org/ggml/issues/287\n    struct ggml_cplan {\n        size_t    work_size; // size of work buffer, calculated by `ggml_graph_plan()`\n        uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`\n\n        int n_threads;\n        struct ggml_threadpool * threadpool;\n\n        // abort ggml_graph_compute when true\n        ggml_abort_callback abort_callback;\n        void *              abort_callback_data;\n\n        // use only reference implementations\n        bool use_ref;\n    };\n\n    // numa strategies\n    enum ggml_numa_strategy {\n        GGML_NUMA_STRATEGY_DISABLED   = 0,\n        GGML_NUMA_STRATEGY_DISTRIBUTE = 1,\n        GGML_NUMA_STRATEGY_ISOLATE    = 2,\n        GGML_NUMA_STRATEGY_NUMACTL    = 3,\n        GGML_NUMA_STRATEGY_MIRROR     = 4,\n        GGML_NUMA_STRATEGY_COUNT\n    };\n\n    GGML_BACKEND_API void    ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems\n    GGML_BACKEND_API bool    ggml_is_numa(void); // true if init detected that system has >1 NUMA node\n\n    GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);\n    GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);\n\n    GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);\n    GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);\n\n    GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);\n    GGML_BACKEND_API void    ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);\n\n    GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);\n    GGML_BACKEND_API void    ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);\n\n    GGML_BACKEND_API float   ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);\n    GGML_BACKEND_API void    ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);\n\n    GGML_BACKEND_API float   ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);\n    GGML_BACKEND_API void    ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);\n\n    GGML_BACKEND_API struct ggml_threadpool *      ggml_threadpool_new           (struct ggml_threadpool_params  * params);\n    GGML_BACKEND_API void                          ggml_threadpool_free          (struct ggml_threadpool * threadpool);\n    GGML_BACKEND_API int                           ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool);\n    GGML_BACKEND_API void                          ggml_threadpool_pause         (struct ggml_threadpool * threadpool);\n    GGML_BACKEND_API void                          ggml_threadpool_resume        (struct ggml_threadpool * threadpool);\n\n    // ggml_graph_plan() has to be called before ggml_graph_compute()\n    // when plan.work_size > 0, caller must allocate memory for plan.work_data\n    GGML_BACKEND_API struct ggml_cplan ggml_graph_plan(\n                  const struct ggml_cgraph * cgraph,\n                                       int   n_threads, /* = GGML_DEFAULT_N_THREADS */\n                    struct ggml_threadpool * threadpool /* = NULL */ );\n    GGML_BACKEND_API enum ggml_status  ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);\n\n    // same as ggml_graph_compute() but the work data is allocated as a part of the context\n    // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data\n    GGML_BACKEND_API enum ggml_status  ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);\n\n    //\n    // system info\n    //\n\n    // x86\n    GGML_BACKEND_API int ggml_cpu_has_sse3       (void);\n    GGML_BACKEND_API int ggml_cpu_has_ssse3      (void);\n    GGML_BACKEND_API int ggml_cpu_has_avx        (void);\n    GGML_BACKEND_API int ggml_cpu_has_avx_vnni   (void);\n    GGML_BACKEND_API int ggml_cpu_has_avx2       (void);\n    GGML_BACKEND_API int ggml_cpu_has_bmi2       (void);\n    GGML_BACKEND_API int ggml_cpu_has_f16c       (void);\n    GGML_BACKEND_API int ggml_cpu_has_fma        (void);\n    GGML_BACKEND_API int ggml_cpu_has_avx512     (void);\n    GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void);\n    GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void);\n    GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void);\n    GGML_BACKEND_API int ggml_cpu_has_amx_int8   (void);\n    // ARM\n    GGML_BACKEND_API int ggml_cpu_has_neon       (void);\n    GGML_BACKEND_API int ggml_cpu_has_arm_fma    (void);\n    GGML_BACKEND_API int ggml_cpu_has_fp16_va    (void);\n    GGML_BACKEND_API int ggml_cpu_has_dotprod    (void);\n    GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void);\n    GGML_BACKEND_API int ggml_cpu_has_sve        (void);\n    GGML_BACKEND_API int ggml_cpu_get_sve_cnt    (void);  // sve vector length in bytes\n    GGML_BACKEND_API int ggml_cpu_has_sme        (void);\n    // other\n    GGML_BACKEND_API int ggml_cpu_has_riscv_v    (void);\n    GGML_BACKEND_API int ggml_cpu_get_rvv_vlen   (void);  // risc-v vector length in bytes\n    GGML_BACKEND_API int ggml_cpu_has_vsx        (void);\n    GGML_BACKEND_API int ggml_cpu_has_vxe        (void);\n    GGML_BACKEND_API int ggml_cpu_has_wasm_simd  (void);\n    GGML_BACKEND_API int ggml_cpu_has_llamafile  (void);\n\n    // Internal types and functions exposed for tests and benchmarks\n\n    typedef void (*ggml_vec_dot_t)  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,\n                                       const void * GGML_RESTRICT y, size_t by, int nrc);\n\n    struct ggml_type_traits_cpu {\n        ggml_from_float_t        from_float;\n        ggml_vec_dot_t           vec_dot;\n        enum ggml_type           vec_dot_type;\n        int64_t                  nrows; // number of rows to process simultaneously\n    };\n\n    GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type);\n\n    GGML_BACKEND_API void ggml_cpu_init(void);\n\n    //\n    // CPU backend\n    //\n\n    GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void);\n\n    GGML_BACKEND_API bool ggml_backend_is_cpu                (ggml_backend_t backend);\n    GGML_BACKEND_API void ggml_backend_cpu_set_n_threads     (ggml_backend_t backend_cpu, int n_threads);\n    GGML_BACKEND_API void ggml_backend_cpu_set_threadpool    (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool);\n    GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);\n\n    GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref);\n\n    GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void);\n\n    GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *,       float *, int64_t);\n    GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *,     int32_t *, int64_t);\n    GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t);\n    GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t);\n    GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t);\n    GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-cuda.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n#ifdef GGML_USE_HIP\n#define GGML_CUDA_NAME \"ROCm\"\n#define GGML_CUBLAS_NAME \"hipBLAS\"\n#elif defined(GGML_USE_MUSA)\n#define GGML_CUDA_NAME \"MUSA\"\n#define GGML_CUBLAS_NAME \"muBLAS\"\n#else\n#define GGML_CUDA_NAME \"CUDA\"\n#define GGML_CUBLAS_NAME \"cuBLAS\"\n#endif\n#define GGML_CUDA_MAX_DEVICES       16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);\n\n// split tensor buffer that splits matrices by rows across multiple devices\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);\n\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);\n\nGGML_BACKEND_API int  ggml_backend_cuda_get_device_count(void);\nGGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);\nGGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);\n\nGGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);\nGGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-hexagon.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_hexagon_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_hexagon(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_hexagon_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-metal.h",
    "content": "// Note: this description is outdated\n//\n// An interface allowing to compute ggml_cgraph with Metal\n//\n// This is a fully functional interface that extends ggml with GPU support for Apple devices.\n// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)\n//\n// How it works?\n//\n// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this\n// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you\n// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)\n//\n// You only need to make sure that all memory buffers that you used during the graph creation\n// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is\n// used during the graph evaluation to determine the arguments of the compute kernels.\n//\n// Synchronization between device and host memory (for example for input and output tensors)\n// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.\n//\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include <stddef.h>\n#include <stdbool.h>\n\nstruct ggml_tensor;\nstruct ggml_cgraph;\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// backend API\n// user-code should use only these functions\n//\n\n// TODO: remove in the future\nGGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend);\n\nGGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data);\n\n// helper to check if the device supports a specific family\n// ideally, the user code should be doing these checks\n// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf\nGGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);\n\n// capture all command buffers committed the next time `ggml_backend_graph_compute` is called\nGGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-opencl.h",
    "content": "#ifndef GGML_OPENCL_H\n#define GGML_OPENCL_H\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n//\n// backend API\n//\nGGML_BACKEND_API ggml_backend_t ggml_backend_opencl_init(void);\nGGML_BACKEND_API bool ggml_backend_is_opencl(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_buffer_type(void);\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_opencl_host_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_opencl_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n\n#endif // GGML_OPENCL_H\n"
  },
  {
    "path": "ggml/include/ggml-openvino.h",
    "content": "#pragma once\n\n#include \"ggml-backend.h\"\n\n#include <cstring>\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_OPENVINO_NAME \"OPENVINO\"\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend);\n\nGGML_BACKEND_API bool ggml_backend_buffer_is_openvino(ggml_backend_buffer_t buffer);\n\nGGML_BACKEND_API bool ggml_backend_buft_is_openvino(ggml_backend_buffer_type_t buft);\n\nGGML_BACKEND_API bool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft);\n\nGGML_BACKEND_API size_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer);\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(int device);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_type(int device);\n\nGGML_BACKEND_API int ggml_backend_openvino_get_device_count(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-opt.h",
    "content": "// This file contains functionality for training models using GGML.\n// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets.\n// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code.\n//\n// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#include <stdint.h>\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    struct ggml_opt_dataset;\n    struct ggml_opt_context;\n    struct ggml_opt_result;\n\n    typedef struct ggml_opt_dataset * ggml_opt_dataset_t;\n    typedef struct ggml_opt_context * ggml_opt_context_t;\n    typedef struct ggml_opt_result  * ggml_opt_result_t;\n\n    // ====== Loss ======\n\n    // built-in loss types, i.e. the built-in quantities minimized by the optimizer\n    // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value\n    enum ggml_opt_loss_type {\n        GGML_OPT_LOSS_TYPE_MEAN,\n        GGML_OPT_LOSS_TYPE_SUM,\n        GGML_OPT_LOSS_TYPE_CROSS_ENTROPY,\n        GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR,\n    };\n\n    // ====== Dataset ======\n\n    GGML_API ggml_opt_dataset_t ggml_opt_dataset_init(\n            enum ggml_type type_data,    // the type for the internal data tensor\n            enum ggml_type type_label,   // the type for the internal labels tensor\n            int64_t        ne_datapoint, // number of elements per datapoint\n            int64_t        ne_label,     // number of elements per label\n            int64_t        ndata,        // total number of datapoints/labels\n            int64_t        ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied)\n    GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset);\n\n    // get underlying tensors that store the data\n    GGML_API int64_t              ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset);\n    GGML_API struct ggml_tensor * ggml_opt_dataset_data  (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata]\n    GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label,     ndata]\n\n    // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative\n    GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata);\n\n    // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch\n    GGML_API void ggml_opt_dataset_get_batch(\n            ggml_opt_dataset_t   dataset,\n            struct ggml_tensor * data_batch,   // shape = [ne_datapoint, ndata_batch]\n            struct ggml_tensor * labels_batch, // shape = [ne_label,     ndata_batch]\n            int64_t              ibatch);\n    GGML_API void ggml_opt_dataset_get_batch_host(\n            ggml_opt_dataset_t   dataset,\n            void               * data_batch,\n            size_t               nb_data_batch,\n            void               * labels_batch,\n            int64_t              ibatch);\n\n    // ====== Model / Context ======\n\n    enum ggml_opt_build_type {\n        GGML_OPT_BUILD_TYPE_FORWARD = 10,\n        GGML_OPT_BUILD_TYPE_GRAD    = 20,\n        GGML_OPT_BUILD_TYPE_OPT     = 30,\n    };\n\n    enum ggml_opt_optimizer_type {\n        GGML_OPT_OPTIMIZER_TYPE_ADAMW,\n        GGML_OPT_OPTIMIZER_TYPE_SGD,\n\n        GGML_OPT_OPTIMIZER_TYPE_COUNT\n    };\n\n    // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss\n    struct ggml_opt_optimizer_params {\n        struct {\n            float alpha; // learning rate\n            float beta1; // first AdamW momentum\n            float beta2; // second AdamW momentum\n            float eps;   // epsilon for numerical stability\n            float wd;    // weight decay - 0.0f to disable\n        } adamw;\n        struct {\n            float alpha; // learning rate\n            float wd;    // weight decay\n        } sgd;\n    };\n\n    // callback to calculate optimizer parameters prior to a backward pass\n    // userdata can be used to pass arbitrary data\n    typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata);\n\n    // returns the default optimizer params (constant, hard-coded values)\n    // userdata is not used\n    GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata);\n\n    // casts userdata to ggml_opt_optimizer_params and returns it\n    GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata);\n\n    // parameters for initializing a new optimization context\n    struct ggml_opt_params {\n        ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs\n\n        // by default the forward graph needs to be reconstructed for each eval\n        // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically\n        struct ggml_context * ctx_compute;\n        struct ggml_tensor  * inputs;\n        struct ggml_tensor  * outputs;\n\n        enum ggml_opt_loss_type  loss_type;\n        enum ggml_opt_build_type build_type;\n\n        int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done\n\n        ggml_opt_get_optimizer_params get_opt_pars;    // callback for calculating optimizer parameters\n        void *                        get_opt_pars_ud; // userdata for calculating optimizer parameters\n\n        // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor\n        enum ggml_opt_optimizer_type optimizer;\n    };\n\n    // get parameters for an optimization context with defaults set where possible\n    // parameters for which no sensible defaults exist are supplied as arguments to this function\n    GGML_API struct ggml_opt_params ggml_opt_default_params(\n            ggml_backend_sched_t    backend_sched,\n            enum ggml_opt_loss_type loss_type);\n\n    GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params);\n    GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx);\n\n    // set gradients to zero, initialize loss, and optionally reset the optimizer\n    GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer);\n\n    GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically\n\n    // get underlying tensors that store data\n    // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc\n    GGML_API struct ggml_tensor * ggml_opt_inputs(  ggml_opt_context_t opt_ctx); // forward graph input tensor\n    GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor\n    GGML_API struct ggml_tensor * ggml_opt_labels(  ggml_opt_context_t opt_ctx); // labels to compare outputs against\n    GGML_API struct ggml_tensor * ggml_opt_loss(    ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss\n    GGML_API struct ggml_tensor * ggml_opt_pred(    ggml_opt_context_t opt_ctx); // predictions made by outputs\n    GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels\n\n    // get the gradient accumulator for a node from the forward graph\n    GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node);\n\n    GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme\n\n    GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type);\n\n    // ====== Optimization Result ======\n\n    GGML_API ggml_opt_result_t ggml_opt_result_init(void);\n    GGML_API void ggml_opt_result_free(ggml_opt_result_t result);\n    GGML_API void ggml_opt_result_reset(ggml_opt_result_t result);\n\n    // get data from result, uncertainties are optional and can be ignored by passing NULL\n    GGML_API void ggml_opt_result_ndata(   ggml_opt_result_t result, int64_t * ndata);                  // writes 1 value, number of datapoints\n    GGML_API void ggml_opt_result_loss(    ggml_opt_result_t result, double  * loss,     double * unc); // writes 1 value\n    GGML_API void ggml_opt_result_pred(    ggml_opt_result_t result, int32_t * pred);                   // writes ndata values\n    GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double  * accuracy, double * unc); // writes 1 value\n\n    // ====== Computation ======\n\n    // if not using static graphs, this function must be called prior to ggml_opt_alloc\n    GGML_API void ggml_opt_prepare_alloc(\n        ggml_opt_context_t    opt_ctx,\n        struct ggml_context * ctx_compute,\n        struct ggml_cgraph  * gf,\n        struct ggml_tensor  * inputs,\n        struct ggml_tensor  * outputs);\n\n    // allocate the next graph for evaluation, either forward or forward + backward\n    // must be called exactly once prior to calling ggml_opt_eval\n    GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward);\n\n    // do forward pass, increment result if not NULL, do backward pass if allocated\n    GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result);\n\n    // ############################################################################\n    // ## The high-level functions start here. They do not depend on any private ##\n    // ## functions or structs and can be copied to and adapted for user code.   ##\n    // ############################################################################\n\n    // ====== Intended Usage ======\n    //\n    // 1. Select the appropriate loss for your problem.\n    // 2. Create a dataset and set the data for the \"data\" tensor. Also set the \"labels\" tensor if your loss needs them.\n    //    Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster).\n    // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors.\n    //    The first context should contain the model parameters and inputs and be allocated statically in user code.\n    //    The second context should contain all other tensors and will be (re)allocated automatically.\n    //    Due to this automated allocation the data of the second context is not defined when accessed in user code.\n    //    Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors.\n    // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead.\n\n    // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation\n    typedef void (*ggml_opt_epoch_callback)(\n            bool               train,       // true after training evaluation, false after validation evaluation\n            ggml_opt_context_t opt_ctx,\n            ggml_opt_dataset_t dataset,\n            ggml_opt_result_t  result,      // result associated with the dataset subsection\n            int64_t            ibatch,      // number of batches that have been evaluated so far\n            int64_t            ibatch_max,  // total number of batches in this dataset subsection\n            int64_t            t_start_us); // time at which the evaluation on the dataset subsection was started\n\n    // do training on front of dataset, do evaluation only on back of dataset\n    GGML_API void ggml_opt_epoch(\n            ggml_opt_context_t      opt_ctx,\n            ggml_opt_dataset_t      dataset,\n            ggml_opt_result_t       result_train,   // result to increment during training, ignored if NULL\n            ggml_opt_result_t       result_eval,    // result to increment during evaluation, ignored if NULL\n            int64_t                 idata_split,    // data index at which to split training and evaluation\n            ggml_opt_epoch_callback callback_train,\n            ggml_opt_epoch_callback callback_eval);\n\n    // callback that prints a progress bar on stderr\n    GGML_API void ggml_opt_epoch_callback_progress_bar(\n            bool               train,\n            ggml_opt_context_t opt_ctx,\n            ggml_opt_dataset_t dataset,\n            ggml_opt_result_t  result,\n            int64_t            ibatch,\n            int64_t            ibatch_max,\n            int64_t            t_start_us);\n\n    // fit model defined by inputs and outputs to dataset\n    GGML_API void ggml_opt_fit(\n            ggml_backend_sched_t            backend_sched,  // backend scheduler for constructing the compute graphs\n            struct ggml_context           * ctx_compute,    // context with temporarily allocated tensors to calculate the outputs\n            struct ggml_tensor            * inputs,         // input tensor with shape [ne_datapoint, ndata_batch]\n            struct ggml_tensor            * outputs,        // output tensor, must have shape [ne_label, ndata_batch] if labels are used\n            ggml_opt_dataset_t              dataset,        // dataset with data and optionally also labels\n            enum ggml_opt_loss_type         loss_type,      // loss to minimize\n            enum ggml_opt_optimizer_type    optimizer,      // sgd or adamw\n            ggml_opt_get_optimizer_params   get_opt_pars,   // callback to get optimizer params, userdata is pointer to epoch (of type int64_t)\n            int64_t                         nepoch,         // how many times the dataset should be iterated over\n            int64_t                         nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs\n            float                           val_split,      // fraction of the dataset to use for validation, must be in [0.0f, 1.0f)\n            bool                            silent);        // whether or not info prints to stderr should be suppressed\n\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-rpc.h",
    "content": "#pragma once\n\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n#define RPC_PROTO_MAJOR_VERSION    3\n#define RPC_PROTO_MINOR_VERSION    6\n#define RPC_PROTO_PATCH_VERSION    1\n\n#ifdef  __cplusplus\nstatic_assert(GGML_OP_COUNT == 96, \"GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION\");\n#endif\n\n#define GGML_RPC_MAX_SERVERS       16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device);\nGGML_BACKEND_API bool ggml_backend_is_rpc(ggml_backend_t backend);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device);\n\nGGML_BACKEND_API void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total);\n\nGGML_BACKEND_API void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,\n                                                    size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_reg(void);\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-sycl.h",
    "content": "//\n//  MIT license\n//  Copyright (C) 2024 Intel Corporation\n//  SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#define GGML_SYCL_NAME \"SYCL\"\n#define GGML_SYCL_MAX_DEVICES 48\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_sycl_init(int device);\n\nGGML_BACKEND_API bool ggml_backend_is_sycl(ggml_backend_t backend);\n\n// devide buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_buffer_type(int device);\n\n// split tensor buffer that splits matrices by rows across multiple devices\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split);\n\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_sycl_host_buffer_type(void);\n\nGGML_BACKEND_API void ggml_backend_sycl_print_sycl_devices(void);\nGGML_BACKEND_API void ggml_backend_sycl_get_gpu_list(int *id_list, int max_len);\nGGML_BACKEND_API void ggml_backend_sycl_get_device_description(int device,\n                                                       char *description,\n                                                       size_t description_size);\nGGML_BACKEND_API int  ggml_backend_sycl_get_device_count();\nGGML_BACKEND_API void ggml_backend_sycl_get_device_memory(int device, size_t *free, size_t *total);\n\n// SYCL doesn't support registering host memory, keep here for reference\n// GGML_BACKEND_API bool ggml_backend_sycl_register_host_buffer(void * buffer, size_t size);\n// GGML_BACKEND_API void ggml_backend_sycl_unregister_host_buffer(void * buffer);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_sycl_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-virtgpu.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_virtgpu_reg();\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-vulkan.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_VK_NAME \"Vulkan\"\n#define GGML_VK_MAX_DEVICES 16\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_vk_init(size_t dev_num);\n\nGGML_BACKEND_API bool ggml_backend_is_vk(ggml_backend_t backend);\nGGML_BACKEND_API int  ggml_backend_vk_get_device_count(void);\nGGML_BACKEND_API void ggml_backend_vk_get_device_description(int device, char * description, size_t description_size);\nGGML_BACKEND_API void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total);\n\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num);\n// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_vk_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-webgpu.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n#define GGML_WEBGPU_NAME \"WebGPU\"\n\n// Needed for examples in ggml\nGGML_BACKEND_API ggml_backend_t ggml_backend_webgpu_init(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_webgpu_reg(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-zdnn.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-backend.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// device buffer\nGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_zdnn_buffer_type(void);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_zdnn_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml-zendnn.h",
    "content": "#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// backend API\nGGML_BACKEND_API ggml_backend_t ggml_backend_zendnn_init(void);\n\nGGML_BACKEND_API bool ggml_backend_is_zendnn(ggml_backend_t backend);\n\n// number of threads used for zendnn operations\nGGML_BACKEND_API void ggml_backend_zendnn_set_n_threads(ggml_backend_t backend_zendnn, int n_threads);\n\nGGML_BACKEND_API ggml_backend_reg_t ggml_backend_zendnn_reg(void);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/ggml.h",
    "content": "#pragma once\n\n//\n// GGML Tensor Library\n//\n// This documentation is still a work in progress.\n// If you wish some specific topics to be covered, feel free to drop a comment:\n//\n//   https://github.com/ggml-org/whisper.cpp/issues/40\n//\n// ## Overview\n//\n// This library implements:\n//\n//  - a set of tensor operations\n//  - automatic differentiation\n//  - basic optimization algorithms\n//\n// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,\n// but is not limited to, the following:\n//\n//  - linear regression\n//  - support vector machines\n//  - neural networks\n//\n// The library allows the user to define a certain function using the available tensor operations. This function\n// definition is represented internally via a computation graph. Each tensor operation in the function definition\n// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the\n// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized\n// using one of the available optimization algorithms.\n//\n// For example, here we define the function: f(x) = a*x^2 + b\n//\n//   {\n//       struct ggml_init_params params = {\n//           .mem_size   = 16*1024*1024,\n//           .mem_buffer = NULL,\n//       };\n//\n//       // memory allocation happens here\n//       struct ggml_context * ctx = ggml_init(params);\n//\n//       struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n//\n//       ggml_set_param(ctx, x); // x is an input variable\n//\n//       struct ggml_tensor * a  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n//       struct ggml_tensor * b  = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n//       struct ggml_tensor * x2 = ggml_mul(ctx, x, x);\n//       struct ggml_tensor * f  = ggml_add(ctx, ggml_mul(ctx, a, x2), b);\n//\n//       ...\n//   }\n//\n// Notice that the function definition above does not involve any actual computation. The computation is performed only\n// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:\n//\n//   {\n//       ...\n//\n//       struct ggml_cgraph * gf = ggml_new_graph(ctx);\n//       ggml_build_forward_expand(gf, f);\n//\n//       // set the input variable and parameter values\n//       ggml_set_f32(x, 2.0f);\n//       ggml_set_f32(a, 3.0f);\n//       ggml_set_f32(b, 4.0f);\n//\n//       ggml_graph_compute_with_ctx(ctx, &gf, n_threads);\n//\n//       printf(\"f = %f\\n\", ggml_get_f32_1d(f, 0));\n//\n//       ...\n//   }\n//\n// The actual computation is performed in the ggml_graph_compute() function.\n//\n// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the\n// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know\n// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory\n// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was\n// actually needed.\n//\n// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic\n// differentiation and optimization algorithms.\n//\n// The described approach allows to define the function graph once and then compute its forward or backward graphs\n// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way\n// the user can avoid the memory allocation overhead at runtime.\n//\n// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class\n// citizens, but in theory the library can be extended to support FP8 and integer data types.\n//\n// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary\n// and binary operations. Most of the available operations fall into one of these two categories. With time, it became\n// clear that the library needs to support more complex operations. The way to support these operations is not clear\n// yet, but a few examples are demonstrated in the following operations:\n//\n//   - ggml_permute()\n//   - ggml_conv_1d_1s()\n//   - ggml_conv_1d_2s()\n//\n// For each tensor operator, the library implements a forward and backward computation function. The forward function\n// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the\n// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a\n// calculus class, or watch the following video:\n//\n//   What is Automatic Differentiation?\n//   https://www.youtube.com/watch?v=wG_nF1awSSY\n//\n//\n// ## Tensor data (struct ggml_tensor)\n//\n// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of\n// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains\n// pointers to the \"source\" tensors - i.e. the tensors that were used to compute the current tensor. For example:\n//\n//   {\n//       struct ggml_tensor * c = ggml_add(ctx, a, b);\n//\n//       assert(c->src[0] == a);\n//       assert(c->src[1] == b);\n//   }\n//\n// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the\n// number of elements in each dimension (\"ne\") as well as the number of bytes (\"nb\", a.k.a. stride). This allows\n// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and\n// permutation. All tensor operations have to take the stride into account and not assume that the tensor is\n// contiguous in memory.\n//\n// The data of the tensor is accessed via the \"data\" pointer. For example:\n//\n//   {\n//       const int nx = 2;\n//       const int ny = 3;\n//\n//       struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);\n//\n//       for (int y = 0; y < ny; y++) {\n//           for (int x = 0; x < nx; x++) {\n//               *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;\n//           }\n//       }\n//\n//       ...\n//   }\n//\n// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.\n//\n// ## The matrix multiplication operator (ggml_mul_mat)\n//\n// TODO\n//\n//\n// ## Multi-threading\n//\n// TODO\n//\n//\n// ## Overview of ggml.c\n//\n// TODO\n//\n//\n// ## SIMD optimizations\n//\n// TODO\n//\n//\n// ## Debugging ggml\n//\n// TODO\n//\n//\n\n#ifdef GGML_SHARED\n#    if defined(_WIN32) && !defined(__MINGW32__)\n#        ifdef GGML_BUILD\n#            define GGML_API __declspec(dllexport) extern\n#        else\n#            define GGML_API __declspec(dllimport) extern\n#        endif\n#    else\n#        define GGML_API __attribute__ ((visibility (\"default\"))) extern\n#    endif\n#else\n#    define GGML_API extern\n#endif\n\n// TODO: support for clang\n#ifdef __GNUC__\n#    define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))\n#elif defined(_MSC_VER)\n#    define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func\n#else\n#    define GGML_DEPRECATED(func, hint) func\n#endif\n\n#ifndef __GNUC__\n#    define GGML_ATTRIBUTE_FORMAT(...)\n#elif defined(__MINGW32__) && !defined(__clang__)\n#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))\n#else\n#    define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))\n#endif\n\n#if defined(_WIN32) && !defined(_WIN32_WINNT)\n#    define _WIN32_WINNT 0x0A00\n#endif\n\n#include <stdbool.h>\n#include <stddef.h>\n#include <stdint.h>\n#include <stdio.h>\n\n#define GGML_FILE_MAGIC   0x67676d6c // \"ggml\"\n#define GGML_FILE_VERSION 2\n\n#define GGML_QNT_VERSION        2    // bump this on quantization format changes\n#define GGML_QNT_VERSION_FACTOR 1000 // do not change this\n\n#define GGML_MAX_DIMS           4\n#define GGML_MAX_PARAMS         2048\n#define GGML_MAX_SRC            10\n#define GGML_MAX_N_THREADS      512\n#define GGML_MAX_OP_PARAMS      64\n\n#ifndef GGML_MAX_NAME\n#   define GGML_MAX_NAME        64\n#endif\n\n#define GGML_DEFAULT_N_THREADS  4\n#define GGML_DEFAULT_GRAPH_SIZE 2048\n\n#if UINTPTR_MAX == 0xFFFFFFFF\n    #define GGML_MEM_ALIGN 4\n#elif defined(__EMSCRIPTEN__)\n// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm.\n// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.)\n// ref: https://github.com/ggml-org/llama.cpp/pull/18628\n    #define GGML_MEM_ALIGN 8\n#else\n    #define GGML_MEM_ALIGN 16\n#endif\n\n#define GGML_EXIT_SUCCESS 0\n#define GGML_EXIT_ABORTED 1\n\n// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726\n#define GGML_ROPE_TYPE_NORMAL 0\n#define GGML_ROPE_TYPE_NEOX   2\n#define GGML_ROPE_TYPE_MROPE  8\n#define GGML_ROPE_TYPE_VISION 24\n#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000\n\n#define GGML_MROPE_SECTIONS   4\n\n#define GGML_UNUSED(x) (void)(x)\n#ifdef __CUDACC__\ntemplate<typename... Args>\n__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}\n#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)\n#else\n#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)\n#endif // __CUDACC__\n\n#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))\n\n#ifndef NDEBUG\n#   define GGML_UNREACHABLE() do { fprintf(stderr, \"statement should be unreachable\\n\"); abort(); } while(0)\n#elif defined(__GNUC__)\n#   define GGML_UNREACHABLE() __builtin_unreachable()\n#elif defined(_MSC_VER)\n#   define GGML_UNREACHABLE() __assume(0)\n#else\n#   define GGML_UNREACHABLE() ((void) 0)\n#endif\n\n#ifdef __cplusplus\n#   define GGML_NORETURN [[noreturn]]\n#elif defined(_MSC_VER)\n#   define GGML_NORETURN __declspec(noreturn)\n#else\n#   define GGML_NORETURN _Noreturn\n#endif\n\n#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)\n#define GGML_ASSERT(x) if (!(x)) GGML_ABORT(\"GGML_ASSERT(%s) failed\", #x)\n\n// used to copy the number of elements and stride in bytes of tensors into local variables.\n// main purpose is to reduce code duplication and improve readability.\n//\n// example:\n//\n//    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);\n//    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb);\n//\n#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \\\n    const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \\\n    GGML_UNUSED(prefix##0);\n#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \\\n    GGML_TENSOR_LOCALS_1    (type, prefix, pointer, array) \\\n    const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \\\n    GGML_UNUSED(prefix##1);\n#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \\\n    GGML_TENSOR_LOCALS_2    (type, prefix, pointer, array) \\\n    const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \\\n    GGML_UNUSED(prefix##2);\n#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \\\n    GGML_TENSOR_LOCALS_3  (type, prefix, pointer, array) \\\n    const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \\\n    GGML_UNUSED(prefix##3);\n\n#define GGML_TENSOR_UNARY_OP_LOCALS \\\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n#define GGML_TENSOR_BINARY_OP_LOCALS \\\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n#define GGML_TENSOR_TERNARY_OP_LOCALS \\\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb2, src2, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n#define GGML_TENSOR_BINARY_OP_LOCALS01 \\\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb) \\\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \\\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb)\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    // Function type used in fatal error callbacks\n    typedef void (*ggml_abort_callback_t)(const char * error_message);\n\n    // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)\n    // Returns the old callback for chaining\n    GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);\n\n    GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)\n    GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);\n\n    enum ggml_status {\n        GGML_STATUS_ALLOC_FAILED = -2,\n        GGML_STATUS_FAILED = -1,\n        GGML_STATUS_SUCCESS = 0,\n        GGML_STATUS_ABORTED = 1,\n    };\n\n    // get ggml_status name string\n    GGML_API const char * ggml_status_to_string(enum ggml_status status);\n\n    // ieee 754-2008 half-precision float16\n    // todo: make this not an integral type\n    typedef uint16_t ggml_fp16_t;\n    GGML_API float       ggml_fp16_to_fp32(ggml_fp16_t);\n    GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);\n    GGML_API void        ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);\n    GGML_API void        ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);\n\n    // google brain half-precision bfloat16\n    typedef struct { uint16_t bits; } ggml_bf16_t;\n    GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);\n    GGML_API float       ggml_bf16_to_fp32(ggml_bf16_t);  // consider just doing << 16\n    GGML_API void        ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);\n    GGML_API void        ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);\n    GGML_API void        ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);\n\n    struct ggml_object;\n    struct ggml_context;\n    struct ggml_cgraph;\n\n    // NOTE: always add types at the end of the enum to keep backward compatibility\n    enum ggml_type {\n        GGML_TYPE_F32     = 0,\n        GGML_TYPE_F16     = 1,\n        GGML_TYPE_Q4_0    = 2,\n        GGML_TYPE_Q4_1    = 3,\n        // GGML_TYPE_Q4_2 = 4, support has been removed\n        // GGML_TYPE_Q4_3 = 5, support has been removed\n        GGML_TYPE_Q5_0    = 6,\n        GGML_TYPE_Q5_1    = 7,\n        GGML_TYPE_Q8_0    = 8,\n        GGML_TYPE_Q8_1    = 9,\n        GGML_TYPE_Q2_K    = 10,\n        GGML_TYPE_Q3_K    = 11,\n        GGML_TYPE_Q4_K    = 12,\n        GGML_TYPE_Q5_K    = 13,\n        GGML_TYPE_Q6_K    = 14,\n        GGML_TYPE_Q8_K    = 15,\n        GGML_TYPE_IQ2_XXS = 16,\n        GGML_TYPE_IQ2_XS  = 17,\n        GGML_TYPE_IQ3_XXS = 18,\n        GGML_TYPE_IQ1_S   = 19,\n        GGML_TYPE_IQ4_NL  = 20,\n        GGML_TYPE_IQ3_S   = 21,\n        GGML_TYPE_IQ2_S   = 22,\n        GGML_TYPE_IQ4_XS  = 23,\n        GGML_TYPE_I8      = 24,\n        GGML_TYPE_I16     = 25,\n        GGML_TYPE_I32     = 26,\n        GGML_TYPE_I64     = 27,\n        GGML_TYPE_F64     = 28,\n        GGML_TYPE_IQ1_M   = 29,\n        GGML_TYPE_BF16    = 30,\n        // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files\n        // GGML_TYPE_Q4_0_4_8 = 32,\n        // GGML_TYPE_Q4_0_8_8 = 33,\n        GGML_TYPE_TQ1_0   = 34,\n        GGML_TYPE_TQ2_0   = 35,\n        // GGML_TYPE_IQ4_NL_4_4 = 36,\n        // GGML_TYPE_IQ4_NL_4_8 = 37,\n        // GGML_TYPE_IQ4_NL_8_8 = 38,\n        GGML_TYPE_MXFP4   = 39, // MXFP4 (1 block)\n        GGML_TYPE_NVFP4   = 40, // NVFP4 (4 blocks, E4M3 scale)\n        GGML_TYPE_COUNT   = 41,\n    };\n\n    // precision\n    enum ggml_prec {\n        GGML_PREC_DEFAULT =  0, // stored as ggml_tensor.op_params, 0 by default\n        GGML_PREC_F32     = 10,\n    };\n\n    // model file types\n    enum ggml_ftype {\n        GGML_FTYPE_UNKNOWN        = -1,\n        GGML_FTYPE_ALL_F32        = 0,\n        GGML_FTYPE_MOSTLY_F16     = 1,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q4_0    = 2,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q4_1    = 3,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16\n        GGML_FTYPE_MOSTLY_Q8_0    = 7,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q5_0    = 8,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q5_1    = 9,  // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q2_K    = 10, // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q3_K    = 11, // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q4_K    = 12, // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q5_K    = 13, // except 1d tensors\n        GGML_FTYPE_MOSTLY_Q6_K    = 14, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ2_XS  = 16, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ1_S   = 18, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ4_NL  = 19, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ3_S   = 20, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ2_S   = 21, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ4_XS  = 22, // except 1d tensors\n        GGML_FTYPE_MOSTLY_IQ1_M   = 23, // except 1d tensors\n        GGML_FTYPE_MOSTLY_BF16    = 24, // except 1d tensors\n        GGML_FTYPE_MOSTLY_MXFP4   = 25, // except 1d tensors\n        GGML_FTYPE_MOSTLY_NVFP4   = 26, // except 1d tensors\n    };\n\n    // available tensor operations:\n    enum ggml_op {\n        GGML_OP_NONE = 0,\n\n        GGML_OP_DUP,\n        GGML_OP_ADD,\n        GGML_OP_ADD_ID,\n        GGML_OP_ADD1,\n        GGML_OP_ACC,\n        GGML_OP_SUB,\n        GGML_OP_MUL,\n        GGML_OP_DIV,\n        GGML_OP_SQR,\n        GGML_OP_SQRT,\n        GGML_OP_LOG,\n        GGML_OP_SIN,\n        GGML_OP_COS,\n        GGML_OP_SUM,\n        GGML_OP_SUM_ROWS,\n        GGML_OP_CUMSUM,\n        GGML_OP_MEAN,\n        GGML_OP_ARGMAX,\n        GGML_OP_COUNT_EQUAL,\n        GGML_OP_REPEAT,\n        GGML_OP_REPEAT_BACK,\n        GGML_OP_CONCAT,\n        GGML_OP_SILU_BACK,\n        GGML_OP_NORM, // normalize\n        GGML_OP_RMS_NORM,\n        GGML_OP_RMS_NORM_BACK,\n        GGML_OP_GROUP_NORM,\n        GGML_OP_L2_NORM,\n\n        GGML_OP_MUL_MAT,\n        GGML_OP_MUL_MAT_ID,\n        GGML_OP_OUT_PROD,\n\n        GGML_OP_SCALE,\n        GGML_OP_SET,\n        GGML_OP_CPY,\n        GGML_OP_CONT,\n        GGML_OP_RESHAPE,\n        GGML_OP_VIEW,\n        GGML_OP_PERMUTE,\n        GGML_OP_TRANSPOSE,\n        GGML_OP_GET_ROWS,\n        GGML_OP_GET_ROWS_BACK,\n        GGML_OP_SET_ROWS,\n        GGML_OP_DIAG,\n        GGML_OP_DIAG_MASK_INF,\n        GGML_OP_DIAG_MASK_ZERO,\n        GGML_OP_SOFT_MAX,\n        GGML_OP_SOFT_MAX_BACK,\n        GGML_OP_ROPE,\n        GGML_OP_ROPE_BACK,\n        GGML_OP_CLAMP,\n        GGML_OP_CONV_TRANSPOSE_1D,\n        GGML_OP_IM2COL,\n        GGML_OP_IM2COL_BACK,\n        GGML_OP_IM2COL_3D,\n        GGML_OP_CONV_2D,\n        GGML_OP_CONV_3D,\n        GGML_OP_CONV_2D_DW,\n        GGML_OP_CONV_TRANSPOSE_2D,\n        GGML_OP_POOL_1D,\n        GGML_OP_POOL_2D,\n        GGML_OP_POOL_2D_BACK,\n        GGML_OP_UPSCALE,\n        GGML_OP_PAD,\n        GGML_OP_PAD_REFLECT_1D,\n        GGML_OP_ROLL,\n        GGML_OP_ARANGE,\n        GGML_OP_TIMESTEP_EMBEDDING,\n        GGML_OP_ARGSORT,\n        GGML_OP_TOP_K,\n        GGML_OP_LEAKY_RELU,\n        GGML_OP_TRI,\n        GGML_OP_FILL,\n\n        GGML_OP_FLASH_ATTN_EXT,\n        GGML_OP_FLASH_ATTN_BACK,\n        GGML_OP_SSM_CONV,\n        GGML_OP_SSM_SCAN,\n        GGML_OP_WIN_PART,\n        GGML_OP_WIN_UNPART,\n        GGML_OP_GET_REL_POS,\n        GGML_OP_ADD_REL_POS,\n        GGML_OP_RWKV_WKV6,\n        GGML_OP_GATED_LINEAR_ATTN,\n        GGML_OP_RWKV_WKV7,\n        GGML_OP_SOLVE_TRI,\n        GGML_OP_GATED_DELTA_NET,\n\n        GGML_OP_UNARY,\n\n        GGML_OP_MAP_CUSTOM1,\n        GGML_OP_MAP_CUSTOM2,\n        GGML_OP_MAP_CUSTOM3,\n\n        GGML_OP_CUSTOM,\n\n        GGML_OP_CROSS_ENTROPY_LOSS,\n        GGML_OP_CROSS_ENTROPY_LOSS_BACK,\n        GGML_OP_OPT_STEP_ADAMW,\n        GGML_OP_OPT_STEP_SGD,\n\n        GGML_OP_GLU,\n\n        GGML_OP_COUNT,\n    };\n\n    enum ggml_unary_op {\n        GGML_UNARY_OP_ABS,\n        GGML_UNARY_OP_SGN,\n        GGML_UNARY_OP_NEG,\n        GGML_UNARY_OP_STEP,\n        GGML_UNARY_OP_TANH,\n        GGML_UNARY_OP_ELU,\n        GGML_UNARY_OP_RELU,\n        GGML_UNARY_OP_SIGMOID,\n        GGML_UNARY_OP_GELU,\n        GGML_UNARY_OP_GELU_QUICK,\n        GGML_UNARY_OP_SILU,\n        GGML_UNARY_OP_HARDSWISH,\n        GGML_UNARY_OP_HARDSIGMOID,\n        GGML_UNARY_OP_EXP,\n        GGML_UNARY_OP_EXPM1,\n        GGML_UNARY_OP_SOFTPLUS,\n        GGML_UNARY_OP_GELU_ERF,\n        GGML_UNARY_OP_XIELU,\n        GGML_UNARY_OP_FLOOR,\n        GGML_UNARY_OP_CEIL,\n        GGML_UNARY_OP_ROUND,\n        GGML_UNARY_OP_TRUNC,\n\n        GGML_UNARY_OP_COUNT,\n    };\n\n    enum ggml_glu_op {\n        GGML_GLU_OP_REGLU,\n        GGML_GLU_OP_GEGLU,\n        GGML_GLU_OP_SWIGLU,\n        GGML_GLU_OP_SWIGLU_OAI,\n        GGML_GLU_OP_GEGLU_ERF,\n        GGML_GLU_OP_GEGLU_QUICK,\n\n        GGML_GLU_OP_COUNT,\n    };\n\n    enum ggml_object_type {\n        GGML_OBJECT_TYPE_TENSOR,\n        GGML_OBJECT_TYPE_GRAPH,\n        GGML_OBJECT_TYPE_WORK_BUFFER\n    };\n\n    enum ggml_log_level {\n        GGML_LOG_LEVEL_NONE  = 0,\n        GGML_LOG_LEVEL_DEBUG = 1,\n        GGML_LOG_LEVEL_INFO  = 2,\n        GGML_LOG_LEVEL_WARN  = 3,\n        GGML_LOG_LEVEL_ERROR = 4,\n        GGML_LOG_LEVEL_CONT  = 5, // continue previous log\n    };\n\n    // this tensor...\n    enum ggml_tensor_flag {\n        GGML_TENSOR_FLAG_INPUT   =  1, // ...is an input for the GGML compute graph\n        GGML_TENSOR_FLAG_OUTPUT  =  2, // ...is an output for the GGML compute graph\n        GGML_TENSOR_FLAG_PARAM   =  4, // ...contains trainable parameters\n        GGML_TENSOR_FLAG_LOSS    =  8, // ...defines loss for numerical optimization (multiple loss tensors add up)\n        GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed\n    };\n\n    enum ggml_tri_type {\n        GGML_TRI_TYPE_UPPER_DIAG = 0,\n        GGML_TRI_TYPE_UPPER      = 1,\n        GGML_TRI_TYPE_LOWER_DIAG = 2,\n        GGML_TRI_TYPE_LOWER      = 3\n    };\n\n    struct ggml_init_params {\n        // memory pool\n        size_t mem_size;   // bytes\n        void * mem_buffer; // if NULL, memory will be allocated internally\n        bool   no_alloc;   // don't allocate memory for the tensor data\n    };\n\n    // n-dimensional tensor\n    struct ggml_tensor {\n        enum ggml_type type;\n\n        struct ggml_backend_buffer * buffer;\n\n        int64_t ne[GGML_MAX_DIMS]; // number of elements\n        size_t  nb[GGML_MAX_DIMS]; // stride in bytes:\n                                   // nb[0] = ggml_type_size(type)\n                                   // nb[1] = nb[0]   * (ne[0] / ggml_blck_size(type)) + padding\n                                   // nb[i] = nb[i-1] * ne[i-1]\n\n        // compute data\n        enum ggml_op op;\n\n        // op params - allocated as int32_t for alignment\n        int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n\n        int32_t flags;\n\n        struct ggml_tensor * src[GGML_MAX_SRC];\n\n        // source tensor and offset for views\n        struct ggml_tensor * view_src;\n        size_t               view_offs;\n\n        void * data;\n\n        char name[GGML_MAX_NAME];\n\n        void * extra; // extra things e.g. for ggml-cuda.cu\n\n        char padding[8];\n    };\n\n    static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);\n\n    // Abort callback\n    // If not NULL, called before ggml computation\n    // If it returns true, the computation is aborted\n    typedef bool (*ggml_abort_callback)(void * data);\n\n\n    //\n    // GUID\n    //\n\n    // GUID types\n    typedef uint8_t ggml_guid[16];\n    typedef ggml_guid * ggml_guid_t;\n\n    GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);\n\n    // misc\n\n    GGML_API const char * ggml_version(void);\n    GGML_API const char * ggml_commit(void);\n\n    GGML_API void    ggml_time_init(void); // call this once at the beginning of the program\n    GGML_API int64_t ggml_time_ms(void);\n    GGML_API int64_t ggml_time_us(void);\n    GGML_API int64_t ggml_cycles(void);\n    GGML_API int64_t ggml_cycles_per_ms(void);\n\n    // accepts a UTF-8 path, even on Windows\n    GGML_API FILE *  ggml_fopen(const char * fname, const char * mode);\n\n    GGML_API void    ggml_print_object (const struct ggml_object * obj);\n    GGML_API void    ggml_print_objects(const struct ggml_context * ctx);\n\n    GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);\n    GGML_API int64_t ggml_nrows     (const struct ggml_tensor * tensor);\n    GGML_API size_t  ggml_nbytes    (const struct ggml_tensor * tensor);\n    GGML_API size_t  ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN\n\n    GGML_API int64_t ggml_blck_size(enum ggml_type type);\n    GGML_API size_t  ggml_type_size(enum ggml_type type);             // size in bytes for all elements in a block\n    GGML_API size_t  ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row\n\n    GGML_DEPRECATED(\n    GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float\n    \"use ggml_row_size() instead\");\n\n    GGML_API const char * ggml_type_name(enum ggml_type type);\n    GGML_API const char * ggml_op_name  (enum ggml_op   op);\n    GGML_API const char * ggml_op_symbol(enum ggml_op   op);\n\n    GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);\n    GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);\n    GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name\n\n    GGML_API size_t  ggml_element_size(const struct ggml_tensor * tensor);\n\n    GGML_API bool    ggml_is_quantized(enum ggml_type type);\n\n    // TODO: temporary until model loading of ggml examples is refactored\n    GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);\n\n    GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_permuted  (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_empty     (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_view      (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_scalar    (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_vector    (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_matrix    (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_3d        (const struct ggml_tensor * tensor);\n    GGML_API int  ggml_n_dims       (const struct ggml_tensor * tensor); // returns 1 for scalars\n\n    // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)\n    GGML_API bool ggml_is_contiguous  (const struct ggml_tensor * tensor);\n    GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()\n    GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1\n    GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2\n\n    // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)\n    GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);\n\n    // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN\n    GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);\n\n    // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements\n    GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);\n\n    GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n    GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n\n    GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);\n\n    // use this to compute the memory overhead of a tensor\n    GGML_API size_t ggml_tensor_overhead(void);\n\n    GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);\n\n    // main\n\n    GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);\n    GGML_API void                  ggml_reset(struct ggml_context * ctx);\n    GGML_API void                  ggml_free (struct ggml_context * ctx);\n\n    GGML_API size_t  ggml_used_mem(const struct ggml_context * ctx);\n\n    GGML_API bool    ggml_get_no_alloc(struct ggml_context * ctx);\n    GGML_API void    ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);\n\n    GGML_API void *  ggml_get_mem_buffer     (const struct ggml_context * ctx);\n    GGML_API size_t  ggml_get_mem_size       (const struct ggml_context * ctx);\n    GGML_API size_t  ggml_get_max_tensor_size(const struct ggml_context * ctx);\n\n    GGML_API struct ggml_tensor * ggml_new_tensor(\n            struct ggml_context * ctx,\n            enum   ggml_type type,\n            int    n_dims,\n            const int64_t *ne);\n\n    GGML_API struct ggml_tensor * ggml_new_tensor_1d(\n            struct ggml_context * ctx,\n            enum   ggml_type type,\n            int64_t ne0);\n\n    GGML_API struct ggml_tensor * ggml_new_tensor_2d(\n            struct ggml_context * ctx,\n            enum   ggml_type type,\n            int64_t ne0,\n            int64_t ne1);\n\n    GGML_API struct ggml_tensor * ggml_new_tensor_3d(\n            struct ggml_context * ctx,\n            enum   ggml_type type,\n            int64_t ne0,\n            int64_t ne1,\n            int64_t ne2);\n\n    GGML_API struct ggml_tensor * ggml_new_tensor_4d(\n            struct ggml_context * ctx,\n            enum   ggml_type type,\n            int64_t ne0,\n            int64_t ne1,\n            int64_t ne2,\n            int64_t ne3);\n\n    GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);\n\n    GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);\n    GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);\n\n    // Context tensor enumeration and lookup\n    GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);\n    GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);\n    GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);\n\n    // Converts a flat index into coordinates\n    GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);\n\n    GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);\n    GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);\n\n    GGML_API void *  ggml_get_data    (const struct ggml_tensor * tensor);\n    GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);\n\n    GGML_API const char *         ggml_get_name   (const struct ggml_tensor * tensor);\n    GGML_API struct ggml_tensor * ggml_set_name   (      struct ggml_tensor * tensor, const char * name);\n    GGML_ATTRIBUTE_FORMAT(2, 3)\n    GGML_API struct ggml_tensor * ggml_format_name(      struct ggml_tensor * tensor, const char * fmt, ...);\n\n    // Tensor flags\n    GGML_API void ggml_set_input(struct ggml_tensor * tensor);\n    GGML_API void ggml_set_output(struct ggml_tensor * tensor);\n    GGML_API void ggml_set_param(struct ggml_tensor * tensor);\n    GGML_API void ggml_set_loss(struct ggml_tensor * tensor);\n\n    //\n    // operations on tensors with backpropagation\n    //\n\n    GGML_API struct ggml_tensor * ggml_dup(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_dup_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_add(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_add_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_add_cast(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            enum   ggml_type      type);\n\n    // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]\n    GGML_API struct ggml_tensor * ggml_add_id(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * ids);\n\n    GGML_API struct ggml_tensor * ggml_add1(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_add1_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // dst = a\n    // view(dst, nb1, nb2, nb3, offset) += b\n    // return dst\n    GGML_API struct ggml_tensor * ggml_acc(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                nb2,\n            size_t                nb3,\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_acc_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                nb2,\n            size_t                nb3,\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_sub(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_sub_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_mul(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_mul_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_div(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_div_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_sqr(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sqr_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sqrt(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sqrt_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_log(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_log_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_expm1(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_expm1_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_softplus(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_softplus_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sin(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sin_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_cos(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_cos_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // return scalar\n    GGML_API struct ggml_tensor * ggml_sum(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]\n    GGML_API struct ggml_tensor * ggml_sum_rows(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_cumsum(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a);\n\n    // mean along rows\n    GGML_API struct ggml_tensor * ggml_mean(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // argmax along rows\n    GGML_API struct ggml_tensor * ggml_argmax(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // count number of equal elements in a and b\n    GGML_API struct ggml_tensor * ggml_count_equal(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // if a is the same shape as b, and a is not parameter, return a\n    // otherwise, return a new tensor: repeat(a) to fit in b\n    GGML_API struct ggml_tensor * ggml_repeat(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // repeat a to the specified shape\n    GGML_API struct ggml_tensor * ggml_repeat_4d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n                       int64_t    ne0,\n                       int64_t    ne1,\n                       int64_t    ne2,\n                       int64_t    ne3);\n\n    // sums repetitions in a into shape of b\n    GGML_API struct ggml_tensor * ggml_repeat_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride\n\n    // concat a and b along dim\n    // used in stable-diffusion\n    GGML_API struct ggml_tensor * ggml_concat(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   dim);\n\n    GGML_API struct ggml_tensor * ggml_abs(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_abs_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sgn(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sgn_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_neg(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_neg_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_step(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_step_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_tanh(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_tanh_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_elu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_elu_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_relu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_leaky_relu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a, float negative_slope, bool inplace);\n\n    GGML_API struct ggml_tensor * ggml_relu_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sigmoid(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_sigmoid_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_gelu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_gelu_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // GELU using erf (error function) when possible\n    // some backends may fallback to approximation based on Abramowitz and Stegun formula\n    GGML_API struct ggml_tensor * ggml_gelu_erf(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_gelu_quick(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_silu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_silu_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // a - x\n    // b - dy\n    GGML_API struct ggml_tensor * ggml_silu_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // hardswish(x) = x * relu6(x + 3) / 6\n    GGML_API struct ggml_tensor * ggml_hardswish(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // hardsigmoid(x) = relu6(x + 3) / 6\n    GGML_API struct ggml_tensor * ggml_hardsigmoid(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_exp(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_exp_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_floor(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_floor_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_ceil(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_ceil_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_round(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_round_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n     /**\n     * Truncates the fractional part of each element in the tensor (towards zero).\n     * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0\n     * Similar to std::trunc in C/C++.\n     */\n\n    GGML_API struct ggml_tensor * ggml_trunc(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_trunc_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n\n\n    // xIELU activation function\n    // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)\n    // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions\n    // that constrain the positive and negative source alpha values respectively\n    GGML_API struct ggml_tensor * ggml_xielu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float alpha_n,\n            float alpha_p,\n            float beta,\n            float eps);\n\n    // gated linear unit ops\n    // A: n columns, r rows,\n    // result is n / 2 columns, r rows,\n    // expects gate in second half of row, unless swapped is true\n    GGML_API struct ggml_tensor * ggml_glu(\n            struct ggml_context * ctx,\n             struct ggml_tensor * a,\n             enum ggml_glu_op     op,\n             bool                 swapped);\n\n    GGML_API struct ggml_tensor * ggml_reglu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_reglu_swapped(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu_swapped(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_swiglu(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_swiglu_swapped(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu_erf(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu_quick(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // A: n columns, r rows,\n    // B: n columns, r rows,\n    GGML_API struct ggml_tensor * ggml_glu_split(\n            struct ggml_context * ctx,\n             struct ggml_tensor * a,\n             struct ggml_tensor * b,\n             enum ggml_glu_op     op);\n\n    GGML_API struct ggml_tensor * ggml_reglu_split(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_geglu_split(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_swiglu_split(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_geglu_erf_split(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_geglu_quick_split(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    GGML_API struct ggml_tensor * ggml_swiglu_oai(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            float                 alpha,\n            float                 limit);\n\n    // normalize along rows\n    GGML_API struct ggml_tensor * ggml_norm(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    GGML_API struct ggml_tensor * ggml_norm_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    GGML_API struct ggml_tensor * ggml_rms_norm(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    GGML_API struct ggml_tensor * ggml_rms_norm_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    // group normalize along ne0*ne1*n_groups\n    // used in stable-diffusion\n    GGML_API struct ggml_tensor * ggml_group_norm(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_groups,\n            float                 eps);\n\n    GGML_API struct ggml_tensor * ggml_group_norm_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_groups,\n            float                 eps);\n\n    // l2 normalize along rows\n    // used in rwkv v7\n    GGML_API struct ggml_tensor * ggml_l2_norm(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    GGML_API struct ggml_tensor * ggml_l2_norm_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 eps);\n\n    // a - x\n    // b - dy\n    GGML_API struct ggml_tensor * ggml_rms_norm_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            float                 eps);\n\n    // A: k columns, n rows => [ne03, ne02, n, k]\n    // B: k columns, m rows  (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]\n    // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]\n    GGML_API struct ggml_tensor * ggml_mul_mat(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // change the precision of a matrix multiplication\n    // set to GGML_PREC_F32 for higher precision (useful for phi-2)\n    GGML_API void ggml_mul_mat_set_prec(\n            struct ggml_tensor * a,\n            enum ggml_prec       prec);\n\n    // indirect matrix multiplication\n    GGML_API struct ggml_tensor * ggml_mul_mat_id(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * as,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * ids);\n\n    // A: m columns, n rows,\n    // B: p columns, n rows,\n    // result is m columns, p rows\n    GGML_API struct ggml_tensor * ggml_out_prod(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    //\n    // operations on tensors without backpropagation\n    //\n\n    GGML_API struct ggml_tensor * ggml_scale(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 s);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_scale_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 s);\n\n    // x = s * a + b\n    GGML_API struct ggml_tensor * ggml_scale_bias(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a,\n        float                 s,\n        float                 b);\n\n    GGML_API struct ggml_tensor * ggml_scale_bias_inplace(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a,\n        float                 s,\n        float                 b);\n\n    // b -> view(a,offset,nb1,nb2,3), return modified a\n    GGML_API struct ggml_tensor * ggml_set(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                nb2,\n            size_t                nb3,\n            size_t                offset); // in bytes\n\n    // b -> view(a,offset,nb1,nb2,3), return view(a)\n    GGML_API struct ggml_tensor * ggml_set_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                nb2,\n            size_t                nb3,\n            size_t                offset); // in bytes\n\n    GGML_API struct ggml_tensor * ggml_set_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                offset); // in bytes\n\n    GGML_API struct ggml_tensor * ggml_set_1d_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                offset); // in bytes\n\n    // b -> view(a,offset,nb1,nb2,3), return modified a\n    GGML_API struct ggml_tensor * ggml_set_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                offset); // in bytes\n\n    // b -> view(a,offset,nb1,nb2,3), return view(a)\n    GGML_API struct ggml_tensor * ggml_set_2d_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            size_t                nb1,\n            size_t                offset); // in bytes\n\n    // a -> b, return view(b)\n    GGML_API struct ggml_tensor * ggml_cpy(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // note: casting from f32 to i32 will discard the fractional part\n    GGML_API struct ggml_tensor * ggml_cast(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            enum   ggml_type      type);\n\n    // make contiguous\n    GGML_API struct ggml_tensor * ggml_cont(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // make contiguous, with new shape\n    GGML_API struct ggml_tensor * ggml_cont_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0);\n\n    GGML_API struct ggml_tensor * ggml_cont_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1);\n\n    GGML_API struct ggml_tensor * ggml_cont_3d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2);\n\n    GGML_API struct ggml_tensor * ggml_cont_4d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            int64_t               ne3);\n\n    // return view(a), b specifies the new shape\n    // TODO: when we start computing gradient, make a copy instead of view\n    GGML_API struct ggml_tensor * ggml_reshape(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // return view(a)\n    // TODO: when we start computing gradient, make a copy instead of view\n    GGML_API struct ggml_tensor * ggml_reshape_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0);\n\n    GGML_API struct ggml_tensor * ggml_reshape_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1);\n\n    // return view(a)\n    // TODO: when we start computing gradient, make a copy instead of view\n    GGML_API struct ggml_tensor * ggml_reshape_3d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2);\n\n    GGML_API struct ggml_tensor * ggml_reshape_4d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            int64_t               ne3);\n\n    // offset in bytes\n    GGML_API struct ggml_tensor * ggml_view_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_view_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            size_t                nb1, // row stride in bytes\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_view_3d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            size_t                nb1, // row   stride in bytes\n            size_t                nb2, // slice stride in bytes\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_view_4d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            int64_t               ne3,\n            size_t                nb1, // row   stride in bytes\n            size_t                nb2, // slice stride in bytes\n            size_t                nb3,\n            size_t                offset);\n\n    GGML_API struct ggml_tensor * ggml_permute(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   axis0,\n            int                   axis1,\n            int                   axis2,\n            int                   axis3);\n\n    // alias for ggml_permute(ctx, a, 1, 0, 2, 3)\n    GGML_API struct ggml_tensor * ggml_transpose(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // supports 4D a:\n    // a     [n_embd, ne1, ne2, ne3]\n    // b I32 [n_rows, ne2, ne3, 1]\n    //\n    // return [n_embd, n_rows, ne2, ne3]\n    GGML_API struct ggml_tensor * ggml_get_rows(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // data\n            struct ggml_tensor  * b); // row indices\n\n    GGML_API struct ggml_tensor * ggml_get_rows_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // gradients of ggml_get_rows result\n            struct ggml_tensor  * b,  // row indices\n            struct ggml_tensor  * c); // data for ggml_get_rows, only used for its shape\n\n    // a TD  [n_embd, ne1,    ne2,    ne3]\n    // b TS  [n_embd, n_rows, ne02,   ne03] | ne02 == ne2, ne03 == ne3\n    // c I64 [n_rows, ne11,   ne12,   1]    | c[i] in [0, ne1)\n    //\n    // undefined behavior if destination rows overlap\n    //\n    // broadcast:\n    //   ne2 % ne11 == 0\n    //   ne3 % ne12 == 0\n    //\n    // return view(a)\n    GGML_API struct ggml_tensor * ggml_set_rows(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // destination\n            struct ggml_tensor  * b,  // source\n            struct ggml_tensor  * c); // row indices\n\n    GGML_API struct ggml_tensor * ggml_diag(\n        struct ggml_context     * ctx,\n        struct ggml_tensor      * a);\n\n    // set elements above the diagonal to -INF\n    GGML_API struct ggml_tensor * ggml_diag_mask_inf(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_past);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_past);\n\n    // set elements above the diagonal to 0\n    GGML_API struct ggml_tensor * ggml_diag_mask_zero(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_past);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   n_past);\n\n    GGML_API struct ggml_tensor * ggml_soft_max(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_soft_max_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a);\n\n    // a    [ne0, ne01, ne02, ne03]\n    // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional\n    //\n    // broadcast:\n    //   ne02 % ne12 == 0\n    //   ne03 % ne13 == 0\n    //\n    // fused soft_max(a*scale + mask*(ALiBi slope))\n    // max_bias = 0.0f for no ALiBi\n    GGML_API struct ggml_tensor * ggml_soft_max_ext(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * mask,\n            float                 scale,\n            float                 max_bias);\n\n    GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * mask,\n            float                 scale,\n            float                 max_bias);\n\n    GGML_API void ggml_soft_max_add_sinks(\n            struct ggml_tensor * a,\n            struct ggml_tensor * sinks);\n\n    GGML_API struct ggml_tensor * ggml_soft_max_ext_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            float                 scale,\n            float                 max_bias);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            float                 scale,\n            float                 max_bias);\n\n    // rotary position embedding\n    // if (mode & 1) - skip n_past elements (NOT SUPPORTED)\n    // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style\n    //\n    // b is an int32 vector with size a->ne[2], it contains the positions\n    GGML_API struct ggml_tensor * ggml_rope(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   n_dims,\n            int                   mode);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_rope_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   n_dims,\n            int                   mode);\n\n    // custom RoPE\n    // c is freq factors (e.g. phi3-128k), (optional)\n    GGML_API struct ggml_tensor * ggml_rope_ext(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * c,\n            int                   n_dims,\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n    GGML_API struct ggml_tensor * ggml_rope_multi(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * c,\n            int                   n_dims,\n            int                   sections[GGML_MROPE_SECTIONS],\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_rope_ext_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * c,\n            int                   n_dims,\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n    GGML_API struct ggml_tensor * ggml_rope_multi_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * c,\n            int                   n_dims,\n            int                   sections[GGML_MROPE_SECTIONS],\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   n_dims,\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow),\n        \"use ggml_rope_ext instead\");\n\n    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   n_dims,\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow),\n        \"use ggml_rope_ext_inplace instead\");\n\n    // compute correction dims for YaRN RoPE scaling\n    GGML_API void ggml_rope_yarn_corr_dims(\n        int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);\n\n    // rotary position embedding backward, i.e compute dx from dy\n    // a - dy\n    GGML_API struct ggml_tensor * ggml_rope_ext_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a, // gradients of ggml_rope result\n            struct ggml_tensor  * b, // positions\n            struct ggml_tensor  * c, // freq factors\n            int                   n_dims,\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n    GGML_API struct ggml_tensor * ggml_rope_multi_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * c,\n            int                   n_dims,\n            int                   sections[4],\n            int                   mode,\n            int                   n_ctx_orig,\n            float                 freq_base,\n            float                 freq_scale,\n            float                 ext_factor,\n            float                 attn_factor,\n            float                 beta_fast,\n            float                 beta_slow);\n\n\n    // clamp\n    // in-place, returns view(a)\n    GGML_API struct ggml_tensor * ggml_clamp(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 min,\n            float                 max);\n\n    // im2col\n    // converts data into a format that effectively results in a convolution when combined with matrix multiplication\n    GGML_API struct ggml_tensor * ggml_im2col(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // convolution kernel\n            struct ggml_tensor  * b,  // data\n            int                   s0, // stride dimension 0\n            int                   s1, // stride dimension 1\n            int                   p0, // padding dimension 0\n            int                   p1, // padding dimension 1\n            int                   d0, // dilation dimension 0\n            int                   d1, // dilation dimension 1\n            bool                  is_2D,\n            enum ggml_type        dst_type);\n\n    GGML_API struct ggml_tensor * ggml_im2col_back(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a,  // convolution kernel\n        struct ggml_tensor  * b,  // gradient of im2col output\n        int64_t             * ne, // shape of im2col input\n        int                   s0, // stride dimension 0\n        int                   s1, // stride dimension 1\n        int                   p0, // padding dimension 0\n        int                   p1, // padding dimension 1\n        int                   d0, // dilation dimension 0\n        int                   d1, // dilation dimension 1\n        bool                  is_2D);\n\n    GGML_API struct ggml_tensor * ggml_conv_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel\n            struct ggml_tensor  * b,   // data\n            int                   s0,  // stride\n            int                   p0,  // padding\n            int                   d0); // dilation\n\n    // conv_1d with padding = half\n    // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)\n    GGML_API struct ggml_tensor* ggml_conv_1d_ph(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // convolution kernel\n            struct ggml_tensor  * b,  // data\n            int                   s,  // stride\n            int                   d); // dilation\n\n    // depthwise\n    // TODO: this is very likely wrong for some cases! - needs more testing\n    GGML_API struct ggml_tensor * ggml_conv_1d_dw(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel\n            struct ggml_tensor  * b,   // data\n            int                   s0,  // stride\n            int                   p0,  // padding\n            int                   d0); // dilation\n\n    GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel\n            struct ggml_tensor  * b,   // data\n            int                   s0,  // stride\n            int                   d0); // dilation\n\n    GGML_API struct ggml_tensor * ggml_conv_transpose_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel\n            struct ggml_tensor  * b,   // data\n            int                   s0,  // stride\n            int                   p0,  // padding\n            int                   d0); // dilation\n\n    GGML_API struct ggml_tensor * ggml_conv_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel\n            struct ggml_tensor  * b,   // data\n            int                   s0,  // stride dimension 0\n            int                   s1,  // stride dimension 1\n            int                   p0,  // padding dimension 0\n            int                   p1,  // padding dimension 1\n            int                   d0,  // dilation dimension 0\n            int                   d1); // dilation dimension 1\n\n    GGML_API struct ggml_tensor * ggml_im2col_3d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int64_t               IC,\n            int                   s0, // stride width\n            int                   s1, // stride height\n            int                   s2, // stride depth\n            int                   p0, // padding width\n            int                   p1, // padding height\n            int                   p2, // padding depth\n            int                   d0, // dilation width\n            int                   d1, // dilation height\n            int                   d2, // dilation depth\n            enum ggml_type        dst_type);\n\n    // a: [OC*IC, KD, KH, KW]\n    // b: [N*IC, ID, IH, IW]\n    // result: [N*OC, OD, OH, OW]\n    GGML_API struct ggml_tensor * ggml_conv_3d(\n                struct ggml_context * ctx,\n                struct ggml_tensor  * a,\n                struct ggml_tensor  * b,\n                int64_t               IC,\n                int                   s0, // stride width\n                int                   s1, // stride height\n                int                   s2, // stride depth\n                int                   p0, // padding width\n                int                   p1, // padding height\n                int                   p2, // padding depth\n                int                   d0, // dilation width\n                int                   d1, // dilation height\n                int                   d2  // dilation depth\n        );\n\n    // kernel size is a->ne[0] x a->ne[1]\n    // stride is equal to kernel size\n    // padding is zero\n    // example:\n    // a:     16   16    3  768\n    // b:   1024 1024    3    1\n    // res:   64   64  768    1\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // kernel size is a->ne[0] x a->ne[1]\n    // stride is 1\n    // padding is half\n    // example:\n    // a:      3    3    256  256\n    // b:     64   64    256    1\n    // res:   64   64    256    1\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b);\n\n    // depthwise (via im2col and mul_mat)\n    GGML_API struct ggml_tensor * ggml_conv_2d_dw(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // convolution kernel\n            struct ggml_tensor  * b,  // data\n            int                  s0,  // stride dimension 0\n            int                  s1,  // stride dimension 1\n            int                  p0,  // padding dimension 0\n            int                  p1,  // padding dimension 1\n            int                  d0,  // dilation dimension 0\n            int                  d1); // dilation dimension 1\n\n    // Depthwise 2D convolution\n    // may be faster than ggml_conv_2d_dw, but not available in all backends\n    // a:   KW    KH    1    C    convolution kernel\n    // b:   W     H     C    N    input data\n    // res: W_out H_out C    N\n    GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   stride0,\n            int                   stride1,\n            int                   pad0,\n            int                   pad1,\n            int                   dilation0,\n            int                   dilation1);\n\n    GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            int                   stride);\n\n    GGML_API struct ggml_tensor * ggml_conv_2d_direct(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // convolution kernel [KW, KH, IC, OC]\n            struct ggml_tensor  * b,   // input data [W, H, C, N]\n            int                   s0,  // stride dimension 0\n            int                   s1,  // stride dimension 1\n            int                   p0,  // padding dimension 0\n            int                   p1,  // padding dimension 1\n            int                   d0,  // dilation dimension 0\n            int                   d1); // dilation dimension 1\n\n    GGML_API struct ggml_tensor * ggml_conv_3d_direct(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,   // kernel [KW, KH, KD, IC * OC]\n            struct ggml_tensor  * b,   // input  [W, H, D, C * N]\n            int                   s0,  // stride\n            int                   s1,\n            int                   s2,\n            int                   p0,  // padding\n            int                   p1,\n            int                   p2,\n            int                   d0,  // dilation\n            int                   d1,\n            int                   d2,\n            int                   n_channels,\n            int                   n_batch,\n            int                   n_channels_out);\n\n    enum ggml_op_pool {\n        GGML_OP_POOL_MAX,\n        GGML_OP_POOL_AVG,\n        GGML_OP_POOL_COUNT,\n    };\n\n    GGML_API struct ggml_tensor * ggml_pool_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            enum ggml_op_pool     op,\n            int                   k0, // kernel size\n            int                   s0, // stride\n            int                   p0); // padding\n\n    // the result will have 2*p0 padding for the first dimension\n    // and 2*p1 padding for the second dimension\n    GGML_API struct ggml_tensor * ggml_pool_2d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            enum ggml_op_pool     op,\n            int                   k0,\n            int                   k1,\n            int                   s0,\n            int                   s1,\n            float                 p0,\n            float                 p1);\n\n    GGML_API struct ggml_tensor * ggml_pool_2d_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * af, // \"a\"/input used in forward pass\n            enum ggml_op_pool     op,\n            int                   k0,\n            int                   k1,\n            int                   s0,\n            int                   s1,\n            float                 p0,\n            float                 p1);\n\n    enum ggml_scale_mode {\n        GGML_SCALE_MODE_NEAREST  = 0,\n        GGML_SCALE_MODE_BILINEAR = 1,\n        GGML_SCALE_MODE_BICUBIC  = 2,\n\n        GGML_SCALE_MODE_COUNT\n    };\n\n    enum ggml_scale_flag {\n        GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8),\n        GGML_SCALE_FLAG_ANTIALIAS     = (1 << 9),\n    };\n\n    // interpolate\n    // multiplies ne0 and ne1 by scale factor\n    GGML_API struct ggml_tensor * ggml_upscale(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   scale_factor,\n            enum ggml_scale_mode  mode);\n\n    // interpolate\n    // interpolate scale to specified dimensions\n    GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   ne0,\n            int                   ne1,\n            int                   ne2,\n            int                   ne3,\n            enum ggml_scale_mode  mode),\n        \"use ggml_interpolate instead\");\n\n    // Up- or downsamples the input to the specified size.\n    // 2D scale modes (eg. bilinear) are applied to the first two dimensions.\n    GGML_API struct ggml_tensor * ggml_interpolate(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            int64_t               ne3,\n            uint32_t              mode); // ggml_scale_mode [ | ggml_scale_flag...]\n\n    // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]\n    GGML_API struct ggml_tensor * ggml_pad(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                  p0,\n            int                  p1,\n            int                  p2,\n            int                  p3);\n\n    // pad each dimension with values on the other side of the torus (looping around)\n    GGML_API struct ggml_tensor * ggml_pad_circular(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   p0,\n            int                   p1,\n            int                   p2,\n            int                   p3);\n\n    GGML_API struct ggml_tensor * ggml_pad_ext(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                  lp0,\n            int                  rp0,\n            int                  lp1,\n            int                  rp1,\n            int                  lp2,\n            int                  rp2,\n            int                  lp3,\n            int                  rp3\n            );\n\n    // pad each dimension with values on the other side of the torus (looping around)\n    GGML_API struct ggml_tensor * ggml_pad_ext_circular(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   lp0,\n            int                   rp0,\n            int                   lp1,\n            int                   rp1,\n            int                   lp2,\n            int                   rp2,\n            int                   lp3,\n            int                   rp3);\n\n    // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]\n    GGML_API struct ggml_tensor * ggml_pad_reflect_1d(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   p0,\n            int                   p1);\n\n    // Move tensor elements by an offset given for each dimension. Elements that\n    // are shifted beyond the last position are wrapped around to the beginning.\n    GGML_API struct ggml_tensor * ggml_roll(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   shift0,\n            int                   shift1,\n            int                   shift2,\n            int                   shift3);\n\n    // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing\n    // zeroes everywhere outside the masked area\n    GGML_API struct ggml_tensor * ggml_tri(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            enum ggml_tri_type    type);\n\n    // Fill tensor a with constant c\n    GGML_API struct ggml_tensor * ggml_fill(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 c);\n\n    GGML_API struct ggml_tensor * ggml_fill_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            float                 c);\n\n    // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151\n    // timesteps: [N,]\n    // return: [N, dim]\n    GGML_API struct ggml_tensor * ggml_timestep_embedding(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * timesteps,\n            int                   dim,\n            int                   max_period);\n\n    // sort rows\n    enum ggml_sort_order {\n        GGML_SORT_ORDER_ASC,\n        GGML_SORT_ORDER_DESC,\n    };\n\n    GGML_API struct ggml_tensor * ggml_argsort(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            enum ggml_sort_order  order);\n\n    // similar to ggml_top_k but implemented as `argsort` + `view`\n    GGML_API struct ggml_tensor * ggml_argsort_top_k(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   k);\n\n    // top k elements per row\n    // note: the resulting top k indices are in no particular order\n    GGML_API struct ggml_tensor * ggml_top_k(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   k);\n\n    GGML_API struct ggml_tensor * ggml_arange(\n            struct ggml_context * ctx,\n            float                 start,\n            float                 stop,\n            float                 step);\n\n    // q:    [n_embd_k, n_batch, n_head,    ne3 ]\n    // k:    [n_embd_k, n_kv,    n_head_kv, ne3 ]\n    // v:    [n_embd_v, n_kv,    n_head_kv, ne3 ] !! not transposed !!\n    // mask: [n_kv,     n_batch, ne32,      ne33]\n    // res:  [n_embd_v, n_head,  n_batch,   ne3 ] !! permuted !!\n    //\n    // broadcast:\n    //   n_head % n_head_kv == 0\n    //   n_head % ne32      == 0\n    //   ne3    % ne33      == 0\n    //\n    GGML_API struct ggml_tensor * ggml_flash_attn_ext(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * q,\n            struct ggml_tensor  * k,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * mask,\n            float                 scale,\n            float                 max_bias,\n            float                 logit_softcap);\n\n    GGML_API void ggml_flash_attn_ext_set_prec(\n            struct ggml_tensor * a,\n            enum ggml_prec       prec);\n\n    GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(\n            const struct ggml_tensor * a);\n\n    GGML_API void ggml_flash_attn_ext_add_sinks(\n            struct ggml_tensor * a,\n            struct ggml_tensor * sinks);\n\n    // TODO: needs to be adapted to ggml_flash_attn_ext\n    GGML_API struct ggml_tensor * ggml_flash_attn_back(\n           struct ggml_context * ctx,\n           struct ggml_tensor  * q,\n           struct ggml_tensor  * k,\n           struct ggml_tensor  * v,\n           struct ggml_tensor  * d,\n           bool                  masked);\n\n    GGML_API struct ggml_tensor * ggml_ssm_conv(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * sx,\n            struct ggml_tensor  * c);\n\n    GGML_API struct ggml_tensor * ggml_ssm_scan(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * s,\n            struct ggml_tensor  * x,\n            struct ggml_tensor  * dt,\n            struct ggml_tensor  * A,\n            struct ggml_tensor  * B,\n            struct ggml_tensor  * C,\n            struct ggml_tensor  * ids);\n\n    // partition into non-overlapping windows with padding if needed\n    // example:\n    // a:   768   64   64    1\n    // w:    14\n    // res: 768   14   14    25\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_win_part(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   w);\n\n    // reverse of ggml_win_part\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_win_unpart(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   w0,\n            int                   h0,\n            int                   w);\n\n    GGML_API struct ggml_tensor * ggml_unary(\n            struct ggml_context * ctx,\n             struct ggml_tensor * a,\n             enum ggml_unary_op op);\n\n    GGML_API struct ggml_tensor * ggml_unary_inplace(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a,\n        enum ggml_unary_op op);\n\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_get_rel_pos(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            int                   qh,\n            int                   kh);\n\n    // used in sam\n    GGML_API struct ggml_tensor * ggml_add_rel_pos(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * pw,\n            struct ggml_tensor  * ph);\n\n    GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * pw,\n            struct ggml_tensor  * ph);\n\n    GGML_API struct ggml_tensor * ggml_rwkv_wkv6(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * k,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * r,\n            struct ggml_tensor  * tf,\n            struct ggml_tensor  * td,\n            struct ggml_tensor  * state);\n\n    GGML_API struct ggml_tensor * ggml_gated_linear_attn(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * k,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * q,\n            struct ggml_tensor  * g,\n            struct ggml_tensor  * state,\n            float scale);\n\n    GGML_API struct ggml_tensor * ggml_rwkv_wkv7(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * r,\n            struct ggml_tensor  * w,\n            struct ggml_tensor  * k,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * b,\n            struct ggml_tensor  * state);\n\n    /* Solves a specific equation of the form Ax=B, where A is a triangular matrix\n    *  without zeroes on the diagonal (i.e. invertible).\n    *  B can have any number of columns, but must have the same number of rows as A\n    *  If A is [n, n] and B is [n, m], then the result will be [n, m] as well\n    *  Has O(n^3) complexity (unlike most matrix ops out there), so use on cases\n    *  where n > 100 sparingly, pre-chunk if necessary.\n    *\n    *  If left = false, solves xA=B instead\n    *  If lower = false, assumes upper triangular instead\n    *  If uni = true, assumes diagonal of A to be all ones (will override actual values)\n    *\n    *  TODO: currently only lower, right, non-unitriangular variant is implemented\n    */\n    GGML_API struct ggml_tensor * ggml_solve_tri(\n        struct ggml_context * ctx,\n        struct ggml_tensor  * a,\n        struct ggml_tensor  * b,\n        bool                  left,\n        bool                  lower,\n        bool                  uni);\n\n    // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]\n    // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306\n    GGML_API struct ggml_tensor * ggml_gated_delta_net(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * q,\n            struct ggml_tensor  * k,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * g,\n            struct ggml_tensor  * beta,\n            struct ggml_tensor  * state);\n\n    // custom operators\n\n    typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);\n    typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);\n    typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);\n\n#define GGML_N_TASKS_MAX (-1)\n    // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks\n\n    GGML_API struct ggml_tensor * ggml_map_custom1(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            ggml_custom1_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    GGML_API struct ggml_tensor * ggml_map_custom1_inplace(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            ggml_custom1_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    GGML_API struct ggml_tensor * ggml_map_custom2(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            struct ggml_tensor    * b,\n            ggml_custom2_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    GGML_API struct ggml_tensor * ggml_map_custom2_inplace(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            struct ggml_tensor    * b,\n            ggml_custom2_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    GGML_API struct ggml_tensor * ggml_map_custom3(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            struct ggml_tensor    * b,\n            struct ggml_tensor    * c,\n            ggml_custom3_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    GGML_API struct ggml_tensor * ggml_map_custom3_inplace(\n            struct ggml_context   * ctx,\n            struct ggml_tensor    * a,\n            struct ggml_tensor    * b,\n            struct ggml_tensor    * c,\n            ggml_custom3_op_t       fun,\n            int                     n_tasks,\n            void                  * userdata);\n\n    typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);\n\n    GGML_API struct ggml_tensor * ggml_custom_4d(\n            struct ggml_context * ctx,\n            enum ggml_type        type,\n            int64_t               ne0,\n            int64_t               ne1,\n            int64_t               ne2,\n            int64_t               ne3,\n            struct ggml_tensor ** args,\n            int                   n_args,\n            ggml_custom_op_t      fun,\n            int                   n_tasks,\n            void                * userdata);\n\n    GGML_API struct ggml_tensor * ggml_custom_inplace(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor ** args,\n            int                   n_args,\n            ggml_custom_op_t      fun,\n            int                   n_tasks,\n            void                * userdata);\n\n    // loss function\n\n    GGML_API struct ggml_tensor * ggml_cross_entropy_loss(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // logits\n            struct ggml_tensor  * b); // labels\n\n    GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,  // logits\n            struct ggml_tensor  * b,  // labels\n            struct ggml_tensor  * c); // gradients of cross_entropy_loss result\n\n    // AdamW optimizer step\n    // Paper: https://arxiv.org/pdf/1711.05101v3.pdf\n    // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html\n    GGML_API struct ggml_tensor * ggml_opt_step_adamw(\n            struct ggml_context * ctx,\n            struct ggml_tensor  * a,\n            struct ggml_tensor  * grad,\n            struct ggml_tensor  * m,\n            struct ggml_tensor  * v,\n            struct ggml_tensor  * adamw_params); // parameters such as the learning rate\n\n    // stochastic gradient descent step (with weight decay)\n    GGML_API struct ggml_tensor * ggml_opt_step_sgd(\n        struct ggml_context * ctx,\n        struct ggml_tensor *  a,\n        struct ggml_tensor *  grad,\n        struct ggml_tensor *  sgd_params); // alpha, weight decay\n\n    // build forward multiple tensors and select one of them for computing\n    // this is useful for creating graphs that have constant topology but compute different things based on the input\n    // ref: https://github.com/ggml-org/llama.cpp/pull/18550\n    //\n    // nodes:\n    //   | - build forward into the graph but do not compute\n    //   c - build forward into the graph and compute\n    //\n    //    |  |  ...  c  ...  |\n    //    |  |  ...  c  ...  |\n    //    |  |  ...  c  ...  |\n    //   [0  1  ... idx ...  n-1]        <-- ggml_build_forward_select(..., n, idx)\n    //               c\n    //               c\n    //\n    // example:\n    //   struct ggml_tensor * curs[3];\n    //\n    //   curs[0]  = compute0(...);\n    //   curs[1]  = compute1(...);\n    //   curs[2]  = compute2(...);\n    //\n    //   int idx = select_branch(some_input);\n    //\n    //   struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx);\n    //\n    GGML_API struct ggml_tensor * ggml_build_forward_select(\n            struct ggml_cgraph  * cgraph,\n            struct ggml_tensor ** tensors,\n            int                   n_tensors,\n            int                   idx);\n\n    GGML_API void ggml_build_forward_expand(\n            struct ggml_cgraph * cgraph,\n            struct ggml_tensor * tensor);\n\n    GGML_API void ggml_build_backward_expand(\n        struct ggml_context *  ctx,        // context for gradient computation\n        struct ggml_cgraph  *  cgraph,\n        struct ggml_tensor  ** grad_accs);\n\n    // graph allocation in a context\n    GGML_API struct ggml_cgraph * ggml_new_graph       (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false\n    GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);\n    GGML_API struct ggml_cgraph * ggml_graph_dup       (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);\n    GGML_API void                 ggml_graph_cpy       (struct ggml_cgraph * src, struct ggml_cgraph * dst);\n    GGML_API void                 ggml_graph_reset     (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1\n    GGML_API void                 ggml_graph_clear     (struct ggml_cgraph * cgraph);\n\n    GGML_API int                   ggml_graph_size   (struct ggml_cgraph * cgraph);\n    GGML_API struct ggml_tensor *  ggml_graph_node   (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]\n    GGML_API struct ggml_tensor ** ggml_graph_nodes  (struct ggml_cgraph * cgraph);\n    GGML_API int                   ggml_graph_n_nodes(struct ggml_cgraph * cgraph);\n\n    GGML_API void   ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);\n\n    GGML_API size_t ggml_graph_overhead(void);\n    GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);\n\n    GGML_API struct ggml_tensor * ggml_graph_get_tensor  (const struct ggml_cgraph * cgraph, const char * name);\n    GGML_API struct ggml_tensor * ggml_graph_get_grad    (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);\n    GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);\n\n    // print info and performance information for the graph\n    GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);\n\n    // dump the graph into a file using the dot format\n    GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename);\n\n    // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?\n    typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);\n\n    // Set callback for all future logging events.\n    // If this is not called, or NULL is supplied, everything is output on stderr.\n    GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data);\n    GGML_API void ggml_log_set(ggml_log_callback   log_callback, void *  user_data);\n\n    GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);\n\n    //\n    // quantization\n    //\n\n    // - ggml_quantize_init can be called multiple times with the same type\n    //   it will only initialize the quantization tables for the first call or after ggml_quantize_free\n    //   automatically called by ggml_quantize_chunk for convenience\n    //\n    // - ggml_quantize_free will free any memory allocated by ggml_quantize_init\n    //   call this at the end of the program to avoid memory leaks\n    //\n    // note: these are thread-safe\n    //\n    GGML_API void ggml_quantize_init(enum ggml_type type);\n    GGML_API void ggml_quantize_free(void);\n\n    // some quantization type cannot be used without an importance matrix\n    GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);\n\n    // calls ggml_quantize_init internally (i.e. can allocate memory)\n    GGML_API size_t ggml_quantize_chunk(\n            enum ggml_type   type,\n               const float * src,\n                      void * dst,\n                   int64_t   start,\n                   int64_t   nrows,\n                   int64_t   n_per_row,\n               const float * imatrix);\n\n#ifdef __cplusplus\n    // restrict not standard in C++\n#    if defined(__GNUC__)\n#        define GGML_RESTRICT __restrict__\n#    elif defined(__clang__)\n#        define GGML_RESTRICT __restrict\n#    elif defined(_MSC_VER)\n#        define GGML_RESTRICT __restrict\n#    else\n#        define GGML_RESTRICT\n#    endif\n#else\n#    if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)\n#        define GGML_RESTRICT __restrict\n#    else\n#        define GGML_RESTRICT restrict\n#    endif\n#endif\n    typedef void (*ggml_to_float_t)  (const void  * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);\n    typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void  * GGML_RESTRICT y, int64_t k);\n\n    struct ggml_type_traits {\n        const char             * type_name;\n        int64_t                  blck_size;\n        int64_t                  blck_size_interleave; // interleave elements in blocks\n        size_t                   type_size;\n        bool                     is_quantized;\n        ggml_to_float_t          to_float;\n        ggml_from_float_t        from_float_ref;\n    };\n\n    GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);\n\n    // ggml threadpool\n    // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend\n    // the goal should be to create an API that other backends can use move everything to the ggml base\n\n    // scheduling priorities\n    enum ggml_sched_priority {\n        GGML_SCHED_PRIO_LOW = -1,\n        GGML_SCHED_PRIO_NORMAL,\n        GGML_SCHED_PRIO_MEDIUM,\n        GGML_SCHED_PRIO_HIGH,\n        GGML_SCHED_PRIO_REALTIME\n    };\n\n    // threadpool params\n    // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults\n    struct ggml_threadpool_params {\n        bool                cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)\n        int                 n_threads;                   // number of threads\n        enum ggml_sched_priority prio;                   // thread priority\n        uint32_t            poll;                        // polling level (0 - no polling, 100 - aggressive polling)\n        bool                strict_cpu;                  // strict cpu placement\n        bool                paused;                      // start in paused state\n    };\n\n    struct ggml_threadpool;     // forward declaration, see ggml.c\n\n    typedef struct ggml_threadpool * ggml_threadpool_t;\n\n    GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);\n    GGML_API void                          ggml_threadpool_params_init   (struct ggml_threadpool_params * p, int n_threads);\n    GGML_API bool                          ggml_threadpool_params_match  (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/include/gguf.h",
    "content": "// This file contains functionality related to \"GGUF\" files, the binary file format used by ggml.\n// GGUF files have the following structure:\n//\n// 1. File magic \"GGUF\" (4 bytes).\n// 2. File version (uint32_t).\n// 3. Number of ggml tensors in file (int64_t).\n// 4. Number of key-value-pairs in file (int64_t).\n// 5. For each KV pair:\n//   1. The key (string).\n//   2. The value type (gguf_type).\n//   3a. If the value type is GGUF_TYPE_ARRAY:\n//     1. The type of the array (gguf_type).\n//     2. The number of elements in the array (uint64_t).\n//     3. The binary representation of each element in the array.\n//   3b. Otherwise:\n//     1. The binary representation of the value.\n// 6. For each ggml tensor:\n//   1. The tensor name (string).\n//   2. The number of dimensions of the tensor (uint32_t).\n//   3. For each dimension:\n//     1. The size of the tensor in the dimension (int64_t).\n//   4. The tensor data type (ggml_type).\n//   5. The tensor data offset in the tensor data binary blob (uint64_t).\n// 7. The tensor data binary blob (optional, aligned).\n//\n// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator.\n// All enums are stored as int32_t.\n// All bool values are stored as int8_t.\n// If the special key \"general.alignment\" (uint32_t) is defined it is used for alignment,\n//   otherwise GGUF_DEFAULT_ALIGNMENT is used.\n//\n// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de)\n\n#pragma once\n\n#include \"ggml.h\"\n\n#include <stdbool.h>\n#include <stdint.h>\n\n#define GGUF_MAGIC   \"GGUF\"\n#define GGUF_VERSION 3\n\n#define GGUF_KEY_GENERAL_ALIGNMENT \"general.alignment\"\n\n#define GGUF_DEFAULT_ALIGNMENT 32\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    // types that can be stored as GGUF KV data\n    enum gguf_type {\n        GGUF_TYPE_UINT8   = 0,\n        GGUF_TYPE_INT8    = 1,\n        GGUF_TYPE_UINT16  = 2,\n        GGUF_TYPE_INT16   = 3,\n        GGUF_TYPE_UINT32  = 4,\n        GGUF_TYPE_INT32   = 5,\n        GGUF_TYPE_FLOAT32 = 6,\n        GGUF_TYPE_BOOL    = 7,\n        GGUF_TYPE_STRING  = 8,\n        GGUF_TYPE_ARRAY   = 9,\n        GGUF_TYPE_UINT64  = 10,\n        GGUF_TYPE_INT64   = 11,\n        GGUF_TYPE_FLOAT64 = 12,\n        GGUF_TYPE_COUNT,       // marks the end of the enum\n    };\n\n    struct gguf_context;\n\n    struct gguf_init_params {\n        bool no_alloc;\n\n        // if not NULL, create a ggml_context and allocate the tensor data in it\n        struct ggml_context ** ctx;\n    };\n\n    GGML_API struct gguf_context * gguf_init_empty(void);\n    GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);\n    //GGML_API struct gguf_context * gguf_init_from_buffer(..);\n\n    GGML_API void gguf_free(struct gguf_context * ctx);\n\n    GGML_API const char * gguf_type_name(enum gguf_type type);\n\n    GGML_API uint32_t gguf_get_version    (const struct gguf_context * ctx);\n    GGML_API size_t   gguf_get_alignment  (const struct gguf_context * ctx);\n    GGML_API size_t   gguf_get_data_offset(const struct gguf_context * ctx);\n\n    GGML_API int64_t      gguf_get_n_kv(const struct gguf_context * ctx);\n    GGML_API int64_t      gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found\n    GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id);\n\n    GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id);\n\n    // will abort if the wrong type is used for the key\n    GGML_API uint8_t      gguf_get_val_u8  (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API int8_t       gguf_get_val_i8  (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API uint16_t     gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API int16_t      gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API uint32_t     gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API int32_t      gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API float        gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API uint64_t     gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API int64_t      gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API double       gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API bool         gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id);\n    GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id);\n    GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id);\n    GGML_API size_t       gguf_get_arr_n   (const struct gguf_context * ctx, int64_t key_id);\n\n    // get raw pointer to the first element of the array with the given key_id\n    // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference)\n    GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id);\n\n    // get ith C string from array with given key_id\n    GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);\n\n    GGML_API int64_t        gguf_get_n_tensors    (const struct gguf_context * ctx);\n    GGML_API int64_t        gguf_find_tensor      (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found\n    GGML_API size_t         gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);\n    GGML_API const char *   gguf_get_tensor_name  (const struct gguf_context * ctx, int64_t tensor_id);\n    GGML_API enum ggml_type gguf_get_tensor_type  (const struct gguf_context * ctx, int64_t tensor_id);\n    GGML_API size_t         gguf_get_tensor_size  (const struct gguf_context * ctx, int64_t tensor_id);\n\n    // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)\n    GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);\n\n    // overrides an existing KV pair or adds a new one, the new KV pair is always at the back\n    GGML_API void gguf_set_val_u8  (struct gguf_context * ctx, const char * key, uint8_t      val);\n    GGML_API void gguf_set_val_i8  (struct gguf_context * ctx, const char * key, int8_t       val);\n    GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t     val);\n    GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t      val);\n    GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t     val);\n    GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t      val);\n    GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float        val);\n    GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t     val);\n    GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t      val);\n    GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double       val);\n    GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool         val);\n    GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);\n\n    // creates a new array with n elements of the given type and copies the corresponding number of bytes from data\n    GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n);\n\n    // creates a new array with n strings and copies the corresponding strings from data\n    GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n);\n\n    // set or add KV pairs from another context\n    GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src);\n\n    // add tensor to GGUF context, tensor name must be unique\n    GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);\n\n    // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated\n    //   in such a way that the tensor data remains as one contiguous block (except for padding)\n    GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);\n\n    // assumes that at least gguf_get_tensor_size bytes can be read from data\n    GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data);\n\n    // writing gguf files can be done in 3 ways:\n    //\n    // - write the entire gguf_context to a binary file in a single pass:\n    //\n    //   gguf_write_to_file(ctx, fname, /*only_meta =*/ false);\n    //\n    // - write only the meta data to a file, then re-open the file and append the tensor data:\n    //\n    //   gguf_write_to_file(ctx, fname, /*only_meta =*/ true);\n    //   FILE * f = fopen(fname, \"ab\");\n    //   fwrite(f, ...); // write tensor data\n    //   fclose(f);\n    //\n    // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:\n    //\n    //   FILE * f = fopen(fname, \"wb\");\n    //   const size_t size_meta = gguf_get_meta_size(ctx);\n    //   fseek(f, size_meta, SEEK_SET);\n    //   fwrite(f, ...); // write tensor data\n    //   void * data = malloc(size_meta);\n    //   gguf_get_meta_data(ctx, data);\n    //   rewind(f);\n    //   fwrite(data, 1, data, f);\n    //   free(data);\n    //   fclose(f);\n    //\n\n    // write the entire context to a binary file\n    GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);\n\n    // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding\n    GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);\n\n    // writes the meta data to pointer \"data\"\n    GGML_API void   gguf_get_meta_data(const struct gguf_context * ctx, void * data);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/CMakeLists.txt",
    "content": "include(CheckCXXCompilerFlag)\ninclude(\"../cmake/common.cmake\")\n\nadd_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})\n\n# enable libstdc++ assertions for debug builds\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n    add_compile_definitions($<$<CONFIG:Debug>:_GLIBCXX_ASSERTIONS>)\nendif()\n\nif (NOT MSVC)\n    if (GGML_SANITIZE_THREAD)\n        add_compile_options(-fsanitize=thread)\n        link_libraries     (-fsanitize=thread)\n    endif()\n\n    if (GGML_SANITIZE_ADDRESS)\n        add_compile_options(-fsanitize=address -fno-omit-frame-pointer)\n        link_libraries     (-fsanitize=address)\n    endif()\n\n    if (GGML_SANITIZE_UNDEFINED)\n        add_compile_options(-fsanitize=undefined)\n        link_libraries     (-fsanitize=undefined)\n    endif()\nendif()\n\nif (GGML_FATAL_WARNINGS)\n    if (CMAKE_CXX_COMPILER_ID MATCHES \"GNU\" OR CMAKE_CXX_COMPILER_ID MATCHES \"Clang\")\n        list(APPEND C_FLAGS   -Werror)\n        list(APPEND CXX_FLAGS -Werror)\n    elseif (CMAKE_CXX_COMPILER_ID STREQUAL \"MSVC\")\n        add_compile_options(/WX)\n    endif()\nendif()\n\nif (GGML_ALL_WARNINGS)\n    if (NOT MSVC)\n        list(APPEND WARNING_FLAGS -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function)\n        list(APPEND C_FLAGS       -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes\n                                  -Werror=implicit-int -Werror=implicit-function-declaration)\n        list(APPEND CXX_FLAGS     -Wmissing-declarations -Wmissing-noreturn)\n\n        list(APPEND C_FLAGS   ${WARNING_FLAGS})\n        list(APPEND CXX_FLAGS ${WARNING_FLAGS})\n\n        ggml_get_flags(${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})\n\n        add_compile_options(\"$<$<COMPILE_LANGUAGE:C>:${C_FLAGS};${GF_C_FLAGS}>\"\n                            \"$<$<COMPILE_LANGUAGE:CXX>:${CXX_FLAGS};${GF_CXX_FLAGS}>\")\n    else()\n        # todo : msvc\n        set(C_FLAGS   \"\")\n        set(CXX_FLAGS \"\")\n    endif()\nendif()\n\nif (GGML_LTO)\n    include(CheckIPOSupported)\n    check_ipo_supported(RESULT result OUTPUT output)\n    if (result)\n        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)\n    else()\n        message(WARNING \"IPO is not supported: ${output}\")\n    endif()\nendif()\n\nif (GGML_CCACHE AND NOT CMAKE_C_COMPILER_LAUNCHER AND NOT CMAKE_CXX_COMPILER_LAUNCHER)\n    find_program(GGML_CCACHE_FOUND ccache)\n    find_program(GGML_SCCACHE_FOUND sccache)\n\n    if (GGML_CCACHE_FOUND OR GGML_SCCACHE_FOUND)\n        if(GGML_CCACHE_FOUND)\n            set(GGML_CCACHE_VARIANT ccache)\n        else()\n            set(GGML_CCACHE_VARIANT sccache)\n        endif()\n        # TODO: should not be set globally\n        if (GGML_SYCL AND GGML_CCACHE_FOUND AND WIN32)\n            set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE \"ccache compiler_type=icl\")\n        else ()\n            set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE \"${GGML_CCACHE_VARIANT}\")\n        endif ()\n        set(ENV{CCACHE_SLOPPINESS} time_macros)\n        message(STATUS \"${GGML_CCACHE_VARIANT} found, compilation results will be cached. Disable with GGML_CCACHE=OFF.\")\n    else()\n        message(STATUS \"Warning: ccache not found - consider installing it for faster compilation or disable this warning with GGML_CCACHE=OFF\")\n    endif ()\nendif()\n\n# this version of Apple ld64 is buggy\nexecute_process(\n    COMMAND ${CMAKE_C_COMPILER} ${CMAKE_EXE_LINKER_FLAGS} -Wl,-v\n    ERROR_VARIABLE output\n    OUTPUT_QUIET\n)\n\nif (output MATCHES \"dyld-1015\\.7\")\n    add_compile_definitions(HAVE_BUGGY_APPLE_LINKER)\nendif()\n\n# architecture specific\n# TODO: probably these flags need to be tweaked on some architectures\n#       feel free to update the Makefile for your architecture and send a pull request or issue\nmessage(STATUS \"CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}\")\nif (MSVC)\n    string(TOLOWER \"${CMAKE_GENERATOR_PLATFORM}\" CMAKE_GENERATOR_PLATFORM_LWR)\n    message(STATUS \"CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}\")\nelse ()\n    set(CMAKE_GENERATOR_PLATFORM_LWR \"\")\nendif ()\nggml_get_system_arch()\nmessage(STATUS \"GGML_SYSTEM_ARCH: ${GGML_SYSTEM_ARCH}\")\n\nif (NOT MSVC)\n    if (GGML_STATIC)\n        if (UNIX AND NOT APPLE)\n            set(CMAKE_FIND_LIBRARY_SUFFIXES \".a;.so\")\n        endif()\n        add_link_options(-static)\n        if (MINGW)\n            add_link_options(-static-libgcc -static-libstdc++)\n        endif()\n    endif()\n    if (GGML_GPROF)\n        add_compile_options(-pg)\n    endif()\nendif()\n\n#\n# POSIX conformance\n#\n\n# clock_gettime came in POSIX.1b (1993)\n# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional\n# posix_memalign came in POSIX.1-2001 / SUSv3\n# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)\n\n# Somehow in OpenBSD whenever POSIX conformance is specified\n# some string functions rely on locale_t availability,\n# which was introduced in POSIX.1-2008, forcing us to go higher\nif (CMAKE_SYSTEM_NAME MATCHES \"OpenBSD\")\n    add_compile_definitions(_XOPEN_SOURCE=700)\nelseif (CMAKE_SYSTEM_NAME MATCHES \"AIX\")\n    # Don't define _XOPEN_SOURCE.  We need _ALL_SOURCE, which is the default,\n    # in order to define _SC_PHYS_PAGES.\nelse()\n    add_compile_definitions(_XOPEN_SOURCE=600)\nendif()\n\n# Data types, macros and functions related to controlling CPU affinity and\n# some memory allocation are available on Linux through GNU extensions in libc\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\" OR CMAKE_SYSTEM_NAME MATCHES \"Android\")\n    add_compile_definitions(_GNU_SOURCE)\nendif()\n\n# RLIMIT_MEMLOCK came in BSD, is not specified in POSIX.1,\n# and on macOS its availability depends on enabling Darwin extensions\n# similarly on DragonFly, enabling BSD extensions is necessary\nif (\n    CMAKE_SYSTEM_NAME MATCHES \"Darwin\" OR\n    CMAKE_SYSTEM_NAME MATCHES \"iOS\"    OR\n    CMAKE_SYSTEM_NAME MATCHES \"tvOS\"   OR\n    CMAKE_SYSTEM_NAME MATCHES \"DragonFly\"\n)\n    add_compile_definitions(_DARWIN_C_SOURCE)\nendif()\n\n# alloca is a non-standard interface that is not visible on BSDs when\n# POSIX conformance is specified, but not all of them provide a clean way\n# to enable it in such cases\nif (CMAKE_SYSTEM_NAME MATCHES \"FreeBSD\")\n    add_compile_definitions(__BSD_VISIBLE)\nendif()\nif (CMAKE_SYSTEM_NAME MATCHES \"NetBSD\")\n    add_compile_definitions(_NETBSD_SOURCE)\nendif()\nif (CMAKE_SYSTEM_NAME MATCHES \"OpenBSD\")\n    add_compile_definitions(_BSD_SOURCE)\nendif()\n\nif (WIN32)\n    add_compile_definitions(_CRT_SECURE_NO_WARNINGS)\nendif()\n\n# ggml\n\nif (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)\n    message(FATAL_ERROR \"GGML_BACKEND_DL requires BUILD_SHARED_LIBS\")\nendif()\n\nadd_library(ggml-base\n            ../include/ggml.h\n            ../include/ggml-alloc.h\n            ../include/ggml-backend.h\n            ../include/ggml-cpp.h\n            ../include/ggml-opt.h\n            ../include/gguf.h\n            ggml.c\n            ggml.cpp\n            ggml-alloc.c\n            ggml-backend.cpp\n            ggml-opt.cpp\n            ggml-threading.cpp\n            ggml-threading.h\n            ggml-quants.c\n            ggml-quants.h\n            gguf.cpp)\n\nset_target_properties(ggml-base PROPERTIES\n    VERSION ${GGML_VERSION}\n    SOVERSION ${GGML_VERSION_MAJOR}\n)\n\ntarget_include_directories(ggml-base PRIVATE .)\nif (GGML_BACKEND_DL)\n    target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL)\nendif()\n\nif (GGML_SCHED_NO_REALLOC)\n    target_compile_definitions(ggml-base PUBLIC GGML_SCHED_NO_REALLOC)\nendif()\n\nadd_library(ggml\n            ggml-backend-dl.cpp\n            ggml-backend-reg.cpp)\nadd_library(ggml::ggml ALIAS ggml)\n\nset_target_properties(ggml PROPERTIES\n    VERSION ${GGML_VERSION}\n    SOVERSION ${GGML_VERSION_MAJOR}\n)\n\nif (GGML_BACKEND_DIR)\n    if (NOT GGML_BACKEND_DL)\n        message(FATAL_ERROR \"GGML_BACKEND_DIR requires GGML_BACKEND_DL\")\n    endif()\n    target_compile_definitions(ggml PUBLIC GGML_BACKEND_DIR=\"${GGML_BACKEND_DIR}\")\nendif()\n\ntarget_link_libraries(ggml PUBLIC ggml-base)\n\nif (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n    target_link_libraries(ggml PRIVATE dl)\nendif()\n\nfunction(ggml_add_backend_library backend)\n    if (GGML_BACKEND_DL)\n        add_library(${backend} MODULE ${ARGN})\n        # write the shared library to the output directory\n        set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})\n        target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)\n        add_dependencies(ggml ${backend})\n        if (GGML_BACKEND_DIR)\n            install(TARGETS ${backend} LIBRARY DESTINATION ${GGML_BACKEND_DIR})\n        else()\n            install(TARGETS ${backend} LIBRARY DESTINATION ${CMAKE_INSTALL_BINDIR})\n        endif()\n    else()\n        add_library(${backend} ${ARGN})\n        target_link_libraries(ggml PUBLIC ${backend})\n        install(TARGETS ${backend} LIBRARY)\n    endif()\n\n    target_link_libraries(${backend} PRIVATE ggml-base)\n    target_include_directories(${backend} PRIVATE ..)\n\n    if (${BUILD_SHARED_LIBS})\n        target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)\n        target_compile_definitions(${backend} PUBLIC  GGML_BACKEND_SHARED)\n    endif()\n\n    # Set versioning properties for all backend libraries\n    # Building a MODULE library with a version is not supported on macOS (https://gitlab.kitware.com/cmake/cmake/-/issues/20782)\n    if (NOT (APPLE AND GGML_BACKEND_DL))\n        set_target_properties(${backend} PROPERTIES\n            VERSION ${GGML_VERSION}\n            SOVERSION ${GGML_VERSION_MAJOR}\n        )\n    endif()\n\n    if(NOT GGML_AVAILABLE_BACKENDS)\n        set(GGML_AVAILABLE_BACKENDS \"${backend}\"\n            CACHE INTERNAL \"List of backends for cmake package\")\n    else()\n        list(FIND GGML_AVAILABLE_BACKENDS \"${backend}\" has_backend)\n        if(has_backend EQUAL -1)\n            set(GGML_AVAILABLE_BACKENDS \"${GGML_AVAILABLE_BACKENDS};${backend}\"\n                CACHE INTERNAL \"List of backends for cmake package\")\n        endif()\n    endif()\nendfunction()\n\nfunction(ggml_add_backend backend)\n    string(TOUPPER \"GGML_${backend}\" backend_id)\n    if (${backend_id})\n        string(TOLOWER \"ggml-${backend}\" backend_target)\n        add_subdirectory(${backend_target})\n        message(STATUS \"Including ${backend} backend\")\n        if (NOT GGML_BACKEND_DL)\n            string(TOUPPER \"GGML_USE_${backend}\" backend_use)\n            target_compile_definitions(ggml PUBLIC ${backend_use})\n        endif()\n    endif()\nendfunction()\n\nfunction(ggml_add_cpu_backend_variant tag_name)\n    set(GGML_CPU_TAG_NAME ${tag_name})\n    # other: OPENMP LLAMAFILE CPU_HBM\n    if (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n        foreach (feat NATIVE\n                      SSE42\n                      AVX AVX2 BMI2 AVX_VNNI FMA F16C\n                      AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16\n                      AMX_TILE AMX_INT8 AMX_BF16)\n            set(GGML_${feat} OFF)\n        endforeach()\n\n        foreach (feat ${ARGN})\n            set(GGML_${feat} ON)\n        endforeach()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n        foreach (feat ${ARGN})\n            set(GGML_INTERNAL_${feat} ON)\n        endforeach()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n        foreach (feat ${ARGN})\n            set(GGML_INTERNAL_${feat} ON)\n        endforeach()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n        foreach (feat VXE2 NNPA)\n            set(GGML_INTERNAL_${feat} OFF)\n        endforeach()\n\n        foreach (feat ${ARGN})\n            set(GGML_INTERNAL_${feat} ON)\n        endforeach()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n        foreach (feat RVV)\n            set(GGML_INTERNAL_${feat} OFF)\n        endforeach()\n\n        foreach (feat ${ARGN})\n            set(GGML_INTERNAL_${feat} ON)\n        endforeach()\n    endif()\n\n    ggml_add_cpu_backend_variant_impl(${tag_name})\nendfunction()\n\nggml_add_backend(CPU)\n\nif (GGML_CPU_ALL_VARIANTS)\n    if (NOT GGML_BACKEND_DL)\n        message(FATAL_ERROR \"GGML_CPU_ALL_VARIANTS requires GGML_BACKEND_DL\")\n    elseif (GGML_CPU_ARM_ARCH)\n        message(FATAL_ERROR \"Cannot use both GGML_CPU_ARM_ARCH and GGML_CPU_ALL_VARIANTS\")\n    endif()\n    if (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n        ggml_add_cpu_backend_variant(x64)\n        ggml_add_cpu_backend_variant(sse42              SSE42)\n        ggml_add_cpu_backend_variant(sandybridge        SSE42 AVX)\n        if (NOT MSVC)\n            # __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n            ggml_add_cpu_backend_variant(ivybridge      SSE42 AVX F16C)\n            ggml_add_cpu_backend_variant(piledriver     SSE42 AVX F16C FMA)\n        endif()\n        ggml_add_cpu_backend_variant(haswell            SSE42 AVX F16C FMA AVX2 BMI2)\n        ggml_add_cpu_backend_variant(skylakex           SSE42 AVX F16C FMA AVX2 BMI2 AVX512)\n        ggml_add_cpu_backend_variant(cannonlake         SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI)\n        ggml_add_cpu_backend_variant(cascadelake        SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VNNI)\n        ggml_add_cpu_backend_variant(icelake            SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI)\n        if (NOT MSVC)\n            # MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!\n            # https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170\n            # https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170\n            ggml_add_cpu_backend_variant(cooperlake     SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VNNI AVX512_BF16)\n            ggml_add_cpu_backend_variant(zen4           SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16)\n        endif()\n        ggml_add_cpu_backend_variant(alderlake          SSE42 AVX F16C FMA AVX2 BMI2 AVX_VNNI)\n        if (NOT MSVC)\n            # MSVC doesn't support AMX\n            ggml_add_cpu_backend_variant(sapphirerapids SSE42 AVX F16C FMA AVX2 BMI2 AVX512 AVX512_VBMI AVX512_VNNI AVX512_BF16 AMX_TILE AMX_INT8)\n        endif()\n    elseif(GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n        if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n            # Many of these features are optional so we build versions with popular\n            # combinations and name the backends based on the version they were\n            # first released with\n            ggml_add_cpu_backend_variant(armv8.0_1)\n            ggml_add_cpu_backend_variant(armv8.2_1    DOTPROD)\n            ggml_add_cpu_backend_variant(armv8.2_2    DOTPROD FP16_VECTOR_ARITHMETIC)\n            ggml_add_cpu_backend_variant(armv8.2_3    DOTPROD FP16_VECTOR_ARITHMETIC SVE)\n            ggml_add_cpu_backend_variant(armv8.6_1    DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8)\n            ggml_add_cpu_backend_variant(armv8.6_2    DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SVE2)\n            ggml_add_cpu_backend_variant(armv9.2_1    DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SME)\n            ggml_add_cpu_backend_variant(armv9.2_2    DOTPROD FP16_VECTOR_ARITHMETIC SVE MATMUL_INT8 SVE2 SME)\n        elseif (CMAKE_SYSTEM_NAME MATCHES \"Android\")\n            # Android-specific backends with SoC-compatible feature sets\n            ggml_add_cpu_backend_variant(android_armv8.0_1)\n            ggml_add_cpu_backend_variant(android_armv8.2_1    DOTPROD)\n            ggml_add_cpu_backend_variant(android_armv8.2_2    DOTPROD FP16_VECTOR_ARITHMETIC)\n            ggml_add_cpu_backend_variant(android_armv8.6_1    DOTPROD FP16_VECTOR_ARITHMETIC MATMUL_INT8)\n            ggml_add_cpu_backend_variant(android_armv9.0_1    DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE2)\n            ggml_add_cpu_backend_variant(android_armv9.2_1    DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE SME)\n            ggml_add_cpu_backend_variant(android_armv9.2_2    DOTPROD MATMUL_INT8 FP16_VECTOR_ARITHMETIC SVE SVE2 SME)\n        elseif (APPLE)\n            ggml_add_cpu_backend_variant(apple_m1             DOTPROD)\n            ggml_add_cpu_backend_variant(apple_m2_m3          DOTPROD MATMUL_INT8)\n            ggml_add_cpu_backend_variant(apple_m4             DOTPROD MATMUL_INT8 NOSVE SME)\n        else()\n            message(FATAL_ERROR \"Unsupported ARM target OS: ${CMAKE_SYSTEM_NAME}\")\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n        if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n            ggml_add_cpu_backend_variant(power0)\n            ggml_add_cpu_backend_variant(power7_1       POWER7)\n            ggml_add_cpu_backend_variant(power7_2       POWER7  VSX)\n            ggml_add_cpu_backend_variant(power8_1       POWER8)\n            ggml_add_cpu_backend_variant(power8_2       POWER8  VSX)\n            ggml_add_cpu_backend_variant(power9         POWER9  VSX)\n            ggml_add_cpu_backend_variant(power10        POWER10 VSX)\n            ggml_add_cpu_backend_variant(power11        POWER11 VSX)\n        else()\n            message(FATAL_ERROR \"Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}\")\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n        if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n            ggml_add_cpu_backend_variant(z15    Z15 VXE2)\n            ggml_add_cpu_backend_variant(z16    Z16 VXE2 NNPA)\n        else()\n            message(FATAL_ERROR \"Unsupported s390x target OS: ${CMAKE_SYSTEM_NAME}\")\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n        if (CMAKE_SYSTEM_NAME MATCHES \"Linux\")\n            ggml_add_cpu_backend_variant(riscv64_0)\n            ggml_add_cpu_backend_variant(riscv64_v   RVV)\n        else()\n            message(FATAL_ERROR \"Unsupported RISC-V target OS: ${CMAKE_SYSTEM_NAME}\")\n        endif()\n    else()\n        message(FATAL_ERROR \"GGML_CPU_ALL_VARIANTS not yet supported with ${GGML_SYSTEM_ARCH} on ${CMAKE_SYSTEM_NAME}\")\n    endif()\nelseif (GGML_CPU)\n    ggml_add_cpu_backend_variant_impl(\"\")\nendif()\n\nggml_add_backend(BLAS)\nggml_add_backend(CANN)\nggml_add_backend(CUDA)\nggml_add_backend(HIP)\nggml_add_backend(METAL)\nggml_add_backend(MUSA)\nggml_add_backend(RPC)\nggml_add_backend(VirtGPU)\nggml_add_backend(SYCL)\nggml_add_backend(Vulkan)\nggml_add_backend(WebGPU)\nggml_add_backend(zDNN)\nggml_add_backend(OpenCL)\nggml_add_backend(Hexagon)\nggml_add_backend(ZenDNN)\nggml_add_backend(OPENVINO)\n\nforeach (target ggml-base ggml)\n    target_include_directories(${target} PUBLIC    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)\n    target_compile_features   (${target} PRIVATE c_std_11 cxx_std_17) # don't bump\nendforeach()\n\ntarget_link_libraries(ggml-base PRIVATE Threads::Threads)\n\nfind_library(MATH_LIBRARY m)\nif (MATH_LIBRARY)\n    if (NOT WIN32 OR NOT DEFINED ENV{ONEAPI_ROOT})\n        target_link_libraries(ggml-base PRIVATE m)\n    endif()\nendif()\n\nif (CMAKE_SYSTEM_NAME MATCHES \"Android\")\n    target_link_libraries(ggml-base PRIVATE dl)\nendif()\n\nif(CMAKE_SYSTEM_NAME MATCHES \"visionOS\")\n    target_compile_definitions(ggml-base PUBLIC _DARWIN_C_SOURCE)\nendif()\n\nif (BUILD_SHARED_LIBS)\n    foreach (target ggml-base ggml)\n        set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)\n        target_compile_definitions(${target} PRIVATE GGML_BUILD)\n        target_compile_definitions(${target} PUBLIC  GGML_SHARED)\n    endforeach()\nendif()\n"
  },
  {
    "path": "ggml/src/ggml-alloc.c",
    "content": "#include \"ggml-alloc.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include <assert.h>\n#include <limits.h>\n#include <stdarg.h>\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\n#define MAX(a, b) ((a) > (b) ? (a) : (b))\n#define MAX_FREE_BLOCKS 256\n\n//#define GGML_ALLOCATOR_DEBUG\n\n//#define AT_PRINTF(...) GGML_LOG_DEBUG(__VA_ARGS__)\n#define AT_PRINTF(...)\n\n// ops that return true for this function must not use restrict pointers for their backend implementations\nbool ggml_op_can_inplace(enum ggml_op op) {\n    switch (op) {\n        case GGML_OP_FILL:\n        case GGML_OP_SCALE:\n        case GGML_OP_DIAG_MASK_ZERO:\n        case GGML_OP_DIAG_MASK_INF:\n        case GGML_OP_ADD:\n        case GGML_OP_ADD_ID:\n        case GGML_OP_ADD1:\n        case GGML_OP_SUB:\n        case GGML_OP_MUL:\n        case GGML_OP_DIV:\n        case GGML_OP_SQR:\n        case GGML_OP_SQRT:\n        case GGML_OP_LOG:\n        case GGML_OP_UNARY:\n        case GGML_OP_ROPE:\n        case GGML_OP_ROPE_BACK:\n        case GGML_OP_SILU_BACK:\n        case GGML_OP_RMS_NORM:\n        case GGML_OP_RMS_NORM_BACK:\n        case GGML_OP_SOFT_MAX:\n        case GGML_OP_SOFT_MAX_BACK:\n            return true;\n\n        default:\n            return false;\n    }\n}\n\nstatic size_t aligned_offset(const void * buffer, size_t offset, size_t alignment) {\n    assert(alignment && !(alignment & (alignment - 1))); // power of 2\n    size_t align = (alignment - (((uintptr_t)buffer + offset) % alignment)) % alignment;\n    return offset + align;\n}\n\n// tallocr\n\nstruct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer) {\n    void * base = ggml_backend_buffer_get_base(buffer);\n    size_t align = ggml_backend_buffer_get_alignment(buffer);\n\n    assert(align && !(align & (align - 1))); // power of 2\n\n    struct ggml_tallocr talloc = (struct ggml_tallocr) {\n        /*.buffer    = */ buffer,\n        /*.base      = */ base,\n        /*.alignment = */ align,\n        /*.offset    = */ aligned_offset(base, 0, align),\n    };\n    return talloc;\n}\n\nenum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor) {\n    size_t size = ggml_backend_buffer_get_alloc_size(talloc->buffer, tensor);\n    size = GGML_PAD(size, talloc->alignment);\n\n    if (talloc->offset + size > ggml_backend_buffer_get_size(talloc->buffer)) {\n        GGML_LOG_ERROR(\"%s: not enough space in the buffer to allocate %s (needed %zu, available %zu)\\n\",\n                __func__, tensor->name, size, ggml_backend_buffer_get_size(talloc->buffer) - talloc->offset);\n        GGML_ABORT(\"not enough space in the buffer\");\n    }\n\n    void * addr = (char *)ggml_backend_buffer_get_base(talloc->buffer) + talloc->offset;\n    talloc->offset += size;\n\n    assert(((uintptr_t)addr % talloc->alignment) == 0);\n\n    return ggml_backend_tensor_alloc(talloc->buffer, tensor, addr);\n}\n\n// dynamic tensor allocator\n\n#define GGML_VBUFFER_MAX_CHUNKS 16\n\n// relative memory address within an allocation that can be split into multiple buffers (chunks)\nstruct buffer_address {\n    int chunk;     // index of a backend buffer\n    size_t offset; // local memory offset within the buffer\n};\n\nstatic const struct buffer_address GGML_BUFFER_ADDRESS_INVALID = { -1, SIZE_MAX };\n\nstatic bool ggml_buffer_address_less(struct buffer_address a, struct buffer_address b) {\n    return a.chunk != b.chunk ? a.chunk < b.chunk : a.offset < b.offset;\n}\n\nstruct free_block {\n    size_t offset;\n    size_t size;\n};\n\nstruct tallocr_chunk {\n    struct free_block free_blocks[MAX_FREE_BLOCKS];\n    int n_free_blocks;\n    size_t max_size;\n};\n\nstruct ggml_dyn_tallocr {\n    size_t alignment;\n    size_t max_chunk_size;\n    struct tallocr_chunk * chunks[GGML_VBUFFER_MAX_CHUNKS];\n    int n_chunks;\n\n#ifdef GGML_ALLOCATOR_DEBUG\n    struct {\n        const struct ggml_tensor * tensor;\n        struct buffer_address addr;\n    } allocated_tensors[1024];\n#endif\n};\n\nstatic void ggml_dyn_tallocr_insert_block(struct tallocr_chunk * chunk, size_t offset, size_t size) {\n    GGML_ASSERT(chunk->n_free_blocks < MAX_FREE_BLOCKS && \"out of free blocks\");\n    // insert the new block in the correct position to keep the array sorted by address (to make merging blocks faster)\n    int insert_pos = 0;\n    while (insert_pos < chunk->n_free_blocks && chunk->free_blocks[insert_pos].offset < offset) {\n        insert_pos++;\n    }\n    // shift all blocks from insert_pos onward to make room for the new block\n    for (int i = chunk->n_free_blocks; i > insert_pos; i--) {\n        chunk->free_blocks[i] = chunk->free_blocks[i-1];\n    }\n    // insert the new block\n    chunk->free_blocks[insert_pos].offset = offset;\n    chunk->free_blocks[insert_pos].size = size;\n    chunk->n_free_blocks++;\n}\n\nstatic void ggml_dyn_tallocr_remove_block(struct tallocr_chunk * chunk, int idx) {\n    // shift all elements after idx by 1 to the left, overwriting the element at idx\n    for (int i = idx; i < chunk->n_free_blocks; i++) {\n        chunk->free_blocks[i] = chunk->free_blocks[i+1];\n    }\n    chunk->n_free_blocks--;\n}\n\nstatic int ggml_dyn_tallocr_new_chunk(struct ggml_dyn_tallocr * alloc, size_t min_size) {\n    if (alloc->n_chunks >= GGML_VBUFFER_MAX_CHUNKS) {\n        return -1;\n    }\n    struct tallocr_chunk * chunk = calloc(1, sizeof(struct tallocr_chunk));\n    chunk->n_free_blocks = 1;\n    chunk->free_blocks[0].offset = 0;\n    // available space in a chunk is limited to max_chunk_size, but can be higher if:\n    // 1. a single tensor exceeds the maximum, and cannot fit any other way\n    // 2. we are running out of chunks\n    // backends will either manage to allocate the larger size, or report an error.\n    chunk->free_blocks[0].size = MAX(min_size, alloc->max_chunk_size);\n    if (alloc->n_chunks == GGML_VBUFFER_MAX_CHUNKS - 1) {\n        chunk->free_blocks[0].size = SIZE_MAX/2;\n    }\n    alloc->chunks[alloc->n_chunks] = chunk;\n    alloc->n_chunks++;\n    return alloc->n_chunks - 1;\n}\n\n#ifdef GGML_ALLOCATOR_DEBUG\nstatic void add_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) {\n    for (int i = 0; i < 1024; i++) {\n        if (alloc->allocated_tensors[i].tensor == NULL) {\n            alloc->allocated_tensors[i].tensor = tensor;\n            alloc->allocated_tensors[i].addr = addr;\n            return;\n        }\n    }\n    GGML_ABORT(\"out of allocated_tensors\");\n}\nstatic void remove_allocated_tensor(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, const struct ggml_tensor * tensor) {\n    for (int i = 0; i < 1024; i++) {\n        if (alloc->allocated_tensors[i].addr.chunk == addr.chunk && alloc->allocated_tensors[i].addr.offset == addr.offset) {\n            alloc->allocated_tensors[i].tensor = NULL;\n            return;\n        }\n    }\n    GGML_ABORT(\"tried to free tensor %s not found\\n\", tensor->name);\n}\n#endif\n\nstatic struct buffer_address ggml_dyn_tallocr_alloc(struct ggml_dyn_tallocr * alloc, size_t size, const struct ggml_tensor * tensor) {\n    size = aligned_offset(NULL, size, alloc->alignment);\n\n    AT_PRINTF(\"%s: allocating %s (%zu bytes) - \", __func__, tensor->name, size);\n\n    int best_fit_chunk = -1;\n    int best_fit_block = -1;\n    size_t max_avail = 0;\n\n    // find the best fitting free block besides the last block, within any chunk\n    for (int c = 0; c < alloc->n_chunks; ++c) {\n        struct tallocr_chunk * chunk = alloc->chunks[c];\n        size_t best_fit_size = SIZE_MAX;\n        for (int i = 0; i < chunk->n_free_blocks - 1; i++) {\n            struct free_block * block = &chunk->free_blocks[i];\n            max_avail = MAX(max_avail, block->size);\n            if (block->size >= size && block->size <= best_fit_size) {\n                best_fit_chunk = c;\n                best_fit_block = i;\n                best_fit_size = block->size;\n            }\n        }\n    }\n\n    if (best_fit_block == -1) {\n        // no suitable block found, try the last block (this may grow a chunks size)\n        int64_t best_reuse = INT64_MIN;\n        for (int c = 0; c < alloc->n_chunks; ++c) {\n            struct tallocr_chunk * chunk = alloc->chunks[c];\n            if (chunk->n_free_blocks > 0) {\n                struct free_block * block = &chunk->free_blocks[chunk->n_free_blocks - 1];\n                max_avail = MAX(max_avail, block->size);\n                int64_t reuse_factor = chunk->max_size - block->offset - size;\n                // reuse_factor < 0 : amount of extra memory that needs to be allocated\n                // reuse_factor = 0 : allocated free space exactly matches tensor size\n                // reuse_factor > 0 : superfluous memory that will remain unused\n                bool better_reuse = best_reuse < 0 && reuse_factor > best_reuse;\n                bool better_fit = reuse_factor >= 0 && reuse_factor < best_reuse;\n                if (block->size >= size && (better_reuse || better_fit)) {\n                    best_fit_chunk = c;\n                    best_fit_block = chunk->n_free_blocks - 1;\n                    best_reuse = reuse_factor;\n                }\n            }\n        }\n    }\n\n    if (best_fit_block == -1) {\n        // none of the existing chunks have enough space left\n        best_fit_chunk = ggml_dyn_tallocr_new_chunk(alloc, size);\n        best_fit_block = 0;\n    }\n    if (best_fit_chunk == -1) {\n        // since the last chunk always has virtually endless memory, this should never happen\n        GGML_LOG_ERROR(\"%s: not enough space in the buffer to allocate %zu bytes, largest block available %zu bytes\\n\",\n            __func__, size, max_avail);\n        GGML_ABORT(\"graph allocation: failed to reserve memory\");\n    }\n\n    struct tallocr_chunk * chunk = alloc->chunks[best_fit_chunk];\n    struct free_block    * block = &chunk->free_blocks[best_fit_block];\n    struct buffer_address  addr  = {.chunk = best_fit_chunk, .offset = block->offset };\n    block->offset += size;\n    block->size -= size;\n    if (block->size == 0) {\n        // remove block if empty\n        ggml_dyn_tallocr_remove_block(chunk, best_fit_block);\n    }\n\n    AT_PRINTF(\"block %d, offset %zu, chunk %d\\n\", best_fit_block, addr.offset, addr.chunk);\n\n#ifdef GGML_ALLOCATOR_DEBUG\n    add_allocated_tensor(alloc, addr, tensor);\n    size_t cur_max = addr.offset + size;\n    if (cur_max > chunk->max_size) {\n        // sort allocated_tensors by chunk/offset\n        for (int i = 0; i < 1024; i++) {\n            for (int j = i + 1; j < 1024; j++) {\n                if (ggml_buffer_address_less(alloc->allocated_tensors[j].addr, alloc->allocated_tensors[i].addr)) {\n                    const struct ggml_tensor * tmp_tensor = alloc->allocated_tensors[i].tensor;\n                    struct buffer_address tmp_addr = alloc->allocated_tensors[i].addr;\n                    alloc->allocated_tensors[i].tensor = alloc->allocated_tensors[j].tensor;\n                    alloc->allocated_tensors[i].addr = alloc->allocated_tensors[j].addr;\n                    alloc->allocated_tensors[j].tensor = tmp_tensor;\n                    alloc->allocated_tensors[j].addr = tmp_addr;\n                }\n            }\n        }\n        GGML_LOG_DEBUG(\"max_size[%d] = %.2f MB: tensors: \", addr.chunk, cur_max / 1024.0 / 1024.0);\n        for (int i = 0; i < 1024; i++) {\n            if (alloc->allocated_tensors[i].tensor) {\n                GGML_LOG_DEBUG(\"%s [%d: %zx-%zx] (%.2f MB) \", alloc->allocated_tensors[i].tensor->name,\n                    alloc->allocated_tensors[i].addr.chunk,\n                    alloc->allocated_tensors[i].addr.offset,\n                    alloc->allocated_tensors[i].addr.offset + ggml_nbytes(alloc->allocated_tensors[i].tensor),\n                    ggml_nbytes(alloc->allocated_tensors[i].tensor) / 1024.0 / 1024.0);\n            }\n        }\n        GGML_LOG_DEBUG(\"\\n\");\n    }\n#endif\n\n    chunk->max_size = MAX(chunk->max_size, addr.offset + size);\n\n    return addr;\n\n    GGML_UNUSED(tensor);\n}\n\n// this is a very naive implementation, but for our case the number of free blocks should be very small\nstatic void ggml_dyn_tallocr_free_bytes(struct ggml_dyn_tallocr * alloc, struct buffer_address addr, size_t size) {\n    size = aligned_offset(NULL, size, alloc->alignment);\n\n    struct tallocr_chunk * chunk = alloc->chunks[addr.chunk];\n\n    // see if we can merge with an existing block\n    for (int i = 0; i < chunk->n_free_blocks; i++) {\n        struct free_block * block = &chunk->free_blocks[i];\n        // check if ptr is at the end of the block\n        if (block->offset + block->size == addr.offset) {\n            block->size += size;\n            // check if we can merge with the next block\n            if (i < chunk->n_free_blocks - 1) {\n                struct free_block * next = &chunk->free_blocks[i+1];\n                if (block->offset + block->size == next->offset) {\n                    block->size += next->size;\n                    ggml_dyn_tallocr_remove_block(chunk, i+1);\n                }\n            }\n            return;\n        }\n        // check if ptr is at the beginning of the block\n        if (addr.offset + size == block->offset) {\n            block->offset = addr.offset;\n            block->size += size;\n            // check if we can merge with the previous block\n            if (i > 0) {\n                struct free_block * prev = &chunk->free_blocks[i-1];\n                if (prev->offset + prev->size == block->offset) {\n                    prev->size += block->size;\n                    ggml_dyn_tallocr_remove_block(chunk, i);\n                }\n            }\n            return;\n        }\n    }\n    // otherwise, add a new block\n    ggml_dyn_tallocr_insert_block(chunk, addr.offset, size);\n}\n\nstatic void ggml_dyn_tallocr_reset(struct ggml_dyn_tallocr * alloc) {\n    for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; i++) {\n        free(alloc->chunks[i]);\n        alloc->chunks[i] = NULL;\n    }\n    alloc->n_chunks = 0;\n\n#ifdef GGML_ALLOCATOR_DEBUG\n    for (int i = 0; i < 1024; i++) {\n        alloc->allocated_tensors[i].tensor = NULL;\n    }\n#endif\n}\n\nstatic struct ggml_dyn_tallocr * ggml_dyn_tallocr_new(size_t alignment, size_t max_buffer_size) {\n    struct ggml_dyn_tallocr * alloc = (struct ggml_dyn_tallocr *)malloc(sizeof(struct ggml_dyn_tallocr));\n\n    *alloc = (struct ggml_dyn_tallocr) {\n        /*.alignment      = */ alignment,\n        /*.max_chunk_size = */ MIN(max_buffer_size, SIZE_MAX/2), // clamp to avoid overflows\n        /*.chunks         = */ {NULL},\n        /*.n_chunks       = */ 0,\n#ifdef GGML_ALLOCATOR_DEBUG\n        /*.allocated_tensors = */ {{0}},\n#endif\n    };\n\n    ggml_dyn_tallocr_reset(alloc);\n\n    return alloc;\n}\n\nstatic void ggml_dyn_tallocr_free(struct ggml_dyn_tallocr * alloc) {\n    for (int i = 0; i < alloc->n_chunks; ++i) {\n        free(alloc->chunks[i]);\n    }\n    free(alloc);\n}\n\nstatic size_t ggml_dyn_tallocr_max_size(struct ggml_dyn_tallocr * alloc, int chunk) {\n    return chunk < alloc->n_chunks ? alloc->chunks[chunk]->max_size : 0;\n}\n\n\n// virtual buffer with contiguous memory range, split into multiple backend buffers (chunks)\n\nstruct vbuffer {\n    ggml_backend_buffer_t chunks[GGML_VBUFFER_MAX_CHUNKS];\n};\n\nstatic void ggml_vbuffer_free(struct vbuffer * buf) {\n    if (buf == NULL) {\n        return;\n    }\n    for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS; ++i) {\n        ggml_backend_buffer_free(buf->chunks[i]);\n    }\n    free(buf);\n}\n\nstatic size_t ggml_vbuffer_chunk_size(struct vbuffer * buf, int chunk) {\n    return buf->chunks[chunk] ? ggml_backend_buffer_get_size(buf->chunks[chunk]) : 0;\n}\n\nstatic size_t ggml_vbuffer_size(struct vbuffer * buf) {\n    size_t size = 0;\n    for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) {\n        size += ggml_backend_buffer_get_size(buf->chunks[i]);\n    }\n    return size;\n}\n\nstatic struct vbuffer * ggml_vbuffer_alloc(ggml_backend_buffer_type_t buft, const struct ggml_dyn_tallocr * talloc, enum ggml_backend_buffer_usage usage) {\n    struct vbuffer * buf = (struct vbuffer *)calloc(1, sizeof(struct vbuffer));\n    if (buf == NULL) {\n        return NULL;\n    }\n\n    for (int n = 0; n < talloc->n_chunks; n++) {\n        size_t chunk_size = talloc->chunks[n]->max_size;\n        buf->chunks[n] = ggml_backend_buft_alloc_buffer(buft, chunk_size);\n        if (buf->chunks[n] == NULL) {\n            ggml_vbuffer_free(buf);\n            return NULL;\n        }\n        ggml_backend_buffer_set_usage(buf->chunks[n], usage);\n    }\n    return buf;\n}\n\nstatic void ggml_vbuffer_tensor_alloc(struct vbuffer * buf, struct ggml_tensor * tensor, struct buffer_address buf_addr) {\n    void * base = ggml_backend_buffer_get_base(buf->chunks[buf_addr.chunk]);\n    void * addr = (char *)base + buf_addr.offset;\n    ggml_backend_tensor_alloc(buf->chunks[buf_addr.chunk], tensor, addr);\n}\n\nstatic void ggml_vbuffer_reset(struct vbuffer * buf) {\n    for (int i = 0; i < GGML_VBUFFER_MAX_CHUNKS && buf->chunks[i]; ++i) {\n        ggml_backend_buffer_reset(buf->chunks[i]);\n    }\n}\n\n\n/////////////////////////////////////\n\n// graph allocator\n\nstruct hash_node {\n    int n_children;\n    int n_views;\n    int buffer_id;\n    struct buffer_address addr;\n    bool allocated;\n};\n\nstruct tensor_alloc {\n    int buffer_id;\n    struct buffer_address addr;\n    size_t size_max; // 0 = pre-allocated, unused, or view\n};\n\nstruct leaf_alloc {\n    struct tensor_alloc leaf;\n};\n\nstruct node_alloc {\n    struct tensor_alloc dst;\n    struct tensor_alloc src[GGML_MAX_SRC];\n};\n\nstruct ggml_gallocr {\n    ggml_backend_buffer_type_t * bufts; // [n_buffers]\n    struct vbuffer ** buffers; // [n_buffers]\n    struct ggml_dyn_tallocr ** buf_tallocs; // [n_buffers]\n    int n_buffers;\n\n    struct ggml_hash_set hash_set;\n    struct hash_node * hash_values; // [hash_set.size]\n\n    struct node_alloc * node_allocs; // [n_nodes]\n    int n_nodes;\n\n    struct leaf_alloc * leaf_allocs; // [n_leafs]\n    int n_leafs;\n};\n\nggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs) {\n    ggml_gallocr_t galloc = (ggml_gallocr_t)calloc(1, sizeof(struct ggml_gallocr));\n    GGML_ASSERT(galloc != NULL);\n\n    galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));\n    GGML_ASSERT(galloc->bufts != NULL);\n\n    galloc->buffers = calloc(n_bufs, sizeof(struct vbuffer *));\n    GGML_ASSERT(galloc->buffers != NULL);\n\n    galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));\n    GGML_ASSERT(galloc->buf_tallocs != NULL);\n\n    for (int i = 0; i < n_bufs; i++) {\n        galloc->bufts[i] = bufts[i];\n        galloc->buffers[i] = NULL;\n\n        // check if the same buffer type is used multiple times and reuse the same allocator\n        for (int j = 0; j < i; j++) {\n            if (bufts[i] == bufts[j]) {\n                galloc->buf_tallocs[i] = galloc->buf_tallocs[j];\n                break;\n            }\n        }\n\n        if (galloc->buf_tallocs[i] == NULL) {\n            size_t alignment = ggml_backend_buft_get_alignment(bufts[i]);\n            size_t max_size = ggml_backend_buft_get_max_size(bufts[i]);\n            galloc->buf_tallocs[i] = ggml_dyn_tallocr_new(alignment, max_size);\n        }\n    }\n    galloc->n_buffers = n_bufs;\n\n    return galloc;\n}\n\nggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft) {\n    return ggml_gallocr_new_n(&buft, 1);\n}\n\nvoid ggml_gallocr_free(ggml_gallocr_t galloc) {\n    if (galloc == NULL) {\n        return;\n    }\n\n    for (int i = 0; i < galloc->n_buffers; i++) {\n        if (galloc->buffers != NULL) {\n            // skip if already freed\n            bool freed = false;\n            for (int j = 0; j < i; j++) {\n                if (galloc->buffers[j] == galloc->buffers[i]) {\n                    freed = true;\n                    break;\n                }\n            }\n            if (!freed) {\n                ggml_vbuffer_free(galloc->buffers[i]);\n            }\n        }\n        if (galloc->buf_tallocs != NULL) {\n            // skip if already freed\n            bool freed = false;\n            for (int j = 0; j < i; j++) {\n                if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {\n                    freed = true;\n                    break;\n                }\n            }\n            if (!freed) {\n                ggml_dyn_tallocr_free(galloc->buf_tallocs[i]);\n            }\n        }\n    }\n\n    ggml_hash_set_free(&galloc->hash_set);\n    free(galloc->hash_values);\n    free(galloc->bufts);\n    free(galloc->buffers);\n    free(galloc->buf_tallocs);\n    free(galloc->node_allocs);\n    free(galloc->leaf_allocs);\n    free(galloc);\n}\n\ntypedef struct ggml_gallocr * ggml_gallocr_t;\n\nstatic struct hash_node * ggml_gallocr_hash_get(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n    size_t i = ggml_hash_find_or_insert(&galloc->hash_set, t);\n    return &galloc->hash_values[i];\n}\n\nstatic bool ggml_gallocr_is_own(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n    return ggml_gallocr_hash_get(galloc, t)->allocated;\n}\n\nstatic bool ggml_gallocr_is_allocated(ggml_gallocr_t galloc, struct ggml_tensor * t) {\n    return t->data != NULL // tensor data already set externally\n        || t->buffer // tensor on external buffer (but not yet allocated)\n        || ggml_gallocr_is_own(galloc, t); // tensor will be allocated by galloc\n}\n\n// free the extra space at the end if the new tensor is smaller\nstatic void ggml_gallocr_free_extra_space(ggml_gallocr_t galloc, struct ggml_tensor * node, struct ggml_tensor * parent) {\n    struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n    struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n\n    size_t parent_size = ggml_backend_buft_get_alloc_size(galloc->bufts[p_hn->buffer_id], parent);\n    size_t node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);\n\n    GGML_ASSERT(parent_size >= node_size);\n\n    // note: we want after the freeing the chunks to continue to be aligned\n    struct ggml_dyn_tallocr * p_alloc = galloc->buf_tallocs[p_hn->buffer_id];\n    parent_size = aligned_offset(NULL, parent_size, p_alloc->alignment);\n    node_size = aligned_offset(NULL, node_size, p_alloc->alignment);\n\n    if (parent_size > node_size) {\n        struct buffer_address p_addr = p_hn->addr;\n        p_addr.offset += node_size;\n        size_t extra_size = parent_size - node_size;\n        AT_PRINTF(\"freeing extra %zu bytes from parent %s for %s\\n\", extra_size, parent->name, node->name);\n        ggml_dyn_tallocr_free_bytes(p_alloc, p_addr, extra_size);\n    }\n}\n\nstatic void ggml_gallocr_allocate_node(ggml_gallocr_t galloc, struct ggml_tensor * node, int buffer_id) {\n    GGML_ASSERT(buffer_id >= 0);\n    struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n\n    if (!ggml_gallocr_is_allocated(galloc, node) && !ggml_impl_is_view(node)) {\n        hn->allocated = true;\n        assert(hn->addr.offset == 0);\n\n        // try to reuse a parent's buffer (inplace)\n        if (ggml_op_can_inplace(node->op)) {\n            for (int i = 0; i < GGML_MAX_SRC; i++) {\n                struct ggml_tensor * parent = node->src[i];\n                if (parent == NULL) {\n                    continue;\n                }\n\n                // if the node's data is external, then we cannot re-use it\n                if (!ggml_gallocr_is_own(galloc, parent)) {\n                    AT_PRINTF(\"not reusing parent %s for %s as %p is external\\n\", parent->name, node->name, parent->data);\n                    continue;\n                }\n\n                // outputs cannot be reused\n                if (parent->flags & GGML_TENSOR_FLAG_OUTPUT || (parent->view_src != NULL && parent->view_src->flags & GGML_TENSOR_FLAG_OUTPUT)) {\n                    AT_PRINTF(\"not reusing parent %s for %s as it is an output\\n\", parent->name, node->name);\n                    continue;\n                }\n\n                if (!ggml_are_same_layout(node, parent)) {\n                    AT_PRINTF(\"not reusing parent %s for %s as layouts are different\\n\", parent->name, node->name);\n                    continue;\n                }\n\n                struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n                if (p_hn->n_children == 1 && p_hn->n_views == 0) {\n                    if (ggml_impl_is_view(parent)) {\n                        struct ggml_tensor * view_src = parent->view_src;\n                        struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);\n                        if (view_src_hn->n_views == 1 && view_src_hn->n_children == 0 && view_src->data == parent->data) {\n                            AT_PRINTF(\"reusing view parent %s (%s) for %s\\n\", parent->name, view_src->name, node->name);\n                            assert(view_src_hn->addr.chunk == p_hn->addr.chunk && view_src_hn->addr.offset == p_hn->addr.offset);\n                            hn->buffer_id = p_hn->buffer_id;\n                            hn->addr = p_hn->addr;\n                            p_hn->allocated = false; // avoid freeing the parent\n                            view_src_hn->allocated = false;\n                            ggml_gallocr_free_extra_space(galloc, node, view_src);\n                            return;\n                        }\n                    } else {\n                        AT_PRINTF(\"reusing parent %s for %s\\n\", parent->name, node->name);\n                        hn->buffer_id = p_hn->buffer_id;\n                        hn->addr = p_hn->addr;\n                        p_hn->allocated = false; // avoid freeing the parent\n                        ggml_gallocr_free_extra_space(galloc, node, parent);\n                        return;\n                    }\n                }\n            }\n        }\n        // allocate tensor from the buffer\n        struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];\n        ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];\n        size_t size = ggml_backend_buft_get_alloc_size(buft, node);\n        hn->buffer_id = buffer_id;\n        hn->addr = ggml_dyn_tallocr_alloc(alloc, size, node);\n    }\n}\n\nstatic void ggml_gallocr_free_node(ggml_gallocr_t galloc, struct ggml_tensor * node) {\n    // graph outputs are never freed\n    if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {\n        AT_PRINTF(\"not freeing output %s\\n\", node->name);\n        return;\n    }\n\n    struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n    int buffer_id = hn->buffer_id;\n    struct ggml_dyn_tallocr * alloc = galloc->buf_tallocs[buffer_id];\n    ggml_backend_buffer_type_t buft = galloc->bufts[buffer_id];\n    size_t size = ggml_backend_buft_get_alloc_size(buft, node);\n\n    AT_PRINTF(\"%s: freeing %s at {chunk=%d, offset=%zu} (%zu bytes) - n_free_blocks = %d\\n\",\n        __func__, node->name, hn->addr.chunk, hn->addr.offset, size, alloc->chunks[hn->addr.chunk]->n_free_blocks);\n#ifdef GGML_ALLOCATOR_DEBUG\n    remove_allocated_tensor(alloc, hn->addr, node);\n#endif\n\n    ggml_dyn_tallocr_free_bytes(alloc, hn->addr, size);\n    hn->allocated = false;\n}\n\nstatic int get_node_buffer_id(const int * node_buffer_ids, int i) {\n    return node_buffer_ids ? node_buffer_ids[i] : 0;\n}\n\nstatic void ggml_gallocr_alloc_graph_impl(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {\n    // clear hash tables\n    ggml_hash_set_reset(&galloc->hash_set);\n    memset(galloc->hash_values, 0, sizeof(struct hash_node) * galloc->hash_set.size);\n\n    // allocate leafs\n    // these may be tensors that the application is not using in the graph, but may still want to allocate for other purposes\n    for (int i = 0; i < graph->n_leafs; i++) {\n        struct ggml_tensor * leaf = graph->leafs[i];\n        ggml_gallocr_allocate_node(galloc, leaf, get_node_buffer_id(leaf_buffer_ids, i));\n    }\n\n    // count number of children and views\n    // allocate other graph inputs and leafs first to avoid overwriting them\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n\n        // TODO: better way to add external dependencies\n        // GGML_OP_NONE does not appear normally in the graph nodes, but is used by ggml-backend to add dependencies to\n        // control when some tensors are allocated and freed. in this case, the dependencies are in `src`, but the node\n        // itself is never used and should not be considered a dependency\n        if (ggml_impl_is_view(node) && node->op != GGML_OP_NONE) {\n            struct ggml_tensor * view_src = node->view_src;\n            ggml_gallocr_hash_get(galloc, view_src)->n_views += 1;\n        }\n\n        if (node->flags & GGML_TENSOR_FLAG_INPUT) {\n            ggml_gallocr_allocate_node(galloc, graph->nodes[i], get_node_buffer_id(node_buffer_ids, i));\n        }\n\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * src = node->src[j];\n            if (src == NULL) {\n                continue;\n            }\n\n            ggml_gallocr_hash_get(galloc, src)->n_children += 1;\n\n            // allocate explicit inputs\n            if (src->flags & GGML_TENSOR_FLAG_INPUT) {\n                ggml_gallocr_allocate_node(galloc, src, get_node_buffer_id(node_buffer_ids, i));\n            }\n        }\n    }\n\n    // allocate tensors\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        int buffer_id = get_node_buffer_id(node_buffer_ids, i);\n\n        // allocate parents (only leafs need to be allocated at this point)\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * parent = node->src[j];\n            if (parent == NULL) {\n                continue;\n            }\n            ggml_gallocr_allocate_node(galloc, parent, buffer_id);\n        }\n\n        // allocate node\n        ggml_gallocr_allocate_node(galloc, node, buffer_id);\n\n        AT_PRINTF(\"exec: %s (%s) <= \", ggml_op_desc(node), node->name);\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * parent = node->src[j];\n            if (parent == NULL) {\n                continue;\n            }\n            AT_PRINTF(\"%s\", parent->name);\n            if (j < GGML_MAX_SRC - 1 && node->src[j + 1] != NULL) {\n                AT_PRINTF(\", \");\n            }\n        }\n        AT_PRINTF(\"\\n\");\n\n        // update parents\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * parent = node->src[j];\n            if (parent == NULL) {\n                continue;\n            }\n            struct hash_node * p_hn = ggml_gallocr_hash_get(galloc, parent);\n            p_hn->n_children -= 1;\n\n            AT_PRINTF(\"parent %s: %d children, %d views, allocated: %d\\n\",\n                parent->name, p_hn->n_children, p_hn->n_views, p_hn->allocated);\n\n            if (p_hn->n_children == 0 && p_hn->n_views == 0) {\n                if (ggml_impl_is_view(parent)) {\n                    struct ggml_tensor * view_src = parent->view_src;\n                    struct hash_node * view_src_hn = ggml_gallocr_hash_get(galloc, view_src);\n                    view_src_hn->n_views -= 1;\n                    AT_PRINTF(\"view_src %s: %d children, %d views\\n\",\n                        view_src->name, view_src_hn->n_children, view_src_hn->n_views);\n                    if (view_src_hn->n_views == 0 && view_src_hn->n_children == 0 && view_src_hn->allocated) {\n                        ggml_gallocr_free_node(galloc, view_src);\n                    }\n                }\n                else if (p_hn->allocated) {\n                    ggml_gallocr_free_node(galloc, parent);\n                }\n            }\n            AT_PRINTF(\"\\n\");\n        }\n    }\n}\n\nstatic bool ggml_gallocr_reserve_n_impl(\n        ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, bool no_alloc) {\n    size_t min_hash_size = graph->n_nodes + graph->n_leafs;\n    // add 25% margin to avoid hash collisions\n    min_hash_size += min_hash_size / 4;\n\n    // initialize hash table\n    if (galloc->hash_set.size < min_hash_size) {\n        ggml_hash_set_free(&galloc->hash_set);\n        galloc->hash_set = ggml_hash_set_new(min_hash_size);\n        GGML_ASSERT(galloc->hash_set.keys != NULL);\n\n        free(galloc->hash_values);\n        galloc->hash_values = malloc(sizeof(struct hash_node) * galloc->hash_set.size);\n        GGML_ASSERT(galloc->hash_values != NULL);\n    }\n\n    // reset allocators\n    for (int i = 0; i < galloc->n_buffers; i++) {\n        ggml_dyn_tallocr_reset(galloc->buf_tallocs[i]);\n    }\n\n    // allocate in hash table\n    ggml_gallocr_alloc_graph_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids);\n\n    // set the node_allocs from the hash table\n    if (galloc->n_nodes < graph->n_nodes) {\n        free(galloc->node_allocs);\n        galloc->node_allocs = calloc(graph->n_nodes, sizeof(struct node_alloc));\n        GGML_ASSERT(galloc->node_allocs != NULL);\n    }\n    galloc->n_nodes = graph->n_nodes;\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        struct node_alloc * node_alloc = &galloc->node_allocs[i];\n        if (node->view_src || node->data) {\n            node_alloc->dst.buffer_id = -1;\n            node_alloc->dst.addr = GGML_BUFFER_ADDRESS_INVALID;\n            node_alloc->dst.size_max = 0;\n        } else {\n            struct hash_node * hn = ggml_gallocr_hash_get(galloc, node);\n            node_alloc->dst.buffer_id = hn->buffer_id;\n            node_alloc->dst.addr = hn->addr;\n            node_alloc->dst.size_max  = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], node);\n        }\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * src = node->src[j];\n            if (!src || src->view_src || src->data) {\n                node_alloc->src[j].buffer_id = -1;\n                node_alloc->src[j].addr = GGML_BUFFER_ADDRESS_INVALID;\n                node_alloc->src[j].size_max = 0;\n            } else {\n                struct hash_node * hn = ggml_gallocr_hash_get(galloc, src);\n                node_alloc->src[j].buffer_id = hn->buffer_id;\n                node_alloc->src[j].addr = hn->addr;\n                node_alloc->src[j].size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], src);\n            }\n        }\n    }\n    if (galloc->n_leafs < graph->n_leafs) {\n        free(galloc->leaf_allocs);\n        galloc->leaf_allocs = calloc(graph->n_leafs, sizeof(galloc->leaf_allocs[0]));\n        GGML_ASSERT(galloc->leaf_allocs != NULL);\n    }\n    galloc->n_leafs = graph->n_leafs;\n    for (int i = 0; i < graph->n_leafs; i++) {\n        struct ggml_tensor * leaf = graph->leafs[i];\n        struct hash_node * hn = ggml_gallocr_hash_get(galloc, leaf);\n        if (leaf->view_src || leaf->data) {\n            galloc->leaf_allocs[i].leaf.buffer_id = -1;\n            galloc->leaf_allocs[i].leaf.addr = GGML_BUFFER_ADDRESS_INVALID;\n            galloc->leaf_allocs[i].leaf.size_max = 0;\n        } else {\n            galloc->leaf_allocs[i].leaf.buffer_id = hn->buffer_id;\n            galloc->leaf_allocs[i].leaf.addr = hn->addr;\n            galloc->leaf_allocs[i].leaf.size_max = ggml_backend_buft_get_alloc_size(galloc->bufts[hn->buffer_id], leaf);\n        }\n    }\n\n    // reallocate buffers if needed\n    for (int i = 0; i < galloc->n_buffers; i++) {\n        // if the buffer type is used multiple times, we reuse the same buffer\n        for (int j = 0; j < i; j++) {\n            if (galloc->buf_tallocs[j] == galloc->buf_tallocs[i]) {\n                galloc->buffers[i] = galloc->buffers[j];\n                break;\n            }\n        }\n\n        // even if there are no tensors allocated in this buffer, we still need to allocate it to initialize views\n        bool realloc = galloc->buffers[i] == NULL;\n        size_t new_size = 0;\n        for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) {\n            size_t cur_chunk_size = galloc->buffers[i] ? ggml_vbuffer_chunk_size(galloc->buffers[i], c) : 0;\n            size_t new_chunk_size = ggml_dyn_tallocr_max_size(galloc->buf_tallocs[i], c);\n            new_size += new_chunk_size;\n            if (new_chunk_size > cur_chunk_size) {\n                realloc = true;\n            }\n        }\n        if (realloc) {\n#ifndef NDEBUG\n            {\n                size_t cur_size = galloc->buffers[i] ? ggml_vbuffer_size(galloc->buffers[i]) : 0;\n                if (cur_size > 0) {\n                    GGML_LOG_DEBUG(\"%s: reallocating %s buffer from size %.02f MiB to %.02f MiB\\n\",\n                        __func__, ggml_backend_buft_name(galloc->bufts[i]), cur_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);\n                }\n            }\n#endif\n            ggml_vbuffer_free(galloc->buffers[i]);\n            if (no_alloc) {\n                galloc->buffers[i] = NULL;\n            } else {\n                galloc->buffers[i] = ggml_vbuffer_alloc(galloc->bufts[i], galloc->buf_tallocs[i], GGML_BACKEND_BUFFER_USAGE_COMPUTE);\n                if (galloc->buffers[i] == NULL) {\n                    GGML_LOG_ERROR(\"%s: failed to allocate %s buffer of size %zu\\n\", __func__, ggml_backend_buft_name(galloc->bufts[i]), new_size);\n                    return false;\n                }\n            }\n        }\n    }\n\n    return true;\n}\n\nvoid ggml_gallocr_reserve_n_size(\n        ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids, size_t * sizes) {\n    GGML_ASSERT(ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ true));\n    for (int i = 0; i < galloc->n_buffers; i++) {\n        sizes[i] = 0;\n        for (int c = 0; c < galloc->buf_tallocs[i]->n_chunks; c++) {\n            sizes[i] += galloc->buf_tallocs[i]->chunks[c]->max_size;\n        }\n    }\n}\n\nbool ggml_gallocr_reserve_n(ggml_gallocr_t galloc, struct ggml_cgraph * graph, const int * node_buffer_ids, const int * leaf_buffer_ids) {\n    return ggml_gallocr_reserve_n_impl(galloc, graph, node_buffer_ids, leaf_buffer_ids, /*no_alloc =*/ false);\n}\n\nbool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph *graph) {\n    return ggml_gallocr_reserve_n(galloc, graph, NULL, NULL);\n}\n\nstatic void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor * tensor, struct tensor_alloc * tensor_alloc) {\n    int buffer_id = tensor_alloc->buffer_id;\n    assert(tensor->data || tensor->view_src || ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max);\n\n    if (tensor->view_src != NULL) {\n        if (tensor->buffer == NULL) {\n            assert(tensor_alloc->addr.offset == SIZE_MAX);\n            if (tensor->view_src->buffer == NULL) {\n                // this tensor was allocated without ggml-backend\n                return;\n            }\n            ggml_backend_view_init(tensor);\n        }\n    } else {\n        if (tensor->data == NULL) {\n            assert(tensor_alloc->addr.offset != SIZE_MAX);\n            assert(ggml_backend_buft_get_alloc_size(galloc->bufts[buffer_id], tensor) <= tensor_alloc->size_max);\n            ggml_vbuffer_tensor_alloc(galloc->buffers[buffer_id], tensor, tensor_alloc->addr);\n        } else {\n            if (tensor->buffer == NULL) {\n                // this tensor was allocated without ggml-backend\n                return;\n            }\n        }\n    }\n}\n\nstatic bool ggml_gallocr_node_needs_realloc(ggml_gallocr_t galloc, struct ggml_tensor * node, struct tensor_alloc * talloc) {\n    size_t node_size = 0;\n    if (!node->data && !node->view_src) {\n        // If we previously had data but don't now then reallocate\n        if (talloc->buffer_id < 0) {\n            return false;\n        }\n        node_size = ggml_backend_buft_get_alloc_size(galloc->bufts[talloc->buffer_id], node);\n    }\n    return talloc->size_max >= node_size;\n}\n\nstatic bool ggml_gallocr_needs_realloc(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {\n    if (galloc->n_nodes != graph->n_nodes) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: graph has different number of nodes\\n\", __func__);\n#endif\n        return true;\n    }\n\n    if (galloc->n_leafs != graph->n_leafs) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: graph has different number of leafs\\n\", __func__);\n#endif\n        return true;\n    }\n\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        struct node_alloc * node_alloc = &galloc->node_allocs[i];\n\n        if (!ggml_gallocr_node_needs_realloc(galloc, node, &node_alloc->dst)) {\n#ifndef NDEBUG\n            GGML_LOG_DEBUG(\"%s: node %s is not valid\\n\", __func__, node->name);\n#endif\n            return true;\n        }\n\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * src = node->src[j];\n            if (src == NULL) {\n                continue;\n            }\n            if (!ggml_gallocr_node_needs_realloc(galloc, src, &node_alloc->src[j])) {\n#ifndef NDEBUG\n                GGML_LOG_DEBUG(\"%s: src %d (%s) of node %s is not valid\\n\", __func__, j, src->name, node->name);\n#endif\n                return true;\n            }\n        }\n    }\n\n    return false;\n}\n\nbool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph) {\n    if (ggml_gallocr_needs_realloc(galloc, graph)) {\n        if (galloc->n_buffers == 1) {\n#ifndef NDEBUG\n            GGML_LOG_DEBUG(\"%s: reallocating buffers automatically\\n\", __func__);\n#endif\n            if (!ggml_gallocr_reserve(galloc, graph)) {\n                return false;\n            }\n        } else {\n#ifndef NDEBUG\n            GGML_LOG_DEBUG(\"%s: cannot reallocate multi buffer graph automatically, call reserve\\n\", __func__);\n#endif\n            return false;\n        }\n    }\n\n    // reset buffers\n    for (int i = 0; i < galloc->n_buffers; i++) {\n        if (galloc->buffers[i] != NULL) {\n            ggml_vbuffer_reset(galloc->buffers[i]);\n        }\n    }\n\n    // allocate the graph tensors from the previous assignments\n    // leafs\n    for (int i = 0; i < graph->n_leafs; i++) {\n        struct ggml_tensor * leaf = graph->leafs[i];\n        struct leaf_alloc * leaf_alloc = &galloc->leaf_allocs[i];\n        ggml_gallocr_init_tensor(galloc, leaf, &leaf_alloc->leaf);\n    }\n    // nodes\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        struct node_alloc * node_alloc = &galloc->node_allocs[i];\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * src = node->src[j];\n            if (src == NULL) {\n                continue;\n            }\n            ggml_gallocr_init_tensor(galloc, src, &node_alloc->src[j]);\n        }\n        ggml_gallocr_init_tensor(galloc, node, &node_alloc->dst);\n    }\n\n    return true;\n}\n\nsize_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id) {\n    GGML_ASSERT(buffer_id >= 0 && buffer_id < galloc->n_buffers);\n\n    if (galloc->buffers[buffer_id] == NULL) {\n        return 0;\n    }\n\n    for (int i = 0; i < buffer_id; i++) {\n        if (galloc->buffers[i] == galloc->buffers[buffer_id]) {\n            // this buffer is the same as a previous one due to the same buffer type being used multiple times\n            // only return the buffer size the first time it appears to avoid double counting\n            return 0;\n        }\n    }\n\n    return ggml_vbuffer_size(galloc->buffers[buffer_id]);\n}\n\n// utils\n\nstatic void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buffers) {\n    for (size_t i = 0; i < *n_buffers; i++) {\n        ggml_backend_buffer_free((*buffers)[i]);\n    }\n    free(*buffers);\n}\n\nstatic bool alloc_tensor_range(struct ggml_context * ctx,\n        struct ggml_tensor * first, struct ggml_tensor * last,\n        ggml_backend_buffer_type_t buft, size_t size,\n        ggml_backend_buffer_t ** buffers, size_t * n_buffers) {\n\n    ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size);\n    if (buffer == NULL) {\n        GGML_LOG_ERROR(\"%s: failed to allocate %s buffer of size %zu\\n\", __func__, ggml_backend_buft_name(buft), size);\n        free_buffers(buffers, n_buffers);\n        return false;\n    }\n\n    *buffers = realloc(*buffers, sizeof(ggml_backend_buffer_t) * (*n_buffers + 1));\n    (*buffers)[(*n_buffers)++] = buffer;\n\n    struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);\n\n    for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {\n        enum ggml_status status = GGML_STATUS_SUCCESS;\n        if (t->data == NULL) {\n            if (t->view_src == NULL) {\n                status = ggml_tallocr_alloc(&tallocr, t);\n            } else if (t->buffer == NULL) {\n                status = ggml_backend_view_init(t);\n            }\n        } else {\n            if (t->view_src != NULL && t->buffer == NULL) {\n                // view of a pre-allocated tensor\n                status = ggml_backend_view_init(t);\n            }\n        }\n        if (status != GGML_STATUS_SUCCESS) {\n            GGML_LOG_ERROR(\"%s: failed to initialize tensor %s\\n\", __func__, t->name);\n            free_buffers(buffers, n_buffers);\n            return false;\n        }\n    }\n\n    return true;\n}\n\nstatic ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft_impl(\n        struct ggml_context * ctx, ggml_backend_buffer_type_t buft, size_t * nbytes_total, bool no_alloc) {\n    GGML_ASSERT(ggml_get_no_alloc(ctx) == true);\n\n    size_t alignment = ggml_backend_buft_get_alignment(buft);\n    size_t max_size = ggml_backend_buft_get_max_size(buft);\n\n    ggml_backend_buffer_t * buffers = NULL;\n    size_t n_buffers = 0;\n    *nbytes_total = 0;\n\n    size_t cur_buf_size = 0;\n    struct ggml_tensor * first = ggml_get_first_tensor(ctx);\n    for (struct ggml_tensor * t = first; t != NULL; t = ggml_get_next_tensor(ctx, t)) {\n        size_t this_size = 0;\n        if (t->data == NULL && t->view_src == NULL) {\n            this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);\n        }\n\n        if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) {\n            // allocate tensors in the current buffer\n            if (!no_alloc && !alloc_tensor_range(ctx, first, t, buft, cur_buf_size, &buffers, &n_buffers)) {\n                return NULL;\n            }\n            first = t;\n            *nbytes_total += cur_buf_size;\n            cur_buf_size = this_size;\n        } else {\n            cur_buf_size += this_size;\n        }\n    }\n\n    // allocate remaining tensors\n    if (cur_buf_size > 0) {\n        *nbytes_total += cur_buf_size;\n        if (!no_alloc && !alloc_tensor_range(ctx, first, NULL, buft, cur_buf_size, &buffers, &n_buffers)) {\n            return NULL;\n        }\n    }\n\n    if (no_alloc) {\n        return NULL;\n    }\n\n    if (n_buffers == 0) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: all tensors in the context are already allocated\\n\", __func__);\n#endif\n        GGML_ASSERT(!buffers);\n        return NULL;\n    }\n\n    ggml_backend_buffer_t buffer;\n    if (n_buffers == 1) {\n        buffer = buffers[0];\n    } else {\n        buffer = ggml_backend_multi_buffer_alloc_buffer(buffers, n_buffers);\n    }\n    if (buffers) {\n        free(buffers); // can be NULL if context is empty or no_alloc\n    }\n    return buffer;\n}\n\nsize_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {\n    size_t nbytes_total = 0;\n    ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc=*/ true);\n    GGML_ASSERT(!buf);\n    return nbytes_total;\n}\n\nggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft) {\n    size_t nbytes_total = 0;\n    return ggml_backend_alloc_ctx_tensors_from_buft_impl(ctx, buft, &nbytes_total, /*no_alloc =*/ false);\n}\n\nggml_backend_buffer_t ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend) {\n    return ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_get_default_buffer_type(backend));\n}\n"
  },
  {
    "path": "ggml/src/ggml-backend-dl.cpp",
    "content": "#include \"ggml-backend-dl.h\"\n\n#ifdef _WIN32\n\ndl_handle * dl_load_library(const fs::path & path) {\n    // suppress error dialogs for missing DLLs\n    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n    HMODULE handle = LoadLibraryW(path.wstring().c_str());\n\n    SetErrorMode(old_mode);\n\n    return handle;\n}\n\nvoid * dl_get_sym(dl_handle * handle, const char * name) {\n    DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);\n    SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);\n\n    void * p = (void *) GetProcAddress(handle, name);\n\n    SetErrorMode(old_mode);\n\n    return p;\n}\n\nconst char * dl_error() {\n    return \"\";\n}\n\n#else\n\ndl_handle * dl_load_library(const fs::path & path) {\n    dl_handle * handle = dlopen(path.string().c_str(), RTLD_NOW | RTLD_LOCAL);\n    return handle;\n}\n\nvoid * dl_get_sym(dl_handle * handle, const char * name) {\n    return dlsym(handle, name);\n}\n\nconst char * dl_error() {\n    const char *rslt = dlerror();\n    return rslt != nullptr ? rslt : \"\";\n}\n\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-backend-dl.h",
    "content": "#pragma once\n\n#ifdef _WIN32\n#   define WIN32_LEAN_AND_MEAN\n#   ifndef NOMINMAX\n#       define NOMINMAX\n#   endif\n#   include <windows.h>\n#   include <winevt.h>\n#else\n#    include <dlfcn.h>\n#    include <unistd.h>\n#endif\n#include <filesystem>\n\nnamespace fs = std::filesystem;\n\n#ifdef _WIN32\n\nusing dl_handle = std::remove_pointer_t<HMODULE>;\n\nstruct dl_handle_deleter {\n    void operator()(HMODULE handle) {\n        FreeLibrary(handle);\n    }\n};\n\n#else\n\nusing dl_handle = void;\n\nstruct dl_handle_deleter {\n    void operator()(void * handle) {\n        dlclose(handle);\n    }\n};\n\n#endif\n\nusing dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;\n\ndl_handle * dl_load_library(const fs::path & path);\nvoid * dl_get_sym(dl_handle * handle, const char * name);\nconst char * dl_error();\n\n"
  },
  {
    "path": "ggml/src/ggml-backend-impl.h",
    "content": "#pragma once\n\n// ggml-backend internal header\n\n#include \"ggml-backend.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\n    #define GGML_BACKEND_API_VERSION 2\n\n    //\n    // Backend buffer type\n    //\n\n    struct ggml_backend_buffer_type_i {\n        const char *          (*get_name)      (ggml_backend_buffer_type_t buft);\n        // allocate a buffer of this type\n        ggml_backend_buffer_t (*alloc_buffer)  (ggml_backend_buffer_type_t buft, size_t size);\n        // tensor alignment\n        size_t                (*get_alignment) (ggml_backend_buffer_type_t buft);\n        // (optional) max buffer size that can be allocated (defaults to SIZE_MAX)\n        size_t                (*get_max_size)  (ggml_backend_buffer_type_t buft);\n        // (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes)\n        size_t                (*get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);\n        // (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)\n        bool                  (*is_host)       (ggml_backend_buffer_type_t buft);\n    };\n\n    struct ggml_backend_buffer_type {\n        struct ggml_backend_buffer_type_i  iface;\n        ggml_backend_dev_t device;\n        void * context;\n    };\n\n    //\n    // Backend buffer\n    //\n\n    struct ggml_backend_buffer_i {\n        // (optional) free the buffer\n        void         (*free_buffer)  (ggml_backend_buffer_t buffer);\n        // base address of the buffer\n        void *       (*get_base)     (ggml_backend_buffer_t buffer);\n        // (optional) initialize a tensor in the buffer (eg. add tensor extras)\n        enum ggml_status (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);\n        // tensor data access\n        void         (*memset_tensor)(ggml_backend_buffer_t buffer,       struct ggml_tensor * tensor,     uint8_t value, size_t offset, size_t size);\n        void         (*set_tensor)   (ggml_backend_buffer_t buffer,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n        void         (*get_tensor)   (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);\n        // (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)\n        bool         (*cpy_tensor)   (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);\n        // clear the entire buffer\n        void         (*clear)        (ggml_backend_buffer_t buffer, uint8_t value);\n        // (optional) reset any internal state due to tensor initialization, such as tensor extras\n        void         (*reset)        (ggml_backend_buffer_t buffer);\n    };\n\n    struct ggml_backend_buffer {\n        struct ggml_backend_buffer_i  iface;\n        ggml_backend_buffer_type_t    buft;\n        void * context;\n        size_t size;\n        enum ggml_backend_buffer_usage usage;\n    };\n\n    GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(\n                   ggml_backend_buffer_type_t buft,\n            struct ggml_backend_buffer_i      iface,\n                   void *                     context,\n                   size_t                     size);\n\n    // do not use directly, use ggml_backend_tensor_copy instead\n    GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);\n\n    // multi-buffer\n    // buffer that contains a collection of buffers\n    GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);\n    GGML_API bool                  ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);\n    GGML_API void                  ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);\n\n    //\n    // Backend (stream)\n    //\n\n    struct ggml_backend_i {\n        const char * (*get_name)(ggml_backend_t backend);\n\n        void (*free)(ggml_backend_t backend);\n\n        // (optional) asynchronous tensor data access\n        void (*set_tensor_async)(ggml_backend_t backend,       struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n        void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor,       void * data, size_t offset, size_t size);\n        bool (*cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);\n\n        // (optional) complete all pending operations (required if the backend supports async operations)\n        void (*synchronize)(ggml_backend_t backend);\n\n        // (optional) graph plans (not used currently)\n        // compute graph with a plan\n        ggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);\n        void                      (*graph_plan_free)   (ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n        // update the plan with a new graph - this should be faster than creating a new plan when the graph has the same topology\n        void                      (*graph_plan_update) (ggml_backend_t backend, ggml_backend_graph_plan_t plan, const struct ggml_cgraph * cgraph);\n        // compute the graph with the plan\n        enum ggml_status          (*graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);\n\n        // compute graph (always async if supported by the backend)\n        enum ggml_status          (*graph_compute)     (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n\n        // (optional) event synchronization\n        // record an event on this stream\n        void (*event_record)(ggml_backend_t backend, ggml_backend_event_t event);\n        // wait for an event on on a different stream\n        void (*event_wait)  (ggml_backend_t backend, ggml_backend_event_t event);\n\n        // (optional) sort/optimize the nodes in the graph\n        void                      (*graph_optimize)    (ggml_backend_t backend, struct ggml_cgraph * cgraph);\n    };\n\n    struct ggml_backend {\n        ggml_guid_t guid;\n        struct ggml_backend_i iface;\n        ggml_backend_dev_t device;\n        void * context;\n    };\n\n    struct ggml_backend_event {\n        struct ggml_backend_device * device;\n        void * context;\n    };\n\n    //\n    // Backend device\n    //\n\n    // Note: if additional properties are needed, we should add a struct with all of them\n    //       the current functions to obtain the properties can remain, since they are more convenient for often used properties\n    struct ggml_backend_device_i {\n        // device name: short identifier for this device, such as \"CPU\" or \"CUDA0\"\n        const char * (*get_name)(ggml_backend_dev_t dev);\n\n        // device description: short informative description of the device, could be the model name\n        const char * (*get_description)(ggml_backend_dev_t dev);\n\n        // device memory in bytes: 0 bytes to indicate no memory to report\n        void         (*get_memory)(ggml_backend_dev_t dev, size_t * free, size_t * total);\n\n        // device type\n        enum ggml_backend_dev_type (*get_type)(ggml_backend_dev_t dev);\n\n        // device properties\n        void (*get_props)(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props);\n\n        // backend (stream) initialization\n        ggml_backend_t (*init_backend)(ggml_backend_dev_t dev, const char * params);\n\n        // preferred buffer type\n        ggml_backend_buffer_type_t (*get_buffer_type)(ggml_backend_dev_t dev);\n\n        // (optional) host buffer type (in system memory, typically this is a pinned memory buffer for faster transfers between host and device)\n        ggml_backend_buffer_type_t (*get_host_buffer_type)(ggml_backend_dev_t dev);\n\n        // (optional) buffer from pointer: create a buffer from a host pointer (useful for memory mapped models and importing data from other libraries)\n        ggml_backend_buffer_t (*buffer_from_host_ptr)(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size);\n\n        // check if the backend can compute an operation\n        bool (*supports_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);\n\n        // check if the backend can use tensors allocated in a buffer type\n        bool (*supports_buft)(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft);\n\n        // (optional) check if the backend wants to run an operation, even if the weights are allocated in an incompatible buffer\n        // these should be expensive operations that may benefit from running on this backend instead of the CPU backend\n        bool (*offload_op)(ggml_backend_dev_t dev, const struct ggml_tensor * op);\n\n        // (optional) event synchronization\n        ggml_backend_event_t (*event_new)         (ggml_backend_dev_t dev);\n        void                 (*event_free)        (ggml_backend_dev_t dev, ggml_backend_event_t event);\n        void                 (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event);\n    };\n\n    struct ggml_backend_device {\n        struct ggml_backend_device_i iface;\n        ggml_backend_reg_t reg;\n        void * context;\n    };\n\n    //\n    // Backend (reg)\n    //\n\n    struct ggml_backend_reg_i {\n        const char * (*get_name)(ggml_backend_reg_t reg);\n\n        // enumerate available devices\n        size_t             (*get_device_count)(ggml_backend_reg_t reg);\n        ggml_backend_dev_t (*get_device)(ggml_backend_reg_t reg, size_t index);\n\n        // (optional) get a pointer to a function in the backend\n        // backends can add custom functions that are not part of the standard ggml-backend interface\n        void * (*get_proc_address)(ggml_backend_reg_t reg, const char * name);\n    };\n\n    struct ggml_backend_reg {\n        int api_version; // initialize to GGML_BACKEND_API_VERSION\n        struct ggml_backend_reg_i iface;\n        void * context;\n    };\n\n    // Add backend dynamic loading support to the backend\n\n    // Initialize the backend\n    typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);\n    // Optional: obtain a score for the backend based on the system configuration\n    // Higher scores are preferred, 0 means the backend is not supported in the current system\n    typedef int                (*ggml_backend_score_t)(void);\n\n#ifdef GGML_BACKEND_DL\n#    ifdef __cplusplus\n#        define GGML_BACKEND_DL_IMPL(reg_fn)                             \\\n            extern \"C\" {                                                 \\\n            GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \\\n            }                                                            \\\n            ggml_backend_reg_t ggml_backend_init(void) {                 \\\n                return reg_fn();                                         \\\n            }\n#        define GGML_BACKEND_DL_SCORE_IMPL(score_fn)       \\\n            extern \"C\" {                                   \\\n            GGML_BACKEND_API int ggml_backend_score(void); \\\n            }                                              \\\n            int ggml_backend_score(void) {                 \\\n                return score_fn();                         \\\n            }\n#    else\n#        define GGML_BACKEND_DL_IMPL(reg_fn)                              \\\n            GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void);  \\\n            ggml_backend_reg_t                  ggml_backend_init(void) { \\\n                return reg_fn();                                          \\\n            }\n#        define GGML_BACKEND_DL_SCORE_IMPL(score_fn)        \\\n            GGML_BACKEND_API int ggml_backend_score(void);  \\\n            int                  ggml_backend_score(void) { \\\n                return score_fn();                          \\\n            }\n#    endif\n#else\n#    define GGML_BACKEND_DL_IMPL(reg_fn)\n#    define GGML_BACKEND_DL_SCORE_IMPL(score_fn)\n#endif\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-backend-reg.cpp",
    "content": "#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-backend-dl.h\"\n#include \"ggml-impl.h\"\n#include <algorithm>\n#include <cstring>\n#include <filesystem>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <vector>\n#include <cctype>\n\n#ifdef _WIN32\n#    define WIN32_LEAN_AND_MEAN\n#    ifndef NOMINMAX\n#        define NOMINMAX\n#    endif\n#    include <windows.h>\n#elif defined(__APPLE__)\n#    include <mach-o/dyld.h>\n#    include <dlfcn.h>\n#else\n#    include <dlfcn.h>\n#    include <unistd.h>\n#endif\n\n// Backend registry\n#ifdef GGML_USE_CPU\n#include \"ggml-cpu.h\"\n#endif\n\n#ifdef GGML_USE_CUDA\n#include \"ggml-cuda.h\"\n#endif\n\n#ifdef GGML_USE_METAL\n#include \"ggml-metal.h\"\n#endif\n\n#ifdef GGML_USE_SYCL\n#include \"ggml-sycl.h\"\n#endif\n\n#ifdef GGML_USE_VULKAN\n#include \"ggml-vulkan.h\"\n#endif\n\n#ifdef GGML_USE_WEBGPU\n#include \"ggml-webgpu.h\"\n#endif\n\n#ifdef GGML_USE_ZDNN\n#include \"ggml-zdnn.h\"\n#endif\n\n#ifdef GGML_USE_OPENCL\n#include \"ggml-opencl.h\"\n#endif\n\n#ifdef GGML_USE_HEXAGON\n#include \"ggml-hexagon.h\"\n#endif\n\n#ifdef GGML_USE_BLAS\n#include \"ggml-blas.h\"\n#endif\n\n#ifdef GGML_USE_RPC\n#include \"ggml-rpc.h\"\n#endif\n\n#ifdef GGML_USE_VIRTGPU_FRONTEND\n#include \"ggml-virtgpu.h\"\n#endif\n\n#ifdef GGML_USE_CANN\n#include \"ggml-cann.h\"\n#endif\n\n#ifdef GGML_USE_ZENDNN\n#include \"ggml-zendnn.h\"\n#endif\n\n#ifdef GGML_USE_OPENVINO\n#include \"ggml-openvino.h\"\n#endif\n\nnamespace fs = std::filesystem;\n\nstatic std::string path_str(const fs::path & path) {\n    try {\n#if defined(__cpp_lib_char8_t)\n        // C++20 and later: u8string() returns std::u8string\n        const std::u8string u8str = path.u8string();\n        return std::string(reinterpret_cast<const char *>(u8str.data()), u8str.size());\n#else\n        // C++17: u8string() returns std::string\n        return path.u8string();\n#endif\n    } catch (...) {\n        return std::string();\n    }\n}\n\nstruct ggml_backend_reg_entry {\n    ggml_backend_reg_t reg;\n    dl_handle_ptr handle;\n};\n\nstruct ggml_backend_registry {\n    std::vector<ggml_backend_reg_entry> backends;\n    std::vector<ggml_backend_dev_t> devices;\n\n    ggml_backend_registry() {\n#ifdef GGML_USE_CUDA\n        register_backend(ggml_backend_cuda_reg());\n#endif\n#ifdef GGML_USE_METAL\n        register_backend(ggml_backend_metal_reg());\n#endif\n#ifdef GGML_USE_SYCL\n        register_backend(ggml_backend_sycl_reg());\n#endif\n#ifdef GGML_USE_VULKAN\n    // Add runtime disable check\n    if (getenv(\"GGML_DISABLE_VULKAN\") == nullptr) {\n        register_backend(ggml_backend_vk_reg());\n    } else {\n        GGML_LOG_DEBUG(\"Vulkan backend disabled by GGML_DISABLE_VULKAN environment variable\\n\");\n    }\n#endif\n#ifdef GGML_USE_WEBGPU\n        register_backend(ggml_backend_webgpu_reg());\n#endif\n#ifdef GGML_USE_ZDNN\n        register_backend(ggml_backend_zdnn_reg());\n#endif\n#ifdef GGML_USE_VIRTGPU_FRONTEND\n        register_backend(ggml_backend_virtgpu_reg());\n#endif\n\n#ifdef GGML_USE_OPENCL\n        register_backend(ggml_backend_opencl_reg());\n#endif\n#ifdef GGML_USE_ZENDNN\n        register_backend(ggml_backend_zendnn_reg());\n#endif\n#ifdef GGML_USE_HEXAGON\n        register_backend(ggml_backend_hexagon_reg());\n#endif\n#ifdef GGML_USE_CANN\n        register_backend(ggml_backend_cann_reg());\n#endif\n#ifdef GGML_USE_BLAS\n        register_backend(ggml_backend_blas_reg());\n#endif\n#ifdef GGML_USE_RPC\n        register_backend(ggml_backend_rpc_reg());\n#endif\n#ifdef GGML_USE_OPENVINO\n        register_backend(ggml_backend_openvino_reg());\n#endif\n#ifdef GGML_USE_CPU\n        register_backend(ggml_backend_cpu_reg());\n#endif\n    }\n\n    ~ggml_backend_registry() {\n        // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,\n        // since backend threads may still be running and accessing resources from the dynamic library\n        for (auto & entry : backends) {\n            if (entry.handle) {\n                entry.handle.release(); // NOLINT\n            }\n        }\n    }\n\n    void register_backend(ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {\n        if (!reg) {\n            return;\n        }\n\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: registered backend %s (%zu devices)\\n\",\n            __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));\n#endif\n        backends.push_back({ reg, std::move(handle) });\n        for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {\n            register_device(ggml_backend_reg_dev_get(reg, i));\n        }\n    }\n\n    void register_device(ggml_backend_dev_t device) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: registered device %s (%s)\\n\", __func__, ggml_backend_dev_name(device), ggml_backend_dev_description(device));\n#endif\n        devices.push_back(device);\n    }\n\n    ggml_backend_reg_t load_backend(const fs::path & path, bool silent) {\n        dl_handle_ptr handle { dl_load_library(path) };\n        if (!handle) {\n            if (!silent) {\n                GGML_LOG_ERROR(\"%s: failed to load %s: %s\\n\", __func__, path_str(path).c_str(), dl_error());\n            }\n            return nullptr;\n        }\n\n        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), \"ggml_backend_score\");\n        if (score_fn && score_fn() == 0) {\n            if (!silent) {\n                GGML_LOG_INFO(\"%s: backend %s is not supported on this system\\n\", __func__, path_str(path).c_str());\n            }\n            return nullptr;\n        }\n\n        auto backend_init_fn = (ggml_backend_init_t) dl_get_sym(handle.get(), \"ggml_backend_init\");\n        if (!backend_init_fn) {\n            if (!silent) {\n                GGML_LOG_ERROR(\"%s: failed to find ggml_backend_init in %s\\n\", __func__, path_str(path).c_str());\n            }\n            return nullptr;\n        }\n\n        ggml_backend_reg_t reg = backend_init_fn();\n        if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {\n            if (!silent) {\n                if (!reg) {\n                    GGML_LOG_ERROR(\"%s: failed to initialize backend from %s: ggml_backend_init returned NULL\\n\",\n                        __func__, path_str(path).c_str());\n                } else {\n                    GGML_LOG_ERROR(\"%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\\n\",\n                        __func__, path_str(path).c_str(), reg->api_version, GGML_BACKEND_API_VERSION);\n                }\n            }\n            return nullptr;\n        }\n\n        GGML_LOG_INFO(\"%s: loaded %s backend from %s\\n\", __func__, ggml_backend_reg_name(reg), path_str(path).c_str());\n\n        register_backend(reg, std::move(handle));\n\n        return reg;\n    }\n\n    void unload_backend(ggml_backend_reg_t reg, bool silent) {\n        auto it = std::find_if(backends.begin(), backends.end(),\n                               [reg](const ggml_backend_reg_entry & entry) { return entry.reg == reg; });\n\n        if (it == backends.end()) {\n            if (!silent) {\n                GGML_LOG_ERROR(\"%s: backend not found\\n\", __func__);\n            }\n            return;\n        }\n\n        if (!silent) {\n            GGML_LOG_DEBUG(\"%s: unloading %s backend\\n\", __func__, ggml_backend_reg_name(reg));\n        }\n\n        // remove devices\n        devices.erase(\n            std::remove_if(devices.begin(), devices.end(),\n                            [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),\n            devices.end());\n\n        // remove backend\n        backends.erase(it);\n    }\n};\n\nstatic ggml_backend_registry & get_reg() {\n    static ggml_backend_registry reg;\n    return reg;\n}\n\n// Internal API\nvoid ggml_backend_register(ggml_backend_reg_t reg) {\n    get_reg().register_backend(reg);\n}\n\nvoid ggml_backend_device_register(ggml_backend_dev_t device) {\n    get_reg().register_device(device);\n}\n\n// Backend (reg) enumeration\nstatic bool striequals(const char * a, const char * b) {\n    for (; *a && *b; a++, b++) {\n        if (std::tolower(*a) != std::tolower(*b)) {\n            return false;\n        }\n    }\n    return *a == *b;\n}\n\nsize_t ggml_backend_reg_count() {\n    return get_reg().backends.size();\n}\n\nggml_backend_reg_t ggml_backend_reg_get(size_t index) {\n    GGML_ASSERT(index < ggml_backend_reg_count());\n    return get_reg().backends[index].reg;\n}\n\nggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {\n    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {\n        ggml_backend_reg_t reg = ggml_backend_reg_get(i);\n        if (striequals(ggml_backend_reg_name(reg), name)) {\n            return reg;\n        }\n    }\n    return nullptr;\n}\n\n// Device enumeration\nsize_t ggml_backend_dev_count() {\n    return get_reg().devices.size();\n}\n\nggml_backend_dev_t ggml_backend_dev_get(size_t index) {\n    GGML_ASSERT(index < ggml_backend_dev_count());\n    return get_reg().devices[index];\n}\n\nggml_backend_dev_t ggml_backend_dev_by_name(const char * name) {\n    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {\n        ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n        if (striequals(ggml_backend_dev_name(dev), name)) {\n            return dev;\n        }\n    }\n    return nullptr;\n}\n\nggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {\n    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {\n        ggml_backend_dev_t dev = ggml_backend_dev_get(i);\n        if (ggml_backend_dev_type(dev) == type) {\n            return dev;\n        }\n    }\n    return nullptr;\n}\n\n// Convenience functions\nggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {\n    ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);\n    if (!dev) {\n        return nullptr;\n    }\n    return ggml_backend_dev_init(dev, params);\n}\n\nggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {\n    ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);\n    if (!dev) {\n        return nullptr;\n    }\n    return ggml_backend_dev_init(dev, params);\n}\n\nggml_backend_t ggml_backend_init_best(void) {\n    ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);\n    dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU);\n    dev = dev ? dev : ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);\n    if (!dev) {\n        return nullptr;\n    }\n    return ggml_backend_dev_init(dev, nullptr);\n}\n\n// Dynamic loading\nggml_backend_reg_t ggml_backend_load(const char * path) {\n    return get_reg().load_backend(path, false);\n}\n\nvoid ggml_backend_unload(ggml_backend_reg_t reg) {\n    get_reg().unload_backend(reg, true);\n}\n\nstatic fs::path get_executable_path() {\n#if defined(__APPLE__)\n    // get executable path\n    std::vector<char> path;\n    uint32_t size;\n    while (true) {\n        size = path.size();\n        if (_NSGetExecutablePath(path.data(), &size) == 0) {\n            break;\n        }\n        path.resize(size);\n    }\n    std::string base_path(path.data(), size);\n    // remove executable name\n    auto last_slash = base_path.find_last_of('/');\n    if (last_slash != std::string::npos) {\n        base_path = base_path.substr(0, last_slash);\n    }\n    return base_path + \"/\";\n#elif defined(__linux__) || defined(__FreeBSD__)\n    std::string base_path = \".\";\n    std::vector<char> path(1024);\n    while (true) {\n        // get executable path\n#    if defined(__linux__)\n        ssize_t len = readlink(\"/proc/self/exe\", path.data(), path.size());\n#    elif defined(__FreeBSD__)\n        ssize_t len = readlink(\"/proc/curproc/file\", path.data(), path.size());\n#    endif\n        if (len == -1) {\n            break;\n        }\n        if (len < (ssize_t) path.size()) {\n            base_path = std::string(path.data(), len);\n            // remove executable name\n            auto last_slash = base_path.find_last_of('/');\n            if (last_slash != std::string::npos) {\n                base_path = base_path.substr(0, last_slash);\n            }\n            break;\n        }\n        path.resize(path.size() * 2);\n    }\n\n    return base_path + \"/\";\n#elif defined(_WIN32)\n    std::vector<wchar_t> path(MAX_PATH);\n    DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());\n    if (len == 0) {\n        return {};\n    }\n    std::wstring base_path(path.data(), len);\n    // remove executable name\n    auto last_slash = base_path.find_last_of('\\\\');\n    if (last_slash != std::string::npos) {\n        base_path = base_path.substr(0, last_slash);\n    }\n    return base_path + L\"\\\\\";\n#else\n    return {};\n#endif\n}\n\nstatic fs::path backend_filename_prefix() {\n#ifdef _WIN32\n    return fs::u8path(\"ggml-\");\n#else\n    return fs::u8path(\"libggml-\");\n#endif\n}\n\nstatic fs::path backend_filename_extension() {\n#ifdef _WIN32\n    return fs::u8path(\".dll\");\n#else\n    return fs::u8path(\".so\");\n#endif\n}\n\nstatic ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {\n    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths\n    const fs::path name_path = fs::u8path(name);\n    const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path(\"-\").native();\n    const fs::path file_extension = backend_filename_extension();\n\n    std::vector<fs::path> search_paths;\n    if (user_search_path == nullptr) {\n#ifdef GGML_BACKEND_DIR\n        search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));\n#endif\n        // default search paths: executable directory, current directory\n        search_paths.push_back(get_executable_path());\n        search_paths.push_back(fs::current_path());\n    } else {\n        search_paths.push_back(fs::u8path(user_search_path));\n    }\n\n    int best_score = 0;\n    fs::path best_path;\n    std::error_code ec;\n\n    for (const auto & search_path : search_paths) {\n        if (!fs::exists(search_path, ec)) {\n            if (ec) {\n                GGML_LOG_DEBUG(\"%s: posix_stat(%s) failure, error-message: %s\\n\", __func__, path_str(search_path).c_str(), ec.message().c_str());\n            } else {\n                GGML_LOG_DEBUG(\"%s: search path %s does not exist\\n\", __func__, path_str(search_path).c_str());\n            }\n            continue;\n        }\n        fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);\n        for (const auto & entry : dir_it) {\n            if (entry.is_regular_file(ec)) {\n                auto filename = entry.path().filename();\n                auto ext = entry.path().extension();\n                if (filename.native().find(file_prefix) == 0 && ext == file_extension) {\n                    dl_handle_ptr handle { dl_load_library(entry) };\n                    if (!handle && !silent) {\n                        GGML_LOG_ERROR(\"%s: failed to load %s: %s\\n\", __func__, path_str(entry.path()).c_str(), dl_error());\n                    }\n                    if (handle) {\n                        auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), \"ggml_backend_score\");\n                        if (score_fn) {\n                            int s = score_fn();\n#ifndef NDEBUG\n                            GGML_LOG_DEBUG(\"%s: %s score: %d\\n\", __func__, path_str(entry.path()).c_str(), s);\n#endif\n                            if (s > best_score) {\n                                best_score = s;\n                                best_path = entry.path();\n                            }\n                        } else {\n                            if (!silent) {\n                                GGML_LOG_INFO(\"%s: failed to find ggml_backend_score in %s\\n\", __func__, path_str(entry.path()).c_str());\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    if (best_score == 0) {\n        // try to load the base backend\n        for (const auto & search_path : search_paths) {\n            fs::path filename = backend_filename_prefix().native() + name_path.native() + backend_filename_extension().native();\n            fs::path path = search_path / filename;\n            if (std::error_code ec; fs::exists(path, ec)) {\n                return get_reg().load_backend(path, silent);\n            } else {\n                if (ec) {\n                    GGML_LOG_DEBUG(\"%s: posix_stat(%s) failure, error-message: %s\\n\", __func__, path_str(path).c_str(), ec.message().c_str());\n                }\n            }\n        }\n        return nullptr;\n    }\n\n    return get_reg().load_backend(best_path, silent);\n}\n\nvoid ggml_backend_load_all() {\n    ggml_backend_load_all_from_path(nullptr);\n}\n\nvoid ggml_backend_load_all_from_path(const char * dir_path) {\n#ifdef NDEBUG\n    bool silent = true;\n#else\n    bool silent = false;\n#endif\n\n    ggml_backend_load_best(\"blas\", silent, dir_path);\n    ggml_backend_load_best(\"zendnn\", silent, dir_path);\n    ggml_backend_load_best(\"cann\", silent, dir_path);\n    ggml_backend_load_best(\"cuda\", silent, dir_path);\n    ggml_backend_load_best(\"hip\", silent, dir_path);\n    ggml_backend_load_best(\"metal\", silent, dir_path);\n    ggml_backend_load_best(\"rpc\", silent, dir_path);\n    ggml_backend_load_best(\"sycl\", silent, dir_path);\n    ggml_backend_load_best(\"vulkan\", silent, dir_path);\n    ggml_backend_load_best(\"virtgpu\", silent, dir_path);\n    ggml_backend_load_best(\"opencl\", silent, dir_path);\n    ggml_backend_load_best(\"hexagon\", silent, dir_path);\n    ggml_backend_load_best(\"musa\", silent, dir_path);\n    ggml_backend_load_best(\"openvino\", silent, dir_path);\n    ggml_backend_load_best(\"cpu\", silent, dir_path);\n    // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend\n    const char * backend_path = std::getenv(\"GGML_BACKEND_PATH\");\n    if (backend_path) {\n        ggml_backend_load(backend_path);\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-backend.cpp",
    "content": "// Note: porting this file to C++ is a work in progress\n\n#ifdef _WIN32\n#define WIN32_LEAN_AND_MEAN\n#ifndef NOMINMAX\n#   define NOMINMAX\n#endif\n#include <windows.h>\n#endif\n\n#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-alloc.h\"\n#include \"ggml-impl.h\"\n\n#include <assert.h>\n#include <limits.h>\n#include <stdarg.h>\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <algorithm>\n#include <vector>\n\n#ifdef __APPLE__\n#include <sys/types.h>\n#include <sys/sysctl.h>\n#endif\n\n\n// backend buffer type\n\nconst char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(buft);\n    return buft->iface.get_name(buft);\n}\n\nggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    GGML_ASSERT(buft);\n    if (size == 0) {\n        // return a dummy buffer for zero-sized allocations\n        return ggml_backend_buffer_init(buft, {}, NULL, 0);\n    }\n    return buft->iface.alloc_buffer(buft, size);\n}\n\nsize_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(buft);\n    return buft->iface.get_alignment(buft);\n}\n\nsize_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(buft);\n    // get_max_size is optional, defaults to SIZE_MAX\n    if (buft->iface.get_max_size) {\n        return buft->iface.get_max_size(buft);\n    }\n    return SIZE_MAX;\n}\n\nsize_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {\n    GGML_ASSERT(buft);\n    // get_alloc_size is optional, defaults to ggml_nbytes\n    if (buft->iface.get_alloc_size) {\n        size_t size = buft->iface.get_alloc_size(buft, tensor);\n        assert(size >= ggml_nbytes(tensor));\n        return size;\n    }\n    return ggml_nbytes(tensor);\n}\n\nbool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(buft);\n    if (buft->iface.is_host) {\n        return buft->iface.is_host(buft);\n    }\n    return false;\n}\n\nggml_backend_dev_t ggml_backend_buft_get_device(ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(buft);\n    return buft->device;\n}\n\n// backend buffer\n\nggml_backend_buffer_t ggml_backend_buffer_init(\n               ggml_backend_buffer_type_t buft,\n        struct ggml_backend_buffer_i      iface,\n               void *                     context,\n               size_t                     size) {\n    ggml_backend_buffer_t buffer = new ggml_backend_buffer {\n        /* .interface = */ iface,\n        /* .buft      = */ buft,\n        /* .context   = */ context,\n        /* .size      = */ size,\n        /* .usage     = */ GGML_BACKEND_BUFFER_USAGE_ANY\n    };\n\n    return buffer;\n}\n\nconst char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {\n    return ggml_backend_buft_name(ggml_backend_buffer_get_type(buffer));\n}\n\nvoid ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {\n    if (buffer == NULL) {\n        return;\n    }\n\n    if (buffer->iface.free_buffer != NULL) {\n        buffer->iface.free_buffer(buffer);\n    }\n    delete buffer;\n}\n\nsize_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    return buffer->size;\n}\n\nvoid * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    // get_base is optional if the buffer is zero-sized\n    if (buffer->size == 0) {\n        return NULL;\n    }\n\n    // FIXME JG: a multi_buffer has a non-zero size, according to the above comment get_base is not optional,\n    //     I don't know whether the above comment is correct\n    if (!buffer->iface.get_base) {\n        return NULL;\n    }\n\n    void * base = buffer->iface.get_base(buffer);\n\n    GGML_ASSERT(base != NULL && \"backend buffer base cannot be NULL\");\n\n    return base;\n}\n\nenum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n    GGML_ASSERT(buffer);\n    // init_tensor is optional\n    if (buffer->iface.init_tensor) {\n        return buffer->iface.init_tensor(buffer, tensor);\n    }\n    return GGML_STATUS_SUCCESS;\n}\n\nvoid ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n    GGML_ASSERT(buffer);\n    // clear is optional if the buffer is zero-sized\n    if (buffer->size == 0) {\n        return;\n    }\n\n    buffer->iface.clear(buffer, value);\n}\n\nsize_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {\n    return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));\n}\n\nsize_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {\n    return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));\n}\n\nsize_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor) {\n    return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);\n}\n\nbool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {\n    return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));\n}\n\nvoid ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {\n    GGML_ASSERT(buffer);\n    buffer->usage = usage;\n\n    // FIXME: add a generic callback to the buffer interface\n    if (ggml_backend_buffer_is_multi_buffer(buffer)) {\n        ggml_backend_multi_buffer_set_usage(buffer, usage);\n    }\n}\n\nenum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    return buffer->usage;\n}\n\nggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    return buffer->buft;\n}\n\nvoid ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    if (buffer->iface.reset) {\n        buffer->iface.reset(buffer);\n    }\n}\n\nbool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {\n    ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;\n    if (dst_buf->iface.cpy_tensor) {\n        return dst_buf->iface.cpy_tensor(dst_buf, src, dst);\n    }\n    return false;\n}\n\n// backend\n\nggml_guid_t ggml_backend_guid(ggml_backend_t backend) {\n    if (backend == NULL) {\n        return NULL;\n    }\n    return backend->guid;\n}\n\nconst char * ggml_backend_name(ggml_backend_t backend) {\n    if (backend == NULL) {\n        return \"NULL\";\n    }\n    return backend->iface.get_name(backend);\n}\n\nvoid ggml_backend_free(ggml_backend_t backend) {\n    if (backend == NULL) {\n        return;\n    }\n\n    backend->iface.free(backend);\n}\n\nggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {\n    GGML_ASSERT(backend);\n    return ggml_backend_dev_buffer_type(backend->device);\n}\n\nggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {\n    return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);\n}\n\nsize_t ggml_backend_get_alignment(ggml_backend_t backend) {\n    return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));\n}\n\nsize_t ggml_backend_get_max_size(ggml_backend_t backend) {\n    return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));\n}\n\nvoid ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(tensor);\n    GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n\n    if (backend->iface.set_tensor_async == NULL) {\n        ggml_backend_synchronize(backend);\n        ggml_backend_tensor_set(tensor, data, offset, size);\n    } else {\n        backend->iface.set_tensor_async(backend, tensor, data, offset, size);\n    }\n}\n\nvoid ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(tensor);\n    GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor read out of bounds\");\n\n    if (backend->iface.get_tensor_async == NULL) {\n        ggml_backend_synchronize(backend);\n        ggml_backend_tensor_get(tensor, data, offset, size);\n    } else {\n        backend->iface.get_tensor_async(backend, tensor, data, offset, size);\n    }\n}\n\nvoid ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n    if (size == 0) {\n        return;\n    }\n\n    GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n    GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n\n    buf->iface.set_tensor(buf, tensor, data, offset, size);\n}\n\nvoid ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n    if (size == 0) {\n        return;\n    }\n\n    GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n    GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor read out of bounds\");\n\n    buf->iface.get_tensor(buf, tensor, data, offset, size);\n}\n\nvoid ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n    if (size == 0) {\n        return;\n    }\n\n    GGML_ASSERT(buf != NULL && \"tensor buffer not set\");\n    GGML_ASSERT(tensor->data != NULL && \"tensor not allocated\");\n    GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && \"tensor write out of bounds\");\n    GGML_ASSERT(buf->iface.memset_tensor != NULL && \"memset not implemented by backend buffer\");\n\n    buf->iface.memset_tensor(buf, tensor, value, offset, size);\n}\n\nvoid ggml_backend_synchronize(ggml_backend_t backend) {\n    GGML_ASSERT(backend);\n    if (backend->iface.synchronize == NULL) {\n        return;\n    }\n\n    backend->iface.synchronize(backend);\n}\n\nggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(backend->iface.graph_plan_create != NULL);\n\n    return backend->iface.graph_plan_create(backend, cgraph);\n}\n\nvoid ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(backend->iface.graph_plan_free != NULL);\n\n    backend->iface.graph_plan_free(backend, plan);\n}\n\nenum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(backend->iface.graph_plan_compute != NULL);\n\n    return backend->iface.graph_plan_compute(backend, plan);\n}\n\nenum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);\n    ggml_backend_synchronize(backend);\n    return err;\n}\n\nenum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    GGML_ASSERT(backend);\n    return backend->iface.graph_compute(backend, cgraph);\n}\n\nbool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {\n    GGML_ASSERT(backend);\n    return ggml_backend_dev_supports_op(backend->device, op);\n}\n\nbool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(backend);\n    return ggml_backend_dev_supports_buft(backend->device, buft);\n}\n\nbool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {\n    GGML_ASSERT(backend);\n    return ggml_backend_dev_offload_op(backend->device, op);\n}\n\nggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend) {\n    GGML_ASSERT(backend);\n    return backend->device;\n}\n\n// backend copy\n\nvoid ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst) {\n    GGML_ASSERT(ggml_are_same_layout(src, dst) && \"cannot copy tensors with different layouts\");\n\n    if (src == dst) {\n        return;\n    }\n\n    if (ggml_backend_buffer_is_host(src->buffer)) {\n        ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));\n    } else if (ggml_backend_buffer_is_host(dst->buffer)) {\n        ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));\n    } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: warning: slow copy from %s to %s\\n\", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));\n#endif\n        size_t nbytes = ggml_nbytes(src);\n        void * data = malloc(nbytes);\n        ggml_backend_tensor_get(src, data, 0, nbytes);\n        ggml_backend_tensor_set(dst, data, 0, nbytes);\n        free(data);\n    }\n}\n\nvoid ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst) {\n    GGML_ASSERT(ggml_are_same_layout(src, dst) && \"cannot copy tensors with different layouts\");\n\n    if (src == dst) {\n        return;\n    }\n\n    GGML_ASSERT(backend_dst);\n    if (backend_dst->iface.cpy_tensor_async != NULL) {\n        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {\n            return;\n        }\n    }\n\n    // an async copy would normally happen after all the queued operations on both backends are completed\n    // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy\n    ggml_backend_synchronize(backend_src);\n    ggml_backend_synchronize(backend_dst);\n    ggml_backend_tensor_copy(src, dst);\n}\n\n// events\n\nggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device) {\n    // null device is allowed for the transition period to the device interface\n    if (device == NULL || device->iface.event_new == NULL) {\n        return NULL;\n    }\n    return device->iface.event_new(device);\n}\n\nvoid ggml_backend_event_free(ggml_backend_event_t event) {\n    if (event == NULL) {\n        return;\n    }\n    event->device->iface.event_free(event->device, event);\n}\n\nvoid ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(backend->iface.event_record != NULL);\n\n    backend->iface.event_record(backend, event);\n}\n\nvoid ggml_backend_event_synchronize(ggml_backend_event_t event) {\n    GGML_ASSERT(event);\n    GGML_ASSERT(event->device->iface.event_synchronize);\n\n    event->device->iface.event_synchronize(event->device, event);\n}\n\nvoid ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n    GGML_ASSERT(backend);\n    GGML_ASSERT(backend->iface.event_wait != NULL);\n\n    backend->iface.event_wait(backend, event);\n}\n\nstatic void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    GGML_ASSERT(backend);\n    if (backend->iface.graph_optimize != NULL) {\n        backend->iface.graph_optimize(backend, cgraph);\n    }\n}\n\n// Backend device\n\nconst char * ggml_backend_dev_name(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    return device->iface.get_name(device);\n}\n\nconst char * ggml_backend_dev_description(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    return device->iface.get_description(device);\n}\n\nvoid ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total) {\n    GGML_ASSERT(device);\n    device->iface.get_memory(device, free, total);\n}\n\nenum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    return device->iface.get_type(device);\n}\n\nvoid ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props) {\n    memset(props, 0, sizeof(*props));\n    device->iface.get_props(device, props);\n}\n\nggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    return device->reg;\n}\n\nggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params) {\n    GGML_ASSERT(device);\n    return device->iface.init_backend(device, params);\n}\n\nggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    return device->iface.get_buffer_type(device);\n}\n\nggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device) {\n    GGML_ASSERT(device);\n    if (device->iface.get_host_buffer_type == NULL) {\n        return NULL;\n    }\n\n    return device->iface.get_host_buffer_type(device);\n}\n\nggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) {\n    GGML_ASSERT(device);\n    return device->iface.buffer_from_host_ptr(device, ptr, size, max_tensor_size);\n}\n\nbool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {\n    GGML_ASSERT(device);\n    return device->iface.supports_op(device, op);\n}\n\nbool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {\n    GGML_ASSERT(device);\n    return device->iface.supports_buft(device, buft);\n}\n\nbool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {\n    GGML_ASSERT(device);\n    if (device->iface.offload_op != NULL) {\n        return device->iface.offload_op(device, op);\n    }\n\n    return false;\n}\n\n// Backend (reg)\n\nconst char * ggml_backend_reg_name(ggml_backend_reg_t reg) {\n    GGML_ASSERT(reg);\n    return reg->iface.get_name(reg);\n}\n\nsize_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg) {\n    GGML_ASSERT(reg);\n    return reg->iface.get_device_count(reg);\n}\n\nggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index) {\n    GGML_ASSERT(reg);\n    return reg->iface.get_device(reg, index);\n}\n\nvoid * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n    GGML_ASSERT(reg);\n    if (!reg->iface.get_proc_address) {\n        return NULL;\n    }\n    return reg->iface.get_proc_address(reg, name);\n}\n\n// multi-buffer buffer\n\nstruct ggml_backend_multi_buffer_context {\n    ggml_backend_buffer_t * buffers;\n    size_t n_buffers;\n};\n\nstatic void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n    for (size_t i = 0; i < ctx->n_buffers; i++) {\n        ggml_backend_buffer_free(ctx->buffers[i]);\n    }\n\n    free(ctx->buffers);\n    free(ctx);\n}\n\nstatic void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n    GGML_ASSERT(buffer);\n    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n    for (size_t i = 0; i < ctx->n_buffers; i++) {\n        ggml_backend_buffer_clear(ctx->buffers[i], value);\n    }\n}\n\nstatic const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {\n    /* .free_buffer     = */ ggml_backend_multi_buffer_free_buffer,\n    /* .get_base        = */ NULL,\n    /* .init_tensor     = */ NULL,\n    /* .memset_tensor   = */ NULL,\n    /* .set_tensor      = */ NULL,\n    /* .get_tensor      = */ NULL,\n    /* .cpy_tensor      = */ NULL,\n    /* .clear           = */ ggml_backend_multi_buffer_clear,\n    /* .reset           = */ NULL,\n};\n\nggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {\n    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) malloc(sizeof(struct ggml_backend_multi_buffer_context));\n    ctx->n_buffers = n_buffers;\n    ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));\n\n    GGML_ASSERT(ctx->buffers != NULL);\n\n    size_t total_size = 0;\n    for (size_t i = 0; i < n_buffers; i++) {\n        ctx->buffers[i] = buffers[i];\n        total_size += ggml_backend_buffer_get_size(buffers[i]);\n    }\n\n    return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_i, ctx, total_size);\n}\n\nbool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    return buffer->iface.free_buffer == ggml_backend_multi_buffer_free_buffer;\n}\n\nvoid ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {\n    GGML_ASSERT(buffer);\n    GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));\n    ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;\n    for (size_t i = 0; i < ctx->n_buffers; i++) {\n        ggml_backend_buffer_set_usage(ctx->buffers[i], usage);\n    }\n}\n\n// creates a copy of the tensor with the same memory layout\nstatic struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {\n    struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);\n    for (int i = 0; i < GGML_MAX_DIMS; i++) {\n        dup->nb[i] = tensor->nb[i];\n    }\n    return dup;\n}\n\nstatic bool ggml_is_view_op(enum ggml_op op) {\n    return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;\n}\n\n// scheduler\n\n#ifndef GGML_SCHED_MAX_BACKENDS\n#define GGML_SCHED_MAX_BACKENDS 16\n#endif\n\n#ifndef GGML_SCHED_MAX_SPLIT_INPUTS\n#define GGML_SCHED_MAX_SPLIT_INPUTS 30\n#endif\n\n#ifndef GGML_SCHED_MAX_COPIES\n#define GGML_SCHED_MAX_COPIES 4\n#endif\n\nstruct ggml_backend_sched_split {\n    int backend_id;\n    int i_start;\n    int i_end;\n    struct ggml_tensor * inputs[GGML_SCHED_MAX_SPLIT_INPUTS];\n    int n_inputs;\n    // graph view of this split\n    struct ggml_cgraph graph;\n};\n\nstruct ggml_backend_sched {\n    bool is_reset; // true if the scheduler has been reset since the last graph split\n    bool is_alloc;\n\n    int n_backends;\n\n    ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];\n    ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];\n    ggml_gallocr_t galloc;\n\n    // hash map of the nodes in the graph\n    struct ggml_hash_set  hash_set;\n    int                 * hv_tensor_backend_ids; // [hash_set.size]\n    struct ggml_tensor ** hv_tensor_copies;      // [hash_set.size][n_backends][n_copies]\n\n    int * node_backend_ids; // [graph_size]\n    int * leaf_backend_ids; // [graph_size]\n\n    int * prev_node_backend_ids; // [graph_size]\n    int * prev_leaf_backend_ids; // [graph_size]\n\n    // copy of the graph with modified inputs\n    struct ggml_cgraph graph;\n\n    // graph splits\n    struct ggml_backend_sched_split * splits;\n    int n_splits;\n    int splits_capacity;\n\n    // pipeline parallelism support\n    int n_copies;\n    int cur_copy;\n    int next_copy;\n    ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];\n    struct ggml_tensor * graph_inputs[GGML_SCHED_MAX_SPLIT_INPUTS];\n    int n_graph_inputs;\n\n    struct ggml_context * ctx;\n\n    ggml_backend_sched_eval_callback callback_eval;\n    void * callback_eval_user_data;\n\n    char * context_buffer;\n    size_t context_buffer_size;\n\n    bool op_offload;\n\n    int debug;\n\n    // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]\n    // ref: https://github.com/ggml-org/llama.cpp/pull/17617\n    int debug_realloc;\n    int debug_graph_size;\n    int debug_prev_graph_size;\n};\n\n#define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)\n#define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]\n#define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]\n#define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)\n\n// returns the priority of the backend, lower id is higher priority\nstatic int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {\n    for (int i = 0; i < sched->n_backends; i++) {\n        if (sched->backends[i] == backend) {\n            return i;\n        }\n    }\n    return -1;\n}\n\nstatic int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {\n    ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n    if (buffer == NULL) {\n        return -1;\n    }\n\n    // find highest prio backend that supports the buffer type and the op\n    for (int i = 0; i < sched->n_backends; i++) {\n        if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&\n            ggml_backend_supports_op(sched->backends[i], op)) {\n            return i;\n        }\n    }\n\n#ifndef NDEBUG\n    GGML_LOG_DEBUG(\"%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\\n\",\n        __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);\n#endif\n\n    return -1;\n}\n\n#if 0\n#define GGML_SCHED_MAX_SPLITS_DEBUG 4096\nstatic char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only\n#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)\n#define GET_CAUSE(node) causes[hash_id(node)]\n#else\n#define SET_CAUSE(node, ...)\n#define GET_CAUSE(node) \"\"\n#endif\n\n// returns the backend that should be used for the node based on the current locations\nstatic int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {\n    // assign pre-allocated nodes to their backend\n    int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);\n    if (cur_backend_id != -1) {\n        SET_CAUSE(tensor, \"1.dst\");\n        return cur_backend_id;\n    }\n\n    // view_src\n    if (tensor->view_src != NULL) {\n        cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);\n        if (cur_backend_id != -1) {\n            SET_CAUSE(tensor, \"1.vsrc\");\n            return cur_backend_id;\n        }\n    }\n\n    if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {\n        // since the tensor is pre-allocated, it cannot be moved to another backend\n        ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n        GGML_ABORT(\"pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)\", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));\n    }\n\n    // graph input\n    if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {\n        cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)\n        SET_CAUSE(tensor, \"1.inp\");\n        return cur_backend_id;\n    }\n\n    // operations with weights are preferably run on the same backend as the weights\n    for (int i = 0; i < GGML_MAX_SRC; i++) {\n        const struct ggml_tensor * src = tensor->src[i];\n        if (src == NULL) {\n            continue;\n        }\n        // skip ROPE since the rope freqs tensor is too small to choose a backend based on it\n        // not an ideal solution\n        if (tensor->op != GGML_OP_ROPE && src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {\n            int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);\n            // check if a backend with higher prio wants to offload the op\n            if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) {\n                for (int b = 0; b < src_backend_id; b++) {\n                    if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {\n                        SET_CAUSE(tensor, \"1.off\");\n                        return b;\n                    }\n                }\n            }\n            SET_CAUSE(tensor, \"1.wgt%d\", i);\n            return src_backend_id;\n        }\n    }\n\n    return -1;\n}\n\nstatic char * fmt_size(size_t size) {\n    static char buffer[128];\n    if (size >= 1024*1024) {\n        snprintf(buffer, sizeof(buffer), \"%zuM\", size/1024/1024);\n    } else {\n        snprintf(buffer, sizeof(buffer), \"%zuK\", size/1024);\n    }\n    return buffer;\n}\n\nstatic void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n    int cur_split = 0;\n    for (int i = 0; i < graph->n_nodes; i++) {\n        if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {\n            ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];\n            GGML_LOG_DEBUG(\"\\n## SPLIT #%d: %s # %d inputs\", cur_split, ggml_backend_name(split_backend),\n                sched->splits[cur_split].n_inputs);\n            for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {\n                if (j == 0) {\n                    GGML_LOG_DEBUG(\": \");\n                }\n                GGML_LOG_DEBUG(\"[%s (%5.5s)] \", sched->splits[cur_split].inputs[j]->name,\n                    fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));\n            }\n            GGML_LOG_DEBUG(\"\\n\");\n            cur_split++;\n        }\n        struct ggml_tensor * node = graph->nodes[i];\n        if (ggml_is_view_op(node->op)) {\n            continue;\n        }\n        if (sched->debug > 1) {\n            ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);\n            GGML_LOG_DEBUG(\"node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:\", i, ggml_op_name(node->op), node->name,\n                fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : \"NULL\", GET_CAUSE(node),\n                graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);\n            for (int j = 0; j < GGML_MAX_SRC; j++) {\n                struct ggml_tensor * src = node->src[j];\n                if (src == NULL) {\n                    continue;\n                }\n                ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);\n                GGML_LOG_DEBUG(\" %20.20s (%5.5s) [%5.5s %8.8s]\", src->name,\n                    fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : \"NULL\", GET_CAUSE(src));\n            }\n            GGML_LOG_DEBUG(\"\\n\");\n        }\n    }\n}\n\nstatic bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {\n    ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;\n    ggml_backend_buffer_type_t buft = NULL;\n\n    if (buf) {\n        // the tensor is already allocated\n        buft = buf->buft;\n    } else {\n        // see if the tensor already has a backend assigned, and use the buffer type of that backend\n        int tensor_backend_id = tensor_backend_id(t);\n        if (tensor_backend_id == -1 && t->view_src) {\n            tensor_backend_id = tensor_backend_id(t->view_src);\n        }\n        if (tensor_backend_id != -1) {\n            buft = sched->bufts[tensor_backend_id];\n        }\n    }\n\n    return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);\n}\n\nstatic void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {\n    if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {\n        *node_backend_id = cur_backend_id;\n        SET_CAUSE(node, \"2.sup\");\n    }\n}\n\n// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend\nvoid ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n    // reset splits\n    sched->n_splits = 0;\n    sched->n_graph_inputs = 0;\n    sched->is_reset = false;\n\n    struct ggml_init_params params = {\n        /* .mem_size =   */ sched->context_buffer_size,\n        /* .mem_buffer = */ sched->context_buffer,\n        /* .no_alloc =   */ true\n    };\n\n    ggml_free(sched->ctx);\n\n    sched->ctx = ggml_init(params);\n    if (sched->ctx == NULL) {\n        GGML_ABORT(\"%s: failed to initialize context\\n\", __func__);\n    }\n\n    // pass 1: assign backends to ops with pre-allocated inputs\n    for (int i = 0; i < graph->n_leafs; i++) {\n        struct ggml_tensor * leaf = graph->leafs[i];\n        int * leaf_backend_id = &tensor_backend_id(leaf);\n        // do not overwrite user assignments\n        if (*leaf_backend_id == -1) {\n            *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);\n        }\n    }\n\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        int * node_backend_id = &tensor_backend_id(node);\n        // do not overwrite user assignments\n        if (*node_backend_id == -1) {\n            *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);\n\n#if 0\n            // src\n            if (node->op == GGML_OP_NONE) {\n                continue;\n            }\n\n            for (int j = 0; j < GGML_MAX_SRC; j++) {\n                struct ggml_tensor * src = node->src[j];\n                if (src == NULL) {\n                    continue;\n                }\n                int * src_backend_id = &tensor_backend_id(src);\n                if (*src_backend_id == -1) {\n                    *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);\n                }\n            }\n#endif\n        }\n    }\n\n    // pass 2: expand current backend assignments\n    // assign the same backend to adjacent nodes\n    // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)\n    // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops\n    // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known\n    // expand gpu down\n    {\n        int cur_backend_id = -1;\n        for (int i = 0; i < graph->n_nodes; i++) {\n            struct ggml_tensor * node = graph->nodes[i];\n            if (ggml_is_view_op(node->op)) {\n                continue;\n            }\n            int * node_backend_id = &tensor_backend_id(node);\n            if (*node_backend_id != -1) {\n                if (*node_backend_id == sched->n_backends - 1) {\n                    // skip cpu (lowest prio backend)\n                    cur_backend_id = -1;\n                } else {\n                    cur_backend_id = *node_backend_id;\n                }\n            } else if (cur_backend_id != -1) {\n                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n            }\n        }\n    }\n    // expand gpu up\n    {\n        int cur_backend_id = -1;\n        for (int i = graph->n_nodes - 1; i >= 0; i--) {\n            struct ggml_tensor * node = graph->nodes[i];\n            if (ggml_is_view_op(node->op)) {\n                continue;\n            }\n            int * node_backend_id = &tensor_backend_id(node);\n            if (*node_backend_id != -1) {\n                if (*node_backend_id == sched->n_backends - 1) {\n                    // skip cpu (lowest prio backend)\n                    cur_backend_id = -1;\n                } else {\n                    cur_backend_id = *node_backend_id;\n                }\n            } else if (cur_backend_id != -1) {\n                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n            }\n        }\n    }\n    // expand rest down\n    {\n        int cur_backend_id = -1;\n        for (int i = 0; i < graph->n_nodes; i++) {\n            struct ggml_tensor * node = graph->nodes[i];\n            if (ggml_is_view_op(node->op)) {\n                continue;\n            }\n            int * node_backend_id = &tensor_backend_id(node);\n            if (*node_backend_id != -1) {\n                cur_backend_id = *node_backend_id;\n            } else if (cur_backend_id != -1) {\n                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n            }\n        }\n    }\n    // expand rest up\n    {\n        int cur_backend_id = -1;\n        for (int i = graph->n_nodes - 1; i >= 0; i--) {\n            struct ggml_tensor * node = graph->nodes[i];\n            if (ggml_is_view_op(node->op)) {\n                continue;\n            }\n            int * node_backend_id = &tensor_backend_id(node);\n            if (*node_backend_id != -1) {\n                cur_backend_id = *node_backend_id;\n            } else if (cur_backend_id != -1) {\n                ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);\n            }\n        }\n    }\n\n    // pass 3: upgrade nodes to higher prio backends with compatible buffer types\n    // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there\n    // however, we also need to verify that the sources are in compatible buffer types\n    // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph\n    // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same\n    // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)\n    // additionally, set remaining unassigned nodes to the backend with the most supported inputs\n    // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        if (ggml_is_view_op(node->op)) {\n            continue;\n        }\n        int * node_backend_id = &tensor_backend_id(node);\n        if (*node_backend_id == -1) {\n            // unassigned node: find the backend with the most supported inputs\n            int n_supported_best = -1;\n            for (int b = 0; b < sched->n_backends; b++) {\n                if (ggml_backend_supports_op(sched->backends[b], node)) {\n                    int n_supported = 0;\n                    for (int j = 0; j < GGML_MAX_SRC; j++) {\n                        struct ggml_tensor * src = node->src[j];\n                        if (src == NULL) {\n                            continue;\n                        }\n                        if ((tensor_backend_id(src) != -1 || tensor_backend_id(src->view_src) != -1) && ggml_backend_sched_buffer_supported(sched, src, b)) {\n                            n_supported++;\n                        }\n                    }\n                    if (n_supported > n_supported_best) {\n                        n_supported_best = n_supported;\n                        *node_backend_id = b;\n                        SET_CAUSE(node, \"3.best\");\n                    }\n                }\n            }\n        } else {\n            // assigned node: upgrade to higher prio backend if possible\n            for (int b = 0; b < *node_backend_id; b++) {\n                if (sched->bufts[b] == sched->bufts[*node_backend_id] && ggml_backend_supports_op(sched->backends[b], node)) {\n                    bool supported = true;\n                    for (int j = 0; j < GGML_MAX_SRC; j++) {\n                        struct ggml_tensor * src = node->src[j];\n                        if (src == NULL) {\n                            continue;\n                        }\n                        if (!ggml_backend_sched_buffer_supported(sched, src, b)) {\n                            supported = false;\n                            break;\n                        }\n                    }\n                    if (supported) {\n                        *node_backend_id = b;\n                        SET_CAUSE(node, \"3.upg\");\n                        break;\n                    }\n                }\n            }\n        }\n    }\n\n    // pass 4: assign backends to remaining src from dst and view_src\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        int * cur_backend_id = &tensor_backend_id(node);\n        if (node->view_src != NULL && *cur_backend_id == -1) {\n            *cur_backend_id = tensor_backend_id(node->view_src);\n            SET_CAUSE(node, \"4.vsrc\");\n        }\n        for (int j = 0; j < GGML_MAX_SRC; j++) {\n            struct ggml_tensor * src = node->src[j];\n            if (src == NULL) {\n                continue;\n            }\n            int * src_backend_id = &tensor_backend_id(src);\n            if (*src_backend_id == -1) {\n                if (src->view_src != NULL) {\n                    // views are always on the same backend as the source\n                    *src_backend_id = tensor_backend_id(src->view_src);\n                    SET_CAUSE(src, \"4.vsrc\");\n                } else {\n                    *src_backend_id = *cur_backend_id;\n                    SET_CAUSE(src, \"4.cur\");\n                }\n            }\n        }\n        // if the node is still unassigned, assign it to the first backend that supports it\n        for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) {\n            ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id);\n        }\n        GGML_ASSERT(*cur_backend_id != -1);\n    }\n\n    // pass 5: split graph, find tensors that need to be copied\n    {\n        int i_split = 0;\n        struct ggml_backend_sched_split * split = &sched->splits[0];\n        // find the backend of the first split, skipping view ops\n        int i = 0;\n        for (; i < graph->n_nodes; i++) {\n            struct ggml_tensor * node = graph->nodes[i];\n            if (!ggml_is_view_op(node->op)) {\n                split->backend_id = tensor_backend_id(node);\n                break;\n            }\n        }\n        split->i_start = 0;\n        split->n_inputs = 0;\n        int cur_backend_id = split->backend_id;\n        for (; i < graph->n_nodes; i++) {\n            struct ggml_tensor * node = graph->nodes[i];\n\n            if (ggml_is_view_op(node->op)) {\n                continue;\n            }\n\n            const int node_backend_id = tensor_backend_id(node);\n\n            GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now, this can happen if there is no CPU fallback\n\n            // check if we should start a new split based on the sources of the current node\n            bool need_new_split = false;\n            if (node_backend_id == cur_backend_id && split->n_inputs > 0) {\n                for (int j = 0; j < GGML_MAX_SRC; j++) {\n                    struct ggml_tensor * src = node->src[j];\n                    if (src == NULL) {\n                        continue;\n                    }\n                    // check if a weight is on a different and incompatible backend\n                    // by starting a new split, the memory of the previously offloaded weights can be reused\n                    if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {\n                        int src_backend_id = tensor_backend_id(src);\n                        if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {\n                            need_new_split = true;\n                            break;\n                        }\n                    }\n                    // check if the split has too many inputs\n                    // FIXME: count the number of inputs instead of only checking when full\n                    if (split->n_inputs == GGML_SCHED_MAX_SPLIT_INPUTS) {\n                        const size_t id = hash_id(src);\n                        int src_backend_id = sched->hv_tensor_backend_ids[id];\n                        bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);\n                        if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {\n                            need_new_split = true;\n                            break;\n                        }\n                    }\n                }\n            }\n\n            if (node_backend_id != cur_backend_id || need_new_split) {\n                split->i_end = i;\n                i_split++;\n                if (i_split >= sched->splits_capacity) {\n                    sched->splits_capacity *= 2;\n                    sched->splits = (ggml_backend_sched_split *)\n                        realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));\n                    GGML_ASSERT(sched->splits != NULL);\n                }\n                split = &sched->splits[i_split];\n                split->backend_id = node_backend_id;\n                split->i_start = i;\n                split->n_inputs = 0;\n                cur_backend_id = node_backend_id;\n            }\n\n            // find inputs that are not on the same backend\n            for (int j = 0; j < GGML_MAX_SRC; j++) {\n                struct ggml_tensor * src = node->src[j];\n                if (src == NULL) {\n                    continue;\n                }\n\n                size_t src_id = hash_id(src);\n                const int src_backend_id = sched->hv_tensor_backend_ids[src_id];\n                GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now\n\n                if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {\n                    if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {\n                        ggml_backend_t backend = sched->backends[src_backend_id];\n                        for (int c = 0; c < sched->n_copies; c++) {\n                            struct ggml_tensor * tensor_copy;\n                            if (c == sched->cur_copy) {\n                                tensor_copy = src; // use the original tensor as the current copy\n                            } else {\n                                tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);\n                                ggml_format_name(tensor_copy, \"%s#%s#%d\", ggml_backend_name(backend), src->name, c);\n                            }\n                            ggml_set_input(tensor_copy);\n                            ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor\n                            tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;\n                            SET_CAUSE(tensor_copy, \"4.cpy\");\n                        }\n                        int n_graph_inputs = sched->n_graph_inputs++;\n                        GGML_ASSERT(n_graph_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);\n                        sched->graph_inputs[n_graph_inputs] = src;\n                    }\n                }\n\n                if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {\n                    // create a copy of the input in the split's backend\n                    if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {\n                        ggml_backend_t backend = sched->backends[cur_backend_id];\n                        for (int c = 0; c < sched->n_copies; c++) {\n                            struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);\n                            ggml_format_name(tensor_copy, \"%s#%s#%d\", ggml_backend_name(backend), src->name, c);\n                            if (sched->n_copies > 1) {\n                                ggml_set_input(tensor_copy);\n                                ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor\n                            }\n                            tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;\n                            SET_CAUSE(tensor_copy, \"4.cpy\");\n                        }\n                        int n_inputs = split->n_inputs++;\n                        GGML_ASSERT(n_inputs < GGML_SCHED_MAX_SPLIT_INPUTS);\n                        split->inputs[n_inputs] = src;\n                    }\n                    node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);\n                }\n            }\n        }\n        split->i_end = graph->n_nodes;\n        sched->n_splits = i_split + 1;\n    }\n\n    if (sched->debug) {\n        ggml_backend_sched_print_assignments(sched, graph);\n    }\n\n    // swap node_backend_ids and leaf _backend_ids with prevs\n    {\n        int * tmp = sched->node_backend_ids;\n        sched->node_backend_ids = sched->prev_node_backend_ids;\n        sched->prev_node_backend_ids = tmp;\n\n        tmp = sched->leaf_backend_ids;\n        sched->leaf_backend_ids = sched->prev_leaf_backend_ids;\n        sched->prev_leaf_backend_ids = tmp;\n    }\n\n    int graph_size = std::max(graph->n_nodes, graph->n_leafs) + sched->n_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sched->n_copies;\n\n    // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]\n    sched->debug_prev_graph_size = sched->debug_graph_size;\n    sched->debug_graph_size = graph_size;\n\n    if (sched->graph.size < graph_size) {\n        sched->graph.size = graph_size;\n        sched->graph.nodes = (ggml_tensor **) realloc(sched->graph.nodes, graph_size * sizeof(struct ggml_tensor *));\n        sched->graph.leafs = (ggml_tensor **) realloc(sched->graph.leafs, graph_size * sizeof(struct ggml_tensor *));\n        GGML_ASSERT(sched->graph.nodes != NULL);\n        GGML_ASSERT(sched->graph.leafs != NULL);\n    }\n    sched->graph.n_nodes = 0;\n    sched->graph.n_leafs = 0;\n\n    struct ggml_cgraph * graph_copy = &sched->graph;\n\n    for (int i = 0; i < sched->n_splits; i++) {\n        struct ggml_backend_sched_split * split = &sched->splits[i];\n        split->graph = ggml_graph_view(graph, split->i_start, split->i_end);\n\n        // Optimize this split of the graph. This needs to happen before we make graph_copy,\n        // so they are in sync.\n        ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph);\n\n        // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split\n        for (int j = 0; j < split->n_inputs; j++) {\n            assert(graph_copy->size > (graph_copy->n_nodes + 1));\n\n            struct ggml_tensor * input = split->inputs[j];\n            const size_t input_id = hash_id(input);\n            struct ggml_tensor * input_cpy = tensor_id_copy(input_id, split->backend_id, sched->cur_copy);\n\n            // add a dependency to the input source so that it is not freed before the copy is done\n            struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);\n            input_dep->src[0] = input;\n            sched->node_backend_ids[graph_copy->n_nodes] = sched->hv_tensor_backend_ids[input_id];\n            graph_copy->nodes[graph_copy->n_nodes++] = input_dep;\n\n            // add a dependency to the input copy so that it is allocated at the start of the split\n            sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;\n            graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;\n        }\n\n        for (int j = split->i_start; j < split->i_end; j++) {\n            assert(graph_copy->size > graph_copy->n_nodes);\n            sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);\n            graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];\n        }\n    }\n\n    if (sched->n_copies > 1) {\n        // add input copies as leafs so that they are allocated first\n        for (int i = 0; i < sched->n_graph_inputs; i++) {\n            struct ggml_tensor * input = sched->graph_inputs[i];\n            size_t id = hash_id(input);\n            int backend_id = tensor_backend_id(input);\n            for (int c = 0; c < sched->n_copies; c++) {\n                struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);\n                sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;\n                assert(graph_copy->size > graph_copy->n_leafs);\n                graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;\n            }\n        }\n\n        for (int i = 0; i < sched->n_splits; i++) {\n            struct ggml_backend_sched_split * split = &sched->splits[i];\n            int backend_id = split->backend_id;\n            for (int j = 0; j < split->n_inputs; j++) {\n                struct ggml_tensor * input = split->inputs[j];\n                size_t id = hash_id(input);\n                for (int c = 0; c < sched->n_copies; c++) {\n                    struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);\n                    sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;\n                    assert(graph_copy->size > graph_copy->n_leafs);\n                    graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;\n                }\n            }\n        }\n    }\n\n    // add leafs from the original graph\n    for (int i = 0; i < graph->n_leafs; i++) {\n        struct ggml_tensor * leaf = graph->leafs[i];\n        sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf);\n        assert(graph_copy->size > graph_copy->n_leafs);\n        graph_copy->leafs[graph_copy->n_leafs++] = leaf;\n    }\n}\n\nstatic bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {\n    bool backend_ids_changed = false;\n    for (int i = 0; i < sched->graph.n_nodes; i++) {\n        if (sched->node_backend_ids[i] != sched->prev_node_backend_ids[i] &&\n            sched->bufts[sched->node_backend_ids[i]] != sched->bufts[sched->prev_node_backend_ids[i]]) {\n            backend_ids_changed = true;\n            break;\n        }\n    }\n    if (!backend_ids_changed) {\n        for (int i = 0; i < sched->graph.n_leafs; i++) {\n            if (sched->leaf_backend_ids[i] != sched->prev_leaf_backend_ids[i] &&\n                sched->bufts[sched->leaf_backend_ids[i]] != sched->bufts[sched->prev_leaf_backend_ids[i]]) {\n                backend_ids_changed = true;\n                break;\n            }\n        }\n    }\n\n    // allocate graph\n    if (backend_ids_changed || !ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {\n#ifndef NDEBUG\n        GGML_LOG_DEBUG(\"%s: failed to allocate graph, reserving (backend_ids_changed = %d)\\n\", __func__, backend_ids_changed);\n#endif\n\n        if (sched->debug_realloc > 0) {\n            // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC]\n            // example: https://github.com/ggml-org/llama.cpp/pull/17143\n            const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size;\n\n            if (unexpected || sched->debug_realloc > 1) {\n                GGML_ABORT(\"%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\\n\", __func__,\n                        sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);\n            }\n        }\n\n        // the re-allocation may cause the split inputs to be moved to a different address\n        // synchronize without ggml_backend_sched_synchronize to avoid changing cur_copy\n        for (int i = 0; i < sched->n_backends; i++) {\n            ggml_backend_synchronize(sched->backends[i]);\n        }\n\n        ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);\n        if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {\n            GGML_LOG_ERROR(\"%s: failed to allocate graph\\n\", __func__);\n            return false;\n        }\n    }\n\n    return true;\n}\n\nstatic enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    struct ggml_backend_sched_split * splits = sched->splits;\n\n    ggml_tensor * prev_ids_tensor = nullptr;\n    std::vector<int32_t> ids;\n    std::vector<ggml_bitset_t> used_ids;\n\n    for (int split_id = 0; split_id < sched->n_splits; split_id++) {\n        struct ggml_backend_sched_split * split = &splits[split_id];\n        int split_backend_id = split->backend_id;\n        ggml_backend_t split_backend = sched->backends[split_backend_id];\n\n        // copy the input tensors to the split backend\n        for (int input_id = 0; input_id < split->n_inputs; input_id++) {\n            ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);\n            struct ggml_tensor * input = split->inputs[input_id];\n            struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);\n\n            if (input->flags & GGML_TENSOR_FLAG_INPUT) {\n                // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done\n                if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n                    ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);\n                } else {\n                    ggml_backend_synchronize(split_backend);\n                }\n                ggml_backend_tensor_copy(input, input_cpy);\n            } else {\n                // wait for the split backend to finish using the input before overwriting it\n                if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n                    ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);\n                } else {\n                    ggml_backend_synchronize(split_backend);\n                }\n\n                // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used\n                ggml_tensor * node = split->graph.nodes[0];\n                if (split->graph.n_nodes > 0 &&\n                    ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&\n                    ggml_backend_buffer_is_host(input->buffer) && (\n                    (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID)\n                    //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */\n                    )) {\n\n                    const int64_t n_expert   = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1];\n                    const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1];\n\n                    ggml_backend_synchronize(input_backend);\n\n                    // get the ids\n                    ggml_tensor * ids_tensor = node->src[2];\n                    ggml_backend_t ids_backend = split_backend;\n\n                    // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend\n                    // in that case, we use the original ids tensor\n                    for (int i = input_id + 1; i < split->n_inputs; i++) {\n                        if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) {\n                            ids_tensor = split->inputs[i];\n                            ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]);\n                            break;\n                        }\n                    }\n\n                    if (ids_tensor != prev_ids_tensor) {\n                        ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t));\n                        ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor));\n                        ggml_backend_synchronize(ids_backend);\n\n                        // find the used experts\n                        used_ids.clear();\n                        used_ids.resize(ggml_bitset_size(n_expert));\n                        for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) {\n                            for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) {\n                                int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)];\n                                GGML_ASSERT(id >= 0 && id < n_expert);\n                                ggml_bitset_set(used_ids.data(), id);\n                            }\n                        }\n\n                        prev_ids_tensor = ids_tensor;\n                    }\n\n                    // group consecutive experts and copy them together\n                    auto copy_experts = [&](int32_t first_id, int32_t last_id) {\n                        const size_t expert_offset = first_id * expert_size;\n                        const size_t expert_size_copy =  (last_id - first_id + 1) * expert_size;\n                        const size_t padding = std::min<size_t>(expert_size, 512);\n                        const size_t padding_end = last_id < n_expert - 1 ? padding : 0;\n\n                        ggml_backend_tensor_set_async(split_backend,\n                            input_cpy,\n                            (const uint8_t *)input->data + expert_offset, expert_offset,\n                            // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert\n                            // this is necessary for MMQ in the CUDA backend\n                            expert_size_copy + padding_end);\n                    };\n\n                    int id = 0;\n                    while (!ggml_bitset_get(used_ids.data(), id)) {\n                        id++;\n                    }\n                    int32_t first_id = id;\n                    int32_t last_id = first_id;\n\n                    for (++id; id < n_expert; ++id) {\n                        if (!ggml_bitset_get(used_ids.data(), id)) {\n                            continue;\n                        }\n\n                        if (id == last_id + 1) {\n                            last_id = id;\n                            continue;\n                        }\n\n                        copy_experts(first_id, last_id);\n\n                        first_id = id;\n                        last_id = id;\n                    }\n                    copy_experts(first_id, last_id);\n                } else {\n                    // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events\n                    // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface\n                    if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {\n                        ggml_backend_synchronize(input_backend);\n                        if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n                            ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);\n                        } else {\n                            ggml_backend_synchronize(split_backend);\n                        }\n                        ggml_backend_tensor_copy(input, input_cpy);\n                    }\n                }\n            }\n        }\n\n        if (!sched->callback_eval) {\n            enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);\n            if (ec != GGML_STATUS_SUCCESS) {\n                return ec;\n            }\n        } else {\n            // similar to ggml_backend_compare_graph_backend\n            for (int j0 = 0; j0 < split->graph.n_nodes; j0++) {\n                struct ggml_tensor * t = split->graph.nodes[j0];\n\n                // check if the user needs data from this node\n                bool need = sched->callback_eval(t, true, sched->callback_eval_user_data);\n\n                int j1 = j0;\n\n                // determine the range [j0, j1] of nodes that can be computed together\n                while (!need && j1 < split->graph.n_nodes - 1) {\n                    t = split->graph.nodes[++j1];\n                    need = sched->callback_eval(t, true, sched->callback_eval_user_data);\n                }\n\n                struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);\n\n                enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);\n                if (ec != GGML_STATUS_SUCCESS) {\n                    return ec;\n                }\n\n                // TODO: pass backend to the callback, then the user can decide if they want to synchronize\n                ggml_backend_synchronize(split_backend);\n\n                if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {\n                    break;\n                }\n\n                j0 = j1;\n            }\n        }\n\n        // record the event of this copy\n        if (split->n_inputs > 0) {\n            if (sched->events[split_backend_id][sched->cur_copy] != NULL) {\n                ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);\n            }\n        }\n    }\n\n    return GGML_STATUS_SUCCESS;\n}\n\nggml_backend_sched_t ggml_backend_sched_new(\n        ggml_backend_t * backends,\n        ggml_backend_buffer_type_t * bufts,\n        int n_backends,\n        size_t graph_size,\n        bool parallel,\n        bool op_offload) {\n    GGML_ASSERT(n_backends > 0);\n    GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS);\n    GGML_ASSERT(ggml_backend_dev_type(ggml_backend_get_device(backends[n_backends - 1])) == GGML_BACKEND_DEVICE_TYPE_CPU);\n\n    struct ggml_backend_sched * sched = (ggml_backend_sched *) calloc(1, sizeof(struct ggml_backend_sched));\n\n    const char * GGML_SCHED_DEBUG = getenv(\"GGML_SCHED_DEBUG\");\n    sched->debug = GGML_SCHED_DEBUG ? atoi(GGML_SCHED_DEBUG) : 0;\n\n    sched->debug_realloc = 0;\n#ifdef GGML_SCHED_NO_REALLOC\n    sched->debug_realloc = 1;\n#endif\n    const char * GGML_SCHED_DEBUG_REALLOC = getenv(\"GGML_SCHED_DEBUG_REALLOC\");\n    sched->debug_realloc = GGML_SCHED_DEBUG_REALLOC ? atoi(GGML_SCHED_DEBUG_REALLOC) : sched->debug_realloc;\n\n    sched->n_backends = n_backends;\n    sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1;\n\n    // initialize hash table\n    // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead)\n    sched->hash_set    = ggml_hash_set_new(graph_size);\n    sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));\n    sched->hv_tensor_copies      = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));\n\n    const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph\n    const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;\n    sched->node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->node_backend_ids[0]));\n    sched->leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));\n    sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));\n    sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));\n\n    sched->debug_graph_size = 0;\n    sched->debug_prev_graph_size = 0;\n\n    sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);\n    sched->context_buffer = (char *) malloc(sched->context_buffer_size);\n\n    const int initial_splits_capacity = 16;\n    sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0]));\n    sched->splits_capacity = initial_splits_capacity;\n\n    for (int b = 0; b < n_backends; b++) {\n        sched->backends[b] = backends[b];\n        sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);\n        GGML_ASSERT(ggml_backend_supports_buft(backends[b], sched->bufts[b]));\n\n        if (sched->n_copies > 1) {\n            for (int c = 0; c < sched->n_copies; c++) {\n                sched->events[b][c] = ggml_backend_event_new(backends[b]->device);\n            }\n        }\n    }\n\n    sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);\n    sched->op_offload = op_offload;\n\n    ggml_backend_sched_reset(sched);\n\n    return sched;\n}\n\nvoid ggml_backend_sched_free(ggml_backend_sched_t sched) {\n    if (sched == NULL) {\n        return;\n    }\n    for (int b = 0; b < sched->n_backends; b++) {\n        for (int c = 0; c < sched->n_copies; c++) {\n            ggml_backend_event_free(sched->events[b][c]);\n        }\n    }\n    ggml_gallocr_free(sched->galloc);\n    ggml_free(sched->ctx);\n    ggml_hash_set_free(&sched->hash_set);\n    free(sched->splits);\n    free(sched->hv_tensor_backend_ids);\n    free(sched->hv_tensor_copies);\n    free(sched->node_backend_ids);\n    free(sched->leaf_backend_ids);\n    free(sched->prev_node_backend_ids);\n    free(sched->prev_leaf_backend_ids);\n    free(sched->context_buffer);\n    free(sched->graph.nodes);\n    free(sched->graph.leafs);\n    free(sched);\n}\n\nvoid ggml_backend_sched_reset(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    // reset state for the next run\n    if (!sched->is_reset) {\n        ggml_hash_set_reset(&sched->hash_set);\n        memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));\n        memset(sched->hv_tensor_copies,       0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));\n        sched->is_reset = true;\n    }\n    sched->is_alloc = false;\n}\n\nvoid ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes) {\n    GGML_ASSERT(sched);\n    GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);\n    GGML_ASSERT(sizes);\n\n    ggml_backend_sched_reset(sched);\n\n    ggml_backend_sched_synchronize(sched);\n\n    ggml_backend_sched_split_graph(sched, measure_graph);\n\n    ggml_gallocr_reserve_n_size(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids, sizes);\n}\n\nbool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {\n    GGML_ASSERT(sched);\n    GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);\n\n    ggml_backend_sched_synchronize(sched);\n\n    ggml_backend_sched_split_graph(sched, measure_graph);\n\n    if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {\n        return false;\n    }\n\n    ggml_backend_sched_reset(sched);\n\n    return true;\n}\n\nbool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n    GGML_ASSERT(sched);\n    GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + graph->n_leafs);\n    GGML_ASSERT(!sched->is_alloc);\n\n    sched->cur_copy = sched->next_copy;\n    sched->next_copy = (sched->next_copy + 1) % sched->n_copies;\n\n    ggml_backend_sched_split_graph(sched, graph);\n\n    if (!ggml_backend_sched_alloc_splits(sched)) {\n        return false;\n    }\n\n    sched->is_alloc = true;\n\n    return true;\n}\n\nenum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n    enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);\n    ggml_backend_sched_synchronize(sched);\n    return err;\n}\n\nenum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {\n    GGML_ASSERT(sched);\n    if (!sched->is_reset && !sched->is_alloc) {\n        ggml_backend_sched_reset(sched);\n    }\n\n    if (!sched->is_alloc) {\n        if (!ggml_backend_sched_alloc_graph(sched, graph)) {\n            return GGML_STATUS_ALLOC_FAILED;\n        }\n    }\n\n    return ggml_backend_sched_compute_splits(sched);\n}\n\nvoid ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    for (int i = 0; i < sched->n_backends; i++) {\n        ggml_backend_synchronize(sched->backends[i]);\n    }\n    if (!sched->is_alloc) {\n        // if the graph is not already allocated, always use copy 0 after a synchronization\n        // this ensures that during generation the same copy is used every time,\n        // which avoids changes in the graph that could cause CUDA or other graphs to be disabled\n        sched->next_copy = 0;\n    }\n}\n\nvoid ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {\n    GGML_ASSERT(sched);\n    sched->callback_eval = callback;\n    sched->callback_eval_user_data = user_data;\n}\n\nint ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    return sched->n_splits;\n}\n\nint ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    return sched->n_copies;\n}\n\nint ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched) {\n    GGML_ASSERT(sched);\n    return sched->n_backends;\n}\n\nggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i) {\n    GGML_ASSERT(sched);\n    GGML_ASSERT(i >= 0 && i < sched->n_backends);\n    return sched->backends[i];\n}\n\nggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend) {\n    GGML_ASSERT(sched);\n    int backend_index = ggml_backend_sched_backend_id(sched, backend);\n    GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n\n    return sched->bufts[backend_index];\n}\n\nsize_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {\n    GGML_ASSERT(sched);\n    int backend_index = ggml_backend_sched_backend_id(sched, backend);\n    GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n\n    return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);\n}\n\nvoid ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {\n    GGML_ASSERT(sched);\n    int backend_index = ggml_backend_sched_backend_id(sched, backend);\n    GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);\n    tensor_backend_id(node) = backend_index;\n    SET_CAUSE(node, \"usr\");\n    sched->is_reset = false;\n}\n\nggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {\n    GGML_ASSERT(sched);\n    int backend_index = tensor_backend_id(node);\n    if (backend_index == -1) {\n        return NULL;\n    }\n    return sched->backends[backend_index];\n}\n\n// utils\n\nenum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) {\n    GGML_ASSERT(tensor);\n    GGML_ASSERT(tensor->buffer == NULL);\n    GGML_ASSERT(tensor->view_src != NULL);\n    GGML_ASSERT(tensor->view_src->buffer != NULL);\n    GGML_ASSERT(tensor->view_src->data != NULL);\n\n    tensor->buffer = tensor->view_src->buffer;\n    tensor->data = (char *)tensor->view_src->data + tensor->view_offs;\n    return ggml_backend_buffer_init_tensor(tensor->buffer, tensor);\n}\n\nenum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {\n    GGML_ASSERT(tensor);\n    GGML_ASSERT(tensor->buffer == NULL);\n    GGML_ASSERT(tensor->data == NULL);\n    GGML_ASSERT(tensor->view_src == NULL);\n    GGML_ASSERT(addr >= ggml_backend_buffer_get_base(buffer));\n    GGML_ASSERT((char *)addr + ggml_backend_buffer_get_alloc_size(buffer, tensor) <=\n                (char *)ggml_backend_buffer_get_base(buffer) + ggml_backend_buffer_get_size(buffer));\n\n    tensor->buffer = buffer;\n    tensor->data = addr;\n    return ggml_backend_buffer_init_tensor(buffer, tensor);\n}\n\nstatic struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,\n    struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {\n\n    GGML_ASSERT(src != NULL);\n    GGML_ASSERT(src->data && \"graph must be allocated\");\n\n    size_t id = ggml_hash_insert(&hash_set, src);\n    if (id == GGML_HASHSET_ALREADY_EXISTS) {\n        return node_copies[ggml_hash_find(&hash_set, src)];\n    }\n\n    struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);\n    if (src->view_src != NULL) {\n        dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);\n        dst->view_offs = src->view_offs;\n    }\n    dst->op = src->op;\n    dst->flags = src->flags;\n    memcpy(dst->op_params, src->op_params, sizeof(dst->op_params));\n    ggml_set_name(dst, src->name);\n\n    // copy src\n    for (int i = 0; i < GGML_MAX_SRC; i++) {\n        struct ggml_tensor * s = src->src[i];\n        if (s == NULL) {\n            continue;\n        }\n        dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);\n    }\n\n    node_copies[id] = dst;\n    return dst;\n}\n\nstatic void graph_copy_init_tensor(struct ggml_hash_set * hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {\n    size_t id = ggml_hash_find(hash_set, src);\n    if (node_init[id]) {\n        return;\n    }\n    node_init[id] = true;\n\n    struct ggml_tensor * dst = node_copies[id];\n    if (dst->view_src != NULL) {\n        graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);\n        enum ggml_status status = ggml_backend_view_init(dst);\n        GGML_ASSERT(status == GGML_STATUS_SUCCESS);\n    }\n    else {\n        ggml_backend_tensor_copy(src, dst);\n    }\n\n    // init src\n    for (int i = 0; i < GGML_MAX_SRC; i++) {\n        struct ggml_tensor * s = src->src[i];\n        if (s == NULL) {\n            continue;\n        }\n        graph_copy_init_tensor(hash_set, node_copies, node_init, s);\n    }\n}\n\nstruct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {\n    GGML_ASSERT(graph);\n    struct ggml_hash_set hash_set = ggml_hash_set_new(graph->visited_hash_set.size);\n    struct ggml_tensor ** node_copies = (ggml_tensor **) calloc(hash_set.size, sizeof(node_copies[0])); // NOLINT\n    bool * node_init = (bool *) calloc(hash_set.size, sizeof(node_init[0]));\n\n    struct ggml_init_params params = {\n        /* .mem_size   = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),\n        /* .mem_buffer = */ NULL,\n        /* .no_alloc   = */ true\n    };\n\n    struct ggml_context * ctx_allocated = ggml_init(params);\n    struct ggml_context * ctx_unallocated = ggml_init(params);\n\n    if (ctx_allocated == NULL || ctx_unallocated == NULL) {\n        GGML_LOG_ERROR(\"%s: failed to allocate context for graph copy\\n\", __func__);\n        ggml_hash_set_free(&hash_set);\n        free(node_copies);\n        free(node_init);\n        ggml_free(ctx_allocated);\n        ggml_free(ctx_unallocated);\n        return {\n            /* .buffer           = */ NULL,\n            /* .ctx_allocated    = */ NULL,\n            /* .ctx_unallocated  = */ NULL,\n            /* .graph            = */ NULL,\n        };\n    }\n\n    // dup nodes\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);\n    }\n\n    // allocate nodes\n    ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);\n    if (buffer == NULL) {\n        GGML_LOG_ERROR(\"%s: failed to allocate buffer for graph copy\\n\", __func__);\n        ggml_hash_set_free(&hash_set);\n        free(node_copies);\n        free(node_init);\n        ggml_free(ctx_allocated);\n        ggml_free(ctx_unallocated);\n        return {\n            /* .buffer           = */ NULL,\n            /* .ctx_allocated    = */ NULL,\n            /* .ctx_unallocated  = */ NULL,\n            /* .graph            = */ NULL,\n        };\n    }\n\n    //printf(\"copy buffer size: %zu MB\\n\", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);\n\n    // copy data and init views\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        graph_copy_init_tensor(&hash_set, node_copies, node_init, node);\n    }\n\n    // build graph copy\n    struct ggml_cgraph * graph_copy = ggml_new_graph_custom(ctx_allocated, graph->size, false);\n    for (int i = 0; i < graph->n_nodes; i++) {\n        struct ggml_tensor * node = graph->nodes[i];\n        struct ggml_tensor * node_copy = node_copies[ggml_hash_find(&hash_set, node)];\n        graph_copy->nodes[i] = node_copy;\n    }\n    graph_copy->n_nodes = graph->n_nodes;\n\n    ggml_hash_set_free(&hash_set);\n    free(node_copies);\n    free(node_init);\n\n    return {\n        /* .buffer           = */ buffer,\n        /* .ctx_allocated    = */ ctx_allocated,\n        /* .ctx_unallocated  = */ ctx_unallocated,\n        /* .graph            = */ graph_copy,\n    };\n}\n\nvoid ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {\n    ggml_backend_buffer_free(copy.buffer);\n    ggml_free(copy.ctx_allocated);\n    ggml_free(copy.ctx_unallocated);\n}\n\nbool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes) {\n    struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);\n    if (copy.buffer == NULL) {\n        return false;\n    }\n\n    struct ggml_cgraph * g1 = graph;\n    struct ggml_cgraph * g2 = copy.graph;\n\n    assert(g1->n_nodes == g2->n_nodes);\n\n    if (num_test_nodes != 0) {\n        GGML_ASSERT(test_nodes);\n        // Compute the whole graph and only test the output for specific tensors\n        ggml_backend_graph_compute(backend1, g1);\n        ggml_backend_graph_compute(backend2, g2);\n\n        bool verified = false;\n        for (int i = 0; i < g1->n_nodes; i++) {\n            for (size_t j = 0; j < num_test_nodes; ++j) {\n                if (g1->nodes[i] == test_nodes[j]) {\n                    callback(i, g1->nodes[i], g2->nodes[i], user_data);\n                    verified = true;\n                }\n            }\n        }\n        GGML_ASSERT(verified);\n    } else {\n        for (int i = 0; i < g1->n_nodes; i++) {\n            struct ggml_tensor * t1 = g1->nodes[i];\n            struct ggml_tensor * t2 = g2->nodes[i];\n\n            assert(t1->op == t2->op && ggml_are_same_layout(t1, t2));\n\n            struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);\n            struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);\n\n            ggml_backend_graph_compute(backend1, &g1v);\n            ggml_backend_graph_compute(backend2, &g2v);\n\n            if (ggml_is_view_op(t1->op)) {\n                continue;\n            }\n\n            // compare results, calculate rms etc\n            if (!callback(i, t1, t2, user_data)) {\n                break;\n            }\n        }\n    }\n    ggml_backend_graph_copy_free(copy);\n\n    return true;\n}\n\n// CPU backend - buffer\n\nstatic void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    uintptr_t data = (uintptr_t)buffer->context;\n\n    // align the buffer\n    if (data % TENSOR_ALIGNMENT != 0) {\n        data = GGML_PAD(data, TENSOR_ALIGNMENT);\n    }\n\n    return (void *)data;\n}\n\nstatic void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n    GGML_ASSERT(buffer);\n    ggml_aligned_free(buffer->context, buffer->size);\n}\n\nstatic void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    memset((char *)tensor->data + offset, value, size);\n\n    GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    memcpy((char *)tensor->data + offset, data, size);\n\n    GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n    GGML_ASSERT(tensor);\n    memcpy(data, (const char *)tensor->data + offset, size);\n\n    GGML_UNUSED(buffer);\n}\n\nstatic bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {\n    GGML_ASSERT(src);\n    if (ggml_backend_buffer_is_host(src->buffer)) {\n        memcpy(dst->data, src->data, ggml_nbytes(src));\n        return true;\n    }\n    return false;\n\n    GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n    GGML_ASSERT(buffer);\n    memset(buffer->context, value, buffer->size);\n}\n\nstatic const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {\n    /* .free_buffer     = */ ggml_backend_cpu_buffer_free_buffer,\n    /* .get_base        = */ ggml_backend_cpu_buffer_get_base,\n    /* .init_tensor     = */ NULL, // no initialization required\n    /* .memset_tensor   = */ ggml_backend_cpu_buffer_memset_tensor,\n    /* .set_tensor      = */ ggml_backend_cpu_buffer_set_tensor,\n    /* .get_tensor      = */ ggml_backend_cpu_buffer_get_tensor,\n    /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,\n    /* .clear           = */ ggml_backend_cpu_buffer_clear,\n    /* .reset           = */ NULL,\n};\n\nstatic const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {\n    /* .free_buffer     = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed\n    /* .get_base        = */ ggml_backend_cpu_buffer_get_base,\n    /* .init_tensor     = */ NULL, // no initialization required\n    /* .memset_tensor   = */ ggml_backend_cpu_buffer_memset_tensor,\n    /* .set_tensor      = */ ggml_backend_cpu_buffer_set_tensor,\n    /* .get_tensor      = */ ggml_backend_cpu_buffer_get_tensor,\n    /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,\n    /* .clear           = */ ggml_backend_cpu_buffer_clear,\n    /* .reset           = */ NULL,\n};\n\n// CPU backend buffer type\n\n// this buffer type is defined here to make it available to all backends\n\nstatic const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"CPU\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    void * data = ggml_aligned_malloc(size);\n\n    if (data == NULL) {\n        GGML_LOG_ERROR(\"%s: failed to allocate buffer of size %zu\\n\", __func__, size);\n        return NULL;\n    }\n\n    return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);\n}\n\nstatic size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    return TENSOR_ALIGNMENT;\n\n    GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n    return true;\n\n    GGML_UNUSED(buft);\n}\n\nggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {\n        /* .iface   = */ {\n            /* .get_name         = */ ggml_backend_cpu_buffer_type_get_name,\n            /* .alloc_buffer     = */ ggml_backend_cpu_buffer_type_alloc_buffer,\n            /* .get_alignment    = */ ggml_backend_cpu_buffer_type_get_alignment,\n            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX\n            /* .get_alloc_size   = */ NULL, // defaults to ggml_nbytes\n            /* .is_host          = */ ggml_backend_cpu_buffer_type_is_host,\n        },\n        /* .device  = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ NULL,\n    };\n\n    return &ggml_backend_cpu_buffer_type;\n}\n\nstatic const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"CPU_Mapped\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cpu_buffer_from_ptr_type(void) {\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {\n        /* .iface   = */ {\n            /* .get_name         = */ ggml_backend_cpu_buffer_from_ptr_type_get_name,\n            /* .alloc_buffer     = */ ggml_backend_cpu_buffer_type_alloc_buffer,\n            /* .get_alignment    = */ ggml_backend_cpu_buffer_type_get_alignment,\n            /* .get_max_size     = */ NULL, // defaults to SIZE_MAX\n            /* .get_alloc_size   = */ NULL, // defaults to ggml_nbytes\n            /* .is_host          = */ ggml_backend_cpu_buffer_type_is_host,\n        },\n        /* .device  = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ NULL,\n    };\n\n    return &ggml_backend_cpu_buffer_type;\n}\n\nggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {\n    GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && \"buffer pointer must be aligned\");\n    return ggml_backend_buffer_init(ggml_backend_cpu_buffer_from_ptr_type(), ggml_backend_cpu_buffer_from_ptr_i, ptr, size);\n}\n"
  },
  {
    "path": "ggml/src/ggml-blas/CMakeLists.txt",
    "content": "if (GGML_STATIC)\n    set(BLA_STATIC ON)\nendif()\n#if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)\n#    set(BLA_SIZEOF_INTEGER 8)\n#endif()\n\nset(BLA_VENDOR ${GGML_BLAS_VENDOR})\nfind_package(BLAS)\n\nif (BLAS_FOUND)\n    message(STATUS \"BLAS found, Libraries: ${BLAS_LIBRARIES}\")\n\n    ggml_add_backend_library(ggml-blas\n                             ggml-blas.cpp\n                            )\n\n    if (${GGML_BLAS_VENDOR} MATCHES \"Apple\")\n        add_compile_definitions(ACCELERATE_NEW_LAPACK)\n        add_compile_definitions(ACCELERATE_LAPACK_ILP64)\n        add_compile_definitions(GGML_BLAS_USE_ACCELERATE)\n    elseif (\"${BLAS_INCLUDE_DIRS}\" STREQUAL \"\")\n        # BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.\n        # see https://gitlab.kitware.com/cmake/cmake/-/issues/20268\n        find_package(PkgConfig REQUIRED)\n        if (${GGML_BLAS_VENDOR} MATCHES \"Generic\")\n            pkg_check_modules(DepBLAS blas)\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"OpenBLAS\")\n            # As of openblas v0.3.22, the 64-bit is named openblas64.pc\n            pkg_check_modules(DepBLAS openblas64)\n            if (NOT DepBLAS_FOUND)\n                pkg_check_modules(DepBLAS openblas)\n            endif()\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"FLAME\")\n            pkg_check_modules(DepBLAS blis)\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"ATLAS\")\n            pkg_check_modules(DepBLAS blas-atlas)\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"FlexiBLAS\")\n            pkg_check_modules(DepBLAS flexiblas_api)\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"Intel\")\n            # all Intel* libraries share the same include path\n            pkg_check_modules(DepBLAS mkl-sdl)\n        elseif (${GGML_BLAS_VENDOR} MATCHES \"NVHPC\")\n            # this doesn't provide pkg-config\n            # suggest to assign BLAS_INCLUDE_DIRS on your own\n            if (\"${NVHPC_VERSION}\" STREQUAL \"\")\n                message(WARNING \"Better to set NVHPC_VERSION\")\n            else()\n                set(DepBLAS_FOUND ON)\n                set(DepBLAS_INCLUDE_DIRS \"/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include\")\n            endif()\n        endif()\n        if (DepBLAS_FOUND)\n            set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})\n        else()\n            message(WARNING \"BLAS_INCLUDE_DIRS neither been provided nor been automatically\"\n            \" detected by pkgconfig, trying to find cblas.h from possible paths...\")\n            find_path(BLAS_INCLUDE_DIRS\n                NAMES cblas.h\n                HINTS\n                    /usr/include\n                    /usr/local/include\n                    /usr/include/openblas\n                    /opt/homebrew/opt/openblas/include\n                    /usr/local/opt/openblas/include\n                    /usr/include/x86_64-linux-gnu/openblas/include\n            )\n        endif()\n    endif()\n\n    message(STATUS \"BLAS found, Includes: ${BLAS_INCLUDE_DIRS}\")\n\n    target_compile_options(ggml-blas PRIVATE ${BLAS_LINKER_FLAGS})\n\n    if (\"${GGML_BLAS_VENDOR}\" STREQUAL \"\")\n        message(WARNING \"GGML_BLAS_VENDOR is not set; some methods may not link properly.\")\n    endif()\n\n    if (\"${GGML_BLAS_VENDOR}\" MATCHES \"Intel\" OR (\"${BLAS_INCLUDE_DIRS}\" MATCHES \"mkl\" AND \"${GGML_BLAS_VENDOR}\" MATCHES \"Generic\"))\n        add_compile_definitions(GGML_BLAS_USE_MKL)\n    endif()\n\n    if (\"${GGML_BLAS_VENDOR}\" MATCHES \"OpenBLAS\")\n        add_compile_definitions(GGML_BLAS_USE_OPENBLAS)\n    endif()\n\n    if (\"${GGML_BLAS_VENDOR}\" MATCHES \"FLAME\" OR \"${GGML_BLAS_VENDOR}\" MATCHES \"AOCL\" OR \"${GGML_BLAS_VENDOR}\" MATCHES \"AOCL_mt\")\n        add_compile_definitions(GGML_BLAS_USE_BLIS)\n    endif()\n\n    if (\"${GGML_BLAS_VENDOR}\" MATCHES \"NVPL\")\n        add_compile_definitions(GGML_BLAS_USE_NVPL)\n    endif()\n\n    target_link_libraries     (ggml-blas PRIVATE ${BLAS_LIBRARIES})\n    target_include_directories(ggml-blas SYSTEM PRIVATE ${BLAS_INCLUDE_DIRS})\nelse()\n    message(FATAL_ERROR \"BLAS not found, please refer to \"\n                        \"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors\"\n                        \" to set correct GGML_BLAS_VENDOR\")\nendif()\n"
  },
  {
    "path": "ggml/src/ggml-blas/ggml-blas.cpp",
    "content": "#include \"ggml-impl.h\"\n#include \"ggml-blas.h\"\n#include \"ggml-backend-impl.h\"\n\n#include <future>\n#include <vector>\n#include <cstring>\n\n#if defined(GGML_BLAS_USE_ACCELERATE)\n#   include <Accelerate/Accelerate.h>\n#elif defined(GGML_BLAS_USE_MKL)\n#   include <mkl.h>\n#elif defined(GGML_BLAS_USE_BLIS)\n#   include <blis.h>\n#elif defined(GGML_BLAS_USE_NVPL)\n#   include <nvpl_blas.h>\n#else\n#   include <cblas.h>\n#endif\n\nstruct ggml_backend_blas_context {\n    int n_threads = GGML_DEFAULT_N_THREADS;\n    std::unique_ptr<char[]> work_data;\n    size_t work_size = 0;\n#ifndef GGML_USE_OPENMP\n    std::vector<std::future<void>> tasks;\n#endif\n};\n\nstatic void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {\n    const struct ggml_tensor * src0 = dst->src[0];\n    const struct ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const enum ggml_type type = src0->type;\n\n    GGML_ASSERT(ne0 == ne01);\n    GGML_ASSERT(ne1 == ne11);\n    GGML_ASSERT(ne2 == ne12);\n    GGML_ASSERT(ne3 == ne13);\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == ggml_type_size(type));\n    GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    // broadcast factors\n    const int64_t r2 = ne12/ne02;\n    const int64_t r3 = ne13/ne03;\n\n    const int64_t ne_plane      = ne01*ne00;\n    const size_t  desired_wsize = type == GGML_TYPE_F32 ? 0 : ne03*ne02*ne_plane*sizeof(float);\n\n    if (ctx->work_size < desired_wsize) {\n        ctx->work_data.reset(new char[desired_wsize]);\n        ctx->work_size = desired_wsize;\n    }\n    void * wdata = ctx->work_data.get();\n\n    // convert src0 to float\n    if (type != GGML_TYPE_F32) {\n        const auto * type_traits = ggml_get_type_traits(type);\n        ggml_to_float_t const to_float = type_traits->to_float;\n\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                const void  *       x      = (char *)  src0->data + i02*nb02          + i03*nb03;\n                      float * const wplane = (float *) wdata      + i02*ne_plane      + i03*ne02*ne_plane;\n\n                const int min_cols_per_thread = 4096;\n                const int min_rows_per_thread = std::max((int)(min_cols_per_thread/ne00), 1);\n                const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01/min_rows_per_thread)), 1);\n\n#ifdef GGML_USE_OPENMP\n                #pragma omp parallel for num_threads(n_threads)\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n                }\n#else\n                for (int i = 1; i < n_threads; i++) {\n                    const int64_t start =       i*ne01/n_threads;\n                    const int64_t end   = (i + 1)*ne01/n_threads;\n                    if (start < end) {\n                        ctx->tasks.push_back(std::async(std::launch::async, [=]() {\n                            for (int64_t i01 = start; i01 < end; i01++) {\n                                to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n                            }\n                        }));\n                    }\n                }\n                {\n                    // reuse the current thread for the first task\n                    const int64_t start = 0;\n                    const int64_t end   = ne01/n_threads;\n                    for (int64_t i01 = start; i01 < end; i01++) {\n                        to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00);\n                    }\n                }\n#endif\n            }\n        }\n\n#ifndef GGML_USE_OPENMP\n        // wait for all tasks to finish\n        for (auto & task : ctx->tasks) {\n            task.get();\n        }\n        ctx->tasks.clear();\n#endif\n    }\n\n#if defined(GGML_BLAS_USE_OPENBLAS)\n    openblas_set_num_threads(ctx->n_threads);\n#elif defined(GGML_BLAS_USE_BLIS)\n    bli_thread_set_num_threads(ctx->n_threads);\n#elif defined(GGML_BLAS_USE_NVPL)\n    nvpl_blas_set_num_threads(ctx->n_threads);\n#endif\n\n    for (int64_t i13 = 0; i13 < ne13; i13++) {\n        for (int64_t i12 = 0; i12 < ne12; i12++) {\n            const int64_t i03 = i13/r3;\n            const int64_t i02 = i12/r2;\n\n            const float * x = (float *) ((char *) src0->data + i02*nb02 + i03*nb03);\n            const float * y = (float *) ((char *) src1->data + i12*nb12 + i13*nb13);\n                  float * d = (float *) ((char *)  dst->data + i12*nb2  + i13*nb3);\n\n            if (type != GGML_TYPE_F32) {\n                x = (float *) wdata + i02*ne_plane + i03*ne02*ne_plane;\n            }\n\n            cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,\n                        ne1, ne01, ne10,\n                        1.0f,   y, ne10,\n                                x, ne00,\n                        0.0f,   d, ne01);\n        }\n    }\n}\n\nstatic void ggml_backend_blas_out_prod(ggml_backend_blas_context * ctx, struct ggml_tensor * dst) {\n    const struct ggml_tensor * src0 = dst->src[0];\n    const struct ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    GGML_ASSERT(ne0  == ne00);\n    GGML_ASSERT(ne1  == ne10);\n    GGML_ASSERT(ne2  == ne02);\n    GGML_ASSERT(ne02 == ne12);\n    GGML_ASSERT(ne3  == ne13);\n    GGML_ASSERT(ne03 == ne13);\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    // GGML_ASSERT(nb0 <= nb1);\n    // GGML_ASSERT(nb1 <= nb2);\n    // GGML_ASSERT(nb2 <= nb3);\n\n    // Arguments to ggml_compute_forward_out_prod (expressed as major,minor)\n    // src0: (k,n)\n    // src1: (k,m)\n    // dst:  (m,n)\n    //\n    // Arguments to sgemm (see https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/sgemm.f)\n    // Also expressed as (major,minor)\n    // a: (m,k): so src1 transposed\n    // b: (k,n): so src0\n    // c: (m,n)\n    //\n    // However, if ggml_is_transposed(src1) is true, then\n    // src1->data already contains a transposed version, so sgemm mustn't\n    // transpose it further.\n\n    int n = src0->ne[0];\n    int k = src0->ne[1];\n    int m = src1->ne[0];\n\n    CBLAS_TRANSPOSE transposeA;\n    int lda;\n\n    if (!ggml_is_transposed(src1)) {\n        transposeA = CblasTrans;\n        lda = m;\n    } else {\n        transposeA = CblasNoTrans;\n        lda = k;\n    }\n\n    float * a = (float *) ((char *) src1->data);\n    float * b = (float *) ((char *) src0->data);\n    float * c = (float *) ((char *) dst->data);\n\n    cblas_sgemm(CblasRowMajor, transposeA, CblasNoTrans, m, n, k, 1.0, a, lda, b, n, 0.0, c, n);\n\n    GGML_UNUSED(ctx);\n}\n\n// backend interface\n\nstatic const char * ggml_backend_blas_get_name(ggml_backend_t backend) {\n    return \"BLAS\";\n\n    GGML_UNUSED(backend);\n}\n\nstatic void ggml_backend_blas_free(ggml_backend_t backend) {\n    ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;\n    delete ctx;\n    delete backend;\n}\n\nstatic enum ggml_status ggml_backend_blas_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend->context;\n\n    for (int i = 0; i < cgraph->n_nodes; i++) {\n        struct ggml_tensor * node = cgraph->nodes[i];\n\n        if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n            continue;\n        }\n\n        switch (node->op) {\n            case GGML_OP_MUL_MAT:\n                ggml_backend_blas_mul_mat(ctx, node);\n                break;\n\n            case GGML_OP_OUT_PROD:\n                ggml_backend_blas_out_prod(ctx, node);\n                break;\n\n            case GGML_OP_NONE:\n            case GGML_OP_RESHAPE:\n            case GGML_OP_VIEW:\n            case GGML_OP_PERMUTE:\n            case GGML_OP_TRANSPOSE:\n                break;\n\n            default:\n                GGML_ABORT(\"%s: unsupported op %s\\n\", __func__, ggml_op_desc(node));\n        }\n    }\n\n    return GGML_STATUS_SUCCESS;\n\n    GGML_UNUSED(backend);\n}\n\nstatic struct ggml_backend_i blas_backend_i = {\n    /* .get_name                = */ ggml_backend_blas_get_name,\n    /* .free                    = */ ggml_backend_blas_free,\n    /* .set_tensor_async        = */ NULL,\n    /* .get_tensor_async        = */ NULL,\n    /* .cpy_tensor_async        = */ NULL,\n    /* .synchronize             = */ NULL,\n    /* .graph_plan_create       = */ NULL,\n    /* .graph_plan_free         = */ NULL,\n    /* .graph_plan_update       = */ NULL,\n    /* .graph_plan_compute      = */ NULL,\n    /* .graph_compute           = */ ggml_backend_blas_graph_compute,\n    /* .event_record            = */ NULL,\n    /* .event_wait              = */ NULL,\n    /* .graph_optimize          = */ NULL,\n};\n\nstatic ggml_guid_t ggml_backend_blas_guid(void) {\n    static ggml_guid guid = { 0x12, 0xa8, 0xae, 0xf4, 0xc0, 0x1e, 0x61, 0x97, 0x8f, 0xeb, 0x33, 0x04, 0xa1, 0x33, 0x51, 0x2d };\n    return &guid;\n}\n\nggml_backend_t ggml_backend_blas_init(void) {\n    ggml_backend_blas_context * ctx = new ggml_backend_blas_context;\n\n    ggml_backend_t backend = new ggml_backend {\n        /* .guid    = */ ggml_backend_blas_guid(),\n        /* .iface   = */ blas_backend_i,\n        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_blas_reg(), 0),\n        /* .context = */ ctx,\n    };\n\n#if defined(GGML_BLAS_USE_OPENBLAS) && defined(GGML_USE_OPENMP)\n    if (openblas_get_parallel() != OPENBLAS_OPENMP) {\n        GGML_LOG_DEBUG(\"%s: warning: ggml is using OpenMP, but OpenBLAS was compiled without OpenMP support\\n\", __func__);\n    }\n#endif\n\n#if defined(BLIS_ENABLE_CBLAS) && defined(GGML_USE_OPENMP) && !defined(BLIS_ENABLE_OPENMP)\n    GGML_LOG_DEBUG(\"%s: warning: ggml is using OpenMP, but BLIS was compiled without OpenMP support\\n\", __func__);\n#endif\n\n    return backend;\n}\n\nbool ggml_backend_is_blas(ggml_backend_t backend) {\n    return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_blas_guid());\n}\n\nvoid ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads) {\n    GGML_ASSERT(ggml_backend_is_blas(backend_blas));\n\n    ggml_backend_blas_context * ctx = (ggml_backend_blas_context *)backend_blas->context;\n    ctx->n_threads = n_threads;\n}\n\n// device interface\n\nstatic const char * ggml_backend_blas_device_get_name(ggml_backend_dev_t dev) {\n    return \"BLAS\";\n\n    GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_blas_device_get_description(ggml_backend_dev_t dev) {\n    #if defined(GGML_BLAS_USE_ACCELERATE)\n        return \"Accelerate\";\n    #elif defined(GGML_BLAS_USE_MKL)\n        return \"MKL\";\n    #elif defined(GGML_BLAS_USE_BLIS)\n        return \"BLIS\";\n    #elif defined(GGML_BLAS_USE_NVPL)\n        return \"NVPL\";\n    #elif defined(GGML_BLAS_USE_OPENBLAS)\n        return \"OpenBLAS\";\n    #else\n        return \"BLAS\";\n    #endif\n\n    GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_blas_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n    // no memory to report\n    *free  = 0;\n    *total = 0;\n\n    GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_blas_device_get_type(ggml_backend_dev_t dev) {\n    return GGML_BACKEND_DEVICE_TYPE_ACCEL;\n\n    GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n    props->name        = ggml_backend_blas_device_get_name(dev);\n    props->description = ggml_backend_blas_device_get_description(dev);\n    props->type        = ggml_backend_blas_device_get_type(dev);\n    ggml_backend_blas_device_get_memory(dev, &props->memory_free, &props->memory_total);\n    props->caps = {\n        /* .async                 = */ false,\n        /* .host_buffer           = */ false,\n        /* .buffer_from_host_ptr  = */ true,\n        /* .events                = */ false,\n    };\n}\n\nstatic ggml_backend_t ggml_backend_blas_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n    return ggml_backend_blas_init();\n\n    GGML_UNUSED(dev);\n    GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_blas_device_get_buffer_type(ggml_backend_dev_t dev) {\n    return ggml_backend_cpu_buffer_type();\n\n    GGML_UNUSED(dev);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_blas_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n    return ggml_backend_cpu_buffer_from_ptr(ptr, size);\n\n    GGML_UNUSED(dev);\n    GGML_UNUSED(max_tensor_size);\n}\n\nstatic bool ggml_backend_blas_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n    const struct ggml_tensor * src0 = op->src[0];\n    const struct ggml_tensor * src1 = op->src[1];\n\n    switch (op->op) {\n        case GGML_OP_NONE:\n        case GGML_OP_RESHAPE:\n        case GGML_OP_VIEW:\n        case GGML_OP_PERMUTE:\n        case GGML_OP_TRANSPOSE:\n            return true;\n\n        case GGML_OP_MUL_MAT:\n        {\n            // BLAS usually is only faster for large matrices\n            const struct ggml_tensor * src0 = op->src[0];\n            const struct ggml_tensor * src1 = op->src[1];\n\n            const int64_t ne10 = src1->ne[0];\n\n            const int64_t ne0 = op->ne[0];\n            const int64_t ne1 = op->ne[1];\n\n            // TODO: find the optimal value\n            const int64_t min_batch = 32;\n\n            return ggml_is_contiguous(src0) &&\n                   ggml_is_contiguous(src1) &&\n                   src1->type == GGML_TYPE_F32 &&\n                   (ne0 >= min_batch && ne1 >= min_batch && ne10 >= min_batch) &&\n                   (src0->type == GGML_TYPE_F32 || ggml_get_type_traits(src0->type)->to_float != NULL);\n        }\n\n        case GGML_OP_OUT_PROD:\n            return op->src[0]->type == GGML_TYPE_F32 &&\n                   op->src[1]->type == GGML_TYPE_F32 &&\n                   ggml_is_matrix(src0) &&\n                   ggml_is_matrix(src1) &&\n                   ggml_is_contiguous(src0) &&\n                   (ggml_is_contiguous(src1) || ggml_is_transposed(src1)) &&\n                   (src0->type == GGML_TYPE_F32 || ggml_get_type_traits(src0->type)->to_float != NULL);\n\n        default:\n            return false;\n\n    }\n\n    GGML_UNUSED(dev);\n}\n\nstatic bool ggml_backend_blas_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n    return ggml_backend_buft_is_host(buft);\n\n    GGML_UNUSED(dev);\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_blas_device_i = {\n    /* .get_name             = */ ggml_backend_blas_device_get_name,\n    /* .get_description      = */ ggml_backend_blas_device_get_description,\n    /* .get_memory           = */ ggml_backend_blas_device_get_memory,\n    /* .get_type             = */ ggml_backend_blas_device_get_type,\n    /* .get_props            = */ ggml_backend_blas_device_get_props,\n    /* .init_backend         = */ ggml_backend_blas_device_init_backend,\n    /* .get_buffer_type      = */ ggml_backend_blas_device_get_buffer_type,\n    /* .get_host_buffer_type = */ NULL,\n    /* .buffer_from_host_ptr = */ ggml_backend_blas_device_buffer_from_host_ptr,\n    /* .supports_op          = */ ggml_backend_blas_device_supports_op,\n    /* .supports_buft        = */ ggml_backend_blas_device_supports_buft,\n    /* .offload_op           = */ NULL,\n    /* .event_new            = */ NULL,\n    /* .event_free           = */ NULL,\n    /* .event_synchronize    = */ NULL,\n};\n\n// backend reg interface\n\nstatic const char * ggml_backend_blas_reg_get_name(ggml_backend_reg_t reg) {\n    return \"BLAS\";\n\n    GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_blas_reg_get_device_count(ggml_backend_reg_t reg) {\n    return 1;\n\n    GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_blas_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n    GGML_ASSERT(index == 0);\n\n    static ggml_backend_device ggml_backend_blas_device = {\n        /* .iface   = */ ggml_backend_blas_device_i,\n        /* .reg     = */ reg,\n        /* .context = */ nullptr,\n    };\n\n    return &ggml_backend_blas_device;\n\n    GGML_UNUSED(reg);\n    GGML_UNUSED(index);\n}\n\nstatic void * ggml_backend_blas_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n    if (std::strcmp(name, \"ggml_backend_set_n_threads\") == 0) {\n        return (void *)ggml_backend_blas_set_n_threads;\n    }\n    return NULL;\n\n    GGML_UNUSED(reg);\n    GGML_UNUSED(name);\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_blas_reg_i = {\n    /* .get_name         = */ ggml_backend_blas_reg_get_name,\n    /* .get_device_count = */ ggml_backend_blas_reg_get_device_count,\n    /* .get_device       = */ ggml_backend_blas_reg_get_device,\n    /* .get_proc_address = */ ggml_backend_blas_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_blas_reg(void) {\n    static struct ggml_backend_reg ggml_backend_blas_reg = {\n        /* .api_version = */ GGML_BACKEND_API_VERSION,\n        /* .iface       = */ ggml_backend_blas_reg_i,\n        /* .context     = */ NULL,\n    };\n\n    return &ggml_backend_blas_reg;\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_blas_reg)\n"
  },
  {
    "path": "ggml/src/ggml-cann/CMakeLists.txt",
    "content": "if (\"cann${CANN_INSTALL_DIR}\" STREQUAL \"cann\" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})\n    set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})\n    message(STATUS \"CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}\")\nendif()\n\n# Auto-detech Soc type and Soc version, if detect failed, will abort build\nset(SOC_VERSION \"\")\nfunction(detect_ascend_soc_type SOC_VERSION)\n    execute_process(\n        COMMAND bash -c \"npu-smi info|awk -F' ' 'NF > 0 && NR==7 {print $3}'\"\n        OUTPUT_VARIABLE npu_info\n        RESULT_VARIABLE npu_result\n        OUTPUT_STRIP_TRAILING_WHITESPACE\n    )\n    if(\"${npu_info}\" STREQUAL \"\" OR ${npu_result})\n        message(FATAL_ERROR \"Auto-detech ascend soc type failed, please specify manually or check ascend device working normally.\")\n    endif()\n    set(${SOC_VERSION} \"Ascend${npu_info}\" PARENT_SCOPE)\nendfunction()\n\nif(NOT SOC_TYPE)\n    detect_ascend_soc_type(SOC_VERSION)\n    set(SOC_TYPE \"${SOC_VERSION}\")\n    message(STATUS \"CANN: SOC_VERSION auto-detected is:${SOC_VERSION}\")\nendif()\n\nstring(TOLOWER ${SOC_TYPE} SOC_VERSION) # SOC_VERSION need lower\n\n# Construct Soc specify compile option: ASCEND_#Soc_Major_SN. Such as ASCEND_910B, ASCEND_310P.\nstring(REGEX MATCH \"[0-9]+[a-zA-Z]\" SOC_TYPE_MAJOR_SN \"${SOC_VERSION}\")\nset(SOC_TYPE_COMPILE_OPTION \"ASCEND_${SOC_TYPE_MAJOR_SN}\")\nstring(TOUPPER ${SOC_TYPE_COMPILE_OPTION} SOC_TYPE_COMPILE_OPTION)\nmessage(STATUS \"CANN: SOC_VERSION =  ${SOC_VERSION}\")\noption(USE_ACL_GRAPH \"Enable CANN graph execution (ACL graph mode)\" OFF)\n\nif(USE_ACL_GRAPH AND (SOC_TYPE_MAJOR_SN STREQUAL \"310P\" OR SOC_TYPE_COMPILE_OPTION STREQUAL \"ASCEND_310P\"))\n    message(FATAL_ERROR\n        \"CANN Graph (ACL graph mode) is not supported on 310P devices. \"\n        \"Please build with -DUSE_ACL_GRAPH=OFF or use a supported SOC.\")\nendif()\n\nif (CANN_INSTALL_DIR)\n    # Only Support Linux.\n    if (NOT UNIX)\n        message(FATAL_ERROR \"CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}\")\n    endif()\n\n    # Supported platforms: x86-64, arm64\n    if (CMAKE_SYSTEM_PROCESSOR STREQUAL \"aarch64\")\n    elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL \"x86_64\" OR CMAKE_SYSTEM_PROCESSOR STREQUAL \"amd64\")\n    else()\n        message(FATAL_ERROR \"CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}\")\n    endif()\n\n    # Set header and libs\n    set(CANN_INCLUDE_DIRS\n        ${CANN_INSTALL_DIR}/include\n        ${CANN_INSTALL_DIR}/include/aclnn\n        ${CANN_INSTALL_DIR}/acllib/include\n    )\n\n    list(APPEND CANN_LIBRARIES\n        ascendcl\n        nnopbase\n        opapi\n        acl_op_compiler\n    )\n\n    file(GLOB GGML_SOURCES_CANN \"*.cpp\")\n\n    ggml_add_backend_library(ggml-cann ${GGML_SOURCES_CANN})\n    target_link_libraries(ggml-cann PRIVATE ${CANN_LIBRARIES})\n    target_include_directories(ggml-cann PRIVATE ${CANN_INCLUDE_DIRS})\n    target_link_directories(ggml-cann PRIVATE ${CANN_INSTALL_DIR}/lib64)\n\n    target_compile_definitions(ggml-cann PRIVATE \"-D${SOC_TYPE_COMPILE_OPTION}\")\n\n    if (USE_ACL_GRAPH)\n        target_compile_definitions(ggml-cann PRIVATE USE_ACL_GRAPH)\n        message(STATUS \"CANN: USE_ACL_GRAPH is enabled.\")\n    else()\n        message(STATUS \"CANN: USE_ACL_GRAPH is disabled.\")\n    endif()\n\n    message(STATUS \"CANN: CANN_INCLUDE_DIRS =  ${CANN_INCLUDE_DIRS}\")\n    message(STATUS \"CANN: CANN_LIBRARIES =  ${CANN_LIBRARIES}\")\nelse()\n    message(FATAL_ERROR \"CANN: Can't find CANN_INSTALL_DIR, did you forget to source set_var.sh?\")\nendif()\n"
  },
  {
    "path": "ggml/src/ggml-cann/acl_tensor.cpp",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"acl_tensor.h\"\n\n#include <algorithm>\n#include <cstring>\n\naclDataType ggml_cann_type_mapping(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_F32:\n            return ACL_FLOAT;\n        case GGML_TYPE_F16:\n            return ACL_FLOAT16;\n        case GGML_TYPE_BF16:\n            return ACL_BF16;\n        case GGML_TYPE_I8:\n            return ACL_INT8;\n        case GGML_TYPE_I16:\n            return ACL_INT16;\n        case GGML_TYPE_I32:\n            return ACL_INT32;\n        case GGML_TYPE_Q4_0:\n            return ACL_INT4;\n        case GGML_TYPE_Q8_0:\n            return ACL_INT8;\n        case GGML_TYPE_I64:\n            return ACL_INT64;\n        default:\n            return ACL_DT_UNDEFINED;\n    }\n}\n\nacl_tensor_ptr ggml_cann_create_tensor(const ggml_tensor * tensor,\n                                       int64_t *           ne,\n                                       size_t *            nb,\n                                       int64_t             dims,\n                                       aclFormat           format,\n                                       size_t              offset) {\n    // If tensor is bcasted, Up to GGML_MAX_DIMS additional dimensions will be\n    // added.\n    int64_t acl_ne[GGML_MAX_DIMS * 2], acl_stride[GGML_MAX_DIMS * 2];\n\n    if (ne == nullptr) {\n        for (int i = 0; i < GGML_MAX_DIMS; i++) {\n            acl_ne[i]     = tensor->ne[i];\n            // The step size of acl is in elements.\n            acl_stride[i] = tensor->nb[i] / ggml_element_size(tensor);\n        }\n    } else {\n        // With bcast\n        for (int i = 0; i < dims; i++) {\n            acl_ne[i]     = ne[i];\n            acl_stride[i] = nb[i] / ggml_element_size(tensor);\n        }\n    }\n\n    int64_t final_dims      = (dims == 0 ? GGML_MAX_DIMS : dims);\n    int64_t acl_storage_len = 1;\n    for (int i = 0; i < final_dims; i++) {\n        acl_storage_len += (acl_ne[i] - 1) * acl_stride[i];\n    }\n    size_t elem_offset = offset / ggml_element_size(tensor);\n    acl_storage_len += elem_offset;\n\n    // Reverse ne and stride.\n    std::reverse(acl_ne, acl_ne + final_dims);\n    std::reverse(acl_stride, acl_stride + final_dims);\n\n    aclTensor * raw = aclCreateTensor(acl_ne, final_dims, ggml_cann_type_mapping(tensor->type), acl_stride, elem_offset,\n                                      format, &acl_storage_len, 1, tensor->data);\n\n    return acl_tensor_ptr(raw);\n}\n\nacl_int_array_ptr ggml_cann_create_int_array(const int64_t * value, uint64_t size) {\n    aclIntArray * raw = aclCreateIntArray(value, size);\n    return acl_int_array_ptr(raw);\n}\n\nacl_scalar_ptr ggml_cann_create_scalar(void * value, aclDataType dataType) {\n    aclScalar * raw = aclCreateScalar(value, dataType);\n    return acl_scalar_ptr(raw);\n}\n\nbool ggml_cann_need_bcast(const ggml_tensor * t0, const ggml_tensor * t1) {\n    for (int i = 0; i < GGML_MAX_DIMS; i++) {\n        if (t1->ne[i] != t0->ne[i] && t1->ne[i] != 1) {\n            return true;\n        }\n    }\n    return false;\n}\n\nint64_t ggml_cann_get_bcast_shape(const ggml_tensor * src0,\n                                  const ggml_tensor * src1,\n                                  int64_t *           bcast_src0_ne,\n                                  int64_t *           bcast_src1_ne,\n                                  size_t *            bcast_src0_nb,\n                                  size_t *            bcast_src1_nb) {\n    GGML_ASSERT(ggml_can_repeat(src1, src0));\n    int bcast_dim_cnt = 0;\n    for (int i = 0; i < GGML_MAX_DIMS; i++) {\n        int64_t nr                   = src0->ne[i] / src1->ne[i];\n        bcast_src0_ne[bcast_dim_cnt] = src0->ne[i] / nr;\n        bcast_src1_ne[bcast_dim_cnt] = src1->ne[i];\n        bcast_src0_nb[bcast_dim_cnt] = src0->nb[i];\n        bcast_src1_nb[bcast_dim_cnt] = src1->nb[i];\n        bcast_dim_cnt++;\n        if (nr != 1) {\n            // Need to add an extra dim.\n            bcast_src0_ne[bcast_dim_cnt] = nr;\n            bcast_src1_ne[bcast_dim_cnt] = 1;\n            bcast_src0_nb[bcast_dim_cnt] = bcast_src0_nb[bcast_dim_cnt - 1] * bcast_src0_ne[bcast_dim_cnt - 1];\n            bcast_src1_nb[bcast_dim_cnt] = bcast_src1_nb[bcast_dim_cnt - 1] * bcast_src1_ne[bcast_dim_cnt - 1];\n            bcast_dim_cnt++;\n        }\n    }\n    return bcast_dim_cnt;\n}\n\nint64_t ggml_cann_get_mulmat_bcast_shape(const int64_t * input_ne,\n                                         const int64_t * weight_ne,\n                                         const int64_t * dst_ne,\n                                         const size_t *  input_nb,\n                                         const size_t *  weight_nb,\n                                         const size_t *  dst_nb,\n                                         int64_t *       bcast_input_ne,\n                                         int64_t *       bcast_weight_ne,\n                                         int64_t *       bcast_dst_ne,\n                                         size_t *        bcast_input_nb,\n                                         size_t *        bcast_weight_nb,\n                                         size_t *        bcast_dst_nb) {\n    // input and dst shoule in same shape, except first two dims.\n    GGML_ASSERT(input_ne[2] == dst_ne[2]);\n    GGML_ASSERT(input_ne[3] == dst_ne[3]);\n\n    int bcast_dim_cnt = 0;\n\n    // For mul_mat, a dimension needs to be added before the dimension that\n    // weight needs to be expanded to satisfy the bcast rule of matrix\n    // multiplication.\n    for (int i = 0; i < GGML_MAX_DIMS; i++) {\n        int64_t nr = input_ne[i] / weight_ne[i];\n        // Do not use bcast in the first two dimensions because we only support\n        // the bcast batch dimension. Just copy them.\n        if (i < 2 || nr == 1) {\n            bcast_input_ne[bcast_dim_cnt]  = input_ne[i];\n            bcast_weight_ne[bcast_dim_cnt] = weight_ne[i];\n            bcast_dst_ne[bcast_dim_cnt]    = dst_ne[i];\n\n            bcast_input_nb[bcast_dim_cnt]  = input_nb[i];\n            bcast_weight_nb[bcast_dim_cnt] = weight_nb[i];\n            bcast_dst_nb[bcast_dim_cnt]    = dst_nb[i];\n            bcast_dim_cnt++;\n        } else {\n            // Need to add an extra dim.\n            bcast_input_ne[bcast_dim_cnt]  = nr;\n            bcast_dst_ne[bcast_dim_cnt]    = nr;\n            bcast_weight_ne[bcast_dim_cnt] = 1;\n            bcast_input_nb[bcast_dim_cnt]  = input_nb[i];\n            bcast_dst_nb[bcast_dim_cnt]    = dst_nb[i];\n            bcast_weight_nb[bcast_dim_cnt] = weight_nb[i];\n            bcast_dim_cnt++;\n\n            bcast_input_ne[bcast_dim_cnt]  = input_ne[i] / nr;\n            bcast_dst_ne[bcast_dim_cnt]    = dst_ne[i] / nr;\n            bcast_weight_ne[bcast_dim_cnt] = weight_ne[i];\n            bcast_input_nb[bcast_dim_cnt]  = bcast_input_nb[bcast_dim_cnt - 1] * bcast_input_ne[bcast_dim_cnt - 1];\n            bcast_dst_nb[bcast_dim_cnt]    = bcast_dst_nb[bcast_dim_cnt - 1] * bcast_dst_ne[bcast_dim_cnt - 1];\n            bcast_weight_nb[bcast_dim_cnt] = bcast_weight_nb[bcast_dim_cnt - 1] * bcast_weight_ne[bcast_dim_cnt - 1];\n            bcast_dim_cnt++;\n        }\n    }\n    return bcast_dim_cnt;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cann/acl_tensor.h",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_ACL_TENSOR_H\n#define CANN_ACL_TENSOR_H\n\n#include \"common.h\"\n\n#include <aclnn/aclnn_base.h>\n\n#include <algorithm>\n#include <cstring>\n\n/**\n * @brief\tMaps a ggml_type to its corresponding aclDataType.\n *\n * @details\tThis function takes a ggml_type as input and returns the corresponding\n *\t\t\taclDataType. It supports mapping for various ggml_types. If the input type\n *\t\t\tdoes not match any of the predefined ggml_types, the function returns\n *          ACL_DT_UNDEFINED.\n *\n * @param\ttype    The ggml_type to be mapped.\n * @return\tThe corresponding aclDataType. If the input type is not recognized,\n *\t\t\tACL_DT_UNDEFINED is returned.\n */\naclDataType ggml_cann_type_mapping(ggml_type type);\n\n// Deleter for acl objects.\ntemplate <typename T, aclError (*DestroyFunc)(const T *)> struct acl_deleter {\n    void operator()(T * ptr) const noexcept {\n        if (ptr) {\n            ACL_CHECK(DestroyFunc(ptr));\n        }\n    }\n};\n\nusing acl_tensor_ptr      = std::unique_ptr<aclTensor, acl_deleter<aclTensor, aclDestroyTensor>>;\nusing acl_int_array_ptr   = std::unique_ptr<aclIntArray, acl_deleter<aclIntArray, aclDestroyIntArray>>;\nusing acl_scalar_ptr      = std::unique_ptr<aclScalar, acl_deleter<aclScalar, aclDestroyScalar>>;\nusing acl_tensor_list_ptr = std::unique_ptr<aclTensorList, acl_deleter<aclTensorList, aclDestroyTensorList>>;\n\n/**\n * @brief   Creates an ACL tensor from a ggml_tensor with optional shape.\n *\n * @details This function creates an ACL tensor based on the properties of the\n *          provided ggml_tensor. It supports customer shape by adjusting dimensions\n *          and strides accordingly. If customer shape is applied, additional\n *          dimensions and strides are calculated based on the provided parameters.\n *\n * @param   tensor      Pointer to the ggml_tensor to be converted to ACL tensor.\n * @param   ne          Pointer to an array containing dimensions. Defaults to nullptr\n *                      if no customer shape is applied.\n * @param   nb          Pointer to an array containing strides. Defaults to nullptr\n *                      if no customer shape is applied.\n * @param   dims        Number of dimensions in the tensor. Defaults to 0 if no customer\n *                      shape is applied.\n * @param   format      ACL tensor format. Defaults to ACL_FORMAT_ND.\n * @param   offset      Offset in bytes for the ACL tensor data. Defaults to 0.\n * @return  Pointer to the created ACL tensor.\n */\nacl_tensor_ptr ggml_cann_create_tensor(const ggml_tensor * tensor,\n                                       int64_t *           ne     = nullptr,\n                                       size_t *            nb     = nullptr,\n                                       int64_t             dims   = 0,\n                                       aclFormat           format = ACL_FORMAT_ND,\n                                       size_t              offset = 0);\n\n/**\n * @brief   Template for creating an ACL tensor from provided parameters. typename TYPE\n *          should be size_t or float.\n *\n * @details This function creates an ACL tensor using the provided data pointer,\n *          data type, dimensions, strides, format, offset, and additional parameters.\n *          It calculates necessary dimensions and strides based on the provided ne and nb\n *          arrays, adjusting them for the ACL tensor creation. The ACL storage length\n *          is also calculated based on the provided dimensions and strides.\n *\n * @param   data_ptr    Pointer to the data buffer for the ACL tensor.\n * @param   dtype       ACL data type of the tensor.\n * @param   type_size   Size of each element in the tensor data buffer.\n * @param   ne          Pointer to an array containing tensor dimensions.\n * @param   nb          Pointer to an array containing tensor strides.\n * @param   dims        Number of dimensions of the tensor.\n * @param   format      ACL tensor format. Defaults to ACL_FORMAT_ND.\n * @param   offset      Offset in bytes for the ACL tensor data. Defaults to 0.\n * @return  Pointer to the created ACL tensor.\n */\ntemplate <typename TYPE>\nacl_tensor_ptr ggml_cann_create_tensor(void *      data_ptr,\n                                       aclDataType dtype,\n                                       TYPE        type_size,\n                                       int64_t *   ne,\n                                       TYPE *      nb,\n                                       int64_t     dims,\n                                       aclFormat   format = ACL_FORMAT_ND,\n                                       size_t      offset = 0) {\n    int64_t tmp_ne[GGML_MAX_DIMS * 2];\n    int64_t tmp_stride[GGML_MAX_DIMS * 2];\n\n    memcpy(tmp_ne, ne, dims * sizeof(int64_t));\n    for (int i = 0; i < dims; i++) {\n        tmp_stride[i] = nb[i] / type_size;\n    }\n\n    int64_t acl_storage_len = 1;\n    for (int i = 0; i < dims; i++) {\n        acl_storage_len += (tmp_ne[i] - 1) * tmp_stride[i];\n    }\n\n    std::reverse(tmp_ne, tmp_ne + dims);\n    std::reverse(tmp_stride, tmp_stride + dims);\n\n    aclTensor * raw =\n        aclCreateTensor(tmp_ne, dims, dtype, tmp_stride, offset / type_size, format, &acl_storage_len, 1, data_ptr);\n\n    return acl_tensor_ptr(raw);\n}\n\n/**\n * @brief Create an ACL int array resource wrapped in a smart pointer.\n *\n * This function constructs an aclIntArray from the provided int64_t values\n * and returns it as an acl_int_array_ptr (a std::unique_ptr with a custom\n * deleter). The returned pointer owns the ACL resource and will automatically\n * destroy it via aclDestroyIntArray().\n *\n * @param value  Pointer to the int64_t elements.\n * @param size   Number of elements in value.\n *\n * @return A smart pointer managing the created ACL int array.\n */\nacl_int_array_ptr ggml_cann_create_int_array(const int64_t * value, uint64_t size);\n\n/**\n * @brief Create an ACL scalar resource wrapped in a smart pointer.\n *\n * This function constructs an aclScalar from the raw value pointer and ACL\n * data type, then returns it as an acl_scalar_ptr (a std::unique_ptr with\n * a custom deleter). The returned pointer owns the ACL scalar and will\n * automatically destroy it via aclDestroyScalar().\n *\n * @param value     Pointer to the raw scalar memory.\n * @param dataType  ACL data type of the scalar.\n *\n * @return A smart pointer managing the created ACL scalar.\n */\nacl_scalar_ptr ggml_cann_create_scalar(void * value, aclDataType dataType);\n\n/**\n * @brief Create an ACL tensor list from multiple tensor smart pointers.\n *\n * This function accepts a variadic list of acl_tensor_ptr (a unique_ptr with\n * custom deleter) and produces an aclTensorList using aclCreateTensorList().\n *\n * The lifecycle management of the tensor objects changes as follows:\n *  - aclCreateTensorList() takes ownership of the tensors\n *  - Each input smart pointer releases ownership using release()\n *  - As a result, the tensors will NOT be destroyed by unique_ptr\n *  - Instead, they will be destroyed when aclDestroyTensorList() is called\n *\n * This ensures correct ownership transfer and prevents double-free situations.\n *\n * @param acl_tensor_ptr  Variadic template parameter; each argument must be\n *                         a unique_ptr-like type supporting get() and release().\n *\n * @param tensors  Variadic list of acl_tensor_ptr objects. Ownership of\n *                         each tensor is transferred away from these smart pointers.\n *\n * @return A smart pointer (acl_tensor_list_ptr) owning the created ACL tensor list.\n *\n * @note This implementation is C++11 compatible. The ownership-release process is\n *       executed using a pack expansion inside an initializer list.\n */\ntemplate <typename... acl_tensor_ptr> acl_tensor_list_ptr ggml_cann_create_tensor_list(acl_tensor_ptr &&... tensors) {\n    aclTensor *     raw_tensors[] = { tensors.get()... };\n    aclTensorList * raw           = aclCreateTensorList(raw_tensors, sizeof...(tensors));\n    // aclTensor will release by aclTensorList, so release ownership without\n    // destroying the tensor\n    int             dummy[]       = { (tensors.release(), 0)... };\n    GGML_UNUSED(dummy);\n    return acl_tensor_list_ptr(raw);\n}\n\n/**\n * @brief   Checks if tensors require broadcasting based on their shapes.\n *\n * @details This function determines if two ggml_tensors need to be broadcasted for\n *          element-wise operations. Broadcasting is necessary if the shapes of the\n *          tensors are not identical and no dimension in either tensor equals 1.\n *\n * @param   t0      Pointer to the first ggml_tensor.\n * @param   t1      Pointer to the second ggml_tensor.\n * @return  True if broadcasting is needed, False otherwise.\n *\n * @remarks This function iterates over the dimensions of t0 and t1. It checks if each\n *          dimension in t1 differs from t0's corresponding dimension and is not equal\n *          to 1. If such a dimension is found, broadcasting is required to align t1\n *          with t0 for element-wise operations.\n */\nbool ggml_cann_need_bcast(const ggml_tensor * t0, const ggml_tensor * t1);\n\n/**\n * @brief   Computes broadcast shapes and strides for two ggml_tensors.\n *\n * @details This function calculates the broadcast shapes and strides for two ggml_tensors,\n *          following the broadcasting rules similar to numpy. It adjusts dimensions and\n *          strides to ensure compatibility for element-wise operations where one tensor\n *          can be broadcasted to match the shape of another tensor.\n *\n * @param   src0                Pointer to the first ggml_tensor.\n * @param   src1                Pointer to the second ggml_tensor.\n * @param   bcast_ne_src0       Output array to store broadcasted dimensions for src0.\n * @param   bcast_ne_src1       Output array to store broadcasted dimensions for src1.\n * @param   bcast_nb_src0       Output array to store broadcasted strides for src0.\n * @param   bcast_nb_src1       Output array to store broadcasted strides for src1.\n * @return  Number of dimensions in the broadcasted shape.\n *\n * @pre     ggml_can_repeat(src1, src0) must return true, indicating src1 can be broadcasted\n *          to match src0.\n *\n * @remarks This function iterates over the dimensions of src0 and src1, calculating the\n *          necessary broadcast dimensions and strides. If a dimension requires broadcasting\n *          (i.e., its size in src1 is smaller than in src0), an additional dimension is\n *          added with size calculated to match src0's dimension. This adjustment ensures\n *          that src1 can be element-wise broadcasted to src0's shape.\n *\n *  How it works:\n *\n *  if dim0 has padding.\n *  a -> (2, 2) padding = 2\n *   a: [[1, 2, *, *]\n *       [2, 3, *, *]]\n *  nb = (8, 4, 2)\n *\n *  if a should bcast with b -> (2, 4)\n *  b' -> (2, 2, 2)\n *  b : [[1, 2, 3, 4, *, *]\n *       [5, 6, 7, 8, *, *]]\n *  nb = (12, 6, 1)\n *\n *  after bcast:\n *  a' -> (2, 1, 2)\n *  a': [[[1, 2], *, *]\n *       [[2, 3], *, *]]\n *  nb = (8, 4, 2, 1)\n *\n *  b' : [[[1, 2], [3, 4], *, *]\n *        [[5, 6], [7, 8], *, *]]\n *  nb = (12, 6, 2, 1)\n *  \\endcode\n *\n *  dim1 in a inserted dim, should add nb for dim1,\n *  and all other nb moves to next in order.\n */\nint64_t ggml_cann_get_bcast_shape(const ggml_tensor * src0,\n                                  const ggml_tensor * src1,\n                                  int64_t *           bcast_ne_src0,\n                                  int64_t *           bcast_ne_src1,\n                                  size_t *            bcast_nb_src0,\n                                  size_t *            bcast_nb_src1);\n\n// Bcast macro to avoid duplicate code.\n#define BCAST_SHAPE(src0, src1)                                                                      \\\n    int64_t bcast_##src0##_ne[GGML_MAX_DIMS * 2];                                                    \\\n    int64_t bcast_##src1##_ne[GGML_MAX_DIMS * 2];                                                    \\\n    size_t  bcast_##src0##_nb[GGML_MAX_DIMS * 2];                                                    \\\n    size_t  bcast_##src1##_nb[GGML_MAX_DIMS * 2];                                                    \\\n    int64_t bcast_dims = ggml_cann_get_bcast_shape(src0, src1, bcast_##src0##_ne, bcast_##src1##_ne, \\\n                                                   bcast_##src0##_nb, bcast_##src1##_nb);\n\n#define BCAST_PARAM(tensor) bcast_##tensor##_ne, bcast_##tensor##_nb, bcast_dims\n\n/**\n * @brief Calculates broadcast shapes for matrix multiplication.\n *\n * @details This function computes the broadcast shapes required for matrix multiplication\n *          based on the input, weight, and destination tensor shapes. It ensures that the\n *          dimensions of weight tensors are expanded appropriately to satisfy matrix\n *          multiplication broadcast rules.\n *\n * @param input_ne      Array containing the dimensions of the input tensor.\n * @param weight_ne     Array containing the dimensions of the weight tensor.\n * @param dst_ne        Array containing the dimensions of the destination tensor.\n * @param input_nb      Array containing the strides of the input tensor.\n * @param weight_nb     Array containing the strides of the weight tensor.\n * @param dst_nb        Array containing the strides of the destination tensor.\n * @param bcast_input_ne    Output array for broadcasted input tensor dimensions.\n * @param bcast_weight_ne   Output array for broadcasted weight tensor dimensions.\n * @param bcast_dst_ne      Output array for broadcasted destination tensor dimensions.\n * @param bcast_input_nb    Output array for broadcasted input tensor strides.\n * @param bcast_weight_nb   Output array for broadcasted weight tensor strides.\n * @param bcast_dst_nb      Output array for broadcasted destination tensor strides.\n * @return The number of dimensions in the broadcasted tensors.\n *\n * @remarks This function iterates over the tensor dimensions and calculates the broadcast\n *          shapes needed for matrix multiplication. It ensures that dimensions where\n *          weight tensor requires expansion are appropriately handled to conform with\n *          broadcasting rules.\n * @note compare with ggml_cann_get_bcast_shape, mul_mat broadcast need add this new dim\n *       before cast dim.\n * @sa ggml_cann_get_bcast_shape\n */\nint64_t ggml_cann_get_mulmat_bcast_shape(const int64_t * input_ne,\n                                         const int64_t * weight_ne,\n                                         const int64_t * dst_ne,\n                                         const size_t *  input_nb,\n                                         const size_t *  weight_nb,\n                                         const size_t *  dst_nb,\n                                         int64_t *       bcast_input_ne,\n                                         int64_t *       bcast_weight_ne,\n                                         int64_t *       bcast_dst_ne,\n                                         size_t *        bcast_input_nb,\n                                         size_t *        bcast_weight_nb,\n                                         size_t *        bcast_dst_nb);\n\n// Bcast macro to avoid duplicate code.\n#define BCAST_MUL_MAT_SHAPE(input, weight, dst)                                                                  \\\n    int64_t bcast_##input##_ne[GGML_MAX_DIMS * 2];                                                               \\\n    int64_t bcast_##weight##_ne[GGML_MAX_DIMS * 2];                                                              \\\n    int64_t bcast_##dst##_ne[GGML_MAX_DIMS * 2];                                                                 \\\n    size_t  bcast_##input##_nb[GGML_MAX_DIMS * 2];                                                               \\\n    size_t  bcast_##weight##_nb[GGML_MAX_DIMS * 2];                                                              \\\n    size_t  bcast_##dst##_nb[GGML_MAX_DIMS * 2];                                                                 \\\n    int64_t bcast_dims = ggml_cann_get_mulmat_bcast_shape(                                                       \\\n        input->ne, weight->ne, dst->ne, input->nb, weight->nb, dst->nb, bcast_##input##_ne, bcast_##weight##_ne, \\\n        bcast_##dst##_ne, bcast_##input##_nb, bcast_##weight##_nb, bcast_##dst##_nb);\n\n#define BCAST_MUL_MAT_PARAM(tensor) bcast_##tensor##_ne, bcast_##tensor##_nb, bcast_dims\n\n#endif  // CANN_ACL_TENSOR_H\n"
  },
  {
    "path": "ggml/src/ggml-cann/aclnn_ops.cpp",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"aclnn_ops.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include <aclnnop/aclnn_add.h>\n#include <aclnnop/aclnn_add_rms_norm.h>\n#include <aclnnop/aclnn_addcdiv.h>\n#include <aclnnop/aclnn_argmax.h>\n#include <aclnnop/aclnn_avgpool2d.h>\n#include <aclnnop/aclnn_batch_matmul.h>\n#include <aclnnop/aclnn_cast.h>\n#include <aclnnop/aclnn_clamp.h>\n#include <aclnnop/aclnn_constant_pad_nd.h>\n#include <aclnnop/aclnn_convolution.h>\n#include <aclnnop/aclnn_copy.h>\n#include <aclnnop/aclnn_div.h>\n#include <aclnnop/aclnn_elu.h>\n#include <aclnnop/aclnn_embedding.h>\n#include <aclnnop/aclnn_eq_tensor.h>\n#include <aclnnop/aclnn_exp.h>\n#include <aclnnop/aclnn_fill_scalar.h>\n#include <aclnnop/aclnn_fused_infer_attention_score_v2.h>\n#include <aclnnop/aclnn_ger.h>\n#include <aclnnop/aclnn_group_norm.h>\n#include <aclnnop/aclnn_grouped_matmul_v3.h>\n#include <aclnnop/aclnn_gt_scalar.h>\n#include <aclnnop/aclnn_im2col.h>\n#include <aclnnop/aclnn_index_copy.h>\n#include <aclnnop/aclnn_index_fill_tensor.h>\n#include <aclnnop/aclnn_index_select.h>\n#include <aclnnop/aclnn_layer_norm.h>\n#include <aclnnop/aclnn_log.h>\n#include <aclnnop/aclnn_matmul.h>\n#include <aclnnop/aclnn_max_pool.h>\n#include <aclnnop/aclnn_mean.h>\n#include <aclnnop/aclnn_mm.h>\n#include <aclnnop/aclnn_mul.h>\n#include <aclnnop/aclnn_mv.h>\n#include <aclnnop/aclnn_permute.h>\n#include <aclnnop/aclnn_pow.h>\n#include <aclnnop/aclnn_pow_tensor_tensor.h>\n#include <aclnnop/aclnn_reduce_sum.h>\n#include <aclnnop/aclnn_reflection_pad1d.h>\n#include <aclnnop/aclnn_repeat.h>\n#include <aclnnop/aclnn_repeat_interleave.h>\n#include <aclnnop/aclnn_rms_norm.h>\n#include <aclnnop/aclnn_roll.h>\n#include <aclnnop/aclnn_softmax.h>\n#include <aclnnop/aclnn_sub.h>\n#include <aclnnop/aclnn_sum.h>\n#include <aclnnop/aclnn_threshold.h>\n#include <aclnnop/aclnn_tril.h>\n#include <aclnnop/aclnn_triu.h>\n#include <aclnnop/aclnn_upsample_nearest_2d.h>\n#include <aclnnop/aclnn_weight_quant_batch_matmul_v2.h>\n#include <aclnnop/aclnn_zero.h>\n#include <float.h>\n\n#include <cmath>\n#include <cstring>\n#include <exception>\n#include <vector>\n\n#define GGML_COMMON_DECL_C\n\n#include \"../ggml-common.h\"\n\nvoid bcast_shape(ggml_tensor *    src0,\n                 ggml_tensor *    src1,\n                 ggml_tensor *    dst,\n                 acl_tensor_ptr & acl_src0,\n                 acl_tensor_ptr & acl_src1,\n                 acl_tensor_ptr & acl_dst) {\n    GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_can_repeat(src1, src0));\n    // Need bcast\n    if (!ggml_are_same_shape(src0, src1) && ggml_cann_need_bcast(src0, src1)) {\n        BCAST_SHAPE(src0, src1)\n        acl_src0 = ggml_cann_create_tensor(src0, BCAST_PARAM(src0));\n        acl_src1 = ggml_cann_create_tensor(src1, BCAST_PARAM(src1));\n        acl_dst  = ggml_cann_create_tensor(dst, BCAST_PARAM(src0));\n    } else {\n        acl_src0 = ggml_cann_create_tensor(src0);\n        acl_src1 = ggml_cann_create_tensor(src1);\n        acl_dst  = ggml_cann_create_tensor(dst);\n    }\n}\n\nvoid ggml_cann_op_unary(std::function<void(ggml_backend_cann_context &, aclTensor *, aclTensor *)> unary_op,\n                        ggml_backend_cann_context &                                                ctx,\n                        ggml_tensor *                                                              dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    unary_op(ctx, acl_src.get(), acl_dst.get());\n}\n\nvoid ggml_cann_op_unary_gated(std::function<void(ggml_backend_cann_context &, aclTensor *, aclTensor *)> unary_op,\n                              ggml_backend_cann_context &                                                ctx,\n                              ggml_tensor *                                                              dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n    acl_tensor_ptr acl_src0, acl_src1;\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n\n        acl_src0 = ggml_cann_create_tensor(src0);\n        acl_src1 = ggml_cann_create_tensor(src1);\n    } else {\n        int64_t ne[] = { src0->ne[0] / 2, src0->ne[1], src0->ne[2], src0->ne[3] };\n        size_t  nb[] = { src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3] };\n        acl_src0     = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, 0);\n        acl_src1 = ggml_cann_create_tensor(src0, ne, nb, GGML_MAX_DIMS, ACL_FORMAT_ND, ne[0] * ggml_element_size(src0));\n        if (swapped) {\n            std::swap(acl_src0, acl_src1);\n        }\n    }\n\n    unary_op(ctx, acl_src0.get(), acl_dst.get());\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_dst.get(), acl_src1.get());\n}\n\n/**\n * @brief Repeats elements of a tensor along each dimension according to the\n * specified repeat array.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor to be repeated.\n * @param acl_dst The destination tensor after repeating.\n * @param repeat_array The array specifying the number of repetitions along each\n * dimension.\n */\nstatic void aclnn_repeat(ggml_backend_cann_context & ctx,\n                         aclTensor *                 acl_src,\n                         aclTensor *                 acl_dst,\n                         int64_t *                   repeat_array) {\n    // repeat tensor along each dim with repeat_array\n    acl_int_array_ptr repeats = ggml_cann_create_int_array(repeat_array, GGML_MAX_DIMS);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Repeat, acl_src, repeats.get(), acl_dst);\n}\n\n/**\n * @brief Casts the data type of a source tensor to a destination tensor.\n *\n * This function casts the data type of the source tensor `acl_src` to the\n * specified data type `cast_data_type` and stores the result in the destination\n * tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose data type will be casted.\n * @param acl_dst The destination tensor where the casted result will be stored.\n * @param cast_data_type The target data type to which the source tensor will be\n * casted.\n */\nstatic void aclnn_cast(ggml_backend_cann_context & ctx,\n                       aclTensor *                 acl_src,\n                       aclTensor *                 acl_dst,\n                       aclDataType                 cast_data_type) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src, cast_data_type, acl_dst);\n}\n\nvoid ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n    GGML_ASSERT(ggml_can_repeat(src, dst));\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    int64_t repeatsArray[] = { dst->ne[3] / src->ne[3], dst->ne[2] / src->ne[2], dst->ne[1] / src->ne[1],\n                               dst->ne[0] / src->ne[0] };\n\n    aclnn_repeat(ctx, acl_src.get(), acl_dst.get(), repeatsArray);\n}\n\nvoid aclnn_add(ggml_backend_cann_context & ctx, aclTensor * acl_src0, aclTensor * acl_src1, aclTensor * acl_dst) {\n    float          alphaValue = 1.0f;\n    acl_scalar_ptr alpha      = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n    if (acl_dst != nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha.get(), acl_dst);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_src0, acl_src1, alpha.get());\n    }\n}\n\nvoid aclnn_sub(ggml_backend_cann_context & ctx, aclTensor * acl_src0, aclTensor * acl_src1, aclTensor * acl_dst) {\n    float          alphaValue = 1.0f;\n    acl_scalar_ptr alpha      = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n    if (acl_dst != nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Sub, acl_src0, acl_src1, alpha.get(), acl_dst);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSub, acl_src0, acl_src1, alpha.get());\n    }\n}\n\nvoid aclnn_mul(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_other, aclTensor * acl_dst) {\n    if (acl_dst != nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_src, acl_other, acl_dst);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_src, acl_other);\n    }\n}\n\nvoid aclnn_div(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_other, aclTensor * acl_dst) {\n    if (acl_dst != nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src, acl_other, acl_dst);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDiv, acl_src, acl_other);\n    }\n}\n\n/**\n * @brief Multiplies elements of a tensor by a scalar value, optionally\n * in-place.\n *\n * This function multiplies each element of the source tensor `acl_src` by the\n * scalar `scale` and stores the result in the destination tensor `acl_dst`. If\n * `inplace` is true, `acl_dst` will not be used and the operation is performed\n *  in-place on `acl_src`.\n * The operation is defined as:\n * \\f[\n *     \\text {acl_dst }_i=\\text {acl_src }_i \\times \\text {scale}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be multiplied.\n * @param scale The scalar value by which each element of `acl_src` will be\n *  multiplied.\n * @param acl_dst The destination tensor where the result will be stored if\n * `inplace` is false.\n * @param inplace Flag indicating whether to perform the operation in-place on\n * `acl_src`.\n */\nstatic void aclnn_muls(ggml_backend_cann_context & ctx,\n                       aclTensor *                 acl_src,\n                       float                       scale,\n                       aclTensor *                 acl_dst,\n                       bool                        inplace) {\n    acl_scalar_ptr acl_scale = ggml_cann_create_scalar(&scale, aclDataType::ACL_FLOAT);\n    if (inplace) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_src, acl_scale.get());\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, acl_scale.get(), acl_dst);\n    }\n}\n\nvoid ggml_cann_leaky_relu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    float negative_slope;\n    memcpy(&negative_slope, dst->op_params, sizeof(float));\n    acl_scalar_ptr acl_negative_slope = ggml_cann_create_scalar(&negative_slope, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, LeakyRelu, acl_src.get(), acl_negative_slope.get(), acl_dst.get());\n}\n\n/**\n * @brief Concatenates a list of tensors along a specified dimension and stores\n * the result in a destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param tensorList The list of tensors to be concatenated.\n * @param acl_dst The destination tensor where the concatenated result will be\n * stored.\n * @param concat_dim The dimension along which the tensors will be concatenated.\n */\nstatic void aclnn_concat(ggml_backend_cann_context & ctx,\n                         aclTensorList *             tensorList,\n                         aclTensor *                 acl_dst,\n                         int64_t                     concat_dim) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, Cat, tensorList, concat_dim, acl_dst);\n}\n\nvoid ggml_cann_concat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor *  src0     = dst->src[0];\n    ggml_tensor *  src1     = dst->src[1];\n    acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1);\n    acl_tensor_ptr acl_dst  = ggml_cann_create_tensor(dst);\n\n    const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(dim >= 0 && dim < 4);\n    int32_t acl_dim = 3 - dim;\n\n    acl_tensor_list_ptr tensor_list = ggml_cann_create_tensor_list(acl_src0, acl_src1);\n    aclnn_concat(ctx, tensor_list.get(), acl_dst.get(), acl_dim);\n}\n\n/**\n * @brief Creates a tensor with values starting from `start`, incremented by\n * `step`, and ending before `stop`.\n *\n * This function performs the operation:\n * \\f[\n *    \\text {out }_{i+1}=\\text {out }_i+\\text {step}\n * \\f]\n * the range is [start, stop).\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_dst The destination tensor where the values will be stored.\n * @param start The starting value of the range.\n * @param stop The ending value of the range (exclusive).\n * @param step The step size between consecutive values.\n * @param n_elements The number of elements in the destination tensor.\n */\nstatic void aclnn_arange(ggml_backend_cann_context & ctx,\n                         aclTensor *                 acl_dst,\n                         float                       start,\n                         float                       stop,\n                         float                       step,\n                         int64_t                     n_elements) {\n    int64_t steps = (int64_t) std::ceil((stop - start) / step);\n    GGML_ASSERT(n_elements == steps);\n\n    acl_scalar_ptr acl_start = ggml_cann_create_scalar(&start, aclDataType::ACL_FLOAT);\n    acl_scalar_ptr acl_end   = ggml_cann_create_scalar(&stop, aclDataType::ACL_FLOAT);\n    acl_scalar_ptr acl_step  = ggml_cann_create_scalar(&step, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Arange, acl_start.get(), acl_end.get(), acl_step.get(), acl_dst);\n}\n\nvoid ggml_cann_arange(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    int64_t n_elements = ggml_nelements(dst);\n    float   start;\n    float   stop;\n    float   step;\n    memcpy(&start, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&stop, (float *) dst->op_params + 1, sizeof(float));\n    memcpy(&step, (float *) dst->op_params + 2, sizeof(float));\n\n    aclnn_arange(ctx, acl_dst.get(), start, stop, step, n_elements);\n}\n\nvoid ggml_cann_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    float min;\n    float max;\n    memcpy(&min, dst->op_params, sizeof(float));\n    memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    acl_scalar_ptr acl_min = ggml_cann_create_scalar(&min, aclDataType::ACL_FLOAT);\n    acl_scalar_ptr acl_max = ggml_cann_create_scalar(&max, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_src.get(), acl_min.get(), acl_max.get(), acl_dst.get());\n}\n\nvoid ggml_cann_scale(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    // scale factor\n    float v;\n    memcpy(&v, dst->op_params, sizeof(float));\n\n    acl_scalar_ptr scale   = ggml_cann_create_scalar(&v, aclDataType::ACL_FLOAT);\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src.get(), scale.get(), acl_dst.get());\n}\n\nvoid ggml_cann_argsort(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor *        src   = dst->src[0];\n    enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];\n\n    acl_tensor_ptr       acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr       acl_dst = ggml_cann_create_tensor(dst);\n    ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), ggml_nelements(dst) * sizeof(int64_t));\n    void *               buffer = temp_buffer_allocator.get();\n    acl_tensor_ptr       tmp_tensor =\n        ggml_cann_create_tensor(buffer, ACL_INT64, ggml_type_size(dst->type), dst->ne, dst->nb, GGML_MAX_DIMS);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Argsort, acl_src.get(), -1, (order == GGML_SORT_ORDER_DESC ? true : false),\n                            tmp_tensor.get());\n    GGML_CANN_CALL_ACLNN_OP(ctx, Cast, tmp_tensor.get(), ggml_cann_type_mapping(dst->type), acl_dst.get());\n}\n\nvoid ggml_cann_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    std::vector<int64_t> normData = { dst->ne[0] };\n    acl_int_array_ptr    norm     = ggml_cann_create_int_array(normData.data(), normData.size());\n    GGML_CANN_CALL_ACLNN_OP(ctx, LayerNorm, acl_src.get(), norm.get(), nullptr, nullptr, eps, acl_dst.get(), nullptr,\n                            nullptr);\n}\n\nvoid ggml_cann_l2_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    size_t               type_size = ggml_type_size(src->type);\n    int64_t              n_bytes   = src->ne[3] * src->ne[2] * src->ne[1] * type_size;\n    ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes);\n    void *               buffer = temp_buffer_allocator.get();\n\n    int64_t div_ne[] = { 1, src->ne[1], src->ne[2], src->ne[3] };\n    size_t  div_nb[GGML_MAX_DIMS];\n    div_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n        div_nb[i] = div_nb[i - 1] * div_ne[i - 1];\n    }\n    acl_tensor_ptr acl_div = ggml_cann_create_tensor(buffer, ACL_FLOAT, type_size, div_ne, div_nb, GGML_MAX_DIMS);\n\n    std::vector<int64_t> norm_dims  = { 3 };\n    acl_int_array_ptr    dims_array = ggml_cann_create_int_array(norm_dims.data(), norm_dims.size());\n\n    float          p_value  = 2.0f;\n    acl_scalar_ptr p_scalar = ggml_cann_create_scalar(&p_value, aclDataType::ACL_FLOAT);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Norm, acl_src.get(), p_scalar.get(), dims_array.get(), true, acl_div.get());\n    GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src.get(), acl_div.get(), acl_dst.get());\n}\n\nvoid ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    const int64_t nc = src0->ne[0];\n    const int64_t nr = ggml_nrows(src0);\n\n    int64_t logits_ne[] = { nc, nr };\n    size_t  logits_nb[2];\n    logits_nb[0]              = ggml_type_size(src0->type);\n    logits_nb[1]              = logits_nb[0] * logits_ne[0];\n    acl_tensor_ptr acl_logits = ggml_cann_create_tensor(src0->data, ACL_FLOAT, sizeof(float), logits_ne, logits_nb, 2);\n\n    size_t               log_softmax_type_size = sizeof(float);\n    int64_t              log_softmax_n_bytes   = nr * nc * log_softmax_type_size;\n    ggml_cann_pool_alloc log_softmax_allocator(ctx.pool(), log_softmax_n_bytes);\n    void *               log_softmax_buffer = log_softmax_allocator.get();\n\n    int64_t log_softmax_ne[] = { nc, nr };\n    size_t  log_softmax_nb[2];\n    log_softmax_nb[0]              = log_softmax_type_size;\n    log_softmax_nb[1]              = log_softmax_nb[0] * log_softmax_ne[0];\n    acl_tensor_ptr acl_log_softmax = ggml_cann_create_tensor(log_softmax_buffer, ACL_FLOAT, log_softmax_type_size,\n                                                             log_softmax_ne, log_softmax_nb, 2);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, LogSoftmax, acl_logits.get(), 1, acl_log_softmax.get());\n\n    int64_t labels_ne[] = { nc, nr };\n    size_t  labels_nb[2];\n    labels_nb[0]              = ggml_type_size(src1->type);\n    labels_nb[1]              = labels_nb[0] * labels_ne[0];\n    acl_tensor_ptr acl_labels = ggml_cann_create_tensor(src1->data, ACL_FLOAT, sizeof(float), labels_ne, labels_nb, 2);\n\n    size_t               mul_type_size = sizeof(float);\n    int64_t              mul_n_bytes   = nr * nc * mul_type_size;\n    ggml_cann_pool_alloc mul_allocator(ctx.pool(), mul_n_bytes);\n    void *               mul_buffer = mul_allocator.get();\n\n    int64_t mul_ne[] = { nc, nr };\n    size_t  mul_nb[2];\n    mul_nb[0]                     = mul_type_size;\n    mul_nb[1]                     = mul_nb[0] * mul_ne[0];\n    acl_tensor_ptr acl_mul_result = ggml_cann_create_tensor(mul_buffer, ACL_FLOAT, mul_type_size, mul_ne, mul_nb, 2);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_log_softmax.get(), acl_labels.get(), acl_mul_result.get());\n\n    size_t               sum_per_sample_type_size = sizeof(float);\n    int64_t              sum_per_sample_n_bytes   = nr * sum_per_sample_type_size;\n    ggml_cann_pool_alloc sum_per_sample_allocator(ctx.pool(), sum_per_sample_n_bytes);\n    void *               sum_per_sample_buffer = sum_per_sample_allocator.get();\n\n    int64_t sum_per_sample_ne[] = { nr };\n    size_t  sum_per_sample_nb[1];\n    sum_per_sample_nb[0]              = sum_per_sample_type_size;\n    acl_tensor_ptr acl_sum_per_sample = ggml_cann_create_tensor(\n        sum_per_sample_buffer, ACL_FLOAT, sum_per_sample_type_size, sum_per_sample_ne, sum_per_sample_nb, 1);\n\n    std::vector<int64_t> sum_dims   = { 1 };\n    acl_int_array_ptr    dims_array = ggml_cann_create_int_array(sum_dims.data(), sum_dims.size());\n    bool                 keep_dims  = false;\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_mul_result.get(), dims_array.get(), keep_dims, ACL_FLOAT,\n                            acl_sum_per_sample.get());\n\n    size_t               total_sum_type_size = sizeof(float);\n    int64_t              total_sum_n_bytes   = 1 * total_sum_type_size;\n    ggml_cann_pool_alloc total_sum_allocator(ctx.pool(), total_sum_n_bytes);\n    void *               total_sum_buffer = total_sum_allocator.get();\n\n    int64_t total_sum_ne[] = { 1 };\n    size_t  total_sum_nb[1];\n    total_sum_nb[0] = total_sum_type_size;\n\n    acl_tensor_ptr acl_total_sum =\n        ggml_cann_create_tensor(total_sum_buffer, ACL_FLOAT, total_sum_type_size, total_sum_ne, total_sum_nb, 1);\n\n    std::vector<int64_t> total_sum_dims    = { 0 };\n    acl_int_array_ptr total_sum_dims_array = ggml_cann_create_int_array(total_sum_dims.data(), total_sum_dims.size());\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_sum_per_sample.get(), total_sum_dims_array.get(), keep_dims, ACL_FLOAT,\n                            acl_total_sum.get());\n\n    float          value        = -1.0f / static_cast<float>(nr);\n    acl_scalar_ptr scale_factor = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n    acl_tensor_ptr acl_dst =\n        ggml_cann_create_tensor(dst->data, ACL_FLOAT, sizeof(float), total_sum_ne, total_sum_nb, 1);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_total_sum.get(), scale_factor.get(), acl_dst.get());\n}\n\nvoid ggml_cann_group_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    int n_groups = dst->op_params[0];\n\n    float eps;\n    memcpy(&eps, dst->op_params + 1, sizeof(float));\n\n    int64_t N   = src->ne[3];\n    int64_t C   = src->ne[2];\n    int64_t HxW = src->ne[1] * src->ne[0];\n\n    size_t  type_size = ggml_type_size(src->type);\n    int64_t ne[]      = { n_groups, N };\n    size_t  nb[]      = { type_size, type_size * n_groups };\n    size_t  n_bytes   = N * n_groups;\n\n    ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes * 2);\n    void *               buffer       = temp_buffer_allocator.get();\n    acl_tensor_ptr       acl_mean_out = ggml_cann_create_tensor(buffer, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);\n    acl_tensor_ptr       acl_rstd_out =\n        ggml_cann_create_tensor((char *) buffer + n_bytes, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, GroupNorm, acl_src.get(), nullptr, nullptr, N, C, HxW, n_groups, eps, acl_dst.get(),\n                            acl_mean_out.get(), acl_rstd_out.get());\n}\n\nvoid ggml_cann_acc(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    size_t nb1     = ((int32_t *) dst->op_params)[0];\n    size_t nb2     = ((int32_t *) dst->op_params)[1];\n    size_t nb3     = ((int32_t *) dst->op_params)[2];\n    size_t offset  = ((int32_t *) dst->op_params)[3];\n    bool   inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n    size_t param_nb[] = { ggml_element_size(src0), nb1, nb2, nb3 };\n\n    acl_tensor_ptr acl_dst  = ggml_cann_create_tensor(dst, src1->ne, param_nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);\n    acl_tensor_ptr acl_src1 = ggml_cann_create_tensor(src1);\n\n    acl_scalar_ptr alpha      = nullptr;\n    float          alphaValue = 1.0f;\n    alpha                     = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n    if (!inplace) {\n        size_t cpy_size = ggml_nbytes(dst);\n        ACL_CHECK(\n            aclrtMemcpyAsync(dst->data, cpy_size, src0->data, cpy_size, ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n        acl_tensor_ptr acl_src0 =\n            ggml_cann_create_tensor(src0, src1->ne, src0->nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0.get(), acl_src1.get(), alpha.get(), acl_dst.get());\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), acl_src1.get(), alpha.get());\n    }\n}\n\n/**\n * @brief Performs sum reduction on a given tensor along specified dimensions.\n *\n * This function reduces the input tensor by summing along the specified dimensions.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the reduced result will be stored.\n * @param dim An array of dimension indices.\n * @param dim_size The number of dimensions.\n */\nstatic void aclnn_reduce_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst, int64_t * dim, size_t dim_size) {\n    GGML_ASSERT(dst->ne[0] == 1);\n    ggml_tensor *     src         = dst->src[0];\n    acl_tensor_ptr    acl_src     = ggml_cann_create_tensor(src);\n    acl_tensor_ptr    acl_dst     = ggml_cann_create_tensor(dst);\n    acl_int_array_ptr reduce_dims = ggml_cann_create_int_array(dim, dim_size);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_src.get(), reduce_dims.get(), true, ggml_cann_type_mapping(dst->type),\n                            acl_dst.get());\n}\n\nvoid ggml_cann_sum_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    int64_t reduce_dims[] = { 3 };\n    aclnn_reduce_sum(ctx, dst, reduce_dims, 1);\n}\n\nvoid ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    int64_t reduce_dims[] = { 0, 1, 2, 3 };\n    aclnn_reduce_sum(ctx, dst, reduce_dims, 4);\n}\n\nvoid ggml_cann_upsample_nearest2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor *  src     = dst->src[0];\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n    std::vector<int64_t> output_size{ dst->ne[1], dst->ne[0] };\n    acl_int_array_ptr    output_size_array = ggml_cann_create_int_array(output_size.data(), 2);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, UpsampleNearest2d, acl_src.get(), output_size_array.get(), acl_dst.get());\n}\n\n/**\n * @brief Pads a tensor with a specified value along each dimension.\n *\n * This function performs padding of the source tensor `acl_src` and stores the\n * result in the destination tensor `acl_dst`. The padding values for each\n * dimension are specified in the `paddings` array.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor to be padded.\n * @param acl_dst The destination tensor where the padded result will be stored.\n * @param paddings An array specifying the padding values for each dimension.\n * The size of the array should be twice the number of dimensions of the tensor.\n * @param value The value to be used for padding. The default value is 0.0.\n */\nstatic void aclnn_pad(ggml_backend_cann_context & ctx,\n                      aclTensor *                 acl_src,\n                      aclTensor *                 acl_dst,\n                      int64_t *                   paddings,\n                      float                       value = 0.0f) {\n    acl_int_array_ptr acl_pad   = ggml_cann_create_int_array(paddings, GGML_MAX_DIMS * 2);\n    acl_scalar_ptr    acl_value = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_src, acl_pad.get(), acl_value.get(), acl_dst);\n}\n\nvoid ggml_cann_pad(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor *  src     = dst->src[0];\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    // padding: value in the array means how much distance will be padding.\n    // the position of elements in the array means which dirction to padding,\n    // each position means: [dim0.front, dim0.behind, dim1.front, dim1.behind,\n    //                       dim2.front, dim2.behind, dim3.front, dim3.behind]\n    const int32_t lp0 = ggml_get_op_params_i32(dst, 0);\n    const int32_t rp0 = ggml_get_op_params_i32(dst, 1);\n    const int32_t lp1 = ggml_get_op_params_i32(dst, 2);\n    const int32_t rp1 = ggml_get_op_params_i32(dst, 3);\n    const int32_t lp2 = ggml_get_op_params_i32(dst, 4);\n    const int32_t rp2 = ggml_get_op_params_i32(dst, 5);\n    const int32_t lp3 = ggml_get_op_params_i32(dst, 6);\n    const int32_t rp3 = ggml_get_op_params_i32(dst, 7);\n\n    int64_t paddings[] = { lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3 };\n    aclnn_pad(ctx, acl_src.get(), acl_dst.get(), paddings);\n}\n\n/**\n * @brief Performs 2D average pooling on the input tensor and stores the result\n * in the destination tensor.\n *\n * This function performs average pooling on the source tensor and stores the\n * result in the destination tensor. The pooling parameters (kernel size,\n * strides, padding) are specified in the `op_params` of the destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result will be stored. The source\n * tensor is referenced by `dst->src[0]`.\n */\nstatic void ggml_cann_avg_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n    GGML_ASSERT(src->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n    const int32_t * opts = (const int32_t *) dst->op_params;\n    const int       k0   = opts[1];\n    const int       k1   = opts[2];\n    const int       s0   = opts[3];\n    const int       s1   = opts[4];\n    const int       p0   = opts[5];\n    const int       p1   = opts[6];\n\n    std::vector<int64_t> kernel_dims      = { k1, k0 };\n    std::vector<int64_t> stride_dims      = { s1, s0 };\n    std::vector<int64_t> padding_avg_dims = { p1, p0 };  // (padH, padW)\n\n    acl_int_array_ptr kernel_size  = ggml_cann_create_int_array(kernel_dims.data(), 2);\n    acl_int_array_ptr strides      = ggml_cann_create_int_array(stride_dims.data(), 2);\n    acl_int_array_ptr paddings_avg = ggml_cann_create_int_array(padding_avg_dims.data(), 2);\n\n    bool    ceil_mode         = false;\n    bool    count_include_pad = true;\n    int64_t divisor_override  = 0;\n    int8_t  cube_math_type    = 0;\n#ifdef ASCEND_310P\n    cube_math_type = 1;\n#endif\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, AvgPool2d, acl_src.get(), kernel_size.get(), strides.get(), paddings_avg.get(),\n                            ceil_mode, count_include_pad, divisor_override, cube_math_type, acl_dst.get());\n}\n\n/**\n * @brief Performs 2D max pooling on the input tensor and stores the result in\n * the destination tensor.\n *\n * This function performs max pooling on the source tensor and stores the result\n * in the destination tensor. The pooling parameters (kernel size, strides,\n * padding) are specified in the `op_params` of the destination tensor.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result will be stored. The source\n * tensor is referenced by `dst->src[0]`.\n */\nstatic void ggml_cann_max_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n    GGML_ASSERT(src->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);\n\n    const int32_t * opts = (const int32_t *) dst->op_params;\n    const int       k0   = opts[1];\n    const int       k1   = opts[2];\n    const int       s0   = opts[3];\n    const int       s1   = opts[4];\n    const int       p0   = opts[5];\n    const int       p1   = opts[6];\n\n    int64_t temp_ne[] = { src->ne[0] + p0 * 2, src->ne[1] + p1 * 2, src->ne[2], src->ne[3] };\n    size_t  temp_nb[GGML_MAX_DIMS];\n\n    temp_nb[0] = ggml_element_size(src);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        temp_nb[i] = temp_nb[i - 1] * temp_ne[i - 1];\n    }\n\n    ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), ggml_nbytes(src) + p0 * 2 + p1 * 2 * src->nb[1]);\n    void *               buffer = temp_buffer_allocator.get();\n    acl_tensor_ptr tmp_tensor   = ggml_cann_create_tensor(buffer, ACL_FLOAT, ggml_element_size(src), temp_ne, temp_nb,\n                                                          GGML_MAX_DIMS, ACL_FORMAT_NCHW);\n\n    // pad: see padding in ggml_cann_pad()\n    int64_t paddings[] = { p0, p0, p1, p1, 0, 0, 0, 0 };\n    float   value      = -FLT_MAX;\n    aclnn_pad(ctx, acl_src.get(), tmp_tensor.get(), paddings, value);\n\n    // max_pool\n    std::vector<int64_t> kernel_dims      = { k1, k0 };\n    std::vector<int64_t> stride_dims      = { s1, s0 };\n    // padding_max_dims: [dim0_start, dim0_end, dim1_start, dim1_end]\n    std::vector<int64_t> padding_max_dims = { 0, 0, 0, 0 };\n    std::vector<int64_t> dilation_size    = { 1, 1 };\n    acl_int_array_ptr    kernel_size      = ggml_cann_create_int_array(kernel_dims.data(), 2);\n    acl_int_array_ptr    strides          = ggml_cann_create_int_array(stride_dims.data(), 2);\n    acl_int_array_ptr    paddings_max     = ggml_cann_create_int_array(padding_max_dims.data(), 4);\n    acl_int_array_ptr    dilations        = ggml_cann_create_int_array(dilation_size.data(), 2);\n\n    bool    ceil_mode = false;\n    int64_t auto_pads = 0;\n    GGML_CANN_CALL_ACLNN_OP(ctx, MaxPool, tmp_tensor.get(), kernel_size.get(), strides.get(), auto_pads,\n                            paddings_max.get(), dilations.get(), ceil_mode, acl_dst.get());\n}\n\nvoid ggml_cann_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    const int32_t *   opts = (const int32_t *) dst->op_params;\n    enum ggml_op_pool op   = static_cast<ggml_op_pool>(opts[0]);\n    switch (op) {\n        case GGML_OP_POOL_AVG:\n            ggml_cann_avg_pool2d(ctx, dst);\n            break;\n        case GGML_OP_POOL_MAX:\n            ggml_cann_max_pool2d(ctx, dst);\n            break;\n        case GGML_OP_POOL_COUNT:\n            GGML_ABORT(\"fatal error\");\n            break;\n    }\n}\n\n/**\n * @brief Copies data from the source tensor to the destination tensor.\n *\n * This function copies data from the source tensor `acl_src` to the destination\n * tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor from which data will be copied.\n * @param acl_dst The destination tensor where the data will be copied to.\n */\nstatic void cann_copy(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst, acl_src);\n}\n\nvoid ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    if (ggml_are_same_shape(src0, dst)) {\n        acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n        acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n        if (dst->type == src0->type) {\n            cann_copy(ctx, acl_src.get(), acl_dst.get());\n        } else {\n            aclnn_cast(ctx, acl_src.get(), acl_dst.get(), ggml_cann_type_mapping(dst->type));\n        }\n    } else {\n        void *               src_trans_buffer = src0->data;\n        ggml_cann_pool_alloc src_buffer_allocator;\n        if (!ggml_is_contiguous(src0)) {\n            acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n            src_buffer_allocator.alloc(ctx.pool(), ggml_nelements(src0) * ggml_type_size(src0->type));\n            src_trans_buffer = src_buffer_allocator.get();\n            size_t src_trans_nb[GGML_MAX_DIMS];\n            src_trans_nb[0] = ggml_type_size(src0->type);\n            for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n            }\n            acl_tensor_ptr src_trans_tensor =\n                ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(src0->type),\n                                        ggml_type_size(src0->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n            cann_copy(ctx, acl_src.get(), src_trans_tensor.get());\n        }\n\n        size_t src_reshape_nb[GGML_MAX_DIMS];\n        src_reshape_nb[0] = ggml_type_size(src0->type);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            src_reshape_nb[i] = src_reshape_nb[i - 1] * dst->ne[i - 1];\n        }\n\n        acl_tensor_ptr trans_acl_src =\n            ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n                                    dst->ne, src_reshape_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n        acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n        if (dst->type == src0->type) {\n            cann_copy(ctx, trans_acl_src.get(), acl_dst.get());\n        } else {\n            aclnn_cast(ctx, trans_acl_src.get(), acl_dst.get(), ggml_cann_type_mapping(dst->type));\n        }\n    }\n}\n\n/**\n * @brief Creates an ACL tensor initialized with zeros using a provided buffer.\n *\n * This function initializes a tensor with zeros using the specified buffer and\n * tensor parameters.\n *\n * @param ctx The context for the CANN backend operations.\n * @param buffer The buffer to be used for the tensor data.\n * @param n_bytes The size of the buffer in bytes.\n * @param ne An array specifying the extents (sizes) of each dimension of the\n * tensor.\n * @param dims The number of dimensions of the tensor.\n * @param type The data type of the tensor.\n * @param type_size The size of each element in the tensor data type.\n * @return A tensor smart pointer initialized with zeros.\n */\nstatic acl_tensor_ptr aclnn_zero(ggml_backend_cann_context & ctx,\n                                 void *                      buffer,\n                                 size_t                      n_bytes,\n                                 int64_t *                   ne,\n                                 int64_t                     dims,\n                                 aclDataType                 type,\n                                 size_t                      type_size) {\n    size_t nb[GGML_MAX_DIMS];\n    nb[0] = type_size;\n    for (int i = 1; i < dims; i++) {\n        nb[i] = nb[i - 1] * ne[i - 1];\n    }\n\n    acl_tensor_ptr zero = ggml_cann_create_tensor(buffer, type, type_size, ne, nb, dims);\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, zero.get());\n    return zero;\n    GGML_UNUSED(n_bytes);\n}\n\n/**\n * @brief Creates an ACL tensor initialized with value using a provided buffer.\n *\n * This function initializes a tensor with value using the specified buffer and\n * tensor parameters.\n *\n * @param ctx The context for the CANN backend operations.\n * @param buffer The buffer to be used for the tensor data.\n * @param n_bytes The size of the buffer in bytes.\n * @param ne An array specifying the extents (sizes) of each dimension of the\n * tensor.\n * @param dims The number of dimensions of the tensor.\n * @param type The data type of the tensor.\n * @param type_size The size of each element in the tensor data type.\n * @param value The value to be used for initializing the tensor (default\n * is 1.0).\n * @return A tensor smart pointer initialized with value.\n */\nstatic acl_tensor_ptr aclnn_values(ggml_backend_cann_context & ctx,\n                                   void *                      buffer,\n                                   size_t                      n_bytes,\n                                   int64_t *                   ne,\n                                   int64_t                     dims,\n                                   aclDataType                 type,\n                                   size_t                      type_size,\n                                   float                       value = 1.0f) {\n    acl_tensor_ptr acl_tensor = aclnn_zero(ctx, buffer, n_bytes, ne, dims, type, type_size);\n    float          alpha_host = 1.0f;\n    acl_scalar_ptr alpha      = ggml_cann_create_scalar(&alpha_host, aclDataType::ACL_FLOAT);\n    acl_scalar_ptr other      = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_tensor.get(), other.get(), alpha.get());\n    return acl_tensor;\n}\n\n/**\n * @brief Fills a tensor with a scalar value.\n *\n * This function fills the destination tensor `acl_dst` with the scalar value\n * `scalar`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param scalar The scalar value used to fill the tensor.\n * @param acl_dst The destination tensor to be filled with the scalar value.\n */\nstatic void aclnn_fill_scalar(ggml_backend_cann_context & ctx, float scalar, aclTensor * acl_dst) {\n    acl_scalar_ptr acl_scalar = ggml_cann_create_scalar(&scalar, aclDataType::ACL_FLOAT);\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst, acl_scalar.get());\n}\n\n/**\n * @brief Get or expand a cached tensor filled with a scalar value.\n *\n * This function manages cached device memory for tensors. If the current\n * cache size is insufficient for the requested tensor shape, the old memory will\n * be released and new memory will be allocated. The allocated buffer is\n * initialized  with the given scalar value using CANN operations.\n * Finally, an aclTensor object is created from the cached memory and returned.\n *\n * @param ctx           The CANN backend context that manages device memory.\n * @param buffer        A pointer to the cached device buffer (will be allocated\n *                      or reallocated if necessary).\n * @param cache_element The current number of cached elements. This will be\n *                      updated when the cache is expanded.\n * @param ne            The tensor shape array (number of elements in each dimension).\n * @param nb            The stride size for each dimension.\n * @param dtype         Data type of cached tensor.\n * @param dims          The number of tensor dimensions.\n * @param value         The scalar value used to fill the tensor (supports zero\n *                      initialization via memset or arbitrary values via fill_scalar).\n * @return              A tensor smart pointer created from the cached buffer.\n */\nstatic acl_tensor_ptr get_cache_acl_tensor(ggml_backend_cann_context & ctx,\n                                           void **                     buffer,\n                                           int64_t &                   cache_element,\n                                           int64_t *                   ne,\n                                           size_t *                    nb,\n                                           ggml_type                   dtype,\n                                           int64_t                     dims,\n                                           float                       value) {\n    // Calculate total number of elements\n    int64_t n_element = 1;\n    for (int i = 0; i < dims; i++) {\n        n_element *= ne[i];\n    }\n    size_t size = n_element * ggml_type_size(dtype);\n\n    // Allocate or expand cache if needed\n    if (cache_element < n_element) {\n        if (*buffer != nullptr) {\n            aclrtFree(*buffer);\n            *buffer = nullptr;\n        }\n\n        ACL_CHECK(aclrtMalloc(buffer, size, ACL_MEM_MALLOC_HUGE_FIRST));\n        cache_element = n_element;\n\n        // Initialize cache\n        int64_t        pool_ne[1] = { n_element };\n        size_t         pool_nb[1] = { ggml_type_size(dtype) };\n        acl_tensor_ptr acl_value =\n            ggml_cann_create_tensor(*buffer, ggml_cann_type_mapping(dtype), ggml_type_size(dtype), pool_ne, pool_nb, 1);\n        aclnn_fill_scalar(ctx, value, acl_value.get());\n    }\n\n    return ggml_cann_create_tensor(*buffer, ggml_cann_type_mapping(dtype), ggml_type_size(dtype), ne, nb, dims);\n}\n\nvoid ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    // build gamma.\n    size_t acl_gamma_nb[GGML_MAX_DIMS];\n    // gamma's type is the same with dst.\n    acl_gamma_nb[0] = ggml_type_size(dst->type);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        acl_gamma_nb[i] = acl_gamma_nb[i - 1] * src->ne[i - 1];\n    }\n    acl_tensor_ptr acl_gamma = get_cache_acl_tensor(\n        ctx, &ctx.rms_norm_one_tensor_cache.cache, ctx.rms_norm_one_tensor_cache.size, src->ne, acl_gamma_nb, dst->type,\n        1,    // dims\n        1.0f  // value\n    );\n\n    // build rstd.\n    int64_t acl_rstd_ne[] = { src->ne[1], src->ne[2], src->ne[3] };\n    size_t  acl_rstd_nb[GGML_MAX_DIMS - 1];\n    // rstd will always be F32.\n    acl_rstd_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n        acl_rstd_nb[i] = acl_rstd_nb[i - 1] * acl_rstd_ne[i - 1];\n    }\n    acl_tensor_ptr acl_rstd =\n        get_cache_acl_tensor(ctx, &ctx.rms_norm_zero_tensor_cache.cache, ctx.rms_norm_zero_tensor_cache.size,\n                             acl_rstd_ne, acl_rstd_nb, GGML_TYPE_F32, GGML_MAX_DIMS - 1,\n                             0.0f  // value\n        );\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src.get(), acl_gamma.get(), eps, acl_dst.get(), acl_rstd.get());\n}\n\n// TODO: performace is low.\nvoid ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    const int n_past = ((int32_t *) dst->op_params)[0];\n\n    ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), ggml_nbytes(src));\n    void *               buffer = one_tensor_allocator.get();\n\n    acl_tensor_ptr mask_tensor = ggml_cann_create_tensor(buffer, ggml_cann_type_mapping(src->type),\n                                                         ggml_type_size(src->type), src->ne, src->nb, GGML_MAX_DIMS);\n\n    aclnn_fill_scalar(ctx, value, mask_tensor.get());\n\n    float          alphaValue = 1.0f;\n    acl_scalar_ptr alpha      = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor.get(), n_past + 1);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src.get(), n_past + 1, acl_dst.get());\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), mask_tensor.get(), alpha.get());\n}\n\n/**\n * @brief Permutes the dimensions of a tensor according to a specified order.\n *\n * This function permutes the dimensions of the source tensor `acl_src`\n * according to the order specified in the `new_dim` array and stores the result\n * in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose dimensions will be permuted.\n * @param acl_dst The destination tensor where the permuted result will be\n * stored.\n * @param new_dim An array specifying the new order of dimensions for the\n * tensor.\n * @param dims The number of dimensions in the tensor.\n */\nstatic void aclnn_permute(ggml_backend_cann_context & ctx,\n                          aclTensor *                 acl_src,\n                          aclTensor *                 acl_dst,\n                          int64_t *                   new_dim,\n                          uint64_t                    dims) {\n    acl_int_array_ptr acl_dims = ggml_cann_create_int_array(new_dim, dims);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Permute, acl_src, acl_dims.get(), acl_dst);\n}\n\nstatic void ggml_cann_im2col_2d_post_process(ggml_backend_cann_context & ctx,\n                                             ggml_tensor *               dst,\n                                             ggml_tensor *               src1,\n                                             aclTensor *                 tmp_cast_tensor,\n                                             aclTensor *                 tmp_im2col_tensor) {\n    // Permute: [N, IC * KH * KW, OW * OH] -> [N, OW * OH, IC * KH * KW]\n    int64_t        dst_ne[] = { dst->ne[0], dst->ne[1] * dst->ne[2], dst->ne[3] };\n    size_t         dst_nb[] = { dst->nb[0], dst->nb[1], dst->nb[3] };\n    acl_tensor_ptr acl_dst  = ggml_cann_create_tensor(dst, dst_ne, dst_nb, GGML_MAX_DIMS - 1);\n\n    int64_t permute_dim[] = { 0, 2, 1 };\n    if (src1->type != dst->type) {\n        aclnn_permute(ctx, tmp_cast_tensor, acl_dst.get(), permute_dim, 3);\n    } else {\n        aclnn_permute(ctx, tmp_im2col_tensor, acl_dst.get(), permute_dim, 3);\n    }\n}\n\nstatic void ggml_cann_im2col_1d_post_process(ggml_backend_cann_context &  ctx,\n                                             ggml_tensor *                dst,\n                                             ggml_tensor *                src1,\n                                             aclTensor *                  tmp_cast_tensor,\n                                             aclTensor *                  tmp_im2col_tensor,\n                                             const std::vector<int64_t> & im2col_op_params) {\n    // get params\n    const int64_t KH             = im2col_op_params[0];\n    const int64_t KW             = im2col_op_params[1];\n    const int64_t IW             = im2col_op_params[2];\n    const int64_t IC             = im2col_op_params[3];\n    const int64_t N              = im2col_op_params[4];\n    const int64_t OH             = im2col_op_params[5];\n    const int64_t OW             = im2col_op_params[6];\n    const int64_t s0             = im2col_op_params[7];\n    const int64_t p0             = im2col_op_params[8];\n    const int64_t d0             = im2col_op_params[9];\n    const int64_t n_bytes_factor = im2col_op_params[10];\n\n    // Permute: [N, IC * KH * KW, OW * OH] ->\n    // [N, OW * OH * n_bytes_factor, IC * KH * KW]\n    ggml_cann_pool_alloc tmp_permute_allocator(ctx.pool());\n    tmp_permute_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);\n    void * tmp_permute_buffer = tmp_permute_allocator.get();\n\n    int64_t tmp_permute_ne[] = { IC * KH * KW, OW * OH * n_bytes_factor, N };\n    size_t  tmp_permute_nb[GGML_MAX_DIMS - 1];\n    tmp_permute_nb[0] = ggml_type_size(dst->type);\n    for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n        tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];\n    }\n\n    acl_tensor_ptr tmp_permute_tensor =\n        ggml_cann_create_tensor(tmp_permute_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n                                tmp_permute_ne, tmp_permute_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n\n    int64_t permute_dim[] = { 0, 2, 1 };\n    if (src1->type != dst->type) {\n        aclnn_permute(ctx, tmp_cast_tensor, tmp_permute_tensor.get(), permute_dim, 3);\n    } else {\n        aclnn_permute(ctx, tmp_im2col_tensor, tmp_permute_tensor.get(), permute_dim, 3);\n    }\n\n    // number of times the kernel moves in W dimension\n    const int n_step_w = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1;\n    size_t    offset;\n    void *    cur_dst_buffer = dst->data, *cur_permute_buffer = tmp_permute_buffer;\n\n    // memory copy with offset to restore 1D im2col from 2d\n    if (IC > 1) {\n        offset          = IC * KH * KW * n_step_w * ggml_type_size(dst->type);\n        size_t cpy_size = KH * KW * ggml_type_size(dst->type);\n\n        for (int c = 0; c < IC; c++) {\n            cur_permute_buffer = (char *) tmp_permute_buffer + offset + KH * KW * c * ggml_type_size(dst->type);\n            cur_dst_buffer     = (char *) dst->data + c * KH * KW * n_step_w * ggml_type_size(dst->type);\n\n            for (int i = 0; i < n_step_w; i++) {\n                ACL_CHECK(aclrtMemcpyAsync(cur_dst_buffer, cpy_size, cur_permute_buffer, cpy_size,\n                                           ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n                cur_dst_buffer     = (char *) cur_dst_buffer + KH * KW * ggml_type_size(dst->type);\n                cur_permute_buffer = (char *) cur_permute_buffer + KH * KW * IC * ggml_type_size(dst->type);\n            }\n        }\n    } else {\n        offset = KH * KW * n_step_w * ggml_type_size(dst->type);  // equal to ggml_nbytes(dst)\n        ACL_CHECK(aclrtMemcpyAsync(dst->data, offset, (char *) tmp_permute_buffer + offset, offset,\n                                   ACL_MEMCPY_DEVICE_TO_DEVICE, ctx.stream()));\n    }\n}\n\nvoid ggml_cann_im2col(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // kernel\n    ggml_tensor * src1 = dst->src[1];  // input\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    // aclnnIm2col only works on 2D. set s1, p1, d1 to 1 to perform 2D\n    // im2col and do post-processing to restore it to 1D.\n    const bool    is_2D = ((const int32_t *) (dst->op_params))[6] == 1;\n    const int32_t s0    = ((const int32_t *) (dst->op_params))[0];\n    const int32_t s1    = is_2D ? ((const int32_t *) (dst->op_params))[1] : 1;\n    const int32_t p0    = ((const int32_t *) (dst->op_params))[2];\n    const int32_t p1    = is_2D ? ((const int32_t *) (dst->op_params))[3] : 1;\n    const int32_t d0    = ((const int32_t *) (dst->op_params))[4];\n    const int32_t d1    = is_2D ? ((const int32_t *) (dst->op_params))[5] : 1;\n\n    const int64_t N  = ne13;\n    const int64_t IC = ne12;\n    const int64_t KH = ne01;\n    const int64_t KW = ne00;\n    const int64_t IW = ne10;\n\n    const int64_t OH = is_2D ? ne2 : 1;\n    const int64_t OW = ne1;\n\n    // memory allocated increased to 3x when is_2D == false\n    const int64_t n_bytes_factor = is_2D ? 1 : 3;\n\n    // im2col: [N,C,H,W] -> [N, IC * KH * KW, OW * OH * n_bytes_factor]\n    acl_tensor_ptr acl_src1        = ggml_cann_create_tensor(src1);\n    int64_t        tmp_im2col_ne[] = { OW * OH * n_bytes_factor, IC * KH * KW, N };\n    size_t         tmp_im2col_nb[GGML_MAX_DIMS - 1];\n\n    tmp_im2col_nb[0] = ggml_type_size(src1->type);\n    for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n        tmp_im2col_nb[i] = tmp_im2col_nb[i - 1] * tmp_im2col_ne[i - 1];\n    }\n\n    // Calculate im2col.\n    // If dst is f16, tmp_buffer is f32, we need alloc src.typesize *\n    // dst.elemcount.\n    ggml_cann_pool_alloc im2col_allocator(ctx.pool(), ggml_nelements(dst) * ggml_element_size(src1) * n_bytes_factor);\n    void *               tmp_im2col_buffer = im2col_allocator.get();\n\n    acl_tensor_ptr tmp_im2col_tensor =\n        ggml_cann_create_tensor(tmp_im2col_buffer, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n                                tmp_im2col_ne, tmp_im2col_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n\n    std::vector<int64_t> kernel_dims   = { KH, KW };\n    std::vector<int64_t> dilation_size = { d1, d0 };\n    std::vector<int64_t> padding_dims  = { p1, p0 };\n    std::vector<int64_t> stride_dims   = { s1, s0 };\n    acl_int_array_ptr    kernel_size   = ggml_cann_create_int_array(kernel_dims.data(), 2);\n    acl_int_array_ptr    dilations     = ggml_cann_create_int_array(dilation_size.data(), 2);\n    acl_int_array_ptr    paddings      = ggml_cann_create_int_array(padding_dims.data(), 2);\n    acl_int_array_ptr    strides       = ggml_cann_create_int_array(stride_dims.data(), 2);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Im2col, acl_src1.get(), kernel_size.get(), dilations.get(), paddings.get(),\n                            strides.get(), tmp_im2col_tensor.get());\n\n    // Cast if dst is f16.\n    acl_tensor_ptr       tmp_cast_tensor;\n    ggml_cann_pool_alloc tmp_cast_allocator(ctx.pool());\n    void *               tmp_cast_buffer = nullptr;\n    if (src1->type != dst->type) {\n        tmp_cast_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);\n        tmp_cast_buffer = tmp_cast_allocator.get();\n        size_t temp_cast_nb[GGML_MAX_DIMS - 1];\n        temp_cast_nb[0] = ggml_type_size(dst->type);\n        for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n            temp_cast_nb[i] = temp_cast_nb[i - 1] * tmp_im2col_ne[i - 1];\n        }\n\n        tmp_cast_tensor =\n            ggml_cann_create_tensor(tmp_cast_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n                                    tmp_im2col_ne, temp_cast_nb, GGML_MAX_DIMS - 1, ACL_FORMAT_ND);\n        aclnn_cast(ctx, tmp_im2col_tensor.get(), tmp_cast_tensor.get(), ggml_cann_type_mapping(dst->type));\n    }\n\n    // post-processing\n    if (is_2D) {\n        ggml_cann_im2col_2d_post_process(ctx, dst, src1, tmp_cast_tensor.get(), tmp_im2col_tensor.get());\n    } else {\n        std::vector<int64_t> im2col_op_params = { KH, KW, IW, IC, N, OH, OW, s0, p0, d0, n_bytes_factor };\n        ggml_cann_im2col_1d_post_process(ctx, dst, src1, tmp_cast_tensor.get(), tmp_im2col_tensor.get(),\n                                         im2col_op_params);\n    }\n}\n\n/**\n * @brief Applies element-wise exponential function to the elements of a tensor.\n *\n * This function computes the exponential of each element in the source tensor\n * `acl_src` and stores the result back into the same tensor.\n * The operation is defined as:\n * \\f[\n *     \\text {acl_src }_i=e^{acl\\_src_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The tensor on which the exponential function will be applied.\n */\nstatic void aclnn_exp(ggml_backend_cann_context & ctx, aclTensor * acl_src) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceExp, acl_src);\n}\n\nvoid aclnn_cos(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n    if (acl_dst == nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCos, acl_src);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Cos, acl_src, acl_dst);\n    }\n}\n\nvoid aclnn_sin(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n    if (acl_dst == nullptr) {\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSin, acl_src);\n    } else {\n        GGML_CANN_CALL_ACLNN_OP(ctx, Sin, acl_src, acl_dst);\n    }\n}\n\nvoid ggml_cann_timestep_embedding(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src = dst->src[0];\n\n    GGML_ASSERT(src->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    const int dim        = dst->op_params[0];\n    const int max_period = dst->op_params[1];\n    int       half       = dim / 2;\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n\n    // arange: [0, ..., half)\n    float   start             = 0;\n    float   stop              = half;\n    float   step              = 1;\n    int64_t n_elements_arange = half;\n    int64_t tmp_arange_ne[]   = { half };\n    size_t  tmp_arange_nb[]   = { sizeof(dst->type) };\n\n    ggml_cann_pool_alloc arange_allocator(ctx.pool(), half * sizeof(dst->type));\n    void *               tmp_arange_buffer = arange_allocator.get();\n    acl_tensor_ptr       tmp_arange_tensor =\n        ggml_cann_create_tensor(tmp_arange_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n                                tmp_arange_ne, tmp_arange_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);\n\n    aclnn_arange(ctx, tmp_arange_tensor.get(), start, stop, step, n_elements_arange);\n\n    // freq\n    float freq_param = -logf(max_period) / half;\n    bool  inplace    = true;\n    aclnn_muls(ctx, tmp_arange_tensor.get(), freq_param, nullptr, inplace);\n    aclnn_exp(ctx, tmp_arange_tensor.get());\n\n    // permute: src [0,1,2,3]->[0,1,3,2]\n    int64_t tmp_permute_ne[] = { src->ne[1], src->ne[0], src->ne[2], src->ne[3] };\n    size_t  tmp_permute_nb[GGML_MAX_DIMS];\n    tmp_permute_nb[0] = ggml_type_size(src->type);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];\n    }\n\n    ggml_cann_pool_alloc permute_allocator(ctx.pool(), ggml_nbytes(src));\n    void *               tmp_permute_buffer = permute_allocator.get();\n    acl_tensor_ptr       tmp_permute_tensor =\n        ggml_cann_create_tensor(tmp_permute_buffer, ggml_cann_type_mapping(src->type), ggml_type_size(src->type),\n                                tmp_permute_ne, tmp_permute_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n    int64_t permute_dim[] = { 0, 1, 3, 2 };\n    int64_t num_dims      = 4;\n    aclnn_permute(ctx, acl_src.get(), tmp_permute_tensor.get(), permute_dim, num_dims);\n\n    // timestep * freq\n    int64_t tmp_mul_ne[] = { src->ne[1] * half, src->ne[0], src->ne[2], src->ne[3] };\n    size_t  tmp_mul_nb[GGML_MAX_DIMS];\n    tmp_mul_nb[0] = ggml_type_size(src->type);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        tmp_mul_nb[i] = tmp_mul_nb[i - 1] * tmp_mul_ne[i - 1];\n    }\n\n    int mul_nelements = src->ne[1] * half * src->ne[0] * src->ne[2] * src->ne[3];\n\n    ggml_cann_pool_alloc mul_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n    void *               tmp_mul_buffer = mul_allocator.get();\n    acl_tensor_ptr       tmp_mul_tensor =\n        ggml_cann_create_tensor(tmp_mul_buffer, ggml_cann_type_mapping(src->type), ggml_type_size(src->type),\n                                tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n    aclnn_mul(ctx, tmp_permute_tensor.get(), tmp_arange_tensor.get(), tmp_mul_tensor.get());\n\n    // cos\n    ggml_cann_pool_alloc cos_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n    void *               tmp_cos_buffer = cos_allocator.get();\n    acl_tensor_ptr       tmp_cos_tensor =\n        ggml_cann_create_tensor(tmp_cos_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n                                tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n\n    aclnn_cos(ctx, tmp_mul_tensor.get(), tmp_cos_tensor.get());\n\n    // sin\n    ggml_cann_pool_alloc sin_allocator(ctx.pool(), mul_nelements * ggml_type_size(src->type));\n    void *               tmp_sin_buffer = sin_allocator.get();\n    acl_tensor_ptr       tmp_sin_tensor =\n        ggml_cann_create_tensor(tmp_sin_buffer, ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),\n                                tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n\n    aclnn_sin(ctx, tmp_mul_tensor.get(), tmp_sin_tensor.get());\n\n    // concat\n    int64_t             concat_dim  = 3;\n    acl_tensor_ptr      acl_dst     = ggml_cann_create_tensor(dst);\n    acl_tensor_list_ptr tensor_list = ggml_cann_create_tensor_list(tmp_cos_tensor, tmp_sin_tensor);\n    aclnn_concat(ctx, tensor_list.get(), acl_dst.get(), concat_dim);\n}\n\n/**\n * @brief Raises each element of a tensor to the power of the corresponding\n * element in another tensor.\n *\n * This function computes the element-wise power of the destination tensor\n * `acl_dst` raised to the power of the exponent tensor `acl_exp`.\n * The operation is defined as:\n * \\f[\n *     \\text {acl_dst }_i=acl\\_dst_i^{\\text {acl_exp }_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_dst The destination tensor, which also serves as the base tensor.\n * @param acl_exp The exponent tensor, each element of which is used to raise\n * the corresponding element in the destination tensor.\n */\nstatic void aclnn_pow_tensor_tensor(ggml_backend_cann_context & ctx, aclTensor * acl_dst, aclTensor * acl_exp) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplacePowTensorTensor, acl_dst, acl_exp);\n}\n\n/**\n * @brief Generate a range of values and apply a scalar base exponentiation.\n *\n * This function creates an evenly spaced sequence from `start` to `stop` (exclusive),\n * with step size `step`, stores it in a temporary buffer, and then computes:\n *\n * @f[\n * slope[i] = m^{\\left( start + i \\cdot step \\right)}, \\quad 0 \\le i < size\n * @f]\n *\n * The results are written to the provided @p slope_buffer.\n *\n * @param ctx           CANN backend context for memory allocation and operator execution.\n * @param slope_buffer  Pointer to the output buffer (float array) for the computed slope values.\n * @param m             Scalar base for the exponentiation.\n * @param size          Number of elements in the generated sequence.\n * @param start         Starting exponent offset.\n * @param stop          Stopping exponent offset (exclusive).\n * @param step          Step size for the exponent increment.\n * @param dtype         Data type for slope tensor.\n */\nstatic void aclnn_get_slope_inner(ggml_backend_cann_context & ctx,\n                                  void *                      slope_buffer,\n                                  float                       m,\n                                  int64_t                     size,\n                                  float                       start,\n                                  float                       stop,\n                                  float                       step,\n                                  ggml_type                   dtype) {\n    aclDataType acl_type  = ggml_cann_type_mapping(dtype);\n    size_t      type_size = ggml_type_size(dtype);\n\n    int64_t ne[] = { size };\n    size_t  nb[] = { type_size };\n\n    ggml_cann_pool_alloc arange_allocator(ctx.pool(), size * type_size);\n    void *               arange_buffer = arange_allocator.get();\n\n    acl_tensor_ptr arange_tensor = ggml_cann_create_tensor(arange_buffer, acl_type, type_size, ne, nb, 1);\n    aclnn_arange(ctx, arange_tensor.get(), start, stop, step, size);\n\n    acl_tensor_ptr slope_tensor = ggml_cann_create_tensor(slope_buffer, acl_type, type_size, ne, nb, 1);\n\n    acl_scalar_ptr sc = ggml_cann_create_scalar(&m, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, PowScalarTensor, sc.get(), arange_tensor.get(), slope_tensor.get());\n}\n\n/**\n * @brief Compute slope values for multiple attention heads based on ALiBi bias parameters.\n *\n * This function generates slope values for each attention head according to the ALiBi\n * (Attention with Linear Biases) method. It splits the computation into two ranges depending\n * on whether the head index is less than @p n_head_log2 or not, and uses different base values\n * (`m0` and `m1`) for the exponentiation.\n *\n * @f[\n * slope[h] =\n * \\begin{cases}\n * m_0^{(h + 1)}, & h < n\\_head\\_log2 \\\\\n * m_1^{\\left( 2 \\cdot (h - n\\_head\\_log2) + 1 \\right)}, & h \\geq n\\_head\\_log2\n * \\end{cases}\n * \\quad , \\quad \\text{if } max\\_bias > 0\n * @f]\n *\n * If @p max_bias <= 0, all slope values are set to 1.0.\n *\n * @param ctx           CANN backend context for memory allocation and operator execution.\n * @param n_head        Total number of attention heads.\n * @param slope_buffer  Pointer to the output buffer (float array) for storing slopes.\n * @param max_bias      Maximum bias value for slope computation.\n * @param dtype         Data type for slope tensor.\n *\n*/\nstatic void aclnn_get_slope(ggml_backend_cann_context & ctx,\n                            int64_t                     n_head,\n                            void *                      slope_buffer,\n                            float                       max_bias,\n                            ggml_type                   dtype) {\n    const int n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n    float m0 = powf(2.0f, -(max_bias) / n_head_log2);\n    float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n    // const float slope = (max_bias > 0.0f) ?\n    //                          h < n_head_log2 ?\n    //                              powf(m0, h + 1) :\n    //                              powf(m1, 2*(h - n_head_log2) + 1) :\n    //                          1.0f;\n    // arange1\n    float start = 0 + 1;\n    float end   = (n_head_log2 - 1) + 1;\n    float step  = 1;\n    float count = n_head_log2;\n    // end needs to be +1 because aclnn uses a left-closed, right-open interval.\n    aclnn_get_slope_inner(ctx, slope_buffer, m0, count, start, end + 1, step, dtype);\n    if (n_head_log2 < n_head) {\n        // arange2\n        start = 2 * (n_head_log2 - n_head_log2) + 1;\n        end   = 2 * ((n_head - 1) - n_head_log2) + 1;\n        step  = 2;\n        count = n_head - n_head_log2;\n        aclnn_get_slope_inner(ctx, (char *) slope_buffer + n_head_log2 * sizeof(float), m1, count, start, end + 1, step,\n                              dtype);\n    }\n}\n\n/**\n * @brief Add ALiBi (Attention with Linear Biases) positional biases to the attention mask.\n *\n * This function computes the ALiBi slopes for each attention head (if max_bias > 0),\n * multiplies them with the attention mask to produce bias tensors, and adds these biases\n * to the destination tensor (@p dst).\n *\n * The function performs necessary broadcasting of the mask and slope tensors to match\n * the shape of the destination tensor, then applies element-wise multiplication and addition\n * using CANN operators.\n *\n * @param ctx         CANN backend context for memory management and operator execution.\n * @param mask        Input attention mask tensor, assumed to be contiguous.\n * @param dst         Destination tensor to which ALiBi biases will be added.\n * @param dst_ptr     Pointer to the memory of the destination tensor.\n * @param max_bias    Maximum bias value controlling the slope scaling.\n *\n * @note\n * - Write data into dst_ptr using only the shape information of the dst tensor.\n * - `GGML_MAX_DIMS + 2` is used to extend tensor dimensions for broadcasting.\n */\nstatic void aclnn_add_alibi(ggml_backend_cann_context & ctx,\n                            ggml_tensor *               mask,\n                            ggml_tensor *               dst,\n                            void *                      dst_ptr,\n                            float                       max_bias) {\n    void * slope_buffer = nullptr;\n    void * bias_buffer  = nullptr;\n\n    if (max_bias > 0.0f) {\n        int64_t              n_heads = dst->ne[2];\n        ggml_cann_pool_alloc slope_allocator(ctx.pool(), n_heads * sizeof(float));\n        slope_buffer = slope_allocator.get();\n        ggml_cann_pool_alloc bias_allocator(ctx.pool(), ggml_nelements(dst) * ggml_element_size(dst));\n        bias_buffer = bias_allocator.get();\n        aclnn_get_slope(ctx, n_heads, slope_buffer, max_bias, GGML_TYPE_F32);\n    }\n\n    // broadcast for mask, slop and dst;\n    int64_t nr2 = dst->ne[2] / mask->ne[2];\n    int64_t nr3 = dst->ne[3] / mask->ne[3];\n\n    // broadcast the mask across rows\n    int64_t mask_ne[] = { mask->ne[0], dst->ne[1], mask->ne[2], 1, mask->ne[3], 1 };\n    size_t  mask_nb[] = { mask_nb[0] = mask->nb[0], mask_nb[1] = mask->nb[1], mask_nb[2] = mask->nb[2],\n                          mask_nb[3] = mask->nb[2], mask_nb[4] = mask->nb[3], mask_nb[5] = mask->nb[3] };\n\n    int64_t dst_ne[] = { dst->ne[0], dst->ne[1], mask->ne[2], nr2, mask->ne[3], nr3 };\n    size_t  dst_nb[] = { dst_nb[0] = dst->nb[0], dst_nb[1] = dst->nb[1], dst_nb[2] = dst->nb[2],\n                         dst_nb[3] = dst->nb[2], dst_nb[4] = dst->nb[3], dst_nb[5] = dst->nb[3] };\n\n    // slope is a 1 dim tensor, slope.ne2 == dst.ne2\n    int64_t slope_ne[] = { 1, 1, mask->ne[2], nr2, 1, 1 };\n    size_t  slope_nb[GGML_MAX_DIMS + 2];\n    slope_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS + 2; i++) {\n        slope_nb[i] = slope_nb[i - 1] * slope_ne[i - 1];\n    }\n\n    acl_tensor_ptr acl_slope =\n        ggml_cann_create_tensor(slope_buffer, ACL_FLOAT, sizeof(float), slope_ne, slope_nb, GGML_MAX_DIMS + 2);\n    acl_tensor_ptr acl_mask = ggml_cann_create_tensor(mask, mask_ne, mask_nb, GGML_MAX_DIMS + 2);\n\n    // write data into dst_ptr using only the shape information of the dst tensor.\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst_ptr, ggml_cann_type_mapping(dst->type),\n                                                     ggml_type_size(dst->type), dst_ne, dst_nb, GGML_MAX_DIMS + 2);\n\n    if (max_bias > 0.0f) {\n        int64_t bias_ne[] = { mask->ne[0], dst->ne[1], mask->ne[2], nr2, mask->ne[3], 1 };\n        size_t  bias_nb[GGML_MAX_DIMS + 2];\n        bias_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS + 2; i++) {\n            bias_nb[i] = bias_nb[i - 1] * bias_ne[i - 1];\n        }\n        acl_tensor_ptr bias_tensor =\n            ggml_cann_create_tensor(bias_buffer, ACL_FLOAT, sizeof(float), bias_ne, bias_nb, GGML_MAX_DIMS + 2);\n\n        aclnn_mul(ctx, acl_slope.get(), acl_mask.get(), bias_tensor.get());\n        aclnn_add(ctx, acl_dst.get(), bias_tensor.get());\n    } else {\n        aclnn_add(ctx, acl_dst.get(), acl_mask.get());\n    }\n}\n\nvoid ggml_cann_cpy(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_cann_dup(ctx, dst);\n}\n\n/**\n * @brief Applies the softmax function to a tensor along a specified dimension.\n *\n * This function computes the softmax of the source tensor `acl_src` along the\n * specified dimension `dim` and stores the result in the destination tensor\n * `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the softmax function will be\n * applied.\n * @param dim The dimension along which the softmax function will be computed.\n * @param acl_dst The destination tensor where the softmax results will be\n * stored.\n */\nstatic void aclnn_softmax(ggml_backend_cann_context & ctx, aclTensor * acl_src, int64_t dim, aclTensor * acl_dst) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, Softmax, acl_src, dim, acl_dst);\n}\n\nvoid ggml_cann_softmax(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];  // mask\n\n    acl_tensor_ptr acl_src0 = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst  = ggml_cann_create_tensor(dst);\n\n    float scale    = 1.0f;\n    float max_bias = 0.0f;\n\n    memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n\n    // input mul scale\n    acl_scalar_ptr       acl_scale = ggml_cann_create_scalar(&scale, aclDataType::ACL_FLOAT);\n    ggml_cann_pool_alloc src_tensor_allocator(ctx.pool(), ggml_nbytes(src0));\n    void *               src_tensor_buffer = src_tensor_allocator.get();\n    acl_tensor_ptr       softmax_tensor = ggml_cann_create_tensor(src_tensor_buffer, ggml_cann_type_mapping(src0->type),\n                                                                  ggml_element_size(src0), src0->ne, src0->nb, GGML_MAX_DIMS);\n\n    aclnn_muls(ctx, acl_src0.get(), scale, softmax_tensor.get(), false);\n\n    // mask\n    if (src1) {\n        aclnn_add_alibi(ctx, src1, src0, src_tensor_buffer, max_bias);\n    }\n    // softmax\n    aclnn_softmax(ctx, softmax_tensor.get(), 3, acl_dst.get());\n}\n\n/**\n * @brief Performs index select operation on a 4D tensor using the CANN backend.\n *\n * This function applies the `IndexSelect` operation along a specific dimension\n * of the source tensor (`src_buffer`) using the indices from the index tensor (`index`).\n * It iterates over the last two dimensions of the source tensor, creates the corresponding\n * CANN tensors for the source, index, and output slices, and executes the `IndexSelect`\n * operation for each slice.\n *\n * @param ctx The context for CANN backend operations.\n * @param src_buffer The source buffer containing the 4D input tensor data.\n * @param src_ne The dimensions of the source tensor.\n * @param src_nb The strides (byte offsets) of the source tensor.\n * @param dst_buffer The destination buffer where the output tensor data will be written.\n * @param dst_ne The dimensions of the destination tensor.\n * @param dst_nb The strides (byte offsets) of the destination tensor.\n * @param index The index tensor specifying the indices to select from the source tensor.\n * @param type The data type of the source and destination tensors.\n */\nstatic void aclnn_index_select_4d(ggml_backend_cann_context & ctx,\n                                  void *                      src_buffer,\n                                  int64_t *                   src_ne,\n                                  size_t *                    src_nb,\n                                  void *                      dst_buffer,\n                                  int64_t *                   dst_ne,\n                                  size_t *                    dst_nb,\n                                  ggml_tensor *               index,\n                                  ggml_type                   type) {\n    for (int64_t i = 0; i < src_ne[3]; i++) {\n        for (int64_t j = 0; j < src_ne[2]; j++) {\n            // src\n            acl_tensor_ptr acl_src_tensor =\n                ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2],\n                                        ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2);\n\n            // index\n            acl_tensor_ptr acl_index = ggml_cann_create_tensor(\n                (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1],\n                ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1);\n\n            // out\n            acl_tensor_ptr acl_out =\n                ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2],\n                                        ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2);\n            GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, acl_src_tensor.get(), 0, acl_index.get(), acl_out.get());\n        }\n    }\n}\n\n/**\n * @brief Performs inplace index copy operation on a 4D tensor using the CANN backend.\n *\n * This function applies the `IndexCopy` operation along a specific dimension of the\n * destination tensor (`dst_buffer`) by copying elements from the source tensor (`src_buffer`)\n * to positions specified by the index tensor (`index`).\n * It iterates over the last two dimensions of the tensors, creates the corresponding\n * CANN tensors for source, index, and destination slices, and performs the index copy\n * operation for each slice.\n *\n * @param ctx The context for CANN backend operations.\n * @param src_buffer The source buffer containing the 4D input tensor data to be copied.\n * @param src_ne The dimensions of the source tensor.\n * @param src_nb The strides (byte offsets) of the source tensor.\n * @param dst_buffer The destination buffer where values will be copied to.\n * @param dst_ne The dimensions of the destination tensor.\n * @param dst_nb The strides (byte offsets) of the destination tensor.\n * @param index The index tensor specifying target positions in the destination tensor.\n * @param type The data type of the source and destination tensors.\n */\nstatic void aclnn_index_copy_4d(ggml_backend_cann_context & ctx,\n                                void *                      src_buffer,\n                                int64_t *                   src_ne,\n                                size_t *                    src_nb,\n                                void *                      dst_buffer,\n                                int64_t *                   dst_ne,\n                                size_t *                    dst_nb,\n                                ggml_tensor *               index,\n                                ggml_type                   type) {\n    for (int64_t i = 0; i < src_ne[3]; i++) {\n        for (int64_t j = 0; j < src_ne[2]; j++) {\n            // src\n            acl_tensor_ptr acl_src_tensor =\n                ggml_cann_create_tensor((char *) src_buffer + i * src_nb[3] + j * src_nb[2],\n                                        ggml_cann_type_mapping(type), ggml_type_size(type), src_ne, src_nb, 2);\n\n            // index\n            acl_tensor_ptr acl_index = ggml_cann_create_tensor(\n                (char *) index->data + (i % index->ne[2]) * index->nb[2] + (j % index->ne[1]) * index->nb[1],\n                ggml_cann_type_mapping(index->type), ggml_element_size(index), index->ne, index->nb, 1);\n\n            // out\n            acl_tensor_ptr acl_out =\n                ggml_cann_create_tensor((char *) dst_buffer + i * dst_nb[3] + j * dst_nb[2],\n                                        ggml_cann_type_mapping(type), ggml_type_size(type), dst_ne, dst_nb, 2);\n            GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexCopy, acl_out.get(), 0, acl_index.get(), acl_src_tensor.get());\n        }\n    }\n}\n\nvoid ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // src\n    ggml_tensor * src1 = dst->src[1];  // index\n\n    GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n        case GGML_TYPE_F32:\n            if (src0->type == dst->type) {\n                aclnn_index_select_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1,\n                                      dst->type);\n            } else {\n                acl_tensor_ptr       acl_src0 = ggml_cann_create_tensor(src0);\n                ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * ggml_element_size(dst));\n                void *               src_trans_buffer = src_buffer_allocator.get();\n                size_t               src_trans_nb[GGML_MAX_DIMS];\n                src_trans_nb[0] = dst->nb[0];\n                for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                    src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n                }\n                acl_tensor_ptr src_trans_tensor =\n                    ggml_cann_create_tensor(src_trans_buffer, ggml_cann_type_mapping(dst->type),\n                                            ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n                aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type));\n                aclnn_index_select_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1,\n                                      dst->type);\n            }\n            break;\n        case GGML_TYPE_Q8_0:\n            {\n                // add 1 dim for bcast mul.\n                size_t  weight_nb[GGML_MAX_DIMS + 1], scale_nb[GGML_MAX_DIMS + 1], dequant_nb[GGML_MAX_DIMS + 1];\n                int64_t weight_ne[GGML_MAX_DIMS + 1], scale_ne[GGML_MAX_DIMS + 1], *dequant_ne;\n                int64_t scale_offset = 0;\n                // [3,4,5,64] -> [3,4,5,2,32]\n                weight_ne[0]         = QK8_0;\n                weight_ne[1]         = src0->ne[0] / QK8_0;\n                weight_nb[0]         = sizeof(int8_t);\n                weight_nb[1]         = weight_nb[0] * weight_ne[0];\n                for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {\n                    weight_ne[i] = src0->ne[i - 1];\n                    weight_nb[i] = weight_nb[i - 1] * weight_ne[i - 1];\n                }\n                // [3,4,5,64] -> [3,4,5,2,1]\n                scale_ne[0] = 1;\n                scale_ne[1] = src0->ne[0] / QK8_0;\n                scale_nb[0] = sizeof(uint16_t);\n                scale_nb[1] = scale_nb[0] * scale_ne[0];\n                for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {\n                    scale_ne[i] = src0->ne[i - 1];\n                    scale_nb[i] = scale_nb[i - 1] * scale_ne[i - 1];\n                }\n                // [3,4,5,64] -> [3,4,5,2,32]\n                dequant_ne    = weight_ne;\n                dequant_nb[0] = ggml_type_size(dst->type);\n                for (int i = 1; i < GGML_MAX_DIMS + 1; i++) {\n                    dequant_nb[i] = dequant_nb[i - 1] * dequant_ne[i - 1];\n                }\n                scale_offset = ggml_nelements(src0) * sizeof(int8_t);\n                ggml_cann_pool_alloc dequant_buffer_allocator(ctx.pool(),\n                                                              ggml_nelements(src0) * ggml_type_size(dst->type));\n                acl_tensor_ptr       acl_weight_tensor = ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t),\n                                                                                 weight_ne, weight_nb, GGML_MAX_DIMS + 1);\n                acl_tensor_ptr       acl_scale_tensor =\n                    ggml_cann_create_tensor(src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb,\n                                            GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset);\n                acl_tensor_ptr dequant_tensor =\n                    ggml_cann_create_tensor(dequant_buffer_allocator.get(), ggml_cann_type_mapping(dst->type),\n                                            ggml_type_size(dst->type), dequant_ne, dequant_nb, GGML_MAX_DIMS + 1);\n                aclnn_mul(ctx, acl_weight_tensor.get(), acl_scale_tensor.get(), dequant_tensor.get());\n                dequant_nb[0] = ggml_type_size(dst->type);\n                dequant_ne    = src0->ne;\n                for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                    dequant_nb[i] = dequant_nb[i - 1] * src0->ne[i - 1];\n                }\n                aclnn_index_select_4d(ctx, dequant_buffer_allocator.get(), dequant_ne, dequant_nb, dst->data, dst->ne,\n                                      dst->nb, src1, dst->type);\n                break;\n            }\n        default:\n            GGML_ABORT(\"Unsupported tensor type for GGML_OP_GET_ROWS\");\n            break;\n    }\n}\n\nvoid ggml_cann_set_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // src\n    ggml_tensor * src1 = dst->src[1];  // index\n\n    switch (dst->type) {\n        case GGML_TYPE_F32:\n            {\n                aclnn_index_copy_4d(ctx, src0->data, src0->ne, src0->nb, dst->data, dst->ne, dst->nb, src1, dst->type);\n                break;\n            }\n        case GGML_TYPE_F16:\n            {\n                acl_tensor_ptr       acl_src0 = ggml_cann_create_tensor(src0);\n                ggml_cann_pool_alloc src_buffer_allocator(ctx.pool(), ggml_nelements(src0) * sizeof(uint16_t));\n                void *               src_trans_buffer = src_buffer_allocator.get();\n                size_t               src_trans_nb[GGML_MAX_DIMS];\n                src_trans_nb[0] = sizeof(uint16_t);\n                for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                    src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];\n                }\n                acl_tensor_ptr src_trans_tensor = ggml_cann_create_tensor(\n                    src_trans_buffer, ACL_FLOAT16, ggml_type_size(dst->type), src0->ne, src_trans_nb, GGML_MAX_DIMS);\n                aclnn_cast(ctx, acl_src0.get(), src_trans_tensor.get(), ggml_cann_type_mapping(dst->type));\n                aclnn_index_copy_4d(ctx, src_trans_buffer, src0->ne, src_trans_nb, dst->data, dst->ne, dst->nb, src1,\n                                    dst->type);\n                break;\n            }\n        default:\n            GGML_ABORT(\"Unsupported tensor type for GGML_OP_SET_ROWS\");\n            break;\n    }\n}\n\n/**\n * @brief Repeats elements of a tensor along a specified dimension.\n *\n * This function repeats each element of the source tensor `acl_src` a specified\n * number of times (`repeats`) along the specified dimension `dim` and stores\n * the result in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be repeated.\n * @param acl_dst The destination tensor where the repeated elements will be\n * stored.\n * @param dim The dimension along which the elements will be repeated.\n * @param repeats The number of times each element will be repeated.\n * @param output_size The size of the output tensor.\n */\nstatic void aclnn_repeat_interleave(ggml_backend_cann_context & ctx,\n                                    aclTensor *                 acl_src,\n                                    aclTensor *                 acl_dst,\n                                    int64_t                     dim,\n                                    int64_t                     repeats,\n                                    int64_t                     output_size) {\n    GGML_CANN_CALL_ACLNN_OP(ctx, RepeatInterleaveIntWithDim, acl_src, repeats, dim, output_size, acl_dst);\n}\n\n/**\n * @brief Performs matrix multiplication with floating-point precision on\n * tensors using the CANN backend.\n *\n * This function performs matrix multiplication of the input tensor and the\n * weight tensor, handling broadcasting and transposing as needed, and stores\n * the result in the destination tensor `dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result of the matrix\n * multiplication will be stored.\n */\nstatic void ggml_cann_mat_mul_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * weight = dst->src[0];  // weight\n    ggml_tensor * input  = dst->src[1];  // input\n\n    // when weight ne2 or ne3 is 1, aclnnMatmulGetWorkspaceSize will auto\n    // broadcast, when weight ne2 or ne3 is not 1, weight need repeat.\n    BCAST_MUL_MAT_SHAPE(input, weight, dst);\n\n    int64_t n_dims = bcast_dims;\n    if (bcast_input_ne[3] == bcast_weight_ne[3] && bcast_input_ne[3] == 1) {\n        if (bcast_input_ne[2] == 1 && bcast_weight_ne[2] == 1) {\n            n_dims = 2;\n        } else if (bcast_input_ne[2] == 1) {\n            n_dims = 3;\n        }\n    }\n\n    acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(input, bcast_input_ne, bcast_input_nb, n_dims);\n    int64_t        transpose_ne[]   = { bcast_weight_ne[1], bcast_weight_ne[0], bcast_weight_ne[2],\n                                        bcast_weight_ne[3], bcast_weight_ne[4], bcast_weight_ne[5] };\n    size_t         transpose_nb[]   = { bcast_weight_nb[1], bcast_weight_nb[0], bcast_weight_nb[2],\n                                        bcast_weight_nb[3], bcast_weight_nb[4], bcast_weight_nb[5] };\n    acl_tensor_ptr acl_weight_tensor;\n\n    // Only check env once.\n    static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n    if (weight_to_nz && is_matmul_weight(weight)) {\n        acl_weight_tensor = ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims, ACL_FORMAT_FRACTAL_NZ);\n    } else {\n        acl_weight_tensor = ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims, ACL_FORMAT_ND);\n    }\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, bcast_dst_ne, bcast_dst_nb, n_dims);\n\n    switch (n_dims) {\n        case 2:\n            GGML_CANN_CALL_ACLNN_OP(ctx, Mm, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(), 2);\n            break;\n        case 3:\n            GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(),\n                                    2);\n            break;\n        default:\n            // ALLOW_FP32_DOWN_PRECISION, when input is\n            // fp32, atlas a2 will transpose it to HFLOAT32.\n            GGML_CANN_CALL_ACLNN_OP(ctx, Matmul, acl_input_tensor.get(), acl_weight_tensor.get(), acl_dst.get(), 1);\n            break;\n    }\n}\n\n/**\n * @brief Performs matrix multiplication with quantized weights and\n * floating-point inputs using the CANN backend.\n *\n * This function performs matrix multiplication of the input tensor `src1` and\n * the weight tensor `src0`, handling broadcasting, transposing, and\n * quantization as needed, and stores the result in the destination tensor\n * `dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The destination tensor where the result of the matrix\n * multiplication will be stored.\n */\nstatic void ggml_cann_mul_mat_quant(ggml_backend_cann_context & ctx, ggml_tensor * dst, const enum ggml_type type) {\n    ggml_tensor * src0 = dst->src[0];  // weight\n    ggml_tensor * src1 = dst->src[1];  // input\n\n    // The shape of the weight is NCHW.\n    // Matrix multiplication uses HW dims.\n    // HC is regarded as batch.\n    // weight need transpose.\n    float weight_elem_size;\n    if (type == GGML_TYPE_Q4_0) {\n        weight_elem_size = float(sizeof(uint8_t)) / 2;\n    } else if (type == GGML_TYPE_Q8_0) {\n        weight_elem_size = float(sizeof(uint8_t));\n    } else {\n        GGML_ABORT(\"Only support Q4_0 and Q8_0 MUL_MAT\");\n    }\n    float  weight_nb[]   = { src0->ne[0] * weight_elem_size, weight_elem_size };\n    size_t weight_stride = src0->ne[1] * src0->ne[0] * weight_elem_size;\n    size_t weight_size   = weight_stride * src0->ne[2] * src0->ne[3];\n\n    // scale stored at the end of weight. Also need transpose.\n    size_t scale_elem_size = sizeof(uint16_t);\n    size_t scale_nb[]      = { src0->ne[0] / QK8_0 * scale_elem_size, scale_elem_size };\n    size_t scale_stride    = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;\n    char * scale_offset    = (char *) src0->data + weight_size;\n\n    // input\n    size_t               input_elem_size = sizeof(uint16_t);\n    int64_t              input_ne[]      = { src1->ne[0], src1->ne[1] };\n    size_t               input_nb[]      = { input_elem_size, input_ne[0] * input_elem_size };\n    size_t               input_stride    = input_ne[0] * input_ne[1] * input_elem_size;\n    ggml_cann_pool_alloc input_alloctor(ctx.pool());\n    void *               input_buffer = src1->data;\n\n    // case in\n    if (src1->type != GGML_TYPE_F16) {\n        acl_tensor_ptr acl_src1_tensor = ggml_cann_create_tensor(src1);\n        input_buffer                   = input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);\n\n        int64_t * input_cast_ne = src1->ne;\n        size_t    input_cast_nb[GGML_MAX_DIMS];\n        input_cast_nb[0] = sizeof(uint16_t);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            input_cast_nb[i] = input_cast_nb[i - 1] * input_cast_ne[i - 1];\n        }\n\n        acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(input_buffer, ACL_FLOAT16, input_elem_size,\n                                                                  input_cast_ne, input_cast_nb, GGML_MAX_DIMS);\n        aclnn_cast(ctx, acl_src1_tensor.get(), acl_input_tensor.get(), ACL_FLOAT16);\n    }\n\n    // output\n    size_t               output_elem_size = sizeof(uint16_t);\n    size_t               output_nb[]      = { output_elem_size, dst->ne[0] * output_elem_size };\n    ggml_cann_pool_alloc output_allocator(ctx.pool());\n    void *               output_buffer = output_allocator.alloc(ggml_nelements(dst) * output_elem_size);\n    size_t               output_stride = dst->ne[0] * dst->ne[1] * output_elem_size;\n\n    // aclnn\n    int64_t              max_elem_size = 65535;\n    int64_t              split_size    = (src0->ne[1] / max_elem_size) + 1;\n    ggml_cann_pool_alloc workspace_allocator(ctx.pool());\n    for (int64_t n1 = 0; n1 < src1->ne[3]; n1++) {\n        for (int64_t c1 = 0; c1 < src1->ne[2]; c1++) {\n            int64_t n0 = n1 / (src1->ne[3] / src0->ne[3]);\n            int64_t c0 = c1 / (src1->ne[2] / src0->ne[2]);\n\n            int64_t batch1 = (n1 * src1->ne[2]) + c1;\n            int64_t batch0 = (n0 * src0->ne[2]) + c0;\n\n            acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(\n                (char *) input_buffer + batch1 * input_stride, ACL_FLOAT16, input_elem_size, input_ne, input_nb, 2);\n\n            // first split\n            int64_t weight_ne_offset = 0;\n            int64_t weight_ne[2]     = { max_elem_size > src0->ne[1] ? src0->ne[1] : max_elem_size, src0->ne[0] };\n            int64_t scale_ne_offset  = 0;\n            int64_t scale_ne[2]      = { weight_ne[0], weight_ne[1] / QK8_0 };\n            int64_t output_ne_offset = 0;\n            int64_t output_ne[2]     = { weight_ne[0], dst->ne[1] };\n\n            acl_tensor_ptr acl_weight_tensor =\n                ggml_cann_create_tensor((char *) src0->data + batch0 * weight_stride, ggml_cann_type_mapping(type),\n                                        weight_elem_size, weight_ne, weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);\n            acl_tensor_ptr acl_scale_tensor =\n                ggml_cann_create_tensor(scale_offset + batch0 * scale_stride, ACL_FLOAT16, scale_elem_size, scale_ne,\n                                        scale_nb, 2, ACL_FORMAT_ND, scale_ne_offset);\n            acl_tensor_ptr acl_output_tensor =\n                ggml_cann_create_tensor((char *) output_buffer + batch1 * output_stride, ACL_FLOAT16, output_elem_size,\n                                        output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset);\n            int64_t antiquantGroupSize = 0;\n            if (src0->ne[0] > QK8_0) {\n                antiquantGroupSize = QK8_0;\n            }\n            GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor.get(), acl_weight_tensor.get(),\n                                    acl_scale_tensor.get(), nullptr, nullptr, nullptr, nullptr, antiquantGroupSize,\n                                    acl_output_tensor.get());\n\n            // other splits\n            for (int64_t split = 1; split < split_size; split++) {\n                weight_ne_offset += weight_elem_size * weight_ne[0] * weight_ne[1];\n                weight_ne[0] =\n                    max_elem_size * (split + 1) > src0->ne[1] ? src0->ne[1] - (max_elem_size * split) : max_elem_size;\n                scale_ne_offset += scale_elem_size * scale_ne[0] * scale_ne[1];\n                scale_ne[0] = weight_ne[0];\n                output_ne_offset += output_elem_size * output_ne[0] * output_ne[1];\n                output_ne[0] = weight_ne[0];\n\n                acl_weight_tensor =\n                    ggml_cann_create_tensor((char *) src0->data + batch0 * weight_stride, ggml_cann_type_mapping(type),\n                                            weight_elem_size, weight_ne, weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);\n                acl_scale_tensor =\n                    ggml_cann_create_tensor(scale_offset + batch0 * scale_stride, ACL_FLOAT16, scale_elem_size,\n                                            scale_ne, scale_nb, 2, ACL_FORMAT_ND, scale_ne_offset);\n                acl_output_tensor =\n                    ggml_cann_create_tensor((char *) output_buffer + batch1 * output_stride, ACL_FLOAT16,\n                                            output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND, output_ne_offset);\n                GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor.get(), acl_weight_tensor.get(),\n                                        acl_scale_tensor.get(), nullptr, nullptr, nullptr, nullptr, antiquantGroupSize,\n                                        acl_output_tensor.get());\n            }\n        }\n    }\n\n    // cast out\n    if (dst->type != GGML_TYPE_F16) {\n        int64_t * output_cast_ne = dst->ne;\n        size_t    output_cast_nb[GGML_MAX_DIMS];\n        output_cast_nb[0] = sizeof(uint16_t);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];\n        }\n\n        acl_tensor_ptr acl_output_tensor = ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, output_elem_size,\n                                                                   output_cast_ne, output_cast_nb, GGML_MAX_DIMS);\n        acl_tensor_ptr acl_dst_tensor    = ggml_cann_create_tensor(dst);\n        aclnn_cast(ctx, acl_output_tensor.get(), acl_dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n    }\n}\n\nvoid ggml_cann_mul_mat(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    const enum ggml_type type = dst->src[0]->type;\n    switch (type) {\n        case GGML_TYPE_F32:\n        case GGML_TYPE_F16:\n            ggml_cann_mat_mul_fp(ctx, dst);\n            break;\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q8_0:\n            ggml_cann_mul_mat_quant(ctx, dst, type);\n            break;\n        default:\n            GGML_ABORT(\"Unsupported type for mul_mat\");\n            break;\n    }\n}\n\n/**\n * @brief Rolls the elements of a tensor along a specified dimension.\n *\n * This function rolls the elements of the source tensor `acl_src` by the\n * specified shifts `shifts` along the specified dimensions `dims`, and stores\n * the result in the destination tensor `acl_dst`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor whose elements will be rolled.\n * @param acl_dst The destination tensor where the rolled elements will be\n * stored.\n * @param shifts An array specifying the number of positions by which elements\n * are shifted.\n * @param dims An array specifying the dimensions along which elements are\n * shifted.\n */\nstatic void aclnn_roll(ggml_backend_cann_context & ctx,\n                       aclTensor *                 acl_src,\n                       aclTensor *                 acl_dst,\n                       int64_t *                   shifts,\n                       int64_t *                   dims) {\n    acl_int_array_ptr acl_shifts = ggml_cann_create_int_array(shifts, 1);\n    acl_int_array_ptr acl_dims   = ggml_cann_create_int_array(dims, 1);\n    GGML_CANN_CALL_ACLNN_OP(ctx, Roll, acl_src, acl_shifts.get(), acl_dims.get(), acl_dst);\n}\n\n/**\n * @brief Fills specified positions of a tensor with a scalar value.\n *\n * This function fills the positions in the source tensor `acl_src` specified by\n * `index` along the dimension `dim` with the scalar value `value`.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor where the positions will be filled.\n * @param dim The dimension along which the positions are specified.\n * @param index An array specifying the positions to be filled.\n * @param index_num The number of positions specified in the index array.\n * @param value The scalar value used to fill the specified positions.\n */\nstatic void aclnn_index_fill_tensor(ggml_backend_cann_context & ctx,\n                                    aclTensor *                 acl_src,\n                                    int64_t                     dim,\n                                    int64_t *                   index,\n                                    int64_t                     index_num,\n                                    float                       value) {\n    acl_int_array_ptr acl_index = ggml_cann_create_int_array(index, index_num);\n    acl_scalar_ptr    acl_value = ggml_cann_create_scalar(&value, aclDataType::ACL_FLOAT);\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexFillTensor, acl_src, dim, acl_index.get(), acl_value.get());\n}\n\n/**\n * @brief Initializes and caches all intermediate tensors required for RoPE\n *        (Rotary Position Embedding), including support for Yarn, mRoPE,\n *        i-mRoPE, Neox repeat strategy, independent sectors, frequency factors，\n *        and multi-section rotary groups.\n *\n * This function computes and caches the per-dimension θ coefficients used for\n * Q/K rotary embedding. The cache is shared across layers, and recomputed only\n * when any dependent parameter changes.\n *\n * The function now supports:\n *   - Yarn RoPE extrapolation (via @param corr_dims and @param ext_factor)\n *   - Per-dimension independent sector exponent rules (indep_sects + sections[])\n *   - Multi-section RoPE (mRoPE) index mapping (mrope_used + is_imrope)\n *   - Frequency factor division (src2)\n *   - Neox / normal repeat expansion modes\n *\n * @param ctx                CANN backend context, containing memory pool,\n *                           cached buffers, and runtime stream.\n * @param dst                Destination ggml_tensor whose computation\n *                           depends on RoPE (typically Qcur or Kcur).\n * @param corr_dims          [low, high] Yarn correction range.\n * @param ext_factor         Yarn extrapolation strength. 0 = disabled.\n * @param theta_scale        Base multiplier for per-dimension θ exponent.\n * @param freq_scale         Global frequency scaling factor.\n * @param attn_factor        Optional scaling applied to sin/cos (if needed).\n * @param is_neox            Whether to use Neox-style dimension interleave.\n * @param sections           4-way sector sizes for independent-section RoPE\n *                           and multi-section mRoPE (t/h/w/e).\n * @param mrope_used         Whether to enable multi-section rotary embedding.\n * @param is_imrope          Whether to apply interleaved mRoPE rules.\n * @param indep_sects        Whether each dimension runs independent exponent\n *                           resets based on @p sections.\n */\nstatic void aclnn_rope_cache_init(ggml_backend_cann_context & ctx,\n                                  ggml_tensor *               dst,\n                                  float *                     corr_dims,\n                                  float                       ext_factor,\n                                  float                       theta_scale,\n                                  float                       freq_scale,\n                                  float                       attn_factor,\n                                  bool                        is_neox,\n                                  int                         sections[4],\n                                  bool                        mrope_used,\n                                  bool                        is_imrope,\n                                  bool                        indep_sects,\n                                  int64_t                     rope_dims) {\n    ggml_tensor * src1 = dst->src[1];  // position\n    ggml_tensor * src2 = dst->src[2];  // freq_factors\n\n    int64_t theta_scale_length = rope_dims / 2;\n    int64_t position_length    = dst->ne[2];\n\n    // TODO: check theta_scale_length and position_length.\n    if (src2 == nullptr && ctx.rope_cache.cached &&\n        ctx.rope_cache.equal(theta_scale_length, position_length, ext_factor, theta_scale, freq_scale, attn_factor,\n                             is_neox, indep_sects, mrope_used, is_imrope, sections)) {\n        // use cache.\n        return;\n    }\n\n    // Step0: calculate tensor shape.\n    int64_t theta_scale_ne[] = { theta_scale_length, 1, 1, 1 };\n    size_t  theta_scale_nb[] = { sizeof(float), theta_scale_length * sizeof(float), theta_scale_length * sizeof(float),\n                                 theta_scale_length * sizeof(float) };\n\n    GGML_ASSERT(src1->type == GGML_TYPE_I32);\n    int64_t position_ne[] = { 1, 1, position_length, 1 };\n    size_t  position_nb[] = { sizeof(int32_t), sizeof(int32_t), sizeof(int32_t), sizeof(int32_t) * position_length };\n\n    int64_t cache_ne[] = { theta_scale_length, 1, position_length, 1 };\n    size_t  cache_nb[GGML_MAX_DIMS];\n    cache_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        cache_nb[i] = cache_nb[i - 1] * cache_ne[i - 1];\n    }\n\n    // Step1: Compute the coefficient of theta. During the cache_init process, aside from\n    // (1) multiplying by the position,\n    // (2) dividing by freq_factors,\n    // (3) computing the sine and cosine,\n    // the other parameters used in the computation generally do not change in most scenarios.\n    // Therefore, we can first compute this part of the result and then cache it.\n\n    // Step1.1: prepare theta_scale exponent. if this exponent updated, should update theta_scale_tensor.\n    acl_tensor_ptr acl_theta_scale_tensor;\n    bool           theta_scale_updated = false;\n    if (ctx.rope_cache.theta_scale_length != theta_scale_length || ctx.rope_cache.theta_scale != theta_scale ||\n        ctx.rope_cache.indep_sects != indep_sects) {\n        theta_scale_updated = true;\n        if (ctx.rope_cache.theta_scale_exp_host != nullptr) {\n            free(ctx.rope_cache.theta_scale_exp_host);\n        }\n        ctx.rope_cache.theta_scale_exp_host = (float *) malloc(theta_scale_length * sizeof(float));\n        GGML_ASSERT(ctx.rope_cache.theta_scale_exp_host != nullptr);\n        if (!indep_sects) {\n            ctx.rope_cache.theta_scale_exp_host[0] = 1;\n            for (int i = 1; i < theta_scale_length; i++) {\n                ctx.rope_cache.theta_scale_exp_host[i] = ctx.rope_cache.theta_scale_exp_host[i - 1] * theta_scale;\n            }\n        } else {\n            int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n            int sec_w     = sections[1] + sections[0];\n            int sec_e     = sections[2] + sec_w;\n\n            ctx.rope_cache.theta_scale_exp_host[0] = 1;\n            for (int i = 1; i < theta_scale_length; i++) {\n                int sector = i % sect_dims;\n                if (sector == 0 || sector == sections[0] || sector == sec_w || sector == sec_e) {\n                    ctx.rope_cache.theta_scale_exp_host[i] = 1;\n                    continue;\n                }\n                ctx.rope_cache.theta_scale_exp_host[i] = ctx.rope_cache.theta_scale_exp_host[i - 1] * theta_scale;\n            }\n        }\n\n        if (ctx.rope_cache.theta_scale_cache != nullptr) {\n            ACL_CHECK(aclrtFree(ctx.rope_cache.theta_scale_cache));\n        }\n        ACL_CHECK(aclrtMalloc(&ctx.rope_cache.theta_scale_cache, theta_scale_length * sizeof(float),\n                              ACL_MEM_MALLOC_HUGE_FIRST));\n\n        ACL_CHECK(aclrtMemcpyAsync(ctx.rope_cache.theta_scale_cache, theta_scale_length * sizeof(float),\n                                   ctx.rope_cache.theta_scale_exp_host, theta_scale_length * sizeof(float),\n                                   ACL_MEMCPY_HOST_TO_DEVICE, ctx.stream()));\n    }\n    acl_theta_scale_tensor = ggml_cann_create_tensor(ctx.rope_cache.theta_scale_cache, ACL_FLOAT, sizeof(float),\n                                                     theta_scale_ne, theta_scale_nb, 1);\n\n    // Step1.2: prepare rope_yarn_ramp, if this part updated, should update theta_scale_tensor.\n    // TODO: acl_yarn_ramp_tensor use rope cache.\n    bool           yarn_ramp_tensor_updated = false;\n    acl_tensor_ptr acl_yarn_ramp_tensor;\n    if (ext_factor != 0 && (theta_scale_updated || ctx.rope_cache.theta_scale_length != theta_scale_length ||\n                            ctx.rope_cache.freq_scale != freq_scale)) {\n        yarn_ramp_tensor_updated = true;\n        if (ctx.rope_cache.yarn_ramp_cache != nullptr) {\n            ACL_CHECK(aclrtFree(ctx.rope_cache.yarn_ramp_cache));\n        }\n        ACL_CHECK(aclrtMalloc(&ctx.rope_cache.yarn_ramp_cache, theta_scale_length * sizeof(float),\n                              ACL_MEM_MALLOC_HUGE_FIRST));\n        // -rope_yarn_ramp\n        // const float y = (i0 / 2 - low) / MAX(0.001f, high - low);\n        // return MIN(1, MAX(0, y)) - 1;\n        acl_yarn_ramp_tensor      = ggml_cann_create_tensor(ctx.rope_cache.yarn_ramp_cache, ACL_FLOAT, sizeof(float),\n                                                            theta_scale_ne, theta_scale_nb, 1);\n        float          zero_value = 0, one_value = 1;\n        float          denom_safe_value = MAX(0.001f, corr_dims[1] - corr_dims[0]);\n        acl_scalar_ptr low              = ggml_cann_create_scalar(&corr_dims[0], aclDataType::ACL_FLOAT);\n        acl_scalar_ptr zero             = ggml_cann_create_scalar(&zero_value, aclDataType::ACL_FLOAT);\n        acl_scalar_ptr one              = ggml_cann_create_scalar(&one_value, aclDataType::ACL_FLOAT);\n        acl_scalar_ptr denom_safe       = ggml_cann_create_scalar(&denom_safe_value, aclDataType::ACL_FLOAT);\n        acl_scalar_ptr ext_factor_sc    = ggml_cann_create_scalar(&ext_factor, aclDataType::ACL_FLOAT);\n\n        aclnn_arange(ctx, acl_yarn_ramp_tensor.get(), 0, theta_scale_length, 1, theta_scale_length);\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSubs, acl_yarn_ramp_tensor.get(), low.get(), one.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDivs, acl_yarn_ramp_tensor.get(), denom_safe.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceThreshold, acl_yarn_ramp_tensor.get(), zero.get(), zero.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceClampMax, acl_yarn_ramp_tensor.get(), one.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSubs, acl_yarn_ramp_tensor.get(), one.get(), one.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_yarn_ramp_tensor.get(), ext_factor_sc.get());\n\n        // theta_interp = freq_scale * theta_extrap;\n        // theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n        // theta = freq_scale * theta_extrap * (1 - ramp_mix) + theta_extrap * ramp_mix;\n        // theta = freq_scale * theta_extrap - freq_scale * theta_extrap * ramp_mix + theta_extrap * ramp_mix;\n        // theta = theta_extrap * (freq_scale - freq_scale * ramp_mix + ramp_mix);\n        //\n        // we cache (freq_scale - freq_scale * ramp_mix + ramp_mix), Considering that the rope_yarn_ramp here is the inverse\n        // cache freq_scale + (freq_scale - 1) * ramp_mix\n        float          freq_scale_1    = freq_scale - 1;\n        acl_scalar_ptr freq_scale_sc   = ggml_cann_create_scalar(&freq_scale, aclDataType::ACL_FLOAT);\n        acl_scalar_ptr freq_scale_1_sc = ggml_cann_create_scalar(&freq_scale_1, aclDataType::ACL_FLOAT);\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_yarn_ramp_tensor.get(), freq_scale_1_sc.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_yarn_ramp_tensor.get(), freq_scale_sc.get(), one.get());\n    } else {\n        acl_yarn_ramp_tensor = ggml_cann_create_tensor(ctx.rope_cache.yarn_ramp_cache, ACL_FLOAT, sizeof(float),\n                                                       theta_scale_ne, theta_scale_nb, 1);\n    }\n    // Step 1.3: update theta_scale_tensor according to ext_factor or freq_scale.\n    if (ext_factor != 0) {\n        if (theta_scale_updated || yarn_ramp_tensor_updated) {\n            theta_scale_updated = true;\n            aclnn_mul(ctx, acl_theta_scale_tensor.get(), acl_yarn_ramp_tensor.get());\n        }\n    } else {\n        if (freq_scale != 1 && (ctx.rope_cache.freq_scale != freq_scale || theta_scale_updated)) {\n            theta_scale_updated = true;\n            aclnn_muls(ctx, acl_theta_scale_tensor.get(), freq_scale, nullptr, true);\n        }\n    }\n\n    // Nothing changed, use cache.\n    if (!theta_scale_updated) {\n        acl_theta_scale_tensor = ggml_cann_create_tensor(ctx.rope_cache.theta_scale_cache, ACL_FLOAT, sizeof(float),\n                                                         theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n    }\n\n    // Step 1.4: prepare select index if mrope\n    acl_tensor_ptr position_select_index_tensor;\n    if (mrope_used) {\n        if (ctx.rope_cache.sections[0] != sections[0] || ctx.rope_cache.sections[1] != sections[1] ||\n            ctx.rope_cache.sections[2] != sections[2] || ctx.rope_cache.sections[3] != sections[3] ||\n            ctx.rope_cache.theta_scale_length != theta_scale_length || ctx.rope_cache.is_imrope != is_imrope) {\n            if (ctx.rope_cache.position_select_index_host != nullptr) {\n                free(ctx.rope_cache.position_select_index_host);\n            }\n            ctx.rope_cache.position_select_index_host = (int *) malloc(theta_scale_length * sizeof(int));\n            GGML_ASSERT(ctx.rope_cache.position_select_index_host != nullptr);\n            int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n            int sec_w     = sections[1] + sections[0];\n            int sec_e     = sections[2] + sec_w;\n            // t,h,w,e\n            for (int i = 0; i < theta_scale_length; i++) {\n                int sector = i % sect_dims;\n\n                if (is_imrope) {  // qwen3vl apply interleaved mrope\n                    if (sector % 3 == 1 && sector < 3 * sections[1]) {\n                        ctx.rope_cache.position_select_index_host[i] = 1;\n                    } else if (sector % 3 == 2 && sector < 3 * sections[2]) {\n                        ctx.rope_cache.position_select_index_host[i] = 2;\n                    } else if (sector % 3 == 0 && sector < 3 * sections[0]) {\n                        ctx.rope_cache.position_select_index_host[i] = 0;\n                    } else {\n                        ctx.rope_cache.position_select_index_host[i] = 3;\n                    }\n                } else {\n                    if (sector >= sections[0] && sector < sec_w) {\n                        ctx.rope_cache.position_select_index_host[i] = 1;\n                    } else if (sector >= sec_w && sector < sec_e) {\n                        ctx.rope_cache.position_select_index_host[i] = 2;\n                    } else if (sector >= sec_e) {\n                        ctx.rope_cache.position_select_index_host[i] = 3;\n                    } else {\n                        ctx.rope_cache.position_select_index_host[i] = 0;\n                    }\n                }\n            }\n\n            if (ctx.rope_cache.position_select_index != nullptr) {\n                ACL_CHECK(aclrtFree(ctx.rope_cache.position_select_index));\n            }\n            ACL_CHECK(aclrtMalloc(&ctx.rope_cache.position_select_index, theta_scale_length * sizeof(int),\n                                  ACL_MEM_MALLOC_HUGE_FIRST));\n\n            ACL_CHECK(aclrtMemcpyAsync(ctx.rope_cache.position_select_index, theta_scale_length * sizeof(int),\n                                       ctx.rope_cache.position_select_index_host, theta_scale_length * sizeof(int),\n                                       ACL_MEMCPY_HOST_TO_DEVICE, ctx.stream()));\n        }\n\n        position_select_index_tensor = ggml_cann_create_tensor(ctx.rope_cache.position_select_index, ACL_INT32,\n                                                               sizeof(int), theta_scale_ne, theta_scale_nb, 1);\n    }\n\n    // Step2: divide by freq_factors\n    ggml_cann_pool_alloc freq_fac_res_allocator(ctx.pool());\n    if (src2) {\n        freq_fac_res_allocator.alloc(theta_scale_length * sizeof(float));\n        void *         freq_fac_res_ptr = freq_fac_res_allocator.get();\n        acl_tensor_ptr acl_freq_factors_tensor =\n            ggml_cann_create_tensor(src2->data, ggml_cann_type_mapping(src2->type), ggml_type_size(src2->type),\n                                    theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n        acl_tensor_ptr acl_freq_fac_res_tensor = ggml_cann_create_tensor(freq_fac_res_ptr, ACL_FLOAT, sizeof(float),\n                                                                         theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);\n        aclnn_div(ctx, acl_theta_scale_tensor.get(), acl_freq_factors_tensor.get(), acl_freq_fac_res_tensor.get());\n        std::swap(acl_theta_scale_tensor, acl_freq_fac_res_tensor);\n    }\n\n    // Step3: prepare position_tensor\n    acl_tensor_ptr       acl_position_tensor;\n    ggml_cann_pool_alloc mrope_position_acllocator(ctx.pool());\n    if (mrope_used) {\n        // Step3.1: select current position;\n        // position :\n        // pos1: [[0, 1 ,2 ,3 ],\n        // pos2:  [4, 5 ,6 ,7 ],\n        // pos3:  [8, 9 ,10,11],\n        // pos4:  [12,13,14,15] ]\n        //\n        // select index = [0, 1, 2, 2, 1, 0]\n        //\n        // selected_tensor:\n        // [[0, 1 ,2 ,3 ],\n        //  [4, 5 ,6 ,7 ],\n        //  [8, 9 ,10,11],\n        //  [8, 9 ,10,11],\n        //  [4, 5 ,6 ,7 ],\n        //  [0, 1 ,2 ,3 ]]\n        //\n        // transpose, from [seq_len:dims] to [dims:seq_len]\n        // [0, 4, 8 ,8 ,4, 0],\n        // [1, 5, 9, 9, 5, 1],\n        // [2, 6, 10,10,6 ,2],\n        // [3, 7, 11,11,7 3 ]]\n        //\n        // multipy by theta_scale_tensor\n        // [theta_scale^0, theta_scale^1, ..., theta_scale ^ n]\n\n        int64_t        mrope_position_ne[] = { position_length, 4 };\n        size_t         mrope_position_nb[] = { sizeof(int), position_length * sizeof(int) };\n        acl_tensor_ptr mrope_position =\n            ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n                                    mrope_position_ne, mrope_position_nb, 2);\n\n        // selected position tensor's shape is a transpose of cache tensor.\n        int64_t selected_position_ne[] = { position_length, theta_scale_length };\n        size_t  selected_position_nb[] = { sizeof(float), position_length * sizeof(float) };\n        mrope_position_acllocator.alloc(theta_scale_length * position_length * sizeof(float));\n        void * mrope_position_buffer = mrope_position_acllocator.get();\n        acl_position_tensor =\n            ggml_cann_create_tensor(mrope_position_buffer, ggml_cann_type_mapping(src1->type),\n                                    ggml_type_size(src1->type), selected_position_ne, selected_position_nb, 2);\n        GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, mrope_position.get(), 0, position_select_index_tensor.get(),\n                                acl_position_tensor.get());\n\n        // transpose\n        int64_t transposed_ne[] = { position_length, 1, theta_scale_length, 1 };\n        size_t  transposed_nb[GGML_MAX_DIMS];\n        transposed_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            transposed_nb[i] = transposed_nb[i - 1] * transposed_ne[i - 1];\n        }\n\n        std::swap(transposed_ne[0], transposed_ne[2]);\n        std::swap(transposed_nb[0], transposed_nb[2]);\n\n        acl_position_tensor =\n            ggml_cann_create_tensor(mrope_position_buffer, ggml_cann_type_mapping(src1->type),\n                                    ggml_type_size(src1->type), transposed_ne, transposed_nb, GGML_MAX_DIMS);\n\n    } else {\n        // auto bcast.\n        acl_position_tensor =\n            ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type), ggml_type_size(src1->type),\n                                    position_ne, position_nb, GGML_MAX_DIMS);\n    }\n\n    // Step4: multiply by the position\n    int64_t              theta_length = theta_scale_length * position_length;\n    ggml_cann_pool_alloc theta_allocator(ctx.pool(), theta_length * sizeof(float));\n    void *               theta_buffer = theta_allocator.get();\n\n    acl_tensor_ptr acl_theta_tensor =\n        ggml_cann_create_tensor(theta_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS);\n    aclnn_mul(ctx, acl_position_tensor.get(), acl_theta_scale_tensor.get(), acl_theta_tensor.get());\n\n    // Step5: calculate sin cos.\n    // init sin_repeat && cos_repeat, only to accelerate first layer on each device\n    if (position_length > ctx.rope_cache.position_length) {\n        ctx.rope_cache.position_length = position_length;\n        if (ctx.rope_cache.sin_cache != nullptr) {\n            ACL_CHECK(aclrtFree(ctx.rope_cache.sin_cache));\n        }\n        if (ctx.rope_cache.cos_cache != nullptr) {\n            ACL_CHECK(aclrtFree(ctx.rope_cache.cos_cache));\n        }\n        int64_t repeat_theta_length = theta_scale_length * position_length * 2;\n        ACL_CHECK(\n            aclrtMalloc(&ctx.rope_cache.sin_cache, repeat_theta_length * sizeof(float), ACL_MEM_MALLOC_HUGE_FIRST));\n        ACL_CHECK(\n            aclrtMalloc(&ctx.rope_cache.cos_cache, repeat_theta_length * sizeof(float), ACL_MEM_MALLOC_HUGE_FIRST));\n    }\n\n    // sin/cos\n    ggml_cann_pool_alloc sin_allocator(ctx.pool(), theta_length * sizeof(float));\n    void *               sin_buffer = sin_allocator.get();\n    acl_tensor_ptr       acl_sin_tensor =\n        ggml_cann_create_tensor(sin_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n    aclnn_sin(ctx, acl_theta_tensor.get(), acl_sin_tensor.get());\n\n    ggml_cann_pool_alloc cos_allocator(ctx.pool(), theta_length * sizeof(float));\n    void *               cos_buffer = cos_allocator.get();\n    acl_tensor_ptr       acl_cos_tensor =\n        ggml_cann_create_tensor(cos_buffer, ACL_FLOAT, sizeof(float), cache_ne, cache_nb, GGML_MAX_DIMS, ACL_FORMAT_ND);\n    aclnn_cos(ctx, acl_theta_tensor.get(), acl_cos_tensor.get());\n\n    if (ext_factor != 0) {\n        attn_factor *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n    }\n\n    // Step 5: multiply by attn_factor\n    if (attn_factor != 1) {\n        aclnn_muls(ctx, acl_sin_tensor.get(), attn_factor, nullptr, true);\n        aclnn_muls(ctx, acl_cos_tensor.get(), attn_factor, nullptr, true);\n    }\n\n    int64_t sin_reshape_ne[4] = { rope_dims, 1, dst->ne[2], 1 };\n    size_t  sin_reshape_nb[GGML_MAX_DIMS];\n    sin_reshape_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];\n    }\n    acl_tensor_ptr acl_sin_repeat_tensor = ggml_cann_create_tensor(ctx.rope_cache.sin_cache, ACL_FLOAT, sizeof(float),\n                                                                   sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n    acl_tensor_ptr acl_cos_repeat_tensor = ggml_cann_create_tensor(ctx.rope_cache.cos_cache, ACL_FLOAT, sizeof(float),\n                                                                   sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n\n    // Step 6: repeat\n    if (is_neox) {\n        // [sinθ1, sinθ1, sinθ2, sinθ2, ..., sinθn, sinθn]\n        int64_t repeatsArray[] = { 1, 1, 1, 2 };\n        aclnn_repeat(ctx, acl_sin_tensor.get(), acl_sin_repeat_tensor.get(), repeatsArray);\n        aclnn_repeat(ctx, acl_cos_tensor.get(), acl_cos_repeat_tensor.get(), repeatsArray);\n    } else {\n        int64_t num_repeats = 2;\n        int64_t dim         = 3;\n        int64_t output_size = theta_scale_length * num_repeats;\n        // [sinθ1, sinθ2, ..., sinθn, sinθ1, sinθ2, ..., sinθn]\n        aclnn_repeat_interleave(ctx, acl_sin_tensor.get(), acl_sin_repeat_tensor.get(), dim, num_repeats, output_size);\n        aclnn_repeat_interleave(ctx, acl_cos_tensor.get(), acl_cos_repeat_tensor.get(), dim, num_repeats, output_size);\n    }\n\n    // Update cached value.\n    ctx.rope_cache.cached = true;\n    ctx.rope_cache.set(theta_scale_length, position_length, ext_factor, theta_scale, freq_scale, attn_factor, is_neox,\n                       indep_sects, mrope_used, is_imrope, sections);\n}\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\naclnnStatus aclnnRotaryPositionEmbeddingGetWorkspaceSize(const aclTensor * x,\n                                                         const aclTensor * cos,\n                                                         const aclTensor * sin,\n                                                         int64_t           mode,\n                                                         const aclTensor * yOut,\n                                                         uint64_t *        workspaceSize,\n                                                         aclOpExecutor **  executor);\naclnnStatus aclnnRotaryPositionEmbedding(void *          workspace,\n                                         uint64_t        workspaceSize,\n                                         aclOpExecutor * executor,\n                                         aclrtStream     stream);\n#ifdef __cplusplus\n}\n#endif\n\nvoid ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // input\n\n    // param\n    float     freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;\n    int       sections[4];\n    // const int n_past     = ((int32_t *) dst->op_params)[0];\n    const int n_dims     = ((int32_t *) dst->op_params)[1];\n    const int mode       = ((int32_t *) dst->op_params)[2];\n    // const int n_ctx      = ((int32_t *) dst->op_params)[3];\n    const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));\n    memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));\n    memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));\n    memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));\n    memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));\n    memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));\n    memcpy(&sections, (int32_t *) dst->op_params + 11, sizeof(int) * 4);\n\n    GGML_ASSERT(n_dims % 2 == 0);\n    GGML_ASSERT(n_dims <= ne00);\n\n    const float theta_scale = powf(freq_base, -2.0f / n_dims);\n\n    float corr_dims[2];\n    ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);\n\n    bool       is_neox    = mode & GGML_ROPE_TYPE_NEOX;\n    const bool is_imrope  = mode == GGML_ROPE_TYPE_IMROPE;  // qwen3vl apply interleaved mrope\n    // mrope_used means the GGML_ROPE_TYPE_MROPE bit is set.\n    // Note: this bit is also set for imrope and some vision modes,\n    // so mrope_used does NOT exclusively indicate pure mrope.\n    const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE;\n    const bool is_vision  = mode == GGML_ROPE_TYPE_VISION;\n\n    if (mrope_used) {\n        GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);\n    }\n\n    if (is_vision) {\n        GGML_ASSERT(n_dims == ne0 / 2);\n    }\n\n    if (is_imrope || mrope_used) {\n        is_neox = true;\n    }\n\n    int64_t rope_dims = n_dims;\n\n    //Our current RotaryPositionEmbedding does not support the VISION mode,\n    //but essentially it only modifies theta_base in mrope,\n    //then repeats it at the end in the same way as is_neox.\n    //In fact, RoPE is still applied across all dimensions.\n    if (is_vision) {\n        rope_dims = src0->ne[0];\n    }\n    int64_t tail_dims = ne00 - rope_dims;\n    bool    has_tail  = tail_dims > 0;\n\n    // init ctx.rope_cos/rope_sin cache\n    aclnn_rope_cache_init(ctx, dst, corr_dims, ext_factor, theta_scale, freq_scale, attn_factor, is_neox, sections,\n                          mrope_used, is_imrope, is_vision, rope_dims);\n\n    // Cache is generated with ne00 dimensions, so we use ne00 for reshape\n    int64_t sin_reshape_ne[4] = { rope_dims, 1, ne02, 1 };\n    size_t  sin_reshape_nb[GGML_MAX_DIMS];\n    sin_reshape_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];\n    }\n    acl_tensor_ptr acl_sin_reshape_tensor = ggml_cann_create_tensor(ctx.rope_cache.sin_cache, ACL_FLOAT, sizeof(float),\n                                                                    sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n    acl_tensor_ptr acl_cos_reshape_tensor = ggml_cann_create_tensor(ctx.rope_cache.cos_cache, ACL_FLOAT, sizeof(float),\n                                                                    sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n#ifdef ASCEND_310P\n    // Special ROPE operation for 310P\n\n    // roll input\n    void *               input_roll_buffer;\n    acl_tensor_ptr       acl_minus_one_tensor;\n    void *               minus_one_scale_buffer = nullptr;\n    ggml_cann_pool_alloc roll_allocator(ctx.pool(), ggml_nbytes(src0));\n    ggml_cann_pool_alloc minus_one_scale_allocator(ctx.pool(), sizeof(float) * src0->ne[0]);\n    if (!is_neox) {\n        // roll input: [q0,q1,q2,q3,...] -> [q1,q0,q3,q2,...]\n        input_roll_buffer        = roll_allocator.get();\n        int64_t input_roll_ne[4] = { 2, src0->ne[1] * (src0->ne[0] / 2), src0->ne[2], src0->ne[3] };\n        size_t  input_roll_nb[GGML_MAX_DIMS];\n        input_roll_nb[0] = ggml_type_size(src0->type);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            input_roll_nb[i] = input_roll_nb[i - 1] * input_roll_ne[i - 1];\n        }\n        acl_tensor_ptr acl_input_roll_tensor =\n            ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n                                    input_roll_ne, input_roll_nb, GGML_MAX_DIMS);\n        acl_tensor_ptr acl_input_tensor =\n            ggml_cann_create_tensor(src0->data, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n                                    input_roll_ne, input_roll_nb, GGML_MAX_DIMS);\n\n        int64_t shifts[] = { 1 };\n        int64_t dims[]   = { 3 };\n        aclnn_roll(ctx, acl_input_tensor.get(), acl_input_roll_tensor.get(), shifts, dims);\n\n        // init [-1, 1, -1, 1, ...]\n        minus_one_scale_buffer = minus_one_scale_allocator.get();\n\n        int64_t minus_one_ne[4] = { src0->ne[0], 1, 1, 1 };\n        size_t  minus_one_nb[GGML_MAX_DIMS];\n        minus_one_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];\n        }\n        acl_minus_one_tensor = aclnn_values(ctx, minus_one_scale_buffer, sizeof(float) * src0->ne[0], minus_one_ne,\n                                            GGML_MAX_DIMS, ACL_FLOAT, sizeof(float), 1);\n        int64_t   dim        = 3;\n        int64_t * index      = new int64_t[src0->ne[0]];\n        for (int i = 0; i < src0->ne[0]; i++) {\n            index[i] = i / 2 * 2;\n        }\n        int64_t index_num = src0->ne[0];\n        float   value     = -1;\n        aclnn_index_fill_tensor(ctx, acl_minus_one_tensor.get(), dim, index, index_num, value);\n    } else {\n        // roll input: [q0,q1,q2,...] ->\n        // [q_half,q_half+1,...,q_end,q0,q1,...q_half-1]\n        input_roll_buffer = roll_allocator.get();\n        acl_tensor_ptr acl_input_roll_tensor =\n            ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n                                    src0->ne, src0->nb, GGML_MAX_DIMS);\n        acl_tensor_ptr acl_input_tensor = ggml_cann_create_tensor(src0);\n\n        int64_t shifts[] = { src0->ne[0] / 2 };\n        int64_t dims[]   = { 3 };\n        aclnn_roll(ctx, acl_input_tensor.get(), acl_input_roll_tensor.get(), shifts, dims);\n\n        // init [-1, -1, -1, 1, 1，1，...]\n        minus_one_scale_buffer  = minus_one_scale_allocator.get();\n        int64_t minus_one_ne[4] = { src0->ne[0], 1, 1, 1 };\n        size_t  minus_one_nb[GGML_MAX_DIMS];\n        minus_one_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];\n        }\n        acl_minus_one_tensor     = aclnn_values(ctx, minus_one_scale_buffer, sizeof(float) * src0->ne[0], minus_one_ne,\n                                                GGML_MAX_DIMS, ACL_FLOAT, sizeof(float), 1);\n        // -1 * first half\n        int64_t first_half_ne[4] = { src0->ne[0] / 2, 1, 1, 1 };\n        size_t  first_half_nb[GGML_MAX_DIMS];\n        first_half_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            first_half_nb[i] = first_half_nb[i - 1] * first_half_ne[i - 1];\n        }\n        acl_tensor_ptr acl_first_half_tensor = ggml_cann_create_tensor(minus_one_scale_buffer, ACL_FLOAT, sizeof(float),\n                                                                       first_half_ne, first_half_nb, GGML_MAX_DIMS);\n        bool           inplace               = true;\n        float          scale                 = -1;\n        aclnn_muls(ctx, acl_first_half_tensor.get(), scale, nullptr, inplace);\n    }\n\n    // TODO: n_dims < ne0\n    GGML_ASSERT(n_dims == src0->ne[0]);\n\n    // input * scale\n    ggml_cann_pool_alloc roll_mul_scale_allocator(ctx.pool(), ggml_nbytes(src0));\n    void *               input_roll_mul_scale_buffer = roll_mul_scale_allocator.get();\n    size_t               input_nb[GGML_MAX_DIMS];\n    input_nb[0] = ggml_type_size(src0->type);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        input_nb[i] = input_nb[i - 1] * src0->ne[i - 1];\n    }\n    acl_tensor_ptr acl_input_roll_mul_scale_tensor =\n        ggml_cann_create_tensor(input_roll_mul_scale_buffer, ggml_cann_type_mapping(src0->type),\n                                ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);\n    acl_tensor_ptr acl_input_roll_reshape_tensor =\n        ggml_cann_create_tensor(input_roll_buffer, ggml_cann_type_mapping(src0->type), ggml_type_size(src0->type),\n                                src0->ne, input_nb, GGML_MAX_DIMS);\n\n    aclnn_mul(ctx, acl_input_roll_reshape_tensor.get(), acl_minus_one_tensor.get(),\n              acl_input_roll_mul_scale_tensor.get());\n\n    // output\n    void * output_fp32_buffer;\n    if (src0->type == GGML_TYPE_F32) {\n        aclnn_mul(ctx, acl_src.get(), acl_cos_reshape_tensor.get());\n        aclnn_mul(ctx, acl_input_roll_mul_scale_tensor.get(), acl_sin_reshape_tensor.get());\n        aclnn_add(ctx, acl_src.get(), acl_input_roll_mul_scale_tensor.get(), acl_dst.get());\n        // TODO: ne0 != n_dims in mode2\n    } else if (src0->type == GGML_TYPE_F16) {\n        size_t input_fp32_nb[GGML_MAX_DIMS];\n        input_fp32_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            input_fp32_nb[i] = input_fp32_nb[i - 1] * dst->ne[i - 1];\n        }\n        ggml_cann_pool_alloc fp32_allocator1(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n        void *               input_fp32_buffer1 = fp32_allocator1.get();\n        acl_tensor_ptr       input_fp32_tensor1 = ggml_cann_create_tensor(input_fp32_buffer1, ACL_FLOAT, sizeof(float),\n                                                                          dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n        ggml_cann_pool_alloc fp32_allocator2(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n        void *               input_fp32_buffer2 = fp32_allocator2.get();\n        acl_tensor_ptr       input_fp32_tensor2 = ggml_cann_create_tensor(input_fp32_buffer2, ACL_FLOAT, sizeof(float),\n                                                                          dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n\n        ggml_cann_pool_alloc fp32_allocator(ctx.pool(), ggml_nelements(dst) * sizeof(float));\n        output_fp32_buffer                = fp32_allocator.get();\n        acl_tensor_ptr output_fp32_tensor = ggml_cann_create_tensor(output_fp32_buffer, ACL_FLOAT, sizeof(float),\n                                                                    dst->ne, input_fp32_nb, GGML_MAX_DIMS);\n        aclnn_mul(ctx, acl_src.get(), acl_cos_reshape_tensor.get(), input_fp32_tensor1.get());\n        aclnn_mul(ctx, acl_input_roll_mul_scale_tensor.get(), acl_sin_reshape_tensor.get(), input_fp32_tensor2.get());\n        aclnn_add(ctx, input_fp32_tensor1.get(), input_fp32_tensor2.get(), output_fp32_tensor.get());\n        aclnn_cast(ctx, output_fp32_tensor.get(), acl_dst.get(), ACL_FLOAT16);\n    }\n    return;\n#endif\n    int64_t acl_mode = is_neox ? 0 : 1;\n\n    // Pre-define head and tail dimensions for reuse\n    int64_t head_ne[GGML_MAX_DIMS] = { rope_dims, ne01, ne02, ne03 };\n    int64_t tail_ne[GGML_MAX_DIMS] = { tail_dims, ne01, ne02, ne03 };\n\n    // Step 1: Prepare trans tensors for F16 type conversion to F32 if needed\n    bool                 src_dst_need_trans = false;\n    ggml_cann_pool_alloc src_trans_allocator(ctx.pool());\n    ggml_cann_pool_alloc dst_trans_allocator(ctx.pool());\n    acl_tensor_ptr       acl_src_trans_tensor;\n    acl_tensor_ptr       acl_dst_trans_tensor;\n    void *               src_trans_buffer = nullptr;\n    void *               dst_trans_buffer = nullptr;\n    size_t               src_dst_trans_nb[GGML_MAX_DIMS];\n    if (src0->type == GGML_TYPE_F16) {\n        src_dst_need_trans = true;\n        src_trans_buffer   = src_trans_allocator.alloc(ggml_nelements(src0) * sizeof(float));\n        dst_trans_buffer   = dst_trans_allocator.alloc(ggml_nelements(dst) * sizeof(float));\n\n        src_dst_trans_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            src_dst_trans_nb[i] = src_dst_trans_nb[i - 1] * src0->ne[i - 1];\n        }\n        acl_src_trans_tensor = ggml_cann_create_tensor(src_trans_buffer, ACL_FLOAT, sizeof(float), src0->ne,\n                                                       src_dst_trans_nb, GGML_MAX_DIMS);\n        acl_dst_trans_tensor = ggml_cann_create_tensor(dst_trans_buffer, ACL_FLOAT, sizeof(float), dst->ne,\n                                                       src_dst_trans_nb, GGML_MAX_DIMS);\n        aclnn_cast(ctx, acl_src.get(), acl_src_trans_tensor.get(), ACL_FLOAT);\n    }\n\n    // Step 2: Prepare head tensors for tail splitting if needed\n    acl_tensor_ptr acl_src_head;\n    acl_tensor_ptr acl_dst_head;\n    if (has_tail) {\n        // Create head views for RotaryPositionEmbedding (only first rope_dims dimensions)\n        // RotaryPositionEmbedding requires contiguous dst tensor, so we use a temporary buffer\n        if (src_dst_need_trans) {\n            // Use F32 trans tensor strides\n            acl_src_head = ggml_cann_create_tensor((char *) src_trans_buffer, ACL_FLOAT, sizeof(float), head_ne,\n                                                   src_dst_trans_nb, GGML_MAX_DIMS);\n        } else {\n            // Use original F32 tensor strides\n            acl_src_head = ggml_cann_create_tensor((char *) src0->data, ACL_FLOAT, sizeof(float), head_ne, src0->nb,\n                                                   GGML_MAX_DIMS);\n        }\n\n        int64_t              head_elements = rope_dims * ne01 * ne02 * ne03;\n        ggml_cann_pool_alloc dst_head_contiguous_allocator(ctx.pool(), head_elements * sizeof(float));\n        void *               dst_head_contiguous_buffer = dst_head_contiguous_allocator.get();\n\n        size_t head_contiguous_nb[GGML_MAX_DIMS];\n        head_contiguous_nb[0] = sizeof(float);\n        for (int i = 1; i < GGML_MAX_DIMS; i++) {\n            head_contiguous_nb[i] = head_contiguous_nb[i - 1] * head_ne[i - 1];\n        }\n        acl_dst_head = ggml_cann_create_tensor(dst_head_contiguous_buffer, ACL_FLOAT, sizeof(float), head_ne,\n                                               head_contiguous_nb, GGML_MAX_DIMS);\n    }\n\n    // Step 3: Execute RotaryPositionEmbedding\n    if (has_tail) {\n        // Rotate only the head portion (first rope_dims dimensions)\n        GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_head.get(), acl_cos_reshape_tensor.get(),\n                                acl_sin_reshape_tensor.get(), acl_mode, acl_dst_head.get());\n\n        // Copy head result from contiguous buffer back to destination tensor\n        if (src_dst_need_trans) {\n            acl_tensor_ptr acl_dst_head_target = ggml_cann_create_tensor(\n                (char *) dst_trans_buffer, ACL_FLOAT, sizeof(float), head_ne, src_dst_trans_nb, GGML_MAX_DIMS);\n            cann_copy(ctx, acl_dst_head.get(), acl_dst_head_target.get());\n        } else {\n            acl_tensor_ptr acl_dst_head_target =\n                ggml_cann_create_tensor((char *) dst->data, ACL_FLOAT, sizeof(float), head_ne, dst->nb, GGML_MAX_DIMS);\n            cann_copy(ctx, acl_dst_head.get(), acl_dst_head_target.get());\n        }\n    } else if (src_dst_need_trans) {\n        // Rotate full tensor (no tail), using trans tensors\n        GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_trans_tensor.get(), acl_cos_reshape_tensor.get(),\n                                acl_sin_reshape_tensor.get(), acl_mode, acl_dst_trans_tensor.get());\n    } else {\n        // Rotate full tensor (no tail), using original tensors\n        GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src.get(), acl_cos_reshape_tensor.get(),\n                                acl_sin_reshape_tensor.get(), acl_mode, acl_dst.get());\n    }\n\n    // Step 4: Copy unrotated tail portion from source to destination\n    if (has_tail) {\n        size_t src_tail_offset;\n        size_t dst_tail_offset;\n\n        auto copy_tail_device = [&](void * src_ptr, void * dst_ptr, aclDataType dtype, size_t elem_size,\n                                    size_t * nb_src_arr, size_t * nb_dst_arr) {\n            acl_tensor_ptr acl_src_tail =\n                ggml_cann_create_tensor(src_ptr, dtype, elem_size, tail_ne, nb_src_arr, GGML_MAX_DIMS);\n            acl_tensor_ptr acl_dst_tail =\n                ggml_cann_create_tensor(dst_ptr, dtype, elem_size, tail_ne, nb_dst_arr, GGML_MAX_DIMS);\n            cann_copy(ctx, acl_src_tail.get(), acl_dst_tail.get());\n        };\n\n        if (src_dst_need_trans) {\n            // Use F32 trans tensor strides and offsets\n            src_tail_offset = rope_dims * src_dst_trans_nb[0];\n            dst_tail_offset = rope_dims * src_dst_trans_nb[0];\n            copy_tail_device((char *) src_trans_buffer + src_tail_offset, (char *) dst_trans_buffer + dst_tail_offset,\n                             ACL_FLOAT, sizeof(float), src_dst_trans_nb, src_dst_trans_nb);\n        } else {\n            // Use original tensor strides and offsets\n            src_tail_offset = rope_dims * nb00;\n            dst_tail_offset = rope_dims * nb0;\n            copy_tail_device((char *) src0->data + src_tail_offset, (char *) dst->data + dst_tail_offset,\n                             ggml_cann_type_mapping(dst->type), ggml_element_size(dst), src0->nb, dst->nb);\n        }\n    }\n\n    // Step 5: Cast back to F16 if needed\n    if (src_dst_need_trans) {\n        aclnn_cast(ctx, acl_dst_trans_tensor.get(), acl_dst.get(), ACL_FLOAT16);\n    }\n}\n\nvoid ggml_cann_argmax(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, ArgMax, acl_src.get(), 3, false, acl_dst.get());\n}\n\nvoid ggml_cann_conv_transpose_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    // stride\n    int64_t s0 = ((const int32_t *) (dst->op_params))[0];\n\n    acl_tensor_ptr acl_input  = ggml_cann_create_tensor(src1, src1->ne, src1->nb, 3, ACL_FORMAT_NCL);\n    acl_tensor_ptr acl_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);\n    acl_tensor_ptr acl_dst    = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3, ACL_FORMAT_NCL);\n\n    // get base information of input and kernel\n    int64_t input_len   = *(src1->ne);\n    int64_t dst_len     = *(dst->ne);\n    int64_t kernel_size = *(src0->ne);\n\n    // set the max kernel size for each conv\n    int64_t max_kernel_size = 255;\n\n    // compute the partition of kernel\n    int64_t part_num = 1;\n    part_num         = (kernel_size + max_kernel_size - 1) / max_kernel_size;\n\n    int64_t strideVal[1];\n    strideVal[0]                    = s0;\n    acl_int_array_ptr stride        = ggml_cann_create_int_array(strideVal, 1);\n    int64_t           paddingVal[]  = { 0 };\n    acl_int_array_ptr padding       = ggml_cann_create_int_array(paddingVal, 1);\n    int64_t           dilationVal[] = { 1 };\n    acl_int_array_ptr dilation      = ggml_cann_create_int_array(dilationVal, 1);\n    bool              transposed    = true;\n    int64_t           groups        = 1;\n    int8_t            cubeMathType  = 0;\n\n#ifdef ASCEND_310P\n    cubeMathType = 1;\n#endif\n\n    auto weight_type = ggml_cann_type_mapping(src0->type);\n    auto dst_type    = ggml_cann_type_mapping(dst->type);\n\n    // slice the kernel to make each conv available\n    int64_t slice_dim   = -1;\n    int64_t slice_start = 0;\n    int64_t slice_end   = max_kernel_size;\n    int64_t slice_step  = 1;\n    int64_t interval    = max_kernel_size;\n\n    int64_t left_pad_len  = dilationVal[0] * (max_kernel_size - 1) + 1 - 2 * paddingVal[0];\n    int64_t right_pad_len = 0;\n\n    acl_scalar_ptr alpha      = nullptr;\n    float          alphaValue = 1.0;\n    alpha                     = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n    // set zero to destination\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_dst.get());\n\n    for (int k = 0; k < part_num; k++) {\n        // create part kernel tensor and slice from big kernel\n        slice_start = max_kernel_size * k;\n        if (k == part_num - 1) {\n            slice_end = kernel_size;\n            interval  = kernel_size - max_kernel_size * k;\n        } else {\n            slice_end = max_kernel_size * (k + 1);\n        }\n\n        int64_t part_ne[4];\n        for (int i = 0; i < 4; i++) {\n            part_ne[i] = *(src0->ne + i);\n        }\n        part_ne[0] = interval;\n\n        size_t part_nb[4];\n        part_nb[0] = sizeof(weight_type);\n        for (int i = 1; i < 4; i++) {\n            part_nb[i] = part_nb[i - 1] * part_ne[i - 1];\n        }\n\n        ggml_cann_pool_alloc part_kernel_allocator;\n        part_kernel_allocator.alloc(ctx.pool(), part_nb[3]);\n        void * part_kernel_buf = part_kernel_allocator.get();\n\n        acl_tensor_ptr part_kernel = ggml_cann_create_tensor(part_kernel_buf, weight_type, ggml_element_size(src0),\n                                                             part_ne, part_nb, 3, ACL_FORMAT_NCL);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, Slice, acl_weight.get(), slice_dim, slice_start, slice_end, slice_step,\n                                part_kernel.get());\n\n        // create the part conv result tensor\n        int64_t part_dst_ne[4];\n        for (int i = 0; i < 4; i++) {\n            part_dst_ne[i] = *(dst->ne + i);\n        }\n        part_dst_ne[0] = (input_len - 1) * strideVal[0] - 2 * paddingVal[0] + dilationVal[0] * (part_ne[0] - 1) + 1;\n\n        size_t part_dst_nb[4];\n        part_dst_nb[0] = sizeof(weight_type);\n        for (int i = 1; i < 4; i++) {\n            part_dst_nb[i] = part_dst_nb[i - 1] * part_dst_ne[i - 1];\n        }\n        ggml_cann_pool_alloc part_dst_allocator;\n        part_dst_allocator.alloc(ctx.pool(), part_dst_nb[3]);\n        void * part_dst_buf = part_dst_allocator.get();\n\n        acl_tensor_ptr acl_part_dst = ggml_cann_create_tensor(part_dst_buf, dst_type, ggml_element_size(dst),\n                                                              part_dst_ne, part_dst_nb, 3, ACL_FORMAT_NCL);\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_part_dst.get());\n\n        // compute part conv transpose 1d\n        GGML_CANN_CALL_ACLNN_OP(ctx, Convolution, acl_input.get(), part_kernel.get(), nullptr, stride.get(),\n                                padding.get(), dilation.get(), transposed, padding.get(), groups, acl_part_dst.get(),\n                                cubeMathType);\n\n        // compute the position of part result in final result\n        int64_t global_start = slice_start;\n        int64_t global_end   = std::min((input_len - 1) * strideVal[0] + slice_end, dst_len);\n\n        left_pad_len  = global_start;\n        right_pad_len = dst_len - global_end;\n\n        std::vector<int64_t> padDataVal = { left_pad_len, right_pad_len };\n        acl_int_array_ptr    padData    = ggml_cann_create_int_array(padDataVal.data(), 2);\n\n        acl_scalar_ptr pad_value    = nullptr;\n        float          pad_valueVal = 0.0;\n        pad_value                   = ggml_cann_create_scalar(&pad_valueVal, aclDataType::ACL_FLOAT);\n\n        int64_t conv_result_ne[4];\n        for (int i = 0; i < 4; i++) {\n            conv_result_ne[i] = *(dst->ne + i);\n        }\n\n        size_t conv_result_nb[4];\n        conv_result_nb[0] = sizeof(weight_type);\n        for (int i = 1; i < 4; i++) {\n            conv_result_nb[i] = conv_result_nb[i - 1] * conv_result_ne[i - 1];\n        }\n\n        ggml_cann_pool_alloc conv_result_allocator;\n        conv_result_allocator.alloc(ctx.pool(), conv_result_nb[3]);\n        void * conv_result_buf = conv_result_allocator.get();\n\n        acl_tensor_ptr conv_result = ggml_cann_create_tensor(conv_result_buf, dst_type, ggml_element_size(dst),\n                                                             conv_result_ne, conv_result_nb, 3, ACL_FORMAT_NCL);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, conv_result.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_part_dst.get(), padData.get(), pad_value.get(),\n                                conv_result.get());\n        GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst.get(), conv_result.get(), alpha.get());\n    }\n}\n\nvoid ggml_cann_elu(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    acl_tensor_ptr acl_input = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst   = ggml_cann_create_tensor(dst);\n\n    float          alphaValue = 1.0f;\n    acl_scalar_ptr alpha      = nullptr;\n    alpha                     = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Elu, acl_input.get(), alpha.get(), alpha.get(), alpha.get(), acl_dst.get());\n}\n\nvoid ggml_cann_mean(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    int64_t           reduceDimValue[] = { 3 };\n    acl_int_array_ptr reduceDim        = ggml_cann_create_int_array(reduceDimValue, 1);\n    bool              keepDim          = true;\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Mean, acl_src.get(), reduceDim.get(), keepDim, ACL_FLOAT, acl_dst.get());\n}\n\nvoid ggml_cann_pad_reflect_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor *     src0             = dst->src[0];\n    int32_t *         opts             = (int32_t *) dst->op_params;\n    int64_t           paddingsArray[2] = { opts[0], opts[1] };\n    acl_int_array_ptr paddings         = ggml_cann_create_int_array(paddingsArray, 2);\n\n    for (int64_t i = 0; i < src0->ne[3]; i++) {\n        acl_tensor_ptr acl_src =\n            ggml_cann_create_tensor((char *) src0->data + i * src0->ne[3], ggml_cann_type_mapping(src0->type),\n                                    ggml_element_size(src0), src0->ne, src0->nb, 3);\n\n        acl_tensor_ptr acl_dst =\n            ggml_cann_create_tensor((char *) dst->data + i * src0->ne[3], ggml_cann_type_mapping(dst->type),\n                                    ggml_element_size(dst), dst->ne, dst->nb, 3);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src.get(), paddings.get(), acl_dst.get());\n    }\n}\n\nvoid ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    acl_tensor_ptr acl_self  = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_other = ggml_cann_create_tensor(src1);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceEqTensor, acl_self.get(), acl_other.get());\n\n    ggml_cann_sum(ctx, dst);\n}\n\nvoid ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src0);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    float          alphaValue = 0.0f;\n    acl_scalar_ptr alpha      = nullptr;\n    alpha                     = ggml_cann_create_scalar(&alphaValue, aclDataType::ACL_FLOAT);\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, GtScalar, acl_src.get(), alpha.get(), acl_dst.get());\n}\n\n/**\n * @brief Performs expert-specific matrix multiplication (MoE) with\n * floating-point precision using the CANN backend.\n *\n * This function executes a matrix multiplication operation tailored for\n * Mixture of Experts (MoE) models, where the input tensor is multiplied\n * with expert-specific weight matrices. It uses the CANN backend for\n * efficient computation and stores the result in the destination tensor `dst`.\n * The operation may leverage identity-based optimizations or routing masks\n * as part of sparse expert selection.\n *\n * @param ctx The context for executing CANN backend operations.\n * @param dst The destination tensor where the MoE multiplication result\n * will be stored.\n *\n * @note This function assumes floating-point data types and is designed for\n * MoE architectures, possibly involving sparse expert routing.\n */\nstatic void ggml_cann_mul_mat_id_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    //dst   [M, K, N, 1]\n    ggml_tensor * src0 = dst->src[0];  //src0\t[D, M, A, 1]  -> [D, M, K, 1]\n    ggml_tensor * src1 = dst->src[1];  //src1\t[D, B, N, 1], B = K or B = 1 -> [D, 1, K, 1]\n    ggml_tensor * ids  = dst->src[2];  //ids\t[K, N]\n\n    GGML_ASSERT(src0->ne[3] == 1);\n    GGML_ASSERT(src1->ne[3] == 1);\n    GGML_ASSERT(dst->ne[3] == 1);\n\n    int64_t batch = src1->ne[2];\n    GGML_ASSERT(batch == ids->ne[1]);\n\n    ggml_cann_pool_alloc export_allocator(ctx.pool(), src0->ne[0] * src0->ne[1] * ids->ne[0] * ggml_element_size(src0));\n    void *               export_ptr = export_allocator.get();\n    for (int64_t i = 0; i < batch; i++) {\n        acl_tensor_ptr select_index  = ggml_cann_create_tensor(ids, ids->ne, ids->nb, 1, ACL_FORMAT_ND, i * ids->nb[1]);\n        acl_tensor_ptr export_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3);\n\n        int64_t select_export_ne[] = { src0->ne[0], src0->ne[1], ids->ne[0] };\n        size_t  select_export_nb[3];\n        select_export_nb[0] = src0->nb[0];\n        for (int k = 1; k < 3; k++) {\n            select_export_nb[k] = select_export_nb[k - 1] * select_export_ne[k - 1];\n        }\n\n        acl_tensor_ptr select_export =\n            ggml_cann_create_tensor(export_ptr, ggml_cann_type_mapping(src0->type), ggml_element_size(src0),\n                                    select_export_ne, select_export_nb, 3);\n        GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, export_weight.get(), 0, select_index.get(), select_export.get());\n\n        int64_t        select_transpose_ne[] = { select_export_ne[1], select_export_ne[0], select_export_ne[2] };\n        size_t         select_transpose_nb[] = { select_export_nb[1], select_export_nb[0], select_export_nb[2] };\n        acl_tensor_ptr select_export_transpose =\n            ggml_cann_create_tensor(export_ptr, ggml_cann_type_mapping(src0->type), ggml_element_size(src0),\n                                    select_transpose_ne, select_transpose_nb, 3);\n\n        int64_t        active_tensor_ne[] = { src1->ne[0], 1, src1->ne[1] };\n        size_t         active_tensor_nb[] = { src1->nb[0], src1->nb[1], src1->nb[1] };\n        acl_tensor_ptr active_tensor =\n            ggml_cann_create_tensor(src1, active_tensor_ne, active_tensor_nb, 3, ACL_FORMAT_ND, i * src1->nb[2]);\n\n        int64_t        dst_ne[] = { dst->ne[0], 1, dst->ne[1] };\n        size_t         dst_nb[] = { dst->nb[0], dst->nb[1], dst->nb[1] };\n        acl_tensor_ptr acl_dst  = ggml_cann_create_tensor(dst, dst_ne, dst_nb, 3, ACL_FORMAT_ND, i * dst->nb[2]);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, active_tensor.get(), select_export_transpose.get(), acl_dst.get(), 2);\n    }\n}\n\n/**\n * @brief Performs quantized matrix multiplication for Mixture of Experts (MoE)\n * models using the CANN backend.\n *\n * This function implements MUL_MAT_ID operation for quantized weight matrices\n * (Q4_0 and Q8_0 formats). It selects expert-specific weight matrices based on\n * the provided expert indices, and computes matrix multiplication using CANN's\n * WeightQuantBatchMatmulV2 operator.\n *\n * The function performs the following steps:\n * 1. Converts input/output tensors to F16 format if necessary\n * 2. Uses IndexSelect to extract expert-specific weights and scales based on indices\n * 3. Performs quantized matrix multiplication for each expert using WeightQuantBatchMatmulV2\n * 4. Converts output back to the target type if needed\n *\n * Tensor shapes:\n * - dst:  [M, K, N, 1] - output tensor\n * - src0: [D, M, A, 1] - quantized weight matrices (Q4_0 or Q8_0)\n * - src1: [D, B, N, 1] - input activations (B = K for per-expert input, or B = 1 for broadcast)\n * - ids:  [K, N] - expert indices for routing\n *\n * @param ctx The CANN backend context for operation execution.\n * @param dst The destination tensor where the multiplication result will be stored.\n *\n * @note Only Q4_0 and Q8_0 quantization formats are supported.\n * @note The function handles automatic type conversion to/from F16 as needed by the hardware.\n */\nstatic void ggml_cann_mul_mat_id_quant(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    // dst:  [M, K, N, 1]\n    // src0: [D, M, A, 1] - quantized weights\n    // src1: [D, B, N, 1] - input activations, B = K or B = 1\n    // ids:  [K, N] - expert indices\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n    ggml_tensor * ids  = dst->src[2];\n\n    GGML_ASSERT(src0->ne[3] == 1);\n    GGML_ASSERT(src1->ne[3] == 1);\n    GGML_ASSERT(dst->ne[3] == 1);\n    GGML_ASSERT(src1->ne[2] == ids->ne[1]);\n\n    const int64_t        n_batches        = ids->ne[1];\n    const int64_t        n_select_experts = ids->ne[0];\n    const enum ggml_type type             = src0->type;\n\n    const int32_t group_size = QK8_0;  // Both Q4_0 and Q8_0 use group size of 32\n    GGML_ASSERT(group_size == QK4_0);\n\n    // Calculate element size for quantized weights\n    const float weight_elem_size =\n        (type == GGML_TYPE_Q4_0) ? 0.5f :\n        (type == GGML_TYPE_Q8_0) ? 1.0f :\n                                   (GGML_ABORT(\"MUL_MAT_ID only supports Q4_0 and Q8_0\"), 0.0f);\n\n    // Calculate scale offset in memory\n    const size_t weight_size     = src0->ne[0] * src0->ne[1] * src0->ne[2] * weight_elem_size;\n    const size_t scale_elem_size = sizeof(uint16_t);\n    char *       scale_data      = (char *) src0->data + weight_size;\n\n    // Allocate buffers for selected expert weights and scales\n    const size_t         selected_weight_size = src0->ne[0] * src0->ne[1] * n_select_experts * weight_elem_size;\n    ggml_cann_pool_alloc selected_weight_alloc(ctx.pool(), selected_weight_size);\n    void *               selected_weight_buffer = selected_weight_alloc.get();\n\n    const size_t selected_scale_size = (src0->ne[0] / group_size) * src0->ne[1] * n_select_experts * scale_elem_size;\n    ggml_cann_pool_alloc selected_scale_alloc(ctx.pool(), selected_scale_size);\n    void *               selected_scale_buffer = selected_scale_alloc.get();\n\n    // Helper lambda to allocate and cast tensor to F16 if needed\n    constexpr size_t f16_elem_size      = sizeof(uint16_t);\n    auto             prepare_f16_buffer = [&](ggml_tensor * tensor, ggml_cann_pool_alloc & allocator,\n                                  bool need_cast = false) -> void * {\n        if (tensor->type == GGML_TYPE_F16) {\n            return tensor->data;\n        }\n\n        size_t total_size = f16_elem_size;\n        for (int i = 0; i < GGML_MAX_DIMS; i++) {\n            total_size *= tensor->ne[i];\n        }\n        void * buffer = allocator.alloc(total_size);\n\n        if (need_cast == false) {\n            return buffer;\n        }\n\n        int64_t ne[GGML_MAX_DIMS];\n        size_t  nb[GGML_MAX_DIMS] = { f16_elem_size };\n        for (int i = 0; i < GGML_MAX_DIMS; i++) {\n            ne[i] = tensor->ne[i];\n            if (i > 0) {\n                nb[i] = nb[i - 1] * ne[i - 1];\n            }\n        }\n\n        acl_tensor_ptr src_tensor = ggml_cann_create_tensor(tensor);\n        acl_tensor_ptr f16_tensor = ggml_cann_create_tensor(buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);\n        aclnn_cast(ctx, src_tensor.get(), f16_tensor.get(), ACL_FLOAT16);\n\n        return buffer;\n    };\n\n    // Prepare input and output buffers\n    ggml_cann_pool_alloc input_alloc(ctx.pool());\n    void *               input_buffer = prepare_f16_buffer(src1, input_alloc, true);\n\n    ggml_cann_pool_alloc output_alloc(ctx.pool());\n    void *               output_buffer = prepare_f16_buffer(dst, output_alloc, false);\n\n    // Process each batch\n    for (int64_t batch_idx = 0; batch_idx < n_batches; batch_idx++) {\n        // Create index tensor for current batch\n        const size_t   index_offset  = batch_idx * ids->nb[1];\n        acl_tensor_ptr batch_indices = ggml_cann_create_tensor(ids, ids->ne, ids->nb, 1, ACL_FORMAT_ND, index_offset);\n\n        // Select quantized weights using expert indices\n        // Q4_0 stores 2 values per byte, Q8_0 stores 1 value per byte\n        const int64_t weight_d         = (type == GGML_TYPE_Q4_0) ? src0->ne[0] / 2 : src0->ne[0];\n        const int64_t weight_m         = src0->ne[1];\n        const int64_t weight_n_experts = src0->ne[2];\n\n        int64_t weight_ne[3] = { weight_d, weight_m, weight_n_experts };\n        size_t  weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t), weight_d * weight_m * sizeof(int8_t) };\n\n        acl_tensor_ptr all_weights =\n            ggml_cann_create_tensor(src0->data, ACL_INT8, sizeof(int8_t), weight_ne, weight_nb, 3);\n\n        int64_t selected_weight_ne[3] = { weight_d, weight_m, n_select_experts };\n        size_t  selected_weight_nb[3] = { sizeof(int8_t), weight_d * sizeof(int8_t),\n                                          weight_d * weight_m * sizeof(int8_t) };\n\n        acl_tensor_ptr selected_weights = ggml_cann_create_tensor(selected_weight_buffer, ACL_INT8, sizeof(int8_t),\n                                                                  selected_weight_ne, selected_weight_nb, 3);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_weights.get(), 0, batch_indices.get(), selected_weights.get());\n\n        // Select scales using the same expert indices\n        const int64_t scale_d     = src0->ne[0] / group_size;\n        int64_t       scale_ne[3] = { scale_d, weight_m, weight_n_experts };\n        size_t scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size, scale_d * weight_m * scale_elem_size };\n\n        acl_tensor_ptr all_scales =\n            ggml_cann_create_tensor(scale_data, ACL_FLOAT16, scale_elem_size, scale_ne, scale_nb, 3);\n\n        int64_t selected_scale_ne[3] = { scale_d, weight_m, n_select_experts };\n        size_t  selected_scale_nb[3] = { scale_elem_size, scale_d * scale_elem_size,\n                                         scale_d * weight_m * scale_elem_size };\n\n        acl_tensor_ptr selected_scales = ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size,\n                                                                 selected_scale_ne, selected_scale_nb, 3);\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, IndexSelect, all_scales.get(), 0, batch_indices.get(), selected_scales.get());\n\n        // Process each expert for current batch\n        // IndexSelect output layout: [D, M, K] in contiguous format\n        // WeightQuantBatchMatmulV2 expects: [M, D] with row-major stride\n        for (int64_t expert_idx = 0; expert_idx < n_select_experts; expert_idx++) {\n            // Determine input offset: broadcast if src1->ne[1]==1, otherwise use per-expert input\n            const size_t input_offset =\n                (batch_idx * src1->ne[1] + (src1->ne[1] == 1 ? 0 : expert_idx)) * src1->ne[0] * f16_elem_size;\n            const size_t output_offset = (batch_idx * dst->ne[1] + expert_idx) * dst->ne[0] * f16_elem_size;\n\n            // Create weight view for current expert: [D, M, K] -> [M, D]\n            int64_t      weight_view_ne[2]  = { weight_m, src0->ne[0] };\n            float        weight_view_nb[2]  = { src0->ne[0] * weight_elem_size, weight_elem_size };\n            const size_t weight_view_offset = expert_idx * selected_weight_nb[2];\n\n            acl_tensor_ptr weight_view =\n                ggml_cann_create_tensor(selected_weight_buffer, ggml_cann_type_mapping(type), weight_elem_size,\n                                        weight_view_ne, weight_view_nb, 2, ACL_FORMAT_ND, weight_view_offset);\n\n            // Create scale view for current expert: [D, M, K] -> [M, D]\n            int64_t      scale_view_ne[2]  = { weight_m, scale_d };\n            size_t       scale_view_nb[2]  = { selected_scale_nb[1], selected_scale_nb[0] };\n            const size_t scale_view_offset = expert_idx * selected_scale_nb[2];\n\n            acl_tensor_ptr scale_view =\n                ggml_cann_create_tensor(selected_scale_buffer, ACL_FLOAT16, scale_elem_size, scale_view_ne,\n                                        scale_view_nb, 2, ACL_FORMAT_ND, scale_view_offset);\n\n            // Create input activation tensor [D, 1]\n            int64_t input_ne[2] = { src1->ne[0], 1 };\n            size_t  input_nb[2] = { f16_elem_size, src1->ne[0] * f16_elem_size };\n\n            acl_tensor_ptr input_tensor = ggml_cann_create_tensor(input_buffer, ACL_FLOAT16, f16_elem_size, input_ne,\n                                                                  input_nb, 2, ACL_FORMAT_ND, input_offset);\n\n            // Create output tensor [M, 1]\n            int64_t output_ne[2] = { dst->ne[0], 1 };\n            size_t  output_nb[2] = { f16_elem_size, dst->ne[0] * f16_elem_size };\n\n            acl_tensor_ptr output_tensor = ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, output_ne,\n                                                                   output_nb, 2, ACL_FORMAT_ND, output_offset);\n\n            // Perform quantized matrix multiplication\n            GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, input_tensor.get(), weight_view.get(),\n                                    scale_view.get(), nullptr, nullptr, nullptr, nullptr, group_size,\n                                    output_tensor.get());\n        }\n    }\n\n    // Cast output back to original type if we used a temporary F16 buffer\n    if (dst->type != GGML_TYPE_F16) {\n        int64_t ne[GGML_MAX_DIMS];\n        size_t  nb[GGML_MAX_DIMS] = { f16_elem_size };\n        for (int i = 0; i < GGML_MAX_DIMS; i++) {\n            ne[i] = dst->ne[i];\n            if (i > 0) {\n                nb[i] = nb[i - 1] * ne[i - 1];\n            }\n        }\n\n        acl_tensor_ptr f16_output =\n            ggml_cann_create_tensor(output_buffer, ACL_FLOAT16, f16_elem_size, ne, nb, GGML_MAX_DIMS);\n        acl_tensor_ptr dst_tensor = ggml_cann_create_tensor(dst);\n\n        aclnn_cast(ctx, f16_output.get(), dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n    }\n}\n\nvoid ggml_cann_mul_mat_id(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    const enum ggml_type type = dst->src[0]->type;\n    switch (type) {\n        case GGML_TYPE_F32:\n        case GGML_TYPE_F16:\n            ggml_cann_mul_mat_id_fp(ctx, dst);\n            break;\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q8_0:\n            ggml_cann_mul_mat_id_quant(ctx, dst);\n            break;\n        default:\n            GGML_ABORT(\"Unsupported type for mul_mat_id\");\n            break;\n    }\n}\n\nvoid ggml_cann_flash_attn_ext(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // q, fp32 | B, N, S, D (uncont) -> B, S, N, D (cont)\n    ggml_tensor * src1 = dst->src[1];  // k, fp16 | B, N, S, D (uncont) -> B, S, N, D (cont)\n    ggml_tensor * src2 = dst->src[2];  // v, fp16 | B, N, S, D (uncont) -> B, S, N, D (cont)\n    ggml_tensor * src3 = dst->src[3];  // mask, fp16\n\n    // B, N, S, D (uncont) -> B, S, N, D (cont)\n    int64_t src0_bsnd_ne[GGML_MAX_DIMS];\n    memcpy(src0_bsnd_ne, src0->ne, GGML_MAX_DIMS * sizeof(int64_t));\n    size_t src0_bsnd_nb[GGML_MAX_DIMS];\n    memcpy(src0_bsnd_nb, src0->nb, GGML_MAX_DIMS * sizeof(size_t));\n    int64_t src1_bsnd_ne[GGML_MAX_DIMS];\n    memcpy(src1_bsnd_ne, src1->ne, GGML_MAX_DIMS * sizeof(int64_t));\n    size_t src1_bsnd_nb[GGML_MAX_DIMS];\n    memcpy(src1_bsnd_nb, src1->nb, GGML_MAX_DIMS * sizeof(size_t));\n    int64_t src2_bsnd_ne[GGML_MAX_DIMS];\n    memcpy(src2_bsnd_ne, src2->ne, GGML_MAX_DIMS * sizeof(int64_t));\n    size_t src2_bsnd_nb[GGML_MAX_DIMS];\n    memcpy(src2_bsnd_nb, src2->nb, GGML_MAX_DIMS * sizeof(size_t));\n\n    auto transpose12 = [](int64_t * ne, size_t * nb) {\n        int64_t ne_tmp = ne[1];\n        size_t  nb_tmp = nb[1];\n        ne[1]          = ne[2];\n        nb[1]          = nb[2];\n        ne[2]          = ne_tmp;\n        nb[2]          = nb_tmp;\n    };\n\n    transpose12(src0_bsnd_ne, src0_bsnd_nb);\n    transpose12(src1_bsnd_ne, src1_bsnd_nb);\n    transpose12(src2_bsnd_ne, src2_bsnd_nb);\n\n    float maxBias      = 0.0f;\n    float scaleValue   = 1.0f;\n    float logitSoftcap = 0.0f;\n    memcpy(&scaleValue, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&maxBias, (float *) dst->op_params + 1, sizeof(float));\n    memcpy(&logitSoftcap, (float *) dst->op_params + 2, sizeof(float));\n\n    if (logitSoftcap == 0.0f) {\n        size_t faElemSize = sizeof(uint16_t);\n        auto   faDataType = ACL_FLOAT16;  //ACL_BF16;\n\n        acl_tensor_ptr acl_q_tensor = nullptr;\n        acl_tensor_ptr acl_k_tensor = nullptr;\n        acl_tensor_ptr acl_v_tensor = nullptr;\n\n        // Step 1: cast the src0 (Query) to fp16 if needed\n        ggml_cann_pool_alloc src0_f16_allocator(ctx.pool());\n        void *               src0_f16_buffer = nullptr;\n\n        if (ggml_cann_type_mapping(src0->type) != faDataType) {\n            acl_tensor_ptr acl_src0_f32_tensor =\n                ggml_cann_create_tensor(src0, src0_bsnd_ne, src0_bsnd_nb, GGML_MAX_DIMS);\n            src0_f16_buffer = src0_f16_allocator.alloc(ggml_nelements(src0) * faElemSize);\n\n            int64_t * src0_f16_ne = src0_bsnd_ne;\n            size_t    src0_f16_nb[GGML_MAX_DIMS];\n            src0_f16_nb[0] = sizeof(uint16_t);\n            for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n                src0_f16_nb[i] = src0_f16_nb[i - 1] * src0_f16_ne[i - 1];\n            }\n\n            acl_q_tensor = ggml_cann_create_tensor(src0_f16_buffer, faDataType, faElemSize, src0_f16_ne, src0_f16_nb,\n                                                   GGML_MAX_DIMS);\n            aclnn_cast(ctx, acl_src0_f32_tensor.get(), acl_q_tensor.get(), faDataType);\n        } else {\n            acl_q_tensor = ggml_cann_create_tensor(src0, src0_bsnd_ne, src0_bsnd_nb, GGML_MAX_DIMS);\n        }\n\n        // Step 2: create the acl tensors for src1 (Key), src2 (Value),\n        //         and the direct output from FusedInferAttention\n\n        acl_k_tensor = ggml_cann_create_tensor(src1, src1_bsnd_ne, src1_bsnd_nb, GGML_MAX_DIMS);\n        acl_v_tensor = ggml_cann_create_tensor(src2, src2_bsnd_ne, src2_bsnd_nb, GGML_MAX_DIMS);\n\n        // Step 3: create the PSEShift tensor if needed\n        //         this tensor is considered as mask (f16) in the llama.cpp\n        acl_tensor_ptr       bcast_pse_tensor;\n        ggml_cann_pool_alloc bcast_pse_allocator(ctx.pool());\n        if (src3 != nullptr) {\n            // Construct the truncated pse tensor (common for prefill/decode)\n            int64_t trunc_pse_ne[GGML_MAX_DIMS] = {\n                src3->ne[0],  // D\n                src0->ne[1],  // S (number of Q tokens)\n                src3->ne[2],  // mask N\n                src3->ne[3]   // B\n            };\n            size_t * trunc_pse_nb = src3->nb;\n\n            acl_tensor_ptr acl_mask_f16_trunc_tensor = ggml_cann_create_tensor(\n                src3->data, ACL_FLOAT16, sizeof(uint16_t), trunc_pse_ne, trunc_pse_nb, GGML_MAX_DIMS);\n\n            int64_t bcast_pse_ne[GGML_MAX_DIMS];\n            size_t  bcast_pse_nb[GGML_MAX_DIMS];\n            bcast_pse_ne[0] = src3->ne[0];  // D\n            bcast_pse_ne[1] = src0->ne[1];  // S\n            bcast_pse_ne[2] = src0->ne[2];  // N (num_heads)\n            bcast_pse_ne[3] = src3->ne[3];  // B\n            if (maxBias == 0.0f) {\n                // When maxBias == 0.0f, use nb = 0 reduce once repeat (Qwen2)\n                // Construct the bcast tensor (simulate repeat on the head dimension using stride=0)\n                bcast_pse_nb[0] = sizeof(uint16_t);\n                bcast_pse_nb[1] = bcast_pse_nb[0] * bcast_pse_ne[0];\n                bcast_pse_nb[2] = 0;  // <---- the head dimension shares the same data\n                bcast_pse_nb[3] = src3->nb[3];\n\n                bcast_pse_tensor = ggml_cann_create_tensor(src3->data, ACL_FLOAT16, sizeof(uint16_t), bcast_pse_ne,\n                                                           bcast_pse_nb, GGML_MAX_DIMS);\n\n            } else {\n                bcast_pse_nb[0] = sizeof(uint16_t);\n                for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                    bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];\n                }\n\n                void * bcast_pse_buffer =\n                    bcast_pse_allocator.alloc(ggml_nelements(src3) * src0->ne[2] * sizeof(uint16_t));\n\n                bcast_pse_tensor = ggml_cann_create_tensor(bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),\n                                                           bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);\n\n                int64_t repeats[] = { 1, src0->ne[2], 1, 1 };\n                aclnn_repeat(ctx, acl_mask_f16_trunc_tensor.get(), bcast_pse_tensor.get(), repeats);\n\n                // alibi\n                // Compute the slope if needed. Derived from ggml_cann_softmax().\n                const int64_t        n_heads = src0->ne[2];\n                ggml_cann_pool_alloc slope_allocator(ctx.pool(), n_heads * sizeof(uint16_t));\n                void *               slope_buffer = slope_allocator.get();\n                aclnn_get_slope(ctx, n_heads, slope_buffer, maxBias, GGML_TYPE_F16);\n\n                int64_t slope_ne[] = { 1, 1, n_heads, 1 };\n                size_t  slope_nb[GGML_MAX_DIMS];\n                slope_nb[0] = sizeof(uint16_t);\n                for (int i = 1; i < GGML_MAX_DIMS; i++) {\n                    slope_nb[i] = slope_nb[i - 1] * slope_ne[0];\n                }\n\n                acl_tensor_ptr slope_tensor = ggml_cann_create_tensor(slope_buffer, ACL_FLOAT16, sizeof(uint16_t),\n                                                                      slope_ne, slope_nb, GGML_MAX_DIMS);\n                GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, bcast_pse_tensor.get(), slope_tensor.get());\n            }\n        }\n\n        // Step 4: set the inputs for FusedInferAttention.\n        acl_tensor_list_ptr acl_k_tensor_list = ggml_cann_create_tensor_list(acl_k_tensor);\n        acl_tensor_list_ptr acl_v_tensor_list = ggml_cann_create_tensor_list(acl_v_tensor);\n\n        int64_t numHeads           = src0->ne[2];  // N\n        int64_t numKeyValueHeads   = src1->ne[2];\n        // double  scaleValue = 1 / sqrt(src0->ne[0]); // 1/sqrt(d)\n        int64_t preTokens          = 65535;\n        int64_t nextTokens         = 65535;\n        char    layout[5]          = { 'B', 'S', 'N', 'D', 0 };\n        int64_t sparseMode         = 0;\n        int64_t innerPrecise       = (src0->ne[1] == 1) ? 0 : 2;\n        int64_t blockSize          = 0;\n        int64_t antiquantMode      = 0;\n        bool    softmaxLseFlag     = false;\n        int64_t keyAntiquantMode   = 0;\n        int64_t valueAntiquantMode = 0;\n\n        GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n        acl_tensor_ptr       fa_dst_tensor;\n        acl_tensor_ptr       acl_dst_tensor;\n        ggml_cann_pool_alloc out_f16_allocator(ctx.pool());\n        if (dst->type == GGML_TYPE_F32) {\n            void * out_f16_buffer = out_f16_allocator.alloc(ggml_nelements(dst) * faElemSize);\n\n            int64_t * out_f16_ne = src0_bsnd_ne;\n            size_t    out_f16_nb[GGML_MAX_DIMS];\n            out_f16_nb[0] = faElemSize;\n            for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n                out_f16_nb[i] = out_f16_nb[i - 1] * out_f16_ne[i - 1];\n            }\n\n            fa_dst_tensor =\n                ggml_cann_create_tensor(out_f16_buffer, faDataType, faElemSize, out_f16_ne, out_f16_nb, GGML_MAX_DIMS);\n        } else {\n            fa_dst_tensor = ggml_cann_create_tensor(dst);\n        }\n\n        GGML_CANN_CALL_ACLNN_OP(ctx, FusedInferAttentionScoreV2, acl_q_tensor.get(), acl_k_tensor_list.get(),\n                                acl_v_tensor_list.get(),               // q, k, v\n                                bcast_pse_tensor.get(), nullptr,       // pse, mask\n                                nullptr, nullptr,                      // actSeqLen, actSeqLenkv\n                                nullptr, nullptr,                      // deqScale1, quantScale1\n                                nullptr, nullptr, nullptr,             // deqScale2, quantScale2, quantOffset2\n                                nullptr, nullptr,                      // antiquantScale, antiquantOffset\n                                nullptr,                               // blockTable\n                                nullptr, nullptr,                      // qPadSize, kvPadSize\n                                nullptr, nullptr,                      // kAntiquantScale, kAntiQuantOffset\n                                nullptr, nullptr,                      // vAntiquantScale, vAntiQuantOffset\n                                nullptr, nullptr, nullptr,             // kSharedPrefix, vSharedPrefix, actSharedLen\n                                numHeads, scaleValue,                  // heads, scaleValue\n                                preTokens, nextTokens,                 // preTokens, nextTokens\n                                layout,                                // inputLayout\n                                numKeyValueHeads,                      // numKVHeads\n                                sparseMode, innerPrecise,              // sparseMode, innerPrecise\n                                blockSize, antiquantMode,              // blockSize, antiquantMode\n                                softmaxLseFlag,                        // softmaxLseFlag\n                                keyAntiquantMode, valueAntiquantMode,  // keyAntiqMode, valueAntiqMode\n                                fa_dst_tensor.get(),                   // attentionOut\n                                nullptr                                // softmaxLse\n        );\n\n        if (dst->type == GGML_TYPE_F32) {\n            // Step 6: post-processing, permute and cast to f32\n            acl_tensor_ptr acl_dst_tensor = ggml_cann_create_tensor(dst);\n            aclnn_cast(ctx, fa_dst_tensor.get(), acl_dst_tensor.get(), ggml_cann_type_mapping(dst->type));\n        }\n    } else {\n        GGML_ABORT(\"Function is not implemented.\");\n    }\n}\n\nstatic void ggml_cann_out_prod_fp(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // weight\n    ggml_tensor * src1 = dst->src[1];  // input\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n    GGML_CANN_CALL_ACLNN_OP(ctx, InplaceZero, acl_dst.get());\n\n    const int64_t dps2 = ne2 / ne02;\n    const int64_t dps3 = ne3 / ne03;\n    for (int64_t i3 = 0; i3 < ne3; i3++) {\n        for (int64_t i2 = 0; i2 < ne2; i2++) {\n            const int64_t i02 = i2 / dps2;\n            const int64_t i03 = i3 / dps3;\n\n            const int64_t  i12 = i2;\n            const int64_t  i13 = i3;\n            acl_tensor_ptr accumulator =\n                ggml_cann_create_tensor((char *) dst->data + i2 * nb2 + i3 * nb3, ggml_cann_type_mapping(dst->type),\n                                        ggml_type_size(dst->type), dst->ne, dst->nb, 2);\n\n            // The outer product needs to be accumulated in this dimension.\n            for (int64_t i1 = 0; i1 < ne11; i1++) {\n                acl_tensor_ptr acl_input = ggml_cann_create_tensor(\n                    (char *) src1->data + i1 * nb11 + i12 * nb12 + i13 * nb13, ggml_cann_type_mapping(src0->type),\n                    ggml_type_size(src0->type), src1->ne, src1->nb, 1);\n\n                acl_tensor_ptr acl_weight = ggml_cann_create_tensor(\n                    (char *) src0->data + i1 * nb01 + i02 * nb02 + i03 * nb03, ggml_cann_type_mapping(src0->type),\n                    ggml_type_size(src0->type), src0->ne, src0->nb, 1);\n\n                ggml_cann_pool_alloc output_allocator(ctx.pool());\n                void *               output_buffer = output_allocator.alloc(ggml_nbytes(dst));\n                acl_tensor_ptr       acl_out = ggml_cann_create_tensor(output_buffer, ggml_cann_type_mapping(dst->type),\n                                                                       ggml_type_size(dst->type), dst->ne, dst->nb, 2);\n\n                GGML_CANN_CALL_ACLNN_OP(ctx, Ger, acl_input.get(), acl_weight.get(), acl_out.get());\n                float       alpha_value = 1.0f;\n                aclScalar * alpha       = aclCreateScalar(&alpha_value, ACL_FLOAT);\n                GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, accumulator.get(), acl_out.get(), alpha);\n            }\n        }\n    }\n}\n\nvoid ggml_cann_out_prod(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n\n    const enum ggml_type type = src0->type;\n\n    switch (type) {\n        case GGML_TYPE_F32:\n        case GGML_TYPE_F16:\n            ggml_cann_out_prod_fp(ctx, dst);\n            break;\n        default:\n            GGML_ABORT(\"Unsupport type for GGML_OP_OUT_PROD\");\n            break;\n    }\n}\n\nvoid ggml_cann_ssm_conv(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];  // conv_x\n    ggml_tensor * src1 = dst->src[1];  // conv1d.weight\n\n    // This op is currently defined only for F32 in ggml_cpu\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    // Shapes follow ggml_compute_forward_ssm_conv_f32\n    const int64_t nc  = src1->ne[0];   // d_conv\n    const int64_t ncs = src0->ne[0];   // d_conv - 1 + n_t\n    const int64_t nr  = src0->ne[1];   // d_inner\n    const int64_t n_s = src0->ne[2];   // n_seqs\n\n    const int64_t n_t = dst->ne[1];    // tokens per sequence\n\n    GGML_ASSERT(dst->ne[0] == nr);     // dst: {d_inner, n_t, n_s}\n    GGML_ASSERT(src1->ne[1] == nr);    // weight: {d_conv, d_inner}\n    GGML_ASSERT(ncs == nc - 1 + n_t);  // conv_x: {d_conv - 1 + n_t, d_inner, n_s}\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n    // --- Build CANN tensors ---\n\n    // 1) Input: conv_x as NCL\n    //\n    // src0->ne = { ncs, nr, n_s, 1 }  // {L_in, C, N}\n    // Passing ACL_FORMAT_NCL here means:\n    //   reversed dims -> [N, C, L_in] = [n_s, nr, ncs]\n    acl_tensor_ptr acl_x = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);\n\n    // 2) Weights: depthwise conv kernel, view src1 as {K, 1, C}\n    //\n    // src1 original:   ne = { nc, nr, 1, 1 }  // [K, C, 1, 1]\n    // we want a view:  ne_w = { nc, 1, nr }   // [K, 1, C]\n    // so that reversed dims -> [C, 1, K] which matches\n    //   [out_channels, in_channels/groups, kernel_size]\n    int64_t w_ne[GGML_MAX_DIMS] = { nc, 1, nr, 1 };  // [K, 1 input ch. per group, C groups]\n    // Layout: src1 data is [K, C] with\n    //   offset(k, c) = k*nb0 + c*nb1\n    // We want offset_w(k, 0, c) = k*nb0 + c*nb1,\n    // so we can reuse nb0 and nb1, and set nb2 = nb1.\n    size_t  w_nb[GGML_MAX_DIMS] = { src1->nb[0], src1->nb[1], src1->nb[1], src1->nb[3] };  // same as src1\n\n    acl_tensor_ptr acl_w = ggml_cann_create_tensor(src1->data, ggml_cann_type_mapping(src1->type),\n                                                   ggml_type_size(src1->type), w_ne, w_nb, 3, ACL_FORMAT_NCL);\n\n    // 3) Output: dst is { d_inner, n_t, n_s } (CLN)\n    //\n    // We need an NCL view of the same buffer:\n    //   desired NCL logical shape: { L_out = n_t, C = nr, N = n_s }\n    //\n    // Original CLN layout:\n    //   dst->ne = { nr, n_t, n_s }\n    //   dst->nb[0] = sizeof(float)\n    //   dst->nb[1] = nr * sizeof(float)\n    //   dst->nb[2] = nr * n_t * sizeof(float)\n    //\n    // We want offset_new(L, C, N) = offset_orig(C, L, N).\n    // Choose:\n    //   nb_y[0] = nr * sizeof(float);           // step in L\n    //   nb_y[1] = sizeof(float);                // step in C\n    //   nb_y[2] = nr * n_t * sizeof(float);     // step in N\n    int64_t y_ne[GGML_MAX_DIMS] = { n_t, nr, n_s, 1 };  // [L_out, C, N]\n    size_t  y_nb[GGML_MAX_DIMS] = { dst->ne[0] * sizeof(float), sizeof(float), dst->ne[0] * dst->ne[1] * sizeof(float),\n                                    dst->nb[3] };       // [nr, 1, nr * n_t]\n\n    acl_tensor_ptr acl_y = ggml_cann_create_tensor(dst->data, ggml_cann_type_mapping(dst->type),\n                                                   ggml_type_size(dst->type), y_ne, y_nb, 3, ACL_FORMAT_NCL);\n\n    // --- Conv1d parameters: depthwise, stride 1, no padding (\"valid\") ---\n    int64_t strideVal[1]   = { 1 };\n    int64_t paddingVal[1]  = { 0 };\n    int64_t dilationVal[1] = { 1 };\n\n    acl_int_array_ptr stride   = ggml_cann_create_int_array(strideVal, 1);\n    acl_int_array_ptr padding  = ggml_cann_create_int_array(paddingVal, 1);\n    acl_int_array_ptr dilation = ggml_cann_create_int_array(dilationVal, 1);\n\n    const bool    transposed   = false;\n    const int64_t groups       = nr;  // depthwise: one group per inner dim\n    int8_t        cubeMathType = 0;\n\n#ifdef ASCEND_310P\n    cubeMathType = 1;\n#endif\n\n    GGML_CANN_CALL_ACLNN_OP(ctx, Convolution,\n                            acl_x.get(),    // input:  N, C, L_in = ncs\n                            acl_w.get(),    // weight: [C, 1, K] with groups=nr\n                            nullptr,        // bias\n                            stride.get(), padding.get(), dilation.get(), transposed,\n                            padding.get(),  // output padding (unused for non-transposed)\n                            groups, acl_y.get(), cubeMathType);\n}\n\nvoid ggml_cann_op_add_rms_norm_fused(ggml_backend_cann_context & ctx,\n                                     ggml_tensor *               add_node,\n                                     ggml_tensor *               rms_norm_node) {\n    // Get the two input tensors for ADD operation\n    ggml_tensor * x1 = add_node->src[0];\n    ggml_tensor * x2 = add_node->src[1];\n\n    // Create ACL tensors for the two ADD inputs\n    acl_tensor_ptr acl_x1 = ggml_cann_create_tensor(x1);\n    acl_tensor_ptr acl_x2 = ggml_cann_create_tensor(x2);\n\n    // Get epsilon parameter from rms_norm_tensor\n    float eps;\n    memcpy(&eps, rms_norm_node->op_params, sizeof(float));\n\n    // Build gamma tensor (RMS normalization scaling factor)\n    // Gamma should match the normalized dimensions (last dimension of x1)\n    size_t acl_gamma_nb[GGML_MAX_DIMS];\n    acl_gamma_nb[0] = ggml_type_size(rms_norm_node->type);\n    for (int i = 1; i < GGML_MAX_DIMS; i++) {\n        acl_gamma_nb[i] = acl_gamma_nb[i - 1] * x1->ne[i - 1];\n    }\n    acl_tensor_ptr acl_gamma =\n        get_cache_acl_tensor(ctx, &ctx.rms_norm_one_tensor_cache.cache, ctx.rms_norm_one_tensor_cache.size, x1->ne,\n                             acl_gamma_nb, rms_norm_node->type,\n                             1,    // dims - only the last dimension\n                             1.0f  // value\n        );\n\n    // Build rstdOut tensor (output for normalized standard deviation)\n    // Shape should be the dimensions that are NOT normalized\n    int64_t acl_rstd_ne[] = { 1, x1->ne[1], x1->ne[2], x1->ne[3] };\n    size_t  acl_rstd_nb[GGML_MAX_DIMS - 1];\n    acl_rstd_nb[0] = sizeof(float);\n    for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {\n        acl_rstd_nb[i] = acl_rstd_nb[i - 1] * acl_rstd_ne[i - 1];\n    }\n    acl_tensor_ptr acl_rstd =\n        get_cache_acl_tensor(ctx, &ctx.rms_norm_zero_tensor_cache.cache, ctx.rms_norm_zero_tensor_cache.size,\n                             acl_rstd_ne, acl_rstd_nb, GGML_TYPE_F32, GGML_MAX_DIMS,\n                             0.0f  // value\n        );\n\n    acl_tensor_ptr acl_xout = ggml_cann_create_tensor(add_node);\n\n    // Create yOut tensor (final output after RMS normalization)\n    acl_tensor_ptr acl_yout = ggml_cann_create_tensor(rms_norm_node);\n\n    // Call fused ADD + RMS_NORM operator\n    GGML_CANN_CALL_ACLNN_OP(ctx, AddRmsNorm, acl_x1.get(), acl_x2.get(), acl_gamma.get(),\n                            eps,  // double type\n                            acl_yout.get(), acl_rstd.get(), acl_xout.get());\n}\n\nvoid ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * k = dst->src[0];\n    ggml_tensor * v = dst->src[1];\n    ggml_tensor * q = dst->src[2];\n    ggml_tensor * g = dst->src[3];\n    ggml_tensor * s = dst->src[4];\n\n    int64_t B = dst->src[4]->ne[1];\n    int64_t T = dst->src[0]->ne[2];\n    int64_t H = dst->src[0]->ne[1];\n    int64_t C = dst->ne[0];\n    int64_t D = C / H;\n    int64_t L = T / B;\n\n    int64_t ne_qkg[2] = { 1, D };\n    int64_t ne_s[2]   = { D, D };\n    int64_t ne_st[2]  = { ne_s[1], ne_s[0] };\n    int64_t ne_vo[2]  = { D, 1 };\n    int64_t ne_q[1]   = { D };\n    size_t  nb_base   = ggml_type_size(k->type);\n    size_t  nb_qkg[2] = { nb_base, nb_base };\n    size_t  nb_s[2]   = { nb_base, D * nb_base };\n    size_t  nb_st[2]  = { nb_s[1], nb_s[0] };\n    size_t  nb_vo[2]  = { nb_base, D * nb_base };\n    size_t  nb_q[1]   = { nb_base };\n\n    const float scale = ggml_get_op_params_f32(dst, 0);\n\n    acl_tensor_ptr acl_s     = ggml_cann_create_tensor(s, s->ne, s->nb, 2, ACL_FORMAT_ND);\n    acl_tensor_ptr new_state = ggml_cann_create_tensor(dst, s->ne, s->nb, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base);\n    cann_copy(ctx, acl_s.get(), new_state.get());\n\n    for (int64_t b = 0; b < B; b++) {\n        for (int64_t h = 0; h < H; h++) {\n            size_t         s_offset = (b * (H * D * D) + h * (D * D)) * nb_base;\n            // D * D\n            acl_tensor_ptr acl_s_new =\n                ggml_cann_create_tensor(dst, ne_s, nb_s, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base + s_offset);\n            acl_tensor_ptr acl_s_new_t =\n                ggml_cann_create_tensor(dst, ne_st, nb_st, 2, ACL_FORMAT_ND, (B * L * H * D) * nb_base + s_offset);\n            for (int64_t l = 0; l < L; l++) {\n                size_t               qkvgo_offset = (b * (L * H * D) + l * (H * D) + h * (D)) * nb_base;\n                // D * 1\n                acl_tensor_ptr       acl_k = ggml_cann_create_tensor(k, ne_qkg, nb_qkg, 2, ACL_FORMAT_ND, qkvgo_offset);\n                acl_tensor_ptr       acl_g = ggml_cann_create_tensor(g, ne_qkg, nb_qkg, 2, ACL_FORMAT_ND, qkvgo_offset);\n                // D\n                acl_tensor_ptr       acl_q = ggml_cann_create_tensor(q, ne_q, nb_q, 1, ACL_FORMAT_ND, qkvgo_offset);\n                // 1 * D\n                acl_tensor_ptr       acl_v = ggml_cann_create_tensor(v, ne_vo, nb_vo, 2, ACL_FORMAT_ND, qkvgo_offset);\n                // D\n                acl_tensor_ptr       acl_o = ggml_cann_create_tensor(dst, ne_q, nb_q, 1, ACL_FORMAT_ND, qkvgo_offset);\n                // k ⊗ v\n                size_t               buf_size = D * D * nb_base;\n                ggml_cann_pool_alloc buffer_allocator(ctx.pool(), buf_size);\n                acl_tensor_ptr       tmp_tensor = ggml_cann_create_tensor(\n                    buffer_allocator.get(), ggml_cann_type_mapping(k->type), nb_base, ne_s, nb_s, 2);\n                aclnn_mul(ctx, acl_k.get(), acl_v.get(), tmp_tensor.get());\n                //s_new = g ⊗ s_old + k ⊗ v\n                aclnn_mul(ctx, acl_s_new.get(), acl_g.get(), nullptr);\n                aclnn_add(ctx, acl_s_new.get(), tmp_tensor.get(), nullptr);\n                // compute output\n                GGML_CANN_CALL_ACLNN_OP(ctx, Mv, acl_s_new_t.get(), acl_q.get(), acl_o.get(), 1);\n                aclnn_muls(ctx, acl_o.get(), scale, nullptr, true);\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cann/aclnn_ops.h",
    "content": "/**\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_ACLNN_OPS\n#define CANN_ACLNN_OPS\n\n#include \"acl_tensor.h\"\n#include \"common.h\"\n\n#include <aclnnop/aclnn_abs.h>\n#include <aclnnop/aclnn_arange.h>\n#include <aclnnop/aclnn_argsort.h>\n#include <aclnnop/aclnn_cat.h>\n#include <aclnnop/aclnn_clamp.h>\n#include <aclnnop/aclnn_cos.h>\n#include <aclnnop/aclnn_exp.h>\n#include <aclnnop/aclnn_gelu.h>\n#include <aclnnop/aclnn_gelu_v2.h>\n#include <aclnnop/aclnn_hardsigmoid.h>\n#include <aclnnop/aclnn_hardswish.h>\n#include <aclnnop/aclnn_leaky_relu.h>\n#include <aclnnop/aclnn_log.h>\n#include <aclnnop/aclnn_logsoftmax.h>\n#include <aclnnop/aclnn_neg.h>\n#include <aclnnop/aclnn_norm.h>\n#include <aclnnop/aclnn_relu.h>\n#include <aclnnop/aclnn_sigmoid.h>\n#include <aclnnop/aclnn_sign.h>\n#include <aclnnop/aclnn_silu.h>\n#include <aclnnop/aclnn_sin.h>\n#include <aclnnop/aclnn_slice.h>\n#include <aclnnop/aclnn_sqrt.h>\n#include <aclnnop/aclnn_tanh.h>\n\n#include <functional>\n#include <unordered_set>\n\n/**\n * @brief   Repeats a ggml tensor along each dimension to match the dimensions\n *          of another tensor.\n *\n * @details This function repeats the elements of a source ggml tensor along\n *          each dimension to create a destination tensor with the specified\n *          dimensions. The operation is performed using the ACL backend and\n *          executed asynchronously on the device.\n *\n * @param   ctx The CANN context used for operations.\n * @param   dst The ggml tensor representing the destination, which op is\n *              GGML_OP_REPEAT and specifies the desired dimensions.\n */\nvoid ggml_cann_repeat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies the Leaky ReLU activation function to a tensor using the CANN\n *          backend.\n *\n * @details This function computes the Leaky ReLU activation for each element of\n *          the input tensor. The Leaky ReLU function allows a small gradient\n *          when the unit is not active (i.e., when the input is negative). The\n *          Leaky ReLU function is defined as:\n *          \\f[\n *              \\text{dst} = \\max(0, src) + \\text{negativeSlope} \\cdot \\min(0,\n *               src)\n *          \\f]\n *          `negativeSlope` is in dst->params.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result of the Leaky ReLU\n *            activation is stored, which op is `GGML_OP_LEAKY_RELU`\n */\nvoid ggml_cann_leaky_relu(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief    Concatenates multiple tensors along a specified dimension using the\n *           CANN backend.\n *\n * @param ctx        The CANN context used for operations.\n * @param tensorList A pointer to the list of tensors to be concatenated.\n * @param dst        The destination tensor where the result of the\n *                   concatenation is stored. dst->op is `GGML_OP_CONCAT`.\n * @param concat_dim The dimension along which the tensors are concatenated.\n *\n * @attention tensorList length should be 2 and the dimension using for concat\n *            default to 1.\n */\nvoid ggml_cann_concat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Generates a sequence of evenly spaced values within a specified\n *          interval for a ggml tensor using the CANN backend.\n *\n * @details This function creates a sequence of numbers over a specified i\n *          nterval, starting from `start`, ending before `stop`, and\n *          incrementing by `step`. The sequence is stored in the destination\n *          tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the generated sequence will be stored.\n *            `start`, 'stop' and 'step' are in dst->op_params and dst->op is\n *            `GGML_OP_ARANGE`.\n */\nvoid ggml_cann_arange(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies a clamp operation to the elements of a ggml tensor using the\n *          CANN backend.\n *\n * @details This function clamps the elements of the input tensor `src` to a\n *          specified range defined by `min` and `max` values. The result is\n *          stored in the destination tensor `dst`. The operation is defined as:\n *          \\f[\n *              y = \\max(\\min(x, max\\_value), min\\_value)\n *           \\f]\n *          where `x` is an element of the input tensor, and `y` is the\n *          corresponding element in the output tensor.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the clamped values will be stored.\n *            dst->op is `GGML_OP_CLAMP`, `min` and `max` value is in dst->params.\n */\nvoid ggml_cann_clamp(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Scales the elements of a ggml tensor by a constant factor using the\n *          CANN backend.\n *\n * @details This function multiplies each element of the input tensor `src` by\n *          a scaling factor `scale`, storing the result in the destination\n *          tensor `dst`. The operation is defined as:\n *          \\f[\n *             dst = src \\times scale\n *          \\f]\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the scaled values will be stored.\n *            dst->op is `GGML_OP_SCALE` and `scale` value is in dst->params.\n */\nvoid ggml_cann_scale(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Sorts the elements of a ggml tensor and returns the indices that\n *          would sort the tensor using the CANN backend.\n *\n * @details This function performs an argsort operation on the input tensor\n *          `src`. It sorts the elements of `src` in either ascending or\n *          descending order, depending on the `GGML_SORT_ORDER_DESC`,\n *          and returns the indices that would sort the original tensor.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the sorted indices will be stored.\n *            dst->op is `GGML_OP_ARGSORT`.\n */\nvoid ggml_cann_argsort(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the Layer Normalization for a ggml tensor using the CANN\n *          backend.\n *\n * @details This function applies the Layer Normalization operation on the\n *          input tensor `src` and stores the result in the destination tensor\n *          `dst`. Layer Normalization normalizes the features at each sample in\n *          a mini-batch independently. It is commonly used in neural networks\n *          to normalize the activations of a layer by adjusting and scaling\n *          the outputs.\n *          The operation is defined as:\n *          \\f[\n *              \\text { out }=\\frac{x-\\mathrm{E}[x]}{\\sqrt{\\text{Var}[x]+eps}}\n *          \\f]\n *          `Var` defaults dst->ne[0]. `eps` is in dst->params.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * @attention `Var` defaults to dst->ne[0].\n */\nvoid ggml_cann_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the L2 Normalization for a ggml tensor using the CANN\n *          backend.\n *\n * @details This function applies the L2 Normalization operation on the\n *          input tensor `src` and stores the result in the destination tensor\n *          `dst`. L2 Normalization scales the input tensor such that the\n *          L2 norm along the specified dimension equals 1. This operation\n *          is commonly used in neural networks for feature normalization\n *          and vector scaling.\n *          The operation is defined as:\n *          \\f[\n *              \\text{out} = \\frac{x}{\\sqrt{\\sum{x^2}}}\n *          \\f]\n *          The normalization is performed along the last dimension by default.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n * @attention The normalization is performed along the last dimension of the\n *            input tensor by default.\n */\nvoid ggml_cann_l2_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the Cross Entropy Loss for a ggml tensor using the CANN\n *          backend.\n *\n * @details This function computes the cross entropy loss between the predicted\n *          logits and target probability distributions. The operation follows\n *          the same computation pattern as the CPU implementation:\n *          1. Applies log_softmax to the logits along the class dimension\n *          2. Element-wise multiplication with target distributions\n *          3. Summation along the class dimension to get per-sample losses\n *          4. Global summation and scaling by -1/nr to get final loss\n *\n *          The computation can be expressed as:\n *          \\f[\n *              \\text{loss} = -\\frac{1}{N} \\sum_{i=1}^{N} \\sum_{j=1}^{C} y_{ij} \\cdot \\log(\\text{softmax}(x_{ij}))\n *          \\f]\n *          where \\f$N\\f$ is the total number of samples, \\f$C\\f$ is the number\n *          of classes, \\f$x\\f$ are the logits, and \\f$y\\f$ are the target\n *          probability distributions.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the computed loss will be stored.\n *            This should be a scalar tensor containing the final loss value.\n *\n * @note This implementation computes cross entropy between probability\n *       distributions, not the typical classification cross entropy that\n *       expects class indices as targets. Both input tensors (src0 and src1)\n *       should have the same shape and represent probability distributions\n *       over the class dimension.\n * @note The function expects two source tensors:\n *       - dst->src[0]: Logits tensor (before softmax)\n *       - dst->src[1]: Target probability distributions tensor\n * @note The computation is performed using CANN backend operators including\n *       LogSoftmax, Mul, ReduceSum, and Muls for the final scaling.\n */\nvoid ggml_cann_cross_entropy_loss(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief  Computes the Group Normalization for a ggml tensor using the CANN\n *         backend.\n *\n * @brief  This function applies the Group Normalization operation on the input\n *         tensor `src` and stores the result in the destination tensor `dst`.\n *         Group Normalization divides the channels into groups and normalizes\n *         the features within each group across spatial locations.\n *         It is commonly used in convolutional neural networks to improve\n *         training stability and performance.\n *         The operation is defined as:\n *         \\f[\n *             \\text { out }=\\frac{x-\\mathrm{E}[x]}{\\sqrt{\\text{Var}[x]+eps}}\n *         \\f]\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n *            `n_groups` is in dst->params, which split C channel to `n_groups`.\n *            dst->op is `GGML_OP_GROUP_NORM`.\n *\n * @attention eps defaults to 1e-6f.\n */\nvoid ggml_cann_group_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the accumulation of tensors using the CANN backend.\n *\n * @details This function performs an accumulation operation on two tensors.\n *          Depending on the `inplace` flag, it either updates the destination\n *          tensor `dst` in place by adding `alpha * src1` to it, or it creates\n *          a new tensor as the result of `src0 + alpha * src1` and stores it in\n *          `dst`.\n *          The operation is defined as:\n *          \\f[\n *               dst = src0 + alpha \\times src1\n *          \\f]\n *          if `inplace` is `true`, `src0` is equal to 'dst'.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the accumulated values will be stored.\n *            `inplace` is in dst->params, and dst->op is `GGML_OP_ACC`.\n */\nvoid ggml_cann_acc(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the sum of elements along the last dimension of a ggml tensor\n *          using the CANN backend.\n *\n * @details This function performs a reduction sum operation along the last\n *          dimension of the input tensor `src`. The result of the sum is stored\n *          in the destination tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the reduced values will be stored。\n *            dst->op is `GGML_OP_SUM_ROWS`.\n *\n * @attention `reduce_dims` defaults to 3, which means the last dimension.\n */\nvoid ggml_cann_sum_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the sum of elements in a ggml tensor.\n *\n * @details This function performs a reduction sum operation along the last\n *          dimension of the input tensor `src`. The result of the sum is stored\n *          in the destination tensor `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the reduced values will be stored。\n *\n */\n\nvoid ggml_cann_sum(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Upsamples a ggml tensor using nearest neighbor interpolation using\n *          the CANN backend.\n *\n * @details This function performs upsampling of the input tensor `src` using\n *          nearest neighbor interpolation. The upsampling is applied to the\n *          height and width dimensions (last two dimensions) of the tensor. The\n *          result is stored in the destination tensor `dst`, which must have\n *          the appropriate dimensions for the upsampled output.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the upsampled values will be stored.\n *            dst->op is `GGML_OP_UPSCALE`.\n */\nvoid ggml_cann_upsample_nearest2d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Pads a ggml tensor to match the dimensions of the destination tensor\n *          using the CANN backend.\n *\n * @details This function pads the input tensor `src` so that it matches the\n *          dimensions of the destination tensor `dst`. The amount of padding\n *          is calculated based on the difference in sizes between `src` and\n *          `dst` along each dimension. The padded tensor is stored in `dst`.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor, which specifies the target dimensions for\n *            padding. dst->op is `GGML_OP_PAD`.\n */\nvoid ggml_cann_pad(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Executes a 2D pooling operation on a ggml tensor using the CANN\n *          backend.\n *\n * @details This function dispatches the execution of a 2D pooling operation on\n *          the input tensor `dst`. The type of pooling (average or max) is\n *          determined by the `op` parameter, which is read from the operation\n *          parameters of `dst`. The function supports average pooling\n *          (`GGML_OP_POOL_AVG`) and max pooling (`GGML_OP_POOL_MAX`). If an\n *          invalid operation is encountered, the function asserts a failure.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor on which the pooling operation is to be\n *            performed. dst->op is `GGML_OP_POOL_2D`.\n */\nvoid ggml_cann_pool2d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Duplicates a ggml tensor using the CANN backend.\n *\n * @details This function duplicates the contents of the source tensor `src` to\n *          the destination tensor `dst`. The function supports various tensor\n *          types and configurations, including handling of extra data, type\n *          conversions, and special cases for contiguous and non-contiguous\n *          tensors.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the duplicated data will be stored.\n *            dst->op is `GGML_OP_DUP`\n *\n * @attention Only support Fp16/FP32. Not support when src and dst have\n *            different shape and dst is no-contiguous.\n * @note:     This func need to simplify.\n */\nvoid ggml_cann_dup(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the Root Mean Square (RMS) normalization of a ggml tensor\n *          using the CANN backend.\n *\n * @details This function applies RMS normalization to the input tensor `src`\n *          and stores the result in the destination tensor `dst`. RMS\n *          normalization involves computing the root mean square of the input\n *          tensor along a specified dimension and then dividing each element of\n *          the tensor by this value, adjusted by a small epsilon value to\n *          prevent division by zero.\n *          The operation is defined as:\n *          \\f[\n *               \\text{RmsNorm}\\left(x_i\\right)=\\frac{x_i}{\\text{Rms}(\\mathbf{x})} g_i,\n *               \\quad \\text { where } \\text{Rms}(\\mathbf{x})=\\sqrt{\\frac{1}{n} \\sum_{i=1}^n x_i^2+e p s}\n *          \\f]\n *          `eps` is in dst->op_params.\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the normalized values will be stored.\n *            dst->op is `GGML_OP_RMS_NORM`.\n */\nvoid ggml_cann_rms_norm(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies a diagonal mask to the tensor with a specified value.\n *\n * @details This function creates a mask tensor filled with ones, then applies\n *          an upper triangular and lower triangular operation to it based on\n *          the number of past elements specified. Afterward, it adds the masked\n *          tensor to the destination tensor in-place.\n *\n * @param ctx The backend CANN context used for operations.\n * @param dst The destination tensor where the result will be stored. dst->op is\n *            `GGML_OP_DIAG_MASK`\n * @param value The value to use for masking.\n */\nvoid ggml_cann_diag_mask(ggml_backend_cann_context & ctx, ggml_tensor * dst, float value);\n\n/**\n * @brief   Performs an image-to-column transformation on the input tensor.\n *\n * @details This function takes an input tensor and applies an image-to-column\n *          operation, converting spatial dimensions into column-like\n *          structures suitable for convolutional operations. It supports both\n *          half-precision (F16) and single-precision (F32) floating-point data\n *          types.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor that stores the result of the operation.\n *            dst->op is `GGML_OP_IM2COL`.\n */\nvoid ggml_cann_im2col(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes time step embeddings using sine and cosine functions.\n *\n * @details This function calculates time step embeddings by applying sine and\n *          cosine transformations to a given input tensor, which is typically\n *          used in temporal models like diffusion models or transformers to\n *          encode time information effectively.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the result of the embedding operation\n *            will be stored. dst->op is `GGML_OP_TIMESTEP_EMBEDDING`.\n */\nvoid ggml_cann_timestep_embedding(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n// @see ggml_cann_dup.\nvoid ggml_cann_cpy(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the softmax activation with optional masking.\n *\n * @details This function computes the softmax activation over the input tensor,\n *          optionally applying a mask and scaling factor. It supports both FP16\n *          and FP32 data types and can handle masking by broadcasting the mask\n *          across rows if necessary.\n *          The function performs the following steps:\n *          1. Multiplies the input tensor by a scale factor.\n *          2. Optionally casts the mask tensor to FP32 if it is in FP16 format.\n *          3. Broadcasts the mask tensor if its dimensions do not match the\n *             input tensor's dimensions.\n *          4. Adds the mask to the scaled input tensor.\n *          5. Applies the softmax activation function along the specified\n *             dimension.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the result will be stored. dst->op is\n *            `GGML_OP_SOFTMAX`.\n */\nvoid ggml_cann_softmax(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Extracts specific rows from a tensor based on indices.\n *\n * @details This function retrieves rows from a source tensor src0 according to\n *          the indices provided in another tensor src1 and stores the result in\n *          a destination tensor (\\p dst).\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the extracted rows will be stored.\n */\nvoid ggml_cann_get_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Writes specific rows into a tensor at positions specified by indices.\n *\n * @details This function copies rows from a source tensor into a destination\n *          tensor (\\p dst) at the positions indicated by the indices in another\n *          tensor.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the specified rows will be updated.\n */\nvoid ggml_cann_set_rows(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Executes matrix multiplication for the given tensor.\n *\n * @details This function performs matrix multiplication on the source tensors\n *          associated with the destination tensor. It supports matrix\n *          multiplication F32, F16, and Q8_0.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor for storing the result of the matrix\n *            multiplication. dst->op is `GGML_OP_MUL_MAT`.\n */\nvoid ggml_cann_mul_mat(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies Rotary Positional Embedding (RoPE) to the input tensor.\n *\n * @details This function implements the RoPE mechanism, which is a method to\n *          encode positional information into sequence data, particularly\n *          useful in transformer models. It supports both F32 and F16 data\n *          types.\n *\n * @param ctx The backend CANN context for executing operations.\n * @param dst The destination tensor where the RoPE-transformed data will be\n *            stored. dst->op is `GGML_OP_ROPE`.\n *\n * @note The function currently does not support cases where the n_dims is less\n *       than the input tensor's first dimension.\n * @note The function currently does not support cases where the freq_factors is\n *       not NULL.\n * @note The function currently does not support cases where the ext_factor is\n *       not equal 0.\n * @note The function currently does not support cases where the freq_scale is\n *       not equal 1.\n */\nvoid ggml_cann_rope(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the index of the maximum value along the specified dimension\n *          of a ggml tensor using the CANN backend.\n *\n * @details This function performs an argmax operation on the input tensor.\n *          It finds the index of the maximum value along the specified axis\n *          and stores these indices in the destination tensor `dst`. The\n *          operation is executed using the CANN backend for optimized performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the indices of the maximum values will\n *            be stored. dst->op is `GGML_OP_ARGMAX`.\n */\nvoid ggml_cann_argmax(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Adds two tensors element-wise and stores the result in a destination\n * tensor.\n *\n * This function performs the operation:\n * \\f[\n *    dst = acl\\_src0 + alpha \\times acl\\_src1\n * \\f]\n * where alpha is a scalar value and defaults to 1.0f.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src0 The first source tensor.\n * @param acl_src1 The second source tensor.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_add(ggml_backend_cann_context & ctx,\n               aclTensor *                 acl_src0,\n               aclTensor *                 acl_src1,\n               aclTensor *                 acl_dst = nullptr);\n\n/**\n * @brief Sub two tensors element-wise and stores the result in a destination\n * tensor.\n *\n * This function performs the operation:\n * \\f[\n *    dst = acl\\_src0 - alpha \\times acl\\_src1\n * \\f]\n * where alpha is a scalar value and defaults to 1.0f.\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src0 The first source tensor.\n * @param acl_src1 The second source tensor.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_sub(ggml_backend_cann_context & ctx,\n               aclTensor *                 acl_src0,\n               aclTensor *                 acl_src1,\n               aclTensor *                 acl_dst = nullptr);\n\n/**\n * @brief Performs element-wise multiplication of two tensors and stores the\n * result in a destination tensor.\n *\n * This function performs element-wise multiplication of the tensors `acl_src`\n * and `acl_other` and stores the result in the destination tensor `acl_dst`.\n * The operation is defined as:\n * \\f[\n *     \\text {acl_dst }_i=\\text {acl_src }_i \\times \\text {acl_other }_i\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The first tensor for element-wise multiplication.\n * @param acl_other The second tensor for element-wise multiplication.\n * @param acl_dst The destination tensor where the result will be stored.\n */\nvoid aclnn_mul(ggml_backend_cann_context & ctx,\n               aclTensor *                 acl_src,\n               aclTensor *                 acl_other,\n               aclTensor *                 acl_dst = nullptr);\n\n/**\n * @brief Matrix division, optionally in-place.\n *\n * This function division each element of the source tensor `acl_src` by the\n * tensor `acl_other` and stores the result in the destination tensor `acl_dst`.\n * If `inplace` is true, `acl_dst` will not be used and the operation is\n * performed in-place on `acl_src`. The operation is defined as: \\f[\n *     \\text{dst}_i = \\frac{\\text{acl_src}_i}{\\text{acl_other}_i}\n * \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src Numerator tensor..\n * @param acl_other Denominator tensor.\n * @param acl_dst The destination tensor where the result will be stored if\n * `inplace` is false.\n * @param inplace Flag indicating whether to perform the operation in-place on\n * `acl_src`.\n */\nvoid aclnn_div(ggml_backend_cann_context & ctx,\n               aclTensor *                 acl_src,\n               aclTensor *                 acl_other,\n               aclTensor *                 acl_dst = nullptr);\n\n/**\n * @brief Applies element-wise cosine function to the elements of a tensor.\n *\n * This function computes the cosine of each element in the source tensor\n * `acl_src` and stores the result in the destination tensor `acl_dst`. The\n * operation is defined as: \\f[ \\text {acl_dst }_i=\\cos \\left(\\text {acl_src\n * }_i\\right) \\f]\n *\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the cosine function will be\n * applied.\n * @param acl_dst The destination tensor where the cosine results will be\n * stored.\n */\nvoid aclnn_cos(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst);\n\n/**\n * @brief Applies element-wise sine function to the elements of a tensor.\n *\n * This function computes the sine of each element in the source tensor\n `acl_src`\n * and stores the result in the destination tensor `acl_dst`.\n * The operation is defined as:\n * \\f[\n *     \\text {acl_dst }_i=\\sin \\left(\\text {acl_src }_i\\right)\n * \\f]\n\n * @param ctx The context for the CANN backend operations.\n * @param acl_src The source tensor on which the sine function will be applied.\n * @param acl_dst The destination tensor where the sine results will be stored.\n */\nvoid aclnn_sin(ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst);\n\n/**\n * @brief Prepares broadcast-compatible ACL tensors for two input tensors and one\n * output tensor.\n *\n * This function checks whether broadcasting is needed between `src0` and `src1`.\n * If broadcasting is required, it calculates the proper shapes and creates\n * ACL tensors with broadcast parameters. Otherwise, it directly creates ACL tensors\n * based on the original tensor shapes.\n *\n * @param src0     The first input tensor (reference shape).\n * @param src1     The second input tensor (possibly broadcasted).\n * @param dst      The destination/output tensor.\n * @param acl_src0 Output pointer to the created ACL tensor corresponding to src0.\n * @param acl_src1 Output pointer to the created ACL tensor corresponding to src1.\n * @param acl_dst  Output pointer to the created ACL tensor corresponding to dst.\n */\nvoid bcast_shape(ggml_tensor *    src0,\n                 ggml_tensor *    src1,\n                 ggml_tensor *    dst,\n                 acl_tensor_ptr & acl_src0,\n                 acl_tensor_ptr & acl_src1,\n                 acl_tensor_ptr & acl_dst);\n\n/**\n * @brief   Computes the 1D transposed convolution (deconvolution) of a ggml\n * tensor using the CANN backend.\n *\n * @details This function performs a 1D transposed convolution (also known as\n * deconvolution) operation on the input tensor. The computed result is stored\n * in the destination tensor `dst`. The operation is optimized using the CANN\n * backend for improved performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the transposed convolution result\n * will be stored. dst->op is `GGML_OP_CONV_TRANSPOSE_1D`.\n */\nvoid ggml_cann_conv_transpose_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies the ELU (Exponential Linear Unit) activation to a ggml tensor\n * using the CANN backend.\n *\n * @details This function performs an element-wise ELU activation on the input\n *          tensor.\n *          The result is written to the destination tensor `dst` in-place.\n *          The ELU function is defined as:\n *\n *          \\text{ELU}(x) =\n *          \\begin{cases}\n *          x, & \\text{if } x > 0 \\\\\n *          \\alpha \\left( \\exp(x) - 1 \\right), & \\text{if } x \\leq 0\n *          \\end{cases}\n *\n *          where α (alpha) is a hyperparameter, typically set to 1.0.\n *          This operation is optimized using the CANN backend for high-performance\n *          inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the ELU-activated result will be stored.\n *            dst->op is expected to be `GGML_OP_ELU`.\n */\nvoid ggml_cann_elu(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Computes the mean of a ggml tensor element-wise using the CANN backend.\n *\n * @details This function calculates the element-wise mean of the input tensor.\n *          The result is written to the destination tensor `dst`.\n *          The mean is computed by averaging the values across the entire tensor.\n *\n *          This operation is optimized using the CANN backend for high-performance inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the mean result will be stored.\n *            dst->op is expected to be `GGML_OP_MEAN`.\n */\nvoid ggml_cann_mean(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies 1D reflect padding to a ggml tensor using the CANN backend.\n *\n * @details This function performs 1D reflect padding on the input tensor.\n *          The amount of padding on each side is specified by parameters stored in `dst->op_params`.\n *          The operation reflects the values at the borders of the tensor to generate the padded output.\n *\n *          This operation is optimized using the CANN backend for high-performance inference or training.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the padded result will be stored.\n *            dst->op is expected to be `GGML_OP_PAD_REFLECT_1D`.\n */\nvoid ggml_cann_pad_reflect_1d(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Counts the number of equal elements in two ggml tensors using the CANN backend.\n *\n * @details This function performs an element-wise comparison between two input tensors,\n *          and counts the number of positions where the elements are equal. The result is\n *          stored in the destination tensor `dst` as a scalar.\n *\n *          The operation is optimized using the CANN backend, making it suitable for\n *          high-performance inference or training scenarios.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n *            dst->op is expected to be `GGML_OP_COUNT_EQUAL`.\n */\nvoid ggml_cann_count_equal(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Applies the Step activation function to a ggml tensor using the CANN backend.\n *\n * @details This function applies a step function element-wise to the input tensor, where\n *          each element is transformed to 1.0 if it is greater than 0, and 0.0 otherwise.\n *          The result is stored in the destination tensor `dst`.\n *\n *          This operation is accelerated using the CANN backend to improve runtime performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n *            dst->op is expected to be `GGML_OP_STEP`.\n */\nvoid ggml_cann_step(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief   Performs the Flash Attention extended operator using the CANN backend.\n *\n * @details This function implements the memory-efficient Flash Attention algorithm\n *          for computing scaled dot-product attention with hardware acceleration.\n *          The result is stored in the destination tensor `dst`.\n *\n *          This operation is accelerated using the CANN backend to improve runtime performance.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the result will be stored.\n *            dst->op is expected to be `GGML_OP_FLASH_ATTN_EXT`.\n */\nvoid ggml_cann_flash_attn_ext(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Forward Gated Linear Attention on the CANN backend.\n *\n * Expects dst->src[0..4] = {k, v, q, g, s} with shape conventions:\n *   k, v, q, g: [D] with outer dims T x H batched as ne[2]=T, ne[1]=H\n *   s: initial state [B, H, D, D], where B is batch and D=C/H\n * dst holds both outputs (o) and updated state; a scale factor is read from op params.\n *\n * The kernel updates per time step l: S_new = g ⊗ S_old + k ⊗ v, then computes o = (S_new^T q) * scale.\n *\n * @param ctx Backend context providing stream/allocator utilities.\n * @param dst Output tensor; src deps are k, v, q, g, s as above.\n */\nvoid ggml_cann_gated_linear_attn(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Launches an asynchronous task using the memory allocator.\n *\n * This macro submit an asynchronous task on the specified stream.\n * The task uses memory allocated by the allocator. It is guaranteed\n * that the memory will not be accessed by other tasks until this task\n * completes, due to the sequential execution order within the same stream.\n *\n * @param OP_NAME aclnn operator name.\n * @param args Additional arguments required by the task.\n *\n * @note\n * Memory from the allocator will be \"freed\" immediately and can be\n * reallocated to other pointers. However, it won't be accessed by any\n * other task before this asynchronous task ends, because all tasks in the\n * same stream are executed in queue order.\n */\n\n#    define GGML_CANN_CALL_ACLNN_OP(CTX, OP_NAME, ...)                                           \\\n        do {                                                                                     \\\n            uint64_t        workspaceSize = 0;                                                   \\\n            aclOpExecutor * executor;                                                            \\\n            void *          workspaceAddr = nullptr;                                             \\\n            ACL_CHECK(aclnn##OP_NAME##GetWorkspaceSize(__VA_ARGS__, &workspaceSize, &executor)); \\\n            /* workspace should alloced in main thread to keep malloc order when using vmm. */   \\\n            if (workspaceSize > 0) {                                                             \\\n                ggml_cann_pool_alloc workspace_allocator(CTX.pool(), workspaceSize);             \\\n                workspaceAddr = workspace_allocator.get();                                       \\\n            }                                                                                    \\\n            ACL_CHECK(aclnn##OP_NAME(workspaceAddr, workspaceSize, executor, CTX.stream()));     \\\n        } while (0)\n\n/**\n * @brief   Performs sparse expert-based matrix multiplication using the CANN backend.\n *\n * @details This function implements a MoE-style batched matrix multiplication, where each input token\n *          is routed to one or more experts, and each expert corresponds to a specific [D, M] weight matrix\n *          in the source tensor `src0`. The routing indices are provided via the `ids` tensor.\n *\n *          For each token (from `src1`), the function selects the corresponding expert(s) as specified by `ids`,\n *          performs the matrix multiplication with the selected expert's weight submatrix (from `src0`),\n *          and stores the results in `dst`. This operation is optimized and executed on the CANN backend.\n *\n *          Dimensions:\n *              - src0: [D, M, A, 1], where A is the number of experts\n *              - src1: [D, B, N, 1], where N is batch size and B is the slot count per sample\n *              - ids : [K, N],       where K is the number of experts each token is routed to\n *              - dst : [M, K, N, 1], output tensor storing the result of expert × token multiplication\n *\n *          The function handles two main modes:\n *              - If `ne12 == 1`, a simpler per-token loop is used.\n *              - TODO: If `ne12 > 1`, grouped multiplication and memory copying is used for efficiency.\n *\n * @param ctx The CANN context used for operations.\n * @param dst The destination tensor where the expert-weighted token outputs are stored.\n *            Expected to be of shape [M, K, N, 1].\n */\nvoid ggml_cann_mul_mat_id(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Performs fused ADD + RMS_NORM operation using the CANN backend.\n *\n * This function fuses the ADD and RMS_NORM operations into a single kernel call\n * for better performance. It first adds two input tensors (x1 + x2), then applies\n * RMS normalization to the result.\n *\n * @param ctx The context for the CANN backend operations.\n * @param dst The ADD operation node, contains the two input tensors to be added.\n * @param rms_norm_tensor The RMS_NORM operation node, contains the gamma weights\n *                        and epsilon parameter.\n */\nvoid ggml_cann_op_add_rms_norm_fused(ggml_backend_cann_context & ctx,\n                                     ggml_tensor *               add_node,\n                                     ggml_tensor *               rms_norm_node);\n\n/**\n * @brief   Check whether a tensor is a weight tensor for matrix multiplication.\n *\n * @details Checks whether the given tensor serves as weight parameters in matrix multiplication operations,\n *          typically within neural network layers. The function maintains a static set of canonical weight\n *          naming suffixes from Transformer-based architectures. Uses substring matching to identify weight\n *          tensors even with hierarchical naming patterns.\n *\n * @param tensor Pointer to the target ggml_tensor object (const-qualified).\n */\nstatic bool is_matmul_weight(const ggml_tensor * tensor) {\n    std::string                                  name = ggml_get_name(tensor);\n    static const std::unordered_set<std::string> weight_suffixes{ \"output.weight\",      \"attn_q.weight\",\n                                                                  \"attn_k.weight\",      \"attn_v.weight\",\n                                                                  \"attn_output.weight\", \"ffn_gate.weight\",\n                                                                  \"ffn_up.weight\",      \"ffn_down.weight\" };\n\n    for (const auto & suffix : weight_suffixes) {\n        if (name.find(suffix) != std::string::npos) {\n            return true;\n        }\n    }\n    return false;\n}\n\n/**\n * @brief Applies a element-wise operation to two input tensors using the CANN\n * backend.\n *\n * This templated function takes a binary operator and applies it to two source\n * tensors\n * associated with the destination tensor. The function handles broadcasting as\n * needed.\n *\n * @tparam binary_op A callable object (e.g., lambda or function pointer) representing\n *         the binary operation to be performed. It must take three arguments:\n *         (ggml_backend_cann_context&, aclTensor*, aclTensor*, aclTensor*).\n *\n * @param ctx The CANN backend context used to manage execution and resources.\n * @param dst The destination tensor.\n */\ntemplate <auto binary_op> void ggml_cann_binary_op(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src0 = dst->src[0];\n    ggml_tensor * src1 = dst->src[1];\n\n    acl_tensor_ptr acl_src0, acl_src1, acl_dst;\n\n    // Need bcast\n    bcast_shape(src0, src1, dst, acl_src0, acl_src1, acl_dst);\n    binary_op(ctx, acl_src0.get(), acl_src1.get(), acl_dst.get());\n}\n\n/**\n * @brief Applies a unary operation to an input tensor using the CANN backend.\n *\n * This templated function applies a unary operator to the source tensor of `dst`\n * and stores the result in the destination tensor.\n *\n * @tparam unary_op A callable with the signature:\n *         void(ggml_backend_cann_context&, aclTensor *, aclTensor *)\n *         where the first aclTensor is the source and the second is the destination.\n * @param ctx The CANN backend context for managing resources and execution.\n * @param dst The destination tensor. Its src[0] is treated as the input tensor.\n */\ntemplate <void unary_op(ggml_backend_cann_context &, aclTensor *, aclTensor *)>\nvoid ggml_cann_op_unary(ggml_backend_cann_context & ctx, ggml_tensor * dst) {\n    ggml_tensor * src = dst->src[0];\n\n    acl_tensor_ptr acl_src = ggml_cann_create_tensor(src);\n    acl_tensor_ptr acl_dst = ggml_cann_create_tensor(dst);\n\n    unary_op(ctx, acl_src.get(), acl_dst.get());\n}\n\n/**\n * @brief Applies a unary operation to a ggml tensor using the CANN backend.\n *\n * @details This function applies a unary operation to the input tensor using\n * a user-provided lambda or callable `unary_op`. The lambda receives the\n * CANN backend context and two ACL tensors: the source and the destination.\n *\n * Internally, this function handles the conversion from GGML tensors to ACL tensors,\n * calls the provided unary op, and manages resource cleanup. The input is assumed\n * to be `dst->src[0]`, and the result is written to `dst`.\n *\n * This utility simplifies writing unary op wrappers by abstracting tensor preparation.\n *\n * @param unary_op A callable that performs the unary operation using CANN ACL APIs.\n * @param ctx The CANN context for operation execution.\n * @param dst The destination ggml_tensor where the result will be stored.\n *            The input tensor is assumed to be `dst->src[0]`.\n *\n * @see GGML_CANN_CALL_OP_UNARY\n */\nvoid ggml_cann_op_unary(std::function<void(ggml_backend_cann_context &, aclTensor *, aclTensor *)> unary_op,\n                        ggml_backend_cann_context &                                                ctx,\n                        ggml_tensor *                                                              dst);\n\nvoid ggml_cann_ssm_conv(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n\n/**\n * @brief Applies a gated (GLU-style) unary operation using the CANN backend.\n *\n * @details This function performs a gated activation such as GEGLU or ReGLU.\n * It supports two input modes:\n *\n * 1. **Dual input mode**: `dst->src[0]` and `dst->src[1]` are both valid tensors.\n *    These are used directly as the value and gate tensors.\n *\n * 2. **Packed input mode**: Only `dst->src[0]` is valid, and it is assumed to\n *    contain a concatenation of value and gate along the first dimension. This tensor\n *    will be split into two equal halves to form the value and gate inputs.\n *\n * The function applies a user-provided unary operation (e.g., GELU) to the value tensor,\n * then multiplies the result in-place with the gate tensor:\n *\n * @code\n * dst = unary_op(value) * gate;\n * @endcode\n *\n * The `swapped` parameter (from `dst->op_params[1]`) allows flipping the\n * order of value/gate in the packed input case.\n *\n * @param unary_op A callable that performs the unary operation using CANN ACL APIs.\n *                 It receives (ctx, acl_value_tensor, acl_output_tensor).\n * @param ctx      The CANN context used for execution.\n * @param dst      The destination ggml_tensor. Source tensors are in `dst->src[0]` and optionally `src[1]`.\n *\n * @see GGML_CANN_CALL_OP_UNARY_GATED\n */\nvoid ggml_cann_op_unary_gated(std::function<void(ggml_backend_cann_context &, aclTensor *, aclTensor *)> unary_op,\n                              ggml_backend_cann_context &                                                ctx,\n                              ggml_tensor *                                                              dst);\n\n/**\n * @brief Helper macro to call a unary ACL operator via ggml_cann_op_unary.\n *\n * This macro wraps the specified ACLNN unary operator name into a lambda expression,\n * and passes it to `ggml_cann_op_unary`, which handles the common logic for executing\n * unary ops in the CANN backend.\n *\n * Internally, this macro expands to a lambda like:\n * @code\n * [](ggml_backend_cann_context& ctx, aclTensor* acl_src, aclTensor* acl_dst) {\n *     GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);\n * };\n * @endcode\n *\n * This lambda is then passed to `ggml_cann_op_unary`, which applies the operation.\n *\n * @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.\n *\n * @see ggml_cann_op_unary\n * @see GGML_CANN_CALL_ACLNN_OP\n */\n#    define GGML_CANN_CALL_OP_UNARY(OP_NAME)                                                              \\\n        do {                                                                                              \\\n            auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { \\\n                GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);                                  \\\n            };                                                                                            \\\n            ggml_cann_op_unary(lambda, ctx, dst);                                                         \\\n        } while (0)\n\n/**\n * @brief Helper macro to call a gated unary ACL operator via ggml_cann_op_unary_gated.\n *\n * This macro wraps the specified ACLNN unary operator name into a lambda expression,\n * and passes it to `ggml_cann_op_unary_gated`, which handles the common logic for\n * executing gated unary ops in the CANN backend.\n *\n * Internally, this macro expands to a lambda like:\n * @code\n * [](ggml_backend_cann_context& ctx, aclTensor* acl_src, aclTensor* acl_dst) {\n *     GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);\n * };\n * @endcode\n *\n * This lambda is then passed to `ggml_cann_op_unary_gated`, which applies the operation.\n *\n * @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.\n *\n * @see ggml_cann_op_unary_gated\n * @see GGML_CANN_CALL_ACLNN_OP\n */\n#    define GGML_CANN_CALL_OP_UNARY_GATED(OP_NAME)                                                        \\\n        do {                                                                                              \\\n            auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) { \\\n                GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);                                  \\\n            };                                                                                            \\\n            ggml_cann_op_unary_gated(lambda, ctx, dst);                                                   \\\n        } while (0)\n\n#endif  // CANN_ACLNN_OPS\n\n/**\n * @brief Performs outer product operation on two ggml tensors using the CANN backend.\n *\n * @details This function computes the outer product of two input tensors (src0 and src1)\n * and stores the result in the destination tensor. The outer product operation is defined as:\n * dst[i,j,k,l] = sum_m (src0[i,m,k,l] * src1[j,m,k,l])\n *\n * The function supports multiple data types including F32, F16. For floating-point\n * types, it uses batch matrix multiplication for efficient computation.\n *\n * The implementation handles 4D tensor broadcasting and batch processing automatically.\n *\n * @param ctx The CANN backend context for operation execution and memory management.\n * @param dst The destination ggml_tensor where the outer product result will be stored.\n *            The input tensors are assumed to be `dst->src[0]` and `dst->src[1]`.\n *\n * @see GGML_CANN_CALL_ACLNN_OP for CANN operator invocation\n */\nvoid ggml_cann_out_prod(ggml_backend_cann_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cann/common.h",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#ifndef CANN_COMMON_H\n#define CANN_COMMON_H\n\n#include \"../ggml-impl.h\"\n#include \"../include/ggml-cann.h\"\n#include \"../include/ggml.h\"\n\n#include <acl/acl.h>\n#include <unistd.h>\n\n#include <atomic>\n#include <condition_variable>\n#include <cstdio>\n#include <functional>\n#include <iostream>\n#include <list>\n#include <map>\n#include <memory>\n#include <mutex>\n#include <optional>\n#include <string>\n#include <thread>\n#include <vector>\n\n#define MATRIX_ROW_PADDING    512\n#define GGML_CANN_MAX_STREAMS 8\n\n/**\n * @brief Handles CANN-related errors by printing an error message and\n *        terminating the program.\n * @param stmt The statement that caused the error.\n * @param func The function in which the error occurred.\n * @param file The file in which the error occurred.\n * @param line The line number at which the error occurred.\n * @param msg The error message.\n */\n[[noreturn]] void ggml_cann_error(const char * stmt, const char * func, const char * file, int line, const char * msg);\n\n/**\n * @brief Checks the result of a CANN function call and invokes the error\n *        handler if the call fails.\n * @param stmt The CANN function call to check.\n * @param success The success code that indicates the call was successful.\n * @param error_fn The function to call to retrieve the error message.\n */\n#define ACL_CHECK_GEN(stmt, success, error_fn)                                \\\n    do {                                                                      \\\n        int err_code = (stmt);                                                \\\n        if (err_code != (success)) {                                          \\\n            ggml_cann_error(#stmt, __func__, __FILE__, __LINE__, error_fn()); \\\n        }                                                                     \\\n    } while (0);\n\n#define ACL_CHECK(stmt) ACL_CHECK_GEN(stmt, 0, aclGetRecentErrMsg)\n\n/**\n * @brief Contains information about CANN devices.\n */\nstruct ggml_cann_device_info {\n    /**\n     * @brief Number of CANN devices available.\n     */\n    int32_t device_count;\n\n    /**\n     * @brief Information about a single CANN device.\n     */\n    struct cann_device_info {\n        int    cc;              /**< Compute capability.                   */\n        size_t smpb;            /**< Maximum shared memory per block.      */\n        bool   vmm;             /**< Virtual memory support.               */\n        size_t vmm_granularity; /**< Granularity of virtual memory.        */\n        size_t total_vram;      /**< Total video RAM available on the device. */\n    };\n\n    cann_device_info devices[GGML_CANN_MAX_DEVICES] = {}; /**< Array of CANN device information. */\n};\n\nconst ggml_cann_device_info & ggml_cann_info();\n\nvoid    ggml_cann_set_device(int32_t device);\n\nstd::optional<std::string> get_env_as_lowercase(const std::string & name);\nbool                       parse_bool(const std::string & value);\nint                        parse_integer(const std::string & value);\n\n/**\n * @brief Abstract base class for memory pools used by CANN.\n */\nstruct ggml_cann_pool {\n    /**\n     * @brief Virtual destructor for the memory pool.\n     */\n    virtual ~ggml_cann_pool() = default;\n\n    /**\n     * @brief Allocates memory from the pool.\n     *\n     * @param size         The size of the memory block to allocate.\n     * @param actual_size  Pointer to a variable where the actual allocated size\n     *                     will be stored.\n     * @return             Pointer to the allocated memory block.\n     */\n    virtual void * alloc(size_t size, size_t * actual_size) = 0;\n\n    /**\n     * @brief Frees a previously allocated memory block.\n     *\n     * @param ptr   Pointer to the memory block to free.\n     * @param size  Size of the memory block to free.\n     * @note Note that all CANN opertors are running async. Make sure memory is\n     *       still avaiable before this operator finished.\n     */\n    virtual void free(void * ptr, size_t size) = 0;\n};\n\n/**\n * @brief RAII wrapper for managing memory allocations from a CANN memory pool.\n */\nstruct ggml_cann_pool_alloc {\n    ggml_cann_pool * pool        = nullptr; /**< Pointer to the memory pool. */\n    void *           ptr         = nullptr; /**< Pointer to the allocated memory block. */\n    size_t           actual_size = 0;       /**< Actual size of the allocated memory block. */\n\n    /**\n     * @brief Default constructor.\n     */\n    ggml_cann_pool_alloc() = default;\n\n    /**\n     * @brief Constructor that initializes the memory pool.\n     * @param pool Reference to the memory pool.\n     */\n    explicit ggml_cann_pool_alloc(ggml_cann_pool & pool) : pool(&pool) {}\n\n    /**\n     * @brief Constructor that initializes the memory pool and allocates memory.\n     * @param pool Reference to the memory pool.\n     * @param size Size of the memory block to allocate.\n     */\n    ggml_cann_pool_alloc(ggml_cann_pool & pool, size_t size) : pool(&pool) { alloc(size); }\n\n    /**\n     * @brief Destructor that frees the allocated memory block.\n     */\n    ~ggml_cann_pool_alloc() {\n        if (ptr != nullptr) {\n            pool->free(ptr, actual_size);\n        }\n    }\n\n    /**\n     * @brief Allocates memory from the pool.\n     * @param size Size of the memory block to allocate.\n     * @return Pointer to the allocated memory block.\n     */\n    void * alloc(size_t size) {\n        GGML_ASSERT(pool != nullptr);\n        GGML_ASSERT(ptr == nullptr);\n        ptr = pool->alloc(size, &this->actual_size);\n        return ptr;\n    }\n\n    /**\n     * @brief Allocates memory from a specific memory pool.\n     * @param pool Reference to the memory pool.\n     * @param size Size of the memory block to allocate.\n     * @return Pointer to the allocated memory block.\n     */\n    void * alloc(ggml_cann_pool & pool, size_t size) {\n        this->pool = &pool;\n        return alloc(size);\n    }\n\n    /**\n     * @brief Gets the pointer to the allocated memory block.\n     * @return Pointer to the allocated memory block.\n     */\n    void * get() { return ptr; }\n\n    // Deleted copy constructor\n    ggml_cann_pool_alloc(const ggml_cann_pool_alloc &) = delete;\n\n    // Deleted move constructor\n    ggml_cann_pool_alloc(ggml_cann_pool_alloc &&) = delete;\n\n    // Deleted copy assignment operator\n    ggml_cann_pool_alloc & operator=(const ggml_cann_pool_alloc &) = delete;\n\n    // Deleted move assignment operator\n    ggml_cann_pool_alloc & operator=(ggml_cann_pool_alloc &&) = delete;\n};\n\n#ifdef USE_ACL_GRAPH\nstruct ggml_graph_node_properties {\n    // dst tensor\n    void *  node_address;\n    int64_t ne[GGML_MAX_DIMS];\n    size_t  nb[GGML_MAX_DIMS];\n\n    // src tensor\n    void *  src_address[GGML_MAX_SRC];\n    int64_t src_ne[GGML_MAX_SRC][GGML_MAX_DIMS];\n    size_t  src_nb[GGML_MAX_SRC][GGML_MAX_DIMS];\n\n    // op\n    ggml_op node_op;\n    int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n\n    /**\n     * @brief Check if a ggml tensor node matches this property set.\n     *\n     * This function compares all relevant fields (address, op type, shape, source inputs, op params)\n     * to determine whether the current node matches these previously recorded properties.\n     *\n     * @param node The current ggml tensor node.\n     * @return true if all fields match (excluding GGML_OP_VIEW); false otherwise.\n     */\n    bool has_matching_properties(ggml_tensor * node) {\n        if (node->data != this->node_address && node->op != GGML_OP_VIEW) {\n            return false;\n        }\n\n        if (node->op != this->node_op) {\n            return false;\n        }\n\n        for (int i = 0; i < GGML_MAX_DIMS; i++) {\n            if (node->ne[i] != this->ne[i]) {\n                return false;\n            }\n            if (node->nb[i] != this->nb[i]) {\n                return false;\n            }\n        }\n\n        for (int i = 0; i < GGML_MAX_SRC; i++) {\n            if (node->src[i]) {\n                if (node->src[i]->data != this->src_address[i] && node->op != GGML_OP_VIEW) {\n                    return false;\n                }\n\n                for (int d = 0; d < GGML_MAX_DIMS; d++) {\n                    if (node->src[i]->ne[d] != this->src_ne[i][d]) {\n                        return false;\n                    }\n                    if (node->src[i]->nb[d] != this->src_nb[i][d]) {\n                        return false;\n                    }\n                }\n            } else {\n                if (this->src_address[i] != nullptr) {\n                    return false;\n                }\n            }\n        }\n\n        if (node->op == GGML_OP_SCALE || node->op == GGML_OP_UNARY || node->op == GGML_OP_GLU) {\n            return memcmp(this->op_params, node->op_params, GGML_MAX_OP_PARAMS) == 0;\n        }\n        return true;\n    }\n};\n\nstruct ggml_cann_graph {\n    ~ggml_cann_graph() {\n        if (graph != nullptr) {\n            ACL_CHECK(aclmdlRIDestroy(graph));\n        }\n    }\n\n    aclmdlRI graph = nullptr;\n\n    std::vector<ggml_graph_node_properties> ggml_graph_properties;\n\n    /**\n     * @brief Create a new CANN graph from a ggml computation graph.\n     *\n     * This function creates a new ggml_cann_graph object and fills its node properties\n     * (operation type, dimensions, strides, input sources, and operation parameters)\n     * based on the current ggml computation graph.\n     *\n     * Each node in the ggml graph is mapped to a property entry in the new CANN graph:\n     * - node address\n     * - operation type\n     * - shape (ne) and strides (nb)\n     * - source tensor addresses\n     * - operation parameters\n     *\n     * @param cgraph The current ggml computation graph.\n     * @return Pointer to the newly created ggml_cann_graph object.\n     */\n    static ggml_cann_graph * create_from_cgraph(ggml_cgraph * cgraph) {\n        ggml_cann_graph * new_graph = new ggml_cann_graph();\n        new_graph->ggml_graph_properties.resize(cgraph->n_nodes);\n\n        for (int node_idx = 0; node_idx < cgraph->n_nodes; ++node_idx) {\n            ggml_tensor * node = cgraph->nodes[node_idx];\n            auto &        prop = new_graph->ggml_graph_properties[node_idx];\n\n            prop.node_address = node->data;\n            prop.node_op      = node->op;\n\n            std::copy_n(node->ne, GGML_MAX_DIMS, prop.ne);\n            std::copy_n(node->nb, GGML_MAX_DIMS, prop.nb);\n\n            for (int src = 0; src < GGML_MAX_SRC; ++src) {\n                if (node->src[src]) {\n                    prop.src_address[src] = node->src[src]->data;\n                    std::copy_n(node->src[src]->ne, GGML_MAX_DIMS, prop.src_ne[src]);\n                    std::copy_n(node->src[src]->nb, GGML_MAX_DIMS, prop.src_nb[src]);\n                } else {\n                    prop.src_address[src] = nullptr;\n                    std::fill_n(prop.src_ne[src], GGML_MAX_DIMS, 0);\n                    std::fill_n(prop.src_nb[src], GGML_MAX_DIMS, 0);\n                }\n            }\n\n            memcpy(prop.op_params, node->op_params, GGML_MAX_OP_PARAMS);\n        }\n\n        return new_graph;\n    }\n\n    /**\n     * @brief Check whether this CANN graph matches the given ggml computation graph.\n     *\n     * This function compares the number of nodes and each node's properties\n     * (operation type, dimensions, strides, inputs, and operation parameters)\n     * to determine whether this CANN graph matches the given ggml graph.\n     *\n     * @param cgraph The current ggml computation graph.\n     * @return true if this CANN graph matches the ggml graph; false otherwise.\n     */\n    bool matches_cgraph(ggml_cgraph * cgraph) {\n        if (this->ggml_graph_properties.size() != static_cast<size_t>(cgraph->n_nodes)) {\n            return false;\n        }\n\n        for (int i = 0; i < cgraph->n_nodes; ++i) {\n            if (!this->ggml_graph_properties[i].has_matching_properties(cgraph->nodes[i])) {\n                return false;\n            }\n        }\n\n        return true;\n    }\n};\n\n/**\n * @brief LRU cache for managing ggml_cann_graph objects.\n *\n * This class maintains a list of shared_ptr to ggml_cann_graph objects\n * and enforces a maximum capacity. It provides methods to push new graphs,\n * move existing graphs to the front (most recently used), and clear the cache.\n */\nstruct ggml_cann_graph_lru_cache {\n    size_t capacity;                         /**< Maximum number of graphs in the cache. */\n\n    std::list<ggml_cann_graph *> cache_list; /**< List storing cached graphs as raw pointers. */\n\n    ggml_cann_graph_lru_cache() { capacity = parse_integer(get_env_as_lowercase(\"GGML_CANN_GRAPH_CACHE_CAPACITY\").value_or(\"12\")); }\n\n    /**\n     * @brief Push a new graph to the front of the cache.\n     * If the cache exceeds capacity, the least recently used graph is deleted.\n     * @param new_node Pointer to the new ggml_cann_graph to cache.\n     *        Ownership is transferred to the cache (cache will delete it).\n     */\n    void push(ggml_cann_graph * new_node) {\n        if (cache_list.size() >= capacity) {\n            ggml_cann_graph * old = cache_list.back();\n            cache_list.pop_back();\n            delete old;  // free the old graph\n        }\n        cache_list.push_front(new_node);\n    }\n\n    /**\n     * @brief Clear all graphs from the cache (also frees memory).\n     */\n    void clear() {\n        for (auto ptr : cache_list) {\n            delete ptr;\n        }\n        cache_list.clear();\n    }\n\n    /**\n     * @brief Destructor that clears the cache and frees all cached graphs.\n     */\n    ~ggml_cann_graph_lru_cache() { clear(); }\n\n    /**\n     * @brief Find a cached CANN graph that matches the given ggml graph and move it to front.\n     *\n     * This function iterates through the cached CANN graphs stored in the LRU cache and\n     * compares them against the given ggml computation graph. If a matching graph is found,\n     * it is promoted to the front of the LRU cache and returned. Otherwise, the function\n     * returns nullptr.\n     *\n     * @param cgraph The current ggml computation graph.\n     * @return true if found; false otherwise.\n     */\n    bool find_and_move_to_front(ggml_cgraph * cgraph) {\n        for (auto & graph_ptr : this->cache_list) {\n            if (graph_ptr->matches_cgraph(cgraph)) {\n                cache_list.remove(graph_ptr);\n                cache_list.push_front(graph_ptr);\n                return true;\n            }\n        }\n        return false;\n    }\n};\n#endif  // USE_ACL_GRAPH\n\nstruct ggml_cann_rope_cache {\n    ~ggml_cann_rope_cache() {\n        if (theta_scale_cache) {\n            ACL_CHECK(aclrtFree(theta_scale_cache));\n        }\n        if (sin_cache) {\n            ACL_CHECK(aclrtFree(sin_cache));\n        }\n        if (cos_cache) {\n            ACL_CHECK(aclrtFree(cos_cache));\n        }\n        if (position_select_index) {\n            ACL_CHECK(aclrtFree(position_select_index));\n        }\n        if (theta_scale_exp_host) {\n            free(theta_scale_exp_host);\n        }\n        if (position_select_index_host) {\n            free(position_select_index_host);\n        }\n        if (yarn_ramp_cache) {\n            ACL_CHECK(aclrtFree(yarn_ramp_cache));\n        }\n    }\n\n    bool equal(int64_t theta_scale_length,\n               int64_t position_length,\n               float   ext_factor,\n               float   theta_scale,\n               float   freq_scale,\n               float   attn_factor,\n               bool    is_neox,\n               bool    indep_sects,\n               bool    mrope_used,\n               bool    is_imrope,\n               int     sections[4]) {\n        return this->theta_scale_length == theta_scale_length && this->position_length == position_length &&\n               this->ext_factor == ext_factor && this->theta_scale == theta_scale && this->freq_scale == freq_scale &&\n               this->attn_factor == attn_factor && this->is_neox == is_neox && this->indep_sects == indep_sects &&\n               this->mrope_used == mrope_used && this->is_imrope == is_imrope && this->sections[0] == sections[0] &&\n               this->sections[1] == sections[1] && this->sections[2] == sections[2] && this->sections[3] == sections[3];\n    }\n\n    void set(int64_t theta_scale_length,\n             int64_t position_length,\n             float   ext_factor,\n             float   theta_scale,\n             float   freq_scale,\n             float   attn_factor,\n             bool    is_neox,\n             bool    indep_sects,\n             bool    mrope_used,\n             bool    is_imrope,\n             int     sections[4]) {\n        this->theta_scale_length = theta_scale_length;\n        this->position_length    = position_length;\n        this->ext_factor         = ext_factor;\n        this->theta_scale        = theta_scale;\n        this->freq_scale         = freq_scale;\n        this->attn_factor        = attn_factor;\n        this->is_neox            = is_neox;\n        this->indep_sects        = indep_sects;\n        this->mrope_used         = mrope_used;\n        this->is_imrope          = is_imrope;\n        this->sections[0]        = sections[0];\n        this->sections[1]        = sections[1];\n        this->sections[2]        = sections[2];\n        this->sections[3]        = sections[3];\n    }\n\n    // memory cache, prepare before inferencing.\n    void *  theta_scale_cache          = nullptr;\n    float * theta_scale_exp_host       = nullptr;\n    int *   position_select_index_host = nullptr;\n    void *  position_select_index      = nullptr;\n    void *  yarn_ramp_cache            = nullptr;\n    // sin/cos cache, used only to accelerate first layer on each device\n    void *  sin_cache                  = nullptr;\n    void *  cos_cache                  = nullptr;\n    // Properties to check before reusing the sincos cache\n    int64_t theta_scale_length         = 0;\n    int64_t position_length            = 0;\n    bool    cached                     = false;\n    float   ext_factor                 = 0.0f;\n    float   theta_scale                = 0.0f;\n    float   freq_scale                 = 0.0f;\n    float   attn_factor                = 0.0f;\n    bool    is_neox                    = false;\n    bool    indep_sects                = false;\n    bool    mrope_used                 = false;\n    int     sections[4]                = { 0, 0, 0, 0 };\n    bool    is_imrope                  = false;\n};\n\nstruct ggml_cann_tensor_cache {\n    ~ggml_cann_tensor_cache() {\n        if (cache != nullptr) {\n            ACL_CHECK(aclrtFree(cache));\n        }\n    }\n\n    void *  cache = nullptr;\n    int64_t size  = 0;\n};\n\n/**\n * @brief Context for managing CANN backend operations.\n */\nstruct ggml_backend_cann_context {\n    int32_t     device;               /**< Device ID. */\n    std::string name;                 /**< Name of the device. */\n    std::string description;          /**< Description of the device. */\n    aclrtEvent  copy_event = nullptr; /**< Event for managing copy operations. */\n#ifdef USE_ACL_GRAPH\n    /// Cached CANN ACL graph used for executing the current ggml computation graph.\n    ggml_cann_graph_lru_cache graph_lru_cache;\n    bool                      acl_graph_mode = true;\n#endif\n    bool                   async_mode;\n    // Rope Cache\n    ggml_cann_rope_cache   rope_cache;\n    // Constant Pool\n    ggml_cann_tensor_cache rms_norm_one_tensor_cache;\n    ggml_cann_tensor_cache rms_norm_zero_tensor_cache;\n\n    aclrtStream streams[GGML_CANN_MAX_STREAMS] = { nullptr }; /**< Array of streams for the device. */\n\n    /**\n     * @brief Constructor for initializing the context with a given device.\n     * @param device Device ID.\n     */\n    explicit ggml_backend_cann_context(int device) : device(device), name(\"CANN\" + std::to_string(device)) {\n        ggml_cann_set_device(device);\n        description = aclrtGetSocName();\n\n#ifdef USE_ACL_GRAPH\n        acl_graph_mode = parse_bool(get_env_as_lowercase(\"GGML_CANN_ACL_GRAPH\").value_or(\"on\"));\n        GGML_LOG_INFO(\"%s: device %d execution mode is %s (%s)\\n\", __func__, device, acl_graph_mode ? \"GRAPH\" : \"EAGER\",\n                      acl_graph_mode ? \"acl graph enabled\" : \"acl graph disabled\");\n#endif\n    }\n\n    /**\n     * @brief Destructor for cleaning up resources.\n     */\n    ~ggml_backend_cann_context() {\n        ggml_cann_set_device(device);\n        if (copy_event != nullptr) {\n            ACL_CHECK(aclrtDestroyEvent(copy_event));\n        }\n        for (int i = 0; i < GGML_CANN_MAX_STREAMS; ++i) {\n            if (streams[i] != nullptr) {\n                ACL_CHECK(aclrtDestroyStream(streams[i]));\n            }\n        }\n    }\n\n    /**\n     * @brief Get or create a stream for a given index.\n     * @param stream Index of the stream.\n     * @return The stream corresponding to the given index.\n     */\n    aclrtStream stream(int stream) {\n        if (streams[stream] == nullptr) {\n            // If the device is not set here, destroying the stream later may cause a mismatch\n            // between the thread contexts where the stream was created and destroyed.\n            // However, I printed the device_id, thread_id, and stream, and they are all consistent.\n            ACL_CHECK(aclrtSetDevice(device));\n            ACL_CHECK(aclrtCreateStream(&streams[stream]));\n        }\n        return streams[stream];\n    }\n\n    /**\n     * @brief Get or create the default stream (index 0).\n     * @return The default stream.\n     */\n    aclrtStream stream() { return stream(0); }\n\n    // TODO: each stream should have a memory pool.\n    std::unique_ptr<ggml_cann_pool> mem_pool; /**< Memory pool for the device. */\n\n    /**\n     * @brief Create a new memory pool for a given device.\n     * @param device Device ID.\n     * @return A unique pointer to the new memory pool.\n     */\n    static std::unique_ptr<ggml_cann_pool> new_pool_for_device(int device);\n\n    /**\n     * @brief Get or create the memory pool for the context.\n     * @return Reference to the memory pool.\n     */\n    ggml_cann_pool & pool() {\n        if (mem_pool == nullptr) {\n            mem_pool = new_pool_for_device(device);\n        }\n        return *mem_pool;\n    }\n};\n\n#endif  // CANN_COMMON_H\n"
  },
  {
    "path": "ggml/src/ggml-cann/ggml-cann.cpp",
    "content": "/*\n * Copyright (c) 2023-2026 The ggml authors\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to\n * deal in the Software without restriction, including without limitation the\n * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or\n * sell copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS\n * IN THE SOFTWARE.\n */\n\n#include \"ggml-cann.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cann/aclnn_ops.h\"\n#include \"ggml-cann/common.h\"\n#include \"ggml-impl.h\"\n#include \"ggml.h\"\n\n#include <acl/acl.h>\n#include <aclnnop/aclnn_trans_matmul_weight.h>\n#include <stdarg.h>\n\n#include <chrono>\n#include <cmath>\n#include <cstdio>\n#include <cstring>\n#include <mutex>\n#include <optional>\n#include <queue>\n#include <unordered_set>\n\n#define GGML_COMMON_DECL_C\n\n#include \"ggml-common.h\"\n\n#define GGML_CANN_NAME \"CANN\"\n\n/**\n * @brief Handles CANN errors by printing an error message and aborting.\n *\n * @param stmt The statement that caused the error.\n * @param func The function in which the error occurred.\n * @param file The file in which the error occurred.\n * @param line The line number where the error occurred.\n * @param msg The error message.\n */\n[[noreturn]] void ggml_cann_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {\n    int32_t id = -1;\n    aclrtGetDevice(&id);\n\n    GGML_LOG_ERROR(\"CANN error: %s\\n\", msg);\n    GGML_LOG_ERROR(\"  current device: %d, in function %s at %s:%d\\n\", id, func, file, line);\n    GGML_LOG_ERROR(\"  %s\\n\", stmt);\n    // abort with GGML_ASSERT to get a stack trace\n    GGML_ABORT(\"CANN error\");\n}\n\n// Thread-local variable to record the current device of this thread.\nthread_local int g_current_cann_device = -1;\n\n/**\n * @brief Set the CANN device to be used.\n *\n * @param device The target device ID to set.\n */\nvoid ggml_cann_set_device(const int32_t device) {\n    // int current_device = -1;\n    // Note: In some CANN versions, if no device has been set yet,\n    //       aclrtGetDevice(&current_device) may return 0 by default.\n    // aclrtGetDevice(&current_device);\n\n    // If the current device is already the target one, no need to switch.\n    if (device == g_current_cann_device) {\n        return;\n    }\n\n    // Switch to the new device.\n    ACL_CHECK(aclrtSetDevice(device));\n\n    // Update the global device record.\n    g_current_cann_device = device;\n}\n\n/**\n * @brief Get the value of the specified environment variable (name) as lowercase.\n *        if not empty, return a std::string object\n */\nstd::optional<std::string> get_env_as_lowercase(const std::string & name) {\n    const char * val = std::getenv(name.c_str());\n    if (!val) {\n        return std::nullopt;\n    }\n    std::string res = std::string(val);\n    std::transform(res.begin(), res.end(), res.begin(), ::tolower);\n    return res;\n}\n\n/**\n * @brief Verify whether the environment variable is a valid value.\n */\nbool parse_bool(const std::string & value) {\n    static const std::unordered_set<std::string> valid_values = { \"on\", \"1\", \"yes\", \"y\", \"enable\", \"true\" };\n    return valid_values.find(value) != valid_values.end();\n}\n\n/**\n * @brief Parse a string as an integer, returning 0 if invalid.\n *\n * This function attempts to convert the input string `value` to an `int`.\n * If the string is not a valid integer or is out of the `int` range,\n * it returns 0.\n *\n * @param value The string to parse.\n * @return The parsed integer, or 0 if conversion fails.\n */\nint parse_integer(const std::string & value) {\n    try {\n        return std::stoi(value);\n    } catch (...) {\n        return 0;\n    }\n}\n\n/**\n * @brief Initialize the CANN device information.\n *\n * This function initializes the CANN device information by obtaining the\n * device count and setting the memory allocation granularity for each device.\n *\n * @return A structure containing the device information.\n */\nstatic ggml_cann_device_info ggml_cann_init() {\n    ggml_cann_device_info info = {};\n\n    aclError err = aclrtGetDeviceCount((uint32_t *) &info.device_count);\n\n    if (err != ACL_SUCCESS) {\n        GGML_LOG_ERROR(\"%s: failed to initialize CANN: %s\\n\", __func__, aclGetRecentErrMsg());\n        return info;\n    }\n\n    GGML_ASSERT(info.device_count <= GGML_CANN_MAX_DEVICES);\n\n    for (int id = 0; id < info.device_count; ++id) {\n        aclrtPhysicalMemProp prop = {};\n        prop.handleType           = ACL_MEM_HANDLE_TYPE_NONE;\n        prop.allocationType       = ACL_MEM_ALLOCATION_TYPE_PINNED;\n        prop.memAttr              = ACL_HBM_MEM_HUGE;\n        prop.location.type        = ACL_MEM_LOCATION_TYPE_DEVICE;\n        prop.location.id          = id;\n        prop.reserve              = 0;\n        err                       = aclrtMemGetAllocationGranularity(&prop, ACL_RT_MEM_ALLOC_GRANULARITY_RECOMMENDED,\n                                                                     &info.devices[id].vmm_granularity);\n        info.devices[id].vmm      = err == ACL_SUCCESS;\n\n        size_t free, total;\n        ggml_backend_cann_get_device_memory(id, &free, &total);\n        info.devices[id].total_vram = free;\n    }\n\n    // TODO: add more device info later.\n    return info;\n}\n\n/**\n * @brief Retrieve the CANN device information.\n *\n * This function returns a reference to a structure containing the CANN device\n * information. The device information is initialized once and reused on\n * subsequent calls.\n *\n * @return A reference to the structure containing the device information.\n */\nconst ggml_cann_device_info & ggml_cann_info() {\n    static ggml_cann_device_info info = ggml_cann_init();\n    return info;\n}\n\n//#define DEBUG_CANN_MALLOC\n/**\n * @brief A pool of CANN buffers(priority segment buffer).\n *\n * This class manages a pool of CANN buffers for a specific device.\n */\nstruct ggml_cann_pool_buf_prio : public ggml_cann_pool {\n    /**\n     * @brief The maximum reuse margin for a buffer.\n     */\n    static const size_t max_reuse_margin = 1ull << 22;  // 4MB\n\n    /**\n     * @brief The minimum free margin for a buffer.\n     */\n    static const size_t min_free_margin = 1ull << 20;  // 1MB\n\n    /**\n     * @brief The alignment for buffer allocation.\n     */\n    static const size_t alignment = 128;\n\n    /**\n     * @brief The device ID associated with this buffer pool.\n     */\n    int device;\n\n    /**\n     * @brief Whether to disable clean during buffer allocation.\n     */\n    bool disable_clean = false;\n\n    /**\n     * @brief Structure representing a CANN buffer.\n     */\n    struct ggml_cann_buffer {\n        void *                                ptr  = nullptr;  ///< Pointer to the buffer.\n        size_t                                size = 0;        ///< Size of the buffer.\n        std::chrono::steady_clock::time_point last_used;       ///< Last used time.\n\n        bool operator>(const ggml_cann_buffer & other) const { return size > other.size; }\n    };\n\n    /**\n     * @brief Array of CANN buffers in the pool.\n     */\n    std::unordered_map<void *, size_t>                                                   buffer_pool;\n    std::priority_queue<ggml_cann_buffer, std::vector<ggml_cann_buffer>, std::greater<>> free_buffers;\n\n    /**\n     * @brief Total size of all buffers in the pool.\n     */\n    size_t pool_size = 0;\n\n    /**\n     * @brief Constructor to initialize the buffer pool for a specific device.\n     *\n     * @param device The device ID to associate with this buffer pool.\n     */\n    explicit ggml_cann_pool_buf_prio(int device) : device(device) {\n        disable_clean = parse_bool(get_env_as_lowercase(\"GGML_CANN_DISABLE_BUF_POOL_CLEAN\").value_or(\"\"));\n    }\n\n    /**\n     * @brief Destructor to free all buffers in the pool.\n     */\n    ~ggml_cann_pool_buf_prio() {\n        ggml_cann_set_device(device);\n        for (auto & [b_ptr, b_size] : buffer_pool) {\n            aclrtFree(b_ptr);\n            pool_size -= b_size;\n        }\n        buffer_pool.clear();\n        GGML_ASSERT(pool_size == 0);\n    }\n\n    /**\n     * @brief Allocate a buffer of the given size.\n     *\n     * @param size The size of the buffer to allocate.\n     * @param actual_size A pointer to a variable to receive the actual size of\n     * the allocated buffer.\n     * @return A pointer to the allocated buffer.\n     */\n    void * alloc(size_t size, size_t * actual_size) override {\n        size = GGML_PAD(size, alignment);\n        if (size == 0) {\n            size = alignment;\n        }\n\n        void * ptr = nullptr;\n        auto   now = std::chrono::steady_clock::now();\n\n        std::vector<ggml_cann_buffer> free_buffers_rest;\n        free_buffers_rest.reserve(free_buffers.size());\n        while (!free_buffers.empty()) {\n            auto b = free_buffers.top();\n            free_buffers.pop();\n\n            if (b.size >= size) {\n                // reuse the buffer if the size is enough\n                const size_t margin = b.size - size;\n                if (margin <= max_reuse_margin) {\n                    *actual_size = b.size;\n                    ptr          = b.ptr;\n#ifdef DEBUG_CANN_MALLOC\n                    GGML_LOG_INFO(\n                        \"cann pool[%d]: reused   %p, \"\n                        \"pool_size = %5u MB, \"\n                        \"size = %5u MB, \"\n                        \"margin = %5u MB\\n\",\n                        device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n                        (uint32_t) (GGML_PAD(size, 1048576) / 1048576),\n                        (uint32_t) (GGML_PAD(margin, 1048576) / 1048576));\n#endif\n                    break;\n                }\n            }\n\n            bool should_clean = !disable_clean && b.size > min_free_margin &&\n                                std::chrono::duration_cast<std::chrono::milliseconds>(now - b.last_used).count() > 100;\n            if (should_clean) {\n                // free the buffer if the size is needed to be freed\n                ACL_CHECK(aclrtFree(b.ptr));\n                pool_size -= b.size;\n                buffer_pool.erase(b.ptr);\n#ifdef DEBUG_CANN_MALLOC\n                GGML_LOG_INFO(\n                    \"cann pool[%d]: clean    %p, \"\n                    \"pool_size = %5u MB, \"\n                    \"size = %5u MB\\n\",\n                    device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n                    (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n                continue;\n            }\n            free_buffers_rest.push_back(b);\n        }\n        for (ggml_cann_buffer & b : free_buffers_rest) {\n            free_buffers.push(std::move(b));\n        }\n\n#ifdef DEBUG_CANN_MALLOC\n        GGML_LOG_INFO(\"cann pool[%d] free pool_size = %5u MB\\n\\n\", device,\n                      (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n        if (ptr != nullptr) {\n            return ptr;\n        }\n\n        // allocate a new buffer if no buffer can be reused\n        ggml_cann_set_device(device);\n        ACL_CHECK(aclrtMalloc(&ptr, size, ACL_MEM_MALLOC_HUGE_FIRST));\n        *actual_size = size;\n        pool_size += size;\n#ifdef DEBUG_CANN_MALLOC\n        GGML_LOG_INFO(\n            \"cann pool[%d]: allocate %p, \"\n            \"pool_size = %5u MB, \"\n            \"size = %5u MB\\n\",\n            device, ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n            (uint32_t) (GGML_PAD(size, 1048576) / 1048576));\n#endif\n        buffer_pool.emplace(ptr, size);\n        return ptr;\n    }\n\n    /**\n     * @brief Free a buffer and return it to the pool.\n     *\n     * @param ptr Pointer to the buffer to free.\n     * @param size Size of the buffer to free.\n     */\n    void free(void * ptr, size_t size) override {\n        GGML_UNUSED(size);\n        auto it = buffer_pool.find(ptr);\n        if (it == buffer_pool.end()) {\n            GGML_ABORT(\"cann pool[%d]: buffer %p not found in pool\\n\", device, ptr);\n        }\n\n        auto now = std::chrono::steady_clock::now();\n        free_buffers.emplace(ggml_cann_buffer{ ptr, it->second, now });\n#ifdef DEBUG_CANN_MALLOC\n        GGML_LOG_INFO(\n            \"cann pool[%d]: return   %p, \"\n            \"pool_size = %5u MB\\n\",\n            device, ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n    }\n};\n\n/**\n * @brief A pool of CANN buffers(segment buffer).\n *\n * This class manages a pool of CANN buffers for a specific device.\n */\nstruct ggml_cann_pool_buf : public ggml_cann_pool {\n    /**\n     * @brief The maximum reuse margin for a buffer.\n     */\n    static const size_t max_reuse_margin = 1ull << 22;  // 4MB\n\n    /**\n     * @brief The minimum free margin for a buffer.\n     */\n    static const size_t min_free_margin = 1ull << 20;  // 1MB\n\n    /**\n     * @brief The alignment for buffer allocation.\n     */\n    static const size_t alignment = 128;\n\n    /**\n     * @brief The maximum number of buffers in the pool.\n     */\n    static const int MAX_BUFFERS = 256;\n\n    /**\n     * @brief The device ID associated with this buffer pool.\n     */\n    int device;\n\n    /**\n     * @brief Whether to disable clean during buffer allocation.\n     */\n    bool disable_clean = false;\n\n    /**\n     * @brief Structure representing a CANN buffer.\n     */\n    struct ggml_cann_buffer {\n        void *                                ptr  = nullptr;  ///< Pointer to the buffer memory.\n        size_t                                size = 0;        ///< Size of the buffer.\n        bool                                  used = false;    ///< Whether the buffer is currently in use.\n        std::chrono::steady_clock::time_point last_used;       ///< Last used time.\n    };\n\n    /**\n     * @brief Array of CANN buffers in the pool.\n     */\n    ggml_cann_buffer buffer_pool[MAX_BUFFERS] = {};\n\n    /**\n     * @brief Total size of all buffers in the pool.\n     */\n    size_t pool_size = 0;\n\n    /**\n     * @brief Constructor to initialize the buffer pool for a specific device.\n     *\n     * @param device The device ID to associate with this buffer pool.\n     */\n    explicit ggml_cann_pool_buf(int device) : device(device) {\n        disable_clean = parse_bool(get_env_as_lowercase(\"GGML_CANN_DISABLE_BUF_POOL_CLEAN\").value_or(\"\"));\n    }\n\n    /**\n     * @brief Destructor to free all buffers in the pool.\n     */\n    ~ggml_cann_pool_buf() {\n        ggml_cann_set_device(device);\n        for (int i = 0; i < MAX_BUFFERS; ++i) {\n            ggml_cann_buffer & b = buffer_pool[i];\n            if (b.ptr != nullptr) {\n                aclrtFree(b.ptr);\n                pool_size -= b.size;\n            }\n        }\n        GGML_ASSERT(pool_size == 0);\n    }\n\n    /**\n     * @brief Allocate a buffer of the given size.\n     *\n     * @param size The size of the buffer to allocate.\n     * @param actual_size A pointer to a variable to receive the actual size of\n     * the allocated buffer.\n     * @return A pointer to the allocated buffer.\n     */\n    void * alloc(size_t size, size_t * actual_size) override {\n        size = GGML_PAD(size, alignment);\n        if (size == 0) {\n            size = alignment;\n        }\n\n        void * ptr = nullptr;\n        auto   now = std::chrono::steady_clock::now();\n\n        int i = 0;\n        for (; i < MAX_BUFFERS; ++i) {\n            ggml_cann_buffer & b = buffer_pool[i];\n            if (b.ptr == nullptr) {\n                break;\n            }\n            if (b.used) {\n                continue;\n            }\n            if (b.size >= size) {\n                // reuse the buffer if the size is enough\n                const size_t margin = b.size - size;\n                if (margin <= max_reuse_margin) {\n                    *actual_size = b.size;\n                    b.used       = true;\n                    ptr          = b.ptr;\n#ifdef DEBUG_CANN_MALLOC\n                    GGML_LOG_INFO(\n                        \"cann pool[%d]: reused   %p, \"\n                        \"pool_size = %5u MB, \"\n                        \"size = %5u MB, \"\n                        \"margin = %5u MB\\n\",\n                        device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n                        (uint32_t) (GGML_PAD(size, 1048576) / 1048576),\n                        (uint32_t) (GGML_PAD(margin, 1048576) / 1048576));\n#endif\n                    break;\n                }\n            }\n\n            bool should_clean = !disable_clean && b.size > min_free_margin &&\n                                std::chrono::duration_cast<std::chrono::milliseconds>(now - b.last_used).count() > 100;\n            if (should_clean) {\n                // free the buffer if the size is needed to be freed\n                ACL_CHECK(aclrtFree(b.ptr));\n                pool_size -= b.size;\n#ifdef DEBUG_CANN_MALLOC\n                GGML_LOG_INFO(\n                    \"cann pool[%d]: clean    %p, \"\n                    \"pool_size = %5u MB, \"\n                    \"size = %5u MB\\n\",\n                    device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n                    (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n                b.ptr = nullptr;\n            }\n        }\n        if (ptr != nullptr) {\n            return ptr;\n        }\n\n        if (i < MAX_BUFFERS) {\n            // allocate a new buffer if no buffer can be reused\n            ggml_cann_buffer & b = buffer_pool[i];\n            ggml_cann_set_device(device);\n            ACL_CHECK(aclrtMalloc(&b.ptr, size, ACL_MEM_MALLOC_HUGE_FIRST));\n            pool_size += size;\n            *actual_size = size;\n            b.size       = size;\n            b.used       = true;\n            if (i >= MAX_BUFFERS - 8) {\n                GGML_LOG_WARN(\"cann pool[%d]: slots almost full\\n\", device);\n            }\n#ifdef DEBUG_CANN_MALLOC\n            GGML_LOG_INFO(\n                \"cann pool[%d]: allocate %p, \"\n                \"pool_size = %5u MB, \"\n                \"size = %5u MB\\n\",\n                device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576),\n                (uint32_t) (GGML_PAD(b.size, 1048576) / 1048576));\n#endif\n            return b.ptr;\n        }\n\n        GGML_ABORT(\"cann pool[%d]: slots full\\n\", device);\n    }\n\n    /**\n     * @brief Free a buffer and return it to the pool.\n     *\n     * @param ptr Pointer to the buffer to free.\n     * @param size Size of the buffer to free.\n     */\n    void free(void * ptr, size_t size) override {\n        GGML_UNUSED(size);\n        for (int i = 0; i < MAX_BUFFERS; ++i) {\n            ggml_cann_buffer & b = buffer_pool[i];\n            if (b.ptr != ptr) {\n                continue;\n            }\n            b.used      = false;\n            b.last_used = std::chrono::steady_clock::now();\n#ifdef DEBUG_CANN_MALLOC\n            GGML_LOG_INFO(\n                \"cann pool[%d]: return   %p, \"\n                \"pool_size = %5u MB\\n\",\n                device, b.ptr, (uint32_t) (GGML_PAD(pool_size, 1048576) / 1048576));\n#endif\n            return;\n        }\n        GGML_ABORT(\"cann pool[%d]: slots full\\n\", device);\n    }\n};\n\n/**\n * @brief A pool of CANN buffers with virtual memory.\n *\n * This class manages a pool of CANN buffers with virtual memory for a specific\n * device.\n */\nstruct ggml_cann_pool_vmm : public ggml_cann_pool {\n    /**\n     * @brief The maximum size of the virtual memory pool (32 GB).\n     */\n    size_t max_size;\n\n    /**\n     * @brief The device ID associated with this buffer pool.\n     */\n    int device;\n\n    /**\n     * @brief Pointer to the start of the virtual memory pool.\n     */\n    void * pool_addr = 0;\n\n    /**\n     * @brief Amount of virtual memory used in the pool.\n     */\n    size_t pool_used = 0;\n\n    /**\n     * @brief Total size of the virtual memory pool.\n     */\n    size_t pool_size = 0;\n\n    /**\n     * @brief Allocation granularity for the virtual memory pool.\n     */\n    size_t granularity;\n\n    /**\n     * @brief Handles for the physical memory allocated.\n     */\n    std::vector<aclrtDrvMemHandle> handles;\n\n    /**\n     * @brief Offsets for the mapped memory regions.\n     */\n    std::vector<void *> map_offsets;\n\n    /**\n     * @brief Constructor to initialize the buffer pool with virtual memory for\n     * a specific device.\n     *\n     * @param device The device ID to associate with this buffer pool.\n     */\n    explicit ggml_cann_pool_vmm(int device) : device(device) {\n        auto dev    = ggml_cann_info().devices[device];\n        granularity = dev.vmm_granularity;\n        max_size    = dev.total_vram;\n    }\n\n    /**\n     * @brief Destructor to free all buffers in the virtual memory pool.\n     */\n    ~ggml_cann_pool_vmm() {\n        if (pool_addr != 0) {\n            for (auto & offset : map_offsets) {\n                ACL_CHECK(aclrtUnmapMem(offset));\n            }\n            for (auto & handle : handles) {\n                ACL_CHECK(aclrtFreePhysical(handle));\n            }\n            ACL_CHECK(aclrtReleaseMemAddress(pool_addr));\n        }\n    }\n\n    /**\n     * @brief Allocate a buffer of the given size in the virtual memory pool.\n     *\n     * @param size The size of the buffer to allocate.\n     * @param actual_size A pointer to a variable to receive the actual size of\n     * the allocated buffer.\n     * @return A pointer to the allocated buffer.\n     */\n    void * alloc(size_t size, size_t * actual_size) override {\n        // round up the allocation size to the alignment to ensure that all\n        // allocations are aligned for all data types\n        const size_t alignment = 128;\n        size                   = GGML_PAD(size, alignment);\n        if (size == 0) {\n            size = alignment;\n        }\n\n        size_t avail = pool_size - pool_used;\n\n        if (size > avail) {\n            // round up to the next multiple of the granularity\n            size_t reserve_size = size - avail;\n            reserve_size        = GGML_PAD(reserve_size, granularity);\n\n            GGML_ASSERT(pool_size + reserve_size <= max_size);\n\n            // allocate more physical memory\n            aclrtPhysicalMemProp prop = {};\n            prop.handleType           = ACL_MEM_HANDLE_TYPE_NONE;\n            prop.allocationType       = ACL_MEM_ALLOCATION_TYPE_PINNED;\n            prop.memAttr              = ACL_HBM_MEM_HUGE;\n            prop.location.type        = ACL_MEM_LOCATION_TYPE_DEVICE;\n            prop.location.id          = device;\n            prop.reserve              = 0;\n            aclrtDrvMemHandle handle;\n            ACL_CHECK(aclrtMallocPhysical(&handle, reserve_size, &prop, 0));\n\n            // reserve virtual address space (if not already reserved)\n            if (pool_addr == 0) {\n                ACL_CHECK(aclrtReserveMemAddress(&pool_addr, max_size, 0, NULL, 1));\n            }\n\n            // map at the end of the pool\n            ACL_CHECK(aclrtMapMem((char *) pool_addr + pool_size, reserve_size, 0, handle, 0));\n\n            handles.push_back(handle);\n            map_offsets.push_back((char *) pool_addr + pool_size);\n\n            // add to the pool\n            pool_size += reserve_size;\n\n#ifdef DEBUG_CANN_MALLOC\n            GGML_LOG_INFO(\"cann pool[%d]: size increased to %llu MB (reserved %llu MB)\\n\", device,\n                          (unsigned long long) (pool_size / 1024 / 1024),\n                          (unsigned long long) (reserve_size / 1024 / 1024));\n#endif\n        }\n\n        GGML_ASSERT(pool_addr != 0);\n\n        void * ptr   = (void *) ((char *) pool_addr + pool_used);\n        *actual_size = size;\n        pool_used += size;\n\n#ifdef DEBUG_CANN_MALLOC\n        GGML_LOG_INFO(\"cann pool[%d]: allocated %llu bytes at %llx\\n\", device, (unsigned long long) size,\n                      (unsigned long long) ptr);\n#endif\n        return ptr;\n    }\n\n    /**\n     * @brief Free a buffer and return it to the virtual memory pool.\n     *\n     * @param ptr Pointer to the buffer to free.\n     * @param size Size of the buffer to free.\n     */\n    void free(void * ptr, size_t size) override {\n#ifdef DEBUG_CANN_MALLOC\n        GGML_LOG_INFO(\"cann pool[%d]: freed %llu bytes at %llx\\n\", device, (unsigned long long) size,\n                      (unsigned long long) ptr);\n#endif\n\n        pool_used -= size;\n\n        // all deallocations must be in reverse order of the allocations\n        GGML_ASSERT(ptr == (void *) ((char *) pool_addr + pool_used));\n    }\n};\n\n/**\n * @brief Create a new CANN pool for a specific device.\n *\n * Factory method to create a new CANN pool object based on the device type.\n *\n * @param device The device ID for which to create the pool.\n * @return A unique pointer to the created CANN pool.\n */\nstd::unique_ptr<ggml_cann_pool> ggml_backend_cann_context::new_pool_for_device(int device) {\n    std::string mem_pool_type = get_env_as_lowercase(\"GGML_CANN_MEM_POOL\").value_or(\"\");\n\n    if (mem_pool_type == \"prio\") {\n        GGML_LOG_INFO(\"%s: device %d use buffer pool with priority queue\\n\", __func__, device);\n        return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_buf_prio(device));\n    }\n\n    if (ggml_cann_info().devices[device].vmm && mem_pool_type != \"leg\") {\n        GGML_LOG_INFO(\"%s: device %d use vmm pool\\n\", __func__, device);\n        return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_vmm(device));\n    }\n\n    GGML_LOG_INFO(\"%s: device %d use buffer pool\\n\", __func__, device);\n    return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_buf(device));\n}\n\n// cann buffer\n/**\n * @brief Context for managing a CANN buffer associated with a specific device.\n *\n * This structure holds information about a CANN buffer, including the device\n * ID, device pointer, and a name derived from GGML_CANN_NAME and the device ID.\n */\nstruct ggml_backend_cann_buffer_context {\n    int32_t device;             ///< The device ID associated with this buffer context.\n    void *  dev_ptr = nullptr;  ///< Pointer to the device memory allocated for the buffer.\n\n    /**\n     * @brief Constructor to initialize the CANN buffer context.\n     *\n     * @param device The device ID associated with this buffer context.\n     * @param dev_ptr Pointer to the device memory allocated for the buffer.\n     */\n    ggml_backend_cann_buffer_context(int32_t device, void * dev_ptr) : device(device), dev_ptr(dev_ptr) {}\n\n    /**\n     * @brief Destructor to free the device memory allocated for the buffer.\n     */\n    ~ggml_backend_cann_buffer_context() { ACL_CHECK(aclrtFree(dev_ptr)); }\n};\n\n// cann buffer type\n/**\n * @brief Structure representing context information for a specific backend\n * buffer type.\n */\nstruct ggml_backend_cann_buffer_type_context {\n    int32_t     device; /**< Device identifier associated with the buffer context. */\n    std::string name;   /**< Name associated with the buffer context. */\n};\n\n/**\n * @brief Retrieves the name associated with a CANN buffer type.\n *\n * This function returns the descriptive name associated with the specified\n * CANN buffer type context.\n *\n * @param buft Pointer to the buffer type context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_buffer_type_name(ggml_backend_buffer_type_t buft) {\n    ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n\n    return buft_ctx->name.c_str();\n}\n\n/**\n * @brief Checks if the backend buffer type is associated with the CANN backend.\n *\n * This function checks whether the provided backend buffer type is associated\n * with the CANN backend based on the comparison of its name retrieval function\n * pointer.\n *\n * @param buft Pointer to the backend buffer type to check.\n * @return bool Returns true if the buffer type is associated with the CANN\n * backend, otherwise false.\n */\nstatic bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft) {\n    return buft->iface.get_name == ggml_backend_cann_buffer_type_name;\n}\n\n/**\n * @brief Free resources associated with a CANN buffer.\n *\n * This function frees the resources associated with a CANN buffer, including\n * its context.\n *\n * @param buffer The CANN buffer to free.\n */\nstatic void ggml_backend_cann_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n    ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n    delete ctx;\n}\n\n/**\n * @brief Retrieve the base pointer of a CANN buffer.\n *\n * This function returns the base pointer of a CANN buffer, which points to the\n * device memory allocated for the buffer.\n *\n * @param buffer The CANN buffer whose base pointer is to be retrieved.\n * @return A pointer to the base of the device memory allocated for the buffer.\n */\nstatic void * ggml_backend_cann_buffer_get_base(ggml_backend_buffer_t buffer) {\n    ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n    return ctx->dev_ptr;\n}\n\n/**\n * @brief Transform quantized Q4.0 tensor data into a format suitable for CANN\n * processing.\n *\n * This function transforms quantized Q4.0 tensor data into a format suitable\n * for CANN processing. It extracts quantization values and scales from the\n * source data and prepares them in a format expected by CANN operations.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data in Q4.0 format.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform_q4_0(ggml_tensor * tensor, const void * src, void * dst) {\n    int64_t n_elems     = ggml_nelements(tensor);\n    int64_t groups      = n_elems / QK4_0;\n    size_t  quant_bytes = n_elems * sizeof(uint8_t) / 2;\n\n    uint8_t *  quant_offset = (uint8_t *) dst;\n    uint16_t * scale_offset = (uint16_t *) ((char *) dst + quant_bytes);\n\n    for (int i = 0; i < groups; i++) {\n        const block_q4_0 * group = (const block_q4_0 *) ((const char *) src + i * sizeof(block_q4_0));\n        *scale_offset            = group->d;\n        scale_offset++;\n\n        // 0-15\n        for (int j = 0; j < QK4_0 / 2; j += 2) {\n            (*quant_offset) = (group->qs[j] & 0x0F);\n            (*quant_offset) |= ((group->qs[j + 1] << 4));\n            quant_offset++;\n        }\n\n        // 16-31\n        for (int j = 0; j < QK4_0 / 2; j += 2) {\n            (*quant_offset) = (group->qs[j] >> 4);\n            (*quant_offset) |= (group->qs[j + 1] & 0xF0);\n            quant_offset++;\n        }\n    }\n\n    // put (uint4b_t -8) into int4b_t\n    for (quant_offset = (uint8_t *) dst; quant_offset < (uint8_t *) dst + quant_bytes; quant_offset++) {\n        (*quant_offset) ^= 0x88;\n    }\n}\n\n/**\n * @brief Transform CANN processed data back into quantized Q4.0 format.\n *\n * This function transforms CANN processed data back into quantized Q4.0 format.\n * It reverses the transformation performed by\n * ggml_backend_cann_transform_q4_0(), converting the data back into its\n * original quantized form.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source buffer containing transformed data.\n * @param dst Pointer to the destination buffer where the Q4.0 formatted data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back_q4_0(const ggml_tensor * tensor, void * src, void * dst) {\n    int64_t n_elems     = ggml_nelements(tensor);\n    int64_t groups      = n_elems / QK4_0;\n    size_t  quant_bytes = n_elems * sizeof(uint8_t) / 2;\n\n    uint8_t *  quant_offset = (uint8_t *) src;\n    uint16_t * scale_offset = (uint16_t *) ((char *) src + quant_bytes);\n\n    for (; quant_offset < (uint8_t *) src + quant_bytes; quant_offset++) {\n        (*quant_offset) ^= 0x88;\n    }\n    quant_offset = (uint8_t *) src;\n\n    for (int i = 0; i < groups; i++) {\n        block_q4_0 * group = (block_q4_0 *) ((char *) dst + i * sizeof(block_q4_0));\n        group->d           = *scale_offset;\n        scale_offset++;\n\n        // 0-15\n        for (int j = 0; j < QK4_0 / 2; j += 2) {\n            group->qs[j]     = ((*quant_offset) & 0x0F);\n            group->qs[j + 1] = ((*quant_offset) >> 4);\n            quant_offset++;\n        }\n\n        // 16-31\n        for (int j = 0; j < QK4_0 / 2; j += 2) {\n            group->qs[j] |= ((*quant_offset) << 4);\n            group->qs[j + 1] |= ((*quant_offset) & 0xF0);\n            quant_offset++;\n        }\n    }\n}\n\n/**\n * @brief Transform quantized Q8.0 tensor data into a format suitable for CANN\n * processing.\n *\n * This function transforms quantized Q8.0 tensor data into a format suitable\n * for CANN processing. It extracts quantization values and scales from the\n * source data and prepares them in a format expected by CANN operations.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data in Q8.0 format.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform_q8_0(ggml_tensor * tensor, const void * src, void * dst) {\n    int64_t n_elems     = ggml_nelements(tensor);\n    int64_t groups      = n_elems / QK8_0;\n    size_t  quant_bytes = n_elems * sizeof(uint8_t);\n\n    uint8_t *  quant_offset = (uint8_t *) dst;\n    uint16_t * scale_offset = (uint16_t *) ((char *) dst + quant_bytes);\n\n    for (int i = 0; i < groups; i++) {\n        const block_q8_0 * group = (const block_q8_0 *) ((const char *) src + i * sizeof(block_q8_0));\n        *scale_offset            = group->d;\n        scale_offset++;\n        size_t group_quant_size = QK8_0 * sizeof(uint8_t);\n        memcpy(quant_offset, group->qs, group_quant_size);\n        quant_offset += group_quant_size;\n    }\n}\n\n/**\n * @brief Transform CANN processed data back into quantized Q8.0 format.\n *\n * This function transforms CANN processed data back into quantized Q8.0 format.\n * It reverses the transformation performed by\n * ggml_backend_cann_transform_q8_0(), converting the data back into its\n * original quantized form.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source buffer containing transformed data.\n * @param dst Pointer to the destination buffer where the Q8.0 formatted data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back_q8_0(const ggml_tensor * tensor, const void * src, void * dst) {\n    int64_t n_elems     = ggml_nelements(tensor);\n    int64_t groups      = n_elems / QK8_0;\n    size_t  quant_bytes = n_elems * sizeof(uint8_t);\n\n    const uint8_t *  quant_offset = (const uint8_t *) src;\n    const uint16_t * scale_offset = (const uint16_t *) ((const char *) src + quant_bytes);\n\n    for (int i = 0; i < groups; i++) {\n        block_q8_0 * group = (block_q8_0 *) ((char *) dst + i * sizeof(block_q8_0));\n        group->d           = *scale_offset;\n        scale_offset++;\n        size_t group_quant_size = QK8_0 * sizeof(uint8_t);\n        memcpy(group->qs, quant_offset, group_quant_size);\n        quant_offset += group_quant_size;\n    }\n}\n\n/**\n * @brief Transform tensor data based on its type for CANN processing.\n *\n * This function transforms tensor data based on its quantization type for CANN\n * processing. It dispatches the transformation based on the tensor's type to\n * specialized functions handling Q4.0 and Q8.0 formats.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data to be transformed.\n * @param dst Pointer to the destination buffer where transformed data will be\n * stored.\n */\nstatic void ggml_backend_cann_transform(ggml_tensor * tensor, const void * src, void * dst) {\n    switch (tensor->type) {\n        case GGML_TYPE_Q4_0:\n            ggml_backend_cann_transform_q4_0(tensor, src, dst);\n            break;\n        case GGML_TYPE_Q8_0:\n            ggml_backend_cann_transform_q8_0(tensor, src, dst);\n            break;\n        default:\n            break;\n    }\n}\n\n/**\n * @brief Transform CANN processed data back into tensor data based on its type.\n *\n * This function transforms CANN processed data back into tensor data based on\n * its quantization type for Q4.0 and Q8.0 formats. It dispatches the\n * transformation based on the tensor's type to specialized functions.\n *\n * @param tensor Pointer to the tensor information.\n * @param src Pointer to the source data containing CANN processed data.\n * @param dst Pointer to the destination buffer where transformed tensor data\n * will be stored.\n */\nstatic void ggml_backend_cann_transform_back(const ggml_tensor * tensor, void * src, void * dst) {\n    switch (tensor->type) {\n        case GGML_TYPE_Q4_0:\n            ggml_backend_cann_transform_back_q4_0(tensor, src, dst);\n            break;\n        case GGML_TYPE_Q8_0:\n            ggml_backend_cann_transform_back_q8_0(tensor, src, dst);\n            break;\n        default:\n            break;\n    }\n}\n\n/**\n * @brief Check if transformation is needed for a given tensor type.\n *\n * This function checks if transformation is needed for a given tensor type\n * to prepare data for CANN processing.\n *\n * @param type The tensor type to check.\n * @return true if transformation is needed, false otherwise.\n */\nstatic bool need_transform(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q8_0:\n            return true;\n        default:\n            return false;\n    }\n}\n\n/**\n * @brief Initialize a tensor using data from a CANN buffer.\n *\n * This function initializes a tensor using data from a CANN buffer.\n * It handles special cases such as views and quantization.\n *\n * @param buffer The CANN buffer from which to initialize the tensor.\n * @param tensor Pointer to the tensor to be initialized.\n */\nstatic enum ggml_status ggml_backend_cann_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {\n    if (tensor->view_src != NULL && tensor->view_offs == 0) {\n        GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);\n        return GGML_STATUS_SUCCESS;\n    }\n\n    // TODO: cann backend doesn't support quantized yet. Just leave the code\n    // here.\n    if (ggml_is_quantized(tensor->type)) {\n        // Initialize padding to 0 to avoid possible NaN values\n        size_t original_size = ggml_nbytes(tensor);\n        size_t padded_size   = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);\n\n        if (padded_size > original_size && tensor->view_src == nullptr) {\n            size_t memset_size = padded_size - original_size;\n            ACL_CHECK(aclrtMemset((char *) tensor->data + original_size, memset_size, 0, memset_size));\n        }\n    }\n    return GGML_STATUS_SUCCESS;\n}\n\n/**\n * @brief Workspace for caching NZ buffers per device.\n *\n * This struct manages a device buffer used in NZ computations. It supports\n * allocation, reallocation, and clearing of cached memory. The struct is\n * designed to be used with a global array, one per device.\n */\nstruct ggml_cann_nz_workspace {\n    void * ptr;        // Pointer to allocated device buffer\n    size_t allocated;  // Size of currently allocated buffer in bytes\n\n    /**\n     * @brief Constructor. Initializes the workspace with no allocated memory.\n     */\n    ggml_cann_nz_workspace() : ptr(nullptr), allocated(0) {}\n\n    /**\n     * @brief Free cached memory and reset the workspace.\n     *\n     * If a buffer has been allocated, this function releases it using\n     * aclrtFree and resets internal state.\n     */\n    void clear() {\n        if (ptr) {\n            ACL_CHECK(aclrtFree(ptr));\n            ptr       = nullptr;\n            allocated = 0;\n        }\n    }\n\n    /**\n     * @brief Allocate or reallocate the workspace buffer.\n     *\n     * If the requested size is larger than the currently allocated size,\n     * the old buffer will be freed and a new buffer of the requested size\n     * will be allocated on the device.\n     *\n     * @param new_size Size in bytes to allocate for the workspace.\n     */\n    void realloc(size_t new_size) {\n        if (new_size > allocated) {\n            clear();\n            ACL_CHECK(aclrtMalloc(&ptr, new_size, ACL_MEM_MALLOC_HUGE_FIRST));\n            allocated = new_size;\n        }\n    }\n\n    /**\n     * @brief Get the device buffer pointer.\n     *\n     * @return Pointer to the allocated buffer, or nullptr if not allocated.\n     */\n    void * get() const { return ptr; }\n};\n\n/**\n * @brief Global array of NZ workspaces, one per device.\n */\nstatic ggml_cann_nz_workspace g_nz_workspaces[GGML_CANN_MAX_DEVICES];\n\n/**\n * @brief Convert tensor weights to NZ format using Ascend CANN API.\n *\n * This function creates a transposed tensor descriptor and performs the\n * TransMatmulWeight operation. Converting tensor formats can significantly\n * improve performance on certain hardware.\n *\n * @param tensor Pointer to the input ggml_tensor containing the weights.\n * @param offset Byte offset within the tensor data buffer where weights start.\n * @param device device id.\n *\n * @note The workspace buffer used in this function is managed globally and reused\n *       across calls. This reduces overhead from repeated memory allocation and deallocation.\n */\nstatic void weight_format_to_nz(ggml_tensor * tensor, size_t offset, int device) {\n    acl_tensor_ptr weightTransposed = ggml_cann_create_tensor(tensor, tensor->ne, tensor->nb, 2, ACL_FORMAT_ND, offset);\n    uint64_t       workspaceSize    = 0;\n    aclOpExecutor * executor;\n\n    // TransMatmulWeight\n    ACL_CHECK(aclnnTransMatmulWeightGetWorkspaceSize(weightTransposed.get(), &workspaceSize, &executor));\n    // Avoid frequent malloc/free of the workspace.\n    g_nz_workspaces[device].realloc(workspaceSize);\n\n    void * g_nz_workspace = g_nz_workspaces[device].get();\n\n    ACL_CHECK(aclnnTransMatmulWeight(g_nz_workspace, workspaceSize, executor, nullptr));\n}\n\n// TODO: need handle tensor which has paddings.\n/**\n * @brief Set tensor data in a CANN buffer.\n *\n * This function sets tensor data in a CANN buffer, handling transformations\n * if needed based on the tensor's type.\n *\n * @param buffer The CANN buffer where the tensor data will be set.\n * @param tensor Pointer to the tensor whose data will be set.\n * @param data Pointer to the source data to be copied into the tensor.\n * @param offset Offset in the source data from where to start copying.\n * @param size Size of the data to be copied, in bytes.\n */\nstatic void ggml_backend_cann_buffer_set_tensor(ggml_backend_buffer_t buffer,\n                                                ggml_tensor *         tensor,\n                                                const void *          data,\n                                                size_t                offset,\n                                                size_t                size) {\n    ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n    ggml_cann_set_device(ctx->device);\n    // TODO: refer to cann(#6017), it use thread's default stream.\n    // For acl, synchronous functions use this default stream.\n    // Why aclrtSynchronizeDevice?\n\n    // Only check env once.\n    static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n    if (!need_transform(tensor->type)) {\n        ACL_CHECK(aclrtMemcpy((char *) tensor->data + offset, size, data, size, ACL_MEMCPY_HOST_TO_DEVICE));\n        if (weight_to_nz && is_matmul_weight((const ggml_tensor *) tensor)) {\n            GGML_ASSERT(tensor->ne[2] == 1);\n            GGML_ASSERT(tensor->ne[3] == 1);\n            weight_format_to_nz(tensor, offset, ctx->device);\n        }\n    } else {\n        void * transform_buffer = malloc(size);\n        ggml_backend_cann_transform(tensor, data, transform_buffer);\n\n        ACL_CHECK(aclrtMemcpy((char *) tensor->data + offset, size, transform_buffer, size, ACL_MEMCPY_HOST_TO_DEVICE));\n        free(transform_buffer);\n    }\n}\n\n/**\n * @brief Get tensor data from a CANN buffer.\n *\n * This function retrieves tensor data from a CANN buffer, handling\n * transformations if needed based on the tensor's type.\n *\n * @param buffer The CANN buffer from which to retrieve tensor data.\n * @param tensor Pointer to the tensor whose data will be retrieved.\n * @param data Pointer to the destination buffer where the tensor data will be\n * copied.\n * @param offset Offset in the destination buffer where to start copying.\n * @param size Size of the data to be copied, in bytes.\n */\nstatic void ggml_backend_cann_buffer_get_tensor(ggml_backend_buffer_t buffer,\n                                                const ggml_tensor *   tensor,\n                                                void *                data,\n                                                size_t                offset,\n                                                size_t                size) {\n    ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n    ggml_cann_set_device(ctx->device);\n\n    if (!need_transform(tensor->type)) {\n        ACL_CHECK(aclrtMemcpy(data, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST));\n    } else {\n        void * transform_buffer = malloc(size);\n        ACL_CHECK(aclrtMemcpy(transform_buffer, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST));\n        ggml_backend_cann_transform_back(tensor, transform_buffer, data);\n        free(transform_buffer);\n    }\n}\n\n/**\n * @brief Copy tensor data between CANN buffers if possible.\n *\n * This function copies tensor data between CANN buffers if the source and\n * destination buffers are CANN buffers and they meet the necessary conditions\n * (same device or devices can access each other).\n *\n * @param buffer The destination CANN buffer where the tensor data will be\n * copied.\n * @param src Pointer to the source tensor whose data will be copied.\n * @param dst Pointer to the destination tensor where the data will be copied.\n * @return true if the copy operation succeeded, false otherwise.\n */\nstatic bool ggml_backend_cann_buffer_cpy_tensor(ggml_backend_buffer_t buffer,\n                                                const ggml_tensor *   src,\n                                                ggml_tensor *         dst) {\n    if (ggml_backend_buft_is_cann(src->buffer->buft)) {\n        ggml_backend_cann_buffer_context * src_ctx = (ggml_backend_cann_buffer_context *) src->buffer->context;\n        ggml_backend_cann_buffer_context * dst_ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n        size_t memcpy_size = ggml_nbytes(src);\n        // Same device.\n        if (src_ctx->device == dst_ctx->device) {\n            ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size,\n                                  ACL_MEMCPY_DEVICE_TO_DEVICE));\n            return true;\n        } else {\n#ifdef ASCEND_310P\n            // TODO: Support 310p P2P copy\n            return false;\n#endif\n            // Different device but can access by peer.\n            int32_t canAccessPeer = 0;\n            ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device, dst_ctx->device));\n            if (canAccessPeer) {\n                ggml_cann_set_device(src_ctx->device);\n                ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0));\n                ACL_CHECK(aclrtMemcpy((char *) dst->data, memcpy_size, (const char *) src->data, memcpy_size,\n                                      ACL_MEMCPY_DEVICE_TO_DEVICE));\n                return true;\n            }\n        }\n    }\n    return false;\n}\n\n/**\n * @brief Clear a CANN buffer by setting all its memory to a specified value.\n *\n * This function clears a CANN buffer by setting all its memory to a specified\n * value.\n *\n * @param buffer The CANN buffer to be cleared.\n * @param value The value to which each byte in the buffer will be set.\n */\nstatic void ggml_backend_cann_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n    ggml_backend_cann_buffer_context * ctx = (ggml_backend_cann_buffer_context *) buffer->context;\n\n    ggml_cann_set_device(ctx->device);\n    ACL_CHECK(aclrtMemset(ctx->dev_ptr, buffer->size, value, buffer->size));\n}\n\n/**\n * @brief Interface for a CANN buffer in the backend.\n *\n * This structure defines function pointers to operations that can be performed\n * on a CANN buffer within the backend.\n */\nstatic const ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {\n    /* .free_buffer     = */ ggml_backend_cann_buffer_free_buffer,\n    /* .get_base        = */ ggml_backend_cann_buffer_get_base,\n    /* .init_tensor     = */ ggml_backend_cann_buffer_init_tensor,\n    /* .memset_tensor   = */ NULL,\n    /* .set_tensor      = */ ggml_backend_cann_buffer_set_tensor,\n    /* .get_tensor      = */ ggml_backend_cann_buffer_get_tensor,\n    /* .cpy_tensor      = */ ggml_backend_cann_buffer_cpy_tensor,\n    /* .clear           = */ ggml_backend_cann_buffer_clear,\n    /* .reset           = */ NULL,\n};\n\n/**\n * @brief Allocates a new CANN buffer of the specified type and size.\n *\n * This function allocates a new CANN buffer on the specified device with the\n * given size.\n *\n * @param buft Pointer to the buffer type context.\n * @param size Size in bytes of the buffer to allocate.\n * @return Pointer to the allocated buffer, or nullptr if allocation fails.\n */\nstatic ggml_backend_buffer_t ggml_backend_cann_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n\n    ggml_cann_set_device(buft_ctx->device);\n\n    const size_t alignment = 128;\n    size                   = GGML_PAD(size, alignment);\n    if (size == 0) {\n        size = alignment;\n    }\n    void *   dev_ptr;\n    aclError err = aclrtMalloc(&dev_ptr, size, ACL_MEM_MALLOC_HUGE_FIRST);\n    if (err != ACL_SUCCESS) {\n        GGML_LOG_ERROR(\"%s: allocating %.2f MiB on device %d: aclrtMalloc failed: %s\\n\", __func__,\n                       size / 1024.0 / 1024.0, buft_ctx->device, aclGetRecentErrMsg());\n        return nullptr;\n    }\n\n    ggml_backend_cann_buffer_context * ctx = new ggml_backend_cann_buffer_context(buft_ctx->device, dev_ptr);\n\n    return ggml_backend_buffer_init(buft, ggml_backend_cann_buffer_interface, ctx, size);\n}\n\n/**\n * @brief Retrieves the memory alignment requirement for CANN buffers of this\n * type.\n *\n * This function returns the alignment requirement in bytes for memory allocated\n * by the CANN buffer type.\n *\n * @param buft Pointer to the buffer type context (unused in this\n * implementation).\n * @return The alignment requirement in bytes (fixed at 128 bytes for CANN\n * buffers).\n */\nstatic size_t ggml_backend_cann_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    return 128;\n\n    GGML_UNUSED(buft);\n}\n\n/**\n * @brief Calculates the allocation size required for a tensor in a CANN buffer.\n *\n * Computes the total allocation size needed for storing the tensor's data in a\n * CANN buffer, considering any necessary padding or adjustments for quantized\n * types.\n *\n * @param buft Pointer to the buffer type context (unused in this\n * implementation).\n * @param tensor Pointer to the tensor for which the allocation size is\n * calculated.\n * @return The total allocation size in bytes required for the tensor in the\n * CANN buffer.\n */\nstatic size_t ggml_backend_cann_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft,\n                                                           const ggml_tensor *        tensor) {\n    size_t  size = ggml_nbytes(tensor);\n    int64_t ne0  = tensor->ne[0];\n\n    // Only check env once.\n    static bool weight_to_nz = parse_bool(get_env_as_lowercase(\"GGML_CANN_WEIGHT_NZ\").value_or(\"on\"));\n\n    // last line must bigger than 32, because every single op deal at\n    // least 32 bytes.\n    // TODO: quantized type?\n    // int64_t line_size = ne0 * ggml_element_size(tensor);\n    // int64_t line_size_align_32 = (line_size + 31) & ~31;\n    // size += (line_size_align_32 - line_size);\n    if (ggml_is_quantized(tensor->type)) {\n        if (ne0 % MATRIX_ROW_PADDING != 0) {\n            size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);\n        }\n    } else if (weight_to_nz && is_matmul_weight((const ggml_tensor *) tensor)) {\n        // NZ format weight are not support quantized yet.\n        // If ND tensor transform to NZ, size may changed.\n        int64_t shape[] = { tensor->ne[1], tensor->ne[0] };\n        GGML_ASSERT(tensor->ne[2] == 1);\n        GGML_ASSERT(tensor->ne[3] == 1);\n        const aclIntArray * acl_shape = aclCreateIntArray(shape, 2);\n        size_t              new_size;\n        ACL_CHECK(aclnnCalculateMatmulWeightSizeV2(acl_shape, ggml_cann_type_mapping(tensor->type), &new_size));\n        ACL_CHECK(aclDestroyIntArray(acl_shape));\n        size = std::max(size, new_size);\n    }\n\n    return size;\n\n    GGML_UNUSED(buft);\n}\n\nstatic bool ggml_backend_cann_buffer_type_is_host(ggml_backend_buffer_type_t buft) {\n    return false;\n\n    GGML_UNUSED(buft);\n}\n\n/**\n * @brief Interface for managing CANN buffer types in the GGML backend.\n *\n * Provides function pointers for allocating, querying properties, and managing\n * memory for CANN buffer types in the GGML backend.\n */\nstatic const ggml_backend_buffer_type_i ggml_backend_cann_buffer_type_interface = {\n    /* .get_name         = */ ggml_backend_cann_buffer_type_name,\n    /* .alloc_buffer     = */ ggml_backend_cann_buffer_type_alloc_buffer,\n    /* .get_alignment    = */ ggml_backend_cann_buffer_type_get_alignment,\n    /* .get_max_size     = */ NULL,  // defaults to SIZE_MAX\n    /* .get_alloc_size   = */ ggml_backend_cann_buffer_type_get_alloc_size,\n    /* .is_host          = */ ggml_backend_cann_buffer_type_is_host,\n};\n\n/**\n * @brief Retrieves the CANN buffer type for a specified device.\n *\n * This function initializes and returns the buffer type interface associated\n * with the given device. It ensures thread-safe access using a mutex.\n *\n * @param device The device index for which to retrieve the buffer type.\n * @return A pointer to the buffer type interface for the specified device, or\n * nullptr if the device index is out of range.\n */\nggml_backend_buffer_type_t ggml_backend_cann_buffer_type(int32_t device) {\n    static std::mutex           mutex;\n    std::lock_guard<std::mutex> lock(mutex);\n\n    if (device >= ggml_backend_cann_get_device_count()) {\n        return nullptr;\n    }\n\n    static ggml_backend_buffer_type ggml_backend_cann_buffer_types[GGML_CANN_MAX_DEVICES];\n\n    static bool ggml_backend_cann_buffer_type_initialized = false;\n\n    if (!ggml_backend_cann_buffer_type_initialized) {\n        for (int32_t i = 0; i < ggml_cann_info().device_count; i++) {\n            ggml_backend_cann_buffer_types[i] = {\n                /* .iface    = */ ggml_backend_cann_buffer_type_interface,\n                /* .device    = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), i),\n                /* .context  = */\n                new ggml_backend_cann_buffer_type_context{ i, \"CANN\" + std::to_string(i) },\n            };\n        }\n        ggml_backend_cann_buffer_type_initialized = true;\n    }\n\n    return &ggml_backend_cann_buffer_types[device];\n}\n\n/**\n * @brief Retrieves the name associated with a CANN host buffer type.\n *\n * This function returns the descriptive name associated with the specified\n * CANN host buffer type context.\n *\n * @param buft Pointer to the host buffer type context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_host_buffer_type_name(ggml_backend_buffer_type_t buft) {\n    return \"CANN_Host\";\n\n    GGML_UNUSED(buft);\n}\n\n/**\n * @brief Retrieves the name associated with a CANN host buffer.\n *\n * This function returns the descriptive name associated with the specified\n * CANN host buffer context.\n *\n * @param buft Pointer to the host buffer context.\n * @return Const pointer to the C-style string containing the name.\n */\nstatic const char * ggml_backend_cann_host_buffer_name(ggml_backend_buffer_t buffer) {\n    return \"CANN_Host\";\n\n    GGML_UNUSED(buffer);\n}\n\n/**\n * @brief Free resources associated with a CANN host buffer.\n *\n * This function frees the resources associated with a CANN host buffer, including\n * its context.\n *\n * @param buffer The CANN host buffer to free.\n */\nstatic void ggml_backend_cann_host_buffer_free(ggml_backend_buffer_t buffer) {\n    ACL_CHECK(aclrtFreeHost(buffer->context));\n}\n\n/**\n * @brief Allocates a new CANN host buffer of the specified size.\n *\n * This function allocates a new CANN host buffer with the given size.\n * @param size Size in bytes of the host buffer to allocate.\n * @return Pointer to the allocated host buffer, or nullptr if allocation fails.\n */\nstatic void * ggml_cann_host_malloc(size_t size) {\n    if (getenv(\"GGML_CANN_NO_PINNED\") != nullptr) {\n        return nullptr;\n    }\n\n    const size_t alignment = 128;\n    size                   = GGML_PAD(size, alignment);\n    if (size == 0) {\n        size = alignment;\n    }\n\n    void *   hostPtr = nullptr;\n    aclError err     = aclrtMallocHost((void **) &hostPtr, size);\n    if (err != ACL_SUCCESS) {\n        GGML_LOG_WARN(\"%s: failed to allocate %.2f MiB of pinned memory: %s\\n\", __func__, size / 1024.0 / 1024.0,\n                      aclGetRecentErrMsg());\n        return nullptr;\n    }\n    return hostPtr;\n}\n\n/**\n * @brief Allocates a new CANN host buffer of the specified type and size.\n *\n * @param buft Pointer to the host buffer type context.\n * @param size Size in bytes of the host buffer to allocate.\n * @return Pointer to the allocated host buffer, or CPU buffer pointer if allocation fails.\n */\nstatic ggml_backend_buffer_t ggml_backend_cann_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n                                                                             size_t                     size) {\n    void * hostPtr = ggml_cann_host_malloc(size);\n\n    if (hostPtr == nullptr) {\n        // fallback to cpu buffer\n        return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n    }\n\n    ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(hostPtr, size);\n    buffer->buft                 = buft;\n    buffer->iface.free_buffer    = ggml_backend_cann_host_buffer_free;\n\n    return buffer;\n}\n\n/**\n * @brief Interface for managing CANN host buffer types in the GGML backend.\n *\n * Provides function pointers for allocating, querying properties, and managing\n * memory for CANN buffer types in the GGML backend.\n */\nggml_backend_buffer_type_t ggml_backend_cann_host_buffer_type() {\n    static struct ggml_backend_buffer_type ggml_backend_cann_buffer_type_host = {\n        /* .iface    = */ {\n                           /* .get_name         = */ ggml_backend_cann_host_buffer_type_name,\n                           /* .alloc_buffer     = */ ggml_backend_cann_host_buffer_type_alloc_buffer,\n                           /* .get_alignment    = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,\n                           /* .get_max_size     = */ NULL,  // defaults to SIZE_MAX\n            /* .get_alloc_size   = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,\n                           /* .is_host          = */ ggml_backend_cpu_buffer_type()->iface.is_host,\n                           },\n        /* .device   = */\n        ggml_backend_reg_dev_get(ggml_backend_cann_reg(), 0),\n        /* .context  = */ nullptr,\n    };\n\n    return &ggml_backend_cann_buffer_type_host;\n}\n\n/**\n * @brief Computes the forward operation for a given tensor using CANN\n * operations.\n *\n * This function selects the appropriate CANN operation based on the type of\n * operation specified in the tensor and performs the computation.\n *\n * @param ctx The CANN context containing necessary resources and\n * configurations.\n * @param dst The destination tensor where the result of the computation will be\n * stored.\n * @return true if the computation was successful; false otherwise.\n */\nstatic bool ggml_cann_compute_forward(ggml_backend_cann_context & ctx, struct ggml_tensor * dst) {\n    switch (dst->op) {\n        case GGML_OP_REPEAT:\n            ggml_cann_repeat(ctx, dst);\n            break;\n        case GGML_OP_GET_ROWS:\n            ggml_cann_get_rows(ctx, dst);\n            break;\n        case GGML_OP_SET_ROWS:\n            ggml_cann_set_rows(ctx, dst);\n            break;\n        case GGML_OP_DUP:\n            ggml_cann_dup(ctx, dst);\n            break;\n        case GGML_OP_ADD:\n        case GGML_OP_ADD1:\n            ggml_cann_binary_op<aclnn_add>(ctx, dst);\n            break;\n        case GGML_OP_SUB:\n            ggml_cann_binary_op<aclnn_sub>(ctx, dst);\n            break;\n        case GGML_OP_ACC:\n            ggml_cann_acc(ctx, dst);\n            break;\n        case GGML_OP_MUL:\n            ggml_cann_binary_op<aclnn_mul>(ctx, dst);\n            break;\n        case GGML_OP_DIV:\n            ggml_cann_binary_op<aclnn_div>(ctx, dst);\n            break;\n        case GGML_OP_UNARY:\n            switch (ggml_get_unary_op(dst)) {\n                case GGML_UNARY_OP_ABS:\n                    GGML_CANN_CALL_OP_UNARY(Abs);\n                    break;\n                case GGML_UNARY_OP_NEG:\n                    GGML_CANN_CALL_OP_UNARY(Neg);\n                    break;\n                case GGML_UNARY_OP_GELU:\n                case GGML_UNARY_OP_GELU_ERF:\n                    // aclnnGelu internally uses the erf-based approximation.\n                    GGML_CANN_CALL_OP_UNARY(Gelu);\n                    break;\n                case GGML_UNARY_OP_SILU:\n                    GGML_CANN_CALL_OP_UNARY(Silu);\n                    break;\n                case GGML_UNARY_OP_GELU_QUICK:\n                    {\n                        auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n                            GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst);\n                        };\n                        ggml_cann_op_unary(lambda, ctx, dst);\n                    }\n                    break;\n                case GGML_UNARY_OP_TANH:\n                    GGML_CANN_CALL_OP_UNARY(Tanh);\n                    break;\n                case GGML_UNARY_OP_RELU:\n                    GGML_CANN_CALL_OP_UNARY(Relu);\n                    break;\n                case GGML_UNARY_OP_SIGMOID:\n                    GGML_CANN_CALL_OP_UNARY(Sigmoid);\n                    break;\n                case GGML_UNARY_OP_HARDSIGMOID:\n                    GGML_CANN_CALL_OP_UNARY(Hardsigmoid);\n                    break;\n                case GGML_UNARY_OP_HARDSWISH:\n                    GGML_CANN_CALL_OP_UNARY(Hardswish);\n                    break;\n                case GGML_UNARY_OP_EXP:\n                    GGML_CANN_CALL_OP_UNARY(Exp);\n                    break;\n                case GGML_UNARY_OP_ELU:\n                    ggml_cann_elu(ctx, dst);\n                    break;\n                case GGML_UNARY_OP_SGN:\n                    GGML_CANN_CALL_OP_UNARY(Sign);\n                    break;\n                case GGML_UNARY_OP_STEP:\n                    ggml_cann_step(ctx, dst);\n                    break;\n                default:\n                    return false;\n            }\n            break;\n        case GGML_OP_GLU:\n            switch (ggml_get_glu_op(dst)) {\n                case GGML_GLU_OP_REGLU:\n                    GGML_CANN_CALL_OP_UNARY_GATED(Relu);\n                    break;\n                case GGML_GLU_OP_GEGLU:\n                case GGML_GLU_OP_GEGLU_ERF:\n                    // aclnnGelu internally uses the erf-based approximation.\n                    GGML_CANN_CALL_OP_UNARY_GATED(Gelu);\n                    break;\n                case GGML_GLU_OP_SWIGLU:\n                    GGML_CANN_CALL_OP_UNARY_GATED(Silu);\n                    break;\n                case GGML_GLU_OP_GEGLU_QUICK:\n                    {\n                        auto lambda = [](ggml_backend_cann_context & ctx, aclTensor * acl_src, aclTensor * acl_dst) {\n                            GGML_CANN_CALL_ACLNN_OP(ctx, GeluV2, acl_src, 0, acl_dst);\n                        };\n                        ggml_cann_op_unary_gated(lambda, ctx, dst);\n                    }\n                    break;\n                default:\n                    return false;\n            }\n            break;\n        case GGML_OP_NORM:\n            ggml_cann_norm(ctx, dst);\n            break;\n        case GGML_OP_GROUP_NORM:\n            ggml_cann_group_norm(ctx, dst);\n            break;\n        case GGML_OP_L2_NORM:\n            ggml_cann_l2_norm(ctx, dst);\n            break;\n        case GGML_OP_CROSS_ENTROPY_LOSS:\n            ggml_cann_cross_entropy_loss(ctx, dst);\n            break;\n        case GGML_OP_CONCAT:\n            ggml_cann_concat(ctx, dst);\n            break;\n        case GGML_OP_UPSCALE:\n            ggml_cann_upsample_nearest2d(ctx, dst);\n            break;\n        case GGML_OP_PAD:\n            ggml_cann_pad(ctx, dst);\n            break;\n        case GGML_OP_ARANGE:\n            ggml_cann_arange(ctx, dst);\n            break;\n        case GGML_OP_TIMESTEP_EMBEDDING:\n            ggml_cann_timestep_embedding(ctx, dst);\n            break;\n        case GGML_OP_LEAKY_RELU:\n            ggml_cann_leaky_relu(ctx, dst);\n            break;\n        case GGML_OP_RMS_NORM:\n            ggml_cann_rms_norm(ctx, dst);\n            break;\n        case GGML_OP_MUL_MAT:\n            ggml_cann_mul_mat(ctx, dst);\n            break;\n        case GGML_OP_MUL_MAT_ID:\n            ggml_cann_mul_mat_id(ctx, dst);\n            break;\n        case GGML_OP_SCALE:\n            ggml_cann_scale(ctx, dst);\n            break;\n        case GGML_OP_SQR:\n            GGML_ASSERT(dst->src[1] == nullptr);\n            dst->src[1] = dst->src[0];\n            ggml_cann_binary_op<aclnn_mul>(ctx, dst);\n            break;\n        case GGML_OP_SQRT:\n            GGML_CANN_CALL_OP_UNARY(Sqrt);\n            break;\n        case GGML_OP_CLAMP:\n            ggml_cann_clamp(ctx, dst);\n            break;\n        case GGML_OP_CPY:\n            ggml_cann_cpy(ctx, dst);\n            break;\n        case GGML_OP_CONT:\n            ggml_cann_dup(ctx, dst);\n            break;\n        case GGML_OP_NONE:\n        case GGML_OP_RESHAPE:\n        case GGML_OP_VIEW:\n        case GGML_OP_PERMUTE:\n        case GGML_OP_TRANSPOSE:\n            break;\n        case GGML_OP_DIAG_MASK_INF:\n            ggml_cann_diag_mask(ctx, dst, -INFINITY);\n            break;\n        case GGML_OP_SOFT_MAX:\n            ggml_cann_softmax(ctx, dst);\n            break;\n        case GGML_OP_ROPE:\n            ggml_cann_rope(ctx, dst);\n            break;\n        case GGML_OP_IM2COL:\n            ggml_cann_im2col(ctx, dst);\n            break;\n        case GGML_OP_POOL_2D:\n            ggml_cann_pool2d(ctx, dst);\n            break;\n        case GGML_OP_SUM:\n            ggml_cann_sum(ctx, dst);\n            break;\n        case GGML_OP_SUM_ROWS:\n            ggml_cann_sum_rows(ctx, dst);\n            break;\n        case GGML_OP_ARGSORT:\n            ggml_cann_argsort(ctx, dst);\n            break;\n        case GGML_OP_ARGMAX:\n            ggml_cann_argmax(ctx, dst);\n            break;\n        case GGML_OP_COS:\n            ggml_cann_op_unary<aclnn_cos>(ctx, dst);\n            break;\n        case GGML_OP_SIN:\n            ggml_cann_op_unary<aclnn_sin>(ctx, dst);\n            break;\n        case GGML_OP_CONV_TRANSPOSE_1D:\n            ggml_cann_conv_transpose_1d(ctx, dst);\n            break;\n        case GGML_OP_LOG:\n            GGML_CANN_CALL_OP_UNARY(Log);\n            break;\n        case GGML_OP_MEAN:\n            ggml_cann_mean(ctx, dst);\n            break;\n        case GGML_OP_PAD_REFLECT_1D:\n            ggml_cann_pad_reflect_1d(ctx, dst);\n            break;\n        case GGML_OP_COUNT_EQUAL:\n            ggml_cann_count_equal(ctx, dst);\n            break;\n        case GGML_OP_FLASH_ATTN_EXT:\n            ggml_cann_flash_attn_ext(ctx, dst);\n            break;\n        case GGML_OP_OUT_PROD:\n            ggml_cann_out_prod(ctx, dst);\n            break;\n        case GGML_OP_GATED_LINEAR_ATTN:\n            ggml_cann_gated_linear_attn(ctx, dst);\n            break;\n        case GGML_OP_SSM_CONV:\n            ggml_cann_ssm_conv(ctx, dst);\n            break;\n        default:\n            return false;\n    }\n\n    return true;\n}\n\n// backend\n/**\n * @brief Retrieves the name associated with the CANN backend.\n *\n * This function returns the name assigned to the CANN backend, which is stored\n * in the context of the provided backend structure.\n *\n * @param backend Pointer to the CANN backend structure.\n * @return A pointer to a constant string representing the backend name.\n */\nstatic const char * ggml_backend_cann_name(ggml_backend_t backend) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n\n    return cann_ctx->name.c_str();\n}\n\n/**\n * @brief Frees resources associated with the CANN backend.\n *\n * This function releases resources associated with the CANN backend context\n * and resets the device associated with the backend to its initial state.\n *\n * @param backend Pointer to the CANN backend structure to be freed.\n */\nstatic void ggml_backend_cann_free(ggml_backend_t backend) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ACL_CHECK(aclrtSynchronizeDevice());\n    ACL_CHECK(aclrtResetDevice(cann_ctx->device));\n\n    delete cann_ctx;\n    delete backend;\n}\n\n/**\n * @brief Sets tensor data asynchronously in the CANN backend.\n *\n * This function asynchronously sets tensor data in the CANN backend.\n *\n * @param backend Pointer to the CANN backend structure.\n * @param tensor Pointer to the tensor structure to set data for.\n * @param data Pointer to the host data to copy to the tensor.\n * @param offset Offset in bytes within the host data.\n * @param size Size of the data to copy in bytes.\n */\nstatic void ggml_backend_cann_set_tensor_async(ggml_backend_t backend,\n                                               ggml_tensor *  tensor,\n                                               const void *   data,\n                                               size_t         offset,\n                                               size_t         size) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ggml_backend_buffer_t       buf      = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n    GGML_ASSERT(buf->buft == ggml_backend_cann_buffer_type(cann_ctx->device) && \"unsupported buffer type\");\n    GGML_ASSERT(!ggml_is_quantized(tensor->type));\n\n    ACL_CHECK(aclrtMemcpyAsync((char *) tensor->data + offset, size, data, size, ACL_MEMCPY_HOST_TO_DEVICE,\n                               cann_ctx->stream()));\n}\n\n/**\n * @brief Gets tensor data asynchronously in the CANN backend.\n *\n * This function asynchronously gets tensor data in the CANN backend.\n *\n * @param backend Pointer to the CANN backend structure.\n * @param tensor Pointer to the tensor structure to get data from.\n * @param data Pointer to the host data to copy from the tensor.\n * @param offset Offset in bytes within the host data.\n * @param size Size of the data to copy in bytes.\n */\nstatic void ggml_backend_cann_get_tensor_async(ggml_backend_t      backend,\n                                               const ggml_tensor * tensor,\n                                               void *              data,\n                                               size_t              offset,\n                                               size_t              size) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ggml_backend_buffer_t       buf      = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;\n\n    GGML_ASSERT(buf->buft == ggml_backend_cann_buffer_type(cann_ctx->device) && \"unsupported buffer type\");\n    GGML_ASSERT(!ggml_is_quantized(tensor->type));\n\n    ACL_CHECK(aclrtMemcpyAsync(data, size, (char *) tensor->data + offset, size, ACL_MEMCPY_DEVICE_TO_HOST,\n                               cann_ctx->stream()));\n}\n\n/**\n * @brief Asynchronously copies tensor data between CANN backends.\n *\n * This function copies tensor data asynchronously between two CANN backends. It\n * checks if both tensors reside in CANN buffers and whether the devices support\n * peer-to-peer access for direct copying. If not, it returns false.\n *\n * @param backend_src Pointer to the source CANN backend structure.\n * @param backend_dst Pointer to the destination CANN backend structure.\n * @param src Pointer to the source tensor to copy data from.\n * @param dst Pointer to the destination tensor to copy data to.\n * @return true if the copy operation succeeds, false otherwise.\n */\nstatic bool ggml_backend_cann_cpy_tensor_async(ggml_backend_t      backend_src,\n                                               ggml_backend_t      backend_dst,\n                                               const ggml_tensor * src,\n                                               ggml_tensor *       dst) {\n    GGML_ASSERT(ggml_backend_is_cann(backend_src) || ggml_backend_is_cann(backend_dst));\n\n    GGML_ASSERT(!is_matmul_weight((const ggml_tensor *) src));\n\n    if (!ggml_backend_buft_is_cann(src->buffer->buft) || !ggml_backend_buft_is_cann(dst->buffer->buft)) {\n        return false;\n    }\n\n    ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;\n    ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;\n\n    ggml_backend_cann_context * cann_ctx_src = (ggml_backend_cann_context *) backend_src->context;\n    ggml_backend_cann_context * cann_ctx_dst = (ggml_backend_cann_context *) backend_dst->context;\n\n    size_t copy_size = ggml_nbytes(dst);\n    if (copy_size == 0) {\n        return true;\n    }\n    if (backend_src != backend_dst) {\n#ifdef ASCEND_310P\n        // TODO: Support 310p P2P copy\n        return false;\n#endif\n        ggml_backend_cann_buffer_context * buf_ctx_src = (ggml_backend_cann_buffer_context *) buf_src->context;\n        ggml_backend_cann_buffer_context * buf_ctx_dst = (ggml_backend_cann_buffer_context *) buf_dst->context;\n\n        GGML_ASSERT(cann_ctx_src->device == buf_ctx_src->device);\n        GGML_ASSERT(cann_ctx_dst->device == buf_ctx_dst->device);\n\n        int32_t canAccessPeer = 0;\n        ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, cann_ctx_src->device, cann_ctx_dst->device));\n        if (!canAccessPeer) {\n            return false;\n        }\n\n        // need open both directions for memcpyasync between devices.\n        ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_src->device, 0));\n        ggml_cann_set_device(cann_ctx_src->device);\n        ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_dst->device, 0));\n\n        // wait for task_queue empty to keep task order.\n        ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size, ACL_MEMCPY_DEVICE_TO_DEVICE,\n                                   cann_ctx_src->stream()));\n        // record event on src stream after the copy\n        // TODO: this event is not effective with acl graph mode, change to use aclrtSynchronizeStream\n        // if (!cann_ctx_src->copy_event) {\n        //     ACL_CHECK(aclrtCreateEventWithFlag(&cann_ctx_src->copy_event, ACL_EVENT_SYNC));\n        // }\n        // ACL_CHECK(aclrtRecordEvent(cann_ctx_src->copy_event, cann_ctx_src->stream()));\n\n        // // wait on dst stream for the copy to complete\n        // ggml_cann_set_device(cann_ctx_dst->device);\n        // ACL_CHECK(aclrtStreamWaitEvent(cann_ctx_dst->stream(), cann_ctx_src->copy_event));\n        ACL_CHECK(aclrtSynchronizeStream(cann_ctx_src->stream()));\n    } else {\n        // src and dst are on the same backend\n        ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size, ACL_MEMCPY_DEVICE_TO_DEVICE,\n                                   cann_ctx_dst->stream()));\n    }\n\n    return true;\n}\n\n/**\n * @brief Synchronizes a CANN backend.\n *\n * This function synchronizes the specified CANN backend by waiting for all\n * operations in its associated stream to complete.\n *\n * @param backend Pointer to the CANN backend structure to synchronize.\n */\nstatic void ggml_backend_cann_synchronize(ggml_backend_t backend) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ggml_cann_set_device(cann_ctx->device);\n    ACL_CHECK(aclrtSynchronizeStream(cann_ctx->stream()));\n}\n\n/**\n * @brief Check if CANN backend can fuse the specified operation sequence\n *\n * This function determines whether an operation sequence starting from the specified node\n * can be fused into an optimized operation in the CANN backend. Operation fusion can reduce\n * memory access overhead and improve computational efficiency.\n *\n * @param cgraph Pointer to the computation graph\n * @param node_idx Index of the starting node in the computation graph\n * @param ops Sequence of operation types to check for fusion\n * @return true if the operations can be fused\n * @return false if the operations cannot be fused\n */\nstatic bool ggml_cann_can_fuse(const struct ggml_cgraph *          cgraph,\n                               int                                 node_idx,\n                               std::initializer_list<enum ggml_op> ops) {\n    if (!ggml_can_fuse(cgraph, node_idx, ops)) {\n        return false;\n    }\n\n    // CANN backend supports fusing ADD + RMS_NORM operations\n    if ((ops.size() == 2) && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM) {\n        ggml_tensor * add_node = cgraph->nodes[node_idx];\n        // TODO: support broadcast for ADD + RMS_NORM\n        if (add_node->src[0]->ne[0] != add_node->src[1]->ne[0] || add_node->src[0]->ne[1] != add_node->src[1]->ne[1] ||\n            add_node->src[0]->ne[2] != add_node->src[1]->ne[2] || add_node->src[0]->ne[3] != add_node->src[1]->ne[3]) {\n            return false;\n        }\n        return true;\n    }\n\n    return false;\n}\n\n/**\n * @brief Evaluate the computation graph and optionally capture or execute it using CANN graph API.\n *\n * If CANN graph execution is enabled and graph capture is required, this function begins\n * graph capture, runs the graph, ends capture, and stores the captured graph.\n *\n * Otherwise, it falls back to op-by-op execution using the CANN compute kernel dispatcher.\n *\n * @param cann_ctx                     The CANN backend context.\n * @param cgraph                       The ggml computation graph.\n * @param use_cann_graph               Whether to use CANN graph execution.\n * @param cann_graph_capture_required  Whether graph capture is needed due to graph changes.\n */\nstatic void evaluate_and_capture_cann_graph(ggml_backend_cann_context * cann_ctx,\n                                            ggml_cgraph *               cgraph,\n                                            bool                        use_cann_graph,\n                                            bool                        cann_graph_capture_required) {\n#ifdef USE_ACL_GRAPH\n    if (use_cann_graph && cann_graph_capture_required) {  // Begin CANN graph capture\n        ACL_CHECK(aclmdlRICaptureBegin(cann_ctx->stream(), ACL_MODEL_RI_CAPTURE_MODE_GLOBAL));\n    }\n#endif  // USE_ACL_GRAPH\n    // Only perform the graph execution if CANN graphs are not enabled, or we are capturing the graph.\n    // With the use of CANN graphs, the execution will be performed by the graph launch.\n    static bool opt_fusion = parse_bool(get_env_as_lowercase(\"GGML_CANN_OPERATOR_FUSION\").value_or(\"\"));\n\n    if (!use_cann_graph || cann_graph_capture_required) {\n        for (int i = 0; i < cgraph->n_nodes; i++) {\n            ggml_tensor * node = cgraph->nodes[i];\n            if (opt_fusion) {\n                if (ggml_cann_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM })) {\n                    ggml_cann_op_add_rms_norm_fused(*cann_ctx, node, cgraph->nodes[i + 1]);\n                    i++;\n                    continue;\n                }\n            }\n\n            if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE ||\n                node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {\n                continue;\n            }\n\n            if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n                continue;\n            }\n\n            bool ok = ggml_cann_compute_forward(*cann_ctx, node);\n            if (!ok) {\n                GGML_LOG_ERROR(\"%s: op not supported %s (%s)\\n\", __func__, node->name, ggml_op_name(node->op));\n            }\n            GGML_ASSERT(ok);\n        }\n    }\n\n#ifdef USE_ACL_GRAPH\n    if (use_cann_graph) {\n        GGML_ASSERT(!cann_ctx->graph_lru_cache.cache_list.empty());\n        ggml_cann_graph * matched_graph = cann_ctx->graph_lru_cache.cache_list.front();\n\n        if (cann_graph_capture_required) {  // End CANN graph capture\n            ACL_CHECK(aclmdlRICaptureEnd(cann_ctx->stream(), &matched_graph->graph));\n        }\n\n        // Execute CANN graph\n        ACL_CHECK(aclmdlRIExecuteAsync(matched_graph->graph, cann_ctx->stream()));\n    }\n#endif  // USE_ACL_GRAPH\n}\n\n/**\n * @brief Computes a computational graph using a CANN backend.\n *\n * This function computes the operations defined in the computational graph\n * using the specified CANN backend.\n *\n * @param backend Pointer to the CANN backend structure to use for computation.\n * @param cgraph Pointer to the computational graph structure containing nodes\n *               representing operations to be computed.\n * @return enum ggml_status Returns GGML_STATUS_SUCCESS if computation\n *         completes successfully, otherwise an appropriate error status.\n */\nstatic enum ggml_status ggml_backend_cann_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ggml_cann_set_device(cann_ctx->device);\n    g_nz_workspaces[cann_ctx->device].clear();\n\n    // calculate rope cache for fist layer in current device.\n    cann_ctx->rope_cache.cached = false;\n\n    bool graph_capture_required = false;\n#ifdef USE_ACL_GRAPH\n    bool use_cann_graph = true;\n\n    static bool prefill_use_graph = parse_bool(get_env_as_lowercase(\"GGML_CANN_PREFILL_USE_GRAPH\").value_or(\"\"));\n    if (!prefill_use_graph) {\n        // Do not use acl_graph for prefill.\n        for (int i = 0; i < cgraph->n_nodes; i++) {\n            ggml_tensor * node = cgraph->nodes[i];\n            // TODO: Optimize here. Currently, we can only\n            // get seq_len by FA's input.\n            if (node->op == GGML_OP_FLASH_ATTN_EXT) {\n                // Q -> src[0], shape: [B, S, N, D]\n                use_cann_graph = (node->src[0]->ne[1] == 1);\n                break;\n            }\n        }\n    }\n\n    if (!cann_ctx->acl_graph_mode) {\n        use_cann_graph = false;\n    }\n\n    if (use_cann_graph) {\n        // If no matching graph is found, the graph needs to be recaptured.\n        graph_capture_required = !cann_ctx->graph_lru_cache.find_and_move_to_front(cgraph);\n        if (graph_capture_required) {\n            // If no matching graph is found, add a new ACL graph.\n            ggml_cann_graph * new_graph = ggml_cann_graph::create_from_cgraph(cgraph);\n            cann_ctx->graph_lru_cache.push(new_graph);\n        }\n    }\n#else\n    bool use_cann_graph = false;\n#endif  // USE_ACL_GRAPH\n    evaluate_and_capture_cann_graph(cann_ctx, cgraph, use_cann_graph, graph_capture_required);\n\n    return GGML_STATUS_SUCCESS;\n}\n\n/**\n * @brief Checks if the CANN backend supports a specific operation.\n *\n * This function checks whether the specified operation is supported by the\n * CANN backend.\n *\n * @param backend Pointer to the CANN backend structure to check support for\n *                the operation.\n * @param op Pointer to the tensor representing the operation to check.\n * @return bool Returns true if the operation is supported by the backend,\n *              otherwise false.\n */\nstatic bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n    switch (op->op) {\n        case GGML_OP_UNARY:\n            switch (ggml_get_unary_op(op)) {\n                case GGML_UNARY_OP_ABS:\n                case GGML_UNARY_OP_NEG:\n                case GGML_UNARY_OP_GELU:\n                case GGML_UNARY_OP_SILU:\n                case GGML_UNARY_OP_RELU:\n                case GGML_UNARY_OP_SIGMOID:\n                case GGML_UNARY_OP_HARDSIGMOID:\n                case GGML_UNARY_OP_HARDSWISH:\n                case GGML_UNARY_OP_GELU_QUICK:\n                case GGML_UNARY_OP_TANH:\n                case GGML_UNARY_OP_EXP:\n                case GGML_UNARY_OP_ELU:\n                case GGML_UNARY_OP_SGN:\n                case GGML_UNARY_OP_STEP:\n                case GGML_UNARY_OP_GELU_ERF:\n                    return true;\n                default:\n                    return false;\n            }\n        case GGML_OP_GLU:\n            switch (ggml_get_glu_op(op)) {\n                case GGML_GLU_OP_REGLU:\n                case GGML_GLU_OP_GEGLU:\n                case GGML_GLU_OP_SWIGLU:\n                case GGML_GLU_OP_GEGLU_ERF:\n                case GGML_GLU_OP_GEGLU_QUICK:\n                    return true;\n                default:\n                    return false;\n            }\n            break;\n        case GGML_OP_MUL_MAT:\n            {\n                switch (op->src[0]->type) {\n                    case GGML_TYPE_F16:\n                    case GGML_TYPE_F32:\n                        return true;\n                    case GGML_TYPE_Q8_0:\n                    case GGML_TYPE_Q4_0:\n#ifdef ASCEND_310P\n                        // Q4 && Q8 per group is not support on 310p device\n                        return false;\n#endif\n                        // only support contiguous for quantized types.\n                        return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n                    default:\n                        return false;\n                }\n            }\n        case GGML_OP_MUL_MAT_ID:\n            switch (op->src[0]->type) {\n                case GGML_TYPE_F16:\n                case GGML_TYPE_F32:\n                    return true;\n                case GGML_TYPE_Q8_0:\n                case GGML_TYPE_Q4_0:\n#ifdef ASCEND_310P\n                    // Q4 && Q8 per group is not support on 310p device\n                    return false;\n#endif\n                    // only support contiguous for quantized types.\n                    return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);\n                default:\n                    return false;\n            }\n        // embedding\n        case GGML_OP_GET_ROWS:\n            {\n                switch (op->src[0]->type) {\n                    case GGML_TYPE_F32:\n                    case GGML_TYPE_F16:\n                    case GGML_TYPE_Q8_0:\n                        return true;\n                    default:\n                        return false;\n                }\n            }\n            break;\n        case GGML_OP_SET_ROWS:\n            {\n                switch (op->type) {\n                    case GGML_TYPE_F32:\n                    case GGML_TYPE_F16:\n                        return true;\n                    default:\n                        return false;\n                }\n            }\n            break;\n        case GGML_OP_CPY:\n            {\n                ggml_tensor * src = op->src[0];\n                if ((op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) ||\n                    (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16)) {\n                    // only support F32 and F16.\n                    return false;\n                }\n                return true;\n            }\n            break;\n        case GGML_OP_CONT:\n            {\n                // TODO: support GGML_TYPE_BF16\n                switch (op->src[0]->type) {\n                    case GGML_TYPE_F32:\n                    case GGML_TYPE_F16:\n                        return true;\n                    default:\n                        return false;\n                }\n            }\n        case GGML_OP_ROPE:\n            {\n                if (op->src[0]->ne[0] > 896) {\n                    return false;\n                }\n#ifdef ASCEND_310P\n                // TODO: Support rope_dim < ne00(dim)\n                if (op->src[0]->ne[0] != op->op_params[1]) {\n                    return false;\n                }\n                if (!ggml_is_contiguous(op->src[0])) {\n                    return false;\n                }\n#endif\n                return true;\n            }\n        case GGML_OP_UPSCALE:\n            {\n                // aclnnUpsampleNearest2dGetWorkspaceSize not support\n                // selfDimN[2]/outDimN[2] or selfDimC[3]/outDimC[3] not equal\n                if (op->src[0]->ne[2] * op->ne[3] != op->src[0]->ne[3] * op->ne[2]) {\n                    return false;\n                }\n                if (op->op_params[0] != GGML_SCALE_MODE_NEAREST) {\n                    return false;\n                }\n                if (op->op_params[0] & GGML_SCALE_FLAG_ANTIALIAS) {\n                    return false;\n                }\n                return true;\n            }\n        case GGML_OP_POOL_2D:\n            {\n                const int32_t * opts = (const int32_t *) op->op_params;\n#ifdef ASCEND_310P\n                enum ggml_op_pool opt = static_cast<ggml_op_pool>(opts[0]);\n                if (opt == GGML_OP_POOL_MAX) {\n                    return false;\n                }\n#endif\n                const int k0 = opts[1];\n                const int k1 = opts[2];\n                const int p0 = opts[5];\n                const int p1 = opts[6];\n                // value of paddingH should be at most half of kernelH\n                // value of paddingW should be at most half of kernelW\n                return (p0 <= (k0 / 2)) && (p1 <= (k1 / 2));\n            }\n        case GGML_OP_SUM:\n            return ggml_is_contiguous_rows(op->src[0]);\n        case GGML_OP_L2_NORM:\n        case GGML_OP_CROSS_ENTROPY_LOSS:\n        case GGML_OP_DUP:\n        case GGML_OP_IM2COL:\n        case GGML_OP_CONCAT:\n        case GGML_OP_REPEAT:\n        case GGML_OP_NONE:\n        case GGML_OP_RESHAPE:\n        case GGML_OP_VIEW:\n        case GGML_OP_PERMUTE:\n        case GGML_OP_TRANSPOSE:\n        case GGML_OP_NORM:\n        case GGML_OP_ADD:\n        case GGML_OP_ADD1:\n        case GGML_OP_SUB:\n        case GGML_OP_MUL:\n        case GGML_OP_DIV:\n        case GGML_OP_RMS_NORM:\n        case GGML_OP_SQR:\n        case GGML_OP_SQRT:\n        case GGML_OP_CLAMP:\n        case GGML_OP_DIAG_MASK_INF:\n        case GGML_OP_SUM_ROWS:\n        case GGML_OP_ARGSORT:\n        case GGML_OP_ACC:\n        case GGML_OP_GROUP_NORM:\n            return true;\n        case GGML_OP_PAD:\n            // TODO: add circular padding support for cann, see https://github.com/ggml-org/llama.cpp/pull/16985\n            return ggml_get_op_params_i32(op, 8) == 0;\n        case GGML_OP_ARANGE:\n        case GGML_OP_TIMESTEP_EMBEDDING:\n        case GGML_OP_LEAKY_RELU:\n        case GGML_OP_ARGMAX:\n        case GGML_OP_COS:\n        case GGML_OP_SIN:\n        case GGML_OP_LOG:\n        case GGML_OP_MEAN:\n        case GGML_OP_PAD_REFLECT_1D:\n        case GGML_OP_COUNT_EQUAL:\n        case GGML_OP_GATED_LINEAR_ATTN:\n            return true;\n        case GGML_OP_OUT_PROD:\n            {\n#ifdef ASCEND_310P\n                // Ger is not supported on 310p device\n                return false;\n#endif\n                switch (op->src[0]->type) {\n                    case GGML_TYPE_F16:\n                    case GGML_TYPE_F32:\n                        return true;\n                    default:\n                        return false;\n                }\n            }\n        case GGML_OP_CONV_TRANSPOSE_1D:\n            return true;\n        case GGML_OP_SCALE:\n            float bias;\n            memcpy(&bias, (const float *) (op->op_params) + 1, sizeof(float));\n            return bias == 0.0f;  // TODO: support bias != 0.0f\n        case GGML_OP_SOFT_MAX:\n            // TODO: support attention sinks [TAG_ATTN_SINKS]\n            if (op->src[2]) {\n                return false;\n            }\n            return true;\n        case GGML_OP_FLASH_ATTN_EXT:\n            {\n#ifdef ASCEND_310P\n                // FA not support on 310p device\n                return false;\n#endif\n                // derived from [ggml-cuda.cu]\n                if (op->src[1]->type != GGML_TYPE_F16 || op->src[2]->type != GGML_TYPE_F16) {\n                    return false;\n                }\n                if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_F32 &&\n                    op->src[1]->type != GGML_TYPE_BF16) {\n                    return false;\n                }\n                if (op->type != GGML_TYPE_F16 && op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_BF16) {\n                    return false;\n                }\n                // TODO: support attention sinks [TAG_ATTN_SINKS]\n                if (op->src[4]) {\n                    return false;\n                }\n                if (op->src[1]->ne[0] != op->src[2]->ne[0]) {\n                    // different head sizes of K and V are not supported yet\n                    return false;\n                }\n                if (op->src[0]->ne[0] % 16 != 0) {\n                    // TODO: padding to support\n                    return false;\n                }\n                float logitSoftcap = 0.0f;\n                memcpy(&logitSoftcap, (const float *) (op->op_params) + 2, sizeof(float));\n                if (logitSoftcap != 0.0f) {\n                    return false;\n                }\n                return true;\n            }\n        case GGML_OP_SSM_CONV:\n            return true;\n        default:\n            return false;\n    }\n\n    GGML_UNUSED(dev);\n}\n\n/**\n * @brief Records an event on the CANN backend stream.\n *\n * This function records the given event on the ACL runtime stream associated\n * with the backend context.\n *\n * @param event Pointer to the event structure to be recorded.\n */\nstatic void ggml_backend_cann_event_record(ggml_backend_t backend, ggml_backend_event_t event) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    ACL_CHECK(aclrtRecordEvent((aclrtEvent) event->context, cann_ctx->stream()));\n}\n\n/**\n * @brief Waits for a recorded event to complete on the CANN backend stream.\n *\n * This function makes the given backend wait for the event to complete on its\n * ACL runtime stream.\n *\n * @param backend Pointer to the backend structure.\n * @param event Pointer to the event structure that the backend needs to wait\n * for.\n */\nstatic void ggml_backend_cann_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {\n    ggml_backend_cann_context * cann_ctx = (ggml_backend_cann_context *) backend->context;\n    if (ggml_backend_is_cann(backend)) {\n        ACL_CHECK(aclrtStreamWaitEvent(cann_ctx->stream(), (aclrtEvent) event->context));\n    } else {\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\n/**\n * @brief Structure defining the interface for the CANN backend.\n *\n * This structure contains function pointers for various operations\n * supported by the CANN backend, including name retrieval, memory\n * management, tensor operations, synchronization, and event handling.\n */\nstatic const ggml_backend_i ggml_backend_cann_interface = {\n    /* .get_name                = */ ggml_backend_cann_name,\n    /* .free                    = */ ggml_backend_cann_free,\n    /* .set_tensor_async        = */ ggml_backend_cann_set_tensor_async,\n    /* .get_tensor_async        = */ ggml_backend_cann_get_tensor_async,\n    /* .cpy_tensor_async        = */ ggml_backend_cann_cpy_tensor_async,\n    /* .synchronize             = */ ggml_backend_cann_synchronize,\n    /* .graph_plan_create       = */ NULL,\n    /* .graph_plan_free         = */ NULL,\n    /* .graph_plan_update       = */ NULL,\n    /* .graph_plan_compute      = */ NULL,\n    /* .graph_compute           = */ ggml_backend_cann_graph_compute,\n    /* .event_record            = */ ggml_backend_cann_event_record,\n    /* .event_wait              = */ ggml_backend_cann_event_wait,\n    /* .graph_optimize          = */ NULL,\n};\n\n/**\n * @brief Return the hardcoded GUID for the CANN backend.\n *\n * This function returns a static GUID which uniquely identifies the CANN\n * backend.\n *\n * @return A pointer to the static GUID.\n */\nstatic ggml_guid_t ggml_backend_cann_guid() {\n    static ggml_guid guid = { 0xa1, 0x94, 0xaf, 0xac, 0xbd, 0x4f, 0x47, 0x34,\n                              0xbe, 0x1a, 0x9e, 0x71, 0x1f, 0x9e, 0xed, 0x64 };\n    return &guid;\n}\n\n// backend device\nstruct ggml_backend_cann_device_context {\n    int         device;\n    std::string name;\n    std::string description;\n    int op_offload_min_batch_size;\n};\n\nstatic const char * ggml_backend_cann_device_get_name(ggml_backend_dev_t dev) {\n    ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n    return ctx->name.c_str();\n}\n\nstatic const char * ggml_backend_cann_device_get_description(ggml_backend_dev_t dev) {\n    ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n    return ctx->description.c_str();\n}\n\nstatic void ggml_backend_cann_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n    ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n    ggml_backend_cann_get_device_memory(ctx->device, free, total);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_cann_device_get_type(ggml_backend_dev_t dev) {\n    GGML_UNUSED(dev);\n    return GGML_BACKEND_DEVICE_TYPE_GPU;\n}\n\nstatic void ggml_backend_cann_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {\n    props->name        = ggml_backend_cann_device_get_name(dev);\n    props->description = ggml_backend_cann_device_get_description(dev);\n    props->type        = ggml_backend_cann_device_get_type(dev);\n    ggml_backend_cann_device_get_memory(dev, &props->memory_free, &props->memory_total);\n\n    bool host_buffer = getenv(\"GGML_CANN_NO_PINNED\") == nullptr;\n\n    props->caps = {\n        /* .async                 = */ false,\n        /* .host_buffer           = */ host_buffer,\n        /* .buffer_from_host_ptr  = */ false,\n        /* .events                = */ true,\n    };\n}\n\nstatic ggml_backend_t ggml_backend_cann_device_init(ggml_backend_dev_t dev, const char * params) {\n    GGML_UNUSED(params);\n    ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n    return ggml_backend_cann_init(ctx->device);\n}\n\n/**\n * @brief Checks if the CANN backend supports a specific backend buffer type.\n *\n * This function determines whether the CANN backend supports the given backend\n * buffer type by comparing the device context of the backend and buffer type.\n * It returns true if the devices are same between the backend context and\n * buffer type context.\n *\n * @param backend Pointer to the CANN backend.\n * @param buft Pointer to the backend buffer type to check.\n * @return bool Returns true if the CANN backend supports the buffer type,\n *              otherwise false.\n */\nstatic bool ggml_backend_cann_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n    if (ggml_backend_buft_is_cann(buft)) {\n        ggml_backend_cann_device_context *      dev_ctx  = (ggml_backend_cann_device_context *) dev->context;\n        ggml_backend_cann_buffer_type_context * buft_ctx = (ggml_backend_cann_buffer_type_context *) buft->context;\n        return buft_ctx->device == dev_ctx->device;\n    }\n    return false;\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cann_device_get_buffer_type(ggml_backend_dev_t dev) {\n    ggml_backend_cann_device_context * ctx = (ggml_backend_cann_device_context *) dev->context;\n    return ggml_backend_cann_buffer_type(ctx->device);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cann_device_get_host_buffer_type(ggml_backend_dev_t dev) {\n    GGML_UNUSED(dev);\n    return ggml_backend_cann_host_buffer_type();\n}\n\n/**\n * @brief Determines if a tensor operation should be offloaded to the CANN\n * backend.\n *\n * This function checks if a given tensor operation should be offloaded to the\n * CANN backend based on the operation type and the size of the tensor. It\n * returns true if the second dimension (ne[1]) of the tensor is greater than or\n * equal to the minimum batch size and the operation is not GGML_OP_GET_ROWS.\n *\n * @param backend Pointer to the CANN backend.\n * @param op Pointer to the tensor operation to check.\n * @return bool Returns true if the operation should be offloaded, otherwise\n * false.\n */\nstatic bool ggml_backend_cann_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {\n    ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *)dev->context;\n\n    return op->ne[1] >= dev_ctx->op_offload_min_batch_size && op->op != GGML_OP_GET_ROWS;\n}\n\n/**\n * @brief Creates a new event for the CANN backend device.\n *\n * This function initializes a new event for the CANN backend by setting the\n * device and creating an ACL runtime event. The created event is then wrapped\n * in a ggml_backend_event structure and returned.\n *\n * @param backend Pointer to the CANN backend.\n * @return ggml_backend_event_t Returns a pointer to the new event structure.\n */\nstatic ggml_backend_event_t ggml_backend_cann_device_event_new(ggml_backend_dev_t dev) {\n    ggml_backend_cann_device_context * dev_ctx = (ggml_backend_cann_device_context *) dev->context;\n\n    ggml_cann_set_device(dev_ctx->device);\n\n    aclrtEvent event;\n    ACL_CHECK(aclrtCreateEvent(&event));\n\n    return new ggml_backend_event{\n        /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), dev_ctx->device),\n        /* .context = */ event,\n    };\n}\n\n/**\n * @brief Frees a CANN backend event.\n *\n * This function destroys the ACL runtime event associated with the given CANN\n * backend event and then deletes the event structure itself.\n *\n * @param event Pointer to the event structure to be freed.\n */\nstatic void ggml_backend_cann_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n    ACL_CHECK(aclrtDestroyEvent((aclrtEvent) event->context));\n\n    delete event;\n    GGML_UNUSED(dev);\n}\n\n/**\n * @brief Synchronizes the given event on the CANN backend.\n *\n * This function waits for the specified event to complete on the ACL runtime.\n *\n * @param event Pointer to the event structure to be synchronized.\n */\nstatic void ggml_backend_cann_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {\n    ACL_CHECK(aclrtSynchronizeEvent((aclrtEvent) event->context));\n\n    GGML_UNUSED(dev);\n}\n\nstatic const ggml_backend_device_i ggml_backend_cann_device_interface = {\n    /* .get_name                = */ ggml_backend_cann_device_get_name,\n    /* .get_description         = */ ggml_backend_cann_device_get_description,\n    /* .get_memory              = */ ggml_backend_cann_device_get_memory,\n    /* .get_type                = */ ggml_backend_cann_device_get_type,\n    /* .get_props               = */ ggml_backend_cann_device_get_props,\n    /* .init_backend            = */ ggml_backend_cann_device_init,  // called for every card\n    /* .get_buffer_type         = */ ggml_backend_cann_device_get_buffer_type,\n    /* .get_host_buffer_type    = */ ggml_backend_cann_device_get_host_buffer_type,\n    /* .buffer_from_host_ptr    = */ NULL,  // not supported for CANN\n    /* .supports_op             = */ ggml_backend_cann_supports_op,\n    /* .supports_buft           = */ ggml_backend_cann_supports_buft,\n    /* .offload_op              = */ ggml_backend_cann_offload_op,\n    /* .event_new               = */ ggml_backend_cann_device_event_new,\n    /* .event_free              = */ ggml_backend_cann_device_event_free,\n    /* .event_synchronize       = */ ggml_backend_cann_device_event_synchronize,\n};\n\n// backend reg\nstruct ggml_backend_cann_reg_context {\n    std::vector<ggml_backend_dev_t> devices;\n};\n\nstatic const char * ggml_backend_cann_reg_get_name(ggml_backend_reg_t reg) {\n    GGML_UNUSED(reg);\n    return GGML_CANN_NAME;\n}\n\nstatic size_t ggml_backend_cann_reg_get_device_count(ggml_backend_reg_t reg) {\n    ggml_backend_cann_reg_context * ctx = (ggml_backend_cann_reg_context *) reg->context;\n    return ctx->devices.size();\n}\n\nstatic ggml_backend_dev_t ggml_backend_cann_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n    ggml_backend_cann_reg_context * ctx = (ggml_backend_cann_reg_context *) reg->context;\n    GGML_ASSERT(index < ctx->devices.size());\n    return ctx->devices[index];\n}\n\nstatic void * ggml_backend_cann_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n    GGML_UNUSED(reg);\n    GGML_UNUSED(name);\n    // reserved for future use\n    return nullptr;\n}\n\nstatic const ggml_backend_reg_i ggml_backend_cann_reg_interface = {\n    /* .get_name          = */ ggml_backend_cann_reg_get_name,\n    /* .get_device_count  = */ ggml_backend_cann_reg_get_device_count,\n    /* .get_device        = */ ggml_backend_cann_reg_get_device,\n    /* .get_proc_address  = */ ggml_backend_cann_reg_get_proc_address,\n};\n\n// backend registry, called only once for cann backend\nggml_backend_reg_t ggml_backend_cann_reg() {\n    static ggml_backend_reg reg;\n    static bool             initialized = false;\n\n    {\n        static std::mutex           mutex;\n        std::lock_guard<std::mutex> lock(mutex);\n        if (!initialized) {\n            aclInit(nullptr);\n            ggml_backend_cann_reg_context * ctx = new ggml_backend_cann_reg_context;\n            const int min_batch_size = getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\") ? atoi(getenv(\"GGML_OP_OFFLOAD_MIN_BATCH\")) : 32;\n\n            for (int i = 0; i < ggml_cann_info().device_count; i++) {\n                ggml_backend_cann_device_context * dev_ctx = new ggml_backend_cann_device_context();\n                dev_ctx->description                       = aclrtGetSocName();\n                dev_ctx->device                            = i;\n                dev_ctx->name                              = GGML_CANN_NAME + std::to_string(i);\n                dev_ctx->op_offload_min_batch_size         = min_batch_size;\n                ggml_cann_set_device(i);\n                ggml_backend_dev_t dev = new ggml_backend_device{ /* .iface   = */ ggml_backend_cann_device_interface,\n                                                                  /* .reg     = */ &reg,\n                                                                  /* .context = */ dev_ctx };\n                ctx->devices.push_back(dev);\n            }\n\n            reg = ggml_backend_reg{ /* .api_version = */ GGML_BACKEND_API_VERSION,\n                                    /* .iface       = */ ggml_backend_cann_reg_interface,\n                                    /* .context     = */ ctx };\n        }\n\n        initialized = true;\n    }\n\n    return &reg;\n}\n\nggml_backend_t ggml_backend_cann_init(int32_t device) {\n    aclInit(nullptr);\n    if (device < 0 || device >= ggml_backend_cann_get_device_count()) {\n        GGML_LOG_ERROR(\"%s: error: invalid device %d\\n\", __func__, device);\n        return nullptr;\n    }\n\n    ggml_backend_cann_context * ctx = new ggml_backend_cann_context(device);\n    if (ctx == nullptr) {\n        GGML_LOG_ERROR(\"%s: error: failed to allocate context\\n\", __func__);\n        return nullptr;\n    }\n    ggml_cann_set_device(ctx->device);\n    ggml_backend_t cann_backend =\n        new ggml_backend{ /* .guid      = */ ggml_backend_cann_guid(),\n                          /* .interface = */ ggml_backend_cann_interface,\n                          /* .device    = */ ggml_backend_reg_dev_get(ggml_backend_cann_reg(), device),\n                          /* .context   = */ ctx };\n\n    return cann_backend;\n}\n\nbool ggml_backend_is_cann(ggml_backend_t backend) {\n    return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cann_guid());\n}\n\nint32_t ggml_backend_cann_get_device_count() {\n    return ggml_cann_info().device_count;\n}\n\nvoid ggml_backend_cann_get_device_description(int32_t device, char * description, size_t description_size) {\n    ggml_cann_set_device(device);\n    const char * soc_name = aclrtGetSocName();\n    snprintf(description, description_size, \"%s\", soc_name);\n}\n\nvoid ggml_backend_cann_get_device_memory(int32_t device, size_t * free, size_t * total) {\n    ggml_cann_set_device(device);\n    ACL_CHECK(aclrtGetMemInfo(ACL_HBM_MEM, free, total));\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_cann_reg)\n"
  },
  {
    "path": "ggml/src/ggml-common.h",
    "content": "#ifndef GGML_COMMON_DECL\n\n#if defined(GGML_COMMON_DECL_C)\n#include <stdint.h>\n\ntypedef uint16_t ggml_half;\ntypedef uint32_t ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_CPP)\n#include <cstdint>\n\ntypedef uint16_t ggml_half;\ntypedef uint32_t ggml_half2;\n\n// std-c++ allow anonymous unions but some compiler warn on it\n#define GGML_COMMON_AGGR_U data\n// std-c++ do not allow it.\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_METAL)\n#include <metal_stdlib>\n\ntypedef half  ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_CUDA)\n#if defined(GGML_COMMON_DECL_MUSA)\n#include <musa_fp16.h>\n#else\n#include <cuda_fp16.h>\n#endif\n#include <cstdint>\n\ntypedef half  ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_HIP)\n#include <hip/hip_fp16.h>\n#include <cstdint>\n\ntypedef half  ggml_half;\ntypedef half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#elif defined(GGML_COMMON_DECL_SYCL)\n#include <sycl/half_type.hpp>\n#include <cstdint>\n\ntypedef sycl::half  ggml_half;\ntypedef sycl::half2 ggml_half2;\n\n#define GGML_COMMON_AGGR_U\n#define GGML_COMMON_AGGR_S data\n\n#define GGML_COMMON_DECL\n#endif\n\n#if defined(GGML_COMMON_DECL)\n\n#ifndef __cplusplus\n#ifndef static_assert\n#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)\n#define static_assert(cond, msg) _Static_assert(cond, msg)\n#else\n#define static_assert(cond, msg) struct global_scope_noop_trick\n#endif\n#endif\n#endif // __cplusplus\n\n// QK = number of values after dequantization\n// QK_K = super-block size\n\n#define QK_K 256\n#define K_SCALE_SIZE 12\n\n#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)\n// QR = QK / number of values before dequantization\n// QI = number of 32 bit integers before dequantization\n\n#define QI4_0 (QK4_0 / (4 * QR4_0))\n#define QR4_0 2\n\n#define QI4_1 (QK4_1 / (4 * QR4_1))\n#define QR4_1 2\n\n#define QI_MXFP4 (QK_MXFP4 / (4 * QR_MXFP4))\n#define QR_MXFP4 2\n\n#define QI_NVFP4 (QK_NVFP4 / (4 * QR_NVFP4))\n#define QR_NVFP4 2\n\n#define QI5_0 (QK5_0 / (4 * QR5_0))\n#define QR5_0 2\n\n#define QI5_1 (QK5_1 / (4 * QR5_1))\n#define QR5_1 2\n\n#define QI8_0 (QK8_0 / (4 * QR8_0))\n#define QR8_0 1\n\n#define QI8_1 (QK8_1 / (4 * QR8_1))\n#define QR8_1 1\n\n#define QI2_K (QK_K / (4*QR2_K))\n#define QR2_K 4\n\n#define QI3_K (QK_K / (4*QR3_K))\n#define QR3_K 4\n\n#define QI4_K (QK_K / (4*QR4_K))\n#define QR4_K 2\n\n#define QI5_K (QK_K / (4*QR5_K))\n#define QR5_K 2\n\n#define QI6_K (QK_K / (4*QR6_K))\n#define QR6_K 2\n\n#define QI2_XXS (QK_K / (4*QR2_XXS))\n#define QR2_XXS 4\n\n#define QI2_XS (QK_K / (4*QR2_XS))\n#define QR2_XS 4\n\n#define QI2_S (QK_K / (4*QR2_S))\n#define QR2_S 4\n\n#define QI3_XXS (QK_K / (4*QR3_XXS))\n#define QR3_XXS 4\n\n#define QI3_XS (QK_K / (4*QR3_XS))\n#define QR3_XS 4\n\n#define QI1_S (QK_K / (4*QR1_S))\n#define QR1_S 8\n\n#define QI1_M (QK_K / (4*QR1_M))\n#define QR1_M 8\n\n#define QI4_NL (QK4_NL / (4*QR4_NL))\n#define QR4_NL 2\n\n#define QI4_XS (QK_K / (4*QR4_XS))\n#define QR4_XS 2\n\n#define QI3_S (QK_K / (4*QR3_S))\n#define QR3_S 4\n\n#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP\n\n#ifdef _MSC_VER\n#define GGML_EXTENSION\n#else // _MSC_VER\n#define GGML_EXTENSION __extension__\n#endif // _MSC_VER\n\n#define QK4_0 32\ntypedef struct {\n    ggml_half d;           // delta\n    uint8_t qs[QK4_0 / 2]; // nibbles / quants\n} block_q4_0;\nstatic_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, \"wrong q4_0 block size/padding\");\n\n#define QK4_1 32\ntypedef struct {\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d; // delta\n            ggml_half m; // min\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 dm;\n    } GGML_COMMON_AGGR_U;\n    uint8_t qs[QK4_1 / 2]; // nibbles / quants\n} block_q4_1;\nstatic_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, \"wrong q4_1 block size/padding\");\n\n#define QK_MXFP4 32\ntypedef struct {\n    uint8_t e; // E8M0\n    uint8_t qs[QK_MXFP4/2];\n} block_mxfp4;\nstatic_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, \"wrong mxfp4 block size/padding\");\n\n#define QK_NVFP4 64\n#define QK_NVFP4_SUB 16  // sub-block size for per-group scales\ntypedef struct {\n    uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; // UE4M3 scales (4 bytes, one per 16-element sub-block)\n    uint8_t qs[QK_NVFP4/2];           // packed 4-bit E2M1 values (32 bytes)\n} block_nvfp4;\nstatic_assert(sizeof(block_nvfp4) == sizeof(uint8_t)*(QK_NVFP4/QK_NVFP4_SUB) + QK_NVFP4/2, \"wrong nvfp4 block size/padding\");\n\n#define QK5_0 32\ntypedef struct {\n    ggml_half d;           // delta\n    uint8_t qh[4];         // 5-th bit of quants\n    uint8_t qs[QK5_0 / 2]; // nibbles / quants\n} block_q5_0;\nstatic_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, \"wrong q5_0 block size/padding\");\n\n#define QK5_1 32\ntypedef struct {\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d; // delta\n            ggml_half m; // min\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 dm;\n    } GGML_COMMON_AGGR_U;\n    uint8_t qh[4];         // 5-th bit of quants\n    uint8_t qs[QK5_1 / 2]; // nibbles / quants\n} block_q5_1;\nstatic_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, \"wrong q5_1 block size/padding\");\n\n#define QK8_0 32\ntypedef struct {\n    ggml_half d;       // delta\n    int8_t  qs[QK8_0]; // quants\n} block_q8_0;\nstatic_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, \"wrong q8_0 block size/padding\");\n\n#define QK8_1 32\ntypedef struct {\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d; // delta\n            ggml_half s; // d * sum(qs[i])\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 ds;\n    } GGML_COMMON_AGGR_U;\n    int8_t qs[QK8_1]; // quants\n} block_q8_1;\nstatic_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, \"wrong q8_1 block size/padding\");\n\n//\n// Ternary quantization\n//\n\n// 1.6875 bpw\ntypedef struct {\n    uint8_t qs[(QK_K - 4 * QK_K / 64) / 5]; // 5 elements per byte (3^5 = 243 < 256)\n    uint8_t qh[QK_K/64]; // 4 elements per byte\n    ggml_half d;\n} block_tq1_0;\nstatic_assert(sizeof(block_tq1_0) == sizeof(ggml_half) + QK_K / 64 + (QK_K - 4 * QK_K / 64) / 5, \"wrong tq1_0 block size/padding\");\n\n// 2.0625 bpw\ntypedef struct {\n    uint8_t qs[QK_K/4]; // 2 bits per element\n    ggml_half d;\n} block_tq2_0;\nstatic_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, \"wrong tq2_0 block size/padding\");\n\n//\n// Super-block quantization structures\n//\n\n// 2-bit quantization\n// weight is represented as x = a * q + b\n// 16 blocks of 16 elements each\n// Effectively 2.625 bits per weight\ntypedef struct {\n    uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits\n    uint8_t qs[QK_K/4];      // quants\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d;    // super-block scale for quantized scales\n            ggml_half dmin; // super-block scale for quantized mins\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 dm;\n    } GGML_COMMON_AGGR_U;\n} block_q2_K;\nstatic_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, \"wrong q2_K block size/padding\");\n\n// 3-bit quantization\n// weight is represented as x = a * q\n// 16 blocks of 16 elements each\n// Effectively 3.4375 bits per weight\ntypedef struct {\n    uint8_t hmask[QK_K/8]; // quants - high bit\n    uint8_t qs[QK_K/4];    // quants - low 2 bits\n    uint8_t scales[12];    // scales, quantized with 6 bits\n    ggml_half d;           // super-block scale\n} block_q3_K;\nstatic_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, \"wrong q3_K block size/padding\");\n\n// 4-bit quantization\n// 8 blocks of 32 elements each\n// weight is represented as x = a * q + b\n// Effectively 4.5 bits per weight\ntypedef struct {\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d;    // super-block scale for quantized scales\n            ggml_half dmin; // super-block scale for quantized mins\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 dm;\n    } GGML_COMMON_AGGR_U;\n    uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits\n    uint8_t qs[QK_K/2];           // 4--bit quants\n} block_q4_K;\nstatic_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, \"wrong q4_K block size/padding\");\n\n// 5-bit quantization\n// 8 blocks of 32 elements each\n// weight is represented as x = a * q + b\n// Effectively 5.5 bits per weight\ntypedef struct {\n    GGML_EXTENSION union {\n        struct {\n            ggml_half d;    // super-block scale for quantized scales\n            ggml_half dmin; // super-block scale for quantized mins\n        } GGML_COMMON_AGGR_S;\n        ggml_half2 dm;\n    } GGML_COMMON_AGGR_U;\n    uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits\n    uint8_t qh[QK_K/8];           // quants, high bit\n    uint8_t qs[QK_K/2];           // quants, low 4 bits\n} block_q5_K;\nstatic_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, \"wrong q5_K block size/padding\");\n\n// 6-bit quantization\n// weight is represented as x = a * q\n// 16 blocks of 16 elements each\n// Effectively 6.5625 bits per weight\ntypedef struct {\n    uint8_t ql[QK_K/2];      // quants, lower 4 bits\n    uint8_t qh[QK_K/4];      // quants, upper 2 bits\n    int8_t  scales[QK_K/16]; // scales, quantized with 8 bits\n    ggml_half d;             // super-block scale\n} block_q6_K;\nstatic_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, \"wrong q6_K block size/padding\");\n\n// This is only used for intermediate quantization and dot products\ntypedef struct {\n    float   d;              // delta\n    int8_t  qs[QK_K];       // quants\n    int16_t bsums[QK_K/16]; // sum of quants in groups of 16\n} block_q8_K;\nstatic_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), \"wrong q8_K block size/padding\");\n\n// (Almost) \"true\" 2-bit quantization.\n// Due to the need to use blocks as per ggml design, it ends up using\n// 2.0625 bpw because of the 16-bit scale for each block of 256.\ntypedef struct {\n    ggml_half d;\n    uint16_t qs[QK_K/8];\n} block_iq2_xxs;\nstatic_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), \"wrong iq2_xxs block size/padding\");\n\n// 2.3125 bpw quants\ntypedef struct {\n    ggml_half d;\n    uint16_t qs[QK_K/8];\n    uint8_t  scales[QK_K/32];\n} block_iq2_xs;\nstatic_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, \"wrong iq2_xs block size/padding\");\n\n// 2.5625 bpw quants\ntypedef struct {\n    ggml_half d;\n    uint8_t qs[QK_K/4];\n    uint8_t qh[QK_K/32];\n    uint8_t scales[QK_K/32];\n} block_iq2_s;\nstatic_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, \"wrong iq2_s block size/padding\");\n\n// (Almost) \"true\" 3-bit quantization.\n// Due to the need to use blocks as per ggml design, it ends up using\n// 3.0625 bpw because of the 16-bit scale for each block of 256.\ntypedef struct {\n    ggml_half d;\n    uint8_t qs[3*QK_K/8];\n} block_iq3_xxs;\nstatic_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), \"wrong iq3_xxs block size/padding\");\n\n// 3.4375 bpw\n#define IQ3S_N_SCALE QK_K/64\ntypedef struct {\n    ggml_half d;\n    uint8_t qs[QK_K/4];\n    uint8_t qh[QK_K/32];\n    uint8_t signs[QK_K/8];\n    uint8_t scales[IQ3S_N_SCALE];\n} block_iq3_s;\nstatic_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, \"wrong iq3_s block size/padding\");\n\n// 1.5625 bpw\ntypedef struct {\n    ggml_half d;\n    uint8_t  qs[QK_K/8];\n    uint16_t qh[QK_K/32];\n} block_iq1_s;\nstatic_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, \"wrong iq1_s block size/padding\");\n\n// 1.75 bpw\ntypedef struct {\n    uint8_t  qs[QK_K/8];      // grid index, low 8 bits\n    uint8_t  qh[QK_K/16];     // grid index, high 3 bits + grid shift bit (for two groups of 8)\n    uint8_t  scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)\n} block_iq1_m;\nstatic_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, \"wrong iq1_m block size/padding\");\n\n// Used by IQ1_M quants\ntypedef union {\n    ggml_half f16;\n    uint16_t  u16;\n} iq1m_scale_t;\n\n// Non-linear quants\n#define QK4_NL 32\ntypedef struct {\n    ggml_half d;\n    uint8_t qs[QK4_NL/2];\n} block_iq4_nl;\nstatic_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, \"wrong iq4_nl block size/padding\");\n\ntypedef struct {\n    ggml_half d;\n    uint16_t scales_h;\n    uint8_t  scales_l[QK_K/64];\n    uint8_t  qs[QK_K/2];\n} block_iq4_xs;\nstatic_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, \"wrong iq4_xs block size/padding\");\n\n#endif // GGML_COMMON_DECL\n#endif // GGML_COMMON_DECL\n\n////////////////////////////////////////////////////////////////////////////////\n\n#ifndef GGML_COMMON_IMPL\n\n#if defined(GGML_COMMON_IMPL_C)\n#include <stdint.h>\n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_CPP)\n#include <cstdint>\n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_METAL)\n#include <metal_stdlib>\n\n#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA)\n#include <cstdint>\n\n#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#elif defined(GGML_COMMON_IMPL_SYCL)\n\n#include <cstdint>\n\n#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {\n#define GGML_TABLE_END() };\n\n#define GGML_COMMON_IMPL\n#endif\n\n#if defined(GGML_COMMON_IMPL)\n\nGGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)\n    1, 2, 4, 8, 16, 32, 64, 128\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)\n      0, 129, 130,   3, 132,   5,   6, 135, 136,   9,  10, 139,  12, 141, 142,  15,\n    144,  17,  18, 147,  20, 149, 150,  23,  24, 153, 154,  27, 156,  29,  30, 159,\n    160,  33,  34, 163,  36, 165, 166,  39,  40, 169, 170,  43, 172,  45,  46, 175,\n     48, 177, 178,  51, 180,  53,  54, 183, 184,  57,  58, 187,  60, 189, 190,  63,\n    192,  65,  66, 195,  68, 197, 198,  71,  72, 201, 202,  75, 204,  77,  78, 207,\n     80, 209, 210,  83, 212,  85,  86, 215, 216,  89,  90, 219,  92, 221, 222,  95,\n     96, 225, 226,  99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,\n    240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, ksigns64, 128)\n    0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,\n    0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,\n    0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,\n    0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,\n    0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,\n    0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,\n    0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,\n    0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,\n    0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,\n    0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,\n    0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,\n    0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,\n    0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,\n    0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,\n    0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,\n    0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,\n    0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,\n    0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,\n    0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,\n    0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,\n    0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,\n    0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,\n    0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,\n    0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,\n    0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,\n    0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,\n    0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,\n    0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,\n    0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,\n    0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,\n    0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,\n    0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,\nGGML_TABLE_END()\n\n\nGGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)\n    0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n    0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,\n    0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,\n    0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,\n    0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,\n    0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,\n    0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,\n    0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,\n    0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,\n    0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,\n    0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,\n    0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,\n    0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,\n    0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,\n    0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,\n    0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,\n    0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,\n    0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,\n    0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,\n    0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,\n    0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,\n    0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,\n    0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,\n    0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,\n    0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,\n    0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,\n    0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,\n    0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,\n    0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,\n    0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,\n    0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,\n    0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,\n    0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,\n    0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,\n    0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,\n    0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,\n    0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,\n    0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,\n    0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,\n    0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,\n    0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,\n    0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,\n    0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,\n    0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,\n    0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,\n    0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,\n    0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,\n    0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,\n    0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,\n    0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,\n    0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,\n    0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,\n    0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,\n    0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,\n    0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,\n    0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,\n    0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,\n    0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,\n    0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,\n    0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,\n    0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,\n    0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,\n    0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,\n    0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)\n    0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n    0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,\n    0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,\n    0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,\n    0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,\n    0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,\n    0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,\n    0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,\n    0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,\n    0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,\n    0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,\n    0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,\n    0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,\n    0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,\n    0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,\n    0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,\n    0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,\n    0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,\n    0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,\n    0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,\n    0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,\n    0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,\n    0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,\n    0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,\n    0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,\n    0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,\n    0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,\n    0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,\n    0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,\n    0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,\n    0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,\n    0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,\n    0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,\n    0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,\n    0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,\n    0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,\n    0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,\n    0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,\n    0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,\n    0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,\n    0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,\n    0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,\n    0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,\n    0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,\n    0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,\n    0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,\n    0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,\n    0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,\n    0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,\n    0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,\n    0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,\n    0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,\n    0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,\n    0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,\n    0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,\n    0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,\n    0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,\n    0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,\n    0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,\n    0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,\n    0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,\n    0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,\n    0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,\n    0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,\n    0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,\n    0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,\n    0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,\n    0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,\n    0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,\n    0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,\n    0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,\n    0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,\n    0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,\n    0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,\n    0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,\n    0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,\n    0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,\n    0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,\n    0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,\n    0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,\n    0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,\n    0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,\n    0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,\n    0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,\n    0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,\n    0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,\n    0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,\n    0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,\n    0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,\n    0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,\n    0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,\n    0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,\n    0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,\n    0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,\n    0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,\n    0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,\n    0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,\n    0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,\n    0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,\n    0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,\n    0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,\n    0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,\n    0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,\n    0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,\n    0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,\n    0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,\n    0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,\n    0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,\n    0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,\n    0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,\n    0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,\n    0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,\n    0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,\n    0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,\n    0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,\n    0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,\n    0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,\n    0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,\n    0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,\n    0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,\n    0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,\n    0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,\n    0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,\n    0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,\n    0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,\n    0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,\n    0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,\n    0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)\n    0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,\n    0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,\n    0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,\n    0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,\n    0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,\n    0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,\n    0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,\n    0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,\n    0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,\n    0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,\n    0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,\n    0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,\n    0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,\n    0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,\n    0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,\n    0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,\n    0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,\n    0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,\n    0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,\n    0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,\n    0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,\n    0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,\n    0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,\n    0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,\n    0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,\n    0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,\n    0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,\n    0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,\n    0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,\n    0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,\n    0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,\n    0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,\n    0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,\n    0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,\n    0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,\n    0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,\n    0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,\n    0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,\n    0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,\n    0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,\n    0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,\n    0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,\n    0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,\n    0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,\n    0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,\n    0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,\n    0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,\n    0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,\n    0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,\n    0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,\n    0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,\n    0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,\n    0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,\n    0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,\n    0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,\n    0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,\n    0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,\n    0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,\n    0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,\n    0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,\n    0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,\n    0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,\n    0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,\n    0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,\n    0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,\n    0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,\n    0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,\n    0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,\n    0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,\n    0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,\n    0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,\n    0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,\n    0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,\n    0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,\n    0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,\n    0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,\n    0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,\n    0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,\n    0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,\n    0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,\n    0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,\n    0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,\n    0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,\n    0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,\n    0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,\n    0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,\n    0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,\n    0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,\n    0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,\n    0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,\n    0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,\n    0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,\n    0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,\n    0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,\n    0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,\n    0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,\n    0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,\n    0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,\n    0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,\n    0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,\n    0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,\n    0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,\n    0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,\n    0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,\n    0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,\n    0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,\n    0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,\n    0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,\n    0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,\n    0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,\n    0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,\n    0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,\n    0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,\n    0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,\n    0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,\n    0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,\n    0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,\n    0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,\n    0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,\n    0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,\n    0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,\n    0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,\n    0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,\n    0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,\n    0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,\n    0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,\n    0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,\n    0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,\n    0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,\n    0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,\n    0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,\n    0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,\n    0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,\n    0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,\n    0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,\n    0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,\n    0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,\n    0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,\n    0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,\n    0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,\n    0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,\n    0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,\n    0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,\n    0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,\n    0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,\n    0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,\n    0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,\n    0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,\n    0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,\n    0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,\n    0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,\n    0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,\n    0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,\n    0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,\n    0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,\n    0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,\n    0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,\n    0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,\n    0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,\n    0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,\n    0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,\n    0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,\n    0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,\n    0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,\n    0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,\n    0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,\n    0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,\n    0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,\n    0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,\n    0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,\n    0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,\n    0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,\n    0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,\n    0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,\n    0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,\n    0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,\n    0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,\n    0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,\n    0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,\n    0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,\n    0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,\n    0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,\n    0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,\n    0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,\n    0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,\n    0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,\n    0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,\n    0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,\n    0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,\n    0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,\n    0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,\n    0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,\n    0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,\n    0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,\n    0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,\n    0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,\n    0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,\n    0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,\n    0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,\n    0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,\n    0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,\n    0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,\n    0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,\n    0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,\n    0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,\n    0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,\n    0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,\n    0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,\n    0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,\n    0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,\n    0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,\n    0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,\n    0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,\n    0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,\n    0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,\n    0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,\n    0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,\n    0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,\n    0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,\n    0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,\n    0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,\n    0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,\n    0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,\n    0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,\n    0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,\n    0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,\n    0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,\n    0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,\n    0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,\n    0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,\n    0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,\n    0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,\n    0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,\n    0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,\n    0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,\n    0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,\n    0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,\n    0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,\n    0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,\n    0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,\n    0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,\n    0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,\n    0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,\n    0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,\n    0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,\n    0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,\n    0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,\n    0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,\n    0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,\n    0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,\n    0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,\n    0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,\n    0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,\n    0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,\n    0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,\n    0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)\n    0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,\n    0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,\n    0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,\n    0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,\n    0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,\n    0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,\n    0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,\n    0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,\n    0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,\n    0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,\n    0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,\n    0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,\n    0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,\n    0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,\n    0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,\n    0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,\n    0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,\n    0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,\n    0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,\n    0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,\n    0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,\n    0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,\n    0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,\n    0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,\n    0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,\n    0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,\n    0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,\n    0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,\n    0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,\n    0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,\n    0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,\n    0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,\nGGML_TABLE_END()\n\nGGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)\n    0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,\n    0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,\n    0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,\n    0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,\n    0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,\n    0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,\n    0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,\n    0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,\n    0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,\n    0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,\n    0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,\n    0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,\n    0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,\n    0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,\n    0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,\n    0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,\n    0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,\n    0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,\n    0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,\n    0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,\n    0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,\n    0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,\n    0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,\n    0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,\n    0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,\n    0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,\n    0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,\n    0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,\n    0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,\n    0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,\n    0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,\n    0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,\n    0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,\n    0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,\n    0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,\n    0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,\n    0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,\n    0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,\n    0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,\n    0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,\n    0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,\n    0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,\n    0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,\n    0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,\n    0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,\n    0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,\n    0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,\n    0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,\n    0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,\n    0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,\n    0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,\n    0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,\n    0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,\n    0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,\n    0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,\n    0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,\n    0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,\n    0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,\n    0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,\n    0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,\n    0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,\n    0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,\n    0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,\n    0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,\nGGML_TABLE_END()\n\n// TODO: fix name to kvalues_iq4_nl\nGGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)\n    -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,\nGGML_TABLE_END()\n\n// e2m1 values (doubled)\n// ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf\nGGML_TABLE_BEGIN(int8_t, kvalues_mxfp4, 16)\n    0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12,\nGGML_TABLE_END()\n\n#define NGRID_IQ1S 2048\n#define IQ1S_DELTA 0.125f\n#define IQ1M_DELTA 0.125f\n#if defined(GGML_COMMON_IMPL_C)\nGGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)\n    0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,\n    0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,\n    0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,\n    0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,\n    0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,\n    0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,\n    0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,\n    0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,\n    0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,\n    0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,\n    0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,\n    0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,\n    0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,\n    0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,\n    0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,\n    0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,\n    0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,\n    0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,\n    0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,\n    0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,\n    0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,\n    0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,\n    0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,\n    0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,\n    0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,\n    0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,\n    0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,\n    0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,\n    0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,\n    0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,\n    0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,\n    0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,\n    0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,\n    0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,\n    0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,\n    0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,\n    0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,\n    0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,\n    0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,\n    0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,\n    0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,\n    0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,\n    0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,\n    0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,\n    0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,\n    0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,\n    0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,\n    0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,\n    0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,\n    0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,\n    0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,\n    0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,\n    0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,\n    0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,\n    0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,\n    0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,\n    0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,\n    0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,\n    0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,\n    0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,\n    0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,\n    0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,\n    0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,\n    0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,\n    0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,\n    0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,\n    0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,\n    0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,\n    0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,\n    0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,\n    0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,\n    0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,\n    0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,\n    0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,\n    0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,\n    0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,\n    0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,\n    0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,\n    0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,\n    0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,\n    0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,\n    0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,\n    0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,\n    0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,\n    0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,\n    0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,\n    0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,\n    0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,\n    0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,\n    0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,\n    0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,\n    0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,\n    0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,\n    0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,\n    0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,\n    0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,\n    0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,\n    0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,\n    0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,\n    0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,\n    0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,\n    0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,\n    0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,\n    0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,\n    0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,\n    0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,\n    0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,\n    0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,\n    0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,\n    0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,\n    0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,\n    0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,\n    0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,\n    0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,\n    0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,\n    0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,\n    0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,\n    0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,\n    0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,\n    0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,\n    0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,\n    0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,\n    0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,\n    0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,\n    0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,\n    0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,\n    0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,\n    0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,\n    0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,\n    0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,\n    0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,\n    0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,\n    0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,\n    0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,\n    0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,\n    0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,\n    0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,\n    0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,\n    0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,\n    0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,\n    0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,\n    0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,\n    0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,\n    0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,\n    0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,\n    0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,\n    0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,\n    0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,\n    0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,\n    0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,\n    0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,\n    0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,\n    0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,\n    0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,\n    0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,\n    0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,\n    0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,\n    0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,\n    0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,\n    0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,\n    0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,\n    0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,\n    0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,\n    0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,\n    0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,\n    0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,\n    0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,\n    0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,\n    0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,\n    0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,\n    0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,\n    0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,\n    0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,\n    0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,\n    0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,\n    0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,\n    0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,\n    0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,\n    0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,\n    0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,\n    0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,\n    0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,\n    0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,\n    0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,\n    0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,\n    0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,\n    0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,\n    0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,\n    0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,\n    0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,\n    0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,\n    0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,\n    0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,\n    0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,\n    0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,\n    0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,\n    0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,\n    0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,\n    0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,\n    0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,\n    0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,\n    0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,\n    0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,\n    0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,\n    0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,\n    0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,\n    0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,\n    0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,\n    0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,\n    0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,\n    0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,\n    0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,\n    0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,\n    0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,\n    0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,\n    0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,\n    0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,\n    0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,\n    0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,\n    0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,\n    0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,\n    0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,\n    0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,\n    0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,\n    0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,\n    0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,\n    0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,\n    0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,\n    0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,\n    0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,\n    0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,\n    0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,\n    0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,\n    0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,\n    0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,\n    0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,\n    0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,\n    0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,\n    0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,\n    0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,\n    0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,\n    0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,\n    0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,\n    0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,\n    0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,\n    0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,\n    0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,\n    0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,\n    0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,\n    0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,\n    0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,\n    0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,\n    0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,\n    0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,\n    0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,\n    0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,\n    0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,\n    0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,\n    0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,\n    0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,\n    0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,\n    0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,\n    0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,\n    0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,\n    0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,\n    0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,\n    0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,\n    0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,\n    0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,\n    0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,\n    0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,\n    0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,\n    0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,\n    0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,\n    0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,\n    0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,\n    0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,\n    0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,\n    0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,\n    0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,\n    0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,\n    0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,\n    0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,\n    0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,\n    0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,\n    0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,\n    0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,\n    0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,\n    0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,\n    0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,\n    0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,\n    0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,\n    0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,\n    0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,\n    0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,\n    0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,\n    0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,\n    0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,\n    0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,\n    0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,\n    0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,\n    0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,\n    0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,\n    0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,\n    0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,\n    0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,\n    0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,\n    0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,\n    0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,\n    0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,\n    0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,\n    0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,\n    0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,\n    0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,\n    0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,\n    0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,\n    0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,\n    0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,\n    0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,\n    0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,\n    0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,\n    0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,\n    0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,\n    0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,\n    0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,\n    0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,\n    0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,\n    0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,\n    0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,\n    0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,\n    0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,\n    0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,\n    0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,\n    0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,\n    0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,\n    0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,\n    0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,\n    0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,\n    0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,\n    0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,\n    0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,\n    0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,\n    0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,\n    0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,\n    0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,\n    0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,\n    0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,\n    0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,\n    0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,\n    0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,\n    0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,\n    0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,\n    0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,\n    0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,\n    0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,\n    0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,\n    0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,\n    0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,\n    0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,\n    0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,\n    0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,\n    0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,\n    0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,\n    0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,\n    0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,\n    0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,\n    0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,\n    0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,\n    0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,\n    0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,\n    0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,\n    0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,\n    0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,\n    0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,\n    0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,\n    0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,\n    0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,\n    0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,\n    0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,\n    0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,\n    0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,\n    0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,\n    0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,\n    0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,\n    0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,\n    0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,\n    0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,\n    0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,\n    0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,\n    0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,\n    0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,\n    0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,\n    0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,\n    0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,\n    0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,\n    0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,\n    0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,\n    0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,\n    0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,\n    0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,\n    0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,\n    0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,\n    0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,\n    0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,\n    0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,\n    0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,\n    0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,\n    0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,\n    0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,\n    0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,\n    0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,\n    0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,\n    0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,\n    0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,\n    0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,\n    0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,\n    0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,\n    0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,\n    0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,\n    0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,\n    0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,\n    0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,\n    0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,\n    0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,\n    0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,\n    0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,\n    0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,\n    0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,\n    0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,\n    0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,\n    0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,\n    0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,\n    0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,\n    0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,\n    0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,\n    0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,\n    0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,\n    0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,\n    0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,\n    0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,\n    0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,\n    0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,\n    0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,\n    0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,\n    0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,\n    0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,\n    0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,\n    0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,\n    0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,\n    0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,\n    0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,\n    0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,\n    0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,\n    0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,\n    0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,\n    0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,\n    0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,\n    0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,\n    0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,\n    0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,\n    0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,\n    0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,\n    0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,\n    0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,\n    0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,\n    0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,\n    0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,\n    0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,\n    0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,\n    0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,\n    0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,\n    0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,\n    0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,\n    0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,\n    0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,\n    0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,\n    0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,\n    0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,\n    0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,\n    0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,\n    0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,\n    0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,\n    0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,\n    0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,\n    0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,\n    0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,\n    0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,\n    0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,\n    0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,\n    0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,\n    0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,\n    0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,\n    0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,\n    0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,\n    0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,\n    0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,\n    0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,\n    0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,\n    0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,\n    0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,\n    0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,\n    0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,\n    0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,\n    0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,\n    0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,\n    0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,\n    0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,\n    0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,\n    0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,\n    0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,\n    0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,\n    0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,\nGGML_TABLE_END()\n#else\nGGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)\n    0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,\n    0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,\n    0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,\n    0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,\n    0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,\n    0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,\n    0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,\n    0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,\n    0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,\n    0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,\n    0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,\n    0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,\n    0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,\n    0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,\n    0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,\n    0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,\n    0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,\n    0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,\n    0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,\n    0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,\n    0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,\n    0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,\n    0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,\n    0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,\n    0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,\n    0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,\n    0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,\n    0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,\n    0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,\n    0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,\n    0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,\n    0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,\n    0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,\n    0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,\n    0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,\n    0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,\n    0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,\n    0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,\n    0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,\n    0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,\n    0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,\n    0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,\n    0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,\n    0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,\n    0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,\n    0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,\n    0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,\n    0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,\n    0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,\n    0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,\n    0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,\n    0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,\n    0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,\n    0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,\n    0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,\n    0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,\n    0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,\n    0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,\n    0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,\n    0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,\n    0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,\n    0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,\n    0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,\n    0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,\n    0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,\n    0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,\n    0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,\n    0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,\n    0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,\n    0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,\n    0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,\n    0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,\n    0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,\n    0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,\n    0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,\n    0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,\n    0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,\n    0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,\n    0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,\n    0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,\n    0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,\n    0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,\n    0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,\n    0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,\n    0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,\n    0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,\n    0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,\n    0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,\n    0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,\n    0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,\n    0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,\n    0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,\n    0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,\n    0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,\n    0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,\n    0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,\n    0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,\n    0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,\n    0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,\n    0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,\n    0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,\n    0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,\n    0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,\n    0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,\n    0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,\n    0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,\n    0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,\n    0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,\n    0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,\n    0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,\n    0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,\n    0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,\n    0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,\n    0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,\n    0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,\n    0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,\n    0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,\n    0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,\n    0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,\n    0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,\n    0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,\n    0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,\n    0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,\n    0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,\n    0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,\n    0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,\n    0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,\n    0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,\n    0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,\n    0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,\n    0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,\n    0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,\n    0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,\n    0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,\n    0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,\n    0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,\n    0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,\n    0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,\n    0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,\n    0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,\n    0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,\n    0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,\n    0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,\n    0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,\n    0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,\n    0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,\n    0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,\n    0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,\n    0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,\n    0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,\n    0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,\n    0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,\n    0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,\n    0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,\n    0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,\n    0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,\n    0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,\n    0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,\n    0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,\n    0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,\n    0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,\n    0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,\n    0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,\n    0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,\n    0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,\n    0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,\n    0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,\n    0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,\n    0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,\n    0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,\n    0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,\n    0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,\n    0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,\n    0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,\n    0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,\n    0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,\n    0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,\n    0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,\n    0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,\n    0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,\n    0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,\n    0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,\n    0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,\n    0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,\n    0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,\n    0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,\n    0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,\n    0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,\n    0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,\n    0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,\n    0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,\n    0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,\n    0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,\n    0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,\n    0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,\n    0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,\n    0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,\n    0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,\n    0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,\n    0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,\n    0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,\n    0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,\n    0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,\n    0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,\n    0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,\n    0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,\n    0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,\n    0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,\n    0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,\n    0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,\n    0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,\n    0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,\n    0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,\n    0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,\n    0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,\n    0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,\n    0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,\n    0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,\n    0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,\n    0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,\n    0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,\n    0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,\n    0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,\n    0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,\n    0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,\n    0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,\n    0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,\n    0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,\n    0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,\n    0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,\n    0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,\n    0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,\n    0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,\n    0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,\n    0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,\n    0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,\n    0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,\n    0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,\n    0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,\n    0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,\n    0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,\n    0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,\n    0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,\n    0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,\n    0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,\n    0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,\n    0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,\n    0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,\n    0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,\n    0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,\n    0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,\n    0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,\n    0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,\n    0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,\n    0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,\n    0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,\nGGML_TABLE_END()\n#endif\n\n#endif // GGML_COMMON_IMPL\n#endif // GGML_COMMON_IMPL\n"
  },
  {
    "path": "ggml/src/ggml-cpu/CMakeLists.txt",
    "content": "function(ggml_add_cpu_backend_features cpu_name arch)\n    # The feature detection code is compiled as a separate target so that\n    # it can be built without the architecture flags\n    # Since multiple variants of the CPU backend may be included in the same\n    # build, using set_source_files_properties() to set the arch flags is not possible\n    set(GGML_CPU_FEATS_NAME ${cpu_name}-feats)\n    add_library(${GGML_CPU_FEATS_NAME} OBJECT ggml-cpu/arch/${arch}/cpu-feats.cpp)\n    target_include_directories(${GGML_CPU_FEATS_NAME} PRIVATE . ../include)\n    target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE ${ARGN})\n    target_compile_definitions(${GGML_CPU_FEATS_NAME} PRIVATE GGML_BACKEND_DL GGML_BACKEND_BUILD GGML_BACKEND_SHARED)\n    set_target_properties(${GGML_CPU_FEATS_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON)\n    # Disable LTO for the feature detection code to prevent cross-module optimization\n    # from inlining architecture-specific instructions into the score function.\n    # Without this, LTO can cause SIGILL when loading backends on older CPUs\n    # (e.g., loading power10 backend on power9 crashes before feature check runs).\n    target_compile_options(${GGML_CPU_FEATS_NAME} PRIVATE -fno-lto)\n    target_link_libraries(${cpu_name} PRIVATE ${GGML_CPU_FEATS_NAME})\nendfunction()\n\nfunction(ggml_add_cpu_backend_variant_impl tag_name)\n    if (tag_name)\n        set(GGML_CPU_NAME ggml-cpu-${tag_name})\n    else()\n        set(GGML_CPU_NAME ggml-cpu)\n    endif()\n\n    ggml_add_backend_library(${GGML_CPU_NAME})\n\n    list (APPEND GGML_CPU_SOURCES\n        ggml-cpu/ggml-cpu.c\n        ggml-cpu/ggml-cpu.cpp\n        ggml-cpu/repack.cpp\n        ggml-cpu/repack.h\n        ggml-cpu/hbm.cpp\n        ggml-cpu/hbm.h\n        ggml-cpu/quants.c\n        ggml-cpu/quants.h\n        ggml-cpu/traits.cpp\n        ggml-cpu/traits.h\n        ggml-cpu/amx/amx.cpp\n        ggml-cpu/amx/amx.h\n        ggml-cpu/amx/mmq.cpp\n        ggml-cpu/amx/mmq.h\n        ggml-cpu/ggml-cpu-impl.h\n        ggml-cpu/common.h\n        ggml-cpu/binary-ops.h\n        ggml-cpu/binary-ops.cpp\n        ggml-cpu/unary-ops.h\n        ggml-cpu/unary-ops.cpp\n        ggml-cpu/simd-mappings.h\n        ggml-cpu/vec.h\n        ggml-cpu/vec.cpp\n        ggml-cpu/ops.h\n        ggml-cpu/ops.cpp\n        )\n\n    target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)\n    target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)\n\n    if (APPLE AND GGML_ACCELERATE)\n        find_library(ACCELERATE_FRAMEWORK Accelerate)\n        if (ACCELERATE_FRAMEWORK)\n            message(STATUS \"Accelerate framework found\")\n\n            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_ACCELERATE)\n            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_NEW_LAPACK)\n            target_compile_definitions(${GGML_CPU_NAME} PRIVATE ACCELERATE_LAPACK_ILP64)\n\n            target_link_libraries(${GGML_CPU_NAME} PRIVATE ${ACCELERATE_FRAMEWORK})\n        else()\n            message(WARNING \"Accelerate framework not found\")\n        endif()\n    endif()\n\n    if (GGML_OPENMP)\n        find_package(OpenMP)\n        if (OpenMP_FOUND)\n            set(GGML_OPENMP_ENABLED \"ON\" CACHE INTERNAL \"\")\n            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_OPENMP)\n\n            target_link_libraries(${GGML_CPU_NAME} PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)\n        else()\n            set(GGML_OPENMP_ENABLED \"OFF\" CACHE INTERNAL \"\")\n            message(WARNING \"OpenMP not found\")\n        endif()\n    endif()\n\n    if (GGML_LLAMAFILE)\n        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_LLAMAFILE)\n\n        list(APPEND GGML_CPU_SOURCES\n                    ggml-cpu/llamafile/sgemm.cpp\n                    ggml-cpu/llamafile/sgemm.h)\n    endif()\n\n    if (GGML_CPU_HBM)\n        find_library(memkind memkind REQUIRED)\n\n        message(STATUS \"Using memkind for CPU HBM\")\n\n        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_HBM)\n\n        target_link_libraries(${GGML_CPU_NAME} PUBLIC memkind)\n    endif()\n\n    if (GGML_SYSTEM_ARCH STREQUAL \"ARM\")\n        message(STATUS \"ARM detected\")\n        list(APPEND GGML_CPU_SOURCES\n            ggml-cpu/arch/arm/quants.c\n            ggml-cpu/arch/arm/repack.cpp\n            )\n\n        if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n            message(FATAL_ERROR \"MSVC is not supported for ARM, use clang\")\n        else()\n            check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)\n            if (NOT \"${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}\" STREQUAL \"\")\n                list(APPEND ARCH_FLAGS -mfp16-format=ieee)\n            endif()\n\n            if (GGML_NATIVE)\n                # -mcpu=native does not always enable all the features in some compilers,\n                # so we check for them manually and enable them if available\n\n                execute_process(\n                    COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v -\n                    INPUT_FILE \"/dev/null\"\n                    OUTPUT_QUIET\n                    ERROR_VARIABLE ARM_MCPU\n                    RESULT_VARIABLE ARM_MCPU_RESULT\n                )\n                if (NOT ARM_MCPU_RESULT)\n                    string(REGEX MATCH \"-mcpu=[^ ']+\" ARM_MCPU_FLAG \"${ARM_MCPU}\")\n                    string(REGEX MATCH \"-march=[^ ']+\" ARM_MARCH_FLAG \"${ARM_MCPU}\")\n\n                    # on some old GCC we need to read -march=\n                    if (ARM_MARCH_FLAG AND NOT \"${ARM_MARCH_FLAG}\" STREQUAL \"-march=native\")\n                        set(ARM_NATIVE_FLAG \"${ARM_MARCH_FLAG}\")\n                    elseif(ARM_MCPU_FLAG AND NOT \"${ARM_MCPU_FLAG}\" STREQUAL \"-mcpu=native\")\n                        set(ARM_NATIVE_FLAG \"${ARM_MCPU_FLAG}\")\n                    endif()\n                endif()\n\n                if (\"${ARM_NATIVE_FLAG}\" STREQUAL \"\")\n                    set(ARM_NATIVE_FLAG -mcpu=native)\n                    message(WARNING \"ARM -march/-mcpu not found, -mcpu=native will be used\")\n                else()\n                    message(STATUS \"ARM detected flags: ${ARM_NATIVE_FLAG}\")\n                endif()\n\n                include(CheckCXXSourceRuns)\n\n                macro(check_arm_feature tag feature code)\n                    set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n                    set(CMAKE_REQUIRED_FLAGS \"${ARM_NATIVE_FLAG}+${tag}\")\n                    check_cxx_source_runs(\"${code}\" GGML_MACHINE_SUPPORTS_${tag})\n                    if (GGML_MACHINE_SUPPORTS_${tag})\n                        set(ARM_NATIVE_FLAG_FIX \"${ARM_NATIVE_FLAG_FIX}+${tag}\")\n                    else()\n                        set(CMAKE_REQUIRED_FLAGS \"${ARM_NATIVE_FLAG}+no${tag}\")\n                        check_cxx_source_compiles(\"int main() { return 0; }\" GGML_MACHINE_SUPPORTS_no${tag})\n                        if (GGML_MACHINE_SUPPORTS_no${tag})\n                            set(ARM_NATIVE_FLAG_FIX \"${ARM_NATIVE_FLAG_FIX}+no${tag}\")\n                            list(APPEND ARCH_FLAGS -U__ARM_FEATURE_${feature})\n                        endif()\n                    endif()\n                    set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n                endmacro()\n\n                check_arm_feature(dotprod DOTPROD     \"#include <arm_neon.h>\\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }\")\n                check_arm_feature(i8mm    MATMUL_INT8 \"#include <arm_neon.h>\\nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }\")\n                check_arm_feature(sve     SVE         \"#include <arm_sve.h>\\nint main()  { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }\")\n                check_arm_feature(sme     SME         \"#include <arm_sme.h>\\n__arm_locally_streaming int main() { __asm__ volatile(\\\"smstart; smstop;\\\"); return 0; }\")\n\n                list(APPEND ARCH_FLAGS \"${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}\")\n            else()\n                if (GGML_CPU_ARM_ARCH)\n                    list(APPEND ARCH_FLAGS -march=${GGML_CPU_ARM_ARCH})\n                elseif(GGML_CPU_ALL_VARIANTS)\n                    # Begin with the lowest baseline\n                    set(ARM_MCPU \"armv8-a\")\n                    set(ARCH_TAGS \"\")\n                    set(ARCH_DEFINITIONS \"\")\n\n                    # When a feature is selected, bump the MCPU to the first\n                    # version that supported it\n                    if (GGML_INTERNAL_DOTPROD)\n                        set(ARM_MCPU \"armv8.2-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+dotprod\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_DOTPROD)\n                    endif()\n                    if (GGML_INTERNAL_FP16_VECTOR_ARITHMETIC)\n                        set(ARM_MCPU \"armv8.2-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+fp16\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_FP16_VECTOR_ARITHMETIC)\n                    endif()\n                    if (GGML_INTERNAL_SVE)\n                        set(ARM_MCPU \"armv8.2-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+sve\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_SVE)\n                    endif()\n                    if (GGML_INTERNAL_MATMUL_INT8)\n                        set(ARM_MCPU \"armv8.6-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+i8mm\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_MATMUL_INT8)\n                    endif()\n                    if (GGML_INTERNAL_SVE2)\n                        set(ARM_MCPU \"armv8.6-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+sve2\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_SVE2)\n                    endif()\n                    if (GGML_INTERNAL_NOSVE)\n                        set(ARCH_TAGS \"${ARCH_TAGS}+nosve\")\n                    endif()\n                    if (GGML_INTERNAL_SME)\n                        set(ARM_MCPU \"armv9.2-a\")\n                        set(ARCH_TAGS \"${ARCH_TAGS}+sme\")\n                        list(APPEND ARCH_DEFINITIONS GGML_USE_SME)\n                    endif()\n                    list(APPEND ARCH_FLAGS \"-march=${ARM_MCPU}${ARCH_TAGS}\")\n                    ggml_add_cpu_backend_features(${GGML_CPU_NAME} arm ${ARCH_DEFINITIONS})\n                endif()\n            endif()\n\n            message(STATUS \"Checking for ARM features using flags:\")\n            foreach(flag IN LISTS ARCH_FLAGS)\n                message(STATUS \"  ${flag}\")\n            endforeach()\n\n            include(CheckCXXSourceCompiles)\n            set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n            string(REPLACE \";\" \" \" ARCH_FLAGS_STR \"${ARCH_FLAGS}\")\n            set(CMAKE_REQUIRED_FLAGS \"${ARCH_FLAGS_STR}\")\n            foreach(feature DOTPROD SVE MATMUL_INT8 FMA FP16_VECTOR_ARITHMETIC SME)\n                set(ARM_FEATURE \"HAVE_${feature}\")\n                check_cxx_source_compiles(\n                    \"\n                    #if !defined(__ARM_FEATURE_${feature})\n                    #  error \\\"Feature ${feature} is not defined\\\"\n                    #endif\n                    int main() { return 0; }\n                    \"\n                    ${ARM_FEATURE}\n                )\n            endforeach()\n            set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"x86\")\n        message(STATUS \"x86 detected\")\n        list(APPEND GGML_CPU_SOURCES\n            ggml-cpu/arch/x86/quants.c\n            ggml-cpu/arch/x86/repack.cpp\n            )\n\n        if (MSVC)\n            # instruction set detection for MSVC only\n            if (GGML_NATIVE)\n                include(ggml-cpu/cmake/FindSIMD.cmake)\n            endif ()\n            if (GGML_AVX512)\n                list(APPEND ARCH_FLAGS /arch:AVX512)\n                # /arch:AVX512 includes: __AVX512F__, __AVX512CD__, __AVX512BW__, __AVX512DQ__, and __AVX512VL__\n                # MSVC has no compile-time flags enabling specific\n                # AVX512 extensions, neither it defines the\n                # macros corresponding to the extensions.\n                # Do it manually.\n                list(APPEND ARCH_DEFINITIONS GGML_AVX512)\n                if (GGML_AVX512_VBMI)\n                    list(APPEND ARCH_DEFINITIONS __AVX512VBMI__)\n                    if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n                        list(APPEND ARCH_FLAGS -mavx512vbmi)\n                    endif()\n                endif()\n                if (GGML_AVX512_VNNI)\n                    list(APPEND ARCH_DEFINITIONS __AVX512VNNI__ GGML_AVX512_VNNI)\n                    if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n                        list(APPEND ARCH_FLAGS -mavx512vnni)\n                    endif()\n                endif()\n                if (GGML_AVX512_BF16)\n                    list(APPEND ARCH_DEFINITIONS __AVX512BF16__ GGML_AVX512_BF16)\n                    if (CMAKE_C_COMPILER_ID STREQUAL \"Clang\")\n                        list(APPEND ARCH_FLAGS -mavx512bf16)\n                    endif()\n                endif()\n                if (GGML_AMX_TILE)\n                    list(APPEND ARCH_DEFINITIONS __AMX_TILE__ GGML_AMX_TILE)\n                endif()\n                if (GGML_AMX_INT8)\n                    list(APPEND ARCH_DEFINITIONS __AMX_INT8__ GGML_AMX_INT8)\n                endif()\n                if (GGML_AMX_BF16)\n                    list(APPEND ARCH_DEFINITIONS __AMX_BF16__ GGML_AMX_BF16)\n                endif()\n            elseif (GGML_AVX2)\n                list(APPEND ARCH_FLAGS /arch:AVX2)\n                list(APPEND ARCH_DEFINITIONS GGML_AVX2 GGML_FMA GGML_F16C)\n            elseif (GGML_AVX)\n                list(APPEND ARCH_FLAGS /arch:AVX)\n                list(APPEND ARCH_DEFINITIONS GGML_AVX)\n            elseif (GGML_SSE42)\n                list(APPEND ARCH_FLAGS /arch:SSE4.2)\n                list(APPEND ARCH_DEFINITIONS GGML_SSE42)\n            endif()\n            if (GGML_AVX_VNNI)\n                list(APPEND ARCH_DEFINITIONS __AVXVNNI__ GGML_AVX_VNNI)\n            endif()\n            if (GGML_BMI2)\n                # MSVC does not define macro __BMI2__\n                list(APPEND ARCH_DEFINITIONS __BMI2__ GGML_BMI2)\n            endif()\n        else ()\n            if (GGML_NATIVE)\n                list(APPEND ARCH_FLAGS -march=native)\n            else ()\n                if (GGML_SSE42)\n                    list(APPEND ARCH_FLAGS -msse4.2)\n                    list(APPEND ARCH_DEFINITIONS GGML_SSE42)\n                endif()\n                if (GGML_F16C)\n                    list(APPEND ARCH_FLAGS -mf16c)\n                    list(APPEND ARCH_DEFINITIONS GGML_F16C)\n                endif()\n                if (GGML_FMA)\n                    list(APPEND ARCH_FLAGS -mfma)\n                    list(APPEND ARCH_DEFINITIONS GGML_FMA)\n                endif()\n                if (GGML_BMI2)\n                    list(APPEND ARCH_FLAGS -mbmi2)\n                    list(APPEND ARCH_DEFINITIONS GGML_BMI2)\n                endif()\n                if (GGML_AVX)\n                    list(APPEND ARCH_FLAGS -mavx)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX)\n                endif()\n                if (GGML_AVX2)\n                    list(APPEND ARCH_FLAGS -mavx2)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX2)\n                endif()\n                if (GGML_AVX_VNNI)\n                    list(APPEND ARCH_FLAGS -mavxvnni)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX_VNNI)\n                endif()\n                if (GGML_AVX512)\n                    list(APPEND ARCH_FLAGS -mavx512f)\n                    list(APPEND ARCH_FLAGS -mavx512cd)\n                    list(APPEND ARCH_FLAGS -mavx512vl)\n                    list(APPEND ARCH_FLAGS -mavx512dq)\n                    list(APPEND ARCH_FLAGS -mavx512bw)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX512)\n                endif()\n                if (GGML_AVX512_VBMI)\n                    list(APPEND ARCH_FLAGS -mavx512vbmi)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VBMI)\n                endif()\n                if (GGML_AVX512_VNNI)\n                    list(APPEND ARCH_FLAGS -mavx512vnni)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_VNNI)\n                endif()\n                if (GGML_AVX512_BF16)\n                    list(APPEND ARCH_FLAGS -mavx512bf16)\n                    list(APPEND ARCH_DEFINITIONS GGML_AVX512_BF16)\n                endif()\n                if (GGML_AMX_TILE)\n                    list(APPEND ARCH_FLAGS -mamx-tile)\n                    list(APPEND ARCH_DEFINITIONS GGML_AMX_TILE)\n                endif()\n                if (GGML_AMX_INT8)\n                    list(APPEND ARCH_FLAGS -mamx-int8)\n                    list(APPEND ARCH_DEFINITIONS GGML_AMX_INT8)\n                endif()\n                if (GGML_AMX_BF16)\n                    list(APPEND ARCH_FLAGS -mamx-bf16)\n                    list(APPEND ARCH_DEFINITIONS GGML_AMX_BF16)\n                endif()\n            endif()\n        endif()\n\n        if (GGML_BACKEND_DL)\n            if (GGML_NATIVE)\n                # the feature check relies on ARCH_DEFINITIONS, but it is not set with GGML_NATIVE\n                message(FATAL_ERROR \"GGML_NATIVE is not compatible with GGML_BACKEND_DL, consider using GGML_CPU_ALL_VARIANTS\")\n            endif()\n            ggml_add_cpu_backend_features(${GGML_CPU_NAME} x86 ${ARCH_DEFINITIONS})\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"PowerPC\")\n        message(STATUS \"PowerPC detected\")\n        list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/powerpc/quants.c)\n        if (GGML_NATIVE)\n            if (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc64\")\n                file(READ \"/proc/cpuinfo\" POWER10_M)\n            elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"powerpc\")\n                execute_process(COMMAND bash -c \"prtconf |grep 'Implementation' | head -n 1\" OUTPUT_VARIABLE POWER10_M)\n            endif()\n\n            string(TOUPPER \"${POWER10_M}\" POWER10_M_UPPER)\n            string(REGEX MATCHALL \"POWER *([0-9]+)\" MATCHED_STRING \"${POWER10_M_UPPER}\")\n            string(REGEX REPLACE \"POWER *([0-9]+)\" \"\\\\1\" EXTRACTED_NUMBER \"${MATCHED_STRING}\")\n\n            if (EXTRACTED_NUMBER GREATER_EQUAL 10)\n                list(APPEND ARCH_FLAGS -mcpu=power10)\n            elseif (EXTRACTED_NUMBER EQUAL 9)\n                list(APPEND ARCH_FLAGS -mcpu=power9)\n            elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES \"ppc64le\")\n                list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native)\n            else()\n                list(APPEND ARCH_FLAGS -mcpu=native -mtune=native -mpowerpc64)\n            endif()\n        elseif(GGML_CPU_ALL_VARIANTS)\n            # Begin with the lowest baseline\n            set(ARCH_DEFINITIONS \"\")\n\n            # When a feature is selected, bump the MCPU to the first\n            # version that supported it\n            foreach(PVER RANGE 7 11)\n                if(DEFINED GGML_INTERNAL_POWER${PVER})\n                    set(POWERPC_MCPU \"power${PVER}\")\n                    list(APPEND ARCH_DEFINITIONS GGML_USE_POWER${PVER})\n                endif()\n            endforeach()\n            if (GGML_INTERNAL_VSX)\n                list(APPEND ARCH_DEFINITIONS GGML_USE_VSX)\n                list(APPEND ARCH_FLAGS -mvsx)\n            endif()\n\n            if (DEFINED POWERPC_MCPU)\n                list(APPEND ARCH_FLAGS -mcpu=${POWERPC_MCPU})\n            endif()\n            ggml_add_cpu_backend_features(${GGML_CPU_NAME} powerpc ${ARCH_DEFINITIONS})\n        else()\n            if (GGML_CPU_POWERPC_CPUTYPE)\n                list(APPEND ARCH_FLAGS -mcpu=${GGML_CPU_POWERPC_CPUTYPE})\n            endif()\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"loongarch64\")\n        message(STATUS \"loongarch64 detected\")\n        list(APPEND GGML_CPU_SOURCES ggml-cpu/arch/loongarch/quants.c)\n\n        list(APPEND ARCH_FLAGS -march=loongarch64)\n        if (GGML_LASX)\n            list(APPEND ARCH_FLAGS -mlasx)\n        endif()\n        if (GGML_LSX)\n            list(APPEND ARCH_FLAGS -mlsx)\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"riscv64\")\n        message(STATUS \"riscv64 detected\")\n        list(APPEND GGML_CPU_SOURCES\n            ggml-cpu/arch/riscv/quants.c\n            ggml-cpu/arch/riscv/repack.cpp\n            )\n        if (GGML_CPU_RISCV64_SPACEMIT)\n            target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_RISCV64_SPACEMIT ${RISCV64_SPACEMIT_IME_SPEC})\n            list(APPEND GGML_CPU_SOURCES\n                ggml-cpu/spacemit/ime.cpp\n                ggml-cpu/spacemit/ime.h\n                ggml-cpu/spacemit/ime1_kernels.cpp\n                ggml-cpu/spacemit/ime_kernels.h\n            )\n        endif()\n        if(NOT GGML_CPU_ALL_VARIANTS)\n            set(MARCH_STR \"rv64gc\")\n            if (GGML_RV_ZFH)\n                string(APPEND MARCH_STR \"_zfh\")\n            endif()\n\n            if (GGML_XTHEADVECTOR)\n                string(APPEND MARCH_STR \"_xtheadvector\")\n            elseif (GGML_RVV)\n                string(APPEND MARCH_STR \"_v\")\n                if (GGML_RV_ZVFH)\n                    string(APPEND MARCH_STR \"_zvfh\")\n                endif()\n                if (GGML_RV_ZVFBFWMA)\n                    string(APPEND MARCH_STR \"_zvfbfwma\")\n                endif()\n            endif()\n            if (GGML_RV_ZICBOP)\n                string(APPEND MARCH_STR \"_zicbop\")\n            endif()\n            if (GGML_RV_ZIHINTPAUSE)\n                string(APPEND MARCH_STR \"_zihintpause\")\n            endif()\n            list(APPEND ARCH_FLAGS \"-march=${MARCH_STR}\" -mabi=lp64d)\n        else()\n            # Begin with the lowest baseline\n            set(ARCH_DEFINITIONS \"\")\n\n            if (GGML_INTERNAL_RVV)\n                message(STATUS \"RVV enabled\")\n                list(APPEND ARCH_DEFINITIONS GGML_USE_RVV)\n                list(APPEND ARCH_FLAGS -march=rv64gc_v -mabi=lp64d)\n            endif()\n\n            ggml_add_cpu_backend_features(${GGML_CPU_NAME} riscv ${ARCH_DEFINITIONS})\n        endif()\n    elseif (GGML_SYSTEM_ARCH STREQUAL \"s390x\")\n        message(STATUS \"s390x detected\")\n        list(APPEND GGML_CPU_SOURCES\n            ggml-cpu/arch/s390/quants.c)\n\n        # for native compilation\n        if (GGML_NATIVE)\n            # check machine level to determine target\n            file(READ \"/proc/cpuinfo\" CPUINFO_CONTENTS)\n            string(REGEX REPLACE \"machine[ \\t\\r\\n]*=[ \\t\\r\\n]*([0-9]+)\" \"\\\\1\" S390X_M ${CPUINFO_CONTENTS})\n\n            # TODO: Separation to determine activation of VX/VXE/VXE2\n            if (${S390X_M} MATCHES \"8561|8562\")\n                message(STATUS \"z15 target\")\n                list(APPEND ARCH_FLAGS -march=z15)\n            elseif (${S390X_M} MATCHES \"3931\")\n                message(STATUS \"z16 target\")\n                list(APPEND ARCH_FLAGS -march=z16)\n            elseif (${S390X_M} MATCHES \"9175|9176\")\n                # NOTE: Only available from GCC 15.1.0 onwards. Any z17 machine with compile issues must first verify their GCC version.\n                #       binutils must also be updated to the latest for the -march=z17 flag to work. Otherwise, use -march=arch15.\n                message(STATUS \"z17 target\")\n                list(APPEND ARCH_FLAGS -march=arch15)\n            else()\n                message(STATUS \"Unknown target\")\n                message(WARNING \"Unknown target. If you are compiling for z14 and earlier, you might have to add -DGGML_VXE=OFF.\")\n                list(APPEND ARCH_FLAGS -march=native -mtune=native)\n            endif()\n        # for cross-compilation\n        elseif(GGML_CPU_ALL_VARIANTS)\n            # range through IBM z15 to z17\n            # NOTE: update when a new hardware level is released\n            foreach (ZHW RANGE 15 17)\n                if(DEFINED GGML_INTERNAL_Z${ZHW})\n                    message(STATUS \"z${ZHW} cross-compile target\")\n                    list(APPEND ARCH_FLAGS -march=z${ZHW})\n                endif()\n            endforeach()\n        endif()\n\n        if (GGML_VXE OR GGML_INTERNAL_VXE2)\n            message(STATUS \"VXE2 enabled\")\n            list(APPEND ARCH_FLAGS -mvx -mzvector)\n            list(APPEND ARCH_DEFINITIONS GGML_USE_VXE2)\n        endif()\n\n        if (GGML_INTERNAL_NNPA)\n            message(STATUS \"NNPA enabled\")\n            list(APPEND ARCH_DEFINITIONS GGML_USE_NNPA)\n        endif()\n\n        ggml_add_cpu_backend_features(${GGML_CPU_NAME} s390 ${ARCH_DEFINITIONS})\n    elseif (CMAKE_SYSTEM_PROCESSOR MATCHES \"wasm\")\n        message(STATUS \"Wasm detected\")\n        list (APPEND GGML_CPU_SOURCES ggml-cpu/arch/wasm/quants.c)\n    else()\n        message(WARNING \"Unknown CPU architecture. Falling back to generic implementations.\")\n        list(APPEND ARCH_FLAGS -DGGML_CPU_GENERIC)\n    endif()\n\n    if (GGML_CPU_REPACK)\n        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_USE_CPU_REPACK)\n    endif()\n\n    if (GGML_CPU_KLEIDIAI)\n        message(STATUS \"Using KleidiAI optimized kernels if applicable\")\n\n        # Disable the KleidiAI tests\n        set(KLEIDIAI_BUILD_TESTS  OFF)\n\n        # Fetch KleidiAI sources:\n        include(FetchContent)\n        set(KLEIDIAI_COMMIT_TAG \"v1.22.0\")\n        set(KLEIDIAI_DOWNLOAD_URL \"https://github.com/ARM-software/kleidiai/archive/refs/tags/${KLEIDIAI_COMMIT_TAG}.tar.gz\")\n        set(KLEIDIAI_ARCHIVE_MD5  \"54049037570ab0ee0a0d126b2ba5ece1\")\n\n        if (POLICY CMP0135)\n            cmake_policy(SET CMP0135 NEW)\n        endif()\n\n        # TODO: Use FetchContent_MakeAvailable with EXCLUDE_FROM_ALL after bumping minimum CMake version to 3.28+\n        # Using FetchContent_Populate instead to avoid EXCLUDE_FROM_ALL which requires CMake 3.28\n        FetchContent_Declare(KleidiAI_Download\n            URL ${KLEIDIAI_DOWNLOAD_URL}\n            DOWNLOAD_EXTRACT_TIMESTAMP NEW\n            URL_HASH MD5=${KLEIDIAI_ARCHIVE_MD5})\n\n        FetchContent_GetProperties(KleidiAI_Download\n            SOURCE_DIR  KLEIDIAI_SRC\n            POPULATED   KLEIDIAI_POPULATED)\n\n        if (NOT KLEIDIAI_POPULATED)\n            FetchContent_Populate(KleidiAI_Download)\n            FetchContent_GetProperties(KleidiAI_Download SOURCE_DIR KLEIDIAI_SRC)\n        endif()\n\n        add_compile_definitions(GGML_USE_CPU_KLEIDIAI)\n\n        list(APPEND GGML_CPU_SOURCES\n            ggml-cpu/kleidiai/kleidiai.cpp\n            ggml-cpu/kleidiai/kernels.cpp\n            ggml-cpu/kleidiai/kleidiai.h\n            ggml-cpu/kleidiai/kernels.h\n            )\n\n        # KleidiAI\n        include_directories(\n            ${KLEIDIAI_SRC}/\n            ${KLEIDIAI_SRC}/kai/\n            ${KLEIDIAI_SRC}/kai/ukernels/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/)\n\n        set(ARCH_FLAGS_TEMP \"${ARCH_FLAGS}\")\n        if (NOT ARCH_FLAGS_TEMP)\n            string(REGEX MATCH \"-march=[^ ]+\" ARCH_FLAGS_TEMP \"${CMAKE_C_FLAGS}\")\n        endif()\n        string(FIND \"${ARCH_FLAGS_TEMP}\" \"+dotprod\" DOTPROD_ENABLED)\n        string(FIND \"${ARCH_FLAGS_TEMP}\" \"+i8mm\" I8MM_ENABLED)\n        string(FIND \"${ARCH_FLAGS_TEMP}\" \"+sme\" SME_ENABLED)\n        string(FIND \"${ARCH_FLAGS_TEMP}\" \"+sve\" SVE_ENABLED)\n\n        set(PRIVATE_ARCH_FLAGS ${ARCH_FLAGS_TEMP})\n\n        list(APPEND GGML_KLEIDIAI_SOURCES\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.c\n            ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.c)\n\n        if (NOT DOTPROD_ENABLED MATCHES -1)\n            list(APPEND GGML_KLEIDIAI_SOURCES\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.c)\n        endif()\n\n        if (NOT I8MM_ENABLED MATCHES -1)\n            list(APPEND GGML_KLEIDIAI_SOURCES\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.c)\n        endif()\n\n        if (NOT SME_ENABLED MATCHES -1)\n            list(APPEND GGML_KLEIDIAI_SOURCES\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_fp32_bf16p_bf16p/kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_f16p_qsi4c32p/kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_bf16p2vlx2_f32_sme.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/pack/kai_lhs_pack_f16pmrx2_f32_neon.c\n                ${KLEIDIAI_SRC}/kai/kai_common_sme_asm.S)\n            set(PRIVATE_ARCH_FLAGS \"-fno-tree-vectorize;${PRIVATE_ARCH_FLAGS}+sve+sve2+sme2+fp16\")\n        endif()\n\n        if (NOT SVE_ENABLED MATCHES -1)\n            list(APPEND GGML_KLEIDIAI_SOURCES\n                ${KLEIDIAI_SRC}/kai/kai_common_sve_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.c\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm_asm.S\n                ${KLEIDIAI_SRC}/kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.c)\n        endif()\n\n        set_source_files_properties(${GGML_KLEIDIAI_SOURCES} PROPERTIES COMPILE_OPTIONS \"${PRIVATE_ARCH_FLAGS}\")\n        list(APPEND GGML_CPU_SOURCES ${GGML_KLEIDIAI_SOURCES})\n    endif()\n\n    message(STATUS \"Adding CPU backend variant ${GGML_CPU_NAME}: ${ARCH_FLAGS} ${ARCH_DEFINITIONS}\")\n    target_sources(${GGML_CPU_NAME} PRIVATE ${GGML_CPU_SOURCES})\n    target_compile_options(${GGML_CPU_NAME} PRIVATE ${ARCH_FLAGS})\n    target_compile_definitions(${GGML_CPU_NAME} PRIVATE ${ARCH_DEFINITIONS})\n\n    if (EMSCRIPTEN)\n        set_target_properties(${GGML_CPU_NAME} PROPERTIES COMPILE_FLAGS \"-msimd128\")\n    endif()\n\n    if (CMAKE_CXX_COMPILER_ID STREQUAL \"IntelLLVM\")\n        # The compiler automatically enables \"-ffast-math\" which can cause NaNs in tests due to \"-fassociative-math\"\n        target_compile_options(${GGML_CPU_NAME} PRIVATE \"-fno-associative-math\")\n    endif()\nendfunction()\n"
  },
  {
    "path": "ggml/src/ggml-cpu/amx/amx.cpp",
    "content": "#include \"amx.h\"\n#include \"common.h\"\n#include \"mmq.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"traits.h\"\n\n#if defined(__linux__)\n#include <sys/syscall.h>\n#include <unistd.h>\n#endif\n\n#include <cstdlib>\n#include <cstring>\n#include <memory>\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n\n// AMX type_trais\nnamespace ggml::cpu::amx {\nclass tensor_traits : public ggml::cpu::tensor_traits {\n    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n        size = ggml_backend_amx_desired_wsize(op);\n        return true;\n    }\n\n    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n        if (op->op == GGML_OP_MUL_MAT) {\n            ggml_backend_amx_mul_mat(params, op);\n            return true;\n        }\n        return false;\n    }\n};\n\nstatic ggml::cpu::tensor_traits * get_tensor_traits(ggml_backend_buffer_t, struct ggml_tensor *) {\n    static tensor_traits traits;\n    return &traits;\n}\n}  // namespace ggml::cpu::amx\n\n// AMX buffer interface\nstatic void ggml_backend_amx_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n    free(buffer->context);\n}\n\nstatic void * ggml_backend_amx_buffer_get_base(ggml_backend_buffer_t buffer) {\n    return (void *) (buffer->context);\n}\n\nstatic enum ggml_status ggml_backend_amx_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n    tensor->extra = (void *) ggml::cpu::amx::get_tensor_traits(buffer, tensor);\n\n    GGML_UNUSED(buffer);\n    return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_amx_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n                                                  uint8_t value, size_t offset, size_t size) {\n    memset((char *) tensor->data + offset, value, size);\n\n    GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_amx_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n                                               const void * data, size_t offset, size_t size) {\n    if (qtype_has_amx_kernels(tensor->type)) {\n        GGML_LOG_DEBUG(\"%s: amx repack tensor %s of type %s\\n\", __func__, tensor->name, ggml_type_name(tensor->type));\n        ggml_backend_amx_convert_weight(tensor, data, offset, size);\n    } else {\n        memcpy((char *) tensor->data + offset, data, size);\n    }\n\n    GGML_UNUSED(buffer);\n}\n\n/*\n// need to figure what we need to do with buffer->extra.\nstatic void ggml_backend_amx_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {\n    GGML_ASSERT(!qtype_has_amx_kernels(tensor->type));\n    memcpy(data, (const char *)tensor->data + offset, size);\n\n    GGML_UNUSED(buffer);\n}\n\nstatic bool ggml_backend_amx_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {\n    if (ggml_backend_buffer_is_host(src->buffer)) {\n        if (qtype_has_amx_kernels(src->type)) {\n            ggml_backend_amx_convert_weight(dst, src->data, 0, ggml_nbytes(dst));\n        } else {\n            memcpy(dst->data, src->data, ggml_nbytes(src));\n        }\n        return true;\n    }\n    return false;\n\n    GGML_UNUSED(buffer);\n}\n*/\n\nstatic void ggml_backend_amx_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {\n    memset(buffer->context, value, buffer->size);\n}\n\nstatic ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {\n    /* .free_buffer     = */ ggml_backend_amx_buffer_free_buffer,\n    /* .get_base        = */ ggml_backend_amx_buffer_get_base,\n    /* .init_tensor     = */ ggml_backend_amx_buffer_init_tensor,\n    /* .memset_tensor   = */ ggml_backend_amx_buffer_memset_tensor,\n    /* .set_tensor      = */ ggml_backend_amx_buffer_set_tensor,\n    /* .get_tensor      = */ nullptr,\n    /* .cpy_tensor      = */ nullptr,\n    /* .clear           = */ ggml_backend_amx_buffer_clear,\n    /* .reset           = */ nullptr,\n};\n\nstatic const char * ggml_backend_amx_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"AMX\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    void * data = ggml_aligned_malloc(size);\n    if (data == NULL) {\n        fprintf(stderr, \"%s: failed to allocate buffer of size %zu\\n\", __func__, size);\n        return NULL;\n    }\n\n    return ggml_backend_buffer_init(buft, ggml_backend_amx_buffer_interface, data, size);\n}\n\nstatic size_t ggml_backend_amx_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    return TENSOR_ALIGNMENT;\n\n    GGML_UNUSED(buft);\n}\n\nnamespace ggml::cpu::amx {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n    bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n        if (op->op != GGML_OP_MUL_MAT) {\n            return false;\n        }\n        auto * src0 = op->src[0];\n        auto * src1 = op->src[1];\n\n        if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {\n            return false;\n        }\n        if (!src0->buffer || src0->buffer->buft != ggml_backend_amx_buffer_type()) {\n            return false;\n        }\n        if (src1->buffer && !ggml_backend_buft_is_host(src1->buffer->buft)) {\n            return false;\n        }\n        if (op->ne[0] % (TILE_N * 2)) {\n            return false;\n        }\n        int alignment;\n        switch (src0->type) {\n            case GGML_TYPE_Q4_0:\n            case GGML_TYPE_Q4_1:\n            case GGML_TYPE_Q8_0:\n                alignment = TILE_K;\n                break;\n            case GGML_TYPE_Q4_K:\n            case GGML_TYPE_Q5_K:\n            case GGML_TYPE_Q6_K:\n            case GGML_TYPE_IQ4_XS:\n                alignment = 256; // QK_K\n                break;\n            case GGML_TYPE_F16:\n                alignment = 16;\n                break;\n            default:\n                return false;\n        }\n        if (src0->ne[0] % alignment) {\n            return false;\n        }\n        if (src1->type != GGML_TYPE_F32) {\n            return false;\n        }\n        return true;\n    }\n\n    ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n        if (op->op == GGML_OP_MUL_MAT && op->src[0]->buffer &&\n            op->src[0]->buffer->buft == ggml_backend_amx_buffer_type()) {\n            return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n        }\n\n        return nullptr;\n    }\n};\n}  // namespace ggml::cpu::amx\n\nstatic size_t ggml_backend_amx_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {\n    return ggml_backend_amx_get_alloc_size(tensor);\n\n    GGML_UNUSED(buft);\n}\n\n#define ARCH_GET_XCOMP_PERM     0x1022\n#define ARCH_REQ_XCOMP_PERM     0x1023\n#define XFEATURE_XTILECFG       17\n#define XFEATURE_XTILEDATA      18\n\nstatic bool ggml_amx_init() {\n#if defined(__linux__)\n    if (syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA)) {\n        fprintf(stderr, \"AMX is not ready to be used!\\n\");\n        return false;\n    }\n    return true;\n#elif defined(_WIN32)\n    return true;\n#else\n    return false;\n#endif\n}\n\nggml_backend_buffer_type_t ggml_backend_amx_buffer_type() {\n    static struct ggml_backend_buffer_type ggml_backend_buffer_type_amx = {\n        /* .iface = */ {\n                        /* .get_name         = */ ggml_backend_amx_buffer_type_get_name,\n                        /* .alloc_buffer     = */ ggml_backend_amx_buffer_type_alloc_buffer,\n                        /* .get_alignment    = */ ggml_backend_amx_buffer_type_get_alignment,\n                        /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX\n                        /* .get_alloc_size   = */ ggml_backend_amx_buffer_type_get_alloc_size,\n                        /* .is_host          = */ nullptr,\n                        },\n        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ new ggml::cpu::amx::extra_buffer_type(),\n    };\n\n    if (!ggml_amx_init()) {\n        return nullptr;\n    }\n\n    return &ggml_backend_buffer_type_amx;\n}\n\n#endif  // defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n"
  },
  {
    "path": "ggml/src/ggml-cpu/amx/amx.h",
    "content": "#include \"ggml-backend.h\"\n#include \"ggml-cpu-impl.h\"\n\n// GGML internal header\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\nggml_backend_buffer_type_t ggml_backend_amx_buffer_type(void);\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/amx/common.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-cpu-impl.h\"\n\n#include <algorithm>\n#include <memory>\n#include <type_traits>\n\n#if defined(GGML_USE_OPENMP)\n#include <omp.h>\n#else\n#include <thread>\n#endif\n\n#define TILE_M 16\n#define TILE_N 16\n#define TILE_K 32\n#define VNNI_BLK 4\n\n#define AMX_BLK_SIZE 32\n\n#define TMM0 0\n#define TMM1 1\n#define TMM2 2\n#define TMM3 3\n#define TMM4 4\n#define TMM5 5\n#define TMM6 6\n#define TMM7 7\n\n// parallel routines\ntemplate <typename T, typename std::enable_if<std::is_integral<T>::value, int>::type = 0>\ninline T div_up(T x, T y) { return (x + y - 1) / y; }\n\ntemplate <typename T>\ninline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {\n#if 0\n    // onednn partition pattern\n    T& n_my = n_end;\n    if (nth <= 1 || n == 0) {\n        n_start = 0;\n        n_my = n;\n    } else {\n        T n1 = div_up(n, nth);\n        T n2 = n1 - 1;\n        T T1 = n - n2 * nth;\n        n_my = ith < T1 ? n1 : n2;\n        n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2;\n    }\n    n_end += n_start;\n#else\n    // pytorch aten partition pattern\n    T n_my = div_up(n, nth);\n    n_start = ith * n_my;\n    n_end = std::min(n_start + n_my, n);\n#endif\n}\n\ntemplate <typename func_t>\ninline void parallel_for(int n, const func_t & f) {\n    if (n <= 0) {\n        return;\n    }\n#if defined(GGML_USE_OPENMP)\n    #pragma omp parallel\n    {\n        int nth = omp_get_num_threads();\n        int ith = omp_get_thread_num();\n        int tbegin, tend;\n        balance211(n, nth, ith, tbegin, tend);\n        f(tbegin, tend);\n    }\n#else\n    int nth = std::thread::hardware_concurrency();\n    if (nth <= 1) {\n        f(0, n);\n        return;\n    }\n    if (nth > n) {\n        nth = n;\n    }\n    std::vector<std::thread> threads;\n    threads.reserve(nth);\n    for (int ith = 0; ith < nth; ++ith) {\n        threads.emplace_back([&f, n, ith, nth] {\n            int tbegin, tend;\n            balance211(n, nth, ith, tbegin, tend);\n            f(tbegin, tend);\n        });\n    }\n    for (auto & t : threads) {\n        t.join();\n    }\n#endif\n}\n\ntemplate <typename func_t>\ninline void parallel_for_ggml(const ggml_compute_params * params, int n, const func_t & f) {\n    int tbegin, tend;\n    balance211(n, params->nth, params->ith, tbegin, tend);\n    f(tbegin, tend);\n}\n\n// quantized types that have AMX support\ninline bool qtype_has_amx_kernels(const enum ggml_type type) {\n    // TODO: fix padding for vnni format\n    return (type == GGML_TYPE_Q4_0) ||\n        (type == GGML_TYPE_Q4_1) ||\n        (type == GGML_TYPE_Q8_0) ||\n        (type == GGML_TYPE_Q4_K) ||\n        (type == GGML_TYPE_Q5_K) ||\n        (type == GGML_TYPE_Q6_K) ||\n        (type == GGML_TYPE_IQ4_XS);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/amx/mmq.cpp",
    "content": "#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wpedantic\"\n#pragma GCC diagnostic ignored \"-Wunused-local-typedefs\"\n#endif\n\n#include \"amx.h\"\n#include \"mmq.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"quants.h\"\n#include \"ggml-quants.h\"\n#include <algorithm>\n#include <type_traits>\n\n#if defined(__gnu_linux__)\n#include <sys/syscall.h>\n#include <unistd.h>\n#endif\n\n#if (defined(_WIN32) || defined(_WIN64))\n#define RESTRICT __restrict\n#else\n#define RESTRICT __restrict__\n#endif\n\n#if (defined(_WIN32) || defined(_WIN64))\n#define ALWAYS_INLINE __forceinline\n#elif __has_attribute(always_inline) || defined(__GNUC__)\n#define ALWAYS_INLINE __attribute__((__always_inline__)) inline\n#else\n#define ALWAYS_INLINE inline\n#endif\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n\nnamespace {\n\n// Forced unrolling\ntemplate <int n>\nstruct Unroll {\n    template <typename Func, typename... Args>\n    ALWAYS_INLINE void operator()(const Func& f, Args... args) const {\n        Unroll<n - 1>{}(f, args...);\n        f(std::integral_constant<int, n - 1>{}, args...);\n    }\n};\n\ntemplate <>\nstruct Unroll<1> {\n    template <typename Func, typename... Args>\n    ALWAYS_INLINE void operator()(const Func& f, Args... args) const {\n        f(std::integral_constant<int, 0>{}, args...);\n    }\n};\n\n// type traits\ntemplate <typename T> struct PackedTypes {};\ntemplate <> struct PackedTypes<block_q4_0> { using type = int8_t; };\ntemplate <> struct PackedTypes<block_q4_1> { using type = uint8_t; };\ntemplate <> struct PackedTypes<block_q8_0> { using type = int8_t; };\ntemplate <typename T> using packed_B_type = typename PackedTypes<T>::type;\n\ntemplate <typename T>\nstruct do_compensate : std::integral_constant<bool,\n    std::is_same<T, block_q8_0>::value> {};\n\ntemplate <typename T>\nstruct do_unpack : std::integral_constant<bool,\n    std::is_same<T, block_q4_0>::value ||\n    std::is_same<T, block_q4_1>::value> {};\n\ntemplate <typename T>\nstruct is_type_qkk : std::integral_constant<bool,\n    std::is_same<T, block_q4_K>::value ||\n    std::is_same<T, block_q5_K>::value ||\n    std::is_same<T, block_q6_K>::value ||\n    std::is_same<T, block_iq4_xs>::value> {};\n\n#define GGML_DISPATCH_FLOATING_TYPES(TYPE, ...)                                        \\\n    [&] {                                                                              \\\n        switch (TYPE) {                                                                \\\n            case GGML_TYPE_F16: {                                                      \\\n                using type = ggml_fp16_t;                                              \\\n                constexpr int blck_size = 16;                                          \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_BF16: {                                                     \\\n                using type = ggml_bf16_t;                                              \\\n                constexpr int blck_size = 32;                                          \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            default:                                                                   \\\n                fprintf(stderr, \"Unsupported floating data type\\n\");                   \\\n        }                                                                              \\\n    }()\n\n#define GGML_DISPATCH_QTYPES(QT, ...)                                                  \\\n    [&] {                                                                              \\\n        switch (QT) {                                                                  \\\n            case GGML_TYPE_Q4_0: {                                                     \\\n                using type = block_q4_0;                                               \\\n                using vec_dot_type = block_q8_0;                                       \\\n                constexpr int blck_size = QK4_0;                                       \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_Q4_1: {                                                     \\\n                using type = block_q4_1;                                               \\\n                using vec_dot_type = block_q8_1;                                       \\\n                constexpr int blck_size = QK4_1;                                       \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_Q8_0: {                                                     \\\n                using type = block_q8_0;                                               \\\n                using vec_dot_type = block_q8_0;                                       \\\n                constexpr int blck_size = QK8_0;                                       \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_Q4_K: {                                                     \\\n                using type = block_q4_K;                                               \\\n                using vec_dot_type = block_q8_K;                                       \\\n                constexpr int blck_size = QK_K;                                        \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_Q5_K: {                                                     \\\n                using type = block_q5_K;                                               \\\n                using vec_dot_type = block_q8_K;                                       \\\n                constexpr int blck_size = QK_K;                                        \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_Q6_K: {                                                     \\\n                using type = block_q6_K;                                               \\\n                using vec_dot_type = block_q8_K;                                       \\\n                constexpr int blck_size = QK_K;                                        \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            case GGML_TYPE_IQ4_XS: {                                                   \\\n                using type = block_iq4_xs;                                             \\\n                using vec_dot_type = block_q8_K;                                       \\\n                constexpr int blck_size = QK_K;                                        \\\n                return __VA_ARGS__();                                                  \\\n            }                                                                          \\\n            default:                                                                   \\\n                fprintf(stderr, \"Unsupported quantized data type: %d\\n\", int(TYPE));   \\\n        }                                                                              \\\n    }()\n\n#define GGML_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...)                                     \\\n    [&] {                                                                              \\\n        if (BOOL_V) {                                                                  \\\n            constexpr bool BOOL_NAME = true;                                           \\\n            return __VA_ARGS__();                                                      \\\n        } else {                                                                       \\\n            constexpr bool BOOL_NAME = false;                                          \\\n            return __VA_ARGS__();                                                      \\\n        }                                                                              \\\n    }()\n\n// define amx tile config data structure\nstruct tile_config_t{\n    uint8_t palette_id = 0;\n    uint8_t start_row = 0;\n    uint8_t reserved_0[14] = {0};\n    uint16_t colsb[16] = {0};\n    uint8_t rows[16] = {0};\n};\n\n// Notes: amx tile config\n//\n// Typically, TMUL calculates A and B of size 16 x 64 containing INT8 values,\n// and accumulate the result to a 16 x 16 matrix C containing INT32 values,\n//\n// As many GGUF quantized types as `block_size` of 32, so a 16-16-32 config is used\n// instead of the normally used 16-16-64 config.\n//\n//    Block A: {16, 32}, dtype = int8_t\n//    Block B: {16, 32}, dtype = uint8_t/int8_t\n//    Block C: {16, 16}, dtype = int32_t\n//\n// Block B needs to be prepacked to vnni format before feeding into  TMUL:\n//    packed_B: from {n, k} to {k/vnni_blk, n, vnni_blck}, viewed in 2d, we get {8, 64}\n//\n// Therefore, we get tileconfig:\n//             A    B    C\n//    rows    16    8   16\n//    colsb   32   64   16\n//\n// For tile distribution, follow a 2-2-4 pattern, e.g. A used TMM2-TMM3, B used TMM0-TMM1,\n// C used TMM4-TMM7:\n//            B TMM0  B TMM1\n//    A TMM2  C TMM4  C TMM6\n//    A TMM3  C TMM5  C TMM7\n//\n// Each `amx` kernel handles 4 blocks at a time: 2MB * 2NB, when m < 2 * BLOCK_M, unpack A\n// will be needed.\n//\n// Here another commonly used pattern 1-3-3 is skipped, as it is mostly used when m <=16;\n// and the single batch gemm (m=1) has a special fast path with `avx512-vnni`.\n//\n// ref: https://www.intel.com/content/www/us/en/developer/articles/code-sample/\n//    advanced-matrix-extensions-intrinsics-functions.html\n//\n\ninline void ggml_tile_config_init(void) {\n    static thread_local bool done = false;\n\n    if (done) {\n        return;\n    }\n\n    alignas(64) tile_config_t tc = {};\n    tc.palette_id = 1;\n    tc.start_row = 0;\n    tc.rows[0] = 8;   tc.colsb[0] = 64;\n    tc.rows[1] = 8;   tc.colsb[1] = 64;\n    tc.rows[2] = 16;  tc.colsb[2] = 32;\n    tc.rows[3] = 16;  tc.colsb[3] = 32;\n    tc.rows[4] = 16;  tc.colsb[4] = 64;\n    tc.rows[5] = 16;  tc.colsb[5] = 64;\n    tc.rows[6] = 16;  tc.colsb[6] = 64;\n    tc.rows[7] = 16;  tc.colsb[7] = 64;\n\n    _tile_loadconfig(&tc);\n    done = true;\n}\n\n// we need an extra 16 * 4B (TILE_N * int32_t) for each NB/KB block for compensation.\n// See the notes `s8s8 igemm compensation in avx512-vnni` for detail.\ntemplate <typename TB>\nint get_tile_size() {\n    int tile_size = TILE_N * sizeof(TB);\n    if (do_compensate<TB>::value) {\n        tile_size += TILE_N * sizeof(int32_t);\n    }\n    if (std::is_same<TB, block_q4_K>::value ||\n        std::is_same<TB, block_q5_K>::value) {\n        tile_size += TILE_N * 4;\n    }\n    if (std::is_same<TB, block_iq4_xs>::value) {\n        tile_size += TILE_N * 2;\n    }\n    return tile_size;\n}\n\ntemplate <typename TB, int BLOCK_K>\nint get_row_size(int K) {\n    int KB = K / BLOCK_K;\n    int row_size = KB * sizeof(TB);\n    if (do_compensate<TB>::value) {\n        row_size += KB * sizeof(int32_t);\n    }\n    if (std::is_same<TB, block_q4_K>::value ||\n        std::is_same<TB, block_q5_K>::value) {\n        row_size += KB * 4;\n    }\n    if (std::is_same<TB, block_iq4_xs>::value) {\n        row_size += KB * 2;\n    }\n    return row_size;\n}\n\n// transpose utils\n#define SHUFFLE_EPI32(a, b, mask) \\\n    _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), mask))\ninline void transpose_8x8_32bit(__m256i * v, __m256i * v1) {\n    // unpacking and 32-bit elements\n    v1[0] = _mm256_unpacklo_epi32(v[0], v[1]);\n    v1[1] = _mm256_unpackhi_epi32(v[0], v[1]);\n    v1[2] = _mm256_unpacklo_epi32(v[2], v[3]);\n    v1[3] = _mm256_unpackhi_epi32(v[2], v[3]);\n    v1[4] = _mm256_unpacklo_epi32(v[4], v[5]);\n    v1[5] = _mm256_unpackhi_epi32(v[4], v[5]);\n    v1[6] = _mm256_unpacklo_epi32(v[6], v[7]);\n    v1[7] = _mm256_unpackhi_epi32(v[6], v[7]);\n\n    // shuffling the 32-bit elements\n    v[0] = SHUFFLE_EPI32(v1[0], v1[2], 0x44);\n    v[1] = SHUFFLE_EPI32(v1[0], v1[2], 0xee);\n    v[2] = SHUFFLE_EPI32(v1[4], v1[6], 0x44);\n    v[3] = SHUFFLE_EPI32(v1[4], v1[6], 0xee);\n    v[4] = SHUFFLE_EPI32(v1[1], v1[3], 0x44);\n    v[5] = SHUFFLE_EPI32(v1[1], v1[3], 0xee);\n    v[6] = SHUFFLE_EPI32(v1[5], v1[7], 0x44);\n    v[7] = SHUFFLE_EPI32(v1[5], v1[7], 0xee);\n\n    // shuffling 128-bit elements\n    v1[0] = _mm256_permute2f128_si256(v[2], v[0], 0x02);\n    v1[1] = _mm256_permute2f128_si256(v[3], v[1], 0x02);\n    v1[2] = _mm256_permute2f128_si256(v[6], v[4], 0x02);\n    v1[3] = _mm256_permute2f128_si256(v[7], v[5], 0x02);\n    v1[4] = _mm256_permute2f128_si256(v[2], v[0], 0x13);\n    v1[5] = _mm256_permute2f128_si256(v[3], v[1], 0x13);\n    v1[6] = _mm256_permute2f128_si256(v[6], v[4], 0x13);\n    v1[7] = _mm256_permute2f128_si256(v[7], v[5], 0x13);\n}\n\ninline void transpose_16x4_32bit(__m512i * r, __m512i * d) {\n\n    static const __m512i index1 = _mm512_set_epi32(\n        0x0f, 0x0b, 0x07, 0x03,\n        0x0e, 0x0a, 0x06, 0x02,\n        0x0d, 0x09, 0x05, 0x01,\n        0x0c, 0x08, 0x04, 0x00);\n\n    d[0] = _mm512_permutexvar_epi32(index1, r[0]);\n    d[1] = _mm512_permutexvar_epi32(index1, r[1]);\n    d[2] = _mm512_permutexvar_epi32(index1, r[2]);\n    d[3] = _mm512_permutexvar_epi32(index1, r[3]);\n\n    r[0] = _mm512_shuffle_i32x4(d[0], d[1], 0x44);\n    r[1] = _mm512_shuffle_i32x4(d[0], d[1], 0xee);\n    r[2] = _mm512_shuffle_i32x4(d[2], d[3], 0x44);\n    r[3] = _mm512_shuffle_i32x4(d[2], d[3], 0xee);\n\n    d[0] = _mm512_shuffle_i32x4(r[0], r[2], 0x88);\n    d[1] = _mm512_shuffle_i32x4(r[0], r[2], 0xdd);\n    d[2] = _mm512_shuffle_i32x4(r[1], r[3], 0x88);\n    d[3] = _mm512_shuffle_i32x4(r[1], r[3], 0xdd);\n}\n\ninline void transpose_16x16_32bit(__m512i * v) {\n    __m512i v1[16];\n    v1[0] = _mm512_unpacklo_epi32(v[0], v[1]);\n    v1[1] = _mm512_unpackhi_epi32(v[0], v[1]);\n    v1[2] = _mm512_unpacklo_epi32(v[2], v[3]);\n    v1[3] = _mm512_unpackhi_epi32(v[2], v[3]);\n    v1[4] = _mm512_unpacklo_epi32(v[4], v[5]);\n    v1[5] = _mm512_unpackhi_epi32(v[4], v[5]);\n    v1[6] = _mm512_unpacklo_epi32(v[6], v[7]);\n    v1[7] = _mm512_unpackhi_epi32(v[6], v[7]);\n    v1[8] = _mm512_unpacklo_epi32(v[8], v[9]);\n    v1[9] = _mm512_unpackhi_epi32(v[8], v[9]);\n    v1[10] = _mm512_unpacklo_epi32(v[10], v[11]);\n    v1[11] = _mm512_unpackhi_epi32(v[10], v[11]);\n    v1[12] = _mm512_unpacklo_epi32(v[12], v[13]);\n    v1[13] = _mm512_unpackhi_epi32(v[12], v[13]);\n    v1[14] = _mm512_unpacklo_epi32(v[14], v[15]);\n    v1[15] = _mm512_unpackhi_epi32(v[14], v[15]);\n\n    v[0] = _mm512_unpacklo_epi64(v1[0], v1[2]);\n    v[1] = _mm512_unpackhi_epi64(v1[0], v1[2]);\n    v[2] = _mm512_unpacklo_epi64(v1[1], v1[3]);\n    v[3] = _mm512_unpackhi_epi64(v1[1], v1[3]);\n    v[4] = _mm512_unpacklo_epi64(v1[4], v1[6]);\n    v[5] = _mm512_unpackhi_epi64(v1[4], v1[6]);\n    v[6] = _mm512_unpacklo_epi64(v1[5], v1[7]);\n    v[7] = _mm512_unpackhi_epi64(v1[5], v1[7]);\n    v[8] = _mm512_unpacklo_epi64(v1[8], v1[10]);\n    v[9] = _mm512_unpackhi_epi64(v1[8], v1[10]);\n    v[10] = _mm512_unpacklo_epi64(v1[9], v1[11]);\n    v[11] = _mm512_unpackhi_epi64(v1[9], v1[11]);\n    v[12] = _mm512_unpacklo_epi64(v1[12], v1[14]);\n    v[13] = _mm512_unpackhi_epi64(v1[12], v1[14]);\n    v[14] = _mm512_unpacklo_epi64(v1[13], v1[15]);\n    v[15] = _mm512_unpackhi_epi64(v1[13], v1[15]);\n\n    v1[0] = _mm512_shuffle_i32x4(v[0], v[4], 0x88);\n    v1[1] = _mm512_shuffle_i32x4(v[1], v[5], 0x88);\n    v1[2] = _mm512_shuffle_i32x4(v[2], v[6], 0x88);\n    v1[3] = _mm512_shuffle_i32x4(v[3], v[7], 0x88);\n    v1[4] = _mm512_shuffle_i32x4(v[0], v[4], 0xdd);\n    v1[5] = _mm512_shuffle_i32x4(v[1], v[5], 0xdd);\n    v1[6] = _mm512_shuffle_i32x4(v[2], v[6], 0xdd);\n    v1[7] = _mm512_shuffle_i32x4(v[3], v[7], 0xdd);\n    v1[8] = _mm512_shuffle_i32x4(v[8], v[12], 0x88);\n    v1[9] = _mm512_shuffle_i32x4(v[9], v[13], 0x88);\n    v1[10] = _mm512_shuffle_i32x4(v[10], v[14], 0x88);\n    v1[11] = _mm512_shuffle_i32x4(v[11], v[15], 0x88);\n    v1[12] = _mm512_shuffle_i32x4(v[8], v[12], 0xdd);\n    v1[13] = _mm512_shuffle_i32x4(v[9], v[13], 0xdd);\n    v1[14] = _mm512_shuffle_i32x4(v[10], v[14], 0xdd);\n    v1[15] = _mm512_shuffle_i32x4(v[11], v[15], 0xdd);\n\n    v[0] = _mm512_shuffle_i32x4(v1[0], v1[8], 0x88);\n    v[1] = _mm512_shuffle_i32x4(v1[1], v1[9], 0x88);\n    v[2] = _mm512_shuffle_i32x4(v1[2], v1[10], 0x88);\n    v[3] = _mm512_shuffle_i32x4(v1[3], v1[11], 0x88);\n    v[4] = _mm512_shuffle_i32x4(v1[4], v1[12], 0x88);\n    v[5] = _mm512_shuffle_i32x4(v1[5], v1[13], 0x88);\n    v[6] = _mm512_shuffle_i32x4(v1[6], v1[14], 0x88);\n    v[7] = _mm512_shuffle_i32x4(v1[7], v1[15], 0x88);\n    v[8] = _mm512_shuffle_i32x4(v1[0], v1[8], 0xdd);\n    v[9] = _mm512_shuffle_i32x4(v1[1], v1[9], 0xdd);\n    v[10] = _mm512_shuffle_i32x4(v1[2], v1[10], 0xdd);\n    v[11] = _mm512_shuffle_i32x4(v1[3], v1[11], 0xdd);\n    v[12] = _mm512_shuffle_i32x4(v1[4], v1[12], 0xdd);\n    v[13] = _mm512_shuffle_i32x4(v1[5], v1[13], 0xdd);\n    v[14] = _mm512_shuffle_i32x4(v1[6], v1[14], 0xdd);\n    v[15] = _mm512_shuffle_i32x4(v1[7], v1[15], 0xdd);\n}\n\nvoid quantize_row_q8_K_vnni(const float * RESTRICT x, void * RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    const int KB = k / QK_K;\n    constexpr int kVecs = QK_K / 16;\n\n    block_q8_K * y = reinterpret_cast<block_q8_K *>(vy);\n\n    // hold 16 float vecs from x\n    __m512  v[kVecs];\n\n    // hold the quants vecs\n    __m512i vq[kVecs / 4];\n\n    // hold the packed quants vecs\n    __m512i vq_packed[kVecs / 4];\n\n    const __m512 signBit = _mm512_set1_ps(-0.f);\n\n    for (int i = 0; i < KB; ++i) {\n        // Compute max(abs(e)) for the block\n        __m512 vamax = _mm512_set1_ps(0.f);\n        for (int j = 0; j < kVecs; ++j) {\n            v[j] = _mm512_loadu_ps(x); x += 16;\n            vamax = _mm512_max_ps(vamax, _mm512_andnot_ps(signBit, v[j]));\n        }\n        const float amax = _mm512_reduce_max_ps(vamax);\n\n        // Quantize these floats\n        const float iscale = 127.f / amax;\n        y[i].d = GGML_CPU_FP32_TO_FP16(1 / iscale);\n        const float id = ( amax != 0.0f ) ? iscale : 0.f;\n        const __m512 vscale = _mm512_set1_ps(id);\n\n        // Apply multiplier and round to nearest integer\n        for (int j = 0; j < kVecs; ++j) {\n            v[j] = _mm512_mul_ps(v[j], vscale);\n            v[j] = _mm512_roundscale_ps(v[j], (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC));\n        }\n\n        // Pack to epi8 vecs\n        for (int j = 0; j < kVecs / 4; ++j) {\n            __m128i q8_0 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 0]));\n            __m128i q8_1 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 1]));\n            __m128i q8_2 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 2]));\n            __m128i q8_3 = _mm512_cvtepi32_epi8(_mm512_cvtps_epi32(v[j * 4 + 3]));\n\n            __m256i q8_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(q8_0), (q8_1), 1);\n            __m256i q8_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(q8_2), (q8_3), 1);\n\n            vq[j] = _mm512_inserti32x8(_mm512_castsi256_si512(q8_01), q8_23, 1);\n            _mm512_storeu_si512((__m512i *)(y[i].qs + j * 64), vq[j]);\n        }\n\n        // Compute the bsums with vnni\n        transpose_16x4_32bit(vq, vq_packed);\n\n        const __m512i one = _mm512_set1_epi8(1);\n        __m512i sum = _mm512_setzero_si512();\n        for (int k = 0; k < 4; ++k) {\n            sum = _mm512_dpbusd_epi32(sum, one, vq_packed[k]);\n        }\n        _mm256_storeu_si256((__m256i *)(y[i].bsums), _mm512_cvtepi32_epi16(sum));\n    }\n}\n\n// quantize A from float to `vec_dot_type`\ntemplate <typename T>\ninline void from_float(const float * x, char * vy, int64_t k);\n\ntemplate <>\ninline void from_float<block_q8_0>(const float * x, char * vy, int64_t k) {\n    quantize_row_q8_0(x, (block_q8_0 *)vy, k);\n}\n\ntemplate <>\ninline void from_float<block_q8_1>(const float * x, char * vy, int64_t k) {\n    quantize_row_q8_1(x, (block_q8_1 *)vy, k);\n}\n\ntemplate <>\ninline void from_float<block_q8_K>(const float * x, char * vy, int64_t k) {\n#if 1\n    // TODO: this is reference impl!\n    quantize_row_q8_K_ref(x, (block_q8_K *)vy, k);\n#else\n    quantize_row_q8_K_vnni(x, vy, k);\n#endif\n}\n\n// load A from memory to array when nrows can not fill in whole tile\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_0 * RESTRICT A, int lda, int nr) {\n    assert(nr != TILE_M);\n    for (int m = 0; m < nr; ++m) {\n        const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs));\n        _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n    }\n}\n\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_1 * RESTRICT A, int lda, int nr) {\n    assert(nr != TILE_M);\n    for (int m = 0; m < nr; ++m) {\n        const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs));\n        _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n    }\n}\n\ntemplate <typename TB>\nvoid unpack_A(int8_t * RESTRICT tile, const block_q8_K * RESTRICT A, int lda, int k, int nr) {\n    assert(nr <= TILE_M);\n    for (int m = 0; m < nr; ++m) {\n        const __m256i v = _mm256_loadu_si256((const __m256i *)(A[m * lda].qs + k * 32));\n        _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), v);\n    }\n}\n\ntemplate <>\nvoid unpack_A<block_q6_K>(int8_t * RESTRICT tile, const block_q8_K * RESTRICT A, int lda, int k, int nr) {\n    assert(nr <= TILE_M);\n    // zero padding k from 16 to 32, so that we don't have to re-config amx\n    const __m128i zero = _mm_setzero_si128();\n    for (int m = 0; m < nr; ++m) {\n        const __m128i v = _mm_loadu_si128((const __m128i *)(A[m * lda].qs + k * 16));\n        const __m256i r = _mm256_insertf128_si256(_mm256_castsi128_si256(v), zero, 1);\n        _mm256_storeu_si256((__m256i *)(tile + m * TILE_K), r);\n    }\n}\n\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\ninline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {\n    const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);\n    const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);\n    const __m256i lowMask = _mm256_set1_epi8(0xF);\n    return _mm256_and_si256(lowMask, bytes);\n}\n\n// used for block_q4_K\ninline __m512i bytes_from_nibbles_64(const uint8_t * rsi) {\n    const __m256i tmp = _mm256_loadu_si256((const __m256i *)rsi);\n    const __m256i lowMask = _mm256_set1_epi8(0xF);\n    const __m256i q4l = _mm256_and_si256(tmp, lowMask);\n    const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(tmp, 4), lowMask);\n    return _mm512_inserti32x8(_mm512_castsi256_si512(q4l), q4h, 1);\n}\n\n// used for block_q5_K\ninline __m512i bytes_from_nibbles_64(const uint8_t * qs, const uint8_t * qh, int k) {\n    const __m256i lowMask = _mm256_set1_epi8(0xF);\n    __m256i hmask = _mm256_set1_epi8(1);\n    hmask = _mm256_slli_epi16(hmask, k);\n\n    const __m256i q5bits = _mm256_loadu_si256((const __m256i *)qs);\n    const __m256i hbits = _mm256_loadu_si256((const __m256i *)qh);\n\n    const __m256i q5l_0 = _mm256_and_si256(q5bits, lowMask);\n    const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), k + 0), 4);\n    const __m256i q5_0  = _mm256_add_epi8(q5l_0, q5h_0);\n    hmask = _mm256_slli_epi16(hmask, 1);\n\n    const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), lowMask);\n    const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), k + 1), 4);\n    const __m256i q5_1  = _mm256_add_epi8(q5l_1, q5h_1);\n\n    return _mm512_inserti32x8(_mm512_castsi256_si512(q5_0), q5_1, 1);\n}\n\n// used for block_q6_K\ninline void bytes_from_nibbles_128(__m512i& r0, __m512i& r1, const uint8_t * qs, const uint8_t * qh) {\n    const __m256i m4 = _mm256_set1_epi8(0xF);\n    const __m256i m2 = _mm256_set1_epi8(0x3);\n\n    const __m256i q6bits1 = _mm256_loadu_si256((const __m256i *)qs);\n    const __m256i q6bits2 = _mm256_loadu_si256((const __m256i *)(qs + 32));\n    const __m256i q6bitsH = _mm256_loadu_si256((const __m256i *)qh);\n\n    const __m256i q6h_0 = _mm256_slli_epi16(_mm256_and_si256(                  q6bitsH,     m2), 4);\n    const __m256i q6h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 2), m2), 4);\n    const __m256i q6h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 4), m2), 4);\n    const __m256i q6h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q6bitsH, 6), m2), 4);\n\n    const __m256i q6_0 = _mm256_or_si256(_mm256_and_si256(q6bits1, m4), q6h_0);\n    const __m256i q6_1 = _mm256_or_si256(_mm256_and_si256(q6bits2, m4), q6h_1);\n    const __m256i q6_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q6bits1, 4), m4), q6h_2);\n    const __m256i q6_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q6bits2, 4), m4), q6h_3);\n\n    r0 = _mm512_inserti32x8(_mm512_castsi256_si512(q6_0), q6_1, 1);\n    r1 = _mm512_inserti32x8(_mm512_castsi256_si512(q6_2), q6_3, 1);\n}\n\ninline __m512i packNibbles(__m512i r0, __m512i r1) {\n    return _mm512_or_si512(r0, _mm512_slli_epi16(r1, 4));\n}\n\ntemplate <typename TB>\ninline void pack_qs(void * RESTRICT packed_B, const TB * RESTRICT B, int KB) {\n    int8_t tmp[8 * 64];\n    __m256i v[8], v2[8];\n    for (int n = 0; n < 8; ++n) {\n        v[n] = bytes_from_nibbles_32(B[n * KB].qs);\n    }\n    transpose_8x8_32bit(v, v2);\n    for (int n = 0; n < 8; ++n) {\n        _mm256_storeu_si256((__m256i *)(tmp + n * 64), v2[n]);\n    }\n    for (int n = 0; n < 8; ++n) {\n        v[n] = bytes_from_nibbles_32(B[(n + 8) * KB].qs);\n    }\n    transpose_8x8_32bit(v, v2);\n    for (int n = 0; n < 8; ++n) {\n        _mm256_storeu_si256((__m256i *)(tmp + n * 64 + 32), v2[n]);\n    }\n\n    // pack again with 128 to fully utilize vector length\n    for (int n = 0; n < 8; n += 2) {\n        __m512i r0 = _mm512_loadu_si512((const __m512i *)(tmp + n * 64));\n        __m512i r1 = _mm512_loadu_si512((const __m512i *)(tmp + n * 64 + 64));\n        __m512i r1r0 = packNibbles(r0, r1);\n        _mm512_storeu_si512((__m512i *)((char *)packed_B + n * 32), r1r0);\n    }\n}\n\ntemplate <>\ninline void pack_qs<block_q8_0>(void * RESTRICT packed_B, const block_q8_0 * RESTRICT B, int KB) {\n    __m256i v[8], v2[8];\n    for (int n = 0; n < 8; ++n) {\n        v[n] = _mm256_loadu_si256((const __m256i *)(B[n * KB].qs));\n    }\n    transpose_8x8_32bit(v, v2);\n    for (int n = 0; n < 8; ++n) {\n        _mm256_storeu_si256((__m256i *)((char *)packed_B + n * 64), v2[n]);\n    }\n    for (int n = 0; n < 8; ++n) {\n        v[n] = _mm256_loadu_si256((const __m256i *)(B[(n + 8) * KB].qs));\n    }\n    transpose_8x8_32bit(v, v2);\n    for (int n = 0; n < 8; ++n) {\n        _mm256_storeu_si256((__m256i *)((char *)packed_B + n * 64 + 32), v2[n]);\n    }\n}\n\ntemplate <>\ninline void pack_qs<block_q4_K>(void * RESTRICT packed_B, const block_q4_K * RESTRICT B, int KB) {\n    __m512i v[16];\n    // QK_K 256 with 8 groups, handle 2 groups at a time\n    char * pb = (char *)packed_B;\n    for (int k = 0; k < QK_K / 64; ++k) {\n        // pack 2 groups { n, g,  k} to {g, k/4, 4n}\n        //          e.g. {16, 2, 32} to {2,   8, 64}\n        for (int n = 0; n < TILE_N; ++n) {\n            v[n] = bytes_from_nibbles_64(B[n * KB].qs + k * 32);\n        }\n\n        transpose_16x16_32bit(v);\n\n        // pack again with 128 to fully utilize vector length\n        for (int n = 0; n < TILE_N; n += 2) {\n            _mm512_storeu_si512((__m512i *)pb, packNibbles(v[n], v[n + 1]));\n            pb += 64;\n        }\n    }\n}\n\ntemplate <>\ninline void pack_qs<block_q5_K>(void * RESTRICT packed_B, const block_q5_K * RESTRICT B, int KB) {\n    __m512i v[16];\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    // QK_K 256 with 8 groups, handle 2 groups at a time\n    char * pb = (char *)packed_B;\n    char * ph = (char *)packed_B + (QK_K / 2) * TILE_N;\n    for (int k = 0; k < QK_K / 64; ++k) {\n        // pack 2 groups { n, g,  k} to {g, k/4, 4n}\n        //          e.g. {16, 2, 32} to {2,   8, 64}\n        for (int n = 0; n < TILE_N; ++n) {\n            v[n] = bytes_from_nibbles_64(B[n * KB].qs + k * 32, B[n * KB].qh, /* group */2 * k);\n        }\n\n        transpose_16x16_32bit(v);\n\n        // 1. pack lower 4bits with 2 groups\n        for (int n = 0; n < TILE_N; n += 2) {\n            // get lower 4 bits\n            const __m512i r0 = _mm512_and_si512(v[n], lowMask);\n            const __m512i r1 = _mm512_and_si512(v[n + 1], lowMask);\n            _mm512_storeu_si512((__m512i *)pb, packNibbles(r0, r1)); pb += 64;\n        }\n\n        // 2. pack higher 1bit with 2 groups\n        const __m512i hmask = _mm512_set1_epi8(0x10);\n        for (int g = 0; g < 2; ++g) {\n            __m512i hbits = _mm512_setzero_si512();\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 0], hmask), 4));\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 1], hmask), 3));\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 2], hmask), 2));\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 8 + 3], hmask), 1));\n            hbits = _mm512_add_epi8(hbits,                   _mm512_and_si512(v[g * 8 + 4], hmask)    );\n            hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 5], hmask), 1));\n            hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 6], hmask), 2));\n            hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 8 + 7], hmask), 3));\n            _mm512_storeu_si512((__m512i *)ph, hbits); ph += 64;\n        }\n    }\n}\n\ntemplate <>\ninline void pack_qs<block_q6_K>(void * RESTRICT packed_B, const block_q6_K * RESTRICT B, int KB) {\n    __m512i v[32];\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    // QK_K 256 with 8 groups, handle 4 groups at a time\n    char * pb = (char *)packed_B;\n    char * ph = (char *)packed_B + (QK_K / 2) * TILE_N;\n    for (int k = 0; k < QK_K / 128; ++k) {\n        for (int n = 0; n < TILE_N; ++n) {\n            bytes_from_nibbles_128(v[n], v[n + 16], B[n * KB].ql + k * 64, B[n * KB].qh + k * 32);\n        }\n\n        // top half: group 0,1 or 4,5; bottom half: group 2,3 or 6,7\n        transpose_16x16_32bit(v);\n        transpose_16x16_32bit(v + 16);\n\n        // 1. pack lower 4bits with 4 groups\n        for (int n = 0; n < 32; n += 2) {\n            const __m512i r0 = _mm512_and_si512(v[n], lowMask);\n            const __m512i r1 = _mm512_and_si512(v[n + 1], lowMask);\n            _mm512_storeu_si512((__m512i *)pb, packNibbles(r0, r1)); pb += 64;\n        }\n\n        // 2. pack higher 2bit with 4 groups\n        const __m512i hmask = _mm512_set1_epi8(0x30);\n        for (int g = 0; g < 8; ++g) {\n            __m512i hbits = _mm512_setzero_si512();\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 4 + 0], hmask), 4));\n            hbits = _mm512_add_epi8(hbits, _mm512_srli_epi16(_mm512_and_si512(v[g * 4 + 1], hmask), 2));\n            hbits = _mm512_add_epi8(hbits,                   _mm512_and_si512(v[g * 4 + 2], hmask)    );\n            hbits = _mm512_add_epi8(hbits, _mm512_slli_epi16(_mm512_and_si512(v[g * 4 + 3], hmask), 2));\n            _mm512_storeu_si512((__m512i *)ph, hbits); ph += 64;\n        }\n    }\n}\n\ntemplate <>\ninline void pack_qs<block_iq4_xs>(void * RESTRICT packed_B, const block_iq4_xs * RESTRICT B, int KB) {\n    __m512i v[16];\n    char * pb = (char *)packed_B;\n    for (int k = 0; k < QK_K / 64; ++k) {\n        for (int n = 0; n < TILE_N; ++n) {\n            __m256i r0 = bytes_from_nibbles_32(B[n * KB].qs + k * 32 +  0);\n            __m256i r1 = bytes_from_nibbles_32(B[n * KB].qs + k * 32 + 16);\n            v[n] = _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);\n        }\n\n        transpose_16x16_32bit(v);\n\n        // pack again with 128 to fully utilize vector length\n        for (int n = 0; n < TILE_N; n += 2) {\n            _mm512_storeu_si512((__m512i *)pb, packNibbles(v[n], v[n + 1]));\n            pb += 64;\n        }\n    }\n}\n\n// pack B to vnni formats in 4bits or 8 bits\nvoid pack_B(void * RESTRICT packed_B, const block_q4_0 * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n    ggml_half * d0 = reinterpret_cast<ggml_half *>((char *)packed_B + TILE_N * TILE_K / 2);\n    for (int n = 0; n < TILE_N; ++n) {\n        d0[n] = B[n * KB].d;\n    }\n}\n\nvoid pack_B(void * RESTRICT packed_B, const block_q4_1 * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n    ggml_half * d0 = reinterpret_cast<ggml_half *>((char *)packed_B + TILE_N * TILE_K / 2);\n    ggml_half * m0 = d0 + TILE_N;\n    for (int n = 0; n < TILE_N; ++n) {\n        d0[n] = B[n * KB].d;\n        m0[n] = B[n * KB].m;\n    }\n}\n\ninline void s8s8_compensation(void * RESTRICT packed_B) {\n    // packed_B layout:\n    //   quants {TILE_N, TILEK}  int8_t\n    //   d0     {TILE_N}      ggml_half\n    //   comp   {TILE_N}        int32_t\n    const int offset = TILE_N * TILE_K + TILE_N * sizeof(ggml_half);\n    __m512i vcomp = _mm512_setzero_si512();\n    const __m512i off = _mm512_set1_epi8(static_cast<char>(0x80));\n    for (int k = 0; k < 8; ++k) {\n        __m512i vb = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + k * 64));\n        vcomp = _mm512_dpbusd_epi32(vcomp, off, vb);\n    }\n    _mm512_storeu_si512((__m512i *)((char *)(packed_B) + offset), vcomp);\n}\n\nvoid pack_B(void * RESTRICT packed_B, const block_q8_0 * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n    ggml_half * d0 = reinterpret_cast<ggml_half *>((char *)packed_B + TILE_N * TILE_K);\n    for (int n = 0; n < TILE_N; ++n) {\n        d0[n] = B[n * KB].d;\n    }\n    s8s8_compensation(packed_B);\n}\n\n// convert 8 * {min, scale} from int6 to int8\ninline void unpack_mins_and_scales(const uint8_t * scales, uint32_t * utmp) {\n    const uint32_t kmask1 = 0x3f3f3f3f;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n    const uint32_t kmask3 = 0x03030303;\n\n    memcpy(utmp, scales, 12);\n    utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n    const uint32_t uaux = utmp[1] & kmask1;\n    utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n    utmp[2] = uaux;\n    utmp[0] &= kmask1;\n}\n\n// packed_B layout:\n//   quants {8, TILE_N, 16}  uint8\n//   scales {8, TILE_N}      uint8\n//   mins   {8, TILE_N}      uint8\n//   d      {TILE_N}     ggml_half\n//   dmin   {TILE_N}     ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q4_K * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n\n    uint8_t * scales = reinterpret_cast<uint8_t *>((char *)packed_B + (QK_K / 2) * TILE_N);\n    uint8_t * mins = scales + 8 * TILE_N;\n    ggml_half * d = reinterpret_cast<ggml_half *>(mins + 8 * TILE_N);\n    ggml_half * dmin = d + TILE_N;\n\n    union {\n        uint32_t u32[4];\n        uint8_t  u8[16];\n    } s;\n\n    for (int n = 0; n < TILE_N; ++n) {\n        unpack_mins_and_scales(B[n * KB].scales, s.u32);\n        for (int k = 0; k < 8; ++k) {\n            scales[k * TILE_N + n] = s.u8[k];\n            mins[(k >> 1) * TILE_N * 2 + n * 2 + (k & 0x1)] = s.u8[k + 8];\n        }\n        d[n] = B[n * KB].d;\n        dmin[n] = B[n * KB].dmin;\n    }\n}\n\n// packed_B layout:\n//   quants {8, TILE_N, 16}  uint8\n//   qh     {8, TILE_N,  4}  uint8\n//   scales {8, TILE_N}      uint8\n//   mins   {8, TILE_N}      uint8\n//   d      {TILE_N}     ggml_half\n//   dmin   {TILE_N}     ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q5_K * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n\n    uint8_t * scales = reinterpret_cast<uint8_t *>((char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N);\n    uint8_t * mins = scales + 8 * TILE_N;\n    ggml_half * d = reinterpret_cast<ggml_half *>(mins + 8 * TILE_N);\n    ggml_half * dmin = d + TILE_N;\n\n    union {\n        uint32_t u32[4];\n        uint8_t  u8[16];\n    } s;\n\n    for (int n = 0; n < TILE_N; ++n) {\n        unpack_mins_and_scales(B[n * KB].scales, s.u32);\n        for (int k = 0; k < 8; ++k) {\n            scales[k * TILE_N + n] = s.u8[k];\n            mins[(k >> 1) * TILE_N * 2 + n * 2 + (k & 0x1)] = s.u8[k + 8];\n        }\n        d[n] = B[n * KB].d;\n        dmin[n] = B[n * KB].dmin;\n    }\n}\n\n// packed_B layout:\n//   quants {16, TILE_N, 8}  uint8\n//   qh     {16, TILE_N, 4}  uint8\n//   scales {16, TILE_N}      uint8\n//   d      {TILE_N}     ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_q6_K * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n\n    uint8_t * scales = reinterpret_cast<uint8_t *>((char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N);\n    ggml_half * d = reinterpret_cast<ggml_half *>(scales + 16 * TILE_N);\n    for (int n = 0; n < TILE_N; ++n) {\n        const int8_t * ps = B[n * KB].scales;\n        for (int k = 0; k < 16; ++k) {\n            scales[k * TILE_N + n] = ps[k];\n        }\n        d[n] = B[n * KB].d;\n    }\n}\n\n// packed_B layout:\n//   quants {8, TILE_N, 16}  uint8\n//   scales {8, TILE_N}       int8\n//   d      {TILE_N}     ggml_half\nvoid pack_B(void * RESTRICT packed_B, const block_iq4_xs * RESTRICT B, int KB) {\n    pack_qs(packed_B, B, KB);\n\n    int8_t * scales = reinterpret_cast<int8_t *>((char *)packed_B + (QK_K / 2) * TILE_N);\n    ggml_half * d = reinterpret_cast<ggml_half *>(scales + 8 * TILE_N);\n\n    // pack the scales\n    for (int n = 0; n < TILE_N; ++n) {\n        uint16_t sh = B[n * KB].scales_h;\n        for (int k = 0; k < 8; k += 2) {\n            const int16_t ls1 = ((B[n * KB].scales_l[k / 2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n            const int16_t ls2 = ((B[n * KB].scales_l[k / 2] >>  4) | ((sh << 2) & 0x30)) - 32;\n            scales[(k + 0) * TILE_N + n] = ls1;\n            scales[(k + 1) * TILE_N + n] = ls2;\n            sh >>= 4;\n        }\n        d[n] = B[n * KB].d;\n    }\n}\n\ntemplate<typename TB, typename packed_B_t = packed_B_type<TB>>\nvoid unpack_B(packed_B_t * RESTRICT tile, const void * RESTRICT packed_B) {\n    GGML_UNUSED(tile);\n    GGML_UNUSED(packed_B);\n}\n\ntemplate <>\nvoid unpack_B<block_q4_0>(int8_t * RESTRICT tile, const void * RESTRICT packed_B) {\n  const __m512i off = _mm512_set1_epi8(8);\n  const __m512i lowMask = _mm512_set1_epi8(0xF);\n  for (int n = 0; n < 8; n += 2) {\n    __m512i bytes = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + n * 32));\n    const __m512i r0 = _mm512_sub_epi8(_mm512_and_si512(bytes, lowMask), off);\n    const __m512i r1 = _mm512_sub_epi8(_mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask), off);\n    _mm512_storeu_si512((__m512i *)(tile + n * 64 +  0), r0);\n    _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n  }\n}\n\ntemplate <>\nvoid unpack_B<block_q4_1>(uint8_t * RESTRICT tile, const void * RESTRICT packed_B) {\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    for (int n = 0; n < 8; n += 2) {\n        __m512i bytes = _mm512_loadu_si512((const __m512i *)((const char *)packed_B + n * 32));\n        const __m512i r0 = _mm512_and_si512(bytes, lowMask);\n        const __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 +  0), r0);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n    }\n}\n\n// packed_B_t for QKK is int8_t\ntemplate <typename TB>\nvoid unpack_B(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n    const int packed_B_group_size = QK_K / 2 * TILE_N / 8;\n    const char * packed_B_group = (const char *)packed_B + k * packed_B_group_size;\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    for (int n = 0; n < 8; n += 2) {\n        __m512i bytes = _mm512_loadu_si512(packed_B_group + n * 32);\n        const __m512i r0 = _mm512_and_si512(bytes, lowMask);\n        const __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 +  0), r0);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n    }\n}\n\ntemplate <>\nvoid unpack_B<block_q5_K>(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n    // lower 4bits, stride 256 bytes\n    const int packed_l4_group_size = QK_K / 2 * TILE_N / 8;\n    const char * pb = (const char *)packed_B + k * packed_l4_group_size;\n\n    // higher 1bit, stride 64 bytes\n    const int packed_h1_group_size = QK_K / 8 * TILE_N / 8;\n    const char * ph = (const char *)packed_B + (QK_K / 2) * TILE_N + k * packed_h1_group_size;\n    const __m512i hbits = _mm512_loadu_si512(ph);\n\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    __m512i hmask0 = _mm512_set1_epi8(0x1);\n    __m512i hmask1 = _mm512_set1_epi8(0x2);\n\n    for (int n = 0; n < 8; n += 2) {\n        __m512i bytes = _mm512_loadu_si512(pb + n * 32);\n        __m512i r0 = _mm512_and_si512(bytes, lowMask);\n        __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n        __m512i h0 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask0), n), 4);\n        __m512i h1 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), n + 1), 4);\n\n        hmask0 = _mm512_slli_epi16(hmask0, 2);\n        hmask1 = _mm512_slli_epi16(hmask1, 2);\n        r0 = _mm512_add_epi8(r0, h0);\n        r1 = _mm512_add_epi8(r1, h1);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 +  0), r0);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n    }\n}\n\ntemplate <>\nvoid unpack_B<block_q6_K>(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n    // lower 4bits, stride 128 bytes\n    const int packed_l4_group_size = QK_K / 2 * TILE_N / 16;\n    const char * pb = (const char *)packed_B + k * packed_l4_group_size;\n\n    // higher 2bits, stride 64 bytes\n    const int packed_h2_group_size = QK_K / 4 * TILE_N / 16;\n    const char * ph = (const char *)packed_B + (QK_K / 2) * TILE_N + k * packed_h2_group_size;\n    const __m512i hbits = _mm512_loadu_si512(ph);\n\n    const __m512i off = _mm512_set1_epi8(32);\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n    __m512i hmask0 = _mm512_set1_epi8(0x3); // 0011\n    __m512i hmask1 = _mm512_set1_epi8(0xC); // 1100\n\n    // notes: skip zero padding from row4 to row7 as we have done so in `unpack_A`\n    __m512i bytes = _mm512_loadu_si512(pb);\n    __m512i r0 = _mm512_and_si512(bytes, lowMask);\n    __m512i r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n    __m512i h0 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask0), 4);\n    __m512i h1 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask1), 2);\n    _mm512_storeu_si512((__m512i *)(tile +  0), _mm512_sub_epi8(_mm512_add_epi8(r0, h0), off));\n    _mm512_storeu_si512((__m512i *)(tile + 64), _mm512_sub_epi8(_mm512_add_epi8(r1, h1), off));\n\n    hmask0 = _mm512_slli_epi16(hmask0, 4);\n    hmask1 = _mm512_slli_epi16(hmask1, 4);\n\n    bytes = _mm512_loadu_si512(pb + 64);\n    r0 = _mm512_and_si512(bytes, lowMask);\n    r1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n    h0 =                   _mm512_and_si512(hbits, hmask0);\n    h1 = _mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), 2);\n    _mm512_storeu_si512((__m512i *)(tile + 128), _mm512_sub_epi8(_mm512_add_epi8(r0, h0), off));\n    _mm512_storeu_si512((__m512i *)(tile + 192), _mm512_sub_epi8(_mm512_add_epi8(r1, h1), off));\n}\n\ntemplate <>\nvoid unpack_B<block_iq4_xs>(int8_t * RESTRICT tile, const void * RESTRICT packed_B, int k) {\n    static const __m512i values128 = _mm512_set_epi8(\n        113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n        113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n        113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n        113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127\n    );\n\n    const int packed_B_group_size = QK_K / 2 * TILE_N / 8;\n    const char * pb = (const char *)packed_B + k * packed_B_group_size;\n    const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n    for (int n = 0; n < 8; n += 2) {\n        __m512i bytes = _mm512_loadu_si512(pb + n * 32);\n        const __m512i r0 = _mm512_shuffle_epi8(values128, _mm512_and_si512(bytes, lowMask));\n        const __m512i r1 = _mm512_shuffle_epi8(values128, _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask));\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 +  0), r0);\n        _mm512_storeu_si512((__m512i *)(tile + n * 64 + 64), r1);\n    }\n}\n\ntemplate <typename TA, typename TB, bool is_acc>\nstruct acc_C {};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_0, block_q4_0, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_0 * A, int lda, const void * packed_B, int nr) {\n        const int offset = TILE_N * TILE_K / 2;\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n\n        for (int m = 0; m < nr; ++m) {\n            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n            vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_1, block_q4_1, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_1 * A, int lda, const void * packed_B, int nr) {\n        const int offset = TILE_N * TILE_K / 2;\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n        const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset + TILE_N * sizeof(ggml_half))));\n\n        for (int m = 0; m < nr; ++m) {\n            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n            const __m512 vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].s));\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n            vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n            vsum = _mm512_fmadd_ps(vm0, vs1, vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_0, block_q8_0, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_0 * A, int lda, const void * packed_B, int nr) {\n        const int offset = TILE_N * TILE_K;\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)((const char *)packed_B + offset)));\n\n        for (int m = 0; m < nr; ++m) {\n            const __m512 vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[m * lda].d));\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n            vsum = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_K, block_q4_K, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n        const uint8_t * scales = reinterpret_cast<const uint8_t *>((const char *)packed_B + (QK_K / 2) * TILE_N);\n        const uint8_t * mins = scales + 8 * TILE_N;\n        const ggml_half * d0 = reinterpret_cast<const ggml_half *>(mins + 8 * TILE_N);\n        const ggml_half * dmin = d0 + TILE_N;\n\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n        const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)dmin));\n\n        for (int m = 0; m < nr; ++m) {\n            const float d1 = A[m * lda].d;\n            const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n            const __m512 vdm = _mm512_mul_ps(_mm512_set1_ps(-d1), vdmin);\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n\n            const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[m * lda].bsums);\n            const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n\n            __m512i acc_m = _mm512_setzero_si512();\n            for (int k = 0; k < 4; ++k) {\n                __m512i vmask = _mm512_set1_epi32(k);\n                __m512i va = _mm512_permutexvar_epi32(vmask, _mm512_castsi128_si512(q8s));\n                __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(mins + k * 32)));\n                acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n            }\n\n            vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n            vsum = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc_m), vdm, vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_K, block_q5_K, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n        const uint8_t * scales = reinterpret_cast<const uint8_t *>((const char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N);\n        const uint8_t * mins = scales + 8 * TILE_N;\n        const ggml_half * d0 = reinterpret_cast<const ggml_half *>(mins + 8 * TILE_N);\n        const ggml_half * dmin = d0 + TILE_N;\n\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n        const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)dmin));\n\n        for (int m = 0; m < nr; ++m) {\n            const float d1 = A[m * lda].d;\n            const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n            const __m512 vdm = _mm512_mul_ps(_mm512_set1_ps(-d1), vdmin);\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n\n            const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[m * lda].bsums);\n            const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n\n            __m512i acc_m = _mm512_setzero_si512();\n            for (int k = 0; k < 4; ++k) {\n                __m512i vmask = _mm512_set1_epi32(k);\n                __m512i va = _mm512_permutexvar_epi32(vmask, _mm512_castsi128_si512(q8s));\n                __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(mins + k * 32)));\n                acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n            }\n\n            vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n            vsum = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc_m), vdm, vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_K, block_q6_K, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n        const uint8_t * scales = reinterpret_cast<const uint8_t *>((const char *)packed_B + (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N);\n        const ggml_half * d0 = reinterpret_cast<const ggml_half *>(scales + 16 * TILE_N);\n\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n\n        for (int m = 0; m < nr; ++m) {\n            const float d1 = A[m * lda].d;\n            const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n\n            vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <bool is_acc>\nstruct acc_C<block_q8_K, block_iq4_xs, is_acc> {\n    static void apply(float * RESTRICT C, int ldc, const int32_t * RESTRICT tile, const block_q8_K * A, int lda, const void * packed_B, int nr) {\n        const int8_t * scales = reinterpret_cast<const int8_t *>((const char *)packed_B + (QK_K / 2) * TILE_N);\n        const ggml_half * d0 = reinterpret_cast<const ggml_half *>(scales + 8 * TILE_N);\n\n        const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)d0));\n\n        for (int m = 0; m < nr; ++m) {\n            const float d1 = A[m * lda].d;\n            const __m512 vd = _mm512_mul_ps(_mm512_set1_ps(d1), vd0);\n            const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));\n\n            __m512 vsum;\n            if (is_acc) {\n                vsum = _mm512_loadu_ps(C + m * ldc);\n            } else {\n                vsum = _mm512_set1_ps(0.f);\n            }\n\n            vsum = _mm512_fmadd_ps(vtile, vd, vsum);\n            _mm512_storeu_ps(C + m * ldc, vsum);\n        }\n    }\n};\n\ntemplate <typename TB> constexpr int get_quants_size();\ntemplate <> constexpr int get_quants_size<block_q4_K>() { return (QK_K / 2) * TILE_N; }\ntemplate <> constexpr int get_quants_size<block_q5_K>() { return (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N; }\ntemplate <> constexpr int get_quants_size<block_q6_K>() { return (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N; }\ntemplate <> constexpr int get_quants_size<block_iq4_xs>() { return (QK_K / 2) * TILE_N; }\n\n// used for QKK format\ntemplate <typename TB, bool is_acc,\n          typename std::enable_if<is_type_qkk<TB>::value, int>::type = 0>\ninline void scale_C(const int32_t * RESTRICT tile, int32_t * RESTRICT sumi, const void * packed_B, int k, int nr) {\n    const uint8_t * scales = reinterpret_cast<const uint8_t *>((const char *)packed_B + get_quants_size<TB>());\n    const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(scales + k * TILE_N)));\n\n    for (int m = 0; m < nr; ++m) {\n        __m512i vsumi;\n        if (is_acc) {\n            vsumi = _mm512_loadu_si512(sumi + m * TILE_N);\n        } else {\n            vsumi = _mm512_setzero_si512();\n        }\n        __m512i vtile = _mm512_loadu_si512(tile + m * TILE_N);\n        vsumi = _mm512_add_epi32(vsumi, _mm512_mullo_epi32(vtile, vscale));\n        _mm512_storeu_si512((__m512i *)(sumi + m * TILE_N), vsumi);\n    }\n}\n\ntemplate <typename TA, typename TB, typename TC, int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_avx {\n    static void apply(int K, const TA * RESTRICT A, const TB * RESTRICT B, TC * RESTRICT C, int ldc) {\n        GGML_UNUSED(K);\n        GGML_UNUSED(A);\n        GGML_UNUSED(B);\n        GGML_UNUSED(C);\n        GGML_UNUSED(ldc);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_avx<float, ggml_fp16_t, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int K, const float * RESTRICT A, const ggml_fp16_t * RESTRICT B, float * RESTRICT C, int ldc) {\n        constexpr int ROWS = BLOCK_M;\n        constexpr int COLS = BLOCK_N;\n        assert(BLOCK_K == 16);\n\n        __m512 va;\n        __m512 vb[COLS];\n        __m512 vc[ROWS * COLS];\n\n        auto loadc = [&](auto idx) {\n            vc[idx] = _mm512_setzero_ps();\n        };\n        Unroll<ROWS * COLS>{}(loadc);\n\n        auto compute = [&](auto idx, auto k) {\n            constexpr int row = idx / COLS;\n            constexpr int col = idx % COLS;\n\n            if constexpr (col == 0) {\n                va = _mm512_loadu_ps(A + row * K + k);\n            }\n            if constexpr (row == 0) {\n                vb[col] =  _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(B + col * K + k)));\n            }\n            vc[idx] = _mm512_fmadd_ps(va, vb[col], vc[idx]);\n        };\n\n        for (int k = 0; k < K; k += 16) {\n            Unroll<ROWS * COLS>{}(compute, k);\n        }\n\n        auto storec = [&](auto idx) {\n            constexpr int row = idx / COLS;\n            constexpr int col = idx % COLS;\n            C[row * ldc + col] = _mm512_reduce_add_ps(vc[idx]);\n        };\n        Unroll<ROWS * COLS>{}(storec);\n    }\n};\n\n#define LAUNCH_TINYGEMM_KERNEL_AVX(MB_SIZE, NB_SIZE)                                \\\n    tinygemm_kernel_avx<float, type, float, MB_SIZE, NB_SIZE, blck_size>::apply(    \\\n        K, (const float *)src1->data + src1_offset + mb_start * K,                  \\\n        (const type *)src0->data + src0_offset + nb_start * K,                      \\\n        (float *)dst->data + dst_offset + mb_start * ldc + nb_start, ldc)\n\n\n// re-organize in the format {NB, KB, TILE_SIZE}:\n#define PACKED_INDEX(n, k, KB, tile_size) (n * KB + k) * tile_size\n\ntemplate<typename TB, int BLOCK_K>\nvoid convert_B_packed_format(void * RESTRICT packed_B, const TB * RESTRICT B, int N, int K) {\n    const int NB = N / TILE_N;\n    const int KB = K / BLOCK_K;\n    const int TILE_SIZE = get_tile_size<TB>();\n\n    // parallel on NB should be enough\n    parallel_for(NB, [&](int begin, int end) {\n        for (int n = begin; n < end; ++n) {\n            for (int k = 0; k < KB; ++k) {\n                int n0 = n * TILE_N;\n                pack_B((char *)packed_B + PACKED_INDEX(n, k, KB, TILE_SIZE), &B[n0 * KB + k], KB);\n            }\n        }\n    });\n}\n\ntemplate <typename TA, typename TB, typename TC, int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni {};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_0, block_q4_0, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q4_0);\n\n        const block_q8_0 * RESTRICT A = static_cast<const block_q8_0 *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        __m512i va[8];\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // sum of offsets, shared across COLS\n        //\n        // avx512-vnni does not have `_mm512_dpbssd_epi32`,\n        // need to transform ss to us:\n        //   a * (b - 8) is equivalent to b * a - 8 * a\n        //   s    u   u                   u   s   u   s\n        //\n        __m512i vcomp;\n\n        const __m512i off = _mm512_set1_epi8(8);\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        auto compute = [&](auto col, auto i) {\n            // load a and compute compensation\n            if constexpr (col == 0) {\n                const int32_t * a_ptr = reinterpret_cast<const int32_t *>(A[0 * KB + i].qs);\n                vcomp = _mm512_setzero_si512();\n                for (int k = 0; k < 8; ++k) {\n                    va[k] = _mm512_set1_epi32(a_ptr[k]);\n                    vcomp = _mm512_dpbusd_epi32(vcomp, off, va[k]);\n                }\n                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n            }\n\n            // load b\n            __m512i vsum = _mm512_setzero_si512();\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            for (int k = 0; k < 8; k += 2) {\n                __m512i bytes = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 32));\n                __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n                vsum = _mm512_dpbusd_epi32(vsum, vb0, va[k + 0]);\n                __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n                vsum = _mm512_dpbusd_epi32(vsum, vb1, va[k + 1]);\n            }\n            const int offset = TILE_N * TILE_K / 2;\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n            vsum = _mm512_sub_epi32(vsum, vcomp);\n\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_1, block_q4_1, float, 1, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q4_1);\n\n        const block_q8_1 * RESTRICT A = static_cast<const block_q8_1 *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        __m512i va[8];\n        __m512i vb[8];\n        __m512 vc[COLS];\n        __m512 vd1, vs1;\n\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        auto compute = [&](auto col, auto i) {\n            // load a\n            if constexpr (col == 0) {\n                const int32_t * a_ptr = reinterpret_cast<const int32_t *>(A[0 * KB + i].qs);\n                for (int k = 0; k < 8; ++k) {\n                    va[k] = _mm512_set1_epi32(a_ptr[k]);\n                }\n                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n                vs1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].s));\n            }\n\n            // load b\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            for (int k = 0; k < 8; k += 2) {\n                __m512i bytes = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 32));\n                vb[k + 0] = _mm512_and_si512(bytes, lowMask);\n                vb[k + 1] = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n            }\n            const int offset = TILE_N * TILE_K / 2;\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n            const __m512 vm0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset + TILE_N * sizeof(ggml_half))));\n\n            __m512i vsum = _mm512_setzero_si512();\n            for (int k = 0; k < 8; ++k) {\n                vsum = _mm512_dpbusd_epi32(vsum, vb[k], va[k]);\n            }\n\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n            vc[col] = _mm512_fmadd_ps(vm0, vs1, vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_0, block_q8_0, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q8_0) + TILE_N * sizeof(int32_t);\n\n        const block_q8_0 * RESTRICT A = static_cast<const block_q8_0 *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        __m512i va[8];\n        __m512i vb[8];\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // Notes: s8s8 igemm compensation in avx512-vnni\n        // change s8s8 to u8s8 with compensate\n        //   a * b = (a + 128) * b - 128 * b\n        //   s   s       u       s    u    s\n        //\n        // (128 * b is pre-computed when packing B to vnni formats)\n        //\n        const __m512i off = _mm512_set1_epi8(static_cast<char>(0x80));\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        auto compute = [&](auto col, auto i) {\n            // load a and add offset 128\n            if constexpr (col == 0) {\n                const int32_t * a_ptr = reinterpret_cast<const int32_t *>(A[0 * KB + i].qs);\n                for (int k = 0; k < 8; ++k) {\n                    va[k] = _mm512_set1_epi32(a_ptr[k]);\n                    va[k] = _mm512_add_epi8(va[k], off);\n                }\n                vd1 = _mm512_set1_ps(GGML_CPU_FP16_TO_FP32(A[0 * KB + i].d));\n            }\n\n            // load b\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            for (int k = 0; k < 8; ++k) {\n                vb[k] = _mm512_loadu_si512((const __m512i *)(b_ptr + k * 64));\n            }\n            const int offset = TILE_N * TILE_K;\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset)));\n            const int offset2 = TILE_N * TILE_K + TILE_N * sizeof(ggml_half);\n            const __m512i vcomp = _mm512_loadu_si512((const __m512i *)(b_ptr + offset2));\n\n            __m512i vsum = _mm512_setzero_si512();\n            for (int k = 0; k < 8; ++k) {\n                vsum = _mm512_dpbusd_epi32(vsum, va[k], vb[k]);\n            }\n            vsum = _mm512_sub_epi32(vsum, vcomp);\n\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(vsum), _mm512_mul_ps(vd0, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_K, block_q4_K, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q4_K) + TILE_N * 4;\n\n        const block_q8_K * RESTRICT A = static_cast<const block_q8_K *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        // a.qs:   8 groups, 32 bytes each group (m256i)\n        __m512i va[8];\n        // a.bsum: 8 groups,  2 bytes each group (m128i)\n        __m512i va_bsum;\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // packed_B:\n        const int offset_scales = (QK_K / 2) * TILE_N;\n        const int offset_mins   = (QK_K / 2) * TILE_N +  8 * TILE_N;\n        const int offset_d0     = (QK_K / 2) * TILE_N + 16 * TILE_N;\n        const int offset_dmin   = (QK_K / 2) * TILE_N + 16 * TILE_N + TILE_N * sizeof(ggml_half);\n\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        // Notes: vnni formats in QK_K\n        //   a) quants vnni format\n        //     int8  {k/4, n, 4}, viewed as 2d {k/4, 4n}, k = 32\n        //     from {16, 32} to {8, 64}\n        //\n        //   b) min vnni format\n        //     int16 {k/2, n, 2}, viewed as 2d {k/2, 2n}, k = 8\n        //     from {16,  8} to {4, 32}\n        //\n        auto compute = [&](auto col, auto i) {\n            // load a\n            if constexpr (col == 0) {\n                for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n                    va[k_group] = _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)(A[0 * KB + i].qs + k_group * 32)));\n                }\n                const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n                const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n                va_bsum = _mm512_castsi128_si512(q8s);\n                vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n            }\n\n            // step 1: accumultate the quants\n            __m512i acc = _mm512_setzero_si512();\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            const char * b_qs  = b_ptr;\n            for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n                __m512i vsum = _mm512_setzero_si512();\n                for (int k = 0; k < 8; k += 2) {\n                    __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 0), va[k_group]);\n                    __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 1), va[k_group]);\n\n                    __m512i bytes = _mm512_loadu_si512((const __m512i *)b_qs);\n                    __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n                    vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n                    __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n                    vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n\n                    b_qs += 64;\n                }\n                // vacc += scale * (q8 @ q4)\n                const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n                acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n            }\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n\n            // step 2: accumulate the mins\n            __m512i acc_m = _mm512_setzero_si512();\n            for (int k = 0; k < 4; ++k) {\n                __m512i vmask = _mm512_set1_epi32(k);\n                __m512i va = _mm512_permutexvar_epi32(vmask, va_bsum);\n                __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_mins + k * 32)));\n                acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n            }\n            const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_dmin)));\n            vc[col] = _mm512_fnmadd_ps(_mm512_cvtepi32_ps(acc_m), _mm512_mul_ps(vdmin, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_K, block_q5_K, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q5_K) + TILE_N * 4;\n\n        const block_q8_K * RESTRICT A = static_cast<const block_q8_K *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        // a.qs:   8 groups, 32 bytes each group (m256i)\n        __m512i va[8];\n        // a.bsum: 8 groups,  2 bytes each group (m128i)\n        __m512i va_bsum;\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // packed_B:\n        const int offset_qh     = (QK_K / 2) * TILE_N;\n        const int offset_scales = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N;\n        const int offset_mins   = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N +  8 * TILE_N;\n        const int offset_d0     = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N + 16 * TILE_N;\n        const int offset_dmin   = (QK_K / 2) * TILE_N + (QK_K / 8) * TILE_N + 16 * TILE_N + TILE_N * sizeof(ggml_half);\n\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        // Q5_K and Q4_K shares the same vnni formats, refer to notes above.\n        auto compute = [&](auto col, auto i) {\n            // load a\n            if constexpr (col == 0) {\n                for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n                    va[k_group] = _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)(A[0 * KB + i].qs + k_group * 32)));\n                }\n                const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n                const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n                va_bsum = _mm512_castsi128_si512(q8s);\n                vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n            }\n\n            // step 1: accumultate the quants\n            __m512i acc = _mm512_setzero_si512();\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            const char * b_qs  = b_ptr;\n            const char * b_qh  = b_ptr + offset_qh;\n            for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n                __m512i vsum = _mm512_setzero_si512();\n                __m512i hmask0 = _mm512_set1_epi8(0x1);\n                __m512i hmask1 = _mm512_set1_epi8(0x2);\n                __m512i hbits = _mm512_loadu_si512((const __m512i *)(b_qh + k_group * 64));\n                for (int k = 0; k < 8; k += 2) {\n                    __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 0), va[k_group]);\n                    __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(k + 1), va[k_group]);\n\n                    __m512i bytes = _mm512_loadu_si512((const __m512i *)b_qs);\n                    __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n                    __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n\n                    __m512i vh0 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask0), k), 4);\n                    __m512i vh1 = _mm512_slli_epi16(_mm512_srli_epi16(_mm512_and_si512(hbits, hmask1), k + 1), 4);\n\n                    hmask0 = _mm512_slli_epi16(hmask0, 2);\n                    hmask1 = _mm512_slli_epi16(hmask1, 2);\n                    vb0 = _mm512_add_epi8(vb0, vh0);\n                    vb1 = _mm512_add_epi8(vb1, vh1);\n\n                    vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n                    vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n\n                    b_qs += 64;\n                }\n                // vacc += scale * (q8 @ q5)\n                const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n                acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n            }\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n\n            // step 2: accumulate the mins\n            __m512i acc_m = _mm512_setzero_si512();\n            for (int k = 0; k < 4; ++k) {\n                __m512i vmask = _mm512_set1_epi32(k);\n                __m512i va = _mm512_permutexvar_epi32(vmask, va_bsum);\n                __m512i vb = _mm512_cvtepi8_epi16(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_mins + k * 32)));\n                acc_m = _mm512_dpwssds_epi32(acc_m, va, vb);\n            }\n            const __m512 vdmin = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_dmin)));\n            vc[col] = _mm512_fnmadd_ps(_mm512_cvtepi32_ps(acc_m), _mm512_mul_ps(vdmin, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_K, block_q6_K, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_q6_K);\n\n        const block_q8_K * RESTRICT A = static_cast<const block_q8_K *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        // load the 256 bytes from A to 4 avx512 vectors\n        __m512i va[4];\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // packed_B:\n        const int offset_qh     = (QK_K / 2) * TILE_N;\n        const int offset_scales = (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N;\n        const int offset_d0     = (QK_K / 2) * TILE_N + (QK_K / 4) * TILE_N + 16 * TILE_N;\n\n        // compensation\n        __m512i vcomp;\n\n        const __m512i m32s = _mm512_set1_epi32(32);\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        auto compute = [&](auto col, auto i) {\n            if constexpr (col == 0) {\n                // load a\n                va[0] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs +   0));\n                va[1] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs +  64));\n                va[2] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 128));\n                va[3] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 192));\n\n                const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n                vcomp = _mm512_mullo_epi32(_mm512_cvtepi16_epi32(q8sums), m32s);\n                vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n            }\n\n            // accmulate the quants\n            __m512i acc = _mm512_setzero_si512();\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            const char * b_qs = b_ptr;\n            const char * b_qh = b_ptr + offset_qh;\n            int mask = 0;\n            for (int k_group = 0; k_group < QK_K / 16; ++k_group) {\n                int r = k_group >> 2;\n                __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n                __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n                __m512i vsum = _mm512_setzero_si512();\n                __m512i hmask = _mm512_set1_epi8(0x3);\n\n                __m512i bytes = _mm512_loadu_si512(b_qs);\n                __m512i hbits = _mm512_loadu_si512(b_qh);\n                __m512i vb0 = _mm512_and_si512(bytes, lowMask);\n                __m512i vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n                __m512i vh0 = _mm512_slli_epi16(_mm512_and_si512(hbits, hmask), 4);\n                __m512i vh1 = _mm512_slli_epi16(_mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 2)), 2);\n\n                vb0 = _mm512_add_epi8(vb0, vh0);\n                vb1 = _mm512_add_epi8(vb1, vh1);\n                vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n                vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n                b_qs += 64;\n\n                va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n                va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n                bytes = _mm512_loadu_si512(b_qs);\n                vb0 = _mm512_and_si512(bytes, lowMask);\n                vb1 = _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask);\n                vh0 =                   _mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 4));\n                vh1 = _mm512_srli_epi16(_mm512_and_si512(hbits, _mm512_slli_epi16(hmask, 6)), 2);\n                vb0 = _mm512_add_epi8(vb0, vh0);\n                vb1 = _mm512_add_epi8(vb1, vh1);\n                vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n                vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n                b_qs += 64;\n                b_qh += 64;\n\n                // B * A - 32 * A\n                __m512i vmask = _mm512_set1_epi32(k_group);\n                vsum = _mm512_sub_epi32(vsum, _mm512_permutexvar_epi32(vmask, vcomp));\n\n                // vacc += scale * (q8 @ q6)\n                const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n                acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n            }\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](int col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\ntemplate <int BLOCK_M, int BLOCK_N, int BLOCK_K>\nstruct tinygemm_kernel_vnni<block_q8_K, block_iq4_xs, float, BLOCK_M, BLOCK_N, BLOCK_K> {\n    static void apply(int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n\n        constexpr int COLS = BLOCK_N / 16;\n        const int TILE_SIZE = TILE_N * sizeof(block_iq4_xs) + TILE_N * 2;\n\n        const block_q8_K * RESTRICT A = static_cast<const block_q8_K *>(_A);\n        const char * RESTRICT B = static_cast<const char *>(_B);\n\n        // load the 256 bytes from A to 4 avx512 vectors\n        __m512i va[4];\n        __m512 vc[COLS];\n        __m512 vd1;\n\n        // packed_B:\n        const int offset_scales = (QK_K / 2) * TILE_N ;\n        const int offset_d0     = (QK_K / 2) * TILE_N + 8 * TILE_N;\n\n        // compensation\n        __m512i vcomp;\n\n        const __m256i m128s = _mm256_set1_epi16(128);\n        const __m512i lowMask = _mm512_set1_epi8(0xF);\n\n        const __m512i values128 = _mm512_set_epi8(\n            113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n            113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n            113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127,\n            113, 89, 69, 53, 38, 25, 13, 1, -10, -22, -35, -49, -65, -83, -104, -127\n        );\n        const __m512i off = _mm512_set1_epi8(static_cast<char>(0x80));\n        const __m512i values256 = _mm512_add_epi8(values128, off);\n\n        auto loadc = [&](auto col) {\n            vc[col] = _mm512_setzero_ps();\n        };\n        Unroll<COLS>{}(loadc);\n\n        auto compute = [&](auto col, auto i) {\n            if constexpr (col == 0) {\n                // load a\n                va[0] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs +   0));\n                va[1] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs +  64));\n                va[2] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 128));\n                va[3] = _mm512_loadu_si512((const __m512i *)(A[0 * KB + i].qs + 192));\n\n                // compensation: 128 * A\n                const __m256i q8sums = _mm256_loadu_si256((const __m256i *)A[0 * KB + i].bsums);\n                vcomp = _mm512_castsi256_si512(_mm256_madd_epi16(q8sums, m128s));\n                vd1 = _mm512_set1_ps(A[0 * KB + i].d);\n            }\n\n            // accmulate the quants\n            __m512i acc = _mm512_setzero_si512();\n            const char * b_ptr = B + PACKED_INDEX(col, i, KB, TILE_SIZE);\n            const char * b_qs = b_ptr;\n            int mask = 0;\n            for (int k_group = 0; k_group < QK_K / 32; ++k_group) {\n                int r = k_group >> 1;\n                __m512i vmask = _mm512_set1_epi32(k_group);\n                __m512i vsum = _mm512_setzero_si512();\n                for (int k = 0; k < 8; k += 2) {\n                    __m512i va0 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n                    __m512i va1 = _mm512_permutexvar_epi32(_mm512_set1_epi32(mask++), va[r]);\n\n                    __m512i bytes = _mm512_loadu_si512(b_qs);\n                    __m512i vb0 = _mm512_shuffle_epi8(values256, _mm512_and_si512(bytes, lowMask));\n                    __m512i vb1 = _mm512_shuffle_epi8(values256, _mm512_and_si512(_mm512_srli_epi16(bytes, 4), lowMask));\n\n                    vsum = _mm512_dpbusd_epi32(vsum, vb0, va0);\n                    vsum = _mm512_dpbusd_epi32(vsum, vb1, va1);\n                    b_qs += 64;\n                }\n                // (B + 128) * A - 128 * A\n                vsum = _mm512_sub_epi32(vsum, _mm512_permutexvar_epi32(vmask, vcomp));\n\n                // vacc += scale * (q8 @ q4)\n                const __m512i vscale = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i *)(b_ptr + offset_scales + k_group * TILE_N)));\n                acc = _mm512_add_epi32(acc, _mm512_mullo_epi32(vsum, vscale));\n            }\n            const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(b_ptr + offset_d0)));\n            vc[col] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(acc), _mm512_mul_ps(vd0, vd1), vc[col]);\n        };\n\n        for (int i = 0; i < KB; ++i) {\n            Unroll<COLS>{}(compute, i);\n        }\n\n        //store to C\n        auto storec = [&](auto col) {\n            _mm512_storeu_ps((__m512i*)(C + 0 * ldc + col * 16), vc[col]);\n        };\n        Unroll<COLS>{}(storec);\n    }\n};\n\n#define LAUNCH_TINYGEMM_KERNEL_VNNI(NB_SIZE)                                                   \\\n    tinygemm_kernel_vnni<vec_dot_type, type, float, 1, NB_SIZE, blck_size>::apply(             \\\n        KB, wdata_batch,                                                                       \\\n        (const char *)src0->data + src0_offset + PACKED_INDEX(nb * kTilesN, 0, KB, TILE_SIZE), \\\n        (float *) dst->data + dst_offset + nb_start, ldc)\n\ntemplate <typename TA, typename TB, typename TC, int BLOCK_K,\n          typename std::enable_if<!is_type_qkk<TB>::value, int>::type = 0>\nvoid tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const void * RESTRICT _B, TC * RESTRICT C, int ldc) {\n    using packed_B_t = packed_B_type<TB>;\n    const int TILE_SIZE = get_tile_size<TB>();\n    const bool need_unpack = do_unpack<TB>::value;\n\n    GGML_ASSERT(M <= 2 * TILE_M && N == 2 * TILE_N);\n    const TA * RESTRICT A = static_cast<const TA *>(_A);\n    const char * RESTRICT B = static_cast<const char *>(_B);\n\n    const int m0 = std::min(M, TILE_M);\n    const int m1 = std::max(M - TILE_M, 0);\n    const int lda = KB * sizeof(TA);\n    //const int ldb = KB * sizeof(TB);\n\n    static thread_local packed_B_t Tile0[TILE_N * TILE_K];\n    static thread_local packed_B_t Tile1[TILE_N * TILE_K];\n    static thread_local int8_t Tile23[TILE_M * TILE_K];\n\n    static thread_local int32_t TileC0[TILE_M * TILE_N * 4];\n    static thread_local int32_t TileC1[TILE_M * TILE_N * 4];\n\n    // double buffering C to interleave avx512 and amx\n    int32_t * C_cur = TileC0;\n    int32_t * C_pre = TileC1;\n\n    auto Tile4 = [&](int32_t * base) { return base; };\n    auto Tile5 = [&](int32_t * base) { return base + TILE_M * TILE_N; };\n    auto Tile6 = [&](int32_t * base) { return base + 2 * TILE_M * TILE_N; };\n    auto Tile7 = [&](int32_t * base) { return base + 3 * TILE_M * TILE_N; };\n\n    if (M == 2 * TILE_M) {\n        // i = 0\n        const char * B_blk0 = B + PACKED_INDEX(0, 0, KB, TILE_SIZE);\n        const char * B_blk1 = B + PACKED_INDEX(1, 0, KB, TILE_SIZE);\n        if (need_unpack) {\n            unpack_B<TB>(Tile0, B_blk0);\n            _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n        } else {\n            _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n        }\n\n        _tile_zero(TMM4);\n        _tile_loadd(TMM2, A[0].qs, lda);\n        _tile_dpbssd(TMM4, TMM2, TMM0);\n        _tile_stored(TMM4, Tile4(C_pre), TILE_N * sizeof(int32_t));\n\n        _tile_zero(TMM5);\n        _tile_loadd(TMM3, A[TILE_M * KB + 0].qs, lda);\n        _tile_dpbssd(TMM5, TMM3, TMM0);\n        _tile_stored(TMM5, Tile5(C_pre), TILE_N * sizeof(int32_t));\n\n        if (need_unpack) {\n            unpack_B<TB>(Tile1, B_blk1);\n            _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n        } else {\n            _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n        }\n\n        _tile_zero(TMM6);\n        _tile_dpbssd(TMM6, TMM2, TMM1);\n        _tile_stored(TMM6, Tile6(C_pre), TILE_N * sizeof(int32_t));\n\n        _tile_zero(TMM7);\n        _tile_dpbssd(TMM7, TMM3, TMM1);\n        _tile_stored(TMM7, Tile7(C_pre), TILE_N * sizeof(int32_t));\n\n        for (int i = 1; i < KB; ++i) {\n            // index of previous iter\n            const int ii = i - 1;\n            const char * B_blk0 = B + PACKED_INDEX(0, i, KB, TILE_SIZE);\n            const char * B_blk1 = B + PACKED_INDEX(1, i, KB, TILE_SIZE);\n            GGML_DISPATCH_BOOL(ii > 0, is_acc, [&] {\n                if (need_unpack) {\n                    unpack_B<TB>(Tile0, B_blk0);\n                    _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n                } else {\n                    _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n                }\n                _tile_zero(TMM4);\n                _tile_loadd(TMM2, A[i].qs, lda);\n                acc_C<TA, TB, is_acc>::apply(C, ldc, Tile4(C_pre), &A[ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n\n                _tile_dpbssd(TMM4, TMM2, TMM0);\n                _tile_stored(TMM4, Tile4(C_cur), TILE_N * sizeof(int32_t));\n\n                _tile_zero(TMM5);\n                _tile_loadd(TMM3, A[TILE_M * KB + i].qs, lda);\n                acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc, ldc, Tile5(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n\n                _tile_dpbssd(TMM5, TMM3, TMM0);\n                _tile_stored(TMM5, Tile5(C_cur), TILE_N * sizeof(int32_t));\n\n                if (need_unpack) {\n                    unpack_B<TB>(Tile1, B_blk1);\n                    _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n                } else {\n                    _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n                }\n                _tile_zero(TMM6);\n                acc_C<TA, TB, is_acc>::apply(C + TILE_N, ldc, Tile6(C_pre), &A[ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n\n                _tile_dpbssd(TMM6, TMM2, TMM1);\n                _tile_stored(TMM6, Tile6(C_cur), TILE_N * sizeof(int32_t));\n\n                _tile_zero(TMM7);\n                acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n\n                _tile_dpbssd(TMM7, TMM3, TMM1);\n                _tile_stored(TMM7, Tile7(C_cur), TILE_N * sizeof(int32_t));\n\n                std::swap(C_cur, C_pre);\n            });\n        }\n        // final accumulation\n        {\n            int ii = KB - 1;\n            acc_C<TA, TB, true>::apply(C, ldc, Tile4(C_pre), &A[ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n            acc_C<TA, TB, true>::apply(C + TILE_M * ldc, ldc, Tile5(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(0, ii, KB, TILE_SIZE), TILE_M);\n            acc_C<TA, TB, true>::apply(C + TILE_N, ldc, Tile6(C_pre), &A[ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n            acc_C<TA, TB, true>::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_pre), &A[TILE_M * KB + ii], KB, B + PACKED_INDEX(1, ii, KB, TILE_SIZE), TILE_M);\n        }\n    } else {\n        for (int i = 0; i < KB; ++i) {\n            _tile_zero(TMM4);\n            _tile_zero(TMM6);\n            if (m1 != 0) {\n                _tile_zero(TMM5);\n                _tile_zero(TMM7);\n            }\n\n            const char * B_blk0 = B + PACKED_INDEX(0, i, KB, TILE_SIZE);\n            const char * B_blk1 = B + PACKED_INDEX(1, i, KB, TILE_SIZE);\n            if (need_unpack) {\n                unpack_B<TB>(Tile0, B_blk0);\n                _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n            } else {\n                _tile_loadd(TMM0, B_blk0, TILE_N * VNNI_BLK);\n            }\n\n            if (need_unpack) {\n                unpack_B<TB>(Tile1, B_blk1);\n                _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n            } else {\n                _tile_loadd(TMM1, B_blk1, TILE_N * VNNI_BLK);\n            }\n\n            if (m0 == TILE_M) {\n                _tile_loadd(TMM2, A[i].qs, lda);\n            } else {\n                unpack_A(Tile23, &A[i], KB, m0);\n                _tile_loadd(TMM2, Tile23, TILE_K);\n            }\n\n            _tile_dpbssd(TMM4, TMM2, TMM0);\n            _tile_dpbssd(TMM6, TMM2, TMM1);\n\n            _tile_stored(TMM4, Tile4(C_cur), TILE_N * sizeof(int32_t));\n            _tile_stored(TMM6, Tile6(C_cur), TILE_N * sizeof(int32_t));\n\n            GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n                acc_C<TA, TB, is_acc>::apply(C,          ldc, Tile4(C_cur), &A[i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m0);\n                acc_C<TA, TB, is_acc>::apply(C + TILE_N, ldc, Tile6(C_cur), &A[i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m0);\n            });\n\n            if (m1 != 0) {\n                unpack_A(Tile23, &A[TILE_M * KB + i], KB, m1);\n                _tile_loadd(TMM3, Tile23, TILE_K);\n\n                _tile_dpbssd(TMM5, TMM3, TMM0);\n                _tile_dpbssd(TMM7, TMM3, TMM1);\n                _tile_stored(TMM5, Tile5(C_cur), TILE_N * sizeof(int32_t));\n                _tile_stored(TMM7, Tile7(C_cur), TILE_N * sizeof(int32_t));\n                GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n                    acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc,          ldc, Tile5(C_cur), &A[TILE_M * KB + i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m1);\n                    acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc + TILE_N, ldc, Tile7(C_cur), &A[TILE_M * KB + i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m1);\n                });\n            }\n        }\n    }\n    return;\n}\n\ntemplate <typename TA, typename TB, typename TC, int BLOCK_K,\n          typename std::enable_if<is_type_qkk<TB>::value, int>::type = 0>\nvoid tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const void * RESTRICT _B, float * RESTRICT C, int ldc) {\n    static_assert(std::is_same<TA, block_q8_K>::value);\n    const int TILE_SIZE = get_tile_size<TB>();\n\n    GGML_ASSERT(M <= 2 * TILE_M && N == 2 * TILE_N);\n    const TA * RESTRICT A = static_cast<const TA *>(_A);\n    const char * RESTRICT B = static_cast<const char *>(_B);\n\n    const int m0 = std::min(M, TILE_M);\n    const int m1 = std::max(M - TILE_M, 0);\n    //const int lda = KB * sizeof(TA);\n\n    static thread_local int8_t Tile0[TILE_N * TILE_K];\n    static thread_local int8_t Tile1[TILE_N * TILE_K];\n    static thread_local int8_t Tile23[TILE_M * TILE_K];\n\n    // mat mul result for each group\n    static thread_local int32_t Tile4[TILE_M * TILE_N];\n    static thread_local int32_t Tile5[TILE_M * TILE_N];\n    static thread_local int32_t Tile6[TILE_M * TILE_N];\n    static thread_local int32_t Tile7[TILE_M * TILE_N];\n\n    // sum of each QK_K block, contains 8 groups, int32\n    static thread_local int32_t Sumi4[TILE_M * TILE_N];\n    static thread_local int32_t Sumi5[TILE_M * TILE_N];\n    static thread_local int32_t Sumi6[TILE_M * TILE_N];\n    static thread_local int32_t Sumi7[TILE_M * TILE_N];\n\n    const int k_group_size = std::is_same<TB, block_q6_K>::value ? 16 : 32;\n    for (int i = 0; i < KB; ++i) {\n        // step 1: accumulate the quants across 8 groups, each group with 32\n        for (int k = 0; k < QK_K / k_group_size; ++k) {\n            GGML_DISPATCH_BOOL(k > 0, is_acc, [&] {\n                _tile_zero(TMM4);\n                _tile_zero(TMM6);\n\n                unpack_B<TB>(Tile0, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k);\n                _tile_loadd(TMM0, Tile0, TILE_N * VNNI_BLK);\n\n                unpack_B<TB>(Tile1, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k);\n                _tile_loadd(TMM1, Tile1, TILE_N * VNNI_BLK);\n\n                unpack_A<TB>(Tile23, &A[i], KB, k, m0);\n                _tile_loadd(TMM2, Tile23, TILE_K);\n\n                _tile_dpbssd(TMM4, TMM2, TMM0);\n                _tile_dpbssd(TMM6, TMM2, TMM1);\n\n                _tile_stored(TMM4, Tile4, TILE_N * sizeof(int32_t));\n                _tile_stored(TMM6, Tile6, TILE_N * sizeof(int32_t));\n\n                scale_C<TB, is_acc>(Tile4, Sumi4, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k, m0);\n                scale_C<TB, is_acc>(Tile6, Sumi6, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k, m0);\n\n                if (m1 != 0) {\n                    _tile_zero(TMM5);\n                    _tile_zero(TMM7);\n\n                    unpack_A<TB>(Tile23, &A[TILE_M * KB + i], KB, k, m1);\n                    _tile_loadd(TMM3, Tile23, TILE_K);\n\n                    _tile_dpbssd(TMM5, TMM3, TMM0);\n                    _tile_dpbssd(TMM7, TMM3, TMM1);\n\n                    _tile_stored(TMM5, Tile5, TILE_N * sizeof(int32_t));\n                    _tile_stored(TMM7, Tile7, TILE_N * sizeof(int32_t));\n\n                    scale_C<TB, is_acc>(Tile5, Sumi5, B + PACKED_INDEX(0, i, KB, TILE_SIZE), k, m1);\n                    scale_C<TB, is_acc>(Tile7, Sumi7, B + PACKED_INDEX(1, i, KB, TILE_SIZE), k, m1);\n                }\n            });\n        }\n\n        // step 2: accmulate the mins\n        GGML_DISPATCH_BOOL(i > 0, is_acc, [&] {\n            acc_C<TA, TB, is_acc>::apply(C,          ldc, Sumi4, &A[i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m0);\n            acc_C<TA, TB, is_acc>::apply(C + TILE_N, ldc, Sumi6, &A[i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m0);\n            if (m1 != 0) {\n                acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc,          ldc, Sumi5, &A[TILE_M * KB + i], KB, B + PACKED_INDEX(0, i, KB, TILE_SIZE), m1);\n                acc_C<TA, TB, is_acc>::apply(C + TILE_M * ldc + TILE_N, ldc, Sumi7, &A[TILE_M * KB + i], KB, B + PACKED_INDEX(1, i, KB, TILE_SIZE), m1);\n            }\n        });\n    }\n    return;\n}\n\n} // anonymous namespace\n\n// get the packed tensor size for quantized weights\nsize_t ggml_backend_amx_get_alloc_size(const struct ggml_tensor * tensor) {\n    const enum ggml_type TYPE = tensor->type;\n\n    const int K = tensor->ne[0]; // ne0: in_features\n    const int N = tensor->ne[1]; // ne1: out_features\n\n    auto get_tensor_size = [&] {\n        size_t row_size_B{0};\n        GGML_DISPATCH_QTYPES(TYPE, [&] {\n            row_size_B = get_row_size<type, blck_size>(K);\n        });\n        return N * row_size_B;\n    };\n\n    if (qtype_has_amx_kernels(TYPE)) {\n        return get_tensor_size();\n    } else {\n        // for f16, bf16 we don't do packing\n        return ggml_nbytes(tensor);\n    }\n}\n\n// pack weight to vnni format\nvoid ggml_backend_amx_convert_weight(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); // only full tensor conversion is supported for now\n\n    const enum ggml_type TYPE = tensor->type;\n\n    const int K = tensor->ne[0]; // ne0: in_features\n    const int N = tensor->ne[1]; // ne1: out_features\n\n    GGML_DISPATCH_QTYPES(TYPE, [&] {\n        convert_B_packed_format<type, blck_size>((void *)((char *)tensor->data + offset), (const type *)data, N, K);\n    });\n}\n\n// ne2 is passed explicitly to help compiler optimize repeated calls\ninline int64_t ggml_batch_offset(const ggml_tensor * t, int64_t batch_idx, int64_t ne2) {\n    const int64_t i2 = batch_idx % ne2;\n    const int64_t i3 = batch_idx / ne2;\n    return i3 * t->nb[3] + i2 * t->nb[2];\n}\n\nsize_t ggml_backend_amx_desired_wsize(const struct ggml_tensor * dst) {\n    struct ggml_tensor * src0 = dst->src[0];\n\n    const enum ggml_type TYPE = src0->type;\n\n    const bool is_floating_type = TYPE == GGML_TYPE_F16;\n    if (is_floating_type) {\n        return 0;\n    }\n\n    const int M = dst->ne[1];\n    const int K = src0->ne[0];\n    const int64_t n_batch = dst->ne[2] * dst->ne[3];\n\n    size_t desired_wsize = 0;\n\n    GGML_DISPATCH_QTYPES(TYPE, [&] {\n        const size_t row_size_A = K / blck_size * sizeof(vec_dot_type);\n        desired_wsize = n_batch * M * row_size_A;\n    });\n\n    return desired_wsize;\n}\n\n// NB: mixed dtype gemm with Advanced Matrix Extensions (Intel AMX)\n//\n// src0: weight in shape of {N, K}, quantized\n// src1: input  in shape of {M, K}, float32\n// dst:  output in shape of {M, N}, float32\n//\n// the function performs: dst = src1 @ src0.T for each batch\n//\nvoid ggml_backend_amx_mul_mat(const ggml_compute_params * params, struct ggml_tensor * dst) {\n    struct ggml_tensor * src0 = dst->src[0];\n    struct ggml_tensor * src1 = dst->src[1];\n\n    const enum ggml_type TYPE = src0->type;\n\n    // f16 only has avx512 kernels for now,\n    // amx kernels will be added once 6th gen xeon is released.\n    const bool is_floating_type = TYPE == GGML_TYPE_F16;\n\n    const int M = dst->ne[1];\n    const int N = dst->ne[0];\n    const int K = src0->ne[0];\n    const int ldc = dst->nb[1] / dst->nb[0];\n\n    const int64_t ne2 = dst->ne[2];\n    const int64_t n_batch = ne2 * dst->ne[3];\n\n    if (is_floating_type) {\n        constexpr int BLOCK_M = 4;\n        constexpr int BLOCK_N = 6;\n        const int MB = div_up(M, BLOCK_M);\n        const int NB = div_up(N, BLOCK_N);\n\n        parallel_for_ggml(params, n_batch * MB * NB, [&](int begin, int end) {\n            GGML_DISPATCH_FLOATING_TYPES(TYPE, [&] {\n                for (int i = begin; i < end; ++i) {\n                    int batch_idx = i / (MB * NB);\n                    int remaining = i % (MB * NB);\n                    int mb = remaining / NB;\n                    int nb = remaining % NB;\n\n                    int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n                    int64_t src1_offset = ggml_batch_offset(src1, batch_idx, ne2);\n                    int64_t dst_offset  = ggml_batch_offset(dst,  batch_idx, ne2);\n\n                    int mb_start = mb * BLOCK_M;\n                    int mb_size = std::min(BLOCK_M, M - mb_start);\n                    int nb_start = nb * BLOCK_N;\n                    int nb_size = std::min(BLOCK_N, N - nb_start);\n\n                    switch (mb_size << 4 | nb_size) {\n                        case 0x12: LAUNCH_TINYGEMM_KERNEL_AVX(1, 2); break;\n                        case 0x14: LAUNCH_TINYGEMM_KERNEL_AVX(1, 4); break;\n                        case 0x16: LAUNCH_TINYGEMM_KERNEL_AVX(1, 6); break;\n                        case 0x22: LAUNCH_TINYGEMM_KERNEL_AVX(2, 2); break;\n                        case 0x24: LAUNCH_TINYGEMM_KERNEL_AVX(2, 4); break;\n                        case 0x26: LAUNCH_TINYGEMM_KERNEL_AVX(2, 6); break;\n                        case 0x32: LAUNCH_TINYGEMM_KERNEL_AVX(3, 2); break;\n                        case 0x34: LAUNCH_TINYGEMM_KERNEL_AVX(3, 4); break;\n                        case 0x36: LAUNCH_TINYGEMM_KERNEL_AVX(3, 6); break;\n                        case 0x42: LAUNCH_TINYGEMM_KERNEL_AVX(4, 2); break;\n                        case 0x44: LAUNCH_TINYGEMM_KERNEL_AVX(4, 4); break;\n                        case 0x46: LAUNCH_TINYGEMM_KERNEL_AVX(4, 6); break;\n                        default: fprintf(stderr, \"Unexpected block size!\\n\");\n                    }\n                }\n            });\n        });\n        return;\n    }\n\n    // pointer to work space, used convert A from float to quantized type\n    void * wdata = params->wdata;\n\n    //TODO: performance improvement: merge quant A\n // if (params->ith == 0) {\n        GGML_DISPATCH_QTYPES(TYPE, [&] {\n            const size_t row_size_A = K / blck_size * sizeof(vec_dot_type);\n            const size_t desired_wsize = n_batch * M * row_size_A;\n            if (params->wsize < desired_wsize) {\n                GGML_ABORT(\"insufficient work space size\");\n            }\n\n            // Q4_0, Q4_1, Q8_0 handles 1 TILE_K per blck_size\n            // Q4_K, Q5_K, Q6_K, IQ4_XS handles 8 TILE_K per blck_size\n            GGML_ASSERT(TILE_K == blck_size || TILE_K * 8 == blck_size);\n\n            parallel_for_ggml(params, n_batch, [&](int begin, int end) {\n                for (int batch_idx = begin; batch_idx < end; ++batch_idx) {\n                    int64_t src1_offset = ggml_batch_offset(src1, batch_idx, ne2);\n                    const float * A_data = (const float *)((const char *)src1->data + src1_offset);\n                    char * wdata_batch = (char *)wdata + batch_idx * M * row_size_A;\n\n                    for (int m = 0; m < M; ++m) {\n                        from_float<vec_dot_type>(A_data + m * K, wdata_batch + m * row_size_A, K);\n                    }\n                }\n            });\n        });\n // }\n\n    ggml_barrier(params->threadpool);\n\n    if (M == 1) {\n        // MB = 1 and handle 8 tiles in each block\n        constexpr int kTilesN = 4;\n        constexpr int BLOCK_N = TILE_N * kTilesN;\n        const int NB = div_up(N, BLOCK_N);\n\n        parallel_for_ggml(params, n_batch * NB, [&](int begin, int end) {\n            GGML_DISPATCH_QTYPES(TYPE, [&] {\n                const int KB = K / blck_size;\n                const int TILE_SIZE = get_tile_size<type>();\n                const int row_size_A = KB * sizeof(vec_dot_type);\n                for (int i = begin; i < end; ++i) {\n                    int batch_idx = i / NB;\n                    int nb = i % NB;\n\n                    int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n                    int64_t dst_offset  = ggml_batch_offset(dst,  batch_idx, ne2);\n                    const char * wdata_batch = (const char *)wdata + batch_idx * row_size_A;\n\n                    int nb_start = nb * BLOCK_N;\n                    int nb_size = std::min(BLOCK_N, N - nb_start); // 32, 64, 96\n\n                    switch (nb_size) {\n                        //case 160: LAUNCH_TINYGEMM_KERNEL_VNNI(160); break;\n                        case 128: LAUNCH_TINYGEMM_KERNEL_VNNI(128); break;\n                        case 96: LAUNCH_TINYGEMM_KERNEL_VNNI(96); break;\n                        case 64: LAUNCH_TINYGEMM_KERNEL_VNNI(64); break;\n                        case 32: LAUNCH_TINYGEMM_KERNEL_VNNI(32); break;\n                        default: fprintf(stderr, \"Unexpected n block size!\\n\");\n                    }\n                }\n            });\n        });\n        return;\n    }\n\n    // handle 4 tiles at a tile\n    constexpr int BLOCK_M = TILE_M * 2;\n    constexpr int BLOCK_N = TILE_N * 2;\n    const int MB = div_up(M, BLOCK_M);\n    const int NB = div_up(N, BLOCK_N);\n\n    parallel_for_ggml(params, n_batch * MB * NB, [&](int begin, int end) {\n        // init tile config for each thread\n        ggml_tile_config_init();\n\n        GGML_DISPATCH_QTYPES(TYPE, [&] {\n            const int KB = K / blck_size;\n            const int TILE_SIZE = get_tile_size<type>();\n            const int row_size_A = KB * sizeof(vec_dot_type);\n\n            for (int i = begin; i < end; ++i) {\n                int batch_idx = i / (MB * NB);\n                int remaining = i % (MB * NB);\n                int mb = remaining / NB;\n                int nb = remaining % NB;\n\n                int64_t src0_offset = ggml_batch_offset(src0, batch_idx, ne2);\n                int64_t dst_offset  = ggml_batch_offset(dst,  batch_idx, ne2);\n                const char * wdata_batch = (const char *)wdata + batch_idx * M * row_size_A;\n\n                int mb_start = mb * BLOCK_M;\n                int mb_size = std::min(BLOCK_M, M - mb_start);\n                int nb_start = nb * BLOCK_N;\n                int nb_size = BLOCK_N;\n\n                tinygemm_kernel_amx<vec_dot_type, type, float, blck_size>(\n                    mb_size, nb_size, KB,\n                    wdata_batch + mb_start * row_size_A,\n                    (const char *)src0->data + src0_offset + PACKED_INDEX(nb * 2, 0, KB, TILE_SIZE),\n                    (float *) dst->data + dst_offset + mb_start * N + nb_start, ldc);\n            }\n        });\n    });\n}\n\n#endif // if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n"
  },
  {
    "path": "ggml/src/ggml-cpu/amx/mmq.h",
    "content": "#pragma once\n#include \"common.h\"\n\nsize_t ggml_backend_amx_desired_wsize(const struct ggml_tensor * dst);\n\nsize_t ggml_backend_amx_get_alloc_size(const struct ggml_tensor * tensor);\n\nvoid ggml_backend_amx_convert_weight(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);\n\nvoid ggml_backend_amx_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp",
    "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__aarch64__)\n\n#if defined(__linux__)\n#include <sys/auxv.h>\n#elif defined(__APPLE__)\n#include <sys/sysctl.h>\n#endif\n\n#if !defined(HWCAP2_SVE2)\n#define HWCAP2_SVE2 (1 << 1)\n#endif\n\n#if !defined(HWCAP2_I8MM)\n#define HWCAP2_I8MM (1 << 13)\n#endif\n\n#if !defined(HWCAP2_SME)\n#define HWCAP2_SME (1 << 23)\n#endif\n\nstruct aarch64_features {\n    // has_neon not needed, aarch64 has NEON guaranteed\n    bool has_dotprod     = false;\n    bool has_fp16_va     = false;\n    bool has_sve         = false;\n    bool has_sve2        = false;\n    bool has_i8mm        = false;\n    bool has_sme         = false;\n\n    aarch64_features() {\n#if defined(__linux__)\n        uint32_t hwcap = getauxval(AT_HWCAP);\n        uint32_t hwcap2 = getauxval(AT_HWCAP2);\n\n        has_dotprod = !!(hwcap & HWCAP_ASIMDDP);\n        has_fp16_va = !!(hwcap & HWCAP_FPHP);\n        has_sve     = !!(hwcap & HWCAP_SVE);\n        has_sve2    = !!(hwcap2 & HWCAP2_SVE2);\n        has_i8mm    = !!(hwcap2 & HWCAP2_I8MM);\n        has_sme     = !!(hwcap2 & HWCAP2_SME);\n#elif defined(__APPLE__)\n        int oldp = 0;\n        size_t size = sizeof(oldp);\n\n        if (sysctlbyname(\"hw.optional.arm.FEAT_DotProd\", &oldp, &size, NULL, 0) == 0) {\n            has_dotprod = static_cast<bool>(oldp);\n        }\n\n        if (sysctlbyname(\"hw.optional.arm.FEAT_I8MM\", &oldp, &size, NULL, 0) == 0) {\n            has_i8mm = static_cast<bool>(oldp);\n        }\n\n        if (sysctlbyname(\"hw.optional.arm.FEAT_SME\", &oldp, &size, NULL, 0) == 0) {\n            has_sme = static_cast<bool>(oldp);\n        }\n\n        // Apple apparently does not implement SVE yet\n#endif\n    }\n};\n\nstatic int ggml_backend_cpu_aarch64_score() {\n    int score = 1;\n    aarch64_features af;\n\n#ifdef GGML_USE_DOTPROD\n    if (!af.has_dotprod) { return 0; }\n    score += 1<<1;\n#endif\n#ifdef GGML_USE_FP16_VECTOR_ARITHMETIC\n    if (!af.has_fp16_va) { return 0; }\n    score += 1<<2;\n#endif\n#ifdef GGML_USE_SVE\n    if (!af.has_sve) { return 0; }\n    score += 1<<3;\n#endif\n#ifdef GGML_USE_MATMUL_INT8\n    if (!af.has_i8mm) { return 0; }\n    score += 1<<4;\n#endif\n#ifdef GGML_USE_SVE2\n    if (!af.has_sve2) { return 0; }\n    score += 1<<5;\n#endif\n#ifdef GGML_USE_SME\n    if (!af.has_sme) { return 0; }\n    score += 1<<6;\n#endif\n\n    return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)\n\n# endif // defined(__aarch64__)\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/arm/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__ARM_NEON)\n#define B1(c,s,n)  0x ## n ## c ,  0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s  ) B7(c,s,     c), B7(c,s,     s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n    for (int i = 0; i < nb; i++) {\n        float32x4_t srcv [8];\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = vld1q_f32(x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = vmaxvq_f32(amaxv[0]);\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        for (int j = 0; j < 8; j++) {\n            const float32x4_t v  = vmulq_n_f32(srcv[j], id);\n            const int32x4_t   vi = vcvtnq_s32_f32(v);\n\n            y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);\n            y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);\n            y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);\n            y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);\n        }\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n#if defined(__ARM_NEON)\n    for (int i = 0; i < nb; i++) {\n        float32x4_t srcv [8];\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = vld1q_f32(x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vmaxq_f32(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vmaxq_f32(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vmaxq_f32(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = vmaxvq_f32(amaxv[0]);\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        int32x4_t accv = vdupq_n_s32(0);\n\n        for (int j = 0; j < 8; j++) {\n            const float32x4_t v  = vmulq_n_f32(srcv[j], id);\n            const int32x4_t   vi = vcvtnq_s32_f32(v);\n\n            y[i].qs[4*j + 0] = vgetq_lane_s32(vi, 0);\n            y[i].qs[4*j + 1] = vgetq_lane_s32(vi, 1);\n            y[i].qs[4*j + 2] = vgetq_lane_s32(vi, 2);\n            y[i].qs[4*j + 3] = vgetq_lane_s32(vi, 3);\n\n            accv = vaddq_s32(accv, vi);\n        }\n\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * vaddvq_s32(accv));\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n// placeholder implementation for Apple targets\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    assert((nrc == 2) || (nrc == 1));\n#else\n    assert(nrc == 1);\n#endif\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const block_q4_0 * GGML_RESTRICT vx0 = vx;\n        const block_q4_0 * GGML_RESTRICT vx1 = (const block_q4_0 *) ((const uint8_t*)vx + bx);\n        const block_q8_0 * GGML_RESTRICT vy0 = vy;\n        const block_q8_0 * GGML_RESTRICT vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);\n\n        float32x4_t sumv0 = vdupq_n_f32(0.0f);\n\n        for (int i = 0; i < nb; i++) {\n            const block_q4_0 * GGML_RESTRICT b_x0 = &vx0[i];\n            const block_q4_0 * GGML_RESTRICT b_x1 = &vx1[i];\n            const block_q8_0 * GGML_RESTRICT b_y0 = &vy0[i];\n            const block_q8_0 * GGML_RESTRICT b_y1 = &vy1[i];\n\n            const uint8x16_t m4b = vdupq_n_u8(0x0F);\n            const int8x16_t  s8b = vdupq_n_s8(0x8);\n\n            const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);\n            const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);\n\n            // 4-bit -> 8-bit\n            const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n            const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n            const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n            const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n            // sub 8\n            const int8x16_t x0_l = vsubq_s8(v0_0l, s8b);\n            const int8x16_t x0_h = vsubq_s8(v0_0h, s8b);\n            const int8x16_t x1_l = vsubq_s8(v0_1l, s8b);\n            const int8x16_t x1_h = vsubq_s8(v0_1h, s8b);\n\n            // load y\n            const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n            const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n            const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n            const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n            float32_t _scale[4] = {\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n            };\n            float32x4_t scale = vld1q_f32(_scale);\n\n            int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n            int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n            int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n            int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n            int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n            int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n            int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n            int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n\n            sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n                                                l1, r1)), l2, r2)), l3, r3))), scale);\n        }\n\n        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n        float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n        vst1_f32(s,      vget_low_f32 (sumv2));\n        vst1_f32(s + bs, vget_high_f32(sumv2));\n\n        return;\n    }\n#endif\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__ARM_FEATURE_SVE)\n    svfloat32_t sumv0 = svdup_n_f32(0.0f);\n    svfloat32_t sumv1 = svdup_n_f32(0.0f);\n\n    const int vector_length = ggml_cpu_get_sve_cnt()*8;\n\n    // VLA Implementation using switch case\n    switch (vector_length) {\n        case 128:\n            {\n                // predicate for activating higher lanes for 4 float32 elements\n                const svbool_t ph4 = svptrue_pat_b32(SV_VL4);\n\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    // load x\n                    const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs);\n                    const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs);\n\n                    // 4-bit -> 8-bit\n                    const svint8_t qx0l = svreinterpret_s8_u8(svand_n_u8_m(svptrue_b8(), qx0r, 0x0F));\n                    const svint8_t qx0h = svreinterpret_s8_u8(svlsr_n_u8_m(svptrue_b8(), qx0r, 0x04));\n                    const svint8_t qx1l = svreinterpret_s8_u8(svand_n_u8_m(svptrue_b8(), qx1r, 0x0F));\n                    const svint8_t qx1h = svreinterpret_s8_u8(svlsr_n_u8_m(svptrue_b8(), qx1r, 0x04));\n\n                    // sub 8\n                    const svint8_t qx0ls = svsub_n_s8_x(svptrue_b8(), qx0h, 8);\n                    const svint8_t qx0hs = svsub_n_s8_x(svptrue_b8(), qx0l, 8);\n                    const svint8_t qx1ls = svsub_n_s8_x(svptrue_b8(), qx1h, 8);\n                    const svint8_t qx1hs = svsub_n_s8_x(svptrue_b8(), qx1l, 8);\n\n                    // load y\n                    const svint8_t qy0h = svld1_s8(svptrue_b8(), y0->qs);\n                    const svint8_t qy0l = svld1_s8(svptrue_b8(), y0->qs + 16);\n                    const svint8_t qy1h = svld1_s8(svptrue_b8(), y1->qs);\n                    const svint8_t qy1l = svld1_s8(svptrue_b8(), y1->qs + 16);\n\n                    // dot product\n                    sumv0 = svmla_n_f32_x(ph4, sumv0, svcvt_f32_s32_x(ph4, svadd_x(ph4,\n                                    svdot_s32(svdup_n_s32(0), qx0ls, qy0l),\n                                    svdot_s32(svdup_n_s32(0), qx0hs, qy0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n                    sumv1 = svmla_n_f32_x(ph4, sumv1, svcvt_f32_s32_x(ph4, svadd_x(ph4,\n                                    svdot_s32(svdup_n_s32(0), qx1ls, qy1l),\n                                    svdot_s32(svdup_n_s32(0), qx1hs, qy1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n                }\n\n                sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n            } break;\n        case 256:\n            {\n                // predicate for activating higher lanes for 16 int8 elements\n                const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n                // predicate for activating lower lanes for  16 int8 elements\n                const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16);\n\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    // load x\n                    const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs);\n                    const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs);\n\n                    // 4-bit -> 8-bit\n                    const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx0r, 0x0F), 0x04));\n                    const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx1r, 0x0F), 0x04));\n\n                    // sub 8\n                    const svint8_t qx0s = svsub_n_s8_x(svptrue_b8(), qx0, 8);\n                    const svint8_t qx1s = svsub_n_s8_x(svptrue_b8(), qx1, 8);\n\n                    // load y\n                    const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);\n                    const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);\n\n                    // dot product\n                    sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(),\n                                svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n                    sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(),\n                                svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n                }\n\n                sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n            } break;\n        case 512:\n            {\n                // predicate for activating higher lanes for 32 int8 elements\n                const svbool_t ph32 = svptrue_pat_b8(SV_VL32);\n\n                // predicate for activating higher lanes for 16 int8 elements\n                const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n                // predicate for activating lower lanes for 16 int8 elements from first 32 int8 activated lanes\n                const svbool_t pl16 = svnot_b_z(ph32, ph16);\n\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    // load x\n                    const svuint8_t qx0r = svld1rq_u8(ph32, x0->qs);\n                    const svuint8_t qx1r = svld1rq_u8(ph32, x1->qs);\n\n                    // 4-bit -> 8-bit\n                    const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx0r, 0x0F), 0x04));\n                    const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_n_u8_m(ph16, qx1r, 0x0F), 0x04));\n\n                    // sub 8\n                    const svint8_t qx0s = svsub_n_s8_x(ph32, qx0, 8);\n                    const svint8_t qx1s = svsub_n_s8_x(ph32, qx1, 8);\n\n                    // load y\n                    const svint8_t qy0 = svld1_s8(ph32, y0->qs);\n                    const svint8_t qy1 = svld1_s8(ph32, y1->qs);\n\n                    // dot product\n                    sumv0 = svmla_n_f32_x(ph32, sumv0, svcvt_f32_s32_x(ph32,\n                                svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n                    sumv1 = svmla_n_f32_x(ph32, sumv1, svcvt_f32_s32_x(ph32,\n                                svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n                }\n\n                sumf = svaddv_f32(ph32, svadd_f32_x(ph32, sumv0, sumv1));\n            } break;\n        default:\n            assert(false && \"Unsupported vector length\");\n            break;\n    }\n\n#elif defined(__ARM_NEON)\n    float32x4_t sumv0 = vdupq_n_f32(0.0f);\n    float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q4_0 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        const uint8x16_t m4b = vdupq_n_u8(0x0F);\n        const int8x16_t  s8b = vdupq_n_s8(0x8);\n\n        const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n        const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n        // 4-bit -> 8-bit\n        const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n        const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n        const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n        const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n        // sub 8\n        const int8x16_t v0_0ls = vsubq_s8(v0_0l, s8b);\n        const int8x16_t v0_0hs = vsubq_s8(v0_0h, s8b);\n        const int8x16_t v0_1ls = vsubq_s8(v0_1l, s8b);\n        const int8x16_t v0_1hs = vsubq_s8(v0_1h, s8b);\n\n        // load y\n        const int8x16_t v1_0l = vld1q_s8(y0->qs);\n        const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n        const int8x16_t v1_1l = vld1q_s8(y1->qs);\n        const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n        // dot product into int32x4_t\n        const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0ls, v1_0l), v0_0hs, v1_0h);\n        const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1ls, v1_1l), v0_1hs, v1_1h);\n\n        sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n        sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n    }\n\n    sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n            const int v1 = (x[ib].qs[j] >>   4) - 8;\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    assert((nrc == 2) || (nrc == 1));\n#else\n    assert(nrc == 1);\n#endif\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const block_q4_1 * GGML_RESTRICT vx0 = vx;\n        const block_q4_1 * GGML_RESTRICT vx1 = (const block_q4_1 *) ((const uint8_t*)vx + bx);\n        const block_q8_1 * GGML_RESTRICT vy0 = vy;\n        const block_q8_1 * GGML_RESTRICT vy1 = (const block_q8_1 *) ((const uint8_t*)vy + by);\n\n        float32x4_t sumv0 = vdupq_n_f32(0.0f);\n        float32x4_t summs0 = vdupq_n_f32(0.0f);\n\n        for (int i = 0; i < nb; i++) {\n            const block_q4_1 * GGML_RESTRICT b_x0 = &vx0[i];\n            const block_q4_1 * GGML_RESTRICT b_x1 = &vx1[i];\n            const block_q8_1 * GGML_RESTRICT b_y0 = &vy0[i];\n            const block_q8_1 * GGML_RESTRICT b_y1 = &vy1[i];\n\n            float32_t summs_t[4] = {\n                GGML_CPU_FP16_TO_FP32(b_x0->m) * GGML_CPU_FP16_TO_FP32(b_y0->s),\n                GGML_CPU_FP16_TO_FP32(b_x1->m) * GGML_CPU_FP16_TO_FP32(b_y0->s),\n                GGML_CPU_FP16_TO_FP32(b_x0->m) * GGML_CPU_FP16_TO_FP32(b_y1->s),\n                GGML_CPU_FP16_TO_FP32(b_x1->m) * GGML_CPU_FP16_TO_FP32(b_y1->s)\n            };\n            summs0 = vaddq_f32(summs0, vld1q_f32(summs_t));\n\n            const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n            const uint8x16_t v0_0 = vld1q_u8(b_x0->qs);\n            const uint8x16_t v0_1 = vld1q_u8(b_x1->qs);\n\n            // 4-bit -> 8-bit\n            const int8x16_t x0_l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n            const int8x16_t x0_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n            const int8x16_t x1_l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n            const int8x16_t x1_h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n            // load y\n            const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n            const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n            const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n            const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n            // mmla into int32x4_t\n            float32_t _scale[4] = {\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n            };\n            float32x4_t scale = vld1q_f32(_scale);\n\n            int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n            int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n            int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n            int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n            int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n            int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n            int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n            int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n            sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n                                                l1, r1)), l2, r2)), l3, r3))), scale);\n        }\n\n        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n        float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n        sumv2 = vaddq_f32(sumv2, summs0);\n\n        vst1_f32(s,      vget_low_f32 (sumv2));\n        vst1_f32(s + bs, vget_high_f32(sumv2));\n\n        return;\n    }\n#endif\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__ARM_NEON)\n    float32x4_t sumv0 = vdupq_n_f32(0.0f);\n    float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n    float summs = 0;\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q4_1 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q4_1 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_1 * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s) + GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n        const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n        const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n        const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n        // 4-bit -> 8-bit\n        const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n        const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n        const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n        const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n        // load y\n        const int8x16_t v1_0l = vld1q_s8(y0->qs);\n        const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n        const int8x16_t v1_1l = vld1q_s8(y1->qs);\n        const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n        // dot product into int32x4_t\n        const int32x4_t p_0 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_0l, v1_0l), v0_0h, v1_0h);\n        const int32x4_t p_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), v0_1l, v1_1l), v0_1h, v1_1h);\n\n        sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(p_0), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n        sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(p_1), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n    }\n\n    sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs;\n\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F);\n            const int v1 = (x[ib].qs[j] >>   4);\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __ARM_NEON\n    const int8x16_t values = vld1q_s8(kvalues_mxfp4);\n    const uint8x16_t m4b = vdupq_n_u8(0x0f);\n    uint8x16x2_t q4bits;\n    int8x16x4_t q4b;\n    int8x16x4_t q8b;\n    int32x4_t prod_1;\n    int32x4_t prod_2;\n\n    for (; ib + 1 < nb; ib += 2) {\n        q4bits.val[0] = vld1q_u8(x[ib + 0].qs);\n        q4bits.val[1] = vld1q_u8(x[ib + 1].qs);\n        q8b.val[0]    = vld1q_s8(y[ib + 0].qs);\n        q8b.val[1]    = vld1q_s8(y[ib + 0].qs + 16);\n        q8b.val[2]    = vld1q_s8(y[ib + 1].qs);\n        q8b.val[3]    = vld1q_s8(y[ib + 1].qs + 16);\n\n        q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[0], m4b));\n        q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n        q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[1], m4b));\n        q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n        prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n        prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n        sumf +=\n            GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * vaddvq_s32(prod_1) +\n            GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * vaddvq_s32(prod_2);\n    }\n\n#endif\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_E8M0_TO_FP32_HALF(x[ib].e);\n        int sumi1 = 0;\n        int sumi2 = 0;\n        for (int j = 0; j < QK_MXFP4/2; ++j) {\n            sumi1 += y[ib].qs[j +          0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_NVFP4 == 0);\n\n    const block_nvfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    // Each NVFP4 super-block (64 elements) spans 2 q8_0 blocks\n    const int nb = n / QK_NVFP4;\n\n    float sumf = 0;\n\n#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)\n    const int8x16_t values = vld1q_s8(kvalues_mxfp4);\n    const uint8x16_t m4b = vdupq_n_u8(0x0f);\n    float32x4_t acc = vdupq_n_f32(0.0f);\n\n    for (int ib = 0; ib < nb; ++ib) {\n        const uint8x16_t q4bits_0 = vld1q_u8(x[ib].qs);\n        const uint8x16_t q4bits_1 = vld1q_u8(x[ib].qs + 16);\n\n        const int8x16_t q4_lo_0 = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits_0, m4b));\n        const int8x16_t q4_hi_0 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_0, 4));\n        const int8x16_t q4_lo_1 = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits_1, m4b));\n        const int8x16_t q4_hi_1 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_1, 4));\n\n        const int8x16_t q8_0a = vld1q_s8(y[2*ib].qs);\n        const int8x16_t q8_0b = vld1q_s8(y[2*ib].qs + 16);\n        const int8x16_t q8_lo_0 = vcombine_s8(vget_low_s8(q8_0a), vget_low_s8(q8_0b));\n        const int8x16_t q8_hi_0 = vcombine_s8(vget_high_s8(q8_0a), vget_high_s8(q8_0b));\n\n        const int8x16_t q8_1a = vld1q_s8(y[2*ib+1].qs);\n        const int8x16_t q8_1b = vld1q_s8(y[2*ib+1].qs + 16);\n        const int8x16_t q8_lo_1 = vcombine_s8(vget_low_s8(q8_1a), vget_low_s8(q8_1b));\n        const int8x16_t q8_hi_1 = vcombine_s8(vget_high_s8(q8_1a), vget_high_s8(q8_1b));\n\n        const int32x4_t p0 = vaddq_s32(\n            ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_0, q8_lo_0),\n            ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_0, q8_hi_0));\n        const int32x4_t p1 = vaddq_s32(\n            ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_1, q8_lo_1),\n            ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_1, q8_hi_1));\n\n        const int32x4_t sums = vpaddq_s32(p0, p1);\n\n        // Decode 4 UE4M3 scales to f32 and multiply with q8 scales\n        const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d);\n        const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d);\n        const float32x4_t nvsc = {\n            ggml_ue4m3_to_fp32(x[ib].d[0]),\n            ggml_ue4m3_to_fp32(x[ib].d[1]),\n            ggml_ue4m3_to_fp32(x[ib].d[2]),\n            ggml_ue4m3_to_fp32(x[ib].d[3])\n        };\n        const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1});\n\n        acc = vfmaq_f32(acc, vcvtq_f32_s32(sums), scales);\n    }\n    sumf = vaddvq_f32(acc);\n#else\n    for (int ib = 0; ib < nb; ++ib) {\n        for (int si = 0; si < 4; ++si) {\n            const float d = ggml_ue4m3_to_fp32(x[ib].d[si]);\n            const int q8b = si / 2;\n            const int q8o = (si % 2) * QK_NVFP4_SUB;\n            const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8b].d);\n\n            int sumi_lo = 0, sumi_hi = 0;\n            for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {\n                const uint8_t qv = x[ib].qs[si*(QK_NVFP4_SUB/2) + j];\n                sumi_lo += y[2*ib + q8b].qs[q8o + j +               0] * kvalues_mxfp4[qv & 0xf];\n                sumi_hi += y[2*ib + q8b].qs[q8o + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >>  4];\n            }\n            sumf += dy * d * (sumi_lo + sumi_hi);\n        }\n    }\n#endif\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n    float32x4_t sumv0 = vdupq_n_f32(0.0f);\n    float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n    uint32_t qh0;\n    uint32_t qh1;\n\n    uint64_t tmp0[4];\n    uint64_t tmp1[4];\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n        const block_q5_0 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n        // extract the 5th bit via lookup table ((!b) << 4)\n        memcpy(&qh0, x0->qh, sizeof(qh0));\n        memcpy(&qh1, x1->qh, sizeof(qh1));\n\n        tmp0[0] = table_b2b_1[(qh0 >>  0) & 0xFF];\n        tmp0[1] = table_b2b_1[(qh0 >>  8) & 0xFF];\n        tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];\n        tmp0[3] = table_b2b_1[(qh0 >> 24)       ];\n\n        tmp1[0] = table_b2b_1[(qh1 >>  0) & 0xFF];\n        tmp1[1] = table_b2b_1[(qh1 >>  8) & 0xFF];\n        tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];\n        tmp1[3] = table_b2b_1[(qh1 >> 24)       ];\n\n        const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));\n        const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));\n        const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));\n        const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));\n\n        const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n        const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n        // 4-bit -> 8-bit\n        int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n        int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n        int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n        int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n        // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)\n        const int8x16_t v0_0lf = vsubq_s8(v0_0l, qhl0);\n        const int8x16_t v0_0hf = vsubq_s8(v0_0h, qhh0);\n        const int8x16_t v0_1lf = vsubq_s8(v0_1l, qhl1);\n        const int8x16_t v0_1hf = vsubq_s8(v0_1h, qhh1);\n\n        // load y\n        const int8x16_t v1_0l = vld1q_s8(y0->qs);\n        const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n        const int8x16_t v1_1l = vld1q_s8(y1->qs);\n        const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n        sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n        sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n    }\n\n    sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n\n#endif\n    for (; ib < nb; ++ib) {\n        uint32_t qh;\n        memcpy(&qh, x[ib].qh, sizeof(qh));\n\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;\n            const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));\n\n            const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);\n            const int32_t x1 = (int8_t)(((x[ib].qs[j] >>   4) | xh_1) - 16);\n\n            sumi0 += (x0 * y[ib].qs[j]);\n            sumi1 += (x1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_NEON)\n    float32x4_t sumv0 = vdupq_n_f32(0.0f);\n    float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n    float summs0 = 0.0f;\n    float summs1 = 0.0f;\n\n    uint32_t qh0;\n    uint32_t qh1;\n\n    uint64_t tmp0[4];\n    uint64_t tmp1[4];\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n        const block_q5_1 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n        const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        const uint8x16_t m4b = vdupq_n_u8(0x0F);\n\n        summs0 += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n        summs1 += GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n        // extract the 5th bit via lookup table ((b) << 4)\n        memcpy(&qh0, x0->qh, sizeof(qh0));\n        memcpy(&qh1, x1->qh, sizeof(qh1));\n\n        tmp0[0] = table_b2b_0[(qh0 >>  0) & 0xFF];\n        tmp0[1] = table_b2b_0[(qh0 >>  8) & 0xFF];\n        tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];\n        tmp0[3] = table_b2b_0[(qh0 >> 24)       ];\n\n        tmp1[0] = table_b2b_0[(qh1 >>  0) & 0xFF];\n        tmp1[1] = table_b2b_0[(qh1 >>  8) & 0xFF];\n        tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];\n        tmp1[3] = table_b2b_0[(qh1 >> 24)       ];\n\n        const int8x16_t qhl0 = vld1q_s8((const int8_t *)(tmp0 + 0));\n        const int8x16_t qhh0 = vld1q_s8((const int8_t *)(tmp0 + 2));\n        const int8x16_t qhl1 = vld1q_s8((const int8_t *)(tmp1 + 0));\n        const int8x16_t qhh1 = vld1q_s8((const int8_t *)(tmp1 + 2));\n\n        const uint8x16_t v0_0 = vld1q_u8(x0->qs);\n        const uint8x16_t v0_1 = vld1q_u8(x1->qs);\n\n        // 4-bit -> 8-bit\n        const int8x16_t v0_0l = vreinterpretq_s8_u8(vandq_u8  (v0_0, m4b));\n        const int8x16_t v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4));\n        const int8x16_t v0_1l = vreinterpretq_s8_u8(vandq_u8  (v0_1, m4b));\n        const int8x16_t v0_1h = vreinterpretq_s8_u8(vshrq_n_u8(v0_1, 4));\n\n        // add high bit\n        const int8x16_t v0_0lf = vorrq_s8(v0_0l, qhl0);\n        const int8x16_t v0_0hf = vorrq_s8(v0_0h, qhh0);\n        const int8x16_t v0_1lf = vorrq_s8(v0_1l, qhl1);\n        const int8x16_t v0_1hf = vorrq_s8(v0_1h, qhh1);\n\n        // load y\n        const int8x16_t v1_0l = vld1q_s8(y0->qs);\n        const int8x16_t v1_0h = vld1q_s8(y0->qs + 16);\n        const int8x16_t v1_1l = vld1q_s8(y1->qs);\n        const int8x16_t v1_1h = vld1q_s8(y1->qs + 16);\n\n        sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_0lf, v1_0l),\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_0hf, v1_0h))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n        sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_1lf, v1_1l),\n                        ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n    }\n\n    sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1) + summs0 + summs1;\n\n#endif\n    for (; ib < nb; ++ib) {\n        uint32_t qh;\n        memcpy(&qh, x[ib].qh, sizeof(qh));\n\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const uint8_t xh_0 = ((qh >> (j +  0)) << 4) & 0x10;\n            const uint8_t xh_1 = ((qh >> (j + 12))     ) & 0x10;\n\n            const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;\n            const int32_t x1 = (x[ib].qs[j] >>  4) | xh_1;\n\n            sumi0 += (x0 * y[ib].qs[j]);\n            sumi1 += (x1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    assert((nrc == 2) || (nrc == 1));\n#else\n    assert(nrc == 1);\n#endif\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const block_q8_0 * GGML_RESTRICT vx0 = vx;\n        const block_q8_0 * GGML_RESTRICT vx1 = (const block_q8_0 *) ((const uint8_t*)vx + bx);\n        const block_q8_0 * GGML_RESTRICT vy0 = vy;\n        const block_q8_0 * GGML_RESTRICT vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by);\n\n        float32x4_t sumv0 = vdupq_n_f32(0.0f);\n\n        for (int i = 0; i < nb; i++) {\n            const block_q8_0 * GGML_RESTRICT b_x0 = &vx0[i];\n            const block_q8_0 * GGML_RESTRICT b_y0 = &vy0[i];\n\n            const block_q8_0 * GGML_RESTRICT b_x1 = &vx1[i];\n            const block_q8_0 * GGML_RESTRICT b_y1 = &vy1[i];\n\n            const int8x16_t x0_l = vld1q_s8(b_x0->qs);\n            const int8x16_t x0_h = vld1q_s8(b_x0->qs + 16);\n            const int8x16_t x1_l = vld1q_s8(b_x1->qs);\n            const int8x16_t x1_h = vld1q_s8(b_x1->qs + 16);\n\n            // load y\n            const int8x16_t y0_l = vld1q_s8(b_y0->qs);\n            const int8x16_t y0_h = vld1q_s8(b_y0->qs + 16);\n            const int8x16_t y1_l = vld1q_s8(b_y1->qs);\n            const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16);\n\n            float32_t _scale[4] = {\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x0->d)*GGML_CPU_FP16_TO_FP32(b_y1->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y0->d),\n                GGML_CPU_FP16_TO_FP32(b_x1->d)*GGML_CPU_FP16_TO_FP32(b_y1->d)\n            };\n            float32x4_t scale = vld1q_f32(_scale);\n\n            int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n            int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l)));\n\n            int8x16_t l2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n            int8x16_t l3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_h), vreinterpretq_s64_s8(x1_h)));\n\n            int8x16_t r0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n            int8x16_t r1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_l), vreinterpretq_s64_s8(y1_l)));\n\n            int8x16_t r2 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n            int8x16_t r3 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(y0_h), vreinterpretq_s64_s8(y1_h)));\n\n            sumv0 = vmlaq_f32(sumv0,(vcvtq_f32_s32(vmmlaq_s32((vmmlaq_s32((vmmlaq_s32((vmmlaq_s32(vdupq_n_s32(0), l0, r0)),\n                                                l1, r1)), l2, r2)), l3, r3))), scale);\n        }\n\n        float32x4_t sumv1 = vextq_f32 (sumv0, sumv0, 2);\n        float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1);\n\n        vst1_f32(s,      vget_low_f32 (sumv2));\n        vst1_f32(s + bs, vget_high_f32(sumv2));\n\n        return;\n    }\n#endif\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__ARM_FEATURE_SVE)\n    svfloat32_t sumv0 = svdup_n_f32(0.0f);\n    svfloat32_t sumv1 = svdup_n_f32(0.0f);\n\n    const int vector_length = ggml_cpu_get_sve_cnt()*8;\n\n    //VLA Implementation for SVE\n    switch (vector_length) {\n        case 128:\n            {\n                // predicate for activating lanes for 16 Int8 elements\n                const svbool_t ph16 = svptrue_pat_b8 (SV_VL16);\n                const svbool_t pl16 = svptrue_pat_b32(SV_VL4);\n\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    // load x\n                    const svint8_t qx0_0 = svld1_s8(ph16, x0->qs);\n                    const svint8_t qx0_1 = svld1_s8(ph16, x0->qs+16);\n                    const svint8_t qx1_0 = svld1_s8(ph16, x1->qs);\n                    const svint8_t qx1_1 = svld1_s8(ph16, x1->qs+16);\n\n                    // load y\n                    const svint8_t qy0_0 = svld1_s8(ph16, y0->qs);\n                    const svint8_t qy0_1 = svld1_s8(ph16, y0->qs+16);\n                    const svint8_t qy1_0 = svld1_s8(ph16, y1->qs);\n                    const svint8_t qy1_1 = svld1_s8(ph16, y1->qs+16);\n\n                    sumv0 = svmla_n_f32_x(pl16, sumv0, svcvt_f32_s32_x(pl16, svadd_x(pl16,\n                                    svdot_s32(svdup_n_s32(0), qx0_0, qy0_0),\n                                    svdot_s32(svdup_n_s32(0), qx0_1, qy0_1))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n                    sumv1 = svmla_n_f32_x(pl16, sumv1, svcvt_f32_s32_x(pl16, svadd_x(pl16,\n                                    svdot_s32(svdup_n_s32(0), qx1_0, qy1_0),\n                                    svdot_s32(svdup_n_s32(0), qx1_1, qy1_1))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n                }\n\n                sumf = svaddv_f32(pl16, svadd_f32_x(pl16, sumv0, sumv1));\n            } break;\n        case 256:\n            {\n                //printf(\"sve256\");\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    // load x\n                    const svint8_t qx0 = svld1_s8(svptrue_b8(), x0->qs);\n                    const svint8_t qx1 = svld1_s8(svptrue_b8(), x1->qs);\n\n                    // load y\n                    const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs);\n                    const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs);\n\n                    sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(),\n                                svdot_s32(svdup_n_s32(0), qx0, qy0)), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n                    sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(),\n                                svdot_s32(svdup_n_s32(0), qx1, qy1)), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n                }\n\n                sumf = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1));\n            } break;\n        case 512:\n            {\n                // predicate for activating high 256 bit\n                const svbool_t ph32 = svptrue_pat_b8(SV_VL32);\n                // predicate for activating low 256 bit\n                const svbool_t pl32 = svnot_b_z(svptrue_b8(), ph32);\n\n                // predicate for activating high lanes for 8 float32 elements\n                const svbool_t ph8 = svptrue_pat_b32(SV_VL8);\n                // predicate for activating low lanes for 8 float32 elements\n                const svbool_t pl8 = svnot_b_z(svptrue_b32(), ph8);\n\n                svfloat32_t sumv00 = svdup_n_f32(0.0f);\n\n                for (; ib + 1 < nb; ib += 2) {\n                    const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n                    const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n                    const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n                    //load 32 int8_t in first half of vector and put another 32 int8_t in second vector lower bits\n                    // and add them to make one 64 element vector\n                    // load x\n                    const svint8_t qx_32 = svld1_s8(ph32, x0->qs);\n                          svint8_t qx_64 = svld1_s8(pl32, x0->qs + 2);\n\n                    qx_64 = svadd_s8_x(svptrue_b8(), qx_32, qx_64);\n\n                    // load y\n                    const svint8_t qy_32 = svld1_s8(ph32, y0->qs);\n                          svint8_t qy_64 = svld1_s8(pl32, y0->qs + 2);\n\n                    qy_64 = svadd_s8_x(svptrue_b8(), qy_32, qy_64);\n\n                    // scale creation\n                    const float32_t deq1 = GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d);\n                    const float32_t deq2 = GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d);\n\n                    // duplicate deq1 in first half of vector and deq2 in second half of vector\n                    const svfloat32_t temp = svdup_f32_m(svdup_f32_z(ph8, deq1), pl8, deq2);\n\n                    const svfloat32_t sumvt = svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx_64, qy_64));\n\n                    sumv00 = svmla_f32_m(svptrue_b32(), sumv00, sumvt, temp);\n                }\n\n                sumf = svaddv_f32(svptrue_b32(), sumv00);\n                break;\n            }\n        default:\n            assert(false && \"Unsupported vector length\");\n            break;\n    }\n#elif defined(__ARM_NEON)\n    float32x4_t sumv0 = vdupq_n_f32(0.0f);\n    float32x4_t sumv1 = vdupq_n_f32(0.0f);\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q8_0 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q8_0 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        const int8x16_t x0_0 = vld1q_s8(x0->qs);\n        const int8x16_t x0_1 = vld1q_s8(x0->qs + 16);\n        const int8x16_t x1_0 = vld1q_s8(x1->qs);\n        const int8x16_t x1_1 = vld1q_s8(x1->qs + 16);\n\n        // load y\n        const int8x16_t y0_0 = vld1q_s8(y0->qs);\n        const int8x16_t y0_1 = vld1q_s8(y0->qs + 16);\n        const int8x16_t y1_0 = vld1q_s8(y1->qs);\n        const int8x16_t y1_1 = vld1q_s8(y1->qs + 16);\n\n        sumv0 = vmlaq_n_f32(sumv0, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), x0_0, y0_0),\n                        ggml_vdotq_s32(vdupq_n_s32(0), x0_1, y0_1))), GGML_CPU_FP16_TO_FP32(x0->d)*GGML_CPU_FP16_TO_FP32(y0->d));\n\n        sumv1 = vmlaq_n_f32(sumv1, vcvtq_f32_s32(vaddq_s32(\n                        ggml_vdotq_s32(vdupq_n_s32(0), x1_0, y1_0),\n                        ggml_vdotq_s32(vdupq_n_s32(0), x1_1, y1_1))), GGML_CPU_FP16_TO_FP32(x1->d)*GGML_CPU_FP16_TO_FP32(y1->d));\n    }\n\n    sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi = 0;\n\n        for (int j = 0; j < qk; j++) {\n            sumi += x[ib].qs[j]*y[ib].qs[j];\n        }\n\n        sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq1_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n    float sumf = 0.0f;\n\n    uint8_t k_shift[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27};\n\n    const uint8x16_t shift = vld1q_u8(k_shift);\n\n    for (int i = 0; i < nb; ++i) {\n#if defined(__ARM_FEATURE_DOTPROD)\n        int32x4_t sumi0 = vdupq_n_s32(0);\n        int32x4_t sumi1 = vdupq_n_s32(0);\n#else\n        int16x8_t sumi0 = vdupq_n_s16(0);\n        int16x8_t sumi1 = vdupq_n_s16(0);\n#endif\n\n        // first 32 bytes of 5 elements\n        {\n            uint8x16_t qx0 = vld1q_u8(x[i].qs + 0);\n            uint8x16_t qx1 = vld1q_u8(x[i].qs + 16);\n            uint8x16_t qx2 = vmulq_u8(qx0, vdupq_n_u8(3));\n            uint8x16_t qx3 = vmulq_u8(qx1, vdupq_n_u8(3));\n            uint8x16_t qx4 = vmulq_u8(qx0, vdupq_n_u8(9));\n            uint8x16_t qx5 = vmulq_u8(qx1, vdupq_n_u8(9));\n            uint8x16_t qx6 = vmulq_u8(qx0, vdupq_n_u8(27));\n            uint8x16_t qx7 = vmulq_u8(qx1, vdupq_n_u8(27));\n            uint8x16_t qx8 = vmulq_u8(qx0, vdupq_n_u8(81));\n            uint8x16_t qx9 = vmulq_u8(qx1, vdupq_n_u8(81));\n\n            // multiply by 3 and keep the 2 bits above 8 bits\n            int8x16_t sqx0 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx0, vshrq_n_u8(qx0, 1)), 6));\n            int8x16_t sqx1 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx1, vshrq_n_u8(qx1, 1)), 6));\n            int8x16_t sqx2 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx2, vshrq_n_u8(qx2, 1)), 6));\n            int8x16_t sqx3 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx3, vshrq_n_u8(qx3, 1)), 6));\n            int8x16_t sqx4 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx4, vshrq_n_u8(qx4, 1)), 6));\n            int8x16_t sqx5 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx5, vshrq_n_u8(qx5, 1)), 6));\n            int8x16_t sqx6 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx6, vshrq_n_u8(qx6, 1)), 6));\n            int8x16_t sqx7 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx7, vshrq_n_u8(qx7, 1)), 6));\n            int8x16_t sqx8 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx8, vshrq_n_u8(qx8, 1)), 6));\n            int8x16_t sqx9 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx9, vshrq_n_u8(qx9, 1)), 6));\n\n            const int8x16_t qy0 = vld1q_s8(y[i].qs +   0);\n            const int8x16_t qy1 = vld1q_s8(y[i].qs +  16);\n            const int8x16_t qy2 = vld1q_s8(y[i].qs +  32);\n            const int8x16_t qy3 = vld1q_s8(y[i].qs +  48);\n            const int8x16_t qy4 = vld1q_s8(y[i].qs +  64);\n            const int8x16_t qy5 = vld1q_s8(y[i].qs +  80);\n            const int8x16_t qy6 = vld1q_s8(y[i].qs +  96);\n            const int8x16_t qy7 = vld1q_s8(y[i].qs + 112);\n            const int8x16_t qy8 = vld1q_s8(y[i].qs + 128);\n            const int8x16_t qy9 = vld1q_s8(y[i].qs + 144);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n            sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n            sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n            sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n            sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n            sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n            sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n            sumi0 = vdotq_s32(sumi0, sqx6, qy6);\n            sumi1 = vdotq_s32(sumi1, sqx7, qy7);\n            sumi0 = vdotq_s32(sumi0, sqx8, qy8);\n            sumi1 = vdotq_s32(sumi1, sqx9, qy9);\n#else\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx6), vget_low_s8(qy6));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx6), vget_high_s8(qy6));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx7), vget_low_s8(qy7));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx7), vget_high_s8(qy7));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx8), vget_low_s8(qy8));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx8), vget_high_s8(qy8));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx9), vget_low_s8(qy9));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx9), vget_high_s8(qy9));\n#endif\n        }\n\n        // last 16 bytes of 5-element, along with the 4 bytes of 4 elements\n        {\n            uint8x16_t qx0 = vld1q_u8(x[i].qs + 32);\n            uint8x16_t qx1 = vmulq_u8(qx0, vdupq_n_u8(3));\n            uint8x16_t qx2 = vmulq_u8(qx0, vdupq_n_u8(9));\n            uint8x16_t qx3 = vmulq_u8(qx0, vdupq_n_u8(27));\n            uint8x16_t qx4 = vmulq_u8(qx0, vdupq_n_u8(81));\n            uint32_t qh;\n            memcpy(&qh, x[i].qh, sizeof(qh)); // potentially unaligned\n            uint8x16_t qx5 = vreinterpretq_u8_u32(vdupq_n_u32(qh));\n            qx5 = vmulq_u8(qx5, shift);\n\n            // multiply by 3 and keep the 2 bits above 8 bits\n            int8x16_t sqx0 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx0, vshrq_n_u8(qx0, 1)), 6));\n            int8x16_t sqx1 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx1, vshrq_n_u8(qx1, 1)), 6));\n            int8x16_t sqx2 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx2, vshrq_n_u8(qx2, 1)), 6));\n            int8x16_t sqx3 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx3, vshrq_n_u8(qx3, 1)), 6));\n            int8x16_t sqx4 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx4, vshrq_n_u8(qx4, 1)), 6));\n            int8x16_t sqx5 = vreinterpretq_s8_u8(vshrq_n_u8(vhaddq_u8(qx5, vshrq_n_u8(qx5, 1)), 6));\n\n            const int8x16_t qy0 = vld1q_s8(y[i].qs + 160);\n            const int8x16_t qy1 = vld1q_s8(y[i].qs + 176);\n            const int8x16_t qy2 = vld1q_s8(y[i].qs + 192);\n            const int8x16_t qy3 = vld1q_s8(y[i].qs + 208);\n            const int8x16_t qy4 = vld1q_s8(y[i].qs + 224);\n            const int8x16_t qy5 = vld1q_s8(y[i].qs + 240);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n            sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n            sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n            sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n            sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n            sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n            sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n#else\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n#endif\n        }\n\n        const int16x8_t ysum0 = vld1q_s16(y[i].bsums);\n        const int16x8_t ysum1 = vld1q_s16(y[i].bsums + 8);\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n#if defined(__ARM_FEATURE_DOTPROD)\n        sumi0 = vaddq_s32(sumi0, sumi1);\n        sumi0 = vsubq_s32(sumi0, vpaddlq_s16(vaddq_s16(ysum0, ysum1)));\n\n        sumf += d * (float) vaddvq_s32(sumi0);\n#else\n        sumi0 = vaddq_s16(sumi0, sumi1);\n        sumi0 = vsubq_s16(sumi0, vaddq_s16(ysum0, ysum1));\n\n        sumf += d * (float) vaddlvq_s16(sumi0);\n#endif\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq2_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n    float sumf = 0.0f;\n\n    const uint8x16_t m3 = vdupq_n_u8(3);\n\n    for (int i = 0; i < nb; ++i) {\n#if defined(__ARM_FEATURE_DOTPROD)\n        int32x4_t sumi0 = vdupq_n_s32(0);\n        int32x4_t sumi1 = vdupq_n_s32(0);\n#else\n        int16x8_t sumi0 = vdupq_n_s16(0);\n        int16x8_t sumi1 = vdupq_n_s16(0);\n#endif\n\n        for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n            uint8x16_t qx0 = vld1q_u8(x[i].qs + j);\n            uint8x16_t qx1 = vld1q_u8(x[i].qs + j + 16);\n            uint8x16_t qx2 = vshrq_n_u8(qx0, 2);\n            uint8x16_t qx3 = vshrq_n_u8(qx1, 2);\n            uint8x16_t qx4 = vshrq_n_u8(qx0, 4);\n            uint8x16_t qx5 = vshrq_n_u8(qx1, 4);\n            uint8x16_t qx6 = vshrq_n_u8(qx0, 6);\n            uint8x16_t qx7 = vshrq_n_u8(qx1, 6);\n\n            int8x16_t sqx0 = vreinterpretq_s8_u8(vandq_u8(qx0, m3));\n            int8x16_t sqx1 = vreinterpretq_s8_u8(vandq_u8(qx1, m3));\n            int8x16_t sqx2 = vreinterpretq_s8_u8(vandq_u8(qx2, m3));\n            int8x16_t sqx3 = vreinterpretq_s8_u8(vandq_u8(qx3, m3));\n            int8x16_t sqx4 = vreinterpretq_s8_u8(vandq_u8(qx4, m3));\n            int8x16_t sqx5 = vreinterpretq_s8_u8(vandq_u8(qx5, m3));\n            int8x16_t sqx6 = vreinterpretq_s8_u8(vandq_u8(qx6, m3));\n            int8x16_t sqx7 = vreinterpretq_s8_u8(vandq_u8(qx7, m3));\n\n            const int8x16_t qy0 = vld1q_s8(y[i].qs + j*4 +   0);\n            const int8x16_t qy1 = vld1q_s8(y[i].qs + j*4 +  16);\n            const int8x16_t qy2 = vld1q_s8(y[i].qs + j*4 +  32);\n            const int8x16_t qy3 = vld1q_s8(y[i].qs + j*4 +  48);\n            const int8x16_t qy4 = vld1q_s8(y[i].qs + j*4 +  64);\n            const int8x16_t qy5 = vld1q_s8(y[i].qs + j*4 +  80);\n            const int8x16_t qy6 = vld1q_s8(y[i].qs + j*4 +  96);\n            const int8x16_t qy7 = vld1q_s8(y[i].qs + j*4 + 112);\n\n#if defined(__ARM_FEATURE_DOTPROD)\n            sumi0 = vdotq_s32(sumi0, sqx0, qy0);\n            sumi1 = vdotq_s32(sumi1, sqx1, qy1);\n            sumi0 = vdotq_s32(sumi0, sqx2, qy2);\n            sumi1 = vdotq_s32(sumi1, sqx3, qy3);\n            sumi0 = vdotq_s32(sumi0, sqx4, qy4);\n            sumi1 = vdotq_s32(sumi1, sqx5, qy5);\n            sumi0 = vdotq_s32(sumi0, sqx6, qy6);\n            sumi1 = vdotq_s32(sumi1, sqx7, qy7);\n#else\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx0), vget_low_s8(qy0));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx0), vget_high_s8(qy0));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx1), vget_low_s8(qy1));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx1), vget_high_s8(qy1));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx2), vget_low_s8(qy2));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx2), vget_high_s8(qy2));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx3), vget_low_s8(qy3));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx3), vget_high_s8(qy3));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx4), vget_low_s8(qy4));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx4), vget_high_s8(qy4));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx5), vget_low_s8(qy5));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx5), vget_high_s8(qy5));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx6), vget_low_s8(qy6));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx6), vget_high_s8(qy6));\n            sumi0 = vmlal_s8(sumi0, vget_low_s8(sqx7), vget_low_s8(qy7));\n            sumi1 = vmlal_s8(sumi1, vget_high_s8(sqx7), vget_high_s8(qy7));\n#endif\n        }\n\n        const int16x8_t ysum0 = vld1q_s16(y[i].bsums);\n        const int16x8_t ysum1 = vld1q_s16(y[i].bsums + 8);\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n#if defined(__ARM_FEATURE_DOTPROD)\n        sumi0 = vaddq_s32(sumi0, sumi1);\n        sumi0 = vsubq_s32(sumi0, vpaddlq_s16(vaddq_s16(ysum0, ysum1)));\n\n        sumf += d * (float) vaddvq_s32(sumi0);\n#else\n        sumi0 = vaddq_s16(sumi0, sumi1);\n        sumi0 = vsubq_s16(sumi0, vaddq_s16(ysum0, ysum1));\n\n        sumf += d * (float) vaddlvq_s16(sumi0);\n#endif\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#ifdef __ARM_FEATURE_SVE\n    const int vector_length = svcntb()*8;\n    const svuint8_t m3s = svdup_n_u8(0x3);\n    const svuint32_t m4s = svdup_n_u32(0xF);\n    const svint32_t vzero_sv = svdup_n_s32(0);\n    svfloat32_t acc_sum = svdup_n_f32(0);\n    svbool_t pred_s32 = svptrue_pat_b32(SV_VL4);\n\n    switch (vector_length) {\n        case 128:\n            for (int i = 0; i < nb; ++i) {\n                const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n                svfloat32_t d_broad = svdup_n_f32((float32_t)d);\n                const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n                svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin);\n\n                const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n                const int8_t  * GGML_RESTRICT q8_sv = y[i].qs;\n                const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n                svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc);\n                const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n                mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+4);\n                const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n                svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums);\n                svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+4);\n\n                const svint32_t s0 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_2, q8sums_sv_2));\n\n                mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+8);\n                const svint32_t mins_sv_3 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n                mins_and_scales_sve = svld1ub_u32(svptrue_b32(), sc+12);\n                const svint32_t mins_sv_4 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), mins_and_scales_sve, 4));\n\n                q8sums_sv_1 = svld1sh_s32(svptrue_b32(), y[i].bsums+8);\n                q8sums_sv_2 = svld1sh_s32(svptrue_b32(), y[i].bsums+12);\n\n                svint32_t s1 = svadd_s32_x(svptrue_b32(), svmul_s32_x(svptrue_b32(), mins_sv_3, q8sums_sv_1), svmul_s32_x(svptrue_b32(), mins_sv_4, q8sums_sv_2));\n\n                svfloat32_t temp = svcvt_f32_s32_x(svptrue_b32(), svadd_s32_x(svptrue_b32(), s0, s1));\n\n                acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, temp, dmin_broad);\n\n                svint32_t sumi1 = svdup_n_s32(0);\n\n                {\n                    const svuint8_t q2bits_1 = svld1_u8(svptrue_b8(), q2);\n                    svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_1, m3s));\n                    svint8_t q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n                    const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc), m4s));\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 0));\n\n                    const svuint8_t q2bits_3 = svld1_u8(svptrue_b8(), q2+16);\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_3, m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 1));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 2));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv, 3));\n\n\n                    const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+4), m4s));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 0));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 1));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_1, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 2));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_3, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_1, 3));\n\n                    //-------------------------------\n\n                    q2 += 32;\n                    const svint32_t scales_sv_2 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+8), m4s));\n                    const svuint8_t q2bits_2 = svld1_u8(svptrue_b8(), q2);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_2, m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 0));\n\n                    const svuint8_t q2bits_4 = svld1_u8(svptrue_b8(), q2+16);\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), q2bits_4, m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 1));\n\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 2));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_2, 3));\n\n\n                    const svint32_t scales_sv_3 = svreinterpret_s32_u32(svand_u32_m(svptrue_b32(), svld1ub_u32(svptrue_b32(), sc+12), m4s));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 0));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 1));\n\n\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_2, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 2));\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q2bits_4, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                    sumi1 = svmla_s32_m(svptrue_b32(), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), svdup_lane_s32(scales_sv_3, 3));\n                }\n                acc_sum = svmla_f32_m(svptrue_b32(), acc_sum, svcvt_f32_s32_x(svptrue_b32(), sumi1), d_broad);\n            }\n            *s = svaddv_f32(svptrue_b32(), acc_sum);\n            break;\n\n        case 256:\n        case 512:\n            for (int i = 0; i < nb; ++i) {\n                const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n                svfloat32_t d_broad = svdup_n_f32((float32_t)d);\n                const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n                svfloat32_t dmin_broad = svdup_n_f32((float32_t)dmin);\n\n                const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n                const int8_t  * GGML_RESTRICT q8_sv = y[i].qs;\n                const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n                const svuint32_t mins_and_scales_sve = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc); sc += 8;\n                const svint32_t scales_sv = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, m4s));\n                const svint32_t mins_sv_1 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve, 4));\n                svint32_t q8sums_sv_1 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums);\n\n                const svuint32_t mins_and_scales_sve_1 = svld1ub_u32(svptrue_pat_b32(SV_VL8), sc);\n                const svint32_t scales_sv_1 = svreinterpret_s32_u32(svand_u32_m(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, m4s));\n                const svint32_t mins_sv_2 = svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_pat_b32(SV_VL8), mins_and_scales_sve_1, 4));\n\n                svint32_t q8sums_sv_2 = svld1sh_s32(svptrue_pat_b32(SV_VL8), y[i].bsums+8);\n\n                svfloat32_t temp = svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), svadd_s32_x(svptrue_pat_b32(SV_VL8), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_1, q8sums_sv_1), svmul_s32_x(svptrue_pat_b32(SV_VL8), mins_sv_2, q8sums_sv_2)));\n\n                acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, temp, dmin_broad);\n\n                svint32_t sumi1 = svdup_n_s32(0);\n\n                {\n                    const svuint8_t q2bits_1 = svld1_u8(svptrue_pat_b8(SV_VL32), q2);\n                    svint8_t q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_1, m3s));\n                    svint8_t q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    svint32_t scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 0), svdup_lane_s32(scales_sv, 1));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    svint32_t scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 2), svdup_lane_s32(scales_sv, 3));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(svdup_n_s32(0), q2bytes_sv, q8bytes_sv), scale_2);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv, 4), svdup_lane_s32(scales_sv, 5));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_1, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv, 6), svdup_lane_s32(scales_sv, 7));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n\n                    q2 += 32;\n\n                    const svuint8_t q2bits_2 = svld1_u8(svptrue_pat_b8(SV_VL32), q2);\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q2bits_2, m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 0), svdup_lane_s32(scales_sv_1, 1));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 2), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 2), svdup_lane_s32(scales_sv_1, 3));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 4), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_1 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 4), svdup_lane_s32(scales_sv_1, 5));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_1);\n\n                    q2bytes_sv = svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q2bits_2, 6), m3s));\n                    q8bytes_sv = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                    scale_2 = svsel(pred_s32, svdup_lane_s32(scales_sv_1, 6), svdup_lane_s32(scales_sv_1, 7));\n                    sumi1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(vzero_sv, q2bytes_sv, q8bytes_sv), scale_2);\n                }\n                acc_sum = svmla_f32_m(svptrue_pat_b32(SV_VL8), acc_sum, svcvt_f32_s32_x(svptrue_pat_b32(SV_VL8), sumi1), d_broad);\n            }\n            *s = svaddv_f32(svptrue_pat_b32(SV_VL8), acc_sum);\n            break;\n\n        default:\n            assert(false && \"Unsupported vector length\");\n            break;\n    }\n\n#elif __ARM_NEON\n    const uint8x16_t m3 = vdupq_n_u8(0x3);\n    const uint8x16_t m4 = vdupq_n_u8(0xF);\n\n    const int32x4_t vzero = vdupq_n_s32(0);\n\n    ggml_int8x16x2_t q2bytes;\n    uint8_t aux[16];\n\n    float sum = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        const uint8_t * GGML_RESTRICT sc = x[i].scales;\n\n        const uint8x16_t mins_and_scales = vld1q_u8(sc);\n        const uint8x16_t scales = vandq_u8(mins_and_scales, m4);\n        vst1q_u8(aux, scales);\n\n        const uint8x16_t mins = vshrq_n_u8(mins_and_scales, 4);\n        const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);\n        const ggml_int16x8x2_t mins16 = {{vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins)))}};\n        const int32x4_t s0 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[0]), vget_low_s16 (q8sums.val[0])),\n                                       vmull_s16(vget_high_s16(mins16.val[0]), vget_high_s16(q8sums.val[0])));\n        const int32x4_t s1 = vaddq_s32(vmull_s16(vget_low_s16 (mins16.val[1]), vget_low_s16 (q8sums.val[1])),\n                                       vmull_s16(vget_high_s16(mins16.val[1]), vget_high_s16(q8sums.val[1])));\n        sum += dmin * vaddvq_s32(vaddq_s32(s0, s1));\n\n        int isum = 0;\n        int is = 0;\n\n// We use this macro instead of a function call because for some reason\n// the code runs 2-3% slower, even if the function is declared inline\n#define MULTIPLY_ACCUM_WITH_SCALE(index)\\\n        isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * aux[is+(index)];\\\n        isum += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * aux[is+1+(index)];\n\n#define SHIFT_MULTIPLY_ACCUM_WITH_SCALE(shift, index)\\\n        q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\\\n        q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[0], (shift)), m3));\\\n        q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.val[1], (shift)), m3));\\\n        MULTIPLY_ACCUM_WITH_SCALE((index));\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            const ggml_uint8x16x2_t q2bits = ggml_vld1q_u8_x2(q2); q2 += 32;\n\n            ggml_int8x16x2_t q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n            q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[0], m3));\n            q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(q2bits.val[1], m3));\n\n            MULTIPLY_ACCUM_WITH_SCALE(0);\n\n            SHIFT_MULTIPLY_ACCUM_WITH_SCALE(2, 2);\n            SHIFT_MULTIPLY_ACCUM_WITH_SCALE(4, 4);\n            SHIFT_MULTIPLY_ACCUM_WITH_SCALE(6, 6);\n\n            is += 8;\n        }\n\n        sum += d * isum;\n    }\n\n    *s = sum;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_FEATURE_SVE)\n\n    uint32_t aux[3];\n    uint32_t utmp[4];\n\n    const int8_t m32 = 32;\n    const int vector_length = svcntb()*8;\n    const svuint8_t m3b_sv = svdup_n_u8(0x3);\n    const svint32_t vzero_sv = svdup_n_s32(0);\n\n    const svuint8_t m0_sv = svdup_n_u8(1);\n    const svuint8_t m1_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 1);\n    const svuint8_t m2_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 2);\n    const svuint8_t m3_sv = svlsl_n_u8_x(svptrue_b8(), m0_sv, 3);\n\n    float sum = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q3_sv = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh_sv = x[i].hmask;\n        const int8_t  * GGML_RESTRICT q8_sv = y[i].qs;\n\n        // Set up scales\n        memcpy(aux, x[i].scales, 12);\n        utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n        utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n        utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n        utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n        int8_t * scale = (int8_t *)utmp;\n\n        for (int j = 0; j < 16; ++j) scale[j] -= m32;\n\n        switch (vector_length) {\n            case 128:\n                {\n                    svuint8_t qhbits_sv_1 = svld1_u8(svptrue_b8(), qh_sv);\n                    svuint8_t qhbits_sv_2 = svld1_u8(svptrue_b8(), qh_sv+16);\n                    svuint8_t q3h_sv;\n\n                    svint32_t sumi1_1 = svdup_n_s32(0);\n                    svint8_t q3bytes_sv;\n\n                    for (int j = 0; j < QK_K/128; ++j) {\n\n                        const svuint8_t q3bits_sv = svld1_u8(svptrue_b8(), q3_sv); q3_sv += 16;\n                        const svuint8_t q3bits_sv_1 = svld1_u8(svptrue_b8(), q3_sv); q3_sv += 16;\n                        svint8_t q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n                        svint8_t q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m0_sv, qhbits_sv_1), 2);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), q3bits_sv, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[0]));\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m0_sv, qhbits_sv_2), 2);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), q3bits_sv_1, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[1]));\n\n                        q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n                        q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m1_sv, qhbits_sv_1), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[2]));\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m1_sv, qhbits_sv_2), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[3]));\n\n\n                        scale += 4;\n                        q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n                        q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                        q3h_sv = svbic_u8_x(svptrue_b8(), m2_sv, qhbits_sv_1);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[0]));\n\n                        q3h_sv = svbic_u8_x(svptrue_b8(), m2_sv, qhbits_sv_2);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[1]));\n\n\n                        q8bytes_1_sv_1 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n                        q8bytes_1_sv_2 = svld1_s8(svptrue_b8(), q8_sv); q8_sv += 16;\n\n                        q3h_sv = svlsr_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m3_sv, qhbits_sv_1), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), svdup_n_s32((int32_t)scale[2]));\n\n                        q3h_sv = svlsr_n_u8_x(svptrue_b8(), svbic_u8_x(svptrue_b8(), m3_sv, qhbits_sv_2), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_b8(), svreinterpret_s8_u8(svand_u8_m(svptrue_b8(), svlsr_n_u8_x(svptrue_b8(), q3bits_sv_1, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        sumi1_1 = svmla_s32_m(svptrue_b32(), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), svdup_n_s32((int32_t)scale[3]));\n\n                        if (j == 0) {\n                            qhbits_sv_1 = svlsr_n_u8_x(svptrue_b8(), qhbits_sv_1, 4);\n                            qhbits_sv_2 = svlsr_n_u8_x(svptrue_b8(), qhbits_sv_2, 4);\n                        }\n\n                        scale += 4;\n                    }\n\n                    sum += d * (svaddv_s32(svptrue_b32(), sumi1_1));\n                } break;\n            case 256:\n            case 512:\n                {\n                    svuint8_t qhbits_sv = svld1_u8(svptrue_pat_b8(SV_VL32), qh_sv);\n                    svuint8_t q3h_sv;\n\n                    svint32_t sumi1_1 = svdup_n_s32(0);\n                    svint8_t q3bytes_sv;\n\n                    for (int j = 0; j < QK_K/128; ++j) {\n\n                        const svuint8_t q3bits_sv = svld1_u8(svptrue_pat_b8(SV_VL32), q3_sv); q3_sv += 32;\n                        svint8_t q8bytes_1_sv_1 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n                        svint8_t q8bytes_1_sv_2 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m0_sv, qhbits_sv), 2);\n                        q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), q3bits_sv, m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n\n                        svint32_t scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[0]), svdup_n_s32((int32_t)scale[1]));\n                        sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), scale_1);\n\n                        q3h_sv = svlsl_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m1_sv, qhbits_sv), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 2), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[2]), svdup_n_s32((int32_t)scale[3]));\n                        sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), scale_1);\n\n                        scale += 4;\n                        q8bytes_1_sv_1 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n                        q8bytes_1_sv_2 = svld1_s8(svptrue_pat_b8(SV_VL32), q8_sv); q8_sv += 32;\n\n                        q3h_sv = svbic_u8_x(svptrue_pat_b8(SV_VL32), m2_sv, qhbits_sv);\n                        q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 4), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[0]), svdup_n_s32((int32_t)scale[1]));\n                        sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_1), scale_1);\n\n                        q3h_sv = svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), svbic_u8_x(svptrue_pat_b8(SV_VL32), m3_sv, qhbits_sv), 1);\n                        q3bytes_sv = svsub_s8_x(svptrue_pat_b8(SV_VL32), svreinterpret_s8_u8(svand_u8_m(svptrue_pat_b8(SV_VL32), svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q3bits_sv, 6), m3b_sv)), svreinterpret_s8_u8(q3h_sv));\n\n                        scale_1 = svsel_s32(svptrue_pat_b32(SV_VL4), svdup_n_s32((int32_t)scale[2]), svdup_n_s32((int32_t)scale[3]));\n                        sumi1_1 = svmla_s32_m(svptrue_pat_b32(SV_VL8), sumi1_1, svdot_s32(vzero_sv, q3bytes_sv, q8bytes_1_sv_2), scale_1);\n\n                        if (j == 0) {\n                            qhbits_sv = svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), qhbits_sv, 4);\n                        }\n\n                        scale += 4;\n                    }\n\n                    sum += d * (svaddv_s32(svptrue_pat_b32(SV_VL8), sumi1_1));\n                } break;\n            default:\n                assert(false && \"Unsupported vector length\");\n                break;\n        }\n    }\n    *s = sum;\n\n#elif __ARM_NEON\n\n    uint32_t aux[3];\n    uint32_t utmp[4];\n\n    const uint8x16_t m3b = vdupq_n_u8(0x3);\n    const int32x4_t  vzero = vdupq_n_s32(0);\n\n    const uint8x16_t m0 = vdupq_n_u8(1);\n    const uint8x16_t m1 = vshlq_n_u8(m0, 1);\n    const uint8x16_t m2 = vshlq_n_u8(m0, 2);\n    const uint8x16_t m3 = vshlq_n_u8(m0, 3);\n    const int8_t m32 = 32;\n\n    ggml_int8x16x4_t q3bytes;\n\n    float sum = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].hmask;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);\n\n        ggml_uint8x16x4_t q3h;\n\n        int32_t isum = 0;\n\n        // Set up scales\n        memcpy(aux, x[i].scales, 12);\n        utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n        utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n        utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n        utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n        int8_t * scale = (int8_t *)utmp;\n        for (int j = 0; j < 16; ++j) scale[j] -= m32;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const ggml_uint8x16x2_t q3bits = ggml_vld1q_u8_x2(q3); q3 += 32;\n            const ggml_int8x16x4_t q8bytes_1 = ggml_vld1q_s8_x4(q8); q8 += 64;\n            const ggml_int8x16x4_t q8bytes_2 = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            q3h.val[0] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[0]), 2);\n            q3h.val[1] = vshlq_n_u8(vbicq_u8(m0, qhbits.val[1]), 2);\n            q3h.val[2] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[0]), 1);\n            q3h.val[3] = vshlq_n_u8(vbicq_u8(m1, qhbits.val[1]), 1);\n\n            q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[0], m3b)), vreinterpretq_s8_u8(q3h.val[0]));\n            q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q3bits.val[1], m3b)), vreinterpretq_s8_u8(q3h.val[1]));\n            q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 2), m3b)), vreinterpretq_s8_u8(q3h.val[2]));\n            q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 2), m3b)), vreinterpretq_s8_u8(q3h.val[3]));\n\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_1.val[0])) * scale[0];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_1.val[1])) * scale[1];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_1.val[2])) * scale[2];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_1.val[3])) * scale[3];\n\n            scale += 4;\n\n            q3h.val[0] = vbicq_u8(m2, qhbits.val[0]);\n            q3h.val[1] = vbicq_u8(m2, qhbits.val[1]);\n            q3h.val[2] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[0]), 1);\n            q3h.val[3] = vshrq_n_u8(vbicq_u8(m3, qhbits.val[1]), 1);\n\n            q3bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 4), m3b)), vreinterpretq_s8_u8(q3h.val[0]));\n            q3bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 4), m3b)), vreinterpretq_s8_u8(q3h.val[1]));\n            q3bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[0], 6), m3b)), vreinterpretq_s8_u8(q3h.val[2]));\n            q3bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.val[1], 6), m3b)), vreinterpretq_s8_u8(q3h.val[3]));\n\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes_2.val[0])) * scale[0];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes_2.val[1])) * scale[1];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes_2.val[2])) * scale[2];\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes_2.val[3])) * scale[3];\n\n            scale += 4;\n\n            if (j == 0) {\n                qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 4);\n                qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 4);\n            }\n\n        }\n        sum += d * isum;\n\n    }\n\n    *s = sum;\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\n#ifdef __ARM_FEATURE_SVE\nstatic inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t * vx_scales) {\n    const svbool_t pg_all   = svptrue_pat_b32(SV_VL4);\n    const svbool_t pg_false = svpfalse_b();            // 0x0000\n    const svbool_t pg_lo_8  = svwhilelt_b8_s32(0,  8); // 0x00ff\n    const svbool_t pg_odd   = svzip1_b32(pg_false, pg_lo_8);\n\n    svuint32_t vutmp_hi, vutmp_lo;\n    svuint32_t vx01 = svld1_u32(pg_lo_8, vx_scales);\n    vutmp_hi = svzip1_u32(vx01, vx01);\n    vutmp_hi = svlsr_n_u32_m(pg_odd, vutmp_hi, 2);\n    vutmp_hi = svreinterpret_u32_u64(svand_n_u64_x(pg_all, svreinterpret_u64_u32(vutmp_hi), UINT64_C(0x303030303f3f3f3f)));\n    const svuint32_t vx2 = svdup_u32(vx_scales[2]);\n    vutmp_lo = svlsr_u32_x(pg_all, vx2, svreinterpret_u32_s32(svindex_s32(-2, 2)));\n    vutmp_lo = svand_n_u32_z(pg_odd, vutmp_lo, UINT32_C(0x0f0f0f0f));\n    svuint32_t vutmp = svorr_u32_z(pg_all, vutmp_hi, vutmp_lo);\n    return vutmp;\n}\n#endif\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n#ifdef __ARM_FEATURE_MATMUL_INT8\n    assert((nrc == 2) || (nrc == 1));\n#else\n    assert(nrc == 1);\n#endif\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n#ifdef __ARM_FEATURE_SVE\n    const int vector_length = ggml_cpu_get_sve_cnt()*8;\n#endif\n\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        svbool_t pg32_2 = svptrue_pat_b32(SV_VL2);\n\n        const block_q4_K * GGML_RESTRICT vx0 = vx;\n        const block_q8_K * GGML_RESTRICT vy0 = vy;\n        const block_q4_K * GGML_RESTRICT vx1 = (const block_q4_K *) ((const uint8_t*)vx + bx);\n        const block_q8_K * GGML_RESTRICT vy1 = (const block_q8_K *) ((const uint8_t*)vy + by);\n\n        union {\n            uint32_t u32[8];\n            uint64_t u64[4];\n        } new_utmp;\n\n        svfloat32_t sumf1 = svdup_n_f32(0);\n\n        switch (vector_length) {\n            case 128:\n                {\n                    svbool_t pg_false = svpfalse_b();\n                    svbool_t pg_lo_8  = svwhilelt_b8_s32(0,  8);\n                    svbool_t vmins_mask1= svzip1_b32(pg_lo_8, pg_false);\n                    svbool_t vmins_mask2 = svzip1_b32(pg_false, pg_lo_8);\n                    svbool_t pg128_all  = svptrue_pat_b8(SV_VL16);\n                    for (int i = 0; i < nb; ++i) {\n                        svfloat32_t vy_d = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n                        svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n                        svfloat32_t svsuper_block_scales = svmul_f32_x(pg128_all, vy_d, vx_d);\n                        svfloat32_t vx_dmins = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].dmin)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].dmin)));\n                        svfloat32_t vy_dmins = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n                        svfloat32_t svdmins = svmul_n_f32_x(pg128_all, svmul_f32_x(pg128_all, vy_dmins, vx_dmins), -1);\n                        const uint8_t * GGML_RESTRICT q4_0 = vx0[i].qs;\n                        const int8_t  * GGML_RESTRICT q8_0 = vy0[i].qs;\n                        const uint8_t * GGML_RESTRICT q4_1 = vx1[i].qs;\n                        const int8_t  * GGML_RESTRICT q8_1 = vy1[i].qs;\n                        svint16_t lo = svld1_s16(pg128_all, vy0[i].bsums + 0);\n                        svint16_t hi = svld1_s16(pg128_all, vy0[i].bsums + 8);\n                        svint16_t sum_tmp1 = svuzp1_s16(lo, hi);\n                        svint16_t sum_tmp2 = svuzp2_s16(lo, hi);\n                        svint16_t svq8sums_0 = svadd_s16_x(pg128_all, sum_tmp1, sum_tmp2);\n                        lo = svld1_s16(pg128_all, vy1[i].bsums + 0);\n                        hi = svld1_s16(pg128_all, vy1[i].bsums + 8);\n                        sum_tmp1 = svuzp1(lo, hi);\n                        sum_tmp2 = svuzp2(lo, hi);\n                        svint16_t svq8sums_1 = svadd_s16_x(pg128_all, sum_tmp1, sum_tmp2);\n                        svuint32_t decoded_scales0 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx0[i].scales);\n                        svuint32_t decoded_scales1 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx1[i].scales);\n                        svuint32x2_t decoded_scales = svcreate2_u32(decoded_scales0, decoded_scales1);\n                        svst2_u32(pg128_all, new_utmp.u32, decoded_scales);\n                        svint16_t svmins8_0 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(svuzp1_u32(svld1_u32(vmins_mask1, new_utmp.u32+4), svdup_n_u32(0)))));\n                        svint16_t svmins8_1 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(svuzp2_u32(svld1_u32(vmins_mask2, new_utmp.u32+4), svdup_n_u32(0)))));\n                        svint32_t svsumfs_tmp1 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_0, svmins8_0));\n                        svint32_t svsumfs_tmp2 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_0, svmins8_1));\n                        svint32_t svsumfs_tmp3 = svtrn1_s32(svsumfs_tmp1, svsumfs_tmp2);\n                        svint32_t svsumfs_tmp4 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_1, svmins8_0));\n                        svint32_t svsumfs_tmp5 = svreinterpret_s32_s64(svdot_s64(svdup_n_s64(0), svq8sums_1, svmins8_1));\n                        svint32_t svsumfs_tmp6 = svtrn1_s32(svsumfs_tmp4, svsumfs_tmp5);\n                        svint32_t svsumfs_tmp7 = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(svsumfs_tmp3), svreinterpret_s64_s32(svsumfs_tmp6)));\n                        svint32_t svsumfs_tmp8 = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(svsumfs_tmp3), svreinterpret_s64_s32(svsumfs_tmp6)));\n                        svint32_t svsumfs_tmp = svadd_s32_x(pg128_all, svsumfs_tmp7, svsumfs_tmp8);\n                        svint32_t svscales, sumi1, sumi2;\n                        svint32_t acc_sumif1 = svdup_n_s32(0);\n                        svint32_t acc_sumif2 = svdup_n_s32(0);\n                        svint8_t q4bytes_0_l, q4bytes_0_h, q4bytes_1_l, q4bytes_1_h, l0, l1, l2, l3,\n                                 q8bytes_0_h, q8bytes_0_l, q8bytes_1_h, q8bytes_1_l, r0, r1, r2, r3;\n#pragma GCC unroll 1\n                        for (int j = 0; j < QK_K/64; ++j) {\n                            q4bytes_0_l = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0), 0xf));\n                            q4bytes_1_l = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1), 0xf));\n                            q4bytes_0_h = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0+16), 0xf));\n                            q4bytes_1_h = svreinterpret_s8_u8(svand_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1+16), 0xf));\n                            l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n                            l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n                            l2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n                            l3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n                            q8bytes_0_h = svld1_s8(pg128_all, q8_0);\n                            q8bytes_1_h = svld1_s8(pg128_all, q8_1);\n                            q8bytes_0_l = svld1_s8(pg128_all, q8_0+16);\n                            q8bytes_1_l = svld1_s8(pg128_all, q8_1+16);\n                            r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n                            r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n                            r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n                            r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n                            sumi1 = svmmla_s32(svmmla_s32(svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), r2, l2), r3, l3);\n                            svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg128_all, svlsl_n_u32_x(pg128_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-1)), 24));\n                            acc_sumif1 = svmla_s32_x(pg128_all, acc_sumif1, svscales, sumi1);\n\n                            q4bytes_0_l = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0), 4));\n                            q4bytes_1_l = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1), 4));\n                            q4bytes_0_h = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_0+16), 4));\n                            q4bytes_1_h = svreinterpret_s8_u8(svlsr_n_u8_x(pg128_all, svld1_u8(pg128_all, q4_1+16), 4));\n                            l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n                            l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_l), svreinterpret_s64_s8(q4bytes_1_l)));\n                            l2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n                            l3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q4bytes_0_h), svreinterpret_s64_s8(q4bytes_1_h)));\n                            q8bytes_0_h = svld1_s8(pg128_all, q8_0+32);\n                            q8bytes_1_h = svld1_s8(pg128_all, q8_1+32);\n                            q8bytes_0_l = svld1_s8(pg128_all, q8_0+48);\n                            q8bytes_1_l = svld1_s8(pg128_all, q8_1+48);\n                            r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n                            r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_h), svreinterpret_s64_s8(q8bytes_1_h)));\n                            r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n                            r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0_l), svreinterpret_s64_s8(q8bytes_1_l)));\n                            sumi2 = svmmla_s32(svmmla_s32(svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), r2, l2), r3, l3);\n                            svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg128_all, svlsl_n_u32_x(pg128_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-2)), 24));\n                            acc_sumif2 = svmla_s32_x(pg128_all, acc_sumif2, svscales, sumi2);\n                            q4_0 += 32; q4_1 += 32; q8_0 += 64; q8_1 += 64;\n                        }\n                        sumf1 = svmla_f32_x(pg128_all,\n                                svmla_f32_x(pg128_all,\n                                    sumf1,\n                                    svcvt_f32_x(pg128_all,\n                                        svadd_s32_x(pg128_all, acc_sumif1, acc_sumif2)),\n                                    svsuper_block_scales),\n                                svdmins,\n                                svcvt_f32_s32_x(pg128_all, svsumfs_tmp));\n                    }  //end of for nb\n                } // end of case 128\n                break;\n            case 256:\n            case 512:\n                {\n                    const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n                    const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16);\n                    const svbool_t pg256_all = svptrue_pat_b8(SV_ALL);\n                    for (int i = 0; i < nb; ++i) {\n                        const uint8_t * GGML_RESTRICT q4_0 = vx0[i].qs;\n                        const int8_t  * GGML_RESTRICT q8_0 = vy0[i].qs;\n                        const uint8_t * GGML_RESTRICT q4_1 = vx1[i].qs;\n                        const int8_t  * GGML_RESTRICT q8_1 = vy1[i].qs;\n                        svint32_t svscales, sumi1, sumi2;\n                        svint32_t acc_sumif1 = svdup_n_s32(0);\n                        svint32_t acc_sumif2 = svdup_n_s32(0);\n                        svint8_t l0, l1, l2, l3, r0, r1, r2, r3;\n                        svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n                        svfloat64_t vy_d_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n                        svfloat32_t vy_d = svreinterpret_f32_f64(svuzp1_f64(vy_d_tmp, vy_d_tmp));\n                        svfloat32_t svsuper_block_scales = svmul_f32_z(pg32_4, vy_d, vx_d);\n                        svfloat32_t vx_dmins = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].dmin)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].dmin)));\n                        svfloat64_t vy_dmins_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n                        svfloat32_t vy_dmins = svreinterpret_f32_f64(svuzp1_f64(vy_dmins_tmp, vy_dmins_tmp));\n                        svfloat32_t svdmins = svmul_n_f32_x(pg32_4, svmul_f32_x(pg32_4, vx_dmins, vy_dmins), -1);\n                        svint16_t rc1 = svuzp1_s16(svld1_s16(pg256_all, vy0[i].bsums), svld1_s16(pg256_all, vy1[i].bsums));\n                        svint16_t rc2 = svuzp2_s16(svld1_s16(pg256_all, vy0[i].bsums), svld1_s16(pg256_all, vy1[i].bsums));\n                        svint16_t svq8sums = svadd_s16_x(pg256_all, rc1, rc2);\n                        svuint32_t decoded_scales0 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx0[i].scales);\n                        svuint32_t decoded_scales1 = ggml_decode_q4scales_and_mins_for_mmla((const uint32_t *)vx1[i].scales);\n                        svuint32x2_t decoded_scales = svcreate2_u32(decoded_scales0, decoded_scales1);\n                        svst2_u32(pg8_16, new_utmp.u32, decoded_scales);\n                        svint16_t new_svq8sums_0 = svreinterpret_s16_u64(svtrn1_u64(svreinterpret_u64_s16(svq8sums), svreinterpret_u64_s16(svq8sums)));\n                        svint16_t new_svq8sums_1 = svreinterpret_s16_u64(svtrn2_u64(svreinterpret_u64_s16(svq8sums), svreinterpret_u64_s16(svq8sums)));\n                        svuint64_t new_mins_0 = svdup_u64(new_utmp.u64[2]);\n                        svuint64_t new_mins_1 = svdup_u64(new_utmp.u64[3]);\n                        svint16_t new_svmins8_0 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u64(new_mins_0)));\n                        svint16_t new_svmins8_1 = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u64(new_mins_1)));\n                        svint64_t dot_prod_0 = svdot_s64(svdup_s64(0), new_svmins8_0, new_svq8sums_0);\n                        svint64_t dot_prod_1 = svdot_s64(dot_prod_0, new_svmins8_1, new_svq8sums_1);\n                        svfloat32_t converted_dot_prod_1 = svcvt_f32_s64_x(pg256_all, dot_prod_1);\n                        svfloat32_t svsumfs_tmp = svuzp1_f32(converted_dot_prod_1, converted_dot_prod_1);\n\n#pragma GCC unroll 1\n                        for (int j = 0; j < QK_K/64; ++j) {\n                            svuint8_t q4bytes_0 = svand_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_0), 0xf);\n                            svuint8_t q4bytes_1 = svand_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_1), 0xf);\n                            svuint8_t q4bytes_2 = svlsr_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_0), 4);\n                            svuint8_t q4bytes_3 = svlsr_n_u8_x(pg256_all, svld1_u8(pg256_all, q4_1), 4);\n                            l0 = svreinterpret_s8_u64(svzip1_u64(svreinterpret_u64_u8(q4bytes_0), svreinterpret_u64_u8(q4bytes_1)));\n                            l1 = svreinterpret_s8_u64(svzip2_u64(svreinterpret_u64_u8(q4bytes_0), svreinterpret_u64_u8(q4bytes_1)));\n                            l2 = svreinterpret_s8_u64(svzip1_u64(svreinterpret_u64_u8(q4bytes_2), svreinterpret_u64_u8(q4bytes_3)));\n                            l3 = svreinterpret_s8_u64(svzip2_u64(svreinterpret_u64_u8(q4bytes_2), svreinterpret_u64_u8(q4bytes_3)));\n                            svint8_t q8bytes_0 = svld1_s8(pg256_all, q8_0);\n                            svint8_t q8bytes_1 = svld1_s8(pg256_all, q8_1);\n                            svint8_t q8bytes_2 = svld1_s8(pg256_all, q8_0+32);\n                            svint8_t q8bytes_3 = svld1_s8(pg256_all, q8_1+32);\n                            r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                            r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                            r2 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_2), svreinterpret_s64_s8(q8bytes_3)));\n                            r3 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_2), svreinterpret_s64_s8(q8bytes_3)));\n                            sumi1 = svmmla(svmmla(svdup_n_s32(0), r0, l0), r1, l1);\n                            svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg256_all, svlsl_n_u32_x(pg256_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-1)), 24));\n                            acc_sumif1 = svmla_s32_x(pg256_all, acc_sumif1, svscales, sumi1);\n                            sumi2 = svmmla(svmmla(svdup_n_s32(0), r2, l2), r3, l3);\n                            svscales = svreinterpret_s32_u32(svlsr_n_u32_x(pg256_all, svlsl_n_u32_x(pg256_all, svreinterpret_u32_u64(svdup_n_u64(new_utmp.u64[j/2])), 8*(4-2*(j%2)-2)), 24));\n                            acc_sumif2 = svmla_s32_x(pg256_all, acc_sumif2, svscales, sumi2);\n                            q4_0 += 32; q4_1 += 32; q8_0 += 64; q8_1 += 64;\n                        }\n                        svint32_t acc_sumif = svadd_s32_x(pg256_all, acc_sumif1, acc_sumif2);\n                        svint32_t swap_acc_sumif = svext_s32(acc_sumif, acc_sumif, 4);\n                        acc_sumif = svadd_s32_x(pg32_4, acc_sumif, swap_acc_sumif);\n                        sumf1 = svmla_f32_x(pg32_4,\n                                svmla_f32_x(pg32_4,\n                                    sumf1,\n                                    svcvt_f32_x(pg32_4, acc_sumif),\n                                    svsuper_block_scales),\n                                svdmins,\n                                svsumfs_tmp);\n                    } // end of for nb\n                } // end of case 256-512\n                break;\n            default:\n                assert(false && \"Unsupported vector length\");\n                break;\n        }\n\n        svst1_f32(pg32_2, s, sumf1);\n        svst1_f32(pg32_2, s + bs, svreinterpret_f32_u8(svext_u8(svreinterpret_u8_f32(sumf1), svdup_n_u8(0), 8)));\n\n        return;\n    }\n#elif defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const block_q4_K * GGML_RESTRICT x0 = x;\n        const block_q4_K * GGML_RESTRICT x1 = (const block_q4_K *) ((const uint8_t *)vx + bx);\n        const block_q8_K * GGML_RESTRICT y0 = y;\n        const block_q8_K * GGML_RESTRICT y1 = (const block_q8_K *) ((const uint8_t *)vy + by);\n\n        const uint8x16_t m4b = vdupq_n_u8(0x0f);\n\n        float32x4_t vfsum = vdupq_n_f32(0.0f);\n\n        for (int i = 0; i < nb; ++i, ++x0, ++x1, ++y0, ++y1) {\n            const uint8_t * GGML_RESTRICT qx0 = x0->qs;\n            const uint8_t * GGML_RESTRICT qx1 = x1->qs;\n            const  int8_t * GGML_RESTRICT qy0 = y0->qs;\n            const  int8_t * GGML_RESTRICT qy1 = y1->qs;\n\n            // decode scales and mins\n            int8_t x0_scales[8], x1_scales[8];\n            int16x8_t x0_mins, x1_mins;\n            {\n                uint32_t scales_mins[3];\n                memcpy(scales_mins, x0->scales, 12);\n                const uint32_t mins_0_3 = scales_mins[1] & kmask1;\n                const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4);\n                const uint32x2_t mins = {mins_0_3, mins_4_7};\n                x0_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins)));\n                uint32_t scales[2];\n                scales[0] = scales_mins[0] & kmask1; // scales 0~3\n                scales[1] = (scales_mins[2] & kmask2) | (((scales_mins[0] >> 6) & kmask3) << 4); // scales 4~7\n                memcpy(x0_scales, scales, 8);\n            }\n            {\n                uint32_t scales_mins[3];\n                memcpy(scales_mins, x1->scales, 12);\n                const uint32_t mins_0_3 = scales_mins[1] & kmask1;\n                const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4);\n                const uint32x2_t mins = {mins_0_3, mins_4_7};\n                x1_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins)));\n                uint32_t scales[2];\n                scales[0] = scales_mins[0] & kmask1; // scales 0~3\n                scales[1] = (scales_mins[2] & kmask2) | (((scales_mins[0] >> 6) & kmask3) << 4); // scales 4~7\n                memcpy(x1_scales, scales, 8);\n            }\n\n            int32x4_t visum = {0};\n\n            // process 64 data points per iteration, totally 256 data points\n            for (int j = 0; j < QK_K / 64; ++j, qx0 += 32, qx1 += 32, qy0 += 64, qy1 += 64) {\n                const int8x16x4_t vy0 = vld1q_s8_x4(qy0);\n                const int8x16x4_t vy1 = vld1q_s8_x4(qy1);\n\n                int8x16_t vx0[4], vx1[4];\n                {\n                    const uint8x16x2_t vv = vld1q_u8_x2(qx0);\n                    vx0[0] = vreinterpretq_s8_u8(vandq_u8(vv.val[0], m4b));\n                    vx0[1] = vreinterpretq_s8_u8(vandq_u8(vv.val[1], m4b));\n                    vx0[2] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[0], 4));\n                    vx0[3] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[1], 4));\n                }\n                {\n                    const uint8x16x2_t vv = vld1q_u8_x2(qx1);\n                    vx1[0] = vreinterpretq_s8_u8(vandq_u8(vv.val[0], m4b));\n                    vx1[1] = vreinterpretq_s8_u8(vandq_u8(vv.val[1], m4b));\n                    vx1[2] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[0], 4));\n                    vx1[3] = vreinterpretq_s8_u8(vshrq_n_u8(vv.val[1], 4));\n                }\n\n                // process 32 data points (share same block scale) per iteration\n                for (int k = 0; k < 2; ++k) {\n                    const int blk = j * 2 + k;\n                    const int32x4_t block_scale = {\n                        x0_scales[blk],\n                        x0_scales[blk],\n                        x1_scales[blk],\n                        x1_scales[blk],\n                    };\n\n                    int32x4_t vr = {0};\n                    for (int l = 0; l < 2; ++l) {\n                        const int idx = k * 2 + l;\n                        const int64x2_t vx0_s64 = vreinterpretq_s64_s8(vx0[idx]);\n                        const int64x2_t vx1_s64 = vreinterpretq_s64_s8(vx1[idx]);\n                        const int64x2_t vy0_s64 = vreinterpretq_s64_s8(vy0.val[idx]);\n                        const int64x2_t vy1_s64 = vreinterpretq_s64_s8(vy1.val[idx]);\n                        const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vx0_s64, vx1_s64));\n                        const int8x16_t vx_h = vreinterpretq_s8_s64(vzip2q_s64(vx0_s64, vx1_s64));\n                        const int8x16_t vy_l = vreinterpretq_s8_s64(vzip1q_s64(vy0_s64, vy1_s64));\n                        const int8x16_t vy_h = vreinterpretq_s8_s64(vzip2q_s64(vy0_s64, vy1_s64));\n                        vr = vmmlaq_s32(vr, vx_l, vy_l);\n                        vr = vmmlaq_s32(vr, vx_h, vy_h);\n                    }\n                    // apply block scale, will NOT overflow\n                    // block_scale * sum_256(int4*int8) <= 2^(8+8+4+8) = 28 bits\n                    visum = vmlaq_s32(visum, vr, block_scale);\n                }\n            }\n\n            // adjust bias, apply superblock scale\n            {\n                int32_t bias[4];\n                // no obvious uplift from sve sdot-16, just use neon mul add\n                const int16x8_t y0_sums = vpaddq_s16(vld1q_s16(y0->bsums), vld1q_s16(y0->bsums+8));\n                const int16x8_t y1_sums = vpaddq_s16(vld1q_s16(y1->bsums), vld1q_s16(y1->bsums+8));\n                bias[0] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y0_sums), vget_low_s16(x0_mins)),\n                                               vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x0_mins))));\n                bias[1] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x0_mins)),\n                                               vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x0_mins))));\n                bias[2] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y0_sums), vget_low_s16(x1_mins)),\n                                               vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x1_mins))));\n                bias[3] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x1_mins)),\n                                               vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x1_mins))));\n                const float32x4_t dmins = {\n                    GGML_CPU_FP16_TO_FP32(x0->dmin) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x0->dmin) * y1->d,\n                    GGML_CPU_FP16_TO_FP32(x1->dmin) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x1->dmin) * y1->d,\n                };\n                vfsum = vmlsq_f32(vfsum, vcvtq_f32_s32(vld1q_s32(bias)), dmins);\n\n                const float32x4_t superblock_scale = {\n                    GGML_CPU_FP16_TO_FP32(x0->d) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x0->d) * y1->d,\n                    GGML_CPU_FP16_TO_FP32(x1->d) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x1->d) * y1->d,\n                };\n                vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale);\n            }\n        }\n\n        // vfsum = ABCD -> ACBD\n        // AC -> s, BD -> (s+bs)\n        vfsum = vzip1q_f32(vfsum, vextq_f32(vfsum, vfsum, 2));\n        vst1_f32(s,      vget_low_f32 (vfsum));\n        vst1_f32(s + bs, vget_high_f32(vfsum));\n\n        return;\n    }\n#endif\n\n#ifdef __ARM_FEATURE_SVE\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n        memcpy(utmp, x[i].scales, K_SCALE_SIZE);\n\n        uint32x2_t mins8 = { 0 };\n        mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);\n        mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);\n\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[0] &= kmask1;\n\n        const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));\n        const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n                                         vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n        sumf -= dmin * vaddvq_s32(prod);\n\n        const uint8_t * scales = (const uint8_t *)utmp;\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const svuint8_t m4b = svdup_n_u8(0xf);\n        const svint32_t mzero = svdup_n_s32(0);\n        svint32_t sumi1 = svdup_n_s32(0);\n        svint32_t sumi1_1 = svdup_n_s32(0);\n        svint32_t sumi1_2 = svdup_n_s32(0);\n        svint32_t sumi2 = svdup_n_s32(0);\n        svint32_t sumi2_1 = svdup_n_s32(0);\n        svint32_t sumi2_2 = svdup_n_s32(0);\n        switch (vector_length) {\n            case 128:\n                {\n                    for (int j = 0; j < QK_K/64; ++j) {\n                        svint8_t q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4), m4b));\n                        svint8_t q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n                        sumi1_1 = svmla_n_s32_x(svptrue_b32(), sumi1_1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n                        q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4+16), m4b));\n                        q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n                        sumi1_2 = svmla_n_s32_x(svptrue_b32(), sumi1_2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n\n                        q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4), 4));\n                        q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n                        sumi2_1 = svmla_n_s32_x(svptrue_b32(), sumi2_1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n                        q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_b8(), svld1_u8(svptrue_b8(), q4+16), 4));\n                        q8bytes = svld1_s8(svptrue_b8(), q8); q8 += 16;\n                        sumi2_2 = svmla_n_s32_x(svptrue_b32(), sumi2_2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n                        q4 += 32;\n                    }\n                    sumi1 = svadd_s32_x(svptrue_b32(), sumi1_1, sumi1_2);\n                    sumi2 = svadd_s32_x(svptrue_b32(), sumi2_1, sumi2_2);\n                    sumf += d * (svaddv_s32(svptrue_b32(), svadd_s32_x(svptrue_b32(), sumi1, sumi2)));\n                } break;\n            case 256:\n            case 512:\n                {\n                    for (int j = 0; j < QK_K/64; ++j) {\n                        const svuint8_t q4bits  = svld1_u8(svptrue_pat_b8(SV_VL32), q4); q4 += 32;\n                        svint8_t q4bytes = svreinterpret_s8_u8(svand_u8_x(svptrue_pat_b8(SV_VL32), q4bits, m4b));\n                        svint8_t q8bytes = svld1_s8(svptrue_pat_b8(SV_VL32), q8); q8 += 32;\n                        sumi1 = svmla_n_s32_x(svptrue_pat_b32(SV_VL8), sumi1, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+0]);\n\n                        q4bytes = svreinterpret_s8_u8(svlsr_n_u8_x(svptrue_pat_b8(SV_VL32), q4bits, 4));\n                        q8bytes = svld1_s8(svptrue_pat_b8(SV_VL32), q8); q8 += 32;\n                        sumi2 = svmla_n_s32_x(svptrue_pat_b32(SV_VL8), sumi2, svdot_s32(mzero, q4bytes, q8bytes), scales[2*j+1]);\n                    }\n                    sumf += d * (svaddv_s32(svptrue_pat_b32(SV_VL8), svadd_s32_x(svptrue_pat_b32(SV_VL8), sumi1, sumi2)));\n                } break;\n            default:\n                assert(false && \"Unsupported vector length\");\n                break;\n        }\n    }\n    *s = sumf;\n#elif defined __ARM_NEON\n    const uint8x16_t m4b = vdupq_n_u8(0xf);\n    const int32x4_t mzero = vdupq_n_s32(0);\n\n    ggml_int8x16x2_t q4bytes;\n    ggml_int8x16x2_t q8bytes;\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n        memcpy(utmp, x[i].scales, 12);\n\n        uint32x2_t mins8 = { 0 };\n        mins8 = vset_lane_u32(utmp[1] & kmask1, mins8, 0);\n        mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), mins8, 1);\n\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[0] &= kmask1;\n\n        const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8)));\n        const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n                                         vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n        sumf -= dmin * vaddvq_s32(prod);\n\n        const uint8_t * scales = (const uint8_t *)utmp;\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        int32_t sumi1 = 0;\n        int32_t sumi2 = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;\n\n            q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n            q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8  (q4bits.val[0], m4b));\n            q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8  (q4bits.val[1], m4b));\n\n            const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);\n            sumi1 += vaddvq_s32(p1) * scales[2*j+0];\n\n            q8bytes = ggml_vld1q_s8_x2(q8); q8 += 32;\n            q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4));\n            q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4));\n\n            const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]);\n\n            sumi2 += vaddvq_s32(p2) * scales[2*j+1];\n        }\n\n        sumf += d * (sumi1 + sumi2);\n\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n\n#ifdef __ARM_NEON\n    const uint8x16_t m4b = vdupq_n_u8(0xf);\n    const uint8x16_t mone = vdupq_n_u8(1);\n    const uint8x16_t mtwo = vdupq_n_u8(2);\n    const int32x4_t mzero = vdupq_n_s32(0);\n\n    ggml_int8x16x4_t q5bytes;\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const int16x8_t q8sums = vpaddq_s16(vld1q_s16(y[i].bsums), vld1q_s16(y[i].bsums + 8));\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const uint8x8_t mins8 = vld1_u8((const uint8_t*)utmp + 8);\n        const int16x8_t mins = vreinterpretq_s16_u16(vmovl_u8(mins8));\n        const int32x4_t prod = vaddq_s32(vmull_s16(vget_low_s16 (q8sums), vget_low_s16 (mins)),\n                                         vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)));\n        int32_t sumi_mins = vaddvq_s32(prod);\n\n        const uint8_t * scales = (const uint8_t *)utmp;\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh);\n\n        ggml_uint8x16x4_t q5h;\n\n        int32_t sumi = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); q5 += 32;\n            const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            q5h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);\n            q5h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);\n            q5h.val[2] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[0]), 3);\n            q5h.val[3] = vshlq_n_u8(vandq_u8(mtwo, qhbits.val[1]), 3);\n            qhbits.val[0] = vshrq_n_u8(qhbits.val[0], 2);\n            qhbits.val[1] = vshrq_n_u8(qhbits.val[1], 2);\n\n            q5bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[0], m4b), q5h.val[0]));\n            q5bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.val[1], m4b), q5h.val[1]));\n            q5bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[0], 4), q5h.val[2]));\n            q5bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.val[1], 4), q5h.val[3]));\n\n            sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0]), q5bytes.val[1], q8bytes.val[1])) * *scales++;\n            sumi += vaddvq_s32(ggml_vdotq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2]), q5bytes.val[3], q8bytes.val[3])) * *scales++;\n        }\n\n        sumf += d * sumi - dmin * sumi_mins;\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n#ifdef __ARM_FEATURE_MATMUL_INT8\n    assert((nrc == 2) || (nrc == 1));\n#else\n    assert(nrc == 1);\n#endif\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#ifdef __ARM_FEATURE_SVE\n    const int vector_length = ggml_cpu_get_sve_cnt()*8;\n#endif\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const svbool_t pg32_2 = svptrue_pat_b32(SV_VL2);\n\n        svfloat32_t sum = svdup_n_f32(0);\n\n        const block_q6_K * GGML_RESTRICT vx0 = vx;\n        const block_q8_K * GGML_RESTRICT vy0 = vy;\n        const block_q6_K * GGML_RESTRICT vx1 = (const block_q6_K *) ((const uint8_t*)vx + bx);\n        const block_q8_K * GGML_RESTRICT vy1 = (const block_q8_K *) ((const uint8_t*)vy + by);\n\n        switch (vector_length) {\n            case 128:\n                {\n                    const svbool_t pg128_all = svptrue_pat_b8(SV_ALL);\n                    for (int i = 0; i < nb; ++i) {\n                        const uint8_t * GGML_RESTRICT ql0 = vx0[i].ql;\n                        const uint8_t * GGML_RESTRICT qh0 = vx0[i].qh;\n                        const uint8_t * GGML_RESTRICT ql1 = vx1[i].ql;\n                        const uint8_t * GGML_RESTRICT qh1 = vx1[i].qh;\n                        const int8_t  * GGML_RESTRICT q80 = vy0[i].qs;\n                        const int8_t  * GGML_RESTRICT q81 = vy1[i].qs;\n\n                        const int8_t * GGML_RESTRICT scale0 = vx0[i].scales;\n                        const int8_t * GGML_RESTRICT scale1 = vx1[i].scales;\n\n                        svfloat32_t vy_d = svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d));\n                        svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n                        svfloat32_t svsuper_block_scales = svmul_f32_x(pg128_all, vy_d, vx_d);\n                        // process q8sum summation 128 bit route\n                        const svint16_t q8sums_01 = svld1_s16(pg128_all, vy0[i].bsums);\n                        const svint16_t q8sums_02 = svld1_s16(pg128_all, vy0[i].bsums + 8);\n                        const svint16_t q8sums_11 = svld1_s16(pg128_all, vy1[i].bsums);\n                        const svint16_t q8sums_12 = svld1_s16(pg128_all, vy1[i].bsums + 8);\n                        const svint64x2_t q6scales_0_tmp = svld2_s64(pg128_all, (const int64_t *)scale0);\n                        const svint16_t q6scales_01 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_0_tmp, 0)));\n                        const svint16_t q6scales_02 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_0_tmp, 1)));\n                        const svint64x2_t q6scales_1_tmp = svld2_s64(pg128_all, (const int64_t *)scale1);\n                        const svint16_t q6scales_11 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_1_tmp, 0)));\n                        const svint16_t q6scales_12 = svunpklo_s16(svreinterpret_s8_s64(svget2_s64(q6scales_1_tmp, 1)));\n                        const svint64_t prod = svdup_n_s64(0);\n\n                        svint32_t isum_tmp1 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_01, q6scales_01), q8sums_02, q6scales_02));\n                        svint32_t isum_tmp2 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_01, q6scales_11), q8sums_02, q6scales_12));\n                        svint32_t isum_tmp3 = svtrn1_s32(isum_tmp1, isum_tmp2);\n                        svint32_t isum_tmp4 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_11, q6scales_01), q8sums_12, q6scales_02));\n                        svint32_t isum_tmp5 = svreinterpret_s32_s64(svdot_s64(svdot_s64(prod, q8sums_11, q6scales_11), q8sums_12, q6scales_12));\n                        svint32_t isum_tmp6 = svtrn1_s32(isum_tmp4, isum_tmp5);\n                        svint32_t isum_tmp7 = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(isum_tmp3), svreinterpret_s64_s32(isum_tmp6)));\n                        svint32_t isum_tmp8 = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(isum_tmp3), svreinterpret_s64_s32(isum_tmp6)));\n                        svint32_t svisum_mins = svadd_s32_x(pg128_all, isum_tmp7, isum_tmp8);\n\n                        // process mmla\n                        svint8_t  l0, l1, r0, r1;\n                        svint32_t isum_tmp = svdup_n_s32(0);\n                        for (int j = 0; j < QK_K/128; ++j) {\n                            for (int k = 0; k < 8; ++k) {\n                                svuint8_t qhbits_0 = svld1_u8(pg128_all, qh0+16*(k%2));\n                                svuint8_t qhbits_1 = svld1_u8(pg128_all, qh1+16*(k%2));\n                                svuint8_t q6bits_0 = svld1_u8(pg128_all, ql0+16*(k%4));\n                                svuint8_t q6bits_1 = svld1_u8(pg128_all, ql1+16*(k%4));\n                                const int ql_pos = (k/4)*4;\n                                svuint8_t q6bytes_0_lo = (ql_pos < 4) ? svand_n_u8_x(pg128_all, q6bits_0, 0xf) : svlsr_n_u8_x(pg128_all, q6bits_0, 4);\n                                svuint8_t q6bytes_1_lo = (ql_pos < 4) ? svand_n_u8_x(pg128_all, q6bits_1, 0xf) : svlsr_n_u8_x(pg128_all, q6bits_1, 4);\n                                const int qh_pos = (k/2)*2;\n                                svuint8_t q6bytes_0_hi = svand_n_u8_x(pg128_all, qhbits_0, 0x3 << qh_pos);\n                                svuint8_t q6bytes_1_hi = svand_n_u8_x(pg128_all, qhbits_1, 0x3 << qh_pos);\n                                svint8_t  q6bytes_0, q6bytes_1;\n                                if (qh_pos <= 4) {\n                                    q6bytes_0 = svreinterpret_s8_u8(svmla_n_u8_x(pg128_all, q6bytes_0_lo, q6bytes_0_hi, 1 << (4 - qh_pos)));\n                                    q6bytes_1 = svreinterpret_s8_u8(svmla_n_u8_x(pg128_all, q6bytes_1_lo, q6bytes_1_hi, 1 << (4 - qh_pos)));\n                                } else {\n                                    q6bytes_0 = svreinterpret_s8_u8(svorr_u8_x(pg128_all, q6bytes_0_lo, svlsr_n_u8_x(pg128_all, q6bytes_0_hi, (qh_pos - 4))));\n                                    q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg128_all, q6bytes_1_lo, svlsr_n_u8_x(pg128_all, q6bytes_1_hi, (qh_pos - 4))));\n                                }\n                                svint8_t  q8bytes_0 = svld1_s8(pg128_all, q80+16*(k%8));\n                                svint8_t  q8bytes_1 = svld1_s8(pg128_all, q81+16*(k%8));\n                                l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n                                l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n                                r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                                r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                                svint32_t svscale = svzip1_s32(svdup_n_s32(scale0[k]), svdup_n_s32(scale1[k]));\n                                isum_tmp = svmla_s32_x(pg128_all, isum_tmp, svmmla_s32(svmmla_s32(svdup_n_s32(0), r0, l0), r1, l1), svscale);\n                            }\n                            qh0 += 32;  qh1 += 32;\n                            ql0 += 64;  ql1 += 64;\n                            q80 += 128; q81 += 128;\n                            scale0 += 8; scale1 += 8;\n                        }\n                        sum = svmla_f32_x(pg128_all, sum,\n                                svcvt_f32_x(pg128_all, svmla_s32_x(pg128_all, isum_tmp,\n                                        svisum_mins, svdup_n_s32(-32))),\n                                svsuper_block_scales);\n                    }\n                } // end of case 128\n                break;\n            case 256:\n            case 512:\n                {\n                    const svbool_t pg256_all = svptrue_pat_b8(SV_ALL);\n                    const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n                    for (int i = 0; i < nb; ++i) {\n                        const uint8_t * GGML_RESTRICT ql0 = vx0[i].ql;\n                        const uint8_t * GGML_RESTRICT qh0 = vx0[i].qh;\n                        const uint8_t * GGML_RESTRICT ql1 = vx1[i].ql;\n                        const uint8_t * GGML_RESTRICT qh1 = vx1[i].qh;\n                        const int8_t  * GGML_RESTRICT q80 = vy0[i].qs;\n                        const int8_t  * GGML_RESTRICT q81 = vy1[i].qs;\n\n                        const int8_t * GGML_RESTRICT scale0 = vx0[i].scales;\n                        const int8_t * GGML_RESTRICT scale1 = vx1[i].scales;\n                        svfloat32_t vx_d = svzip1_f32(svdup_n_f32(GGML_FP16_TO_FP32(vx0[i].d)), svdup_n_f32(GGML_FP16_TO_FP32(vx1[i].d)));\n                        svfloat64_t vy_d_tmp = svreinterpret_f64_f32(svuzp1_f32(svdup_n_f32(vy0[i].d), svdup_n_f32(vy1[i].d)));\n                        svfloat32_t vy_d = svreinterpret_f32_f64(svuzp1_f64(vy_d_tmp, vy_d_tmp));\n                        svfloat32_t svsuper_block_scales = svmul_f32_x(pg32_4, vy_d, vx_d);\n                        // process q8sum summation 256 bit route\n                        const svint16_t q8sums_0 = svld1_s16(pg256_all, vy0[i].bsums);\n                        const svint16_t q8sums_1 = svld1_s16(pg256_all, vy1[i].bsums);\n                        const svint16_t q6scales_0 = svunpklo_s16(svld1_s8(pg256_all, scale0));\n                        const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(pg256_all, scale1));\n                        const svint64_t prod = svdup_n_s64(0);\n                        svint32_t isum_tmp1  = svreinterpret_s32_s64(svdot_s64(prod, q8sums_0, q6scales_0));\n                        svint32_t isum_tmp2  = svreinterpret_s32_s64(svdot_s64(prod, q8sums_0, q6scales_1));\n                        svint32_t isum_tmp3  = svreinterpret_s32_s64(svdot_s64(prod, q8sums_1, q6scales_0));\n                        svint32_t isum_tmp4  = svreinterpret_s32_s64(svdot_s64(prod, q8sums_1, q6scales_1));\n                        svint32_t isum_tmp5  = svtrn1_s32(isum_tmp1, isum_tmp2);\n                        svint32_t isum_tmp6  = svtrn1_s32(isum_tmp3, isum_tmp4);\n                        svint32_t isum_tmp7  = svreinterpret_s32_s64(svtrn2_s64(svreinterpret_s64_s32(isum_tmp5), svreinterpret_s64_s32(isum_tmp6)));\n                        svint32_t isum_tmp8  = svreinterpret_s32_s64(svtrn1_s64(svreinterpret_s64_s32(isum_tmp5), svreinterpret_s64_s32(isum_tmp6)));\n                        svint32_t isum_tmp9  = svadd_s32_x(pg256_all, isum_tmp7, isum_tmp8);\n                        svint32_t isum_tmp10 = svreinterpret_s32_u8(svext_u8(svreinterpret_u8_s32(isum_tmp9), svreinterpret_u8_s32(isum_tmp9), 16));\n                        svint32_t svisum_mins = svadd_s32_z(pg32_4, isum_tmp9, isum_tmp10);\n\n                        // process mmla\n                        svint8_t l0, l1, r0, r1;\n                        svint32_t isum_tmp = svdup_n_s32(0);\n                        for (int j = 0; j < QK_K/128; ++j) {\n                            for (int k = 0; k < 8; k+=2) { // process 2 block\n                                svuint8_t qhbits_0  = svld1_u8(pg256_all, qh0);\n                                svuint8_t qhbits_1  = svld1_u8(pg256_all, qh1);\n                                svuint8_t q6bits_0  = svld1_u8(pg256_all, ql0+32*((k%4)/2));\n                                svuint8_t q6bits_1  = svld1_u8(pg256_all, ql1+32*((k%4)/2));\n                                const int ql_pos = (k/4)*4;\n                                svuint8_t q6bytes_0_lo = (ql_pos < 4) ? svand_n_u8_x(pg256_all, q6bits_0, 0xf) : svlsr_n_u8_x(pg256_all, q6bits_0, 4);\n                                svuint8_t q6bytes_1_lo = (ql_pos < 4) ? svand_n_u8_x(pg256_all, q6bits_1, 0xf) : svlsr_n_u8_x(pg256_all, q6bits_1, 4);\n                                const int qh_pos = (k/2)*2;\n                                svuint8_t q6bytes_0_hi = svand_n_u8_x(pg256_all, qhbits_0, 0x3 << qh_pos);\n                                svuint8_t q6bytes_1_hi = svand_n_u8_x(pg256_all, qhbits_1, 0x3 << qh_pos);\n                                svint8_t  q6bytes_0, q6bytes_1;\n                                if (qh_pos <= 4) {\n                                    q6bytes_0 = svreinterpret_s8_u8(svmla_n_u8_x(pg256_all, q6bytes_0_lo, q6bytes_0_hi, 1 << (4 - qh_pos)));\n                                    q6bytes_1 = svreinterpret_s8_u8(svmla_n_u8_x(pg256_all, q6bytes_1_lo, q6bytes_1_hi, 1 << (4 - qh_pos)));\n                                } else {\n                                    q6bytes_0 = svreinterpret_s8_u8(svorr_u8_x(pg256_all, q6bytes_0_lo, svlsr_n_u8_x(pg256_all, q6bytes_0_hi, (qh_pos - 4))));\n                                    q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg256_all, q6bytes_1_lo, svlsr_n_u8_x(pg256_all, q6bytes_1_hi, (qh_pos - 4))));\n                                }\n                                svint8_t  q8bytes_0 = svld1_s8(pg256_all, q80+32*(k/2));\n                                svint8_t  q8bytes_1 = svld1_s8(pg256_all, q81+32*(k/2));\n                                l0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n                                l1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q6bytes_0), svreinterpret_s64_s8(q6bytes_1)));\n                                r0 = svreinterpret_s8_s64(svzip1_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                                r1 = svreinterpret_s8_s64(svzip2_s64(svreinterpret_s64_s8(q8bytes_0), svreinterpret_s64_s8(q8bytes_1)));\n                                svint32_t svscale0 = svzip1_s32(svdup_n_s32(scale0[k]), svdup_n_s32(scale1[k]));\n                                svint32_t svscale1 = svzip1_s32(svdup_n_s32(scale0[k+1]), svdup_n_s32(scale1[k+1]));\n                                isum_tmp = svmla_s32_x(pg256_all, isum_tmp, svmmla_s32(svdup_n_s32(0), r0, l0), svscale0);\n                                isum_tmp = svmla_s32_x(pg256_all, isum_tmp, svmmla_s32(svdup_n_s32(0), r1, l1), svscale1);\n                            }\n                            qh0 += 32;  qh1 += 32;\n                            ql0 += 64;  ql1 += 64;\n                            q80 += 128; q81 += 128;\n                            scale0 += 8; scale1 += 8;\n                        } // end of for\n                        svint32_t swap_isum_tmp = svext_s32(isum_tmp, isum_tmp, 4);\n                        isum_tmp = svadd_s32_x(pg32_4, isum_tmp, swap_isum_tmp);\n                        sum = svmla_f32_x(pg32_4, sum,\n                                svcvt_f32_x(pg32_4, svmla_s32_x(pg32_4, isum_tmp,\n                                        svisum_mins, svdup_n_s32(-32))),\n                                svsuper_block_scales);\n                    }\n                } // end of case 256\n                break;\n            default:\n                assert(false && \"Unsupported vector length\");\n                break;\n        } // end of switch\n\n        svst1_f32(pg32_2, s, sum);\n        svst1_f32(pg32_2, s + bs, svreinterpret_f32_u8(svext_u8(svreinterpret_u8_f32(sum), svdup_n_u8(0), 8)));\n\n        return;\n    }\n#elif defined(__ARM_FEATURE_MATMUL_INT8)\n    if (nrc == 2) {\n        const block_q6_K * GGML_RESTRICT x0 = x;\n        const block_q6_K * GGML_RESTRICT x1 = (const block_q6_K *) ((const uint8_t *)vx + bx);\n        const block_q8_K * GGML_RESTRICT y0 = y;\n        const block_q8_K * GGML_RESTRICT y1 = (const block_q8_K *) ((const uint8_t *)vy + by);\n\n        float32x4_t vfsum = vdupq_n_f32(0.0f);\n\n        for (int i = 0; i < nb; ++i, ++x0, ++x1, ++y0, ++y1) {\n            const uint8_t * GGML_RESTRICT ql0 = x0->ql;\n            const uint8_t * GGML_RESTRICT ql1 = x1->ql;\n            const uint8_t * GGML_RESTRICT qh0 = x0->qh;\n            const uint8_t * GGML_RESTRICT qh1 = x1->qh;\n            const  int8_t * GGML_RESTRICT qy0 = y0->qs;\n            const  int8_t * GGML_RESTRICT qy1 = y1->qs;\n\n            const uint8x16_t mone = vdupq_n_u8(0x30);\n            const uint8x16_t  m4b = vdupq_n_u8(0x0f);\n\n            int32x4_t visum = vdupq_n_s32(0);\n\n            // process 8 blocks per iteration, totally 16 blocks\n            for (int j = 0; j < 2; ++j, qh0 += 32, ql0 += 64, qh1 += 32, ql1 += 64) {\n                int8x16_t vx0[8], vx1[8];\n\n                // de-quantize vx0[8]\n                {\n                    const uint8x16x2_t qh_bits = vld1q_u8_x2(qh0);\n                    const uint8x16x4_t ql_bits = vld1q_u8_x4(ql0);\n\n                    uint8x16_t q6h_0 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 4));\n                    uint8x16_t q6h_1 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 4));\n                    uint8x16_t q6h_2 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 2));\n                    uint8x16_t q6h_3 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 2));\n\n                    vx0[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[0], m4b), q6h_0));\n                    vx0[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[1], m4b), q6h_1));\n                    vx0[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[2], m4b), q6h_2));\n                    vx0[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[3], m4b), q6h_3));\n\n                    q6h_0 = vandq_u8(mone, qh_bits.val[0]);\n                    q6h_1 = vandq_u8(mone, qh_bits.val[1]);\n                    q6h_2 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[0], 2));\n                    q6h_3 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[1], 2));\n\n                    vx0[4] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[0], 4), q6h_0));\n                    vx0[5] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[1], 4), q6h_1));\n                    vx0[6] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[2], 4), q6h_2));\n                    vx0[7] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[3], 4), q6h_3));\n                }\n\n                // de-quantize vx1[8]\n                {\n                    const uint8x16x2_t qh_bits = vld1q_u8_x2(qh1);\n                    const uint8x16x4_t ql_bits = vld1q_u8_x4(ql1);\n\n                    uint8x16_t q6h_0 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 4));\n                    uint8x16_t q6h_1 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 4));\n                    uint8x16_t q6h_2 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[0], 2));\n                    uint8x16_t q6h_3 = vandq_u8(mone, vshlq_n_u8(qh_bits.val[1], 2));\n\n                    vx1[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[0], m4b), q6h_0));\n                    vx1[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[1], m4b), q6h_1));\n                    vx1[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[2], m4b), q6h_2));\n                    vx1[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(ql_bits.val[3], m4b), q6h_3));\n\n                    q6h_0 = vandq_u8(mone, qh_bits.val[0]);\n                    q6h_1 = vandq_u8(mone, qh_bits.val[1]);\n                    q6h_2 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[0], 2));\n                    q6h_3 = vandq_u8(mone, vshrq_n_u8(qh_bits.val[1], 2));\n\n                    vx1[4] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[0], 4), q6h_0));\n                    vx1[5] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[1], 4), q6h_1));\n                    vx1[6] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[2], 4), q6h_2));\n                    vx1[7] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(ql_bits.val[3], 4), q6h_3));\n                }\n\n                // process 16 elements (one block with same scale) per iteration\n                // - vx = concat(ql, qh) - 32\n                // - r1,r2,r3,r4 = smmla(vx, vy)\n                for (int k = 0; k < 8; ++k) {\n                    const int blk = j * 8 + k;\n\n                    const int8x16_t vy0 = vld1q_s8(qy0);\n                    const int8x16_t vy1 = vld1q_s8(qy1);\n                    qy0 += 16;\n                    qy1 += 16;\n\n                    const int32x4_t block_scale = {\n                        x0->scales[blk],\n                        x0->scales[blk],\n                        x1->scales[blk],\n                        x1->scales[blk],\n                    };\n\n                    // calculate four results at once with outer product\n                    const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k])));\n                    const int8x16_t vx_h = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k])));\n                    const int8x16_t vy_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vy0), vreinterpretq_s64_s8(vy1)));\n                    const int8x16_t vy_h = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(vy0), vreinterpretq_s64_s8(vy1)));\n                    int32x4_t vr = vdupq_n_s32(0);\n                    vr = vmmlaq_s32(vr, vx_l, vy_l);\n                    vr = vmmlaq_s32(vr, vx_h, vy_h);\n\n                    // apply block scale, will NOT overflow\n                    // block_scale * sum_256(int6*int8) <= 2^(8+8+6+8) = 30 bits\n                    visum = vmlaq_s32(visum, vr, block_scale);\n                }\n            }\n\n            // adjust bias, apply superblock scale\n            {\n                int32_t bias[4];\n                // NEON doesn't support int16 dot product, fallback to separated mul and add\n                const int16x8x2_t q8sums0 = vld1q_s16_x2(y0->bsums);\n                const int16x8x2_t q8sums1 = vld1q_s16_x2(y1->bsums);\n\n                int8x16_t scales_s8 = vld1q_s8(x0->scales);\n                const int16x8x2_t q6scales0 = {{vmovl_s8(vget_low_s8(scales_s8)), vmovl_s8(vget_high_s8(scales_s8))}};\n                scales_s8 = vld1q_s8(x1->scales);\n                const int16x8x2_t q6scales1 = {{vmovl_s8(vget_low_s8(scales_s8)), vmovl_s8(vget_high_s8(scales_s8))}};\n\n                int32x4_t prod;\n                prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[0]), vget_low_s16 (q6scales0.val[0])),\n                                           vmull_s16(vget_high_s16(q8sums0.val[0]), vget_high_s16(q6scales0.val[0]))),\n                                 vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[1]), vget_low_s16 (q6scales0.val[1])),\n                                           vmull_s16(vget_high_s16(q8sums0.val[1]), vget_high_s16(q6scales0.val[1]))));\n                bias[0] = vaddvq_s32(prod);\n                prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[0]), vget_low_s16 (q6scales0.val[0])),\n                                           vmull_s16(vget_high_s16(q8sums1.val[0]), vget_high_s16(q6scales0.val[0]))),\n                                 vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[1]), vget_low_s16 (q6scales0.val[1])),\n                                           vmull_s16(vget_high_s16(q8sums1.val[1]), vget_high_s16(q6scales0.val[1]))));\n                bias[1] = vaddvq_s32(prod);\n                prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[0]), vget_low_s16 (q6scales1.val[0])),\n                                           vmull_s16(vget_high_s16(q8sums0.val[0]), vget_high_s16(q6scales1.val[0]))),\n                                 vaddq_s32(vmull_s16(vget_low_s16 (q8sums0.val[1]), vget_low_s16 (q6scales1.val[1])),\n                                           vmull_s16(vget_high_s16(q8sums0.val[1]), vget_high_s16(q6scales1.val[1]))));\n                bias[2] = vaddvq_s32(prod);\n                prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[0]), vget_low_s16 (q6scales1.val[0])),\n                                           vmull_s16(vget_high_s16(q8sums1.val[0]), vget_high_s16(q6scales1.val[0]))),\n                                 vaddq_s32(vmull_s16(vget_low_s16 (q8sums1.val[1]), vget_low_s16 (q6scales1.val[1])),\n                                           vmull_s16(vget_high_s16(q8sums1.val[1]), vget_high_s16(q6scales1.val[1]))));\n                bias[3] = vaddvq_s32(prod);\n\n                const int32x4_t vibias = vmulq_n_s32(vld1q_s32(bias), 32);\n\n                const float32x4_t superblock_scale = {\n                    GGML_CPU_FP16_TO_FP32(x0->d) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x0->d) * y1->d,\n                    GGML_CPU_FP16_TO_FP32(x1->d) * y0->d,\n                    GGML_CPU_FP16_TO_FP32(x1->d) * y1->d,\n                };\n\n                visum = vsubq_s32(visum, vibias);\n                vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale);\n            }\n        }\n\n        // vfsum = ABCD -> ACBD\n        // AC -> s, BD -> (s+bs)\n        vfsum = vzip1q_f32(vfsum, vextq_f32(vfsum, vfsum, 2));\n        vst1_f32(s,      vget_low_f32 (vfsum));\n        vst1_f32(s + bs, vget_high_f32(vfsum));\n\n        return;\n    }\n#endif\n\n#ifdef __ARM_FEATURE_SVE\n    float sum = 0;\n    svuint8_t m4b = svdup_n_u8(0xf);\n    svint32_t vzero = svdup_n_s32(0);\n    svuint8_t mone = svdup_n_u8(0x30);\n    svint8_t q6bytes_1, q6bytes_2, q6bytes_3, q6bytes_4;\n    svuint8_t q6h_1, q6h_2, q6h_3, q6h_4;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n        const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8);\n        const svint16_t q8sums_1 = svld1_s16(pg16_8, y[i].bsums);\n        const svint16_t q8sums_2 = svld1_s16(pg16_8, y[i].bsums + 8);\n        const svint16_t q6scales_1 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale));\n        const svint16_t q6scales_2 = svunpklo_s16(svld1_s8(svptrue_pat_b8(SV_VL8), scale + 8));\n        const svint64_t prod = svdup_n_s64(0);\n        int32_t isum_mins = svaddv_s64(svptrue_b64(), svadd_s64_x(svptrue_b64(), svdot_s64(prod, q8sums_1, q6scales_1),\n                                                                                 svdot_s64(prod, q8sums_2, q6scales_2)));\n        int32_t isum = 0;\n\n        switch (vector_length) {\n            case 128:\n                {\n                    const svbool_t pg32_4 = svptrue_pat_b32(SV_VL4);\n                    const svbool_t pg8_16 = svptrue_pat_b8(SV_VL16);\n                    svint32_t isum_tmp = svdup_n_s32(0);\n                    for (int j = 0; j < QK_K/128; ++j) {\n                        svuint8_t qhbits_1 = svld1_u8(pg8_16, qh);\n                        svuint8_t qhbits_2 = svld1_u8(pg8_16, qh+16);\n                        qh += 32;\n                        svuint8_t q6bits_1 = svld1_u8(pg8_16, q6);\n                        svuint8_t q6bits_2 = svld1_u8(pg8_16, q6+16);\n                        svuint8_t q6bits_3 = svld1_u8(pg8_16, q6+32);\n                        svuint8_t q6bits_4 = svld1_u8(pg8_16, q6+48);\n                        q6 += 64;\n                        svint8_t q8bytes_1 = svld1_s8(pg8_16, q8);\n                        svint8_t q8bytes_2 = svld1_s8(pg8_16, q8+16);\n                        svint8_t q8bytes_3 = svld1_s8(pg8_16, q8+32);\n                        svint8_t q8bytes_4 = svld1_s8(pg8_16, q8+48);\n                        q8 += 64;\n\n                        q6h_1 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 4));\n                        q6h_2 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 4));\n                        q6h_3 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_1, 2));\n                        q6h_4 = svand_u8_x(pg16_8, mone, svlsl_n_u8_x(pg16_8, qhbits_2, 2));\n                        q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_1, m4b), q6h_1));\n                        q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_2, m4b), q6h_2));\n                        q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_3, m4b), q6h_3));\n                        q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svand_u8_x(pg8_16, q6bits_4, m4b), q6h_4));\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]);\n\n                        scale += 4;\n                        q8bytes_1 = svld1_s8(pg8_16, q8);\n                        q8bytes_2 = svld1_s8(pg8_16, q8+16);\n                        q8bytes_3 = svld1_s8(pg8_16, q8+32);\n                        q8bytes_4 = svld1_s8(pg8_16, q8+48);\n                        q8 += 64;\n\n                        q6h_1 = svand_u8_x(pg16_8, mone, qhbits_1);\n                        q6h_2 = svand_u8_x(pg16_8, mone, qhbits_2);\n                        q6h_3 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_1, 2));\n                        q6h_4 = svand_u8_x(pg16_8, mone, svlsr_n_u8_x(pg16_8, qhbits_2, 2));\n                        q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_1, 4), q6h_1));\n                        q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_2, 4), q6h_2));\n                        q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_3, 4), q6h_3));\n                        q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_16, svlsr_n_u8_x(pg8_16, q6bits_4, 4), q6h_4));\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale[0]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale[1]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale[2]);\n                        isum_tmp = svmla_n_s32_x(pg32_4, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale[3]);\n                        scale += 4;\n                    }\n                    isum += svaddv_s32(pg32_4, isum_tmp);\n                    sum += d_all * y[i].d * (isum - 32 * isum_mins);\n                }\n                break;\n            case 256:\n            case 512:\n                {\n                    const svbool_t pg8_2 = svptrue_pat_b8(SV_VL2);\n                    const svbool_t pg32_8 = svptrue_pat_b32(SV_VL8);\n                    const svbool_t pg8_32 = svptrue_pat_b8(SV_VL32);\n                    svint32_t isum_tmp = svdup_n_s32(0);\n                    for (int j = 0; j < QK_K/128; j++) {\n                        svuint8_t qhbits_1 = svld1_u8(pg8_32, qh);\n                        qh += 32;\n                        svuint8_t q6bits_1 = svld1_u8(pg8_32, q6);\n                        svuint8_t q6bits_2 = svld1_u8(pg8_32, q6+32);\n                        q6 += 64;\n                        svint8_t q8bytes_1 = svld1_s8(pg8_32, q8);\n                        svint8_t q8bytes_2 = svld1_s8(pg8_32, q8+32);\n                        svint8_t q8bytes_3 = svld1_s8(pg8_32, q8+64);\n                        svint8_t q8bytes_4 = svld1_s8(pg8_32, q8+96);\n                        q8 += 128;\n                        q6h_1 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 4));\n                        q6h_2 = svand_u8_x(pg8_32, mone, svlsl_n_u8_x(pg8_32, qhbits_1, 2));\n                        q6h_3 = svand_u8_x(pg8_32, mone, qhbits_1);\n                        q6h_4 = svand_u8_x(pg8_32, mone, svlsr_n_u8_x(pg8_32, qhbits_1, 2));\n                        q6bytes_1 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_1, m4b), q6h_1));\n                        q6bytes_2 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svand_u8_x(pg8_32, q6bits_2, m4b), q6h_2));\n                        q6bytes_3 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_1, 4), q6h_3));\n                        q6bytes_4 = svreinterpret_s8_u8(svorr_u8_x(pg8_32, svlsr_n_u8_x(pg8_32, q6bits_2, 4), q6h_4));\n\n                        svint8_t scale_lane_1_tmp = svld1_s8(pg8_2, scale);\n                        scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp);\n                        scale_lane_1_tmp= svzip1_s8(scale_lane_1_tmp, scale_lane_1_tmp);\n                        svint8_t scale_lane_2_tmp = svld1_s8(pg8_2, scale+2);\n                        scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp);\n                        scale_lane_2_tmp = svzip1_s8(scale_lane_2_tmp, scale_lane_2_tmp);\n                        svint8_t scale_lane_3_tmp = svld1_s8(pg8_2, scale+4);\n                        scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp);\n                        scale_lane_3_tmp = svzip1_s8(scale_lane_3_tmp, scale_lane_3_tmp);\n                        svint8_t scale_lane_4_tmp = svld1_s8(pg8_2, scale+6);\n                        scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp);\n                        scale_lane_4_tmp = svzip1_s8(scale_lane_4_tmp, scale_lane_4_tmp);\n                        svint32_t scale_lane_1 = svunpklo_s32(svunpklo_s16(scale_lane_1_tmp));\n                        svint32_t scale_lane_2 = svunpklo_s32(svunpklo_s16(scale_lane_2_tmp));\n                        svint32_t scale_lane_3 = svunpklo_s32(svunpklo_s16(scale_lane_3_tmp));\n                        svint32_t scale_lane_4 = svunpklo_s32(svunpklo_s16(scale_lane_4_tmp));\n\n                        isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_1, q8bytes_1), scale_lane_1);\n                        isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_2, q8bytes_2), scale_lane_2);\n                        isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_3, q8bytes_3), scale_lane_3);\n                        isum_tmp = svmla_s32_x(pg32_8, isum_tmp, svdot_s32(vzero, q6bytes_4, q8bytes_4), scale_lane_4);\n                        scale += 8;\n                    }\n                    isum += svaddv_s32(pg32_8, isum_tmp);\n                    sum += d_all * y[i].d * (isum - 32 * isum_mins);\n                }\n                break;\n            default:\n                assert(false && \"Unsupported vector length\");\n                break;\n        }\n    }\n\n    *s = sum;\n\n#elif __ARM_NEON\n    float sum = 0;\n\n    const uint8x16_t m4b = vdupq_n_u8(0xF);\n    const int32x4_t  vzero = vdupq_n_s32(0);\n    //const int8x16_t  m32s = vdupq_n_s8(32);\n\n    const uint8x16_t mone = vdupq_n_u8(3);\n\n    ggml_int8x16x4_t q6bytes;\n    ggml_uint8x16x4_t q6h;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n        const ggml_int16x8x2_t q8sums = ggml_vld1q_s16_x2(y[i].bsums);\n        const int8x16_t scales = vld1q_s8(scale);\n        const ggml_int16x8x2_t q6scales = {{vmovl_s8(vget_low_s8(scales)), vmovl_s8(vget_high_s8(scales))}};\n\n        const int32x4_t prod = vaddq_s32(vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[0]), vget_low_s16 (q6scales.val[0])),\n                                                   vmull_s16(vget_high_s16(q8sums.val[0]), vget_high_s16(q6scales.val[0]))),\n                                         vaddq_s32(vmull_s16(vget_low_s16 (q8sums.val[1]), vget_low_s16 (q6scales.val[1])),\n                                                   vmull_s16(vget_high_s16(q8sums.val[1]), vget_high_s16(q6scales.val[1]))));\n        int32_t isum_mins = vaddvq_s32(prod);\n\n        int32_t isum = 0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            ggml_uint8x16x2_t qhbits = ggml_vld1q_u8_x2(qh); qh += 32;\n            ggml_uint8x16x4_t q6bits = ggml_vld1q_u8_x4(q6); q6 += 64;\n            ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits.val[0]), 4);\n            q6h.val[1] = vshlq_n_u8(vandq_u8(mone, qhbits.val[1]), 4);\n            uint8x16_t shifted = vshrq_n_u8(qhbits.val[0], 2);\n            q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n            shifted = vshrq_n_u8(qhbits.val[1], 2);\n            q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n\n            //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s);\n            //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s);\n            //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2])), m32s);\n            //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3])), m32s);\n            q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0]));\n            q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1]));\n            q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[2], m4b), q6h.val[2]));\n            q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[3], m4b), q6h.val[3]));\n\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];\n\n            scale += 4;\n\n            q8bytes = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            shifted = vshrq_n_u8(qhbits.val[0], 4);\n            q6h.val[0] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n            shifted = vshrq_n_u8(qhbits.val[1], 4);\n            q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n            shifted = vshrq_n_u8(qhbits.val[0], 6);\n            q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n            shifted = vshrq_n_u8(qhbits.val[1], 6);\n            q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4);\n\n            //q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0])), m32s);\n            //q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1])), m32s);\n            //q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2])), m32s);\n            //q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3])), m32s);\n            q6bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[0]));\n            q6bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[1]));\n            q6bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[2], 4), q6h.val[2]));\n            q6bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[3], 4), q6h.val[3]));\n\n            isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] +\n                    vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3];\n            scale += 4;\n        }\n        //sum += isum * d_all * y[i].d;\n        sum += d_all * y[i].d * (isum - 32 * isum_mins);\n\n    }\n    *s = sum;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__ARM_NEON)\nstatic const int8_t keven_signs_q2xs[1024] = {\n     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[4];\n    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n    ggml_int8x16x4_t q2u;\n    ggml_int8x16x4_t q2s;\n    ggml_int8x16x4_t q8b;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        float sumf1 = 0, sumf2 = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n            memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n            q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 0])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 1])));\n            q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 2])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 3])));\n            q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[ 8])), vld1_s8((const void *)(iq2xxs_grid + aux8[ 9])));\n            q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xxs_grid + aux8[10])), vld1_s8((const void *)(iq2xxs_grid + aux8[11])));\n            q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >>  0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >>  7) & 127))));\n            q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));\n            q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >>  0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >>  7) & 127))));\n            q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[3] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[3] >> 21) & 127))));\n            q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);\n            q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);\n            q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);\n            q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);\n            const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]), q2u.val[1], q8b.val[1]);\n            const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]), q2u.val[3], q8b.val[3]);\n            sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28));\n            sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[3] >> 28));\n        }\n        sumf += d*(sumf1 + sumf2);\n    }\n    *s = 0.25f * sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    ggml_int8x16x4_t q2u;\n    ggml_int8x16x4_t q2s;\n    ggml_int8x16x4_t q8b;\n\n    int32x4x4_t scales32;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        const uint8x8_t scales8 = vld1_u8(x[i].scales);\n        const uint8x8_t scales_l = vand_u8(scales8, vdup_n_u8(0xf));\n        const uint8x8_t scales_h = vshr_n_u8(scales8, 4);\n        uint8x16_t scales = vcombine_u8(vzip1_u8(scales_l, scales_h), vzip2_u8(scales_l, scales_h));\n        scales = vaddq_u8(vshlq_n_u8(scales, 1), vdupq_n_u8(1));\n        const uint16x8_t scales1 = vmovl_u8(vget_low_u8(scales));\n        const uint16x8_t scales2 = vmovl_u8(vget_high_u8(scales));\n        scales32.val[0] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales1)));\n        scales32.val[1] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales1)));\n        scales32.val[2] = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(scales2)));\n        scales32.val[3] = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(scales2)));\n        int32x4_t sumi = vdupq_n_s32(0);\n        for (int ib64 = 0; ib64 < QK_K/64; ++ib64) {\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n            q2u.val[0] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[0] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[1] & 511))));\n            q2u.val[1] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[2] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[3] & 511))));\n            q2u.val[2] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[4] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[5] & 511))));\n            q2u.val[3] = vcombine_s8(vld1_s8((const void *)(iq2xs_grid + (q2[6] & 511))), vld1_s8((const void *)(iq2xs_grid + (q2[7] & 511))));\n            q2s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[0] >> 9))), vld1_s8((const void *)(signs64 + (q2[1] >> 9))));\n            q2s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[2] >> 9))), vld1_s8((const void *)(signs64 + (q2[3] >> 9))));\n            q2s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[4] >> 9))), vld1_s8((const void *)(signs64 + (q2[5] >> 9))));\n            q2s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + (q2[6] >> 9))), vld1_s8((const void *)(signs64 + (q2[7] >> 9))));\n            q2u.val[0] = vmulq_s8(q2u.val[0], q2s.val[0]);\n            q2u.val[1] = vmulq_s8(q2u.val[1], q2s.val[1]);\n            q2u.val[2] = vmulq_s8(q2u.val[2], q2s.val[2]);\n            q2u.val[3] = vmulq_s8(q2u.val[3], q2s.val[3]);\n            const int32x4_t p1 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[0], q8b.val[0]);\n            const int32x4_t p2 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[1], q8b.val[1]);\n            const int32x4_t p3 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[2], q8b.val[2]);\n            const int32x4_t p4 = ggml_vdotq_s32(vdupq_n_s32(0), q2u.val[3], q8b.val[3]);\n            const int32x4_t p = vpaddq_s32(vpaddq_s32(p1, p2), vpaddq_s32(p3, p4));\n            sumi = vmlaq_s32(sumi, p, scales32.val[ib64]);\n            q2 += 8;\n        }\n        sumf += d*vaddvq_s32(sumi);\n    }\n    *s = 0.125f * sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n    const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);\n    const uint8x16_t        mask2 = vld1q_u8(k_mask2);\n    const uint8x16_t m1 = vdupq_n_u8(1);\n    const int32x4_t vzero = vdupq_n_s32(0);\n\n    uint8x16x2_t vs;\n    ggml_int8x16x4_t q2s;\n    ggml_int8x16x4_t q8b;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        int sumi1 = 0, sumi2 = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n            q2s.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[0] | ((qh[ib32+0] << 8) & 0x300)))),\n                                     vld1_s8((const int8_t *)(iq2s_grid + (qs[1] | ((qh[ib32+0] << 6) & 0x300)))));\n            q2s.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[2] | ((qh[ib32+0] << 4) & 0x300)))),\n                                     vld1_s8((const int8_t *)(iq2s_grid + (qs[3] | ((qh[ib32+0] << 2) & 0x300)))));\n            q2s.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[4] | ((qh[ib32+1] << 8) & 0x300)))),\n                                     vld1_s8((const int8_t *)(iq2s_grid + (qs[5] | ((qh[ib32+1] << 6) & 0x300)))));\n            q2s.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq2s_grid + (qs[6] | ((qh[ib32+1] << 4) & 0x300)))),\n                                     vld1_s8((const int8_t *)(iq2s_grid + (qs[7] | ((qh[ib32+1] << 2) & 0x300)))));\n            qs += 8;\n\n            vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));\n            vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n            vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n            vs.val[0] = vceqq_u8(vs.val[0], mask2);\n            vs.val[1] = vceqq_u8(vs.val[1], mask2);\n\n            q2s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[0]);\n            q2s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[1]);\n\n            vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));\n            vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n            vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n            vs.val[0] = vceqq_u8(vs.val[0], mask2);\n            vs.val[1] = vceqq_u8(vs.val[1], mask2);\n\n            signs += 4;\n\n            q2s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[0], m1)), q2s.val[2]);\n            q2s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vorrq_u8(vs.val[1], m1)), q2s.val[3]);\n\n            const int32x4_t p1 = ggml_vdotq_s32(vzero, q2s.val[0], q8b.val[0]);\n            const int32x4_t p2 = ggml_vdotq_s32(vzero, q2s.val[1], q8b.val[1]);\n            const int32x4_t p3 = ggml_vdotq_s32(vzero, q2s.val[2], q8b.val[2]);\n            const int32x4_t p4 = ggml_vdotq_s32(vzero, q2s.val[3], q8b.val[3]);\n\n            sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32+0] & 0xf));\n            sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32+0] >>  4));\n            sumi1 += vaddvq_s32(p3) * (1 + 2*(x[i].scales[ib32+1] & 0xf));\n            sumi2 += vaddvq_s32(p4) * (1 + 2*(x[i].scales[ib32+1] >>  4));\n        }\n        sumf += d*(sumi1 + sumi2);\n    }\n\n    *s = 0.125f * sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[2];\n\n    ggml_int8x16x4_t q3s;\n    ggml_int8x16x4_t q8b;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        float sumf1 = 0, sumf2 = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n            memcpy(aux32, gas, 2*sizeof(uint32_t)); gas += 2*sizeof(uint32_t);\n            const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]);\n            const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]);\n            const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]);\n            const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]);\n            q3 += 16;\n            q3s.val[0] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >>  0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >>  7) & 127))));\n            q3s.val[1] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[0] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[0] >> 21) & 127))));\n            q3s.val[2] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >>  0) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >>  7) & 127))));\n            q3s.val[3] = vcombine_s8(vld1_s8((const void *)(signs64 + ((aux32[1] >> 14) & 127))), vld1_s8((const void *)(signs64 + ((aux32[1] >> 21) & 127))));\n            q3s.val[0] = vmulq_s8(q3s.val[0], vreinterpretq_s8_u32(aux32x4_0));\n            q3s.val[1] = vmulq_s8(q3s.val[1], vreinterpretq_s8_u32(aux32x4_1));\n            q3s.val[2] = vmulq_s8(q3s.val[2], vreinterpretq_s8_u32(aux32x4_2));\n            q3s.val[3] = vmulq_s8(q3s.val[3], vreinterpretq_s8_u32(aux32x4_3));\n            const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);\n            const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);\n            sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28));\n            sumf2 += vaddvq_s32(p2) * (0.5f + (aux32[1] >> 28));\n        }\n        sumf += d*(sumf1 + sumf2);\n    }\n    *s = 0.5f * sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__ARM_NEON)\n\n    typedef union {\n        uint16x8_t vec_index;\n        uint16_t   index[8];\n    } vec_index_t;\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n    static const int16_t k_shift[8] = {8, 7, 6, 5, 4, 3, 2, 1};\n\n    const ggml_uint8x16x2_t mask1 = ggml_vld1q_u8_x2(k_mask1);\n    const uint8x16_t        mask2 = vld1q_u8(k_mask2);\n\n    const int16x8_t  hshift = vld1q_s16(k_shift);\n    const uint16x8_t m256   = vdupq_n_u16(256);\n    const uint8x16_t m1     = vdupq_n_u8(1);\n\n    uint8x16x2_t vs;\n    ggml_int8x16x4_t q3s;\n    ggml_int8x16x4_t q8b;\n    vec_index_t idx;\n\n    uint32_t scales32[2];\n    const uint8_t * scales8 = (const uint8_t *)scales32;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(scales32, x[i].scales, 4);\n        scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101;\n        scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101;\n\n        int sumi1 = 0, sumi2 = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            const uint8x16_t idx_l = vld1q_u8(qs); qs += 16;\n            idx.vec_index = vorrq_u16(vmovl_u8(vget_low_u8 (idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+0]), hshift), m256));\n            const uint32x4_t aux32x4_0 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],\n                                                        iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);\n            const uint32x4_t aux32x4_1 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],\n                                                        iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);\n            idx.vec_index = vorrq_u16(vmovl_u8(vget_high_u8(idx_l)), vandq_u16(vshlq_u16(vdupq_n_u16(qh[ib32+1]), hshift), m256));\n            const uint32x4_t aux32x4_2 = ggml_vld1q_u32(iq3s_grid[idx.index[0]], iq3s_grid[idx.index[1]],\n                                                        iq3s_grid[idx.index[2]], iq3s_grid[idx.index[3]]);\n            const uint32x4_t aux32x4_3 = ggml_vld1q_u32(iq3s_grid[idx.index[4]], iq3s_grid[idx.index[5]],\n                                                        iq3s_grid[idx.index[6]], iq3s_grid[idx.index[7]]);\n\n\n            vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[0] | ((uint32_t) signs[1] << 16)));\n            vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n            vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n            vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);\n            vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);\n\n            q3s.val[0] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_0));\n            q3s.val[1] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_1));\n\n            vs.val[0] = vreinterpretq_u8_u32(vdupq_n_u32(signs[2] | ((uint32_t) signs[3] << 16)));\n            vs.val[1] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[1]), mask2);\n            vs.val[0] = vandq_u8(ggml_vqtbl1q_u8(vs.val[0], mask1.val[0]), mask2);\n            vs.val[0] = vorrq_u8(vceqq_u8(vs.val[0], mask2), m1);\n            vs.val[1] = vorrq_u8(vceqq_u8(vs.val[1], mask2), m1);\n\n            signs += 4;\n\n            q3s.val[2] = vmulq_s8(vreinterpretq_s8_u8(vs.val[0]), vreinterpretq_s8_u32(aux32x4_2));\n            q3s.val[3] = vmulq_s8(vreinterpretq_s8_u8(vs.val[1]), vreinterpretq_s8_u32(aux32x4_3));\n\n            const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]);\n            const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]);\n\n            sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0];\n            sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4];\n        }\n        sumf += d*(sumi1 + sumi2);\n    }\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __ARM_NEON\n\n    ggml_int8x16x4_t q1b;\n    ggml_int8x16x4_t q8b;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint16_t * qh = x[i].qh;\n\n        int sumi1 = 0, sumi2 = 0, sumi3 = 0;\n\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n\n            q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[ib+0] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[ib+0] << 5) & 0x700)))));\n            q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[ib+0] << 2) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[ib+0] >> 1) & 0x700)))));\n            q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[ib+1] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[ib+1] << 5) & 0x700)))));\n            q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[ib+1] << 2) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[ib+1] >> 1) & 0x700)))));\n            qs += 8;\n\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[0], q8b.val[0]), q1b.val[1], q8b.val[1]);\n            const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q1b.val[2], q8b.val[2]), q1b.val[3], q8b.val[3]);\n\n            const int ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n            const int ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n            sumi1 += vaddvq_s32(p1) * ls1;\n            sumi2 += vaddvq_s32(p2) * ls2;\n            sumi3 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * ls1 * (qh[ib+0] & 0x8000 ? -1 : 1)\n                   + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * ls2 * (qh[ib+1] & 0x8000 ? -1 : 1);\n\n        }\n\n        sumf += y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d) * (sumi1 + sumi2 + IQ1S_DELTA * sumi3);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_m * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    iq1m_scale_t scale;\n\n#if defined __ARM_NEON\n    const int32x4_t mask  = vdupq_n_s32(0x7);\n    const int32x4_t mone  = vdupq_n_s32(1);\n    const int32x4_t mzero = vdupq_n_s32(0);\n\n    ggml_int8x16x4_t deltas;\n    deltas.val[0] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(+1));\n    deltas.val[1] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(+1));\n    deltas.val[2] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(-1));\n    deltas.val[3] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(-1));\n\n    ggml_int8x16x4_t q1b;\n    ggml_int8x16x4_t q8b;\n\n    uint32_t aux32;\n    const uint8_t * aux8 = (const uint8_t *)&aux32;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint8_t  * qh = x[i].qh;\n        const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n        scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n        int32x4_t sumi1 = mzero;\n        int32x4_t sumi2 = mzero;\n\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n\n            q1b.val[0] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[0] | ((qh[0] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[1] | ((qh[0] << 4) & 0x700)))));\n            q1b.val[1] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[2] | ((qh[1] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[3] | ((qh[1] << 4) & 0x700)))));\n            q1b.val[2] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[4] | ((qh[2] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[5] | ((qh[2] << 4) & 0x700)))));\n            q1b.val[3] = vcombine_s8(vld1_s8((const int8_t *)(iq1s_grid + (qs[6] | ((qh[3] << 8) & 0x700)))),\n                                     vld1_s8((const int8_t *)(iq1s_grid + (qs[7] | ((qh[3] << 4) & 0x700)))));\n\n            q8b = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            const int32x4_t p1 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[0], q8b.val[0]), ggml_vdotq_s32(mzero, q1b.val[1], q8b.val[1]));\n            const int32x4_t p2 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, q1b.val[3], q8b.val[3]));\n            const int32x4_t p12 = vpaddq_s32(p1, p2);\n\n            const uint32_t * qh32 = (const uint32_t *)qh; // we are 4-byte aligned, so we can do that\n            aux32 = ((qh32[0] >> 3) & 0x01010101) | ((qh32[0] >> 6) & 0x02020202);\n\n            const int32x4_t p3 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[0]], q8b.val[0]), ggml_vdotq_s32(mzero, deltas.val[aux8[1]], q8b.val[1]));\n            const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[2]], q8b.val[2]), ggml_vdotq_s32(mzero, deltas.val[aux8[3]], q8b.val[3]));\n            const int32x4_t p34 = vpaddq_s32(p3, p4);\n\n            int32x4_t scales_4 = ggml_vld1q_u32(sc[ib/2] >> 0, sc[ib/2] >> 3, sc[ib/2] >> 6, sc[ib/2] >> 9);\n\n            scales_4 = vaddq_s32(vshlq_n_s32(vandq_s32(scales_4, mask), 1), mone);\n\n            sumi1 = vmlaq_s32(sumi1, scales_4, p12);\n            sumi2 = vmlaq_s32(sumi2, scales_4, p34);\n\n            qs += 8; qh += 4;\n\n        }\n\n        sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (vaddvq_s32(sumi1) + IQ1M_DELTA * vaddvq_s32(sumi2));\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(scale);\n    ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __ARM_NEON\n    const int8x16_t values = vld1q_s8(kvalues_iq4nl);\n    const uint8x16_t m4b = vdupq_n_u8(0x0f);\n    uint8x16x2_t q4bits;\n    int8x16x4_t q4b;\n    int8x16x4_t q8b;\n    int32x4_t prod_1, prod_2;\n\n    for (; ib + 1 < nb; ib += 2) {\n\n        q4bits.val[0] = vld1q_u8(x[ib + 0].qs);\n        q4bits.val[1] = vld1q_u8(x[ib + 1].qs);\n        q8b.val[0]    = vld1q_s8(y[ib + 0].qs);\n        q8b.val[1]    = vld1q_s8(y[ib + 0].qs + 16);\n        q8b.val[2]    = vld1q_s8(y[ib + 1].qs);\n        q8b.val[3]    = vld1q_s8(y[ib + 1].qs + 16);\n\n        q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[0], m4b));\n        q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n        q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[1], m4b));\n        q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n        prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n        prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n        sumf +=\n            GGML_CPU_FP16_TO_FP32(x[ib+0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * vaddvq_s32(prod_1) +\n            GGML_CPU_FP16_TO_FP32(x[ib+1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * vaddvq_s32(prod_2);\n    }\n\n#endif\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n        int sumi1 = 0, sumi2 = 0;\n        for (int j = 0; j < QK4_NL/2; ++j) {\n            sumi1 += y[ib].qs[j+       0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __ARM_NEON\n    const int8x16_t values = vld1q_s8(kvalues_iq4nl);\n    const uint8x16_t m4b = vdupq_n_u8(0x0f);\n    ggml_uint8x16x2_t q4bits;\n    ggml_int8x16x4_t q4b;\n    ggml_int8x16x4_t q8b;\n    int32x4_t prod_1, prod_2;\n\n    float sumf = 0;\n\n    for (int ibl = 0; ibl < nb; ++ibl) {\n\n        const int8_t  * q8 = y[ibl].qs;\n        const uint8_t * q4 = x[ibl].qs;\n        uint16_t h = x[ibl].scales_h;\n\n        int sumi1 = 0, sumi2 = 0;\n        for (int ib = 0; ib < QK_K/64; ++ib) {\n\n            q4bits = ggml_vld1q_u8_x2(q4); q4 += 32;\n            q8b    = ggml_vld1q_s8_x4(q8); q8 += 64;\n\n            q4b.val[0] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[0], m4b));\n            q4b.val[1] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[0], 4));\n            q4b.val[2] = ggml_vqtbl1q_s8(values, vandq_u8  (q4bits.val[1], m4b));\n            q4b.val[3] = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits.val[1], 4));\n\n            prod_1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]);\n            prod_2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]);\n\n            int ls1 = ((x[ibl].scales_l[ib] & 0xf) | ((h << 4) & 0x30)) - 32;\n            int ls2 = ((x[ibl].scales_l[ib] >>  4) | ((h << 2) & 0x30)) - 32;\n            h >>= 4;\n            sumi1 += vaddvq_s32(prod_1) * ls1;\n            sumi2 += vaddvq_s32(prod_2) * ls2;\n\n        }\n\n        sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/arm/repack.cpp",
    "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include <cmath>\n#include <cstring>\n#include <cassert>\n#include <cstdlib> // for qsort\n#include <cstdio>  // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD))\n// Helper for decoding scales and mins of Q4_K and Q5_K block formats\nstatic inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t * out_mins, int8_t * out_scales) {\n    constexpr uint32_t kmask1 = 0x3f3f3f3f;\n    constexpr uint32_t kmask2 = 0x0f0f0f0f;\n    constexpr uint32_t kmask3 = 0x03030303;\n    constexpr uint8_t  scales_size = 12;\n\n    uint32_t sm[3];\n    memcpy(sm, scales_in, scales_size);\n\n    const uint32_t   mins_0_3 = sm[1] & kmask1;\n    const uint32_t   mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4);\n    const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 };\n\n    *out_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins_u32)));\n\n    uint32_t scales_u32[2];\n    scales_u32[0] = sm[0] & kmask1;\n    scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4);\n    memcpy(out_scales, scales_u32, 8);\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__ARM_NEON)\n    float32x4_t srcv[4][8];\n    float id[4];\n\n    for (int i = 0; i < nb; i++) {\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);\n            for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);\n\n            for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);\n            for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);\n            for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);\n\n            const float amax = vmaxvq_f32(amaxv[0]);\n\n            const float d = amax / ((1 << 7) - 1);\n            id[row_iter] = d ? 1.0f / d : 0.0f;\n\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n        }\n\n        for (int j = 0; j < 8; j++) {\n            float32x4_t v = vmulq_n_f32(srcv[0][j], id[0]);\n            int32x4_t vi = vcvtnq_s32_f32(v);\n            y[i].qs[16 * j + 0] = vgetq_lane_s32(vi, 0);\n            y[i].qs[16 * j + 1] = vgetq_lane_s32(vi, 1);\n            y[i].qs[16 * j + 2] = vgetq_lane_s32(vi, 2);\n            y[i].qs[16 * j + 3] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[1][j], id[1]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[16 * j + 4] = vgetq_lane_s32(vi, 0);\n            y[i].qs[16 * j + 5] = vgetq_lane_s32(vi, 1);\n            y[i].qs[16 * j + 6] = vgetq_lane_s32(vi, 2);\n            y[i].qs[16 * j + 7] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[2][j], id[2]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[16 * j + 8] = vgetq_lane_s32(vi, 0);\n            y[i].qs[16 * j + 9] = vgetq_lane_s32(vi, 1);\n            y[i].qs[16 * j + 10] = vgetq_lane_s32(vi, 2);\n            y[i].qs[16 * j + 11] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[3][j], id[3]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[16 * j + 12] = vgetq_lane_s32(vi, 0);\n            y[i].qs[16 * j + 13] = vgetq_lane_s32(vi, 1);\n            y[i].qs[16 * j + 14] = vgetq_lane_s32(vi, 2);\n            y[i].qs[16 * j + 15] = vgetq_lane_s32(vi, 3);\n        }\n    }\n#else\n    UNUSED(nb);\n    UNUSED(y);\n    ggml_quantize_mat_q8_0_4x4_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__ARM_NEON)\n    float32x4_t srcv[4][8];\n    float id[4];\n\n    for (int i = 0; i < nb; i++) {\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);\n            for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);\n\n            for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);\n            for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);\n            for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);\n\n            const float amax = vmaxvq_f32(amaxv[0]);\n\n            const float d = amax / ((1 << 7) - 1);\n            id[row_iter] = d ? 1.0f / d : 0.0f;\n\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n        }\n\n        for (int j = 0; j < 4; j++) {\n            float32x4_t v = vmulq_n_f32(srcv[0][2 * j], id[0]);\n            int32x4_t vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 0] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 1] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 2] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 3] = vgetq_lane_s32(vi, 3);\n            v = vmulq_n_f32(srcv[0][2 * j + 1], id[0]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 4] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 5] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 6] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 7] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[1][2 * j], id[1]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 8] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 9] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 10] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 11] = vgetq_lane_s32(vi, 3);\n            v = vmulq_n_f32(srcv[1][2 * j + 1], id[1]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 12] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 13] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 14] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 15] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[2][2 * j], id[2]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 16] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 17] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 18] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 19] = vgetq_lane_s32(vi, 3);\n            v = vmulq_n_f32(srcv[2][2 * j + 1], id[2]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 20] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 21] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 22] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 23] = vgetq_lane_s32(vi, 3);\n\n            v = vmulq_n_f32(srcv[3][2 * j], id[3]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 24] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 25] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 26] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 27] = vgetq_lane_s32(vi, 3);\n            v = vmulq_n_f32(srcv[3][2 * j + 1], id[3]);\n            vi = vcvtnq_s32_f32(v);\n            y[i].qs[32 * j + 28] = vgetq_lane_s32(vi, 0);\n            y[i].qs[32 * j + 29] = vgetq_lane_s32(vi, 1);\n            y[i].qs[32 * j + 30] = vgetq_lane_s32(vi, 2);\n            y[i].qs[32 * j + 31] = vgetq_lane_s32(vi, 3);\n        }\n    }\n\n#else\n    UNUSED(nb);\n    UNUSED(y);\n    ggml_quantize_mat_q8_0_4x8_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx;\n\n    for (int c = 0; c < nc; c += ncols_interleaved) {\n        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n        float32x4_t acc = vdupq_n_f32(0);\n        for (int b = 0; b < nb; b++) {\n            int8x16_t b0 = vld1q_s8((const int8_t *) b_ptr->qs);\n            int8x16_t b1 = vld1q_s8((const int8_t *) b_ptr->qs + 16);\n            int8x16_t b2 = vld1q_s8((const int8_t *) b_ptr->qs + 32);\n            int8x16_t b3 = vld1q_s8((const int8_t *) b_ptr->qs + 48);\n            float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n            int8x16_t a0 = vld1q_s8(a_ptr->qs);\n            int8x16_t a1 = vld1q_s8(a_ptr->qs + qk/2);\n            float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n            int32x4_t ret = vdupq_n_s32(0);\n\n            ret = vdotq_laneq_s32(ret, b0 << 4, a0, 0);\n            ret = vdotq_laneq_s32(ret, b1 << 4, a0, 1);\n            ret = vdotq_laneq_s32(ret, b2 << 4, a0, 2);\n            ret = vdotq_laneq_s32(ret, b3 << 4, a0, 3);\n\n            ret = vdotq_laneq_s32(ret, b0 & 0xf0U, a1, 0);\n            ret = vdotq_laneq_s32(ret, b1 & 0xf0U, a1, 1);\n            ret = vdotq_laneq_s32(ret, b2 & 0xf0U, a1, 2);\n            ret = vdotq_laneq_s32(ret, b3 & 0xf0U, a1, 3);\n\n            acc = vfmaq_f32(acc, vcvtq_n_f32_s32(ret, 4),\n                            vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n            a_ptr++;\n            b_ptr++;\n        }\n        vst1q_f32(s, acc);\n        s += ncols_interleaved;\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx;\n\n    for (int c = 0; c < nc; c += ncols_interleaved) {\n        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n        float32x4_t acc = vdupq_n_f32(0);\n        for (int b = 0; b < nb; b++) {\n            int8x16_t b0 = vld1q_s8((const int8_t *) b_ptr->qs);\n            int8x16_t b1 = vld1q_s8((const int8_t *) b_ptr->qs + 16);\n            int8x16_t b2 = vld1q_s8((const int8_t *) b_ptr->qs + 32);\n            int8x16_t b3 = vld1q_s8((const int8_t *) b_ptr->qs + 48);\n            float16x4_t bd = vld1_f16((const __fp16 *) b_ptr->d);\n\n            int8x16_t a0 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs);\n            int8x16_t a1 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 1);\n            int8x16_t a2 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 2);\n            int8x16_t a3 = (int8x16_t) vld1q_dup_s64((const int64_t *) a_ptr->qs + 3);\n            float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n            int32x4_t ret0 = vdupq_n_s32(0);\n            int32x4_t ret1 = vdupq_n_s32(0);\n\n            ret0 = vdotq_s32(ret0, b0 << 4, a0);\n            ret1 = vdotq_s32(ret1, b1 << 4, a0);\n            ret0 = vdotq_s32(ret0, b2 << 4, a1);\n            ret1 = vdotq_s32(ret1, b3 << 4, a1);\n\n            ret0 = vdotq_s32(ret0, b0 & 0xf0U, a2);\n            ret1 = vdotq_s32(ret1, b1 & 0xf0U, a2);\n            ret0 = vdotq_s32(ret0, b2 & 0xf0U, a3);\n            ret1 = vdotq_s32(ret1, b3 & 0xf0U, a3);\n\n            int32x4_t ret = vpaddq_s32(ret0, ret1);\n\n            acc = vfmaq_f32(acc, vcvtq_n_f32_s32(ret, 4),\n                    vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n            a_ptr++;\n            b_ptr++;\n        }\n        vst1q_f32(s, acc);\n        s += ncols_interleaved;\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q4_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n#if defined(__ARM_FEATURE_SVE)\n    if (ggml_cpu_get_sve_cnt() == QK8_0) {\n        const void * b_ptr = vx;\n        const void * a_ptr = vy;\n        float * res_ptr = s;\n\n        __asm__ __volatile__(\n            \"ptrue p0.b\\n\"\n            \"add %x[b_ptr], %x[b_ptr], #0x10\\n\"\n            \"1:\"  // Column loop\n            \"add x22, %x[a_ptr], #0x2\\n\"\n            \"mov z31.b, #0x0\\n\"\n            \"mov x21, %x[nb]\\n\"\n            \"2:\"  // Block loop\n            \"ld1b { z30.b }, p0/Z, [%x[b_ptr]]\\n\"\n            \"ld1b { z29.b }, p0/Z, [%x[b_ptr], #1, MUL VL]\\n\"\n            \"mov z28.s, #0x0\\n\"\n            \"mov z27.s, #0x0\\n\"\n            \"ld1rd { z26.d }, p0/Z, [x22]\\n\"\n            \"ld1b { z25.b }, p0/Z, [%x[b_ptr], #2, MUL VL]\\n\"\n            \"sub x20, x22, #0x2\\n\"\n            \"sub x21, x21, #0x1\\n\"\n            \"ld1b { z24.b }, p0/Z, [%x[b_ptr], #3, MUL VL]\\n\"\n            \"ld1rd { z23.d }, p0/Z, [x22, #8]\\n\"\n            \"lsl z22.b, z30.b, #0x4\\n\"\n            \"lsl z16.b, z29.b, #0x4\\n\"\n            \"and z30.b, z30.b, #0xf0\\n\"\n            \"and z29.b, z29.b, #0xf0\\n\"\n            \"ld1rd { z21.d }, p0/Z, [x22, #16]\\n\"\n            \"ld1rd { z20.d }, p0/Z, [x22, #24]\\n\"\n            \"lsl z19.b, z25.b, #0x4\\n\"\n            \"and z25.b, z25.b, #0xf0\\n\"\n            \"ld1rh { z17.h }, p0/Z, [x20]\\n\"\n            \"ld1h { z18.s }, p0/Z, [%x[b_ptr], #-1, MUL VL]\\n\"\n            \"sdot z28.s, z22.b, z26.b\\n\"\n            \"sdot z27.s, z16.b, z26.b\\n\"\n            \"lsl z16.b, z24.b, #0x4\\n\"\n            \"add x22, x22, #0x22\\n\"\n            \"and z24.b, z24.b, #0xf0\\n\"\n            \"add %x[b_ptr], %x[b_ptr], #0x90\\n\"\n            \"fcvt z17.s, p0/m, z17.h\\n\"\n            \"fcvt z18.s, p0/m, z18.h\\n\"\n            \"sdot z28.s, z19.b, z23.b\\n\"\n            \"sdot z27.s, z16.b, z23.b\\n\"\n            \"fmul z18.s, z18.s, z17.s\\n\"\n            \"sdot z28.s, z30.b, z21.b\\n\"\n            \"sdot z27.s, z29.b, z21.b\\n\"\n            \"sdot z28.s, z25.b, z20.b\\n\"\n            \"sdot z27.s, z24.b, z20.b\\n\"\n            \"uzp1 z17.s, z28.s, z27.s\\n\"\n            \"uzp2 z16.s, z28.s, z27.s\\n\"\n            \"add z17.s, z17.s, z16.s\\n\"\n            \"asr z17.s, z17.s, #0x4\\n\"\n            \"scvtf z17.s, p0/m, z17.s\\n\"\n            \"fmla z31.s, p0/M, z17.s, z18.s\\n\"\n            \"cbnz x21, 2b\\n\"\n            \"sub %x[nc], %x[nc], #0x8\\n\"\n            \"st1w { z31.s }, p0, [%x[res_ptr]]\\n\"\n            \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n            \"cbnz %x[nc], 1b\\n\"\n            : [b_ptr] \"+&r\" (b_ptr), [res_ptr] \"+&r\" (res_ptr), [nc] \"+&r\" (nc)\n            : [a_ptr] \"r\" (a_ptr), [nb] \"r\" (nb)\n            : \"memory\", \"p0\", \"x20\", \"x21\", \"x22\", \"z16\", \"z17\", \"z18\", \"z19\", \"z20\", \"z21\", \"z22\", \"z23\", \"z24\", \"z25\", \"z26\", \"z27\", \"z28\", \"z29\", \"z30\", \"z31\"\n        );\n        return;\n    }\n#endif // #if defined(__ARM_FEATURE_SVE)\n\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n    ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const int8x16_t kvalues = vld1q_s8(kvalues_iq4nl);\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    float * res_ptr = s;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n        float32x4_t sumf = vdupq_n_f32(0);\n        for (int l = 0; l < nb; l++) {\n            uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);\n            uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);\n            uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);\n            uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);\n\n            int8x16_t b_0_hi = vqtbl1q_s8(kvalues, b_0 >> 4);\n            int8x16_t b_0_lo = vqtbl1q_s8(kvalues, b_0 & 0x0F);\n            int8x16_t b_1_hi = vqtbl1q_s8(kvalues, b_1 >> 4);\n            int8x16_t b_1_lo = vqtbl1q_s8(kvalues, b_1 & 0x0F);\n            int8x16_t b_2_hi = vqtbl1q_s8(kvalues, b_2 >> 4);\n            int8x16_t b_2_lo = vqtbl1q_s8(kvalues, b_2 & 0x0F);\n            int8x16_t b_3_hi = vqtbl1q_s8(kvalues, b_3 >> 4);\n            int8x16_t b_3_lo = vqtbl1q_s8(kvalues, b_3 & 0x0F);\n\n            int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);\n            int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);\n\n            int32x4_t sumi = vdupq_n_s32(0);\n            sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);\n            sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);\n            sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);\n            sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);\n            sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);\n            sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);\n            sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);\n            sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);\n\n            float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));\n            float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *)b_ptr[l].d));\n            float32x4_t d = a_d * b_d;\n\n            sumf = vmlaq_f32(sumf, d, vcvtq_f32_s32(sumi));\n        }\n\n        vst1q_f32(res_ptr + x * 4, sumf);\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n    ggml_gemv_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    float * res_ptr = s;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n        float32x4_t sumf = vdupq_n_f32(0);\n        for (int l = 0; l < nb; l++) {\n            uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);\n            uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);\n            uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);\n            uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);\n\n            int8x16_t b_0_hi = vqtbl1q_s8(kvalues, b_0 >> 4);\n            int8x16_t b_0_lo = vqtbl1q_s8(kvalues, b_0 & 0x0F);\n            int8x16_t b_1_hi = vqtbl1q_s8(kvalues, b_1 >> 4);\n            int8x16_t b_1_lo = vqtbl1q_s8(kvalues, b_1 & 0x0F);\n            int8x16_t b_2_hi = vqtbl1q_s8(kvalues, b_2 >> 4);\n            int8x16_t b_2_lo = vqtbl1q_s8(kvalues, b_2 & 0x0F);\n            int8x16_t b_3_hi = vqtbl1q_s8(kvalues, b_3 >> 4);\n            int8x16_t b_3_lo = vqtbl1q_s8(kvalues, b_3 & 0x0F);\n\n            int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);\n            int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);\n\n            int32x4_t sumi = vdupq_n_s32(0);\n            sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);\n            sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);\n            sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);\n            sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);\n            sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);\n            sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);\n            sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);\n            sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);\n\n            float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));\n            float32x4_t b_d = {\n                GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),\n                GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),\n                GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),\n                GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),\n            };\n            float32x4_t d = a_d * b_d;\n\n            sumf = vmlaq_f32(sumf, d, vcvtq_f32_s32(sumi));\n        }\n\n        vst1q_f32(res_ptr + x * 4, sumf);\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n    ggml_gemv_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_groups = ncols_interleaved / 4; // 0123 and 4567\n    const uint8x16_t m4b        = vdupq_n_u8(0x0f);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[col_groups];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n        for (int i = 0; i < col_groups; i++) {\n            acc_f32[i] = vdupq_n_f32(0);\n        }\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q4_d_0        = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q4_d_1        = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d          = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0123 = vmulq_f32(q4_d_0, q8_d);\n            float32x4_t sb_scale_4567 = vmulq_f32(q4_d_1, q8_d);\n            float32x4_t q4_dmin_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin));      // dmin 0..3\n            float32x4_t q4_dmin_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4));  // dmin 4..7\n            float32x4_t sb_min_0123   = vmulq_f32(q4_dmin_0, q8_d);\n            float32x4_t sb_min_4567   = vmulq_f32(q4_dmin_1, q8_d);\n\n            // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n            int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n            int32x4_t acc_lo[col_groups];\n            int32x4_t acc_hi[col_groups];\n\n            // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n            const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n            int16_t         bsums_arr[8];\n            vst1q_s16(bsums_arr, bsums);\n            for (int sb = 0; sb < QK_K / 64; sb++) {\n                for (int i = 0; i < col_groups; i++) {\n                    acc_lo[i] = vdupq_n_s32(0);\n                    acc_hi[i] = vdupq_n_s32(0);\n                }\n                // Need scales for the low and high nibbles\n                // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                int16x8_t q4sb_mins[2];\n                int16x8_t q4sb_scales[2];\n                for (int i = 0; i < 2; i++) {\n                    int8_t    aux_q4sb[8];\n                    const int offset = sb * 24 + i * 12;\n                    decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n                    q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n                }\n\n                int8x16_t q8_qs[64 / 16];\n                for (int i = 0; i < 64 / 16; i++) {\n                    q8_qs[i] = vld1q_s8(q8_ptr[b].qs + sb * 64 + i * 16);\n                }\n\n                for (int c = 0; c < col_groups; c++) {\n                    uint8x16_t q4_cols[8];\n                    for (int i = 0; i < 8; i++) {\n                        q4_cols[i] = vld1q_u8(q4_ptr[b].qs + sb * QK_K + i * 32 + 16 * c);\n                    }\n\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[0], m4b)), q8_qs[0], 0);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[1], m4b)), q8_qs[0], 1);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[2], m4b)), q8_qs[0], 2);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[3], m4b)), q8_qs[0], 3);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[4], m4b)), q8_qs[1], 0);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[5], m4b)), q8_qs[1], 1);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[6], m4b)), q8_qs[1], 2);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], vreinterpretq_s8_u8(vandq_u8(q4_cols[7], m4b)), q8_qs[1], 3);\n\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[0], 4)), q8_qs[2], 0);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[1], 4)), q8_qs[2], 1);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[2], 4)), q8_qs[2], 2);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[3], 4)), q8_qs[2], 3);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[4], 4)), q8_qs[3], 0);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[5], 4)), q8_qs[3], 1);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[6], 4)), q8_qs[3], 2);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], vreinterpretq_s8_u8(vshrq_n_u8(q4_cols[7], 4)), q8_qs[3], 3);\n                }\n\n                // Scales\n                // row c0123 blk0 and blk1\n                const int16x4_t   sc_0123_lo = vget_low_s16(q4sb_scales[0]);\n                const int16x4_t   sc_0123_hi = vget_low_s16(q4sb_scales[1]);\n                const float32x4_t sumf_0123  = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[0]),\n                                                                       vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[0])));\n                acc_f32[0]                   = vfmaq_f32(acc_f32[0], sb_scale_0123, sumf_0123);\n                // row c4567 blk0 and blk1\n                const int16x4_t   sc_4567_lo = vget_high_s16(q4sb_scales[0]);\n                const int16x4_t   sc_4567_hi = vget_high_s16(q4sb_scales[1]);\n                const float32x4_t sumf_4567  = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[1]),\n                                                                       vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[1])));\n                acc_f32[1]                   = vfmaq_f32(acc_f32[1], sb_scale_4567, sumf_4567);\n\n                // Bias Correction\n                const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n                const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n            }  // for sb\n\n            acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0123);\n            acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_4567);\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q4_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_pairs = ncols_interleaved / 2;\n    const uint8x16_t m4b       = vdupq_n_u8(0x0f);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[ncols_interleaved / 4];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n        for (int i = 0; i < ncols_interleaved / 4; i++) {\n            acc_f32[i] = vdupq_n_f32(0);\n        }\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q4_d_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q4_d_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d       = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0 = vmulq_f32(q4_d_0, q8_d);\n            float32x4_t sb_scale_1 = vmulq_f32(q4_d_1, q8_d);\n            float32x4_t q4_dmin_0  = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin));      // dmin 0..3\n            float32x4_t q4_dmin_1  = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4));  // dmin 4..7\n            float32x4_t sb_min_0   = vmulq_f32(q4_dmin_0, q8_d);\n            float32x4_t sb_min_1   = vmulq_f32(q4_dmin_1, q8_d);\n\n            // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n            int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n            // 2 sb each iteration\n            int32x4_t acc_lo[col_pairs];\n            int32x4_t acc_hi[col_pairs];\n\n            // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n            const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n            int16_t         bsums_arr[8];\n            vst1q_s16(bsums_arr, bsums);\n            for (int sb = 0; sb < QK_K / 64; sb++) {\n                for (int i = 0; i < col_pairs; i++) {\n                    acc_lo[i] = vdupq_n_s32(0);\n                    acc_hi[i] = vdupq_n_s32(0);\n                }\n                // Need scales for the low and high nibbles\n                // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                int16x8_t q4sb_mins[2];  // int16 as its needed for bias_acc later\n                int16x8_t q4sb_scales[2];\n                for (int i = 0; i < 2; i++) {\n                    int8_t    aux_q4sb[8];\n                    const int offset = sb * 24 + i * 12;\n                    decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n                    q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n                }\n\n                const uint8_t * q4_base = q4_ptr[b].qs + sb * QK_K;\n\n                // Load the 64 quants from q8K duplicated to use vecdots with the interleaved columns\n                // but still need the qs to use the low and hi bits from q4\n                const int8_t * q8_base = q8_ptr[b].qs + sb * 64;\n                int8x16_t      q8_qs[8];\n                for (int i = 0; i < 8; i++) {\n                    q8_qs[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base + i * 8));\n                }\n\n                // Q4s columns iterated in pairs (01, 23, 45, 67)\n                for (int cp = 0; cp < col_pairs; cp++) {\n                    uint8x16_t q4_qs_cp_0 = vld1q_u8(q4_base + 16 * cp);\n                    uint8x16_t q4_qs_cp_1 = vld1q_u8(q4_base + 16 * cp + 64);\n                    uint8x16_t q4_qs_cp_2 = vld1q_u8(q4_base + 16 * cp + 128);\n                    uint8x16_t q4_qs_cp_3 = vld1q_u8(q4_base + 16 * cp + 192);\n\n                    acc_lo[cp] =\n                        ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_0, m4b)), q8_qs[0]);  // 0 .. 7\n                    acc_lo[cp] =\n                        ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_1, m4b)), q8_qs[1]);  // 8 ..15\n                    acc_lo[cp] =\n                        ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_2, m4b)), q8_qs[2]);  // 16..23\n                    acc_lo[cp] =\n                        ggml_vdotq_s32(acc_lo[cp], vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_3, m4b)), q8_qs[3]);  // 24..31\n\n                    acc_hi[cp] =\n                        ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)), q8_qs[4]);  // 32..39\n                    acc_hi[cp] =\n                        ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)), q8_qs[5]);  // 40..47\n                    acc_hi[cp] =\n                        ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)), q8_qs[6]);  // 48..55\n                    acc_hi[cp] =\n                        ggml_vdotq_s32(acc_hi[cp], vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)), q8_qs[7]);  // 56..63\n                }\n\n                // Iterates over a pair of column pairs (4 columns) to use a single 128 register\n                // p = 0 -> 0123  p2 -> 4567\n                for (int i = 0, p = 0; p < col_pairs; i++, p += 2) {\n                    int16x4_t   group_scales_lo = p == 0 ? vget_low_s16(q4sb_scales[0]) : vget_high_s16(q4sb_scales[0]);\n                    int16x4_t   group_scales_hi = p == 0 ? vget_low_s16(q4sb_scales[1]) : vget_high_s16(q4sb_scales[1]);\n                    float32x4_t sb_scale        = p == 0 ? sb_scale_0 : sb_scale_1;\n\n                    // 0123 or 4567\n                    float32x4_t sumf_0 =\n                        vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_lo), vpaddq_s32(acc_lo[p], acc_lo[p + 1])));\n                    acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_0);\n\n                    float32x4_t sumf_1 =\n                        vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_hi), vpaddq_s32(acc_hi[p], acc_hi[p + 1])));\n                    acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_1);\n                }\n\n                // Multiply Acc bsum + mins\n                // Each pair of subblocks share the same bsums\n                // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n                int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n                int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n                // cols 0-3 bias\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n\n                // cols 4-7 bias\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n            }  // for sb\n\n            acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0);\n            acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_1);\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x4_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_groups = ncols_interleaved / 4;  // 0123 and 4567\n    const uint8x16_t m4b        = vdupq_n_u8(0x0f);\n    const uint8x16_t mone       = vdupq_n_u8(1);\n    const uint8x16_t mtwo       = vdupq_n_u8(2);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[col_groups];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n        for (int i = 0; i < col_groups; i++) {\n            acc_f32[i] = vdupq_n_f32(0);\n        }\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q5_d_0        = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q5_d_1        = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d          = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0123 = vmulq_f32(q5_d_0, q8_d);\n            float32x4_t sb_scale_4567 = vmulq_f32(q5_d_1, q8_d);\n            float32x4_t q5_dmin_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin));      // dmin 0..3\n            float32x4_t q5_dmin_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4));  // dmin 4..7\n            float32x4_t sb_min_0123   = vmulq_f32(q5_dmin_0, q8_d);\n            float32x4_t sb_min_4567   = vmulq_f32(q5_dmin_1, q8_d);\n\n            // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567\n            int32x4_t bias_acc[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };\n            int32x4_t acc_lo[col_groups];\n            int32x4_t acc_hi[col_groups];\n\n            // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n            const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n            int16_t         bsums_arr[8];\n            vst1q_s16(bsums_arr, bsums);\n\n            uint8x16_t qh[col_groups][8];\n            for (int c = 0; c < col_groups; c++) {\n                for (int i = 0; i < 8; i++) {\n                    qh[c][i] = vld1q_u8(q5_ptr[b].qh + i * 32 + 16 * c);\n                }\n            }\n\n            for (int sb = 0; sb < QK_K / 64; sb++) {\n                for (int i = 0; i < col_groups; i++) {\n                    acc_lo[i] = vdupq_n_s32(0);\n                    acc_hi[i] = vdupq_n_s32(0);\n                }\n                // Need scales for the low and high nibbles\n                // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                int16x8_t q5sb_mins[2];\n                int16x8_t q5sb_scales[2];\n                for (int i = 0; i < 2; i++) {\n                    int8_t    aux_q5sb[8];\n                    const int offset = sb * 24 + i * 12;\n                    decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n                    q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n                }\n\n                int8x16_t q8_qs[4];\n                for (int i = 0; i < 4; i++) {\n                    q8_qs[i] = vld1q_s8(q8_ptr[b].qs + sb * 64 + i * 16);\n                }\n\n                for (int c = 0; c < col_groups; c++) {\n                    uint8x16_t q5_cols[8];\n                    uint8x16_t hbit_lo[8];\n                    uint8x16_t hbit_hi[8];\n                    int8x16_t  q5_lo[8];\n                    int8x16_t  q5_hi[8];\n\n                    for (int i = 0; i < 8; i++) {\n                        q5_cols[i] = vld1q_u8(q5_ptr[b].qs + sb * QK_K + i * 32 + 16 * c);\n                        hbit_lo[i] = vandq_u8(qh[c][i], mone);\n                        hbit_hi[i] = vshlq_n_u8(vandq_u8(qh[c][i], mtwo), 3);\n                        qh[c][i]   = vshrq_n_u8(qh[c][i], 2);\n                        q5_lo[i]   = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_cols[i], m4b), hbit_lo[i], 4));\n                        q5_hi[i]   = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_cols[i], 4), hbit_hi[i]));\n                    }\n\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[0], q8_qs[0], 0);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[1], q8_qs[0], 1);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[2], q8_qs[0], 2);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[3], q8_qs[0], 3);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[4], q8_qs[1], 0);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[5], q8_qs[1], 1);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[6], q8_qs[1], 2);\n                    acc_lo[c] = vdotq_laneq_s32(acc_lo[c], q5_lo[7], q8_qs[1], 3);\n\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[0], q8_qs[2], 0);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[1], q8_qs[2], 1);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[2], q8_qs[2], 2);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[3], q8_qs[2], 3);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[4], q8_qs[3], 0);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[5], q8_qs[3], 1);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[6], q8_qs[3], 2);\n                    acc_hi[c] = vdotq_laneq_s32(acc_hi[c], q5_hi[7], q8_qs[3], 3);\n                }\n\n                // Scales\n                // row c0123 blk0 and blk1\n                const int16x4_t   sc_0123_lo = vget_low_s16(q5sb_scales[0]);\n                const int16x4_t   sc_0123_hi = vget_low_s16(q5sb_scales[1]);\n                const float32x4_t sumf_0123  = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[0]),\n                                                                       vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[0])));\n                acc_f32[0]                   = vfmaq_f32(acc_f32[0], sb_scale_0123, sumf_0123);\n                // row c4567 blk0 and blk1\n                const int16x4_t   sc_4567_lo = vget_high_s16(q5sb_scales[0]);\n                const int16x4_t   sc_4567_hi = vget_high_s16(q5sb_scales[1]);\n                const float32x4_t sumf_4567  = vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[1]),\n                                                                       vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[1])));\n                acc_f32[1]                   = vfmaq_f32(acc_f32[1], sb_scale_4567, sumf_4567);\n\n                // Bias Correction\n                const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n                const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n                bias_acc[0] = vmlal_s16(bias_acc[0], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n                bias_acc[1] = vmlal_s16(bias_acc[1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n            }  // for sb\n\n            acc_f32[0] = vmlsq_f32(acc_f32[0], vcvtq_f32_s32(bias_acc[0]), sb_min_0123);\n            acc_f32[1] = vmlsq_f32(acc_f32[1], vcvtq_f32_s32(bias_acc[1]), sb_min_4567);\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q5_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_pairs = ncols_interleaved / 2;\n    const uint8x16_t m4b       = vdupq_n_u8(0x0f);\n    const uint8x16_t mone      = vdupq_n_u8(1);\n    const uint8x16_t mtwo      = vdupq_n_u8(2);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[ncols_interleaved / 4];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n        for (int i = 0; i < ncols_interleaved / 4; i++) {\n            acc_f32[i] = vdupq_n_f32(0);\n        }\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q5_d_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q5_d_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d       = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0 = vmulq_f32(q5_d_0, q8_d);\n            float32x4_t sb_scale_1 = vmulq_f32(q5_d_1, q8_d);\n            float32x4_t q5_dmin_0  = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin));      // dmin 0..3\n            float32x4_t q5_dmin_1  = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4));  // dmin 4..7\n            float32x4_t sb_min_0   = vmulq_f32(q5_dmin_0, q8_d);\n            float32x4_t sb_min_1   = vmulq_f32(q5_dmin_1, q8_d);\n\n            // 2 sb each iteration\n            int32x4_t acc_lo[col_pairs];\n            int32x4_t acc_hi[col_pairs];\n\n            // Each bsum is 16 elements, pairwise add leaves us with the 8 bsums of the entire block\n            const int16x8_t bsums = vpaddq_s16(vld1q_s16(q8_ptr[b].bsums), vld1q_s16(q8_ptr[b].bsums + 8));\n            int16_t         bsums_arr[8];\n            vst1q_s16(bsums_arr, bsums);\n\n            // Load qh once per block and shift after each subblock\n            const uint8_t * qh_base = q5_ptr[b].qh;\n            uint8x16_t      qh[col_pairs][4];\n            for (int cp = 0; cp < col_pairs; cp++) {\n                qh[cp][0] = vld1q_u8(qh_base + 16 * cp);\n                qh[cp][1] = vld1q_u8(qh_base + 16 * cp + 64);\n                qh[cp][2] = vld1q_u8(qh_base + 16 * cp + 128);\n                qh[cp][3] = vld1q_u8(qh_base + 16 * cp + 192);\n            }\n\n            for (int sb = 0; sb < QK_K / 64; sb++) {\n                for (int i = 0; i < col_pairs; i++) {\n                    acc_lo[i] = vdupq_n_s32(0);\n                    acc_hi[i] = vdupq_n_s32(0);\n                }\n                // Need scales for the low and high nibbles\n                // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                int16x8_t q5sb_mins[2];  // int16 as its needed for bias_acc later\n                int16x8_t q5sb_scales[2];\n                for (int i = 0; i < 2; i++) {\n                    int8_t    aux_q5sb[8];\n                    const int offset = sb * 24 + i * 12;\n                    decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n                    q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n                }\n\n                const uint8_t * qs_base = q5_ptr[b].qs + sb * QK_K;\n\n                // Load the 64 quants from q8K duplicated to use vecdots with the interleaved columns\n                const int8_t * q8_base = q8_ptr[b].qs + sb * 64;\n                int8x16_t      q8_qs[8];\n                for (int i = 0; i < 8; i++) {\n                    q8_qs[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base + i * 8));\n                }\n\n                // Q5s column pair loop unrolled\n                {\n                    // Cols 01\n                    uint8x16_t qs_0 = vld1q_u8(qs_base);\n                    uint8x16_t qs_1 = vld1q_u8(qs_base + 64);\n                    uint8x16_t qs_2 = vld1q_u8(qs_base + 128);\n                    uint8x16_t qs_3 = vld1q_u8(qs_base + 192);\n\n                    uint8x16_t hbit_lo_0 = vandq_u8(qh[0][0], mone);\n                    uint8x16_t hbit_lo_1 = vandq_u8(qh[0][1], mone);\n                    uint8x16_t hbit_lo_2 = vandq_u8(qh[0][2], mone);\n                    uint8x16_t hbit_lo_3 = vandq_u8(qh[0][3], mone);\n                    uint8x16_t hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[0][0], mtwo), 3);\n                    uint8x16_t hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[0][1], mtwo), 3);\n                    uint8x16_t hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[0][2], mtwo), 3);\n                    uint8x16_t hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[0][3], mtwo), 3);\n\n                    qh[0][0] = vshrq_n_u8(qh[0][0], 2);\n                    qh[0][1] = vshrq_n_u8(qh[0][1], 2);\n                    qh[0][2] = vshrq_n_u8(qh[0][2], 2);\n                    qh[0][3] = vshrq_n_u8(qh[0][3], 2);\n\n                    acc_lo[0] = ggml_vdotq_s32(\n                        acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n                    acc_lo[0] = ggml_vdotq_s32(\n                        acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n                    acc_lo[0] = ggml_vdotq_s32(\n                        acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n                    acc_lo[0] = ggml_vdotq_s32(\n                        acc_lo[0], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n                    acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n                                               q8_qs[4]);\n                    acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n                                               q8_qs[5]);\n                    acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n                                               q8_qs[6]);\n                    acc_hi[0] = ggml_vdotq_s32(acc_hi[0], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n                                               q8_qs[7]);\n\n                    // Cols 23\n                    qs_0 = vld1q_u8(qs_base + 16);\n                    qs_1 = vld1q_u8(qs_base + 80);\n                    qs_2 = vld1q_u8(qs_base + 144);\n                    qs_3 = vld1q_u8(qs_base + 208);\n\n                    hbit_lo_0 = vandq_u8(qh[1][0], mone);\n                    hbit_lo_1 = vandq_u8(qh[1][1], mone);\n                    hbit_lo_2 = vandq_u8(qh[1][2], mone);\n                    hbit_lo_3 = vandq_u8(qh[1][3], mone);\n                    hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[1][0], mtwo), 3);\n                    hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[1][1], mtwo), 3);\n                    hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[1][2], mtwo), 3);\n                    hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[1][3], mtwo), 3);\n\n                    qh[1][0] = vshrq_n_u8(qh[1][0], 2);\n                    qh[1][1] = vshrq_n_u8(qh[1][1], 2);\n                    qh[1][2] = vshrq_n_u8(qh[1][2], 2);\n                    qh[1][3] = vshrq_n_u8(qh[1][3], 2);\n\n                    acc_lo[1] = ggml_vdotq_s32(\n                        acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n                    acc_lo[1] = ggml_vdotq_s32(\n                        acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n                    acc_lo[1] = ggml_vdotq_s32(\n                        acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n                    acc_lo[1] = ggml_vdotq_s32(\n                        acc_lo[1], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n                    acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n                                               q8_qs[4]);\n                    acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n                                               q8_qs[5]);\n                    acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n                                               q8_qs[6]);\n                    acc_hi[1] = ggml_vdotq_s32(acc_hi[1], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n                                               q8_qs[7]);\n\n                    // Cols 45\n                    qs_0 = vld1q_u8(qs_base + 32);\n                    qs_1 = vld1q_u8(qs_base + 96);\n                    qs_2 = vld1q_u8(qs_base + 160);\n                    qs_3 = vld1q_u8(qs_base + 224);\n\n                    hbit_lo_0 = vandq_u8(qh[2][0], mone);\n                    hbit_lo_1 = vandq_u8(qh[2][1], mone);\n                    hbit_lo_2 = vandq_u8(qh[2][2], mone);\n                    hbit_lo_3 = vandq_u8(qh[2][3], mone);\n                    hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[2][0], mtwo), 3);\n                    hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[2][1], mtwo), 3);\n                    hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[2][2], mtwo), 3);\n                    hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[2][3], mtwo), 3);\n\n                    qh[2][0] = vshrq_n_u8(qh[2][0], 2);\n                    qh[2][1] = vshrq_n_u8(qh[2][1], 2);\n                    qh[2][2] = vshrq_n_u8(qh[2][2], 2);\n                    qh[2][3] = vshrq_n_u8(qh[2][3], 2);\n\n                    acc_lo[2] = ggml_vdotq_s32(\n                        acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n                    acc_lo[2] = ggml_vdotq_s32(\n                        acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n                    acc_lo[2] = ggml_vdotq_s32(\n                        acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n                    acc_lo[2] = ggml_vdotq_s32(\n                        acc_lo[2], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n                    acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n                                               q8_qs[4]);\n                    acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n                                               q8_qs[5]);\n                    acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n                                               q8_qs[6]);\n                    acc_hi[2] = ggml_vdotq_s32(acc_hi[2], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n                                               q8_qs[7]);\n\n                    // Cols 45\n                    qs_0 = vld1q_u8(qs_base + 48);\n                    qs_1 = vld1q_u8(qs_base + 112);\n                    qs_2 = vld1q_u8(qs_base + 176);\n                    qs_3 = vld1q_u8(qs_base + 240);\n\n                    hbit_lo_0 = vandq_u8(qh[3][0], mone);\n                    hbit_lo_1 = vandq_u8(qh[3][1], mone);\n                    hbit_lo_2 = vandq_u8(qh[3][2], mone);\n                    hbit_lo_3 = vandq_u8(qh[3][3], mone);\n                    hbit_hi_0 = vshlq_n_u8(vandq_u8(qh[3][0], mtwo), 3);\n                    hbit_hi_1 = vshlq_n_u8(vandq_u8(qh[3][1], mtwo), 3);\n                    hbit_hi_2 = vshlq_n_u8(vandq_u8(qh[3][2], mtwo), 3);\n                    hbit_hi_3 = vshlq_n_u8(vandq_u8(qh[3][3], mtwo), 3);\n\n                    qh[3][0] = vshrq_n_u8(qh[3][0], 2);\n                    qh[3][1] = vshrq_n_u8(qh[3][1], 2);\n                    qh[3][2] = vshrq_n_u8(qh[3][2], 2);\n                    qh[3][3] = vshrq_n_u8(qh[3][3], 2);\n\n                    acc_lo[3] = ggml_vdotq_s32(\n                        acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_0, m4b), hbit_lo_0, 4)), q8_qs[0]);\n                    acc_lo[3] = ggml_vdotq_s32(\n                        acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_1, m4b), hbit_lo_1, 4)), q8_qs[1]);\n                    acc_lo[3] = ggml_vdotq_s32(\n                        acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_2, m4b), hbit_lo_2, 4)), q8_qs[2]);\n                    acc_lo[3] = ggml_vdotq_s32(\n                        acc_lo[3], vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_3, m4b), hbit_lo_3, 4)), q8_qs[3]);\n                    acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_0, 4), hbit_hi_0)),\n                                               q8_qs[4]);\n                    acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_1, 4), hbit_hi_1)),\n                                               q8_qs[5]);\n                    acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_2, 4), hbit_hi_2)),\n                                               q8_qs[6]);\n                    acc_hi[3] = ggml_vdotq_s32(acc_hi[3], vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_3, 4), hbit_hi_3)),\n                                               q8_qs[7]);\n                }\n\n                // Prepare bsum vectors for bias computation\n                // Each pair of subblocks share the same bsums\n                int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[2 * sb + 0]);\n                int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[2 * sb + 1]);\n\n                // Iterates over a pair of column pairs (4 columns) to use a single 128 register\n                // p = 0 -> 0123  p2 -> 4567\n                for (int i = 0, p = 0; p < col_pairs; i++, p += 2) {\n                    int16x4_t   group_scales_lo = p == 0 ? vget_low_s16(q5sb_scales[0]) : vget_high_s16(q5sb_scales[0]);\n                    int16x4_t   group_scales_hi = p == 0 ? vget_low_s16(q5sb_scales[1]) : vget_high_s16(q5sb_scales[1]);\n                    int16x4_t   group_mins_lo   = p == 0 ? vget_low_s16(q5sb_mins[0]) : vget_high_s16(q5sb_mins[0]);\n                    int16x4_t   group_mins_hi   = p == 0 ? vget_low_s16(q5sb_mins[1]) : vget_high_s16(q5sb_mins[1]);\n                    float32x4_t sb_scale        = p == 0 ? sb_scale_0 : sb_scale_1;\n                    float32x4_t sb_min          = p == 0 ? sb_min_0 : sb_min_1;\n\n                    // 0123 or 4567\n                    float32x4_t sumf_0 =\n                        vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_lo), vpaddq_s32(acc_lo[p], acc_lo[p + 1])));\n                    acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_0);\n\n                    float32x4_t sumf_1 =\n                        vcvtq_f32_s32(vmulq_s32(vmovl_s16(group_scales_hi), vpaddq_s32(acc_hi[p], acc_hi[p + 1])));\n                    acc_f32[i] = vfmaq_f32(acc_f32[i], sb_scale, sumf_1);\n\n                    // FUSED BIAS: Compute and subtract bias immediately\n                    // bias = (bsums_lo * mins_lo + bsums_hi * mins_hi) * sb_min\n                    int32x4_t bias       = vmull_s16(bsums_vec_lo, group_mins_lo);\n                    bias                 = vmlal_s16(bias, bsums_vec_hi, group_mins_hi);\n                    float32x4_t bias_f32 = vcvtq_f32_s32(bias);\n                    acc_f32[i]           = vmlsq_f32(acc_f32[i], sb_min, bias_f32);\n                }\n            }  // for sb\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x4_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_groups = ncols_interleaved / 4;\n    const uint8x16_t m4b        = vdupq_n_u8(0x0f);\n    const uint8x16_t mask_lo    = vdupq_n_u8(0x03);\n    const uint8x16_t mask_hi    = vdupq_n_u8(0x30);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[2];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n        for (int i = 0; i < col_groups; i++) {\n            acc_f32[i] = vdupq_n_f32(0);\n        }\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q6_d_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q6_d_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d       = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0 = vmulq_f32(q6_d_0, q8_d);\n            float32x4_t sb_scale_1 = vmulq_f32(q6_d_1, q8_d);\n\n            int32x4_t acc[col_groups];\n            for (int i = 0; i < col_groups; i++) {\n                acc[i] = vdupq_n_s32(0);\n            }\n\n            // Load all 16 scales once and widen to int16 (Q6_K has 16 scales per block)\n            // Reused for bias and dequantization later\n            int16_t q6_scales[16 * 8];\n            for (int i = 0; i < 16; i++) {\n                int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n                vst1q_s16(q6_scales + i * 8, scales);\n            }\n\n            // Compute bias per column using q8 bsums and preloaded scales to skip the -32 shift\n            int32x4_t bias_lo = vdupq_n_s32(0);\n            int32x4_t bias_hi = vdupq_n_s32(0);\n\n            // Load bsums in chunks of 4 to process with vectorized operations\n            for (int i = 0; i < 16; i += 4) {\n                int16x4_t bsums_vec   = vld1_s16(q8_ptr[b].bsums + i);\n                int16x4_t scales_lo_0 = vld1_s16(q6_scales + (i + 0) * 8);\n                int16x4_t scales_hi_0 = vld1_s16(q6_scales + (i + 0) * 8 + 4);\n                int16x4_t scales_lo_1 = vld1_s16(q6_scales + (i + 1) * 8);\n                int16x4_t scales_hi_1 = vld1_s16(q6_scales + (i + 1) * 8 + 4);\n                int16x4_t scales_lo_2 = vld1_s16(q6_scales + (i + 2) * 8);\n                int16x4_t scales_hi_2 = vld1_s16(q6_scales + (i + 2) * 8 + 4);\n                int16x4_t scales_lo_3 = vld1_s16(q6_scales + (i + 3) * 8);\n                int16x4_t scales_hi_3 = vld1_s16(q6_scales + (i + 3) * 8 + 4);\n\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_0, bsums_vec, 0);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_0, bsums_vec, 0);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_1, bsums_vec, 1);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_1, bsums_vec, 1);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_2, bsums_vec, 2);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_2, bsums_vec, 2);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_3, bsums_vec, 3);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_3, bsums_vec, 3);\n            }\n            bias_lo = vshlq_n_s32(bias_lo, 5);\n            bias_hi = vshlq_n_s32(bias_hi, 5);\n\n            // Process two 128-value halves per superblock\n            for (int half = 0; half < 2; half++) {\n                const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n                const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n                // A subblock (sb) is a set of weights that share the scale\n                // Since q6_K scales are per 16 elements\n                // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    const int8_t * q8_base_l = q8_ptr[b].qs + half * 128 + sb * 16;\n                    const int8_t * q8_base_h = q8_base_l + 64;\n\n                    // Load and duplicate q8 values (each register covers four interleaved columns of q6)\n                    int8x16_t q8_l[4];\n                    int8x16_t q8_h[4];\n                    for (int i = 0; i < 4; i++) {\n                        q8_l[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_l + i * 4));\n                        q8_h[i] = (int8x16_t) vld1q_dup_s32((const int32_t *) (q8_base_h + i * 4));\n                    }\n\n                    const int ql_off_base = sb * QK_K / 2;\n                    const int qh_off_base = ql_off_base & 255;  // wraps after 256 bytes\n\n                    // Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)\n                    uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);\n                    uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);\n                    uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);\n                    uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);\n\n                    // Adjust qh for subblocks 2 and 3 (shift right by 2)\n                    if (sb > 1) {\n                        q6_qh_0.val[0] = vshrq_n_u8(q6_qh_0.val[0], 2);\n                        q6_qh_0.val[1] = vshrq_n_u8(q6_qh_0.val[1], 2);\n                        q6_qh_0.val[2] = vshrq_n_u8(q6_qh_0.val[2], 2);\n                        q6_qh_0.val[3] = vshrq_n_u8(q6_qh_0.val[3], 2);\n                        q6_qh_1.val[0] = vshrq_n_u8(q6_qh_1.val[0], 2);\n                        q6_qh_1.val[1] = vshrq_n_u8(q6_qh_1.val[1], 2);\n                        q6_qh_1.val[2] = vshrq_n_u8(q6_qh_1.val[2], 2);\n                        q6_qh_1.val[3] = vshrq_n_u8(q6_qh_1.val[3], 2);\n                    }\n\n                    const uint8x16_t q6_ql[8] = { q6_ql_0.val[0], q6_ql_0.val[1], q6_ql_0.val[2], q6_ql_0.val[3],\n                                                  q6_ql_1.val[0], q6_ql_1.val[1], q6_ql_1.val[2], q6_ql_1.val[3] };\n                    const uint8x16_t q6_qh[8] = { q6_qh_0.val[0], q6_qh_0.val[1], q6_qh_0.val[2], q6_qh_0.val[3],\n                                                  q6_qh_1.val[0], q6_qh_1.val[1], q6_qh_1.val[2], q6_qh_1.val[3] };\n\n                    // Process column groups (0-3, 4-7)\n                    for (int g = 0; g < col_groups; g++) {\n                        int32x4_t sb_acc_l = vdupq_n_s32(0);\n                        int32x4_t sb_acc_h = vdupq_n_s32(0);\n\n                        for (int chunk = 0; chunk < 4; chunk++) {\n                            const int idx = chunk * 2 + g;\n\n                            const uint8x16_t q6_qs_l = q6_ql[idx];\n                            const uint8x16_t q6_qs_h = q6_qh[idx];\n\n                            // Extract high 2 bits for upper nibble reconstruction\n                            const uint8x16_t q6_qs_hh = vandq_u8(q6_qs_h, mask_hi);\n\n                            // q6 = (low4 | high2<<4), without -32 bias (handled via bsums)\n                            const int8x16_t q6_l =\n                                vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_l, m4b), vandq_u8(q6_qs_h, mask_lo), 4));\n                            const int8x16_t q6_h = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_l, 4), q6_qs_hh));\n\n                            sb_acc_l = vdotq_s32(sb_acc_l, q6_l, q8_l[chunk]);\n                            sb_acc_h = vdotq_s32(sb_acc_h, q6_h, q8_h[chunk]);\n                        }\n\n                        const int scale_idx_l = half * 8 + sb;\n                        const int scale_idx_h = half * 8 + sb + 4;\n\n                        const int32x4_t scale_vec_l = vmovl_s16(vld1_s16(q6_scales + scale_idx_l * 8 + g * 4));\n                        const int32x4_t scale_vec_h = vmovl_s16(vld1_s16(q6_scales + scale_idx_h * 8 + g * 4));\n\n                        acc[g] = vmlaq_s32(acc[g], sb_acc_l, scale_vec_l);\n                        acc[g] = vmlaq_s32(acc[g], sb_acc_h, scale_vec_h);\n                    }\n                }\n            }  // for half\n\n            // Bias correction\n            acc[0] = vsubq_s32(acc[0], bias_lo);\n            acc[1] = vsubq_s32(acc[1], bias_hi);\n\n            // Apply superblock scale (no mins for q6_K)\n            // acc[g] has [c0, c1, c2, c3]\n            float32x4_t w_0123 = vmulq_f32(vcvtq_f32_s32(acc[0]), sb_scale_0);\n            float32x4_t w_4567 = vmulq_f32(vcvtq_f32_s32(acc[1]), sb_scale_1);\n\n            acc_f32[0] = vaddq_f32(acc_f32[0], w_0123);\n            acc_f32[1] = vaddq_f32(acc_f32[1], w_4567);\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    col_pairs = ncols_interleaved / 2;\n    const uint8x16_t m4b       = vdupq_n_u8(0x0f);\n    const uint8x16_t mask_lo   = vdupq_n_u8(0x03);\n    const uint8x16_t mask_hi   = vdupq_n_u8(0x30);\n\n    // 1x8 tile = 2 x 4\n    float32x4_t acc_f32[2];\n\n    const block_q8_K * GGML_RESTRICT q8_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n        acc_f32[0] = vdupq_n_f32(0);\n        acc_f32[1] = vdupq_n_f32(0);\n\n        for (int b = 0; b < nb; b++) {\n            float32x4_t q6_d_0     = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d));      // d0 d1 d2 d3\n            float32x4_t q6_d_1     = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4));  // d4 d5 d6 d7\n            float32x4_t q8_d       = vdupq_n_f32(q8_ptr[b].d);\n            float32x4_t sb_scale_0 = vmulq_f32(q6_d_0, q8_d);\n            float32x4_t sb_scale_1 = vmulq_f32(q6_d_1, q8_d);\n\n            int32x2_t acc[col_pairs];\n            for (int i = 0; i < col_pairs; i++) {\n                acc[i] = vdup_n_s32(0);\n            }\n\n            // Load all 16 scales once and widen to int16 (Q6_K has 16 scales per block)\n            // Reused for bias and dequantization later\n            int16_t q6_scales[16 * 8];\n            for (int i = 0; i < 16; i++) {\n                int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n                vst1q_s16(q6_scales + i * 8, scales);\n            }\n\n            // Compute bias per column using q8 bsums and preloaded scales to skip the -32 shift\n            int32x4_t bias_lo = vdupq_n_s32(0);\n            int32x4_t bias_hi = vdupq_n_s32(0);\n\n            // Load bsums in chunks of 4 to process with vectorized operations\n            for (int i = 0; i < 16; i += 4) {\n                int16x4_t bsums_vec   = vld1_s16(q8_ptr[b].bsums + i);\n                int16x4_t scales_lo_0 = vld1_s16(q6_scales + (i + 0) * 8);\n                int16x4_t scales_hi_0 = vld1_s16(q6_scales + (i + 0) * 8 + 4);\n                int16x4_t scales_lo_1 = vld1_s16(q6_scales + (i + 1) * 8);\n                int16x4_t scales_hi_1 = vld1_s16(q6_scales + (i + 1) * 8 + 4);\n                int16x4_t scales_lo_2 = vld1_s16(q6_scales + (i + 2) * 8);\n                int16x4_t scales_hi_2 = vld1_s16(q6_scales + (i + 2) * 8 + 4);\n                int16x4_t scales_lo_3 = vld1_s16(q6_scales + (i + 3) * 8);\n                int16x4_t scales_hi_3 = vld1_s16(q6_scales + (i + 3) * 8 + 4);\n\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_0, bsums_vec, 0);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_0, bsums_vec, 0);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_1, bsums_vec, 1);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_1, bsums_vec, 1);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_2, bsums_vec, 2);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_2, bsums_vec, 2);\n                bias_lo = vmlal_lane_s16(bias_lo, scales_lo_3, bsums_vec, 3);\n                bias_hi = vmlal_lane_s16(bias_hi, scales_hi_3, bsums_vec, 3);\n            }\n            bias_lo = vshlq_n_s32(bias_lo, 5);\n            bias_hi = vshlq_n_s32(bias_hi, 5);\n\n            // Process two 128-value halves per superblock\n            for (int half = 0; half < 2; half++) {\n                const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n                const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n                // A subblock (sb) is a set of weights that share the scale\n                // Since q6_K scales are per 16 elements\n                // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    const int8_t * q8_base_l = q8_ptr[b].qs + half * 128 + sb * 16;\n                    const int8_t * q8_base_h = q8_base_l + 64;\n\n                    // Load and duplicate q8 values (each register covers two interleaved columns of q6)\n                    int8x16_t q8_l[2];\n                    int8x16_t q8_h[2];\n                    for (int i = 0; i < 2; i++) {\n                        q8_l[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base_l + i * 8));\n                        q8_h[i] = (int8x16_t) vld1q_dup_s64((const int64_t *) (q8_base_h + i * 8));\n                    }\n\n                    const int ql_off_base = sb * QK_K / 2;\n                    const int qh_off_base = ql_off_base & 255;  // wraps after 256 bytes\n\n                    // Load 4 vectors at once (64 bytes each for ql_0, ql_1, qh_0, qh_1)\n                    uint8x16x4_t q6_ql_0 = vld1q_u8_x4(ql_base + ql_off_base);\n                    uint8x16x4_t q6_ql_1 = vld1q_u8_x4(ql_base + ql_off_base + 64);\n                    uint8x16x4_t q6_qh_0 = vld1q_u8_x4(qh_base + qh_off_base);\n                    uint8x16x4_t q6_qh_1 = vld1q_u8_x4(qh_base + qh_off_base + 64);\n\n                    // Adjust qh for subblocks 2 and 3 (shift right by 2)\n                    if (sb > 1) {\n                        q6_qh_0.val[0] = vshrq_n_u8(q6_qh_0.val[0], 2);\n                        q6_qh_0.val[1] = vshrq_n_u8(q6_qh_0.val[1], 2);\n                        q6_qh_0.val[2] = vshrq_n_u8(q6_qh_0.val[2], 2);\n                        q6_qh_0.val[3] = vshrq_n_u8(q6_qh_0.val[3], 2);\n                        q6_qh_1.val[0] = vshrq_n_u8(q6_qh_1.val[0], 2);\n                        q6_qh_1.val[1] = vshrq_n_u8(q6_qh_1.val[1], 2);\n                        q6_qh_1.val[2] = vshrq_n_u8(q6_qh_1.val[2], 2);\n                        q6_qh_1.val[3] = vshrq_n_u8(q6_qh_1.val[3], 2);\n                    }\n\n                    // Process column pairs (0-1, 2-3, 4-5, 6-7)\n                    for (int cp = 0; cp < col_pairs; cp++) {\n                        const uint8x16_t q6_qs_cp_0_l = q6_ql_0.val[cp];\n                        const uint8x16_t q6_qs_cp_1_l = q6_ql_1.val[cp];\n                        const uint8x16_t q6_qs_cp_0_h = q6_qh_0.val[cp];\n                        const uint8x16_t q6_qs_cp_1_h = q6_qh_1.val[cp];\n\n                        // Extract high 2 bits for upper nibble reconstruction\n                        const uint8x16_t q6_qs_cp_0_hh = vandq_u8(q6_qs_cp_0_h, mask_hi);\n                        const uint8x16_t q6_qs_cp_1_hh = vandq_u8(q6_qs_cp_1_h, mask_hi);\n\n                        // q6 = (low4 | high2<<4), without -32 bias (handled via bsums)\n                        const int8x16_t q6_l0 = vreinterpretq_s8_u8(\n                            vsliq_n_u8(vandq_u8(q6_qs_cp_0_l, m4b), vandq_u8(q6_qs_cp_0_h, mask_lo), 4));\n                        const int8x16_t q6_l1 = vreinterpretq_s8_u8(\n                            vsliq_n_u8(vandq_u8(q6_qs_cp_1_l, m4b), vandq_u8(q6_qs_cp_1_h, mask_lo), 4));\n                        const int8x16_t q6_h0 =\n                            vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_0_l, 4), q6_qs_cp_0_hh));\n                        const int8x16_t q6_h1 =\n                            vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_1_l, 4), q6_qs_cp_1_hh));\n\n                        int32x4_t sb_acc_l = vdupq_n_s32(0);\n                        sb_acc_l           = vdotq_s32(sb_acc_l, q6_l0, q8_l[0]);\n                        sb_acc_l           = vdotq_s32(sb_acc_l, q6_l1, q8_l[1]);\n\n                        int32x4_t sb_acc_h = vdupq_n_s32(0);\n                        sb_acc_h           = vdotq_s32(sb_acc_h, q6_h0, q8_h[0]);\n                        sb_acc_h           = vdotq_s32(sb_acc_h, q6_h1, q8_h[1]);\n\n                        // Pairwise add to get per-column sums: [col0, col1]\n                        int32x2_t sum_l = vpadd_s32(vget_low_s32(sb_acc_l), vget_high_s32(sb_acc_l));\n                        int32x2_t sum_h = vpadd_s32(vget_low_s32(sb_acc_h), vget_high_s32(sb_acc_h));\n\n                        const int scale_idx_l = half * 8 + sb;\n                        const int scale_idx_h = half * 8 + sb + 4;\n\n                        // Access scales using array indexing (scales are interleaved by column)\n                        const int32x2_t scale_vec_l = { (int32_t) q6_scales[scale_idx_l * 8 + cp * 2],\n                                                        (int32_t) q6_scales[scale_idx_l * 8 + cp * 2 + 1] };\n                        const int32x2_t scale_vec_h = { (int32_t) q6_scales[scale_idx_h * 8 + cp * 2],\n                                                        (int32_t) q6_scales[scale_idx_h * 8 + cp * 2 + 1] };\n\n                        // Accumulate scaled results\n                        acc[cp] = vmla_s32(acc[cp], sum_l, scale_vec_l);\n                        acc[cp] = vmla_s32(acc[cp], sum_h, scale_vec_h);\n                    }\n                }\n            }  // for half\n\n            // Bias correction\n            acc[0] = vsub_s32(acc[0], vget_low_s32(bias_lo));\n            acc[1] = vsub_s32(acc[1], vget_high_s32(bias_lo));\n            acc[2] = vsub_s32(acc[2], vget_low_s32(bias_hi));\n            acc[3] = vsub_s32(acc[3], vget_high_s32(bias_hi));\n\n            // Apply superblock scale (no mins for q6_K)\n            // acc[cp] has [c0, c1]\n            float32x2_t w_01 = vmul_f32(vcvt_f32_s32(acc[0]), vget_low_f32(sb_scale_0));\n            float32x2_t w_23 = vmul_f32(vcvt_f32_s32(acc[1]), vget_high_f32(sb_scale_0));\n            float32x2_t w_45 = vmul_f32(vcvt_f32_s32(acc[2]), vget_low_f32(sb_scale_1));\n            float32x2_t w_67 = vmul_f32(vcvt_f32_s32(acc[3]), vget_high_f32(sb_scale_1));\n\n            acc_f32[0] = vaddq_f32(acc_f32[0], vcombine_f32(w_01, w_23));\n            acc_f32[1] = vaddq_f32(acc_f32[1], vcombine_f32(w_45, w_67));\n        }  // for b\n\n        int base = x * ncols_interleaved;\n        vst1q_f32(s + base, acc_f32[0]);\n        vst1q_f32(s + base + 4, acc_f32[1]);\n    }  // for x\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q6_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_4x4_q8_0(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;\n\n    for (int c = 0; c < nc; c += ncols_interleaved) {\n        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n        float32x4_t        acc   = vdupq_n_f32(0);\n        for (int b = 0; b < nb; b++) {\n            int8x16x4_t b_low  = vld1q_s8_x4((const int8_t *) b_ptr->qs);\n            int8x16x4_t b_high = vld1q_s8_x4((const int8_t *) b_ptr->qs + 64);\n            float16x4_t bd     = vld1_f16((const __fp16 *) b_ptr->d);\n\n            int8x16x2_t a  = vld1q_s8_x2(a_ptr->qs);\n            float16x4_t ad = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n            int32x4_t ret = vdupq_n_s32(0);\n\n            ret = vdotq_laneq_s32(ret, b_low.val[0], a.val[0], 0);\n            ret = vdotq_laneq_s32(ret, b_low.val[1], a.val[0], 1);\n            ret = vdotq_laneq_s32(ret, b_low.val[2], a.val[0], 2);\n            ret = vdotq_laneq_s32(ret, b_low.val[3], a.val[0], 3);\n\n            ret = vdotq_laneq_s32(ret, b_high.val[0], a.val[1], 0);\n            ret = vdotq_laneq_s32(ret, b_high.val[1], a.val[1], 1);\n            ret = vdotq_laneq_s32(ret, b_high.val[2], a.val[1], 2);\n            ret = vdotq_laneq_s32(ret, b_high.val[3], a.val[1], 3);\n\n            acc = vfmaq_f32(acc, vcvtq_f32_s32(ret), vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n            a_ptr++;\n            b_ptr++;\n        }\n        vst1q_f32(s, acc);\n        s += ncols_interleaved;\n    }\n    return;\n\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q8_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_4x8_q8_0(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx;\n\n    for (int c = 0; c < nc; c += ncols_interleaved) {\n        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n        float32x4_t        acc   = vdupq_n_f32(0);\n\n        for (int b = 0; b < nb; b++) {\n            int8x16x4_t b_low  = vld1q_s8_x4((const int8_t *) b_ptr->qs);\n            int8x16x4_t b_high = vld1q_s8_x4((const int8_t *) b_ptr->qs + 64);\n            float16x4_t bd     = vld1_f16((const __fp16 *) b_ptr->d);\n\n            int8x8x4_t  a_chunks = vld1_s8_x4(a_ptr->qs);\n            int8x16_t   a0       = vcombine_s8(a_chunks.val[0], a_chunks.val[0]);\n            int8x16_t   a1       = vcombine_s8(a_chunks.val[1], a_chunks.val[1]);\n            int8x16_t   a2       = vcombine_s8(a_chunks.val[2], a_chunks.val[2]);\n            int8x16_t   a3       = vcombine_s8(a_chunks.val[3], a_chunks.val[3]);\n            float16x4_t ad       = vld1_dup_f16((const __fp16 *) &a_ptr->d);\n\n            int32x4_t ret0 = vdupq_n_s32(0);\n            int32x4_t ret1 = vdupq_n_s32(0);\n\n            // 0..7\n            ret0 = vdotq_s32(ret0, b_low.val[0], a0);\n            ret1 = vdotq_s32(ret1, b_low.val[1], a0);\n            // 8..15\n            ret0 = vdotq_s32(ret0, b_low.val[2], a1);\n            ret1 = vdotq_s32(ret1, b_low.val[3], a1);\n            // 16..23\n            ret0 = vdotq_s32(ret0, b_high.val[0], a2);\n            ret1 = vdotq_s32(ret1, b_high.val[1], a2);\n            // 24..31\n            ret0 = vdotq_s32(ret0, b_high.val[2], a3);\n            ret1 = vdotq_s32(ret1, b_high.val[3], a3);\n\n            int32x4_t ret = vpaddq_s32(ret0, ret1);\n\n            acc = vfmaq_f32(acc, vcvtq_f32_s32(ret), vmulq_f32(vcvt_f32_f16(ad), vcvt_f32_f16(bd)));\n            a_ptr++;\n            b_ptr++;\n        }\n        vst1q_f32(s, acc);\n        s += ncols_interleaved;\n    }\n    return;\n\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemv_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const void * b_ptr = vx;\n    const void * a_ptr = vy;\n    float * res_ptr = s;\n    size_t res_stride = bs * sizeof(float);\n\n    __asm__ __volatile__(\n        \"mov x10, %x[nr]\\n\"\n        \"mov x9, #0x88\\n\"\n        \"cmp x10, #0x10\\n\"\n        \"mul x9, %x[nb], x9\\n\"\n        \"blt 4f\\n\"\n        \"1:\"  // Row loop\n        \"add x28, %x[b_ptr], #0x8\\n\"\n        \"mov x27, %x[nc]\\n\"\n        \"add x26, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n        \"2:\"  // Column loop\n        \"add x25, %x[a_ptr], #0x8\\n\"\n        \"movi v15.16b, #0x0\\n\"\n        \"movi v19.16b, #0x0\\n\"\n        \"mov x24, %x[nb]\\n\"\n        \"add x23, x25, x9\\n\"\n        \"movi v18.16b, #0x0\\n\"\n        \"movi v14.16b, #0x0\\n\"\n        \"add x22, x23, x9\\n\"\n        \"movi v11.16b, #0x0\\n\"\n        \"movi v13.16b, #0x0\\n\"\n        \"add x21, x22, x9\\n\"\n        \"movi v23.16b, #0x0\\n\"\n        \"movi v16.16b, #0x0\\n\"\n        \"movi v25.16b, #0x0\\n\"\n        \"movi v7.16b, #0x0\\n\"\n        \"movi v0.16b, #0x0\\n\"\n        \"movi v4.16b, #0x0\\n\"\n        \"movi v5.16b, #0x0\\n\"\n        \"movi v21.16b, #0x0\\n\"\n        \"movi v8.16b, #0x0\\n\"\n        \"movi v1.16b, #0x0\\n\"\n        \"3:\"  // Block loop\n        \"ldr q3, [x28, #0x0]\\n\"\n        \"ldr q31, [x25, #0x0]\\n\"\n        \"movi v28.16b, #0x4\\n\"\n        \"movi v10.4s, #0x0\\n\"\n        \"ldr q22, [x28, #0x10]\\n\"\n        \"ldr q6, [x25, #0x10]\\n\"\n        \"movi v29.4s, #0x0\\n\"\n        \"movi v9.4s, #0x0\\n\"\n        \"ldr q27, [x28, #0x20]\\n\"\n        \"ldr q30, [x28, #0x30]\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v24.16b, #0xf0\\n\"\n        \"ldr d2, [x25, #-0x8]\\n\"\n        \"ldr d26, [x23, #-0x8]\\n\"\n        \"sshl v12.16b, v3.16b, v28.16b\\n\"\n        \"sub x20, x28, #0x8\\n\"\n        \"ldr d17, [x20, #0x0]\\n\"\n        \"and v3.16b, v3.16b, v24.16b\\n\"\n        \"subs x24, x24, #0x1\\n\"\n        \"add x28, x28, #0x48\\n\"\n        \".inst 0x4f9fe18a  // sdot v10.4s, v12.16b, v31.4b[0]\\n\"\n        \".inst 0x4fbfe19d  // sdot v29.4s, v12.16b, v31.4b[1]\\n\"\n        \".inst 0x4f9fe989  // sdot v9.4s, v12.16b, v31.4b[2]\\n\"\n        \".inst 0x4fbfe994  // sdot v20.4s, v12.16b, v31.4b[3]\\n\"\n        \"sshl v31.16b, v22.16b, v28.16b\\n\"\n        \"and v22.16b, v22.16b, v24.16b\\n\"\n        \"fcvtl v17.4s, v17.4h\\n\"\n        \"fcvtl v2.4s, v2.4h\\n\"\n        \"fcvtl v26.4s, v26.4h\\n\"\n        \".inst 0x4f86e3ea  // sdot v10.4s, v31.16b, v6.4b[0]\\n\"\n        \".inst 0x4fa6e3fd  // sdot v29.4s, v31.16b, v6.4b[1]\\n\"\n        \".inst 0x4f86ebe9  // sdot v9.4s, v31.16b, v6.4b[2]\\n\"\n        \".inst 0x4fa6ebf4  // sdot v20.4s, v31.16b, v6.4b[3]\\n\"\n        \"sshl v6.16b, v27.16b, v28.16b\\n\"\n        \"sshl v28.16b, v30.16b, v28.16b\\n\"\n        \"and v27.16b, v27.16b, v24.16b\\n\"\n        \"and v30.16b, v30.16b, v24.16b\\n\"\n        \"ldr q24, [x25, #0x20]\\n\"\n        \".inst 0x4f98e0ca  // sdot v10.4s, v6.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e0dd  // sdot v29.4s, v6.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e8c9  // sdot v9.4s, v6.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e8d4  // sdot v20.4s, v6.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x25, #0x30]\\n\"\n        \".inst 0x4f98e38a  // sdot v10.4s, v28.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e39d  // sdot v29.4s, v28.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98eb89  // sdot v9.4s, v28.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8eb94  // sdot v20.4s, v28.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x25, #0x40]\\n\"\n        \".inst 0x4f98e06a  // sdot v10.4s, v3.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e07d  // sdot v29.4s, v3.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e869  // sdot v9.4s, v3.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e874  // sdot v20.4s, v3.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x25, #0x50]\\n\"\n        \".inst 0x4f98e2ca  // sdot v10.4s, v22.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e2dd  // sdot v29.4s, v22.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98eac9  // sdot v9.4s, v22.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8ead4  // sdot v20.4s, v22.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x25, #0x60]\\n\"\n        \".inst 0x4f98e36a  // sdot v10.4s, v27.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e37d  // sdot v29.4s, v27.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98eb69  // sdot v9.4s, v27.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8eb74  // sdot v20.4s, v27.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x25, #0x70]\\n\"\n        \"add x25, x25, #0x88\\n\"\n        \".inst 0x4f98e3ca  // sdot v10.4s, v30.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e3dd  // sdot v29.4s, v30.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98ebc9  // sdot v9.4s, v30.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8ebd4  // sdot v20.4s, v30.16b, v24.4b[3]\\n\"\n        \"fmul v24.4s, v17.4s, v2.s[0]\\n\"\n        \"scvtf v10.4s, v10.4s, #0x4\\n\"\n        \"scvtf v29.4s, v29.4s, #0x4\\n\"\n        \"scvtf v9.4s, v9.4s, #0x4\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"fmla v15.4s, v10.4s, v24.4s\\n\"\n        \"ldr q24, [x23, #0x0]\\n\"\n        \"fmul v10.4s, v17.4s, v2.s[1]\\n\"\n        \"fmla v19.4s, v29.4s, v10.4s\\n\"\n        \"ldr q10, [x23, #0x10]\\n\"\n        \"fmul v29.4s, v17.4s, v2.s[2]\\n\"\n        \"fmul v2.4s, v17.4s, v2.s[3]\\n\"\n        \"fmla v18.4s, v9.4s, v29.4s\\n\"\n        \"movi v9.4s, #0x0\\n\"\n        \"movi v29.4s, #0x0\\n\"\n        \".inst 0x4f98e189  // sdot v9.4s, v12.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e19d  // sdot v29.4s, v12.16b, v24.4b[1]\\n\"\n        \"fmla v14.4s, v20.4s, v2.4s\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v2.4s, #0x0\\n\"\n        \".inst 0x4f98e994  // sdot v20.4s, v12.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e982  // sdot v2.4s, v12.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x23, #0x20]\\n\"\n        \".inst 0x4f8ae3e9  // sdot v9.4s, v31.16b, v10.4b[0]\\n\"\n        \".inst 0x4faae3fd  // sdot v29.4s, v31.16b, v10.4b[1]\\n\"\n        \".inst 0x4f8aebf4  // sdot v20.4s, v31.16b, v10.4b[2]\\n\"\n        \".inst 0x4faaebe2  // sdot v2.4s, v31.16b, v10.4b[3]\\n\"\n        \"ldr q10, [x23, #0x30]\\n\"\n        \".inst 0x4f98e0c9  // sdot v9.4s, v6.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e0dd  // sdot v29.4s, v6.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e8d4  // sdot v20.4s, v6.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e8c2  // sdot v2.4s, v6.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x23, #0x40]\\n\"\n        \".inst 0x4f8ae389  // sdot v9.4s, v28.16b, v10.4b[0]\\n\"\n        \".inst 0x4faae39d  // sdot v29.4s, v28.16b, v10.4b[1]\\n\"\n        \".inst 0x4f8aeb94  // sdot v20.4s, v28.16b, v10.4b[2]\\n\"\n        \".inst 0x4faaeb82  // sdot v2.4s, v28.16b, v10.4b[3]\\n\"\n        \"ldr q10, [x23, #0x50]\\n\"\n        \".inst 0x4f98e069  // sdot v9.4s, v3.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e07d  // sdot v29.4s, v3.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e874  // sdot v20.4s, v3.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e862  // sdot v2.4s, v3.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x23, #0x60]\\n\"\n        \".inst 0x4f8ae2c9  // sdot v9.4s, v22.16b, v10.4b[0]\\n\"\n        \".inst 0x4faae2dd  // sdot v29.4s, v22.16b, v10.4b[1]\\n\"\n        \".inst 0x4f8aead4  // sdot v20.4s, v22.16b, v10.4b[2]\\n\"\n        \".inst 0x4faaeac2  // sdot v2.4s, v22.16b, v10.4b[3]\\n\"\n        \"ldr q10, [x23, #0x70]\\n\"\n        \"add x23, x23, #0x88\\n\"\n        \".inst 0x4f98e369  // sdot v9.4s, v27.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e37d  // sdot v29.4s, v27.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98eb74  // sdot v20.4s, v27.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8eb62  // sdot v2.4s, v27.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x22, #0x0]\\n\"\n        \".inst 0x4f8ae3c9  // sdot v9.4s, v30.16b, v10.4b[0]\\n\"\n        \".inst 0x4faae3dd  // sdot v29.4s, v30.16b, v10.4b[1]\\n\"\n        \".inst 0x4f8aebd4  // sdot v20.4s, v30.16b, v10.4b[2]\\n\"\n        \".inst 0x4faaebc2  // sdot v2.4s, v30.16b, v10.4b[3]\\n\"\n        \"fmul v10.4s, v17.4s, v26.s[0]\\n\"\n        \"scvtf v9.4s, v9.4s, #0x4\\n\"\n        \"scvtf v29.4s, v29.4s, #0x4\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"scvtf v2.4s, v2.4s, #0x4\\n\"\n        \"fmla v11.4s, v9.4s, v10.4s\\n\"\n        \"ldr q9, [x22, #0x10]\\n\"\n        \"fmul v10.4s, v17.4s, v26.s[1]\\n\"\n        \"fmla v13.4s, v29.4s, v10.4s\\n\"\n        \"ldr d29, [x22, #-0x8]\\n\"\n        \"fmul v10.4s, v17.4s, v26.s[2]\\n\"\n        \"fmul v26.4s, v17.4s, v26.s[3]\\n\"\n        \"fcvtl v29.4s, v29.4h\\n\"\n        \"fmla v23.4s, v20.4s, v10.4s\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v10.4s, #0x0\\n\"\n        \"fmla v16.4s, v2.4s, v26.4s\\n\"\n        \"movi v26.4s, #0x0\\n\"\n        \"movi v2.4s, #0x0\\n\"\n        \".inst 0x4f98e194  // sdot v20.4s, v12.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e18a  // sdot v10.4s, v12.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e99a  // sdot v26.4s, v12.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e982  // sdot v2.4s, v12.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x22, #0x20]\\n\"\n        \".inst 0x4f89e3f4  // sdot v20.4s, v31.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e3ea  // sdot v10.4s, v31.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89ebfa  // sdot v26.4s, v31.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9ebe2  // sdot v2.4s, v31.16b, v9.4b[3]\\n\"\n        \"ldr q9, [x22, #0x30]\\n\"\n        \".inst 0x4f98e0d4  // sdot v20.4s, v6.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e0ca  // sdot v10.4s, v6.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e8da  // sdot v26.4s, v6.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e8c2  // sdot v2.4s, v6.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x22, #0x40]\\n\"\n        \".inst 0x4f89e394  // sdot v20.4s, v28.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e38a  // sdot v10.4s, v28.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89eb9a  // sdot v26.4s, v28.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9eb82  // sdot v2.4s, v28.16b, v9.4b[3]\\n\"\n        \"ldr q9, [x22, #0x50]\\n\"\n        \".inst 0x4f98e074  // sdot v20.4s, v3.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e06a  // sdot v10.4s, v3.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e87a  // sdot v26.4s, v3.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e862  // sdot v2.4s, v3.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x22, #0x60]\\n\"\n        \".inst 0x4f89e2d4  // sdot v20.4s, v22.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e2ca  // sdot v10.4s, v22.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89eada  // sdot v26.4s, v22.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9eac2  // sdot v2.4s, v22.16b, v9.4b[3]\\n\"\n        \"ldr q9, [x22, #0x70]\\n\"\n        \"add x22, x22, #0x88\\n\"\n        \".inst 0x4f98e374  // sdot v20.4s, v27.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e36a  // sdot v10.4s, v27.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98eb7a  // sdot v26.4s, v27.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8eb62  // sdot v2.4s, v27.16b, v24.4b[3]\\n\"\n        \"ldr q24, [x21, #0x0]\\n\"\n        \".inst 0x4f89e3d4  // sdot v20.4s, v30.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e3ca  // sdot v10.4s, v30.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89ebda  // sdot v26.4s, v30.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9ebc2  // sdot v2.4s, v30.16b, v9.4b[3]\\n\"\n        \"fmul v9.4s, v17.4s, v29.s[0]\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"scvtf v10.4s, v10.4s, #0x4\\n\"\n        \"scvtf v26.4s, v26.4s, #0x4\\n\"\n        \"scvtf v2.4s, v2.4s, #0x4\\n\"\n        \"fmla v25.4s, v20.4s, v9.4s\\n\"\n        \"ldr q9, [x21, #0x10]\\n\"\n        \"fmul v20.4s, v17.4s, v29.s[1]\\n\"\n        \"fmla v7.4s, v10.4s, v20.4s\\n\"\n        \"ldr d20, [x21, #-0x8]\\n\"\n        \"fmul v10.4s, v17.4s, v29.s[2]\\n\"\n        \"fmul v29.4s, v17.4s, v29.s[3]\\n\"\n        \"fcvtl v20.4s, v20.4h\\n\"\n        \"fmla v0.4s, v26.4s, v10.4s\\n\"\n        \"movi v26.4s, #0x0\\n\"\n        \"movi v10.4s, #0x0\\n\"\n        \"fmla v4.4s, v2.4s, v29.4s\\n\"\n        \"movi v2.4s, #0x0\\n\"\n        \"movi v29.4s, #0x0\\n\"\n        \".inst 0x4f98e19a  // sdot v26.4s, v12.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e18a  // sdot v10.4s, v12.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e982  // sdot v2.4s, v12.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e99d  // sdot v29.4s, v12.16b, v24.4b[3]\\n\"\n        \"ldr q12, [x21, #0x20]\\n\"\n        \"fmul v24.4s, v17.4s, v20.s[0]\\n\"\n        \".inst 0x4f89e3fa  // sdot v26.4s, v31.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e3ea  // sdot v10.4s, v31.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89ebe2  // sdot v2.4s, v31.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9ebfd  // sdot v29.4s, v31.16b, v9.4b[3]\\n\"\n        \"ldr q9, [x21, #0x30]\\n\"\n        \"fmul v31.4s, v17.4s, v20.s[1]\\n\"\n        \".inst 0x4f8ce0da  // sdot v26.4s, v6.16b, v12.4b[0]\\n\"\n        \".inst 0x4face0ca  // sdot v10.4s, v6.16b, v12.4b[1]\\n\"\n        \".inst 0x4f8ce8c2  // sdot v2.4s, v6.16b, v12.4b[2]\\n\"\n        \".inst 0x4face8dd  // sdot v29.4s, v6.16b, v12.4b[3]\\n\"\n        \"ldr q12, [x21, #0x40]\\n\"\n        \"fmul v6.4s, v17.4s, v20.s[2]\\n\"\n        \"fmul v20.4s, v17.4s, v20.s[3]\\n\"\n        \".inst 0x4f89e39a  // sdot v26.4s, v28.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e38a  // sdot v10.4s, v28.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89eb82  // sdot v2.4s, v28.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9eb9d  // sdot v29.4s, v28.16b, v9.4b[3]\\n\"\n        \"ldr q9, [x21, #0x50]\\n\"\n        \".inst 0x4f8ce07a  // sdot v26.4s, v3.16b, v12.4b[0]\\n\"\n        \".inst 0x4face06a  // sdot v10.4s, v3.16b, v12.4b[1]\\n\"\n        \".inst 0x4f8ce862  // sdot v2.4s, v3.16b, v12.4b[2]\\n\"\n        \".inst 0x4face87d  // sdot v29.4s, v3.16b, v12.4b[3]\\n\"\n        \"ldr q12, [x21, #0x60]\\n\"\n        \".inst 0x4f89e2da  // sdot v26.4s, v22.16b, v9.4b[0]\\n\"\n        \".inst 0x4fa9e2ca  // sdot v10.4s, v22.16b, v9.4b[1]\\n\"\n        \".inst 0x4f89eac2  // sdot v2.4s, v22.16b, v9.4b[2]\\n\"\n        \".inst 0x4fa9eadd  // sdot v29.4s, v22.16b, v9.4b[3]\\n\"\n        \"ldr q17, [x21, #0x70]\\n\"\n        \"add x21, x21, #0x88\\n\"\n        \".inst 0x4f8ce37a  // sdot v26.4s, v27.16b, v12.4b[0]\\n\"\n        \".inst 0x4face36a  // sdot v10.4s, v27.16b, v12.4b[1]\\n\"\n        \".inst 0x4f8ceb62  // sdot v2.4s, v27.16b, v12.4b[2]\\n\"\n        \".inst 0x4faceb7d  // sdot v29.4s, v27.16b, v12.4b[3]\\n\"\n        \".inst 0x4f91e3da  // sdot v26.4s, v30.16b, v17.4b[0]\\n\"\n        \".inst 0x4fb1e3ca  // sdot v10.4s, v30.16b, v17.4b[1]\\n\"\n        \".inst 0x4f91ebc2  // sdot v2.4s, v30.16b, v17.4b[2]\\n\"\n        \".inst 0x4fb1ebdd  // sdot v29.4s, v30.16b, v17.4b[3]\\n\"\n        \"scvtf v26.4s, v26.4s, #0x4\\n\"\n        \"scvtf v10.4s, v10.4s, #0x4\\n\"\n        \"fmla v5.4s, v26.4s, v24.4s\\n\"\n        \"scvtf v2.4s, v2.4s, #0x4\\n\"\n        \"scvtf v29.4s, v29.4s, #0x4\\n\"\n        \"fmla v21.4s, v10.4s, v31.4s\\n\"\n        \"fmla v8.4s, v2.4s, v6.4s\\n\"\n        \"fmla v1.4s, v29.4s, v20.4s\\n\"\n        \"bgt 3b\\n\"\n        \"mov x20, %x[res_ptr]\\n\"\n        \"subs x27, x27, #0x4\\n\"\n        \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n        \"str q15, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q19, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q18, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q14, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q11, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q13, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q23, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q16, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q25, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q7, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q0, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q4, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q5, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q21, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q8, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q1, [x20, #0x0]\\n\"\n        \"bne 2b\\n\"\n        \"mov x20, #0x4\\n\"\n        \"sub x10, x10, #0x10\\n\"\n        \"cmp x10, #0x10\\n\"\n        \"mov %x[res_ptr], x26\\n\"\n        \"madd %x[a_ptr], x20, x9, %x[a_ptr]\\n\"\n        \"bge 1b\\n\"\n        \"4:\"  // Row loop skip\n        \"cbz x10, 9f\\n\"\n        \"5:\"  // Row tail: Row loop\n        \"add x24, %x[b_ptr], #0x8\\n\"\n        \"mov x23, %x[nc]\\n\"\n        \"add x22, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n        \"6:\"  // Row tail: Column loop\n        \"movi v15.16b, #0x0\\n\"\n        \"movi v19.16b, #0x0\\n\"\n        \"add x25, %x[a_ptr], #0x8\\n\"\n        \"mov x21, %x[nb]\\n\"\n        \"movi v18.16b, #0x0\\n\"\n        \"movi v14.16b, #0x0\\n\"\n        \"7:\"  // Row tail: Block loop\n        \"ldr q7, [x24, #0x0]\\n\"\n        \"ldr q5, [x25, #0x0]\\n\"\n        \"movi v9.16b, #0x4\\n\"\n        \"movi v4.4s, #0x0\\n\"\n        \"ldr q3, [x24, #0x10]\\n\"\n        \"ldr q2, [x25, #0x10]\\n\"\n        \"movi v1.4s, #0x0\\n\"\n        \"movi v0.4s, #0x0\\n\"\n        \"ldr q13, [x24, #0x20]\\n\"\n        \"ldr q31, [x25, #0x20]\\n\"\n        \"movi v30.4s, #0x0\\n\"\n        \"movi v29.16b, #0xf0\\n\"\n        \"ldr q28, [x24, #0x30]\\n\"\n        \"ldr q27, [x25, #0x30]\\n\"\n        \"sshl v20.16b, v7.16b, v9.16b\\n\"\n        \"sub x20, x24, #0x8\\n\"\n        \"ldr q26, [x25, #0x40]\\n\"\n        \"ldr q25, [x25, #0x50]\\n\"\n        \"sshl v17.16b, v3.16b, v9.16b\\n\"\n        \"and v7.16b, v7.16b, v29.16b\\n\"\n        \"ldr q24, [x25, #0x60]\\n\"\n        \"ldr q16, [x25, #0x70]\\n\"\n        \"sshl v22.16b, v13.16b, v9.16b\\n\"\n        \"and v3.16b, v3.16b, v29.16b\\n\"\n        \"ldr d21, [x20, #0x0]\\n\"\n        \"ldr d12, [x25, #-0x8]\\n\"\n        \".inst 0x4f85e284  // sdot v4.4s, v20.16b, v5.4b[0]\\n\"\n        \".inst 0x4fa5e281  // sdot v1.4s, v20.16b, v5.4b[1]\\n\"\n        \".inst 0x4f85ea80  // sdot v0.4s, v20.16b, v5.4b[2]\\n\"\n        \".inst 0x4fa5ea9e  // sdot v30.4s, v20.16b, v5.4b[3]\\n\"\n        \"sshl v9.16b, v28.16b, v9.16b\\n\"\n        \"subs x21, x21, #0x1\\n\"\n        \"and v13.16b, v13.16b, v29.16b\\n\"\n        \"and v28.16b, v28.16b, v29.16b\\n\"\n        \"add x25, x25, #0x88\\n\"\n        \"add x24, x24, #0x48\\n\"\n        \"fcvtl v21.4s, v21.4h\\n\"\n        \"fcvtl v12.4s, v12.4h\\n\"\n        \".inst 0x4f82e224  // sdot v4.4s, v17.16b, v2.4b[0]\\n\"\n        \".inst 0x4fa2e221  // sdot v1.4s, v17.16b, v2.4b[1]\\n\"\n        \".inst 0x4f82ea20  // sdot v0.4s, v17.16b, v2.4b[2]\\n\"\n        \".inst 0x4fa2ea3e  // sdot v30.4s, v17.16b, v2.4b[3]\\n\"\n        \"fmul v11.4s, v21.4s, v12.s[0]\\n\"\n        \"fmul v23.4s, v21.4s, v12.s[1]\\n\"\n        \"fmul v17.4s, v21.4s, v12.s[2]\\n\"\n        \".inst 0x4f9fe2c4  // sdot v4.4s, v22.16b, v31.4b[0]\\n\"\n        \"fmul v6.4s, v21.4s, v12.s[3]\\n\"\n        \".inst 0x4fbfe2c1  // sdot v1.4s, v22.16b, v31.4b[1]\\n\"\n        \".inst 0x4f9feac0  // sdot v0.4s, v22.16b, v31.4b[2]\\n\"\n        \".inst 0x4fbfeade  // sdot v30.4s, v22.16b, v31.4b[3]\\n\"\n        \".inst 0x4f9be124  // sdot v4.4s, v9.16b, v27.4b[0]\\n\"\n        \".inst 0x4fbbe121  // sdot v1.4s, v9.16b, v27.4b[1]\\n\"\n        \".inst 0x4f9be920  // sdot v0.4s, v9.16b, v27.4b[2]\\n\"\n        \".inst 0x4fbbe93e  // sdot v30.4s, v9.16b, v27.4b[3]\\n\"\n        \".inst 0x4f9ae0e4  // sdot v4.4s, v7.16b, v26.4b[0]\\n\"\n        \".inst 0x4fbae0e1  // sdot v1.4s, v7.16b, v26.4b[1]\\n\"\n        \".inst 0x4f9ae8e0  // sdot v0.4s, v7.16b, v26.4b[2]\\n\"\n        \".inst 0x4fbae8fe  // sdot v30.4s, v7.16b, v26.4b[3]\\n\"\n        \".inst 0x4f99e064  // sdot v4.4s, v3.16b, v25.4b[0]\\n\"\n        \".inst 0x4fb9e061  // sdot v1.4s, v3.16b, v25.4b[1]\\n\"\n        \".inst 0x4f99e860  // sdot v0.4s, v3.16b, v25.4b[2]\\n\"\n        \".inst 0x4fb9e87e  // sdot v30.4s, v3.16b, v25.4b[3]\\n\"\n        \".inst 0x4f98e1a4  // sdot v4.4s, v13.16b, v24.4b[0]\\n\"\n        \".inst 0x4fb8e1a1  // sdot v1.4s, v13.16b, v24.4b[1]\\n\"\n        \".inst 0x4f98e9a0  // sdot v0.4s, v13.16b, v24.4b[2]\\n\"\n        \".inst 0x4fb8e9be  // sdot v30.4s, v13.16b, v24.4b[3]\\n\"\n        \".inst 0x4f90e384  // sdot v4.4s, v28.16b, v16.4b[0]\\n\"\n        \".inst 0x4fb0e381  // sdot v1.4s, v28.16b, v16.4b[1]\\n\"\n        \".inst 0x4f90eb80  // sdot v0.4s, v28.16b, v16.4b[2]\\n\"\n        \".inst 0x4fb0eb9e  // sdot v30.4s, v28.16b, v16.4b[3]\\n\"\n        \"scvtf v4.4s, v4.4s, #0x4\\n\"\n        \"scvtf v1.4s, v1.4s, #0x4\\n\"\n        \"scvtf v0.4s, v0.4s, #0x4\\n\"\n        \"fmla v15.4s, v4.4s, v11.4s\\n\"\n        \"scvtf v30.4s, v30.4s, #0x4\\n\"\n        \"fmla v19.4s, v1.4s, v23.4s\\n\"\n        \"fmla v18.4s, v0.4s, v17.4s\\n\"\n        \"fmla v14.4s, v30.4s, v6.4s\\n\"\n        \"bgt 7b\\n\"\n        \"mov x20, %x[res_ptr]\\n\"\n        \"cmp x10, #0x1\\n\"\n        \"str q15, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"cmp x10, #0x2\\n\"\n        \"str q19, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"cmp x10, #0x3\\n\"\n        \"str q18, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"str q14, [x20, #0x0]\\n\"\n        \"8:\"  // Row tail: Accumulator store skip\n        \"subs x23, x23, #0x4\\n\"\n        \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n        \"bne 6b\\n\"\n        \"subs x10, x10, #0x4\\n\"\n        \"add %x[a_ptr], %x[a_ptr], x9\\n\"\n        \"mov %x[res_ptr], x22\\n\"\n        \"bgt 5b\\n\"\n        \"9:\"  // Row tail: Row loop skip\n        : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n        : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n        : \"cc\", \"memory\", \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\", \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\", \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\", \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\", \"x9\", \"x10\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\"\n    );\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n    ggml_gemm_q4_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    const void * b_ptr = vx;\n    const void * a_ptr = vy;\n    float * res_ptr = s;\n    size_t res_stride = bs * sizeof(float);\n\n    __asm__ __volatile__(\n        \"mov x10, %x[nr]\\n\"\n        \"mov x9, #0x88\\n\"\n        \"cmp x10, #0x10\\n\"\n        \"mul x9, %x[nb], x9\\n\"\n        \"blt 4f\\n\"\n        \"1:\"  // Row loop\n        \"add x28, %x[b_ptr], #0x8\\n\"\n        \"mov x27, %x[nc]\\n\"\n        \"add x26, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n        \"2:\"  // Column loop\n        \"add x25, %x[a_ptr], #0x8\\n\"\n        \"movi v2.16b, #0x0\\n\"\n        \"movi v10.16b, #0x0\\n\"\n        \"mov x24, %x[nb]\\n\"\n        \"add x23, x25, x9\\n\"\n        \"movi v12.16b, #0x0\\n\"\n        \"movi v28.16b, #0x0\\n\"\n        \"add x22, x23, x9\\n\"\n        \"movi v11.16b, #0x0\\n\"\n        \"movi v13.16b, #0x0\\n\"\n        \"add x21, x22, x9\\n\"\n        \"movi v22.16b, #0x0\\n\"\n        \"movi v23.16b, #0x0\\n\"\n        \"movi v25.16b, #0x0\\n\"\n        \"movi v5.16b, #0x0\\n\"\n        \"movi v7.16b, #0x0\\n\"\n        \"movi v4.16b, #0x0\\n\"\n        \"movi v6.16b, #0x0\\n\"\n        \"movi v30.16b, #0x0\\n\"\n        \"movi v24.16b, #0x0\\n\"\n        \"movi v14.16b, #0x0\\n\"\n        \"3:\"  // Block loop\n        \"ldr q21, [x28, #0x0]\\n\"\n        \"ldr q16, [x28, #0x10]\\n\"\n        \"movi v1.16b, #0x4\\n\"\n        \"movi v19.4s, #0x0\\n\"\n        \"ldr q27, [x25, #0x0]\\n\"\n        \"ldr q15, [x25, #0x10]\\n\"\n        \"movi v26.4s, #0x0\\n\"\n        \"movi v18.4s, #0x0\\n\"\n        \"ldr q29, [x28, #0x20]\\n\"\n        \"ldr q3, [x28, #0x30]\\n\"\n        \"movi v17.4s, #0x0\\n\"\n        \"movi v0.16b, #0xf0\\n\"\n        \"ldr d20, [x25, #-0x8]\\n\"\n        \"ldr d9, [x23, #-0x8]\\n\"\n        \"sshl v8.16b, v21.16b, v1.16b\\n\"\n        \"sshl v31.16b, v16.16b, v1.16b\\n\"\n        \"and v21.16b, v21.16b, v0.16b\\n\"\n        \"and v16.16b, v16.16b, v0.16b\\n\"\n        \"sub x20, x28, #0x8\\n\"\n        \"subs x24, x24, #0x1\\n\"\n        \"add x28, x28, #0x48\\n\"\n        \".inst 0x4e88a773  // smmla v19.4s, v27.16b, v8.16b\\n\"\n        \".inst 0x4e9fa77a  // smmla v26.4s, v27.16b, v31.16b\\n\"\n        \"ldr q27, [x25, #0x20]\\n\"\n        \".inst 0x4e88a5f2  // smmla v18.4s, v15.16b, v8.16b\\n\"\n        \".inst 0x4e9fa5f1  // smmla v17.4s, v15.16b, v31.16b\\n\"\n        \"sshl v15.16b, v29.16b, v1.16b\\n\"\n        \"sshl v1.16b, v3.16b, v1.16b\\n\"\n        \"and v29.16b, v29.16b, v0.16b\\n\"\n        \"and v3.16b, v3.16b, v0.16b\\n\"\n        \"ldr q0, [x25, #0x30]\\n\"\n        \"fcvtl v20.4s, v20.4h\\n\"\n        \".inst 0x4e8fa773  // smmla v19.4s, v27.16b, v15.16b\\n\"\n        \"fcvtl v9.4s, v9.4h\\n\"\n        \".inst 0x4e81a77a  // smmla v26.4s, v27.16b, v1.16b\\n\"\n        \"ldr q27, [x25, #0x40]\\n\"\n        \".inst 0x4e8fa412  // smmla v18.4s, v0.16b, v15.16b\\n\"\n        \".inst 0x4e81a411  // smmla v17.4s, v0.16b, v1.16b\\n\"\n        \"ldr q0, [x25, #0x50]\\n\"\n        \".inst 0x4e95a773  // smmla v19.4s, v27.16b, v21.16b\\n\"\n        \".inst 0x4e90a77a  // smmla v26.4s, v27.16b, v16.16b\\n\"\n        \"ldr q27, [x25, #0x60]\\n\"\n        \".inst 0x4e95a412  // smmla v18.4s, v0.16b, v21.16b\\n\"\n        \".inst 0x4e90a411  // smmla v17.4s, v0.16b, v16.16b\\n\"\n        \"ldr q0, [x25, #0x70]\\n\"\n        \"add x25, x25, #0x88\\n\"\n        \".inst 0x4e9da773  // smmla v19.4s, v27.16b, v29.16b\\n\"\n        \".inst 0x4e83a77a  // smmla v26.4s, v27.16b, v3.16b\\n\"\n        \"ldr d27, [x20, #0x0]\\n\"\n        \".inst 0x4e9da412  // smmla v18.4s, v0.16b, v29.16b\\n\"\n        \".inst 0x4e83a411  // smmla v17.4s, v0.16b, v3.16b\\n\"\n        \"fcvtl v27.4s, v27.4h\\n\"\n        \"uzp1 v0.2d, v19.2d, v26.2d\\n\"\n        \"uzp2 v26.2d, v19.2d, v26.2d\\n\"\n        \"fmul v19.4s, v27.4s, v20.s[0]\\n\"\n        \"scvtf v0.4s, v0.4s, #0x4\\n\"\n        \"scvtf v26.4s, v26.4s, #0x4\\n\"\n        \"fmla v2.4s, v0.4s, v19.4s\\n\"\n        \"ldr q19, [x23, #0x0]\\n\"\n        \"uzp1 v0.2d, v18.2d, v17.2d\\n\"\n        \"uzp2 v18.2d, v18.2d, v17.2d\\n\"\n        \"fmul v17.4s, v27.4s, v20.s[1]\\n\"\n        \"scvtf v0.4s, v0.4s, #0x4\\n\"\n        \"scvtf v18.4s, v18.4s, #0x4\\n\"\n        \"fmla v10.4s, v26.4s, v17.4s\\n\"\n        \"ldr q17, [x23, #0x10]\\n\"\n        \"fmul v26.4s, v27.4s, v20.s[2]\\n\"\n        \"fmul v20.4s, v27.4s, v20.s[3]\\n\"\n        \"fmla v12.4s, v0.4s, v26.4s\\n\"\n        \"ldr d0, [x22, #-0x8]\\n\"\n        \"ldr d26, [x21, #-0x8]\\n\"\n        \"fcvtl v0.4s, v0.4h\\n\"\n        \"fmla v28.4s, v18.4s, v20.4s\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v18.4s, #0x0\\n\"\n        \".inst 0x4e88a674  // smmla v20.4s, v19.16b, v8.16b\\n\"\n        \".inst 0x4e9fa672  // smmla v18.4s, v19.16b, v31.16b\\n\"\n        \"ldr q19, [x23, #0x20]\\n\"\n        \"fcvtl v26.4s, v26.4h\\n\"\n        \".inst 0x4e8fa674  // smmla v20.4s, v19.16b, v15.16b\\n\"\n        \".inst 0x4e81a672  // smmla v18.4s, v19.16b, v1.16b\\n\"\n        \"ldr q19, [x23, #0x40]\\n\"\n        \".inst 0x4e95a674  // smmla v20.4s, v19.16b, v21.16b\\n\"\n        \".inst 0x4e90a672  // smmla v18.4s, v19.16b, v16.16b\\n\"\n        \"ldr q19, [x23, #0x60]\\n\"\n        \".inst 0x4e9da674  // smmla v20.4s, v19.16b, v29.16b\\n\"\n        \".inst 0x4e83a672  // smmla v18.4s, v19.16b, v3.16b\\n\"\n        \"uzp1 v19.2d, v20.2d, v18.2d\\n\"\n        \"scvtf v19.4s, v19.4s, #0x4\\n\"\n        \"uzp2 v20.2d, v20.2d, v18.2d\\n\"\n        \"fmul v18.4s, v27.4s, v9.s[0]\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"fmla v11.4s, v19.4s, v18.4s\\n\"\n        \"ldr q18, [x22, #0x0]\\n\"\n        \"fmul v19.4s, v27.4s, v9.s[1]\\n\"\n        \"fmla v13.4s, v20.4s, v19.4s\\n\"\n        \"movi v19.4s, #0x0\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \".inst 0x4e88a633  // smmla v19.4s, v17.16b, v8.16b\\n\"\n        \".inst 0x4e9fa634  // smmla v20.4s, v17.16b, v31.16b\\n\"\n        \"ldr q17, [x23, #0x30]\\n\"\n        \".inst 0x4e8fa633  // smmla v19.4s, v17.16b, v15.16b\\n\"\n        \".inst 0x4e81a634  // smmla v20.4s, v17.16b, v1.16b\\n\"\n        \"ldr q17, [x23, #0x50]\\n\"\n        \".inst 0x4e95a633  // smmla v19.4s, v17.16b, v21.16b\\n\"\n        \".inst 0x4e90a634  // smmla v20.4s, v17.16b, v16.16b\\n\"\n        \"ldr q17, [x23, #0x70]\\n\"\n        \"add x23, x23, #0x88\\n\"\n        \".inst 0x4e9da633  // smmla v19.4s, v17.16b, v29.16b\\n\"\n        \".inst 0x4e83a634  // smmla v20.4s, v17.16b, v3.16b\\n\"\n        \"uzp1 v17.2d, v19.2d, v20.2d\\n\"\n        \"scvtf v17.4s, v17.4s, #0x4\\n\"\n        \"uzp2 v20.2d, v19.2d, v20.2d\\n\"\n        \"fmul v19.4s, v27.4s, v9.s[2]\\n\"\n        \"fmul v9.4s, v27.4s, v9.s[3]\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"fmla v22.4s, v17.4s, v19.4s\\n\"\n        \"ldr q17, [x22, #0x10]\\n\"\n        \"movi v19.4s, #0x0\\n\"\n        \".inst 0x4e88a653  // smmla v19.4s, v18.16b, v8.16b\\n\"\n        \"fmla v23.4s, v20.4s, v9.4s\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v9.4s, #0x0\\n\"\n        \".inst 0x4e9fa654  // smmla v20.4s, v18.16b, v31.16b\\n\"\n        \"ldr q18, [x22, #0x20]\\n\"\n        \".inst 0x4e88a629  // smmla v9.4s, v17.16b, v8.16b\\n\"\n        \".inst 0x4e8fa653  // smmla v19.4s, v18.16b, v15.16b\\n\"\n        \".inst 0x4e81a654  // smmla v20.4s, v18.16b, v1.16b\\n\"\n        \"ldr q18, [x22, #0x40]\\n\"\n        \".inst 0x4e95a653  // smmla v19.4s, v18.16b, v21.16b\\n\"\n        \".inst 0x4e90a654  // smmla v20.4s, v18.16b, v16.16b\\n\"\n        \"ldr q18, [x22, #0x60]\\n\"\n        \".inst 0x4e9da653  // smmla v19.4s, v18.16b, v29.16b\\n\"\n        \".inst 0x4e83a654  // smmla v20.4s, v18.16b, v3.16b\\n\"\n        \"movi v18.4s, #0x0\\n\"\n        \".inst 0x4e9fa632  // smmla v18.4s, v17.16b, v31.16b\\n\"\n        \"ldr q17, [x22, #0x30]\\n\"\n        \".inst 0x4e8fa629  // smmla v9.4s, v17.16b, v15.16b\\n\"\n        \".inst 0x4e81a632  // smmla v18.4s, v17.16b, v1.16b\\n\"\n        \"ldr q17, [x22, #0x50]\\n\"\n        \".inst 0x4e95a629  // smmla v9.4s, v17.16b, v21.16b\\n\"\n        \".inst 0x4e90a632  // smmla v18.4s, v17.16b, v16.16b\\n\"\n        \"ldr q17, [x22, #0x70]\\n\"\n        \"add x22, x22, #0x88\\n\"\n        \".inst 0x4e9da629  // smmla v9.4s, v17.16b, v29.16b\\n\"\n        \".inst 0x4e83a632  // smmla v18.4s, v17.16b, v3.16b\\n\"\n        \"uzp1 v17.2d, v19.2d, v20.2d\\n\"\n        \"uzp2 v20.2d, v19.2d, v20.2d\\n\"\n        \"fmul v19.4s, v27.4s, v0.s[0]\\n\"\n        \"scvtf v17.4s, v17.4s, #0x4\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"fmla v25.4s, v17.4s, v19.4s\\n\"\n        \"ldr q19, [x21, #0x0]\\n\"\n        \"fmul v17.4s, v27.4s, v0.s[1]\\n\"\n        \"fmla v5.4s, v20.4s, v17.4s\\n\"\n        \"ldr q17, [x21, #0x10]\\n\"\n        \"uzp1 v20.2d, v9.2d, v18.2d\\n\"\n        \"uzp2 v9.2d, v9.2d, v18.2d\\n\"\n        \"fmul v18.4s, v27.4s, v0.s[2]\\n\"\n        \"fmul v0.4s, v27.4s, v0.s[3]\\n\"\n        \"scvtf v20.4s, v20.4s, #0x4\\n\"\n        \"scvtf v9.4s, v9.4s, #0x4\\n\"\n        \"fmla v7.4s, v20.4s, v18.4s\\n\"\n        \"movi v20.4s, #0x0\\n\"\n        \"movi v18.4s, #0x0\\n\"\n        \".inst 0x4e88a674  // smmla v20.4s, v19.16b, v8.16b\\n\"\n        \".inst 0x4e9fa672  // smmla v18.4s, v19.16b, v31.16b\\n\"\n        \"ldr q19, [x21, #0x20]\\n\"\n        \"fmla v4.4s, v9.4s, v0.4s\\n\"\n        \"movi v9.4s, #0x0\\n\"\n        \"movi v0.4s, #0x0\\n\"\n        \".inst 0x4e88a629  // smmla v9.4s, v17.16b, v8.16b\\n\"\n        \"fmul v8.4s, v27.4s, v26.s[0]\\n\"\n        \".inst 0x4e9fa620  // smmla v0.4s, v17.16b, v31.16b\\n\"\n        \"ldr q17, [x21, #0x30]\\n\"\n        \".inst 0x4e8fa674  // smmla v20.4s, v19.16b, v15.16b\\n\"\n        \"fmul v31.4s, v27.4s, v26.s[1]\\n\"\n        \".inst 0x4e81a672  // smmla v18.4s, v19.16b, v1.16b\\n\"\n        \"ldr q19, [x21, #0x40]\\n\"\n        \".inst 0x4e8fa629  // smmla v9.4s, v17.16b, v15.16b\\n\"\n        \"fmul v15.4s, v27.4s, v26.s[2]\\n\"\n        \"fmul v27.4s, v27.4s, v26.s[3]\\n\"\n        \".inst 0x4e81a620  // smmla v0.4s, v17.16b, v1.16b\\n\"\n        \"ldr q1, [x21, #0x50]\\n\"\n        \".inst 0x4e95a674  // smmla v20.4s, v19.16b, v21.16b\\n\"\n        \".inst 0x4e90a672  // smmla v18.4s, v19.16b, v16.16b\\n\"\n        \"ldr q26, [x21, #0x60]\\n\"\n        \".inst 0x4e95a429  // smmla v9.4s, v1.16b, v21.16b\\n\"\n        \".inst 0x4e90a420  // smmla v0.4s, v1.16b, v16.16b\\n\"\n        \"ldr q21, [x21, #0x70]\\n\"\n        \"add x21, x21, #0x88\\n\"\n        \".inst 0x4e9da754  // smmla v20.4s, v26.16b, v29.16b\\n\"\n        \".inst 0x4e83a752  // smmla v18.4s, v26.16b, v3.16b\\n\"\n        \".inst 0x4e9da6a9  // smmla v9.4s, v21.16b, v29.16b\\n\"\n        \".inst 0x4e83a6a0  // smmla v0.4s, v21.16b, v3.16b\\n\"\n        \"uzp1 v29.2d, v20.2d, v18.2d\\n\"\n        \"uzp2 v21.2d, v20.2d, v18.2d\\n\"\n        \"scvtf v29.4s, v29.4s, #0x4\\n\"\n        \"uzp1 v18.2d, v9.2d, v0.2d\\n\"\n        \"uzp2 v16.2d, v9.2d, v0.2d\\n\"\n        \"scvtf v21.4s, v21.4s, #0x4\\n\"\n        \"fmla v6.4s, v29.4s, v8.4s\\n\"\n        \"scvtf v18.4s, v18.4s, #0x4\\n\"\n        \"scvtf v16.4s, v16.4s, #0x4\\n\"\n        \"fmla v30.4s, v21.4s, v31.4s\\n\"\n        \"fmla v24.4s, v18.4s, v15.4s\\n\"\n        \"fmla v14.4s, v16.4s, v27.4s\\n\"\n        \"bgt 3b\\n\"\n        \"mov x20, %x[res_ptr]\\n\"\n        \"subs x27, x27, #0x4\\n\"\n        \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n        \"str q2, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q10, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q12, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q28, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q11, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q13, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q22, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q23, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q25, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q5, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q7, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q4, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q6, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q30, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q24, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"str q14, [x20, #0x0]\\n\"\n        \"bne 2b\\n\"\n        \"mov x20, #0x4\\n\"\n        \"sub x10, x10, #0x10\\n\"\n        \"cmp x10, #0x10\\n\"\n        \"mov %x[res_ptr], x26\\n\"\n        \"madd %x[a_ptr], x20, x9, %x[a_ptr]\\n\"\n        \"bge 1b\\n\"\n        \"4:\"  // Row loop skip\n        \"cbz x10, 9f\\n\"\n        \"5:\"  // Row tail: Row loop\n        \"add x24, %x[b_ptr], #0x8\\n\"\n        \"mov x23, %x[nc]\\n\"\n        \"add x22, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n        \"6:\"  // Row tail: Column loop\n        \"movi v2.16b, #0x0\\n\"\n        \"movi v10.16b, #0x0\\n\"\n        \"add x25, %x[a_ptr], #0x8\\n\"\n        \"mov x21, %x[nb]\\n\"\n        \"movi v12.16b, #0x0\\n\"\n        \"movi v28.16b, #0x0\\n\"\n        \"7:\"  // Row tail: Block loop\n        \"ldr q6, [x24, #0x0]\\n\"\n        \"ldr q5, [x24, #0x10]\\n\"\n        \"movi v17.16b, #0x4\\n\"\n        \"movi v8.4s, #0x0\\n\"\n        \"ldr q4, [x25, #0x0]\\n\"\n        \"ldr q13, [x25, #0x10]\\n\"\n        \"movi v27.4s, #0x0\\n\"\n        \"movi v0.4s, #0x0\\n\"\n        \"ldr q31, [x24, #0x20]\\n\"\n        \"ldr q14, [x24, #0x30]\\n\"\n        \"movi v29.4s, #0x0\\n\"\n        \"movi v22.16b, #0xf0\\n\"\n        \"ldr q11, [x25, #0x20]\\n\"\n        \"ldr q23, [x25, #0x30]\\n\"\n        \"sshl v21.16b, v6.16b, v17.16b\\n\"\n        \"sshl v16.16b, v5.16b, v17.16b\\n\"\n        \"ldr q20, [x25, #0x40]\\n\"\n        \"ldr q26, [x25, #0x50]\\n\"\n        \"and v6.16b, v6.16b, v22.16b\\n\"\n        \"and v5.16b, v5.16b, v22.16b\\n\"\n        \"ldr q25, [x25, #0x60]\\n\"\n        \"ldr q3, [x25, #0x70]\\n\"\n        \"sshl v19.16b, v31.16b, v17.16b\\n\"\n        \"sshl v18.16b, v14.16b, v17.16b\\n\"\n        \"ldr d17, [x25, #-0x8]\\n\"\n        \".inst 0x4e95a488  // smmla v8.4s, v4.16b, v21.16b\\n\"\n        \".inst 0x4e90a49b  // smmla v27.4s, v4.16b, v16.16b\\n\"\n        \"and v31.16b, v31.16b, v22.16b\\n\"\n        \".inst 0x4e95a5a0  // smmla v0.4s, v13.16b, v21.16b\\n\"\n        \".inst 0x4e90a5bd  // smmla v29.4s, v13.16b, v16.16b\\n\"\n        \"and v14.16b, v14.16b, v22.16b\\n\"\n        \"sub x20, x24, #0x8\\n\"\n        \"ldr d16, [x20, #0x0]\\n\"\n        \"subs x21, x21, #0x1\\n\"\n        \"add x25, x25, #0x88\\n\"\n        \"fcvtl v17.4s, v17.4h\\n\"\n        \"add x24, x24, #0x48\\n\"\n        \".inst 0x4e93a568  // smmla v8.4s, v11.16b, v19.16b\\n\"\n        \".inst 0x4e92a57b  // smmla v27.4s, v11.16b, v18.16b\\n\"\n        \".inst 0x4e93a6e0  // smmla v0.4s, v23.16b, v19.16b\\n\"\n        \".inst 0x4e92a6fd  // smmla v29.4s, v23.16b, v18.16b\\n\"\n        \"fcvtl v16.4s, v16.4h\\n\"\n        \".inst 0x4e86a688  // smmla v8.4s, v20.16b, v6.16b\\n\"\n        \".inst 0x4e85a69b  // smmla v27.4s, v20.16b, v5.16b\\n\"\n        \"fmul v23.4s, v16.4s, v17.s[0]\\n\"\n        \"fmul v21.4s, v16.4s, v17.s[1]\\n\"\n        \"fmul v1.4s, v16.4s, v17.s[2]\\n\"\n        \"fmul v20.4s, v16.4s, v17.s[3]\\n\"\n        \".inst 0x4e86a740  // smmla v0.4s, v26.16b, v6.16b\\n\"\n        \".inst 0x4e85a75d  // smmla v29.4s, v26.16b, v5.16b\\n\"\n        \".inst 0x4e9fa728  // smmla v8.4s, v25.16b, v31.16b\\n\"\n        \".inst 0x4e8ea73b  // smmla v27.4s, v25.16b, v14.16b\\n\"\n        \".inst 0x4e9fa460  // smmla v0.4s, v3.16b, v31.16b\\n\"\n        \".inst 0x4e8ea47d  // smmla v29.4s, v3.16b, v14.16b\\n\"\n        \"uzp1 v19.2d, v8.2d, v27.2d\\n\"\n        \"uzp2 v18.2d, v8.2d, v27.2d\\n\"\n        \"scvtf v19.4s, v19.4s, #0x4\\n\"\n        \"uzp1 v17.2d, v0.2d, v29.2d\\n\"\n        \"uzp2 v16.2d, v0.2d, v29.2d\\n\"\n        \"scvtf v18.4s, v18.4s, #0x4\\n\"\n        \"fmla v2.4s, v19.4s, v23.4s\\n\"\n        \"scvtf v17.4s, v17.4s, #0x4\\n\"\n        \"scvtf v16.4s, v16.4s, #0x4\\n\"\n        \"fmla v10.4s, v18.4s, v21.4s\\n\"\n        \"fmla v12.4s, v17.4s, v1.4s\\n\"\n        \"fmla v28.4s, v16.4s, v20.4s\\n\"\n        \"bgt 7b\\n\"\n        \"mov x20, %x[res_ptr]\\n\"\n        \"cmp x10, #0x1\\n\"\n        \"str q2, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"cmp x10, #0x2\\n\"\n        \"str q10, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"cmp x10, #0x3\\n\"\n        \"str q12, [x20, #0x0]\\n\"\n        \"add x20, x20, %x[res_stride]\\n\"\n        \"ble 8f\\n\"\n        \"str q28, [x20, #0x0]\\n\"\n        \"8:\"  // Row tail: Accumulator store skip\n        \"subs x23, x23, #0x4\\n\"\n        \"add %x[res_ptr], %x[res_ptr], #0x10\\n\"\n        \"bne 6b\\n\"\n        \"subs x10, x10, #0x4\\n\"\n        \"add %x[a_ptr], %x[a_ptr], x9\\n\"\n        \"mov %x[res_ptr], x22\\n\"\n        \"bgt 5b\\n\"\n        \"9:\"  // Row tail: Row loop skip\n        : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n        : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n        : \"cc\", \"memory\", \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\", \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\", \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\", \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\", \"x9\", \"x10\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\"\n    );\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    ggml_gemm_q4_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n    if (ggml_cpu_get_sve_cnt() == QK8_0) {\n        const void * b_ptr = vx;\n        const void * a_ptr = vy;\n        float * res_ptr = s;\n        size_t res_stride = bs * sizeof(float);\n\n        __asm__ __volatile__(\n            \"mov x20, #0x4\\n\"\n            \"mov x13, %x[nr]\\n\"\n            \"mov z28.s, #-0x4\\n\"\n            \"mov x12, #0x88\\n\"\n            \"ptrue p1.b\\n\"\n            \"whilelt p0.s, XZR, x20\\n\"\n            \"cmp x13, #0x10\\n\"\n            \"mul x12, %x[nb], x12\\n\"\n            \"blt 4f\\n\"\n            \"1:\"  // Row loop\n            \"add x11, %x[b_ptr], #0x10\\n\"\n            \"mov x10, %x[nc]\\n\"\n            \"add x9, %x[res_ptr], %x[res_stride], LSL #4\\n\"\n            \"2:\"  // Column loop\n            \"add x28, %x[a_ptr], #0x8\\n\"\n            \"mov z24.b, #0x0\\n\"\n            \"mov z15.b, #0x0\\n\"\n            \"mov x27, %x[nb]\\n\"\n            \"add x26, x28, x12\\n\"\n            \"mov z12.b, #0x0\\n\"\n            \"mov z0.b, #0x0\\n\"\n            \"add x25, x26, x12\\n\"\n            \"mov z13.b, #0x0\\n\"\n            \"mov z1.b, #0x0\\n\"\n            \"add x24, x25, x12\\n\"\n            \"mov z20.b, #0x0\\n\"\n            \"mov z25.b, #0x0\\n\"\n            \"mov z11.b, #0x0\\n\"\n            \"mov z16.b, #0x0\\n\"\n            \"mov z19.b, #0x0\\n\"\n            \"mov z26.b, #0x0\\n\"\n            \"mov z8.b, #0x0\\n\"\n            \"mov z29.b, #0x0\\n\"\n            \"mov z27.b, #0x0\\n\"\n            \"mov z10.b, #0x0\\n\"\n            \"3:\"  // Block loop\n            \"ld1b { z30.b }, p1/Z, [x11]\\n\"\n            \"ld1b { z21.b }, p1/Z, [x11, #1, MUL VL]\\n\"\n            \"mov z18.s, #0x0\\n\"\n            \"mov z7.s, #0x0\\n\"\n            \"ld1rqb { z3.b }, p1/Z, [x28]\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x28, #16]\\n\"\n            \"mov z9.s, #0x0\\n\"\n            \"mov z22.s, #0x0\\n\"\n            \"ld1b { z4.b }, p1/Z, [x11, #2, MUL VL]\\n\"\n            \"ld1b { z17.b }, p1/Z, [x11, #3, MUL VL]\\n\"\n            \"sub x20, x11, #0x10\\n\"\n            \"sub x23, x28, #0x8\\n\"\n            \"lsl z31.b, z30.b, #0x4\\n\"\n            \"lsl z6.b, z21.b, #0x4\\n\"\n            \"ld1h { z23.s }, p1/Z, [x20]\\n\"\n            \"sub x22, x26, #0x8\\n\"\n            \"and z30.b, z30.b, #0xf0\\n\"\n            \"and z21.b, z21.b, #0xf0\\n\"\n            \"sub x21, x25, #0x8\\n\"\n            \"sub x20, x24, #0x8\\n\"\n            \"lsl z14.b, z4.b, #0x4\\n\"\n            \"lsl z2.b, z17.b, #0x4\\n\"\n            \"subs x27, x27, #0x1\\n\"\n            \"add x11, x11, #0x90\\n\"\n            \".inst 0x451f9872  // smmla z18.s, z3.b, z31.b\\n\"\n            \".inst 0x45069867  // smmla z7.s, z3.b, z6.b\\n\"\n            \"ld1rqb { z3.b }, p1/Z, [x28, #32]\\n\"\n            \"and z4.b, z4.b, #0xf0\\n\"\n            \".inst 0x451f98a9  // smmla z9.s, z5.b, z31.b\\n\"\n            \".inst 0x450698b6  // smmla z22.s, z5.b, z6.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x28, #48]\\n\"\n            \"and z17.b, z17.b, #0xf0\\n\"\n            \"fcvt z23.s, p1/m, z23.h\\n\"\n            \".inst 0x450e9872  // smmla z18.s, z3.b, z14.b\\n\"\n            \".inst 0x45029867  // smmla z7.s, z3.b, z2.b\\n\"\n            \"ld1rqb { z3.b }, p1/Z, [x28, #64]\\n\"\n            \".inst 0x450e98a9  // smmla z9.s, z5.b, z14.b\\n\"\n            \".inst 0x450298b6  // smmla z22.s, z5.b, z2.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x28, #80]\\n\"\n            \"fscale z23.s, p1/m, z23.s, z28.s\\n\"\n            \".inst 0x451e9872  // smmla z18.s, z3.b, z30.b\\n\"\n            \".inst 0x45159867  // smmla z7.s, z3.b, z21.b\\n\"\n            \"ld1rqb { z3.b }, p1/Z, [x28, #96]\\n\"\n            \".inst 0x451e98a9  // smmla z9.s, z5.b, z30.b\\n\"\n            \".inst 0x451598b6  // smmla z22.s, z5.b, z21.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x28, #112]\\n\"\n            \"add x28, x28, #0x88\\n\"\n            \".inst 0x45049872  // smmla z18.s, z3.b, z4.b\\n\"\n            \".inst 0x45119867  // smmla z7.s, z3.b, z17.b\\n\"\n            \"ld1h { z3.s }, p0/Z, [x23]\\n\"\n            \".inst 0x450498a9  // smmla z9.s, z5.b, z4.b\\n\"\n            \".inst 0x451198b6  // smmla z22.s, z5.b, z17.b\\n\"\n            \"fcvt z3.s, p1/m, z3.h\\n\"\n            \"uzp1 z5.d, z18.d, z7.d\\n\"\n            \"uzp2 z18.d, z18.d, z7.d\\n\"\n            \"mov z3.q, z3.q[0]\\n\"\n            \"uzp1 z7.d, z9.d, z22.d\\n\"\n            \"uzp2 z22.d, z9.d, z22.d\\n\"\n            \"fmul z9.s, z23.s, z3.s[0]\\n\"\n            \"scvtf z5.s, p1/m, z5.s\\n\"\n            \"scvtf z18.s, p1/m, z18.s\\n\"\n            \"scvtf z7.s, p1/m, z7.s\\n\"\n            \"scvtf z22.s, p1/m, z22.s\\n\"\n            \"fmla z24.s, p1/M, z5.s, z9.s\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x26]\\n\"\n            \"fmul z9.s, z23.s, z3.s[1]\\n\"\n            \"fmla z15.s, p1/M, z18.s, z9.s\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x26, #16]\\n\"\n            \"fmul z9.s, z23.s, z3.s[2]\\n\"\n            \"fmul z3.s, z23.s, z3.s[3]\\n\"\n            \"fmla z12.s, p1/M, z7.s, z9.s\\n\"\n            \"mov z9.s, #0x0\\n\"\n            \"ld1h { z7.s }, p0/Z, [x22]\\n\"\n            \".inst 0x451f98a9  // smmla z9.s, z5.b, z31.b\\n\"\n            \"fmla z0.s, p1/M, z22.s, z3.s\\n\"\n            \"mov z22.s, #0x0\\n\"\n            \"ld1h { z3.s }, p0/Z, [x21]\\n\"\n            \".inst 0x450698b6  // smmla z22.s, z5.b, z6.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x26, #32]\\n\"\n            \"fcvt z7.s, p1/m, z7.h\\n\"\n            \"fcvt z3.s, p1/m, z3.h\\n\"\n            \".inst 0x450e98a9  // smmla z9.s, z5.b, z14.b\\n\"\n            \".inst 0x450298b6  // smmla z22.s, z5.b, z2.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x26, #64]\\n\"\n            \"mov z7.q, z7.q[0]\\n\"\n            \"mov z3.q, z3.q[0]\\n\"\n            \".inst 0x451e98a9  // smmla z9.s, z5.b, z30.b\\n\"\n            \".inst 0x451598b6  // smmla z22.s, z5.b, z21.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x26, #96]\\n\"\n            \".inst 0x450498a9  // smmla z9.s, z5.b, z4.b\\n\"\n            \".inst 0x451198b6  // smmla z22.s, z5.b, z17.b\\n\"\n            \"uzp1 z5.d, z9.d, z22.d\\n\"\n            \"scvtf z5.s, p1/m, z5.s\\n\"\n            \"uzp2 z22.d, z9.d, z22.d\\n\"\n            \"fmul z9.s, z23.s, z7.s[0]\\n\"\n            \"scvtf z22.s, p1/m, z22.s\\n\"\n            \"fmla z13.s, p1/M, z5.s, z9.s\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x25]\\n\"\n            \"fmul z5.s, z23.s, z7.s[1]\\n\"\n            \"fmla z1.s, p1/M, z22.s, z5.s\\n\"\n            \"mov z5.s, #0x0\\n\"\n            \"mov z22.s, #0x0\\n\"\n            \".inst 0x451f9a45  // smmla z5.s, z18.b, z31.b\\n\"\n            \".inst 0x45069a56  // smmla z22.s, z18.b, z6.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x26, #48]\\n\"\n            \".inst 0x450e9a45  // smmla z5.s, z18.b, z14.b\\n\"\n            \".inst 0x45029a56  // smmla z22.s, z18.b, z2.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x26, #80]\\n\"\n            \".inst 0x451e9a45  // smmla z5.s, z18.b, z30.b\\n\"\n            \".inst 0x45159a56  // smmla z22.s, z18.b, z21.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x26, #112]\\n\"\n            \"add x26, x26, #0x88\\n\"\n            \".inst 0x45049a45  // smmla z5.s, z18.b, z4.b\\n\"\n            \".inst 0x45119a56  // smmla z22.s, z18.b, z17.b\\n\"\n            \"uzp1 z18.d, z5.d, z22.d\\n\"\n            \"scvtf z18.s, p1/m, z18.s\\n\"\n            \"uzp2 z22.d, z5.d, z22.d\\n\"\n            \"fmul z5.s, z23.s, z7.s[2]\\n\"\n            \"fmul z7.s, z23.s, z7.s[3]\\n\"\n            \"scvtf z22.s, p1/m, z22.s\\n\"\n            \"fmla z20.s, p1/M, z18.s, z5.s\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x25, #16]\\n\"\n            \"ld1h { z5.s }, p0/Z, [x20]\\n\"\n            \"fcvt z5.s, p1/m, z5.h\\n\"\n            \"fmla z25.s, p1/M, z22.s, z7.s\\n\"\n            \"mov z22.s, #0x0\\n\"\n            \"mov z7.s, #0x0\\n\"\n            \".inst 0x451f9936  // smmla z22.s, z9.b, z31.b\\n\"\n            \".inst 0x45069927  // smmla z7.s, z9.b, z6.b\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x25, #32]\\n\"\n            \"mov z5.q, z5.q[0]\\n\"\n            \".inst 0x450e9936  // smmla z22.s, z9.b, z14.b\\n\"\n            \".inst 0x45029927  // smmla z7.s, z9.b, z2.b\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x25, #64]\\n\"\n            \".inst 0x451e9936  // smmla z22.s, z9.b, z30.b\\n\"\n            \".inst 0x45159927  // smmla z7.s, z9.b, z21.b\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x25, #96]\\n\"\n            \".inst 0x45049936  // smmla z22.s, z9.b, z4.b\\n\"\n            \".inst 0x45119927  // smmla z7.s, z9.b, z17.b\\n\"\n            \"uzp1 z9.d, z22.d, z7.d\\n\"\n            \"scvtf z9.s, p1/m, z9.s\\n\"\n            \"uzp2 z22.d, z22.d, z7.d\\n\"\n            \"fmul z7.s, z23.s, z3.s[0]\\n\"\n            \"scvtf z22.s, p1/m, z22.s\\n\"\n            \"fmla z11.s, p1/M, z9.s, z7.s\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x24]\\n\"\n            \"fmul z7.s, z23.s, z3.s[1]\\n\"\n            \"fmla z16.s, p1/M, z22.s, z7.s\\n\"\n            \"mov z22.s, #0x0\\n\"\n            \"mov z7.s, #0x0\\n\"\n            \".inst 0x451f9a56  // smmla z22.s, z18.b, z31.b\\n\"\n            \".inst 0x45069a47  // smmla z7.s, z18.b, z6.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x25, #48]\\n\"\n            \".inst 0x450e9a56  // smmla z22.s, z18.b, z14.b\\n\"\n            \".inst 0x45029a47  // smmla z7.s, z18.b, z2.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x25, #80]\\n\"\n            \".inst 0x451e9a56  // smmla z22.s, z18.b, z30.b\\n\"\n            \".inst 0x45159a47  // smmla z7.s, z18.b, z21.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x25, #112]\\n\"\n            \"add x25, x25, #0x88\\n\"\n            \".inst 0x45049a56  // smmla z22.s, z18.b, z4.b\\n\"\n            \".inst 0x45119a47  // smmla z7.s, z18.b, z17.b\\n\"\n            \"uzp1 z18.d, z22.d, z7.d\\n\"\n            \"scvtf z18.s, p1/m, z18.s\\n\"\n            \"uzp2 z7.d, z22.d, z7.d\\n\"\n            \"fmul z22.s, z23.s, z3.s[2]\\n\"\n            \"fmul z3.s, z23.s, z3.s[3]\\n\"\n            \"scvtf z7.s, p1/m, z7.s\\n\"\n            \"fmla z19.s, p1/M, z18.s, z22.s\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x24, #16]\\n\"\n            \"fmul z22.s, z23.s, z5.s[0]\\n\"\n            \"fmla z26.s, p1/M, z7.s, z3.s\\n\"\n            \"mov z3.s, #0x0\\n\"\n            \"mov z7.s, #0x0\\n\"\n            \".inst 0x451f9923  // smmla z3.s, z9.b, z31.b\\n\"\n            \".inst 0x45069927  // smmla z7.s, z9.b, z6.b\\n\"\n            \"ld1rqb { z9.b }, p1/Z, [x24, #32]\\n\"\n            \".inst 0x450e9923  // smmla z3.s, z9.b, z14.b\\n\"\n            \".inst 0x45029927  // smmla z7.s, z9.b, z2.b\\n\"\n            \"mov z9.s, #0x0\\n\"\n            \".inst 0x451f9a49  // smmla z9.s, z18.b, z31.b\\n\"\n            \"mov z31.s, #0x0\\n\"\n            \".inst 0x45069a5f  // smmla z31.s, z18.b, z6.b\\n\"\n            \"ld1rqb { z6.b }, p1/Z, [x24, #48]\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x24, #64]\\n\"\n            \".inst 0x450e98c9  // smmla z9.s, z6.b, z14.b\\n\"\n            \"fmul z14.s, z23.s, z5.s[1]\\n\"\n            \".inst 0x450298df  // smmla z31.s, z6.b, z2.b\\n\"\n            \"ld1rqb { z6.b }, p1/Z, [x24, #80]\\n\"\n            \"fmul z2.s, z23.s, z5.s[2]\\n\"\n            \"fmul z23.s, z23.s, z5.s[3]\\n\"\n            \".inst 0x451e9a43  // smmla z3.s, z18.b, z30.b\\n\"\n            \".inst 0x45159a47  // smmla z7.s, z18.b, z21.b\\n\"\n            \"ld1rqb { z5.b }, p1/Z, [x24, #96]\\n\"\n            \".inst 0x451e98c9  // smmla z9.s, z6.b, z30.b\\n\"\n            \".inst 0x451598df  // smmla z31.s, z6.b, z21.b\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x24, #112]\\n\"\n            \"add x24, x24, #0x88\\n\"\n            \".inst 0x450498a3  // smmla z3.s, z5.b, z4.b\\n\"\n            \".inst 0x451198a7  // smmla z7.s, z5.b, z17.b\\n\"\n            \".inst 0x45049a49  // smmla z9.s, z18.b, z4.b\\n\"\n            \".inst 0x45119a5f  // smmla z31.s, z18.b, z17.b\\n\"\n            \"uzp1 z18.d, z3.d, z7.d\\n\"\n            \"uzp2 z5.d, z3.d, z7.d\\n\"\n            \"scvtf z18.s, p1/m, z18.s\\n\"\n            \"uzp1 z6.d, z9.d, z31.d\\n\"\n            \"uzp2 z9.d, z9.d, z31.d\\n\"\n            \"scvtf z5.s, p1/m, z5.s\\n\"\n            \"fmla z8.s, p1/M, z18.s, z22.s\\n\"\n            \"scvtf z6.s, p1/m, z6.s\\n\"\n            \"scvtf z9.s, p1/m, z9.s\\n\"\n            \"fmla z29.s, p1/M, z5.s, z14.s\\n\"\n            \"fmla z27.s, p1/M, z6.s, z2.s\\n\"\n            \"fmla z10.s, p1/M, z9.s, z23.s\\n\"\n            \"bgt 3b\\n\"\n            \"mov x20, %x[res_ptr]\\n\"\n            \"subs x10, x10, #0x8\\n\"\n            \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n            \"st1w { z24.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z15.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z12.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z0.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z13.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z1.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z20.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z25.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z11.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z16.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z19.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z26.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z8.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z29.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z27.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"st1w { z10.s }, p1, [x20]\\n\"\n            \"bne 2b\\n\"\n            \"mov x20, #0x4\\n\"\n            \"sub x13, x13, #0x10\\n\"\n            \"cmp x13, #0x10\\n\"\n            \"mov %x[res_ptr], x9\\n\"\n            \"madd %x[a_ptr], x20, x12, %x[a_ptr]\\n\"\n            \"bge 1b\\n\"\n            \"4:\"  // Row loop skip\n            \"cbz x13, 9f\\n\"\n            \"5:\"  // Row tail: Row loop\n            \"add x25, %x[b_ptr], #0x10\\n\"\n            \"mov x24, %x[nc]\\n\"\n            \"add x23, %x[res_ptr], %x[res_stride], LSL #2\\n\"\n            \"6:\"  // Row tail: Column loop\n            \"mov z24.b, #0x0\\n\"\n            \"mov z15.b, #0x0\\n\"\n            \"add x28, %x[a_ptr], #0x8\\n\"\n            \"mov x22, %x[nb]\\n\"\n            \"mov z12.b, #0x0\\n\"\n            \"mov z0.b, #0x0\\n\"\n            \"7:\"  // Row tail: Block loop\n            \"ld1b { z3.b }, p1/Z, [x25]\\n\"\n            \"ld1b { z6.b }, p1/Z, [x25, #1, MUL VL]\\n\"\n            \"mov z2.s, #0x0\\n\"\n            \"mov z25.s, #0x0\\n\"\n            \"ld1rqb { z26.b }, p1/Z, [x28]\\n\"\n            \"ld1rqb { z21.b }, p1/Z, [x28, #16]\\n\"\n            \"mov z27.s, #0x0\\n\"\n            \"mov z19.s, #0x0\\n\"\n            \"ld1b { z29.b }, p1/Z, [x25, #2, MUL VL]\\n\"\n            \"ld1b { z16.b }, p1/Z, [x25, #3, MUL VL]\\n\"\n            \"sub x21, x25, #0x10\\n\"\n            \"sub x20, x28, #0x8\\n\"\n            \"lsl z20.b, z3.b, #0x4\\n\"\n            \"lsl z4.b, z6.b, #0x4\\n\"\n            \"ld1rqb { z10.b }, p1/Z, [x28, #32]\\n\"\n            \"ld1rqb { z23.b }, p1/Z, [x28, #48]\\n\"\n            \"and z3.b, z3.b, #0xf0\\n\"\n            \"and z6.b, z6.b, #0xf0\\n\"\n            \"ld1rqb { z11.b }, p1/Z, [x28, #64]\\n\"\n            \"ld1rqb { z7.b }, p1/Z, [x28, #80]\\n\"\n            \"lsl z8.b, z29.b, #0x4\\n\"\n            \"lsl z14.b, z16.b, #0x4\\n\"\n            \"ld1rqb { z18.b }, p1/Z, [x28, #96]\\n\"\n            \"ld1rqb { z30.b }, p1/Z, [x28, #112]\\n\"\n            \".inst 0x45149b42  // smmla z2.s, z26.b, z20.b\\n\"\n            \".inst 0x45049b59  // smmla z25.s, z26.b, z4.b\\n\"\n            \"and z29.b, z29.b, #0xf0\\n\"\n            \"ld1h { z17.s }, p1/Z, [x21]\\n\"\n            \".inst 0x45149abb  // smmla z27.s, z21.b, z20.b\\n\"\n            \".inst 0x45049ab3  // smmla z19.s, z21.b, z4.b\\n\"\n            \"and z16.b, z16.b, #0xf0\\n\"\n            \"ld1h { z4.s }, p0/Z, [x20]\\n\"\n            \"subs x22, x22, #0x1\\n\"\n            \"add x28, x28, #0x88\\n\"\n            \"fcvt z17.s, p1/m, z17.h\\n\"\n            \"add x25, x25, #0x90\\n\"\n            \".inst 0x45089942  // smmla z2.s, z10.b, z8.b\\n\"\n            \".inst 0x450e9959  // smmla z25.s, z10.b, z14.b\\n\"\n            \"fcvt z4.s, p1/m, z4.h\\n\"\n            \".inst 0x45089afb  // smmla z27.s, z23.b, z8.b\\n\"\n            \".inst 0x450e9af3  // smmla z19.s, z23.b, z14.b\\n\"\n            \"fscale z17.s, p1/m, z17.s, z28.s\\n\"\n            \"mov z4.q, z4.q[0]\\n\"\n            \".inst 0x45039962  // smmla z2.s, z11.b, z3.b\\n\"\n            \".inst 0x45069979  // smmla z25.s, z11.b, z6.b\\n\"\n            \"fmul z23.s, z17.s, z4.s[0]\\n\"\n            \"fmul z9.s, z17.s, z4.s[1]\\n\"\n            \"fmul z21.s, z17.s, z4.s[2]\\n\"\n            \"fmul z4.s, z17.s, z4.s[3]\\n\"\n            \".inst 0x450398fb  // smmla z27.s, z7.b, z3.b\\n\"\n            \".inst 0x450698f3  // smmla z19.s, z7.b, z6.b\\n\"\n            \".inst 0x451d9a42  // smmla z2.s, z18.b, z29.b\\n\"\n            \".inst 0x45109a59  // smmla z25.s, z18.b, z16.b\\n\"\n            \".inst 0x451d9bdb  // smmla z27.s, z30.b, z29.b\\n\"\n            \".inst 0x45109bd3  // smmla z19.s, z30.b, z16.b\\n\"\n            \"uzp1 z31.d, z2.d, z25.d\\n\"\n            \"uzp2 z13.d, z2.d, z25.d\\n\"\n            \"scvtf z31.s, p1/m, z31.s\\n\"\n            \"uzp1 z17.d, z27.d, z19.d\\n\"\n            \"uzp2 z18.d, z27.d, z19.d\\n\"\n            \"scvtf z13.s, p1/m, z13.s\\n\"\n            \"fmla z24.s, p1/M, z31.s, z23.s\\n\"\n            \"scvtf z17.s, p1/m, z17.s\\n\"\n            \"scvtf z18.s, p1/m, z18.s\\n\"\n            \"fmla z15.s, p1/M, z13.s, z9.s\\n\"\n            \"fmla z12.s, p1/M, z17.s, z21.s\\n\"\n            \"fmla z0.s, p1/M, z18.s, z4.s\\n\"\n            \"bgt 7b\\n\"\n            \"mov x20, %x[res_ptr]\\n\"\n            \"cmp x13, #0x1\\n\"\n            \"st1w { z24.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"ble 8f\\n\"\n            \"cmp x13, #0x2\\n\"\n            \"st1w { z15.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"ble 8f\\n\"\n            \"cmp x13, #0x3\\n\"\n            \"st1w { z12.s }, p1, [x20]\\n\"\n            \"add x20, x20, %x[res_stride]\\n\"\n            \"ble 8f\\n\"\n            \"st1w { z0.s }, p1, [x20]\\n\"\n            \"8:\"  // Row tail: Accumulator store skip\n            \"subs x24, x24, #0x8\\n\"\n            \"add %x[res_ptr], %x[res_ptr], #0x20\\n\"\n            \"bne 6b\\n\"\n            \"subs x13, x13, #0x4\\n\"\n            \"add %x[a_ptr], %x[a_ptr], x12\\n\"\n            \"mov %x[res_ptr], x23\\n\"\n            \"bgt 5b\\n\"\n            \"9:\"  // Row tail: Row loop skip\n            : [a_ptr] \"+&r\" (a_ptr), [res_ptr] \"+&r\" (res_ptr)\n            : [b_ptr] \"r\" (b_ptr), [nr] \"r\" (nr), [nb] \"r\" (nb), [res_stride] \"r\" (res_stride), [nc] \"r\" (nc)\n            : \"cc\", \"memory\", \"p0\", \"p1\", \"x9\", \"x10\", \"x11\", \"x12\", \"x13\", \"x20\", \"x21\", \"x22\", \"x23\", \"x24\", \"x25\", \"x26\", \"x27\", \"x28\", \"z0\", \"z1\", \"z2\", \"z3\", \"z4\", \"z5\", \"z6\", \"z7\", \"z8\", \"z9\", \"z10\", \"z11\", \"z12\", \"z13\", \"z14\", \"z15\", \"z16\", \"z17\", \"z18\", \"z19\", \"z20\", \"z21\", \"z22\", \"z23\", \"z24\", \"z25\", \"z26\", \"z27\", \"z28\", \"z29\", \"z30\", \"z31\"\n        );\n        return;\n    }\n#endif // #if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__)\n    ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const int8x16_t kvalues = vld1q_s8(kvalues_iq4nl);\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n            float32x4_t sumf[4];\n            for (int m = 0; m < 4; m++) {\n                sumf[m] = vdupq_n_f32(0);\n            }\n\n            for (int l = 0; l < nb; l++) {\n                float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *)a_ptr[l].d));\n                float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *)b_ptr[l].d));\n\n                int32x4_t sumi_0 = vdupq_n_s32(0);\n                int32x4_t sumi_1 = vdupq_n_s32(0);\n                int32x4_t sumi_2 = vdupq_n_s32(0);\n                int32x4_t sumi_3 = vdupq_n_s32(0);\n\n                for (int k = 0; k < 4; k++) {\n                    int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 16 * k + 0);\n                    int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16 * k + 64);\n\n                    uint8x16_t b = vld1q_u8(b_ptr[l].qs + 16 * k);\n                    int8x16_t b_hi = vqtbl1q_s8(kvalues, b >> 4);\n                    int8x16_t b_lo = vqtbl1q_s8(kvalues, b & 0xF);\n\n                    sumi_0 = vdotq_laneq_s32(sumi_0, b_lo, a_0, 0);\n                    sumi_1 = vdotq_laneq_s32(sumi_1, b_lo, a_0, 1);\n                    sumi_2 = vdotq_laneq_s32(sumi_2, b_lo, a_0, 2);\n                    sumi_3 = vdotq_laneq_s32(sumi_3, b_lo, a_0, 3);\n                    sumi_0 = vdotq_laneq_s32(sumi_0, b_hi, a_1, 0);\n                    sumi_1 = vdotq_laneq_s32(sumi_1, b_hi, a_1, 1);\n                    sumi_2 = vdotq_laneq_s32(sumi_2, b_hi, a_1, 2);\n                    sumi_3 = vdotq_laneq_s32(sumi_3, b_hi, a_1, 3);\n                }\n\n                sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n                sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n                sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n                sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n            }\n\n            for (int m = 0; m < 4; m++) {\n                vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n            }\n        }\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n    ggml_gemm_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n            float32x4_t sumf[4];\n            for (int m = 0; m < 4; m++) {\n                sumf[m] = vdupq_n_f32(0);\n            }\n\n            for (int l = 0; l < nb; l++) {\n                float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *)a_ptr[l].d));\n                float32x4_t b_d = {\n                    GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),\n                    GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),\n                    GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),\n                    GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),\n                };\n\n                int32x4_t sumi_0 = vdupq_n_s32(0);\n                int32x4_t sumi_1 = vdupq_n_s32(0);\n                int32x4_t sumi_2 = vdupq_n_s32(0);\n                int32x4_t sumi_3 = vdupq_n_s32(0);\n\n                for (int k = 0; k < 4; k++) {\n                    int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 16 * k + 0);\n                    int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16 * k + 64);\n\n                    uint8x16_t b = vld1q_u8(b_ptr[l].qs + 16 * k);\n                    int8x16_t b_hi = vqtbl1q_s8(kvalues, b >> 4);\n                    int8x16_t b_lo = vqtbl1q_s8(kvalues, b & 0xF);\n\n                    sumi_0 = vdotq_laneq_s32(sumi_0, b_lo, a_0, 0);\n                    sumi_1 = vdotq_laneq_s32(sumi_1, b_lo, a_0, 1);\n                    sumi_2 = vdotq_laneq_s32(sumi_2, b_lo, a_0, 2);\n                    sumi_3 = vdotq_laneq_s32(sumi_3, b_lo, a_0, 3);\n                    sumi_0 = vdotq_laneq_s32(sumi_0, b_hi, a_1, 0);\n                    sumi_1 = vdotq_laneq_s32(sumi_1, b_hi, a_1, 1);\n                    sumi_2 = vdotq_laneq_s32(sumi_2, b_hi, a_1, 2);\n                    sumi_3 = vdotq_laneq_s32(sumi_3, b_hi, a_1, 3);\n                }\n\n                sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n                sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n                sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n                sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n            }\n\n            for (int m = 0; m < 4; m++) {\n                vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n            }\n        }\n    }\n    return;\n#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)\n    ggml_gemm_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    q8_k_blocklen = 4;\n    constexpr int    acc_size  = 2 * 4;  // 2 row pairs × 4 col pairs\n    const uint8x16_t m4b       = vdupq_n_u8(0x0f);\n\n    // 8 accumulators: 2 row pairs × 4 col pairs\n    float32x4_t acc_f32[acc_size];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < acc_size; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                // d4 0 1 2 3, 4 5 6 7\n                float32x4_t q4_d_0123    = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d));\n                float32x4_t q4_d_4567    = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].d + 4));\n                // d8 0 1 2 3\n                float32x4_t q8_d_0123    = vld1q_f32(q8_ptr[b].d);\n                // mins\n                float32x4_t q4_dmin_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin));\n                float32x4_t q4_dmin_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q4_ptr[b].dmin + 4));\n\n                // Precomputation of scales and mins\n                float32x4_t sbd_scale_0123[q8_k_blocklen];\n                float32x4_t sbd_scale_4567[q8_k_blocklen];\n                float32x4_t sbd_min_0123[q8_k_blocklen];\n                float32x4_t sbd_min_4567[q8_k_blocklen];\n\n                sbd_scale_0123[0] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 0);\n                sbd_scale_4567[0] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 0);\n                sbd_min_0123[0]   = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 0);\n                sbd_min_4567[0]   = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 0);\n\n                sbd_scale_0123[1] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 1);\n                sbd_scale_4567[1] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 1);\n                sbd_min_0123[1]   = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 1);\n                sbd_min_4567[1]   = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 1);\n\n                sbd_scale_0123[2] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 2);\n                sbd_scale_4567[2] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 2);\n                sbd_min_0123[2]   = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 2);\n                sbd_min_4567[2]   = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 2);\n\n                sbd_scale_0123[3] = vmulq_laneq_f32(q4_d_0123, q8_d_0123, 3);\n                sbd_scale_4567[3] = vmulq_laneq_f32(q4_d_4567, q8_d_0123, 3);\n                sbd_min_0123[3]   = vmulq_laneq_f32(q4_dmin_0123, q8_d_0123, 3);\n                sbd_min_4567[3]   = vmulq_laneq_f32(q4_dmin_4567, q8_d_0123, 3);\n\n                // Precomputation of bsums, each vpaddq calcs all the bsums for each row\n                const int16x8_t bsums[q8_k_blocklen] = {\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n                };\n                int16_t bsums_arr[QK_K / 64][8];\n                for (int q8_row = 0; q8_row < 4; q8_row++) {\n                    vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n                }\n\n                // interleaved bias_acc: [0]->r0 0123, [1]->r1 0123, .., [4]->r0 4567, [5]->r1 4567 ..\n                int32x4_t bias_acc[acc_size];\n                for (int i = 0; i < acc_size; i++) {\n                    bias_acc[i] = vdupq_n_s32(0);\n                }\n\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    // Int accumulators for qs vecdot (4 row x 2 col quartets)\n                    int32x4_t acc_lo[acc_size];\n                    int32x4_t acc_hi[acc_size];\n                    for (int i = 0; i < acc_size; i++) {\n                        acc_lo[i] = vdupq_n_s32(0);\n                        acc_hi[i] = vdupq_n_s32(0);\n                    }\n                    // Need scales for the low and high nibbles\n                    // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                    int16x8_t q4sb_scales[2];\n                    int16x8_t q4sb_mins[2];\n                    for (int i = 0; i < 2; i++) {\n                        int8_t    aux_q4sb[8];\n                        const int offset = sb * 24 + i * 12;\n                        decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], aux_q4sb);\n                        q4sb_scales[i] = vmovl_s8(vld1_s8(aux_q4sb));\n                    }\n\n                    constexpr int reads_per_sb = 8;  // 8 * 16 bytes each => 32 qs * 4 rows\n                    for (int k = 0; k < reads_per_sb; k++) {\n                        const int8x16_t q8_blk0 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k);\n                        const int8x16_t q8_blk1 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k + 128);\n\n                        // 0..3 & 32..35\n                        const uint8x16_t q4_0123 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 32 * k);\n                        const uint8x16_t q4_4567 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 32 * k + 16);\n\n                        const int8x16_t q4_0123_lo = vreinterpretq_s8_u8(vandq_u8(q4_0123, m4b));\n                        const int8x16_t q4_0123_hi = vreinterpretq_s8_u8(vshrq_n_u8(q4_0123, 4));\n\n                        acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q4_0123_lo, q8_blk0, 0);  //  0..3  r0 c0123\n                        acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q4_0123_lo, q8_blk0, 1);  //  0..3  r1 c0123\n                        acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q4_0123_lo, q8_blk0, 2);  //  0..3  r2 c0123\n                        acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q4_0123_lo, q8_blk0, 3);  //  0..3  r3 c0123\n\n                        acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q4_0123_hi, q8_blk1, 0);  // 32..35 r0 c0123\n                        acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q4_0123_hi, q8_blk1, 1);  // 32..35 r1 c0123\n                        acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q4_0123_hi, q8_blk1, 2);  // 32..35 r2 c0123\n                        acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q4_0123_hi, q8_blk1, 3);  // 32..35 r3 c0123\n\n                        const int8x16_t q4_4567_lo = vreinterpretq_s8_u8(vandq_u8(q4_4567, m4b));\n                        const int8x16_t q4_4567_hi = vreinterpretq_s8_u8(vshrq_n_u8(q4_4567, 4));\n\n                        acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q4_4567_lo, q8_blk0, 0);  //  0..3  r0 c4567\n                        acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q4_4567_lo, q8_blk0, 1);  //  0..3  r1 c4567\n                        acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q4_4567_lo, q8_blk0, 2);  //  0..3  r2 c4567\n                        acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q4_4567_lo, q8_blk0, 3);  //  0..3  r3 c4567\n\n                        acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q4_4567_hi, q8_blk1, 0);  // 32..35 r0 c4567\n                        acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q4_4567_hi, q8_blk1, 1);  // 32..35 r1 c4567\n                        acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q4_4567_hi, q8_blk1, 2);  // 32..35 r2 c4567\n                        acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q4_4567_hi, q8_blk1, 3);  // 32..35 r3 c4567\n                    }\n\n                    // Scale and bias application\n                    // acc is stored interleaved to match output layout\n                    const int16x4_t sc_0123_lo = vget_low_s16(q4sb_scales[0]);\n                    const int16x4_t sc_4567_lo = vget_high_s16(q4sb_scales[0]);\n                    const int16x4_t sc_0123_hi = vget_low_s16(q4sb_scales[1]);\n                    const int16x4_t sc_4567_hi = vget_high_s16(q4sb_scales[1]);\n                    for (int row = 0; row < q8_k_blocklen; row++) {\n                        // Bias correction\n                        // row c0123 blk0 and blk1\n                        const float32x4_t sumf_0123 =\n                            vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[row]),\n                                                    vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[row])));\n                        acc_f32[2 * row] = vfmaq_f32(acc_f32[2 * row], sbd_scale_0123[row], sumf_0123);\n\n                        // row c4567 blk0 and blk1\n                        const float32x4_t sumf_4567 =\n                            vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[row + 4]),\n                                                    vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[row + 4])));\n                        acc_f32[2 * row + 1] = vfmaq_f32(acc_f32[2 * row + 1], sbd_scale_4567[row], sumf_4567);\n\n                        // Bias\n                        const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][row * 2]);\n                        const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][row * 2 + 1]);\n\n                        // row c0123 blk0 and blk1\n                        bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n                        bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n\n                        // row c4567 blk0 and blk1\n                        bias_acc[2 * row + 1] =\n                            vmlal_s16(bias_acc[2 * row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n                        bias_acc[2 * row + 1] =\n                            vmlal_s16(bias_acc[2 * row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n                    }\n                }  // for sb\n\n                for (int row = 0; row < q8_k_blocklen; row++) {\n                    acc_f32[2 * row] = vmlsq_f32(acc_f32[2 * row], vcvtq_f32_s32(bias_acc[2 * row]), sbd_min_0123[row]);\n                    acc_f32[2 * row + 1] =\n                        vmlsq_f32(acc_f32[2 * row + 1], vcvtq_f32_s32(bias_acc[2 * row + 1]), sbd_min_4567[row]);\n                }\n            }  // for b\n\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemm_q4_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x4_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    q8_k_blocklen = 4;\n    constexpr int    acc_size      = 2 * 4;  // 2 row pairs, 4 col pairs\n    constexpr int    col_groups    = ncols_interleaved / 4;\n    const uint8x16_t m4b           = vdupq_n_u8(0x0f);\n    const uint8x16_t mone          = vdupq_n_u8(1);\n    const uint8x16_t mtwo          = vdupq_n_u8(2);\n\n    // 8 accumulators: 2 row pairs, 4 col pairs\n    float32x4_t acc_f32[acc_size];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < acc_size; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                // d5 0 1 2 3, 4 5 6 7\n                float32x4_t q5_d_0123    = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d));\n                float32x4_t q5_d_4567    = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].d + 4));\n                // d8 0 1 2 3\n                float32x4_t q8_d_0123    = vld1q_f32(q8_ptr[b].d);\n                // mins\n                float32x4_t q5_dmin_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin));\n                float32x4_t q5_dmin_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q5_ptr[b].dmin + 4));\n\n                // Precomputation of scales and mins\n                float32x4_t sbd_scale_0123[q8_k_blocklen];\n                float32x4_t sbd_scale_4567[q8_k_blocklen];\n                float32x4_t sbd_min_0123[q8_k_blocklen];\n                float32x4_t sbd_min_4567[q8_k_blocklen];\n\n                sbd_scale_0123[0] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 0);\n                sbd_scale_4567[0] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 0);\n                sbd_min_0123[0]   = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 0);\n                sbd_min_4567[0]   = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 0);\n\n                sbd_scale_0123[1] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 1);\n                sbd_scale_4567[1] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 1);\n                sbd_min_0123[1]   = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 1);\n                sbd_min_4567[1]   = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 1);\n\n                sbd_scale_0123[2] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 2);\n                sbd_scale_4567[2] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 2);\n                sbd_min_0123[2]   = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 2);\n                sbd_min_4567[2]   = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 2);\n\n                sbd_scale_0123[3] = vmulq_laneq_f32(q5_d_0123, q8_d_0123, 3);\n                sbd_scale_4567[3] = vmulq_laneq_f32(q5_d_4567, q8_d_0123, 3);\n                sbd_min_0123[3]   = vmulq_laneq_f32(q5_dmin_0123, q8_d_0123, 3);\n                sbd_min_4567[3]   = vmulq_laneq_f32(q5_dmin_4567, q8_d_0123, 3);\n\n                // Precomputation of bsums, each vpaddq calcs all the bsums for each row\n                const int16x8_t bsums[q8_k_blocklen] = {\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n                };\n                int16_t bsums_arr[QK_K / 64][8];\n                for (int q8_row = 0; q8_row < 4; q8_row++) {\n                    vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n                }\n\n                // interleaved bias_acc: [0]->r0 0123, [1]->r1 0123, .., [4]->r0 4567, [5]->r1 4567 ..\n                int32x4_t bias_acc[acc_size];\n                for (int i = 0; i < acc_size; i++) {\n                    bias_acc[i] = vdupq_n_s32(0);\n                }\n\n                uint8x16_t qh[col_groups][8];\n                for (int c = 0; c < col_groups; c++) {\n                    for (int i = 0; i < 8; i++) {\n                        qh[c][i] = vld1q_u8(q5_ptr[b].qh + i * 32 + 16 * c);\n                    }\n                }\n\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    // Int accumulators for qs vecdot (4 row * 2 col quartets)\n                    int32x4_t acc_lo[acc_size];\n                    int32x4_t acc_hi[acc_size];\n                    for (int i = 0; i < acc_size; i++) {\n                        acc_lo[i] = vdupq_n_s32(0);\n                        acc_hi[i] = vdupq_n_s32(0);\n                    }\n                    // Need scales for the low and high nibbles\n                    // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                    int16x8_t q5sb_scales[2];\n                    int16x8_t q5sb_mins[2];\n                    for (int i = 0; i < 2; i++) {\n                        int8_t    aux_q5sb[8];\n                        const int offset = sb * 24 + i * 12;\n                        decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], aux_q5sb);\n                        q5sb_scales[i] = vmovl_s8(vld1_s8(aux_q5sb));\n                    }\n\n                    constexpr int reads_per_sb = 8;  // 8 * 16 bytes each => 32 qs * 4 rows\n                    for (int k = 0; k < reads_per_sb; k++) {\n                        const int8x16_t q8_blk0 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k);\n                        const int8x16_t q8_blk1 = vld1q_s8(q8_ptr[b].qs + sb * 256 + 16 * k + 128);\n\n                        // 0..3 & 32..35\n                        const uint8x16_t q5_0123 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 32 * k);\n                        const uint8x16_t q5_4567 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 32 * k + 16);\n\n                        // NOTE: This is the only difference with q4_K\n                        const uint8x16_t hbit_lo_0123 = vandq_u8(qh[0][k], mone);\n                        const uint8x16_t hbit_hi_0123 = vshlq_n_u8(vandq_u8(qh[0][k], mtwo), 3);\n                        qh[0][k]                      = vshrq_n_u8(qh[0][k], 2);\n                        const uint8x16_t hbit_lo_4567 = vandq_u8(qh[1][k], mone);\n                        const uint8x16_t hbit_hi_4567 = vshlq_n_u8(vandq_u8(qh[1][k], mtwo), 3);\n                        qh[1][k]                      = vshrq_n_u8(qh[1][k], 2);\n                        // From here, same as q4_K\n\n                        const int8x16_t q5_0123_lo =\n                            vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_0123, m4b), hbit_lo_0123, 4));\n                        const int8x16_t q5_0123_hi =\n                            vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_0123, 4), hbit_hi_0123));\n\n                        acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q5_0123_lo, q8_blk0, 0);  //  0..3  r0 c0123\n                        acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q5_0123_lo, q8_blk0, 1);  //  0..3  r1 c0123\n                        acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q5_0123_lo, q8_blk0, 2);  //  0..3  r2 c0123\n                        acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q5_0123_lo, q8_blk0, 3);  //  0..3  r3 c0123\n\n                        acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q5_0123_hi, q8_blk1, 0);  // 32..35 r0 c0123\n                        acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q5_0123_hi, q8_blk1, 1);  // 32..35 r1 c0123\n                        acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q5_0123_hi, q8_blk1, 2);  // 32..35 r2 c0123\n                        acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q5_0123_hi, q8_blk1, 3);  // 32..35 r3 c0123\n\n                        const int8x16_t q5_4567_lo =\n                            vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q5_4567, m4b), hbit_lo_4567, 4));\n                        const int8x16_t q5_4567_hi =\n                            vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5_4567, 4), hbit_hi_4567));\n\n                        acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q5_4567_lo, q8_blk0, 0);  //  0..3  r0 c4567\n                        acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q5_4567_lo, q8_blk0, 1);  //  0..3  r1 c4567\n                        acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q5_4567_lo, q8_blk0, 2);  //  0..3  r2 c4567\n                        acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q5_4567_lo, q8_blk0, 3);  //  0..3  r3 c4567\n\n                        acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q5_4567_hi, q8_blk1, 0);  // 32..35 r0 c4567\n                        acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q5_4567_hi, q8_blk1, 1);  // 32..35 r1 c4567\n                        acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q5_4567_hi, q8_blk1, 2);  // 32..35 r2 c4567\n                        acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q5_4567_hi, q8_blk1, 3);  // 32..35 r3 c4567\n                    }\n\n                    // Scale and bias application\n                    // acc is stored interleaved to match output layout\n                    const int16x4_t sc_0123_lo = vget_low_s16(q5sb_scales[0]);\n                    const int16x4_t sc_4567_lo = vget_high_s16(q5sb_scales[0]);\n                    const int16x4_t sc_0123_hi = vget_low_s16(q5sb_scales[1]);\n                    const int16x4_t sc_4567_hi = vget_high_s16(q5sb_scales[1]);\n                    for (int row = 0; row < q8_k_blocklen; row++) {\n                        // Bias correction\n                        // row c0123 blk0 and blk1\n                        const float32x4_t sumf_0123 =\n                            vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_0123_lo), acc_lo[row]),\n                                                    vmulq_s32(vmovl_s16(sc_0123_hi), acc_hi[row])));\n                        acc_f32[2 * row] = vfmaq_f32(acc_f32[2 * row], sbd_scale_0123[row], sumf_0123);\n\n                        // row c4567 blk0 and blk1\n                        const float32x4_t sumf_4567 =\n                            vcvtq_f32_s32(vaddq_s32(vmulq_s32(vmovl_s16(sc_4567_lo), acc_lo[row + 4]),\n                                                    vmulq_s32(vmovl_s16(sc_4567_hi), acc_hi[row + 4])));\n                        acc_f32[2 * row + 1] = vfmaq_f32(acc_f32[2 * row + 1], sbd_scale_4567[row], sumf_4567);\n\n                        // Bias\n                        const int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][row * 2]);\n                        const int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][row * 2 + 1]);\n\n                        // row c0123 blk0 and blk1\n                        bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n                        bias_acc[2 * row] = vmlal_s16(bias_acc[2 * row], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n\n                        // row c4567 blk0 and blk1\n                        bias_acc[2 * row + 1] =\n                            vmlal_s16(bias_acc[2 * row + 1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n                        bias_acc[2 * row + 1] =\n                            vmlal_s16(bias_acc[2 * row + 1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n                    }\n                }  // for sb\n\n                for (int row = 0; row < q8_k_blocklen; row++) {\n                    acc_f32[2 * row] = vmlsq_f32(acc_f32[2 * row], vcvtq_f32_s32(bias_acc[2 * row]), sbd_min_0123[row]);\n                    acc_f32[2 * row + 1] =\n                        vmlsq_f32(acc_f32[2 * row + 1], vcvtq_f32_s32(bias_acc[2 * row + 1]), sbd_min_4567[row]);\n                }\n            }  // for b\n\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemm_q5_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)\n    if (svcntb() * 8 == 256) {\n        constexpr int    q8_k_blocklen = 4;\n        const svuint8_t m4b_1          = svdup_n_u8(0x0f);\n        // 8 accumulators: 2 row pairs × 4 col pairs\n        svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67;\n        uint32_t idx_arr[8] = { 0, 2, 4, 6,  1, 3, 5, 7 };\n        svbool_t pg = svptrue_pat_b32(SV_VL8);\n        svuint32_t idx = svld1(pg, idx_arr);\n\n        static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7};\n        svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data);\n\n        for (int y = 0; y < nr / q8_k_blocklen; y++) {\n            const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n            for (int x = 0; x < nc / ncols_interleaved; x++) {\n                const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n                acc_f32_01 = svdup_n_f32(0);\n                acc_f32_23 = svdup_n_f32(0);\n                acc_f32_45 = svdup_n_f32(0);\n                acc_f32_67 = svdup_n_f32(0);\n\n                for (int b = 0; b < nb; b++) {\n                    // bsums pairs belongs to the same q8_k subblock\n                    // 64 elements loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum\n                    const int16x8_t bsums[4]{\n                        vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n                        vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n                        vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n                        vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n                    };\n\n                    int32_t bsums_arr32[4][8];\n\n                    for (int q8_row = 0; q8_row < 4; q8_row++) {\n                        int16x8_t v16 = bsums[q8_row];\n\n                        // low 4\n                        int32x4_t v32_lo = vmovl_s16(vget_low_s16(v16));\n                        vst1q_s32(&bsums_arr32[q8_row][0], v32_lo);\n\n                        // high 4\n                        int32x4_t v32_hi = vmovl_s16(vget_high_s16(v16));\n                        vst1q_s32(&bsums_arr32[q8_row][4], v32_hi);\n                    }\n\n                    svint32_t sb_acc_0 = svdup_n_s32(0);\n                    svint32_t sb_acc_2 = svdup_n_s32(0);\n\n                    svint32_t acc_00 = svdup_n_s32(0);\n                    svint32_t acc_11 = svdup_n_s32(0);\n                    svint32_t acc_22 = svdup_n_s32(0);\n                    svint32_t acc_33 = svdup_n_s32(0);\n                    svint32_t acc_44 = svdup_n_s32(0);\n                    svint32_t acc_55 = svdup_n_s32(0);\n                    svint32_t acc_66 = svdup_n_s32(0);\n                    svint32_t acc_77 = svdup_n_s32(0);\n\n                    svint32_t bias_acc_00 = svdup_n_s32(0);\n                    svint32_t bias_acc_22 = svdup_n_s32(0);\n                    svint32_t bias_acc_44 = svdup_n_s32(0);\n                    svint32_t bias_acc_66 = svdup_n_s32(0);\n\n                    for (int sb = 0; sb < QK_K / 64; sb++) {\n                        // Need scales for the low and high nibbles\n                        // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                        svint32_t block_scale_0, block_scale_1, block_scale_2, block_scale_3;\n                        svint32_t q4sb_mins_0, q4sb_mins_1;\n                        {\n                            // 2-superblock I am working on\n                            const int offset = sb * 24 + 0 * 12;\n                            const uint8_t * scales_in = &q4_ptr[b].scales[offset];\n\n                            const int offset1 = sb * 24 + 12;\n                            const uint8_t * scales_in1 = &q4_ptr[b].scales[offset1];\n\n                            constexpr uint32_t kmask1 = 0x3f3f3f3f;\n                            constexpr uint32_t kmask2 = 0x0f0f0f0f;\n                            constexpr uint32_t kmask3 = 0x03030303;\n                            constexpr uint8_t  scales_size = 12;\n\n                            uint32_t sm[3];\n                            memcpy(sm, scales_in, scales_size);\n\n                            uint32_t sm1[3];\n                            memcpy(sm1, scales_in1, scales_size);\n\n                            const uint32_t mins_0_3 = sm[1] & kmask1;\n                            const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4);\n\n                            const uint32_t mins_0_3_1 = sm1[1] & kmask1;\n                            const uint32_t mins_4_7_1 = ((sm1[2] >> 4) & kmask2) | (((sm1[1] >> 6) & kmask3) << 4);\n\n                            svuint32_t mins_u32_temp = svzip1_u32(svdup_n_u32(mins_0_3), svdup_n_u32(mins_4_7));\n                            svuint32_t mins_u32_temp_1 = svzip1_u32(svdup_n_u32(mins_0_3_1), svdup_n_u32(mins_4_7_1));\n\n                            /* reinterpret u32 → u8 */\n                            svuint8_t mins_u8 = svreinterpret_u8_u32(mins_u32_temp);\n                            svuint8_t mins_u8_1 = svreinterpret_u8_u32(mins_u32_temp_1);\n\n                            /* widen u8 → u16->u32 (lower half only) */\n                            svuint32_t mins_u16 = svunpklo_u32(svunpklo_u16(mins_u8));\n                            svuint32_t mins_u16_1 = svunpklo_u32(svunpklo_u16(mins_u8_1));\n\n                            q4sb_mins_0 = svreinterpret_s32_u32(mins_u16);\n                            q4sb_mins_1 = svreinterpret_s32_u32(mins_u16_1);\n\n                            uint32_t scales_u32_0 = sm[0] & kmask1;\n                            uint32_t scales_u32_1 = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4);\n                            uint32_t scales_u32_2 = sm1[0] & kmask1;\n                            uint32_t scales_u32_3 = (sm1[2] & kmask2) | (((sm1[0] >> 6) & kmask3) << 4);\n\n                            svuint32_t S01 = svdup_n_u32(scales_u32_0);\n                            svuint32_t S23 = svdup_n_u32(scales_u32_1);\n                            svuint32_t R01 = svdup_n_u32(scales_u32_2);\n                            svuint32_t R23 = svdup_n_u32(scales_u32_3);\n\n                            svint8_t S01_b = svreinterpret_s8_u32(S01);\n                            svint8_t S23_b = svreinterpret_s8_u32(S23);\n                            svint8_t R01_b = svreinterpret_s8_u32(R01);\n                            svint8_t R23_b = svreinterpret_s8_u32(R23);\n\n                            svint32_t S01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S01_b, S01_b)));\n                            svint32_t R01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R01_b, R01_b)));\n                            svint32_t S23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S23_b, S23_b)));\n                            svint32_t R23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R23_b, R23_b)));\n\n                            block_scale_0 = svtbl_s32(svzip1_s32(S01_d, R01_d), idx);\n                            block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx);\n                            block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx);\n                            block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx);\n                        }\n\n                        const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256;\n\n                        // Load 32-byte per row pair, 1 subblock each time\n                        // predicate for activating higher lanes for 16 int8 elements\n                        const svbool_t ph16 = svptrue_pat_b8(SV_VL16);\n                        // predicate for activating lower lanes for  16 int8 elements\n                        const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16);\n\n                        svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 0), svld1_s8(pl16, q8_base_1 + 112));\n                        svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 32), svld1_s8(pl16, q8_base_1 + 144));\n                        svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 64), svld1_s8(pl16, q8_base_1 + 176));\n                        svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 96), svld1_s8(pl16, q8_base_1 + 208));\n\n                        svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 16), svld1_s8(pl16, q8_base_1 + 128));\n                        svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 48), svld1_s8(pl16, q8_base_1 + 160));\n                        svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 80), svld1_s8(pl16, q8_base_1 + 192));\n                        svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 112), svld1_s8(pl16, q8_base_1 + 224));\n\n                        // Q4s columns iterated in pairs (01, 23, 45, 67)\n                        for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n\n                            sb_acc_0 = svdup_n_s32(0);\n                            sb_acc_2 = svdup_n_s32(0);\n\n                            svuint8_t q4_qs_cp_00 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 0);\n                            svuint8_t q4_qs_cp_01 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 64);\n                            svuint8_t q4_qs_cp_02 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 128);\n                            svuint8_t q4_qs_cp_03 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 192);\n\n                            svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_00, m4b_1), 4));\n                            svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_01, m4b_1), 4));\n                            svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_02, m4b_1), 4));\n                            svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_03, m4b_1), 4));\n\n                            sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_0);\n                            sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_2);\n\n                            sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_4);\n                            sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_6);\n\n                            sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_1);\n                            sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_3);\n\n                            sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_5);\n                            sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_7);\n\n                            if(cp == 0) {\n                                acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0, block_scale_0);\n                                acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2, block_scale_0);\n                            }\n                            if(cp == 1) {\n                                acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0, block_scale_1);\n                                acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2, block_scale_1);\n                            }\n                            if(cp == 2) {\n                                acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0, block_scale_2);\n                                acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2, block_scale_2);\n                            }\n                            if(cp == 3) {\n                                acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0, block_scale_3);\n                                acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2, block_scale_3);\n                            }\n                        }\n\n                        bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][0]), q4sb_mins_0);\n                        bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][1]), q4sb_mins_1);\n\n                        bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][2]), q4sb_mins_0);\n                        bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][3]), q4sb_mins_1);\n\n                        bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][4]), q4sb_mins_0);\n                        bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][5]), q4sb_mins_1);\n\n                        bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][6]), q4sb_mins_0);\n                        bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][7]), q4sb_mins_1);\n                    }  // for sb\n\n\n                    acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4));\n                    acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4));\n                    acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4));\n                    acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4));\n                    acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4));\n                    acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4));\n                    acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4));\n                    acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4));\n\n                    svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1);\n                    svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1);\n\n                    svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1);\n                    svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1);\n\n                    // Broadcast q8 scalar\n                    svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]);\n\n                    svfloat32_t q4_dmin_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].dmin), svdup_f16(0)));\n\n                    svfloat32_t q4_d_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].d), svdup_f16(0)));\n\n                    svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n                    svfloat32_t dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n                    acc_f32_01 = svmls_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_00), dmins1);\n                    acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1);\n\n                    q8_d = svdup_f32(q8_ptr[b].d[1]);\n\n                    scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n                    dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n                    acc_f32_23 = svmls_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_22), dmins1);\n                    acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1);\n\n                    q8_d = svdup_f32(q8_ptr[b].d[2]);\n\n\n                    scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n                    dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n                    acc_f32_45 = svmls_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_44), dmins1);\n                    acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1);\n\n                    q8_d = svdup_f32(q8_ptr[b].d[3]);\n\n                    scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d);\n                    dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d);\n\n                    acc_f32_67 = svmls_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_66), dmins1);\n                    acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1);\n\n                }  // for b\n\n                // With the previous reorder, the tile is already in the correct memory layout.\n                // Predicate for exactly 4 lanes\n                svbool_t pg4 = svptrue_pat_b32(SV_VL4);\n                for (int i = 0; i < q8_k_blocklen; i++) {\n                    int row = y * q8_k_blocklen + i;\n                    for (int j = 0; j < 2; j++) {\n                        int col    = x * ncols_interleaved + j * 4;\n                        int offset = row * bs + col;\n\n                        if (i == 0 && j == 0) {\n                            // acc_f32_0 → lower half of acc_f32_01\n                            svst1_f32(pg4, s + offset, acc_f32_01);\n                        } else if (i == 0 && j == 1) {\n                            // acc_f32_1 → upper half of acc_f32_01\n                            svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4));\n                        } else if (i == 1 && j == 0) {\n                            // acc_f32_2\n                            svst1_f32(pg4, s + offset, acc_f32_23);\n                        } else if (i == 1 && j == 1) {\n                            // acc_f32_3\n                            svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4));\n                        } else if (i == 2 && j == 0) {\n                            // acc_f32_4\n                            svst1_f32(pg4, s + offset, acc_f32_45);\n                        } else if (i == 2 && j == 1) {\n                            // acc_f32_5\n                            svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4));\n                        } else if (i == 3 && j == 0) {\n                            // acc_f32_6\n                            svst1_f32(pg4, s + offset, acc_f32_67);\n                        } else if (i == 3 && j == 1) {\n                            // acc_f32_7\n                            svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4));\n                        }\n                    }\n                }\n            }  // for x\n        }  // for y\n        return;\n    }\n#endif  // SVE compile-time end\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    constexpr int    q8_k_blocklen = 4;\n    const uint8x16_t m4b           = vdupq_n_u8(0x0f);\n\n    // 8 accumulators: 2 row pairs × 4 col pairs\n    float32x4_t acc_f32[blocklen];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < blocklen; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                // bsums pairs belongs to the same q8_k subblock\n                const int16x8_t bsums[4]{\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n                };\n                int16_t bsums_arr[4][8];\n                for (int q8_row = 0; q8_row < 4; q8_row++) {\n                    vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n                }\n\n                int32x4_t sb_acc[4];    // Aux accumulators to store subblock (partial) results\n                int32x4_t acc[8];       // rows 01 stored in [0][1][2][3] rows 23 stored in [4][5][6][7]\n                int32x4_t bias_acc[8];  // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567, [2]->r1 0123 ...\n                for (int i = 0; i < 8; i++) {\n                    acc[i]      = vdupq_n_s32(0);\n                    bias_acc[i] = vdupq_n_s32(0);\n                }\n\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    // Need scales for the low and high nibbles\n                    // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                    int8_t    q4sb_scales[2][8];\n                    int16x8_t q4sb_mins[2];  // int16 as its needed for bias_acc later\n                    for (int i = 0; i < 2; i++) {\n                        const int offset = sb * 24 + i * 12;\n                        decode_q_Kx8_6bit_scales(&q4_ptr[b].scales[offset], &q4sb_mins[i], q4sb_scales[i]);\n                    }\n\n                    // q8_ptr[b].qs has interleaved Q8 rows (01, 23)\n                    const int8_t * q8_base = q8_ptr[b].qs + sb * 256;\n\n                    int8x16_t q8_qs_01[8];\n                    int8x16_t q8_qs_23[8];\n\n                    // Load 32-byte per row pair, 1 subblock each time\n                    for (int i = 0; i < 8; i++) {\n                        const int offset = i * 32;  // 16 for row 01, 16 for row 23\n                        q8_qs_01[i]      = vld1q_s8(q8_base + offset);\n                        q8_qs_23[i]      = vld1q_s8(q8_base + offset + 16);\n                    }\n\n                    const int8x16_t q8s[2][8] = {\n                        { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3],\n                          q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] },\n                        { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3],\n                          q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] },\n                    };\n\n                    // Q4s columns iterated in pairs (01, 23, 45, 67)\n                    for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n                        for (int i = 0; i < 4; i++) {\n                            sb_acc[i] = vdupq_n_s32(0);\n                        }\n\n                        uint8x16_t q4_qs_cp_0 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 0);    // 0 .. 7 & 32..39\n                        uint8x16_t q4_qs_cp_1 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 64);   // 8 ..15 & 40..47\n                        uint8x16_t q4_qs_cp_2 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 128);  // 16..23 & 48..55\n                        uint8x16_t q4_qs_cp_3 = vld1q_u8(q4_ptr[b].qs + sb * QK_K + 16 * cp + 192);  // 24..31 & 56..63\n                        const int8x16_t q4_nibbles[2][4] = {\n                            {\n                                vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_0, m4b)),\n                                vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_1, m4b)),\n                                vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_2, m4b)),\n                                vreinterpretq_s8_u8(vandq_u8(q4_qs_cp_3, m4b)),\n                            },\n                            {\n                                vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)),\n                                vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)),\n                                vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)),\n                                vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)),\n                            }\n                        };\n\n                        // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8\n                        // for each of the internal 32 qs subblock (blk)\n                        for (int rp = 0; rp < 2; rp++) {\n                            for (int blk = 0; blk < 2; blk++) {\n                                const int8x16_t * q8  = &q8s[rp][4 * blk];\n                                const int8x16_t * q4  = q4_nibbles[blk];\n                                int32x4_t         acc = sb_acc[2 * rp + blk];\n                                // mul add for each qs in the same subblock\n                                for (int qs_offset = 0; qs_offset < 4; qs_offset++) {\n                                    acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]);\n                                }\n                                sb_acc[2 * rp + blk] = acc;\n                            }\n                        }\n\n                        // Scales[i] corresponds to column i\n                        const int scale_offset = cp * 2;\n                        const int32_t scale_00 = q4sb_scales[0][scale_offset];\n                        const int32_t scale_01 = q4sb_scales[0][scale_offset + 1];\n                        const int32_t scale_10 = q4sb_scales[1][scale_offset];\n                        const int32_t scale_11 = q4sb_scales[1][scale_offset + 1];\n                        const int32x4_t block_scale_0 = vcombine_s32(vdup_n_s32(scale_00), vdup_n_s32(scale_01));\n                        const int32x4_t block_scale_1 = vcombine_s32(vdup_n_s32(scale_10), vdup_n_s32(scale_11));\n\n                        acc[cp]     = vmlaq_s32(acc[cp], sb_acc[0], block_scale_0);\n                        acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale_0);\n                        acc[cp]     = vmlaq_s32(acc[cp], sb_acc[1], block_scale_1);\n                        acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale_1);\n                    }\n\n                    // Multiply Acc bsum + mins\n                    for (int q8_row = 0; q8_row < 4; q8_row++) {\n                        // Each pair of subblocks share the same bsums\n                        // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n                        int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]);\n                        int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]);\n\n                        bias_acc[2 * q8_row] =\n                            vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0]));\n                        bias_acc[2 * q8_row] =\n                            vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q4sb_mins[1]));\n                        bias_acc[2 * q8_row + 1] =\n                            vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0]));\n                        bias_acc[2 * q8_row + 1] =\n                            vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1]));\n                    }\n                }  // for sb\n\n                // Reorder of i8mm output with bias and output layout\n                for (int i = 0; i < 8; i++) {\n                    int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n                    acc[i]          = vcombine_s32(aux.val[0], aux.val[1]);\n                }\n                int32x4_t reorder_acc[8] = {\n                    vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n                    vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n                    vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n                    vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n                    vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n                    vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n                    vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n                    vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n                };\n\n                for (int i = 0; i < q8_k_blocklen; i++) {\n                    for (int j = 0; j < 2; j++) {\n                        float32x4_t       q8_d    = vdupq_n_f32(q8_ptr[b].d[i]);\n                        float32x4_t       q4_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].dmin + j * 4)));\n                        const float32x4_t dmins   = vmulq_f32(q4_dmin, q8_d);\n\n                        float32x4_t       q4_d  = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].d + j * 4)));\n                        const float32x4_t scale = vmulq_f32(q4_d, q8_d);\n\n                        acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins);\n                        acc_f32[2 * i + j] =\n                            vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n                    }\n                }\n            }  // for b\n\n            // With the previous reorder, the tile is already in the correct memory layout.\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    ggml_gemm_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    constexpr int    q8_k_blocklen = 4;\n    constexpr int    col_pairs     = ncols_interleaved / 2;\n    const uint8x16_t m4b           = vdupq_n_u8(0x0f);\n    const uint8x16_t mone          = vdupq_n_u8(1);\n    const uint8x16_t mtwo          = vdupq_n_u8(2);\n\n    // 8 accumulators: 2 row pairs × 4 col pairs\n    float32x4_t acc_f32[blocklen];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q5_Kx8 * GGML_RESTRICT q5_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < blocklen; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                // bsums pairs belongs to the same q8_k subblock\n                const int16x8_t bsums[4]{\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)),\n                    vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)),\n                };\n                int16_t bsums_arr[4][8];\n                for (int q8_row = 0; q8_row < 4; q8_row++) {\n                    vst1q_s16(bsums_arr[q8_row], bsums[q8_row]);\n                }\n\n                int32x4_t sb_acc[4];    // Aux accumulators to store subblock (partial) results\n                int32x4_t acc[8];       // rows 01 stored in [0][1][2][3] rows 23 stored in [4][5][6][7]\n                int32x4_t bias_acc[8];  // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567, [2]->r1 0123 ...\n                for (int i = 0; i < 8; i++) {\n                    acc[i]      = vdupq_n_s32(0);\n                    bias_acc[i] = vdupq_n_s32(0);\n                }\n\n                // Load qh once per block and shift after each subblock\n                const uint8_t * qh_base = q5_ptr[b].qh;\n                uint8x16_t      qh[col_pairs][4];\n                for (int cp = 0; cp < col_pairs; cp++) {\n                    qh[cp][0] = vld1q_u8(qh_base + 16 * cp);\n                    qh[cp][1] = vld1q_u8(qh_base + 16 * cp + 64);\n                    qh[cp][2] = vld1q_u8(qh_base + 16 * cp + 128);\n                    qh[cp][3] = vld1q_u8(qh_base + 16 * cp + 192);\n                }\n\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n                    // Need scales for the low and high nibbles\n                    // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total\n                    int8_t    q5sb_scales[2][8];\n                    int16x8_t q5sb_mins[2];  // int16 as its needed for bias_acc later\n                    for (int i = 0; i < 2; i++) {\n                        const int offset = sb * 24 + i * 12;\n                        decode_q_Kx8_6bit_scales(&q5_ptr[b].scales[offset], &q5sb_mins[i], q5sb_scales[i]);\n                    }\n\n                    // q8_ptr[b].qs has interleaved Q8 rows (01, 23)\n                    const int8_t * q8_base = q8_ptr[b].qs + sb * 256;\n\n                    int8x16_t q8_qs_01[8];\n                    int8x16_t q8_qs_23[8];\n\n                    // Load 32-byte per row pair, 1 subblock each time\n                    for (int i = 0; i < 8; i++) {\n                        const int offset = i * 32;  // 16 for row 01, 16 for row 23\n                        q8_qs_01[i]      = vld1q_s8(q8_base + offset);\n                        q8_qs_23[i]      = vld1q_s8(q8_base + offset + 16);\n                    }\n\n                    const int8x16_t q8s[2][8] = {\n                        { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6],\n                         q8_qs_01[7] },\n                        { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6],\n                         q8_qs_23[7] },\n                    };\n\n                    // Q5s columns iterated in pairs (01, 23, 45, 67)\n                    for (int cp = 0; cp < col_pairs; cp++) {\n                        for (int i = 0; i < 4; i++) {\n                            sb_acc[i] = vdupq_n_s32(0);\n                        }\n\n                        uint8x16_t qs_cp_0 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 0);    // 0 .. 7 & 32..39\n                        uint8x16_t qs_cp_1 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 64);   // 8 ..15 & 40..47\n                        uint8x16_t qs_cp_2 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 128);  // 16..23 & 48..55\n                        uint8x16_t qs_cp_3 = vld1q_u8(q5_ptr[b].qs + sb * QK_K + 16 * cp + 192);  // 24..31 & 56..63\n\n                        // This is the only part of the algorithm that differs with Q4_K\n                        // Extract High bits and pack into 5 bit weights\n                        uint8x16_t hbit_lo_0    = vandq_u8(qh[cp][0], mone);\n                        uint8x16_t hbit_hi_0    = vshlq_n_u8(vandq_u8(qh[cp][0], mtwo), 3);\n                        qh[cp][0]               = vshrq_n_u8(qh[cp][0], 2);\n                        // Same as Q4_K, i8mm to dequantize the weights.\n                        const int8x16_t qs_lo_0 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_0, m4b), hbit_lo_0, 4));\n                        int32x4_t       acc_0   = sb_acc[0];\n                        acc_0                   = vmmlaq_s32(acc_0, qs_lo_0, q8s[0][0]);\n                        int32x4_t acc_2         = sb_acc[2];\n                        acc_2                   = vmmlaq_s32(acc_2, qs_lo_0, q8s[1][0]);\n                        const int8x16_t qs_hi_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_0, 4), hbit_hi_0));\n                        int32x4_t       acc_1   = sb_acc[1];\n                        acc_1                   = vmmlaq_s32(acc_1, qs_hi_0, q8s[0][4]);\n                        int32x4_t acc_3         = sb_acc[3];\n                        acc_3                   = vmmlaq_s32(acc_3, qs_hi_0, q8s[1][4]);\n\n                        // Repeat for the other 3 columns (8..15, 16..23, 24..31)\n                        uint8x16_t hbit_hi_1    = vshlq_n_u8(vandq_u8(qh[cp][1], mtwo), 3);\n                        uint8x16_t hbit_lo_1    = vandq_u8(qh[cp][1], mone);\n                        qh[cp][1]               = vshrq_n_u8(qh[cp][1], 2);\n                        const int8x16_t qs_lo_1 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_1, m4b), hbit_lo_1, 4));\n                        acc_0                   = vmmlaq_s32(acc_0, qs_lo_1, q8s[0][1]);\n                        acc_2                   = vmmlaq_s32(acc_2, qs_lo_1, q8s[1][1]);\n                        const int8x16_t qs_hi_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_1, 4), hbit_hi_1));\n                        acc_1                   = vmmlaq_s32(acc_1, qs_hi_1, q8s[0][5]);\n                        acc_3                   = vmmlaq_s32(acc_3, qs_hi_1, q8s[1][5]);\n\n                        uint8x16_t hbit_hi_2    = vshlq_n_u8(vandq_u8(qh[cp][2], mtwo), 3);\n                        uint8x16_t hbit_lo_2    = vandq_u8(qh[cp][2], mone);\n                        qh[cp][2]               = vshrq_n_u8(qh[cp][2], 2);\n                        const int8x16_t qs_lo_2 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_2, m4b), hbit_lo_2, 4));\n                        acc_0                   = vmmlaq_s32(acc_0, qs_lo_2, q8s[0][2]);\n                        acc_2                   = vmmlaq_s32(acc_2, qs_lo_2, q8s[1][2]);\n                        const int8x16_t qs_hi_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_2, 4), hbit_hi_2));\n                        acc_1                   = vmmlaq_s32(acc_1, qs_hi_2, q8s[0][6]);\n                        acc_3                   = vmmlaq_s32(acc_3, qs_hi_2, q8s[1][6]);\n\n                        uint8x16_t hbit_lo_3    = vandq_u8(qh[cp][3], mone);\n                        uint8x16_t hbit_hi_3    = vshlq_n_u8(vandq_u8(qh[cp][3], mtwo), 3);\n                        qh[cp][3]               = vshrq_n_u8(qh[cp][3], 2);\n                        const int8x16_t qs_lo_3 = vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(qs_cp_3, m4b), hbit_lo_3, 4));\n                        acc_0                   = vmmlaq_s32(acc_0, qs_lo_3, q8s[0][3]);\n                        sb_acc[0]               = acc_0;\n                        acc_2                   = vmmlaq_s32(acc_2, qs_lo_3, q8s[1][3]);\n                        sb_acc[2]               = acc_2;\n\n                        // Scales[i] corresponds to column i\n                        const int       scale_offset = cp * 2;\n                        const int32_t   s0           = q5sb_scales[0][scale_offset];\n                        const int32_t   s1           = q5sb_scales[0][scale_offset + 1];\n                        const int32x4_t block_scale  = vcombine_s32(vdup_n_s32(s0), vdup_n_s32(s1));\n                        acc[cp]                      = vmlaq_s32(acc[cp], sb_acc[0], block_scale);\n                        acc[cp + 4]                  = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale);\n\n                        const int8x16_t qs_hi_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(qs_cp_3, 4), hbit_hi_3));\n                        acc_1                   = vmmlaq_s32(acc_1, qs_hi_3, q8s[0][7]);\n                        sb_acc[1]               = acc_1;\n                        acc_3                   = vmmlaq_s32(acc_3, qs_hi_3, q8s[1][7]);\n                        sb_acc[3]               = acc_3;\n\n                        const int32_t   s2           = q5sb_scales[1][scale_offset];\n                        const int32_t   s3           = q5sb_scales[1][scale_offset + 1];\n                        const int32x4_t block_scale2 = vcombine_s32(vdup_n_s32(s2), vdup_n_s32(s3));\n                        acc[cp]                      = vmlaq_s32(acc[cp], sb_acc[1], block_scale2);\n                        acc[cp + 4]                  = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale2);\n                    }\n\n                    // Multiply Acc bsum + mins\n                    for (int q8_row = 0; q8_row < 4; q8_row++) {\n                        // Each pair of subblocks share the same bsums\n                        // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)).\n                        int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]);\n                        int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]);\n\n                        bias_acc[2 * q8_row] =\n                            vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q5sb_mins[0]));\n                        bias_acc[2 * q8_row] =\n                            vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q5sb_mins[1]));\n                        bias_acc[2 * q8_row + 1] =\n                            vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q5sb_mins[0]));\n                        bias_acc[2 * q8_row + 1] =\n                            vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q5sb_mins[1]));\n                    }\n                }  // for sb\n\n                // Reorder of i8mm output with bias and output layout\n                for (int i = 0; i < 8; i++) {\n                    int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n                    acc[i]          = vcombine_s32(aux.val[0], aux.val[1]);\n                }\n                int32x4_t reorder_acc[8] = {\n                    vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n                    vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n                    vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n                    vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n                    vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n                    vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n                    vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n                    vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n                };\n\n                for (int i = 0; i < q8_k_blocklen; i++) {\n                    for (int j = 0; j < 2; j++) {\n                        float32x4_t       q8_d    = vdupq_n_f32(q8_ptr[b].d[i]);\n                        float32x4_t       q5_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q5_ptr[b].dmin + j * 4)));\n                        const float32x4_t dmins   = vmulq_f32(q5_dmin, q8_d);\n\n                        float32x4_t       q5_d  = vcvt_f32_f16(vld1_f16((const __fp16 *) (q5_ptr[b].d + j * 4)));\n                        const float32x4_t scale = vmulq_f32(q5_d, q8_d);\n\n                        acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins);\n                        acc_f32[2 * i + j] =\n                            vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n                    }\n                }\n            }  // for b\n\n            // With the previous reorder, the tile is already in the correct memory layout.\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    ggml_gemm_q5_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x4_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    constexpr int    q8_k_blocklen = 4;\n    constexpr int    col_groups    = ncols_interleaved / 4;\n    constexpr int    acc_size      = q8_k_blocklen * col_groups;  // 4 rows, 2 column groups\n    const uint8x16_t m4b           = vdupq_n_u8(0x0f);\n    const uint8x16_t mask_lo       = vdupq_n_u8(0x03);\n    const uint8x16_t mask_hi       = vdupq_n_u8(0x30);\n    const int8x16_t  m32s          = vdupq_n_s8(32);\n\n    float32x4_t acc_f32[acc_size];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < acc_size; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                float32x4_t q6_d_0123 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d));\n                float32x4_t q6_d_4567 = vcvt_f32_f16(vld1_f16((const __fp16 *) q6_ptr[b].d + 4));\n                float32x4_t q8_d_0123 = vld1q_f32(q8_ptr[b].d);\n\n                float32x4_t sbd_scale_0123[q8_k_blocklen];\n                float32x4_t sbd_scale_4567[q8_k_blocklen];\n\n                sbd_scale_0123[0] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 0);\n                sbd_scale_4567[0] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 0);\n                sbd_scale_0123[1] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 1);\n                sbd_scale_4567[1] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 1);\n                sbd_scale_0123[2] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 2);\n                sbd_scale_4567[2] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 2);\n                sbd_scale_0123[3] = vmulq_laneq_f32(q6_d_0123, q8_d_0123, 3);\n                sbd_scale_4567[3] = vmulq_laneq_f32(q6_d_4567, q8_d_0123, 3);\n\n                int32x4_t acc_s32[acc_size];\n                for (int i = 0; i < acc_size; i++) {\n                    acc_s32[i] = vdupq_n_s32(0);\n                }\n\n                int16_t q6_scales[8 * 16];\n                for (int i = 0; i < 16; i++) {\n                    int16x8_t scales = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n                    vst1q_s16(q6_scales + i * 8, scales);\n                }\n\n                for (int half = 0; half < 2; half++) {\n                    const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n                    const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n                    for (int sb = 0; sb < QK_K / 64; sb++) {\n                        int32x4_t acc_lo[acc_size];\n                        int32x4_t acc_hi[acc_size];\n                        for (int i = 0; i < acc_size; i++) {\n                            acc_lo[i] = vdupq_n_s32(0);\n                            acc_hi[i] = vdupq_n_s32(0);\n                        }\n\n                        const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;\n                        const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;\n\n                        // 4 rows * 16 elements per scale\n                        // 4 reads of 16 bytes each\n                        constexpr int reads_per_sb = 4;\n                        int8x16_t     q8_l[reads_per_sb];\n                        int8x16_t     q8_h[reads_per_sb];\n                        for (int k = 0; k < reads_per_sb; k++) {\n                            q8_l[k] = vld1q_s8(q8_base_l + 16 * k);\n                            q8_h[k] = vld1q_s8(q8_base_h + 16 * k);\n                        }\n\n                        const int ql_off_base = sb * QK_K / 2;\n                        const int qh_off_base = ql_off_base & 255;\n\n                        uint8x16_t q6_ql_0123[reads_per_sb];\n                        uint8x16_t q6_ql_4567[reads_per_sb];\n                        uint8x16_t q6_qh_0123[reads_per_sb];\n                        uint8x16_t q6_qh_4567[reads_per_sb];\n\n                        for (int k = 0; k < reads_per_sb; k++) {\n                            q6_ql_0123[k] = vld1q_u8(ql_base + ql_off_base + k * 32);\n                            q6_ql_4567[k] = vld1q_u8(ql_base + ql_off_base + k * 32 + 16);\n                            q6_qh_0123[k] = vld1q_u8(qh_base + qh_off_base + k * 32);\n                            q6_qh_4567[k] = vld1q_u8(qh_base + qh_off_base + k * 32 + 16);\n                        }\n\n                        if (sb > 1) {\n                            for (int k = 0; k < reads_per_sb; k++) {\n                                q6_qh_0123[k] = vshrq_n_u8(q6_qh_0123[k], 2);\n                                q6_qh_4567[k] = vshrq_n_u8(q6_qh_4567[k], 2);\n                            }\n                        }\n\n                        for (int k = 0; k < reads_per_sb; k++) {\n                            // q = (ql | qh) - 32\n                            const uint8x16_t hbit_lo_0123 = vandq_u8(q6_qh_0123[k], mask_lo);\n                            const uint8x16_t hbit_hi_0123 = vandq_u8(q6_qh_0123[k], mask_hi);\n                            const uint8x16_t hbit_lo_4567 = vandq_u8(q6_qh_4567[k], mask_lo);\n                            const uint8x16_t hbit_hi_4567 = vandq_u8(q6_qh_4567[k], mask_hi);\n\n                            const int8x16_t q6_0123_lo = vsubq_s8(\n                                vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_0123[k], m4b), hbit_lo_0123, 4)), m32s);\n                            const int8x16_t q6_0123_hi = vsubq_s8(\n                                vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_0123[k], 4), hbit_hi_0123)), m32s);\n\n                            acc_lo[0] = vdotq_laneq_s32(acc_lo[0], q6_0123_lo, q8_l[k], 0);  //  0..3  r0 c0123\n                            acc_lo[1] = vdotq_laneq_s32(acc_lo[1], q6_0123_lo, q8_l[k], 1);  //  0..3  r1 c0123\n                            acc_lo[2] = vdotq_laneq_s32(acc_lo[2], q6_0123_lo, q8_l[k], 2);  //  0..3  r2 c0123\n                            acc_lo[3] = vdotq_laneq_s32(acc_lo[3], q6_0123_lo, q8_l[k], 3);  //  0..3  r3 c0123\n\n                            acc_hi[0] = vdotq_laneq_s32(acc_hi[0], q6_0123_hi, q8_h[k], 0);  // 64..67 r0 c0123\n                            acc_hi[1] = vdotq_laneq_s32(acc_hi[1], q6_0123_hi, q8_h[k], 1);  // 64..67 r1 c0123\n                            acc_hi[2] = vdotq_laneq_s32(acc_hi[2], q6_0123_hi, q8_h[k], 2);  // 64..67 r2 c0123\n                            acc_hi[3] = vdotq_laneq_s32(acc_hi[3], q6_0123_hi, q8_h[k], 3);  // 64..67 r3 c0123\n\n                            const int8x16_t q6_4567_lo = vsubq_s8(\n                                vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_ql_4567[k], m4b), hbit_lo_4567, 4)), m32s);\n                            const int8x16_t q6_4567_hi = vsubq_s8(\n                                vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_ql_4567[k], 4), hbit_hi_4567)), m32s);\n\n                            acc_lo[4] = vdotq_laneq_s32(acc_lo[4], q6_4567_lo, q8_l[k], 0);  //  0..3  r0 c4567\n                            acc_lo[5] = vdotq_laneq_s32(acc_lo[5], q6_4567_lo, q8_l[k], 1);  //  0..3  r1 c4567\n                            acc_lo[6] = vdotq_laneq_s32(acc_lo[6], q6_4567_lo, q8_l[k], 2);  //  0..3  r2 c4567\n                            acc_lo[7] = vdotq_laneq_s32(acc_lo[7], q6_4567_lo, q8_l[k], 3);  //  0..3  r3 c4567\n\n                            acc_hi[4] = vdotq_laneq_s32(acc_hi[4], q6_4567_hi, q8_h[k], 0);  // 64..67 r0 c4567\n                            acc_hi[5] = vdotq_laneq_s32(acc_hi[5], q6_4567_hi, q8_h[k], 1);  // 64..67 r1 c4567\n                            acc_hi[6] = vdotq_laneq_s32(acc_hi[6], q6_4567_hi, q8_h[k], 2);  // 64..67 r2 c4567\n                            acc_hi[7] = vdotq_laneq_s32(acc_hi[7], q6_4567_hi, q8_h[k], 3);  // 64..67 r3 c4567\n                        }\n\n                        // Scale and bias\n                        const int scale_idx_l = half * 8 + sb;\n                        const int scale_idx_h = half * 8 + sb + 4;\n\n                        for (int g = 0; g < col_groups; g++) {\n                            const int16x4_t scales_l16  = vld1_s16(q6_scales + scale_idx_l * 8 + g * 4);\n                            const int16x4_t scales_h16  = vld1_s16(q6_scales + scale_idx_h * 8 + g * 4);\n                            const int32x4_t scale_vec_l = vmovl_s16(scales_l16);\n                            const int32x4_t scale_vec_h = vmovl_s16(scales_h16);\n                            const int       acc_offset  = g * q8_k_blocklen;\n\n                            for (int row = 0; row < q8_k_blocklen; row++) {\n                                const int idx = row * 2 + g;\n                                acc_s32[idx]  = vmlaq_s32(acc_s32[idx], acc_lo[acc_offset + row], scale_vec_l);\n                                acc_s32[idx]  = vmlaq_s32(acc_s32[idx], acc_hi[acc_offset + row], scale_vec_h);\n                            }\n                        }\n                    }\n                }\n\n                // Finally we apply the superblock scales\n                for (int row = 0; row < q8_k_blocklen; row++) {\n                    const int       idx0     = 2 * row;\n                    const int       idx1     = 2 * row + 1;\n                    const int32x4_t acc_0123 = acc_s32[idx0];\n                    const int32x4_t acc_4567 = acc_s32[idx1];\n\n                    acc_f32[idx0] = vmlaq_f32(acc_f32[idx0], vcvtq_f32_s32(acc_0123), sbd_scale_0123[row]);\n                    acc_f32[idx1] = vmlaq_f32(acc_f32[idx1], vcvtq_f32_s32(acc_4567), sbd_scale_4567[row]);\n                }\n            }  // for b\n\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemm_q6_K_8x4_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x8_q8_K(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    constexpr int qk = QK_K;\n    const int     nb = n / qk;\n\n    constexpr int ncols_interleaved = 8;\n    constexpr int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    constexpr int    q8_k_blocklen = 4;\n    const uint8x16_t m4b           = vdupq_n_u8(0x0f);\n    const uint8x16_t mask_lo       = vdupq_n_u8(0x03);\n    const uint8x16_t mask_hi       = vdupq_n_u8(0x30);\n    const int8x16_t  m32s          = vdupq_n_s8(32);\n\n    // 8 accumulators: 4 q8 rows × 2 col groups (0-3, 4-7)\n    float32x4_t acc_f32[blocklen];\n\n    for (int y = 0; y < nr / q8_k_blocklen; y++) {\n        const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n            for (int i = 0; i < blocklen; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                int32x4_t acc[8];  // rows 01 stored in [0][1][2][3], rows 23 stored in [4][5][6][7]\n                for (int i = 0; i < 8; i++) {\n                    acc[i] = vdupq_n_s32(0);\n                }\n\n                // Q6_K has simple 8-bit scales, 16 per block (one per 16 values)\n                // Reused for bias and dequantization later\n                int16_t q6_scales[16 * 8];\n                for (int i = 0; i < 16; ++i) {\n                    int16x8_t s16 = vmovl_s8(vld1_s8(q6_ptr[b].scales + i * 8));\n                    vst1q_s16(q6_scales + i * 8, s16);\n                }\n\n                // Process two 128-value halves per superblock\n                for (int half = 0; half < 2; half++) {\n\n                    const uint8_t * ql_base = q6_ptr[b].ql + half * 512;\n                    const uint8_t * qh_base = q6_ptr[b].qh + half * 256;\n\n                    // A subblock (sb) is a set of weights that share the scale\n                    // Since q6_K scales are per 16 elements\n                    // num sbs -> 256 elements / (16 elements/scale * 2 elements/byte * 2 halves)\n                    for (int sb = 0; sb < QK_K / 64; sb++) {\n                        // Q6_K weight index increasing by 64 instead of 32 requires\n                        // loading various q8 memory regions\n                        const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64;\n                        const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64;\n\n                        int8x16_t q8_l_01[2];\n                        int8x16_t q8_l_23[2];\n                        for (int i = 0; i < 2; i++) {\n                            const int offset = i * 32;\n                            q8_l_01[i]       = vld1q_s8(q8_base_l + offset);       // 0..7 & 8..15 (r01)\n                            q8_l_23[i]       = vld1q_s8(q8_base_l + offset + 16);  // 0..7 & 8..15 (r23)\n                        }\n\n                        int8x16_t q8_h_01[2];\n                        int8x16_t q8_h_23[2];\n                        for (int i = 0; i < 2; i++) {\n                            const int offset = i * 32;\n                            q8_h_01[i]       = vld1q_s8(q8_base_h + offset);\n                            q8_h_23[i]       = vld1q_s8(q8_base_h + offset + 16);\n                        }\n\n                        const int ql_off_base = sb * QK_K / 2;\n\n                        uint8x16_t q6_ql_0[4];\n                        uint8x16_t q6_ql_1[4];\n                        for (int k = 0; k < 4; k++) {\n                            q6_ql_0[k] = vld1q_u8(ql_base + ql_off_base + 16 * k);\n                            q6_ql_1[k] = vld1q_u8(ql_base + ql_off_base + 64 + 16 * k);\n                        }\n\n                        const int  qh_off_base = (sb * QK_K / 2) & 255;  // wrap after 256 bytes\n                        uint8x16_t q6_qh_0[4];\n                        uint8x16_t q6_qh_1[4];\n                        for (int k = 0; k < 4; k++) {\n                            q6_qh_0[k] = vld1q_u8(qh_base + qh_off_base + 16 * k);\n                            q6_qh_1[k] = vld1q_u8(qh_base + qh_off_base + 64 + 16 * k);\n                        }\n\n                        // Adjust for the proper high bits (Sb 2 and 3)\n                        if (sb > 1) {\n                            for (int k = 0; k < 4; k++) {\n                                q6_qh_0[k] = vshrq_n_u8(q6_qh_0[k], 2);\n                                q6_qh_1[k] = vshrq_n_u8(q6_qh_1[k], 2);\n                            }\n                        }\n\n                        // Process column pairs (0-1, 2-3, 4-5, 6-7)\n                        for (int cp = 0; cp < ncols_interleaved / 2; cp++) {\n                            const uint8x16_t q6_qs_cp_0_l = q6_ql_0[cp];\n                            const uint8x16_t q6_qs_cp_1_l = q6_ql_1[cp];\n                            const uint8x16_t q6_qs_cp_0_h = q6_qh_0[cp];\n                            const uint8x16_t q6_qs_cp_1_h = q6_qh_1[cp];\n\n                            // Extract high 2 bits for upper nibble reconstruction\n                            const uint8x16_t q6_qs_cp_0_hh = vandq_u8(q6_qs_cp_0_h, mask_hi);\n                            const uint8x16_t q6_qs_cp_1_hh = vandq_u8(q6_qs_cp_1_h, mask_hi);\n\n                            // q6 = (low4 | high2<<4) - 32\n                            // Use vsliq_n_u8 to combine shift-left-insert in one instruction (like Q5_K)\n                            const int8x16_t q6_l0 = vsubq_s8(\n                                vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_cp_0_l, m4b), vandq_u8(q6_qs_cp_0_h, mask_lo), 4)),\n                                m32s);\n                            const int8x16_t q6_l1 = vsubq_s8(\n                                vreinterpretq_s8_u8(vsliq_n_u8(vandq_u8(q6_qs_cp_1_l, m4b), vandq_u8(q6_qs_cp_1_h, mask_lo), 4)),\n                                m32s);\n                            const int8x16_t q6_h0 = vsubq_s8(\n                                vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_0_l, 4), q6_qs_cp_0_hh)), m32s);\n                            const int8x16_t q6_h1 = vsubq_s8(\n                                vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6_qs_cp_1_l, 4), q6_qs_cp_1_hh)), m32s);\n\n                            // row pair 0, base_l\n                            int32x4_t sb_acc_0l = vmmlaq_s32(vdupq_n_s32(0), q6_l0, q8_l_01[0]);\n                            sb_acc_0l           = vmmlaq_s32(sb_acc_0l, q6_l1, q8_l_01[1]);\n                            // row pair 0, base_h\n                            int32x4_t sb_acc_0h = vmmlaq_s32(vdupq_n_s32(0), q6_h0, q8_h_01[0]);\n                            sb_acc_0h           = vmmlaq_s32(sb_acc_0h, q6_h1, q8_h_01[1]);\n                            // row pair 1, base_l\n                            int32x4_t sb_acc_1l = vmmlaq_s32(vdupq_n_s32(0), q6_l0, q8_l_23[0]);\n                            sb_acc_1l           = vmmlaq_s32(sb_acc_1l, q6_l1, q8_l_23[1]);\n                            // row pair 1, base_h\n                            int32x4_t sb_acc_1h = vmmlaq_s32(vdupq_n_s32(0), q6_h0, q8_h_23[0]);\n                            sb_acc_1h           = vmmlaq_s32(sb_acc_1h, q6_h1, q8_h_23[1]);\n\n                            const int scale_idx_l = half * 8 + sb;\n                            const int scale_idx_h = half * 8 + sb + 4;\n\n                            const int32x4_t scale_vec_l = {\n                                q6_scales[scale_idx_l * 8 + cp * 2 + 0],\n                                q6_scales[scale_idx_l * 8 + cp * 2 + 0],\n                                q6_scales[scale_idx_l * 8 + cp * 2 + 1],\n                                q6_scales[scale_idx_l * 8 + cp * 2 + 1],\n                            };\n                            const int32x4_t scale_vec_h = {\n                                q6_scales[scale_idx_h * 8 + cp * 2 + 0],\n                                q6_scales[scale_idx_h * 8 + cp * 2 + 0],\n                                q6_scales[scale_idx_h * 8 + cp * 2 + 1],\n                                q6_scales[scale_idx_h * 8 + cp * 2 + 1],\n                            };\n\n                            acc[cp]     = vmlaq_s32(acc[cp], sb_acc_0l, scale_vec_l);\n                            acc[cp]     = vmlaq_s32(acc[cp], sb_acc_0h, scale_vec_h);\n                            acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc_1l, scale_vec_l);\n                            acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc_1h, scale_vec_h);\n                        }\n                    }\n                }  // for half\n\n                // Reorder i8mm output to match memory layout\n                for (int i = 0; i < 8; i++) {\n                    int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i]));\n                    acc[i]          = vcombine_s32(aux.val[0], aux.val[1]);\n                }\n                int32x4_t reorder_acc[8] = {\n                    vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])),\n                    vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])),\n                    vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])),\n                    vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])),\n                    vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])),\n                    vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])),\n                    vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])),\n                    vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])),\n                };\n\n                // Apply superblock scale (no mins for q6_K)\n                for (int i = 0; i < q8_k_blocklen; i++) {\n                    for (int j = 0; j < 2; j++) {\n                        float32x4_t       q8_d  = vdupq_n_f32(q8_ptr[b].d[i]);\n                        float32x4_t       q6_d  = vcvt_f32_f16(vld1_f16((const __fp16 *) (q6_ptr[b].d + j * 4)));\n                        const float32x4_t scale = vmulq_f32(q6_d, q8_d);\n\n                        acc_f32[2 * i + j] =\n                            vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale);\n                    }\n                }\n            }  // for b\n\n            // Store results\n            for (int i = 0; i < q8_k_blocklen; i++) {\n                int row = y * q8_k_blocklen + i;\n                for (int j = 0; j < 2; j++) {\n                    int col    = x * ncols_interleaved + j * 4;\n                    int offset = row * bs + col;\n                    vst1q_f32(s + offset, acc_f32[2 * i + j]);\n                }\n            }\n        }  // for x\n    }  // for y\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    ggml_gemm_q6_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_4x4_q8_0(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n            float32x4_t sumf[4];\n            for (int m = 0; m < 4; m++) {\n                sumf[m] = vdupq_n_f32(0);\n            }\n\n            for (int l = 0; l < nb; l++) {\n                float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *) a_ptr[l].d));\n                float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *) b_ptr[l].d));\n\n                int32x4_t sumi_0 = vdupq_n_s32(0);\n                int32x4_t sumi_1 = vdupq_n_s32(0);\n                int32x4_t sumi_2 = vdupq_n_s32(0);\n                int32x4_t sumi_3 = vdupq_n_s32(0);\n\n                for (int k_group = 0; k_group < 8; k_group += 4) {\n                    int8x16x4_t a = vld1q_s8_x4(a_ptr[l].qs + 16 * k_group);\n                    int8x16x4_t b = vld1q_s8_x4(b_ptr[l].qs + 16 * k_group);\n\n                    for (int k = 0; k < 4; k++) {\n                        sumi_0 = vdotq_laneq_s32(sumi_0, b.val[k], a.val[k], 0);\n                        sumi_1 = vdotq_laneq_s32(sumi_1, b.val[k], a.val[k], 1);\n                        sumi_2 = vdotq_laneq_s32(sumi_2, b.val[k], a.val[k], 2);\n                        sumi_3 = vdotq_laneq_s32(sumi_3, b.val[k], a.val[k], 3);\n                    }\n                }\n\n                sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));\n                sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));\n                sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));\n                sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));\n            }\n\n            for (int m = 0; m < 4; m++) {\n                vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);\n            }\n        }\n    }\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)\n    ggml_gemm_q8_0_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_4x8_q8_0(int                        n,\n                             float * GGML_RESTRICT      s,\n                             size_t                     bs,\n                             const void * GGML_RESTRICT vx,\n                             const void * GGML_RESTRICT vy,\n                             int                        nr,\n                             int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    const block_q8_0x4 * b_ptr_base = (const block_q8_0x4 *) vx;\n\n    for (int y = 0; y < nr; y += 4) {\n        const block_q8_0x4 * a_ptr_base = (const block_q8_0x4 *) vy + (y / 4) * nb;\n\n        for (int x = 0; x < nc; x += ncols_interleaved) {\n            const block_q8_0x4 * b_ptr = b_ptr_base + (x / 4) * nb;\n            const block_q8_0x4 * a_ptr = a_ptr_base;\n\n            float32x4_t acc_f32[4];\n            for (int i = 0; i < 4; i++) {\n                acc_f32[i] = vdupq_n_f32(0);\n            }\n\n            for (int b = 0; b < nb; b++) {\n                int32x4_t acc[4];\n                for (int i = 0; i < 4; i++) {\n                    acc[i] = vdupq_n_s32(0);\n                }\n\n                // Process 4 chunks of 8 positions each\n                for (int chunk = 0; chunk < 4; chunk++) {\n                    int8x16_t a01 = vld1q_s8(a_ptr->qs + chunk * 32);\n                    int8x16_t a23 = vld1q_s8(a_ptr->qs + chunk * 32 + 16);\n                    int8x16_t b01 = vld1q_s8(b_ptr->qs + chunk * 32);\n                    int8x16_t b23 = vld1q_s8(b_ptr->qs + chunk * 32 + 16);\n\n                    acc[0] = vmmlaq_s32(acc[0], a01, b01);\n                    acc[1] = vmmlaq_s32(acc[1], a01, b23);\n                    acc[2] = vmmlaq_s32(acc[2], a23, b01);\n                    acc[3] = vmmlaq_s32(acc[3], a23, b23);\n                }\n\n                // Reorder outputs from 2×2 tiles to row-major\n                // acc[0] = [r0c0, r0c1, r1c0, r1c1]\n                // acc[1] = [r0c2, r0c3, r1c2, r1c3]\n                // acc[2] = [r2c0, r2c1, r3c0, r3c1]\n                // acc[3] = [r2c2, r2c3, r3c2, r3c3]\n                int32x4_t row0 = vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1]));\n                int32x4_t row1 = vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1]));\n                int32x4_t row2 = vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3]));\n                int32x4_t row3 = vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3]));\n\n                // Scales\n                float32x4_t a_d = vcvt_f32_f16(vld1_f16((const __fp16 *) a_ptr->d));\n                float32x4_t b_d = vcvt_f32_f16(vld1_f16((const __fp16 *) b_ptr->d));\n\n                acc_f32[0] = vfmaq_f32(acc_f32[0], vcvtq_f32_s32(row0), vmulq_laneq_f32(b_d, a_d, 0));\n                acc_f32[1] = vfmaq_f32(acc_f32[1], vcvtq_f32_s32(row1), vmulq_laneq_f32(b_d, a_d, 1));\n                acc_f32[2] = vfmaq_f32(acc_f32[2], vcvtq_f32_s32(row2), vmulq_laneq_f32(b_d, a_d, 2));\n                acc_f32[3] = vfmaq_f32(acc_f32[3], vcvtq_f32_s32(row3), vmulq_laneq_f32(b_d, a_d, 3));\n\n                a_ptr++;\n                b_ptr++;\n            }\n\n            for (int row = 0; row < 4; row++) {\n                vst1q_f32(s + (y + row) * bs + x, acc_f32[row]);\n            }\n        }\n    }\n    return;\n#endif  // defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)\n    ggml_gemm_q8_0_4x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/loongarch/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__loongarch_sx)\n\nstatic __m128i lsx_packs_w(__m128i a, __m128i b) {\n    __m128i tmp, tmp1;\n    tmp = __lsx_vsat_w(a, 15);\n    tmp1 = __lsx_vsat_w(b, 15);\n    return __lsx_vpickev_h(tmp1, tmp);\n}\n\nstatic __m128i lsx_packs_h(__m128i a, __m128i b) {\n    __m128i tmp, tmp1;\n    tmp = __lsx_vsat_h(a, 7);\n    tmp1 = __lsx_vsat_h(b, 7);\n    return __lsx_vpickev_b(tmp1, tmp);\n}\n\nstatic __m128i lsx_packus_h(__m128i a, __m128i b) {\n    __m128i tmp, tmp1;\n    tmp = __lsx_vsat_hu(a, 7);\n    tmp1 = __lsx_vsat_hu(b, 7);\n    return __lsx_vpickev_b(tmp1, tmp);\n}\n\nstatic __m128i lsx_maddubs_h(__m128i a, __m128i b) {\n    __m128i tmp1, tmp2;\n    tmp1 = __lsx_vmulwev_h_b(a, b);\n    tmp2 = __lsx_vmulwod_h_b(a, b);\n    return __lsx_vsadd_h(tmp1, tmp2);\n}\n\nstatic __m128i lsx_madd_h(__m128i a, __m128i b) {\n    __m128i tmp1, tmp2;\n    tmp1 = __lsx_vmulwev_w_h(a, b);\n    tmp2 = __lsx_vmulwod_w_h(a, b);\n    return __lsx_vadd_w(tmp1, tmp2);\n}\n\nstatic __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) {\n    v4i32 __ret = {d, c, b, a};\n    return (__m128i)__ret;\n}\n\nstatic __m128i lsx_shuffle_b(__m128i a, __m128i b) {\n    __m128i mask_f, zero, tmp0, tmp2, mask;\n    int f = 0x8f;\n    mask_f = __lsx_vreplgr2vr_b(f);\n    zero = __lsx_vldi(0);\n    tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits\n    tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or  with 0x10 prepare for positive\n    mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask\n    tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones\n    return __lsx_vshuf_b(a, zero, tmp2);\n}\n\nstatic __m128i lsx_hadd_h(__m128i a, __m128i b) {\n    __m128i tmp1 = __lsx_vpickev_h(b, a);\n    __m128i tmp2 = __lsx_vpickod_h(b, a);\n    return __lsx_vadd_h(tmp1, tmp2);\n}\n\nstatic __m128i lsx_hadd_w(__m128i a, __m128i b) {\n    __m128i tmp1 = __lsx_vpickev_w(b, a);\n    __m128i tmp2 = __lsx_vpickod_w(b, a);\n    return __lsx_vadd_w(tmp1, tmp2);\n}\n\nstatic __m128 lsx_hadd_s(__m128 a, __m128 b) {\n    __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a);\n    __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a);\n\n    return __lsx_vfadd_s(tmp1, tmp2);\n}\n\nstatic inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {\n    __m128 res_0 =lsx_hadd_s(a, b);\n    __m128 res_1 =lsx_hadd_s(c, d);\n    __m128 res =lsx_hadd_s(res_0, res_1);\n    res =lsx_hadd_s(res, res);\n    res =lsx_hadd_s(res, res);\n\n    return ((v4f32)res)[0];\n}\n\n// multiply int8_t, add results pairwise twice\nstatic inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {\n    // Get absolute values of x vectors\n    const __m128i ax = __lsx_vsigncov_b(x, x);\n    // Sign the values of the y vectors\n    const __m128i sy = __lsx_vsigncov_b(x, y);\n    // Perform multiplication and create 16-bit values\n    const __m128i dot = lsx_maddubs_h(ax, sy);\n    const __m128i ones = __lsx_vreplgr2vr_h(1);\n    return lsx_madd_h(ones, dot);\n}\n#endif\n\n#if defined(__loongarch_asx)\n\n#ifdef __clang__\n#define VREGS_PREFIX \"$vr\"\n#define XREGS_PREFIX \"$xr\"\n#else // GCC\n#define VREGS_PREFIX \"$f\"\n#define XREGS_PREFIX \"$f\"\n#endif\n#define __ALL_REGS \"0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\"\n// Convert __m128i to __m256i\nstatic inline __m256i ____m256i(__m128i in) {\n    __m256i out = __lasx_xvldi(0);\n    __asm__ volatile (\n        \".irp i,\" __ALL_REGS                \"\\n\\t\"\n        \" .ifc %[out], \" XREGS_PREFIX\"\\\\i    \\n\\t\"\n        \"  .irp j,\" __ALL_REGS              \"\\n\\t\"\n        \"   .ifc %[in], \" VREGS_PREFIX \"\\\\j  \\n\\t\"\n        \"    xvpermi.q $xr\\\\i, $xr\\\\j, 0x20  \\n\\t\"\n        \"   .endif                           \\n\\t\"\n        \"  .endr                             \\n\\t\"\n        \" .endif                             \\n\\t\"\n        \".endr                               \\n\\t\"\n        : [out] \"+f\" (out) : [in] \"f\" (in)\n    );\n    return out;\n}\n// Convert two __m128i to __m256i\nstatic inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) {\n    __m256i out;\n    __asm__ volatile (\n        \".irp i,\" __ALL_REGS                \"\\n\\t\"\n        \" .ifc %[hi], \" VREGS_PREFIX \"\\\\i    \\n\\t\"\n        \"  .irp j,\" __ALL_REGS              \"\\n\\t\"\n        \"   .ifc %[lo], \" VREGS_PREFIX \"\\\\j  \\n\\t\"\n        \"    xvpermi.q $xr\\\\i, $xr\\\\j, 0x20  \\n\\t\"\n        \"   .endif                           \\n\\t\"\n        \"  .endr                             \\n\\t\"\n        \" .endif                             \\n\\t\"\n        \".endr                               \\n\\t\"\n        \".ifnc %[out], %[hi]                 \\n\\t\"\n        \".irp i,\" __ALL_REGS                \"\\n\\t\"\n        \" .ifc %[out], \" XREGS_PREFIX \"\\\\i   \\n\\t\"\n        \"  .irp j,\" __ALL_REGS              \"\\n\\t\"\n        \"   .ifc %[hi], \" VREGS_PREFIX \"\\\\j  \\n\\t\"\n        \"    xvori.b $xr\\\\i, $xr\\\\j, 0       \\n\\t\"\n        \"   .endif                           \\n\\t\"\n        \"  .endr                             \\n\\t\"\n        \" .endif                             \\n\\t\"\n        \".endr                               \\n\\t\"\n        \".endif                              \\n\\t\"\n        : [out] \"=f\" (out), [hi] \"+f\" (inhi)\n        : [lo] \"f\" (inlo)\n    );\n    return out;\n}\n// Convert __m256i low part to __m128i\nstatic inline __m128i lasx_extracti128_lo(__m256i in) {\n    __m128i out;\n    __asm__ volatile (\n        \".ifnc %[out], %[in]                 \\n\\t\"\n        \".irp i,\" __ALL_REGS                \"\\n\\t\"\n        \" .ifc %[out], \" VREGS_PREFIX \"\\\\i   \\n\\t\"\n        \"  .irp j,\" __ALL_REGS              \"\\n\\t\"\n        \"   .ifc %[in], \" XREGS_PREFIX \"\\\\j  \\n\\t\"\n        \"    vori.b $vr\\\\i, $vr\\\\j, 0        \\n\\t\"\n        \"   .endif                           \\n\\t\"\n        \"  .endr                             \\n\\t\"\n        \" .endif                             \\n\\t\"\n        \".endr                               \\n\\t\"\n        \".endif                              \\n\\t\"\n        : [out] \"=f\" (out) : [in] \"f\" (in)\n    );\n    return out;\n}\n// Convert __m256i high part to __m128i\nstatic inline __m128i lasx_extracti128_hi(__m256i in) {\n    __m128i out;\n    __asm__ volatile (\n        \".irp i,\" __ALL_REGS                \"\\n\\t\"\n        \" .ifc %[out], \" VREGS_PREFIX \"\\\\i   \\n\\t\"\n        \"  .irp j,\" __ALL_REGS              \"\\n\\t\"\n        \"   .ifc %[in], \" XREGS_PREFIX \"\\\\j  \\n\\t\"\n        \"    xvpermi.q $xr\\\\i, $xr\\\\j, 0x11  \\n\\t\"\n        \"   .endif                           \\n\\t\"\n        \"  .endr                             \\n\\t\"\n        \" .endif                             \\n\\t\"\n        \".endr                               \\n\\t\"\n        : [out] \"=f\" (out) : [in] \"f\" (in)\n    );\n    return out;\n}\n\nstatic __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) {\n    v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7};\n    return (__m256i)__ret;\n}\n\nstatic __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) {\n    v4i64 __ret = {d, c, b, a};\n    return (__m256i)__ret;\n}\n\nstatic __m256i lasx_insertf128( __m128i x, __m128i y) {\n    return lasx_set_q(x, y);\n}\n\nstatic __m256i lasx_shuffle_b(__m256i a, __m256i b) {\n    __m256i mask_f, zero, tmp0, tmp2, mask;\n    int f = 0x8f;\n    mask_f = __lasx_xvreplgr2vr_b(f);\n    zero = __lasx_xvldi(0);\n    tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits\n    tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or  with 0x10 prepare for positive\n    mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask\n    tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones\n    return __lasx_xvshuf_b(a, zero, tmp2);\n}\n\nstatic __m256i lasx_extu8_16(__m128i a) {\n    return __lasx_vext2xv_hu_bu(____m256i(a));\n}\n\nstatic __m256i lasx_ext8_16(__m128i a) {\n    return __lasx_vext2xv_h_b(____m256i(a));\n}\n\nstatic __m256i lasx_ext16_32(__m128i a) {\n    return __lasx_vext2xv_w_h(____m256i(a));\n}\n\nstatic __m128i lasx_extracti128( __m256i a, int pos) {\n    __m128i ret;\n    if( pos == 0)\n    {\n       ret = lasx_extracti128_lo(a);\n    } else {\n       ret = lasx_extracti128_hi(a);\n    }\n    return ret;\n}\n\nstatic __m128 lasx_extractf128( __m256 a, int pos) {\n    __m128 ret;\n    if( pos == 0)\n    {\n       ret = (__m128)lasx_extracti128_lo((__m256i)a);\n    } else {\n       ret = (__m128)lasx_extracti128_hi((__m256i)a);\n    }\n    return ret;\n}\n\nstatic __m256i lasx_maddubs_h(__m256i a, __m256i b) {\n    __m256i tmp1, tmp2;\n    tmp1 = __lasx_xvmulwev_h_b(a, b);\n    tmp2 = __lasx_xvmulwod_h_b(a, b);\n    return __lasx_xvsadd_h(tmp1, tmp2);\n}\n\nstatic __m256i lasx_madd_h(__m256i a, __m256i b) {\n    __m256i tmp1, tmp2;\n    tmp1 = __lasx_xvmulwev_w_h(a, b);\n    tmp2 = __lasx_xvmulwod_w_h(a, b);\n    return __lasx_xvadd_w(tmp1, tmp2);\n}\n\nstatic __m256i lasx_packs_w(__m256i a, __m256i b) {\n    __m256i tmp, tmp1;\n    tmp = __lasx_xvsat_w(a, 15);\n    tmp1 = __lasx_xvsat_w(b, 15);\n    return __lasx_xvpickev_h(tmp1, tmp);\n}\n\nstatic __m256i lasx_packs_h(__m256i a, __m256i b) {\n    __m256i tmp, tmp1;\n    tmp = __lasx_xvsat_h(a, 7);\n    tmp1 = __lasx_xvsat_h(b, 7);\n    return __lasx_xvpickev_b(tmp1, tmp);\n}\n\nstatic inline __m256i lasx_madd_h_b(__m256i a, __m256i b) {\n    __m256i tmp1, tmp2;\n    tmp1 = __lasx_xvmulwev_h_b(a, b);\n    tmp2 = __lasx_xvmulwod_h_b(a, b);\n    return __lasx_xvadd_h(tmp1, tmp2);\n}\n\nstatic inline __m256i lasx_xvrepl128vei_h(__m256i a, const unsigned int b) {\n    switch (b) {\n        case 0: return __lasx_xvrepl128vei_h(a, 0);\n        case 1: return __lasx_xvrepl128vei_h(a, 1);\n        case 2: return __lasx_xvrepl128vei_h(a, 2);\n        case 3: return __lasx_xvrepl128vei_h(a, 3);\n        case 4: return __lasx_xvrepl128vei_h(a, 4);\n        case 5: return __lasx_xvrepl128vei_h(a, 5);\n        case 6: return __lasx_xvrepl128vei_h(a, 6);\n        case 7: return __lasx_xvrepl128vei_h(a, 7);\n        default: __builtin_unreachable();\n    }\n}\n\nstatic inline __m256i lasx_xvandi_b_bit(__m256i a, const unsigned int b) {\n    switch (b) {\n        case 0: return __lasx_xvandi_b(a, 1 << 0);\n        case 1: return __lasx_xvandi_b(a, 1 << 1);\n        case 2: return __lasx_xvandi_b(a, 1 << 2);\n        case 3: return __lasx_xvandi_b(a, 1 << 3);\n        case 4: return __lasx_xvandi_b(a, 1 << 4);\n        case 5: return __lasx_xvandi_b(a, 1 << 5);\n        case 6: return __lasx_xvandi_b(a, 1 << 6);\n        case 7: return __lasx_xvandi_b(a, 1 << 7);\n        default: __builtin_unreachable();\n    }\n}\n\n// horizontally add 8 floats\nstatic inline float hsum_float_8(const __m256 x) {\n    __m128 res = lasx_extractf128(x, 1);\n    res = __lsx_vfadd_s(res, lasx_extractf128(x, 0));\n    res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res));\n    res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0));\n    return ((v4f32)res)[0];\n}\n\n// horizontally add 8 int32_t\nstatic inline int hsum_i32_8(const __m256i a) {\n\n    __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11);\n    __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00);\n\n    __m128i  tmp1_128 = lasx_extracti128_lo(tmp1);\n    __m128i  tmp2_128 = lasx_extracti128_lo(tmp2);\n\n    __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128);\n\n    __m128i ev = __lsx_vpickev_w(sum128, sum128);\n    __m128i od = __lsx_vpickod_w(sum128, sum128);\n    __m128i sum64 = __lsx_vadd_w(ev, od);\n\n    int sum64_1, sum64_2;\n    sum64_1 = __lsx_vpickve2gr_w(sum64, 0);\n    sum64_2 = __lsx_vpickve2gr_w(sum64, 1);\n\n    return  sum64_1 + sum64_2;\n}\n\n// horizontally add 4 int32_t\nstatic inline int hsum_i32_4(const __m128i a) {\n    __m128i ev = __lsx_vpickev_w(a, a);\n    __m128i od = __lsx_vpickod_w(a, a);\n    __m128i sum64 = __lsx_vadd_w(ev, od);\n\n    int sum64_1, sum64_2;\n    sum64_1 = __lsx_vpickve2gr_w(sum64, 0);\n    sum64_2 = __lsx_vpickve2gr_w(sum64, 1);\n\n    return  sum64_1 + sum64_2;\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n\n    uint32_t x32;\n    memcpy(&x32, x, sizeof(uint32_t));\n    const __m256i shuf_mask = lasx_set_d(\n            0x0303030303030303, 0x0202020202020202,\n            0x0101010101010101, 0x0000000000000000);\n\n    __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask);\n    const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe);\n    bytes = __lasx_xvor_v(bytes, bit_mask);\n    return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1));\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) {\n    const __m128i lo = __lsx_vld((const __m128i *)rsi, 0);\n    __m128i hi = __lsx_vsrli_h(lo, 4);\n    return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m256i x) {\n    __m256i v = __lasx_xvpackod_h(x, x);\n    __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v);\n    return __lasx_xvffint_s_w(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n    // Perform multiplication and create 16-bit values\n    const __m256i dot = lasx_maddubs_h(ax, sy);\n    return sum_i16_pairs_float(dot);\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n    const __m256i dot = lasx_madd_h_b(x, y);\n    return sum_i16_pairs_float(dot);\n}\n\nstatic inline __m128i packNibbles( __m256i bytes ) {\n    // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n    const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF);\n     __m256i high = __lasx_xvandn_v(lowByte, bytes);\n    __m256i low = __lasx_xvand_v(lowByte, bytes);\n    high = __lasx_xvsrli_h(high, 4);\n    bytes = __lasx_xvor_v(low, high);\n    // Compress uint16_t lanes into bytes\n    __m128i *r0 = (__m128i *)&bytes;\n    __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11);\n    __m128i *r1 = (__m128i *)&tmp_h128;\n\n    __m128i zero = __lsx_vldi(0);\n    __m128i tmp, tmp2, tmp3;\n\n    tmp = __lsx_vmax_h(zero, *r0);\n    tmp2 = __lsx_vsat_hu(tmp, 7);\n\n    tmp = __lsx_vmax_h(zero, *r1);\n    tmp3 = __lsx_vsat_hu(tmp, 7);\n    return  __lsx_vpickev_b(tmp3, tmp2);\n}\n#endif  //__loongarch_asx\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n    for (int i = 0; i < nb; i++) {\n        __m256 v0 = (__m256)__lasx_xvld( x , 0);\n        __m256 v1 = (__m256)__lasx_xvld( x , 32);\n        __m256 v2 = (__m256)__lasx_xvld( x , 64);\n        __m256 v3 = (__m256)__lasx_xvld( x , 96);\n        x += 32;\n\n        // Compute max(abs(e)) for the block\n        const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );\n        __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );\n\n        __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) );\n        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );\n        __m128 tmp = max4;\n        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 ));\n        const float max_scalar = ((v4f32)max4)[0];\n\n        // Quantize these floats\n        const float d = max_scalar / 127.f;\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n        const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n        const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id );\n\n        // Apply the multiplier\n        v0 = __lasx_xvfmul_s( v0, mul );\n        v1 = __lasx_xvfmul_s( v1, mul );\n        v2 = __lasx_xvfmul_s( v2, mul );\n        v3 = __lasx_xvfmul_s( v3, mul );\n\n        // Round to nearest integer\n        __m256i i0 = __lasx_xvftintrne_w_s( v0 );\n        __m256i i1 = __lasx_xvftintrne_w_s( v1 );\n        __m256i i2 = __lasx_xvftintrne_w_s( v2 );\n        __m256i i3 = __lasx_xvftintrne_w_s( v3 );\n\n        __m128i ni0 = lasx_extracti128( i0, 0 );\n        __m128i ni1 = lasx_extracti128( i0, 1);\n        __m128i ni2 = lasx_extracti128( i1, 0);\n        __m128i ni3 = lasx_extracti128( i1, 1);\n        __m128i ni4 = lasx_extracti128( i2, 0);\n        __m128i ni5 = lasx_extracti128( i2, 1);\n        __m128i ni6 = lasx_extracti128( i3, 0);\n        __m128i ni7 = lasx_extracti128( i3, 1);\n\n        // Convert int32 to int16\n        ni0 = lsx_packs_w( ni0, ni1 );\n        ni2 = lsx_packs_w( ni2, ni3 );\n        ni4 = lsx_packs_w( ni4, ni5 );\n        ni6 = lsx_packs_w( ni6, ni7 );\n        // Convert int16 to int8\n        ni0 = lsx_packs_h( ni0, ni2 );\n        ni4 = lsx_packs_h( ni4, ni6 );\n\n        __lsx_vst(ni0, (__m128i *)(y[i].qs +  0), 0);\n        __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);\n\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n    for (int i = 0; i < nb; i++) {\n        __m256 v0 = (__m256)__lasx_xvld( x , 0 );\n        __m256 v1 = (__m256)__lasx_xvld( x , 32 );\n        __m256 v2 = (__m256)__lasx_xvld( x , 64 );\n        __m256 v3 = (__m256)__lasx_xvld( x , 96 );\n        x += 32;\n\n        // Compute max(abs(e)) for the block\n        const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f );\n        __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) );\n        max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) );\n\n        __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) );\n        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) );\n        __m128 tmp = max4;\n        max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x1 ));\n        const float max_scalar = ((v4f32)max4)[0];\n\n        // Quantize these floats\n        const float d = max_scalar / 127.f;\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n        const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n        const __m256 mul = __lasx_xvreplfr2vr_s( id );\n\n        // Apply the multiplier\n        v0 = __lasx_xvfmul_s( v0, mul );\n        v1 = __lasx_xvfmul_s( v1, mul );\n        v2 = __lasx_xvfmul_s( v2, mul );\n        v3 = __lasx_xvfmul_s( v3, mul );\n\n        // Round to nearest integer\n        __m256i i0 = __lasx_xvftintrne_w_s( v0 );\n        __m256i i1 = __lasx_xvftintrne_w_s( v1 );\n        __m256i i2 = __lasx_xvftintrne_w_s( v2 );\n        __m256i i3 = __lasx_xvftintrne_w_s( v3 );\n\n        __m128i ni0 = lasx_extracti128(i0, 0);\n        __m128i ni1 = lasx_extracti128( i0, 1);\n        __m128i ni2 = lasx_extracti128( i1, 0);\n        __m128i ni3 = lasx_extracti128( i1, 1);\n        __m128i ni4 = lasx_extracti128( i2, 0 );\n        __m128i ni5 = lasx_extracti128( i2, 1);\n        __m128i ni6 = lasx_extracti128( i3, 0);\n        __m128i ni7 = lasx_extracti128( i3, 1);\n\n        // Compute the sum of the quants and set y[i].s\n        const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3));\n        const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7));\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1)));\n\n        // Convert int32 to int16\n        ni0 = lsx_packs_w( ni0, ni1 );\n        ni2 = lsx_packs_w( ni2, ni3 );\n        ni4 = lsx_packs_w( ni4, ni5 );\n        ni6 = lsx_packs_w( ni6, ni7 );\n        // Convert int16 to int8\n        ni0 = lsx_packs_h( ni0, ni2 );\n        ni4 = lsx_packs_h( ni4, ni6 );\n\n        __lsx_vst(ni0, (__m128i *)(y[i].qs +  0), 0);\n        __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0);\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\n//\n// Helper functions\n//\n\n#if defined(__loongarch_asx)\n// shuffles to pick the required scales in dot products\nstatic inline __m256i get_scale_shuffle_q3k(int i) {\n    static const uint8_t k_shuffle[128] = {\n         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,     2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,     6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,    10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,    14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,\n    };\n    return __lasx_xvld((const __m256i*)k_shuffle + i, 0);\n}\nstatic inline __m256i get_scale_shuffle_k4(int i) {\n    static const uint8_t k_shuffle[256] = {\n         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n         2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,\n         6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,\n        10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,\n        14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15\n    };\n    return __lasx_xvld((const __m256i*)k_shuffle + i, 0);\n}\nstatic inline __m128i get_scale_shuffle(int i) {\n    static const uint8_t k_shuffle[128] = {\n         0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n         2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,\n         4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,\n         6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,\n         8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,\n        10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,\n        12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,\n        14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15\n    };\n    return __lsx_vld((const __m128i*)k_shuffle + i, 0);\n}\n#endif\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__loongarch_asx)\n    // Initialize accumulator with zeros\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        /* Compute combined scale for the block */\n        const __m256 d = __lasx_xvreplfr2vr_s( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n\n        // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.\n        const __m256i off = __lasx_xvreplgr2vr_b( 8 );\n        qx = __lasx_xvsub_b( qx, off );\n\n        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        /* Multiply q with scale and accumulate */\n        acc = __lasx_xvfmadd_s( d, q, acc );\n    }\n\n    sumf = hsum_float_8(acc);\n\n#elif defined(__loongarch_sx)\n    // set constants\n    const __m128i low_mask = __lsx_vreplgr2vr_b(0xF);\n    const __m128i off = __lsx_vreplgr2vr_b(8);\n\n    // Initialize accumulator with zeros\n    __m128 acc_0 = (__m128)__lsx_vldi(0);\n    __m128 acc_1 = (__m128)__lsx_vldi(0);\n    __m128 acc_2 = (__m128)__lsx_vldi(0);\n    __m128 acc_3 = (__m128)__lsx_vldi(0);\n\n    for (; ib + 1 < nb; ib += 2) {\n\n        // Compute combined scale for the block 0 and 1\n        const float ft0 = GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d);\n        const __m128 d_0_1 = (__m128)(v4f32){ft0, ft0, ft0, ft0};\n\n        const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[ib].qs, 0);\n\n        __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1);\n        __m128i by_0 = __lsx_vld((const __m128i *)y[ib].qs, 0);\n        bx_0 = __lsx_vsub_b(bx_0, off);\n        const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);\n\n        __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4));\n        __m128i by_1 = __lsx_vld((const __m128i *)(y[ib].qs + 16), 0);\n        bx_1 = __lsx_vsub_b(bx_1, off);\n        const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);\n\n        // Compute combined scale for the block 2 and 3\n        const float ft1 = GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d);\n        const __m128 d_2_3 = (__m128)(v4f32){ft1, ft1, ft1, ft1};\n\n        const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[ib + 1].qs, 0);\n\n        __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3);\n        __m128i by_2 = __lsx_vld((const __m128i *)y[ib + 1].qs, 0);\n        bx_2 = __lsx_vsub_b(bx_2, off);\n        const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);\n\n        __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4));\n        __m128i by_3 = __lsx_vld((const __m128i *)(y[ib + 1].qs + 16), 0);\n        bx_3 = __lsx_vsub_b(bx_3, off);\n        const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);\n\n        // Convert int32_t to float\n        __m128 p0 = __lsx_vffint_s_w(i32_0);\n        __m128 p1 = __lsx_vffint_s_w(i32_1);\n        __m128 p2 = __lsx_vffint_s_w(i32_2);\n        __m128 p3 = __lsx_vffint_s_w(i32_3);\n\n        // Apply the scale\n        __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 );\n        __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 );\n        __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 );\n        __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 );\n\n        // Acummulate\n        acc_0 = __lsx_vfadd_s(p0_d, acc_0);\n        acc_1 = __lsx_vfadd_s(p1_d, acc_1);\n        acc_2 = __lsx_vfadd_s(p2_d, acc_2);\n        acc_3 = __lsx_vfadd_s(p3_d, acc_3);\n    }\n\n    sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);\n\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n            const int v1 = (x[ib].qs[j] >>   4) - 8;\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__loongarch_asx)\n    // Initialize accumulator with zeros\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    float summs = 0;\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);\n        const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        const __m256 d0v = __lasx_xvreplfr2vr_s( d0 );\n        const __m256 d1v = __lasx_xvreplfr2vr_s( d1 );\n\n        // Compute combined scales\n        const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v );\n\n        // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes\n        const __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        const __m256i qy = __lasx_xvld( (const __m256i *)y[ib].qs, 0);\n\n        const __m256 xy = mul_sum_us8_pairs_float(qx, qy);\n\n        // Accumulate d0*d1*x*y\n        acc = __lasx_xvfmadd_s( d0d1, xy, acc );\n    }\n\n    sumf = hsum_float_8(acc) + summs;\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n    // Initialize accumulator with zeros\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        /* Compute combined scale for the block */\n        const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)); //FIXME\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));\n        qx = __lasx_xvor_v(qx, bxhi);\n\n        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        /* Multiply q with scale and accumulate */\n        acc = __lasx_xvfmadd_s(d, q, acc);\n    }\n\n    sumf = hsum_float_8(acc);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__loongarch_asx)\n    // Initialize accumulator with zeros\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    float summs = 0.0f;\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        const __m256 dx = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10));\n        qx = __lasx_xvor_v(qx, bxhi);\n\n        const __m256 dy = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        const __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n        const __m256 q = mul_sum_us8_pairs_float(qx, qy);\n\n        acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc);\n    }\n\n    sumf = hsum_float_8(acc) + summs;\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__loongarch_asx)\n    // Initialize accumulator with zeros\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        // Compute combined scale for the block\n        const __m256 d = __lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n        __m256i qx = __lasx_xvld((const __m256i *)x[ib].qs, 0);\n        __m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        // Multiply q with scale and accumulate\n        acc = __lasx_xvfmadd_s( d, q, acc );\n    }\n\n    sumf = hsum_float_8(acc);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m128i mins_and_scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);\n        const __m128i scales128 = __lsx_vandi_b(mins_and_scales128, 0xf);\n        const __m256i mins = lasx_ext8_16(__lsx_vsrli_b(mins_and_scales128, 4));\n        const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0));\n\n        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc);\n\n        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n        __m256i sumi = __lasx_xvldi(0);\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32;\n\n            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n            const __m256i q2_0 = __lasx_xvandi_b(q2bits, 3);\n            const __m256i q2_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 2), 3);\n            const __m256i q2_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 4), 3);\n            const __m256i q2_3 = __lasx_xvsrli_b(q2bits, 6);\n\n            __m256i p0 = lasx_madd_h_b(q2_0, q8_0);\n            __m256i p1 = lasx_madd_h_b(q2_1, q8_1);\n            __m256i p2 = lasx_madd_h_b(q2_2, q8_2);\n            __m256i p3 = lasx_madd_h_b(q2_3, q8_3);\n\n            p0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p0);\n            p1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p1);\n            p2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p2);\n            p3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p3);\n\n            p0 = __lasx_xvadd_w(p0, p1);\n            p2 = __lasx_xvadd_w(p2, p3);\n\n            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2));\n        }\n\n        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n    const __m128i m32 = __lsx_vreplgr2vr_b(32);\n\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    uint32_t aux[3];\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        // Set up scales\n        memcpy(aux, x[i].scales, 12);\n        __m128i scales128 = lsx_set_w(\n                ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n                ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n                (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n                (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n        scales128 = __lsx_vsub_b(scales128, m32);\n\n        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n        // high bit\n        const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0);\n\n        // integer accumulator\n        __m256i sumi = __lasx_xvldi(0);\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            // load low 2 bits\n            const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32;\n\n            // prepare low and high bits\n            const __m256i q3l_0 = __lasx_xvandi_b(q3bits, 3);\n            const __m256i q3l_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 2), 3);\n            const __m256i q3l_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 4), 3);\n            const __m256i q3l_3 = __lasx_xvsrli_b(q3bits, 6);\n            const __m256i q3h_0 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 0), 0), 2);\n            const __m256i q3h_1 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 1), 0), 2);\n            const __m256i q3h_2 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 2), 0), 2);\n            const __m256i q3h_3 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 3), 0), 2);\n            const __m256i q3_0 = __lasx_xvor_v(q3h_0, q3l_0);\n            const __m256i q3_1 = __lasx_xvor_v(q3h_1, q3l_1);\n            const __m256i q3_2 = __lasx_xvor_v(q3h_2, q3l_2);\n            const __m256i q3_3 = __lasx_xvor_v(q3h_3, q3l_3);\n\n            // load Q8 quants\n            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n            __m256i p16_0 = lasx_madd_h_b(q8_0, q3_0);\n            __m256i p16_1 = lasx_madd_h_b(q8_1, q3_1);\n            __m256i p16_2 = lasx_madd_h_b(q8_2, q3_2);\n            __m256i p16_3 = lasx_madd_h_b(q8_3, q3_3);\n\n            // multiply with scales\n            p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);\n            p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);\n            p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);\n            p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);\n\n            // accumulate\n            p16_0 = __lasx_xvadd_w(p16_0, p16_1);\n            p16_2 = __lasx_xvadd_w(p16_2, p16_3);\n            sumi  = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2));\n        }\n        // multiply with block scale and accumulate\n        acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __loongarch_asx\n\n    __m256 acc = (__m256)__lasx_xvldi(0);\n    __m128 acc_m = (__m128)__lsx_vldi(0);\n\n   for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);\n        const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);\n        const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);\n\n        const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);\n        const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));\n        const __m128i prod = lsx_madd_h(mins128, q8s);\n        acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);\n\n        const __m256i scales = lasx_insertf128(scales128, scales128);\n\n        __m256i sumi = __lasx_xvldi(0);\n\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m256i scale_l = lasx_xvrepl128vei_h(scales, 2 * j + 0);\n            const __m256i scale_h = lasx_xvrepl128vei_h(scales, 2 * j + 1);\n\n            const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n            const __m256i q4l = __lasx_xvandi_b(q4bits, 0xf);\n            const __m256i q4h = __lasx_xvsrli_b(q4bits, 4);\n\n            const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            __m256i p16l = lasx_madd_h_b(q4l, q8l);\n            p16l = lasx_madd_h(scale_l, p16l);\n\n            const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            __m256i p16h = lasx_madd_h_b(q4h, q8h);\n            p16h = lasx_madd_h(scale_h, p16h);\n            const __m256i sumj = __lasx_xvadd_w(p16l, p16h);\n\n            sumi = __lasx_xvadd_w(sumi, sumj);\n        }\n\n        __m256 vd = __lasx_xvreplfr2vr_s(d);\n        acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);\n\n    }\n\n    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee));\n    __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0);\n    acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1);\n\n\n    *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __loongarch_asx\n\n    __m256 acc = (__m256)__lasx_xvldi(0);\n    __m128 acc_m = (__m128)__lsx_vldi(0);\n\n    for (int i = 0; i < nb; ++i) {\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]);\n        const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128);\n        const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0);\n\n        const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0);\n        const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1));\n        const __m128i prod = lsx_madd_h(mins128, q8s);\n        acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m);\n\n        const __m256i scales = lasx_insertf128(scales128, scales128);\n\n        const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0);\n\n        __m256i sumi = __lasx_xvldi(0);\n\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m256i scale_0 = lasx_xvrepl128vei_h(scales, 2 * j + 0);\n            const __m256i scale_1 = lasx_xvrepl128vei_h(scales, 2 * j + 1);\n\n            const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32;\n\n            const __m256i q5l_0 = __lasx_xvandi_b(q5bits, 0xf);\n            const __m256i q5l_1 = __lasx_xvsrli_b(q5bits, 4);\n            const __m256i q5h_0 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 0), 0), 0xef);\n            const __m256i q5h_1 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 1), 0), 0xef);\n            const __m256i q5_0  = __lasx_xvor_v(q5l_0, q5h_0);\n            const __m256i q5_1  = __lasx_xvor_v(q5l_1, q5h_1);\n\n            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n            __m256i p16_0 = lasx_madd_h_b(q5_0, q8_0);\n            __m256i p16_1 = lasx_madd_h_b(q5_1, q8_1);\n\n            p16_0 = lasx_madd_h(scale_0, p16_0);\n            p16_1 = lasx_madd_h(scale_1, p16_1);\n\n            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));\n\n        }\n\n        __m256 vd = __lasx_xvreplfr2vr_s(d);\n        acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc);\n\n    }\n\n    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 8));\n    acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 4));\n\n    *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __loongarch_asx\n\n    const __m256i m32s = __lasx_xvreplgr2vr_b(32);\n\n    __m256 acc = (__m256)__lasx_xvldi(0);\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m128i scales128 = __lsx_vld((const __m128i*)x[i].scales, 0);\n        const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15};\n        const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask));\n\n        __m256i sumi = __lasx_xvldi(0);\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n            const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32;\n            const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32;\n\n            const __m256i q4h_0 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3), 4);\n            const __m256i q4h_1 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3 << 2), 2);\n            const __m256i q4h_2 = __lasx_xvandi_b(q4bitsH, 3 << 4);\n            const __m256i q4h_3 = __lasx_xvsrli_b(__lasx_xvandi_b(q4bitsH, 3 << 6), 2);\n\n            const __m256i q4_0 = __lasx_xvor_v(__lasx_xvandi_b(q4bits1, 0xf), q4h_0);\n            const __m256i q4_1 = __lasx_xvor_v(__lasx_xvandi_b(q4bits2, 0xf), q4h_1);\n            const __m256i q4_2 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits1, 4), q4h_2);\n            const __m256i q4_3 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits2, 4), q4h_3);\n\n            const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n            __m256i p16_0 = lasx_madd_h_b(__lasx_xvsub_b(q4_0, m32s), q8_0);\n            __m256i p16_1 = lasx_madd_h_b(__lasx_xvsub_b(q4_1, m32s), q8_1);\n            __m256i p16_2 = lasx_madd_h_b(__lasx_xvsub_b(q4_2, m32s), q8_2);\n            __m256i p16_3 = lasx_madd_h_b(__lasx_xvsub_b(q4_3, m32s), q8_3);\n\n            p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0);\n            p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1);\n            p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2);\n            p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3);\n\n            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1));\n            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3));\n        }\n\n        acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__loongarch_asx)\nstatic const int8_t keven_signs_q2xs[1024] = {\n     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[4];\n    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n    __m256 accumf = (__m256)__lasx_xvldi(0);\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n\n            const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n            const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n            const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],\n                                                   signs64[(aux32[3] >>  7) & 127], signs64[(aux32[3] >>  0) & 127]);\n            const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);\n            const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);\n            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);\n            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);\n            const uint16_t ls1 = aux32[1] >> 28;\n            const uint16_t ls2 = aux32[3] >> 28;\n            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n            sumi1 = __lasx_xvadd_w(sumi1, p1);\n            sumi2 = __lasx_xvadd_w(sumi2, p2);\n        }\n\n        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n    const __m256i mone = __lasx_xvreplgr2vr_b(1);\n    static const char block_sign_shuffle_mask_1[32] = {\n        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n        0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n    };\n    static const char block_sign_shuffle_mask_2[32] = {\n        0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n        0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n    };\n    static const uint8_t bit_selector_mask_bytes[32] = {\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0);\n    const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0);\n    const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0);\n\n    static const uint8_t k_bit_helper[32] = {\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n    };\n    const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0);\n    const __m256i m511 = __lasx_xvreplgr2vr_h(511);\n    const __m128i m4 = __lsx_vreplgr2vr_b(0xf);\n    const __m128i m1 = __lsx_vreplgr2vr_b(1);\n\n    uint64_t aux64;\n\n    // somewhat hacky, but gives a significant boost in performance\n    __m256i aux_gindex;\n    const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n    __m256 accumf = (__m256)__lasx_xvldi(0);\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(&aux64, x[i].scales, 8);\n        __m128i stmp = __lsx_vreplgr2vr_d(aux64);\n        stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4));\n        const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1);\n\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n            const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0);  q2 += 16;\n            aux_gindex = __lasx_xvand_v(q2_data, m511);\n\n            const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9);\n            const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13);\n            const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper);\n\n            const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting);\n            const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits);\n\n            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n\n            const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],\n                                                   iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);\n            const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],\n                                                   iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);\n            const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],\n                                                   iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);\n            const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],\n                                                   iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n\n            const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0);\n            const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1);\n            const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l);\n            const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h);\n\n            __m256i signs;\n            signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1);\n            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1);\n\n            signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2);\n            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2);\n\n            signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1);\n            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3);\n\n            signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2);\n            signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4);\n\n            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);\n            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);\n            const __m256i dot3  = lasx_maddubs_h(q2_3, q8s_3);\n            const __m256i dot4  = lasx_maddubs_h(q2_4, q8s_4);\n\n            const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0)));\n            const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1)));\n            const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2)));\n            const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3)));\n\n            sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1));\n            sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2));\n            sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3));\n            sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4));\n        }\n\n        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n\n    const __m128i m4 = __lsx_vreplgr2vr_b(0xf);\n    const __m128i m1 = __lsx_vreplgr2vr_b(1);\n\n    const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);\n    const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);\n    uint64_t aux64;\n\n    __m256 accumf = (__m256)__lasx_xvldi(0);\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        __m128i tmp1;\n        memcpy(&aux64, x[i].scales, 8);\n        tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0);\n        tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1);\n        const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1);\n        const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15\n\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n                                                   iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],\n                                                   iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n                                                   iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n            const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n                                                   iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],\n                                                   iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n                                                   iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n            qs += 8;\n\n            __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16));\n            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n            const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);\n            const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);\n\n            aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16));\n            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n            const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);\n            const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);\n\n            signs += 4;\n\n            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1\n            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3\n\n            const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0)));\n            const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1)));\n            sumi1 = __lasx_xvadd_w(sumi1, p1);\n            sumi2 = __lasx_xvadd_w(sumi2, p2);\n        }\n\n        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[2];\n\n    __m256 accumf = (__m256)__lasx_xvldi(0);\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n                                                iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            q3 += 8;\n            const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n                                                iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            q3 += 8;\n            memcpy(aux32, gas, 8); gas += 8;\n\n            const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],\n                                                   signs64[(aux32[0] >>  7) & 127], signs64[(aux32[0] >>  0) & 127]);\n            const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1);\n            const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2);\n            const __m256i dot1  = lasx_maddubs_h(q2_1, q8s_1);\n            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);\n            const uint16_t ls1 = aux32[0] >> 28;\n            const uint16_t ls2 = aux32[1] >> 28;\n\n            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n            sumi1 = __lasx_xvadd_w(sumi1, p1);\n            sumi2 = __lasx_xvadd_w(sumi2, p2);\n        }\n\n        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n    }\n\n    *s = 0.25f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0);\n    const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0);\n\n    __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8);\n    const __m256i idx_mask  = __lasx_xvreplgr2vr_w(256);\n\n    typedef union {\n        __m256i  vec[2];\n        uint32_t index[16];\n    } index_t;\n\n    index_t idx;\n\n    __m256 accumf = (__m256)__lasx_xvldi(0);\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16;\n            idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]);\n            idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]);\n            idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask);\n            idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask);\n            idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0)));\n            idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1)));\n\n            // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.\n            //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);\n            //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);\n            const __m256i q2_1 = lasx_set_w(\n                    iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],\n                    iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]\n            );\n            const __m256i q2_2 = lasx_set_w(\n                    iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],\n                    iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]\n            );\n\n            __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16));\n            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n            const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2);\n            const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1);\n\n            aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16));\n            aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2);\n            const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2);\n            const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2);\n\n            signs += 4;\n\n            const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1);\n            const __m256i dot2  = lasx_maddubs_h(q2_2, q8s_2);\n            const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n            const uint16_t ls2 = x[i].scales[ib32/2] >>  4;\n            const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1));\n            const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1));\n            sumi1 = __lasx_xvadd_w(sumi1, p1);\n            sumi2 = __lasx_xvadd_w(sumi2, p2);\n        }\n\n        accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf);\n    }\n\n    *s = hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__loongarch_asx)\nstatic inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {\n    const __m256i a = __lasx_xvmulwev_h_b(x, y);\n    const __m256i b = __lasx_xvmulwod_h_b(x, y);\n    return __lasx_xvadd_h(a, b);\n}\n#endif\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n    __m256 accum = (__m256)__lasx_xvldi(0);\n    float accum1 = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint16_t * qh = x[i].qh;\n\n        __m256i sumi = __lasx_xvldi(0);\n        int sumi1 = 0;\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0);\n            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1);\n            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2);\n            q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3);\n\n            __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0);\n            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1);\n            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2);\n            q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3);\n\n            qs += 8;\n            const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n            const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32;\n\n            const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n            const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n            const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n            const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n\n            __m256i tmp1, tmp5, tmp6;\n            tmp1 = __lasx_xvreplgr2vr_h(ls1);\n            tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1);\n            tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1);\n            const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6);\n\n            tmp1 = __lasx_xvreplgr2vr_h(ls2);\n            tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1);\n            tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1);\n            const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6);\n\n            sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2));\n            sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n                   + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n        }\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum);\n        accum1 += d * sumi1;\n    }\n\n    *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined (__loongarch_asx)\n\n    const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);\n    const __m128i m4b  = __lsx_vreplgr2vr_b(0x0f);\n    const __m256i mone = __lasx_xvreplgr2vr_h(1);\n\n    __m256 accum1 = (__m256)__lasx_xvldi(0);\n    __m256 accum2 = (__m256)__lasx_xvldi(0);\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[ib + 0].qs, 0);\n        const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[ib + 1].qs, 0);\n        const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[ib + 0].qs, 0);\n        const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[ib + 1].qs, 0);\n        const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)),\n                                              lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b)));\n        const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)),\n                                              lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b)));\n        const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n        const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n        const __m256i p_1 = lasx_madd_h(p16_1, mone);\n        const __m256i p_2 = lasx_madd_h(p16_2, mone);\n        accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),\n                __lasx_xvffint_s_w(p_1), accum1);\n        accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),\n                __lasx_xvffint_s_w(p_2), accum2);\n    }\n\n    sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2));\n\n#endif\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n        int sumi1 = 0, sumi2 = 0;\n        for (int j = 0; j < QK4_NL/2; ++j) {\n            sumi1 += y[ib].qs[j+       0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__loongarch_asx)\n\n    const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0);\n\n    __m256 accum = (__m256)__lasx_xvldi(0);\n\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const uint8_t * qs = x[ibl].qs;\n        const int8_t  * q8 = y[ibl].qs;\n        uint16_t sh = x[ibl].scales_h;\n        __m256i sumi1 = __lasx_xvldi(0);\n        __m256i sumi2 = __lasx_xvldi(0);\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16;\n            const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16;\n            const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32;\n            const __m256i q4b_1 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_1, 4)),\n                                                  __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_1, 0xf)));\n            const __m256i q4b_2 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_2, 4)),\n                                                  __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_2, 0xf)));\n            const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n            const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n            const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n            const int16_t ls2 = ((x[ibl].scales_l[ib/2] >>  4) | ((sh << 2) & 0x30)) - 32;\n            sh >>= 4;\n            const __m256i p_1 = lasx_madd_h(p16_1, __lasx_xvreplgr2vr_h(ls1));\n            const __m256i p_2 = lasx_madd_h(p16_2, __lasx_xvreplgr2vr_h(ls2));\n            sumi1 = __lasx_xvadd_w(p_1, sumi1);\n            sumi2 = __lasx_xvadd_w(p_2, sumi2);\n        }\n        accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n                __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum);\n    }\n\n    *s = hsum_float_8(accum);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp",
    "content": "# include \"ggml-backend-impl.h\"\n\n#if defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)\n\n#if defined(__linux__)\n#include <sys/auxv.h>\n#endif\n\n#include <string>\n\nstruct powerpc_features {\n    std::string platform = \"\";\n    int power_version    = -1;\n\n    bool has_vsx         = false;\n\n    powerpc_features() {\n#if defined(__linux__)\n        unsigned long auxval = getauxval(AT_PLATFORM);\n        if (auxval) {\n            platform = std::string(reinterpret_cast<const char*>(auxval));\n            // TBD: Do systems exist that return this in uppercase?\n            if (platform.substr(0, 5) == \"power\") {\n                // Extractt a numeric suffix, if one exists\n                int vpos = -1;\n                for (int i = platform.length() - 1; i >= 0; i--) {\n                    if (std::isdigit(platform[i])) {\n                        vpos = i;\n                    } else {\n                        break;\n                    }\n                }\n                if (vpos > -1) {\n                    power_version = std::stoi(platform.substr(vpos));\n                }\n            }\n        }\n#endif\n        if (power_version >= 9) {\n            has_vsx = true;\n        }\n    }\n};\n\nstatic int ggml_backend_cpu_powerpc_score() {\n    int score = 1;\n    powerpc_features pf;\n\n// Platform scores\n#if defined(GGML_USE_POWER7)\n    if (pf.power_version < 7) { return 0; }\n    score += 1<<1;\n#endif\n#if defined(GGML_USE_POWER8)\n    if (pf.power_version < 8) { return 0; }\n    score += 1<<2;\n#endif\n#if defined(GGML_USE_POWER9)\n    if (pf.power_version < 9) { return 0; }\n    score += 1<<3;\n#endif\n#if defined(GGML_USE_POWER10)\n    if (pf.power_version < 10) { return 0; }\n    score += 1<<4;\n#endif\n#if defined(GGML_USE_POWER11)\n    if (pf.power_version < 11) { return 0; }\n    score += 1<<5;\n#endif\n\n// Feature scores\n#if defined(GGML_USE_VSX)\n    if (!pf.has_vsx) { return 0; }\n    score += 1<<6;\n#endif\n\n    return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_powerpc_score)\n\n#endif // defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/powerpc/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__POWER9_VECTOR__)\n#define B1(c,s,n)  0x ## n ## c ,  0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s  ) B7(c,s,     c), B7(c,s,     s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n    for (int i = 0; i < nb; i++) {\n        vector float srcv [8];\n        vector float asrcv[8];\n        vector float amaxv[8];\n        vector signed int vi[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = vec_xl(0, x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n                                   vec_extract(amaxv[0], 1)),\n                               MAX(vec_extract(amaxv[0], 2),\n                                   vec_extract(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n        const vector float vid = vec_splats(id);\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        for (int j = 0; j < 8; j++) {\n            const vector float v  = vec_round(vec_mul(srcv[j], vid));\n            vi[j] = vec_cts(v, 0);\n        }\n        vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])),  0, &y[i].qs[0]);\n        vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n    for (int i = 0; i < nb; i++) {\n        vector float srcv [8];\n        vector float asrcv[8];\n        vector float amaxv[8];\n        vector signed int vi[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = vec_xl(0, x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n                                   vec_extract(amaxv[0], 1)),\n                               MAX(vec_extract(amaxv[0], 2),\n                                   vec_extract(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n        const vector float vid = vec_splats(id);\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        vector int accv = vec_splats(0);\n\n        for (int j = 0; j < 8; j++) {\n            const vector float v  = vec_round(vec_mul(srcv[j], vid));\n            vi[j] = vec_cts(v, 0);\n\n            accv = vec_add(accv, vi[j]);\n        }\n        vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])),  0, &y[i].qs[0]);\n        vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]);\n\n        accv = vec_add(accv, vec_sld(accv, accv, 4));\n        accv = vec_add(accv, vec_sld(accv, accv, 8));\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * vec_extract(accv, 0));\n    }\n\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n    const vector signed char v8 = vec_splats((signed char)0x8);\n\n    vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n        vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n        vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n        vector signed char q4x0 = vec_and(qxs, lowMask);\n        vector signed char q4x1 = vec_sr(qxs, v4);\n\n        q4x0 = vec_sub(q4x0, v8);\n        q4x1 = vec_sub(q4x1, v8);\n\n        vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n        vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n        vector signed int vsumi0 = v0;\n\n        vsumi0 = vec_sum4s(qv0, vsumi0);\n        vsumi0 = vec_sum4s(qv1, vsumi0);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].m));\n        vector float vys = {GGML_CPU_FP16_TO_FP32(y[ib].s), 0.0f, 0.0f, 0.0f};\n        vsumf0 = vec_madd(vxmin, vys, vsumf0);\n\n        vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n        vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n        vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n        vector unsigned char q4x0 = (vector unsigned char)vec_and(qxs, lowMask);\n        vector unsigned char q4x1 = (vector unsigned char)vec_sr(qxs, v4);\n\n        vector signed int vsumi0 = v0;\n\n        vsumi0 = vec_msum(q8y0, q4x0, vsumi0);\n        vsumi0 = vec_msum(q8y1, q4x1, vsumi0);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector unsigned char vshift4 = vec_splats((unsigned char)4);\n    vector float vsumf0 = vec_splats(0.0f);\n\n    vector signed char kv = vec_xl(0, (const signed char *)kvalues_mxfp4);\n\n#pragma GCC unroll 8\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d) *\n                                      GGML_E8M0_TO_FP32_HALF(x[ib].e));\n\n        vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n        vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n        vector signed char qxs = (vector signed char)vec_xl(0, x[ib].qs);\n\n        vector unsigned char lo_nibbles = (vector unsigned char)vec_and(qxs, lowMask);\n        vector unsigned char hi_nibbles = (vector unsigned char)vec_sr(qxs, vshift4);\n\n        vector signed char q4x0 = vec_perm(kv, kv, lo_nibbles);\n        vector signed char q4x1 = vec_perm(kv, kv, hi_nibbles);\n\n        vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n        vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n        vector signed int vsumi0 = vec_splats((int32_t)0);\n        vsumi0 = vec_sum4s(qv0, vsumi0);\n        vsumi0 = vec_sum4s(qv1, vsumi0);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vyd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n    sumf = vec_extract(vsumf0, 0);\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector unsigned char v4 = vec_splats((unsigned char)4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[ib].qh[0]]), (uint64_t)(table_b2b_1[x[ib].qh[1]])};\n        vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[ib].qh[2]]), (uint64_t)(table_b2b_1[x[ib].qh[3]])};\n\n        vector signed char qh0 = (vector signed char)aux64x2_0;\n        vector signed char qh1 = (vector signed char)aux64x2_1;\n\n        vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n\n        vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0);\n        vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1);\n\n        vector signed char q8y0 = vec_xl(  0, y[ib].qs);\n        vector signed char q8y1 = vec_xl( 16, y[ib].qs);\n\n        vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));\n        vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));\n\n        qv0 = vec_add(qv0, qv1);\n\n        vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0));\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].m));\n        vector float vys = {GGML_CPU_FP16_TO_FP32(y[ib].s), 0.f, 0.f, 0.f};\n        vsumf0 = vec_madd(vxmin, vys, vsumf0);\n\n        vector unsigned long long aux64x2_0 = {(uint64_t)(table_b2b_0[x[ib].qh[0]]), (uint64_t)(table_b2b_0[x[ib].qh[1]])};\n        vector unsigned long long aux64x2_1 = {(uint64_t)(table_b2b_0[x[ib].qh[2]]), (uint64_t)(table_b2b_0[x[ib].qh[3]])};\n\n        vector signed char qh0 = (vector signed char)aux64x2_0;\n        vector signed char qh1 = (vector signed char)aux64x2_1;\n\n        vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n\n        vector unsigned char q5x0 = (vector unsigned char)vec_or(vec_and(qxs, lowMask), qh0);\n        vector unsigned char q5x1 = (vector unsigned char)vec_or(vec_sr(qxs, v4), qh1);\n\n        vector signed char q8y0 = vec_xl(  0, y[ib].qs);\n        vector signed char q8y1 = vec_xl( 16, y[ib].qs);\n\n        vector signed int vsumi0 = v0;\n\n        vsumi0 = vec_msum(q8y0, q5x0, vsumi0);\n        vsumi0 = vec_msum(q8y1, q5x1, vsumi0);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed int v0 = vec_splats((int32_t)0);\n    vector float vsumf0 = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed char q8x0 = vec_xl( 0, x[ib].qs);\n        vector signed char q8x1 = vec_xl(16, x[ib].qs);\n        vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n        vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n        vector signed short qv0 = vec_mule(q8x0, q8y0);\n        vector signed short qv1 = vec_mulo(q8x0, q8y0);\n        vector signed short qv2 = vec_mule(q8x1, q8y1);\n        vector signed short qv3 = vec_mulo(q8x1, q8y1);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n\n        vsumi0 = vec_sum4s(qv0, vsumi0);\n        vsumi1 = vec_sum4s(qv1, vsumi1);\n        vsumi0 = vec_sum4s(qv2, vsumi0);\n        vsumi1 = vec_sum4s(qv3, vsumi1);\n\n        vsumi0 = vec_add(vsumi0, vsumi1);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0x3);\n    const vector signed char lowScaleMask = vec_splats((signed char)0xF);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n    const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n        vector float vdmin = vec_mul(vxmin, vyd);\n\n        vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n        vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n        vector signed char q2xmins = (vector signed char)vec_xl( 0, x[i].scales);\n        vector signed char vscales = vec_and(q2xmins, lowScaleMask);\n\n        q2xmins = vec_sr(q2xmins, v4);\n        vector signed short q2xmins0 = vec_unpackh(q2xmins);\n        vector signed short q2xmins1 = vec_unpackl(q2xmins);\n\n        vector signed int prod0 = vec_mule(q2xmins0, q8ysums0);\n        vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0);\n        vector signed int prod2 = vec_mule(q2xmins1, q8ysums1);\n        vector signed int prod3 = vec_mulo(q2xmins1, q8ysums1);\n\n        vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n        vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n        vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n        vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n        vector signed int vsumi4 = v0;\n        vector signed int vsumi5 = v0;\n        vector signed int vsumi6 = v0;\n        vector signed int vsumi7 = v0;\n\n        const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            __builtin_prefetch(q2, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q2);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q2);\n            q2 += 32;\n\n            vector unsigned char q2x00 = (vector unsigned char)vec_and(qxs0, lowMask);\n            vector unsigned char q2x01 = (vector unsigned char)vec_and(vec_sr(qxs0, v2), lowMask);\n            vector unsigned char q2x02 = (vector unsigned char)vec_and(vec_sr(qxs0, v4), lowMask);\n            vector unsigned char q2x03 = (vector unsigned char)vec_and(vec_sr(qxs0, v6), lowMask);\n            vector unsigned char q2x10 = (vector unsigned char)vec_and(qxs1, lowMask);\n            vector unsigned char q2x11 = (vector unsigned char)vec_and(vec_sr(qxs1, v2), lowMask);\n            vector unsigned char q2x12 = (vector unsigned char)vec_and(vec_sr(qxs1, v4), lowMask);\n            vector unsigned char q2x13 = (vector unsigned char)vec_and(vec_sr(qxs1, v6), lowMask);\n\n            vector signed char q8y00 = vec_xl(  0, q8);\n            vector signed char q8y10 = vec_xl( 16, q8);\n            vector signed char q8y01 = vec_xl( 32, q8);\n            vector signed char q8y11 = vec_xl( 48, q8);\n            vector signed char q8y02 = vec_xl( 64, q8);\n            vector signed char q8y12 = vec_xl( 80, q8);\n            vector signed char q8y03 = vec_xl( 96, q8);\n            vector signed char q8y13 = vec_xl(112, q8);\n            q8 += 128;\n\n            vector signed int qv0 = vec_msum(q8y00, q2x00, v0);\n            vector signed int qv1 = vec_msum(q8y01, q2x01, v0);\n            vector signed int qv2 = vec_msum(q8y02, q2x02, v0);\n            vector signed int qv3 = vec_msum(q8y03, q2x03, v0);\n            vector signed int qv4 = vec_msum(q8y10, q2x10, v0);\n            vector signed int qv5 = vec_msum(q8y11, q2x11, v0);\n            vector signed int qv6 = vec_msum(q8y12, q2x12, v0);\n            vector signed int qv7 = vec_msum(q8y13, q2x13, v0);\n\n            vector signed short vscales_07 = vec_unpackh(vscales);\n            vector signed int vscales_03 = vec_unpackh(vscales_07);\n            vector signed int vscales_47 = vec_unpackl(vscales_07);\n            vector signed int vs0 = vec_splat(vscales_03, 0);\n            vector signed int vs1 = vec_splat(vscales_03, 1);\n            vector signed int vs2 = vec_splat(vscales_03, 2);\n            vector signed int vs3 = vec_splat(vscales_03, 3);\n            vector signed int vs4 = vec_splat(vscales_47, 0);\n            vector signed int vs5 = vec_splat(vscales_47, 1);\n            vector signed int vs6 = vec_splat(vscales_47, 2);\n            vector signed int vs7 = vec_splat(vscales_47, 3);\n            vscales = vec_sld(vscales, vscales, 8);\n\n            vsumi0 = vec_add(vec_mul(qv0, vs0), vsumi0);\n            vsumi1 = vec_add(vec_mul(qv1, vs2), vsumi1);\n            vsumi2 = vec_add(vec_mul(qv2, vs4), vsumi2);\n            vsumi3 = vec_add(vec_mul(qv3, vs6), vsumi3);\n            vsumi4 = vec_add(vec_mul(qv4, vs1), vsumi4);\n            vsumi5 = vec_add(vec_mul(qv5, vs3), vsumi5);\n            vsumi6 = vec_add(vec_mul(qv6, vs5), vsumi6);\n            vsumi7 = vec_add(vec_mul(qv7, vs7), vsumi7);\n        }\n\n        vsumi0 = vec_add(vsumi0, vsumi4);\n        vsumi1 = vec_add(vsumi1, vsumi5);\n        vsumi2 = vec_add(vsumi2, vsumi6);\n        vsumi3 = vec_add(vsumi3, vsumi7);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0x3);\n    const vector signed char lowMask1 = vec_splats((int8_t)0xf);\n    const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector signed char v1 = vec_splats((signed char)0x1);\n    const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n    const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n    const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n    const vector signed char off = vec_splats((signed char)0x20);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        UNUSED(kmask1);\n        UNUSED(kmask2);\n\n        vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n        vector signed char u1 = vec_and(u0, lowMask1);\n        vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n        vector signed char u3 = (vector signed char)vec_mergeh((vector signed int)u2, (vector signed int)vec_sr(u2, v2));\n        vector signed char u30 = vec_sl(vec_and(u3, lowMask), v4);\n        vector signed char u31 = vec_and(u3, lowMask2);\n\n        u1 = vec_or(u1, u30);\n        u2 = vec_or(vec_sr(u0, v4), u31);\n\n        vector signed char vscales = (vector signed char)vec_mergeh((vector signed long long)u1, (vector signed long long)u2);\n        vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].hmask);\n        vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].hmask);\n\n        vscales = vec_sub(vscales, off);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n        vector signed int vsumi4 = v0;\n        vector signed int vsumi5 = v0;\n        vector signed int vsumi6 = v0;\n        vector signed int vsumi7 = v0;\n\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            __builtin_prefetch(q3, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q3);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q3);\n            q3 += 32;\n\n            //the low 2 bits\n            vector signed char qxs00 = vec_and(qxs0, lowMask);\n            vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask);\n            vector signed char qxs02 = vec_and(vec_sr(qxs0, v4), lowMask);\n            vector signed char qxs03 = vec_and(vec_sr(qxs0, v6), lowMask);\n            vector signed char qxs10 = vec_and(qxs1, lowMask);\n            vector signed char qxs11 = vec_and(vec_sr(qxs1, v2), lowMask);\n            vector signed char qxs12 = vec_and(vec_sr(qxs1, v4), lowMask);\n            vector signed char qxs13 = vec_and(vec_sr(qxs1, v6), lowMask);\n\n            //the 3rd bit\n            vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2);\n            vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, (vector unsigned char)v1)), v2);\n            vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2);\n            vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v3)), v2);\n            vector signed char qxh10 = vec_sl(vec_andc(v1, qxhs1), v2);\n            vector signed char qxh11 = vec_sl(vec_andc(v1, vec_sr(qxhs1, (vector unsigned char)v1)), v2);\n            vector signed char qxh12 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v2)), v2);\n            vector signed char qxh13 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v3)), v2);\n            qxhs0 = vec_sr(qxhs0, v4);\n            qxhs1 = vec_sr(qxhs1, v4);\n\n            vector signed char q3x00 = vec_sub(qxs00, qxh00);\n            vector signed char q3x01 = vec_sub(qxs01, qxh01);\n            vector signed char q3x02 = vec_sub(qxs02, qxh02);\n            vector signed char q3x03 = vec_sub(qxs03, qxh03);\n            vector signed char q3x10 = vec_sub(qxs10, qxh10);\n            vector signed char q3x11 = vec_sub(qxs11, qxh11);\n            vector signed char q3x12 = vec_sub(qxs12, qxh12);\n            vector signed char q3x13 = vec_sub(qxs13, qxh13);\n\n            vector signed char q8y00 = vec_xl(  0, q8);\n            vector signed char q8y10 = vec_xl( 16, q8);\n            vector signed char q8y01 = vec_xl( 32, q8);\n            vector signed char q8y11 = vec_xl( 48, q8);\n            vector signed char q8y02 = vec_xl( 64, q8);\n            vector signed char q8y12 = vec_xl( 80, q8);\n            vector signed char q8y03 = vec_xl( 96, q8);\n            vector signed char q8y13 = vec_xl(112, q8);\n            q8 += 128;\n\n            vector signed short vscales_h = vec_unpackh(vscales);\n            vector signed short vs0 = vec_splat(vscales_h, 0);\n            vector signed short vs1 = vec_splat(vscales_h, 1);\n            vector signed short vs2 = vec_splat(vscales_h, 2);\n            vector signed short vs3 = vec_splat(vscales_h, 3);\n            vector signed short vs4 = vec_splat(vscales_h, 4);\n            vector signed short vs5 = vec_splat(vscales_h, 5);\n            vector signed short vs6 = vec_splat(vscales_h, 6);\n            vector signed short vs7 = vec_splat(vscales_h, 7);\n            vscales = vec_sld(vscales, vscales, 8);\n\n            vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00));\n            vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01));\n            vector signed short qv02 = vec_add(vec_mule(q3x02, q8y02), vec_mulo(q3x02, q8y02));\n            vector signed short qv03 = vec_add(vec_mule(q3x03, q8y03), vec_mulo(q3x03, q8y03));\n            vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10));\n            vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11));\n            vector signed short qv12 = vec_add(vec_mule(q3x12, q8y12), vec_mulo(q3x12, q8y12));\n            vector signed short qv13 = vec_add(vec_mule(q3x13, q8y13), vec_mulo(q3x13, q8y13));\n\n            vsumi0 = vec_msum(qv00, vs0, vsumi0);\n            vsumi1 = vec_msum(qv01, vs2, vsumi1);\n            vsumi2 = vec_msum(qv02, vs4, vsumi2);\n            vsumi3 = vec_msum(qv03, vs6, vsumi3);\n            vsumi4 = vec_msum(qv10, vs1, vsumi4);\n            vsumi5 = vec_msum(qv11, vs3, vsumi5);\n            vsumi6 = vec_msum(qv12, vs5, vsumi6);\n            vsumi7 = vec_msum(qv13, vs7, vsumi7);\n        }\n\n        vsumi0 = vec_add(vsumi0, vsumi4);\n        vsumi1 = vec_add(vsumi1, vsumi5);\n        vsumi2 = vec_add(vsumi2, vsumi6);\n        vsumi3 = vec_add(vsumi3, vsumi7);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed char lowMask1 = vec_splats((int8_t)0x3f);\n    const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v2 = vec_splats((uint8_t)2);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n        vector float vdmin = vec_mul(vxmin, vyd);\n\n        vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n        vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n        UNUSED(kmask1);\n        UNUSED(kmask2);\n        UNUSED(kmask3);\n        UNUSED(utmp);\n\n        vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n        vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);\n        vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n        vector signed char u3 = vec_sr(u2, v4);\n\n        vector signed char u30 = u1;\n        vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);\n\n        u1 = vec_and(u0, lowMask1);\n        u2 = vec_or(u30, u31);\n\n        vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);\n\n        vector signed short vscales = vec_unpackh(utmps);\n        vector signed short q4xmins = vec_unpackl(utmps);\n        vector signed short q4xmins0 = vec_mergeh(q4xmins, q4xmins);\n        vector signed short q4xmins1 = vec_mergel(q4xmins, q4xmins);\n\n        vector signed int prod0 = vec_mule(q4xmins0, q8ysums0);\n        vector signed int prod1 = vec_mule(q4xmins1, q8ysums1);\n        vector signed int prod2 = vec_mulo(q4xmins0, q8ysums0);\n        vector signed int prod3 = vec_mulo(q4xmins1, q8ysums1);\n\n        vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n        vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n        vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n        vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/64; j+=2) {\n            __builtin_prefetch(q4, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q4);\n            vector signed char qxs2 = (vector signed char)vec_xl(32, q4);\n            vector signed char qxs3 = (vector signed char)vec_xl(48, q4);\n            q4 += 64;\n\n            vector unsigned char q4x00 = (vector unsigned char)vec_and(qxs0, lowMask);\n            vector unsigned char q4x01 = (vector unsigned char)vec_sr(qxs0, v4);\n            vector unsigned char q4x10 = (vector unsigned char)vec_and(qxs1, lowMask);\n            vector unsigned char q4x11 = (vector unsigned char)vec_sr(qxs1, v4);\n            vector unsigned char q4x20 = (vector unsigned char)vec_and(qxs2, lowMask);\n            vector unsigned char q4x21 = (vector unsigned char)vec_sr(qxs2, v4);\n            vector unsigned char q4x30 = (vector unsigned char)vec_and(qxs3, lowMask);\n            vector unsigned char q4x31 = (vector unsigned char)vec_sr(qxs3, v4);\n\n            vector signed char q8y00 = vec_xl(  0, q8);\n            vector signed char q8y10 = vec_xl( 16, q8);\n            vector signed char q8y01 = vec_xl( 32, q8);\n            vector signed char q8y11 = vec_xl( 48, q8);\n            vector signed char q8y20 = vec_xl( 64, q8);\n            vector signed char q8y30 = vec_xl( 80, q8);\n            vector signed char q8y21 = vec_xl( 96, q8);\n            vector signed char q8y31 = vec_xl(112, q8);\n            q8 += 128;\n\n            vector signed int qv00 = vec_msum(q8y00, q4x00, v0);\n            vector signed int qv01 = vec_msum(q8y01, q4x01, v0);\n            vector signed int qv10 = vec_msum(q8y10, q4x10, v0);\n            vector signed int qv11 = vec_msum(q8y11, q4x11, v0);\n            vector signed int qv20 = vec_msum(q8y20, q4x20, v0);\n            vector signed int qv21 = vec_msum(q8y21, q4x21, v0);\n            vector signed int qv30 = vec_msum(q8y30, q4x30, v0);\n            vector signed int qv31 = vec_msum(q8y31, q4x31, v0);\n\n            vector signed int vscales_h = vec_unpackh(vscales);\n            vector signed int vs0 = vec_splat(vscales_h, 0);\n            vector signed int vs1 = vec_splat(vscales_h, 1);\n            vector signed int vs2 = vec_splat(vscales_h, 2);\n            vector signed int vs3 = vec_splat(vscales_h, 3);\n            vscales = vec_sld(vscales, vscales, 8);\n\n            vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);\n            vsumi1 = vec_add(vec_mul(qv01, vs1), vsumi1);\n            vsumi2 = vec_add(vec_mul(qv20, vs2), vsumi2);\n            vsumi3 = vec_add(vec_mul(qv21, vs3), vsumi3);\n\n            vsumi0 = vec_add(vec_mul(qv10, vs0), vsumi0);\n            vsumi1 = vec_add(vec_mul(qv11, vs1), vsumi1);\n            vsumi2 = vec_add(vec_mul(qv30, vs2), vsumi2);\n            vsumi3 = vec_add(vec_mul(qv31, vs3), vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed char lowMask1 = vec_splats((int8_t)0x3f);\n    const vector signed char lowMask2 = vec_splats((int8_t)0x30);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v1 = vec_splats((unsigned char)0x1);\n    const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n    const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector float vxmin = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].dmin));\n        vector float vdmin = vec_mul(vxmin, vyd);\n\n        UNUSED(kmask1);\n        UNUSED(kmask2);\n        UNUSED(kmask3);\n        UNUSED(utmp);\n\n        vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8);\n        vector signed char u1 = vec_and(vec_sr(u0, v2), lowMask2);\n        vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4);\n        vector signed char u3 = vec_sr(u2, v4);\n\n        vector signed char u30 = u1;\n        vector signed char u31 = (vector signed char)vec_mergeh((vector signed int)vec_and(u2, lowMask), (vector signed int)u3);\n\n        u1 = vec_and(u0, lowMask1);\n        u2 = vec_or(u30, u31);\n\n        vector signed char utmps = (vector signed char)vec_mergeh((vector signed int)u1, (vector signed int)u2);\n\n        vector signed short q8ysums0 = vec_xl( 0, y[i].bsums);\n        vector signed short q8ysums1 = vec_xl(16, y[i].bsums);\n\n        vector signed short vscales = vec_unpackh(utmps);\n\n        vector signed short q5xmins = vec_unpackl(utmps);\n        vector signed short q5xmins0 = vec_mergeh(q5xmins, q5xmins);\n        vector signed short q5xmins1 = vec_mergel(q5xmins, q5xmins);\n\n        vector signed int prod0 = vec_mule(q5xmins0, q8ysums0);\n        vector signed int prod1 = vec_mule(q5xmins1, q8ysums1);\n        vector signed int prod2 = vec_mulo(q5xmins0, q8ysums0);\n        vector signed int prod3 = vec_mulo(q5xmins1, q8ysums1);\n\n        vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0);\n        vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1);\n        vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2);\n        vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3);\n\n        vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh);\n        vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].qh);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            __builtin_prefetch(q5, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q5);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q5);\n            q5 += 32;\n\n            vector signed char qxs00 = vec_and(qxs0, lowMask);\n            vector signed char qxs01 = vec_sr(qxs0, v4);\n            vector signed char qxs10 = vec_and(qxs1, lowMask);\n            vector signed char qxs11 = vec_sr(qxs1, v4);\n\n            vector signed char q5h00 = vec_sl(vec_and((vector signed char)v1, qxhs0), v4);\n            vector signed char q5h01 = vec_sl(vec_and((vector signed char)v2, qxhs0), v3);\n            vector signed char q5h10 = vec_sl(vec_and((vector signed char)v1, qxhs1), v4);\n            vector signed char q5h11 = vec_sl(vec_and((vector signed char)v2, qxhs1), v3);\n            qxhs0 = vec_sr(qxhs0, v2);\n            qxhs1 = vec_sr(qxhs1, v2);\n\n            vector unsigned char q5x00 = (vector unsigned char)vec_or(q5h00, qxs00);\n            vector unsigned char q5x01 = (vector unsigned char)vec_or(q5h01, qxs01);\n            vector unsigned char q5x10 = (vector unsigned char)vec_or(q5h10, qxs10);\n            vector unsigned char q5x11 = (vector unsigned char)vec_or(q5h11, qxs11);\n\n            vector signed char q8y00 = vec_xl( 0, q8);\n            vector signed char q8y10 = vec_xl(16, q8);\n            vector signed char q8y01 = vec_xl(32, q8);\n            vector signed char q8y11 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed int qv00 = vec_msum(q8y00, q5x00, v0);\n            vector signed int qv01 = vec_msum(q8y01, q5x01, v0);\n            vector signed int qv10 = vec_msum(q8y10, q5x10, v0);\n            vector signed int qv11 = vec_msum(q8y11, q5x11, v0);\n\n            vector signed int vscales_h = vec_unpackh(vscales);\n            vector signed int vs0 = vec_splat(vscales_h, 0);\n            vector signed int vs1 = vec_splat(vscales_h, 1);\n            vscales = vec_sld(vscales, vscales, 12);\n\n            vsumi0 = vec_add(vec_mul(qv00, vs0), vsumi0);\n            vsumi1 = vec_add(vec_mul(qv10, vs0), vsumi1);\n            vsumi2 = vec_add(vec_mul(qv01, vs1), vsumi2);\n            vsumi3 = vec_add(vec_mul(qv11, vs1), vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v2 = vec_splats((unsigned char)0x2);\n    const vector unsigned char v3 = vec_splats((unsigned char)0x3);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n    const vector unsigned char v6 = vec_splats((unsigned char)0x6);\n    const vector signed char off = vec_splats((signed char)0x20);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n        vector signed int vsumi4 = v0;\n        vector signed int vsumi5 = v0;\n        vector signed int vsumi6 = v0;\n        vector signed int vsumi7 = v0;\n\n        const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT qs = x[i].scales;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            __builtin_prefetch(q6, 0, 0);\n            __builtin_prefetch(qh, 0, 0);\n            __builtin_prefetch(q8, 0, 0);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q6);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q6);\n            vector signed char qxs2 = (vector signed char)vec_xl(32, q6);\n            vector signed char qxs3 = (vector signed char)vec_xl(48, q6);\n            q6 += 64;\n\n            vector signed char qxs00 = vec_and(qxs0, lowMask);\n            vector signed char qxs01 = vec_sr(qxs0, v4);\n            vector signed char qxs10 = vec_and(qxs1, lowMask);\n            vector signed char qxs11 = vec_sr(qxs1, v4);\n            vector signed char qxs20 = vec_and(qxs2, lowMask);\n            vector signed char qxs21 = vec_sr(qxs2, v4);\n            vector signed char qxs30 = vec_and(qxs3, lowMask);\n            vector signed char qxs31 = vec_sr(qxs3, v4);\n\n            vector signed char qxhs0 = (vector signed char)vec_xl( 0, qh);\n            vector signed char qxhs1 = (vector signed char)vec_xl(16, qh);\n            qh += 32;\n\n            vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4);\n            vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4);\n            vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, qxhs1), v4);\n            vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v4)), v4);\n            vector signed char qxh20 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4);\n            vector signed char qxh21 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4);\n            vector signed char qxh30 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v2)), v4);\n            vector signed char qxh31 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v6)), v4);\n\n            vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off);\n            vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off);\n            vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off);\n            vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off);\n            vector signed char q6x20 = vec_sub(vec_or(qxh20, qxs20), off);\n            vector signed char q6x21 = vec_sub(vec_or(qxh21, qxs21), off);\n            vector signed char q6x30 = vec_sub(vec_or(qxh30, qxs30), off);\n            vector signed char q6x31 = vec_sub(vec_or(qxh31, qxs31), off);\n\n            vector signed char q8y00 = vec_xl(  0, q8);\n            vector signed char q8y10 = vec_xl( 16, q8);\n            vector signed char q8y20 = vec_xl( 32, q8);\n            vector signed char q8y30 = vec_xl( 48, q8);\n            vector signed char q8y01 = vec_xl( 64, q8);\n            vector signed char q8y11 = vec_xl( 80, q8);\n            vector signed char q8y21 = vec_xl( 96, q8);\n            vector signed char q8y31 = vec_xl(112, q8);\n            q8 += 128;\n\n            vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00));\n            vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10));\n            vector signed short qv20 = vec_add(vec_mule(q6x20, q8y20), vec_mulo(q6x20, q8y20));\n            vector signed short qv30 = vec_add(vec_mule(q6x30, q8y30), vec_mulo(q6x30, q8y30));\n            vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01));\n            vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11));\n            vector signed short qv21 = vec_add(vec_mule(q6x21, q8y21), vec_mulo(q6x21, q8y21));\n            vector signed short qv31 = vec_add(vec_mule(q6x31, q8y31), vec_mulo(q6x31, q8y31));\n\n            vector signed short vscales = vec_unpackh(vec_xl_len(qs, 8));\n            qs += 8;\n\n            vector signed short vs0 = vec_splat(vscales, 0);\n            vector signed short vs1 = vec_splat(vscales, 1);\n            vector signed short vs2 = vec_splat(vscales, 2);\n            vector signed short vs3 = vec_splat(vscales, 3);\n            vector signed short vs4 = vec_splat(vscales, 4);\n            vector signed short vs5 = vec_splat(vscales, 5);\n            vector signed short vs6 = vec_splat(vscales, 6);\n            vector signed short vs7 = vec_splat(vscales, 7);\n\n            vsumi0 = vec_msum(qv00, vs0, vsumi0);\n            vsumi1 = vec_msum(qv01, vs4, vsumi1);\n            vsumi2 = vec_msum(qv10, vs1, vsumi2);\n            vsumi3 = vec_msum(qv11, vs5, vsumi3);\n            vsumi4 = vec_msum(qv20, vs2, vsumi4);\n            vsumi5 = vec_msum(qv21, vs6, vsumi5);\n            vsumi6 = vec_msum(qv30, vs3, vsumi6);\n            vsumi7 = vec_msum(qv31, vs7, vsumi7);\n        }\n\n        vsumi0 = vec_add(vsumi0, vsumi4);\n        vsumi1 = vec_add(vsumi1, vsumi5);\n        vsumi2 = vec_add(vsumi2, vsumi6);\n        vsumi3 = vec_add(vsumi3, vsumi7);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__POWER9_VECTOR__)\nstatic const int8_t keven_signs_q2xs[1024] = {\n     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector int v0 = vec_splats((int32_t)0);\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  *  GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/32; j += 2) {\n            __builtin_prefetch(q2, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            uint32_t aux32[4];\n            const uint8_t * aux8 = (const uint8_t *)aux32;\n\n            memcpy(aux32, q2, 4*sizeof(uint32_t));\n            q2 += 8;\n\n            vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1])};\n            vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3])};\n            vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9])};\n            vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11])};\n\n            vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((aux32[1] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >>  7) & 127))};\n            vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127))};\n            vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((aux32[3] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >>  7) & 127))};\n            vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127))};\n\n            vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);\n            vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);\n            vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);\n            vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n            const uint16_t ls0 = aux32[1] >> 28;\n            const uint16_t ls1 = aux32[3] >> 28;\n\n            vector signed short vscales01 = vec_splats((int16_t)(2*ls0+1));\n            vector signed short vscales23 = vec_splats((int16_t)(2*ls1+1));\n\n            vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector int v0 = vec_splats((int32_t)0);\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const uint8_t  * GGML_RESTRICT sc = x[i].scales;\n        const int8_t  *  GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            __builtin_prefetch(q2, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xs_grid + (q2[0] & 511)), *(const int64_t *)(iq2xs_grid + (q2[1] & 511))};\n            vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xs_grid + (q2[2] & 511)), *(const int64_t *)(iq2xs_grid + (q2[3] & 511))};\n            vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xs_grid + (q2[4] & 511)), *(const int64_t *)(iq2xs_grid + (q2[5] & 511))};\n            vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xs_grid + (q2[6] & 511)), *(const int64_t *)(iq2xs_grid + (q2[7] & 511))};\n\n            vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((q2[0] >> 9))), *(const int64_t *)(signs64 + ((q2[1] >> 9)))};\n            vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((q2[2] >> 9))), *(const int64_t *)(signs64 + ((q2[3] >> 9)))};\n            vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((q2[4] >> 9))), *(const int64_t *)(signs64 + ((q2[5] >> 9)))};\n            vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((q2[6] >> 9))), *(const int64_t *)(signs64 + ((q2[7] >> 9)))};\n            q2 += 8;\n\n            vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0);\n            vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1);\n            vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2);\n            vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n            const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n            const uint16_t ls1 = (uint16_t)(sc[0] >>  4);\n            const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);\n            const uint16_t ls3 = (uint16_t)(sc[1] >>  4);\n            sc += 2;\n\n            vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));\n            vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));\n            vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));\n            vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));\n\n            vsumi0 = vec_msum(qv0, vscales0, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales1, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales2, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales3, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                        0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n    };\n\n    static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n    const vector int v0 = vec_splats((int32_t)0);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    const vector unsigned char mask0 = vec_xl( 0, k_mask1);\n    const vector unsigned char mask1 = vec_xl(16, k_mask1);\n    const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint8_t *  GGML_RESTRICT q2 = x[i].qs;\n        const uint8_t *  GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n        const uint8_t *  GGML_RESTRICT sc = x[i].scales;\n        const int8_t  *  GGML_RESTRICT q8 = y[i].qs;\n\n        for (int j = 0; j < QK_K/32; j += 2) {\n            __builtin_prefetch(q2, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed long long aux64x2_0 = {*(const int64_t *)(iq2s_grid + (q2[0] | ((qh[0] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[1] | ((qh[0] << 6) & 0x300)))};\n            vector signed long long aux64x2_1 = {*(const int64_t *)(iq2s_grid + (q2[2] | ((qh[0] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[3] | ((qh[0] << 2) & 0x300)))};\n            vector signed long long aux64x2_2 = {*(const int64_t *)(iq2s_grid + (q2[4] | ((qh[1] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[5] | ((qh[1] << 6) & 0x300)))};\n            vector signed long long aux64x2_3 = {*(const int64_t *)(iq2s_grid + (q2[6] | ((qh[1] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[7] | ((qh[1] << 2) & 0x300)))};\n            q2 += 8;\n            qh += 2;\n\n            vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);\n            vector signed char vsigns23 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);\n            signs += 4;\n\n            vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);\n            vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);\n            vector signed char vsigns2 = vec_perm(vsigns23, vsigns23, mask0);\n            vector signed char vsigns3 = vec_perm(vsigns23, vsigns23, mask1);\n\n            vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);\n            vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);\n            vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);\n            vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);\n\n            vector signed char q2x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux64x2_0), vsigns0);\n            vector signed char q2x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux64x2_1), vsigns1);\n            vector signed char q2x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux64x2_2), vsigns2);\n            vector signed char q2x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux64x2_3), vsigns3);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3));\n\n            const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n            const uint16_t ls1 = (uint16_t)(sc[0] >>  4);\n            const uint16_t ls2 = (uint16_t)(sc[1] & 0xf);\n            const uint16_t ls3 = (uint16_t)(sc[1] >>  4);\n            sc += 2;\n\n            vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1));\n            vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1));\n            vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1));\n            vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1));\n\n            vsumi0 = vec_msum(qv0, vscales0, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales1, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales2, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales3, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = 0.125f * vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    const vector int v0 = vec_splats((int32_t)0);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint32_t * GGML_RESTRICT signs = (const uint32_t *)(x[i].qs + QK_K/4);\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n#pragma GCC unroll 1\n        for (int j = 0; j < QK_K/32; j += 2) {\n            __builtin_prefetch(q3, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector unsigned int aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]};\n            vector unsigned int aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]};\n            vector unsigned int aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]};\n            vector unsigned int aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]};\n            q3 += 16;\n\n            vector unsigned long long aux64x2_0 = {(uint64_t)(signs64[(signs[0] >>  0) & 127]), (uint64_t)(signs64[(signs[0] >>  7) & 127])};\n            vector unsigned long long aux64x2_1 = {(uint64_t)(signs64[(signs[0] >> 14) & 127]), (uint64_t)(signs64[(signs[0] >> 21) & 127])};\n            vector unsigned long long aux64x2_2 = {(uint64_t)(signs64[(signs[1] >>  0) & 127]), (uint64_t)(signs64[(signs[1] >>  7) & 127])};\n            vector unsigned long long aux64x2_3 = {(uint64_t)(signs64[(signs[1] >> 14) & 127]), (uint64_t)(signs64[(signs[1] >> 21) & 127])};\n\n            vector signed char q3x0 = vec_mul((vector signed char)aux64x2_0, (vector signed char)aux32x4_0);\n            vector signed char q3x1 = vec_mul((vector signed char)aux64x2_1, (vector signed char)aux32x4_1);\n            vector signed char q3x2 = vec_mul((vector signed char)aux64x2_2, (vector signed char)aux32x4_2);\n            vector signed char q3x3 = vec_mul((vector signed char)aux64x2_3, (vector signed char)aux32x4_3);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));\n\n            const uint16_t ls0 = (uint16_t)(signs[0] >> 28);\n            const uint16_t ls1 = (uint16_t)(signs[1] >> 28);\n            signs += 2;\n\n            vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n            vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n\n            vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = 0.25f * vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                        0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n    };\n\n    static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,};\n\n    const vector int v0 = vec_splats((int32_t)0);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    const vector unsigned char mask0 = vec_xl( 0, k_mask1);\n    const vector unsigned char mask1 = vec_xl(16, k_mask1);\n    const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        const uint8_t *  GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t *  GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].signs);\n        const uint8_t *  GGML_RESTRICT sc = x[i].scales;\n        const int8_t  *  GGML_RESTRICT q8 = y[i].qs;\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        for (int j = 0; j < QK_K/32; j += 2) {\n            __builtin_prefetch(q3, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector unsigned int aux32x4_0 = {iq3s_grid[q3[ 0] | ((qh[0] << 8) & 256)], iq3s_grid[q3[ 1] | ((qh[0] << 7) & 256)],\n                                             iq3s_grid[q3[ 2] | ((qh[0] << 6) & 256)], iq3s_grid[q3[ 3] | ((qh[0] << 5) & 256)]};\n            vector unsigned int aux32x4_1 = {iq3s_grid[q3[ 4] | ((qh[0] << 4) & 256)], iq3s_grid[q3[ 5] | ((qh[0] << 3) & 256)],\n                                             iq3s_grid[q3[ 6] | ((qh[0] << 2) & 256)], iq3s_grid[q3[ 7] | ((qh[0] << 1) & 256)]};\n            vector unsigned int aux32x4_2 = {iq3s_grid[q3[ 8] | ((qh[1] << 8) & 256)], iq3s_grid[q3[ 9] | ((qh[1] << 7) & 256)],\n                                             iq3s_grid[q3[10] | ((qh[1] << 6) & 256)], iq3s_grid[q3[11] | ((qh[1] << 5) & 256)]};\n            vector unsigned int aux32x4_3 = {iq3s_grid[q3[12] | ((qh[1] << 4) & 256)], iq3s_grid[q3[13] | ((qh[1] << 3) & 256)],\n                                             iq3s_grid[q3[14] | ((qh[1] << 2) & 256)], iq3s_grid[q3[15] | ((qh[1] << 1) & 256)]};\n            q3 += 16;\n            qh += 2;\n\n            vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]);\n            vector signed char vsigns02 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]);\n            signs += 4;\n\n            vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0);\n            vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1);\n            vector signed char vsigns2 = vec_perm(vsigns02, vsigns02, mask0);\n            vector signed char vsigns3 = vec_perm(vsigns02, vsigns02, mask1);\n\n            vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2);\n            vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2);\n            vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2);\n            vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2);\n\n            vector signed char q3x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux32x4_0), vsigns0);\n            vector signed char q3x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux32x4_1), vsigns1);\n            vector signed char q3x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux32x4_2), vsigns2);\n            vector signed char q3x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux32x4_3), vsigns3);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3));\n\n            const uint16_t ls0 = (uint16_t)(sc[0] & 0xf);\n            const uint16_t ls1 = (uint16_t)(sc[0] >>  4);\n            sc ++;\n\n            vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n            vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n\n            vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector unsigned char v0 = vec_splats((unsigned char)0x0);\n    const vector unsigned short vsign = vec_splats((unsigned short)0x8000);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    for (int i = 0; i < nb; ++i) {\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[i].d));\n        vector float vyd = vec_splats(y[i].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = vec_splats((int32_t)0);\n        vector signed int vsumi1 = vec_splats((int32_t)0);\n        vector signed int vsumi2 = vec_splats((int32_t)0);\n        vector signed int vsumi3 = vec_splats((int32_t)0);\n        vector signed int vsumi8 = vec_splats((int32_t)0);\n\n        const uint8_t  * GGML_RESTRICT q1 = x[i].qs;\n        const uint16_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        const int16_t  * GGML_RESTRICT qs = y[i].bsums;\n\n        for (int j = 0; j < QK_K/32; j += 2) {\n            __builtin_prefetch(q1, 0, 1);\n            __builtin_prefetch(qh, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed long long aux64x2_0 = {*(const int64_t *)(iq1s_grid + (q1[0] | ((qh[0] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[1] | ((qh[0] << 5) & 0x700)))};\n            vector signed long long aux64x2_1 = {*(const int64_t *)(iq1s_grid + (q1[2] | ((qh[0] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[3] | ((qh[0] >> 1) & 0x700)))};\n            vector signed long long aux64x2_2 = {*(const int64_t *)(iq1s_grid + (q1[4] | ((qh[1] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[5] | ((qh[1] << 5) & 0x700)))};\n            vector signed long long aux64x2_3 = {*(const int64_t *)(iq1s_grid + (q1[6] | ((qh[1] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[7] | ((qh[1] >> 1) & 0x700)))};\n            q1 += 8;\n\n            vector signed char q1x0 = (vector signed char)aux64x2_0;\n            vector signed char q1x1 = (vector signed char)aux64x2_1;\n            vector signed char q1x2 = (vector signed char)aux64x2_2;\n            vector signed char q1x3 = (vector signed char)aux64x2_3;\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q1x0, q8y0), vec_mulo(q1x0, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q1x1, q8y1), vec_mulo(q1x1, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q1x2, q8y2), vec_mulo(q1x2, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q1x3, q8y3), vec_mulo(q1x3, q8y3));\n\n            const uint16_t ls0 = (uint16_t)((qh[0] >> 12) & 7);\n            const uint16_t ls1 = (uint16_t)((qh[1] >> 12) & 7);\n\n            vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1));\n            vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1));\n            vector signed short vscales = vec_sld(vscales23, vscales01, 8);\n\n            vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n\n            vector signed short q8ysums = vec_xl_len(qs, 8);\n            qs += 4;\n            q8ysums = vec_mergeh(q8ysums, (vector signed short)v0);\n\n            vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8);\n            qh += 2;\n            vector __bool short vsel = vec_cmpge(qxh, (vector signed short)v0);\n\n            vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel);\n\n            vsumi8 = vec_add(vec_mule(q8ysum, vscales), vsumi8);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi8, 0), vec_mul(vd, vec_splats(IQ1S_DELTA)), vsumf0);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector signed int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n\n    const vector signed char values = vec_xl( 0, kvalues_iq4nl);\n\n#pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d));\n        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);\n        vector signed char q4x0 = vec_and(qxs, lowMask);\n        vector signed char q4x1 = vec_sr(qxs, v4);\n\n        q4x0 = vec_perm(values, values, (vector unsigned char)q4x0);\n        q4x1 = vec_perm(values, values, (vector unsigned char)q4x1);\n\n        vector signed char q8y0 = vec_xl( 0, y[ib].qs);\n        vector signed char q8y1 = vec_xl(16, y[ib].qs);\n\n        vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));\n        vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n\n        vsumi0 = vec_sum4s(qv0, vsumi0);\n        vsumi1 = vec_sum4s(qv1, vsumi1);\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    sumf = vec_extract(vsumf0, 0);\n\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__POWER9_VECTOR__)\n    const vector signed char lowMask = vec_splats((signed char)0xF);\n    const vector int v0 = vec_splats((int32_t)0);\n    const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n\n    vector float vsumf0 = vec_splats(0.0f);\n    vector float vsumf1 = vec_splats(0.0f);\n    vector float vsumf2 = vec_splats(0.0f);\n    vector float vsumf3 = vec_splats(0.0f);\n\n    const vector signed char values = vec_xl( 0, kvalues_iq4nl);\n\n    for (int ibl = 0; ibl < nb; ++ibl) {\n\n        vector float vxd = vec_splats(GGML_CPU_FP16_TO_FP32(x[ibl].d));\n        vector float vyd = vec_splats(y[ibl].d);\n        vector float vd = vec_mul(vxd, vyd);\n\n        vector signed int vsumi0 = v0;\n        vector signed int vsumi1 = v0;\n        vector signed int vsumi2 = v0;\n        vector signed int vsumi3 = v0;\n\n        uint16_t h = x[ibl].scales_h;\n\n        const uint8_t * GGML_RESTRICT q4 = x[ibl].qs;\n        const uint8_t * GGML_RESTRICT sc = x[ibl].scales_l;\n        const int8_t  * GGML_RESTRICT q8 = y[ibl].qs;\n\n        for (int ib = 0; ib < QK_K/64; ib ++ ) {\n            __builtin_prefetch(q4, 0, 1);\n            __builtin_prefetch(q8, 0, 1);\n\n            vector signed char qxs0 = (vector signed char)vec_xl( 0, q4);\n            vector signed char qxs1 = (vector signed char)vec_xl(16, q4);\n            q4 += 32;\n\n            vector signed char q4x00 = (vector signed char)vec_and(qxs0, lowMask);\n            vector signed char q4x01 = (vector signed char)vec_sr(qxs0, v4);\n            vector signed char q4x10 = (vector signed char)vec_and(qxs1, lowMask);\n            vector signed char q4x11 = (vector signed char)vec_sr(qxs1, v4);\n\n            q4x00 = vec_perm(values, values, (vector unsigned char)q4x00);\n            q4x01 = vec_perm(values, values, (vector unsigned char)q4x01);\n            q4x10 = vec_perm(values, values, (vector unsigned char)q4x10);\n            q4x11 = vec_perm(values, values, (vector unsigned char)q4x11);\n\n            vector signed char q8y0 = vec_xl( 0, q8);\n            vector signed char q8y1 = vec_xl(16, q8);\n            vector signed char q8y2 = vec_xl(32, q8);\n            vector signed char q8y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            vector signed short qv0 = vec_add(vec_mule(q4x00, q8y0), vec_mulo(q4x00, q8y0));\n            vector signed short qv1 = vec_add(vec_mule(q4x01, q8y1), vec_mulo(q4x01, q8y1));\n            vector signed short qv2 = vec_add(vec_mule(q4x10, q8y2), vec_mulo(q4x10, q8y2));\n            vector signed short qv3 = vec_add(vec_mule(q4x11, q8y3), vec_mulo(q4x11, q8y3));\n\n            const uint16_t ls0 = (uint16_t)(((sc[0] & 0xf) | ((h << 4) & 0x30)) - 32);\n            const uint16_t ls1 = (uint16_t)(((sc[0] >>  4) | ((h << 2) & 0x30)) - 32);\n            h >>= 4;\n            sc ++;\n\n            vector signed short vscales01 = vec_splats((int16_t)ls0);\n            vector signed short vscales23 = vec_splats((int16_t)ls1);\n\n            vsumi0 = vec_msum(qv0, vscales01, vsumi0);\n            vsumi1 = vec_msum(qv1, vscales01, vsumi1);\n            vsumi2 = vec_msum(qv2, vscales23, vsumi2);\n            vsumi3 = vec_msum(qv3, vscales23, vsumi3);\n        }\n\n        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0);\n        vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1);\n        vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2);\n        vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3);\n    }\n\n    vsumf0 = vec_add(vsumf0, vsumf2);\n    vsumf1 = vec_add(vsumf1, vsumf3);\n\n    vsumf0 = vec_add(vsumf0, vsumf1);\n\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));\n    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));\n\n    *s = vec_extract(vsumf0, 0);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp",
    "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__riscv) && __riscv_xlen == 64\n#include <asm/hwprobe.h>\n#include <asm/unistd.h>\n#include <unistd.h>\n\nstruct riscv64_features {\n    bool has_rvv = false;\n\n    riscv64_features() {\n        struct riscv_hwprobe probe;\n        probe.key = RISCV_HWPROBE_KEY_IMA_EXT_0;\n        probe.value = 0;\n\n        int ret = syscall(__NR_riscv_hwprobe, &probe, 1, 0, NULL, 0);\n\n        if (0 == ret) {\n            has_rvv = !!(probe.value & RISCV_HWPROBE_IMA_V);\n        }\n    }\n};\n\nstatic int ggml_backend_cpu_riscv64_score() {\n    int score = 1;\n    riscv64_features rf;\n\n#ifdef GGML_USE_RVV\n    if (!rf.has_rvv) { return 0; }\n    score += 1 << 1;\n#endif\n\n    return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_riscv64_score)\n\n#endif  // __riscv && __riscv_xlen == 64\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/riscv/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__riscv_v)\n\n    size_t vl = QK8_0;\n\n    for (int i = 0; i < nb; i++) {\n        // load elements\n        vfloat32m8_t v_x   = __riscv_vle32_v_f32m8(x+i*QK8_0, vl);\n\n        vfloat32m8_t vfabs = __riscv_vfabs_v_f32m8(v_x, vl);\n        vfloat32m1_t tmp   = __riscv_vfmv_v_f_f32m1(0.0f, vl);\n        vfloat32m1_t vmax  = __riscv_vfredmax_vs_f32m8_f32m1(vfabs, tmp, vl);\n        float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        vfloat32m8_t x0 = __riscv_vfmul_vf_f32m8(v_x, id, vl);\n\n        // convert to integer\n        vint16m4_t   vi = __riscv_vfncvt_x_f_w_i16m4(x0, vl);\n        vint8m2_t    vs = __riscv_vncvt_x_x_w_i8m2(vi, vl);\n\n        // store result\n        __riscv_vse8_v_i8m2(y[i].qs , vs, vl);\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__riscv_v)\n\n    size_t vl = QK8_1;\n\n    for (int i = 0; i < nb; i++) {\n        // load elements\n        vfloat32m8_t v_x   = __riscv_vle32_v_f32m8(x+i*QK8_1, vl);\n\n        vfloat32m8_t vfabs = __riscv_vfabs_v_f32m8(v_x, vl);\n        vfloat32m1_t tmp   = __riscv_vfmv_v_f_f32m1(0.0, vl);\n        vfloat32m1_t vmax  = __riscv_vfredmax_vs_f32m8_f32m1(vfabs, tmp, vl);\n        float amax = __riscv_vfmv_f_s_f32m1_f32(vmax);\n\n        const float d  = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        vfloat32m8_t x0 = __riscv_vfmul_vf_f32m8(v_x, id, vl);\n\n        // convert to integer\n        vint16m4_t   vi = __riscv_vfncvt_x_f_w_i16m4(x0, vl);\n        vint8m2_t    vs = __riscv_vncvt_x_x_w_i8m2(vi, vl);\n\n        // store result\n        __riscv_vse8_v_i8m2(y[i].qs , vs, vl);\n\n        // compute sum for y[i].s\n        vint16m1_t tmp2 = __riscv_vmv_v_x_i16m1(0, vl);\n        vint16m1_t vwrs = __riscv_vwredsum_vs_i8m2_i16m1(vs, tmp2, vl);\n\n        // set y[i].s\n        int sum = __riscv_vmv_x_s_i16m1_i16(vwrs);\n        y[i].s = GGML_CPU_FP32_TO_FP16(sum*d);\n    }\n\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    assert(k % QK_K == 0);\n    block_q8_K * y_blocks = (block_q8_K *)y;\n    size_t nb = k / QK_K;\n\n#if defined(__riscv_v_intrinsic)\n    const size_t vlmax_f32m8 = __riscv_vsetvlmax_e32m8();\n\n    for (size_t i = 0; i < nb; i++) {\n        const float* x_block = x + i * QK_K;\n        block_q8_K* y_block = &y_blocks[i];\n\n        // 1. Calculate Min/Max\n        vfloat32m8_t max_v = __riscv_vfmv_v_f_f32m8(-__builtin_inff(), vlmax_f32m8);\n        vfloat32m8_t min_v = __riscv_vfmv_v_f_f32m8(__builtin_inff(), vlmax_f32m8);\n\n        size_t rem = QK_K;\n        size_t offset = 0;\n        while (rem > 0) {\n            size_t vl = __riscv_vsetvl_e32m8(rem);\n            vfloat32m8_t v_curr = __riscv_vle32_v_f32m8(x_block + offset, vl);\n            max_v = __riscv_vfmax_vv_f32m8(max_v, v_curr, vl);\n            min_v = __riscv_vfmin_vv_f32m8(min_v, v_curr, vl);\n            rem -= vl;\n            offset += vl;\n        }\n\n        vfloat32m1_t v_init_max = __riscv_vfmv_s_f_f32m1(-__builtin_inff(), 1);\n        vfloat32m1_t v_init_min = __riscv_vfmv_s_f_f32m1(__builtin_inff(), 1);\n\n        vfloat32m1_t v_scalar_max = __riscv_vfredmax_vs_f32m8_f32m1(max_v, v_init_max, vlmax_f32m8);\n        vfloat32m1_t v_scalar_min = __riscv_vfredmin_vs_f32m8_f32m1(min_v, v_init_min, vlmax_f32m8);\n\n        float max_val = __riscv_vfmv_f_s_f32m1_f32(v_scalar_max);\n        float min_val = __riscv_vfmv_f_s_f32m1_f32(v_scalar_min);\n\n        float amax = fabsf(max_val) > fabsf(min_val) ? fabsf(max_val) : fabsf(min_val);\n\n        if (amax == 0.0f) {\n            y_block->d = 0.0f;\n            memset(y_block->qs, 0, QK_K);\n            memset(y_block->bsums, 0, sizeof(y_block->bsums));\n            continue;\n        }\n\n        const float iscale = -127.f / (fabsf(max_val) > fabsf(min_val) ? max_val : min_val);\n        y_block->d = 1.0f / iscale;\n\n        // 2. Quantize and Calculate Sums\n        offset = 0;\n        rem = QK_K;\n        vint16m1_t v_zero_sum = __riscv_vmv_v_x_i16m1(0, 1);\n\n        while (rem > 0) {\n            size_t vl = __riscv_vsetvl_e32m8(rem);\n            vfloat32m8_t v_f = __riscv_vle32_v_f32m8(x_block + offset, vl);\n\n            v_f = __riscv_vfmul_vf_f32m8(v_f, iscale, vl);\n\n            vint32m8_t v_i32 = __riscv_vfcvt_x_f_v_i32m8_rm(v_f, __RISCV_FRM_RNE, vl);\n            vint16m4_t v_i16 = __riscv_vnclip_wx_i16m4(v_i32, 0, __RISCV_VXRM_RNE, vl);\n            vint8m2_t v_q = __riscv_vnclip_wx_i8m2(v_i16, 0, __RISCV_VXRM_RNE, vl);\n\n            __riscv_vse8_v_i8m2(y_block->qs + offset, v_q, vl);\n\n            // first iteration clear\n\n            int sum_idx;\n            vint8m1_t chunk_m1;\n            vint16m1_t v_sum;\n            sum_idx = offset / 16;\n            chunk_m1 = __riscv_vget_v_i8m2_i8m1(v_q, 0);\n            v_sum = __riscv_vwredsum_vs_i8m1_i16m1(chunk_m1, v_zero_sum, 16);\n            y_block->bsums[sum_idx] = (int16_t)__riscv_vmv_x_s_i16m1_i16(v_sum);\n\n            // remaining iterations\n            vint8m2_t slid_q = v_q;\n            for (size_t k = 16; k < vl; k += 16) {\n                slid_q = __riscv_vslidedown_vx_i8m2(slid_q, 16, vl);\n\n                sum_idx = (offset + k) / 16;\n                chunk_m1 = __riscv_vget_v_i8m2_i8m1(slid_q, 0);\n\n                v_sum = __riscv_vwredsum_vs_i8m1_i16m1(chunk_m1, v_zero_sum, 16);\n                y_block->bsums[sum_idx] =(int16_t)__riscv_vmv_x_s_i16m1_i16(v_sum);\n            }\n\n            rem -= vl;\n            offset += vl;\n        }\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_K_ref(x, y, k);\n#endif\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n    size_t vl = qk / 2;\n\n    for (; ib < nb; ++ib) {\n        // load elements\n        vuint8m1_t tx = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n\n        vint8m1_t y0 = __riscv_vle8_v_i8m1(y[ib].qs, vl);\n        vint8m1_t y1 = __riscv_vle8_v_i8m1(y[ib].qs+16, vl);\n\n        // mask and store lower part of x, and then upper part\n        vuint8m1_t x_a = __riscv_vand_vx_u8m1(tx, 0x0F, vl);\n        vuint8m1_t x_l = __riscv_vsrl_vx_u8m1(tx, 0x04, vl);\n\n        vint8m1_t x_ai = __riscv_vreinterpret_v_u8m1_i8m1(x_a);\n        vint8m1_t x_li = __riscv_vreinterpret_v_u8m1_i8m1(x_l);\n\n        // subtract offset\n        vint8m1_t v0 = __riscv_vsub_vx_i8m1(x_ai, 8, vl);\n        vint8m1_t v1 = __riscv_vsub_vx_i8m1(x_li, 8, vl);\n\n        vint16m2_t vec_mul1 = __riscv_vwmul_vv_i16m2(v0, y0, vl);\n        vint16m2_t vec_mul2 = __riscv_vwmacc_vv_i16m2(vec_mul1, v1, y1, vl);\n\n        vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);\n        vint32m1_t vs2 = __riscv_vwredsum_vs_i16m2_i32m1(vec_mul2, vec_zero, vl);\n\n        int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);\n\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n#else\n    ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n    size_t vl = qk / 2;\n\n    for (; ib < nb; ++ib) {\n        // load elements\n        vuint8m1_t tx = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n\n        vint8m1_t y0 = __riscv_vle8_v_i8m1(y[ib].qs, vl);\n        vint8m1_t y1 = __riscv_vle8_v_i8m1(y[ib].qs+16, vl);\n\n        // mask and store lower part of x, and then upper part\n        vuint8m1_t x_a = __riscv_vand_vx_u8m1(tx, 0x0F, vl);\n        vuint8m1_t x_l = __riscv_vsrl_vx_u8m1(tx, 0x04, vl);\n\n        vint8m1_t v0 = __riscv_vreinterpret_v_u8m1_i8m1(x_a);\n        vint8m1_t v1 = __riscv_vreinterpret_v_u8m1_i8m1(x_l);\n\n        vint16m2_t vec_mul1 = __riscv_vwmul_vv_i16m2(v0, y0, vl);\n        vint16m2_t vec_mul2 = __riscv_vwmacc_vv_i16m2(vec_mul1, v1, y1, vl);\n\n        vint32m1_t vec_zero = __riscv_vmv_v_x_i32m1(0, vl);\n        vint32m1_t vs2 = __riscv_vwredsum_vs_i16m2_i32m1(vec_mul2, vec_zero, vl);\n\n        int sumi = __riscv_vmv_x_s_i32m1_i32(vs2);\n\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n#else\n    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    size_t vl;\n    size_t vlenb = __riscv_vlenb();\n\n    for (; ib < nb; ++ib) {\n        vl = qk / 2;\n        vuint8m1_t v0 = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n        vint8m1_t v0l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(v0, 0x0F, vl));\n        vint8m1_t v0h = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(v0, 4, vl));\n        vint8m2_t v0c;\n        if (vlenb == 16) {\n            v0c = __riscv_vcreate_v_i8m1_i8m2(v0l, v0h);\n        } else {\n            v0l = __riscv_vslideup_vx_i8m1(v0l, v0h, 16, 32);\n            v0c = __riscv_vlmul_ext_v_i8m1_i8m2(v0l);\n        }\n\n        vl = qk;\n        vbool4_t qh = __riscv_vlm_v_b4(x[ib].qh, vl);\n        qh = __riscv_vmnand_mm_b4(qh, qh, vl);\n        vint8m2_t v0f = __riscv_vsub_vx_i8m2_mu(qh, v0c, v0c, 0x10, vl);\n        vint8m2_t v1 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n        vint16m4_t mul = __riscv_vwmul_vv_i16m4(v0f, v1, vl);\n        vint32m1_t zero = __riscv_vmv_v_x_i32m1(0, vl);\n        vint32m1_t sum = __riscv_vwredsum_vs_i16m4_i32m1(mul, zero, vl);\n        int32_t sumi = __riscv_vmv_x_s_i32m1_i32(sum);\n\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n    }\n\n    *s = sumf;\n#else\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined(__riscv_v)\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    size_t vl;\n    size_t vlenb = __riscv_vlenb();\n\n    for (; ib < nb; ++ib) {\n        vl = qk / 2;\n        vuint8m1_t v0 = __riscv_vle8_v_u8m1(x[ib].qs, vl);\n        vint8m1_t v0l = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(v0, 0x0F, vl));\n        vint8m1_t v0h = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(v0, 4, vl));\n        vint8m2_t v0c;\n        if (vlenb == 16) {\n            v0c = __riscv_vcreate_v_i8m1_i8m2(v0l, v0h);\n        } else {\n            v0l = __riscv_vslideup_vx_i8m1(v0l, v0h, 16, 32);\n            v0c = __riscv_vlmul_ext_v_i8m1_i8m2(v0l);\n        }\n\n        vl = qk;\n        vbool4_t qh = __riscv_vlm_v_b4(x[ib].qh, vl);\n        vint8m2_t v0f = __riscv_vor_vx_i8m2_mu(qh, v0c, v0c, 0x10, vl);\n        vint8m2_t v1 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n        vint16m4_t mul = __riscv_vwmul_vv_i16m4(v0f, v1, vl);\n        vint32m1_t zero = __riscv_vmv_v_x_i32m1(0, vl);\n        vint32m1_t sum = __riscv_vwredsum_vs_i16m4_i32m1(mul, zero, vl);\n        int32_t sumi = __riscv_vmv_x_s_i32m1_i32(sum);\n\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n#else\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__riscv_v)\n    size_t vl = qk;\n\n    for (; ib < nb; ++ib) {\n        // load elements\n        vint8m2_t bx_0 = __riscv_vle8_v_i8m2(x[ib].qs, vl);\n        vint8m2_t by_0 = __riscv_vle8_v_i8m2(y[ib].qs, vl);\n\n        vint16m4_t vw_mul = __riscv_vwmul_vv_i16m4(bx_0, by_0, vl);\n\n        vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, vl);\n        vint32m1_t v_sum = __riscv_vwredsum_vs_i16m4_i32m1(vw_mul, v_zero, vl);\n\n        int sumi = __riscv_vmv_x_s_i32m1_i32(v_sum);\n\n        sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n    }\n\n    *s = sumf;\n#else\n\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n\n    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n    float sumf = 0;\n    uint8_t atmp[16];\n\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * q2 = x[i].qs;\n        const  int8_t * q8 = y[i].qs;\n        const uint8_t * sc = x[i].scales;\n        const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n        uint8_t *patmp = atmp;\n        int vsums;\n        int tmp;\n        __asm__ __volatile__(\n            \"th.vsetvli zero, %[vl16], e8, m1\\n\\t\"\n            \"th.vmv.v.x v8, zero\\n\\t\"\n            \"th.vlb.v v1, (%[sc])\\n\\t\"\n            \"th.vand.vi v0, v1, 0xF\\n\\t\"\n            \"th.vsrl.vi v1, v1, 4\\n\\t\"\n            \"th.vsb.v v0, (%[scale])\\n\\t\"\n            \"th.vwaddu.vx v16, v1, zero\\n\\t\"\n            \"th.vsetvli zero, %[vl16], e16, m2\\n\\t\"\n            \"th.vlh.v v2, (%[bsums])\\n\\t\"\n            \"th.vwmul.vv v4, v16, v2\\n\\t\"\n            \"th.vsetvli zero, %[vl16], e32, m4\\n\\t\"\n            \"th.vredsum.vs v8, v4, v8\\n\\t\"\n            \"th.vmv.x.s %[vsums], v8\"\n            : [tmp] \"=&r\" (tmp), [vsums] \"=&r\" (vsums)\n            : [sc] \"r\" (sc), [scale] \"r\" (atmp), [bsums] \"r\" (y[i].bsums)\n            , [vl16] \"r\" (16)\n            : \"memory\"\n            , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n            , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n            , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n            , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n        );\n        sumf += dmin * vsums;\n        int isum = 0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            __asm__ __volatile__(\n                \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n                \"th.vlb.v v0, (%[q2])\\n\\t\"\n                \"th.vsrl.vi v2, v0, 2\\n\\t\"\n                \"th.vsrl.vi v4, v0, 4\\n\\t\"\n                \"th.vsrl.vi v6, v0, 6\\n\\t\"\n                \"th.vand.vi v0, v0, 0x3\\n\\t\"\n                \"th.vand.vi v2, v2, 0x3\\n\\t\"\n                \"th.vand.vi v4, v4, 0x3\\n\\t\"\n                \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n                \"th.vlb.v v8, (%[q8])\\n\\t\"\n                \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n                \"th.vwmul.vv v16, v0, v8\\n\\t\"\n                \"th.vwmul.vv v24, v4, v12\\n\\t\"\n                \"th.vsetvli zero, %[vl16], e16, m2\\n\\t\"\n                \"th.vmv.v.x v0, zero\\n\\t\"\n                \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n                \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n                \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n                \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n                \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n                \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n                \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n                \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n                \"li %[tmp], 4\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n                \"th.vslideup.vi v10, v9, 1\\n\\t\"\n                \"th.vslideup.vi v8, v7, 1\\n\\t\"\n                \"th.vslideup.vi v11, v12, 1\\n\\t\"\n                \"th.vslideup.vi v13, v14, 1\\n\\t\"\n                \"th.vslideup.vi v10, v8, 2\\n\\t\"\n                \"th.vslideup.vi v11, v13, 2\\n\\t\"\n                \"li %[tmp], 8\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m2\\n\\t\"\n                \"th.vlbu.v v12, (%[scale])\\n\\t\"\n                \"th.vmul.vv v10, v10, v12\\n\\t\"\n                \"th.vredsum.vs v0, v10, v0\\n\\t\"\n                \"th.vmv.x.s %[tmp], v0\\n\\t\"\n                \"add %[isum], %[isum], %[tmp]\"\n                : [tmp] \"=&r\" (tmp), [isum] \"+&r\" (isum)\n                : [q2] \"r\" (q2), [scale] \"r\" (patmp), [q8] \"r\" (q8)\n                , [vl16] \"r\" (16), [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n            q2 += 32; q8 += 128; patmp += 8;\n        }\n\n        sumf += dall * isum;\n    }\n\n    *s = sumf;\n\n#elif defined __riscv_v\n\n    float sumf = 0;\n    uint8_t atmp[16];\n\n    const int vector_length = __riscv_vlenb() * 8;\n    uint8_t temp_01[32] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n                            1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };\n\n    switch (vector_length) {\n    case 256:\n        for (int i = 0; i < nb; ++i) {\n            const uint8_t * q2 = x[i].qs;\n            const int8_t *  q8 = y[i].qs;\n            const uint8_t * sc = x[i].scales;\n\n            const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n            const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n            size_t vl = 16;\n\n            vuint8m1_t scales = __riscv_vle8_v_u8m1(sc, vl);\n            vuint8m1_t aux    = __riscv_vand_vx_u8m1(scales, 0x0F, vl);\n\n            vint16m1_t q8sums = __riscv_vle16_v_i16m1(y[i].bsums, vl);\n\n            vuint8mf2_t scales_2 = __riscv_vle8_v_u8mf2(sc, vl);\n            vuint8mf2_t mins8    = __riscv_vsrl_vx_u8mf2(scales_2, 0x4, vl);\n            vint16m1_t  mins     = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));\n            vint32m2_t  prod     = __riscv_vwmul_vv_i32m2(q8sums, mins, vl);\n            vint32m1_t  vsums    = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n\n            sumf += dmin * __riscv_vmv_x_s_i32m1_i32(vsums);\n\n            vl = 32;\n\n            vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n            vuint8m1_t v_b   = __riscv_vle8_v_u8m1(temp_01, vl);\n\n            uint8_t is   = 0;\n            int     isum = 0;\n\n            for (int j = 0; j < QK_K / 128; ++j) {\n                // load Q2\n                vuint8m1_t q2_x = __riscv_vle8_v_u8m1(q2, vl);\n\n                vuint8m1_t q2_0 = __riscv_vand_vx_u8m1(q2_x, 0x03, vl);\n                vuint8m1_t q2_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x2, vl), 0x03, vl);\n                vuint8m1_t q2_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x4, vl), 0x03, vl);\n                vuint8m1_t q2_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q2_x, 0x6, vl), 0x03, vl);\n\n                // duplicate scale elements for product\n                vuint8m1_t sc0 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 0 + is, vl), vl);\n                vuint8m1_t sc1 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 2 + is, vl), vl);\n                vuint8m1_t sc2 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 4 + is, vl), vl);\n                vuint8m1_t sc3 = __riscv_vrgather_vv_u8m1(aux, __riscv_vadd_vx_u8m1(v_b, 6 + is, vl), vl);\n\n                vint16m2_t p0 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_0, sc0, vl));\n                vint16m2_t p1 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_1, sc1, vl));\n                vint16m2_t p2 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_2, sc2, vl));\n                vint16m2_t p3 = __riscv_vreinterpret_v_u16m2_i16m2(__riscv_vwmulu_vv_u16m2(q2_3, sc3, vl));\n\n                // load Q8\n                vint8m1_t q8_0 = __riscv_vle8_v_i8m1(q8, vl);\n                vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8 + 32, vl);\n                vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8 + 64, vl);\n                vint8m1_t q8_3 = __riscv_vle8_v_i8m1(q8 + 96, vl);\n\n                vint32m4_t s0 = __riscv_vwmul_vv_i32m4(p0, __riscv_vwcvt_x_x_v_i16m2(q8_0, vl), vl);\n                vint32m4_t s1 = __riscv_vwmul_vv_i32m4(p1, __riscv_vwcvt_x_x_v_i16m2(q8_1, vl), vl);\n                vint32m4_t s2 = __riscv_vwmul_vv_i32m4(p2, __riscv_vwcvt_x_x_v_i16m2(q8_2, vl), vl);\n                vint32m4_t s3 = __riscv_vwmul_vv_i32m4(p3, __riscv_vwcvt_x_x_v_i16m2(q8_3, vl), vl);\n\n                vint32m1_t isum0 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s0, s1, vl), vzero, vl);\n                vint32m1_t isum1 = __riscv_vredsum_vs_i32m4_i32m1(__riscv_vadd_vv_i32m4(s2, s3, vl), isum0, vl);\n\n                isum += __riscv_vmv_x_s_i32m1_i32(isum1);\n\n                q2 += 32;\n                q8 += 128;\n                is = 8;\n            }\n\n            sumf += dall * isum;\n        }\n        break;\n    case 128:\n        for (int i = 0; i < nb; ++i) {\n            const uint8_t * q2 = x[i].qs;\n            const  int8_t * q8 = y[i].qs;\n            const uint8_t * sc = x[i].scales;\n            const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n            const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n            uint8_t *patmp = atmp;\n            int vsums;\n            int tmp, t1, t2, t3, t4, t5, t6, t7;\n            __asm__ __volatile__(\n                \"vsetivli zero, 16, e8, m1\\n\\t\"\n                \"vmv.v.x v8, zero\\n\\t\"\n                \"lb zero, 15(%[sc])\\n\\t\"\n                \"vle8.v v1, (%[sc])\\n\\t\"\n                \"vle8.v v2, (%[bsums])\\n\\t\"\n                \"addi %[tmp], %[bsums], 16\\n\\t\"\n                \"vand.vi v0, v1, 0xF\\n\\t\"\n                \"vsrl.vi v1, v1, 4\\n\\t\"\n                \"vle8.v v3, (%[tmp])\\n\\t\"\n                \"vse8.v v0, (%[scale])\\n\\t\"\n                \"vsetivli zero, 16, e16, m2\\n\\t\"\n                \"vzext.vf2 v0, v1\\n\\t\"\n                \"vwmul.vv v4, v0, v2\\n\\t\"\n                \"vsetivli zero, 16, e32, m4\\n\\t\"\n                \"vredsum.vs v8, v4, v8\\n\\t\"\n                \"vmv.x.s %[vsums], v8\"\n                : [tmp] \"=&r\" (tmp), [vsums] \"=&r\" (vsums)\n                : [sc] \"r\" (sc), [scale] \"r\" (atmp), [bsums] \"r\" (y[i].bsums)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n            sumf += dmin * vsums;\n            int isum = 0;\n\n            for (int j = 0; j < QK_K/128; ++j) {\n                __asm__ __volatile__(\n                    \"lb zero, 31(%[q2])\\n\\t\"\n                    \"addi %[tmp], %[q2], 16\\n\\t\"\n                    \"addi %[t1], %[q8], 16\\n\\t\"\n                    \"vsetivli zero, 16, e8, m1\\n\\t\"\n                    \"vle8.v v0, (%[q2])\\n\\t\"\n                    \"vle8.v v1, (%[tmp])\\n\\t\"\n                    \"vsrl.vi v2, v0, 2\\n\\t\"\n                    \"vsrl.vi v3, v1, 2\\n\\t\"\n                    \"vsrl.vi v4, v0, 4\\n\\t\"\n                    \"addi %[tmp], %[q8], 32\\n\\t\"\n                    \"vle8.v v8, (%[q8])\\n\\t\"\n                    \"vle8.v v9, (%[t1])\\n\\t\"\n                    \"addi %[t1], %[t1], 32\\n\\t\"\n                    \"vsrl.vi v5, v1, 4\\n\\t\"\n                    \"vsrl.vi v6, v0, 6\\n\\t\"\n                    \"vsrl.vi v7, v1, 6\\n\\t\"\n                    \"vle8.v v10, (%[tmp])\\n\\t\"\n                    \"vle8.v v11, (%[t1])\\n\\t\"\n                    \"addi %[tmp], %[tmp], 32\\n\\t\"\n                    \"addi %[t1], %[t1], 32\\n\\t\"\n                    \"vand.vi v0, v0, 0x3\\n\\t\"\n                    \"vand.vi v1, v1, 0x3\\n\\t\"\n                    \"vand.vi v2, v2, 0x3\\n\\t\"\n                    \"vle8.v v12, (%[tmp])\\n\\t\"\n                    \"vle8.v v13, (%[t1])\\n\\t\"\n                    \"addi %[tmp], %[tmp], 32\\n\\t\"\n                    \"addi %[t1], %[t1], 32\\n\\t\"\n                    \"vand.vi v3, v3, 0x3\\n\\t\"\n                    \"vand.vi v4, v4, 0x3\\n\\t\"\n                    \"vand.vi v5, v5, 0x3\\n\\t\"\n                    \"vle8.v v14, (%[tmp])\\n\\t\"\n                    \"vle8.v v15, (%[t1])\\n\\t\"\n                    \"vwmul.vv v16, v0, v8\\n\\t\"\n                    \"vwmul.vv v18, v1, v9\\n\\t\"\n                    \"vwmul.vv v20, v2, v10\\n\\t\"\n                    \"vwmul.vv v22, v3, v11\\n\\t\"\n                    \"vwmul.vv v24, v4, v12\\n\\t\"\n                    \"vwmul.vv v26, v5, v13\\n\\t\"\n                    \"vwmul.vv v28, v6, v14\\n\\t\"\n                    \"vwmul.vv v30, v7, v15\\n\\t\"\n                    \"vsetivli zero, 8, e16, m1\\n\\t\"\n                    \"vmv.v.x v0, zero\\n\\t\"\n                    \"lbu %[tmp], 0(%[scale])\\n\\t\"\n                    \"vwredsum.vs v8, v16, v0\\n\\t\"\n                    \"vwredsum.vs v9, v18, v0\\n\\t\"\n                    \"lbu %[t1], 1(%[scale])\\n\\t\"\n                    \"vwredsum.vs v10, v20, v0\\n\\t\"\n                    \"vwredsum.vs v11, v22, v0\\n\\t\"\n                    \"lbu %[t2], 2(%[scale])\\n\\t\"\n                    \"vwredsum.vs v12, v24, v0\\n\\t\"\n                    \"vwredsum.vs v13, v26, v0\\n\\t\"\n                    \"lbu %[t3], 3(%[scale])\\n\\t\"\n                    \"vwredsum.vs v14, v28, v0\\n\\t\"\n                    \"vwredsum.vs v15, v30, v0\\n\\t\"\n                    \"lbu %[t4], 4(%[scale])\\n\\t\"\n                    \"vwredsum.vs v8, v17, v8\\n\\t\"\n                    \"vwredsum.vs v9, v19, v9\\n\\t\"\n                    \"lbu %[t5], 5(%[scale])\\n\\t\"\n                    \"vwredsum.vs v10, v21, v10\\n\\t\"\n                    \"vwredsum.vs v11, v23, v11\\n\\t\"\n                    \"lbu %[t6], 6(%[scale])\\n\\t\"\n                    \"vwredsum.vs v12, v25, v12\\n\\t\"\n                    \"vwredsum.vs v13, v27, v13\\n\\t\"\n                    \"lbu %[t7], 7(%[scale])\\n\\t\"\n                    \"vwredsum.vs v14, v29, v14\\n\\t\"\n                    \"vwredsum.vs v15, v31, v15\\n\\t\"\n                    \"vsetivli zero, 4, e32, m1\\n\\t\"\n                    \"vmul.vx v0, v8, %[tmp]\\n\\t\"\n                    \"vmul.vx v1, v9, %[t1]\\n\\t\"\n                    \"vmacc.vx v0, %[t2], v10\\n\\t\"\n                    \"vmacc.vx v1, %[t3], v11\\n\\t\"\n                    \"vmacc.vx v0, %[t4], v12\\n\\t\"\n                    \"vmacc.vx v1, %[t5], v13\\n\\t\"\n                    \"vmacc.vx v0, %[t6], v14\\n\\t\"\n                    \"vmacc.vx v1, %[t7], v15\\n\\t\"\n                    \"vmv.x.s %[tmp], v0\\n\\t\"\n                    \"vmv.x.s %[t1], v1\\n\\t\"\n                    \"add %[isum], %[isum], %[tmp]\\n\\t\"\n                    \"add %[isum], %[isum], %[t1]\"\n                    : [tmp] \"=&r\" (tmp), [t1] \"=&r\" (t1), [t2] \"=&r\" (t2), [t3] \"=&r\" (t3)\n                    , [t4] \"=&r\" (t4), [t5] \"=&r\" (t5), [t6] \"=&r\" (t6), [t7] \"=&r\" (t7)\n                    , [isum] \"+&r\" (isum)\n                    : [q2] \"r\" (q2), [scale] \"r\" (patmp), [q8] \"r\" (q8)\n                    : \"memory\"\n                    , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                    , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                    , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                    , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n                );\n                q2 += 32; q8 += 128; patmp += 8;\n            }\n\n            sumf += dall * isum;\n        }\n        break;\n    default:\n        assert(false && \"Unsupported vector length\");\n        break;\n    }\n\n    *s = sumf;\n\n#else\n\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n    uint32_t utmp[4];\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * restrict q3 = x[i].qs;\n        const uint8_t * restrict qh = x[i].hmask;\n        const  int8_t * restrict q8 = y[i].qs;\n\n        int8_t * scale = (int8_t *)utmp;\n        int tmp;\n        __asm__ __volatile__(\n            \"li %[tmp], 12\\n\\t\"\n            \"th.vsetvli zero, %[tmp], e8, m1\\n\\t\"\n            \"th.vlb.v v0, (%[s6b])\\n\\t\"\n            \"th.vmv.v.v v2, v0\\n\\t\"\n            \"li %[tmp], 2\\n\\t\"\n            \"th.vsetvli zero, %[tmp], e64, m1\\n\\t\"\n            \"th.vmv.v.x v9, %[sh]\\n\\t\"\\\n            \"th.vslidedown.vi v1, v0, 1\\n\\t\"\n            \"th.vslide1up.vx v8, v9, zero\\n\\t\" // {0, 0, 4, 4}\n            \"th.vslideup.vi v0, v2, 1\\n\\t\" // {aux[0], aux[1], aux[0], aux[1]}\n            \"li %[tmp], 4\\n\\t\"\n            \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n            \"th.vid.v v9\\n\\t\"\n            \"th.vmv.x.s %[tmp], v1\\n\\t\"\n            \"th.vsll.vi v9, v9, 1\\n\\t\" // {0, 2, 4, 6}\n            \"th.vmv.v.x v1, %[tmp]\\n\\t\" // {aux[2], aux[2], aux[2], aux[2]}\n            \"th.vsrl.vv v4, v1, v9\\n\\t\"\n            \"th.vsrl.vv v2, v0, v8\\n\\t\"\n            \"th.vand.vx v5, v4, %[kmask1]\\n\\t\"\n            \"th.vand.vx v3, v2, %[kmask2]\\n\\t\"\n            \"th.vsll.vi v6, v5, 4\\n\\t\"\n            \"th.vor.vv v7, v6, v3\\n\\t\"\n            \"li %[tmp], 16\\n\\t\"\n            \"th.vsetvli zero, %[tmp], e8, m1\\n\\t\"\n            \"th.vsub.vx v0, v7, %[c]\\n\\t\"\n            \"th.vsb.v v0, (%[scale])\"\n            : [tmp] \"=&r\" (tmp)\n            : [sh] \"r\" (0x0000000400000004), [s6b] \"r\" (x[i].scales), [c] \"r\" (32)\n            , [scale] \"r\" (scale), [kmask1] \"r\" (kmask1), [kmask2] \"r\" (kmask2)\n            : \"memory\"\n            , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n            , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n            , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n            , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n        );\n\n        uint8_t m = 1;\n        int isum = 0;\n        for (int j = 0; j < QK_K; j += 128) {\n            __asm__ __volatile__(\n                // fixme: use v0p7 mask layout directly\n                \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n                \"th.vlb.v v8, (%[q3])\\n\\t\"\n                \"th.vsrl.vi v10, v8, 2\\n\\t\"\n                \"th.vsrl.vi v12, v8, 4\\n\\t\"\n                \"th.vsrl.vi v14, v8, 6\\n\\t\"\n                \"th.vand.vi v8, v8, 3\\n\\t\"\n                \"th.vand.vi v10, v10, 3\\n\\t\"\n                \"th.vand.vi v12, v12, 3\\n\\t\"\n                \"th.vlb.v v2, (%[qh])\\n\\t\"\n                \"th.vand.vx v4, v2, %[m]\\n\\t\"\n                \"slli %[m], %[m], 1\\n\\t\"\n                \"th.vmseq.vx v0, v4, zero\\n\\t\"\n                \"th.vadd.vi v8, v8, -4, v0.t\\n\\t\"\n                \"th.vand.vx v4, v2, %[m]\\n\\t\"\n                \"slli %[m], %[m], 1\\n\\t\"\n                \"th.vmseq.vx v0, v4, zero\\n\\t\"\n                \"th.vadd.vi v10, v10, -4, v0.t\\n\\t\"\n                \"th.vand.vx v4, v2, %[m]\\n\\t\"\n                \"slli %[m], %[m], 1\\n\\t\"\n                \"th.vmseq.vx v0, v4, zero\\n\\t\"\n                \"th.vadd.vi v12, v12, -4, v0.t\\n\\t\"\n                \"th.vand.vx v4, v2, %[m]\\n\\t\"\n                \"slli %[m], %[m], 1\\n\\t\"\n                \"th.vmseq.vx v0, v4, zero\\n\\t\"\n                \"th.vadd.vi v14, v14, -4, v0.t\\n\\t\"\n                \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n                \"th.vlb.v v0, (%[q8])\\n\\t\"\n                \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n                \"th.vwmul.vv v16, v0, v8\\n\\t\"\n                \"th.vwmul.vv v24, v4, v12\\n\\t\"\n                \"li %[tmp], 16\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e16, m2\\n\\t\"\n                \"th.vmv.v.x v0, zero\\n\\t\"\n                \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n                \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n                \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n                \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n                \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n                \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n                \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n                \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n                \"li %[tmp], 4\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n                \"th.vslideup.vi v10, v9, 1\\n\\t\"\n                \"th.vslideup.vi v8, v7, 1\\n\\t\"\n                \"th.vslideup.vi v11, v12, 1\\n\\t\"\n                \"th.vslideup.vi v13, v14, 1\\n\\t\"\n                \"th.vslideup.vi v10, v8, 2\\n\\t\"\n                \"th.vslideup.vi v11, v13, 2\\n\\t\"\n                \"li %[tmp], 8\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m2\\n\\t\"\n                \"th.vlb.v v12, (%[scale])\\n\\t\"\n                \"th.vmul.vv v10, v10, v12\\n\\t\"\n                \"th.vredsum.vs v0, v10, v0\\n\\t\"\n                \"th.vmv.x.s %[tmp], v0\\n\\t\"\n                \"add %[isum], %[isum], %[tmp]\"\n                : [tmp] \"=&r\" (tmp), [m] \"+&r\" (m), [isum] \"+&r\" (isum)\n                : [vl128] \"r\" (128), [vl64] \"r\" (64), [vl32] \"r\" (32)\n                , [q3] \"r\" (q3), [qh] \"r\" (qh), [scale] \"r\" (scale), [q8] \"r\" (q8)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n            q3 += 32;    q8 += 128;   scale += 8;\n        }\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        sumf += d * isum;\n    }\n\n    *s = sumf;\n\n#elif defined __riscv_v\n\n    uint32_t utmp[4];\n    float sumf = 0;\n    uint32_t aux[3];\n    const int vector_length = __riscv_vlenb() * 8;\n\n    switch (vector_length) {\n    case 256:\n        for (int i = 0; i < nb; ++i) {\n\n            const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n            const uint8_t * GGML_RESTRICT qh = x[i].hmask;\n            const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n            memcpy(aux, x[i].scales, 12);\n            utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n            utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n            utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n            utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n            int8_t * scale = (int8_t *)utmp;\n            for (int j = 0; j < 16; ++j) scale[j] -= 32;\n\n\n            size_t vl = 32;\n            uint8_t m =  1;\n\n            vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n            vuint8m1_t vqh = __riscv_vle8_v_u8m1(qh, vl);\n\n            int sum_t = 0;\n\n            for (int j = 0; j < QK_K; j += 128) {\n\n                vl = 32;\n\n                // load Q3\n                vuint8m1_t q3_x = __riscv_vle8_v_u8m1(q3, vl);\n\n                vint8m1_t q3_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q3_x, 0x03, vl));\n                vint8m1_t q3_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x2, vl), 0x03 , vl));\n                vint8m1_t q3_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x4, vl), 0x03 , vl));\n                vint8m1_t q3_3 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(q3_x, 0x6, vl), 0x03 , vl));\n\n                // compute mask for subtraction\n                vuint8m1_t qh_m0 = __riscv_vand_vx_u8m1(vqh, m, vl);\n                vbool8_t vmask_0 = __riscv_vmseq_vx_u8m1_b8(qh_m0, 0, vl);\n                vint8m1_t q3_m0 = __riscv_vsub_vx_i8m1_mu(vmask_0, q3_0, q3_0, 0x4, vl);\n                m <<= 1;\n\n                vuint8m1_t qh_m1 = __riscv_vand_vx_u8m1(vqh, m, vl);\n                vbool8_t vmask_1 = __riscv_vmseq_vx_u8m1_b8(qh_m1, 0, vl);\n                vint8m1_t q3_m1 = __riscv_vsub_vx_i8m1_mu(vmask_1, q3_1, q3_1, 0x4, vl);\n                m <<= 1;\n\n                vuint8m1_t qh_m2 = __riscv_vand_vx_u8m1(vqh, m, vl);\n                vbool8_t vmask_2 = __riscv_vmseq_vx_u8m1_b8(qh_m2, 0, vl);\n                vint8m1_t q3_m2 = __riscv_vsub_vx_i8m1_mu(vmask_2, q3_2, q3_2, 0x4, vl);\n                m <<= 1;\n\n                vuint8m1_t qh_m3 = __riscv_vand_vx_u8m1(vqh, m, vl);\n                vbool8_t vmask_3 = __riscv_vmseq_vx_u8m1_b8(qh_m3, 0, vl);\n                vint8m1_t q3_m3 = __riscv_vsub_vx_i8m1_mu(vmask_3, q3_3, q3_3, 0x4, vl);\n                m <<= 1;\n\n                // load Q8 and take product with Q3\n                vint16m2_t a0 = __riscv_vwmul_vv_i16m2(q3_m0, __riscv_vle8_v_i8m1(q8, vl), vl);\n                vint16m2_t a1 = __riscv_vwmul_vv_i16m2(q3_m1, __riscv_vle8_v_i8m1(q8+32, vl), vl);\n                vint16m2_t a2 = __riscv_vwmul_vv_i16m2(q3_m2, __riscv_vle8_v_i8m1(q8+64, vl), vl);\n                vint16m2_t a3 = __riscv_vwmul_vv_i16m2(q3_m3, __riscv_vle8_v_i8m1(q8+96, vl), vl);\n\n                vl = 16;\n\n                // retrieve lane to multiply with scale\n                vint32m2_t aux0_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 0), (scale[0]), vl);\n                vint32m2_t aux0_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a0, 1), (scale[1]), vl);\n                vint32m2_t aux1_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 0), (scale[2]), vl);\n                vint32m2_t aux1_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a1, 1), (scale[3]), vl);\n                vint32m2_t aux2_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 0), (scale[4]), vl);\n                vint32m2_t aux2_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a2, 1), (scale[5]), vl);\n                vint32m2_t aux3_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 0), (scale[6]), vl);\n                vint32m2_t aux3_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(a3, 1), (scale[7]), vl);\n\n                vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux0_0, aux0_1, vl), vzero, vl);\n                vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux1_0, aux1_1, vl), isum0, vl);\n                vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux2_0, aux2_1, vl), isum1, vl);\n                vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(aux3_0, aux3_1, vl), isum2, vl);\n\n                sum_t +=  __riscv_vmv_x_s_i32m1_i32(isum3);\n\n                q3 += 32;    q8 += 128;   scale += 8;\n\n            }\n\n            const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n            sumf += d*sum_t;\n\n        }\n        break;\n    case 128:\n        for (int i = 0; i < nb; ++i) {\n            const uint8_t * restrict q3 = x[i].qs;\n            const uint8_t * restrict qh = x[i].hmask;\n            const  int8_t * restrict q8 = y[i].qs;\n\n            int8_t * scale = (int8_t *)utmp;\n            int tmp, t1, t2, t3, t4, t5, t6, t7;\n            __asm__ __volatile__(\n                \"vsetivli zero, 12, e8, m1\\n\\t\"\n                \"vle8.v v0, (%[s6b])\\n\\t\"\n                \"vmv1r.v v2, v0\\n\\t\"\n                \"vsetivli zero, 2, e64, m1\\n\\t\"\n                \"vmv.v.x v9, %[sh]\\n\\t\"\\\n                \"vslidedown.vi v1, v0, 1\\n\\t\"\n                \"vslide1up.vx v8, v9, zero\\n\\t\" // {0, 0, 4, 4}\n                \"vslideup.vi v0, v2, 1\\n\\t\" // {aux[0], aux[1], aux[0], aux[1]}\n                \"vsetivli zero, 4, e32, m1\\n\\t\"\n                \"vid.v v9\\n\\t\"\n                \"vmv.x.s %[tmp], v1\\n\\t\"\n                \"vsll.vi v9, v9, 1\\n\\t\" // {0, 2, 4, 6}\n                \"vmv.v.x v1, %[tmp]\\n\\t\" // {aux[2], aux[2], aux[2], aux[2]}\n                \"vsrl.vv v4, v1, v9\\n\\t\"\n                \"vsrl.vv v2, v0, v8\\n\\t\"\n                \"vand.vx v5, v4, %[kmask1]\\n\\t\"\n                \"vand.vx v3, v2, %[kmask2]\\n\\t\"\n                \"vsll.vi v6, v5, 4\\n\\t\"\n                \"vor.vv v7, v6, v3\\n\\t\"\n                \"vsetivli zero, 16, e8, m1\\n\\t\"\n                \"vsub.vx v0, v7, %[c]\\n\\t\"\n                \"vse8.v v0, (%[scale])\"\n                : [tmp] \"=&r\" (tmp)\n                : [sh] \"r\" (0x0000000400000004), [s6b] \"r\" (x[i].scales), [c] \"r\" (32)\n                , [scale] \"r\" (scale), [kmask1] \"r\" (kmask1), [kmask2] \"r\" (kmask2)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n\n            uint8_t m = 1;\n            int isum = 0;\n            for (int j = 0; j < QK_K; j += 128) {\n                __asm__ __volatile__(\n                    \"lb zero, 31(%[q3])\\n\\t\"\n                    \"vsetvli zero, %[vl32], e8, m2, ta, mu\\n\\t\"\n                    \"vle8.v v8, (%[q3])\\n\\t\"\n                    \"vsrl.vi v10, v8, 2\\n\\t\"\n                    \"vsrl.vi v12, v8, 4\\n\\t\"\n                    \"vsrl.vi v14, v8, 6\\n\\t\"\n                    \"lb zero, 64(%[q8])\\n\\t\"\n                    \"vand.vi v8, v8, 3\\n\\t\"\n                    \"vand.vi v10, v10, 3\\n\\t\"\n                    \"vand.vi v12, v12, 3\\n\\t\"\n                    \"vle8.v v2, (%[qh])\\n\\t\"\n                    \"lb zero, 127(%[q8])\\n\\t\"\n                    \"vand.vx v4, v2, %[m]\\n\\t\"\n                    \"slli %[m], %[m], 1\\n\\t\"\n                    \"vmseq.vx v0, v4, zero\\n\\t\"\n                    \"vadd.vi v8, v8, -4, v0.t\\n\\t\"\n                    \"lb zero, 0(%[q8])\\n\\t\"\n                    \"vand.vx v4, v2, %[m]\\n\\t\"\n                    \"slli %[m], %[m], 1\\n\\t\"\n                    \"vmseq.vx v0, v4, zero\\n\\t\"\n                    \"vadd.vi v10, v10, -4, v0.t\\n\\t\"\n                    \"vand.vx v4, v2, %[m]\\n\\t\"\n                    \"slli %[m], %[m], 1\\n\\t\"\n                    \"vmseq.vx v0, v4, zero\\n\\t\"\n                    \"vadd.vi v12, v12, -4, v0.t\\n\\t\"\n                    \"vand.vx v4, v2, %[m]\\n\\t\"\n                    \"slli %[m], %[m], 1\\n\\t\"\n                    \"vmseq.vx v0, v4, zero\\n\\t\"\n                    \"vadd.vi v14, v14, -4, v0.t\\n\\t\"\n                    \"vsetvli zero, %[vl128], e8, m8\\n\\t\"\n                    \"vle8.v v0, (%[q8])\\n\\t\"\n                    \"lb %[tmp], 0(%[scale])\\n\\t\"\n                    \"lb %[t1], 1(%[scale])\\n\\t\"\n                    \"lb %[t2], 2(%[scale])\\n\\t\"\n                    \"lb %[t3], 3(%[scale])\\n\\t\"\n                    \"vsetvli zero, %[vl64], e8, m4\\n\\t\"\n                    \"vwmul.vv v16, v0, v8\\n\\t\"\n                    \"vwmul.vv v24, v4, v12\\n\\t\"\n                    \"vsetivli zero, 16, e16, m2\\n\\t\"\n                    \"vmv.v.x v0, zero\\n\\t\"\n                    \"vwredsum.vs v8, v16, v0\\n\\t\"\n                    \"lb %[t4], 4(%[scale])\\n\\t\"\n                    \"lb %[t5], 5(%[scale])\\n\\t\"\n                    \"vwredsum.vs v9, v18, v0\\n\\t\"\n                    \"vwredsum.vs v10, v20, v0\\n\\t\"\n                    \"vwredsum.vs v11, v22, v0\\n\\t\"\n                    \"vwredsum.vs v12, v24, v0\\n\\t\"\n                    \"lb %[t6], 6(%[scale])\\n\\t\"\n                    \"lb %[t7], 7(%[scale])\\n\\t\"\n                    \"vwredsum.vs v13, v26, v0\\n\\t\"\n                    \"vwredsum.vs v14, v28, v0\\n\\t\"\n                    \"vwredsum.vs v15, v30, v0\\n\\t\"\n                    \"vsetivli zero, 4, e32, m1\\n\\t\"\n                    \"vmul.vx v0, v8, %[tmp]\\n\\t\"\n                    \"vmul.vx v1, v9, %[t1]\\n\\t\"\n                    \"vmacc.vx v0, %[t2], v10\\n\\t\"\n                    \"vmacc.vx v1, %[t3], v11\\n\\t\"\n                    \"vmacc.vx v0, %[t4], v12\\n\\t\"\n                    \"vmacc.vx v1, %[t5], v13\\n\\t\"\n                    \"vmacc.vx v0, %[t6], v14\\n\\t\"\n                    \"vmacc.vx v1, %[t7], v15\\n\\t\"\n                    \"vmv.x.s %[tmp], v0\\n\\t\"\n                    \"vmv.x.s %[t1], v1\\n\\t\"\n                    \"add %[isum], %[isum], %[tmp]\\n\\t\"\n                    \"add %[isum], %[isum], %[t1]\"\n                    : [tmp] \"=&r\" (tmp), [t1] \"=&r\" (t1), [t2] \"=&r\" (t2), [t3] \"=&r\" (t3)\n                    , [t4] \"=&r\" (t4), [t5] \"=&r\" (t5), [t6] \"=&r\" (t6), [t7] \"=&r\" (t7)\n                    , [m] \"+&r\" (m), [isum] \"+&r\" (isum)\n                    : [vl128] \"r\" (128), [vl64] \"r\" (64), [vl32] \"r\" (32)\n                    , [q3] \"r\" (q3), [qh] \"r\" (qh), [scale] \"r\" (scale), [q8] \"r\" (q8)\n                    : \"memory\"\n                    , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                    , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                    , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                    , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n                );\n                q3 += 32;    q8 += 128;   scale += 8;\n            }\n\n            const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n            sumf += d * isum;\n        }\n        break;\n    default:\n        assert(false && \"Unsupported vector length\");\n        break;\n    }\n\n    *s = sumf;\n\n#else\n\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __riscv_xtheadvector\n\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    const uint8_t * mins   = (const uint8_t*)&utmp[2];\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        int tmp, tmp2, sumi;\n        __asm__ __volatile__(\n            \"li %[t1], 12\\n\\t\"\n            \"th.vsetvli zero, %[t1], e8, m1\\n\\t\"\n            \"th.vlb.v v1, (%[s6b])\\n\\t\" // {aux[0], aux[1], aux[2]}\n            \"li %[t1], 4\\n\\t\"\n            \"th.vsetvli zero, %[t1], e32, m1\\n\\t\"\n            \"th.vslidedown.vi v2, v1, 2\\n\\t\"\n            \"th.vmv.v.v v3, v2\\n\\t\"\n            \"th.vslideup.vi v2, v3, 1\\n\\t\" // {aux[2], aux[2]}\n            \"li %[t1], 2\\n\\t\"\n            \"th.vsetvli zero, %[t1], e32, m1\\n\\t\"\n            \"th.vmv.v.i v4, 4\\n\\t\"\n            \"th.vand.vx v8, v1, %[kmask1]\\n\\t\"\n            \"th.vslide1up.vx v5, v4, zero\\n\\t\" // {0, 4}\n            \"th.vsrl.vi v6, v1, 6\\n\\t\"\n            \"th.vsrl.vv v7, v2, v5\\n\\t\"\n            \"th.vand.vx v0, v6, %[kmask3]\\n\\t\"\n            \"th.vand.vx v2, v7, %[kmask2]\\n\\t\"\n            \"th.vsll.vi v6, v0, 4\\n\\t\"\n            \"li %[t2], 8\\n\\t\"\n            \"addi %[t1], %[utmp], 4\\n\\t\"\n            \"th.vor.vv v1, v6, v2\\n\\t\"\n            \"th.vssw.v v8, (%[utmp]), %[t2]\\n\\t\"\n            \"th.vssw.v v1, (%[t1]), %[t2]\\n\\t\"\n            \"th.vsetvli zero, zero, e32, m2\\n\\t\" // vl == 8\n            \"th.vlw.v v2, (%[bsums])\\n\\t\"\n            \"th.vsetvli zero, %[t2], e16, m1\\n\\t\"\n            \"th.vnsrl.vi v0, v2, 0\\n\\t\"\n            \"th.vnsrl.vi v1, v2, 16\\n\\t\"\n            \"th.vadd.vv v2, v0, v1\\n\\t\"\n            \"th.vlbu.v v4, (%[mins])\\n\\t\"\n            \"th.vwmul.vv v6, v4, v2\\n\\t\"\n            \"th.vmv.v.x v0, zero\\n\\t\"\n            \"th.vsetvli zero, %[t2], e32, m2\\n\\t\"\n            \"th.vredsum.vs v0, v6, v0\\n\\t\"\n            \"th.vmv.x.s %[sumi], v0\"\n            : [t1] \"=&r\" (tmp), [t2] \"=&r\" (tmp2), [sumi] \"=&r\" (sumi)\n            : [bsums] \"r\" (y[i].bsums), [mins] \"r\" (mins), [utmp] \"r\" (utmp)\n            , [s6b] \"r\" (x[i].scales), [kmask1] \"r\" (kmask1)\n            , [kmask2] \"r\" (kmask2), [kmask3] \"r\" (kmask3)\n            : \"memory\"\n            , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n            , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n            , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n            , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n        );\n        sumf -= dmin * sumi;\n\n        const uint8_t * restrict q4 = x[i].qs;\n        const int8_t  * restrict q8 = y[i].qs;\n\n        sumi = 0;\n        const uint8_t * scale = scales;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            int vl128 = 128, vl64 = 64, vl32 = 32;\n            __asm__ __volatile__(\n                \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\"\n                \"th.vlb.v v8, (%[q8])\\n\\t\"\n                \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\"\n                \"th.vlb.v v0, (%[q4])\\n\\t\"\n                \"th.vsrl.vi v4, v0, 4\\n\\t\"\n                \"th.vand.vi v0, v0, 0xF\\n\\t\"\n                \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\"\n                \"th.vwmul.vv v28, v6, v14\\n\\t\"\n                \"th.vwmul.vv v20, v4, v10\\n\\t\"\n                \"th.vwmul.vv v24, v2, v12\\n\\t\"\n                \"th.vwmul.vv v16, v0, v8\\n\\t\"\n                \"li %[tmp], 4\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n                \"th.vlbu.v v1, (%[scale])\\n\\t\"\n                \"th.vmv.v.x v0, zero\\n\\t\"\n                \"th.vsetvli zero, %[vl32], e16, m4\\n\\t\"\n                \"th.vwredsum.vs v6, v24, v0\\n\\t\"\n                \"th.vwredsum.vs v7, v28, v0\\n\\t\"\n                \"th.vwredsum.vs v4, v16, v0\\n\\t\"\n                \"th.vwredsum.vs v5, v20, v0\\n\\t\"\n                \"th.vsetvli zero, %[tmp], e32, m1\\n\\t\"\n                \"th.vslideup.vi v6, v7, 1\\n\\t\"\n                \"th.vslideup.vi v4, v5, 1\\n\\t\"\n                \"th.vslideup.vi v4, v6, 2\\n\\t\"\n                \"th.vmul.vv v8, v4, v1\\n\\t\"\n                \"th.vredsum.vs v0, v8, v0\\n\\t\"\n                \"th.vmv.x.s %[tmp], v0\\n\\t\"\n                \"add %[sumi], %[sumi], %[tmp]\"\n                : [tmp] \"=&r\" (tmp), [sumi] \"+&r\" (sumi)\n                : [vl128] \"r\" (vl128), [vl64] \"r\" (vl64), [vl32] \"r\" (vl32)\n                , [q4] \"r\" (q4), [q8] \"r\" (q8), [scale] \"r\" (scale)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n\n            q4 += 64;    q8 += 128;    scale += 4;\n        }\n\n        sumf += d * sumi;\n\n    }\n\n    *s = sumf;\n\n#elif defined __riscv_v\n\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    const uint8_t * mins   = (const uint8_t*)&utmp[2];\n\n    float sumf = 0;\n    const int vector_length = __riscv_vlenb() * 8;\n\n    switch (vector_length) {\n    case 256:\n        for (int i = 0; i < nb; ++i) {\n\n            size_t vl = 8;\n\n            const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n            const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n            vint16mf2_t q8sums_0 = __riscv_vlse16_v_i16mf2(y[i].bsums, 4, vl);\n            vint16mf2_t q8sums_1 = __riscv_vlse16_v_i16mf2(y[i].bsums+1, 4, vl);\n            vint16mf2_t q8sums   = __riscv_vadd_vv_i16mf2(q8sums_0, q8sums_1, vl);\n\n            memcpy(utmp, x[i].scales, 12);\n            utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n            const uint32_t uaux = utmp[1] & kmask1;\n            utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n            utmp[2] = uaux;\n            utmp[0] &= kmask1;\n\n            vuint8mf4_t mins8  = __riscv_vle8_v_u8mf4(mins, vl);\n            vint16mf2_t v_mins = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vzext_vf2_u16mf2(mins8, vl));\n            vint32m1_t  prod   = __riscv_vwmul_vv_i32m1(q8sums, v_mins, vl);\n\n            vint32m1_t sumi = __riscv_vredsum_vs_i32m1_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n            sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);\n\n            const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n            const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n            vl = 32;\n\n            int32_t sum_1 = 0;\n            int32_t sum_2 = 0;\n\n            vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1);\n\n            for (int j = 0; j < QK_K/64; ++j) {\n                // load Q4\n                vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl);\n\n                // load Q8 and multiply it with lower Q4 nibble\n                vint8m1_t  q8_0 = __riscv_vle8_v_i8m1(q8, vl);\n                vint8m1_t  q4_0 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl));\n                vint16m2_t qv_0 = __riscv_vwmul_vv_i16m2(q4_0, q8_0, vl);\n                vint16m1_t vs_0 = __riscv_vredsum_vs_i16m2_i16m1(qv_0, vzero, vl);\n\n                sum_1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[2*j+0];\n\n                // load Q8 and multiply it with upper Q4 nibble\n                vint8m1_t  q8_1 = __riscv_vle8_v_i8m1(q8+32, vl);\n                vint8m1_t  q4_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl));\n                vint16m2_t qv_1 = __riscv_vwmul_vv_i16m2(q4_1, q8_1, vl);\n                vint16m1_t vs_1 = __riscv_vredsum_vs_i16m2_i16m1(qv_1, vzero, vl);\n\n                sum_2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[2*j+1];\n\n                q4 += 32;    q8 += 64;\n\n            }\n\n            sumf += d*(sum_1 + sum_2);\n\n        }\n        break;\n    case 128:\n        for (int i = 0; i < nb; ++i) {\n            const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n            const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n            float ftmp, ft2;\n            const uint8_t * restrict q40;\n            const uint8_t * restrict q41;\n            const uint8_t * restrict q42;\n            const uint8_t * restrict q43;\n            const int8_t  * restrict q80;\n            const int8_t  * restrict q81;\n            const int8_t  * restrict q82;\n            const int8_t  * restrict q83;\n            int s0, s1, s2, s3;\n\n            __asm__ __volatile__(\n                \"li %[s1], 8\\n\\t\"\n                \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n                \"vle32.v v1, (%[s6b])\\n\\t\"\n                \"vslide1down.vx v1, v1, zero\\n\\t\"\n                \"vmv.v.x v16, zero\\n\\t\"\n                \"vslidedown.vi v2, v1, 2\\n\\t\"\n                \"vmv1r.v v3, v2\\n\\t\"\n                \"vslideup.vi v2, v3, 1\\n\\t\" // {aux[2], aux[2]}\n                \"vsetivli zero, 2, e32, m1, ta, ma\\n\\t\"\n                \"vmv.v.i v4, 4\\n\\t\"\n                \"vand.vx v8, v1, %[kmask1]\\n\\t\"\n                \"vslide1up.vx v5, v4, zero\\n\\t\" // {0, 4}\n                \"vsrl.vi v6, v1, 6\\n\\t\"\n                \"vsrl.vv v7, v2, v5\\n\\t\"\n                \"vsse32.v v8, (%[utmp]), %[s1]\\n\\t\"\n                \"vand.vx v0, v6, %[kmask3]\\n\\t\"\n                \"vand.vx v2, v7, %[kmask2]\\n\\t\"\n                \"vsll.vi v6, v0, 4\\n\\t\"\n                \"addi %[s0], %[utmp], 4\\n\\t\"\n                \"vor.vv v1, v6, v2\\n\\t\"\n                \"vsse32.v v1, (%[s0]), %[s1]\\n\\t\"\n                \"vsetivli zero, 8, e16, m1, ta, ma\\n\\t\"\n                \"vle32.v v2, (%[bsums])\\n\\t\"\n                \"vnsrl.wi v0, v2, 0\\n\\t\"\n                \"vnsrl.wi v1, v2, 16\\n\\t\"\n                \"vadd.vv v2, v0, v1\\n\\t\"\n                \"vle8.v v3, (%[mins])\\n\\t\"\n                \"vzext.vf2 v4, v3\\n\\t\"\n                \"vwmul.vv v6, v4, v2\\n\\t\"\n                \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n                \"vredsum.vs v0, v6, v16\\n\\t\"\n                \"vredsum.vs v0, v7, v0\\n\\t\"\n                \"vfcvt.f.x.v v0, v0\\n\\t\"\n                \"vfmv.f.s %[ftmp], v0\\n\\t\"\n                \"vsetivli zero, 16, e8, m1, ta, ma\\n\\t\"\n                \"vle8.v v0, (%[xs])\\n\\t\"\n                \"fnmsub.s %[sumf], %[dmin], %[ftmp], %[sumf]\\n\\t\"\n                \"addi %[q40], %[xs], 64\\n\\t\"\n                \"addi %[q41], %[xs], 16\\n\\t\"\n                \"addi %[q42], %[xs], 32\\n\\t\"\n                \"addi %[q43], %[xs], 48\\n\\t\"\n                \"addi %[q80], %[ys], 64\\n\\t\"\n                \"vle8.v v1, (%[q41])\\n\\t\"\n                \"vle8.v v2, (%[q42])\\n\\t\"\n                \"addi %[q81], %[ys], 16\\n\\t\"\n                \"addi %[q41], %[q41], 64\\n\\t\"\n                \"addi %[q82], %[ys], 32\\n\\t\"\n                \"vle8.v v3, (%[q43])\\n\\t\"\n                \"vle8.v v8, (%[ys])\\n\\t\"\n                \"addi %[q42], %[q42], 64\\n\\t\"\n                \"addi %[q83], %[ys], 48\\n\\t\"\n                \"addi %[q43], %[q43], 64\\n\\t\"\n                \"vsrl.vi v4, v0, 4\\n\\t\"\n                \"vle8.v v9, (%[q81])\\n\\t\"\n                \"vle8.v v10, (%[q82])\\n\\t\"\n                \"vand.vi v0, v0, 0xF\\n\\t\"\n                \"addi %[q81], %[q81], 64\\n\\t\"\n                \"vsrl.vi v5, v1, 4\\n\\t\"\n                \"addi %[q82], %[q82], 64\\n\\t\"\n                \"vle8.v v11, (%[q83])\\n\\t\"\n                \"vle8.v v12, (%[q80])\\n\\t\"\n                \"vand.vi v1, v1, 0xF\\n\\t\"\n                \"addi %[q83], %[q83], 64\\n\\t\"\n                \"vsrl.vi v6, v2, 4\\n\\t\"\n                \"addi %[q80], %[q80], 64\\n\\t\"\n                \"vle8.v v13, (%[q81])\\n\\t\"\n                \"vle8.v v14, (%[q82])\\n\\t\"\n                \"vand.vi v2, v2, 0xF\\n\\t\"\n                \"addi %[q81], %[q81], 64\\n\\t\"\n                \"vsrl.vi v7, v3, 4\\n\\t\"\n                \"addi %[q82], %[q82], 64\\n\\t\"\n                \"vwmul.vv v16, v0, v8\\n\\t\"\n                \"vle8.v v15, (%[q83])\\n\\t\"\n                \"vle8.v v0, (%[q40])\\n\\t\"\n                \"vand.vi v3, v3, 0xF\\n\\t\"\n                \"addi %[q83], %[q83], 64\\n\\t\"\n                \"vwmul.vv v24, v2, v12\\n\\t\"\n                \"vwmul.vv v20, v4, v10\\n\\t\"\n                \"vwmul.vv v28, v6, v14\\n\\t\"\n                \"vwmacc.vv v16, v1, v9\\n\\t\"\n                \"vle8.v v1, (%[q41])\\n\\t\"\n                \"vle8.v v2, (%[q42])\\n\\t\"\n                \"vwmacc.vv v24, v3, v13\\n\\t\"\n                \"vwmacc.vv v20, v5, v11\\n\\t\"\n                \"vwmacc.vv v28, v7, v15\\n\\t\"\n                \"addi %[q40], %[q80], 64\\n\\t\"\n                \"addi %[q41], %[q81], 64\\n\\t\"\n                \"vle8.v v3, (%[q43])\\n\\t\"\n                \"vle8.v v8, (%[q80])\\n\\t\"\n                \"addi %[q42], %[q82], 64\\n\\t\"\n                \"addi %[q43], %[q83], 64\\n\\t\"\n                \"vsrl.vi v4, v0, 4\\n\\t\"\n                \"vle8.v v9, (%[q81])\\n\\t\"\n                \"vle8.v v10, (%[q82])\\n\\t\"\n                \"vand.vi v0, v0, 0xF\\n\\t\"\n                \"vsrl.vi v5, v1, 4\\n\\t\"\n                \"vsrl.vi v7, v3, 4\\n\\t\"\n                \"vand.vi v3, v3, 0xF\\n\\t\"\n                \"vle8.v v11, (%[q83])\\n\\t\"\n                \"vle8.v v12, (%[q40])\\n\\t\"\n                \"vand.vi v1, v1, 0xF\\n\\t\"\n                \"vsrl.vi v6, v2, 4\\n\\t\"\n                \"vand.vi v2, v2, 0xF\\n\\t\"\n                \"vwmul.vv v18, v0, v8\\n\\t\"\n                \"vle8.v v13, (%[q41])\\n\\t\"\n                \"vle8.v v14, (%[q42])\\n\\t\"\n                \"vwmul.vv v26, v2, v12\\n\\t\"\n                \"vwmul.vv v22, v4, v10\\n\\t\"\n                \"vwmul.vv v30, v6, v14\\n\\t\"\n                \"vwmacc.vv v18, v1, v9\\n\\t\"\n                \"vle8.v v15, (%[q43])\\n\\t\"\n                \"vwmacc.vv v26, v3, v13\\n\\t\"\n                \"vwmacc.vv v22, v5, v11\\n\\t\"\n                \"vwmacc.vv v30, v7, v15\\n\\t\"\n                \"vmv.v.x v0, zero\\n\\t\"\n                \"vsetivli zero, 16, e16, m2, ta, ma\\n\\t\"\n                \"vwredsum.vs v4, v16, v0\\n\\t\"\n                \"lbu %[s0], 0(%[scale])\\n\\t\"\n                \"vwredsum.vs v5, v20, v0\\n\\t\"\n                \"lbu %[s1], 1(%[scale])\\n\\t\"\n                \"vwredsum.vs v6, v24, v0\\n\\t\"\n                \"lbu %[s2], 2(%[scale])\\n\\t\"\n                \"vwredsum.vs v7, v28, v0\\n\\t\"\n                \"lbu %[s3], 3(%[scale])\\n\\t\"\n                \"vwredsum.vs v8, v18, v0\\n\\t\"\n                \"lbu %[q40], 4(%[scale])\\n\\t\"\n                \"vwredsum.vs v9, v22, v0\\n\\t\"\n                \"lbu %[q41], 5(%[scale])\\n\\t\"\n                \"vwredsum.vs v10, v26, v0\\n\\t\"\n                \"lbu %[q42], 6(%[scale])\\n\\t\"\n                \"vwredsum.vs v11, v30, v0\\n\\t\"\n                \"lbu %[q43], 7(%[scale])\\n\\t\"\n                \"vsetivli zero, 4, e32, m1, ta, ma\\n\\t\"\n                \"vmul.vx v0, v4, %[s0]\\n\\t\"\n                \"vmul.vx v1, v8, %[q40]\\n\\t\"\n                \"vmacc.vx v0, %[s1], v5\\n\\t\"\n                \"vmacc.vx v1, %[q41], v9\\n\\t\"\n                \"vmacc.vx v0, %[s2], v6\\n\\t\"\n                \"vmacc.vx v1, %[q42], v10\\n\\t\"\n                \"vmacc.vx v0, %[s3], v7\\n\\t\"\n                \"vmacc.vx v1, %[q43], v11\\n\\t\"\n                \"vfcvt.f.x.v v0, v0\\n\\t\"\n                \"vfcvt.f.x.v v1, v1\\n\\t\"\n                \"vfmv.f.s %[ft2], v0\\n\\t\"\n                \"vfmv.f.s %[ftmp], v1\\n\\t\"\n                \"fadd.s %[ft2], %[ft2], %[ftmp]\\n\\t\"\n                \"fmadd.s %[sumf], %[d], %[ft2], %[sumf]\"\n                : [ftmp] \"=&f\" (ftmp), [sumf] \"+&f\" (sumf), [ft2] \"=&f\" (ft2)\n                , [s0] \"=&r\" (s0), [s1] \"=&r\" (s1), [s2] \"=&r\" (s2), [s3] \"=&r\" (s3)\n                , [q40] \"=&r\" (q40), [q41] \"=&r\" (q41), [q42] \"=&r\" (q42), [q43] \"=&r\" (q43)\n                , [q80] \"=&r\" (q80), [q81] \"=&r\" (q81), [q82] \"=&r\" (q82), [q83] \"=&r\" (q83)\n                : [d] \"f\" (d), [ys] \"r\" (y[i].qs), [xs] \"r\" (x[i].qs), [scale] \"r\" (scales)\n                , [bsums] \"r\" (y[i].bsums), [mins] \"r\" (mins), [utmp] \"r\" (utmp)\n                , [s6b] \"r\" (&x[i]), [kmask1] \"r\" (kmask1), [dmin] \"f\" (dmin)\n                , [kmask2] \"r\" (kmask2), [kmask3] \"r\" (kmask3)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n        }\n        break;\n    default:\n        assert(false && \"Unsupported vector length\");\n        break;\n    }\n\n    *s = sumf;\n\n#else\n\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(nb);\n    UNUSED(utmp);\n\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __riscv_v\n\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    const uint8_t * mins   = (const uint8_t*)&utmp[2];\n\n    float sumf = 0;\n    float sums = 0.0;\n\n    size_t vl;\n\n    for (int i = 0; i < nb; ++i) {\n\n        vl = 8;\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const uint8_t * GGML_RESTRICT hm = x[i].qh;\n        const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n\n        vint16m1_t q8sums_0 = __riscv_vlse16_v_i16m1(y[i].bsums, 4, vl);\n        vint16m1_t q8sums_1 = __riscv_vlse16_v_i16m1(y[i].bsums+1, 4, vl);\n        vint16m1_t q8sums = __riscv_vadd_vv_i16m1(q8sums_0, q8sums_1, vl);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        vuint8mf2_t mins8 = __riscv_vle8_v_u8mf2(mins, vl);\n        vint16m1_t v_mins = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(mins8, vl));\n        vint32m2_t prod = __riscv_vwmul_vv_i32m2(q8sums, v_mins, vl);\n\n        vint32m1_t sumi = __riscv_vredsum_vs_i32m2_i32m1(prod, __riscv_vmv_v_x_i32m1(0, 1), vl);\n        sumf -= dmin * __riscv_vmv_x_s_i32m1_i32(sumi);\n\n        vl = 32;\n        int32_t aux32 = 0;\n        int is = 0;\n\n        uint8_t m = 1;\n        vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n        vuint8m2_t vqh = __riscv_vle8_v_u8m2(hm, vl);\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            // load Q5 and Q8\n            vuint8m2_t q5_x = __riscv_vle8_v_u8m2(q5, vl);\n            vint8m2_t  q8_y1 = __riscv_vle8_v_i8m2(q8, vl);\n            vint8m2_t  q8_y2 = __riscv_vle8_v_i8m2(q8+32, vl);\n\n            // compute mask for addition\n            vint8m2_t q5_a = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vand_vx_u8m2(q5_x, 0x0F, vl));\n            vuint8m2_t qh_m1 = __riscv_vand_vx_u8m2(vqh, m, vl);\n            vbool4_t vmask_1 = __riscv_vmsne_vx_u8m2_b4(qh_m1, 0, vl);\n            vint8m2_t q5_m1 = __riscv_vadd_vx_i8m2_mu(vmask_1, q5_a, q5_a, 16, vl);\n            m <<= 1;\n\n            vint8m2_t q5_l = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vsrl_vx_u8m2(q5_x, 0x04, vl));\n            vuint8m2_t qh_m2 = __riscv_vand_vx_u8m2(vqh, m, vl);\n            vbool4_t vmask_2 = __riscv_vmsne_vx_u8m2_b4(qh_m2, 0, vl);\n            vint8m2_t q5_m2 = __riscv_vadd_vx_i8m2_mu(vmask_2, q5_l, q5_l, 16, vl);\n            m <<= 1;\n\n            vint16m4_t v0 = __riscv_vwmul_vv_i16m4(q5_m1, q8_y1, vl);\n            vint16m4_t v1 = __riscv_vwmul_vv_i16m4(q5_m2, q8_y2, vl);\n\n            vint32m8_t vs1 = __riscv_vwmul_vx_i32m8(v0, scales[is++], vl);\n            vint32m8_t vs2 = __riscv_vwmul_vx_i32m8(v1, scales[is++], vl);\n\n            vint32m1_t vacc1 = __riscv_vredsum_vs_i32m8_i32m1(vs1, vzero, vl);\n            vint32m1_t vacc2 = __riscv_vredsum_vs_i32m8_i32m1(vs2, vacc1, vl);\n\n            aux32 += __riscv_vmv_x_s_i32m1_i32(vacc2);\n            q5 += 32;    q8 += 64;\n\n        }\n\n        sums += aux32 * d;\n\n    }\n\n    *s = sumf+sums;\n\n#else\n\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(nb);\n    UNUSED(utmp);\n\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __riscv_xtheadvector\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t * restrict q6 = x[i].ql;\n        const uint8_t * restrict qh = x[i].qh;\n        const  int8_t * restrict q8 = y[i].qs;\n\n        const int8_t * restrict scale = x[i].scales;\n\n        int sum_t = 0;\n        int t0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            __asm__ __volatile__(\n                \"th.vsetvli zero, %[vl32], e8, m2\\n\\t\" // vl == 32\n                \"th.vlb.v v4, (%[qh])\\n\\t\"\n                \"th.vsll.vi v0, v4, 4\\n\\t\"\n                \"th.vsll.vi v2, v4, 2\\n\\t\"\n                \"th.vsrl.vi v6, v4, 2\\n\\t\"\n                \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\" // vl == 64\n                \"th.vlb.v v8, (%[q6])\\n\\t\"\n                \"th.vsrl.vi v12, v8, 4\\n\\t\"\n                \"th.vand.vi v8, v8, 0xF\\n\\t\"\n                \"th.vsetvli zero, %[vl128], e8, m8\\n\\t\" // vl == 128\n                \"th.vand.vx v0, v0, %[mask]\\n\\t\"\n                \"th.vor.vv v8, v8, v0\\n\\t\"\n                \"th.vlb.v v0, (%[q8])\\n\\t\"\n                \"th.vsub.vx v8, v8, %[vl32]\\n\\t\"\n                \"th.vsetvli zero, %[vl64], e8, m4\\n\\t\" // vl == 64\n                \"th.vwmul.vv v16, v0, v8\\n\\t\"\n                \"th.vwmul.vv v24, v4, v12\\n\\t\"\n                \"li %[t0], 16\\n\\t\"\n                \"th.vsetvli zero, %[t0], e16, m2\\n\\t\" // vl == 16\n                \"th.vmv.v.x v0, zero\\n\\t\"\n                \"th.vwredsum.vs v10, v16, v0\\n\\t\"\n                \"th.vwredsum.vs v9, v18, v0\\n\\t\"\n                \"th.vwredsum.vs v8, v20, v0\\n\\t\"\n                \"th.vwredsum.vs v7, v22, v0\\n\\t\"\n                \"th.vwredsum.vs v11, v24, v0\\n\\t\"\n                \"th.vwredsum.vs v12, v26, v0\\n\\t\"\n                \"th.vwredsum.vs v13, v28, v0\\n\\t\"\n                \"th.vwredsum.vs v14, v30, v0\\n\\t\"\n                \"li %[t0], 4\\n\\t\"\n                \"th.vsetvli zero, %[t0], e32, m1\\n\\t\" // vl == 4\n                \"th.vslideup.vi v10, v9, 1\\n\\t\"\n                \"th.vslideup.vi v8, v7, 1\\n\\t\"\n                \"th.vslideup.vi v11, v12, 1\\n\\t\"\n                \"th.vslideup.vi v13, v14, 1\\n\\t\"\n                \"th.vslideup.vi v10, v8, 2\\n\\t\"\n                \"th.vslideup.vi v11, v13, 2\\n\\t\"\n                \"li %[t0], 8\\n\\t\"\n                \"th.vsetvli zero, %[t0], e32, m2\\n\\t\" // vl == 8\n                \"th.vlb.v v4, (%[scale])\\n\\t\"\n                \"th.vmul.vv v2, v4, v10\\n\\t\"\n                \"th.vredsum.vs v0, v2, v0\\n\\t\"\n                \"th.vmv.x.s %[t0], v0\\n\\t\"\n                \"add %[sumi], %[sumi], %[t0]\"\n                : [sumi] \"+&r\" (sum_t), [t0] \"=&r\" (t0)\n                : [qh] \"r\" (qh), [q6] \"r\" (q6), [q8] \"r\" (q8), [scale] \"r\" (scale)\n                , [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n                , [mask] \"r\" (0x30)\n                : \"memory\"\n                , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n            );\n            q6 += 64;   qh += 32;   q8 += 128;   scale += 8;\n        }\n\n        sumf += d * sum_t;\n\n    }\n\n    *s = sumf;\n\n#elif defined __riscv_v\n\n    float sumf = 0;\n    const int vector_length = __riscv_vlenb() * 8;\n\n    switch (vector_length) {\n    case 256:\n        for (int i = 0; i < nb; ++i) {\n\n            const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n            const uint8_t * GGML_RESTRICT q6 = x[i].ql;\n            const uint8_t * GGML_RESTRICT qh = x[i].qh;\n            const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n\n            const int8_t * GGML_RESTRICT scale = x[i].scales;\n\n            size_t vl;\n\n            vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1);\n\n            int sum_t = 0;\n            int is = 0;\n\n            for (int j = 0; j < QK_K/128; ++j) {\n\n                vl = 32;\n\n                // load qh\n                vuint8m1_t qh_x = __riscv_vle8_v_u8m1(qh, vl);\n\n                // load Q6\n                vuint8m1_t q6_0 = __riscv_vle8_v_u8m1(q6, vl);\n                vuint8m1_t q6_1 = __riscv_vle8_v_u8m1(q6+32, vl);\n\n                vuint8m1_t q6a_0 = __riscv_vand_vx_u8m1(q6_0, 0x0F, vl);\n                vuint8m1_t q6a_1 = __riscv_vand_vx_u8m1(q6_1, 0x0F, vl);\n                vuint8m1_t q6s_0 = __riscv_vsrl_vx_u8m1(q6_0, 0x04, vl);\n                vuint8m1_t q6s_1 = __riscv_vsrl_vx_u8m1(q6_1, 0x04, vl);\n\n                vuint8m1_t qh_0 = __riscv_vand_vx_u8m1(qh_x, 0x03, vl);\n                vuint8m1_t qh_1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x2, vl), 0x03 , vl);\n                vuint8m1_t qh_2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x4, vl), 0x03 , vl);\n                vuint8m1_t qh_3 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(qh_x, 0x6, vl), 0x03 , vl);\n\n                vuint8m1_t qhi_0 = __riscv_vor_vv_u8m1(q6a_0, __riscv_vsll_vx_u8m1(qh_0, 0x04, vl), vl);\n                vuint8m1_t qhi_1 = __riscv_vor_vv_u8m1(q6a_1, __riscv_vsll_vx_u8m1(qh_1, 0x04, vl), vl);\n                vuint8m1_t qhi_2 = __riscv_vor_vv_u8m1(q6s_0, __riscv_vsll_vx_u8m1(qh_2, 0x04, vl), vl);\n                vuint8m1_t qhi_3 = __riscv_vor_vv_u8m1(q6s_1, __riscv_vsll_vx_u8m1(qh_3, 0x04, vl), vl);\n\n                vint8m1_t a_0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_0), 32, vl);\n                vint8m1_t a_1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_1), 32, vl);\n                vint8m1_t a_2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_2), 32, vl);\n                vint8m1_t a_3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(qhi_3), 32, vl);\n\n                // load Q8 and take product\n                vint16m2_t va_q_0 = __riscv_vwmul_vv_i16m2(a_0, __riscv_vle8_v_i8m1(q8, vl), vl);\n                vint16m2_t va_q_1 = __riscv_vwmul_vv_i16m2(a_1, __riscv_vle8_v_i8m1(q8+32, vl), vl);\n                vint16m2_t va_q_2 = __riscv_vwmul_vv_i16m2(a_2, __riscv_vle8_v_i8m1(q8+64, vl), vl);\n                vint16m2_t va_q_3 = __riscv_vwmul_vv_i16m2(a_3, __riscv_vle8_v_i8m1(q8+96, vl), vl);\n\n                vl = 16;\n\n                vint32m2_t vaux_0 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 0), scale[is+0], vl);\n                vint32m2_t vaux_1 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_0, 1), scale[is+1], vl);\n                vint32m2_t vaux_2 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 0), scale[is+2], vl);\n                vint32m2_t vaux_3 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_1, 1), scale[is+3], vl);\n                vint32m2_t vaux_4 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 0), scale[is+4], vl);\n                vint32m2_t vaux_5 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_2, 1), scale[is+5], vl);\n                vint32m2_t vaux_6 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 0), scale[is+6], vl);\n                vint32m2_t vaux_7 = __riscv_vwmul_vx_i32m2(__riscv_vget_v_i16m2_i16m1(va_q_3, 1), scale[is+7], vl);\n\n                vint32m1_t isum0 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_0, vaux_1, vl), vzero, vl);\n                vint32m1_t isum1 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_2, vaux_3, vl), isum0, vl);\n                vint32m1_t isum2 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_4, vaux_5, vl), isum1, vl);\n                vint32m1_t isum3 = __riscv_vredsum_vs_i32m2_i32m1(__riscv_vadd_vv_i32m2(vaux_6, vaux_7, vl), isum2, vl);\n\n                sum_t += __riscv_vmv_x_s_i32m1_i32(isum3);\n\n                q6 += 64;   qh += 32;   q8 += 128;   is=8;\n\n            }\n\n            sumf += d * sum_t;\n\n        }\n        break;\n    case 128:\n        for (int i = 0; i < nb; ++i) {\n\n            __builtin_prefetch(&x[i + 1].d, 0, 1);\n\n            const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n            const uint8_t * restrict q6 = x[i].ql;\n            const uint8_t * restrict qh = x[i].qh;\n            const  int8_t * restrict q8 = y[i].qs;\n\n            const int8_t * restrict scale = x[i].scales;\n\n            int q6h;\n            float ftmp;\n\n            for (int j = 0; j < QK_K/128; ++j) {\n                __asm__ __volatile__(\n                    \"addi %[q6h], %[q6], 32\\n\\t\"\n                    \"ld t0, 0(%[scale])\\n\\t\"\n                    \"addi %[scale], %[scale], 8\\n\\t\"\n                    \"slli t6, t0, 1 * 8\\n\\t\"\n                    \"lb zero, 0(%[q6])\\n\\t\"\n                    \"slli t5, t0, 2 * 8\\n\\t\"\n                    \"slli t4, t0, 3 * 8\\n\\t\"\n                    \"lb zero, 0(%[q6h])\\n\\t\"\n                    \"slli t3, t0, 4 * 8\\n\\t\"\n                    \"slli t2, t0, 5 * 8\\n\\t\"\n                    \"lb zero, 0(%[qh])\\n\\t\"\n                    \"lb zero, 31(%[q6h])\\n\\t\"\n                    \"slli t1, t0, 6 * 8\\n\\t\"\n                    \"srai a7, t0, 56\\n\\t\"\n                    \"vsetvli zero, %[vl32], e8, m2\\n\\t\"\n                    \"vle8.v v8, (%[q6])\\n\\t\"\n                    \"srai t6, t6, 56\\n\\t\"\n                    \"srai t5, t5, 56\\n\\t\"\n                    \"srai t4, t4, 56\\n\\t\"\n                    \"srai t3, t3, 56\\n\\t\"\n                    \"vle8.v v10, (%[q6h])\\n\\t\"\n                    \"addi %[q6], %[q6], 64\\n\\t\"\n                    \"slli t0, t0, 7 * 8\\n\\t\"\n                    \"srai t2, t2, 56\\n\\t\"\n                    \"srai t1, t1, 56\\n\\t\"\n                    \"srai t0, t0, 56\\n\\t\"\n                    \"vle8.v v4, (%[qh])\\n\\t\"\n                    \"vsrl.vi v12, v8, 4\\n\\t\"\n                    \"vsrl.vi v14, v10, 4\\n\\t\"\n                    \"lb zero, 0(%[q8])\\n\\t\"\n                    \"vand.vi v8, v8, 0xF\\n\\t\"\n                    \"vand.vi v10, v10, 0xF\\n\\t\"\n                    \"lb zero, 32(%[q8])\\n\\t\"\n                    \"vsll.vi v0, v4, 4\\n\\t\"\n                    \"vsll.vi v2, v4, 2\\n\\t\"\n                    \"lb zero, 64(%[q8])\\n\\t\"\n                    \"vsrl.vi v6, v4, 2\\n\\t\"\n                    \"vand.vx v0, v0, %[mask]\\n\\t\"\n                    \"lb zero, 96(%[q8])\\n\\t\"\n                    \"vand.vx v2, v2, %[mask]\\n\\t\"\n                    \"vand.vx v4, v4, %[mask]\\n\\t\"\n                    \"vand.vx v6, v6, %[mask]\\n\\t\"\n                    \"vor.vv v8, v8, v0\\n\\t\"\n                    \"lb zero, 127(%[q8])\\n\\t\"\n                    \"vor.vv v10, v10, v2\\n\\t\"\n                    \"vor.vv v12, v12, v4\\n\\t\"\n                    \"vor.vv v14, v14, v6\\n\\t\"\n                    \"vsetvli zero, %[vl128], e8, m8\\n\\t\"\n                    \"vle8.v v0, (%[q8])\\n\\t\"\n                    \"vsub.vx v8, v8, %[vl32]\\n\\t\"\n                    \"vsetvli zero, %[vl64], e8, m4\\n\\t\"\n                    \"vwmul.vv v16, v0, v8\\n\\t\"\n                    \"vwmul.vv v24, v4, v12\\n\\t\"\n                    \"vsetivli zero, 16, e16, m2\\n\\t\"\n                    \"vmv.v.x v0, zero\\n\\t\"\n                    \"vwredsum.vs v10, v16, v0\\n\\t\"\n                    \"vwredsum.vs v9, v18, v0\\n\\t\"\n                    \"vwredsum.vs v8, v20, v0\\n\\t\"\n                    \"vwredsum.vs v7, v22, v0\\n\\t\"\n                    \"vwredsum.vs v11, v24, v0\\n\\t\"\n                    \"vwredsum.vs v12, v26, v0\\n\\t\"\n                    \"vwredsum.vs v13, v28, v0\\n\\t\"\n                    \"vwredsum.vs v14, v30, v0\\n\\t\"\n                    \"vsetivli zero, 4, e32, m1\\n\\t\"\n                    \"vmul.vx v0, v10, t0\\n\\t\"\n                    \"vmul.vx v1, v9, t1\\n\\t\"\n                    \"vmacc.vx v0, t2, v8\\n\\t\"\n                    \"vmacc.vx v1, t3, v7\\n\\t\"\n                    \"vmacc.vx v0, t4, v11\\n\\t\"\n                    \"vmacc.vx v1, t5, v12\\n\\t\"\n                    \"vmacc.vx v0, t6, v13\\n\\t\"\n                    \"vmacc.vx v1, a7, v14\\n\\t\"\n                    \"vadd.vv v0, v0, v1\\n\\t\"\n                    \"vfcvt.f.x.v v0, v0\\n\\t\"\n                    \"vfmv.f.s %[ftmp], v0\\n\\t\"\n                    \"fmadd.s %[sumf], %[d], %[ftmp], %[sumf]\"\n                    : [q6] \"+&r\" (q6), [q6h] \"=&r\" (q6h)\n                    , [scale] \"+&r\" (scale)\n                    , [sumf] \"+&f\" (sumf), [ftmp] \"=&f\" (ftmp)\n                    : [qh] \"r\" (qh), [q8] \"r\" (q8)\n                    , [vl32] \"r\" (32), [vl64] \"r\" (64), [vl128] \"r\" (128)\n                    , [mask] \"r\" (0x30), [d] \"f\" (d)\n                    : \"memory\"\n                    , \"v0\", \"v1\", \"v2\", \"v3\", \"v4\", \"v5\", \"v6\", \"v7\"\n                    , \"v8\", \"v9\", \"v10\", \"v11\", \"v12\", \"v13\", \"v14\", \"v15\"\n                    , \"v16\", \"v17\", \"v18\", \"v19\", \"v20\", \"v21\", \"v22\", \"v23\"\n                    , \"v24\", \"v25\", \"v26\", \"v27\", \"v28\", \"v29\", \"v30\", \"v31\"\n                    , \"t0\", \"t1\", \"t2\", \"t3\", \"t4\", \"t5\", \"t6\", \"a7\"\n                    , \"a6\", \"a5\", \"a4\", \"a3\"\n                );\n                qh += 32;   q8 += 128;\n            }\n        }\n        break;\n    default:\n        assert(false && \"Unsupported vector length\");\n        break;\n    }\n\n    *s = sumf;\n\n#else\n\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        // Load qh once for the entire superblock.\n        vuint16mf2_t qh = __riscv_vle16_v_u16mf2(x[i].qh, 8);\n\n        // Calculate ls.\n        vuint16mf2_t temp = __riscv_vsrl_vx_u16mf2(qh, 12, 8);\n        temp = __riscv_vand_vx_u16mf2(temp, 7, 8);\n        vint32m1_t ls = __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vwmulu_vx_u32m1(temp, 2, 8));\n        ls = __riscv_vadd_vx_i32m1(ls, 1, 8);\n\n        // Calculate delta.\n        vbool32_t mask = __riscv_vmseq_vx_u16mf2_b32(__riscv_vand_vx_u16mf2(qh, 0x8000, 8), 0, 8);\n        vint32m1_t delta_neg = __riscv_vmv_v_x_i32m1(-1, 8);\n        vint32m1_t delta_pos = __riscv_vmv_v_x_i32m1(1, 8);\n        vint32m1_t delta = __riscv_vmerge_vvm_i32m1(delta_neg, delta_pos, mask, 8);\n\n        // Load qs.\n        vuint8m1_t qs = __riscv_vle8_v_u8m1(x[i].qs, 32);\n\n        // Prepare the indices.\n        const uint64_t shift = 0x0009000600030000;\n        vuint16m2_t qh_shift = __riscv_vreinterpret_v_u64m2_u16m2(__riscv_vmv_v_x_u64m2(shift, 8));\n        vuint16m2_t qh_gather_index = __riscv_vreinterpret_v_i16m2_u16m2(\n            __riscv_vdiv_vx_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vid_v_u16m2(32)), 4, 32));\n        vuint16m2_t qh_ext = __riscv_vlmul_ext_v_u16m1_u16m2(__riscv_vlmul_ext_v_u16mf2_u16m1(qh));\n        vuint16m2_t qh_index = __riscv_vrgather_vv_u16m2(qh_ext, qh_gather_index, 32);\n        qh_index = __riscv_vsrl_vv_u16m2(qh_index, qh_shift, 32);\n        qh_index = __riscv_vand_vx_u16m2(qh_index, 7, 32);\n        qh_index = __riscv_vsll_vx_u16m2(qh_index, 8, 32);\n        qh_index = __riscv_vor_vv_u16m2(qh_index, __riscv_vzext_vf2_u16m2(qs, 32), 32);\n        vuint16m2_t index = __riscv_vsll_vx_u16m2(qh_index, 3, 32);\n\n        // Final lsums.\n        int32_t lsums_s[8];\n        vint32m1_t one_scalar = __riscv_vmv_v_x_i32m1(0, 1);\n\n        // Sub-blocks 1-4\n        {\n            vuint16m1_t grid_index0 = __riscv_vget_v_u16m2_u16m1(index, 0);\n            vint8m4_t grid0 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index0, 16));\n            vint8m4_t q80 = __riscv_vle8_v_i8m4(y[i].qs, 128);\n            vint16m8_t lsum0 = __riscv_vwmul_vv_i16m8(grid0, q80, 128);\n            lsums_s[0] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 0), one_scalar, 32));\n            lsums_s[1] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 1), one_scalar, 32));\n            lsums_s[2] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 2), one_scalar, 32));\n            lsums_s[3] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum0, 3), one_scalar, 32));\n        }\n        __asm__ __volatile__(\"\" ::: \"memory\");\n        // Sub-blocks 5-8\n        {\n            vuint16m1_t grid_index1 = __riscv_vget_v_u16m2_u16m1(index, 1);\n            vint8m4_t grid1 = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vluxei16_v_i64m4((const int64_t*)iq1s_grid, grid_index1, 16));\n            vint8m4_t q81 = __riscv_vle8_v_i8m4(&y[i].qs[128], 128);\n            vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(grid1, q81, 128);\n            lsums_s[4] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 0), one_scalar, 32));\n            lsums_s[5] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 1), one_scalar, 32));\n            lsums_s[6] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 2), one_scalar, 32));\n            lsums_s[7] = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(lsum1, 3), one_scalar, 32));\n        }\n        __asm__ __volatile__(\"\" ::: \"memory\");\n        vint32m1_t lsums = __riscv_vle32_v_i32m1(&lsums_s[0], 8);\n\n        // Calculate the bsums.\n        vint16m1_t bsums_0 = __riscv_vle16_v_i16m1(y[i].bsums, 16);\n        const vuint32m1_t bsums_i32 = __riscv_vreinterpret_v_u16m1_u32m1(__riscv_vreinterpret_v_i16m1_u16m1(bsums_0));\n        const vint16mf2_t bsums_i32_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 0, 8));\n        const vint16mf2_t bsums_i32_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(bsums_i32, 16, 8));\n        const vint32m1_t bsums = __riscv_vwadd_vv_i32m1(bsums_i32_0, bsums_i32_1, 8);\n\n        // Accumulation.\n        vint32m1_t sumi_v = __riscv_vmul_vv_i32m1(ls, lsums, 8);\n        vint32m1_t sumi1_v = __riscv_vmul_vv_i32m1(__riscv_vmul_vv_i32m1(ls, delta, 8), bsums, 8);\n\n        // Update sumf.\n        int sumi = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));\n        int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m1_i32m1(sumi1_v, __riscv_vmv_v_x_i32m1(0.0f, 1), 8));\n        sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_m * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    iq1m_scale_t scale;\n    float sumf = 0.0f;\n    for (int i = 0; i < nb; ++i) {\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint8_t  * qh = x[i].qh;\n        const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n        scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n        // Accumulators.\n        vint32m2_t acc1 = __riscv_vmv_v_x_i32m2(0, 16);\n        vint32m2_t acc2 = __riscv_vmv_v_x_i32m2(0, 16);\n\n        // We process 4 sub-blocks together.\n        for (int ib = 0; ib < QK_K/128; ib++) {\n            // Load qh for 4 sub-blocks.\n            const vuint8mf4_t qh_8 = __riscv_vle8_v_u8mf4(qh, 8);\n            const vuint16mf2_t qh_16_lo = __riscv_vzext_vf2_u16mf2(qh_8, 8);\n            const vuint16mf2_t qh_16_hi = __riscv_vsll_vx_u16mf2(qh_16_lo, 8, 8);\n            const vuint16m1_t qhb = __riscv_vzext_vf2_u16m1(\n                __riscv_vreinterpret_v_u16mf2_u8mf2(__riscv_vor_vv_u16mf2(qh_16_lo, qh_16_hi, 8)), 16);\n            qh += 8;\n\n            // Prepare grid indices.\n            const vuint16m1_t qsb = __riscv_vzext_vf2_u16m1(__riscv_vle8_v_u8mf2(&qs[0], 16), 16);\n            const vuint16m1_t shift = __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00040008, 8));\n            vuint16m1_t index = __riscv_vor_vv_u16m1(qsb, __riscv_vand_vx_u16m1(__riscv_vsll_vv_u16m1(qhb, shift, 16), 0x700, 16), 16);\n            index = __riscv_vsll_vx_u16m1(index, 3, 16);\n            qs += 16;\n\n            // Load the grid.\n            const vint8m4_t iq1b = __riscv_vreinterpret_v_i64m4_i8m4(__riscv_vreinterpret_v_u64m4_i64m4(\n                __riscv_vluxei16_v_u64m4(iq1s_grid, index, 16)));\n\n            // Prepare the deltas.\n            const vbool16_t mask = __riscv_vmsgtu_vx_u16m1_b16(\n                __riscv_vand_vv_u16m1(qhb, __riscv_vreinterpret_v_u32m1_u16m1(__riscv_vmv_v_x_u32m1(0x00800008, 8)), 16), 0, 16);\n            const vint64m4_t delta_pos = __riscv_vmv_v_x_i64m4(0x0101010101010101, 16);\n            const vint64m4_t delta_neg = __riscv_vmv_v_x_i64m4(0xffffffffffffffff, 16);\n            const vint8m4_t delta = __riscv_vreinterpret_v_i64m4_i8m4(\n                __riscv_vmerge_vvm_i64m4(delta_pos, delta_neg, mask, 16));\n\n            // Load q8 for sub-blocks.\n            const vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 128);\n            q8 += 128;\n\n            // Calculate the lsums.\n            const vint16m8_t lsum1 = __riscv_vwmul_vv_i16m8(iq1b, q8b, 128);\n            const vint16m8_t lsum2 = __riscv_vwmul_vv_i16m8(delta, q8b, 128);\n\n            // Prepare the scales.\n            const int16_t ls_0_0 = 2*((sc[0] >> 0) & 0x7) + 1;\n            const int16_t ls_0_1 = 2*((sc[0] >> 3) & 0x7) + 1;\n            const int16_t ls_1_0 = 2*((sc[0] >> 6) & 0x7) + 1;\n            const int16_t ls_1_1 = 2*((sc[0] >> 9) & 0x7) + 1;\n            const int16_t ls_2_0 = 2*((sc[1] >> 0) & 0x7) + 1;\n            const int16_t ls_2_1 = 2*((sc[1] >> 3) & 0x7) + 1;\n            const int16_t ls_3_0 = 2*((sc[1] >> 6) & 0x7) + 1;\n            const int16_t ls_3_1 = 2*((sc[1] >> 9) & 0x7) + 1;\n            sc += 2;\n\n            // Accumulate in acc0 and acc1 for each sub-block.\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum1, 0), 16);\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum1, 1), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_0, __riscv_vget_v_i16m8_i16m1(lsum2, 0), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_0_1, __riscv_vget_v_i16m8_i16m1(lsum2, 1), 16);\n            //\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum1, 2), 16);\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum1, 3), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_0, __riscv_vget_v_i16m8_i16m1(lsum2, 2), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_1_1, __riscv_vget_v_i16m8_i16m1(lsum2, 3), 16);\n            //\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum1, 4), 16);\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum1, 5), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_0, __riscv_vget_v_i16m8_i16m1(lsum2, 4), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_2_1, __riscv_vget_v_i16m8_i16m1(lsum2, 5), 16);\n            //\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum1, 6), 16);\n            acc1 = __riscv_vwmacc_vx_i32m2(acc1, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum1, 7), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_0, __riscv_vget_v_i16m8_i16m1(lsum2, 6), 16);\n            acc2 = __riscv_vwmacc_vx_i32m2(acc2, ls_3_1, __riscv_vget_v_i16m8_i16m1(lsum2, 7), 16);\n        }\n\n        // Reduce and accumulate in `sumf`.\n        vint32m1_t one = __riscv_vmv_v_x_i32m1(0, 1);\n        int sumi1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc1, one, 16));\n        int sumi2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vredsum_vs_i32m2_i32m1(acc2, one, 16));\n        sumf += y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16) * (sumi1 + IQ1M_DELTA * sumi2);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic const uint8_t sign_gather_indices_arr[64] = {\n    0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,\n    4,4,4,4,4,4,4,4, 5,5,5,5,5,5,5,5, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7\n};\n\nstatic const uint8_t sign_bit_masks_arr[64] = {\n    1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128,\n    1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128\n};\n\n\nstatic void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    const uint64_t * grid64 = (const uint64_t *)iq2s_grid;\n\n    // Pre-load Constants\n    vuint8m2_t v_ids = __riscv_vid_v_u8m2(32);\n    vuint8m2_t v_sign_gather_indices = __riscv_vsrl_vx_u8m2(v_ids, 3, 32);\n    vuint8m2_t v_ones = __riscv_vmv_v_x_u8m2(1, 32);\n    vuint8m2_t v_shift_amts = __riscv_vand_vx_u8m2(v_ids, 7, 32);\n    vuint8m2_t v_sign_masks = __riscv_vsll_vv_u8m2(v_ones, v_shift_amts, 32);\n    uint16_t shift_qh_arr[4] = {11, 9, 7, 5};\n    vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 4);\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint8_t * GGML_RESTRICT scales = x[i].scales;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const uint8_t * signs_ptr = qs + 32;\n        float sum_block = 0.0f;\n\n        for (int ib = 0; ib < 8; ++ib) {\n\n            // Load Low Bits [4 bytes]\n            vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 4);\n            qs += 4;\n\n            // Load 1 byte. It contains bits for 4 mini-blocks.\n            uint8_t qh_val = *qh++;\n\n            // Combine Low + High bits of 10bit indices\n            vuint8mf4_t v_qh_raw = __riscv_vmv_v_x_u8mf4(qh_val, 4);\n            vuint16mf2_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qh_raw, 4);\n            vuint16mf2_t v_qh_mf2 = __riscv_vsll_vv_u16mf2(v_qh_u16, v_shift_qh, 4);\n            v_qh_mf2 = __riscv_vand_vx_u16mf2(v_qh_mf2, 0x1800, 4);\n            vuint16mf2_t v_qs_u16_mf2 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 4);\n            vuint16mf2_t v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16_mf2, 3, 4);\n            vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_mf2, 4);\n\n            // Lookup Grid\n            vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(__riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 4)));\n\n            vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 4);\n            signs_ptr += 4;\n            vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n            vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 32);\n\n            // generating sign mask\n            vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 32);\n            vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 32);\n\n            vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 32);\n            q8 += 32;\n\n            // apply signs\n            vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative,v_q8, v_q8, 0, 32);\n            vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 32);\n\n            // Reduction\n            vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n            // Reduce 0-15 (First Half)\n            int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(\n                __riscv_vget_v_i16m4_i16m2(v_dot, 0), v_zero, 16));\n\n            // Reduce 16-31 (Second Half)\n            int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(\n                __riscv_vget_v_i16m4_i16m2(v_dot, 1), v_zero, 16));\n\n            // Apply sub Scales\n            uint8_t sc = *scales++;\n\n            sum_block += s0 * (2 * (sc & 0xF) + 1);\n            sum_block += s1 * (2 * (sc >> 4)  + 1);\n        }\n        sumf += sum_block * combined_scale;\n    }\n    *s = 0.125f * sumf;\n}\n\nstatic void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    const uint64_t * grid64 = (const uint64_t *)iq2s_grid;\n\n    // --- Pre-load Constants ---\n    uint16_t gather_qh_arr[8] = {0, 0, 0, 0, 1, 1, 1, 1};\n    vuint16mf2_t v_gather_qh = __riscv_vle16_v_u16mf2(gather_qh_arr, 8);\n    uint16_t shift_qh_arr[8] = {11, 9, 7, 5, 11, 9, 7, 5};\n    vuint16mf2_t v_shift_qh = __riscv_vle16_v_u16mf2(shift_qh_arr, 8);\n\n    // Constants for sign extraction\n    vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);\n    vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint8_t * GGML_RESTRICT scales = x[i].scales;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const uint8_t * signs_ptr = qs + 32;\n\n        float sum_block = 0.0f;\n\n        for (int ib = 0; ib < 4; ++ib) {\n            // Combine low + high bits\n            vuint8mf4_t v_qs_u8 = __riscv_vle8_v_u8mf4(qs, 8);\n            qs += 8;\n            uint16_t qh_val;\n            memcpy(&qh_val, qh, 2);\n            qh += 2;\n            vuint8mf8_t v_qh_raw = __riscv_vle8_v_u8mf8((const uint8_t*)&qh_val, 2);\n            vuint16mf4_t v_qh_u16 = __riscv_vwcvtu_x_x_v_u16mf4(v_qh_raw, 2);\n            vuint16mf2_t v_qh_u16_ext = __riscv_vlmul_ext_v_u16mf4_u16mf2(v_qh_u16);\n            vuint16mf2_t v_qh_expanded = __riscv_vrgather_vv_u16mf2(v_qh_u16_ext, v_gather_qh, 8);\n            v_qh_expanded = __riscv_vsll_vv_u16mf2(v_qh_expanded, v_shift_qh, 8);\n\n            // Mask: We want bits 11-12. 0x1800 = 0001 1000 0000 0000\n            v_qh_expanded = __riscv_vand_vx_u16mf2(v_qh_expanded, 0x1800, 8);\n            vuint16mf2_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16mf2(v_qs_u8, 8);\n\n            // Multiply by 8 to get byte offset, instead of element offset\n            v_qs_u16 = __riscv_vsll_vx_u16mf2(v_qs_u16, 3, 8);\n            vuint16mf2_t v_grid_offsets = __riscv_vor_vv_u16mf2(v_qs_u16, v_qh_expanded, 8);\n\n            // Lookup Grid using Byte Offsets\n            vuint64m2_t v_grid_vals = __riscv_vluxei16_v_u64m2(grid64, v_grid_offsets, 8);\n\n            vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u64m2_u8m2(v_grid_vals);\n            vint8m2_t v_grid_i8 = __riscv_vreinterpret_v_u8m2_i8m2(v_grid_u8);\n\n            // Load signs and generate sign mask\n            vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs_ptr, 8);\n            signs_ptr += 8;\n\n            vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n            vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);\n\n            vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);\n            vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);\n\n            vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n            q8 += 64;\n\n            vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);\n            vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_grid_i8, v_q8_signed, 64);\n\n            vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n            int32_t s0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                __riscv_vget_v_i16m4_i16m1(v_dot, 0), v_zero, 16));\n            int32_t s1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                __riscv_vget_v_i16m4_i16m1(v_dot, 1), v_zero, 16));\n            int32_t s2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                __riscv_vget_v_i16m4_i16m1(v_dot, 2), v_zero, 16));\n            int32_t s3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                __riscv_vget_v_i16m4_i16m1(v_dot, 3), v_zero, 16));\n\n            uint8_t sc0 = scales[0];\n            uint8_t sc1 = scales[1];\n            scales += 2;\n\n            sum_block += s0 * (2 * (sc0 & 0xF) + 1);\n            sum_block += s1 * (2 * (sc0 >> 4)  + 1);\n            sum_block += s2 * (2 * (sc1 & 0xF) + 1);\n            sum_block += s3 * (2 * (sc1 >> 4)  + 1);\n        }\n        sumf += sum_block * combined_scale;\n    }\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 128:\n            ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        case 256:\n            ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined(__riscv_v)\nstatic const int8_t keven_signs_q2xs[1024] = {\n     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nstatic void ggml_vec_dot_iq2_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n    const uint64_t * grid64  = (const uint64_t *)iq2xs_grid;\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT qs = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        const uint8_t  * GGML_RESTRICT scales = x[i].scales;\n\n        int32_t sum_int = 0;\n\n        // Loop over 4 subblocks of 64 elements (QK_K = 256)\n        for (int ib64 = 0; ib64 < QK_K / 64; ++ib64) {\n            // Load 8 uint16 indices (controls 64 values)\n            vuint16mf2_t v_qs = __riscv_vle16_v_u16mf2(qs, 8);\n            qs += 8;\n\n            // Extract indices for grid (low 9 bits) and signs (high 7 bits)\n            // Multiply by 8 (<< 3) for byte offsets into the uint64 tables\n            vuint16mf2_t vidx_grid = __riscv_vsll_vx_u16mf2(__riscv_vand_vx_u16mf2(v_qs, 511, 8), 3, 8);\n            vuint16mf2_t vidx_sign = __riscv_vsll_vx_u16mf2(__riscv_vsrl_vx_u16mf2(v_qs, 9, 8), 3, 8);\n\n            vuint64m2_t vq2_64 = __riscv_vluxei16_v_u64m2(grid64, vidx_grid, 8);\n            vuint64m2_t vs2_64 = __riscv_vluxei16_v_u64m2(signs64, vidx_sign, 8);\n\n            vint8m2_t q2u = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64));\n            vint8m2_t q2s = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64));\n\n            vint8m2_t q2_final = __riscv_vmul_vv_i8m2(q2u, q2s, 64);\n\n            vint8m2_t q8v = __riscv_vle8_v_i8m2(q8, 64);\n            q8 += 64;\n\n            vint16m4_t prod = __riscv_vwmul_vv_i16m4(q2_final, q8v, 64);\n\n            vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n\n            int32_t sum0 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                           __riscv_vget_v_i16m4_i16m1(prod, 0), zero_vec, 16));\n            int32_t sum1 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                           __riscv_vget_v_i16m4_i16m1(prod, 1), zero_vec, 16));\n            int32_t sum2 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                           __riscv_vget_v_i16m4_i16m1(prod, 2), zero_vec, 16));\n            int32_t sum3 = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m1_i32m1(\n                           __riscv_vget_v_i16m4_i16m1(prod, 3), zero_vec, 16));\n\n            const uint8_t scale_byte_1 = scales[0];\n            const uint8_t scale_byte_2 = scales[1];\n            scales += 2;\n\n            sum_int += sum0 * ((scale_byte_1 & 0x0F) * 2 + 1);\n            sum_int += sum1 * ((scale_byte_1 >> 4)   * 2 + 1);\n            sum_int += sum2 * ((scale_byte_2 & 0x0F) * 2 + 1);\n            sum_int += sum3 * ((scale_byte_2 >> 4)   * 2 + 1);\n        }\n\n        sumf += d * sum_int;\n    }\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n      switch (__riscv_vlenb() * 8) {\n          case 256:\n              ggml_vec_dot_iq2_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n              break;\n          default:\n              ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n              break;\n      }\n#else\n    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n    const uint64_t * grid64  = (const uint64_t *)iq2xxs_grid;\n\n    uint32_t shift_constants[4] = {0, 7, 14, 21};\n    vuint32m1_t v_shifts = __riscv_vle32_v_u32m1(shift_constants, 4);\n\n    float sumf = 0.0f;\n    for (int i = 0; i < nb; ++i) {\n        const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t  * GGML_RESTRICT q2_ptr = (const uint8_t *) x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        float sum = 0.0f;\n\n        #pragma GCC unroll 1\n        for (int ib32 = 0; ib32 < QK_K / 32; ib32 += 2) {\n            vint8m2_t q8_1 = __riscv_vle8_v_i8m2(q8, 32); q8 += 32;\n            vint8m2_t q8_2 = __riscv_vle8_v_i8m2(q8, 32); q8 += 32;\n\n            vuint8mf4_t v_raw_q2_1 = __riscv_vle8_v_u8mf4(q2_ptr, 4);\n            vuint8mf4_t v_raw_q2_2 = __riscv_vle8_v_u8mf4(q2_ptr + 8, 4);\n\n            vuint16mf2_t vidx_q2_1 = __riscv_vwcvtu_x_x_v_u16mf2(v_raw_q2_1, 4);\n            vuint16mf2_t vidx_q2_2 = __riscv_vwcvtu_x_x_v_u16mf2(v_raw_q2_2, 4);\n\n            vidx_q2_1 = __riscv_vsll_vx_u16mf2(vidx_q2_1, 3, 4);\n            vidx_q2_2 = __riscv_vsll_vx_u16mf2(vidx_q2_2, 3, 4);\n\n            uint32_t s_packed_1, s_packed_2;\n            memcpy(&s_packed_1, q2_ptr + 4, 4);\n            memcpy(&s_packed_2, q2_ptr + 12, 4);\n\n            vuint32m1_t v_s_1 = __riscv_vmv_v_x_u32m1(s_packed_1, 4);\n            vuint32m1_t v_s_2 = __riscv_vmv_v_x_u32m1(s_packed_2, 4);\n            v_s_1 = __riscv_vsrl_vv_u32m1(v_s_1, v_shifts, 4);\n            v_s_2 = __riscv_vsrl_vv_u32m1(v_s_2, v_shifts, 4);\n\n            v_s_1 = __riscv_vand_vx_u32m1(v_s_1, 127, 4);\n            v_s_2 = __riscv_vand_vx_u32m1(v_s_2, 127, 4);\n\n            vuint16mf2_t vidx_s2_1 = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_1, 4), 3, 4);\n            vuint16mf2_t vidx_s2_2 = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_2, 4), 3, 4);\n\n            vuint64m2_t vq2_64_1 = __riscv_vluxei16_v_u64m2(grid64, vidx_q2_1, 4);\n            vuint64m2_t vq2_64_2 = __riscv_vluxei16_v_u64m2(grid64, vidx_q2_2, 4);\n\n            vint8m2_t q2_1 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64_1));\n            vint8m2_t q2_2 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vq2_64_2));\n\n            vuint64m2_t vs2_64_1 = __riscv_vluxei16_v_u64m2(signs64, vidx_s2_1, 4);\n            vuint64m2_t vs2_64_2 = __riscv_vluxei16_v_u64m2(signs64, vidx_s2_2, 4);\n            vint8m2_t s2_1 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64_1));\n            vint8m2_t s2_2 = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(vs2_64_2));\n\n            vint8m2_t q8s_1 = __riscv_vmul_vv_i8m2(q8_1, s2_1, 32);\n            vint8m2_t q8s_2 = __riscv_vmul_vv_i8m2(q8_2, s2_2, 32);\n\n            vint16m4_t dot1 = __riscv_vwmul_vv_i16m4(q8s_1, q2_1, 32);\n            vint16m4_t dot2 = __riscv_vwmul_vv_i16m4(q8s_2, q2_2, 32);\n\n            vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n            vint32m1_t sumv1 = __riscv_vwredsum_vs_i16m4_i32m1(dot1, zero_vec, 32);\n            vint32m1_t sumv2 = __riscv_vwredsum_vs_i16m4_i32m1(dot2, zero_vec, 32);\n\n            int32_t scalar_sum1 = __riscv_vmv_x_s_i32m1_i32(sumv1);\n            int32_t scalar_sum2 = __riscv_vmv_x_s_i32m1_i32(sumv2);\n\n            int16_t scale1 = 2 * ((s_packed_1 >> 28) & 0xF) + 1;\n            int16_t scale2 = 2 * ((s_packed_2 >> 28) & 0xF) + 1;\n\n            sum += scalar_sum1 * scale1 + scalar_sum2 * scale2;\n            q2_ptr += 16;\n        }\n        sumf += sum * combined_scale;\n    }\n    *s = 0.125f * sumf;\n}\n\nstatic void ggml_vec_dot_iq2_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n    const uint64_t * grid64  = (const uint64_t *)iq2xxs_grid;\n\n    uint32_t shift_constants[4] = {0, 7, 14, 21};\n    vuint32mf2_t v_shifts = __riscv_vle32_v_u32mf2(shift_constants, 4);\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float combined_scale = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t  * GGML_RESTRICT q2_ptr = (const uint8_t *) x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        float sum = 0.0f;\n\n        for (int ib32 = 0; ib32 < QK_K / 32; ib32 += 2) {\n            vint8m1_t q8_1 = __riscv_vle8_v_i8m1(q8, 32); q8 += 32;\n            vint8m1_t q8_2 = __riscv_vle8_v_i8m1(q8, 32); q8 += 32;\n\n            vuint8mf8_t v_raw_q2_1 = __riscv_vle8_v_u8mf8(q2_ptr, 4);\n            vuint8mf8_t v_raw_q2_2 = __riscv_vle8_v_u8mf8(q2_ptr + 8, 4);\n\n            vuint16mf4_t vidx_q2_1 = __riscv_vwcvtu_x_x_v_u16mf4(v_raw_q2_1, 4);\n            vuint16mf4_t vidx_q2_2 = __riscv_vwcvtu_x_x_v_u16mf4(v_raw_q2_2, 4);\n\n            vidx_q2_1 = __riscv_vsll_vx_u16mf4(vidx_q2_1, 3, 4);\n            vidx_q2_2 = __riscv_vsll_vx_u16mf4(vidx_q2_2, 3, 4);\n\n            uint32_t s_packed_1, s_packed_2;\n            memcpy(&s_packed_1, q2_ptr + 4, 4);\n            memcpy(&s_packed_2, q2_ptr + 12, 4);\n\n            vuint32mf2_t v_s_1 = __riscv_vmv_v_x_u32mf2(s_packed_1, 4);\n            vuint32mf2_t v_s_2 = __riscv_vmv_v_x_u32mf2(s_packed_2, 4);\n\n            v_s_1 = __riscv_vsrl_vv_u32mf2(v_s_1, v_shifts, 4);\n            v_s_2 = __riscv_vsrl_vv_u32mf2(v_s_2, v_shifts, 4);\n\n            v_s_1 = __riscv_vand_vx_u32mf2(v_s_1, 127, 4);\n            v_s_2 = __riscv_vand_vx_u32mf2(v_s_2, 127, 4);\n\n            // Narrow u32 -> u16 (vncvt) and Scale by 8 to get byte offsets\n            vuint16mf4_t vidx_s2_1 = __riscv_vsll_vx_u16mf4(__riscv_vncvt_x_x_w_u16mf4(v_s_1, 4), 3, 4);\n            vuint16mf4_t vidx_s2_2 = __riscv_vsll_vx_u16mf4(__riscv_vncvt_x_x_w_u16mf4(v_s_2, 4), 3, 4);\n\n            // Load q2 values from lookup grid\n            vuint64m1_t vq2_64_1 = __riscv_vluxei16_v_u64m1(grid64, vidx_q2_1, 4);\n            vuint64m1_t vq2_64_2 = __riscv_vluxei16_v_u64m1(grid64, vidx_q2_2, 4);\n            vint8m1_t q2_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vq2_64_1));\n            vint8m1_t q2_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vq2_64_2));\n\n            // Load sign values\n            vuint64m1_t vs2_64_1 = __riscv_vluxei16_v_u64m1(signs64, vidx_s2_1, 4);\n            vuint64m1_t vs2_64_2 = __riscv_vluxei16_v_u64m1(signs64, vidx_s2_2, 4);\n            vint8m1_t s2_1 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vs2_64_1));\n            vint8m1_t s2_2 = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vreinterpret_v_u64m1_u8m1(vs2_64_2));\n\n            // Apply signs to q8\n            vint8m1_t q8s_1 = __riscv_vmul_vv_i8m1(q8_1, s2_1, 32);\n            vint8m1_t q8s_2 = __riscv_vmul_vv_i8m1(q8_2, s2_2, 32);\n\n            // multiplying q2 with q8\n            vint16m2_t dot1 = __riscv_vwmul_vv_i16m2(q8s_1, q2_1, 32);\n            vint16m2_t dot2 = __riscv_vwmul_vv_i16m2(q8s_2, q2_2, 32);\n\n            vint32m1_t zero_vec = __riscv_vmv_v_x_i32m1(0, 1);\n            vint32m1_t sumv1 = __riscv_vwredsum_vs_i16m2_i32m1(dot1, zero_vec, 32);\n            vint32m1_t sumv2 = __riscv_vwredsum_vs_i16m2_i32m1(dot2, zero_vec, 32);\n            int32_t scalar_sum1 = __riscv_vmv_x_s_i32m1_i32(sumv1);\n            int32_t scalar_sum2 = __riscv_vmv_x_s_i32m1_i32(sumv2);\n            int16_t scale1 = 2 * ((s_packed_1 >> 28) & 0xF) + 1;\n            int16_t scale2 = 2 * ((s_packed_2 >> 28) & 0xF) + 1;\n\n            sum += scalar_sum1 * scale1 + scalar_sum2 * scale2;\n            q2_ptr += 16;\n        }\n        sumf += sum * combined_scale;\n    }\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 128:\n            ggml_vec_dot_iq2_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq2_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq2_xxs_q8_K(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    const uint64_t * grid64 = (const uint64_t *)iq3s_grid;\n\n    // --- Pre-load Constants ---\n    const uint16_t qh_bit_shifts_arr[16] = {\n        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\n    };\n    vuint8m2_t v_sign_gather_indices = __riscv_vle8_v_u8m2(sign_gather_indices_arr, 64);\n    vuint8m2_t v_sign_masks = __riscv_vle8_v_u8m2(sign_bit_masks_arr, 64);\n    vuint16m1_t v_qh_shifts = __riscv_vle16_v_u16m1(qh_bit_shifts_arr, 16);\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float combined_scale = d * y[i].d;\n\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint8_t * GGML_RESTRICT scales = x[i].scales;\n        const uint8_t * GGML_RESTRICT signs = x[i].signs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        float sum_block = 0.0f;\n\n        // Loop: Process 64 weights (16 mini-blocks of 4) per iteration\n        for (int ib = 0; ib < 4; ++ib) {\n\n            vuint8mf2_t v_qs_u8 = __riscv_vle8_v_u8mf2(qs, 16);\n            qs += 16;\n\n            uint16_t qh_val;\n            memcpy(&qh_val, qh, 2);\n            qh += 2;\n\n            vuint16m1_t v_qh_val = __riscv_vmv_v_x_u16m1(qh_val, 16);\n            // Extract bits: (qh >> i) & 1\n            v_qh_val = __riscv_vsrl_vv_u16m1(v_qh_val, v_qh_shifts, 16);\n            v_qh_val = __riscv_vand_vx_u16m1(v_qh_val, 1, 16);\n\n            vuint16m1_t v_qs_u16 = __riscv_vwcvtu_x_x_v_u16m1(v_qs_u8, 16);\n            v_qs_u16 = __riscv_vsll_vx_u16m1(v_qs_u16, 2, 16);\n            v_qh_val = __riscv_vsll_vx_u16m1(v_qh_val, 10, 16);\n            vuint16m1_t v_grid_offsets = __riscv_vor_vv_u16m1(v_qs_u16, v_qh_val, 16);\n\n            // Grid value is 4xuint8\n            vuint32m2_t v_grid_packed = __riscv_vluxei16_v_u32m2((const uint32_t *)grid64, v_grid_offsets, 16);\n            vuint8m2_t v_grid_u8 = __riscv_vreinterpret_v_u32m2_u8m2(v_grid_packed);\n            vuint8mf4_t v_signs_raw = __riscv_vle8_v_u8mf4(signs, 8);\n            signs += 8;\n\n            // Generate sign mask\n            vuint8m2_t v_signs_source = __riscv_vlmul_ext_v_u8mf4_u8m2(v_signs_raw);\n            vuint8m2_t v_signs_bcast = __riscv_vrgather_vv_u8m2(v_signs_source, v_sign_gather_indices, 64);\n            vuint8m2_t v_sign_bits = __riscv_vand_vv_u8m2(v_signs_bcast, v_sign_masks, 64);\n            vbool4_t m_negative = __riscv_vmsne_vx_u8m2_b4(v_sign_bits, 0, 64);\n\n            vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n            q8 += 64;\n\n            // Apply Signs\n            vint8m2_t v_q8_signed = __riscv_vrsub_vx_i8m2_mu(m_negative, v_q8, v_q8, 0, 64);\n            vint16m4_t v_dot = __riscv_vwmulsu_vv_i16m4(v_q8_signed, v_grid_u8, 64);\n\n            // Reduction\n            vint16m2_t v_dot_lo = __riscv_vget_v_i16m4_i16m2(v_dot, 0);\n            vint16m2_t v_dot_hi = __riscv_vget_v_i16m4_i16m2(v_dot, 1);\n            vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n\n            int32_t s_lo = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_lo, v_zero, 32));\n            int32_t s_hi = __riscv_vmv_x_s_i32m1_i32(__riscv_vwredsum_vs_i16m2_i32m1(v_dot_hi, v_zero, 32));\n\n            // Apply sub-scales\n            uint8_t sc_byte = *scales++;\n            int sc_lo = (sc_byte & 0xF) * 2 + 1;\n            int sc_hi = (sc_byte >> 4)  * 2 + 1;\n\n            sum_block += s_lo * sc_lo + s_hi * sc_hi;\n        }\n        sumf += sum_block * combined_scale;\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq3_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n    const int nb = n / QK_K;\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n    const uint32_t * grid32  = (const uint32_t *)iq3xxs_grid;\n\n    // constants for unpacking logic\n    const uint32_t shifts_val[8] = {0, 7, 14, 21, 0, 7, 14, 21};\n    vuint32m1_t v_shifts = __riscv_vle32_v_u32m1(shifts_val, 8);\n\n    const uint32_t gather_idx_val[8] = {0, 0, 0, 0, 1, 1, 1, 1};\n    vuint32m1_t v_gather_idx = __riscv_vle32_v_u32m1(gather_idx_val, 8);\n\n    uint32_t aux32[2];\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n\n        const uint8_t * GGML_RESTRICT q3_indices = x[i].qs;\n        const uint8_t * GGML_RESTRICT metadata   = x[i].qs + QK_K/4;\n        const int8_t  * GGML_RESTRICT q8         = y[i].qs;\n\n        float block_sum = 0.0f;\n\n        for (int ib = 0; ib < QK_K / 64; ++ib) {\n            // Load q8 (64 bytes)\n            vint8m2_t v_q8 = __riscv_vle8_v_i8m2(q8, 64);\n            q8 += 64;\n\n            // load of metadata via memcpy\n            memcpy(aux32, metadata, 2 * sizeof(uint32_t));\n            metadata += 2 * sizeof(uint32_t);\n\n            // Load q3 indices and gather magnitudes\n            vuint8mf2_t v_q3_idx_u8 = __riscv_vle8_v_u8mf2(q3_indices, 16);\n            q3_indices += 16;\n\n            vuint16m1_t v_q3_idx_u16 = __riscv_vwmulu_vx_u16m1(v_q3_idx_u8, 4, 16);\n            vuint32m2_t v_q3_magnitudes_u32 = __riscv_vluxei16_v_u32m2(grid32, v_q3_idx_u16, 16);\n            vint8m2_t v_q3_magnitudes = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u32m2_u8m2(v_q3_magnitudes_u32));\n\n            // --- Unpacking of Sign Indices ---\n\n            // 1. Load the 2 auxiliary 32-bit integers into a vector\n            vuint32m1_t v_aux = __riscv_vle32_v_u32m1(aux32, 2);\n\n            // 2. Broadcast/Gather: replicate aux[0] to first 4 lanes, aux[1] to next 4 lanes\n            vuint32m1_t v_aux_expanded = __riscv_vrgather_vv_u32m1(v_aux, v_gather_idx, 8);\n\n            // 3. Apply Shifts and Mask: ((val >> shift) & 127)\n            vuint32m1_t v_s_vals_raw = __riscv_vand_vx_u32m1(__riscv_vsrl_vv_u32m1(v_aux_expanded, v_shifts, 8), 127, 8);\n\n            // 4. Narrow to u16 (required for vluxei index) and multiply by 8 (byte offset for u64 table)\n            vuint16mf2_t sign_indices_byte_offset = __riscv_vsll_vx_u16mf2(__riscv_vncvt_x_x_w_u16mf2(v_s_vals_raw, 8), 3, 8);\n\n            // 5. Gather Signs\n            vuint64m2_t v_s_vals_u64 = __riscv_vluxei16_v_u64m2(signs64, sign_indices_byte_offset, 8);\n            vint8m2_t v_s_vals = __riscv_vreinterpret_v_u8m2_i8m2(__riscv_vreinterpret_v_u64m2_u8m2(v_s_vals_u64));\n\n            vint8m2_t v_q3_signed = __riscv_vmul_vv_i8m2(v_q3_magnitudes, v_s_vals, 64);\n            vint16m4_t v_dot = __riscv_vwmul_vv_i16m4(v_q8, v_q3_signed, 64);\n\n            vint16m2_t v_dot_1 = __riscv_vget_v_i16m4_i16m2(v_dot, 0);\n            vint16m2_t v_dot_2 = __riscv_vget_v_i16m4_i16m2(v_dot, 1);\n\n            vint32m1_t v_zero = __riscv_vmv_v_x_i32m1(0, 1);\n            vint32m1_t v_sum_1 = __riscv_vwredsum_vs_i16m2_i32m1(v_dot_1, v_zero, 32);\n            vint32m1_t v_sum_2 = __riscv_vwredsum_vs_i16m2_i32m1(v_dot_2, v_zero, 32);\n\n            int32_t sum1_i = __riscv_vmv_x_s_i32m1_i32(v_sum_1);\n            int32_t sum2_i = __riscv_vmv_x_s_i32m1_i32(v_sum_2);\n\n            const float scale1_f = (float)(2 * (aux32[0] >> 28) + 1);\n            const float scale2_f = (float)(2 * (aux32[1] >> 28) + 1);\n\n            block_sum += sum1_i * scale1_f + sum2_i * scale2_f;\n        }\n\n        sumf += d * block_sum;\n    }\n    *s = 0.25f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_iq3_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n    // Load the lookup table once.\n    const vint8m2_t values = __riscv_vle8_v_i8m2(kvalues_iq4nl, 16);\n    int acc1, acc2;\n\n    // We process 2 blocks at once.\n    for (; ib + 1 < nb; ib += 2) {\n        // Weights and activations.\n        vuint8m1_t iq4_packed1 = __riscv_vle8_v_u8m1(x[ib + 0].qs, 16);\n        vint8m2_t q8b1 = __riscv_vle8_v_i8m2(y[ib + 0].qs, 32);\n        vuint8m1_t iq4_packed2 = __riscv_vle8_v_u8m1(x[ib + 1].qs, 16);\n        vint8m2_t q8b2 = __riscv_vle8_v_i8m2(y[ib + 1].qs, 32);\n\n        // Unpack the weight blocks.\n        vuint8m2_t iq4bits1;\n        iq4bits1 = __riscv_vset_v_u8m1_u8m2(iq4bits1, 0, __riscv_vand_vx_u8m1(iq4_packed1, 0xf, 16));\n        iq4bits1 = __riscv_vset_v_u8m1_u8m2(iq4bits1, 1, __riscv_vsrl_vx_u8m1(iq4_packed1, 4, 16));\n        vuint8m2_t iq4bits2;\n        iq4bits2 = __riscv_vset_v_u8m1_u8m2(iq4bits2, 0, __riscv_vand_vx_u8m1(iq4_packed2, 0xf, 16));\n        iq4bits2 = __riscv_vset_v_u8m1_u8m2(iq4bits2, 1, __riscv_vsrl_vx_u8m1(iq4_packed2, 4, 16));\n\n        // Gather values from the lookup table.\n        vint8m2_t iq4b1 = __riscv_vrgather_vv_i8m2(values, iq4bits1, 32);\n        vint8m2_t iq4b2 = __riscv_vrgather_vv_i8m2(values, iq4bits2, 32);\n\n        // Accumulation.\n        vint16m4_t sum1 = __riscv_vwmul_vv_i16m4(q8b1, iq4b1, 32);\n        vint16m4_t sum2 = __riscv_vwmul_vv_i16m4(q8b2, iq4b2, 32);\n        __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m4_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n        __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m4_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n        sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n        sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n    }\n\n    *s = sumf;\n}\n\nstatic void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n    // Load the lookup table once.\n    const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n    int acc1, acc2;\n\n    // We process 2 blocks at once.\n    for (; ib + 1 < nb; ib += 2) {\n        // Weights and activations.\n        vuint8mf2_t iq4_packed1 = __riscv_vle8_v_u8mf2(x[ib + 0].qs, 16);\n        vint8mf2_t q8b_lo1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs, 16);\n        vint8mf2_t q8b_hi1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs + 16, 16);\n        vuint8mf2_t iq4_packed2 = __riscv_vle8_v_u8mf2(x[ib + 1].qs, 16);\n        vint8mf2_t q8b_lo2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs, 16);\n        vint8mf2_t q8b_hi2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs + 16, 16);\n\n        // Unpack the weight blocks.\n        vuint8mf2_t iq4bits_lo1 = __riscv_vand_vx_u8mf2(iq4_packed1, 0xf, 16);\n        vuint8mf2_t iq4bits_hi1 = __riscv_vsrl_vx_u8mf2(iq4_packed1, 4, 16);\n        vuint8mf2_t iq4bits_lo2 = __riscv_vand_vx_u8mf2(iq4_packed2, 0xf, 16);\n        vuint8mf2_t iq4bits_hi2 = __riscv_vsrl_vx_u8mf2(iq4_packed2, 4, 16);\n\n        // Gather values from the lookup table.\n        vint8mf2_t iq4b_lo1 = __riscv_vrgather_vv_i8mf2(values, iq4bits_lo1, 16);\n        vint8mf2_t iq4b_hi1 = __riscv_vrgather_vv_i8mf2(values, iq4bits_hi1, 16);\n        vint8mf2_t iq4b_lo2 = __riscv_vrgather_vv_i8mf2(values, iq4bits_lo2, 16);\n        vint8mf2_t iq4b_hi2 = __riscv_vrgather_vv_i8mf2(values, iq4bits_hi2, 16);\n\n        // Accumulation.\n        vint16m1_t sum1 = __riscv_vwmul_vv_i16m1(q8b_lo1, iq4b_lo1, 16);\n        sum1 = __riscv_vwmacc_vv_i16m1(sum1, q8b_hi1, iq4b_hi1, 16);\n        vint16m1_t sum2 = __riscv_vwmul_vv_i16m1(q8b_lo2, iq4b_lo2, 16);\n        sum2 = __riscv_vwmacc_vv_i16m1(sum2, q8b_hi2, iq4b_hi2, 16);\n        __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m1_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n        __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m1_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n        sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 0].d) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n        sumf += ((GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 128:\n            ggml_vec_dot_iq4_nl_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq4_nl_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_iq4_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __riscv_v_intrinsic\n    const vint8m4_t values = __riscv_vle8_v_i8m4(kvalues_iq4nl, 16);\n    float sumf = 0;\n    int acc[4];\n\n    // Indices for re-ordering IQ4 data.\n    uint64_t index[16] = {\n        0, 1, 8, 9,\n        2, 3, 10, 11,\n        4, 5,12, 13,\n        6, 7, 14, 15,\n    };\n    vuint64m4_t i_vec = __riscv_vle64_v_u64m4(index, 16);\n\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const int8_t  * q8 = y[ibl].qs;\n        const uint8_t * iq4 = x[ibl].qs;\n        uint16_t h = x[ibl].scales_h;\n\n        int sumi1 = 0, sumi2 = 0, sumi3 = 0, sumi4 = 0;\n\n        for (int ib = 0; ib < QK_K / 128; ++ib) {\n            // Weights and activations.\n            vuint8m2_t iq4_packed = __riscv_vle8_v_u8m2(iq4, 64);\n            vint8m4_t q8b = __riscv_vle8_v_i8m4(q8, 128);\n            iq4 += 64;\n            q8 += 128;\n\n            // Unpack the weight blocks.\n            vuint8m2_t iq4bits_lo = __riscv_vand_vx_u8m2(iq4_packed, 0xf, 64);\n            vuint8m2_t iq4bits_hi = __riscv_vsrl_vx_u8m2(iq4_packed, 4, 64);\n            vuint8m4_t iq4bits;\n            iq4bits = __riscv_vset_v_u8m2_u8m4(iq4bits, 0, iq4bits_lo);\n            iq4bits = __riscv_vset_v_u8m2_u8m4(iq4bits, 1, iq4bits_hi);\n            vuint8m4_t iq4bits_reorder = __riscv_vreinterpret_v_u64m4_u8m4(__riscv_vrgather_vv_u64m4(__riscv_vreinterpret_v_u8m4_u64m4(iq4bits), i_vec, 16));\n            vint8m4_t iq4b = __riscv_vrgather_vv_i8m4(values, iq4bits_reorder, 128);\n\n            // Multiply with activations.\n            vint16m8_t prod = __riscv_vwmul_vv_i16m8(iq4b, q8b, 128);\n\n            // Reduce separately.\n            __riscv_vse32_v_i32m1(&acc[0],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 0), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n            __riscv_vse32_v_i32m1(&acc[1],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 1), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n            __riscv_vse32_v_i32m1(&acc[2],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 2), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n            __riscv_vse32_v_i32m1(&acc[3],__riscv_vwredsum_vs_i16m2_i32m1(__riscv_vget_v_i16m8_i16m2(prod, 3), __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n\n            int ls1 = ((x[ibl].scales_l[ib * 2 + 0] & 0xf)  | ((h << 4) & 0x30)) - 32;\n            int ls2 = ((x[ibl].scales_l[ib * 2 + 0] >>  4)  | ((h << 2) & 0x30)) - 32;\n            int ls3 = ((x[ibl].scales_l[ib * 2 + 1] &  0xf) | ((h << 0) & 0x30)) - 32;\n            int ls4 = ((x[ibl].scales_l[ib * 2 + 1] >>  4)  | ((h >> 2) & 0x30)) - 32;\n            h >>= 8;\n\n            sumi1 += acc[0] * ls1;\n            sumi2 += acc[1] * ls2;\n            sumi3 += acc[2] * ls3;\n            sumi4 += acc[3] * ls4;\n        }\n\n        sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2 + sumi3 + sumi4);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_iq4_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq1_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0.0f;\n    uint8_t pow[16] = {1, 1, 1, 1, 3, 3, 3, 3, 9, 9, 9, 9, 27, 27, 27, 27};\n\n    for (int i = 0; i < nb; i++) {\n        // First loop.\n        vint32m4_t suml1;\n        {\n            const int vl = 32;\n            vuint8m1_t tq = __riscv_vle8_v_u8m1(x[i].qs, vl);\n\n            vuint16m2_t tq0 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(tq, 3, vl), 8, vl);\n            vuint16m2_t tq1 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 3, vl), 3, vl), 8, vl);\n            vuint16m2_t tq2 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 9, vl), 3, vl), 8, vl);\n            vuint16m2_t tq3 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 27, vl), 3, vl), 8, vl);\n            vuint16m2_t tq4 = __riscv_vsrl_vx_u16m2(__riscv_vwmulu_vx_u16m2(__riscv_vmul_vx_u8m1(tq, 81, vl), 3, vl), 8, vl);\n\n            vint16m2_t q80 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 0, vl), vl);\n            vint16m2_t q81 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 32, vl), vl);\n            vint16m2_t q82 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 64, vl), vl);\n            vint16m2_t q83 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 96, vl), vl);\n            vint16m2_t q84 = __riscv_vwcvt_x_x_v_i16m2(__riscv_vle8_v_i8m1(y[i].qs + 128, vl), vl);\n\n            vint16m2_t sum0 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq0, 1, vl)), q80, vl);\n            vint16m2_t sum1 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq1, 1, vl)), q81, vl);\n            vint16m2_t sum2 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq2, 1, vl)), q82, vl);\n            vint16m2_t sum3 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq3, 1, vl)), q83, vl);\n            vint16m2_t sum4 = __riscv_vmul_vv_i16m2(__riscv_vreinterpret_v_u16m2_i16m2(__riscv_vsub_vx_u16m2(tq4, 1, vl)), q84, vl);\n\n            vint32m4_t sumi0 = __riscv_vwadd_vv_i32m4(sum0, sum1, vl);\n            vint32m4_t sumi1 = __riscv_vwadd_vv_i32m4(sum2, sum3, vl);\n            suml1 = __riscv_vadd_vv_i32m4(__riscv_vwcvt_x_x_v_i32m4(sum4, vl), __riscv_vadd_vv_i32m4(sumi0, sumi1, vl), vl);\n        }\n\n        // Second loop.\n        vint32m2_t suml2;\n        {\n            const int vl = 16;\n            vuint8mf2_t tq = __riscv_vle8_v_u8mf2(x[i].qs + 32, vl);\n\n            vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(tq, 3 * 1, vl), 8, vl);\n            vuint16m1_t tq1 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 3, vl), 3, vl), 8, vl);\n            vuint16m1_t tq2 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 9, vl), 3, vl), 8, vl);\n            vuint16m1_t tq3 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 27, vl), 3, vl), 8, vl);\n            vuint16m1_t tq4 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vx_u8mf2(tq, 81, vl), 3, vl), 8, vl);\n\n            vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 160, vl), vl);\n            vint16m1_t q81 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 176, vl), vl);\n            vint16m1_t q82 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 192, vl), vl);\n            vint16m1_t q83 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 208, vl), vl);\n            vint16m1_t q84 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 224, vl), vl);\n\n            vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);\n            vint16m1_t sum1 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq1, 1, vl)), q81, vl);\n            vint16m1_t sum2 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq2, 1, vl)), q82, vl);\n            vint16m1_t sum3 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq3, 1, vl)), q83, vl);\n            vint16m1_t sum4 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq4, 1, vl)), q84, vl);\n\n            vint32m2_t sumi0 = __riscv_vwadd_vv_i32m2(sum0, sum1, vl);\n            vint32m2_t sumi1 = __riscv_vwadd_vv_i32m2(sum2, sum3, vl);\n            suml2 = __riscv_vadd_vv_i32m2(__riscv_vwcvt_x_x_v_i32m2(sum4, vl), __riscv_vadd_vv_i32m2(sumi0, sumi1, vl), vl);\n        }\n\n        // Third loop.\n        vint32m2_t suml3;\n        {\n            const int vl = 16;\n\n            uint32_t qh;\n            memcpy(&qh, &x[i].qh[0], 4);\n            // Prevent fusion with vmv.\n            __asm__ __volatile__(\"\" : \"+r\"(qh));\n            vuint8mf2_t tq = __riscv_vreinterpret_v_u32mf2_u8mf2(__riscv_vmv_v_x_u32mf2(qh, vl / 4));\n\n            vuint8mf2_t p = __riscv_vle8_v_u8mf2(pow, vl);\n\n            vuint16m1_t tq0 = __riscv_vsrl_vx_u16m1(__riscv_vwmulu_vx_u16m1(__riscv_vmul_vv_u8mf2(tq, p, vl), 3, vl), 8, vl);\n\n            vint16m1_t q80 = __riscv_vwcvt_x_x_v_i16m1(__riscv_vle8_v_i8mf2(y[i].qs + 240, vl), vl);\n\n            vint16m1_t sum0 = __riscv_vmul_vv_i16m1(__riscv_vreinterpret_v_u16m1_i16m1(__riscv_vsub_vx_u16m1(tq0, 1, vl)), q80, vl);\n            suml3 = __riscv_vwcvt_x_x_v_i32m2(sum0, vl);\n        }\n\n        vint32m2_t sumb = __riscv_vadd_vv_i32m2(__riscv_vget_v_i32m4_i32m2(suml1, 0), __riscv_vget_v_i32m4_i32m2(suml1, 1), 16);\n        sumb = __riscv_vadd_vv_i32m2(sumb, suml2, 16);\n        sumb = __riscv_vadd_vv_i32m2(sumb, suml3, 16);\n\n        vint32m1_t sum = __riscv_vredsum_vs_i32m2_i32m1(sumb, __riscv_vmv_v_x_i32m1(0, 1), 16);\n        sumf += __riscv_vmv_x_s_i32m1_i32(sum) * y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq2_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0.0f;\n    for (int i = 0; i < nb; ++i) {\n        int32_t sumi = 0;\n\n        for (size_t j = 0; j < sizeof(x[0].qs); j += 32) {\n            const int8_t * py0 = &y[i].qs[j * 4 + 0 * 32];\n            const int8_t * py1 = &y[i].qs[j * 4 + 1 * 32];\n            const int8_t * py2 = &y[i].qs[j * 4 + 2 * 32];\n            const int8_t * py3 = &y[i].qs[j * 4 + 3 * 32];\n            const uint8_t* px  = &x[i].qs[j];\n\n            size_t vlmax_16m2 = __riscv_vsetvl_e16m2(32);\n            vint16m2_t vacc16 = __riscv_vmv_v_x_i16m2(0, vlmax_16m2);\n\n            size_t vl = __riscv_vsetvl_e8m1(32);\n\n            vuint8m1_t vx_u8 = __riscv_vle8_v_u8m1(px, vl);\n\n            vint8m1_t vy0 = __riscv_vle8_v_i8m1(py0 , vl);\n            vint8m1_t vy1 = __riscv_vle8_v_i8m1(py1, vl);\n            vint8m1_t vy2 = __riscv_vle8_v_i8m1(py2, vl);\n            vint8m1_t vy3 = __riscv_vle8_v_i8m1(py3, vl);\n\n            // l=0 (bits 1:0)\n            vuint8m1_t t0 = __riscv_vand_vx_u8m1(vx_u8, 0x03, vl);\n            vint8m1_t vq0 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t0), 1, vl);\n\n            // l=1 (bits 3:2)\n            vuint8m1_t t1 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 2, vl), 0x03, vl);\n            vint8m1_t vq1 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t1), 1, vl);\n\n            // l=2 (bits 5:4)\n            vuint8m1_t t2 = __riscv_vand_vx_u8m1(__riscv_vsrl_vx_u8m1(vx_u8, 4, vl), 0x03, vl);\n            vint8m1_t vq2 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t2), 1, vl);\n\n            // l=3 (bits 7:6)\n            vuint8m1_t t3 = __riscv_vsrl_vx_u8m1(vx_u8, 6, vl); // No final AND needed as vsrl shifts in zeros\n            vint8m1_t vq3 = __riscv_vsub_vx_i8m1(__riscv_vreinterpret_v_u8m1_i8m1(t3), 1, vl);\n\n            // 4. Multiply and accumulate\n            vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq0, vy0, vl);\n            vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq1, vy1, vl);\n            vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq2, vy2, vl);\n            vacc16 = __riscv_vwmacc_vv_i16m2(vacc16, vq3, vy3, vl);\n\n            vlmax_16m2 = __riscv_vsetvl_e16m2(32);\n            vint32m1_t vzero32 = __riscv_vmv_v_x_i32m1(0, 1);\n            vint32m1_t vred32 = __riscv_vwredsum_vs_i16m2_i32m1(vacc16, vzero32, vlmax_16m2);\n\n            sumi += __riscv_vmv_x_s_i32m1_i32(vred32);\n        }\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        sumf += (float)sumi * d;\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 256:\n            ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nstatic void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n    // Load the lookup table once.\n    const vint8m2_t values = __riscv_vle8_v_i8m2(kvalues_mxfp4, 16);\n    int acc1, acc2;\n\n    // We process 2 blocks at once.\n    for (; ib + 1 < nb; ib += 2) {\n        // Weights and activations.\n        vuint8m1_t mx_packed1 = __riscv_vle8_v_u8m1(x[ib + 0].qs, 16);\n        vint8m2_t q8b1 = __riscv_vle8_v_i8m2(y[ib + 0].qs, 32);\n        vuint8m1_t mx_packed2 = __riscv_vle8_v_u8m1(x[ib + 1].qs, 16);\n        vint8m2_t q8b2 = __riscv_vle8_v_i8m2(y[ib + 1].qs, 32);\n\n        // Unpack the weight blocks.\n        vuint8m2_t mxbits1;\n        mxbits1 = __riscv_vset_v_u8m1_u8m2(mxbits1, 0, __riscv_vand_vx_u8m1(mx_packed1, 0xf, 16));\n        mxbits1 = __riscv_vset_v_u8m1_u8m2(mxbits1, 1, __riscv_vsrl_vx_u8m1(mx_packed1, 4, 16));\n        vuint8m2_t mxbits2;\n        mxbits2 = __riscv_vset_v_u8m1_u8m2(mxbits2, 0, __riscv_vand_vx_u8m1(mx_packed2, 0xf, 16));\n        mxbits2 = __riscv_vset_v_u8m1_u8m2(mxbits2, 1, __riscv_vsrl_vx_u8m1(mx_packed2, 4, 16));\n\n        // Gather values from the lookup table.\n        vint8m2_t mxb1 = __riscv_vrgather_vv_i8m2(values, mxbits1, 32);\n        vint8m2_t mxb2 = __riscv_vrgather_vv_i8m2(values, mxbits2, 32);\n\n        // Accumulation.\n        vint16m4_t sum1 = __riscv_vwmul_vv_i16m4(q8b1, mxb1, 32);\n        vint16m4_t sum2 = __riscv_vwmul_vv_i16m4(q8b2, mxb2, 32);\n        __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m4_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n        __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m4_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 32), 1);\n        sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n        sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n    }\n\n    *s = sumf;\n}\n\nstatic void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n    // Load the lookup table once.\n    const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_mxfp4, 16);\n    int acc1, acc2;\n\n    // We process 2 blocks at once.\n    for (; ib + 1 < nb; ib+=2) {\n        // Weights and activations.\n        vuint8mf2_t mx_packed1 = __riscv_vle8_v_u8mf2(x[ib + 0].qs, 16);\n        vint8mf2_t q8b_lo1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs, 16);\n        vint8mf2_t q8b_hi1 = __riscv_vle8_v_i8mf2(y[ib + 0].qs + 16, 16);\n        vuint8mf2_t mx_packed2 = __riscv_vle8_v_u8mf2(x[ib + 1].qs, 16);\n        vint8mf2_t q8b_lo2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs, 16);\n        vint8mf2_t q8b_hi2 = __riscv_vle8_v_i8mf2(y[ib + 1].qs + 16, 16);\n\n        // Unpack the weight blocks.\n        vuint8mf2_t mxbits_lo1 = __riscv_vand_vx_u8mf2(mx_packed1, 0xf, 16);\n        vuint8mf2_t mxbits_hi1 = __riscv_vsrl_vx_u8mf2(mx_packed1, 4, 16);\n        vuint8mf2_t mxbits_lo2 = __riscv_vand_vx_u8mf2(mx_packed2, 0xf, 16);\n        vuint8mf2_t mxbits_hi2 = __riscv_vsrl_vx_u8mf2(mx_packed2, 4, 16);\n\n        // Gather values from the lookup table.\n        vint8mf2_t mxb_lo1 = __riscv_vrgather_vv_i8mf2(values, mxbits_lo1, 16);\n        vint8mf2_t mxb_hi1 = __riscv_vrgather_vv_i8mf2(values, mxbits_hi1, 16);\n        vint8mf2_t mxb_lo2 = __riscv_vrgather_vv_i8mf2(values, mxbits_lo2, 16);\n        vint8mf2_t mxb_hi2 = __riscv_vrgather_vv_i8mf2(values, mxbits_hi2, 16);\n\n        // Accumulation.\n        vint16m1_t sum1 = __riscv_vwmul_vv_i16m1(q8b_lo1, mxb_lo1, 16);\n        sum1 = __riscv_vwmacc_vv_i16m1(sum1, q8b_hi1, mxb_hi1, 16);\n        vint16m1_t sum2 = __riscv_vwmul_vv_i16m1(q8b_lo2, mxb_lo2, 16);\n        sum2 = __riscv_vwmacc_vv_i16m1(sum2, q8b_hi2, mxb_hi2, 16);\n        __riscv_vse32_v_i32m1(&acc1,__riscv_vwredsum_vs_i16m1_i32m1(sum1, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n        __riscv_vse32_v_i32m1(&acc2,__riscv_vwredsum_vs_i16m1_i32m1(sum2, __riscv_vmv_v_x_i32m1(0, 1), 16), 1);\n        sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 0].e) * GGML_CPU_FP16_TO_FP32(y[ib + 0].d) * acc1));\n        sumf += ((GGML_E8M0_TO_FP32_HALF(x[ib + 1].e) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) * acc2));\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n#if defined __riscv_v_intrinsic\n    switch (__riscv_vlenb() * 8) {\n        case 128:\n            ggml_vec_dot_mxfp4_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n        default:\n            ggml_vec_dot_mxfp4_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc);\n            break;\n    }\n#else\n    return ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/riscv/repack.cpp",
    "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include <cmath>\n#include <cstring>\n#include <cassert>\n#include <cstdlib> // for qsort\n#include <cstdio>  // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n#if defined(__riscv_v_intrinsic)\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n    const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0);\n    const size_t vl_save = __riscv_vsetvl_e64m2(4);\n    vfloat32m1_t v_scalar_zero = __riscv_vfmv_s_f_f32m1(0.0f, __riscv_vsetvl_e32m1(1));\n\n    for (int i = 0; i < nb; i++) {\n        const float *x_block_base = x + i * QK8_0;\n        vint8m2_t q_r0, q_r1, q_r2, q_r3;\n        {\n            vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 0 * k, vl_calc);\n            vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n            vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n            float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n            float d = amax / 127.0f;\n            y[i].d[0] = GGML_CPU_FP32_TO_FP16(d);\n\n            float id = d ? 1.0f / d : 0.0f;\n            vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n            vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n            q_r0 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n        }\n        asm volatile (\"\" ::: \"memory\");\n\n        {\n            vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 1 * k, vl_calc);\n            vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n            vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n            float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n            float d = amax / 127.0f;\n            y[i].d[1] = GGML_CPU_FP32_TO_FP16(d);\n            float id = d ? 1.0f / d : 0.0f;\n\n            vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n            vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n            q_r1 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n        }\n        asm volatile (\"\" ::: \"memory\");\n        {\n            vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 2 * k, vl_calc);\n            vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n            vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n            float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n            float d = amax / 127.0f;\n            y[i].d[2] = GGML_CPU_FP32_TO_FP16(d);\n            float id = d ? 1.0f / d : 0.0f;\n\n            vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n            vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n            q_r2 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n        }\n        asm volatile (\"\" ::: \"memory\");\n        {\n            vfloat32m8_t v_src = __riscv_vle32_v_f32m8(x_block_base + 3 * k, vl_calc);\n            vfloat32m8_t v_abs = __riscv_vfabs_v_f32m8(v_src, vl_calc);\n            vfloat32m1_t v_max = __riscv_vfredmax_vs_f32m8_f32m1(v_abs, v_scalar_zero, vl_calc);\n            float amax = __riscv_vfmv_f_s_f32m1_f32(v_max);\n\n            float d = amax / 127.0f;\n            y[i].d[3] = GGML_CPU_FP32_TO_FP16(d);\n            float id = d ? 1.0f / d : 0.0f;\n\n            vfloat32m8_t v_scaled = __riscv_vfmul_vf_f32m8(v_src, id, vl_calc);\n            vint16m4_t v_i16 = __riscv_vfncvt_x_f_w_i16m4_rm(v_scaled, 4, vl_calc);\n            q_r3 = __riscv_vncvt_x_x_w_i8m2(v_i16, vl_calc);\n        }\n        vint64m2_t v_q64_r0 = __riscv_vreinterpret_v_i8m2_i64m2(q_r0);\n        vint64m2_t v_q64_r1 = __riscv_vreinterpret_v_i8m2_i64m2(q_r1);\n        vint64m2_t v_q64_r2 = __riscv_vreinterpret_v_i8m2_i64m2(q_r2);\n        vint64m2_t v_q64_r3 = __riscv_vreinterpret_v_i8m2_i64m2(q_r3);\n        vint64m2x4_t v_quant_tuple = __riscv_vcreate_v_i64m2x4(v_q64_r0, v_q64_r1, v_q64_r2, v_q64_r3);\n        __riscv_vsseg4e64_v_i64m2x4((int64_t*)y[i].qs, v_quant_tuple, vl_save);\n    }\n#else\n    UNUSED(nb);\n    UNUSED(y);\n    ggml_quantize_mat_q8_0_4x4_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v\n    if (__riscv_vlenb() >= QK4_0) {\n        const size_t vl = QK4_0;\n\n        const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n\n            vfloat32m1_t sumf = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n            for (int l = 0; l < nb; l++) {\n                const int64_t a0 = *(const int64_t *)&a_ptr[l].qs[0];\n                const int64_t a1 = *(const int64_t *)&a_ptr[l].qs[8];\n                const int64_t a2 = *(const int64_t *)&a_ptr[l].qs[16];\n                const int64_t a3 = *(const int64_t *)&a_ptr[l].qs[24];\n                __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment constraints\n                const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a0, vl / 4));\n                const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a1, vl / 4));\n                const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a2, vl / 4));\n                const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(a3, vl / 4));\n\n                const vint8m4_t rhs_raw_vec = __riscv_vle8_v_i8m4((const int8_t *)b_ptr[l].qs, vl * 4);\n                const vint8m4_t rhs_vec_lo = __riscv_vsra_vx_i8m4(__riscv_vsll_vx_i8m4(rhs_raw_vec, 4, vl * 4), 4, vl * 4);\n                const vint8m4_t rhs_vec_hi = __riscv_vsra_vx_i8m4(rhs_raw_vec, 4, vl * 4);\n                const vint8m2_t rhs_vec_lo_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 0);\n                const vint8m2_t rhs_vec_lo_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 1);\n                const vint8m2_t rhs_vec_hi_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 0);\n                const vint8m2_t rhs_vec_hi_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 1);\n\n                const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n                const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n                const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n                const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n                const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_hi_m));\n                const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n                const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n                const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n                const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n                const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n                const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n                const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n                const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n                const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n                const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n                const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n                const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n                // vector version needs Zvfhmin extension\n                const float a_scale = GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                const float b_scales[8] = {\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[4]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[5]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[6]),\n                    GGML_CPU_FP16_TO_FP32(b_ptr[l].d[7])\n                };\n                const vfloat32m1_t b_scales_vec = __riscv_vle32_v_f32m1(b_scales, vl / 4);\n                const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scale, vl / 4);\n                sumf = __riscv_vfmacc_vv_f32m1(sumf, tmp1, b_scales_vec, vl / 4);\n            }\n            __riscv_vse32_v_f32m1(s + x * ncols_interleaved, sumf, vl / 4);\n        }\n        return;\n    }\n\n#endif\n    ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n        // 1x16 Accumulator\n        vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n        for (int l = 0; l < nb; l++) {\n            // 1x16 Integer Accumulator\n            vint16m1_t sumi_0_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n            vint16m1_t sumi_0_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n            // Accumulation loop.\n            for (int i = 0; i < QK4_0 / 2; i++) {\n                // Load `b_ptr`.\n                const vint8mf2_t b_0_packed = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n                const vint8mf2_t b_0_lo = __riscv_vsra_vx_i8mf2(__riscv_vsll_vx_i8mf2(b_0_packed, 4, 16), 4, 16);\n                const vint8mf2_t b_0_hi = __riscv_vsra_vx_i8mf2(b_0_packed, 4, 16);\n\n                sumi_0_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_0_lo_16, a_ptr[l].qs[i], b_0_lo, 16);\n                sumi_0_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_0_hi_16, a_ptr[l].qs[16 + i], b_0_hi, 16);\n            }\n\n            const vint32m2_t sumi = __riscv_vwadd_vv_i32m2(sumi_0_lo_16, sumi_0_hi_16, 16);\n\n            const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n            const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n            sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n        }\n\n        __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n    }\n    return;\n#endif\n    ggml_gemv_q4_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n\n        // 1x16 Accumulator\n        vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n        for (int l = 0; l < nb; l++) {\n            vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0, 16);\n\n            // Load `dmin`.\n            const vfloat32m2_t dmins_d = __riscv_vfmul_vf_f32m2(\n                __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].dmin, 16), 16), a_ptr[l].d, 16);\n\n            // We process 4 sub-blocks at once.\n            for (int j = 0; j < QK_K / 128; j++) {\n                // Extract the scales and the mins.\n                //\n                // Low bits.\n                vuint8m2_t scales_mins_lo = __riscv_vle8_v_u8m2(&b_ptr[l].scales[j * 64], 64);\n                vuint8m2_t scales_lo = __riscv_vand_vx_u8m2(scales_mins_lo, 0x0F, 64);\n                vuint8m2_t mins_lo = __riscv_vsrl_vx_u8m2(scales_mins_lo, 4, 64);\n\n                // High bits.\n                vuint8m2_t scales_mins_hi = __riscv_vle8_v_u8m2(&b_ptr[l].scales[128], 64);\n                vuint8m2_t scales_hi;\n                vuint8m2_t mins_hi;\n                if (!j) {\n                    scales_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x03, 64), 4, 64);\n                    mins_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x0C, 64), 2, 64);\n                } else {\n                    scales_hi = __riscv_vand_vx_u8m2(scales_mins_hi, 0x30, 64);\n                    mins_hi = __riscv_vsrl_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0xC0, 64), 2, 64);\n                }\n                vuint16m4_t scales = __riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(scales_hi, scales_lo, 64), 64);\n                vint16m4_t mins = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(mins_hi, mins_lo, 64), 64));\n\n                // Reduce the mins and multiply with `dmin`.\n                //\n                // Correct in `sumf`.\n                vint32m2_t bsums = __riscv_vmv_v_x_i32m2(0, 16);\n                bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8] + a_ptr[l].bsums[j * 8 + 1], __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n                bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 2] + a_ptr[l].bsums[j * 8 + 3], __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n                bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 4] + a_ptr[l].bsums[j * 8 + 5], __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n                bsums = __riscv_vwmacc_vx_i32m2(bsums, a_ptr[l].bsums[j * 8 + 6] + a_ptr[l].bsums[j * 8 + 7], __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n\n                sumf = __riscv_vfsub_vv_f32m2(sumf, __riscv_vfmul_vv_f32m2(dmins_d, __riscv_vfcvt_f_x_v_f32m2(bsums, 16), 16), 16);\n\n                // Accumulation for 2 sub-blocks.\n                //\n                // This might overflow, so we accumulate in two steps.\n                //\n                // Recheck.\n                for (int k = 0; k < 2; k++) {\n                    vint16m1_t sumi_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                    vint16m1_t sumi_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n                    for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n                        // Load `b_ptr`.\n                        const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + i * 16], 16);\n                        const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n                        const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n                        sumi_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_s_0_16, a_ptr[l].qs[j * 128 + i], b_s_0, 16);\n                        sumi_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_s_1_16, a_ptr[l].qs[j * 128 + 32 + i], b_s_1, 16);\n                    }\n\n                    sumi = __riscv_vwmacc_vv_i32m2(sumi,\n                        __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n                        sumi_s_0_16, 16);\n                    sumi = __riscv_vwmacc_vv_i32m2(sumi,\n                        __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n                        sumi_s_1_16, 16);\n                }\n                // Accumulation for 2 sub-blocks.\n                //\n                // This might overflow, so we accumulate in two steps.\n                //\n                // Recheck.\n                for (int k = 0; k < 2; k++) {\n                    vint16m1_t sumi_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                    vint16m1_t sumi_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n                    for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n                        // Load `b_ptr`.\n                        const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + 512 + i * 16], 16);\n                        const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n                        const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n                        sumi_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_s_0_16, a_ptr[l].qs[j * 128 + 64 + i], b_s_0, 16);\n                        sumi_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_s_1_16, a_ptr[l].qs[j * 128 + 96 + i], b_s_1, 16);\n                    }\n\n                    sumi = __riscv_vwmacc_vv_i32m2(sumi,\n                        __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n                        sumi_s_0_16, 16);\n                    sumi = __riscv_vwmacc_vv_i32m2(sumi,\n                        __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n                        sumi_s_1_16, 16);\n                }\n            }\n\n            const vfloat32m2_t b_d = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)&b_ptr[l].d[0], 16), 16);\n            const vfloat32m2_t d_0 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d, 16);\n\n            sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n        }\n\n        __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n    }\n    return;\n#endif\n    ggml_gemv_q4_K_16x1_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n        // 1x16 Accumulator1\n        vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n        for (int l = 0; l < nb; l++) {\n            // 1x16 integer accumulator\n            vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n            // Accumulation loop.\n            for (int i = 0; i < QK4_NL / 2; i++) {\n                // Load `b_ptr`.\n                const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2((const uint8_t *)&b_ptr[l].qs[i * 16], 16);\n                const vint8mf2_t b_0_lo = __riscv_vrgather_vv_i8mf2(values, __riscv_vand_vx_u8mf2(b_0_packed, 0xf, 16), 16);\n                const vint8mf2_t b_0_hi = __riscv_vrgather_vv_i8mf2(values, __riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16), 16);\n                // const vint16m1_t b_0_lo_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_lo, 16);\n                // const vint16m1_t b_0_hi_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_hi, 16);\n\n                const vint16m1_t sumi_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i], 16);\n                const vint16m1_t sumi_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[16 + i], 16);\n                sumi = __riscv_vadd_vv_i32m2(sumi, __riscv_vwadd_vv_i32m2(sumi_lo, sumi_hi, 16), 16);\n            }\n\n            const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n            const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n            sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n        }\n\n        __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n    }\n    return;\n#endif\n    ggml_gemv_iq4_nl_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n    UNUSED(bs);\n\n#if defined __riscv_v_intrinsic\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n        // 1x16 Accumulator\n        vfloat32m2_t sumf = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n        for (int l = 0; l < nb; l++) {\n            // 1x16 Integer Accumulator\n            vint32m2_t sumi = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n            // Accumulation loop.\n            for (int i = 0; i < QK8_0; i++) {\n                // Load `b_ptr`.\n                const vint8mf2_t b_0 = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n                // const vint16m1_t b_0_16 = __riscv_vwcvt_x_x_v_i16m1(b_0, 16);\n\n                sumi = __riscv_vwadd_wv_i32m2(sumi, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i], 16), 16);\n            }\n\n            const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n            const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d, 16);\n\n            sumf = __riscv_vfmacc_vv_f32m2(sumf, __riscv_vfcvt_f_x_v_f32m2(sumi, 16), d_0, 16);\n        }\n\n        __riscv_vse32_v_f32m2(s + x * 16, sumf, 16);\n    }\n    return;\n#endif\n    ggml_gemv_q8_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    assert(n % QK_K == 0);\n    assert(nr == 1);\n    assert(nc % 16 == 0);\n\n    UNUSED(bs);\n\n    const int N_COLS_TILE = 16;\n    const int num_k_blocks = n / QK_K;\n\n    const size_t vl = __riscv_vsetvl_e32m2(N_COLS_TILE);\n    for (int col_tile = 0; col_tile < nc; col_tile += N_COLS_TILE) {\n\n        const block_q8_K* lhs_base_ptr = (const block_q8_K*)vy;\n        const block_q2_Kx16* rhs_base_ptr = (const block_q2_Kx16*)vx + (col_tile / N_COLS_TILE) * num_k_blocks;\n\n        vfloat32m2_t v_sumf = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n\n        for (int k_block = 0; k_block < num_k_blocks; ++k_block) {\n            const block_q8_K* lhs_current = &lhs_base_ptr[k_block];\n            const block_q2_Kx16* rhs_current = &rhs_base_ptr[k_block];\n\n            // 1. Prepare Global Min Scales\n            vfloat16m1_t v_g_min_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->dmin, vl);\n            vfloat32m2_t v_g_min_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_min_f16, vl);\n\n            vfloat32m2_t v_g_min_final = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d, vl);\n\n            vint32m2_t v_isum = __riscv_vmv_v_x_i32m2(0, vl);\n\n            const uint8_t* rhs_qs_ptr = rhs_current->qs;\n            const uint8_t* rhs_sc_ptr = rhs_current->scales;\n            const int8_t*  lhs_qs_ptr = lhs_current->qs;\n\n            // --- Phase Loop (4 phases x 64 elements) ---\n            for (int phase = 0; phase < 4; ++phase) {\n\n                // A. Load Scales/Mins\n                vuint16m1_t v_d_sb_0, v_d_sb_1, v_d_sb_2, v_d_sb_3;\n                vuint16m1_t v_m_sb_0, v_m_sb_1, v_m_sb_2, v_m_sb_3;\n\n                {\n                    vuint8mf2_t v_raw;\n                    // Sub-block 0\n                    v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 0, vl);\n                    v_d_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                    v_m_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                    // Sub-block 1\n                    v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 16, vl);\n                    v_d_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                    v_m_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                    // Sub-block 2\n                    v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 32, vl);\n                    v_d_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                    v_m_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                    // Sub-block 3\n                    v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 48, vl);\n                    v_d_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                    v_m_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                    rhs_sc_ptr += 64;\n                }\n\n                int base_k_phase = (phase < 2) ? (phase * 16) : (128 + (phase-2)*16);\n                int k_offsets[4] = {0, 32, 64, 96};\n\n                // B. Inner Dot Product Loop\n                for (int l = 0; l < 16; ++l) {\n                    vuint8mf2_t v_rhs_data = __riscv_vle8_v_u8mf2(rhs_qs_ptr, vl);\n                    rhs_qs_ptr += 16;\n\n                    // Sub-block 0\n                    {\n                        vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(v_rhs_data, 3, vl);\n                        vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                            __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                            __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_0), vl);\n\n                        int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[0] + l];\n                        v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n                    }\n                    // Sub-block 1\n                    {\n                        vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 2, vl), 3, vl);\n                        vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                            __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                            __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_1), vl);\n\n                        int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[1] + l];\n                        v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n                    }\n                    // Sub-block 2\n                    {\n                        vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 4, vl), 3, vl);\n                        vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                            __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                            __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_2), vl);\n\n                        int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[2] + l];\n                        v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n                    }\n                    // Sub-block 3\n                    {\n                        vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 6, vl), 3, vl);\n                        vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                            __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                            __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_3), vl);\n\n                        int8_t q8 = lhs_qs_ptr[base_k_phase + k_offsets[3] + l];\n                        v_isum = __riscv_vwmacc_vx_i32m2(v_isum, (int16_t)q8, v_w, vl);\n                    }\n                }\n\n                // correction\n                int sb_base_abs = base_k_phase / 16;\n\n                // Sub-block 0\n                {\n                    int sb_idx = sb_base_abs + (k_offsets[0] / 16);\n                    int16_t bsum = lhs_current->bsums[sb_idx];\n                    vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_0);\n                    vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n                    vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n                    v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n                }\n                // Sub-block 1\n                {\n                    int sb_idx = sb_base_abs + (k_offsets[1] / 16);\n                    int16_t bsum = lhs_current->bsums[sb_idx];\n                    vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_1);\n                    vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n                    vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n                    v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n                }\n                // Sub-block 2\n                {\n                    int sb_idx = sb_base_abs + (k_offsets[2] / 16);\n                    int16_t bsum = lhs_current->bsums[sb_idx];\n                    vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_2);\n                    vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n                    vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n                    v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n                }\n                // Sub-block 3\n                {\n                    int sb_idx = sb_base_abs + (k_offsets[3] / 16);\n                    int16_t bsum = lhs_current->bsums[sb_idx];\n                    vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_3);\n                    vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, bsum, vl);\n                    vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min_final, vl);\n                    v_sumf = __riscv_vfsub_vv_f32m2(v_sumf, vf_c, vl);\n                }\n\n            } // End Phase Loop\n\n            // Apply global Scales\n            vfloat16m1_t v_g_all_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->d, vl);\n            vfloat32m2_t v_g_all_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_all_f16, vl);\n\n            vfloat32m2_t v_g_all_final = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d, vl);\n            vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum, vl);\n            v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all_final, vl);\n            v_sumf = __riscv_vfadd_vv_f32m2(v_sumf, v_sum, vl);\n\n        } // End K-Block\n        __riscv_vse32_v_f32m2(s + col_tile, v_sumf, vl);\n\n    }\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v\n    if (__riscv_vlenb() >= QK4_0) {\n        const size_t vl = QK4_0;\n\n        for (int y = 0; y < nr / 4; y++) {\n            const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n            for (int x = 0; x < nc / ncols_interleaved; x++) {\n                const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n                vfloat32m1_t sumf0 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n                vfloat32m1_t sumf1 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n                vfloat32m1_t sumf2 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n                vfloat32m1_t sumf3 = __riscv_vfmv_v_f_f32m1(0.0, vl / 4);\n                for (int l = 0; l < nb; l++) {\n                    const vint8m4_t rhs_raw_vec = __riscv_vle8_v_i8m4((const int8_t *)b_ptr[l].qs, vl * 4);\n                    const vint8m4_t rhs_vec_lo = __riscv_vsra_vx_i8m4(__riscv_vsll_vx_i8m4(rhs_raw_vec, 4, vl * 4), 4, vl * 4);\n                    const vint8m4_t rhs_vec_hi = __riscv_vsra_vx_i8m4(rhs_raw_vec, 4, vl * 4);\n                    const vint8m2_t rhs_vec_lo_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 0);\n                    const vint8m2_t rhs_vec_lo_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_lo, 1);\n                    const vint8m2_t rhs_vec_hi_0 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 0);\n                    const vint8m2_t rhs_vec_hi_1 = __riscv_vget_v_i8m4_i8m2(rhs_vec_hi, 1);\n\n                    // vector version needs Zvfhmin extension\n                    const float a_scales[4] = {\n                        GGML_CPU_FP16_TO_FP32(a_ptr[l].d[0]),\n                        GGML_CPU_FP16_TO_FP32(a_ptr[l].d[1]),\n                        GGML_CPU_FP16_TO_FP32(a_ptr[l].d[2]),\n                        GGML_CPU_FP16_TO_FP32(a_ptr[l].d[3])\n                    };\n                    const float b_scales[8] = {\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[0]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[1]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[2]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[3]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[4]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[5]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[6]),\n                        GGML_CPU_FP16_TO_FP32(b_ptr[l].d[7])\n                    };\n                    const vfloat32m1_t b_scales_vec = __riscv_vle32_v_f32m1(b_scales, vl / 4);\n\n                    const int64_t A0 = *(const int64_t *)&a_ptr[l].qs[0];\n                    const int64_t A4 = *(const int64_t *)&a_ptr[l].qs[32];\n                    const int64_t A8 = *(const int64_t *)&a_ptr[l].qs[64];\n                    const int64_t Ac = *(const int64_t *)&a_ptr[l].qs[96];\n                    __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n                    vint16m4_t sumi_l0;\n                    {\n                        const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A0, vl / 4));\n                        const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A4, vl / 4));\n                        const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A8, vl / 4));\n                        const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ac, vl / 4));\n                        const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n                        const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n                        const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n                        const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n                        sumi_l0 = sumi_hi_m;\n                    }\n\n                    {\n                        const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l0));\n                        const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n                        const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n                        const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n                        const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n                        const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n                        const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n                        const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n                        const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n                        const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n                        const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n                        const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n                        const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n                        const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[0], vl / 4);\n                        sumf0 = __riscv_vfmacc_vv_f32m1(sumf0, tmp1, b_scales_vec, vl / 4);\n                    }\n\n                    const int64_t A1 = *(const int64_t *)&a_ptr[l].qs[8];\n                    const int64_t A5 = *(const int64_t *)&a_ptr[l].qs[40];\n                    const int64_t A9 = *(const int64_t *)&a_ptr[l].qs[72];\n                    const int64_t Ad = *(const int64_t *)&a_ptr[l].qs[104];\n                    __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n                    vint16m4_t sumi_l1;\n                    {\n                        const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A1, vl / 4));\n                        const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A5, vl / 4));\n                        const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A9, vl / 4));\n                        const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ad, vl / 4));\n                        const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n                        const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n                        const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n                        const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n                        sumi_l1 = sumi_hi_m;\n                    }\n\n                    {\n                        const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l1));\n                        const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n                        const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n                        const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n                        const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n                        const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n                        const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n                        const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n                        const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n                        const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n                        const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n                        const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n                        const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n                        const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[1], vl / 4);\n                        sumf1 = __riscv_vfmacc_vv_f32m1(sumf1, tmp1, b_scales_vec, vl / 4);\n                    }\n\n                    const int64_t A2 = *(const int64_t *)&a_ptr[l].qs[16];\n                    const int64_t A6 = *(const int64_t *)&a_ptr[l].qs[48];\n                    const int64_t Aa = *(const int64_t *)&a_ptr[l].qs[80];\n                    const int64_t Ae = *(const int64_t *)&a_ptr[l].qs[112];\n                    __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n                    vint16m4_t sumi_l2;\n                    {\n                        const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A2, vl / 4));\n                        const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A6, vl / 4));\n                        const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Aa, vl / 4));\n                        const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ae, vl / 4));\n                        const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n                        const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n                        const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n                        const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n                        sumi_l2 = sumi_hi_m;\n                    }\n\n                    {\n                        const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l2));\n                        const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n                        const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n                        const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n                        const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n                        const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n                        const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n                        const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n                        const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n                        const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n                        const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n                        const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n                        const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n                        const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[2], vl / 4);\n                        sumf2 = __riscv_vfmacc_vv_f32m1(sumf2, tmp1, b_scales_vec, vl / 4);\n                    }\n\n                    const int64_t A3 = *(const int64_t *)&a_ptr[l].qs[24];\n                    const int64_t A7 = *(const int64_t *)&a_ptr[l].qs[56];\n                    const int64_t Ab = *(const int64_t *)&a_ptr[l].qs[88];\n                    const int64_t Af = *(const int64_t *)&a_ptr[l].qs[120];\n                    __asm__ __volatile__(\"\" ::: \"memory\"); // prevent gcc from emitting fused vlse64, violating alignment\n                    vint16m4_t sumi_l3;\n                    {\n                        const vint8m2_t lhs_0_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A3, vl / 4));\n                        const vint8m2_t lhs_1_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(A7, vl / 4));\n                        const vint8m2_t lhs_2_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Ab, vl / 4));\n                        const vint8m2_t lhs_3_8 =__riscv_vreinterpret_v_i64m2_i8m2(__riscv_vmv_v_x_i64m2(Af, vl / 4));\n                        const vint16m4_t sumi_lo_0 = __riscv_vwmul_vv_i16m4(rhs_vec_lo_0, lhs_0_8, vl * 2);\n                        const vint16m4_t sumi_lo_1 = __riscv_vwmacc_vv_i16m4(sumi_lo_0, rhs_vec_lo_1, lhs_1_8, vl * 2);\n                        const vint16m4_t sumi_hi_0 = __riscv_vwmacc_vv_i16m4(sumi_lo_1, rhs_vec_hi_0, lhs_2_8, vl * 2);\n                        const vint16m4_t sumi_hi_m = __riscv_vwmacc_vv_i16m4(sumi_hi_0, rhs_vec_hi_1, lhs_3_8, vl * 2);\n\n                        sumi_l3 = sumi_hi_m;\n                    }\n\n                    {\n                        const vuint32m4_t sumi_i32 = __riscv_vreinterpret_v_i32m4_u32m4(__riscv_vreinterpret_v_i16m4_i32m4(sumi_l3));\n                        const vuint16m2_t sumi_h2_0 = __riscv_vnsrl_wx_u16m2(sumi_i32, 0, vl);\n                        const vuint16m2_t sumi_h2_1 = __riscv_vnsrl_wx_u16m2(sumi_i32, 16, vl);\n                        const vuint16m2_t sumi_h2 = __riscv_vadd_vv_u16m2(sumi_h2_0, sumi_h2_1, vl);\n                        const vuint32m2_t sumi_h2_i32 = __riscv_vreinterpret_v_u16m2_u32m2(sumi_h2);\n                        const vuint16m1_t sumi_h4_0 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 0, vl / 2);\n                        const vuint16m1_t sumi_h4_1 = __riscv_vnsrl_wx_u16m1(sumi_h2_i32, 16, vl / 2);\n                        const vuint16m1_t sumi_h4 = __riscv_vadd_vv_u16m1(sumi_h4_0, sumi_h4_1, vl / 2);\n                        const vuint32m1_t sumi_h4_i32 = __riscv_vreinterpret_v_u16m1_u32m1(sumi_h4);\n                        const vint16mf2_t sumi_h8_0 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 0, vl / 4));\n                        const vint16mf2_t sumi_h8_1 = __riscv_vreinterpret_v_u16mf2_i16mf2(__riscv_vnsrl_wx_u16mf2(sumi_h4_i32, 16, vl / 4));\n                        const vint32m1_t sumi_h8 = __riscv_vwadd_vv_i32m1(sumi_h8_0, sumi_h8_1, vl / 4);\n                        const vfloat32m1_t facc = __riscv_vfcvt_f_x_v_f32m1(sumi_h8, vl / 4);\n\n                        const vfloat32m1_t tmp1 = __riscv_vfmul_vf_f32m1(facc, a_scales[3], vl / 4);\n                        sumf3 = __riscv_vfmacc_vv_f32m1(sumf3, tmp1, b_scales_vec, vl / 4);\n                    }\n                }\n                __riscv_vse32_v_f32m1(&s[(y * 4 + 0) * bs + x * ncols_interleaved], sumf0, vl / 4);\n                __riscv_vse32_v_f32m1(&s[(y * 4 + 1) * bs + x * ncols_interleaved], sumf1, vl / 4);\n                __riscv_vse32_v_f32m1(&s[(y * 4 + 2) * bs + x * ncols_interleaved], sumf2, vl / 4);\n                __riscv_vse32_v_f32m1(&s[(y * 4 + 3) * bs + x * ncols_interleaved], sumf3, vl / 4);\n            }\n        }\n\n        return;\n    }\n\n#endif\n    ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n            // 4x16 Accumulators\n            vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n            for (int l = 0; l < nb; l++) {\n                // 4x16 integer accumulators\n                vint16m1_t sumi_0_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_1_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_2_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_3_lo_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_0_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_1_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_2_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                vint16m1_t sumi_3_hi_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n                // Accumulation loop.\n                for (int i = 0; i < QK4_0 / 2; i++) {\n                    // Load `b_ptr`.\n                    const vint8mf2_t b_0_packed = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n                    const vint8mf2_t b_0_lo = __riscv_vsra_vx_i8mf2(__riscv_vsll_vx_i8mf2(b_0_packed, 4, 16), 4, 16);\n                    const vint8mf2_t b_0_hi = __riscv_vsra_vx_i8mf2(b_0_packed, 4, 16);\n\n                    sumi_0_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_0_lo_16, a_ptr[l].qs[i * 4], b_0_lo, 16);\n                    sumi_1_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_1_lo_16, a_ptr[l].qs[i * 4 + 1], b_0_lo, 16);\n                    sumi_2_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_2_lo_16, a_ptr[l].qs[i * 4 + 2], b_0_lo, 16);\n                    sumi_3_lo_16 = __riscv_vwmacc_vx_i16m1(sumi_3_lo_16, a_ptr[l].qs[i * 4 + 3], b_0_lo, 16);\n\n                    sumi_0_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_0_hi_16, a_ptr[l].qs[64 + i * 4], b_0_hi, 16);\n                    sumi_1_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_1_hi_16, a_ptr[l].qs[64 + i * 4 + 1], b_0_hi, 16);\n                    sumi_2_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_2_hi_16, a_ptr[l].qs[64 + i * 4 + 2], b_0_hi, 16);\n                    sumi_3_hi_16 = __riscv_vwmacc_vx_i16m1(sumi_3_hi_16, a_ptr[l].qs[64 + i * 4 + 3], b_0_hi, 16);\n                }\n\n                // Do the final accumulation in i32 to prevent overflow.\n                const vint32m2_t sumi_0 = __riscv_vwadd_vv_i32m2(sumi_0_lo_16, sumi_0_hi_16, 16);\n                const vint32m2_t sumi_1 = __riscv_vwadd_vv_i32m2(sumi_1_lo_16, sumi_1_hi_16, 16);\n                const vint32m2_t sumi_2 = __riscv_vwadd_vv_i32m2(sumi_2_lo_16, sumi_2_hi_16, 16);\n                const vint32m2_t sumi_3 = __riscv_vwadd_vv_i32m2(sumi_3_lo_16, sumi_3_hi_16, 16);\n\n                const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n                const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n                const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n                const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n                const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n                sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n                sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n                sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n                sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n            }\n\n            __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n        }\n    }\n    return;\n#endif\n    ggml_gemm_q4_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n\n            // 4x16 Accumulators\n            vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n            for (int l = 0; l < nb; l++) {\n                vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0, 16);\n                vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0, 16);\n                vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0, 16);\n                vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0, 16);\n\n                // Load `dmin`.\n                const vfloat32m2_t dmins = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].dmin, 16), 16);\n\n                // We process 4 sub-blocks at once.\n                for (int j = 0; j < QK_K / 128; j++) {\n                    // Extract the scales and the mins.\n                    //\n                    // Low bits.\n                    vuint8m2_t scales_mins_lo = __riscv_vle8_v_u8m2(&b_ptr[l].scales[j * 64], 64);\n                    vuint8m2_t scales_lo = __riscv_vand_vx_u8m2(scales_mins_lo, 0x0F, 64);\n                    vuint8m2_t mins_lo = __riscv_vsrl_vx_u8m2(scales_mins_lo, 4, 64);\n\n                    // High bits.\n                    vuint8m2_t scales_mins_hi = __riscv_vle8_v_u8m2(&b_ptr[l].scales[128], 64);\n                    vuint8m2_t scales_hi;\n                    vuint8m2_t mins_hi;\n                    if (!j) {\n                        scales_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x03, 64), 4, 64);\n                        mins_hi = __riscv_vsll_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0x0C, 64), 2, 64);\n                    } else {\n                        scales_hi = __riscv_vand_vx_u8m2(scales_mins_hi, 0x30, 64);\n                        mins_hi = __riscv_vsrl_vx_u8m2(__riscv_vand_vx_u8m2(scales_mins_hi, 0xC0, 64), 2, 64);\n                    }\n                    vuint16m4_t scales = __riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(scales_hi, scales_lo, 64), 64);\n                    vint16m4_t mins = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vzext_vf2_u16m4(__riscv_vor_vv_u8m2(mins_hi, mins_lo, 64), 64));\n\n                    // Reduce the mins and multiply with `dmin`.\n                    //\n                    // Correct in `sumf`.\n                    vint32m2_t bsums_0 = __riscv_vmv_v_x_i32m2(0, 16);\n                    vint32m2_t bsums_1 = __riscv_vmv_v_x_i32m2(0, 16);\n                    vint32m2_t bsums_2 = __riscv_vmv_v_x_i32m2(0, 16);\n                    vint32m2_t bsums_3 = __riscv_vmv_v_x_i32m2(0, 16);\n\n                    bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n                                a_ptr[l].bsums[j * 32] + a_ptr[l].bsums[j * 32 + 4],\n                                __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n                    bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n                                a_ptr[l].bsums[j * 32 + 1] + a_ptr[l].bsums[j * 32 + 5],\n                                __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n                    bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n                                a_ptr[l].bsums[j * 32 + 2] + a_ptr[l].bsums[j * 32 + 6],\n                                __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n                    bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n                                a_ptr[l].bsums[j * 32 + 3] + a_ptr[l].bsums[j * 32 + 7],\n                                __riscv_vget_v_i16m4_i16m1(mins, 0), 16);\n                    bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n                                a_ptr[l].bsums[j * 32 + 8] + a_ptr[l].bsums[j * 32 + 8 + 4],\n                                __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n                    bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n                                a_ptr[l].bsums[j * 32 + 8 + 1] + a_ptr[l].bsums[j * 32 + 8 + 5],\n                                __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n                    bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n                                a_ptr[l].bsums[j * 32 + 8 + 2] + a_ptr[l].bsums[j * 32 + 8 + 6],\n                                __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n                    bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n                                a_ptr[l].bsums[j * 32 + 8 + 3] + a_ptr[l].bsums[j * 32 + 8 + 7],\n                                __riscv_vget_v_i16m4_i16m1(mins, 1), 16);\n                    bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n                                a_ptr[l].bsums[j * 32 + 16] + a_ptr[l].bsums[j * 32 + 16 + 4],\n                                __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n                    bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n                                a_ptr[l].bsums[j * 32 + 16 + 1] + a_ptr[l].bsums[j * 32 + 16 + 5],\n                                __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n                    bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n                                a_ptr[l].bsums[j * 32 + 16 + 2] + a_ptr[l].bsums[j * 32 + 16 + 6],\n                                __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n                    bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n                                a_ptr[l].bsums[j * 32 + 16 + 3] + a_ptr[l].bsums[j * 32 + 16 + 7],\n                                __riscv_vget_v_i16m4_i16m1(mins, 2), 16);\n                    bsums_0 = __riscv_vwmacc_vx_i32m2(bsums_0,\n                                a_ptr[l].bsums[j * 32 + 24 + 0] + a_ptr[l].bsums[j * 32 + 24 + 4],\n                                __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n                    bsums_1 = __riscv_vwmacc_vx_i32m2(bsums_1,\n                                a_ptr[l].bsums[j * 32 + 24 + 1] + a_ptr[l].bsums[j * 32 + 24 + 5],\n                                __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n                    bsums_2 = __riscv_vwmacc_vx_i32m2(bsums_2,\n                                a_ptr[l].bsums[j * 32 + 24 + 2] + a_ptr[l].bsums[j * 32 + 24 + 6],\n                                __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n                    bsums_3 = __riscv_vwmacc_vx_i32m2(bsums_3,\n                                a_ptr[l].bsums[j * 32 + 24 + 3] + a_ptr[l].bsums[j * 32 + 24 + 7],\n                                __riscv_vget_v_i16m4_i16m1(mins, 3), 16);\n\n                    const vfloat32m2_t dmins_d_0 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[0], 16);\n                    const vfloat32m2_t dmins_d_1 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[1], 16);\n                    const vfloat32m2_t dmins_d_2 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[2], 16);\n                    const vfloat32m2_t dmins_d_3 = __riscv_vfmul_vf_f32m2(dmins, a_ptr[l].d[3], 16);\n\n                    sumf_0 = __riscv_vfsub_vv_f32m2(sumf_0, __riscv_vfmul_vv_f32m2(dmins_d_0, __riscv_vfcvt_f_x_v_f32m2(bsums_0, 16), 16), 16);\n                    sumf_1 = __riscv_vfsub_vv_f32m2(sumf_1, __riscv_vfmul_vv_f32m2(dmins_d_1, __riscv_vfcvt_f_x_v_f32m2(bsums_1, 16), 16), 16);\n                    sumf_2 = __riscv_vfsub_vv_f32m2(sumf_2, __riscv_vfmul_vv_f32m2(dmins_d_2, __riscv_vfcvt_f_x_v_f32m2(bsums_2, 16), 16), 16);\n                    sumf_3 = __riscv_vfsub_vv_f32m2(sumf_3, __riscv_vfmul_vv_f32m2(dmins_d_3, __riscv_vfcvt_f_x_v_f32m2(bsums_3, 16), 16), 16);\n\n\n                    // Accumulation for 2 sub-blocks.\n                    //\n                    // This might overflow, so we accumulate in two steps.\n                    //\n                    // Recheck.\n                    for (int k = 0; k < 2; k++) {\n                        // 4x16 integer accumulators\n                        vint16m1_t sumi_0_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_1_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_2_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_3_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_0_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_1_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_2_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_3_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n                        for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n                            // Load `b_ptr`.\n                            const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + i * 16], 16);\n                            const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n                            const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n                            sumi_0_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_0_16, a_ptr[l].qs[j * 512 + i * 4], b_s_0, 16);\n                            sumi_1_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 1], b_s_0, 16);\n                            sumi_2_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 2], b_s_0, 16);\n                            sumi_3_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_0_16, a_ptr[l].qs[j * 512 + i * 4 + 3], b_s_0, 16);\n\n                            sumi_0_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4], b_s_1, 16);\n                            sumi_1_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 1], b_s_1, 16);\n                            sumi_2_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 2], b_s_1, 16);\n                            sumi_3_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_1_16, a_ptr[l].qs[j * 512 + 128 + i * 4 + 3], b_s_1, 16);\n                        }\n\n                        sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n                                    sumi_0_s_0_16, 16);\n                        sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n                                    sumi_0_s_1_16, 16);\n                        sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n                                    sumi_1_s_0_16, 16);\n                        sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n                                    sumi_1_s_1_16, 16);\n                        sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n                                    sumi_2_s_0_16, 16);\n                        sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n                                    sumi_2_s_1_16, 16);\n                        sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 0)),\n                                    sumi_3_s_0_16, 16);\n                        sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 1)),\n                                    sumi_3_s_1_16, 16);\n                    }\n                    // Accumulation for 2 sub-blocks.\n                    //\n                    // This might overflow, so we accumulate in two steps.\n                    //\n                    // Recheck.\n                    for (int k = 0; k < 2; k++) {\n                        // 4x16 integer accumulators\n                        vint16m1_t sumi_0_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_1_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_2_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_3_s_0_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_0_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_1_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_2_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n                        vint16m1_t sumi_3_s_1_16 = __riscv_vmv_v_x_i16m1(0.0f, 16);\n\n                        for (int i = k * 16; i < k * 16 + QK4_0 / 2; i++) {\n                            // Load `b_ptr`.\n                            const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2(&b_ptr[l].qs[j * 1024 + 512 + i * 16], 16);\n                            const vint8mf2_t b_s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(b_0_packed, 0xF, 16));\n                            const vint8mf2_t b_s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16));\n\n                            sumi_0_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4], b_s_0, 16);\n                            sumi_1_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 1], b_s_0, 16);\n                            sumi_2_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 2], b_s_0, 16);\n                            sumi_3_s_0_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_0_16, a_ptr[l].qs[j * 512 + 256 + i * 4 + 3], b_s_0, 16);\n\n                            sumi_0_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_0_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4], b_s_1, 16);\n                            sumi_1_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_1_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 1], b_s_1, 16);\n                            sumi_2_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_2_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 2], b_s_1, 16);\n                            sumi_3_s_1_16 = __riscv_vwmacc_vx_i16m1(sumi_3_s_1_16, a_ptr[l].qs[j * 512 + 384 + i * 4 + 3], b_s_1, 16);\n                        }\n\n                        sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n                                    sumi_0_s_0_16, 16);\n                        sumi_0 = __riscv_vwmacc_vv_i32m2(sumi_0,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n                                    sumi_0_s_1_16, 16);\n                        sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n                                    sumi_1_s_0_16, 16);\n                        sumi_1 = __riscv_vwmacc_vv_i32m2(sumi_1,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n                                    sumi_1_s_1_16, 16);\n                        sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n                                    sumi_2_s_0_16, 16);\n                        sumi_2 = __riscv_vwmacc_vv_i32m2(sumi_2,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n                                    sumi_2_s_1_16, 16);\n                        sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 2)),\n                                    sumi_3_s_0_16, 16);\n                        sumi_3 = __riscv_vwmacc_vv_i32m2(sumi_3,\n                                    __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vget_v_u16m4_u16m1(scales, 3)),\n                                    sumi_3_s_1_16, 16);\n                    }\n                }\n\n                const vfloat32m2_t b_d = __riscv_vfwcvt_f_f_v_f32m2(__riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16), 16);\n                const vfloat32m2_t d_0 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[0], 16);\n                const vfloat32m2_t d_1 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[1], 16);\n                const vfloat32m2_t d_2 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[2], 16);\n                const vfloat32m2_t d_3 = __riscv_vfmul_vf_f32m2(b_d, a_ptr[l].d[3], 16);\n\n                sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n                sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n                sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n                sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n            }\n\n            __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n        }\n    }\n    return;\n#endif\n    ggml_gemm_q4_K_16x1_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    const vint8mf2_t values = __riscv_vle8_v_i8mf2(kvalues_iq4nl, 16);\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n            // 4x16 Accumulators\n            vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n            for (int l = 0; l < nb; l++) {\n                // 4x16 integer accumulators\n                vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n                // Accumulation loop.\n                for (int i = 0; i < QK4_NL / 2; i++) {\n                    // Load `b_ptr`.\n                    const vuint8mf2_t b_0_packed = __riscv_vle8_v_u8mf2((const uint8_t *)&b_ptr[l].qs[i * 16], 16);\n                    const vint8mf2_t b_0_lo = __riscv_vrgather_vv_i8mf2(values, __riscv_vand_vx_u8mf2(b_0_packed, 0xf, 16), 16);\n                    const vint8mf2_t b_0_hi = __riscv_vrgather_vv_i8mf2(values, __riscv_vsrl_vx_u8mf2(b_0_packed, 4, 16), 16);\n                    // const vint16m1_t b_0_lo_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_lo, 16);\n                    // const vint16m1_t b_0_hi_16 = __riscv_vwcvt_x_x_v_i16m1(b_0_hi, 16);\n\n                    const vint16m1_t sumi_0_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4], 16);\n                    const vint16m1_t sumi_1_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 1], 16);\n                    const vint16m1_t sumi_2_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 2], 16);\n                    const vint16m1_t sumi_3_lo = __riscv_vwmul_vx_i16m1(b_0_lo, a_ptr[l].qs[i * 4 + 3], 16);\n\n                    const vint16m1_t sumi_0_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4], 16);\n                    const vint16m1_t sumi_1_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 1], 16);\n                    const vint16m1_t sumi_2_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 2], 16);\n                    const vint16m1_t sumi_3_hi = __riscv_vwmul_vx_i16m1(b_0_hi, a_ptr[l].qs[64 + i * 4 + 3], 16);\n\n                    sumi_0 = __riscv_vadd_vv_i32m2(sumi_0, __riscv_vwadd_vv_i32m2(sumi_0_lo, sumi_0_hi, 16), 16);\n                    sumi_1 = __riscv_vadd_vv_i32m2(sumi_1, __riscv_vwadd_vv_i32m2(sumi_1_lo, sumi_1_hi, 16), 16);\n                    sumi_2 = __riscv_vadd_vv_i32m2(sumi_2, __riscv_vwadd_vv_i32m2(sumi_2_lo, sumi_2_hi, 16), 16);\n                    sumi_3 = __riscv_vadd_vv_i32m2(sumi_3, __riscv_vwadd_vv_i32m2(sumi_3_lo, sumi_3_hi, 16), 16);\n                }\n\n                const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n                const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n                const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n                const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n                const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n                sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n                sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n                sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n                sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n            }\n\n            __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n        }\n    }\n    return;\n#endif\n    ggml_gemm_iq4_nl_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined __riscv_v_intrinsic\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n            // 4x16 Accumulators\n            vfloat32m2_t sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n            vfloat32m2_t sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, 16);\n\n            for (int l = 0; l < nb; l++) {\n                // 4x16 Integer Accumulators\n                vint32m2_t sumi_0 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_1 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_2 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n                vint32m2_t sumi_3 = __riscv_vmv_v_x_i32m2(0.0f, 16);\n\n                // Accumulation loop.\n                for (int i = 0; i < QK8_0; i++) {\n                    // Load `b_ptr`.\n                    const vint8mf2_t b_0 = __riscv_vle8_v_i8mf2((const int8_t *)&b_ptr[l].qs[i * 16], 16);\n                    // const vint16m1_t b_0_16 = __riscv_vwcvt_x_x_v_i16m1(b_0, 16);\n\n                    sumi_0 = __riscv_vwadd_wv_i32m2(sumi_0, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 0], 16), 16);\n                    sumi_1 = __riscv_vwadd_wv_i32m2(sumi_1, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 1], 16), 16);\n                    sumi_2 = __riscv_vwadd_wv_i32m2(sumi_2, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 2], 16), 16);\n                    sumi_3 = __riscv_vwadd_wv_i32m2(sumi_3, __riscv_vwmul_vx_i16m1(b_0, a_ptr[l].qs[i * 4 + 3], 16), 16);\n                }\n\n                const vfloat16m1_t b_d = __riscv_vle16_v_f16m1((const _Float16 *)b_ptr[l].d, 16);\n                const vfloat32m2_t d_0 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[0], 16);\n                const vfloat32m2_t d_1 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[1], 16);\n                const vfloat32m2_t d_2 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[2], 16);\n                const vfloat32m2_t d_3 = __riscv_vfwmul_vf_f32m2(b_d, *(const _Float16 *)&a_ptr[l].d[3], 16);\n\n                sumf_0 = __riscv_vfmacc_vv_f32m2(sumf_0, __riscv_vfcvt_f_x_v_f32m2(sumi_0, 16), d_0, 16);\n                sumf_1 = __riscv_vfmacc_vv_f32m2(sumf_1, __riscv_vfcvt_f_x_v_f32m2(sumi_1, 16), d_1, 16);\n                sumf_2 = __riscv_vfmacc_vv_f32m2(sumf_2, __riscv_vfcvt_f_x_v_f32m2(sumi_2, 16), d_2, 16);\n                sumf_3 = __riscv_vfmacc_vv_f32m2(sumf_3, __riscv_vfcvt_f_x_v_f32m2(sumi_3, 16), d_3, 16);\n            }\n\n            __riscv_vse32_v_f32m2(s + (y * 4 + 0) * bs + x * 16, sumf_0, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 1) * bs + x * 16, sumf_1, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 2) * bs + x * 16, sumf_2, 16);\n            __riscv_vse32_v_f32m2(s + (y * 4 + 3) * bs + x * 16, sumf_3, 16);\n        }\n    }\n    return;\n#endif\n    ggml_gemm_q8_0_16x1_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    assert(n % QK_K == 0);\n    const int num_k_blocks = n / QK_K;\n    const int N_ROWS_TILE = 4;\n    const int N_COLS_TILE = 16;\n    assert(nr % N_ROWS_TILE == 0);\n    assert(nc % N_COLS_TILE == 0);\n\n    const size_t vl = __riscv_vsetvl_e32m2(N_COLS_TILE);\n    // --- Tiling Loops ---\n#pragma GCC unroll 1\n    for (int row_tile = 0; row_tile < nr; row_tile += N_ROWS_TILE) {\n#pragma GCC unroll 1\n        for (int col_tile = 0; col_tile < nc; col_tile += N_COLS_TILE) {\n            // Base Pointers\n            const block_q8_Kx4* lhs_base_ptr = (const block_q8_Kx4*)vy + (row_tile / N_ROWS_TILE) * num_k_blocks;\n            const block_q2_Kx16* rhs_base_ptr = (const block_q2_Kx16*)vx + (col_tile / N_COLS_TILE) * num_k_blocks;\n\n            // Persistent Float Accumulators\n            vfloat32m2_t v_sumf_0 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n            vfloat32m2_t v_sumf_1 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n            vfloat32m2_t v_sumf_2 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n            vfloat32m2_t v_sumf_3 = __riscv_vfmv_v_f_f32m2(0.0f, vl);\n\n            // --- Super-Block Loop (K=0..255) ---\n#pragma GCC unroll 1\n            for (int k_block = 0; k_block < num_k_blocks; ++k_block) {\n                const block_q8_Kx4* lhs_current = &lhs_base_ptr[k_block];\n                const block_q2_Kx16* rhs_current = &rhs_base_ptr[k_block];\n\n                // 1. Load Global Min Scales (Keep as F16/LMUL=1 to save registers)\n                vfloat16m1_t v_g_min_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->dmin, vl);\n                vfloat32m2_t v_g_min_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_min_f16, vl);\n\n                // 2. Initialize Integer Accumulators\n                vint32m2_t v_isum_0 = __riscv_vmv_v_x_i32m2(0, vl);\n                vint32m2_t v_isum_1 = __riscv_vmv_v_x_i32m2(0, vl);\n                vint32m2_t v_isum_2 = __riscv_vmv_v_x_i32m2(0, vl);\n                vint32m2_t v_isum_3 = __riscv_vmv_v_x_i32m2(0, vl);\n\n                const uint8_t* rhs_qs_ptr = rhs_current->qs;\n                const uint8_t* rhs_sc_ptr = rhs_current->scales;\n                const int8_t*  lhs_qs_ptr = lhs_current->qs;\n\n                // --- Phase Loop (4 phases x 64 elements) ---\n#pragma GCC unroll 1\n                for (int phase = 0; phase < 4; ++phase) {\n\n                    // A. Load Scales/Mins for the 4 interleaved sub-blocks\n                    vuint16m1_t v_d_sb_0, v_d_sb_1, v_d_sb_2, v_d_sb_3;\n                    vuint16m1_t v_m_sb_0, v_m_sb_1, v_m_sb_2, v_m_sb_3;\n\n                    // Unrolled Load Logic\n                    {\n                        vuint8mf2_t v_raw;\n                        // Sub-block 0\n                        v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 0, vl);\n                        v_d_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                        v_m_sb_0 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                        // Sub-block 1\n                        v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 16, vl);\n                        v_d_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                        v_m_sb_1 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                        // Sub-block 2\n                        v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 32, vl);\n                        v_d_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                        v_m_sb_2 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                        // Sub-block 3\n                        v_raw = __riscv_vle8_v_u8mf2(rhs_sc_ptr + 48, vl);\n                        v_d_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vand_vx_u8mf2(v_raw, 0xF, vl), vl);\n                        v_m_sb_3 = __riscv_vzext_vf2_u16m1(__riscv_vsrl_vx_u8mf2(v_raw, 4, vl), vl);\n\n                        rhs_sc_ptr += 64;\n                    }\n\n                    int base_k_phase = (phase < 2) ? (phase * 16) : (128 + (phase-2)*16);\n                    int k_offsets[4] = {0, 32, 64, 96};\n\n                    // B. Inner Dot Product Loop\n#pragma GCC unroll 1\n                    for (int l = 0; l < 16; ++l) {\n                        vuint8mf2_t v_rhs_data = __riscv_vle8_v_u8mf2(rhs_qs_ptr, vl);\n                        rhs_qs_ptr += 16;\n\n                        // Unroll over 4 sub-blocks (0, 1, 2, 3 relative to phase)\n\n                        // --- Sub-block 0 ---\n                        {\n                            vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(v_rhs_data, 3, vl);\n                            vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                                __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                                __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_0), vl);\n\n                            const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[0] + l) * 4];\n                            v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n                            v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n                            v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n                            v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n                        }\n                        // --- Sub-block 1 ---\n                        {\n                            vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 2, vl), 3, vl);\n                            vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                                __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                                __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_1), vl);\n\n                            const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[1] + l) * 4];\n                            v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n                            v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n                            v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n                            v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n                        }\n                        // --- Sub-block 2 ---\n                        {\n                            vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 4, vl), 3, vl);\n                            vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                                __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                                __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_2), vl);\n\n                            const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[2] + l) * 4];\n                            v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n                            v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n                            v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n                            v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n                        }\n                        // --- Sub-block 3 ---\n                        {\n                            vuint8mf2_t v_q2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(v_rhs_data, 6, vl), 3, vl);\n                            vint16m1_t v_w = __riscv_vmul_vv_i16m1(\n                                __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vzext_vf2_u16m1(v_q2, vl)),\n                                __riscv_vreinterpret_v_u16m1_i16m1(v_d_sb_3), vl);\n\n                            const int8_t* q8 = &lhs_qs_ptr[(base_k_phase + k_offsets[3] + l) * 4];\n                            v_isum_0 = __riscv_vwmacc_vx_i32m2(v_isum_0, (int16_t)q8[0], v_w, vl);\n                            v_isum_1 = __riscv_vwmacc_vx_i32m2(v_isum_1, (int16_t)q8[1], v_w, vl);\n                            v_isum_2 = __riscv_vwmacc_vx_i32m2(v_isum_2, (int16_t)q8[2], v_w, vl);\n                            v_isum_3 = __riscv_vwmacc_vx_i32m2(v_isum_3, (int16_t)q8[3], v_w, vl);\n                        }\n                    }\n\n                    // C CORRECTION\n                    int sb_base_abs = base_k_phase / 16;\n\n                    // --- Correction Sub-block 0 ---\n                    {\n                        int sb_abs = sb_base_abs + (k_offsets[0] / 16);\n                        vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_0);\n\n                        // Row 0\n                        vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n                        vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n                        vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n                        // Row 1\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n                        // Row 2\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n                        // Row 3\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n                    }\n\n                    // --- Correction Sub-block 1 ---\n                    {\n                        int sb_abs = sb_base_abs + (k_offsets[1] / 16);\n                        vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_1);\n\n                        vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n                        vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n                        vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n                    }\n\n                    // --- Correction Sub-block 2 ---\n                    {\n                        int sb_abs = sb_base_abs + (k_offsets[2] / 16);\n                        vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_2);\n\n                        vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n                        vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n                        vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n                    }\n\n                    // --- Correction Sub-block 3 ---\n                    {\n                        int sb_abs = sb_base_abs + (k_offsets[3] / 16);\n                        vint16m1_t v_min = __riscv_vreinterpret_v_u16m1_i16m1(v_m_sb_3);\n\n                        vfloat32m2_t v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[0], vl);\n                        vint32m2_t v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 0], vl);\n                        vfloat32m2_t vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_0 = __riscv_vfsub_vv_f32m2(v_sumf_0, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[1], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 1], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_1 = __riscv_vfsub_vv_f32m2(v_sumf_1, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[2], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 2], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_2 = __riscv_vfsub_vv_f32m2(v_sumf_2, vf_c, vl);\n\n                        v_g_min = __riscv_vfmul_vf_f32m2(v_g_min_base, lhs_current->d[3], vl);\n                        v_c = __riscv_vwmul_vx_i32m2(v_min, lhs_current->bsums[sb_abs * 4 + 3], vl);\n                        vf_c = __riscv_vfmul_vv_f32m2(__riscv_vfcvt_f_x_v_f32m2(v_c, vl), v_g_min, vl);\n                        v_sumf_3 = __riscv_vfsub_vv_f32m2(v_sumf_3, vf_c, vl);\n                    }\n\n                } // End Phase Loop\n\n                // --- Apply Main Scales ---\n                vfloat16m1_t v_g_all_f16 = __riscv_vle16_v_f16m1((const _Float16*)rhs_current->d, vl);\n                vfloat32m2_t v_g_all_base = __riscv_vfwcvt_f_f_v_f32m2(v_g_all_f16, vl);\n\n                {\n                    vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[0], vl);\n                    vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_0, vl);\n                    v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n                    v_sumf_0 = __riscv_vfadd_vv_f32m2(v_sumf_0, v_sum, vl);\n                }\n                // Row 1\n                {\n                    vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[1], vl);\n                    vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_1, vl);\n                    v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n                    v_sumf_1 = __riscv_vfadd_vv_f32m2(v_sumf_1, v_sum, vl);\n                }\n                // Row 2\n                {\n                    vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[2], vl);\n                    vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_2, vl);\n                    v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n                    v_sumf_2 = __riscv_vfadd_vv_f32m2(v_sumf_2, v_sum, vl);\n                }\n                // Row 3\n                {\n                    vfloat32m2_t v_g_all = __riscv_vfmul_vf_f32m2(v_g_all_base, lhs_current->d[3], vl);\n                    vfloat32m2_t v_sum = __riscv_vfcvt_f_x_v_f32m2(v_isum_3, vl);\n                    v_sum = __riscv_vfmul_vv_f32m2(v_sum, v_g_all, vl);\n                    v_sumf_3 = __riscv_vfadd_vv_f32m2(v_sumf_3, v_sum, vl);\n                }\n\n            } // End K-Block\n\n            __riscv_vse32_v_f32m2(s + (row_tile + 0) * bs + col_tile, v_sumf_0, vl);\n            __riscv_vse32_v_f32m2(s + (row_tile + 1) * bs + col_tile, v_sumf_1, vl);\n            __riscv_vse32_v_f32m2(s + (row_tile + 2) * bs + col_tile, v_sumf_2, vl);\n            __riscv_vse32_v_f32m2(s + (row_tile + 3) * bs + col_tile, v_sumf_3, vl);\n        }\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp",
    "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__s390x__)\n#include <sys/auxv.h>\n\n// find hwcap bits in asm/elf.h\n#ifndef HWCAP_VXRS_EXT2\n#define HWCAP_VXRS_EXT2 (1 << 15)\n#endif\n\n#ifndef HWCAP_NNPA\n#define HWCAP_NNPA (1 << 20)\n#endif\n\nstruct s390x_features {\n    bool has_vxe2 = false;\n    bool has_nnpa = false;\n\n    s390x_features() {\n        uint32_t hwcap = getauxval(AT_HWCAP);\n        // NOTE: use hwcap2 with DFLT for z17 and later\n        // uint32_t hwcap2 = getauxval(AT_HWCAP2);\n\n        has_vxe2 = !!(hwcap & HWCAP_VXRS_EXT2);\n        has_nnpa = !!(hwcap & HWCAP_NNPA);\n    }\n};\n\nstatic int ggml_backend_cpu_s390x_score() {\n    int score = 1;\n    s390x_features sf;\n\n// IBM z15 / LinuxONE 3\n#ifdef GGML_USE_VXE2\n    if (!sf.has_vxe2) { return 0; }\n    score += 1 << 1;\n#endif\n\n// IBM z16 / LinuxONE 4 and z17 / LinuxONE 5\n#ifdef GGML_USE_NNPA\n    if (!sf.has_nnpa) { return 0; }\n    score += 1 << 2;\n#endif\n\n    return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_s390x_score)\n\n#endif  // __s390x__\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/s390/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__VXE__) || defined(__VXE2__)\n#define B1(c,s,n)  0x ## n ## c ,  0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s  ) B7(c,s,     c), B7(c,s,     s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const __attribute__((aligned(16))) uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b ) << 4\nstatic const __attribute__((aligned(16))) uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n\n// permute mask for byteswapping\nstatic const uint8x16_t v_kperm = (const uint8x16_t){\n     7,  6,  5,  4,  3,  2, 1, 0,\n    15, 14, 13, 12, 11, 10, 9, 8\n};\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    for (int i = 0; i < nb; i++) {\n        float32x4_t srcv [8];\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n                                   vec_extract(amaxv[0], 1)),\n                               MAX(vec_extract(amaxv[0], 2),\n                                   vec_extract(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f / d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        for (int j = 0; j < 8; j++) {\n            const float32x4_t v = vec_mul(srcv[j], vec_splats(id));\n            /* Uses non-default rounding for vec_signed or vec_round */\n            const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));\n\n            y[i].qs[4*j + 0] = vec_extract(vi, 0);\n            y[i].qs[4*j + 1] = vec_extract(vi, 1);\n            y[i].qs[4*j + 2] = vec_extract(vi, 2);\n            y[i].qs[4*j + 3] = vec_extract(vi, 3);\n        }\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    for (int i = 0; i < nb; i++) {\n        float32x4_t srcv [8];\n        float32x4_t asrcv[8];\n        float32x4_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]);\n        for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(vec_extract(amaxv[0], 0),\n                                   vec_extract(amaxv[0], 1)),\n                               MAX(vec_extract(amaxv[0], 2),\n                                   vec_extract(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f / d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        int32x4_t acc = vec_splats(0);\n\n        for (int j = 0; j < 8; j++) {\n            const float32x4_t v = vec_mul(srcv[j], vec_splats(id));\n            /* Uses non-default rounding for vec_signed or vec_round */\n            const int32x4_t vi = vec_signed(__builtin_s390_vfisb(v, 4, 1));\n\n            y[i].qs[4*j + 0] = vec_extract(vi, 0);\n            y[i].qs[4*j + 1] = vec_extract(vi, 1);\n            y[i].qs[4*j + 2] = vec_extract(vi, 2);\n            y[i].qs[4*j + 3] = vec_extract(vi, 3);\n\n            acc = vec_add(acc, vi);\n        }\n\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * (acc[0] + acc[1] + acc[2] + acc[3]));\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float32x4_t acc = vec_splats(0.0f);\n\n    const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);\n    const int8x16_t  v_s = vec_splats( (const int8_t)0x08);\n\n    for (; ib < nb; ++ib) {\n        const uint8x16_t v_x = vec_xl(0, x[ib].qs);\n        const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);\n        const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);\n\n        const int8x16_t v_xls = vec_sub(v_xl, v_s);\n        const int8x16_t v_xhs = vec_sub(v_xh, v_s);\n\n        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);\n        const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);\n\n        const int16x8_t v_xylso = vec_mulo(v_xls, v_yl);\n        const int16x8_t v_xyl = vec_meadd(v_xls, v_yl, v_xylso);\n        const int16x8_t v_xyhso = vec_mulo(v_xhs, v_yh);\n        const int16x8_t v_xyh = vec_meadd(v_xhs, v_yh, v_xyhso);\n\n        int16x8_t v_xy_ = v_xyl + v_xyh; v_xy_ += vec_reve(v_xy_);\n\n        const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_));\n        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n        acc = vec_madd(v_xy, v_d, acc);\n    }\n\n    sumf = vec_hsum_f32x4(acc);\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float summs = 0;\n    float32x4_t acc = vec_splats(0.0f);\n\n    const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n#pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        const uint8x16_t v_x = vec_xl(0, x[ib].qs);\n        const int8x16_t v_xl = (const int8x16_t)(v_x & v_m);\n        const int8x16_t v_xh = (const int8x16_t)(v_x >> 4);\n\n        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);\n        const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs);\n\n        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n        const float32x4_t v_xy = vec_float(v_xy_);\n\n        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n        acc = vec_madd(v_xy, v_d, acc);\n    }\n\n    sumf = vec_hsum_f32x4(acc) + summs;\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const int qk = QK_MXFP4;\n    const int nb = n / qk;\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0  * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    const int8x16_t  v_k = vec_xl(0, kvalues_mxfp4);\n    const uint8x16_t v_m = vec_splats((const uint8_t)0x0F);\n\n    float32x4_t v_acc = vec_splats(0.0f);\n\n    #pragma GCC unroll 8\n    for (; ib + 1 < nb; ib += 2) {\n        const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_mxfp4 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0  * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_0  * GGML_RESTRICT y1 = &y[ib + 1];\n\n        const uint8x16_t v_x0 = vec_xl(0, x0->qs);\n        const uint8x16_t v_x1 = vec_xl(0, x1->qs);\n\n        int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n        int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n        int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n        int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n        v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);\n        v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);\n        v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);\n        v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);\n\n        const int8x16_t v_y0l = vec_xl(0,       y0->qs);\n        const int8x16_t v_y0h = vec_xl(QK8_0/2, y0->qs);\n        const int8x16_t v_y1l = vec_xl(0,       y1->qs);\n        const int8x16_t v_y1h = vec_xl(QK8_0/2, y1->qs);\n\n        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0l), v_x0h, v_y0h);\n        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y1l), v_x1h, v_y1h);\n\n        const float32x4_t v_xy0f = vec_float(v_xy0);\n        const float32x4_t v_xy1f = vec_float(v_xy1);\n\n        const float32x4_t v_d0 = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));\n        const float32x4_t v_d1 = vec_splats(GGML_E8M0_TO_FP32_HALF(x1->e) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n        v_acc = vec_madd(v_xy0f, v_d0, v_acc);\n        v_acc = vec_madd(v_xy1f, v_d1, v_acc);\n    }\n\n    for (; ib < nb; ++ib) {\n        const block_mxfp4 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q8_0  * GGML_RESTRICT y0 = &y[ib + 0];\n\n        const uint8x16_t v_x = vec_xl(0, x0->qs);\n\n        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n        v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);\n        v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);\n\n        const int8x16_t v_yl = vec_xl(0,       y0->qs);\n        const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);\n\n        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n        const float32x4_t v_xyf = vec_float(v_xy);\n\n        const float32x4_t v_d = vec_splats(GGML_E8M0_TO_FP32_HALF(x0->e) * GGML_CPU_FP16_TO_FP32(y0->d));\n        v_acc = vec_madd(v_xyf, v_d, v_acc);\n    }\n\n    sumf = vec_hsum_f32x4(v_acc);\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float32x4_t v_sum0 = vec_splats(0.0f);\n    float32x4_t v_sum1 = vec_splats(0.0f);\n\n    uint32_t qh0, qh1;\n    uint64_t tmp0[4], tmp1[4];\n\n    const uint8x16_t v_m = vec_splats((uint8_t)0x0F);\n\n    #pragma GCC unroll 4\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q5_0 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q5_0 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        memcpy(&qh0, x0->qh, sizeof(qh0));\n        memcpy(&qh1, x1->qh, sizeof(qh1));\n\n        tmp0[0] = table_b2b_1[(qh0 >>  0) & 0xFF];\n        tmp0[1] = table_b2b_1[(qh0 >>  8) & 0xFF];\n        tmp0[2] = table_b2b_1[(qh0 >> 16) & 0xFF];\n        tmp0[3] = table_b2b_1[(qh0 >> 24)       ];\n\n        tmp1[0] = table_b2b_1[(qh1 >>  0) & 0xFF];\n        tmp1[1] = table_b2b_1[(qh1 >>  8) & 0xFF];\n        tmp1[2] = table_b2b_1[(qh1 >> 16) & 0xFF];\n        tmp1[3] = table_b2b_1[(qh1 >> 24)       ];\n\n        int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));\n        int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));\n        int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));\n        int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));\n\n        // required for fixing the byteorder\n        v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);\n        v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);\n        v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);\n        v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);\n\n        const uint8x16_t v_x0 = vec_xl(0, (const uint8_t *)x0->qs);\n        const uint8x16_t v_x1 = vec_xl(0, (const uint8_t *)x1->qs);\n\n        int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n        int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n        int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n        int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n        const int8x16_t v_x0lf = vec_sub(v_x0l, v_qh0l);\n        const int8x16_t v_x0hf = vec_sub(v_x0h, v_qh0h);\n        const int8x16_t v_x1lf = vec_sub(v_x1l, v_qh1l);\n        const int8x16_t v_x1hf = vec_sub(v_x1h, v_qh1h);\n\n        const int8x16_t v_y0l = vec_xl(0,       (const int8_t *)y0->qs);\n        const int8x16_t v_y0h = vec_xl(QK8_0/2, (const int8_t *)y0->qs);\n        const int8x16_t v_y1l = vec_xl(0,       (const int8_t *)y1->qs);\n        const int8x16_t v_y1h = vec_xl(QK8_0/2, (const int8_t *)y1->qs);\n\n        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);\n        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);\n\n        const float32x4_t v_xy0f = vec_float(v_xy0);\n        const float32x4_t v_xy1f = vec_float(v_xy1);\n\n        const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n        const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n        v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);\n        v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);\n    }\n\n    sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1);\n\n    #pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n        uint32_t qh;\n        memcpy(&qh, x0->qh, sizeof(qh));\n\n        uint64_t tmp[4];\n        tmp[0] = table_b2b_1[(qh >>  0) & 0xFF];\n        tmp[1] = table_b2b_1[(qh >>  8) & 0xFF];\n        tmp[2] = table_b2b_1[(qh >> 16) & 0xFF];\n        tmp[3] = table_b2b_1[(qh >> 24)       ];\n\n        int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));\n        int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));\n\n        // required for fixing the byteorder\n        v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);\n        v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);\n\n        const uint8x16_t v_x = vec_xl(0, (const uint8_t *)x0->qs);\n        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n        const int8x16_t v_xlf = vec_sub(v_xl, v_qhl);\n        const int8x16_t v_xhf = vec_sub(v_xh, v_qhh);\n\n        const int8x16_t v_yl = vec_xl(0,       (const int8_t *)y0->qs);\n        const int8x16_t v_yh = vec_xl(QK8_0/2, (const int8_t *)y0->qs);\n\n        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);\n        const float32x4_t v_xyf = vec_float(v_xy);\n\n        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n        const float32x4_t v_acc = vec_madd(v_xyf, v_d, vec_splats(0.0f));\n\n        sumf += vec_hsum_f32x4(v_acc);\n    }\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0.0f;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float32x4_t v_sum0 = vec_splats(0.0f);\n    float32x4_t v_sum1 = vec_splats(0.0f);\n\n    float summs0 = 0.0f;\n    float summs1 = 0.0f;\n\n    uint32_t qh0;\n    uint32_t qh1;\n\n    uint64_t tmp0[4];\n    uint64_t tmp1[4];\n\n    const uint8x16_t v_m = vec_splats((uint8_t)0x0F);\n\n    #pragma GCC unroll 4\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q5_1 * GGML_RESTRICT x0 = &x[ib + 0];\n        const block_q5_1 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_1 * GGML_RESTRICT y0 = &y[ib + 0];\n        const block_q8_1 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        summs0 += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n        summs1 += GGML_CPU_FP16_TO_FP32(x1->m) * GGML_CPU_FP16_TO_FP32(y1->s);\n\n        memcpy(&qh0, x0->qh, sizeof(qh0));\n        memcpy(&qh1, x1->qh, sizeof(qh1));\n\n        tmp0[0] = table_b2b_0[(qh0 >>  0) & 0xFF];\n        tmp0[1] = table_b2b_0[(qh0 >>  8) & 0xFF];\n        tmp0[2] = table_b2b_0[(qh0 >> 16) & 0xFF];\n        tmp0[3] = table_b2b_0[(qh0 >> 24)       ];\n\n        tmp1[0] = table_b2b_0[(qh1 >>  0) & 0xFF];\n        tmp1[1] = table_b2b_0[(qh1 >>  8) & 0xFF];\n        tmp1[2] = table_b2b_0[(qh1 >> 16) & 0xFF];\n        tmp1[3] = table_b2b_0[(qh1 >> 24)       ];\n\n        int8x16_t v_qh0l = vec_xl(0, (const int8_t *)(tmp0 + 0));\n        int8x16_t v_qh0h = vec_xl(0, (const int8_t *)(tmp0 + 2));\n        int8x16_t v_qh1l = vec_xl(0, (const int8_t *)(tmp1 + 0));\n        int8x16_t v_qh1h = vec_xl(0, (const int8_t *)(tmp1 + 2));\n\n        // required for fixing the byteorder\n        v_qh0l = vec_perm(v_qh0l, v_qh0l, v_kperm);\n        v_qh0h = vec_perm(v_qh0h, v_qh0h, v_kperm);\n        v_qh1l = vec_perm(v_qh1l, v_qh1l, v_kperm);\n        v_qh1h = vec_perm(v_qh1h, v_qh1h, v_kperm);\n\n        const uint8x16_t v_x0 = vec_xl(0, x0->qs);\n        const uint8x16_t v_x1 = vec_xl(0, x1->qs);\n\n        const int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n        const int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n        const int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n        const int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n        const int8x16_t v_x0lf = vec_or(v_x0l, v_qh0l);\n        const int8x16_t v_x0hf = vec_or(v_x0h, v_qh0h);\n        const int8x16_t v_x1lf = vec_or(v_x1l, v_qh1l);\n        const int8x16_t v_x1hf = vec_or(v_x1h, v_qh1h);\n\n        const int8x16_t v_y0l = vec_xl(0      , y0->qs);\n        const int8x16_t v_y0h = vec_xl(QK8_1/2, y0->qs);\n        const int8x16_t v_y1l = vec_xl(0      , y1->qs);\n        const int8x16_t v_y1h = vec_xl(QK8_1/2, y1->qs);\n\n        const int32x4_t v_xy0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0lf, v_y0l), v_x0hf, v_y0h);\n        const int32x4_t v_xy1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1lf, v_y1l), v_x1hf, v_y1h);\n\n        const float32x4_t v_xy0f = vec_float(v_xy0);\n        const float32x4_t v_xy1f = vec_float(v_xy1);\n\n        const float32x4_t v_d0 = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n        const float32x4_t v_d1 = vec_splats(GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d));\n\n        v_sum0 = vec_madd(v_xy0f, v_d0, v_sum0);\n        v_sum1 = vec_madd(v_xy1f, v_d1, v_sum1);\n    }\n\n    sumf += vec_hsum_f32x4(v_sum0) + vec_hsum_f32x4(v_sum1) + summs0 + summs1;\n\n    #pragma GCC unroll 4\n    for (; ib < nb; ++ib) {\n        const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n\n        float summs = GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n\n        uint32_t qh;\n        memcpy(&qh, x0->qh, sizeof(qh));\n\n        uint64_t tmp[4];\n        tmp[0] = table_b2b_0[(qh >>  0) & 0xFF];\n        tmp[1] = table_b2b_0[(qh >>  8) & 0xFF];\n        tmp[2] = table_b2b_0[(qh >> 16) & 0xFF];\n        tmp[3] = table_b2b_0[(qh >> 24)       ];\n\n        int8x16_t v_qhl = vec_xl(0, (const int8_t *)(tmp + 0));\n        int8x16_t v_qhh = vec_xl(0, (const int8_t *)(tmp + 2));\n\n        // required for fixing the byteorder\n        v_qhl = vec_perm(v_qhl, v_qhl, v_kperm);\n        v_qhh = vec_perm(v_qhh, v_qhh, v_kperm);\n\n        const uint8x16_t v_x = vec_xl(0, x0->qs);\n        const int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n        const int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n        const int8x16_t v_xlf = vec_or(v_xl, v_qhl);\n        const int8x16_t v_xhf = vec_or(v_xh, v_qhh);\n\n        const int8x16_t v_yl = vec_xl(0      , y0->qs);\n        const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs);\n\n        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xlf, v_yl), v_xhf, v_yh);\n        const float32x4_t v_xyf = vec_float(v_xy);\n\n        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d));\n        const float32x4_t v_acc = vec_madd(v_xyf, v_d, v_acc);\n\n        sumf += vec_hsum_f32x4(v_acc) + summs;\n    }\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float32x4_t acc = vec_splats(0.0f);\n\n#pragma GCC unroll 8\n    for (; ib < nb; ++ib) {\n        __builtin_prefetch(x[ib].qs, 0, 1);\n        __builtin_prefetch(y[ib].qs, 0, 1);\n\n        const int8x16_t v_xl = vec_xl(0      , x[ib].qs);\n        const int8x16_t v_xh = vec_xl(QK8_0/2, x[ib].qs);\n        const int8x16_t v_yl = vec_xl(0      , y[ib].qs);\n        const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs);\n\n        const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n        const float32x4_t v_xy = vec_float(v_xy_);\n        const float32x4_t v_d = vec_splats(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n        acc = vec_madd(v_xy, v_d, acc);\n    }\n\n    sumf = vec_hsum_f32x4(acc);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    uint32_t aux[3];\n    uint32_t utmp[4];\n\n    const int32x4_t v_z = vec_splat_s32(0);\n    const uint8x16_t v_3m = vec_splat_u8(0x03);\n\n    const uint8x16_t v_0c = vec_splat_u8(1);\n    const uint8x16_t v_1c = vec_sl(v_0c, 1);\n    const uint8x16_t v_2c = vec_sl(v_0c, 2);\n    const uint8x16_t v_3c = vec_sl(v_0c, 3);\n\n    uint8x16_t q3h[4];\n    uint8x16_t q3b[2];\n    int8x16_t q3bytes[4];\n    int8x16_t q8bytes[8];\n    uint8x16_t qhbits[2];\n\n    float sum = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * restrict x0l = x[i].qs;\n        const uint8_t * restrict x0h = x[i].hmask;\n        const int8_t  * restrict y0  = y[i].qs;\n\n        qhbits[0] = vec_xl(0 , x0h);\n        qhbits[1] = vec_xl(16, x0h);\n\n        int32_t isum = 0;\n\n        memcpy(aux, x[i].scales, 12);\n        utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4);\n        utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4);\n        utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4);\n        utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4);\n\n        int8_t * scale = (int8_t *)utmp;\n        for (int j = 0; j < 16; ++j) scale[j] -= 32;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            int32x4_t isum0, isum1, isum2, isum3;\n\n            q3b[0] = vec_xl(0 , x0l);\n            q3b[1] = vec_xl(16, x0l);\n            x0l += 32;\n\n            q8bytes[0] = vec_xl(0  , y0);\n            q8bytes[1] = vec_xl(16 , y0);\n            q8bytes[2] = vec_xl(32 , y0);\n            q8bytes[3] = vec_xl(48 , y0);\n            q8bytes[4] = vec_xl(64 , y0);\n            q8bytes[5] = vec_xl(80 , y0);\n            q8bytes[6] = vec_xl(96 , y0);\n            q8bytes[7] = vec_xl(112, y0);\n            y0 += 128;\n\n            q3h[0] = vec_sl(vec_andc(v_0c, qhbits[0]), 2);\n            q3h[1] = vec_sl(vec_andc(v_0c, qhbits[1]), 2);\n            q3h[2] = vec_sl(vec_andc(v_1c, qhbits[0]), 1);\n            q3h[3] = vec_sl(vec_andc(v_1c, qhbits[1]), 1);\n\n            q3bytes[0] = vec_sub((int8x16_t)vec_and(q3b[0], v_3m), (int8x16_t)q3h[0]);\n            q3bytes[1] = vec_sub((int8x16_t)vec_and(q3b[1], v_3m), (int8x16_t)q3h[1]);\n            q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 2), v_3m), (int8x16_t)q3h[2]);\n            q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 2), v_3m), (int8x16_t)q3h[3]);\n\n            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[0]);\n            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[1]);\n            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[2]);\n            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[3]);\n\n            isum += (isum0[0] + isum0[1] + isum0[2] + isum0[3]) * scale[0];\n            isum += (isum1[0] + isum1[1] + isum1[2] + isum1[3]) * scale[1];\n            isum += (isum2[0] + isum2[1] + isum2[2] + isum2[3]) * scale[2];\n            isum += (isum3[0] + isum3[1] + isum3[2] + isum3[3]) * scale[3];\n\n            scale += 4;\n\n            q3h[0] = vec_andc(v_2c, qhbits[0]);\n            q3h[1] = vec_andc(v_2c, qhbits[1]);\n            q3h[2] = vec_sr(vec_andc(v_3c, qhbits[0]), 1);\n            q3h[3] = vec_sr(vec_andc(v_3c, qhbits[1]), 1);\n\n            q3bytes[0] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 4), v_3m), (int8x16_t)q3h[0]);\n            q3bytes[1] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 4), v_3m), (int8x16_t)q3h[1]);\n            q3bytes[2] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[0], 6), v_3m), (int8x16_t)q3h[2]);\n            q3bytes[3] = vec_sub((int8x16_t)vec_and(vec_sr(q3b[1], 6), v_3m), (int8x16_t)q3h[3]);\n\n            isum0 = ggml_vec_dot(v_z, q3bytes[0], q8bytes[4]);\n            isum1 = ggml_vec_dot(v_z, q3bytes[1], q8bytes[5]);\n            isum2 = ggml_vec_dot(v_z, q3bytes[2], q8bytes[6]);\n            isum3 = ggml_vec_dot(v_z, q3bytes[3], q8bytes[7]);\n\n            isum += vec_hsum_i32x4(isum0) * scale[0];\n            isum += vec_hsum_i32x4(isum1) * scale[1];\n            isum += vec_hsum_i32x4(isum2) * scale[2];\n            isum += vec_hsum_i32x4(isum3) * scale[3];\n\n            scale += 4;\n\n            if (j == 0) {\n                qhbits[0] = vec_sr(qhbits[0], 4);\n                qhbits[1] = vec_sr(qhbits[1], 4);\n            }\n        }\n\n        sum += d * isum;\n    }\n\n    *s = sum;\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined(__VXE__) || defined(__VXE2__)\n    const uint8x16_t v_lm = vec_splat_u8(0x0F);\n    const int32x4_t v_z = vec_splat_s32(0);\n\n    uint8x16_t v_x[2];\n    int8x16_t  v_xl[2];\n    int8x16_t  v_y[2];\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n        const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);\n\n        memcpy(utmp, x[i].scales, 12);\n\n        uint32x4_t v_mins8 = { 0 };\n        v_mins8 = vec_insert(utmp[1] & kmask1, v_mins8, 0);\n        v_mins8 = vec_insert(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1);\n\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[0] &= kmask1;\n\n        const int16x8_t v_minsh = (int16x8_t)vec_unpackh((uint8x16_t)v_mins8);\n\n        const int32x4_t v_minso = vec_mulo(v_ysums, v_minsh);\n        const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minso);\n        sumf -= dmin * (v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]);\n\n        const uint8_t * scales = (const uint8_t *)utmp;\n        const uint8_t * GGML_RESTRICT x0 = x[i].qs;\n        const int8_t  * GGML_RESTRICT y0 = y[i].qs;\n\n        int32_t sumi1 = 0;\n        int32_t sumi2 = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            v_x[0] = vec_xl(0 , x0);\n            v_x[1] = vec_xl(16, x0);\n            x0 += 32;\n\n            v_y[0] = vec_xl(0 , y0);\n            v_y[1] = vec_xl(16, y0);\n            y0 += 32;\n\n            v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm);\n            v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm);\n\n            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);\n            sumi1 += vec_hsum_i32x4(p1) * scales[2*j+0];\n\n            v_y[0] = vec_xl(0 , y0);\n            v_y[1] = vec_xl(16, y0);\n            y0 += 32;\n\n            v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4);\n            v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4);\n\n            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]);\n            sumi2 += vec_hsum_i32x4(p2) * scales[2*j+1];\n        }\n\n        sumf += d * (sumi1 + sumi2);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined(__VXE__) || defined(__VXE2__)\n    const uint8x16_t v_lm = vec_splat_u8(0x0F);\n    const uint8x16_t v_1m = vec_splat_u8(0x01);\n    const uint8x16_t v_2m = vec_splat_u8(0x02);\n\n    const int32x4_t v_z = vec_splat_s32(0);\n\n    const uchar8x16_t v_minsm = {\n        0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,\n        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF\n    };\n\n    int8x16_t  q5b[4];\n    uint8x16_t q5h[4];\n\n    uint8x16_t v_xl[2];\n    uint8x16_t v_xh[2];\n    int8x16_t  v_y[4];\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n        const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const uint8x16_t v_mins16 = vec_xl(0, (const uint8_t *)utmp);\n        const uint8x16_t v_mins8 = vec_perm(v_mins16, v_mins16, v_minsm);\n        const int16x8_t v_minsh = (int16x8_t)vec_unpackh(v_mins8);\n\n        const int32x4_t v_minsho = vec_mulo(v_ysums, v_minsh);\n        const int32x4_t v_mins = vec_meadd(v_ysums, v_minsh, v_minsho);\n        const int32_t mins = vec_hsum_i32x4(v_mins);\n\n        const uint8_t * scales = (const uint8_t *)utmp;\n        const uint8_t * GGML_RESTRICT x0l = x[i].qs;\n        const uint8_t * GGML_RESTRICT x0h = x[i].qh;\n        const int8_t  * GGML_RESTRICT y0 = y[i].qs;\n\n        v_xh[0] = vec_xl(0 , x0h);\n        v_xh[1] = vec_xl(16, x0h);\n\n        int32_t sumi = 0;\n        for (int j = 0; j < QK_K/64; ++j) {\n            v_xl[0] = vec_xl(0 , x0l);\n            v_xl[1] = vec_xl(16, x0l);\n            x0l += 32;\n\n            v_y[0] = vec_xl(0 , y0);\n            v_y[1] = vec_xl(16, y0);\n            v_y[2] = vec_xl(32, y0);\n            v_y[3] = vec_xl(48, y0);\n            y0 += 64;\n\n            q5h[0] = vec_sl(vec_and(v_1m, v_xh[0]), 4);\n            q5h[1] = vec_sl(vec_and(v_1m, v_xh[1]), 4);\n            q5h[2] = vec_sl(vec_and(v_2m, v_xh[0]), 3);\n            q5h[3] = vec_sl(vec_and(v_2m, v_xh[1]), 3);\n            v_xh[0] = vec_sr(v_xh[0], 2);\n            v_xh[1] = vec_sr(v_xh[1], 2);\n\n            q5b[0] = (int8x16_t)vec_or(vec_and(v_xl[0], v_lm), q5h[0]);\n            q5b[1] = (int8x16_t)vec_or(vec_and(v_xl[1], v_lm), q5h[1]);\n            q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]);\n            q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]);\n\n            int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]);\n            int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]);\n\n            sumi += vec_hsum_i32x4(sumi0) * *scales++;\n            sumi += vec_hsum_i32x4(sumi1) * *scales++;\n        }\n\n        sumf += d * sumi - dmin * mins;\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    float sum = 0;\n\n    // Lower 4-bit and upper 2-bit masks\n    const uint8x16_t v_lm = vec_splat_u8(0x0F);\n    const uint8x16_t v_um = vec_splat_u8(0x03);\n\n    const int32x4_t v_z = vec_splat_s32(0);\n\n    int8x16_t  q6b[4];\n    uint8x16_t q6h[4];\n\n    uint8x16_t v_xl[4];\n    uint8x16_t v_xh[2];\n    int8x16_t  v_y[4];\n\n    for (int i = 0; i < nb; ++i) {\n        const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT x0l = x[i].ql;\n        const uint8_t * GGML_RESTRICT x0h = x[i].qh;\n        const int8_t  * GGML_RESTRICT y0 = y[i].qs;\n\n        const int8_t  * GGML_RESTRICT scale = x[i].scales;\n\n        const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums);\n        const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums);\n\n        const int8x16_t v_scale  = vec_xl(0, scale);\n        const int16x8_t v_scalel = vec_unpackh(v_scale);\n        const int16x8_t v_scaleh = vec_unpackl(v_scale);\n\n        const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel);\n        const int32x4_t v_minsl = vec_meadd(v_ysumsl, v_scalel, v_minslo);\n        const int32x4_t v_minsho = vec_mulo(v_ysumsh, v_scaleh);\n        const int32x4_t v_minsh = vec_meadd(v_ysumsh, v_scaleh, v_minsho);\n        const int32x4_t v_mins = vec_add(v_minsl, v_minsh);\n\n        const int32_t mins = vec_hsum_i32x4(v_mins);\n\n        int32_t isum = 0;\n        for (int j = 0; j < QK_K/128; ++j) {\n            // Load model upper 2 bits\n            v_xh[0] = vec_xl(0 , x0h);\n            v_xh[1] = vec_xl(16, x0h);\n            x0h += 32;\n\n            // Load model lower 4 bits\n            v_xl[0] = vec_xl(0 , x0l);\n            v_xl[1] = vec_xl(16, x0l);\n            v_xl[2] = vec_xl(32, x0l);\n            v_xl[3] = vec_xl(48, x0l);\n            x0l += 64;\n\n            // Load activation quants\n            v_y[0] = vec_xl(0 , y0);\n            v_y[1] = vec_xl(16, y0);\n            v_y[2] = vec_xl(32, y0);\n            v_y[3] = vec_xl(48, y0);\n            y0 += 64;\n\n            q6h[0] = vec_sl(vec_and(v_um, v_xh[0]), 4);\n            q6h[1] = vec_sl(vec_and(v_um, v_xh[1]), 4);\n            uint8x16_t shifted = vec_sr(v_xh[0], 2);\n            q6h[2] = vec_sl(vec_and(v_um, shifted), 4);\n            shifted = vec_sr(v_xh[1], 2);\n            q6h[3] = vec_sl(vec_and(v_um, shifted), 4);\n\n            q6b[0] = (int8x16_t)(vec_or(vec_and(v_xl[0], v_lm), q6h[0]));\n            q6b[1] = (int8x16_t)(vec_or(vec_and(v_xl[1], v_lm), q6h[1]));\n            q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2]));\n            q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3]));\n\n            int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);\n            int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);\n            int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);\n            int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);\n\n            isum += vec_hsum_i32x4(summs0) * scale[0] +\n                    vec_hsum_i32x4(summs1) * scale[1] +\n                    vec_hsum_i32x4(summs2) * scale[2] +\n                    vec_hsum_i32x4(summs3) * scale[3];\n\n            scale += 4;\n\n\n            // Load activation quants\n            v_y[0] = vec_xl(0 , y0);\n            v_y[1] = vec_xl(16, y0);\n            v_y[2] = vec_xl(32, y0);\n            v_y[3] = vec_xl(48, y0);\n            y0 += 64;\n\n            shifted = vec_sr(v_xh[0], 4);\n            q6h[0] = vec_sl(vec_and(v_um, shifted), 4);\n            shifted = vec_sr(v_xh[1], 4);\n            q6h[1] = vec_sl(vec_and(v_um, shifted), 4);\n            shifted = vec_sr(v_xh[0], 6);\n            q6h[2] = vec_sl(vec_and(v_um, shifted), 4);\n            shifted = vec_sr(v_xh[1], 6);\n            q6h[3] = vec_sl(vec_and(v_um, shifted), 4);\n\n            q6b[0] = (int8x16_t)(vec_or(vec_sr(v_xl[0], 4), q6h[0]));\n            q6b[1] = (int8x16_t)(vec_or(vec_sr(v_xl[1], 4), q6h[1]));\n            q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2]));\n            q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3]));\n\n            summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]);\n            summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]);\n            summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]);\n            summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]);\n\n            isum += vec_hsum_i32x4(summs0) * scale[0] +\n                    vec_hsum_i32x4(summs1) * scale[1] +\n                    vec_hsum_i32x4(summs2) * scale[2] +\n                    vec_hsum_i32x4(summs3) * scale[3];\n\n            scale += 4;\n        }\n\n        sum += d_all * y[i].d * (isum - 32 * mins);\n    }\n\n    *s = sum;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n// #if defined(__VXE__) || defined(__VXE2__)\n// static const int8_t keven_signs_q2xs[1024] = {\n//      1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n//      1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n//      1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n//      1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n//      1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n//      1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n//      1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n//      1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n//      1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n//      1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n//      1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n//      1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n//      1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n//      1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n//      1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n//      1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n//      1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n//      1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n//      1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n//      1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n//      1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n//      1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n//      1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n//      1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n//      1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n//      1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n//      1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n//      1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n//      1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n//      1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n//      1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n//      1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n// };\n// #endif\n\n// void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n//     assert(n % QK_K == 0);\n//     assert(nrc == 1);\n//     UNUSED(nrc);\n//     UNUSED(bx);\n//     UNUSED(by);\n//     UNUSED(bs);\n\n//     const block_iq2_xxs * GGML_RESTRICT x = vx;\n//     const block_q8_K    * GGML_RESTRICT y = vy;\n\n//     const int nb = n / QK_K;\n\n// #if defined(__VXE__) || defined(__VXE2__)\n//    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n//    uint32_t aux32[4];\n//    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n//    float sumf = 0;\n\n//    for (int i = 0; i < nb; ++i) {\n//        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n//        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n//        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n//        float sumf1 = 0, sumf2 = 0;\n\n//        for (int ib32 = 0; ib32 < QK_K/32; ib += 2) {\n//            int8x16_t q8b0 = vec_xl( 0, q8);\n//            int8x16_t qb81 = vec_xl(16, q8);\n//            int8x16_t q8b2 = vec_xl(32, q8);\n//            int8x16_t q8b3 = vec_xl(48, q8);\n//            q8 += 64;\n\n//            memcpy(aux32, q2, 4 * sizeof(uint32_t));\n//            q2 += 8;\n\n//            int8x16_t q2u0 = { *(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1]) };\n//            int8x16_t q2u1 = { *(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3]) };\n//            int8x16_t q2u2 = { *(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9]) };\n//            int8x16_t q2u3 = { *(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11]) };\n\n//            int8x16_t q2s0 = { *(const int64_t *)(signs64 + ((aux32[1] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >>  7) & 127)) };\n//            int8x16_t q2s1 = { *(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127)) };\n//            int8x16_t q2s2 = { *(const int64_t *)(signs64 + ((aux32[3] >>  0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >>  7) & 127)) };\n//            int8x16_t q2s3 = { *(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127)) };\n\n//            q2u0 = vec_mul(q2u0, q2s0);\n//            q2u1 = vec_mul(q2u1, q2s1);\n//            q2u2 = vec_mul(q2u2, q2s2);\n//            q2u3 = vec_mul(q2u3, q2s3);\n\n//            const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1);\n//            const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3);\n\n//            sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28));\n//            sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28));\n//        }\n\n//        sumf += d * (sumf1 + sumf2);\n//    }\n\n//    *s = 0.25f * sumf;\n\n// #else\n\n//     uint32_t aux32[2];\n//     const uint8_t * aux8 = (const uint8_t *)aux32;\n\n//     float sumf = 0.f;\n//     for (int i = 0; i < nb; ++i) {\n//         const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n//         const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n//         const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n//         int32_t bsum = 0;\n//         for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n//             memcpy(aux32, q2, 2*sizeof(uint32_t));\n//             q2 += 4;\n//             const uint32_t ls = 2*(aux32[1] >> 28) + 1;\n//             int32_t sumi = 0;\n//             for (int l = 0; l < 4; ++l) {\n//                 const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);\n//                 const uint8_t  signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];\n//                 for (int j = 0; j < 8; ++j) {\n//                     sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n//                 }\n//                 q8 += 8;\n//             }\n//             bsum += sumi * ls;\n//         }\n//         sumf += d * bsum;\n//     }\n//     *s = 0.125f * sumf;\n// #endif\n// }\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);\n    const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n    for (; ib < nb; ++ib) {\n        const block_iq4_nl * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_0   * GGML_RESTRICT y0 = &y[ib];\n\n        const uint8x16_t v_x = vec_xl(0, x0->qs);\n        int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m);\n        int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4);\n\n        v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl);\n        v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh);\n\n        const int8x16_t v_yl = vec_xl(0      , y0->qs);\n        const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs);\n        const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh);\n\n        sumf += GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d) * vec_hsum_i32x4(v_xy);\n    }\n\n    *s = sumf;\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__VXE__) || defined(__VXE2__)\n    const int8x16_t v_k = vec_xl(0, kvalues_iq4nl);\n    const uint8x16_t v_m = vec_splat_u8(0x0F);\n\n    float sumf = 0;\n\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const uint8_t * GGML_RESTRICT q4 = x[ibl].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[ibl].qs;\n\n        uint16_t h = x[ibl].scales_h;\n\n        int sumi1 = 0, sumi2 = 0;\n        for (int ib = 0; ib < QK_K/64; ++ib) {\n            const uint8x16_t v_x0 = vec_xl(0       , q4);\n            const uint8x16_t v_x1 = vec_xl(QK4_NL/2, q4);\n            q4 += 32;\n\n            int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m);\n            int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4);\n            int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m);\n            int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4);\n\n            v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l);\n            v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h);\n            v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l);\n            v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h);\n\n            const int8x16_t v_y0 = vec_xl( 0, q8);\n            const int8x16_t v_y1 = vec_xl(16, q8);\n            const int8x16_t v_y2 = vec_xl(32, q8);\n            const int8x16_t v_y3 = vec_xl(48, q8);\n            q8 += 64;\n\n            int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1);\n            int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3);\n\n            int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32;\n            int ls2 = ((x[ibl].scales_l[ib] >>  4) | ((h << 2) & 0x30)) - 32;\n\n            h >>= 4;\n\n            sumi1 += vec_hsum_i32x4(vsumi0) * ls1;\n            sumi2 += vec_hsum_i32x4(vsumi1) * ls2;\n        }\n\n        sumf += GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/wasm/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__wasm_simd128__)\n#define B1(c,s,n)  0x ## n ## c ,  0x ## n ## s\n#define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s)\n#define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s)\n#define B4(c,s,n) B3(c,s,n ## c), B3(c,s,n ## s)\n#define B5(c,s,n) B4(c,s,n ## c), B4(c,s,n ## s)\n#define B6(c,s,n) B5(c,s,n ## c), B5(c,s,n ## s)\n#define B7(c,s,n) B6(c,s,n ## c), B6(c,s,n ## s)\n#define B8(c,s  ) B7(c,s,     c), B7(c,s,     s)\n\n// precomputed tables for expanding 8bits to 8 bytes:\nstatic const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4\nstatic const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4\n#endif\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n    for (int i = 0; i < nb; i++) {\n        v128_t srcv [8];\n        v128_t asrcv[8];\n        v128_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = wasm_v128_load(x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),\n                                   wasm_f32x4_extract_lane(amaxv[0], 1)),\n                               MAX(wasm_f32x4_extract_lane(amaxv[0], 2),\n                                   wasm_f32x4_extract_lane(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        for (int j = 0; j < 8; j++) {\n            const v128_t v  = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));\n            const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);\n\n            y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);\n            y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);\n            y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);\n            y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);\n        }\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n#if defined __wasm_simd128__\n    for (int i = 0; i < nb; i++) {\n        v128_t srcv [8];\n        v128_t asrcv[8];\n        v128_t amaxv[8];\n\n        for (int j = 0; j < 8; j++) srcv[j]  = wasm_v128_load(x + i*32 + 4*j);\n        for (int j = 0; j < 8; j++) asrcv[j] = wasm_f32x4_abs(srcv[j]);\n\n        for (int j = 0; j < 4; j++) amaxv[2*j] = wasm_f32x4_max(asrcv[2*j], asrcv[2*j+1]);\n        for (int j = 0; j < 2; j++) amaxv[4*j] = wasm_f32x4_max(amaxv[4*j], amaxv[4*j+2]);\n        for (int j = 0; j < 1; j++) amaxv[8*j] = wasm_f32x4_max(amaxv[8*j], amaxv[8*j+4]);\n\n        const float amax = MAX(MAX(wasm_f32x4_extract_lane(amaxv[0], 0),\n                                   wasm_f32x4_extract_lane(amaxv[0], 1)),\n                               MAX(wasm_f32x4_extract_lane(amaxv[0], 2),\n                                   wasm_f32x4_extract_lane(amaxv[0], 3)));\n\n        const float d = amax / ((1 << 7) - 1);\n        const float id = d ? 1.0f/d : 0.0f;\n\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n\n        v128_t accv = wasm_i32x4_splat(0);\n\n        for (int j = 0; j < 8; j++) {\n            const v128_t v  = wasm_f32x4_mul(srcv[j], wasm_f32x4_splat(id));\n            const v128_t vi = wasm_i32x4_trunc_sat_f32x4(v);\n\n            y[i].qs[4*j + 0] = wasm_i32x4_extract_lane(vi, 0);\n            y[i].qs[4*j + 1] = wasm_i32x4_extract_lane(vi, 1);\n            y[i].qs[4*j + 2] = wasm_i32x4_extract_lane(vi, 2);\n            y[i].qs[4*j + 3] = wasm_i32x4_extract_lane(vi, 3);\n\n            accv = wasm_i32x4_add(accv, vi);\n        }\n\n        y[i].s = GGML_CPU_FP32_TO_FP16(\n                d * (wasm_i32x4_extract_lane(accv, 0) +\n                     wasm_i32x4_extract_lane(accv, 1) +\n                     wasm_i32x4_extract_lane(accv, 2) +\n                     wasm_i32x4_extract_lane(accv, 3)));\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n//===================================== Q8_K ==============================================\n\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n#ifdef __wasm_simd128__\n    assert(k % QK_K == 0);\n    const int64_t nb = k / QK_K;\n    block_q8_K * GGML_RESTRICT yc = y; // Cast to proper type\n\n    for (int i = 0; i < nb; i++) {\n        const float * x_block = x + i * QK_K;\n\n        v128_t min_vec = wasm_v128_load(x_block);\n        v128_t max_vec = min_vec;\n\n        for (int j = 4; j < QK_K; j += 4) {\n            v128_t x_vec = wasm_v128_load(x_block + j);\n            max_vec = wasm_f32x4_pmax(max_vec, x_vec);\n            min_vec = wasm_f32x4_pmin(min_vec, x_vec);\n        }\n        max_vec = wasm_f32x4_pmax(max_vec, wasm_i32x4_shuffle(max_vec, max_vec, 2, 3, 0, 1));\n        max_vec = wasm_f32x4_pmax(max_vec, wasm_i32x4_shuffle(max_vec, max_vec, 1, 0, 3, 2));\n        min_vec = wasm_f32x4_pmin(min_vec, wasm_i32x4_shuffle(min_vec, min_vec, 2, 3, 0, 1));\n        min_vec = wasm_f32x4_pmin(min_vec, wasm_i32x4_shuffle(min_vec, min_vec, 1, 0, 3, 2));\n        float max = wasm_f32x4_extract_lane(max_vec, 0);\n        float min = wasm_f32x4_extract_lane(min_vec, 0);\n        float amax = -min > max ? min : max;\n\n        if (amax == 0.0f) {\n            yc[i].d = 0.0f;\n            const v128_t zero = wasm_i8x16_splat(0);\n            for (int j = 0; j < QK_K; j += 16) {\n                wasm_v128_store(yc[i].qs + j, zero);\n            }\n            continue;\n        }\n\n        const float iscale = -127.0f / amax;\n        const v128_t scale_vec = wasm_f32x4_splat(iscale);\n\n        // Process 16 elements per iteration\n        for (int j = 0, jb = 0; j < QK_K; j += 16, jb++) {\n            // Load and quantize 16 floats\n            v128_t x0 = wasm_v128_load(x_block + j);\n            v128_t x1 = wasm_v128_load(x_block + j + 4);\n            v128_t x2 = wasm_v128_load(x_block + j + 8);\n            v128_t x3 = wasm_v128_load(x_block + j + 12);\n\n            v128_t q0 = wasm_f32x4_nearest(wasm_f32x4_mul(x0, scale_vec));\n            v128_t q1 = wasm_f32x4_nearest(wasm_f32x4_mul(x1, scale_vec));\n            v128_t q2 = wasm_f32x4_nearest(wasm_f32x4_mul(x2, scale_vec));\n            v128_t q3 = wasm_f32x4_nearest(wasm_f32x4_mul(x3, scale_vec));\n\n            // Convert to i32 with saturation\n            v128_t i0 = wasm_i32x4_trunc_sat_f32x4(q0);\n            v128_t i1 = wasm_i32x4_trunc_sat_f32x4(q1);\n            v128_t i2 = wasm_i32x4_trunc_sat_f32x4(q2);\n            v128_t i3 = wasm_i32x4_trunc_sat_f32x4(q3);\n\n            // Pack into 16 i8 values\n            v128_t i8 = wasm_i8x16_narrow_i16x8(\n                wasm_i16x8_narrow_i32x4(i0, i1),\n                wasm_i16x8_narrow_i32x4(i2, i3)\n            );\n            wasm_v128_store(yc[i].qs + j, i8);\n\n            // Calculate bsums using SIMD\n            v128_t sum16 = wasm_i16x8_add(\n                wasm_i16x8_extend_low_i8x16(i8),\n                wasm_i16x8_extend_high_i8x16(i8)\n            );\n            v128_t sum32 = wasm_i32x4_add(\n                wasm_i32x4_extend_low_i16x8(sum16),\n                wasm_i32x4_extend_high_i16x8(sum16)\n            );\n            sum32 = wasm_i32x4_add(sum32, wasm_i32x4_shuffle(sum32, sum32, 2, 3, 0, 1));\n            sum32 = wasm_i32x4_add(sum32, wasm_i32x4_shuffle(sum32, sum32, 1, 0, 3, 2));\n            yc[i].bsums[jb] = wasm_i32x4_extract_lane(sum32, 0);\n        }\n\n        yc[i].d = 1.0f / iscale;\n    }\n#else\n    quantize_row_q8_K_ref(x, y, k);\n#endif\n}\n\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __wasm_simd128__\n    v128_t sumv = wasm_f32x4_splat(0.0f);\n\n    const v128_t m4b = wasm_i8x16_splat(0x0F);\n    const v128_t s8b = wasm_i8x16_splat(0x8);\n\n    for (; ib + 1 < nb; ib += 2) {\n        const block_q4_0 * GGML_RESTRICT x0 = &x[ib];\n        const block_q4_0 * GGML_RESTRICT x1 = &x[ib + 1];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n        const block_q8_0 * GGML_RESTRICT y1 = &y[ib + 1];\n\n        // Load and process x0\n        v128_t v0_0 = wasm_v128_load(x0->qs);\n        v128_t v0_0l = wasm_v128_and(v0_0, m4b);\n        v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);\n        v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);\n        v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);\n\n        // Load y0 vectors\n        v128_t y0_l = wasm_v128_load(y0->qs);\n        v128_t y0_h = wasm_v128_load(y0->qs + 16);\n\n        // Extend to i16x8 and compute dot products\n        v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);\n        v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);\n        v128_t dx0hl = wasm_i16x8_extend_low_i8x16(v0_0hs);\n        v128_t dx0hh = wasm_i16x8_extend_high_i8x16(v0_0hs);\n\n        v128_t dy0ll = wasm_i16x8_extend_low_i8x16(y0_l);\n        v128_t dy0lh = wasm_i16x8_extend_high_i8x16(y0_l);\n        v128_t dy0hl = wasm_i16x8_extend_low_i8x16(y0_h);\n        v128_t dy0hh = wasm_i16x8_extend_high_i8x16(y0_h);\n\n        v128_t dp0 = wasm_i32x4_add(\n            wasm_i32x4_add(\n                wasm_i32x4_dot_i16x8(dx0l, dy0ll),\n                wasm_i32x4_dot_i16x8(dx0h, dy0lh)\n            ),\n            wasm_i32x4_add(\n                wasm_i32x4_dot_i16x8(dx0hl, dy0hl),\n                wasm_i32x4_dot_i16x8(dx0hh, dy0hh)\n            )\n        );\n\n        // Load and process x1\n        v128_t v0_1 = wasm_v128_load(x1->qs);\n        v128_t v0_1l = wasm_v128_and(v0_1, m4b);\n        v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);\n        v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);\n        v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);\n\n        // Load y1 vectors\n        v128_t y1_l = wasm_v128_load(y1->qs);\n        v128_t y1_h = wasm_v128_load(y1->qs + 16);\n\n        // Extend to i16x8 and compute dot products\n        v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);\n        v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);\n        v128_t dx1hl = wasm_i16x8_extend_low_i8x16(v0_1hs);\n        v128_t dx1hh = wasm_i16x8_extend_high_i8x16(v0_1hs);\n\n        v128_t dy1ll = wasm_i16x8_extend_low_i8x16(y1_l);\n        v128_t dy1lh = wasm_i16x8_extend_high_i8x16(y1_l);\n        v128_t dy1hl = wasm_i16x8_extend_low_i8x16(y1_h);\n        v128_t dy1hh = wasm_i16x8_extend_high_i8x16(y1_h);\n\n        v128_t dp1 = wasm_i32x4_add(\n            wasm_i32x4_add(\n                wasm_i32x4_dot_i16x8(dx1l, dy1ll),\n                wasm_i32x4_dot_i16x8(dx1h, dy1lh)\n            ),\n            wasm_i32x4_add(\n                wasm_i32x4_dot_i16x8(dx1hl, dy1hl),\n                wasm_i32x4_dot_i16x8(dx1hh, dy1hh)\n            )\n        );\n\n        // Accumulate results with scaling\n        float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);\n        float scale1 = GGML_CPU_FP16_TO_FP32(x1->d) * GGML_CPU_FP16_TO_FP32(y1->d);\n\n        sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp0), wasm_f32x4_splat(scale0)));\n        sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp1), wasm_f32x4_splat(scale1)));\n    }\n\n    sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n           wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n            const int v1 = (x[ib].qs[j] >>   4) - 8;\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n    v128_t sumv = wasm_f32x4_splat(0.0f);\n\n    uint32_t qh_;\n    uint64_t tmp[4];\n\n    // TODO: check if unrolling this is better\n    for (; ib < nb; ++ib) {\n        const block_q5_0 * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n        const v128_t m4b  = wasm_i8x16_splat(0x0F);\n\n        // extract the 5th bit\n        memcpy(&qh_, x0->qh, sizeof(qh_));\n\n        tmp[0] = table_b2b_1[(qh_ >>  0) & 0xFF];\n        tmp[1] = table_b2b_1[(qh_ >>  8) & 0xFF];\n        tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];\n        tmp[3] = table_b2b_1[(qh_ >> 24)       ];\n\n        const v128_t qhl = wasm_v128_load(tmp + 0);\n        const v128_t qhh = wasm_v128_load(tmp + 2);\n\n        const v128_t v0 = wasm_v128_load(x0->qs);\n\n        // 4-bit -> 8-bit\n        const v128_t v0l = wasm_v128_and (v0, m4b);\n        const v128_t v0h = wasm_u8x16_shr(v0, 4);\n\n        // add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)\n        const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);\n        const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);\n\n        // load y\n        const v128_t v1l = wasm_v128_load(y0->qs);\n        const v128_t v1h = wasm_v128_load(y0->qs + 16);\n\n        // int8x16 -> int16x8\n        const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);\n        const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);\n        const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);\n        const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);\n\n        const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);\n        const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);\n        const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);\n        const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);\n\n        // dot product\n        sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(\n                        wasm_i32x4_add(\n                            wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),\n                                           wasm_i32x4_dot_i16x8(v0lfh, v1lh)),\n                            wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),\n                                           wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),\n                    wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));\n    }\n\n    sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n           wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined __wasm_simd128__\n    v128_t sumv = wasm_f32x4_splat(0.0f);\n\n    float summs = 0.0f;\n\n    uint32_t qh_;\n    uint64_t tmp[4];\n\n    // TODO: check if unrolling this is better\n    for (; ib < nb; ++ib) {\n        const block_q5_1 * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_1 * GGML_RESTRICT y0 = &y[ib];\n\n        summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);\n\n        const v128_t m4b = wasm_i8x16_splat(0x0F);\n\n        // extract the 5th bit\n        memcpy(&qh_, x0->qh, sizeof(qh_));\n\n        tmp[0] = table_b2b_0[(qh_ >>  0) & 0xFF];\n        tmp[1] = table_b2b_0[(qh_ >>  8) & 0xFF];\n        tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];\n        tmp[3] = table_b2b_0[(qh_ >> 24)       ];\n\n        const v128_t qhl = wasm_v128_load(tmp + 0);\n        const v128_t qhh = wasm_v128_load(tmp + 2);\n\n        const v128_t v0 = wasm_v128_load(x0->qs);\n\n        // 4-bit -> 8-bit\n        const v128_t v0l = wasm_v128_and (v0, m4b);\n        const v128_t v0h = wasm_u8x16_shr(v0, 4);\n\n        // add high bit\n        const v128_t v0lf = wasm_v128_or(v0l, qhl);\n        const v128_t v0hf = wasm_v128_or(v0h, qhh);\n\n        // load y\n        const v128_t v1l = wasm_v128_load(y0->qs);\n        const v128_t v1h = wasm_v128_load(y0->qs + 16);\n\n        // int8x16 -> int16x8\n        const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);\n        const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);\n        const v128_t v0hfl = wasm_i16x8_extend_low_i8x16 (v0hf);\n        const v128_t v0hfh = wasm_i16x8_extend_high_i8x16(v0hf);\n\n        const v128_t v1ll = wasm_i16x8_extend_low_i8x16 (v1l);\n        const v128_t v1lh = wasm_i16x8_extend_high_i8x16(v1l);\n        const v128_t v1hl = wasm_i16x8_extend_low_i8x16 (v1h);\n        const v128_t v1hh = wasm_i16x8_extend_high_i8x16(v1h);\n\n        // dot product\n        sumv = wasm_f32x4_add(sumv,\n                wasm_f32x4_mul(wasm_f32x4_convert_i32x4(wasm_i32x4_add(\n                            wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0lfl, v1ll),\n                                           wasm_i32x4_dot_i16x8(v0lfh, v1lh)),\n                            wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),\n                                           wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),\n                    wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));\n    }\n\n    sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n           wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(sumf);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __wasm_simd128__\n    v128_t sumv = wasm_f32x4_splat(0.0f);\n\n    for (; ib < nb; ++ib) {\n        const block_q8_0 * GGML_RESTRICT x0 = &x[ib];\n        const block_q8_0 * GGML_RESTRICT y0 = &y[ib];\n\n        const v128_t x0_0 = wasm_v128_load(x0->qs);\n        const v128_t x0_1 = wasm_v128_load(x0->qs + 16);\n        const v128_t y0_0 = wasm_v128_load(y0->qs);\n        const v128_t y0_1 = wasm_v128_load(y0->qs + 16);\n\n        // Extend 8-bit to 16-bit\n        const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);\n        const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);\n        const v128_t x0_1l = wasm_i16x8_extend_low_i8x16(x0_1);\n        const v128_t x0_1h = wasm_i16x8_extend_high_i8x16(x0_1);\n\n        const v128_t y0_0l = wasm_i16x8_extend_low_i8x16(y0_0);\n        const v128_t y0_0h = wasm_i16x8_extend_high_i8x16(y0_0);\n        const v128_t y0_1l = wasm_i16x8_extend_low_i8x16(y0_1);\n        const v128_t y0_1h = wasm_i16x8_extend_high_i8x16(y0_1);\n\n        // Compute dot products\n        const v128_t dx0_0 = wasm_i32x4_dot_i16x8(x0_0l, y0_0l);\n        const v128_t dx0_1 = wasm_i32x4_dot_i16x8(x0_0h, y0_0h);\n        const v128_t dx1_0 = wasm_i32x4_dot_i16x8(x0_1l, y0_1l);\n        const v128_t dx1_1 = wasm_i32x4_dot_i16x8(x0_1h, y0_1h);\n\n        // Sum all dot products\n        const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));\n\n        // Convert to float and accumulate\n        const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);\n        sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));\n    }\n\n    sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +\n           wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);\n\n    *s = sumf;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    UNUSED(sumf);\n    ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * q2 = x[i].qs;\n        const int8_t * q8 = y[i].qs;\n        const uint8_t * sc = x[i].scales;\n\n        // Vectorized summs calculation\n        v128_t summs_vec = wasm_i32x4_splat(0);\n        {\n            v128_t sc_vec = wasm_v128_load(sc);\n            v128_t sc_upper = wasm_u8x16_shr(sc_vec, 4);\n\n            v128_t sc_low = wasm_u16x8_extend_low_u8x16(sc_upper);\n            v128_t sc_high = wasm_u16x8_extend_high_u8x16(sc_upper);\n\n            v128_t bsums1 = wasm_v128_load(&y[i].bsums[0]);\n            v128_t bsums2 = wasm_v128_load(&y[i].bsums[8]);\n\n            summs_vec = wasm_i32x4_add(\n                wasm_i32x4_add(wasm_i32x4_dot_i16x8(sc_low, bsums1),\n                               wasm_i32x4_dot_i16x8(sc_high, bsums2)),\n                summs_vec\n            );\n\n            summs_vec = wasm_i32x4_add(summs_vec, wasm_i32x4_shuffle(summs_vec, summs_vec, 2, 3, 0, 1));\n            summs_vec = wasm_i32x4_add(summs_vec, wasm_i32x4_shuffle(summs_vec, summs_vec, 1, 0, 3, 2));\n        }\n        int32_t summs = wasm_i32x4_extract_lane(summs_vec, 0);\n\n        // Vectorized isum calculation\n        int32_t isum = 0;\n        const uint8_t * sc_ptr = sc;\n        const int k_iters = QK_K/128;\n\n        for (int k = 0; k < k_iters; ++k) {\n            v128_t isum_vec = wasm_i32x4_splat(0);\n            int shift = 0;\n\n            for (int j = 0; j < 4; ++j) {\n                const int d0 = (sc_ptr[0] & 0xF);\n                const int d1 = (sc_ptr[1] & 0xF);\n                sc_ptr += 2;\n\n                // Process first 16 elements\n                v128_t q2_0 = wasm_v128_load(q2);\n                v128_t q8_0 = wasm_v128_load(q8);\n                v128_t q2_shift_0 = wasm_u8x16_shr(q2_0, shift);\n                v128_t q2_bits_0 = wasm_v128_and(q2_shift_0, wasm_i8x16_splat(0x03));\n\n                // Process next 16 elements\n                v128_t q2_1 = wasm_v128_load(q2 + 16);\n                v128_t q8_1 = wasm_v128_load(q8 + 16);\n                v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);\n                v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));\n\n                // Calculate dot products\n                v128_t p0 = wasm_i32x4_dot_i16x8(\n                    wasm_i16x8_extend_low_i8x16(q8_0),\n                    wasm_i16x8_extend_low_i8x16(q2_bits_0)\n                );\n                v128_t p1 = wasm_i32x4_dot_i16x8(\n                    wasm_i16x8_extend_high_i8x16(q8_0),\n                    wasm_i16x8_extend_high_i8x16(q2_bits_0)\n                );\n                v128_t p2 = wasm_i32x4_dot_i16x8(\n                    wasm_i16x8_extend_low_i8x16(q8_1),\n                    wasm_i16x8_extend_low_i8x16(q2_bits_1)\n                );\n                v128_t p3 = wasm_i32x4_dot_i16x8(\n                    wasm_i16x8_extend_high_i8x16(q8_1),\n                    wasm_i16x8_extend_high_i8x16(q2_bits_1)\n                );\n\n                // Accumulate scaled results\n                v128_t scaled = wasm_i32x4_add(\n                    wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),\n                    wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))\n                );\n\n                isum_vec = wasm_i32x4_add(isum_vec, scaled);\n                q8 += 32;\n                shift += 2;\n            }\n            q2 += 32;\n\n            // Horizontal sum of isum_vec\n            isum_vec = wasm_i32x4_add(isum_vec, wasm_i32x4_shuffle(isum_vec, isum_vec, 2, 3, 0, 1));\n            isum_vec = wasm_i32x4_add(isum_vec, wasm_i32x4_shuffle(isum_vec, isum_vec, 1, 0, 3, 2));\n            isum += wasm_i32x4_extract_lane(isum_vec, 0);\n        }\n\n        const float dall = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n        sumf += dall * isum - dmin * summs;\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n    int8_t  aux8[QK_K];\n    float   sums[8] = {0};\n    uint32_t auxs[4];\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT hm = x[i].hmask;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // Process blocks with SIMD\n        int8_t * a = aux8;\n        uint8_t m = 1;\n        for (int j = 0; j < QK_K; j += 128) {\n            for (int shift = 0; shift <= 6; shift += 2) {\n                v128_t v_m = wasm_i8x16_splat(m);\n                for (int l = 0; l < 32; l += 16) {\n                    v128_t v_q3 = wasm_v128_load(q3 + l);\n                    v128_t v_shift = wasm_i8x16_shr(v_q3, shift);\n                    v128_t v_low2 = wasm_v128_and(v_shift, wasm_i8x16_splat(0x03));\n\n                    v128_t v_hm = wasm_v128_load(hm + l);\n                    v128_t v_mask = wasm_v128_and(v_hm, v_m);\n                    v_mask = wasm_i8x16_ne(v_mask, wasm_i8x16_splat(0));\n\n                    v_low2 = wasm_i8x16_sub(v_low2, wasm_v128_and(wasm_i8x16_splat(4), wasm_v128_not(v_mask)));\n                    wasm_v128_store(a + l, v_low2);\n                }\n                a += 32;\n                m <<= 1;\n            }\n            q3 += 32;\n        }\n\n        // Extract scales\n        memcpy(auxs, x[i].scales, 12);\n        uint32_t tmp = auxs[2];\n        auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);\n        auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);\n        auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);\n        auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);\n        const int8_t * scales = (const int8_t *)auxs;\n\n        // SIMD dot product with register accumulators\n        v128_t v_acc0 = wasm_i32x4_splat(0);\n        v128_t v_acc1 = wasm_i32x4_splat(0);\n        a = aux8;\n        for (int j = 0; j < QK_K/16; ++j) {\n            const v128_t v_scale = wasm_i16x8_splat(scales[j] - 32);\n\n            // Process 16 elements per iteration\n            for (int k = 0; k < 2; ++k) {\n                const v128_t v_q8 = wasm_i16x8_load8x8(q8);\n                const v128_t v_a = wasm_i16x8_load8x8(a);\n\n                v128_t v_prod = wasm_i16x8_mul(v_q8, v_a);\n                v_prod = wasm_i16x8_mul(v_prod, v_scale);\n\n                v_acc0 = wasm_i32x4_add(v_acc0, wasm_i32x4_extend_low_i16x8(v_prod));\n                v_acc1 = wasm_i32x4_add(v_acc1, wasm_i32x4_extend_high_i16x8(v_prod));\n\n                q8 += 8;\n                a += 8;\n            }\n        }\n\n        // Accumulate results\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const v128_t v_d = wasm_f32x4_splat(d);\n        v128_t v_sum = wasm_f32x4_add(\n            wasm_f32x4_mul(wasm_f32x4_convert_i32x4(v_acc0), v_d),\n            wasm_f32x4_mul(wasm_f32x4_convert_i32x4(v_acc1), v_d)\n        );\n\n        // Accumulate into sums vector\n        wasm_v128_store(sums, wasm_f32x4_add(wasm_v128_load(sums), v_sum));\n    }\n\n    // Horizontal sum\n    v128_t v_sum = wasm_f32x4_add(wasm_v128_load(sums), wasm_v128_load(sums + 4));\n    sumf = wasm_f32x4_extract_lane(v_sum, 0) +\n           wasm_f32x4_extract_lane(v_sum, 1) +\n           wasm_f32x4_extract_lane(v_sum, 2) +\n           wasm_f32x4_extract_lane(v_sum, 3);\n\n    *s = sumf;\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __wasm_simd128__\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); // Corrected sign\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // Process scales and mins\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        // Sum mins * q8sums\n        int32_t sumi = 0;\n        const int16_t * GGML_RESTRICT q8sums = y[i].bsums;\n        const uint8_t * m = (const uint8_t *)&utmp[2];\n        for (int j = 0; j < 16; j += 2) {\n            sumi += (q8sums[j] + q8sums[j+1]) * m[j/2];\n        }\n        sumf -= dmin * sumi;\n\n        int32_t sumi1 = 0;\n        int32_t sumi2 = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            // Load 64 4-bit weights (32 bytes)\n            const v128_t q4x0 = wasm_v128_load(q4);\n            const v128_t q4x1 = wasm_v128_load(q4 + 16);\n            q4 += 32;\n\n            // Split into low/high nibbles\n            const v128_t q4l0 = wasm_v128_and(q4x0, wasm_i8x16_splat(0x0F));\n            const v128_t q4h0 = wasm_u8x16_shr(q4x0, 4);\n            const v128_t q4l1 = wasm_v128_and(q4x1, wasm_i8x16_splat(0x0F));\n            const v128_t q4h1 = wasm_u8x16_shr(q4x1, 4);\n\n            // Load 64 8-bit values (64 bytes)\n            const v128_t q8x0 = wasm_v128_load(q8);\n            const v128_t q8x1 = wasm_v128_load(q8 + 16);\n            const v128_t q8x2 = wasm_v128_load(q8 + 32);\n            const v128_t q8x3 = wasm_v128_load(q8 + 48);\n            q8 += 64;\n\n            // Low nibble products\n            v128_t vacc1 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q4l0),\n                wasm_i16x8_extend_low_i8x16(q8x0)\n            );\n            vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q4l0),\n                wasm_i16x8_extend_high_i8x16(q8x0)\n            ));\n            vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q4l1),\n                wasm_i16x8_extend_low_i8x16(q8x1)\n            ));\n            vacc1 = wasm_i32x4_add(vacc1, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q4l1),\n                wasm_i16x8_extend_high_i8x16(q8x1)\n            ));\n\n            // High nibble products\n            v128_t vacc2 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q4h0),\n                wasm_i16x8_extend_low_i8x16(q8x2)\n            );\n            vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q4h0),\n                wasm_i16x8_extend_high_i8x16(q8x2)\n            ));\n            vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q4h1),\n                wasm_i16x8_extend_low_i8x16(q8x3)\n            ));\n            vacc2 = wasm_i32x4_add(vacc2, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q4h1),\n                wasm_i16x8_extend_high_i8x16(q8x3)\n            ));\n\n            // Accumulate scaled results\n            int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +\n                                wasm_i32x4_extract_lane(vacc1, 2) + wasm_i32x4_extract_lane(vacc1, 3);\n            sumi1 += vacc1_sum * scales[2*j];\n\n            int32_t vacc2_sum = wasm_i32x4_extract_lane(vacc2, 0) + wasm_i32x4_extract_lane(vacc2, 1) +\n                                wasm_i32x4_extract_lane(vacc2, 2) + wasm_i32x4_extract_lane(vacc2, 3);\n            sumi2 += vacc2_sum * scales[2*j+1];\n        }\n\n        sumf += d * (sumi1 + sumi2);\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __wasm_simd128__\n    //const uint8_t * scales = (const uint8_t*)&utmp[0];\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); // Fixed sign\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // Process scales and mins\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        // Sum mins * q8sums\n        int32_t sumi_mins = 0;\n        const int16_t * GGML_RESTRICT q8sums = y[i].bsums;\n        const uint8_t * m = (const uint8_t *)&utmp[2];\n        for (int j = 0; j < 16; j += 2) {\n            sumi_mins += (q8sums[j] + q8sums[j+1]) * m[j/2];\n        }\n        sumf -= dmin * sumi_mins; // Correct subtraction\n\n        v128_t qh0 = wasm_v128_load(qh);\n        v128_t qh1 = wasm_v128_load(qh + 16);\n        const uint8_t * sc = (const uint8_t *)utmp;\n\n        int32_t sumi = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n            const int shift = j * 2;\n            v128_t qh_shift0 = wasm_u8x16_shr(qh0, shift);\n            v128_t qh_shift1 = wasm_u8x16_shr(qh1, shift);\n\n            v128_t qh_low0 = wasm_i8x16_shl(wasm_v128_and(qh_shift0, wasm_i8x16_splat(0x01)), 4);\n            v128_t qh_high0 = wasm_i8x16_shl(wasm_v128_and(qh_shift0, wasm_i8x16_splat(0x02)), 3);\n            v128_t qh_low1 = wasm_i8x16_shl(wasm_v128_and(qh_shift1, wasm_i8x16_splat(0x01)), 4);\n            v128_t qh_high1 = wasm_i8x16_shl(wasm_v128_and(qh_shift1, wasm_i8x16_splat(0x02)), 3);\n\n            v128_t q5_0 = wasm_v128_load(q5);\n            v128_t q5_1 = wasm_v128_load(q5 + 16);\n            q5 += 32;\n\n            v128_t q5l_0 = wasm_v128_or(wasm_v128_and(q5_0, wasm_i8x16_splat(0x0F)), qh_low0);\n            v128_t q5h_0 = wasm_v128_or(wasm_u8x16_shr(q5_0, 4), qh_high0);\n            v128_t q5l_1 = wasm_v128_or(wasm_v128_and(q5_1, wasm_i8x16_splat(0x0F)), qh_low1);\n            v128_t q5h_1 = wasm_v128_or(wasm_u8x16_shr(q5_1, 4), qh_high1);\n\n            v128_t q8_0 = wasm_v128_load(q8);\n            v128_t q8_1 = wasm_v128_load(q8 + 16);\n            v128_t q8_2 = wasm_v128_load(q8 + 32);\n            v128_t q8_3 = wasm_v128_load(q8 + 48);\n            q8 += 64;\n\n            // Process low quants\n            v128_t pl0 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q5l_0),\n                wasm_i16x8_extend_low_i8x16(q8_0)\n            );\n            pl0 = wasm_i32x4_add(pl0, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q5l_0),\n                wasm_i16x8_extend_high_i8x16(q8_0)\n            ));\n            v128_t pl1 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q5l_1),\n                wasm_i16x8_extend_low_i8x16(q8_1)\n            );\n            pl1 = wasm_i32x4_add(pl1, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q5l_1),\n                wasm_i16x8_extend_high_i8x16(q8_1)\n            ));\n            v128_t sum_low = wasm_i32x4_add(pl0, pl1);\n\n            // Process high quants\n            v128_t ph0 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q5h_0),\n                wasm_i16x8_extend_low_i8x16(q8_2)\n            );\n            ph0 = wasm_i32x4_add(ph0, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q5h_0),\n                wasm_i16x8_extend_high_i8x16(q8_2)\n            ));\n            v128_t ph1 = wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_low_i8x16(q5h_1),\n                wasm_i16x8_extend_low_i8x16(q8_3)\n            );\n            ph1 = wasm_i32x4_add(ph1, wasm_i32x4_dot_i16x8(\n                wasm_i16x8_extend_high_i8x16(q5h_1),\n                wasm_i16x8_extend_high_i8x16(q8_3)\n            ));\n            v128_t sum_high = wasm_i32x4_add(ph0, ph1);\n\n            // Accumulate with scale factors\n            int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +\n                        wasm_i32x4_extract_lane(sum_low, 2) + wasm_i32x4_extract_lane(sum_low, 3);\n            int32_t sh = wasm_i32x4_extract_lane(sum_high, 0) + wasm_i32x4_extract_lane(sum_high, 1) +\n                        wasm_i32x4_extract_lane(sum_high, 2) + wasm_i32x4_extract_lane(sum_high, 3);\n\n            sumi += sl * sc[2*j] + sh * sc[2*j+1];\n        }\n\n        sumf += d * sumi;\n    }\n\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __wasm_simd128__\n    int8_t aux8[QK_K] __attribute__((aligned(16)));\n    int32_t aux32[8] __attribute__((aligned(16))) = {0};\n    float sums[8] __attribute__((aligned(16))) = {0};\n\n    for (int i = 0; i < nb; ++i) {\n        // Unpack 6-bit quantized data into aux8 (unchanged)\n        const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        int8_t * a = aux8;\n        for (int j = 0; j < QK_K; j += 128) {\n            for (int l = 0; l < 32; ++l) {\n                a[l +  0] = (int8_t)((q4[l +  0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;\n                a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;\n                a[l + 64] = (int8_t)((q4[l +  0] >>  4) | (((qh[l] >> 4) & 3) << 4)) - 32;\n                a[l + 96] = (int8_t)((q4[l + 32] >>  4) | (((qh[l] >> 6) & 3) << 4)) - 32;\n            }\n            a += 128;\n            q4 += 64;\n            qh += 32;\n        }\n\n        const int8_t * GGML_RESTRICT a_ptr = aux8;\n        const int8_t * GGML_RESTRICT q8 = y[i].qs;\n        v128_t acc0 = wasm_i32x4_splat(0);\n        v128_t acc1 = wasm_i32x4_splat(0);\n\n        for (int j = 0; j < QK_K/16; ++j) {\n            const int scale = x[i].scales[j];\n            const v128_t vscale = wasm_i32x4_splat(scale);\n\n            // Load 16 elements from a and q8\n            const v128_t a_vec = wasm_v128_load(a_ptr);\n            const v128_t q8_vec = wasm_v128_load(q8);\n\n            // Process low 8 elements\n            v128_t a_low = wasm_i16x8_extend_low_i8x16(a_vec);\n            v128_t q8_low = wasm_i16x8_extend_low_i8x16(q8_vec);\n            v128_t prod_low = wasm_i16x8_mul(a_low, q8_low);\n            v128_t prod_lo_lo = wasm_i32x4_extend_low_i16x8(prod_low);\n            v128_t prod_lo_hi = wasm_i32x4_extend_high_i16x8(prod_low);\n\n            // Process high 8 elements\n            v128_t a_high = wasm_i16x8_extend_high_i8x16(a_vec);\n            v128_t q8_high = wasm_i16x8_extend_high_i8x16(q8_vec);\n            v128_t prod_high = wasm_i16x8_mul(a_high, q8_high);\n            v128_t prod_hi_lo = wasm_i32x4_extend_low_i16x8(prod_high);\n            v128_t prod_hi_hi = wasm_i32x4_extend_high_i16x8(prod_high);\n\n            // Scale and accumulate\n            prod_lo_lo = wasm_i32x4_mul(prod_lo_lo, vscale);\n            prod_lo_hi = wasm_i32x4_mul(prod_lo_hi, vscale);\n            prod_hi_lo = wasm_i32x4_mul(prod_hi_lo, vscale);\n            prod_hi_hi = wasm_i32x4_mul(prod_hi_hi, vscale);\n\n            acc0 = wasm_i32x4_add(acc0, wasm_i32x4_add(prod_lo_lo, prod_hi_lo));\n            acc1 = wasm_i32x4_add(acc1, wasm_i32x4_add(prod_lo_hi, prod_hi_hi));\n\n            a_ptr += 16;\n            q8 += 16;\n        }\n\n        // Store accumulated results\n        wasm_v128_store(&aux32[0], acc0);\n        wasm_v128_store(&aux32[4], acc1);\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        for (int l = 0; l < 8; ++l) {\n            sums[l] += d * aux32[l];\n        }\n    }\n\n    // Sum final results\n    float sumf = 0;\n    for (int l = 0; l < 8; ++l) {\n        sumf += sums[l];\n    }\n    *s = sumf;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp",
    "content": "#include \"ggml-backend-impl.h\"\n\n#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n\n#ifdef _MSC_VER\n#include <intrin.h>\n#endif\n\n#include <cstring>\n#include <vector>\n#include <bitset>\n#include <array>\n#include <string>\n\n// ref: https://cdrdv2-public.intel.com/782156/325383-sdm-vol-2abcd.pdf\nstruct cpuid_x86 {\n    bool SSE3(void) { return f_1_ecx[0]; }\n    bool PCLMULQDQ(void) { return f_1_ecx[1]; }\n    bool MONITOR(void) { return f_1_ecx[3]; }\n    bool SSSE3(void) { return f_1_ecx[9]; }\n    bool FMA(void) { return f_1_ecx[12]; }\n    bool CMPXCHG16B(void) { return f_1_ecx[13]; }\n    bool SSE41(void) { return f_1_ecx[19]; }\n    bool SSE42(void) { return f_1_ecx[20]; }\n    bool MOVBE(void) { return f_1_ecx[22]; }\n    bool POPCNT(void) { return f_1_ecx[23]; }\n    bool AES(void) { return f_1_ecx[25]; }\n    bool XSAVE(void) { return f_1_ecx[26]; }\n    bool OSXSAVE(void) { return f_1_ecx[27]; }\n    bool AVX(void) { return f_1_ecx[28]; }\n    bool F16C(void) { return f_1_ecx[29]; }\n    bool RDRAND(void) { return f_1_ecx[30]; }\n\n    bool MSR(void) { return f_1_edx[5]; }\n    bool CX8(void) { return f_1_edx[8]; }\n    bool SEP(void) { return f_1_edx[11]; }\n    bool CMOV(void) { return f_1_edx[15]; }\n    bool CLFSH(void) { return f_1_edx[19]; }\n    bool MMX(void) { return f_1_edx[23]; }\n    bool FXSR(void) { return f_1_edx[24]; }\n    bool SSE(void) { return f_1_edx[25]; }\n    bool SSE2(void) { return f_1_edx[26]; }\n\n    bool FSGSBASE(void) { return f_7_ebx[0]; }\n    bool BMI1(void) { return f_7_ebx[3]; }\n    bool HLE(void) { return is_intel && f_7_ebx[4]; }\n    bool AVX2(void) { return f_7_ebx[5]; }\n    bool BMI2(void) { return f_7_ebx[8]; }\n    bool ERMS(void) { return f_7_ebx[9]; }\n    bool INVPCID(void) { return f_7_ebx[10]; }\n    bool RTM(void) { return is_intel && f_7_ebx[11]; }\n    bool AVX512F(void) { return f_7_ebx[16]; }\n    bool AVX512DQ(void) { return f_7_ebx[17]; }\n    bool RDSEED(void) { return f_7_ebx[18]; }\n    bool ADX(void) { return f_7_ebx[19]; }\n    bool AVX512PF(void) { return f_7_ebx[26]; }\n    bool AVX512ER(void) { return f_7_ebx[27]; }\n    bool AVX512CD(void) { return f_7_ebx[28]; }\n    bool AVX512BW(void) { return f_7_ebx[30]; }\n    bool AVX512VL(void) { return f_7_ebx[31]; }\n\n    bool SHA(void) { return f_7_ebx[29]; }\n\n    bool PREFETCHWT1(void) { return f_7_ecx[0]; }\n\n    bool LAHF(void) { return f_81_ecx[0]; }\n    bool LZCNT(void) { return is_intel && f_81_ecx[5]; }\n    bool ABM(void) { return is_amd && f_81_ecx[5]; }\n    bool SSE4a(void) { return is_amd && f_81_ecx[6]; }\n    bool XOP(void) { return is_amd && f_81_ecx[11]; }\n    bool TBM(void) { return is_amd && f_81_ecx[21]; }\n\n    bool SYSCALL(void) { return is_intel && f_81_edx[11]; }\n    bool MMXEXT(void) { return is_amd && f_81_edx[22]; }\n    bool RDTSCP(void) { return is_intel && f_81_edx[27]; }\n    bool _3DNOWEXT(void) { return is_amd && f_81_edx[30]; }\n    bool _3DNOW(void) { return is_amd && f_81_edx[31]; }\n\n    bool AVX512_VBMI(void) { return f_7_ecx[1]; }\n    bool AVX512_VNNI(void) { return f_7_ecx[11]; }\n    bool AVX512_FP16(void) { return f_7_edx[23]; }\n    bool AVX512_BF16(void) { return f_7_1_eax[5]; }\n    bool AVX_VNNI(void) { return f_7_1_eax[4]; }\n\n    bool AMX_TILE(void) { return f_7_edx[24]; }\n    bool AMX_INT8(void) { return f_7_edx[25]; }\n    bool AMX_FP16(void) { return f_7_1_eax[21]; }\n    bool AMX_BF16(void) { return f_7_edx[22]; }\n\n#ifdef _MSC_VER\n    static void cpuid(int cpu_info[4], int eax) {\n        __cpuid(cpu_info, eax);\n    }\n    static void cpuidex(int cpu_info[4], int eax, int ecx) {\n        __cpuidex(cpu_info, eax, ecx);\n    }\n#else\n    static void cpuid(int cpu_info[4], int eax) {\n        __asm__ __volatile__(\n            \"cpuid\"\n            : \"=a\"(cpu_info[0]), \"=b\"(cpu_info[1]), \"=c\"(cpu_info[2]), \"=d\"(cpu_info[3])\n            : \"a\"(eax), \"c\"(0));\n    }\n    static void cpuidex(int cpu_info[4], int eax, int ecx) {\n        __asm__ __volatile__(\n            \"cpuid\"\n            : \"=a\"(cpu_info[0]), \"=b\"(cpu_info[1]), \"=c\"(cpu_info[2]), \"=d\"(cpu_info[3])\n            : \"a\"(eax), \"c\"(ecx));\n    }\n#endif\n\n    cpuid_x86() {\n        std::array<int, 4> cpui;\n        std::vector<std::array<int, 4>> data;\n\n        // calling __cpuid with 0x0 as the function_id argument\n        // gets the number of the highest valid function ID.\n        cpuid(cpui.data(), 0);\n        int n_ids = cpui[0];\n\n        for (int i = 0; i <= n_ids; ++i) {\n            cpuidex(cpui.data(), i, 0);\n            data.push_back(cpui);\n        }\n\n        // capture vendor string\n        char vendor[0x20] = {};\n        *reinterpret_cast<int *>(vendor)     = data[0][1];\n        *reinterpret_cast<int *>(vendor + 4) = data[0][3];\n        *reinterpret_cast<int *>(vendor + 8) = data[0][2];\n        this->vendor = vendor;\n        if (this->vendor == \"GenuineIntel\") {\n            is_intel = true;\n        } else if (this->vendor == \"AuthenticAMD\") {\n            is_amd = true;\n        }\n\n        // load bitset with flags for function 0x00000001\n        if (n_ids >= 1) {\n            f_1_ecx = data[1][2];\n            f_1_edx = data[1][3];\n        }\n\n        // load bitset with flags for function 0x00000007\n        if (n_ids >= 7) {\n            f_7_ebx = data[7][1];\n            f_7_ecx = data[7][2];\n            f_7_edx = data[7][3];\n            cpuidex(cpui.data(), 7, 1);\n            f_7_1_eax = cpui[0];\n        }\n\n        // calling __cpuid with 0x80000000 as the function_id argument\n        // gets the number of the highest valid extended ID.\n        cpuid(cpui.data(), 0x80000000);\n        unsigned int n_ex_ids = cpui[0];\n\n        std::vector<std::array<int, 4>> ext_data;\n        for (unsigned int i = 0x80000000; i <= n_ex_ids; ++i) {\n            cpuidex(cpui.data(), i, 0);\n            ext_data.push_back(cpui);\n        }\n\n        // load bitset with flags for function 0x80000001\n        if (n_ex_ids >= 0x80000001) {\n            f_81_ecx = ext_data[1][2];\n            f_81_edx = ext_data[1][3];\n        }\n\n        // interpret CPU brand string if reported\n        char brand[0x40] = {};\n        if (n_ex_ids >= 0x80000004) {\n            std::memcpy(brand, ext_data[2].data(), sizeof(cpui));\n            std::memcpy(brand + 16, ext_data[3].data(), sizeof(cpui));\n            std::memcpy(brand + 32, ext_data[4].data(), sizeof(cpui));\n            this->brand = brand;\n        }\n    }\n\n    bool is_intel = false;\n    bool is_amd = false;\n    std::string vendor;\n    std::string brand;\n    std::bitset<32> f_1_ecx;\n    std::bitset<32> f_1_edx;\n    std::bitset<32> f_7_ebx;\n    std::bitset<32> f_7_ecx;\n    std::bitset<32> f_7_edx;\n    std::bitset<32> f_7_1_eax;\n    std::bitset<32> f_81_ecx;\n    std::bitset<32> f_81_edx;\n};\n\n#if 0\nvoid test_x86_is() {\n    cpuid_x86 is;\n    printf(\"CPU Vendor: %s\\n\", is.vendor.c_str());\n    printf(\"Brand: %s\\n\", is.brand.c_str());\n    printf(\"is_intel: %d\\n\", is.is_intel);\n    printf(\"is_amd: %d\\n\", is.is_amd);\n    printf(\"sse3: %d\\n\", is.SSE3());\n    printf(\"pclmulqdq: %d\\n\", is.PCLMULQDQ());\n    printf(\"ssse3: %d\\n\", is.SSSE3());\n    printf(\"fma: %d\\n\", is.FMA());\n    printf(\"cmpxchg16b: %d\\n\", is.CMPXCHG16B());\n    printf(\"sse41: %d\\n\", is.SSE41());\n    printf(\"sse42: %d\\n\", is.SSE42());\n    printf(\"movbe: %d\\n\", is.MOVBE());\n    printf(\"popcnt: %d\\n\", is.POPCNT());\n    printf(\"aes: %d\\n\", is.AES());\n    printf(\"xsave: %d\\n\", is.XSAVE());\n    printf(\"osxsave: %d\\n\", is.OSXSAVE());\n    printf(\"avx: %d\\n\", is.AVX());\n    printf(\"f16c: %d\\n\", is.F16C());\n    printf(\"rdrand: %d\\n\", is.RDRAND());\n    printf(\"msr: %d\\n\", is.MSR());\n    printf(\"cx8: %d\\n\", is.CX8());\n    printf(\"sep: %d\\n\", is.SEP());\n    printf(\"cmov: %d\\n\", is.CMOV());\n    printf(\"clflush: %d\\n\", is.CLFSH());\n    printf(\"mmx: %d\\n\", is.MMX());\n    printf(\"fxsr: %d\\n\", is.FXSR());\n    printf(\"sse: %d\\n\", is.SSE());\n    printf(\"sse2: %d\\n\", is.SSE2());\n    printf(\"fsgsbase: %d\\n\", is.FSGSBASE());\n    printf(\"bmi1: %d\\n\", is.BMI1());\n    printf(\"hle: %d\\n\", is.HLE());\n    printf(\"avx2: %d\\n\", is.AVX2());\n    printf(\"bmi2: %d\\n\", is.BMI2());\n    printf(\"erms: %d\\n\", is.ERMS());\n    printf(\"invpcid: %d\\n\", is.INVPCID());\n    printf(\"rtm: %d\\n\", is.RTM());\n    printf(\"avx512f: %d\\n\", is.AVX512F());\n    printf(\"rdseed: %d\\n\", is.RDSEED());\n    printf(\"adx: %d\\n\", is.ADX());\n    printf(\"avx512pf: %d\\n\", is.AVX512PF());\n    printf(\"avx512er: %d\\n\", is.AVX512ER());\n    printf(\"avx512cd: %d\\n\", is.AVX512CD());\n    printf(\"sha: %d\\n\", is.SHA());\n    printf(\"prefetchwt1: %d\\n\", is.PREFETCHWT1());\n    printf(\"lahf: %d\\n\", is.LAHF());\n    printf(\"lzcnt: %d\\n\", is.LZCNT());\n    printf(\"abm: %d\\n\", is.ABM());\n    printf(\"sse4a: %d\\n\", is.SSE4a());\n    printf(\"xop: %d\\n\", is.XOP());\n    printf(\"tbm: %d\\n\", is.TBM());\n    printf(\"syscall: %d\\n\", is.SYSCALL());\n    printf(\"mmxext: %d\\n\", is.MMXEXT());\n    printf(\"rdtscp: %d\\n\", is.RDTSCP());\n    printf(\"3dnowext: %d\\n\", is._3DNOWEXT());\n    printf(\"3dnow: %d\\n\", is._3DNOW());\n    printf(\"avx512_vbmi: %d\\n\", is.AVX512_VBMI());\n    printf(\"avx512_vnni: %d\\n\", is.AVX512_VNNI());\n    printf(\"avx512_fp16: %d\\n\", is.AVX512_FP16());\n    printf(\"avx512_bf16: %d\\n\", is.AVX512_BF16());\n    printf(\"amx_tile: %d\\n\", is.AMX_TILE());\n    printf(\"amx_int8: %d\\n\", is.AMX_INT8());\n    printf(\"amx_fp16: %d\\n\", is.AMX_FP16());\n    printf(\"amx_bf16: %d\\n\", is.AMX_BF16());\n}\n#endif\n\nstatic int ggml_backend_cpu_x86_score() {\n    // FIXME: this does not check for OS support\n\n    int score = 1;\n    cpuid_x86 is;\n\n#ifdef GGML_FMA\n    if (!is.FMA()) { return 0; }\n    score += 1;\n#endif\n#ifdef GGML_F16C\n    if (!is.F16C()) { return 0; }\n    score += 1<<1;\n#endif\n#ifdef GGML_SSE42\n    if (!is.SSE42()) { return 0; }\n    score += 1<<2;\n#endif\n#ifdef GGML_BMI2\n    if (!is.BMI2()) { return 0; }\n    score += 1<<3;\n#endif\n#ifdef GGML_AVX\n    if (!is.AVX()) { return 0; }\n    score += 1<<4;\n#endif\n#ifdef GGML_AVX2\n    if (!is.AVX2()) { return 0; }\n    score += 1<<5;\n#endif\n#ifdef GGML_AVX_VNNI\n    if (!is.AVX_VNNI()) { return 0; }\n    score += 1<<6;\n#endif\n#ifdef GGML_AVX512\n    if (!is.AVX512F()) { return 0; }\n    if (!is.AVX512CD()) { return 0; }\n    if (!is.AVX512VL()) { return 0; }\n    if (!is.AVX512DQ()) { return 0; }\n    if (!is.AVX512BW()) { return 0; }\n    score += 1<<7;\n#endif\n#ifdef GGML_AVX512_VBMI\n    if (!is.AVX512_VBMI()) { return 0; }\n    score += 1<<8;\n#endif\n#ifdef GGML_AVX512_BF16\n    if (!is.AVX512_BF16()) { return 0; }\n    score += 1<<9;\n#endif\n#ifdef GGML_AVX512_VNNI\n    if (!is.AVX512_VNNI()) { return 0; }\n    score += 1<<10;\n#endif\n#ifdef GGML_AMX_INT8\n    if (!is.AMX_INT8()) { return 0; }\n    score += 1<<11;\n#endif\n\n    return score;\n}\n\nGGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_x86_score)\n\n#endif // defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/x86/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n#include \"ggml-quants.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"simd-mappings.h\"\n\n#include \"../../quants.h\"\n#include \"../../ggml-cpu-impl.h\"\n\n#include <math.h>\n#include <string.h>\n#include <assert.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\n// some compilers don't provide _mm256_set_m128i, e.g. gcc 7\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)\n// multiply int8_t, add results pairwise twice\nstatic inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) {\n    // Get absolute values of x vectors\n    const __m128i ax = _mm_sign_epi8(x, x);\n    // Sign the values of the y vectors\n    const __m128i sy = _mm_sign_epi8(y, x);\n    // Perform multiplication and create 16-bit values\n    const __m128i dot = _mm_maddubs_epi16(ax, sy);\n    const __m128i ones = _mm_set1_epi16(1);\n    return _mm_madd_epi16(ones, dot);\n}\n\n#if __AVX__ || __AVX2__ || __AVX512F__\n// horizontally add 8 floats\nstatic inline float hsum_float_8(const __m256 x) {\n    __m128 res = _mm256_extractf128_ps(x, 1);\n    res = _mm_add_ps(res, _mm256_castps256_ps128(x));\n    res = _mm_add_ps(res, _mm_movehl_ps(res, res));\n    res = _mm_add_ss(res, _mm_movehdup_ps(res));\n    return _mm_cvtss_f32(res);\n}\n\n// horizontally add 8 int32_t\nstatic inline int hsum_i32_8(const __m256i a) {\n    const __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));\n    const __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);\n    const __m128i sum64 = _mm_add_epi32(hi64, sum128);\n    const __m128i hi32  = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));\n    return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));\n}\n\n// horizontally add 4 int32_t\nstatic inline int hsum_i32_4(const __m128i a) {\n    const __m128i hi64 = _mm_unpackhi_epi64(a, a);\n    const __m128i sum64 = _mm_add_epi32(hi64, a);\n    const __m128i hi32  = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));\n    return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));\n}\n\n#if defined(__AVX2__) || defined(__AVX512F__)\nstatic inline __m256i mul_add_epi8(const __m256i x, const __m256i y) {\n    const __m256i ax = _mm256_sign_epi8(x, x);\n    const __m256i sy = _mm256_sign_epi8(y, x);\n    return _mm256_maddubs_epi16(ax, sy);\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n    uint32_t x32;\n    memcpy(&x32, x, sizeof(uint32_t));\n    const __m256i shuf_mask = _mm256_set_epi64x(\n            0x0303030303030303, 0x0202020202020202,\n            0x0101010101010101, 0x0000000000000000);\n    __m256i bytes = _mm256_shuffle_epi8(_mm256_set1_epi32(x32), shuf_mask);\n    const __m256i bit_mask = _mm256_set1_epi64x(0x7fbfdfeff7fbfdfe);\n    bytes = _mm256_or_si256(bytes, bit_mask);\n    return _mm256_cmpeq_epi8(bytes, _mm256_set1_epi64x(-1));\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)\n{\n    const __m128i tmp = _mm_loadu_si128((const __m128i *)rsi);\n    const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);\n    const __m256i lowMask = _mm256_set1_epi8( 0xF );\n    return _mm256_and_si256(lowMask, bytes);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m256i x) {\n    const __m256i ones = _mm256_set1_epi16(1);\n    const __m256i summed_pairs = _mm256_madd_epi16(ones, x);\n    return _mm256_cvtepi32_ps(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n    const __m256i zero = _mm256_setzero_si256();\n    const __m256i summed_pairs = _mm256_dpbusd_epi32(zero, ax, sy);\n    return _mm256_cvtepi32_ps(summed_pairs);\n#elif defined(__AVXVNNI__)\n    const __m256i zero = _mm256_setzero_si256();\n    const __m256i summed_pairs = _mm256_dpbusd_avx_epi32(zero, ax, sy);\n    return _mm256_cvtepi32_ps(summed_pairs);\n#else\n    // Perform multiplication and create 16-bit values\n    const __m256i dot = _mm256_maddubs_epi16(ax, sy);\n    return sum_i16_pairs_float(dot);\n#endif\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n#if __AVXVNNIINT8__\n    const __m256i zero = _mm256_setzero_si256();\n    const __m256i summed_pairs = _mm256_dpbssd_epi32(zero, x, y);\n    return _mm256_cvtepi32_ps(summed_pairs);\n#else\n    // Get absolute values of x vectors\n    const __m256i ax = _mm256_sign_epi8(x, x);\n    // Sign the values of the y vectors\n    const __m256i sy = _mm256_sign_epi8(y, x);\n    return mul_sum_us8_pairs_float(ax, sy);\n#endif\n}\n\nstatic inline __m128i packNibbles( __m256i bytes )\n{\n    // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n#if __AVX512F__\n    const __m256i bytes_srli_4 = _mm256_srli_epi16(bytes, 4);   // 0000_0000_abcd_0000\n    bytes = _mm256_or_si256(bytes, bytes_srli_4);               // 0000_abcd_abcd_efgh\n    return _mm256_cvtepi16_epi8(bytes);                         // abcd_efgh\n#else\n    const __m256i lowByte = _mm256_set1_epi16( 0xFF );\n    __m256i high = _mm256_andnot_si256( lowByte, bytes );\n    __m256i low = _mm256_and_si256( lowByte, bytes );\n    high = _mm256_srli_epi16( high, 4 );\n    bytes = _mm256_or_si256( low, high );\n\n    // Compress uint16_t lanes into bytes\n    __m128i r0 = _mm256_castsi256_si128( bytes );\n    __m128i r1 = _mm256_extracti128_si256( bytes, 1 );\n    return _mm_packus_epi16( r0, r1 );\n#endif\n}\n#elif defined(__AVX__)\nstatic inline __m128i packNibbles( __m128i bytes1, __m128i bytes2 )\n{\n    // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh\n    const __m128i lowByte = _mm_set1_epi16( 0xFF );\n    __m128i high = _mm_andnot_si128( lowByte, bytes1 );\n    __m128i low = _mm_and_si128( lowByte, bytes1 );\n    high = _mm_srli_epi16( high, 4 );\n    bytes1 = _mm_or_si128( low, high );\n    high = _mm_andnot_si128( lowByte, bytes2 );\n    low = _mm_and_si128( lowByte, bytes2 );\n    high = _mm_srli_epi16( high, 4 );\n    bytes2 = _mm_or_si128( low, high );\n\n    return _mm_packus_epi16( bytes1, bytes2);\n}\n\nstatic inline __m128i mul_add_epi8_sse(const __m128i x, const __m128i y) {\n    const __m128i ax = _mm_sign_epi8(x, x);\n    const __m128i sy = _mm_sign_epi8(y, x);\n    return _mm_maddubs_epi16(ax, sy);\n}\n\n// spread 32 bits to 32 bytes { 0x00, 0xFF }\nstatic inline __m256i bytes_from_bits_32(const uint8_t * x) {\n    uint32_t x32;\n    memcpy(&x32, x, sizeof(uint32_t));\n    const __m128i shuf_maskl = _mm_set_epi64x(0x0101010101010101, 0x0000000000000000);\n    const __m128i shuf_maskh = _mm_set_epi64x(0x0303030303030303, 0x0202020202020202);\n    __m128i bytesl = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskl);\n    __m128i bytesh = _mm_shuffle_epi8(_mm_set1_epi32(x32), shuf_maskh);\n    const __m128i bit_mask = _mm_set1_epi64x(0x7fbfdfeff7fbfdfe);\n    bytesl = _mm_or_si128(bytesl, bit_mask);\n    bytesh = _mm_or_si128(bytesh, bit_mask);\n    bytesl = _mm_cmpeq_epi8(bytesl, _mm_set1_epi64x(-1));\n    bytesh = _mm_cmpeq_epi8(bytesh, _mm_set1_epi64x(-1));\n    return MM256_SET_M128I(bytesh, bytesl);\n}\n\n// Unpack 32 4-bit fields into 32 bytes\n// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval\nstatic inline __m256i bytes_from_nibbles_32(const uint8_t * rsi)\n{\n    // Load 16 bytes from memory\n    __m128i tmpl = _mm_loadu_si128((const __m128i *)rsi);\n    __m128i tmph = _mm_srli_epi16(tmpl, 4);\n    const __m128i lowMask = _mm_set1_epi8(0xF);\n    tmpl = _mm_and_si128(lowMask, tmpl);\n    tmph = _mm_and_si128(lowMask, tmph);\n    return MM256_SET_M128I(tmph, tmpl);\n}\n\n// add int16_t pairwise and return as float vector\nstatic inline __m256 sum_i16_pairs_float(const __m128i xh, const __m128i xl) {\n    const __m128i ones = _mm_set1_epi16(1);\n    const __m128i summed_pairsl = _mm_madd_epi16(ones, xl);\n    const __m128i summed_pairsh = _mm_madd_epi16(ones, xh);\n    const __m256i summed_pairs = MM256_SET_M128I(summed_pairsh, summed_pairsl);\n    return _mm256_cvtepi32_ps(summed_pairs);\n}\n\nstatic inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) {\n    const __m128i axl = _mm256_castsi256_si128(ax);\n    const __m128i axh = _mm256_extractf128_si256(ax, 1);\n    const __m128i syl = _mm256_castsi256_si128(sy);\n    const __m128i syh = _mm256_extractf128_si256(sy, 1);\n    // Perform multiplication and create 16-bit values\n    const __m128i dotl = _mm_maddubs_epi16(axl, syl);\n    const __m128i doth = _mm_maddubs_epi16(axh, syh);\n    return sum_i16_pairs_float(doth, dotl);\n}\n\n// multiply int8_t, add results pairwise twice and return as float vector\nstatic inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) {\n    const __m128i xl = _mm256_castsi256_si128(x);\n    const __m128i xh = _mm256_extractf128_si256(x, 1);\n    const __m128i yl = _mm256_castsi256_si128(y);\n    const __m128i yh = _mm256_extractf128_si256(y, 1);\n    // Get absolute values of x vectors\n    const __m128i axl = _mm_sign_epi8(xl, xl);\n    const __m128i axh = _mm_sign_epi8(xh, xh);\n    // Sign the values of the y vectors\n    const __m128i syl = _mm_sign_epi8(yl, xl);\n    const __m128i syh = _mm_sign_epi8(yh, xh);\n    // Perform multiplication and create 16-bit values\n    const __m128i dotl = _mm_maddubs_epi16(axl, syl);\n    const __m128i doth = _mm_maddubs_epi16(axh, syh);\n    return sum_i16_pairs_float(doth, dotl);\n}\n\n// larger version of mul_sum_i8_pairs_float where x and y are each represented by four 128-bit vectors\nstatic inline __m256 mul_sum_i8_quad_float(const __m128i x_1_0, const __m128i x_1_1, const __m128i x_2_0, const __m128i x_2_1,\n                                           const __m128i y_1_0, const __m128i y_1_1, const __m128i y_2_0, const __m128i y_2_1) {\n    const __m128i mone = _mm_set1_epi16(1);\n\n    const __m128i p16_1_0 = mul_add_epi8_sse(x_1_0, y_1_0);\n    const __m128i p16_1_1 = mul_add_epi8_sse(x_1_1, y_1_1);\n    const __m128i p16_2_0 = mul_add_epi8_sse(x_2_0, y_2_0);\n    const __m128i p16_2_1 = mul_add_epi8_sse(x_2_1, y_2_1);\n    const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, mone);\n    const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, mone);\n    const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, mone);\n    const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, mone);\n    const __m128i p_1 = _mm_add_epi32(p_1_0, p_1_1);\n    const __m128i p_2 = _mm_add_epi32(p_2_0, p_2_1);\n    return _mm256_cvtepi32_ps(MM256_SET_M128I(p_2, p_1));\n}\n\n// quad fp16 delta calculation\nstatic inline __m256 quad_fp16_delta_float(const float x0, const float y0, const float x1, const float y1) {\n    // GGML_CPU_FP16_TO_FP32 is faster than Intel F16C\n    return _mm256_set_m128(_mm_set1_ps(GGML_CPU_FP16_TO_FP32(x1) * GGML_CPU_FP16_TO_FP32(y1)),\n                           _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x0) * GGML_CPU_FP16_TO_FP32(y0)));\n}\n\nstatic inline __m256 quad_mx_delta_float(const uint8_t x0, const float y0, const uint8_t x1, const float y1) {\n    return _mm256_set_m128(_mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x1) * GGML_CPU_FP16_TO_FP32(y1)),\n                           _mm_set1_ps(GGML_CPU_E8M0_TO_FP32_HALF(x0) * GGML_CPU_FP16_TO_FP32(y0)));\n}\n#endif\n#elif defined(__SSSE3__)\n// horizontally add 4x4 floats\nstatic inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) {\n    __m128 res_0 =_mm_hadd_ps(a, b);\n    __m128 res_1 =_mm_hadd_ps(c, d);\n    __m128 res =_mm_hadd_ps(res_0, res_1);\n    res =_mm_hadd_ps(res, res);\n    res =_mm_hadd_ps(res, res);\n\n    return _mm_cvtss_f32(res);\n}\n#endif // __AVX__ || __AVX2__ || __AVX512F__\n#endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__)\n\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__) || defined(__AVX__)\n    for (int i = 0; i < nb; i++) {\n        // Load elements into 4 AVX vectors\n        __m256 v0 = _mm256_loadu_ps( x );\n        __m256 v1 = _mm256_loadu_ps( x + 8 );\n        __m256 v2 = _mm256_loadu_ps( x + 16 );\n        __m256 v3 = _mm256_loadu_ps( x + 24 );\n        x += 32;\n\n        // Compute max(abs(e)) for the block\n        const __m256 signBit = _mm256_set1_ps( -0.0f );\n        __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n        __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n        max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n        max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n        const float maxScalar = _mm_cvtss_f32( max4 );\n\n        // Quantize these floats\n        const float d = maxScalar / 127.f;\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n        const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;\n        const __m256 mul = _mm256_set1_ps( id );\n\n        // Apply the multiplier\n        v0 = _mm256_mul_ps( v0, mul );\n        v1 = _mm256_mul_ps( v1, mul );\n        v2 = _mm256_mul_ps( v2, mul );\n        v3 = _mm256_mul_ps( v3, mul );\n\n        // Round to nearest integer\n        v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n        v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n        v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n        v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n        // Convert floats to integers\n        __m256i i0 = _mm256_cvtps_epi32( v0 );\n        __m256i i1 = _mm256_cvtps_epi32( v1 );\n        __m256i i2 = _mm256_cvtps_epi32( v2 );\n        __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n        // Convert int32 to int16\n        i0 = _mm256_packs_epi32( i0, i1 );\t// 0, 1, 2, 3,  8, 9, 10, 11,  4, 5, 6, 7, 12, 13, 14, 15\n        i2 = _mm256_packs_epi32( i2, i3 );\t// 16, 17, 18, 19,  24, 25, 26, 27,  20, 21, 22, 23, 28, 29, 30, 31\n                                            // Convert int16 to int8\n        i0 = _mm256_packs_epi16( i0, i2 );\t// 0, 1, 2, 3,  8, 9, 10, 11,  16, 17, 18, 19,  24, 25, 26, 27,  4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31\n\n        // We got our precious signed bytes, but the order is now wrong\n        // These AVX2 pack instructions process 16-byte pieces independently\n        // The following instruction is fixing the order\n        const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n        i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n        _mm256_storeu_si256((__m256i *)y[i].qs, i0);\n#else\n        // Since we don't have in AVX some necessary functions,\n        // we split the registers in half and call AVX2 analogs from SSE\n        __m128i ni0 = _mm256_castsi256_si128( i0 );\n        __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n        __m128i ni2 = _mm256_castsi256_si128( i1 );\n        __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n        __m128i ni4 = _mm256_castsi256_si128( i2 );\n        __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n        __m128i ni6 = _mm256_castsi256_si128( i3 );\n        __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n        // Convert int32 to int16\n        ni0 = _mm_packs_epi32( ni0, ni1 );\n        ni2 = _mm_packs_epi32( ni2, ni3 );\n        ni4 = _mm_packs_epi32( ni4, ni5 );\n        ni6 = _mm_packs_epi32( ni6, ni7 );\n        // Convert int16 to int8\n        ni0 = _mm_packs_epi16( ni0, ni2 );\n        ni4 = _mm_packs_epi16( ni4, ni6 );\n\n        _mm_storeu_si128((__m128i *)(y[i].qs +  0), ni0);\n        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);\n#endif\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_0_ref(x, y, k);\n#endif\n}\n\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK8_1 == 0);\n    const int nb = k / QK8_1;\n\n    block_q8_1 * GGML_RESTRICT y = vy;\n#if defined(__AVX2__) || defined(__AVX__)\n    for (int i = 0; i < nb; i++) {\n        // Load elements into 4 AVX vectors\n        __m256 v0 = _mm256_loadu_ps( x );\n        __m256 v1 = _mm256_loadu_ps( x + 8 );\n        __m256 v2 = _mm256_loadu_ps( x + 16 );\n        __m256 v3 = _mm256_loadu_ps( x + 24 );\n        x += 32;\n\n        // Compute max(abs(e)) for the block\n        const __m256 signBit = _mm256_set1_ps( -0.0f );\n        __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n        maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n        __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n        max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n        max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n        const float max_scalar = _mm_cvtss_f32( max4 );\n\n        // Quantize these floats\n        const float d = max_scalar / 127.f;\n        y[i].d = GGML_CPU_FP32_TO_FP16(d);\n        const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f;\n        const __m256 mul = _mm256_set1_ps( id );\n\n        // Apply the multiplier\n        v0 = _mm256_mul_ps( v0, mul );\n        v1 = _mm256_mul_ps( v1, mul );\n        v2 = _mm256_mul_ps( v2, mul );\n        v3 = _mm256_mul_ps( v3, mul );\n\n        // Round to nearest integer\n        v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n        v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n        v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n        v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n        // Convert floats to integers\n        __m256i i0 = _mm256_cvtps_epi32( v0 );\n        __m256i i1 = _mm256_cvtps_epi32( v1 );\n        __m256i i2 = _mm256_cvtps_epi32( v2 );\n        __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n        // Compute the sum of the quants and set y[i].s\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_8(_mm256_add_epi32(_mm256_add_epi32(i0, i1), _mm256_add_epi32(i2, i3))));\n\n        // Convert int32 to int16\n        i0 = _mm256_packs_epi32( i0, i1 );\t// 0, 1, 2, 3,  8, 9, 10, 11,  4, 5, 6, 7, 12, 13, 14, 15\n        i2 = _mm256_packs_epi32( i2, i3 );\t// 16, 17, 18, 19,  24, 25, 26, 27,  20, 21, 22, 23, 28, 29, 30, 31\n                                            // Convert int16 to int8\n        i0 = _mm256_packs_epi16( i0, i2 );\t// 0, 1, 2, 3,  8, 9, 10, 11,  16, 17, 18, 19,  24, 25, 26, 27,  4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31\n\n        // We got our precious signed bytes, but the order is now wrong\n        // These AVX2 pack instructions process 16-byte pieces independently\n        // The following instruction is fixing the order\n        const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n        i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n        _mm256_storeu_si256((__m256i *)y[i].qs, i0);\n#else\n        // Since we don't have in AVX some necessary functions,\n        // we split the registers in half and call AVX2 analogs from SSE\n        __m128i ni0 = _mm256_castsi256_si128( i0 );\n        __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n        __m128i ni2 = _mm256_castsi256_si128( i1 );\n        __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n        __m128i ni4 = _mm256_castsi256_si128( i2 );\n        __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n        __m128i ni6 = _mm256_castsi256_si128( i3 );\n        __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n        // Compute the sum of the quants and set y[i].s\n        const __m128i s0 = _mm_add_epi32(_mm_add_epi32(ni0, ni1), _mm_add_epi32(ni2, ni3));\n        const __m128i s1 = _mm_add_epi32(_mm_add_epi32(ni4, ni5), _mm_add_epi32(ni6, ni7));\n        y[i].s = GGML_CPU_FP32_TO_FP16(d * hsum_i32_4(_mm_add_epi32(s0, s1)));\n\n        // Convert int32 to int16\n        ni0 = _mm_packs_epi32( ni0, ni1 );\n        ni2 = _mm_packs_epi32( ni2, ni3 );\n        ni4 = _mm_packs_epi32( ni4, ni5 );\n        ni6 = _mm_packs_epi32( ni6, ni7 );\n        // Convert int16 to int8\n        ni0 = _mm_packs_epi16( ni0, ni2 );\n        ni4 = _mm_packs_epi16( ni4, ni6 );\n\n        _mm_storeu_si128((__m128i *)(y[i].qs +  0), ni0);\n        _mm_storeu_si128((__m128i *)(y[i].qs + 16), ni4);\n#endif\n    }\n#else\n    GGML_UNUSED(nb);\n    // scalar\n    quantize_row_q8_1_ref(x, y, k);\n#endif\n}\n\n// placeholder implementation for Apple targets\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\n//\n// Helper functions\n//\n\n#if __AVX__ || __AVX2__ || __AVX512F__\n\n// shuffles to pick the required scales in dot products\nstatic inline __m256i get_scale_shuffle_q3k(int i) {\n    static const uint8_t k_shuffle[128] = {\n         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,     2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,     6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,    10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,    14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,\n    };\n    return _mm256_loadu_si256((const __m256i*)k_shuffle + i);\n}\nstatic inline __m256i get_scale_shuffle_k4(int i) {\n    static const uint8_t k_shuffle[256] = {\n         0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n         2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,\n         4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,\n         6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,\n         8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9,\n        10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,\n        12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,\n        14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15\n    };\n    return _mm256_loadu_si256((const __m256i*)k_shuffle + i);\n}\nstatic inline __m128i get_scale_shuffle(int i) {\n    static const uint8_t k_shuffle[128] = {\n         0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n         2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,\n         4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,\n         6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,\n         8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,\n        10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11,\n        12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13,\n        14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15\n    };\n    return _mm_loadu_si128((const __m128i*)k_shuffle + i);\n}\n#endif\n\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__AVX2__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        /* Compute combined scale for the block */\n        const __m256 d = _mm256_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n\n        // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval.\n        const __m256i off = _mm256_set1_epi8( 8 );\n        qx = _mm256_sub_epi8( qx, off );\n\n        __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        /* Multiply q with scale and accumulate */\n        acc = _mm256_fmadd_ps( d, q, acc );\n    }\n\n    sumf = hsum_float_8(acc);\n#elif defined(__AVX__)\n    __m256 accum = _mm256_setzero_ps();\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n        const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n        const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n        const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n        const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n        const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n        const __m128i q4b_1_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_1), _mm_set1_epi8(8));\n        const __m128i q4b_1_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_1, 4)), _mm_set1_epi8(8));\n        const __m128i q4b_2_0 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), q4bits_2), _mm_set1_epi8(8));\n        const __m128i q4b_2_1 = _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(q4bits_2, 4)), _mm_set1_epi8(8));\n\n        const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);\n        const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);\n        const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);\n        const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);\n        const __m128i p_1 = _mm_add_epi16(p16_1_0, p16_1_1);\n        const __m128i p_2 = _mm_add_epi16(p16_2_0, p16_2_1);\n        const __m256 p =  sum_i16_pairs_float(p_2, p_1);\n\n        const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n        accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n    }\n\n    sumf = hsum_float_8(accum);\n#elif defined(__SSSE3__)\n    // set constants\n    const __m128i lowMask = _mm_set1_epi8(0xF);\n    const __m128i off = _mm_set1_epi8(8);\n\n    // Initialize accumulator with zeros\n    __m128 acc_0 = _mm_setzero_ps();\n    __m128 acc_1 = _mm_setzero_ps();\n    __m128 acc_2 = _mm_setzero_ps();\n    __m128 acc_3 = _mm_setzero_ps();\n\n    for (; ib + 1 < nb; ib += 2) {\n        _mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);\n        _mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);\n\n        // Compute combined scale for the block 0 and 1\n        const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );\n\n        const __m128i tmp_0_1 = _mm_loadu_si128((const __m128i *)x[ib].qs);\n\n        __m128i bx_0 = _mm_and_si128(lowMask, tmp_0_1);\n        __m128i by_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);\n        bx_0 = _mm_sub_epi8(bx_0, off);\n        const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0);\n\n        __m128i bx_1 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_0_1, 4));\n        __m128i by_1 = _mm_loadu_si128((const __m128i *)(y[ib].qs + 16));\n        bx_1 = _mm_sub_epi8(bx_1, off);\n        const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);\n\n        _mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);\n        _mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);\n\n        // Compute combined scale for the block 2 and 3\n        const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );\n\n        const __m128i tmp_2_3 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n\n        __m128i bx_2 = _mm_and_si128(lowMask, tmp_2_3);\n        __m128i by_2 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n        bx_2 = _mm_sub_epi8(bx_2, off);\n        const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2);\n\n        __m128i bx_3 = _mm_and_si128(lowMask, _mm_srli_epi64(tmp_2_3, 4));\n        __m128i by_3 = _mm_loadu_si128((const __m128i *)(y[ib + 1].qs + 16));\n        bx_3 = _mm_sub_epi8(bx_3, off);\n        const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3);\n\n        // Convert int32_t to float\n        __m128 p0 = _mm_cvtepi32_ps(i32_0);\n        __m128 p1 = _mm_cvtepi32_ps(i32_1);\n        __m128 p2 = _mm_cvtepi32_ps(i32_2);\n        __m128 p3 = _mm_cvtepi32_ps(i32_3);\n\n        // Apply the scale\n        __m128 p0_d = _mm_mul_ps( d_0_1, p0 );\n        __m128 p1_d = _mm_mul_ps( d_0_1, p1 );\n        __m128 p2_d = _mm_mul_ps( d_2_3, p2 );\n        __m128 p3_d = _mm_mul_ps( d_2_3, p3 );\n\n        // Acummulate\n        acc_0 = _mm_add_ps(p0_d, acc_0);\n        acc_1 = _mm_add_ps(p1_d, acc_1);\n        acc_2 = _mm_add_ps(p2_d, acc_2);\n        acc_3 = _mm_add_ps(p3_d, acc_3);\n    }\n\n    sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3);\n\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n            const int v1 = (x[ib].qs[j] >>   4) - 8;\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n\n#if defined(__AVX2__) || defined(__AVX__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n\n    float summs = 0;\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d);\n        const float d1 = GGML_CPU_FP16_TO_FP32(y[ib].d);\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        const __m256 d0v = _mm256_set1_ps( d0 );\n        const __m256 d1v = _mm256_set1_ps( d1 );\n\n        // Compute combined scales\n        const __m256 d0d1 = _mm256_mul_ps( d0v, d1v );\n\n        // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes\n        const __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        const __m256i qy = _mm256_loadu_si256( (const __m256i *)y[ib].qs );\n\n        const __m256 xy = mul_sum_us8_pairs_float(qx, qy);\n\n        // Accumulate d0*d1*x*y\n#if defined(__AVX2__)\n        acc = _mm256_fmadd_ps( d0d1, xy, acc );\n#else\n        acc = _mm256_add_ps( _mm256_mul_ps( d0d1, xy ), acc );\n#endif\n    }\n\n    *s = hsum_float_8(acc) + summs;\n#else\n    UNUSED(nb);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(ib);\n    ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __AVX2__\n\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n    const __m256i mone = _mm256_set1_epi16(1);\n\n    __m256 accum1 = _mm256_setzero_ps();\n    __m256 accum2 = _mm256_setzero_ps();\n\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);\n        const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);\n        const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);\n        const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);\n        const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n                                              _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n        const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n                                              _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n        const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n        const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n        const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);\n        const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);\n        const __m256 scale0 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 0].e));\n        const __m256 scale1 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib + 1].e));\n        accum1 = _mm256_fmadd_ps(scale0, _mm256_cvtepi32_ps(p_1), accum1);\n        accum2 = _mm256_fmadd_ps(scale1, _mm256_cvtepi32_ps(p_2), accum2);\n    }\n\n    sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));\n\n#elif defined __AVX__\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_mxfp4);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n\n    __m256 accum = _mm256_setzero_ps();\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n        const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n        const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n        const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n        const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n        const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n        const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n        const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n        const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n        const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n\n        const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);\n        const __m256 deltas = quad_mx_delta_float(x[ib].e, y[ib].d, x[ib + 1].e, y[ib + 1].d);\n        accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n    }\n\n    sumf = hsum_float_8(accum);\n\n#endif\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_E8M0_TO_FP32_HALF(x[ib].e);\n        int sumi1 = 0;\n        int sumi2 = 0;\n        for (int j = 0; j < QK_MXFP4/2; ++j) {\n            sumi1 += y[ib].qs[j +          0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        /* Compute combined scale for the block */\n        const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));\n        qx = _mm256_or_si256(qx, bxhi);\n\n        __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        /* Multiply q with scale and accumulate */\n        acc = _mm256_fmadd_ps(d, q, acc);\n    }\n\n    *s = hsum_float_8(acc);\n#elif defined(__AVX__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n    __m128i mask = _mm_set1_epi8((char)0xF0);\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        /* Compute combined scale for the block */\n        const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n\n        __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);\n        const __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        __m128i bxhil = _mm256_castsi256_si128(bxhi);\n        __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);\n        bxhil = _mm_andnot_si128(bxhil, mask);\n        bxhih = _mm_andnot_si128(bxhih, mask);\n        __m128i bxl = _mm256_castsi256_si128(bx_0);\n        __m128i bxh = _mm256_extractf128_si256(bx_0, 1);\n        bxl = _mm_or_si128(bxl, bxhil);\n        bxh = _mm_or_si128(bxh, bxhih);\n        bx_0 = MM256_SET_M128I(bxh, bxl);\n\n        const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);\n\n        /* Multiply q with scale and accumulate */\n        acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);\n    }\n\n    *s = hsum_float_8(acc);\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n#if defined(__AVX2__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n\n    float summs = 0.0f;\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        __m256i qx = bytes_from_nibbles_32(x[ib].qs);\n        __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        bxhi = _mm256_and_si256(bxhi, _mm256_set1_epi8(0x10));\n        qx = _mm256_or_si256(qx, bxhi);\n\n        const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        const __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_us8_pairs_float(qx, qy);\n\n        acc = _mm256_fmadd_ps(q, _mm256_mul_ps(dx, dy), acc);\n    }\n\n    *s = hsum_float_8(acc) + summs;\n#elif defined(__AVX__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n    __m128i mask = _mm_set1_epi8(0x10);\n\n    float summs = 0.0f;\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        const __m256 dx = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d));\n\n        summs += GGML_CPU_FP16_TO_FP32(x[ib].m) * GGML_CPU_FP16_TO_FP32(y[ib].s);\n\n        __m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);\n        const __m256i bxhi = bytes_from_bits_32(x[ib].qh);\n        __m128i bxhil = _mm256_castsi256_si128(bxhi);\n        __m128i bxhih = _mm256_extractf128_si256(bxhi, 1);\n        bxhil = _mm_and_si128(bxhil, mask);\n        bxhih = _mm_and_si128(bxhih, mask);\n        __m128i bxl = _mm256_castsi256_si128(bx_0);\n        __m128i bxh = _mm256_extractf128_si256(bx_0, 1);\n        bxl = _mm_or_si128(bxl, bxhil);\n        bxh = _mm_or_si128(bxh, bxhih);\n        bx_0 = MM256_SET_M128I(bxh, bxl);\n\n        const __m256 dy = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib].d));\n        const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_us8_pairs_float(bx_0, by_0);\n\n        acc = _mm256_add_ps(_mm256_mul_ps(q, _mm256_mul_ps(dx, dy)), acc);\n    }\n\n    *s = hsum_float_8(acc) + summs;\n#else\n    UNUSED(nb);\n    UNUSED(ib);\n    UNUSED(x);\n    UNUSED(y);\n    ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined(__AVX2__)\n    // Initialize accumulator with zeros\n    __m256 acc = _mm256_setzero_ps();\n\n    // Main loop\n    for (; ib < nb; ++ib) {\n        // Compute combined scale for the block\n        const __m256 d = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d));\n        __m256i qx = _mm256_loadu_si256((const __m256i *)x[ib].qs);\n        __m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);\n\n        const __m256 q = mul_sum_i8_pairs_float(qx, qy);\n\n        // Multiply q with scale and accumulate\n        acc = _mm256_fmadd_ps( d, q, acc );\n    }\n\n    sumf = hsum_float_8(acc);\n#elif defined(__AVX__)\n    __m256 accum = _mm256_setzero_ps();\n\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i qx_1_0 = _mm_loadu_si128((const __m128i *)x[ib].qs);\n        const __m128i qx_1_1 = _mm_loadu_si128((const __m128i *)x[ib].qs + 1);\n        const __m128i qx_2_0 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n        const __m128i qx_2_1 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs + 1);\n        const __m128i qy_1_0 = _mm_loadu_si128((const __m128i *)y[ib].qs);\n        const __m128i qy_1_1 = _mm_loadu_si128((const __m128i *)y[ib].qs + 1);\n        const __m128i qy_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n        const __m128i qy_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n        const __m256 p = mul_sum_i8_quad_float(qx_1_0, qx_1_1, qx_2_0, qx_2_1, qy_1_0, qy_1_1, qy_2_0, qy_2_1);\n        const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n        accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n    }\n\n    sumf = hsum_float_8(accum);\n#endif\n    for (; ib < nb; ++ib) {\n        int sumi = 0;\n\n        for (int j = 0; j < qk; j++) {\n            sumi += x[ib].qs[j]*y[ib].qs[j];\n        }\n\n        sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq1_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n    __m256 sumf = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n        // 16-bit sums\n        __m256i sumi0 = _mm256_setzero_si256();\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n\n        // first 32 bytes of 5 elements\n        {\n            __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs));\n            // 8-bit multiplies with shifts, masks and adds\n            __m256i qx1 = _mm256_add_epi8(qx0, _mm256_add_epi8(qx0, qx0)); // 1 * 3\n            __m256i qx2 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx0, 3), _mm256_set1_epi8(-8)), qx0); // 1 * 9\n            __m256i qx3 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx1, 3), _mm256_set1_epi8(-8)), qx1); // 3 * 9\n            __m256i qx4 = _mm256_add_epi8(_mm256_and_si256(_mm256_slli_epi16(qx2, 3), _mm256_set1_epi8(-8)), qx2); // 9 * 9\n\n            // TODO: can _mm256_mulhi_epu16 be faster even if 16-bits?\n\n            // Cancel the +1 from avg so that it behaves like a halving add\n            qx0 = _mm256_subs_epu8(qx0, _mm256_set1_epi8(1));\n            qx1 = _mm256_subs_epu8(qx1, _mm256_set1_epi8(1));\n            qx2 = _mm256_subs_epu8(qx2, _mm256_set1_epi8(1));\n            qx3 = _mm256_subs_epu8(qx3, _mm256_set1_epi8(1));\n            qx4 = _mm256_subs_epu8(qx4, _mm256_set1_epi8(1));\n            // Multiply by 3 and get the top 2 bits\n            qx0 = _mm256_avg_epu8(qx0, _mm256_avg_epu8(qx0, _mm256_setzero_si256()));\n            qx1 = _mm256_avg_epu8(qx1, _mm256_avg_epu8(qx1, _mm256_setzero_si256()));\n            qx2 = _mm256_avg_epu8(qx2, _mm256_avg_epu8(qx2, _mm256_setzero_si256()));\n            qx3 = _mm256_avg_epu8(qx3, _mm256_avg_epu8(qx3, _mm256_setzero_si256()));\n            qx4 = _mm256_avg_epu8(qx4, _mm256_avg_epu8(qx4, _mm256_setzero_si256()));\n            qx0 = _mm256_and_si256(_mm256_srli_epi16(qx0, 6), _mm256_set1_epi8(3));\n            qx1 = _mm256_and_si256(_mm256_srli_epi16(qx1, 6), _mm256_set1_epi8(3));\n            qx2 = _mm256_and_si256(_mm256_srli_epi16(qx2, 6), _mm256_set1_epi8(3));\n            qx3 = _mm256_and_si256(_mm256_srli_epi16(qx3, 6), _mm256_set1_epi8(3));\n            qx4 = _mm256_and_si256(_mm256_srli_epi16(qx4, 6), _mm256_set1_epi8(3));\n\n            const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs +   0));\n            const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs +  32));\n            const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs +  64));\n            const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs +  96));\n            const __m256i qy4 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 128));\n\n            qx0 = _mm256_maddubs_epi16(qx0, qy0);\n            qx1 = _mm256_maddubs_epi16(qx1, qy1);\n            qx2 = _mm256_maddubs_epi16(qx2, qy2);\n            qx3 = _mm256_maddubs_epi16(qx3, qy3);\n            qx4 = _mm256_maddubs_epi16(qx4, qy4);\n\n            sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));\n            sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));\n            sumi2 = _mm256_add_epi16(sumi2, qx4);\n        }\n\n        // last 16 bytes of 5-element, along with the 4 bytes of 4 elements\n        {\n            __m128i qx0 = _mm_loadu_si128((const __m128i *) (x[i].qs + 32));\n            uint32_t qh;\n            memcpy(&qh, x[i].qh, sizeof(qh)); // potentially unaligned\n            __m256i qx5_l = _mm256_cvtepu8_epi16(_mm_set1_epi32(qh));\n            __m128i qx1 = _mm_add_epi8(qx0, _mm_add_epi8(qx0, qx0)); // 1 * 3\n            __m128i qx2 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx0, 3), _mm_set1_epi8(-8)), qx0); // 1 * 9\n            __m128i qx3 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx1, 3), _mm_set1_epi8(-8)), qx1); // 3 * 9\n            __m128i qx4 = _mm_add_epi8(_mm_and_si128(_mm_slli_epi16(qx2, 3), _mm_set1_epi8(-8)), qx2); // 9 * 9\n            __m256i qx01 = MM256_SET_M128I(qx1, qx0);\n            __m256i qx23 = MM256_SET_M128I(qx3, qx2);\n\n            // avx2 does not have 8-bit multiplies, so 16-bit it is.\n            qx5_l = _mm256_mullo_epi16(qx5_l, _mm256_set_epi16(27, 27, 27, 27, 9, 9, 9, 9, 3, 3, 3, 3, 1, 1, 1, 1));\n            qx5_l = _mm256_and_si256(qx5_l, _mm256_set1_epi16(0xFF));\n            __m128i qx5 = _mm_packus_epi16(_mm256_castsi256_si128(qx5_l), _mm256_extracti128_si256(qx5_l, 1));\n\n            __m256i qx45 = MM256_SET_M128I(qx5, qx4);\n\n            // Cancel the +1 from avg so that it behaves like a halving add\n            qx01 = _mm256_subs_epu8(qx01, _mm256_set1_epi8(1));\n            qx23 = _mm256_subs_epu8(qx23, _mm256_set1_epi8(1));\n            qx45 = _mm256_subs_epu8(qx45, _mm256_set1_epi8(1));\n            // Multiply by 3 and get the top 2 bits\n            qx01 = _mm256_avg_epu8(qx01, _mm256_avg_epu8(qx01, _mm256_setzero_si256()));\n            qx23 = _mm256_avg_epu8(qx23, _mm256_avg_epu8(qx23, _mm256_setzero_si256()));\n            qx45 = _mm256_avg_epu8(qx45, _mm256_avg_epu8(qx45, _mm256_setzero_si256()));\n            qx01 = _mm256_and_si256(_mm256_srli_epi16(qx01, 6), _mm256_set1_epi8(3));\n            qx23 = _mm256_and_si256(_mm256_srli_epi16(qx23, 6), _mm256_set1_epi8(3));\n            qx45 = _mm256_and_si256(_mm256_srli_epi16(qx45, 6), _mm256_set1_epi8(3));\n\n            const __m256i qy01 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 160));\n            const __m256i qy23 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 192));\n            const __m256i qy45 = _mm256_loadu_si256((const __m256i *) (y[i].qs + 224));\n\n            qx01 = _mm256_maddubs_epi16(qx01, qy01);\n            qx23 = _mm256_maddubs_epi16(qx23, qy23);\n            qx45 = _mm256_maddubs_epi16(qx45, qy45);\n\n            sumi0 = _mm256_add_epi16(sumi0, qx01);\n            sumi1 = _mm256_add_epi16(sumi1, qx23);\n            sumi2 = _mm256_add_epi16(sumi2, qx45);\n        }\n\n        const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);\n        const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));\n\n        sumi0 = _mm256_sub_epi16(sumi0, ysum);\n        sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(sumi1, sumi2));\n        sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));\n\n        sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);\n    }\n\n    *s = hsum_float_8(sumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq2_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n    __m256 sumf = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n        // 16-bit sums, because 256*127 still fits\n        __m256i sumi0 = _mm256_setzero_si256();\n        __m256i sumi1 = _mm256_setzero_si256();\n\n        for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n            __m256i qx0 = _mm256_loadu_si256((const __m256i *) (x[i].qs + j));\n            __m256i qx1 = _mm256_srli_epi16(qx0, 2);\n            __m256i qx2 = _mm256_srli_epi16(qx0, 4);\n            __m256i qx3 = _mm256_srli_epi16(qx0, 6);\n\n            // 0, 1, 2 (should not be 3)\n            qx0 = _mm256_and_si256(qx0, _mm256_set1_epi8(3));\n            qx1 = _mm256_and_si256(qx1, _mm256_set1_epi8(3));\n            qx2 = _mm256_and_si256(qx2, _mm256_set1_epi8(3));\n            qx3 = _mm256_and_si256(qx3, _mm256_set1_epi8(3));\n\n            const __m256i qy0 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 +  0));\n            const __m256i qy1 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 32));\n            const __m256i qy2 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 64));\n            const __m256i qy3 = _mm256_loadu_si256((const __m256i *) (y[i].qs + j*4 + 96));\n\n            qx0 = _mm256_maddubs_epi16(qx0, qy0);\n            qx1 = _mm256_maddubs_epi16(qx1, qy1);\n            qx2 = _mm256_maddubs_epi16(qx2, qy2);\n            qx3 = _mm256_maddubs_epi16(qx3, qy3);\n\n            sumi0 = _mm256_add_epi16(sumi0, _mm256_add_epi16(qx0, qx1));\n            sumi1 = _mm256_add_epi16(sumi1, _mm256_add_epi16(qx2, qx3));\n        }\n\n        const __m256i ysum = _mm256_loadu_si256((const __m256i *) y[i].bsums);\n        const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d));\n\n        sumi0 = _mm256_add_epi16(sumi0, sumi1);\n        sumi0 = _mm256_sub_epi16(sumi0, ysum);\n        sumi0 = _mm256_madd_epi16(sumi0, _mm256_set1_epi16(1));\n\n        sumf = _mm256_add_ps(_mm256_mul_ps(_mm256_cvtepi32_ps(sumi0), d), sumf);\n    }\n\n    *s = hsum_float_8(sumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n    const __m256i m3 = _mm256_set1_epi8(3);\n    const __m128i m4 = _mm_set1_epi8(0xF);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n        const __m128i scales8 = _mm_and_si128(mins_and_scales, m4);\n        const __m128i mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);\n        const __m256i mins = _mm256_cvtepi8_epi16(mins8);\n        const __m256i prod = _mm256_madd_epi16(mins, _mm256_loadu_si256((const __m256i*)y[i].bsums));\n\n        acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc);\n\n        const __m256i all_scales = _mm256_cvtepi8_epi16(scales8);\n        const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);\n        const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);\n        const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};\n\n        __m256i sumi = _mm256_setzero_si256();\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const __m256i q2bits = _mm256_loadu_si256((const __m256i*)q2); q2 += 32;\n\n            const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            const __m256i q2_0 = _mm256_and_si256(q2bits, m3);\n            const __m256i q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3);\n            const __m256i q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3);\n            const __m256i q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3);\n\n            __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0);\n            __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1);\n            __m256i p2 = _mm256_maddubs_epi16(q2_2, q8_2);\n            __m256i p3 = _mm256_maddubs_epi16(q2_3, q8_3);\n\n            p0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(0)), p0);\n            p1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(1)), p1);\n            p2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(2)), p2);\n            p3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(3)), p3);\n\n            p0 = _mm256_add_epi32(p0, p1);\n            p2 = _mm256_add_epi32(p2, p3);\n\n            sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2));\n        }\n\n        acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n\n    }\n\n    *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n    const __m128i m3 = _mm_set1_epi8(0x3);\n    const __m128i m4 = _mm_set1_epi8(0xF);\n    const __m128i m2 = _mm_set1_epi8(0x2);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // load mins and scales from block_q2_K.scales[QK_K/16]\n        const __m128i mins_and_scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n        const __m128i scales16 = _mm_and_si128(mins_and_scales, m4);\n        const __m128i mins16 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4);\n        const __m128i mins_0 = _mm_cvtepi8_epi16(mins16);\n        const __m128i mins_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(mins16, mins16));\n\n        // summs = y[i].bsums * (x[i].scales >> 4) in 16bits*8*2 to 32bits*4*2\n        const __m128i summs_0 = _mm_madd_epi16(mins_0, _mm_loadu_si128((const __m128i*)&y[i].bsums[0]));\n        const __m128i summs_1 = _mm_madd_epi16(mins_1, _mm_loadu_si128((const __m128i*)&y[i].bsums[8]));\n\n        // sumf += -dmin * summs in 32bits*8\n        acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(MM256_SET_M128I(summs_1, summs_0))), acc);\n\n        const __m128i scales_0 = _mm_cvtepi8_epi16(scales16);\n        const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales16, scales16));\n        const __m128i scales[2] = { scales_0, scales_1 };\n\n        __m128i sumi_0 = _mm_setzero_si128();\n        __m128i sumi_1 = _mm_setzero_si128();\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            // load Q8 quants int8*16*8 from block_q8_K.qs[QK_K]\n            const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n            // load 2bits*16*8 from block_q2_K.qs[QK_K/4]\n            __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;\n            const __m128i q2_0 = _mm_and_si128(q2bits, m3);\n            const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);\n            const __m128i q2_4 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);\n            const __m128i q2_6 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);\n            q2bits = _mm_loadu_si128((const __m128i*)q2); q2 += 16;\n            const __m128i q2_1 = _mm_and_si128(q2bits, m3);\n            const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3);\n            const __m128i q2_5 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3);\n            const __m128i q2_7 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3);\n\n            // isuml = q8[l] * ((q2[l] >> shift) & 3) in 8bits*16*8 to 16bits*8*8\n            __m128i p0 = _mm_maddubs_epi16(q2_0, q8_0);\n            __m128i p1 = _mm_maddubs_epi16(q2_1, q8_1);\n            __m128i p2 = _mm_maddubs_epi16(q2_2, q8_2);\n            __m128i p3 = _mm_maddubs_epi16(q2_3, q8_3);\n            __m128i p4 = _mm_maddubs_epi16(q2_4, q8_4);\n            __m128i p5 = _mm_maddubs_epi16(q2_5, q8_5);\n            __m128i p6 = _mm_maddubs_epi16(q2_6, q8_6);\n            __m128i p7 = _mm_maddubs_epi16(q2_7, q8_7);\n\n            // isum += (x[i].scales[is++] & 0xF) * isuml in 16bits*8*8 to 32bits*4*8\n            __m128i shuffle = _mm_set1_epi16(0x0100);\n            p0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p0);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p1);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p2);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p3);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p4);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p5);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p6);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p7);\n\n            p0 = _mm_add_epi32(p0, p1);\n            p2 = _mm_add_epi32(p2, p3);\n            p4 = _mm_add_epi32(p4, p5);\n            p6 = _mm_add_epi32(p6, p7);\n\n            // isum in 32bits*4*2\n            sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p0, p2));\n            sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p4, p6));\n        }\n\n        // sumf += dall * isum - dmin * summs in 32bits\n        __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n        acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&dall), _mm256_cvtepi32_ps(sumi)), acc);\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n    const __m256i m3 = _mm256_set1_epi8(3);\n    const __m256i mone = _mm256_set1_epi8(1);\n    const __m128i m32 = _mm_set1_epi8(32);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    uint32_t aux[3];\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // Set up scales\n        memcpy(aux, x[i].scales, 12);\n        __m128i scales128 = _mm_set_epi32(\n                ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n                ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n                (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n                (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n        scales128 = _mm_sub_epi8(scales128, m32);\n        const __m256i all_scales = _mm256_cvtepi8_epi16(scales128);\n        const __m128i l_scales = _mm256_extracti128_si256(all_scales, 0);\n        const __m128i h_scales = _mm256_extracti128_si256(all_scales, 1);\n        const __m256i scales[2] = {MM256_SET_M128I(l_scales, l_scales), MM256_SET_M128I(h_scales, h_scales)};\n\n        // high bit\n        const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].hmask);\n\n        // integer accumulator\n        __m256i sumi = _mm256_setzero_si256();\n\n        int bit = 0;\n        int is  = 0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            // load low 2 bits\n            const __m256i q3bits = _mm256_loadu_si256((const __m256i*)q3); q3 += 32;\n\n            // prepare low and high bits\n            const __m256i q3l_0 = _mm256_and_si256(q3bits, m3);\n            const __m256i q3h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n            ++bit;\n\n            const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3);\n            const __m256i q3h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n            ++bit;\n\n            const __m256i q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3);\n            const __m256i q3h_2 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n            ++bit;\n\n            const __m256i q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3);\n            const __m256i q3h_3 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, bit)), bit), 2);\n            ++bit;\n\n            // load Q8 quants\n            const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,\n            // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,\n            // and 2 if the high bit was set)\n            __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0);\n            __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1);\n            __m256i q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2);\n            __m256i q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3);\n\n            __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0);\n            __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1);\n            __m256i p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2);\n            __m256i p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3);\n\n            p16_0 = _mm256_sub_epi16(p16_0, q8s_0);\n            p16_1 = _mm256_sub_epi16(p16_1, q8s_1);\n            p16_2 = _mm256_sub_epi16(p16_2, q8s_2);\n            p16_3 = _mm256_sub_epi16(p16_3, q8s_3);\n\n            // multiply with scales\n            p16_0 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0);\n            p16_1 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1);\n            p16_2 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2);\n            p16_3 = _mm256_madd_epi16(_mm256_shuffle_epi8(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3);\n\n            // accumulate\n            p16_0 = _mm256_add_epi32(p16_0, p16_1);\n            p16_2 = _mm256_add_epi32(p16_2, p16_3);\n            sumi  = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2));\n\n        }\n\n        // multiply with block scale and accumulate\n        acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n\n    }\n\n    *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n    const __m128i m3 = _mm_set1_epi8(3);\n    const __m128i mone = _mm_set1_epi8(1);\n    const __m128i m32 = _mm_set1_epi8(32);\n    const __m128i m2 = _mm_set1_epi8(2);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    const uint32_t *aux;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // Set up scales\n        aux = (const uint32_t *)x[i].scales;\n        __m128i scales128 = _mm_set_epi32(\n                ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4),\n                ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4),\n                (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4),\n                (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4));\n        scales128 = _mm_sub_epi8(scales128, m32);\n        const __m128i scales_0 = _mm_cvtepi8_epi16(scales128);\n        const __m128i scales_1 = _mm_cvtepi8_epi16(_mm_unpackhi_epi64(scales128, scales128));\n        const __m128i scales[2] = { scales_0, scales_1 };\n\n        // high bit *128*2 from block_q3_K.hmask[QK_K/8]\n        const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].hmask[0]);\n        const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].hmask[16]);\n\n        // integer accumulator\n        __m128i sumi_0 = _mm_setzero_si128();\n        __m128i sumi_1 = _mm_setzero_si128();\n\n        for (int j = 0; j < QK_K/128; ++j) {\n            // load low 2 bits *64*2 from block_q3_K.qs[QK_K/4]\n            const __m128i q3bits_0 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;\n            const __m128i q3bits_1 = _mm_loadu_si128((const __m128i*)q3); q3 += 16;\n\n            // prepare low and high bits\n            const int bit = j << 2;\n\n            const __m128i q3l_0 = _mm_and_si128(q3bits_0, m3);\n            const __m128i q3l_1 = _mm_and_si128(q3bits_1, m3);\n            const __m128i q3h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit)), bit), 2);\n            const __m128i q3h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit)), bit), 2);\n\n            const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 2), m3);\n            const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 2), m3);\n            const __m128i q3h_2 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+1)), bit+1), 2);\n            const __m128i q3h_3 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+1)), bit+1), 2);\n\n            const __m128i q3l_4 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 4), m3);\n            const __m128i q3l_5 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 4), m3);\n            const __m128i q3h_4 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+2)), bit+2), 2);\n            const __m128i q3h_5 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+2)), bit+2), 2);\n\n            const __m128i q3l_6 = _mm_and_si128(_mm_srli_epi16(q3bits_0, 6), m3);\n            const __m128i q3l_7 = _mm_and_si128(_mm_srli_epi16(q3bits_1, 6), m3);\n            const __m128i q3h_6 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_0, _mm_slli_epi16(mone, bit+3)), bit+3), 2);\n            const __m128i q3h_7 = _mm_slli_epi16(_mm_srli_epi16(_mm_andnot_si128(hbits_1, _mm_slli_epi16(mone, bit+3)), bit+3), 2);\n\n            // load Q8 quants from block_q8_K.qs[QK_K]\n            const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n            // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16,\n            // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set,\n            // and 2 if the high bit was set)\n            __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, q8_0);\n            __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, q8_1);\n            __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, q8_2);\n            __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, q8_3);\n            __m128i q8s_4 = _mm_maddubs_epi16(q3h_4, q8_4);\n            __m128i q8s_5 = _mm_maddubs_epi16(q3h_5, q8_5);\n            __m128i q8s_6 = _mm_maddubs_epi16(q3h_6, q8_6);\n            __m128i q8s_7 = _mm_maddubs_epi16(q3h_7, q8_7);\n\n            __m128i p16_0 = _mm_maddubs_epi16(q3l_0, q8_0);\n            __m128i p16_1 = _mm_maddubs_epi16(q3l_1, q8_1);\n            __m128i p16_2 = _mm_maddubs_epi16(q3l_2, q8_2);\n            __m128i p16_3 = _mm_maddubs_epi16(q3l_3, q8_3);\n            __m128i p16_4 = _mm_maddubs_epi16(q3l_4, q8_4);\n            __m128i p16_5 = _mm_maddubs_epi16(q3l_5, q8_5);\n            __m128i p16_6 = _mm_maddubs_epi16(q3l_6, q8_6);\n            __m128i p16_7 = _mm_maddubs_epi16(q3l_7, q8_7);\n\n            p16_0 = _mm_sub_epi16(p16_0, q8s_0);\n            p16_1 = _mm_sub_epi16(p16_1, q8s_1);\n            p16_2 = _mm_sub_epi16(p16_2, q8s_2);\n            p16_3 = _mm_sub_epi16(p16_3, q8s_3);\n            p16_4 = _mm_sub_epi16(p16_4, q8s_4);\n            p16_5 = _mm_sub_epi16(p16_5, q8s_5);\n            p16_6 = _mm_sub_epi16(p16_6, q8s_6);\n            p16_7 = _mm_sub_epi16(p16_7, q8s_7);\n\n            // multiply with scales\n            __m128i shuffle = _mm_set1_epi16(0x0100);\n            p16_0 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_0);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_1 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_1);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_2 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_2);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_3 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_3);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_4 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_4);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_5 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_5);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_6 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_6);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            p16_7 = _mm_madd_epi16(_mm_shuffle_epi8(scales[j], shuffle), p16_7);\n\n            // accumulate\n            p16_0 = _mm_add_epi32(p16_0, p16_1);\n            p16_2 = _mm_add_epi32(p16_2, p16_3);\n            p16_4 = _mm_add_epi32(p16_4, p16_5);\n            p16_6 = _mm_add_epi32(p16_6, p16_7);\n            sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n            sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_4, p16_6));\n\n        }\n\n        // multiply with block scale and accumulate\n        __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n        acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi)), acc);\n\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __AVX2__\n\n    const __m256i m4 = _mm256_set1_epi8(0xF);\n\n    __m256 acc = _mm256_setzero_ps();\n    __m128 acc_m = _mm_setzero_ps();\n\n   for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));\n\n        const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);\n        const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n        const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);\n        acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);\n\n        const __m128i sc128  = _mm256_extracti128_si256(mins_and_scales, 0);\n        const __m256i scales = MM256_SET_M128I(sc128, sc128);\n\n        __m256i sumi = _mm256_setzero_si256();\n\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m256i scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));\n            const __m256i scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));\n\n            const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n            const __m256i q4l = _mm256_and_si256(q4bits, m4);\n            const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4);\n\n            const __m256i q8l = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            __m256i p16l = _mm256_maddubs_epi16(q4l, q8l);\n            p16l = _mm256_madd_epi16(scale_l, p16l);\n\n            const __m256i q8h = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            __m256i p16h = _mm256_maddubs_epi16(q4h, q8h);\n            p16h = _mm256_madd_epi16(scale_h, p16h);\n            const __m256i sumj = _mm256_add_epi32(p16l, p16h);\n\n            sumi = _mm256_add_epi32(sumi, sumj);\n        }\n\n        __m256 vd = _mm256_set1_ps(d);\n        acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);\n\n    }\n\n    acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));\n    acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));\n\n    *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);\n\n#elif defined __AVX__\n\n    const __m128i m4 = _mm_set1_epi8(0xF);\n    const __m128i m2 = _mm_set1_epi8(0x2);\n\n    __m256 acc = _mm256_setzero_ps();\n    __m128 acc_m = _mm_setzero_ps();\n\n   for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);\n        const __m128i scales = _mm_cvtepu8_epi16(utmps);\n        const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));\n\n        const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);\n        const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);\n        const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);\n        const __m128i prod = _mm_madd_epi16(mins, q8s);\n        acc_m = _mm_add_ps(_mm_mul_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod)), acc_m);\n\n        __m128i sumi_0 = _mm_setzero_si128();\n        __m128i sumi_1 = _mm_setzero_si128();\n\n        __m128i shuffle = _mm_set1_epi16(0x0100);\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m128i scale_l = _mm_shuffle_epi8(scales, shuffle);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            const __m128i scale_h = _mm_shuffle_epi8(scales, shuffle);\n            shuffle = _mm_add_epi16(shuffle, m2);\n\n            __m128i q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n            const __m128i q4l_0 = _mm_and_si128(q4bits, m4);\n            const __m128i q4h_0 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);\n            q4bits = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n            const __m128i q4l_1 = _mm_and_si128(q4bits, m4);\n            const __m128i q4h_1 = _mm_and_si128(_mm_srli_epi16(q4bits, 4), m4);\n\n            const __m128i q8l_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            __m128i p16l = _mm_maddubs_epi16(q4l_0, q8l_0);\n            p16l = _mm_madd_epi16(scale_l, p16l);\n            sumi_0 = _mm_add_epi32(sumi_0, p16l);\n            const __m128i q8l_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            p16l = _mm_maddubs_epi16(q4l_1, q8l_1);\n            p16l = _mm_madd_epi16(scale_l, p16l);\n            sumi_1 = _mm_add_epi32(sumi_1, p16l);\n\n            const __m128i q8h_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            __m128i p16h = _mm_maddubs_epi16(q4h_0, q8h_0);\n            p16h = _mm_madd_epi16(scale_h, p16h);\n            sumi_0 = _mm_add_epi32(sumi_0, p16h);\n            const __m128i q8h_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            p16h = _mm_maddubs_epi16(q4h_1, q8h_1);\n            p16h = _mm_madd_epi16(scale_h, p16h);\n            sumi_1 = _mm_add_epi32(sumi_1, p16h);\n\n        }\n\n        __m256 vd = _mm256_set1_ps(d);\n        __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n        acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);\n\n    }\n\n    acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));\n    acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));\n\n    *s = hsum_float_8(acc) + _mm_cvtss_f32(acc_m);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n#if defined __AVX2__\n\n    const __m256i m4 = _mm256_set1_epi8(0xF);\n    const __m128i mzero = _mm_setzero_si128();\n    const __m256i mone  = _mm256_set1_epi8(1);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    float summs = 0.f;\n\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));\n\n        const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);\n        const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));\n        const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);\n        const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);\n        summs += dmin * _mm_extract_epi32(hsum, 0);\n\n        const __m128i sc128  = _mm256_extracti128_si256(mins_and_scales, 0);\n        const __m256i scales = MM256_SET_M128I(sc128, sc128);\n\n        const __m256i hbits = _mm256_loadu_si256((const __m256i*)x[i].qh);\n        __m256i hmask = mone;\n\n        __m256i sumi = _mm256_setzero_si256();\n\n        int bit = 0;\n\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m256i scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+0));\n            const __m256i scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2*j+1));\n\n            const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); q5 += 32;\n\n            const __m256i q5l_0 = _mm256_and_si256(q5bits, m4);\n            const __m256i q5h_0 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);\n            const __m256i q5_0  = _mm256_add_epi8(q5l_0, q5h_0);\n            hmask = _mm256_slli_epi16(hmask, 1);\n\n            const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4);\n            const __m256i q5h_1 = _mm256_slli_epi16(_mm256_srli_epi16(_mm256_and_si256(hbits, hmask), bit++), 4);\n            const __m256i q5_1  = _mm256_add_epi8(q5l_1, q5h_1);\n            hmask = _mm256_slli_epi16(hmask, 1);\n\n            const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            __m256i p16_0 = _mm256_maddubs_epi16(q5_0, q8_0);\n            __m256i p16_1 = _mm256_maddubs_epi16(q5_1, q8_1);\n\n            p16_0 = _mm256_madd_epi16(scale_0, p16_0);\n            p16_1 = _mm256_madd_epi16(scale_1, p16_1);\n\n            sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));\n\n        }\n\n        __m256 vd = _mm256_set1_ps(d);\n        acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc);\n\n    }\n\n    *s = hsum_float_8(acc) + summs;\n\n#elif defined __AVX__\n\n    const __m128i m4 = _mm_set1_epi8(0xF);\n    const __m128i mzero = _mm_setzero_si128();\n    const __m128i mone  = _mm_set1_epi8(1);\n    const __m128i m2 = _mm_set1_epi8(2);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    float summs = 0.f;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        const uint8_t * GGML_RESTRICT q5 = x[i].qs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        const __m128i utmps = _mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]);\n        const __m128i scales = _mm_cvtepu8_epi16(utmps);\n        const __m128i mins = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(utmps, utmps));\n\n        const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)&y[i].bsums[0]);\n        const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)&y[i].bsums[8]);\n        const __m128i q8s = _mm_hadd_epi16(q8sums_0, q8sums_1);\n        const __m128i prod = _mm_madd_epi16(mins, q8s);\n        const __m128i hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero);\n        summs += dmin * _mm_extract_epi32(hsum, 0);\n\n        const __m128i hbits_0 = _mm_loadu_si128((const __m128i*)&x[i].qh[0]);\n        const __m128i hbits_1 = _mm_loadu_si128((const __m128i*)&x[i].qh[16]);\n        __m128i hmask = mone;\n\n        __m128i sumi_0 = _mm_setzero_si128();\n        __m128i sumi_1 = _mm_setzero_si128();\n\n        int bit = 0;\n\n        __m128i shuffle = _mm_set1_epi16(0x0100);\n        for (int j = 0; j < QK_K/64; ++j) {\n\n            const __m128i scale_0 = _mm_shuffle_epi8(scales, shuffle);\n            shuffle = _mm_add_epi16(shuffle, m2);\n            const __m128i scale_1 = _mm_shuffle_epi8(scales, shuffle);\n            shuffle = _mm_add_epi16(shuffle, m2);\n\n            const __m128i q5bits_0 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;\n            const __m128i q5bits_1 = _mm_loadu_si128((const __m128i*)q5); q5 += 16;\n\n            __m128i q5l_0 = _mm_and_si128(q5bits_0, m4);\n            __m128i q5l_1 = _mm_and_si128(q5bits_1, m4);\n            __m128i q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);\n            __m128i q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);\n            __m128i q5_0  = _mm_add_epi8(q5l_0, q5h_0);\n            __m128i q5_1  = _mm_add_epi8(q5l_1, q5h_1);\n            hmask = _mm_slli_epi16(hmask, 1);\n\n            __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            __m128i p16_0 = _mm_maddubs_epi16(q5_0, q8_0);\n            __m128i p16_1 = _mm_maddubs_epi16(q5_1, q8_1);\n            p16_0 = _mm_madd_epi16(scale_0, p16_0);\n            p16_1 = _mm_madd_epi16(scale_0, p16_1);\n\n            q5l_0 = _mm_and_si128(_mm_srli_epi16(q5bits_0, 4), m4);\n            q5l_1 = _mm_and_si128(_mm_srli_epi16(q5bits_1, 4), m4);\n            q5h_0 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_0, hmask), bit), 4);\n            q5h_1 = _mm_slli_epi16(_mm_srli_epi16(_mm_and_si128(hbits_1, hmask), bit++), 4);\n            q5_0  = _mm_add_epi8(q5l_0, q5h_0);\n            q5_1  = _mm_add_epi8(q5l_1, q5h_1);\n            hmask = _mm_slli_epi16(hmask, 1);\n\n            q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            __m128i p16_2 = _mm_maddubs_epi16(q5_0, q8_0);\n            __m128i p16_3 = _mm_maddubs_epi16(q5_1, q8_1);\n            p16_2 = _mm_madd_epi16(scale_1, p16_2);\n            p16_3 = _mm_madd_epi16(scale_1, p16_3);\n\n            sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n            sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));\n\n        }\n\n        __m256 vd = _mm256_set1_ps(d);\n        __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n        acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(sumi)), acc);\n\n    }\n\n    *s = hsum_float_8(acc) + summs;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    UNUSED(utmp);\n    ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n    const __m256i m4 = _mm256_set1_epi8(0xF);\n    const __m256i m2 = _mm256_set1_epi8(3);\n    const __m256i m32s = _mm256_set1_epi8(32);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n\n        __m256i sumi = _mm256_setzero_si256();\n\n        int is = 0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));\n            const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));\n            const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));\n            const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));\n            is += 4;\n\n            const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n            const __m256i q4bits2 = _mm256_loadu_si256((const __m256i*)q4); q4 += 32;\n            const __m256i q4bitsH = _mm256_loadu_si256((const __m256i*)qh); qh += 32;\n\n            const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bitsH, m2), 4);\n            const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 2), m2), 4);\n            const __m256i q4h_2 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 4), m2), 4);\n            const __m256i q4h_3 = _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bitsH, 6), m2), 4);\n\n            const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);\n            const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);\n            const __m256i q4_2 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2);\n            const __m256i q4_3 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3);\n\n            const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8_3 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0);\n            __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1);\n            __m256i q8s_2 = _mm256_maddubs_epi16(m32s, q8_2);\n            __m256i q8s_3 = _mm256_maddubs_epi16(m32s, q8_3);\n\n            __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0);\n            __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1);\n            __m256i p16_2 = _mm256_maddubs_epi16(q4_2, q8_2);\n            __m256i p16_3 = _mm256_maddubs_epi16(q4_3, q8_3);\n\n            p16_0 = _mm256_sub_epi16(p16_0, q8s_0);\n            p16_1 = _mm256_sub_epi16(p16_1, q8s_1);\n            p16_2 = _mm256_sub_epi16(p16_2, q8s_2);\n            p16_3 = _mm256_sub_epi16(p16_3, q8s_3);\n\n            p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0);\n            p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1);\n            p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2);\n            p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3);\n\n            sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1));\n            sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3));\n\n        }\n\n        acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc);\n    }\n\n    *s = hsum_float_8(acc);\n\n#elif defined __AVX__\n\n    const __m128i m3 = _mm_set1_epi8(3);\n    const __m128i m15 = _mm_set1_epi8(15);\n\n    __m256 acc = _mm256_setzero_ps();\n\n    for (int i = 0; i < nb; ++i) {\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        // handle the q6_k -32 offset separately using bsums\n        const __m128i q8sums_0 = _mm_loadu_si128((const __m128i*)y[i].bsums);\n        const __m128i q8sums_1 = _mm_loadu_si128((const __m128i*)y[i].bsums + 1);\n        const __m128i scales = _mm_loadu_si128((const __m128i*)x[i].scales);\n        const __m128i scales_16_0 = _mm_cvtepi8_epi16(scales);\n        const __m128i scales_16_1 = _mm_cvtepi8_epi16(_mm_bsrli_si128(scales, 8));\n        const __m128i q8sclsub_0 = _mm_slli_epi32(_mm_madd_epi16(q8sums_0, scales_16_0), 5);\n        const __m128i q8sclsub_1 = _mm_slli_epi32(_mm_madd_epi16(q8sums_1, scales_16_1), 5);\n\n        __m128i sumi_0 = _mm_setzero_si128();\n        __m128i sumi_1 = _mm_setzero_si128();\n\n        int is = 0;\n\n        for (int j = 0; j < QK_K/128; ++j) {\n\n            const __m128i q4bitsH_0 = _mm_loadu_si128((const __m128i*)qh); qh += 16;\n            const __m128i q4bitsH_1 = _mm_loadu_si128((const __m128i*)qh); qh += 16;\n\n            const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, m3), 4);\n            const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, m3), 4);\n            const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(12)), 2);\n            const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(12)), 2);\n            const __m128i q4h_4 = _mm_and_si128(q4bitsH_0, _mm_set1_epi8(48));\n            const __m128i q4h_5 = _mm_and_si128(q4bitsH_1, _mm_set1_epi8(48));\n            const __m128i q4h_6 = _mm_srli_epi16(_mm_and_si128(q4bitsH_0, _mm_set1_epi8(-64)), 2);\n            const __m128i q4h_7 = _mm_srli_epi16(_mm_and_si128(q4bitsH_1, _mm_set1_epi8(-64)), 2);\n\n            const __m128i q4bits1_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n            const __m128i q4bits1_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n            const __m128i q4bits2_0 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n            const __m128i q4bits2_1 = _mm_loadu_si128((const __m128i*)q4); q4 += 16;\n\n            const __m128i q4_0 = _mm_or_si128(_mm_and_si128(q4bits1_0, m15), q4h_0);\n            const __m128i q4_1 = _mm_or_si128(_mm_and_si128(q4bits1_1, m15), q4h_1);\n            const __m128i q4_2 = _mm_or_si128(_mm_and_si128(q4bits2_0, m15), q4h_2);\n            const __m128i q4_3 = _mm_or_si128(_mm_and_si128(q4bits2_1, m15), q4h_3);\n            const __m128i q4_4 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_0, 4), m15), q4h_4);\n            const __m128i q4_5 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits1_1, 4), m15), q4h_5);\n            const __m128i q4_6 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_0, 4), m15), q4h_6);\n            const __m128i q4_7 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(q4bits2_1, 4), m15), q4h_7);\n\n            const __m128i q8_0 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_1 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_2 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_3 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_4 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_5 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_6 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n            const __m128i q8_7 = _mm_loadu_si128((const __m128i*)q8); q8 += 16;\n\n            __m128i p16_0 = _mm_maddubs_epi16(q4_0, q8_0);\n            __m128i p16_1 = _mm_maddubs_epi16(q4_1, q8_1);\n            __m128i p16_2 = _mm_maddubs_epi16(q4_2, q8_2);\n            __m128i p16_3 = _mm_maddubs_epi16(q4_3, q8_3);\n            __m128i p16_4 = _mm_maddubs_epi16(q4_4, q8_4);\n            __m128i p16_5 = _mm_maddubs_epi16(q4_5, q8_5);\n            __m128i p16_6 = _mm_maddubs_epi16(q4_6, q8_6);\n            __m128i p16_7 = _mm_maddubs_epi16(q4_7, q8_7);\n\n            const __m128i scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 0));\n            const __m128i scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1));\n            const __m128i scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2));\n            const __m128i scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3));\n            is += 4;\n\n            p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0);\n            p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_0, 8)), p16_1);\n            p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2);\n            p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_1, 8)), p16_3);\n            p16_4 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_2), p16_4);\n            p16_5 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_2, 8)), p16_5);\n            p16_6 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_3), p16_6);\n            p16_7 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_bsrli_si128(scale_3, 8)), p16_7);\n\n            sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2));\n            sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3));\n            sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_4, p16_6));\n            sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_5, p16_7));\n\n        }\n\n        sumi_0 = _mm_sub_epi32(sumi_0, q8sclsub_0);\n        sumi_1 = _mm_sub_epi32(sumi_1, q8sclsub_1);\n        const __m256i sumi = MM256_SET_M128I(sumi_1, sumi_0);\n        acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi)), acc);\n    }\n\n    *s = hsum_float_8(acc);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\n#if defined (__AVX__) || defined (__AVX2__)\nstatic const int8_t keven_signs_q2xs[1024] = {\n     1,  1,  1,  1,  1,  1,  1,  1, -1,  1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1,  1,\n     1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1,  1,  1, -1, -1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1, -1,\n     1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1,  1,  1, -1,  1, -1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1, -1,\n     1,  1, -1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1,  1,\n     1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1, -1,\n     1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1,  1,\n     1,  1,  1, -1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1,  1,\n     1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1,  1,  1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1, -1,\n     1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1,  1, -1,  1,  1, -1, -1,  1,  1,  1, -1,  1, -1,\n     1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1,  1,\n     1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1,  1,\n     1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1, -1,\n     1,  1,  1,  1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1,  1,\n     1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1, -1,\n     1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1,  1,  1,  1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1, -1,\n     1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1,  1,\n     1,  1,  1,  1,  1,  1, -1, -1, -1,  1,  1,  1,  1,  1, -1,  1,  1, -1,  1,  1,  1,  1, -1,  1, -1, -1,  1,  1,  1,  1, -1, -1,\n     1,  1, -1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1, -1,  1, -1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1,  1,\n     1,  1,  1, -1,  1,  1, -1,  1, -1,  1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1,  1,\n     1,  1, -1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1, -1,\n     1,  1,  1,  1, -1,  1, -1,  1, -1,  1,  1,  1, -1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1,  1,\n     1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1,  1,  1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1, -1,\n     1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1,  1, -1,  1,  1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1, -1,\n     1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1,  1,\n     1,  1,  1,  1,  1, -1, -1,  1, -1,  1,  1,  1,  1, -1, -1, -1,  1, -1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1,  1,\n     1,  1, -1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1,  1, -1, -1,  1, -1, -1, -1,  1,  1, -1, -1, -1,\n     1,  1,  1, -1,  1, -1, -1, -1, -1,  1,  1, -1,  1, -1, -1,  1,  1, -1,  1, -1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1, -1,\n     1,  1, -1, -1,  1, -1, -1,  1, -1,  1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  1,\n     1,  1,  1,  1, -1, -1, -1, -1, -1,  1,  1,  1, -1, -1, -1,  1,  1, -1,  1,  1, -1, -1, -1,  1, -1, -1,  1,  1, -1, -1, -1, -1,\n     1,  1, -1,  1, -1, -1, -1,  1, -1,  1, -1,  1, -1, -1, -1, -1,  1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1,  1,\n     1,  1,  1, -1, -1, -1, -1,  1, -1,  1,  1, -1, -1, -1, -1, -1,  1, -1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1,  1,\n     1,  1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1,  1,  1, -1, -1, -1, -1, -1, -1,  1, -1, -1, -1, -1, -1, -1, -1, -1,\n};\n#endif\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[4];\n    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n            const __m256i q2_1 = _mm256_set_epi64x(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n            const __m256i q2_2 = _mm256_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n            const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127],\n                                                   signs64[(aux32[3] >>  7) & 127], signs64[(aux32[3] >>  0) & 127]);\n            const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);\n            const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);\n            const __m256i dot1  = _mm256_maddubs_epi16(q2_1, q8s_1);\n            const __m256i dot2  = _mm256_maddubs_epi16(q2_2, q8s_2);\n            const uint16_t ls1 = aux32[1] >> 28;\n            const uint16_t ls2 = aux32[3] >> 28;\n            const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n            const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n            sumi1 = _mm256_add_epi32(sumi1, p1);\n            sumi2 = _mm256_add_epi32(sumi2, p2);\n        }\n\n        accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[4];\n    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8;\n            const __m128i q2_1_0 = _mm_set_epi64x(iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]);\n            const __m128i q2_1_1 = _mm_set_epi64x(iq2xxs_grid[aux8[3]], iq2xxs_grid[aux8[2]]);\n            const __m128i q2_2_0 = _mm_set_epi64x(iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]);\n            const __m128i q2_2_1 = _mm_set_epi64x(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]]);\n            const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);\n            const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[3] >>  7) & 127], signs64[(aux32[3] >>  0) & 127]);\n            const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127]);\n            const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);\n            const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);\n            const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);\n            const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);\n            const __m128i dot1_0  = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n            const __m128i dot1_1  = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n            const __m128i dot2_0  = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n            const __m128i dot2_1  = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n            const uint16_t ls1 = aux32[1] >> 28;\n            const uint16_t ls2 = aux32[3] >> 28;\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n            sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n            sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n            sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n            sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n        }\n\n        accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n    const __m256i mone = _mm256_set1_epi8(1);\n    static const char block_sign_shuffle_mask_1[32] = {\n        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n        0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n    };\n    static const char block_sign_shuffle_mask_2[32] = {\n        0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n        0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n    };\n    static const uint8_t bit_selector_mask_bytes[32] = {\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m256i bit_selector_mask = _mm256_loadu_si256((const __m256i*)bit_selector_mask_bytes);\n    const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1);\n    const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2);\n\n    static const uint8_t k_bit_helper[32] = {\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n    };\n    const __m256i bit_helper = _mm256_loadu_si256((const __m256i*)k_bit_helper);\n    const __m256i m511 = _mm256_set1_epi16(511);\n    const __m128i m4 = _mm_set1_epi8(0xf);\n    const __m128i m1 = _mm_set1_epi8(1);\n\n    uint64_t aux64;\n\n    // somewhat hacky, but gives a significant boost in performance\n    __m256i aux_gindex;\n    const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(&aux64, x[i].scales, 8);\n        __m128i stmp = _mm_set1_epi64x(aux64);\n        stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));\n        const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);\n\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n            const __m256i q2_data = _mm256_loadu_si256((const __m256i*)q2);  q2 += 16;\n            aux_gindex = _mm256_and_si256(q2_data, m511);\n\n            const __m256i partial_sign_bits = _mm256_srli_epi16(q2_data, 9);\n            const __m256i partial_sign_bits_upper = _mm256_srli_epi16(q2_data, 13);\n            const __m256i partial_sign_bits_for_counting = _mm256_xor_si256(partial_sign_bits, partial_sign_bits_upper);\n\n            const __m256i odd_bits = _mm256_shuffle_epi8(bit_helper, partial_sign_bits_for_counting);\n            const __m256i full_sign_bits = _mm256_or_si256(partial_sign_bits, odd_bits);\n\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_3 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_4 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n\n            const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]],\n                                                   iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]);\n            const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]],\n                                                   iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]);\n            const __m256i q2_3 = _mm256_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]],\n                                                   iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]);\n            const __m256i q2_4 = _mm256_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]],\n                                                   iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n\n            const __m128i full_signs_l = _mm256_castsi256_si128(full_sign_bits);\n            const __m128i full_signs_h = _mm256_extractf128_si256(full_sign_bits, 1);\n            const __m256i full_signs_1 = MM256_SET_M128I(full_signs_l, full_signs_l);\n            const __m256i full_signs_2 = MM256_SET_M128I(full_signs_h, full_signs_h);\n\n            __m256i signs;\n            signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_1);\n            signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone));\n\n            signs = _mm256_shuffle_epi8(full_signs_1, block_sign_shuffle_2);\n            signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone));\n\n            signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_1);\n            signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_3 = _mm256_sign_epi8(q8_3, _mm256_or_si256(signs, mone));\n\n            signs = _mm256_shuffle_epi8(full_signs_2, block_sign_shuffle_2);\n            signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask);\n            const __m256i q8s_4 = _mm256_sign_epi8(q8_4, _mm256_or_si256(signs, mone));\n\n            const __m256i dot1  = _mm256_maddubs_epi16(q2_1, q8s_1);\n            const __m256i dot2  = _mm256_maddubs_epi16(q2_2, q8s_2);\n            const __m256i dot3  = _mm256_maddubs_epi16(q2_3, q8s_3);\n            const __m256i dot4  = _mm256_maddubs_epi16(q2_4, q8s_4);\n\n            const __m256i sc1 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0)));\n            const __m256i sc2 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1)));\n            const __m256i sc3 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2)));\n            const __m256i sc4 = _mm256_cvtepi8_epi16(_mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3)));\n\n            sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot1, sc1));\n            sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot2, sc2));\n            sumi1 = _mm256_add_epi32(sumi1, _mm256_madd_epi16(dot3, sc3));\n            sumi2 = _mm256_add_epi32(sumi2, _mm256_madd_epi16(dot4, sc4));\n        }\n\n        accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n    const __m128i mone = _mm_set1_epi8(1);\n    static const char block_sign_shuffle_mask_1[32] = {\n        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,\n        0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,\n    };\n    static const char block_sign_shuffle_mask_2[32] = {\n        0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a,\n        0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e,\n    };\n    static const uint8_t bit_selector_mask_bytes[32] = {\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m128i bit_selector_mask_0 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes);\n    const __m128i bit_selector_mask_1 = _mm_loadu_si128((const __m128i*)bit_selector_mask_bytes + 1);\n    const __m128i block_sign_shuffle_1_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1);\n    const __m128i block_sign_shuffle_1_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_1 + 1);\n    const __m128i block_sign_shuffle_2_0 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2);\n    const __m128i block_sign_shuffle_2_1 = _mm_loadu_si128((const __m128i*)block_sign_shuffle_mask_2 + 1);\n\n    static const uint8_t k_bit_helper[32] = {\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n        0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00,\n    };\n    const __m128i bit_helper_0 = _mm_loadu_si128((const __m128i*)k_bit_helper);\n    const __m128i bit_helper_1 = _mm_loadu_si128((const __m128i*)k_bit_helper + 1);\n    const __m128i m511 = _mm_set1_epi16(511);\n    const __m128i m4 = _mm_set1_epi8(0xf);\n    const __m128i m1 = _mm_set1_epi8(1);\n\n    uint64_t aux64;\n\n    // somewhat hacky, but gives a significant boost in performance\n    __m256i aux_gindex;\n    const uint16_t * gindex = (const uint16_t *)&aux_gindex;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(&aux64, x[i].scales, 8);\n        __m128i stmp = _mm_set1_epi64x(aux64);\n        stmp = _mm_unpacklo_epi8(_mm_and_si128(stmp, m4), _mm_and_si128(_mm_srli_epi16(stmp, 4), m4));\n        const __m128i scales = _mm_add_epi8(_mm_slli_epi16(stmp, 1), m1);\n\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) {\n\n            const __m128i q2_data_0 = _mm_loadu_si128((const __m128i*)q2);\n            const __m128i q2_data_1 = _mm_loadu_si128((const __m128i*)q2 + 1);  q2 += 16;\n            aux_gindex = MM256_SET_M128I(_mm_and_si128(q2_data_1, m511), _mm_and_si128(q2_data_0, m511));\n\n            const __m128i partial_sign_bits_0 = _mm_srli_epi16(q2_data_0, 9);\n            const __m128i partial_sign_bits_1 = _mm_srli_epi16(q2_data_1, 9);\n            const __m128i partial_sign_bits_upper_0 = _mm_srli_epi16(q2_data_0, 13);\n            const __m128i partial_sign_bits_upper_1 = _mm_srli_epi16(q2_data_1, 13);\n            const __m128i partial_sign_bits_for_counting_0 = _mm_xor_si128(partial_sign_bits_0, partial_sign_bits_upper_0);\n            const __m128i partial_sign_bits_for_counting_1 = _mm_xor_si128(partial_sign_bits_1, partial_sign_bits_upper_1);\n\n            const __m128i odd_bits_0 = _mm_shuffle_epi8(bit_helper_0, partial_sign_bits_for_counting_0);\n            const __m128i odd_bits_1 = _mm_shuffle_epi8(bit_helper_1, partial_sign_bits_for_counting_1);\n            const __m128i full_sign_bits_0 = _mm_or_si128(partial_sign_bits_0, odd_bits_0);\n            const __m128i full_sign_bits_1 = _mm_or_si128(partial_sign_bits_1, odd_bits_1);\n\n            const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_3_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_3_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_4_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_4_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n            const __m128i q2_1_0 = _mm_set_epi64x(iq2xs_grid[gindex[1]], iq2xs_grid[gindex[0]]);\n            const __m128i q2_1_1 = _mm_set_epi64x(iq2xs_grid[gindex[3]], iq2xs_grid[gindex[2]]);\n            const __m128i q2_2_0 = _mm_set_epi64x(iq2xs_grid[gindex[5]], iq2xs_grid[gindex[4]]);\n            const __m128i q2_2_1 = _mm_set_epi64x(iq2xs_grid[gindex[7]], iq2xs_grid[gindex[6]]);\n            const __m128i q2_3_0 = _mm_set_epi64x(iq2xs_grid[gindex[9]], iq2xs_grid[gindex[8]]);\n            const __m128i q2_3_1 = _mm_set_epi64x(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]]);\n            const __m128i q2_4_0 = _mm_set_epi64x(iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]);\n            const __m128i q2_4_1 = _mm_set_epi64x(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]]);\n\n            // AVX2 full_signs_1 is full_sign_bits_0 here\n            // AVX2 full_signs_2 is full_sign_bits_1 here\n            __m128i signs_0, signs_1;\n            signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_0);\n            signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_1_1);\n            signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n            signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n            const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, _mm_or_si128(signs_0, mone));\n            const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, _mm_or_si128(signs_1, mone));\n\n            signs_0 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_0);\n            signs_1 = _mm_shuffle_epi8(full_sign_bits_0, block_sign_shuffle_2_1);\n            signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n            signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n            const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, _mm_or_si128(signs_0, mone));\n            const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, _mm_or_si128(signs_1, mone));\n\n            signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_0);\n            signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_1_1);\n            signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n            signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n            const __m128i q8s_3_0 = _mm_sign_epi8(q8_3_0, _mm_or_si128(signs_0, mone));\n            const __m128i q8s_3_1 = _mm_sign_epi8(q8_3_1, _mm_or_si128(signs_1, mone));\n\n            signs_0 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_0);\n            signs_1 = _mm_shuffle_epi8(full_sign_bits_1, block_sign_shuffle_2_1);\n            signs_0 = _mm_cmpeq_epi8(_mm_and_si128(signs_0, bit_selector_mask_0), bit_selector_mask_0);\n            signs_1 = _mm_cmpeq_epi8(_mm_and_si128(signs_1, bit_selector_mask_1), bit_selector_mask_1);\n            const __m128i q8s_4_0 = _mm_sign_epi8(q8_4_0, _mm_or_si128(signs_0, mone));\n            const __m128i q8s_4_1 = _mm_sign_epi8(q8_4_1, _mm_or_si128(signs_1, mone));\n\n            const __m128i dot1_0  = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n            const __m128i dot1_1  = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n            const __m128i dot2_0  = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n            const __m128i dot2_1  = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n            const __m128i dot3_0  = _mm_maddubs_epi16(q2_3_0, q8s_3_0);\n            const __m128i dot3_1  = _mm_maddubs_epi16(q2_3_1, q8s_3_1);\n            const __m128i dot4_0  = _mm_maddubs_epi16(q2_4_0, q8s_4_0);\n            const __m128i dot4_1  = _mm_maddubs_epi16(q2_4_1, q8s_4_1);\n\n            __m128i sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+0));\n            const __m128i sc1_0 = _mm_cvtepi8_epi16(sc_tmp);\n            const __m128i sc1_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n            sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+1));\n            const __m128i sc2_0 = _mm_cvtepi8_epi16(sc_tmp);\n            const __m128i sc2_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n            sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+2));\n            const __m128i sc3_0 = _mm_cvtepi8_epi16(sc_tmp);\n            const __m128i sc3_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n            sc_tmp = _mm_shuffle_epi8(scales, get_scale_shuffle(ib32+3));\n            const __m128i sc4_0 = _mm_cvtepi8_epi16(sc_tmp);\n            const __m128i sc4_1 = _mm_cvtepi8_epi16(_mm_srli_si128(sc_tmp, 8));\n\n            sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot1_0, sc1_0));\n            sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot1_1, sc1_1));\n            sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot2_0, sc2_0));\n            sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot2_1, sc2_1));\n            sumi1_0 = _mm_add_epi32(sumi1_0, _mm_madd_epi16(dot3_0, sc3_0));\n            sumi1_1 = _mm_add_epi32(sumi1_1, _mm_madd_epi16(dot3_1, sc3_1));\n            sumi2_0 = _mm_add_epi32(sumi2_0, _mm_madd_epi16(dot4_0, sc4_0));\n            sumi2_1 = _mm_add_epi32(sumi2_1, _mm_madd_epi16(dot4_1, sc4_1));\n        }\n\n        accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m128i m4 = _mm_set1_epi8(0xf);\n    const __m128i m1 = _mm_set1_epi8(1);\n\n    const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);\n    const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);\n\n    uint64_t aux64;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(&aux64, x[i].scales, 8);\n        const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);\n        const __m256i scales16 = _mm256_cvtepi8_epi16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15\n\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q2_1 = _mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n                                                   iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)],\n                                                   iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n                                                   iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n            const __m256i q2_2 = _mm256_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n                                                   iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)],\n                                                   iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n                                                   iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n            qs += 8;\n\n            __m256i aux256 = _mm256_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));\n            aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n            const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);\n            const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);\n\n            aux256 = _mm256_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));\n            aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n            const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);\n            const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);\n\n            signs += 4;\n\n            const __m256i dot1  = _mm256_maddubs_epi16(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1\n            const __m256i dot2  = _mm256_maddubs_epi16(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3\n\n            const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+0)));\n            const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32+1)));\n            sumi1 = _mm256_add_epi32(sumi1, p1);\n            sumi2 = _mm256_add_epi32(sumi2, p2);\n        }\n\n        accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m128i m4 = _mm_set1_epi8(0xf);\n    const __m128i m1 = _mm_set1_epi8(1);\n\n    const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);\n    const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);\n    const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);\n    const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);\n\n    uint64_t aux64;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)(x[i].qs + QK_K/8);\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n\n        memcpy(&aux64, x[i].scales, 8);\n        const __m128i scales8 = _mm_add_epi8(_mm_slli_epi16(_mm_and_si128(_mm_set_epi64x(aux64 >> 4, aux64), m4), 1), m1);\n        const __m128i scales16_0 = _mm_cvtepi8_epi16(scales8);\n        const __m128i scales16_1 = _mm_cvtepi8_epi16(_mm_srli_si128(scales8, 8));\n\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q2_1_0 = _mm_set_epi64x(iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)],\n                                                  iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]);\n            const __m128i q2_1_1 = _mm_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)],\n                                                  iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)]);\n            const __m128i q2_2_0 = _mm_set_epi64x(iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)],\n                                                  iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]);\n            const __m128i q2_2_1 = _mm_set_epi64x(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)],\n                                                  iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)]);\n            qs += 8;\n\n            __m128i aux128_0 = _mm_set1_epi32(signs[0] | ((uint32_t) signs[1] << 16));\n            __m128i aux128_1 = aux128_0;\n            aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n            aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n            const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n            const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n            const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);\n            const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);\n\n            aux128_0 = _mm_set1_epi32(signs[2] | ((uint32_t) signs[3] << 16));\n            aux128_1 = aux128_0;\n            aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n            aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n            const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n            const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n            const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);\n            const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);\n\n            signs += 4;\n\n            const __m128i dot1_0  = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n            const __m128i dot1_1  = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n            const __m128i dot2_0  = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n            const __m128i dot2_1  = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 0)));\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+0), 1)));\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_shuffle_epi8(scales16_0, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 0)));\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_shuffle_epi8(scales16_1, _mm256_extractf128_si256(get_scale_shuffle_k4(ib32+1), 1)));\n            sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n            sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n            sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n            sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n        }\n\n        accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n    }\n\n    *s = 0.125f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[2];\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q2_1 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n                                                  iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            q3 += 8;\n            const __m256i q2_2 = _mm256_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]],\n                                                  iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            q3 += 8;\n            memcpy(aux32, gas, 8); gas += 8;\n            const __m256i s2_1 = _mm256_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127],\n                                                   signs64[(aux32[0] >>  7) & 127], signs64[(aux32[0] >>  0) & 127]);\n            const __m256i s2_2 = _mm256_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127],\n                                                   signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m256i q8s_1 = _mm256_sign_epi8(q8_1, s2_1);\n            const __m256i q8s_2 = _mm256_sign_epi8(q8_2, s2_2);\n            const __m256i dot1  = _mm256_maddubs_epi16(q2_1, q8s_1);\n            const __m256i dot2  = _mm256_maddubs_epi16(q2_2, q8s_2);\n            const uint16_t ls1 = aux32[0] >> 28;\n            const uint16_t ls2 = aux32[1] >> 28;\n            const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n            const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n            sumi1 = _mm256_add_epi32(sumi1, p1);\n            sumi2 = _mm256_add_epi32(sumi2, p2);\n        }\n\n        accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = 0.25f * hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n    const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs;\n\n    uint32_t aux32[2];\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q2_1_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            const __m128i q2_1_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);\n            q3 += 8;\n            const __m128i q2_2_0 = _mm_set_epi32(iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]);\n            const __m128i q2_2_1 = _mm_set_epi32(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]]);\n            q3 += 8;\n            memcpy(aux32, gas, 8); gas += 8;\n            const __m128i s2_1_0 = _mm_set_epi64x(signs64[(aux32[0] >>  7) & 127], signs64[(aux32[0] >>  0) & 127]);\n            const __m128i s2_1_1 = _mm_set_epi64x(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127]);\n            const __m128i s2_2_0 = _mm_set_epi64x(signs64[(aux32[1] >>  7) & 127], signs64[(aux32[1] >>  0) & 127]);\n            const __m128i s2_2_1 = _mm_set_epi64x(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127]);\n            const __m128i q8s_1_0 = _mm_sign_epi8(q8_1_0, s2_1_0);\n            const __m128i q8s_1_1 = _mm_sign_epi8(q8_1_1, s2_1_1);\n            const __m128i q8s_2_0 = _mm_sign_epi8(q8_2_0, s2_2_0);\n            const __m128i q8s_2_1 = _mm_sign_epi8(q8_2_1, s2_2_1);\n            const __m128i dot1_0  = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n            const __m128i dot1_1  = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n            const __m128i dot2_0  = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n            const __m128i dot2_1  = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n            const uint16_t ls1 = aux32[0] >> 28;\n            const uint16_t ls2 = aux32[1] >> 28;\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n            sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n            sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n            sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n            sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n        }\n\n        accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n    }\n\n    *s = 0.25f * hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined(__AVX2__)\n\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m256i mask1 = _mm256_loadu_si256((const __m256i*)k_mask1);\n    const __m256i mask2 = _mm256_loadu_si256((const __m256i*)k_mask2);\n\n    const __m256i idx_shift = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);\n    const __m256i idx_mask  = _mm256_set1_epi32(256);\n\n    typedef union {\n        __m256i  vec[2];\n        uint32_t index[16];\n    } index_t;\n\n    index_t idx;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i idx_l = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *)qs)); qs += 16;\n            idx.vec[0] = _mm256_set1_epi32(qh[ib32+0]);\n            idx.vec[1] = _mm256_set1_epi32(qh[ib32+1]);\n            idx.vec[0] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[0], idx_shift), idx_mask);\n            idx.vec[1] = _mm256_and_si256(_mm256_sllv_epi32(idx.vec[1], idx_shift), idx_mask);\n            idx.vec[0] = _mm256_or_si256(idx.vec[0], _mm256_cvtepi16_epi32(_mm256_castsi256_si128(idx_l)));\n            idx.vec[1] = _mm256_or_si256(idx.vec[1], _mm256_cvtepi16_epi32(_mm256_extractf128_si256(idx_l, 1)));\n\n            // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange.\n            //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4);\n            //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4);\n            const __m256i q2_1 = _mm256_set_epi32(\n                    iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]],\n                    iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]\n            );\n            const __m256i q2_2 = _mm256_set_epi32(\n                    iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]],\n                    iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]]\n            );\n\n            __m256i aux256 = _mm256_set1_epi32(signs[0] | (signs[1] << 16));\n            aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n            const __m256i s2_1 = _mm256_cmpeq_epi8(aux256, mask2);\n            const __m256i q8s_1 = _mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1);\n\n            aux256 = _mm256_set1_epi32(signs[2] | (signs[3] << 16));\n            aux256 = _mm256_and_si256(_mm256_shuffle_epi8(aux256,mask1), mask2);\n            const __m256i s2_2 = _mm256_cmpeq_epi8(aux256, mask2);\n            const __m256i q8s_2 = _mm256_sub_epi8(_mm256_xor_si256(s2_2, q8_2), s2_2);\n\n            signs += 4;\n\n            const __m256i dot1  = _mm256_maddubs_epi16(q2_1, q8s_1);\n            const __m256i dot2  = _mm256_maddubs_epi16(q2_2, q8s_2);\n            const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n            const uint16_t ls2 = x[i].scales[ib32/2] >>  4;\n            const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(2*ls1+1));\n            const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(2*ls2+1));\n            sumi1 = _mm256_add_epi32(sumi1, p1);\n            sumi2 = _mm256_add_epi32(sumi2, p2);\n        }\n\n        accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accumf);\n\n    }\n\n    *s = hsum_float_8(accumf);\n\n#elif defined(__AVX__)\n   static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\n                                       0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03\n   };\n\n    static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n                                        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,\n    };\n\n    const __m128i mask1_0 = _mm_loadu_si128((const __m128i*)k_mask1);\n    const __m128i mask1_1 = _mm_loadu_si128((const __m128i*)k_mask1 + 1);\n    const __m128i mask2_0 = _mm_loadu_si128((const __m128i*)k_mask2);\n    const __m128i mask2_1 = _mm_loadu_si128((const __m128i*)k_mask2 + 1);\n\n    const __m128i idx_mul_0 = _mm_set_epi32(32, 64, 128, 256);\n    const __m128i idx_mul_1 = _mm_set_epi32(2, 4, 8, 16);\n    const __m128i idx_mask  = _mm_set1_epi32(256);\n\n    typedef union {\n        __m128i  vec[4];\n        uint32_t index[16];\n    } index_t;\n\n    index_t idx;\n\n    __m256 accumf = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint16_t * GGML_RESTRICT signs = (const uint16_t *)x[i].signs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const __m128i q8_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i qs_tmp = _mm_loadu_si128((const __m128i *)qs);\n            const __m128i idx_l_0 = _mm_cvtepu8_epi16(qs_tmp);\n            const __m128i idx_l_1 = _mm_cvtepu8_epi16(_mm_srli_si128(qs_tmp, 8)); qs += 16;\n            idx.vec[0] = _mm_set1_epi32(qh[ib32+0]);\n            idx.vec[1] = idx.vec[0];\n            idx.vec[2] = _mm_set1_epi32(qh[ib32+1]);\n            idx.vec[3] = idx.vec[2];\n\n            idx.vec[0] = _mm_and_si128(_mm_mullo_epi32(idx.vec[0], idx_mul_0), idx_mask);\n            idx.vec[1] = _mm_and_si128(_mm_mullo_epi32(idx.vec[1], idx_mul_1), idx_mask);\n            idx.vec[2] = _mm_and_si128(_mm_mullo_epi32(idx.vec[2], idx_mul_0), idx_mask);\n            idx.vec[3] = _mm_and_si128(_mm_mullo_epi32(idx.vec[3], idx_mul_1), idx_mask);\n\n            idx.vec[0] = _mm_or_si128(idx.vec[0], _mm_cvtepi16_epi32(idx_l_0));\n            idx.vec[1] = _mm_or_si128(idx.vec[1], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_0, 8)));\n            idx.vec[2] = _mm_or_si128(idx.vec[2], _mm_cvtepi16_epi32(idx_l_1));\n            idx.vec[3] = _mm_or_si128(idx.vec[3], _mm_cvtepi16_epi32(_mm_srli_si128(idx_l_1, 8)));\n\n            const __m128i q2_1_0 = _mm_set_epi32(iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]]);\n            const __m128i q2_1_1 = _mm_set_epi32(iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]]);\n            const __m128i q2_2_0 = _mm_set_epi32(iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[9]], iq3s_grid[idx.index[8]]);\n            const __m128i q2_2_1 = _mm_set_epi32(iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]]);\n\n            __m128i aux128_0 = _mm_set1_epi32(signs[0] | (signs[1] << 16));\n            __m128i aux128_1 = aux128_0;\n            aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n            aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n            const __m128i s2_1_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n            const __m128i s2_1_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n            const __m128i q8s_1_0 = _mm_sub_epi8(_mm_xor_si128(s2_1_0, q8_1_0), s2_1_0);\n            const __m128i q8s_1_1 = _mm_sub_epi8(_mm_xor_si128(s2_1_1, q8_1_1), s2_1_1);\n\n            aux128_0 = _mm_set1_epi32(signs[2] | (signs[3] << 16));\n            aux128_1 = aux128_0;\n            aux128_0 = _mm_and_si128(_mm_shuffle_epi8(aux128_0,mask1_0), mask2_0);\n            aux128_1 = _mm_and_si128(_mm_shuffle_epi8(aux128_1,mask1_1), mask2_1);\n            const __m128i s2_2_0 = _mm_cmpeq_epi8(aux128_0, mask2_0);\n            const __m128i s2_2_1 = _mm_cmpeq_epi8(aux128_1, mask2_1);\n            const __m128i q8s_2_0 = _mm_sub_epi8(_mm_xor_si128(s2_2_0, q8_2_0), s2_2_0);\n            const __m128i q8s_2_1 = _mm_sub_epi8(_mm_xor_si128(s2_2_1, q8_2_1), s2_2_1);\n\n            signs += 4;\n\n            const __m128i dot1_0  = _mm_maddubs_epi16(q2_1_0, q8s_1_0);\n            const __m128i dot1_1  = _mm_maddubs_epi16(q2_1_1, q8s_1_1);\n            const __m128i dot2_0  = _mm_maddubs_epi16(q2_2_0, q8s_2_0);\n            const __m128i dot2_1  = _mm_maddubs_epi16(q2_2_1, q8s_2_1);\n            const uint16_t ls1 = x[i].scales[ib32/2] & 0xf;\n            const uint16_t ls2 = x[i].scales[ib32/2] >>  4;\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(2*ls1+1));\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(2*ls1+1));\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(2*ls2+1));\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(2*ls2+1));\n            sumi1_0 = _mm_add_epi32(sumi1_0, p1_0);\n            sumi1_1 = _mm_add_epi32(sumi1_1, p1_1);\n            sumi2_0 = _mm_add_epi32(sumi2_0, p2_0);\n            sumi2_1 = _mm_add_epi32(sumi2_1, p2_1);\n        }\n\n        accumf = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_add_epi32(sumi1_1, sumi2_1), _mm_add_epi32(sumi1_0, sumi2_0)))), accumf);\n\n    }\n\n    *s = hsum_float_8(accumf);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n    __m256 accum = _mm256_setzero_ps();\n    float accum1 = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint16_t * qh = x[i].qh;\n\n        __m256i sumi = _mm256_setzero_si256();\n        int sumi1 = 0;\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n#ifdef __BMI2__\n            const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib], 0x700070007000700ULL);\n            const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL) | _pdep_u64(qh[ib + 1], 0x700070007000700ULL);\n            const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);\n            const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);\n            const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);\n            const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);\n#else\n            const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)],\n                                                    iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);\n            const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)],\n                                                    iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);\n#endif\n            qs += 8;\n            const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n            const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n            const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n            const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n            const __m256i p1 = _mm256_madd_epi16(dot1, _mm256_set1_epi16(ls1));\n            const __m256i p2 = _mm256_madd_epi16(dot2, _mm256_set1_epi16(ls2));\n\n            sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p1, p2));\n            sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n                   + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n        }\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        accum = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), accum);\n        accum1 += d * sumi1;\n\n    }\n\n    *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#elif defined __AVX__\n    __m256 accum = _mm256_setzero_ps();\n    float accum1 = 0;\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint16_t * qh = x[i].qh;\n\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        int sumi1 = 0;\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const __m128i q1b_1_0 = _mm_set_epi64x(iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)]);\n            const __m128i q1b_1_1 = _mm_set_epi64x(iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)]);\n            const __m128i q1b_2_0 = _mm_set_epi64x(iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)]);\n            const __m128i q1b_2_1 = _mm_set_epi64x(iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)]);\n            qs += 8;\n            const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n            const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);\n            const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);\n            const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);\n            const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);\n            const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1;\n            const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1;\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, _mm_set1_epi16(ls1));\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, _mm_set1_epi16(ls1));\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, _mm_set1_epi16(ls2));\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, _mm_set1_epi16(ls2));\n\n            sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));\n            sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));\n            sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1\n                   + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2;\n        }\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum);\n        accum1 += d * sumi1;\n\n    }\n\n    *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_m * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    iq1m_scale_t scale;\n\n#if defined __AVX2__\n\n    const __m256i mask = _mm256_set1_epi16(0x7);\n    const __m256i mone = _mm256_set1_epi16(1);\n    const __m256i mone8 = _mm256_set1_epi8(1);\n    const __m256i mtwo8 = _mm256_set1_epi8(2);\n    // VPSHUFB cannot cross 128-bit lanes so odd shifts go to upper half.\n    const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);\n\n    __m256 accum1 = _mm256_setzero_ps();\n    __m256 accum2 = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint8_t  * qh = x[i].qh;\n        const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n        scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n        // Extract 3-bit scales (16 values)\n        __m256i scales = _mm256_set1_epi64x(*(const uint64_t*)sc);\n        scales = _mm256_srlv_epi64(scales, scales_shift);\n        scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);\n\n        // Indices to repeat each scale 8 times.\n        __m256i scales_idx1 = _mm256_set1_epi16(0x0100);\n        __m256i scales_idx2 = _mm256_add_epi8(scales_idx1, _mm256_set1_epi8(8));\n\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n#ifdef __BMI2__\n            const uint64_t packed_idx1 = _pdep_u64(*(const uint32_t *)qs, 0x00ff00ff00ff00ffULL)\n                                       | _pdep_u64(*(const uint16_t*)(qh) & 0x7777, 0xf000f000f000f00ULL);\n            const uint64_t packed_idx2 = _pdep_u64(*(const uint32_t *)(qs + 4), 0x00ff00ff00ff00ffULL)\n                                       | _pdep_u64(*(const uint16_t*)(qh + 2) & 0x7777, 0xf000f000f000f00ULL);\n            const uint16_t *idx1 = (const uint16_t *)(&packed_idx1);\n            const uint16_t *idx2 = (const uint16_t *)(&packed_idx2);\n            const __m256i q1b_1 = _mm256_set_epi64x(iq1s_grid[idx1[3]], iq1s_grid[idx1[2]], iq1s_grid[idx1[1]], iq1s_grid[idx1[0]]);\n            const __m256i q1b_2 = _mm256_set_epi64x(iq1s_grid[idx2[3]], iq1s_grid[idx2[2]], iq1s_grid[idx2[1]], iq1s_grid[idx2[0]]);\n\n            // Convert signs to bytes 0x81 (negative) or 0x01 (positive)\n            const uint64_t delta_sign = _pdep_u64(*(const uint32_t*)(qh) & 0x88888888, 0xf0f0f0f0f0f0f0f0ULL);\n            const __m256i delta1 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign)));\n            const __m256i delta2 = _mm256_or_si256(mone8, _mm256_cvtepi8_epi64(_mm_set1_epi32(delta_sign >> 32)));\n#else\n            const __m256i q1b_1 = _mm256_set_epi64x(\n                    iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)],\n                    iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]\n            );\n            const __m256i q1b_2 = _mm256_set_epi64x(\n                    iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)],\n                    iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]\n            );\n\n            const __m256i delta1 = _mm256_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n            const __m256i delta2 = _mm256_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n#endif\n            const __m256i q8b_1 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n            const __m256i q8b_2 = _mm256_loadu_si256((const __m256i*)q8); q8 += 32;\n\n            const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1);\n            const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2);\n            const __m256i dot3 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_1, delta1));\n            const __m256i dot4 = _mm256_maddubs_epi16(mone8, _mm256_sign_epi8(q8b_2, delta2));\n\n            __m256i scale1 = _mm256_shuffle_epi8(scales, scales_idx1);\n            __m256i scale2 = _mm256_shuffle_epi8(scales, scales_idx2);\n\n            scales_idx1 = _mm256_add_epi8(scales_idx1, mtwo8);\n            scales_idx2 = _mm256_add_epi8(scales_idx2, mtwo8);\n\n            const __m256i p1 = _mm256_madd_epi16(dot1, scale1);\n            const __m256i p2 = _mm256_madd_epi16(dot2, scale2);\n            const __m256i p3 = _mm256_madd_epi16(dot3, scale1);\n            const __m256i p4 = _mm256_madd_epi16(dot4, scale2);\n\n            sumi1 = _mm256_add_epi32(sumi1, _mm256_add_epi32(p1, p2));\n            sumi2 = _mm256_add_epi32(sumi2, _mm256_add_epi32(p3, p4));\n\n            qs += 8; qh += 4;\n        }\n\n        const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));\n\n        accum1 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi1), accum1);\n        accum2 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi2), accum2);\n    }\n\n    *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);\n\n#elif defined __AVX__\n    const __m128i mask = _mm_set1_epi16(0x7);\n    const __m128i mone = _mm_set1_epi16(1);\n\n    __m256 accum1 = _mm256_setzero_ps();\n    __m256 accum2 = _mm256_setzero_ps();\n    for (int i = 0; i < nb; ++i) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint8_t  * qh = x[i].qh;\n        const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n        scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const __m128i q1b_1_0 = _mm_set_epi64x(\n                    iq1s_grid[qs[1] | (((uint16_t)qh[0] << 4) & 0x700)], iq1s_grid[qs[0] | (((uint16_t)qh[0] << 8) & 0x700)]);\n            const __m128i q1b_1_1 = _mm_set_epi64x(\n                    iq1s_grid[qs[3] | (((uint16_t)qh[1] << 4) & 0x700)], iq1s_grid[qs[2] | (((uint16_t)qh[1] << 8) & 0x700)]);\n            const __m128i q1b_2_0 = _mm_set_epi64x(\n                    iq1s_grid[qs[5] | (((uint16_t)qh[2] << 4) & 0x700)], iq1s_grid[qs[4] | (((uint16_t)qh[2] << 8) & 0x700)]);\n            const __m128i q1b_2_1 = _mm_set_epi64x(\n                    iq1s_grid[qs[7] | (((uint16_t)qh[3] << 4) & 0x700)], iq1s_grid[qs[6] | (((uint16_t)qh[3] << 8) & 0x700)]);\n            const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n\n            const __m128i dot1_0 = mul_add_epi8_sse(q1b_1_0, q8b_1_0);\n            const __m128i dot1_1 = mul_add_epi8_sse(q1b_1_1, q8b_1_1);\n            const __m128i dot2_0 = mul_add_epi8_sse(q1b_2_0, q8b_2_0);\n            const __m128i dot2_1 = mul_add_epi8_sse(q1b_2_1, q8b_2_1);\n\n            const __m128i delta1_0 = _mm_set_epi64x(qh[0] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[0] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n            const __m128i delta1_1 = _mm_set_epi64x(qh[1] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[1] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n            const __m128i delta2_0 = _mm_set_epi64x(qh[2] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[2] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n            const __m128i delta2_1 = _mm_set_epi64x(qh[3] & 0x80 ? 0xffffffffffffffff : 0x0101010101010101,\n                                                     qh[3] & 0x08 ? 0xffffffffffffffff : 0x0101010101010101);\n\n            const __m128i dot3_0 = mul_add_epi8_sse(delta1_0, q8b_1_0);\n            const __m128i dot3_1 = mul_add_epi8_sse(delta1_1, q8b_1_1);\n            const __m128i dot4_0 = mul_add_epi8_sse(delta2_0, q8b_2_0);\n            const __m128i dot4_1 = mul_add_epi8_sse(delta2_1, q8b_2_1);\n\n            __m128i scale1_0 = _mm_set1_epi16(sc[ib/2] >> 0);\n            __m128i scale1_1 = _mm_set1_epi16(sc[ib/2] >> 3);\n            __m128i scale2_0 = _mm_set1_epi16(sc[ib/2] >> 6);\n            __m128i scale2_1 = _mm_set1_epi16(sc[ib/2] >> 9);\n\n            scale1_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_0, mask), 1), mone);\n            scale1_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale1_1, mask), 1), mone);\n            scale2_0 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_0, mask), 1), mone);\n            scale2_1 = _mm_add_epi16(_mm_slli_epi16(_mm_and_si128(scale2_1, mask), 1), mone);\n            const __m128i p1_0 = _mm_madd_epi16(dot1_0, scale1_0);\n            const __m128i p1_1 = _mm_madd_epi16(dot1_1, scale1_1);\n            const __m128i p2_0 = _mm_madd_epi16(dot2_0, scale2_0);\n            const __m128i p2_1 = _mm_madd_epi16(dot2_1, scale2_1);\n            const __m128i p3_0 = _mm_madd_epi16(dot3_0, scale1_0);\n            const __m128i p3_1 = _mm_madd_epi16(dot3_1, scale1_1);\n            const __m128i p4_0 = _mm_madd_epi16(dot4_0, scale2_0);\n            const __m128i p4_1 = _mm_madd_epi16(dot4_1, scale2_1);\n\n            sumi1_0 = _mm_add_epi32(sumi1_0, _mm_add_epi32(p1_0, p2_0));\n            sumi1_1 = _mm_add_epi32(sumi1_1, _mm_add_epi32(p1_1, p2_1));\n            sumi2_0 = _mm_add_epi32(sumi2_0, _mm_add_epi32(p3_0, p4_0));\n            sumi2_1 = _mm_add_epi32(sumi2_1, _mm_add_epi32(p3_1, p4_1));\n\n            qs += 8; qh += 4;\n        }\n\n        const __m256 d = _mm256_set1_ps(y[i].d * GGML_CPU_FP16_TO_FP32(scale.f16));\n\n        accum1 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi1_1, sumi1_0))), accum1);\n        accum2 = _mm256_add_ps(_mm256_mul_ps(d, _mm256_cvtepi32_ps(MM256_SET_M128I(sumi2_1, sumi2_0))), accum2);\n    }\n\n    *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    UNUSED(scale);\n    ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n#if defined __AVX2__\n\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n    const __m256i mone = _mm256_set1_epi16(1);\n\n    __m256 accum1 = _mm256_setzero_ps();\n    __m256 accum2 = _mm256_setzero_ps();\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs);\n        const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs);\n        const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs);\n        const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs);\n        const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n                                              _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n        const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n                                              _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n        const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n        const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n        const __m256i p_1 = _mm256_madd_epi16(p16_1, mone);\n        const __m256i p_2 = _mm256_madd_epi16(p16_2, mone);\n        accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)),\n                _mm256_cvtepi32_ps(p_1), accum1);\n        accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)),\n                _mm256_cvtepi32_ps(p_2), accum2);\n    }\n\n    sumf = hsum_float_8(_mm256_add_ps(accum1, accum2));\n\n#elif defined __AVX__\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n\n    __m256 accum = _mm256_setzero_ps();\n    for (; ib + 1 < nb; ib += 2) {\n        const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs);\n        const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs);\n        const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs);\n        const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1);\n        const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs);\n        const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1);\n\n        const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n        const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n        const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n        const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n\n        const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1);\n        const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d);\n        accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum);\n    }\n\n    sumf = hsum_float_8(accum);\n\n#endif\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n        int sumi1 = 0, sumi2 = 0;\n        for (int j = 0; j < QK4_NL/2; ++j) {\n            sumi1 += y[ib].qs[j+       0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n#if defined __AVX2__\n\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n\n    __m256 accum = _mm256_setzero_ps();\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const uint8_t * qs = x[ibl].qs;\n        const int8_t  * q8 = y[ibl].qs;\n        uint16_t sh = x[ibl].scales_h;\n        __m256i sumi1 = _mm256_setzero_si256();\n        __m256i sumi2 = _mm256_setzero_si256();\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)qs);  qs += 16;\n            const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)qs);  qs += 16;\n            const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)q8); q8 += 32;\n            const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)),\n                                                  _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)));\n            const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)),\n                                                  _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)));\n            const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1);\n            const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2);\n            const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n            const int16_t ls2 = ((x[ibl].scales_l[ib/2] >>  4) | ((sh << 2) & 0x30)) - 32;\n            sh >>= 4;\n            const __m256i p_1 = _mm256_madd_epi16(p16_1, _mm256_set1_epi16(ls1));\n            const __m256i p_2 = _mm256_madd_epi16(p16_2, _mm256_set1_epi16(ls2));\n            sumi1 = _mm256_add_epi32(p_1, sumi1);\n            sumi2 = _mm256_add_epi32(p_2, sumi2);\n        }\n        accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n                _mm256_cvtepi32_ps(_mm256_add_epi32(sumi1, sumi2)), accum);\n    }\n\n    *s = hsum_float_8(accum);\n\n#elif defined __AVX__\n    const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);\n    const __m128i m4b  = _mm_set1_epi8(0x0f);\n\n    __m256 accum = _mm256_setzero_ps();\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const uint8_t * qs = x[ibl].qs;\n        const int8_t  * q8 = y[ibl].qs;\n        uint16_t sh = x[ibl].scales_h;\n        __m128i sumi1_0 = _mm_setzero_si128();\n        __m128i sumi1_1 = _mm_setzero_si128();\n        __m128i sumi2_0 = _mm_setzero_si128();\n        __m128i sumi2_1 = _mm_setzero_si128();\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)qs); qs += 16;\n            const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)qs); qs += 16;\n            const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)q8); q8 += 16;\n            const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b));\n            const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b));\n            const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b));\n            const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b));\n            const __m128i p16_1_0 = mul_add_epi8_sse(q4b_1_0, q8b_1_0);\n            const __m128i p16_1_1 = mul_add_epi8_sse(q4b_1_1, q8b_1_1);\n            const __m128i p16_2_0 = mul_add_epi8_sse(q4b_2_0, q8b_2_0);\n            const __m128i p16_2_1 = mul_add_epi8_sse(q4b_2_1, q8b_2_1);\n            const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32;\n            const int16_t ls2 = ((x[ibl].scales_l[ib/2] >>  4) | ((sh << 2) & 0x30)) - 32;\n            sh >>= 4;\n            const __m128i p_1_0 = _mm_madd_epi16(p16_1_0, _mm_set1_epi16(ls1));\n            const __m128i p_1_1 = _mm_madd_epi16(p16_1_1, _mm_set1_epi16(ls1));\n            const __m128i p_2_0 = _mm_madd_epi16(p16_2_0, _mm_set1_epi16(ls2));\n            const __m128i p_2_1 = _mm_madd_epi16(p16_2_1, _mm_set1_epi16(ls2));\n            sumi1_0 = _mm_add_epi32(p_1_0, sumi1_0);\n            sumi1_1 = _mm_add_epi32(p_1_1, sumi1_1);\n            sumi2_0 = _mm_add_epi32(p_2_0, sumi2_0);\n            sumi2_1 = _mm_add_epi32(p_2_1, sumi2_1);\n        }\n        __m128i sumi12_0 = _mm_add_epi32(sumi1_0, sumi2_0);\n        __m128i sumi12_1 = _mm_add_epi32(sumi1_1, sumi2_1);\n        accum = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(x[ibl].d)*y[ibl].d),\n                _mm256_cvtepi32_ps(MM256_SET_M128I(sumi12_1, sumi12_0))), accum);\n    }\n\n    *s = hsum_float_8(accum);\n\n#else\n    UNUSED(x);\n    UNUSED(y);\n    UNUSED(nb);\n    ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);\n#endif\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch/x86/repack.cpp",
    "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include <cmath>\n#include <cstring>\n#include <cassert>\n#include <cstdlib> // for qsort\n#include <cstdio>  // for GGML_ASSERT\n\n#define GGML_CPU_CLANG_WORKAROUND\n#include \"../../repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\n#if defined(__AVX__)\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n#define GGML_F32Cx8x2_LOAD(x, y)     _mm512_cvtph_ps(_mm256_set_m128i(_mm_loadu_si128((const __m128i *)(y)), _mm_loadu_si128((const __m128i *)(x))))\n#define GGML_F32Cx16_REPEAT_LOAD(x)  _mm512_cvtph_ps(_mm256_set_m128i(x, x))\n#endif\n// the  _mm256_cvt intrinsics require F16C\n#define GGML_F32Cx8_LOAD(x)     _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))\n#define GGML_F32Cx8_REPEAT_LOAD(x, loadMask)     _mm256_cvtph_ps(_mm_shuffle_epi32(_mm_maskload_epi32((int const*)(x), loadMask), 68))\n#define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask)     _mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask))\n#else\n#if defined(__AVX512F__)\nstatic inline __m512 __avx512_f32cx8x2_load(ggml_fp16_t *x, ggml_fp16_t *y) {\n    float tmp[16];\n\n    for (int i = 0; i < 8; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n\n    for (int i = 0; i < 8; i++) {\n        tmp[i + 8] = GGML_CPU_FP16_TO_FP32(y[i]);\n    }\n\n    return _mm512_loadu_ps(tmp);\n}\nstatic inline __m512 __avx512_repeat_f32cx16_load(__m128i x) {\n    float tmp[16];\n    uint16_t tmphalf[8];\n    _mm_storeu_si128((__m128i*)tmphalf, x);\n\n    for (int i = 0; i < 4; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n        tmp[i + 4] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n        tmp[i + 8] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n        tmp[i + 12] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n    }\n\n    return _mm512_loadu_ps(tmp);\n}\n#endif\nstatic inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) {\n    float tmp[8];\n\n    for (int i = 0; i < 8; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n\n    return _mm256_loadu_ps(tmp);\n}\nstatic inline __m256 __avx_repeat_f32cx8_load(ggml_fp16_t *x) {\n    float tmp[8];\n\n    for (int i = 0; i < 4; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n        tmp[i + 4] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n\n    return _mm256_loadu_ps(tmp);\n}\nstatic inline __m256 __avx_rearranged_f32cx8_load(ggml_fp16_t *x, __m128i arrangeMask) {\n    uint16_t tmphalf[8];\n    float tmp[8];\n\n    _mm_storeu_si128((__m128i*)tmphalf, _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask));\n    for (int i = 0; i < 8; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i]);\n    }\n\n    return _mm256_loadu_ps(tmp);\n}\n\n#define GGML_F32Cx8_LOAD(x)     __avx_f32cx8_load(x)\n#define GGML_F32Cx8_REPEAT_LOAD(x, loadMask)     __avx_repeat_f32cx8_load(x)\n#define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask)     __avx_rearranged_f32cx8_load(x, arrangeMask)\n#if defined(__AVX512F__)\n#define GGML_F32Cx8x2_LOAD(x, y)     __avx512_f32cx8x2_load(x, y)\n#define GGML_F32Cx16_REPEAT_LOAD(x)  __avx512_repeat_f32cx16_load(x)\n#endif\n#endif\n#endif\n\nstatic inline int nearest_int(float fval) {\n    assert(fabsf(fval) <= 4194303.f);\n    float val = fval + 12582912.f;\n    int i; memcpy(&i, &val, sizeof(int));\n    return (i & 0x007fffff) - 0x00400000;\n}\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n#if defined(__AVX512F__)\n// add int16_t pairwise and return as 512 bit int vector, then add the accumulator\nstatic inline __m512i sum_i16_pairs_acc_int32x16(const __m512i acc, const __m512i x) {\n    const __m512i ones = _mm512_set1_epi16(1);\n    return _mm512_add_epi32(acc, _mm512_madd_epi16(ones, x));\n}\n\nstatic inline __m512i mul_sum_us8_pairs_acc_int32x16(const __m512i acc, const __m512i ax, const __m512i sy) {\n#if defined(__AVX512VNNI__)\n    return _mm512_dpbusd_epi32(acc, ax, sy);\n#else\n    // Perform multiplication and create 16-bit values\n    const __m512i dot = _mm512_maddubs_epi16(ax, sy);\n    return sum_i16_pairs_acc_int32x16(acc, dot);\n#endif\n}\n\n// multiply int8_t, add results pairwise twice and return as 512 bit int vector，then add the accumulator\nstatic inline __m512i mul_sum_i8_pairs_acc_int32x16(const __m512i acc, const __m512i x, const __m512i y) {\n    const __m512i zero = _mm512_setzero_si512();\n    // Get absolute values of x vectors\n    const __m512i ax = _mm512_abs_epi8(x);\n    // Sign the values of the y vectors\n    __mmask64 blt0 = _mm512_movepi8_mask(x);\n    const __m512i sy = _mm512_mask_sub_epi8(y, blt0, zero, y);\n    return mul_sum_us8_pairs_acc_int32x16(acc, ax, sy);\n}\n#endif\n\n// add int16_t pairwise and return as 256 bit int vector, then add the accumulator\nstatic inline __m256i sum_i16_pairs_acc_int32x8(const __m256i acc, const __m256i x) {\n    const __m256i ones = _mm256_set1_epi16(1);\n    return _mm256_add_epi32(acc, _mm256_madd_epi16(ones, x));\n}\n\nstatic inline __m256i mul_sum_us8_pairs_acc_int32x8(const __m256i acc, const __m256i ax, const __m256i sy) {\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n    return _mm256_dpbusd_epi32(acc, ax, sy);\n#elif defined(__AVXVNNI__)\n    return _mm256_dpbusd_avx_epi32(acc, ax, sy);\n#else\n    // Perform multiplication and create 16-bit values\n    const __m256i dot = _mm256_maddubs_epi16(ax, sy);\n    return sum_i16_pairs_acc_int32x8(acc, dot);\n#endif\n}\n\n// Integer variant of the function defined in ggml-quants.c\n// multiply int8_t, add results pairwise twice and return as 256 bit int vector, then add the accumulator\nstatic inline __m256i mul_sum_i8_pairs_acc_int32x8(const __m256i acc, const __m256i x, const __m256i y) {\n#if defined(__AVXVNNIINT8__)\n    return _mm256_dpbssd_epi32(acc, x, y);\n#else\n    // Get absolute values of x vectors\n    const __m256i ax = _mm256_sign_epi8(x, x);\n    // Sign the values of the y vectors\n    const __m256i sy = _mm256_sign_epi8(y, x);\n    return mul_sum_us8_pairs_acc_int32x8(acc, ax, sy);\n#endif\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n#if defined(__AVX2__) || defined(__AVX__)\n    float id[4];\n    __m256 srcv[4][4];\n    __m256 idvec[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            // Load elements into 4 AVX vectors\n            __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 32 );\n            __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 8 );\n            __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 16 );\n            __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 24 );\n\n            // Compute max(abs(e)) for the block\n            const __m256 signBit = _mm256_set1_ps( -0.0f );\n            __m256 maxAbs = _mm256_andnot_ps( signBit, v0 );\n            maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) );\n            maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) );\n            maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) );\n\n            __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n            max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n            max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n            const float maxScalar = _mm_cvtss_f32( max4 );\n\n            // Divided by 127.f to mirror results in quantize_row_q8_0\n            const float d = maxScalar  / 127.f;\n            id[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; //d ? 1.0f / d : 0.0f;\n\n            // Store the scale for the individual block\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n\n            // Store the values in blocks of eight values - Aim is to use these later for block interleaving\n            srcv[row_iter][0] = v0;\n            srcv[row_iter][1] = v1;\n            srcv[row_iter][2] = v2;\n            srcv[row_iter][3] = v3;\n            idvec[row_iter] = _mm256_set1_ps(id[row_iter]);\n        }\n\n        // The loop iterates four times - The aim is to get 4 corresponding chunks of eight bytes from the original weight blocks that are interleaved\n        for (int j = 0; j < 4; j++) {\n            // Apply the multiplier\n            __m256 v0 = _mm256_mul_ps(srcv[0][j], idvec[0]);\n            __m256 v1 = _mm256_mul_ps(srcv[1][j], idvec[1]);\n            __m256 v2 = _mm256_mul_ps(srcv[2][j], idvec[2]);\n            __m256 v3 = _mm256_mul_ps(srcv[3][j], idvec[3]);\n\n            // Round to nearest integer\n            v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n            v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n            v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n            v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n            // Convert floats to integers\n            __m256i i0 = _mm256_cvtps_epi32( v0 );\n            __m256i i1 = _mm256_cvtps_epi32( v1 );\n            __m256i i2 = _mm256_cvtps_epi32( v2 );\n            __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n#if defined(__AVX2__)\n            // Convert int32 to int16\n            i0 = _mm256_packs_epi32( i0, i1 );\n            i2 = _mm256_packs_epi32( i2, i3 );\n            // Convert int16 to int8\n            i0 = _mm256_packs_epi16( i0, i2 );\n\n            //  Permute and store the quantized weights in the required order after the pack instruction\n            const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n            i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n            _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0);\n#else\n            // Since we don't have in AVX some necessary functions,\n            // we split the registers in half and call AVX2 analogs from SSE\n            __m128i ni0 = _mm256_castsi256_si128( i0 );\n            __m128i ni1 = _mm256_extractf128_si256( i0, 1);\n            __m128i ni2 = _mm256_castsi256_si128( i1 );\n            __m128i ni3 = _mm256_extractf128_si256( i1, 1);\n            __m128i ni4 = _mm256_castsi256_si128( i2 );\n            __m128i ni5 = _mm256_extractf128_si256( i2, 1);\n            __m128i ni6 = _mm256_castsi256_si128( i3 );\n            __m128i ni7 = _mm256_extractf128_si256( i3, 1);\n\n            // Convert int32 to int16\n            ni0 = _mm_packs_epi32( ni0, ni1 );\n            ni2 = _mm_packs_epi32( ni2, ni3 );\n            ni4 = _mm_packs_epi32( ni4, ni5 );\n            ni6 = _mm_packs_epi32( ni6, ni7 );\n            // Convert int16 to int8\n            ni0 = _mm_packs_epi16( ni0, ni2 );\n            ni4 = _mm_packs_epi16( ni4, ni6 );\n            _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j), ni0);\n            _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j + 16), ni4);\n#endif\n        }\n    }\n\n#else\n    UNUSED(nb);\n    UNUSED(y);\n    ggml_quantize_mat_q8_0_4x8_generic(x, vy, k);\n#endif\n}\n\nvoid ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK_K == 256);\n    assert(k % QK_K == 0);\n    const int nb = k / QK_K;\n\n    block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n#if defined(__AVX2__)\n    float iscale[4];\n    __m256 srcv[4][32];\n    __m256 iscale_vec[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            // Load elements into 4 AVX vectors\n            __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 );\n            __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 8 );\n            __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 16 );\n            __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 24 );\n\n            // Compute max(abs(e)) for the block\n            const __m256 signBit = _mm256_set1_ps( -0.0f );\n            __m256 abs0 = _mm256_andnot_ps( signBit, v0 );\n            __m256 abs1 = _mm256_andnot_ps( signBit, v1 );\n            __m256 abs2 = _mm256_andnot_ps( signBit, v2 );\n            __m256 abs3 = _mm256_andnot_ps( signBit, v3 );\n\n            __m256 maxAbs = _mm256_max_ps( abs0, abs1 );\n            maxAbs = _mm256_max_ps( maxAbs, abs2 );\n            maxAbs = _mm256_max_ps( maxAbs, abs3 );\n\n            __m256 mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ );\n            __m256 mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ );\n            __m256 mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ );\n            __m256 mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ );\n\n            __m256 maskAbs = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3));\n\n            srcv[row_iter][0] = v0;\n            srcv[row_iter][1] = v1;\n            srcv[row_iter][2] = v2;\n            srcv[row_iter][3] = v3;\n\n            for (int sb = 1; sb < 8; sb++) {\n                // Temporarily stores absolute quant values\n                __m256 tempAbs = maxAbs;\n\n                // Load elements into 4 AVX vectors\n                __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32);\n                __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 8 );\n                __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 16 );\n                __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 24 );\n\n                // Compute max(abs(e)) for the block\n                __m256 abs0 = _mm256_andnot_ps( signBit, v0 );\n                __m256 abs1 = _mm256_andnot_ps( signBit, v1 );\n                __m256 abs2 = _mm256_andnot_ps( signBit, v2 );\n                __m256 abs3 = _mm256_andnot_ps( signBit, v3 );\n\n                maxAbs = _mm256_max_ps( maxAbs, abs0 );\n                maxAbs = _mm256_max_ps( maxAbs, abs1 );\n                maxAbs = _mm256_max_ps( maxAbs, abs2 );\n                maxAbs = _mm256_max_ps( maxAbs, abs3 );\n\n                __m256 mask_prev = _mm256_cmp_ps( tempAbs, maxAbs, _CMP_EQ_OQ );\n                maskAbs = _mm256_and_ps( maskAbs, mask_prev );\n\n                mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ );\n                mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ );\n                mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ );\n                mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ );\n\n                __m256 mask_curr = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3));\n                maskAbs =  _mm256_or_ps(maskAbs, mask_curr);\n\n                srcv[row_iter][sb * 4] = v0;\n                srcv[row_iter][sb * 4 + 1] = v1;\n                srcv[row_iter][sb * 4 + 2] = v2;\n                srcv[row_iter][sb * 4 + 3] = v3;\n            }\n\n            __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) );\n            max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) );\n            max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) );\n            const float maxScalar = _mm_cvtss_f32( max4 );\n\n            __m256 maxScalarVec = _mm256_set1_ps(maxScalar);\n\n            __m256 mask_next = _mm256_cmp_ps( maxScalarVec, maxAbs, _CMP_EQ_OQ );\n            __m256 finalMask = _mm256_and_ps(maskAbs, mask_next);\n\n            const int mask = _mm256_movemask_ps(finalMask);\n            iscale[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f;\n\n            if(mask) {\n                iscale[row_iter] = ( maxScalar != 0.0f ) ? -127.f / maxScalar: 0.0f;\n            }\n\n            y[i].d[row_iter] = maxScalar ? 1/iscale[row_iter] : 0;\n            iscale_vec[row_iter] = _mm256_set1_ps(iscale[row_iter]);\n        }\n\n        __m256i quants_interleaved[32];\n        for (int j = 0; j < 32; j++) {\n            // Apply the multiplier\n            __m256 v0 = _mm256_mul_ps(srcv[0][j], iscale_vec[0]);\n            __m256 v1 = _mm256_mul_ps(srcv[1][j], iscale_vec[1]);\n            __m256 v2 = _mm256_mul_ps(srcv[2][j], iscale_vec[2]);\n            __m256 v3 = _mm256_mul_ps(srcv[3][j], iscale_vec[3]);\n\n            // Round to nearest integer\n            v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST );\n            v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST );\n            v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST );\n            v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST );\n\n            // Convert floats to integers\n            __m256i i0 = _mm256_cvtps_epi32( v0 );\n            __m256i i1 = _mm256_cvtps_epi32( v1 );\n            __m256i i2 = _mm256_cvtps_epi32( v2 );\n            __m256i i3 = _mm256_cvtps_epi32( v3 );\n\n            // Convert int32 to int16\n            i0 = _mm256_packs_epi32( i0, i1 );\n            i2 = _mm256_packs_epi32( i2, i3 );\n            // Convert int16 to int8\n            i0 = _mm256_packs_epi16( i0, i2 );\n\n            //  Permute and store the quantized weights in the required order after the pack instruction\n            const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 );\n            i0 = _mm256_permutevar8x32_epi32( i0, perm );\n\n            _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0);\n            quants_interleaved[j] = i0;\n        }\n\n        // Masks to shuffle the quants of corresponding sub blocks for rearranging quants for vectorized bsums computation\n        __m256i shuffle_mask_sb2 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 0, 1, 4, 5, 6, 7, 8, 9, 8, 9, 12, 13, 14, 15));\n        shuffle_mask_sb2 = _mm256_permute2f128_si256(shuffle_mask_sb2, shuffle_mask_sb2, 0);\n        __m256i shuffle_mask_sb3 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 8, 9, 14, 15));\n        shuffle_mask_sb3 = _mm256_permute2f128_si256(shuffle_mask_sb3, shuffle_mask_sb3, 0);\n        __m256i shuffle_mask_sb4 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 0, 1, 8, 9, 10, 11, 12, 13, 8, 9));\n        shuffle_mask_sb4 = _mm256_permute2f128_si256(shuffle_mask_sb4, shuffle_mask_sb4, 0);\n\n        for (int k = 0; k < 4; k++) {\n            // Quants from four different sub blocks are taken\n            __m256i q0 = quants_interleaved[k * 8 + 0];\n            __m256i q1 = quants_interleaved[k * 8 + 1];\n            __m256i q2 = quants_interleaved[k * 8 + 2];\n            __m256i q3 = quants_interleaved[k * 8 + 3];\n            __m256i q4 = quants_interleaved[k * 8 + 4];\n            __m256i q5 = quants_interleaved[k * 8 + 5];\n            __m256i q6 = quants_interleaved[k * 8 + 6];\n            __m256i q7 = quants_interleaved[k * 8 + 7];\n\n\n            // The below code block has the first half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time\n            __m256i sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2);\n            __m256i sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34);\n            __m256i sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3);\n            sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68);\n            __m256i sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4);\n            sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136);\n\n            __m256i one = _mm256_set1_epi8(1);\n            __m256i bsums_r1 = _mm256_maddubs_epi16(one, sb_h1_interleaved);\n\n            for (int l = 0; l < 3; l++) {\n                // Quants value shifted to process next two values from each sub block\n                q0 = _mm256_srli_epi64(q0, 16);\n                q2 = _mm256_srli_epi64(q2, 16);\n                q4 = _mm256_srli_epi64(q4, 16);\n                q6 = _mm256_srli_epi64(q6, 16);\n\n                sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2);\n                sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34);\n                sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3);\n                sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68);\n                sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4);\n                sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136);\n\n                bsums_r1 = _mm256_add_epi16(bsums_r1, _mm256_maddubs_epi16(one, sb_h1_interleaved));\n            }\n\n            // The below code block has the second half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time\n            __m256i sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2);\n            __m256i sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34);\n            __m256i sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3);\n            sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68);\n            __m256i sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4);\n            sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136);\n\n            __m256i bsums_r2 = _mm256_maddubs_epi16(one, sb_h2_interleaved);\n\n            for (int l = 0; l < 3; l++) {\n                // Quants value shifted to process next two values from each sub block\n                q1 = _mm256_srli_epi64(q1, 16);\n                q3 = _mm256_srli_epi64(q3, 16);\n                q5 = _mm256_srli_epi64(q5, 16);\n                q7 = _mm256_srli_epi64(q7, 16);\n\n                sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2);\n                sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34);\n                sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3);\n                sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68);\n                sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4);\n                sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136);\n\n                bsums_r2 = _mm256_add_epi16(bsums_r2, _mm256_maddubs_epi16(one, sb_h2_interleaved));\n            }\n\n            // Overall bsums in interleaved fashion computed by adding results of both halves\n            __m256i bsums_r = _mm256_add_epi16(bsums_r1, bsums_r2);\n            _mm256_storeu_si256((__m256i *)(y[i].bsums + 16 * k), bsums_r);\n        }\n    }\n\n#else\n    UNUSED(nb);\n    UNUSED(y);\n    ggml_quantize_mat_q8_K_4x8_generic(x, vy, k);\n#endif\n}\n\n//\n// GEMV/GEMM templates\n//\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n\n// GEMV for 8x blocks of 32 4-bit quants with a single scale factor per block\ntemplate<typename block_tx8>\nstatic void gemv_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc, __m256i signextendlut) {\n    static_assert(\n            std::is_same_v<block_tx8, block_q4_0x8> ||\n            std::is_same_v<block_tx8, block_iq4_nlx8> ||\n            std::is_same_v<block_tx8, block_mxfp4x8>,\n            \"Unsupported block type\");\n\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    UNUSED(bs);\n\n    __m128i changemask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n    __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n    // Permute mask used for easier vector processing at later stages\n    const __m256i m4b = _mm256_set1_epi8(0x0F);\n\n    int64_t b_nb = n / 32;\n\n    const block_tx8  * b_ptr_start = (const block_tx8  *)vx;\n    const block_q8_0 * a_ptr_start = (const block_q8_0 *)vy;\n\n    // Process Q8_0 blocks one by one\n    for (int64_t y = 0; y < nr; y++) {\n\n        // Pointers to LHS blocks of block_q8_0 format\n        const block_q8_0 * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight blocks at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < nc / 8; x++) {\n\n            // Pointers to RHS blocks\n            const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulator\n            __m256 acc_row = _mm256_setzero_ps();\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Load 8 blocks of 32 interleaved as 8 bytes (B0 - B7)\n                const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n                const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 1);\n                const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 2);\n                const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 3);\n\n                // 4-bit -> 8-bit - Sign is maintained\n                const __m256i rhs_vec_0123_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_0, m4b)); // B0(0-7) B1(0-7) B2(0-7) B3(0-7)\n                const __m256i rhs_vec_4567_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_0, m4b)); // B4(0-7) B5(0-7) B6(0-7) B7(0-7)\n                const __m256i rhs_vec_0123_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15)\n                const __m256i rhs_vec_4567_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15)\n\n                const __m256i rhs_vec_0123_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b)); // B0(16-23) B1(16-23) B2(16-23) B3(16-23)\n                const __m256i rhs_vec_4567_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b)); // B4(16-23) B5(16-23) B6(16-23) B7(16-23)\n                const __m256i rhs_vec_0123_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b)); // B0(24-31) B1(24-31) B2(24-31) B3(24-31)\n                const __m256i rhs_vec_4567_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b)); // B4(24-31) B5(24-31) B6(24-31) B7(24-31)\n\n                // Load the scale values for the 8 blocks interleaved in block_tx8\n                __m256 col_scale_f32;\n                if constexpr (\n                        std::is_same_v<block_tx8, block_q4_0x8> ||\n                        std::is_same_v<block_tx8, block_iq4_nlx8>) {\n                    col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, changemask);\n                } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) {\n                    // Load 8 E8M0 exponents and convert to float via LUT\n                    // Rearranged to match changemask order: 0,4,1,5,2,6,3,7\n                    col_scale_f32 = _mm256_set_ps(\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n                }\n\n                // Load and convert to FP32 scale from block_q8_0\n                const __m256 row_scale_f32 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(a_ptr[b].d));\n\n                // Load the block values in block_q8_0 in batches of 16 bytes and replicate the same across 256 bit vector\n                __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)a_ptr[b].qs));\n                __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16)));\n\n                lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0); // A0 (0-15) A0(0-15)\n                lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0); // A0 (16-31) A0(16-31))\n\n                __m256i iacc = _mm256_setzero_si256();\n\n                // Dot product done within 32 bit lanes and accumulated in the same vector\n                // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n                // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n                // ...........................................................................\n                // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31)\n\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_0 ,_mm256_shuffle_epi32(rhs_vec_4567_0, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 0));\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_0, 177) ,rhs_vec_4567_0, 170), _mm256_shuffle_epi32(lhs_vec_0, 85));\n\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_1 ,_mm256_shuffle_epi32(rhs_vec_4567_1, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 170));\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_1, 177) ,rhs_vec_4567_1, 170), _mm256_shuffle_epi32(lhs_vec_0, 255));\n\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_2 ,_mm256_shuffle_epi32(rhs_vec_4567_2, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 0));\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_2, 177) ,rhs_vec_4567_2, 170), _mm256_shuffle_epi32(lhs_vec_1, 85));\n\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_3 ,_mm256_shuffle_epi32(rhs_vec_4567_3, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 170));\n                iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_3, 177) ,rhs_vec_4567_3, 170), _mm256_shuffle_epi32(lhs_vec_1, 255));\n\n                // Accumulated values multiplied with appropriate scales\n                acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n            }\n\n            // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n            acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n            _mm256_storeu_ps(s + (y * nr + x * 8), acc_row);\n        }\n    }\n}\n\n// GEMM for 8x blocks of 32 4-bit quants with a single scale factor per block\ntemplate<typename block_tx8>\nstatic void gemm_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc, __m256i signextendlut) {\n    static_assert(\n            std::is_same_v<block_tx8, block_q4_0x8> ||\n            std::is_same_v<block_tx8, block_iq4_nlx8> ||\n            std::is_same_v<block_tx8, block_mxfp4x8>,\n            \"Unsupported block type\");\n\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    const block_tx8    * b_ptr_start = (const block_tx8    *)vx;\n    const block_q8_0x4 * a_ptr_start = (const block_q8_0x4 *)vy;\n\n    int64_t b_nb = n / 32;\n    int64_t y = 0;\n    // Mask to mask out nibbles from packed bytes\n    const __m256i m4b = _mm256_set1_epi8(0x0F);\n    const __m128i loadMask = _mm_blend_epi32(_mm_setzero_si128(), _mm_set1_epi32(0xFFFFFFFF), 3);\n    // Permute mask used for easier vector processing at later stages\n    __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n    int64_t xstart = 0;\n    int anr = nr - nr%16; // Used to align nr with boundary of 16\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n    int anc = nc - nc%16; // Used to align nc with boundary of 16\n                          // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n    const __m512i m4bexpanded = _mm512_set1_epi8(0x0F);\n    // Lookup table to convert signed nibbles to signed bytes expanded to 512 bit length\n    __m512i signextendlutexpanded = _mm512_inserti32x8(_mm512_castsi256_si512(signextendlut), signextendlut, 1);\n\n    // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation\n    for (; y < anr / 4; y += 4) {\n\n        const block_q8_0x4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_tx8 * b_ptr_0 = b_ptr_start + ((x)     * b_nb);\n            const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF\n                const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs));\n                const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32));\n                const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64));\n                const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96));\n\n                const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs));\n                const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32));\n                const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64));\n                const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96));\n\n                // Save the values in the following vectors in the formats B0B1B4B5B8B9BCBD, B2B3B6B7BABBBEBF for further processing and storing of values\n                const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n\n                const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                // 4-bit -> 8-bit - Sign is maintained\n                const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7)\n                const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7)\n\n                const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15)\n                const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15)\n\n                const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23)\n                const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23)\n\n                const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31)\n                const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31)\n\n                // Shuffle pattern one - right side input\n                const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3)\n                const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3)\n\n                const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11)\n                const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11)\n\n                const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19)\n                const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19)\n\n                const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27)\n                const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27)\n\n                // Shuffle pattern two - right side input\n\n                const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7)\n                const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7)\n\n                const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15)\n                const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15)\n\n                const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23)\n                const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23)\n\n                const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31)\n                const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31)\n\n                // Scale values - Load the weight scale values of two block_tx8\n                __m512 col_scale_f32;\n                if constexpr (\n                        std::is_same_v<block_tx8, block_q4_0x8> ||\n                        std::is_same_v<block_tx8, block_iq4_nlx8>) {\n                    col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n                } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) {\n                    //TODO: simd-ify\n                    col_scale_f32 = _mm512_set_ps(\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0]));\n                }\n\n                // Process LHS in pairs of rows\n                for (int rp = 0; rp < 4; rp++) {\n\n                    // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n                    __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs)));\n                    __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0);\n                    __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17);\n                    __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32)));\n                    __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0);\n                    __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17);\n                    __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64)));\n                    __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0);\n                    __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17);\n                    __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96)));\n                    __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0);\n                    __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17);\n\n                    __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1);\n                    __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1);\n                    __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1);\n                    __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1);\n                    __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1);\n                    __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1);\n                    __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1);\n                    __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1);\n\n                    // Shuffle pattern one - left side input\n\n                    const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160);  //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n                    const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160);  //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n                    const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160);  //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n                    const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160);  //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n                    const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160);  //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n                    const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160);  //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n                    const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160);  //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n                    const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160);  //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n                    // Shuffle pattern two - left side input\n\n                    const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245);  //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n                    const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245);  //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n                    const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245);  //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n                    const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245);  //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n                    const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245);  //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n                    const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245);  //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n                    const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245);  //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n                    const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245);  //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    // Resembles MMLAs into 2x2 matrices in ARM Version\n                    const __m512i zero = _mm512_setzero_epi32();\n                    __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1);\n                    __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1);\n                    __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1);\n                    __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1);\n                    __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2);\n                    __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2);\n                    __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2);\n                    __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2);\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n                    __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n                    __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n                    __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n                    // Straighten out to make 4 row vectors\n                    __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n                    __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n                    // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n                    const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptrs[rp][b].d), loadMask), 68);\n                    const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16);\n\n                    // Multiply with appropriate scales and accumulate\n                    acc_rows[rp * 4]     = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)),   acc_rows[rp * 4]);\n                    acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)),  acc_rows[rp * 4 + 1]);\n                    acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                    acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n                }\n            }\n\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n            }\n        }\n    }\n\n    // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation\n    for (; y < nr / 4; y ++) {\n        const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_tx8 * b_ptr_0 = b_ptr_start + ((x)     * b_nb);\n            const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF\n                const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs));\n                const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32));\n                const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64));\n                const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96));\n\n                const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs));\n                const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32));\n                const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64));\n                const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96));\n\n                // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n\n                const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                // 4-bit -> 8-bit - Sign is maintained\n                const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7)\n                const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7)\n\n                const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15)\n                const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15)\n\n                const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23)\n                const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23)\n\n                const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31)\n                const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31)\n\n                // Shuffle pattern one - right side input\n                const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3)\n                const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3)\n\n                const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11)\n                const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11)\n\n                const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19)\n                const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19)\n\n                const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27)\n                const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27)\n\n                // Shuffle pattern two - right side input\n\n                const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7)\n                const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7)\n\n                const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15)\n                const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15)\n\n                const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23)\n                const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23)\n\n                const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31)\n                const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31)\n\n\n                // Scale values - Load the weight scale values of two block_tx8\n                __m512 col_scale_f32;\n                if constexpr (\n                        std::is_same_v<block_tx8, block_q4_0x8> ||\n                        std::is_same_v<block_tx8, block_iq4_nlx8>) {\n                    col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n                } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) {\n                    //TODO: simd-ify\n                    col_scale_f32 = _mm512_set_ps(\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0]));\n                }\n\n                // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n                // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n                __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs)));\n                __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0);\n                __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17);\n                __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32)));\n                __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0);\n                __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17);\n                __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64)));\n                __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0);\n                __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17);\n                __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96)));\n                __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0);\n                __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17);\n\n                __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1);\n                __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1);\n                __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1);\n                __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1);\n                __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1);\n                __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1);\n                __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1);\n                __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1);\n\n                // Shuffle pattern one - left side input\n\n                const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160);  //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n                const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160);  //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n                const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160);  //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n                const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160);  //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n                const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160);  //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n                const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160);  //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n                const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160);  //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n                const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160);  //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n                // Shuffle pattern two - left side input\n\n                const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245);  //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n                const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245);  //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n                const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245);  //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n                const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245);  //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n                const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245);  //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n                const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245);  //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n                const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245);  //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n                const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245);  //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n                // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                // Resembles MMLAs into 2x2 matrices in ARM Version\n                const __m512i zero = _mm512_setzero_epi32();\n                __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1);\n                __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1);\n                __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1);\n                __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1);\n                __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2);\n                __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2);\n                __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2);\n                __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2);\n\n                // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n                __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n                __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n                __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n                // Straighten out to make 4 row vectors\n                __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n                __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n                __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n                __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n                // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n                const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptr[b].d), loadMask), 68);\n                const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16);\n\n                // Multiply with appropriate scales and accumulate\n                acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)),   acc_rows[0]);\n                acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)),  acc_rows[1]);\n                acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n            }\n\n            // Store the accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n            }\n        }\n    }\n    if (anc != nc) {\n        xstart = anc/8;\n        y = 0;\n    }\n#endif // __AVX512BW__ && __AVX512DQ__\n\n    // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation\n\n    for (; y < anr / 4; y += 4) {\n        const block_q8_0x4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of eight block_tx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = xstart; x < nc / 8; x++) {\n\n            const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n                const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n                const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32));\n                const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64));\n                const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96));\n\n                // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                // 4-bit -> 8-bit - Sign is maintained\n                const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7)\n                const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7)\n\n                const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15)\n                const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15)\n\n                const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23)\n                const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23)\n\n                const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31)\n                const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31)\n\n                // Shuffle pattern one - right side input\n                const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136);  //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3)\n                const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136);  //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3)\n\n                const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136);  //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11)\n                const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136);  //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11)\n\n                const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136);  //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19)\n                const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136);  //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19)\n\n                const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136);  //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27)\n                const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136);  //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27)\n\n                // Shuffle pattern two - right side input\n\n                const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221);  //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7)\n                const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221);  //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7)\n\n                const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221);  //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15)\n                const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221);  //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15)\n\n                const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221);  //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23)\n                const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221);  //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23)\n\n                const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221);  //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31)\n                const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221);  //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31)\n\n                // Scale values - Load the wight scale values of block_tx8\n                __m256 col_scale_f32;\n                if constexpr (\n                        std::is_same_v<block_tx8, block_q4_0x8> ||\n                        std::is_same_v<block_tx8, block_iq4_nlx8>) {\n                    col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n                } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) {\n                    col_scale_f32 = _mm256_set_ps(\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n                }\n\n                // Process LHS in groups of four\n                for (int rp = 0; rp < 4; rp++) {\n                    // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                    __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs)));\n                    __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0);\n                    __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17);\n                    __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32)));\n                    __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0);\n                    __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17);\n                    __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64)));\n                    __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0);\n                    __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17);\n                    __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96)));\n                    __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0);\n                    __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17);\n\n                    // Shuffle pattern one - left side input\n                    const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160);  //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n                    const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160);  //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n                    const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160);  //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n                    const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160);  //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n                    const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160);  //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n                    const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160);  //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n                    const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160);  //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n                    const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160);  //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n                    // Shuffle pattern two - left side input\n                    const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245);  //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n                    const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245);  //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n                    const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245);  //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n                    const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245);  //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n                    const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245);  //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n                    const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245);  //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n                    const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245);  //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n                    const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245);  //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    // Resembles MMLAs into 2x2 matrices in ARM Version\n                    const __m256i zero = _mm256_setzero_si256();\n                    __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1);\n                    __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1);\n                    __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1);\n                    __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1);\n                    __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2);\n                    __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2);\n                    __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2);\n                    __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2);\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n                    __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n                    __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n                    __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n                    // Straighten out to make 4 row vectors\n                    __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n                    __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n                    __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n                    __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n                    // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n                    const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n                    // Multiply with appropriate scales and accumulate\n                    acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n                    acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n                    acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                    acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32,  255)), acc_rows[rp * 4 + 3]);\n                }\n            }\n\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n            }\n        }\n    }\n\n    // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation\n    for (; y < nr / 4; y ++) {\n        const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb);\n\n        // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n        for (int64_t x = xstart; x < nc / 8; x++) {\n            const block_tx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Load the eight block_q8_0 quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n                const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs));\n                const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32));\n                const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64));\n                const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96));\n\n                // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                // 4-bit -> 8-bit - Sign is maintained\n                const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b));  //B0(0-7) B1(0-7) B4(0-7) B5(0-7)\n                const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b));  //B2(0-7) B3(0-7) B6(0-7) B7(0-7)\n\n                const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b));  //B0(8-15) B1(8-15) B4(8-15) B5(8-15)\n                const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b));  //B2(8-15) B3(8-15) B6(8-15) B7(8-15)\n\n                const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b));  //B0(16-23) B1(16-23) B4(16-23) B5(16-23)\n                const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b));  //B2(16-23) B3(16-23) B6(16-23) B7(16-23)\n\n                const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b));  //B0(24-31) B1(24-31) B4(24-31) B5(24-31)\n                const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b));  //B2(24-31) B3(24-31) B6(24-31) B7(24-31)\n\n                // Shuffle pattern one - right side input\n                const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136);  //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3)\n                const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136);  //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3)\n\n                const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136);  //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11)\n                const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136);  //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11)\n\n                const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136);  //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19)\n                const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136);  //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19)\n\n                const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136);  //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27)\n                const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136);  //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27)\n\n                // Shuffle pattern two - right side input\n\n                const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221);  //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7)\n                const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221);  //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7)\n\n                const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221);  //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15)\n                const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221);  //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15)\n\n                const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221);  //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23)\n                const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221);  //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23)\n\n                const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221);  //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31)\n                const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221);  //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31)\n\n                // Scale values - Load the wight scale values of block_tx8\n                __m256 col_scale_f32;\n                if constexpr (\n                        std::is_same_v<block_tx8, block_q4_0x8> ||\n                        std::is_same_v<block_tx8, block_iq4_nlx8>) {\n                    col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n                } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) {\n                    col_scale_f32 = _mm256_set_ps(\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1]),\n                        GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0]));\n                }\n\n                // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3\n                // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs)));\n                __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0);\n                __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17);\n                __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32)));\n                __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0);\n                __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17);\n                __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64)));\n                __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0);\n                __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17);\n                __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96)));\n                __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0);\n                __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17);\n\n                // Shuffle pattern one - left side input\n\n                const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160);  //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3)\n                const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160);  //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3)\n\n                const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160);  //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11)\n                const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160);  //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11)\n\n                const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160);  //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19)\n                const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160);  //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19)\n\n                const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160);  //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27)\n                const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160);  //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27)\n\n                // Shuffle pattern two - left side input\n\n                const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245);  //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7)\n                const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245);  //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7)\n\n                const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245);  //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15)\n                const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245);  //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15)\n\n                const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245);  //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23)\n                const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245);  //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23)\n\n                const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245);  //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31)\n                const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245);  //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31)\n\n                // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                // Resembles MMLAs into 2x2 matrices in ARM Version\n                const __m256i zero = _mm256_setzero_si256();\n                __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1);\n                __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1);\n                __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1);\n                __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1);\n                __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2);\n                __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2);\n                __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2);\n                __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2);\n\n                // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2);\n                __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2);\n                __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2);\n                __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2);\n\n\n                // Straighten out to make 4 row vectors\n                __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n                __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n                __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n                __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n                // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes\n                const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptr[b].d, loadMask);\n\n                // Multiply with appropriate scales and accumulate\n                acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n                acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n                acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n            }\n\n            // Store the accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]);\n            }\n        }\n    }\n}\n\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n    {\n        // Lookup table to convert signed nibbles to signed bytes\n        __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n        signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n        gemv_q4_b32_8x8_q8_0_lut_avx<block_q4_0x8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n        return;\n    }\n#endif\n\n    ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__AVX2__)\n    // Lookup table to convert signed nibbles to signed bytes\n    __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n    signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n    // Shuffle masks to rearrange delta and scale values to multiply with appropriate scales\n    __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n    __m128i scalemask = _mm_set_epi8(7, 7, 3, 3, 6, 6, 2, 2, 5, 5, 1, 1, 4, 4, 0, 0);\n    // Permute mask used for easier vector processing at later stages\n    __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n    // Mask to extract nibbles from bytes\n    const __m256i m4b = _mm256_set1_epi8(0x0F);\n\n    int64_t b_nb = n / QK_K;\n\n    const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 *)vx;\n    const block_q8_K * a_ptr_start = (const block_q8_K *)vy;\n\n    // Process Q8_K blocks one by one\n    for (int64_t y = 0; y < nr; y++) {\n\n        // Pointers to LHS blocks of block_q8_K format\n        const block_q8_K * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight interleaved block_q4_K structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < nc / 8; x++) {\n\n            // Pointers to RHS blocks\n            const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_row = _mm256_setzero_ps();\n            __m256 acc_min_rows = _mm256_setzero_ps();\n\n            for (int64_t b = 0; b < nb; b++) {\n\n                // Load and convert to FP32 scale from block_q8_K\n                const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d));\n\n                // Load the scale values for the 8 blocks interleaved in block_q4_Kx8\n                // col_scale_f32 rearranged so as to multiply with appropriate quants\n                const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask);\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                __m256i iacc_b = _mm256_setzero_si256();\n                __m256i iacc_min_b = _mm256_setzero_si256();\n\n                const __m256i q8sums = _mm256_loadu_si256((const __m256i * )(a_ptr[b].bsums));\n                __m256i q8s = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(q8sums), _mm256_extracti128_si256(q8sums, 1)));\n                q8s = _mm256_permute2f128_si256(q8s, q8s, 0);\n\n                // Processes two sub blocks from each Q4_K in each iteration\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n\n                    // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n                    // 4-bit -> 8-bit\n                    // Values of the first sub block of eight block_q4_K structures for the sb loop\n                    const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m4b);\n                    const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m4b);\n                    const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m4b);\n                    const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m4b);\n                    const __m256i rhs_vec_0123_02 = _mm256_and_si256(rhs_raw_vec_0123_2, m4b);\n                    const __m256i rhs_vec_4567_02 = _mm256_and_si256(rhs_raw_vec_4567_2, m4b);\n                    const __m256i rhs_vec_0123_03 = _mm256_and_si256(rhs_raw_vec_0123_3, m4b);\n                    const __m256i rhs_vec_4567_03 = _mm256_and_si256(rhs_raw_vec_4567_3, m4b);\n\n                    // Values of the second sub block of eight block_q4_K structures when sb = 1\n                    const __m256i rhs_vec_0123_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b);\n                    const __m256i rhs_vec_4567_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b);\n                    const __m256i rhs_vec_0123_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b);\n                    const __m256i rhs_vec_4567_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b);\n                    const __m256i rhs_vec_0123_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m4b);\n                    const __m256i rhs_vec_4567_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m4b);\n                    const __m256i rhs_vec_0123_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m4b);\n                    const __m256i rhs_vec_4567_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m4b);\n\n                    uint32_t utmp_0[4], utmp_1[4];\n\n                    // Scales and Mins of corresponding sub blocks from different Q8_K structures are stored together\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n                    utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp_0[1] & kmask1;\n                    utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n                    utmp_0[2] = uaux_0;\n                    utmp_0[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n                    utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_1 = utmp_1[1] & kmask1;\n                    utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n                    utmp_1[2] = uaux_1;\n                    utmp_1[0] &= kmask1;\n\n                    // Scales of first sub block in the sb loop\n                    const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n                    __m128i scales_rearrange_0 = _mm_shuffle_epi8(mins_and_scales_0, scalemask);\n                    __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0);\n\n                    // Scales of second sub block in the sb loop\n                    __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n                    __m128i scales_rearrange_1 = _mm_shuffle_epi8(mins_and_scales_1, scalemask);\n                    __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1);\n\n                    // Mins of first and second sub block of Q4_K block are arranged side by side\n                    __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n                    // Load the two sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector\n                    __m256i lhs_vec_00 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 64)));\n                    __m256i lhs_vec_01 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 64)));\n                    __m256i lhs_vec_10 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 64)));\n                    __m256i lhs_vec_11 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 64)));\n\n                    lhs_vec_00 = _mm256_permute2f128_si256(lhs_vec_00, lhs_vec_00, 0);\n                    lhs_vec_01 = _mm256_permute2f128_si256(lhs_vec_01, lhs_vec_01, 0);\n                    lhs_vec_10 = _mm256_permute2f128_si256(lhs_vec_10, lhs_vec_10, 0);\n                    lhs_vec_11 = _mm256_permute2f128_si256(lhs_vec_11, lhs_vec_11, 0);\n\n                    // Dot product done within 32 bit lanes and accumulated in the same vector\n                    // First done for first sub block and then for second sub block in each sb\n                    // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n                    // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n                    // ...........................................................................\n                    // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31)\n\n\n                    __m256i iacc_0 = _mm256_setzero_si256();\n                    __m256i iacc_1 = _mm256_setzero_si256();\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 0)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_00, 85)));\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_00, 170)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_00, 255)));\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_02 ,_mm256_shuffle_epi32(rhs_vec_4567_02, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 0)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_02, 177) ,rhs_vec_4567_02, 170), _mm256_shuffle_epi32(lhs_vec_01, 85)));\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_03 ,_mm256_shuffle_epi32(rhs_vec_4567_03, 177), 170), _mm256_shuffle_epi32(lhs_vec_01, 170)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_03, 177) ,rhs_vec_4567_03, 170), _mm256_shuffle_epi32(lhs_vec_01, 255)));\n\n                    iacc_0 = _mm256_madd_epi16(iacc_0, scales_0);\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 0)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_10, 85)));\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_10, 170)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_10, 255)));\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_12 ,_mm256_shuffle_epi32(rhs_vec_4567_12, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 0)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_12, 177) ,rhs_vec_4567_12, 170), _mm256_shuffle_epi32(lhs_vec_11, 85)));\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_13 ,_mm256_shuffle_epi32(rhs_vec_4567_13, 177), 170), _mm256_shuffle_epi32(lhs_vec_11, 170)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_13, 177) ,rhs_vec_4567_13, 170), _mm256_shuffle_epi32(lhs_vec_11, 255)));\n\n                    iacc_1 = _mm256_madd_epi16(iacc_1, scales_1);\n\n                    // Accumulate the iacc value for one sb\n                    __m256i iacc_sb = _mm256_add_epi32(iacc_0, iacc_1);\n\n                    // Broadcast the bsums of the two sub blocks  of the iteration of Q8_K across the vector\n                    // Multiply-Add with corresponding mins of Q4_Kx8 with bsums\n                    __m256i q8s_sb = _mm256_shuffle_epi32(q8s, 0);\n                    __m256i iacc_min_sb = _mm256_madd_epi16(q8s_sb, mins_01);\n                    q8s = _mm256_bsrli_epi128(q8s, 4);\n\n                    // Accumulate for the complete block\n                    iacc_b = _mm256_add_epi32(iacc_b, iacc_sb);\n                    iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb);\n                }\n\n                // Multiply-Add with scale values for the complete super block\n                acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n                acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows);\n\n            }\n\n            // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n            acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n            _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows));\n        }\n    }\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    ggml_gemv_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n#endif\n}\n\nvoid ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__)\n    __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl));\n    signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n    gemv_q4_b32_8x8_q8_0_lut_avx<block_iq4_nlx8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n    return;\n#endif\n\n    ggml_gemv_iq4_nl_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__)\n    __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4));\n    signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n    gemv_q4_b32_8x8_q8_0_lut_avx<block_mxfp4x8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n    return;\n#endif\n\n    ggml_gemv_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__AVX2__)\n    // Lookup table to convert signed nibbles to signed bytes\n    __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n    signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n    // Shuffle masks to rearrange delta values to multiply with appropriate scales\n    __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0);\n    // Permute mask used for easier vector processing at later stages\n    __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0);\n\n    const __m256i m3b = _mm256_set1_epi8(3);\n    const __m128i m4b_sse = _mm_set1_epi8(0xF);\n\n    //Mask to get appropriate scales\n    __m128i scalemask1 = _mm_set_epi8(14,14,6,6,12,12,4,4,10,10,2,2,8,8,0,0);\n    __m128i scalemask2 = _mm_set_epi8(15,15,7,7,13,13,5,5,11,11,3,3,9,9,1,1);\n\n    int64_t b_nb = n / QK_K;\n\n    const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 *)vx;\n    const block_q8_K * a_ptr_start = (const block_q8_K *)vy;\n\n    // Process Q8_K blocks one by one\n    for (int64_t y = 0; y < nr; y++) {\n\n        // Pointers to LHS blocks of block_q8_K format\n        const block_q8_K * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight interleaved block_q2_K structures at each pass of the loop and perform dot product operation\n        for(int64_t x = 0; x < nc / 8; x++) {\n\n            // Pointers to RHS blocks\n            const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_row = _mm256_setzero_ps();\n            __m256 acc_min_rows = _mm256_setzero_ps();\n\n            for (int64_t b = 0; b < nb; b++) {\n\n                // Load and convert to FP32 delta from block_q8_K\n                const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d));\n\n                // Load the delta values for the 8 blocks interleaved in block_q2_Kx8\n                // col_scale_f32 rearranged so as to multiply with appropriate quants\n                const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask);\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                __m256i iacc_b = _mm256_setzero_si256();\n                __m256i iacc_min_b = _mm256_setzero_si256();\n\n                // Processes eight sub blocks from each Q2_K in each iteration\n                for(int sb = 0; sb < QK_K / 128; sb++) {\n\n                    // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n                    // 2-bit -> 8-bit\n                    // Values of the 0th,2nd,4th,6th sub blocks of eight block_q2_K structures for the sb loop\n                    const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m3b); //B00(0-7) B01(0-7) B02(0-7) B03(0-7)\n                    const __m256i rhs_vec_0123_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 2), m3b); //B20(0-7) B21(0-7) B22(0-7) B23(0-7)\n                    const __m256i rhs_vec_0123_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m3b); //B40(0-7) B41(0-7) B42(0-7) B43(0-7)\n                    const __m256i rhs_vec_0123_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 6), m3b); //B60(0-7) B61(0-7) B62(0-7) B63(0-7)\n\n                    const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m3b); //B04(0-7) B05(0-7) B06(0-7) B07(0-7)\n                    const __m256i rhs_vec_4567_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 2), m3b); //B24(0-7) B25(0-7) B26(0-7) B27(0-7)\n                    const __m256i rhs_vec_4567_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m3b); //B44(0-7) B45(0-7) B46(0-7) B47(0-7)\n                    const __m256i rhs_vec_4567_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 6), m3b); //B64(0-7) B65(0-7) B66(0-7) B67(0-7)\n\n                    const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m3b); //B00(8-15) B01(8-15) B02(8-15) B03(8-15)\n                    const __m256i rhs_vec_0123_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 2), m3b); //B20(8-15) B21(8-15) B22(8-15) B23(8-15)\n                    const __m256i rhs_vec_0123_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m3b); //B40(8-15) B41(8-15) B42(8-15) B43(8-15)\n                    const __m256i rhs_vec_0123_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 6), m3b); //B60(8-15) B61(8-15) B62(8-15) B63(8-15)\n\n                    const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m3b); //B04(8-15) B05(8-15) B06(8-15) B07(8-15)\n                    const __m256i rhs_vec_4567_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 2), m3b); //B24(8-15) B25(8-15) B26(8-15) B27(8-15)\n                    const __m256i rhs_vec_4567_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m3b); //B44(8-15) B45(8-15) B46(8-15) B47(8-15)\n                    const __m256i rhs_vec_4567_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 6), m3b); //B64(8-15) B65(8-15) B66(8-15) B67(8-15)\n\n                    // Values of the 1st,3rd,5th,7th sub blocks of eight block_q2_K structures for the sb loop\n                    const __m256i rhs_vec_0123_10 = _mm256_and_si256(rhs_raw_vec_0123_2, m3b); //B10(0-7) B11(0-7) B12(0-7) B13(0-7)\n                    const __m256i rhs_vec_0123_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 2), m3b); //B30(0-7) B31(0-7) B32(0-7) B33(0-7)\n                    const __m256i rhs_vec_0123_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m3b); //B50(0-7) B51(0-7) B52(0-7) B53(0-7)\n                    const __m256i rhs_vec_0123_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 6), m3b); //B70(0-7) B71(0-7) B72(0-7) B73(0-7)\n\n                    const __m256i rhs_vec_4567_10 = _mm256_and_si256(rhs_raw_vec_4567_2, m3b); //B14(0-7) B15(0-7) B16(0-7) B17(0-7)\n                    const __m256i rhs_vec_4567_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 2), m3b); //B34(0-7) B35(0-7) B36(0-7) B37(0-7)\n                    const __m256i rhs_vec_4567_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m3b); //B54(0-7) B55(0-7) B56(0-7) B57(0-7)\n                    const __m256i rhs_vec_4567_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 6), m3b); //B74(0-7) B75(0-7) B76(0-7) B77(0-7)\n\n                    const __m256i rhs_vec_0123_11 = _mm256_and_si256(rhs_raw_vec_0123_3, m3b); //B10(8-15) B11(8-15) B12(8-15) B13(8-15)\n                    const __m256i rhs_vec_0123_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 2), m3b); //B30(8-15) B31(8-15) B32(8-15) B33(8-15)\n                    const __m256i rhs_vec_0123_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m3b); //B50(8-15) B51(8-15) B52(8-15) B53(8-15)\n                    const __m256i rhs_vec_0123_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 6), m3b); //B70(8-15) B71(8-15) B72(8-15) B73(8-15)\n\n                    const __m256i rhs_vec_4567_11 = _mm256_and_si256(rhs_raw_vec_4567_3, m3b); //B14(8-15) B15(8-15) B16(8-15) B17(8-15)\n                    const __m256i rhs_vec_4567_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 2), m3b); //B34(8-15) B35(8-15) B36(8-15) B37(8-15)\n                    const __m256i rhs_vec_4567_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m3b); //B54(8-15) B55(8-15) B56(8-15) B57(8-15)\n                    const __m256i rhs_vec_4567_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 6), m3b); //B74(8-15) B75(8-15) B76(8-15) B77(8-15)\n\n                    //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n                    //s00 m00  s01 m01   s10 m10  s11 m11  s20 m20  s21 m21   s30 m30  s31 m31  s40 m40  s41 m41   s50 m50  s51 m51  s60 m60  s61 m61   s70 m70  s71 m71\n\n                    const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n                    // Extract scales which is lower half from mins_and_scales\n                    const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n                    const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n                    const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n                    const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n                    // Extract mins which is upper half from mins_and_scales\n                    const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n                    const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n                    const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n                    const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n                    // Scales of sub blocks in the sb loop\n                    // Scales of the 0th sub block from each super block\n                    __m128i scales_rearrange_0 = _mm_shuffle_epi8(scales_01, scalemask1);\n                    __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0);\n\n                    // Scales of the 1st sub block from each super block\n                    __m128i scales_rearrange_1 = _mm_shuffle_epi8(scales_01, scalemask2);\n                    __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1);\n\n                    // Scales of the 2nd sub block from each super block\n                    __m128i scales_rearrange_2 = _mm_shuffle_epi8(scales_23, scalemask1);\n                    __m256i scales_2 = _mm256_cvtepu8_epi16(scales_rearrange_2);\n\n                    // Scales of the 3rd sub block from each super block\n                    __m128i scales_rearrange_3 = _mm_shuffle_epi8(scales_23, scalemask2);\n                    __m256i scales_3 = _mm256_cvtepu8_epi16(scales_rearrange_3);\n\n                    // Scales of the 4th sub block from each super block\n                    __m128i scales_rearrange_4 = _mm_shuffle_epi8(scales_45, scalemask1);\n                    __m256i scales_4 = _mm256_cvtepu8_epi16(scales_rearrange_4);\n\n                    // Scales of the 5th sub block from each super block\n                    __m128i scales_rearrange_5 = _mm_shuffle_epi8(scales_45, scalemask2);\n                    __m256i scales_5 = _mm256_cvtepu8_epi16(scales_rearrange_5);\n\n                    // Scales of the 6th sub block from each super block\n                    __m128i scales_rearrange_6 = _mm_shuffle_epi8(scales_67, scalemask1);\n                    __m256i scales_6 = _mm256_cvtepu8_epi16(scales_rearrange_6);\n\n                    // Scales of the 7th sub block from each super block\n                    __m128i scales_rearrange_7 = _mm_shuffle_epi8(scales_67, scalemask2);\n                    __m256i scales_7 = _mm256_cvtepu8_epi16(scales_rearrange_7);\n\n                    // Load the sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector\n                    __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 128)));\n                    __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 128)));\n                    __m256i lhs_vec_2 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 128)));\n                    __m256i lhs_vec_3 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 128)));\n                    __m256i lhs_vec_4 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 64 + sb * 128)));\n                    __m256i lhs_vec_5 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 80 + sb * 128)));\n                    __m256i lhs_vec_6 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 96 + sb * 128)));\n                    __m256i lhs_vec_7 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 112 + sb * 128)));\n\n                    lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0);\n                    lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0);\n                    lhs_vec_2 = _mm256_permute2f128_si256(lhs_vec_2, lhs_vec_2, 0);\n                    lhs_vec_3 = _mm256_permute2f128_si256(lhs_vec_3, lhs_vec_3, 0);\n                    lhs_vec_4 = _mm256_permute2f128_si256(lhs_vec_4, lhs_vec_4, 0);\n                    lhs_vec_5 = _mm256_permute2f128_si256(lhs_vec_5, lhs_vec_5, 0);\n                    lhs_vec_6 = _mm256_permute2f128_si256(lhs_vec_6, lhs_vec_6, 0);\n                    lhs_vec_7 = _mm256_permute2f128_si256(lhs_vec_7, lhs_vec_7, 0);\n\n                    __m256i iacc_0 = _mm256_setzero_si256();\n                    __m256i iacc_1 = _mm256_setzero_si256();\n                    __m256i iacc_2 = _mm256_setzero_si256();\n                    __m256i iacc_3 = _mm256_setzero_si256();\n                    __m256i iacc_4 = _mm256_setzero_si256();\n                    __m256i iacc_5 = _mm256_setzero_si256();\n                    __m256i iacc_6 = _mm256_setzero_si256();\n                    __m256i iacc_7 = _mm256_setzero_si256();\n\n                    // Dot product done within 32 bit lanes and accumulated in the same vector\n                    // First done for 0th sub block and then for seven (1st - 7th) other sub blocks processed for each sb (sb < QK_K/128 loop)                    // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3)\n                    // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7)\n                    // B0(8-11) B4(8-11) B1(8-11) B5(8-11) B2(8-11) B6(8-11) B3(8-11) B7(8-11) with A0(8-11)\n                    // B0(12-15) B4(12-15) B1(12-15) B5(12-15) B2(12-15) B6(12-15) B3(12-15) B7(12-15) with A0(12-15)\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 0)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170), _mm256_shuffle_epi32(lhs_vec_0, 85)));\n\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170), _mm256_shuffle_epi32(lhs_vec_0, 170)));\n                    iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170), _mm256_shuffle_epi32(lhs_vec_0, 255)));\n\n                    iacc_0 = _mm256_madd_epi16(iacc_0, scales_0);\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 0)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170), _mm256_shuffle_epi32(lhs_vec_1, 85)));\n\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170), _mm256_shuffle_epi32(lhs_vec_1, 170)));\n                    iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170), _mm256_shuffle_epi32(lhs_vec_1, 255)));\n\n                    iacc_1 = _mm256_madd_epi16(iacc_1, scales_1);\n\n                    iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_20 ,_mm256_shuffle_epi32(rhs_vec_4567_20, 177), 170), _mm256_shuffle_epi32(lhs_vec_2, 0)));\n                    iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_20, 177) ,rhs_vec_4567_20, 170), _mm256_shuffle_epi32(lhs_vec_2, 85)));\n\n                    iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_21 ,_mm256_shuffle_epi32(rhs_vec_4567_21, 177), 170), _mm256_shuffle_epi32(lhs_vec_2, 170)));\n                    iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_21, 177) ,rhs_vec_4567_21, 170), _mm256_shuffle_epi32(lhs_vec_2, 255)));\n\n                    iacc_2 = _mm256_madd_epi16(iacc_2, scales_2);\n\n                    iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_30 ,_mm256_shuffle_epi32(rhs_vec_4567_30, 177), 170), _mm256_shuffle_epi32(lhs_vec_3, 0)));\n                    iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_30, 177) ,rhs_vec_4567_30, 170), _mm256_shuffle_epi32(lhs_vec_3, 85)));\n\n                    iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_31 ,_mm256_shuffle_epi32(rhs_vec_4567_31, 177), 170), _mm256_shuffle_epi32(lhs_vec_3, 170)));\n                    iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_31, 177) ,rhs_vec_4567_31, 170), _mm256_shuffle_epi32(lhs_vec_3, 255)));\n\n                    iacc_3 = _mm256_madd_epi16(iacc_3, scales_3);\n\n                    iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_40 ,_mm256_shuffle_epi32(rhs_vec_4567_40, 177), 170), _mm256_shuffle_epi32(lhs_vec_4, 0)));\n                    iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_40, 177) ,rhs_vec_4567_40, 170), _mm256_shuffle_epi32(lhs_vec_4, 85)));\n\n                    iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_41 ,_mm256_shuffle_epi32(rhs_vec_4567_41, 177), 170), _mm256_shuffle_epi32(lhs_vec_4, 170)));\n                    iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_41, 177) ,rhs_vec_4567_41, 170), _mm256_shuffle_epi32(lhs_vec_4, 255)));\n\n                    iacc_4 = _mm256_madd_epi16(iacc_4, scales_4);\n\n                    iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_50 ,_mm256_shuffle_epi32(rhs_vec_4567_50, 177), 170), _mm256_shuffle_epi32(lhs_vec_5, 0)));\n                    iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_50, 177) ,rhs_vec_4567_50, 170), _mm256_shuffle_epi32(lhs_vec_5, 85)));\n\n                    iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_51 ,_mm256_shuffle_epi32(rhs_vec_4567_51, 177), 170), _mm256_shuffle_epi32(lhs_vec_5, 170)));\n                    iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_51, 177) ,rhs_vec_4567_51, 170), _mm256_shuffle_epi32(lhs_vec_5, 255)));\n\n                    iacc_5 = _mm256_madd_epi16(iacc_5, scales_5);\n\n                    iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_60 ,_mm256_shuffle_epi32(rhs_vec_4567_60, 177), 170), _mm256_shuffle_epi32(lhs_vec_6, 0)));\n                    iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_60, 177) ,rhs_vec_4567_60, 170), _mm256_shuffle_epi32(lhs_vec_6, 85)));\n\n                    iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_61 ,_mm256_shuffle_epi32(rhs_vec_4567_61, 177), 170), _mm256_shuffle_epi32(lhs_vec_6, 170)));\n                    iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_61, 177) ,rhs_vec_4567_61, 170), _mm256_shuffle_epi32(lhs_vec_6, 255)));\n\n                    iacc_6 = _mm256_madd_epi16(iacc_6, scales_6);\n\n                    iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_70 ,_mm256_shuffle_epi32(rhs_vec_4567_70, 177), 170), _mm256_shuffle_epi32(lhs_vec_7, 0)));\n                    iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_70, 177) ,rhs_vec_4567_70, 170), _mm256_shuffle_epi32(lhs_vec_7, 85)));\n\n                    iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_71 ,_mm256_shuffle_epi32(rhs_vec_4567_71, 177), 170), _mm256_shuffle_epi32(lhs_vec_7, 170)));\n                    iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_71, 177) ,rhs_vec_4567_71, 170), _mm256_shuffle_epi32(lhs_vec_7, 255)));\n\n                    iacc_7 = _mm256_madd_epi16(iacc_7, scales_7);\n\n                    // Accumulate the iacc value for one sb\n                    __m256i iacc_sb = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_0, iacc_1), _mm256_add_epi32(iacc_2, iacc_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_4, iacc_5), _mm256_add_epi32(iacc_6, iacc_7)));\n\n                    __m128i q8sums = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + sb * 8));\n                    __m256i q8s = _mm256_castsi128_si256(q8sums);\n                    q8s= _mm256_permute2f128_si256(q8s, q8s, 0);\n\n                    // Broadcast the bsums of the two corresponding subblocks of q8_k\n                    // Multiply-Add with corresponding mins of Q2_Kx8 with bsums\n                    __m256i iacc_min_sb_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 0), mins_01);\n                    __m256i iacc_min_sb_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 85), mins_23);\n                    __m256i iacc_min_sb_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 170), mins_45);\n                    __m256i iacc_min_sb_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 255), mins_67);\n\n                    __m256i iacc_min_sb = _mm256_add_epi32(_mm256_add_epi32(iacc_min_sb_01, iacc_min_sb_23), _mm256_add_epi32(iacc_min_sb_45,iacc_min_sb_67));\n\n                    // Accumulate for the complete block\n                    iacc_b = _mm256_add_epi32(iacc_b, iacc_sb);\n                    iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb);\n                }\n\n                //Multiply-Add with scale values for complete super block\n                acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row);\n                acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows);\n            }\n            // Accumulated output values permuted so as to be stored in appropriate order post accumulation\n            acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask);\n            _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows));\n        }\n    }\n#else\n\n    ggml_gemv_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n\n#endif\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n    {\n        // Lookup table to convert signed nibbles to signed bytes\n        __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0));\n        signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n        gemm_q4_b32_8x8_q8_0_lut_avx<block_q4_0x8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n        return;\n    }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n    ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n    const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 * ) vx;\n    const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy;\n    int64_t b_nb = n / QK_K;\n    int64_t y = 0;\n\n    // Mask to mask out nibbles from packed bytes\n    const __m256i m4b = _mm256_set1_epi8(0x0F);\n    // Permute mask used for easier vector processing at later stages\n    __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n    int64_t xstart = 0;\n    int anr = nr - nr % 16;; // Used to align nr with boundary of 16\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n    int anc = nc - nc % 16; // Used to align nc with boundary of 16\n    // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n    const __m512i m4bexpanded = _mm512_set1_epi8(0x0F);\n    //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n    for (; y < anr / 4; y += 4) {\n\n        const block_q8_Kx4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n            const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            __m512 acc_min_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_min_rows[i] = _mm512_setzero_ps();\n            }\n\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n                // Scale values - Load the sixteen scale values from two block_q4_kx8 structures\n                const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n                // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures\n                const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n                // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                    const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n                    const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n                    const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n                    const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                    const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                    const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n                    const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n                    //4-bit -> 8-bit\n                    const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n                    const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n                    const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n                    const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n\n                    const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23)\n                    const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23)\n                    const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31)\n                    const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31)\n\n                    const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n                    const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n                    const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n                    const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n                    const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23)\n                    const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23)\n                    const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31)\n                    const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31)\n\n                    // Shuffle pattern one - right side input\n                    const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n                    const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n                    const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n                    const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19)\n                    const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19)\n                    const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27)\n                    const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27)\n\n                    const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n                    const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n                    const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n                    const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n                    const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19)\n                    const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19)\n                    const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27)\n                    const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27)\n\n                    // Shuffle pattern two - right side input\n                    const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n                    const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n                    const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n                    const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n                    const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23)\n                    const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23)\n                    const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31)\n                    const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31)\n\n                    const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n                    const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n                    const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n                    const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n                    const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23)\n                    const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23)\n                    const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31)\n                    const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31)\n\n                    uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4];\n\n                    // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12);\n                    utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_00 = utmp_00[1] & kmask1;\n                    utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4);\n                    utmp_00[2] = uaux_00;\n                    utmp_00[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12);\n                    utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_01 = utmp_01[1] & kmask1;\n                    utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4);\n                    utmp_01[2] = uaux_01;\n                    utmp_01[0] &= kmask1;\n\n                    memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12);\n                    utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_10 = utmp_10[1] & kmask1;\n                    utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4);\n                    utmp_10[2] = uaux_10;\n                    utmp_10[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12);\n                    utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_11 = utmp_11[1] & kmask1;\n                    utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4);\n                    utmp_11[2] = uaux_11;\n                    utmp_11[0] &= kmask1;\n\n                    // Scales of first sub block in the sb loop\n                    const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]);\n                    const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n                    // Scales of second sub block in the sb loop\n                    const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]);\n                    const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n                    // Mins of first and second sub block of Q4_K block are arranged side by side\n                    const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78), _mm256_shuffle_epi32(mins_and_scales_1, 78)));\n\n                    const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n                    for (int rp = 0; rp < 4; rp++) {\n\n                        // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                        // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n                        __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n                        __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n                        __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n                        __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n                        __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0);\n                        __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17);\n                        __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0);\n                        __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17);\n                        __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n                        __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n                        __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n                        __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n                        __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0);\n                        __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17);\n                        __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb)));\n                        __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0);\n                        __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17);\n\n                        __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n                        __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n                        __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n                        __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n                        __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1);\n                        __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1);\n                        __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1);\n                        __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1);\n\n                        __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n                        __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n                        __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n                        __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n                        __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1);\n                        __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1);\n                        __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1);\n                        __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1);\n\n                        // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n                        __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb)));\n                        __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n                        lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0);\n                        __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1);\n\n                        // Shuffle pattern one - left side input\n                        const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                        const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n                        const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                        const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n                        const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n                        const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19)\n                        const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n                        const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27)\n\n                        const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                        const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n                        const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                        const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n                        const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n                        const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19)\n                        const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n                        const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27)\n\n                        const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                        const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n                        const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                        const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n                        const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n                        const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23)\n                        const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n                        const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31)\n\n                        const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                        const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n                        const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                        const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n                        const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n                        const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23)\n                        const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n                        const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31)\n\n                        // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                        __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1));\n                        __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1));\n                        __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1));\n                        __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1));\n                        __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1));\n                        __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1));\n                        __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1));\n                        __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1));\n\n                        __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2));\n                        __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2));\n                        __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2));\n                        __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2));\n                        __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2));\n                        __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2));\n                        __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2));\n                        __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2));\n\n                        // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                        __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                        __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                        __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                        __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                        __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                        __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                        __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                        __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                        iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n                        iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n                        iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n                        iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n                        iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n                        iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n                        iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n                        iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n                        // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n                        __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78), iacc_mat_01_0);\n                        __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78), iacc_mat_11_0);\n                        __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78), iacc_mat_01_1);\n                        __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78), iacc_mat_11_1);\n\n                        __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1);\n                        __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1);\n                        __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1);\n                        __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n                        // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                        const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n                        const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n                        const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n                        // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                        acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n                        acc_rows[rp * 4  + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n                        acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                        acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n                        __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0), mins_01);\n                        __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85), mins_01);\n                        __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170), mins_01);\n                        __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255), mins_01);\n\n                        acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n                        acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n                        acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n                        acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n                    }\n                }\n            }\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n\n    for (; y < nr / 4; y++) {\n\n        const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n            const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            __m512 acc_min_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_min_rows[i] = _mm512_setzero_ps();\n            }\n\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n                // Scale values - Load the sixteen scale values from two block_q4_kx8 structures\n                const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n                // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures\n                const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n                // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                    const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n                    const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n                    const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n                    const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                    const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                    const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n                    const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n                    //4-bit -> 8-bit\n                    const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n                    const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n                    const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n                    const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n\n                    const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23)\n                    const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23)\n                    const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31)\n                    const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31)\n\n                    const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n                    const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n                    const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n                    const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n                    const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23)\n                    const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23)\n                    const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31)\n                    const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31)\n\n                    // Shuffle pattern one - right side input\n                    const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n                    const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n                    const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n                    const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19)\n                    const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19)\n                    const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27)\n                    const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27)\n\n                    const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n                    const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n                    const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n                    const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n                    const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19)\n                    const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19)\n                    const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27)\n                    const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27)\n\n                    // Shuffle pattern two - right side input\n                    const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n                    const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n                    const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n                    const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n                    const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23)\n                    const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23)\n                    const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31)\n                    const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31)\n\n                    const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n                    const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n                    const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n                    const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n                    const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23)\n                    const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23)\n                    const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31)\n                    const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31)\n\n                    uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4];\n\n                    // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12);\n                    utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_00 = utmp_00[1] & kmask1;\n                    utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4);\n                    utmp_00[2] = uaux_00;\n                    utmp_00[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12);\n                    utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_01 = utmp_01[1] & kmask1;\n                    utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4);\n                    utmp_01[2] = uaux_01;\n                    utmp_01[0] &= kmask1;\n\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12);\n                    utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_10 = utmp_10[1] & kmask1;\n                    utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4);\n                    utmp_10[2] = uaux_10;\n                    utmp_10[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12);\n                    utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_11 = utmp_11[1] & kmask1;\n                    utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4);\n                    utmp_11[2] = uaux_11;\n                    utmp_11[0] &= kmask1;\n\n                    // Scales of first sub block in the sb loop\n                    const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]);\n                    const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n                    // Scales of second sub block in the sb loop\n                    const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]);\n                    const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n                    // Mins of first and second sub block of Q4_K block are arranged side by side\n                    const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78), _mm256_shuffle_epi32(mins_and_scales_1, 78)));\n\n                    const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n                    // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                    __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n                    __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n                    __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n                    __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n                    __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0);\n                    __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17);\n                    __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0);\n                    __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17);\n                    __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n                    __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n                    __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n                    __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n                    __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0);\n                    __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17);\n                    __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb)));\n                    __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0);\n                    __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17);\n\n                    //Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into a 512 bit vector\n                    __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n                    __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n                    __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n                    __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n                    __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1);\n                    __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1);\n                    __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1);\n                    __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1);\n\n                    __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n                    __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n                    __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n                    __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n                    __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1);\n                    __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1);\n                    __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1);\n                    __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1);\n\n                    // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n                    __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb)));\n                    __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n                    lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0);\n                    __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1);\n\n                    // Shuffle pattern one - left side input\n                    const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                    const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n                    const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                    const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n                    const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n                    const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19)\n                    const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n                    const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27)\n\n                    const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                    const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n                    const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                    const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n                    const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n                    const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19)\n                    const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n                    const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27)\n\n                    const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                    const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n                    const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                    const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n                    const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n                    const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23)\n                    const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n                    const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31)\n\n                    const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                    const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n                    const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                    const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n                    const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n                    const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23)\n                    const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n                    const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1));\n                    __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1));\n                    __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1));\n                    __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1));\n                    __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1));\n                    __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1));\n                    __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1));\n                    __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1));\n\n                    __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2));\n                    __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2));\n                    __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2));\n                    __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2));\n                    __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2));\n                    __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2));\n                    __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2));\n                    __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2));\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                    __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                    __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                    __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                    __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                    __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                    __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                    __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                    iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n                    iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n                    iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n                    iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n                    iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n                    iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n                    iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n                    iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n                    // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n                    __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78), iacc_mat_01_0);\n                    __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78), iacc_mat_11_0);\n                    __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78), iacc_mat_01_1);\n                    __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78), iacc_mat_11_1);\n\n                    __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1);\n                    __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1);\n                    __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1);\n                    __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n                    // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                    const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n                    const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n                    const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n                    // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                    acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n                    acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n                    acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                    acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n                    __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0), mins_01);\n                    __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85), mins_01);\n                    __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170), mins_01);\n                    __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255), mins_01);\n\n                    acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n                    acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n                    acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n                    acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n                }\n            }\n            // Store accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n    if (anc != nc) {\n        xstart = anc/8;\n        y = 0;\n    }\n#endif // __AVX512BW__ && __AVX512DQ__\n\n    // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n    for (; y < anr / 4; y += 4) {\n\n        const block_q8_Kx4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = xstart; x < nc / 8; x++) {\n\n            const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            __m256 acc_min_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_min_rows[i] = _mm256_setzero_ps();\n            }\n\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n\n                // Scale values - Load the eight scale values of block_q4_kx8\n                const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n                // dmin values - Load the eight dmin values of block_q4_kx8\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n\n                    // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n                    // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    // 4-bit -> 8-bit\n                    // First sub block of the two sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n                    const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n                    const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n                    const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n                    const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23)\n                    const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23)\n\n                    const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31)\n                    const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31)\n\n                    // Second sub block of the two sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n                    const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n                    const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n                    const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n                    const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23)\n                    const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23)\n\n                    const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31)\n                    const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31)\n\n                    // Shuffle pattern one - right side input\n                    const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n                    const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n                    const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n                    const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n                    const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19)\n                    const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19)\n\n                    const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27)\n                    const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27)\n\n                    const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n                    const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n                    const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n                    const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n                    const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19)\n                    const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19)\n\n                    const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27)\n                    const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27)\n\n\n                    // Shuffle pattern two - right side input\n                    const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n                    const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n                    const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n                    const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n                    const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23)\n                    const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23)\n\n                    const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31)\n                    const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31)\n\n                    const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n                    const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n                    const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n                    const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n                    const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23)\n                    const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23)\n\n                    const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31)\n                    const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31)\n\n                    uint32_t utmp_0[4], utmp_1[4];\n\n                    // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n                    utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp_0[1] & kmask1;\n                    utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n                    utmp_0[2] = uaux_0;\n                    utmp_0[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n                    utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_1 = utmp_1[1] & kmask1;\n                    utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n                    utmp_1[2] = uaux_1;\n                    utmp_1[0] &= kmask1;\n\n                    // Scales of first sub block in the sb loop\n                    const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n                    const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n                    // Scales of second sub block in the sb loop\n                    const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n                    const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n                    // Mins of first and second sub block of Q4_K block are arranged side by side\n                    const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n                    const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n                    const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n                    const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n                    const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n                    for (int rp = 0; rp < 4; rp++) {\n\n                        // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                        // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                        __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb)));\n                        __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n                        __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n                        __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb)));\n                        __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n                        __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n                        __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb)));\n                        __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0);\n                        __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17);\n                        __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb)));\n                        __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0);\n                        __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17);\n                        __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb)));\n                        __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n                        __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n                        __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb)));\n                        __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n                        __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n                        __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb)));\n                        __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0);\n                        __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17);\n                        __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb)));\n                        __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0);\n                        __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17);\n\n                        // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n                        __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb)));\n                        __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n                        lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0);\n\n                        // Shuffle pattern one - left side input\n                        const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                        const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n                        const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160);  //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                        const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160);  //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n                        const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160);  //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n                        const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160);  //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19)\n\n                        const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160);  //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n                        const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27)\n\n                        const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                        const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n                        const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160);  //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                        const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160);  //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n                        const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160);  //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n                        const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160);  //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19)\n\n                        const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n                        const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27)\n\n                        // Shuffle pattern two- left side input\n                        const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                        const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n                        const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                        const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n                        const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n                        const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23)\n\n                        const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n                        const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31)\n\n                        const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                        const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n                        const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                        const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n                        const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n                        const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23)\n\n                        const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n                        const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31)\n\n                        // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                        __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1));\n                        __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1));\n                        __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1));\n                        __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1));\n                        __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1));\n                        __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1));\n                        __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1));\n                        __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1));\n\n                        __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2));\n                        __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2));\n                        __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2));\n                        __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2));\n                        __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2));\n                        __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2));\n                        __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2));\n                        __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2));\n\n                        // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                        __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                        __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                        __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                        __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                        __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                        __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                        __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                        __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                        // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                        iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n                        iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n                        iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n                        iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n                        iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n                        iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n                        iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n                        iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n                        // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n                        __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204);\n                        __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204);\n                        __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204);\n                        __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204);\n                        __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204);\n                        __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204);\n                        __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204);\n                        __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204);\n\n                        __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1);\n                        __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1);\n                        __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1);\n                        __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n                        // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                        const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n                        const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);//GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n                        // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                        acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n                        acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n                        acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                        acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n                        __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01);\n                        __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01);\n                        __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01);\n                        __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01);\n\n                        acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n                        acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n                        acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n                        acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n\n                    }\n                }\n            }\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n    for (; y < nr / 4; y++) {\n\n        const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n        for (int64_t x = xstart; x < nc / 8; x++) {\n\n            const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            __m256 acc_min_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_min_rows[i] = _mm256_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n\n                // Scale values - Load the eight scale values of block_q4_Kx8\n                const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n                // dmin values - Load the eight dmin values of block_q4_Kx8\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 64; sb++) {\n\n                    // Load the eight block_q4_k for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256));\n\n                    // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    // 4-bit -> 8-bit\n                    // First sub block of the two sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n                    const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n                    const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n                    const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n                    const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23)\n                    const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23)\n\n                    const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31)\n                    const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31)\n\n                    // Second sub block of the two sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n                    const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n                    const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n                    const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n                    const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23)\n                    const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23)\n\n                    const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31)\n                    const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31)\n\n                    // Shuffle pattern one - right side input\n                    const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n                    const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n                    const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n                    const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n                    const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19)\n                    const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19)\n\n                    const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27)\n                    const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27)\n\n                    const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n                    const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n                    const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n                    const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n                    const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19)\n                    const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19)\n\n                    const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27)\n                    const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27)\n\n                    // Shuffle pattern two - right side input\n                    const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n                    const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n                    const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n                    const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n                    const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23)\n                    const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23)\n\n                    const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31)\n                    const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31)\n\n                    const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n                    const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n                    const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n                    const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n                    const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23)\n                    const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23)\n\n                    const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31)\n                    const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31)\n\n                    uint32_t utmp_0[4], utmp_1[4];\n\n                    // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together\n                    // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop\n                    memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12);\n                    utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp_0[1] & kmask1;\n                    utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4);\n                    utmp_0[2] = uaux_0;\n                    utmp_0[0] &= kmask1;\n\n                    // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures when sb = 1\n                    memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12);\n                    utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_1 = utmp_1[1] & kmask1;\n                    utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4);\n                    utmp_1[2] = uaux_1;\n                    utmp_1[0] &= kmask1;\n\n                    // Scales of first sub block in the sb loop\n                    const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]);\n                    const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0));\n\n                    // Scales of second sub block in the sb loop\n                    const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]);\n                    const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1));\n\n                    // Mins of first and second sub block of Q4_K block are arranged side by side\n                    const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78), _mm_shuffle_epi32(mins_and_scales_1, 78)));\n\n                    const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n                    const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n                    const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n                    const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n                    // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                    __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb)));\n                    __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n                    __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n                    __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb)));\n                    __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n                    __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n                    __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb)));\n                    __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0);\n                    __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17);\n                    __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb)));\n                    __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0);\n                    __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17);\n                    __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb)));\n                    __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n                    __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n                    __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb)));\n                    __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n                    __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n                    __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb)));\n                    __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0);\n                    __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17);\n                    __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb)));\n                    __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0);\n                    __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17);\n\n                    // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks\n                    __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb)));\n                    __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)));\n                    lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0);\n\n                    // Shuffle pattern one - left side input\n                    const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                    const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n                    const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160);  //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                    const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160);  //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n                    const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160);  //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19)\n                    const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160);  //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19)\n\n                    const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160);  //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27)\n                    const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27)\n\n                    const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                    const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n                    const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160);  //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                    const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160);  //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n                    const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160);  //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19)\n                    const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160);  //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19)\n\n                    const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27)\n                    const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27)\n\n                    // Shuffle pattern two- left side input\n                    const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                    const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n                    const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                    const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n                    const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23)\n                    const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23)\n\n                    const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31)\n                    const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31)\n\n                    const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                    const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n                    const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                    const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n                    const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23)\n                    const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23)\n\n                    const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31)\n                    const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1));\n                    __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1));\n                    __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1));\n                    __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1));\n                    __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1));\n                    __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1));\n                    __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1));\n                    __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1));\n\n                    __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2));\n                    __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2));\n                    __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2));\n                    __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2));\n                    __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2));\n                    __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2));\n                    __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2));\n                    __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2));\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                    __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                    __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                    __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                    __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                    __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                    __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                    __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n                    iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n                    iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n                    iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n                    iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n                    iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n                    iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n                    iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n                    // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step)\n                    __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204);\n                    __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204);\n                    __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204);\n                    __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204);\n                    __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204);\n                    __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204);\n                    __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204);\n                    __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204);\n\n                    __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1);\n                    __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1);\n                    __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1);\n                    __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1);\n\n                    // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                    const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n                    const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); //GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask);\n\n                    // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                    acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n                    acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n                    acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                    acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n                    __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0), mins_01);\n                    __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85), mins_01);\n                    __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170), mins_01);\n                    __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255), mins_01);\n\n                    acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n                    acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n                    acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n                    acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n                }\n            }\n\n            // Store the accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n\n#else\n    UNUSED(kmask1);\n    UNUSED(kmask2);\n    UNUSED(kmask3);\n    ggml_gemm_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n#endif\n}\n\nvoid ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n    {\n        __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl));\n        signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n        gemm_q4_b32_8x8_q8_0_lut_avx<block_iq4_nlx8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n        return;\n    }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n    ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n#if defined(__AVX2__) || defined(__AVX512F__)\n    {\n        __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4));\n        signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0);\n\n        gemm_q4_b32_8x8_q8_0_lut_avx<block_mxfp4x8>(n, s, bs, vx, vy, nr, nc, signextendlut);\n\n        return;\n    }\n#endif // defined(__AVX2__) || defined(__AVX512F__)\n\n    ggml_gemm_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n#if defined(__AVX2__) || defined(__AVX512F__)\n    const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 * ) vx;\n    const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy;\n    int64_t b_nb = n / QK_K;\n    int64_t y = 0;\n\n    // Permute mask used for easier vector processing at later stages\n    __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4);\n    int64_t xstart = 0;\n    int anr = nr - nr % 16; // Used to align nr with boundary of 16\n\n    // Mask to convert 2 bit and 4 bit values into a bytes\n    const __m256i m3b = _mm256_set1_epi8(3);\n    const __m128i m4b_sse = _mm_set1_epi8(0xF);\n\n    //Mask to get appropriate scales\n    __m128i scalesmask1_sse = _mm_set_epi8(14,14,12,12,10,10,8,8,6,6,4,4,2,2,0,0);\n    __m128i scalesmask2_sse = _mm_set_epi8(15,15,13,13,11,11,9,9,7,7,5,5,3,3,1,1);\n\n    __m256i scalesmask1 = _mm256_castsi128_si256(scalesmask1_sse);\n    scalesmask1 = _mm256_permute2f128_si256(scalesmask1, scalesmask1, 0);\n    __m256i scalesmask2 = _mm256_castsi128_si256(scalesmask2_sse);\n    scalesmask2 = _mm256_permute2f128_si256(scalesmask2, scalesmask2, 0);\n\n#if defined(__AVX512BW__) && defined(__AVX512DQ__)\n\n    int anc = nc - nc % 16; // Used to align nc with boundary of 16\n\n    // Mask to mask out nibbles from packed bytes\n    const __m256i m4b = _mm256_set1_epi8(0x0F);\n    // Mask to mask out nibbles from packed bytes expanded to 512 bit length\n    const __m512i m3bexpanded = _mm512_set1_epi8(3);\n    //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n    for (; y < anr / 4; y += 4) {\n\n        const block_q8_Kx4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n            const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            __m512 acc_min_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_min_rows[i] = _mm512_setzero_ps();\n            }\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n                // Delta values - Load the sixteen scale values from two block_q2_kx8 structures\n                const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n                // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures\n                const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n                // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 128; sb++) {\n\n                    // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                    const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n                    const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n                    const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n                    const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                    const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                    const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n                    const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n                    //2-bit -> 8-bit\n                    const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n                    const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n                    const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n                    const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n                    const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n                    const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n                    const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n                    const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n                    const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7)\n                    const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7)\n\n                    const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15)\n                    const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15)\n\n                    const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7)\n                    const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7)\n\n                    const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15)\n                    const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15)\n\n                    const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7)\n                    const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7)\n\n                    const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15)\n                    const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15)\n\n                    const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7)\n                    const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7)\n\n                    const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15)\n                    const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15)\n\n                    const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7)\n                    const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7)\n\n                    const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15)\n                    const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15)\n\n                    const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7)\n                    const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7)\n\n                    const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15)\n                    const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15)\n\n                    const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n                    const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n\n                    const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n\n                    const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n                    const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n\n                    const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n                    const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n\n                    const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3)\n                    const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3)\n\n                    const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11)\n                    const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11)\n\n                    const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3)\n                    const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3)\n\n                    const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11)\n                    const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11)\n\n                    const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3)\n                    const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3)\n\n                    const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11)\n                    const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11)\n\n                    const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3)\n                    const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3)\n\n                    const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11)\n                    const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11)\n\n                    const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3)\n                    const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3)\n\n                    const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11)\n                    const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11)\n\n                    const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3)\n                    const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3)\n\n                    const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11)\n\n                    const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n                    const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n\n                    const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n                    const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n\n                    const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n                    const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n\n                    const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n                    const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n\n                    const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7)\n                    const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7)\n\n                    const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15)\n                    const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15)\n\n                    const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7)\n                    const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7)\n\n                    const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15)\n                    const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15)\n\n                    const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7)\n                    const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7)\n\n                    const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15)\n                    const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15)\n\n                    const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7)\n                    const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7)\n\n                    const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15)\n                    const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15)\n\n                    const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7)\n                    const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7)\n\n                    const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15)\n                    const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15)\n\n                    const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7)\n                    const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7)\n\n                    const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15)\n                    const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15)\n\n                    //notation:superblock subblock\n                    //s00 m00  s01 m01   s10 m10  s11 m11  s20 m20  s21 m21   s30 m30  s31 m31  s40 m40  s41 m41   s50 m50  s51 m51  s60 m60  s61 m61   s70 m70  s71 m71\n\n                    const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64));\n\n                    const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64));\n\n                    // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n                    const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1);\n                    const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1);\n                    const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1);\n                    const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1);\n\n                    // Extract scales which is lower half from mins_and_scales\n                    const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b);\n                    const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b);\n                    const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b);\n                    const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b);\n\n                    // Extract mins which is upper half from mins_and_scales\n                    const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b));\n                    const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b));\n                    const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b));\n                    const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b));\n\n                    const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask1));\n                    const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask2));\n                    const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask1));\n                    const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask2));\n                    const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask1));\n                    const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask2));\n                    const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask1));\n                    const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask2));\n\n                    const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238);\n\n\n                    for (int rp = 0; rp < 4; rp++) {\n\n                        // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                        // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector\n                        __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n                        __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n                        __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n                        __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n                        __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n                        __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n                        __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n                        __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n                        __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0);\n                        __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17);\n                        __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0);\n                        __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17);\n                        __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0);\n                        __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17);\n                        __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0);\n                        __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17);\n\n                        __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0);\n                        __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17);\n                        __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0);\n                        __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17);\n                        __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0);\n                        __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17);\n                        __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0);\n                        __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17);\n                        __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0);\n                        __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17);\n                        __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0);\n                        __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17);\n                        __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0);\n                        __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17);\n                        __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb)));\n                        __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0);\n                        __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17);\n\n\n                        __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n                        __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n                        __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n                        __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n\n                        __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n                        __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n                        __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n                        __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n\n                        __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1);\n                        __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1);\n                        __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1);\n                        __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1);\n\n                        __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1);\n                        __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1);\n                        __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1);\n                        __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1);\n\n                        __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1);\n                        __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1);\n                        __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1);\n                        __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1);\n\n                        __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1);\n                        __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1);\n                        __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1);\n                        __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1);\n\n                        __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1);\n                        __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1);\n                        __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1);\n                        __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1);\n\n                        __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1);\n                        __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1);\n                        __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1);\n                        __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1);\n\n                        // Bsums are loaded for the different Q8_K blocks\n                        __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb)));\n                        __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb));\n                        __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb)));\n                        __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb));\n\n                        __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n                        __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1);\n                        __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n                        __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1);                        __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n                        __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1);\n                        __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n                        __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1);\n\n                        // Shuffle pattern one - left side input\n                        const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                        const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n\n                        const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                        const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n\n                        const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                        const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n\n                        const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                        const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n\n                        const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n                        const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3)\n\n                        const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n                        const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11)\n\n                        const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n                        const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3)\n\n                        const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n                        const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11)\n\n                        const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n                        const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3)\n\n                        const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n                        const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11)\n\n                        const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n                        const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3)\n\n                        const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n                        const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11)\n\n                        const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n                        const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3)\n\n                        const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n                        const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11)\n\n                        const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n                        const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3)\n\n                        const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n                        const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11)\n\n                        const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                        const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n\n                        const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                        const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n\n                        const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                        const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n\n                        const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                        const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n\n                        const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n                        const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7)\n\n                        const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n                        const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15)\n\n                        const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n                        const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7)\n\n                        const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n                        const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15)\n\n                        const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n                        const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7)\n\n                        const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n                        const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15)\n\n                        const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n                        const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7)\n\n                        const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n                        const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15)\n\n                        const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n                        const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7)\n\n                        const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n                        const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15)\n\n                        const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n                        const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7)\n\n                        const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n                        const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15)\n\n                        // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                        __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1));\n                        __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1));\n\n                        __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1));\n                        __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1));\n\n                        __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1));\n                        __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1));\n\n                        __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1));\n                        __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1));\n\n                        __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1));\n                        __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1));\n\n                        __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1));\n                        __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1));\n\n                        __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1));\n                        __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1));\n\n                        __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1));\n                        __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1));\n\n                        __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1));\n                        __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1));\n\n                        __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1));\n                        __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1));\n\n                        __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1));\n                        __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1));\n\n                        __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1));\n                        __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1));\n\n                        __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1));\n                        __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1));\n\n                        __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1));\n                        __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1));\n\n                        __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1));\n                        __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1));\n\n                        __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1));\n                        __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1));\n\n\n                        __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2));\n                        __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2));\n\n                        __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2));\n                        __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2));\n\n                        __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2));\n                        __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2));\n\n                        __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2));\n                        __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2));\n\n                        __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2));\n                        __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2));\n\n                        __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2));\n                        __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2));\n\n                        __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2));\n                        __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2));\n\n                        __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2));\n                        __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2));\n\n                        __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2));\n                        __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2));\n\n                        __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2));\n                        __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2));\n\n                        __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2));\n                        __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2));\n\n                        __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2));\n                        __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2));\n\n                        __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2));\n                        __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2));\n\n                        __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2));\n                        __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2));\n\n                        __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2));\n                        __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2));\n\n                        __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2));\n                        __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2));\n\n                        // Combine results from both shuffle patterns for each output block\n                        __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                        __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                        __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                        __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                        __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                        __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                        __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                        __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                        __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n                        __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n                        __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n                        __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n                        __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n                        __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n                        __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n                        __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n                        __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n                        __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n                        __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n                        __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n                        __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n                        __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n                        __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n                        __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n                        __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n                        __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n                        __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n                        __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n                        __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n                        __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n                        __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n                        __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n                        // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                        iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n                        iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n                        iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n                        iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n                        iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n                        iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n                        iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n                        iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n                        iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2);\n                        iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2);\n                        iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2);\n                        iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2);\n\n                        iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3);\n                        iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3);\n                        iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3);\n                        iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3);\n\n                        iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4);\n                        iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4);\n                        iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4);\n                        iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4);\n\n                        iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5);\n                        iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5);\n                        iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5);\n                        iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5);\n\n                        iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6);\n                        iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6);\n                        iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6);\n                        iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6);\n\n                        iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7);\n                        iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7);\n                        iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7);\n                        iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7);\n\n                        __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n                        __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n                        __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n                        __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n                        // Straighten out to make 4 row vectors\n                        __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n                        __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n                        __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n                        // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                        const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n                        const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n                        const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n                        // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                        acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n                        acc_rows[rp * 4  + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n                        acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                        acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n                        // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n                        __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0), mins_01);\n                        __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170), mins_01);\n                        __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0), mins_01);\n                        __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170), mins_01);\n\n                        __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85), mins_23);\n                        __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255), mins_23);\n                        __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85), mins_23);\n                        __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255), mins_23);\n\n                        __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0), mins_45);\n                        __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170), mins_45);\n                        __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0), mins_45);\n                        __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170), mins_45);\n\n                        __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85), mins_67);\n                        __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255), mins_67);\n                        __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85), mins_67);\n                        __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255), mins_67);\n\n                        __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n                        __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n                        __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n                        __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n                        acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n                        acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n                        acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n                        acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n                    }\n                }\n            }\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n\n    for (; y < nr / 4; y ++) {\n\n        const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = 0; x < anc / 8; x += 2) {\n\n            const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb);\n            const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb);\n\n            // Master FP accumulators\n            __m512 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm512_setzero_ps();\n            }\n\n            __m512 acc_min_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_min_rows[i] = _mm512_setzero_ps();\n            }\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n                // Delta values - Load the sixteen scale values from two block_q2_kx8 structures\n                const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d);\n\n                // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures\n                const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin);\n\n                // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 128; sb++) {\n\n                    // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256));\n\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240);\n                    const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240);\n                    const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240);\n                    const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240);\n\n                    const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1);\n                    const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1);\n\n                    const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1);\n                    const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1);\n                    const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1);\n\n                    //2-bit -> 8-bit\n                    const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7)\n                    const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7)\n                    const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15)\n                    const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15)\n                    const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7)\n                    const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7)\n                    const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15)\n                    const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15)\n\n                    const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7)\n                    const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7)\n\n                    const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15)\n                    const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15)\n\n                    const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7)\n                    const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7)\n\n                    const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15)\n                    const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15)\n\n                    const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7)\n                    const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7)\n\n                    const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15)\n                    const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15)\n\n                    const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7)\n                    const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7)\n\n                    const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15)\n                    const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15)\n\n                    const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7)\n                    const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7)\n\n                    const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15)\n                    const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15)\n\n                    const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7)\n                    const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7)\n\n                    const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15)\n                    const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15)\n\n                    const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3)\n                    const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3)\n\n                    const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11)\n\n                    const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3)\n                    const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3)\n\n                    const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11)\n                    const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11)\n\n                    const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3)\n                    const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3)\n\n                    const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11)\n                    const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11)\n                    const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3)\n                    const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3)\n\n                    const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11)\n                    const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11)\n\n                    const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3)\n                    const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3)\n\n                    const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11)\n                    const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11)\n\n                    const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3)\n                    const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3)\n\n                    const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11)\n                    const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11)\n\n                    const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3)\n                    const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3)\n\n                    const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11)\n                    const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11)\n\n                    const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3)\n                    const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3)\n\n                    const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11)\n                    const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11)\n\n                    const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7)\n                    const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7)\n\n                    const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15)\n                    const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15)\n\n                    const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7)\n                    const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7)\n\n                    const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15)\n                    const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15)\n\n                    const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7)\n                    const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7)\n\n                    const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15)\n                    const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15)\n\n                    const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7)\n                    const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7)\n\n                    const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15)\n                    const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15)\n\n                    const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7)\n                    const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7)\n\n                    const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15)\n                    const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15)\n\n                    const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7)\n                    const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7)\n\n                    const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15)\n                    const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15)\n\n                    const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7)\n                    const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7)\n\n                    const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15)\n                    const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15)\n\n                    const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7)\n                    const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7)\n\n                    const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15)\n                    const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15)\n\n                    //notation:superblock subblock\n                    //s00 m00  s01 m01   s10 m10  s11 m11  s20 m20  s21 m21   s30 m30  s31 m31  s40 m40  s41 m41   s50 m50  s51 m51  s60 m60  s61 m61   s70 m70  s71 m71\n\n                    const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64));\n\n                    const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64));\n\n                    // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n                    const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1);\n                    const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1);\n                    const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1);\n                    const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1);\n\n                    // Extract scales which is lower half from mins_and_scales\n                    const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b);\n                    const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b);\n                    const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b);\n                    const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b);\n\n                    // Extract mins which is upper half from mins_and_scales\n                    const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b));\n                    const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b));\n                    const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b));\n                    const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b));\n\n                    const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask1));\n                    const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask2));\n                    const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask1));\n                    const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask2));\n                    const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask1));\n                    const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask2));\n                    const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask1));\n                    const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask2));\n\n                    const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238);\n\n                    const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68);\n                    const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238);\n\n                    // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                    __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0);\n                    __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17);\n                    __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0);\n                    __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17);\n                    __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0);\n                    __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17);\n                    __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0);\n                    __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17);\n                    __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0);\n                    __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17);\n                    __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0);\n                    __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17);\n                    __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0);\n                    __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17);\n                    __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0);\n                    __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17);\n\n                    __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0);\n                    __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17);\n                    __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0);\n                    __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17);\n                    __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0);\n                    __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17);\n                    __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0);\n                    __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17);\n                    __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0);\n                    __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17);\n                    __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0);\n                    __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17);\n                    __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0);\n                    __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17);\n                    __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb)));\n                    __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0);\n                    __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17);\n\n                    __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1);\n                    __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1);\n                    __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1);\n                    __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1);\n\n                    __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1);\n                    __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1);\n                    __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1);\n                    __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1);\n\n                    __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1);\n                    __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1);\n                    __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1);\n                    __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1);\n\n                    __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1);\n                    __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1);\n                    __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1);\n                    __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1);\n\n                    __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1);\n                    __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1);\n                    __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1);\n                    __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1);\n\n                    __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1);\n                    __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1);\n                    __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1);\n                    __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1);\n\n                    __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1);\n                    __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1);\n                    __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1);\n                    __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1);\n\n                    __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1);\n                    __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1);\n                    __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1);\n                    __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1);\n\n                    // Bsums are loaded for the different Q8_K blocks\n                    __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb)));\n                    __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb));\n                    __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb)));\n                    __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb));\n\n                    __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n                    __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1);\n                    __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n                    __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1);\n                    __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n                    __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1);\n                    __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n                    __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1);\n\n                    // Shuffle pattern one - left side input\n                    const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                    const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3)\n\n                    const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                    const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11)\n\n                    const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                    const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3)\n\n                    const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                    const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11)\n\n                    const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n                    const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3)\n\n                    const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n                    const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11)\n\n                    const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n                    const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3)\n\n                    const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n                    const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11)\n\n                    const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n                    const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3)\n\n                    const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n                    const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11)\n\n                    const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n                    const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3)\n\n                    const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n                    const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11)\n\n                    const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n                    const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3)\n\n                    const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n                    const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11)\n\n                    const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n                    const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3)\n\n                    const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n                    const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11)\n\n                    const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                    const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7)\n\n                    const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                    const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15)\n\n                    const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                    const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7)\n\n                    const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                    const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15)\n\n                    const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n                    const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7)\n\n                    const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n                    const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15)\n\n                    const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n                    const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7)\n\n                    const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n                    const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15)\n\n                    const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n                    const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7)\n\n                    const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n                    const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15)\n\n                    const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n                    const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7)\n\n                    const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n                    const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15)\n\n                    const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n                    const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7)\n\n                    const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n                    const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15)\n\n                    const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n                    const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7)\n\n                    const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n                    const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1));\n                    __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1));\n\n                    __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1));\n                    __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1));\n\n                    __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1));\n                    __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1));\n\n                    __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1));\n                    __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1));\n\n                    __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1));\n                    __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1));\n\n                    __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1));\n                    __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1));\n\n                    __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1));\n                    __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1));\n\n                    __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1));\n                    __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1));\n\n                    __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1));\n                    __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1));\n\n                    __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1));\n                    __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1));\n\n                    __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1));\n                    __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1));\n\n                    __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1));\n                    __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1));\n\n                    __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1));\n                    __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1));\n\n                    __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1));\n                    __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1));\n\n                    __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1));\n                    __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1));\n\n                    __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1));\n                    __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1));\n\n\n                    __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2));\n                    __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2));\n\n                    __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2));\n                    __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2));\n\n                    __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2));\n                    __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2));\n\n                    __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2));\n                    __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2));\n\n                    __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2));\n                    __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2));\n\n                    __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2));\n                    __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2));\n\n                    __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2));\n                    __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2));\n\n                    __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2));\n                    __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2));\n\n                    __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2));\n                    __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2));\n\n                    __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2));\n                    __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2));\n\n                    __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2));\n                    __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2));\n\n                    __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2));\n                    __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2));\n\n                    __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2));\n                    __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2));\n\n                    __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2));\n                    __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2));\n\n                    __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2));\n                    __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2));\n\n                    __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2));\n                    __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2));\n\n                    // Combine results from both shuffle patterns for each output block\n                    __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                    __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                    __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                    __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                    __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                    __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                    __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                    __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                    __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n                    __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n                    __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n                    __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n                    __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n                    __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n                    __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n                    __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n                    __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n                    __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n                    __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n                    __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n                    __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n                    __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n                    __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n                    __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n                    __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n                    __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n                    __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n                    __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n                    __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n                    __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n                    __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n                    __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0);\n                    iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0);\n                    iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0);\n                    iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0);\n\n                    iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1);\n                    iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1);\n                    iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1);\n                    iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1);\n\n                    iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2);\n                    iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2);\n                    iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2);\n                    iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2);\n\n                    iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3);\n                    iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3);\n                    iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3);\n                    iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3);\n\n                    iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4);\n                    iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4);\n                    iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4);\n                    iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4);\n\n                    iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5);\n                    iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5);\n                    iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5);\n                    iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5);\n\n                    iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6);\n                    iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6);\n                    iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6);\n                    iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6);\n\n                    iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7);\n                    iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7);\n                    iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7);\n                    iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7);\n\n                    __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n                    __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n                    __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n                    __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n                    // Straighten out to make 4 row vectors\n                    __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78), iacc_mat_01);\n                    __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78));\n                    __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78), iacc_mat_11);\n\n                    // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                    const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n                    const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n                    const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1);\n\n                    // Multiply with appropiate scales and accumulate (for both d and dmin) below\n                    acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n                    acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n                    acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                    acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n                    // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n                    __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0), mins_01);\n                    __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170), mins_01);\n                    __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0), mins_01);\n                    __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170), mins_01);\n\n                    __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85), mins_23);\n                    __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255), mins_23);\n                    __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85), mins_23);\n                    __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255), mins_23);\n\n                    __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0), mins_45);\n                    __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170), mins_45);\n                    __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0), mins_45);\n                    __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170), mins_45);\n\n                    __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85), mins_67);\n                    __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255), mins_67);\n                    __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85), mins_67);\n                    __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255), mins_67);\n\n                    __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n                    __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n                    __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n                    __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n                    acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n                    acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n                    acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n                    acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n                }\n            }\n            // Store accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n\n    if (anc != nc) {\n        xstart = anc/8;\n        y = 0;\n    }\n\n#endif // __AVX512BW__ && __AVX512DQ__\n\n    // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation\n    for (; y < anr / 4; y += 4) {\n\n        const block_q8_Kx4 * a_ptrs[4];\n\n        a_ptrs[0] = a_ptr_start + (y * nb);\n        for (int i = 0; i < 3; ++i) {\n            a_ptrs[i + 1] = a_ptrs[i] + nb;\n        }\n\n        // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = xstart; x < nc / 8; x++) {\n\n            const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            __m256 acc_min_rows[16];\n            for (int i = 0; i < 16; i++) {\n                acc_min_rows[i] = _mm256_setzero_ps();\n            }\n\n            // For super block\n            for (int64_t b = 0; b < nb; b++) {\n                // Delta values - Load the eight scale values of block_q2_kx8\n                const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n                // dmin values - Load the eight dmin values of block_q2_kx8\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 128; sb++) {\n\n                    // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256));\n\n                    // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                    //superblock    sub block   which part of sub block\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    // 2-bit -> 8-bit\n                    // First sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n                    const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n                    const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n                    const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n                    // Second sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n                    const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n                    const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n                    const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n                    // Third sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7)\n                    const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7)\n\n                    const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15)\n                    const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15)\n\n                    // Fourth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7)\n                    const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7)\n\n                    const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15)\n                    const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15)\n\n                    // Fifth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7)\n                    const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7)\n\n                    const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15)\n                    const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15)\n\n                    // Sixth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7)\n                    const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7)\n\n                    const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15)\n                    const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15)\n\n                    // Seventh sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7)\n                    const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7)\n\n                    const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15)\n                    const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15)\n\n                    // Eighth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7)\n                    const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7)\n\n                    const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15)\n                    const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15)\n\n                    // Shuffle pattern one - right side input\n                    const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n                    const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n                    const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n                    const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n                    const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n                    const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n                    const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n                    const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n                    const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3)\n                    const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3)\n\n                    const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11)\n                    const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11)\n\n                    const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3)\n                    const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3)\n\n                    const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11\n                    const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11)\n\n                    const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3)\n                    const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3)\n\n                    const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11)\n                    const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11)\n\n                    const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3)\n                    const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3)\n\n                    const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11)\n                    const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11)\n\n                    const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3)\n                    const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3)\n\n                    const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11)\n                    const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11)\n\n                    const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3)\n                    const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3)\n\n                    const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11)\n                    const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11)\n\n\n                    // Shuffle pattern two - right side input\n                    const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n                    const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n                    const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n                    const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n                    const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n                    const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n                    const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n                    const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n                    const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7)\n                    const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7)\n\n                    const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15)\n                    const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15)\n\n                    const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7)\n                    const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7)\n\n                    const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15)\n                    const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15)\n\n                    const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7)\n                    const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7)\n\n                    const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15)\n                    const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15)\n\n                    const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7)\n                    const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7)\n\n                    const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15)\n                    const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15)\n\n                    const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7)\n                    const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7)\n\n                    const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15)\n                    const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15)\n\n                    const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7)\n                    const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7)\n\n                    const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15)\n                    const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15)\n\n                    //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n                    //s00 m00  s01 m01   s10 m10  s11 m11  s20 m20  s21 m21   s30 m30  s31 m31  s40 m40  s41 m41   s50 m50  s51 m51  s60 m60  s61 m61   s70 m70  s71 m71\n\n                    // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n                    const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n                    // Extract scales which is lower half from mins_and_scales\n                    const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n                    const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n                    const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n                    const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n                    // Extract mins which is upper half from mins_and_scales\n                    const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n                    const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n                    const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n                    const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n                    const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse));\n                    const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse));\n\n                    const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse));\n                    const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse));\n\n                    const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse));\n                    const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse));\n\n                    const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse));\n                    const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse));\n\n                    const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n                    const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n                    const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n                    const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n                    const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68);\n                    const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238);\n\n                    const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68);\n                    const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238);\n\n                    const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68);\n                    const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238);\n\n                    const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68);\n                    const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238);\n\n                    const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68);\n                    const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238);\n\n                    const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68);\n                    const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238);\n\n\n                    for (int rp = 0; rp < 4; rp++) {\n\n                        // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                        // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                        __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb)));\n                        __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n                        __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n                        __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb)));\n                        __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n                        __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n                        __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb)));\n                        __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n                        __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n                        __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb)));\n                        __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n                        __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n                        __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb)));\n                        __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0);\n                        __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17);\n                        __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb)));\n                        __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0);\n                        __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17);\n                        __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb)));\n                        __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0);\n                        __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17);\n                        __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb)));\n                        __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0);\n                        __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17);\n\n                        __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb)));\n                        __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0);\n                        __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17);\n                        __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb)));\n                        __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0);\n                        __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17);\n                        __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb)));\n                        __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0);\n                        __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17);\n                        __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb)));\n                        __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0);\n                        __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17);\n                        __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb)));\n                        __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0);\n                        __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17);\n                        __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb)));\n                        __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0);\n                        __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17);\n                        __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb)));\n                        __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0);\n                        __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17);\n                        __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb)));\n                        __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0);\n                        __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17);\n\n                        // Bsums are loaded for the different Q8_K blocks\n                        __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb)));\n                        __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb));\n                        __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb)));\n                        __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb));\n\n                        // Shuffle pattern one - left side input\n                        const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                        const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n                        const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                        const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n                        const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                        const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n                        const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                        const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n                        const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n                        const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3)\n\n                        const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n                        const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11)\n\n                        const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n                        const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3)\n\n                        const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n                        const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11)\n\n                        const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n                        const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3)\n\n                        const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n                        const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11)\n\n                        const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n                        const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3)\n\n                        const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n                        const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11)\n\n                        const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n                        const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3)\n\n                        const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n                        const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11)\n\n                        const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n                        const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3)\n\n                        const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n                        const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11)\n\n                        // Shuffle pattern two- left side input\n                        const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                        const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n                        const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                        const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n                        const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                        const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n                        const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                        const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n                        const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n                        const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7)\n\n                        const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n                        const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15)\n\n                        const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n                        const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7)\n\n                        const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n                        const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15)\n\n                        const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n                        const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7)\n\n                        const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n                        const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15)\n\n                        const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n                        const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7)\n\n                        const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n                        const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15)\n\n                        const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n                        const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7)\n\n                        const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n                        const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15)\n\n                        const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n                        const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7)\n\n                        const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n                        const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15)\n\n                        // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                        __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1));\n                        __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1));\n\n                        __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1));\n                        __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1));\n\n                        __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1));\n                        __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1));\n\n                        __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1));\n                        __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1));\n\n                        __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1));\n                        __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1));\n\n                        __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1));\n                        __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1));\n\n                        __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1));\n                        __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1));\n\n                        __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1));\n                        __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1));\n\n                        __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1));\n                        __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1));\n\n                        __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1));\n                        __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1));\n\n                        __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1));\n                        __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1));\n\n                        __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1));\n                        __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1));\n\n                        __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1));\n                        __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1));\n\n                        __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1));\n                        __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1));\n\n                        __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1));\n                        __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1));\n\n                        __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1));\n                        __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1));\n\n\n                        __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2));\n                        __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2));\n\n                        __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2));\n                        __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2));\n\n                        __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2));\n                        __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2));\n\n                        __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2));\n                        __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2));\n\n                        __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2));\n                        __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2));\n\n                        __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2));\n                        __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2));\n\n                        __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2));\n                        __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2));\n\n                        __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2));\n                        __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2));\n\n                        __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2));\n                        __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2));\n\n                        __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2));\n                        __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2));\n\n                        __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2));\n                        __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2));\n\n                        __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2));\n                        __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2));\n\n                        __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2));\n                        __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2));\n\n                        __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2));\n                        __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2));\n\n                        __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2));\n                        __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2));\n\n                        __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2));\n                        __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2));\n\n                        // Combine results from both shuffle patterns for each output block\n                        __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                        __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                        __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                        __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                        __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                        __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                        __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                        __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                        __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n                        __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n                        __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n                        __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n                        __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n                        __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n                        __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n                        __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n                        __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n                        __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n                        __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n                        __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n                        __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n                        __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n                        __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n                        __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n                        __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n                        __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n                        __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n                        __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n                        __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n                        __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n                        __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n                        __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n                        // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                        iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n                        iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n                        iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n                        iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n                        iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n                        iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n                        iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n                        iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n                        iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2);\n                        iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2);\n                        iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2);\n                        iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2);\n\n                        iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3);\n                        iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3);\n                        iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3);\n                        iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3);\n\n                        iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4);\n                        iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4);\n                        iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4);\n                        iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4);\n\n                        iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5);\n                        iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5);\n                        iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5);\n                        iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5);\n\n                        iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6);\n                        iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6);\n                        iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6);\n                        iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6);\n\n                        iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7);\n                        iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7);\n                        iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7);\n                        iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7);\n\n                        __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n                        __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n                        __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n                        __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n                        // Straighten out to make 4 row vectors\n                        __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n                        __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n                        __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n                        __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n                        // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                        const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d);\n                        const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n\n                        // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                        acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[rp * 4]);\n                        acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[rp * 4 + 1]);\n                        acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[rp * 4 + 2]);\n                        acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[rp * 4 + 3]);\n\n                        __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n                        __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n                        __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n                        __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n\n                       // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n                        __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0), mins_01);\n                        __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170), mins_01);\n                        __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0), mins_01);\n                        __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170), mins_01);\n\n                        __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85), mins_23);\n                        __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255), mins_23);\n                        __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85), mins_23);\n                        __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255), mins_23);\n\n                        __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0), mins_45);\n                        __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170), mins_45);\n                        __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0), mins_45);\n                        __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170), mins_45);\n\n                        __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85), mins_67);\n                        __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255), mins_67);\n                        __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85), mins_67);\n                        __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255), mins_67);\n\n                        __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n                        __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n                        __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n                        __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n                        acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[rp * 4]);\n                        acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[rp * 4 + 1]);\n                        acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[rp * 4 + 2]);\n                        acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[rp * 4 + 3]);\n\n                    }\n                }\n            }\n            // Store the accumulated values\n            for (int i = 0; i < 16; i++) {\n                _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n\n            }\n        }\n    }\n\n    for (; y < nr / 4; y ++) {\n\n        const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb);\n\n        // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation\n        for (int64_t x = xstart; x < nc / 8; x++) {\n\n            const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb);\n\n            // Master FP accumulators\n            __m256 acc_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_rows[i] = _mm256_setzero_ps();\n            }\n\n            __m256 acc_min_rows[4];\n            for (int i = 0; i < 4; i++) {\n                acc_min_rows[i] = _mm256_setzero_ps();\n            }\n\n            for (int64_t b = 0; b < nb; b++) {\n                // Delta values - Load the eight scale values of block_q2_kx8\n                const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d);\n\n                // dmin values - Load the eight dmin values of block_q2_kx8\n                const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin);\n\n                // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration\n                for (int sb = 0; sb < QK_K / 128; sb++) {\n\n                    // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7\n                    const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256));\n                    const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256));\n                    const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256));\n                    const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256));\n\n                    // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values\n                    //superblock    sub block   which part of sub block\n                    const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240);\n\n                    const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240);\n\n                    const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240);\n\n                    const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240);\n                    const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240);\n\n                    // 2-bit -> 8-bit\n                    // First sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7)\n                    const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7)\n\n                    const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15)\n                    const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15)\n\n                    // Second sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7)\n                    const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7)\n\n                    const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15)\n                    const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15)\n\n                    // Third sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7)\n                    const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7)\n\n                    const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15)\n                    const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15)\n\n                    // Fourth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7)\n                    const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7)\n\n                    const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15)\n                    const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15)\n\n                    // Fifth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7)\n                    const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7)\n\n                    const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15)\n                    const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15)\n\n                    // Sixth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7)\n                    const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7)\n\n                    const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15)\n                    const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15)\n\n                    // Seventh sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7)\n                    const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7)\n\n                    const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15)\n                    const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15)\n\n                    // Eighth sub block of the eight sub blocks processed in the iteration\n                    const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7)\n                    const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7)\n\n                    const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15)\n                    const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15)\n\n                    // Shuffle pattern one - right side input\n                    const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3)\n                    const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3)\n\n                    const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11)\n                    const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11)\n\n                    const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3)\n                    const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3)\n\n                    const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11)\n                    const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11)\n\n                    const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3)\n                    const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3)\n\n                    const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11)\n                    const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11)\n\n                    const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3)\n                    const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3)\n\n                    const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11\n                    const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11)\n\n                    const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3)\n                    const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3)\n\n                    const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11)\n                    const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11)\n\n                    const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3)\n                    const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3)\n\n                    const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11)\n                    const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11)\n\n                    const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3)\n                    const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3)\n\n                    const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11)\n                    const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11)\n\n                    const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3)\n                    const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3)\n\n                    const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11)\n                    const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11)\n\n\n                    // Shuffle pattern two - right side input\n                    const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7)\n                    const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7)\n\n                    const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15)\n                    const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15)\n\n                    const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7)\n                    const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7)\n\n                    const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15)\n                    const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15)\n\n                    const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7)\n                    const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7)\n\n                    const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15)\n                    const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15)\n\n                    const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7)\n                    const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7)\n\n                    const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15)\n                    const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15)\n\n                    const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7)\n                    const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7)\n\n                    const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15)\n                    const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15)\n\n                    const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7)\n                    const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7)\n\n                    const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15)\n                    const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15)\n\n                    const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7)\n                    const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7)\n\n                    const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15)\n                    const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15)\n\n                    const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7)\n                    const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7)\n\n                    const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15)\n                    const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15)\n\n\n                    //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together\n                    //s00 m00  s01 m01   s10 m10  s11 m11  s20 m20  s21 m21   s30 m30  s31 m31  s40 m40  s41 m41   s50 m50  s51 m51  s60 m60  s61 m61   s70 m70  s71 m71\n\n                    // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop\n                    const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64));\n                    const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64));\n                    const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64));\n                    const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64));\n\n                    // Extract scales which is lower half from mins_and_scales\n                    const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse);\n                    const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse);\n                    const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse);\n                    const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse);\n\n                    // Extract mins which is upper half from mins_and_scales\n                    const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse));\n                    const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse));\n                    const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse));\n                    const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse));\n\n                    const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse));\n                    const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse));\n\n                    const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse));\n                    const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse));\n\n                    const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse));\n                    const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse));\n\n                    const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse));\n                    const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse));\n\n                    const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68);\n                    const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238);\n\n                    const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68);\n                    const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238);\n\n                    const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68);\n                    const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238);\n\n                    const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68);\n                    const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238);\n\n                    const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68);\n                    const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238);\n\n                    const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68);\n                    const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238);\n\n                    const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68);\n                    const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238);\n\n                    const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68);\n                    const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238);\n\n                    // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3\n                    // Loaded as set of 128 bit vectors and repeated into a 256 bit vector\n                    __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb)));\n                    __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0);\n                    __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17);\n                    __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb)));\n                    __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0);\n                    __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17);\n                    __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb)));\n                    __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0);\n                    __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17);\n                    __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb)));\n                    __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0);\n                    __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17);\n                    __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb)));\n                    __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0);\n                    __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17);\n                    __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb)));\n                    __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0);\n                    __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17);\n                    __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb)));\n                    __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0);\n                    __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17);\n                    __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb)));\n                    __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0);\n                    __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17);\n\n                    __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb)));\n                    __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0);\n                    __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17);\n                    __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb)));\n                    __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0);\n                    __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17);\n                    __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb)));\n                    __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0);\n                    __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17);\n                    __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb)));\n                    __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0);\n                    __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17);\n                    __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb)));\n                    __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0);\n                    __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17);\n                    __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb)));\n                    __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0);\n                    __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17);\n                    __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb)));\n                    __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0);\n                    __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17);\n                    __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb)));\n                    __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0);\n                    __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17);\n\n                    // Bsums are loaded for the different Q8_K blocks\n                    __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb)));\n                    __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb));\n                    __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb)));\n                    __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb));\n\n                    // Shuffle pattern one - left side input\n                    const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3)\n                    const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3)\n\n                    const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11)\n                    const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11)\n\n                    const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3)\n                    const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3)\n\n                    const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11)\n                    const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11)\n\n                    const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3)\n                    const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3)\n\n                    const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11)\n                    const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11)\n\n                    const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3)\n                    const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3)\n\n                    const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11)\n                    const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11)\n\n                    const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3)\n                    const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3)\n\n                    const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11)\n                    const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11)\n\n                    const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3)\n                    const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3)\n\n                    const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11)\n                    const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11)\n\n                    const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3)\n                    const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3)\n\n                    const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11)\n                    const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11)\n\n                    const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3)\n                    const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3)\n\n                    const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11)\n                    const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11)\n\n                    // Shuffle pattern two- left side input\n                    const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7)\n                    const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7)\n\n                    const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15)\n                    const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15)\n\n                    const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7)\n                    const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7)\n\n                    const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15)\n                    const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15)\n\n                    const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7)\n                    const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7)\n\n                    const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15)\n                    const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15)\n\n                    const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7)\n                    const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7)\n\n                    const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15)\n                    const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15)\n\n                    const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7)\n                    const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7)\n\n                    const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15)\n                    const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15)\n\n                    const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7)\n                    const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7)\n\n                    const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15)\n                    const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15)\n\n                    const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7)\n                    const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7)\n\n                    const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15)\n                    const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15)\n\n                    const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7)\n                    const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7)\n\n                    const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15)\n                    const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15)\n\n                    // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane\n                    __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1));\n                    __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1));\n\n                    __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1));\n                    __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1));\n\n                    __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1));\n                    __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1));\n\n                    __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1));\n                    __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1));\n\n                    __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1));\n                    __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1));\n\n                    __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1));\n                    __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1));\n\n                    __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1));\n                    __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1));\n\n                    __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1));\n                    __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1));\n\n                    __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1));\n                    __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1));\n\n                    __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1));\n                    __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1));\n\n                    __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1));\n                    __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1));\n\n                    __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1));\n                    __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1));\n\n                    __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1));\n                    __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1));\n\n                    __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1));\n                    __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1));\n\n                    __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1));\n                    __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1));\n\n                    __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1));\n                    __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1));\n\n\n                    __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2));\n                    __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2));\n\n                    __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2));\n                    __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2));\n\n                    __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2));\n                    __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2));\n\n                    __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2));\n                    __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2));\n\n                    __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2));\n                    __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2));\n\n                    __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2));\n                    __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2));\n\n                    __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2));\n                    __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2));\n\n                    __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2));\n                    __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2));\n\n                    __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2));\n                    __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2));\n\n                    __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2));\n                    __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2));\n\n                    __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2));\n                    __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2));\n\n                    __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2));\n                    __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2));\n\n                    __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2));\n                    __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2));\n\n                    __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2));\n                    __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2));\n\n                    __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2));\n                    __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2));\n\n                    __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2));\n                    __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2));\n\n                    // Combine results from both shuffle patterns for each output block.\n                    __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2);\n                    __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2);\n                    __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2);\n                    __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2);\n\n                    __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2);\n                    __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2);\n                    __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2);\n                    __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2);\n\n                    __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2);\n                    __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2);\n                    __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2);\n                    __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2);\n\n                    __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2);\n                    __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2);\n                    __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2);\n                    __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2);\n\n                    __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2);\n                    __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2);\n                    __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2);\n                    __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2);\n\n                    __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2);\n                    __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2);\n                    __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2);\n                    __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2);\n\n                    __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2);\n                    __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2);\n                    __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2);\n                    __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2);\n\n                    __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2);\n                    __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2);\n                    __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2);\n                    __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2);\n\n                    // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block\n                    iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0);\n                    iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0);\n                    iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0);\n                    iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0);\n\n                    iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1);\n                    iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1);\n                    iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1);\n                    iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1);\n\n                    iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2);\n                    iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2);\n                    iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2);\n                    iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2);\n\n                    iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3);\n                    iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3);\n                    iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3);\n                    iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3);\n\n                    iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4);\n                    iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4);\n                    iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4);\n                    iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4);\n\n                    iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5);\n                    iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5);\n                    iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5);\n                    iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5);\n\n                    iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6);\n                    iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6);\n                    iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6);\n                    iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6);\n\n                    iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7);\n                    iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7);\n                    iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7);\n                    iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7);\n\n                    __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7)));\n                    __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7)));\n                    __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7)));\n                    __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7)));\n\n                    // Straighten out to make 4 row vectors\n                    __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204);\n                    __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204);\n                    __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204);\n                    __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204);\n\n                    // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes\n                    const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d);\n                    const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);\n\n                    // Multiply with appropriate scales and accumulate (for both d and dmin) below\n                    acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_rows[0]);\n                    acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_rows[1]);\n                    acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_rows[2]);\n                    acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_rows[3]);\n\n                    __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1);\n                    __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1);\n                    __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1);\n                    __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1);\n\n                    // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K\n                    __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0), mins_01);\n                    __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170), mins_01);\n                    __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0), mins_01);\n                    __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170), mins_01);\n\n                    __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85), mins_23);\n                    __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255), mins_23);\n                    __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85), mins_23);\n                    __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255), mins_23);\n\n                    __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0), mins_45);\n                    __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170), mins_45);\n                    __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0), mins_45);\n                    __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170), mins_45);\n\n                    __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85), mins_67);\n                    __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255), mins_67);\n                    __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85), mins_67);\n                    __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255), mins_67);\n\n                    __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67));\n                    __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67));\n                    __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67));\n                    __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67));\n\n                    acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)), acc_min_rows[0]);\n                    acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)), acc_min_rows[1]);\n                    acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)), acc_min_rows[2]);\n                    acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)), acc_min_rows[3]);\n                }\n            }\n            // Store the accumulated values\n            for (int i = 0; i < 4; i++) {\n                _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i]));\n            }\n        }\n    }\n#else\n\n    ggml_gemm_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc);\n\n\n#endif\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/arch-fallback.h",
    "content": "\n#pragma once\n\n// Rename `_generic` functions if no native implementation is available.\n// This effectively selects the generic implementation.\n\n#if defined(GGML_CPU_GENERIC)\n// quants.c\n#define quantize_row_q8_0_generic quantize_row_q8_0\n#define quantize_row_q8_1_generic quantize_row_q8_1\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_q4_0_q8_0_generic ggml_vec_dot_q4_0_q8_0\n#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1\n#define ggml_vec_dot_q5_0_q8_0_generic ggml_vec_dot_q5_0_q8_0\n#define ggml_vec_dot_q5_1_q8_1_generic ggml_vec_dot_q5_1_q8_1\n#define ggml_vec_dot_q8_0_q8_0_generic ggml_vec_dot_q8_0_q8_0\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K\n#define ggml_vec_dot_q3_K_q8_K_generic ggml_vec_dot_q3_K_q8_K\n#define ggml_vec_dot_q4_K_q8_K_generic ggml_vec_dot_q4_K_q8_K\n#define ggml_vec_dot_q5_K_q8_K_generic ggml_vec_dot_q5_K_q8_K\n#define ggml_vec_dot_q6_K_q8_K_generic ggml_vec_dot_q6_K_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0\n#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)\n// repack.cpp\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)\n// quants.c\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__POWERPC__) || defined(__powerpc__)\n// ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__loongarch64)\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__riscv)\n// quants.c\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_K_4x1_generic ggml_quantize_mat_q8_K_4x1\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__s390x__)\n// quants.c\n#define quantize_row_q8_K_generic quantize_row_q8_K\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#elif defined(__wasm__)\n// quants.c\n#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1\n#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K\n#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K\n#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K\n#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K\n#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K\n#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K\n#define ggml_vec_dot_iq3_s_q8_K_generic ggml_vec_dot_iq3_s_q8_K\n#define ggml_vec_dot_iq1_s_q8_K_generic ggml_vec_dot_iq1_s_q8_K\n#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K\n#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0\n#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K\n#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0\n#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0\n// repack.cpp\n#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4\n#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8\n#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4\n#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8\n#define ggml_gemv_q4_0_4x4_q8_0_generic ggml_gemv_q4_0_4x4_q8_0\n#define ggml_gemv_q4_0_4x8_q8_0_generic ggml_gemv_q4_0_4x8_q8_0\n#define ggml_gemv_q4_0_8x8_q8_0_generic ggml_gemv_q4_0_8x8_q8_0\n#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K\n#define ggml_gemv_q4_K_8x4_q8_K_generic ggml_gemv_q4_K_8x4_q8_K\n#define ggml_gemv_q4_K_8x8_q8_K_generic ggml_gemv_q4_K_8x8_q8_K\n#define ggml_gemv_q5_K_8x4_q8_K_generic ggml_gemv_q5_K_8x4_q8_K\n#define ggml_gemv_q5_K_8x8_q8_K_generic ggml_gemv_q5_K_8x8_q8_K\n#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K\n#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K\n#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0\n#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0\n#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0\n#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0\n#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0\n#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0\n#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0\n#define ggml_gemm_q4_0_4x8_q8_0_generic ggml_gemm_q4_0_4x8_q8_0\n#define ggml_gemm_q4_0_8x8_q8_0_generic ggml_gemm_q4_0_8x8_q8_0\n#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K\n#define ggml_gemm_q4_K_8x4_q8_K_generic ggml_gemm_q4_K_8x4_q8_K\n#define ggml_gemm_q4_K_8x8_q8_K_generic ggml_gemm_q4_K_8x8_q8_K\n#define ggml_gemm_q5_K_8x4_q8_K_generic ggml_gemm_q5_K_8x4_q8_K\n#define ggml_gemm_q5_K_8x8_q8_K_generic ggml_gemm_q5_K_8x8_q8_K\n#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K\n#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K\n#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0\n#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0\n#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0\n#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0\n#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0\n#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/binary-ops.cpp",
    "content": "#include \"binary-ops.h\"\n\n#if defined(GGML_USE_ACCELERATE)\n#include <Accelerate/Accelerate.h>\n\nusing vDSP_fn_t = void (*)(const float *, vDSP_Stride, const float *, vDSP_Stride, float *, vDSP_Stride, vDSP_Length);\n#endif\n\nstatic inline float op_add(float a, float b) {\n    return a + b;\n}\n\nstatic inline float op_sub(float a, float b) {\n    return a - b;\n}\n\nstatic inline float op_mul(float a, float b) {\n    return a * b;\n}\n\nstatic inline float op_div(float a, float b) {\n    return a / b;\n}\n\ntemplate <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>\nstatic inline void vec_binary_op_contiguous(const int64_t n, dst_t * z, const src0_t * x, const src1_t * y) {\n    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;\n    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;\n    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;\n\n    for (int i = 0; i < n; i++) {\n        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(y[i])));\n    }\n}\n\ntemplate <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>\nstatic inline void vec_binary_op_non_contiguous(const int64_t n, const int64_t ne10, const int64_t nb10, dst_t * z, const src0_t * x, const src1_t * y) {\n    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;\n    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;\n    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;\n\n    for (int i = 0; i < n; i++) {\n        int i10 = i % ne10;\n        const src1_t * y_ptr = (const src1_t *)((const char *)y + i10*nb10);\n        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(*y_ptr)));\n    }\n}\n\ntemplate <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>\nstatic void apply_binary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    GGML_ASSERT( nb0 == sizeof(dst_t));\n    GGML_ASSERT(nb00 == sizeof(src0_t));\n\n    const auto [ir0, ir1] = get_thread_range(params, src0);\n    const bool is_src1_contiguous_rows = ggml_is_contiguous_rows(src1);\n\n#ifdef GGML_USE_ACCELERATE\n    vDSP_fn_t vDSP_op = nullptr;\n    // TODO - avoid the f32-only check using type 'trait' lookup tables and row-based src-to-float conversion functions\n    if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n        if (op == op_add) {\n            vDSP_op = vDSP_vadd;\n        } else if (op == op_sub) {\n            vDSP_op = vDSP_vsub;\n        } else if (op == op_mul) {\n            vDSP_op = vDSP_vmul;\n        } else if (op == op_div) {\n            vDSP_op = vDSP_vdiv;\n        }\n    }\n#endif\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        const int64_t i13 = i03 % ne13;\n        const int64_t i12 = i02 % ne12;\n        const int64_t i11 = i01 % ne11;\n\n        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );\n        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n        const src1_t * src1_ptr = (const src1_t *) ((const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);\n\n        if (is_src1_contiguous_rows) {\n            // src1 is broadcastable across src0 and dst in i1, i2, i3\n            const int64_t nr0 = ne00 / ne10;\n\n            for (int64_t r = 0; r < nr0; ++r) {\n#ifdef GGML_USE_ACCELERATE\n                if constexpr (std::is_same_v<src0_t, float> && std::is_same_v<src1_t, float> && std::is_same_v<dst_t, float>) {\n                    if (vDSP_op != nullptr) {\n                        vDSP_op(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);\n                        continue;\n                    }\n                }\n#endif\n                vec_binary_op_contiguous<op>(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);\n            }\n        } else {\n            vec_binary_op_non_contiguous<op>(ne0, ne10, nb10, dst_ptr, src0_ptr, src1_ptr);\n        }\n    }\n}\n\n// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates\ntemplate <float (*op)(float, float)>\nstatic void binary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    /*  */ if (src0->type == GGML_TYPE_F32  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) { // all f32\n        apply_binary_op<op, float, float, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F16) { // all f16\n        apply_binary_op<op, ggml_fp16_t, ggml_fp16_t, ggml_fp16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n        apply_binary_op<op, ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_BF16) {\n        apply_binary_op<op, ggml_bf16_t, float, ggml_bf16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) {\n        apply_binary_op<op, ggml_bf16_t, float, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F16) {\n        apply_binary_op<op, ggml_fp16_t, float, ggml_fp16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) {\n        apply_binary_op<op, ggml_fp16_t, float, float>(params, dst);\n    } else {\n        GGML_ABORT(\"%s: unsupported types: dst: %s, src0: %s, src1: %s\\n\", __func__,\n            ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n    }\n}\n\nvoid ggml_compute_forward_add_non_quantized(const ggml_compute_params * params, ggml_tensor * dst) {\n    binary_op<op_add>(params, dst);\n}\n\nvoid ggml_compute_forward_sub(const ggml_compute_params * params, ggml_tensor * dst) {\n    binary_op<op_sub>(params, dst);\n}\n\nvoid ggml_compute_forward_mul(const ggml_compute_params * params, ggml_tensor * dst) {\n    binary_op<op_mul>(params, dst);\n}\n\nvoid ggml_compute_forward_div(const ggml_compute_params * params, ggml_tensor * dst) {\n    binary_op<op_div>(params, dst);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/binary-ops.h",
    "content": "#pragma once\n\n#include \"common.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_add_non_quantized(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sub(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mul(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_div(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/cmake/FindSIMD.cmake",
    "content": "include(CheckCSourceRuns)\n\nset(AVX_CODE \"\n    #include <immintrin.h>\n    int main()\n    {\n        __m256 a;\n        a = _mm256_set1_ps(0);\n        return 0;\n    }\n\")\n\nset(AVX512_CODE \"\n    #include <immintrin.h>\n    int main()\n    {\n        __m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0,\n                                    0, 0, 0, 0, 0, 0, 0, 0);\n        __m512i b = a;\n        __mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ);\n        return 0;\n    }\n\")\n\nset(AVX2_CODE \"\n    #include <immintrin.h>\n    int main()\n    {\n        __m256i a = {0};\n        a = _mm256_abs_epi16(a);\n        __m256i x;\n        _mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code\n        return 0;\n    }\n\")\n\nset(FMA_CODE \"\n    #include <immintrin.h>\n    int main()\n    {\n        __m256 acc = _mm256_setzero_ps();\n        const __m256 d = _mm256_setzero_ps();\n        const __m256 p = _mm256_setzero_ps();\n        acc = _mm256_fmadd_ps( d, p, acc );\n        return 0;\n    }\n\")\n\nmacro(check_sse type flags)\n    set(__FLAG_I 1)\n    set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})\n    foreach (__FLAG ${flags})\n        if (NOT ${type}_FOUND)\n            set(CMAKE_REQUIRED_FLAGS ${__FLAG})\n            check_c_source_runs(\"${${type}_CODE}\" HAS_${type}_${__FLAG_I})\n            if (HAS_${type}_${__FLAG_I})\n                set(${type}_FOUND TRUE CACHE BOOL \"${type} support\")\n                set(${type}_FLAGS \"${__FLAG}\" CACHE STRING \"${type} flags\")\n            endif()\n            math(EXPR __FLAG_I \"${__FLAG_I}+1\")\n        endif()\n    endforeach()\n    set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})\n\n    if (NOT ${type}_FOUND)\n        set(${type}_FOUND FALSE CACHE BOOL \"${type} support\")\n        set(${type}_FLAGS \"\" CACHE STRING \"${type} flags\")\n    endif()\n\n    mark_as_advanced(${type}_FOUND ${type}_FLAGS)\nendmacro()\n\n# flags are for MSVC only!\ncheck_sse(\"AVX\" \" ;/arch:AVX\")\nif (NOT ${AVX_FOUND})\n    set(GGML_AVX OFF)\nelse()\n    set(GGML_AVX ON)\nendif()\n\ncheck_sse(\"AVX2\" \" ;/arch:AVX2\")\ncheck_sse(\"FMA\" \" ;/arch:AVX2\")\nif ((NOT ${AVX2_FOUND}) OR (NOT ${FMA_FOUND}))\n    set(GGML_AVX2 OFF)\nelse()\n    set(GGML_AVX2 ON)\nendif()\n\ncheck_sse(\"AVX512\" \" ;/arch:AVX512\")\nif (NOT ${AVX512_FOUND})\n    set(GGML_AVX512 OFF)\nelse()\n    set(GGML_AVX512 ON)\nendif()\n"
  },
  {
    "path": "ggml/src/ggml-cpu/common.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"traits.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-impl.h\"\n#include \"simd-mappings.h\"\n\n#define GGML_FA_TILE_Q  64\n#define GGML_FA_TILE_KV 64\n\n#ifdef __cplusplus\n\n#include <utility>\n\n// convenience functions/macros for use in template calls\n// note: these won't be required after the 'traits' lookup table is used.\nstatic inline ggml_fp16_t f32_to_f16(float x) {\n    return GGML_CPU_FP32_TO_FP16(x);\n}\n\nstatic inline float f16_to_f32(ggml_fp16_t x) {\n    return GGML_CPU_FP16_TO_FP32(x);\n}\n\nstatic inline ggml_bf16_t f32_to_bf16(float x) {\n    return GGML_FP32_TO_BF16(x);\n}\n\nstatic inline float bf16_to_f32(ggml_bf16_t x) {\n    return GGML_BF16_TO_FP32(x);\n}\n\nstatic inline float i32_to_f32(int32_t x) {\n    return x;\n}\n\nstatic inline int32_t f32_to_i32(float x) {\n    return x;\n}\n\nstatic inline float f32_to_f32(float x) {\n    return x;\n}\n\n// TODO - merge this into the traits table, after using row-based conversions\ntemplate <class T>\nstruct type_conversion_table;\n\ntemplate <>\nstruct type_conversion_table<ggml_fp16_t> {\n    static constexpr float (*to_f32)(ggml_fp16_t) = f16_to_f32;\n    static constexpr ggml_fp16_t (*from_f32)(float) = f32_to_f16;\n};\n\ntemplate <>\nstruct type_conversion_table<float> {\n    static constexpr float (*to_f32)(float) = f32_to_f32;\n    static constexpr float (*from_f32)(float) = f32_to_f32;\n};\n\ntemplate <>\nstruct type_conversion_table<ggml_bf16_t> {\n    static constexpr float (*to_f32)(ggml_bf16_t) = bf16_to_f32;\n    static constexpr ggml_bf16_t (*from_f32)(float) = f32_to_bf16;\n};\n\ntemplate <>\nstruct type_conversion_table<int32_t> {\n    static constexpr float (*to_f32)(int32_t) = i32_to_f32;\n    static constexpr int32_t (*from_f32)(float) = f32_to_i32;\n};\n\nstatic std::pair<int64_t, int64_t> get_thread_range(const struct ggml_compute_params * params, const struct ggml_tensor * src0) {\n    const int64_t ith = params->ith;\n    const int64_t nth = params->nth;\n\n    const int64_t nr  = ggml_nrows(src0);\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    return {ir0, ir1};\n}\n\nstruct ggml_fa_tile_config {\n    static constexpr size_t Q  = GGML_FA_TILE_Q;\n    static constexpr size_t KV = GGML_FA_TILE_KV;\n};\n\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/ggml-cpu-impl.h",
    "content": "#pragma once\n\n// GGML CPU internal header\n\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n\n#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/\n//#include <stddef.h>\n#include <stdbool.h>\n#include <string.h> // memcpy\n#include <math.h>   // fabsf\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nstruct ggml_compute_params {\n    // ith = thread index, nth = number of threads\n    int ith, nth;\n\n    // work buffer for all threads\n    size_t wsize;\n    void * wdata;\n\n    struct ggml_threadpool * threadpool;\n\n    // use reference implementation\n    bool use_ref;\n};\n\n\n#if defined(_MSC_VER)\n\n#define m512bh(p) p\n#define m512i(p) p\n\n#else\n\n#define m512bh(p) (__m512bh)(p)\n#define m512i(p) (__m512i)(p)\n\n#endif\n\n// __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512\n#if defined(_MSC_VER) && (defined(__AVX2__) || defined(__AVX512F__))\n#ifndef __FMA__\n#define __FMA__\n#endif\n#ifndef __F16C__\n#define __F16C__\n#endif\n#endif\n\n// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available\n#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__))\n#ifndef __SSE3__\n#define __SSE3__\n#endif\n#ifndef __SSSE3__\n#define __SSSE3__\n#endif\n#endif\n\n#if defined(__s390x__) && defined(__VEC__)\n#ifndef __VXE__\n#define __VXE__\n#endif  // __VXE__\n#ifndef __VXE2__\n#define __VXE2__\n#endif  // __VXE2__\n#endif  // __s390x__ && __VEC__\n\n#if defined(__ARM_FEATURE_SVE) && defined(__linux__)\n#include <sys/prctl.h>\n#endif\n\n#if defined(__ARM_NEON)\n\n// ref: https://github.com/ggml-org/llama.cpp/pull/5404\n#ifdef _MSC_VER\n#define ggml_vld1q_u32(w,x,y,z) { ((w) + ((uint64_t)(x) << 32)), ((y) + ((uint64_t)(z) << 32)) }\n#else\n#define ggml_vld1q_u32(w,x,y,z) { (w), (x), (y), (z) }\n#endif // _MSC_VER\n\n#if !defined(__aarch64__)\n\n// 32-bit ARM compatibility\n\n// vaddlvq_s16\n// vpaddq_s16\n// vpaddq_s32\n// vaddvq_s32\n// vaddvq_f32\n// vmaxvq_f32\n// vcvtnq_s32_f32\n// vzip1_u8\n// vzip2_u8\n\ninline static int32_t vaddlvq_s16(int16x8_t v) {\n    int32x4_t v0 = vreinterpretq_s32_s64(vpaddlq_s32(vpaddlq_s16(v)));\n    return vgetq_lane_s32(v0, 0) + vgetq_lane_s32(v0, 2);\n}\n\ninline static int16x8_t vpaddq_s16(int16x8_t a, int16x8_t b) {\n    int16x4_t a0 = vpadd_s16(vget_low_s16(a), vget_high_s16(a));\n    int16x4_t b0 = vpadd_s16(vget_low_s16(b), vget_high_s16(b));\n    return vcombine_s16(a0, b0);\n}\n\ninline static int32x4_t vpaddq_s32(int32x4_t a, int32x4_t b) {\n    int32x2_t a0 = vpadd_s32(vget_low_s32(a), vget_high_s32(a));\n    int32x2_t b0 = vpadd_s32(vget_low_s32(b), vget_high_s32(b));\n    return vcombine_s32(a0, b0);\n}\n\ninline static int32_t vaddvq_s32(int32x4_t v) {\n    return vgetq_lane_s32(v, 0) + vgetq_lane_s32(v, 1) + vgetq_lane_s32(v, 2) + vgetq_lane_s32(v, 3);\n}\n\ninline static float vaddvq_f32(float32x4_t v) {\n    return vgetq_lane_f32(v, 0) + vgetq_lane_f32(v, 1) + vgetq_lane_f32(v, 2) + vgetq_lane_f32(v, 3);\n}\n\ninline static float vmaxvq_f32(float32x4_t v) {\n    return\n        MAX(MAX(vgetq_lane_f32(v, 0), vgetq_lane_f32(v, 1)),\n            MAX(vgetq_lane_f32(v, 2), vgetq_lane_f32(v, 3)));\n}\n\ninline static int32x4_t vcvtnq_s32_f32(float32x4_t v) {\n    int32x4_t res;\n\n    res[0] = roundf(vgetq_lane_f32(v, 0));\n    res[1] = roundf(vgetq_lane_f32(v, 1));\n    res[2] = roundf(vgetq_lane_f32(v, 2));\n    res[3] = roundf(vgetq_lane_f32(v, 3));\n\n    return res;\n}\n\ninline static uint8x8_t vzip1_u8(uint8x8_t a, uint8x8_t b) {\n    uint8x8_t res;\n\n    res[0] = a[0]; res[1] = b[0];\n    res[2] = a[1]; res[3] = b[1];\n    res[4] = a[2]; res[5] = b[2];\n    res[6] = a[3]; res[7] = b[3];\n\n    return res;\n}\n\ninline static uint8x8_t vzip2_u8(uint8x8_t a, uint8x8_t b) {\n    uint8x8_t res;\n\n    res[0] = a[4]; res[1] = b[4];\n    res[2] = a[5]; res[3] = b[5];\n    res[4] = a[6]; res[5] = b[6];\n    res[6] = a[7]; res[7] = b[7];\n\n    return res;\n}\n\n// vld1q_s16_x2\n// vld1q_u8_x2\n// vld1q_u8_x4\n// vld1q_s8_x2\n// vld1q_s8_x4\n// TODO: double-check these work correctly\n\ntypedef struct ggml_int16x8x2_t {\n    int16x8_t val[2];\n} ggml_int16x8x2_t;\n\ninline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) {\n    ggml_int16x8x2_t res;\n\n    res.val[0] = vld1q_s16(ptr + 0);\n    res.val[1] = vld1q_s16(ptr + 8);\n\n    return res;\n}\n\ntypedef struct ggml_uint8x16x2_t {\n    uint8x16_t val[2];\n} ggml_uint8x16x2_t;\n\ninline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) {\n    ggml_uint8x16x2_t res;\n\n    res.val[0] = vld1q_u8(ptr + 0);\n    res.val[1] = vld1q_u8(ptr + 16);\n\n    return res;\n}\n\ntypedef struct ggml_uint8x16x4_t {\n    uint8x16_t val[4];\n} ggml_uint8x16x4_t;\n\ninline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) {\n    ggml_uint8x16x4_t res;\n\n    res.val[0] = vld1q_u8(ptr + 0);\n    res.val[1] = vld1q_u8(ptr + 16);\n    res.val[2] = vld1q_u8(ptr + 32);\n    res.val[3] = vld1q_u8(ptr + 48);\n\n    return res;\n}\n\ntypedef struct ggml_int8x16x2_t {\n    int8x16_t val[2];\n} ggml_int8x16x2_t;\n\ninline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) {\n    ggml_int8x16x2_t res;\n\n    res.val[0] = vld1q_s8(ptr + 0);\n    res.val[1] = vld1q_s8(ptr + 16);\n\n    return res;\n}\n\ntypedef struct ggml_int8x16x4_t {\n    int8x16_t val[4];\n} ggml_int8x16x4_t;\n\ninline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) {\n    ggml_int8x16x4_t res;\n\n    res.val[0] = vld1q_s8(ptr + 0);\n    res.val[1] = vld1q_s8(ptr + 16);\n    res.val[2] = vld1q_s8(ptr + 32);\n    res.val[3] = vld1q_s8(ptr + 48);\n\n    return res;\n}\n\n// NOTE: not tested\ninline static int8x16_t ggml_vqtbl1q_s8(int8x16_t a, uint8x16_t b) {\n    int8x16_t res;\n\n    res[ 0] = a[b[ 0]];\n    res[ 1] = a[b[ 1]];\n    res[ 2] = a[b[ 2]];\n    res[ 3] = a[b[ 3]];\n    res[ 4] = a[b[ 4]];\n    res[ 5] = a[b[ 5]];\n    res[ 6] = a[b[ 6]];\n    res[ 7] = a[b[ 7]];\n    res[ 8] = a[b[ 8]];\n    res[ 9] = a[b[ 9]];\n    res[10] = a[b[10]];\n    res[11] = a[b[11]];\n    res[12] = a[b[12]];\n    res[13] = a[b[13]];\n    res[14] = a[b[14]];\n    res[15] = a[b[15]];\n\n    return res;\n}\n\n// NOTE: not tested\ninline static uint8x16_t ggml_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) {\n    uint8x16_t res;\n\n    res[ 0] = a[b[ 0]];\n    res[ 1] = a[b[ 1]];\n    res[ 2] = a[b[ 2]];\n    res[ 3] = a[b[ 3]];\n    res[ 4] = a[b[ 4]];\n    res[ 5] = a[b[ 5]];\n    res[ 6] = a[b[ 6]];\n    res[ 7] = a[b[ 7]];\n    res[ 8] = a[b[ 8]];\n    res[ 9] = a[b[ 9]];\n    res[10] = a[b[10]];\n    res[11] = a[b[11]];\n    res[12] = a[b[12]];\n    res[13] = a[b[13]];\n    res[14] = a[b[14]];\n    res[15] = a[b[15]];\n\n    return res;\n}\n\n#else\n\n#define ggml_int16x8x2_t  int16x8x2_t\n#define ggml_uint8x16x2_t uint8x16x2_t\n#define ggml_uint8x16x4_t uint8x16x4_t\n#define ggml_int8x16x2_t  int8x16x2_t\n#define ggml_int8x16x4_t  int8x16x4_t\n\n#define ggml_vld1q_s16_x2 vld1q_s16_x2\n#define ggml_vld1q_u8_x2  vld1q_u8_x2\n#define ggml_vld1q_u8_x4  vld1q_u8_x4\n#define ggml_vld1q_s8_x2  vld1q_s8_x2\n#define ggml_vld1q_s8_x4  vld1q_s8_x4\n#define ggml_vqtbl1q_s8   vqtbl1q_s8\n#define ggml_vqtbl1q_u8   vqtbl1q_u8\n\n#endif // !defined(__aarch64__)\n\n#if !defined(__ARM_FEATURE_DOTPROD)\n\ninline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) {\n    const int16x8_t p0 = vmull_s8(vget_low_s8 (a), vget_low_s8 (b));\n    const int16x8_t p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b));\n\n    return vaddq_s32(acc, vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)));\n}\n\n#else\n\n#define ggml_vdotq_s32(a, b, c) vdotq_s32(a, b, c)\n\n#endif // !defined(__ARM_FEATURE_DOTPROD)\n\n#endif // defined(__ARM_NEON)\n\n#ifdef __wasm_simd128__\n#include <wasm_simd128.h>\n#endif\n\n#ifdef __POWER9_VECTOR__\n#include <altivec.h>\n#endif\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n#include <intrin.h>\n#elif defined(__SSE__) || defined(__SSE3__) || defined(__SSSE3__) || defined(__AVX__) || defined(__F16C__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX512BF16__)\n#include <immintrin.h>\n#endif\n\n#ifdef __riscv_v_intrinsic\n#include <riscv_vector.h>\n#endif\n\n#if defined(__loongarch64)\n#if defined(__loongarch_asx)\n#include <lasxintrin.h>\n#endif\n#if defined(__loongarch_sx)\n#include <lsxintrin.h>\n#endif\n#endif\n\n#if defined(__VXE__) || defined(__VXE2__)\n#include <vecintrin.h>\n\n#define vec_neg(a)    (-(a))                // Vector Negate\n#define vec_add(a, b) ((a) + (b))           // Vector Add\n#define vec_sub(a, b) ((a) - (b))           // Vector Subtract\n#define vec_mul(a, b) ((a) * (b))           // Vector Multiply\n#define vec_div(a, b) ((a) / (b))           // Vector Divide\n#define vec_sl(a, b)  ((a) << (b))          // Vector Shift Left\n#define vec_sra(a, b) ((a) >> (b))          // Vector Shift Right\n#define vec_sr(a, b)  ((a) >> (b))          // Vector Shift Right Algebraic\n#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet\n#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet\n\n#ifndef vec_and\n#define vec_and(a, b) ((a) & (b)) // Vector AND\n#endif\n\n#ifndef vec_or\n#define vec_or(a, b)  ((a) | (b)) // Vector OR\n#endif\n\n#ifndef vec_xor\n#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR\n#endif\n\ntypedef signed   char char8x16_t  __attribute__((vector_size(16)));\ntypedef unsigned char uchar8x16_t __attribute__((vector_size(16)));\n\ntypedef int8_t  int8x16_t __attribute__((vector_size(16)));\ntypedef int16_t int16x8_t __attribute__((vector_size(16)));\ntypedef int32_t int32x4_t __attribute__((vector_size(16)));\n\ntypedef uint8_t  uint8x16_t __attribute__((vector_size(16)));\ntypedef uint16_t uint16x8_t __attribute__((vector_size(16)));\ntypedef uint32_t uint32x4_t __attribute__((vector_size(16)));\n\ntypedef float  float32x4_t  __attribute__((vector_size(16)));\ntypedef double double64x2_t __attribute__((vector_size(16)));\n\ntypedef signed   long long long64x2_t  __attribute__((vector_size(16)));\ntypedef unsigned long long ulong64x2_t __attribute__((vector_size(16)));\n\ntypedef struct ggml_uint8x16x2_t {\n    uint8x16_t val[2];\n} ggml_uint8x16x2_t;\n\ninline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) {\n    ggml_uint8x16x2_t res;\n\n    res.val[0] = vec_xl( 0, ptr);\n    res.val[1] = vec_xl(16, ptr);\n\n    return res;\n}\n\ntypedef struct ggml_uint8x16x4_t {\n    uint8x16_t val[4];\n} ggml_uint8x16x4_t;\n\ninline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) {\n    ggml_uint8x16x4_t res;\n\n    res.val[0] = vec_xl( 0, ptr);\n    res.val[1] = vec_xl(16, ptr);\n    res.val[2] = vec_xl(32, ptr);\n    res.val[3] = vec_xl(48, ptr);\n\n    return res;\n}\n\ntypedef struct ggml_int8x16x4_t {\n    int8x16_t val[4];\n} ggml_int8x16x4_t;\n\ninline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) {\n    ggml_int8x16x4_t res;\n\n    res.val[0] = vec_xl( 0, ptr);\n    res.val[1] = vec_xl(16, ptr);\n    res.val[2] = vec_xl(32, ptr);\n    res.val[3] = vec_xl(48, ptr);\n\n    return res;\n}\n\ntypedef struct ggml_int16x8x2_t {\n    int16x8_t val[2];\n} ggml_int16x8x2_t;\n\ninline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) {\n    ggml_int16x8x2_t res;\n\n    res.val[0] = vec_xl( 0, ptr);\n    res.val[1] = vec_xl(16, ptr);\n\n    return res;\n}\n\n/*\n    ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs\n    !          or iq4_nl for example implementation.\n*/\ninline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) {\n    int8x16_t res;\n\n    res[ 0] = a[b[ 0]];\n    res[ 1] = a[b[ 1]];\n    res[ 2] = a[b[ 2]];\n    res[ 3] = a[b[ 3]];\n    res[ 4] = a[b[ 4]];\n    res[ 5] = a[b[ 5]];\n    res[ 6] = a[b[ 6]];\n    res[ 7] = a[b[ 7]];\n    res[ 8] = a[b[ 8]];\n    res[ 9] = a[b[ 9]];\n    res[10] = a[b[10]];\n    res[11] = a[b[11]];\n    res[12] = a[b[12]];\n    res[13] = a[b[13]];\n    res[14] = a[b[14]];\n    res[15] = a[b[15]];\n\n    return res;\n}\n\ninline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) {\n    const uchar8x16_t v_maske = {  0,  1,  4,  5,  8,  9, 12, 13,\n                                  16, 17, 20, 21, 24, 25, 28, 29 };\n\n    const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b);\n    const int16x8_t v_abe = vec_perm(a, b, v_maske);\n    return v_abo + v_abe;\n}\n\n/**\n * @see https://github.com/ggml-org/llama.cpp/pull/14037\n */\ninline static float vec_hsum_f32x4(float32x4_t v) {\n    float32x4_t v_temp = v + vec_reve(v);\n    return v_temp[0] + v_temp[1];\n}\n\ninline static int32_t vec_hsum_i32x4(int32x4_t v) {\n    int32x4_t v_temp = v + vec_reve(v);\n    return v_temp[0] + v_temp[1];\n}\n\ninline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) {\n    const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b);\n    return acc + (vec_unpackh(p) + vec_unpackl(p));\n}\n\n#endif\n\n#if defined(__loongarch_sx)\n/* float type data load instructions */\nstatic __m128 __lsx_vreplfr2vr_s(const float val) {\n    v4f32 res = {val, val, val, val};\n    return (__m128)res;\n}\n#endif\n\n#if defined(__loongarch_asx)\nstatic __m256 __lasx_xvreplfr2vr_s(const float val) {\n    v8f32 res = {val, val, val, val, val, val, val, val};\n    return (__m256)res;\n}\n#endif\n\n// TODO: move to ggml-threading\nvoid ggml_barrier(struct ggml_threadpool * tp);\n\nvoid ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value);\nint  ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/ggml-cpu.c",
    "content": "#define _CRT_SECURE_NO_DEPRECATE // Disables \"unsafe\" warnings on Windows\n#define _USE_MATH_DEFINES // For M_PI on MSVC\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n#include \"traits.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-impl.h\"\n#include \"quants.h\"\n#include \"ggml-threading.h\"\n#include \"unary-ops.h\"\n#include \"binary-ops.h\"\n#include \"vec.h\"\n#include \"ops.h\"\n#include \"ggml.h\"\n#include \"common.h\"\n\n#if defined(_MSC_VER) || defined(__MINGW32__)\n#include <malloc.h> // using malloc.h with MSC/MINGW\n#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)\n#include <alloca.h>\n#endif\n\n#include <assert.h>\n#include <errno.h>\n#include <time.h>\n#include <math.h>\n#include <stdlib.h>\n#include <string.h>\n#include <stdint.h>\n#include <inttypes.h>\n#include <stdio.h>\n#include <float.h>\n#include <limits.h>\n#include <stdarg.h>\n#include <signal.h>\n#if defined(__gnu_linux__)\n#include <syscall.h>\n#endif\n\n#ifdef GGML_USE_OPENMP\n#include <omp.h>\n#endif\n\n#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)\n#undef GGML_USE_LLAMAFILE\n#endif\n\n#ifdef GGML_USE_LLAMAFILE\n#include \"llamafile/sgemm.h\"\n#endif\n\n// Note: once we move threading into a separate C++ file\n// will use std::hardware_destructive_interference_size instead of hardcoding it here\n// and we'll use C++ attribute syntax.\n#define GGML_CACHE_LINE  64\n\n#if defined(__clang__) || defined(__GNUC__)\n#define GGML_CACHE_ALIGN __attribute__((aligned(GGML_CACHE_LINE)))\n#endif\n\n#if defined(__has_feature)\n#if __has_feature(thread_sanitizer)\n#define GGML_TSAN_ENABLED 1\n#endif\n#else  // __has_feature\n#if defined(__SANITIZE_THREAD__)\n#define GGML_TSAN_ENABLED 1\n#endif\n#endif // __has_feature\n\n#define UNUSED GGML_UNUSED\n#define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)\n\n// precomputed f32 table for f16 (256 KB) (simd-mappings.h)\nfloat ggml_table_f32_f16[1 << 16];\n\n// precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h)\nfloat ggml_table_f32_e8m0_half[1 << 8];\n\n#if defined(__ARM_ARCH)\nstruct ggml_arm_arch_features_type {\n    int sve_cnt;\n} ggml_arm_arch_features = { 0 };\n#endif\n\n#if defined(__riscv)\nstruct ggml_riscv_arch_features_type {\n    int rvv_vlen;\n} ggml_riscv_arch_features = { 0 };\n#endif\n\n#if defined(_WIN32)\n\n#define WIN32_LEAN_AND_MEAN\n#ifndef NOMINMAX\n    #define NOMINMAX\n#endif\n#include <windows.h>\n\n#if defined(_MSC_VER) && !defined(__clang__)\n#define GGML_CACHE_ALIGN __declspec(align(GGML_CACHE_LINE))\n\ntypedef volatile LONG atomic_int;\ntypedef atomic_int atomic_bool;\ntypedef atomic_int atomic_flag;\n\n#define ATOMIC_FLAG_INIT 0\n\ntypedef enum {\n    memory_order_relaxed,\n    memory_order_consume,\n    memory_order_acquire,\n    memory_order_release,\n    memory_order_acq_rel,\n    memory_order_seq_cst\n} memory_order;\n\nstatic void atomic_store(atomic_int * ptr, LONG val) {\n    InterlockedExchange(ptr, val);\n}\nstatic void atomic_store_explicit(atomic_int * ptr, LONG val, memory_order mo) {\n    // TODO: add support for explicit memory order\n    InterlockedExchange(ptr, val);\n}\nstatic LONG atomic_load(atomic_int * ptr) {\n    return InterlockedCompareExchange(ptr, 0, 0);\n}\nstatic LONG atomic_load_explicit(atomic_int * ptr, memory_order mo) {\n    // TODO: add support for explicit memory order\n    return InterlockedCompareExchange(ptr, 0, 0);\n}\nstatic LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {\n    return InterlockedExchangeAdd(ptr, inc);\n}\nstatic LONG atomic_fetch_add_explicit(atomic_int * ptr, LONG inc, memory_order mo) {\n    // TODO: add support for explicit memory order\n    return InterlockedExchangeAdd(ptr, inc);\n}\nstatic atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {\n    return InterlockedExchange(ptr, 1);\n}\nstatic void atomic_flag_clear(atomic_flag * ptr) {\n    InterlockedExchange(ptr, 0);\n}\nstatic void atomic_thread_fence(memory_order mo) {\n    MemoryBarrier();\n}\n#else // clang\n#include <stdatomic.h>\n#endif\n\ntypedef HANDLE pthread_t;\n\ntypedef DWORD thread_ret_t;\nstatic int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {\n    (void) unused;\n    HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);\n    if (handle == NULL)\n    {\n        return EAGAIN;\n    }\n\n    *out = handle;\n    return 0;\n}\n\nstatic int pthread_join(pthread_t thread, void * unused) {\n    (void) unused;\n    int ret = (int) WaitForSingleObject(thread, INFINITE);\n    CloseHandle(thread);\n    return ret;\n}\n\nstatic int sched_yield (void) {\n    Sleep (0);\n    return 0;\n}\n#else\n\n#include <pthread.h>\n#include <stdatomic.h>\n#include <sched.h>\n#if defined(__FreeBSD__)\n#include <pthread_np.h>\n#endif\n\ntypedef void * thread_ret_t;\n\n#include <sys/types.h>\n#include <sys/stat.h>\n#include <unistd.h>\n\n#endif\n\ntypedef pthread_t ggml_thread_t;\n\n#define GGML_THREADPOOL_N_THREADS_MASK (0xffffU)\n#define GGML_THREADPOOL_N_THREADS_BITS (16)\n\n#if defined(__APPLE__)\n#include <unistd.h>\n#include <mach/mach.h>\n#include <TargetConditionals.h>\n#endif\n\nstatic const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {\n    [GGML_TYPE_F32] = {\n        .from_float               = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,\n        .vec_dot                  = (ggml_vec_dot_t) ggml_vec_dot_f32,\n        .vec_dot_type             = GGML_TYPE_F32,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_F16] = {\n        .from_float               = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,\n        .vec_dot                  = (ggml_vec_dot_t) ggml_vec_dot_f16,\n        .vec_dot_type             = GGML_TYPE_F16,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q4_0] = {\n        .from_float               = quantize_row_q4_0,\n        .vec_dot                  = ggml_vec_dot_q4_0_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n        .nrows                    = 2,\n#else\n        .nrows                    = 1,\n#endif\n    },\n    [GGML_TYPE_Q4_1] = {\n        .from_float               = quantize_row_q4_1,\n        .vec_dot                  = ggml_vec_dot_q4_1_q8_1,\n        .vec_dot_type             = GGML_TYPE_Q8_1,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n        .nrows                    = 2,\n#else\n        .nrows                    = 1,\n#endif\n    },\n    [GGML_TYPE_Q5_0] = {\n        .from_float               = quantize_row_q5_0,\n        .vec_dot                  = ggml_vec_dot_q5_0_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q5_1] = {\n        .from_float               = quantize_row_q5_1,\n        .vec_dot                  = ggml_vec_dot_q5_1_q8_1,\n        .vec_dot_type             = GGML_TYPE_Q8_1,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q8_0] = {\n        .from_float               = quantize_row_q8_0,\n        .vec_dot                  = ggml_vec_dot_q8_0_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n        .nrows                    = 2,\n#else\n        .nrows                    = 1,\n#endif\n    },\n    [GGML_TYPE_Q8_1] = {\n        .from_float               = quantize_row_q8_1,\n        .vec_dot_type             = GGML_TYPE_Q8_1,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_MXFP4] = {\n        .from_float               = quantize_row_mxfp4,\n        .vec_dot                  = ggml_vec_dot_mxfp4_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_NVFP4] = {\n        .from_float               = quantize_row_nvfp4,\n        .vec_dot                  = ggml_vec_dot_nvfp4_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q2_K] = {\n        .from_float               = quantize_row_q2_K,\n        .vec_dot                  = ggml_vec_dot_q2_K_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q3_K] = {\n        .from_float               = quantize_row_q3_K,\n        .vec_dot                  = ggml_vec_dot_q3_K_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q4_K] = {\n        .from_float               = quantize_row_q4_K,\n        .vec_dot                  = ggml_vec_dot_q4_K_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n        .nrows                    = 2,\n#else\n        .nrows                    = 1,\n#endif\n    },\n    [GGML_TYPE_Q5_K] = {\n        .from_float               = quantize_row_q5_K,\n        .vec_dot                  = ggml_vec_dot_q5_K_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q6_K] = {\n        .from_float               = quantize_row_q6_K,\n        .vec_dot                  = ggml_vec_dot_q6_K_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n#if defined (__ARM_FEATURE_MATMUL_INT8)\n        .nrows                    = 2,\n#else\n        .nrows                    = 1,\n#endif\n    },\n    [GGML_TYPE_IQ2_XXS] = {\n        .from_float               = NULL,\n        .vec_dot                  = ggml_vec_dot_iq2_xxs_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ2_XS] = {\n        .from_float               = NULL,\n        .vec_dot                  = ggml_vec_dot_iq2_xs_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ3_XXS] = {\n        // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init\n        //.from_float               = quantize_row_iq3_xxs,\n        .vec_dot                  = ggml_vec_dot_iq3_xxs_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ3_S] = {\n        //.from_float               = quantize_row_iq3_s,\n        .vec_dot                  = ggml_vec_dot_iq3_s_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ2_S] = {\n        //.from_float               = quantize_row_iq2_s,\n        .vec_dot                  = ggml_vec_dot_iq2_s_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ1_S] = {\n        .from_float               = NULL,\n        .vec_dot                  = ggml_vec_dot_iq1_s_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ1_M] = {\n        .from_float               = NULL,\n        .vec_dot                  = ggml_vec_dot_iq1_m_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ4_NL] = {\n        .from_float               = quantize_row_iq4_nl,\n        .vec_dot                  = ggml_vec_dot_iq4_nl_q8_0,\n        .vec_dot_type             = GGML_TYPE_Q8_0,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_IQ4_XS] = {\n        .from_float               = quantize_row_iq4_xs,\n        .vec_dot                  = ggml_vec_dot_iq4_xs_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_Q8_K] = {\n        .from_float               = quantize_row_q8_K,\n    },\n    [GGML_TYPE_BF16] = {\n        .from_float               = (ggml_from_float_t) ggml_cpu_fp32_to_bf16,\n        .vec_dot                  = (ggml_vec_dot_t) ggml_vec_dot_bf16,\n        .vec_dot_type             = GGML_TYPE_BF16,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_TQ1_0] = {\n        .from_float               = quantize_row_tq1_0,\n        .vec_dot                  = ggml_vec_dot_tq1_0_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_TQ2_0] = {\n        .from_float               = quantize_row_tq2_0,\n        .vec_dot                  = ggml_vec_dot_tq2_0_q8_K,\n        .vec_dot_type             = GGML_TYPE_Q8_K,\n        .nrows                    = 1,\n    },\n    [GGML_TYPE_I32] = {\n        .from_float               = (ggml_from_float_t) ggml_cpu_fp32_to_i32,\n    },\n};\n\nconst struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {\n    return &type_traits_cpu[type];\n}\n\n//\n// Threading defs\n//\n\ntypedef pthread_t          ggml_thread_t;\n\n#if defined(_WIN32)\n\ntypedef CONDITION_VARIABLE ggml_cond_t;\ntypedef SRWLOCK            ggml_mutex_t;\n\n#define ggml_mutex_init(m)   InitializeSRWLock(m)\n#define ggml_mutex_destroy(m)\n#define ggml_mutex_lock(m)   AcquireSRWLockExclusive(m)\n#define ggml_mutex_unlock(m) ReleaseSRWLockExclusive(m)\n#define ggml_mutex_lock_shared(m)   AcquireSRWLockShared(m)\n#define ggml_mutex_unlock_shared(m) ReleaseSRWLockShared(m)\n\n#define ggml_cond_init(c)    InitializeConditionVariable(c)\n#define ggml_cond_destroy(c)\n#define ggml_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)\n#define ggml_cond_broadcast(c) WakeAllConditionVariable(c)\n\n#define ggml_thread_create pthread_create\n#define ggml_thread_join   pthread_join\n\n#else\n\ntypedef pthread_cond_t     ggml_cond_t;\ntypedef pthread_mutex_t    ggml_mutex_t;\n\n#define ggml_mutex_init(m)          pthread_mutex_init(m, NULL)\n#define ggml_mutex_destroy(m)       pthread_mutex_destroy(m)\n#define ggml_mutex_lock(m)          pthread_mutex_lock(m)\n#define ggml_mutex_unlock(m)        pthread_mutex_unlock(m)\n#define ggml_mutex_lock_shared(m)   pthread_mutex_lock(m)\n#define ggml_mutex_unlock_shared(m) pthread_mutex_unlock(m)\n\n#define ggml_lock_init(x)    UNUSED(x)\n#define ggml_lock_destroy(x) UNUSED(x)\n#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))\n#define ggml_lock_lock(x)    _mm_pause()\n#else\n#define ggml_lock_lock(x)    UNUSED(x)\n#endif\n#define ggml_lock_unlock(x)  UNUSED(x)\n\n#define GGML_LOCK_INITIALIZER 0\n#define ggml_cond_init(c)      pthread_cond_init(c, NULL)\n#define ggml_cond_destroy(c)   pthread_cond_destroy(c)\n#define ggml_cond_wait(c, m)   pthread_cond_wait(c, m)\n#define ggml_cond_broadcast(c) pthread_cond_broadcast(c)\n\n#define ggml_thread_create pthread_create\n#define ggml_thread_join   pthread_join\n\n#endif\n\n// Threadpool def\nstruct ggml_threadpool {\n    ggml_mutex_t mutex;       // mutex for cond.var\n    ggml_cond_t  cond;        // cond.var for waiting for new work\n\n    struct ggml_cgraph * cgraph;\n    struct ggml_cplan  * cplan;\n\n    // synchronization primitives\n    atomic_int n_graph;       // updated when there is work to be done (i.e each graph) holds graph and active thread counts.\n    atomic_int GGML_CACHE_ALIGN n_barrier;\n    atomic_int GGML_CACHE_ALIGN n_barrier_passed;\n    atomic_int GGML_CACHE_ALIGN current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.\n\n    // these are atomic as an annotation for thread-sanitizer\n    atomic_bool stop;         // Used for stopping the threadpool altogether\n    atomic_bool pause;        // Used for pausing the threadpool or individual threads\n    atomic_int  abort;        // Used for aborting processing of a graph\n\n    struct ggml_compute_state * workers;   // per thread state\n    int          n_threads;   // Number of threads in the pool\n    int32_t      prio;        // Scheduling priority\n    uint32_t     poll;        // Polling level (0 - no polling)\n\n    enum ggml_status ec;\n};\n\n// Per-thread state\nstruct ggml_compute_state {\n#ifndef GGML_USE_OPENMP\n    ggml_thread_t thrd;\n    int  last_graph;\n    bool pending;\n#endif\n    bool cpumask[GGML_MAX_N_THREADS];\n    struct ggml_threadpool * threadpool;\n    int ith;\n};\n\n// Helpers for polling loops\n#if defined(__aarch64__) && ( defined(__clang__) || defined(__GNUC__) )\nstatic inline void ggml_thread_cpu_relax(void) {\n    __asm__ volatile(\"yield\" ::: \"memory\");\n}\n#elif defined(__x86_64__)\nstatic inline void ggml_thread_cpu_relax(void) {\n    _mm_pause();\n}\n#elif defined(__riscv)\nstatic inline void ggml_thread_cpu_relax(void) {\n    #ifdef __riscv_zihintpause\n        __asm__ __volatile__ (\"pause\");\n    #else\n        /* Encoding of the pause instruction */\n        __asm__ __volatile__ (\".4byte 0x100000F\");\n    #endif\n}\n#else\nstatic inline void ggml_thread_cpu_relax(void) {;}\n#endif\n\n//\n// NUMA support\n//\n\n#define GGML_NUMA_MAX_NODES 8\n#define GGML_NUMA_MAX_CPUS 512\n\nstruct ggml_numa_node {\n    uint32_t cpus[GGML_NUMA_MAX_CPUS]; // hardware threads on this node\n    uint32_t n_cpus;\n};\n\nstruct ggml_numa_nodes {\n    enum ggml_numa_strategy numa_strategy;\n    struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];\n    uint32_t n_nodes;\n    uint32_t total_cpus; // hardware threads on system\n    uint32_t current_node; // node on which main process is execting\n#if defined(__gnu_linux__)\n    cpu_set_t cpuset; // cpuset from numactl\n#else\n    uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype\n#endif\n};\n\n//\n// ggml state\n//\n\nstruct ggml_state {\n    struct ggml_numa_nodes numa;\n};\n\nstatic struct ggml_state g_state = {0};\n\nvoid ggml_barrier(struct ggml_threadpool * tp) {\n    int n_threads = atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK;\n    if (n_threads == 1) {\n        return;\n    }\n\n#ifdef GGML_USE_OPENMP\n    #pragma omp barrier\n#else\n    int n_passed = atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed);\n\n    // enter barrier (full seq-cst fence)\n    int n_barrier = atomic_fetch_add_explicit(&tp->n_barrier, 1, memory_order_seq_cst);\n\n    if (n_barrier == (n_threads - 1)) {\n        // last thread\n        atomic_store_explicit(&tp->n_barrier, 0, memory_order_relaxed);\n\n        // exit barrier (full seq-cst fence)\n        atomic_fetch_add_explicit(&tp->n_barrier_passed, 1, memory_order_seq_cst);\n        return;\n    }\n\n    // wait for other threads\n    while (atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) {\n        ggml_thread_cpu_relax();\n    }\n\n    // exit barrier (full seq-cst fence)\n    // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead\n    #ifdef GGML_TSAN_ENABLED\n    atomic_fetch_add_explicit(&tp->n_barrier_passed, 0, memory_order_seq_cst);\n    #else\n    atomic_thread_fence(memory_order_seq_cst);\n    #endif\n#endif\n}\n\nvoid ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) {\n    atomic_store_explicit(&tp->current_chunk, value, memory_order_relaxed);\n}\n\nint ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) {\n    return atomic_fetch_add_explicit(&tp->current_chunk, value, memory_order_relaxed);\n}\n\n#if defined(__gnu_linux__)\nstatic cpu_set_t ggml_get_numa_affinity(void) {\n    cpu_set_t cpuset;\n    pthread_t thread;\n    thread = pthread_self();\n    CPU_ZERO(&cpuset);\n    pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);\n    return cpuset;\n}\n#else\nstatic uint32_t ggml_get_numa_affinity(void) {\n    return 0; // no NUMA support\n}\n#endif\n\nvoid ggml_numa_init(enum ggml_numa_strategy numa_flag) {\n    if (g_state.numa.n_nodes > 0) {\n        fprintf(stderr, \"ggml_numa_init: NUMA already initialized\\n\");\n\n        return;\n    }\n\n#if defined(__gnu_linux__)\n    struct stat st;\n    char path[256];\n    int rv;\n\n    // set numa scheme\n    g_state.numa.numa_strategy = numa_flag;\n\n    GGML_PRINT_DEBUG(\"numa strategy %u\\n\",g_state.numa.numa_strategy);\n\n    g_state.numa.cpuset = ggml_get_numa_affinity();\n\n    // enumerate nodes\n    while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {\n        rv = snprintf(path, sizeof(path), \"/sys/devices/system/node/node%u\", g_state.numa.n_nodes);\n        GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n        if (stat(path, &st) != 0) { break; }\n        ++g_state.numa.n_nodes;\n    }\n\n    // enumerate CPUs\n    while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS) {\n        rv = snprintf(path, sizeof(path), \"/sys/devices/system/cpu/cpu%u\", g_state.numa.total_cpus);\n        GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n        if (stat(path, &st) != 0) { break; }\n        ++g_state.numa.total_cpus;\n    }\n\n    GGML_PRINT_DEBUG(\"found %u numa nodes, %u CPUs\\n\", g_state.numa.n_nodes, g_state.numa.total_cpus);\n\n    // figure out which node we're on\n    uint current_cpu;\n    int getcpu_ret = 0;\n#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 33) || defined(__COSMOPOLITAN__)\n    getcpu_ret = getcpu(&current_cpu, &g_state.numa.current_node);\n#else\n    // old glibc doesn't have a wrapper for this call. Fall back on direct syscall\n#   if !defined(SYS_getcpu) && defined(SYS_get_cpu)\n#       define SYS_getcpu SYS_get_cpu // some older glibc versions use this name\n#   endif\n    getcpu_ret = syscall(SYS_getcpu, &current_cpu, &g_state.numa.current_node);\n#endif\n\n    if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {\n        g_state.numa.n_nodes = 0;\n        return;\n    }\n\n    GGML_PRINT_DEBUG(\"found our process on numa node %u, CPU %u\\n\", g_state.numa.current_node, current_cpu);\n\n    for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {\n        struct ggml_numa_node * node = &g_state.numa.nodes[n];\n        GGML_PRINT_DEBUG(\"CPUs on node %u:\", n);\n        node->n_cpus = 0;\n        for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {\n            rv = snprintf(path, sizeof(path), \"/sys/devices/system/node/node%u/cpu%u\", n, c);\n            GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));\n            if (stat(path, &st) == 0) {\n                node->cpus[node->n_cpus++] = c;\n                GGML_PRINT_DEBUG(\" %u\", c);\n            }\n        }\n        GGML_PRINT_DEBUG(\"\\n\");\n    }\n\n    if (ggml_is_numa()) {\n        FILE *fptr = fopen(\"/proc/sys/kernel/numa_balancing\", \"r\");\n        if (fptr != NULL) {\n            char buf[42];\n            if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, \"0\\n\", sizeof(buf)) != 0) {\n                GGML_LOG_WARN(\"/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\\n\");\n            }\n            fclose(fptr);\n        }\n    }\n#else\n    UNUSED(numa_flag);\n    // TODO\n#endif\n}\n\nbool ggml_is_numa(void) {\n    return g_state.numa.n_nodes > 1;\n}\n\n#if defined(__ARM_ARCH)\n#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)\n#include <arm_sve.h>\nstatic void ggml_init_arm_arch_features(void) {\n    ggml_arm_arch_features.sve_cnt = svcntb();\n}\n#else\nstatic void ggml_init_arm_arch_features(void) {}\n#endif\n#endif // __ARM_ARCH\n\n#if defined(__riscv) && defined(__riscv_v_intrinsic)\n#include <riscv_vector.h>\nstatic void ggml_init_riscv_arch_features(void) {\n    ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb();\n}\n#else\nstatic void ggml_init_riscv_arch_features(void) {}\n#endif\n\nstruct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {\n    GGML_ASSERT(!ggml_get_no_alloc(ctx));\n\n    struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);\n\n    ggml_set_i32(result, value);\n\n    return result;\n}\n\nstruct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {\n    GGML_ASSERT(!ggml_get_no_alloc(ctx));\n\n    struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);\n\n    ggml_set_f32(result, value);\n\n    return result;\n}\n\nstruct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {\n    const int n     = ggml_nrows(tensor);\n    const int nc    = tensor->ne[0];\n    const size_t n1 = tensor->nb[1];\n\n    char * const data = tensor->data;\n\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                assert(tensor->nb[0] == sizeof(int8_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_I16:\n            {\n                assert(tensor->nb[0] == sizeof(int16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_I32:\n            {\n                assert(tensor->nb[0] == sizeof(int32_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_F16:\n            {\n                assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));\n                }\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));\n                }\n            } break;\n        case GGML_TYPE_F32:\n            {\n                assert(tensor->nb[0] == sizeof(float));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_f32(nc, (float *)(data + i*n1), value);\n                }\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n\n    return tensor;\n}\n\nstruct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {\n    const int n     = ggml_nrows(tensor);\n    const int nc    = tensor->ne[0];\n    const size_t n1 = tensor->nb[1];\n\n    char * const data = tensor->data;\n\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                assert(tensor->nb[0] == sizeof(int8_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_I16:\n            {\n                assert(tensor->nb[0] == sizeof(int16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_I32:\n            {\n                assert(tensor->nb[0] == sizeof(int32_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);\n                }\n            } break;\n        case GGML_TYPE_F16:\n            {\n                assert(tensor->nb[0] == sizeof(ggml_fp16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));\n                }\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                assert(tensor->nb[0] == sizeof(ggml_bf16_t));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));\n                }\n            } break;\n        case GGML_TYPE_F32:\n            {\n                assert(tensor->nb[0] == sizeof(float));\n                for (int i = 0; i < n; i++) {\n                    ggml_vec_set_f32(nc, (float *)(data + i*n1), value);\n                }\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n\n    return tensor;\n}\n\nint32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {\n    if (!ggml_is_contiguous(tensor)) {\n        int64_t id[4] = { 0, 0, 0, 0 };\n        ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n        return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]);\n    }\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));\n                return ((int8_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_I16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));\n                return ((int16_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_I32:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));\n                return ((int32_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_F16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));\n                return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);\n            }\n        case GGML_TYPE_BF16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));\n                return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);\n            }\n        case GGML_TYPE_F32:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(float));\n                return ((float *)(tensor->data))[i];\n            }\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nvoid ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {\n    if (!ggml_is_contiguous(tensor)) {\n        int64_t id[4] = { 0, 0, 0, 0 };\n        ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n        ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value);\n        return;\n    }\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));\n                ((int8_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_I16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));\n                ((int16_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_I32:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));\n                ((int32_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_F16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));\n                ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));\n                ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                GGML_ASSERT(tensor->nb[0] == sizeof(float));\n                ((float *)(tensor->data))[i] = value;\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nint32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {\n    void * data   = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            return ((int8_t *) data)[0];\n        case GGML_TYPE_I16:\n            return ((int16_t *) data)[0];\n        case GGML_TYPE_I32:\n            return ((int32_t *) data)[0];\n        case GGML_TYPE_F16:\n            return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);\n        case GGML_TYPE_BF16:\n            return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);\n        case GGML_TYPE_F32:\n            return ((float *) data)[0];\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n}\n\nvoid ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {\n    void * data   = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                ((int8_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_I16:\n            {\n                ((int16_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_I32:\n            {\n                ((int32_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ((float *)(data))[0] = value;\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nfloat ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {\n    if (!ggml_is_contiguous(tensor)) {\n        int64_t id[4] = { 0, 0, 0, 0 };\n        ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n        return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]);\n    }\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                return ((int8_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_I16:\n            {\n                return ((int16_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_I32:\n            {\n                return ((int32_t *)(tensor->data))[i];\n            }\n        case GGML_TYPE_F16:\n            {\n                return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);\n            }\n        case GGML_TYPE_BF16:\n            {\n                return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);\n            }\n        case GGML_TYPE_F32:\n            {\n                return ((float *)(tensor->data))[i];\n            }\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nvoid ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {\n    if (!ggml_is_contiguous(tensor)) {\n        int64_t id[4] = { 0, 0, 0, 0 };\n        ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);\n        ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);\n        return;\n    }\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                ((int8_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_I16:\n            {\n                ((int16_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_I32:\n            {\n                ((int32_t *)(tensor->data))[i] = value;\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ((float *)(tensor->data))[i] = value;\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nfloat ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {\n    void * data   = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            return ((int8_t *) data)[0];\n        case GGML_TYPE_I16:\n            return ((int16_t *) data)[0];\n        case GGML_TYPE_I32:\n            return ((int32_t *) data)[0];\n        case GGML_TYPE_F16:\n            return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);\n        case GGML_TYPE_BF16:\n            return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);\n        case GGML_TYPE_F32:\n            return ((float *) data)[0];\n        default:\n            GGML_ABORT(\"fatal error\");\n    }\n}\n\nvoid ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {\n    void * data   = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];\n    switch (tensor->type) {\n        case GGML_TYPE_I8:\n            {\n                ((int8_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_I16:\n            {\n                ((int16_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_I32:\n            {\n                ((int32_t *)(data))[0] = value;\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ((float *)(data))[0] = value;\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// ggml_compute_forward_mul_mat\n\nstatic void ggml_compute_forward_mul_mat_one_chunk(\n    const struct ggml_compute_params * params,\n    struct ggml_tensor * dst,\n    const enum ggml_type type,\n    const int64_t num_rows_per_vec_dot,\n    const int64_t ir0_start,\n    const int64_t ir0_end,\n    const int64_t ir1_start,\n    const int64_t ir1_end) {\n\n    const struct ggml_tensor * src0 = dst->src[0];\n    const struct ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const bool src1_cont = ggml_is_contiguous(src1);\n\n    ggml_vec_dot_t const vec_dot      = type_traits_cpu[type].vec_dot;\n    enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;\n\n    // broadcast factors\n    const int64_t r2 = ne12 / ne02;\n    const int64_t r3 = ne13 / ne03;\n\n    //printf(\"ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\\n\", ir0_start, ir0_end, ir1_start, ir1_end);\n\n    // threads with no work simply yield (not sure if it helps)\n    if (ir0_start >= ir0_end || ir1_start >= ir1_end) {\n        return;\n    }\n\n    const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n    const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n    assert(ne12 % ne02 == 0);\n    assert(ne13 % ne03 == 0);\n\n    // block-tiling attempt\n    const int64_t blck_0 = 16;\n    const int64_t blck_1 = 16;\n\n    const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;\n\n    // attempt to reduce false-sharing (does not seem to make a difference)\n    // 16 * 2, accounting for mmla kernels\n    float tmp[32];\n\n    for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {\n        for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {\n            for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {\n                const int64_t i13 = (ir1 / (ne12 * ne1));\n                const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;\n                const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);\n\n                // broadcast src0 into src1\n                const int64_t i03 = i13 / r3;\n                const int64_t i02 = i12 / r2;\n\n                const int64_t i1 = i11;\n                const int64_t i2 = i12;\n                const int64_t i3 = i13;\n\n                const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);\n\n                // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides\n                //       if it is, then we have either copied the data to params->wdata and made it contiguous or we are using\n                //       the original src1 data pointer, so we should index using the indices directly\n                // TODO: this is a bit of a hack, we should probably have a better way to handle this\n                const char * src1_col = (const char*)wdata +\n                    (src1_cont || src1->type != vec_dot_type\n                        ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size\n                        : (i11 * nb11 + i12 * nb12 + i13 * nb13));\n                float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));\n\n                //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {\n                //    vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);\n                //}\n\n                for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {\n                    vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);\n                }\n\n                for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {\n                    memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_mul_mat(\n        const struct ggml_compute_params * params,\n              struct ggml_tensor * dst) {\n\n    const struct ggml_tensor * src0 = dst->src[0];\n    const struct ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    enum ggml_type           const vec_dot_type         = type_traits_cpu[src0->type].vec_dot_type;\n    ggml_from_float_t        const from_float           = type_traits_cpu[vec_dot_type].from_float;\n    int64_t                  const vec_dot_num_rows     = type_traits_cpu[src0->type].nrows;\n\n    GGML_ASSERT(ne0 == ne01);\n    GGML_ASSERT(ne1 == ne11);\n    GGML_ASSERT(ne2 == ne12);\n    GGML_ASSERT(ne3 == ne13);\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == ggml_type_size(src0->type));\n    GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    // nb01 >= nb00 - src0 is not transposed\n    //   compute by src0 rows\n\n    // TODO: extract to \"extra_op\"\n#if GGML_USE_LLAMAFILE\n    // broadcast factors\n    const int64_t r2 = ne12 / ne02;\n    const int64_t r3 = ne13 / ne03;\n\n    const bool src1_cont = ggml_is_contiguous(src1);\n\n    if (src1_cont) {\n        for (int64_t i13 = 0; i13 < ne13; i13++)\n            for (int64_t i12 = 0; i12 < ne12; i12++)\n                if (!llamafile_sgemm(params,\n                                     ne01, ne11, ne00/ggml_blck_size(src0->type),\n                                     (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,\n                                     nb01/ggml_type_size(src0->type),\n                                     (const char *)src1->data + i12*nb12 + i13*nb13,\n                                     nb11/ggml_type_size(src1->type),\n                                     (char *)dst->data + i12*nb2 + i13*nb3,\n                                     nb1/ggml_type_size(dst->type),\n                                     src0->type,\n                                     src1->type,\n                                     dst->type))\n                    goto UseGgmlGemm1;\n        return;\n    }\nUseGgmlGemm1:;\n#endif\n\n    if (src1->type != vec_dot_type) {\n        char * wdata = params->wdata;\n\n        const size_t nbw0 = ggml_type_size(vec_dot_type);\n        const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);\n        const size_t nbw2 = nbw1*ne11;\n        const size_t nbw3 = nbw2*ne12;\n\n        assert(params->wsize >= ne13*nbw3);\n        GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n    #if 0\n        for (int64_t i13 = 0; i13 < ne13; ++i13) {\n            for (int64_t i12 = 0; i12 < ne12; ++i12) {\n                for (int64_t i11 = ith; i11 < ne11; i11 += nth) {\n                    from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),\n                               (void *)               (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),\n                                ne10);\n                }\n            }\n        }\n    #else\n        for (int64_t i13 = 0; i13 < ne13; ++i13) {\n            for (int64_t i12 = 0; i12 < ne12; ++i12) {\n                for (int64_t i11 = 0; i11 < ne11; ++i11) {\n                    size_t bs = ggml_blck_size(vec_dot_type);\n                    int64_t ne10_block_start = (ith * ne10/bs) / nth;\n                    int64_t ne10_block_end   = ((ith + 1) * ne10/bs) / nth;\n                    from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),\n                               (void *)               (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),\n                               (ne10_block_end - ne10_block_start) * bs);\n                }\n            }\n        }\n    #endif\n    }\n\n    if (ith == 0) {\n        // Every thread starts at ith, so the first unprocessed chunk is nth.  This save a bit of coordination right at the start.\n        atomic_store_explicit(&params->threadpool->current_chunk, nth, memory_order_relaxed);\n    }\n\n    ggml_barrier(params->threadpool);\n\n#if GGML_USE_LLAMAFILE\n    if (src1->type != vec_dot_type) {\n        const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n        const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n        for (int64_t i13 = 0; i13 < ne13; i13++)\n            for (int64_t i12 = 0; i12 < ne12; i12++)\n                if (!llamafile_sgemm(params,\n                                     ne01, ne11, ne00/ggml_blck_size(src0->type),\n                                     (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,\n                                     nb01/ggml_type_size(src0->type),\n                                     (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,\n                                     row_size/ggml_type_size(vec_dot_type),\n                                     (char *)dst->data + i12*nb2 + i13*nb3,\n                                     nb1/ggml_type_size(dst->type),\n                                     src0->type,\n                                     vec_dot_type,\n                                     dst->type))\n                    goto UseGgmlGemm2;\n        return;\n    }\nUseGgmlGemm2:;\n#endif\n\n    // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)\n    const int64_t nr0 = ne0;\n\n    // This is the size of the rest of the dimensions of the result\n    const int64_t nr1 = ne1 * ne2 * ne3;\n\n    // Now select a reasonable chunk size.\n    int chunk_size = 16;\n\n    // We need to step up the size if it's small\n    if (nr0 == 1 || nr1 == 1) {\n        chunk_size = 64;\n    }\n\n    // distribute the work across the inner or outer loop based on which one is larger\n    // The number of chunks in the 0/1 dim.\n    // CEIL(nr0/chunk_size)\n    int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;\n    int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;\n\n    // If the chunking is poor for the number of threads on this setup, scrap the whole plan.  Re-chunk it by thread.\n    //   Also, chunking by thread was measured to have perform better on NUMA systems.  See https://github.com/ggml-org/llama.cpp/pull/6915\n    //   In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.\n    if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {\n        // distribute the thread work across the inner or outer loop based on which one is larger\n        nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows\n        nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows\n    }\n\n    // The number of elements in each chunk\n    const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n    const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;\n\n    // The first chunk comes from our thread_id, the rest will get auto-assigned.\n    int current_chunk = ith;\n\n    while (current_chunk < nchunk0 * nchunk1) {\n        const int64_t ith0 = current_chunk % nchunk0;\n        const int64_t ith1 = current_chunk / nchunk0;\n\n        const int64_t ir0_start = dr0 * ith0;\n        const int64_t ir0_end = MIN(ir0_start + dr0, nr0);\n\n        const int64_t ir1_start = dr1 * ith1;\n        const int64_t ir1_end = MIN(ir1_start + dr1, nr1);\n\n        // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols\n        int64_t num_rows_per_vec_dot = vec_dot_num_rows;\n\n        // these checks are needed to avoid crossing dim1 boundaries\n        // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity\n        if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {\n            num_rows_per_vec_dot = 1;\n        }\n        ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);\n\n        if (nth >= nchunk0 * nchunk1) {\n            break;\n        }\n\n        current_chunk = atomic_fetch_add_explicit(&params->threadpool->current_chunk, 1, memory_order_relaxed);\n    }\n}\n\n// ggml_compute_forward_mul_mat_id\n\n#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)]\n\nstruct mmid_row_mapping {\n    int32_t i1;\n    int32_t i2;\n};\n\nstatic void ggml_compute_forward_mul_mat_id_one_chunk(\n    struct ggml_tensor * dst,\n    const struct ggml_tensor * src0,\n    const struct ggml_tensor * src1,\n    const struct ggml_tensor * ids,\n    const int64_t cur_a,\n    const int64_t ir0_start,\n    const int64_t ir0_end,\n    const int64_t ir1_start,\n    const int64_t ir1_end,\n    const char * src0_cur,\n    const struct mmid_row_mapping * matrix_rows,\n    const size_t row_size,\n    const bool src1_cont,\n    const void * wdata) {\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const enum ggml_type type = src0->type;\n\n    ggml_vec_dot_t    const vec_dot      = type_traits_cpu[type].vec_dot;\n    enum ggml_type    const vec_dot_type = type_traits_cpu[type].vec_dot_type;\n\n    const int64_t blck_0 = 16;\n    const int64_t blck_1 = 16;\n\n    float tmp[16];\n\n    for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {\n        for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {\n            for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ++ir1) {\n                const int64_t _i12 = ir1; // logical row index for this expert\n\n                struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12);\n                const int id       = row_mapping.i1; // selected expert index\n\n                const int64_t  i11 = id % ne11;\n                const int64_t  i12 = row_mapping.i2; // row index in src1\n\n                const int64_t  i1 = id;  // selected expert index\n                const int64_t  i2 = i12; // row\n\n                // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides\n                //       if it is, then we have either copied the data to params->wdata and made it contiguous or we are using\n                //       the original src1 data pointer, so we should index using the indices directly\n                // TODO: this is a bit of a hack, we should probably have a better way to handle this\n                const char * src1_col = (const char *) wdata +\n                    (src1_cont || src1->type != vec_dot_type\n                    ? (i11      + i12*ne11)*row_size\n                    : (i11*nb11 + i12*nb12));\n\n                float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));\n\n                for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {\n                    vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);\n                }\n\n                memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float));\n            }\n        }\n    }\n}\n\nstatic void * incr_ptr_aligned(void ** p, size_t size, size_t align) {\n\n    void * ptr = *p;\n    ptr = (void *) GGML_PAD((uintptr_t) ptr, align);\n    *p = (void *) ((char *) ptr + size);\n    return ptr;\n}\n\nstatic void ggml_compute_forward_mul_mat_id(\n        const struct ggml_compute_params * params,\n              struct ggml_tensor * dst) {\n\n    const struct ggml_tensor * src0 = dst->src[0];\n    const struct ggml_tensor * src1 = dst->src[1];\n    const struct ggml_tensor * ids = dst->src[2];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const enum ggml_type type = src0->type;\n\n    const bool src1_cont = ggml_is_contiguous(src1);\n\n    enum ggml_type    const vec_dot_type    = type_traits_cpu[type].vec_dot_type;\n    ggml_from_float_t const from_float      = type_traits_cpu[vec_dot_type].from_float;\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == ggml_type_size(type));\n    GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    // row groups\n    const int n_ids = ids->ne[0]; // n_expert_used\n    const int n_as  = ne02;       // n_expert\n\n    void * wdata_cur = params->wdata;\n\n    if (src1->type != vec_dot_type) {\n        incr_ptr_aligned(&wdata_cur, ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));\n    }\n\n    int64_t * matrix_row_counts = // [n_as]\n        incr_ptr_aligned(&wdata_cur, n_as*sizeof(int64_t), sizeof(int64_t));\n\n    struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]]\n        incr_ptr_aligned(&wdata_cur, n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), sizeof(int64_t));\n\n    char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]\n        incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);\n\n    GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));\n\n    if (src1->type != vec_dot_type) {\n        char * wdata = params->wdata;\n\n        const size_t nbw0 = ggml_type_size(vec_dot_type);\n        const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);\n        const size_t nbw2 = nbw1*ne11;\n        const size_t nbw3 = nbw2*ne12;\n\n        assert(params->wsize >= ne13*nbw3);\n        GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n#if 0\n        for (int64_t i13 = 0; i13 < ne13; ++i13) {\n            for (int64_t i12 = ith; i12 < ne12; i12 += nth) {\n                for (int64_t i11 = 0; i11 < ne11; ++i11) {\n                    from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),\n                               (void *)               (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),\n                               ne10);\n                }\n            }\n        }\n#else\n        for (int64_t i13 = 0; i13 < ne13; ++i13) {\n            for (int64_t i12 = 0; i12 < ne12; ++i12) {\n                for (int64_t i11 = 0; i11 < ne11; ++i11) {\n                    size_t bs = ggml_blck_size(vec_dot_type);\n                    int64_t ne10_block_start = (ith * ne10/bs) / nth;\n                    int64_t ne10_block_end   = ((ith + 1) * ne10/bs) / nth;\n                    from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),\n                               (void *)               (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),\n                               (ne10_block_end - ne10_block_start) * bs);\n                }\n            }\n        }\n#endif\n    }\n\n    if (ith == 0) {\n        // initialize matrix_row_counts\n        memset(matrix_row_counts, 0, n_as*sizeof(int64_t));\n\n        // group rows by src0 matrix\n        for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {\n            for (int id = 0; id < n_ids; ++id) {\n                const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);\n\n                assert(i02 >= 0 && i02 < n_as);\n\n                MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};\n                matrix_row_counts[i02] += 1;\n            }\n        }\n    }\n\n    // reset current_chunk\n    for (int cur_a = ith; cur_a < n_as; cur_a += nth) {\n        atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);\n        *current_chunk_ctr = nth;\n    }\n\n    ggml_barrier(params->threadpool);\n\n    for (int cur_a = 0; cur_a < n_as; ++cur_a) {\n        const int64_t cne1 = matrix_row_counts[cur_a];\n\n        if (cne1 == 0) {\n            continue;\n        }\n\n        const char * src0_cur = (const char *) src0->data + cur_a * nb02;\n        const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;\n        const size_t row_size = ggml_row_size(vec_dot_type, ne10);\n\n        const int64_t nr0 = ne01;\n        const int64_t nr1 = cne1;\n\n        int chunk_size = 16;\n        if (nr0 == 1 || nr1 == 1) {\n            chunk_size = 64;\n        }\n\n        // disable for NUMA\n        const bool disable_chunking = ggml_is_numa();\n\n        int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;\n        int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;\n\n        if (nchunk0 * nchunk1 < nth * 4 || disable_chunking) {\n            nchunk0 = nr0 > nr1 ? nth : 1;\n            nchunk1 = nr0 > nr1 ? 1 : nth;\n        }\n\n        const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n        const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;\n\n        int current_chunk = ith;\n\n        atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);\n\n        while (current_chunk < nchunk0 * nchunk1) {\n            const int64_t ith0 = current_chunk % nchunk0;\n            const int64_t ith1 = current_chunk / nchunk0;\n\n            const int64_t ir0_start = dr0 * ith0;\n            const int64_t ir0_end = MIN(ir0_start + dr0, nr0);\n\n            const int64_t ir1_start = dr1 * ith1;\n            const int64_t ir1_end = MIN(ir1_start + dr1, nr1);\n\n            ggml_compute_forward_mul_mat_id_one_chunk(\n                dst, src0, src1, ids, cur_a,\n                ir0_start, ir0_end, ir1_start, ir1_end,\n                src0_cur, matrix_rows, row_size, src1_cont, wdata\n            );\n\n            if (nth >= nchunk0 * nchunk1) {\n                break;\n            }\n\n            current_chunk = atomic_fetch_add_explicit(current_chunk_ctr, 1, memory_order_relaxed);\n        }\n    }\n}\n\n/////////////////////////////////\n\nstatic void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {\n    GGML_ASSERT(params);\n\n    if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {\n        return;\n    }\n\n    // extra_buffer op?\n    if (ggml_cpu_extra_compute_forward(params, tensor)) {\n        return;\n    }\n\n    switch (tensor->op) {\n        case GGML_OP_DUP:\n            {\n                ggml_compute_forward_dup(params, tensor);\n            } break;\n        case GGML_OP_ADD:\n            {\n                ggml_compute_forward_add(params, tensor);\n            } break;\n        case GGML_OP_ADD_ID:\n            {\n                ggml_compute_forward_add_id(params, tensor);\n            } break;\n        case GGML_OP_ADD1:\n            {\n                ggml_compute_forward_add1(params, tensor);\n            } break;\n        case GGML_OP_ACC:\n            {\n                ggml_compute_forward_acc(params, tensor);\n            } break;\n        case GGML_OP_SUB:\n            {\n                ggml_compute_forward_sub(params, tensor);\n            } break;\n        case GGML_OP_MUL:\n            {\n                ggml_compute_forward_mul(params, tensor);\n            } break;\n        case GGML_OP_DIV:\n            {\n                ggml_compute_forward_div(params, tensor);\n            } break;\n        case GGML_OP_SQR:\n            {\n                ggml_compute_forward_sqr(params, tensor);\n            } break;\n        case GGML_OP_SQRT:\n            {\n                ggml_compute_forward_sqrt(params, tensor);\n            } break;\n        case GGML_OP_LOG:\n            {\n                ggml_compute_forward_log(params, tensor);\n            } break;\n        case GGML_OP_SIN:\n            {\n                ggml_compute_forward_sin(params, tensor);\n            } break;\n        case GGML_OP_COS:\n            {\n                ggml_compute_forward_cos(params, tensor);\n            } break;\n        case GGML_OP_SUM:\n            {\n                ggml_compute_forward_sum(params, tensor);\n            } break;\n        case GGML_OP_SUM_ROWS:\n            {\n                ggml_compute_forward_sum_rows(params, tensor);\n            } break;\n        case GGML_OP_CUMSUM:\n            {\n                ggml_compute_forward_cumsum(params, tensor);\n            } break;\n        case GGML_OP_MEAN:\n            {\n                ggml_compute_forward_mean(params, tensor);\n            } break;\n        case GGML_OP_ARGMAX:\n            {\n                ggml_compute_forward_argmax(params, tensor);\n            } break;\n        case GGML_OP_COUNT_EQUAL:\n            {\n                ggml_compute_forward_count_equal(params, tensor);\n            } break;\n        case GGML_OP_REPEAT:\n            {\n                ggml_compute_forward_repeat(params, tensor);\n            } break;\n        case GGML_OP_REPEAT_BACK:\n            {\n                ggml_compute_forward_repeat_back(params, tensor);\n            } break;\n        case GGML_OP_CONCAT:\n            {\n                ggml_compute_forward_concat(params, tensor);\n            } break;\n        case GGML_OP_SILU_BACK:\n            {\n                ggml_compute_forward_silu_back(params, tensor);\n            } break;\n        case GGML_OP_NORM:\n            {\n                ggml_compute_forward_norm(params, tensor);\n            } break;\n        case GGML_OP_RMS_NORM:\n            {\n                ggml_compute_forward_rms_norm(params, tensor);\n            } break;\n        case GGML_OP_RMS_NORM_BACK:\n            {\n                ggml_compute_forward_rms_norm_back(params, tensor);\n            } break;\n        case GGML_OP_GROUP_NORM:\n            {\n                ggml_compute_forward_group_norm(params, tensor);\n            } break;\n        case GGML_OP_L2_NORM:\n            {\n                ggml_compute_forward_l2_norm(params, tensor);\n            } break;\n        case GGML_OP_MUL_MAT:\n            {\n                ggml_compute_forward_mul_mat(params, tensor);\n            } break;\n        case GGML_OP_MUL_MAT_ID:\n            {\n                ggml_compute_forward_mul_mat_id(params, tensor);\n            } break;\n        case GGML_OP_OUT_PROD:\n            {\n                ggml_compute_forward_out_prod(params, tensor);\n            } break;\n        case GGML_OP_SCALE:\n            {\n                ggml_compute_forward_scale(params, tensor);\n            } break;\n        case GGML_OP_SET:\n            {\n                ggml_compute_forward_set(params, tensor);\n            } break;\n        case GGML_OP_CPY:\n            {\n                ggml_compute_forward_cpy(params, tensor);\n            } break;\n        case GGML_OP_CONT:\n            {\n                ggml_compute_forward_cont(params, tensor);\n            } break;\n        case GGML_OP_GET_ROWS:\n            {\n                ggml_compute_forward_get_rows(params, tensor);\n            } break;\n        case GGML_OP_GET_ROWS_BACK:\n            {\n                ggml_compute_forward_get_rows_back(params, tensor);\n            } break;\n        case GGML_OP_SET_ROWS:\n            {\n                ggml_compute_forward_set_rows(params, tensor);\n            } break;\n        case GGML_OP_DIAG:\n            {\n                ggml_compute_forward_diag(params, tensor);\n            } break;\n        case GGML_OP_DIAG_MASK_INF:\n            {\n                ggml_compute_forward_diag_mask_inf(params, tensor);\n            } break;\n        case GGML_OP_DIAG_MASK_ZERO:\n            {\n                ggml_compute_forward_diag_mask_zero(params, tensor);\n            } break;\n        case GGML_OP_SOFT_MAX:\n            {\n                ggml_compute_forward_soft_max(params, tensor);\n            } break;\n        case GGML_OP_SOFT_MAX_BACK:\n            {\n                ggml_compute_forward_soft_max_ext_back(params, tensor);\n            } break;\n        case GGML_OP_ROPE:\n            {\n                ggml_compute_forward_rope(params, tensor);\n            } break;\n        case GGML_OP_ROPE_BACK:\n            {\n                ggml_compute_forward_rope_back(params, tensor);\n            } break;\n        case GGML_OP_CLAMP:\n            {\n                ggml_compute_forward_clamp(params, tensor);\n            } break;\n        case GGML_OP_CONV_TRANSPOSE_1D:\n            {\n                ggml_compute_forward_conv_transpose_1d(params, tensor);\n            } break;\n        case GGML_OP_IM2COL:\n            {\n                ggml_compute_forward_im2col(params, tensor);\n            } break;\n        case GGML_OP_IM2COL_BACK:\n            {\n                ggml_compute_forward_im2col_back_f32(params, tensor);\n            } break;\n        case GGML_OP_IM2COL_3D:\n            {\n                ggml_compute_forward_im2col_3d(params, tensor);\n            } break;\n        case GGML_OP_CONV_2D:\n            {\n                ggml_compute_forward_conv_2d(params, tensor);\n            } break;\n        case GGML_OP_CONV_3D:\n            {\n                ggml_compute_forward_conv_3d(params, tensor);\n            } break;\n        case GGML_OP_CONV_2D_DW:\n            {\n                ggml_compute_forward_conv_2d_dw(params, tensor);\n            } break;\n        case GGML_OP_CONV_TRANSPOSE_2D:\n            {\n                ggml_compute_forward_conv_transpose_2d(params, tensor);\n            } break;\n        case GGML_OP_POOL_1D:\n            {\n                ggml_compute_forward_pool_1d(params, tensor);\n            } break;\n        case GGML_OP_POOL_2D:\n            {\n                ggml_compute_forward_pool_2d(params, tensor);\n            } break;\n        case GGML_OP_POOL_2D_BACK:\n            {\n                ggml_compute_forward_pool_2d_back(params, tensor);\n            } break;\n        case GGML_OP_UPSCALE:\n            {\n                ggml_compute_forward_upscale(params, tensor);\n            } break;\n        case GGML_OP_PAD:\n            {\n                ggml_compute_forward_pad(params, tensor);\n            } break;\n        case GGML_OP_PAD_REFLECT_1D:\n            {\n                ggml_compute_forward_pad_reflect_1d(params, tensor);\n            } break;\n        case GGML_OP_ROLL:\n            {\n                ggml_compute_forward_roll(params, tensor);\n            } break;\n        case GGML_OP_ARANGE:\n            {\n                ggml_compute_forward_arange(params, tensor);\n            } break;\n        case GGML_OP_TIMESTEP_EMBEDDING:\n            {\n                ggml_compute_forward_timestep_embedding(params, tensor);\n            } break;\n        case GGML_OP_ARGSORT:\n            {\n                ggml_compute_forward_argsort(params, tensor);\n            } break;\n        case GGML_OP_TOP_K:\n            {\n                ggml_compute_forward_top_k(params, tensor);\n            } break;\n        case GGML_OP_LEAKY_RELU:\n            {\n                ggml_compute_forward_leaky_relu(params, tensor);\n            } break;\n        case GGML_OP_TRI:\n            {\n                ggml_compute_forward_tri(params, tensor);\n            } break;\n        case GGML_OP_FILL:\n            {\n                ggml_compute_forward_fill(params, tensor);\n            } break;\n        case GGML_OP_FLASH_ATTN_EXT:\n            {\n                ggml_compute_forward_flash_attn_ext(params, tensor);\n            } break;\n        case GGML_OP_FLASH_ATTN_BACK:\n            {\n                int32_t t = ggml_get_op_params_i32(tensor, 0);\n                GGML_ASSERT(t == 0 || t == 1);\n                bool masked = t != 0;\n                ggml_compute_forward_flash_attn_back(params, masked, tensor);\n            } break;\n        case GGML_OP_SSM_CONV:\n            {\n                ggml_compute_forward_ssm_conv(params, tensor);\n            } break;\n        case GGML_OP_SSM_SCAN:\n            {\n                ggml_compute_forward_ssm_scan(params, tensor);\n            } break;\n        case GGML_OP_WIN_PART:\n            {\n                ggml_compute_forward_win_part(params, tensor);\n            } break;\n        case GGML_OP_WIN_UNPART:\n            {\n                ggml_compute_forward_win_unpart(params, tensor);\n            } break;\n        case GGML_OP_UNARY:\n            {\n                ggml_compute_forward_unary(params, tensor);\n            } break;\n        case GGML_OP_GLU:\n            {\n                ggml_compute_forward_glu(params, tensor);\n            } break;\n        case GGML_OP_GET_REL_POS:\n            {\n                ggml_compute_forward_get_rel_pos(params, tensor);\n            } break;\n        case GGML_OP_ADD_REL_POS:\n            {\n                ggml_compute_forward_add_rel_pos(params, tensor);\n            } break;\n        case GGML_OP_RWKV_WKV6:\n            {\n                ggml_compute_forward_rwkv_wkv6(params, tensor);\n            } break;\n        case GGML_OP_GATED_LINEAR_ATTN:\n            {\n                ggml_compute_forward_gla(params, tensor);\n            } break;\n        case GGML_OP_RWKV_WKV7:\n            {\n                ggml_compute_forward_rwkv_wkv7(params, tensor);\n            } break;\n        case GGML_OP_SOLVE_TRI:\n            {\n                ggml_compute_forward_solve_tri(params, tensor);\n            } break;\n        case GGML_OP_GATED_DELTA_NET:\n            {\n                ggml_compute_forward_gated_delta_net(params, tensor);\n            } break;\n        case GGML_OP_MAP_CUSTOM1:\n            {\n                ggml_compute_forward_map_custom1(params, tensor);\n            }\n            break;\n        case GGML_OP_MAP_CUSTOM2:\n            {\n                ggml_compute_forward_map_custom2(params, tensor);\n            }\n            break;\n        case GGML_OP_MAP_CUSTOM3:\n            {\n                ggml_compute_forward_map_custom3(params, tensor);\n            }\n            break;\n        case GGML_OP_CUSTOM:\n            {\n                ggml_compute_forward_custom(params, tensor);\n            }\n            break;\n        case GGML_OP_CROSS_ENTROPY_LOSS:\n            {\n                ggml_compute_forward_cross_entropy_loss(params, tensor);\n            }\n            break;\n        case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n            {\n                ggml_compute_forward_cross_entropy_loss_back(params, tensor);\n            }\n            break;\n        case GGML_OP_OPT_STEP_ADAMW:\n            {\n                ggml_compute_forward_opt_step_adamw(params, tensor);\n            }\n            break;\n        case GGML_OP_OPT_STEP_SGD:\n            {\n                ggml_compute_forward_opt_step_sgd(params, tensor);\n            }\n            break;\n        case GGML_OP_NONE:\n            {\n                // nop\n            } break;\n        case GGML_OP_RESHAPE:\n            {\n                // nop\n            } break;\n        case GGML_OP_PERMUTE:\n            {\n                // nop\n            } break;\n        case GGML_OP_VIEW:\n            {\n                // nop\n            } break;\n        case GGML_OP_TRANSPOSE:\n            {\n                // nop\n            } break;\n        case GGML_OP_COUNT:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// Android's libc implementation \"bionic\" does not support setting affinity\n#if defined(__gnu_linux__)\nstatic void set_numa_thread_affinity(int thread_n) {\n    if (!ggml_is_numa()) {\n        return;\n    }\n\n    int node_num;\n    int rv;\n    size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);\n\n    switch(g_state.numa.numa_strategy) {\n        case GGML_NUMA_STRATEGY_DISTRIBUTE:\n            // run thread on node_num thread_n / (threads per node)\n            node_num = thread_n % g_state.numa.n_nodes;\n            break;\n        case GGML_NUMA_STRATEGY_ISOLATE:\n            // run thread on current_node\n            node_num = g_state.numa.current_node;\n            break;\n        case GGML_NUMA_STRATEGY_NUMACTL:\n            // use the cpuset that numactl gave us\n            rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);\n            if (rv) {\n                fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\",strerror(rv));\n            }\n            return;\n        default:\n            return;\n    }\n\n    struct ggml_numa_node * node = &g_state.numa.nodes[node_num];\n\n    cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);\n    CPU_ZERO_S(setsize, cpus);\n    for (size_t i = 0; i < node->n_cpus; ++i) {\n        CPU_SET_S(node->cpus[i], setsize, cpus);\n    }\n\n    rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);\n    if (rv) {\n            fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\", strerror(rv));\n    }\n\n    CPU_FREE(cpus);\n}\n\nstatic void clear_numa_thread_affinity(void) {\n    if (!ggml_is_numa()) {\n        return;\n    }\n\n    size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);\n\n    cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);\n    CPU_ZERO_S(setsize, cpus);\n    for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {\n        CPU_SET_S(i, setsize, cpus);\n    }\n\n    int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);\n    if (rv) {\n        fprintf(stderr, \"warning: pthread_setaffinity_np() failed: %s\\n\", strerror(rv));\n    }\n\n    CPU_FREE(cpus);\n}\n#else\n// TODO: Windows etc.\n// (the linux implementation may also work on BSD, someone should test)\nstatic void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n);  }\nstatic void clear_numa_thread_affinity(void) {}\n#endif\n\nstatic int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {\n    int n_tasks = 0;\n\n    if (ggml_is_empty(node)) {\n        // no need to multi-thread a no-op\n        n_tasks = 1;\n        return n_tasks;\n    }\n\n    switch (node->op) {\n        case GGML_OP_CPY:\n        case GGML_OP_DUP:\n        case GGML_OP_CONT:\n        case GGML_OP_ADD:\n        case GGML_OP_ADD_ID:\n        case GGML_OP_ADD1:\n        case GGML_OP_ACC:\n        case GGML_OP_CUMSUM:\n        case GGML_OP_TRI:\n        case GGML_OP_FILL:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_SUB:\n        case GGML_OP_SQR:\n        case GGML_OP_SQRT:\n        case GGML_OP_LOG:\n        case GGML_OP_SIN:\n        case GGML_OP_COS:\n        case GGML_OP_SUM:\n        case GGML_OP_SUM_ROWS:\n        case GGML_OP_MEAN:\n        case GGML_OP_ARGMAX:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_COUNT_EQUAL:\n        case GGML_OP_SOLVE_TRI:\n        case GGML_OP_GATED_DELTA_NET:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_REPEAT:\n        case GGML_OP_REPEAT_BACK:\n        case GGML_OP_LEAKY_RELU:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_UNARY:\n            switch (ggml_get_unary_op(node)) {\n                case GGML_UNARY_OP_ABS:\n                case GGML_UNARY_OP_SGN:\n                case GGML_UNARY_OP_NEG:\n                case GGML_UNARY_OP_STEP:\n                case GGML_UNARY_OP_TANH:\n                case GGML_UNARY_OP_ELU:\n                case GGML_UNARY_OP_RELU:\n                case GGML_UNARY_OP_SIGMOID:\n                case GGML_UNARY_OP_HARDSWISH:\n                case GGML_UNARY_OP_HARDSIGMOID:\n                case GGML_UNARY_OP_EXP:\n                case GGML_UNARY_OP_SOFTPLUS:\n                case GGML_UNARY_OP_EXPM1:\n                case GGML_UNARY_OP_FLOOR:\n                case GGML_UNARY_OP_CEIL:\n                case GGML_UNARY_OP_ROUND:\n                case GGML_UNARY_OP_TRUNC:\n                    {\n                        n_tasks = 1;\n                    } break;\n\n                case GGML_UNARY_OP_GELU:\n                case GGML_UNARY_OP_GELU_ERF:\n                case GGML_UNARY_OP_GELU_QUICK:\n                case GGML_UNARY_OP_SILU:\n                case GGML_UNARY_OP_XIELU:\n                    {\n                        n_tasks = n_threads;\n                    } break;\n                default:\n                    GGML_ABORT(\"fatal error\");\n            }\n            break;\n        case GGML_OP_GLU:\n            switch (ggml_get_glu_op(node)) {\n                case GGML_GLU_OP_REGLU:\n                case GGML_GLU_OP_GEGLU:\n                case GGML_GLU_OP_SWIGLU:\n                case GGML_GLU_OP_SWIGLU_OAI:\n                case GGML_GLU_OP_GEGLU_ERF:\n                case GGML_GLU_OP_GEGLU_QUICK:\n                    {\n                        n_tasks = n_threads;\n                    } break;\n                default:\n                    GGML_ABORT(\"fatal error\");\n            }\n            break;\n        case GGML_OP_SILU_BACK:\n        case GGML_OP_MUL:\n        case GGML_OP_DIV:\n        case GGML_OP_NORM:\n        case GGML_OP_RMS_NORM:\n        case GGML_OP_RMS_NORM_BACK:\n        case GGML_OP_L2_NORM:\n        case GGML_OP_GROUP_NORM:\n        case GGML_OP_CONCAT:\n        case GGML_OP_MUL_MAT:\n        case GGML_OP_MUL_MAT_ID:\n        case GGML_OP_OUT_PROD:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_GET_ROWS:\n        case GGML_OP_SET_ROWS:\n            {\n                // FIXME: get_rows can use additional threads, but the cost of launching additional threads\n                // decreases performance with GPU offloading\n                //n_tasks = n_threads;\n                n_tasks = 1;\n            } break;\n        case GGML_OP_SCALE:\n        case GGML_OP_SET:\n        case GGML_OP_RESHAPE:\n        case GGML_OP_VIEW:\n        case GGML_OP_PERMUTE:\n        case GGML_OP_TRANSPOSE:\n        case GGML_OP_GET_ROWS_BACK:\n        case GGML_OP_DIAG:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_DIAG_MASK_ZERO:\n        case GGML_OP_DIAG_MASK_INF:\n        case GGML_OP_SOFT_MAX_BACK:\n        case GGML_OP_ROPE:\n        case GGML_OP_ROPE_BACK:\n        case GGML_OP_ADD_REL_POS:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_CLAMP:\n            {\n                n_tasks = 1; //TODO\n            } break;\n        case GGML_OP_SOFT_MAX:\n            {\n                n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));\n            } break;\n        case GGML_OP_IM2COL:\n        case GGML_OP_IM2COL_BACK:\n        case GGML_OP_IM2COL_3D:\n        case GGML_OP_CONV_2D:\n        case GGML_OP_CONV_3D:\n        case GGML_OP_CONV_2D_DW:\n        case GGML_OP_CONV_TRANSPOSE_1D:\n        case GGML_OP_CONV_TRANSPOSE_2D:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_POOL_1D:\n        case GGML_OP_POOL_2D:\n        case GGML_OP_POOL_2D_BACK:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_UPSCALE:\n        case GGML_OP_PAD:\n        case GGML_OP_PAD_REFLECT_1D:\n        case GGML_OP_ROLL:\n        case GGML_OP_ARANGE:\n        case GGML_OP_TIMESTEP_EMBEDDING:\n        case GGML_OP_ARGSORT:\n        case GGML_OP_TOP_K:\n        case GGML_OP_FLASH_ATTN_EXT:\n        case GGML_OP_FLASH_ATTN_BACK:\n        case GGML_OP_SSM_CONV:\n        case GGML_OP_SSM_SCAN:\n        case GGML_OP_RWKV_WKV6:\n        case GGML_OP_GATED_LINEAR_ATTN:\n        case GGML_OP_RWKV_WKV7:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_WIN_PART:\n        case GGML_OP_WIN_UNPART:\n        case GGML_OP_GET_REL_POS:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_MAP_CUSTOM1:\n            {\n                struct ggml_map_custom1_op_params p;\n                memcpy(&p, node->op_params, sizeof(p));\n                if (p.n_tasks == GGML_N_TASKS_MAX) {\n                    n_tasks = n_threads;\n                } else {\n                    n_tasks = MIN(p.n_tasks, n_threads);\n                }\n            } break;\n        case GGML_OP_MAP_CUSTOM2:\n            {\n                struct ggml_map_custom2_op_params p;\n                memcpy(&p, node->op_params, sizeof(p));\n                if (p.n_tasks == GGML_N_TASKS_MAX) {\n                    n_tasks = n_threads;\n                } else {\n                    n_tasks = MIN(p.n_tasks, n_threads);\n                }\n            } break;\n        case GGML_OP_MAP_CUSTOM3:\n            {\n                struct ggml_map_custom3_op_params p;\n                memcpy(&p, node->op_params, sizeof(p));\n                if (p.n_tasks == GGML_N_TASKS_MAX) {\n                    n_tasks = n_threads;\n                } else {\n                    n_tasks = MIN(p.n_tasks, n_threads);\n                }\n            } break;\n        case GGML_OP_CUSTOM:\n            {\n                struct ggml_custom_op_params p;\n                memcpy(&p, node->op_params, sizeof(p));\n                if (p.n_tasks == GGML_N_TASKS_MAX) {\n                    n_tasks = n_threads;\n                } else {\n                    n_tasks = MIN(p.n_tasks, n_threads);\n                }\n            } break;\n        case GGML_OP_CROSS_ENTROPY_LOSS:\n        case GGML_OP_CROSS_ENTROPY_LOSS_BACK:\n        case GGML_OP_OPT_STEP_ADAMW:\n        case GGML_OP_OPT_STEP_SGD:\n            {\n                n_tasks = n_threads;\n            } break;\n        case GGML_OP_NONE:\n            {\n                n_tasks = 1;\n            } break;\n        case GGML_OP_COUNT:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n        default:\n            {\n                fprintf(stderr, \"%s: op not implemented: \", __func__);\n                if (node->op < GGML_OP_COUNT) {\n                    fprintf(stderr, \"%s\\n\", ggml_op_name(node->op));\n                } else {\n                    fprintf(stderr, \"%d\\n\", node->op);\n                }\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n\n    assert(n_tasks > 0);\n\n    return n_tasks;\n}\n\nstatic thread_ret_t ggml_graph_compute_secondary_thread(void* data);\n\n#if defined(_WIN32)\n#include \"windows.h\"\n\n// TODO: support > 64 CPUs\nstatic bool ggml_thread_apply_affinity(bool * mask) {\n    HANDLE    h = GetCurrentThread();\n    uint64_t  bitmask = 0ULL;\n\n    assert(GGML_MAX_N_THREADS >= 64);\n\n    for (int32_t i = 0; i < 8; i++) {\n        int32_t idx = i * 8;\n        uint8_t val = 0;\n        val |= mask[idx + 0] << 0;\n        val |= mask[idx + 1] << 1;\n        val |= mask[idx + 2] << 2;\n        val |= mask[idx + 3] << 3;\n        val |= mask[idx + 4] << 4;\n        val |= mask[idx + 5] << 5;\n        val |= mask[idx + 6] << 6;\n        val |= mask[idx + 7] << 7;\n        bitmask |= (uint64_t)val << idx;\n    }\n\n    for (int32_t i = 64; i < GGML_MAX_N_THREADS; i++) {\n        if (mask[i]) {\n            fprintf(stderr, \"warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\\n\");\n            break;\n        }\n    }\n\n    DWORD_PTR m = (DWORD_PTR)bitmask;\n\n    m = SetThreadAffinityMask(h, m);\n\n    return m != 0;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n    // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.\n    // This is up to the applications.\n    DWORD p = THREAD_PRIORITY_NORMAL;\n    switch (prio) {\n        case GGML_SCHED_PRIO_LOW:      p = THREAD_PRIORITY_BELOW_NORMAL;  break;\n        case GGML_SCHED_PRIO_NORMAL:   p = THREAD_PRIORITY_NORMAL;        break;\n        case GGML_SCHED_PRIO_MEDIUM:   p = THREAD_PRIORITY_ABOVE_NORMAL;  break;\n        case GGML_SCHED_PRIO_HIGH:     p = THREAD_PRIORITY_HIGHEST;       break;\n        case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;\n    }\n\n    if (prio != GGML_SCHED_PRIO_LOW) {\n        // Tell Windows that this thread should not be throttled (needs its own CPU core).\n        // Newer Windows 11 versions aggressively park (offline) CPU cores and often place\n        // all our threads onto the first 4 cores which results in terrible performance with\n        // n_threads > 4\n        #if _WIN32_WINNT >= 0x0602\n        THREAD_POWER_THROTTLING_STATE t;\n        ZeroMemory(&t, sizeof(t));\n        t.Version     = THREAD_POWER_THROTTLING_CURRENT_VERSION;\n        t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;\n        t.StateMask   = 0;\n\n        if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) {\n            GGML_LOG_DEBUG(\"failed to disable thread power throttling %d : (%d)\\n\", prio, (int) GetLastError());\n            return false;\n        }\n        #endif\n    }\n\n    if (prio == GGML_SCHED_PRIO_NORMAL) {\n        // Keep inherited policy/priority\n        return true;\n    }\n\n    if (!SetThreadPriority(GetCurrentThread(), p)) {\n        fprintf(stderr, \"warn: failed to set thread priority %d : (%d)\\n\", prio, (int) GetLastError());\n        return false;\n    }\n\n    return true;\n}\n\n#elif defined(__APPLE__)\n#include <sys/types.h>\n#include <sys/resource.h>\n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n    // Not supported on Apple platforms\n    UNUSED(mask);\n    return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n    struct sched_param p;\n    int32_t policy = SCHED_OTHER;\n    switch (prio) {\n        // TODO: there seems to be no way to set lower prio on Apple platforms\n        case GGML_SCHED_PRIO_LOW:      policy = SCHED_OTHER; p.sched_priority = 0;  break;\n        case GGML_SCHED_PRIO_NORMAL:   policy = SCHED_OTHER; p.sched_priority = 0;  break;\n        case GGML_SCHED_PRIO_MEDIUM:   policy = SCHED_FIFO;  p.sched_priority = 40; break;\n        case GGML_SCHED_PRIO_HIGH:     policy = SCHED_FIFO;  p.sched_priority = 80; break;\n        case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO;  p.sched_priority = 90; break;\n    }\n\n    if (prio == GGML_SCHED_PRIO_NORMAL) {\n        // Keep inherited policy/priority\n        return true;\n    }\n\n    int32_t err = pthread_setschedparam(pthread_self(), policy, &p);\n    if (err != 0) {\n        fprintf(stderr, \"warn: failed to set thread priority %d : %s (%d)\\n\", prio, strerror(err), err);\n        return false;\n    }\n\n    return true;\n}\n\n#elif defined(__gnu_linux__)\n// TODO: this may not work on BSD, to be verified\n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n    cpu_set_t cpuset;\n    int err;\n\n    CPU_ZERO(&cpuset);\n\n    for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {\n        if (mask[i]) {\n            GGML_PRINT_DEBUG(\"Thread %lx: adding %d to cpuset\\n\", pthread_self(), i);\n            CPU_SET(i, &cpuset);\n        }\n    }\n\n#ifdef __ANDROID__\n    err = sched_setaffinity(0, sizeof(cpuset), &cpuset);\n    if (err < 0) {\n        err = errno;\n    }\n#else\n    err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);\n#endif\n    if (err != 0) {\n        fprintf(stderr, \"warn: failed to set affinity mask 0x%llx : %s (%d)\\n\", (unsigned long long)mask, strerror(err), err);\n        return false;\n    }\n\n    return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n    struct sched_param p;\n    int32_t policy = SCHED_OTHER;\n    switch (prio) {\n        case GGML_SCHED_PRIO_LOW:      policy = SCHED_BATCH; p.sched_priority = 0;  break;\n        case GGML_SCHED_PRIO_NORMAL:   policy = SCHED_OTHER; p.sched_priority = 0;  break;\n        case GGML_SCHED_PRIO_MEDIUM:   policy = SCHED_FIFO;  p.sched_priority = 40; break;\n        case GGML_SCHED_PRIO_HIGH:     policy = SCHED_FIFO;  p.sched_priority = 80; break;\n        case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO;  p.sched_priority = 90; break;\n    }\n\n    if (prio == GGML_SCHED_PRIO_NORMAL) {\n        // Keep inherited policy/priority\n        return true;\n    }\n\n    int32_t err = pthread_setschedparam(pthread_self(), policy, &p);\n    if (err != 0) {\n        fprintf(stderr, \"warn: failed to set thread priority %d : %s (%d)\\n\", prio, strerror(err), err);\n        return false;\n    }\n\n    return true;\n}\n\n#else // unsupported platforms\n\nstatic bool ggml_thread_apply_affinity(const bool * mask) {\n    UNUSED(mask);\n    return true;\n}\n\nstatic bool ggml_thread_apply_priority(int32_t prio) {\n    UNUSED(prio);\n    return true;\n}\n\n#endif\n\nstatic bool ggml_thread_cpumask_is_valid(const bool * mask) {\n    for (int i = 0; i < GGML_MAX_N_THREADS; i++) {\n        if (mask[i]) { return true; }\n    }\n    return false;\n}\n\nstatic void ggml_thread_cpumask_next(const bool * global_mask, bool * local_mask, bool strict, int32_t* iter) {\n    if (!strict) {\n        memcpy(local_mask, global_mask, GGML_MAX_N_THREADS);\n        return;\n    } else {\n        memset(local_mask, 0, GGML_MAX_N_THREADS);\n        int32_t base_idx = *iter;\n        for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {\n            int32_t idx = base_idx + i;\n            if (idx >= GGML_MAX_N_THREADS) {\n                // Just a cheaper modulo\n                idx -= GGML_MAX_N_THREADS;\n            }\n            if (global_mask[idx]) {\n                local_mask[idx] = 1;\n                *iter = idx + 1;\n                return;\n            }\n        }\n    }\n}\n\nvoid ggml_threadpool_free(struct ggml_threadpool* threadpool) {\n    if (!threadpool) return;\n\n    const int n_threads = threadpool->n_threads;\n\n#ifndef GGML_USE_OPENMP\n    struct ggml_compute_state* workers = threadpool->workers;\n\n    ggml_mutex_lock(&threadpool->mutex);\n\n    threadpool->stop = true;\n    threadpool->pause = false;\n\n    ggml_cond_broadcast(&threadpool->cond);\n    ggml_mutex_unlock(&threadpool->mutex);\n\n    for (int j = 1; j < n_threads; j++) {\n        int32_t rc = ggml_thread_join(workers[j].thrd, NULL);\n        GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED);\n        UNUSED(rc);\n    }\n\n    ggml_mutex_destroy(&threadpool->mutex);\n    ggml_cond_destroy(&threadpool->cond);\n#endif // GGML_USE_OPENMP\n\n    const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;\n    ggml_aligned_free(threadpool->workers, workers_size);\n    ggml_aligned_free(threadpool, sizeof(struct ggml_threadpool));\n}\n\n#ifndef GGML_USE_OPENMP\n// pause/resume must be called under mutex\nstatic void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {\n    GGML_PRINT_DEBUG(\"Pausing threadpool\\n\");\n    threadpool->pause = true;\n    ggml_cond_broadcast(&threadpool->cond);\n}\n\nstatic void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {\n    GGML_PRINT_DEBUG(\"Resuming threadpool\\n\");\n    threadpool->pause = false;\n    ggml_cond_broadcast(&threadpool->cond);\n}\n#endif\n\nvoid ggml_threadpool_pause(struct ggml_threadpool * threadpool) {\n#ifndef GGML_USE_OPENMP\n    ggml_mutex_lock(&threadpool->mutex);\n    if (!threadpool->pause) {\n       ggml_threadpool_pause_locked(threadpool);\n    }\n    ggml_mutex_unlock(&threadpool->mutex);\n#else\n    UNUSED(threadpool);\n#endif\n}\n\nvoid ggml_threadpool_resume(struct ggml_threadpool * threadpool) {\n#ifndef GGML_USE_OPENMP\n    ggml_mutex_lock(&threadpool->mutex);\n    if (threadpool->pause) {\n       ggml_threadpool_resume_locked(threadpool);\n    }\n    ggml_mutex_unlock(&threadpool->mutex);\n#else\n    UNUSED(threadpool);\n#endif\n}\n\nstruct ggml_cplan ggml_graph_plan(\n          const struct ggml_cgraph * cgraph,\n                               int   n_threads,\n            struct ggml_threadpool * threadpool) {\n\n    if (threadpool == NULL) {\n        //GGML_PRINT_DEBUG(\"Threadpool is not specified. Will create a disposable threadpool : n_threads %d\\n\", n_threads);\n    }\n    if (n_threads <= 0) {\n        n_threads = threadpool ? threadpool->n_threads : GGML_DEFAULT_N_THREADS;\n    }\n\n#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)\n    // Emscripten without pthreads support can only use a single thread\n    n_threads = 1;\n#endif\n\n    size_t work_size = 0;\n\n    struct ggml_cplan cplan;\n    memset(&cplan, 0, sizeof(struct ggml_cplan));\n\n    int max_tasks = 1;\n\n    // thread scheduling for the different operations + work buffer size estimation\n    for (int i = 0; i < cgraph->n_nodes; i++) {\n        struct ggml_tensor * node = cgraph->nodes[i];\n\n        const int n_tasks = ggml_get_n_tasks(node, n_threads);\n\n        max_tasks = MAX(max_tasks, n_tasks);\n\n        size_t cur = 0;\n\n        if (!ggml_cpu_extra_work_size(n_threads, node, &cur)) {\n            switch (node->op) {\n                case GGML_OP_CPY:\n                case GGML_OP_DUP:\n                    {\n                        if (ggml_is_quantized(node->type) ||\n                            // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32\n                            (node->src[0]->type == GGML_TYPE_F16  && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||\n                            (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) ||\n                            // conversion between F32 and I32\n                            (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||\n                            (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {\n                            cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;\n                        }\n                    } break;\n                case GGML_OP_ADD:\n                case GGML_OP_ADD_ID:\n                case GGML_OP_ADD1:\n                    {\n                        if (ggml_is_quantized(node->src[0]->type)) {\n                            cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;\n                        }\n                    } break;\n                case GGML_OP_ACC:\n                    {\n                        if (ggml_is_quantized(node->src[0]->type)) {\n                            cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;\n                        }\n                    } break;\n                case GGML_OP_COUNT_EQUAL:\n                    {\n                        cur = ggml_type_size(node->type)*n_tasks;\n                    } break;\n                case GGML_OP_MUL_MAT:\n                    {\n                        const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;\n\n                        if (node->src[1]->type != vec_dot_type) {\n                            cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));\n                        }\n                    } break;\n                case GGML_OP_MUL_MAT_ID:\n                    {\n                        cur = 0;\n                        const struct ggml_tensor * src0 = node->src[0];\n                        const struct ggml_tensor * src1 = node->src[1];\n                        const struct ggml_tensor * ids = node->src[2];\n                        const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;\n                        const int n_as = src0->ne[2];\n                        // src1\n                        if (src1->type != vec_dot_type) {\n                            cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)) + sizeof(int64_t);\n                        }\n                        // matrix_row_counts\n                        cur += n_as * sizeof(int64_t) + sizeof(int64_t);\n                        // matrix_rows\n                        cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);\n                        // atomic_current_chunk\n                        cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;\n                    } break;\n                case GGML_OP_OUT_PROD:\n                    {\n                        if (ggml_is_quantized(node->src[0]->type)) {\n                            cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;\n                        }\n                    } break;\n                case GGML_OP_SOFT_MAX:\n                case GGML_OP_ROPE:\n                case GGML_OP_ROPE_BACK:\n                    {\n                        cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;\n                    } break;\n                case GGML_OP_CONV_TRANSPOSE_1D:\n                    {\n                        GGML_ASSERT(node->src[0]->ne[3] == 1);\n                        GGML_ASSERT(node->src[1]->ne[2] == 1);\n                        GGML_ASSERT(node->src[1]->ne[3] == 1);\n\n                        const int64_t ne00 = node->src[0]->ne[0];  // K\n                        const int64_t ne01 = node->src[0]->ne[1];  // Cout\n                        const int64_t ne02 = node->src[0]->ne[2];  // Cin\n                        const int64_t ne10 = node->src[1]->ne[0];  // L\n                        const int64_t ne11 = node->src[1]->ne[1];  // Cin\n\n                        if ((node->src[0]->type == GGML_TYPE_F16 ||\n                             node->src[0]->type == GGML_TYPE_BF16) &&\n                            node->src[1]->type == GGML_TYPE_F32) {\n                            cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;\n                            cur += sizeof(ggml_fp16_t)*ne10*ne11;\n                        } else if (node->src[0]->type == GGML_TYPE_F32 &&\n                                   node->src[1]->type == GGML_TYPE_F32) {\n                            cur += sizeof(float)*ne00*ne01*ne02;\n                            cur += sizeof(float)*ne10*ne11;\n                        } else {\n                            GGML_ABORT(\"fatal error\");\n                        }\n                    } break;\n                case GGML_OP_CONV_2D:\n                case GGML_OP_CONV_3D:\n                    {\n                        cur = GGML_IM2COL_WORK_SIZE;\n                    } break;\n                case GGML_OP_CONV_TRANSPOSE_2D:\n                    {\n                        const int64_t ne00 = node->src[0]->ne[0]; // W\n                        const int64_t ne01 = node->src[0]->ne[1]; // H\n                        const int64_t ne02 = node->src[0]->ne[2]; // Channels Out\n                        const int64_t ne03 = node->src[0]->ne[3]; // Channels In\n\n                        const int64_t ne10 = node->src[1]->ne[0]; // W\n                        const int64_t ne11 = node->src[1]->ne[1]; // H\n                        const int64_t ne12 = node->src[1]->ne[2]; // Channels In\n\n                        cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03;\n                        cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12;\n                    } break;\n                case GGML_OP_TOP_K:\n                    {\n                        cur += sizeof(int32_t)*node->src[0]->ne[0]*n_tasks;\n                    } break;\n                case GGML_OP_FLASH_ATTN_EXT:\n                    {\n                        const int64_t neq2 = node->src[0]->ne[2]; // number of query heads\n                        const int64_t DK = node->src[1]->ne[0];\n                        const int64_t DV = node->src[2]->ne[0];\n\n                        // Tiled flash attention scratch (tile sizes defined in common.h)\n                        // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding\n                        size_t prefill  = sizeof(float)*(GGML_FA_TILE_Q*DK + 2*GGML_FA_TILE_Q*GGML_FA_TILE_KV + GGML_FA_TILE_Q*DV + GGML_FA_TILE_KV*DV + GGML_FA_TILE_KV*DK)*n_tasks;\n\n                        // Decode path: n_kv_chunks = n_tasks (one chunk per thread)\n                        // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ\n                        size_t n_chunks = n_tasks;\n                        size_t decode   = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV));\n\n                        cur += MAX(prefill, decode);\n                    } break;\n                case GGML_OP_FLASH_ATTN_BACK:\n                    {\n                        const int64_t    D = node->src[0]->ne[0];\n                        const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);\n                        const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back\n                        if (node->src[1]->type == GGML_TYPE_F32) {\n                            cur  = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n                            cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n                        } else if (node->src[1]->type == GGML_TYPE_F16) {\n                            cur  = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n                            cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n                        } else if (node->src[1]->type == GGML_TYPE_BF16) {\n                            cur  = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)\n                            cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2\n                        }\n                    } break;\n\n                case GGML_OP_CROSS_ENTROPY_LOSS:\n                    {\n                        cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);\n                    } break;\n                case GGML_OP_GATED_DELTA_NET:\n                    {\n                        const int64_t S_v = node->src[2]->ne[0];\n                        cur = S_v * sizeof(float) * n_tasks;\n                    } break;\n                case GGML_OP_COUNT:\n                    {\n                        GGML_ABORT(\"fatal error\");\n                    }\n                default:\n                    break;\n            }\n        }\n\n        work_size = MAX(work_size, cur);\n    }\n\n    if (work_size > 0) {\n        work_size += CACHE_LINE_SIZE*(n_threads);\n    }\n\n    cplan.threadpool = threadpool;\n    cplan.n_threads  = MIN(max_tasks, n_threads);\n    cplan.work_size  = work_size;\n    cplan.work_data  = NULL;\n\n    return cplan;\n}\n\nstatic thread_ret_t ggml_graph_compute_thread(void * data) {\n    struct ggml_compute_state * state = (struct ggml_compute_state *) data;\n    struct ggml_threadpool    * tp    = state->threadpool;\n\n    const struct ggml_cgraph * cgraph = tp->cgraph;\n    const struct ggml_cplan  * cplan  = tp->cplan;\n\n    set_numa_thread_affinity(state->ith);\n\n    struct ggml_compute_params params = {\n        /*.ith        =*/ state->ith,\n        /*.nth        =*/ atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK,\n        /*.wsize      =*/ cplan->work_size,\n        /*.wdata      =*/ cplan->work_data,\n        /*.threadpool =*/ tp,\n        /*.use_ref    =*/ cplan->use_ref,\n    };\n\n#ifdef GGML_USE_OPENMP\n    GGML_PRINT_DEBUG(\"thread #%d compute-start cplan %p\\n\", state->ith, (const void *)cplan);\n#else\n    GGML_PRINT_DEBUG(\"thread #%d compute-start cplan %p last-graph %d\\n\", state->ith, (const void *)cplan, state->last_graph);\n#endif\n\n    for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {\n        struct ggml_tensor * node = cgraph->nodes[node_n];\n\n        if (ggml_op_is_empty(node->op)) {\n            // skip NOPs\n            continue;\n        }\n\n        if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {\n            continue;\n        }\n\n        ggml_compute_forward(&params, node);\n\n        if (state->ith == 0 && cplan->abort_callback &&\n                cplan->abort_callback(cplan->abort_callback_data)) {\n            atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);\n            tp->ec    = GGML_STATUS_ABORTED;\n        }\n\n        if (node_n + 1 < cgraph->n_nodes) {\n            ggml_barrier(state->threadpool);\n        }\n    }\n\n#ifdef GGML_USE_OPENMP\n    GGML_PRINT_DEBUG(\"thread #%d compute-done cplan %p\\n\", state->ith, (const void *)cplan);\n#else\n    GGML_PRINT_DEBUG(\"thread #%d compute-done cplan %p last-graph %d\\n\", state->ith, (const void *)cplan, state->last_graph);\n#endif\n\n    ggml_barrier(state->threadpool);\n\n    return 0;\n}\n\n#ifndef GGML_USE_OPENMP\n\n// check if thread is ready to proceed (exit from polling or sleeping)\n// returns true if loops should exit, sets state->pending to indicate new work\nstatic inline bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) {\n    struct ggml_threadpool * threadpool = state->threadpool;\n\n    if (state->pending || threadpool->stop || threadpool->pause) { return true; }\n\n    // check for new graph/work\n    int n_graph   = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed);\n    int n_threads = n_graph & GGML_THREADPOOL_N_THREADS_MASK;\n    if (n_graph != state->last_graph) {\n        state->pending    = (state->ith < n_threads);\n        state->last_graph = n_graph;\n        return true;\n    }\n\n    return false;\n}\n\n// sync thread state after polling\nstatic inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) {\n    // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead\n    #ifdef GGML_TSAN_ENABLED\n    atomic_fetch_add_explicit(&state->threadpool->n_graph, 0, memory_order_seq_cst);\n    #else\n    atomic_thread_fence(memory_order_seq_cst);\n    #endif\n    UNUSED(state);\n}\n\nstatic inline bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {\n    struct ggml_threadpool * threadpool = state->threadpool;\n\n    // This seems to make 0 ... 100 a decent range for polling level across modern processors.\n    // Perhaps, we can adjust it dynamically based on load and things.\n    const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;\n\n    for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) {\n        // No new work. Keep polling.\n        ggml_thread_cpu_relax();\n    }\n\n    return state->pending;\n}\n\nstatic inline bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {\n    struct ggml_threadpool * threadpool = state->threadpool;\n\n    if (ggml_graph_compute_poll_for_work(state)) {\n        ggml_graph_compute_thread_sync(state);\n        return state->pending;\n    }\n\n    ggml_mutex_lock_shared(&threadpool->mutex);\n    while (!ggml_graph_compute_thread_ready(state)) {\n        // No new work. Wait for the signal.\n        GGML_PRINT_DEBUG(\"thread #%d waiting for work (sleeping)\\n\", state->ith);\n        ggml_cond_wait(&threadpool->cond, &threadpool->mutex);\n    }\n    ggml_mutex_unlock_shared(&threadpool->mutex);\n\n    return state->pending;\n}\n\nstatic thread_ret_t ggml_graph_compute_secondary_thread(void* data) {\n    struct ggml_compute_state * state = (struct ggml_compute_state *) data;\n    struct ggml_threadpool * threadpool = state->threadpool;\n\n    ggml_thread_apply_priority(threadpool->prio);\n    if (ggml_thread_cpumask_is_valid(state->cpumask)) {\n        ggml_thread_apply_affinity(state->cpumask);\n    }\n\n    while (true) {\n        // Check if we need to sleep\n        while (threadpool->pause) {\n            GGML_PRINT_DEBUG(\"thread #%d inside pause loop\\n\", state->ith);\n            ggml_mutex_lock_shared(&threadpool->mutex);\n            if (threadpool->pause) {\n                ggml_cond_wait(&threadpool->cond, &threadpool->mutex);\n            }\n            GGML_PRINT_DEBUG(\"thread #%d resuming after wait\\n\", state->ith);\n            ggml_mutex_unlock_shared(&threadpool->mutex);\n        }\n\n        // This needs to be checked for after the cond_wait\n        if (threadpool->stop) break;\n\n        // Check if there is new work\n        // The main thread is the only one that can dispatch new work\n\n        ggml_graph_compute_check_for_work(state);\n        if (state->pending) {\n            state->pending = false;\n            ggml_graph_compute_thread(state);\n        }\n    }\n\n    return (thread_ret_t) 0;\n}\n\n// Start processing new graph\nstatic void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads)\n{\n    // Always take the mutex here because the worker threads are doing hybrid poll/wait\n\n    ggml_mutex_lock(&threadpool->mutex);\n\n    // Update the number of active threads and the graph count\n    int n_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed) >> GGML_THREADPOOL_N_THREADS_BITS;\n    n_graph = ((n_graph + 1) << GGML_THREADPOOL_N_THREADS_BITS) | (n_threads & GGML_THREADPOOL_N_THREADS_MASK);\n\n    GGML_PRINT_DEBUG(\"compute-kickoff: n_threads %d n_graph %d\\n\", n_threads, n_graph);\n\n    // Indicate the graph is ready to be processed\n    // We need the full seq-cst fence here because of the polling threads (used in thread_sync)\n    atomic_store_explicit(&threadpool->n_graph, n_graph, memory_order_seq_cst);\n\n    if (threadpool->pause) {\n       // Update main thread prio and affinity to match the threadpool settings\n       ggml_thread_apply_priority(threadpool->prio);\n       if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {\n           ggml_thread_apply_affinity(threadpool->workers[0].cpumask);\n       }\n\n       // resume does cond broadcast\n       ggml_threadpool_resume_locked(threadpool);\n    } else {\n       ggml_cond_broadcast(&threadpool->cond);\n    }\n\n    ggml_mutex_unlock(&threadpool->mutex);\n}\n\n#endif // GGML_USE_OPENMP\n\nstatic struct ggml_threadpool * ggml_threadpool_new_impl(\n    struct ggml_threadpool_params * tpp,\n               struct ggml_cgraph * cgraph,\n                struct ggml_cplan * cplan) {\n\n    struct ggml_threadpool * threadpool =\n        ggml_aligned_malloc(sizeof(struct ggml_threadpool));\n    {\n        threadpool->cgraph           = cgraph;\n        threadpool->cplan            = cplan;\n        threadpool->n_graph          = 0;\n        threadpool->n_barrier        = 0;\n        threadpool->n_barrier_passed = 0;\n        threadpool->current_chunk    = 0;\n        threadpool->stop             = false;\n        threadpool->pause            = tpp->paused;\n        threadpool->abort            = -1;\n        threadpool->workers          = NULL;\n        threadpool->n_threads        = tpp->n_threads;\n        threadpool->poll             = tpp->poll;\n        threadpool->prio             = tpp->prio;\n        threadpool->ec               = GGML_STATUS_SUCCESS;\n    }\n\n    // Allocate and init workers state\n    const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;\n    struct ggml_compute_state * workers = ggml_aligned_malloc(workers_size);\n\n    memset(workers, 0, workers_size);\n    for (int j = 0; j < tpp->n_threads; j++) {\n        workers[j].threadpool = threadpool;\n        workers[j].ith        = j;\n    }\n\n    threadpool->workers = workers;\n\n#ifdef GGML_USE_OPENMP\n    int32_t cpumask_iter = 0;\n\n    // Compute CPU masks for each thread\n    for (int j = 0; j < tpp->n_threads; j++) {\n        ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);\n    }\n#else // GGML_USE_OPENMP\n    ggml_mutex_init(&threadpool->mutex);\n    ggml_cond_init(&threadpool->cond);\n\n    // Spin the threads for all workers, and update CPU placements.\n    // Place the main thread last (towards the higher numbered CPU cores).\n\n    int32_t cpumask_iter = 0;\n\n    for (int j = 1; j < tpp->n_threads; j++) {\n        ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);\n\n        int32_t rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_secondary_thread, &workers[j]);\n        GGML_ASSERT(rc == 0);\n    }\n\n    ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);\n\n    if (!threadpool->pause) {\n        // Update main thread prio and affinity at the start, otherwise we'll do it in resume\n        ggml_thread_apply_priority(threadpool->prio);\n        if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {\n            ggml_thread_apply_affinity(threadpool->workers[0].cpumask);\n        }\n    }\n#endif // GGML_USE_OPENMP\n\n    return threadpool;\n}\n\nstruct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {\n    return ggml_threadpool_new_impl(tpp, NULL, NULL);\n}\n\nenum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {\n    ggml_cpu_init();\n\n    GGML_ASSERT(cplan);\n    GGML_ASSERT(cplan->n_threads > 0);\n    GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL);\n\n    int n_threads                               = cplan->n_threads;\n    struct ggml_threadpool * threadpool = cplan->threadpool;\n\n    bool disposable_threadpool = false;\n\n    if (threadpool == NULL) {\n        //GGML_PRINT_DEBUG(\"Threadpool is not specified. Will create a disposable threadpool : n_threads %d\\n\", n_threads);\n        disposable_threadpool = true;\n\n        struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);\n        threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan);\n    } else {\n        // Reset some of the parameters that need resetting\n        // No worker threads should be accessing the parameters below at this stage\n        threadpool->cgraph           = cgraph;\n        threadpool->cplan            = cplan;\n        threadpool->current_chunk    = 0;\n        threadpool->abort            = -1;\n        threadpool->ec               = GGML_STATUS_SUCCESS;\n    }\n\n#ifdef GGML_USE_OPENMP\n    if (n_threads > 1) {\n        #pragma omp parallel num_threads(n_threads)\n        {\n            #pragma omp single\n            {\n                // update the number of threads from the actual number of threads that we got from OpenMP\n                n_threads = omp_get_num_threads();\n                atomic_store_explicit(&threadpool->n_graph, n_threads, memory_order_relaxed);\n            }\n\n            // Apply thread CPU mask and priority\n            int ith = omp_get_thread_num();\n\n            ggml_thread_apply_priority(threadpool->prio);\n            if (ggml_thread_cpumask_is_valid(threadpool->workers[ith].cpumask)) {\n                ggml_thread_apply_affinity(threadpool->workers[ith].cpumask);\n            }\n            ggml_graph_compute_thread(&threadpool->workers[ith]);\n        }\n    } else {\n        atomic_store_explicit(&threadpool->n_graph, 1, memory_order_relaxed);\n        ggml_graph_compute_thread(&threadpool->workers[0]);\n    }\n#else\n    if (n_threads > threadpool->n_threads) {\n        GGML_LOG_WARN(\"cplan requested more threads (%d) than available (%d)\\n\", n_threads, threadpool->n_threads);\n        n_threads = threadpool->n_threads;\n    }\n\n    // Kick all threads to start the new graph\n    ggml_graph_compute_kickoff(threadpool, n_threads);\n\n    // This is a work thread too\n    ggml_graph_compute_thread(&threadpool->workers[0]);\n#endif\n\n    // don't leave affinity set on the main thread\n    clear_numa_thread_affinity();\n\n    enum ggml_status ret = threadpool->ec;\n\n    if (disposable_threadpool) {\n        ggml_threadpool_free(threadpool);\n    }\n\n    return ret;\n}\n\nenum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {\n    struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL);\n\n    cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, cplan.work_size);\n\n    return ggml_graph_compute(cgraph, &cplan);\n}\n\nvoid ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {\n    memcpy(y, x, n * sizeof(float));\n}\n\nvoid ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {\n    int64_t i = 0;\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n    for (; i + 15 < n; i += 16) {\n        __m512 x_vec = _mm512_loadu_ps(x + i);\n        __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n        _mm256_storeu_si256((__m256i *)(y + i), y_vec);\n    }\n#endif\n    for (; i + 7 < n; i += 8) {\n        __m256 x_vec = _mm256_loadu_ps(x + i);\n        __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n        _mm_storeu_si128((__m128i *)(y + i), y_vec);\n    }\n    for (; i + 3 < n; i += 4) {\n        __m128 x_vec = _mm_loadu_ps(x + i);\n        __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);\n        _mm_storel_epi64((__m128i *)(y + i), y_vec);\n    }\n#elif defined(__riscv_zvfh)\n    for (int vl; i < n; i += vl) {\n        vl = __riscv_vsetvl_e32m2(n - i);\n        vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n        vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);\n        __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);\n    }\n#endif\n    for (; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(x[i]);\n    }\n}\n\nvoid ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {\n    int64_t i = 0;\n#if defined(__F16C__)\n#if defined(__AVX512F__)\n    for (; i + 15 < n; i += 16) {\n        __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i));\n        __m512 y_vec = _mm512_cvtph_ps(x_vec);\n        _mm512_storeu_ps(y + i, y_vec);\n    }\n#endif\n    for (; i + 7 < n; i += 8) {\n        __m128i x_vec = _mm_loadu_si128((const __m128i *)(x + i));\n        __m256 y_vec = _mm256_cvtph_ps(x_vec);\n        _mm256_storeu_ps(y + i, y_vec);\n    }\n    for (; i + 3 < n; i += 4) {\n        __m128i x_vec = _mm_loadl_epi64((const __m128i *)(x + i));\n        __m128 y_vec = _mm_cvtph_ps(x_vec);\n        _mm_storeu_ps(y + i, y_vec);\n    }\n\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin)\n    // calculate step size\n    const int epr = __riscv_vsetvlmax_e16m2();\n    const int step = epr * 2;\n    const int np = (n & ~(step - 1));\n\n    // unroll by 2\n    for (; i < np; i += step) {\n        vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, epr);\n        vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, epr);\n        __riscv_vse32_v_f32m4(y + i, ay0, epr);\n\n        vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16*)x + i + epr, epr);\n        vfloat32m4_t ay1 = __riscv_vfwcvt_f_f_v_f32m4(ax1, epr);\n        __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);\n    }\n\n    // leftovers\n    int vl;\n    for (i = np; i < n; i += vl) {\n        vl = __riscv_vsetvl_e16m2(n - i);\n        vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, vl);\n        vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, vl);\n        __riscv_vse32_v_f32m4(y + i, ay0, vl);\n    }\n\n#endif\n\n    for (; i < n; ++i) {\n        y[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n}\n\nvoid ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) {\n    int64_t i = 0;\n    for (; i < n; ++i) {\n        y[i] = GGML_FP32_TO_BF16(x[i]);\n    }\n}\n\nvoid ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) {\n    int64_t i = 0;\n    for (; i < n; ++i) {\n        y[i] = x[i];\n    }\n}\n\nvoid ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {\n    int64_t i = 0;\n#if defined(__AVX2__)\n#if defined(__AVX512F__)\n    for (; i + 15 < n; i += 16) {\n        _mm512_storeu_ps(y + i,\n                        _mm512_castsi512_ps(\n                            _mm512_slli_epi32(\n                                _mm512_cvtepu16_epi32(\n                                    _mm256_loadu_si256(\n                                        (const __m256i *)(x + i))),\n                                16)));\n    }\n#endif\n    for (; i + 7 < n; i += 8) {\n        _mm256_storeu_ps(y + i,\n                        _mm256_castsi256_ps(\n                            _mm256_slli_epi32(\n                                _mm256_cvtepu16_epi32(\n                                    _mm_loadu_si128(\n                                        (const __m128i *)(x + i))),\n                                16)));\n    }\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin)\n    // calculate step size\n    const int epr = __riscv_vsetvlmax_e16m2();\n    const int step = epr * 2;\n    const int np = (n & ~(step - 1));\n\n    // unroll by 2\n    for (; i < np; i += step) {\n        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, epr);\n        vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, epr);\n        __riscv_vse32_v_f32m4(y + i, ay0, epr);\n\n        vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16*)x + i + epr, epr);\n        vfloat32m4_t ay1 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax1, epr);\n        __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);\n    }\n\n    // leftovers\n    int vl;\n    for (i = np; i < n; i += vl) {\n        vl = __riscv_vsetvl_e16m2(n - i);\n        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, vl);\n        vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, vl);\n        __riscv_vse32_v_f32m4(y + i, ay0, vl);\n    }\n#endif\n    for (; i < n; i++) {\n        y[i] = GGML_BF16_TO_FP32(x[i]);\n    }\n}\n\nint ggml_cpu_has_avx(void) {\n#if defined(__AVX__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx_vnni(void) {\n#if defined(__AVXVNNI__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx2(void) {\n#if defined(__AVX2__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512(void) {\n#if defined(__AVX512F__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_vbmi(void) {\n#if defined(__AVX512VBMI__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_vnni(void) {\n#if defined(__AVX512VNNI__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_avx512_bf16(void) {\n#if defined(__AVX512BF16__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_amx_int8(void) {\n#if defined(__AMX_INT8__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_bmi2(void) {\n#if defined(__BMI2__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_fma(void) {\n#if defined(__FMA__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_arm_fma(void) {\n#if defined(__ARM_FEATURE_FMA)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_riscv_v(void) {\n#if defined(__riscv_v_intrinsic)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_get_rvv_vlen(void) {\n#if defined(__riscv) && defined(__riscv_v_intrinsic)\n    return ggml_riscv_arch_features.rvv_vlen;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_f16c(void) {\n#if defined(__F16C__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_fp16_va(void) {\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_wasm_simd(void) {\n#if defined(__wasm_simd128__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_llamafile(void) {\n#if defined(GGML_USE_LLAMAFILE)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_sse3(void) {\n#if defined(__SSE3__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_ssse3(void) {\n#if defined(__SSSE3__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_vsx(void) {\n#if defined(__POWER9_VECTOR__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_vxe(void) {\n#if defined(__VXE__) || defined(__VXE2__)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_neon(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_NEON)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_dotprod(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_sve(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_matmul_int8(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_get_sve_cnt(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)\n    return ggml_arm_arch_features.sve_cnt;\n#else\n    return 0;\n#endif\n}\n\nint ggml_cpu_has_sme(void) {\n#if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME)\n    return 1;\n#else\n    return 0;\n#endif\n}\n\nvoid ggml_cpu_init(void) {\n    // needed to initialize ggml_time\n    {\n        struct ggml_init_params params = { 0, NULL, false };\n        struct ggml_context * ctx = ggml_init(params);\n        ggml_free(ctx);\n    }\n\n    ggml_critical_section_start();\n\n    static bool is_first_call = true;\n\n    if (is_first_call) {\n        // initialize GELU, Quick GELU, SILU and EXP F32 tables\n        {\n            const uint64_t t_start = ggml_time_us(); UNUSED(t_start);\n\n            for (int i = 0; i < (1 << 16); ++i) {\n                union {\n                    uint16_t u16;\n                    ggml_fp16_t fp16;\n                } u = {i};\n                float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16);\n                ggml_table_f32_f16[i] = f;\n                ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f));\n                ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f));\n            }\n\n            // initialize E8M0 half table (256 entries)\n            for (int i = 0; i < (1 << 8); ++i) {\n                ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i);\n            }\n\n            const uint64_t t_end = ggml_time_us(); UNUSED(t_end);\n\n            GGML_PRINT_DEBUG(\"%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\\n\", __func__, (t_end - t_start)/1000.0);\n\n#ifdef GGML_USE_OPENMP\n            //if (!getenv(\"OMP_WAIT_POLICY\")) {\n            //    // set the wait policy to active, so that OpenMP threads don't sleep\n            //    setenv(\"OMP_WAIT_POLICY\", \"active\", 0)\n            //}\n\n            if (!getenv(\"KMP_BLOCKTIME\")) {\n                // set the time to wait before sleeping a thread\n                // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases\n#ifdef _WIN32\n                _putenv_s(\"KMP_BLOCKTIME\", \"200\"); // 200ms\n#else\n                setenv(\"KMP_BLOCKTIME\", \"200\", 0); // 200ms\n#endif\n            }\n#endif\n        }\n\n#if defined(__ARM_ARCH)\n        ggml_init_arm_arch_features();\n#endif\n\n#if defined(__riscv)\n        ggml_init_riscv_arch_features();\n#endif\n\n        is_first_call = false;\n    }\n\n    ggml_critical_section_end();\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/ggml-cpu.cpp",
    "content": "#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"repack.h\"\n#include \"traits.h\"\n#include \"ggml-impl.h\"\n#include \"amx/amx.h\"\n\n#include <cctype>\n#include <string>\n#include <vector>\n\n#ifdef GGML_USE_CPU_HBM\n#    include \"hbm.h\"\n#endif\n\n#ifdef GGML_USE_CPU_KLEIDIAI\n#    include \"kleidiai/kleidiai.h\"\n#endif\n\n#ifdef GGML_USE_CPU_RISCV64_SPACEMIT\n#    include \"spacemit/ime.h\"\n#endif\n\n#if defined(_WIN32)\n#    define WIN32_LEAN_AND_MEAN\n#    ifndef NOMINMAX\n#        define NOMINMAX\n#    endif\n#    include <windows.h>\n#else\n#    include <unistd.h>\n#endif\n\n#if defined(__APPLE__)\n#    include <sys/sysctl.h>\n#    include <sys/types.h>\n#endif\n\n// ggml-backend interface\n\nstd::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffer_types() {\n    static std::vector<ggml_backend_buffer_type_t> bufts = []() {\n        std::vector<ggml_backend_buffer_type_t> bufts;\n\n#if defined(__AMX_INT8__) && defined(__AVX512VNNI__)\n        if (ggml_backend_amx_buffer_type()) {\n            bufts.push_back(ggml_backend_amx_buffer_type());\n        }\n#endif\n\n#ifdef GGML_USE_CPU_RISCV64_SPACEMIT\n        if (ggml_backend_cpu_riscv64_spacemit_buffer_type()) {\n            bufts.push_back(ggml_backend_cpu_riscv64_spacemit_buffer_type());\n        }\n#endif\n\n#ifdef GGML_USE_CPU_KLEIDIAI\n        if (ggml_backend_cpu_kleidiai_buffer_type()) {\n            bufts.push_back(ggml_backend_cpu_kleidiai_buffer_type());\n        }\n#endif\n\n#ifdef GGML_USE_CPU_REPACK\n        if (ggml_backend_cpu_repack_buffer_type()) {\n            bufts.push_back(ggml_backend_cpu_repack_buffer_type());\n        }\n#endif\n\n        return bufts;\n    }();\n\n    return bufts;\n}\n\nstatic ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) {\n    static std::vector<ggml_backend_buffer_type_t> extra_bufts = [] {\n        std::vector<ggml_backend_buffer_type_t> bufts = ggml_backend_cpu_get_extra_buffer_types();\n        bufts.push_back(nullptr);\n        return bufts;\n    }();\n\n    return extra_bufts.data();\n\n    GGML_UNUSED(device);\n}\n\nstatic bool ggml_backend_cpu_is_extra_buffer_type(ggml_backend_buffer_type_t buft) {\n    for (auto * extra : ggml_backend_cpu_get_extra_buffer_types()) {\n        if (extra == buft) {\n            return true;\n        }\n    }\n    return false;\n}\n\n// CPU backend - backend (stream)\n\nstruct ggml_backend_cpu_context {\n    int                 n_threads;\n    ggml_threadpool_t   threadpool;\n\n    uint8_t *           work_data;\n    size_t              work_size;\n\n    ggml_abort_callback abort_callback;\n    void *              abort_callback_data;\n\n    bool                use_ref;  // use reference implementation\n};\n\nstatic const char * ggml_backend_cpu_get_name(ggml_backend_t backend) {\n    return \"CPU\";\n\n    GGML_UNUSED(backend);\n}\n\nstatic void ggml_backend_cpu_free(ggml_backend_t backend) {\n    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n    delete[] cpu_ctx->work_data;\n    delete cpu_ctx;\n    delete backend;\n}\n\nstruct ggml_backend_plan_cpu {\n    struct ggml_cplan cplan;\n    struct ggml_cgraph cgraph;\n};\n\nstatic ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backend_t backend, const struct ggml_cgraph * cgraph) {\n    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n\n    struct ggml_backend_plan_cpu * cpu_plan = new ggml_backend_plan_cpu;\n\n    cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);\n    cpu_plan->cgraph = *cgraph; // FIXME: deep copy\n\n    if (cpu_plan->cplan.work_size > 0) {\n        cpu_plan->cplan.work_data = new uint8_t[cpu_plan->cplan.work_size];\n        if (cpu_plan->cplan.work_data == NULL) {\n            delete cpu_plan;\n            return NULL;\n        }\n    }\n\n    cpu_plan->cplan.abort_callback      = cpu_ctx->abort_callback;\n    cpu_plan->cplan.abort_callback_data = cpu_ctx->abort_callback_data;\n    cpu_plan->cplan.use_ref             = cpu_ctx->use_ref;\n\n    return cpu_plan;\n}\n\nstatic void ggml_backend_cpu_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n    struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;\n\n    delete[] cpu_plan->cplan.work_data;\n    delete cpu_plan;\n\n    GGML_UNUSED(backend);\n}\n\nstatic enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {\n    struct ggml_backend_plan_cpu * cpu_plan = (struct ggml_backend_plan_cpu *)plan;\n\n    return ggml_graph_compute(&cpu_plan->cgraph, &cpu_plan->cplan);\n\n    GGML_UNUSED(backend);\n}\n\nstatic enum ggml_status ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {\n    struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;\n\n    struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);\n\n    if (cpu_ctx->work_size < cplan.work_size) {\n        delete[] cpu_ctx->work_data;\n        cpu_ctx->work_data = new uint8_t[cplan.work_size];\n        if (cpu_ctx->work_data == NULL) {\n            cpu_ctx->work_size = 0;\n            return GGML_STATUS_ALLOC_FAILED;\n        }\n        cpu_ctx->work_size = cplan.work_size;\n    }\n    cplan.work_data = (uint8_t *)cpu_ctx->work_data;\n\n    cplan.abort_callback      = cpu_ctx->abort_callback;\n    cplan.abort_callback_data = cpu_ctx->abort_callback_data;\n    cplan.use_ref             = cpu_ctx->use_ref;\n\n    return ggml_graph_compute(cgraph, &cplan);\n}\n\nstatic const struct ggml_backend_i ggml_backend_cpu_i = {\n    /* .get_name                = */ ggml_backend_cpu_get_name,\n    /* .free                    = */ ggml_backend_cpu_free,\n    /* .set_tensor_async        = */ NULL,\n    /* .get_tensor_async        = */ NULL,\n    /* .cpy_tensor_async        = */ NULL,\n    /* .synchronize             = */ NULL,\n    /* .graph_plan_create       = */ ggml_backend_cpu_graph_plan_create,\n    /* .graph_plan_free         = */ ggml_backend_cpu_graph_plan_free,\n    /* .graph_plan_update       = */ NULL,\n    /* .graph_plan_compute      = */ ggml_backend_cpu_graph_plan_compute,\n    /* .graph_compute           = */ ggml_backend_cpu_graph_compute,\n    /* .event_record            = */ NULL,\n    /* .event_wait              = */ NULL,\n    /* .graph_optimize          = */ NULL,\n};\n\nstatic ggml_guid_t ggml_backend_cpu_guid(void) {\n    static ggml_guid guid = { 0xaa, 0x67, 0xc7, 0x43, 0x96, 0xe6, 0xa3, 0x8a, 0xe3, 0xaf, 0xea, 0x92, 0x36, 0xbc, 0xfc, 0x89 };\n    return &guid;\n}\n\nggml_backend_t ggml_backend_cpu_init(void) {\n    // initialize CPU backend now to avoid slowing the first graph computation\n    ggml_cpu_init();\n\n    struct ggml_backend_cpu_context * ctx = new ggml_backend_cpu_context;\n    if (ctx == NULL) {\n        return NULL;\n    }\n\n    ctx->n_threads           = GGML_DEFAULT_N_THREADS;\n    ctx->threadpool          = NULL;\n    ctx->work_data           = NULL;\n    ctx->work_size           = 0;\n    ctx->abort_callback      = NULL;\n    ctx->abort_callback_data = NULL;\n    ctx->use_ref             = false;\n\n    ggml_backend_t cpu_backend = new ggml_backend {\n        /* .guid    = */ ggml_backend_cpu_guid(),\n        /* .iface   = */ ggml_backend_cpu_i,\n        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ ctx,\n    };\n\n    if (cpu_backend == NULL) {\n        delete ctx;\n        return NULL;\n    }\n\n    return cpu_backend;\n}\n\nbool ggml_backend_is_cpu(ggml_backend_t backend) {\n    return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());\n}\n\nvoid ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {\n    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n    ctx->n_threads = n_threads;\n}\n\nvoid ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool_t threadpool) {\n    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n\n    if (ctx->threadpool && ctx->threadpool != threadpool) {\n        // already had a different threadpool, pause/suspend it before switching\n        ggml_threadpool_pause(ctx->threadpool);\n    }\n    ctx->threadpool = threadpool;\n}\n\nvoid ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {\n    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n    ctx->abort_callback = abort_callback;\n    ctx->abort_callback_data = abort_callback_data;\n}\n\nvoid ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref) {\n    GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));\n\n    struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;\n    ctx->use_ref = use_ref;\n}\n\n// CPU backend - device\n\nstruct ggml_backend_cpu_device_context {\n    std::string description = \"CPU\";\n\n    ggml_backend_cpu_device_context() {\n#ifdef __APPLE__\n        size_t len = 0;\n        if (!sysctlbyname(\"machdep.cpu.brand_string\", NULL, &len, NULL, 0)) {\n            description.resize(len);\n            sysctlbyname(\"machdep.cpu.brand_string\", &description[0], &len, NULL, 0); // NOLINT\n        }\n#elif defined(__linux__)\n        FILE * f = fopen(\"/proc/cpuinfo\", \"r\");\n        if (f) {\n            char buf[1024];\n            while (fgets(buf, sizeof(buf), f)) {\n                if (strncmp(buf, \"model name\", 10) == 0) {\n                    char * p = strchr(buf, ':');\n                    if (p) {\n                        p++;\n                        while (std::isspace(*p)) {\n                            p++;\n                        }\n                        while (std::isspace(p[strlen(p) - 1])) {\n                            p[strlen(p) - 1] = '\\0';\n                        }\n                        description = p;\n                        break;\n                    }\n                }\n            }\n            fclose(f);\n        }\n#elif defined(_WIN32)\n        HKEY hKey;\n        if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,\n                        TEXT(\"HARDWARE\\\\DESCRIPTION\\\\System\\\\CentralProcessor\\\\0\"),\n                        0,\n                        KEY_READ,\n                        &hKey) == ERROR_SUCCESS) {\n            DWORD cpu_brand_size = 0;\n            if (RegQueryValueExA(hKey,\n                                \"ProcessorNameString\",\n                                NULL,\n                                NULL,\n                                NULL,\n                                &cpu_brand_size) == ERROR_SUCCESS) {\n                description.resize(cpu_brand_size);\n                if (RegQueryValueExA(hKey,\n                                    \"ProcessorNameString\",\n                                    NULL,\n                                    NULL,\n                                    (LPBYTE)&description[0], // NOLINT\n                                    &cpu_brand_size) == ERROR_SUCCESS) {\n                    if (description.find('\\0') != std::string::npos) {\n                        description.resize(description.find('\\0'));\n                    }\n                }\n            }\n            RegCloseKey(hKey);\n        }\n#endif\n    }\n};\n\nstatic const char * ggml_backend_cpu_device_get_name(ggml_backend_dev_t dev) {\n    return \"CPU\";\n\n    GGML_UNUSED(dev);\n}\n\nstatic const char * ggml_backend_cpu_device_get_description(ggml_backend_dev_t dev) {\n    struct ggml_backend_cpu_device_context * ctx = (struct ggml_backend_cpu_device_context *)dev->context;\n\n    return ctx->description.c_str();\n}\n\nstatic void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {\n#ifdef _WIN32\n    MEMORYSTATUSEX status;\n    status.dwLength = sizeof(status);\n    GlobalMemoryStatusEx(&status);\n    *total = status.ullTotalPhys;\n    *free = status.ullAvailPhys;\n#else\n    long pages = sysconf(_SC_PHYS_PAGES);\n    long page_size = sysconf(_SC_PAGE_SIZE);\n    *total = pages * page_size;\n\n    // \"free\" system memory is ill-defined, for practical purposes assume that all of it is free:\n    *free = *total;\n#endif // _WIN32\n\n    GGML_UNUSED(dev);\n}\n\nstatic enum ggml_backend_dev_type ggml_backend_cpu_device_get_type(ggml_backend_dev_t dev) {\n    return GGML_BACKEND_DEVICE_TYPE_CPU;\n\n    GGML_UNUSED(dev);\n}\n\nstatic void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {\n    props->name        = ggml_backend_cpu_device_get_name(dev);\n    props->description = ggml_backend_cpu_device_get_description(dev);\n    props->type        = ggml_backend_cpu_device_get_type(dev);\n    ggml_backend_cpu_device_get_memory(dev, &props->memory_free, &props->memory_total);\n    props->caps = {\n        /* .async                 = */ false,\n        /* .host_buffer           = */ false,\n        /* .buffer_from_host_ptr  = */ true,\n        /* .events                = */ false,\n    };\n}\n\nstatic ggml_backend_t ggml_backend_cpu_device_init_backend(ggml_backend_dev_t dev, const char * params) {\n    return ggml_backend_cpu_init();\n\n    GGML_UNUSED(dev);\n    GGML_UNUSED(params);\n}\n\nstatic ggml_backend_buffer_type_t ggml_backend_cpu_device_get_buffer_type(ggml_backend_dev_t dev) {\n    return ggml_backend_cpu_buffer_type();\n\n    GGML_UNUSED(dev);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {\n    return ggml_backend_cpu_buffer_from_ptr(ptr, size);\n\n    GGML_UNUSED(dev);\n    GGML_UNUSED(max_tensor_size);\n}\n\nstatic bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {\n    const struct ggml_tensor * src0 = op->src[0];\n    const struct ggml_tensor * src1 = op->src[1];\n\n    if (op->op == GGML_OP_NONE || op->op == GGML_OP_RESHAPE || op->op == GGML_OP_VIEW || op->op == GGML_OP_PERMUTE || op->op == GGML_OP_TRANSPOSE) {\n        return true;\n    }\n\n    // check extra buffer types\n    // note: only the first sources are checked for extra buffer types to reduce overhead, increase if necessary\n    for (int i = 0; i < 4; i++) {\n        if (op->src[i] && op->src[i]->buffer &&\n            ggml_backend_cpu_is_extra_buffer_type(op->src[i]->buffer->buft)) {\n            auto * buf_extra = (ggml::cpu::extra_buffer_type *) op->src[i]->buffer->buft->context;\n            return buf_extra->supports_op(dev, op);\n        }\n    }\n\n    switch (op->op) {\n        case GGML_OP_CPY:\n        case GGML_OP_SET_ROWS:\n            return\n                op->type != GGML_TYPE_IQ3_XXS &&\n                op->type != GGML_TYPE_IQ3_S   &&\n                op->type != GGML_TYPE_IQ2_XXS &&\n                op->type != GGML_TYPE_IQ2_XS  &&\n                op->type != GGML_TYPE_IQ2_S   &&\n                op->type != GGML_TYPE_IQ1_S   &&\n                op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float\n        case GGML_OP_MUL_MAT:\n            return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;\n        case GGML_OP_SOFT_MAX_BACK: {\n            if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {\n                return false;\n            }\n            float max_bias = 0.0f;\n\n            memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float));\n\n            return max_bias == 0.0f;\n        }\n        case GGML_OP_IM2COL_BACK:\n            return src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32;\n        case GGML_OP_GET_ROWS_BACK:\n            return src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16;\n        case GGML_OP_OUT_PROD:\n            return (src0->type == GGML_TYPE_F32 || (ggml_is_quantized(src0->type) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) &&\n                src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;\n        default:\n            return true;\n    }\n}\n\nstatic bool ggml_backend_cpu_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {\n    return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);\n    GGML_UNUSED(dev);\n}\n\nstatic const struct ggml_backend_device_i ggml_backend_cpu_device_i = {\n    /* .get_name             = */ ggml_backend_cpu_device_get_name,\n    /* .get_description      = */ ggml_backend_cpu_device_get_description,\n    /* .get_memory           = */ ggml_backend_cpu_device_get_memory,\n    /* .get_type             = */ ggml_backend_cpu_device_get_type,\n    /* .get_props            = */ ggml_backend_cpu_device_get_props,\n    /* .init_backend         = */ ggml_backend_cpu_device_init_backend,\n    /* .get_buffer_type      = */ ggml_backend_cpu_device_get_buffer_type,\n    /* .get_host_buffer_type = */ NULL,\n    /* .buffer_from_host_ptr = */ ggml_backend_cpu_device_buffer_from_host_ptr,\n    /* .supports_op          = */ ggml_backend_cpu_device_supports_op,\n    /* .supports_buft        = */ ggml_backend_cpu_device_supports_buft,\n    /* .offload_op           = */ NULL,\n    /* .event_new            = */ NULL,\n    /* .event_free           = */ NULL,\n    /* .event_synchronize    = */ NULL,\n};\n\n// CPU backend - backend (reg)\n\nstatic const char * ggml_backend_cpu_reg_get_name(ggml_backend_reg_t reg) {\n    return \"CPU\";\n\n    GGML_UNUSED(reg);\n}\n\nstatic size_t ggml_backend_cpu_reg_get_device_count(ggml_backend_reg_t reg) {\n    return 1;\n\n    GGML_UNUSED(reg);\n}\n\nstatic ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg, size_t index) {\n    GGML_ASSERT(index == 0);\n\n    static ggml_backend_cpu_device_context ctx;\n    static ggml_backend_device ggml_backend_cpu_device = {\n        /* .iface   = */ ggml_backend_cpu_device_i,\n        /* .reg     = */ reg,\n        /* .context = */ &ctx,\n    };\n\n    return &ggml_backend_cpu_device;\n}\n\n// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,\n// and additionally to allow other backends to expose their own list of features that applications can query using the same API\nstatic ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t reg) {\n    static std::vector<ggml_backend_feature> features = []() {\n        ggml_cpu_init();\n\n        std::vector<ggml_backend_feature> features;\n        if (ggml_cpu_has_sse3()) {\n            features.push_back({ \"SSE3\", \"1\" });\n        }\n        if (ggml_cpu_has_ssse3()) {\n            features.push_back({ \"SSSE3\", \"1\" });\n        }\n        if (ggml_cpu_has_avx()) {\n            features.push_back({ \"AVX\", \"1\" });\n        }\n        if (ggml_cpu_has_avx_vnni()) {\n            features.push_back({ \"AVX_VNNI\", \"1\" });\n        }\n        if (ggml_cpu_has_avx2()) {\n            features.push_back({ \"AVX2\", \"1\" });\n        }\n        if (ggml_cpu_has_f16c()) {\n            features.push_back({ \"F16C\", \"1\" });\n        }\n        if (ggml_cpu_has_fma()) {\n            features.push_back({ \"FMA\", \"1\" });\n        }\n        if (ggml_cpu_has_bmi2()) {\n            features.push_back({ \"BMI2\", \"1\" });\n        }\n        if (ggml_cpu_has_avx512()) {\n            features.push_back({ \"AVX512\", \"1\" });\n        }\n        if (ggml_cpu_has_avx512_vbmi()) {\n            features.push_back({ \"AVX512_VBMI\", \"1\" });\n        }\n        if (ggml_cpu_has_avx512_vnni()) {\n            features.push_back({ \"AVX512_VNNI\", \"1\" });\n        }\n        if (ggml_cpu_has_avx512_bf16()) {\n            features.push_back({ \"AVX512_BF16\", \"1\" });\n        }\n        if (ggml_cpu_has_amx_int8()) {\n            features.push_back({ \"AMX_INT8\", \"1\" });\n        }\n        if (ggml_cpu_has_neon()) {\n            features.push_back({ \"NEON\", \"1\" });\n        }\n        if (ggml_cpu_has_arm_fma()) {\n            features.push_back({ \"ARM_FMA\", \"1\" });\n        }\n        if (ggml_cpu_has_fp16_va()) {\n            features.push_back({ \"FP16_VA\", \"1\" });\n        }\n        if (ggml_cpu_has_matmul_int8()) {\n            features.push_back({ \"MATMUL_INT8\", \"1\" });\n        }\n        if (ggml_cpu_has_sve()) {\n            features.push_back({ \"SVE\", \"1\" });\n        }\n        if (ggml_cpu_has_dotprod()) {\n            features.push_back({ \"DOTPROD\", \"1\" });\n        }\n        if (ggml_cpu_get_sve_cnt() > 0) {\n            static std::string sve_cnt = std::to_string(ggml_cpu_get_sve_cnt());\n            features.push_back({ \"SVE_CNT\", sve_cnt.c_str() });\n        }\n        if (ggml_cpu_has_sme()) {\n            features.push_back({ \"SME\", \"1\" });\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            features.push_back({ \"RISCV_V\", \"1\" });\n        }\n        if (ggml_cpu_get_rvv_vlen() > 0) {\n            static std::string rvv_vlen = std::to_string(ggml_cpu_get_rvv_vlen());\n            features.push_back({ \"RVV_VLEN\", rvv_vlen.c_str() });\n        }\n        if (ggml_cpu_has_vsx()) {\n            features.push_back({ \"VSX\", \"1\" });\n        }\n        if (ggml_cpu_has_vxe()) {\n            features.push_back({ \"VXE\", \"1\" });\n        }\n        if (ggml_cpu_has_wasm_simd()) {\n            features.push_back({ \"WASM_SIMD\", \"1\" });\n        }\n        if (ggml_cpu_has_llamafile()) {\n            features.push_back({ \"LLAMAFILE\", \"1\" });\n        }\n    #ifdef GGML_USE_ACCELERATE\n        features.push_back({ \"ACCELERATE\", \"1\" });\n    #endif\n    #ifdef GGML_USE_CPU_HBM\n        features.push_back({ \"CPU_HBM\", \"1\" });\n    #endif\n    #ifdef GGML_USE_OPENMP\n        features.push_back({ \"OPENMP\", \"1\" });\n    #endif\n    #ifdef GGML_USE_CPU_KLEIDIAI\n        features.push_back({ \"KLEIDIAI\", \"1\" });\n    #endif\n    #ifdef GGML_USE_CPU_REPACK\n        features.push_back({ \"REPACK\", \"1\" });\n    #endif\n\n        features.push_back({ nullptr, nullptr });\n\n        return features;\n    }();\n\n    return features.data();\n\n    GGML_UNUSED(reg);\n}\n\nstatic void * ggml_backend_cpu_get_proc_address(ggml_backend_reg_t reg, const char * name) {\n    if (strcmp(name, \"ggml_backend_set_n_threads\") == 0) {\n        ggml_backend_set_n_threads_t fct = ggml_backend_cpu_set_n_threads;\n        return (void *)fct;\n    }\n    if (strcmp(name, \"ggml_backend_dev_get_extra_bufts\") == 0) {\n        ggml_backend_dev_get_extra_bufts_t fct = ggml_backend_cpu_device_get_extra_buffers_type;\n        return (void *)fct;\n    }\n    if (strcmp(name, \"ggml_backend_get_features\") == 0) {\n        return (void *)ggml_backend_cpu_get_features;\n    }\n    if (strcmp(name, \"ggml_backend_set_abort_callback\") == 0) {\n        return (void *)ggml_backend_cpu_set_abort_callback;\n    }\n    if (strcmp(name, \"ggml_backend_cpu_numa_init\") == 0) {\n        return (void *)ggml_numa_init;\n    }\n    if (strcmp(name, \"ggml_backend_cpu_is_numa\") == 0) {\n        return (void *)ggml_is_numa;\n    }\n    if (strcmp(name, \"ggml_backend_cpu_set_use_ref\") == 0) {\n        return (void *)ggml_backend_cpu_set_use_ref;\n    }\n\n    // threadpool - TODO:  move to ggml-base\n    if (strcmp(name, \"ggml_threadpool_new\") == 0) {\n        return (void *)ggml_threadpool_new;\n    }\n    if (strcmp(name, \"ggml_threadpool_free\") == 0) {\n        return (void *)ggml_threadpool_free;\n    }\n    if (strcmp(name, \"ggml_backend_cpu_set_threadpool\") == 0) {\n        return (void *)ggml_backend_cpu_set_threadpool;\n    }\n\n    return NULL;\n\n    GGML_UNUSED(reg);\n}\n\nstatic const struct ggml_backend_reg_i ggml_backend_cpu_reg_i = {\n    /* .get_name         = */ ggml_backend_cpu_reg_get_name,\n    /* .get_device_count = */ ggml_backend_cpu_reg_get_device_count,\n    /* .get_device       = */ ggml_backend_cpu_reg_get_device,\n    /* .get_proc_address = */ ggml_backend_cpu_get_proc_address,\n};\n\nggml_backend_reg_t ggml_backend_cpu_reg(void) {\n    // init CPU feature detection\n    ggml_cpu_init();\n\n    static struct ggml_backend_reg ggml_backend_cpu_reg = {\n        /* .api_version = */ GGML_BACKEND_API_VERSION,\n        /* .iface       = */ ggml_backend_cpu_reg_i,\n        /* .context     = */ NULL,\n    };\n\n    return &ggml_backend_cpu_reg;\n}\n\nGGML_BACKEND_DL_IMPL(ggml_backend_cpu_reg)\n"
  },
  {
    "path": "ggml/src/ggml-cpu/hbm.cpp",
    "content": "#ifdef GGML_USE_CPU_HBM\n\n#include \"ggml-backend.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n\n#include \"hbm.h\"\n\n// buffer type HBM\n\n#include <hbwmalloc.h>\n\nstatic const char * ggml_backend_cpu_hbm_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"CPU_HBM\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {\n    hbw_free(buffer->context);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n                                                                           size_t                     size) {\n    void * ptr;\n    int    result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);\n    if (result != 0) {\n        GGML_LOG_ERROR(\"failed to allocate HBM buffer of size %zu\\n\", size);\n        return NULL;\n    }\n\n    ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);\n    buffer->buft                 = buft;\n    buffer->iface.free_buffer    = ggml_backend_cpu_hbm_buffer_free_buffer;\n\n    return buffer;\n}\n\nggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {\n        /* .iface    = */ {\n                           /* .get_name         = */ ggml_backend_cpu_hbm_buffer_type_get_name,\n                           /* .alloc_buffer     = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,\n                           /* .get_alignment    = */ ggml_backend_cpu_buffer_type_get_alignment,\n                           /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX\n                           /* .get_alloc_size   = */ nullptr,  // defaults to ggml_nbytes\n                           /* .is_host          = */ ggml_backend_cpu_buffer_type_is_host,\n                           },\n        /* .context  = */ nullptr,\n    };\n\n    return &ggml_backend_cpu_buffer_type_hbm;\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/hbm.h",
    "content": "#pragma once\n\n#include \"ggml-backend.h\"\n#include \"ggml.h\"\n\n// GGML CPU internal header\n\nggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);\n"
  },
  {
    "path": "ggml/src/ggml-cpu/kleidiai/kernels.cpp",
    "content": "// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates <open-source-office@arm.com>\n// SPDX-License-Identifier: MIT\n//\n\n// KleidiAI micro-kernels\n#include \"kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot.h\"\n#include \"kai_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod.h\"\n#include \"kai_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm.h\"\n#include \"kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod.h\"\n#include \"kai_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa.h\"\n\n#include \"kai_lhs_pack_bf16p2vlx2_f32_sme.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p_f32.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p4x8sb_f32_neon.h\"\n#include \"kai_lhs_quant_pack_qsi8d32p_f32_neon.h\"\n#include \"kai_lhs_quant_pack_qai8dxp_f32.h\"\n\n#include \"kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h\"\n#include \"kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h\"\n#include \"kai_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon.h\"\n#include \"kai_rhs_pack_nxk_qsi8cxp_qsi8cx_neon.h\"\n#include \"kai_lhs_pack_f16pmrx2_f32_neon.h\"\n\n#include \"kai_common.h\"\n\n#include \"simd-mappings.h\"\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\n#include \"kernels.h\"\n\n#define NELEMS(x) (sizeof(x) / sizeof(*x))\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t)>\nstatic inline size_t kernel_offs_fn3(size_t a, size_t b, size_t c) {\n    return Fn(a, b, c);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t)>\nstatic inline size_t kernel_offs_fn2(size_t a, size_t b, size_t) {\n    return Fn(a, b);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,const void*,const void*,float*,size_t,size_t,float,float)>\nstatic inline void kernel_run_fn11(size_t m, size_t n, size_t k, size_t bl,\n                                     const void* lhs, const void* rhs, void* dst,\n                                     size_t dst_stride_row, size_t dst_stride_col,\n                                     float clamp_min, float clamp_max) {\n    Fn(m, n, k, bl, lhs, rhs, static_cast<float*>(dst), dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,const void*,const void*,void*,size_t,size_t,float,float)>\nstatic inline void kernel_run_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,\n                                   const void* lhs, const void* rhs, void* dst,\n                                   size_t dst_stride_row, size_t dst_stride_col,\n                                   float clamp_min, float clamp_max) {\n    Fn(m, n, k, lhs, rhs, dst, dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,const void*,const void*,float*,size_t,size_t,float,float)>\nstatic inline void kernel_run_float_fn10(size_t m, size_t n, size_t k, size_t /*bl*/,\n                                         const void* lhs, const void* rhs, void* dst,\n                                         size_t dst_stride_row, size_t dst_stride_col,\n                                         float clamp_min, float clamp_max) {\n    Fn(m, n, k, lhs, rhs, static_cast<float*>(dst), dst_stride_row, dst_stride_col, clamp_min, clamp_max);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t)>\nstatic inline size_t lhs_ps_fn6(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {\n    return Fn(m, k, bl, mr, kr, sr);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t,size_t)>\nstatic inline size_t lhs_ps_fn5(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr) {\n    return Fn(m, k, mr, kr, sr);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t)>\nstatic inline size_t lhs_offs_fn6(size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {\n    return Fn(m_idx, k, bl, mr, kr, sr);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t,size_t)>\nstatic inline size_t lhs_offs_fn5(size_t m_idx, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr) {\n    return Fn(m_idx, k, mr, kr, sr);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const float*,size_t,void*)>\nstatic inline void lhs_pack_float_fn10(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n                                            size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n    Fn(m, k, bl, mr, kr, sr, m_idx_start, static_cast<const float*>(lhs), lhs_stride, lhs_packed);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const void*,size_t,void*)>\nstatic inline void lhs_pack_void_fn10(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n                                           size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n    Fn(m, k, bl, mr, kr, sr, m_idx_start, lhs, lhs_stride, lhs_packed);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,const void*,size_t,void*)>\nstatic inline void lhs_pack_void_fn9(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr,\n                                             size_t m_idx_start, const void* lhs, size_t lhs_stride, void* lhs_packed) {\n    Fn(m, k, mr, kr, sr, m_idx_start, lhs, lhs_stride, lhs_packed);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,const float*,size_t,void*)>\nstatic inline void lhs_pack_float_fn9_no_bl(size_t m, size_t k, size_t /*bl*/, size_t mr, size_t kr, size_t sr,\n                                            size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) {\n    Fn(m, k, mr, kr, sr, m_idx_start, static_cast<const float*>(lhs), lhs_stride, lhs_packed);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t,size_t)>\nstatic inline size_t rhs_ps_fn5(size_t n, size_t k, size_t nr, size_t kr, size_t bl) {\n    return Fn(n, k, nr, kr, bl);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t)>\nstatic inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {\n    return Fn(n, k);\n}\n\ntemplate<size_t(*Fn)(size_t,size_t,size_t,size_t)>\nstatic inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) {\n    return Fn(k, nr, kr, bl);\n}\n\ntemplate<size_t(*Fn)(size_t)>\nstatic inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {\n    return Fn(k);\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const uint8_t*,const float*,void*,size_t,const struct kai_rhs_pack_qs4cxs1s0_param*)>\nstatic inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,\n                                      size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/,\n                                      void* rhs_packed, size_t extra_bytes, const void* params) {\n    Fn(num_groups, n, k, nr, kr, sr, bl,\n       static_cast<const uint8_t*>(rhs),\n       static_cast<const float*>(bias),\n       rhs_packed, extra_bytes,\n       static_cast<const kai_rhs_pack_qs4cxs1s0_param*>(params));\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,const int8_t*,const float*,const float*,void*,size_t,const struct kai_rhs_pack_qsi8cx_params*)>\nstatic inline void rhs_pack_scale_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t /*bl*/,\n                                       size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* scale,\n                                       void* rhs_packed, size_t extra_bytes, const void* params) {\n    Fn(num_groups, n, k, nr, kr, sr,\n       static_cast<const int8_t*>(rhs),\n       static_cast<const float*>(bias),\n       static_cast<const float*>(scale),\n       rhs_packed, extra_bytes,\n       static_cast<const kai_rhs_pack_qsi8cx_params*>(params));\n}\n\ntemplate<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const void*,const void*,const void*,void*,size_t,const void*)>\nstatic inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t /*bl*/,\n                                               size_t rhs_stride, const void* rhs, const void* bias, const void* scale,\n                                               void* rhs_packed, size_t extra_bytes, const void* params) {\n    Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params);\n}\n\nstatic const size_t INT4_PER_BYTE = 2;\nstatic const size_t INT4_BITS     = 4;\nstatic const int Q4_0_ZERO_POINT  = 8;\nconst size_t INT4_PER_UINT16      = 4;\n\nstatic void dequantize_row_qsi4c32pscalef16(\n    const void *packed_data,\n    int32_t row_idx,\n    int64_t nc,\n    float *out,\n    size_t nr_pack,\n    size_t packed_row_stride,\n    size_t kr,\n    size_t bl,\n    size_t num_bytes_multiplier\n) {\n    size_t group_idx = row_idx / nr_pack;\n    size_t row_in_group = row_idx % nr_pack;\n    const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;\n    size_t num_blocks = nc / bl;\n    const uint8_t *block_ptr = packed_group;\n\n    for (size_t b = 0; b < num_blocks; ++b) {\n        uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier));\n        float scale = GGML_CPU_FP16_TO_FP32(scale_f16);\n\n        const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier;\n        size_t num_segments = bl / kr;\n        size_t num_bytes_per_segment = kr / INT4_PER_BYTE;\n\n        for (size_t s = 0; s < num_segments; ++s) {\n            const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment;\n            const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment;\n            for (size_t k = 0; k < num_bytes_per_segment; ++k) {\n                uint8_t byte = qbytes[k] ^ 0x88;\n                int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT;\n                int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT;\n                out[b * bl + s * num_bytes_per_segment + k] = x0 * scale;\n                out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale;\n            }\n        }\n        block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment;\n    }\n}\n\nstatic void dequantize_row_qsi4c32ps1s0scalef16(\n    const void *packed_data,\n    int32_t row_idx,\n    int64_t k,\n    float *out,\n    size_t nr,\n    size_t packed_row_stride,\n    size_t kr,\n    size_t bl,\n    size_t num_bytes_multiplier\n) {\n    const size_t num_blocks = k / bl;\n    const size_t bl4 = bl / INT4_PER_UINT16;\n\n    size_t group_idx = row_idx / nr;\n    size_t row_in_group = row_idx % nr;\n\n    const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;\n    const uint16_t *qdata = (const uint16_t *)packed_group;\n    const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier));\n\n    for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) {\n        uint16_t scale_f16 = scales[row_in_group + block_idx * nr];\n        float scale = GGML_CPU_FP16_TO_FP32(scale_f16);\n\n        for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) {\n            uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group];\n\n            for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) {\n                int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT;\n                out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale;\n            }\n        }\n    }\n    GGML_UNUSED(kr);\n}\n\nstatic void dequantize_row_qsi8cxp(\n    const void *packed_data,\n    int32_t row_idx,\n    int64_t k,\n    float *out,\n    size_t nr,\n    size_t packed_row_stride,\n    size_t kr,\n    size_t bl,\n    size_t num_bytes_multiplier\n) {\n    GGML_UNUSED(bl);\n    GGML_UNUSED(num_bytes_multiplier);\n\n    const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0;\n    const size_t group_idx = row_idx / nr;\n    const size_t row_in_group = row_idx % nr;\n\n    const uint8_t * group_ptr = static_cast<const uint8_t *>(packed_data) + group_idx * packed_row_stride;\n    const int8_t  * data_base = reinterpret_cast<const int8_t *>(group_ptr);\n\n    const size_t num_blocks = k_internal / kr;\n\n    for (size_t block = 0; block < num_blocks; ++block) {\n        const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr;\n        for (size_t i = 0; i < kr; ++i) {\n            const size_t k_idx = block * kr + i;\n            if (k_idx < (size_t) k) {\n                out[k_idx] = static_cast<float>(block_ptr[i]);\n            }\n        }\n    }\n\n    const uint8_t * sums_ptr = group_ptr + nr * k_internal;\n    GGML_UNUSED(sums_ptr);\n\n    const float * scale_ptr = reinterpret_cast<const float *>(sums_ptr + nr * sizeof(int32_t));\n    const float scale = scale_ptr[row_in_group];\n\n    if (scale == 0.0f) {\n        for (size_t i = 0; i < (size_t) k; ++i) {\n            out[i] = 0.0f;\n        }\n        return;\n    }\n\n    for (size_t i = 0; i < (size_t) k; ++i) {\n        out[i] *= scale;\n    }\n}\n\nstatic ggml_kleidiai_kernels gemm_gemv_kernels[] = {\n#if defined(__ARM_FEATURE_SME)\n    {\n        /* SME GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,\n        },\n\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_pack_f16pmrx2_f32_neon>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>,\n            /* .pack_func_ex          = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>,\n        },\n        /* SME GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32_neon,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32_neon>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32_neon>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32_neon>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon,\n            /* .to_float              = */ dequantize_row_qsi4c32ps1s0scalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_SME,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n    {\n        /* SME GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,\n            /* .run_kernel_ex         = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_bf16p2vlx2_f32_sme,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_bf16p2vlx2_f32_sme>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_bf16p2vlx2_f32_sme>,\n            /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,\n        },\n        /* SME GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,\n            /* .get_lhs_offset_ex     = */ nullptr,\n            /* .get_rhs_packed_offset_ex = */ nullptr,\n            /* .run_kernel_ex         = */ nullptr,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_pack_bf16p2vlx2_f32_sme,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_pack_bf16p2vlx2_f32_sme>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_pack_bf16p2vlx2_f32_sme>,\n            /* .pack_func_ex          = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ nullptr,\n            /* .to_float              = */ nullptr,\n            /* .packed_size_ex        = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,\n            /* .pack_func_ex          = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_SME,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_F16,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#if defined(__APPLE__)\n#if defined(__ARM_FEATURE_DOTPROD)\n    {\n        /* DOTPROD GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* DOTPROD GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n            /* .to_float              = */ dequantize_row_qsi4c32pscalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_DOTPROD,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    {\n        /* i8mm GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n        },\n        /* i8mm GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n            /* .to_float              = */ dequantize_row_qsi4c32pscalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_I8MM,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#else\n#if defined(__ARM_FEATURE_SVE)\n    {\n        /* SVE i8mm GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n        },\n        /* SVE dotprod GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n            /* .to_float              = */ dequantize_row_qsi4c32pscalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    {\n        /* i8mm GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p4x8sb_f32_neon>,\n        },\n        /* i8mm GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n            /* .to_float              = */ dequantize_row_qsi4c32pscalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_I8MM,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif // __ARM_FEATURE_MATMUL_INT8\n#if defined(__ARM_FEATURE_DOTPROD)\n    {\n        /* DOTPROD GEMM */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* DOTPROD GEMV */\n        /* .kern_info = */ {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qsi8d32p_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_quant_pack_qsi8d32p_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32p_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,\n            /* .to_float              = */ dequantize_row_qsi4c32pscalef16,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n            /* .pack_func_ex          = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_DOTPROD,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q4_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#endif\n    { /* Sentinel */ }\n};\n\nstatic ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {\n#if defined(__ARM_FEATURE_SME)\n    {\n        /* SME GEMM */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* SME GEMV */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n            /* .to_float              = */ dequantize_row_qsi8cxp,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .pack_func_ex          = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_SME,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q8_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#if defined(__ARM_FEATURE_MATMUL_INT8)\n    {\n        /* I8MM GEMM */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* I8MM GEMV (dotprod fallback) */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n            /* .to_float              = */ dequantize_row_qsi8cxp,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .pack_func_ex          = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_I8MM,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q8_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n#if defined(__ARM_FEATURE_DOTPROD)\n    {\n        /* DOTPROD GEMM */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,\n        },\n        /* .gemm_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* DOTPROD GEMV */\n        {\n            /* .get_m_step            = */ kai_get_m_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_n_step            = */ kai_get_n_step_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_mr                = */ kai_get_mr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_nr                = */ kai_get_nr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_kr                = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_sr                = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_dst_offset        = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_dst_size          = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,\n            /* .get_lhs_offset_ex     = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,\n            /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,\n            /* .run_kernel_ex         = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,\n        },\n        /* .gemv_lhs_info = */ {\n            /* .get_offset            = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,\n            /* .get_packed_offset_ex  = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,\n            /* .packed_size_ex        = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,\n            /* .pack_func_ex          = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,\n        },\n        /* .rhs_info = */ {\n            /* .packed_stride         = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,\n            /* .to_float              = */ dequantize_row_qsi8cxp,\n            /* .packed_size_ex        = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .packed_stride_ex      = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n            /* .pack_func_ex          = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,\n        },\n        /* .required_cpu       = */ CPU_FEATURE_DOTPROD,\n        /* .lhs_type           = */ GGML_TYPE_F32,\n        /* .rhs_type           = */ GGML_TYPE_Q8_0,\n        /* .op_type            = */ GGML_TYPE_F32,\n    },\n#endif\n    { /* Sentinel */ }\n};\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor) {\n    ggml_kleidiai_kernels * kernel = nullptr;\n\n    if (tensor->op == GGML_OP_MUL_MAT && tensor->src[0] != nullptr && tensor->src[1] != nullptr) {\n#if defined(__ARM_FEATURE_SME)          ||  \\\n    defined(__ARM_FEATURE_DOTPROD)      ||  \\\n    defined(__ARM_FEATURE_MATMUL_INT8)  ||  \\\n    defined(__ARM_FEATURE_SVE)\n        auto try_table = [&](auto & table) {\n            for (size_t i = 0; i < NELEMS(table) - 1; ++i) {\n                if ((cpu_features & table[i].required_cpu) == table[i].required_cpu &&\n                    table[i].lhs_type == tensor->src[1]->type &&\n                    table[i].rhs_type == tensor->src[0]->type &&\n                    table[i].op_type  == tensor->type) {\n                    kernel = &table[i];\n                    return true;\n                }\n            }\n            return false;\n        };\n\n        if (tensor->src[0]->type == GGML_TYPE_Q8_0) {\n            try_table(gemm_gemv_kernels_q8);\n        } else {\n            try_table(gemm_gemv_kernels);\n        }\n#else\n    GGML_UNUSED(gemm_gemv_kernels);\n    GGML_UNUSED(gemm_gemv_kernels_q8);\n    GGML_UNUSED(cpu_features);\n#endif\n    }\n\n    return kernel;\n}\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) {\n    ggml_kleidiai_kernels * kernels = nullptr;\n\n#if defined(__ARM_FEATURE_SME)          ||  \\\n    defined(__ARM_FEATURE_DOTPROD)      ||  \\\n    defined(__ARM_FEATURE_MATMUL_INT8)  ||  \\\n    defined(__ARM_FEATURE_SVE)\n    for (size_t i = 0; i < NELEMS(gemm_gemv_kernels) - 1; ++i) {\n        if ((features & gemm_gemv_kernels[i].required_cpu) == gemm_gemv_kernels[i].required_cpu) {\n            kernels = &gemm_gemv_kernels[i];\n            break;\n        }\n    }\n#else\n    GGML_UNUSED(features);\n#endif\n\n    return kernels;\n}\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {\n    ggml_kleidiai_kernels * kernels = nullptr;\n\n#if defined(__ARM_FEATURE_SME) || defined(__ARM_FEATURE_DOTPROD) || defined(__ARM_FEATURE_MATMUL_INT8)\n    for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {\n        if ((features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu) {\n            kernels = &gemm_gemv_kernels_q8[i];\n            break;\n        }\n    }\n#else\n    GGML_UNUSED(features);\n#endif\n\n    return kernels;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/kleidiai/kernels.h",
    "content": "// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>\n// SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml.h\"\n\nenum cpu_feature {\n    CPU_FEATURE_NONE    = 0,\n    CPU_FEATURE_DOTPROD = 1,\n    CPU_FEATURE_I8MM    = 2,\n    CPU_FEATURE_SVE     = 4,\n    CPU_FEATURE_SME     = 8\n};\n\ninline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {\n    lhs = static_cast<cpu_feature>(lhs | rhs);\n    return lhs;\n}\ninline cpu_feature operator|(cpu_feature lhs, cpu_feature rhs) {\n    return static_cast<cpu_feature>(static_cast<int>(lhs) | static_cast<int>(rhs));\n}\n\nstruct kernel_info {\n    size_t (*get_m_step)(void);\n    size_t (*get_n_step)(void);\n    size_t (*get_mr)(void);\n    size_t (*get_nr)(void);\n    size_t (*get_kr)(void);\n    size_t (*get_sr)(void);\n\n    size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride);\n    size_t (*get_dst_size)(size_t m, size_t n);\n\n    size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl);\n\n    size_t (*get_rhs_packed_offset_ex)(size_t n_idx, size_t k, size_t bl);\n\n    void (*run_kernel_ex)(\n        size_t m, size_t n, size_t k, size_t bl,\n        const void* lhs_packed, const void* rhs_packed,\n        void* dst, size_t dst_stride_row, size_t dst_stride_col,\n        float clamp_min, float clamp_max);\n};\n\nstruct lhs_packing_info {\n    size_t (*get_offset)(size_t m_idx, size_t lhs_stride);\n\n    size_t (*get_packed_offset_ex)(size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr);\n\n    size_t (*packed_size_ex)(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr);\n\n    void (*pack_func_ex)(size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,\n        size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed);\n};\n\nstruct rhs_packing_info {\n    size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl);\n\n    void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out,\n                     size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl,\n                     size_t num_bytes_multiplier);\n\n    size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl);\n\n    size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl);\n\n    void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,\n        size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params);\n};\n\nstruct ggml_kleidiai_kernels {\n    kernel_info      gemm;\n    lhs_packing_info gemm_lhs_info;\n\n    kernel_info      gemv;\n    lhs_packing_info gemv_lhs_info;\n\n    rhs_packing_info rhs_info;\n\n    cpu_feature required_cpu;\n    ggml_type lhs_type;\n    ggml_type rhs_type;\n    ggml_type op_type;\n};\n\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor);\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features);\nggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features);\n"
  },
  {
    "path": "ggml/src/ggml-cpu/kleidiai/kleidiai.cpp",
    "content": "// SPDX-FileCopyrightText: Copyright 2025-2026 Arm Limited and/or its affiliates <open-source-office@arm.com>\n// SPDX-License-Identifier: MIT\n//\n#include <arm_neon.h>\n#include <assert.h>\n#include <stdio.h>\n#include <atomic>\n#include <cfloat>\n#include <algorithm>\n#include <cmath>\n#include <stdexcept>\n#include <stdint.h>\n#include <string.h>\n#include <string>\n#include <vector>\n#include <array>\n#include <cstddef>\n#include <cstdint>\n#include <fstream>\n#include <set>\n#include <iostream>\n#include <climits>\n#if defined(__linux__)\n#include <asm/hwcap.h>\n#include <sys/auxv.h>\n#include <sys/types.h>\n#include <sys/stat.h>\n#include <unistd.h>\n#elif defined(__APPLE__)\n#include <string_view>\n#include <sys/sysctl.h>\n#include <sys/types.h>\n#elif defined(_WIN32)\n#include <windows.h>\n#include <excpt.h>\n#endif\n\n#include \"kleidiai.h\"\n\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-backend-impl.h\"\n#include \"ggml-threading.h\"\n#include \"traits.h\"\n\n#include \"kernels.h\"\n\n#include \"kai_common.h\"\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\nstatic constexpr int      GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2;\nstatic constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC       = 0x4b4c4149; // \"KLAI\"\nstatic constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION     = 1;\nstatic constexpr size_t   GGML_KLEIDIAI_PACK_ALIGN       = 64;\n\nstruct ggml_kleidiai_context {\n    cpu_feature features;\n    ggml_kleidiai_kernels * kernels_q4;\n    ggml_kleidiai_kernels * kernels_q8;\n    int sme_thread_cap; // <= 0 means “SME disabled/unknown”;\n    int thread_hint;    // <= 0 means “no hint”\n} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, 0, -1 };\n\nstatic const char* cpu_feature_to_string(cpu_feature f) {\n    if (f == CPU_FEATURE_NONE) {\n        return \"NONE\";\n    } else if ((f & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n        return \"SME\";\n    } else if ((f & CPU_FEATURE_SVE) == CPU_FEATURE_SVE) {\n        return \"SVE\";\n    }\n    else if ((f & CPU_FEATURE_I8MM) == CPU_FEATURE_I8MM) {\n        return \"I8MM\";\n    } else if ((f & CPU_FEATURE_DOTPROD) == CPU_FEATURE_DOTPROD) {\n        return \"DOTPROD\";\n    }\n    else {\n        return \"UNKNOWN\";\n    }\n}\n\nstatic size_t detect_num_smcus() {\n    if (!ggml_cpu_has_sme()) {\n        return 0;\n    }\n\n#if defined(__linux__) && defined(__aarch64__)\n    // Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs.\n    size_t num_private = 0;\n    std::set<uint32_t> shared_ids;\n\n    for (size_t cpu = 0;; ++cpu) {\n        const std::string path =\n            \"/sys/devices/system/cpu/cpu\" + std::to_string(cpu) +\n            \"/regs/identification/smidr_el1\";\n\n        std::ifstream file(path);\n        if (!file.is_open()) {\n            break;\n        }\n\n        uint64_t smidr = 0;\n        if (!(file >> std::hex >> smidr)) {\n            continue;\n        }\n\n        // Arm ARM: SMIDR_EL1\n        const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);\n        // Build an \"affinity-like\" identifier for shared SMCUs.\n        // Keep the original packing logic, but isolate it here.\n        const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u));\n\n        switch (sh) {\n            case 0b10: // private SMCU\n                ++num_private;\n                break;\n            case 0b11: // shared SMCU\n                shared_ids.emplace(id);\n                break;\n            case 0b00:\n                // Ambiguous / implementation-defined. Be conservative:\n                // treat id==0 as private, otherwise as shared.\n                if (id == 0) ++num_private;\n                else shared_ids.emplace(id);\n                break;\n            default:\n                break;\n        }\n    }\n\n    return num_private + shared_ids.size();\n\n#elif defined(__APPLE__) && defined(__aarch64__)\n    // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=<n>.\n    char chip_name[256] = {};\n    size_t size = sizeof(chip_name);\n\n    if (sysctlbyname(\"machdep.cpu.brand_string\", chip_name, &size, nullptr, 0) == 0) {\n        const std::string brand(chip_name);\n\n        struct ModelSMCU { const char *match; size_t smcus; };\n        static const ModelSMCU table[] = {\n            { \"M4 Ultra\", 2 },\n            { \"M4 Max\",   2 },\n            { \"M4 Pro\",   2 },\n            { \"M4\",       1 },\n        };\n\n        for (const auto &e : table) {\n            if (brand.find(e.match) != std::string::npos) {\n                return e.smcus;\n            }\n        }\n    }\n    return 1;\n\n#else\n    return 1;\n#endif\n}\n\nstatic int parse_uint_env(const char *s, const char *name, bool *ok) {\n    if (!s) { *ok = false; return 0; }\n    char *end = nullptr;\n    long v = strtol(s, &end, 10);\n    if (end == s || *end != '\\0') {\n        GGML_LOG_WARN(\"kleidiai: invalid %s='%s' (expected integer)\\n\", name, s);\n        *ok = false;\n        return 0;\n    }\n    if (v < 0 || v > INT_MAX) {\n        GGML_LOG_WARN(\"kleidiai: out-of-range %s='%s'\\n\", name, s);\n        *ok = false;\n        return 0;\n    }\n    *ok = true;\n    return (int)v;\n}\n\nstatic void init_kleidiai_context(void) {\n    ggml_critical_section_start();\n    static bool initialized = false;\n\n    if (!initialized) {\n        initialized = true;\n\n        const char *env_sme     = getenv(\"GGML_KLEIDIAI_SME\");\n        const char *env_threads = getenv(\"GGML_TOTAL_THREADS\");\n\n        const bool cpu_has_sme = ggml_cpu_has_sme();\n        size_t detected_smcus = 0;\n\n        ctx.features  = (ggml_cpu_has_dotprod()     ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |\n                        (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM    : CPU_FEATURE_NONE) |\n                        ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);\n\n        if (env_threads) {\n            bool ok = false;\n            int hint = parse_uint_env(env_threads, \"GGML_TOTAL_THREADS\", &ok);\n            if (ok && hint > 0) {\n                ctx.thread_hint = hint;\n            }\n        }\n\n        // SME policy:\n        // - If CPU doesn't support SME: SME always off.\n        // - Else:\n        //   - env unset => auto-detect cores; enable if detected > 0.\n        //   - env=0     => force off.\n        //   - env>0     => force N cores (skip detection).\n        int sme_cores = 0;\n        bool sme_env_ok = false;\n        bool sme_env_set = (env_sme != nullptr);\n\n        if (!cpu_has_sme) {\n            if (sme_env_set) {\n                bool ok = false;\n                int req = parse_uint_env(env_sme, \"GGML_KLEIDIAI_SME\", &ok);\n                if (ok && req > 0) {\n                    GGML_LOG_WARN(\"kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME\\n\", req);\n                }\n            }\n            sme_cores = 0;\n        } else {\n            if (sme_env_set) {\n                bool ok = false;\n                int v = parse_uint_env(env_sme, \"GGML_KLEIDIAI_SME\", &ok);\n                sme_env_ok = ok;\n\n                if (!ok) {\n                    GGML_LOG_WARN(\"kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\\n\");\n                    detected_smcus = detect_num_smcus();\n                    sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;\n                } else if (v == 0) {\n                    sme_cores = 0;\n                } else {\n                    sme_cores = v;\n                }\n            } else {\n                detected_smcus = detect_num_smcus();\n                sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;\n            }\n\n            if (!sme_env_set && sme_cores == 0) {\n                GGML_LOG_WARN(\"kleidiai: SME supported but runtime SME-core detection returned 0; falling back to NEON\\n\");\n            }\n\n            if (sme_cores > 0) {\n                ctx.features |= CPU_FEATURE_SME;\n            }\n        }\n\n        // Kernel selection\n        ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features);\n        ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features);\n\n        if (!ctx.kernels_q4) {\n            GGML_LOG_INFO(\"kleidiai: no compatible q4 kernels found for CPU features mask %d\\n\", (int)ctx.features);\n        } else {\n            GGML_LOG_INFO(\"kleidiai: primary q4 kernel feature %s\\n\", cpu_feature_to_string(ctx.kernels_q4->required_cpu));\n        }\n\n        if (!ctx.kernels_q8) {\n            GGML_LOG_INFO(\"kleidiai: no compatible q8 kernels found for CPU features mask %d\\n\", (int)ctx.features);\n        } else {\n            GGML_LOG_INFO(\"kleidiai: primary q8 kernel feature %s\\n\", cpu_feature_to_string(ctx.kernels_q8->required_cpu));\n        }\n\n        ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;\n\n        if (ctx.features & CPU_FEATURE_SME) {\n            if (sme_env_set && sme_env_ok && sme_cores > 0) {\n                GGML_LOG_INFO(\"kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d override)\\n\", sme_cores);\n            } else {\n                GGML_LOG_INFO(\"kleidiai: SME enabled (runtime-detected SME cores=%d)\\n\", sme_cores);\n            }\n        } else {\n            GGML_LOG_INFO(\"kleidiai: SME disabled\\n\");\n        }\n    }\n\n    ggml_critical_section_end();\n}\n\nstatic inline int kleidiai_sme_thread_cap() {\n    return ctx.sme_thread_cap;\n}\n\nstatic inline size_t align_up(size_t value, size_t alignment) {\n    if (alignment == 0) {\n        return value;\n    }\n    const size_t remainder = value % alignment;\n    return remainder == 0 ? value : value + (alignment - remainder);\n}\n\nstatic inline bool kleidiai_pack_fallback_allowed() {\n    if (ctx.sme_thread_cap <= 0) {\n        return false;\n    }\n    if (ctx.thread_hint <= 0) {\n        return true;\n    }\n    return ctx.thread_hint > ctx.sme_thread_cap;\n}\n\nstruct kleidiai_weight_header {\n    uint32_t magic;\n    uint16_t version;\n    uint16_t slot_count;\n    uint64_t offsets[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n    uint64_t sizes[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n};\n\nstatic inline kleidiai_weight_header * kleidiai_weight_header_from_ptr(void * data) {\n    return reinterpret_cast<kleidiai_weight_header *>(data);\n}\n\nstatic inline const kleidiai_weight_header * kleidiai_weight_header_from_ptr(const void * data) {\n    return reinterpret_cast<const kleidiai_weight_header *>(data);\n}\n\nstatic inline bool kleidiai_is_weight_header_valid(const kleidiai_weight_header * header) {\n    if (!header) {\n        return false;\n    }\n    if (header->magic != GGML_KLEIDIAI_PACK_MAGIC || header->version != GGML_KLEIDIAI_PACK_VERSION) {\n        return false;\n    }\n    if (header->slot_count == 0 || header->slot_count > GGML_KLEIDIAI_MAX_KERNEL_SLOTS) {\n        return false;\n    }\n    return true;\n}\n\nstatic inline uint8_t * kleidiai_weight_slot_ptr(kleidiai_weight_header * header, int slot) {\n    if (!kleidiai_is_weight_header_valid(header)) {\n        return nullptr;\n    }\n    if (slot < 0 || slot >= header->slot_count) {\n        return nullptr;\n    }\n    return reinterpret_cast<uint8_t *>(header) + header->offsets[slot];\n}\n\nstatic inline const uint8_t * kleidiai_weight_slot_ptr(const kleidiai_weight_header * header, int slot) {\n    if (!kleidiai_is_weight_header_valid(header)) {\n        return nullptr;\n    }\n    if (slot < 0 || slot >= header->slot_count) {\n        return nullptr;\n    }\n    return reinterpret_cast<const uint8_t *>(header) + header->offsets[slot];\n}\n\nstatic inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() {\n    return ctx.kernels_q4;\n}\n\nstatic inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() {\n    return ctx.kernels_q8;\n}\n\ntemplate <typename SelectFallback>\nstatic int kleidiai_collect_kernel_chain_common(\n        ggml_kleidiai_kernels * primary,\n        cpu_feature features,\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out,\n        SelectFallback select_fallback) {\n    int count = 0;\n    if (!primary) {\n        return 0;\n    }\n    out[count++] = primary;\n\n    if ((primary->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n        const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME);\n        if (fallback_mask != CPU_FEATURE_NONE) {\n            ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);\n            if (fallback && fallback != primary &&\n                fallback->lhs_type == primary->lhs_type &&\n                fallback->rhs_type == primary->rhs_type &&\n                fallback->op_type  == primary->op_type) {\n                out[count++] = fallback;\n            }\n        }\n    }\n\n    return count;\n}\n\nstatic int kleidiai_collect_kernel_chain(const struct ggml_tensor * op,\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {\n    ggml_kleidiai_kernels * primary = ggml_kleidiai_select_kernels(ctx.features, op);\n    return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n        [&](cpu_feature mask) { return ggml_kleidiai_select_kernels(mask, op); });\n}\n\nstatic int kleidiai_collect_q4_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {\n    ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4();\n    return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n        [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); });\n}\n\nstatic int kleidiai_collect_q8_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {\n    ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8();\n    return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,\n        [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q8_0(mask); });\n}\n\nstatic inline int64_t ggml_ne(const ggml_tensor * tensor, int dim) {\n    GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);\n    return tensor->ne[dim];\n}\n\nnamespace ggml::cpu::kleidiai {\n\nstatic size_t round_down(size_t x, size_t y) {\n    return y == 0 ? x : x - (x % y);\n}\n\nstatic void transpose_f32kxn_f16nxk(size_t n, size_t k, float * dst, const uint16_t * src, size_t rhs_stride) {\n    size_t src_stride = rhs_stride / sizeof(uint16_t);\n    size_t dst_stride = n;\n\n    for (size_t k_idx = 0; k_idx < k; ++k_idx) {\n        for (size_t n_idx = 0; n_idx < n; ++n_idx) {\n            uint16_t v = *(src + k_idx + n_idx * src_stride);\n            *(dst + n_idx + k_idx * dst_stride) = kai_cast_f32_f16(v);\n        }\n    }\n}\n\nclass tensor_traits : public ggml::cpu::tensor_traits {\n    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n        if (op->op != GGML_OP_MUL_MAT) {\n            return false;\n        }\n\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n        const int slot_count = kleidiai_collect_kernel_chain(op, kernel_chain);\n        if (slot_count == 0) {\n            return false;\n        }\n\n        const bool is_gemv = op->src[1]->ne[1] == 1;\n        const size_t k = op->src[0]->ne[0];\n        const size_t n = op->src[0]->ne[1];\n        const size_t m = op->src[1]->ne[1];\n\n        if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {\n            const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0;\n\n            size_t cursor = 0;\n            bool any_slot = false;\n\n            for (int slot = 0; slot < slot_count; ++slot) {\n                ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n                lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n                kernel_info * kernel        = is_gemv ? &kernels->gemv : &kernels->gemm;\n\n                if (!lhs_info || !lhs_info->packed_size_ex || !kernel) {\n                    return false;\n                }\n\n                const size_t mr = kernel->get_mr();\n                const size_t kr = kernel->get_kr();\n                const size_t sr = kernel->get_sr();\n\n                const size_t packed = lhs_info->packed_size_ex(m, k, qk, mr, kr, sr);\n\n                cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n                cursor += packed;\n                any_slot = true;\n            }\n\n            if (!any_slot) {\n                return false;\n            }\n\n            size = cursor;\n            return true;\n        }\n\n        if (op->src[0]->type == GGML_TYPE_F16) {\n            const int64_t lhs_batch_size0 = op->src[1]->ne[2];\n            const int64_t rhs_batch_size0 = op->src[0]->ne[2];\n            GGML_ASSERT(rhs_batch_size0 > 0);\n            const int64_t r = lhs_batch_size0 / rhs_batch_size0;\n\n            size_t cursor = 0;\n            bool any_slot = false;\n\n            for (int slot = 0; slot < slot_count; ++slot) {\n                ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n                lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n                kernel_info * kernel        = is_gemv ? &kernels->gemv : &kernels->gemm;\n                if (!lhs_info || !lhs_info->packed_size_ex || !kernels->rhs_info.packed_size_ex || !kernel) {\n                    return false;\n                }\n\n                const size_t mr = kernel->get_mr();\n                const size_t kr = kernel->get_kr();\n                const size_t sr = kernel->get_sr();\n\n                cursor  = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n                cursor += lhs_info->packed_size_ex(m * r, k, 0, mr, kr, sr);\n                any_slot = true;\n            }\n\n            for (int slot = 0; slot < slot_count; ++slot) {\n                ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n                kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n                if (!kernel || !kernels->rhs_info.packed_size_ex) {\n                    return false;\n                }\n                cursor  = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n                cursor += kernels->rhs_info.packed_size_ex(n, k, kernel->get_nr(), kernel->get_kr(), 0);\n            }\n\n            cursor  = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n            cursor += k * n * sizeof(float);\n            cursor  = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n            cursor += n * sizeof(float);\n\n            if (!any_slot) {\n                return false;\n            }\n\n            size = cursor;\n            return true;\n        }\n\n        return false;\n    }\n\n    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override {\n        if (dst->op == GGML_OP_MUL_MAT) {\n            if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {\n                return compute_forward_qx(params, dst);\n            } else if (dst->src[0]->type == GGML_TYPE_F16) {\n                return compute_forward_fp16(params, dst);\n            }\n        } else if (dst->op == GGML_OP_GET_ROWS) {\n            if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {\n                return compute_forward_get_rows(params, dst);\n            }\n        }\n        return false;\n    }\n\n    bool compute_forward_fp16(ggml_compute_params * params, struct ggml_tensor * dst) {\n        const ggml_tensor * src0 = dst->src[0];\n        const ggml_tensor * src1 = dst->src[1];\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        ggml_kleidiai_kernels *kernels = ggml_kleidiai_select_kernels(ctx.features, dst);\n        if (!kernels) {\n            return false;\n        }\n\n        const bool is_gemv = src1->ne[1] == 1;\n        kernel_info * kernel = is_gemv ? &kernels->gemv : &kernels->gemm;\n        lhs_packing_info * lhs_info = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n        GGML_ASSERT(kernel);\n        if (!kernels->rhs_info.pack_func_ex ||\n            !kernel->get_lhs_offset_ex || !kernel->get_rhs_packed_offset_ex || !kernel->run_kernel_ex) {\n            return false;\n        }\n\n        const int nth = params->nth;\n        const int ith = params->ith;\n\n        const int64_t lhs_batch_size0 = ne12;\n        const int64_t rhs_batch_size0 = ne02;\n        const int64_t batch_size      = lhs_batch_size0;\n\n        GGML_ASSERT(rhs_batch_size0 > 0);\n        GGML_ASSERT(lhs_batch_size0 % rhs_batch_size0 == 0);\n        const int64_t r = lhs_batch_size0 / rhs_batch_size0;\n\n        const int64_t m_group = ne11;\n        const int64_t m       = m_group;\n        const int64_t n       = ne01;\n        const int64_t k       = ne00;\n\n        const size_t lhs_stride = src1->nb[1];\n        const size_t rhs_stride = src0->nb[1];\n        const size_t dst_stride = dst->nb[1];\n\n        const int64_t mr = (int64_t) kernel->get_mr();\n        const int64_t nr = (int64_t) kernel->get_nr();\n        const int64_t kr = (int64_t) kernel->get_kr();\n        const int64_t sr = (int64_t) kernel->get_sr();\n\n        const size_t lhs_packed_size = lhs_info->packed_size_ex(m, k, 0, mr, kr, sr);\n        const size_t rhs_packed_size = kernels->rhs_info.packed_size_ex(n, k, nr, kr, 0);\n        const size_t kxn_size        = k * n * sizeof(float);\n        const size_t bias_size       = n * sizeof(float);\n\n        const size_t wsize_required = lhs_packed_size + rhs_packed_size + kxn_size + bias_size;\n        GGML_ASSERT(wsize_required <= params->wsize);\n\n        uint8_t * lhs_packed = static_cast<uint8_t *>(params->wdata);\n        uint8_t * rhs_packed = lhs_packed + lhs_packed_size;\n        uint8_t * rhs_kxn    = rhs_packed + rhs_packed_size;\n        uint8_t * bias       = rhs_kxn + kxn_size;\n\n        for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) {\n            const int64_t rhs_batch_idx = batch_idx / r;\n            const uint8_t * rhs_batch_base = static_cast<const uint8_t *>(src0->data) + rhs_batch_idx * src0->nb[2];\n            uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];\n\n            // LHS packing (threaded over m, honoring mr alignment and KV groups)\n            {\n                const int64_t m_roundup_mr = kai_roundup(m, mr);\n                const int64_t num_threads  = KAI_MIN(m_roundup_mr / mr, nth);\n\n                if (ith < num_threads) {\n                    const int64_t num_m_per_thread0   = round_down((size_t)(m_roundup_mr / num_threads), (size_t)mr);\n                    const int64_t num_m_per_threadN_1 = m - (num_threads - 1) * num_m_per_thread0;\n\n                    const int64_t m_start = ith * num_m_per_thread0;\n                    const int64_t m_count = (ith == num_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0;\n\n                    // Base packed offset (aligned) and per-row stride in bytes\n                    const size_t base_packed_off  = lhs_info->get_packed_offset_ex(m_start, k, 0, mr, kr, sr);\n                    const size_t next_block_off   = lhs_info->get_packed_offset_ex(m_start + mr, k, 0, mr, kr, sr);\n                    const size_t row_stride_bytes = (next_block_off - base_packed_off) / (size_t)mr;\n\n                    int64_t remaining = m_count;\n                    int64_t cur       = m_start;\n\n                    while (remaining > 0) {\n                        const int64_t row_in_group = cur;\n                        const int64_t avail        = m_group - row_in_group;\n                        const int64_t take         = std::min(avail, remaining);\n\n                        const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];\n                        const void * src_ptr = lhs_batch_base + (size_t)row_in_group * lhs_stride;\n                        const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes;\n                        void * dst_ptr       = lhs_packed + dst_off;\n\n                        lhs_info->pack_func_ex(take, k, 0, mr, kr, sr, 0, src_ptr, lhs_stride, dst_ptr);\n\n                        cur       += take;\n                        remaining -= take;\n                    }\n                }\n            }\n\n            // RHS packing (single thread), then synchronize\n            if (ith == 0) {\n                memset(bias, 0, (size_t)n * sizeof(float));\n                transpose_f32kxn_f16nxk((size_t)n, (size_t)k,\n                                        reinterpret_cast<float *>(rhs_kxn),\n                                        reinterpret_cast<const uint16_t *>(rhs_batch_base),\n                                        rhs_stride);\n\n                kernels->rhs_info.pack_func_ex(1, n, k, nr, kr, sr, 0, n * sizeof(float),\n                             rhs_kxn, bias, nullptr, rhs_packed, 0, nullptr);\n            }\n\n            ggml_barrier(params->threadpool);\n\n            // Matmul (threaded over n)\n            {\n                const int64_t n_step  = (int64_t) kernel->get_n_step();\n                int64_t num_threads_n = KAI_MIN(n / n_step, nth);\n                if (num_threads_n <= 0) {\n                    num_threads_n = 1;\n                }\n\n                if (ith < num_threads_n) {\n                    const int64_t num_n_per_thread0   = round_down((size_t)(n / num_threads_n), (size_t)n_step);\n                    const int64_t num_n_per_threadN_1 = n - (num_threads_n - 1) * num_n_per_thread0;\n\n                    const int64_t n_start      = ith * num_n_per_thread0;\n                    const int64_t n_to_process = (ith == num_threads_n - 1) ? num_n_per_threadN_1 : num_n_per_thread0;\n\n                    // LHS packed base at row 0 (consistent with packing above)\n                    const size_t lhs_packed_offset0 = lhs_info->get_packed_offset_ex(0, k, 0, mr, kr, sr);\n                    const size_t rhs_packed_offset  = kernel->get_rhs_packed_offset_ex(n_start, k, 0);\n                    const size_t dst_offset         = kernel->get_dst_offset((size_t)0, (size_t)n_start, dst_stride);\n\n                    const void * lhs_ptr = lhs_packed + lhs_packed_offset0;\n                    const void * rhs_ptr = rhs_packed + rhs_packed_offset;\n                    float * dst_ptr      = reinterpret_cast<float *>(dst_batch_base + dst_offset);\n\n                    kernel->run_kernel_ex(m, n_to_process, k, 0, lhs_ptr, rhs_ptr, dst_ptr, dst_stride, sizeof(float), -FLT_MAX, FLT_MAX);\n                }\n            }\n\n            if (batch_idx != batch_size - 1) {\n                ggml_barrier(params->threadpool);\n            }\n        }\n\n        return true;\n    }\n\n    bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) {\n        GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);\n\n        const ggml_tensor * src0 = dst->src[0];\n        const ggml_tensor * src1 = dst->src[1];\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);\n        const bool has_header = kleidiai_is_weight_header_valid(header);\n        const bool is_gemv = src1->ne[1] == 1;\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n        const int slot_total = kleidiai_collect_kernel_chain(dst, kernel_chain);\n\n        auto weight_for_slot = [&](int slot_index, size_t & size_out) -> const uint8_t * {\n            if (slot_index < 0 || slot_index >= slot_total) {\n                return nullptr;\n            }\n            if (has_header) {\n                if (slot_index < header->slot_count) {\n                    size_out = static_cast<size_t>(header->sizes[slot_index]);\n                    return kleidiai_weight_slot_ptr(header, slot_index);\n                }\n                return nullptr;\n            }\n            if (slot_index == 0) {\n                size_out = ggml_nbytes(src0);\n                return static_cast<const uint8_t *>(src0->data);\n            }\n            return nullptr;\n        };\n\n        struct runtime_slot {\n            int slot_index;\n            ggml_kleidiai_kernels * kernels;\n            kernel_info * kernel;\n            lhs_packing_info * lhs_info;\n            size_t mr;\n            size_t nr;\n            size_t kr;\n            size_t sr;\n            size_t n_step;\n            size_t lhs_packed_size;\n            size_t lhs_offset;\n            size_t n_offset;\n            size_t n_cols;\n            int assigned_threads;\n            int thread_begin;\n            int thread_end;\n            const uint8_t * rhs_base;\n        };\n\n        std::array<runtime_slot, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> runtime{};\n        int runtime_count = 0;\n\n        for (int slot = 0; slot < slot_total && runtime_count < GGML_KLEIDIAI_MAX_KERNEL_SLOTS; ++slot) {\n            ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n            kernel_info * kinfo      = is_gemv ? &kernels->gemv : &kernels->gemm;\n            lhs_packing_info * linfo = is_gemv ? &kernels->gemv_lhs_info : &kernels->gemm_lhs_info;\n            if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex || !linfo->get_offset ||\n                !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset) {\n                continue;\n            }\n\n            size_t rhs_size = 0;\n            const uint8_t * rhs_ptr = weight_for_slot(slot, rhs_size);\n            if (!rhs_ptr || rhs_size == 0) {\n                continue;\n            }\n\n            runtime[runtime_count] = {\n                slot,\n                kernels,\n                kinfo,\n                linfo,\n                kinfo->get_mr(),\n                kinfo->get_nr(),\n                kinfo->get_kr(),\n                kinfo->get_sr(),\n                kinfo->get_n_step(),\n                0,\n                0,\n                0,\n                0,\n                0,\n                0,\n                0,\n                rhs_ptr\n            };\n            ++runtime_count;\n        }\n\n        if (runtime_count == 0) {\n            ggml_kleidiai_kernels * fallback = ggml_kleidiai_select_kernels(ctx.features, dst);\n            if (!fallback) {\n                return false;\n            }\n            kernel_info * kinfo      = is_gemv ? &fallback->gemv : &fallback->gemm;\n            lhs_packing_info * linfo = is_gemv ? &fallback->gemv_lhs_info : &fallback->gemm_lhs_info;\n            rhs_packing_info * rinfo = &fallback->rhs_info;\n            if (!kinfo || !linfo || !linfo->packed_size_ex || !linfo->pack_func_ex ||\n                !kinfo->get_rhs_packed_offset_ex || !kinfo->run_kernel_ex || !kinfo->get_dst_offset ||\n                !rinfo || !rinfo->pack_func_ex || !rinfo->packed_size_ex) {\n                return false;\n            }\n            kernel_chain[0] = fallback;\n            runtime[0] = {\n                0,\n                fallback,\n                kinfo,\n                linfo,\n                kinfo->get_mr(),\n                kinfo->get_nr(),\n                kinfo->get_kr(),\n                kinfo->get_sr(),\n                kinfo->get_n_step(),\n                0,\n                0,\n                0,\n                0,\n                0,\n                0,\n                0,\n                nullptr\n            };\n            size_t rhs_size_fallback = 0;\n            const uint8_t * rhs_base = weight_for_slot(0, rhs_size_fallback);\n            if (!rhs_base) {\n                rhs_base = static_cast<const uint8_t *>(src0->data);\n            }\n            runtime[0].rhs_base = rhs_base;\n            runtime_count = 1;\n        }\n\n        const int nth_total = params->nth > 0 ? params->nth : 1;\n        const int ith_total = params->ith;\n\n        int sme_slot = -1;\n        for (int i = 0; i < runtime_count; ++i) {\n            if ((runtime[i].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n                sme_slot = i;\n                break;\n            }\n        }\n\n        const int sme_cap_limit = ctx.sme_thread_cap;\n        const bool use_hybrid = sme_cap_limit > 0 &&\n                                 runtime_count > 1 &&\n                                 nth_total > sme_cap_limit;\n        // Heuristic: disable hybrid for very small workloads where per-slot overhead dominates.\n        // If rows are small or average columns per thread are small, keep single-slot.\n        size_t min_cols_per_thread = 0;\n        if (runtime_count > 0 && nth_total > 0) {\n            min_cols_per_thread = (size_t) std::max<int64_t>(1, (int64_t)ne01 / (int64_t)nth_total);\n        }\n        const bool too_small_for_hybrid = (min_cols_per_thread < 2) || (ne11 < 128);\n\n        const bool hybrid_enabled = use_hybrid && !too_small_for_hybrid;\n\n        if (!hybrid_enabled) {\n            int chosen_slot = 0;\n            if (too_small_for_hybrid && sme_slot != -1) {\n                chosen_slot = sme_slot;\n            } else if (runtime_count > 1 && ctx.sme_thread_cap > 0 && nth_total > ctx.sme_thread_cap) {\n                chosen_slot = 1;\n            }\n            if (chosen_slot != 0 && chosen_slot < runtime_count) {\n                runtime[0] = runtime[chosen_slot];\n            }\n            runtime_count = runtime_count > 0 ? 1 : 0;\n\n            // Recompute SME slot based on the collapsed runtime[0]\n            sme_slot = -1;\n            if (runtime_count > 0 &&\n                (runtime[0].kernels->required_cpu & CPU_FEATURE_SME) == CPU_FEATURE_SME) {\n                sme_slot = 0;\n            }\n        }\n\n        int sme_cap = kleidiai_sme_thread_cap();\n        if (sme_cap < 0) {\n            sme_cap = nth_total;\n        }\n        sme_cap = std::min(sme_cap, nth_total);\n\n        int threads_remaining = nth_total;\n        if (sme_slot != -1) {\n            int sme_threads = std::min(std::max(sme_cap, 0), threads_remaining);\n            runtime[sme_slot].assigned_threads = sme_threads;\n            threads_remaining -= sme_threads;\n        }\n\n        int fallback_indices[GGML_KLEIDIAI_MAX_KERNEL_SLOTS];\n        int fallback_count = 0;\n        for (int i = 0; i < runtime_count; ++i) {\n            if (i == sme_slot) {\n                continue;\n            }\n            fallback_indices[fallback_count++] = i;\n        }\n\n        for (int fi = 0; fi < fallback_count; ++fi) {\n            if (threads_remaining <= 0) {\n                break;\n            }\n            const int slot_index = fallback_indices[fi];\n            const int slots_left = fallback_count - fi;\n            int share = (threads_remaining + slots_left - 1) / slots_left;\n            share     = std::min(share, threads_remaining);\n            runtime[slot_index].assigned_threads = share;\n            threads_remaining -= share;\n        }\n\n        if (threads_remaining > 0) {\n            const int fallback_slot = (sme_slot != -1) ? sme_slot : 0;\n            runtime[fallback_slot].assigned_threads += threads_remaining;\n            threads_remaining = 0;\n        }\n\n        int thread_cursor = 0;\n        for (int i = 0; i < runtime_count; ++i) {\n            runtime[i].thread_begin = thread_cursor;\n            thread_cursor += runtime[i].assigned_threads;\n            runtime[i].thread_end = thread_cursor;\n        }\n\n        if (thread_cursor < nth_total && runtime_count > 0) {\n            runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor;\n            runtime[runtime_count - 1].thread_end = nth_total;\n        }\n\n        int local_slot = -1;\n        int local_ith  = 0;\n        for (int i = 0; i < runtime_count; ++i) {\n            if (ith_total >= runtime[i].thread_begin && ith_total < runtime[i].thread_end) {\n                local_slot = i;\n                local_ith  = ith_total - runtime[i].thread_begin;\n                break;\n            }\n        }\n        if (local_slot == -1) {\n            return false;\n        }\n\n        const size_t k = ne00;\n        const size_t m = ne11;\n        const size_t n = ne01;\n\n        size_t cursor = 0;\n        for (int i = 0; i < runtime_count; ++i) {\n            const ggml_type slot_rhs_type = runtime[i].kernels->rhs_type;\n            const size_t slot_pack_size_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n                                              slot_rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0;\n            runtime[i].lhs_packed_size = runtime[i].lhs_info->packed_size_ex(m, k, slot_pack_size_arg, runtime[i].mr, runtime[i].kr, runtime[i].sr);\n            cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n            runtime[i].lhs_offset = cursor;\n            cursor += runtime[i].lhs_packed_size;\n        }\n\n        GGML_ASSERT(cursor <= params->wsize);\n        uint8_t * scratch = static_cast<uint8_t *>(params->wdata);\n\n        size_t assigned_cols = 0;\n        uint64_t weighted_total = 0;\n        if (runtime_count > 1 && sme_slot != -1) {\n            for (int i = 0; i < runtime_count; ++i) {\n                const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;\n                weighted_total += (uint64_t)runtime[i].assigned_threads * weight;\n            }\n        }\n        for (int i = 0; i < runtime_count; ++i) {\n            runtime[i].n_offset = assigned_cols;\n            if (runtime[i].assigned_threads == 0) {\n                runtime[i].n_cols = 0;\n                continue;\n            }\n            const size_t remaining_cols = n - assigned_cols;\n            if (remaining_cols == 0) {\n                runtime[i].n_cols = 0;\n                continue;\n            }\n            const size_t step = runtime[i].n_step ? runtime[i].n_step : 1;\n            size_t target      = 0;\n            if (weighted_total > 0) {\n                const uint64_t weight = (i == sme_slot) ? (sme_cap << 1) : 1;\n                target = (size_t)(((uint64_t)n * runtime[i].assigned_threads * weight) / weighted_total);\n            } else {\n                target = (size_t)(((uint64_t)n * runtime[i].assigned_threads) / nth_total);\n            }\n            target             = std::min(target, remaining_cols);\n            size_t aligned     = round_down(target, step);\n            if (aligned == 0 && remaining_cols >= step) {\n                aligned = step;\n            }\n            runtime[i].n_cols = aligned;\n            assigned_cols += aligned;\n        }\n\n        if (assigned_cols < n) {\n            for (int i = runtime_count - 1; i >= 0; --i) {\n                if (runtime[i].assigned_threads > 0) {\n                    runtime[i].n_cols += n - assigned_cols;\n                    break;\n                }\n            }\n        }\n        const size_t dst_stride = dst->nb[1];\n\n        for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {\n            const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];\n            uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];\n\n            if (runtime[local_slot].assigned_threads > 0) {\n                runtime_slot & slot = runtime[local_slot];\n                const ggml_type slot_rhs_type = slot.kernels->rhs_type;\n                const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n                                                 slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n                const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);\n                int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;\n                max_threads = std::max<int64_t>(1, max_threads);\n                const int64_t use_threads = std::min<int64_t>(slot.assigned_threads, max_threads);\n\n                if (local_ith < use_threads) {\n                    const int64_t num_m_per_thread0   = round_down((size_t)(m_roundup_mr / use_threads), slot.mr);\n                    const int64_t num_m_per_threadN_1 = (int64_t)m - (use_threads - 1) * num_m_per_thread0;\n\n                    const int64_t m_start = (int64_t)local_ith * num_m_per_thread0;\n                    const int64_t m_count = (local_ith == use_threads - 1) ? num_m_per_threadN_1 : num_m_per_thread0;\n\n                    const size_t base_packed_off  = slot.lhs_info->get_packed_offset_ex(m_start, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n                    const size_t next_block_off   = slot.lhs_info->get_packed_offset_ex(m_start + slot.mr, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n                    const size_t row_stride_bytes = slot.mr ? (next_block_off - base_packed_off) / slot.mr : 0;\n\n                    int64_t remaining = m_count;\n                    int64_t cur       = m_start;\n\n                    uint8_t * lhs_packed = scratch + slot.lhs_offset;\n                    while (remaining > 0) {\n                        const int64_t row_in_group = cur;\n                        const int64_t avail        = (int64_t)m - row_in_group;\n                        const int64_t take         = std::min(avail, remaining);\n\n                        const size_t src_off = slot.lhs_info->get_offset(row_in_group, src1->nb[1]);\n                        const void * src_ptr = lhs_batch_base + src_off;\n                        const size_t dst_off = base_packed_off + (size_t)(cur - m_start) * row_stride_bytes;\n                        void * dst_ptr       = lhs_packed + dst_off;\n\n                        slot.lhs_info->pack_func_ex(take, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr, 0, src_ptr, src1->nb[1], dst_ptr);\n\n                        cur       += take;\n                        remaining -= take;\n                    }\n                }\n            }\n\n            ggml_barrier(params->threadpool);\n\n            runtime_slot & slot = runtime[local_slot];\n            if (slot.n_cols > 0 && slot.assigned_threads > 0) {\n                int64_t active_threads = slot.assigned_threads;\n                const int64_t max_threads = slot.n_step ? (slot.n_cols / slot.n_step) : slot.assigned_threads;\n                if (max_threads > 0) {\n                    active_threads = std::min<int64_t>(active_threads, std::max<int64_t>(1, max_threads));\n                }\n                active_threads = std::max<int64_t>(1, active_threads);\n\n                if (local_ith < active_threads) {\n                    const size_t step = slot.n_step ? slot.n_step : 1;\n                    const size_t chunk0 = round_down((size_t)(slot.n_cols / active_threads), step);\n                    const size_t chunkN = slot.n_cols - (active_threads - 1) * chunk0;\n                    const size_t local_start = (size_t)local_ith * chunk0;\n                    const size_t cols = (local_ith == active_threads - 1) ? chunkN : chunk0;\n\n                    if (cols > 0) {\n                        const ggml_type slot_rhs_type = slot.kernels->rhs_type;\n                        const size_t slot_lhs_exec_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n                                                         slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n                        const size_t slot_rhs_block_arg = slot_rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n                                                          slot_rhs_type == GGML_TYPE_Q8_0 ? 0 : 0;\n                        const size_t global_start = slot.n_offset + local_start;\n                        const size_t lhs_packed_offset = slot.lhs_info->get_packed_offset_ex(0, k, slot_lhs_exec_arg, slot.mr, slot.kr, slot.sr);\n                        const size_t rhs_packed_offset = slot.kernel->get_rhs_packed_offset_ex(global_start, k, slot_rhs_block_arg);\n                        const size_t dst_offset        = slot.kernel->get_dst_offset(0, global_start, dst_stride);\n\n                        const uint8_t * lhs_ptr = scratch + slot.lhs_offset + lhs_packed_offset;\n                        const uint8_t * rhs_ptr = slot.rhs_base + rhs_packed_offset;\n                        float * dst_ptr         = reinterpret_cast<float *>(dst_batch_base + dst_offset);\n\n                        slot.kernel->run_kernel_ex(m, cols, k, slot_rhs_block_arg,\n                                                   lhs_ptr,\n                                                   rhs_ptr,\n                                                   dst_ptr,\n                                                   dst_stride,\n                                                   sizeof(float),\n                                                   -FLT_MAX,\n                                                   FLT_MAX);\n                    }\n                }\n            }\n\n            if (batch_idx != ne12 - 1) {\n                ggml_barrier(params->threadpool);\n            }\n        }\n\n        return true;\n    }\n\n    bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) {\n        GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);\n        const ggml_tensor * src0 = dst->src[0];\n        const ggml_tensor * src1 = dst->src[1];\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);\n        const bool has_header = kleidiai_is_weight_header_valid(header);\n\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n        const bool want_q8 = src0->type == GGML_TYPE_Q8_0;\n        const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n                                        : kleidiai_collect_q4_chain(kernel_chain);\n\n        ggml_kleidiai_kernels * kernels = nullptr;\n        const uint8_t * packed_base = static_cast<const uint8_t *>(src0->data);\n\n        if (has_header && chain_count > 0) {\n            int select_slot = 0;\n            if (select_slot >= header->slot_count) {\n                select_slot = header->slot_count - 1;\n            }\n            if (select_slot >= 0 && select_slot < chain_count) {\n                kernels = kernel_chain[select_slot];\n                const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot);\n                if (slot_ptr) {\n                    packed_base = slot_ptr;\n                }\n            }\n        }\n\n        if (!kernels && chain_count > 0) {\n            kernels = kernel_chain[0];\n            if (has_header) {\n                const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0);\n                if (slot_ptr) {\n                    packed_base = slot_ptr;\n                }\n            }\n        }\n\n        if (!kernels) {\n            return false;\n        }\n\n        rhs_packing_info * rhs_info = &kernels->rhs_info;\n        kernel_info * kernel        = &kernels->gemm;\n        if (!rhs_info->to_float || !kernel->get_nr) {\n            return false;\n        }\n\n        const int64_t nc     = ne00;\n        const int64_t nr     = ggml_nelements(src1);\n\n        const ggml_type rhs_type = kernels->rhs_type;\n        size_t block_len = 0;\n        size_t num_bytes_multiplier = 0;\n        if (rhs_type == GGML_TYPE_Q4_0) {\n            block_len = QK4_0;\n            num_bytes_multiplier = sizeof(uint16_t);\n        } else if (rhs_type == GGML_TYPE_Q8_0) {\n            block_len = QK8_0;\n            num_bytes_multiplier = sizeof(float);\n        } else {\n            return false;\n        }\n\n        const size_t block_rows = kernel->get_nr();\n        const size_t kr         = kernel->get_kr();\n\n        const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len);\n\n        const int ith = params->ith;\n        const int nth = params->nth;\n\n        const int dr = (nr + nth - 1) / nth;\n        const int ir0 = dr * ith;\n        const int ir1 = MIN(ir0 + dr, nr);\n\n        for (int64_t i = ir0; i < ir1; ++i) {\n            GGML_ASSERT(src1->type == GGML_TYPE_I32);\n            int64_t row_idx = ((const int32_t *)src1->data)[i];\n            GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]);\n\n            float *out = (float *)((char *)dst->data + i * nb1);\n            rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier);\n        }\n\n        return true;\n    }\n\npublic:\n    int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) {\n        GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0);\n        const size_t n = tensor->ne[1];\n        const size_t k = tensor->ne[0];\n\n        kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(tensor->data);\n        if (!header) {\n            return -1;\n        }\n\n        header->magic      = GGML_KLEIDIAI_PACK_MAGIC;\n        header->version    = GGML_KLEIDIAI_PACK_VERSION;\n        header->slot_count = 0;\n\n        uint8_t * base_ptr = static_cast<uint8_t *>(tensor->data);\n        size_t cursor = sizeof(kleidiai_weight_header);\n        cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n        const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;\n        const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n                                       : kleidiai_collect_q4_chain(kernel_chain);\n        const bool allow_fallback = kleidiai_pack_fallback_allowed();\n\n        std::vector<int8_t> qdata;\n        std::vector<float>  scales;\n\n        if (want_q8 && slot_total > 0) {\n            qdata.resize(n * k, 0);\n            scales.resize(n, 0.0f);\n\n            const size_t row_stride = tensor->nb[1];\n            const size_t k_blocks   = (k + QK8_0 - 1) / QK8_0;\n\n            for (size_t row = 0; row < n; ++row) {\n                const auto * row_blocks = reinterpret_cast<const block_q8_0 *>(\n                    static_cast<const uint8_t *>(data) + row * row_stride);\n\n                float max_abs = 0.0f;\n                for (size_t block = 0; block < k_blocks; ++block) {\n                    const block_q8_0 & blk = row_blocks[block];\n                    const float d = GGML_FP16_TO_FP32(blk.d);\n                    for (size_t l = 0; l < QK8_0; ++l) {\n                        const size_t linear_idx = block * QK8_0 + l;\n                        if (linear_idx >= k) {\n                            break;\n                        }\n                        const float value = d * static_cast<float>(blk.qs[l]);\n                        max_abs = std::max(max_abs, std::fabs(value));\n                    }\n                }\n\n                float scale = max_abs > 0.0f ? max_abs / 127.0f : 0.0f;\n                scales[row] = scale;\n                const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f;\n\n                for (size_t block = 0; block < k_blocks; ++block) {\n                    const block_q8_0 & blk = row_blocks[block];\n                    const float d = GGML_FP16_TO_FP32(blk.d);\n                    for (size_t l = 0; l < QK8_0; ++l) {\n                        const size_t linear_idx = block * QK8_0 + l;\n                        if (linear_idx >= k) {\n                            break;\n                        }\n                        const float value = d * static_cast<float>(blk.qs[l]);\n                        int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;\n                        q = std::clamp(q, -127, 127);\n                        qdata[row * k + linear_idx] = static_cast<int8_t>(q);\n                    }\n                }\n            }\n        }\n\n        for (int slot = 0; slot < slot_total && slot < GGML_KLEIDIAI_MAX_KERNEL_SLOTS; ++slot) {\n            if (!allow_fallback && slot > 0) {\n                break;\n            }\n            ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n            kernel_info * kernel = &kernels->gemm;\n            rhs_packing_info * rhs_info = &kernels->rhs_info;\n            if (!rhs_info || !rhs_info->pack_func_ex || !rhs_info->packed_size_ex || !kernel) {\n                continue;\n            }\n\n            const size_t nr = kernel->get_nr();\n            const size_t kr = kernel->get_kr();\n            const size_t sr = kernel->get_sr();\n            const ggml_type rhs_type = kernels->rhs_type;\n            const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :\n                                     rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : 0;\n            if (block_len == 0) {\n                continue;\n            }\n\n            const size_t packed_size = rhs_info->packed_size_ex(n, k, nr, kr, block_len);\n            const size_t aligned_cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n            uint8_t * dst_ptr = base_ptr + aligned_cursor;\n\n            if (rhs_type == GGML_TYPE_Q4_0) {\n                struct kai_rhs_pack_qs4cxs1s0_param params;\n                params.lhs_zero_point = 1;\n                params.rhs_zero_point = 8;\n                rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0,\n                                       static_cast<const uint8_t *>(data), nullptr, nullptr,\n                                       dst_ptr, 0, &params);\n            } else if (rhs_type == GGML_TYPE_Q8_0) {\n                struct kai_rhs_pack_qsi8cx_params params;\n                params.lhs_zero_point = 1;\n                params.scale_multiplier = 1.0f;\n                rhs_info->pack_func_ex(1, n, k, nr, kr, sr, 0, 0,\n                                       qdata.data(), nullptr, scales.data(),\n                                       dst_ptr, 0, &params);\n            } else {\n                continue;\n            }\n\n            header->offsets[header->slot_count] = aligned_cursor;\n            header->sizes[header->slot_count]   = packed_size;\n            ++header->slot_count;\n\n            cursor = aligned_cursor + packed_size;\n        }\n\n        if (header->slot_count == 0) {\n            header->magic   = 0;\n            header->version = 0;\n            memcpy(tensor->data, data, data_size);\n        }\n\n        return 0;\n    }\n};\n\nstatic ggml::cpu::tensor_traits * get_tensor_traits(ggml_backend_buffer_t, struct ggml_tensor *) {\n    static tensor_traits traits;\n    return &traits;\n}\n}  // namespace ggml::cpu::kleidiai\n\nstatic enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n    tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor);\n\n    return GGML_STATUS_SUCCESS;\n    GGML_UNUSED(buffer);\n}\n\nstatic void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n                                                       const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(offset == 0);\n    GGML_ASSERT(size == ggml_nbytes(tensor));\n\n    auto tensor_traits = (ggml::cpu::kleidiai::tensor_traits *) tensor->extra;\n    auto OK            = tensor_traits->repack(tensor, data, size);\n\n    GGML_ASSERT(OK == 0);\n    GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_kleidiai_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    GGML_UNUSED(buft);\n    return \"CPU_KLEIDIAI\";\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n    if (buffer == nullptr) {\n        return nullptr;\n    }\n\n    buffer->buft              = buft;\n    buffer->iface.init_tensor = ggml_backend_cpu_kleidiai_buffer_init_tensor;\n    buffer->iface.set_tensor  = ggml_backend_cpu_kleidiai_buffer_set_tensor;\n    buffer->iface.get_tensor  = nullptr;\n    buffer->iface.cpy_tensor  = nullptr;\n    return buffer;\n}\n\nstatic size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    GGML_UNUSED(buft);\n    return TENSOR_ALIGNMENT;\n}\n\nstatic size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {\n    GGML_UNUSED(buft);\n\n    if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0) {\n        return ggml_nbytes(tensor);\n    }\n\n    const size_t n = tensor->ne[1];\n    const size_t k = tensor->ne[0];\n\n    size_t cursor = sizeof(kleidiai_weight_header);\n    cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n\n    std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n    const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;\n    const int slot_total = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)\n                                   : kleidiai_collect_q4_chain(kernel_chain);\n    const bool allow_fallback = kleidiai_pack_fallback_allowed();\n\n    size_t slot_count = 0;\n    for (int slot = 0; slot < slot_total; ++slot) {\n        if (!allow_fallback && slot > 0) {\n            break;\n        }\n        ggml_kleidiai_kernels * kernels = kernel_chain[slot];\n        if (!kernels) {\n            continue;\n        }\n        kernel_info * kernel = &kernels->gemm;\n        rhs_packing_info * rhs_info = &kernels->rhs_info;\n        if (!kernel || !rhs_info || !rhs_info->packed_size_ex) {\n            continue;\n        }\n\n        const ggml_type rhs_type = kernels->rhs_type;\n        const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :\n                                 rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : 0;\n        if (block_len == 0) {\n            continue;\n        }\n\n        cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);\n        cursor += rhs_info->packed_size_ex(n, k, kernel->get_nr(), kernel->get_kr(), block_len);\n        ++slot_count;\n    }\n\n    if (slot_count == 0) {\n        return ggml_nbytes(tensor);\n    }\n\n    return std::max(cursor, ggml_nbytes(tensor));\n}\n\nnamespace ggml::cpu::kleidiai {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n    bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n        std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n        const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);\n        if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&\n            (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) &&\n            op->src[0]->buffer &&\n            (ggml_n_dims(op->src[0]) == 2) &&\n            op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type() &&\n            slot_total > 0) {\n            if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) {\n                return false;\n            }\n            if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) {\n                return false;\n            }\n            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n                return false;\n            }\n            if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) &&\n                ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n        if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {\n            if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type()) {\n                return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n            } else {\n                std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;\n                const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);\n                const bool has_kernel = slot_total > 0;\n                if (has_kernel && op->src[1]->ne[1] > 1) {\n                    if ((op->src[0]->nb[1] * op->src[0]->ne[1] != op->src[0]->nb[2]) ||\n                        (op->src[1]->nb[1] * op->src[1]->ne[1] != op->src[1]->nb[2])) {\n                        return nullptr;\n                    }\n                    return ggml::cpu::kleidiai::get_tensor_traits(NULL, NULL);\n                }\n            }\n        }\n        return nullptr;\n    }\n};\n}  // namespace ggml::cpu::kleidiai\n\nggml_backend_buffer_type_t ggml_backend_cpu_kleidiai_buffer_type(void) {\n    static ggml::cpu::kleidiai::extra_buffer_type ctx;\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_kleidiai = {\n        /* .iface    = */ {\n                           /* .get_name         = */ ggml_backend_cpu_kleidiai_buffer_type_get_name,\n                           /* .alloc_buffer     = */ ggml_backend_cpu_kleidiai_buffer_type_alloc_buffer,\n                           /* .get_alignment    = */ ggml_backend_cpu_kleidiai_buffer_type_get_alignment,\n                           /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX\n                           /* .get_alloc_size   = */ ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size,\n                           /* .is_host          = */ nullptr,\n                           },\n        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ &ctx,\n    };\n\n    init_kleidiai_context();\n\n    return &ggml_backend_cpu_buffer_type_kleidiai;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/kleidiai/kleidiai.h",
    "content": "// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>\n// SPDX-License-Identifier: MIT\n//\n\n#pragma once\n\n#include \"ggml-alloc.h\"\n\n#ifdef  __cplusplus\nextern \"C\" {\n#endif\n\nggml_backend_buffer_type_t ggml_backend_cpu_kleidiai_buffer_type(void);\n\n#ifdef  __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/llamafile/sgemm.cpp",
    "content": "// Copyright 2024 Mozilla Foundation\n//\n// Permission is hereby granted, free of charge, to any person obtaining\n// a copy of this software and associated documentation files (the\n// \"Software\"), to deal in the Software without restriction, including\n// without limitation the rights to use, copy, modify, merge, publish,\n// distribute, sublicense, and/or sell copies of the Software, and to\n// permit persons to whom the Software is furnished to do so, subject to\n// the following conditions:\n//\n// The above copyright notice and this permission notice shall be\n// included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\n// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\n// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\n// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n// SOFTWARE.\n\n//\n//                   _   _          ___ _      _   ___\n//                  | |_(_)_ _ _  _| _ ) |    /_\\ / __|\n//                  |  _| | ' \\ || | _ \\ |__ / _ \\\\__ \\.\n//                   \\__|_|_||_\\_, |___/____/_/ \\_\\___/\n//                             |__/\n//\n//                    BASIC LINEAR ALGEBRA SUBPROGRAMS\n//\n//\n// This file implements multithreaded CPU matrix multiplication for the\n// common contiguous use case C = Aᵀ * B. These kernels are designed to\n// have excellent performance[1] for matrices that fit in the CPU cache\n// without imposing any overhead such as cache filling or malloc calls.\n//\n// This implementation does not guarantee any upper bound with rounding\n// errors, which grow along with k. Our goal's to maximally exploit the\n// hardware for performance, and then use whatever resources remain for\n// improving numerical accuracy.\n//\n// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].\n//     Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Wpedantic\"\n#pragma GCC diagnostic ignored \"-Wignored-attributes\"\n#endif\n\n#include \"sgemm.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml-quants.h\"\n#include \"simd-mappings.h\"\n\n#include <array>\n#include <type_traits>\n\n#ifdef _MSC_VER\n#define NOINLINE __declspec(noinline)\n#else\n#define NOINLINE __attribute__((__noinline__))\n#endif\n\n#if defined(__ARM_NEON) || defined(__AVX512F__) || defined(__VXE__) || defined(__VXE2__)\n#define VECTOR_REGISTERS 32\n#else\n#define VECTOR_REGISTERS 16\n#endif\n\n#if defined(__riscv_v_intrinsic)\n#define LMUL 4\n#endif\n\n#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)\n\nnamespace {\n\ninline float unhalf(ggml_fp16_t d) {\n    return GGML_CPU_FP16_TO_FP32(d);\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED ARITHMETIC OPERATIONS\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }\ninline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }\ninline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }\n#endif  // __SSE__\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }\ninline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }\ninline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }\n#endif // __AVX__\n\n#if defined(__AVX512F__)\ninline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }\ninline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }\ninline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }\n#endif // __AVX512F__\n\n#if defined(__ARM_NEON)\ninline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }\ninline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }\ninline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }\n#endif // __ARM_NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\ninline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }\ninline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }\ninline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }\n#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\n\n#if defined(__VXE__) || defined(__VXE2__)\ninline float32x4_t add(float32x4_t x, float32x4_t y) { return vec_add(x, y); }\ninline float32x4_t sub(float32x4_t x, float32x4_t y) { return vec_sub(x, y); }\ninline float32x4_t mul(float32x4_t x, float32x4_t y) { return vec_mul(x, y); }\n#endif\n\n#if defined(__MMA__)\ntypedef vector unsigned char vec_t;\ntypedef __vector_quad acc_t;\n#endif\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED FUSED MULTIPLY ADD\n\n/**\n * Computes a * b + c.\n */\ntemplate <typename T, typename U>\ninline U madd(T a, T b, U c) {\n    return add(mul(a, b), c);\n}\n\n#if defined(__FMA__)\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <>\ninline __m256 madd(__m256 a, __m256 b, __m256 c) {\n    return _mm256_fmadd_ps(a, b, c);\n}\n#endif\n#if defined(__AVX512F__)\ntemplate <>\ninline __m512 madd(__m512 a, __m512 b, __m512 c) {\n    return _mm512_fmadd_ps(a, b, c);\n}\n#endif\n#if defined(__AVX512BF16__)\ntemplate <>\ninline __m512 madd(__m512bh a, __m512bh b, __m512 c) {\n    return _mm512_dpbf16_ps(c, a, b);\n}\ntemplate <>\ninline __m256 madd(__m256bh a, __m256bh b, __m256 c) {\n    return _mm256_dpbf16_ps(c, a, b);\n}\n#endif\n#endif\n\n#if defined(__ARM_FEATURE_FMA)\ntemplate <>\ninline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {\n    return vfmaq_f32(c, b, a);\n}\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\ntemplate <>\ninline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {\n    return vfmaq_f16(c, b, a);\n}\n#endif\n#endif\n\n#if defined(__VXE__) || defined(__VXE2__)\ntemplate <>\ninline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {\n    return vec_madd(a, b, c);\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate <>\ninline vfloat32m1_t madd(vfloat16mf2_t a, vfloat16mf2_t b, vfloat32m1_t c) {\n    return __riscv_vfwmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vfloat16m1_t a, vfloat16m1_t b, vfloat32m2_t c) {\n    return __riscv_vfwmacc_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vfloat16m2_t a, vfloat16m2_t b, vfloat32m4_t c) {\n    return __riscv_vfwmacc_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\ninline vfloat32m8_t madd(vfloat16m4_t a, vfloat16m4_t b, vfloat32m8_t c) {\n    return __riscv_vfwmacc_vv_f32m8(c, a, b, __riscv_vsetvlmax_e32m8());\n}\ninline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) {\n    return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vfloat32m2_t a, vfloat32m2_t b, vfloat32m2_t c) {\n    return __riscv_vfmacc_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vfloat32m4_t a, vfloat32m4_t b, vfloat32m4_t c) {\n    return __riscv_vfmacc_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\ninline vfloat32m8_t madd(vfloat32m8_t a, vfloat32m8_t b, vfloat32m8_t c) {\n    return __riscv_vfmacc_vv_f32m8(c, a, b, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_zvfbfwma)\ninline vfloat32m1_t madd(vbfloat16mf2_t a, vbfloat16mf2_t b, vfloat32m1_t c) {\n    return __riscv_vfwmaccbf16_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1());\n}\ninline vfloat32m2_t madd(vbfloat16m1_t a, vbfloat16m1_t b, vfloat32m2_t c) {\n    return __riscv_vfwmaccbf16_vv_f32m2(c, a, b, __riscv_vsetvlmax_e32m2());\n}\ninline vfloat32m4_t madd(vbfloat16m2_t a, vbfloat16m2_t b, vfloat32m4_t c) {\n    return __riscv_vfwmaccbf16_vv_f32m4(c, a, b, __riscv_vsetvlmax_e32m4());\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED HORIZONTAL SUM\n\n#if defined(__ARM_NEON)\ninline float hsum(float32x4_t x) {\n    return vaddvq_f32(x);\n}\n#endif // __ARM_NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\ninline float hsum(float16x8_t x) {\n    return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)),\n                                vcvt_f32_f16(vget_high_f16(x))));\n}\n#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\n\n#if defined(__VXE__) || defined(__VXE2__)\ninline float hsum(float32x4_t x) {\n    float32x4_t tmp = x + vec_reve(x);\n    return tmp[0] + tmp[1];\n}\n#endif\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline float hsum(__m128 x) {\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n    x = _mm_add_ps(x, _mm_movehl_ps(x, x));\n    x = _mm_add_ss(x, _mm_movehdup_ps(x));\n#else\n    __m128 t;\n    t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));\n    x = _mm_add_ps(x, t);\n    t = _mm_movehl_ps(t, x);\n    x = _mm_add_ss(x, t);\n#endif\n    return _mm_cvtss_f32(x);\n}\n#endif\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ninline float hsum(__m256 x) {\n    return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1),\n                           _mm256_castps256_ps128(x)));\n}\n#endif // __AVX__\n\n#if defined(__AVX512F__)\ninline float hsum(__m512 x) {\n    return _mm512_reduce_add_ps(x);\n}\n#endif // __AVX512F__\n\n#if defined(__riscv_zvfh)\ninline float hsum(vfloat32m1_t x) {\n    return __riscv_vfmv_f_s_f32m1_f32(\n        __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1()));\n}\ninline float hsum(vfloat32m2_t x) {\n    return __riscv_vfmv_f_s_f32m1_f32(\n        __riscv_vfredusum_vs_f32m2_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m2()));\n}\ninline float hsum(vfloat32m4_t x) {\n    return __riscv_vfmv_f_s_f32m1_f32(\n        __riscv_vfredusum_vs_f32m4_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m4()));\n}\ninline float hsum(vfloat32m8_t x) {\n    return __riscv_vfmv_f_s_f32m1_f32(\n        __riscv_vfredusum_vs_f32m8_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m8()));\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// VECTORIZED MEMORY LOADING\n\ntemplate <typename T, typename U> T load(const U *);\n\n#if defined(__ARM_NEON)\ntemplate <> inline float32x4_t load(const float *p) {\n    return vld1q_f32(p);\n}\n#if !defined(_MSC_VER)\n// FIXME: this should check for __ARM_FEATURE_FP16_VECTOR_ARITHMETIC\ntemplate <> inline float16x8_t load(const ggml_fp16_t *p) {\n    return vld1q_f16((const float16_t *)p);\n}\ntemplate <> inline float32x4_t load(const ggml_fp16_t *p) {\n    return vcvt_f32_f16(vld1_f16((const float16_t *)p));\n}\n#endif // _MSC_VER\n#endif // __ARM_NEON\n\n#if defined(__VXE__) || defined(__VXE2__)\ntemplate <> inline float32x4_t load(const ggml_fp16_t * p) {\n    float tmp[4];\n\n    for (int i = 0; i < 4; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(p[i]);\n    }\n\n    return vec_xl(0, (const float *)(tmp));\n}\ntemplate <> inline float32x4_t load(const float * p) {\n    return vec_xl(0, p);\n}\n#endif\n\n#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m128 load(const float *p) {\n    return _mm_loadu_ps(p);\n}\n#endif  // __SSE__\n\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m256 load(const float *p) {\n    return _mm256_loadu_ps(p);\n}\n#endif // __AVX__\n\n#if defined(__AVX2__) || defined(__AVX512F__)\ntemplate <> inline __m256 load(const ggml_bf16_t *p) {\n    return _mm256_castsi256_ps(\n        _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));\n}\n#endif // __AVX2__\n\n#if defined(__F16C__)\ntemplate <> inline __m256 load(const ggml_fp16_t *p) {\n    return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));\n}\n#endif // __F16C__\n\n#if defined(__AVX512F__)\ntemplate <> inline __m512 load(const float *p) {\n    return _mm512_loadu_ps(p);\n}\ntemplate <> inline __m512 load(const ggml_fp16_t *p) {\n    return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));\n}\ntemplate <> inline __m512 load(const ggml_bf16_t *p) {\n    return _mm512_castsi512_ps(\n        _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));\n}\n#endif // __AVX512F__\n\n#if defined(__AVX512BF16__)\ntemplate <> inline __m512bh load(const ggml_bf16_t *p) {\n    return (__m512bh)_mm512_loadu_ps((const float *)p);\n}\ntemplate <> inline __m256bh load(const ggml_bf16_t *p) {\n    return (__m256bh)_mm256_loadu_ps((const float *)p);\n}\ntemplate <> inline __m512bh load(const float *p) {\n    return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));\n}\ntemplate <> inline __m256bh load(const float *p) {\n    return _mm512_cvtneps_pbh(_mm512_loadu_ps(p));\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate <> inline vfloat16mf2_t load(const ggml_fp16_t *p) {\n    return __riscv_vle16_v_f16mf2(reinterpret_cast<const _Float16 *>(p), __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vfloat16m1_t load(const ggml_fp16_t *p) {\n    return __riscv_vle16_v_f16m1(reinterpret_cast<const _Float16 *>(p), __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vfloat16m2_t load(const ggml_fp16_t *p) {\n    return __riscv_vle16_v_f16m2(reinterpret_cast<const _Float16 *>(p), __riscv_vsetvlmax_e16m2());\n}\ntemplate <> inline vfloat16m4_t load(const ggml_fp16_t *p) {\n    return __riscv_vle16_v_f16m4(reinterpret_cast<const _Float16 *>(p), __riscv_vsetvlmax_e16m4());\n}\ntemplate <> inline vfloat32m1_t load(const float *p) {\n    return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1());\n}\ntemplate <> inline vfloat32m2_t load(const float *p) {\n    return __riscv_vle32_v_f32m2(p, __riscv_vsetvlmax_e32m2());\n}\ntemplate <> inline vfloat32m4_t load(const float *p) {\n    return __riscv_vle32_v_f32m4(p, __riscv_vsetvlmax_e32m4());\n}\ntemplate <> inline vfloat32m8_t load(const float *p) {\n    return __riscv_vle32_v_f32m8(p, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_zvfbfwma)\ntemplate <> inline vbfloat16mf2_t load(const ggml_bf16_t *p) {\n    return __riscv_vle16_v_bf16mf2(reinterpret_cast<const __bf16*>(p), __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vbfloat16m1_t load(const ggml_bf16_t *p) {\n    return __riscv_vle16_v_bf16m1(reinterpret_cast<const __bf16*>(p), __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vbfloat16m2_t load(const ggml_bf16_t *p) {\n    return __riscv_vle16_v_bf16m2(reinterpret_cast<const __bf16*>(p), __riscv_vsetvlmax_e16m2());\n}\n#endif\n\n#if defined(__riscv_zvfh)\ntemplate <typename T> T set_zero();\n\ntemplate <> inline vfloat16mf2_t set_zero() {\n    return __riscv_vfmv_v_f_f16mf2(0, __riscv_vsetvlmax_e16mf2());\n}\ntemplate <> inline vfloat16m1_t set_zero() {\n    return __riscv_vfmv_v_f_f16m1(0, __riscv_vsetvlmax_e16m1());\n}\ntemplate <> inline vfloat16m2_t set_zero() {\n    return __riscv_vfmv_v_f_f16m2(0, __riscv_vsetvlmax_e16m2());\n}\ntemplate <> inline vfloat16m4_t set_zero() {\n    return __riscv_vfmv_v_f_f16m4(0, __riscv_vsetvlmax_e16m4());\n}\ntemplate <> inline vfloat32m1_t set_zero() {\n    return __riscv_vfmv_v_f_f32m1(0.0f, __riscv_vsetvlmax_e32m1());\n}\ntemplate <> inline vfloat32m2_t set_zero() {\n    return __riscv_vfmv_v_f_f32m2(0, __riscv_vsetvlmax_e32m2());\n}\ntemplate <> inline vfloat32m4_t set_zero() {\n    return __riscv_vfmv_v_f_f32m4(0, __riscv_vsetvlmax_e32m4());\n}\ntemplate <> inline vfloat32m8_t set_zero() {\n    return __riscv_vfmv_v_f_f32m8(0, __riscv_vsetvlmax_e32m8());\n}\n#endif\n\n#if defined(__riscv_v_intrinsic)\ntemplate <typename T> size_t vlmax() {\n    if constexpr (std::is_same_v<T, vfloat16mf2_t>) { return  __riscv_vsetvlmax_e16mf2(); }\n    else if constexpr (std::is_same_v<T, vfloat16m1_t>) { return  __riscv_vsetvlmax_e16m1(); }\n    else if constexpr (std::is_same_v<T, vfloat16m2_t>) { return  __riscv_vsetvlmax_e16m2(); }\n    else if constexpr (std::is_same_v<T, vfloat16m4_t>) { return  __riscv_vsetvlmax_e16m4(); }\n    else if constexpr (std::is_same_v<T, vfloat32m1_t>) { return  __riscv_vsetvlmax_e32m1(); }\n    else if constexpr (std::is_same_v<T, vfloat32m2_t>) { return  __riscv_vsetvlmax_e32m2(); }\n    else if constexpr (std::is_same_v<T, vfloat32m4_t>) { return  __riscv_vsetvlmax_e32m4(); }\n    else if constexpr (std::is_same_v<T, vfloat32m8_t>) { return  __riscv_vsetvlmax_e32m8(); }\n    return 0;\n}\n#endif\n\n////////////////////////////////////////////////////////////////////////////////////////////////////\n// FLOATING POINT MATRIX MULTIPLICATION\n\ntemplate <int M>\nstatic inline int64_t BLOCK_SIZE(size_t m) {\n    const int64_t NB_BLOC_M = (m + M - 1) / M;\n    return (m % NB_BLOC_M == 0) ? m / NB_BLOC_M : (m / NB_BLOC_M) + 1;\n}\n\nstatic constexpr inline int64_t BLOC_POS(int64_t ib, int64_t ibN, int64_t bloc_size) {\n    return ib < ibN ? ib * bloc_size : ibN * bloc_size + (ib - ibN) * (bloc_size - 1);\n}\n\ntemplate <int KN, typename D, typename V, typename TA, typename TB, typename TC>\nclass tinyBLAS {\n  public:\n    tinyBLAS(const ggml_compute_params * params, int64_t k,\n             const TA *A, int64_t lda,\n             const TB *B, int64_t ldb,\n             TC *C, int64_t ldc)\n        : params(params), A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc) {\n    }\n\n    bool matmul(int64_t m, int64_t n) {\n        if (k % KN != 0)\n            return false;\n        // compute RM for only need tile with size RM&RM-1\n#if VECTOR_REGISTERS == 32\n        if (m % 16 == 0 && (m/16 >= params->nth)) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 4>(m, n, SIZE_N, 12);\n            return true;\n        }\n        if (m % 8 == 0 ) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 2>(m, n, SIZE_N, 12);\n            return true;\n        }\n        if (m % 4 == 0) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 1>(m, n, SIZE_N, 12);\n            return true;\n        }\n#else  // VECTOR_REGISTERS == 16\n        if (m % 16 == 0 && (m/16 >= params->nth)) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 4>(m, n, SIZE_N, 24);\n            return true;\n        }\n        if (m % 8 == 0 ) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 2>(m, n, SIZE_N, 24);\n            return true;\n        }\n        if (m % 4 == 0) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 1>(m, n, SIZE_N, 24);\n            return true;\n        }\n#endif\n        return false;\n    }\n\n  private:\n    template <int RM, int RN, int BM>\n    inline void mnpack(int64_t m, int64_t n, int64_t SIZE_N, int64_t BN) {\n        if (SIZE_N == RN) {\n            return gemm<RM, RN, BM>(m, n, BN);\n        }\n        if constexpr (RN > 1) {\n            return mnpack<RM, RN-1, BM>(m, n, SIZE_N, BN);\n        } else {\n            GGML_LOG_ERROR(\"mnpack<%d, %d> block size not supported\\n\", RM, (int)SIZE_N);\n            GGML_ASSERT(false); // we have miss something.\n        }\n    }\n\n    template <int RM, int RN>\n    inline void gemm_bloc(int64_t ii, int64_t jj) {\n        D Cv[RN][RM] = {};\n        for (int64_t l = 0; l < k; l += KN) {\n            // help compiler for op order.\n            if constexpr (RM <= RN) {\n                V Av[RM];\n                for (int64_t i = 0; i < RM; ++i) {\n                    Av[i] = load<V>(A + lda * (ii + i) + l);\n                }\n                for (int64_t j = 0; j < RN; ++j) {\n                    V Bv = load<V>(B + ldb * (jj + j) + l);\n                    for (int64_t i = 0; i < RM; ++i) {\n                        Cv[j][i] = madd(Av[i], Bv, Cv[j][i]);\n                    }\n                }\n            } else {\n                V Bv[RN];\n                for (int64_t j = 0; j < RN; ++j) {\n                    Bv[j] = load<V>(B + ldb * (jj + j) + l);\n                }\n                for (int64_t i = 0; i < RM; ++i) {\n                    V Av = load<V>(A + lda * (ii + i) + l);\n                    for (int64_t j = 0; j < RN; ++j) {\n                        Cv[j][i] = madd(Av, Bv[j], Cv[j][i]);\n                    }\n                }\n            }\n        }\n        for (int64_t j = 0; j < RN; ++j)\n            for (int64_t i = 0; i < RM; ++i)\n                C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n    }\n\n    template <int RM, int RN, int BM>\n    NOINLINE void gemm(int64_t m, int64_t n, int64_t BN) {\n        GGML_ASSERT(m % (RM * BM) == 0);\n        const int64_t ytiles = m / (RM * BM);\n        const int64_t xtiles = (n + RN -1) / RN;\n        const int64_t jj_RN = (xtiles - (xtiles * RN - n));\n\n        // \"round\" bloc_size to \"nearest\" BN\n        const int64_t NB_BN = xtiles < BN ? 1 : (xtiles + BN / 2) / BN;\n        const int64_t SIZE_BN = xtiles % NB_BN == 0 ? xtiles / NB_BN : xtiles / NB_BN + 1;\n        const int64_t jj_BN = (NB_BN - (NB_BN * SIZE_BN - xtiles));\n        const int64_t nb_job = ytiles * NB_BN;\n\n        if (params->ith == 0) {\n            GGML_ASSERT( jj_BN * SIZE_BN + (NB_BN - jj_BN) * (SIZE_BN - 1) == xtiles);\n            // Every thread starts at ith, so the first unprocessed chunk is nth.  This save a bit of coordination right at the start.\n            ggml_threadpool_chunk_set(params->threadpool, params->nth);\n        }\n\n        ggml_barrier(params->threadpool);\n\n        int64_t job = params->ith;\n        while (job < nb_job) {\n            const int64_t ii = (job % ytiles) * RM * BM;\n            const int64_t jb =  job / ytiles;\n            const int64_t jr0 = BLOC_POS(jb  , jj_BN, SIZE_BN);\n            const int64_t jrN = BLOC_POS(jb+1, jj_BN, SIZE_BN);\n\n            const int64_t jj0 = BLOC_POS(jr0, jj_RN, RN);\n            const int64_t jj2 = BLOC_POS(jrN, jj_RN, RN);\n            const int64_t jj1 = jj2 < jj_RN * RN ? jj2 : jj_RN * RN;\n\n            for (int64_t bi = 0; bi < BM * RM; bi += RM) {\n                int64_t jj = jj0;\n                for (; jj < jj1; jj += RN) {\n                    gemm_bloc<RM, RN>(ii + bi, jj);\n                }\n                if constexpr (RN > 1) {\n                    for (; jj < jj2; jj += RN - 1) {\n                        gemm_bloc<RM, RN-1>(ii + bi, jj);\n                    }\n                }\n                GGML_ASSERT(jj == jj2);\n            }\n\n            job = ggml_threadpool_chunk_add(params->threadpool, 1);\n        }\n\n        ggml_barrier(params->threadpool);\n        return;\n    }\n\n    const ggml_compute_params * params;\n    const TA *const A;\n    const TB *const B;\n    TC *const C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n};\n\n#if defined(__riscv_v_intrinsic)\ntemplate <typename D, typename V, typename TA, typename TB, typename TC>\nclass tinyBLAS_RVV {\n  public:\n    tinyBLAS_RVV(const ggml_compute_params * params, int64_t k,\n             const TA *A, int64_t lda,\n             const TB *B, int64_t ldb,\n             TC *C, int64_t ldc)\n        : params(params), A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc) {\n    }\n\n    bool matmul(int64_t m, int64_t n) {\n        if (k % vlmax<V>() != 0) {\n            return false;\n        }\n\n#if LMUL == 1\n        if (m % 16 == 0 && (m/16 >= params->nth)) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 4>(m, n, SIZE_N, 12);\n            return true;\n        }\n        if (m % 8 == 0 ) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 2>(m, n, SIZE_N, 12);\n            return true;\n        }\n        if (m % 4 == 0) {\n            const int64_t SIZE_N = BLOCK_SIZE<6>(n);\n            mnpack<4, 6, 1>(m, n, SIZE_N, 12);\n            return true;\n        }\n#elif LMUL == 2\n        if (m % 16 == 0 && (m/16 >= params->nth)) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 4>(m, n, SIZE_N, 24);\n            return true;\n        }\n        if (m % 8 == 0 ) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 2>(m, n, SIZE_N, 24);\n            return true;\n        }\n        if (m % 4 == 0) {\n            const int64_t SIZE_N = BLOCK_SIZE<3>(n);\n            mnpack<4, 3, 1>(m, n, SIZE_N, 24);\n            return true;\n        }\n#else // LMUL = 4\n        if (m % 16 == 0 && (m/16 >= params->nth)) {\n            const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n            mnpack<2, 2, 8>(m, n, SIZE_N, 36);\n            return true;\n        }\n        if (m % 8 == 0 ) {\n            const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n            mnpack<2, 2, 4>(m, n, SIZE_N, 36);\n            return true;\n        }\n        if (m % 4 == 0) {\n            const int64_t SIZE_N = BLOCK_SIZE<2>(n);\n            mnpack<2, 2, 2>(m, n, SIZE_N, 36);\n            return true;\n        }\n#endif\n        return false;\n    }\n\n  private:\n    template<int RM, int RN, int BM>\n    inline void mnpack(int64_t m, int64_t n, int64_t SIZE_N, int64_t BN) {\n        if (SIZE_N == RN) {\n            return gemm<RM, RN, BM>(m, n, BN);\n        }\n        if constexpr (RN > 1) {\n            return mnpack<RM, RN-1, BM>(m, n, SIZE_N, BN);\n        } else {\n            GGML_LOG_ERROR(\"mnpack<%d, %d> block size not supported\\n\", RM, (int)SIZE_N);\n            GGML_ASSERT(false); // we have miss something.\n        }\n    }\n\n    inline void gemm_bloc_4x6(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n        D Cv12 = set_zero<D>();\n        D Cv13 = set_zero<D>();\n        D Cv20 = set_zero<D>();\n        D Cv21 = set_zero<D>();\n        D Cv22 = set_zero<D>();\n        D Cv23 = set_zero<D>();\n        D Cv30 = set_zero<D>();\n        D Cv31 = set_zero<D>();\n        D Cv32 = set_zero<D>();\n        D Cv33 = set_zero<D>();\n        D Cv40 = set_zero<D>();\n        D Cv41 = set_zero<D>();\n        D Cv42 = set_zero<D>();\n        D Cv43 = set_zero<D>();\n        D Cv50 = set_zero<D>();\n        D Cv51 = set_zero<D>();\n        D Cv52 = set_zero<D>();\n        D Cv53 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            V Bv2 = load<V>(B + ldb * (jj + 2) + l);\n            V Bv3 = load<V>(B + ldb * (jj + 3) + l);\n            V Bv4 = load<V>(B + ldb * (jj + 4) + l);\n            V Bv5 = load<V>(B + ldb * (jj + 5) + l);\n\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv20 = madd(Av0, Bv2, Cv20);\n            Cv30 = madd(Av0, Bv3, Cv30);\n            Cv40 = madd(Av0, Bv4, Cv40);\n            Cv50 = madd(Av0, Bv5, Cv50);\n\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv11 = madd(Av1, Bv1, Cv11);\n            Cv21 = madd(Av1, Bv2, Cv21);\n            Cv31 = madd(Av1, Bv3, Cv31);\n            Cv41 = madd(Av1, Bv4, Cv41);\n            Cv51 = madd(Av1, Bv5, Cv51);\n\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv12 = madd(Av2, Bv1, Cv12);\n            Cv22 = madd(Av2, Bv2, Cv22);\n            Cv32 = madd(Av2, Bv3, Cv32);\n            Cv42 = madd(Av2, Bv4, Cv42);\n            Cv52 = madd(Av2, Bv5, Cv52);\n\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n            Cv03 = madd(Av3, Bv0, Cv03);\n            Cv13 = madd(Av3, Bv1, Cv13);\n            Cv23 = madd(Av3, Bv2, Cv23);\n            Cv33 = madd(Av3, Bv3, Cv33);\n            Cv43 = madd(Av3, Bv4, Cv43);\n            Cv53 = madd(Av3, Bv5, Cv53);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n        C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n        C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n        C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n        C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n        C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n        C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n        C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n        C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n        C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n        C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n        C[ldc * (jj + 4) + (ii + 0)] = hsum(Cv40);\n        C[ldc * (jj + 4) + (ii + 1)] = hsum(Cv41);\n        C[ldc * (jj + 4) + (ii + 2)] = hsum(Cv42);\n        C[ldc * (jj + 4) + (ii + 3)] = hsum(Cv43);\n        C[ldc * (jj + 5) + (ii + 0)] = hsum(Cv50);\n        C[ldc * (jj + 5) + (ii + 1)] = hsum(Cv51);\n        C[ldc * (jj + 5) + (ii + 2)] = hsum(Cv52);\n        C[ldc * (jj + 5) + (ii + 3)] = hsum(Cv53);\n    }\n\n    inline void gemm_bloc_4x5(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n        D Cv12 = set_zero<D>();\n        D Cv13 = set_zero<D>();\n        D Cv20 = set_zero<D>();\n        D Cv21 = set_zero<D>();\n        D Cv22 = set_zero<D>();\n        D Cv23 = set_zero<D>();\n        D Cv30 = set_zero<D>();\n        D Cv31 = set_zero<D>();\n        D Cv32 = set_zero<D>();\n        D Cv33 = set_zero<D>();\n        D Cv40 = set_zero<D>();\n        D Cv41 = set_zero<D>();\n        D Cv42 = set_zero<D>();\n        D Cv43 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            V Bv2 = load<V>(B + ldb * (jj + 2) + l);\n            V Bv3 = load<V>(B + ldb * (jj + 3) + l);\n            V Bv4 = load<V>(B + ldb * (jj + 4) + l);\n\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv20 = madd(Av0, Bv2, Cv20);\n            Cv30 = madd(Av0, Bv3, Cv30);\n            Cv40 = madd(Av0, Bv4, Cv40);\n\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv11 = madd(Av1, Bv1, Cv11);\n            Cv21 = madd(Av1, Bv2, Cv21);\n            Cv31 = madd(Av1, Bv3, Cv31);\n            Cv41 = madd(Av1, Bv4, Cv41);\n\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv12 = madd(Av2, Bv1, Cv12);\n            Cv22 = madd(Av2, Bv2, Cv22);\n            Cv32 = madd(Av2, Bv3, Cv32);\n            Cv42 = madd(Av2, Bv4, Cv42);\n\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n            Cv03 = madd(Av3, Bv0, Cv03);\n            Cv13 = madd(Av3, Bv1, Cv13);\n            Cv23 = madd(Av3, Bv2, Cv23);\n            Cv33 = madd(Av3, Bv3, Cv33);\n            Cv43 = madd(Av3, Bv4, Cv43);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n        C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n        C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n        C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n        C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n        C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n        C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n        C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n        C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n        C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n        C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n        C[ldc * (jj + 4) + (ii + 0)] = hsum(Cv40);\n        C[ldc * (jj + 4) + (ii + 1)] = hsum(Cv41);\n        C[ldc * (jj + 4) + (ii + 2)] = hsum(Cv42);\n        C[ldc * (jj + 4) + (ii + 3)] = hsum(Cv43);\n    }\n\n    inline void gemm_bloc_4x4(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n        D Cv12 = set_zero<D>();\n        D Cv13 = set_zero<D>();\n        D Cv20 = set_zero<D>();\n        D Cv21 = set_zero<D>();\n        D Cv22 = set_zero<D>();\n        D Cv23 = set_zero<D>();\n        D Cv30 = set_zero<D>();\n        D Cv31 = set_zero<D>();\n        D Cv32 = set_zero<D>();\n        D Cv33 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv03 = madd(Av3, Bv0, Cv03);\n\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv11 = madd(Av1, Bv1, Cv11);\n            Cv12 = madd(Av2, Bv1, Cv12);\n            Cv13 = madd(Av3, Bv1, Cv13);\n\n            V Bv2 = load<V>(B + ldb * (jj + 2) + l);\n            Cv20 = madd(Av0, Bv2, Cv20);\n            Cv21 = madd(Av1, Bv2, Cv21);\n            Cv22 = madd(Av2, Bv2, Cv22);\n            Cv23 = madd(Av3, Bv2, Cv23);\n\n            V Bv3 = load<V>(B + ldb * (jj + 3) + l);\n            Cv30 = madd(Av0, Bv3, Cv30);\n            Cv31 = madd(Av1, Bv3, Cv31);\n            Cv32 = madd(Av2, Bv3, Cv32);\n            Cv33 = madd(Av3, Bv3, Cv33);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n        C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n        C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n        C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n        C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n        C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n        C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n        C[ldc * (jj + 3) + (ii + 0)] = hsum(Cv30);\n        C[ldc * (jj + 3) + (ii + 1)] = hsum(Cv31);\n        C[ldc * (jj + 3) + (ii + 2)] = hsum(Cv32);\n        C[ldc * (jj + 3) + (ii + 3)] = hsum(Cv33);\n    }\n\n    inline void gemm_bloc_4x3(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n        D Cv12 = set_zero<D>();\n        D Cv13 = set_zero<D>();\n        D Cv20 = set_zero<D>();\n        D Cv21 = set_zero<D>();\n        D Cv22 = set_zero<D>();\n        D Cv23 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv03 = madd(Av3, Bv0, Cv03);\n\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv11 = madd(Av1, Bv1, Cv11);\n            Cv12 = madd(Av2, Bv1, Cv12);\n            Cv13 = madd(Av3, Bv1, Cv13);\n\n            V Bv2 = load<V>(B + ldb * (jj + 2) + l);\n            Cv20 = madd(Av0, Bv2, Cv20);\n            Cv21 = madd(Av1, Bv2, Cv21);\n            Cv22 = madd(Av2, Bv2, Cv22);\n            Cv23 = madd(Av3, Bv2, Cv23);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n        C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n        C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n        C[ldc * (jj + 2) + (ii + 0)] = hsum(Cv20);\n        C[ldc * (jj + 2) + (ii + 1)] = hsum(Cv21);\n        C[ldc * (jj + 2) + (ii + 2)] = hsum(Cv22);\n        C[ldc * (jj + 2) + (ii + 3)] = hsum(Cv23);\n    }\n\n    inline void gemm_bloc_4x2(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n        D Cv12 = set_zero<D>();\n        D Cv13 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv03 = madd(Av3, Bv0, Cv03);\n\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv11 = madd(Av1, Bv1, Cv11);\n            Cv12 = madd(Av2, Bv1, Cv12);\n            Cv13 = madd(Av3, Bv1, Cv13);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n        C[ldc * (jj + 1) + (ii + 2)] = hsum(Cv12);\n        C[ldc * (jj + 1) + (ii + 3)] = hsum(Cv13);\n    }\n\n    inline void gemm_bloc_4x1(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv02 = set_zero<D>();\n        D Cv03 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n            V Av2 = load<V>(A + lda * (ii + 2) + l);\n            V Av3 = load<V>(A + lda * (ii + 3) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n            Cv02 = madd(Av2, Bv0, Cv02);\n            Cv03 = madd(Av3, Bv0, Cv03);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 0) + (ii + 2)] = hsum(Cv02);\n        C[ldc * (jj + 0) + (ii + 3)] = hsum(Cv03);\n    }\n\n    inline void gemm_bloc_2x2(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n        D Cv10 = set_zero<D>();\n        D Cv11 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n\n            V Bv1 = load<V>(B + ldb * (jj + 1) + l);\n            Cv10 = madd(Av0, Bv1, Cv10);\n            Cv11 = madd(Av1, Bv1, Cv11);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n        C[ldc * (jj + 1) + (ii + 0)] = hsum(Cv10);\n        C[ldc * (jj + 1) + (ii + 1)] = hsum(Cv11);\n    }\n\n    inline void gemm_bloc_2x1(int64_t ii, int64_t jj) {\n        size_t vl = vlmax<V>();\n        D Cv00 = set_zero<D>();\n        D Cv01 = set_zero<D>();\n\n        for (int64_t l = 0; l < k; l += vl) {\n            V Av0 = load<V>(A + lda * (ii + 0) + l);\n            V Av1 = load<V>(A + lda * (ii + 1) + l);\n\n            V Bv0 = load<V>(B + ldb * (jj + 0) + l);\n            Cv00 = madd(Av0, Bv0, Cv00);\n            Cv01 = madd(Av1, Bv0, Cv01);\n        }\n\n        C[ldc * (jj + 0) + (ii + 0)] = hsum(Cv00);\n        C[ldc * (jj + 0) + (ii + 1)] = hsum(Cv01);\n    }\n\n    template <int RM, int RN>\n    inline void gemm_bloc(int64_t ii, int64_t jj) {\n        if constexpr (RM == 4) {\n            if constexpr (RN == 6) { return gemm_bloc_4x6(ii, jj); }\n            if constexpr (RN == 5) { return gemm_bloc_4x5(ii, jj); }\n            if constexpr (RN == 4) { return gemm_bloc_4x4(ii, jj); }\n            if constexpr (RN == 3) { return gemm_bloc_4x3(ii, jj); }\n            if constexpr (RN == 2) { return gemm_bloc_4x2(ii, jj); }\n            if constexpr (RN == 1) { return gemm_bloc_4x1(ii, jj); }\n        } else if constexpr (RM == 2) {\n            if constexpr (RN == 2) { return gemm_bloc_2x2(ii, jj); }\n            if constexpr (RN == 1) { return gemm_bloc_2x1(ii, jj); }\n        }\n    }\n\n    template <int RM, int RN, int BM>\n    NOINLINE void gemm(int64_t m, int64_t n, int64_t BN) {\n        GGML_ASSERT(m % (RM * BM) == 0);\n        const int64_t ytiles = m / (RM * BM);\n        const int64_t xtiles = (n + RN -1) / RN;\n        const int64_t jj_RN = (xtiles - (xtiles * RN - n));\n\n        // \"round\" bloc_size to \"nearest\" BN\n        const int64_t NB_BN = xtiles < BN ? 1 : (xtiles + BN / 2) / BN;\n        const int64_t SIZE_BN = xtiles % NB_BN == 0 ? xtiles / NB_BN : xtiles / NB_BN + 1;\n        const int64_t jj_BN = (NB_BN - (NB_BN * SIZE_BN - xtiles));\n        const int64_t nb_job = ytiles * NB_BN;\n\n        if (params->ith == 0) {\n            GGML_ASSERT( jj_BN * SIZE_BN + (NB_BN - jj_BN) * (SIZE_BN - 1) == xtiles);\n            // Every thread starts at ith, so the first unprocessed chunk is nth.  This save a bit of coordination right at the start.\n            ggml_threadpool_chunk_set(params->threadpool, params->nth);\n        }\n\n        ggml_barrier(params->threadpool);\n\n        int64_t job = params->ith;\n        while (job < nb_job) {\n            const int64_t ii = (job % ytiles) * RM * BM;\n            const int64_t jb =  job / ytiles;\n            const int64_t jr0 = BLOC_POS(jb  , jj_BN, SIZE_BN);\n            const int64_t jrN = BLOC_POS(jb+1, jj_BN, SIZE_BN);\n\n            const int64_t jj0 = BLOC_POS(jr0, jj_RN, RN);\n            const int64_t jj2 = BLOC_POS(jrN, jj_RN, RN);\n            const int64_t jj1 = jj2 < jj_RN * RN ? jj2 : jj_RN * RN;\n\n            for (int64_t bi = 0; bi < BM * RM; bi += RM) {\n                int64_t jj = jj0;\n                for (; jj < jj1; jj += RN) {\n                    gemm_bloc<RM, RN>(ii + bi, jj);\n                }\n                if constexpr (RN > 1) {\n                    for (; jj < jj2; jj += RN - 1) {\n                        gemm_bloc<RM, RN-1>(ii + bi, jj);\n                    }\n                }\n                GGML_ASSERT(jj == jj2);\n            }\n\n            job = ggml_threadpool_chunk_add(params->threadpool, 1);\n        }\n\n        ggml_barrier(params->threadpool);\n        return;\n    }\n\n    const ggml_compute_params * params;\n    const TA *const A;\n    const TB *const B;\n    TC *const C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n};\n#endif\n\n//////////////////////////////////////////////////////////////////////////////////////////\n// QUANT ZERO MATRIX MULTIPLICATION\n\n#if defined(__ARM_FEATURE_DOTPROD)\ntemplate <typename TA>\nclass tinyBLAS_Q0_ARM {\n  public:\n    tinyBLAS_Q0_ARM(int64_t k,\n                    const TA *A, int64_t lda,\n                    const block_q8_0 *B, int64_t ldb,\n                    float *C, int64_t ldc,\n                    int ith, int nth)\n        : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n    }\n\n    void matmul(int64_t m, int64_t n) {\n        mnpack(0, m, 0, n);\n    }\n\n  private:\n    NOINLINE void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t mc, nc, mp, np;\n        switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3ll)) {\n        case 0x33:\n            mc = 3;\n            nc = 3;\n            gemm<3, 3>(m0, m, n0, n);\n            break;\n        case 0x32:\n            mc = 3;\n            nc = 2;\n            gemm<3, 2>(m0, m, n0, n);\n            break;\n        case 0x23:\n            mc = 2;\n            nc = 3;\n            gemm<2, 3>(m0, m, n0, n);\n            break;\n        case 0x22:\n            mc = 2;\n            nc = 2;\n            gemm<2, 2>(m0, m, n0, n);\n            break;\n        case 0x31:\n            mc = 3;\n            nc = 1;\n            gemm<3, 1>(m0, m, n0, n);\n            break;\n        case 0x13:\n            mc = 1;\n            nc = 3;\n            gemm<1, 3>(m0, m, n0, n);\n            break;\n        case 0x21:\n            mc = 2;\n            nc = 1;\n            gemm<2, 1>(m0, m, n0, n);\n            break;\n        case 0x12:\n            mc = 1;\n            nc = 2;\n            gemm<1, 2>(m0, m, n0, n);\n            break;\n        case 0x11:\n            mc = 1;\n            nc = 1;\n            gemm<1, 1>(m0, m, n0, n);\n            break;\n        default:\n            return;\n        }\n        mp = m0 + (m - m0) / mc * mc;\n        np = n0 + (n - n0) / nc * nc;\n        mnpack(mp, m, n0, np);\n        mnpack(m0, m, np, n);\n    }\n\n    template <int RM, int RN>\n    NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            float32x4_t Cv[RN][RM] = {};\n            for (int64_t l = 0; l < k; ++l)\n                for (int64_t j = 0; j < RN; ++j)\n                    for (int64_t i = 0; i < RM; ++i)\n                        Cv[j][i] = vmlaq_n_f32(Cv[j][i],\n                                               vcvtq_f32_s32(vdotq_s32(\n                                                   vdotq_s32(vdupq_n_s32(0),\n                                                             load_lo(A + lda * (ii + i) + l),\n                                                             load_lo(B + ldb * (jj + j) + l)),\n                                                   load_hi(A + lda * (ii + i) + l),\n                                                   load_hi(B + ldb * (jj + j) + l))),\n                                               unhalf(A[lda * (ii + i) + l].d) *\n                                               unhalf(B[ldb * (jj + j) + l].d));\n            for (int64_t j = 0; j < RN; ++j)\n                for (int64_t i = 0; i < RM; ++i)\n                    C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n        }\n    }\n\n    inline int8x16_t load_lo(const block_q8_0 *b) {\n        return vld1q_s8(b->qs);\n    }\n\n    inline int8x16_t load_hi(const block_q8_0 *b) {\n        return vld1q_s8(b->qs + 16);\n    }\n\n    inline int8x16_t load_lo(const block_q4_0 *b) {\n        return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs),\n                                                     vdupq_n_u8(0x0f))),\n                        vdupq_n_s8(0x8));\n    }\n\n    inline int8x16_t load_hi(const block_q4_0 *b) {\n        return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)),\n                        vdupq_n_s8(0x8));\n    }\n\n    const TA *const A;\n    const block_q8_0 *const B;\n    float *const C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n    const int ith;\n    const int nth;\n};\n#endif // __ARM_FEATURE_DOTPROD\n\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\ntemplate <typename TA, typename TB, typename TC>\nclass tinyBLAS_Q0_AVX {\n  public:\n    tinyBLAS_Q0_AVX(int64_t k,\n                    const TA *A, int64_t lda,\n                    const TB *B, int64_t ldb,\n                    TC *C, int64_t ldc,\n                    int ith, int nth)\n        : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n        const int8_t kvalues_iq4nl[16] = {\n            -127, -104, -83, -65,\n            -49,  -35,  -22, -10,\n              1,   13,   25,  38,\n             53,   69,   89, 113\n        };\n\n        iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl);\n    }\n\n    void matmul(int64_t m, int64_t n) {\n        mnpack(0, m, 0, n);\n    }\n\n  private:\n    void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t mc, nc, mp, np;\n        switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {\n#if VECTOR_REGISTERS == 32\n        case 0x44:\n            mc = 4;\n            nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemm4xN<4>(m0, m, n0, n);\n#else\n            gemm<4, 4>(m0, m, n0, n);\n#endif\n            break;\n        case 0x43:\n            mc = 4;\n            nc = 3;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemm4xN<3>(m0, m, n0, n);\n#else\n            gemm<4, 3>(m0, m, n0, n);\n#endif\n            break;\n        case 0x34:\n            mc = 3;\n            nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemmMx4<3>(m0, m, n0, n);\n#else\n            gemm<3, 4>(m0, m, n0, n);\n#endif\n            break;\n        case 0x33:\n            mc = 3;\n            nc = 3;\n            gemm<3, 3>(m0, m, n0, n);\n            break;\n        case 0x42:\n            mc = 4;\n            nc = 2;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemm4xN<2>(m0, m, n0, n);\n#else\n            gemm<4, 2>(m0, m, n0, n);\n#endif\n            break;\n        case 0x24:\n            mc = 2;\n            nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemmMx4<2>(m0, m, n0, n);\n#else\n            gemm<2, 4>(m0, m, n0, n);\n#endif\n            break;\n#else\n        case 0x44:\n        case 0x43:\n        case 0x42:\n            mc = 4;\n            nc = 2;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemm4xN<2>(m0, m, n0, n);\n#else\n            gemm<4, 2>(m0, m, n0, n);\n#endif\n            break;\n        case 0x34:\n        case 0x24:\n            mc = 2;\n            nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemmMx4<2>(m0, m, n0, n);\n#else\n            gemm<2, 4>(m0, m, n0, n);\n#endif\n            break;\n        case 0x33:\n#endif\n        case 0x32:\n            mc = 3;\n            nc = 2;\n            gemm<3, 2>(m0, m, n0, n);\n            break;\n        case 0x23:\n            mc = 2;\n            nc = 3;\n            gemm<2, 3>(m0, m, n0, n);\n            break;\n        case 0x41:\n            mc = 4;\n            nc = 1;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemm4xN<1>(m0, m, n0, n);\n#else\n            gemm<4, 1>(m0, m, n0, n);\n#endif\n            break;\n        case 0x22:\n            mc = 2;\n            nc = 2;\n            gemm<2, 2>(m0, m, n0, n);\n            break;\n        case 0x14:\n            mc = 1;\n            nc = 4;\n#if defined(__AVX2__) && defined(__F16C__)\n            gemmMx4<1>(m0, m, n0, n);\n#else\n            gemm<1, 4>(m0, m, n0, n);\n#endif\n            break;\n        case 0x31:\n            mc = 3;\n            nc = 1;\n            gemm<3, 1>(m0, m, n0, n);\n            break;\n        case 0x13:\n            mc = 1;\n            nc = 3;\n            gemm<1, 3>(m0, m, n0, n);\n            break;\n        case 0x21:\n            mc = 2;\n            nc = 1;\n            gemm<2, 1>(m0, m, n0, n);\n            break;\n        case 0x12:\n            mc = 1;\n            nc = 2;\n            gemm<1, 2>(m0, m, n0, n);\n            break;\n        case 0x11:\n            mc = 1;\n            nc = 1;\n            gemm<1, 1>(m0, m, n0, n);\n            break;\n        default:\n            return;\n        }\n        mp = m0 + (m - m0) / mc * mc;\n        np = n0 + (n - n0) / nc * nc;\n        mnpack(mp, m, n0, np);\n        mnpack(m0, m, np, n);\n    }\n\n#if defined(__AVX2__) && defined(__F16C__)\n// Templated functions for gemm of dimensions 4xN\n    template <int RN>\n    NOINLINE void gemm4xN(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / 4;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * 4;\n            int64_t jj = n0 + job % xtiles * RN;\n            __m256 Cv[RN][4] = {};\n            for (int64_t l = 0; l < k; ++l) {\n                uint64_t a_delta = ((uint64_t)A[lda * (ii + 3) + l].d << 48) | ((uint64_t)A[lda * (ii + 2) + l].d << 32) | ((uint64_t)A[lda * (ii + 1) + l].d << 16) | (A[lda * (ii + 0) + l].d);\n                // Convert delta values for four blocks to float values\n                __m128 da = _mm_cvtph_ps(_mm_set_epi64x(0, a_delta));\n                __m256i avec0 = load(A + lda * (ii + 0) + l);\n                __m256i avec1 = load(A + lda * (ii + 1) + l);\n                __m256i avec2 = load(A + lda * (ii + 2) + l);\n                __m256i avec3 = load(A + lda * (ii + 3) + l);\n                for (int64_t j = 0; j < RN; ++j) {\n                        __m128 db = _mm_set1_ps(unhalf(B[ldb * (jj + j) + l].d));\n                        // Computation of product of delta values for four blocks and replicate it across 256 bit lane\n                        __m256 dvec =  _mm256_castps128_ps256(_mm_mul_ps(da, db));\n                        dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);\n                        // Computation of dot product and multiplication with appropriate delta value products\n                        Cv[j][0] = madd(_mm256_shuffle_ps(dvec, dvec, 0),\n                                    updot(_mm256_sign_epi8(avec0, avec0),\n                                          _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec0)),\n                                    Cv[j][0]);\n                        Cv[j][1] = madd(_mm256_shuffle_ps(dvec, dvec, 85),\n                                    updot(_mm256_sign_epi8(avec1, avec1),\n                                            _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec1)),\n                                    Cv[j][1]);\n                        Cv[j][2] = madd(_mm256_shuffle_ps(dvec, dvec, 170),\n                                    updot(_mm256_sign_epi8(avec2, avec2),\n                                            _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec2)),\n                                    Cv[j][2]);\n                        Cv[j][3] = madd(_mm256_shuffle_ps(dvec, dvec, 255),\n                                    updot(_mm256_sign_epi8(avec3, avec3),\n                                            _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec3)),\n                                    Cv[j][3]);\n                }\n            }\n\n            for (int64_t j = 0; j < RN; ++j)\n                for (int64_t i = 0; i < 4; ++i)\n                    C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n        }\n    }\n\n    // Templated functions for gemm of dimensions Mx4\n    template <int RM>\n    NOINLINE void gemmMx4(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / 4;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * 4;\n            __m256 Cv[4][RM] = {};\n            for (int64_t l = 0; l < k; ++l) {\n                uint64_t b_delta = ((uint64_t)B[ldb * (jj + 3) + l].d << 48) | ((uint64_t)B[ldb * (jj + 2) + l].d << 32) | ((uint64_t)B[ldb * (jj + 1) + l].d << 16) | (B[ldb * (jj + 0) + l].d);\n                // Convert delta values for four blocks to float values\n                __m128 db = _mm_cvtph_ps(_mm_set_epi64x(0, b_delta));\n                __m256i bvec0 = load(B + ldb * (jj + 0) + l);\n                __m256i bvec1 = load(B + ldb * (jj + 1) + l);\n                __m256i bvec2 = load(B + ldb * (jj + 2) + l);\n                __m256i bvec3 = load(B + ldb * (jj + 3) + l);\n                for (int64_t i = 0; i < RM; ++i) {\n                    __m128 da = _mm_set1_ps(unhalf((A[lda * (ii + i) + l].d)));\n                    // Computation of product of delta values for four blocks and replicate it across 256 bit lane\n                    __m256 dvec =  _mm256_castps128_ps256(_mm_mul_ps(da, db));\n                    dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);\n                    // Computation of dot product and multiplication with appropriate delta value products\n                    Cv[0][i] = madd(_mm256_shuffle_ps(dvec, dvec, 0),\n                                    updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n                                                            load(A + lda * (ii + i) + l)),\n                                            _mm256_sign_epi8(bvec0, load(A + lda * (ii + i) + l))),\n                                    Cv[0][i]);\n                    Cv[1][i] = madd(_mm256_shuffle_ps(dvec, dvec, 85),\n                                    updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n                                                            load(A + lda * (ii + i) + l)),\n                                            _mm256_sign_epi8(bvec1, load(A + lda * (ii + i) + l))),\n                                    Cv[1][i]);\n                    Cv[2][i] = madd(_mm256_shuffle_ps(dvec, dvec, 170),\n                                    updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n                                                            load(A + lda * (ii + i) + l)),\n                                            _mm256_sign_epi8(bvec2, load(A + lda * (ii + i) + l))),\n                                    Cv[2][i]);\n                    Cv[3][i] = madd(_mm256_shuffle_ps(dvec, dvec, 255),\n                                    updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n                                                            load(A + lda * (ii + i) + l)),\n                                            _mm256_sign_epi8(bvec3, load(A + lda * (ii + i) + l))),\n                                    Cv[3][i]);\n                }\n            }\n            for (int64_t j = 0; j < 4; ++j)\n                for (int64_t i = 0; i < RM; ++i)\n                    C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n        }\n    }\n#endif\n\n    template <int RM, int RN>\n    NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            __m256 Cv[RN][RM] = {};\n            for (int64_t l = 0; l < k; ++l)\n                for (int64_t j = 0; j < RN; ++j)\n                    for (int64_t i = 0; i < RM; ++i) {\n#if defined(__AVX2__)\n                        __m256 udTmp = updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),\n                                                              load(A + lda * (ii + i) + l)),\n                                             _mm256_sign_epi8(load(B + ldb * (jj + j) + l),\n                                                              load(A + lda * (ii + i) + l)));\n#else\n                        __m128i ali0 = load0(A + lda * (ii + i) + l);\n                        __m128i ali1 = load1(A + lda * (ii + i) + l);\n                        __m128i blj0 = load0(B + ldb * (jj + j) + l);\n                        __m128i blj1 = load1(B + ldb * (jj + j) + l);\n\n                        __m128i sepAA0 = _mm_sign_epi8(ali0, ali0);\n                        __m128i sepAA1 = _mm_sign_epi8(ali1, ali1);\n                        __m128i sepBA0 = _mm_sign_epi8(blj0, ali0);\n                        __m128i sepBA1 = _mm_sign_epi8(blj1, ali1);\n\n                        // updot\n                        const __m128i oneFill = _mm_set1_epi16(1);\n                        __m128i mad0 = _mm_maddubs_epi16(sepAA0, sepBA0);\n                        __m128i mad1 = _mm_maddubs_epi16(sepAA1, sepBA1);\n                        __m256 udTmp = _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_madd_epi16(oneFill, mad1), _mm_madd_epi16(oneFill, mad0)));\n#endif\n                        Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *\n                                                       unhalf(B[ldb * (jj + j) + l].d)),\n                                                       udTmp,\n                                                       Cv[j][i]);\n                    }\n            for (int64_t j = 0; j < RN; ++j)\n                for (int64_t i = 0; i < RM; ++i)\n                    C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);\n        }\n    }\n\n    inline __m256i load(const block_q8_0 *b) {\n        return _mm256_loadu_si256((const __m256i *)b->qs);\n    }\n\n    inline __m128i load0(const block_q8_0 *b) {\n        return _mm_loadu_si128((const __m128i *)b->qs);\n    }\n\n    inline __m128i load1(const block_q8_0 *b) {\n        return _mm_loadu_si128(((const __m128i *)b->qs) + 1);\n    }\n\n    inline __m256i load(const block_q4_0 *b) {\n        return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));\n    }\n\n    inline __m128i load0(const block_q4_0 *b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), x), _mm_set1_epi8(8));\n    }\n\n    inline __m128i load1(const block_q4_0 *b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)), _mm_set1_epi8(8));\n    }\n\n    inline __m256i load(const block_q5_0 *b) {\n        return _mm256_or_si256(denibble(b->qs), bittobyte(b->qh));\n    }\n\n    inline __m128i load0(const block_q5_0* b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        uint32_t x32;\n        memcpy(&x32, b->qh, sizeof(uint32_t));\n        __m128i qxl = _mm_and_si128(_mm_set1_epi8(15), x);\n        __m128i bytesl = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),\n                                        _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),\n                                                     _mm_shuffle_epi8(_mm_set1_epi32(x32),\n                                                                      _mm_set_epi64x(0x0101010101010101, 0x0000000000000000))));\n        bytesl = _mm_andnot_si128(bytesl, _mm_set1_epi8((char)0xF0));\n        return _mm_or_si128(qxl, bytesl);\n    }\n\n    inline __m128i load1(const block_q5_0* b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        uint32_t x32;\n        memcpy(&x32, b->qh, sizeof(uint32_t));\n        __m128i qxh = _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4));\n        __m128i bytesh = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),\n                                        _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),\n                                                     _mm_shuffle_epi8(_mm_set1_epi32(x32),\n                                                                      _mm_set_epi64x(0x0303030303030303, 0x0202020202020202))));\n        bytesh = _mm_andnot_si128(bytesh, _mm_set1_epi8((char)0xF0));\n        return _mm_or_si128(qxh, bytesh);\n    }\n\n    inline __m256i load(const block_iq4_nl *b) {\n        return MM256_SET_M128I(load1(b), load0(b));\n    }\n\n    inline __m128i load0(const block_iq4_nl *b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), x));\n    }\n\n    inline __m128i load1(const block_iq4_nl *b) {\n        const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));\n        return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)));\n    }\n\n    inline __m256 updot(__m256i u, __m256i s) {\n        __m256i res;\n#if defined(__AVX512VNNI__) && defined(__AVX512VL__)\n        res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);\n#elif defined(__AVXVNNI__)\n        res = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), u, s);\n#else\n        res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));\n#endif\n        return _mm256_cvtepi32_ps(res);\n    }\n\n    static inline __m256i denibble(const uint8_t *p) {\n        __m128i x = _mm_loadu_si128((const __m128i *)p);\n        return _mm256_and_si256(_mm256_set1_epi8(15),\n                                _mm256_insertf128_si256(_mm256_castsi128_si256(x),\n                                                        _mm_srli_epi16(x, 4), 1));\n    }\n\n    static inline __m256i bittobyte(const uint8_t *p) {\n        uint32_t x32;\n        memcpy(&x32, p, sizeof(uint32_t));\n        __m256i bytes = _mm256_cmpeq_epi8(_mm256_set1_epi64x(-1),\n                                          _mm256_or_si256(_mm256_set1_epi64x(0x7fbfdfeff7fbfdfe),\n                                                          _mm256_shuffle_epi8(_mm256_set1_epi32(x32),\n                                                                              _mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,\n                                                                                                0x0101010101010101, 0x0000000000000000))));\n        return _mm256_andnot_si256(bytes, _mm256_set1_epi8((char)0xF0));\n    }\n\n    const TA *const A;\n    const TB *const B;\n    TC *const C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n    const int ith;\n    const int nth;\n    __m128i iq4nlt;\n};\n#endif // __AVX__\n\n//PPC Implementation\n#if defined(__MMA__)\n\n#define SAVE_ACC(ACC, ii, jj) \\\n   __builtin_mma_disassemble_acc(vec_C, ACC); \\\n   for (int I = 0; I < 4; I++) { \\\n      for (int J = 0; J < 4; J++) { \\\n         *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); \\\n      } \\\n   } \\\n\ntemplate<typename T>\nstruct mma_instr;\n\ntemplate<>\nstruct mma_instr<ggml_bf16_t> {\n    static inline void outer_product(acc_t *acc, vec_t a, vec_t b) {\n        __builtin_mma_xvbf16ger2pp(acc, a, b);\n    }\n};\n\ntemplate<>\nstruct mma_instr<ggml_fp16_t> {\n    static inline void outer_product(acc_t *acc, vec_t a, vec_t b) {\n        __builtin_mma_xvf16ger2pp(acc, a, b);\n    }\n};\n\ntemplate <typename TA, typename TB, typename TC>\nclass tinyBLAS_HP16_PPC {\n  public:\n    tinyBLAS_HP16_PPC(int64_t k,\n                const TA *A, int64_t lda,\n                const TB *B, int64_t ldb,\n                TC *C, int64_t ldc,\n                int ith, int nth)\n        : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n    }\n\n    void matmul(int64_t m, int64_t n) {\n        mnpack(0, m, 0, n);\n    }\n\n  private:\n    void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) {\n        vec_t t[8], s[8];\n        vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n        vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n        vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n        vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n\n        if (numVec == 2) {\n            t[0] = vec_perm(c[0], c[1], swiz1);\n            t[1] = vec_perm(c[2], c[3], swiz1);\n            s[0] = vec_perm(t[0], t[1], swiz3);\n            s[1] = vec_perm(t[0], t[1], swiz4);\n            vec_xst(s[0], 0, (vec_t*)vecOffset);\n            vec_xst(s[1], 0, (vec_t*)(vecOffset + 16));\n        } else if (numVec == 4) {\n            t[0] = vec_perm(c[0], c[1], swiz1);\n            t[1] = vec_perm(c[0], c[1], swiz2);\n            t[2] = vec_perm(c[2], c[3], swiz1);\n            t[3] = vec_perm(c[2], c[3], swiz2);\n            s[0] = vec_perm(t[0], t[2], swiz3);\n            s[1] = vec_perm(t[0], t[2], swiz4);\n            s[2] = vec_perm(t[1], t[3], swiz3);\n            s[3] = vec_perm(t[1], t[3], swiz4);\n            for (int i = 0; i < 4; ++i)\n                vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));\n        } else if (numVec == 8) {\n            for (int i = 0; i < 4; i += 2) {\n                t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);\n                t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);\n            }\n            for (int i = 4; i < 8; i += 2) {\n                t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);\n                t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);\n            }\n            s[0] = vec_perm(t[0], t[2], swiz3);\n            s[1] = vec_perm(t[0], t[2], swiz4);\n            s[2] = vec_perm(t[1], t[3], swiz3);\n            s[3] = vec_perm(t[1], t[3], swiz4);\n            s[4] = vec_perm(t[4], t[6], swiz3);\n            s[5] = vec_perm(t[4], t[6], swiz4);\n            s[6] = vec_perm(t[5], t[7], swiz3);\n            s[7] = vec_perm(t[5], t[7], swiz4);\n            for (int i = 0; i < 8; ++i)\n                vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));\n        }\n    }\n\n    void packNormal(const TA* a, int64_t lda, int rows, int cols, unsigned char* vec) {\n        int64_t i, j;\n        TA *aoffset = NULL;\n        unsigned char *vecOffset = NULL;\n        TA * aoffsets[8];\n        vector unsigned char c_arr[8];\n        aoffset = const_cast<TA*>(a);\n        vecOffset = vec;\n        j = (rows >> 3);\n        if (j > 0) {\n            do {\n                if (cols == 4) {\n                    aoffsets[0] = aoffset;\n                    for (int it = 1; it < 4; ++it)\n                        aoffsets[it] = aoffsets[it-1] + lda;\n                    aoffset += 4 * lda;\n                    for (int i = 0; i < 4; ++i)\n                        c_arr[i] = vec_xl(0, (vector unsigned char*)aoffsets[i]);\n                    vector_permute_store(c_arr, 4, vecOffset);\n                    for (int i = 0; i<4; i++)\n                        aoffsets[i] = aoffsets[i]+lda;\n                    vecOffset +=64;\n                }\n                i = (cols >> 3);\n                if (i > 0) {\n                    aoffsets[0] = aoffset;\n                    for (int it = 1; it < 8; ++it) {\n                        aoffsets[it] = aoffsets[it-1] + lda;\n                    }\n                    aoffset += 8 * lda;\n                    do {\n                        for (int it = 0; it < 8; ++it)\n                            c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n                        vector_permute_store(c_arr, 8, vecOffset);\n                        for (int it = 0; it < 8; ++it)\n                            aoffsets[it] = aoffsets[it] + 8*lda;\n                        vecOffset += 128;\n                        i--;\n                    } while(i > 0);\n                }\n                j--;\n            } while(j > 0);\n        }\n        if (rows & 4) {\n            aoffsets[0] = aoffset;\n            for (int it = 1; it < 4; ++it)\n                aoffsets[it] = aoffsets[it-1] + lda;\n            aoffset += 4 * lda;\n            if (cols == 4) {\n                for (int it = 0; it < 4; ++it)\n                    c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n                vector_permute_store(c_arr, 2, vecOffset);\n                for (int it = 0; it< 4; it++)\n                    aoffsets[it] = aoffsets[it] + lda;\n                vecOffset += 32;\n            }\n            i = (cols >> 3);\n            if (i > 0) {\n                do {\n                    for (int it = 0; it < 4; ++it)\n                        c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);\n                    vector_permute_store(c_arr, 4, vecOffset);\n                    for (int it = 0; it< 4; it++)\n                        aoffsets[it] = aoffsets[it] + 8*lda;\n                    vecOffset += 64;\n                    i--;\n                } while(i > 0);\n            }\n        }\n        if (rows & 3) {\n            aoffsets[0] = aoffset;\n            for (int it = 1; it < 4; ++it)\n                aoffsets[it] = aoffsets[it-1] + lda;\n            if (cols == 4) {\n                switch(rows) {\n                    case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);\n                    case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);\n                    case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);\n                        break;\n                }\n                vector_permute_store(c_arr, 2, vecOffset);\n                for (int it = 0; it< 4; it++)\n                     aoffsets[it] = aoffsets[it] + lda;\n                vecOffset += 32;\n            }\n            i = (cols >> 3);\n            if (i > 0) {\n                do {\n                    switch(rows) {\n                        case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);\n                        case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);\n                        case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);\n                            break;\n                    }\n                    vector_permute_store(c_arr, 4, vecOffset);\n                    for (int it = 0; it <4; it++)\n                         aoffsets[it] = aoffsets[it] + 8* lda;\n                    vecOffset += 64;\n                    i--;\n                } while(i > 0);\n            }\n        }\n    }\n\n    void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t mc, nc, mp, np;\n        int m_rem = MIN(m - m0, 8);\n        int n_rem = MIN(n - n0, 8);\n\n        if (m_rem >= 8 && n_rem >= 8) {\n            mc = 8;\n            nc = 8;\n            gemm<8,8>(m0, m, n0, n);\n        } else if (m_rem >= 4 && n_rem >= 8) {\n            mc = 4;\n            nc = 8;\n            gemm<4,8>(m0, m, n0, n);\n        } else if (m_rem >=8 && n_rem >=4){\n                mc = 8;\n                nc = 4;\n                gemm<8,4>(m0, m, n0, n);\n        } else if ((m_rem < 4) && (n_rem >= 8)) {\n            nc = 8;\n            switch(m_rem) {\n                case 1:\n                    mc = 1;\n                    gemm_Mx8<1>(m0, m, n0, n);\n                    break;\n                case 2:\n                    mc = 2;\n                    gemm_Mx8<2>(m0, m, n0, n);\n                    break;\n                case 3:\n                    mc = 3;\n                    gemm_Mx8<3>(m0, m, n0, n);\n                    break;\n                default:\n                    return;\n            }\n        } else if (m_rem >= 4 && n_rem >= 4) {\n            mc = 4;\n            nc = 4;\n            gemm_small<4, 4>(m0, m, n0, n);\n        } else if ((m_rem > 4) && (n_rem < 4)) {\n            mc = 4;\n            switch(n_rem) {\n                case 1:\n                    nc = 1;\n                    gemm_small<4, 1>(m0, m, n0, n);\n                    break;\n                case 2:\n                    nc = 2;\n                    gemm_small<4, 2>(m0, m, n0, n);\n                    break;\n                case 3:\n                    nc = 3;\n                    gemm_small<4, 3>(m0, m, n0, n);\n                    break;\n\n                default:\n                    return;\n            }\n        } else {\n            switch((m_rem << 4) | n_rem) {\n                case 0x43:\n                    mc = 4;\n                    nc = 3;\n                    gemm_small<4, 3>(m0, m, n0, n);\n                    break;\n                case 0x42:\n                    mc = 4;\n                    nc = 2;\n                    gemm_small<4, 2>(m0, m, n0, n);\n                    break;\n                case 0x41:\n                    mc = 4;\n                    nc = 1;\n                    gemm_small<4, 1>(m0, m, n0, n);\n                    break;\n                case 0x34:\n                    mc = 3;\n                    nc = 4;\n                    gemm_small<3, 4>(m0, m, n0, n);\n                    break;\n                case 0x33:\n                    mc = 3;\n                    nc = 3;\n                    gemm_small<3, 3>(m0, m, n0, n);\n                    break;\n                case 0x32:\n                    mc = 3;\n                    nc = 2;\n                    gemm_small<3, 2>(m0, m, n0, n);\n                    break;\n                case 0x31:\n                    mc = 3;\n                    nc = 1;\n                    gemm_small<3, 1>(m0, m, n0, n);\n                    break;\n                case 0x24:\n                    mc = 2;\n                    nc = 4;\n                    gemm_small<2,4>(m0, m, n0, n);\n                    break;\n                case 0x23:\n                    mc = 2;\n                    nc = 3;\n                    gemm_small<2, 3>(m0, m, n0, n);\n                    break;\n                case 0x22:\n                    mc = 2;\n                    nc = 2;\n                    gemm_small<2, 2>(m0, m, n0, n);\n                    break;\n                case 0x21:\n                    mc = 2;\n                    nc = 1;\n                    gemm_small<2, 1>(m0, m, n0, n);\n                    break;\n                case 0x14:\n                    mc = 1;\n                    nc = 4;\n                    gemm_small<1, 4>(m0, m, n0, n);\n                    break;\n                case 0x13:\n                    mc = 1;\n                    nc = 3;\n                    gemm_small<1, 3>(m0, m, n0, n);\n                    break;\n                case 0x12:\n                    mc = 1;\n                    nc = 2;\n                    gemm_small<1, 2>(m0, m, n0, n);\n                    break;\n                case 0x11:\n                    mc = 1;\n                    nc = 1;\n                    gemm_small<1, 1>(m0, m, n0, n);\n                    break;\n                default:\n                    return;\n            }\n        }\n        mp = m0 + (m - m0) / mc * mc;\n        np = n0 + (n - n0) / nc * nc;\n        mnpack(mp, m, n0, np);\n        mnpack(m0, m, np, n);\n    }\n\n    void KERNEL_4x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[4], vec_B[8] , vec_C[4];\n        acc_t acc_0, acc_1;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        for (int l = 0; l < k; l+=8) {\n            packNormal((A+(ii*lda)+l), lda, 4, 8, (uint8_t*)vec_A);\n            packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B);\n            for (int x = 0; x < 4; x++) {\n                mma_instr<TA>::outer_product(&acc_0, vec_A[x], vec_B[x]);\n                mma_instr<TA>::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n            }\n        }\n        SAVE_ACC(&acc_0, ii, jj);\n        SAVE_ACC(&acc_1, ii, jj+4);\n    }\n\n    void KERNEL_8x4(int64_t ii, int64_t jj) {\n        vec_t vec_A[8], vec_B[4] , vec_C[4];\n        acc_t acc_0, acc_1;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        for (int l = 0; l < k; l+=8) {\n            packNormal((A+(ii*lda)+l), lda, 8, 8, (uint8_t*)vec_A);\n            packNormal((B+(jj*ldb)+l), ldb, 8, 4, (uint8_t*)vec_B);\n            for (int x = 0; x < 4; x++) {\n                mma_instr<TA>::outer_product(&acc_0, vec_A[x], vec_B[x]);\n                mma_instr<TA>::outer_product(&acc_1, vec_A[x+4], vec_B[x]);\n            }\n        }\n        SAVE_ACC(&acc_0, ii, jj);\n        SAVE_ACC(&acc_1, ii+4, jj);\n    }\n\n\n    void KERNEL_8x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[8], vec_B[8], vec_C[4];\n        acc_t acc_0, acc_1, acc_2, acc_3;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        __builtin_mma_xxsetaccz(&acc_2);\n        __builtin_mma_xxsetaccz(&acc_3);\n        for (int l = 0; l < k; l+=8) {\n            packNormal(A+(ii*lda)+l, lda, 8, 8, (uint8_t*)vec_A);\n            packNormal(B+(jj*ldb)+l, ldb, 8, 8, (uint8_t*)vec_B);\n            for (int x = 0; x < 4; x++) {\n                mma_instr<TA>::outer_product(&acc_0, vec_A[x], vec_B[x]);\n                mma_instr<TA>::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n                mma_instr<TA>::outer_product(&acc_2, vec_A[x+4], vec_B[x]);\n                mma_instr<TA>::outer_product(&acc_3, vec_A[x+4], vec_B[x+4]);\n            }\n        }\n\n        SAVE_ACC(&acc_0, ii, jj);\n        SAVE_ACC(&acc_1, ii, jj+4);\n        SAVE_ACC(&acc_2, ii+4, jj);\n        SAVE_ACC(&acc_3, ii+4, jj+4);\n    }\n\n    template<int RM, int RN>\n    void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            vec_t vec_C[4];\n            acc_t acc_0;\n            __builtin_mma_xxsetaccz(&acc_0);\n            vec_t vec_A[2], vec_B[2];\n            for (int l=0; l<k; l+=4) {\n                packNormal(A+(ii*lda)+l, lda, RM, 4, (uint8_t*)vec_A);\n                packNormal(B+(jj*ldb)+l, ldb, RN, 4, (uint8_t*)vec_B);\n                for (int x = 0; x<2; x++) {\n                    mma_instr<TA>::outer_product(&acc_0, vec_A[x], vec_B[x]);\n                }\n            }\n            __builtin_mma_disassemble_acc(vec_C, &acc_0);\n            for (int I = 0; I < RM; I++) {\n                for (int J = 0; J < RN; J++) {\n                    *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n                }\n            }\n        }\n    }\n\n    template<int RM>\n    void gemm_Mx8(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int RN = 8;\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            vec_t vec_C[4];\n            acc_t acc_0, acc_1;\n            __builtin_mma_xxsetaccz(&acc_0);\n            __builtin_mma_xxsetaccz(&acc_1);\n            vec_t vec_A[4], vec_B[8];\n            for (int l=0; l<k; l+=8) {\n                packNormal(A+(ii*lda)+l, lda, RM, 8, (uint8_t*)vec_A);\n                packNormal(B+(jj*ldb)+l, ldb, RN, 8, (uint8_t*)vec_B);\n                for (int x = 0; x<4; x++) {\n                    mma_instr<TA>::outer_product(&acc_0, vec_A[x], vec_B[x]);\n                    mma_instr<TA>::outer_product(&acc_1, vec_A[x], vec_B[x+4]);\n                }\n            }\n            __builtin_mma_disassemble_acc(vec_C, &acc_0);\n            for (int I = 0; I < RM; I++) {\n                for (int J = 0; J < 4; J++) {\n                    *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n                }\n            }\n            __builtin_mma_disassemble_acc(vec_C, &acc_1);\n            for (int I = 0; I < RM; I++) {\n                for (int J = 0; J < 4; J++) {\n                    *((TC*)(C+ii+((jj+4+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);\n                }\n            }\n        }\n    }\n\n    template<int RM, int RN>\n    inline void kernel(int64_t ii, int64_t jj) {\n       if constexpr(RM == 4 && RN == 8) {\n          KERNEL_4x8(ii,jj);\n       } else if constexpr(RM == 8 && RN == 8) {\n          KERNEL_8x8(ii,jj);\n       } else if constexpr(RM == 8 && RN == 4) {\n          KERNEL_8x4(ii,jj);\n       } else {\n          assert(false && \"RN/RM values not supported\");\n       }\n    }\n\n    template <int RM, int RN>\n    NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            kernel<RM, RN>(ii, jj);\n        }\n    }\n\n    const TA *const A;\n    const TB *const B;\n    TC *C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n    const int ith;\n    const int nth;\n};\n\ntemplate <typename TA>\nclass tinyBLAS_Q0_PPC {\n  public:\n    tinyBLAS_Q0_PPC(int64_t k,\n             const TA * A, int64_t lda,\n             const block_q8_0 * B, int64_t ldb,\n             float * C, int64_t ldc,\n             int ith, int nth)\n        : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n    }\n\n    void matmul(int64_t m, int64_t n) {\n        const int64_t mc = 64;\n        const int64_t kc = 64;\n        int64_t nc = 64;\n        int64_t n_aligned = 0;\n        if (n % 64 == 0) {\n            n_aligned = n;\n        } else if (n == 4) {\n            n_aligned = 4;\n        } else if (n < 64) {\n            n_aligned = (n / 8) * 8;\n        } else {\n            n_aligned = (n / 64) * 64;\n        }\n\n        if (n_aligned > 0) {\n            if (n_aligned % 64 == 0)      nc = 64;\n            else if (n_aligned == n)      nc = n;\n            else if (n_aligned % 32 == 0) nc = 32;\n            else if (n_aligned % 24 == 0) nc = 24;\n            else if (n_aligned % 16 == 0) nc = 16;\n            else                          nc = 8;\n        }\n        bool can_use_tiled = n_aligned > 0 && (m % mc == 0) && (k % kc == 0);\n        if (can_use_tiled) {\n            matmul_tiled(m, n_aligned, mc, nc, kc);\n            if (n > n_aligned) {\n                mnpack(0, m, n_aligned, n);\n            }\n        } else {\n            mnpack(0, m, 0, n);\n        }\n    }\n\n  private:\n    inline void save_res(int ii, int jj, int idx, vector float * fin_res, int RM = 4, int RN = 4) {\n        for (int I = 0; I < RM; I++) {\n            for (int J = 0; J < RN; J++) {\n                *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&fin_res[idx + I] + J);\n            }\n        }\n    }\n\n    inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n        vec_t vec_C[4];\n        __builtin_mma_disassemble_acc(vec_C, ACC);\n        for (int I = 0; I < 4; I++) {\n            for (int J = 0; J < 4; J++) {\n                *((float *)(C + ii + ((jj + J) * ldc) + I)) = *((float *)&vec_C[I] + J);\n            }\n        }\n    }\n\n    inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n        vec_t vec_C[4];\n        __builtin_mma_disassemble_acc(vec_C, ACC);\n        for (int I = 0; I < 4; I++) {\n            for (int J = 0; J < 4; J++) {\n                float * c_ptr = (float *)(C + ii+ ((jj + J) * ldc) + I);\n                *c_ptr += *((float *)&vec_C[I] + J);\n            }\n        }\n    }\n\n    template<typename ArrayType>\n    inline void compute(acc_t * ACC, int c_idx, int s_idx, ArrayType & comparray, vector float * vs, vector float * fin_res) {\n        vector signed int vec_C[4];\n        vector float CA[4] = {0};\n        vector float res[4] = {0};\n        __builtin_mma_disassemble_acc(vec_C, ACC);\n        for (int i = 0; i < 4; i++) {\n            CA[i] = vec_splats((float)(((double)comparray[c_idx + i]) * -128.0));\n            res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);\n            fin_res[s_idx + i] = vec_madd(res[i], vs[s_idx + i], fin_res[s_idx + i]);\n        }\n    }\n\n    inline void process_q4_elements(vector signed char (&c)[2], int * ca) {\n        const vector signed char lowMask = vec_splats((signed char)0xF);\n        const vector unsigned char v4 = vec_splats((unsigned char)0x4);\n        const vector signed char v8 = vec_splats((signed char)0x8);\n        vector signed int vsum = {0};\n        vector signed int vsum2 = {0};\n        c[0] = vec_and(c[1], lowMask);\n        c[1] = vec_sr(c[1], v4);\n        c[0] = vec_sub(c[0], v8);\n        c[1] = vec_sub(c[1], v8);\n        vsum = vec_sum4s(c[0], vsum);\n        vsum2 = vec_sum4s(c[1], vsum2);\n        vsum = vec_add(vsum, vsum2);\n        *(ca) = vsum[0] + vsum[1] + vsum[2] + vsum[3];\n    }\n\n    template <typename V1, typename V2>\n    inline void vector_permute_store(V2 & s1, V2 & s2, V2 & s3, V2 & s4, V1 * vecOffset, bool flip) {\n        vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n        vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n        vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};\n        vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};\n        V2 t1, t2, t3, t4, t5, t6, t7, t8;\n        vector unsigned char xor_vector;\n        uint8_t flip_vec = 0x80;\n        xor_vector = vec_splats(flip_vec);\n        t1 = vec_perm(s1, s2, swiz1);\n        t2 = vec_perm(s1, s2, swiz2);\n        t3 = vec_perm(s3, s4, swiz1);\n        t4 = vec_perm(s3, s4, swiz2);\n        t5 = vec_perm(t1, t3, swiz3);\n        t6 = vec_perm(t1, t3, swiz4);\n        t7 = vec_perm(t2, t4, swiz3);\n        t8 = vec_perm(t2, t4, swiz4);\n        if (flip == true) {\n            t5 = vec_xor(t5, xor_vector);\n            t6 = vec_xor(t6, xor_vector);\n            t7 = vec_xor(t7, xor_vector);\n            t8 = vec_xor(t8, xor_vector);\n        }\n        vec_xst(t5, 0, vecOffset);\n        vec_xst(t6, 0, vecOffset + 16);\n        vec_xst(t7, 0, vecOffset + 32);\n        vec_xst(t8, 0, vecOffset + 48);\n    }\n\n    inline void unpack_q4_to_q8(vector signed char packed, vector signed char & lo, vector signed char & hi) {\n        const vector signed char lowMask = vec_splats((signed char)0x0F);\n        const vector signed char v8      = vec_splats((signed char)0x08);\n        const vector unsigned char v4    = vec_splats((unsigned char)4);\n        lo = vec_and(packed, lowMask);\n        hi = vec_sr(packed, v4);\n        lo = vec_sub(lo, v8);\n        hi = vec_sub(hi, v8);\n    }\n\n    inline void vector_permute_store_fp16(vec_t * c, unsigned char * vecOffset) {\n        vec_t t[8], s[8];\n        vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n        vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n        vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n        vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n        for (int i = 0; i < 4; i += 2) {\n            t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);\n            t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);\n        }\n        for (int i = 4; i < 8; i += 2) {\n            t[i + 0] = vec_perm(c[i + 0], c[i + 1], swiz1);\n            t[i + 1] = vec_perm(c[i + 0], c[i + 1], swiz2);\n        }\n        s[0] = vec_perm(t[0], t[2], swiz3);\n        s[1] = vec_perm(t[0], t[2], swiz4);\n        s[2] = vec_perm(t[1], t[3], swiz3);\n        s[3] = vec_perm(t[1], t[3], swiz4);\n        s[4] = vec_perm(t[4], t[6], swiz3);\n        s[5] = vec_perm(t[4], t[6], swiz4);\n        s[6] = vec_perm(t[5], t[7], swiz3);\n        s[7] = vec_perm(t[5], t[7], swiz4);\n        for (int i = 0; i < 8; ++i) {\n            vec_xst(s[i], 0, (vec_t *)(vecOffset + i * 16));\n        }\n    }\n\n    static inline void convert_and_scale_q8(vector signed char raw, vector float v_scale, vector unsigned short & out_hi, vector unsigned short & out_lo) {\n        vector signed short i16_hi = vec_unpackh(raw);\n        vector signed short i16_lo = vec_unpackl(raw);\n\n        vector float f_hi_h = vec_ctf(vec_unpackh(i16_hi), 0);\n        vector float f_hi_l = vec_ctf(vec_unpackl(i16_hi), 0);\n        vector float f_lo_h = vec_ctf(vec_unpackh(i16_lo), 0);\n        vector float f_lo_l = vec_ctf(vec_unpackl(i16_lo), 0);\n        out_hi = vec_pack_to_short_fp32(vec_mul(f_hi_h, v_scale), vec_mul(f_hi_l, v_scale));\n        out_lo = vec_pack_to_short_fp32(vec_mul(f_lo_h, v_scale), vec_mul(f_lo_l, v_scale));\n    }\n\n    void packNormal_q4_fp16(const block_q4_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n        unsigned char * vecOffset = vec;\n        for (int i = 0; i < rows; i += 8) {\n            const block_q4_0 * rows_base[8];\n            for (int r = 0; r < 8; r++) {\n                rows_base[r] = a + (i + r) * lda;\n            }\n            for (int blk = 0; blk < blocks; blk++) {\n                vector unsigned short hp_res[8][4];\n                for (int r = 0; r < 8; r++) {\n                    const block_q4_0 * current_blk = rows_base[r] + blk;\n                    vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(current_blk->d));\n                    vector signed char v_qs = vec_xl(0, (const vector signed char *)current_blk->qs);\n                    vector signed char c1, c2;\n                    unpack_q4_to_q8(v_qs, c1, c2);\n                    convert_and_scale_q8(c1, v_scale, hp_res[r][0], hp_res[r][1]);\n                    convert_and_scale_q8(c2, v_scale, hp_res[r][2], hp_res[r][3]);\n                }\n                for (int c = 0; c < 4; c++) {\n                    vector unsigned char c_arr[8];\n                    for (int r = 0; r < 8; r++) {\n                        c_arr[r] = (vector unsigned char)hp_res[r][c];\n                    }\n                    vector_permute_store_fp16((vec_t *)c_arr, vecOffset);\n                    vecOffset += 128;\n                }\n            }\n        }\n    }\n\n    template <int chunk_size>\n    static inline void pack_q8_block(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n        unsigned char * vecOffset = vec;\n        const vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};\n        const vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};\n        const vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};\n        const vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};\n\n        for (int i = 0; i < rows; i += chunk_size) {\n            const block_q8_0 * rows_base[chunk_size];\n            for (int r = 0; r < chunk_size; r++) {\n                rows_base[r] = a + (i + r) * lda;\n            }\n            for (int blk = 0; blk < blocks; blk++) {\n                vector unsigned short hp_res[chunk_size][4];\n                for (int r = 0; r < chunk_size; r++) {\n                    const block_q8_0 * b = rows_base[r] + blk;\n                    vector float v_scale = vec_extract_fp32_from_shorth(vec_splats(b->d));\n                    vector signed char c[2];\n                    __vector_pair pair = __builtin_vsx_lxvp(0, (__vector_pair *)b->qs);\n                    __builtin_vsx_disassemble_pair(c, & pair);\n                    convert_and_scale_q8(c[0], v_scale, hp_res[r][0], hp_res[r][1]);\n                    convert_and_scale_q8(c[1], v_scale, hp_res[r][2], hp_res[r][3]);\n                }\n                for (int col = 0; col < 4; col++) {\n                    if constexpr (chunk_size == 8) {\n                        vec_t t[8];\n                        t[0] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);\n                        t[1] = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);\n                        t[2] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);\n                        t[3] = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);\n                        t[4] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz1);\n                        t[5] = vec_perm((vec_t)hp_res[4][col], (vec_t)hp_res[5][col], swiz2);\n                        t[6] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz1);\n                        t[7] = vec_perm((vec_t)hp_res[6][col], (vec_t)hp_res[7][col], swiz2);\n\n                        vec_xst(vec_perm(t[0], t[2], swiz3), 0, (vec_t *)(vecOffset + 0));\n                        vec_xst(vec_perm(t[0], t[2], swiz4), 0, (vec_t *)(vecOffset + 16));\n                        vec_xst(vec_perm(t[1], t[3], swiz3), 0, (vec_t *)(vecOffset + 32));\n                        vec_xst(vec_perm(t[1], t[3], swiz4), 0, (vec_t *)(vecOffset + 48));\n                        vec_xst(vec_perm(t[4], t[6], swiz3), 0, (vec_t *)(vecOffset + 64));\n                        vec_xst(vec_perm(t[4], t[6], swiz4), 0, (vec_t *)(vecOffset + 80));\n                        vec_xst(vec_perm(t[5], t[7], swiz3), 0, (vec_t *)(vecOffset + 96));\n                        vec_xst(vec_perm(t[5], t[7], swiz4), 0, (vec_t *)(vecOffset + 112));\n                        vecOffset += 128;\n                    } else {\n                        vec_t t0 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz1);\n                        vec_t t1 = vec_perm((vec_t)hp_res[0][col], (vec_t)hp_res[1][col], swiz2);\n                        vec_t t2 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz1);\n                        vec_t t3 = vec_perm((vec_t)hp_res[2][col], (vec_t)hp_res[3][col], swiz2);\n\n                        vec_xst(vec_perm(t0, t2, swiz3), 0, (vec_t *)(vecOffset + 0));\n                        vec_xst(vec_perm(t0, t2, swiz4), 0, (vec_t *)(vecOffset + 16));\n                        vec_xst(vec_perm(t1, t3, swiz3), 0, (vec_t *)(vecOffset + 32));\n                        vec_xst(vec_perm(t1, t3, swiz4), 0, (vec_t *)(vecOffset + 48));\n                        vecOffset += 64;\n                    }\n                }\n            }\n        }\n    }\n\n    void packNormal_q8_fp16(const block_q8_0 * a, int64_t lda, int rows, int blocks, unsigned char * vec) {\n        if (rows == 4) {\n            pack_q8_block<4>(a, lda, rows, blocks, vec);\n        } else {\n            pack_q8_block<8>(a, lda, rows, blocks, vec);\n        }\n    }\n\n    template<int size>\n    void packNormalInt4(const TA * a, int64_t lda, int rows, int cols, int8_t * vec, std::array<int, size> & comparray) {\n        int64_t i, j;\n        TA * aoffset = NULL;\n        int8_t * vecOffset = NULL;\n        TA * aoffset1 = NULL, * aoffset2 = NULL, * aoffset3 = NULL, * aoffset4 = NULL;\n        TA * aoffset5 = NULL, * aoffset6 = NULL, * aoffset7 = NULL, * aoffset8 = NULL;\n        vector signed char c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};\n        vector signed char c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};\n        aoffset = const_cast<TA *>(a);\n        vecOffset = vec;\n        j = (rows >> 3);\n        if (j > 0) {\n            do {\n                aoffset1 = aoffset;\n                aoffset2 = aoffset1 + lda;\n                aoffset3 = aoffset2 + lda;\n                aoffset4 = aoffset3 + lda;\n                aoffset5 = aoffset4 + lda;\n                aoffset6 = aoffset5 + lda;\n                aoffset7 = aoffset6 + lda;\n                aoffset8 = aoffset7 + lda;\n                aoffset += 8 * lda;\n                i = (cols >> 2);\n                if (i > 0) {\n                    do {\n                        c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n                        c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n                        c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n                        c4[1] = vec_xl(0, (const vector signed char *)aoffset4->qs);\n                        c5[1] = vec_xl(0, (const vector signed char *)aoffset5->qs);\n                        c6[1] = vec_xl(0, (const vector signed char *)aoffset6->qs);\n                        c7[1] = vec_xl(0, (const vector signed char *)aoffset7->qs);\n                        c8[1] = vec_xl(0, (const vector signed char *)aoffset8->qs);\n\n                        process_q4_elements(c1, & comparray[0]);\n                        process_q4_elements(c2, & comparray[1]);\n                        process_q4_elements(c3, & comparray[2]);\n                        process_q4_elements(c4, & comparray[3]);\n                        process_q4_elements(c5, & comparray[4]);\n                        process_q4_elements(c6, & comparray[5]);\n                        process_q4_elements(c7, & comparray[6]);\n                        process_q4_elements(c8, & comparray[7]);\n                        vector_permute_store<int8_t, vector signed char>(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n                        vector_permute_store<int8_t, vector signed char>(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n                        vector_permute_store<int8_t, vector signed char>(c5[0], c6[0], c7[0], c8[0], vecOffset + 128, false);\n                        vector_permute_store<int8_t, vector signed char>(c5[1], c6[1], c7[1], c8[1], vecOffset + 192, false);\n                        aoffset1 += lda;\n                        aoffset2 += lda;\n                        aoffset3 += lda;\n                        aoffset4 += lda;\n                        aoffset5 += lda;\n                        aoffset6 += lda;\n                        aoffset7 += lda;\n                        aoffset8 += lda;\n                        vecOffset += 256;\n                        i--;\n                    } while (i > 0);\n                }\n                j--;\n            } while (j > 0);\n        }\n\n        if (rows & 4) {\n            aoffset1 = aoffset;\n            aoffset2 = aoffset1 + lda;\n            aoffset3 = aoffset2 + lda;\n            aoffset4 = aoffset3 + lda;\n            aoffset += 4 * lda;\n            i = (cols >> 2);\n            if (i > 0) {\n                do {\n                    c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n                    c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n                    c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n                    c4[1] = vec_xl(0, (const vector signed char *)aoffset4->qs);\n\n                    process_q4_elements(c1, & comparray[0]);\n                    process_q4_elements(c2, & comparray[1]);\n                    process_q4_elements(c3, & comparray[2]);\n                    process_q4_elements(c4, & comparray[3]);\n                    vector_permute_store<int8_t, vector signed char>(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n                    vector_permute_store<int8_t, vector signed char>(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n                    aoffset1 += lda;\n                    aoffset2 += lda;\n                    aoffset3 += lda;\n                    aoffset4 += lda;\n                    vecOffset += 128;\n                    i--;\n                } while (i > 0);\n            }\n        }\n\n        if (rows & 3) {\n            aoffset1 = aoffset;\n            aoffset2 = aoffset1 + lda;\n            aoffset3 = aoffset2 + lda;\n            i = (cols >> 2);\n            if (i > 0) {\n                do {\n                    switch(rows) {\n                        case 3: c3[1] = vec_xl(0, (const vector signed char *)aoffset3->qs);\n                        case 2: c2[1] = vec_xl(0, (const vector signed char *)aoffset2->qs);\n                        case 1: c1[1] = vec_xl(0, (const vector signed char *)aoffset1->qs);\n                            break;\n                    }\n                    process_q4_elements(c1, & comparray[0]);\n                    process_q4_elements(c2, & comparray[1]);\n                    process_q4_elements(c3, & comparray[2]);\n                    process_q4_elements(c4, & comparray[3]);\n                    vector_permute_store<int8_t, vector signed char>(c1[0], c2[0], c3[0], c4[0], vecOffset, false);\n                    vector_permute_store<int8_t, vector signed char>(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false);\n                    aoffset1 += lda;\n                    aoffset2 += lda;\n                    aoffset3 += lda;\n                    vecOffset += 128;\n                    i--;\n                } while(i > 0);\n            }\n        }\n    }\n\n    template<typename VA, typename VB>\n    void packNormal(const block_q8_0 * a, int64_t lda, int rows, int cols, VA * vec, bool flip) {\n        int64_t i, j;\n        block_q8_0 * aoffset = NULL;\n        VA * vecOffset = NULL;\n        block_q8_0 * aoffsets[8];\n        __vector_pair arr[8];\n        VB c[8][2] = {0};\n        VB c1[8] = {0}; VB c2[8] = {0};\n        aoffset = const_cast<block_q8_0 *>(a);\n        vecOffset = vec;\n        j = (rows >> 3);\n        if (j > 0) {\n            do {\n                aoffsets[0] = aoffset;\n                for (int it = 1; it < 8; it++)\n                    aoffsets[it] = aoffsets[it - 1] + lda;\n                aoffset += 8 * lda;\n\n                i = (cols >> 3);\n                if (i > 0) {\n                do {\n                    for (int it = 0; it < 8; it++) {\n                        arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);\n                        __builtin_vsx_disassemble_pair(c[it], & arr[it]);\n                        c1[it] = c[it][0];\n                        c2[it] = c[it][1];\n                    }\n                    vector_permute_store<VA, VB>(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n                    vector_permute_store<VA, VB>(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n                    vector_permute_store<VA, VB>(c1[4], c1[5], c1[6], c1[7], vecOffset + 128, flip);\n                    vector_permute_store<VA, VB>(c2[4], c2[5], c2[6], c2[7], vecOffset + 192, flip);\n                    for (int it = 0; it < 8; it++)\n                        aoffsets[it] += lda;\n                    vecOffset += 256;\n                    i--;\n               } while(i > 0);\n            }\n            j--;\n        } while(j > 0);\n    }\n    if (rows & 4) {\n            aoffsets[0]  = aoffset;\n            for (int it = 1; it < 4; it++ )\n                aoffsets[it] = aoffsets[it-1] + lda;\n            aoffset += 4 * lda;\n        i = (cols >> 3);\n            if (i > 0) {\n               do {\n                    for (int it = 0; it < 4; it++) {\n                        arr[it] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[it]->qs);\n                        __builtin_vsx_disassemble_pair(c[it], & arr[it]);\n                        c1[it] = c[it][0];\n                        c2[it] = c[it][1];\n                    }\n                    vector_permute_store<VA, VB>(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n                    vector_permute_store<VA, VB>(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n                    for (int it = 0; it < 4; it++) {\n                        aoffsets[it] += lda;\n                    }\n                    vecOffset += 128;\n                    i--;\n               } while(i > 0);\n            }\n        }\n\n        if (rows & 3) {\n            aoffsets[0]  = aoffset;\n            for (int it = 1; it < 3; it++ )\n                aoffsets[it] = aoffsets[it - 1] + lda;\n            i = (cols >> 3);\n            if (i > 0) {\n                do {\n                    switch(rows) {\n                        case 3: arr[2] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[2]->qs);\n                                __builtin_vsx_disassemble_pair(c[2], & arr[2]);\n                                c1[2] = c[2][0]; c2[2] = c[2][1];\n                        case 2: arr[1] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[1]->qs);\n                                __builtin_vsx_disassemble_pair(c[1], & arr[1]);\n                                c1[1] = c[1][0]; c2[1] = c[1][1];\n                        case 1: arr[0] = __builtin_vsx_lxvp(0, (__vector_pair *)aoffsets[0]->qs);\n                                __builtin_vsx_disassemble_pair(c[0], & arr[0]);\n                                c1[0] = c[0][0]; c2[0] = c[0][1];\n                                break;\n                    }\n                    vector_permute_store<VA, VB>(c1[0], c1[1], c1[2], c1[3], vecOffset, flip);\n                    vector_permute_store<VA, VB>(c2[0], c2[1], c2[2], c2[3], vecOffset + 64, flip);\n                    for (int it = 0; it < 3; it++)\n                         aoffsets[it] += lda;\n                    vecOffset += 128;\n                    i--;\n               } while(i > 0);\n            }\n        }\n    }\n\n    void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int m_rem = MIN(m - m0, 16);\n        int n_rem = MIN(n - n0, 16);\n\n        int mc = 0, nc = 0;\n\n        if (m_rem >= 8 && n_rem >= 8) {\n           mc = 8;\n           nc = 8;\n           gemm<8, 8>(m0, m, n0, n);\n        } else if (m_rem >= 4 && n_rem >= 8) {\n            mc = 4;\n            nc = 8;\n            gemm<4, 8>(m0, m, n0, n);\n        } else if (m_rem >= 8 && n_rem >= 4) {\n            mc = 8;\n            nc = 4;\n            gemm<8, 4>(m0, m, n0, n);\n        } else if (m_rem >= 4 && n_rem >= 4) {\n            mc = 4;\n            nc = 4;\n            gemm_small(m0, m, n0, n, mc, nc);\n        } else {\n            mc = (m_rem >= 4) ? 4 : m_rem;\n            nc = (n_rem >= 4) ? 4 : n_rem;\n            if (mc == 0 || nc == 0)\n               return;\n            gemm_small(m0, m, n0, n, mc, nc);\n        }\n\n        int64_t mp = m0 + ((m - m0) / mc) * mc;\n        int64_t np = n0 + ((n - n0) / nc) * nc;\n        mnpack(mp, m, n0, np);\n        mnpack(m0, m, np, n);\n    }\n\n\n    void KERNEL_4x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[8], vec_B[16] = {0};\n        acc_t acc_0, acc_1;\n        std::array<int, 4> comparray {};\n        vector float fin_res[8] = {0};\n        vector float vs[8] = {0};\n        bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;\n        for (int l = 0; l < k; l++) {\n            __builtin_mma_xxsetaccz(& acc_0);\n            __builtin_mma_xxsetaccz(& acc_1);\n            if (std::is_same_v<TA, block_q4_0>) {\n               packNormalInt4<4>((A + (ii * lda) + l), lda, 4, 4, (int8_t *)vec_A, comparray);\n            } else {\n               packNormal<int8_t, vector signed char>((const block_q8_0 *)(A + (ii * lda) + l), lda, 4, 8, (int8_t *)vec_A, false);\n            }\n            packNormal<uint8_t, vector unsigned char>((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);\n            for(int x = 0; x < 8; x++) {\n                __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n                __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x], vec_B[x+8]);\n            }\n            for (int I = 0; I<4; I++) {\n                for (int J = 0; J<4; J++) {\n                    *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n                    *((float *)& vs[I + 4] + J) = (unhalf((A +((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));\n                }\n            }\n            if (!isAblock_q4) {\n                auto aoffset = A + (ii * lda) + l;\n                for (int i = 0; i < 4; i++) {\n                    comparray[i] = 0;\n                    int ca = 0;\n                    auto *at = aoffset->qs;\n                    for (int j = 0; j < 32; j++)\n                        ca += (int)*at++;\n                    comparray[i] = ca;\n                    aoffset += lda;\n                }\n            }\n            compute(& acc_0, 0, 0, comparray, vs, fin_res);\n            compute(& acc_1, 0, 4, comparray, vs, fin_res);\n        }\n        save_res(ii, jj, 0, fin_res);\n        save_res(ii, jj + 4, 4, fin_res);\n    }\n\n    void KERNEL_8x4(int64_t ii, int64_t jj) {\n        vec_t vec_A[16], vec_B[8] = {0};\n        acc_t acc_0, acc_1;\n        std::array<int, 8> comparray {};\n        vector float fin_res[8] = {0};\n        vector float vs[8] = {0};\n        bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;\n        for (int l = 0; l < k; l++) {\n            __builtin_mma_xxsetaccz(& acc_0);\n            __builtin_mma_xxsetaccz(& acc_1);\n            if (std::is_same_v<TA, block_q4_0>) {\n               packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);\n            } else {\n               packNormal<int8_t, vector signed char>((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);\n            }\n            packNormal<uint8_t, vector unsigned char>((B + (jj * ldb) + l), ldb, 4, 8, (uint8_t *)vec_B, true);\n            for(int x = 0; x < 8; x++) {\n                __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n                __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);\n            }\n            for (int I = 0; I < 8; I++) {\n                for (int J = 0; J < 4; J++) {\n                    *((float *)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n                }\n            }\n            if (!isAblock_q4) {\n                auto aoffset = A + (ii * lda) + l;\n                for (int i = 0; i < 8; i++) {\n                    comparray[i] = 0;\n                    int ca = 0;\n                    auto *at = aoffset->qs;\n                    for (int j = 0; j < 32; j++)\n                        ca += (int)*at++;\n                    comparray[i] = ca;\n                    aoffset += lda;\n                }\n            }\n            compute(& acc_0, 0, 0, comparray, vs, fin_res);\n            compute(& acc_1, 4, 4, comparray, vs, fin_res);\n        }\n        save_res(ii, jj, 0, fin_res);\n        save_res(ii + 4, jj, 4, fin_res);\n    }\n\n    void KERNEL_8x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[16], vec_B[16] = {0};\n        acc_t acc_0, acc_1, acc_2, acc_3;\n        acc_t acc_4, acc_5, acc_6, acc_7;\n        std::array<int, 8> comparray {};\n        vector float fin_res[16] = {0};\n        vector float vs[16] = {0};\n        bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;\n        for (int l = 0; l < k; l++) {\n            __builtin_mma_xxsetaccz(& acc_0);\n            __builtin_mma_xxsetaccz(& acc_1);\n            __builtin_mma_xxsetaccz(& acc_2);\n            __builtin_mma_xxsetaccz(& acc_3);\n            if (std::is_same_v<TA, block_q4_0>) {\n               packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray);\n            } else {\n               packNormal<int8_t, vector signed char>((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false);\n            }\n            packNormal<uint8_t, vector unsigned char>((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true);\n            for(int x = 0; x < 8; x++) {\n                __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n                __builtin_mma_xvi8ger4pp(& acc_1, vec_A[x + 8], vec_B[x]);\n                __builtin_mma_xvi8ger4pp(& acc_2, vec_A[x], vec_B[x + 8]);\n                __builtin_mma_xvi8ger4pp(& acc_3, vec_A[x + 8], vec_B[x + 8]);\n            }\n            for (int I = 0; I < 8 ; I++) {\n                for (int J = 0; J < 4; J++) {\n                    *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n                    *((float *)& vs[I + 8] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d));\n                }\n            }\n            if (!isAblock_q4) {\n                auto aoffset = A + (ii * lda) + l;\n                for (int i = 0; i < 8; i++) {\n                    comparray[i] = 0;\n                    int ca = 0;\n                    auto *at = aoffset->qs;\n                    for (int j = 0; j < 32; j++)\n                        ca += (int)*at++;\n                    comparray[i] = ca;\n                    aoffset += lda;\n                }\n            }\n            compute(& acc_0, 0, 0, comparray, vs, fin_res);\n            compute(& acc_1, 4, 4, comparray, vs, fin_res);\n            compute(& acc_2, 0, 8, comparray, vs, fin_res);\n            compute(& acc_3, 4, 12, comparray, vs, fin_res);\n        }\n        save_res(ii, jj, 0, fin_res);\n        save_res(ii + 4, jj, 4, fin_res);\n        save_res(ii, jj + 4, 8, fin_res);\n        save_res(ii + 4, jj + 4, 12, fin_res);\n    }\n\n    void KERNEL_Q0(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, int64_t l, vec_t * vec_A, vec_t * vec_B) {\n        acc_t acc[8];\n        for (int i = 0; i < mc ; i += 16) {\n            for (int j = 0; j < nc; j += 8) {\n                int A0_base = (i / 16) * (2 * 32 * kc);\n                int B0_base = (j / 8) * (32 * kc);\n                for (int x = 0; x < 8; x++) {\n                     __builtin_mma_xxsetaccz(&acc[x]);\n                }\n                for (int64_t kk = 0; kk < kc; kk++) {\n                    int A0_block_idx = A0_base + kk * 32;\n                    int B0_block_idx = B0_base + kk * 32;\n                    int A1_block_idx = A0_block_idx + 32 * kc;\n                    int B1_block_idx = B0_block_idx + 32 * kc;\n                    vec_t * A0_block = & vec_A[A0_block_idx];\n                    vec_t * B0_block = & vec_B[B0_block_idx];\n                    vec_t * A1_block = & vec_A[A1_block_idx];\n                    for (int it = 0; it < 4; it++) {\n                        for (int x = 0; x < 4; x++) {\n                            __builtin_mma_xvf16ger2pp(& acc[0], A0_block[8 * it + x], B0_block[8 * it + x]);\n                            __builtin_mma_xvf16ger2pp(& acc[1], A0_block[8 * it + x], B0_block[8 * it + x + 4]);\n                            __builtin_mma_xvf16ger2pp(& acc[2], A0_block[8 * it + x + 4], B0_block[8 * it + x]);\n                            __builtin_mma_xvf16ger2pp(& acc[3], A0_block[8 * it + x + 4], B0_block[8 * it + x + 4]);\n                            __builtin_mma_xvf16ger2pp(& acc[4], A1_block[8 * it + x], B0_block[8 * it + x]);\n                            __builtin_mma_xvf16ger2pp(& acc[5], A1_block[8 * it + x], B0_block[8 * it+ x + 4]);\n                            __builtin_mma_xvf16ger2pp(& acc[6], A1_block[8 * it + x + 4], B0_block[8 * it + x]);\n                            __builtin_mma_xvf16ger2pp(& acc[7], A1_block[8 * it + x + 4], B0_block[8 * it + x + 4]);\n                        }\n                    }\n                }\n                if (l == 0) {\n                    save_acc(& acc[0], ii + i, jj + j);\n                    save_acc(& acc[1], ii + i, jj + j + 4);\n                    save_acc(& acc[2], ii + i + 4, jj + j);\n                    save_acc(& acc[3], ii + i + 4, jj + j + 4);\n                    save_acc(& acc[4], ii + i + 8, jj + j);\n                    save_acc(& acc[5], ii + i + 8, jj + j + 4);\n                    save_acc(& acc[6], ii + i + 12, jj + j);\n                    save_acc(& acc[7], ii + i + 12, jj + j + 4);\n                } else {\n                    add_save_acc(& acc[0], ii + i, jj + j);\n                    add_save_acc(& acc[1], ii + i, jj + j + 4);\n                    add_save_acc(& acc[2], ii + i + 4, jj + j);\n                    add_save_acc(& acc[3], ii + i + 4, jj + j + 4);\n                    add_save_acc(& acc[4], ii + i + 8, jj + j);\n                    add_save_acc(& acc[5], ii + i + 8, jj + j + 4);\n                    add_save_acc(& acc[6], ii + i + 12, jj + j);\n                    add_save_acc(& acc[7], ii + i + 12, jj + j + 4);\n                }\n            }\n        }\n    }\n\n    void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) {\n        vec_t A_pack[mc * kc * 4];\n        vec_t B_pack[nc * kc * 4];\n        constexpr bool is_Ablock_q4 = std::is_same_v<TA, block_q4_0>;\n        int64_t ytiles = m / mc;\n        int64_t xtiles = n / nc;\n        int64_t tiles  = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles) {\n            end = tiles;\n        }\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = (job / xtiles) * mc;\n            int64_t jj = (job % xtiles) * nc;\n            for (int64_t kk = 0; kk < k; kk += kc) {\n                if constexpr(is_Ablock_q4) {\n                    packNormal_q4_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);\n                } else {\n                    packNormal_q8_fp16(A + ii * lda + kk, lda, mc, kc, (uint8_t *)A_pack);\n                }\n                packNormal_q8_fp16(B + jj * ldb + kk, ldb, nc, kc, (uint8_t *)B_pack);\n                KERNEL_Q0(ii, jj, mc, nc, kc, kk, A_pack, B_pack);\n            }\n        }\n    }\n\n    void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        vec_t vec_A[8] = {0}, vec_B[8] = {0};\n        vector signed int vec_C[4];\n        acc_t acc_0;\n        bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;\n\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            std::array<int, 4> comparray{};\n            vector float res[4] = {0};\n            vector float fin_res[4] = {0};\n            vector float vs[4] = {0};\n            vector float CA[4] = {0};\n            __builtin_prefetch((A + (ii * lda) + 0)->qs, 0, 1); // prefetch first value\n            __builtin_prefetch((B + (jj * ldb) + 0)->qs, 0, 1); // prefetch first value\n            for (int l = 0; l < k; l++) {\n                __builtin_prefetch((A + (ii * lda) + (l + 1))->qs, 0, 1); // prefetch one loop ahead\n                __builtin_prefetch((B + (jj * ldb) + (l + 1))->qs, 0, 1); // prefetch one loop ahead\n                __builtin_mma_xxsetaccz(& acc_0);\n                if (isAblock_q4) {\n                    packNormalInt4<4>((A + (ii * lda) + l), lda, RM, 4, (int8_t *)vec_A, comparray);\n                } else {\n                    packNormal<int8_t, vector signed char>((const block_q8_0 *)(A + (ii * lda) + l), lda, RM, 8, (int8_t *)vec_A, false);\n                }\n                packNormal<uint8_t, vector unsigned char>((B + (jj * ldb) + l), ldb, RN, 8, (uint8_t *)vec_B, true);\n                for (int x = 0; x < 8; x += 4) {\n                    __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x], vec_B[x]);\n                    __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 1], vec_B[x + 1]);\n                    __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 2], vec_B[x + 2]);\n                    __builtin_mma_xvi8ger4pp(& acc_0, vec_A[x + 3], vec_B[x + 3]);\n                }\n                for (int I = 0; I < RM; I++) {\n                    for (int J = 0; J < RN; J++) {\n                        *((float*)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d));\n                    }\n                }\n                __builtin_mma_disassemble_acc(vec_C, & acc_0);\n                if (!isAblock_q4) {\n                    auto aoffset = A + (ii * lda) + l;\n                    for (int i = 0; i < RM; i++) {\n                        comparray[i] = 0;\n                        int ca = 0;\n                        auto *at = aoffset->qs;\n                        for (int j = 0; j < 32; j++)\n                            ca += (int)*at++;\n                        comparray[i] = ca;\n                        aoffset += lda;\n                    }\n                }\n                for (int i = 0; i < RM; i++) {\n                    CA[i] = vec_splats((float)(((double)comparray[i]) * -128.0));\n                    res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);\n                    fin_res[i] = vec_madd(res[i], vs[i], fin_res[i]);\n                }\n            }\n            save_res(ii, jj, 0, fin_res, RM, RN);\n        }\n    }\n\n    template<int RM, int RN>\n    inline void kernel(int64_t ii, int64_t jj) {\n        if constexpr(RM == 4 && RN == 8) {\n            KERNEL_4x8(ii,jj);\n        } else if constexpr(RM == 8 && RN == 4) {\n            KERNEL_8x4(ii,jj);\n        } else if constexpr(RM == 8 && RN == 8) {\n            KERNEL_8x8(ii,jj);\n        } else {\n            assert(false && \"RN/RM values not supported\");\n        }\n    }\n\n    template <int RM, int RN>\n    NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            kernel<RM, RN>(ii, jj);\n        }\n    }\n    const TA * const A;\n    const block_q8_0 * const B;\n    float * C;\n    const int64_t k;\n    int64_t kc;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n    const int ith;\n    const int nth;\n};\n\nclass tinyBLAS_PPC {\n  public:\n    tinyBLAS_PPC(int64_t k,\n                const float * A, int64_t lda,\n                const float * B, int64_t ldb,\n                float * C, int64_t ldc,\n                int ith, int nth)\n        : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {\n    }\n\n    void matmul(int64_t m, int64_t n) {\n        int64_t mc = 256; int64_t nc = 256; int64_t kc = 256;\n        if (m % mc == 0 && n % nc == 0 && k % kc == 0) {\n            matmul_tiled(m, n, mc, nc, kc);\n        } else {\n            mnpack(0, m, 0, n);\n        }\n    }\n\n  private:\n\n    inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n        vec_t vec_C[4];\n        __builtin_mma_disassemble_acc(vec_C, ACC);\n        for (int I = 0; I < 4; I++) {\n            for (int J = 0; J < 4; J++) {\n                *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J);\n            }\n        }\n    }\n\n    inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) {\n        vec_t vec_C[4];\n        __builtin_mma_disassemble_acc(vec_C, ACC);\n        for (int I = 0; I < 4; I++) {\n            for (int J = 0; J < 4; J++) {\n                float * c_ptr = (float *)(C+ii+((jj+J)*ldc)+I);\n                *c_ptr += *((float *)&vec_C[I]+J);\n            }\n        }\n    }\n\n    inline void vector_permute_store_4(vector float * src, float * vecOffset) {\n        vector float t1, t2, t3, t4, t5, t6, t7, t8;\n        t1 = vec_mergeh(src[0], src[1]);\n        t2 = vec_mergeh(src[2], src[3]);\n        t3 = vec_mergel(src[0], src[1]);\n        t4 = vec_mergel(src[2], src[3]);\n\n        t5 = vec_xxpermdi(t1, t2, 0);\n        t6 = vec_xxpermdi(t1, t2, 3);\n        t7 = vec_xxpermdi(t3, t4, 0);\n        t8 = vec_xxpermdi(t3, t4, 3);\n\n        vec_xst(t5, 0, vecOffset);\n        vec_xst(t6, 0, vecOffset + 4);\n        vec_xst(t7, 0, vecOffset + 8);\n        vec_xst(t8, 0, vecOffset + 12);\n    }\n\n    inline void vector_permute_store_8(vector float * src, float * vecOffset) {\n        vector float t1, t2, t3, t4, t5, t6, t7, t8;\n        t1 = vec_mergeh(src[0], src[1]);\n        t2 = vec_mergeh(src[2], src[3]);\n        t3 = vec_mergeh(src[4], src[5]);\n        t4 = vec_mergeh(src[6], src[7]);\n\n        t5 = vec_xxpermdi(t1, t2, 0);\n        t6 = vec_xxpermdi(t3, t4, 0);\n        t7 = vec_xxpermdi(t1, t2, 3);\n        t8 = vec_xxpermdi(t3, t4, 3);\n\n        vec_xst(t5, 0, vecOffset);\n        vec_xst(t6, 0, vecOffset + 4);\n        vec_xst(t7, 0, vecOffset + 8);\n        vec_xst(t8, 0, vecOffset + 12);\n\n        t1 = vec_mergel(src[0], src[1]);\n        t2 = vec_mergel(src[2], src[3]);\n        t3 = vec_mergel(src[4], src[5]);\n        t4 = vec_mergel(src[6], src[7]);\n\n        t5 = vec_xxpermdi(t1, t2, 0);\n        t6 = vec_xxpermdi(t3, t4, 0);\n        t7 = vec_xxpermdi(t1, t2, 3);\n        t8 = vec_xxpermdi(t3, t4, 3);\n\n        vec_xst(t5, 0, vecOffset + 16);\n        vec_xst(t6, 0, vecOffset + 20);\n        vec_xst(t7, 0, vecOffset + 24);\n        vec_xst(t8, 0, vecOffset + 28);\n    }\n\n    void packTranspose(const float * a, int64_t lda, int rows, int cols, float * vec) {\n        int64_t i, j;\n        float * aoffsets[8];\n        float * aoffset = NULL, * boffset = NULL;\n        __vector_pair arr[8];\n        vector float c[8][2] = {0};\n        vector float c1[8] = {0};\n        vector float c2[8] = {0};\n        aoffset = const_cast<float *>(a);\n        boffset = vec;\n        j = (rows >> 3);\n        if (j > 0) {\n            do {\n                aoffsets[0] = aoffset;\n                for (int it = 1; it < 8; it++)\n                    aoffsets[it] = aoffsets[it-1] + lda;\n                aoffset += 8 * lda;\n                i = (cols >> 3);\n                if (i > 0) {\n                    do {\n                        for (int it = 0; it < 8; it++) {\n                            arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]);\n                            __builtin_vsx_disassemble_pair(c[it], &arr[it]);\n                            c1[it] = c[it][0];\n                            c2[it] = c[it][1];\n                        }\n\n                        vector_permute_store_8(c1, boffset);\n                        vector_permute_store_8(c2, boffset + 32);\n                        boffset += 64;\n                        i--;\n                        if (i > 0) {\n                           for (int it = 0; it < 8; it++) {\n                               aoffsets[it] = aoffsets[it] + 8;\n                           }\n                        }\n                    } while(i > 0);\n                }\n                if (cols & 4) {\n                    for (int it = 0; it < 8 ; it++)\n                        c1[it] = vec_xl(0, aoffsets[it]);\n                    vector_permute_store_8(c1, boffset);\n                }\n            j--;\n            } while(j > 0);\n        }\n\n        if (rows & 4) {\n            aoffsets[0] = aoffset;\n            for (int it = 1; it < 4; it++)\n                aoffsets[it] = aoffsets[it-1] + lda;\n            aoffset += 4 * lda;\n            i = (cols >> 3);\n            if (i > 0) {\n                do {\n                    for (int it = 0; it < 4; it++) {\n                        arr[it] = __builtin_vsx_lxvp(0, (__vector_pair*)aoffsets[it]);\n                        __builtin_vsx_disassemble_pair(c[it], &arr[it]);\n                        c1[it] = c[it][0];\n                        c2[it] = c[it][1];\n                    }\n                    vector_permute_store_4(c1, boffset);\n                    vector_permute_store_4(c2, boffset + 16);\n                    for (int it = 0; it < 4; it++)\n                        aoffsets[it] += 8 * lda;\n                    boffset += 32;\n                    i--;\n                } while(i > 0);\n            }\n\n            if (cols & 4) {\n               for (int it = 0; it < 4; it++)\n                   c1[it] = vec_xl(0, aoffsets[it]);\n                vector_permute_store_4(c1, boffset);\n            }\n        }\n        if (rows & 3) {\n            aoffsets[0] = aoffset;\n            for (int it = 1; it < 3; it++)\n                aoffsets[it] = aoffsets[it-1] + lda;\n            if (cols & 4) {\n                for (int it = 0; it < 3; it++)\n                    c1[it] = vec_xl(0, aoffsets[it]);\n                vector_permute_store_4(c1, boffset);\n            }\n        }\n    }\n\n    void KERNEL_4x4(int64_t ii, int64_t jj) {\n        vec_t vec_A[4], vec_B[4], vec_C[4];\n        acc_t acc_0;\n        __builtin_mma_xxsetaccz(&acc_0);\n        for (int l = 0; l < k; l += 4) {\n            packTranspose(A + (ii * lda) + l, lda, 4, 4, (float *)vec_A);\n            packTranspose(B + (jj * ldb) + l, ldb, 4, 4, (float *)vec_B);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);\n        }\n        save_acc(&acc_0, ii, jj);\n    }\n\n    void KERNEL_4x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[4], vec_B[8], vec_C[4];\n        acc_t acc_0, acc_1;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        for (int64_t l = 0; l < k; l += 4) {\n            packTranspose(A + (ii * lda) + l, lda, 4, 4, (float *)vec_A);\n            packTranspose(B + (jj * ldb) + l, ldb, 8, 4, (float *)vec_B);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], (vec_t)vec_B[0]);\n            __builtin_mma_xvf32gerpp(&acc_1, vec_A[0], (vec_t)vec_B[1]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], (vec_t)vec_B[2]);\n            __builtin_mma_xvf32gerpp(&acc_1, vec_A[1], (vec_t)vec_B[3]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], (vec_t)vec_B[4]);\n            __builtin_mma_xvf32gerpp(&acc_1, vec_A[2], (vec_t)vec_B[5]);\n            __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], (vec_t)vec_B[6]);\n            __builtin_mma_xvf32gerpp(&acc_1, vec_A[3], (vec_t)vec_B[7]);\n        }\n        save_acc(&acc_0, ii, jj);\n        save_acc(&acc_1, ii, jj + 4);\n    }\n\n    void KERNEL_8x4(int64_t ii, int64_t jj) {\n        vec_t vec_A[8], vec_B[4], vec_C[4];\n        acc_t acc_0, acc_1;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        for (int64_t l = 0; l < k; l += 4) {\n            packTranspose(A + (ii * lda) + l, lda, 8, 4, (float *)vec_A);\n            packTranspose(B + (jj * ldb) + l, ldb, 4, 4, (float *)vec_B);\n            __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[0], vec_B[0]);\n            __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[1], vec_B[0]);\n            __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[2], vec_B[1]);\n            __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[3], vec_B[1]);\n            __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[4], vec_B[2]);\n            __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[5], vec_B[2]);\n            __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[6], vec_B[3]);\n            __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[7], vec_B[3]);\n        }\n        save_acc(&acc_0, ii, jj);\n        save_acc(&acc_1, ii + 4, jj);\n    }\n\n    void KERNEL_8x8(int64_t ii, int64_t jj) {\n        vec_t vec_A[16], vec_B[16], vec_C[4];\n        acc_t acc_0, acc_1, acc_2, acc_3;\n        __builtin_mma_xxsetaccz(&acc_0);\n        __builtin_mma_xxsetaccz(&acc_1);\n        __builtin_mma_xxsetaccz(&acc_2);\n        __builtin_mma_xxsetaccz(&acc_3);\n        for (int l = 0; l < k; l+=8) {\n            packTranspose(A + (ii * lda) + l, lda, 8, 8, (float *)vec_A);\n            packTranspose(B + (jj * ldb) + l, ldb, 8, 8, (float *)vec_B);\n            for(int x = 0; x < 16; x+=2) {\n                __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[x], vec_B[x]);\n                __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[x], vec_B[x + 1]);\n                __builtin_mma_xvf32gerpp(&acc_2, (vec_t)vec_A[x + 1], vec_B[x]);\n                __builtin_mma_xvf32gerpp(&acc_3, (vec_t)vec_A[x + 1], vec_B[x + 1]);\n            }\n        }\n        save_acc(&acc_0, ii, jj);\n        save_acc(&acc_1, ii, jj + 4);\n        save_acc(&acc_2, ii + 4, jj);\n        save_acc(&acc_3, ii + 4, jj + 4);\n    }\n\n    inline void MMA_16x8(vec_t * vec_A0, vec_t * vec_A1, vec_t * vec_B, acc_t * acc) {\n        for (int x = 0; x < 16; x += 2) {\n            __builtin_mma_xvf32gerpp(&acc[0], vec_A0[x + 0], vec_B[x]);\n            __builtin_mma_xvf32gerpp(&acc[1], vec_A0[x + 0], vec_B[x + 1]);\n            __builtin_mma_xvf32gerpp(&acc[2], vec_A0[x + 1], vec_B[x]);\n            __builtin_mma_xvf32gerpp(&acc[3], vec_A0[x + 1], vec_B[x + 1]);\n            __builtin_mma_xvf32gerpp(&acc[4], vec_A1[x + 0], vec_B[x]);\n            __builtin_mma_xvf32gerpp(&acc[5], vec_A1[x + 0], vec_B[x + 1]);\n            __builtin_mma_xvf32gerpp(&acc[6], vec_A1[x + 1], vec_B[x]);\n            __builtin_mma_xvf32gerpp(&acc[7], vec_A1[x + 1], vec_B[x + 1]);\n        }\n    }\n\n    void KERNEL(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, vec_t * vec_A, vec_t * vec_B, int64_t kk) {\n        for (int64_t i = 0; i < mc; i += 16) {\n            int A_base_addr = (mc / 8) * (i / 8) * 16;\n            for (int64_t j = 0; j < nc; j += 8) {\n                 int B_base_addr = (nc / 8) * (j / 8) * 16;\n                 acc_t acc[8];\n                 vec_t A0_block[16]; vec_t A1_block[16];\n                 for (int x = 0; x < 8; x++)\n                     __builtin_mma_xxsetaccz(&acc[x]);\n                 for (int64_t l = 0; l < kc; l += 8) {\n                     int A0_block_idx = A_base_addr + (l / 8) * 16;\n                     int A1_block_idx = A0_block_idx + (mc / 8) * 16;\n                     int B_block_idx = B_base_addr + (l / 8) * 16;\n                     vec_t* A0_block = &vec_A[A0_block_idx];\n                     vec_t* A1_block = &vec_A[A1_block_idx];\n                     vec_t* B_block = &vec_B[B_block_idx];\n                     MMA_16x8(A0_block, A1_block, B_block, acc);\n                 }\n                 if (kk == 0) {\n                     save_acc(&acc[0], ii + i, jj + j);\n                     save_acc(&acc[1], ii + i, jj + j + 4);\n                     save_acc(&acc[2], ii + i + 4, jj + j);\n                     save_acc(&acc[3], ii + i + 4, jj + j + 4);\n                     save_acc(&acc[4], ii + i + 8, jj + j);\n                     save_acc(&acc[5], ii + i + 8, jj + j + 4);\n                     save_acc(&acc[6], ii + i + 12, jj + j);\n                     save_acc(&acc[7], ii + i + 12, jj + j + 4);\n                 } else {\n                     add_save_acc(&acc[0], ii + i, jj + j);\n                     add_save_acc(&acc[1], ii + i, jj + j + 4);\n                     add_save_acc(&acc[2], ii + i + 4, jj + j);\n                     add_save_acc(&acc[3], ii + i + 4, jj + j + 4);\n                     add_save_acc(&acc[4], ii + i + 8, jj + j);\n                     add_save_acc(&acc[5], ii + i + 8, jj + j + 4);\n                     add_save_acc(&acc[6], ii + i + 12, jj + j);\n                     add_save_acc(&acc[7], ii + i + 12, jj + j + 4);\n                 }\n            }\n        }\n    }\n\n    void matmul_tiled(int64_t m , int64_t n, int64_t mc, int64_t nc, int64_t kc) {\n        int64_t ytiles = m / mc;\n        int64_t xtiles = n / nc;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles) {\n            end = tiles;\n        }\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = (job / xtiles) * mc;\n            int64_t jj = (job % xtiles) * nc;\n            for (int64_t kk = 0; kk < k; kk += kc) {\n                 vec_t A_pack[kc * mc / 4];\n                 vec_t B_pack[kc * nc / 4];\n                 packTranspose(A + (ii * lda) + kk, lda, kc, mc, (float *)A_pack);\n                 packTranspose(B + (jj * ldb) + kk, ldb, kc, nc, (float *)B_pack);\n                 KERNEL(ii, jj, mc, nc, kc, A_pack, B_pack, kk);\n            }\n        }\n    }\n\n    void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int m_rem = MIN(m - m0, 8);\n        int n_rem = MIN(n - n0, 8);\n        int mc = 0, nc = 0;\n        if (m_rem >= 8 && n_rem >= 8) {\n            mc = 8;\n            nc = 8;\n            gemm<8, 8>(m0, m, n0, n);\n        } else if (m_rem >= 4 && n_rem >= 8) {\n            mc = 4;\n            nc = 8;\n            gemm<4, 8>(m0, m, n0, n);\n        } else if (m_rem >= 8 && n_rem >= 4) {\n            mc = 8;\n            nc = 4;\n            gemm<8, 4>(m0, m, n0, n);\n        } else if (m_rem >= 4 && n_rem >= 4) {\n            mc = 4;\n            nc = 4;\n            gemm<4, 4>(m0, m, n0, n);\n        } else {\n            mc = (m_rem >= 4) ? 4 : m_rem;\n            nc = (n_rem >= 4) ? 4 : n_rem;\n            if (mc == 0 || nc == 0)\n                return;\n            gemm_small(m0, m, n0, n, mc, nc);\n        }\n        int64_t mp = m0 + ((m - m0) / mc) * mc;\n        int64_t np = n0 + ((n - n0) / nc) * nc;\n        mnpack(mp, m, n0, np);\n        mnpack(m0, m, np, n);\n    }\n\n    void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            vec_t vec_C[4];\n            acc_t acc_0;\n            __builtin_mma_xxsetaccz(&acc_0);\n            vec_t vec_A[4] = {0}, vec_B[4] = {0};\n            for (int l = 0; l < k; l += 4) {\n                /* 'GEMV Forwarding' concept is used in first two conditional loops.\n                 * when one of the matrix has a single row/column, the elements are\n                 * broadcasted, instead of using packing routine to prepack the\n                 * matrix elements.\n                 */\n                if (RM == 1) {\n                    float * a = const_cast<float *>(A + (ii) * lda + l);\n                    packTranspose(B + (jj * ldb) + l, ldb, RN, 4, (float *)vec_B);\n                    vec_A[0] = (vec_t)vec_xl(0,a);\n                    vec_A[1] = (vec_t)vec_splats(*((float *)&vec_A+1));\n                    vec_A[2] = (vec_t)vec_splats(*((float *)&vec_A+2));\n                    vec_A[3] = (vec_t)vec_splats(*((float *)&vec_A+3));\n                } else if (RN == 1) {\n                    packTranspose(A + (ii * lda) + l, lda, RM, 4, (float *)vec_A);\n                    float * b = const_cast<float *>(B + (jj) * ldb + l);\n                    vec_B[0] = (vec_t)vec_xl(0,b);\n                    vec_B[1] = (vec_t)vec_splats(*((float *)&vec_B+1));\n                    vec_B[2] = (vec_t)vec_splats(*((float *)&vec_B+2));\n                    vec_B[3] = (vec_t)vec_splats(*((float *)&vec_B+3));\n                } else {\n                    packTranspose(A + (ii * lda) + l, lda, RM, 4, (float *)vec_A);\n                    packTranspose(B + (jj * ldb) + l, ldb, RN, 4, (float *)vec_B);\n                }\n                __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);\n                __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);\n                __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);\n                __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);\n            }\n            __builtin_mma_disassemble_acc(vec_C, &acc_0);\n            for (int I = 0; I < RM; I++) {\n                for (int J = 0; J < RN; J++) {\n                    *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J);\n                }\n            }\n       }\n    }\n\n    template<int RM, int RN>\n    inline void kernel(int64_t ii, int64_t jj) {\n        if constexpr(RM == 4 && RN == 4) {\n            KERNEL_4x4(ii, jj);\n        } else if constexpr(RM == 4 && RN == 8) {\n            KERNEL_4x8(ii, jj);\n        } else if constexpr(RM == 8 && RN == 4) {\n            KERNEL_8x4(ii, jj);\n        } else if constexpr(RM == 8 && RN == 8) {\n            KERNEL_8x8(ii, jj);\n        } else {\n            static_assert(false, \"RN/RM values not supported\");\n        }\n    }\n\n    template <int RM, int RN>\n    NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {\n        int64_t ytiles = (m - m0) / RM;\n        int64_t xtiles = (n - n0) / RN;\n        int64_t tiles = xtiles * ytiles;\n        int64_t duty = (tiles + nth - 1) / nth;\n        int64_t start = duty * ith;\n        int64_t end = start + duty;\n        if (end > tiles)\n            end = tiles;\n        for (int64_t job = start; job < end; ++job) {\n            int64_t ii = m0 + job / xtiles * RM;\n            int64_t jj = n0 + job % xtiles * RN;\n            kernel<RM, RN>(ii, jj);\n        }\n    }\n\n    const float * const A;\n    const float * const B;\n    float * C;\n    const int64_t k;\n    const int64_t lda;\n    const int64_t ldb;\n    const int64_t ldc;\n    const int ith;\n    const int nth;\n};\n#endif\n} // namespace\n\n/**\n * Performs optimized matrix multiplication on CPU.\n *\n * This subroutine may compute C = Aᵀ * B with column major ordering.\n * Despite its name, this isn't a generalized implementation. Work is\n * only performed when a handwritten kernel is written and available.\n * Otherwise the caller should fall back to a general matmul routine.\n *\n * For example, for single-threaded single-precision GEMM you can say\n *\n *     llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,\n *                     0, 1,\n *                     GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);\n *\n * @param m is rows in `A` and `C`\n * @param n is cols in `B` and `C`\n * @param k is cols in `A` and rows in `B`\n * @param A is first input matrix (always transposed)\n * @param lda is row stride of `A`\n * @param B is second input matrix (never transposed)\n * @param ldb is row stride of `B`\n * @param C is input/output array of output matrices\n * @param ldc is row stride of `C`\n * @param ith is thread id (must be less than `nth`)\n * @param nth is number of threads (must be greater than zero)\n * @param Atype is GGML data type of `A`\n * @param Btype is GGML data type of `B`\n * @param Ctype is GGML data type of `C`\n * @return true if this function was able to service the matmul request\n */\nbool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64_t n, int64_t k,\n                     const void *A, int64_t lda, const void *B, int64_t ldb, void *C,\n                     int64_t ldc, int Atype, int Btype, int Ctype) {\n\n    assert(m >= 0);\n    assert(n >= 0);\n    assert(k >= 0);\n    assert(lda >= k);\n    assert(ldb >= k);\n    assert(ldc >= m);\n    assert(params->nth > 0);\n    assert(params->ith < params->nth);\n\n    // only enable sgemm for prompt processing\n#if !defined(__MMA__)\n    if (n < 2)\n        return false;\n#endif\n\n    if (Ctype != GGML_TYPE_F32)\n        return false;\n\n    switch (Atype) {\n\n    case GGML_TYPE_F32: {\n        if (Btype != GGML_TYPE_F32)\n            return false;\n#if defined(__AVX512F__)\n        tinyBLAS<16, __m512, __m512, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n        return tb.matmul(m, n);\n#elif defined(__AVX__) || defined(__AVX2__)\n        tinyBLAS<8, __m256, __m256, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n        return tb.matmul(m, n);\n#elif defined(__ARM_NEON)\n        if (n < 4)\n            return false;\n        tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n        return tb.matmul(m, n);\n#elif defined(__VXE__) || defined(__VXE2__)\n        if (n < 4)\n            return false;\n        tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n        return tb.matmul(m, n);\n#elif defined(__MMA__)\n        if (k % 8)\n            return false;\n        tinyBLAS_PPC tb{\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#elif defined(__riscv_zvfh)\n    #if LMUL == 1\n        tinyBLAS_RVV<vfloat32m1_t, vfloat32m1_t, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n    #elif LMUL == 2\n        tinyBLAS_RVV<vfloat32m2_t, vfloat32m2_t, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n    #else // LMUL = 4\n        tinyBLAS_RVV<vfloat32m4_t, vfloat32m4_t, float, float, float> tb{ params,\n            k, (const float *)A, lda,\n            (const float *)B, ldb,\n            (float *)C, ldc};\n    #endif\n        return tb.matmul(m, n);\n#else\n        return false;\n#endif\n    }\n\n    case GGML_TYPE_BF16: {\n#if defined(__AVX512BF16__)\n        if (Btype == GGML_TYPE_BF16) {\n            tinyBLAS<32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n                (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__AVX512F__)\n        if (Btype == GGML_TYPE_BF16) {\n            tinyBLAS<16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n                (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__AVX2__)\n        if (Btype == GGML_TYPE_BF16) {\n            tinyBLAS<8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,\n                (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__MMA__)\n        if (k % 8) {\n            return false;\n        }\n\n        if (Btype == GGML_TYPE_BF16) {\n            tinyBLAS_HP16_PPC<ggml_bf16_t, ggml_bf16_t, float> tb{ k,\n                (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc,\n                params->ith, params->nth };\n\n            tb.matmul(m, n);\n            return true;\n        }\n#elif defined(__riscv_zvfbfwma)\n        #if LMUL == 1\n            tinyBLAS_RVV<vfloat32m1_t, vbfloat16mf2_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params,\n                k, (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n        #elif LMUL == 2\n            tinyBLAS_RVV<vfloat32m2_t, vbfloat16m1_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params,\n                k, (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n        #else // LMUL = 4\n            tinyBLAS_RVV<vfloat32m4_t, vbfloat16m2_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params,\n                k, (const ggml_bf16_t *)A, lda,\n                (const ggml_bf16_t *)B, ldb,\n                (float *)C, ldc};\n        #endif\n            return tb.matmul(m, n);\n#endif\n        return false;\n    }\n\n    case GGML_TYPE_F16: {\n#if defined(__AVX512F__)\n        if (Btype == GGML_TYPE_F16) {\n            tinyBLAS<16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,\n                (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)\n        if (Btype == GGML_TYPE_F16) {\n            tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,\n                (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)\n        if (n < 8)\n            return false;\n        if (Btype == GGML_TYPE_F16) {\n            tinyBLAS<8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__ARM_NEON) && !defined(_MSC_VER)\n        if (Btype == GGML_TYPE_F32) {\n            tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const float *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__VXE__) || defined(__VXE2__)\n        if (n < 4)\n            return false;\n        if (Btype == GGML_TYPE_F16) {\n            tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n            return tb.matmul(m, n);\n        }\n#elif defined(__riscv_zvfh)\n        if (Btype == GGML_TYPE_F16) {\n        #if LMUL == 1\n            tinyBLAS_RVV<vfloat32m1_t, vfloat16mf2_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n        #elif LMUL == 2\n            tinyBLAS_RVV<vfloat32m2_t, vfloat16m1_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n        #else // LMUL = 4\n            tinyBLAS_RVV<vfloat32m4_t, vfloat16m2_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,\n                k, (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc};\n        #endif\n            return tb.matmul(m, n);\n        }\n#elif defined(__MMA__)\n        if (k % 8) {\n            return false;\n        }\n\n        if (Btype == GGML_TYPE_F16) {\n            tinyBLAS_HP16_PPC<ggml_fp16_t, ggml_fp16_t, float> tb{ k,\n                (const ggml_fp16_t *)A, lda,\n                (const ggml_fp16_t *)B, ldb,\n                (float *)C, ldc,\n                params->ith, params->nth };\n\n            tb.matmul(m, n);\n            return true;\n        }\n#endif\n        return false;\n    }\n\n    case GGML_TYPE_Q8_0: {\n        if (Btype != GGML_TYPE_Q8_0)\n           return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n        tinyBLAS_Q0_AVX<block_q8_0, block_q8_0, float> tb{\n            k, (const block_q8_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#elif defined(__ARM_FEATURE_DOTPROD)\n        tinyBLAS_Q0_ARM<block_q8_0> tb{\n            k, (const block_q8_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#elif defined(__MMA__)\n    //TO-DO: Remove this condition once gemv forwarding is enabled.\n        if (n < 8 && n != 4)\n           return false;\n        if (m < 8 && m != 4)\n           return false;\n        tinyBLAS_Q0_PPC<block_q8_0> tb{\n            k, (const block_q8_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#else\n        return false;\n#endif\n    }\n\n    case GGML_TYPE_Q4_0: {\n        if (Btype != GGML_TYPE_Q8_0)\n            return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n        tinyBLAS_Q0_AVX<block_q4_0, block_q8_0, float> tb{\n            k, (const block_q4_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#elif defined(__ARM_FEATURE_DOTPROD)\n        tinyBLAS_Q0_ARM<block_q4_0> tb{\n            k, (const block_q4_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#elif defined(__MMA__)\n    //TO-DO: Remove this condition once gemv forwarding is enabled.\n        if (n < 8 && n != 4)\n           return false;\n        if (m < 8 && m != 4)\n           return false;\n        tinyBLAS_Q0_PPC<block_q4_0> tb{\n            k, (const block_q4_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#else\n        return false;\n#endif\n    }\n\n    case GGML_TYPE_Q5_0: {\n        if (Btype != GGML_TYPE_Q8_0)\n            return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n        tinyBLAS_Q0_AVX<block_q5_0, block_q8_0, float> tb{\n            k, (const block_q5_0 *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#else\n        return false;\n#endif\n    }\n\n    case GGML_TYPE_IQ4_NL: {\n        if (Btype != GGML_TYPE_Q8_0)\n            return false;\n#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)\n        tinyBLAS_Q0_AVX<block_iq4_nl, block_q8_0, float> tb{\n            k, (const block_iq4_nl *)A, lda,\n            (const block_q8_0 *)B, ldb,\n            (float *)C, ldc,\n            params->ith, params->nth};\n        tb.matmul(m, n);\n        return true;\n#else\n        return false;\n#endif\n    }\n\n    default:\n        return false;\n    }\n\n    (void)params;\n    (void)m;\n    (void)n;\n    (void)k;\n    (void)A;\n    (void)lda;\n    (void)B;\n    (void)ldb;\n    (void)C;\n    (void)ldc;\n    (void)Atype;\n    (void)Btype;\n    (void)Ctype;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/llamafile/sgemm.h",
    "content": "#pragma once\n#include <stdint.h>\n#include <stdbool.h>\n\n#if defined(__VXE__) || defined(__VXE2__)\n#include <vecintrin.h>\n#endif\n\n#ifdef _MSC_VER\n#define NOINLINE __declspec(noinline)\n#else\n#define NOINLINE __attribute__((__noinline__))\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nbool llamafile_sgemm(const struct ggml_compute_params * params, int64_t, int64_t, int64_t,\n                     const void *, int64_t, const void *, int64_t, void *, int64_t,\n                     int, int, int);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/ops.cpp",
    "content": "#include \"ops.h\"\n\n#include \"ggml-cpu.h\"\n#include \"ggml-impl.h\"\n#include \"binary-ops.h\"\n#include \"simd-gemm.h\"\n#include \"ggml.h\"\n#include \"unary-ops.h\"\n#include \"vec.h\"\n\n#include <algorithm>\n#include <cfloat>\n#include <cmath>\n\n// ggml_compute_forward_dup\n\nstatic void ggml_compute_forward_dup_same_cont(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n    GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n    GGML_ASSERT(src0->type == dst->type);\n\n    const size_t nb0 = ggml_type_size(src0->type);\n\n    const int ith = params->ith; // thread index\n    const int nth = params->nth; // number of threads\n\n    // parallelize by blocks\n    const int nk = ggml_nelements(src0)/ggml_blck_size(src0->type);\n    const int dr = (nk + nth - 1) / nth;\n    const int k0 = dr * ith;\n    const int k1 = MIN(k0 + dr, nk);\n\n    if (k0 < k1) {\n        memcpy(\n            ((char *)  dst->data + k0*nb0),\n            ((char *) src0->data + k0*nb0),\n            (k1 - k0) * nb0);\n    }\n}\n\ntemplate<typename src_t, typename dst_t>\nstatic void ggml_compute_forward_dup_flt(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n    GGML_ASSERT(!ggml_is_quantized(src0->type) && !ggml_is_quantized(dst->type));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int ith = params->ith; // thread index\n    const int nth = params->nth; // number of threads\n\n    // parallelize by rows\n    const int nr = ne01;\n    // number of rows per thread\n    const int dr = (nr + nth - 1) / nth;\n    // row range for this thread\n    const int ir0 = dr * ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    // case: type & row size equal\n    if (src0->type == dst->type &&\n        ne00 == ne0 &&\n        nb00 == ggml_type_size(src0->type) && nb0 == ggml_type_size(dst->type)) {\n        // copy by rows\n        const size_t rs = ne00*nb00;\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                    memcpy(\n                        ((char *)  dst->data + i01*nb1  + i02*nb2  + i03*nb3),\n                        ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),\n                        rs);\n                }\n            }\n        }\n        return;\n    }\n\n    // case: dst tensor is contiguous\n    if (ggml_is_contiguous(dst)) {\n        if (nb00 == sizeof(src_t)) {\n            if constexpr (std::is_same_v<dst_t, src_t>) {\n                // same type\n                size_t id = 0;\n                const size_t rs = ne00 * nb00;\n                char * dst_ptr = (char *) dst->data;\n\n                for (int i03 = 0; i03 < ne03; i03++) {\n                    for (int i02 = 0; i02 < ne02; i02++) {\n                        id += rs * ir0;\n                        for (int i01 = ir0; i01 < ir1; i01++) {\n                            const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n                            memcpy(dst_ptr + id, src0_ptr, rs);\n                            id += rs;\n                        }\n                        id += rs * (ne01 - ir1);\n                    }\n                }\n            } else {\n                // casting between non-quantized types\n                size_t id = 0;\n                dst_t * dst_ptr = (dst_t *) dst->data;\n\n                for (int i03 = 0; i03 < ne03; i03++) {\n                    for (int i02 = 0; i02 < ne02; i02++) {\n                        id += ne00 * ir0;\n                        for (int i01 = ir0; i01 < ir1; i01++) {\n                            const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n                            for (int i00 = 0; i00 < ne00; i00++) {\n                                float tmp = type_conversion_table<src_t>::to_f32(src0_ptr[i00]);\n                                dst_ptr[id] = type_conversion_table<dst_t>::from_f32(tmp);\n                                id++;\n                            }\n                        }\n                        id += ne00 * (ne01 - ir1);\n                    }\n                }\n            }\n        } else {\n            //printf(\"%s: this is not optimal - fix me\\n\", __func__);\n\n            size_t id = 0;\n            dst_t * dst_ptr = (dst_t *) dst->data;\n\n            for (int i03 = 0; i03 < ne03; i03++) {\n                for (int i02 = 0; i02 < ne02; i02++) {\n                    id += ne00 * ir0;\n                    for (int i01 = ir0; i01 < ir1; i01++) {\n                        for (int i00 = 0; i00 < ne00; i00++) {\n                            const src_t * src0_ptr = (src_t *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n\n                            float tmp = type_conversion_table<src_t>::to_f32(*src0_ptr);\n                            dst_ptr[id] = type_conversion_table<dst_t>::from_f32(tmp);\n                            id++;\n                        }\n                    }\n                    id += ne00 * (ne01 - ir1);\n                }\n            }\n        }\n        return;\n    }\n\n    // dst counters\n    int64_t i10 = 0;\n    int64_t i11 = 0;\n    int64_t i12 = 0;\n    int64_t i13 = 0;\n\n    if constexpr (std::is_same_v<dst_t, src_t>) {\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                i10 += ne00 * ir0;\n                while (i10 >= ne0) {\n                    i10 -= ne0;\n                    if (++i11 == ne1) {\n                        i11 = 0;\n                        if (++i12 == ne2) {\n                            i12 = 0;\n                            if (++i13 == ne3) {\n                                i13 = 0;\n                            }\n                        }\n                    }\n                }\n                for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n                              char * dst_ptr  = ((char *)  dst->data + i10*nb0  + i11*nb1  + i12*nb2  + i13*nb3);\n\n                        memcpy(dst_ptr, src0_ptr, sizeof(dst_t));\n\n                        if (++i10 == ne00) {\n                            i10 = 0;\n                            if (++i11 == ne01) {\n                                i11 = 0;\n                                if (++i12 == ne02) {\n                                    i12 = 0;\n                                    if (++i13 == ne03) {\n                                        i13 = 0;\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n                i10 += ne00 * (ne01 - ir1);\n                while (i10 >= ne0) {\n                    i10 -= ne0;\n                    if (++i11 == ne1) {\n                        i11 = 0;\n                        if (++i12 == ne2) {\n                            i12 = 0;\n                            if (++i13 == ne3) {\n                                i13 = 0;\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n    } else {\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                i10 += ne00 * ir0;\n                while (i10 >= ne0) {\n                    i10 -= ne0;\n                    if (++i11 == ne1) {\n                        i11 = 0;\n                        if (++i12 == ne2) {\n                            i12 = 0;\n                            if (++i13 == ne3) {\n                                i13 = 0;\n                            }\n                        }\n                    }\n                }\n                for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n                              char * dst_ptr  = ((char *)  dst->data + i10*nb0  + i11*nb1  + i12*nb2  + i13*nb3);\n\n                        float tmp = type_conversion_table<src_t>::to_f32(*(const src_t *) src0_ptr);\n                        *(dst_t *) dst_ptr = type_conversion_table<dst_t>::from_f32(tmp);\n\n                        if (++i10 == ne0) {\n                            i10 = 0;\n                            if (++i11 == ne1) {\n                                i11 = 0;\n                                if (++i12 == ne2) {\n                                    i12 = 0;\n                                    if (++i13 == ne3) {\n                                        i13 = 0;\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n                i10 += ne00 * (ne01 - ir1);\n                while (i10 >= ne0) {\n                    i10 -= ne0;\n                    if (++i11 == ne1) {\n                        i11 = 0;\n                        if (++i12 == ne2) {\n                            i12 = 0;\n                            if (++i13 == ne3) {\n                                i13 = 0;\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\n\ntemplate<typename src_t>\nstatic void ggml_compute_forward_dup_to_q(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n    GGML_ASSERT(!ggml_is_quantized(src0->type));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int ith = params->ith; // thread index\n    const int nth = params->nth; // number of threads\n\n    // parallelize by rows\n    const int nr = ne01;\n    // number of rows per thread\n    const int dr = (nr + nth - 1) / nth;\n    // row range for this thread\n    const int ir0 = dr * ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    if (ggml_is_contiguous(dst) &&\n            nb00 == sizeof(src_t) &&\n            ggml_get_type_traits_cpu(dst->type)->from_float) {\n        // casting non-quantized types --> intermediate f32 --> quantized\n        ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dst->type)->from_float;\n        float * src0_f32 = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n        size_t id = 0;\n        size_t rs = nb0 * (ne00 / ggml_blck_size(dst->type));\n        char * dst_ptr = (char *) dst->data;\n\n        for (int i03 = 0; i03 < ne03; i03++) {\n            for (int i02 = 0; i02 < ne02; i02++) {\n                id += rs * ir0;\n                for (int i01 = ir0; i01 < ir1; i01++) {\n                    const src_t * src0_ptr = (src_t *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n                    for (int i00 = 0; i00 < ne00; i00++) {\n                        src0_f32[i00] = type_conversion_table<src_t>::to_f32(src0_ptr[i00]);\n                    }\n\n                    quantize_row_q(src0_f32, dst_ptr + id, ne00);\n                    id += rs;\n                }\n                id += rs * (ne01 - ir1);\n            }\n        }\n    } else {\n        // printf(\"%s %s\\n\", ggml_type_name(src0->type), ggml_type_name(dst->type));\n        GGML_ABORT(\"not implemented\");\n    }\n}\n\n// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.\nstatic void ggml_compute_forward_dup_bytes(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n    GGML_ASSERT(src0->type == dst->type);\n\n    GGML_TENSOR_UNARY_OP_LOCALS;\n\n    if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {\n        ggml_compute_forward_dup_same_cont(params, dst);\n        return;\n    }\n\n    const size_t type_size = ggml_type_size(src0->type);\n\n    const int ith = params->ith; // thread index\n    const int nth = params->nth; // number of threads\n\n    // parallelize by rows\n    const int nr = ne01;\n    // number of rows per thread\n    const int dr = (nr + nth - 1) / nth;\n    // row range for this thread\n    const int ir0 = dr * ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    if (src0->type == dst->type &&\n        ggml_are_same_shape(src0, dst) &&\n        nb00 == type_size && nb0 == type_size) {\n        // copy by rows\n        const size_t rs = ggml_row_size(src0->type, ne00);\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                    memcpy(\n                        ((char *)  dst->data + i01*nb1  + i02*nb2  + i03*nb3),\n                        ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),\n                        rs);\n                }\n            }\n        }\n        return;\n    }\n\n    if (ggml_is_contiguous(dst)) {\n        size_t id = 0;\n        char * dst_ptr = (char *) dst->data;\n        const size_t rs = ne00 * type_size;\n\n        if (nb00 == type_size) {\n            // src0 is contiguous on first dimension, copy by rows\n            for (int64_t i03 = 0; i03 < ne03; i03++) {\n                for (int64_t i02 = 0; i02 < ne02; i02++) {\n                    id += rs * ir0;\n                    for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                        const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;\n                        memcpy(dst_ptr + id, src0_ptr, rs);\n                        id += rs;\n                    }\n                    id += rs * (ne01 - ir1);\n                }\n            }\n        } else {\n            //printf(\"%s: this is not optimal - fix me\\n\", __func__);\n\n            for (int64_t i03 = 0; i03 < ne03; i03++) {\n                for (int64_t i02 = 0; i02 < ne02; i02++) {\n                    id += rs * ir0;\n                    for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                        for (int64_t i00 = 0; i00 < ne00; i00++) {\n                            const char * src0_ptr = (char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03;\n                            memcpy(dst_ptr + id, src0_ptr, type_size);\n\n                            id += type_size;\n                        }\n                    }\n                    id += rs * (ne01 - ir1);\n                }\n            }\n        }\n\n        return;\n    }\n\n    // dst counters\n    int64_t k10 = 0;\n    int64_t i11 = 0;\n    int64_t i12 = 0;\n    int64_t i13 = 0;\n\n    // number of blocks in a row\n    const int64_t nk00 = ne00 / ggml_blck_size(src0->type);\n    const int64_t nk0  = ne0  / ggml_blck_size(dst->type);\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            k10 += nk00 * ir0;\n            while (k10 >= nk0) {\n                k10 -= nk0;\n                if (++i11 == ne1) {\n                    i11 = 0;\n                    if (++i12 == ne2) {\n                        i12 = 0;\n                        if (++i13 == ne3) {\n                            i13 = 0;\n                        }\n                    }\n                }\n            }\n            for (int64_t i01 = ir0; i01 < ir1; i01++) {\n                for (int64_t k00 = 0; k00 < nk00; k00++) {\n                    const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n                          char * dst_ptr  = ((char *)  dst->data + k10*nb0  + i11*nb1  + i12*nb2  + i13*nb3);\n\n                    memcpy(dst_ptr, src0_ptr, type_size);\n\n                    if (++k10 == nk0) {\n                        k10 = 0;\n                        if (++i11 == ne1) {\n                            i11 = 0;\n                            if (++i12 == ne2) {\n                                i12 = 0;\n                                if (++i13 == ne3) {\n                                    i13 = 0;\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n            k10 += nk00 * (ne01 - ir1);\n            while (k10 >= nk0) {\n                k10 -= nk0;\n                if (++i11 == ne1) {\n                    i11 = 0;\n                    if (++i12 == ne2) {\n                        i12 = 0;\n                        if (++i13 == ne3) {\n                            i13 = 0;\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_dup_from_q(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const ggml_type type = src0->type;\n    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n    size_t qk = ggml_blck_size(type);\n    const int64_t nr = ggml_nelements(src1) / qk;\n\n    // destination must be contiguous in the first dimension\n    GGML_ASSERT(nb10 == ggml_type_size(dst->type));\n    // must either have first dimension large enough to hold a row, or fully contiguous\n    GGML_ASSERT((ne10 % qk) == 0 || ggml_is_contiguous(dst));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n\n        uint32_t i = ir * qk;\n\n        const int64_t i03 = i/(ne00 * ne01 * ne02);\n        const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n        const int64_t i01 = (i - i03*ne00*ne01*ne02  -  i02*ne01*ne00) / ne00;\n        const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n        const int64_t x_offset = (i00/qk)*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n        const int64_t i13 = i/(ne10 * ne11 * ne12);\n        const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n        const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n        const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n        const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n        dequantize_row_q(\n                (const void *) ((char *) src0->data + x_offset),\n                     (float *) ((char *)  dst->data + dst_offset), qk);\n    }\n}\n\nvoid ggml_compute_forward_dup(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (src0->type == dst->type) {\n        ggml_compute_forward_dup_bytes(params, dst);\n        return;\n    }\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n            {\n                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_fp16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_fp16_t, ggml_bf16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_fp16_t, float      >(params, dst);\n                else ggml_compute_forward_dup_to_q<ggml_fp16_t>(params, dst);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_fp16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<ggml_bf16_t, ggml_bf16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<ggml_bf16_t, float      >(params, dst);\n                else ggml_compute_forward_dup_to_q<ggml_bf16_t>(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                /**/ if (dst->type == GGML_TYPE_F16)  ggml_compute_forward_dup_flt<float, ggml_fp16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_BF16) ggml_compute_forward_dup_flt<float, ggml_bf16_t>(params, dst);\n                else if (dst->type == GGML_TYPE_F32)  ggml_compute_forward_dup_flt<float, float      >(params, dst);\n                else if (dst->type == GGML_TYPE_I32)  ggml_compute_forward_dup_flt<float, int32_t    >(params, dst);\n                else ggml_compute_forward_dup_to_q<float>(params, dst);\n            } break;\n        case GGML_TYPE_I32:\n            {\n                if (dst->type == GGML_TYPE_F32) ggml_compute_forward_dup_flt<int32_t, float>(params, dst);\n                else GGML_ABORT(\"not implemented\");\n            } break;\n        default:\n            {\n                if (ggml_is_quantized(src0->type) && dst->type == GGML_TYPE_F32) {\n                    ggml_compute_forward_dup_from_q(params, dst);\n                    break;\n                }\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_add\n\nstatic void ggml_compute_forward_add_q_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const ggml_type type = src0->type;\n    const ggml_type dtype = dst->type;\n    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n    ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dtype)->from_float;\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == ggml_type_size(type));\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    GGML_ASSERT(ggml_is_quantized(src0->type));\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    float * wdata = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 indices\n        const int i03 = ir/(ne02*ne01);\n        const int i02 = (ir - i03*ne02*ne01)/ne01;\n        const int i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        // src1 and dst are same shape as src0 => same indices\n        const int i13 = i03;\n        const int i12 = i02;\n        const int i11 = i01;\n\n        const int i3 = i03;\n        const int i2 = i02;\n        const int i1 = i01;\n\n        void  * src0_row = (void *) ((char *) src0->data + (i01*nb01 + i02*nb02 + i03*nb03));\n        float * src1_row = (float *)((char *) src1->data + (i11*nb11 + i12*nb12 + i13*nb13));\n        void  * dst_row  = (void *) ((char *)  dst->data + ( i1*nb1  +  i2*nb2  +  i3*nb3));\n\n        assert(ne00 % 32 == 0);\n\n        // unquantize row from src0 to temp buffer\n        dequantize_row_q(src0_row, wdata, ne00);\n        // add src1\n        ggml_vec_acc_f32(ne00, wdata, src1_row);\n        // quantize row to dst\n        if (quantize_row_q != NULL) {\n            quantize_row_q(wdata, dst_row, ne00);\n        } else {\n            memcpy(dst_row, wdata, ne0*nb0);\n        }\n    }\n}\n\nvoid ggml_compute_forward_add(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n            {\n                ggml_compute_forward_add_non_quantized(params, dst);\n            } break;\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n            {\n                ggml_compute_forward_add_q_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_add_id\n\nstatic void ggml_compute_forward_add_id_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    const ggml_tensor * src2 = dst->src[2];\n\n    GGML_ASSERT(dst->type  == GGML_TYPE_F32);\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(src2->type == GGML_TYPE_I32);\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_TERNARY_OP_LOCALS\n\n    GGML_ASSERT( nb0 == sizeof(float));\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        // src1 indices\n        const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21);\n\n        GGML_ASSERT(i11 >= 0 && i11 < ne11);\n\n        ggml_vec_add_f32(ne0,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 ),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),\n                (float *) ((char *) src1->data + i11*nb11));\n    }\n}\n\nvoid ggml_compute_forward_add_id(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_add_id_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"unsupported type for ggml_compute_forward_add_id: %s\", ggml_type_name(src0->type));\n            }\n    }\n}\n\n// ggml_compute_forward_add1\n\nstatic void ggml_compute_forward_add1_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT( nb0 == sizeof(float));\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n#ifdef GGML_USE_ACCELERATE\n        GGML_UNUSED(ggml_vec_add1_f32);\n\n        vDSP_vadd(\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1,\n                (float *) ((char *) src1->data), 0,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 ), 1,\n                ne0);\n#else\n        ggml_vec_add1_f32(ne0,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 ),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),\n               *(float *) src1->data);\n#endif\n    }\n}\n\nstatic void ggml_compute_forward_add1_f16_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    // scalar to add\n    const float v = *(float *) src1->data;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type  == GGML_TYPE_F16);\n\n    GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        ggml_fp16_t * dst_ptr  = (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 );\n        ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n        for (int i = 0; i < ne0; i++) {\n            dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_add1_f16_f16(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    // scalar to add\n    const float v = GGML_CPU_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F16);\n    GGML_ASSERT(dst->type  == GGML_TYPE_F16);\n\n    GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        ggml_fp16_t * dst_ptr  = (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 );\n        ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n        for (int i = 0; i < ne0; i++) {\n            dst_ptr[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(src0_ptr[i]) + v);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_add1_q_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    // scalar to add\n    const float v = *(float *) src1->data;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const ggml_type type = src0->type;\n    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n    ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(type)->from_float;\n\n    // we don't support permuted src0\n    GGML_ASSERT(nb00 == ggml_type_size(type));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    GGML_ASSERT(ggml_is_quantized(src0->type));\n    GGML_ASSERT(dst->type == src0->type);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        void  * src0_row = (void *) ((char *) src0->data + (i1*nb01 + i2*nb02 + i3*nb03));\n        void  * dst_row  = (void *) ((char *)  dst->data + (i1*nb1  + i2*nb2  + i3*nb0 ));\n\n        assert(ne0 % 32 == 0);\n\n        // unquantize row from src0 to temp buffer\n        dequantize_row_q(src0_row, wdata, ne0);\n        // add src1\n        ggml_vec_acc1_f32(ne0, wdata, v);\n        // quantize row to dst\n        quantize_row_q(wdata, dst_row, ne0);\n    }\n}\n\nstatic void ggml_compute_forward_add1_bf16_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    // scalar to add\n    const float v = *(float *) src1->data;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(src0->type == GGML_TYPE_BF16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type  == GGML_TYPE_BF16);\n\n    GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        ggml_bf16_t * dst_ptr  = (ggml_bf16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 );\n        ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n        for (int i = 0; i < ne0; i++) {\n            dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_add1_bf16_bf16(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_scalar(src1));\n\n    // scalar to add\n    const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(src0->type == GGML_TYPE_BF16);\n    GGML_ASSERT(src1->type == GGML_TYPE_BF16);\n    GGML_ASSERT(dst->type  == GGML_TYPE_BF16);\n\n    GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are same shape => same indices\n        const int i3 = ir/(ne2*ne1);\n        const int i2 = (ir - i3*ne2*ne1)/ne1;\n        const int i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        ggml_bf16_t * dst_ptr  = (ggml_bf16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1 );\n        ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n        for (int i = 0; i < ne0; i++) {\n            dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);\n        }\n    }\n}\n\nvoid ggml_compute_forward_add1(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_add1_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                if (src1->type == GGML_TYPE_F16) {\n                    ggml_compute_forward_add1_f16_f16(params, dst);\n                }\n                else if (src1->type == GGML_TYPE_F32) {\n                    ggml_compute_forward_add1_f16_f32(params, dst);\n                }\n                else {\n                    GGML_ABORT(\"fatal error\");\n                }\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                if (src1->type == GGML_TYPE_BF16) {\n                    ggml_compute_forward_add1_bf16_bf16(params, dst);\n                }\n                else if (src1->type == GGML_TYPE_F32) {\n                    ggml_compute_forward_add1_bf16_f32(params, dst);\n                }\n                else {\n                    GGML_ABORT(\"fatal error\");\n                }\n            } break;\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_Q8_1:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n            {\n                ggml_compute_forward_add1_q_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_acc\n\nstatic void ggml_compute_forward_acc_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n    // view src0 and dst with these strides and data offset inbytes during acc\n    // nb0 is implicitly element_size because src0 and dst are contiguous\n    size_t nb1     = ((int32_t *) dst->op_params)[0];\n    size_t nb2     = ((int32_t *) dst->op_params)[1];\n    size_t nb3     = ((int32_t *) dst->op_params)[2];\n    size_t offset  = ((int32_t *) dst->op_params)[3];\n    bool   inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n    if (!inplace) {\n        if (params->ith == 0) {\n            // memcpy needs to be synchronized across threads to avoid race conditions.\n            // => do it in INIT phase\n            memcpy(\n                ((char *)  dst->data),\n                ((char *) src0->data),\n                ggml_nbytes(dst));\n        }\n        ggml_barrier(params->threadpool);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr = ggml_nrows(src1);\n    const int nc = src1->ne[0];\n\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb)\n\n    // src0 and dst as viewed during acc\n    const size_t nb0 = ggml_element_size(src0);\n\n    const size_t nb00 = nb0;\n    const size_t nb01 = nb1;\n    const size_t nb02 = nb2;\n    const size_t nb03 = nb3;\n\n    GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb0  + (ne11 == 0 ? 0 : ne11-1)*nb1  + (ne12 == 0 ? 0 : ne12-1)*nb2  + (ne13 == 0 ? 0 : ne13-1)*nb3  < ggml_nbytes(dst));\n    GGML_ASSERT(offset + (ne10 == 0 ? 0 : ne10-1)*nb00 + (ne11 == 0 ? 0 : ne11-1)*nb01 + (ne12 == 0 ? 0 : ne12-1)*nb02 + (ne13 == 0 ? 0 : ne13-1)*nb03 < ggml_nbytes(src0));\n\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are viewed with shape of src1 and offset\n        // => same indices\n        const int i3 = ir/(ne12*ne11);\n        const int i2 = (ir - i3*ne12*ne11)/ne11;\n        const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n#ifdef GGML_USE_ACCELERATE\n        vDSP_vadd(\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset), 1,\n                (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1  + offset), 1, nc);\n#else\n        ggml_vec_add_f32(nc,\n                (float *) ((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1  + offset),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + offset),\n                (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n#endif\n    }\n}\n\nvoid ggml_compute_forward_acc(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_acc_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_Q8_1:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_sum\n\nstatic void ggml_compute_forward_sum_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(ggml_is_scalar(dst));\n    assert(src0->nb[0] == sizeof(float));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n\n    ggml_float sum     = 0;\n    ggml_float row_sum = 0;\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = 0; i01 < ne01; i01++) {\n                ggml_vec_sum_f32_ggf(ne00,\n                        &row_sum,\n                        (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));\n                sum += row_sum;\n            }\n        }\n    }\n    ((float *) dst->data)[0] = sum;\n}\n\nstatic void ggml_compute_forward_sum_f16(\n    const ggml_compute_params * params,\n          ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(ggml_is_scalar(dst));\n\n    assert(src0->nb[0] == sizeof(ggml_fp16_t));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n\n    float sum = 0;\n    float row_sum = 0;\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = 0; i01 < ne01; i01++) {\n                ggml_vec_sum_f16_ggf(ne00,\n                    &row_sum,\n                    (ggml_fp16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));\n                sum += row_sum;\n            }\n        }\n    }\n    ((ggml_fp16_t *) dst->data)[0] = GGML_CPU_FP32_TO_FP16(sum);\n}\n\nstatic void ggml_compute_forward_sum_bf16(\n    const ggml_compute_params * params,\n          ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(ggml_is_scalar(dst));\n\n    assert(src0->nb[0] == sizeof(ggml_bf16_t));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n\n    float sum = 0;\n    float row_sum = 0;\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = 0; i01 < ne01; i01++) {\n                ggml_vec_sum_bf16_ggf(ne00,\n                    &row_sum,\n                    (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));\n                sum += row_sum;\n            }\n        }\n    }\n    ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);\n}\n\nvoid ggml_compute_forward_sum(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_sum_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_sum_f16(params, dst);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                ggml_compute_forward_sum_bf16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_cumsum\n\nstatic void ggml_compute_forward_cumsum_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(ne0 == ne00);\n    GGML_ASSERT(ne1 == ne01);\n    GGML_ASSERT(ne2 == ne02);\n    GGML_ASSERT(ne3 == ne03);\n\n    const auto [ir0, ir1] = get_thread_range(params, src0);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        float * src_row = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n        float * dst_row = (float *) ((char *) dst->data  + i01*nb1  + i02*nb2  + i03*nb3);\n\n        ggml_vec_cumsum_f32(ne00, dst_row, src_row);\n    }\n}\n\nvoid ggml_compute_forward_cumsum(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_cumsum_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_sum_rows\n\nstatic void ggml_compute_forward_sum_rows_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(ne0 == 1);\n    GGML_ASSERT(ne1 == ne01);\n    GGML_ASSERT(ne2 == ne02);\n    GGML_ASSERT(ne3 == ne03);\n\n    for (int64_t i3 = 0; i3 < ne03; i3++) {\n        for (int64_t i2 = 0; i2 < ne02; i2++) {\n            for (int64_t i1 = 0; i1 < ne01; i1++) {\n                float * src_row = (float *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);\n                float * dst_row = (float *) ((char *) dst->data  + i1*nb1  + i2*nb2  + i3*nb3);\n                float row_sum = 0;\n                ggml_vec_sum_f32(ne00, &row_sum, src_row);\n                dst_row[0] = row_sum;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_sum_rows(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_sum_rows_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_mean\n\nstatic void ggml_compute_forward_mean_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(src0->nb[0] == sizeof(float));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    assert(ne0 == 1);\n    assert(ne1 == ne01);\n    assert(ne2 == ne02);\n    assert(ne3 == ne03);\n\n    GGML_UNUSED(ne0);\n    GGML_UNUSED(ne1);\n    GGML_UNUSED(ne2);\n    GGML_UNUSED(ne3);\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = 0; i01 < ne01; i01++) {\n                ggml_vec_sum_f32(ne00,\n                        (float *) ((char *)  dst->data + i01*nb1  + i02*nb2  + i03*nb3),\n                        (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));\n\n                *(float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3) /= (float) ne00;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_mean(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_mean_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_argmax\n\nstatic void ggml_compute_forward_argmax_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(src0->nb[0] == sizeof(float));\n    assert(dst->nb[0] == sizeof(float));\n\n    const int64_t ne00 = src0->ne[0];\n    const int64_t ne01 = src0->ne[1];\n\n    const size_t nb01 = src0->nb[1];\n    const size_t nb0 = dst->nb[0];\n\n    for (int64_t i1 = 0; i1 < ne01; i1++) {\n        float * src = (float *) ((char *) src0->data + i1*nb01);\n        int32_t * dst_ = (int32_t *) ((char *)  dst->data + i1*nb0);\n        int v = 0;\n        ggml_vec_argmax_f32(ne00, &v, src);\n        dst_[0] = v;\n    }\n}\n\nvoid ggml_compute_forward_argmax(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_argmax_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_count_equal\n\nstatic void ggml_compute_forward_count_equal_i32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    GGML_ASSERT(src0->type == GGML_TYPE_I32);\n    GGML_ASSERT(src1->type == GGML_TYPE_I32);\n    GGML_ASSERT(ggml_are_same_shape(src0, src1));\n    GGML_ASSERT(ggml_is_scalar(dst));\n    GGML_ASSERT(dst->type == GGML_TYPE_I64);\n\n    const int64_t nr = ggml_nrows(src0);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    int64_t * sums = (int64_t *) params->wdata;\n    int64_t sum_thread = 0;\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 =  ir                        / (ne02*ne01);\n        const int64_t i02 = (ir - i03*ne03)            /       ne01;\n        const int64_t i01 =  ir - i03*ne03 - i02*ne02;\n\n        const char * data0 = (const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01;\n        const char * data1 = (const char *) src1->data + i03*nb13 + i02*nb12 + i01*nb11;\n\n        for (int64_t i00 = 0; i00 < ne00; ++i00) {\n            const int32_t val0 = *((const int32_t *) (data0 + i00*nb00));\n            const int32_t val1 = *((const int32_t *) (data1 + i00*nb10));\n\n            sum_thread += val0 == val1;\n        }\n    }\n    if (ith != 0) {\n        sums[ith] = sum_thread;\n    }\n    ggml_barrier(params->threadpool);\n\n    if (ith != 0) {\n        return;\n    }\n\n    for (int ith_other = 1; ith_other < nth; ++ith_other) {\n        sum_thread += sums[ith_other];\n    }\n    *((int64_t *) dst->data) = sum_thread;\n}\n\nvoid ggml_compute_forward_count_equal(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_I32:\n            {\n                ggml_compute_forward_count_equal_i32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_repeat\n\nstatic void ggml_compute_forward_repeat_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(ggml_can_repeat(src0, dst));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    // guaranteed to be an integer due to the check in ggml_can_repeat\n    const int nr0 = (int)(ne0/ne00);\n    const int nr1 = (int)(ne1/ne01);\n    const int nr2 = (int)(ne2/ne02);\n    const int nr3 = (int)(ne3/ne03);\n\n    // TODO: support for transposed / permuted tensors\n    GGML_ASSERT(nb0  == sizeof(float));\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // TODO: maybe this is not optimal?\n    for                         (int i3 = 0; i3 < nr3;  i3++) {\n        for                     (int k3 = 0; k3 < ne03; k3++) {\n            for                 (int i2 = 0; i2 < nr2;  i2++) {\n                for             (int k2 = 0; k2 < ne02; k2++) {\n                    for         (int i1 = 0; i1 < nr1;  i1++) {\n                        for     (int k1 = 0; k1 < ne01; k1++) {\n                            for (int i0 = 0; i0 < nr0;  i0++) {\n                                ggml_vec_cpy_f32(ne00,\n                                        (float *) ((char *)  dst->data + (i3*ne03 + k3)*nb3  + (i2*ne02 + k2)*nb2  + (i1*ne01 + k1)*nb1  + (i0*ne00)*nb0),\n                                        (float *) ((char *) src0->data + (          k3)*nb03 + (          k2)*nb02 + (          k1)*nb01));\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_repeat_f16(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(ggml_can_repeat(src0, dst));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    // guaranteed to be an integer due to the check in ggml_can_repeat\n    const int nr0 = (int)(ne0/ne00);\n    const int nr1 = (int)(ne1/ne01);\n    const int nr2 = (int)(ne2/ne02);\n    const int nr3 = (int)(ne3/ne03);\n\n    // TODO: support for transposed / permuted tensors\n    GGML_ASSERT(nb0  == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n    // TODO: maybe this is not optimal?\n    for                         (int i3 = 0; i3 < nr3;  i3++) {\n        for                     (int k3 = 0; k3 < ne03; k3++) {\n            for                 (int i2 = 0; i2 < nr2;  i2++) {\n                for             (int k2 = 0; k2 < ne02; k2++) {\n                    for         (int i1 = 0; i1 < nr1;  i1++) {\n                        for     (int k1 = 0; k1 < ne01; k1++) {\n                            for (int i0 = 0; i0 < nr0;  i0++) {\n                                ggml_fp16_t * y = (ggml_fp16_t *) ((char *)  dst->data + (i3*ne03 + k3)*nb3  + (i2*ne02 + k2)*nb2  + (i1*ne01 + k1)*nb1  + (i0*ne00)*nb0);\n                                ggml_fp16_t * x = (ggml_fp16_t *) ((char *) src0->data + (          k3)*nb03 + (          k2)*nb02 + (          k1)*nb01);\n                                // ggml_vec_cpy_f16(ne00, y, x)\n                                for (int i = 0; i < ne00; ++i) {\n                                    y[i]  = x[i];\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_repeat(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n        case GGML_TYPE_I16:\n            {\n                ggml_compute_forward_repeat_f16(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n        case GGML_TYPE_I32:\n            {\n                ggml_compute_forward_repeat_f32(params, dst);\n            } break;\n        // TODO: templateify the implementation and support for I64\n        //       ref https://github.com/ggml-org/llama.cpp/pull/14274#discussion_r2169492225\n        //case GGML_TYPE_I64:\n        //    {\n        //        ggml_compute_forward_repeat_i64(params, dst);\n        //    } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_repeat_back\n\nstatic void ggml_compute_forward_repeat_back_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(ggml_can_repeat(dst, src0));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    // guaranteed to be an integer due to the check in ggml_can_repeat\n    const int nr0 = (int)(ne00/ne0);\n    const int nr1 = (int)(ne01/ne1);\n    const int nr2 = (int)(ne02/ne2);\n    const int nr3 = (int)(ne03/ne3);\n\n    // TODO: support for transposed / permuted tensors\n    GGML_ASSERT(nb0  == sizeof(float));\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    if (ggml_is_contiguous(dst)) {\n        ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n    } else {\n        for         (int k3 = 0; k3 < ne3; k3++) {\n            for     (int k2 = 0; k2 < ne2; k2++) {\n                for (int k1 = 0; k1 < ne1; k1++) {\n                    ggml_vec_set_f32(ne0,\n                        (float *) ((char *) dst->data + k1*nb1 + k2*nb2 + k3*nb3),\n                        0);\n                }\n            }\n        }\n    }\n\n    // TODO: maybe this is not optimal?\n    for                         (int i3 = 0; i3 < nr3; i3++) {\n        for                     (int k3 = 0; k3 < ne3; k3++) {\n            for                 (int i2 = 0; i2 < nr2; i2++) {\n                for             (int k2 = 0; k2 < ne2; k2++) {\n                    for         (int i1 = 0; i1 < nr1; i1++) {\n                        for     (int k1 = 0; k1 < ne1; k1++) {\n                            for (int i0 = 0; i0 < nr0; i0++) {\n                                ggml_vec_acc_f32(ne0,\n                                        (float *) ((char *)  dst->data + (         k3)*nb3  + (         k2)*nb2  + (         k1)*nb1),\n                                        (float *) ((char *) src0->data + (i3*ne3 + k3)*nb03 + (i2*ne2 + k2)*nb02 + (i1*ne1 + k1)*nb01 + (i0*ne0)*nb00));\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_repeat_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_repeat_back_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_concat\n\nstatic void ggml_compute_forward_concat_any(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    const size_t len = ggml_type_size(src0->type);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(dim >= 0 && dim < 4);\n\n    int64_t o[4] = {0, 0, 0, 0};\n    o[dim] = src0->ne[dim];\n\n    const char * x;\n\n    // TODO: smarter multi-theading\n    for (int i3 = 0; i3 < ne3; i3++) {\n        for (int i2 = ith; i2 < ne2; i2 += nth) {\n            for (int i1 = 0; i1 < ne1; i1++) {\n                for (int i0 = 0; i0 < ne0; i0++) {\n                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n                        x = (const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03;\n                    } else {\n                        x = (const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13;\n                    }\n\n                    char * y = (char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3;\n\n                    memcpy(y, x, len);\n                }\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_concat_i8(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(dim >= 0 && dim < 4);\n\n    int64_t o[4] = {0, 0, 0, 0};\n    o[dim] = src0->ne[dim];\n\n    const int8_t * x;\n\n    // TODO: smarter multi-theading\n    for (int i3 = 0; i3 < ne3; i3++) {\n        for (int i2 = ith; i2 < ne2; i2 += nth) {\n            for (int i1 = 0; i1 < ne1; i1++) {\n                for (int i0 = 0; i0 < ne0; i0++) {\n                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n                        x = (const int8_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);\n                    } else {\n                        x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n                    }\n\n                    int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n                    *y = *x;\n                }\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_concat_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(dim >= 0 && dim < 4);\n\n    int64_t o[4] = {0, 0, 0, 0};\n    o[dim] = src0->ne[dim];\n\n    const ggml_fp16_t * x;\n\n    // TODO: smarter multi-theading\n    for (int i3 = 0; i3 < ne3; i3++) {\n        for (int i2 = ith; i2 < ne2; i2 += nth) {\n            for (int i1 = 0; i1 < ne1; i1++) {\n                for (int i0 = 0; i0 < ne0; i0++) {\n                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n                        x = (const ggml_fp16_t *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);\n                    } else {\n                        x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n                    }\n\n                    ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n                    *y = *x;\n                }\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_concat_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int32_t dim = ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(dim >= 0 && dim < 4);\n\n    int64_t o[4] = {0, 0, 0, 0};\n    o[dim] = src0->ne[dim];\n\n    const float * x;\n\n    // TODO: smarter multi-theading\n    for (int i3 = 0; i3 < ne3; i3++) {\n        for (int i2 = ith; i2 < ne2; i2 += nth) {\n            for (int i1 = 0; i1 < ne1; i1++) {\n                for (int i0 = 0; i0 < ne0; i0++) {\n                    if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n                        x = (const float *) ((const char *)src0->data + (i0       )*nb00 + (i1       )*nb01 + (i2       )*nb02 + (i3       )*nb03);\n                    } else {\n                        x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);\n                    }\n\n                    float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n\n                    *y = *x;\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_concat(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n        case GGML_TYPE_I16:\n            {\n                ggml_compute_forward_concat_f16(params, dst);\n            } break;\n        case GGML_TYPE_I8:\n            {\n                ggml_compute_forward_concat_i8(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n        case GGML_TYPE_I32:\n            {\n                ggml_compute_forward_concat_f32(params, dst);\n            } break;\n        default:\n            {\n                ggml_compute_forward_concat_any(params, dst);\n            }\n    }\n}\n\n// ggml_compute_forward_gelu\n\nstatic void ggml_compute_forward_gelu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_f32(nc,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n            const float v = GGML_CPU_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_gelu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_gelu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_fill\n\nstatic void ggml_compute_forward_fill_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n    const float c = ggml_get_op_params_f32(dst, 0);\n\n    GGML_TENSOR_LOCALS(int64_t, ne, dst, ne);\n    GGML_TENSOR_LOCALS(size_t,  nb, dst, nb);\n\n    const auto [ir0, ir1] = get_thread_range(params, dst);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne2*ne1);\n        const int64_t i02 = (ir - i03*ne2*ne1)/ne1;\n        const int64_t i01 = (ir - i03*ne2*ne1 - i02*ne1);\n\n        float * dst_ptr  = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);\n\n        ggml_vec_set_f32(ne0, dst_ptr, c);\n    }\n}\n\nvoid ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor * dst) {\n    ggml_compute_forward_fill_f32(params, dst);\n}\n\n// ggml_compute_tri\n\nstatic void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const auto [ir0, ir1] = get_thread_range(params, src0);\n\n    bool (*bipred)(int, int);\n\n    switch (ttype) {\n        case GGML_TRI_TYPE_LOWER:      bipred = [](int i, int r) { return i <  r; }; break;\n        case GGML_TRI_TYPE_LOWER_DIAG: bipred = [](int i, int r) { return i <= r; }; break;\n        case GGML_TRI_TYPE_UPPER:      bipred = [](int i, int r) { return i >  r; }; break;\n        case GGML_TRI_TYPE_UPPER_DIAG: bipred = [](int i, int r) { return i >= r; }; break;\n        default: GGML_ABORT(\"invalid tri type\");\n    }\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        const float * src_ptr = (const float  *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n              float * dst_ptr = (      float  *) ((      char *) dst->data  + i03*nb3  + i02*nb2  + i01*nb1);\n\n        for (int i0 = 0; i0 < ne0; ++i0) {\n            dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;\n        }\n    }\n}\n\nvoid ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_tri_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_gelu_erf\n\nstatic void ggml_compute_forward_gelu_erf_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_erf_f32(nc,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu_erf_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_erf_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n            const float v = GGML_CPU_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu_erf(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_gelu_erf_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_gelu_erf_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_gelu_quick\n\nstatic void ggml_compute_forward_gelu_quick_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_quick_f32(nc,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu_quick_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_gelu_quick_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n            const float v = GGML_CPU_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_gelu_quick(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_gelu_quick_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_gelu_quick_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_silu\n\nstatic void ggml_compute_forward_silu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_silu_f32(nc,\n                (float *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*(dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_silu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(ggml_is_contiguous_rows(src0));\n    assert(ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb0, src0, nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst,  nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int i3 = ir/(ne02*ne01);\n        const int i2 = (ir - i3*ne02*ne01)/ne01;\n        const int i1 = (ir - i3*ne02*ne01 - i2*ne01);\n\n        ggml_vec_silu_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i3*nb3  + i2*nb2  + i1*nb1),\n                (ggml_fp16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*( dst->nb[1])))[k];\n            const float v = GGML_CPU_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_silu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_silu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_silu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n// ggml_compute_forward_leaky_relu\n\nstatic void ggml_compute_forward_leaky_relu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(ggml_is_contiguous_1(src0));\n    assert(ggml_is_contiguous_1(dst));\n    assert(ggml_are_same_shape(src0, dst));\n\n    const int n  = ggml_nrows(src0);\n    const int nc = src0->ne[0];\n\n    float negative_slope;\n    memcpy(&negative_slope, dst->op_params, sizeof(float));\n\n    assert(dst->nb[0]  == sizeof(float));\n    assert(src0->nb[0] == sizeof(float));\n\n    for (int i = 0; i < n; i++) {\n        ggml_vec_leaky_relu_f32(nc,\n                (float *) ((char *) dst->data  + i*( dst->nb[1])),\n                (float *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);\n    }\n}\n\nstatic void ggml_compute_forward_leaky_relu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    assert(ggml_is_contiguous_1(src0));\n    assert(ggml_is_contiguous_1(dst));\n    assert(ggml_are_same_shape(src0, dst));\n\n    const int n  = ggml_nrows(src0);\n    const int nc = src0->ne[0];\n\n    float negative_slope;\n    memcpy(&negative_slope, dst->op_params, sizeof(float));\n\n    assert(dst->nb[0]  == sizeof(ggml_fp16_t));\n    assert(src0->nb[0] == sizeof(ggml_fp16_t));\n\n    for (int i = 0; i < n; i++) {\n        ggml_vec_leaky_relu_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i*( dst->nb[1])),\n                (ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);\n    }\n}\n\nvoid ggml_compute_forward_leaky_relu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_leaky_relu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_leaky_relu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_silu_back\n\nstatic void ggml_compute_forward_silu_back_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * grad = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    assert(ggml_is_contiguous_1(grad));\n    assert(ggml_is_contiguous_1(src1));\n    assert(ggml_is_contiguous_1(dst));\n    assert(ggml_are_same_shape(src1, dst));\n    assert(ggml_are_same_shape(src1, grad));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1->ne[0];\n    const int nr = ggml_nrows(src1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_vec_silu_backward_f32(nc,\n                (float *) ((char *) dst->data  + i1*( dst->nb[1])),\n                (float *) ((char *) src1->data + i1*(src1->nb[1])),\n                (float *) ((char *) grad->data + i1*(grad->nb[1])));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_silu_back_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * grad = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    assert(ggml_is_contiguous_1(grad));\n    assert(ggml_is_contiguous_1(src1));\n    assert(ggml_is_contiguous_1(dst));\n    assert(ggml_are_same_shape(src1, dst));\n    assert(ggml_are_same_shape(src1, grad));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1->ne[0];\n    const int nr = ggml_nrows(src1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_vec_silu_backward_f16(nc,\n                (ggml_fp16_t *) ((char *) dst->data  + i1*( dst->nb[1])),\n                (ggml_fp16_t *) ((char *) src1->data + i1*(src1->nb[1])),\n                (ggml_fp16_t *) ((char *) grad->data + i1*(grad->nb[1])));\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_CPU_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nvoid ggml_compute_forward_silu_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_silu_back_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_silu_back_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_reglu\n\nstatic void ggml_compute_forward_reglu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_reglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_reglu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_reglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_reglu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_reglu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_reglu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_geglu\n\nstatic void ggml_compute_forward_geglu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_geglu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_geglu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_swiglu\n\nstatic void ggml_compute_forward_swiglu_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_swiglu_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_swiglu_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_swiglu_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_swiglu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_swiglu_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_swiglu_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_swiglu_oai\n\nstatic void ggml_compute_forward_swiglu_oai_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n    const float alpha = ggml_get_op_params_f32(dst, 2);\n    const float limit = ggml_get_op_params_f32(dst, 3);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n        float * dst_p  = (float *) ((char *) dst->data + i1*(dst->nb[1]));\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        for (int k = 0; k < nc; k++) {\n            const float x = std::min(src0_p[k], limit);\n            const float y = std::clamp(src1_p[k], -limit, limit);\n            const float out_glu = x / (1.f + expf(alpha * (-x)));\n            dst_p[k] = out_glu * (y + 1.f);\n        }\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = dst_p[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_swiglu_oai(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_swiglu_oai_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_geglu_erf\n\nstatic void ggml_compute_forward_geglu_erf_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_erf_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu_erf_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_erf_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu_erf(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_geglu_erf_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_geglu_erf_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_geglu_quick\n\nstatic void ggml_compute_forward_geglu_quick_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * src0_p = (float *) (src0_d + i1*src0_o);\n        float * src1_p = (float *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_quick_f32(nc, (float *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            GGML_UNUSED(x);\n            assert(!isnan(x));\n            assert(!isinf(x));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu_quick_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    char * src0_d = (char *) src0->data;\n    char * src1_d = (char *) (src1 ? src1->data : src0->data);\n    const size_t src0_o = src0->nb[1];\n    const size_t src1_o = src1 ? src1->nb[1] : src0->nb[1];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0));\n    GGML_ASSERT(ggml_is_contiguous_1(dst));\n\n    if (src1) {\n        GGML_ASSERT(ggml_is_contiguous_1(src1));\n        GGML_ASSERT(src0->type == src1->type);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src1 ? src0->ne[0] : src0->ne[0] / 2;\n    const int nr = ggml_nrows(src0);\n\n    GGML_ASSERT(dst->ne[0] == nc);\n    GGML_ASSERT(ggml_nrows(dst) == nr);\n\n    const int32_t swapped = ggml_get_op_params_i32(dst, 1);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);\n        ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);\n\n        if (!src1) {\n            src0_p += swapped ? nc : 0;\n            src1_p += swapped ? 0 : nc;\n        }\n\n        ggml_vec_geglu_quick_f16(nc, (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1])), src0_p, src1_p);\n\n#ifndef NDEBUG\n        for (int k = 0; k < nc; k++) {\n            const ggml_fp16_t x = ((ggml_fp16_t *) ((char *) dst->data + i1*( dst->nb[1])))[k];\n            const float v = GGML_FP16_TO_FP32(x);\n            GGML_UNUSED(v);\n            assert(!isnan(v));\n            assert(!isinf(v));\n        }\n#endif // NDEBUG\n    }\n}\n\nstatic void ggml_compute_forward_geglu_quick(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_geglu_quick_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_geglu_quick_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_norm\n\nstatic void ggml_compute_forward_norm_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    GGML_ASSERT(eps >= 0.0f);\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n                const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n                float sum = 0.0;\n                ggml_vec_sum_f32(ne00, &sum, x);\n                float mean = sum/ne00;\n\n                float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n                float variance = 0;\n\n#ifdef GGML_USE_ACCELERATE\n                mean = -mean;\n                vDSP_vsadd(x, 1, &mean, y, 1, ne00);\n                vDSP_measqv(y, 1, &variance, ne00);\n#else\n                variance = ggml_vec_cvar_f32(ne00, y, x, mean);\n#endif //GGML_USE_ACCELERATE\n\n                const float scale = 1.0f/sqrtf(variance + eps);\n                ggml_vec_scale_f32(ne00, y, scale);\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_norm(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_norm_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_group_rms_norm\n\nstatic void ggml_compute_forward_rms_norm_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    GGML_ASSERT(eps >= 0.0f);\n\n    // TODO: optimize\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n                const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n                ggml_float sum = 0.0;\n                for (int64_t i00 = 0; i00 < ne00; i00++) {\n                    sum += (ggml_float)(x[i00] * x[i00]);\n                }\n\n                const float mean = sum/ne00;\n\n                float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n                memcpy(y, x, ne00 * sizeof(float));\n                // for (int i00 = 0; i00 < ne00; i00++) {\n                //     y[i00] = x[i00];\n                // }\n\n                const float scale = 1.0f/sqrtf(mean + eps);\n\n                // if you hit this, likely you got an inf somewhere earlier\n                assert(scale > 0.0f);\n\n                ggml_vec_scale_f32(ne00, y, scale);\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_rms_norm(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rms_norm_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_rms_norm_back_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0]; // gradients from forward pass output\n    const ggml_tensor * src1 = dst->src[1]; // src1 from forward pass\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(src1->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    // TODO: optimize\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n                // src1 is same shape as src0 => same indices\n                const int64_t i11 = i01;\n                const int64_t i12 = i02;\n                const int64_t i13 = i03;\n\n                const float * dz = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n                const float * x  = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);\n\n                ggml_float sum_xx  = 0.0;\n                ggml_float sum_xdz = 0.0;\n\n                for (int64_t i00 = 0; i00 < ne00; i00++) {\n                    sum_xx  += (ggml_float)(x[i00] * x[i00]);\n                    sum_xdz += (ggml_float)(x[i00] * dz[i00]);\n                }\n\n                //const float mean     = (float)(sum_xx)/ne00;\n                const float mean_eps = (float)(sum_xx)/ne00 + eps;\n                const float sum_eps  = (float)(sum_xx) + eps*ne00;\n                //const float mean_xdz = (float)(sum_xdz)/ne00;\n                // we could cache rms from forward pass to improve performance.\n                // to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms.\n                //const float rms      = sqrtf(mean_eps);\n                const float rrms     = 1.0f / sqrtf(mean_eps);\n                //const float scale    = -rrms/(ne00 * mean_eps); // -1/(n*rms**3)\n\n                {\n                    // z = rms_norm(x)\n                    //\n                    // rms_norm(src1) =\n                    //     scale(\n                    //         src1,\n                    //         div(\n                    //             1,\n                    //             sqrt(\n                    //                 add(\n                    //                     scale(\n                    //                         sum(\n                    //                             sqr(\n                    //                                 src1)),\n                    //                         (1.0/N)),\n                    //                     eps))));\n\n                    // postorder:\n                    // ## op    args         grad\n                    // 00 param src1         grad[#00]\n                    // 01 const 1\n                    // 02 sqr   (#00)        grad[#02]\n                    // 03 sum   (#02)        grad[#03]\n                    // 04 const 1/N\n                    // 05 scale (#03, #04)   grad[#05]\n                    // 06 const eps\n                    // 07 add   (#05, #06)   grad[#07]\n                    // 08 sqrt  (#07)        grad[#08]\n                    // 09 div   (#01,#08)    grad[#09]\n                    // 10 scale (#00,#09)    grad[#10]\n                    //\n                    // backward pass, given grad[#10]\n                    // #10: scale\n                    // grad[#00] += scale(grad[#10],#09)\n                    // grad[#09] += sum(mul(grad[#10],#00))\n                    // #09: div\n                    // grad[#08] += neg(mul(grad[#09], div(#09,#08)))\n                    // #08: sqrt\n                    // grad[#07] += mul(grad[#08], div(0.5, #08))\n                    // #07: add\n                    // grad[#05] += grad[#07]\n                    // #05: scale\n                    // grad[#03] += scale(grad[#05],#04)\n                    // #03: sum\n                    // grad[#02] += repeat(grad[#03], #02)\n                    // #02:\n                    // grad[#00] += scale(mul(#00, grad[#02]), 2.0)\n                    //\n                    // substitute and simplify:\n                    // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)\n                    // grad[#02] = repeat(grad[#03], #02)\n                    // grad[#02] = repeat(scale(grad[#05],#04), #02)\n                    // grad[#02] = repeat(scale(grad[#07],#04), #02)\n                    // grad[#02] = repeat(scale(mul(grad[#08], div(0.5, #08)),#04), #02)\n                    // grad[#02] = repeat(scale(mul(neg(mul(grad[#09], div(#09,#08))), div(0.5, #08)),#04), #02)\n                    // grad[#02] = repeat(scale(mul(neg(mul(sum(mul(grad[#10],#00)), div(#09,#08))), div(0.5, #08)),#04), #02)\n                    // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(#09,#08) * div(0.5, #08) * (1/N)), #02)\n                    // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(div(#01,#08),#08) * div(0.5, #08) * (1/N)), #02)\n                    // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#08*#08) * div(0.5, #08) * (1/N)), #02)\n                    // grad[#02] = repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)\n                    // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, grad[#02]), 2.0)\n                    // grad[#00] = scale(grad(#10), #09) + scale(mul(#00, repeat(-(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N)), #02)), 2.0)\n                    // grad[#00] = scale(grad(#10), #09) + scale(scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(0.5, #08) * (1/N))), 2.0)\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, -(sum(mul(grad[#10],#00)) * div(1,#07) * div(1,#08) * (1/N)))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,#07*#08) * (-1/N))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(1,mean_eps*rms) * (-1/N))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*mean_eps))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*(sum_xx/N+eps)))\n                    // grad[#00] = scale(grad(#10), #09) + scale(#00, sum(mul(grad[#10],#00)) * div(-1,rms*N*sum_xx+rms*N*eps))\n                    // grad[#00] = scale(dz, rrms) + scale(x, sum(mul(dz,x)) * div(-1,rms*N*mean_eps))\n                    // grad[#00] = scale(dz, rrms) + scale(x, sum_xdz * div(-1,rms*N*mean_eps))\n                    // a = b*c + d*e\n                    // a = b*c*f/f + d*e*f/f\n                    // a = (b*c*f + d*e*f)*(1/f)\n                    // a = (b*c*(1/c) + d*e*(1/c))*(1/(1/c))\n                    // a = (b + d*e/c)*c\n                    // b = dz, c = rrms, d = x, e = sum_xdz * div(-1,rms*N*mean_eps)\n                    // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)/rrms)*rrms\n                    // a = (dz + x*sum_xdz * div(-1,rms*N*mean_eps)*rms)*rrms\n                    // a = (dz + x*sum_xdz * div(-rms,rms*N*mean_eps))*rrms\n                    // a = (dz + x*sum_xdz * div(-1,N*mean_eps))*rrms\n                    // a = (dz + x*div(-sum_xdz,N*mean_eps))*rrms\n                    // a = (dz + x*div(-mean_xdz,mean_eps))*rrms\n                    // grad[#00] = scale(dz + scale(x, div(-mean_xdz,mean_eps)),rrms)\n                    // grad[#00] = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n                    // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n                }\n                // dx = scale(dz + scale(x, -mean_xdz/mean_eps),rrms)\n                // post-order:\n                // dx := x\n                // dx := scale(dx,-mean_xdz/mean_eps)\n                // dx := add(dx, dz)\n                // dx := scale(dx, rrms)\n                float * dx = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n                // dx[i00] = (x*(-sum_xdz/sum_eps) + dz) / sqrtf(mean_eps)\n                ggml_vec_cpy_f32  (ne00, dx, x);\n                // ggml_vec_scale_f32(ne00, dx, -mean_xdz/mean_eps);\n                ggml_vec_scale_f32(ne00, dx, (float)(-sum_xdz)/sum_eps);\n                ggml_vec_acc_f32  (ne00, dx, dz);\n                ggml_vec_scale_f32(ne00, dx, rrms);\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_rms_norm_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rms_norm_back_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_group_norm\n\nstatic void ggml_compute_forward_group_norm_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    // TODO: optimize\n\n    float eps;\n    memcpy(&eps, dst->op_params + 1, sizeof(float));\n\n    int n_channels = src0->ne[2];\n    int n_groups = dst->op_params[0];\n    int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;\n    for (int i = ith; i < n_groups; i += nth) {\n        int start = i * n_channels_per_group;\n        int end = start + n_channels_per_group;\n        if (end > n_channels) {\n            end = n_channels;\n        }\n        int step = end - start;\n\n        for (int64_t i03 = 0; i03 < ne03; i03++) {\n            ggml_float sum = 0.0;\n            for (int64_t i02 = start; i02 < end; i02++) {\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);\n\n                    ggml_float sumr = 0.0;\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        sumr += (ggml_float)x[i00];\n                    }\n                    sum += sumr;\n                }\n            }\n            const float mean = sum / (ne00 * ne01 * step);\n\n            ggml_float sum2 = 0.0;\n            for (int64_t i02 = start; i02 < end; i02++) {\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    const float * x = (float *)((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03);\n\n                    float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);\n\n                    ggml_float sumr = 0.0;\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        float v = x[i00] - mean;\n                        y[i00] = v;\n                        sumr += (ggml_float)(v * v);\n                    }\n                    sum2 += sumr;\n                }\n            }\n            const float variance = sum2 / (ne00 * ne01 * step);\n            const float scale = 1.0f / sqrtf(variance + eps);\n\n            for (int64_t i02 = start; i02 < end; i02++) {\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    float * y = (float *)((char *) dst->data + i01 * nb1 + i02 * nb2 + i03 * nb3);\n                    ggml_vec_scale_f32(ne00, y, scale);\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_group_norm(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_group_norm_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_l2_norm\n\nstatic void ggml_compute_forward_l2_norm_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    float eps;\n    memcpy(&eps, dst->op_params, sizeof(float));\n\n    GGML_ASSERT(eps >= 0.0f);\n\n    // TODO: optimize\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n                const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n\n                ggml_float sum = 0.0;\n                for (int64_t i00 = 0; i00 < ne00; i00++) {\n                    sum += (ggml_float)(x[i00] * x[i00]);\n                }\n\n                float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);\n\n                memcpy(y, x, ne00 * sizeof(float));\n\n                const float scale = 1.0f/fmaxf(sqrtf(sum), eps);\n\n                ggml_vec_scale_f32(ne00, y, scale);\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_l2_norm(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_l2_norm_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_out_prod\n\nstatic void ggml_compute_forward_out_prod_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_ASSERT(ne0 == ne00);\n    GGML_ASSERT(ne1 == ne10);\n    GGML_ASSERT(ne2 == ne12);\n    GGML_ASSERT(ne3 == ne13);\n\n    GGML_ASSERT(ne2 % ne02 == 0);\n    GGML_ASSERT(ne3 % ne03 == 0);\n\n    // we don't support permuted src0 or src1\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    // GGML_ASSERT(nb0 <= nb1);\n    // GGML_ASSERT(nb1 <= nb2);\n    // GGML_ASSERT(nb2 <= nb3);\n\n    // nb01 >= nb00 - src0 is not transposed\n    //   compute by src0 rows\n\n    if (ith == 0) {\n        ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n    }\n    ggml_barrier(params->threadpool);\n\n    // dst[:,:,:,:] = 0\n    // for i2,i3:\n    //   for i1:\n    //     for i01:\n    //       for i0:\n    //         dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]\n\n    // parallelize by last three dimensions\n\n    // total rows in dst\n    const int64_t nr = ne1*ne2*ne3;\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    // block-tiling attempt\n    const int64_t blck_0 = MAX(GGML_VEC_MAD_UNROLL, 32);\n    const int64_t blck_1 = 16;\n\n    // dps == dst per src0, used for group query attention\n    const int64_t dps2 = ne2 / ne02;\n    const int64_t dps3 = ne3 / ne03;\n\n    for (int64_t bir = ir0; bir < ir1; bir += blck_1) {\n        const int64_t bir1 = MIN(bir + blck_1, ir1);\n        for (int64_t bi01 = 0; bi01 < ne01; bi01 += blck_0) {\n            const int64_t bne01 = MIN(bi01 + blck_0, ne01);\n            for (int64_t ir = bir; ir < bir1; ++ir) {\n                // dst indices\n                const int64_t i3 = ir/(ne2*ne1);\n                const int64_t i2 = (ir - i3*ne2*ne1)/ne1;\n                const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n                const int64_t i02 = i2 / dps2;\n                const int64_t i03 = i3 / dps3;\n\n                //const int64_t i10 = i1;\n                const int64_t i12 = i2;\n                const int64_t i13 = i3;\n\n#if GGML_VEC_MAD_UNROLL > 2\n                const int64_t bne01_unroll = bne01 - (bne01 % GGML_VEC_MAD_UNROLL);\n                for (int64_t i01 = bi01; i01 < bne01_unroll; i01 += GGML_VEC_MAD_UNROLL) {\n                    const int64_t i11 = i01;\n\n                    float * s0 = (float *) ((char *) src0->data + (          i01*nb01 + i02*nb02 + i03*nb03));\n                    float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n                    float * d  = (float *) ((char *)  dst->data + (          i1*nb1   + i2*nb2   + i3*nb3));\n\n                    ggml_vec_mad_f32_unroll(ne0, nb01, nb11, d, s0, s1);\n                }\n                for (int64_t i01 = bne01_unroll; i01 < bne01; ++i01) {\n                    const int64_t i11 = i01;\n\n                    float * s0 = (float *) ((char *) src0->data + (          i01*nb01 + i02*nb02 + i03*nb03));\n                    float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n                    float * d  = (float *) ((char *)  dst->data + (          i1*nb1   + i2*nb2   + i3*nb3));\n\n                    ggml_vec_mad_f32(ne0, d, s0, *s1);\n                }\n#else\n                for (int64_t i01 = bi01; i01 < bne01; ++i01) {\n                    const int64_t i11 = i01;\n\n                    float * s0 = (float *) ((char *) src0->data + (          i01*nb01 + i02*nb02 + i03*nb03));\n                    float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n                    float * d  = (float *) ((char *)  dst->data + (          i1*nb1 + i2*nb2 + i3*nb3));\n\n                    ggml_vec_mad_f32(ne0, d, s0, *s1);\n                }\n#endif\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_out_prod_q_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const ggml_type type = src0->type;\n    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n    GGML_ASSERT(ne02 == ne12);\n    GGML_ASSERT(ne03 == ne13);\n    GGML_ASSERT(ne2  == ne12);\n    GGML_ASSERT(ne3  == ne13);\n\n    // we don't support permuted src0 dim0\n    GGML_ASSERT(nb00 == ggml_type_size(type));\n\n    // dst dim0 cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    // GGML_ASSERT(nb0 <= nb1);\n    // GGML_ASSERT(nb1 <= nb2);\n    // GGML_ASSERT(nb2 <= nb3);\n\n    GGML_ASSERT(ne0 == ne00);\n    GGML_ASSERT(ne1 == ne10);\n    GGML_ASSERT(ne2 == ne02);\n    GGML_ASSERT(ne3 == ne03);\n\n    // nb01 >= nb00 - src0 is not transposed\n    //   compute by src0 rows\n\n    if (ith == 0) {\n        ggml_vec_set_f32(ne0*ne1*ne2*ne3, (float *)dst->data, 0);\n    }\n    ggml_barrier(params->threadpool);\n\n    // parallelize by last three dimensions\n\n    // total rows in dst\n    const int64_t nr = ne1*ne2*ne3;\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    // dst[:,:,:,:] = 0\n    // for i2,i3:\n    //   for i1:\n    //     for i01:\n    //       for i0:\n    //         dst[i0,i1,i2,i3] += src0[i0,i01,i2,i3] * src1[i1,i01,i2,i3]\n\n    float * wdata = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32) * ith;\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        // dst indices\n        const int64_t i3 = ir/(ne2*ne1);\n        const int64_t i2 = (ir - i3*ne2*ne1)/ne1;\n        const int64_t i1 = (ir - i3*ne2*ne1 - i2*ne1);\n\n        const int64_t i02 = i2;\n        const int64_t i03 = i3;\n\n        //const int64_t i10 = i1;\n        const int64_t i12 = i2;\n        const int64_t i13 = i3;\n\n        for (int64_t i01 = 0; i01 < ne01; ++i01) {\n            const int64_t i11 = i01;\n\n            float * s0 = (float *) ((char *) src0->data + (          i01*nb01 + i02*nb02 + i03*nb03));\n            float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13));\n            float * d  = (float *) ((char *)  dst->data + (          i1*nb1 + i2*nb2 + i3*nb3));\n\n            dequantize_row_q(s0, wdata, ne0);\n            ggml_vec_mad_f32(ne0, d, wdata, *s1);\n        }\n    }\n}\n\nvoid ggml_compute_forward_out_prod(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n            {\n                ggml_compute_forward_out_prod_q_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                GGML_ABORT(\"fatal error\"); // todo\n                // ggml_compute_forward_out_prod_f16_f32(params, dst);\n            }\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_out_prod_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_scale\n\nstatic void ggml_compute_forward_scale_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n\n    float s; // scale factor\n    float b; // bias\n\n    memcpy(&s, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&b, (float *) dst->op_params + 1, sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    const size_t nb01 = src0->nb[1];\n\n    const size_t nb1 = dst->nb[1];\n\n    if (b == 0.0f) {\n        for (int i1 = ir0; i1 < ir1; i1++) {\n            if (dst->data != src0->data) {\n                // src0 is same shape as dst => same indices\n                // TODO: add x parameter to ggml_vec_scale_f32 and remove this memcpy\n                memcpy((char *)dst->data + i1*nb1, (char *)src0->data + i1*nb01, nc * sizeof(float));\n            }\n            ggml_vec_scale_f32(nc, (float *) ((char *) dst->data + i1*nb1), s);\n        }\n    } else {\n        for (int i1 = ir0; i1 < ir1; i1++) {\n            ggml_vec_mad1_f32(nc,\n                (float *) ((char *) dst->data  + i1*nb1),\n                (float *) ((char *) src0->data + i1*nb1),\n                s, b);\n        }\n    }\n}\n\nvoid ggml_compute_forward_scale(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_scale_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_set\n\nstatic void ggml_compute_forward_set_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n    // view src0 and dst with these strides and data offset inbytes during set\n    // nb0 is implicitly element_size because src0 and dst are contiguous\n    size_t nb1     = ((int32_t *) dst->op_params)[0];\n    size_t nb2     = ((int32_t *) dst->op_params)[1];\n    size_t nb3     = ((int32_t *) dst->op_params)[2];\n    size_t offset  = ((int32_t *) dst->op_params)[3];\n    bool   inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n    if (!inplace) {\n        if (params->ith == 0) {\n            // memcpy needs to be synchronized across threads to avoid race conditions.\n            // => do it in INIT phase\n            memcpy(\n                ((char *)  dst->data),\n                ((char *) src0->data),\n                ggml_nbytes(dst));\n        }\n        ggml_barrier(params->threadpool);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr = ggml_nrows(src1);\n    const int nc = src1->ne[0];\n\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb)\n\n    // src0 and dst as viewed during set\n    const size_t nb0 = ggml_element_size(src0);\n\n    const int im0 = (ne10 == 0 ? 0 : ne10-1);\n    const int im1 = (ne11 == 0 ? 0 : ne11-1);\n    const int im2 = (ne12 == 0 ? 0 : ne12-1);\n    const int im3 = (ne13 == 0 ? 0 : ne13-1);\n\n    GGML_ASSERT(offset + im0*nb0  + im1*nb1  + im2*nb2  + im3*nb3  <= ggml_nbytes(dst));\n\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are viewed with shape of src1 and offset\n        // => same indices\n        const int i3 = ir/(ne12*ne11);\n        const int i2 = (ir - i3*ne12*ne11)/ne11;\n        const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n        ggml_vec_cpy_f32(nc,\n                (float *) ((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1  + offset),\n                (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n    }\n}\n\nstatic void ggml_compute_forward_set_i32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n\n    // view src0 and dst with these strides and data offset inbytes during set\n    // nb0 is implicitly element_size because src0 and dst are contiguous\n    size_t nb1     = ((int32_t *) dst->op_params)[0];\n    size_t nb2     = ((int32_t *) dst->op_params)[1];\n    size_t nb3     = ((int32_t *) dst->op_params)[2];\n    size_t offset  = ((int32_t *) dst->op_params)[3];\n    bool   inplace = (bool) ((int32_t *) dst->op_params)[4];\n\n    if (!inplace) {\n        if (params->ith == 0) {\n            // memcpy needs to be synchronized across threads to avoid race conditions.\n            // => do it in INIT phase\n            memcpy(\n                ((char *)  dst->data),\n                ((char *) src0->data),\n                ggml_nbytes(dst));\n        }\n        ggml_barrier(params->threadpool);\n    }\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr = ggml_nrows(src1);\n    const int nc = src1->ne[0];\n\n    GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb1, src1, nb)\n\n    // src0 and dst as viewed during set\n    const size_t nb0 = ggml_element_size(src0);\n\n    const int im0 = (ne10 == 0 ? 0 : ne10-1);\n    const int im1 = (ne11 == 0 ? 0 : ne11-1);\n    const int im2 = (ne12 == 0 ? 0 : ne12-1);\n    const int im3 = (ne13 == 0 ? 0 : ne13-1);\n\n    GGML_ASSERT(offset + im0*nb0  + im1*nb1  + im2*nb2  + im3*nb3  <= ggml_nbytes(dst));\n\n    GGML_ASSERT(nb10 == sizeof(int32_t));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // src0 and dst are viewed with shape of src1 and offset\n        // => same indices\n        const int i3 = ir/(ne12*ne11);\n        const int i2 = (ir - i3*ne12*ne11)/ne11;\n        const int i1 = (ir - i3*ne12*ne11 - i2*ne11);\n\n        ggml_vec_cpy_i32(nc,\n                (int32_t *) ((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1  + offset),\n                (int32_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11));\n    }\n}\n\nvoid ggml_compute_forward_set(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_set_f32(params, dst);\n            } break;\n        case GGML_TYPE_I32:\n            {\n                ggml_compute_forward_set_i32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_Q8_1:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_cpy\n\nvoid ggml_compute_forward_cpy(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    ggml_compute_forward_dup(params, dst);\n}\n\n// ggml_compute_forward_cont\n\nvoid ggml_compute_forward_cont(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    ggml_compute_forward_dup(params, dst);\n}\n\n// ggml_compute_forward_get_rows\n\nstatic void ggml_compute_forward_get_rows_q(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int64_t nc = ne00;\n    const int64_t nr = ggml_nelements(src1);\n\n    const ggml_type type = src0->type;\n    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;\n\n    assert(ne0  == nc);\n    assert(ne02 == ne11);\n    assert(nb00 == ggml_type_size(type));\n    assert(ggml_nrows(dst) == nr);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t i = ir0; i < ir1; ++i) {\n        const int64_t i12 = i/(ne11*ne10);\n        const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n        const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n        const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n        GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n        dequantize_row_q(\n                (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n                     (float *) ((char *)  dst->data + i10*nb1  + i11*nb2  + i12*nb3), nc);\n    }\n}\n\nstatic void ggml_compute_forward_get_rows_f16(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int64_t nc = ne00;\n    const int64_t nr = ggml_nelements(src1);\n\n    assert(ne0  == nc);\n    assert(ne02 == ne11);\n    assert(nb00 == sizeof(ggml_fp16_t));\n    assert(ggml_nrows(dst) == nr);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t i = ir0; i < ir1; ++i) {\n        const int64_t i12 = i/(ne11*ne10);\n        const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n        const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n        const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n        GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n        ggml_cpu_fp16_to_fp32(\n            (const ggml_fp16_t*) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n                       (float *) ((char *)  dst->data + i10*nb1  + i11*nb2  + i12*nb3), nc);\n    }\n}\n\nstatic void ggml_compute_forward_get_rows_bf16(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int64_t nc = ne00;\n    const int64_t nr = ggml_nelements(src1);\n\n    assert(ne0  == nc);\n    assert(ne02 == ne11);\n    assert(nb00 == sizeof(ggml_bf16_t));\n    assert(ggml_nrows(dst) == nr);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t i = ir0; i < ir1; ++i) {\n        const int64_t i12 = i/(ne11*ne10);\n        const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n        const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n        const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n        GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n        ggml_cpu_bf16_to_fp32(\n            (const ggml_bf16_t *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),\n                        (float *) ((char *)  dst->data + i10*nb1  + i11*nb2  + i12*nb3), nc);\n    }\n}\n\nstatic void ggml_compute_forward_get_rows_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int64_t nc = ne00;\n    const int64_t nr = ggml_nelements(src1);\n\n    assert(ne0  == nc);\n    assert(ne02 == ne11);\n    assert(nb00 == sizeof(float));\n    assert(ggml_nrows(dst) == nr);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t i = ir0; i < ir1; ++i) {\n        const int64_t i12 = i/(ne11*ne10);\n        const int64_t i11 = (i - i12*ne11*ne10)/ne10;\n        const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);\n        const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n        GGML_ASSERT(i01 >= 0 && i01 < ne01);\n\n        ggml_vec_cpy_f32(nc,\n                (float *) ((char *)  dst->data + i10*nb1  + i11*nb2  + i12*nb3),\n                (float *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03));\n    }\n}\n\nvoid ggml_compute_forward_get_rows(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_Q8_1:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n            {\n                ggml_compute_forward_get_rows_q(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_get_rows_f16(params, dst);\n            } break;\n        case GGML_TYPE_BF16:\n            {\n                ggml_compute_forward_get_rows_bf16(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n        case GGML_TYPE_I32:\n            {\n                ggml_compute_forward_get_rows_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n\n    //static bool first = true;\n    //printf(\"ne0 = %d, ne1 = %d, ne2 = %d\\n\", dst->ne[0], dst->ne[1], dst->ne[2]);\n    //if (first) {\n    //    first = false;\n    //} else {\n    //    for (int k = 0; k < dst->ne[1]; ++k) {\n    //        for (int j = 0; j < dst->ne[0]/16; ++j) {\n    //            for (int i = 0; i < 16; ++i) {\n    //                printf(\"%8.4f \", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);\n    //            }\n    //            printf(\"\\n\");\n    //        }\n    //        printf(\"\\n\");\n    //    }\n    //    printf(\"\\n\");\n    //    exit(0);\n    //}\n}\n\ntemplate<typename idx_t>\nstatic void ggml_compute_forward_set_rows_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int64_t nc = ne00;\n    const int64_t nr = ne01;\n\n    assert(ne0  == nc);\n    assert(ne2  == ne02);\n    assert(ne3  == ne03);\n    assert(src0->type == GGML_TYPE_F32);\n    assert(ne02 % ne11 == 0);\n    assert(ne03 % ne12 == 0);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = std::min(ir0 + dr, nr);\n\n    ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;\n\n    for (int64_t i03 = 0; i03 < ne03; ++i03) {\n        for (int64_t i02 = 0; i02 < ne02; ++i02) {\n            for (int64_t i = ir0; i < ir1; ++i) {\n                const int64_t i12 = i03%ne12;\n                const int64_t i11 = i02%ne11;\n                const int64_t i10 = i;\n\n                const int64_t i1 = *(idx_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);\n\n                GGML_ASSERT(i1 >= 0 && i1 < ne1);\n\n                from_float(\n                        (const float *) ((char *) src0->data +  i*nb01 + i02*nb02 + i03*nb03),\n                                        ((char *)  dst->data + i1*nb1  + i02*nb2  + i03*nb3), nc);\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_set_rows(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                if (src1->type == GGML_TYPE_I64) {\n                    ggml_compute_forward_set_rows_f32<int64_t>(params, dst);\n                } else if (src1->type == GGML_TYPE_I32) {\n                    ggml_compute_forward_set_rows_f32<int32_t>(params, dst);\n                } else {\n                    GGML_ABORT(\"src1->type = %d (%s) not supported\", src1->type, ggml_type_name(src1->type));\n                }\n            } break;\n        default:\n            {\n                GGML_ABORT(\"src0->type = %d (%s) not supported\", src0->type, ggml_type_name(src0->type));\n            }\n    }\n}\n\n// ggml_compute_forward_get_rows_back\n\nstatic void ggml_compute_forward_get_rows_back_f32_f16(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(ggml_is_contiguous(dst));\n\n    // ggml_compute_forward_dup_same_cont(params, opt0, dst);\n\n    memset(dst->data, 0, ggml_nbytes(dst));\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nelements(src1);\n\n    GGML_ASSERT( dst->ne[0] == nc);\n    GGML_ASSERT(src0->nb[0] == sizeof(ggml_fp16_t));\n\n    for (int i = 0; i < nr; ++i) {\n        const int r = ((int32_t *) src1->data)[i];\n\n        for (int j = 0; j < nc; ++j) {\n            ggml_fp16_t v = ((ggml_fp16_t *) ((char *) src0->data + i*src0->nb[1]))[j];\n            ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_CPU_FP16_TO_FP32(v);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_get_rows_back_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    GGML_ASSERT(ggml_is_contiguous(dst));\n\n    // ggml_compute_forward_dup_same_cont(params, opt0, dst);\n\n    memset(dst->data, 0, ggml_nbytes(dst));\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nelements(src1);\n\n    GGML_ASSERT( dst->ne[0] == nc);\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    for (int i = 0; i < nr; ++i) {\n        const int r = ((int32_t *) src1->data)[i];\n\n        ggml_vec_add_f32(nc,\n                (float *) ((char *)  dst->data + r*dst->nb[1]),\n                (float *) ((char *)  dst->data + r*dst->nb[1]),\n                (float *) ((char *) src0->data + i*src0->nb[1]));\n    }\n}\n\nvoid ggml_compute_forward_get_rows_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_get_rows_back_f32_f16(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_get_rows_back_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n\n    //static bool first = true;\n    //printf(\"ne0 = %d, ne1 = %d, ne2 = %d\\n\", dst->ne[0], dst->ne[1], dst->ne[2]);\n    //if (first) {\n    //    first = false;\n    //} else {\n    //    for (int k = 0; k < dst->ne[1]; ++k) {\n    //        for (int j = 0; j < dst->ne[0]/16; ++j) {\n    //            for (int i = 0; i < 16; ++i) {\n    //                printf(\"%8.4f \", ((float *) dst->data)[k*dst->ne[0] + j*16 + i]);\n    //            }\n    //            printf(\"\\n\");\n    //        }\n    //        printf(\"\\n\");\n    //    }\n    //    printf(\"\\n\");\n    //    exit(0);\n    //}\n}\n\n// ggml_compute_forward_diag\n\nstatic void ggml_compute_forward_diag_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    // TODO: handle transposed/permuted matrices\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(ne00 == ne0);\n    GGML_ASSERT(ne00 == ne1);\n    GGML_ASSERT(ne01 == 1);\n    GGML_ASSERT(ne02 == ne2);\n    GGML_ASSERT(ne03 == ne3);\n\n    GGML_ASSERT(nb00 == sizeof(float));\n    GGML_ASSERT(nb0  == sizeof(float));\n\n    for (int i3 = 0; i3 < ne3; i3++) {\n        for (int i2 = 0; i2 < ne2; i2++) {\n            for (int i1 = 0; i1 < ne1; i1++) {\n                float * d = (float *)((char *)  dst->data + i3*nb3  + i2*nb2 + i1*nb1);\n                float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);\n                for (int i0 = 0; i0 < i1; i0++) {\n                    d[i0] = 0;\n                }\n                d[i1] = s[i1];\n                for (int i0 = i1+1; i0 < ne0; i0++) {\n                    d[i0] = 0;\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_diag(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_diag_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_diag_mask_inf\n\nstatic void ggml_compute_forward_diag_mask_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst,\n        const float value) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int  n_past  = ((int32_t *) dst->op_params)[0];\n    const bool inplace = src0->data == dst->data;\n\n    GGML_ASSERT(n_past >= 0);\n\n    if (!inplace) {\n        if (ith == 0) {\n            // memcpy needs to be synchronized across threads to avoid race conditions.\n            // => do it in INIT phase\n            GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));\n            GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));\n            memcpy(\n                ((char *)  dst->data),\n                ((char *) src0->data),\n                ggml_nbytes(dst));\n        }\n        ggml_barrier(params->threadpool);\n    }\n\n    // TODO: handle transposed/permuted matrices\n\n    const int n  = ggml_nrows(src0);\n    const int nc = src0->ne[0];\n    const int nr = src0->ne[1];\n    const int nz = n/nr;\n\n    GGML_ASSERT( dst->nb[0] == sizeof(float));\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    for (int k = 0; k < nz; k++) {\n        for (int j = ith; j < nr; j += nth) {\n            for (int i = n_past; i < nc; i++) {\n                if (i > n_past + j) {\n                    *(float *)((char *) dst->data + k*dst->nb[2] + j*dst->nb[1] + i*dst->nb[0]) = value;\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_diag_mask_inf(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_diag_mask_f32(params, dst, -INFINITY);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nvoid ggml_compute_forward_diag_mask_zero(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_diag_mask_f32(params, dst, 0);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_soft_max\n\nstatic void ggml_compute_forward_soft_max_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    const ggml_tensor * src2 = dst->src[2];\n\n    assert(ggml_is_contiguous(dst));\n    assert(ggml_are_same_shape(src0, dst));\n\n    float scale    = 1.0f;\n    float max_bias = 0.0f;\n\n    memcpy(&scale,    (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max_bias, (float *) dst->op_params + 1, sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int64_t nb11 = src1 ? src1->nb[1] : 1;\n    const int64_t nb12 = src1 ? src1->nb[2] : 1;\n    const int64_t nb13 = src1 ? src1->nb[3] : 1;\n\n    const int64_t ne12 = src1 ? src1->ne[2] : 1;\n    const int64_t ne13 = src1 ? src1->ne[3] : 1;\n\n    // TODO: is this supposed to be ceil instead of floor?\n    //       https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L370\n    const uint32_t n_head      = ne02;\n    const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n    const float m0 = powf(2.0f, -(max_bias       ) / n_head_log2);\n    const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n    float * wp = (float *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n    const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);\n\n    // sinks\n    const float * sk = src2 ? (float *)((char *) src2->data) : nullptr;\n\n    for (int64_t i03 = 0; i03 < ne03; i03++) {\n        for (int64_t i02 = 0; i02 < ne02; i02++) {\n            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {\n                const int64_t i11 = i01;\n                const int64_t i12 = i02%ne12;\n                const int64_t i13 = i03%ne13;\n\n                // ALiBi\n                const uint32_t h = i02; // head\n                const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n                float * sp = (float *)((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);\n                float * dp = (float *)((char *)  dst->data + i01*nb1  + i02*nb2  + i03*nb3);\n\n                // broadcast the mask across rows\n                ggml_fp16_t * mp_f16 = src1 ? (ggml_fp16_t *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL;\n                float       * mp_f32 = src1 ? (float       *)((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13) : NULL;\n\n                ggml_vec_cpy_f32  (ne00, wp, sp);\n                ggml_vec_scale_f32(ne00, wp, scale);\n                if (mp_f32) {\n                    if (use_f16) {\n                        for (int i = 0; i < ne00; ++i) {\n                            wp[i] += slope*GGML_CPU_FP16_TO_FP32(mp_f16[i]);\n                        }\n                    } else {\n                        for (int i = 0; i < ne00; ++i) {\n                            wp[i] += slope*mp_f32[i];\n                        }\n                    }\n                }\n\n#ifndef NDEBUG\n                for (int i = 0; i < ne00; ++i) {\n                    //printf(\"p[%d] = %f\\n\", i, p[i]);\n                    assert(!isnan(wp[i]));\n                }\n#endif // NDEBUG\n\n                float max = -INFINITY;\n                ggml_vec_max_f32(ne00, &max, wp);\n\n                // if we have sinks, make a correction as if they were included in the softmax\n                if (sk) {\n                    max = MAX(max, sk[i02]);\n                }\n\n                ggml_float sum = ggml_vec_soft_max_f32(ne00, dp, wp, max);\n                assert(sum > 0.0);\n\n                if (sk) {\n                    sum += (ggml_float) expf(sk[i02] - max);\n                }\n\n                sum = 1.0/sum;\n                ggml_vec_scale_f32(ne00, dp, sum);\n\n#ifndef NDEBUG\n                for (int i = 0; i < ne00; ++i) {\n                    assert(!isnan(dp[i]));\n                    assert(!isinf(dp[i]));\n                }\n#endif // NDEBUG\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_soft_max(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_soft_max_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n\n// ggml_compute_forward_soft_max_ext_back\n\nstatic void ggml_compute_forward_soft_max_ext_back_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n    GGML_ASSERT(ggml_is_contiguous(src1));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_are_same_shape(src0, dst));\n    GGML_ASSERT(ggml_are_same_shape(src1, dst));\n\n    float scale    = 1.0f;\n    float max_bias = 0.0f;\n\n    memcpy(&scale,    (const float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));\n\n    GGML_ASSERT(max_bias == 0.0f);\n\n    // TODO: handle transposed/permuted matrices\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc = src0->ne[0];\n    const int nr = ggml_nrows(src0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float *dy = (float *)((char *) src0->data + i1*src0->nb[1]);\n        float *y  = (float *)((char *) src1->data + i1*src1->nb[1]);\n        float *dx = (float *)((char *) dst->data  + i1*dst->nb[1]);\n\n#ifndef NDEBUG\n        for (int i = 0; i < nc; ++i) {\n            //printf(\"p[%d] = %f\\n\", i, p[i]);\n            assert(!isnan(dy[i]));\n            assert(!isnan(y[i]));\n        }\n#endif // NDEBUG\n        // Jii = yi - yi*yi\n        // Jij = -yi*yj\n        // J = diag(y)-y.T*y\n        // dx = J * dy\n        // dxk = sum_i(Jki * dyi)\n        // dxk = sum_i(-yk*yi * dyi) - (-yk*yk)*dyk + (yk - yk*yk)*dyk\n        // dxk = sum_i(-yk*yi * dyi) + yk*yk*dyk + yk*dyk - yk*yk*dyk\n        // dxk = sum_i(-yk*yi * dyi) + yk*dyk\n        // dxk = -yk * sum_i(yi * dyi) + yk*dyk\n        // dxk = -yk * dot(y, dy) + yk*dyk\n        // dxk = yk * (- dot(y, dy) + dyk)\n        // dxk = yk * (dyk - dot(y, dy))\n        //\n        // post-order:\n        // dot_y_dy := dot(y, dy)\n        // dx := dy\n        // dx := dx - dot_y_dy\n        // dx := dx * y\n\n        // linear runtime, no additional memory\n        float dot_y_dy = 0;\n        ggml_vec_dot_f32  (nc, &dot_y_dy, 0, y, 0, dy, 0, 1);\n        ggml_vec_cpy_f32  (nc, dx, dy);\n        ggml_vec_acc1_f32 (nc, dx, -dot_y_dy);\n        ggml_vec_mul_f32  (nc, dx, dx, y);\n        ggml_vec_scale_f32(nc, dx, scale);\n\n#ifndef NDEBUG\n        for (int i = 0; i < nc; ++i) {\n            assert(!isnan(dx[i]));\n            assert(!isinf(dx[i]));\n        }\n#endif // NDEBUG\n    }\n}\n\nvoid ggml_compute_forward_soft_max_ext_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_soft_max_ext_back_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_clamp\n\nstatic void ggml_compute_forward_clamp_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    float min;\n    float max;\n    memcpy(&min, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int n  = ggml_nrows(src0);\n    const int nc = src0->ne[0];\n\n    const size_t nb00 = src0->nb[0];\n    const size_t nb01 = src0->nb[1];\n\n    const size_t nb0 = dst->nb[0];\n    const size_t nb1 = dst->nb[1];\n\n    GGML_ASSERT( nb0 == sizeof(float));\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    for (int j = ith; j < n; j += nth) {\n        float * dst_ptr  = (float *) ((char *)  dst->data + j*nb1);\n        float * src0_ptr = (float *) ((char *) src0->data + j*nb01);\n\n        for (int i = 0; i < nc; i++) {\n            dst_ptr[i] = MAX(MIN(src0_ptr[i], max), min);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_clamp_f16(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    float min;\n    float max;\n    memcpy(&min, (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int n  = ggml_nrows(src0);\n    const int nc = src0->ne[0];\n\n    const size_t nb00 = src0->nb[0];\n    const size_t nb01 = src0->nb[1];\n\n    const size_t nb0 = dst->nb[0];\n    const size_t nb1 = dst->nb[1];\n\n    GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n\n    for (int j = ith; j < n; j += nth) {\n        ggml_fp16_t * dst_ptr  = (ggml_fp16_t *) ((char *)  dst->data + j*nb1);\n        ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + j*nb01);\n\n        for (int i = 0; i < nc; i++) {\n            float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);\n            dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));\n        }\n    }\n}\n\nvoid ggml_compute_forward_clamp(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_clamp_f32(params, dst);\n            } break;\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_clamp_f16(params, dst);\n            } break;\n        case GGML_TYPE_BF16:\n        case GGML_TYPE_Q4_0:\n        case GGML_TYPE_Q4_1:\n        case GGML_TYPE_Q5_0:\n        case GGML_TYPE_Q5_1:\n        case GGML_TYPE_Q8_0:\n        case GGML_TYPE_Q8_1:\n        case GGML_TYPE_MXFP4:\n        case GGML_TYPE_NVFP4:\n        case GGML_TYPE_Q2_K:\n        case GGML_TYPE_Q3_K:\n        case GGML_TYPE_Q4_K:\n        case GGML_TYPE_Q5_K:\n        case GGML_TYPE_Q6_K:\n        case GGML_TYPE_TQ1_0:\n        case GGML_TYPE_TQ2_0:\n        case GGML_TYPE_IQ2_XXS:\n        case GGML_TYPE_IQ2_XS:\n        case GGML_TYPE_IQ3_XXS:\n        case GGML_TYPE_IQ1_S:\n        case GGML_TYPE_IQ1_M:\n        case GGML_TYPE_IQ4_NL:\n        case GGML_TYPE_IQ4_XS:\n        case GGML_TYPE_IQ3_S:\n        case GGML_TYPE_IQ2_S:\n        case GGML_TYPE_Q8_K:\n        case GGML_TYPE_I8:\n        case GGML_TYPE_I16:\n        case GGML_TYPE_I32:\n        case GGML_TYPE_I64:\n        case GGML_TYPE_F64:\n        case GGML_TYPE_COUNT:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_rope\n\nstatic float rope_yarn_ramp(const float low, const float high, const int i0) {\n    const float y = (i0 / 2 - low) / MAX(0.001f, high - low);\n    return 1 - MIN(1, MAX(0, y));\n}\n\n// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn\n// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.\nstatic void rope_yarn(\n    float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,\n    float * cos_theta, float * sin_theta) {\n    // Get n-d rotational scaling corrected for extrapolation\n    float theta_interp = freq_scale * theta_extrap;\n    float theta = theta_interp;\n    if (ext_factor != 0.0f) {\n        float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;\n        theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;\n\n        // Get n-d magnitude scaling corrected for interpolation\n        mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);\n    }\n    *cos_theta = cosf(theta) * mscale;\n    *sin_theta = sinf(theta) * mscale;\n}\n\nstatic void ggml_rope_cache_init(\n     float theta_base, float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,\n     float * cache, float sin_sign, float theta_scale) {\n    // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py\n    float theta = theta_base;\n    for (int64_t i0 = 0; i0 < ne0; i0 += 2) {\n        const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;\n        rope_yarn(\n            theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]\n        );\n        cache[i0 + 1] *= sin_sign;\n\n        theta *= theta_scale;\n    }\n}\n\nstatic void ggml_mrope_cache_init(\n     float theta_base_t, float theta_base_h, float theta_base_w, float theta_base_e, int sections[4], bool is_imrope, bool indep_sects,\n     float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,\n     float * cache, float sin_sign, float theta_scale) {\n    // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py\n    float theta_t = theta_base_t;\n    float theta_h = theta_base_h;\n    float theta_w = theta_base_w;\n    float theta_e = theta_base_e;  // extra position id for vision encoder\n    int sect_dims = sections[0] + sections[1] + sections[2] + sections[3];\n    int sec_w = sections[1] + sections[0];\n    int sec_e = sections[2] + sec_w;\n    GGML_ASSERT(sect_dims <= ne0);\n\n    for (int64_t i0 = 0; i0 < ne0; i0 += 2) {\n        const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;\n\n        int sector = (i0 / 2) % sect_dims;\n        if (indep_sects) {\n            // compute theta independently for each dim sections\n            // (i.e. reset corresponding theta when `i0` go from one section to another)\n            if (sector == 0) {\n                theta_t = theta_base_t;\n            }\n            else if (sector == sections[0]) {\n                theta_h = theta_base_h;;\n            }\n            else if (sector == sec_w) {\n                theta_w = theta_base_w;\n            }\n            else if (sector == sec_e) {\n                theta_e = theta_base_e;\n            }\n        }\n\n        float theta = theta_t;\n        if (is_imrope) { // qwen3vl apply interleaved mrope\n            if (sector % 3 == 1 && sector < 3 * sections[1]) {\n                theta = theta_h;\n            } else if (sector % 3 == 2 && sector < 3 * sections[2]) {\n                theta = theta_w;\n            } else if (sector % 3 == 0 && sector < 3 * sections[0]) {\n                theta = theta_t;\n            } else {\n                theta = theta_e;\n            }\n        } else {\n            if (sector >= sections[0] && sector < sec_w) {\n                theta = theta_h;\n            }\n            else if (sector >= sec_w && sector < sec_w + sections[2]) {\n                theta = theta_w;\n            }\n            else if (sector >= sec_w + sections[2]) {\n                theta = theta_e;\n            }\n        }\n\n        rope_yarn(\n            theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]\n        );\n        cache[i0 + 1] *= sin_sign;\n\n        theta_t *= theta_scale;\n        theta_w *= theta_scale;\n        theta_h *= theta_scale;\n        theta_e *= theta_scale;\n    }\n}\n\n\ntemplate<typename T>\nstatic void rotate_pairs(const int64_t n, const int64_t n_offset, const float * cache, const T * src_data, T * dst_data, const int scale = 2) {\n  for (int64_t i0 = 0; i0 < n; i0 += 2) {\n    const int64_t ic = i0/scale; // hack for GGML_ROPE_TYPE_NORMAL, where we need ic = i0; for all other cases, ic = i0/2\n\n    const float cos_theta = cache[i0 + 0];\n    const float sin_theta = cache[i0 + 1];\n\n    const T * const src = src_data + ic;\n    T * dst             = dst_data + ic;\n\n    const float x0 = type_conversion_table<T>::to_f32(src[0]);\n    const float x1 = type_conversion_table<T>::to_f32(src[n_offset]);\n\n    dst[0]        = type_conversion_table<T>::from_f32(x0*cos_theta - x1*sin_theta);\n    dst[n_offset] = type_conversion_table<T>::from_f32(x0*sin_theta + x1*cos_theta);\n  }\n}\n\ntemplate<typename T> //float or ggml_fp16_t\nstatic void ggml_compute_forward_rope_flt(\n        const ggml_compute_params * params,\n        ggml_tensor * dst,\n        const bool forward) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    const ggml_tensor * src2 = dst->src[2];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_I32);\n\n    float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;\n    int sections[4];\n\n    //const int n_past     = ((int32_t *) dst->op_params)[0];\n    const int n_dims     = ((int32_t *) dst->op_params)[1];\n    const int mode       = ((int32_t *) dst->op_params)[2];\n    //const int n_ctx      = ((int32_t *) dst->op_params)[3];\n    const int n_ctx_orig = ((int32_t *) dst->op_params)[4];\n\n    memcpy(&freq_base,   (int32_t *) dst->op_params +  5, sizeof(float));\n    memcpy(&freq_scale,  (int32_t *) dst->op_params +  6, sizeof(float));\n    memcpy(&ext_factor,  (int32_t *) dst->op_params +  7, sizeof(float));\n    memcpy(&attn_factor, (int32_t *) dst->op_params +  8, sizeof(float));\n    memcpy(&beta_fast,   (int32_t *) dst->op_params +  9, sizeof(float));\n    memcpy(&beta_slow,   (int32_t *) dst->op_params + 10, sizeof(float));\n    memcpy(&sections,    (int32_t *) dst->op_params + 11, sizeof(int)*4);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    //printf(\"ne0: %d, ne1: %d, ne2: %d, ne3: %d\\n\", ne0, ne1, ne2, ne3);\n    //printf(\"n_past = %d, ne2 = %d\\n\", n_past, ne2);\n\n    GGML_ASSERT(nb0 == nb00);\n    GGML_ASSERT(nb0 == sizeof(T));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr = ggml_nrows(dst);\n\n    GGML_ASSERT(n_dims <= ne0);\n    GGML_ASSERT(n_dims % 2 == 0);\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    // row index used to determine which thread to use\n    int ir = 0;\n\n    const float theta_scale = powf(freq_base, -2.0f/n_dims);\n\n    float corr_dims[2];\n    ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);\n\n    const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE; // qwen3vl apply interleaved mrope\n    const bool mrope_used = mode & GGML_ROPE_TYPE_MROPE;  // ggml_rope_multi, note: also true for vision (24 & 8 == true) and for imrope\n    const bool is_vision = mode == GGML_ROPE_TYPE_VISION;\n\n    if (mrope_used) {\n        GGML_ASSERT(sections[0] > 0 || sections[1] > 0 || sections[2] > 0);\n    }\n\n    if (is_vision) {\n        GGML_ASSERT(n_dims == ne0/2);\n    }\n\n    const float * freq_factors = NULL;\n    if (src2 != NULL) {\n        GGML_ASSERT(src2->type == GGML_TYPE_F32);\n        GGML_ASSERT(src2->ne[0] >= n_dims / 2);\n        freq_factors = (const float *) src2->data;\n    }\n\n    // backward process uses inverse rotation by cos and sin.\n    // cos and sin build a rotation matrix, where the inverse is the transpose.\n    // this essentially just switches the sign of sin.\n    const float sin_sign = forward ? 1.0f : -1.0f;\n\n    const int32_t * pos = (const int32_t *) src1->data;\n\n    int64_t last_i2 = -1;\n\n    for (int64_t i3 = 0; i3 < ne3; i3++) { // batch\n        for (int64_t i2 = 0; i2 < ne2; i2++) { // seq-len\n            for (int64_t i1 = 0; i1 < ne1; i1++) { // attn-heads\n                if (ir++ < ir0) continue; // skip rows mapped to other threads\n                if (ir   > ir1) break;\n\n                float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;\n                if (last_i2 != i2) {\n                    if (!mrope_used) {\n                        const int64_t p = pos[i2];\n                        ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);\n                    }\n                    else {\n                        const int64_t p_t = pos[i2];\n                        const int64_t p_h = pos[i2 + ne2];\n                        const int64_t p_w = pos[i2 + ne2 * 2];\n                        const int64_t p_e = pos[i2 + ne2 * 3];\n                        ggml_mrope_cache_init(\n                            p_t, p_h, p_w, p_e, sections, is_imrope, is_vision,\n                            freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);\n                    }\n\n                    last_i2 = i2;\n                }\n\n                T * src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);\n                T * dst_data  = (T *)((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1);\n\n                switch (mode) {\n                    case GGML_ROPE_TYPE_NORMAL:\n                        rotate_pairs<T>(n_dims, 1, cache, src, dst_data, 1);\n                        break;\n                    case GGML_ROPE_TYPE_NEOX:\n                    case GGML_ROPE_TYPE_MROPE:\n                    case GGML_ROPE_TYPE_IMROPE:\n                        rotate_pairs<T>(n_dims, n_dims/2, cache, src, dst_data);\n                        break;\n                    case GGML_ROPE_TYPE_VISION:\n                        rotate_pairs<T>(ne0, n_dims, cache, src, dst_data);\n                        break;\n                    default:\n                        GGML_ABORT(\"rope type not supported\");\n                }\n\n                if (!is_vision) {\n                    // fill the remain channels with data from src tensor\n                    for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {\n                        const T * const src = (T *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);\n                        T * dst_data  = (T *)((char *)  dst->data + i3*nb3  + i2*nb2  + i1*nb1  + i0*nb0);\n\n                        dst_data[0] = src[0];\n                        dst_data[1] = src[1];\n                    }\n                }\n            } //attn-heads\n        }\n    }\n}\n\nvoid ggml_compute_forward_rope(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rope_flt<float>(params, dst, true);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_rope_back\n\nvoid ggml_compute_forward_rope_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rope_flt<float>(params, dst, false);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_conv_transpose_1d\n\nstatic void ggml_compute_forward_conv_transpose_1d_f16_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nk = ne00*ne01*ne02;\n\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    if (ith == 0) {\n        memset(params->wdata, 0, params->wsize);\n\n        // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)\n        {\n            ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i02*nb02 + i01*nb01);\n                    ggml_fp16_t * dst_data = wdata + i01*ne00*ne02;\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        dst_data[i00*ne02 + i02] = src[i00];\n                    }\n                }\n            }\n        }\n\n        // permute source data (src1) from (L x Cin) to (Cin x L)\n        {\n            ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;\n            ggml_fp16_t * dst_data = wdata;\n\n            for (int64_t i11 = 0; i11 < ne11; i11++) {\n                const float * const src = (float *)((char *) src1->data + i11*nb11);\n                for (int64_t i10 = 0; i10 < ne10; i10++) {\n                    dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10]);\n                }\n            }\n        }\n\n        // need to zero dst since we are accumulating into it\n        memset(dst->data, 0, ggml_nbytes(dst));\n    }\n    ggml_barrier(params->threadpool);\n\n    const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n\n    // total rows in dst\n    const int nr = ne1;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    ggml_fp16_t * const wdata     = (ggml_fp16_t *) params->wdata + 0;\n    ggml_fp16_t * const wdata_src = wdata + nk;\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * dst_data = (float *)((char *) dst->data + i1*nb1);\n        ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;\n        for (int i10 = 0; i10 < ne10; i10++) {\n            const int i1n = i10*ne11;\n            for (int i00 = 0; i00 < ne00; i00++) {\n                float v = 0;\n                ggml_vec_dot_f16(ne02, &v, 0,\n                        (ggml_fp16_t *)    wdata_src + i1n, 0,\n                        (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);\n                dst_data[i10*s0 + i00] += v;\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_conv_transpose_1d_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nk = ne00*ne01*ne02;\n\n    GGML_ASSERT(nb00 == sizeof(float));\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    if (ith == 0) {\n        memset(params->wdata, 0, params->wsize);\n\n        // prepare kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)\n        {\n            float * const wdata = (float *) params->wdata + 0;\n\n            for (int64_t i02 = 0; i02 < ne02; i02++) {\n                for (int64_t i01 = 0; i01 < ne01; i01++) {\n                    const float * const src = (float *)((char *) src0->data + i02*nb02 + i01*nb01);\n                    float * dst_data = wdata + i01*ne00*ne02;\n                    for (int64_t i00 = 0; i00 < ne00; i00++) {\n                        dst_data[i00*ne02 + i02] = src[i00];\n                    }\n                }\n            }\n        }\n\n        // prepare source data (src1)\n        {\n            float * const wdata = (float *) params->wdata + nk;\n            float * dst_data = wdata;\n\n            for (int64_t i11 = 0; i11 < ne11; i11++) {\n                const float * const src = (float *)((char *) src1->data + i11*nb11);\n                for (int64_t i10 = 0; i10 < ne10; i10++) {\n                    dst_data[i10*ne11 + i11] = src[i10];\n                }\n            }\n        }\n\n        // need to zero dst since we are accumulating into it\n        memset(dst->data, 0, ggml_nbytes(dst));\n    }\n    ggml_barrier(params->threadpool);\n\n    const int32_t s0 = ((const int32_t*)(dst->op_params))[0];\n\n    // total rows in dst\n    const int nr = ne1;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    float * const wdata     = (float *) params->wdata + 0;\n    float * const wdata_src = wdata + nk;\n\n    for (int i1 = ir0; i1 < ir1; i1++) {\n        float * dst_data = (float *)((char *) dst->data + i1*nb1);\n        float * wdata_kernel = wdata + i1*ne02*ne00;\n        for (int i10 = 0; i10 < ne10; i10++) {\n            const int i1n = i10*ne11;\n            for (int i00 = 0; i00 < ne00; i00++) {\n                float v = 0;\n                ggml_vec_dot_f32(ne02, &v, 0,\n                        wdata_src + i1n, 0,\n                        wdata_kernel + i00*ne02, 0, 1);\n                dst_data[i10*s0 + i00] += v;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_conv_transpose_1d(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_conv_transpose_1d_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_im2col_f32\n// src0: kernel [OC, IC, KH, KW]\n// src1: image [N, IC, IH, IW]\n// dst:  result [N, OH, OW, IC*KH*KW]\nstatic void ggml_compute_forward_im2col_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n    const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n    const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n    const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n    const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t N  = is_2D ? ne13 : ne12;\n    const int64_t IC = is_2D ? ne12 : ne11;\n    const int64_t IH = is_2D ? ne11 : 1;\n    const int64_t IW = ne10;\n\n    const int64_t KH = is_2D ? ne01 : 1;\n    const int64_t KW = ne00;\n\n    const int64_t OH = is_2D ? ne2 : 1;\n    const int64_t OW = ne1;\n\n    int ofs0 = is_2D ? nb13 : nb12;\n    int ofs1 = is_2D ? nb12 : nb11;\n\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n    {\n        float * const wdata = (float *) dst->data;\n\n        for (int64_t in = 0; in < N; in++) {\n            for (int64_t ioh = 0; ioh < OH; ioh++) { // 1\n                for (int64_t iow = 0; iow < OW; iow++) {\n                    for (int64_t iic = ith; iic < IC; iic += nth) {\n\n                        // micro kernel\n                        float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n                        const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]\n\n                        for (int64_t ikh = 0; ikh < KH; ikh++) {  // 1\n                            for (int64_t ikw = 0; ikw < KW; ikw++) {\n                                const int64_t iiw = iow*s0 + ikw*d0 - p0;\n                                const int64_t iih = ioh*s1 + ikh*d1 - p1;\n\n                                if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n                                    dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;\n                                } else {\n                                    dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\n\n// ggml_compute_forward_im2col_f16\n// src0: kernel [OC, IC, KH, KW]\n// src1: image [N, IC, IH, IW]\n// dst:  result [N, OH, OW, IC*KH*KW]\nstatic void ggml_compute_forward_im2col_f16(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F16);\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n    const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n    const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n    const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n    const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t N  = is_2D ? ne13 : ne12;\n    const int64_t IC = is_2D ? ne12 : ne11;\n    const int64_t IH = is_2D ? ne11 : 1;\n    const int64_t IW = ne10;\n\n    const int64_t KH = is_2D ? ne01 : 1;\n    const int64_t KW = ne00;\n\n    const int64_t OH = is_2D ? ne2 : 1;\n    const int64_t OW = ne1;\n\n    int ofs0 = is_2D ? nb13 : nb12;\n    int ofs1 = is_2D ? nb12 : nb11;\n\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n    // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n    {\n        ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;\n\n        for (int64_t in = 0; in < N; in++) {\n            for (int64_t ioh = 0; ioh < OH; ioh++) { // 1\n                for (int64_t iow = 0; iow < OW; iow++) {\n                    for (int64_t iic = ith; iic < IC; iic += nth) {\n\n                        // micro kernel\n                        ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n                        const float * const src_data_f32 = src1->type == GGML_TYPE_F32\n                            ? (const float *)((const char *) src1->data + in*ofs0 + iic*ofs1)\n                            : nullptr; // [IH, IW]\n                        const ggml_fp16_t * const src_data_f16 = src1->type == GGML_TYPE_F16\n                            ? (const ggml_fp16_t *)((const char *) src1->data + in*ofs0 + iic*ofs1)\n                            : nullptr; // [IH, IW]\n\n                        for (int64_t ikh = 0; ikh < KH; ikh++) {  // 1\n                            for (int64_t ikw = 0; ikw < KW; ikw++) {\n                                const int64_t iiw = iow*s0 + ikw*d0 - p0;\n                                const int64_t iih = ioh*s1 + ikh*d1 - p1;\n\n                                if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n                                    dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;\n                                } else {\n                                    if (src_data_f32 != nullptr) {\n                                        dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(src_data_f32[iih*IW + iiw]);\n                                    } else {\n                                        dst_data[iic*(KH*KW) + ikh*KW + ikw] = src_data_f16[iih*IW + iiw];\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_im2col(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n    switch (dst->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_im2col_f16(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_im2col_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_im2col_back_f32\n\nvoid ggml_compute_forward_im2col_back_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output\n    const ggml_tensor * src1 = dst->src[1]; // convolution kernel\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t p0 = ((const int32_t *)(dst->op_params))[2];\n    const int32_t p1 = ((const int32_t *)(dst->op_params))[3];\n    const int32_t d0 = ((const int32_t *)(dst->op_params))[4];\n    const int32_t d1 = ((const int32_t *)(dst->op_params))[5];\n    const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t N  = is_2D ? ne3 : ne2;\n    const int64_t IC = is_2D ? ne2 : ne1;\n    const int64_t IH = is_2D ? ne1 : 1;\n    const int64_t IW = ne0;\n\n    const int64_t KH = is_2D ? ne11 : 1;\n    const int64_t KW = ne10;\n\n    const int64_t OH = is_2D ? ne02 : 1;\n    const int64_t OW = ne01;\n\n    int ofs0 = is_2D ? nb3 : nb2;\n    int ofs1 = is_2D ? nb2 : nb1;\n\n    GGML_ASSERT(nb0  == sizeof(float));\n\n    // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]\n    {\n        float * const wdata = (float *) dst->data;\n\n        for (int64_t in = 0; in < N; in++) {\n            for (int64_t iic = ith; iic < IC; iic += nth) {\n                for (int64_t iih = 0; iih < IH; iih++) {\n                    for (int64_t iiw = 0; iiw < IW; iiw++) {\n\n                        // micro kernel\n                        float grad = 0.0f;\n                        for (int64_t ikh = 0; ikh < KH; ikh++) {\n                            for (int64_t ikw = 0; ikw < KW; ikw++) {\n                                // For s0 > 1 some values were skipped over in the forward pass.\n                                // These values have tmpw % s0 != 0 and need to be skipped in the backwards pass as well.\n                                const int64_t tmpw = (iiw + p0 - ikw*d0);\n                                if (tmpw % s0 != 0) {\n                                    continue;\n                                }\n                                const int64_t iow = tmpw / s0;\n\n                                // Equivalent logic as above except for s1.\n                                int64_t ioh;\n                                if (is_2D) {\n                                    const int64_t tmph = iih + p1 - ikh*d1;\n\n                                    if (tmph % s1 != 0) {\n                                        continue;\n                                    }\n\n                                    ioh = tmph / s1;\n                                } else {\n                                    ioh = 0;\n                                }\n\n                                if (iow < 0 || iow >= OW || ioh < 0 || ioh >= OH) {\n                                    continue;\n                                }\n\n                                const float * const grad_in = (const float *) src0->data\n                                    + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]\n                                grad += grad_in[iic*(KH*KW) + ikh*KW + ikw];\n                            }\n                        }\n                        float * dst_data = (float *)((char *) wdata + (in*ofs0 + iic*ofs1)); // [IH, IW]\n                        dst_data[iih*IW + iiw] = grad;\n                    }\n                }\n            }\n        }\n    }\n}\n\n\n// ggml_compute_forward_im2col_3d_f16\n// src0: kernel [OC*IC, KD, KH, KW]\n// src1: image [N*IC, ID, IH, IW]\n// dst:  result [N*OD, OH, OW, IC * KD * KH * KW]\nstatic void ggml_compute_forward_im2col_3d_f16(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F16);\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t s2 = ((const int32_t *)(dst->op_params))[2];\n    const int32_t p0 = ((const int32_t *)(dst->op_params))[3];\n    const int32_t p1 = ((const int32_t *)(dst->op_params))[4];\n    const int32_t p2 = ((const int32_t *)(dst->op_params))[5];\n    const int32_t d0 = ((const int32_t *)(dst->op_params))[6];\n    const int32_t d1 = ((const int32_t *)(dst->op_params))[7];\n    const int32_t d2 = ((const int32_t *)(dst->op_params))[8];\n    const int32_t IC = ((const int32_t *)(dst->op_params))[9];\n\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t N  = ne13 / IC;\n    const int64_t ID = ne12;\n    const int64_t IH = ne11;\n    const int64_t IW = ne10;\n\n    const int64_t OC = ne03 / IC;\n    GGML_UNUSED(OC);\n    const int64_t KD = ne02;\n    const int64_t KH = ne01;\n    const int64_t KW = ne00;\n\n    const int64_t OD = ne3 / N;\n    const int64_t OH = ne2;\n    const int64_t OW = ne1;\n    const int64_t OH_OW = OH*OW;\n    const int64_t KD_KH_KW = KD*KH*KW;\n    const int64_t KH_KW = KH*KW;\n    const int64_t IC_KD_KH_KW = IC*KD*KH*KW;\n\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\n    {\n        ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;\n\n        for (int64_t in = 0; in < N; in++) {\n            for (int64_t iod = 0; iod < OD; iod++) {\n                for (int64_t ioh = 0; ioh < OH; ioh++) {\n                    for (int64_t iow = 0; iow < OW; iow++) {\n                        for (int64_t iic = ith; iic < IC; iic += nth) {\n\n                            // micro kernel\n                            ggml_fp16_t * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]\n                            const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]\n\n                            for (int64_t ikd = 0; ikd < KD; ikd++) {\n                                for (int64_t ikh = 0; ikh < KH; ikh++) {\n                                    for (int64_t ikw = 0; ikw < KW; ikw++) {\n                                        const int64_t iiw = iow*s0 + ikw*d0 - p0;\n                                        const int64_t iih = ioh*s1 + ikh*d1 - p1;\n                                        const int64_t iid = iod*s2 + ikd*d2 - p2;\n\n                                        if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {\n                                            dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;\n                                        } else {\n                                            const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]\n                                            dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = GGML_CPU_FP32_TO_FP16(*s);\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\n// ggml_compute_forward_im2col_3d_f32\n// src0: kernel [OC*IC, KD, KH, KW]\n// src1: image [N*IC, ID, IH, IW]\n// dst:  result [N*OD, OH, OW, IC * KD * KH * KW]\nstatic void ggml_compute_forward_im2col_3d_f32(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    const int32_t s0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t s1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t s2 = ((const int32_t *)(dst->op_params))[2];\n    const int32_t p0 = ((const int32_t *)(dst->op_params))[3];\n    const int32_t p1 = ((const int32_t *)(dst->op_params))[4];\n    const int32_t p2 = ((const int32_t *)(dst->op_params))[5];\n    const int32_t d0 = ((const int32_t *)(dst->op_params))[6];\n    const int32_t d1 = ((const int32_t *)(dst->op_params))[7];\n    const int32_t d2 = ((const int32_t *)(dst->op_params))[8];\n    const int32_t IC = ((const int32_t *)(dst->op_params))[9];\n\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t N  = ne13 / IC;\n    const int64_t ID = ne12;\n    const int64_t IH = ne11;\n    const int64_t IW = ne10;\n\n    const int64_t OC = ne03 / IC;\n    GGML_UNUSED(OC);\n    const int64_t KD = ne02;\n    const int64_t KH = ne01;\n    const int64_t KW = ne00;\n\n    const int64_t OD = ne3 / N;\n    const int64_t OH = ne2;\n    const int64_t OW = ne1;\n\n    const int64_t OH_OW = OH*OW;\n    const int64_t KD_KH_KW = KD*KH*KW;\n    const int64_t KH_KW = KH*KW;\n    const int64_t IC_KD_KH_KW = IC*KD*KH*KW;\n\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    // im2col: [N*IC, ID, IH, IW] => [N*OD, OH, OW, IC * KD * KH * KW]\n    {\n        float * const wdata = (float *) dst->data;\n\n        for (int64_t in = 0; in < N; in++) {\n            for (int64_t iod = 0; iod < OD; iod++) {\n                for (int64_t ioh = 0; ioh < OH; ioh++) {\n                    for (int64_t iow = 0; iow < OW; iow++) {\n                        for (int64_t iic = ith; iic < IC; iic += nth) {\n\n                            // micro kernel\n                            float * dst_data = wdata + (in*OD*OH_OW + iod*OH_OW + ioh*OW + iow)*IC_KD_KH_KW; // [IC, KD, KH, KW]\n                            const float * const src_data = (const float *) ((const char *)src1->data + (in*IC + iic)*nb13); // [ID, IH, IW]\n\n                            for (int64_t ikd = 0; ikd < KD; ikd++) {\n                                for (int64_t ikh = 0; ikh < KH; ikh++) {\n                                    for (int64_t ikw = 0; ikw < KW; ikw++) {\n                                        const int64_t iiw = iow*s0 + ikw*d0 - p0;\n                                        const int64_t iih = ioh*s1 + ikh*d1 - p1;\n                                        const int64_t iid = iod*s2 + ikd*d2 - p2;\n\n                                        if (iid < 0 || iid >= ID || iih < 0 || iih >= IH || iiw < 0 || iiw >= IW || iid < 0 || iid >= ID) {\n                                            dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = 0;\n                                        } else {\n                                            const float * const s = (const float *) ((const char *)src_data + iid*nb12 + iih*nb11 + iiw*nb10); // [ID, IH, IW]\n                                            dst_data[iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw] = *s;\n                                        }\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\n\nvoid ggml_compute_forward_im2col_3d(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n    switch (dst->type) {\n        case GGML_TYPE_F16:\n            {\n                ggml_compute_forward_im2col_3d_f16(params, dst);\n            } break;\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_im2col_3d_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k,\n                              void * a, void * b, float * c) {\n    const ggml_type_traits * traits = ggml_get_type_traits(type);\n    struct ggml_tensor src1 = {};\n    src1.type  = type;\n    src1.ne[0] = k;\n    src1.ne[1] = m;\n    src1.ne[2] = 1;\n    src1.ne[3] = 1;\n    src1.nb[0] = traits->type_size;\n    src1.nb[1] = k * traits->type_size;\n    src1.nb[2] = src1.nb[1];\n    src1.nb[3] = src1.nb[2];\n    src1.data  = a;\n\n    struct ggml_tensor src0 = {};\n    src0.type  = type;\n    src0.ne[0] = k;\n    src0.ne[1] = n;\n    src0.ne[2] = 1;\n    src0.ne[3] = 1;\n    src0.nb[0] = traits->type_size;\n    src0.nb[1] = k * traits->type_size;\n    src0.nb[2] = src0.nb[1];\n    src0.nb[3] = src0.nb[2];\n    src0.data  = b;\n\n    struct ggml_tensor dst = {};\n    dst.ne[0] = n;\n    dst.ne[1] = m;\n    dst.ne[2] = 1;\n    dst.ne[3] = 1;\n    dst.nb[0] = sizeof(float);\n    dst.nb[1] = n * sizeof(float);\n    dst.nb[2] = dst.nb[1];\n    dst.nb[3] = dst.nb[2];\n    dst.data  = c;\n    dst.src[0] = &src0;\n    dst.src[1] = &src1;\n\n    ggml_compute_forward_mul_mat(params, &dst);\n}\n\nstatic inline int64_t ggml_wrap_around(int64_t coord, int64_t size) {\n    return (coord  + size) % size; // adding size avoids negative number weirdness\n}\n\n// ggml_compute_forward_conv_2d\n\n\nstatic void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params,\n                                              const ggml_tensor *         kernel,  // [KW, KH, IC, OC]\n                                              const ggml_tensor *         src,     // [W, H, C, N]\n                                              ggml_tensor *               dst,     // [OW, OH, OC, N]\n                                              ggml_type                   kernel_type) {\n\n    GGML_ASSERT(ggml_is_contiguous(kernel));\n    GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);\n    GGML_ASSERT(kernel->type == kernel_type);\n\n    const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);\n\n    const int32_t stride_x   = dst->op_params[0];\n    const int32_t stride_y   = dst->op_params[1];\n    const int32_t pad_x      = dst->op_params[2];\n    const int32_t pad_y      = dst->op_params[3];\n    const int32_t dilation_x = dst->op_params[4];\n    const int32_t dilation_y = dst->op_params[5];\n\n    const int64_t c_in  = src->ne[2];\n    const int64_t c_out = kernel->ne[3];\n    GGML_ASSERT(c_in == kernel->ne[2]);\n\n    const int64_t src_w = src->ne[0];\n    const int64_t src_h = src->ne[1];\n    const int64_t knl_w = kernel->ne[0];\n    const int64_t knl_h = kernel->ne[1];\n    const int64_t dst_w = dst->ne[0];\n    const int64_t dst_h = dst->ne[1];\n\n    const float * src_data = (float *) src->data;\n    void  * knl_data       = kernel->data;\n    float * dst_data       = (float *) dst->data;\n\n    const int64_t knl_n           = knl_w * knl_h * c_in;\n    const int64_t patch_total     = dst->ne[3] * dst_w * dst_h;\n\n    const int64_t space_per_patch   = knl_n * traits->type_size + c_out * sizeof(float);\n    const int64_t batch_size        = params->wsize / space_per_patch;\n    const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;\n    const int64_t batch_n           = (patch_total + patches_per_batch - 1) / patches_per_batch;\n\n    GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1);\n\n    void * tmp = params->wdata;\n\n    for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {\n\n        const int64_t patch_start_batch = batch_i * patches_per_batch;\n        const int64_t patch_end_batch   = std::min(patch_start_batch + patches_per_batch,\n                                              patch_total);\n        const int64_t patch_n           = patch_end_batch - patch_start_batch;\n\n        const int64_t patch_per_thread  = (patch_n + params->nth - 1) / params->nth;\n        const int64_t patch_start       = patch_start_batch + params->ith * patch_per_thread;\n        const int64_t patch_end         = std::min(patch_start + patch_per_thread, patch_end_batch);\n\n        //im2col for a patch\n        for (int64_t p = patch_start; p < patch_end; ++p) {\n            const int64_t  batch_n     =  p / (dst_w * dst_h);\n            const int64_t  src_x       = (p / dst_w) % dst_h;\n            const int64_t  src_y       =  p % dst_w;\n\n            const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]);\n            char *        dst_row  = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size;\n\n            for (int64_t ic = 0; ic < c_in; ++ic) {\n                for (int64_t ky = 0; ky < knl_h; ++ky) {\n                    for (int64_t kx = 0; kx < knl_w; ++kx) {\n                        const int64_t sy = src_x * stride_y + ky * dilation_y - pad_y;\n                        const int64_t sx = src_y * stride_x + kx * dilation_x - pad_x;\n\n                        int64_t dst_idx = ic * (knl_h * knl_w) + ky * knl_w + kx;\n\n                        float src_val;\n                        if (sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {\n                            src_val = 0.0f;\n                        } else {\n                            const float * src_ptr = (const float *)((const char *)src_base + sx * src->nb[0] + sy * src->nb[1] + ic * src->nb[2]);\n                            src_val               = *src_ptr;\n                        }\n\n                        char * element_ptr = dst_row + dst_idx * traits->type_size;\n                        if (kernel_type == GGML_TYPE_F32) {\n                            *(float *) element_ptr = src_val;\n                        } else if (kernel_type == GGML_TYPE_F16) {\n                            *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val);\n                        }\n                    }\n                }\n            }\n        }   // patches handled by this thread\n\n        ggml_barrier(params->threadpool);\n\n        float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size);\n\n        GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize);\n\n        // GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out]\n        ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output);\n\n        ggml_barrier(params->threadpool);\n\n\n        //permute back [OC, N, OH, OW] to [N, OC, OH, OW]\n        const int64_t permute_per_thread = (patch_n + params->nth - 1) / params->nth;\n        const int64_t permute_start = params->ith * permute_per_thread;\n        const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n);\n\n        for (int64_t i = permute_start; i < permute_end; ++i) {\n            const int64_t p       = patch_start_batch + i;\n            const int64_t batch_n = p / (dst_w * dst_h);\n            const int64_t dst_y   = (p / dst_w) % dst_h;\n            const int64_t dst_x   = p % dst_w;\n\n            for (int64_t oc = 0; oc < c_out; ++oc) {\n                const float value = gemm_output[i * c_out + oc];\n                float * dst_ptr = (float *)((char *)dst_data + dst_x * dst->nb[0] + dst_y * dst->nb[1] + oc * dst->nb[2] + batch_n * dst->nb[3]);\n                *dst_ptr = value;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_conv_2d(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    ggml_compute_forward_conv_2d_impl(params, src0, src1, dst, src0->type);\n}\n\n// ggml_compute_forward_conv_3d\n\nstatic void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params,\n                                              const ggml_tensor *         kernel,\n                                              const ggml_tensor *         src,\n                                              ggml_tensor *               dst,\n                                              ggml_type                   kernel_type) {\n\n    GGML_ASSERT(ggml_is_contiguous(kernel));\n    GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);\n    GGML_ASSERT(kernel->type == kernel_type);\n\n    const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);\n\n    const int32_t s0 = dst->op_params[0];\n    const int32_t s1 = dst->op_params[1];\n    const int32_t s2 = dst->op_params[2];\n    const int32_t p0 = dst->op_params[3];\n    const int32_t p1 = dst->op_params[4];\n    const int32_t p2 = dst->op_params[5];\n    const int32_t d0 = dst->op_params[6];\n    const int32_t d1 = dst->op_params[7];\n    const int32_t d2 = dst->op_params[8];\n    const int32_t c  = dst->op_params[9];\n    const int32_t n  = dst->op_params[10];\n    const int32_t oc = dst->op_params[11];\n\n    const int64_t src_w = src->ne[0];\n    const int64_t src_h = src->ne[1];\n    const int64_t src_d = src->ne[2];\n    const int64_t knl_w = kernel->ne[0];\n    const int64_t knl_h = kernel->ne[1];\n    const int64_t knl_d = kernel->ne[2];\n    const int64_t dst_w = dst->ne[0];\n    const int64_t dst_h = dst->ne[1];\n    const int64_t dst_d = dst->ne[2];\n\n    const float * src_data = (float *) src->data;\n    void  * knl_data       = kernel->data;\n    float * dst_data       = (float *) dst->data;\n\n    const int64_t knl_n_per_channel = knl_w * knl_h * knl_d;\n    const int64_t knl_n_total       = knl_n_per_channel * c;\n    const int64_t patch_total       = n * dst_w * dst_h * dst_d;\n\n    const int64_t space_per_patch   = knl_n_total * traits->type_size + oc * sizeof(float);\n    const int64_t batch_size        = params->wsize / space_per_patch;\n    const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;\n    const int64_t batch_n           = (patch_total + patches_per_batch - 1) / patches_per_batch;\n\n    GGML_ASSERT(patches_per_batch > 0 && batch_size >= 1);\n\n    void * tmp = params->wdata;\n\n    for (int64_t batch_i = 0; batch_i < batch_n; ++batch_i) {\n        const int64_t patch_start_batch = batch_i * patches_per_batch;\n        const int64_t patch_end_batch   = std::min(patch_start_batch + patches_per_batch, patch_total);\n        const int64_t patch_n_in_batch  = patch_end_batch - patch_start_batch;\n\n        const int64_t patch_per_thread  = (patch_n_in_batch + params->nth - 1) / params->nth;\n        const int64_t patch_start       = patch_start_batch + params->ith * patch_per_thread;\n        const int64_t patch_end         = std::min(patch_start + patch_per_thread, patch_end_batch);\n\n        for (int64_t p = patch_start; p < patch_end; ++p) {\n            const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);\n            const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);\n            const int64_t batch_idx  = p / (dst_w * dst_h * dst_d);\n            const int64_t dst_z      = p_in_batch / (dst_w * dst_h);\n            const int64_t dst_y      = p_in_depth / dst_w;\n            const int64_t dst_x      = p_in_depth % dst_w;\n\n            char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size;\n\n            for (int64_t ic = 0; ic < c; ++ic) {\n                for (int64_t kz = 0; kz < knl_d; ++kz) {\n                    for (int64_t ky = 0; ky < knl_h; ++ky) {\n                        for (int64_t kx = 0; kx < knl_w; ++kx) {\n                            const int64_t sz = dst_z * s2 + kz * d2 - p2;\n                            const int64_t sy = dst_y * s1 + ky * d1 - p1;\n                            const int64_t sx = dst_x * s0 + kx * d0 - p0;\n\n                            int64_t dst_idx = ic * knl_n_per_channel + kz * (knl_h * knl_w) + ky * knl_w + kx;\n\n                            float src_val;\n                            if (sz < 0 || sz >= src_d || sy < 0 || sy >= src_h || sx < 0 || sx >= src_w) {\n                                src_val = 0.0f;\n                            } else {\n                                const int64_t cn_idx = batch_idx * c + ic;\n                                const float * src_ptr = (const float *)((const char *)src_data + sx*src->nb[0] + sy*src->nb[1] + sz*src->nb[2] + cn_idx*src->nb[3]);\n                                src_val = *src_ptr;\n                            }\n\n                            char * element_ptr = dst_row + dst_idx * traits->type_size;\n                            if (kernel_type == GGML_TYPE_F32) {\n                                *(float *)element_ptr = src_val;\n                            } else if (kernel_type == GGML_TYPE_F16) {\n                                *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val);\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n        ggml_barrier(params->threadpool);\n\n        float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size);\n        ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output);\n\n        ggml_barrier(params->threadpool);\n\n        const int64_t permute_per_thread = (patch_n_in_batch + params->nth - 1) / params->nth;\n        const int64_t permute_start = params->ith * permute_per_thread;\n        const int64_t permute_end = std::min(permute_start + permute_per_thread, patch_n_in_batch);\n\n        for (int64_t i = permute_start; i < permute_end; ++i) {\n            const int64_t p = patch_start_batch + i;\n            const int64_t p_in_batch = p % (dst_w * dst_h * dst_d);\n            const int64_t p_in_depth = p_in_batch % (dst_w * dst_h);\n            const int64_t batch_idx  = p / (dst_w * dst_h * dst_d);\n            const int64_t dst_z      = p_in_batch / (dst_w * dst_h);\n            const int64_t dst_y      = p_in_depth / dst_w;\n            const int64_t dst_x      = p_in_depth % dst_w;\n\n            for (int64_t ioc = 0; ioc < oc; ++ioc) {\n                const float value = gemm_output[i * oc + ioc];\n                const int64_t ocn_idx = batch_idx * oc + ioc;\n                float * dst_ptr = (float *)((char *)dst_data + dst_x*dst->nb[0] + dst_y*dst->nb[1] + dst_z*dst->nb[2] + ocn_idx*dst->nb[3]);\n                *dst_ptr = value;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_conv_3d(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    ggml_compute_forward_conv_3d_impl(params, src0, src1, dst, src0->type);\n}\n\n// ggml_compute_forward_conv_transpose_2d\n\nvoid ggml_compute_forward_conv_transpose_2d(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F16);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nk = ne00*ne01*ne02*ne03;\n\n    GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));\n    GGML_ASSERT(nb10 == sizeof(float));\n\n    if (ith == 0) {\n        memset(params->wdata, 0, params->wsize);\n\n        // permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)\n        {\n            ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n\n            for (int64_t i03 = 0; i03 < ne03; i03++) {\n                for (int64_t i02 = 0; i02 < ne02; i02++) {\n                    const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i03*nb03 + i02*nb02);\n                    ggml_fp16_t * dst_data = wdata + i02*ne01*ne00*ne03;\n                    for (int64_t i01 = 0; i01 < ne01; i01++) {\n                        for (int64_t i00 = 0; i00 < ne00; i00++) {\n                            dst_data[i01*ne00*ne03 + i00*ne03 + i03] = src[i01 * ne00 + i00];\n                        }\n                    }\n                }\n            }\n        }\n\n        // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh)\n        {\n            ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;\n            for (int i12 = 0; i12 < ne12; i12++) {\n                for (int i11 = 0; i11 < ne11; i11++) {\n                    const float * const src = (float *)((char *) src1->data + i12*nb12 + i11*nb11);\n                    ggml_fp16_t * dst_data = wdata + i11*ne10*ne12;\n                    for (int i10 = 0; i10 < ne10; i10++) {\n                        dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);\n                    }\n                }\n            }\n        }\n\n        memset(dst->data, 0, ggml_nbytes(dst));\n    }\n    ggml_barrier(params->threadpool);\n\n    const int32_t stride = ggml_get_op_params_i32(dst, 0);\n\n    // total patches in dst\n    const int np = ne2;\n\n    // patches per thread\n    const int dp = (np + nth - 1)/nth;\n\n    // patch range for this thread\n    const int ip0 = dp*ith;\n    const int ip1 = MIN(ip0 + dp, np);\n\n    ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;\n    ggml_fp16_t * const wdata_src = wdata + nk;\n\n    for (int i2 = ip0; i2 < ip1; i2++) { // Cout\n        float * dst_data = (float *)((char *) dst->data + i2*nb2);\n        ggml_fp16_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;\n        for (int i11 = 0; i11 < ne11; i11++) {\n            for (int i10 = 0; i10 < ne10; i10++) {\n                const int i1n = i11*ne10*ne12 + i10*ne12;\n                for (int i01 = 0; i01 < ne01; i01++) {\n                    for (int i00 = 0; i00 < ne00; i00++) {\n                        float v = 0;\n                        ggml_vec_dot_f16(ne03, &v, 0,\n                                wdata_src + i1n, 0,\n                                wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);\n                        dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;\n                    }\n                }\n            }\n        }\n    }\n}\n\n// ggml_compute_forward_conv_2d_dw\n\nstruct ggml_conv_2d_dw_params {\n    int64_t channels;\n    int64_t batch;\n    int64_t src_w;\n    int64_t src_h;\n    int64_t dst_w;\n    int64_t dst_h;\n    int64_t knl_w;\n    int64_t knl_h;\n    int stride_x;\n    int stride_y;\n    int pad_x;\n    int pad_y;\n    int dilation_x;\n    int dilation_y;\n};\n\nstatic void ggml_compute_forward_conv_2d_dw_cwhn(\n        const ggml_compute_params * params,\n        const ggml_tensor * src,\n        const ggml_tensor * kernel,\n        ggml_tensor * dst,\n        const ggml_conv_2d_dw_params & p) {\n\n    const int64_t c = p.channels;\n    const float * knl_data = (const float *)kernel->data;\n\n    const int64_t rows_total = p.dst_h * p.batch;\n    const int64_t rows_per_thread = (rows_total + params->nth - 1) / params->nth;\n    const int64_t row_start = params->ith * rows_per_thread;\n    const int64_t row_end = MIN(row_start + rows_per_thread, rows_total);\n\n#ifdef GGML_SIMD\n    #if defined(__ARM_FEATURE_SVE)\n        const int64_t pkg_size = svcntw();\n    #else\n        const int64_t pkg_size = GGML_F32_EPR;\n    #endif\n    const int64_t pkg_count = c / pkg_size;\n    const int64_t c_pkg_end = pkg_count * pkg_size;\n#else\n    const int64_t c_pkg_end = 0;\n#endif\n\n    for (int64_t row = row_start; row < row_end; ++row) {\n        const int64_t dst_y = row % p.dst_h;\n        const float * src_data = (const float *)src->data + (row / p.dst_h) * p.src_w * p.src_h * c;\n        for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {\n            float * dst_data = (float *)dst->data + (row * p.dst_w + dst_x) * c;\n            const int64_t src_y_base = dst_y * p.stride_y - p.pad_y;\n            const int64_t src_x_base = dst_x * p.stride_x - p.pad_x;\n\n#ifdef GGML_SIMD\n            // Vectorized loop\n            for (int64_t c_i = 0; c_i < c_pkg_end; c_i += pkg_size) {\n                GGML_F32_VEC sum = GGML_F32_VEC_ZERO;\n                for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n                    const int64_t src_y = src_y_base + knl_y * p.dilation_y;\n                    if (src_y < 0 || src_y >= p.src_h) {\n                        continue;\n                    }\n                    for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n                        const int64_t src_x = src_x_base + knl_x * p.dilation_x;\n                        if (src_x < 0 || src_x >= p.src_w) {\n                            continue;\n                        }\n                        GGML_F32_VEC k = GGML_F32_VEC_LOAD(knl_data + (knl_y * p.knl_w + knl_x) * c + c_i);\n                        GGML_F32_VEC s = GGML_F32_VEC_LOAD(src_data + (src_y * p.src_w + src_x) * c + c_i);\n                        sum = GGML_F32_VEC_FMA(sum, k, s);\n                    }\n                }\n                GGML_F32_VEC_STORE(dst_data + c_i, sum);\n            }\n#endif\n            // Scalar loop\n            for (int64_t c_i = c_pkg_end; c_i < c; ++c_i) {\n                float sum = 0.0f;\n                for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n                    const int64_t src_y = src_y_base + knl_y * p.dilation_y;\n                    if (src_y < 0 || src_y >= p.src_h) {\n                        continue;\n                    }\n                    for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n                        const int64_t src_x = src_x_base + knl_x * p.dilation_x;\n                        if (src_x < 0 || src_x >= p.src_w) {\n                            continue;\n                        }\n                        sum += knl_data[(knl_y * p.knl_w + knl_x) * c + c_i]\n                             * src_data[(src_y * p.src_w + src_x) * c + c_i];\n                    }\n                }\n                dst_data[c_i] = sum;\n            }\n        }\n    }\n}\n\nstatic void ggml_compute_forward_conv_2d_dw_whcn(\n        const ggml_compute_params * params,\n        const ggml_tensor * src,\n        const ggml_tensor * kernel,\n        ggml_tensor * dst,\n        const ggml_conv_2d_dw_params & p) {\n\n    const int64_t n = p.channels * p.batch;\n    const int64_t per_thread = (n + params->nth - 1) / params->nth;\n    const int64_t start = params->ith * per_thread;\n    const int64_t end = MIN(start + per_thread, n);\n\n    for (int64_t i = start; i < end; ++i) {\n        const float * knl_data = (const float *)kernel->data + (i % p.channels) * p.knl_w * p.knl_h;\n        const float * src_data = (const float *)src->data + i * p.src_w * p.src_h;\n        float * dst_data = (float *)dst->data + i * p.dst_w * p.dst_h;\n\n        for (int64_t dst_y = 0; dst_y < p.dst_h; ++dst_y) {\n            for (int64_t dst_x = 0; dst_x < p.dst_w; ++dst_x) {\n\n                float sum = 0.0f;\n                for (int64_t knl_y = 0; knl_y < p.knl_h; ++knl_y) {\n                    const int64_t src_y = dst_y * p.stride_y + knl_y * p.dilation_y - p.pad_y;\n                    if (src_y < 0 || src_y >= p.src_h) {\n                        continue;\n                    }\n                    for (int64_t knl_x = 0; knl_x < p.knl_w; ++knl_x) {\n                        const int64_t src_x = dst_x * p.stride_x + knl_x * p.dilation_x - p.pad_x;\n                        if (src_x < 0 || src_x >= p.src_w) {\n                            continue;\n                        }\n                        sum += knl_data[knl_y * p.knl_w + knl_x]\n                             * src_data[src_y * p.src_w + src_x];\n                    }\n                }\n                dst_data[dst_y * p.dst_w + dst_x] = sum;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_conv_2d_dw(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * kernel = dst->src[0];\n    const ggml_tensor * src = dst->src[1];\n    ggml_conv_2d_dw_params p;\n    p.channels = src->ne[2];\n    p.batch = src->ne[3];\n    p.src_w = src->ne[0];\n    p.src_h = src->ne[1];\n    p.dst_w = dst->ne[0];\n    p.dst_h = dst->ne[1];\n    p.knl_w = kernel->ne[0];\n    p.knl_h = kernel->ne[1];\n    p.stride_x = dst->op_params[0];\n    p.stride_y = dst->op_params[1];\n    p.pad_x = dst->op_params[2];\n    p.pad_y = dst->op_params[3];\n    p.dilation_x = dst->op_params[4];\n    p.dilation_y = dst->op_params[5];\n\n    GGML_ASSERT(kernel->ne[3] == p.channels);\n    GGML_ASSERT(dst->ne[3] == p.batch);\n\n    if (ggml_is_contiguous(src)) {\n        ggml_compute_forward_conv_2d_dw_whcn(params, src, kernel, dst, p);\n    } else if (ggml_is_contiguous_channels(src)) {\n        // kernel should also have channels most contiguous in memory\n        GGML_ASSERT(kernel->nb[0] >= kernel->nb[2] && kernel->nb[1] >= kernel->nb[0]);\n        ggml_compute_forward_conv_2d_dw_cwhn(params, src, kernel, dst, p);\n    } else {\n        GGML_ABORT(\"non-contiguous memory layout not supported\");\n    }\n}\n\n// ggml_compute_forward_pool_1d_ksp\nstatic void ggml_compute_forward_pool_1d_ksp(\n        const ggml_compute_params * params,\n        const ggml_op_pool op,\n        const int k,\n        const int s,\n        const int p,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src = dst->src[0];\n\n    assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    const int64_t IW = src->ne[0];\n    const int64_t OW = dst->ne[0];\n\n    const int64_t nr = ggml_nrows(src);\n\n    for (int64_t ir = 0; ir < nr; ++ir) {\n        const char * srow_bytes =            (const char *) src->data + ir * src->nb[1];\n        float      * drow       = (float *) ((      char *) dst->data + ir * dst->nb[1]);\n\n        for (int64_t ow = 0; ow < OW; ++ow) {\n            float res = 0;\n            switch (op) {\n                case GGML_OP_POOL_AVG: res = 0.0f;     break;\n                case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n                case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n            }\n\n            int count = 0;\n            const int base = (int) ow * s - p;\n\n            for (int ki = 0; ki < k; ++ki) {\n                const int j = base + ki;\n                if (j < 0 || j >= (int) IW) {\n                    continue;\n                }\n\n                float v;\n                if (src->type == GGML_TYPE_F32) {\n                    v = ((const float *) srow_bytes)[j];\n                } else {\n                    v = GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) srow_bytes)[j]);\n                }\n\n                switch (op) {\n                    case GGML_OP_POOL_AVG: res += v;                break;\n                    case GGML_OP_POOL_MAX: res =  std::max(v, res); break;\n                    case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n                }\n\n                ++count;\n            }\n\n            switch (op) {\n                case GGML_OP_POOL_AVG: res = (count > 0) ? (res / count) : 0.0f; break;\n                case GGML_OP_POOL_MAX:                                           break;\n                case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n            }\n\n            drow[ow] = res;\n        }\n    }\n}\n\n// ggml_compute_forward_pool_1d\n\nvoid ggml_compute_forward_pool_1d(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const int32_t * opts = (const int32_t *)dst->op_params;\n    ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);\n    const int k0 = opts[1];\n    const int s0 = opts[2];\n    const int p0 = opts[3];\n\n    ggml_compute_forward_pool_1d_ksp(params, op, k0, s0, p0, dst);\n}\n\n// ggml_compute_forward_pool_2d\n\nvoid ggml_compute_forward_pool_2d(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src = dst->src[0];\n\n    assert(src->type == GGML_TYPE_F32 || src->type == GGML_TYPE_F16);\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    const int32_t * opts = (const int32_t *)dst->op_params;\n\n    ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);\n    const int k0 = opts[1];\n    const int k1 = opts[2];\n    const int s0 = opts[3];\n    const int s1 = opts[4];\n    const int p0 = opts[5];\n    const int p1 = opts[6];\n    const char * cdata = (const char*)src->data;\n    const char * const data_end = cdata + ggml_nbytes(src);\n\n    const int64_t px = dst->ne[0];\n    const int64_t py = dst->ne[1];\n    const int64_t pa = px * py;\n\n    float * dplane = (float *)dst->data;\n\n    const int ka = k0 * k1;\n    const int offset0 = -p0;\n    const int offset1 = -p1;\n\n    while (cdata < data_end) {\n        for (int oy = 0; oy < py; ++oy) {\n            float * const drow = dplane + oy * px;\n            float * const out  = drow;\n\n            for (int ox = 0; ox < px; ++ox) {\n                float res = 0;\n                switch (op) {\n                    case GGML_OP_POOL_AVG: res = 0;        break;\n                    case GGML_OP_POOL_MAX: res = -FLT_MAX; break;\n                    case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n                }\n\n                const int ix = offset0 + ox * s0;\n                const int iy = offset1 + oy * s1;\n\n                for (int ky = 0; ky < k1; ++ky) {\n                    if (iy + ky < 0 || iy + ky >= src->ne[1]) {\n                        continue;\n                    }\n\n                    const void * srow = (const void *)(cdata + src->nb[1] * (iy + ky));\n                    for (int kx = 0; kx < k0; ++kx) {\n                        int j = ix + kx;\n                        if (j < 0 || j >= src->ne[0]) {\n                            continue;\n                        }\n\n                        const float srow_j = (src->type == GGML_TYPE_F32) ? ((const float*)srow)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t*)srow)[j]);\n                        switch (op) {\n                            case GGML_OP_POOL_AVG: res += srow_j;                break;\n                            case GGML_OP_POOL_MAX: res =  std::max(srow_j, res); break;\n                            case GGML_OP_POOL_COUNT:               GGML_ABORT(\"fatal error\");\n                        }\n                    }\n                }\n                switch (op) {\n                    case GGML_OP_POOL_AVG:           res /= ka; break;\n                    case GGML_OP_POOL_MAX:                      break;\n                    case GGML_OP_POOL_COUNT: GGML_ABORT(\"fatal error\");\n                }\n\n                out[ox] = res;\n            }\n        }\n\n        cdata  += src->nb[2];\n        dplane += pa;\n    }\n}\n\n// ggml_compute_forward_pool_2d_back\n\nvoid ggml_compute_forward_pool_2d_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src  = dst->src[0];\n    const ggml_tensor * dstf = dst->src[1]; // forward tensor of dst\n\n    assert(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);\n\n    if (params->ith != 0) {\n        return;\n    }\n\n    const int32_t * opts = (const int32_t *)dst->op_params;\n    ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);\n    const int k0 = opts[1];\n    const int k1 = opts[2];\n    const int s0 = opts[3];\n    const int s1 = opts[4];\n    const int p0 = opts[5];\n    const int p1 = opts[6];\n\n    char       * cdata  = (char       *) dst->data;\n    const char * cdataf = (const char *) dstf->data;\n    const char * const data_end = cdata + ggml_nbytes(dst);\n\n    GGML_ASSERT(params->ith == 0);\n    memset(cdata, 0, ggml_nbytes(dst));\n\n    const int64_t px = src->ne[0];\n    const int64_t py = src->ne[1];\n    const int64_t pa = px * py;\n\n    const float * splane = (const float *) src->data;\n\n    const int ka = k0 * k1;\n    const int offset0 = -p0;\n    const int offset1 = -p1;\n\n    while (cdata < data_end) {\n        for (int oy = 0; oy < py; ++oy) {\n            const float * const srow = splane + oy * px;\n            for (int ox = 0; ox < px; ++ox) {\n                const float grad0 = srow[ox];\n\n                const int ix = offset0 + ox * s0;\n                const int iy = offset1 + oy * s1;\n\n                if (op == GGML_OP_POOL_MAX) {\n                    float maxval = -FLT_MAX;\n                    int kxmax = -1;\n                    int kymax = -1;\n\n                    for (int ky = 0; ky < k1; ++ky) {\n                        if (iy + ky < 0 || iy + ky >= dst->ne[1]) {\n                            continue;\n                        }\n                        const void * drowf = (const void *)(cdataf + dst->nb[1] * (iy + ky));\n                        for (int kx = 0; kx < k0; ++kx) {\n                            int j = ix + kx;\n                            if (j < 0 || j >= dst->ne[0]) {\n                                continue;\n                            }\n\n                            const float val = dst->type == GGML_TYPE_F32 ?\n                                ((const float *) drowf)[j] : GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drowf)[j]);\n                            if (val <= maxval) {\n                                continue;\n                            }\n\n                            maxval = val;\n                            kxmax = kx;\n                            kymax = ky;\n                        }\n                    }\n\n                    if (kxmax == -1 || kymax == -1) {\n                        continue;\n                    }\n\n                    void * drow = (void *)(cdata + dst->nb[1] * (iy + kymax));\n                    const int j = ix + kxmax;\n                    if (dst->type == GGML_TYPE_F32) {\n                        ((float *) drow)[j] += grad0;\n                    } else {\n                        ((ggml_fp16_t *) drow)[j] = GGML_CPU_FP32_TO_FP16(grad0 + GGML_CPU_FP16_TO_FP32(((const ggml_fp16_t *) drow)[j]));\n                    }\n                } else if (op == GGML_OP_POOL_AVG) {\n                    const float grad = grad0 / ka;\n\n                    for (int ky = 0; ky < k1; ++ky) {\n                        if (iy + ky < 0 || iy + ky >= dst->ne[1]) {\n                            continue;\n                        }\n                        void * drow = (void *)(cdata + dst->nb[1] * (iy + ky));\n                        for (int kx = 0; kx < k0; ++kx) {\n                            int j = ix + kx;\n                            if (j < 0 || j >= dst->ne[0]) {\n                                continue;\n                            }\n\n                            if (dst->type == GGML_TYPE_F32) {\n                                ((float *) drow)[j] += grad;\n                            } else {\n                                ((ggml_fp16_t *) drow)[j] += GGML_CPU_FP32_TO_FP16(grad);\n                            }\n                        }\n                    }\n                } else {\n                    GGML_ASSERT(false);\n                }\n            }\n        }\n\n        cdata  += dst->nb[2];\n        cdataf += dst->nb[2];\n        splane += pa;\n    }\n}\n\n// ggml_compute_forward_upscale\n\nstatic void ggml_compute_forward_upscale_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    float sf0 = (float)ne0/src0->ne[0];\n    float sf1 = (float)ne1/src0->ne[1];\n    float sf2 = (float)ne2/src0->ne[2];\n    float sf3 = (float)ne3/src0->ne[3];\n    float pixel_offset = 0.5f;\n\n    const int32_t mode_flags = ggml_get_op_params_i32(dst, 0);\n    const ggml_scale_mode mode = (ggml_scale_mode) (mode_flags & 0xFF);\n\n    if (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) {\n        pixel_offset = 0.0f;\n        sf0 = ne0 > 1 && ne00 > 1 ? (float)(ne0 - 1) / (ne00 - 1) : sf0;\n        sf1 = ne1 > 1 && ne01 > 1 ? (float)(ne1 - 1) / (ne01 - 1) : sf1;\n    }\n\n    if (mode == GGML_SCALE_MODE_NEAREST) {\n        for (int64_t i3 = 0; i3 < ne3; i3++) {\n            const int64_t i03 = i3 / sf3;\n            for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n                const int64_t i02 = i2 / sf2;\n                for (int64_t i1 = 0; i1 < ne1; i1++) {\n                    const int64_t i01 = i1 / sf1;\n                    for (int64_t i0 = 0; i0 < ne0; i0++) {\n                        const int64_t i00 = i0 / sf0;\n\n                        const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n                              float * y = (float *)((char *)  dst->data +  i0*nb0  +  i1*nb1  +  i2*nb2  +  i3*nb3);\n\n                        *y = *x;\n                    }\n                }\n            }\n        }\n    } else if (mode == GGML_SCALE_MODE_BILINEAR && (mode_flags & GGML_SCALE_FLAG_ANTIALIAS)) {\n        // Similar to F.interpolate(..., mode=\"bilinear\", align_corners=False, antialias=True)\n        // https://github.com/pytorch/pytorch/blob/8871ff29b743948d1225389d5b7068f37b22750b/aten/src/ATen/native/cpu/UpSampleKernel.cpp\n        auto triangle_filter = [](float x) -> float {\n            return std::max(1.0f - fabsf(x), 0.0f);\n        };\n\n        // support and invscale, minimum 1 pixel for bilinear\n        const float support1  = std::max(1.0f, 1.0f / sf1);\n        const float invscale1 = 1.0f / support1;\n        const float support0  = std::max(1.0f, 1.0f / sf0);\n        const float invscale0 = 1.0f / support0;\n\n        for (int64_t i3 = 0; i3 < ne3; i3++) {\n            const int64_t i03 = i3 / sf3;\n            for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n                const int64_t i02 = i2 / sf2;\n                for (int64_t i1 = 0; i1 < ne1; i1++) {\n                    const float y = ((float) i1 + pixel_offset) / sf1;\n                    for (int64_t i0 = 0; i0 < ne0; i0++) {\n                        const float x = ((float) i0 + pixel_offset) / sf0;\n\n                        // the range of source pixels that contribute\n                        const int64_t x_min = std::max<int64_t>(x - support0 + pixel_offset, 0);\n                        const int64_t x_max = std::min<int64_t>(x + support0 + pixel_offset, ne00);\n                        const int64_t y_min = std::max<int64_t>(y - support1 + pixel_offset, 0);\n                        const int64_t y_max = std::min<int64_t>(y + support1 + pixel_offset, ne01);\n\n                        // bilinear filter with antialiasing\n                        float val = 0.0f;\n                        float total_weight = 0.0f;\n\n                        for (int64_t sy = y_min; sy < y_max; sy++) {\n                            const float weight_y = triangle_filter((sy - y + pixel_offset) * invscale1);\n\n                            for (int64_t sx = x_min; sx < x_max; sx++) {\n                                const float weight_x = triangle_filter((sx - x + pixel_offset) * invscale0);\n                                const float weight = weight_x * weight_y;\n\n                                if (weight <= 0.0f) {\n                                    continue;\n                                }\n\n                                const float pixel = *(const float *)((const char *)src0->data + sx*nb00 + sy*nb01 + i02*nb02 + i03*nb03);\n                                val += pixel * weight;\n                                total_weight += weight;\n                            }\n                        }\n\n                        if (total_weight > 0.0f) {\n                            val /= total_weight;\n                        }\n\n                        float * dst_ptr = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n                        *dst_ptr = val;\n                    }\n                }\n            }\n        }\n    } else if (mode == GGML_SCALE_MODE_BILINEAR) {\n        for (int64_t i3 = 0; i3 < ne3; i3++) {\n            const int64_t i03 = i3 / sf3;\n            for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n                const int64_t i02 = i2 / sf2;\n                for (int64_t i1 = 0; i1 < ne1; i1++) {\n                    const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;\n                    int64_t y0 = (int64_t)floorf(y);\n                    int64_t y1 = y0 + 1;\n\n                    y0 = std::max(int64_t(0), std::min(y0, ne01 - 1));\n                    y1 = std::max(int64_t(0), std::min(y1, ne01 - 1));\n\n                    float dy = y - (float)y0;\n                    dy = std::max(0.0f, std::min(dy, 1.0f));\n\n                    for (int64_t i0 = 0; i0 < ne0; i0++) {\n                        const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;\n                        int64_t x0 = (int64_t)floorf(x);\n                        int64_t x1 = x0 + 1;\n\n                        x0 = std::max(int64_t(0), std::min(x0, ne00 - 1));\n                        x1 = std::max(int64_t(0), std::min(x1, ne00 - 1));\n\n                        float dx = x - (float)x0;\n                        dx = std::max(0.0f, std::min(dx, 1.0f));\n\n                        // fetch the four surrounding pixel values and interpolate\n                        const float a = *(const float *)((const char *)src0->data + x0*nb00 + y0*nb01 + i02*nb02 + i03*nb03);\n                        const float b = *(const float *)((const char *)src0->data + x1*nb00 + y0*nb01 + i02*nb02 + i03*nb03);\n                        const float c = *(const float *)((const char *)src0->data + x0*nb00 + y1*nb01 + i02*nb02 + i03*nb03);\n                        const float d = *(const float *)((const char *)src0->data + x1*nb00 + y1*nb01 + i02*nb02 + i03*nb03);\n\n                        const float val = a*(1 - dx)*(1 - dy) + b*dx*(1 - dy) + c*(1 - dx)*dy + d*dx*dy;\n\n                        float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n                        *y_dst = val;\n                    }\n                }\n            }\n        }\n    } else if (mode == GGML_SCALE_MODE_BICUBIC) {\n        // https://en.wikipedia.org/wiki/Bicubic_interpolation#Bicubic_convolution_algorithm\n        const float a = -0.75f; // use alpha = -0.75 (same as PyTorch)\n        auto weight1 = [a](float x) { return ((a + 2) * x - (a + 3)) * x * x + 1; };\n        auto weight2 = [a](float x) { return ((a * x - 5 * a) * x + 8 * a) * x - 4 * a; };\n        auto bicubic = [=](float p0, float p1, float p2, float p3, float x) {\n            const float w0 = weight2(x + 1);\n            const float w1 = weight1(x + 0);\n            const float w2 = weight1(1 - x);\n            const float w3 = weight2(2 - x);\n            return p0*w0 + p1*w1 + p2*w2 + p3*w3;\n        };\n\n        for (int64_t i3 = 0; i3 < ne3; i3++) {\n            const int64_t i03 = i3 / sf3;\n            for (int64_t i2 = ith; i2 < ne2; i2 += nth) {\n                const int64_t i02 = i2 / sf2;\n                for (int64_t i1 = 0; i1 < ne1; i1++) {\n                    const float y = ((float)i1 + pixel_offset) / sf1 - pixel_offset;\n                    const int64_t y0 = (int64_t)floorf(y);\n                    const float dy = y - (float)y0;\n\n                    for (int64_t i0 = 0; i0 < ne0; i0++) {\n                        const float x = ((float)i0 + pixel_offset) / sf0 - pixel_offset;\n                        const int64_t x0 = (int64_t)floorf(x);\n                        const float dx = x - (float)x0;\n\n                        auto p = [=](int64_t x_off, int64_t y_off) -> float {\n                            int64_t i00 = std::max(int64_t(0), std::min(x0 + x_off, ne00 - 1));\n                            int64_t i01 = std::max(int64_t(0), std::min(y0 + y_off, ne01 - 1));\n                            return *(const float *)((const char *)src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);\n                        };\n\n                        const float val = bicubic(\n                            bicubic(p(-1,-1), p(0,-1), p(1,-1), p(2,-1), dx),\n                            bicubic(p(-1, 0), p(0, 0), p(1, 0), p(2, 0), dx),\n                            bicubic(p(-1, 1), p(0, 1), p(1, 1), p(2, 1), dx),\n                            bicubic(p(-1, 2), p(0, 2), p(1, 2), p(2, 2), dx), dy);\n\n                        float * y_dst = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);\n                        *y_dst = val;\n                    }\n                }\n            }\n        }\n    } else {\n        GGML_ABORT(\"unsupported upscale mode\");\n    }\n}\n\nvoid ggml_compute_forward_upscale(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_upscale_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n\n// ggml_compute_forward_pad\n\ntemplate<bool circular_t>\nstatic void ggml_compute_forward_pad_f32(\n    const ggml_compute_params * params,\n          ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    assert(dst->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    float * dst_ptr = (float *) dst->data;\n    const int32_t lp0 = ggml_get_op_params_i32(dst, 0);\n    const int32_t rp0 = ggml_get_op_params_i32(dst, 1);\n    const int32_t lp1 = ggml_get_op_params_i32(dst, 2);\n    const int32_t rp1 = ggml_get_op_params_i32(dst, 3);\n    const int32_t lp2 = ggml_get_op_params_i32(dst, 4);\n    const int32_t rp2 = ggml_get_op_params_i32(dst, 5);\n    const int32_t lp3 = ggml_get_op_params_i32(dst, 6);\n    const int32_t rp3 = ggml_get_op_params_i32(dst, 7);\n\n    // TODO: optimize\n\n    for (int64_t i2 = 0; i2 < ne2; ++i2) {\n        for (int64_t i1 = ith; i1 < ne1; i1 += nth) {\n            for (int64_t i0 = 0; i0 < ne0; ++i0) {\n                for (int64_t i3 = 0; i3 < ne3; ++i3) {\n                    // circular means wrap around on a torus, so x and y loop around\n                    if constexpr (circular_t) {\n                        const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;\n                        const int64_t src_i0 = ggml_wrap_around(i0 - lp0, ne00);\n                        const int64_t src_i1 = ggml_wrap_around(i1 - lp1, ne01);\n                        const int64_t src_i2 = ggml_wrap_around(i2 - lp2, ne02);\n                        const int64_t src_i3 = ggml_wrap_around(i3 - lp3, ne03);\n\n                        const int64_t src_idx =\n                            src_i3*nb03 +\n                            src_i2*nb02 +\n                            src_i1*nb01 +\n                            src_i0*nb00;\n\n                        const float * src_ptr = (const float *)((char *) src0->data + src_idx);\n                        dst_ptr[dst_idx] = *src_ptr;\n                    } else {\n                        const int64_t dst_idx = i3*(ne0*ne1*ne2) + i2*(ne0*ne1) + i1*ne0 + i0;\n                        if ((i0 >= lp0 && i0 < ne0 - rp0) \\\n                            && (i1 >= lp1 && i1 < ne1 - rp1) \\\n                            && (i2 >= lp2 && i2 < ne2 - rp2) \\\n                            && (i3 >= lp3 && i3 < ne3 - rp3)) {\n                            const int64_t src_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;\n                            const float * src_ptr = (const float *)((char *) src0->data + src_idx);\n                            dst_ptr[dst_idx] = *src_ptr;\n                        } else {\n                            dst_ptr[dst_idx] = 0;\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\n\nvoid ggml_compute_forward_pad(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const bool circular = (bool) ggml_get_op_params_i32(dst, 8);\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                if (circular) {\n                    ggml_compute_forward_pad_f32<true>(params, dst);\n                } else {\n                    ggml_compute_forward_pad_f32<false>(params, dst);\n                }\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_pad_reflect_1d\n\nvoid ggml_compute_forward_pad_reflect_1d(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int32_t * opts = (const int32_t *) dst->op_params;\n    const int p0 = opts[0];\n    const int p1 = opts[1];\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    for (int64_t i3 = 0; i3 < ne3; i3++) {\n        for (int64_t i2 = 0; i2 < ne2; i2++) {\n            for (int64_t i1 = ith; i1 < ne1; i1 += nth) {\n                float * left  = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 +         p0*nb0);\n                float * right = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + (ne0-p1-1)*nb0);\n\n                ggml_vec_cpy_f32(ne00, left, (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01));\n\n                for (int i0 = 1; i0 <= p0; i0++) { left[-i0] = left[i0];   }\n                for (int i0 = 1; i0 <= p1; i0++) { right[i0] = right[-i0]; }\n            }\n        }\n    }\n}\n\n// ggml_compute_forward_roll\n\nstatic int64_t ggml_wrap_index(int64_t i, int64_t ne) {\n    if (i < 0) {\n        return i + ne;\n    } else if (i >= ne) {\n        return i - ne;\n    }\n    return i;\n}\n\nstatic void ggml_compute_forward_roll_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const float * src_data = (const float *) src0->data;\n    float * dst_data = (float *) dst->data;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int s0 = ggml_get_op_params_i32(dst, 0);\n    const int s1 = ggml_get_op_params_i32(dst, 1);\n    const int s2 = ggml_get_op_params_i32(dst, 2);\n    const int s3 = ggml_get_op_params_i32(dst, 3);\n\n    const int64_t total = ne1 * ne2 * ne3;\n    const int64_t per_thread = (total + params->nth) / params->nth;\n    const int64_t start = params->ith * per_thread;\n    const int64_t end   = std::min(start + per_thread, total);\n\n    for (int64_t i = start; i < end; ++i) {\n        const int64_t i1 = i % ne1;\n        const int64_t i2 = (i / ne1) % ne2;\n        const int64_t i3 = i / (ne2 * ne1);\n        float * dst_row = dst_data + (i3*nb3 + i2*nb2 + i1*nb1) / sizeof(float);\n\n        const int64_t i01 = ggml_wrap_index(i1 - s1, ne01);\n        const int64_t i02 = ggml_wrap_index(i2 - s2, ne02);\n        const int64_t i03 = ggml_wrap_index(i3 - s3, ne03);\n        const float * src_row = src_data + (i03*nb03 + i02*nb02 + i01*nb01) / sizeof(float);\n\n        const int64_t s = ggml_wrap_index(-s0, ne00);\n        const int64_t n = ne00 - s;\n        ggml_vec_cpy_f32(n, dst_row,     src_row + s);\n        ggml_vec_cpy_f32(s, dst_row + n, src_row);\n    }\n}\n\nvoid ggml_compute_forward_roll(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_roll_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_arange\n\nstatic void ggml_compute_forward_arange_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    GGML_ASSERT(dst->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const float start = ggml_get_op_params_f32(dst, 0);\n    const float stop  = ggml_get_op_params_f32(dst, 1);\n    const float step  = ggml_get_op_params_f32(dst, 2);\n\n    const int64_t steps = (int64_t) ceilf((stop - start) / step);\n\n    GGML_ASSERT(ggml_nelements(dst) == steps);\n\n    for (int64_t i = ith; i < steps; i+= nth) {\n        float value = start + step * i;\n        ((float *)dst->data)[i] = value;\n    }\n}\n\nvoid ggml_compute_forward_arange(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n    switch (dst->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_arange_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_timestep_embedding_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int dim = ggml_get_op_params_i32(dst, 0);\n    const int max_period = ggml_get_op_params_i32(dst, 1);\n\n    int half = dim / 2;\n\n    for (int64_t i = 0; i < ne00; i++) {\n        float * embed_data = (float *)((char *)  dst->data +  i*nb1);\n        for (int64_t j = ith; j < half; j += nth) {\n            float timestep = ((float *)src0->data)[i];\n            float freq = (float)expf(-logf(max_period) * j / half);\n            float arg = timestep * freq;\n            embed_data[j] = cosf(arg);\n            embed_data[j + half] = sinf(arg);\n        }\n        if (dim % 2 != 0 && ith == 0) {\n            embed_data[2 * half] = 0.f;\n        }\n    }\n}\n\nvoid ggml_compute_forward_timestep_embedding(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_timestep_embedding_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_argsort\n\ntemplate<enum ggml_sort_order order>\nstruct cmp_argsort {\n    const float * data;\n    bool operator()(int32_t a, int32_t b) const {\n        if constexpr (order == GGML_SORT_ORDER_ASC) {\n            return data[a] < data[b];\n        } else {\n            return data[a] > data[b];\n        }\n    }\n};\n\nstatic void ggml_compute_forward_argsort_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(nb0 == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t nr = ggml_nrows(src0);\n\n    ggml_sort_order order = (ggml_sort_order) ggml_get_op_params_i32(dst, 0);\n\n    for (int64_t i = ith; i < nr; i += nth) {\n        const float * src_data = (float *)((char *) src0->data + i*nb01);\n\n        int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);\n\n        for (int64_t j = 0; j < ne0; j++) {\n            dst_data[j] = j;\n        }\n\n        switch (order) {\n            case GGML_SORT_ORDER_ASC:\n                std::sort(dst_data, dst_data + ne0, cmp_argsort<GGML_SORT_ORDER_ASC>{src_data});\n                break;\n\n            case GGML_SORT_ORDER_DESC:\n                std::sort(dst_data, dst_data + ne0, cmp_argsort<GGML_SORT_ORDER_DESC>{src_data});\n                break;\n\n            default:\n                GGML_ABORT(\"invalid sort order\");\n        }\n    }\n}\n\nvoid ggml_compute_forward_argsort(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_argsort_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_top_k\n\nstruct cmp_top_k {\n    const float * data;\n    bool operator()(int32_t a, int32_t b) const {\n        return data[a] > data[b];\n    }\n};\n\nstatic void ggml_compute_forward_top_k_f32(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT(nb0 == sizeof(float));\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t nr = ggml_nrows(src0);\n\n    const int top_k = ne0;\n\n    int32_t * tmp = (int32_t *) params->wdata + (ne00 + CACHE_LINE_SIZE_F32) * ith;\n\n    for (int64_t i = ith; i < nr; i += nth) {\n        const float * src_data = (float *)((char *) src0->data + i*nb01);\n\n        for (int64_t j = 0; j < ne00; j++) {\n            tmp[j] = j;\n        }\n\n        std::partial_sort(tmp, tmp + top_k, tmp + ne00, cmp_top_k{src_data});\n\n        int32_t * dst_data = (int32_t *)((char *) dst->data + i*nb1);\n\n        std::copy(tmp, tmp + top_k, dst_data);\n\n        // emphasize that the order is not important\n        if (top_k > 1) {\n            std::swap(dst_data[0], dst_data[1]);\n        }\n    }\n}\n\nvoid ggml_compute_forward_top_k(\n    const ggml_compute_params * params,\n    ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_top_k_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_f16_one_chunk(\n        const ggml_compute_params * params,\n        ggml_tensor * dst,\n        int ir0, int ir1,\n        int64_t ic_start, int64_t ic_end,\n        float * partials, int64_t partial_stride) {\n\n    const bool write_partials = (partials != nullptr);\n    const ggml_tensor * q     = dst->src[0];\n    const ggml_tensor * k     = dst->src[1];\n    const ggml_tensor * v     = dst->src[2];\n    const ggml_tensor * mask  = dst->src[3];\n    const ggml_tensor * sinks = dst->src[4];\n\n    GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nek, k,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbk, k,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nev, v,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbv, v,   nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst, nb)\n\n    const int64_t DK = nek0;\n    const int64_t DV = nev0;\n    const int64_t N  = neq1;\n\n    GGML_ASSERT(ne0 == DV);\n    GGML_ASSERT(ne2 == N);\n\n    // input tensor rows must be contiguous\n    GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n    GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n    GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n    GGML_ASSERT(neq0 == DK);\n    GGML_ASSERT(nek0 == DK);\n    GGML_ASSERT(nev0 == DV);\n\n    GGML_ASSERT(neq1 == N);\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    // broadcast factors\n    const int64_t rk2 = neq2/nek2;\n    const int64_t rk3 = neq3/nek3;\n\n    const int64_t rv2 = neq2/nev2;\n    const int64_t rv3 = neq3/nev3;\n\n    // parallelize by q rows using ggml_vec_dot_f32\n\n    float scale         = 1.0f;\n    float max_bias      = 0.0f;\n    float logit_softcap = 0.0f;\n\n    memcpy(&scale,         (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max_bias,      (float *) dst->op_params + 1, sizeof(float));\n    memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));\n\n    if (logit_softcap != 0) {\n        scale /= logit_softcap;\n    }\n\n    const uint32_t n_head      = neq2;\n    const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n    const float m0 = powf(2.0f, -(max_bias       ) / n_head_log2);\n    const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n    ggml_type         const k_vec_dot_type = ggml_get_type_traits_cpu(k->type)->vec_dot_type;\n    ggml_from_float_t const q_to_vec_dot   = ggml_get_type_traits_cpu(k_vec_dot_type)->from_float;\n    ggml_vec_dot_t    const kq_vec_dot     = ggml_get_type_traits_cpu(k->type)->vec_dot;\n    ggml_to_float_t   const v_to_float     = ggml_get_type_traits(v->type)->to_float;\n\n    GGML_ASSERT((                            q_to_vec_dot) && \"fattn: unsupported K-type\");\n    GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float  ) && \"fattn: unsupported V-type\");\n\n    int ith = params->ith;\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // q indices\n        const int iq3 = ir/(neq2*neq1);\n        const int iq2 = (ir - iq3*neq2*neq1)/neq1;\n        const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);\n\n        const uint32_t h = iq2; // head index\n        const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n        float S = 0.0f;      // sum\n        float M = -INFINITY; // maximum KQ value\n\n        float       * VKQ32 = (float       *) params->wdata + ith*(1*DK + 2*DV + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator\n        float       * V32   =                 (VKQ32 + 1*DV); // (temporary) FP32 V buffer\n        ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator\n        ggml_fp16_t * Q_q   = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16\n\n        if (v->type == GGML_TYPE_F16) {\n            memset(VKQ16, 0, DV*sizeof(ggml_fp16_t));\n        } else {\n            memset(VKQ32, 0, DV*sizeof(float));\n        }\n\n        const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;\n\n        // k indices\n        const int ik3 = iq3 / rk3;\n        const int ik2 = iq2 / rk2;\n\n        // v indices\n        const int iv3 = iq3 / rv3;\n        const int iv2 = iq2 / rv2;\n\n        const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3));\n        q_to_vec_dot(pq, Q_q, DK);\n\n        // online softmax / attention\n        // loop over n_kv and n_head_kv\n        // ref: https://arxiv.org/pdf/2112.05682.pdf\n\n        for (int64_t ic = ic_start; ic < ic_end; ++ic) {\n            const float mv = mp ? slope*GGML_CPU_FP16_TO_FP32(mp[ic]) : 0.0f;\n            if (mv == -INFINITY) {\n                continue;\n            }\n\n            float s; // KQ value\n\n            const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);\n            kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1);\n\n            s = s*scale; // scale KQ value\n\n            if (logit_softcap != 0.0f) {\n                s = logit_softcap*tanhf(s);\n            }\n\n            s += mv; // apply mask\n\n            const float Mold = M;\n\n            float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value\n            float vs = 1.0f; // post-softmax KQ value, expf(s - M)\n\n            const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));\n\n            if (v->type == GGML_TYPE_F16) {\n                if (s > M) {\n                    // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f\n                    M = s;\n                    ms = expf(Mold - M);\n\n                    // V = V*expf(Mold - M)\n                    ggml_vec_scale_f16(DV, VKQ16, ms);\n                } else {\n                    // no new maximum, ms == 1.0f, vs != 1.0f\n                    vs = expf(s - M);\n                }\n\n                // V += v*expf(s - M)\n                ggml_vec_mad_f16(DV, VKQ16, (const ggml_fp16_t *) v_data, vs);\n            } else {\n                if (s > M) {\n                    // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f\n                    M = s;\n                    ms = expf(Mold - M);\n\n                    // V = V*expf(Mold - M)\n                    ggml_vec_scale_f32(DV, VKQ32, ms);\n                } else {\n                    // no new maximum, ms == 1.0f, vs != 1.0f\n                    vs = expf(s - M);\n                }\n\n                // V += v*expf(s - M)\n                if (v_to_float) {\n                    v_to_float(v_data, V32, DV);\n                    ggml_vec_mad_f32(DV, VKQ32, V32, vs);\n                } else {\n                    // V is F32\n                    ggml_vec_mad_f32(DV, VKQ32, (const float *) v_data, vs);\n                }\n            }\n\n            S = S*ms + vs; // scale and increment sum with partial sum\n        }\n\n        if (v->type == GGML_TYPE_F16) {\n            for (int64_t d = 0; d < DV; ++d) {\n                VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);\n            }\n        }\n\n        // sinks - apply only on the first kv-chunk\n        if (sinks && ic_start == 0) {\n            const float s = ((float *)((char *) sinks->data))[h];\n\n            float ms = 1.0f;\n            float vs = 1.0f;\n\n            if (s > M) {\n                ms = expf(M - s);\n                M = s;\n                ggml_vec_scale_f32(DV, VKQ32, ms);\n            } else {\n                vs = expf(s - M);\n            }\n\n            S = S*ms + vs;\n        }\n\n        if (write_partials) {\n            // Write M, S, VKQ to partials for later reduction\n            // partials layout: [M, S, VKQ[DV]] per query head\n            float * partial = partials + ir * partial_stride;\n            partial[0] = M;\n            partial[1] = S;\n            memcpy(partial + 2, VKQ32, DV * sizeof(float));\n        } else {\n            // V /= S\n            const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;\n            ggml_vec_scale_f32(DV, VKQ32, S_inv);\n\n            // dst indices\n            const int i1 = iq1;\n            const int i2 = iq2;\n            const int i3 = iq3;\n\n            // permute(0, 2, 1, 3)\n            memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1);\n        }\n    }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_tiled(\n        const ggml_compute_params * params,\n        ggml_tensor * dst,\n        int ir0, int ir1) {\n    const ggml_tensor * q     = dst->src[0];\n    const ggml_tensor * k     = dst->src[1];\n    const ggml_tensor * v     = dst->src[2];\n    const ggml_tensor * mask  = dst->src[3];\n    const ggml_tensor * sinks = dst->src[4];\n\n    GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nek, k,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbk, k,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nev, v,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbv, v,   nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst, nb)\n\n    const int64_t DK = nek0;\n    const int64_t DV = nev0;\n    const int64_t N  = neq1;\n\n    GGML_ASSERT(ne0 == DV);\n    GGML_ASSERT(ne2 == N);\n\n    // input tensor rows must be contiguous\n    GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n    GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n    GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n    GGML_ASSERT(neq0 == DK);\n    GGML_ASSERT(nek0 == DK);\n    GGML_ASSERT(nev0 == DV);\n\n    GGML_ASSERT(neq1 == N);\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    GGML_ASSERT(k->type == v->type);\n    const ggml_type kv_type = k->type;\n\n\n    // broadcast factors\n    const int64_t rk2 = neq2/nek2;\n    const int64_t rk3 = neq3/nek3;\n\n    const int64_t rv2 = neq2/nev2;\n    const int64_t rv3 = neq3/nev3;\n\n    float scale         = 1.0f;\n    float max_bias      = 0.0f;\n    float logit_softcap = 0.0f;\n\n    memcpy(&scale,         (float *) dst->op_params + 0, sizeof(float));\n    memcpy(&max_bias,      (float *) dst->op_params + 1, sizeof(float));\n    memcpy(&logit_softcap, (float *) dst->op_params + 2, sizeof(float));\n\n    if (logit_softcap != 0) {\n        scale /= logit_softcap;\n    }\n\n    const uint32_t n_head      = neq2;\n    const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));\n\n    const float m0 = powf(2.0f, -(max_bias       ) / n_head_log2);\n    const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);\n\n    int ith = params->ith;\n\n    static constexpr int Q_TILE_SZ  = ggml_fa_tile_config::Q;\n    static constexpr int KV_TILE_SZ = ggml_fa_tile_config::KV;\n\n    int ir = ir0;\n    while (ir < ir1) {\n        // q indices for the start of this tile\n        const int iq3 = ir/(neq2*neq1);\n        const int iq2 = (ir - iq3*neq2*neq1)/neq1;\n        const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);\n\n        // Number of valid rows in this tile:\n        // - limited by tile size (Q_TILE_SZ)\n        // - limited by chunk boundary (ir1 - ir)\n        // - limited by head boundary (neq1 - iq1) to avoid crossing into next head\n        const int tile_rows = MIN(Q_TILE_SZ, MIN((int)(ir1 - ir), (int)(neq1 - iq1)));\n        GGML_ASSERT(tile_rows > 0);\n\n        const uint32_t h = iq2; // head index\n        const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f;\n\n        float S[Q_TILE_SZ];\n        float M[Q_TILE_SZ];\n\n        for (int i = 0 ; i < Q_TILE_SZ; ++i) {\n            S[i] = 0.;\n            M[i] = -INFINITY;\n        }\n\n        // Per-thread scratch layout:\n        // Q_q:    Q_TILE_SZ * DK (converted Q tile — F32 for GEMM, KV type for scalar)\n        // KQ:     Q_TILE_SZ * KV_TILE_SZ (attention scores in float)\n        // mask:   Q_TILE_SZ * KV_TILE_SZ (mask in float)\n        // VKQ32:  Q_TILE_SZ * DV (FP32 output accumulator)\n        // V32:    KV_TILE_SZ * DV (F32 buffer for V tile)\n        // K_f32:  KV_TILE_SZ * DK (F32 buffer for K tile — GEMM path)\n        float * base  = (float *) params->wdata + ith*(Q_TILE_SZ*DK + 2*Q_TILE_SZ*KV_TILE_SZ + Q_TILE_SZ*DV + KV_TILE_SZ*DV + KV_TILE_SZ*DK + CACHE_LINE_SIZE_F32);\n\n        void  * Q_q    = base;\n        float * KQ     = (float *)((char *)base + Q_TILE_SZ * DK * sizeof(float));\n        float * mask32 = KQ + Q_TILE_SZ * KV_TILE_SZ;\n        float * VKQ32  = mask32 + Q_TILE_SZ * KV_TILE_SZ;\n        float * V32    = VKQ32 + Q_TILE_SZ * DV;\n        float * K_f32  = V32 + KV_TILE_SZ * DV;\n\n        memset(VKQ32, 0, Q_TILE_SZ * DV * sizeof(float));\n        memset(mask32, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));\n\n        // k indices\n        const int ik3 = iq3 / rk3;\n        const int ik2 = iq2 / rk2;\n\n        // v indices\n        const int iv3 = iq3 / rv3;\n        const int iv2 = iq2 / rv2;\n\n        {\n            float * Q_f32 = (float *)Q_q;\n            for (int tq = 0; tq < tile_rows; tq++) {\n                const float * pq = (const float *) ((char *) q->data + ((iq1 + tq)*nbq1 + iq2*nbq2 + iq3*nbq3));\n                memcpy(Q_f32 + tq * DK, pq, DK * sizeof(float));\n            }\n            for (int tq = tile_rows; tq < Q_TILE_SZ; tq++) {\n                memset(Q_f32 + tq * DK, 0, DK * sizeof(float));\n            }\n        }\n\n        memset(K_f32, 0, DK * KV_TILE_SZ * sizeof(float));\n        memset(V32,   0, KV_TILE_SZ * DV * sizeof(float));\n\n        for (int64_t ic = 0; ic < nek1; ic += KV_TILE_SZ) {\n            const int kv_tile = (int)std::min((int64_t)KV_TILE_SZ, nek1 - ic);\n\n            // skip the tile entirely if all the masks are -inf\n            if (mask) {\n                bool can_skip = true;\n                for (int tq = 0; tq < tile_rows; tq++) {\n                    const ggml_fp16_t * mp_row = (const ggml_fp16_t *)((const char *) mask->data + (iq1 + tq)*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]);\n                    for (int tk = 0; tk < kv_tile; tk++) {\n                        mask32[tq * KV_TILE_SZ + tk] = slope * GGML_CPU_FP16_TO_FP32(mp_row[ic + tk]);\n                        if (mask32[tq * KV_TILE_SZ + tk] != -INFINITY) {\n                            can_skip = false;\n                        }\n                    }\n                    // Pad remaining mask entries with -inf\n                    for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {\n                        mask32[tq * KV_TILE_SZ + tk] = -INFINITY;\n                    }\n                }\n\n                if (can_skip) {\n                    continue;\n                }\n            }\n\n            // Pack K tile transposed: K_f32[dk][kv] so KV_TILE is contiguous (SIMD dim)\n            // Zero-pad the last tile so the GEMM always operates on KV_TILE_SZ columns\n            for (int tk = 0; tk < kv_tile; tk++) {\n                const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;\n                if (kv_type == GGML_TYPE_F16) {\n                    const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;\n                    for (int64_t dk = 0; dk < DK; dk++) {\n                        K_f32[dk * KV_TILE_SZ + tk] = GGML_CPU_FP16_TO_FP32(k_f16[dk]);\n                    }\n                } else {\n                    const float * k_f32_src = (const float *)k_data;\n                    for (int64_t dk = 0; dk < DK; dk++) {\n                        K_f32[dk * KV_TILE_SZ + tk] = k_f32_src[dk];\n                    }\n                }\n            }\n            memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));\n            simd_gemm(KQ, (const float *)Q_q, K_f32, Q_TILE_SZ, DK, KV_TILE_SZ);\n            ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);\n\n            // Set padded KQ entries to -inf so softmax gives them zero weight\n            if (kv_tile < KV_TILE_SZ) {\n                for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n                    for (int tk = kv_tile; tk < KV_TILE_SZ; tk++) {\n                        KQ[tq * KV_TILE_SZ + tk] = -INFINITY;\n                    }\n                }\n            }\n\n            if (logit_softcap != 0.0f) {\n                ggml_vec_tanh_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, KQ);\n                ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, logit_softcap);\n            }\n\n            if (mask) {\n                ggml_vec_add_f32(tile_rows * KV_TILE_SZ, KQ, KQ, mask32);\n            }\n\n            bool skip[Q_TILE_SZ] = {};\n\n            for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n                float * kq_row = KQ + tq * KV_TILE_SZ;\n\n                float tile_max;\n                ggml_vec_max_f32(KV_TILE_SZ, &tile_max, kq_row);\n\n                if (tile_max == -INFINITY) {\n                    skip[tq] = true;\n                    continue;\n                }\n\n                const float Mold = M[tq];\n                const float Mnew = fmaxf(Mold, tile_max);\n\n                if (Mnew > Mold) {\n                    const float ms = expf(Mold - Mnew);\n                    ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);\n                    S[tq] *= ms;\n                }\n                M[tq] = Mnew;\n\n\n                S[tq] += ggml_vec_soft_max_f32(KV_TILE_SZ, kq_row, kq_row, Mnew);\n            }\n\n            // V accumulation: VKQ32 += softmax(KQ) * V\n            // Pack V tile to contiguous F32, zero-padded\n            for (int tk = 0; tk < kv_tile; tk++) {\n                const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;\n                if (kv_type == GGML_TYPE_F16) {\n                    ggml_fp16_to_fp32_row((const ggml_fp16_t *)v_data, V32 + tk * DV, DV);\n                } else {\n                    memcpy(V32 + tk * DV, v_data, DV * sizeof(float));\n                }\n            }\n            for (int tq = 0; tq < Q_TILE_SZ; tq++) {\n                if (skip[tq]) {\n                    memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));\n                }\n            }\n            simd_gemm(VKQ32, KQ, V32, Q_TILE_SZ, KV_TILE_SZ, DV);\n        }\n\n        // sinks (apply only to valid rows in the tile)\n        if (sinks) {\n            const float s = ((float *)((char *) sinks->data))[h];\n\n            for (int tq = 0; tq < tile_rows; tq++) {\n                float ms = 1.0f;\n                float vs = 1.0f;\n\n                if (s > M[tq]) {\n                    ms = expf(M[tq] - s);\n                    ggml_vec_scale_f32(DV, VKQ32 + tq * DV, ms);\n                } else {\n                    vs = expf(s - M[tq]);\n                }\n\n                S[tq] = S[tq] * ms + vs;\n            }\n        }\n\n        for (int tq = 0; tq < tile_rows; tq++) {\n            // V /= S\n            const float S_inv = S[tq] == 0.0f ? 0.0f : 1.0f / S[tq];\n            ggml_vec_scale_f32(DV, VKQ32 + tq * DV, S_inv);\n\n            // dst indices\n            const int i1 = iq1 + tq;\n            const int i2 = iq2;\n            const int i3 = iq3;\n\n            // permute(0, 2, 1, 3)\n            memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32 + tq * DV, nb1);\n        }\n\n        ir += tile_rows;\n    }\n}\n\n// Reduction function: combines partial results across KV chunks\n// Partials layout in wdata: [n_q_heads][n_chunks][2 + DV]\nstatic void ggml_flash_attn_ext_reduce_partials(\n        const ggml_compute_params * params,\n        ggml_tensor * dst,\n        const int64_t n_chunks,\n        const int64_t chunk_size) {\n\n    const ggml_tensor * q = dst->src[0];\n    const ggml_tensor * k = dst->src[1];\n    const ggml_tensor * v = dst->src[2];\n\n    const int64_t DK        = k->ne[0];\n    const int64_t DV        = v->ne[0];\n    const int64_t nek1      = k->ne[1];\n    const int64_t n_q_heads = q->ne[2];\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t wdata_per_thread = DK + 2*DV + CACHE_LINE_SIZE_F32;\n    float *       thread_wdata     = (float *) params->wdata + ith * wdata_per_thread;\n\n    const int64_t partials_offset  = nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);\n    const int64_t partial_size     = 2 + DV;\n    const float * partials_base    = (const float *) params->wdata + partials_offset;\n\n    // Output layout\n    const int64_t ne1 = dst->ne[1];\n    const int64_t ne2 = dst->ne[2];\n    const size_t  nb1 = dst->nb[1];\n\n    // Each thread reduces a subset of query heads\n    for (int64_t q_head = ith; q_head < n_q_heads; q_head += nth) {\n        float   M_final   = -INFINITY;\n        float   S_final   = 0.0f;\n        float * VKQ_final = thread_wdata;\n        memset(VKQ_final, 0, DV * sizeof(float));\n\n        // Combine partials from all chunks\n        for (int64_t chunk_idx = 0; chunk_idx < n_chunks; ++chunk_idx) {\n            const int64_t ic_start = chunk_idx * chunk_size;\n            if (ic_start >= nek1) continue;\n\n            const float * partial   = partials_base + (q_head * n_chunks + chunk_idx) * partial_size;\n            const float   M_chunk   = partial[0];\n            const float   S_chunk   = partial[1];\n            const float * VKQ_chunk = partial + 2;\n\n            if (S_chunk == 0.0f) continue;\n\n            const float M_new     = fmaxf(M_final, M_chunk);\n            const float scale_old = expf(M_final - M_new);\n            const float scale_new = expf(M_chunk - M_new);\n\n            for (int64_t d = 0; d < DV; ++d) {\n                VKQ_final[d] = VKQ_final[d] * scale_old + VKQ_chunk[d] * scale_new;\n            }\n            S_final = S_final * scale_old + S_chunk * scale_new;\n            M_final = M_new;\n        }\n\n        // Normalize and write to output\n        if (S_final != 0.0f) {\n            const float S_inv = 1.0f / S_final;\n            ggml_vec_scale_f32(DV, VKQ_final, S_inv);\n        }\n        // iq1=0, iq3=0 for decode\n        memcpy((char *) dst->data + (0*ne2*ne1 + q_head + 0*ne1)*nb1, VKQ_final, nb1);\n    }\n}\n\nstatic void ggml_compute_forward_flash_attn_ext_f16(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * q     = dst->src[0];\n    const ggml_tensor * k     = dst->src[1];\n    const ggml_tensor * v     = dst->src[2];\n\n    GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nek, k,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbk, k,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nev, v,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbv, v,   nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst, nb)\n\n    const int64_t DK = nek0;\n    const int64_t DV = nev0;\n    const int64_t N  = neq1;\n\n\n    GGML_ASSERT(ne0 == DV);\n    GGML_ASSERT(ne2 == N);\n\n    // input tensor rows must be contiguous\n    GGML_ASSERT(nbq0 == ggml_type_size(q->type));\n    GGML_ASSERT(nbk0 == ggml_type_size(k->type));\n    GGML_ASSERT(nbv0 == ggml_type_size(v->type));\n\n    GGML_ASSERT(neq0 == DK);\n    GGML_ASSERT(nek0 == DK);\n    GGML_ASSERT(nev0 == DV);\n\n    GGML_ASSERT(neq1 == N);\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // When use_ref is set, force the vec-only reference implementation (no tiling, no KV-chunking)\n    const bool use_ref = params->use_ref;\n\n    const bool kv_is_f32_or_f16 = (k->type == GGML_TYPE_F32 || k->type == GGML_TYPE_F16);\n    const bool use_split_kv_path = !use_ref && (neq1 == 1 && neq3 == 1) && kv_is_f32_or_f16 && (k->type == v->type) && q->type == GGML_TYPE_F32 && nek1 >= 512;\n\n    if (use_split_kv_path) {\n        const int64_t chunk_size = (nek1 + nth - 1) / nth;\n\n        // Partials buffer layout: [q_head][kv_chunk][M, S, VKQ]\n        const int64_t partial_size  = 2 + DV;\n        float *       partials_base = (float *) params->wdata + nth * (DK + 2*DV + CACHE_LINE_SIZE_F32);\n\n        const int64_t ic_start = ith * chunk_size;\n        const int64_t ic_end   = std::min(ic_start + chunk_size, nek1);\n\n        const int64_t partial_stride = nth * partial_size;\n        float *       chunk_partials = partials_base + ith * partial_size;\n\n        if (ic_start < nek1) {\n            for (int64_t q_head = 0; q_head < neq2; q_head++) {\n                ggml_compute_forward_flash_attn_ext_f16_one_chunk(\n                    params, dst, q_head, q_head + 1, ic_start, ic_end,\n                    chunk_partials, partial_stride);\n            }\n        } else {\n            for (int64_t q_head = 0; q_head < neq2; q_head++) {\n                float * q_partials = chunk_partials + q_head * partial_stride;\n                q_partials[0] = -INFINITY;  // M\n                q_partials[1] = 0.0f;       // S\n            }\n        }\n\n        ggml_barrier(params->threadpool);\n        ggml_flash_attn_ext_reduce_partials(params, dst, nth, chunk_size);\n    } else {\n\n        // total rows in q\n        const int64_t nr = neq1*neq2*neq3;\n\n        // disable for NUMA\n        const bool disable_chunking = ggml_is_numa();\n\n        // 4x chunks per thread\n        int nth_scaled = nth * 4;\n        int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;\n        int64_t nchunk     = (nr + chunk_size - 1) / chunk_size;\n\n        if (nth == 1 || nchunk < nth || disable_chunking) {\n            nchunk = nth;\n        }\n\n        if (ith == 0) {\n            ggml_threadpool_chunk_set(params->threadpool, nth);\n        }\n\n        ggml_barrier(params->threadpool);\n\n        const int64_t dr = (nr + nchunk - 1) / nchunk;\n\n        static constexpr int64_t Q_TILE_SZ  = ggml_fa_tile_config::Q;\n        bool use_tiled = !use_ref &&\n                               (q->type == GGML_TYPE_F32 &&\n                                kv_is_f32_or_f16 &&\n                                k->type == v->type &&\n                                neq1 >= Q_TILE_SZ);\n#ifdef GGML_SIMD\n        use_tiled &= (DV % GGML_F32_EPR == 0);\n#endif\n        int current_chunk = ith;\n\n        while (current_chunk < nchunk) {\n            const int64_t ir0 = dr * current_chunk;\n            const int64_t ir1 = MIN(ir0 + dr, nr);\n\n            if (use_tiled) {\n                ggml_compute_forward_flash_attn_ext_tiled(params, dst, ir0, ir1);\n            } else {\n                ggml_compute_forward_flash_attn_ext_f16_one_chunk(params, dst, ir0, ir1, 0, nek1, nullptr, 0);\n            }\n\n            current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n        }\n    }\n}\n\nvoid ggml_compute_forward_flash_attn_ext(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    switch (dst->op_params[3]) {\n        case GGML_PREC_DEFAULT:\n        case GGML_PREC_F32:\n            {\n                // uses F32 accumulators\n                ggml_compute_forward_flash_attn_ext_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_flash_attn_back\n\nstatic void ggml_compute_forward_flash_attn_back_f32(\n        const ggml_compute_params * params,\n        const bool masked,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * q = dst->src[0];\n    const ggml_tensor * k = dst->src[1];\n    const ggml_tensor * v = dst->src[2];\n    const ggml_tensor * d = dst->src[3];\n\n    GGML_TENSOR_LOCALS(int64_t, neq, q,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbq, q,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nek, k,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbk, k,   nb)\n    GGML_TENSOR_LOCALS(int64_t, nev, v,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbv, v,   nb)\n    GGML_TENSOR_LOCALS(int64_t, ned, d,   ne)\n    GGML_TENSOR_LOCALS(size_t,  nbd, d,   nb)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst, ne)\n    GGML_TENSOR_LOCALS(size_t,  nb,  dst, nb)\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t D = neq0;\n    const int64_t N = neq1;\n    const int64_t P = nek1 - N;\n    const int64_t M = P + N;\n\n    const int Mup  = ggml_up(M, GGML_SOFT_MAX_UNROLL);\n    const int mxDM = MAX(D, Mup);\n\n    // GGML_ASSERT(ne0 == D);\n    // GGML_ASSERT(ne1 == N);\n    GGML_ASSERT(P >= 0);\n\n    GGML_ASSERT(nbq0 == sizeof(float));\n    GGML_ASSERT(nbk0 == sizeof(float));\n    GGML_ASSERT(nbv0 == sizeof(float));\n\n    GGML_ASSERT(neq0 == D);\n    GGML_ASSERT(nek0 == D);\n    GGML_ASSERT(nev1 == D);\n    GGML_ASSERT(ned0 == D);\n\n    GGML_ASSERT(neq1 == N);\n    GGML_ASSERT(nek1 == N + P);\n    GGML_ASSERT(nev1 == D);\n    GGML_ASSERT(ned1 == N);\n\n    // dst cannot be transposed or permuted\n    GGML_ASSERT(nb0 == sizeof(float));\n    GGML_ASSERT(nb0 <= nb1);\n    GGML_ASSERT(nb1 <= nb2);\n    GGML_ASSERT(nb2 <= nb3);\n\n    if (ith == 0) {\n        memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);\n    }\n    ggml_barrier(params->threadpool);\n\n    const int64_t elem_q = ggml_nelements(q);\n    const int64_t elem_k = ggml_nelements(k);\n\n    ggml_type result_type = dst->type;\n    GGML_ASSERT(ggml_blck_size(result_type) == 1);\n    const size_t tsize = ggml_type_size(result_type);\n\n    const size_t offs_q = 0;\n    const size_t offs_k = offs_q + GGML_PAD(elem_q * tsize, GGML_MEM_ALIGN);\n    const size_t offs_v = offs_k + GGML_PAD(elem_k * tsize, GGML_MEM_ALIGN);\n\n    void * grad_q = (char *) dst->data;\n    void * grad_k = (char *) dst->data + offs_k;\n    void * grad_v = (char *) dst->data + offs_v;\n\n    const size_t nbgq1 = nb0*neq0;\n    const size_t nbgq2 = nb0*neq0*neq1;\n    const size_t nbgq3 = nb0*neq0*neq1*neq2;\n\n    const size_t nbgk1 = nb0*nek0;\n    const size_t nbgk2 = nb0*nek0*nek1;\n    const size_t nbgk3 = nb0*nek0*nek1*neq2;\n\n    const size_t nbgv1 = nb0*nev0;\n    const size_t nbgv2 = nb0*nev0*nev1;\n    const size_t nbgv3 = nb0*nev0*nev1*neq2;\n\n    // parallelize by k rows using ggml_vec_dot_f32\n\n    // total rows in k\n    const int nr = nek2*nek3;\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    const float scale = 1.0f/sqrtf(D);\n\n    //printf(\"P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\\n\", P, N, D, ir0, ir1, scale);\n\n    // how often k2 (and v2) is repeated in q2\n    int nrep = neq2/nek2;\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        // q indices\n        const int ik3 = ir/(nek2);\n        const int ik2 = ir - ik3*nek2;\n\n        const int iq3 = ik3;\n        const int id3 = ik3;\n        const int iv3 = ik3;\n        const int iv2 = ik2;\n\n        for (int irep = 0; irep < nrep; ++irep) {\n            const int iq2 = ik2 + irep*nek2;\n            const int id2 = iq2;\n\n            // (ik2 + irep*nek2) % nek2 == ik2\n            for (int iq1 = 0; iq1 < neq1; ++iq1) {\n                const int id1 = iq1;\n\n                // not sure about CACHE_LINE_SIZE_F32..\n                // - maybe it must not be multiplied by 2 and excluded from .. in SM 1*(..) offset?\n                float * S  = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 0*(mxDM+CACHE_LINE_SIZE_F32);\n                float * SM = (float *) params->wdata + ith*2*(mxDM + CACHE_LINE_SIZE_F32) + 1*(mxDM+CACHE_LINE_SIZE_F32);\n\n                for (int i = M; i < Mup; ++i) {\n                    S[i] = -INFINITY;\n                }\n\n                const int64_t masked_begin = masked ? (P + iq1 + 1) : M;\n                for (int64_t ic = 0; ic < masked_begin; ++ic) {\n                    // k indices\n                    const int ik1 = ic;\n\n                    // S indices\n                    const int i1 = ik1;\n\n                    ggml_vec_dot_f32(neq0,\n                            S + i1, 0,\n                            (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,\n                            (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);\n                }\n\n                // scale\n                ggml_vec_scale_f32(masked_begin, S, scale);\n\n                for (int64_t i = masked_begin; i < M; i++) {\n                    S[i] = -INFINITY;\n                }\n\n                // softmax\n                // exclude known -INF S[..] values from max and loop\n                // dont forget to set their SM values to zero\n                {\n                    float max = -INFINITY;\n                    ggml_vec_max_f32(masked_begin, &max, S);\n\n                    ggml_float sum = 0.0;\n                    {\n#ifdef GGML_SOFT_MAX_ACCELERATE\n                        max = -max;\n                        vDSP_vsadd(SM, 1, &max, SM, 1, Mup);\n                        vvexpf(SM, SM, &Mup);\n                        ggml_vec_sum_f32(Mup, &sum, SM);\n#else\n                        sum = ggml_vec_soft_max_f32(Mup, SM, S, max);\n#endif\n                    }\n\n                    assert(sum > 0.0);\n\n                    sum = 1.0/sum;\n                    ggml_vec_scale_f32(masked_begin, SM, sum);\n\n                }\n\n                // step-by-step explanation\n                {\n                    // forward-process                    shape      grads from backward process\n                    // parallel_for ik2,ik3:\n                    //  for irep:\n                    //   iq2 = ik2 + irep*nek2\n                    //   k[:D,:M,:,:]                     [D,M,:,:]  grad[k][:D,:M,ik2,ik3]  += grad[kcur]\n                    //   q[:D,:N,:,:]                     [D,N,:,:]  grad[q][:D,iq1,iq2,iq3] += grad[qcur]\n                    //   v[:M,:D,:,:]                     [M,D,:,:]  grad[v][:M,:D,iv2,iv3]  += grad[vcur]\n                    //   for iq1:\n                    //    kcur   = k[:D,:M,ik2,ik3]       [D,M,1,1]  grad[kcur] = grad[S1].T @ qcur\n                    //    qcur   = q[:D,iq1,iq2,iq3]      [D,1,1,1]  grad[qcur] = grad[S1]   @ kcur\n                    //    vcur   = v[:M,:D,iv2,iv3]       [M,D,1,1]  grad[vcur] = grad[S5].T @ S4\n                    //    S0     = -Inf                   [D,1,1,1]\n                    //   ~S1[i]  = dot(kcur[:D,i], qcur)\n                    //    S1     = qcur @ kcur.T          [M,1,1,1]  grad[S1]   = grad[S2] * scale\n                    //    S2     = S1 * scale             [M,1,1,1]  grad[S2]   = diag_mask_zero(grad[S3], P)\n                    //    S3     = diag_mask_inf(S2, P)   [M,1,1,1]  grad[S3]   = S4 * (grad[S4] - dot(S4, grad[S4]))\n                    //    S4     = softmax(S3)            [M,1,1,1]  grad[S4]   = grad[S5] @ vcur\n                    //   ~S5[i]  = dot(vcur[:,i], S4)\n                    //    S5     = S4 @ vcur.T            [D,1,1,1]  grad[S5]   = d[:D,id1,id2,id3]\n                    //   ~dst[i,iq1,iq2,iq3]  = S5[i]              ^\n                    //    dst[:D,iq1,iq2,iq3] = S5                 | grad[dst[:D,iq1,iq2,iq3]] = d[:D,id1,id2,id3]\n                    // dst                               backward-/ grad[dst]                 = d\n                    //\n                    // output gradients with their dependencies:\n                    //\n                    // grad[kcur] = grad[S1].T @ qcur\n                    // grad[S1]   = diag_mask_zero(grad[S3], P) * scale\n                    // grad[S3]   = S4 * (grad[S4] - dot(S4, grad[S4]))\n                    // grad[S4]   = grad[S5] @ vcur\n                    // grad[S4]   = d[:D,id1,id2,id3] @ vcur\n                    // grad[qcur] = grad[S1]   @ kcur\n                    // grad[vcur] = grad[S5].T @ S4\n                    // grad[vcur] = d[:D,id1,id2,id3].T @ S4\n                    //\n                    // in post-order:\n                    //\n                    // S1         = qcur @ kcur.T\n                    // S2         = S1 * scale\n                    // S3         = diag_mask_inf(S2, P)\n                    // S4         = softmax(S3)\n                    // grad[S4]   = d[:D,id1,id2,id3] @ vcur\n                    // grad[S3]   = S4 * (grad[S4] - dot(S4, grad[S4]))\n                    // grad[S1]   = diag_mask_zero(grad[S3], P) * scale\n                    // grad[qcur] = grad[S1]   @ kcur\n                    // grad[kcur] = grad[S1].T @ qcur\n                    // grad[vcur] = d[:D,id1,id2,id3].T @ S4\n                    //\n                    // using less variables (SM=S4):\n                    //\n                    // S             = diag_mask_inf(qcur @ kcur.T * scale, P)\n                    // SM            = softmax(S)\n                    // S             = d[:D,iq1,iq2,iq3] @ vcur\n                    // dot_SM_gradSM = dot(SM, S)\n                    // S             = SM * (S - dot(SM, S))\n                    // S             = diag_mask_zero(S, P) * scale\n                    //\n                    // grad[q][:D,iq1,iq2,iq3] += S   @ kcur\n                    // grad[k][:D,:M,ik2,ik3]  += S.T @ qcur\n                    // grad[v][:M,:D,iv2,iv3]  += d[:D,id1,id2,id3].T @ SM\n                }\n\n                // S = gradSM = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]\n                // S = d[:D,id1,id2,id3] @ vcur[:,:,iv2,iv3]\n                // for ic:\n                //   S[:M] += vcur[:M,ic,iv2,iv3] * d[ic,id1,id2,id3]\n                // exclude known future zero S[..] values from operation\n                ggml_vec_set_f32(masked_begin, S, 0);\n                for (int64_t ic = 0; ic < D; ++ic) {\n                    ggml_vec_mad_f32(masked_begin,\n                            S,\n                             (float *) ((char *) v->data + (          ic*nbv1  + iv2*nbv2 + iv3*nbv3)),\n                            *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2 + id3*nbd3)));\n                }\n\n                // S = SM * (S - dot(SM, S))\n                float dot_SM_gradSM = 0;\n                ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1);\n                ggml_vec_acc1_f32(M, S, -dot_SM_gradSM);\n                ggml_vec_mul_f32 (masked_begin, S, S, SM);\n\n                // S = diag_mask_zero(S, P) * scale\n                // already done by above ggml_vec_set_f32\n\n                // exclude known zero S[..] values from operation\n                ggml_vec_scale_f32(masked_begin, S, scale);\n\n                // S    shape [M,1]\n                // SM   shape [M,1]\n                // kcur shape [D,M]\n                // qcur shape [D,1]\n                // vcur shape [M,D]\n\n                // grad[q][:D,iq1,iq2,iq3] += S @ kcur\n                // grad[q][:D,iq1,iq2,iq3] += shape[M,1] @ shape[D,M]\n                // for ic:\n                //  grad[q][:D,iq1,iq2,iq3] += S[ic] * kcur[:D,ic,ik2,ik3]\n                // exclude known zero S[..] values from loop\n                for (int64_t ic = 0; ic < masked_begin; ++ic) {\n                    ggml_vec_mad_f32(D,\n                            (float *) ((char *) grad_q  + (iq1*nbgq1 + iq2*nbgq2  + iq3*nbgq3)),\n                            (float *) ((char *) k->data + (ic*nbk1   + ik2*nbk2   + ik3*nbk3)),\n                            S[ic]);\n                }\n\n                // grad[k][:D,:M,iq2,iq3] += S.T @ qcur\n                // for ic:\n                //  grad[k][:D,ic,iq2,iq3] += S.T[0,ic] * qcur[:D,0]\n                //  grad[k][:D,ic,iq2,iq3] += S[ic]     * qcur[:D,0]\n                // exclude known zero S[..] values from loop\n                for (int64_t ic = 0; ic < masked_begin; ++ic) {\n                    ggml_vec_mad_f32(D,\n                            (float *) ((char *) grad_k  + (ic*nbgk1  + ik2*nbgk2  + ik3*nbgk3)),\n                            (float *) ((char *) q->data + (iq1*nbq1  + iq2*nbq2   + iq3*nbq3)),\n                            S[ic]);\n                }\n\n                // grad[v][:M,:D,iv2,iv3] += d[:D,id1,id2,id3].T       @ SM\n                // for ic:\n                //  grad[v][:M,ic,iv2,iv3] += d[:D,id1,id2,id3].T[0,ic] * SM[:M]\n                //  grad[v][:M,ic,iv2,iv3] += d[ic,id1,id2,id3]         * SM[:M]\n                // exclude known zero SM[..] values from mad\n                for (int64_t ic = 0; ic < D; ++ic) {\n                    ggml_vec_mad_f32(masked_begin,\n                            (float *) ((char *) grad_v   + (          ic*nbgv1 + iv2*nbgv2 + iv3*nbgv3)),\n                            SM,\n                            *(float *) ((char *) d->data + (ic*nbd0 + id1*nbd1 + id2*nbd2  + id3*nbd3)));\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_flash_attn_back(\n        const ggml_compute_params * params,\n        const bool masked,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * q = dst->src[0];\n\n    switch (q->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_flash_attn_back_f32(params, masked, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_ssm_conv\n\nstatic void ggml_compute_forward_ssm_conv_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0]; // conv_x\n    const ggml_tensor * src1 = dst->src[1]; // conv1d.weight\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nc  = src1->ne[0]; // d_conv\n    const int ncs = src0->ne[0]; // d_conv - 1 + n_t\n    const int nr  = src0->ne[1]; // d_inner\n    const int n_t =  dst->ne[1]; // tokens per sequence\n    const int n_s =  dst->ne[2]; // number of sequences in the batch\n\n    GGML_ASSERT( dst->ne[0] == nr);\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(src1->nb[0] == sizeof(float));\n    GGML_ASSERT(src0->nb[1] == src0->ne[0]*sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n    const int ir  = ir1 - ir0;\n\n    for (int i3 = 0; i3 < n_s; ++i3) {\n        for (int i2 = 0; i2 < n_t; ++i2) {\n            // {d_conv - 1 + n_t, d_inner, n_seqs}\n            // sliding window\n            const float * s = (const float *) ((const char *) src0->data + ir0*(src0->nb[1]) + i2*(src0->nb[0]) + i3*(src0->nb[2])); // {d_conv, d_inner, n_s}\n            const float * c = (const float *) ((const char *) src1->data + ir0*(src1->nb[1])); // {d_conv, d_inner}\n            float * x = (float *) ((char *) dst->data + ir0*(dst->nb[0]) + i2*(dst->nb[1]) + i3*(dst->nb[2])); // {d_inner, n_t, n_s}\n\n            // TODO: transpose the output for smaller strides for big batches?\n            // d_inner\n            for (int i1 = 0; i1 < ir; ++i1) {\n                // rowwise dot product\n                // NOTE: not using ggml_vec_dot_f32, because its sum is in double precision\n                float sumf = 0.0f;\n\n                // d_conv\n                for (int i0 = 0; i0 < nc; ++i0) {\n                    sumf += s[i0 + i1*ncs] * c[i0 + i1*nc];\n                }\n                x[i1] = sumf;\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_ssm_conv(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    switch (dst->src[0]->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_ssm_conv_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_ssm_scan\n\nstatic void ggml_compute_forward_ssm_scan_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0]; // s  {d_state, dim, n_head, n_seqs+}\n    const ggml_tensor * src1 = dst->src[1]; // x  {dim, n_head, n_seq_tokens, n_seqs}\n    const ggml_tensor * src2 = dst->src[2]; // dt {n_head, n_seq_tokens, n_seqs}\n    const ggml_tensor * src3 = dst->src[3]; // A  {d_state, n_head} or {1, n_head}\n    const ggml_tensor * src4 = dst->src[4]; // B  {d_state, n_group, n_seq_tokens, n_seqs}\n    const ggml_tensor * src5 = dst->src[5]; // C  {d_state, n_group, n_seq_tokens, n_seqs}\n    const ggml_tensor * src6 = dst->src[6]; // ids {n_seqs}\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t nc = src0->ne[0]; // d_state\n    const int64_t nr = src0->ne[1]; // dim\n    const int64_t nh = src1->ne[1]; // n_head\n    const int64_t ng = src4->ne[1];\n    const int64_t nt = src1->ne[2]; // number of tokens per sequence\n    const int64_t ns = src1->ne[3]; // number of sequences in the batch\n\n    // can't use ggml_nbytes because src1 is not necessarily contiguous\n    const int64_t s_off = ggml_nelements(src1) * ggml_element_size(src1);\n\n    GGML_ASSERT(ggml_nelements(src1) + nc*nr*nh*ns == ggml_nelements(dst));\n    GGML_ASSERT(src0->nb[0] == sizeof(float));\n    GGML_ASSERT(src1->nb[0] == sizeof(float));\n    GGML_ASSERT(src2->nb[0] == sizeof(float));\n    GGML_ASSERT(src3->nb[0] == sizeof(float));\n    GGML_ASSERT(src4->nb[0] == sizeof(float));\n    GGML_ASSERT(src5->nb[0] == sizeof(float));\n    GGML_ASSERT(src6->nb[0] == sizeof(int32_t));\n    GGML_ASSERT(nh % ng == 0);\n\n    // heads per thread\n    const int dh = (nh + nth - 1)/nth;\n\n    // head range for this thread\n    const int ih0 = dh*ith;\n    const int ih1 = MIN(ih0 + dh, nh);\n\n    const int32_t * ids = (const int32_t *) src6->data;\n\n    for (int i3 = 0; i3 < ns; ++i3) {\n        const float * s0 = (const float *) ((const char *) src0->data + ids[i3]*(src0->nb[3])); // {d_state, dim, nh, ns}\n              float * s  = (      float *) ((      char *) dst->data  + i3*(src0->nb[3]) + s_off); // {d_state, dim, nh, ns}\n\n        for (int i2 = 0; i2 < nt; ++i2) {\n            const float * x  = (const float *) ((const char *) src1->data + i2*(src1->nb[2]) + i3*(src1->nb[3])); // {dim, nh, nt, ns}\n            const float * dt = (const float *) ((const char *) src2->data + i2*(src2->nb[1]) + i3*(src2->nb[2])); // {nh, nt, ns}\n            const float * A  = (const float *) ((const char *) src3->data); // {d_state, nh} or {1, nh}\n            const float * B  = (const float *) ((const char *) src4->data + i2*(src4->nb[2]) + i3*(src4->nb[3])); // {d_state, ng, nt, ns}\n            const float * C  = (const float *) ((const char *) src5->data + i2*(src5->nb[2]) + i3*(src5->nb[3])); // {d_state, ng, nt, ns}\n                  float * y  = (      float *) ((      char *) dst->data + i2*(nh*nr*sizeof(float)) + i3*(nt*nh*nr*sizeof(float))); // {dim, nh, nt, ns}\n\n            if (src3->ne[0] == 1) {\n                // Mamba-2 has a scalar decay factor per head; dA can be outside the state-wise loop\n\n                // n_head\n                for (int h = ih0; h < ih1; ++h) {\n                    // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16\n                    const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);\n                    const float dA = expf(dt_soft_plus * A[h]);\n                    const int g = h / (nh / ng); // repeat_interleave\n\n                    // dim\n                    for (int i1 = 0; i1 < nr; ++i1) {\n                        const int ii = i1 + h*nr;\n                        const float x_dt = x[ii] * dt_soft_plus;\n                        float sumf = 0.0f;\n#if defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n                        const int ggml_f32_epr = svcntw();\n                        const int ggml_f32_step = 1 * ggml_f32_epr;\n\n                        const int np = (nc & ~(ggml_f32_step - 1));\n\n                        GGML_F32_VEC sum = GGML_F32_VEC_ZERO;\n\n                        GGML_F32_VEC adA = GGML_F32_VEC_SET1(dA);\n                        GGML_F32_VEC axdt = GGML_F32_VEC_SET1(x_dt);\n\n                        for (int i = 0; i < np; i += ggml_f32_step) {\n                            // TODO: maybe unroll more?\n                            for (int j = 0; j < 1; j++) {\n                                GGML_F32_VEC t0 = GGML_F32_VEC_LOAD(s0 + i + j*ggml_f32_epr + ii*nc);\n                                GGML_F32_VEC t1 = GGML_F32_VEC_LOAD(B + i + j*ggml_f32_epr + g*nc);\n                                GGML_F32_VEC t2 = GGML_F32_VEC_LOAD(C + i + j*ggml_f32_epr + g*nc);\n\n                                t0 = GGML_F32_VEC_MUL(t0, adA);\n                                t1 = GGML_F32_VEC_MUL(t1, axdt);\n\n                                t0 = GGML_F32_VEC_ADD(t0, t1);\n\n                                sum = GGML_F32_VEC_FMA(sum, t0, t2);\n\n                                GGML_F32_VEC_STORE(s + i + j*ggml_f32_epr + ii*nc, t0);\n                            }\n                        }\n\n                        sumf = GGML_F32xt_REDUCE_ONE(sum);\n    #elif defined(__riscv_v_intrinsic)\n                        // todo: RVV implementation\n                        const int np = 0;\n    #else\n                        const int np = (nc & ~(GGML_F32_STEP - 1));\n\n                        GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n\n                        GGML_F32_VEC adA = GGML_F32_VEC_SET1(dA);\n                        GGML_F32_VEC axdt = GGML_F32_VEC_SET1(x_dt);\n\n                        GGML_F32_VEC ax[GGML_F32_ARR];\n                        GGML_F32_VEC ay[GGML_F32_ARR];\n                        GGML_F32_VEC az[GGML_F32_ARR];\n\n                        for (int i = 0; i < np; i += GGML_F32_STEP) {\n                            for (int j = 0; j < GGML_F32_ARR; j++) {\n                                ax[j] = GGML_F32_VEC_LOAD(s0 + i + j*GGML_F32_EPR + ii*nc);\n                                ay[j] = GGML_F32_VEC_LOAD(B + i + j*GGML_F32_EPR + g*nc);\n                                az[j] = GGML_F32_VEC_LOAD(C + i + j*GGML_F32_EPR + g*nc);\n\n                                ax[j] = GGML_F32_VEC_MUL(ax[j], adA);\n                                ay[j] = GGML_F32_VEC_MUL(ay[j], axdt);\n\n                                ax[j] = GGML_F32_VEC_ADD(ax[j], ay[j]);\n\n                                sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], az[j]);\n\n                                GGML_F32_VEC_STORE(s + i + j*GGML_F32_EPR + ii*nc, ax[j]);\n                            }\n                        }\n\n                        // reduce sum0..sum3 to sum0\n                        GGML_F32_VEC_REDUCE(sumf, sum);\n    #endif\n#else\n                        const int np = 0;\n#endif\n                        // d_state\n                        for (int i0 = np; i0 < nc; ++i0) {\n                            const int i = i0 + ii*nc;\n                            const int ig = i0 + g*nc;\n                            // state = prev_state * dA + dB * x\n                            const float state = (s0[i] * dA) + (B[ig] * x_dt);\n                            // y = rowwise_dotprod(state, C)\n                            sumf += state * C[ig];\n                            s[i] = state;\n                        }\n                        y[ii] = sumf;\n                    }\n                }\n            } else {\n                // Mamba-1 has an element-wise decay factor for the states\n\n                // n_head\n                for (int h = ih0; h < ih1; ++h) {\n                    // ref: https://github.com/state-spaces/mamba/blob/62db608da60f6fc790b8ed9f4b3225e95ca15fde/mamba_ssm/ops/triton/softplus.py#L16\n                    const float dt_soft_plus = ggml_compute_softplus_f32(dt[h]);\n                    const int g = h / (nh / ng); // repeat_interleave\n\n                    // dim\n                    for (int i1 = 0; i1 < nr; ++i1) {\n                        const int ii = i1 + h*nr;\n                        const float x_dt = x[ii] * dt_soft_plus;\n#if defined(__ARM_FEATURE_SVE)\n                        svfloat32_t vx_dt = GGML_F32_VEC_SET1(x_dt);\n                        svfloat32_t vdt_soft_plus = GGML_F32_VEC_SET1(dt_soft_plus);\n                        svfloat32_t r1_vector = GGML_F32_VEC_ZERO;\n\n                        // d_state\n                        // TODO: what happens when (d_state % svcntw()) != 0?\n                        for (int64_t k = 0; k < nc; k += svcntw()) {\n                            svfloat32_t vA = GGML_F32_VEC_LOAD(&A[h*nc + k]);\n                            svfloat32_t vB = GGML_F32_VEC_LOAD(&B[k + g*nc]);\n                            svfloat32_t vC = GGML_F32_VEC_LOAD(&C[k + g*nc]);\n                            svfloat32_t vs0 = GGML_F32_VEC_LOAD(&s0[ii*nc + k]);\n\n                            svfloat32_t t1 = GGML_F32_VEC_MUL(vdt_soft_plus, vA);\n                            t1 = exp_ps_sve(svptrue_b32(), t1);\n                            svfloat32_t t2 = GGML_F32_VEC_MUL(vx_dt, vB);\n\n                            vs0 = GGML_F32_VEC_FMA(t2, vs0, t1);\n                            r1_vector = GGML_F32_VEC_ADD(GGML_F32_VEC_MUL(vs0, vC), r1_vector);\n\n                            GGML_F32_VEC_STORE(&s[ii*nc + k], vs0);\n                        }\n                        y[ii] = GGML_F32xt_REDUCE_ONE(r1_vector);\n#else\n                        float sumf = 0.0f;\n                        // NOTE: can't really use GGML_SIMD here because d_state is usually 16\n                        //       and also because expf is used within the loop.\n                        // d_state\n                        for (int i0 = 0; i0 < nc; ++i0) {\n                            const int i = i0 + ii*nc;\n                            const int ig = i0 + g*nc;\n                            // state = prev_state * dA + dB * x\n                            const float state = (s0[i] * expf(dt_soft_plus * A[i0 + h*nc])) + (B[ig] * x_dt);\n                            // y = rowwise_dotprod(state, C)\n                            sumf += state * C[ig];\n                            s[i] = state;\n                        }\n                        y[ii] = sumf;\n#endif\n                    }\n                }\n            }\n            // use the output as the source when it's not the first token-wise iteration\n            s0 = s;\n        }\n    }\n}\n\nvoid ggml_compute_forward_ssm_scan(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    switch (dst->src[0]->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_ssm_scan_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_win_part\n\nstatic void ggml_compute_forward_win_part_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    GGML_UNUSED(params);\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n\n    const int32_t nep0 = ((const int32_t *)(dst->op_params))[0];\n    const int32_t nep1 = ((const int32_t *)(dst->op_params))[1];\n    const int32_t w    = ((const int32_t *)(dst->op_params))[2];\n\n    assert(ne00 == ne0);\n    assert(ne3  == nep0*nep1);\n\n    // TODO: optimize / multi-thread\n    for (int py = 0; py < nep1; ++py) {\n        for (int px = 0; px < nep0; ++px) {\n            const int64_t i3 = py*nep0 + px;\n            for (int64_t i2 = 0; i2 < ne2; ++i2) {\n                for (int64_t i1 = 0; i1 < ne1; ++i1) {\n                    for (int64_t i0 = 0; i0 < ne0; ++i0) {\n                        const int64_t i02 = py*w + i2;\n                        const int64_t i01 = px*w + i1;\n                        const int64_t i00 = i0;\n\n                        const int64_t i = i3*ne2*ne1*ne0 + i2*ne1*ne0    + i1*ne0   + i0;\n                        const int64_t j =                  i02*ne01*ne00 + i01*ne00 + i00;\n\n                        if (py*w + i2 >= ne02 || px*w + i1 >= ne01) {\n                            ((float *) dst->data)[i] = 0.0f;\n                        } else {\n                            ((float *) dst->data)[i] = ((float *) src0->data)[j];\n                        }\n                    }\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_win_part(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_win_part_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_win_unpart\n\nstatic void ggml_compute_forward_win_unpart_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    GGML_UNUSED(params);\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)\n    GGML_TENSOR_LOCALS(int64_t, ne,  dst,  ne)\n\n    const int32_t w = ((const int32_t *)(dst->op_params))[0];\n\n    // padding\n    const int px = (w - ne1%w)%w;\n    //const int py = (w - ne2%w)%w;\n\n    const int npx = (px + ne1)/w;\n    //const int npy = (py + ne2)/w;\n\n    assert(ne0 == ne00);\n\n    // TODO: optimize / multi-thread\n    for (int64_t i2 = 0; i2 < ne2; ++i2) {\n        for (int64_t i1 = 0; i1 < ne1; ++i1) {\n            for (int64_t i0 = 0; i0 < ne0; ++i0) {\n                const int ip2 = i2/w;\n                const int ip1 = i1/w;\n\n                const int64_t i02 = i2%w;\n                const int64_t i01 = i1%w;\n                const int64_t i00 = i0;\n\n                const int64_t i = (ip2*npx + ip1)*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00 + i00;\n                const int64_t j =                                  i2*ne1*ne0    + i1*ne0   + i0;\n\n                ((float *) dst->data)[j] = ((float *) src0->data)[i];\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_win_unpart(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_win_unpart_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n//ggml_compute_forward_unary\n\nvoid ggml_compute_forward_unary(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_unary_op op = ggml_get_unary_op(dst);\n\n    switch (op) {\n        case GGML_UNARY_OP_ABS:\n            {\n                ggml_compute_forward_abs(params, dst);\n            } break;\n        case GGML_UNARY_OP_SGN:\n            {\n                ggml_compute_forward_sgn(params, dst);\n            } break;\n        case GGML_UNARY_OP_NEG:\n            {\n                ggml_compute_forward_neg(params, dst);\n            } break;\n        case GGML_UNARY_OP_STEP:\n            {\n                ggml_compute_forward_step(params, dst);\n            } break;\n        case GGML_UNARY_OP_TANH:\n            {\n                ggml_compute_forward_tanh(params, dst);\n            } break;\n        case GGML_UNARY_OP_ELU:\n            {\n                ggml_compute_forward_elu(params, dst);\n            } break;\n        case GGML_UNARY_OP_RELU:\n            {\n                ggml_compute_forward_relu(params, dst);\n            } break;\n        case GGML_UNARY_OP_SIGMOID:\n            {\n                ggml_compute_forward_sigmoid(params, dst);\n            } break;\n        case GGML_UNARY_OP_GELU:\n            {\n                ggml_compute_forward_gelu(params, dst);\n            } break;\n        case GGML_UNARY_OP_GELU_ERF:\n            {\n                ggml_compute_forward_gelu_erf(params, dst);\n            } break;\n        case GGML_UNARY_OP_GELU_QUICK:\n            {\n                ggml_compute_forward_gelu_quick(params, dst);\n            } break;\n        case GGML_UNARY_OP_SILU:\n            {\n                ggml_compute_forward_silu(params, dst);\n            } break;\n        case GGML_UNARY_OP_HARDSWISH:\n            {\n                ggml_compute_forward_hardswish(params, dst);\n            } break;\n        case GGML_UNARY_OP_HARDSIGMOID:\n            {\n                ggml_compute_forward_hardsigmoid(params, dst);\n            } break;\n        case GGML_UNARY_OP_EXP:\n            {\n                ggml_compute_forward_exp(params, dst);\n            } break;\n        case GGML_UNARY_OP_FLOOR:\n            {\n                ggml_compute_forward_floor(params, dst);\n            } break;\n        case GGML_UNARY_OP_CEIL:\n            {\n                ggml_compute_forward_ceil(params, dst);\n            } break;\n        case GGML_UNARY_OP_ROUND:\n            {\n                ggml_compute_forward_round(params, dst);\n            } break;\n        case GGML_UNARY_OP_TRUNC:\n            {\n                ggml_compute_forward_trunc(params, dst);\n            } break;\n        case GGML_UNARY_OP_XIELU:\n            {\n                ggml_compute_forward_xielu(params, dst);\n            } break;\n        case GGML_UNARY_OP_EXPM1:\n            {\n                ggml_compute_forward_expm1(params, dst);\n            } break;\n        case GGML_UNARY_OP_SOFTPLUS:\n            {\n                ggml_compute_forward_softplus(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n//ggml_compute_forward_glu\n\nvoid ggml_compute_forward_glu(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_glu_op op = ggml_get_glu_op(dst);\n\n    switch (op) {\n        case GGML_GLU_OP_REGLU:\n            {\n                ggml_compute_forward_reglu(params, dst);\n            } break;\n        case GGML_GLU_OP_GEGLU:\n            {\n                ggml_compute_forward_geglu(params, dst);\n            } break;\n        case GGML_GLU_OP_SWIGLU:\n            {\n                ggml_compute_forward_swiglu(params, dst);\n            } break;\n        case GGML_GLU_OP_SWIGLU_OAI:\n            {\n                ggml_compute_forward_swiglu_oai(params, dst);\n            } break;\n        case GGML_GLU_OP_GEGLU_ERF:\n            {\n                ggml_compute_forward_geglu_erf(params, dst);\n            } break;\n        case GGML_GLU_OP_GEGLU_QUICK:\n            {\n                ggml_compute_forward_geglu_quick(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_get_rel_pos\n\nstatic void ggml_compute_forward_get_rel_pos_f16(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    GGML_UNUSED(params);\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L292-L322\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    const int64_t w = ne1;\n\n    ggml_fp16_t * src0_data = (ggml_fp16_t *) src0->data;\n    ggml_fp16_t * dst_data  = (ggml_fp16_t *) dst->data;\n\n    for (int64_t i2 = 0; i2 < ne2; ++i2) {\n        for (int64_t i1 = 0; i1 < ne1; ++i1) {\n            const int64_t pos = (w - i1 - 1) + i2;\n            for (int64_t i0 = 0; i0 < ne0; ++i0) {\n                dst_data[i2*ne1*ne0 + i1*ne0 + i0] = src0_data[pos*ne00 + i0];\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_get_rel_pos(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F16:\n        case GGML_TYPE_BF16:\n            {\n                ggml_compute_forward_get_rel_pos_f16(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_add_rel_pos\n\nstatic void ggml_compute_forward_add_rel_pos_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    const ggml_tensor * src2 = dst->src[2];\n\n    const bool inplace = (bool) ((int32_t *) dst->op_params)[0];\n    if (!inplace) {\n        if (params->ith == 0) {\n            memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));\n        }\n        ggml_barrier(params->threadpool);\n    }\n    // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L357-L359\n\n    float * src1_data = (float *) src1->data;\n    float * src2_data = (float *) src2->data;\n    float * dst_data  = (float *) dst->data;\n\n    const int64_t ne10 = src1->ne[0];\n    const int64_t ne11 = src1->ne[1];\n    const int64_t ne12 = src1->ne[2];\n    const int64_t ne13 = src1->ne[3];\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    // total patches in dst\n    const int np = ne13;\n\n    // patches per thread\n    const int dp = (np + nth - 1)/nth;\n\n    // patch range for this thread\n    const int ip0 = dp*ith;\n    const int ip1 = MIN(ip0 + dp, np);\n\n    for (int64_t i13 = ip0; i13 < ip1; ++i13) {\n        for (int64_t i12 = 0; i12 < ne12; ++i12) {\n            for (int64_t i11 = 0; i11 < ne11; ++i11) {\n                const int64_t jp1 = i13*ne12*ne11*ne10 + i12*ne11*ne10 + i11*ne10;\n                for (int64_t i10 = 0; i10 < ne10; ++i10) {\n                    const int64_t jp0  = jp1 + i10;\n                    const float src1_e = src1_data[jp0];\n                    const float src2_e = src2_data[jp0];\n\n                    const int64_t jdh = jp0 * ne10;\n                    const int64_t jdw = jdh - (ne10 - 1) * i10;\n\n                    for (int64_t j = 0; j < ne10; ++j) {\n                        dst_data[jdh + j     ] += src2_e;\n                        dst_data[jdw + j*ne10] += src1_e;\n                    }\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_compute_forward_add_rel_pos(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_add_rel_pos_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_rwkv_wkv6\n\nstatic void ggml_compute_forward_rwkv_wkv6_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const int64_t T = dst->src[1]->ne[2];\n    const int64_t C = dst->ne[0];\n    const int64_t HEADS = dst->src[1]->ne[1];\n    const int64_t n_seqs = dst->src[5]->ne[1];\n    const int64_t head_size = C / HEADS;\n\n    float * dst_data = (float *) dst->data;\n    float * state = ((float *) dst->data) + C * T;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    if (ith >= HEADS) {\n        return;\n    }\n\n    const int h_start = (HEADS * ith) / nth;\n    const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n                (HEADS * (ith + 1)) / nth : HEADS;\n\n    float * k =          (float *) dst->src[0]->data;\n    float * v =          (float *) dst->src[1]->data;\n    float * r =          (float *) dst->src[2]->data;\n    float * time_faaaa = (float *) dst->src[3]->data;\n    float * time_decay = (float *) dst->src[4]->data;\n\n    size_t t_stride = HEADS * head_size; // Same to C\n\n    size_t h_stride = C / HEADS;\n    GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n    size_t h_stride_2d = head_size * head_size;\n\n    if (ith == 0) {\n        memset(dst_data, 0, T * C * sizeof(float));\n    }\n    ggml_barrier(params->threadpool);\n\n\n    #if defined(__AVX__) && !defined(__AVX512F__)\n        #define GGML_F32X GGML_F32x8\n        #define GGML_F32X_SET1 GGML_F32x8_SET1\n        #define GGML_F32X_LOAD GGML_F32x8_LOAD\n        #define GGML_F32X_STORE GGML_F32x8_STORE\n        #define GGML_F32X_MUL GGML_F32x8_MUL\n        #define GGML_F32X_FMA GGML_F32x8_FMA\n        #define WKV_VECTOR_SIZE 8\n    #elif defined(__AVX512F__)\n        #define GGML_F32X GGML_F32x16\n        #define GGML_F32X_SET1 GGML_F32x16_SET1\n        #define GGML_F32X_LOAD GGML_F32x16_LOAD\n        #define GGML_F32X_STORE GGML_F32x16_STORE\n        #define GGML_F32X_MUL GGML_F32x16_MUL\n        #define GGML_F32X_FMA GGML_F32x16_FMA\n        #define WKV_VECTOR_SIZE 16\n    #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n        #define GGML_F32X GGML_F32xt\n        #define GGML_F32X_SET1 GGML_F32xt_SET1\n        #define GGML_F32X_LOAD GGML_F32xt_LOAD\n        #define GGML_F32X_STORE GGML_F32xt_STORE\n        #define GGML_F32X_MUL GGML_F32xt_MUL\n        #define GGML_F32X_FMA GGML_F32xt_FMA\n        #define WKV_VECTOR_SIZE 8\n    #elif defined(__ARM_NEON) && defined(__aarch64__)\n        #define GGML_F32X GGML_F32x4\n        #define GGML_F32X_SET1 GGML_F32x4_SET1\n        #define GGML_F32X_LOAD GGML_F32x4_LOAD\n        #define GGML_F32X_STORE GGML_F32x4_STORE\n        #define GGML_F32X_MUL GGML_F32x4_MUL\n        #define GGML_F32X_FMA GGML_F32x4_FMA\n        #define WKV_VECTOR_SIZE 4\n    #endif\n\n    #ifdef WKV_VECTOR_SIZE\n        int wkv_vector_size;\n        #if defined(__ARM_FEATURE_SVE)\n            wkv_vector_size = svcntw();\n        #else\n            wkv_vector_size = WKV_VECTOR_SIZE;\n        #endif\n        const int64_t vec_count = head_size / wkv_vector_size;\n\n        for (int64_t t = 0; t < T; t++) {\n            size_t t_offset = t * t_stride;\n            size_t state_offset = head_size * C * (t / (T / n_seqs));\n            float * state_cur = state + state_offset;\n            float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;\n\n            for (int64_t h = h_start; h < h_end; h++) {\n                size_t h_offset = h * h_stride;\n                size_t t_h_offset = t_offset + h_offset;\n                size_t h_2d_offset = h * h_stride_2d;\n\n                for (int64_t i = 0; i < head_size; i++) {\n                    size_t t_h_i_offset = t_h_offset + i;\n                    size_t h_i_offset = h_offset + i;\n                    size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                    float k_val = k[t_h_i_offset];\n                    float r_val = r[t_h_i_offset];\n                    float time_faaaa_val = time_faaaa[h_i_offset];\n                    float time_decay_val = time_decay[t_h_i_offset];\n\n                    // Broadcast scalar values to vectors\n                    GGML_F32X k_vec = GGML_F32X_SET1(k_val);\n                    GGML_F32X r_vec = GGML_F32X_SET1(r_val);\n                    GGML_F32X time_faaaa_vec = GGML_F32X_SET1(time_faaaa_val);\n                    GGML_F32X time_decay_vec = GGML_F32X_SET1(time_decay_val);\n\n                    for (int64_t j = 0; j < vec_count; j++) {\n                        size_t base_j = j * wkv_vector_size;\n                        size_t t_h_j_offset = t_h_offset + base_j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + base_j;\n\n                        // Load x elements at once\n                        GGML_F32X v_vec = GGML_F32X_LOAD(&v[t_h_j_offset]);\n                        GGML_F32X prev_state_vec = GGML_F32X_LOAD(&state_prev[h_2d_i_j_offset]);\n                        GGML_F32X dst_vec = GGML_F32X_LOAD(&dst_data[t_h_j_offset]);\n\n                        // Compute kv = v * k\n                        GGML_F32X kv_vec = GGML_F32X_MUL(v_vec, k_vec);\n\n                        // Compute temp = kv * time_faaaa + prev_state\n                        GGML_F32X temp_vec = GGML_F32X_FMA(prev_state_vec, kv_vec, time_faaaa_vec);\n\n                        // Update dst: dst += temp * r\n                        dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, r_vec);\n                        GGML_F32X_STORE(&dst_data[t_h_j_offset], dst_vec);\n\n                        // Update state: state = prev_state * time_decay + kv\n                        GGML_F32X new_state_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, time_decay_vec);\n                        GGML_F32X_STORE(&state_cur[h_2d_i_j_offset], new_state_vec);\n                    }\n\n                    // Handle remaining elements, this will not be used.\n                    for (int64_t j = vec_count * wkv_vector_size; j < head_size; j++) {\n                        size_t t_h_j_offset = t_h_offset + j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + j;\n                        float v_val = v[t_h_j_offset];\n                        float kv_val = v_val * k_val;\n                        float prev_state_val = state_prev[h_2d_i_j_offset];\n                        float temp_val = kv_val * time_faaaa_val + prev_state_val;\n                        dst_data[t_h_j_offset] += temp_val * r_val;\n                        state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;\n                    }\n                }\n            }\n        }\n\n    #else\n        // basically fused operations:\n        // dst = r @ (time_faaaa * (k @ v) + state),\n        // state = time_decay * state + (k @ v),\n        // recursive through each token\n        for (int64_t t = 0; t < T; t++) {\n            size_t t_offset = t * t_stride;\n            size_t state_offset = head_size * C * (t / (T / n_seqs));\n            float * state_cur = state + state_offset;\n            float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[5]->data + state_offset;\n\n            for (int64_t h = h_start; h < h_end; h++) {\n                size_t h_offset = h * h_stride;\n                size_t t_h_offset = t_offset + h_offset;\n                size_t h_2d_offset = h * h_stride_2d;\n\n                for (int64_t i = 0; i < head_size; i++) {\n                    size_t t_h_i_offset = t_h_offset + i;\n                    size_t h_i_offset = h_offset + i;\n                    size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                    float k_val = k[t_h_i_offset];\n                    float r_val = r[t_h_i_offset];\n                    float time_faaaa_val = time_faaaa[h_i_offset];\n                    // RWKV v6: different time_decay for each token.\n                    float time_decay_val = time_decay[t_h_i_offset];\n\n                    for (int64_t j = 0; j < head_size; j++) {\n                        size_t t_h_j_offset = t_h_offset + j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n                        float v_val = v[t_h_j_offset];\n                        float kv_val = v_val * k_val;\n                        float prev_state_val = state_prev[h_2d_i_j_offset];\n                        float temp_val = kv_val * time_faaaa_val + prev_state_val;\n                        dst_data[t_h_j_offset] += temp_val * r_val;\n                        state_cur[h_2d_i_j_offset] = prev_state_val * time_decay_val + kv_val;\n                    }\n                }\n            }\n        }\n    #endif\n}\n\n\nvoid ggml_compute_forward_rwkv_wkv6(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rwkv_wkv6_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_gla\n\nstatic void ggml_compute_forward_gla_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const int64_t T = dst->src[1]->ne[2];\n    const int64_t C = dst->ne[0];\n    const int64_t HEADS = dst->src[1]->ne[1];\n    const int64_t n_seqs = dst->src[4]->ne[1];\n    const int64_t head_size = C / HEADS;\n    const float scale = ggml_get_op_params_f32(dst, 0);\n\n    float * dst_data = (float *) dst->data;\n    float * state = ((float *) dst->data) + C * T;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    if (ith >= HEADS) {\n        return;\n    }\n\n    const int h_start = (HEADS * ith) / nth;\n    const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n                (HEADS * (ith + 1)) / nth : HEADS;\n\n    float * k = (float *) dst->src[0]->data;\n    float * v = (float *) dst->src[1]->data;\n    float * q = (float *) dst->src[2]->data;\n    float * g = (float *) dst->src[3]->data;\n\n    size_t t_stride = HEADS * head_size; // Same to C\n\n    size_t h_stride = C / HEADS;\n    GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n    size_t h_stride_2d = head_size * head_size;\n\n    if (ith == 0) {\n        memset(dst_data, 0, T * C * sizeof(float));\n    }\n    ggml_barrier(params->threadpool);\n\n\n    #if defined(__AVX__) && !defined(__AVX512F__)\n        #define GGML_F32X GGML_F32x8\n        #define GGML_F32X_SET1 GGML_F32x8_SET1\n        #define GGML_F32X_LOAD GGML_F32x8_LOAD\n        #define GGML_F32X_STORE GGML_F32x8_STORE\n        #define GGML_F32X_MUL GGML_F32x8_MUL\n        #define GGML_F32X_FMA GGML_F32x8_FMA\n        #define GLA_VECTOR_SIZE 8\n    #elif defined(__AVX512F__)\n        #define GGML_F32X GGML_F32x16\n        #define GGML_F32X_SET1 GGML_F32x16_SET1\n        #define GGML_F32X_LOAD GGML_F32x16_LOAD\n        #define GGML_F32X_STORE GGML_F32x16_STORE\n        #define GGML_F32X_MUL GGML_F32x16_MUL\n        #define GGML_F32X_FMA GGML_F32x16_FMA\n        #define GLA_VECTOR_SIZE 16\n    #elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n        #define GGML_F32X GGML_F32xt\n        #define GGML_F32X_SET1 GGML_F32xt_SET1\n        #define GGML_F32X_LOAD GGML_F32xt_LOAD\n        #define GGML_F32X_STORE GGML_F32xt_STORE\n        #define GGML_F32X_MUL GGML_F32xt_MUL\n        #define GGML_F32X_FMA GGML_F32xt_FMA\n        #define GLA_VECTOR_SIZE 8\n    #elif defined(__ARM_NEON) && defined(__aarch64__)\n        #define GGML_F32X GGML_F32x4\n        #define GGML_F32X_SET1 GGML_F32x4_SET1\n        #define GGML_F32X_LOAD GGML_F32x4_LOAD\n        #define GGML_F32X_STORE GGML_F32x4_STORE\n        #define GGML_F32X_MUL GGML_F32x4_MUL\n        #define GGML_F32X_FMA GGML_F32x4_FMA\n        #define GLA_VECTOR_SIZE 4\n    #endif\n\n    #ifdef GLA_VECTOR_SIZE\n        int gla_vector_size;\n        #if defined(__ARM_FEATURE_SVE)\n            gla_vector_size = svcntw();\n        #else\n            gla_vector_size = GLA_VECTOR_SIZE;\n        #endif\n        const int64_t vec_count = head_size / gla_vector_size;\n\n        for (int64_t t = 0; t < T; t++) {\n            size_t t_offset = t * t_stride;\n            size_t state_offset = head_size * C * (t / (T / n_seqs));\n            float * state_cur = state + state_offset;\n            float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;\n\n            for (int64_t h = h_start; h < h_end; h++) {\n                size_t h_offset = h * h_stride;\n                size_t t_h_offset = t_offset + h_offset;\n                size_t h_2d_offset = h * h_stride_2d;\n\n                for (int64_t i = 0; i < head_size; i++) {\n                    size_t t_h_i_offset = t_h_offset + i;\n                    size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                    float k_val = k[t_h_i_offset];\n                    float q_val = q[t_h_i_offset] * scale;\n                    float g_val = g[t_h_i_offset];\n\n                    // Broadcast scalar values to vectors\n                    GGML_F32X k_vec = GGML_F32X_SET1(k_val);\n                    GGML_F32X q_vec = GGML_F32X_SET1(q_val);\n                    GGML_F32X g_vec = GGML_F32X_SET1(g_val);\n\n                    for (int64_t j = 0; j < vec_count; j++) {\n                        size_t base_j = j * gla_vector_size;\n                        size_t t_h_j_offset = t_h_offset + base_j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + base_j;\n\n                        // Load x elements at once\n                        GGML_F32X v_vec = GGML_F32X_LOAD(&v[t_h_j_offset]);\n                        GGML_F32X prev_state_vec = GGML_F32X_LOAD(&state_prev[h_2d_i_j_offset]);\n                        GGML_F32X dst_vec = GGML_F32X_LOAD(&dst_data[t_h_j_offset]);\n\n                        // Compute kv = v * k\n                        GGML_F32X kv_vec = GGML_F32X_MUL(v_vec, k_vec);\n\n                        // Compute temp = prev_state * g + kv\n                        GGML_F32X temp_vec = GGML_F32X_FMA(kv_vec, prev_state_vec, g_vec);\n\n                        // Update dst: dst += temp * q\n                        dst_vec = GGML_F32X_FMA(dst_vec, temp_vec, q_vec);\n                        GGML_F32X_STORE(&dst_data[t_h_j_offset], dst_vec);\n\n                        // Update state\n                        GGML_F32X_STORE(&state_cur[h_2d_i_j_offset], temp_vec);\n                    }\n\n                    // Handle remaining elements, this will not be used.\n                    for (int64_t j = vec_count * gla_vector_size; j < head_size; j++) {\n                        size_t t_h_j_offset = t_h_offset + j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + j;\n                        float v_val = v[t_h_j_offset];\n                        float kv_val = v_val * k_val;\n                        float prev_state_val = state_prev[h_2d_i_j_offset];\n                        float temp_val = kv_val + prev_state_val * g_val;\n                        dst_data[t_h_j_offset] += temp_val * q_val;\n                        state_cur[h_2d_i_j_offset] = temp_val;\n                    }\n                }\n            }\n        }\n\n    #else\n        for (int64_t t = 0; t < T; t++) {\n            size_t t_offset = t * t_stride;\n            size_t state_offset = head_size * C * (t / (T / n_seqs));\n            float * state_cur = state + state_offset;\n            float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[4]->data + state_offset;\n\n            for (int64_t h = h_start; h < h_end; h++) {\n                size_t h_offset = h * h_stride;\n                size_t t_h_offset = t_offset + h_offset;\n                size_t h_2d_offset = h * h_stride_2d;\n\n                for (int64_t i = 0; i < head_size; i++) {\n                    size_t t_h_i_offset = t_h_offset + i;\n                    size_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                    float k_val = k[t_h_i_offset];\n                    float q_val = q[t_h_i_offset] * scale;\n                    float g_val = g[t_h_i_offset];\n\n                    for (int64_t j = 0; j < head_size; j++) {\n                        size_t t_h_j_offset = t_h_offset + j;\n                        size_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n                        float v_val = v[t_h_j_offset];\n                        float kv_val = v_val * k_val;\n                        float prev_state_val = state_prev[h_2d_i_j_offset];\n                        float temp_val = prev_state_val * g_val + kv_val;\n                        dst_data[t_h_j_offset] += temp_val * q_val;\n                        state_cur[h_2d_i_j_offset] = temp_val;\n                    }\n                }\n            }\n        }\n    #endif\n}\n\n\nvoid ggml_compute_forward_gla(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_gla_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_solve_tri_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst) {\n    const struct ggml_tensor * src0 = dst->src[0];  // A (lower triangular)\n    const struct ggml_tensor * src1 = dst->src[1];  // B (RHS)\n\n    GGML_TENSOR_BINARY_OP_LOCALS;\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type  == GGML_TYPE_F32);\n\n    GGML_ASSERT(ne00 == ne01); // A must be square\n    GGML_ASSERT(ne0  == ne10); // solution cols == B cols\n    GGML_ASSERT(ne1  == ne11); // solution rows == B rows\n\n    GGML_ASSERT(ne02 == ne12 && ne12 == ne2);\n    GGML_ASSERT(ne03 == ne13 && ne13 == ne3);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int64_t k = ne10;   // number of RHS columns\n    const int64_t n = ne11;   // A is n×n\n    const int64_t nr = ne02 * ne03 * k; // we're parallelizing on columns here, so seq x token x column will be the unit\n\n    // chunks per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // chunk range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    const float * A = (const float *) src0->data;  // [n, n, B1, B2]\n    const float * B = (const float *) src1->data;  // [n, k, B1, B2]\n          float * X = (      float *) dst->data;   // [n, k, B1, B2]\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*k);\n        const int64_t i02 = (ir - i03*ne02*k)/k;\n        const int64_t i01 = (ir - i03*ne02*k - i02*k);\n\n        const float * A_batch = A + i02 * nb02 / sizeof(float) + i03 * nb03 / sizeof(float);\n        const float * B_batch = B + i02 * nb12 / sizeof(float) + i03 * nb13 / sizeof(float);\n\n        float * X_batch = X + i02 * nb2 / sizeof(float) + i03 * nb3 / sizeof(float);\n\n        for (int64_t i00 = 0; i00 < n; ++i00) {\n            float sum = 0.0f;\n            for (int64_t t = 0; t < i00; ++t) {\n                sum += A_batch[i00 * n + t] * X_batch[t * k + i01];\n            }\n\n            const float diag = A_batch[i00 * n + i00];\n            assert(diag != 0.0f && \"Zero diagonal in triangular matrix\");\n\n            X_batch[i00 * k + i01] = (B_batch[i00 * k + i01] - sum) / diag;\n        }\n    }\n}\n\nvoid ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n        ggml_compute_forward_solve_tri_f32(params, dst);\n    } else {\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\n// ggml_compute_forward_gated_delta_net\nstatic void ggml_compute_forward_gated_delta_net_one_chunk(\n    const ggml_compute_params * params,\n    ggml_tensor * dst,\n    int64_t ir0,\n    int64_t ir1) {\n\n    ggml_tensor * src_q     = dst->src[0];\n    ggml_tensor * src_k     = dst->src[1];\n    ggml_tensor * src_v     = dst->src[2];\n    ggml_tensor * src_g     = dst->src[3];\n    ggml_tensor * src_beta  = dst->src[4];\n    ggml_tensor * src_state = dst->src[5];\n\n    const int64_t S_v      = src_v->ne[0];\n    const int64_t H        = src_v->ne[1];\n    const int64_t n_tokens = src_v->ne[2];\n    const int64_t n_seqs   = src_v->ne[3];\n\n    GGML_ASSERT(ggml_is_contiguous_rows(src_q));\n    GGML_ASSERT(ggml_is_contiguous_rows(src_k));\n    GGML_ASSERT(ggml_is_contiguous_rows(src_v));\n    GGML_ASSERT(ggml_is_contiguous(src_g));\n    GGML_ASSERT(ggml_is_contiguous(src_beta));\n    GGML_ASSERT(ggml_is_contiguous(src_state));\n\n    GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v);\n    GGML_ASSERT(src_beta->ne[0] == 1);\n\n    GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne);\n    GGML_TENSOR_LOCALS(size_t,  nbq, src_q, nb);\n    GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne);\n    GGML_TENSOR_LOCALS(size_t,  nbk, src_k, nb);\n    GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne);\n    GGML_TENSOR_LOCALS(size_t,  nbv, src_v, nb);\n    GGML_TENSOR_LOCALS(int64_t, neg, src_g, ne);\n    GGML_TENSOR_LOCALS(size_t,  nbg, src_g, nb);\n    GGML_TENSOR_LOCALS(size_t,  nbb, src_beta, nb);\n\n    const bool kda = (neg0 == S_v);\n\n    // scratch layout per thread: [delta(S_v)]\n    const int64_t scratch_per_thread = S_v;\n    const int ith = params->ith;\n\n    float * delta = (float *)params->wdata + ith * scratch_per_thread + CACHE_LINE_SIZE_F32;\n\n    // output layout: [attn_scores | new_states]\n    // attn_scores: S_v * H * n_tokens * n_seqs floats\n    // new_states:  S_v * S_v * H * n_seqs floats\n    const int64_t attn_score_elems = S_v * H * n_tokens * n_seqs;\n    float * attn_out_base  = (float *)dst->data;\n    float * state_out_base = (float *)dst->data + attn_score_elems;\n\n    const float * state_in_base = (const float *)src_state->data;\n\n  //const int64_t rq1 = nev1 / neq1;\n  //const int64_t rk1 = nev1 / nek1;\n    const int64_t rq3 = nev3 / neq3;\n    const int64_t rk3 = nev3 / nek3;\n\n    const float scale = 1.0f / sqrtf((float) S_v);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t iv1 = ir % H; // head_index\n        const int64_t iv3 = ir / H; // sequence\n\n        const int64_t iq1 = iv1 % neq1;\n        const int64_t ik1 = iv1 % nek1;\n\n        const int64_t iq3 = iv3 / rq3;\n        const int64_t ik3 = iv3 / rk3;\n\n        float * s_out = state_out_base + (iv3 * H + iv1) * S_v * S_v;\n\n        // copy input state into output buffer and operate in-place\n        const float * s_in = state_in_base + (iv3 * H + iv1) * S_v * S_v;\n        memcpy(s_out, s_in, S_v * S_v * sizeof(float));\n\n        // attn output pointer for first token of this (head, seq)\n        float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v;\n\n        for (int64_t t = 0; t < n_tokens; t++) {\n            const float * q_d = (const float *)((const char *)src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1);\n            const float * k_d = (const float *)((const char *)src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1);\n            const float * v_d = (const float *)((const char *)src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1);\n\n            const float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1);\n            const float * g_d    =  (const float *)((const char *)src_g->data    + iv3 * nbg3 + t * nbg2 + iv1 * nbg1);\n\n            // state is stored transposed: s_out[j*S_v + i] = S[i][j]\n            // so row j of s_out = column j of S (contiguous access)\n\n            if (kda) {\n                // precompute exp(g) into delta scratch (reused below)\n                for (int64_t i = 0; i < S_v; ++i) {\n                    delta[i] = expf(g_d[i]);\n                }\n                // S[i][:] *= exp(g[i]) => for each row j of M: M[j][i] *= exp(g[i])\n                for (int64_t j = 0; j < S_v; ++j) {\n                    ggml_vec_mul_f32(S_v, &s_out[j * S_v], &s_out[j * S_v], delta);\n                }\n            } else {\n                ggml_vec_scale_f32(S_v * S_v, s_out, expf(g_d[0]));\n            }\n\n            // delta[j] = sum_i S[i][j] * k[i] = dot(row j of M, k)\n            for (int64_t j = 0; j < S_v; ++j) {\n                float sum = 0.0f;\n                ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, k_d, 0, 1);\n                delta[j] = (v_d[j] - sum) * beta_val;\n            }\n\n            // outer product: S[i][j] += k[i] * delta[j] => M[j][i] += delta[j] * k[i]\n            for (int64_t j = 0; j < S_v; ++j) {\n                ggml_vec_mad_f32(S_v, &s_out[j * S_v], k_d, delta[j]);\n            }\n\n            // attn_out[j] = sum_i S[i][j] * q[i] = dot(row j of M, q)\n            for (int64_t j = 0; j < S_v; ++j) {\n                float sum = 0.0f;\n                ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, q_d, 0, 1);\n                attn_data[j] = sum * scale;\n            }\n\n            attn_data += S_v * H; // advance to next token\n        }\n    }\n}\n\n\nstatic void ggml_compute_forward_gated_delta_net_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    ggml_tensor * V = dst->src[2];\n    int64_t nr = V->ne[1] * V->ne[3];\n\n    // disable for NUMA\n    const bool disable_chunking = ggml_is_numa();\n\n    int nth = params->nth;\n    int ith = params->ith;\n\n    // 4x chunks per thread\n    int nth_scaled = nth * 4;\n    int64_t chunk_size = (nr + nth_scaled - 1) / nth_scaled;\n    int64_t nchunk     = (nr + chunk_size - 1) / chunk_size;\n\n    if (nth == 1 || nchunk < nth || disable_chunking) {\n      nchunk = nth;\n    }\n\n    if (ith == 0) {\n      ggml_threadpool_chunk_set(params->threadpool, nth);\n    }\n\n    ggml_barrier(params->threadpool);\n\n    const int64_t dr = (nr + nchunk - 1) / nchunk;\n\n    int current_chunk = ith;\n\n    while (current_chunk < nchunk) {\n        const int64_t ir0 = dr * current_chunk;\n        const int64_t ir1 = MIN(ir0 + dr, nr);\n\n        ggml_compute_forward_gated_delta_net_one_chunk(params, dst, ir0, ir1);\n        current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n    }\n}\n\nvoid ggml_compute_forward_gated_delta_net(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_gated_delta_net_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_rwkv_wkv7\n\nstatic void ggml_compute_forward_rwkv_wkv7_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n    const int64_t T = dst->src[1]->ne[2];\n    const int64_t C = dst->ne[0];\n    const int64_t HEADS = dst->src[1]->ne[1];\n    const int64_t n_seqs = dst->src[6]->ne[1];\n    const int64_t head_size = C / HEADS;\n\n    float * dst_data = (float *) dst->data;\n    float * state = ((float *) dst->data) + C * T;\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    if (ith >= HEADS) {\n        return;\n    }\n\n    const int h_start = (HEADS * ith) / nth;\n    const int h_end = ((HEADS * (ith + 1)) / nth < HEADS) ?\n                (HEADS * (ith + 1)) / nth : HEADS;\n\n    float * r = (float *) dst->src[0]->data;\n    float * w = (float *) dst->src[1]->data;\n    float * k = (float *) dst->src[2]->data;\n    float * v = (float *) dst->src[3]->data;\n    float * a = (float *) dst->src[4]->data;\n    float * b = (float *) dst->src[5]->data;\n\n    int64_t t_stride = HEADS * head_size; // Same to C\n\n    int64_t h_stride = C / HEADS;\n    GGML_ASSERT(C % HEADS == 0); // C must be divisible by HEADS\n    int64_t h_stride_2d = head_size * head_size;\n\n    #if defined(GGML_SIMD)\n        #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic)\n            // scalar Route to scalar implementation       //TODO: Write SVE code and RVV code\n            for (int64_t t = 0; t < T; t++) {\n                int64_t t_offset = t * t_stride;\n                int64_t state_offset = head_size * C * (t / (T / n_seqs));\n                float * state_cur = state + state_offset;\n                float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n                for (int64_t h = h_start; h < h_end; h++) {\n                    int64_t h_offset = h * h_stride;\n                    int64_t t_h_offset = t_offset + h_offset;\n                    int64_t h_2d_offset = h * h_stride_2d;\n\n                    for (int64_t i = 0; i < head_size; i++) {\n                        int64_t t_h_i_offset = t_h_offset + i;\n                        int64_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                        float v_val = v[t_h_i_offset];\n\n                        float sa = 0, result = 0;\n                        for (int64_t j = 0; j < head_size; j++) {\n                            sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];\n                        }\n\n                        for (int64_t j = 0; j < head_size; j++) {\n                            int64_t t_h_j_offset = t_h_offset + j;\n                            int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n                            float r_val = r[t_h_j_offset];\n                            float w_val = w[t_h_j_offset];\n                            float k_val = k[t_h_j_offset];\n                            float b_val = b[t_h_j_offset];\n                            float kv_val = v_val * k_val;\n                            float prev_state_val = state_prev[h_2d_i_j_offset];\n                            state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n                            result += state_cur[h_2d_i_j_offset] * r_val;\n                        }\n                        dst_data[t_h_i_offset] = result;\n                    }\n                }\n            }\n        #else\n            for (int64_t t = 0; t < T; t++) {\n                int64_t t_offset = t * t_stride;\n                int64_t state_offset = head_size * C * (t / (T / n_seqs));\n                float * state_cur = state + state_offset;\n                float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n                for (int64_t h = h_start; h < h_end; h++) {\n                    int64_t h_offset = h * h_stride;\n                    int64_t t_h_offset = t_offset + h_offset;\n                    int64_t h_2d_offset = h * h_stride_2d;\n\n                    for (int64_t ii = 0; ii < head_size; ii++) {\n                        int64_t t_h_i_offset = t_h_offset + ii;\n                        int64_t h_2d_i_offset = h_2d_offset + ii * h_stride;\n\n                        GGML_F32_VEC v_vec = GGML_F32_VEC_SET1(v[t_h_i_offset]);\n\n                        float sa = 0;\n                        {\n                            GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n                            GGML_F32_VEC ax[GGML_F32_ARR];\n                            GGML_F32_VEC ay[GGML_F32_ARR];\n                            for (int64_t j = 0; j < head_size; j += GGML_F32_STEP) {\n                                for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) {\n                                    ax[kk] = GGML_F32_VEC_LOAD(&a[t_h_offset + j + kk * GGML_F32_EPR]);\n                                    ay[kk] = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_offset + j + kk * GGML_F32_EPR]);\n                                    sum[kk] = GGML_F32_VEC_FMA(sum[kk], ax[kk], ay[kk]);\n                                }\n                            }\n                            GGML_F32_VEC_REDUCE(sa, sum);\n                        }\n\n                        GGML_F32_VEC sa_vec = GGML_F32_VEC_SET1(sa);\n\n                        int64_t j = 0;\n                        GGML_F32_VEC result_vec[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n                        for (; j < head_size; j += GGML_F32_STEP) {\n                            for (int64_t kk = 0; kk < GGML_F32_ARR; kk++) {\n                                int64_t t_h_j_offset = t_h_offset + j + kk * GGML_F32_EPR;\n                                int64_t h_2d_i_j_offset = h_2d_i_offset + j + kk * GGML_F32_EPR;\n\n                                GGML_F32_VEC r_vec = GGML_F32_VEC_LOAD(&r[t_h_j_offset]);\n                                GGML_F32_VEC w_vec = GGML_F32_VEC_LOAD(&w[t_h_j_offset]);\n                                GGML_F32_VEC k_vec = GGML_F32_VEC_LOAD(&k[t_h_j_offset]);\n                                GGML_F32_VEC b_vec = GGML_F32_VEC_LOAD(&b[t_h_j_offset]);\n\n                                k_vec = GGML_F32_VEC_MUL(v_vec, k_vec);\n\n                                GGML_F32_VEC state_vec = GGML_F32_VEC_LOAD(&state_prev[h_2d_i_j_offset]);\n                                // kv + s * decay + sa * b\n                                state_vec = GGML_F32_VEC_FMA(k_vec, state_vec, w_vec);\n                                state_vec = GGML_F32_VEC_FMA(state_vec, sa_vec, b_vec);\n                                GGML_F32_VEC_STORE(&state_cur[h_2d_i_j_offset], state_vec);\n\n                                result_vec[kk] = GGML_F32_VEC_FMA(result_vec[kk], state_vec, r_vec);\n                            }\n                        }\n                        GGML_F32_VEC_REDUCE(dst_data[t_h_i_offset], result_vec);\n\n                        // There shouldn't be left-overs though.\n                        for (; j < head_size; j++) {\n                            int64_t t_h_j_offset = t_h_offset + j;\n                            int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n                            float r_val = r[t_h_j_offset];\n                            float w_val = w[t_h_j_offset];\n                            float k_val = k[t_h_j_offset];\n                            float b_val = b[t_h_j_offset];\n                            float kv_val = v[t_h_i_offset] * k_val;\n\n                            float prev_state_val = state_prev[h_2d_i_j_offset];\n                            state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n                            dst_data[t_h_i_offset] += state_cur[h_2d_i_j_offset] * r_val;\n                        }\n                    }\n                }\n            }\n        #endif\n    #else\n        for (int64_t t = 0; t < T; t++) {\n            int64_t t_offset = t * t_stride;\n            int64_t state_offset = head_size * C * (t / (T / n_seqs));\n            float * state_cur = state + state_offset;\n            float * state_prev = t % (T / n_seqs) ? state_cur : (float*)dst->src[6]->data + state_offset;\n\n            for (int64_t h = h_start; h < h_end; h++) {\n                int64_t h_offset = h * h_stride;\n                int64_t t_h_offset = t_offset + h_offset;\n                int64_t h_2d_offset = h * h_stride_2d;\n\n                for (int64_t i = 0; i < head_size; i++) {\n                    int64_t t_h_i_offset = t_h_offset + i;\n                    int64_t h_2d_i_offset = h_2d_offset + i * h_stride;\n\n                    float v_val = v[t_h_i_offset];\n\n                    float sa = 0, result = 0;\n                    for (int64_t j = 0; j < head_size; j++) {\n                        sa += a[t_h_offset + j] * state_prev[h_2d_i_offset + j];\n                    }\n\n                    for (int64_t j = 0; j < head_size; j++) {\n                        int64_t t_h_j_offset = t_h_offset + j;\n                        int64_t h_2d_i_j_offset = h_2d_i_offset + j;\n\n                        float r_val = r[t_h_j_offset];\n                        float w_val = w[t_h_j_offset];\n                        float k_val = k[t_h_j_offset];\n                        float b_val = b[t_h_j_offset];\n                        float kv_val = v_val * k_val;\n                        float prev_state_val = state_prev[h_2d_i_j_offset];\n                        state_cur[h_2d_i_j_offset] = prev_state_val * w_val + kv_val + sa * b_val;\n                        result += state_cur[h_2d_i_j_offset] * r_val;\n                    }\n                    dst_data[t_h_i_offset] = result;\n                }\n            }\n        }\n    #endif\n}\n\n\nvoid ggml_compute_forward_rwkv_wkv7(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_rwkv_wkv7_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_map_custom1\n\nvoid ggml_compute_forward_map_custom1(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * a = dst->src[0];\n\n    struct ggml_map_custom1_op_params p;\n    memcpy(&p, dst->op_params, sizeof(p));\n\n    p.fun(dst, a, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_map_custom2\n\nvoid ggml_compute_forward_map_custom2(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * a = dst->src[0];\n    const ggml_tensor * b = dst->src[1];\n\n    struct ggml_map_custom2_op_params p;\n    memcpy(&p, dst->op_params, sizeof(p));\n\n    p.fun(dst, a, b, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_map_custom3\n\nvoid ggml_compute_forward_map_custom3(\n        const ggml_compute_params * params,\n              ggml_tensor * dst) {\n\n    const ggml_tensor * a = dst->src[0];\n    const ggml_tensor * b = dst->src[1];\n    const ggml_tensor * c = dst->src[2];\n\n    struct ggml_map_custom3_op_params p;\n    memcpy(&p, dst->op_params, sizeof(p));\n\n    p.fun(dst, a, b, c, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_custom\n\nvoid ggml_compute_forward_custom(\n    const struct ggml_compute_params * params,\n          struct ggml_tensor * dst) {\n\n    struct ggml_custom_op_params p;\n    memcpy(&p, dst->op_params, sizeof(p));\n\n    p.fun(dst, params->ith, params->nth, p.userdata);\n}\n\n// ggml_compute_forward_cross_entropy_loss\n\nstatic void ggml_compute_forward_cross_entropy_loss_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));\n    GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));\n    GGML_ASSERT(ggml_are_same_shape(src0, src1));\n    GGML_ASSERT(ggml_is_scalar(dst));\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    // TODO: handle transposed/permuted matrices\n    const int64_t nc = src0->ne[0];\n    const int64_t nr = ggml_nrows(src0);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    float * sums =  (float *) params->wdata;\n    float * st   = ((float *) params->wdata) + nth + ith*nc;\n    float sum_thread = 0.0f;\n\n    GGML_ASSERT(params->wsize >= sizeof(float) * (nth + nth * nc));\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    for (int64_t i1 = ir0; i1 < ir1; ++i1) {\n        const float * s0 = (const float *)((const char *) src0->data + i1*src0->nb[1]);\n        const float * s1 = (const float *)((const char *) src1->data + i1*src1->nb[1]);\n\n#ifndef NDEBUG\n        for (int64_t i = 0; i < nc; ++i) {\n            //printf(\"p[%d] = %f\\n\", i, p[i]);\n            assert(!isnan(s0[i]));\n            assert(!isnan(s1[i]));\n        }\n#endif // NDEBUG\n\n        float max = -INFINITY;\n        ggml_vec_max_f32(nc, &max, s0);\n        const ggml_float sum_softmax = ggml_vec_log_soft_max_f32(nc, st, s0, max);\n        assert(sum_softmax >= 0.0);\n\n        ggml_vec_add1_f32(nc, st, st, -sum_softmax);\n        ggml_vec_mul_f32(nc, st, st, s1);\n\n        float sum_st = 0.0f;\n        ggml_vec_sum_f32(nc, &sum_st, st);\n        sum_thread += sum_st;\n\n#ifndef NDEBUG\n        for (int64_t i = 0; i < nc; ++i) {\n            assert(!isnan(st[i]));\n            assert(!isinf(st[i]));\n        }\n#endif // NDEBUG\n    }\n    sums[ith] = sum_thread;\n    ggml_barrier(params->threadpool);\n\n    if (ith == 0) {\n        float * dp = (float *) dst->data;\n        ggml_vec_sum_f32(nth, dp, sums);\n        dp[0] *= -1.0f / (float) nr;\n    }\n}\n\nvoid ggml_compute_forward_cross_entropy_loss(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_cross_entropy_loss_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\n// ggml_compute_forward_cross_entropy_loss_back\n\nstatic void ggml_compute_forward_cross_entropy_loss_back_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * grad  = dst->src[0]; // gradient of forward pass output\n    const ggml_tensor * src0f = dst->src[1]; // src0 of forward pass\n    const ggml_tensor * src1f = dst->src[2]; // src1 of forward pass\n\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_is_contiguous(src0f));\n    GGML_ASSERT(ggml_is_contiguous(src1f));\n    GGML_ASSERT(ggml_is_contiguous(grad));\n    GGML_ASSERT(ggml_are_same_shape(src0f, src1f) && ggml_are_same_shape(src0f, dst));\n\n    const int64_t ith = params->ith;\n    const int64_t nth = params->nth;\n\n    // TODO: handle transposed/permuted matrices\n    const int64_t nc = src0f->ne[0];\n    const int64_t nr = ggml_nrows(src0f);\n\n    // rows per thread\n    const int64_t dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int64_t ir0 = dr*ith;\n    const int64_t ir1 = MIN(ir0 + dr, nr);\n\n    const float d_by_nr = ((const float *) grad->data)[0] / (float) nr;\n\n    for (int64_t i1 = ir0; i1 < ir1; i1++) {\n        float       * ds0 = (float       *)((char       *) dst->data   + i1*dst->nb[1]);\n        const float * s0  = (const float *)((const char *) src0f->data + i1*src0f->nb[1]);\n        const float * s1  = (const float *)((const char *) src1f->data + i1*src1f->nb[1]);\n\n#ifndef NDEBUG\n        for (int64_t i = 0; i < nc; ++i) {\n            //printf(\"p[%d] = %f\\n\", i, p[i]);\n            assert(!isnan(s0[i]));\n            assert(!isnan(s1[i]));\n        }\n#endif // NDEBUG\n\n        // soft_max\n        float max = -INFINITY;\n        ggml_vec_max_f32(nc, &max, s0);\n        const ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max);\n        assert(sum > 0.0);\n        ggml_vec_scale_f32(nc, ds0, 1.0/sum);\n\n        // grad(src0f) = (softmax(src0f) - src1f) * grad(cross_entropy_loss(src0f, src1f)) / nr\n        ggml_vec_sub_f32(nc, ds0, ds0, s1);\n        ggml_vec_scale_f32(nc, ds0, d_by_nr);\n\n#ifndef NDEBUG\n        for (int64_t i = 0; i < nc; ++i) {\n            assert(!isnan(ds0[i]));\n            assert(!isinf(ds0[i]));\n        }\n#endif // NDEBUG\n    }\n}\n\nvoid ggml_compute_forward_cross_entropy_loss_back(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_cross_entropy_loss_back_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_opt_step_adamw_f32(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0         = dst->src[0];\n    const ggml_tensor * src0_grad    = dst->src[1];\n    const ggml_tensor * src0_grad_m  = dst->src[2];\n    const ggml_tensor * src0_grad_v  = dst->src[3];\n    const ggml_tensor * adamw_params = dst->src[4];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n    GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_m));\n    GGML_ASSERT(ggml_are_same_shape(src0, src0_grad_v));\n    GGML_ASSERT(ggml_nelements(adamw_params) == 7);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr  = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1)/nth;\n\n    // row range for this thread\n    const int ir0 = dr*ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    const float * adamw_params_ptr = ggml_get_data_f32(adamw_params);\n\n    const float alpha  = adamw_params_ptr[0];\n    const float beta1  = adamw_params_ptr[1];\n    const float beta2  = adamw_params_ptr[2];\n    const float eps    = adamw_params_ptr[3];\n    const float wd     = adamw_params_ptr[4];\n    const float beta1h = adamw_params_ptr[5];\n    const float beta2h = adamw_params_ptr[6];\n    const float keep   = 1.f - alpha * wd;\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        const size_t offset = i03*nb03 + i02*nb02 + i01*nb01;\n\n        float       * w = (float       *) ((char       *) src0->data        + offset); // weight\n        const float * g = (const float *) ((const char *) src0_grad->data   + offset); // grad\n        float       * m = (float       *) ((char       *) src0_grad_m->data + offset);\n        float       * v = (float       *) ((char       *) src0_grad_v->data + offset);\n\n        for (int i00 = 0; i00 < ne00; ++i00) {\n            m[i00] = m[i00]*beta1 +        g[i00]*(1.0f - beta1);\n            v[i00] = v[i00]*beta2 + g[i00]*g[i00]*(1.0f - beta2);\n\n            const float mh =       m[i00]*beta1h;\n            const float vh = sqrtf(v[i00]*beta2h) + eps;\n\n            // The weight decay is applied independently of the Adam momenta m and v.\n            // This is NOT equivalent to l2 regularization that adds w[i00]*w[i00] to the loss.\n            // See: https://arxiv.org/pdf/1711.05101v3.pdf\n            w[i00] = w[i00] * keep - alpha * mh / vh;\n        }\n    }\n}\n\nvoid ggml_compute_forward_opt_step_adamw(\n        const ggml_compute_params * params,\n        ggml_tensor * dst) {\n\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_opt_step_adamw_f32(params, dst);\n            } break;\n        default:\n            {\n                GGML_ABORT(\"fatal error\");\n            }\n    }\n}\n\nstatic void ggml_compute_forward_opt_step_sgd_f32(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0       = dst->src[0];\n    const ggml_tensor * src0_grad  = dst->src[1];\n    const ggml_tensor * sgd_params = dst->src[2];\n\n    GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));\n    GGML_ASSERT(ggml_nelements(sgd_params) == 2);\n\n    const int ith = params->ith;\n    const int nth = params->nth;\n\n    const int nr = ggml_nrows(src0);\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n    GGML_ASSERT(nb00 == sizeof(float));\n\n    // rows per thread\n    const int dr = (nr + nth - 1) / nth;\n\n    // row range for this thread\n    const int ir0 = dr * ith;\n    const int ir1 = MIN(ir0 + dr, nr);\n\n    // using adamw param subset we care about - alpha, wd - could have a separate struct\n    const float * sgd_params_ptr   = ggml_get_data_f32(sgd_params);\n    const float   alpha            = sgd_params_ptr[0];\n    const float   keep             = 1.f - alpha * sgd_params_ptr[1];\n\n    for (int ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir / (ne02 * ne01);\n        const int64_t i02 = (ir - i03 * ne02 * ne01) / ne01;\n        const int64_t i01 = (ir - i03 * ne02 * ne01 - i02 * ne01);\n\n        const size_t offset = i03 * nb03 + i02 * nb02 + i01 * nb01;\n\n        float *       w = (float *) ((char *) src0->data + offset);                   // weight\n        const float * g = (const float *) ((const char *) src0_grad->data + offset);  // grad\n\n        for (int i00 = 0; i00 < ne00; ++i00) {\n            w[i00] = w[i00] * keep - alpha * g[i00];\n        }\n    }\n}\n\nvoid ggml_compute_forward_opt_step_sgd(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    switch (src0->type) {\n        case GGML_TYPE_F32:\n            {\n                ggml_compute_forward_opt_step_sgd_f32(params, dst);\n            }\n            break;\n        default:\n            {\n                GGML_ABORT(\"fatal error - sgd is F32 only\");\n            }\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/ops.h",
    "content": "#pragma once\n\n#include \"ggml.h\"\n\n//\n// cache line\n//\n\n#if defined(__cpp_lib_hardware_interference_size)\n#define CACHE_LINE_SIZE std::hardware_destructive_interference_size\n#else\n#if defined(__POWER9_VECTOR__)\n#define CACHE_LINE_SIZE 128\n#elif defined(__VXE__) || defined(__VXE2__)\n#define CACHE_LINE_SIZE 256\n#else\n#define CACHE_LINE_SIZE 64\n#endif\n#endif\n\nstatic const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);\n\n// Work buffer size for im2col operations in CONV2D\n#define GGML_IM2COL_WORK_SIZE (16 * 1024 * 1024)\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_dup(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add_id(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_acc(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sum(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sum_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cumsum(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mean(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_argmax(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_count_equal(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_repeat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_repeat_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_concat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_out_prod(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_scale(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_set(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cpy(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cont(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rows_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_set_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag_mask_inf(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_diag_mask_zero(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_soft_max(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_soft_max_ext_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rope(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rope_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_clamp(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_transpose_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col_back_f32(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_im2col_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_3d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_transpose_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_conv_2d_dw(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_2d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pool_2d_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_upscale(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pad(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_pad_reflect_1d(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_roll(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_arange(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_timestep_embedding(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_argsort(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_top_k(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_flash_attn_back(\n        const struct ggml_compute_params * params,\n        const bool masked,\n        struct ggml_tensor * dst);\nvoid ggml_compute_forward_ssm_conv(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_ssm_scan(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_win_part(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_win_unpart(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_unary(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_glu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_get_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_add_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rwkv_wkv6(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_custom(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cross_entropy_loss(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cross_entropy_loss_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_opt_step_adamw(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_mul_mat(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_opt_step_sgd(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/quants.c",
    "content": "#define GGML_COMMON_IMPL_C\n#include \"ggml-common.h\"\n\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"ggml-quants.h\"\n#include \"quants.h\"\n\n#include \"arch-fallback.h\"\n\n#include <string.h>\n#include <assert.h>\n#include <float.h>\n#include <stdlib.h> // for qsort\n#include <stdio.h>  // for GGML_ASSERT\n\n#define GROUP_MAX_EPS 1e-15f\n#define GROUP_MAX_EPS_IQ3_XXS 1e-8f\n#define GROUP_MAX_EPS_IQ2_S 1e-8f\n#define GROUP_MAX_EPS_IQ1_M 1e-7f\n#define GROUP_MAX_EPS_IQ1_S 1e-12f\n\n#define UNUSED GGML_UNUSED\n\nvoid quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q4_0_ref(x, y, k);\n}\n\nvoid quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q4_1_ref(x, y, k);\n}\n\nvoid quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q5_0_ref(x, y, k);\n}\n\nvoid quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q5_1_ref(x, y, k);\n}\n\nvoid quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q8_0_ref(x, y, k);\n}\n\nvoid quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q8_1_ref(x, y, k);\n}\n\nvoid quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_mxfp4_ref(x, y, k);\n}\n\nvoid quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_nvfp4_ref(x, y, k);\n}\n\n//\n// 2-6 bit quantization in super-blocks\n//\n\n//========================- 2-bit (de)-quantization\n\nvoid quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    quantize_row_q2_K_ref(x, vy, k);\n}\n\n//========================= 3-bit (de)-quantization\n\nvoid quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    quantize_row_q3_K_ref(x, vy, k);\n}\n\n// ====================== 4-bit (de)-quantization\n\nvoid quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    block_q4_K * GGML_RESTRICT y = vy;\n    quantize_row_q4_K_ref(x, y, k);\n}\n\n// ====================== 5-bit (de)-quantization\n\nvoid quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    block_q5_K * GGML_RESTRICT y = vy;\n    quantize_row_q5_K_ref(x, y, k);\n}\n\n// ====================== 6-bit (de)-quantization\n\nvoid quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    block_q6_K * GGML_RESTRICT y = vy;\n    quantize_row_q6_K_ref(x, y, k);\n}\n\n// ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs)\n\nvoid quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    block_tq1_0 * GGML_RESTRICT y = vy;\n    quantize_row_tq1_0_ref(x, y, k);\n}\n\nvoid quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(k % QK_K == 0);\n    block_tq2_0 * GGML_RESTRICT y = vy;\n    quantize_row_tq2_0_ref(x, y, k);\n}\n\n//===================================== Q8_K ==============================================\n\nvoid quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    quantize_row_q8_K_ref(x, y, k);\n}\n\n//===================================== Dot products =================================\n\nvoid ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F) - 8;\n            const int v1 = (x[ib].qs[j] >>   4) - 8;\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += sumi*GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d);\n    }\n\n    *s = sumf;\n}\n\n// TODO: add WASM SIMD\nvoid ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n    for (; ib < nb; ++ib) {\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const int v0 = (x[ib].qs[j] & 0x0F);\n            const int v1 = (x[ib].qs[j] >>   4);\n\n            sumi0 += (v0 * y[ib].qs[j]);\n            sumi1 += (v1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_MXFP4 == 0);\n    static_assert(QK_MXFP4 == QK8_0, \"QK_MXFP4 and QK8_0 must be the same\");\n\n    const block_mxfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_MXFP4;\n\n    int ib = 0;\n    float sumf = 0;\n\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_E8M0_TO_FP32_HALF(x[ib].e);\n\n        int sumi1 = 0;\n        int sumi2 = 0;\n        for (int j = 0; j < QK_MXFP4/2; ++j) {\n            sumi1 += y[ib].qs[j +          0] * kvalues_mxfp4[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j + QK_MXFP4/2] * kvalues_mxfp4[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\n// NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks\nvoid ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_NVFP4 == 0);\n\n    const block_nvfp4 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_NVFP4;\n\n    float sumf = 0;\n\n    for (int ib = 0; ib < nb; ++ib) {\n        for (int s_idx = 0; s_idx < 4; ++s_idx) {\n            const float d = ggml_ue4m3_to_fp32(x[ib].d[s_idx]);\n            const int q8_block = s_idx / 2;\n            const int q8_off   = (s_idx % 2) * QK_NVFP4_SUB;\n            const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8_block].d);\n\n            int sumi_lo = 0, sumi_hi = 0;\n            for (int j = 0; j < QK_NVFP4_SUB/2; ++j) {\n                const uint8_t qv = x[ib].qs[s_idx*(QK_NVFP4_SUB/2) + j];\n                sumi_lo += y[2*ib + q8_block].qs[q8_off + j +               0] * kvalues_mxfp4[qv & 0xf];\n                sumi_hi += y[2*ib + q8_block].qs[q8_off + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >>  4];\n            }\n\n            sumf += dy * d * (sumi_lo + sumi_hi);\n        }\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    for (; ib < nb; ++ib) {\n        uint32_t qh;\n        memcpy(&qh, x[ib].qh, sizeof(qh));\n\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;\n            const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));\n\n            const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);\n            const int32_t x1 = (int8_t)(((x[ib].qs[j] >>   4) | xh_1) - 16);\n\n            sumi0 += (x0 * y[ib].qs[j]);\n            sumi1 += (x1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_1;\n    const int nb = n / qk;\n\n    int ib = 0;\n    float sumf = 0;\n\n    assert(n % qk == 0);\n    assert(qk == QK5_1);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_1 * GGML_RESTRICT x = vx;\n    const block_q8_1 * GGML_RESTRICT y = vy;\n\n    for (; ib < nb; ++ib) {\n        uint32_t qh;\n        memcpy(&qh, x[ib].qh, sizeof(qh));\n\n        int sumi0 = 0;\n        int sumi1 = 0;\n\n        for (int j = 0; j < qk/2; ++j) {\n            const uint8_t xh_0 = ((qh >> (j +  0)) << 4) & 0x10;\n            const uint8_t xh_1 = ((qh >> (j + 12))     ) & 0x10;\n\n            const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;\n            const int32_t x1 = (x[ib].qs[j] >>  4) | xh_1;\n\n            sumi0 += (x0 * y[ib].qs[j]);\n            sumi1 += (x1 * y[ib].qs[j + qk/2]);\n        }\n\n        int sumi = sumi0 + sumi1;\n        sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n\n    assert(n % qk == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q8_0 * GGML_RESTRICT x = vx;\n    const block_q8_0 * GGML_RESTRICT y = vy;\n\n    int ib = 0;\n    float sumf = 0;\n\n    for (; ib < nb; ++ib) {\n        int sumi = 0;\n\n        for (int j = 0; j < qk; j++) {\n            sumi += x[ib].qs[j]*y[ib].qs[j];\n        }\n\n        sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq1_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243};\n\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        int sum = 0;\n\n        for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) {\n            for (size_t l = 0; l < 5; ++l) {\n                for (size_t m = 0; m < 32; ++m) {\n                    uint8_t q = x[i].qs[j + m] * pow3[l];\n                    uint16_t xi = ((uint16_t) q * 3) >> 8;\n                    sum += (xi - 1) * y[i].qs[j*5 + l*32 + m];\n                }\n            }\n        }\n        for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) {\n            for (size_t l = 0; l < 5; ++l) {\n                for (size_t m = 0; m < 16; ++m) {\n                    uint8_t q = x[i].qs[j + m] * pow3[l];\n                    uint16_t xi = ((uint16_t) q * 3) >> 8;\n                    sum += (xi - 1) * y[i].qs[j*5 + l*16 + m];\n                }\n            }\n        }\n\n        for (size_t l = 0; l < 4; ++l) {\n            for (size_t j = 0; j < sizeof(x->qh); ++j) {\n                uint8_t q = x[i].qh[j] * pow3[l];\n                uint16_t xi = ((uint16_t) q * 3) >> 8;\n                sum += (xi - 1) * y[i].qs[sizeof(x->qs)*5 + l*sizeof(x->qh) + j];\n            }\n        }\n\n        sumf += (float) sum * (GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_tq2_0 * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n    float sumf = 0.0f;\n\n    for (int i = 0; i < nb; ++i) {\n        int32_t sumi = 0;\n\n        for (size_t j = 0; j < sizeof(x->qs); j += 32) {\n            for (size_t l = 0; l < 4; ++l) {\n                for (size_t k = 0; k < 32; ++k) {\n                    sumi += y[i].qs[j*4 + l*32 + k] * (((x[i].qs[j + k] >> (l*2)) & 3) - 1);\n                }\n            }\n        }\n\n        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n\n        sumf += (float) sumi * d;\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q2_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0;\n\n    for (int i = 0; i < nb; ++i) {\n\n        const uint8_t * q2 = x[i].qs;\n        const  int8_t * q8 = y[i].qs;\n        const uint8_t * sc = x[i].scales;\n\n        int summs = 0;\n        for (int j = 0; j < 16; ++j) {\n            summs += y[i].bsums[j] * (sc[j] >> 4);\n        }\n\n        const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);\n        const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);\n\n        int isum = 0;\n        int is = 0;\n        int d;\n        for (int k = 0; k < QK_K/128; ++k) {\n            int shift = 0;\n            for (int j = 0; j < 4; ++j) {\n                d = sc[is++] & 0xF;\n                int isuml = 0;\n                for (int l =  0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);\n                isum += d * isuml;\n                d = sc[is++] & 0xF;\n                isuml = 0;\n                for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);\n                isum += d * isuml;\n                shift += 2;\n                q8 += 32;\n            }\n            q2 += 32;\n        }\n        sumf += dall * isum - dmin * summs;\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const uint32_t kmask1 = 0x03030303;\n    const uint32_t kmask2 = 0x0f0f0f0f;\n\n    const block_q3_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    // scalar version\n    // This function is written like this so the compiler can manage to vectorize most of it\n    // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the\n    // manually vectorized version above. Every other version I tried would run at least 4 times slower.\n    // The ideal situation would be if we could just write the code once, and the compiler would\n    // automatically produce the best possible set of machine instructions, instead of us having to manually\n    // write vectorized versions for AVX, ARM_NEON, etc.\n\n    int8_t  aux8[QK_K];\n    int16_t aux16[8];\n    float   sums [8];\n    int32_t aux32[8];\n    memset(sums, 0, 8*sizeof(float));\n\n    uint32_t auxs[4];\n    const int8_t * scales = (const int8_t*)auxs;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT hm = x[i].hmask;\n        const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n        memset(aux32, 0, 8*sizeof(int32_t));\n        int8_t * GGML_RESTRICT a = aux8;\n        uint8_t m = 1;\n        for (int j = 0; j < QK_K; j += 128) {\n            for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;\n            for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n            a += 32; m <<= 1;\n            for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;\n            for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n            a += 32; m <<= 1;\n            for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;\n            for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n            a += 32; m <<= 1;\n            for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;\n            for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);\n            a += 32; m <<= 1;\n            q3 += 32;\n        }\n        a = aux8;\n\n        memcpy(auxs, x[i].scales, 12);\n        uint32_t tmp = auxs[2];\n        auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);\n        auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);\n        auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);\n        auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);\n        for (int j = 0; j < QK_K/16; ++j) {\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];\n            q8 += 8; a += 8;\n        }\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n    }\n    for (int l = 0; l < 8; ++l) sumf += sums[l];\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q4_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    const uint8_t * mins   = (const uint8_t*)&utmp[2];\n\n    int8_t  aux8[QK_K];\n    int16_t aux16[8];\n    float   sums [8];\n    int32_t aux32[8];\n    memset(sums, 0, 8*sizeof(float));\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n        memset(aux32, 0, 8*sizeof(int32_t));\n        int8_t * GGML_RESTRICT a = aux8;\n        for (int j = 0; j < QK_K/64; ++j) {\n            for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);\n            a += 32;\n            for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l]  >> 4);\n            a += 32; q4 += 32;\n        }\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        int sumi = 0;\n        for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];\n        a = aux8;\n        int is = 0;\n        for (int j = 0; j < QK_K/32; ++j) {\n            int32_t scale = scales[is++];\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n        }\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n        const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n        sumf -= dmin * sumi;\n    }\n    for (int l = 0; l < 8; ++l) sumf += sums[l];\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q5_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    uint32_t utmp[4];\n\n    const uint8_t * scales = (const uint8_t*)&utmp[0];\n    const uint8_t * mins   = (const uint8_t*)&utmp[2];\n\n    int8_t  aux8[QK_K];\n    int16_t aux16[8];\n    float   sums [8];\n    int32_t aux32[8];\n    memset(sums, 0, 8*sizeof(float));\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q4 = x[i].qs;\n        const uint8_t * GGML_RESTRICT hm = x[i].qh;\n        const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n        memset(aux32, 0, 8*sizeof(int32_t));\n        int8_t * GGML_RESTRICT a = aux8;\n        uint8_t m = 1;\n        for (int j = 0; j < QK_K/64; ++j) {\n            for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);\n            for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);\n            a += 32; m <<= 1;\n            for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l]  >> 4);\n            for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);\n            a += 32; m <<= 1;\n            q4 += 32;\n        }\n        memcpy(utmp, x[i].scales, 12);\n        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);\n        const uint32_t uaux = utmp[1] & kmask1;\n        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);\n        utmp[2] = uaux;\n        utmp[0] &= kmask1;\n\n        int sumi = 0;\n        for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];\n        a = aux8;\n        int is = 0;\n        for (int j = 0; j < QK_K/32; ++j) {\n            int32_t scale = scales[is++];\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n        }\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n        const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;\n        sumf -= dmin * sumi;\n    }\n    for (int l = 0; l < 8; ++l) sumf += sums[l];\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_q6_K * GGML_RESTRICT x = vx;\n    const block_q8_K * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    int8_t  aux8[QK_K];\n    int16_t aux16[8];\n    float   sums [8];\n    int32_t aux32[8];\n    memset(sums, 0, 8*sizeof(float));\n\n    float sumf = 0;\n    for (int i = 0; i < nb; ++i) {\n        const uint8_t * GGML_RESTRICT q4 = x[i].ql;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const  int8_t * GGML_RESTRICT q8 = y[i].qs;\n        memset(aux32, 0, 8*sizeof(int32_t));\n        int8_t * GGML_RESTRICT a = aux8;\n        for (int j = 0; j < QK_K; j += 128) {\n            for (int l = 0; l < 32; ++l) {\n                a[l +  0] = (int8_t)((q4[l +  0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;\n                a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;\n                a[l + 64] = (int8_t)((q4[l +  0] >>  4) | (((qh[l] >> 4) & 3) << 4)) - 32;\n                a[l + 96] = (int8_t)((q4[l + 32] >>  4) | (((qh[l] >> 6) & 3) << 4)) - 32;\n            }\n            a  += 128;\n            q4 += 64;\n            qh += 32;\n        }\n        a = aux8;\n        int is = 0;\n        for (int j = 0; j < QK_K/16; ++j) {\n            int scale = x[i].scales[is++];\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n            for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];\n            for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];\n            q8 += 8; a += 8;\n        }\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];\n    }\n    for (int l = 0; l < 8; ++l) sumf += sums[l];\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    uint32_t aux32[2];\n    const uint8_t * aux8 = (const uint8_t *)aux32;\n\n    float sumf = 0.f;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        int32_t bsum = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n            memcpy(aux32, q2, 2*sizeof(uint32_t));\n            q2 += 4;\n            const uint32_t ls = 2*(aux32[1] >> 28) + 1;\n            int32_t sumi = 0;\n            for (int l = 0; l < 4; ++l) {\n                const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);\n                const uint8_t  signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];\n                for (int j = 0; j < 8; ++j) {\n                    sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            bsum += sumi * ls;\n        }\n        sumf += d * bsum;\n    }\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0.f;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint16_t * GGML_RESTRICT q2 = x[i].qs;\n        const uint8_t  * GGML_RESTRICT sc = x[i].scales;\n        const int8_t   * GGML_RESTRICT q8 = y[i].qs;\n        int32_t bsum = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n            const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;\n            const uint16_t ls2 = 2*(sc[ib32] >>  4) + 1;\n            int32_t sumi = 0;\n            for (int l = 0; l < 2; ++l) {\n                const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));\n                const uint8_t  signs = ksigns_iq2xs[q2[l] >> 9];\n                for (int j = 0; j < 8; ++j) {\n                    sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            bsum += sumi * ls1;\n            sumi = 0;\n            for (int l = 2; l < 4; ++l) {\n                const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));\n                const uint8_t  signs = ksigns_iq2xs[q2[l] >> 9];\n                for (int j = 0; j < 8; ++j) {\n                    sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            bsum += sumi * ls2;\n            q2 += 4;\n        }\n        sumf += d * bsum;\n    }\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq2_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; i++) {\n\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const int8_t  * q8 = y[i].qs;\n        const uint8_t * qs = x[i].qs;\n        const uint8_t * qh = x[i].qh;\n        const uint8_t * signs = qs + QK_K/8;\n\n        int bsum = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n            int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf);\n            int ls2 = 1 + 2*(x[i].scales[ib32] >>  4);\n            int sumi1 = 0, sumi2 = 0;\n            for (int l = 0; l < 2; ++l) {\n                const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));\n                for (int j = 0; j < 8; ++j) {\n                    sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            for (int l = 2; l < 4; ++l) {\n                const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));\n                for (int j = 0; j < 8; ++j) {\n                    sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            bsum += ls1 * sumi1 + ls2 * sumi2;\n            qs += 4;\n            signs += 4;\n        }\n\n        sumf += d * bsum;\n    }\n\n    *s = 0.125f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_xxs * GGML_RESTRICT x = vx;\n    const block_q8_K    * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    uint32_t aux32;\n\n    float sumf = 0.f;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT q3 = x[i].qs;\n        const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        int32_t bsum = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {\n            memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);\n            const uint32_t ls = 2*(aux32 >> 28) + 1;\n            int32_t sumi = 0;\n            for (int l = 0; l < 4; ++l) {\n                const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);\n                const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);\n                const uint8_t  signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];\n                for (int j = 0; j < 4; ++j) {\n                    sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);\n                    sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            q3 += 8;\n            bsum += sumi * ls;\n        }\n        sumf += d * bsum;\n    }\n    *s = 0.25f * sumf;\n}\n\nvoid ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq3_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0.f;\n    for (int i = 0; i < nb; ++i) {\n        const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;\n        const uint8_t * GGML_RESTRICT qs = x[i].qs;\n        const uint8_t * GGML_RESTRICT qh = x[i].qh;\n        const uint8_t * GGML_RESTRICT signs = x[i].signs;\n        const int8_t  * GGML_RESTRICT q8 = y[i].qs;\n        int32_t bsum = 0;\n        for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {\n            const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1;\n            const uint32_t ls2 = 2*(x[i].scales[ib32/2] >>  4) + 1;\n            int32_t sumi = 0;\n            for (int l = 0; l < 4; ++l) {\n                const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));\n                const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));\n                for (int j = 0; j < 4; ++j) {\n                    sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);\n                    sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            qs += 8;\n            signs += 4;\n            bsum += sumi * ls1;\n            sumi = 0;\n            for (int l = 0; l < 4; ++l) {\n                const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));\n                const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));\n                for (int j = 0; j < 4; ++j) {\n                    sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);\n                    sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);\n                }\n                q8 += 8;\n            }\n            qs += 8;\n            signs += 4;\n            bsum += sumi * ls2;\n        }\n        sumf += d * bsum;\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_s * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0;\n    for (int i = 0; i < nb; i++) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint16_t * qh = x[i].qh;\n\n        int sumi = 0, sumi1 = 0;\n        for (int ib = 0; ib < QK_K/32; ++ib) {\n            const int ls = 2*((qh[ib] >> 12) & 7) + 1;\n            const int delta = qh[ib] & 0x8000 ? -1 : 1;\n            int lsum = 0;\n            for (int l = 0; l < 4; ++l) {\n                const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));\n                for (int j = 0; j < 8; ++j) {\n                    lsum += q8[j] * grid[j];\n                }\n                q8 += 8;\n            }\n            sumi  += ls * lsum;\n            sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);\n            qs += 4;\n        }\n\n        sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(n % QK_K == 0);\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n\n    const block_iq1_m * GGML_RESTRICT x = vx;\n    const block_q8_K  * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    iq1m_scale_t scale;\n\n    int sum1[2], sum2[2], delta[4];\n\n    float sumf = 0;\n    for (int i = 0; i < nb; i++) {\n\n        const int8_t   * q8 = y[i].qs;\n        const uint8_t  * qs = x[i].qs;\n        const uint8_t  * qh = x[i].qh;\n        const uint16_t * sc = (const uint16_t *)x[i].scales;\n\n        scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n\n        int sumi1 = 0, sumi2 = 0;\n        for (int ib = 0; ib < QK_K/32; ++ib) {\n            delta[0] = qh[0] & 0x08 ? -1 : 1;\n            delta[1] = qh[0] & 0x80 ? -1 : 1;\n            delta[2] = qh[1] & 0x08 ? -1 : 1;\n            delta[3] = qh[1] & 0x80 ? -1 : 1;\n            sum1[0] = sum1[1] = sum2[0] = sum2[1] = 0;\n            for (int l = 0; l < 4; ++l) {\n                const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((uint16_t)qh[l/2] << (8 - 4*(l%2))) & 0x700)));\n                int lsum1 = 0, lsum2 = 0;\n                for (int j = 0; j < 8; ++j) {\n                    lsum1 += q8[j] * grid[j];\n                    lsum2 += q8[j];\n                }\n                q8 += 8;\n                sum1[l/2] += lsum1;\n                sum2[l/2] += lsum2*delta[l];\n            }\n\n            const int ls1 = 2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1;\n            const int ls2 = 2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1;\n\n            sumi1 += sum1[0] * ls1 + sum1[1] * ls2;\n            sumi2 += sum2[0] * ls1 + sum2[1] * ls2;\n            qs += 4;\n            qh += 2;\n        }\n\n        sumf += GGML_CPU_FP16_TO_FP32(scale.f16) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2);\n    }\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK4_NL == 0);\n    static_assert(QK4_NL == QK8_0, \"QK4_NL and QK8_0 must be the same\");\n\n    const block_iq4_nl * GGML_RESTRICT x = vx;\n    const block_q8_0   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK4_NL;\n\n    int ib = 0;\n    float sumf = 0;\n\n    for (; ib < nb; ++ib) {\n        const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d);\n        int sumi1 = 0, sumi2 = 0;\n        for (int j = 0; j < QK4_NL/2; ++j) {\n            sumi1 += y[ib].qs[j+       0] * kvalues_iq4nl[x[ib].qs[j] & 0xf];\n            sumi2 += y[ib].qs[j+QK4_NL/2] * kvalues_iq4nl[x[ib].qs[j] >>  4];\n        }\n        sumf += d * (sumi1 + sumi2);\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {\n    assert(nrc == 1);\n    UNUSED(nrc);\n    UNUSED(bx);\n    UNUSED(by);\n    UNUSED(bs);\n    assert(n % QK_K == 0);\n\n    const block_iq4_xs * GGML_RESTRICT x = vx;\n    const block_q8_K   * GGML_RESTRICT y = vy;\n\n    const int nb = n / QK_K;\n\n    float sumf = 0;\n    for (int ibl = 0; ibl < nb; ++ibl) {\n        const float d4d8 = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;\n        uint16_t h = x[ibl].scales_h;\n        const uint8_t * qs = x[ibl].qs;\n        const int8_t  * q8 = y[ibl].qs;\n        for (int ib = 0; ib < QK_K/32; ib += 2) {\n            const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30);\n            const uint8_t ls2 = (x[ibl].scales_l[ib/2] >>  4) | ((h << 2) & 0x30);\n            h >>= 4;\n            const float d1 = d4d8*(ls1 - 32);\n            const float d2 = d4d8*(ls2 - 32);\n            int sumi1 = 0, sumi2 = 0;\n            for (int j = 0; j < 16; ++j) {\n                sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];\n                sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >>  4];\n            }\n            sumf += d1 * (sumi1 + sumi2);\n            qs += 16;\n            q8 += 32;\n            sumi1 = sumi2 = 0;\n            for (int j = 0; j < 16; ++j) {\n                sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];\n                sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >>  4];\n            }\n            sumf += d2 * (sumi1 + sumi2);\n            qs += 16;\n            q8 += 32;\n        }\n    }\n    *s = sumf;\n}\n\n// ============================ 4-bit non-linear quants\n\nvoid quantize_row_iq4_nl(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    assert(k % QK4_NL == 0);\n    quantize_row_iq4_nl_ref(x, y, k);\n}\n\nvoid quantize_row_iq4_xs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {\n    assert(k % QK_K == 0);\n    quantize_iq4_xs(x, y, 1, k, NULL);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/quants.h",
    "content": "#pragma once\n\n#define GGML_COMMON_DECL_C\n#include \"ggml-common.h\"\n\n#include \"ggml.h\"\n\n// GGML CPU internal header\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// Quantization\nvoid quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_mxfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_nvfp4(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\nvoid quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\n\n// Dot product\nvoid ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_m_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_s_q8_K  (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\n// Generic implementation\nvoid quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);\nvoid ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\nvoid ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy,  size_t by, int nrc);\nvoid ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\nvoid ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/repack.cpp",
    "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n#include \"ggml-backend-impl.h\"\n\n#include \"ggml-impl.h\"\n#include \"ggml-cpu.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"simd-mappings.h\"\n#include \"traits.h\"\n\n#include \"arch-fallback.h\"\n\n#include <cmath>\n#include <cstring>\n#include <cassert>\n#include <cstdio>  // for GGML_ASSERT\n\n#include \"repack.h\"\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#endif\n\n#define UNUSED GGML_UNUSED\n\nstatic inline int nearest_int(float fval) {\n    assert(fabsf(fval) <= 4194303.f);\n    float val = fval + 12582912.f;\n    int i; memcpy(&i, &val, sizeof(int));\n    return (i & 0x007fffff) - 0x00400000;\n}\n\n// Functions to create the interleaved data layout formats\n\n// interleave 4 block_q4_0s in blocks of blck_size_interleave\n// returns an interleaved block_q4_0x4\n// in the interleaved block_q4_0x4, place deltas for 4 block_q4_0 blocks\n// first, then interleave quants from 4 block_q4_0s in blocks of blck_size_interleave\n//\n// - in                  : an array of block_q4_0 pointers\n// - blck_size_interleave : the block_q4_0 quants bytes are interleaved in blocks of\n//                         blck_size_interleave bytes\n// - xor_mask            : the mask to convert the nibbles in block_q4_0 quants bytes\n//                         from bias offset form to pure sign form (this saves subtract\n//                         operations durin unpacking)\n//\n\nextern \"C\" {\n\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n    // scalar\n    const int blck_size_interleave = 1;\n    float srcv[4][QK8_0];\n    float id[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n\n            for (int j = 0; j < QK8_0; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n                amax = MAX(amax, fabsf(srcv[row_iter][j]));\n            }\n\n            const float d = amax / ((1 << 7) - 1);\n            id[row_iter] = d ? 1.0f / d : 0.0f;\n\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n        }\n\n        for (int j = 0; j < QK8_0 * 4; j++) {\n            int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n            int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n            src_offset += (j % blck_size_interleave);\n\n            float x0 = srcv[src_id][src_offset] * id[src_id];\n            y[i].qs[j] = roundf(x0);\n        }\n    }\n}\n\nvoid ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK_K == 256);\n    assert(k % QK_K == 0);\n    const int nb = k / QK_K;\n\n    block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n    const int blck_size_interleave = 1;\n    float srcv[4][QK_K];\n    float iscale[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n            float max = 0;\n\n            for (int j = 0; j < QK_K; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n                // Update the maximum value of the corresponding super block\n                if(amax < fabsf(srcv[row_iter][j])) {\n                    amax = fabsf(srcv[row_iter][j]);\n                    max = srcv[row_iter][j];\n                }\n            }\n\n            iscale[row_iter] = amax ? -127.f/max : 0;\n            y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n        }\n\n        for (int j = 0; j < QK_K / 4; j++) {\n            y[i].bsums[j] = 0;\n        }\n        for (int j = 0; j < QK_K * 4; j++) {\n            int src_id = j % 4;\n            int src_offset = j / 4;\n            int index = ((j >> 6) << 2) + (j & 3);\n\n            float x0 = srcv[src_id][src_offset] * iscale[src_id];\n            y[i].qs[j] = nearest_int(x0);\n            y[i].bsums[index] += y[i].qs[j];\n        }\n    }\n}\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n    // scalar\n    const int blck_size_interleave = 4;\n    float srcv[4][QK8_0];\n    float id[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n\n            for (int j = 0; j < QK8_0; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n                amax = MAX(amax, fabsf(srcv[row_iter][j]));\n            }\n\n            const float d = amax / ((1 << 7) - 1);\n            id[row_iter] = d ? 1.0f / d : 0.0f;\n\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n        }\n\n        for (int j = 0; j < QK8_0 * 4; j++) {\n            int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n            int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n            src_offset += (j % blck_size_interleave);\n\n            float x0 = srcv[src_id][src_offset] * id[src_id];\n            y[i].qs[j] = roundf(x0);\n        }\n    }\n}\n\nvoid ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK8_0 == 32);\n    assert(k % QK8_0 == 0);\n    const int nb = k / QK8_0;\n\n    block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy;\n\n    // scalar\n    const int blck_size_interleave = 8;\n    float srcv[4][QK8_0];\n    float id[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n\n            for (int j = 0; j < QK8_0; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];\n                amax = MAX(amax, fabsf(srcv[row_iter][j]));\n            }\n\n            const float d = amax / ((1 << 7) - 1);\n            id[row_iter] = d ? 1.0f / d : 0.0f;\n\n            y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d);\n        }\n\n        for (int j = 0; j < QK8_0 * 4; j++) {\n            int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n            int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n            src_offset += (j % blck_size_interleave);\n\n            float x0 = srcv[src_id][src_offset] * id[src_id];\n            y[i].qs[j] = roundf(x0);\n        }\n    }\n}\n\nvoid ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK_K == 256);\n    assert(k % QK_K == 0);\n    const int nb = k / QK_K;\n\n    block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n    // scalar\n    const int blck_size_interleave = 4;\n    float srcv[4][QK_K];\n    float iscale[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n            float max = 0;\n\n            for (int j = 0; j < QK_K; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n                // Update the maximum value of the corresponding super block\n                if(amax < fabsf(srcv[row_iter][j])) {\n                    amax = fabsf(srcv[row_iter][j]);\n                    max = srcv[row_iter][j];\n                }\n            }\n\n            iscale[row_iter] = amax ? -127.f/max : 0;\n\n            y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n        }\n\n        for (int j = 0; j < QK_K / 4; j++) {\n            y[i].bsums[j] = 0;\n        }\n\n        // Quants values are interleaved in sequence of four bytes from corresponding super blocks\n        // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving\n        // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on\n        for (int j = 0; j < QK_K * 4; j++) {\n            int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n            int src_id     = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n            src_offset += (j % blck_size_interleave);\n            int index = (((j & 15) >> 2) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3);\n\n            float x0 = srcv[src_id][src_offset] * iscale[src_id];\n            y[i].qs[j] = nearest_int(x0);\n            y[i].bsums[index] += y[i].qs[j];\n        }\n    }\n}\n\nvoid ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {\n    assert(QK_K == 256);\n    assert(k % QK_K == 0);\n    const int nb = k / QK_K;\n\n    block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy;\n\n    // scalar\n    const int blck_size_interleave = 8;\n    float srcv[4][QK_K];\n    float iscale[4];\n\n    for (int i = 0; i < nb; i++) {\n        for (int row_iter = 0; row_iter < 4; row_iter++) {\n            float amax = 0.0f; // absolute max\n            float max = 0;\n\n            for (int j = 0; j < QK_K; j++) {\n                srcv[row_iter][j] = x[row_iter * k + i * QK_K + j];\n                // Update the maximum value of the corresponding super block\n                if(amax < fabsf(srcv[row_iter][j])) {\n                    amax = fabsf(srcv[row_iter][j]);\n                    max = srcv[row_iter][j];\n                }\n            }\n\n            iscale[row_iter] = amax ? -127.f/max : 0;\n\n            y[i].d[row_iter] = amax ? 1/iscale[row_iter] : 0;\n        }\n\n        for (int j = 0; j < QK_K / 4; j++) {\n            y[i].bsums[j] = 0;\n        }\n\n        // Quants values are interleaved in sequence of eight bytes from corresponding super blocks\n        // Bsums values are interleaved in sequence of four bsums from each super block taken for interleaving\n        // i.e first four bsums from the first super block, followed by first four bsums from second super block and so on\n        for (int j = 0; j < QK_K * 4; j++) {\n            int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;\n            int src_id     = (j % (4 * blck_size_interleave)) / blck_size_interleave;\n            src_offset += (j % blck_size_interleave);\n            int index = (((j & 31) >> 3) << 2) + ((j >> 8) << 4) + ((j >> 6) & 3);\n\n            float x0 = srcv[src_id][src_offset] * iscale[src_id];\n            y[i].qs[j] = nearest_int(x0);\n            y[i].bsums[index] += y[i].qs[j];\n        }\n    }\n}\n\n} // extern \"C\"\n\ntemplate <int64_t INTER_SIZE, ggml_type PARAM_TYPE>\nvoid ggml_quantize_mat_t(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row);\n\ntemplate <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_0_4x4(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_0_4x8(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<4, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_K_4x4(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<8, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_K_4x8(x, vy, n_per_row);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_0>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_0_4x1(x, vy, n_per_row);\n}\n\ntemplate <> void ggml_quantize_mat_t<1, GGML_TYPE_Q8_K>(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t nrow, int64_t n_per_row) {\n    assert(nrow == 4);\n    UNUSED(nrow);\n    ggml_quantize_mat_q8_K_4x1(x, vy, n_per_row);\n}\n#endif\n\ntemplate <int M, int N>\nstatic void ggml_gemv_q6_K_NxM_q8_K_generic_impl(int                        n,\n                                                 float * GGML_RESTRICT      s,\n                                                 size_t                     bs,\n                                                 const void * GGML_RESTRICT vx,\n                                                 const void * GGML_RESTRICT vy,\n                                                 int                        nr,\n                                                 int                        nc) {\n    constexpr int blocklen          = M;\n    constexpr int ncols_interleaved = N;\n    const int     qk                = QK_K;\n    const int     nb                = n / qk;\n    const int     blocks_per_half   = 64 / blocklen;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[8];\n\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0f;\n        }\n\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;\n                const int base_h = base_l + 64;\n\n                const int scale_idx_l = base_l / 16;\n                const int scale_idx_h = base_h / 16;\n\n                const int qh_shift_l = ((base_l % 128) / 32) * 2;\n                const int qh_shift_h = ((base_h % 128) / 32) * 2;\n\n                const int qh_half_l = (base_l / 128) * 32;\n                const int qh_half_h = (base_h / 128) * 32;\n\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];\n                    const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];\n\n                    int sumi_l = 0;\n                    int sumi_h = 0;\n\n                    for (int i = 0; i < blocklen; i++) {\n                        const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;\n                        const int l_4    = b_ptr[l].ql[ql_pos] & 0xF;\n                        const int hi_4   = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;\n\n                        const int qh_idx_l    = qh_half_l + ((base_l + i) % 32);\n                        const int qh_chunk_l  = qh_idx_l / blocklen;\n                        const int qh_pos_l    = qh_idx_l % blocklen;\n                        const int qh_offset_l = qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;\n                        const int hi_2_l      = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;\n\n                        const int qh_idx_h    = qh_half_h + ((base_h + i) % 32);\n                        const int qh_chunk_h  = qh_idx_h / blocklen;\n                        const int qh_pos_h    = qh_idx_h % blocklen;\n                        const int qh_offset_h = qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;\n                        const int hi_2_h      = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;\n\n                        const int q_l = ((hi_2_l << 4) | l_4) - 32;\n                        const int q_h = ((hi_2_h << 4) | hi_4) - 32;\n\n                        const int8_t a_l = a_ptr[l].qs[base_l + i];\n                        const int8_t a_h = a_ptr[l].qs[base_h + i];\n\n                        sumi_l += q_l * a_l;\n                        sumi_h += q_h * a_h;\n                    }\n\n                    sumf[j] +=\n                        (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                }\n            }\n        }\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j];\n        }\n    }\n}\n\ntemplate <int M, int N>\nstatic void ggml_gemm_q6_K_NxM_q8_K_generic_impl(int                        n,\n                                                 float * GGML_RESTRICT      s,\n                                                 size_t                     bs,\n                                                 const void * GGML_RESTRICT vx,\n                                                 const void * GGML_RESTRICT vy,\n                                                 int                        nr,\n                                                 int                        nc) {\n    constexpr int blocklen          = M;\n    constexpr int ncols_interleaved = N;\n    const int     qk                = QK_K;\n    const int     nb                = n / qk;\n    const int     blocks_per_half   = 64 / blocklen;\n    const int     q8_half_stride    = 512;\n    const int     q8_low_high_step  = 256;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n\n    float sumf[4][8];\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q6_Kx8 * b_ptr = (const block_q6_Kx8 *) vx + (x * nb);\n\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0f;\n                }\n            }\n\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    const int base_l = (k / blocks_per_half) * 128 + (k % blocks_per_half) * blocklen;\n                    const int base_h = base_l + 64;\n\n                    const int scale_idx_l = base_l / 16;\n                    const int scale_idx_h = base_h / 16;\n\n                    const int qh_shift_l = ((base_l % 128) / 32) * 2;\n                    const int qh_shift_h = ((base_h % 128) / 32) * 2;\n\n                    const int qh_half_l = (base_l / 128) * 32;\n                    const int qh_half_h = (base_h / 128) * 32;\n\n                    const int q8_base = (k / blocks_per_half) * q8_half_stride + (k % blocks_per_half) * (blocklen * 4);\n\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            const int8_t scale_l = b_ptr[l].scales[scale_idx_l * ncols_interleaved + j];\n                            const int8_t scale_h = b_ptr[l].scales[scale_idx_h * ncols_interleaved + j];\n\n                            int sumi_l = 0;\n                            int sumi_h = 0;\n\n                            for (int i = 0; i < blocklen; i++) {\n                                const int ql_pos = k * ncols_interleaved * blocklen + j * blocklen + i;\n                                const int l_4    = b_ptr[l].ql[ql_pos] & 0xF;\n                                const int hi_4   = (b_ptr[l].ql[ql_pos] >> 4) & 0xF;\n\n                                const int qh_idx_l   = qh_half_l + ((base_l + i) % 32);\n                                const int qh_chunk_l = qh_idx_l / blocklen;\n                                const int qh_pos_l   = qh_idx_l % blocklen;\n                                const int qh_offset_l =\n                                    qh_chunk_l * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_l;\n                                const int hi_2_l = (b_ptr[l].qh[qh_offset_l] >> qh_shift_l) & 0x3;\n\n                                const int qh_idx_h   = qh_half_h + ((base_h + i) % 32);\n                                const int qh_chunk_h = qh_idx_h / blocklen;\n                                const int qh_pos_h   = qh_idx_h % blocklen;\n                                const int qh_offset_h =\n                                    qh_chunk_h * (blocklen * ncols_interleaved) + j * blocklen + qh_pos_h;\n                                const int hi_2_h = (b_ptr[l].qh[qh_offset_h] >> qh_shift_h) & 0x3;\n\n                                const int q_l = ((hi_2_l << 4) | l_4) - 32;\n                                const int q_h = ((hi_2_h << 4) | hi_4) - 32;\n\n                                const int8_t q8_l = a_ptr[l].qs[q8_base + m * blocklen + i];\n                                const int8_t q8_h = a_ptr[l].qs[q8_base + m * blocklen + i + q8_low_high_step];\n\n                                sumi_l += q_l * q8_l;\n                                sumi_h += q_h * q8_h;\n                            }\n\n                            sumf[m][j] += (sumi_l * scale_l + sumi_h * scale_h) * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) *\n                                          a_ptr[l].d[m];\n                        }\n                    }\n                }\n            }\n\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\ntemplate <int M, int N>\nstatic void ggml_gemv_q5_K_NxM_q8_K_generic_impl(int                        n,\n                                                 float * GGML_RESTRICT      s,\n                                                 size_t                     bs,\n                                                 const void * GGML_RESTRICT vx,\n                                                 const void * GGML_RESTRICT vy,\n                                                 int                        nr,\n                                                 int                        nc) {\n    constexpr int         blocklen          = M;\n    constexpr int         ncols_interleaved = N;\n    const int             qk                = QK_K;\n    const int             nb                = n / qk;\n    static const uint32_t kmask1            = 0x3f3f3f3f;\n    static const uint32_t kmask2            = 0x0f0f0f0f;\n    static const uint32_t kmask3            = 0x03030303;\n\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float    sumf[ncols_interleaved];\n    float    sum_minf[ncols_interleaved];\n    uint32_t utmp[32];\n    int      sumi1;\n    int      sumi2;\n    int      sumi;\n\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q5_Kx8 * b_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j]     = 0.0;\n            sum_minf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int sb = 0; sb < 8; sb++) {\n                memcpy(utmp + sb * 4, b_ptr[l].scales + sb * K_SCALE_SIZE, K_SCALE_SIZE);\n                utmp[sb * 4 + 3]      = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                utmp[sb * 4 + 1]      = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                utmp[sb * 4 + 2]      = uaux_0;\n                utmp[sb * 4 + 0] &= kmask1;\n            }\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                constexpr int scale_stride = 32;\n                uint8_t *     scales_0     = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride;\n                uint8_t *     scales_1     = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride + 16;\n\n                const int qh_shift = (k / (32 / blocklen)) * 2;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi1 = 0;\n                    sumi2 = 0;\n                    sumi  = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int b_qs_offset = k * ncols_interleaved * blocklen + j * blocklen + i;\n\n                        const int qh_idx      = (k * blocklen + i) % 32;\n                        const int qh_chunk    = qh_idx / blocklen;\n                        const int qh_pos      = qh_idx % blocklen;\n                        const int b_qh_offset = qh_chunk * (blocklen * ncols_interleaved) + j * blocklen + qh_pos;\n\n                        const uint8_t qh_val = b_ptr[l].qh[b_qh_offset];\n                        const uint8_t h0     = (qh_val >> qh_shift) & 1;\n                        const uint8_t h1     = (qh_val >> (qh_shift + 1)) & 1;\n\n                        const int v0 = (int8_t) ((b_ptr[l].qs[b_qs_offset] & 0xF) | (h0 << 4));\n                        const int v1 = (int8_t) ((b_ptr[l].qs[b_qs_offset] >> 4) | (h1 << 4));\n\n                        const int q8_offset = (k / (32 / blocklen)) * 64 + (k % (32 / blocklen)) * blocklen + i;\n\n                        sumi1 = (v0 * a_ptr[l].qs[q8_offset]);\n                        sumi2 = (v1 * a_ptr[l].qs[q8_offset + 32]);\n                        sumi1 = sumi1 * scales_0[j];\n                        sumi2 = sumi2 * scales_1[j];\n                        sumi += sumi1 + sumi2;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                }\n            }\n            for (int sb = 0; sb < 8; sb++) {\n                uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) *\n                                   GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n        }\n    }\n}\n\ntemplate <int M, int N>\nstatic void ggml_gemm_q5_K_NxM_q8_K_generic_impl(int                        n,\n                                                 float * GGML_RESTRICT      s,\n                                                 size_t                     bs,\n                                                 const void * GGML_RESTRICT vx,\n                                                 const void * GGML_RESTRICT vy,\n                                                 int                        nr,\n                                                 int                        nc) {\n    constexpr int         blocklen          = M;\n    constexpr int         ncols_interleaved = N;\n    const int             qk                = QK_K;\n    const int             nb                = n / qk;\n    static const uint32_t kmask1            = 0x3f3f3f3f;\n    static const uint32_t kmask2            = 0x0f0f0f0f;\n    static const uint32_t kmask3            = 0x03030303;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float    sumf[4][ncols_interleaved];\n    float    sum_minf[4][ncols_interleaved];\n    uint32_t utmp[32];\n    int      sumi1;\n    int      sumi2;\n    int      sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q5_Kx8 * b_ptr = (const block_q5_Kx8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j]     = 0.0;\n                    sum_minf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int sb = 0; sb < 8; sb++) {\n                    memcpy(utmp + sb * 4, b_ptr[l].scales + sb * K_SCALE_SIZE, K_SCALE_SIZE);\n                    utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                    utmp[sb * 4 + 1]      = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                    utmp[sb * 4 + 2]      = uaux_0;\n                    utmp[sb * 4 + 0] &= kmask1;\n                }\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    constexpr int scale_stride = 32;\n                    uint8_t *     scales_0     = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride;\n                    uint8_t *     scales_1     = (uint8_t *) utmp + (k / (32 / blocklen)) * scale_stride + 16;\n\n                    const int qh_shift = (k / (32 / blocklen)) * 2;\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi1 = 0;\n                            sumi2 = 0;\n                            sumi  = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int b_qs_offset = k * ncols_interleaved * blocklen + j * blocklen + i;\n\n                                const int qh_idx   = (k * blocklen + i) % 32;\n                                const int qh_chunk = qh_idx / blocklen;\n                                const int qh_pos   = qh_idx % blocklen;\n                                const int b_qh_offset =\n                                    qh_chunk * (blocklen * ncols_interleaved) + j * blocklen + qh_pos;\n\n                                const uint8_t qh_val = b_ptr[l].qh[b_qh_offset];\n                                const uint8_t h0     = (qh_val >> qh_shift) & 1;\n                                const uint8_t h1     = (qh_val >> (qh_shift + 1)) & 1;\n\n                                const int v0 = (int8_t) ((b_ptr[l].qs[b_qs_offset] & 0xF) | (h0 << 4));\n                                const int v1 = (int8_t) ((b_ptr[l].qs[b_qs_offset] >> 4) | (h1 << 4));\n\n                                const int q8_offset = (k / (32 / blocklen)) * 256 +\n                                                      (k % (32 / blocklen)) * 4 * blocklen + m * blocklen + i;\n\n                                sumi1 = (v0 * a_ptr[l].qs[q8_offset]);\n                                sumi2 = (v1 * a_ptr[l].qs[q8_offset + 128]);\n                                sumi1 = sumi1 * scales_0[j];\n                                sumi2 = sumi2 * scales_1[j];\n                                sumi += sumi1 + sumi2;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n                for (int sb = 0; sb < 8; sb++) {\n                    uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n                    for (int m = 0; m < 4; m++) {\n                        const int16_t * bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) *\n                                              GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n                }\n            }\n        }\n    }\n}\n\nextern \"C\" {\n\nvoid ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[8];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 4;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[8];\n    float sum_minf[8];\n    uint32_t utmp[32];\n    int sumi1;\n    int sumi2;\n    int sumi;\n\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n            sum_minf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int sb = 0; sb < 8; sb++) {\n                memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n                utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                utmp[sb * 4 + 2] = uaux_0;\n                utmp[sb * 4 + 0] &= kmask1;\n            }\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                uint8_t * scales_0 = (uint8_t *) utmp + (k / 8) * 32;\n                uint8_t * scales_1 = (uint8_t *) utmp + (k / 8) * 32 + 16;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi1 = 0;\n                    sumi2 = 0;\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n                        sumi1 = (v0 * a_ptr[l].qs[(k / 8) * 64 + (k % 8) * blocklen + i]);\n                        sumi2 = (v1 * a_ptr[l].qs[(k / 8) * 64 + (k % 8) * blocklen + i + 32]);\n                        sumi1 = sumi1 * scales_0[j];\n                        sumi2 = sumi2 * scales_1[j];\n                        sumi += sumi1 + sumi2;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                }\n            }\n            for (int sb = 0; sb < 8; sb++) {\n                uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[8];\n    float sum_minf[8];\n    uint32_t utmp[32];\n    int sumi1;\n    int sumi2;\n    int sumi;\n\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n            sum_minf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int sb = 0; sb < 8; sb++) {\n                memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n                utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                utmp[sb * 4 + 2] = uaux_0;\n                utmp[sb * 4 + 0] &= kmask1;\n            }\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32;\n                uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi1 = 0;\n                    sumi2 = 0;\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n                        sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i]);\n                        sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 64 + (k % 4) * blocklen + i + 32]);\n                        sumi1 = sumi1 * scales_0[j];\n                        sumi2 = sumi2 * scales_1[j];\n                        sumi += sumi1 + sumi2;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                }\n            }\n            for (int sb = 0; sb < 8; sb++) {\n                uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sum_minf[j] += mins[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[8];\n    float sum_minf[8];\n    int sumi1,sumi2,sumi3,sumi4;\n    int sumi;\n\n    const block_q8_K * a_ptr = (const block_q8_K *)vy;\n    for(int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q2_Kx8 * b_ptr = (const block_q2_Kx8 *) vx + (x * nb);\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n            sum_minf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (4 * blocklen)); k++) {\n                const uint8_t *scales_0 = b_ptr[l].scales + (k / 4) * 64 ;\n                const uint8_t *scales_1 = b_ptr[l].scales + (k / 4) * 64 + 16;\n                const uint8_t *scales_2 = b_ptr[l].scales + (k / 4) * 64 + 32;\n                const uint8_t *scales_3 = b_ptr[l].scales + (k / 4) * 64 + 48;\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi1 = 0;\n                    sumi2 = 0;\n                    sumi3 = 0;\n                    sumi4 = 0;\n                    sumi = 0;\n                    int offset = ((k / 2) % 2) + j * 2;\n                    for (int i = 0; i < blocklen; ++i){\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 3);\n                        const int v1 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 2 ) & 3);\n                        const int v2 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4 ) & 3);\n                        const int v3 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 6 ) & 3);\n                        sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i]);\n                        sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 32]);\n                        sumi3 = (v2 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 64]);\n                        sumi4 = (v3 * a_ptr[l].qs[(k >> 2) * 128 + (k % 4) * blocklen + i + 96]);\n\n                        sumi1 = sumi1 * (scales_0[offset] & 0xF);\n                        sumi2 = sumi2 * (scales_1[offset] & 0xF);\n                        sumi3 = sumi3 * (scales_2[offset] & 0xF);\n                        sumi4 = sumi4 * (scales_3[offset] & 0xF);\n                        sumi += sumi1 + sumi2 + sumi3 + sumi4;\n                    }\n                    sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                }\n            }\n            for(int sb = 0; sb < 8; sb++) {\n                const uint8_t *mins = b_ptr[l].scales + sb * 16;\n                for(int j = 0; j < ncols_interleaved; j++){\n                    sum_minf[j] += ((mins[j * 2] >> 4) * a_ptr[l].bsums[sb * 2] + (mins[(j * 2)+ 1] >> 4) * a_ptr[l].bsums[sb * 2 + 1]) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemv_q5_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemv_q5_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\n\nvoid ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemv_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemv_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[4];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                        const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[8];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_iq4_nlx8 * b_ptr = (const block_iq4_nlx8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                        const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[4];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                        const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n                    }\n                    sumf[j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[8];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_mxfp4x8 * b_ptr = (const block_mxfp4x8 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                        const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n                    }\n                    sumf[j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q8_0_4x4_q8_0_generic(int                        n,\n                                     float * GGML_RESTRICT      s,\n                                     size_t                     bs,\n                                     const void * GGML_RESTRICT vx,\n                                     const void * GGML_RESTRICT vy,\n                                     int                        nr,\n                                     int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 4;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[4];\n    int   sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / blocklen); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                        sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_q8_0_4x8_q8_0_generic(int                        n,\n                                     float * GGML_RESTRICT      s,\n                                     size_t                     bs,\n                                     const void * GGML_RESTRICT vx,\n                                     const void * GGML_RESTRICT vy,\n                                     int                        nr,\n                                     int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 8;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[4];\n    int   sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / blocklen); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                        sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j];\n        }\n    }\n}\n\n#if defined __riscv_zvfh\nvoid ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[16];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                        const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n    assert (n % qk == 0);\n    assert (nc % ncols_interleaved == 0);\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n    float sumf[16];\n    float sum_minf[16];\n    uint8_t scales[128];\n    uint8_t mins[128];\n    int sumi1;\n    int sumi2;\n    int sumi;\n    const block_q8_K * a_ptr = (const block_q8_K *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0f;\n            sum_minf[j] = 0.0f;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int i = 0; i < 128; i++) {\n                scales[i] = b_ptr[l].scales[i] & 0x0F;\n                mins[i] = b_ptr[l].scales[i] >> 4;\n            }\n            for (int i = 0; i < 64; i++) {\n                scales[i] |= (b_ptr[l].scales[128 + i] & 0x03) << 4;\n                mins[i] |= (b_ptr[l].scales[128 + i] & 0x0C) << 2;\n                scales[i + 64] |= (b_ptr[l].scales[128 + i] & 0x30);\n                mins[i + 64] |= (b_ptr[l].scales[128 + i] & 0xC0) >> 2;\n            }\n            for (int sb = 0; sb < 8; sb++) {\n                uint8_t *min = &mins[sb * 16];\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sum_minf[j] += min[j] * (a_ptr[l].bsums[sb * 2] + a_ptr[l].bsums[sb * 2 + 1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d;\n                }\n            }\n            for (int sb = 0; sb < 8; sb += 2) {\n                uint8_t *scales_0 = &scales[sb * 16];\n                uint8_t *scales_1 = &scales[(sb + 1) * 16];\n                for (int i = 0; i < QK4_0; i++) {\n                    for (int j = 0; j < ncols_interleaved; j++) {\n                        sumi1 = 0;\n                        sumi2 = 0;\n                        sumi = 0;\n                        const int v0 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] & 0xF);\n                        const int v1 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] >> 4);\n                        sumi1 = (v0 * a_ptr[l].qs[sb * 32 + i]);\n                        sumi2 = (v1 * a_ptr[l].qs[sb * 32 + 32 + i]);\n                        sumi1 = sumi1 * scales_0[j];\n                        sumi2 = sumi2 * scales_1[j];\n                        sumi += sumi1 + sumi2;\n                        sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d;\n                    }\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j] - sum_minf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[16];\n    int sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                        const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                        sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2]));\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];\n    }\n}\n\nvoid ggml_gemv_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen          = 1;\n\n    assert(nr == 1);\n    assert(n % qk == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n    UNUSED(nr);\n\n    float sumf[16];\n    int   sumi;\n\n    const block_q8_0 * a_ptr = (const block_q8_0 *) vy;\n    for (int x = 0; x < nc / ncols_interleaved; x++) {\n        const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n\n        for (int j = 0; j < ncols_interleaved; j++) {\n            sumf[j] = 0.0;\n        }\n        for (int l = 0; l < nb; l++) {\n            for (int k = 0; k < (qk / blocklen); k++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumi = 0;\n                    for (int i = 0; i < blocklen; ++i) {\n                        const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                        sumi += v0 * a_ptr[l].qs[k * blocklen + i];\n                    }\n                    sumf[j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d);\n                }\n            }\n        }\n        for (int j = 0; j < ncols_interleaved; j++) {\n            s[x * ncols_interleaved + j] = sumf[j];\n        }\n    }\n}\n\nvoid ggml_gemv_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    assert(n % QK_K == 0);\n    assert(nr == 1);\n    assert(nc % 16 == 0);\n\n    UNUSED(bs);\n\n    const int nb = n / QK_K;\n    const block_q2_Kx16 * x = (const block_q2_Kx16 *)vx;\n    const block_q8_K    * y = (const block_q8_K *)vy;\n\n    // Layout: Even-Low(0,2,4,6), Odd-Low(1,3,5,7), Even-High(8...), Odd-High(9...)\n    const int sb_perm[16] = {\n        0, 4, 1, 5, 2, 6, 3, 7,  // 0-7\n        8, 12, 9, 13, 10, 14, 11, 15 // 8-15\n    };\n\n    for (int col_tile = 0; col_tile < nc; col_tile += 16) {\n        const block_q2_Kx16 * x_ptr = x + (col_tile / 16) * nb;\n        const block_q8_K    * y_ptr = y;\n\n        float sumf[16] = {0};\n\n        // Loop over K-blocks\n        for (int k_block = 0; k_block < nb; ++k_block) {\n            int32_t isum[16]  = {0};\n            int32_t summs[16] = {0};\n\n            const uint8_t * qs_rhs = x_ptr[k_block].qs;\n            const uint8_t * sc_rhs = x_ptr[k_block].scales;\n            const int8_t  * qs_lhs = y_ptr[k_block].qs;\n            const int16_t * bs_lhs = y_ptr[k_block].bsums;\n\n            // Iterate over sub-blocks 0..15\n            for (int sb = 0; sb < 16; ++sb) {\n                // Correction Term\n                int16_t bsum = bs_lhs[sb];\n                int scale_offset = sb_perm[sb] * 16;\n\n                for (int col = 0; col < 16; ++col) {\n                    uint8_t sc_val = sc_rhs[scale_offset + col];\n                    summs[col] += bsum * (sc_val >> 4); // Min is high 4 bits\n                }\n\n                // Main Dot Product\n                // Calculate base offsets for Q2 unpacking based on SB\n                int byte_base;\n                if (sb < 8) byte_base = (sb % 2 == 0) ? 0 : 16;\n                else        byte_base = (sb % 2 == 0) ? 32 : 48;\n\n                int shift = ((sb / 2) % 4) * 2;\n\n                for (int col = 0; col < 16; ++col) {\n                    uint8_t sc_val = sc_rhs[scale_offset + col];\n                    int32_t d_sb = sc_val & 0xF; // Scale is low 4 bits\n\n                    // Process 16 elements (l=0..15)\n                    for (int l = 0; l < 16; ++l) {\n                        // Q2: Interleaved by column. Byte `l` contains 4 k-values.\n                        int qs_idx = (byte_base + l) * 16 + col;\n                        uint8_t q2_val = (qs_rhs[qs_idx] >> shift) & 3;\n\n                        // Q8: Linear access\n                        int k = sb * 16 + l;\n                        int8_t q8_val = qs_lhs[k];\n\n                        isum[col] += q8_val * q2_val * d_sb;\n                    }\n                }\n            }\n\n            // Finalize K-Block\n            for (int col = 0; col < 16; ++col) {\n                float d_lhs = y_ptr[k_block].d;\n                float d_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].d[col]);\n                float dm_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].dmin[col]);\n\n                float d_all = d_lhs * d_rhs;\n                float d_min = d_lhs * dm_rhs;\n\n                sumf[col] += (isum[col] * d_all) - (summs[col] * d_min);\n            }\n        }\n\n        for (int col = 0; col < 16; ++col) {\n            s[col_tile + col] = sumf[col];\n        }\n    }\n}\n#endif\n\nvoid ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    {\n        float sumf[4][4];\n        int sumi;\n\n        for (int y = 0; y < nr / 4; y++) {\n            const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n            for (int x = 0; x < nc / ncols_interleaved; x++) {\n                const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n                for (int m = 0; m < 4; m++) {\n                    for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n                }\n                for (int l = 0; l < nb; l++) {\n                    for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                        for (int m = 0; m < 4; m++) {\n                            for (int j = 0; j < ncols_interleaved; j++) {\n                                sumi = 0;\n                                for (int i = 0; i < blocklen; ++i) {\n                                    const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                                    const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                                    sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                            (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n                                }\n                                sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                            }\n                        }\n                    }\n                }\n                for (int m = 0; m < 4; m++) {\n                    for (int j = 0; j < ncols_interleaved; j++)\n                        s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][4];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                                const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                        (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][8];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                                const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 4;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][8];\n    float sum_minf[4][8];\n    uint32_t utmp[32];\n    int sumi1;\n    int sumi2;\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                    sum_minf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int sb = 0; sb < 8; sb++) {\n                    memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n                    utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                    utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                    utmp[sb * 4 + 2] = uaux_0;\n                    utmp[sb * 4 + 0] &= kmask1;\n                }\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    uint8_t * scales_0 = (uint8_t *) utmp + (k / 8) * 32;\n                    uint8_t * scales_1 = (uint8_t *) utmp + (k / 8) * 32 + 16;\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi1 = 0;\n                            sumi2 = 0;\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n                                const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n                                sumi1 = (v0 * a_ptr[l].qs[(k / 8) * 256 + (k % 8) * 4 * blocklen + m * blocklen + i]);\n                                sumi2 = (v1 * a_ptr[l].qs[(k / 8) * 256 + (k % 8) * 4 * blocklen + m * blocklen + i + 128]);\n                                sumi1 = sumi1 * scales_0[j];\n                                sumi2 = sumi2 * scales_1[j];\n                                sumi += sumi1 + sumi2;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n                for (int sb = 0; sb < 8; sb++) {\n                    uint8_t * mins = (uint8_t *) utmp + 8 + sb * 16;\n                    for(int m = 0; m < 4; m++) {\n                        const int16_t * bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n                        for(int j = 0; j < ncols_interleaved; j++) {\n                            sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n    static const uint32_t kmask1 = 0x3f3f3f3f;\n    static const uint32_t kmask2 = 0x0f0f0f0f;\n    static const uint32_t kmask3 = 0x03030303;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(bs);\n\n    float sumf[4][8];\n    float sum_minf[4][8];\n    uint32_t utmp[32];\n    int sumi1;\n    int sumi2;\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx8 * b_ptr = (const block_q4_Kx8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                    sum_minf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int sb = 0; sb < 8; sb++) {\n                    memcpy(utmp + sb * 4, b_ptr[l].scales + sb * 12, 12);\n                    utmp[sb * 4 + 3] = ((utmp[sb * 4 + 2] >> 4) & kmask2) | (((utmp[sb * 4 + 1] >> 6) & kmask3) << 4);\n                    const uint32_t uaux_0 = utmp[sb * 4 + 1] & kmask1;\n                    utmp[sb * 4 + 1] = (utmp[sb * 4 + 2] & kmask2) | (((utmp[sb * 4 + 0] >> 6) & kmask3) << 4);\n                    utmp[sb * 4 + 2] = uaux_0;\n                    utmp[sb * 4 + 0] &= kmask1;\n                }\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    uint8_t *scales_0 = (uint8_t*) utmp + (k / 4) * 32;\n                    uint8_t *scales_1 = (uint8_t*) utmp + (k / 4) * 32 + 16;\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi1 = 0;\n                            sumi2 = 0;\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF);\n                                const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4);\n                                sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i]);\n                                sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 256 + (k % 4) * 4 * blocklen + m * blocklen + i + 128]);\n                                sumi1 = sumi1 * scales_0[j];\n                                sumi2 = sumi2 * scales_1[j];\n                                sumi += sumi1 + sumi2;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n                for (int sb = 0; sb < 8; sb++) {\n                    uint8_t *mins = (uint8_t*) utmp + 8 + sb * 16;\n                    for(int m = 0; m < 4; m++) {\n                        const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) * 6);\n                        for(int j = 0; j < ncols_interleaved; j++) {\n                            sum_minf[m][j] += mins[j] * (bsums[0] + bsums[1]) * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][8];\n    float sum_minf[4][8];\n    int sumi1, sumi2, sumi3, sumi4;\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q2_Kx8 * b_ptr = (const block_q2_Kx8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                    sum_minf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (4 * blocklen)); k++) {\n\n                    const uint8_t *scales_0 = b_ptr[l].scales + (k / 4) * 64 ;\n                    const uint8_t *scales_1 = b_ptr[l].scales + (k / 4) * 64 + 16;\n                    const uint8_t *scales_2 = b_ptr[l].scales + (k / 4) * 64 + 32;\n                    const uint8_t *scales_3 = b_ptr[l].scales + (k / 4) * 64 + 48;\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi1 = 0;\n                            sumi2 = 0;\n                            sumi3 = 0;\n                            sumi4 = 0;\n                            sumi = 0;\n                            int offset = ((k / 2) % 2) + j * 2;\n                            for (int i = 0; i < blocklen; ++i){\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 3);\n                                const int v1 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 2 ) & 3);\n                                const int v2 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4 ) & 3);\n                                const int v3 = (int8_t) ((b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 6 ) & 3);\n                                sumi1 = (v0 * a_ptr[l].qs[(k >> 2) * 512 + (k % 4) * 4 * blocklen + m * blocklen + i]);\n                                sumi2 = (v1 * a_ptr[l].qs[(k >> 2) * 512  + (k % 4) * 4 * blocklen + m * blocklen + i + 128]);\n                                sumi3 = (v2 * a_ptr[l].qs[(k >> 2) * 512  + (k % 4) * 4 * blocklen + m * blocklen + i + 256]);\n                                sumi4 = (v3 * a_ptr[l].qs[(k >> 2) * 512  + (k % 4) * 4 * blocklen + m * blocklen + i + 384]);\n                                sumi1 = sumi1 * (scales_0[offset] & 0xF);\n                                sumi2 = sumi2 * (scales_1[offset] & 0xF);\n                                sumi3 = sumi3 * (scales_2[offset] & 0xF);\n                                sumi4 = sumi4 * (scales_3[offset] & 0xF);\n                                sumi += sumi1 + sumi2 + sumi3 + sumi4;\n                            }\n                            sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n                for(int sb = 0; sb < 8; sb++) {\n                    const uint8_t *mins = b_ptr[l].scales + sb * 16;\n                    for(int m = 0; m < 4; m++) {\n                        const int16_t *bsums = a_ptr[l].bsums + (sb * 8) + (m * 4) - ((sb % 2) *  6);\n                        for(int j = 0; j < ncols_interleaved; j++) {\n                            int mins_prod = ((mins[j * 2] >> 4) * bsums[0] + (mins[(j * 2)+ 1] >> 4) * bsums[1]);\n                            sum_minf[m][j] += (mins_prod) * GGML_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n            }\n\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemm_q5_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemm_q5_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    ggml_gemm_q6_K_NxM_q8_K_generic_impl<4, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n   ggml_gemm_q6_K_NxM_q8_K_generic_impl<8, 8>(n, s, bs, vx, vy, nr, nc);\n}\n\nvoid ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    {\n        float sumf[4][4];\n        int sumi;\n\n        for (int y = 0; y < nr / 4; y++) {\n            const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n            for (int x = 0; x < nc / ncols_interleaved; x++) {\n                const block_iq4_nlx4 * b_ptr = (const block_iq4_nlx4 *) vx + (x * nb);\n                for (int m = 0; m < 4; m++) {\n                    for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n                }\n                for (int l = 0; l < nb; l++) {\n                    for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                        for (int m = 0; m < 4; m++) {\n                            for (int j = 0; j < ncols_interleaved; j++) {\n                                sumi = 0;\n                                for (int i = 0; i < blocklen; ++i) {\n                                    const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                                    const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                                    sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                            (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n                                }\n                                sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                            }\n                        }\n                    }\n                }\n                for (int m = 0; m < 4; m++) {\n                    for (int j = 0; j < ncols_interleaved; j++)\n                        s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][8];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_iq4_nlx8 * b_ptr = (const block_iq4_nlx8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                                const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][4];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                                const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 8;\n    const int blocklen = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][8];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_mxfp4x8 * b_ptr = (const block_mxfp4x8 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                                const int v1 = kvalues_mxfp4[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4]));\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q8_0_4x4_q8_0_generic(int                        n,\n                                     float * GGML_RESTRICT      s,\n                                     size_t                     bs,\n                                     const void * GGML_RESTRICT vx,\n                                     const void * GGML_RESTRICT vy,\n                                     int                        nr,\n                                     int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 4;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][4];\n    int   sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / blocklen); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                                sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n                            }\n                            sumf[m][j] +=\n                                sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\n\n\nvoid ggml_gemm_q8_0_4x8_q8_0_generic(int                        n,\n                                     float * GGML_RESTRICT      s,\n                                     size_t                     bs,\n                                     const void * GGML_RESTRICT vx,\n                                     const void * GGML_RESTRICT vy,\n                                     int                        nr,\n                                     int                        nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 4;\n    const int blocklen          = 8;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][4];\n    int   sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q8_0x4 * b_ptr = (const block_q8_0x4 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / blocklen); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                                sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n                            }\n                            sumf[m][j] +=\n                                sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\n#if defined __riscv_zvfh\nvoid ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][16];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_0x16 * b_ptr = (const block_q4_0x16 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);\n                                const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK_K;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert (n % qk == 0);\n    assert (nr % 4 == 0);\n    assert (nc % ncols_interleaved == 0);\n\n    UNUSED(s);\n    UNUSED(bs);\n    UNUSED(vx);\n    UNUSED(vy);\n    UNUSED(nr);\n    UNUSED(nc);\n    UNUSED(nb);\n    UNUSED(ncols_interleaved);\n    UNUSED(blocklen);\n\n    float sumf[4][16];\n    float sum_minf[4][16];\n    uint8_t scales[128];\n    uint8_t mins[128];\n    int sumi1;\n    int sumi2;\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_Kx4 * a_ptr = (const block_q8_Kx4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q4_Kx16 * b_ptr = (const block_q4_Kx16 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                    sum_minf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int i = 0; i < 128; i++) {\n                    scales[i] = b_ptr[l].scales[i] & 0x0F;\n                    mins[i] = b_ptr[l].scales[i] >> 4;\n                }\n                for (int i = 0; i < 64; i++) {\n                    scales[i] |= (b_ptr[l].scales[128 + i] & 0x03) << 4;\n                    mins[i] |= (b_ptr[l].scales[128 + i] & 0x0C) << 2;\n                    scales[i + 64] |= (b_ptr[l].scales[128 + i] & 0x30);\n                    mins[i + 64] |= (b_ptr[l].scales[128 + i] & 0xC0) >> 2;\n                }\n\n                for (int sb = 0; sb < 8; sb++) {\n                    uint8_t *min = &mins[sb * 16];\n                    for(int m = 0; m < 4; m++) {\n                        const int16_t bsums = a_ptr[l].bsums[sb * 8 + m] + a_ptr[l].bsums[sb * 8 + m + 4];\n                        for(int j = 0; j < ncols_interleaved; j++) {\n                            sum_minf[m][j] += min[j] * bsums * GGML_CPU_FP16_TO_FP32(b_ptr[l].dmin[j]) * a_ptr[l].d[m];\n                        }\n                    }\n                }\n\n                for (int sb = 0; sb < 8; sb += 2) {\n                    uint8_t *scales_0 = &scales[sb * 16];\n                    uint8_t *scales_1 = &scales[(sb + 1) * 16];\n\n                    for (int i = 0; i < QK4_0; i++) {\n                        for (int m = 0; m < 4; m++) {\n                            for (int j = 0; j < ncols_interleaved; j++) {\n                                sumi1 = 0;\n                                sumi2 = 0;\n                                sumi = 0;\n\n                                const int v0 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] & 0xF);\n                                const int v1 = (int8_t) (b_ptr[l].qs[sb * 256 + i * 16 + j] >> 4);\n                                sumi1 = (v0 * a_ptr[l].qs[sb * 4 * 32 + i * 4 + m]);\n                                sumi2 = (v1 * a_ptr[l].qs[sb * 4 * 32 + 32 * 4 + i * 4 + m]);\n                                sumi1 = sumi1 * scales_0[j];\n                                sumi2 = sumi2 * scales_1[j];\n                                sumi += sumi1 + sumi2;\n\n                                sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * a_ptr[l].d[m];\n                            }\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j] - sum_minf[m][j];\n                }\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk = QK8_0;\n    const int nb = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen = 1;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][16];\n    int sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_iq4_nlx16 * b_ptr = (const block_iq4_nlx16 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / (2 * blocklen)); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0x0F];\n                                const int v1 = kvalues_iq4nl[b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] >> 4];\n                                sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +\n                                         (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + (qk / 2) * 4]));\n                            }\n                            sumf[m][j] += sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++)\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n            }\n        }\n    }\n}\n\nvoid ggml_gemm_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    const int qk                = QK8_0;\n    const int nb                = n / qk;\n    const int ncols_interleaved = 16;\n    const int blocklen          = 1;\n\n    assert(n % qk == 0);\n    assert(nr % 4 == 0);\n    assert(nc % ncols_interleaved == 0);\n\n    float sumf[4][16];\n    int   sumi;\n\n    for (int y = 0; y < nr / 4; y++) {\n        const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);\n        for (int x = 0; x < nc / ncols_interleaved; x++) {\n            const block_q8_0x16 * b_ptr = (const block_q8_0x16 *) vx + (x * nb);\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    sumf[m][j] = 0.0;\n                }\n            }\n            for (int l = 0; l < nb; l++) {\n                for (int k = 0; k < (qk / blocklen); k++) {\n                    for (int m = 0; m < 4; m++) {\n                        for (int j = 0; j < ncols_interleaved; j++) {\n                            sumi = 0;\n                            for (int i = 0; i < blocklen; ++i) {\n                                const int v0 = b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i];\n                                sumi += v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i];\n                            }\n                            sumf[m][j] +=\n                                sumi * GGML_CPU_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_CPU_FP16_TO_FP32(a_ptr[l].d[m]);\n                        }\n                    }\n                }\n            }\n            for (int m = 0; m < 4; m++) {\n                for (int j = 0; j < ncols_interleaved; j++) {\n                    s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];\n                }\n            }\n        }\n    }\n}\n\n\nvoid ggml_gemm_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {\n    assert(n % QK_K == 0);\n    assert(nr % 4 == 0);\n    assert(nc % 16 == 0);\n    const int nb = n / QK_K;\n    const block_q2_Kx16 * x = (const block_q2_Kx16 *)vx;\n    const block_q8_Kx4  * y = (const block_q8_Kx4 *)vy;\n\n    const int sb_perm[16] = {\n        0, 4, 1, 5, 2, 6, 3, 7,\n        8, 12, 9, 13, 10, 14, 11, 15\n    };\n\n    // Iterate Rows in tiles of 4\n    for (int row_tile = 0; row_tile < nr; row_tile += 4) {\n        // Iterate Columns in tiles of 16\n        for (int col_tile = 0; col_tile < nc; col_tile += 16) {\n\n            const block_q2_Kx16 * x_ptr = x + (col_tile / 16) * nb;\n            const block_q8_Kx4  * y_ptr = y + (row_tile / 4) * nb;\n\n            float sumf[4][16];\n            memset(sumf, 0, sizeof(sumf));\n\n            for (int k_block = 0; k_block < nb; ++k_block) {\n                int32_t isum[4][16];\n                int32_t summs[4][16];\n                memset(isum, 0, sizeof(isum));\n                memset(summs, 0, sizeof(summs));\n\n                const uint8_t * qs_rhs = x_ptr[k_block].qs;\n                const uint8_t * sc_rhs = x_ptr[k_block].scales;\n                const int8_t  * qs_lhs = y_ptr[k_block].qs;\n                const int16_t * bs_lhs = y_ptr[k_block].bsums;\n\n                for (int sb = 0; sb < 16; ++sb) {\n                    int scale_offset = sb_perm[sb] * 16;\n\n                    int byte_base;\n                    if (sb < 8) byte_base = (sb % 2 == 0) ? 0 : 16;\n                    else        byte_base = (sb % 2 == 0) ? 32 : 48;\n                    int shift = ((sb / 2) % 4) * 2;\n\n                    for (int col = 0; col < 16; ++col) {\n                        uint8_t sc_val = sc_rhs[scale_offset + col];\n                        int32_t d_sb = sc_val & 0xF;\n                        int32_t m_sb = sc_val >> 4;\n\n                        // Correction Term\n                        for (int r = 0; r < 4; ++r) {\n                            int bsum_idx = (sb / 4) * 16 + r * 4 + (sb % 4);\n                            summs[r][col] += bs_lhs[bsum_idx] * m_sb;\n                        }\n\n                        // Main Dot Product\n                        for (int l = 0; l < 16; ++l) {\n                            int qs_idx = (byte_base + l) * 16 + col;\n                            uint8_t q2_val = (qs_rhs[qs_idx] >> shift) & 3;\n\n                            // Calculate Q8 index for this specific k and row\n                            int k = sb * 16 + l;\n                            int q8_idx = (k / 4) * 16 + (k % 4);\n\n                            for (int r = 0; r < 4; ++r) {\n                                // Add r*4 to jump to the correct row within the 4x4 chunk\n                                int8_t q8_val = qs_lhs[q8_idx + r * 4];\n                                isum[r][col] += q8_val * q2_val * d_sb;\n                            }\n                        }\n                    }\n                }\n\n                // Finalize K-Block\n                for (int col = 0; col < 16; ++col) {\n                    float d_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].d[col]);\n                    float dm_rhs = GGML_FP16_TO_FP32(x_ptr[k_block].dmin[col]);\n\n                    for (int r = 0; r < 4; ++r) {\n                        float d_lhs = y_ptr[k_block].d[r];\n                        float d_all = d_lhs * d_rhs;\n                        float d_min = d_lhs * dm_rhs;\n                        sumf[r][col] += (isum[r][col] * d_all) - (summs[r][col] * d_min);\n                    }\n                }\n            }\n\n            for (int r = 0; r < 4; ++r) {\n                for (int col = 0; col < 16; ++col) {\n                    s[(row_tile + r) * bs + (col_tile + col)] = sumf[r][col];\n                }\n            }\n        }\n    }\n}\n#endif\n\n} // extern \"C\"\n\nstatic block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_interleave) {\n    block_q8_0x4 out;\n\n    for (int i = 0; i < 4; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK8_0 * 4 / blck_size_interleave;\n    for (int i = 0; i < end; ++i) {\n        int src_id     = i % 4;\n        int src_offset = (i / 4) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n        memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], blck_size_interleave);\n    }\n    return out;\n}\n\nstatic block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) {\n    block_q4_0x4 out;\n\n    for (int i = 0; i < 4; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_0 * 2 / blck_size_interleave;\n\n    if (blck_size_interleave == 8) {\n        const uint64_t xor_mask = 0x8888888888888888ULL;\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 4;\n            int src_offset = (i / 4) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            uint64_t elems;\n            // Using memcpy to avoid unaligned memory accesses\n            memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n            elems ^= xor_mask;\n            memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n        }\n    } else if (blck_size_interleave == 4) {\n        const uint32_t xor_mask = 0x88888888;\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 4;\n            int src_offset = (i / 4) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            uint32_t elems;\n            memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint32_t));\n            elems ^= xor_mask;\n            memcpy(&out.qs[dst_offset], &elems, sizeof(uint32_t));\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\n// interleave 8 block_q4_0s in blocks of blck_size_interleave\n// returns an interleaved block_q4_0x8\n// in the interleaved block_q4_0x8, place deltas for 8 block_q4_0 blocks\n// first, then interleave quants from 8 block_q4_0s in blocks of blck_size_interleave\nstatic block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_interleave) {\n    block_q4_0x8 out;\n\n    for (int i = 0; i < 8; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_0 * 4 / blck_size_interleave;\n    const uint64_t xor_mask = 0x8888888888888888ULL;\n\n    for (int i = 0; i < end; ++i) {\n        int src_id = i % 8;\n        int src_offset = (i / 8) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        uint64_t elems;\n        memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n        elems ^= xor_mask;\n        memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n    }\n\n    return out;\n}\n\nstatic block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) {\n    block_q4_0x16 out;\n\n    for (int i = 0; i < 16; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_0 * 8 / blck_size_interleave;\n\n    if (blck_size_interleave == 1) {\n        const uint8_t xor_mask = 0x88;\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 16;\n            int src_offset = i / 16;\n            int dst_offset = i;\n\n            out.qs[dst_offset] = in[src_id].qs[src_offset] ^ xor_mask;\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic block_q4_Kx8 make_block_q4_Kx8(block_q4_K * in, unsigned int blck_size_interleave) {\n    block_q4_Kx8 out;\n    //Delta(scale) and dmin values of the eight Q4_K structures are copied onto the output interleaved structure\n    for (int i = 0; i < 8; i++) {\n        out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n    }\n\n    for (int i = 0; i < 8; i++) {\n        out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n    }\n\n    const int end = QK_K * 4 / blck_size_interleave;\n\n    // Interleave Q4_K quants by taking 8 bytes at a time\n    for (int i = 0; i < end; ++i) {\n        int src_id = i % 8;\n        int src_offset = (i / 8) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        // buffer large enough for the max interleave block size (8 bytes)\n        uint64_t elems;\n        memcpy(&elems, &in[src_id].qs[src_offset], blck_size_interleave);\n        memcpy(&out.qs[dst_offset], &elems, blck_size_interleave);\n    }\n\n    // The below logic is designed so as to unpack and rearrange scales and mins values in Q4_K\n    // Currently the Q4_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value)\n    // The output Q4_Kx8 structure has 96 bytes\n    // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q4_K structure\n    // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q4_K structures\n    uint8_t s[8], m[8];\n\n    for (int i = 0; i < 4; i++) {\n        for (int j = 0; j < 8; j++) {\n            s[j] = in[j].scales[i] & 63;\n            m[j] = in[j].scales[i + 4] & 63;\n        }\n\n        out.scales[i * 12]      = (s[0] & 63) + ((s[4] & 48) << 2);\n        out.scales[i * 12 + 1]  = (s[1] & 63) + ((s[5] & 48) << 2);\n        out.scales[i * 12 + 2]  = (s[2] & 63) + ((s[6] & 48) << 2);\n        out.scales[i * 12 + 3]  = (s[3] & 63) + ((s[7] & 48) << 2);\n        out.scales[i * 12 + 4]  = (m[0] & 63) + ((m[4] & 48) << 2);\n        out.scales[i * 12 + 5]  = (m[1] & 63) + ((m[5] & 48) << 2);\n        out.scales[i * 12 + 6]  = (m[2] & 63) + ((m[6] & 48) << 2);\n        out.scales[i * 12 + 7]  = (m[3] & 63) + ((m[7] & 48) << 2);\n        out.scales[i * 12 + 8]  = (s[4] & 15) + ((m[4] & 15) << 4);\n        out.scales[i * 12 + 9]  = (s[5] & 15) + ((m[5] & 15) << 4);\n        out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4);\n        out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4);\n\n    }\n\n    for (int i = 0; i < 4; i++) {\n        for (int j = 0; j < 8; j++) {\n            s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15);\n            m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4);\n        }\n\n        out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2);\n        out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2);\n        out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2);\n        out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2);\n        out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2);\n        out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2);\n        out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2);\n        out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2);\n        out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4);\n        out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4);\n        out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4);\n        out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4);\n\n    }\n\n    return out;\n}\n\nstatic block_q4_Kx16 make_block_q4_Kx16(block_q4_K * in, unsigned int blck_size_interleave) {\n    block_q4_Kx16 out;\n    //Delta(scale) and dmin values of the 16 Q4_K structures are copied onto the output interleaved structure\n    for (int i = 0; i < 16; i++) {\n        out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n    }\n\n    for (int i = 0; i < 16; i++) {\n        out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n    }\n\n    const int end = QK_K * 8 / blck_size_interleave;\n\n    if (blck_size_interleave == 1) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 16;\n            int src_offset = i / 16;\n            int dst_offset = i;\n\n            out.qs[dst_offset] = in[src_id].qs[src_offset];\n        }\n\n        // RVV repacking.\n        //\n        // Extract sums and mins for all 8 sub-blocks for each block of Q4_K.\n        uint8_t s[128], m[128];\n        for (int i = 0; i < 4; i++) {\n            for (int j = 0; j < 16; j++) {\n                s[i * 16 + j] = in[j].scales[i] & 63;\n                m[i * 16 + j] = in[j].scales[i + 4] & 63;\n            }\n        }\n        for (int i = 0; i < 4; i++) {\n            for (int j = 0; j < 16; j++) {\n                s[64 + i * 16 + j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i+8] & 15);\n                m[64 + i * 16 + j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i+8] & 240) >> 4);\n            }\n        }\n\n        for (int i = 0; i < 128; i++) {\n            out.scales[i] = (s[i] & 15) | ((m[i] & 15) << 4);\n        }\n        for (int i = 0; i < 64; i++) {\n            out.scales[128 + i] = ((s[i] & 48) >> 4) | ((m[i] & 48) >> 2) | (s[64 + i] & 48) | ((m[64 + i] & 48) << 2);\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic block_q2_Kx8 make_block_q2_Kx8(block_q2_K * in, unsigned int blck_size_interleave) {\n    block_q2_Kx8 out;\n\n    // Delta(scale) and dmin values of the eight Q2_K structures are copied onto the output interleaved structure\n    for (int i = 0; i < 8; i++) {\n        out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n    }\n\n    for (int i = 0; i < 8; i++) {\n        out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n    }\n\n    const int end = QK_K * 2 / blck_size_interleave;\n\n    // Interleave Q2_K quants by taking 8 bytes at a time\n    for (int i = 0; i < end; ++i) {\n        int src_id = i % 8;\n        int src_offset = (i / 8) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        uint64_t elems;\n        memcpy(&elems, &in[src_id].qs[src_offset], sizeof(uint64_t));\n        memcpy(&out.qs[dst_offset], &elems, sizeof(uint64_t));\n    }\n\n    // The below logic is designed so as to unpack and rearrange scales and mins values in Q2_K\n    // Currently the Q2_K structure has 16 scales and 16 mins packed in 16 bytes ( 4 bits for each value)\n    // The output Q2_Kx8 structure has 128 bytes for storing scales and mins\n    // Every 16 byte is packed such that it contains scales and mins for corresponding sub blocks from Q2_K structure\n    // For eg - First 16 bytes contains 16 scales and 16 mins - each of first and second sub blocks from different Q2_K structures\n\n    for (int i = 0; i < 128; i++) {\n        // Index for selecting which q2k super block\n        int src1 = (i % 16) / 2;\n        // Index for selecting scale\n        int src2 = ((i / 16) * 2) + (i % 2);\n\n        out.scales[i] = in[src1].scales[src2];\n    }\n    return out;\n}\n\nstatic block_q5_Kx8 make_block_q5_Kx8(block_q5_K * in, unsigned int blck_size_interleave) {\n    block_q5_Kx8 out;\n    //Delta(scale) and dmin values of the eight Q5_K structures are copied onto the output interleaved structure\n    for (int i = 0; i < 8; i++) {\n        out.d[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n    }\n\n    for (int i = 0; i < 8; i++) {\n        out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n    }\n\n    const int end = QK_K * 4 / blck_size_interleave;\n\n    // Interleave Q5_K quants by taking blck_size_interleave bytes at a time\n    for (int i = 0; i < end; ++i) {\n        int src_id     = i % 8;\n        int src_offset = (i / 8) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], blck_size_interleave);\n    }\n\n    // Repeat for high bits with the same chunk size, since\n    // the high bits are interleaved in Q5_K and the index is\n    // qh_idx = (qs_idx % 32);\n    // qh_val = qh[qh_idx] >> (qs_idx / 32);\n    for (int i = 0; i < end / 4; ++i) {\n        int src_id     = i % 8;\n        int src_offset = (i / 8) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        memcpy(&out.qh[dst_offset], &in[src_id].qh[src_offset], blck_size_interleave);\n    }\n\n    // The below logic is copied over from Q4_K\n    // The point is to unpack all the scales and mins for each sub block every time we load 12 bytes.\n    // Currently the Q5_K structure has 8 scales and 8 mins packed in 12 bytes ( 6 bits for each value)\n    // The output Q5_Kx8 structure has 96 bytes\n    // Every 12 byte is packed such that it contains scales and mins for corresponding sub blocks from Q5_K structure\n    // For eg - First 12 bytes contains 8 scales and 8 mins - each of first sub block from different Q5_K structures\n    uint8_t s[8], m[8];\n\n    for (int i = 0; i < 4; i++) {\n        for (int j = 0; j < 8; j++) {\n            s[j] = in[j].scales[i] & 63;\n            m[j] = in[j].scales[i + 4] & 63;\n        }\n\n        out.scales[i * 12]      = (s[0] & 63) + ((s[4] & 48) << 2);\n        out.scales[i * 12 + 1]  = (s[1] & 63) + ((s[5] & 48) << 2);\n        out.scales[i * 12 + 2]  = (s[2] & 63) + ((s[6] & 48) << 2);\n        out.scales[i * 12 + 3]  = (s[3] & 63) + ((s[7] & 48) << 2);\n        out.scales[i * 12 + 4]  = (m[0] & 63) + ((m[4] & 48) << 2);\n        out.scales[i * 12 + 5]  = (m[1] & 63) + ((m[5] & 48) << 2);\n        out.scales[i * 12 + 6]  = (m[2] & 63) + ((m[6] & 48) << 2);\n        out.scales[i * 12 + 7]  = (m[3] & 63) + ((m[7] & 48) << 2);\n        out.scales[i * 12 + 8]  = (s[4] & 15) + ((m[4] & 15) << 4);\n        out.scales[i * 12 + 9]  = (s[5] & 15) + ((m[5] & 15) << 4);\n        out.scales[i * 12 + 10] = (s[6] & 15) + ((m[6] & 15) << 4);\n        out.scales[i * 12 + 11] = (s[7] & 15) + ((m[7] & 15) << 4);\n    }\n\n    for (int i = 0; i < 4; i++) {\n        for (int j = 0; j < 8; j++) {\n            s[j] = ((in[j].scales[i] & 192) >> 2) | (in[j].scales[i + 8] & 15);\n            m[j] = ((in[j].scales[i + 4] & 192) >> 2) | ((in[j].scales[i + 8] & 240) >> 4);\n        }\n\n        out.scales[i * 12 + 48] = (s[0] & 63) + ((s[4] & 48) << 2);\n        out.scales[i * 12 + 49] = (s[1] & 63) + ((s[5] & 48) << 2);\n        out.scales[i * 12 + 50] = (s[2] & 63) + ((s[6] & 48) << 2);\n        out.scales[i * 12 + 51] = (s[3] & 63) + ((s[7] & 48) << 2);\n        out.scales[i * 12 + 52] = (m[0] & 63) + ((m[4] & 48) << 2);\n        out.scales[i * 12 + 53] = (m[1] & 63) + ((m[5] & 48) << 2);\n        out.scales[i * 12 + 54] = (m[2] & 63) + ((m[6] & 48) << 2);\n        out.scales[i * 12 + 55] = (m[3] & 63) + ((m[7] & 48) << 2);\n        out.scales[i * 12 + 56] = (s[4] & 15) + ((m[4] & 15) << 4);\n        out.scales[i * 12 + 57] = (s[5] & 15) + ((m[5] & 15) << 4);\n        out.scales[i * 12 + 58] = (s[6] & 15) + ((m[6] & 15) << 4);\n        out.scales[i * 12 + 59] = (s[7] & 15) + ((m[7] & 15) << 4);\n    }\n\n    return out;\n}\n\nstatic block_q6_Kx8 make_block_q6_Kx8(block_q6_K * in, unsigned int blck_size_interleave) {\n    block_q6_Kx8  out;\n    constexpr int n_blocks = 8;  // Kx8\n    for (int i = 0; i < n_blocks; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end_ls = QK_K * 4 / blck_size_interleave;\n    // Interleave Q6_K quants by taking blck_size_interleave bytes at a time\n    for (int i = 0; i < end_ls; ++i) {\n        int src_id     = i % n_blocks;\n        int src_offset = (i / n_blocks) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        uint64_t elem_ls;\n        memcpy(&elem_ls, &in[src_id].ql[src_offset], blck_size_interleave);\n        memcpy(&out.ql[dst_offset], &elem_ls, blck_size_interleave);\n    }\n\n    // Interleave high bits using same chunk size as low bits\n    const int end_hs = end_ls / 2;\n    for (int i = 0; i < end_hs; ++i) {\n        int src_id     = i % n_blocks;\n        int src_offset = (i / n_blocks) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n\n        uint64_t elem_hs;\n        memcpy(&elem_hs, &in[src_id].qh[src_offset], blck_size_interleave);\n        memcpy(&out.qh[dst_offset], &elem_hs, blck_size_interleave);\n    }\n\n    // The below logic is designed so as to unpack and rearrange scales in Q6_K\n    // The output Q6_Kx8 structure interleaves the 8 bit scales in the same fashion as the quants\n    // Q6_K structure has an 8-bit scale per 16 elements -> 16 scales\n    // scales: [0 bl0 0 bl1 ... 0 bl7][1 bl0 ... 1 bl7] ... [15 bl0 ... 15 bl7]  (bl = block)\n    constexpr int n_scales = QK_K / 16;\n\n    for (int i = 0; i < n_blocks; i++) {\n        for (int j = 0; j < n_scales; j++) {\n            out.scales[j * n_blocks + i] = in[i].scales[j];\n        }\n    }\n\n    return out;\n}\n\nstatic block_q2_Kx16 make_block_q2_Kx16(const block_q2_K * in, unsigned int blck_size_interleave) {\n    block_q2_Kx16 out;\n    constexpr int N_COLS = 16;\n\n    // 1. Copy Super-Scales (d) and Super-Mins (dmin)\n    for (int i = 0; i < N_COLS; i++) {\n        out.d[i]    = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d;\n        out.dmin[i] = in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin;\n    }\n\n    // 2. Interleave Q2_K Data\n    const int bytes_per_col = 64;\n    const int total_bytes = N_COLS * bytes_per_col;\n    const int end = total_bytes / blck_size_interleave;\n\n    for (int i = 0; i < end; ++i) {\n        int src_col_id = i % N_COLS;\n        int src_offset = (i / N_COLS) * blck_size_interleave;\n        int dst_offset = i * blck_size_interleave;\n        memcpy(&out.qs[dst_offset], &in[src_col_id].qs[src_offset], blck_size_interleave);\n    }\n\n    // 3. Repack Scales into the Optimized \"Sequential-Parallel\" Layout\n    int out_idx = 0;\n\n    // Arrays define the sub-block order for each group\n    const int even_low_sbs[]  = {0, 2, 4, 6};\n    const int odd_low_sbs[]   = {1, 3, 5, 7};\n    const int even_high_sbs[] = {8, 10, 12, 14};\n    const int odd_high_sbs[]  = {9, 11, 13, 15};\n\n    // Pack Group 1: Even-Low\n    for (int sb : even_low_sbs) {\n        for (int col = 0; col < N_COLS; col++) {\n            out.scales[out_idx++] = in[col].scales[sb];\n        }\n    }\n\n    // Pack Group 2: Odd-Low\n    for (int sb : odd_low_sbs) {\n        for (int col = 0; col < N_COLS; col++) {\n            out.scales[out_idx++] = in[col].scales[sb];\n        }\n    }\n\n    // Pack Group 3: Even-High\n    for (int sb : even_high_sbs) {\n        for (int col = 0; col < N_COLS; col++) {\n            out.scales[out_idx++] = in[col].scales[sb];\n        }\n    }\n\n    // Pack Group 4: Odd-High\n    for (int sb : odd_high_sbs) {\n        for (int col = 0; col < N_COLS; col++) {\n            out.scales[out_idx++] = in[col].scales[sb];\n        }\n    }\n\n    return out;\n}\n\nstatic int repack_q4_0_to_q4_0_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n    GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n    constexpr int nrows_interleaved = 4;\n\n    block_q4_0x4 * dst = (block_q4_0x4 *)t->data;\n    const block_q4_0 * src = (const block_q4_0 *)data;\n    block_q4_0 dst_tmp[4];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK4_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_0x4(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_K_to_q4_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n    GGML_ASSERT(interleave_block == 8 || interleave_block == 4);\n    constexpr int nrows_interleaved = 8;\n\n    block_q4_Kx8 * dst = (block_q4_Kx8*)t->data;\n    const block_q4_K * src = (const block_q4_K*) data;\n    block_q4_K dst_tmp[8];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i  = 0; i < nrows_interleaved; i++ ) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_Kx8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_K_to_q4_K_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n    constexpr int nrows_interleaved = 16;\n\n    block_q4_Kx16 * dst = (block_q4_Kx16*)t->data;\n    const block_q4_K * src = (const block_q4_K*) data;\n    block_q4_K dst_tmp[16];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i  = 0; i < nrows_interleaved; i++ ) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_Kx16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q2_K_to_q2_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q2_K);\n    GGML_ASSERT(interleave_block == 8);\n    constexpr int nrows_interleaved = 8;\n\n    block_q2_Kx8 * dst = (block_q2_Kx8*)t->data;\n    const block_q2_K * src = (const block_q2_K*) data;\n    block_q2_K dst_tmp[8];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q2_Kx8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q2_K_to_q2_K_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q2_K);\n    constexpr int nrows_interleaved = 16;\n\n    block_q2_Kx16 * dst = (block_q2_Kx16*)t->data;\n    const block_q2_K * src = (const block_q2_K*) data;\n\n    block_q2_K dst_tmp[nrows_interleaved];\n\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q2_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            // This loop gathers 16 separate blocks (one from each column)\n            // that correspond to the same K-dimension chunk.\n            for (int i  = 0; i < nrows_interleaved; i++ ) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n\n            *dst++ = make_block_q2_Kx16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_0_to_q4_0_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n    constexpr int nrows_interleaved = 16;\n\n    block_q4_0x16 * dst = (block_q4_0x16*)t->data;\n    const block_q4_0 * src = (const block_q4_0*) data;\n    block_q4_0 dst_tmp[16];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK4_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i  = 0; i < nrows_interleaved; i++ ) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_0x16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q5_K_to_q5_K_8_bl(struct ggml_tensor *       t,\n                                    int                        interleave_block,\n                                    const void * GGML_RESTRICT data,\n                                    size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q5_K);\n    GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n    constexpr int nrows_interleaved = 8;\n\n    block_q5_Kx8 *     dst = (block_q5_Kx8 *) t->data;\n    const block_q5_K * src = (const block_q5_K *) data;\n    block_q5_K         dst_tmp[8];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q5_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q5_Kx8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n}\n\nstatic int repack_q6_K_to_q6_K_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q6_K);\n    GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n    constexpr int nrows_interleaved = 8;\n\n    block_q6_Kx8 * dst = (block_q6_Kx8 *)t->data;\n    const block_q6_K * src = (const block_q6_K *) data;\n    block_q6_K dst_tmp[8];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK_K;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q6_K));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q6_Kx8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n}\n\nstatic int repack_q4_0_to_q4_0_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n    GGML_ASSERT(interleave_block == 8);\n    constexpr int nrows_interleaved = 8;\n\n    block_q4_0x8 * dst = (block_q4_0x8*)t->data;\n    const block_q4_0 * src = (const block_q4_0*) data;\n    block_q4_0 dst_tmp[8];\n    int nrow = ggml_nrows(t);\n    int nblocks = t->ne[0] / QK4_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i  = 0; i < nrows_interleaved; i++ ) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_0x8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q8_0_to_q8_0_4_bl(struct ggml_tensor *       t,\n                                    int                        interleave_block,\n                                    const void * GGML_RESTRICT data,\n                                    size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q8_0);\n    GGML_ASSERT(interleave_block == 4 || interleave_block == 8);\n    constexpr int nrows_interleaved = 4;\n\n    block_q8_0x4 *     dst = (block_q8_0x4 *) t->data;\n    const block_q8_0 * src = (const block_q8_0 *) data;\n    block_q8_0         dst_tmp[4];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK8_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q8_0x4(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n}\n\nstatic block_q8_0x16 make_block_q8_0x16(block_q8_0 * in, unsigned int blck_size_interleave) {\n    block_q8_0x16 out;\n\n    for (int i = 0; i < 16; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK8_0 * 16 / blck_size_interleave;\n\n    if (blck_size_interleave == 1) {\n        for (int i = 0; i < end; ++i) {\n            int src_id     = i % 16;\n            int src_offset = i / 16;\n            int dst_offset = i;\n            out.qs[dst_offset] = in[src_id].qs[src_offset];\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_q8_0_to_q8_0_16_bl(struct ggml_tensor *       t,\n                                    int                        interleave_block,\n                                    const void * GGML_RESTRICT data,\n                                    size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q8_0);\n    constexpr int nrows_interleaved = 16;\n\n    block_q8_0x16 *     dst = (block_q8_0x16 *) t->data;\n    const block_q8_0 * src = (const block_q8_0 *) data;\n    block_q8_0         dst_tmp[16];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK8_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q8_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q8_0x16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n}\n\nstatic block_iq4_nlx4 make_block_iq4_nlx4(block_iq4_nl * in, unsigned int blck_size_interleave) {\n    block_iq4_nlx4 out;\n\n    for (int i = 0; i < 4; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_NL * 2 / blck_size_interleave;\n\n    // TODO: this branch seems wrong\n    //if (blck_size_interleave == 8) {\n    //    for (int i = 0; i < end; ++i) {\n    //        int src_id = i % 4;\n    //        int src_offset = (i / 4) * blck_size_interleave;\n    //        int dst_offset = i * blck_size_interleave;\n\n    //        // Using memcpy to avoid unaligned memory accesses\n    //        memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n    //    }\n    //} else\n    if (blck_size_interleave == 4) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 4;\n            int src_offset = (i / 4) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint32_t));\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n    GGML_ASSERT(interleave_block == 4);\n\n    const block_iq4_nl   * src = (const block_iq4_nl   *)data;\n          block_iq4_nlx4 * dst = (      block_iq4_nlx4 *)t->data;\n\n    block_iq4_nl dst_tmp[4];\n\n    int nrow = ggml_nrows(t);\n    int nrows_interleaved = 4;\n    int nblocks = t->ne[0] / QK4_NL;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_iq4_nlx4(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic block_iq4_nlx8 make_block_iq4_nlx8(block_iq4_nl * in, unsigned int blck_size_interleave) {\n    block_iq4_nlx8 out;\n\n    for (int i = 0; i < 8; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_NL * 4 / blck_size_interleave;\n\n    if (blck_size_interleave == 8) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 8;\n            int src_offset = (i / 8) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n    GGML_ASSERT(interleave_block == 8);\n\n    const block_iq4_nl   * src = (const block_iq4_nl   *)data;\n          block_iq4_nlx8 * dst = (      block_iq4_nlx8 *)t->data;\n\n    block_iq4_nl dst_tmp[8];\n\n    int nrow = ggml_nrows(t);\n    int nrows_interleaved = 8;\n    int nblocks = t->ne[0] / QK4_NL;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n    if (t->ne[1] % nrows_interleaved != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_iq4_nlx8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic block_iq4_nlx16 make_block_iq4_nlx16(block_iq4_nl * in, unsigned int blck_size_interleave) {\n    block_iq4_nlx16 out;\n\n    for (int i = 0; i < 16; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    const int end = QK4_NL * 8 / blck_size_interleave;\n\n    if (blck_size_interleave == 1) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 16;\n            int src_offset = i / 16;\n            int dst_offset = i;\n\n            out.qs[dst_offset] = in[src_id].qs[src_offset];\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_iq4_nl_to_iq4_nl_16_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_IQ4_NL);\n    GGML_ASSERT(interleave_block == 1);\n\n    const block_iq4_nl    * src = (const block_iq4_nl   *)data;\n          block_iq4_nlx16 * dst = (      block_iq4_nlx16 *)t->data;\n\n    block_iq4_nl dst_tmp[16];\n\n    int nrow = ggml_nrows(t);\n    int nrows_interleaved = 16;\n    int nblocks = t->ne[0] / QK4_NL;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_iq4_nl));\n\n    if (t->ne[1] % nrows_interleaved != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_iq4_nlx16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic block_mxfp4x4 make_block_mxfp4x4(block_mxfp4 * in, unsigned int blck_size_interleave) {\n    block_mxfp4x4 out;\n\n    for (int i = 0; i < 4; i++) {\n        out.e[i] = in[i].e;\n    }\n\n    const int end = QK_MXFP4 * 2 / blck_size_interleave;\n\n    if (blck_size_interleave == 4) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 4;\n            int src_offset = (i / 4) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint32_t));\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_mxfp4_to_mxfp4_4_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_MXFP4);\n    GGML_ASSERT(interleave_block == 4);\n\n    const block_mxfp4   * src = (const block_mxfp4   *)data;\n          block_mxfp4x4 * dst = (      block_mxfp4x4 *)t->data;\n\n    block_mxfp4 dst_tmp[4];\n\n    int nrow = ggml_nrows(t);\n    int nrows_interleaved = 4;\n    int nblocks = t->ne[0] / QK_MXFP4;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_mxfp4));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % 8 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_mxfp4x4(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic block_mxfp4x8 make_block_mxfp4x8(block_mxfp4 * in, unsigned int blck_size_interleave) {\n    block_mxfp4x8 out;\n\n    for (int i = 0; i < 8; i++) {\n        out.e[i] = in[i].e;\n    }\n\n    const int end = QK_MXFP4 * 4 / blck_size_interleave;\n\n    if (blck_size_interleave == 8) {\n        for (int i = 0; i < end; ++i) {\n            int src_id = i % 8;\n            int src_offset = (i / 8) * blck_size_interleave;\n            int dst_offset = i * blck_size_interleave;\n\n            memcpy(&out.qs[dst_offset], &in[src_id].qs[src_offset], sizeof(uint64_t));\n        }\n    } else {\n        GGML_ASSERT(false);\n    }\n\n    return out;\n}\n\nstatic int repack_mxfp4_to_mxfp4_8_bl(struct ggml_tensor * t, int interleave_block, const void * GGML_RESTRICT data, size_t data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_MXFP4);\n    GGML_ASSERT(interleave_block == 8);\n\n    const block_mxfp4   * src = (const block_mxfp4   *)data;\n          block_mxfp4x8 * dst = (      block_mxfp4x8 *)t->data;\n\n    block_mxfp4 dst_tmp[8];\n\n    int nrow = ggml_nrows(t);\n    int nrows_interleaved = 8;\n    int nblocks = t->ne[0] / QK_MXFP4;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_mxfp4));\n\n    if (t->ne[1] % nrows_interleaved != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_mxfp4x8(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nnamespace ggml::cpu::repack {\n// repack\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS>\nint repack(struct ggml_tensor *, const void *, size_t);\n\n// TODO: generalise.\ntemplate <> int repack<block_q4_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_0_to_q4_0_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_q4_0, 8, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_0_to_q4_0_4_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q4_0, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_0_to_q4_0_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q4_K, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_K_to_q4_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q4_K, 4, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_K_to_q4_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_q2_K, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q2_K_to_q2_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q5_K, 4, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q5_K_to_q5_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_q5_K, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q5_K_to_q5_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q6_K, 4, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q6_K_to_q6_K_8_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_q6_K, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q6_K_to_q6_K_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_iq4_nl, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_iq4_nl_to_iq4_nl_4_bl(t, 4, data, data_size);\n}\n\n// TODO: needs to be revisited\n//template <> int repack<block_iq4_nl, 8, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n//    return repack_iq4_nl_to_iq4_nl_4_bl(t, 8, data, data_size);\n//}\n\ntemplate <> int repack<block_iq4_nl, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_iq4_nl_to_iq4_nl_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_mxfp4, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_mxfp4_to_mxfp4_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_mxfp4, 8, 8>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_mxfp4_to_mxfp4_8_bl(t, 8, data, data_size);\n}\n\ntemplate <> int repack<block_q8_0, 4, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q8_0_to_q8_0_4_bl(t, 4, data, data_size);\n}\n\ntemplate <> int repack<block_q8_0, 8, 4>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q8_0_to_q8_0_4_bl(t, 8, data, data_size);\n}\n\n#if defined __riscv_zvfh\ntemplate <> int repack<block_q4_0, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_0_to_q4_0_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack<block_q4_K, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_K_to_q4_K_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack<block_iq4_nl, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_iq4_nl_to_iq4_nl_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack<block_q8_0, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q8_0_to_q8_0_16_bl(t, 1, data, data_size);\n}\n\ntemplate <> int repack<block_q2_K, 1, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q2_K_to_q2_K_16_bl(t, 1, data, data_size);\n}\n#endif\n\n// gemv\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE>\nvoid gemv(int, float *, size_t, const void *, const void *, int, int);\n\ntemplate <> void gemv<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q4_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q4_0, 8, 8, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <>\nvoid gemv<block_q2_K, 8, 8, GGML_TYPE_Q8_K>(int          n,\n                                            float *      s,\n                                            size_t       bs,\n                                            const void * vx,\n                                            const void * vy,\n                                            int          nr,\n                                            int          nc) {\n    ggml_gemv_q2_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q4_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q4_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q5_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q5_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q5_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q6_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q6_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_iq4_nl, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_iq4_nl, 8, 8, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_mxfp4, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_mxfp4_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_mxfp4, 8, 8, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_mxfp4_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q8_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q8_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q8_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void gemv<block_q4_0, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q4_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q4_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_iq4_nl, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_iq4_nl_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q8_0, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q8_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemv<block_q2_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemv_q2_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n#endif\n\n// gemm\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE>\nvoid gemm(int, float *, size_t, const void *, const void *, int, int);\n\ntemplate <> void gemm<block_q4_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q4_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <>\nvoid gemm<block_q4_0, 8, 8, GGML_TYPE_Q8_0>(int          n,\n                                            float *      s,\n                                            size_t       bs,\n                                            const void * vx,\n                                            const void * vy,\n                                            int          nr,\n                                            int          nc) {\n    ggml_gemm_q4_0_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q2_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q2_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q4_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q4_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q5_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q5_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q5_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q5_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q6_K, 4, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q6_K_8x4_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q6_K, 8, 8, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q6_K_8x8_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_iq4_nl, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_iq4_nl, 8, 8, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_iq4_nl_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_mxfp4, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_mxfp4_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_mxfp4, 8, 8, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_mxfp4_8x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q8_0, 4, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q8_0_4x4_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q8_0, 8, 4, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q8_0_4x8_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\n#if defined __riscv_zvfh\ntemplate <> void gemm<block_q4_0, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q4_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q4_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_iq4_nl, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_iq4_nl_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q8_0, 1, 16, GGML_TYPE_Q8_0>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q8_0_16x1_q8_0(n, s, bs, vx, vy, nr, nc);\n}\n\ntemplate <> void gemm<block_q2_K, 1, 16, GGML_TYPE_Q8_K>(int n, float * s, size_t bs, const void * vx, const void * vy, int nr, int nc) {\n    ggml_gemm_q2_K_16x1_q8_K(n, s, bs, vx, vy, nr, nc);\n}\n#endif\n\nclass tensor_traits_base : public ggml::cpu::tensor_traits {\n  public:\n    virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0;\n};\n\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PARAM_TYPE> class tensor_traits : public tensor_traits_base {\n\n    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n        // not realy a GGML_TYPE_Q8_0 but same size.\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                {\n                    size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1]));\n                    return true;\n                }\n            case GGML_OP_MUL_MAT_ID:\n                {\n                    size = ggml_row_size(PARAM_TYPE, ggml_nelements(op->src[1]));\n                    size = GGML_PAD(size, sizeof(int64_t)); // + padding for next block.\n\n                    const int64_t ne02 = op->src[0]->ne[2]; // n_as, n_expert\n                    const int64_t ne12 = op->src[1]->ne[2]; // n_tokens\n\n                    const size_t sizeof_mmid_row_mapping = sizeof(int64_t);\n\n                    size += sizeof_mmid_row_mapping*ne02*(ne12 + 1);\n\n                    return true;\n                }\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                forward_mul_mat(params, op);\n                return true;\n            case GGML_OP_MUL_MAT_ID:\n                forward_mul_mat_id(params, op);\n                return true;\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    void forward_mul_mat_one_chunk(ggml_compute_params * params,\n                                   ggml_tensor *         op,\n                                   int64_t               src0_start,\n                                   int64_t               src0_end,\n                                   int64_t               src1_start,\n                                   int64_t               src1_end) {\n        const ggml_tensor * src0 = op->src[0];\n        const ggml_tensor * src1 = op->src[1];\n        ggml_tensor *       dst  = op;\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        const size_t src1_col_stride = ggml_row_size(PARAM_TYPE, ne10);\n\n        GGML_ASSERT(ne03 == 1 && ne13 == 1);\n        GGML_ASSERT(ne12 % ne02 == 0);\n        const int64_t r2 = ne12 / ne02;\n\n        const int64_t i12 = src1_start / ne1;\n        const int64_t i11 = src1_start - i12 * ne1;\n\n        // Determine batch index\n        const int64_t i02 = i12 / r2;\n\n        const int64_t i1 = i11;\n        const int64_t i2 = i12;\n\n        const char * src0_ptr = (const char *) src0->data + i02 * nb02;\n        const char * src1_ptr = (const char *) params->wdata + (i11 + i12 * ne11) * src1_col_stride;\n        char *       dst_ptr  = ((char *) dst->data + (i1 * nb1 + i2 * nb2));\n\n        const int64_t nrows = src1_end - src1_start;\n        const int64_t ncols = src0_end - src0_start;\n\n        GGML_ASSERT(src1_ptr + src1_col_stride * nrows <= (const char *) params->wdata + params->wsize);\n\n        // If there are more than three rows in src1, use gemm; otherwise, use gemv.\n        if (nrows > 3) {\n            gemm<BLOC_TYPE, INTER_SIZE, NB_COLS, PARAM_TYPE>(ne00, (float *) (dst_ptr) + src0_start, nb1 / nb0,\n                                                             src0_ptr + src0_start * nb01, src1_ptr,\n                                                             nrows - (nrows % 4), ncols);\n        }\n        for (int iter = nrows - (nrows % 4); iter < nrows; iter++) {\n            gemv<BLOC_TYPE, INTER_SIZE, NB_COLS, PARAM_TYPE>(ne00, (float *) (dst_ptr + (iter * nb1)) + src0_start,\n                                                             ne01, src0_ptr + src0_start * nb01,\n                                                             src1_ptr + (src1_col_stride * iter), 1 /* nrows */, ncols);\n        }\n    }\n\n    void forward_mul_mat(ggml_compute_params * params, ggml_tensor * op) {\n        const ggml_tensor * src0 = op->src[0];\n        const ggml_tensor * src1 = op->src[1];\n        ggml_tensor *       dst  = op;\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        const int ith = params->ith;\n        const int nth = params->nth;\n\n        GGML_ASSERT(ne0 == ne01);\n        GGML_ASSERT(ne1 == ne11);\n        GGML_ASSERT(ne2 == ne12);\n        GGML_ASSERT(ne3 == ne13);\n\n        // dst cannot be transposed or permuted\n        GGML_ASSERT(nb0 == sizeof(float));\n        GGML_ASSERT(nb0 <= nb1);\n        GGML_ASSERT(nb1 <= nb2);\n        GGML_ASSERT(nb2 <= nb3);\n\n        // TODO: General batched mul mat for 4D tensors\n        // Currently only supports 3D tensors\n        GGML_ASSERT(ne03 == 1);\n        GGML_ASSERT(ne13 == 1);\n        GGML_ASSERT(ne3 == 1);\n\n        GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n        GGML_ASSERT(ggml_n_dims(op->src[0]) == 2);\n        // GGML_ASSERT(ggml_n_dims(op->src[1]) == 2);\n\n        char *       wdata = static_cast<char *>(params->wdata);\n        const size_t nbw1  = ggml_row_size(PARAM_TYPE, ne10);\n        const size_t nbw2  = nbw1 * ne11;\n\n        assert(params->wsize >= nbw2 * ne12);\n\n        const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float;\n\n        // INFO: Quantization is done in planes to avoid extra complexity in chunking.\n        // Flattening dimensions not multiple of INTER_SIZE would require extra handling depending on how\n        // the planes are broadcast.\n        for (int64_t i12 = 0; i12 < ne12; i12++) {\n            char * data_ptr  = (char *) src1->data + i12 * nb12;\n            char * wdata_ptr = wdata + i12 * nbw2;\n\n            for (int64_t i11 = ith * 4; i11 < ne11 - ne11 % 4; i11 += nth * 4) {\n                ggml_quantize_mat_t<INTER_SIZE, PARAM_TYPE>((float *) (data_ptr + i11 * nb11),\n                                                            (void *) (wdata_ptr + i11 * nbw1), 4, ne10);\n            }\n\n            const int64_t i11_processed = ne11 - ne11 % 4;\n            for (int64_t i11 = i11_processed + ith; i11 < ne11; i11 += nth) {\n                from_float((float *) (data_ptr + i11 * nb11), (void *) (wdata_ptr + i11 * nbw1), ne10);\n            }\n        }\n\n        // disable for NUMA\n        const bool disable_chunking = ggml_is_numa();\n\n        // 4x chunks per thread\n        const int64_t nr0 = ggml_nrows(op->src[0]);\n\n        int     nth_scaled  = nth * 4;\n        int64_t chunk_size0 = (nr0 + nth_scaled - 1) / nth_scaled;\n        int64_t nchunk0     = (nr0 + chunk_size0 - 1) / chunk_size0;\n\n        // src1 is chunked only by full planes.\n        // When we flatten we need to address dimensions not multiple of the q8 INTER_SIZE\n        // to route them thorugh GEMV.\n        // nchunk1 = ne12 also avoids messing the chunking for models with no 3d tensors\n        // to avoid affecting their performance\n        int64_t nchunk1 = ne12;\n\n        // Ensure minimum chunk size to avoid alignment issues with high thread counts\n        // Minimum chunk size should be at least NB_COLS to prevent overlapping chunks after alignment\n        const int64_t min_chunk_size = NB_COLS;\n        if (nchunk0 > 0 && (nr0 / nchunk0) < min_chunk_size && nr0 >= min_chunk_size) {\n            nchunk0 = (nr0 + min_chunk_size - 1) / min_chunk_size;\n        }\n\n        int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n        // Only increase nchunk0 to nth if it won't make chunks too small\n        if (nth == 1 || ((nchunk0 < nth || disable_chunking) && (nr0 + nth - 1) / nth >= min_chunk_size)) {\n            nchunk0 = nth;\n            dr0 = (nr0 + nchunk0 - 1) / nchunk0;\n        }\n\n        // Ensure nchunk doesn't exceed the number of rows divided by minimum chunk size\n        // This prevents creating too many tiny chunks that could overlap after alignment\n        const int64_t max_nchunk = (nr0 + min_chunk_size - 1) / min_chunk_size;\n        nchunk0                  = MIN(nchunk0, max_nchunk);\n\n        if (ith == 0) {\n            // Every thread starts at ith, so the first unprocessed chunk is nth.  This save a bit of coordination right at the start.\n            ggml_threadpool_chunk_set(params->threadpool, nth);\n        }\n\n        ggml_barrier(params->threadpool);\n\n        // The first chunk comes from our thread_id, the rest will get auto-assigned.\n        int current_chunk = ith;\n\n        while (current_chunk < nchunk0 * nchunk1) {\n            const int64_t ith0 = current_chunk % nchunk0;\n            const int64_t ith1 = current_chunk / nchunk0;\n\n            int64_t src0_start = dr0 * ith0;\n            int64_t src0_end   = MIN(src0_start + dr0, nr0);\n\n            // full-plane range for src1\n            int64_t src1_start = ith1 * ne11;\n            int64_t src1_end = (ith1 + 1) * ne11;\n\n            // Align boundaries to NB_COLS - round up to ensure all data is included\n            // The chunk size limiting above ensures chunks are large enough to prevent overlaps\n            src0_start = (src0_start % NB_COLS) ? src0_start + NB_COLS - (src0_start % NB_COLS) : src0_start;\n            src0_end   = (src0_end % NB_COLS) ? src0_end + NB_COLS - (src0_end % NB_COLS) : src0_end;\n            src0_end   = MIN(src0_end, ne01);\n\n            // Make sure current plane is the last one before exiting\n            if (src0_start >= src0_end) {\n                current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n                continue;\n            }\n\n            forward_mul_mat_one_chunk(params, dst, src0_start, src0_end, src1_start, src1_end);\n\n            current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);\n        }\n    }\n\n    void forward_mul_mat_id(ggml_compute_params * params, ggml_tensor * op) {\n        const ggml_tensor * src0 = op->src[0];\n        const ggml_tensor * src1 = op->src[1];\n        const ggml_tensor * ids  = op->src[2];\n        ggml_tensor *       dst  = op;\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        const int ith = params->ith;\n        const int nth = params->nth;\n\n        const ggml_from_float_t from_float = ggml_get_type_traits_cpu(PARAM_TYPE)->from_float;\n\n        // we don't support permuted src0 or src1\n        GGML_ASSERT(nb00 == ggml_type_size(src0->type));\n        GGML_ASSERT(nb10 == ggml_type_size(src1->type));\n\n        // dst cannot be transposed or permuted\n        GGML_ASSERT(nb0 == sizeof(float));\n        GGML_ASSERT(nb0 <= nb1);\n        GGML_ASSERT(nb1 <= nb2);\n        GGML_ASSERT(nb2 <= nb3);\n\n        GGML_ASSERT(ne03 == 1);\n        GGML_ASSERT(ne13 == 1);\n        GGML_ASSERT(ne3  == 1);\n\n        GGML_ASSERT(src1->type == GGML_TYPE_F32);\n\n        // row groups\n        const int n_ids = ids->ne[0]; // n_expert_used\n        const int n_as  = ne02;       // n_expert\n\n        const size_t nbw1 = ggml_row_size(PARAM_TYPE, ne10);\n        const size_t nbw2 = nbw1*ne11;\n        const size_t nbw3 = nbw2*ne12;\n\n        struct mmid_row_mapping {\n            int32_t i1;\n            int32_t i2;\n        };\n\n        GGML_ASSERT(params->wsize >=\n                (GGML_PAD(nbw3, sizeof(int64_t)) +\n                 n_as*(ne12 + 1)*sizeof(mmid_row_mapping))\n                );\n\n        auto * wdata          = (char *)params->wdata;\n        auto * wdata_src1_end = (char *)wdata + GGML_PAD(nbw3, sizeof(int64_t));\n\n        // total of [n_as][ne12 + 1] elements of type mmid_row_mapping (2*int32_t = int64_t)\n        auto * matrix_row_counts = (int64_t *) (wdata_src1_end);                                        // [n_as]\n        struct mmid_row_mapping * matrix_rows = (struct mmid_row_mapping *) (matrix_row_counts + n_as); // [n_as][ne12]\n\n        // src1: float32 => param type\n        for (int64_t i12 = 0; i12 < ne12; ++i12) {\n            for (int64_t i11 = ith; i11 < ne11; i11 += nth) {\n                from_float((float *)((char *) src1->data + i12 * nb12 + i11 * nb11),\n                           (void *)               (wdata + i12 * nbw2 + i11 * nbw1),\n                           ne10);\n            }\n        }\n\n#define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id) * ne12 + (i1)]\n\n        if (ith == 0) {\n            // initialize matrix_row_counts\n            memset(matrix_row_counts, 0, n_as * sizeof(int64_t));\n\n            // group rows by src0 matrix\n            for (int32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {\n                for (int32_t id = 0; id < n_ids; ++id) {\n                    const int32_t i02 =\n                        *(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]);\n\n                    GGML_ASSERT(i02 >= 0 && i02 < n_as);\n\n                    MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 };\n                    matrix_row_counts[i02] += 1;\n                }\n            }\n        }\n\n        ggml_barrier(params->threadpool);\n\n        // compute each matrix multiplication in sequence\n        for (int cur_a = 0; cur_a < n_as; ++cur_a) {\n            const int64_t cne1 = matrix_row_counts[cur_a];\n\n            if (cne1 == 0) {\n                continue;\n            }\n\n            const auto * src0_cur = (const char *) src0->data + cur_a*nb02;\n\n            //const int64_t nr0 = ne01; // src0 rows\n            const int64_t nr1 = cne1; // src1 rows\n\n            int64_t src0_cur_start = (ith * ne01) / nth;\n            int64_t src0_cur_end   = ((ith + 1) * ne01) / nth;\n\n            // Align boundaries to NB_COLS - round up to ensure all data is included\n            src0_cur_start = (src0_cur_start % NB_COLS) ? src0_cur_start + NB_COLS - (src0_cur_start % NB_COLS) : src0_cur_start;\n            src0_cur_end   = (src0_cur_end   % NB_COLS) ? src0_cur_end   + NB_COLS - (src0_cur_end   % NB_COLS) : src0_cur_end;\n            if (src0_cur_end > ne01) {\n                src0_cur_end = ne01;\n            }\n\n            if (src0_cur_start >= src0_cur_end) {\n                return;\n            }\n\n            for (int ir1 = 0; ir1 < nr1; ir1++) {\n                struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, ir1);\n\n                const int id = row_mapping.i1;  // selected expert index\n\n                const int64_t i11 = id % ne11;\n                const int64_t i12 = row_mapping.i2;  // row index in src1\n\n                const int64_t i1 = id;               // selected expert index\n                const int64_t i2 = i12;              // row\n\n                const auto * src1_col = (const char *) wdata + (i11 * nbw1 + i12 * nbw2);\n\n                gemv<BLOC_TYPE, INTER_SIZE, NB_COLS, PARAM_TYPE>(\n                    ne00, (float *) ((char *) dst->data + (i1 * nb1 + i2 * nb2)) + src0_cur_start, ne01,\n                    src0_cur + src0_cur_start * nb01, src1_col, 1, src0_cur_end - src0_cur_start);\n            }\n        }\n#undef MMID_MATRIX_ROW\n    }\n\n    int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n        GGML_LOG_DEBUG(\"%s: repack tensor %s with %s_%dx%d\\n\", __func__, t->name, ggml_type_name(t->type),\n                       (int) NB_COLS, (int) INTER_SIZE);\n        return ggml::cpu::repack::repack<BLOC_TYPE, INTER_SIZE, NB_COLS>(t, data, data_size);\n    }\n};\n\n}  // namespace ggml::cpu::repack\n\nstatic const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(const struct ggml_tensor * cur) {\n    // instance for Q4\n    static const ggml::cpu::repack::tensor_traits<block_q4_0, 4, 4, GGML_TYPE_Q8_0> q4_0_4x4_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 4, GGML_TYPE_Q8_0> q4_0_4x8_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q4_0, 8, 8, GGML_TYPE_Q8_0> q4_0_8x8_q8_0;\n\n    // instance for Q4_K\n    static const ggml::cpu::repack::tensor_traits<block_q4_K, 4, 8, GGML_TYPE_Q8_K> q4_K_8x4_q8_K;\n    static const ggml::cpu::repack::tensor_traits<block_q4_K, 8, 8, GGML_TYPE_Q8_K> q4_K_8x8_q8_K;\n\n    // instance for Q5_K\n    static const ggml::cpu::repack::tensor_traits<block_q5_K, 4, 8, GGML_TYPE_Q8_K> q5_K_8x4_q8_K;\n    static const ggml::cpu::repack::tensor_traits<block_q5_K, 8, 8, GGML_TYPE_Q8_K> q5_K_8x8_q8_K;\n\n    // instance for Q6_K\n    static const ggml::cpu::repack::tensor_traits<block_q6_K, 4, 8, GGML_TYPE_Q8_K> q6_K_8x4_q8_K;\n    static const ggml::cpu::repack::tensor_traits<block_q6_K, 8, 8, GGML_TYPE_Q8_K> q6_K_8x8_q8_K;\n\n    // instance for Q2\n    static const ggml::cpu::repack::tensor_traits<block_q2_K, 8, 8, GGML_TYPE_Q8_K> q2_K_8x8_q8_K;\n\n    // instance for IQ4\n    static const ggml::cpu::repack::tensor_traits<block_iq4_nl, 4, 4, GGML_TYPE_Q8_0> iq4_nl_4x4_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_iq4_nl, 8, 8, GGML_TYPE_Q8_0> iq4_nl_8x8_q8_0;\n\n    // instance for MXFP4\n    static const ggml::cpu::repack::tensor_traits<block_mxfp4, 4, 4, GGML_TYPE_Q8_0> mxfp4_4x4_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_mxfp4, 8, 8, GGML_TYPE_Q8_0> mxfp4_8x8_q8_0;\n\n    // instance for Q8_0\n    static const ggml::cpu::repack::tensor_traits<block_q8_0, 4, 4, GGML_TYPE_Q8_0> q8_0_4x4_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q8_0, 8, 4, GGML_TYPE_Q8_0> q8_0_4x8_q8_0;\n\n    // instances for RISC-V\n    //\n    // These implement outer-product style matrix multiplication kernels with\n    // an interleave of 1.\n#if defined __riscv_zvfh\n    static const ggml::cpu::repack::tensor_traits<block_q4_0, 1, 16, GGML_TYPE_Q8_0> q4_0_16x1_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q4_K, 1, 16, GGML_TYPE_Q8_K> q4_K_16x1_q8_K;\n    static const ggml::cpu::repack::tensor_traits<block_iq4_nl, 1, 16, GGML_TYPE_Q8_0> iq4_nl_16x1_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q8_0, 1, 16, GGML_TYPE_Q8_0> q8_0_16x1_q8_0;\n    static const ggml::cpu::repack::tensor_traits<block_q2_K, 1, 16, GGML_TYPE_Q8_K> q2_K_16x1_q8_K;\n#endif\n\n    if (cur->type == GGML_TYPE_Q4_0) {\n        if (ggml_cpu_has_avx2() || (ggml_cpu_has_sve() && ggml_cpu_has_matmul_int8() && ggml_cpu_get_sve_cnt() == QK8_0)) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q4_0_8x8_q8_0;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &q4_0_4x8_q8_0;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &q4_0_4x4_q8_0;\n            }\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            #if defined __riscv_zvfh\n            switch (__riscv_vlenb() * 8) {\n                case 128:  { break; } // TODO\n                case 256:  { if (cur->ne[1] % 16 == 0) { return &q4_0_16x1_q8_0; } break; }\n                case 512:  { break; } // TODO\n                case 1024: { break; } // TODO\n                default:   { return nullptr; }\n            }\n            #endif\n        }\n    } else if (cur->type == GGML_TYPE_Q4_K) {\n        if (ggml_cpu_has_avx2()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q4_K_8x8_q8_K;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q4_K_8x8_q8_K;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q4_K_8x4_q8_K;\n            }\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            #if defined __riscv_zvfh\n            switch (__riscv_vlenb() * 8) {\n                case 128:  { break; } // TODO\n                case 256:  { if (cur->ne[1] % 16 == 0) { return &q4_K_16x1_q8_K; } break; }\n                case 512:  { break; } // TODO\n                case 1024: { break; } // TODO\n                default:   { return nullptr; }\n            }\n            #endif\n        }\n    } else if (cur->type == GGML_TYPE_Q2_K) {\n        if (ggml_cpu_has_avx512()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q2_K_8x8_q8_K;\n            }\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            #if defined __riscv_zvfh\n            switch (__riscv_vlenb() * 8) {\n                case 128:  { break; } // TODO\n                case 256:  { if (cur->ne[1] % 16 == 0) { return &q2_K_16x1_q8_K; } break; }\n                case 512:  { break; } // TODO\n                case 1024: { break; } // TODO\n                default:   { return nullptr; }\n            }\n            #endif\n        }\n    } else if (cur->type == GGML_TYPE_Q5_K) {\n        if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q5_K_8x8_q8_K;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q5_K_8x4_q8_K;\n            }\n        }\n    } else if (cur->type == GGML_TYPE_Q6_K) {\n        if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q6_K_8x8_q8_K;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &q6_K_8x4_q8_K;\n            }\n        }\n    } else if (cur->type == GGML_TYPE_IQ4_NL) {\n        if (ggml_cpu_has_avx2()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &iq4_nl_8x8_q8_0;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &iq4_nl_4x4_q8_0;\n            }\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            #if defined __riscv_zvfh\n            switch (__riscv_vlenb() * 8) {\n                case 128:  { break; } // TODO\n                case 256:  { if (cur->ne[1] % 16 == 0) { return &iq4_nl_16x1_q8_0; } break; }\n                case 512:  { break; } // TODO\n                case 1024: { break; } // TODO\n                default:   { return nullptr; }\n            }\n            #endif\n        }\n    } else if (cur->type == GGML_TYPE_MXFP4) {\n        if (ggml_cpu_has_avx2()) {\n            if (cur->ne[1] % 8 == 0) {\n                return &mxfp4_8x8_q8_0;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &mxfp4_4x4_q8_0;\n            }\n        }\n    } else if (cur->type == GGML_TYPE_Q8_0) {\n        if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &q8_0_4x8_q8_0;\n            }\n        }\n        if (ggml_cpu_has_neon() && ggml_cpu_has_dotprod()) {\n            if (cur->ne[1] % 4 == 0) {\n                return &q8_0_4x4_q8_0;\n            }\n        }\n        if (ggml_cpu_has_riscv_v()) {\n            #if defined __riscv_zvfh\n            switch (__riscv_vlenb() * 8) {\n                case 128:  { break; } // TODO\n                case 256:  { if (cur->ne[1] % 16 == 0) { return &q8_0_16x1_q8_0; } break; }\n                case 512:  { break; } // TODO\n                case 1024: { break; } // TODO\n                default:   { return nullptr; }\n            }\n            #endif\n        }\n    }\n\n    return nullptr;\n}\n\nstatic enum ggml_status ggml_backend_cpu_repack_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {\n    tensor->extra = (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_repack_get_optimal_repack_type(tensor));\n\n    GGML_UNUSED(buffer);\n    return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,\n                                                       const void * data, size_t offset, size_t size) {\n    GGML_ASSERT(offset == 0);\n    GGML_ASSERT(size == ggml_nbytes(tensor));\n\n    auto tensor_traits = (ggml::cpu::repack::tensor_traits_base *) tensor->extra;\n    auto OK            = tensor_traits->repack(tensor, data, size);\n\n    GGML_ASSERT(OK == 0);\n    GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_repack_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"CPU_REPACK\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {\n    ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n    if (buffer == nullptr) {\n        return nullptr;\n    }\n\n    buffer->buft              = buft;\n    buffer->iface.init_tensor = ggml_backend_cpu_repack_buffer_init_tensor;\n    buffer->iface.set_tensor  = ggml_backend_cpu_repack_buffer_set_tensor;\n    buffer->iface.get_tensor  = nullptr;\n    buffer->iface.cpy_tensor  = nullptr;\n    return buffer;\n}\n\nstatic size_t ggml_backend_cpu_repack_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    return TENSOR_ALIGNMENT;\n\n    GGML_UNUSED(buft);\n}\n\nnamespace ggml::cpu::repack {\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n    bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n        if (    op->op == GGML_OP_MUL_MAT &&\n                op->src[0]->buffer &&\n                (ggml_n_dims(op->src[0]) == 2) &&\n                op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type() &&\n                ggml_repack_get_optimal_repack_type(op->src[0])\n                ) {\n            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n                return false;\n            }\n            if (op->src[1]->type == GGML_TYPE_F32) {\n                return true;\n            }\n            //if (op->src[1]->type == GGML_TYPE_Q8_0) {\n            //    return true;\n            //}\n            // may be possible if Q8_0 packed...\n        } else if (op->op == GGML_OP_MUL_MAT_ID\n                && op->src[0]->buffer\n                && (ggml_n_dims(op->src[0]) == 3)\n                && op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type()\n                && ggml_repack_get_optimal_repack_type(op->src[0])\n                ) {\n            if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n                return false;\n            }\n            if (op->src[1]->type == GGML_TYPE_F32) {\n                return true;\n            }\n            //if (op->src[1]->type == GGML_TYPE_Q8_0) {\n            //    return true;\n            //}\n        }\n        return false;\n    }\n\n    ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n        if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_MUL_MAT_ID) {\n            if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_repack_buffer_type()) {\n                return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n            }\n        }\n        return nullptr;\n    }\n};\n}  // namespace ggml::cpu::repack\n\nggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void) {\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_repack = {\n        /* .iface    = */ {\n                           /* .get_name         = */ ggml_backend_cpu_repack_buffer_type_get_name,\n                           /* .alloc_buffer     = */ ggml_backend_cpu_repack_buffer_type_alloc_buffer,\n                           /* .get_alignment    = */ ggml_backend_cpu_repack_buffer_type_get_alignment,\n                           /* .get_max_size     = */ nullptr,  // defaults to SIZE_MAX\n                           /* .get_alloc_size   = */ nullptr,  // defaults to ggml_nbytes\n                           /* .is_host          = */ nullptr,\n                           },\n        /* .device  = */ ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n        /* .context = */ new ggml::cpu::repack::extra_buffer_type(),\n    };\n\n    return &ggml_backend_cpu_buffer_type_repack;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/repack.h",
    "content": "#pragma once\n\n#define GGML_COMMON_DECL_CPP\n#include \"ggml-common.h\"\n\n#include \"traits.h\"\n#include \"ggml.h\"\n\n// GGML internal header\n\nggml_backend_buffer_type_t ggml_backend_cpu_repack_buffer_type(void);\n\ntemplate <int K> constexpr int QK_0() {\n    if constexpr (K == 4) {\n        return QK4_0;\n    }\n    if constexpr (K == 8) {\n        return QK8_0;\n    }\n    return -1;\n}\n\ntemplate <int K, int N> struct block {\n    ggml_half d[N];                         // deltas for N qK_0 blocks\n    int8_t    qs[(QK_0<K>() * N * K) / 8];  // quants for N qK_0 blocks\n};\n\n// control size\nstatic_assert(sizeof(block<4, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 2, \"wrong block<4,4> size/padding\");\nstatic_assert(sizeof(block<4, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 4, \"wrong block<4,8> size/padding\");\nstatic_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 8, \"wrong block<4,16> size/padding\");\nstatic_assert(sizeof(block<8, 4>) == 4 * sizeof(ggml_half) + QK8_0 * 4, \"wrong block<8,4> size/padding\");\nstatic_assert(sizeof(block<8, 8>) == 8 * sizeof(ggml_half) + QK8_0 * 8, \"wrong block<8,8> size/padding\");\nstatic_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK8_0 * 16, \"wrong block<8,16> size/padding\");\n\nusing block_q4_0x4 = block<4, 4>;\nusing block_q4_0x8 = block<4, 8>;\nusing block_q4_0x16 = block<4, 16>;\nusing block_q8_0x4 = block<8, 4>;\nusing block_q8_0x8 = block<8, 8>;\nusing block_q8_0x16 = block<8, 16>;\n\nstruct block_q4_Kx8 {\n    ggml_half d[8];      // super-block scale for quantized scales\n    ggml_half dmin[8];   // super-block scale for quantized mins\n    uint8_t scales[96];  // scales and mins, quantized with 6 bits\n    uint8_t qs[1024];    // 4--bit quants\n};\n\nstatic_assert(sizeof(block_q4_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 4, \"wrong q4_K block size/padding\");\nstruct block_q4_Kx16 {\n    ggml_half d[16];      // super-block scale for quantized scales\n    ggml_half dmin[16];   // super-block scale for quantized mins\n    uint8_t scales[192];  // scales and mins, quantized with 6 bits\n    uint8_t qs[2048];    // 4--bit quants\n};\n\nstatic_assert(sizeof(block_q4_Kx16) == sizeof(ggml_half) * 32 + K_SCALE_SIZE * 16 + QK_K * 8, \"wrong q4_K block size/padding\");\nstruct block_q2_Kx8 {\n    ggml_half d[8];      // super-block scale for quantized scales\n    ggml_half dmin[8];   // super-block scale for quantized mins\n    uint8_t scales[128];  // scales and mins, quantized with 4 bits\n    uint8_t qs[512];    // 2--bit quants\n};\n\nstatic_assert(sizeof(block_q2_Kx8) == sizeof(ggml_half) * 16 + QK_K/2 + QK_K * 2, \"wrong q2_K block size/padding\");\nstruct block_q2_Kx16 {\n    ggml_half d[16];       // Super-block scale for quantized scales\n    ggml_half dmin[16];    // Super-block scale for quantized mins\n    uint8_t   scales[256]; // Sub-block scales (16 cols * 16 sub-blocks)\n    uint8_t   qs[1024];    // Data (16 cols * 64 bytes per block)\n};\nstatic_assert(sizeof(block_q2_Kx16) == sizeof(ggml_half) * 32 + QK_K + QK_K * 4, \"wrong q2_K block size/padding\");\n\nstruct block_q5_Kx8 {\n    ggml_half d[8];              // super-block scale for quantized scales\n    ggml_half dmin[8];           // super-block scale for quantized mins\n    uint8_t   scales[96];        // scales and mins, quantized with 6 bits\n    uint8_t   qh[QK_K * 8 / 8];  // high bits of 5-bit quants\n    uint8_t   qs[QK_K * 8 / 2];  // low bits of 5-bit quants (in groups of 4)\n};\n\nstatic_assert(sizeof(block_q5_Kx8) == sizeof(ggml_half) * 16 + K_SCALE_SIZE * 8 + QK_K * 5,\n              \"wrong q5_K block size/padding\");\n\nstruct block_q6_Kx8 {\n    ggml_half d[8];\n    int8_t    scales[QK_K / 16 * 8];\n    uint8_t   ql[QK_K / 2 * 8];  // low bits of 6-bit quants (groups of 2)\n    uint8_t   qh[QK_K / 4 * 8];  // high bits of 6-bit quants (groups of 4)\n};\n\nstatic_assert(sizeof(block_q6_Kx8) == sizeof(ggml_half) * 8 + QK_K / 16 * 8 + 3 * QK_K / 4 * 8,\n              \"wrong q6_K block size/padding\");\n\nstruct block_q8_Kx4 {\n    float d[4];              // delta\n    int8_t qs[QK_K * 4];     // quants\n    int16_t bsums[QK_K / 4]; // sum of quants in groups of 16\n};\n\nstatic_assert(sizeof(block_q8_Kx4) == sizeof(float) * 4 + QK_K * 4 + (QK_K / 4) * sizeof(int16_t), \"wrong q8_K block size/padding\");\n\nstruct block_iq4_nlx4 {\n    ggml_half d[4];            // deltas for 4 iq4_nl blocks\n    uint8_t   qs[QK4_NL * 2];  // nibbles / quants for 4 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(ggml_half) + QK4_NL * 2, \"wrong iq4_nlx4 block size/padding\");\n\nstruct block_iq4_nlx8 {\n    ggml_half d[8];            // deltas for 8 iq4_nl blocks\n    uint8_t   qs[QK4_NL * 4];  // nibbles / quants for 8 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx8) == 8 * sizeof(ggml_half) + QK4_NL * 4, \"wrong iq4_nlx8 block size/padding\");\n\nstruct block_iq4_nlx16 {\n    ggml_half d[16];            // deltas for 16 iq4_nl blocks\n    uint8_t   qs[QK4_NL * 8];  // nibbles / quants for 16 iq4_nl blocks\n};\n\nstatic_assert(sizeof(block_iq4_nlx16) == 16 * sizeof(ggml_half) + QK4_NL * 8, \"wrong iq4_nlx16 block size/padding\");\nstruct block_mxfp4x4 {\n    uint8_t e[4];\n    uint8_t qs[QK_MXFP4 * 2];\n};\nstatic_assert(sizeof(block_mxfp4x4) == 4 + QK_MXFP4 * 2, \"wrong mxfp4x4 block size/padding\");\n\nstruct block_mxfp4x8 {\n    uint8_t e[8];\n    uint8_t qs[QK_MXFP4 * 4];\n};\nstatic_assert(sizeof(block_mxfp4x8) == 8 + QK_MXFP4 * 4, \"wrong mxfp4x8 block size/padding\");\n\n#if defined(__cplusplus)\nextern \"C\" {\n#endif\n\nvoid ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x8_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x8_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_16x1_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_16x1_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#endif\n\n// Native implementations\nvoid ggml_quantize_mat_q8_0_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_0_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x4_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x8_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q5_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x4_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q6_K_8x8_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_mxfp4_8x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_4x8_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#if defined __riscv_zvfh\nvoid ggml_quantize_mat_q8_0_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k);\nvoid ggml_gemv_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemv_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q4_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q8_0_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_q2_K_16x1_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\nvoid ggml_gemm_iq4_nl_16x1_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc);\n#endif\n\n#if defined(__cplusplus)\n} // extern \"C\"\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/simd-gemm.h",
    "content": "#pragma once\n\n// Computes C[M x N] += A[M x K] * B[K x N]\n\n#include \"simd-mappings.h\"\n\n// TODO: add support for sizeless vector types\n#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic)\n\n// TODO: untested on avx512\n// These are in units of GGML_F32_EPR\n#if defined(__AVX512F__) || defined (__ARM_NEON__)\n    static constexpr int GEMM_RM = 4;\n    static constexpr int GEMM_RN = 4; // 16+4+1 = 25/32\n#elif defined(__AVX2__) || defined(__AVX__)\n    static constexpr int GEMM_RM = 6;\n    static constexpr int GEMM_RN = 2; // 12+2+1 = 15/16\n#else\n    static constexpr int GEMM_RM = 2;\n    static constexpr int GEMM_RN = 2;\n#endif\n\ntemplate <int RM, int RN>\nstatic inline void simd_gemm_ukernel(\n    float       * GGML_RESTRICT C,\n    const float * GGML_RESTRICT A,\n    const float * GGML_RESTRICT B,\n    int K, int N)\n{\n    static constexpr int KN = GGML_F32_EPR;\n\n    GGML_F32_VEC acc[RM][RN];\n    for (int64_t i = 0; i < RM; i++) {\n        for (int r = 0; r < RN; r++) {\n            acc[i][r] = GGML_F32_VEC_LOAD(C + i * N + r * KN);\n        }\n    }\n\n    for (int64_t kk = 0; kk < K; kk++) {\n        GGML_F32_VEC Bv[RN];\n        for (int r = 0; r < RN; r++) {\n            Bv[r] = GGML_F32_VEC_LOAD(B + kk * N + r * KN);\n        }\n        for (int64_t i = 0; i < RM; i++) {\n            GGML_F32_VEC p = GGML_F32_VEC_SET1(A[i * K + kk]);\n            for (int r = 0; r < RN; r++) {\n                acc[i][r] = GGML_F32_VEC_FMA(acc[i][r], Bv[r], p);\n            }\n        }\n    }\n\n    for (int64_t i = 0; i < RM; i++) {\n        for (int r = 0; r < RN; r++) {\n            GGML_F32_VEC_STORE(C + i * N + r * KN, acc[i][r]);\n        }\n    }\n}\n\n// C[M x N] += A[M x K] * B[K x N]\nstatic void simd_gemm(\n    float       * GGML_RESTRICT C,\n    const float * GGML_RESTRICT A,\n    const float * GGML_RESTRICT B,\n    int M, int K, int N)\n{\n    static constexpr int KN = GGML_F32_EPR;\n\n    int64_t ii = 0;\n    for (; ii + GEMM_RM <= M; ii += GEMM_RM) {\n        int64_t jj = 0;\n        for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {\n            simd_gemm_ukernel<GEMM_RM, GEMM_RN>(C + jj, A, B + jj, K, N);\n        }\n        for (; jj + KN <= N; jj += KN) {\n            simd_gemm_ukernel<GEMM_RM, 1>(C + jj, A, B + jj, K, N);\n        }\n        for (; jj < N; jj++) {\n            for (int64_t i = 0; i < GEMM_RM; i++) {\n                float a = C[i * N + jj];\n                for (int64_t kk = 0; kk < K; kk++) {\n                    a += A[i + kk] * B[kk * N + jj];\n                }\n                C[i * N + jj] = a;\n            }\n        }\n\n        A += GEMM_RM * K;\n        C += GEMM_RM * N;\n    }\n\n    // Tail rows: one at a time\n    for (; ii < M; ii++) {\n        int64_t jj = 0;\n        for (; jj + GEMM_RN * KN <= N; jj += GEMM_RN * KN) {\n            simd_gemm_ukernel<1, GEMM_RN>(C + jj, A, B + jj, K, N);\n        }\n        for (; jj + KN <= N; jj += KN) {\n            simd_gemm_ukernel<1, 1>(C + jj, A, B + jj, K, N);\n        }\n        for (; jj < N; jj++) {\n            float a = C[jj];\n            for (int64_t kk = 0; kk < K; kk++) {\n                a += A[kk] * B[kk * N + jj];\n            }\n            C[jj] = a;\n        }\n\n        A += K;\n        C += N;\n    }\n}\n\n#if defined(__GNUC__) && !defined(__clang__)\n#pragma GCC diagnostic pop\n#endif\n\n#else // scalar path\n\nstatic void simd_gemm(\n    float       * GGML_RESTRICT C,\n    const float * GGML_RESTRICT A,\n    const float * GGML_RESTRICT B,\n    int M, int K, int N)\n{\n    for (int64_t i = 0; i < M; i++) {\n        for (int64_t j = 0; j < N; j++) {\n            float sum = C[i * N + j];\n            for (int64_t kk = 0; kk < K; kk++) {\n                sum += A[i * K + kk] * B[kk * N + j];\n            }\n            C[i * N + j] = sum;\n        }\n    }\n}\n\n#endif // GGML_SIMD\n"
  },
  {
    "path": "ggml/src/ggml-cpu/simd-mappings.h",
    "content": "#pragma once\n\n#include \"ggml-cpu-impl.h\"\n\n#ifdef __ARM_FEATURE_SVE\n#include <arm_sve.h>\n#endif // __ARM_FEATURE_SVE\n\n#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)\n// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:\n//\n//   $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/\n//\n#include <arm_neon.h>\n#endif\n\n#if defined(__riscv_v_intrinsic)\n#include <riscv_vector.h>\n#endif\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// simd mappings\n//\n\n// FP16 to FP32 conversion\n\n// 16-bit float\n// on Arm, we use __fp16\n// on x86, we use uint16_t\n//\n// for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616\n// for     MUSA compilers        , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843\n//\n#if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__)\n    #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x)\n    #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x)\n\n    #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n\n    static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) {\n        __fp16 tmp;\n        memcpy(&tmp, &h, sizeof(ggml_fp16_t));\n        return (float)tmp;\n    }\n\n    static inline ggml_fp16_t neon_compute_fp32_to_fp16(float f) {\n        ggml_fp16_t res;\n        __fp16 tmp = f;\n        memcpy(&res, &tmp, sizeof(ggml_fp16_t));\n        return res;\n    }\n#elif defined(__F16C__)\n    #ifdef _MSC_VER\n        #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))\n        #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)\n    #else\n        #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)\n        #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)\n    #endif\n#elif defined(__POWER9_VECTOR__)\n    #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) power_compute_fp16_to_fp32(x)\n    #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) power_compute_fp32_to_fp16(x)\n    /* the inline asm below is about 12% faster than the lookup method */\n    #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n    #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)\n\n    static inline float power_compute_fp16_to_fp32(ggml_fp16_t h) {\n        float f;\n        double d;\n        __asm__(\n            \"mtfprd %0,%2\\n\"\n            \"xscvhpdp %0,%0\\n\"\n            \"frsp %1,%0\\n\" :\n            /* temp */ \"=d\"(d),\n            /* out */  \"=f\"(f):\n            /* in */   \"r\"(h));\n        return f;\n    }\n\n    static inline ggml_fp16_t power_compute_fp32_to_fp16(float f) {\n        double d;\n        ggml_fp16_t r;\n        __asm__( /* xscvdphp can work on double or single precision */\n            \"xscvdphp %0,%2\\n\"\n            \"mffprd %1,%0\\n\" :\n            /* temp */ \"=d\"(d),\n            /* out */  \"=r\"(r):\n            /* in */   \"f\"(f));\n        return r;\n    }\n#elif defined(__riscv) && defined(__riscv_zfhmin)\n    static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) {\n        _Float16 hf;\n        memcpy(&hf, &h, sizeof(ggml_fp16_t));\n        return hf;\n    }\n\n    static inline ggml_fp16_t riscv_compute_fp32_to_fp16(float f) {\n        ggml_fp16_t res;\n        _Float16 hf = (_Float16)f;\n        memcpy(&res, &hf, sizeof(ggml_fp16_t));\n        return res;\n    }\n\n    #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x)\n    #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x)\n    #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)\n    #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)\n#endif\n\n// precomputed f32 table for f16 (256 KB)\n// defined in ggml-cpu.c, initialized in ggml_cpu_init()\nextern float ggml_table_f32_f16[1 << 16];\n\n// precomputed f32 table for e8m0 half (1 KB)\n// defined in ggml-cpu.c, initialized in ggml_cpu_init()\nextern float ggml_table_f32_e8m0_half[1 << 8];\n\n// Use lookup table for E8M0 on x86 (faster than bit manipulation)\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n#define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)]\n#else\n#define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x)\n#endif\n\n// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,\n// so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON.\n// This is also true for POWER9.\n#if !defined(GGML_CPU_FP16_TO_FP32)\ninline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {\n    uint16_t s;\n    memcpy(&s, &f, sizeof(uint16_t));\n    return ggml_table_f32_f16[s];\n}\n\n#define GGML_CPU_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)\n#endif\n\n#if !defined(GGML_CPU_FP32_TO_FP16)\n#define GGML_CPU_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)\n#endif\n\n\n// we define a common set of C macros which map to specific intrinsics based on the current architecture\n// we then implement the fundamental computation operations below using only these macros\n// adding support for new architectures requires to define the corresponding SIMD macros\n//\n// GGML_F32_STEP / GGML_F16_STEP\n//   number of elements to process in a single step\n//\n// GGML_F32_EPR / GGML_F16_EPR\n//   number of elements to fit in a single register\n//\n\n#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_FMA)\n\n#define GGML_SIMD\n\n// F32 SVE\n#define GGML_F32_EPR 8\n#define DEFAULT_PG svptrue_b32()\n\n#define GGML_F32xt                        svfloat32_t\n#define GGML_F32xt_ZERO                   svdup_n_f32(0.0f)\n#define GGML_F32xt_SET1(x)                svdup_n_f32(x)\n#define GGML_F32xt_LOAD_IMPL(pg, a)       svld1_f32(pg, a)\n#define GGML_F32xt_LOAD(a)                GGML_F32xt_LOAD_IMPL(DEFAULT_PG, a)\n#define GGML_F32xt_STORE_IMPL(pg, a, b)   svst1_f32(pg, a, b)\n#define GGML_F32xt_STORE(a, b)            GGML_F32xt_STORE_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_FMA_IMPL(pg, a, b, c)  svmad_f32_m(pg, b, c, a)\n#define GGML_F32xt_FMA(a, b, c)           GGML_F32xt_FMA_IMPL(DEFAULT_PG, a, b, c)\n#define GGML_F32xt_ADD_IMPL(pg, a, b)     svadd_f32_m(pg, a, b)\n#define GGML_F32xt_ADD(a, b)              GGML_F32xt_ADD_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_MUL_IMPL(pg, a, b)     svmul_f32_m(pg, a, b)\n#define GGML_F32xt_MUL(a, b)              GGML_F32xt_MUL_IMPL(DEFAULT_PG, a, b)\n#define GGML_F32xt_REDUCE_ONE_IMPL(pg, a) svaddv(pg, a)\n#define GGML_F32xt_REDUCE_ONE(a)          GGML_F32xt_REDUCE_ONE_IMPL(DEFAULT_PG, a)\n#define GGML_F32xt_REDUCE_IMPL(pg, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8)  \\\n{                                                      \\\n    sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum2);        \\\n    sum3 = svadd_f32_m(DEFAULT_PG, sum3, sum4);        \\\n    sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum6);        \\\n    sum7 = svadd_f32_m(DEFAULT_PG, sum7, sum8);        \\\n    sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum3);        \\\n    sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum7);        \\\n    sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum5);        \\\n    (res) = (ggml_float) GGML_F32xt_REDUCE_ONE(sum1);  \\\n}\n#define GGML_F32xt_REDUCE(res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8)  \\\n        GGML_F32xt_REDUCE_IMPL(DEFAULT_PG, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8)\n\n#define GGML_F32_VEC        GGML_F32xt\n#define GGML_F32_VEC_ZERO   GGML_F32xt_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32xt_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32xt_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32xt_STORE\n#define GGML_F32_VEC_FMA    GGML_F32xt_FMA\n#define GGML_F32_VEC_ADD    GGML_F32xt_ADD\n#define GGML_F32_VEC_MUL    GGML_F32xt_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32xt_REDUCE\n\n// F16 SVE\n#define DEFAULT_PG32    svptrue_b32()\n#define DEFAULT_PG16    svptrue_b16()\n\n#define GGML_F32Cxt                         svfloat16_t\n#define GGML_F32Cxt_ZERO                    svdup_n_f16(0.0f)\n#define GGML_F32Cxt_SET1(x)                 svdup_n_f16(x)\n#define GGML_F32Cxt_LOAD(p)                 svld1_f16(DEFAULT_PG16, (const __fp16 *)(p))\n#define GGML_F32Cxt_STORE(dst_ptr, src_vec) svst1_f16(DEFAULT_PG16, (__fp16 *)(dst_ptr), (src_vec))\n\n#define GGML_F32Cxt_FMA_IMPL(pg, a, b, c)   svmad_f16_x(pg, b, c, a)\n#define GGML_F32Cxt_FMA(a, b, c)            GGML_F32Cxt_FMA_IMPL(DEFAULT_PG16, a, b, c)\n#define GGML_F32Cxt_ADD_IMPL(pg, a, b)      svadd_f16_x(pg, a, b)\n#define GGML_F32Cxt_ADD(a, b)               GGML_F32Cxt_ADD_IMPL(DEFAULT_PG16, a, b)\n#define GGML_F32Cxt_MUL_IMPL(pg, a, b)      svmul_f16_x(pg, a, b)\n#define GGML_F32Cxt_MUL(a, b)               GGML_F32Cxt_MUL_IMPL(DEFAULT_PG16, a, b)\n#define GGML_F32Cxt_REDUCE                  GGML_F16xt_REDUCE_MIXED\n\n#define GGML_F16x_VEC                GGML_F32Cxt\n#define GGML_F16x_VEC_ZERO           GGML_F32Cxt_ZERO\n#define GGML_F16x_VEC_SET1           GGML_F32Cxt_SET1\n#define GGML_F16x_VEC_LOAD(p, i)     GGML_F32Cxt_LOAD(p)\n#define GGML_F16x_VEC_STORE(p, r, i) GGML_F32Cxt_STORE((__fp16 *)(p), r)\n#define GGML_F16x_VEC_FMA            GGML_F32Cxt_FMA\n#define GGML_F16x_VEC_ADD            GGML_F32Cxt_ADD\n#define GGML_F16x_VEC_MUL            GGML_F32Cxt_MUL\n#define GGML_F16x_VEC_REDUCE         GGML_F32Cxt_REDUCE\n\n#define GGML_F16xt_REDUCE_ONE_IMPL(pg, a) svaddv_f16(pg, a)\n#define GGML_F16xt_REDUCE_ONE(a)          GGML_F16xt_REDUCE_ONE_IMPL(DEFAULT_PG16, a)\n\n#define GGML_F16xt_REDUCE_MIXED_IMPL(pg16, res, sum1, sum2, sum3, sum4)  \\\n{                                                      \\\n    sum1 = svadd_f16_x(pg16, sum1, sum2);              \\\n    sum3 = svadd_f16_x(pg16, sum3, sum4);              \\\n    sum1 = svadd_f16_x(pg16, sum1, sum3);              \\\n    __fp16 sum_f16 = svaddv_f16(pg16, sum1);           \\\n    (res) = (ggml_float) sum_f16;                      \\\n}\n#define GGML_F16xt_REDUCE_MIXED(res, sum1, sum2, sum3, sum4)  \\\n        GGML_F16xt_REDUCE_MIXED_IMPL(DEFAULT_PG16, res, sum1, sum2, sum3, sum4)\n\n// F16 NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n    #define GGML_F16_STEP 32\n    #define GGML_F16_EPR  8\n\n    #define GGML_F16x8              float16x8_t\n    #define GGML_F16x8_ZERO         vdupq_n_f16(0.0f)\n    #define GGML_F16x8_SET1(x)      vdupq_n_f16(x)\n    #define GGML_F16x8_LOAD(x)      vld1q_f16((const __fp16 *)(x))\n    #define GGML_F16x8_STORE        vst1q_f16\n    #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)\n    #define GGML_F16x8_ADD          vaddq_f16\n    #define GGML_F16x8_MUL          vmulq_f16\n    #define GGML_F16x8_REDUCE(res, x)                               \\\n    do {                                                            \\\n        int offset = GGML_F16_ARR >> 1;                             \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        offset >>= 1;                                               \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        offset >>= 1;                                               \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \\\n        const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \\\n        (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1));         \\\n    } while (0)\n\n    #define GGML_F16_VEC                GGML_F16x8\n    #define GGML_F16_VEC_ZERO           GGML_F16x8_ZERO\n    #define GGML_F16_VEC_SET1           GGML_F16x8_SET1\n    #define GGML_F16_VEC_LOAD(p, i)     GGML_F16x8_LOAD(p)\n    #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])\n    #define GGML_F16_VEC_FMA            GGML_F16x8_FMA\n    #define GGML_F16_VEC_ADD            GGML_F16x8_ADD\n    #define GGML_F16_VEC_MUL            GGML_F16x8_MUL\n    #define GGML_F16_VEC_REDUCE         GGML_F16x8_REDUCE\n#else\n    // if FP16 vector arithmetic is not supported, we use FP32 instead\n    // and take advantage of the vcvt_ functions to convert to/from FP16\n\n    #define GGML_F16_STEP 16\n    #define GGML_F16_EPR  4\n\n    #define GGML_F32Cx4              float32x4_t\n    #define GGML_F32Cx4_ZERO         vdupq_n_f32(0.0f)\n    #define GGML_F32Cx4_SET1(x)      vdupq_n_f32(x)\n    #define GGML_F32Cx4_LOAD(x)      vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))\n    #define GGML_F32Cx4_STORE(x, y)  vst1_f16(x, vcvt_f16_f32(y))\n    #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)\n    #define GGML_F32Cx4_ADD          vaddq_f32\n    #define GGML_F32Cx4_MUL          vmulq_f32\n    #define GGML_F32Cx4_REDUCE       GGML_F32x4_REDUCE\n\n    #define GGML_F16_VEC                GGML_F32Cx4\n    #define GGML_F16_VEC_ZERO           GGML_F32Cx4_ZERO\n    #define GGML_F16_VEC_SET1           GGML_F32Cx4_SET1\n    #define GGML_F16_VEC_LOAD(p, i)     GGML_F32Cx4_LOAD(p)\n    #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])\n    #define GGML_F16_VEC_FMA            GGML_F32Cx4_FMA\n    #define GGML_F16_VEC_ADD            GGML_F32Cx4_ADD\n    #define GGML_F16_VEC_MUL            GGML_F32Cx4_MUL\n    #define GGML_F16_VEC_REDUCE         GGML_F32Cx4_REDUCE\n#endif\n\n#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA)\n\n#define GGML_SIMD\n\n// F32 NEON\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4              float32x4_t\n#define GGML_F32x4_ZERO         vdupq_n_f32(0.0f)\n#define GGML_F32x4_SET1(x)      vdupq_n_f32(x)\n#define GGML_F32x4_LOAD         vld1q_f32\n#define GGML_F32x4_STORE        vst1q_f32\n#define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c)\n#define GGML_F32x4_ADD          vaddq_f32\n#define GGML_F32x4_MUL          vmulq_f32\n#define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x)\n#define GGML_F32x4_REDUCE(res, x)                       \\\n{                                                       \\\n    int offset = GGML_F32_ARR >> 1;                     \\\n    for (int i = 0; i < offset; ++i) {                  \\\n        (x)[i] = vaddq_f32((x)[i], (x)[offset+i]);      \\\n    }                                                   \\\n    offset >>= 1;                                       \\\n    for (int i = 0; i < offset; ++i) {                  \\\n        (x)[i] = vaddq_f32((x)[i], (x)[offset+i]);      \\\n    }                                                   \\\n    offset >>= 1;                                       \\\n    for (int i = 0; i < offset; ++i) {                  \\\n        (x)[i] = vaddq_f32((x)[i], (x)[offset+i]);      \\\n    }                                                   \\\n    (res) = (ggml_float) GGML_F32x4_REDUCE_ONE((x)[0]); \\\n}\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 NEON\n\n#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)\n    #define GGML_F16_STEP 32\n    #define GGML_F16_EPR  8\n\n    #define GGML_F16x8              float16x8_t\n    #define GGML_F16x8_ZERO         vdupq_n_f16(0.0f)\n    #define GGML_F16x8_SET1(x)      vdupq_n_f16(x)\n    #define GGML_F16x8_LOAD(x)      vld1q_f16((const __fp16 *)(x))\n    #define GGML_F16x8_STORE        vst1q_f16\n    #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c)\n    #define GGML_F16x8_ADD          vaddq_f16\n    #define GGML_F16x8_MUL          vmulq_f16\n    #define GGML_F16x8_REDUCE(res, x)                               \\\n    do {                                                            \\\n        int offset = GGML_F16_ARR >> 1;                             \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        offset >>= 1;                                               \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        offset >>= 1;                                               \\\n        for (int i = 0; i < offset; ++i) {                          \\\n            (x)[i] = vaddq_f16((x)[i], (x)[offset+i]);              \\\n        }                                                           \\\n        const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \\\n        const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \\\n        (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1));         \\\n    } while (0)\n\n    #define GGML_F16_VEC                GGML_F16x8\n    #define GGML_F16_VEC_ZERO           GGML_F16x8_ZERO\n    #define GGML_F16_VEC_SET1           GGML_F16x8_SET1\n    #define GGML_F16_VEC_LOAD(p, i)     GGML_F16x8_LOAD(p)\n    #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i])\n    #define GGML_F16_VEC_FMA            GGML_F16x8_FMA\n    #define GGML_F16_VEC_ADD            GGML_F16x8_ADD\n    #define GGML_F16_VEC_MUL            GGML_F16x8_MUL\n    #define GGML_F16_VEC_REDUCE         GGML_F16x8_REDUCE\n#else\n    // if FP16 vector arithmetic is not supported, we use FP32 instead\n    // and take advantage of the vcvt_ functions to convert to/from FP16\n\n    #define GGML_F16_STEP 16\n    #define GGML_F16_EPR  4\n\n    #define GGML_F32Cx4              float32x4_t\n    #define GGML_F32Cx4_ZERO         vdupq_n_f32(0.0f)\n    #define GGML_F32Cx4_SET1(x)      vdupq_n_f32(x)\n    #define GGML_F32Cx4_LOAD(x)      vcvt_f32_f16(vld1_f16((const __fp16 *)(x)))\n    #define GGML_F32Cx4_STORE(x, y)  vst1_f16(x, vcvt_f16_f32(y))\n    #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c)\n    #define GGML_F32Cx4_ADD          vaddq_f32\n    #define GGML_F32Cx4_MUL          vmulq_f32\n    #define GGML_F32Cx4_REDUCE       GGML_F32x4_REDUCE\n\n    #define GGML_F16_VEC                GGML_F32Cx4\n    #define GGML_F16_VEC_ZERO           GGML_F32Cx4_ZERO\n    #define GGML_F16_VEC_SET1           GGML_F32Cx4_SET1\n    #define GGML_F16_VEC_LOAD(p, i)     GGML_F32Cx4_LOAD(p)\n    #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i])\n    #define GGML_F16_VEC_FMA            GGML_F32Cx4_FMA\n    #define GGML_F16_VEC_ADD            GGML_F32Cx4_ADD\n    #define GGML_F16_VEC_MUL            GGML_F32Cx4_MUL\n    #define GGML_F16_VEC_REDUCE         GGML_F32Cx4_REDUCE\n#endif\n\n#elif defined(__AVX512F__)\n\n#define GGML_SIMD\n\n// F32 AVX512\n\n#define GGML_F32_STEP 64\n#define GGML_F32_EPR  16\n\n#define GGML_F32x16         __m512\n#define GGML_F32x16_ZERO    _mm512_setzero_ps()\n#define GGML_F32x16_SET1(x) _mm512_set1_ps(x)\n#define GGML_F32x16_LOAD    _mm512_loadu_ps\n#define GGML_F32x16_STORE   _mm512_storeu_ps\n// _mm512_fmadd_ps is defined in AVX512F so no guard is required\n#define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)\n#define GGML_F32x16_ADD     _mm512_add_ps\n#define GGML_F32x16_MUL     _mm512_mul_ps\n#define GGML_F32x16_REDUCE(res, x)                                    \\\ndo {                                                                  \\\n    int offset = GGML_F32_ARR >> 1;                                   \\\n    for (int i = 0; i < offset; ++i) {                                \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                      \\\n    }                                                                 \\\n    offset >>= 1;                                                     \\\n    for (int i = 0; i < offset; ++i) {                                \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                      \\\n    }                                                                 \\\n    offset >>= 1;                                                     \\\n    for (int i = 0; i < offset; ++i) {                                \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                      \\\n    }                                                                 \\\n    res = (ggml_float) _mm512_reduce_add_ps(x[0]);                    \\\n} while (0)\n\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC        GGML_F32x16\n#define GGML_F32_VEC_ZERO   GGML_F32x16_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x16_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x16_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x16_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x16_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x16_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x16_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE\n\n// F16 AVX512\n\n#if defined(__AVX512FP16__)\n\n#define GGML_F16_STEP 128\n#define GGML_F16_EPR  32\n\n#define GGML_F16x32              __m512h\n#define GGML_F16x32_ZERO         _mm512_setzero_ph()\n#define GGML_F16x32_SET1(x)      _mm512_set1_ph(__extension__(_Float16)(x))\n#define GGML_F16x32_LOAD(x)      _mm512_loadu_ph(x)\n#define GGML_F16x32_STORE(x, y)  _mm512_storeu_ph(x, y)\n#define GGML_F16x32_FMA(a, b, c) _mm512_fmadd_ph(b, c, a)\n#define GGML_F16x32_ADD          _mm512_add_ph\n#define GGML_F16x32_MUL          _mm512_mul_ph\n#define GGML_F16x32_REDUCE(res, x)                                     \\\ndo {                                                                   \\\n    int offset = GGML_F16_ARR >> 1;                                    \\\n    for (int i = 0; i < offset; ++i) {                                 \\\n        x[i] = _mm512_add_ph(x[i], x[offset+i]);                       \\\n    }                                                                  \\\n    offset >>= 1;                                                      \\\n    for (int i = 0; i < offset; ++i) {                                 \\\n        x[i] = _mm512_add_ph(x[i], x[offset+i]);                       \\\n    }                                                                  \\\n    offset >>= 1;                                                      \\\n    for (int i = 0; i < offset; ++i) {                                 \\\n        x[i] = _mm512_add_ph(x[i], x[offset+i]);                       \\\n    }                                                                  \\\n    res = (ggml_float) _mm512_reduce_add_ph(x[0]);                     \\\n} while (0)\n\n#define GGML_F16_VEC                GGML_F16x32\n#define GGML_F16_VEC_ZERO           GGML_F16x32_ZERO\n#define GGML_F16_VEC_SET1           GGML_F16x32_SET1\n#define GGML_F16_VEC_LOAD(p, i)     GGML_F16x32_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x32_STORE(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F16x32_FMA\n#define GGML_F16_VEC_ADD            GGML_F16x32_ADD\n#define GGML_F16_VEC_MUL            GGML_F16x32_MUL\n#define GGML_F16_VEC_REDUCE         GGML_F16x32_REDUCE\n\n#else // Fallback FP16 <-> FP32\n\n#define GGML_F16_STEP 64\n#define GGML_F16_EPR  16\n\n#define GGML_F32Cx16             __m512\n#define GGML_F32Cx16_ZERO        _mm512_setzero_ps()\n#define GGML_F32Cx16_SET1(x)     _mm512_set1_ps(x)\n\n// unlike  _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F\n// so F16C guard isn't required\n#define GGML_F32Cx16_LOAD(x)     _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x)))\n#define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0))\n\n#define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a)\n#define GGML_F32Cx16_ADD         _mm512_add_ps\n#define GGML_F32Cx16_MUL         _mm512_mul_ps\n#define GGML_F32Cx16_REDUCE(res, x)                               \\\ndo {                                                              \\\n    int offset = GGML_F32_ARR >> 1;                               \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm512_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    res = (ggml_float) _mm512_reduce_add_ps(x[0]);                \\\n} while (0)\n\n#define GGML_F16_VEC                GGML_F32Cx16\n#define GGML_F16_VEC_ZERO           GGML_F32Cx16_ZERO\n#define GGML_F16_VEC_SET1           GGML_F32Cx16_SET1\n#define GGML_F16_VEC_LOAD(p, i)     GGML_F32Cx16_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F32Cx16_FMA\n#define GGML_F16_VEC_ADD            GGML_F32Cx16_ADD\n#define GGML_F16_VEC_MUL            GGML_F32Cx16_MUL\n\n#define GGML_F16_VEC_REDUCE         GGML_F32Cx16_REDUCE\n\n#endif // __AVX512FP16__\n#elif defined(__AVX__)\n\n#define GGML_SIMD\n\n// F32 AVX\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  8\n\n#define GGML_F32x8         __m256\n#define GGML_F32x8_ZERO    _mm256_setzero_ps()\n#define GGML_F32x8_SET1(x) _mm256_set1_ps(x)\n#define GGML_F32x8_LOAD    _mm256_loadu_ps\n#define GGML_F32x8_STORE   _mm256_storeu_ps\n#if defined(__FMA__)\n    #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a)\n#else\n    #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a)\n#endif\n#define GGML_F32x8_ADD     _mm256_add_ps\n#define GGML_F32x8_MUL     _mm256_mul_ps\n#define GGML_F32x8_REDUCE(res, x)                                 \\\ndo {                                                              \\\n    int offset = GGML_F32_ARR >> 1;                               \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm256_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm256_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm256_add_ps(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]),    \\\n                                 _mm256_extractf128_ps(x[0], 1)); \\\n    const __m128 t1 = _mm_hadd_ps(t0, t0);                        \\\n    res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1));        \\\n} while (0)\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC        GGML_F32x8\n#define GGML_F32_VEC_ZERO   GGML_F32x8_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x8_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x8_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x8_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x8_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x8_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x8_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE\n\n// F16 AVX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR  8\n\n// F16 arithmetic is not supported by AVX, so we use F32 instead\n\n#define GGML_F32Cx8             __m256\n#define GGML_F32Cx8_ZERO        _mm256_setzero_ps()\n#define GGML_F32Cx8_SET1(x)     _mm256_set1_ps(x)\n\n#if defined(__F16C__)\n// the  _mm256_cvt intrinsics require F16C\n#define GGML_F32Cx8_LOAD(x)     _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x)))\n#define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0))\n#else\nstatic inline __m256 __avx_f32cx8_load(const ggml_fp16_t * x) {\n    float tmp[8];\n\n    for (int i = 0; i < 8; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n\n    return _mm256_loadu_ps(tmp);\n}\nstatic inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) {\n    float arr[8];\n\n    _mm256_storeu_ps(arr, y);\n\n    for (int i = 0; i < 8; i++)\n        x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);\n}\n#define GGML_F32Cx8_LOAD(x)     __avx_f32cx8_load(x)\n#define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y)\n#endif\n\n#define GGML_F32Cx8_FMA         GGML_F32x8_FMA\n#define GGML_F32Cx8_ADD         _mm256_add_ps\n#define GGML_F32Cx8_MUL         _mm256_mul_ps\n#define GGML_F32Cx8_REDUCE      GGML_F32x8_REDUCE\n\n#define GGML_F16_VEC                GGML_F32Cx8\n#define GGML_F16_VEC_ZERO           GGML_F32Cx8_ZERO\n#define GGML_F16_VEC_SET1           GGML_F32Cx8_SET1\n#define GGML_F16_VEC_LOAD(p, i)     GGML_F32Cx8_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F32Cx8_FMA\n#define GGML_F16_VEC_ADD            GGML_F32Cx8_ADD\n#define GGML_F16_VEC_MUL            GGML_F32Cx8_MUL\n#define GGML_F16_VEC_REDUCE         GGML_F32Cx8_REDUCE\n\n#elif defined(__POWER9_VECTOR__)\n\n#define GGML_SIMD\n\n// F32 POWER9\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4              vector float\n#define GGML_F32x4_ZERO         {0.0f}\n#define GGML_F32x4_SET1         vec_splats\n#define GGML_F32x4_LOAD(p)      vec_xl(0, p)\n#define GGML_F32x4_STORE(p, r)  vec_xst(r, 0, p)\n#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)\n#define GGML_F32x4_ADD          vec_add\n#define GGML_F32x4_MUL          vec_mul\n#define GGML_F32x4_REDUCE(res, x)              \\\n{                                              \\\n    int offset = GGML_F32_ARR >> 1;            \\\n    for (int i = 0; i < offset; ++i) {         \\\n        x[i] = vec_add(x[i], x[offset+i]);     \\\n    }                                          \\\n    offset >>= 1;                              \\\n    for (int i = 0; i < offset; ++i) {         \\\n        x[i] = vec_add(x[i], x[offset+i]);     \\\n    }                                          \\\n    offset >>= 1;                              \\\n    for (int i = 0; i < offset; ++i) {         \\\n        x[i] = vec_add(x[i], x[offset+i]);     \\\n    }                                          \\\n    res = vec_extract(x[0], 0) +               \\\n          vec_extract(x[0], 1) +               \\\n          vec_extract(x[0], 2) +               \\\n          vec_extract(x[0], 3);                \\\n}\n#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3)        \\\n{                                                       \\\n    vector float v = vec_add(vec_add(s0, s1),           \\\n                             vec_add(s2, s3));          \\\n    v = vec_add(v, vec_sld(v, v, 8));                   \\\n    v = vec_add(v, vec_sld(v, v, 4));                   \\\n    res += (ggml_float) vec_extract(v, 0);              \\\n}\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 POWER9\n#define GGML_F16_STEP       GGML_F32_STEP\n#define GGML_F16_EPR        GGML_F32_EPR\n#define GGML_F16_VEC        GGML_F32x4\n#define GGML_F16_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F16_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F16_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F16_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F16_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE\n// Use vec_xl, not vec_ld, in case the load address is not aligned.\n#define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ?                   \\\n  vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \\\n  vec_extract_fp32_from_shortl(vec_xl(0, p))\nstatic inline unsigned char ggml_endian_byte(int i) {\n       uint16_t tmp_val = 1;\n       return ((unsigned char *)&tmp_val)[i];\n}\n#define GGML_ENDIAN_BYTE(i) ggml_endian_byte(i)\n#define GGML_F16_VEC_STORE(p, r, i)                             \\\n  if (i & 0x1)                                                  \\\n    vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)],  \\\n                                   r[i - GGML_ENDIAN_BYTE(0)]), \\\n            0, p - GGML_F16_EPR)\n\n//BF16 POWER9\n#define GGML_BF16_STEP 16\n#define GGML_BF16_EPR  8\n\n#define GGML_BF16x8         vector unsigned short\n#define GGML_BF16x8_ZERO    vec_splats((unsigned short)0)\n#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))\n\n#define GGML_BF16_VEC          GGML_BF16x8\n#define GGML_BF16_VEC_ZERO     GGML_BF16x8_ZERO\n#define GGML_BF16_VEC_LOAD     GGML_BF16x8_LOAD\n#if defined(__LITTLE_ENDIAN__)\n#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel(GGML_BF16_VEC_ZERO, (v)))\n#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh(GGML_BF16_VEC_ZERO, (v)))\n#else\n#define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh((v), GGML_BF16_VEC_ZERO))\n#endif\n#define GGML_BF16_FMA_LO(acc, x, y) \\\n    (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))\n#define GGML_BF16_FMA_HI(acc, x, y) \\\n    (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))\n\n#elif defined(__wasm_simd128__)\n\n#define GGML_SIMD\n\n// F32 WASM\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4              v128_t\n#define GGML_F32x4_ZERO         wasm_f32x4_splat(0.0f)\n#define GGML_F32x4_SET1(x)      wasm_f32x4_splat(x)\n#define GGML_F32x4_LOAD         wasm_v128_load\n#define GGML_F32x4_STORE        wasm_v128_store\n#define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a)\n#define GGML_F32x4_ADD          wasm_f32x4_add\n#define GGML_F32x4_MUL          wasm_f32x4_mul\n#define GGML_F32x4_REDUCE(res, x)                  \\\n{                                                  \\\n    int offset = GGML_F32_ARR >> 1;                \\\n    for (int i = 0; i < offset; ++i) {             \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);  \\\n    }                                              \\\n    offset >>= 1;                                  \\\n    for (int i = 0; i < offset; ++i) {             \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);  \\\n    }                                              \\\n    offset >>= 1;                                  \\\n    for (int i = 0; i < offset; ++i) {             \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);  \\\n    }                                              \\\n    res = wasm_f32x4_extract_lane(x[0], 0) +       \\\n          wasm_f32x4_extract_lane(x[0], 1) +       \\\n          wasm_f32x4_extract_lane(x[0], 2) +       \\\n          wasm_f32x4_extract_lane(x[0], 3);        \\\n}\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 WASM\n\n#define GGML_F16_STEP 16\n#define GGML_F16_EPR  4\n\ninline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) {\n    float tmp[4];\n\n    tmp[0] = GGML_CPU_FP16_TO_FP32(p[0]);\n    tmp[1] = GGML_CPU_FP16_TO_FP32(p[1]);\n    tmp[2] = GGML_CPU_FP16_TO_FP32(p[2]);\n    tmp[3] = GGML_CPU_FP16_TO_FP32(p[3]);\n\n    return wasm_v128_load(tmp);\n}\n\ninline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) {\n    float tmp[4];\n\n    wasm_v128_store(tmp, x);\n\n    p[0] = GGML_CPU_FP32_TO_FP16(tmp[0]);\n    p[1] = GGML_CPU_FP32_TO_FP16(tmp[1]);\n    p[2] = GGML_CPU_FP32_TO_FP16(tmp[2]);\n    p[3] = GGML_CPU_FP32_TO_FP16(tmp[3]);\n}\n\n#define GGML_F16x4             v128_t\n#define GGML_F16x4_ZERO        wasm_f32x4_splat(0.0f)\n#define GGML_F16x4_SET1(x)     wasm_f32x4_splat(x)\n#define GGML_F16x4_LOAD(x)     __wasm_f16x4_load(x)\n#define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y)\n#define GGML_F16x4_FMA         GGML_F32x4_FMA\n#define GGML_F16x4_ADD         wasm_f32x4_add\n#define GGML_F16x4_MUL         wasm_f32x4_mul\n#define GGML_F16x4_REDUCE(res, x)                           \\\n{                                                           \\\n    int offset = GGML_F16_ARR >> 1;                         \\\n    for (int i = 0; i < offset; ++i) {                      \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);           \\\n    }                                                       \\\n    offset >>= 1;                                           \\\n    for (int i = 0; i < offset; ++i) {                      \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);           \\\n    }                                                       \\\n    offset >>= 1;                                           \\\n    for (int i = 0; i < offset; ++i) {                      \\\n        x[i] = wasm_f32x4_add(x[i], x[offset+i]);           \\\n    }                                                       \\\n    res = (ggml_float) (wasm_f32x4_extract_lane(x[0], 0) +  \\\n          wasm_f32x4_extract_lane(x[0], 1) +                \\\n          wasm_f32x4_extract_lane(x[0], 2) +                \\\n          wasm_f32x4_extract_lane(x[0], 3));                \\\n}\n\n#define GGML_F16_VEC                GGML_F16x4\n#define GGML_F16_VEC_ZERO           GGML_F16x4_ZERO\n#define GGML_F16_VEC_SET1           GGML_F16x4_SET1\n#define GGML_F16_VEC_LOAD(p, i)     GGML_F16x4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F16x4_FMA\n#define GGML_F16_VEC_ADD            GGML_F16x4_ADD\n#define GGML_F16_VEC_MUL            GGML_F16x4_MUL\n#define GGML_F16_VEC_REDUCE         GGML_F16x4_REDUCE\n\n#elif defined(__SSE3__)\n\n#define GGML_SIMD\n\n// F32 SSE\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4         __m128\n#define GGML_F32x4_ZERO    _mm_setzero_ps()\n#define GGML_F32x4_SET1(x) _mm_set1_ps(x)\n#define GGML_F32x4_LOAD    _mm_loadu_ps\n#define GGML_F32x4_STORE   _mm_storeu_ps\n#if defined(__FMA__)\n    // TODO: Does this work?\n    #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a)\n#else\n    #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a)\n#endif\n#define GGML_F32x4_ADD     _mm_add_ps\n#define GGML_F32x4_MUL     _mm_mul_ps\n#define GGML_F32x4_REDUCE(res, x)                                 \\\n{                                                                 \\\n    int offset = GGML_F32_ARR >> 1;                               \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm_add_ps(x[i], x[offset+i]);                     \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm_add_ps(x[i], x[offset+i]);                     \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = _mm_add_ps(x[i], x[offset+i]);                     \\\n    }                                                             \\\n    const __m128 t0 = _mm_hadd_ps(x[0], x[0]);                    \\\n    res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0));        \\\n}\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 SSE\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR  4\n\nstatic inline __m128 __sse_f16x4_load(const ggml_fp16_t * x) {\n    float tmp[4];\n\n    tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]);\n    tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]);\n    tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]);\n    tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]);\n\n    return _mm_loadu_ps(tmp);\n}\n\nstatic inline void __sse_f16x4_store(ggml_fp16_t * x, __m128 y) {\n    float arr[4];\n\n    _mm_storeu_ps(arr, y);\n\n    x[0] = GGML_CPU_FP32_TO_FP16(arr[0]);\n    x[1] = GGML_CPU_FP32_TO_FP16(arr[1]);\n    x[2] = GGML_CPU_FP32_TO_FP16(arr[2]);\n    x[3] = GGML_CPU_FP32_TO_FP16(arr[3]);\n}\n\n#define GGML_F32Cx4             __m128\n#define GGML_F32Cx4_ZERO        _mm_setzero_ps()\n#define GGML_F32Cx4_SET1(x)     _mm_set1_ps(x)\n#define GGML_F32Cx4_LOAD(x)     __sse_f16x4_load(x)\n#define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y)\n#define GGML_F32Cx4_FMA         GGML_F32x4_FMA\n#define GGML_F32Cx4_ADD         _mm_add_ps\n#define GGML_F32Cx4_MUL         _mm_mul_ps\n#define GGML_F32Cx4_REDUCE      GGML_F32x4_REDUCE\n\n#define GGML_F16_VEC                 GGML_F32Cx4\n#define GGML_F16_VEC_ZERO            GGML_F32Cx4_ZERO\n#define GGML_F16_VEC_SET1            GGML_F32Cx4_SET1\n#define GGML_F16_VEC_LOAD(p, i)      GGML_F32Cx4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i)  GGML_F32Cx4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA             GGML_F32Cx4_FMA\n#define GGML_F16_VEC_ADD             GGML_F32Cx4_ADD\n#define GGML_F16_VEC_MUL             GGML_F32Cx4_MUL\n#define GGML_F16_VEC_REDUCE          GGML_F32Cx4_REDUCE\n\n#elif defined(__loongarch_asx)\n\n#define GGML_SIMD\n\n// F32 LASX\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  8\n\n#define GGML_F32x8         __m256\n#define GGML_F32x8_ZERO    (__m256)__lasx_xvldi(0)\n#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))\n#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0)\n#define GGML_F32x8_STORE(x,y)   __lasx_xvst((y), (x), 0)\n#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a)\n#define GGML_F32x8_ADD     __lasx_xvfadd_s\n#define GGML_F32x8_MUL     __lasx_xvfmul_s\n#define GGML_F32x8_REDUCE(res, x)                                 \\\ndo {                                                              \\\n    int offset = GGML_F32_ARR >> 1;                               \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = __lasx_xvfadd_s(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = __lasx_xvfadd_s(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    offset >>= 1;                                                 \\\n    for (int i = 0; i < offset; ++i) {                            \\\n        x[i] = __lasx_xvfadd_s(x[i], x[offset+i]);                  \\\n    }                                                             \\\n    float *tmp_p = (float *)&x[0]; \\\n    res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7];  \\\n} while (0)\n// TODO: is this optimal ?\n\n#define GGML_F32_VEC        GGML_F32x8\n#define GGML_F32_VEC_ZERO   GGML_F32x8_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x8_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x8_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x8_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x8_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x8_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x8_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE\n\n// F16 LASX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR  8\n\n// F16 arithmetic is not supported by LASX, so we use F32 instead\n\n#define GGML_F32Cx8          __m256\n#define GGML_F32Cx8_ZERO    (__m256)__lasx_xvldi(0)\n#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x))\n\nstatic inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) {\n    __m256i a;\n    memcpy(&a, x, sizeof(ggml_fp16_t) * 8);\n    a = __lasx_xvpermi_d(a, 0 | (1 << 4));\n    return __lasx_xvfcvtl_s_h(a);\n}\n\nstatic inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) {\n    __m256i a = __lasx_xvfcvt_h_s(y, y);\n    a = __lasx_xvpermi_d(a, 0 | (2 << 2));\n    memcpy(x, &a, sizeof(ggml_fp16_t) * 8);\n}\n#define GGML_F32Cx8_LOAD(x)     __lasx_f32cx8_load(x)\n#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)\n\n#define GGML_F32Cx8_FMA         GGML_F32x8_FMA\n#define GGML_F32Cx8_ADD         __lasx_xvfadd_s\n#define GGML_F32Cx8_MUL         __lasx_xvfmul_s\n#define GGML_F32Cx8_REDUCE      GGML_F32x8_REDUCE\n\n#define GGML_F16_VEC                GGML_F32Cx8\n#define GGML_F16_VEC_ZERO           GGML_F32Cx8_ZERO\n#define GGML_F16_VEC_SET1           GGML_F32Cx8_SET1\n#define GGML_F16_VEC_LOAD(p, i)     GGML_F32Cx8_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F32Cx8_FMA\n#define GGML_F16_VEC_ADD            GGML_F32Cx8_ADD\n#define GGML_F16_VEC_MUL            GGML_F32Cx8_MUL\n#define GGML_F16_VEC_REDUCE         GGML_F32Cx8_REDUCE\n\n#elif defined(__loongarch_sx)\n\n#define GGML_SIMD\n\n// F32 LSX\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4         __m128\n#define GGML_F32x4_ZERO    (__m128)__lsx_vldi(0)\n#define GGML_F32x4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x))\n#define GGML_F32x4_LOAD(x) (__m128)__lsx_vld((x), 0)\n#define GGML_F32x4_STORE(x, y)   __lsx_vst(y, x, 0)\n#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a)\n#define GGML_F32x4_ADD     __lsx_vfadd_s\n#define GGML_F32x4_MUL     __lsx_vfmul_s\n\n#define GGML_F32x4_REDUCE(res, x)                               \\\n{                                                               \\\n    int offset = GGML_F32_ARR >> 1;                             \\\n    for (int i = 0; i < offset; ++i) {                          \\\n        x[i] = __lsx_vfadd_s(x[i], x[offset+i]);                \\\n    }                                                           \\\n    offset >>= 1;                                               \\\n    for (int i = 0; i < offset; ++i) {                          \\\n        x[i] = __lsx_vfadd_s(x[i], x[offset+i]);                \\\n    }                                                           \\\n    offset >>= 1;                                               \\\n    for (int i = 0; i < offset; ++i) {                          \\\n        x[i] = __lsx_vfadd_s(x[i], x[offset+i]);                \\\n    }                                                           \\\n    __m128i t0 = __lsx_vpickev_w((__m128i)x[0], (__m128i)x[0]); \\\n    __m128i t1 = __lsx_vpickod_w((__m128i)x[0], (__m128i)x[0]); \\\n    __m128 t2 = __lsx_vfadd_s((__m128)t0, (__m128)t1);          \\\n    __m128i t3 = __lsx_vpickev_w((__m128i)t2, (__m128i)t2);     \\\n    __m128i t4 = __lsx_vpickod_w((__m128i)t2, (__m128i)t2);     \\\n    __m128 t5 = __lsx_vfadd_s((__m128)t3, (__m128)t4);          \\\n    res = (ggml_float) ((v4f32)t5)[0];                          \\\n}\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 LSX\n\n#define GGML_F16_STEP 32\n#define GGML_F16_EPR  4\n\nstatic inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) {\n    float tmp[4];\n\n    tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]);\n    tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]);\n    tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]);\n    tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]);\n\n    return (__m128)__lsx_vld(tmp, 0);\n}\n\nstatic inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {\n    float arr[4];\n\n    __lsx_vst(y, arr, 0);\n\n    x[0] = GGML_CPU_FP32_TO_FP16(arr[0]);\n    x[1] = GGML_CPU_FP32_TO_FP16(arr[1]);\n    x[2] = GGML_CPU_FP32_TO_FP16(arr[2]);\n    x[3] = GGML_CPU_FP32_TO_FP16(arr[3]);\n}\n\n#define GGML_F32Cx4             __m128\n#define GGML_F32Cx4_ZERO        (__m128)__lsx_vldi(0)\n#define GGML_F32Cx4_SET1(x)     (__m128)__lsx_vreplfr2vr_s((x))\n#define GGML_F32Cx4_LOAD(x)     (__m128)__lsx_f16x4_load(x)\n#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y)\n#define GGML_F32Cx4_FMA         GGML_F32x4_FMA\n#define GGML_F32Cx4_ADD         __lsx_vfadd_s\n#define GGML_F32Cx4_MUL         __lsx_vfmul_s\n#define GGML_F32Cx4_REDUCE      GGML_F32x4_REDUCE\n\n#define GGML_F16_VEC                 GGML_F32Cx4\n#define GGML_F16_VEC_ZERO            GGML_F32Cx4_ZERO\n#define GGML_F16_VEC_SET1            GGML_F32Cx4_SET1\n#define GGML_F16_VEC_LOAD(p, i)      GGML_F32Cx4_LOAD(p)\n#define GGML_F16_VEC_STORE(p, r, i)  GGML_F32Cx4_STORE(p, r[i])\n#define GGML_F16_VEC_FMA             GGML_F32Cx4_FMA\n#define GGML_F16_VEC_ADD             GGML_F32Cx4_ADD\n#define GGML_F16_VEC_MUL             GGML_F32Cx4_MUL\n#define GGML_F16_VEC_REDUCE          GGML_F32Cx4_REDUCE\n\n#elif defined(__VXE__) || defined(__VXE2__)\n\n#define GGML_SIMD\n\n// F32 s390x\n\n#define GGML_F32_STEP 32\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4              float32x4_t\n#define GGML_F32x4_ZERO         vec_splats(0.0f)\n#define GGML_F32x4_SET1         vec_splats\n#define GGML_F32x4_LOAD(p)      vec_xl(0, p)\n#define GGML_F32x4_STORE(p, r)  vec_xst(r, 0, p)\n#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a)\n#define GGML_F32x4_ADD          vec_add\n#define GGML_F32x4_MUL          vec_mul\n#define GGML_F32x4_REDUCE(res, x)                   \\\n{                                                   \\\n    int offset = GGML_F32_ARR >> 1;                 \\\n    for (int i = 0; i < offset; ++i) {              \\\n        x[i] = vec_add(x[i], x[offset + i]);        \\\n    }                                               \\\n    offset >>= 1;                                   \\\n    for (int i = 0; i < offset; ++i) {              \\\n        x[i] = vec_add(x[i], x[offset + i]);        \\\n    }                                               \\\n    offset >>= 1;                                   \\\n    for (int i = 0; i < offset; ++i) {              \\\n        x[i] = vec_add(x[i], x[offset + i]);        \\\n    }                                               \\\n    float32x4_t tmp = x[0] + vec_reve(x[0]);        \\\n    res = tmp[0] + tmp[1];                          \\\n}\n#define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \\\n{                                                \\\n    float32x4_t v = vec_add(vec_add(s0, s1),     \\\n                            vec_add(s2, s3));    \\\n    v = vec_add(v, vec_sld(v, v, 8));            \\\n    v = vec_add(v, vec_sld(v, v, 4));            \\\n    res += (ggml_float)vec_extract(v, 0);        \\\n}\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n// F16 s390x\n#define GGML_F16_STEP GGML_F32_STEP\n#define GGML_F16_EPR  GGML_F32_EPR\n\nstatic inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) {\n    float tmp[4];\n\n    for (int i = 0; i < 4; i++) {\n        tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n\n    // note: keep type-cast here to prevent compiler bugs\n    // see: https://github.com/ggml-org/llama.cpp/issues/12846\n    return vec_xl(0, (const float *)(tmp));\n}\n\nstatic inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {\n    float arr[4];\n\n    // note: keep type-cast here to prevent compiler bugs\n    // see: https://github.com/ggml-org/llama.cpp/issues/12846\n    vec_xst(v_y, 0, (float *)(arr));\n\n    for (int i = 0; i < 4; i++) {\n        x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);\n    }\n}\n\n#define GGML_F16_VEC                GGML_F32x4\n#define GGML_F16_VEC_ZERO           GGML_F32x4_ZERO\n#define GGML_F16_VEC_SET1           GGML_F32x4_SET1\n#define GGML_F16_VEC_LOAD(p, i)     __lzs_f16cx4_load(p)\n#define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i])\n#define GGML_F16_VEC_FMA            GGML_F32x4_FMA\n#define GGML_F16_VEC_ADD            GGML_F32x4_ADD\n#define GGML_F16_VEC_MUL            GGML_F32x4_MUL\n#define GGML_F16_VEC_REDUCE         GGML_F32x4_REDUCE\n\n// BF16 s390x\n#define GGML_BF16_STEP 16\n#define GGML_BF16_EPR  8\n\n#define GGML_BF16x8         __vector unsigned short\n#define GGML_BF16x8_ZERO    vec_splats((unsigned short)0)\n#define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p))\n\n#define GGML_BF16_VEC      GGML_BF16x8\n#define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO\n#define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD\n#define GGML_BF16_TO_F32_LO(v) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_TO_F32_HI(v) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO))\n#define GGML_BF16_FMA_LO(acc, x, y) \\\n    (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y))\n#define GGML_BF16_FMA_HI(acc, x, y) \\\n    (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y))\n\n#elif defined(__riscv_v_intrinsic)\n\n// compatible with vlen >= 128\n\n#define GGML_SIMD\n\n// F32\n\n#define GGML_F32_STEP 16\n#define GGML_F32_EPR  4\n\n#define GGML_F32x4              vfloat32m1_t\n#define GGML_F32x4_ZERO         __riscv_vfmv_v_f_f32m1(0.0f, GGML_F32_EPR)\n#define GGML_F32x4_SET1(x)      __riscv_vfmv_v_f_f32m1(x, GGML_F32_EPR)\n#define GGML_F32x4_LOAD(x)      __riscv_vle32_v_f32m1(x, GGML_F32_EPR)\n#define GGML_F32x4_STORE(b, v)  __riscv_vse32_v_f32m1(b, v, GGML_F32_EPR)\n#define GGML_F32x4_FMA(a, b, c) __riscv_vfmacc_vv_f32m1(a, b, c, GGML_F32_EPR)\n#define GGML_F32x4_ADD(a, b)    __riscv_vfadd_vv_f32m1(a, b, GGML_F32_EPR)\n#define GGML_F32x4_MUL(a, b)    __riscv_vfmul_vv_f32m1(a, b, GGML_F32_EPR)\n\n#define GGML_F32_VEC        GGML_F32x4\n#define GGML_F32_VEC_ZERO   GGML_F32x4_ZERO\n#define GGML_F32_VEC_SET1   GGML_F32x4_SET1\n#define GGML_F32_VEC_LOAD   GGML_F32x4_LOAD\n#define GGML_F32_VEC_STORE  GGML_F32x4_STORE\n#define GGML_F32_VEC_FMA    GGML_F32x4_FMA\n#define GGML_F32_VEC_ADD    GGML_F32x4_ADD\n#define GGML_F32_VEC_MUL    GGML_F32x4_MUL\n#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE\n\n#endif\n\n// GGML_F32_ARR / GGML_F16_ARR\n//   number of registers to use per step\n#ifdef GGML_SIMD\n#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR)\n#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR)\n#endif\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/spacemit/ime.cpp",
    "content": "#define GGML_COMMON_IMPL_CPP\n#define GGML_COMMON_DECL_CPP\n\n#include \"ime.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-common.h\"\n#include \"ggml-cpu.h\"\n#include \"ime_kernels.h\"\n#include \"traits.h\"\n\n#include <algorithm>\n#include <cassert>\n#include <cmath>\n#include <cstdio>  // for GGML_ASSERT\n#include <stdexcept>\n#include <thread>\n\n// clang-format off\n#if defined(__riscv)\n\n#if !defined(__riscv_v) || !defined(__riscv_v_intrinsic)\n#error \"riscv v extension or v_intrinsic not enabled\"\n#else\n#include <riscv_vector.h>\n#endif\n\n#if !defined(__riscv_zfh)\n#error \"riscv zfh extension not enabled\"\n#endif\n\n#if defined(RISCV64_SPACEMIT_IME1)\n#else\n#error \"RISCV64_SPACEMIT_IME1 not defined\"\n#endif\n\n#else\n\n#error \"riscv not enabled in this build\"\n\n#endif\n\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#pragma GCC diagnostic ignored \"-Wcast-qual\"\n#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n#endif\n\n#if defined(RISCV64_SPACEMIT_IME1)\n#define QGEMM_STRIDEN_THREAD_ALIGN 16\n#else\n#define QGEMM_STRIDEN_THREAD_ALIGN 32\n#endif\n\n// clang-format on\n\nstruct qnbitgemm_spacemit_ime_args {\n    const float *     a_ptr               = nullptr;\n    size_t            lda                 = 0;\n    const std::byte * packed_quant_b_data = nullptr;\n    const float *     quant_b_scale       = nullptr;\n    const void *      quant_b_zp          = nullptr;\n    const float *     quant_b_blksum      = nullptr;\n    const float *     bias                = nullptr;\n    float *           c_ptr               = nullptr;\n    size_t            ldc                 = 0;\n};\n\nconstexpr size_t div_round_up(size_t up, size_t down) {\n    return (up + down - 1) / down;\n}\n\nconstexpr size_t q8_blk_size(size_t blk_len) {\n    const size_t blk_size = sizeof(float) + blk_len * sizeof(int8_t);\n    // Currently, the strictest alignment requirement of a block is for a float.\n    // Ensure contiguous blocks are suitably aligned.\n    assert(blk_size % alignof(float) == 0);\n    return blk_size;\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\nconst int num_ai_cores = std::thread::hardware_concurrency() / 2;\n\n}  // namespace ggml::cpu::riscv64_spacemit\n\nstatic void sqnbitgemm_spacemit_ime_i8i4(const size_t                        blk_len,\n                                         const size_t                        gemm_k,\n                                         const qnbitgemm_spacemit_ime_args * gemm_args,\n                                         void * const                        per_gemm_ws,\n                                         const size_t                        m_start,\n                                         const size_t                        m_count,\n                                         const size_t                        n_start,\n                                         const size_t                        n_count) {\n    constexpr size_t scale_stride = sizeof(uint16_t);\n    constexpr size_t blk_bitwidth = 4;\n\n    const size_t k_blks = div_round_up(gemm_k, blk_len);\n\n    const size_t      lda         = k_blks * q8_blk_size(blk_len);\n    const size_t      ldc         = gemm_args->ldc;\n    const size_t      ldb         = k_blks * (blk_len * blk_bitwidth / 8);\n    const std::byte * quant_a_ptr = static_cast<const std::byte *>(per_gemm_ws) + m_start * lda;\n\n    const size_t      zero_point_stride   = gemm_args->quant_b_zp != nullptr ? sizeof(uint8_t) : 0;\n    const size_t      packed_b_stride     = ldb + k_blks * (scale_stride + zero_point_stride);\n    const std::byte * packed_quant_b_data = gemm_args->packed_quant_b_data + n_start * packed_b_stride;\n\n    float * c_ptr = gemm_args->c_ptr + m_start * ldc + n_start;\n\n    size_t       count_n               = 0;\n    const size_t compute_block_count_n = m_count == 1 ? n_count : 16;\n    for (size_t n = 0; n < n_count; n += count_n) {\n        count_n = std::min(n_count - n, compute_block_count_n);\n\n        const std::byte * a_row    = quant_a_ptr;\n        const std::byte * b_col    = packed_quant_b_data + n * packed_b_stride;\n        const std::byte * b_col_zp = (zero_point_stride != 0) ? b_col : nullptr;\n        float *           c_blk    = c_ptr + n;\n\n        int32_t rows_remaining = m_count;\n\n        while (rows_remaining > 0) {\n            const auto rows_handled = sqnbitgemm_spacemit_ime::ime1::gemm_kernel_i8i4(\n                blk_len, a_row, b_col, nullptr, b_col_zp, c_blk, rows_remaining, count_n, gemm_k, k_blks, ldc, nullptr,\n                scale_stride);\n\n            c_blk += rows_handled * ldc;\n            a_row += rows_handled * lda;\n\n            rows_remaining -= rows_handled;\n        }\n    }\n}\n\ntemplate <int K> constexpr int QK_0() {\n    if constexpr (K == 4) {\n        return QK4_0;\n    }\n    if constexpr (K == 8) {\n        return QK8_0;\n    }\n    return -1;\n}\n\ntemplate <int K, int N> struct block {\n    ggml_half d[N];                         // deltas for N qK_0 blocks\n    uint8_t   qs[(QK_0<K>() * N * K) / 8];  // quants for N qK_0 blocks\n};\n\ntemplate <int K, int N> struct block_with_zp {\n    ggml_half d[N];                         // deltas for N qK_1 blocks\n    uint8_t   zp[N];                        // zero points for N qK_1 blocks\n    uint8_t   qs[(QK_0<K>() * N * K) / 8];  // quants for N qK_1 blocks\n};\n\n// control size\nstatic_assert(sizeof(block<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8, \"wrong block<4,16> size/padding\");\nstatic_assert(sizeof(block_with_zp<4, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 8 + 16 * sizeof(uint8_t),\n              \"wrong block_with_zp<4,16> size/padding\");\nstatic_assert(sizeof(block<8, 16>) == 16 * sizeof(ggml_half) + QK4_0 * 16, \"wrong block<8,16> size/padding\");\n\nusing block_q4_0x16 = block<4, 16>;\nusing block_q4_1x16 = block_with_zp<4, 16>;\nusing block_q8_0x16 = block<8, 16>;\n\nstatic block_q4_0x16 make_block_q4_0x16(block_q4_0 * in, unsigned int blck_size_interleave) {\n    block_q4_0x16 out;\n    GGML_ASSERT(QK4_0 / blck_size_interleave == 2);\n\n    for (int i = 0; i < 16; i++) {\n        out.d[i] = in[i].d;\n    }\n\n    for (int i = 0; i < 16; i++) {\n        // [0, 15], in.d & 0x0F\n        for (int j = 0; j < QK4_0 / 4; j++) {\n            //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n            //dst [b0 b8] ......... [b7 b15]\n            out.qs[i * QK4_0 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_0 / 4] & 0x0F) << 4);\n        }\n    }\n\n    for (int i = 0; i < 16; i++) {\n        // [16, 31], in.d & 0xF0\n        for (int j = 0; j < QK4_0 / 4; j++) {\n            //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n            //dst [b16 b24] ......... [b23 b31]\n            out.qs[4 * QK4_0 + i * QK4_0 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_0 / 4] & 0xF0);\n        }\n    }\n\n    return out;\n}\n\nstatic block_q4_1x16 make_block_q4_1x16(block_q4_1 * in, unsigned int blck_size_interleave) {\n    block_q4_1x16 out;\n    GGML_ASSERT(QK4_1 / blck_size_interleave == 2);\n\n    for (int i = 0; i < 16; i++) {\n        float d   = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d);\n        float m   = GGML_FP16_TO_FP32(in[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m);\n        float mid = -std::nearbyintf(m / d);\n        mid       = std::min(15.0f, std::max(0.0f, mid));\n        out.d[i]  = GGML_FP32_TO_FP16(d);\n        out.zp[i] = static_cast<uint8_t>(mid);\n    }\n\n    for (int i = 0; i < 16; i++) {\n        // [0, 15], in.d & 0x0F\n        for (int j = 0; j < QK4_1 / 4; j++) {\n            //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n            //dst [b0 b8] ......... [b7 b15]\n            out.qs[i * QK4_1 / 4 + j] = (in[i].qs[j] & 0x0F) | ((in[i].qs[j + QK4_1 / 4] & 0x0F) << 4);\n        }\n    }\n\n    for (int i = 0; i < 16; i++) {\n        // [16, 31], in.d & 0xF0\n        for (int j = 0; j < QK4_1 / 4; j++) {\n            //src [b0 b16] ......... [b8 b24] ......... [b15 b31]\n            //dst [b16 b24] ......... [b23 b31]\n            out.qs[4 * QK4_1 + i * QK4_1 / 4 + j] = ((in[i].qs[j] & 0xF0) >> 4) | (in[i].qs[j + QK4_1 / 4] & 0xF0);\n        }\n    }\n\n    return out;\n}\n\nstatic int repack_q4_0_to_q4_0_16_bl(struct ggml_tensor *       t,\n                                     int                        interleave_block,\n                                     const void * GGML_RESTRICT data,\n                                     size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_0);\n    GGML_ASSERT(interleave_block == 16);\n\n    constexpr int nrows_interleaved = 16;\n\n    block_q4_0x16 *    dst = (block_q4_0x16 *) t->data;\n    const block_q4_0 * src = (const block_q4_0 *) data;\n    block_q4_0         dst_tmp[16];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK4_0;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_0));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_0 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_0x16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic int repack_q4_1_to_q4_1_16_bl(struct ggml_tensor *       t,\n                                     int                        interleave_block,\n                                     const void * GGML_RESTRICT data,\n                                     size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_1);\n    GGML_ASSERT(interleave_block == 16);\n\n    constexpr int nrows_interleaved = 16;\n\n    block_q4_1x16 *    dst = (block_q4_1x16 *) t->data;\n    const block_q4_1 * src = (const block_q4_1 *) data;\n    block_q4_1         dst_tmp[16];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK4_1;\n\n    GGML_ASSERT(data_size == nrow * nblocks * sizeof(block_q4_1));\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK4_1 != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int i = 0; i < nrows_interleaved; i++) {\n                dst_tmp[i] = src[x + i * nblocks];\n            }\n            *dst++ = make_block_q4_1x16(dst_tmp, interleave_block);\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nstatic inline void get_scale_min_k4(int                           j,\n                                    const uint8_t * GGML_RESTRICT q,\n                                    uint8_t * GGML_RESTRICT       d,\n                                    uint8_t * GGML_RESTRICT       m) {\n    if (j < 4) {\n        *d = q[j] & 63;\n        *m = q[j + 4] & 63;\n    } else {\n        *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);\n        *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);\n    }\n}\n\nstatic int repack_q4_k_to_q4_1_16_bl(struct ggml_tensor *       t,\n                                     int                        interleave_block,\n                                     const void * GGML_RESTRICT data,\n                                     size_t                     data_size) {\n    GGML_ASSERT(t->type == GGML_TYPE_Q4_K);\n    GGML_ASSERT(interleave_block == 16);\n    GGML_ASSERT(QK_K / QK4_1 == 8);\n\n    constexpr int nrows_interleaved = 16;\n\n    block_q4_1x16 *    dst = (block_q4_1x16 *) t->data;\n    const block_q4_K * src = (const block_q4_K *) data;\n    block_q4_1         dst_tmp[16];\n    int                nrow    = ggml_nrows(t);\n    int                nblocks = t->ne[0] / QK_K;\n\n    if (t->ne[1] % nrows_interleaved != 0 || t->ne[0] % QK_K != 0) {\n        return -1;\n    }\n\n    for (int b = 0; b < nrow; b += nrows_interleaved) {\n        for (int64_t x = 0; x < nblocks; x++) {\n            for (int j = 0; j < 8; j++) {\n                for (int i = 0; i < nrows_interleaved; i++) {\n                    uint8_t     sc, m;\n                    const float d = GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d);\n                    const float min =\n                        GGML_FP16_TO_FP32(src[x + i * nblocks].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.dmin);\n                    get_scale_min_k4(j, src[x + i * nblocks].scales, &sc, &m);\n                    const float d1 = d * sc;\n                    const float m1 = min * m;\n\n                    dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.d = GGML_FP32_TO_FP16(d1);\n                    dst_tmp[i].GGML_COMMON_AGGR_U.GGML_COMMON_AGGR_S.m = GGML_FP32_TO_FP16(-m1);\n                    // src -> [b0, b32] [b1, b33] ... [b31, b63]\n                    // dst -> [b0, b16] [b1, b17] ... [b15, b31] [b32, b48] [b33, b49] ... [b47, b63]\n                    const uint8_t * q                                  = src[x + i * nblocks].qs + (j / 2) * QK4_1;\n                    if (j % 2 == 0) {\n                        for (int ii = 0; ii < 16; ii++) {\n                            dst_tmp[i].qs[ii] = (q[ii] & 0x0F) | ((q[ii + 16] & 0x0F) << 4);\n                        }\n                    } else {\n                        for (int ii = 0; ii < 16; ii++) {\n                            dst_tmp[i].qs[ii] = ((q[ii] & 0xF0) >> 4) | (q[ii + 16] & 0xF0);\n                        }\n                    }\n                }\n                *dst++ = make_block_q4_1x16(dst_tmp, interleave_block);\n            }\n        }\n        src += nrows_interleaved * nblocks;\n    }\n    return 0;\n\n    GGML_UNUSED(data_size);\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS>\nint repack(struct ggml_tensor *, const void *, size_t);\n\ntemplate <> int repack<block_q4_0, 8, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_0_to_q4_0_16_bl(t, 16, data, data_size);\n}\n\ntemplate <> int repack<block_q4_1, 8, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_1_to_q4_1_16_bl(t, 16, data, data_size);\n}\n\ntemplate <> int repack<block_q4_K, 8, 16>(struct ggml_tensor * t, const void * data, size_t data_size) {\n    return repack_q4_k_to_q4_1_16_bl(t, 16, data, data_size);\n}\n\nclass tensor_traits_base : public ggml::cpu::tensor_traits {\n  public:\n    virtual int repack(struct ggml_tensor * t, const void * data, size_t data_size) = 0;\n};\n\ntemplate <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_traits : public tensor_traits_base {\n    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                size = ggml_row_size(GGML_TYPE_Q8_0, ggml_nelements(op->src[1])) * 4;\n                size = ((size + QK4_0 - 1) / QK4_0) * (QK4_0 * sizeof(float) + sizeof(float));\n                return true;\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                if (op->src[0]->type == GGML_TYPE_Q4_0 ||  //\n                    op->src[0]->type == GGML_TYPE_Q4_1 ||  //\n                    op->src[0]->type == GGML_TYPE_Q4_K) {\n                    forward_mul_mat_q4(params, op);\n                    return true;\n                }\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    void forward_mul_mat_q4(ggml_compute_params * params, ggml_tensor * op) {\n        const ggml_tensor * src0 = op->src[0];\n        const ggml_tensor * src1 = op->src[1];\n        ggml_tensor *       dst  = op;\n\n        GGML_TENSOR_BINARY_OP_LOCALS\n\n        int ith = params->ith;\n        int nth = params->nth;\n\n        [[maybe_unused]] const enum ggml_type type = src0->type;\n\n        void *        w_data  = (void *) src0->data;\n        const float * feature = (const float *) src1->data;\n        float *       output  = (float *) dst->data;\n\n        const size_t                  batch_feature = ne12 * ne13;\n        [[maybe_unused]] const size_t batch_weight  = ne02 * ne03;\n        const size_t                  gemm_m        = ne11;\n        const size_t                  gemm_k        = ne10;\n        const size_t                  gemm_n        = ne01;\n\n        GGML_ASSERT(batch_weight == 1);\n\n        const size_t block_count_k           = div_round_up(gemm_k, QK4_0);\n        const size_t per_gemm_workspace_size = gemm_m * block_count_k * q8_blk_size(QK4_0);\n        const size_t per_gemm_workspace_stride =\n            div_round_up(per_gemm_workspace_size, alignof(uint64_t)) * alignof(uint64_t);\n        const size_t gemm_workspace_size = batch_feature * per_gemm_workspace_stride;\n        const size_t desired_wsize       = gemm_workspace_size + alignof(uint64_t) - 1;\n\n        if (ith == 0 && params->wsize < desired_wsize) {\n            throw std::runtime_error(\"wsize less than desired_wsize\");\n        }\n\n        std::vector<qnbitgemm_spacemit_ime_args> qnbitgemm_args(batch_feature);\n\n        for (size_t i = 0; i < batch_feature; i++) {\n            qnbitgemm_args[i].a_ptr               = feature + gemm_m * gemm_k * i;\n            qnbitgemm_args[i].lda                 = gemm_k;\n            qnbitgemm_args[i].packed_quant_b_data = (const std::byte *) w_data;\n            qnbitgemm_args[i].quant_b_scale       = nullptr;\n\n            if constexpr (std::is_same_v<BLOC_TYPE, block_q4_0>) {\n                qnbitgemm_args[i].quant_b_zp = nullptr;\n            } else {\n                qnbitgemm_args[i].quant_b_zp = w_data;\n            }\n\n            qnbitgemm_args[i].bias  = nullptr;\n            qnbitgemm_args[i].c_ptr = output + gemm_m * gemm_n * i;\n            qnbitgemm_args[i].ldc   = gemm_n;\n        }\n\n        const uintptr_t ws_ptr = reinterpret_cast<uintptr_t>(params->wdata);\n        void *          ws = reinterpret_cast<void *>((ws_ptr + alignof(uint64_t) - 1) & (~(alignof(uint64_t) - 1)));\n        const size_t    quant_a_stride = block_count_k * q8_blk_size(QK4_0);\n\n        {\n            constexpr size_t block_size_m           = 4;\n            size_t           per_gemm_block_count_m = div_round_up(gemm_m, block_size_m);\n            int32_t          task_count             = batch_feature * per_gemm_block_count_m;\n            int32_t          task_per_thread        = (task_count + nth - 1) / nth;\n            int32_t          start                  = ith * task_per_thread;\n            int32_t          end                    = std::min((ith + 1) * task_per_thread, task_count);\n            for (int32_t compute_idx = start; compute_idx < end; compute_idx++) {\n                int32_t                             gemm_idx = compute_idx / per_gemm_block_count_m;\n                int32_t                             block_idx_in_gemm = compute_idx % per_gemm_block_count_m;\n                int32_t                             m_idx    = block_idx_in_gemm * block_size_m;\n                const qnbitgemm_spacemit_ime_args & data     = qnbitgemm_args[gemm_idx];\n                int32_t rows_tobe_handled = (gemm_m - m_idx) > block_size_m ? block_size_m : (gemm_m - m_idx);\n\n                if (rows_tobe_handled == block_size_m) {\n                    const float * a_row_ptr = data.a_ptr + m_idx * data.lda;\n                    std::byte *   quant_a_row_ptr =\n                        static_cast<std::byte *>(ws) + gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride;\n                    sqnbitgemm_spacemit_ime::ime1::quantize_a_4row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr);\n                } else {\n                    while (rows_tobe_handled) {\n                        const float * a_row_ptr       = data.a_ptr + m_idx * data.lda;\n                        std::byte *   quant_a_row_ptr = static_cast<std::byte *>(ws) +\n                                                      gemm_idx * per_gemm_workspace_stride + m_idx * quant_a_stride;\n                        sqnbitgemm_spacemit_ime::ime1::quantize_a_row_i8(QK4_0, a_row_ptr, gemm_k, quant_a_row_ptr);\n                        rows_tobe_handled -= 1;\n                        m_idx += 1;\n                    }\n                }\n            }\n        }\n\n        ggml_barrier(params->threadpool);\n\n        if (ith >= ggml::cpu::riscv64_spacemit::num_ai_cores) {\n            return;\n        }\n        nth = std::min(nth, int{ ggml::cpu::riscv64_spacemit::num_ai_cores });\n\n        size_t           threads_per_gemm = nth / batch_feature;\n        constexpr size_t gemm_m_stride    = 128;\n        size_t           nc               = gemm_n;\n        const size_t     gemm_m_blocked   = div_round_up(gemm_m, gemm_m_stride);\n        const size_t     max_nc           = div_round_up(gemm_n * gemm_m_blocked, threads_per_gemm);\n        if (max_nc < nc) {\n            nc = std::min(nc, div_round_up(max_nc, QGEMM_STRIDEN_THREAD_ALIGN) * QGEMM_STRIDEN_THREAD_ALIGN);\n        }\n        const size_t gemm_n_stride  = nc;\n        const size_t thread_count_m = div_round_up(gemm_m, gemm_m_stride);\n        const size_t thread_count_n = div_round_up(gemm_n, gemm_n_stride);\n        threads_per_gemm            = thread_count_m * thread_count_n;\n\n        {\n            int task_count      = batch_feature * threads_per_gemm;\n            int task_per_thread = (task_count + nth - 1) / nth;\n            int start           = ith * task_per_thread;\n            int end             = std::min((ith + 1) * task_per_thread, task_count);\n            for (int compute_idx = start; compute_idx < end; compute_idx++) {\n                const auto   gemm_i = compute_idx / threads_per_gemm;\n                const auto   blk_i  = compute_idx % threads_per_gemm;\n                const auto * data   = &qnbitgemm_args[gemm_i];\n\n                const auto tid_n = blk_i / thread_count_m;\n                const auto tid_m = blk_i % thread_count_m;\n\n                const size_t m_start = tid_m * gemm_m_stride;\n                const size_t m_count = std::min(gemm_m - m_start, (size_t) gemm_m_stride);\n\n                const size_t n_start = tid_n * gemm_n_stride;\n                const size_t n_count = std::min(gemm_n - n_start, (size_t) gemm_n_stride);\n\n                void * per_gemm_ws = reinterpret_cast<std::byte *>(ws) + gemm_i * per_gemm_workspace_stride;\n\n                sqnbitgemm_spacemit_ime_i8i4(QK4_0, gemm_k, data, per_gemm_ws, m_start, m_count, n_start, n_count);\n            }\n        }\n    }\n\n    int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n        GGML_LOG_DEBUG(\"%s: repack tensor %s with %s_%dx%d\\n\", __func__, t->name, ggml_type_name(t->type),\n                       (int) NB_COLS, (int) INTER_SIZE);\n        return ggml::cpu::riscv64_spacemit::repack<BLOC_TYPE, INTER_SIZE, NB_COLS>(t, data, data_size);\n    }\n};\n\nclass tensor_traits_common : public tensor_traits_base {\n    bool work_size(int /* n_threads */, const struct ggml_tensor * op, size_t & size) override {\n        switch (op->op) {\n            case GGML_OP_NORM:\n            case GGML_OP_RMS_NORM:\n                size = 0;\n                return true;\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) override {\n        switch (op->op) {\n            case GGML_OP_NORM:\n                forward_norm_f32(params, op);\n                return true;\n            case GGML_OP_RMS_NORM:\n                forward_rms_norm_f32(params, op);\n                return true;\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    void forward_norm_f32(ggml_compute_params * params, ggml_tensor * op) {\n        const ggml_tensor * src0 = op->src[0];\n        ggml_tensor *       dst  = op;\n        GGML_ASSERT(ggml_are_same_shape(src0, dst));\n        GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n        const int ith = params->ith;\n        const int nth = params->nth;\n\n        GGML_TENSOR_UNARY_OP_LOCALS\n\n        float epsilon;\n        memcpy(&epsilon, dst->op_params, sizeof(float));\n\n        GGML_ASSERT(epsilon > 0.0f);\n\n        auto * input  = (float *) src0->data;\n        auto * output = (float *) dst->data;\n\n        const auto hidden_size     = ne00;\n        const auto task_count      = ne01 * ne02 * ne03;\n        const auto task_per_thread = (task_count + nth - 1) / nth;\n\n        const auto task_begin = ith * task_per_thread;\n        const auto task_end   = std::min((ith + 1) * task_per_thread, task_count);\n\n        for (auto task_idx = task_begin; task_idx < task_end; task_idx++) {\n            auto   offset  = task_idx * hidden_size;\n            auto * p_input = const_cast<float *>(input + offset);\n\n            auto *       p_output      = output + offset;\n            auto *       p_temp_output = p_output;\n            auto *       p_gamma_data  = (const float *) nullptr;\n            auto *       p_beta_data   = (const float *) nullptr;\n            size_t       gvl           = __riscv_vsetvlmax_e32m4();\n            vfloat32m4_t sum           = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n            vfloat32m4_t sum_sq        = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n            int64_t      length        = hidden_size;\n            while (length > 0) {\n                gvl                   = __riscv_vsetvl_e32m4(length);\n                // load data\n                vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl);\n\n                sum    = __riscv_vfadd_vv_f32m4(sum, src_data, gvl);\n                sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl);\n\n                __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl);\n\n                p_input += gvl;\n                p_temp_output += gvl;\n                length -= gvl;\n            }\n\n            gvl = __riscv_vsetvlmax_e32m1();\n\n            float        mean   = 0.f;\n            vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl);\n            vfloat32m1_t mean_v =\n                __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum, 0), __riscv_vget_v_f32m4_f32m1(sum, 1), gvl);\n            mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 2), gvl);\n            mean_v = __riscv_vfadd_vv_f32m1(mean_v, __riscv_vget_v_f32m4_f32m1(sum, 3), gvl);\n            mean_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_v, zero_v, gvl);\n            mean   = __riscv_vfmv_f_s_f32m1_f32(mean_v);\n            mean /= hidden_size;\n\n            vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0),\n                                                                __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl);\n            mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl);\n            mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl);\n            mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl);\n\n            float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v);\n            mean_square /= hidden_size;\n            mean_square = sqrt(mean_square - mean * mean + epsilon);\n\n            mean_square   = 1.0f / mean_square;\n            length        = hidden_size;\n            p_temp_output = p_output;\n\n            if (p_gamma_data == nullptr && p_beta_data == nullptr) {\n                while (length > 0) {\n                    gvl                   = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    src_data              = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n                    src_data              = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    length -= gvl;\n                }\n            } else if (p_beta_data == nullptr) {\n                while (length > 0) {\n                    gvl                       = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data     = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n                    src_data                  = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n                    src_data                  = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    src_data                  = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    p_gamma_data += gvl;\n                    length -= gvl;\n                }\n            } else if (p_gamma_data != nullptr) {\n                while (length > 0) {\n                    gvl                       = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data     = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n                    src_data                  = __riscv_vfsub_vf_f32m4(src_data, mean, gvl);\n                    src_data                  = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    src_data                  = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n                    vfloat32m4_t beta_data_v  = __riscv_vle32_v_f32m4(p_beta_data, gvl);\n                    src_data                  = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl);\n                    p_beta_data += gvl;\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    p_gamma_data += gvl;\n                    length -= gvl;\n                }\n            }\n        }\n    }\n\n    void forward_rms_norm_f32(ggml_compute_params * params, ggml_tensor * op) {\n        const ggml_tensor * src0 = op->src[0];\n        ggml_tensor *       dst  = op;\n        GGML_ASSERT(ggml_are_same_shape(src0, dst));\n        GGML_ASSERT(src0->nb[0] == sizeof(float));\n\n        const int ith = params->ith;\n        const int nth = params->nth;\n\n        GGML_TENSOR_UNARY_OP_LOCALS\n\n        float epsilon;\n        memcpy(&epsilon, dst->op_params, sizeof(float));\n\n        GGML_ASSERT(epsilon > 0.0f);\n\n        auto * input  = (float *) src0->data;\n        auto * output = (float *) dst->data;\n\n        const auto hidden_size     = ne00;\n        const auto task_count      = ne01 * ne02 * ne03;\n        const auto task_per_thread = (task_count + nth - 1) / nth;\n\n        const auto task_begin = ith * task_per_thread;\n        const auto task_end   = std::min((ith + 1) * task_per_thread, task_count);\n\n        for (auto task_idx = task_begin; task_idx < task_end; task_idx++) {\n            auto   offset        = task_idx * hidden_size;\n            auto * p_input       = const_cast<float *>(input + offset);\n            auto * p_output      = output + offset;\n            auto * p_temp_output = p_output;\n            auto * p_gamma_data  = (const float *) nullptr;\n            auto * p_beta_data   = (const float *) nullptr;\n\n            size_t       gvl    = __riscv_vsetvlmax_e32m4();\n            // vfloat32m4_t sum = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n            vfloat32m4_t sum_sq = __riscv_vfmv_v_f_f32m4(0.f, gvl);\n            int64_t      length = hidden_size;\n            while (length > 0) {\n                gvl                   = __riscv_vsetvl_e32m4(length);\n                // load data\n                vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_input, gvl);\n\n                sum_sq = __riscv_vfmacc_vv_f32m4(sum_sq, src_data, src_data, gvl);\n\n                __riscv_vse32_v_f32m4(p_temp_output, src_data, gvl);\n\n                p_input += gvl;\n                p_temp_output += gvl;\n                length -= gvl;\n            }\n\n            gvl = __riscv_vsetvlmax_e32m1();\n\n            // float mean = 0.f;\n            vfloat32m1_t zero_v = __riscv_vfmv_v_f_f32m1(0.f, gvl);\n\n            vfloat32m1_t mean_square_v = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m4_f32m1(sum_sq, 0),\n                                                                __riscv_vget_v_f32m4_f32m1(sum_sq, 1), gvl);\n            mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 2), gvl);\n            mean_square_v = __riscv_vfadd_vv_f32m1(mean_square_v, __riscv_vget_v_f32m4_f32m1(sum_sq, 3), gvl);\n            mean_square_v = __riscv_vfredusum_vs_f32m1_f32m1(mean_square_v, zero_v, gvl);\n\n            float mean_square = __riscv_vfmv_f_s_f32m1_f32(mean_square_v);\n            mean_square /= hidden_size;\n\n            mean_square = sqrt(mean_square + epsilon);\n\n            mean_square   = 1.0f / mean_square;\n            length        = hidden_size;\n            p_temp_output = p_output;\n\n            if (p_gamma_data == nullptr && p_beta_data == nullptr) {\n                while (length > 0) {\n                    gvl                   = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    src_data              = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    length -= gvl;\n                }\n            } else if (p_beta_data == nullptr) {\n                while (length > 0) {\n                    gvl                       = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data     = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n                    src_data                  = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    src_data                  = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    p_gamma_data += gvl;\n                    length -= gvl;\n                }\n            } else if (p_gamma_data != nullptr) {\n                while (length > 0) {\n                    gvl                       = __riscv_vsetvl_e32m4(length);\n                    vfloat32m4_t src_data     = __riscv_vle32_v_f32m4(p_temp_output, gvl);\n                    vfloat32m4_t gamma_data_v = __riscv_vle32_v_f32m4(p_gamma_data, gvl);\n                    src_data                  = __riscv_vfmul_vf_f32m4(src_data, mean_square, gvl);\n                    src_data                  = __riscv_vfmul_vv_f32m4(src_data, gamma_data_v, gvl);\n                    vfloat32m4_t beta_data_v  = __riscv_vle32_v_f32m4(p_beta_data, gvl);\n                    src_data                  = __riscv_vfadd_vv_f32m4(src_data, beta_data_v, gvl);\n                    p_beta_data += gvl;\n                    __riscv_vse32_v_f32m4(p_output, src_data, gvl);\n                    p_temp_output += gvl;\n                    p_output += gvl;\n                    p_gamma_data += gvl;\n                    length -= gvl;\n                }\n            }\n        }\n    }\n\n    int repack(struct ggml_tensor * t, const void * data, size_t data_size) override {\n        memcpy(t->data, data, data_size);\n        return 0;\n    }\n};\n\nstatic const tensor_traits<block_q4_0, 8, 16> q4_0_16x8_q8_0;\nstatic const tensor_traits<block_q4_1, 8, 16> q4_1_16x8_q8_0;\nstatic const tensor_traits<block_q4_K, 8, 16> q4_k_16x8_q8_0;\nstatic const tensor_traits_common             rvv_impl;\n\n}  // namespace ggml::cpu::riscv64_spacemit\n\nstatic const ggml::cpu::tensor_traits * ggml_riscv64_spacemit_get_optimal_repack_type(const struct ggml_tensor * cur) {\n    if (cur->type == GGML_TYPE_Q4_0) {\n        if (cur->ne[1] % 16 == 0) {\n            return &ggml::cpu::riscv64_spacemit::q4_0_16x8_q8_0;\n        }\n    } else if (cur->type == GGML_TYPE_Q4_1) {\n        if (cur->ne[1] % 16 == 0) {\n            return &ggml::cpu::riscv64_spacemit::q4_1_16x8_q8_0;\n        }\n    } else if (cur->type == GGML_TYPE_Q4_K) {\n        if (cur->ne[1] % 16 == 0) {\n            return &ggml::cpu::riscv64_spacemit::q4_k_16x8_q8_0;\n        }\n    } else if (cur->type == GGML_TYPE_F32) {\n        return &ggml::cpu::riscv64_spacemit::rvv_impl;\n    }\n\n    return nullptr;\n}\n\nstatic enum ggml_status ggml_backend_riscv64_spacemit_buffer_init_tensor(ggml_backend_buffer_t buffer,\n                                                                         struct ggml_tensor *  tensor) {\n    tensor->extra =\n        (void *) const_cast<ggml::cpu::tensor_traits *>(ggml_riscv64_spacemit_get_optimal_repack_type(tensor));\n\n    GGML_UNUSED(buffer);\n\n    return GGML_STATUS_SUCCESS;\n}\n\nstatic void ggml_backend_riscv64_spacemit_buffer_set_tensor(ggml_backend_buffer_t buffer,\n                                                            struct ggml_tensor *  tensor,\n                                                            const void *          data,\n                                                            size_t                offset,\n                                                            size_t                size) {\n    GGML_ASSERT(offset == 0);\n    GGML_ASSERT(size == ggml_nbytes(tensor));\n\n    auto tensor_traits = (ggml::cpu::riscv64_spacemit::tensor_traits_base *) tensor->extra;\n    if (tensor_traits) {\n        auto OK = tensor_traits->repack(tensor, data, size);\n        GGML_ASSERT(OK == 0);\n    }\n\n    GGML_UNUSED(buffer);\n}\n\nstatic const char * ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name(ggml_backend_buffer_type_t buft) {\n    return \"CPU_RISCV64_SPACEMIT\";\n\n    GGML_UNUSED(buft);\n}\n\nstatic ggml_backend_buffer_t ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,\n                                                                                        size_t size) {\n    ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);\n\n    if (buffer == nullptr) {\n        return nullptr;\n    }\n\n    buffer->buft              = buft;\n    buffer->iface.init_tensor = ggml_backend_riscv64_spacemit_buffer_init_tensor;\n    buffer->iface.set_tensor  = ggml_backend_riscv64_spacemit_buffer_set_tensor;\n    buffer->iface.get_tensor  = nullptr;\n    buffer->iface.cpy_tensor  = nullptr;\n    return buffer;\n}\n\nstatic size_t ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {\n    return 64;\n\n    GGML_UNUSED(buft);\n}\n\nstatic size_t ggml_backend_cpu_riscv64_spacemit_nbytes(ggml_backend_buffer_type_t buft,\n                                                       const struct ggml_tensor * tensor) {\n    for (int i = 0; i < GGML_MAX_DIMS; ++i) {\n        if (tensor->ne[i] <= 0) {\n            return 0;\n        }\n    }\n\n    size_t       nbytes;\n    const size_t blck_size = ggml_blck_size(tensor->type);\n    if (blck_size == 1) {\n        nbytes = ggml_type_size(tensor->type);\n        for (int i = 0; i < GGML_MAX_DIMS; ++i) {\n            nbytes += (tensor->ne[i] - 1) * tensor->nb[i];\n        }\n    } else {\n        nbytes = tensor->ne[0] * tensor->nb[0] / blck_size;\n        if (tensor->type == GGML_TYPE_Q4_K) {\n            GGML_ASSERT(nbytes % sizeof(block_q4_K) == 0);\n            nbytes = (nbytes / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8;\n            for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n                nbytes += (tensor->ne[i] - 1) * (tensor->nb[i] / sizeof(block_q4_K)) * sizeof(block_q4_1) * 8;\n            }\n        } else {\n            for (int i = 1; i < GGML_MAX_DIMS; ++i) {\n                nbytes += (tensor->ne[i] - 1) * tensor->nb[i];\n            }\n        }\n    }\n\n    GGML_UNUSED(buft);\n    return nbytes;\n}\n\nnamespace ggml::cpu::riscv64_spacemit {\n\nclass extra_buffer_type : ggml::cpu::extra_buffer_type {\n    bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                if (op->src[0]->buffer && (ggml_n_dims(op->src[0]) == 2) &&\n                    op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type() &&\n                    ggml_riscv64_spacemit_get_optimal_repack_type(op->src[0])) {\n                    if (op->src[1]->buffer && !ggml_backend_buft_is_host(op->src[1]->buffer->buft)) {\n                        return false;\n                    }\n                    if (op->src[1]->type == GGML_TYPE_F32) {\n                        return true;\n                    }\n                }\n                break;\n            case GGML_OP_NORM:\n            case GGML_OP_RMS_NORM:\n                if (op->src[0]->type == GGML_TYPE_F32) {\n                    return true;\n                }\n                break;\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n        return false;\n    }\n\n    ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {\n        switch (op->op) {\n            case GGML_OP_MUL_MAT:\n                if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_riscv64_spacemit_buffer_type()) {\n                    return (ggml::cpu::tensor_traits *) op->src[0]->extra;\n                }\n                break;\n            case GGML_OP_NORM:\n            case GGML_OP_RMS_NORM:\n                return (ggml::cpu::tensor_traits *) (&ggml::cpu::riscv64_spacemit::rvv_impl);\n            default:\n                // GGML_ABORT(\"fatal error\");\n                break;\n        }\n\n        return nullptr;\n    }\n};\n\n}  // namespace ggml::cpu::riscv64_spacemit\n\nggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void) {\n    static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_riscv64_spacemit = {\n  /* .iface    = */\n        {\n         /* .get_name         = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_name,\n         /* .alloc_buffer     = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_alloc_buffer,\n         /* .get_alignment    = */ ggml_backend_cpu_riscv64_spacemit_buffer_type_get_alignment,\n         /* .get_max_size     = */ nullptr,\n         /* .get_alloc_size   = */ ggml_backend_cpu_riscv64_spacemit_nbytes,\n         /* .is_host          = */ nullptr,\n         },\n /* .device  = */\n        ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),\n /* .context = */\n        new ggml::cpu::riscv64_spacemit::extra_buffer_type(),\n    };\n\n    return &ggml_backend_cpu_buffer_type_riscv64_spacemit;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/spacemit/ime.h",
    "content": "#pragma once\n\n#include \"ggml-alloc.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nggml_backend_buffer_type_t ggml_backend_cpu_riscv64_spacemit_buffer_type(void);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp",
    "content": "#include \"ggml.h\"\n#include \"ime_kernels.h\"\n\n#include <algorithm>\n#include <cmath>\n\n// clang-format off\n#if defined(__GNUC__)\n#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n#pragma GCC diagnostic ignored \"-Wcast-qual\"\n#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n#endif\n// clang-format on\nnamespace sqnbitgemm_spacemit_ime {\n\n#define QUANTIZEM4ROW_KERNEL                           \\\n    \"vmv.s.x            v16, zero                \\n\\t\" \\\n    \"vfabs.v            v8, v0                   \\n\\t\" \\\n    \"vfredmax.vs        v16, v8, v16             \\n\\t\" \\\n    \"vfmv.f.s           f10, v16                 \\n\\t\" \\\n    \"fmul.s             f10, f10, %[RMAXREC]     \\n\\t\" \\\n    \"fsw                f10, (a1)                \\n\\t\" \\\n    \"fdiv.s             f11, %[FONE], f10        \\n\\t\" \\\n    \"vfmul.vf           v16, v0, f11             \\n\\t\" \\\n    \"vfcvt.x.f.v        v16, v16                 \\n\\t\" \\\n    \"vsetvli            t0, zero, e16, mf2       \\n\\t\" \\\n    \"vnclip.wx          v16, v16, zero           \\n\\t\" \\\n    \"vnclip.wx          v17, v17, zero           \\n\\t\" \\\n    \"vnclip.wx          v18, v18, zero           \\n\\t\" \\\n    \"vnclip.wx          v19, v19, zero           \\n\\t\" \\\n    \"vnclip.wx          v20, v20, zero           \\n\\t\" \\\n    \"vnclip.wx          v21, v21, zero           \\n\\t\" \\\n    \"vnclip.wx          v22, v22, zero           \\n\\t\" \\\n    \"vnclip.wx          v23, v23, zero           \\n\\t\" \\\n    \"vsetvli            t0, zero, e8, mf4        \\n\\t\" \\\n    \"vnclip.wx          v24, v16, zero           \\n\\t\" \\\n    \"vnclip.wx          v25, v17, zero           \\n\\t\" \\\n    \"vnclip.wx          v26, v18, zero           \\n\\t\" \\\n    \"vnclip.wx          v27, v19, zero           \\n\\t\" \\\n    \"vnclip.wx          v28, v20, zero           \\n\\t\" \\\n    \"vnclip.wx          v29, v21, zero           \\n\\t\" \\\n    \"vnclip.wx          v30, v22, zero           \\n\\t\" \\\n    \"vnclip.wx          v31, v23, zero           \\n\\t\"\n\n#define QUANTIZEM4ROW_STORE                            \\\n    \"addi               t1, %[BlkLen], 0         \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v24, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v25, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v26, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v27, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v28, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v29, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v30, (s1)                \\n\\t\" \\\n    \"addi               s1, s1, 32               \\n\\t\" \\\n    \"sub                t1, t1, t0               \\n\\t\" \\\n    \"vsetvli            t0, t1, e8, mf4          \\n\\t\" \\\n    \"vse8.v             v31, (s1)                \\n\\t\"\n\nnamespace ime1 {\nvoid quantize_a_4row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) {\n    constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1);\n    const float     fone                 = 1.0f;\n\n    if (BlkLen == 16 || BlkLen == 32 || BlkLen == 64) {\n        for (size_t row_index = 0; row_index < 4; ++row_index) {\n            const float * SRC = A + row_index * CountK;\n            std::byte *   DST = QuantA + row_index * sizeof(float);\n\n            const size_t offset = (4 - row_index) * 4 + row_index * 8;\n            const size_t stride = 4 * (sizeof(float) + BlkLen);\n            __asm__ volatile(\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"addi               t2, %[CountK], 0         \\n\\t\"\n                \"addi               a1, %[DST], 0            \\n\\t\"\n                \"blt                t2, %[BlkLen], TAIL%=    \\n\\t\"\n\n                \"LOOP%=:                                     \\n\\t\"\n                \"vsetvli            t0, %[BlkLen], e32, m8   \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"sub                t2, t2, t0               \\n\\t\"\n                \"slli               t1, t0, 2                \\n\\t\"\n                \"add                %[SRC], %[SRC], t1       \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n\n                QUANTIZEM4ROW_KERNEL QUANTIZEM4ROW_STORE\n\n                \"add                a1, a1, %[STRIDE]        \\n\\t\"\n                \"bge                t2, %[BlkLen], LOOP%=    \\n\\t\"\n\n                \"TAIL%=:                                     \\n\\t\"\n                \"blez               t2, QUIT%=               \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vxor.vv            v16, v16, v16            \\n\\t\"\n                \"vxor.vv            v24, v24, v24            \\n\\t\"\n                \"vsetvli            t0, t2, e32, m8          \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n\n                QUANTIZEM4ROW_KERNEL\n\n                \"addi               t3, %[BlkLen], 0         \\n\\t\"\n                \"addi               s2, s1, 0                \\n\\t\"\n                \"vsetvli            t0, zero, e8, mf4        \\n\\t\"\n                \"vxor.vv            v8, v8, v8               \\n\\t\"\n                \"SET_ZERO%=:                                 \\n\\t\"\n                \"vse8.v             v8, (s2)                 \\n\\t\"\n                \"addi               s2, s2, 32               \\n\\t\"\n                \"addi               t3, t3, -8               \\n\\t\"\n                \"bnez               t3, SET_ZERO%=           \\n\\t\"\n\n                QUANTIZEM4ROW_STORE\n\n                \"QUIT%=:                                     \\n\\t\"\n                : [SRC] \"+r\"(SRC)\n                : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n                  [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n                : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n        }\n    } else if (BlkLen == 128) {\n        for (size_t row_index = 0; row_index < 4; ++row_index) {\n            const float * SRC = A + row_index * CountK;\n            std::byte *   DST = QuantA + row_index * sizeof(float);\n\n            const size_t offset = (4 - row_index) * 4 + row_index * 8;\n            const size_t stride = 4 * (sizeof(float) + BlkLen);\n            __asm__ volatile(\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"li                 t6, 32                   \\n\\t\"\n                \"addi               t2, %[CountK], 0         \\n\\t\"\n                \"addi               a1, %[DST], 0            \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n                \"blt                t2, %[BlkLen], TAIL%=    \\n\\t\"\n\n                \"LOOP%=:                                     \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v8, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"addi               t2, t2, -128             \\n\\t\"\n\n                \"QUANTIZE%=:                                 \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n                \"vfabs.v            v16, v0                  \\n\\t\"\n                \"vfabs.v            v24, v8                  \\n\\t\"\n                \"vfmax.vv           v16, v24, v16            \\n\\t\"\n                \"vfredmax.vs        v24, v16, v24            \\n\\t\"\n                \"vfmv.f.s           f10, v24                 \\n\\t\"\n                \"fmul.s             f10, f10, %[RMAXREC]     \\n\\t\"\n                \"fsw                f10, (a1)                \\n\\t\"\n                \"fdiv.s             f11, %[FONE], f10        \\n\\t\"\n                \"vfmul.vf           v16, v0, f11             \\n\\t\"\n                \"vfmul.vf           v24, v8, f11             \\n\\t\"\n                \"vfcvt.x.f.v        v16, v16                 \\n\\t\"\n                \"vfcvt.x.f.v        v24, v24                 \\n\\t\"\n                \"vsetvli            t0, zero, e16, m4        \\n\\t\"\n                \"vnclip.wx          v16, v16, zero           \\n\\t\"\n                \"vnclip.wx          v20, v24, zero           \\n\\t\"\n                \"vsetvli            t0, zero, e8, m4         \\n\\t\"\n                \"vnclip.wx          v16, v16, zero           \\n\\t\"\n                \"vsetvli            t0, zero, e64, m4        \\n\\t\"\n                \"vsse64.v           v16, (s1), t6            \\n\\t\"\n                \"add                a1, a1, %[STRIDE]        \\n\\t\"\n                \"bge                t2, %[BlkLen], LOOP%=    \\n\\t\"\n\n                \"TAIL%=:                                     \\n\\t\"\n                \"blez               t2, QUIT%=               \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vxor.vv             v0, v0, v0              \\n\\t\"\n                \"vxor.vv             v8, v8, v8              \\n\\t\"\n                \"vxor.vv             v16, v16, v16           \\n\\t\"\n                \"vxor.vv             v24, v24, v24           \\n\\t\"\n                \"vsetvli            t0, t2, e32, m8          \\n\\t\"\n                \"sub                t2, t2, t0               \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vsetvli            t0, t2, e32, m8          \\n\\t\"\n                \"vle32.v            v8, (%[SRC])             \\n\\t\"\n                \"sub                t2, t2, t2               \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"jal                x0, QUANTIZE%=           \\n\\t\"\n\n                \"QUIT%=:                                     \\n\\t\"\n                : [SRC] \"+r\"(SRC)\n                : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n                  [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n                : \"cc\", \"t0\", \"t1\", \"t2\", \"t6\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n        }\n    } else if (BlkLen == 256) {\n        for (size_t row_index = 0; row_index < 4; ++row_index) {\n            const float * SRC    = A + row_index * CountK;\n            std::byte *   DST    = QuantA + row_index * sizeof(float);\n            const size_t  offset = (4 - row_index) * 4 + row_index * 8;\n            const size_t  stride = 4 * (sizeof(float) + BlkLen);\n            __asm__ volatile(\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"li                 t6, 32                   \\n\\t\"\n                \"addi               t2, %[CountK], 0         \\n\\t\"\n                \"addi               a1, %[DST], 0            \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n                \"blt                t2, %[BlkLen], TAIL%=    \\n\\t\"\n\n                \"LOOP%=:                                     \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v8, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v16, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v24, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], -768     \\n\\t\"\n                \"addi               t2, t2, -256             \\n\\t\"\n                \"vfabs.v            v0, v0                   \\n\\t\"\n                \"vfabs.v            v8, v8                   \\n\\t\"\n                \"vfabs.v            v16, v16                 \\n\\t\"\n                \"vfabs.v            v24, v24                 \\n\\t\"\n                \"vfmax.vv           v8, v0, v8               \\n\\t\"\n                \"vfmax.vv           v24, v24, v16            \\n\\t\"\n                \"vfmax.vv           v8, v8, v24              \\n\\t\"\n                \"vfredmax.vs        v24, v8, v24             \\n\\t\"\n                \"vfmv.f.s           f10, v24                 \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v8, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v16, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vle32.v            v24, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n\n                \"QUANTIZE%=:                                 \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n                \"fmul.s             f10, f10, %[RMAXREC]     \\n\\t\"\n                \"fsw                f10, (a1)                \\n\\t\"\n                \"fdiv.s             f11, %[FONE], f10        \\n\\t\"\n                \"vfmul.vf           v0, v0, f11              \\n\\t\"\n                \"vfmul.vf           v8, v8, f11              \\n\\t\"\n                \"vfmul.vf           v16, v16, f11            \\n\\t\"\n                \"vfmul.vf           v24, v24, f11            \\n\\t\"\n                \"vfcvt.x.f.v        v0, v0                   \\n\\t\"\n                \"vfcvt.x.f.v        v8, v8                   \\n\\t\"\n                \"vfcvt.x.f.v        v16, v16                 \\n\\t\"\n                \"vfcvt.x.f.v        v24, v24                 \\n\\t\"\n                \"vsetvli            t0, zero, e16, m4        \\n\\t\"\n                \"vnclip.wx          v0, v0, zero             \\n\\t\"\n                \"vnclip.wx          v4, v8, zero             \\n\\t\"\n                \"vnclip.wx          v8, v16, zero            \\n\\t\"\n                \"vnclip.wx          v12, v24, zero           \\n\\t\"\n                \"vsetvli            t0, zero, e8, m4         \\n\\t\"\n                \"vnclip.wx          v0, v0, zero             \\n\\t\"\n                \"vnclip.wx          v4, v8, zero             \\n\\t\"\n                \"vsetvli            t0, zero, e64, m8        \\n\\t\"\n                \"vsse64.v           v0, (s1), t6             \\n\\t\"\n                \"add                a1, a1, %[STRIDE]        \\n\\t\"\n                \"bge                t2, %[BlkLen], LOOP%=    \\n\\t\"\n\n                \"TAIL%=:                                     \\n\\t\"\n                \"blez               t2, QUIT%=               \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vxor.vv            v0, v0, v0               \\n\\t\"\n                \"vxor.vv            v8, v8, v8               \\n\\t\"\n                \"vxor.vv            v16, v16, v16            \\n\\t\"\n                \"vxor.vv            v24, v24, v24            \\n\\t\"\n                \"addi               t1, t2, 0                \\n\\t\"\n                \"vsetvli            t0, t1, e32, m8          \\n\\t\"\n                \"sub                t1, t1, t0               \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vsetvli            t0, t1, e32, m8          \\n\\t\"\n                \"sub                t1, t1, t0               \\n\\t\"\n                \"vle32.v            v8, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vsetvli            t0, t1, e32, m8          \\n\\t\"\n                \"sub                t1, t1, t0               \\n\\t\"\n                \"vle32.v            v16, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], 256      \\n\\t\"\n                \"vsetvli            t0, t1, e32, m8          \\n\\t\"\n                \"vle32.v            v24, (%[SRC])            \\n\\t\"\n                \"addi               %[SRC], %[SRC], -768     \\n\\t\"\n                \"vsetvli            t0, zero, e32, m8        \\n\\t\"\n                \"vfabs.v            v0, v0                   \\n\\t\"\n                \"vfabs.v            v8, v8                   \\n\\t\"\n                \"vfabs.v            v16, v16                 \\n\\t\"\n                \"vfabs.v            v24, v24                 \\n\\t\"\n                \"vfmax.vv           v8, v0, v8               \\n\\t\"\n                \"vfmax.vv           v24, v16, v24            \\n\\t\"\n                \"vfmax.vv           v8, v8, v24              \\n\\t\"\n                \"vfredmax.vs        v24, v8, v24             \\n\\t\"\n                \"vfmv.f.s           f10, v24                 \\n\\t\"\n                \"add                s1, a1, %[OFFSET]        \\n\\t\"\n                \"fmul.s             f10, f10, %[RMAXREC]     \\n\\t\"\n                \"fsw                f10, (a1)                \\n\\t\"\n                \"fdiv.s             f11, %[FONE], f10        \\n\\t\"\n                \"vsetvli            t0, zero, e64, m8        \\n\\t\"\n                \"vxor.vv            v0, v0, v0               \\n\\t\"\n                \"vsse64.v           v0, (s1), t6             \\n\\t\"\n\n                \"TAIL_LOOP%=:                                \\n\\t\"\n                \"vsetvli            t0, zero, e32, m4        \\n\\t\"\n                \"vxor.vv            v0, v0, v0               \\n\\t\"\n                \"vsetvli            t0, t2, e32, m1          \\n\\t\"\n                \"sub                t2, t2, t0               \\n\\t\"\n                \"vle32.v            v0, (%[SRC])             \\n\\t\"\n                \"addi               %[SRC], %[SRC], 32       \\n\\t\"\n                \"vfmul.vf           v1, v0, f11              \\n\\t\"\n                \"vfcvt.x.f.v        v2, v1                   \\n\\t\"\n                \"vsetvli            t0, zero, e16, mf2       \\n\\t\"\n                \"vnclip.wx          v3, v2, zero             \\n\\t\"\n                \"vsetvli            t0, zero, e8, mf4        \\n\\t\"\n                \"vnclip.wx          v3, v3, zero             \\n\\t\"\n                \"vse8.v             v3, (s1)                 \\n\\t\"\n                \"addi               s1, s1, 32               \\n\\t\"\n                \"bnez               t2, TAIL_LOOP%=          \\n\\t\"\n\n                \"QUIT%=:                                     \\n\\t\"\n                : [SRC] \"+r\"(SRC)\n                : [DST] \"r\"(DST), [BlkLen] \"r\"(BlkLen), [OFFSET] \"r\"(offset), [STRIDE] \"r\"(stride),\n                  [CountK] \"r\"(CountK), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n                : \"cc\", \"t0\", \"t1\", \"t2\", \"t6\", \"a1\", \"s1\", \"s2\", \"f10\", \"f11\");\n        }\n    }\n}\n\nvoid quantize_a_row_i8(size_t BlkLen, const float * A, size_t CountK, std::byte * QuantA) {\n    const float *   SRC                  = A;\n    std::byte *     DST                  = QuantA;\n    constexpr float range_max_reciprocal = 1.0f / ((1 << 7) - 1);\n    const float     fone                 = 1.0f;\n    std::byte *     QuantA_offset        = QuantA + CountK + 4 * ((CountK + BlkLen - 1) / BlkLen);\n    size_t          offset               = (CountK + BlkLen - 1) / BlkLen * BlkLen - CountK;\n\n    if (CountK <= BlkLen) {\n        float max_abs_A = 0.0f;\n        for (size_t k = 0; k < CountK; k++) {\n            max_abs_A = std::max(max_abs_A, fabsf(A[k]));\n        }\n        float scale_A = max_abs_A * range_max_reciprocal;\n\n        ((float *) QuantA)[0] = scale_A;\n\n        auto * QuantAData_offset = (int8_t *) (QuantA + sizeof(float));\n\n        for (size_t k = 0; k < CountK; k++) {\n            QuantAData_offset[k] =\n                (int8_t) std::clamp(roundf(A[k] / scale_A), (float) std::numeric_limits<int8_t>::lowest(),\n                                    (float) std::numeric_limits<int8_t>::max());\n        }\n        for (size_t k = CountK; k < BlkLen; k++) {\n            QuantAData_offset[k] = 0;\n        }\n\n        return;\n    }\n\n    if (BlkLen != 32 || BlkLen != 64 || BlkLen != 128) {\n        __asm__ volatile(\n            \"vsetvli      t0, zero, e8, m8        \\n\\t\"\n            \"vxor.vv      v24, v24, v24           \\n\\t\"\n            \"LOOP%=:                              \\n\\t\"\n            \"vsetvli      t0, %[CNT], e8, m8      \\n\\t\"\n            \"vse8.v       v24, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 128     \\n\\t\"\n            \"sub          %[CNT], %[CNT], t0      \\n\\t\"\n            \"bnez         %[CNT], LOOP%=          \\n\\t\"\n            : [DST] \"+r\"(QuantA_offset), [CNT] \"+r\"(offset)\n            :\n            : \"cc\", \"t0\");\n    }\n    if (BlkLen == 16) {\n        float buffer[64] = { 0.0f };\n        __asm__ volatile(\n            \"addi         t3, zero, 16*8          \\n\\t\"\n            \"addi         t2, zero, 16            \\n\\t\"\n            \"blt          %[K], t3, LOOP_K%=      \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_MAIN%=:                         \\n\\t\"\n            \"vsetvli      t1, zero, e32, m2       \\n\\t\"\n            \"addi         %[K], %[K], -128        \\n\\t\"\n            \"vle32.v      v0, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v2, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v4, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v6, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v8, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v10, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v12, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"vle32.v      v14, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"addi         a1, %[BUFFER], 0        \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vfabs.v      v18, v2                 \\n\\t\"\n            \"vfabs.v      v20, v4                 \\n\\t\"\n            \"vfabs.v      v22, v6                 \\n\\t\"\n            \"vfabs.v      v24, v8                 \\n\\t\"\n            \"vfabs.v      v26, v10                \\n\\t\"\n            \"vfabs.v      v28, v12                \\n\\t\"\n            \"vfabs.v      v30, v14                \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vfmax.vv     v18, v18, v19           \\n\\t\"\n            \"vfmax.vv     v20, v20, v21           \\n\\t\"\n            \"vfmax.vv     v22, v22, v23           \\n\\t\"\n            \"vfmax.vv     v24, v24, v25           \\n\\t\"\n            \"vfmax.vv     v26, v26, v27           \\n\\t\"\n            \"vfmax.vv     v28, v28, v29           \\n\\t\"\n            \"vfmax.vv     v30, v30, v31           \\n\\t\"\n            \"vse32.v      v16, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v18, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v20, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v22, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v24, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v26, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v28, (a1)               \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"vse32.v      v30, (a1)               \\n\\t\"\n            \"addi         a1, %[BUFFER], 0        \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f10, f3, f7             \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f10, %[FONE], f10       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f11, f3, f7             \\n\\t\"\n            \"fmul.s       f11, f11, %[RMAXREC]    \\n\\t\"\n            \"fsw          f11, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f11       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f12, f3, f7             \\n\\t\"\n            \"fmul.s       f12, f12, %[RMAXREC]    \\n\\t\"\n            \"fsw          f12, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f12, %[FONE], f12       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f13, f3, f7             \\n\\t\"\n            \"fmul.s       f13, f13, %[RMAXREC]    \\n\\t\"\n            \"fsw          f13, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f13, %[FONE], f13       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f14, f3, f7             \\n\\t\"\n            \"fmul.s       f14, f14, %[RMAXREC]    \\n\\t\"\n            \"fsw          f14, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f14, %[FONE], f14       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f15, f3, f7             \\n\\t\"\n            \"fmul.s       f15, f15, %[RMAXREC]    \\n\\t\"\n            \"fsw          f15, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f15, %[FONE], f15       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f16, f3, f7             \\n\\t\"\n            \"fmul.s       f16, f16, %[RMAXREC]    \\n\\t\"\n            \"fsw          f16, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"fdiv.s       f16, %[FONE], f16       \\n\\t\"\n            \"flw          f0, (a1)                \\n\\t\"\n            \"flw          f1, 4(a1)               \\n\\t\"\n            \"flw          f2, 8(a1)               \\n\\t\"\n            \"flw          f3, 12(a1)              \\n\\t\"\n            \"flw          f4, 16(a1)              \\n\\t\"\n            \"flw          f5, 20(a1)              \\n\\t\"\n            \"flw          f6, 24(a1)              \\n\\t\"\n            \"flw          f7, 28(a1)              \\n\\t\"\n            \"addi         a1, a1, 32              \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f17, f3, f7             \\n\\t\"\n            \"fmul.s       f17, f17, %[RMAXREC]    \\n\\t\"\n            \"fsw          f17, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], -136    \\n\\t\"\n            \"fdiv.s       f17, %[FONE], f17       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmul.vf     v16, v0, f10            \\n\\t\"\n            \"vfmul.vf     v18, v2, f11            \\n\\t\"\n            \"vfmul.vf     v20, v4, f12            \\n\\t\"\n            \"vfmul.vf     v22, v6, f13            \\n\\t\"\n            \"vfmul.vf     v24, v8, f14            \\n\\t\"\n            \"vfmul.vf     v26, v10, f15           \\n\\t\"\n            \"vfmul.vf     v28, v12, f16           \\n\\t\"\n            \"vfmul.vf     v30, v14, f17           \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vfcvt.x.f.v  v18, v18                \\n\\t\"\n            \"vfcvt.x.f.v  v20, v20                \\n\\t\"\n            \"vfcvt.x.f.v  v22, v22                \\n\\t\"\n            \"vfcvt.x.f.v  v24, v24                \\n\\t\"\n            \"vfcvt.x.f.v  v26, v26                \\n\\t\"\n            \"vfcvt.x.f.v  v28, v28                \\n\\t\"\n            \"vfcvt.x.f.v  v30, v30                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m1       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v18, v18, zero          \\n\\t\"\n            \"vnclip.wx    v20, v20, zero          \\n\\t\"\n            \"vnclip.wx    v22, v22, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vnclip.wx    v26, v26, zero          \\n\\t\"\n            \"vnclip.wx    v28, v28, zero          \\n\\t\"\n            \"vnclip.wx    v30, v30, zero          \\n\\t\"\n            \"vsetvli      t0, t1, e8, mf2         \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v18, v18, zero          \\n\\t\"\n            \"vnclip.wx    v20, v20, zero          \\n\\t\"\n            \"vnclip.wx    v22, v22, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vnclip.wx    v26, v26, zero          \\n\\t\"\n            \"vnclip.wx    v28, v28, zero          \\n\\t\"\n            \"vnclip.wx    v30, v30, zero          \\n\\t\"\n            \"vse8.v       v16, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v18, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v20, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v22, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v24, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v26, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v28, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 20      \\n\\t\"\n            \"vse8.v       v30, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 16      \\n\\t\"\n            \"bge          %[K], t3, LOOP_MAIN%=   \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_K%=:                            \\n\\t\"\n            \"vsetvli      t1, %[K], e32, m2       \\n\\t\"\n            \"vle32.v      v0, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 64      \\n\\t\"\n            \"sub          %[K], %[K], t1          \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vse32.v      v16, (%[BUFFER])        \\n\\t\"\n            \"flw          f0, (%[BUFFER])         \\n\\t\"\n            \"flw          f1, 4(%[BUFFER])        \\n\\t\"\n            \"flw          f2, 8(%[BUFFER])        \\n\\t\"\n            \"flw          f3, 12(%[BUFFER])       \\n\\t\"\n            \"flw          f4, 16(%[BUFFER])       \\n\\t\"\n            \"flw          f5, 20(%[BUFFER])       \\n\\t\"\n            \"flw          f6, 24(%[BUFFER])       \\n\\t\"\n            \"flw          f7, 28(%[BUFFER])       \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f10, f3, f7             \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 4       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmul.vf     v16, v0, f11            \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m1       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vsetvli      t0, t1, e8, mf2         \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vse8.v       v16, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 16      \\n\\t\"\n            \"bge          %[K], t2, LOOP_K%=      \\n\\t\"\n            \"TAIL%=:                              \\n\\t\"\n            \"blez         %[K], END%=             \\n\\t\"\n            \"vsetvli      t0, t3, e32, m2         \\n\\t\"\n            \"vxor.vv      v16, v16, v16           \\n\\t\"\n            \"jal          x0, LOOP_K%=            \\n\\t\"\n            \"END%=:                               \\n\\t\"\n            : [SRC] \"+r\"(SRC), [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n            : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [BUFFER] \"r\"(buffer)\n            : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"f0\", \"f1\", \"f2\", \"f3\", \"f4\", \"f5\", \"f6\", \"f7\", \"f10\", \"f11\", \"f12\",\n              \"f13\", \"f14\", \"f15\", \"f16\", \"f17\");\n    } else if (BlkLen == 32) {\n        __asm__ volatile(\n            \"addi         t3, zero, 32*4          \\n\\t\"\n            \"addi         t2, zero, 32            \\n\\t\"\n\n            \"addi         a1, %[SRC], 0           \\n\\t\"\n            \"addi         a2, %[SRC], 128         \\n\\t\"\n            \"addi         a3, %[SRC], 256         \\n\\t\"\n            \"addi         a4, %[SRC], 384         \\n\\t\"\n\n            \"addi         s1, %[DST], 0           \\n\\t\"\n            \"addi         s2, %[DST], 36          \\n\\t\"\n            \"addi         s3, %[DST], 72          \\n\\t\"\n            \"addi         s4, %[DST], 108         \\n\\t\"\n            \"blt          %[K], t3, LOOP_K%=      \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n\n            \"LOOP_MAIN%=:                         \\n\\t\"\n            \"vsetvli      t1, zero, e32, m4       \\n\\t\"\n            \"addi         %[K], %[K], -128        \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"addi         a1, a1, 512             \\n\\t\"\n            \"vle32.v      v4, (a2)                \\n\\t\"\n            \"addi         a2, a2, 512             \\n\\t\"\n            \"vle32.v      v8, (a3)                \\n\\t\"\n            \"addi         a3, a3, 512             \\n\\t\"\n            \"vle32.v      v12, (a4)               \\n\\t\"\n            \"addi         a4, a4, 512             \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vfabs.v      v20, v4                 \\n\\t\"\n            \"vfabs.v      v24, v8                 \\n\\t\"\n            \"vfabs.v      v28, v12                \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v16, v16, v18           \\n\\t\"\n            \"vfmax.vv     v20, v20, v22           \\n\\t\"\n            \"vfmax.vv     v24, v24, v26           \\n\\t\"\n            \"vfmax.vv     v28, v28, v30           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vfmax.vv     v20, v20, v21           \\n\\t\"\n            \"vfmax.vv     v24, v24, v25           \\n\\t\"\n            \"vfmax.vv     v28, v28, v29           \\n\\t\"\n\n            \"vfredmax.vs  v17, v16, v17           \\n\\t\"\n            \"vfredmax.vs  v21, v20, v21           \\n\\t\"\n            \"vfredmax.vs  v25, v24, v25           \\n\\t\"\n            \"vfredmax.vs  v29, v28, v29           \\n\\t\"\n            \"vfmv.f.s     f10,  v17               \\n\\t\"\n            \"vfmv.f.s     f11,  v21               \\n\\t\"\n            \"vfmv.f.s     f12,  v25               \\n\\t\"\n            \"vfmv.f.s     f13,  v29               \\n\\t\"\n\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fmul.s       f11, f11, %[RMAXREC]    \\n\\t\"\n            \"fmul.s       f12, f12, %[RMAXREC]    \\n\\t\"\n            \"fmul.s       f13, f13, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (s1)               \\n\\t\"\n            \"addi         s1, s1, 4               \\n\\t\"\n\n            \"fsw          f11, (s2)               \\n\\t\"\n            \"addi         s2, s2, 4               \\n\\t\"\n            \"fsw          f12, (s3)               \\n\\t\"\n            \"addi         s3, s3, 4               \\n\\t\"\n            \"fsw          f13, (s4)               \\n\\t\"\n            \"addi         s4, s4, 4               \\n\\t\"\n            \"fdiv.s       f10, %[FONE], f10       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f11       \\n\\t\"\n            \"fdiv.s       f12, %[FONE], f12       \\n\\t\"\n            \"fdiv.s       f13, %[FONE], f13       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmul.vf     v16, v0, f10            \\n\\t\"\n            \"vfmul.vf     v20, v4, f11            \\n\\t\"\n            \"vfmul.vf     v24, v8, f12            \\n\\t\"\n            \"vfmul.vf     v28, v12, f13           \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vfcvt.x.f.v  v20, v20                \\n\\t\"\n            \"vfcvt.x.f.v  v24, v24                \\n\\t\"\n            \"vfcvt.x.f.v  v28, v28                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m2       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v20, v20, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vnclip.wx    v28, v28, zero          \\n\\t\"\n            \"vsetvli      t0, t1, e8, m1          \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v20, v20, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vnclip.wx    v28, v28, zero          \\n\\t\"\n            \"vse8.v       v16, (s1)               \\n\\t\"\n            \"addi         s1, s1, 140             \\n\\t\"\n            \"vse8.v       v20, (s2)               \\n\\t\"\n            \"addi         s2, s2, 140             \\n\\t\"\n            \"vse8.v       v24, (s3)               \\n\\t\"\n            \"addi         s3, s3, 140             \\n\\t\"\n            \"vse8.v       v28, (s4)               \\n\\t\"\n            \"addi         s4, s4, 140             \\n\\t\"\n            \"bge          %[K], t3, LOOP_MAIN%=   \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_K%=:                            \\n\\t\"\n            \"vsetvli      t1, %[K], e32, m4       \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"addi         a1, a1, 128             \\n\\t\"\n            \"sub          %[K], %[K], t1          \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v16, v16, v18           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vfredmax.vs  v17, v16, v17           \\n\\t\"\n            \"vfmv.f.s     f10,  v17               \\n\\t\"\n\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (s1)               \\n\\t\"\n            \"addi         s1, s1, 4               \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmul.vf     v16, v0, f11            \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m2       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vse8.v       v16, (s1)               \\n\\t\"\n            \"addi         s1, s1, 32              \\n\\t\"\n            \"bge          %[K], t2, LOOP_K%=      \\n\\t\"\n            \"TAIL%=:                              \\n\\t\"\n            \"blez         %[K], END%=             \\n\\t\"\n            \"vsetvli      t0, t3, e32, m4         \\n\\t\"\n            \"vxor.vv      v0, v0, v0              \\n\\t\"\n            \"vxor.vv      v16, v16, v16           \\n\\t\"\n            \"jal          x0, LOOP_K%=            \\n\\t\"\n            \"END%=:                               \\n\\t\"\n            : [K] \"+r\"(CountK)\n            : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [SRC] \"r\"(SRC), [DST] \"r\"(DST)\n            : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"a3\", \"a4\", \"s1\", \"s2\", \"s3\", \"s4\", \"f10\", \"f11\", \"f12\", \"f13\");\n    } else if (BlkLen == 64) {\n        __asm__ volatile(\n            \"addi         t3, zero, 64*2          \\n\\t\"\n            \"addi         t2, zero, 64            \\n\\t\"\n            \"addi         a1, %[SRC], 0           \\n\\t\"\n            \"addi         a2, %[SRC], 256         \\n\\t\"\n            \"addi         s1, %[DST], 0           \\n\\t\"\n            \"addi         s2, %[DST], 68          \\n\\t\"\n            \"blt          %[K], t3, LOOP_K%=      \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_MAIN%=:                         \\n\\t\"\n            \"vsetvli      t1, zero, e32, m8       \\n\\t\"\n            \"addi         %[K], %[K], -128        \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"addi         a1, a1, 512             \\n\\t\"\n            \"vle32.v      v8, (a2)                \\n\\t\"\n            \"addi         a2, a2, 512             \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vfabs.v      v24, v8                 \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmax.vv     v16, v16, v20           \\n\\t\"\n            \"vfmax.vv     v24, v24, v28           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v16, v16, v18           \\n\\t\"\n            \"vfmax.vv     v24, v24, v26           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vfmax.vv     v24, v24, v25           \\n\\t\"\n            \"vfredmax.vs  v17, v16, v17           \\n\\t\"\n            \"vfredmax.vs  v25, v24, v25           \\n\\t\"\n            \"vfmv.f.s     f10,  v17               \\n\\t\"\n            \"vfmv.f.s     f11,  v25               \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fmul.s       f11, f11, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (s1)               \\n\\t\"\n            \"addi         s1, s1, 4               \\n\\t\"\n            \"fsw          f11, (s2)               \\n\\t\"\n            \"addi         s2, s2, 4               \\n\\t\"\n            \"fdiv.s       f10, %[FONE], f10       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f11       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vfmul.vf     v16, v0, f10            \\n\\t\"\n            \"vfmul.vf     v24, v8, f11            \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vfcvt.x.f.v  v24, v24                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m4       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vsetvli      t0, t1, e8, m2          \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v24, v24, zero          \\n\\t\"\n            \"vse8.v       v16, (s1)               \\n\\t\"\n            \"addi         s1, s1, 132             \\n\\t\"\n            \"vse8.v       v24, (s2)               \\n\\t\"\n            \"addi         s2, s2, 132             \\n\\t\"\n            \"bge          %[K], t3, LOOP_MAIN%=   \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_K%=:                            \\n\\t\"\n            \"vsetvli      t1, %[K], e32, m8       \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"addi         a1, a1, 256             \\n\\t\"\n            \"sub          %[K], %[K], t1          \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmax.vv     v16, v16, v20           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v16, v16, v18           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v16, v16, v17           \\n\\t\"\n            \"vfredmax.vs  v17, v16, v17           \\n\\t\"\n            \"vfmv.f.s     f10,  v17               \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (s1)               \\n\\t\"\n            \"addi         s1, s1, 4               \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vfmul.vf     v16, v0, f11            \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m4       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vsetvli      t0, zero, e8, m2        \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vse8.v       v16, (s1)               \\n\\t\"\n            \"addi         s1, s1, 64              \\n\\t\"\n            \"bge          %[K], t2, LOOP_K%=      \\n\\t\"\n            \"TAIL%=:                              \\n\\t\"\n            \"blez         %[K], END%=             \\n\\t\"\n            \"vsetvli      t0, t3, e32, m8         \\n\\t\"\n            \"vxor.vv      v0, v0, v0              \\n\\t\"\n            \"vxor.vv      v16, v16, v16           \\n\\t\"\n            \"jal          x0, LOOP_K%=            \\n\\t\"\n            \"END%=:                               \\n\\t\"\n            : [K] \"+r\"(CountK)\n            : [SRC] \"r\"(SRC), [DST] \"r\"(DST), [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal)\n            : \"cc\", \"t3\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"s1\", \"s2\", \"f10\", \"f11\");\n    } else if (BlkLen == 128) {\n        __asm__ volatile(\n            \"addi         t2, zero, 128           \\n\\t\"\n            \"addi         a1, %[SRC], 0           \\n\\t\"\n            \"addi         a2, %[SRC], 256         \\n\\t\"\n            \"blt          %[K], t2, TAIL%=        \\n\\t\"\n            \"LOOP_K%=:                            \\n\\t\"\n            \"vsetvli      t1, zero, e32, m8       \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"addi         a1, a1, 512             \\n\\t\"\n            \"vle32.v      v8, (a2)                \\n\\t\"\n            \"addi         a2, a2, 512             \\n\\t\"\n            \"sub          %[K], %[K], t2          \\n\\t\"\n            \"QUANT%=:                             \\n\\t\"\n            \"vfabs.v      v16, v0                 \\n\\t\"\n            \"vfabs.v      v24, v8                 \\n\\t\"\n            \"vfmax.vv     v24, v16, v24           \\n\\t\"\n            \"vsetvli      t1, zero, e32, m4       \\n\\t\"\n            \"vfmax.vv     v28, v24, v28           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v30, v28, v30           \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v30, v30, v31           \\n\\t\"\n            \"vfredmax.vs  v31, v30, v31           \\n\\t\"\n            \"vfmv.f.s     f10, v31                \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 4       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vfmul.vf     v16, v0, f11            \\n\\t\"\n            \"vfmul.vf     v24, v8, f11            \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vfcvt.x.f.v  v24, v24                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m4       \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vnclip.wx    v20, v24, zero          \\n\\t\"\n            \"vsetvli      t0, zero, e8, m4        \\n\\t\"\n            \"vnclip.wx    v16, v16, zero          \\n\\t\"\n            \"vse8.v       v16, (%[DST])           \\n\\t\"\n            \"addi         %[DST], %[DST], 128     \\n\\t\"\n            \"bge          %[K], t2, LOOP_K%=      \\n\\t\"\n            \"TAIL%=:                              \\n\\t\"\n            \"blez         %[K], END%=             \\n\\t\"\n            \"vsetvli      t1, zero, e32, m8       \\n\\t\"\n            \"vxor.vv      v0, v0, v0              \\n\\t\"\n            \"vxor.vv      v8, v8, v8              \\n\\t\"\n            \"vsetvli      t0, %[K], e32, m8       \\n\\t\"\n            \"vle32.v      v0, (a1)                \\n\\t\"\n            \"sub          %[K], %[K], t0          \\n\\t\"\n            \"vsetvli      t0, %[K], e32, m8       \\n\\t\"\n            \"vle32.v      v8, (a2)                \\n\\t\"\n            \"sub          %[K], %[K], t0          \\n\\t\"\n            \"vsetvli      t1, zero, e32, m8       \\n\\t\"\n            \"jal          x0, QUANT%=             \\n\\t\"\n            \"END%=:                               \\n\\t\"\n\n            : [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n            : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [SRC] \"r\"(SRC)\n            : \"cc\", \"t2\", \"t1\", \"t0\", \"a1\", \"a2\", \"f10\", \"f11\");\n    } else {\n        float  buffer[8] = { 0.0f };\n        size_t cnt       = BlkLen / 256;\n\n        __asm__ volatile(\n            \"slli         t3, %[BLK], 2           \\n\\t\"\n            \"blt       %[K], %[BLK], LOOP_TAIL%=  \\n\\t\"\n            \"LOOP_MAIN%=:                         \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vxor.vv      v31, v31, v31           \\n\\t\"\n            \"vse32.v      v31, (%[BUFFER])        \\n\\t\"\n            \"addi         t6, %[CNT], 0           \\n\\t\"\n            \"LOOP_CMP%=:                          \\n\\t\"\n            \"addi         t6, t6, -1              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vle32.v      v0, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v8, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v16, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v24, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vfabs.v      v0, v0                  \\n\\t\"\n            \"vfabs.v      v8, v8                  \\n\\t\"\n            \"vfabs.v      v16, v16                \\n\\t\"\n            \"vfabs.v      v24, v24                \\n\\t\"\n            \"vfmax.vv     v8, v0, v8              \\n\\t\"\n            \"vfmax.vv     v16, v16, v24           \\n\\t\"\n            \"vfmax.vv     v0, v0, v16             \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmax.vv     v0, v0, v4              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v0, v0, v2              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v0, v0, v1              \\n\\t\"\n            \"vle32.v      v30, (%[BUFFER])        \\n\\t\"\n            \"vfmax.vv     v31, v30,  v0           \\n\\t\"\n            \"vse32.v      v31, (%[BUFFER])        \\n\\t\"\n            \"bnez         t6, LOOP_CMP%=          \\n\\t\"\n            \"sub          %[SRC], %[SRC], t3      \\n\\t\"\n            \"addi         t6, %[CNT], 0           \\n\\t\"\n            \"flw          f0, (%[BUFFER])         \\n\\t\"\n            \"flw          f1, 4(%[BUFFER])        \\n\\t\"\n            \"flw          f2, 8(%[BUFFER])        \\n\\t\"\n            \"flw          f3, 12(%[BUFFER])       \\n\\t\"\n            \"flw          f4, 16(%[BUFFER])       \\n\\t\"\n            \"flw          f5, 20(%[BUFFER])       \\n\\t\"\n            \"flw          f6, 24(%[BUFFER])       \\n\\t\"\n            \"flw          f7, 28(%[BUFFER])       \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f10, f3, f7             \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10,  (%[DST])          \\n\\t\"\n            \"addi         %[DST], %[DST], 4       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"addi         t6,  %[CNT], 0          \\n\\t\"\n            \"LOOP_QUANT%=:                        \\n\\t\"\n            \"addi         t6, t6, -1              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vle32.v      v0, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v8, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v16, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vle32.v      v24, (%[SRC])           \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vfmul.vf     v0, v0, f11             \\n\\t\"\n            \"vfmul.vf     v8, v8, f11             \\n\\t\"\n            \"vfmul.vf     v16, v16, f11           \\n\\t\"\n            \"vfmul.vf     v24, v24, f11           \\n\\t\"\n            \"vfcvt.x.f.v  v0, v0                  \\n\\t\"\n            \"vfcvt.x.f.v  v8, v8                  \\n\\t\"\n            \"vfcvt.x.f.v  v16, v16                \\n\\t\"\n            \"vfcvt.x.f.v  v24, v24                \\n\\t\"\n            \"vsetvli      t0, zero, e16, m4       \\n\\t\"\n            \"vnclip.wx    v0, v0, zero            \\n\\t\"\n            \"vnclip.wx    v4, v8, zero            \\n\\t\"\n            \"vnclip.wx    v8, v16, zero           \\n\\t\"\n            \"vnclip.wx    v12, v24, zero          \\n\\t\"\n            \"vsetvli      t0, zero, e8, m4        \\n\\t\"\n            \"vnclip.wx    v0, v0, zero            \\n\\t\"\n            \"vnclip.wx    v4, v8, zero            \\n\\t\"\n            \"vse8.v       v0, (%[DST])            \\n\\t\"\n            \"addi         %[DST], %[DST], 128     \\n\\t\"\n            \"vse8.v       v4, (%[DST])            \\n\\t\"\n            \"addi         %[DST], %[DST], 128     \\n\\t\"\n            \"bnez         t6, LOOP_QUANT%=        \\n\\t\"\n            \"sub           %[K], %[K], %[BLK]     \\n\\t\"\n            \"bge        %[K], %[BLK], LOOP_MAIN%= \\n\\t\"\n            \"blez         %[K], END%=             \\n\\t\"\n            \"LOOP_TAIL%=:                         \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vxor.vv      v31, v31, v31           \\n\\t\"\n            \"vse32.v      v31, (%[BUFFER])        \\n\\t\"\n            \"addi         t6, %[K], 0             \\n\\t\"\n            \"addi         s1, %[SRC], 0           \\n\\t\"\n            \"TAIL_CMP%=:                          \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vxor.vv       v0, v0, v0             \\n\\t\"\n            \"vsetvli      t0, t6, e32, m8         \\n\\t\"\n            \"vle32.v      v0, (%[SRC])            \\n\\t\"\n            \"addi         %[SRC], %[SRC], 256     \\n\\t\"\n            \"sub          t6, t6, t0              \\n\\t\"\n            \"vfabs.v      v0, v0                  \\n\\t\"\n            \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n            \"vfmax.vv     v0, v0, v4              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m2       \\n\\t\"\n            \"vfmax.vv     v0, v0, v2              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m1       \\n\\t\"\n            \"vfmax.vv     v0, v0, v1              \\n\\t\"\n            \"vle32.v      v30, (%[BUFFER])        \\n\\t\"\n            \"vfmax.vv     v31, v30,  v0           \\n\\t\"\n            \"vse32.v      v31, (%[BUFFER])        \\n\\t\"\n            \"bnez         t6, TAIL_CMP%=          \\n\\t\"\n            \"addi         t6, %[K], 0             \\n\\t\"\n            \"flw          f0, (%[BUFFER])         \\n\\t\"\n            \"flw          f1, 4(%[BUFFER])        \\n\\t\"\n            \"flw          f2, 8(%[BUFFER])        \\n\\t\"\n            \"flw          f3, 12(%[BUFFER])       \\n\\t\"\n            \"flw          f4, 16(%[BUFFER])       \\n\\t\"\n            \"flw          f5, 20(%[BUFFER])       \\n\\t\"\n            \"flw          f6, 24(%[BUFFER])       \\n\\t\"\n            \"flw          f7, 28(%[BUFFER])       \\n\\t\"\n            \"fmax.s       f1, f0, f1              \\n\\t\"\n            \"fmax.s       f3, f2, f3              \\n\\t\"\n            \"fmax.s       f5, f4, f5              \\n\\t\"\n            \"fmax.s       f7, f6, f7              \\n\\t\"\n            \"fmax.s       f3, f1, f3              \\n\\t\"\n            \"fmax.s       f7, f5, f7              \\n\\t\"\n            \"fmax.s       f10, f3, f7             \\n\\t\"\n            \"fmul.s       f10, f10, %[RMAXREC]    \\n\\t\"\n            \"fsw          f10,  (%[DST])          \\n\\t\"\n            \"addi         %[DST], %[DST], 4       \\n\\t\"\n            \"fdiv.s       f11, %[FONE], f10       \\n\\t\"\n            \"addi         t6,  %[K], 0            \\n\\t\"\n            \"TAIL_QUANT%=:                        \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vxor.vv       v0, v0, v0             \\n\\t\"\n            \"vsetvli      t1, t6, e32, m8         \\n\\t\"\n            \"vle32.v      v0, (s1)                \\n\\t\"\n            \"addi         s1, s1, 256             \\n\\t\"\n            \"sub          t6, t6, t1              \\n\\t\"\n            \"vsetvli      t0, zero, e32, m8       \\n\\t\"\n            \"vfmul.vf     v0, v0, f11             \\n\\t\"\n            \"vfcvt.x.f.v  v0, v0                  \\n\\t\"\n            \"vsetvli      t0, zero, e16, m4       \\n\\t\"\n            \"vnclip.wx    v0, v0, zero            \\n\\t\"\n            \"vsetvli      t0, t1, e8, m2          \\n\\t\"\n            \"vnclip.wx    v0, v0, zero            \\n\\t\"\n            \"vse8.v       v0, (%[DST])            \\n\\t\"\n            \"addi         %[DST], %[DST], 64      \\n\\t\"\n            \"bnez         t6, TAIL_QUANT%=        \\n\\t\"\n            \"END%=:                               \\n\\t\"\n            : [SRC] \"+r\"(SRC), [DST] \"+r\"(DST), [K] \"+r\"(CountK)\n            : [FONE] \"f\"(fone), [RMAXREC] \"f\"(range_max_reciprocal), [BLK] \"r\"(BlkLen), [BUFFER] \"r\"(buffer),\n              [CNT] \"r\"(cnt)\n            : \"cc\", \"t1\", \"t0\", \"t6\", \"s1\", \"f0\", \"f1\", \"f2\", \"f3\", \"f4\", \"f5\", \"f6\");\n    }\n}\n\n}  // namespace ime1\n\nnamespace {\n#define SQ4BIT_KERNEL_COMP_1x8x2_4X8X4          \\\n    \"vmadot       v16, v14, v0            \\n\\t\" \\\n    \"vmadot       v18, v14, v1            \\n\\t\" \\\n    \"vmadot       v20, v14, v2            \\n\\t\" \\\n    \"vmadot       v22, v14, v3            \\n\\t\" \\\n    \"vmadot       v16, v15, v4            \\n\\t\" \\\n    \"vmadot       v18, v15, v5            \\n\\t\" \\\n    \"vmadot       v20, v15, v6            \\n\\t\" \\\n    \"vmadot       v22, v15, v7            \\n\\t\"\n\n#define SQ4BIT_KERNEL_ACC_1X4X4                 \\\n    \"vfcvt.f.x.v  v16,  v16               \\n\\t\" \\\n    \"vfcvt.f.x.v  v18,  v18               \\n\\t\" \\\n    \"vfcvt.f.x.v  v20,  v20               \\n\\t\" \\\n    \"vfcvt.f.x.v  v22,  v22               \\n\\t\" \\\n    \"addi         s2, s1, 16              \\n\\t\" \\\n    \"addi         s3, s1, 32              \\n\\t\" \\\n    \"addi         s4, s1, 48              \\n\\t\" \\\n    \"addi         s6, s5, 12              \\n\\t\" \\\n    \"vfmacc.vv    v28, v16, v24           \\n\\t\" \\\n    \"vfmacc.vv    v29, v18, v25           \\n\\t\" \\\n    \"vfmacc.vv    v30, v20, v26           \\n\\t\" \\\n    \"vfmacc.vv    v31, v22, v27           \\n\\t\"\n\n#define SQ4BIT_KERNEL_ACC_F16_1X4X4             \\\n    \"vfcvt.f.x.v  v16,  v16               \\n\\t\" \\\n    \"vfcvt.f.x.v  v18,  v18               \\n\\t\" \\\n    \"vfcvt.f.x.v  v20,  v20               \\n\\t\" \\\n    \"vfcvt.f.x.v  v22,  v22               \\n\\t\" \\\n    \"addi         s2, s1, 8               \\n\\t\" \\\n    \"addi         s3, s1, 16              \\n\\t\" \\\n    \"addi         s4, s1, 24              \\n\\t\" \\\n    \"addi         s6, s5, 12              \\n\\t\" \\\n    \"vfmacc.vv    v28, v16, v24           \\n\\t\" \\\n    \"vfmacc.vv    v29, v18, v25           \\n\\t\" \\\n    \"vfmacc.vv    v30, v20, v26           \\n\\t\" \\\n    \"vfmacc.vv    v31, v22, v27           \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4          \\\n    \"vle8.v       v4, (s1)                \\n\\t\" \\\n    \"addi         s1, s1, 128             \\n\\t\" \\\n    \"vle8.v       v5, (s2)                \\n\\t\" \\\n    \"addi         s2, s2, 128             \\n\\t\" \\\n    \"vle8.v       v6, (s3)                \\n\\t\" \\\n    \"addi         s3, s3, 128             \\n\\t\" \\\n    \"vle8.v       v7, (s4)                \\n\\t\" \\\n    \"addi         s4, s4, 128             \\n\\t\" \\\n    \"vsetvli      t0, zero, e8, mf4       \\n\\t\" \\\n    \"vle8.v       v14, (s5)               \\n\\t\" \\\n    \"addi         s5, s5, 16              \\n\\t\" \\\n    \"vle8.v       v15, (s6)               \\n\\t\" \\\n    \"addi         s6, s6, 16              \\n\\t\" \\\n    \"addi         t5, t5, -1              \\n\\t\" \\\n    \"vsetvli      t0, zero, e8, m1        \\n\\t\" \\\n    \"vand.vi      v0, v4, 15              \\n\\t\" \\\n    \"vand.vi      v1, v5, 15              \\n\\t\" \\\n    \"vand.vi      v2, v6, 15              \\n\\t\" \\\n    \"vand.vi      v3, v7, 15              \\n\\t\" \\\n    \"vsrl.vi      v4, v4, 4               \\n\\t\" \\\n    \"vsrl.vi      v5, v5, 4               \\n\\t\" \\\n    \"vsrl.vi      v6, v6, 4               \\n\\t\" \\\n    \"vsrl.vi      v7, v7, 4               \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_ZP_16X1              \\\n    \"vsetvli      t0, zero, e8, mf2       \\n\\t\" \\\n    \"vle8.v       v1, (s7)                \\n\\t\" \\\n    \"vsetvli      t0, zero, e8, m1        \\n\\t\" \\\n    \"vrgather.vv  v8, v1, v13             \\n\\t\" \\\n    \"vadd.vi      v13, v13, 4             \\n\\t\" \\\n    \"vrgather.vv  v9, v1, v13             \\n\\t\" \\\n    \"vadd.vi      v13, v13, 4             \\n\\t\" \\\n    \"vrgather.vv  v10, v1, v13            \\n\\t\" \\\n    \"vadd.vi      v13, v13, 4             \\n\\t\" \\\n    \"vrgather.vv  v11, v1, v13            \\n\\t\" \\\n    \"vadd.vi      v13, v13, -12           \\n\\t\"\n\n// using for M4Kernel\n#define LOAD_B_16x8x2                           \\\n    \"vsetvli      t0, zero, e8, m1        \\n\\t\" \\\n    \"vle8.v       v6, (s1)                \\n\\t\" \\\n    \"addi         s1, s1, 32*4            \\n\\t\" \\\n    \"vle8.v       v7, (s2)                \\n\\t\" \\\n    \"addi         s2, s2, 32*4            \\n\\t\" \\\n    \"vle8.v       v8, (s3)                \\n\\t\" \\\n    \"addi         s3, s3, 32*4            \\n\\t\" \\\n    \"vle8.v       v9, (s4)                \\n\\t\" \\\n    \"addi         s4, s4, 32*4            \\n\\t\" \\\n                                                \\\n    \"vand.vi      v2, v6, 15              \\n\\t\" \\\n    \"vand.vi      v3, v7, 15              \\n\\t\" \\\n    \"vand.vi      v4, v8, 15              \\n\\t\" \\\n    \"vand.vi      v5, v9, 15              \\n\\t\" \\\n                                                \\\n    \"vsrl.vi      v6, v6, 4               \\n\\t\" \\\n    \"vsrl.vi      v7, v7, 4               \\n\\t\" \\\n    \"vsrl.vi      v8, v8, 4               \\n\\t\" \\\n    \"vsrl.vi      v9, v9, 4               \\n\\t\"\n\n// [s2|s5, s3, s4, s6]\n#define LOAD_SCALE_4x16_FP16                    \\\n    \"addi         s2, s5, -8              \\n\\t\" \\\n    \"addi         s3, s5, 8               \\n\\t\" \\\n    \"addi         s4, s5, 16              \\n\\t\" \\\n    \"addi         s6, s5, 24              \\n\\t\" \\\n    \"li           t1, 0xf0                \\n\\t\" \\\n    \"vmv.s.x      v0, t1                  \\n\\t\" \\\n    \"vsetvli      t0, zero, e16, mf4      \\n\\t\" \\\n    \"vle16.v      v9, (s5)                \\n\\t\" \\\n    \"vle16.v      v11, (s3)               \\n\\t\" \\\n    \"vle16.v      v13, (s4)               \\n\\t\" \\\n    \"vle16.v      v15, (s6)               \\n\\t\" \\\n    \"vsetvli      t0, zero, e16, mf2      \\n\\t\" \\\n    \"vle16.v      v9, (s2), v0.t          \\n\\t\" \\\n    \"vle16.v      v11, (s5), v0.t         \\n\\t\" \\\n    \"vle16.v      v13, (s3), v0.t         \\n\\t\" \\\n    \"vle16.v      v15, (s4), v0.t         \\n\\t\" \\\n    \"vfwcvt.f.f.v v8, v9                  \\n\\t\" \\\n    \"vfwcvt.f.f.v v10, v11                \\n\\t\" \\\n    \"vfwcvt.f.f.v v12, v13                \\n\\t\" \\\n    \"vfwcvt.f.f.v v14, v15                \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n    \"vmv.v.v      v9, v8                  \\n\\t\" \\\n    \"vmv.v.v      v11, v10                \\n\\t\" \\\n    \"vmv.v.v      v13, v12                \\n\\t\" \\\n    \"vmv.v.v      v15, v14                \\n\\t\" \\\n    \"li           t1, 0xf0                \\n\\t\" \\\n    \"vmv.s.x      v0, t1                  \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, mf2      \\n\\t\" \\\n    \"vfmul.vf     v8, v8, f1              \\n\\t\" \\\n    \"vfmul.vf     v10, v10, f1            \\n\\t\" \\\n    \"vfmul.vf     v12, v12, f1            \\n\\t\" \\\n    \"vfmul.vf     v14, v14, f1            \\n\\t\" \\\n    \"vfmul.vf     v9, v9, f3              \\n\\t\" \\\n    \"vfmul.vf     v11, v11, f3            \\n\\t\" \\\n    \"vfmul.vf     v13, v13, f3            \\n\\t\" \\\n    \"vfmul.vf     v15, v15, f3            \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n    \"vfmul.vf     v8, v8, f2, v0.t        \\n\\t\" \\\n    \"vfmul.vf     v10, v10, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v12, v12, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v14, v14, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v9, v9, f4, v0.t        \\n\\t\" \\\n    \"vfmul.vf     v11, v11, f4, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v13, v13, f4, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v15, v15, f4, v0.t      \\n\\t\"\n\n// [s2|s5, s3, s4, s6]\n#define LOAD_SCALE_4x16                         \\\n    \"addi         s2, s5, -16             \\n\\t\" \\\n    \"addi         s3, s5, 16              \\n\\t\" \\\n    \"addi         s4, s5, 32              \\n\\t\" \\\n    \"addi         s6, s5, 48              \\n\\t\" \\\n    \"li           t1, 0xf0                \\n\\t\" \\\n    \"vmv.s.x      v0, t1                  \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, mf2      \\n\\t\" \\\n    \"vle32.v      v8, (s5)                \\n\\t\" \\\n    \"vle32.v      v10, (s3)               \\n\\t\" \\\n    \"vle32.v      v12, (s4)               \\n\\t\" \\\n    \"vle32.v      v14, (s6)               \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n    \"vle32.v      v8, (s2), v0.t          \\n\\t\" \\\n    \"vle32.v      v10, (s5), v0.t         \\n\\t\" \\\n    \"vle32.v      v12, (s3), v0.t         \\n\\t\" \\\n    \"vle32.v      v14, (s4), v0.t         \\n\\t\" \\\n    \"vmv.v.v      v9, v8                  \\n\\t\" \\\n    \"vmv.v.v      v11, v10                \\n\\t\" \\\n    \"vmv.v.v      v13, v12                \\n\\t\" \\\n    \"vmv.v.v      v15, v14                \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, mf2      \\n\\t\" \\\n    \"vfmul.vf     v8, v8, f1              \\n\\t\" \\\n    \"vfmul.vf     v10, v10, f1            \\n\\t\" \\\n    \"vfmul.vf     v12, v12, f1            \\n\\t\" \\\n    \"vfmul.vf     v14, v14, f1            \\n\\t\" \\\n    \"vfmul.vf     v9, v9, f3              \\n\\t\" \\\n    \"vfmul.vf     v11, v11, f3            \\n\\t\" \\\n    \"vfmul.vf     v13, v13, f3            \\n\\t\" \\\n    \"vfmul.vf     v15, v15, f3            \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n    \"vfmul.vf     v8, v8, f2, v0.t        \\n\\t\" \\\n    \"vfmul.vf     v10, v10, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v12, v12, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v14, v14, f2, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v9, v9, f4, v0.t        \\n\\t\" \\\n    \"vfmul.vf     v11, v11, f4, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v13, v13, f4, v0.t      \\n\\t\" \\\n    \"vfmul.vf     v15, v15, f4, v0.t      \\n\\t\"\n\n//[s1| BIAS, s2, s3, s4]\n#define LOAD_BIAS                               \\\n    \"vsetvli      t0, zero, e32, mf2      \\n\\t\" \\\n    \"li           t1, 0xf0                \\n\\t\" \\\n    \"vmv.s.x      v0, t1                  \\n\\t\" \\\n    \"addi         s1, %[BIAS], -16        \\n\\t\" \\\n    \"addi         s2, %[BIAS], 16         \\n\\t\" \\\n    \"addi         s3, %[BIAS], 32         \\n\\t\" \\\n    \"addi         s4, %[BIAS], 48         \\n\\t\" \\\n                                                \\\n    \"vle32.v      v24, (%[BIAS])          \\n\\t\" \\\n    \"vle32.v      v26, (s2)               \\n\\t\" \\\n    \"vle32.v      v28, (s3)               \\n\\t\" \\\n    \"vle32.v      v30, (s4)               \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n    \"vle32.v      v24, (s1), v0.t         \\n\\t\" \\\n    \"vle32.v      v26, (%[BIAS]), v0.t    \\n\\t\" \\\n    \"vle32.v      v28, (s2), v0.t         \\n\\t\" \\\n    \"vle32.v      v30, (s3), v0.t         \\n\\t\" \\\n    \"vmv.v.v      v25, v24                \\n\\t\" \\\n    \"vmv.v.v      v27, v26                \\n\\t\" \\\n    \"vmv.v.v      v29, v28                \\n\\t\" \\\n    \"vmv.v.v      v31, v30                \\n\\t\"\n\n#define SQ4BIT_KERNEL_COMP_4x16x16              \\\n    \"vmadot       v16, v10, v2            \\n\\t\" \\\n    \"vmadot       v18, v10, v3            \\n\\t\" \\\n    \"vmadot       v20, v10, v4            \\n\\t\" \\\n    \"vmadot       v22, v10, v5            \\n\\t\" \\\n    \"vmadot       v16, v11, v6            \\n\\t\" \\\n    \"vmadot       v18, v11, v7            \\n\\t\" \\\n    \"vmadot       v20, v11, v8            \\n\\t\" \\\n    \"vmadot       v22, v11, v9            \\n\\t\"\n\n#define SAVE_RESULT_4x16                        \\\n    \"addi         a1, %[C], 0             \\n\\t\" \\\n    \"add          a2, %[C], %[LDC]        \\n\\t\" \\\n    \"add          a3, a2, %[LDC]          \\n\\t\" \\\n    \"add          a4, a3, %[LDC]          \\n\\t\" \\\n    \"addi         a2, a2, -16             \\n\\t\" \\\n    \"addi         a4, a4, -16             \\n\\t\" \\\n    \"li           t1, 0xf0                \\n\\t\" \\\n    \"vmv.s.x      v0, t1                  \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, mf2      \\n\\t\" \\\n                                                \\\n    \"vse32.v      v24, (a1)               \\n\\t\" \\\n    \"addi         a1, a1, 16              \\n\\t\" \\\n    \"vse32.v      v25, (a3)               \\n\\t\" \\\n    \"addi         a3, a3, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v26, (a1)               \\n\\t\" \\\n    \"addi         a1, a1, 16              \\n\\t\" \\\n    \"vse32.v      v27, (a3)               \\n\\t\" \\\n    \"addi         a3, a3, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v28, (a1)               \\n\\t\" \\\n    \"addi         a1, a1, 16              \\n\\t\" \\\n    \"vse32.v      v29, (a3)               \\n\\t\" \\\n    \"addi         a3, a3, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v30, (a1)               \\n\\t\" \\\n    \"vse32.v      v31, (a3)               \\n\\t\" \\\n    \"vsetvli      t0, zero, e32, m1       \\n\\t\" \\\n                                                \\\n    \"vse32.v      v24, (a2), v0.t         \\n\\t\" \\\n    \"addi         a2, a2, 16              \\n\\t\" \\\n    \"vse32.v      v25, (a4), v0.t         \\n\\t\" \\\n    \"addi         a4, a4, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v26, (a2), v0.t         \\n\\t\" \\\n    \"addi         a2, a2, 16              \\n\\t\" \\\n    \"vse32.v      v27, (a4), v0.t         \\n\\t\" \\\n    \"addi         a4, a4, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v28, (a2), v0.t         \\n\\t\" \\\n    \"addi         a2, a2, 16              \\n\\t\" \\\n    \"vse32.v      v29, (a4), v0.t         \\n\\t\" \\\n    \"addi         a4, a4, 16              \\n\\t\" \\\n                                                \\\n    \"vse32.v      v30, (a2), v0.t         \\n\\t\" \\\n    \"vse32.v      v31, (a4), v0.t         \\n\\t\"\n\n#define SQ4BIT_KERNEL_LOAD_ZP_16X1_v2           \\\n    \"vsetvli      t0, zero, e8, mf2       \\n\\t\" \\\n    \"vle8.v       v11, (s6)               \\n\\t\" \\\n    \"vsetvli      t0, zero, e8, m1        \\n\\t\" \\\n    \"vrgather.vv  v12, v11, v1            \\n\\t\" \\\n    \"vadd.vi      v1, v1, 4               \\n\\t\" \\\n    \"vrgather.vv  v13, v11, v1            \\n\\t\" \\\n    \"vadd.vi      v1, v1, 4               \\n\\t\" \\\n    \"vrgather.vv  v14, v11, v1            \\n\\t\" \\\n    \"vadd.vi      v1, v1, 4               \\n\\t\" \\\n    \"vrgather.vv  v15, v11, v1            \\n\\t\" \\\n    \"vadd.vi      v1, v1, -12             \\n\\t\"\n\ntemplate <bool HasZeroPoint>\nvoid SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl(size_t            BlkLen,\n                                                const std::byte * QuantA,\n                                                const std::byte * QuantBData,\n                                                const float *     QuantBScale,\n                                                const std::byte * QuantBZeroPoint,\n                                                float *           C,\n                                                size_t            CountN,\n                                                size_t            BlockCountK,\n                                                const float *     Bias,\n                                                const size_t      ldc) {\n    GGML_UNUSED(QuantBScale);\n    GGML_UNUSED(QuantBZeroPoint);\n    size_t       LDC   = ldc * sizeof(float);\n    const size_t INNER = BlkLen / 16;\n    float        tmp[4 * 16];\n\n    if constexpr (HasZeroPoint) {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      NBLKS         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +           //\n                                        n * BlockCountK * BlkLen / 2 +       // b data\n                                        n * BlockCountK * sizeof(uint8_t) +  // zp\n                                        n * BlockCountK * sizeof(_Float16);    // scale\n            float * CPtr = C + n;\n            if (NBLKS < 16) {\n                CPtr = tmp;\n                LDC  = 16 * sizeof(float);\n            }\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                if (NBLKS < 16) {\n                    __asm__ volatile(\n                        \"vsetvli        t0, %[N], e32, m2     \\n\\t\"\n                        \"vle32.v        v0, (%[SRC])          \\n\\t\"\n                        \"vse32.v        v0, (%[DST])          \\n\\t\"\n                        :\n                        : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n                        : \"cc\", \"t0\");\n                    bias = tmp;\n                }\n                __asm__ volatile(LOAD_BIAS\n\n                                 \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n\n                                 \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                                 \"li                 s1, 24                      \\n\\t\"\n                                 \"vmv.v.i            v1, 3                       \\n\\t\"\n                                 \"vsetvli            t0, s1, e8, m1              \\n\\t\"\n                                 \"vmv.v.i            v1, 2                       \\n\\t\"\n                                 \"vsetvli            t0, zero, e8, mf2           \\n\\t\"\n                                 \"vmv.v.i            v1, 1                       \\n\\t\"\n                                 \"vsetvli            t0, zero, e8, mf4           \\n\\t\"\n                                 \"vmv.v.i            v1, 0                       \\n\\t\"\n\n                                 \"addi               a1, %[A], 0                 \\n\\t\"\n                                 \"addi               s1, %[B], 0                 \\n\\t\"\n\n                                 \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                                 // scale offset\n                                 \"addi               s5, s1, 0                   \\n\\t\"\n                                 // zp offset\n                                 \"addi               s6, s1, 32                  \\n\\t\"\n                                 \"addi               s1, s6, 16                  \\n\\t\"\n                                 \"addi               s2, s1, 32                  \\n\\t\"\n                                 \"addi               s3, s1, 32*2                \\n\\t\"\n                                 \"addi               s4, s1, 32*3                \\n\\t\"\n\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vxor.vv            v16, v16, v16               \\n\\t\"\n                                 // load a scale\n                                 \"flw                f1, (a1)                    \\n\\t\"\n                                 \"flw                f2, 4(a1)                   \\n\\t\"\n                                 \"flw                f3, 8(a1)                   \\n\\t\"\n                                 \"flw                f4, 12(a1)                  \\n\\t\"\n                                 \"addi               a1, a1, 16                  \\n\\t\"\n                                 \"addi               t2, %[INNER], 0             \\n\\t\"\n\n                                 SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n                                 \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                                 LOAD_B_16x8x2\n\n                                 \"vle8.v             v10, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vle8.v             v11, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vsub.vv            v2, v2, v12                 \\n\\t\"\n                                 \"vsub.vv            v6, v6, v12                 \\n\\t\"\n                                 \"vsub.vv            v3, v3, v13                 \\n\\t\"\n                                 \"vsub.vv            v7, v7, v13                 \\n\\t\"\n                                 \"vsub.vv            v4, v4, v14                 \\n\\t\"\n                                 \"vsub.vv            v8, v8, v14                 \\n\\t\"\n                                 \"vsub.vv            v5, v5, v15                 \\n\\t\"\n                                 \"vsub.vv            v9, v9, v15                 \\n\\t\"\n\n                                 SQ4BIT_KERNEL_COMP_4x16x16\n\n                                 \"addi               t2, t2, -1                  \\n\\t\"\n                                 \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                                 LOAD_SCALE_4x16_FP16\n\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                                 \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                                 \"addi               t3, t3, -1                  \\n\\t\"\n                                 \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                                 \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                                 SAVE_RESULT_4x16\n\n                                 :\n                                 : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                                   [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n                                 : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n                                   \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n            } else {\n                __asm__ volatile(\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v24, v24, v24               \\n\\t\"\n                    \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                    \"li                 s1, 24                      \\n\\t\"\n                    \"vmv.v.i            v1, 3                       \\n\\t\"\n                    \"vsetvli            t0, s1, e8, m1              \\n\\t\"\n                    \"vmv.v.i            v1, 2                       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, mf2           \\n\\t\"\n                    \"vmv.v.i            v1, 1                       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, mf4           \\n\\t\"\n                    \"vmv.v.i            v1, 0                       \\n\\t\"\n                    \"addi               a1, %[A], 0                 \\n\\t\"\n                    \"addi               s1, %[B], 0                 \\n\\t\"\n                    \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                    // scale offset\n                    \"addi               s5, s1, 0                   \\n\\t\"\n                    // zp offset\n                    \"addi               s6, s1, 32                  \\n\\t\"\n                    \"addi               s1, s6, 16                  \\n\\t\"\n                    \"addi               s2, s1, 32                  \\n\\t\"\n                    \"addi               s3, s1, 32*2                \\n\\t\"\n                    \"addi               s4, s1, 32*3                \\n\\t\"\n\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v16, v16, v16               \\n\\t\"\n                    // load a scale\n                    \"flw                f1, (a1)                    \\n\\t\"\n                    \"flw                f2, 4(a1)                   \\n\\t\"\n                    \"flw                f3, 8(a1)                   \\n\\t\"\n                    \"flw                f4, 12(a1)                  \\n\\t\"\n                    \"addi               a1, a1, 16                  \\n\\t\"\n                    \"addi               t2, %[INNER], 0             \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n                    \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                    LOAD_B_16x8x2\n\n                    \"vle8.v             v10, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vle8.v             v11, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vsub.vv            v2, v2, v12                 \\n\\t\"\n                    \"vsub.vv            v6, v6, v12                 \\n\\t\"\n                    \"vsub.vv            v3, v3, v13                 \\n\\t\"\n                    \"vsub.vv            v7, v7, v13                 \\n\\t\"\n                    \"vsub.vv            v4, v4, v14                 \\n\\t\"\n                    \"vsub.vv            v8, v8, v14                 \\n\\t\"\n                    \"vsub.vv            v5, v5, v15                 \\n\\t\"\n                    \"vsub.vv            v9, v9, v15                 \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_4x16x16\n\n                    \"addi               t2, t2, -1                  \\n\\t\"\n                    \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                    LOAD_SCALE_4x16_FP16\n\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                    \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                    \"addi               t3, t3, -1                  \\n\\t\"\n                    \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                    \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                    SAVE_RESULT_4x16\n\n                    :\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                      [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n                      \"s4\", \"s5\", \"s6\");\n            }\n        }\n    } else {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      NBLKS         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +         //\n                                        n * BlockCountK * BlkLen / 2 +     // b data\n                                        n * BlockCountK * sizeof(_Float16);  // scale\n            float * CPtr = C + n;\n            if (NBLKS < 16) {\n                CPtr = tmp;\n                LDC  = 16 * sizeof(float);\n            }\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                if (NBLKS < 16) {\n                    __asm__ volatile(\n                        \"vsetvli        t0, %[N], e32, m2     \\n\\t\"\n                        \"vle32.v        v0, (%[SRC])          \\n\\t\"\n                        \"vse32.v        v0, (%[DST])          \\n\\t\"\n                        :\n                        : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n                        : \"cc\", \"t0\");\n                    bias = tmp;\n                }\n                __asm__ volatile(LOAD_BIAS\n\n                                 \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                                 \"addi               a1, %[A], 0                 \\n\\t\"\n                                 \"addi               s1, %[B], 0                 \\n\\t\"\n                                 \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                                 \"addi               s5, s1, 0                   \\n\\t\"\n                                 \"addi               s1, s5, 32                  \\n\\t\"\n                                 \"addi               s2, s1, 32                  \\n\\t\"\n                                 \"addi               s3, s1, 32*2                \\n\\t\"\n                                 \"addi               s4, s1, 32*3                \\n\\t\"\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vxor.vv            v16, v16, v16               \\n\\t\"\n                                 // load a scale\n                                 \"flw                f1, (a1)                    \\n\\t\"\n                                 \"flw                f2, 4(a1)                   \\n\\t\"\n                                 \"flw                f3, 8(a1)                   \\n\\t\"\n                                 \"flw                f4, 12(a1)                  \\n\\t\"\n                                 \"addi               a1, a1, 16                  \\n\\t\"\n                                 \"addi               t2, %[INNER], 0             \\n\\t\"\n                                 \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                                 LOAD_B_16x8x2\n\n                                 \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                                 \"vle8.v             v10, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vle8.v             v11, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vadd.vi            v2, v2, -8                  \\n\\t\"\n                                 \"vadd.vi            v3, v3, -8                  \\n\\t\"\n                                 \"vadd.vi            v4, v4, -8                  \\n\\t\"\n                                 \"vadd.vi            v5, v5, -8                  \\n\\t\"\n                                 \"vadd.vi            v6, v6, -8                  \\n\\t\"\n                                 \"vadd.vi            v7, v7, -8                  \\n\\t\"\n                                 \"vadd.vi            v8, v8, -8                  \\n\\t\"\n                                 \"vadd.vi            v9, v9, -8                  \\n\\t\"\n\n                                 SQ4BIT_KERNEL_COMP_4x16x16\n\n                                 \"addi               t2, t2, -1                  \\n\\t\"\n                                 \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                                 LOAD_SCALE_4x16_FP16\n\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                                 \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                                 \"addi               t3, t3, -1                  \\n\\t\"\n                                 \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n                                 \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                                 SAVE_RESULT_4x16\n\n                                 :\n                                 : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                                   [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n                                 : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n                                   \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n            } else {\n                __asm__ volatile(\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v24, v24, v24               \\n\\t\"\n                    \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                    \"addi               a1, %[A], 0                 \\n\\t\"\n                    \"addi               s1, %[B], 0                 \\n\\t\"\n                    \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                    \"addi               s5, s1, 0                   \\n\\t\"\n                    \"addi               s1, s5, 32                  \\n\\t\"\n                    \"addi               s2, s1, 32                  \\n\\t\"\n                    \"addi               s3, s1, 32*2                \\n\\t\"\n                    \"addi               s4, s1, 32*3                \\n\\t\"\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v16, v16, v16               \\n\\t\"\n                    // load a scale\n                    \"flw                f1, (a1)                    \\n\\t\"\n                    \"flw                f2, 4(a1)                   \\n\\t\"\n                    \"flw                f3, 8(a1)                   \\n\\t\"\n                    \"flw                f4, 12(a1)                  \\n\\t\"\n                    \"addi               a1, a1, 16                  \\n\\t\"\n                    \"addi               t2, %[INNER], 0             \\n\\t\"\n                    \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                    LOAD_B_16x8x2\n\n                    \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                    \"vle8.v             v10, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vle8.v             v11, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vadd.vi            v2, v2, -8                  \\n\\t\"\n                    \"vadd.vi            v3, v3, -8                  \\n\\t\"\n                    \"vadd.vi            v4, v4, -8                  \\n\\t\"\n                    \"vadd.vi            v5, v5, -8                  \\n\\t\"\n                    \"vadd.vi            v6, v6, -8                  \\n\\t\"\n                    \"vadd.vi            v7, v7, -8                  \\n\\t\"\n                    \"vadd.vi            v8, v8, -8                  \\n\\t\"\n                    \"vadd.vi            v9, v9, -8                  \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_4x16x16\n\n                    \"addi               t2, t2, -1                  \\n\\t\"\n                    \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                    LOAD_SCALE_4x16_FP16\n\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                    \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                    \"addi               t3, t3, -1                  \\n\\t\"\n                    \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n                    \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                    SAVE_RESULT_4x16\n\n                    :\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                      [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n                      \"s4\", \"s5\", \"s6\");\n            }\n        }\n    }\n    if (CountN % 16 != 0) {\n        // stroe output from tmp to C when NBLKS less than 16.\n        float *      CPtr = C + CountN / 16 * 16;\n        const size_t N    = CountN % 16;\n        LDC               = ldc * sizeof(float);\n        __asm__ volatile(\n            \"vsetvli            t0, %[N], e32, m2       \\n\\t\"\n            \"vle32.v            v0, (%[SRC])            \\n\\t\"\n            \"addi               s2, %[SRC], 64          \\n\\t\"\n            \"addi               s3, %[SRC], 64*2        \\n\\t\"\n            \"addi               s4, %[SRC], 64*3        \\n\\t\"\n            \"vle32.v            v2, (s2)                \\n\\t\"\n            \"vle32.v            v4, (s3)                \\n\\t\"\n            \"vle32.v            v6, (s4)                \\n\\t\"\n            \"add                t2, %[DST], %[LDC]      \\n\\t\"\n            \"add                t3, t2, %[LDC]          \\n\\t\"\n            \"add                t4, t3, %[LDC]          \\n\\t\"\n            \"vse32.v            v0, (%[DST])            \\n\\t\"\n            \"vse32.v            v2, (t2)                \\n\\t\"\n            \"vse32.v            v4, (t3)                \\n\\t\"\n            \"vse32.v            v6, (t4)                \\n\\t\"\n            :\n            : [N] \"r\"(N), [SRC] \"r\"(tmp), [DST] \"r\"(CPtr), [LDC] \"r\"(LDC)\n            : \"cc\", \"t0\", \"t2\", \"t3\", \"t4\", \"s2\", \"s3\", \"s4\");\n    }\n}\n\ntemplate <bool HasZeroPoint>\nvoid SQ4BitGemmM4Kernel_CompInt8_Impl(size_t            BlkLen,\n                                      const std::byte * QuantA,\n                                      const std::byte * QuantBData,\n                                      const float *     QuantBScale,\n                                      const std::byte * QuantBZeroPoint,\n                                      float *           C,\n                                      size_t            CountN,\n                                      size_t            BlockCountK,\n                                      const float *     Bias,\n                                      const size_t      ldc) {\n    GGML_UNUSED(QuantBScale);\n    GGML_UNUSED(QuantBZeroPoint);\n    size_t       LDC   = ldc * sizeof(float);\n    const size_t INNER = BlkLen / 16;\n    float        tmp[4 * 16];\n\n    if constexpr (HasZeroPoint) {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      NBLKS         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +           //\n                                        n * BlockCountK * BlkLen / 2 +       // b data\n                                        n * BlockCountK * sizeof(uint8_t) +  // zp\n                                        n * BlockCountK * sizeof(float);     // scale\n            float * CPtr = C + n;\n            if (NBLKS < 16) {\n                CPtr = tmp;\n                LDC  = 16 * sizeof(float);\n            }\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                if (NBLKS < 16) {\n                    __asm__ volatile(\n                        \"vsetvli        t0, %[N], e32, m2     \\n\\t\"\n                        \"vle32.v        v0, (%[SRC])          \\n\\t\"\n                        \"vse32.v        v0, (%[DST])          \\n\\t\"\n                        :\n                        : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n                        : \"cc\", \"t0\");\n                    bias = tmp;\n                }\n\n                __asm__ volatile(LOAD_BIAS\n                                 \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                                 \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                                 \"li                 s1, 24                      \\n\\t\"\n                                 \"vmv.v.i            v1, 3                       \\n\\t\"\n                                 \"vsetvli            t0, s1, e8, m1              \\n\\t\"\n                                 \"vmv.v.i            v1, 2                       \\n\\t\"\n                                 \"vsetvli            t0, zero, e8, mf2           \\n\\t\"\n                                 \"vmv.v.i            v1, 1                       \\n\\t\"\n                                 \"vsetvli            t0, zero, e8, mf4           \\n\\t\"\n                                 \"vmv.v.i            v1, 0                       \\n\\t\"\n                                 \"addi               a1, %[A], 0                 \\n\\t\"\n                                 \"addi               s1, %[B], 0                 \\n\\t\"\n                                 \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                                 // scale offset\n                                 \"addi               s5, s1, 0                   \\n\\t\"\n                                 // zp offset\n                                 \"addi               s6, s1, 64                  \\n\\t\"\n                                 \"addi               s1, s6, 16                  \\n\\t\"\n                                 \"addi               s2, s1, 32                  \\n\\t\"\n                                 \"addi               s3, s1, 32*2                \\n\\t\"\n                                 \"addi               s4, s1, 32*3                \\n\\t\"\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vxor.vv            v16, v16, v16               \\n\\t\"\n                                 // load a scale\n                                 \"flw                f1, (a1)                    \\n\\t\"\n                                 \"flw                f2, 4(a1)                   \\n\\t\"\n                                 \"flw                f3, 8(a1)                   \\n\\t\"\n                                 \"flw                f4, 12(a1)                  \\n\\t\"\n                                 \"addi               a1, a1, 16                  \\n\\t\"\n                                 \"addi               t2, %[INNER], 0             \\n\\t\"\n\n                                 SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n                                 \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                                 LOAD_B_16x8x2\n\n                                 \"vle8.v             v10, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vle8.v             v11, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vsub.vv            v2, v2, v12                 \\n\\t\"\n                                 \"vsub.vv            v6, v6, v12                 \\n\\t\"\n                                 \"vsub.vv            v3, v3, v13                 \\n\\t\"\n                                 \"vsub.vv            v7, v7, v13                 \\n\\t\"\n                                 \"vsub.vv            v4, v4, v14                 \\n\\t\"\n                                 \"vsub.vv            v8, v8, v14                 \\n\\t\"\n                                 \"vsub.vv            v5, v5, v15                 \\n\\t\"\n                                 \"vsub.vv            v9, v9, v15                 \\n\\t\"\n\n                                 SQ4BIT_KERNEL_COMP_4x16x16\n\n                                 \"addi               t2, t2, -1                  \\n\\t\"\n                                 \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                                 LOAD_SCALE_4x16\n\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                                 \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                                 \"addi               t3, t3, -1                  \\n\\t\"\n                                 \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                                 \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                                 SAVE_RESULT_4x16\n\n                                 :\n                                 : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                                   [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n                                 : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n                                   \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n            } else {\n                __asm__ volatile(\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v24, v24, v24               \\n\\t\"\n                    \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                    \"li                 s1, 24                      \\n\\t\"\n                    \"vmv.v.i            v1, 3                       \\n\\t\"\n                    \"vsetvli            t0, s1, e8, m1              \\n\\t\"\n                    \"vmv.v.i            v1, 2                       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, mf2           \\n\\t\"\n                    \"vmv.v.i            v1, 1                       \\n\\t\"\n                    \"vsetvli            t0, zero, e8, mf4           \\n\\t\"\n                    \"vmv.v.i            v1, 0                       \\n\\t\"\n                    \"addi               a1, %[A], 0                 \\n\\t\"\n                    \"addi               s1, %[B], 0                 \\n\\t\"\n                    \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                    // scale offset\n                    \"addi               s5, s1, 0                   \\n\\t\"\n                    // zp offset\n                    \"addi               s6, s1, 64                  \\n\\t\"\n                    \"addi               s1, s6, 16                  \\n\\t\"\n                    \"addi               s2, s1, 32                  \\n\\t\"\n                    \"addi               s3, s1, 32*2                \\n\\t\"\n                    \"addi               s4, s1, 32*3                \\n\\t\"\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v16, v16, v16               \\n\\t\"\n                    // load a scale\n                    // load a scale\n                    \"flw                f1, (a1)                    \\n\\t\"\n                    \"flw                f2, 4(a1)                   \\n\\t\"\n                    \"flw                f3, 8(a1)                   \\n\\t\"\n                    \"flw                f4, 12(a1)                  \\n\\t\"\n                    \"addi               a1, a1, 16                  \\n\\t\"\n                    \"addi               t2, %[INNER], 0             \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1_v2\n\n                    \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                    LOAD_B_16x8x2\n\n                    \"vle8.v             v10, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vle8.v             v11, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vsub.vv            v2, v2, v12                 \\n\\t\"\n                    \"vsub.vv            v6, v6, v12                 \\n\\t\"\n                    \"vsub.vv            v3, v3, v13                 \\n\\t\"\n                    \"vsub.vv            v7, v7, v13                 \\n\\t\"\n                    \"vsub.vv            v4, v4, v14                 \\n\\t\"\n                    \"vsub.vv            v8, v8, v14                 \\n\\t\"\n                    \"vsub.vv            v5, v5, v15                 \\n\\t\"\n                    \"vsub.vv            v9, v9, v15                 \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_4x16x16\n\n                    \"addi               t2, t2, -1                  \\n\\t\"\n                    \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                    LOAD_SCALE_4x16\n\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                    \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                    \"addi               t3, t3, -1                  \\n\\t\"\n                    \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                    \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                    SAVE_RESULT_4x16\n\n                    :\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                      [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n                      \"s4\", \"s5\", \"s6\");\n            }\n        }\n    } else {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      NBLKS         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +        //\n                                        n * BlockCountK * BlkLen / 2 +    // b data\n                                        n * BlockCountK * sizeof(float);  // scale\n            float * CPtr = C + n;\n            if (NBLKS < 16) {\n                CPtr = tmp;\n                LDC  = 16 * sizeof(float);\n            }\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                if (NBLKS < 16) {\n                    __asm__ volatile(\n                        \"vsetvli        t0, %[N], e32, m2     \\n\\t\"\n                        \"vle32.v        v0, (%[SRC])          \\n\\t\"\n                        \"vse32.v        v0, (%[DST])          \\n\\t\"\n                        :\n                        : [SRC] \"r\"(bias), [DST] \"r\"(tmp), [N] \"r\"(NBLKS)\n                        : \"cc\", \"t0\");\n                    bias = tmp;\n                }\n                __asm__ volatile(LOAD_BIAS\n                                 \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                                 \"addi               a1, %[A], 0                 \\n\\t\"\n                                 \"addi               s1, %[B], 0                 \\n\\t\"\n                                 \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                                 \"addi               s5, s1, 0                   \\n\\t\"\n                                 \"addi               s1, s5, 64                  \\n\\t\"\n                                 \"addi               s2, s1, 32                  \\n\\t\"\n                                 \"addi               s3, s1, 32*2                \\n\\t\"\n                                 \"addi               s4, s1, 32*3                \\n\\t\"\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vxor.vv            v16, v16, v16               \\n\\t\"\n                                 // load a scale\n                                 \"flw                f1, (a1)                    \\n\\t\"\n                                 \"flw                f2, 4(a1)                   \\n\\t\"\n                                 \"flw                f3, 8(a1)                   \\n\\t\"\n                                 \"flw                f4, 12(a1)                  \\n\\t\"\n                                 \"addi               a1, a1, 16                  \\n\\t\"\n                                 \"addi               t2, %[INNER], 0             \\n\\t\"\n                                 \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                                 LOAD_B_16x8x2\n\n                                 \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                                 \"vle8.v             v10, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vle8.v             v11, (a1)                   \\n\\t\"\n                                 \"addi               a1, a1, 32                  \\n\\t\"\n                                 \"vadd.vi            v2, v2, -8                  \\n\\t\"\n                                 \"vadd.vi            v3, v3, -8                  \\n\\t\"\n                                 \"vadd.vi            v4, v4, -8                  \\n\\t\"\n                                 \"vadd.vi            v5, v5, -8                  \\n\\t\"\n                                 \"vadd.vi            v6, v6, -8                  \\n\\t\"\n                                 \"vadd.vi            v7, v7, -8                  \\n\\t\"\n                                 \"vadd.vi            v8, v8, -8                  \\n\\t\"\n                                 \"vadd.vi            v9, v9, -8                  \\n\\t\"\n\n                                 SQ4BIT_KERNEL_COMP_4x16x16\n\n                                 \"addi               t2, t2, -1                  \\n\\t\"\n                                 \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                                 LOAD_SCALE_4x16\n\n                                 \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                                 \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                                 \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                                 \"addi               t3, t3, -1                  \\n\\t\"\n                                 \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                                 \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                                 SAVE_RESULT_4x16\n\n                                 :\n                                 : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                                   [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr), [BIAS] \"r\"(bias)\n                                 : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\",\n                                   \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n\n            } else {\n                __asm__ volatile(\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v24, v24, v24               \\n\\t\"\n                    \"addi               t3, %[BlockCountK], 0       \\n\\t\"\n                    \"addi               a1, %[A], 0                 \\n\\t\"\n                    \"addi               s1, %[B], 0                 \\n\\t\"\n                    \"BLOCK_COUNTK_LOOP%=:                           \\n\\t\"\n                    \"addi               s5, s1, 0                   \\n\\t\"\n                    \"addi               s1, s5, 64                  \\n\\t\"\n                    \"addi               s2, s1, 32                  \\n\\t\"\n                    \"addi               s3, s1, 32*2                \\n\\t\"\n                    \"addi               s4, s1, 32*3                \\n\\t\"\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vxor.vv            v16, v16, v16               \\n\\t\"\n                    // load a scale\n                    \"flw                f1, (a1)                    \\n\\t\"\n                    \"flw                f2, 4(a1)                   \\n\\t\"\n                    \"flw                f3, 8(a1)                   \\n\\t\"\n                    \"flw                f4, 12(a1)                  \\n\\t\"\n                    \"addi               a1, a1, 16                  \\n\\t\"\n                    \"addi               t2, %[INNER], 0             \\n\\t\"\n                    \"BLOCK_INNER_LOOP%=:                            \\n\\t\"\n\n                    LOAD_B_16x8x2\n\n                    \"vsetvli            t0, zero, e8, m1            \\n\\t\"\n                    \"vle8.v             v10, (a1)                   \\n\\t\"\n\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vle8.v             v11, (a1)                   \\n\\t\"\n                    \"addi               a1, a1, 32                  \\n\\t\"\n                    \"vadd.vi            v2, v2, -8                  \\n\\t\"\n                    \"vadd.vi            v3, v3, -8                  \\n\\t\"\n                    \"vadd.vi            v4, v4, -8                  \\n\\t\"\n                    \"vadd.vi            v5, v5, -8                  \\n\\t\"\n                    \"vadd.vi            v6, v6, -8                  \\n\\t\"\n                    \"vadd.vi            v7, v7, -8                  \\n\\t\"\n                    \"vadd.vi            v8, v8, -8                  \\n\\t\"\n                    \"vadd.vi            v9, v9, -8                  \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_4x16x16\n\n                    \"addi               t2, t2, -1                  \\n\\t\"\n                    \"bnez               t2, BLOCK_INNER_LOOP%=      \\n\\t\"\n\n                    LOAD_SCALE_4x16\n\n                    \"vsetvli            t0, zero, e32, m8           \\n\\t\"\n                    \"vfcvt.f.x.v        v16, v16                    \\n\\t\"\n                    \"vfmacc.vv          v24, v16, v8                \\n\\t\"\n                    \"addi               t3, t3, -1                  \\n\\t\"\n                    \"bnez               t3, BLOCK_COUNTK_LOOP%=     \\n\\t\"\n\n                    \"RESULT_SAVE%=:                                 \\n\\t\"\n\n                    SAVE_RESULT_4x16\n\n                    :\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [LDC] \"r\"(LDC),\n                      [BlockCountK] \"r\"(BlockCountK), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t1\", \"t2\", \"t3\", \"a1\", \"a2\", \"a3\", \"a4\", \"f1\", \"f2\", \"f3\", \"f4\", \"s1\", \"s2\", \"s3\",\n                      \"s4\", \"s5\", \"s6\");\n            }\n        }\n    }\n    if (CountN % 16 != 0) {\n        // stroe output from tmp to C when NBLKS less than 16.\n        float *      CPtr = C + CountN / 16 * 16;\n        const size_t N    = CountN % 16;\n        LDC               = ldc * sizeof(float);\n        __asm__ volatile(\n            \"vsetvli            t0, %[N], e32, m2       \\n\\t\"\n            \"vle32.v            v0, (%[SRC])            \\n\\t\"\n            \"addi               s2, %[SRC], 64          \\n\\t\"\n            \"addi               s3, %[SRC], 64*2        \\n\\t\"\n            \"addi               s4, %[SRC], 64*3        \\n\\t\"\n            \"vle32.v            v2, (s2)                \\n\\t\"\n            \"vle32.v            v4, (s3)                \\n\\t\"\n            \"vle32.v            v6, (s4)                \\n\\t\"\n            \"add                t2, %[DST], %[LDC]      \\n\\t\"\n            \"add                t3, t2, %[LDC]          \\n\\t\"\n            \"add                t4, t3, %[LDC]          \\n\\t\"\n            \"vse32.v            v0, (%[DST])            \\n\\t\"\n            \"vse32.v            v2, (t2)                \\n\\t\"\n            \"vse32.v            v4, (t3)                \\n\\t\"\n            \"vse32.v            v6, (t4)                \\n\\t\"\n            :\n            : [N] \"r\"(N), [SRC] \"r\"(tmp), [DST] \"r\"(CPtr), [LDC] \"r\"(LDC)\n            : \"cc\", \"t0\", \"t2\", \"t3\", \"t4\", \"s2\", \"s3\", \"s4\");\n    }\n}\n\ntemplate <bool HasZeroPoint>\nvoid SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl(size_t            BlkLen,\n                                                const std::byte * QuantA,\n                                                const std::byte * QuantBData,\n                                                const float *     QuantBScale,\n                                                const std::byte * QuantBZeroPoint,\n                                                float *           C,\n                                                size_t            CountN,\n                                                size_t            BlockCountK,\n                                                const float *     Bias) {\n    GGML_UNUSED(QuantBScale);\n    GGML_UNUSED(QuantBZeroPoint);\n    size_t INNER = BlkLen / 16;\n\n    if constexpr (HasZeroPoint) {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      nblks         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +           //\n                                        n * BlockCountK * BlkLen / 2 +       // b data\n                                        n * BlockCountK * sizeof(uint8_t) +  // zp\n                                        n * BlockCountK * sizeof(_Float16);    // scale\n            float * CPtr = C + n;\n            size_t  cnt  = BlockCountK;\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                __asm__ volatile(\n                    \"addi         t3, %[NBLKS], 0         \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n\n                    \"vmv.v.i      v13, 3                  \\n\\t\"\n                    \"li           s1, 24                  \\n\\t\"\n                    \"vsetvli      t0, s1, e8, m1          \\n\\t\"\n                    \"vmv.v.i      v13, 2                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf2       \\n\\t\"\n                    \"vmv.v.i      v13, 1                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf4       \\n\\t\"\n                    \"vmv.v.i      v13, 0                  \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 8             \\n\\t\"\n                    \"addi         s3, %[B], 16            \\n\\t\"\n                    \"addi         s4, %[B], 24            \\n\\t\"\n                    // zp offset\n                    \"addi         s7, %[B], 32            \\n\\t\"\n                    // a offset\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v28, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v29, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v30, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v31, (%[BIAS])          \\n\\t\"\n\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vsetvli      t0, zero, e16, mf4      \\n\\t\"\n\n                    \"vle16.v      v4, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 48              \\n\\t\"\n                    \"vle16.v      v5, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 72              \\n\\t\"\n                    \"vle16.v      v6, (s3)                \\n\\t\"\n                    \"addi         s3, s3, 96              \\n\\t\"\n                    \"vle16.v      v7, (s4)                \\n\\t\"\n                    \"addi         s4, s4, 120             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n                    \"vfwcvt.f.f.v v8, v4                  \\n\\t\"\n                    \"vfwcvt.f.f.v v9, v5                  \\n\\t\"\n                    \"vfwcvt.f.f.v v10, v6                 \\n\\t\"\n                    \"vfwcvt.f.f.v v11, v7                 \\n\\t\"\n\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vsub.vv      v0, v0, v8              \\n\\t\"\n                    \"vsub.vv      v4, v4, v8              \\n\\t\"\n                    \"vsub.vv      v1, v1, v9              \\n\\t\"\n                    \"vsub.vv      v5, v5, v9              \\n\\t\"\n                    \"vsub.vv      v2, v2, v10             \\n\\t\"\n                    \"vsub.vv      v6, v6, v10             \\n\\t\"\n                    \"vsub.vv      v3, v3, v11             \\n\\t\"\n                    \"vsub.vv      v7, v7, v11             \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_F16_1X4X4\n                    \"addi         s7, s1, 32              \\n\\t\"\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n            } else {\n                __asm__ volatile(\n                    \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n                    \"vxor.vv      v28, v28, v28           \\n\\t\"\n\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"vmv.v.i      v13, 3                  \\n\\t\"\n                    \"li           s1, 24                  \\n\\t\"\n                    \"vsetvli      t0, s1, e8, m1          \\n\\t\"\n                    \"vmv.v.i      v13, 2                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf2       \\n\\t\"\n                    \"vmv.v.i      v13, 1                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf4       \\n\\t\"\n                    \"vmv.v.i      v13, 0                  \\n\\t\"\n\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 8             \\n\\t\"\n                    \"addi         s3, %[B], 16            \\n\\t\"\n                    \"addi         s4, %[B], 24            \\n\\t\"\n\n                    \"addi         s7, %[B], 32            \\n\\t\"\n\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vsetvli      t0, zero, e16, mf4      \\n\\t\"\n                    \"vle16.v      v4, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 48              \\n\\t\"\n                    \"vle16.v      v5, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 72              \\n\\t\"\n                    \"vle16.v      v6, (s3)                \\n\\t\"\n                    \"addi         s3, s3, 96              \\n\\t\"\n                    \"vle16.v      v7, (s4)                \\n\\t\"\n                    \"addi         s4, s4, 120             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"vfwcvt.f.f.v v8, v4                  \\n\\t\"\n                    \"vfwcvt.f.f.v v9, v5                  \\n\\t\"\n                    \"vfwcvt.f.f.v v10, v6                 \\n\\t\"\n                    \"vfwcvt.f.f.v v11, v7                 \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vsub.vv      v0, v0, v8              \\n\\t\"\n                    \"vsub.vv      v4, v4, v8              \\n\\t\"\n                    \"vsub.vv      v1, v1, v9              \\n\\t\"\n                    \"vsub.vv      v5, v5, v9              \\n\\t\"\n                    \"vsub.vv      v2, v2, v10             \\n\\t\"\n                    \"vsub.vv      v6, v6, v10             \\n\\t\"\n                    \"vsub.vv      v3, v3, v11             \\n\\t\"\n                    \"vsub.vv      v7, v7, v11             \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_F16_1X4X4\n                    \"addi         s7, s1, 32              \\n\\t\"\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n            }\n        }\n    } else {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      nblks         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +         //\n                                        n * BlockCountK * BlkLen / 2 +     // b data\n                                        n * BlockCountK * sizeof(_Float16);  // scale\n            float * CPtr = C + n;\n            size_t  cnt  = BlockCountK;\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                __asm__ volatile(\n                    \"addi         t3, %[NBLKS], 0         \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 8             \\n\\t\"\n                    \"addi         s3, %[B], 16            \\n\\t\"\n                    \"addi         s4, %[B], 24            \\n\\t\"\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v28, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v29, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v30, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v31, (%[BIAS])          \\n\\t\"\n\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vsetvli      t0, zero, e16, mf4      \\n\\t\"\n\n                    \"vle16.v      v4, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 32              \\n\\t\"\n                    \"vle16.v      v5, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 56              \\n\\t\"\n                    \"vle16.v      v6, (s3)                \\n\\t\"\n                    \"addi         s3, s3, 80              \\n\\t\"\n                    \"vle16.v      v7, (s4)                \\n\\t\"\n                    \"addi         s4, s4, 104             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n                    \"vfwcvt.f.f.v v8, v4                  \\n\\t\"\n                    \"vfwcvt.f.f.v v9, v5                  \\n\\t\"\n                    \"vfwcvt.f.f.v v10, v6                 \\n\\t\"\n                    \"vfwcvt.f.f.v v11, v7                 \\n\\t\"\n\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vadd.vi      v0, v0, -8              \\n\\t\"\n                    \"vadd.vi      v1, v1, -8              \\n\\t\"\n                    \"vadd.vi      v2, v2, -8              \\n\\t\"\n                    \"vadd.vi      v3, v3, -8              \\n\\t\"\n                    \"vadd.vi      v4, v4, -8              \\n\\t\"\n                    \"vadd.vi      v5, v5, -8              \\n\\t\"\n                    \"vadd.vi      v6, v6, -8              \\n\\t\"\n                    \"vadd.vi      v7, v7, -8              \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_F16_1X4X4\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n            } else {\n                __asm__ volatile(\n                    \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n                    \"vxor.vv      v28, v28, v28           \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 8             \\n\\t\"\n                    \"addi         s3, %[B], 16            \\n\\t\"\n                    \"addi         s4, %[B], 24            \\n\\t\"\n\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vsetvli      t0, zero, e16, mf4      \\n\\t\"\n                    \"vle16.v      v4, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 32              \\n\\t\"\n                    \"vle16.v      v5, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 56              \\n\\t\"\n                    \"vle16.v      v6, (s3)                \\n\\t\"\n                    \"addi         s3, s3, 80              \\n\\t\"\n                    \"vle16.v      v7, (s4)                \\n\\t\"\n                    \"addi         s4, s4, 104             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"vfwcvt.f.f.v v8, v4                  \\n\\t\"\n                    \"vfwcvt.f.f.v v9, v5                  \\n\\t\"\n                    \"vfwcvt.f.f.v v10, v6                 \\n\\t\"\n                    \"vfwcvt.f.f.v v11, v7                 \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vadd.vi      v0, v0, -8              \\n\\t\"\n                    \"vadd.vi      v1, v1, -8              \\n\\t\"\n                    \"vadd.vi      v2, v2, -8              \\n\\t\"\n                    \"vadd.vi      v3, v3, -8              \\n\\t\"\n                    \"vadd.vi      v4, v4, -8              \\n\\t\"\n                    \"vadd.vi      v5, v5, -8              \\n\\t\"\n                    \"vadd.vi      v6, v6, -8              \\n\\t\"\n                    \"vadd.vi      v7, v7, -8              \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_F16_1X4X4\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n            }\n        }\n    }\n}\n\ntemplate <bool HasZeroPoint>\nvoid SQ4BitGemmM1Kernel_CompInt8_Impl(size_t            BlkLen,\n                                      const std::byte * QuantA,\n                                      const std::byte * QuantBData,\n                                      const float *     QuantBScale,\n                                      const std::byte * QuantBZeroPoint,\n                                      float *           C,\n                                      size_t            CountN,\n                                      size_t            BlockCountK,\n                                      const float *     Bias) {\n    GGML_UNUSED(QuantBScale);\n    GGML_UNUSED(QuantBZeroPoint);\n    const size_t INNER = BlkLen / 16;\n    if constexpr (HasZeroPoint) {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      nblks         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +           //\n                                        n * BlockCountK * BlkLen / 2 +       // b data\n                                        n * BlockCountK * sizeof(uint8_t) +  // zp\n                                        n * BlockCountK * sizeof(float);     // scale\n            float * CPtr = C + n;\n            size_t  cnt  = BlockCountK;\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                __asm__ volatile(\n                    \"addi         t3, %[NBLKS], 0         \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"vmv.v.i      v13, 3                  \\n\\t\"\n                    \"li           s1, 24                  \\n\\t\"\n                    \"vsetvli      t0, s1, e8, m1          \\n\\t\"\n                    \"vmv.v.i      v13, 2                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf2       \\n\\t\"\n                    \"vmv.v.i      v13, 1                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf4       \\n\\t\"\n                    \"vmv.v.i      v13, 0                  \\n\\t\"\n                    \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n                    \"vxor.vv      v28, v28, v28           \\n\\t\"\n\n                    // scale offset, scale0.0, scale1.0, scale2.0, scale3.0....scale15.0\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 16            \\n\\t\"\n                    \"addi         s3, %[B], 32            \\n\\t\"\n                    \"addi         s4, %[B], 48            \\n\\t\"\n                    // zp offset\n                    \"addi         s7, %[B], 64            \\n\\t\"\n                    // a offset\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v28, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v29, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v30, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v31, (%[BIAS])          \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n                    \"LOOP_K%=:                            \\n\\t\"\n\n                    // load scale\n                    \"vle32.v      v8, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 80              \\n\\t\"\n                    \"vle32.v      v9, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 96              \\n\\t\"\n                    \"vle32.v      v10, (s3)               \\n\\t\"\n                    \"addi         s3, s3, 112             \\n\\t\"\n                    \"vle32.v      v11, (s4)               \\n\\t\"\n                    \"addi         s4, s4, 128             \\n\\t\"\n\n                    // load a scale\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n\n                    // a scale * b scale\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vsub.vv      v0, v0, v8              \\n\\t\"\n                    \"vsub.vv      v4, v4, v8              \\n\\t\"\n                    \"vsub.vv      v1, v1, v9              \\n\\t\"\n                    \"vsub.vv      v5, v5, v9              \\n\\t\"\n                    \"vsub.vv      v2, v2, v10             \\n\\t\"\n                    \"vsub.vv      v6, v6, v10             \\n\\t\"\n                    \"vsub.vv      v3, v3, v11             \\n\\t\"\n                    \"vsub.vv      v7, v7, v11             \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_1X4X4\n                    \"addi         s7, s1, 64              \\n\\t\"\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n            } else {\n                __asm__ volatile(\n                    \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n                    \"vxor.vv      v28, v28, v28           \\n\\t\"\n\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"vmv.v.i      v13, 3                  \\n\\t\"\n                    \"li           s1, 24                  \\n\\t\"\n                    \"vsetvli      t0, s1, e8, m1          \\n\\t\"\n                    \"vmv.v.i      v13, 2                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf2       \\n\\t\"\n                    \"vmv.v.i      v13, 1                  \\n\\t\"\n                    \"vsetvli      t0, zero, e8, mf4       \\n\\t\"\n                    \"vmv.v.i      v13, 0                  \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 16            \\n\\t\"\n                    \"addi         s3, %[B], 32            \\n\\t\"\n                    \"addi         s4, %[B], 48            \\n\\t\"\n\n                    \"addi         s7, %[B], 64            \\n\\t\"\n\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vle32.v      v8, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 80              \\n\\t\"\n                    \"vle32.v      v9, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 96              \\n\\t\"\n                    \"vle32.v      v10, (s3)               \\n\\t\"\n                    \"addi         s3, s3, 112             \\n\\t\"\n                    \"vle32.v      v11, (s4)               \\n\\t\"\n                    \"addi         s4, s4, 128             \\n\\t\"\n\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_ZP_16X1\n\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vsub.vv      v0, v0, v8              \\n\\t\"\n                    \"vsub.vv      v4, v4, v8              \\n\\t\"\n                    \"vsub.vv      v1, v1, v9              \\n\\t\"\n                    \"vsub.vv      v5, v5, v9              \\n\\t\"\n                    \"vsub.vv      v2, v2, v10             \\n\\t\"\n                    \"vsub.vv      v6, v6, v10             \\n\\t\"\n                    \"vsub.vv      v3, v3, v11             \\n\\t\"\n                    \"vsub.vv      v7, v7, v11             \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_1X4X4\n                    \"addi         s7, s1, 64              \\n\\t\"\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\", \"s7\");\n            }\n        }\n    } else {\n        for (size_t n = 0; n < CountN; n += 16) {\n            size_t      nblks         = (CountN - n) > 16 ? 16 : CountN - n;\n            std::byte * QuantBDataPtr = (std::byte *) QuantBData +        //\n                                        n * BlockCountK * BlkLen / 2 +    // b data\n                                        n * BlockCountK * sizeof(float);  // scale\n            float * CPtr = C + n;\n            size_t  cnt  = BlockCountK;\n            if (Bias != nullptr) {\n                const float * bias = Bias + n;\n                __asm__ volatile(\n                    \"addi         t3, %[NBLKS], 0         \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 16            \\n\\t\"\n                    \"addi         s3, %[B], 32            \\n\\t\"\n                    \"addi         s4, %[B], 48            \\n\\t\"\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v28, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v29, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v30, (%[BIAS])          \\n\\t\"\n                    \"sub          t3, t3, t0              \\n\\t\"\n                    \"addi         %[BIAS], %[BIAS], 16    \\n\\t\"\n                    \"vsetvli      t0, t3, e32, mf2        \\n\\t\"\n                    \"vle32.v      v31, (%[BIAS])          \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vle32.v      v8, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 64              \\n\\t\"\n                    \"vle32.v      v9, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 80              \\n\\t\"\n                    \"vle32.v      v10, (s3)               \\n\\t\"\n                    \"addi         s3, s3, 96              \\n\\t\"\n                    \"vle32.v      v11, (s4)               \\n\\t\"\n                    \"addi         s4, s4, 112             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vadd.vi      v0, v0, -8              \\n\\t\"\n                    \"vadd.vi      v1, v1, -8              \\n\\t\"\n                    \"vadd.vi      v2, v2, -8              \\n\\t\"\n                    \"vadd.vi      v3, v3, -8              \\n\\t\"\n                    \"vadd.vi      v4, v4, -8              \\n\\t\"\n                    \"vadd.vi      v5, v5, -8              \\n\\t\"\n                    \"vadd.vi      v6, v6, -8              \\n\\t\"\n                    \"vadd.vi      v7, v7, -8              \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_1X4X4\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks), [BIAS] \"+r\"(bias)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n            } else {\n                __asm__ volatile(\n                    \"vsetvli      t0, zero, e32, m4       \\n\\t\"\n                    \"vxor.vv      v28, v28, v28           \\n\\t\"\n                    \"addi         s1, %[B], 0             \\n\\t\"\n                    \"addi         s2, %[B], 16            \\n\\t\"\n                    \"addi         s3, %[B], 32            \\n\\t\"\n                    \"addi         s4, %[B], 48            \\n\\t\"\n\n                    \"addi         s5, %[A], 0             \\n\\t\"\n                    \"addi         s6, %[A], 12            \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n                    \"LOOP_K%=:                            \\n\\t\"\n                    \"vle32.v      v8, (s1)                \\n\\t\"\n                    \"addi         s1, s1, 64              \\n\\t\"\n                    \"vle32.v      v9, (s2)                \\n\\t\"\n                    \"addi         s2, s2, 80              \\n\\t\"\n                    \"vle32.v      v10, (s3)               \\n\\t\"\n                    \"addi         s3, s3, 96              \\n\\t\"\n                    \"vle32.v      v11, (s4)               \\n\\t\"\n                    \"addi         s4, s4, 112             \\n\\t\"\n                    \"flw          f1, (s5)                \\n\\t\"\n                    \"addi         s5, s5, 4               \\n\\t\"\n\n                    \"addi         t5, %[INNER], 0         \\n\\t\"\n                    \"vxor.vv      v16, v16, v16           \\n\\t\"\n                    \"vxor.vv      v18, v18, v18           \\n\\t\"\n                    \"vxor.vv      v20, v20, v20           \\n\\t\"\n                    \"vxor.vv      v22, v22, v22           \\n\\t\"\n                    \"vfmul.vf     v24, v8, f1             \\n\\t\"\n                    \"vfmul.vf     v25, v9, f1             \\n\\t\"\n                    \"vfmul.vf     v26, v10, f1            \\n\\t\"\n                    \"vfmul.vf     v27, v11, f1            \\n\\t\"\n                    \"addi         %[CNT], %[CNT], -1      \\n\\t\"\n                    \"vsetvli      t0, zero, e8, m1        \\n\\t\"\n                    \"LOOP_INNER%=:                        \\n\\t\"\n\n                    SQ4BIT_KERNEL_LOAD_1x8x2_4X8X4\n\n                    \"vadd.vi      v0, v0, -8              \\n\\t\"\n                    \"vadd.vi      v1, v1, -8              \\n\\t\"\n                    \"vadd.vi      v2, v2, -8              \\n\\t\"\n                    \"vadd.vi      v3, v3, -8              \\n\\t\"\n                    \"vadd.vi      v4, v4, -8              \\n\\t\"\n                    \"vadd.vi      v5, v5, -8              \\n\\t\"\n                    \"vadd.vi      v6, v6, -8              \\n\\t\"\n                    \"vadd.vi      v7, v7, -8              \\n\\t\"\n\n                    SQ4BIT_KERNEL_COMP_1x8x2_4X8X4\n\n                    \"bnez         t5, LOOP_INNER%=        \\n\\t\"\n                    \"vsetvli      t0, zero, e32, mf2      \\n\\t\"\n\n                    SQ4BIT_KERNEL_ACC_1X4X4\n\n                    \"bnez         %[CNT], LOOP_K%=        \\n\\t\"\n                    \"addi         t3, zero, 16            \\n\\t\"\n                    \"addi         s1, %[C], 16            \\n\\t\"\n                    \"addi         s2, %[C], 32            \\n\\t\"\n                    \"addi         s3, %[C], 48            \\n\\t\"\n                    \"blt          %[NBLKS], t3, ST_TAIL%= \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"jal          x0, END%=               \\n\\t\"\n\n                    \"ST_TAIL%=:                           \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v28, (%[C])             \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v29, (s1)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v30, (s2)               \\n\\t\"\n                    \"vsetvli      t0, %[NBLKS], e32, mf2  \\n\\t\"\n                    \"sub          %[NBLKS], %[NBLKS], t0  \\n\\t\"\n                    \"vse32.v      v31, (s3)               \\n\\t\"\n                    \"END%=:                               \\n\\t\"\n\n                    : [CNT] \"+r\"(cnt), [NBLKS] \"+r\"(nblks)\n                    : [INNER] \"r\"(INNER), [A] \"r\"(QuantA), [B] \"r\"(QuantBDataPtr), [C] \"r\"(CPtr)\n                    : \"cc\", \"t0\", \"t5\", \"t3\", \"f1\", \"s1\", \"s2\", \"s3\", \"s4\", \"s5\", \"s6\");\n            }\n        }\n    }\n}\n\ntemplate <bool HasZeroPoint>\ninline void SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen(size_t            BlkLen,\n                                                         const std::byte * QuantA,\n                                                         const std::byte * QuantBData,\n                                                         const float *     QuantBScale,\n                                                         const std::byte * QuantBZeroPoint,\n                                                         float *           C,\n                                                         size_t            CountM,\n                                                         size_t            CountN,\n                                                         size_t            BlockStrideQuantB,\n                                                         const float *     Bias,\n                                                         const size_t      ldc,\n                                                         const size_t      scalestride) {\n    if (scalestride == 4) {\n        SQ4BitGemmM4Kernel_CompInt8_Impl<HasZeroPoint>(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C,\n                                                       CountN, BlockStrideQuantB, Bias, ldc);\n\n    } else if (scalestride == 2) {\n        SQ4BitGemmM4Kernel_CompInt8_ScaleFp16_Impl<HasZeroPoint>(\n            BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias, ldc);\n    }\n}\n\ntemplate <bool HasZeroPoint>\ninline void SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen(size_t            BlkLen,\n                                                         const std::byte * QuantA,\n                                                         const std::byte * QuantBData,\n                                                         const float *     QuantBScale,\n                                                         const std::byte * QuantBZeroPoint,\n                                                         float *           C,\n                                                         size_t            CountM,\n                                                         size_t            CountN,\n                                                         size_t            BlockStrideQuantB,\n                                                         const float *     Bias,\n                                                         const size_t      ldc,\n                                                         const size_t      scalestride) {\n    if (scalestride == 4) {\n        SQ4BitGemmM1Kernel_CompInt8_Impl<HasZeroPoint>(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint, C,\n                                                       CountN, BlockStrideQuantB, Bias);\n    } else if (scalestride == 2) {\n        SQ4BitGemmM1Kernel_CompInt8_ScaleFp16_Impl<HasZeroPoint>(BlkLen, QuantA, QuantBData, QuantBScale,\n                                                                 QuantBZeroPoint, C, CountN, BlockStrideQuantB, Bias);\n    }\n}\n\n}  // namespace\n\nnamespace ime1 {\nsize_t gemm_kernel_i8i4(size_t            BlkLen,\n                        const std::byte * QuantA,\n                        const std::byte * QuantBData,\n                        const float *     QuantBScale,\n                        const std::byte * QuantBZeroPoint,\n                        float *           C,\n                        size_t            CountM,\n                        size_t            CountN,\n                        size_t            CountK,\n                        size_t            BlockCountK,\n                        size_t            ldc,\n                        const float *     Bias,\n                        const size_t      ScaleStride) {\n    GGML_UNUSED(CountM);\n    GGML_UNUSED(CountK);\n    GGML_UNUSED(ldc);\n    if (CountM >= 4) {\n        if (QuantBZeroPoint != nullptr) {\n            SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen<true>(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint,\n                                                               C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride);\n        } else {\n            SQ4BitGemmM4Kernel_CompInt8_DispatchOnBlkLen<false>(BlkLen, QuantA, QuantBData, QuantBScale,\n                                                                QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias,\n                                                                ldc, ScaleStride);\n        }\n        return 4;\n    } else {\n        if (QuantBZeroPoint != nullptr) {\n            SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen<true>(BlkLen, QuantA, QuantBData, QuantBScale, QuantBZeroPoint,\n                                                               C, CountM, CountN, BlockCountK, Bias, ldc, ScaleStride);\n        } else {\n            SQ4BitGemmM1Kernel_CompInt8_DispatchOnBlkLen<false>(BlkLen, QuantA, QuantBData, QuantBScale,\n                                                                QuantBZeroPoint, C, CountM, CountN, BlockCountK, Bias,\n                                                                ldc, ScaleStride);\n        }\n        return 1;\n    }\n}\n}  // namespace ime1\n}  // namespace sqnbitgemm_spacemit_ime\n"
  },
  {
    "path": "ggml/src/ggml-cpu/spacemit/ime_kernels.h",
    "content": "#pragma once\n\n#include <cstddef>\n\nnamespace sqnbitgemm_spacemit_ime {\nnamespace ime1 {\nsize_t gemm_kernel_i8i4(size_t            blk_len,\n                        const std::byte * quant_a_ptr,\n                        const std::byte * quant_b_data,\n                        const float *     quant_b_scale,\n                        const std::byte * quant_b_zp,\n                        float *           c_ptr,\n                        size_t            count_m,\n                        size_t            count_n,\n                        size_t            count_k,\n                        size_t            block_count_k,\n                        size_t            ldc,\n                        const float *     bias,\n                        const size_t      scale_stride);\n\nvoid quantize_a_row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);\n\nvoid quantize_a_4row_i8(size_t blk_len, const float * a_ptr, size_t count_k, std::byte * quant_a_ptr);\n\n}  // namespace ime1\n}  // namespace sqnbitgemm_spacemit_ime\n"
  },
  {
    "path": "ggml/src/ggml-cpu/traits.cpp",
    "content": "#include \"traits.h\"\n\n#include \"ggml-backend-impl.h\"\n#include \"ggml-backend.h\"\n\nnamespace ggml::cpu {\ntensor_traits::~tensor_traits() {}\n\nextra_buffer_type::~extra_buffer_type() {}\n}  // namespace ggml::cpu\n\nbool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) {\n    for (auto extra : ggml_backend_cpu_get_extra_buffer_types()) {\n        if (extra && extra->context) {\n            auto buf_extra     = (ggml::cpu::extra_buffer_type *) extra->context;\n            auto tensor_traits = buf_extra->get_tensor_traits(op);\n            if (tensor_traits && tensor_traits->compute_forward(params, op)) {\n                return true;\n            }\n        }\n    }\n    return false;\n}\n\nbool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size) {\n    for (auto extra : ggml_backend_cpu_get_extra_buffer_types()) {\n        if (extra && extra->context) {\n            auto buf_extra     = (ggml::cpu::extra_buffer_type *) extra->context;\n            auto tensor_traits = buf_extra->get_tensor_traits(op);\n            if (tensor_traits && tensor_traits->work_size(n_threads, op, *size)) {\n                return true;\n            }\n        }\n    }\n    return false;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/traits.h",
    "content": "#pragma once\n#include \"ggml-backend-impl.h\"\n#include \"ggml-cpu-impl.h\"\n#include \"ggml.h\"\n\n#ifdef __cplusplus\n#    include <vector>\nextern \"C\" {\n#endif\n\n// return true if op part of extra \"accelerator\"\nbool ggml_cpu_extra_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op);\nbool ggml_cpu_extra_work_size(int n_threads, const struct ggml_tensor * op, size_t * size);\n\n#ifdef __cplusplus\n}\n\nnamespace ggml::cpu {\n// register in tensor->extra\nclass tensor_traits {\n  public:\n    virtual ~tensor_traits();\n    virtual bool work_size(int n_threads, const struct ggml_tensor * op, size_t & size)        = 0;\n    virtual bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * op) = 0;\n};\n\nclass extra_buffer_type {\n  public:\n    virtual ~extra_buffer_type();\n    virtual bool            supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) = 0;\n    virtual tensor_traits * get_tensor_traits(const struct ggml_tensor * op)                   = 0;\n};\n}  // namespace ggml::cpu\n\n// implemented in ggml-cpu.cpp.\nstd::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffer_types();\n\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/unary-ops.cpp",
    "content": "#include \"unary-ops.h\"\n\nstatic inline float op_abs(float x) {\n    return fabsf(x);\n}\n\nstatic inline float op_sgn(float x) {\n    return (x > 0.f) ? 1.f : ((x < 0.f) ? -1.f : 0.f);\n}\n\nstatic inline float op_neg(float x) {\n    return -x;\n}\n\nstatic inline float op_step(float x) {\n    return (x > 0.f) ? 1.f : 0.f;\n}\n\nstatic inline float op_tanh(float x) {\n    return tanhf(x);\n}\n\nstatic inline float op_elu(float x) {\n    return (x > 0.f) ? x : expm1f(x);\n}\n\nstatic inline float op_relu(float x) {\n    return (x > 0.f) ? x : 0.f;\n}\n\nstatic inline float op_sigmoid(float x) {\n    return 1.f / (1.f + expf(-x));\n}\n\nstatic inline float op_hardsigmoid(float x) {\n    return fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic inline float op_exp(float x) {\n    return expf(x);\n}\n\nstatic inline float op_hardswish(float x) {\n    return x * fminf(1.0f, fmaxf(0.0f, (x + 3.0f) / 6.0f));\n}\n\nstatic inline float op_sqr(float x) {\n    return x * x;\n}\n\nstatic inline float op_sqrt(float x) {\n    return sqrtf(x);\n}\n\nstatic inline float op_xielu(float x, float alpha_n, float alpha_p, float beta, float eps) {\n    if (x > 0.0f) {\n        return alpha_p * x * x + beta * x;\n    } else {\n        const float min_x_eps = fminf(x, eps);\n        return (expm1f(min_x_eps) - x) * alpha_n + beta * x;\n    }\n}\n\nstatic inline float op_sin(float x) {\n    return sinf(x);\n}\n\nstatic inline float op_cos(float x) {\n    return cosf(x);\n}\n\nstatic inline float op_log(float x) {\n    return logf(x);\n}\n\nstatic inline float op_expm1(float x) {\n    return expf(x) - 1.0f;\n}\n\nstatic inline float op_softplus(float x) {\n    return (x > 20.0f) ? x : logf(1.0f + expf(x));\n}\n\nstatic inline float op_floor(float x) {\n    return floorf(x);\n}\n\nstatic inline float op_ceil(float x) {\n    return ceilf(x);\n}\n\nstatic inline float op_round(float x) {\n    return roundf(x);\n}\n\nstatic inline float op_trunc(float x) {\n    return truncf(x);\n}\n\ntemplate <float (*op)(float), typename src0_t, typename dst_t>\nstatic inline void vec_unary_op(int64_t n, dst_t * y, const src0_t * x) {\n    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;\n    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;\n\n    for (int i = 0; i < n; i++) {\n        y[i] = f32_to_dst(op(src0_to_f32(x[i])));\n    }\n}\n\ntemplate <float (*op)(float), typename src0_t, typename dst_t>\nstatic void apply_unary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(dst) && ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT( nb0 == sizeof(dst_t));\n    GGML_ASSERT(nb00 == sizeof(src0_t));\n\n    const auto [ir0, ir1] = get_thread_range(params, src0);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );\n        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n\n        vec_unary_op<op>(ne0, dst_ptr, src0_ptr);\n    }\n}\n\n// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates\ntemplate <float (*op)(float)>\nstatic void unary_op(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    /*  */ if (src0->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) { // all f32\n        apply_unary_op<op, float, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F16) { // all f16\n        apply_unary_op<op, ggml_fp16_t, ggml_fp16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n        apply_unary_op<op, ggml_bf16_t, ggml_bf16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n        apply_unary_op<op, ggml_bf16_t, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F32) {\n        apply_unary_op<op, ggml_fp16_t, float>(params, dst);\n    } else {\n        fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n            ggml_type_name(dst->type), ggml_type_name(src0->type));\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\ntemplate <float (*op)(float, ggml_tensor *)>\nstatic void unary_op_params(const ggml_compute_params * params, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    /*  */ if (src0->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) { // all f32\n        apply_unary_op<op, float, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F16) { // all f16\n        apply_unary_op<op, ggml_fp16_t, ggml_fp16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n        apply_unary_op<op, ggml_bf16_t, ggml_bf16_t>(params, dst);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n        apply_unary_op<op, ggml_bf16_t, float>(params, dst);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F32) {\n        apply_unary_op<op, ggml_fp16_t, float>(params, dst);\n    } else {\n        fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n            ggml_type_name(dst->type), ggml_type_name(src0->type));\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\n// Extend vec_unary_op to support functors\ntemplate <typename Op, typename src0_t, typename dst_t>\nstatic inline void vec_unary_op_functor(int64_t n, dst_t * y, const src0_t * x, Op op) {\n    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;\n    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;\n\n    for (int i = 0; i < n; i++) {\n        y[i] = f32_to_dst(op(src0_to_f32(x[i])));\n    }\n}\n\n// Extend apply_unary_op to support functors\ntemplate <typename Op, typename src0_t, typename dst_t>\nstatic void apply_unary_op_functor(const ggml_compute_params * params, ggml_tensor * dst, Op op) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(ggml_is_contiguous_1(src0) && ggml_is_contiguous_1(dst) && ggml_are_same_shape(src0, dst));\n\n    GGML_TENSOR_UNARY_OP_LOCALS\n\n    GGML_ASSERT( nb0 == sizeof(dst_t));\n    GGML_ASSERT(nb00 == sizeof(src0_t));\n\n    const auto [ir0, ir1] = get_thread_range(params, src0);\n\n    for (int64_t ir = ir0; ir < ir1; ++ir) {\n        const int64_t i03 = ir/(ne02*ne01);\n        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;\n        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);\n\n        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );\n        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);\n\n        vec_unary_op_functor(ne0, dst_ptr, src0_ptr, op);\n    }\n}\n\n// Generic dispatcher for functors\ntemplate <typename Op>\nstatic void unary_op_functor(const ggml_compute_params * params, ggml_tensor * dst, Op op) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    /*  */ if (src0->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) { // all f32\n        apply_unary_op_functor<Op, float, float>(params, dst, op);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F16) { // all f16\n        apply_unary_op_functor<Op, ggml_fp16_t, ggml_fp16_t>(params, dst, op);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16\n        apply_unary_op_functor<Op, ggml_bf16_t, ggml_bf16_t>(params, dst, op);\n    } else if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_F32) {\n        apply_unary_op_functor<Op, ggml_bf16_t, float>(params, dst, op);\n    } else if (src0->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F32) {\n        apply_unary_op_functor<Op, ggml_fp16_t, float>(params, dst, op);\n    } else {\n        fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s\\n\", __func__,\n            ggml_type_name(dst->type), ggml_type_name(src0->type));\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\nvoid ggml_compute_forward_abs(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_abs>(params, dst);\n}\n\nvoid ggml_compute_forward_sgn(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_sgn>(params, dst);\n}\n\nvoid ggml_compute_forward_neg(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_neg>(params, dst);\n}\n\nvoid ggml_compute_forward_step(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_step>(params, dst);\n}\n\nvoid ggml_compute_forward_tanh(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_tanh>(params, dst);\n}\n\nvoid ggml_compute_forward_elu(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_elu>(params, dst);\n}\n\nvoid ggml_compute_forward_relu(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_relu>(params, dst);\n}\n\nvoid ggml_compute_forward_sigmoid(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_sigmoid>(params, dst);\n}\n\nvoid ggml_compute_forward_hardsigmoid(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_hardsigmoid>(params, dst);\n}\n\nvoid ggml_compute_forward_exp(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_exp>(params, dst);\n}\n\nvoid ggml_compute_forward_hardswish(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_hardswish>(params, dst);\n}\n\nvoid ggml_compute_forward_sqr(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_sqr>(params, dst);\n}\n\nvoid ggml_compute_forward_sqrt(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_sqrt>(params, dst);\n}\n\nvoid ggml_compute_forward_sin(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_sin>(params, dst);\n}\n\nvoid ggml_compute_forward_cos(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_cos>(params, dst);\n}\n\nvoid ggml_compute_forward_log(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_log>(params, dst);\n}\n\nvoid ggml_compute_forward_expm1(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_expm1>(params, dst);\n}\n\nvoid ggml_compute_forward_softplus(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_softplus>(params, dst);\n}\n\nvoid ggml_compute_forward_floor(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_floor>(params, dst);\n}\n\nvoid ggml_compute_forward_ceil(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_ceil>(params, dst);\n}\n\nvoid ggml_compute_forward_round(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_round>(params, dst);\n}\n\nvoid ggml_compute_forward_trunc(const ggml_compute_params * params, ggml_tensor * dst) {\n    unary_op<op_trunc>(params, dst);\n}\n\nvoid ggml_compute_forward_xielu(const ggml_compute_params * params, ggml_tensor * dst) {\n    const float alpha_n = ggml_get_op_params_f32(dst, 1);\n    const float alpha_p = ggml_get_op_params_f32(dst, 2);\n    const float beta = ggml_get_op_params_f32(dst, 3);\n    const float eps = ggml_get_op_params_f32(dst, 4);\n\n    const auto xielu_op_params = [alpha_n, alpha_p, beta, eps](float f) {\n        return op_xielu(f, alpha_n, alpha_p, beta, eps);\n    };\n\n    unary_op_functor(params, dst, xielu_op_params);\n}\n\n"
  },
  {
    "path": "ggml/src/ggml-cpu/unary-ops.h",
    "content": "#pragma once\n\n#include \"common.h\"\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nvoid ggml_compute_forward_abs(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sgn(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_neg(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_step(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_tanh(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_elu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sigmoid(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_hardsigmoid(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_exp(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_hardswish(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sqr(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sqrt(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_sin(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_cos(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_log(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_expm1(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_softplus(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_floor(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_ceil(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_round(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_trunc(const struct ggml_compute_params * params, struct ggml_tensor * dst);\nvoid ggml_compute_forward_xielu(const struct ggml_compute_params * params, struct ggml_tensor * dst);\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cpu/vec.cpp",
    "content": "#include \"vec.h\"\n\n#include <cassert>\n\n// precomputed gelu table for f16 (128 KB)\nggml_fp16_t ggml_table_gelu_f16[1 << 16];\n\n// precomputed quick gelu table for f16 (128 KB)\nggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];\n\nvoid ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {\n   assert(nrc == 1);\n   GGML_UNUSED(nrc);\n   GGML_UNUSED(bx);\n   GGML_UNUSED(by);\n   GGML_UNUSED(bs);\n\n#if defined(GGML_SIMD)\n    float sumf = 0.0f;\n\n    #if defined(__ARM_FEATURE_SVE)\n        const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n        const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n        const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers\n\n        const int np = (n & ~(ggml_f32_step - 1));\n        svfloat32_t sum1 = svdup_n_f32(0.0f);\n        svfloat32_t sum2 = svdup_n_f32(0.0f);\n        svfloat32_t sum3 = svdup_n_f32(0.0f);\n        svfloat32_t sum4 = svdup_n_f32(0.0f);\n        svfloat32_t sum5 = svdup_n_f32(0.0f);\n        svfloat32_t sum6 = svdup_n_f32(0.0f);\n        svfloat32_t sum7 = svdup_n_f32(0.0f);\n        svfloat32_t sum8 = svdup_n_f32(0.0f);\n        svfloat32_t ax1,ax2,ax3,ax4,ax5,ax6,ax7,ax8;\n        svfloat32_t ay1,ay2,ay3,ay4,ay5,ay6,ay7,ay8;\n        for (int i = 0; i < np; i += ggml_f32_step) {\n            ax1 = GGML_F32_VEC_LOAD(x + i);\n            ay1 = GGML_F32_VEC_LOAD(y + i);\n            sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);\n\n            ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);\n            ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n            sum2 = GGML_F32_VEC_FMA(sum2, ax2, ay2);\n\n            ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);\n            ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);\n            sum3 = GGML_F32_VEC_FMA(sum3, ax3, ay3);\n\n            ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);\n            ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);\n            sum4 = GGML_F32_VEC_FMA(sum4, ax4, ay4);\n\n            ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);\n            ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);\n            sum5 = GGML_F32_VEC_FMA(sum5, ax5, ay5);\n\n            ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);\n            ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);\n            sum6 = GGML_F32_VEC_FMA(sum6, ax6, ay6);\n\n            ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);\n            ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);\n            sum7 = GGML_F32_VEC_FMA(sum7, ax7, ay7);\n\n            ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);\n            ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);\n            sum8 = GGML_F32_VEC_FMA(sum8, ax8, ay8);\n        }\n        // leftovers\n        // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop\n        const int np2 = (n & ~(ggml_f32_epr - 1));\n        for (int i = np; i < np2; i += ggml_f32_epr) {\n            ax1 = GGML_F32_VEC_LOAD(x + i);\n            ay1 = GGML_F32_VEC_LOAD(y + i);\n            sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);\n        }\n        // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n        if (np2 < n) {\n            svbool_t pg = svwhilelt_b32(np2, n);\n            ax1 = svld1_f32(pg, x + np2);\n            ay1 = svld1_f32(pg, y + np2);\n            sum1 = svmad_f32_m(pg, ax1, ay1, sum1);\n        }\n        // reduce sum1,sum2 to sum1\n        GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);\n    #elif defined(__riscv_v_intrinsic)\n        int vl = __riscv_vsetvlmax_e32m8();\n        vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);\n        vfloat32m8_t vsum;\n        vfloat32m8_t ax;\n        vfloat32m8_t ay;\n        vsum = __riscv_vfmv_v_f_f32m8_tu(vsum, 0.0f, vl);\n        for (int i = 0; i < n; i += vl) {\n            vl = __riscv_vsetvl_e32m8(n - i);\n            ax = __riscv_vle32_v_f32m8_tu(ax, &x[i], vl);\n            ay = __riscv_vle32_v_f32m8_tu(ay, &y[i], vl);\n            vsum = __riscv_vfmacc_vv_f32m8_tu(vsum, ax, ay, vl);\n        }\n        vl = __riscv_vsetvlmax_e32m8();\n        vs = __riscv_vfredusum_vs_f32m8_f32m1(vsum, vs, vl);\n        sumf += __riscv_vfmv_f_s_f32m1_f32(vs);\n    #else\n        const int np = (n & ~(GGML_F32_STEP - 1));\n\n        GGML_F32_VEC sum[GGML_F32_ARR] = { GGML_F32_VEC_ZERO };\n\n        GGML_F32_VEC ax[GGML_F32_ARR];\n        GGML_F32_VEC ay[GGML_F32_ARR];\n\n        for (int i = 0; i < np; i += GGML_F32_STEP) {\n            for (int j = 0; j < GGML_F32_ARR; j++) {\n                ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n                ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n\n                sum[j] = GGML_F32_VEC_FMA(sum[j], ax[j], ay[j]);\n            }\n        }\n\n        // reduce sum0..sum3 to sum0\n        GGML_F32_VEC_REDUCE(sumf, sum);\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            sumf += x[i]*y[i];\n        }\n    #endif\n#else\n    // scalar\n    ggml_float sumf = 0.0;\n    for (int i = 0; i < n; ++i) {\n        sumf += (ggml_float)(x[i]*y[i]);\n    }\n#endif\n\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc) {\n    assert(nrc == 1);\n    GGML_UNUSED(nrc);\n    GGML_UNUSED(bx);\n    GGML_UNUSED(by);\n    GGML_UNUSED(bs);\n    int i = 0;\n    ggml_float sumf = 0;\n\n#if defined(__AVX512BF16__)\n    __m512 c1 = _mm512_setzero_ps();\n    __m512 c2 = _mm512_setzero_ps();\n    for (; i + 64 <= n; i += 64) {\n        c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))),\n                             m512bh(_mm512_loadu_si512((y + i))));\n        c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))),\n                             m512bh(_mm512_loadu_si512((y + i + 32))));\n    }\n    sumf += (ggml_float)_mm512_reduce_add_ps(c1);\n    sumf += (ggml_float)_mm512_reduce_add_ps(c2);\n\n#elif defined(__AVX512F__)\n#define LOAD(p) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)(p))), 16))\n    __m512 c1 = _mm512_setzero_ps();\n    __m512 c2 = _mm512_setzero_ps();\n    for (; i + 32 <= n; i += 32) {\n        c1 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i), LOAD(y + i)), c1);\n        c2 = _mm512_add_ps(_mm512_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c2);\n    }\n    sumf += (ggml_float)_mm512_reduce_add_ps(c1);\n    sumf += (ggml_float)_mm512_reduce_add_ps(c2);\n\n#undef LOAD\n#elif defined(__AVX2__) || defined(__AVX__)\n#if defined(__AVX2__)\n#define LOAD(p) _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16))\n#else\n#define LOAD(p) _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)(p))), 16)), (_mm_slli_epi32(_mm_cvtepu16_epi32(_mm_bsrli_si128(_mm_loadu_si128((const __m128i *)(p)), 8)), 16)), 1))\n#endif\n    __m256 c1 = _mm256_setzero_ps();\n    __m256 c2 = _mm256_setzero_ps();\n    __m256 c3 = _mm256_setzero_ps();\n    __m256 c4 = _mm256_setzero_ps();\n    for (; i + 32 <= n; i += 32) {\n        c1 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i), LOAD(y + i)), c1);\n        c2 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 8), LOAD(y + i + 8)), c2);\n        c3 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 16), LOAD(y + i + 16)), c3);\n        c4 = _mm256_add_ps(_mm256_mul_ps(LOAD(x + i + 24), LOAD(y + i + 24)), c4);\n    }\n    __m128 g;\n    c1 = _mm256_add_ps(_mm256_add_ps(c1, c3),\n                       _mm256_add_ps(c2, c4));\n    g = _mm_add_ps(_mm256_extractf128_ps(c1, 1),\n                   _mm256_castps256_ps128(c1));\n    g = _mm_add_ps(g, _mm_movehl_ps(g, g));\n    g = _mm_add_ss(g, _mm_movehdup_ps(g));\n    sumf += (ggml_float)_mm_cvtss_f32(g);\n\n#undef LOAD\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma)\n    size_t vl = __riscv_vsetvlmax_e32m4();\n\n    // initialize accumulators to all zeroes\n    vfloat32m4_t vsum0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n    vfloat32m4_t vsum1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n\n    // calculate step size\n    const size_t epr = __riscv_vsetvlmax_e16m2();\n    const size_t step = epr * 2;\n    const int np = (n & ~(step - 1));\n\n    // unroll by 2\n    for (; i < np; i += step) {\n        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], epr);\n        vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], epr);\n        vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n\n        vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i + epr], epr);\n        vbfloat16m2_t ay1 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i + epr], epr);\n        vsum1 = __riscv_vfwmaccbf16_vv_f32m4(vsum1, ax1, ay1, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n    }\n\n    // accumulate in 1 register\n    vsum0 = __riscv_vfadd_vv_f32m4(vsum0, vsum1, vl);\n\n    // leftovers\n    for (i = np; i < n; i += vl) {\n        vl = __riscv_vsetvl_e16m2(n - i);\n        vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16 *)&x[i], vl);\n        vbfloat16m2_t ay0 = __riscv_vle16_v_bf16m2((const __bf16 *)&y[i], vl);\n        vsum0 = __riscv_vfwmaccbf16_vv_f32m4(vsum0, ax0, ay0, vl);\n    }\n\n    // reduce\n    vl = __riscv_vsetvlmax_e32m4();\n    vfloat32m1_t redsum = __riscv_vfredusum_vs_f32m4_f32m1(vsum0, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n    sumf += __riscv_vfmv_f_s_f32m1_f32(redsum);\n\n#elif defined(__POWER9_VECTOR__) || defined(__VXE__) || defined(__VXE2__)\n    const int np = (n & ~(GGML_BF16_STEP - 1));\n    if (np > 0) {\n        GGML_F32_VEC sum[4] = {GGML_F32_VEC_ZERO};\n        for (; i < np; i += GGML_BF16_STEP) {\n            GGML_BF16_VEC vx0 = GGML_BF16_VEC_LOAD(x + i);\n            GGML_BF16_VEC vx1 = GGML_BF16_VEC_LOAD(x + i + 8);\n            GGML_BF16_VEC vy0 = GGML_BF16_VEC_LOAD(y + i);\n            GGML_BF16_VEC vy1 = GGML_BF16_VEC_LOAD(y + i + 8);\n            GGML_BF16_FMA_LO(sum[0], vx0, vy0);\n            GGML_BF16_FMA_HI(sum[1], vx0, vy0);\n            GGML_BF16_FMA_LO(sum[2], vx1, vy1);\n            GGML_BF16_FMA_HI(sum[3], vx1, vy1);\n        }\n        GGML_F32x4_REDUCE_4(sumf, sum[0], sum[1], sum[2], sum[3]);\n    }\n#endif\n\n    for (; i < n; ++i) {\n        sumf += (ggml_float)(GGML_BF16_TO_FP32(x[i]) *\n                             GGML_BF16_TO_FP32(y[i]));\n    }\n    *s = sumf;\n}\n\nvoid ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc) {\n    assert(nrc == 1);\n    GGML_UNUSED(nrc);\n    GGML_UNUSED(bx);\n    GGML_UNUSED(by);\n    GGML_UNUSED(bs);\n\n    ggml_float sumf = 0.0;\n\n\n#if defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n        const int sve_register_length = svcntb() * 8; //get vector length\n        const int ggml_f16_epr = sve_register_length / 16; // running when 16\n        const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers\n\n        const int np= (n & ~(ggml_f16_step - 1));\n        svfloat16_t sum1 = svdup_n_f16(0.0f);\n        svfloat16_t sum2 = svdup_n_f16(0.0f);\n        svfloat16_t sum3 = svdup_n_f16(0.0f);\n        svfloat16_t sum4 = svdup_n_f16(0.0f);\n\n        svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n        svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n        for (int i = 0; i < np; i += ggml_f16_step) {\n            ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0);\n            ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0);\n            sum1 = GGML_F16x_VEC_FMA(sum1, ax1, ay1);\n\n            ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1);\n            ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1);\n            sum2 = GGML_F16x_VEC_FMA(sum2, ax2, ay2);\n\n            ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2);\n            ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n            sum3 = GGML_F16x_VEC_FMA(sum3, ax3, ay3);\n\n            ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3);\n            ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n            sum4 = GGML_F16x_VEC_FMA(sum4, ax4, ay4);\n\n            ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4);\n            ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n            sum1 = GGML_F16x_VEC_FMA(sum1, ax5, ay5);\n\n            ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5);\n            ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n            sum2 = GGML_F16x_VEC_FMA(sum2, ax6, ay6);\n\n            ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6);\n            ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n            sum3 = GGML_F16x_VEC_FMA(sum3, ax7, ay7);\n\n            ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7);\n            ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n            sum4 = GGML_F16x_VEC_FMA(sum4, ax8, ay8);\n        }\n\n        const int np2 = (n & ~(ggml_f16_epr - 1)); // round down to multiple of 8\n        for (int k = np; k < np2; k += ggml_f16_epr) {\n            svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0);\n            svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n            sum1 = GGML_F16x_VEC_FMA(sum1, rx, ry);\n        }\n\n        if (np2 < n) {\n            svbool_t pg = svwhilelt_b16(np2, n);\n            svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2));\n            svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n\n            sum1 = svmad_f16_x(pg, hx, hy, sum1);\n        }\n        GGML_F16x_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4);\n    #elif defined(__riscv_v_intrinsic)\n        #if defined(__riscv_zvfh)\n            int vl = __riscv_vsetvlmax_e32m2();\n            vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1);\n            vfloat32m2_t vsum;\n            vfloat16m1_t ax;\n            vfloat16m1_t ay;\n            vsum = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vmv_v_x_u32m2(0, vl));\n            for (int i = 0; i < n; i += vl) {\n                vl = __riscv_vsetvl_e16m1(n - i);\n                ax = __riscv_vle16_v_f16m1_tu(ax, (const _Float16 *)&x[i], vl);\n                ay = __riscv_vle16_v_f16m1_tu(ay, (const _Float16 *)&y[i], vl);\n                vsum = __riscv_vfwmacc_vv_f32m2_tu(vsum, ax, ay, vl);\n            }\n            vl = __riscv_vsetvlmax_e32m1();\n            vfloat32m1_t ac0 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(vsum, 0), __riscv_vget_v_f32m2_f32m1(vsum, 1), vl);\n            vs = __riscv_vfredusum_vs_f32m1_f32m1(ac0, vs, vl);\n            sumf += __riscv_vfmv_f_s_f32m1_f32(vs);\n        #else\n            for (int i = 0; i < n; ++i) {\n                sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n            }\n        #endif // __riscv_zvfh\n    #else\n        const int np = (n & ~(GGML_F16_STEP - 1));\n\n        GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO };\n\n        GGML_F16_VEC ax[GGML_F16_ARR];\n        GGML_F16_VEC ay[GGML_F16_ARR];\n\n        for (int i = 0; i < np; i += GGML_F16_STEP) {\n            for (int j = 0; j < GGML_F16_ARR; j++) {\n                ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);\n                ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n\n                sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]);\n            }\n        }\n\n        // reduce sum0..sum3 to sum0\n        GGML_F16_VEC_REDUCE(sumf, sum);\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n        }\n        // if you hit this, you are likely running outside the FP range\n        assert(!isnan(sumf) && !isinf(sumf));\n    #endif\n#else\n    for (int i = 0; i < n; ++i) {\n        sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*GGML_CPU_FP16_TO_FP32(y[i]));\n    }\n#endif // GGML_SIMD\n\n    *s = sumf;\n}\n\nvoid ggml_vec_silu_f32(const int n, float * y, const float * x) {\n    int i = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n    for (; i + 15 < n; i += 16) {\n        _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i)));\n    }\n#elif defined(__AVX2__) && defined(__FMA__)\n    for (; i + 7 < n; i += 8) {\n        _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i)));\n    }\n#elif defined(__SSE2__)\n    for (; i + 3 < n; i += 4) {\n        _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i)));\n    }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n    const int vlen = svcntw();\n    for (; i < n; i += vlen) {\n        const svbool_t pg = svwhilelt_b32_s32(i, n);\n        svst1_f32(pg, y + i, ggml_v_silu(pg, svld1_f32(pg, x + i)));\n    }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n    for (; i + 3 < n; i += 4) {\n        vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i)));\n    }\n#elif defined(__riscv_v_intrinsic)\n    for (int vl; i < n; i += vl) {\n        vl = __riscv_vsetvl_e32m2(n - i);\n        vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n        vfloat32m2_t vy = ggml_v_silu_m2(vx, vl);\n        __riscv_vse32_v_f32m2(&y[i], vy, vl);\n    }\n#endif\n    for (; i < n; ++i) {\n        y[i] = ggml_silu_f32(x[i]);\n    }\n}\n\nvoid ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g) {\n    int i = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n    for (; i + 15 < n; i += 16) {\n        _mm512_storeu_ps(y + i, _mm512_mul_ps(ggml_v_silu(_mm512_loadu_ps(x + i)), _mm512_loadu_ps(g + i)));\n    }\n#elif defined(__AVX2__) && defined(__FMA__)\n    for (; i + 7 < n; i += 8) {\n        _mm256_storeu_ps(y + i, _mm256_mul_ps(ggml_v_silu(_mm256_loadu_ps(x + i)), _mm256_loadu_ps(g + i)));\n    }\n#elif defined(__SSE2__)\n    for (; i + 3 < n; i += 4) {\n        _mm_storeu_ps(y + i, _mm_mul_ps(ggml_v_silu(_mm_loadu_ps(x + i)), _mm_loadu_ps(g + i)));\n    }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n    const int vlen = svcntw();\n    for (; i < n; i += vlen) {\n        const svbool_t pg = svwhilelt_b32_s32(i, n);\n        svst1_f32(pg, y + i, svmul_f32_x(pg, ggml_v_silu(pg, svld1_f32(pg, x + i)), svld1_f32(pg, g + i)));\n    }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n    for (; i + 3 < n; i += 4) {\n        vst1q_f32(y + i, vmulq_f32(ggml_v_silu(vld1q_f32(x + i)), vld1q_f32(g + i)));\n    }\n#elif defined(__riscv_v_intrinsic)\n    for (int vl; i < n; i += vl) {\n        vl = __riscv_vsetvl_e32m2(n - i);\n        vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);\n        vfloat32m2_t vg = __riscv_vle32_v_f32m2(&g[i], vl);\n        vfloat32m2_t vy = __riscv_vfmul_vv_f32m2(ggml_v_silu_m2(vx, vl), vg, vl);\n        __riscv_vse32_v_f32m2(&y[i], vy, vl);\n    }\n#endif\n    for (; i < n; ++i) {\n        y[i] = ggml_silu_f32(x[i]) * g[i];\n    }\n}\n\nggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean) {\n    int i = 0;\n    ggml_float sum = 0;\n// TODO: optimize to process the remaining elements in groups using the smaller vector sizes from AVX2 and SSE\n// ref: https://github.com/ggml-org/llama.cpp/pull/15953#pullrequestreview-3310928344\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n    for (; i + 15 < n; i += 16) {\n        __m512 val = _mm512_sub_ps(_mm512_loadu_ps(x + i),\n                                   _mm512_set1_ps(mean));\n        _mm512_storeu_ps(y + i, val);\n        sum += (ggml_float)_mm512_reduce_add_ps(_mm512_mul_ps(val, val));\n    }\n#elif defined(__AVX2__) && defined(__FMA__)\n    for (; i + 7 < n; i += 8) {\n        __m256 val = _mm256_sub_ps(_mm256_loadu_ps(x + i),\n                                   _mm256_set1_ps(mean));\n        _mm256_storeu_ps(y + i, val);\n        val = _mm256_mul_ps(val,val);\n        __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),\n                                 _mm256_castps256_ps128(val));\n        val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));\n        val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));\n        sum += (ggml_float)_mm_cvtss_f32(val2);\n    }\n#elif defined(__SSE2__)\n    for (; i + 3 < n; i += 4) {\n        __m128 val = _mm_sub_ps(_mm_loadu_ps(x + i),\n                                _mm_set1_ps(mean));\n        _mm_storeu_ps(y + i, val);\n        val = _mm_mul_ps(val, val);\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n        val = _mm_add_ps(val, _mm_movehl_ps(val, val));\n        val = _mm_add_ss(val, _mm_movehdup_ps(val));\n#else\n        __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));\n        val = _mm_add_ps(val, tmp);\n        tmp = _mm_movehl_ps(tmp, val);\n        val = _mm_add_ss(val, tmp);\n#endif  // __AVX__ || __AVX2__ || __AVX512F__\n        sum += (ggml_float)_mm_cvtss_f32(val);\n    }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n    for (; i + 3 < n; i += 4) {\n        float32x4_t val = vsubq_f32(vld1q_f32(x + i),\n                                    vdupq_n_f32(mean));\n        vst1q_f32(y + i, val);\n        val = vmulq_f32(val, val);\n        sum += (ggml_float)vaddvq_f32(val);\n    }\n#elif defined(__VXE__) || defined(__VXE2__)\n    for (; i + 3 < n; i += 4) {\n        float32x4_t val = vec_sub(vec_xl(0, x + i), vec_splats(mean));\n        vec_xst(val, 0, y + i);\n        val = vec_mul(val, val);\n        sum += (ggml_float)vec_hsum_f32x4(val);\n    }\n#elif defined(__riscv_v_intrinsic)\n    vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);\n    for (int vl; i < n; i += vl) {\n        vl = __riscv_vsetvl_e32m2(n - i);\n        vfloat32m2_t val = __riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], vl), mean, vl);\n        __riscv_vse32_v_f32m2(&y[i], val, vl);\n        val = __riscv_vfmul_vv_f32m2(val, val, vl);\n        vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, vl);\n    }\n    sum = (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);\n#endif\n    for (; i < n; ++i) {\n        float val = x[i] - mean;\n        y[i] = val;\n        val *= val;\n        sum += (ggml_float)val;\n    }\n    return sum/n;\n}\n\nggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) {\n    int i = 0;\n    ggml_float sum = 0;\n#if defined(__AVX512F__) && defined(__AVX512DQ__)\n    for (; i + 15 < n; i += 16) {\n        __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i),\n                                               _mm512_set1_ps(max)));\n        _mm512_storeu_ps(y + i, val);\n        sum += (ggml_float)_mm512_reduce_add_ps(val);\n    }\n#elif defined(__AVX2__) && defined(__FMA__)\n    for (; i + 7 < n; i += 8) {\n        __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i),\n                                               _mm256_set1_ps(max)));\n        _mm256_storeu_ps(y + i, val);\n        __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1),\n                                 _mm256_castps256_ps128(val));\n        val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2));\n        val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2));\n        sum += (ggml_float)_mm_cvtss_f32(val2);\n    }\n#elif defined(__SSE2__)\n    for (; i + 3 < n; i += 4) {\n        __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i),\n                                            _mm_set1_ps(max)));\n        _mm_storeu_ps(y + i, val);\n#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)\n        val = _mm_add_ps(val, _mm_movehl_ps(val, val));\n        val = _mm_add_ss(val, _mm_movehdup_ps(val));\n#else\n        __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1));\n        val = _mm_add_ps(val, tmp);\n        tmp = _mm_movehl_ps(tmp, val);\n        val = _mm_add_ss(val, tmp);\n#endif\n        sum += (ggml_float)_mm_cvtss_f32(val);\n    }\n#elif defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n    const int vlen = svcntw();\n    for (; i < n; i += vlen) {\n        const svbool_t pg = svwhilelt_b32_s32(i, n);\n        svfloat32_t val = ggml_v_expf(pg, svsub_f32_x(pg, svld1_f32(pg, x + i),\n                                                svdup_n_f32_x(pg, max)));\n        svst1_f32(pg, y + i, val);\n        sum += (ggml_float)svaddv_f32(pg, val);\n    }\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n    for (; i + 3 < n; i += 4) {\n        float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i),\n                                                vdupq_n_f32(max)));\n        vst1q_f32(y + i, val);\n        sum += (ggml_float)vaddvq_f32(val);\n    }\n#elif defined(__riscv_v_intrinsic)\n    vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1);\n    for (int avl; i < n; i += avl) {\n        avl = __riscv_vsetvl_e32m2(n - i);\n        vfloat32m2_t val = ggml_v_expf_m2(__riscv_vfsub_vf_f32m2(__riscv_vle32_v_f32m2(&x[i], avl), max, avl), avl);\n        __riscv_vse32_v_f32m2(&y[i], val, avl);\n        vsum = __riscv_vfwredusum_vs_f32m2_f64m1(val, vsum, avl);\n    }\n    return (ggml_float)__riscv_vfmv_f_s_f64m1_f64(vsum);\n#endif\n    for (; i < n; ++i) {\n        float val = expf(x[i] - max);\n        sum += (ggml_float)val;\n        y[i] = val;\n    }\n    return sum;\n}\n\nggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max) {\n    // log(soft_max) = log(soft_max_i / soft_max_sum) = log(soft_max_i) - log(soft_max_sum) = (logit_i - max) - log(soft_max_i)\n\n    int i = 0;\n    ggml_float sum = 0;\n    for (; i < n; ++i) {\n        float val = x[i] - max;\n        y[i] = val;\n        sum += (ggml_float)expf(val);\n    }\n    return sum = (ggml_float)logf(sum);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cpu/vec.h",
    "content": "// Vectorized functions for fundamental operations\n\n#pragma once\n\n#include \"ggml-impl.h\"\n#include \"simd-mappings.h\"\n#include \"ggml.h\"\n#include \"ggml-cpu.h\"\n\n#if defined(GGML_USE_ACCELERATE)\n#include <Accelerate/Accelerate.h>\n#endif\n\n// floating point type used to accumulate sums\ntypedef double ggml_float;\n\n#define GGML_GELU_FP16\n#define GGML_GELU_QUICK_FP16\n\n#define GGML_SOFT_MAX_UNROLL 4\n#define GGML_VEC_DOT_UNROLL  2\n#define GGML_VEC_MAD_UNROLL  32\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n//\n// global data\n//\n\n// precomputed gelu table for f16 (128 KB)\nextern ggml_fp16_t ggml_table_gelu_f16[1 << 16];\n\n// precomputed quick gelu table for f16 (128 KB)\nextern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];\n\n//\n// fundamental operations\n//\n\nvoid ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);\nvoid ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);\nvoid ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);\n\nvoid ggml_vec_silu_f32(const int n, float * y, const float * x);\nggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean )\nggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max);\nggml_float ggml_vec_log_soft_max_f32(const int n, float * y, const float * x, float max);\n\ninline static void ggml_vec_set_i8(const int n, int8_t * x, const int8_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_set_i16(const int n, int16_t * x, const int16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\n\ninline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t   v) { for (int i = 0; i < n; ++i) x[i] = v;    }\ninline static void ggml_vec_cpy_i32(const int n, int32_t * y, const int32_t * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; }\n\ninline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const ggml_fp16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\ninline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }\n\ninline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) {\n    int i = 0;\n#if defined(__AVX2__)\n    for (; i + 7 < n; i += 8) {\n        __m256 vx = _mm256_loadu_ps(x + i);\n        __m256 vy = _mm256_loadu_ps(y + i);\n        __m256 vz = _mm256_add_ps(vx, vy);\n        _mm256_storeu_ps(z + i, vz);\n    }\n#endif\n    for (; i < n; ++i) {\n        z[i] = x[i] + y[i];\n    }\n}\n\ninline static void ggml_vec_add_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n    for (int i = 0; i < n; ++i) {\n        z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) + GGML_CPU_FP16_TO_FP32(y[i]));\n    }\n}\ninline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float   v) { for (int i = 0; i < n; ++i) z[i]  = x[i] + v;    }\ninline static void ggml_vec_acc_f32 (const int n, float * y, const float * x)                  { for (int i = 0; i < n; ++i) y[i] += x[i];        }\ninline static void ggml_vec_acc1_f32(const int n, float * y, const float   v)                  { for (int i = 0; i < n; ++i) y[i] += v;           }\ninline static void ggml_vec_sub_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i]  = x[i] - y[i]; }\ninline static void ggml_vec_sub_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n    for (int i = 0; i < n; ++i) {\n        z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) - GGML_CPU_FP16_TO_FP32(y[i]));\n    }\n}\ninline static void ggml_vec_set_f32 (const int n, float * x, const float   v)                  { for (int i = 0; i < n; ++i) x[i]  = v;           }\ninline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x)                  { for (int i = 0; i < n; ++i) y[i]  = x[i];        }\ninline static void ggml_vec_neg_f32 (const int n, float * y, const float * x)                  { for (int i = 0; i < n; ++i) y[i]  = -x[i];       }\ninline static void ggml_vec_neg_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(-GGML_CPU_FP16_TO_FP32(x[i]));\n    }\n}\n\ninline static void ggml_vec_mul_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i]  = x[i]*y[i];   }\ninline static void ggml_vec_mul_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n    for (int i = 0; i < n; ++i) {\n        z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) * GGML_CPU_FP16_TO_FP32(y[i]));\n    }\n}\ninline static void ggml_vec_div_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i]  = x[i]/y[i];   }\ninline static void ggml_vec_div_f16 (const int n, ggml_fp16_t * z, const ggml_fp16_t * x, const ggml_fp16_t * y) {\n    for (int i = 0; i < n; ++i) {\n        z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) / GGML_CPU_FP16_TO_FP32(y[i]));\n    }\n}\n\n// compute GGML_VEC_DOT_UNROLL dot products at once\n// xs - x row stride in bytes\ninline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GGML_RESTRICT s, void * GGML_RESTRICT xv, ggml_fp16_t * GGML_RESTRICT y) {\n    ggml_float sumf[GGML_VEC_DOT_UNROLL] = { 0.0 };\n\n    ggml_fp16_t * GGML_RESTRICT x[GGML_VEC_DOT_UNROLL];\n\n    for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {\n        x[i] = (ggml_fp16_t *) ((char *) xv + i*xs);\n    }\n\n#if defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n\n        const int sve_register_length = svcntb() * 8;\n        const int ggml_f16_epr = sve_register_length / 16; // running when 16\n        const int ggml_f16_step = 8 * ggml_f16_epr; // choose 8 SVE registers\n\n        const int np = (n & ~(ggml_f16_step - 1));\n\n        svfloat16_t sum_00 = svdup_n_f16(0.0f);\n        svfloat16_t sum_01 = svdup_n_f16(0.0f);\n        svfloat16_t sum_02 = svdup_n_f16(0.0f);\n        svfloat16_t sum_03 = svdup_n_f16(0.0f);\n\n        svfloat16_t sum_10 = svdup_n_f16(0.0f);\n        svfloat16_t sum_11 = svdup_n_f16(0.0f);\n        svfloat16_t sum_12 = svdup_n_f16(0.0f);\n        svfloat16_t sum_13 = svdup_n_f16(0.0f);\n\n        svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n        svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n\n        for (int i = 0; i < np; i += ggml_f16_step) {\n            ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0); // 8 elements\n\n            ax1 = GGML_F16x_VEC_LOAD(x[0] + i + 0*ggml_f16_epr, 0); // 8 elements\n            sum_00 = GGML_F16x_VEC_FMA(sum_00, ax1, ay1);     // sum_00 = sum_00+ax1*ay1\n            ax1 = GGML_F16x_VEC_LOAD(x[1] + i + 0*ggml_f16_epr, 0); // 8 elements\n            sum_10 = GGML_F16x_VEC_FMA(sum_10, ax1, ay1);\n\n            ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1); // next 8 elements\n\n            ax2 = GGML_F16x_VEC_LOAD(x[0] + i + 1*ggml_f16_epr, 1); // next 8 elements\n            sum_01 = GGML_F16x_VEC_FMA(sum_01, ax2, ay2);\n            ax2 = GGML_F16x_VEC_LOAD(x[1] + i + 1*ggml_f16_epr, 1);\n            sum_11 = GGML_F16x_VEC_FMA(sum_11, ax2, ay2);\n\n            ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n\n            ax3 = GGML_F16x_VEC_LOAD(x[0] + i + 2*ggml_f16_epr, 2);\n            sum_02 = GGML_F16x_VEC_FMA(sum_02, ax3, ay3);\n            ax3 = GGML_F16x_VEC_LOAD(x[1] + i + 2*ggml_f16_epr, 2);\n            sum_12 = GGML_F16x_VEC_FMA(sum_12, ax3, ay3);\n\n            ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n\n            ax4 = GGML_F16x_VEC_LOAD(x[0] + i + 3*ggml_f16_epr, 3);\n            sum_03 = GGML_F16x_VEC_FMA(sum_03, ax4, ay4);\n            ax4 = GGML_F16x_VEC_LOAD(x[1] + i + 3*ggml_f16_epr, 3);\n            sum_13 = GGML_F16x_VEC_FMA(sum_13, ax4, ay4);\n\n            ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n\n            ax5 = GGML_F16x_VEC_LOAD(x[0] + i + 4*ggml_f16_epr, 4);\n\n            sum_00 = GGML_F16x_VEC_FMA(sum_00, ax5, ay5);\n            ax5 = GGML_F16x_VEC_LOAD(x[1] + i + 4*ggml_f16_epr, 4);\n            sum_10 = GGML_F16x_VEC_FMA(sum_10, ax5, ay5);\n\n            ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n\n            ax6 = GGML_F16x_VEC_LOAD(x[0] + i + 5*ggml_f16_epr, 5);\n\n            sum_01 = GGML_F16x_VEC_FMA(sum_01, ax6, ay6);\n            ax6 = GGML_F16x_VEC_LOAD(x[1] + i + 5*ggml_f16_epr, 5);\n            sum_11 = GGML_F16x_VEC_FMA(sum_11, ax6, ay6);\n\n            ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n\n            ax7 = GGML_F16x_VEC_LOAD(x[0] + i + 6*ggml_f16_epr, 6);\n\n            sum_02 = GGML_F16x_VEC_FMA(sum_02, ax7, ay7);\n            ax7 = GGML_F16x_VEC_LOAD(x[1] + i + 6*ggml_f16_epr, 6);\n            sum_12 = GGML_F16x_VEC_FMA(sum_12, ax7, ay7);\n\n            ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n\n            ax8 = GGML_F16x_VEC_LOAD(x[0] + i + 7*ggml_f16_epr, 7);\n\n            sum_03 = GGML_F16x_VEC_FMA(sum_03, ax8, ay8);\n            ax8 = GGML_F16x_VEC_LOAD(x[1] + i + 7*ggml_f16_epr, 7);\n            sum_13 = GGML_F16x_VEC_FMA(sum_13, ax8, ay8);\n        }\n\n        const int np2 = (n & ~(ggml_f16_epr - 1));\n        for (int k = np; k < np2; k += ggml_f16_epr) {\n            svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n\n            svfloat16_t rx = GGML_F16x_VEC_LOAD(x[0] + k, 0);\n            sum_00 = GGML_F16x_VEC_FMA(sum_00, rx, ry);\n            rx = GGML_F16x_VEC_LOAD(x[1] + k, 0);\n            sum_10 = GGML_F16x_VEC_FMA(sum_10, rx, ry);\n        }\n\n        if (np2 < n) {\n            svbool_t pg = svwhilelt_b16(np2, n);\n            svfloat16_t hx_0 = svld1_f16(pg, (const __fp16 *)(x[0] + np2));\n            svfloat16_t hx_1 = svld1_f16(pg, (const __fp16 *)(x[1] + np2));\n            svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n\n            sum_00 = svmad_f16_x(pg, hx_0, hy, sum_00);\n            sum_10 = svmad_f16_x(pg, hx_1, hy, sum_10);\n        }\n        GGML_F16x_VEC_REDUCE(sumf[0], sum_00, sum_01, sum_02, sum_03);\n        GGML_F16x_VEC_REDUCE(sumf[1], sum_10, sum_11, sum_12, sum_13);\n\n    #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)\n        size_t vl = __riscv_vsetvlmax_e32m4();\n\n        // initialize accumulators to all zeroes\n        vfloat32m4_t vsum0_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n        vfloat32m4_t vsum0_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n        vfloat32m4_t vsum1_0 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n        vfloat32m4_t vsum1_1 = __riscv_vfmv_v_f_f32m4(0.0f, vl);\n\n        // calculate step size\n        const size_t epr = __riscv_vsetvlmax_e16m2();\n        const size_t step = epr * 2;\n        const int np = (n & ~(step - 1));\n\n        // unroll by 2 along the row dimension\n        for (int i = 0; i < np; i += step) {\n            vfloat16m2_t ay0 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), epr);\n            vfloat16m2_t ax0_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), epr);\n            vfloat16m2_t ax1_0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), epr);\n            vsum0_0 = __riscv_vfwmacc_vv_f32m4(vsum0_0, ax0_0, ay0, epr);\n            vsum1_0 = __riscv_vfwmacc_vv_f32m4(vsum1_0, ax1_0, ay0, epr);\n\n            vfloat16m2_t ay1 = __riscv_vle16_v_f16m2((const _Float16 *)(y + i + epr), epr);\n            vfloat16m2_t ax0_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i + epr), epr);\n            vfloat16m2_t ax1_1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i + epr), epr);\n            vsum0_1 = __riscv_vfwmacc_vv_f32m4(vsum0_1, ax0_1, ay1, epr);\n            vsum1_1 = __riscv_vfwmacc_vv_f32m4(vsum1_1, ax1_1, ay1, epr);\n        }\n\n        vfloat32m4_t vsum0 = __riscv_vfadd_vv_f32m4(vsum0_0, vsum0_1, vl);\n        vfloat32m4_t vsum1 = __riscv_vfadd_vv_f32m4(vsum1_0, vsum1_1, vl);\n\n        // leftovers\n        for (int i = np; i < n; i += vl) {\n            vl = __riscv_vsetvl_e16m2(n - i);\n            vfloat16m2_t ay = __riscv_vle16_v_f16m2((const _Float16 *)(y + i), vl);\n            vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16 *)(x[0] + i), vl);\n            vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16 *)(x[1] + i), vl);\n\n            vsum0 = __riscv_vfwmacc_vv_f32m4(vsum0, ax0, ay, vl);\n            vsum1 = __riscv_vfwmacc_vv_f32m4(vsum1, ax1, ay, vl);\n        }\n\n        // reduce\n        vl = __riscv_vsetvlmax_e32m2();\n        vfloat32m2_t acc0_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum0, 0),\n                                    __riscv_vget_v_f32m4_f32m2(vsum0, 1), vl);\n        vl = __riscv_vsetvlmax_e32m1();\n        vfloat32m1_t acc0_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc0_0, 0),\n        __riscv_vget_v_f32m2_f32m1(acc0_0, 1), vl);\n        vfloat32m1_t redsum0 = __riscv_vfredusum_vs_f32m1_f32m1(\n                                    acc0_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n\n        vl = __riscv_vsetvlmax_e32m2();\n        vfloat32m2_t acc1_0 = __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(vsum1, 0),\n                                    __riscv_vget_v_f32m4_f32m2(vsum1, 1), vl);\n        vl = __riscv_vsetvlmax_e32m1();\n        vfloat32m1_t acc1_1 = __riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(acc1_0, 0),\n                                    __riscv_vget_v_f32m2_f32m1(acc1_0, 1), vl);\n        vfloat32m1_t redsum1 = __riscv_vfredusum_vs_f32m1_f32m1(\n                                    acc1_1, __riscv_vfmv_v_f_f32m1(0.0f, 1), vl);\n        sumf[0] = __riscv_vfmv_f_s_f32m1_f32(redsum0);\n        sumf[1] = __riscv_vfmv_f_s_f32m1_f32(redsum1);\n\n    #else\n        const int np = (n & ~(GGML_F16_STEP - 1));\n\n        GGML_F16_VEC sum[GGML_VEC_DOT_UNROLL][GGML_F16_ARR] = { { GGML_F16_VEC_ZERO } };\n\n        GGML_F16_VEC ax[GGML_F16_ARR];\n        GGML_F16_VEC ay[GGML_F16_ARR];\n\n        for (int i = 0; i < np; i += GGML_F16_STEP) {\n            for (int j = 0; j < GGML_F16_ARR; j++) {\n                ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n\n                for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {\n                    ax[j] = GGML_F16_VEC_LOAD(x[k] + i + j*GGML_F16_EPR, j);\n\n                    sum[k][j] = GGML_F16_VEC_FMA(sum[k][j], ax[j], ay[j]);\n                }\n            }\n        }\n\n        // reduce sum0..sum3 to sum0\n        for (int k = 0; k < GGML_VEC_DOT_UNROLL; ++k) {\n            GGML_F16_VEC_REDUCE(sumf[k], sum[k]);\n        }\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {\n                sumf[j] += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[j][i])*GGML_CPU_FP16_TO_FP32(y[i]));\n            }\n        }\n    #endif\n#else\n    for (int i = 0; i < n; ++i) {\n        for (int j = 0; j < GGML_VEC_DOT_UNROLL; ++j) {\n            sumf[j] += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[j][i])*GGML_CPU_FP16_TO_FP32(y[i]));\n        }\n    }\n#endif\n\n    for (int i = 0; i < GGML_VEC_DOT_UNROLL; ++i) {\n        s[i] = (float)sumf[i];\n    }\n}\n\ninline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const float * GGML_RESTRICT x, const float v) {\n#if defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n\n        const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n        const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n        const int ggml_f32_step = 8 * ggml_f32_epr; // choose 8 SVE registers\n        GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n        const int np = (n & ~(ggml_f32_step - 1));\n        svfloat32_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n        svfloat32_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n        for (int i = 0; i < np; i += ggml_f32_step) {\n\n            ax1 = GGML_F32_VEC_LOAD(x + i);\n            ay1 = GGML_F32_VEC_LOAD(y + i);\n            ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);\n\n            GGML_F32_VEC_STORE(y + i, ay1);\n\n            ax2 = GGML_F32_VEC_LOAD(x + i + 1*ggml_f32_epr);\n            ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n            ay2 = GGML_F32_VEC_FMA(ay2, ax2, vx);\n\n            GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);\n\n            ax3 = GGML_F32_VEC_LOAD(x + i + 2*ggml_f32_epr);\n            ay3 = GGML_F32_VEC_LOAD(y + i + 2*ggml_f32_epr);\n            ay3 = GGML_F32_VEC_FMA(ay3, ax3, vx);\n\n            GGML_F32_VEC_STORE(y + i + 2*ggml_f32_epr, ay3);\n\n            ax4 = GGML_F32_VEC_LOAD(x + i + 3*ggml_f32_epr);\n            ay4 = GGML_F32_VEC_LOAD(y + i + 3*ggml_f32_epr);\n            ay4 = GGML_F32_VEC_FMA(ay4, ax4, vx);\n\n            GGML_F32_VEC_STORE(y + i + 3*ggml_f32_epr, ay4);\n\n            ax5 = GGML_F32_VEC_LOAD(x + i + 4*ggml_f32_epr);\n            ay5 = GGML_F32_VEC_LOAD(y + i + 4*ggml_f32_epr);\n            ay5 = GGML_F32_VEC_FMA(ay5, ax5, vx);\n\n            GGML_F32_VEC_STORE(y + i + 4*ggml_f32_epr, ay5);\n\n            ax6 = GGML_F32_VEC_LOAD(x + i + 5*ggml_f32_epr);\n            ay6 = GGML_F32_VEC_LOAD(y + i + 5*ggml_f32_epr);\n            ay6 = GGML_F32_VEC_FMA(ay6, ax6, vx);\n\n            GGML_F32_VEC_STORE(y + i + 5*ggml_f32_epr, ay6);\n\n            ax7 = GGML_F32_VEC_LOAD(x + i + 6*ggml_f32_epr);\n            ay7 = GGML_F32_VEC_LOAD(y + i + 6*ggml_f32_epr);\n            ay7 = GGML_F32_VEC_FMA(ay7, ax7, vx);\n\n            GGML_F32_VEC_STORE(y + i + 6*ggml_f32_epr, ay7);\n\n            ax8 = GGML_F32_VEC_LOAD(x + i + 7*ggml_f32_epr);\n            ay8 = GGML_F32_VEC_LOAD(y + i + 7*ggml_f32_epr);\n            ay8 = GGML_F32_VEC_FMA(ay8, ax8, vx);\n\n            GGML_F32_VEC_STORE(y + i + 7*ggml_f32_epr, ay8);\n        }\n        // leftovers\n        // Since 8 unrolls are done in above loop, leftovers lie in range [0, ggml_f32_step] which is handled in below loop\n        const int np2 = (n & ~(ggml_f32_epr - 1));\n        for (int i = np; i < np2; i += ggml_f32_epr) {\n            ax1 = GGML_F32_VEC_LOAD(x + i);\n            ay1 = GGML_F32_VEC_LOAD(y + i);\n            ay1 = GGML_F32_VEC_FMA(ay1, ax1, vx);\n\n            GGML_F32_VEC_STORE(y + i, ay1);\n        }\n        // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n        if (np2 < n) {\n            svbool_t pg =svwhilelt_b32(np2, n);\n            ax1 = svld1_f32(pg, x + np2);\n            ay1 = svld1_f32(pg, y + np2);\n            ay1 = svmad_f32_m(pg, ax1, vx, ay1);\n\n            svst1_f32(pg, y + np2, ay1);\n        }\n    #elif defined(__riscv_v_intrinsic)\n        for (int i = 0, avl; i < n; i += avl) {\n            avl = __riscv_vsetvl_e32m8(n - i);\n            vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);\n            vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n            vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, v, ay, avl);\n            __riscv_vse32_v_f32m8(&y[i], ny, avl);\n        }\n    #else\n        const int np = (n & ~(GGML_F32_STEP - 1));\n\n        GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n        GGML_F32_VEC ax[GGML_F32_ARR];\n        GGML_F32_VEC ay[GGML_F32_ARR];\n\n        for (int i = 0; i < np; i += GGML_F32_STEP) {\n            for (int j = 0; j < GGML_F32_ARR; j++) {\n                ax[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n                ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n                ay[j] = GGML_F32_VEC_FMA(ay[j], ax[j], vx);\n\n                GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n            }\n        }\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            y[i] += x[i]*v;\n        }\n    #endif\n#else\n    // scalar\n    for (int i = 0; i < n; ++i) {\n        y[i] += x[i]*v;\n    }\n#endif\n}\n\ninline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, const ggml_fp16_t * GGML_RESTRICT x, const float v) {\n#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)\n    const int sve_register_length = svcntb() * 8;\n    const int ggml_f16_epr = sve_register_length / 16;\n    const int ggml_f16_step = 8 * ggml_f16_epr;\n\n    GGML_F16x_VEC vx = GGML_F16x_VEC_SET1(v);\n\n    int np = (n & ~(ggml_f16_step - 1));\n\n    svfloat16_t ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8;\n    svfloat16_t ay1, ay2, ay3, ay4, ay5, ay6, ay7, ay8;\n    for (int i = 0; i < np; i += ggml_f16_step) {\n        ax1 = GGML_F16x_VEC_LOAD(x + i + 0 * ggml_f16_epr, 0);\n        ay1 = GGML_F16x_VEC_LOAD(y + i + 0 * ggml_f16_epr, 0);\n        ay1 = GGML_F16x_VEC_FMA(ay1, ax1, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 0 * ggml_f16_epr, ay1, 0);\n\n        ax2 = GGML_F16x_VEC_LOAD(x + i + 1 * ggml_f16_epr, 1);\n        ay2 = GGML_F16x_VEC_LOAD(y + i + 1 * ggml_f16_epr, 1);\n        ay2 = GGML_F16x_VEC_FMA(ay2, ax2, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 1 * ggml_f16_epr, ay2, 1);\n\n        ax3 = GGML_F16x_VEC_LOAD(x + i + 2 * ggml_f16_epr, 2);\n        ay3 = GGML_F16x_VEC_LOAD(y + i + 2 * ggml_f16_epr, 2);\n        ay3 = GGML_F16x_VEC_FMA(ay3, ax3, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 2 * ggml_f16_epr, ay3, 2);\n\n        ax4 = GGML_F16x_VEC_LOAD(x + i + 3 * ggml_f16_epr, 3);\n        ay4 = GGML_F16x_VEC_LOAD(y + i + 3 * ggml_f16_epr, 3);\n        ay4 = GGML_F16x_VEC_FMA(ay4, ax4, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 3 * ggml_f16_epr, ay4, 3);\n\n        ax5 = GGML_F16x_VEC_LOAD(x + i + 4 * ggml_f16_epr, 4);\n        ay5 = GGML_F16x_VEC_LOAD(y + i + 4 * ggml_f16_epr, 4);\n        ay5 = GGML_F16x_VEC_FMA(ay5, ax5, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 4 * ggml_f16_epr, ay5, 4);\n\n        ax6 = GGML_F16x_VEC_LOAD(x + i + 5 * ggml_f16_epr, 5);\n        ay6 = GGML_F16x_VEC_LOAD(y + i + 5 * ggml_f16_epr, 5);\n        ay6 = GGML_F16x_VEC_FMA(ay6, ax6, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 5 * ggml_f16_epr, ay6, 5);\n\n        ax7 = GGML_F16x_VEC_LOAD(x + i + 6 * ggml_f16_epr, 6);\n        ay7 = GGML_F16x_VEC_LOAD(y + i + 6 * ggml_f16_epr, 6);\n        ay7 = GGML_F16x_VEC_FMA(ay7, ax7, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 6 * ggml_f16_epr, ay7, 6);\n\n        ax8 = GGML_F16x_VEC_LOAD(x + i + 7 * ggml_f16_epr, 7);\n        ay8 = GGML_F16x_VEC_LOAD(y + i + 7 * ggml_f16_epr, 7);\n        ay8 = GGML_F16x_VEC_FMA(ay8, ax8, vx);\n\n        GGML_F16x_VEC_STORE(y + i + 7 * ggml_f16_epr, ay8, 7);\n    }\n    const int np2 = (n & ~(ggml_f16_epr - 1));\n    for (int k = np; k < np2; k += ggml_f16_epr) {\n        svfloat16_t rx = GGML_F16x_VEC_LOAD(x + k, 0);\n        svfloat16_t ry = GGML_F16x_VEC_LOAD(y + k, 0);\n        ry = GGML_F16x_VEC_FMA(ry, rx, vx);\n\n        GGML_F16x_VEC_STORE(y + k, ry, 0);\n    }\n\n    if (np2 < n) {\n        svbool_t pg = svwhilelt_b16(np2, n);\n        svfloat16_t hx = svld1_f16(pg, (const __fp16 *)(x + np2));\n        svfloat16_t hy = svld1_f16(pg, (const __fp16 *)(y + np2));\n        hy = svmad_f16_x(pg, hx, vx, hy);\n        svst1_f16(pg, (__fp16 *)(y + np2), hy);\n    }\n    np = n;\n#elif defined(__riscv_zvfh) // implies __riscv_v_intrinsic\n    const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);\n    const _Float16 scale = *(const _Float16*)(&s);\n\n    // calculate step size\n    const int epr = __riscv_vsetvlmax_e16m4();\n    const int step = epr * 2;\n    int np = (n & ~(step - 1));\n\n    // unroll by 2\n    for (int i = 0; i < np; i += step) {\n        vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, epr);\n        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);\n        ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, epr);\n        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n\n        vfloat16m4_t ax1 = __riscv_vle16_v_f16m4((const _Float16*)x + i + epr, epr);\n        vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);\n        ay1 = __riscv_vfmacc_vf_f16m4(ay1, scale, ax1, epr);\n        __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n    }\n\n    // leftovers\n    int vl;\n    for (int i = np; i < n; i += vl) {\n        vl = __riscv_vsetvl_e16m4(n - i);\n        vfloat16m4_t ax0 = __riscv_vle16_v_f16m4((const _Float16*)x + i, vl);\n        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);\n        ay0 = __riscv_vfmacc_vf_f16m4(ay0, scale, ax0, vl);\n        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);\n    }\n    np = n;\n#elif defined(GGML_SIMD)\n    const int np = (n & ~(GGML_F16_STEP - 1));\n\n    GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);\n\n    GGML_F16_VEC ax[GGML_F16_ARR];\n    GGML_F16_VEC ay[GGML_F16_ARR];\n\n    for (int i = 0; i < np; i += GGML_F16_STEP) {\n        for (int j = 0; j < GGML_F16_ARR; j++) {\n            ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j);\n            ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n            ay[j] = GGML_F16_VEC_FMA(ay[j], ax[j], vx);\n\n            GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);\n        }\n    }\n#else\n    const int np = 0;\n#endif\n\n    // leftovers\n    for (int i = np; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i]) + GGML_CPU_FP16_TO_FP32(x[i])*v);\n    }\n}\n\n// xs and vs are byte strides of x and v\ninline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int vs, float * GGML_RESTRICT y, const float * GGML_RESTRICT xv, const float * GGML_RESTRICT vv) {\n\n    const float * GGML_RESTRICT x[GGML_VEC_MAD_UNROLL];\n    const float * GGML_RESTRICT v[GGML_VEC_MAD_UNROLL];\n\n    for (int i = 0; i < GGML_VEC_MAD_UNROLL; ++i) {\n        x[i] = (const float *) ((const char *) xv + i*xs);\n        v[i] = (const float *) ((const char *) vv + i*vs);\n    }\n\n#if defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n        // scalar Route to scalar implementation       //TODO: Write SVE code\n        for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n            for (int i = 0; i < n; ++i) {\n                y[i] += x[k][i]*v[k][0];\n            }\n        }\n    #elif defined(__riscv_v_intrinsic)\n        for (int i = 0, avl; i < n; i += avl) {\n            avl = __riscv_vsetvl_e32m8(n - i);\n            vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n            for (int k = 0; k < GGML_VEC_MAD_UNROLL; k++) {\n                vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[k][i], avl);\n                ay = __riscv_vfmadd_vf_f32m8(ax, v[k][0], ay, avl);\n            }\n            __riscv_vse32_v_f32m8(&y[i], ay, avl);\n        }\n    #else\n        const int np = (n & ~(GGML_F32_STEP - 1));\n\n        GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL];\n\n        for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n            vx[k] = GGML_F32_VEC_SET1(v[k][0]);\n        }\n\n        GGML_F32_VEC ax[GGML_VEC_MAD_UNROLL][GGML_F32_ARR];\n        GGML_F32_VEC ay[GGML_F32_ARR];\n\n        for (int i = 0; i < np; i += GGML_F32_STEP) {\n            for (int j = 0; j < GGML_F32_ARR; j++) {\n                ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n\n                for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n                    ax[k][j] = GGML_F32_VEC_LOAD(x[k] + i + j*GGML_F32_EPR);\n                    ay[j] = GGML_F32_VEC_FMA(ay[j], ax[k][j], vx[k]);\n                }\n\n                GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n            }\n        }\n\n        // leftovers\n        for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n            for (int i = np; i < n; ++i) {\n                y[i] += x[k][i]*v[k][0];\n            }\n        }\n    #endif\n#else\n    // scalar\n    for (int k = 0; k < GGML_VEC_MAD_UNROLL; ++k) {\n        for (int i = 0; i < n; ++i) {\n            y[i] += x[k][i]*v[k][0];\n        }\n    }\n#endif\n}\n\ninline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, const float s, const float b) {\n#if defined(GGML_USE_ACCELERATE)\n    vDSP_vsmsa(x, 1, &s, &b, y, 1, n);\n#elif defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n        // scalar ; TODO: Write SVE code\n        for (int i = 0; i < n; ++i) {\n            y[i] = x[i]*s + b;\n        }\n    #elif defined(__riscv_v_intrinsic)\n        for (int i = 0, avl; i < n; i += avl) {\n            avl = __riscv_vsetvl_e32m8(n - i);\n            vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl);\n            vfloat32m8_t vb = __riscv_vfmv_v_f_f32m8(b, avl);\n            vfloat32m8_t ny = __riscv_vfmadd_vf_f32m8(ax, s, vb, avl);\n            __riscv_vse32_v_f32m8(&y[i], ny, avl);\n        }\n    #else\n        const int np = (n & ~(GGML_F32_STEP - 1));\n\n        GGML_F32_VEC vs = GGML_F32_VEC_SET1(s);\n        GGML_F32_VEC vb = GGML_F32_VEC_SET1(b);\n\n        GGML_F32_VEC ay[GGML_F32_ARR];\n\n        for (int i = 0; i < np; i += GGML_F32_STEP) {\n            for (int j = 0; j < GGML_F32_ARR; j++) {\n                ay[j] = GGML_F32_VEC_LOAD(x + i + j*GGML_F32_EPR);\n                ay[j] = GGML_F32_VEC_FMA(vb, ay[j], vs);\n\n                GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n            }\n        }\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            y[i] = x[i]*s + b;\n        }\n    #endif\n#else\n    // scalar\n    for (int i = 0; i < n; ++i) {\n        y[i] = x[i]*s + b;\n    }\n#endif\n}\n\n//inline static void ggml_vec_scale_f32(const int n, float * y, const float   v) { for (int i = 0; i < n; ++i) y[i] *= v;          }\ninline static void ggml_vec_scale_f32(const int n, float * y, const float   v) {\n#if defined(GGML_USE_ACCELERATE)\n    vDSP_vsmul(y, 1, &v, y, 1, n);\n#elif defined(GGML_SIMD)\n    #if defined(__ARM_FEATURE_SVE)\n        const int sve_register_length = ggml_cpu_get_sve_cnt() * 8;\n        const int ggml_f32_epr = sve_register_length / 32;//8;//svcntw(); // SVE128:4, SVE256:8, SVE512:16\n        const int ggml_f32_step = 2 * ggml_f32_epr;\n\n        GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n        const int np = (n & ~(ggml_f32_step - 1));\n        svfloat32_t ay1;\n        svfloat32_t ay2;\n        for (int i = 0; i < np; i += ggml_f32_step) {\n            ay1 = GGML_F32_VEC_LOAD(y + i);\n            ay1 = GGML_F32_VEC_MUL(ay1, vx);\n            GGML_F32_VEC_STORE(y + i, ay1);\n\n            ay2 = GGML_F32_VEC_LOAD(y + i + 1*ggml_f32_epr);\n            ay2 = GGML_F32_VEC_MUL(ay2, vx);\n            GGML_F32_VEC_STORE(y + i + 1*ggml_f32_epr, ay2);\n        }\n        // leftovers\n        // maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only\n        for (int i = np; i < n; i += ggml_f32_epr) {\n            svbool_t pg = svwhilelt_b32(i, n);\n            ay1 = svld1_f32(pg, y + i);\n            ay1 = svmul_f32_m(pg, ay1, vx);\n            svst1_f32(pg, y + i, ay1);\n        }\n    #elif defined(__riscv_v_intrinsic)\n        for (int i = 0, avl; i < n; i += avl) {\n            avl = __riscv_vsetvl_e32m8(n - i);\n            vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl);\n            vfloat32m8_t ny = __riscv_vfmul_vf_f32m8(ay, v, avl);\n            __riscv_vse32_v_f32m8(&y[i], ny, avl);\n        }\n    #else\n        const int np = (n & ~(GGML_F32_STEP - 1));\n\n        GGML_F32_VEC vx = GGML_F32_VEC_SET1(v);\n\n        GGML_F32_VEC ay[GGML_F32_ARR];\n\n        for (int i = 0; i < np; i += GGML_F32_STEP) {\n            for (int j = 0; j < GGML_F32_ARR; j++) {\n                ay[j] = GGML_F32_VEC_LOAD(y + i + j*GGML_F32_EPR);\n                ay[j] = GGML_F32_VEC_MUL(ay[j], vx);\n\n                GGML_F32_VEC_STORE(y + i + j*GGML_F32_EPR, ay[j]);\n            }\n        }\n\n        // leftovers\n        for (int i = np; i < n; ++i) {\n            y[i] *= v;\n        }\n    #endif\n#else\n    // scalar\n    for (int i = 0; i < n; ++i) {\n        y[i] *= v;\n    }\n#endif\n}\n\ninline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) {\n#if defined(GGML_SIMD) && defined(__ARM_FEATURE_SVE)\n    const int sve_register_length = svcntb() * 8;\n    const int ggml_f16_epr = sve_register_length / 16;\n    const int ggml_f16_step = 2 * ggml_f16_epr;\n\n    GGML_F16x_VEC vx =  GGML_F16x_VEC_SET1(v);\n    const int np = (n & ~(ggml_f16_step - 1));\n    svfloat16_t ay1, ay2;\n\n    for (int i = 0; i < np; i += ggml_f16_step) {\n        ay1 = GGML_F16x_VEC_LOAD(y + i + 0*ggml_f16_epr, 0);\n        ay1 = GGML_F16x_VEC_MUL(ay1, vx);\n        GGML_F16x_VEC_STORE(y + i + 0*ggml_f16_epr, ay1, 0);\n\n        ay2 = GGML_F16x_VEC_LOAD(y + i + 1*ggml_f16_epr, 1);\n        ay2 = GGML_F16x_VEC_MUL(ay2, vx);\n        GGML_F16x_VEC_STORE(y + i + 1*ggml_f16_epr, ay2, 1);\n    }\n    // leftovers\n    // maximum number of leftover elements will be less that ggmlF_16x_epr. Apply predicated svmad on available elements only\n    if (np < n) {\n        svbool_t pg = svwhilelt_b16(np, n);\n        svfloat16_t hy = svld1_f16(pg, (__fp16 *)(y + np));\n        svfloat16_t out = svmul_f16_m(pg, hy, vx);\n        svst1_f16(pg, (__fp16 *)(y + np), out);\n    }\n#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfh)\n    const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v);\n    const _Float16 scale = *(const _Float16*)(&s);\n\n    // calculate step size\n    const int epr = __riscv_vsetvlmax_e16m4();\n    const int step = epr * 2;\n    const int np = (n & ~(step - 1));\n\n    // unroll by 2\n    for (int i = 0; i < np; i += step) {\n        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, epr);\n        ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, epr);\n        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n\n        vfloat16m4_t ay1 = __riscv_vle16_v_f16m4((const _Float16*)y + i + epr, epr);\n        ay1 = __riscv_vfmul_vf_f16m4(ay1, scale, epr);\n        __riscv_vse16_v_f16m4((_Float16*)y + i + epr, ay1, epr);\n        __asm__ __volatile__ (\"\" ::: \"memory\");\n    }\n\n    // leftovers\n    int vl;\n    for (int i = np; i < n; i += vl) {\n        vl = __riscv_vsetvl_e16m4(n - i);\n        vfloat16m4_t ay0 = __riscv_vle16_v_f16m4((const _Float16*)y + i, vl);\n        ay0 = __riscv_vfmul_vf_f16m4(ay0, scale, vl);\n        __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl);\n    }\n#elif defined(GGML_SIMD)\n    const int np = (n & ~(GGML_F16_STEP - 1));\n\n    GGML_F16_VEC vx = GGML_F16_VEC_SET1(v);\n\n    GGML_F16_VEC ay[GGML_F16_ARR];\n\n    for (int i = 0; i < np; i += GGML_F16_STEP) {\n        for (int j = 0; j < GGML_F16_ARR; j++) {\n            ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);\n            ay[j] = GGML_F16_VEC_MUL(ay[j], vx);\n\n            GGML_F16_VEC_STORE(y + i + j*GGML_F16_EPR, ay, j);\n        }\n    }\n\n    // leftovers\n    for (int i = np; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i])*v);\n    }\n#else\n    // scalar\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(y[i])*v);\n    }\n#endif\n}\n\ninline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s);   }\ninline static void ggml_vec_sqr_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i];   }\ninline static void ggml_vec_sqr_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(v*v);\n    }\n}\ninline static void ggml_vec_sqrt_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sqrtf(x[i]); }\ninline static void ggml_vec_sqrt_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(sqrtf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_log_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = logf(x[i]);  }\ninline static void ggml_vec_log_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(logf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_sin_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = sinf(x[i]);  }\ninline static void ggml_vec_sin_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(sinf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_cos_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = cosf(x[i]);  }\ninline static void ggml_vec_cos_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(cosf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_abs_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fabsf(x[i]); }\ninline static void ggml_vec_abs_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(fabsf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_sgn_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : ((x[i] < 0.f) ? -1.f : 0.f); }\ninline static void ggml_vec_sgn_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? 1.f : ((v < 0.f) ? -1.f : 0.f));\n    }\n}\ninline static void ggml_vec_step_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? 1.f : 0.f; }\ninline static void ggml_vec_step_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16((GGML_CPU_FP16_TO_FP32(x[i]) > 0.f) ? 1.f : 0.f);\n    }\n}\ninline static void ggml_vec_tanh_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = tanhf(x[i]);  }\ninline static void ggml_vec_tanh_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(tanhf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\ninline static void ggml_vec_elu_f32  (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : expm1f(x[i]); }\ninline static void ggml_vec_elu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        const float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : expm1f(v));\n    }\n}\ninline static void ggml_vec_relu_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = (x[i] > 0.f) ? x[i] : 0.f; }\ninline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v : 0.f);\n    }\n}\ninline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }\ninline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));\n    }\n}\ninline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }\ninline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(1.f / (1.f + expf(-GGML_CPU_FP16_TO_FP32(x[i]))));\n    }\n}\n// TODO: optimize performance\ninline static void ggml_vec_hardswish_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i] * fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }\ninline static void ggml_vec_hardswish_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(v * fminf(1.0f, fmaxf(0.0f, (v + 3.0f) / 6.0f)));\n    }\n}\ninline static void ggml_vec_hardsigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = fminf(1.0f, fmaxf(0.0f, (x[i] + 3.0f) / 6.0f)); }\ninline static void ggml_vec_hardsigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(fminf(1.0f, fmaxf(0.0f, (GGML_CPU_FP16_TO_FP32(x[i]) + 3.0f) / 6.0f)));\n    }\n}\ninline static void ggml_vec_exp_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = expf(x[i]); }\ninline static void ggml_vec_exp_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = GGML_CPU_FP32_TO_FP16(expf(GGML_CPU_FP16_TO_FP32(x[i])));\n    }\n}\n\nstatic const float GELU_COEF_A     = 0.044715f;\nstatic const float GELU_QUICK_COEF = -1.702f;\nstatic const float SQRT_2_OVER_PI  = 0.79788456080286535587989211986876f;\nstatic const float SQRT_2_INV      = 0.70710678118654752440084436210484f;\n\ninline static float ggml_gelu_f32(float x) {\n    return 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)));\n}\n\ninline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    const uint16_t * i16 = (const uint16_t *) x;\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_table_gelu_f16[i16[i]];\n    }\n}\n\ninline static void ggml_vec_gelu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n        float res = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));\n        y[i] = GGML_CPU_FP32_TO_FP16(res);\n    }\n}\n\n#ifdef GGML_GELU_FP16\ninline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {\n    uint16_t t;\n    for (int i = 0; i < n; ++i) {\n        if (x[i] <= -10.0f) {\n            y[i] = 0.0f;\n        } else if (x[i] >= 10.0f) {\n            y[i] = x[i];\n        } else {\n            ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n            memcpy(&t, &fp16, sizeof(uint16_t));\n            y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]);\n        }\n    }\n}\n#else\ninline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_gelu_f32(x[i]);\n    }\n}\n#endif\n\ninline static void ggml_vec_gelu_erf_f32(const int n, float * y, const float * x) {\n    for (int i = 0; i < n; ++i) {\n        float xi = x[i];\n        y[i] = 0.5f*xi*(1.0f + erff(xi*SQRT_2_INV));\n    }\n}\n\ninline static float ggml_gelu_quick_f32(float x) {\n    return x*(1.0f/(1.0f+expf(GELU_QUICK_COEF*x)));\n}\n\n//inline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n//    const uint16_t * i16 = (const uint16_t *) x;\n//    for (int i = 0; i < n; ++i) {\n//        y[i] = ggml_table_gelu_quick_f16[i16[i]];\n//    }\n//}\n\n#ifdef GGML_GELU_QUICK_FP16\ninline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {\n    uint16_t t;\n    for (int i = 0; i < n; ++i) {\n        ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n        memcpy(&t, &fp16, sizeof(uint16_t));\n        y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]);\n    }\n}\n#else\ninline static void ggml_vec_gelu_quick_f32(const int n, float * y, const float * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_gelu_quick_f32(x[i]);\n    }\n}\n#endif\n\ninline static void ggml_vec_gelu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(v*(1.0f/(1.0f+expf(GELU_QUICK_COEF*v))));\n    }\n}\n\n// Sigmoid Linear Unit (SiLU) function\ninline static float ggml_silu_f32(float x) {\n    return x/(1.0f + expf(-x));\n}\ninline static ggml_fp16_t ggml_silu_f16(ggml_fp16_t x) {\n    float v = GGML_CPU_FP16_TO_FP32(x);\n    return GGML_CPU_FP32_TO_FP16(v/(1.0f + expf(-v)));\n}\n\n#if __FINITE_MATH_ONLY__\n#error \"some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix\"\n#error \"ref: https://github.com/ggml-org/llama.cpp/pull/7154#issuecomment-2143844461\"\n#endif\n\n/* Below function was borrowed from the GitHub repository:\nhttps://github.com/openvinotoolkit/openvino/blob/master/src/plugins/intel_cpu/src/nodes/kernels/scaled_attn/common.hpp */\n#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n    inline static svfloat32_t exp_ps_sve(svbool_t pg, svfloat32_t src) {\n        // Constants\n        const svfloat32_t log2_e = svdup_n_f32(1.4426950409f);\n        const svfloat32_t ln2 = svdup_n_f32(0.6931473921f);\n        const svfloat32_t half_ln2_sq = svdup_n_f32(0.2413862043f);\n        const svuint32_t not_mask17 = svdup_n_u32(~((1u << 17) - 1));\n        const svfloat32_t one = svdup_n_f32(1.0f);\n        const svfloat32_t inactive1 = svdup_n_f32(0.0f);\n        const svint32_t inactive2 = svdup_n_s32(0);\n\n        // Algorithm starts here\n        svfloat32_t t0 = svmul_f32_m(pg, src, log2_e);  // y = x * log2(e)\n        svfloat32_t t1 = svrintm_f32_m(inactive1, pg, t0);         // rount to int (float)\n        svint32_t t2 = svcvt_s32_f32_m(inactive2, pg, t1);         // n\n\n        t1 = svsub_f32_m(pg, t0, t1);   // a = y - floor(y)\n        t1 = svadd_f32_m(pg, t1, one);  // b = a + 1\n\n        svuint32_t t3 = svlsr_n_u32_m(pg, svreinterpret_u32_f32(t1), 17);  // v = b >> 17 (u32)\n        svfloat32_t t4 = svexpa_f32(t3);                                   // c = fexpa(v)\n        t4 = svscale_f32_m(pg, t4, t2);                                    // fexpa(v) * 2^(n)\n\n        // and_(t2.d, t1.d, not_mask17.d)\n        svfloat32_t t5 = svreinterpret_f32_u32(svand_u32_m(pg, svreinterpret_u32_f32(t1), not_mask17));\n        t5 = svsub_f32_m(pg, t1, t5);                // z\n        t0 = svmla_f32_m(pg, ln2, t5, half_ln2_sq);  // ln2 + half_ln2_sq * z\n        t0 = svmla_f32_m(pg, one, t5, t0);           // 1 + (ln2 * z) + (half_ln2_sq * z * z)\n        t0 = svmul_f32_m(pg, t0, t4);                // Final result\n\n        return t0;\n    }\n#endif\n\n#if defined(__ARM_FEATURE_SVE) && defined(__aarch64__)\n\ninline static svfloat32_t ggml_v_expf(svbool_t pg, svfloat32_t x) {\n    const svfloat32_t r = svdup_n_f32_x(pg, 0x1.8p23f);\n    const svfloat32_t z = svmla_n_f32_x(pg, r, x, 0x1.715476p+0f);\n    const svfloat32_t n = svsub_f32_x(pg, z, r);\n    const svfloat32_t b = svmls_n_f32_x(pg, svmls_n_f32_x(pg, x, n, 0x1.62e4p-1f), n, 0x1.7f7d1cp-20f);\n    const svuint32_t e = svlsl_n_u32_x(pg, svreinterpret_u32_f32(z), 23);\n    const svfloat32_t k = svreinterpret_f32_u32(svadd_u32_x(pg, e, svreinterpret_u32_f32(svdup_n_f32_x(pg, 1))));\n    const svbool_t c = svacgt_n_f32(pg, n, 126);\n    const svfloat32_t u = svmul_f32_x(pg, b, b);\n    const svfloat32_t j = svmla_f32_x(pg,\n        svmul_n_f32_x(pg, b, 0x1.ffffecp-1f),\n        svmla_f32_x(pg, svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.fffdb6p-2f), svdup_n_f32_x(pg, 0x1.555e66p-3f), b),\n                        svmla_f32_x(pg, svdup_n_f32_x(pg, 0x1.573e2ep-5f), svdup_n_f32_x(pg, 0x1.0e4020p-7f), b), u), u);\n    const svuint32_t d = svdup_n_u32_z(svcmple_n_f32(pg, n, 0.0), 0x82000000);\n    const svfloat32_t s1 = svreinterpret_f32_u32(svadd_n_u32_x(pg, d, 0x7f000000));\n    const svfloat32_t s2 = svreinterpret_f32_u32(svsub_u32_x(pg, e, d));\n    return svsel_f32(svacgt_f32(pg, n, svdup_n_f32_x(pg, 192)), svmul_f32_x(pg, s1, s1),\n                     svsel_f32(c, svmul_f32_x(pg, svmla_f32_x(pg, s2, s2, j), s1), svmla_f32_x(pg, k, k, j)));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static svfloat32_t ggml_v_silu(svbool_t pg, svfloat32_t x) {\n    const svfloat32_t one = svdup_n_f32_x(pg, 1.0f);\n    const svfloat32_t zero = svdup_n_f32_x(pg, 0.0f);\n    const svfloat32_t neg_x = svsub_f32_x(pg, zero, x);\n    const svfloat32_t exp_neg_x = ggml_v_expf(pg, neg_x);\n    const svfloat32_t one_plus_exp_neg_x = svadd_f32_x(pg, one, exp_neg_x);\n    return svdiv_f32_x(pg, x, one_plus_exp_neg_x);\n}\n\n#elif defined(__ARM_NEON) && defined(__aarch64__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static float32x4_t ggml_v_expf(float32x4_t x) {\n    const float32x4_t r = vdupq_n_f32(0x1.8p23f);\n    const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f));\n    const float32x4_t n = vsubq_f32(z, r);\n    const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n,\n                                    vdupq_n_f32(0x1.7f7d1cp-20f));\n    const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23);\n    const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1))));\n    const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126));\n    const float32x4_t u = vmulq_f32(b, b);\n    const float32x4_t j = vfmaq_f32(\n        vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b),\n        vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b),\n                  vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u);\n    if (!vpaddd_u64(vreinterpretq_u64_u32(c)))\n        return vfmaq_f32(k, j, k);\n    const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000));\n    const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000)));\n    const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d));\n    return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1),\n                     vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j)));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static float32x4_t ggml_v_silu(float32x4_t x) {\n    const float32x4_t one = vdupq_n_f32(1.0f);\n    const float32x4_t zero = vdupq_n_f32(0.0f);\n    const float32x4_t neg_x = vsubq_f32(zero, x);\n    const float32x4_t exp_neg_x = ggml_v_expf(neg_x);\n    const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x);\n    return vdivq_f32(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__AVX512F__) && defined(__AVX512DQ__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m512 ggml_v_expf(__m512 x) {\n  const __m512 r = _mm512_set1_ps(0x1.8p23f);\n  const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);\n  const __m512 n = _mm512_sub_ps(z, r);\n  const __m512 b =\n      _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),\n                       _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));\n  const __mmask16 d =\n      _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ);\n  const __m512 u = _mm512_mul_ps(b, b);\n  const __m512 j = _mm512_fmadd_ps(\n      _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,\n                                      _mm512_set1_ps(0x1.573e2ep-5f)),\n                      u,\n                      _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,\n                                      _mm512_set1_ps(0x1.fffdb6p-2f))),\n      u,\n      _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F)));\n  const __m512 res = _mm512_scalef_ps(j, n);\n  if (_mm512_kortestz(d, d))\n    return res;\n  const __m512 zero = _mm512_setzero_ps();\n  const __m512 alt = _mm512_mask_blend_ps(\n      _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero);\n  return _mm512_mask_blend_ps(d, res, alt);\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m512 ggml_v_silu(__m512 x) {\n    const __m512 one = _mm512_set1_ps(1);\n    const __m512 zero = _mm512_setzero_ps();\n    const __m512 neg_x = _mm512_sub_ps(zero, x);\n    const __m512 exp_neg_x = ggml_v_expf(neg_x);\n    const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x);\n    return _mm512_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__AVX2__) && defined(__FMA__)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m256 ggml_v_expf(__m256 x) {\n  const __m256 r = _mm256_set1_ps(0x1.8p23f);\n  const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r);\n  const __m256 n = _mm256_sub_ps(z, r);\n  const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f),\n                                    _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x));\n  const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23);\n  const __m256 k = _mm256_castsi256_ps(\n      _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1))));\n  const __m256i c = _mm256_castps_si256(\n      _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),\n                    _mm256_set1_ps(126), _CMP_GT_OQ));\n  const __m256 u = _mm256_mul_ps(b, b);\n  const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b,\n                                                                   _mm256_set1_ps(0x1.573e2ep-5f)), u,\n                                                   _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b,\n                                                                   _mm256_set1_ps(0x1.fffdb6p-2f))),\n                                   u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b));\n  if (!_mm256_movemask_ps(_mm256_castsi256_ps(c)))\n    return _mm256_fmadd_ps(j, k, k);\n  const __m256i g = _mm256_and_si256(\n      _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)),\n      _mm256_set1_epi32(0x82000000u));\n  const __m256 s1 =\n      _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u)));\n  const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g));\n  const __m256i d = _mm256_castps_si256(\n      _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n),\n                    _mm256_set1_ps(192), _CMP_GT_OQ));\n  return _mm256_or_ps(\n      _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)),\n      _mm256_andnot_ps(\n          _mm256_castsi256_ps(d),\n          _mm256_or_ps(\n              _mm256_and_ps(_mm256_castsi256_ps(c),\n                            _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)),\n              _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k)))));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m256 ggml_v_silu(__m256 x) {\n    const __m256 one = _mm256_set1_ps(1);\n    const __m256 zero = _mm256_setzero_ps();\n    const __m256 neg_x = _mm256_sub_ps(zero, x);\n    const __m256 exp_neg_x = ggml_v_expf(neg_x);\n    const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x);\n    return _mm256_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__SSE2__) // __AVX2__ / __ARM_NEON\n\n#if defined(__FMA__)\n#define MADD128(x, y, z) _mm_fmadd_ps(x, y, z)\n#define NMADD128(x, y, z) _mm_fnmadd_ps(x, y, z)\n#else\n#define MADD128(x, y, z) _mm_add_ps(_mm_mul_ps(x, y), z)\n#define NMADD128(x, y, z) _mm_sub_ps(z, _mm_mul_ps(x, y))\n#endif\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static __m128 ggml_v_expf(__m128 x) {\n    const __m128 r = _mm_set1_ps(0x1.8p23f);\n    const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r);\n    const __m128 n = _mm_sub_ps(z, r);\n    const __m128 b =\n        NMADD128(n, _mm_set1_ps(0x1.7f7d1cp-20f), NMADD128(n, _mm_set1_ps(0x1.62e4p-1f), x));\n    const __m128i e = _mm_slli_epi32(_mm_castps_si128(z), 23);\n    const __m128 k = _mm_castsi128_ps(_mm_add_epi32(e, _mm_castps_si128(_mm_set1_ps(1))));\n    const __m128i c =\n        _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(126)));\n    const __m128 u = _mm_mul_ps(b, b);\n    const __m128 j =\n        MADD128(MADD128(MADD128(_mm_set1_ps(0x1.0e4020p-7f), b, _mm_set1_ps(0x1.573e2ep-5f)), u,\n                        MADD128(_mm_set1_ps(0x1.555e66p-3f), b, _mm_set1_ps(0x1.fffdb6p-2f))),\n                u, _mm_mul_ps(_mm_set1_ps(0x1.ffffecp-1f), b));\n    if (!_mm_movemask_epi8(c))\n        return MADD128(j, k, k);\n    const __m128i g = _mm_and_si128(_mm_castps_si128(_mm_cmple_ps(n, _mm_setzero_ps())),\n                                    _mm_set1_epi32(0x82000000u));\n    const __m128 s1 = _mm_castsi128_ps(_mm_add_epi32(g, _mm_set1_epi32(0x7f000000u)));\n    const __m128 s2 = _mm_castsi128_ps(_mm_sub_epi32(e, g));\n    const __m128i d =\n        _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(192)));\n    return _mm_or_ps(\n        _mm_and_ps(_mm_castsi128_ps(d), _mm_mul_ps(s1, s1)),\n        _mm_andnot_ps(_mm_castsi128_ps(d),\n                      _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(c), _mm_mul_ps(MADD128(s2, j, s2), s1)),\n                                _mm_andnot_ps(_mm_castsi128_ps(c), MADD128(k, j, k)))));\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static __m128 ggml_v_silu(__m128 x) {\n    const __m128 one = _mm_set1_ps(1);\n    const __m128 zero = _mm_setzero_ps();\n    const __m128 neg_x = _mm_sub_ps(zero, x);\n    const __m128 exp_neg_x = ggml_v_expf(neg_x);\n    const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x);\n    return _mm_div_ps(x, one_plus_exp_neg_x);\n}\n\n#elif defined(__riscv_v_intrinsic)\n\n// adapted from arm limited optimized routine\n// the maximum error is 1.45358 plus 0.5 ulps\n// numbers above 88.38 will flush to infinity\n// numbers beneath -103.97 will flush to zero\ninline static vfloat32m2_t ggml_v_expf_m2(vfloat32m2_t x, int vl) {\n    const vfloat32m2_t r = __riscv_vfmv_v_f_f32m2(0x1.8p23f, vl);\n#ifdef __riscv_xtheadvector\n    // workaround for compiler bug (gcc 14.3.0: Error: unrecognized opcode `th.vmv1r.v v2,v4')\n    vfloat32m2_t z = __riscv_vfadd_vf_f32m2(r, 0.0f, vl);\n    z = __riscv_vfmacc_vf_f32m2(z, 0x1.715476p+0f, x, vl);\n#else\n    const vfloat32m2_t z = __riscv_vfmacc_vf_f32m2(r, 0x1.715476p+0f, x, vl);\n#endif\n    const vfloat32m2_t n = __riscv_vfsub_vv_f32m2(z, r, vl);\n    const vfloat32m2_t b = __riscv_vfnmsac_vf_f32m2(__riscv_vfnmsac_vf_f32m2(x, 0x1.62e4p-1f, n, vl),\n                                                    0x1.7f7d1cp-20f, n, vl);\n    const vuint32m2_t e = __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_f32m2_u32m2(z), 23, vl);\n    const vfloat32m2_t k = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(e, 0x3f800000, vl)); // 1.0f\n    const vbool16_t c = __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 126.0f, vl);\n    const vfloat32m2_t u = __riscv_vfmul_vv_f32m2(b, b, vl);\n    const vfloat32m2_t j = __riscv_vfmacc_vv_f32m2(\n        __riscv_vfmul_vf_f32m2(b, 0x1.ffffecp-1f, vl),\n        __riscv_vfmacc_vv_f32m2(\n            __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.fffdb6p-2f, vl), 0x1.555e66p-3f, b, vl),\n            __riscv_vfmacc_vf_f32m2(__riscv_vfmv_v_f_f32m2(0x1.573e2ep-5f, vl), 0x1.0e4020p-7f, b, vl),\n            u, vl), u, vl);\n    if (!__riscv_vcpop_m_b16(c, vl))\n        return __riscv_vfmacc_vv_f32m2(k, j, k, vl);\n    const vbool16_t  dm = __riscv_vmfle_vf_f32m2_b16(n, 0.0f, vl);\n    const vuint32m2_t d = __riscv_vmerge_vxm_u32m2(__riscv_vmv_v_x_u32m2(0, vl), 0x82000000, dm, vl);\n    const vfloat32m2_t s1 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vadd_vx_u32m2(d, 0x7f000000, vl));\n    const vfloat32m2_t s2 = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vsub_vv_u32m2(e, d, vl));\n    const vfloat32m2_t r1 = __riscv_vmerge_vvm_f32m2(\n        __riscv_vfmacc_vv_f32m2(k, k, j, vl),\n        __riscv_vfmul_vv_f32m2(__riscv_vfmacc_vv_f32m2(s2, s2, j, vl), s1, vl),\n        c, vl);\n    return __riscv_vmerge_vvm_f32m2(\n        r1, __riscv_vfmul_vv_f32m2(s1, s1, vl),\n        __riscv_vmfgt_vf_f32m2_b16(__riscv_vfabs_v_f32m2(n, vl), 192.0f, vl),\n        vl);\n}\n\n// computes silu x/(1+exp(-x)) in single precision vector\ninline static vfloat32m2_t ggml_v_silu_m2(vfloat32m2_t x, int vl) {\n    const vfloat32m2_t neg_x = __riscv_vfneg_v_f32m2(x, vl);\n    const vfloat32m2_t exp_neg_x = ggml_v_expf_m2(neg_x, vl);\n    const vfloat32m2_t one_plus_exp_neg_x = __riscv_vfadd_vf_f32m2(exp_neg_x, 1.0f, vl);\n    return __riscv_vfdiv_vv_f32m2(x, one_plus_exp_neg_x, vl);\n}\n\n#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __riscv_v_intrinsic\n\ninline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_silu_f16(x[i]);\n    }\n}\n\ninline static float ggml_silu_backward_f32(float x, float dy) {\n    const float s = 1.0f/(1.0f + expf(-x));\n    return dy*s*(1.0f + x*(1.0f - s));\n}\n\ninline static ggml_fp16_t ggml_silu_backward_f16(ggml_fp16_t x, ggml_fp16_t dy) {\n    const float v = GGML_CPU_FP16_TO_FP32(x);\n    const float s = 1.0f/(1.0f + expf(-v));\n    return GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(dy)*s*(1.0f + v*(1.0f - s)));\n}\n\ninline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) {\n    for (int i = 0; i < n; ++i) {\n        dx[i] = ggml_silu_backward_f32(x[i], dy[i]);\n    }\n}\n\ninline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, const ggml_fp16_t * x, const ggml_fp16_t * dy) {\n    for (int i = 0; i < n; ++i) {\n        dx[i] = ggml_silu_backward_f16(x[i], dy[i]);\n    }\n}\n\ninline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;\n    }\n}\n\ninline static void ggml_vec_reglu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(x[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16((v > 0.f) ? v * GGML_CPU_FP16_TO_FP32(g[i]) : 0.f);\n    }\n}\n\n#ifdef GGML_GELU_FP16\ninline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {\n    uint16_t t;\n    for (int i = 0; i < n; ++i) {\n        if (x[i] <= -10.0f) {\n            y[i] = 0.0f;\n        } else if (x[i] >= 10.0f) {\n            y[i] = x[i] * g[i];\n        } else {\n            ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n            memcpy(&t, &fp16, sizeof(uint16_t));\n            y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[t]) * g[i];\n        }\n    }\n}\n#else\ninline static void ggml_vec_geglu_f32(const int n, float * y, const float * x, const float * g) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_gelu_f32(x[i]) * g[i];\n    }\n}\n#endif\n\ninline static void ggml_vec_geglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n    const uint16_t * i16 = (const uint16_t *) x;\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(g[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_f16[i16[i]]) * v);\n    }\n}\n\nvoid ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * g);\n\ninline static void ggml_vec_swiglu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n    for (int i = 0; i < n; ++i) {\n        float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n        float gi = GGML_CPU_FP16_TO_FP32(g[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16((xi/(1.0f + expf(-xi))) * gi);\n    }\n}\n\ninline static void ggml_vec_geglu_erf_f32(const int n, float * y, const float * x, const float * g) {\n    for (int i = 0; i < n; ++i) {\n        float xi = x[i];\n        y[i] = 0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * g[i];\n    }\n}\n\ninline static void ggml_vec_geglu_erf_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n    for (int i = 0; i < n; ++i) {\n        float xi = GGML_CPU_FP16_TO_FP32(x[i]);\n        float gi = GGML_CPU_FP16_TO_FP32(g[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(0.5f * xi * (1.0f + erff(xi*SQRT_2_INV)) * gi);\n    }\n}\n\n#ifdef GGML_GELU_QUICK_FP16\ninline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {\n    uint16_t t;\n    for (int i = 0; i < n; ++i) {\n        ggml_fp16_t fp16 = GGML_CPU_FP32_TO_FP16(x[i]);\n        memcpy(&t, &fp16, sizeof(uint16_t));\n        y[i] = GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[t]) * g[i];\n    }\n}\n#else\ninline static void ggml_vec_geglu_quick_f32(const int n, float * y, const float * x, const float * g) {\n    for (int i = 0; i < n; ++i) {\n        y[i] = ggml_gelu_quick_f32(x[i]) * g[i];\n    }\n}\n#endif\n\ninline static void ggml_vec_geglu_quick_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const ggml_fp16_t * g) {\n    const uint16_t * i16 = (const uint16_t *) x;\n    for (int i = 0; i < n; ++i) {\n        float v = GGML_CPU_FP16_TO_FP32(g[i]);\n        y[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(ggml_table_gelu_quick_f16[i16[i]]) * v);\n    }\n}\n\ninline static void ggml_vec_sum_f32(const int n, float * s, const float * x) {\n#ifndef GGML_USE_ACCELERATE\n    ggml_float sum = 0.0;\n    for (int i = 0; i < n; ++i) {\n        sum += (ggml_float)x[i];\n    }\n    *s = (float)sum;\n#else\n    vDSP_sve(x, 1, s, n);\n#endif\n}\n\ninline static void ggml_vec_cumsum_f32(const int n, float * y, const float * x) {\n    for (int i = 0; i < n; ++i) {\n        if (i == 0) {\n            y[i] = x[i];\n        } else {\n            y[i] = y[i - 1] + x[i];\n        }\n    }\n}\n\ninline static void ggml_vec_sum_f32_ggf(const int n, ggml_float * s, const float * x) {\n    ggml_float sum = 0.0;\n    for (int i = 0; i < n; ++i) {\n        sum += (ggml_float)x[i];\n    }\n    *s = sum;\n}\n\ninline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_t * x) {\n    float sum = 0.0f;\n    for (int i = 0; i < n; ++i) {\n        sum += GGML_CPU_FP16_TO_FP32(x[i]);\n    }\n    *s = sum;\n}\n\ninline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {\n    float sum = 0.0f;\n    for (int i = 0; i < n; ++i) {\n        sum += GGML_BF16_TO_FP32(x[i]);\n    }\n    *s = sum;\n}\n\ninline static void ggml_vec_max_f32(const int n, float * s, const float * x) {\n#ifndef GGML_USE_ACCELERATE\n    float max = -INFINITY;\n    for (int i = 0; i < n; ++i) {\n        max = MAX(max, x[i]);\n    }\n    *s = max;\n#else\n    vDSP_maxv(x, 1, s, n);\n#endif\n}\n\ninline static void ggml_vec_norm_inv_f32(const int n, float * s, const float * x) {\n    ggml_vec_norm_f32(n, s, x);\n    *s = 1.f/(*s);\n}\n\ninline static void ggml_vec_argmax_f32(const int n, int * s, const float * x) {\n    float max = -INFINITY;\n    int idx = 0;\n    for (int i = 0; i < n; ++i) {\n        max = MAX(max, x[i]);\n        if (max == x[i]) { idx = i; }\n    }\n    *s = idx;\n}\n\n#ifdef __cplusplus\n}\n#endif\n"
  },
  {
    "path": "ggml/src/ggml-cuda/CMakeLists.txt",
    "content": "cmake_minimum_required(VERSION 3.18)  # for CMAKE_CUDA_ARCHITECTURES\n\nfind_package(CUDAToolkit)\n\nif (CUDAToolkit_FOUND)\n    message(STATUS \"CUDA Toolkit found\")\n\n    if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)\n        # native == GPUs available at build time\n        # 50     == Maxwell, lowest CUDA 12 standard\n        # 60     == P100, FP16 CUDA intrinsics\n        # 61     == Pascal, __dp4a instruction (per-byte integer dot product)\n        # 70     == V100, FP16 tensor cores\n        # 75     == Turing, int8 tensor cores\n        # 80     == Ampere, asynchronous data loading, faster tensor core instructions\n        # 86     == RTX 3000, needs CUDA v11.1\n        # 89     == RTX 4000, needs CUDA v11.8\n        # 120    == Blackwell, needs CUDA v12.8, FP4 tensor cores\n        #\n        # XX-virtual == compile CUDA code as PTX, do JIT compilation to binary code on first run\n        # XX-real    == compile CUDA code as device code for this specific architecture\n        # no suffix  == compile as both PTX and device code\n        #\n        # The default behavior for a non-native is to build virtual architectures as needed to cover all features needed\n        #     for best performance and to also build real architectures for the most commonly used GPUs.\n        if (GGML_NATIVE AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"11.6\" AND CMAKE_VERSION VERSION_GREATER_EQUAL \"3.24\")\n            set(CMAKE_CUDA_ARCHITECTURES \"native\")\n        else()\n            if (CUDAToolkit_VERSION VERSION_LESS \"13\")\n                list(APPEND CMAKE_CUDA_ARCHITECTURES 50-virtual 61-virtual 70-virtual)\n            endif ()\n\n            list(APPEND CMAKE_CUDA_ARCHITECTURES 75-virtual 80-virtual 86-real)\n\n            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"11.8\")\n                list(APPEND CMAKE_CUDA_ARCHITECTURES 89-real)\n            endif()\n\n            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.8\")\n                # The CUDA architecture 120f-virtual would in principle work for Blackwell support\n                #     but the newly added \"f\" suffix conflicted with a preexising regex for validating CUDA architectures in CMake.\n                # So either a recent CMake version or one with the backported fix is needed.\n                # The following versions should work:\n                #   - CMake >= v3.31.8 && CMake < v4.0.0\n                #   - CMake >= v4.0.2\n                # This is NOT documented in the CMake release notes,\n                #     check Modules/Internal/CMakeCUDAArchitecturesValidate.cmake in the CMake git repository instead.\n                # However, the architectures 120a-real and 121a-real should work with basically any CMake version and\n                #     until the release of e.g. Rubin there is no benefit to shipping virtual architectures for Blackwell.\n                list(APPEND CMAKE_CUDA_ARCHITECTURES 120a-real)\n            endif()\n            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.9\")\n                list(APPEND CMAKE_CUDA_ARCHITECTURES 121a-real)\n            endif()\n        endif()\n    endif()\n\n    enable_language(CUDA)\n\n    # TODO: Remove once CCCL 3.2 has been released and bundled with CUDA Toolkit\n    if (GGML_CUDA_CUB_3DOT2)\n        include(FetchContent)\n\n        FetchContent_Declare(\n            CCCL\n            GIT_REPOSITORY https://github.com/nvidia/cccl.git\n            GIT_TAG        v3.2.0\n            GIT_SHALLOW    TRUE\n        )\n\n        FetchContent_MakeAvailable(CCCL)\n    endif()\n\n    # Replace any plain 12X CUDA architectures with their \"architecture-specific\" equivalents 12Xa.\n    # 12X is forwards-compatible, 12Xa is not.\n    # Notably the Blackwell FP4 tensor core instructions are not forwards compatible and therefore need 12Xa.\n    # But while 12X vs. 12Xa can be checked in device code there is (to my knowledge) no easy way to do the same check in host code.\n    # So for now just replace all instances of 12X with 12Xa, this should be fine until Rubin is released.\n    foreach(ARCHS IN ITEMS CMAKE_CUDA_ARCHITECTURES CMAKE_CUDA_ARCHITECTURES_NATIVE)\n        set(FIXED_ARCHS \"\")\n        foreach(ARCH IN LISTS ${ARCHS})\n            if (ARCH MATCHES \"^12[0-9](-real|-virtual)?$\")\n                string(REGEX REPLACE \"^(12[0-9])((-real|-virtual)?)$\" \"\\\\1a\\\\2\" FIXED_ARCH ${ARCH})\n                message(STATUS \"Replacing ${ARCH} in ${ARCHS} with ${FIXED_ARCH}\")\n                list(APPEND FIXED_ARCHS \"${FIXED_ARCH}\")\n            else()\n                list(APPEND FIXED_ARCHS \"${ARCH}\")\n            endif()\n        endforeach()\n        set(${ARCHS} ${FIXED_ARCHS})\n    endforeach()\n\n    # If we try to compile a \"native\" build it will use the 12X architectures and fail.\n    # So we should instead use the native architectures as determined by CMake after replacing 12X with 12Xa.\n    # But if at the time of the build no GPUs are connected at all CMAKE_CUDA_ARCHITECTURES will contain garbage that we should not use.\n    if (CMAKE_CUDA_ARCHITECTURES STREQUAL \"native\" AND CMAKE_CUDA_ARCHITECTURES_NATIVE MATCHES \"^[0-9]+(a|f)?(-real|-virtual)?(;[0-9]+(a|f)?(-real|-virtual)?|;)*$\")\n        set(CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_NATIVE})\n    endif()\n    message(STATUS \"Using CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} CMAKE_CUDA_ARCHITECTURES_NATIVE=${CMAKE_CUDA_ARCHITECTURES_NATIVE}\")\n\n    file(GLOB   GGML_HEADERS_CUDA \"*.cuh\")\n    list(APPEND GGML_HEADERS_CUDA \"../../include/ggml-cuda.h\")\n\n    file(GLOB   GGML_SOURCES_CUDA \"*.cu\")\n    file(GLOB   SRCS \"template-instances/fattn-tile*.cu\")\n    list(APPEND GGML_SOURCES_CUDA ${SRCS})\n    file(GLOB   SRCS \"template-instances/fattn-mma*.cu\")\n    list(APPEND GGML_SOURCES_CUDA ${SRCS})\n    file(GLOB   SRCS \"template-instances/mmq*.cu\")\n    list(APPEND GGML_SOURCES_CUDA ${SRCS})\n    file(GLOB   SRCS \"template-instances/mmf*.cu\")\n    list(APPEND GGML_SOURCES_CUDA ${SRCS})\n\n    if (GGML_CUDA_FA_ALL_QUANTS)\n        file(GLOB   SRCS \"template-instances/fattn-vec*.cu\")\n        list(APPEND GGML_SOURCES_CUDA ${SRCS})\n        add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)\n    else()\n        file(GLOB   SRCS \"template-instances/fattn-vec*q4_0-q4_0.cu\")\n        list(APPEND GGML_SOURCES_CUDA ${SRCS})\n        file(GLOB   SRCS \"template-instances/fattn-vec*q8_0-q8_0.cu\")\n        list(APPEND GGML_SOURCES_CUDA ${SRCS})\n        file(GLOB   SRCS \"template-instances/fattn-vec*f16-f16.cu\")\n        list(APPEND GGML_SOURCES_CUDA ${SRCS})\n    endif()\n\n    ggml_add_backend_library(ggml-cuda\n                             ${GGML_HEADERS_CUDA}\n                             ${GGML_SOURCES_CUDA}\n                            )\n\n    add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})\n\n    if (GGML_CUDA_GRAPHS)\n        add_compile_definitions(GGML_CUDA_USE_GRAPHS)\n    endif()\n\n    if (GGML_CUDA_FORCE_MMQ)\n        add_compile_definitions(GGML_CUDA_FORCE_MMQ)\n    endif()\n\n    if (GGML_CUDA_FORCE_CUBLAS)\n        add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)\n    endif()\n\n    if (GGML_CUDA_NO_VMM)\n        add_compile_definitions(GGML_CUDA_NO_VMM)\n    endif()\n\n    if (NOT GGML_CUDA_FA)\n        add_compile_definitions(GGML_CUDA_NO_FA)\n    endif()\n\n    if (GGML_CUDA_NO_PEER_COPY)\n        add_compile_definitions(GGML_CUDA_NO_PEER_COPY)\n    endif()\n\n    if (GGML_STATIC)\n        if (WIN32)\n            # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library\n            target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas)\n        else ()\n            if (GGML_CUDA_CUB_3DOT2)\n                target_link_libraries(ggml-cuda PRIVATE  CCCL::CCCL)\n            endif()\n            if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"10.1\")\n                target_link_libraries(ggml-cuda PRIVATE  CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)\n            else()\n                target_link_libraries(ggml-cuda PRIVATE  CUDA::cudart_static CUDA::cublas_static)\n            endif()\n        endif()\n    else()\n        if (GGML_CUDA_CUB_3DOT2)\n            target_link_libraries(ggml-cuda PRIVATE  CCCL::CCCL)\n        endif()\n        target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas)\n    endif()\n\n    if (GGML_CUDA_NO_VMM)\n        # No VMM requested, no need to link directly with the cuda driver lib (libcuda.so)\n    else()\n        target_link_libraries(ggml-cuda PRIVATE CUDA::cuda_driver)\n    endif()\n\n    set(CUDA_CXX_FLAGS \"\")\n\n    set(CUDA_FLAGS -use_fast_math -extended-lambda)\n\n    if (GGML_CUDA_DEBUG)\n        list(APPEND CUDA_FLAGS -lineinfo)\n        add_compile_definitions(GGML_CUDA_DEBUG)\n    endif()\n\n    if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL \"12.8\")\n        # Options are:\n        # - none (not recommended)\n        # - speed (nvcc's default)\n        # - balance\n        # - size\n        list(APPEND CUDA_FLAGS -compress-mode=${GGML_CUDA_COMPRESSION_MODE})\n    endif()\n\n    if (GGML_FATAL_WARNINGS)\n        list(APPEND CUDA_FLAGS -Werror all-warnings)\n    endif()\n\n    if (GGML_ALL_WARNINGS AND NOT MSVC)\n        set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c)\n        if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL \"\")\n            list(APPEND NVCC_CMD -ccbin ${CMAKE_CUDA_HOST_COMPILER})\n        endif()\n\n        execute_process(\n            COMMAND ${NVCC_CMD} -Xcompiler --version\n            OUTPUT_VARIABLE CUDA_CCFULLVER\n            ERROR_QUIET\n        )\n\n        if (NOT CUDA_CCFULLVER MATCHES clang)\n            set(CUDA_CCID \"GNU\")\n            execute_process(\n                COMMAND ${NVCC_CMD} -Xcompiler \"-dumpfullversion -dumpversion\"\n                OUTPUT_VARIABLE CUDA_CCVER\n                ERROR_QUIET\n                OUTPUT_STRIP_TRAILING_WHITESPACE\n            )\n        else()\n            if (CUDA_CCFULLVER MATCHES Apple)\n                set(CUDA_CCID \"AppleClang\")\n            else()\n                set(CUDA_CCID \"Clang\")\n            endif()\n            string(REGEX REPLACE \"^.* version ([0-9.]*).*$\" \"\\\\1\" CUDA_CCVER ${CUDA_CCFULLVER})\n        endif()\n\n        message(STATUS \"CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}\")\n\n        ggml_get_flags(${CUDA_CCID} ${CUDA_CCVER})\n        list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS})  # This is passed to -Xcompiler later\n    endif()\n\n    if (NOT MSVC)\n        list(APPEND CUDA_CXX_FLAGS -Wno-pedantic)\n    else()\n        # CCCL 3.2 onwards will require a cpp-standard-compliant preprocessor for MSVC\n        # https://github.com/NVIDIA/cccl/pull/6827\n        list(APPEND CUDA_CXX_FLAGS /Zc:preprocessor)\n    endif()\n\n    list(JOIN   CUDA_CXX_FLAGS \" \" CUDA_CXX_FLAGS_JOINED)  # pass host compiler flags as a single argument\n\n    if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL \"\")\n        list(APPEND CUDA_FLAGS -Xcompiler ${CUDA_CXX_FLAGS_JOINED})\n    endif()\n\n    target_compile_options(ggml-cuda PRIVATE \"$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_FLAGS}>\")\nelse()\n    message(FATAL_ERROR \"CUDA Toolkit not found\")\nendif()\n"
  },
  {
    "path": "ggml/src/ggml-cuda/acc.cu",
    "content": "#include \"acc.cuh\"\n\nstatic __global__ void acc_f32(const float * x, const float * y, float * dst, const int64_t ne,\n        const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n        const int64_t s11, const int64_t s12, const int64_t s13, const int64_t offset) {\n    const int64_t i = blockDim.x * blockIdx.x + threadIdx.x;\n\n    if (i >= ne) {\n        return;\n    }\n\n    int64_t src1_idx = i - offset;\n\n    int64_t tmp = src1_idx;\n    const int64_t i13 = tmp / s13;\n    tmp -= i13 * s13;\n    const int64_t i12 = tmp / s12;\n    tmp -= i12 * s12;\n    const int64_t i11 = tmp / s11;\n    tmp -= i11 * s11;\n    const int64_t i10 = tmp;\n\n    float val = x[i];\n    if (src1_idx >= 0 && i10 < ne10 && i11 < ne11 && i12 < ne12 && i13 < ne13) {\n        val += y[((i13*ne12 + i12) * ne11 + i11) * ne10 + i10];\n    }\n    dst[i] = val;\n}\n\nstatic void acc_f32_cuda(const float * x, const float * y, float * dst, const int64_t n_elements,\n        const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,\n        const int64_t s1, const int64_t s2, const int64_t s3, const int64_t offset, cudaStream_t stream) {\n    const int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;\n    acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, ne13, s1, s2, s3, offset);\n}\n\nvoid ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    const float * src0_d = (const float *) src0->data;\n    const float * src1_d = (const float *) src1->data;\n    float       * dst_d  = (float       *)  dst->data;\n\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_ASSERT(ggml_is_contiguous(src1));\n    GGML_ASSERT(dst->nb[0] == ggml_element_size(dst));\n    GGML_ASSERT(ggml_is_contiguously_allocated(dst));\n\n    const int64_t s1     = dst->op_params[0] / sizeof(float);\n    const int64_t s2     = dst->op_params[1] / sizeof(float);\n    const int64_t s3     = dst->op_params[2] / sizeof(float);\n    const int64_t offset = dst->op_params[3] / sizeof(float);\n\n    acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], s1, s2, s3, offset, stream);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/acc.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_ACC_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/add-id.cu",
    "content": "#include \"add-id.cuh\"\n\nstatic __global__ void add_id_kernel(\n        const float * src0, const float * src1, const int32_t * src2, float * dst,\n        int64_t ne0, int64_t ne1,\n        size_t nb01, size_t nb02,\n        size_t nb11,\n        size_t nb21\n    ) {\n\n    const int64_t i1 = blockIdx.x;\n    const int64_t i2 = blockIdx.y;\n\n    const int i11 = *(const int32_t *) ((const char *) src2 + i1*sizeof(int32_t) + i2*nb21);\n\n    const size_t nb1 = ne0 * sizeof(float);\n    const size_t nb2 = ne1 * nb1;\n\n    float * dst_row = (float *)((char *)dst + i1*nb1 + i2*nb2);\n    const float * src0_row = (const float *)((const char *)src0 +  i1*nb01 + i2*nb02);\n    const float * src1_row = (const float *)((const char *)src1 + i11*nb11);\n\n    for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {\n        dst_row[i0] = src0_row[i0] + src1_row[i0];\n    }\n}\n\nvoid ggml_cuda_op_add_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n    const ggml_tensor * src2 = dst->src[2];\n\n    GGML_TENSOR_TERNARY_OP_LOCALS\n\n    GGML_ASSERT(dst->type  == GGML_TYPE_F32);\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(src2->type == GGML_TYPE_I32);\n\n    GGML_ASSERT(nb00 == sizeof(float));\n    GGML_ASSERT(nb10 == sizeof(float));\n    GGML_ASSERT(nb20 == sizeof(int32_t));\n\n    const float * src0_d = (const float *)src0->data;\n    const float * src1_d = (const float *)src1->data;\n    const int32_t * src2_d = (const int32_t *)src2->data;\n    float * dst_d = (float *)dst->data;\n\n    int threads = std::min((int)ne00, 768); // cols\n    dim3 blocks(ne01, ne02); // n_experts_used, n_tokens\n    add_id_kernel<<<blocks, threads, 0, ctx.stream()>>>(\n        src0_d, src1_d, src2_d, dst_d,\n        ne0, ne1,\n        nb01, nb02,\n        nb11,\n        nb21\n    );\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/add-id.cuh",
    "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_add_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/arange.cu",
    "content": "#include \"arange.cuh\"\n\nstatic __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {\n    // blockIDx.x: idx of ne0 / BLOCK_SIZE\n    int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n    if (nidx >= ne0) {\n        return;\n    }\n    dst[nidx] = start + step * nidx;\n}\n\nstatic void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {\n    int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;\n    arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start,  step);\n}\n\nvoid ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    float * dst_d = (float *)dst->data;\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(dst->type == GGML_TYPE_F32);\n\n    float start;\n    float stop;\n    float step;\n    memcpy(&start, (float *)dst->op_params + 0, sizeof(float));\n    memcpy(&stop,  (float *)dst->op_params + 1, sizeof(float));\n    memcpy(&step,  (float *)dst->op_params + 2, sizeof(float));\n\n    int64_t steps = (int64_t)ceil((stop - start) / step);\n    GGML_ASSERT(ggml_nelements(dst) == steps);\n\n    arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/arange.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_ARANGE_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/argmax.cu",
    "content": "#include <algorithm>\n#include <cstdint>\n\n#include \"argmax.cuh\"\n#include \"common.cuh\"\n#include \"sum.cuh\"\n\nstatic __global__ void argmax_f32(const float * __restrict__ x, int32_t * __restrict__ dst, const int64_t ncols) {\n    const int64_t row = blockIdx.x;\n\n    float maxval = -FLT_MAX;\n    int   argmax = -1;\n    const float * rowx = x + row * ncols;\n\n    for (int32_t col = threadIdx.x; col < ncols; col += blockDim.x) {\n        const float val = rowx[col];\n        if (val > maxval) {\n            maxval = val;\n            argmax = col;\n        }\n    }\n\n#pragma unroll\n    for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {\n        const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);\n        const int   col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);\n        if (val > maxval) {\n            maxval = val;\n            argmax = col;\n        }\n    }\n\n    const int n_warps = blockDim.x / WARP_SIZE;\n    const int lane_id = threadIdx.x % WARP_SIZE;\n    const int warp_id = threadIdx.x / WARP_SIZE;\n    if (n_warps > 1) {\n        constexpr int    max_warps = 1024 / WARP_SIZE;\n        __shared__ float shared_maxval[max_warps];\n        __shared__ int   shared_argmax[max_warps];\n        if (lane_id == 0) {\n            shared_maxval[warp_id] = maxval;\n            shared_argmax[warp_id] = argmax;\n        }\n\n        __syncthreads();\n\n        if (warp_id == 0) {\n            if (lane_id < n_warps) {\n                maxval = shared_maxval[lane_id];\n                argmax = shared_argmax[lane_id];\n            }\n#pragma unroll\n            for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {\n                const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);\n                const int   col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);\n                if (val > maxval) {\n                    maxval = val;\n                    argmax = col;\n                }\n            }\n        }\n    }\n\n    if (warp_id == 0 && lane_id == 0) {\n        dst[row] = argmax;\n    }\n}\n\nvoid ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_I32);\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n\n    const int64_t ne00  = src0->ne[0];\n    const int64_t nrows = ggml_nrows(src0);\n\n    const float * src0_d = (const float *) src0->data;\n    int32_t     * dst_d  = (int32_t     *) dst->data;\n\n    cudaStream_t stream = ctx.stream();\n\n    const int64_t num_blocks = nrows;\n    const int64_t num_threads = std::min<int64_t>(1024, (ne00 + WARP_SIZE - 1) / WARP_SIZE * WARP_SIZE);\n    const dim3 blocks_dim(num_threads, 1, 1);\n    const dim3 blocks_num(num_blocks, 1, 1);\n\n    argmax_f32<<<blocks_num, blocks_dim, 0, stream>>>(src0_d, dst_d, ne00);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/argmax.cuh",
    "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_argmax(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/argsort.cu",
    "content": "#include \"argsort.cuh\"\n\n#ifdef GGML_CUDA_USE_CUB\n#    include <cub/cub.cuh>\n#    if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1)\n#        define STRIDED_ITERATOR_AVAILABLE\n#    endif\nusing namespace cub;\n#endif  // GGML_CUDA_USE_CUB\n\nstatic __global__ void init_indices(int * indices, const int ncols, const int nrows) {\n    const int col = blockIdx.x * blockDim.x + threadIdx.x;\n    const int row = blockIdx.y;\n\n    if (col < ncols && row < nrows) {\n        indices[row * ncols + col] = col;\n    }\n}\n\n#ifndef STRIDED_ITERATOR_AVAILABLE\nstatic __global__ void init_offsets(int * offsets, const int ncols, const int nrows) {\n    const int idx = blockIdx.x * blockDim.x + threadIdx.x;\n    if (idx <= nrows) {\n        offsets[idx] = idx * ncols;\n    }\n}\n#endif  // STRIDED_ITERATOR_AVAILABLE\n\n#ifdef GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool,\n                              const float *    x,\n                              int *            dst,\n                              const int        ncols,\n                              const int        nrows,\n                              ggml_sort_order  order,\n                              cudaStream_t     stream) {\n    ggml_cuda_pool_alloc<int>   temp_indices_alloc(pool, ncols * nrows);\n    ggml_cuda_pool_alloc<float> temp_keys_alloc(pool, ncols * nrows);\n\n    int *   temp_indices = temp_indices_alloc.get();\n    float * temp_keys    = temp_keys_alloc.get();\n\n    static const int block_size = 256;\n    const dim3 grid_size((ncols + block_size - 1) / block_size, nrows);\n    init_indices<<<grid_size, block_size, 0, stream>>>(temp_indices, ncols, nrows);\n\n#ifdef STRIDED_ITERATOR_AVAILABLE\n    auto offset_iterator = cuda::make_strided_iterator(cuda::make_counting_iterator(0), ncols);\n#else\n    ggml_cuda_pool_alloc<int> offsets_alloc(pool, nrows + 1);\n    int *                     offset_iterator = offsets_alloc.get();\n    const dim3                offset_grid((nrows + block_size - 1) / block_size);\n    init_offsets<<<offset_grid, block_size, 0, stream>>>(offset_iterator, ncols, nrows);\n#endif\n    CUDA_CHECK(cudaMemcpyAsync(temp_keys, x, ncols * nrows * sizeof(float), cudaMemcpyDeviceToDevice, stream));\n\n    size_t temp_storage_bytes = 0;\n\n    if (order == GGML_SORT_ORDER_ASC) {\n        if (nrows == 1) {\n            DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys,  // keys (in-place)\n                                       temp_indices, dst,                                  // values (indices)\n                                       ncols, 0, sizeof(float) * 8, stream);\n        } else {\n            DeviceSegmentedSort::SortPairs(nullptr, temp_storage_bytes, temp_keys, temp_keys,  // keys (in-place)\n                                           temp_indices, dst,                                  // values (indices)\n                                           ncols * nrows, nrows,  // num items, num segments\n                                           offset_iterator, offset_iterator + 1, stream);\n        }\n    } else {\n        if (nrows == 1) {\n            DeviceRadixSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys,  // keys (in-place)\n                                                 temp_indices, dst,                                  // values (indices)\n                                                 ncols, 0, sizeof(float) * 8, stream);\n        } else {\n            DeviceSegmentedSort::SortPairsDescending(nullptr, temp_storage_bytes, temp_keys, temp_keys, temp_indices,\n                                                     dst, ncols * nrows, nrows, offset_iterator, offset_iterator + 1,\n                                                     stream);\n        }\n    }\n\n    ggml_cuda_pool_alloc<uint8_t> temp_storage_alloc(pool, temp_storage_bytes);\n    void *                        d_temp_storage = temp_storage_alloc.get();\n\n    if (order == GGML_SORT_ORDER_ASC) {\n        if (nrows == 1) {\n            DeviceRadixSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys,  // keys (in-place)\n                                       temp_indices, dst,  // values (indices)\n                                       ncols, 0, sizeof(float) * 8, stream);\n        } else {\n            DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys, temp_indices, dst,\n                                           ncols * nrows, nrows, offset_iterator, offset_iterator + 1, stream);\n        }\n    } else {\n        if (nrows == 1) {\n            DeviceRadixSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys,  // keys (in-place)\n                                                 temp_indices, dst,                                  // values (indices)\n                                                 ncols, 0, sizeof(float) * 8, stream);\n        } else {\n            DeviceSegmentedSort::SortPairsDescending(d_temp_storage, temp_storage_bytes, temp_keys, temp_keys,\n                                                     temp_indices, dst, ncols * nrows, nrows, offset_iterator,\n                                                     offset_iterator + 1, stream);\n        }\n    }\n}\n#endif  // GGML_CUDA_USE_CUB\n\n// Bitonic sort implementation\ntemplate<typename T>\nstatic inline __device__ void ggml_cuda_swap(T & a, T & b) {\n    T tmp = a;\n    a = b;\n    b = tmp;\n}\n\ntemplate<ggml_sort_order order>\nstatic __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {\n    // bitonic sort\n    int col = threadIdx.x;\n    int row = blockIdx.x;\n\n    if (col >= ncols_pad) {\n        return;\n    }\n\n    const float * x_row = x + row * ncols;\n    extern __shared__ int dst_row[];\n\n    // initialize indices\n    dst_row[col] = col;\n\n    __syncthreads();\n\n    for (int k = 2; k <= ncols_pad; k *= 2) {\n        for (int j = k / 2; j > 0; j /= 2) {\n            int ixj = col ^ j;\n            if (ixj > col) {\n                if ((col & k) == 0) {\n                    if (dst_row[col] >= ncols ||\n                        (dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?\n                            x_row[dst_row[col]] > x_row[dst_row[ixj]] :\n                            x_row[dst_row[col]] < x_row[dst_row[ixj]]))\n                    ) {\n                        ggml_cuda_swap(dst_row[col], dst_row[ixj]);\n                    }\n                } else {\n                    if (dst_row[ixj] >= ncols ||\n                        (dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?\n                            x_row[dst_row[col]] < x_row[dst_row[ixj]] :\n                            x_row[dst_row[col]] > x_row[dst_row[ixj]]))\n                    ) {\n                        ggml_cuda_swap(dst_row[col], dst_row[ixj]);\n                    }\n                }\n            }\n            __syncthreads();\n        }\n    }\n\n    // copy the result to dst without the padding\n    if (col < ncols) {\n        dst[row * ncols + col] = dst_row[col];\n    }\n}\n\nstatic int next_power_of_2(int x) {\n    int n = 1;\n    while (n < x) {\n        n *= 2;\n    }\n    return n;\n}\n\nvoid argsort_f32_i32_cuda_bitonic(const float *   x,\n                                  int *           dst,\n                                  const int       ncols,\n                                  const int       nrows,\n                                  ggml_sort_order order,\n                                  cudaStream_t    stream) {\n    // bitonic sort requires ncols to be power of 2\n    const int ncols_pad = next_power_of_2(ncols);\n\n    const dim3 block_dims(ncols_pad, 1, 1);\n    const dim3 block_nums(nrows, 1, 1);\n    const size_t shared_mem = ncols_pad * sizeof(int);\n\n    // FIXME: this limit could be raised by ~2-4x on Ampere or newer\n    GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);\n\n    if (order == GGML_SORT_ORDER_ASC) {\n        k_argsort_f32_i32<GGML_SORT_ORDER_ASC>\n            <<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);\n    } else if (order == GGML_SORT_ORDER_DESC) {\n        k_argsort_f32_i32<GGML_SORT_ORDER_DESC>\n            <<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);\n    } else {\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\nvoid ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const float * src0_d = (const float *)src0->data;\n    float * dst_d = (float *)dst->data;\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_I32);\n    GGML_ASSERT(ggml_is_contiguous(src0));\n\n    const int64_t ncols = src0->ne[0];\n    const int64_t nrows = ggml_nrows(src0);\n\n    enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];\n\n#ifdef GGML_CUDA_USE_CUB\n    const int    ncols_pad      = next_power_of_2(ncols);\n    const size_t shared_mem     = ncols_pad * sizeof(int);\n    const size_t max_shared_mem = ggml_cuda_info().devices[ggml_cuda_get_device()].smpb;\n\n    if (shared_mem > max_shared_mem || ncols > 1024) {\n        ggml_cuda_pool & pool = ctx.pool();\n        argsort_f32_i32_cuda_cub(pool, src0_d, (int *) dst_d, ncols, nrows, order, stream);\n    } else {\n        argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream);\n    }\n#else\n    argsort_f32_i32_cuda_bitonic(src0_d, (int *) dst_d, ncols, nrows, order, stream);\n#endif\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/argsort.cuh",
    "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\n#ifdef GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_cub(ggml_cuda_pool & pool,\n                              const float *    x,\n                              int *            dst,\n                              const int        ncols,\n                              const int        nrows,\n                              ggml_sort_order  order,\n                              cudaStream_t     stream);\n#endif  // GGML_CUDA_USE_CUB\nvoid argsort_f32_i32_cuda_bitonic(const float *   x,\n                                  int *           dst,\n                                  const int       ncols,\n                                  const int       nrows,\n                                  ggml_sort_order order,\n                                  cudaStream_t    stream);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/binbcast.cu",
    "content": "#include \"binbcast.cuh\"\n#include <cstdint>\n#include <utility>\n\nstatic __device__ __forceinline__ float op_repeat(const float a, const float b) {\n    return b;\n    GGML_UNUSED(a);\n}\n\nstatic __device__ __forceinline__ float op_add(const float a, const float b) {\n    return a + b;\n}\n\nstatic __device__ __forceinline__ float op_sub(const float a, const float b) {\n    return a - b;\n}\n\nstatic __device__ __forceinline__ float op_mul(const float a, const float b) {\n    return a * b;\n}\n\nstatic __device__ __forceinline__ float op_div(const float a, const float b) {\n    return a / b;\n}\n\ntemplate <float (*bin_op)(const float, const float),\n          typename src0_t,\n          typename src1_t,\n          typename dst_t,\n          typename... src1_ptrs>\nstatic __global__ void k_bin_bcast(const src0_t *         src0,\n                                   const src1_t *         src1,\n                                   dst_t *                dst,\n                                   const int              ne0,\n                                   const int              ne1,\n                                   const int              ne2,\n                                   const uint3            ne3,\n                                   const uint3            ne10,\n                                   const uint3            ne11,\n                                   const uint3            ne12,\n                                   const uint3            ne13,\n                                 /*const int              s0,*/\n                                   const int              s1,\n                                   const int              s2,\n                                   const int              s3,\n                                   const int              s00,\n                                   const int              s01,\n                                   const int              s02,\n                                   const int              s03,\n                                   const int              s10,\n                                   const int              s11,\n                                   const int              s12,\n                                   const int              s13,\n                                   src1_ptrs... src1s) {\n    const uint32_t i0s = blockDim.x * blockIdx.x + threadIdx.x;\n    const uint32_t i1  = (blockDim.y * blockIdx.y + threadIdx.y);\n    const uint32_t i2  = fastdiv((blockDim.z * blockIdx.z + threadIdx.z), ne3);\n    const uint32_t i3  = (blockDim.z * blockIdx.z + threadIdx.z) - (i2 * ne3.z);\n\n    if (i0s >= (uint32_t)ne0 || i1 >= (uint32_t)ne1 || i2 >= (uint32_t)ne2 || i3 >= ne3.z) {\n        return;\n    }\n\n    const uint32_t i11 = fastmodulo(i1, ne11);\n    const uint32_t i12 = fastmodulo(i2, ne12);\n    const uint32_t i13 = fastmodulo(i3, ne13);\n\n    const size_t i_src0 =  i3*s03 +  i2*s02 +  i1*s01;\n    const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n    const size_t i_dst  =  i3*s3  +  i2*s2  +  i1*s1;\n\n    const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr;\n    dst_t * dst_row = dst + i_dst;\n\n    for (int i0 = i0s; i0 < ne0; i0 += blockDim.x * gridDim.x) {\n        const uint32_t i10 = fastmodulo(i0, ne10);\n\n        float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;\n        if constexpr (sizeof...(src1_ptrs) > 0) {\n            result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));\n        } else {\n            result = bin_op(result, (float)src1[i_src1 + i10*s10]);\n        }\n\n        dst_row[i0] = (dst_t) result;\n    }\n}\n\ntemplate <float (*bin_op)(const float, const float),\n          typename src0_t,\n          typename src1_t,\n          typename dst_t,\n          typename... src1_ptrs>\nstatic __global__ void k_bin_bcast_unravel(const src0_t *         src0,\n                                           const src1_t *         src1,\n                                           dst_t *                dst,\n                                           const uint3            ne0,\n                                           const uint3            ne1,\n                                           const uint3            ne2,\n                                           const uint32_t         ne3,\n                                           const uint3            prod_012,\n                                           const uint3            prod_01,\n                                           const uint3            ne10,\n                                           const uint3            ne11,\n                                           const uint3            ne12,\n                                           const uint3            ne13,\n                                         /*const int              s0,*/\n                                           const int              s1,\n                                           const int              s2,\n                                           const int              s3,\n                                           const int              s00,\n                                           const int              s01,\n                                           const int              s02,\n                                           const int              s03,\n                                           const int              s10,\n                                           const int              s11,\n                                           const int              s12,\n                                           const int              s13,\n                                           src1_ptrs... src1s) {\n    const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n    const uint32_t i3 = fastdiv(i, prod_012);\n    const uint32_t i2 = fastdiv(i - i3 * prod_012.z, prod_01);\n    const uint32_t i1 = fastdiv(i - i3 * prod_012.z - i2 * prod_01.z, ne0);\n    const uint32_t i0 = i - i3 * prod_012.z - i2 * prod_01.z - i1 * ne0.z;\n\n    if (i0 >= ne0.z || i1 >= ne1.z || i2 >= ne2.z || i3 >= ne3) {\n        return;\n    }\n\n    const int i11 = fastmodulo(i1, ne11);\n    const int i12 = fastmodulo(i2, ne12);\n    const int i13 = fastmodulo(i3, ne13);\n\n    const size_t i_src0 =  i3*s03 +  i2*s02 +  i1*s01;\n    const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;\n    const size_t i_dst  =  i3*s3  +  i2*s2  +  i1*s1;\n\n    const src0_t * src0_row = src0 ? (src0 + i_src0) : nullptr;\n    dst_t * dst_row = dst + i_dst;\n\n    const int i10 = fastmodulo(i0, ne10);\n\n    float result = src0_row ? (float) src0_row[i0*s00] : 0.0f;\n    if constexpr (sizeof...(src1_ptrs) > 0) {\n        result = (..., (result = bin_op(result, (float)src1s[i_src1 + i10*s10])));\n    } else {\n        result = bin_op(result, (float)src1[i_src1 + i10*s10]);\n    }\n\n    dst_row[i0] = (dst_t) result;\n}\n\ntemplate <float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t, size_t... I>\nstatic void launch_bin_bcast_pack(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,\n                                  const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,\n                                  cudaStream_t stream, std::index_sequence<I...>) {\n    GGML_TENSOR_BINARY_OP_LOCALS\n\n    int nr0 = ne10 / ne0;\n    int nr1 = ne11 / ne1;\n    int nr2 = ne12 / ne2;\n    int nr3 = ne13 / ne3;\n\n    int nr[4] = { nr0, nr1, nr2, nr3 };\n\n    int64_t cne[]  = { ne0, ne1, ne2, ne3 };\n    int64_t cne0[] = { ne00, ne01, ne02, ne03 };\n    int64_t cne1[] = { ne10, ne11, ne12, ne13 };\n\n    size_t cnb[]  = { nb0, nb1, nb2, nb3 };\n    size_t cnb0[] = { nb00, nb01, nb02, nb03 };\n    size_t cnb1[] = { nb10, nb11, nb12, nb13 };\n\n    auto collapse = [](int64_t cne[]) {\n        cne[0] *= cne[1];\n        cne[1] = cne[2];\n        cne[2] = cne[3];\n        cne[3] = 1;\n    };\n\n    auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {\n        cnb[1] *= cne[1];\n        cnb[2] *= cne[2];\n        cnb[3] *= cne[3];\n    };\n\n    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && !ggml_is_permuted(src0) && !ggml_is_permuted(src1)) {\n        for (int i = 0; i < 4; i++) {\n            if (nr[i] != 1) {\n                break;\n            }\n            if (i > 0) {\n                collapse_nb(cnb, cne);\n                collapse_nb(cnb0, cne0);\n                collapse_nb(cnb1, cne1);\n                collapse(cne);\n                collapse(cne0);\n                collapse(cne1);\n            }\n        }\n    }\n\n    {\n        int64_t ne0 = cne[0];\n        int64_t ne1 = cne[1];\n        int64_t ne2 = cne[2];\n        int64_t ne3 = cne[3];\n\n        //int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);\n        //int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);\n        //int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);\n        //int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);\n\n        size_t nb0 = cnb[0];\n        size_t nb1 = cnb[1];\n        size_t nb2 = cnb[2];\n        size_t nb3 = cnb[3];\n\n        size_t nb00 = cnb0[0];\n        size_t nb01 = cnb0[1];\n        size_t nb02 = cnb0[2];\n        size_t nb03 = cnb0[3];\n\n        size_t nb10 = cnb1[0];\n        size_t nb11 = cnb1[1];\n        size_t nb12 = cnb1[2];\n        size_t nb13 = cnb1[3];\n\n      //size_t s0 = nb0 / sizeof(dst_t);\n        size_t s1 = nb1 / sizeof(dst_t);\n        size_t s2 = nb2 / sizeof(dst_t);\n        size_t s3 = nb3 / sizeof(dst_t);\n\n        size_t s10 = nb10 / sizeof(src1_t);\n        size_t s11 = nb11 / sizeof(src1_t);\n        size_t s12 = nb12 / sizeof(src1_t);\n        size_t s13 = nb13 / sizeof(src1_t);\n\n        size_t s00 = nb00 / sizeof(src0_t);\n        size_t s01 = nb01 / sizeof(src0_t);\n        size_t s02 = nb02 / sizeof(src0_t);\n        size_t s03 = nb03 / sizeof(src0_t);\n\n        GGML_ASSERT(nb0 % sizeof(dst_t) == 0);\n        GGML_ASSERT(nb1 % sizeof(dst_t) == 0);\n        GGML_ASSERT(nb2 % sizeof(dst_t) == 0);\n        GGML_ASSERT(nb3 % sizeof(dst_t) == 0);\n\n        GGML_ASSERT(nb00 % sizeof(src0_t) == 0);\n        GGML_ASSERT(nb01 % sizeof(src0_t) == 0);\n        GGML_ASSERT(nb02 % sizeof(src0_t) == 0);\n        GGML_ASSERT(nb03 % sizeof(src0_t) == 0);\n\n        GGML_ASSERT(nb10 % sizeof(src1_t) == 0);\n        GGML_ASSERT(nb11 % sizeof(src1_t) == 0);\n        GGML_ASSERT(nb12 % sizeof(src1_t) == 0);\n        GGML_ASSERT(nb13 % sizeof(src1_t) == 0);\n\n        const int block_size = 128;\n\n        int64_t hne0 = std::max(ne0 / 2LL, 1LL);\n\n        dim3 block_dims;\n        block_dims.x = std::min<unsigned int>(hne0, block_size);\n        block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);\n        block_dims.z = std::min(std::min<unsigned int>(ne2 * ne3, block_size / block_dims.x / block_dims.y), 64U);\n\n        dim3 block_nums((hne0 + block_dims.x - 1) / block_dims.x, (ne1 + block_dims.y - 1) / block_dims.y,\n                        (ne2 * ne3 + block_dims.z - 1) / block_dims.z);\n\n        const uint3 ne10 = init_fastdiv_values((uint32_t) cne1[0]);\n        const uint3 ne11 = init_fastdiv_values((uint32_t) cne1[1]);\n        const uint3 ne12 = init_fastdiv_values((uint32_t) cne1[2]);\n        const uint3 ne13 = init_fastdiv_values((uint32_t) cne1[3]);\n\n        if (block_nums.z > 65535 || block_nums.y > 65535) {\n            int         block_num  = (ne0 * ne1 * ne2 * ne3 + block_size - 1) / block_size;\n            const uint3 prod_012    = init_fastdiv_values((uint32_t) (ne0 * ne1 * ne2));\n            const uint3 prod_01     = init_fastdiv_values((uint32_t) (ne0 * ne1));\n            const uint3 ne0_fastdiv = init_fastdiv_values((uint32_t) ne0);\n            const uint3 ne1_fastdiv = init_fastdiv_values((uint32_t) ne1);\n            const uint3 ne2_fastdiv = init_fastdiv_values((uint32_t) ne2);\n\n            if constexpr (sizeof...(I) > 0) {\n                k_bin_bcast_unravel<bin_op, src0_t, src1_t, dst_t><<<block_num, block_size, 0, stream>>>(\n                    src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv, ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11,\n                    ne12, ne13,\n                  /*s0,*/ s1,  s2,  s3,\n                    s00, s01, s02, s03,\n                    s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);\n            } else {\n                k_bin_bcast_unravel<bin_op, src0_t, src1_t, dst_t>\n                    <<<block_num, block_size, 0, stream>>>(src0_dd, src1_dd, dst_dd, ne0_fastdiv, ne1_fastdiv,\n                                                           ne2_fastdiv, ne3, prod_012, prod_01, ne10, ne11, ne12, ne13,\n                                                         /*s0,*/ s1,  s2,  s3,\n                                                           s00, s01, s02, s03,\n                                                           s10, s11, s12, s13);\n            }\n        } else {\n            const uint3 ne3_fastdiv = init_fastdiv_values((uint32_t) ne3);\n            if constexpr (sizeof...(I) > 0) {\n                k_bin_bcast<bin_op, src0_t, src1_t, dst_t><<<block_nums, block_dims, 0, stream>>>(\n                    src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,\n                  /*s0,*/ s1, s2,  s3,\n                    s00 ,s01, s02, s03,\n                    s10, s11, s12, s13, (const src1_t *) dst->src[I + 1]->data...);\n            } else {\n                k_bin_bcast<bin_op, src0_t, src1_t, dst_t><<<block_nums, block_dims, 0, stream>>>(\n                    src0_dd, src1_dd, dst_dd, ne0, ne1, ne2, ne3_fastdiv, ne10, ne11, ne12, ne13,\n                  /*s0,*/ s1,  s2,  s3,\n                    s00, s01, s02, s03,\n                    s10, s11, s12, s13);\n            }\n        }\n    }\n}\n\ntemplate <typename T>\nstatic __global__ void k_repeat_back(\n    const T * __restrict__ src, T * __restrict__ dst, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n    const size_t s00, const size_t s01, const size_t s02, const size_t s03,\n    const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3) {\n\n    const int64_t tid0  = int64_t(blockIdx.x)*blockDim.x + threadIdx.x;\n    const int64_t tid1  = int64_t(blockIdx.y)*blockDim.y + threadIdx.y;\n    const int64_t tid23 = int64_t(blockIdx.z)*blockDim.z + threadIdx.z;\n    const int64_t tid2  = tid23 % ne2;\n    const int64_t tid3  = tid23 / ne2;\n\n    if (tid0 >= ne0) {\n        return;\n    }\n\n    T sum = 0;\n    for (int64_t i3 = tid3; i3 < ne03; i3 += ne3) {\n        for (int64_t i2 = tid2; i2 < ne02; i2 += ne2) {\n            for (int64_t i1 = tid1; i1 < ne01; i1 += ne1) {\n                for (int64_t i0 = tid0; i0 < ne00; i0 += ne0) {\n                    sum += src[i3*s03 + i2*s02 + i1*s01 + i0*s00];\n                }\n            }\n        }\n    }\n    dst[tid3*ne2*ne1*ne0 + tid2*ne1*ne0 + tid1*ne0 + tid0] = sum;\n}\n\ntemplate <float (*bin_op)(const float, const float), int n_fuse = 1>\nstruct bin_bcast_cuda {\n    template<typename src0_t, typename src1_t, typename dst_t>\n    void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,\n            const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,\n            cudaStream_t stream) {\n        launch_bin_bcast_pack<bin_op, src0_t, src1_t, dst_t>(\n            src0, src1, dst, src0_dd, src1_dd, dst_dd, stream, std::make_index_sequence<n_fuse>{});\n    }\n};\n\ntemplate <typename T>\nstatic void repeat_back_cuda(\n    const T * src, T * dst, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n    const size_t s00, const size_t s01, const size_t s02, const size_t s03,\n    const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {\n\n    const dim3 block_dims(WARP_SIZE, 1, 1);\n    const dim3 block_nums((ne0 + WARP_SIZE - 1) / WARP_SIZE, ne1, ne2*ne3);\n    k_repeat_back<T><<<block_nums, block_dims, 0, stream>>>\n        (src, dst, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3);\n}\n\ntemplate<class op>\nstatic void ggml_cuda_op_bin_bcast(\n    const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,\n    const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {\n\n    GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);\n\n    if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n        op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n        op()(src0, src1, dst, (const half *) src0_dd, (const half *)src1_dd, (half *) dst_dd, stream);\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n        op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);\n    } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n        op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);\n    } else {\n        fprintf(stderr, \"%s: unsupported types: dst: %s, src0: %s, src1: %s\\n\", __func__,\n            ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\nvoid ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat, 0>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_sub>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\nvoid ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());\n}\n\ntemplate <float (*op)(const float, const float), int n_fuse>\nstatic void ggml_cuda_op_fused_binbcast_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    cudaStream_t stream = ctx.stream();\n\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {\n        launch_bin_bcast_pack<op, float, float, float>(src0, src1, dst,\n            (const float *) src0->data, (const float *) src1->data, (float *) dst->data,\n            stream, std::make_index_sequence<n_fuse>{});\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {\n        launch_bin_bcast_pack<op, half, half, half>(src0, src1, dst,\n            (const half *) src0->data, (const half *) src1->data, (half *) dst->data,\n            stream, std::make_index_sequence<n_fuse>{});\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {\n        launch_bin_bcast_pack<op, half, float, half>(src0, src1, dst,\n            (const half *) src0->data, (const float *) src1->data, (half *) dst->data,\n            stream, std::make_index_sequence<n_fuse>{});\n    } else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {\n        launch_bin_bcast_pack<op, half, float, float>(src0, src1, dst,\n            (const half *) src0->data, (const float *) src1->data, (float *) dst->data,\n            stream, std::make_index_sequence<n_fuse>{});\n    } else {\n        fprintf(stderr,\n                \"%s: unsupported types for fusion: dst: %s, src0: %s, src1: %s\\n\",\n                __func__, ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));\n        GGML_ABORT(\"fatal error\");\n    }\n}\n\n\nvoid ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse) {\n    GGML_ASSERT(2 <= n_fuse && n_fuse <= 8);\n\n    switch (n_fuse) {\n        case 2:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 2>(ctx, dst);\n            break;\n        case 3:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 3>(ctx, dst);\n            break;\n        case 4:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 4>(ctx, dst);\n            break;\n        case 5:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 5>(ctx, dst);\n            break;\n        case 6:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 6>(ctx, dst);\n            break;\n        case 7:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 7>(ctx, dst);\n            break;\n        case 8:\n            ggml_cuda_op_fused_binbcast_impl<op_add, 8>(ctx, dst);\n            break;\n        default:\n            GGML_ASSERT(false && \"Unsupported n_fuse value\");\n    }\n}\n\nvoid ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    GGML_ASSERT(src0->type == dst->type);\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_can_repeat(dst, src0));\n\n    cudaStream_t stream = ctx.stream();\n\n    GGML_TENSOR_UNARY_OP_LOCALS;\n\n    GGML_ASSERT(ne2*ne3 <= (1 << 15));\n\n    const size_t ts = ggml_type_size(src0->type);\n    const size_t s00 = nb00 / ts;\n    const size_t s01 = nb01 / ts;\n    const size_t s02 = nb02 / ts;\n    const size_t s03 = nb03 / ts;\n\n    switch (dst->type) {\n        case GGML_TYPE_F32: {\n            const float * src0_d = (const float *) src0->data;\n            float       * dst_d  = (float       *) dst->data;\n            repeat_back_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s00, s01, s02, s03, ne0, ne1, ne2, ne3, stream);\n        } break;\n        default: {\n            GGML_ASSERT(false);\n        } break;\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/binbcast.cuh",
    "content": "#include \"common.cuh\"\n\nvoid ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\nvoid ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/clamp.cu",
    "content": "#include \"clamp.cuh\"\n\nstatic __device__ __forceinline__ float op_clamp(float x, float min, float max) {\n    return fminf(fmaxf(x, min), max);\n}\n\ntemplate <class T>\nstatic __global__ void op_clamp_kernel(const T * x, T * dst, const T min, const T max, const int k) {\n    const int i = blockDim.x*blockIdx.x + threadIdx.x;\n\n    if (i >= k) {\n        return;\n    }\n\n    dst[i] = (T)op_clamp((float)x[i], (float)min, (float)max);\n}\n\ntemplate <class T>\nstatic void clamp_cuda(const T * x, T * dst, const T min, const T max, const int k, cudaStream_t stream) {\n    const int num_blocks = (k + CUDA_CLAMP_BLOCK_SIZE - 1) / CUDA_CLAMP_BLOCK_SIZE;\n    op_clamp_kernel<<<num_blocks, CUDA_CLAMP_BLOCK_SIZE, 0, stream>>>(x, dst, min, max, k);\n}\n\n\nvoid ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const void * src0_d = src0->data;\n    void * dst_d = dst->data;\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32 ||  dst->type == GGML_TYPE_F16);\n    GGML_ASSERT(src0->type == dst->type);\n\n    float min;\n    float max;\n    memcpy(&min, dst->op_params, sizeof(float));\n    memcpy(&max, (float *) dst->op_params + 1, sizeof(float));\n\n    if (src0->type == GGML_TYPE_F16) {\n        clamp_cuda((const half *)src0_d, (half *)dst_d, (half)min, (half)max, ggml_nelements(src0), stream);\n    } else {\n        clamp_cuda((const float *)src0_d, (float *)dst_d, (float)min, (float)max, ggml_nelements(src0), stream);\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/clamp.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CLAMP_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/common.cuh",
    "content": "#pragma once\n\n#include \"ggml.h\"\n#include \"ggml-impl.h\"\n#include \"ggml-cuda.h\"\n\n#include <cstdint>\n#include <memory>\n\n#if defined(GGML_USE_HIP)\n#define GGML_COMMON_DECL_HIP\n#define GGML_COMMON_IMPL_HIP\n#else\n#define GGML_COMMON_DECL_CUDA\n#define GGML_COMMON_IMPL_CUDA\n#if defined(GGML_USE_MUSA)\n#define GGML_COMMON_DECL_MUSA\n#define GGML_COMMON_IMPL_MUSA\n#endif\n#endif\n#include \"ggml-common.h\"\n\n#include <array>\n#include <algorithm>\n#include <cassert>\n#include <cfloat>\n#include <cstdio>\n#include <string>\n#include <unordered_map>\n#include <vector>\n\n#if defined(GGML_USE_HIP)\n#include \"vendors/hip.h\"\n#elif defined(GGML_USE_MUSA)\n#include \"vendors/musa.h\"\n#else\n#include \"vendors/cuda.h\"\n#endif // defined(GGML_USE_HIP)\n\n#define STRINGIZE_IMPL(...) #__VA_ARGS__\n#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)\n\n#define WARP_SIZE 32\n#define CUDART_HMAX   11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)\n#define CUDART_HMASK  12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons\n\n#define GGML_CUDA_CC_PASCAL          600\n#define GGML_CUDA_CC_DP4A            610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products\n#define GGML_CUDA_CC_VOLTA           700\n#define GGML_CUDA_CC_TURING          750\n#define GGML_CUDA_CC_AMPERE          800\n#define GGML_CUDA_CC_ADA_LOVELACE    890\n// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see\n// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms\n#define GGML_CUDA_CC_BLACKWELL       1200\n#define GGML_CUDA_CC_DGX_SPARK       1210\n#define GGML_CUDA_CC_RUBIN           1300\n#define GGML_CUDA_CC_OFFSET_AMD      0x1000000\n#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000\n#define GGML_CUDA_CC_IS_NVIDIA(cc)   (cc < GGML_CUDA_CC_OFFSET_MTHREADS)\n\n// AMD\n// GCN/CDNA, wave size is 64\n#define GGML_CUDA_CC_GCN4       (GGML_CUDA_CC_OFFSET_AMD + 0x803)  // Tonga, Fiji, Polaris, minimum for fast fp16\n#define GGML_CUDA_CC_VEGA       (GGML_CUDA_CC_OFFSET_AMD + 0x900)  // Vega56/64, minimum for fp16 dual issue\n#define GGML_CUDA_CC_VEGA20     (GGML_CUDA_CC_OFFSET_AMD + 0x906)  // MI50/Radeon VII, minimum for dp4a\n#define GGML_CUDA_CC_CDNA1      (GGML_CUDA_CC_OFFSET_AMD + 0x908)  // MI100, minimum for MFMA, acc registers\n#define GGML_CUDA_CC_CDNA2      (GGML_CUDA_CC_OFFSET_AMD + 0x910)  // MI210, minimum acc register renameing\n#define GGML_CUDA_CC_CDNA3      (GGML_CUDA_CC_OFFSET_AMD + 0x942)  // MI300\n\n// RDNA removes MFMA, dp4a, xnack, acc registers, wave size is 32\n#define GGML_CUDA_CC_RDNA1      (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000\n#define GGML_CUDA_CC_RDNA2      (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a\n#define GGML_CUDA_CC_RDNA3      (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA\n#define GGML_CUDA_CC_RDNA3_5    (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.\n#define GGML_CUDA_CC_RDNA4      (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000\n\n#define GGML_CUDA_CC_IS_AMD(cc)     (cc >= GGML_CUDA_CC_OFFSET_AMD)\n#define GGML_CUDA_CC_IS_RDNA(cc)    (cc >= GGML_CUDA_CC_RDNA1)\n#define GGML_CUDA_CC_IS_RDNA1(cc)   (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)\n#define GGML_CUDA_CC_IS_RDNA2(cc)   (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)\n#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)\n#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)\n#define GGML_CUDA_CC_IS_RDNA3(cc)   (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))\n#define GGML_CUDA_CC_IS_RDNA4(cc)   (cc >= GGML_CUDA_CC_RDNA4)\n#define GGML_CUDA_CC_IS_GCN(cc)     (cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1)\n#define GGML_CUDA_CC_IS_CDNA(cc)    (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)\n#define GGML_CUDA_CC_IS_CDNA1(cc)   (cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)\n#define GGML_CUDA_CC_IS_CDNA2(cc)   (cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)\n#define GGML_CUDA_CC_IS_CDNA3(cc)   (cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_RDNA1)\n\n// Moore Threads\n#define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons\n\n#define GGML_CUDA_CC_QY1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x210) // MTT S80, MTT S3000\n#define GGML_CUDA_CC_QY2 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x220) // MTT S4000\n#define GGML_CUDA_CC_PH1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x310) // MTT S5000\n\n#define GGML_CUDA_CC_IS_MTHREADS(cc) (cc >= GGML_CUDA_CC_OFFSET_MTHREADS && cc < GGML_CUDA_CC_OFFSET_AMD)\n#define GGML_CUDA_CC_IS_QY1(cc)      (cc >= GGML_CUDA_CC_QY1 && cc < GGML_CUDA_CC_QY2)\n#define GGML_CUDA_CC_IS_QY2(cc)      (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1)\n#define GGML_CUDA_CC_IS_PH1(cc)      (cc >= GGML_CUDA_CC_PH1)\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n#    define GGML_CUDA_USE_CUB\n#endif  // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070\n\n#ifdef __CUDA_ARCH_LIST__\nconstexpr bool ggml_cuda_has_arch_impl(int) {\n    return false;\n}\n\ntemplate<class ... Archs>\nconstexpr bool ggml_cuda_has_arch_impl(const int arch, const int first, Archs... rest) {\n    return arch == first || ggml_cuda_has_arch_impl(arch, rest...);\n}\n\nconstexpr bool ggml_cuda_has_arch(const int arch) {\n    return ggml_cuda_has_arch_impl(arch, __CUDA_ARCH_LIST__);\n}\n\nconstexpr int ggml_cuda_highest_compiled_arch_impl(const int /*arch*/, const int cur) {\n    if (cur == 0) {\n        return -1;\n    }\n    return cur;\n}\n\ntemplate<class ... Archs>\nconstexpr int ggml_cuda_highest_compiled_arch_impl(const int arch, const int cur, const int first, Archs... rest) {\n    if (first <= arch && first > cur) {\n        return ggml_cuda_highest_compiled_arch_impl(arch, first, rest...);\n    } else {\n        return ggml_cuda_highest_compiled_arch_impl(arch, cur, rest...);\n    }\n}\n\nconstexpr int ggml_cuda_highest_compiled_arch(const int arch) {\n    return ggml_cuda_highest_compiled_arch_impl(arch, 0, __CUDA_ARCH_LIST__);\n}\n#else\nstatic int ggml_cuda_highest_compiled_arch(const int arch) {\n    return arch;\n}\n#endif // __CUDA_ARCH_LIST__\n\n// ---------------------------------------------------------------------------------------------------------\n\n#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses\n\n#define GGML_CUDA_MAX_STREAMS 8\n\n[[noreturn]]\nvoid ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg);\n\n#define CUDA_CHECK_GEN(err, success, error_fn)                                      \\\n     do {                                                                           \\\n        auto err_ = (err);                                                          \\\n        if (err_ != (success)) {                                                    \\\n            ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_));    \\\n        }                                                                           \\\n    } while (0)\n\n#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)\n\n#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA)\n    static const char * cublas_get_error_str(const cublasStatus_t err) {\n        return cublasGetStatusString(err);\n    }\n#else\n    static const char * cublas_get_error_str(const cublasStatus_t err) {\n        switch (err) {\n            case CUBLAS_STATUS_SUCCESS: return \"CUBLAS_STATUS_SUCCESS\";\n            case CUBLAS_STATUS_NOT_INITIALIZED: return \"CUBLAS_STATUS_NOT_INITIALIZED\";\n            case CUBLAS_STATUS_ALLOC_FAILED: return \"CUBLAS_STATUS_ALLOC_FAILED\";\n            case CUBLAS_STATUS_INVALID_VALUE: return \"CUBLAS_STATUS_INVALID_VALUE\";\n            case CUBLAS_STATUS_ARCH_MISMATCH: return \"CUBLAS_STATUS_ARCH_MISMATCH\";\n            case CUBLAS_STATUS_MAPPING_ERROR: return \"CUBLAS_STATUS_MAPPING_ERROR\";\n            case CUBLAS_STATUS_EXECUTION_FAILED: return \"CUBLAS_STATUS_EXECUTION_FAILED\";\n            case CUBLAS_STATUS_INTERNAL_ERROR: return \"CUBLAS_STATUS_INTERNAL_ERROR\";\n            case CUBLAS_STATUS_NOT_SUPPORTED: return \"CUBLAS_STATUS_NOT_SUPPORTED\";\n            default: return \"unknown error\";\n        }\n    }\n#endif // CUDART_VERSION >= 12000\n\n#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)\nstatic const char * cu_get_error_str(CUresult err) {\n    const char * err_str;\n    cuGetErrorString(err, &err_str);\n    return err_str;\n}\n#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)\n#endif\n\n#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n#    define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes)                                                       \\\n        do {                                                                                                   \\\n            static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false };                         \\\n            const int   id                                                = ggml_cuda_get_device();            \\\n            if (!shared_memory_limit_raised[id]) {                                                             \\\n                CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes)); \\\n                shared_memory_limit_raised[id] = true;                                                         \\\n            }                                                                                                  \\\n        } while (0)\n#else\n#    define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \\\n        do {                                             \\\n            GGML_UNUSED(nbytes);                         \\\n        } while (0)\n#endif // !(defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)\n\n#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)\n#define GGML_CUDA_ASSUME(x) __builtin_assume(x)\n#else\n#define GGML_CUDA_ASSUME(x)\n#endif // CUDART_VERSION >= 11010\n\n#if (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))\n#define GGML_USE_VMM\n#endif // (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))\n\n#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n#define FP16_AVAILABLE\n#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n\n#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610\n#define FAST_FP16_AVAILABLE\n#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610\n\n#if defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)\n#define AMD_MFMA_AVAILABLE\n#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)\n\n#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))\n#define AMD_WMMA_AVAILABLE\n#endif // defined(GGML_USE_HIP) && defined(RDNA4)\n\n// The Volta instructions are in principle available on Turing or newer but they are effectively unusable:\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n#define VOLTA_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING\n#define TURING_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#define AMPERE_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN\n#    define BLACKWELL_MMA_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL\n\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n#define CP_ASYNC_AVAILABLE\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n\n#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)\n#define FLASH_ATTN_AVAILABLE\n#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)\n\n#if defined(TURING_MMA_AVAILABLE)\n#define LDMATRIX_TRANS_AVAILABLE\n#endif // defined(TURING_MMA_AVAILABLE)\n\nstatic bool fp16_available(const int cc) {\n    return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL ||\n        (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);\n}\n\nstatic bool fast_fp16_available(const int cc) {\n    return GGML_CUDA_CC_IS_AMD(cc) ||\n        (GGML_CUDA_CC_IS_NVIDIA(cc) && fp16_available(cc) && ggml_cuda_highest_compiled_arch(cc) != 610) ||\n        (GGML_CUDA_CC_IS_MTHREADS(cc) && fp16_available(cc));\n}\n\n// To be used for feature selection of external libraries, e.g. cuBLAS.\nstatic bool fast_fp16_hardware_available(const int cc) {\n    return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_PASCAL && cc != 610) || GGML_CUDA_CC_IS_AMD(cc) ||\n        (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);\n}\n\n// To be used for feature selection of external libraries, e.g. cuBLAS.\nstatic bool fp16_mma_hardware_available(const int cc) {\n    return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||\n        GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||\n        (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);\n}\n\nstatic bool bf16_mma_hardware_available(const int cc) {\n    return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) ||\n        GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 ||\n        (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);\n}\n\nstatic bool fp32_mma_hardware_available(const int cc) {\n    return GGML_CUDA_CC_IS_CDNA(cc);\n}\n\nstatic bool amd_mfma_available(const int cc) {\n#if !defined(GGML_HIP_NO_MMQ_MFMA)\n    return GGML_CUDA_CC_IS_CDNA(cc);\n#else\n    return false;\n#endif //!defined(GGML_HIP_NO_MMQ_MFMA)\n}\n\nstatic bool amd_wmma_available(const int cc) {\n    return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));\n}\n\nstatic bool volta_mma_available(const int cc) {\n    return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA;\n}\n\nstatic bool turing_mma_available(const int cc) {\n    return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING;\n}\n\nstatic bool ampere_mma_available(const int cc) {\n    return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;\n}\n\nstatic bool cp_async_available(const int cc) {\n    return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;\n}\n\nstatic bool blackwell_mma_available(const int cc) {\n    return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&\n           ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;\n}\n\nstatic constexpr __device__ int ggml_cuda_get_physical_warp_size() {\n#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))\n    return 64;\n#else\n    return 32;\n#endif // defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))\n}\n\n// Maximum number of bytes that can be copied in a single instruction.\nstatic constexpr __device__ int ggml_cuda_get_max_cpy_bytes() {\n#ifdef GGML_USE_HIP\n    return 16;\n#else\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n    return 16;\n#else\n    return 8;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA\n#endif // GGML_USE_HIP\n}\n\n\n[[noreturn]]\nstatic __device__ void no_device_code(\n    const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {\n\n#if defined(GGML_USE_HIP)\n    printf(\"%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\\n\",\n           file_name, line, function_name, arch);\n    GGML_UNUSED(arch_list);\n#else\n    printf(\"%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\\n\",\n           file_name, line, function_name, arch, arch_list);\n#endif // defined(GGML_USE_HIP)\n    __trap();\n\n    GGML_UNUSED(no_device_code); // suppress unused function warning\n\n#if defined(GGML_USE_MUSA)\n    __builtin_unreachable();\n#endif // defined(GGML_USE_MUSA)\n}\n\n#ifdef __CUDA_ARCH__\n#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))\n#else\n#define NO_DEVICE_CODE //GGML_ABORT(\"NO_DEVICE_CODE not valid in host code.\")\n#endif // __CUDA_ARCH__\n\n// The compiler is always able to unroll loops if they contain continue expressions.\n// In such cases loop unrolling can still be achieved via recursion:\ntemplate <int n>\nstruct ggml_cuda_unroll {\n    template <typename Func, typename... Args>\n    __device__ void operator()(const Func & f, Args... args) const {\n        f(n - 1, args...);\n        ggml_cuda_unroll<n - 1>{}(f, args...);\n    }\n};\n\ntemplate <>\nstruct ggml_cuda_unroll<1> {\n    template <typename Func, typename... Args>\n    __device__ void operator()(const Func & f, Args... args) const {\n        f(0, args...);\n    }\n};\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ int warp_reduce_sum(int x) {\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n    return __reduce_add_sync(0xffffffff, x);\n#else\n#pragma unroll\n    for (int offset = width/2; offset > 0; offset >>= 1) {\n        x += __shfl_xor_sync(0xffffffff, x, offset, width);\n    }\n    return x;\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ float warp_reduce_sum(float x) {\n#pragma unroll\n    for (int offset = width/2; offset > 0; offset >>= 1) {\n        x += __shfl_xor_sync(0xffffffff, x, offset, width);\n    }\n    return x;\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {\n#pragma unroll\n    for (int offset = width/2; offset > 0; offset >>= 1) {\n        a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width);\n        a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width);\n    }\n    return a;\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {\n#ifdef FP16_AVAILABLE\n#pragma unroll\n    for (int offset = width/2; offset > 0; offset >>= 1) {\n        a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width));\n    }\n    return a;\n\n#else\n    NO_DEVICE_CODE;\n    return a;\n#endif // FP16_AVAILABLE\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ int warp_reduce_all(int x) {\n    if (width == ggml_cuda_get_physical_warp_size()) {\n        return __all_sync(0xffffffff, x);\n    } else {\n#pragma unroll\n        for (int offset = width/2; offset > 0; offset >>= 1) {\n            x = __shfl_xor_sync(0xffffffff, x, offset, width) && x;\n        }\n        return x;\n    }\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ int warp_reduce_any(int x) {\n    if (width == ggml_cuda_get_physical_warp_size()) {\n        return __any_sync(0xffffffff, x);\n    } else {\n#pragma unroll\n        for (int offset = width/2; offset > 0; offset >>= 1) {\n            x = __shfl_xor_sync(0xffffffff, x, offset, width) || x;\n        }\n        return x;\n    }\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ float warp_reduce_max(float x) {\n#pragma unroll\n    for (int offset = width/2; offset > 0; offset >>= 1) {\n        x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width));\n    }\n    return x;\n}\n\ntemplate<typename T, int width = WARP_SIZE>\nstatic __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) {\n    const int lane_id = threadIdx.x % width;\n#pragma unroll\n    for (int offset = 1; offset < width; offset <<= 1) {\n        const T t = __shfl_up_sync(0xffffffff, x, offset, width);\n        if (lane_id >= offset) {\n            x += t;\n        }\n    }\n    return x;\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) {\n    const int lane_id = threadIdx.x % width;\n#pragma unroll\n    for (int offset = 1; offset < width; offset <<= 1) {\n        const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width);\n        const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width);\n        if (lane_id >= offset) {\n            a.x += t_x;\n            a.y += t_y;\n        }\n    }\n    return a;\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) {\n#ifdef FP16_AVAILABLE\n    const int lane_id = threadIdx.x % width;\n#pragma unroll\n    for (int offset = 1; offset < width; offset <<= 1) {\n        const half2 t = __shfl_up_sync(0xffffffff, a, offset, width);\n        if (lane_id >= offset) {\n            a = __hadd2(a, t);\n        }\n    }\n    return a;\n\n#else\n    NO_DEVICE_CODE;\n    return a;\n#endif // FP16_AVAILABLE\n}\n\nenum class block_reduce_method {\n    MAX,\n    SUM,\n};\n\ntemplate<block_reduce_method method_t, typename T>\nstruct block_reduce_policy;\n\ntemplate <typename T, typename... Ts>\ninline constexpr bool is_any = (std::is_same_v<T, Ts> || ...);\n\ntemplate<typename...>\ninline constexpr bool ggml_cuda_dependent_false_v = false;\n\ntemplate <typename T> struct block_reduce_policy<block_reduce_method::SUM, T> {\n    static __device__ T reduce(T val) {\n        if constexpr(is_any<T, float, float2, half2, int>) {\n            return warp_reduce_sum(val);\n        } else {\n            static_assert(ggml_cuda_dependent_false_v<T>, \"Unsupported type for block reduce sum\");\n        }\n    }\n\n    static __device__ T sentinel() {\n        if constexpr (std::is_same_v<T, float>) {\n            return 0.0f;\n        } else if constexpr (std::is_same_v<T, float2>) {\n            return make_float2(0.0f, 0.0f);\n        } else if constexpr (std::is_same_v<T, half2>) {\n            return make_half2(0.0f, 0.0f);\n        } else if constexpr (std::is_same_v<T, int>) {\n            return 0;\n        } else {\n            static_assert(ggml_cuda_dependent_false_v<T>, \"Unsupported type for block reduce sum\");\n        }\n    }\n};\n\ntemplate <typename T> struct block_reduce_policy<block_reduce_method::MAX, T> {\n    static __device__ T reduce(T val) {\n        if constexpr (is_any<T, float, half2>) {\n            return warp_reduce_max(val);\n        } else {\n            static_assert(ggml_cuda_dependent_false_v<T>, \"Unsupported type for block reduce max\");\n        }\n    }\n\n    static __device__ T sentinel() {\n        if constexpr (std::is_same_v<T, float>) {\n            return -INFINITY;\n        } else if constexpr (std::is_same_v<T, half2>) {\n            return make_half2(-INFINITY, -INFINITY);\n        } else {\n            static_assert(ggml_cuda_dependent_false_v<T>, \"Unsupported type for block reduce max\");\n        }\n    }\n};\n\ntemplate <block_reduce_method reduce_method_t, const unsigned int block_size_template = 0, typename T>\nstatic __device__ T block_reduce(T val, T * shared_vals) {\n    val                           = block_reduce_policy<reduce_method_t, T>::reduce(val);\n    const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;\n    if (block_size > WARP_SIZE) {\n        assert((block_size <= 1024) && (block_size % WARP_SIZE) == 0);\n        const int warp_id = threadIdx.x / WARP_SIZE;\n        const int lane_id = threadIdx.x % WARP_SIZE;\n        if (lane_id == 0) {\n            shared_vals[warp_id] = val;\n        }\n        __syncthreads();\n        val = block_reduce_policy<reduce_method_t, T>::sentinel();\n        if (lane_id < (static_cast<int>(block_size) / WARP_SIZE)) {\n            val = shared_vals[lane_id];\n        }\n        return block_reduce_policy<reduce_method_t, T>::reduce(val);\n    }\n\n    return val;\n}\n\nstatic __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {\n#ifdef FP16_AVAILABLE\n\n#if !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX\n    return __float2half(fmaxf(__half2float(a), __half2float(b)));\n#else\n    return __hmax(a, b);\n#endif // !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX\n\n#else\n   NO_DEVICE_CODE;\n   GGML_UNUSED(b);\n   return a;\n#endif // FP16_AVAILABLE\n}\n\nstatic __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {\n#if defined(GGML_USE_HIP)\n    return half2(__hmax(a.x, b.x), __hmax(a.y, b.y));\n#elif CUDART_VERSION >= CUDART_HMAX\n    return __hmax2(a, b);\n#else\n    half2 ret;\n    reinterpret_cast<half&>(ret.x) = __float2half(fmaxf( __low2float(a),  __low2float(b)));\n    reinterpret_cast<half&>(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));\n    return ret;\n#endif\n}\n\ntemplate<int width = WARP_SIZE>\nstatic __device__ __forceinline__ half2 warp_reduce_max(half2 x) {\n#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP)\n#pragma unroll\n   for (int offset = width/2; offset > 0; offset >>= 1) {\n       x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width));\n   }\n   return x;\n#else\n   GGML_UNUSED(x);\n   NO_DEVICE_CODE;\n#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL || defined(GGML_USE_HIP)\n}\n\n#if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \\\n    (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)\nstatic __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {\n    const uint32_t mask_low  = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));\n    const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));\n    return mask_low | mask_high;\n}\n#endif // (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)\n\nstatic __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) {\n#if defined(GGML_USE_HIP)\n#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)\n    c = __builtin_amdgcn_sdot4(a, b, c, false);\n#elif defined(RDNA3) || defined(RDNA4)\n    c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);\n#elif defined(RDNA1) || defined(__gfx900__)\n    int tmp1;\n    int tmp2;\n    asm(\"\\n \\\n        v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \\n \\\n        v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \\n \\\n        v_add3_u32 %0, %1, %2, %0 \\n \\\n        v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \\n \\\n        v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \\n \\\n        v_add3_u32 %0, %1, %2, %0 \\n \\\n        \"\n        : \"+v\"(c), \"=&v\"(tmp1), \"=&v\"(tmp2)\n        : \"v\"(a), \"v\"(b)\n    );\n#else\n    const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);\n    const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);\n    c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];\n#endif\n    return c;\n\n#else // defined(GGML_USE_HIP)\n\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n    return __dp4a(a, b, c);\n#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n    const int8_t * a8 = (const int8_t *) &a;\n    const int8_t * b8 = (const int8_t *) &b;\n    return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)\n\n#endif // defined(GGML_USE_HIP)\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float v, const float u) {\n    acc += v*u;\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v, const float2 u) {\n    acc += v.x*u.x;\n    acc += v.y*u.y;\n}\n\n#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))\n#define V_DOT2_F32_F16_AVAILABLE\n#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {\n#ifdef V_DOT2_F32_F16_AVAILABLE\n    asm volatile(\"v_dot2_f32_f16 %0, %1, %2, %0\" : \"+v\"(acc) : \"v\"(v), \"v\"(u));\n#else\n#ifdef FAST_FP16_AVAILABLE\n    const float2 tmp = __half22float2(v*u);\n    acc += tmp.x + tmp.y;\n#else\n    const float2 tmpv = __half22float2(v);\n    const float2 tmpu = __half22float2(u);\n    acc += tmpv.x * tmpu.x;\n    acc += tmpv.y * tmpu.y;\n#endif // FAST_FP16_AVAILABLE\n#endif // V_DOT2_F32_F16_AVAILABLE\n}\n\nstatic __device__ __forceinline__ void ggml_cuda_mad(half2 & acc, const half2 v, const half2 u) {\n#ifdef FAST_FP16_AVAILABLE\n    acc += v*u;\n#else\n    const float2 tmpv = __half22float2(v);\n    const float2 tmpu = __half22float2(u);\n    float2 tmpacc = __half22float2(acc);\n    tmpacc.x += tmpv.x * tmpu.x;\n    tmpacc.y += tmpv.y * tmpu.y;\n    acc = make_half2(tmpacc.x, tmpacc.y);\n#endif // FAST_FP16_AVAILABLE\n}\n\n// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes, especially on AMD.\n// Important: do not use this function if dst and src both point at registers.\n//     Due to the strict aliasing rule the compiler can do incorrect optimizations if src and dst have different types.\n//     The function is intended for copies between registers and SRAM/VRAM to make the compiler emit the right instructions.\n//     If dst and src point at different address spaces then they are guaranteed to not be aliased.\ntemplate <int nbytes, int alignment = 0>\nstatic __device__ __forceinline__ void ggml_cuda_memcpy_1(void * __restrict__ dst, const void * __restrict__ src) {\n    static_assert(\n        nbytes <= ggml_cuda_get_max_cpy_bytes() || alignment == 0,\n        \"You are misusing the alignment parameter for ggml_cuda_memcpy_1. \"\n        \"The intent is for the parameter is only as a workaround if either one of the pointers is not properly aligned. \"\n        \"If you use it to do more bytes per copy than ggml_cuda_max_cpy_bytes() the reads and writes may not be coalesced. \"\n        \"Call ggml_cuda_memcpy_1 in a loop instead.\");\n    if constexpr (alignment != 0) {\n        static_assert(nbytes % alignment == 0, \"bad alignment\");\n    }\n    constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment;\n\n#pragma unroll\n    for (int i = 0; i < nbytes/nb_per_cpy; ++i) {\n        if constexpr (nb_per_cpy == 1) {\n            ((char *) dst)[i] = ((const char *) src)[i];\n        } else if constexpr (nb_per_cpy == 2) {\n            ((short *) dst)[i] = ((const short *) src)[i];\n        } else if constexpr (nb_per_cpy == 4) {\n            ((int *) dst)[i] = ((const int *) src)[i];\n        } else if constexpr (nb_per_cpy == 8) {\n            ((int2 *) dst)[i] = ((const int2 *) src)[i];\n        } else if constexpr (nb_per_cpy == 16) {\n            ((int4 *) dst)[i] = ((const int4 *) src)[i];\n        } else {\n            static_assert(nbytes == 0 && nbytes == -1, \"bad nbytes\");\n        }\n    }\n}\n\nstatic __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) {\n#if CUDART_VERSION >= 12080\n    const nv_bfloat16 e = __nv_cvt_e8m0_to_bf16raw(x);\n    return (float) e;\n#else\n    uint32_t bits;\n    if (x == 0) {\n        bits = 0x00400000;\n    } else {\n        bits = (uint32_t) x << 23;\n    }\n\n    float result;\n    memcpy(&result, &bits, sizeof(float));\n    return result;\n#endif // CUDART_VERSION >= 12050\n}\n\n__device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) {\n    const uint8_t sign_bit = (x < 0.0f) << 3;\n    float         ax       = fabsf(x) * e;\n\n    // Positive LUT\n    static constexpr float pos_lut[8] = { 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f };\n\n    int   best_i   = 0;\n    float best_err = fabsf(ax - pos_lut[0]);\n\n#pragma unroll\n    for (int i = 1; i < 8; ++i) {\n        const float err = fabsf(ax - pos_lut[i]);\n        if (err < best_err) {\n            best_err = err;\n            best_i   = i;\n        }\n    }\n\n    return static_cast<uint8_t>(best_i | sign_bit);\n}\n\n// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.\n// Precompute mp (m' in the paper) and L such that division\n// can be computed using a multiply (high 32b of 64b result)\n// and a shift:\n//\n// n/d = (mulhi(n, mp) + n) >> L;\nstatic const uint3 init_fastdiv_values(uint64_t d_64) {\n    GGML_ASSERT(d_64 != 0);\n    GGML_ASSERT(d_64 <= std::numeric_limits<uint32_t>::max());\n\n    uint32_t d = (uint32_t)d_64;\n\n    // compute L = ceil(log2(d));\n    uint32_t L = 0;\n    while (L < 32 && (uint32_t{ 1 } << L) < d) {\n        L++;\n    }\n\n    uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);\n    // pack divisor as well to reduce error surface\n    return make_uint3(mp, L, d);\n}\n\nstatic __device__ __forceinline__ uint32_t fastdiv(uint32_t n, const uint3 fastdiv_values) {\n    // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z>\n    // fastdiv_values.z is unused and optimized away by the compiler.\n    // Compute high 32 bits of n * mp\n    const uint32_t hi = __umulhi(n, fastdiv_values.x);\n    // add n, apply bit shift\n    return (hi + n) >> fastdiv_values.y;\n}\n\nstatic __device__ __forceinline__ uint32_t fastmodulo(uint32_t n, const uint3 fastdiv_values) {\n    // expects  fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)\n    return n - fastdiv(n, fastdiv_values) * fastdiv_values.z;\n}\n\n// Calculate both division and modulo at once, returns <n/divisor, n%divisor>\nstatic __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 fastdiv_values) {\n    // expects  fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)\n    const uint32_t div_val = fastdiv(n, fastdiv_values);\n    const uint32_t mod_val = n - div_val * fastdiv_values.z;\n    return make_uint2(div_val, mod_val);\n}\n\ntypedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v);\n\nstatic __device__ __forceinline__ float get_alibi_slope(\n    const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1\n) {\n    if (max_bias <= 0.0f) {\n        return 1.0f;\n    }\n    const float base = h < n_head_log2 ? m0 : m1;\n    const int   exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;\n\n    return powf(base, exph);\n}\n\ntemplate <ggml_type type>\nstruct ggml_cuda_type_traits;\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_F16> {\n    static constexpr int qk = 1;\n    static constexpr int qr = 1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {\n    static constexpr int qk = QK4_0;\n    static constexpr int qr = QR4_0;\n    static constexpr int qi = QI4_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q4_1> {\n    static constexpr int qk = QK4_1;\n    static constexpr int qr = QR4_1;\n    static constexpr int qi = QI4_1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q5_0> {\n    static constexpr int qk = QK5_0;\n    static constexpr int qr = QR5_0;\n    static constexpr int qi = QI5_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q5_1> {\n    static constexpr int qk = QK5_1;\n    static constexpr int qr = QR5_1;\n    static constexpr int qi = QI5_1;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q8_0> {\n    static constexpr int qk = QK8_0;\n    static constexpr int qr = QR8_0;\n    static constexpr int qi = QI8_0;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_MXFP4> {\n    static constexpr int qk = QK_MXFP4;\n    static constexpr int qr = QR_MXFP4;\n    static constexpr int qi = QI_MXFP4;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q2_K> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR2_K;\n    static constexpr int qi = QI2_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q3_K> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR3_K;\n    static constexpr int qi = QI3_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q4_K> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR4_K;\n    static constexpr int qi = QI4_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q5_K> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR5_K;\n    static constexpr int qi = QI5_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_Q6_K> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR6_K;\n    static constexpr int qi = QI6_K;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR2_XXS;\n    static constexpr int qi = QI2_XXS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR2_XS;\n    static constexpr int qi = QI2_XS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR2_S;\n    static constexpr int qi = QI2_S;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR3_XXS;\n    static constexpr int qi = QI3_XXS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR1_S;\n    static constexpr int qi = QI1_S;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR1_M;\n    static constexpr int qi = QI1_M;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> {\n    static constexpr int qk = QK4_NL;\n    static constexpr int qr = QR4_NL;\n    static constexpr int qi = QI4_NL;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR4_XS;\n    static constexpr int qi = QI4_XS;\n};\n\ntemplate<>\nstruct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {\n    static constexpr int qk = QK_K;\n    static constexpr int qr = QR3_S;\n    static constexpr int qi = QI3_S;\n};\n\n//////////////////////\n\nstruct ggml_cuda_device_info {\n    int device_count;\n\n    struct cuda_device_info {\n        int     cc;                             // compute capability\n        int     nsm;                            // number of streaming multiprocessors\n        size_t  smpb;                           // max. shared memory per block\n        size_t  smpbo;                          // max. shared memory per block (with opt-in)\n        bool    integrated;                     // Device is integrated as opposed to discrete\n        bool    vmm;                            // virtual memory support\n        size_t  vmm_granularity;                // granularity of virtual memory\n        size_t  total_vram;\n        int     warp_size;                      // Number of threads in a dispatch\n        bool    supports_cooperative_launch;    // whether cooperative launch is supported\n    };\n\n    cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};\n\n    std::array<float, GGML_CUDA_MAX_DEVICES> default_tensor_split = {};\n};\n\nconst ggml_cuda_device_info & ggml_cuda_info();\n\nvoid ggml_cuda_set_device(int device);\nint ggml_cuda_get_device();\n\nstruct ggml_cuda_pool {\n    virtual ~ggml_cuda_pool() = default;\n\n    virtual void * alloc(size_t size, size_t * actual_size) = 0;\n    virtual void free(void * ptr, size_t size) = 0;\n};\n\ntemplate<typename T>\nstruct ggml_cuda_pool_alloc {\n    ggml_cuda_pool * pool = nullptr;\n    T * ptr = nullptr;\n    size_t actual_size = 0;\n\n    ggml_cuda_pool_alloc() = default;\n\n    explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {\n    }\n\n    ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {\n        alloc(size);\n    }\n\n    ~ggml_cuda_pool_alloc() {\n        if (ptr != nullptr) {\n            pool->free(ptr, actual_size);\n        }\n    }\n\n    // size is in number of elements\n    T * alloc(size_t size) {\n        GGML_ASSERT(pool != nullptr);\n        GGML_ASSERT(ptr == nullptr);\n        ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);\n        return ptr;\n    }\n\n    T * alloc(ggml_cuda_pool & pool, size_t size) {\n        this->pool = &pool;\n        return alloc(size);\n    }\n\n    T * get() {\n        return ptr;\n    }\n\n    ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;\n    ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;\n    ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;\n    ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;\n};\n\n\n// backend interface\n\nstruct ggml_tensor_extra_gpu {\n    void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors\n    cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs\n};\n\n\n#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) || defined(GGML_MUSA_GRAPHS)\n#define USE_CUDA_GRAPH\n#endif\n\nstruct ggml_cuda_graph_node_properties {\n    void * node_data;\n    ggml_op node_op;\n    enum ggml_type node_type;\n    int32_t flags;\n    int64_t ne[GGML_MAX_DIMS];\n    size_t nb[GGML_MAX_DIMS];\n    void * src_data[GGML_MAX_SRC];\n    int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];\n};\n\nstatic_assert(std::is_trivial<ggml_cuda_graph_node_properties>::value, \"ggml_cuda_graph_node_properties must be trivial\");\n\nstruct ggml_cuda_graph {\n#ifdef USE_CUDA_GRAPH\n    ~ggml_cuda_graph() {\n        if (instance != nullptr) {\n            CUDA_CHECK(cudaGraphExecDestroy(instance));\n        }\n        if (graph != nullptr) {\n            CUDA_CHECK(cudaGraphDestroy(graph));\n        }\n    }\n    cudaGraph_t graph = nullptr;\n    cudaGraphExec_t instance = nullptr;\n    size_t num_nodes = 0;\n    std::vector<cudaGraphNode_t> nodes;\n    bool disable_due_to_gpu_arch = false;\n    bool warmup_complete = false;\n    std::vector<ggml_cuda_graph_node_properties> props;\n\n    // these are extra tensors (inputs) that participate in the ggml graph but are not nodes\n    // they properties also have to match in order to be able to safely reuse a CUDA graph\n    // ref: https://github.com/ggml-org/llama.cpp/pull/18583\n    // ref: https://github.com/ggml-org/llama.cpp/pull/19165\n    std::vector<ggml_cuda_graph_node_properties> extra;\n\n    bool is_enabled() const {\n        static const bool disable_cuda_graphs_due_to_env = (getenv(\"GGML_CUDA_DISABLE_GRAPHS\") != nullptr);\n        return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env);\n    }\n#endif\n};\n\nstruct ggml_cuda_concurrent_event {\n    std::vector<cudaEvent_t> join_events;\n    cudaEvent_t              fork_event = nullptr;\n\n    int                                          n_streams = 0;\n    std::unordered_map<const ggml_tensor *, int> stream_mapping;\n\n    // Original order of nodes in this concurrent region (before interleaving)\n    // Used to restore grouping for fusion within streams\n    std::vector<const ggml_tensor *> original_order;\n\n    const ggml_tensor * join_node;\n\n    ggml_cuda_concurrent_event() = default;\n\n    ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete;\n    ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete;\n\n    explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) {\n        join_events.resize(n_streams);\n\n        for (size_t i = 0; i < join_events.size(); ++i) {\n            CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming));\n        }\n\n        CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming));\n    }\n\n    ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept\n    : join_events(std::move(other.join_events))\n    , fork_event(other.fork_event)\n    , n_streams(other.n_streams)\n    , stream_mapping(std::move(other.stream_mapping))\n    , original_order(std::move(other.original_order))\n    , join_node(other.join_node) {\n        other.fork_event = nullptr;\n    }\n\n    // 1. check if any branches write to overlapping memory ranges (except the join node)\n    // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event\n    // we assume all nodes have the same buffer\n    bool is_valid() const {\n        std::vector<std::vector<std::pair<int64_t, int64_t>>> write_ranges;\n        write_ranges.resize(n_streams);\n\n        // get join_node's memory range to exclude from overlap checking.\n        // multiple nodes can use join_node's buffer; we synchronize on the join node.\n        const ggml_tensor * join_t     = join_node->view_src ? join_node->view_src : join_node;\n        const int64_t       join_start = (int64_t) join_t->data;\n        const int64_t       join_end   = join_start + ggml_nbytes(join_t);\n\n        for (const auto & [tensor, stream] : stream_mapping) {\n            const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;\n            const int64_t       t_start = (int64_t) t->data;\n            const int64_t       t_end   = t_start + ggml_nbytes(t);\n\n            // skip tensors that overlap with join_node's buffer.\n            if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {\n                continue;\n            }\n\n            // concurrent streams begin from 1\n            write_ranges[stream - 1].emplace_back(t_start, t_end);\n        }\n\n        for (int i = 0; i < n_streams; ++i) {\n            // sorts first by start then by end of write range\n            std::sort(write_ranges[i].begin(), write_ranges[i].end());\n        }\n\n        bool writes_overlap = false;\n        bool dependent_srcs = false;\n        for (const auto & [tensor, stream] : stream_mapping) {\n            const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;\n            const int64_t       t_start = (int64_t) t->data;\n            const int64_t       t_end   = t_start + ggml_nbytes(t);\n\n            // skip tensors that overlap with join_node's buffer\n            if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {\n                continue;\n            }\n\n            // check if this buffer's write data overlaps with another stream's\n            std::pair<int64_t, int64_t> data_range = std::make_pair(t_start, t_end);\n            for (int i = 0; i < n_streams; ++i) {\n                if (i == stream - 1) {\n                    continue;\n                }\n                auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range);\n\n                if (it != write_ranges[i].end()) {\n                    const std::pair<int64_t, int64_t> & other = *it;\n\n                    // std::lower_bound returns the first element where other >= data_range (lexicographically).\n                    // This guarantees other.first >= data_range.first.\n                    // Therefore, overlap occurs iff other.first < data_range.second\n                    // (i.e., the other range starts before this range ends).\n                    if (other.first < data_range.second) {\n                        GGML_LOG_DEBUG(\"Writes overlap for %s\", tensor->name);\n                        writes_overlap = true;\n                        break;\n                    }\n                }\n            }\n\n            //check if all srcs are either in branch or don't have a branch\n            for (int i = 0; i < GGML_MAX_SRC; ++i) {\n                if (!tensor->src[i]) {\n                    continue;\n                }\n\n                auto it = stream_mapping.find(tensor->src[i]);\n\n                if (it == stream_mapping.end()) {\n                    continue;\n                }\n\n                if (it->second != stream) {\n                    dependent_srcs = true;\n                    break;\n                }\n            }\n\n            if (dependent_srcs || writes_overlap) {\n                break;\n            }\n        }\n\n        return !writes_overlap && !dependent_srcs;\n    }\n\n    ~ggml_cuda_concurrent_event() {\n        if (fork_event != nullptr) {\n            CUDA_CHECK(cudaEventDestroy(fork_event));\n        }\n        for (cudaEvent_t e : join_events) {\n            if (e != nullptr) {\n                CUDA_CHECK(cudaEventDestroy(e));\n            }\n        }\n    }\n};\n\nstruct ggml_cuda_stream_context {\n    std::unordered_map<const ggml_tensor *, ggml_cuda_concurrent_event> concurrent_events;\n\n    void reset() {\n        concurrent_events.clear();\n    }\n};\n\nstruct ggml_backend_cuda_context {\n    int device;\n    std::string name;\n    cudaEvent_t copy_event = nullptr;\n\n    cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } };\n    cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};\n\n    int curr_stream_no = 0;\n\n#ifdef USE_CUDA_GRAPH\n    // Map from first_node_ptr to cuda_graph - allows multiple graphs per context\n    // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)\n    std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;\n\n    ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {\n        auto it = cuda_graphs.find(first_node_ptr);\n        if (it == cuda_graphs.end()) {\n            cuda_graphs[first_node_ptr] = std::make_unique<ggml_cuda_graph>();\n            return cuda_graphs[first_node_ptr].get();\n        }\n        return it->second.get();\n    }\n\n    // Check if any CUDA graph is enabled for this context (used by kernels that need to know\n    // if graphs are in use without having access to the specific graph key)\n    bool any_cuda_graph_enabled() const {\n        for (const auto & [key, graph] : cuda_graphs) {\n            if (graph && graph->is_enabled()) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    // Check if any CUDA graph has an instance for this context\n    bool any_cuda_graph_has_instance() const {\n        for (const auto & [key, graph] : cuda_graphs) {\n            if (graph && graph->instance != nullptr) {\n                return true;\n            }\n        }\n        return false;\n    }\n#endif // USE_CUDA_GRAPH\n\n    explicit ggml_backend_cuda_context(int device) :\n        device(device),\n        name(GGML_CUDA_NAME + std::to_string(device)) {\n    }\n\n    ggml_cuda_stream_context concurrent_stream_context;\n\n    ~ggml_backend_cuda_context();\n\n    cudaStream_t stream(int device, int stream) {\n        if (streams[device][stream] == nullptr) {\n            ggml_cuda_set_device(device);\n            CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));\n        }\n        return streams[device][stream];\n    }\n\n    cudaStream_t stream() { return stream(device, curr_stream_no); }\n\n    ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; }\n\n    cublasHandle_t cublas_handle(int device) {\n        if (cublas_handles[device] == nullptr) {\n            ggml_cuda_set_device(device);\n            CUBLAS_CHECK(cublasCreate(&cublas_handles[device]));\n            CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device], CUBLAS_TF32_TENSOR_OP_MATH));\n        }\n        return cublas_handles[device];\n    }\n\n    cublasHandle_t cublas_handle() {\n        return cublas_handle(device);\n    }\n\n    // pool\n    std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];\n\n    static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no);\n\n    ggml_cuda_pool & pool(int device) {\n        if (pools[device][curr_stream_no] == nullptr) {\n            pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);\n        }\n        return *pools[device][curr_stream_no];\n    }\n\n    ggml_cuda_pool & pool() {\n        return pool(device);\n    }\n};\n\nstruct ggml_cuda_mm_fusion_args_host {\n    const ggml_tensor * x_bias = nullptr;\n    const ggml_tensor * gate = nullptr;\n    const ggml_tensor * gate_bias = nullptr;\n    ggml_glu_op glu_op;\n};\nstruct ggml_cuda_mm_fusion_args_device {\n    const void * x_bias = nullptr;\n    const void * gate = nullptr;\n    const void * gate_bias = nullptr;\n    ggml_glu_op glu_op;\n};\n"
  },
  {
    "path": "ggml/src/ggml-cuda/concat.cu",
    "content": "#include \"concat.cuh\"\n\n// contiguous kernels\nstatic __global__ void concat_f32_dim0(const float * x, const float * y, float * dst, const int ne0, const int ne00) {\n    int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n    if (nidx >= ne0) {\n        return;\n    }\n\n    int offset_dst =\n        nidx +\n        blockIdx.y * ne0 +\n        blockIdx.z * ne0 * gridDim.y;\n\n    if (nidx < ne00) { // src0\n        int offset_src =\n            nidx +\n            blockIdx.y * ne00 +\n            blockIdx.z * ne00 * gridDim.y;\n        dst[offset_dst] = x[offset_src];\n    } else {\n        int offset_src =\n            (nidx - ne00) +\n            blockIdx.y * (ne0 - ne00) +\n            blockIdx.z * (ne0 - ne00) * gridDim.y;\n        dst[offset_dst] = y[offset_src];\n    }\n}\n\nstatic __global__ void concat_f32_dim1(const float * x, const float * y, float * dst, const int ne0, const int ne01) {\n    int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n    if (nidx >= ne0) {\n        return;\n    }\n\n    int offset_dst =\n        nidx +\n        blockIdx.y * ne0 +\n        blockIdx.z * ne0 * gridDim.y;\n\n    if (blockIdx.y < (unsigned)ne01) { // src0\n        int offset_src =\n            nidx +\n            blockIdx.y * ne0 +\n            blockIdx.z * ne0 * ne01;\n        dst[offset_dst] = x[offset_src];\n    } else {\n        int offset_src =\n            nidx +\n            (blockIdx.y - ne01) * ne0 +\n            blockIdx.z * ne0 * (gridDim.y - ne01);\n        dst[offset_dst] = y[offset_src];\n    }\n}\n\nstatic __global__ void concat_f32_dim2(const float * x, const float * y, float * dst, const int ne0, const int ne02) {\n    int nidx = threadIdx.x + blockIdx.x * blockDim.x;\n    if (nidx >= ne0) {\n        return;\n    }\n\n    int offset_dst =\n        nidx +\n        blockIdx.y * ne0 +\n        blockIdx.z * ne0 * gridDim.y;\n\n    if (blockIdx.z < (unsigned)ne02) { // src0\n        int offset_src =\n            nidx +\n            blockIdx.y * ne0 +\n            blockIdx.z * ne0 * gridDim.y;\n        dst[offset_dst] = x[offset_src];\n    } else {\n        int offset_src =\n            nidx +\n            blockIdx.y * ne0 +\n            (blockIdx.z - ne02) * ne0 *  gridDim.y;\n        dst[offset_dst] = y[offset_src];\n    }\n}\n\nstatic void concat_f32_cuda(const float * x, const float * y, float * dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, cudaStream_t stream) {\n    int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;\n    dim3 gridDim(num_blocks, ne1, ne2);\n    if (dim == 0) {\n        concat_f32_dim0<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne00);\n        return;\n    }\n    if (dim == 1) {\n        concat_f32_dim1<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne01);\n        return;\n    }\n    concat_f32_dim2<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);\n}\n\n// non-contiguous kernel (slow)\ntemplate <int dim>\nstatic __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)\n    concat_f32_non_cont(\n        const char * src0,\n        const char * src1,\n              char * dst,\n           int64_t   ne00,\n           int64_t   ne01,\n           int64_t   ne02,\n           int64_t   ne03,\n          uint64_t   nb00,\n          uint64_t   nb01,\n          uint64_t   nb02,\n          uint64_t   nb03,\n           int64_t /*ne10*/,\n           int64_t /*ne11*/,\n           int64_t /*ne12*/,\n           int64_t /*ne13*/,\n          uint64_t   nb10,\n          uint64_t   nb11,\n          uint64_t   nb12,\n          uint64_t   nb13,\n           int64_t   ne0,\n           int64_t /*ne1*/,\n           int64_t /*ne2*/,\n           int64_t /*ne3*/,\n          uint64_t   nb0,\n          uint64_t   nb1,\n          uint64_t   nb2,\n          uint64_t   nb3){\n    static_assert(dim >= 0 && dim <= 3, \"dim must be in [0, 3]\");\n\n    const int64_t i3 = blockIdx.z;\n    const int64_t i2 = blockIdx.y;\n    const int64_t i1 = blockIdx.x;\n\n    const float * x;\n\n    for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {\n        if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {\n            x = (const float *)(src0 + (i3       )*nb03 + (i2       )*nb02 + (i1       )*nb01 + (i0       )*nb00);\n        } else {\n            if constexpr (dim == 0) {\n                x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + i1 * nb11 + (i0 - ne00) * nb10);\n            } else if constexpr (dim == 1) {\n                x = (const float *) (src1 + i3 * nb13 + i2 * nb12 + (i1 - ne01) * nb11 + i0 * nb10);\n            } else if constexpr (dim == 2) {\n                x = (const float *) (src1 + i3 * nb13 + (i2 - ne02) * nb12 + i1 * nb11 + i0 * nb10);\n            } else if constexpr (dim == 3) {\n                x = (const float *) (src1 + (i3 - ne03) * nb13 + i2 * nb12 + i1 * nb11 + i0 * nb10);\n            }\n        }\n\n        float * y = (float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);\n\n        *y = *x;\n    }\n}\n\n\nvoid ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    cudaStream_t stream = ctx.stream();\n\n    const int32_t dim = ((int32_t *) dst->op_params)[0];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT(dst->type  == GGML_TYPE_F32);\n\n    if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1)) {\n        const float * src0_d = (const float *)src0->data;\n        const float * src1_d = (const float *)src1->data;\n\n        float * dst_d = (float *)dst->data;\n\n        if (dim != 3) {\n            for (int i3 = 0; i3 < dst->ne[3]; i3++) {\n                concat_f32_cuda(\n                        src0_d + i3 * (src0->nb[3] / 4),\n                        src1_d + i3 * (src1->nb[3] / 4),\n                        dst_d + i3 * ( dst->nb[3] / 4),\n                        src0->ne[0], src0->ne[1], src0->ne[2],\n                        dst->ne[0],  dst->ne[1],  dst->ne[2], dim, stream);\n            }\n        } else {\n            const size_t size0 = ggml_nbytes(src0);\n            const size_t size1 = ggml_nbytes(src1);\n\n            CUDA_CHECK(cudaMemcpyAsync(dst_d,           src0_d, size0, cudaMemcpyDeviceToDevice, stream));\n            CUDA_CHECK(cudaMemcpyAsync(dst_d + size0/4, src1_d, size1, cudaMemcpyDeviceToDevice, stream));\n        }\n    } else {\n        dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]);\n        auto launch_kernel = [&](auto dim) {\n            concat_f32_non_cont<dim><<<grid_dim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(\n                (const char *) src0->data, (const char *) src1->data, (char *) dst->data,\n                src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n                src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3],\n                src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n                src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3],\n                dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n                dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]);\n        };\n        switch (dim) {\n            case 0:\n                launch_kernel(std::integral_constant<int, 0>{});\n                break;\n            case 1:\n                launch_kernel(std::integral_constant<int, 1>{});\n                break;\n            case 2:\n                launch_kernel(std::integral_constant<int, 2>{});\n                break;\n            case 3:\n                launch_kernel(std::integral_constant<int, 3>{});\n                break;\n            default:\n                GGML_ABORT(\"Invalid dim: %d\", dim);\n                break;\n        }\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/concat.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CONCAT_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv-transpose-1d.cu",
    "content": "#include \"conv-transpose-1d.cuh\"\n\nstatic  __global__ void conv_transpose_1d_kernel(\n        const int s0, const int p0, const int d0, const int output_size,\n        const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,\n        const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,\n        const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,\n        const float * src0, const float * src1,  float * dst) {\n    int global_index = threadIdx.x + blockIdx.x * blockDim.x;\n    if (global_index >= output_size) {\n        return;\n    }\n\n    int out_index = global_index / dst_ne0;\n\n    float accumulator = 0;\n\n    for (int c = 0; c < src0_ne2; c++) {\n        int idx = global_index % dst_ne0;\n\n        int kernel_offset = (src0_ne0 * src0_ne1 * c) + (out_index * src0_ne0);\n        int input_offset = src1_ne0 * c;\n\n        for (int i = 0; i < src1_ne0; i++) {\n            if (!(idx >= i*s0 && idx < i*s0 + src0_ne0)) {\n                continue;\n            }\n            int weight_idx = idx - i*s0;\n\n            float kernel_weight = src0[kernel_offset + weight_idx];\n            float input_value =  src1[input_offset+i];\n\n            accumulator += kernel_weight * input_value;\n        }\n    }\n    dst[global_index] = accumulator;\n    GGML_UNUSED_VARS(p0, d0, src0_ne3, src1_ne3, dst_ne3, src1_ne1, dst_ne1, src1_ne2, dst_ne2);\n}\n\nstatic void conv_transpose_1d_f32_f32_cuda(\n        const int s0, const int p0, const int d0, const int output_size,\n        const int src0_ne0, const int src0_ne1, const int src0_ne2, const int src0_ne3,\n        const int src1_ne0, const int src1_ne1, const int src1_ne2, const int src1_ne3,\n        const int dst_ne0, const int dst_ne1, const int dst_ne2, const int dst_ne3,\n        const float * src0, const float * src1,  float * dst,\n        cudaStream_t stream) {\n\n    const int num_blocks = (output_size + CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE - 1) / CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE;\n    conv_transpose_1d_kernel<<<num_blocks,CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE, 0, stream>>>(\n        s0,p0,d0,output_size,\n        src0_ne0, src0_ne1,  src0_ne2, src0_ne3,\n        src1_ne0, src1_ne1,  src1_ne2, src1_ne3,\n        dst_ne0,  dst_ne1,   dst_ne2,  dst_ne3,\n        src0,src1, dst);\n}\n\nvoid ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const float * src0_d = (const float *)src0->data;\n\n    const ggml_tensor * src1 = dst->src[1];\n    const float * src1_d = (const float *)src1->data;\n\n    float * dst_d = (float *)dst->data;\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n    GGML_ASSERT(ggml_is_contiguous(src1));\n\n    const int32_t * opts = (const int32_t *)dst->op_params;\n\n    const int s0 = opts[0];\n    const int p0 = 0;//opts[3];\n    const int d0 = 1;//opts[4];\n\n    const int64_t output_size = ggml_nelements(dst);\n\n    conv_transpose_1d_f32_f32_cuda(s0, p0, d0, output_size,\n        src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],\n        src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],\n        dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],\n        src0_d, src1_d, dst_d, stream);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv-transpose-1d.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CONV_TRANPOSE_1D_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_conv_transpose_1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d-dw.cu",
    "content": "#include \"conv2d-dw.cuh\"\n\nstruct conv_params {\n    int in_w, in_h;\n    int out_w, out_h;\n    int kernel_w, kernel_h;\n    int stride_x, stride_y;\n    int padding_x, padding_y;\n    int dilation_x, dilation_y;\n    int channels, batches;\n};\n\nstruct kernel_bounds {\n    int y_min, y_max;\n    int x_min, x_max;\n};\n\n__device__ __forceinline__ kernel_bounds calculate_kernel_bounds(int out_x, int out_y, const conv_params & params) {\n    kernel_bounds bounds;\n    bounds.y_min = max(0, (params.padding_y - out_y * params.stride_y + params.dilation_y - 1) / params.dilation_y);\n    bounds.y_max =\n        min(params.kernel_h,\n            (params.in_h + params.padding_y - out_y * params.stride_y + params.dilation_y - 1) / params.dilation_y);\n    bounds.x_min = max(0, (params.padding_x - out_x * params.stride_x + params.dilation_x - 1) / params.dilation_x);\n    bounds.x_max =\n        min(params.kernel_w,\n            (params.in_w + params.padding_x - out_x * params.stride_x + params.dilation_x - 1) / params.dilation_x);\n    return bounds;\n}\n\n__device__ __forceinline__ int calculate_input_coord(int out_coord, int kern_coord, int stride, int dilation, int padding) {\n    return out_coord * stride + kern_coord * dilation - padding;\n}\n\nstruct whcn_layout {\n    __device__ static int input_index(int n, int c, int y, int x, const conv_params & params) {\n        return n * (params.channels * params.in_w * params.in_h) + c * params.in_w * params.in_h + y * params.in_w + x;\n    }\n\n    __device__ static int kernel_index(int c, int ky, int kx, const conv_params & params) {\n        return c * params.kernel_h * params.kernel_w + ky * params.kernel_w + kx;\n    }\n\n    __device__ static int output_index(int n, int c, int y, int x, const conv_params & params) {\n        return n * (params.channels * params.out_w * params.out_h) + c * params.out_w * params.out_h +\n               y * params.out_w + x;\n    }\n\n    __device__ static void unpack_indices(int global_idx, const conv_params & params, int & n, int & c, int & out_y,\n                                          int & out_x) {\n        out_x = global_idx % params.out_w;\n        out_y = (global_idx / params.out_w) % params.out_h;\n        c     = (global_idx / (params.out_w * params.out_h)) % params.channels;\n        n     = global_idx / (params.out_w * params.out_h * params.channels);\n    }\n};\n\nstruct cwhn_layout {\n    __device__ static int input_index(int n, int c, int y, int x, const conv_params & params) {\n        return n * (params.channels * params.in_w * params.in_h) + (y * params.in_w + x) * params.channels + c;\n    }\n\n    __device__ static int kernel_index(int c, int ky, int kx, const conv_params & params) {\n        return (ky * params.kernel_w + kx) * params.channels + c;\n    }\n\n    __device__ static int output_index(int n, int c, int y, int x, const conv_params & params) {\n        return n * (params.channels * params.out_w * params.out_h) + y * (params.out_w * params.channels) +\n               x * params.channels + c;\n    }\n\n    __device__ static void unpack_indices(int global_idx, const conv_params & params, int & n, int & c, int & out_y,\n                                          int & out_x) {\n        c     = global_idx % params.channels;\n        out_x = (global_idx / params.channels) % params.out_w;\n        out_y = (global_idx / (params.channels * params.out_w)) % params.out_h;\n        n     = global_idx / (params.channels * params.out_w * params.out_h);\n    }\n};\n\ntemplate <typename T, typename Layout>\n__global__ void conv2d_dw_kernel(const T * __restrict__ input, const T * __restrict__ kernel, T * __restrict__ output,\n                                 const int in_w, const int in_h, const int out_w, const int out_h,\n                                 const int kernel_w, const int kernel_h, const int stride_x, const int stride_y,\n                                 const int padding_x, const int padding_y, const int dilation_x, const int dilation_y,\n                                 const int channels, const int batches) {\n    const int global_idx     = blockIdx.x * blockDim.x + threadIdx.x;\n    const int total_elements = batches * channels * out_h * out_w;\n\n    if (global_idx >= total_elements) {\n        return;\n    }\n\n    conv_params params = { in_w,     in_h,      out_w,     out_h,      kernel_w,   kernel_h, stride_x,\n                           stride_y, padding_x, padding_y, dilation_x, dilation_y, channels, batches };\n\n    int batch_idx, channel_idx, out_y_idx, out_x_idx;\n    Layout::unpack_indices(global_idx, params, batch_idx, channel_idx, out_y_idx, out_x_idx);\n\n    T accumulator = 0;\n    kernel_bounds bounds = calculate_kernel_bounds(out_x_idx, out_y_idx, params);\n\n    for (int kern_y = bounds.y_min; kern_y < bounds.y_max; ++kern_y) {\n        int in_y_idx = calculate_input_coord(out_y_idx, kern_y, params.stride_y, params.dilation_y, params.padding_y);\n\n        for (int kern_x = bounds.x_min; kern_x < bounds.x_max; ++kern_x) {\n            int in_x_idx = calculate_input_coord(out_x_idx, kern_x, params.stride_x, params.dilation_x, params.padding_x);\n\n            const T input_val  = input[Layout::input_index(batch_idx, channel_idx, in_y_idx, in_x_idx, params)];\n            const T kernel_val = kernel[Layout::kernel_index(channel_idx, kern_y, kern_x, params)];\n\n            accumulator += input_val * kernel_val;\n        }\n    }\n\n    output[Layout::output_index(batch_idx, channel_idx, out_y_idx, out_x_idx, params)] = accumulator;\n}\n\nvoid ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * kernel = dst->src[0];\n    const ggml_tensor * input  = dst->src[1];\n\n    GGML_ASSERT(kernel->type == GGML_TYPE_F32 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);\n    const float * w_d = (const float *) kernel->data;\n    const float * x_d = (const float *) input->data;\n    float *       y_d = (float *) dst->data;\n\n    const int32_t * p          = (const int32_t *) dst->op_params;\n    const int       stride_x   = p[0];\n    const int       stride_y   = p[1];\n    const int       padding_x  = p[2];\n    const int       padding_y  = p[3];\n    const int       dilation_x = p[4];\n    const int       dilation_y = p[5];\n\n    const int in_w     = input->ne[0];\n    const int in_h     = input->ne[1];\n    const int kernel_w = kernel->ne[0];\n    const int kernel_h = kernel->ne[1];\n    const int out_w    = dst->ne[0];\n    const int out_h    = dst->ne[1];\n    const int channels = dst->ne[2];\n    const int batches  = dst->ne[3];\n\n    cudaStream_t st = ctx.stream();\n\n    const int total  = batches * channels * out_h * out_w;\n    const int blocks = (total + CUDA_CONV2D_DW_BLOCK_SIZE - 1) / CUDA_CONV2D_DW_BLOCK_SIZE;\n\n    if (ggml_is_contiguous(input)) {\n        conv2d_dw_kernel<float, whcn_layout><<<blocks, CUDA_CONV2D_DW_BLOCK_SIZE, 0, st>>>(\n            x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y,\n            dilation_x, dilation_y, channels, batches);\n    } else if (ggml_is_contiguous_channels(input)) {\n        conv2d_dw_kernel<float, cwhn_layout><<<blocks, CUDA_CONV2D_DW_BLOCK_SIZE, 0, st>>>(\n            x_d, w_d, y_d, in_w, in_h, out_w, out_h, kernel_w, kernel_h, stride_x, stride_y, padding_x, padding_y,\n            dilation_x, dilation_y, channels, batches);\n    } else {\n        GGML_ABORT(\"Unsupported memory layout for conv_2d_dw\");\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d-dw.cuh",
    "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_CONV2D_DW_BLOCK_SIZE 256\nvoid ggml_cuda_op_conv2d_dw(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d-transpose.cu",
    "content": "#include <algorithm>\n\n#include \"conv2d-transpose.cuh\"\n#include \"ggml.h\"\n\n__global__ void conv2d_transpose_kernel(const float * __restrict__ input, const half * __restrict__ kernel,\n                                        float * __restrict__ output, const int in_w, const int in_h, const int out_w,\n                                        const int out_h, const int kernel_w, const int kernel_h, const int stride,\n                                        const int c_in, const int c_out, const int batches) {\n    const int global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n    const int total_elements = out_w * out_h * c_out * batches;\n\n    if (global_idx >= total_elements) {\n        return;\n    }\n\n    const int out_x_idx = global_idx % out_w;\n    const int out_y_idx = (global_idx / out_w) % out_h;\n    const int c_idx     = (global_idx / (out_w * out_h)) % c_out;\n    const int n_idx     = global_idx / (out_w * out_h * c_out);\n\n    float accumulator = 0;\n    // For each output idx, find the inputs that contribute to it by checking stride alignment and bounds\n\n    for (int c_in_idx = 0; c_in_idx < c_in; c_in_idx++) {\n        for (int kh = 0; kh < kernel_h; ++kh) {\n            int in_y = out_y_idx - kh;\n            if (in_y < 0 || in_y % stride) continue;\n            in_y /= stride;\n            if (in_y >= in_h) continue;\n\n            for (int kw = 0; kw < kernel_w; ++kw) {\n                int in_x = out_x_idx - kw;\n                if (in_x < 0 || in_x % stride) continue;\n                in_x /= stride;\n                if (in_x >= in_w) continue;\n\n                const int input_idx = (in_w * in_h * c_in) * n_idx + (in_w * in_h) * c_in_idx + (in_w) *in_y + in_x;\n                const int kernel_idx =\n                    (kernel_h * kernel_w * c_out) * c_in_idx + (kernel_h * kernel_w) * c_idx + (kernel_w) *kh + kw;\n\n                float input_val = input[input_idx];\n                half  kern_val  = kernel[kernel_idx];\n\n                accumulator += input_val * (float) kern_val;\n            }\n        }\n    }\n\n    output[(out_w * out_h * c_out) * n_idx + (out_w * out_h) * c_idx + (out_w) *out_y_idx + out_x_idx] = accumulator;\n}\n\n//input is (W, H, C_in, N), Kernel is (W, H, C_out, C_in)\nvoid ggml_cuda_conv_2d_transpose_p0(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * kernel = dst->src[0];\n    const ggml_tensor * input  = dst->src[1];\n\n    GGML_ASSERT(kernel->type == GGML_TYPE_F16 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);\n\n    const float * input_data  = (const float *) input->data;\n    float *       output_data = (float *) dst->data;\n    const half * kernel_data = (const half *) kernel->data;\n\n    const int input_w      = input->ne[0];\n    const int input_h      = input->ne[1];\n    const int output_w     = dst->ne[0];\n    const int output_h     = dst->ne[1];\n    const int channels_in  = input->ne[2];\n    const int channels_out = kernel->ne[2];\n    const int kernel_w     = kernel->ne[0];\n    const int kernel_h     = kernel->ne[1];\n    const int stride       = dst->op_params[0];\n    const int batches      = input->ne[3];\n\n    GGML_ASSERT(channels_in == kernel->ne[3]);\n    GGML_ASSERT(stride > 0);\n\n    cudaStream_t st = ctx.stream();\n\n    GGML_ASSERT(ggml_is_contiguous(input));\n    GGML_ASSERT(ggml_is_contiguous(kernel));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n\n    const int total  = (output_w * output_h * channels_out * batches);\n    const int blocks = (total + CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE - 1) / CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE;\n\n    conv2d_transpose_kernel<<<blocks, CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE, 0, st>>>(\n        input_data, kernel_data, output_data, input_w, input_h, output_w, output_h, kernel_w, kernel_h, stride,\n        channels_in, channels_out, batches);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d-transpose.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CONV2D_TRANSPOSE_BLOCK_SIZE 256\nvoid ggml_cuda_conv_2d_transpose_p0(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d.cu",
    "content": "#include \"conv2d.cuh\"\n#include \"convert.cuh\"\n\nstruct conv_params {\n    const int64_t IW, IH;\n    const int64_t OW, OH;\n    const int64_t KW, KH;\n    const int64_t ST_X, ST_Y;\n    const int64_t PD_X, PD_Y;\n    const int64_t DL_X, DL_Y;\n    const int64_t IC, OC;\n    const int64_t B;\n    const int64_t TOTAL;\n};\n\nstruct kernel_bounds {\n    int64_t y_min, y_max;\n    int64_t x_min, x_max;\n};\n\n__device__ __forceinline__ int64_t max64(int64_t a, int64_t b) {\n    return (a > b) ? a : b;\n}\n\n__device__ __forceinline__ int64_t min64(int64_t a, int64_t b) {\n    return (a < b) ? a : b;\n}\n\n__device__ __forceinline__ kernel_bounds calculate_kernel_bounds(int64_t out_x, int64_t out_y, const conv_params & P) {\n    kernel_bounds bounds;\n    bounds.y_min = max64(0, (P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);\n    bounds.y_max = min64(P.KH, (P.IH + P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y);\n    bounds.x_min = max64(0, (P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);\n    bounds.x_max = min64(P.KW, (P.IW + P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X);\n    return bounds;\n}\n\n__device__ __forceinline__ int calculate_input_coord(int64_t out_coord,\n                                                     int64_t kern_coord,\n                                                     int64_t stride,\n                                                     int64_t dilation,\n                                                     int64_t padding) {\n    return out_coord * stride + kern_coord * dilation - padding;\n}\n\nstruct whcn_layout {\n    __device__ static int64_t input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv_params & P) {\n        return n * (P.IC * P.IW * P.IH) + c * P.IW * P.IH + y * P.IW + x;\n    }\n\n    __device__ static int64_t kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv_params & P) {\n        return c_out * (P.IC * P.KH * P.KW) + c_in * (P.KH * P.KW) + ky * P.KW + kx;\n    }\n\n    __device__ static int64_t output_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv_params & P) {\n        return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x;\n    }\n\n    __device__ static void unpack_indices(int64_t             global_idx,\n                                          const conv_params & P,\n                                          int64_t &           n,\n                                          int64_t &           c,\n                                          int64_t &           out_y,\n                                          int64_t &           out_x) {\n        out_x = global_idx % P.OW;\n        out_y = (global_idx / P.OW) % P.OH;\n        c     = (global_idx / (P.OW * P.OH)) % P.OC;\n        n     = global_idx / (P.OW * P.OH * P.OC);\n    }\n};\n\ntemplate <typename T, typename Layout>\nstatic __global__ void conv2d_kernel(const float * __restrict__ input,\n                                     const T * __restrict__ kernel,\n                                     float * __restrict__ output,\n                                     const conv_params P) {\n    const int64_t global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n    if (global_idx >= P.TOTAL) {\n        return;\n    }\n\n    int64_t n, c_out, out_y, out_x;\n    Layout::unpack_indices(global_idx, P, n, c_out, out_y, out_x);\n\n    float acc = 0.0f;\n\n    for (int64_t c_in = 0; c_in < P.IC; ++c_in) {\n        kernel_bounds bounds = calculate_kernel_bounds(out_x, out_y, P);\n\n        for (int64_t ky = bounds.y_min; ky < bounds.y_max; ++ky) {\n            const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y);\n\n            for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) {\n                const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X);\n\n                const float input_val = input[Layout::input_index(n, c_in, in_y, in_x, P)];\n                const T kernel_val = kernel[Layout::kernel_index(c_out, c_in, ky, kx, P)];\n                acc += (input_val * ggml_cuda_cast<float>(kernel_val));\n            }\n        }\n    }\n\n    // [N, OC, OH, OW]\n    output[Layout::output_index(n, c_out, out_y, out_x, P)] = acc;\n}\n\ntemplate <typename T>\nstatic void conv2d_cuda(const float * X_D, const T * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n    const int blocks = (P.TOTAL + CUDA_CONV2D_BLOCK_SIZE - 1) / CUDA_CONV2D_BLOCK_SIZE;\n    conv2d_kernel<T, whcn_layout><<<blocks, CUDA_CONV2D_BLOCK_SIZE, 0, st>>>(X_D, K_D, Y_D, P);\n}\n\nstatic void conv2d_cuda_f16(const float * X_D, const half * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n    conv2d_cuda<half>(X_D, K_D, Y_D, P, st);\n}\n\nstatic void conv2d_cuda_f32(const float * X_D, const float * K_D, float * Y_D, const conv_params P, cudaStream_t st) {\n    conv2d_cuda<float>(X_D, K_D, Y_D, P, st);\n}\n\nvoid ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * kernel = dst->src[0];\n    const ggml_tensor * input  = dst->src[1];\n    float *             K_D    = (float *) kernel->data;\n    const float *       X_D    = (const float *) input->data;\n    float *             Y_D    = (float *) dst->data;\n\n    GGML_ASSERT(ggml_is_contiguous(kernel));\n    GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32);\n\n    // same number of input channels\n    GGML_ASSERT(input->ne[2] == kernel->ne[2]);\n\n    cudaStream_t st = ctx.stream();\n\n    const int32_t * p    = (const int32_t *) dst->op_params;\n    const int       ST_X = p[0];  // stride_x\n    const int       ST_Y = p[1];  // stride_y\n    const int       PD_X = p[2];  // padding_x\n    const int       PD_Y = p[3];  // padding_y\n    const int       DL_X = p[4];  // dilation_x\n    const int       DL_Y = p[5];  // dilation_y\n\n    // No cwhn\n    GGML_ASSERT(p[6] == false);\n\n    const int IW = input->ne[0];   // input_w\n    const int IH = input->ne[1];   // input_h\n    const int OW = dst->ne[0];     // output_w\n    const int OH = dst->ne[1];     // output_h\n    const int KW = kernel->ne[0];  // kernel_w\n    const int KH = kernel->ne[1];  // kernel_h\n    const int IC = input->ne[2];   // input_channels\n    const int OC = kernel->ne[3];  // ouptut_chanles\n    const int B  = input->ne[3];   // n_batches\n\n    const int64_t total  = B * OC * OH * OW;\n    conv_params   params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total };\n\n    if (kernel->type == GGML_TYPE_F16) {\n        conv2d_cuda_f16(X_D, (half *) K_D, Y_D, params, st);\n    } else {\n        conv2d_cuda_f32(X_D, K_D, Y_D, params, st);\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/conv2d.cuh",
    "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_CONV2D_BLOCK_SIZE 256\nvoid ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/convert.cu",
    "content": "#include \"convert.cuh\"\n#include \"dequantize.cuh\"\n\n#include <cstdint>\n\n#define CUDA_Q8_0_NE_ALIGN 2048\n\ntemplate <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>\nstatic __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y,\n        const int64_t ne00, const int64_t ne01,\n        const int64_t ne0203, const uint3 ne02,\n        const int64_t s01, const int64_t s02, const int64_t s03) {\n    const int64_t i00 = 2 * (int64_t(blockDim.x)*blockIdx.x + threadIdx.x);\n\n    if (i00 >= ne00) {\n        return;\n    }\n\n    for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {\n        for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {\n            const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);\n            const int64_t i02 = dm.y;\n            const int64_t i03 = dm.x;\n\n            const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;\n\n            const int64_t ib = ibx0 + i00/qk; // block index\n            const int64_t iqs = (i00%qk)/qr; // quant index\n            const int64_t iybs = i00 - i00%qk; // y block start index\n            const int64_t y_offset = qr == 1 ? 1 : qk/2;\n\n            // dequantize\n            float2 v;\n            dequantize_kernel(vx, ib, iqs, v);\n\n            const int64_t iy0 = (i0203*ne01 + i01)*ne00 + iybs + iqs;\n            y[iy0 + 0]        = ggml_cuda_cast<dst_t>(v.x);\n            y[iy0 + y_offset] = ggml_cuda_cast<dst_t>(v.y);\n        }\n    }\n}\n\ntemplate <bool need_check>\nstatic __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {\n#if __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n    constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;\n\n    const int64_t   i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;\n    const int * x0 = ((int *) vx) + blockIdx.x * nint;\n    half2 * y2 = (half2 *) (y + i0);\n\n    __shared__ int vals[nint];\n\n#pragma unroll\n    for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {\n        if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {\n            break;\n        }\n\n        const int ix = ix0 + threadIdx.x;\n        vals[ix] = x0[ix];\n    }\n\n    __syncthreads();\n\n#pragma unroll\n    for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {\n        if (need_check && i0 + iy + 2*threadIdx.x >= k) {\n            return;\n        }\n\n        const half * b0 = ((const half  *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);\n        const half    d = *b0;\n        const char2  qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];\n\n        y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));\n    }\n#else\n    GGML_UNUSED_VARS(vx, y, k);\n    NO_DEVICE_CODE;\n#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {\n\n    const int64_t i = blockIdx.x;\n\n    // assume 32 threads\n    const int64_t tid = threadIdx.x;\n    const int64_t il  = tid/8;\n    const int64_t ir  = tid%8;\n    const int64_t ib = 8*i + ir;\n    if (ib >= nb32) {\n        return;\n    }\n\n    dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n    const block_q4_0 * x = (const block_q4_0 *)vx + ib;\n    const float d = __half2float(x->d);\n    const float dm = -8*d;\n\n    const uint8_t * q = x->qs + 4*il;\n\n    for (int l = 0; l < 4; ++l) {\n        y[l+ 0] = d * (q[l] & 0xF) + dm;\n        y[l+16] = d * (q[l] >>  4) + dm;\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {\n\n    const int64_t i = blockIdx.x;\n\n    // assume 32 threads\n    const int64_t tid = threadIdx.x;\n    const int64_t il  = tid/8;\n    const int64_t ir  = tid%8;\n    const int64_t ib = 8*i + ir;\n    if (ib >= nb32) {\n        return;\n    }\n\n    dst_t * y = yy + 256*i + 32*ir + 4*il;\n\n    const block_q4_1 * x = (const block_q4_1 *)vx + ib;\n    const float2 d = __half22float2(x->dm);\n\n    const uint8_t * q = x->qs + 4*il;\n\n    for (int l = 0; l < 4; ++l) {\n        y[l+ 0] = d.x * (q[l] & 0xF) + d.y;\n        y[l+16] = d.x * (q[l] >>  4) + d.y;\n    }\n}\n\n//================================== k-quants\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_q2_K * x = (const block_q2_K *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t n   = tid/32;\n    const int64_t l   = tid - 32*n;\n    const int64_t is  = 8*n + l/16;\n\n    const uint8_t q = x[i].qs[32*n + l];\n    dst_t * y = yy + i*QK_K + 128*n;\n\n    float dall = __low2half(x[i].dm);\n    float dmin = __high2half(x[i].dm);\n    y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4);\n    y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4);\n    y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4);\n    y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4);\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i = blockIdx.x;\n    const block_q3_K * x = (const block_q3_K *) vx;\n\n    const int64_t r = threadIdx.x/4;\n    const int64_t tid = r/2;\n    const int64_t is0 = r%2;\n    const int64_t l0 = 16*is0 + 4*(threadIdx.x%4);\n    const int64_t n = tid / 4;\n    const int64_t j = tid - 4*n;\n\n    uint8_t m = 1 << (4*n + j);\n    int64_t is = 8*n + 2*j + is0;\n    int shift = 2*j;\n\n    int8_t us = is <  4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :\n                is <  8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :\n                is < 12 ? (x[i].scales[is-8] >>  4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :\n                          (x[i].scales[is-8] >>  4) | (((x[i].scales[is-4] >> 6) & 3) << 4);\n    float d_all = x[i].d;\n    float dl = d_all * (us - 32);\n\n    dst_t * y = yy + i*QK_K + 128*n + 32*j;\n    const uint8_t * q = x[i].qs + 32*n;\n    const uint8_t * hm = x[i].hmask;\n\n    for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));\n}\n\nstatic inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {\n    if (j < 4) {\n        d = q[j] & 63; m = q[j + 4] & 63;\n    } else {\n        d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);\n        m = (q[j+4] >>  4) | ((q[j-0] >> 6) << 4);\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n    const block_q4_K * x = (const block_q4_K *) vx;\n\n    const int64_t i = blockIdx.x;\n\n    // assume 32 threads\n    const int64_t tid = threadIdx.x;\n    const int64_t il  = tid/8;\n    const int64_t ir  = tid%8;\n    const int64_t is  = 2*il;\n    const int64_t n   = 4;\n\n    dst_t * y = yy + i*QK_K + 64*il + n*ir;\n\n    const float dall = __low2half(x[i].dm);\n    const float dmin = __high2half(x[i].dm);\n\n    const uint8_t * q = x[i].qs + 32*il + n*ir;\n\n    uint8_t sc, m;\n    get_scale_min_k4(is + 0, x[i].scales, sc, m);\n    const float d1 = dall * sc; const float m1 = dmin * m;\n    get_scale_min_k4(is + 1, x[i].scales, sc, m);\n    const float d2 = dall * sc; const float m2 = dmin * m;\n    for (int l = 0; l < n; ++l) {\n        y[l + 0] = d1 * (q[l] & 0xF) - m1;\n        y[l +32] = d2 * (q[l] >>  4) - m2;\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n    const block_q5_K * x = (const block_q5_K *) vx;\n\n    const int64_t i = blockIdx.x;\n\n    // assume 64 threads - this is very slightly better than the one below\n    const int64_t tid = threadIdx.x;\n    const int64_t il  = tid/16;   // il is in 0...3\n    const int64_t ir  = tid%16;   // ir is in 0...15\n    const int64_t is  = 2*il;     // is is in 0...6\n\n    dst_t * y = yy + i*QK_K + 64*il + 2*ir;\n\n    const float dall = __low2half(x[i].dm);\n    const float dmin = __high2half(x[i].dm);\n\n    const uint8_t * ql = x[i].qs + 32*il + 2*ir;\n    const uint8_t * qh = x[i].qh + 2*ir;\n\n    uint8_t sc, m;\n    get_scale_min_k4(is + 0, x[i].scales, sc, m);\n    const float d1 = dall * sc; const float m1 = dmin * m;\n    get_scale_min_k4(is + 1, x[i].scales, sc, m);\n    const float d2 = dall * sc; const float m2 = dmin * m;\n\n    uint8_t   hm  = 1 << (2*il);\n    y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;\n    y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;\n    hm <<= 1;\n    y[32] = d2 * ((ql[ 0] >>  4) + (qh[ 0] & hm ? 16 : 0)) - m2;\n    y[33] = d2 * ((ql[ 1] >>  4) + (qh[ 1] & hm ? 16 : 0)) - m2;\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n    const block_q6_K * x = (const block_q6_K *) vx;\n\n    const int64_t i = blockIdx.x;\n\n    // assume 64 threads - this is very slightly better than the one below\n    const int64_t tid = threadIdx.x;\n    const int64_t ip  = tid/32;   // ip is 0 or 1\n    const int64_t il  = tid - 32*ip; // 0...32\n    const int64_t is  = 8*ip + il/16;\n\n    dst_t * y = yy + i*QK_K + 128*ip + il;\n\n    const float d = x[i].d;\n\n    const uint8_t * ql = x[i].ql + 64*ip + il;\n    const uint8_t   qh = x[i].qh[32*ip + il];\n    const int8_t  * sc = x[i].scales + is;\n\n    y[ 0] = d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);\n    y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);\n    y[64] = d * sc[4] * ((int8_t)((ql[ 0]  >> 4) | (((qh >> 4) & 3) << 4)) - 32);\n    y[96] = d * sc[6] * ((int8_t)((ql[32]  >> 4) | (((qh >> 6) & 3) << 4)) - 32);\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq2_xxs * x = (const block_iq2_xxs  *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint16_t * q2 = x[i].qs + 4*ib;\n    const uint8_t  * aux8 = (const uint8_t *)q2;\n    const uint8_t  * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);\n    const uint32_t aux32 = q2[2] | (q2[3] << 16);\n    const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f;\n    const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];\n    for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq2_xs * x = (const block_iq2_xs *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint16_t * q2 = x[i].qs + 4*ib;\n    const uint8_t  * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));\n    const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n    const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];\n    for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq2_s * x = (const block_iq2_s *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));\n    const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;\n    const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];\n    for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq3_xxs * x = (const block_iq3_xxs  *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint8_t  * q3 = x[i].qs + 8*ib;\n    const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;\n    const uint8_t  * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);\n    const uint8_t  * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);\n    const uint32_t aux32 = gas[0] | (gas[1] << 16);\n    const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.5f;\n    const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];\n    for (int j = 0; j < 4; ++j) {\n        y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n        y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq3_s * x = (const block_iq3_s *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint8_t * qs = x[i].qs + 8*ib;\n    const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));\n    const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));\n    const float d = (float)x[i].d * (1 + 2*((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf));\n    const uint8_t signs = x[i].signs[4*ib + il];\n    for (int j = 0; j < 4; ++j) {\n        y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);\n        y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq1_s * x = (const block_iq1_s  *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;\n    const float d = (float)x[i].d * (2*((x[i].qh[ib] >> 12) & 7) + 1);\n    uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n    grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];\n    grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n    grid32[0] &= 0x0f0f0f0f;\n    for (int j = 0; j < 8; ++j) {\n        y[j] = d * (q[j] + delta);\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq1_m * x = (const block_iq1_m  *) vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 8*il;\n    const uint16_t * sc = (const uint16_t *)x[i].scales;\n    iq1m_scale_t scale;\n    scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);\n    const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);\n    const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);\n    const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;\n    uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;\n    grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];\n    grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;\n    grid32[0] &= 0x0f0f0f0f;\n    for (int j = 0; j < 8; ++j) {\n        y[j] = d * (q[j] + delta);\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n    const uint8_t  * q4 = x[ib].qs + 4*il;\n    const float d = (float)x[ib].d;\n    for (int j = 0; j < 4; ++j) {\n        y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n        y[j+16] = d * kvalues_iq4nl[q4[j] >>  4];\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n    const int64_t i   = blockIdx.x;\n    const block_iq4_xs * x = (const block_iq4_xs *)vx;\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n    const uint8_t  * q4 = x[i].qs + 16*ib + 4*il;\n    const float d = (float)x[i].d * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);\n    for (int j = 0; j < 4; ++j) {\n        y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];\n        y[j+16] = d * kvalues_iq4nl[q4[j] >>  4];\n    }\n}\n\ntemplate<typename dst_t>\nstatic __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) {\n\n    const int64_t i   = blockIdx.x;\n    const block_mxfp4 * x = (const block_mxfp4 *) vx + i*(QK_K/QK_MXFP4);\n\n    const int64_t tid = threadIdx.x;\n    const int64_t il = tid/8; // 0...3\n    const int64_t ib = tid%8; // 0...7\n    dst_t * y = yy + i*QK_K + 32*ib + 4*il;\n    const uint8_t  * q4 = x[ib].qs + 4*il;\n    const float d = ggml_cuda_e8m0_to_fp32(x[ib].e);\n    for (int j = 0; j < 4; ++j) {\n        y[j+ 0] = d * kvalues_mxfp4[q4[j] & 0xf]*0.5f;\n        y[j+16] = d * kvalues_mxfp4[q4[j] >>  4]*0.5f;\n    }\n}\n\ntemplate <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>\nstatic void dequantize_block_cuda(const void * vx, dst_t * y,\n        const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n        const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {\n    const int64_t ne0203 = ne02*ne03;\n    const uint3 ne02_fdv = init_fastdiv_values(ne02);\n    const dim3 num_blocks((ne00 + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE), (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));\n    dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>\n        (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);\n}\n\ntemplate <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>\nstatic void dequantize_block_cont_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {\n    dequantize_block_cuda<qk, qr, dequantize_kernel, dst_t>(vx, y, k, 1, 1, 1, k/qk, k/qk, k/qk, stream);\n}\n\nstatic void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {\n    const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;\n    if (k % CUDA_Q8_0_NE_ALIGN == 0) {\n        const bool need_check = false;\n        dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);\n    } else {\n        const bool need_check = true;\n        dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);\n    }\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb32 = k / 32;\n    const int nb = (k + 255) / 256;\n    dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb32 = k / 32;\n    const int nb = (k + 255) / 256;\n    dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = (k + QK_K - 1) / QK_K;\n    dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = k / QK_K;\n    dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = (k + QK_K - 1) / QK_K;\n    dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate<typename dst_t>\nstatic void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    const int nb = (k + QK_K - 1) / QK_K;\n    dequantize_block_mxfp4<<<nb, 32, 0, stream>>>(vx, y);\n}\n\ntemplate <typename src_t, typename dst_t>\nstatic __global__ void convert_unary(\n        const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01,\n        const int64_t ne0203, const uint3 ne02,\n        const int64_t s01, const int64_t s02, const int64_t s03) {\n    const int64_t i00 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n    if (i00 >= ne00) {\n        return;\n    }\n\n    const src_t * x = (const src_t *) vx;\n\n    for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {\n        for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {\n            const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);\n            const int64_t i02 = dm.y;\n            const int64_t i03 = dm.x;\n\n            const int64_t ix = i03*s03 + i02*s02 + i01*s01 + i00;\n            const int64_t iy = (i0203*ne01 + i01)*ne00 + i00;\n            y[iy] = ggml_cuda_cast<dst_t>(x[ix]);\n        }\n    }\n}\n\ntemplate <typename src_t, typename dst_t>\nstatic void convert_unary_cuda(const void * vx, dst_t * y,\n        const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,\n        const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {\n    const int64_t ne0203 = ne02*ne03;\n    const uint3 ne02_fdv = init_fastdiv_values(ne02);\n    const dim3 num_blocks((ne00 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE, (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));\n    convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>\n        (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);\n}\n\ntemplate <typename src_t, typename dst_t>\nstatic void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {\n    convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);\n}\n\nto_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_F32:\n            return convert_unary_cont_cuda<float>;\n        case GGML_TYPE_F16:\n            return convert_unary_cont_cuda<half>;\n        default:\n            return nullptr;\n    }\n}\n\nto_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_Q4_0:\n            return dequantize_row_q4_0_cuda;\n        case GGML_TYPE_Q4_1:\n            return dequantize_row_q4_1_cuda;\n        case GGML_TYPE_Q5_0:\n            return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;\n        case GGML_TYPE_Q5_1:\n            return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;\n        case GGML_TYPE_Q8_0:\n            if (fp16_available(ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) {\n                return dequantize_block_q8_0_f16_cuda;\n            }\n            return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;\n        case GGML_TYPE_Q2_K:\n            return dequantize_row_q2_K_cuda;\n        case GGML_TYPE_Q3_K:\n            return dequantize_row_q3_K_cuda;\n        case GGML_TYPE_Q4_K:\n            return dequantize_row_q4_K_cuda;\n        case GGML_TYPE_Q5_K:\n            return dequantize_row_q5_K_cuda;\n        case GGML_TYPE_Q6_K:\n            return dequantize_row_q6_K_cuda;\n        case GGML_TYPE_IQ2_XXS:\n            return dequantize_row_iq2_xxs_cuda;\n        case GGML_TYPE_IQ2_XS:\n            return dequantize_row_iq2_xs_cuda;\n        case GGML_TYPE_IQ2_S:\n            return dequantize_row_iq2_s_cuda;\n        case GGML_TYPE_IQ3_XXS:\n            return dequantize_row_iq3_xxs_cuda;\n        case GGML_TYPE_IQ1_S:\n            return dequantize_row_iq1_s_cuda;\n        case GGML_TYPE_IQ1_M:\n            return dequantize_row_iq1_m_cuda;\n        case GGML_TYPE_IQ4_NL:\n            return dequantize_row_iq4_nl_cuda;\n        case GGML_TYPE_IQ4_XS:\n            return dequantize_row_iq4_xs_cuda;\n        case GGML_TYPE_IQ3_S:\n            return dequantize_row_iq3_s_cuda;\n        case GGML_TYPE_MXFP4:\n            return dequantize_row_mxfp4_cuda;\n        case GGML_TYPE_F32:\n            return convert_unary_cont_cuda<float>;\n        case GGML_TYPE_BF16:\n            return convert_unary_cont_cuda<nv_bfloat16>;\n        default:\n            return nullptr;\n    }\n}\n\nto_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_Q4_0:\n            return dequantize_row_q4_0_cuda;\n        case GGML_TYPE_Q4_1:\n            return dequantize_row_q4_1_cuda;\n        case GGML_TYPE_Q5_0:\n            return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;\n        case GGML_TYPE_Q5_1:\n            return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;\n        case GGML_TYPE_Q8_0:\n            return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;\n        case GGML_TYPE_Q2_K:\n            return dequantize_row_q2_K_cuda;\n        case GGML_TYPE_Q3_K:\n            return dequantize_row_q3_K_cuda;\n        case GGML_TYPE_Q4_K:\n            return dequantize_row_q4_K_cuda;\n        case GGML_TYPE_Q5_K:\n            return dequantize_row_q5_K_cuda;\n        case GGML_TYPE_Q6_K:\n            return dequantize_row_q6_K_cuda;\n        case GGML_TYPE_IQ2_XXS:\n            return dequantize_row_iq2_xxs_cuda;\n        case GGML_TYPE_IQ2_XS:\n            return dequantize_row_iq2_xs_cuda;\n        case GGML_TYPE_IQ2_S:\n            return dequantize_row_iq2_s_cuda;\n        case GGML_TYPE_IQ3_XXS:\n            return dequantize_row_iq3_xxs_cuda;\n        case GGML_TYPE_IQ1_S:\n            return dequantize_row_iq1_s_cuda;\n        case GGML_TYPE_IQ1_M:\n            return dequantize_row_iq1_m_cuda;\n        case GGML_TYPE_IQ4_NL:\n            return dequantize_row_iq4_nl_cuda;\n        case GGML_TYPE_IQ4_XS:\n            return dequantize_row_iq4_xs_cuda;\n        case GGML_TYPE_IQ3_S:\n            return dequantize_row_iq3_s_cuda;\n        case GGML_TYPE_MXFP4:\n            return dequantize_row_mxfp4_cuda;\n        case GGML_TYPE_F16:\n            return convert_unary_cont_cuda<half>;\n        case GGML_TYPE_BF16:\n            return convert_unary_cont_cuda<nv_bfloat16>;\n        default:\n            return nullptr;\n    }\n}\n\nto_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_F32:\n            return convert_unary_cuda<float>;\n        case GGML_TYPE_Q4_0:\n            return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;\n        case GGML_TYPE_Q4_1:\n            return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;\n        case GGML_TYPE_Q5_0:\n            return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;\n        case GGML_TYPE_Q5_1:\n            return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;\n        case GGML_TYPE_Q8_0:\n            return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;\n        case GGML_TYPE_BF16:\n            return convert_unary_cuda<nv_bfloat16>;\n        default:\n            return nullptr;\n    }\n}\n\nto_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_F32:\n            return convert_unary_cuda<float, nv_bfloat16>;\n        case GGML_TYPE_Q4_0:\n            return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;\n        case GGML_TYPE_Q4_1:\n            return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;\n        case GGML_TYPE_Q5_0:\n            return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;\n        case GGML_TYPE_Q5_1:\n            return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;\n        case GGML_TYPE_Q8_0:\n            return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;\n        case GGML_TYPE_F16:\n            return convert_unary_cuda<half, nv_bfloat16>;\n        default:\n            return nullptr;\n    }\n}\n\nto_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {\n    switch (type) {\n        case GGML_TYPE_F16:\n            return convert_unary_cuda<half, float>;\n        case GGML_TYPE_Q4_0:\n            return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;\n        case GGML_TYPE_Q4_1:\n            return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;\n        case GGML_TYPE_Q5_0:\n            return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;\n        case GGML_TYPE_Q5_1:\n            return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;\n        case GGML_TYPE_Q8_0:\n            return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;\n        case GGML_TYPE_BF16:\n            return convert_unary_cuda<nv_bfloat16, float>;\n        default:\n            return nullptr;\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/convert.cuh",
    "content": "#pragma once\n#include \"common.cuh\"\n\n#define CUDA_DEQUANTIZE_BLOCK_SIZE 256\n\ntemplate<typename T>\nusing to_t_cuda_t = void (*)(const void * x, T * y, int64_t k, cudaStream_t stream);\n\ntypedef to_t_cuda_t<float> to_fp32_cuda_t;\ntypedef to_t_cuda_t<half> to_fp16_cuda_t;\ntypedef to_t_cuda_t<nv_bfloat16> to_bf16_cuda_t;\n\nto_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);\n\nto_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type);\n\nto_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);\n\n// TODO more general support for non-contiguous inputs\n\ntemplate<typename T>\nusing to_t_nc_cuda_t = void (*)(const void * x, T * y,\n    int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03,\n    int64_t s01, int64_t s02, int64_t s03, cudaStream_t stream);\n\ntypedef to_t_nc_cuda_t<float> to_fp32_nc_cuda_t;\ntypedef to_t_nc_cuda_t<half> to_fp16_nc_cuda_t;\ntypedef to_t_nc_cuda_t<nv_bfloat16> to_bf16_nc_cuda_t;\n\nto_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type);\nto_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type);\nto_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type);\n\ntemplate<typename dst_t, typename src_t>\n __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) {\n    if constexpr (std::is_same_v<dst_t, src_t>) {\n        return x;\n    } else if constexpr(std::is_same_v<dst_t, nv_bfloat16>) {\n        return __float2bfloat16(float(x));\n    } else if constexpr(std::is_same_v<src_t, nv_bfloat16>) {\n        return __bfloat162float(x);\n    } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, half2>) {\n        return __float22half2_rn(x);\n    } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, nv_bfloat162>) {\n        // bypass compile error on cuda 12.0.1\n#ifdef GGML_USE_HIP\n        return __float22bfloat162_rn(x);\n#else\n        return {x.x, x.y};\n#endif // GGML_USE_HIP\n    } else if constexpr(std::is_same_v<dst_t, int32_t>) {\n        return int32_t(x);\n    } else {\n        return float(x);\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/count-equal.cu",
    "content": "#include \"common.cuh\"\n#include \"count-equal.cuh\"\n\n#include <cstdint>\n\ntemplate <typename T>\nstatic __global__ void count_equal(const T * __restrict__ x, const T * __restrict__ y, int64_t * __restrict__ dst, const int64_t dk, const int64_t k) {\n    const int64_t i0 = (int64_t) blockIdx.x*dk;\n    const int64_t i1 = min(i0 + dk, k);\n\n    int nequal = 0;\n\n    for (int64_t i = i0 + threadIdx.x; i < i1; i += WARP_SIZE) {\n        const T xi = x[i];\n        const T yi = y[i];\n        nequal += xi == yi;\n    }\n\n    nequal = warp_reduce_sum(nequal);\n\n    if (threadIdx.x != 0) {\n        return;\n    }\n\n    atomicAdd((int *) dst, nequal);\n}\n\nvoid ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == src1->type);\n    GGML_ASSERT( dst->type == GGML_TYPE_I64);\n\n    GGML_ASSERT(ggml_are_same_shape(src0, src1));\n    GGML_ASSERT(ggml_is_contiguous(src0));\n    GGML_ASSERT(ggml_is_contiguous(src1));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n\n    int64_t * dst_d  = (int64_t *) dst->data;\n\n    cudaStream_t stream = ctx.stream();\n    const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;\n\n    const int64_t ne = ggml_nelements(src0);\n    GGML_ASSERT(ne < (1 << 30) && \"atomicAdd implementation only supports int\");\n    const int64_t dne = GGML_PAD((ne + 4*nsm - 1) / (4*nsm), CUDA_COUNT_EQUAL_CHUNK_SIZE);\n\n    CUDA_CHECK(cudaMemsetAsync(dst_d, 0, ggml_nbytes(dst), stream));\n\n    const dim3 blocks_dim(WARP_SIZE, 1, 1);\n    const dim3 blocks_num(std::min((int64_t)4*nsm, (ne + CUDA_COUNT_EQUAL_CHUNK_SIZE - 1)/CUDA_COUNT_EQUAL_CHUNK_SIZE), 1, 1);\n\n    switch (src0->type) {\n        case GGML_TYPE_I32: {\n            const int * src0_d = (const int *) src0->data;\n            const int * src1_d = (const int *) src1->data;\n            count_equal<<<blocks_num, blocks_dim, 0, stream>>>(src0_d, src1_d, dst_d, dne, ne);\n        } break;\n        default:\n            GGML_ASSERT(false);\n            break;\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/count-equal.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_COUNT_EQUAL_CHUNK_SIZE 128\n\nvoid ggml_cuda_count_equal(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cp-async.cuh",
    "content": "// Simplified API for asynchronous data loading.\n\n#include \"common.cuh\"\n\n\nstatic __device__ __forceinline__ unsigned int ggml_cuda_cvta_generic_to_shared(void * generic_ptr) {\n#ifdef CP_ASYNC_AVAILABLE\n    return __cvta_generic_to_shared(generic_ptr);\n#else\n    GGML_UNUSED(generic_ptr);\n    NO_DEVICE_CODE;\n    return 0;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// Copies data from global to shared memory, cg == cache global.\n// Both the src and dst pointers must be aligned to 16 bit.\n// Shared memory uses 32 bit addressing, the pointer is passed as unsigned int.\n// Generic pointers can be converted to 32 bit shared memory pointers using __cvta_generic_to_shared.\n// Only the 16 bit copy is exposed because 4 and 8 bit copies did not yield performance improvements.\ntemplate <int preload>\nstatic __device__ __forceinline__ void cp_async_cg_16(const unsigned int dst, const void * src) {\n    static_assert(preload == 0 || preload == 64 || preload == 128 || preload == 256, \"bad preload\");\n#ifdef CP_ASYNC_AVAILABLE\n#if CUDART_VERSION >= 11040\n    if (preload == 256) {\n        asm volatile(\"cp.async.cg.shared.global.L2::256B [%0], [%1], 16;\"\n            : : \"r\"(dst), \"l\"(src));\n    } else if (preload == 128) {\n        asm volatile(\"cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\"\n            : : \"r\"(dst), \"l\"(src));\n    } else if (preload == 64) {\n        asm volatile(\"cp.async.cg.shared.global.L2::64B [%0], [%1], 16;\"\n            : : \"r\"(dst), \"l\"(src));\n    } else\n#endif // CUDART_VERSION >= 11040\n    {\n        asm volatile(\"cp.async.cg.shared.global [%0], [%1], 16;\"\n            : : \"r\"(dst), \"l\"(src));\n    }\n#else\n    GGML_UNUSED(dst);\n    GGML_UNUSED(src);\n    NO_DEVICE_CODE;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// Makes each thread wait until its asynchronous data copies are done.\n// This does NOT provide any additional synchronization.\n// In particular, when copying data with multiple warps a call to __syncthreads will be needed.\nstatic __device__ __forceinline__ void cp_async_wait_all() {\n#ifdef CP_ASYNC_AVAILABLE\n    asm volatile(\"cp.async.wait_all;\");\n#else\n    NO_DEVICE_CODE;\n#endif // CP_ASYNC_AVAILABLE\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cpy-utils.cuh",
    "content": "#pragma once\n\n#include \"ggml-common.h\"\n#include \"convert.cuh\"\n\nstatic __device__ __forceinline__ int best_index_int8(int n, const int8_t * val, float x) {\n    if (x <= val[0]) return 0;\n    if (x >= val[n-1]) return n-1;\n    int ml = 0, mu = n-1;\n    while (mu-ml > 1) {\n        int mav = (ml+mu)/2;\n        if (x < val[mav]) mu = mav; else ml = mav;\n    }\n    return x - val[mu-1] < val[mu] - x ? mu-1 : mu;\n}\n\nstatic __device__ void quantize_f32_q4_0_block(const float * __restrict__ x, block_q4_0 * __restrict__ y) {\n    float amax = 0.0f;\n    float vmax = 0.0f;\n\n    for (int j = 0; j < QK4_0; ++j) {\n        const float v = x[j];\n        if (amax < fabsf(v)) {\n            amax = fabsf(v);\n            vmax = v;\n        }\n    }\n\n    const float d  = vmax / -8;\n    const float id = d ? 1.0f/d : 0.0f;\n\n    y->d = d;\n\n    for (int j = 0; j < QK4_0/2; ++j) {\n        const float x0 = x[0       + j]*id;\n        const float x1 = x[QK4_0/2 + j]*id;\n\n        const uint8_t xi0 = min(15, (int8_t)(x0 + 8.5f));\n        const uint8_t xi1 = min(15, (int8_t)(x1 + 8.5f));\n\n        y->qs[j]  = xi0;\n        y->qs[j] |= xi1 << 4;\n    }\n}\n\nstatic __device__ void quantize_f32_q4_1_block(const float * __restrict__ x, block_q4_1 * __restrict__ y) {\n    float vmin = FLT_MAX;\n    float vmax = -FLT_MAX;\n\n    for (int j = 0; j < QK4_1; ++j) {\n        const float v = x[j];\n        if (v < vmin) vmin = v;\n        if (v > vmax) vmax = v;\n    }\n\n    const float d  = (vmax - vmin) / ((1 << 4) - 1);\n    const float id = d ? 1.0f/d : 0.0f;\n\n    y->dm.x = d;\n    y->dm.y = vmin;\n\n    for (int j = 0; j < QK4_1/2; ++j) {\n        const float x0 = (x[0       + j] - vmin)*id;\n        const float x1 = (x[QK4_1/2 + j] - vmin)*id;\n\n        const uint8_t xi0 = min(15, (int8_t)(x0 + 0.5f));\n        const uint8_t xi1 = min(15, (int8_t)(x1 + 0.5f));\n\n        y->qs[j]  = xi0;\n        y->qs[j] |= xi1 << 4;\n    }\n}\n\nstatic __device__ void quantize_f32_q5_0_block(const float * __restrict__ x, block_q5_0 * __restrict__ y) {\n    float amax = 0.0f;\n    float vmax = 0.0f;\n\n    for (int j = 0; j < QK5_0; ++j) {\n        const float v = x[j];\n        if (amax < fabsf(v)) {\n            amax = fabsf(v);\n            vmax = v;\n        }\n    }\n\n    const float d  = vmax / -16;\n    const float id = d ? 1.0f/d : 0.0f;\n\n    y->d = d;\n\n    uint32_t qh = 0;\n    for (int j = 0; j < QK5_0/2; ++j) {\n        const float x0 = x[0       + j]*id;\n        const float x1 = x[QK5_0/2 + j]*id;\n\n        const uint8_t xi0 = min(31, (int8_t)(x0 + 16.5f));\n        const uint8_t xi1 = min(31, (int8_t)(x1 + 16.5f));\n\n        y->qs[j]  = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n        qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n        qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_0/2);\n    }\n    memcpy(y->qh, &qh, sizeof(qh));\n}\n\nstatic __device__ void quantize_f32_q5_1_block(const float * __restrict__ x, block_q5_1 * __restrict__ y) {\n    float min = x[0];\n    float max = x[0];\n\n    for (int j = 1; j < QK5_1; ++j) {\n        const float v = x[j];\n        min = v < min ? v : min;\n        max = v > max ? v : max;\n    }\n\n    const float d  = (max - min) / 31;\n    const float id = d ? 1.0f/d : 0.0f;\n\n    y->dm.x = d;\n    y->dm.y = min;\n\n    uint32_t qh = 0;\n    for (int j = 0; j < QK5_1/2; ++j) {\n        const float x0 = (x[0       + j] - min)*id;\n        const float x1 = (x[QK5_1/2 + j] - min)*id;\n\n        const uint8_t xi0 = (uint8_t)(x0 + 0.5f);\n        const uint8_t xi1 = (uint8_t)(x1 + 0.5f);\n\n        y->qs[j]  = (xi0 & 0xf) | ((xi1 & 0xf) << 4);\n        qh |= ((xi0 & 0x10u) >> 4) << (j + 0);\n        qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_1/2);\n    }\n    memcpy(y->qh, &qh, sizeof(qh));\n}\n\nstatic __device__ void quantize_f32_q8_0_block(const float * __restrict__ x, block_q8_0 * __restrict__ y) {\n    float amax = 0.0f; // absolute max\n\n    for (int j = 0; j < QK8_0; j++) {\n        const float v = x[j];\n        amax = fmaxf(amax, fabsf(v));\n    }\n\n    const float d = amax / ((1 << 7) - 1);\n    const float id = d ? 1.0f/d : 0.0f;\n\n    y->d = d;\n\n    for (int j = 0; j < QK8_0; ++j) {\n        const float x0 = x[j]*id;\n        y->qs[j] = roundf(x0);\n    }\n}\n\nstatic __device__ void quantize_f32_iq4_nl_block(const float * __restrict__ x, block_iq4_nl * __restrict__ y) {\n    float amax = 0.0f;\n    float vmax = 0.0f;\n\n    for (int j = 0; j < QK4_NL; ++j) {\n        const float v = x[j];\n        if (amax < fabsf(v)) {\n            amax = fabsf(v);\n            vmax = v;\n        }\n    }\n\n    float d = vmax / kvalues_iq4nl[0];\n    const float id = d ? 1.0f/d : 0.0f;\n\n    float sumqx = 0, sumq2 = 0;\n    for (int j = 0; j < QK4_NL/2; ++j) {\n        const float x0 = x[0        + j]*id;\n        const float x1 = x[QK4_NL/2 + j]*id;\n        const uint8_t xi0 = best_index_int8(16, kvalues_iq4nl, x0);\n        const uint8_t xi1 = best_index_int8(16, kvalues_iq4nl, x1);\n        y->qs[j] = xi0 | (xi1 << 4);\n        const float v0 = kvalues_iq4nl[xi0];\n        const float v1 = kvalues_iq4nl[xi1];\n        const float w0 = x[0        + j]*x[0        + j];\n        const float w1 = x[QK4_NL/2 + j]*x[QK4_NL/2 + j];\n        sumqx += w0*v0*x[j] + w1*v1*x[QK4_NL/2 + j];\n        sumq2 += w0*v0*v0 + w1*v1*v1;\n    }\n\n    y->d = sumq2 > 0 ? sumqx/sumq2 : d;\n}\n\n// Wrapper functions for cpy.cu compatibility\nstatic __device__ void cpy_blck_f32_q4_0(const char * cxi, char * cdsti) {\n    quantize_f32_q4_0_block((const float *)cxi, (block_q4_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q4_1(const char * cxi, char * cdsti) {\n    quantize_f32_q4_1_block((const float *)cxi, (block_q4_1 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q5_0(const char * cxi, char * cdsti) {\n    quantize_f32_q5_0_block((const float *)cxi, (block_q5_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q5_1(const char * cxi, char * cdsti) {\n    quantize_f32_q5_1_block((const float *)cxi, (block_q5_1 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_q8_0(const char * cxi, char * cdsti) {\n    quantize_f32_q8_0_block((const float *)cxi, (block_q8_0 *)cdsti);\n}\n\nstatic __device__ void cpy_blck_f32_iq4_nl(const char * cxi, char * cdsti) {\n    quantize_f32_iq4_nl_block((const float *)cxi, (block_iq4_nl *)cdsti);\n}\n\ntemplate<typename src_t, typename dst_t>\nstatic __device__ void cpy_1_scalar(const char * cxi, char * cdsti) {\n    *(dst_t *) cdsti = ggml_cuda_cast<dst_t>(*(const src_t *) cxi);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cpy.cu",
    "content": "#include \"cpy.cuh\"\n#include \"dequantize.cuh\"\n#include \"cpy-utils.cuh\"\n#if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY)\n#include \"ggml-musa/mudnn.cuh\"\n#endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY\n\ntypedef void (*cpy_kernel_t)(const char * cx, char * cdst);\n\nconst int CUDA_CPY_TILE_DIM_2D = 32; // 2D tile dimension for transposed blocks\nconst int CUDA_CPY_BLOCK_NM = 8;     // block size of 3rd dimension if available\nconst int CUDA_CPY_BLOCK_ROWS = 8;   // block dimension for marching through rows\n\ntemplate <cpy_kernel_t cpy_1>\nstatic __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne,\n                                  const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n                                  const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n                                  const int64_t nb12, const int64_t nb13) {\n    const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n    if (i >= ne) {\n        return;\n    }\n\n    // determine indices i03/i13, i02/i12, i01/i11, i00/i10 as a function of index i of flattened tensor\n    // then combine those indices with the corresponding byte offsets to get the total offsets\n    const int64_t i03 = i/(ne00 * ne01 * ne02);\n    const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n    const int64_t i01 = (i - i03*ne00*ne01*ne02  -  i02*ne01*ne00) / ne00;\n    const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n    const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n    const int64_t i13 = i/(ne10 * ne11 * ne12);\n    const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n    const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n    const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n    const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13 * nb13;\n\n    cpy_1(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate <typename T>\nstatic __global__ void cpy_scalar_transpose(const char * cx, char * cdst, const int64_t ne,\n                               const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n                               const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n                               const int64_t nb12, const int64_t nb13) {\n\n    const T* src = reinterpret_cast<const T*>(cx);\n    T* dst = reinterpret_cast<T*>(cdst);\n\n    const int64_t nmat = ne / (ne00 * ne01);\n    const int64_t n = ne00 * ne01;\n\n    const int x = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.x;\n    const int y = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.y;\n    const int tx = blockIdx.y * CUDA_CPY_TILE_DIM_2D + threadIdx.x;  // transpose block offset\n    const int ty = blockIdx.x * CUDA_CPY_TILE_DIM_2D + threadIdx.y;\n\n    __shared__ float tile[2][CUDA_CPY_TILE_DIM_2D][CUDA_CPY_TILE_DIM_2D+1];\n    int cur_tile_buf = 0;\n\n#pragma unroll\n    for (int i = 0; i < CUDA_CPY_BLOCK_NM; ++i) {\n\n        const unsigned int imat = blockIdx.z * CUDA_CPY_BLOCK_NM + i;\n        if (imat >= nmat)\n            break;\n\n#pragma unroll\n        for (int j = 0; j < CUDA_CPY_TILE_DIM_2D; j += CUDA_CPY_BLOCK_ROWS) {\n            if(x < ne01 && y + j < ne00){\n                const int row = threadIdx.y+j;\n                const int col = threadIdx.x * sizeof(float)/sizeof(T);\n                T *tile2 = reinterpret_cast<T*>(tile[cur_tile_buf][row]);\n                tile2[col] = src[imat*n + (y+j)*ne01 + x];\n            }\n        }\n\n        __syncthreads();\n\n#pragma unroll\n        for (int j = 0; j < CUDA_CPY_TILE_DIM_2D; j += CUDA_CPY_BLOCK_ROWS) {\n            if (ty + j < ne01 && tx < ne00) {\n                const int col = (threadIdx.y+j)*sizeof(float)/sizeof(T);\n                const T *tile2 = reinterpret_cast<const T*>(tile[cur_tile_buf][threadIdx.x]);\n                dst[imat*n + (ty+j)*ne00 + tx] = tile2[col];\n            }\n        }\n\n        cur_tile_buf = (cur_tile_buf + 1) % 2;\n    }\n\n    GGML_UNUSED_VARS(ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11,\n        nb12, nb13);\n}\n\nstatic __device__ void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) {\n    float * cdstf = (float *)(cdsti);\n\n#pragma unroll\n    for (int j = 0; j < QK8_0; j += 2) {\n        float2 dq;\n        dequantize_q8_0(cxi, 0, j, dq);\n        *(cdstf + j) = dq.x;\n        *(cdstf + j + 1) = dq.y;\n    }\n}\n\ntemplate<dequantize_kernel_t dequant, int qk>\nstatic __device__ void cpy_blck_q_f32(const char * cxi, char * cdsti) {\n    float * cdstf = (float *)(cdsti);\n\n#pragma unroll\n    for (int j = 0; j < qk/2; j++) {\n        float2 dq;\n        dequant(cxi, 0, j, dq);\n        *(cdstf + j) = dq.x;\n        *(cdstf + j + qk/2) = dq.y;\n    }\n}\n\ntemplate <cpy_kernel_t cpy_blck, int qk>\nstatic __global__ void cpy_f32_q(const char * cx, char * cdst, const int64_t ne,\n                                 const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n                                 const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n                                 const int64_t nb12, const int64_t nb13) {\n    const int64_t i = ((int64_t)blockDim.x*blockIdx.x + threadIdx.x)*qk;\n\n    if (i >= ne) {\n        return;\n    }\n\n    const int64_t i03 = i/(ne00 * ne01 * ne02);\n    const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n    const int64_t i01 = (i - i03*ne00*ne01*ne02  -  i02*ne01*ne00) / ne00;\n    const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n    const int64_t x_offset = i00*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n    const int64_t i13 = i/(ne10 * ne11 * ne12);\n    const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n    const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n    const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n    const int64_t dst_offset = (i10/qk)*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n    cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate <cpy_kernel_t cpy_blck, int qk>\nstatic __global__ void cpy_q_f32(const char * cx, char * cdst, const int64_t ne,\n                                 const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n                                 const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11,\n                                 const int64_t nb12, const int64_t nb13) {\n    const int64_t i = ((int64_t)blockDim.x*blockIdx.x + threadIdx.x)*qk;\n\n    if (i >= ne) {\n        return;\n    }\n\n    const int64_t i03 = i/(ne00 * ne01 * ne02);\n    const int64_t i02 = (i - i03*ne00*ne01*ne02 )/ (ne00*ne01);\n    const int64_t i01 = (i - i03*ne00*ne01*ne02  -  i02*ne01*ne00) / ne00;\n    const int64_t i00 = i - i03*ne00*ne01*ne02 - i02*ne01*ne00 - i01*ne00;\n    const int64_t x_offset = (i00/qk)*nb00 + i01*nb01 + i02*nb02 + i03 * nb03;\n\n    const int64_t i13 = i/(ne10 * ne11 * ne12);\n    const int64_t i12 = (i - i13*ne10*ne11*ne12) / (ne10*ne11);\n    const int64_t i11 = (i - i13*ne10*ne11*ne12 - i12*ne10*ne11) / ne10;\n    const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10;\n    const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13;\n\n    cpy_blck(cx + x_offset, cdst + dst_offset);\n}\n\ntemplate<typename src_t, typename dst_t>\nstatic __global__ void cpy_scalar_contiguous(const char * cx, char * cdst, const int64_t ne) {\n    const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;\n\n    if (i >= ne) {\n        return;\n    }\n\n    const src_t * x = (const src_t *) cx;\n    dst_t *     dst = (dst_t *) cdst;\n\n    dst[i] = ggml_cuda_cast<dst_t>(x[i]);\n}\n\ntemplate<typename src_t, typename dst_t>\nstatic void ggml_cpy_scalar_contiguous_cuda(\n    const char * cx, char * cdst, const int64_t ne,\ncudaStream_t stream) {\n\n    const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_scalar_contiguous<src_t, dst_t><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>\n        (cx, cdst, ne);\n}\n\ntemplate<typename src_t, typename dst_t, bool transposed = false>\nstatic void ggml_cpy_scalar_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    if (transposed) {\n        GGML_ASSERT(ne == ne00*ne01*ne02);  // ne[3] is 1 assumed\n        int64_t ne00n, ne01n, ne02n;\n        if (nb00 <= nb02) { // most likely safe to handle nb00 = nb02 case here\n            ne00n = ne00;\n            ne01n = ne01;\n            ne02n = ne02;\n        } else {\n            ne00n = ne00;\n            ne01n = ne01*ne02;\n            ne02n = 1;\n        }\n\n        int64_t grid_x = (ne01n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D;\n        int64_t grid_y = (ne00n + CUDA_CPY_TILE_DIM_2D - 1) / CUDA_CPY_TILE_DIM_2D;\n        int64_t grid_z = (ne/(ne01n*ne00n) + CUDA_CPY_BLOCK_NM - 1) / CUDA_CPY_BLOCK_NM;\n        GGML_ASSERT(grid_x < UINT_MAX);\n        GGML_ASSERT(grid_y < USHRT_MAX);\n        GGML_ASSERT(grid_z < USHRT_MAX);\n        dim3 dimGrid(grid_x, grid_y, grid_z);\n        dim3 dimBlock(CUDA_CPY_TILE_DIM_2D, CUDA_CPY_BLOCK_ROWS, 1);\n        cpy_scalar_transpose<dst_t><<<dimGrid, dimBlock, 0, stream>>>\n            (cx, cdst, ne, ne00n, ne01n, ne02n, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n    } else {\n        const int64_t num_blocks = (ne + CUDA_CPY_BLOCK_SIZE - 1) / CUDA_CPY_BLOCK_SIZE;\n        GGML_ASSERT(num_blocks < UINT_MAX);\n        cpy_scalar<cpy_1_scalar<src_t, dst_t>><<<num_blocks, CUDA_CPY_BLOCK_SIZE, 0, stream>>>\n            (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n    }\n}\n\nstatic void ggml_cpy_f32_q8_0_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK8_0 == 0);\n    const int64_t num_blocks = ne / QK8_0;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_q8_0, QK8_0><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q8_0_f32_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    const int64_t num_blocks = ne;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_q_f32<cpy_blck_q8_0_f32, QK8_0><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q4_0_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK4_0 == 0);\n    const int64_t num_blocks = ne / QK4_0;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_q4_0, QK4_0><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q4_0_f32_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02,\n    const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n    const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n    cudaStream_t stream) {\n    const int64_t num_blocks = ne;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_q_f32<cpy_blck_q_f32<dequantize_q4_0, QK4_0>, QK4_0><<<num_blocks, 1, 0, stream>>>(\n        cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n         ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q4_1_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK4_1 == 0);\n    const int64_t num_blocks = ne / QK4_1;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_q4_1, QK4_1><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q4_1_f32_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02,\n    const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n    const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n    cudaStream_t stream) {\n    const int64_t num_blocks = ne;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_q_f32<cpy_blck_q_f32<dequantize_q4_1, QK4_1>, QK4_1><<<num_blocks, 1, 0, stream>>>(\n        cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n         ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q5_0_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK5_0 == 0);\n    const int64_t num_blocks = ne / QK5_0;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_q5_0, QK5_0><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q5_0_f32_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02,\n    const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n    const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n    cudaStream_t stream) {\n    const int64_t num_blocks = ne;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_q_f32<cpy_blck_q_f32<dequantize_q5_0, QK5_0>, QK5_0><<<num_blocks, 1, 0, stream>>>(\n        cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n        ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_q5_1_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK5_1 == 0);\n    const int64_t num_blocks = ne / QK5_1;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_q5_1, QK5_1><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_q5_1_f32_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02,\n    const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12,\n    const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13,\n    cudaStream_t stream) {\n    const int64_t num_blocks = ne;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_q_f32<cpy_blck_q_f32<dequantize_q5_1, QK5_1>, QK5_1><<<num_blocks, 1, 0, stream>>>(\n        cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03,\n        ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nstatic void ggml_cpy_f32_iq4_nl_cuda(\n    const char * cx, char * cdst, const int64_t ne,\n    const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02,\n    const int64_t nb03, const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t nb10, const int64_t nb11, const int64_t nb12, const int64_t nb13, cudaStream_t stream) {\n\n    GGML_ASSERT(ne % QK4_NL == 0);\n    const int64_t num_blocks = ne / QK4_NL;\n    GGML_ASSERT(num_blocks < UINT_MAX);\n    cpy_f32_q<cpy_blck_f32_iq4_nl, QK4_NL><<<num_blocks, 1, 0, stream>>>\n        (cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13);\n}\n\nvoid ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1) {\n    const int64_t ne = ggml_nelements(src0);\n    GGML_ASSERT(ne == ggml_nelements(src1));\n\n    const int64_t ne00 = src0->ne[0];\n    const int64_t ne01 = src0->ne[1];\n    const int64_t ne02 = src0->ne[2];\n\n    //GGML_ASSERT(src0->ne[3] == 1);\n\n    const int64_t nb00 = src0->nb[0];\n    const int64_t nb01 = src0->nb[1];\n    const int64_t nb02 = src0->nb[2];\n    const int64_t nb03 = src0->nb[3];\n\n    const int64_t ne10 = src1->ne[0];\n    const int64_t ne11 = src1->ne[1];\n    const int64_t ne12 = src1->ne[2];\n\n    //GGML_ASSERT(src1->ne[3] == 1);\n\n    const int64_t nb10 = src1->nb[0];\n    const int64_t nb11 = src1->nb[1];\n    const int64_t nb12 = src1->nb[2];\n    const int64_t nb13 = src1->nb[3];\n\n    cudaStream_t main_stream = ctx.stream();\n\n    char * src0_ddc = (char *) src0->data;\n    char * src1_ddc = (char *) src1->data;\n\n    const bool contiguous_srcs = ggml_is_contiguous(src0) && ggml_is_contiguous(src1);\n    const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) &&\n        src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0);\n\n    if (src0->type == src1->type && contiguous_srcs) {\n        GGML_ASSERT(ggml_nbytes(src0) == ggml_nbytes(src1));\n#if defined(GGML_USE_MUSA) && defined(GGML_MUSA_MUDNN_COPY)\n        if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16) {\n            CUDA_CHECK(mudnnMemcpyAsync(ctx, src1, src0));\n        } else\n#endif // GGML_USE_MUSA && GGML_MUSA_MUDNN_COPY\n        {\n            CUDA_CHECK(cudaMemcpyAsync(src1_ddc, src0_ddc, ggml_nbytes(src0), cudaMemcpyDeviceToDevice, main_stream));\n        }\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {\n        if (can_be_transposed) {\n            ggml_cpy_scalar_cuda<float, float, true>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<float, float>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_BF16) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<float, nv_bfloat16>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<float, nv_bfloat16>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<float, half>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<float, half>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_0) {\n        ggml_cpy_f32_q8_0_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) {\n        ggml_cpy_q8_0_f32_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_0) {\n        ggml_cpy_f32_q4_0_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_Q4_0 && src1->type == GGML_TYPE_F32) {\n        ggml_cpy_q4_0_f32_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q4_1) {\n        ggml_cpy_f32_q4_1_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_Q4_1 && src1->type == GGML_TYPE_F32) {\n        ggml_cpy_q4_1_f32_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) {\n        ggml_cpy_f32_q5_0_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {\n        ggml_cpy_q5_0_f32_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_IQ4_NL) {\n        ggml_cpy_f32_iq4_nl_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_1) {\n        ggml_cpy_f32_q5_1_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_Q5_1 && src1->type == GGML_TYPE_F32) {\n        ggml_cpy_q5_1_f32_cuda\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16) {\n        if (can_be_transposed) {\n            ggml_cpy_scalar_cuda<half, half, true>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<half, half>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_BF16) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<half, nv_bfloat16>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<half, nv_bfloat16>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<half, float>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<half, float>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16) {\n        if (can_be_transposed) {\n            ggml_cpy_scalar_cuda<nv_bfloat16, nv_bfloat16, true>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<nv_bfloat16, nv_bfloat16>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F16) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<nv_bfloat16, half>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<nv_bfloat16, half>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<nv_bfloat16, float>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<nv_bfloat16, float>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_I32) {\n        if (can_be_transposed) {\n            ggml_cpy_scalar_cuda<int32_t, int32_t, true>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<int32_t, int32_t>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<float, int32_t>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<float, int32_t>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else if (src0->type == GGML_TYPE_I32 && src1->type == GGML_TYPE_F32) {\n        if (contiguous_srcs) {\n            ggml_cpy_scalar_contiguous_cuda<int32_t, float>\n                (src0_ddc, src1_ddc, ne, main_stream);\n        } else {\n            ggml_cpy_scalar_cuda<int32_t, float>\n                (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);\n        }\n    } else {\n        GGML_ABORT(\"%s: unsupported type combination (%s to %s)\\n\", __func__,\n                ggml_type_name(src0->type), ggml_type_name(src1->type));\n    }\n}\n\nvoid ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    ggml_cuda_cpy(ctx, src0, dst);\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cpy.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CPY_BLOCK_SIZE 64\n\nvoid ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggml_tensor * src1);\n\nvoid ggml_cuda_dup(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cross-entropy-loss.cu",
    "content": "#include \"common.cuh\"\n#include \"cross-entropy-loss.cuh\"\n#include \"sum.cuh\"\n\n#include <cmath>\n#include <cstdint>\n\ntemplate <bool use_shared>\nstatic __global__ void cross_entropy_loss_f32(\n        const float * __restrict__ logits, const float * __restrict__ labels, float * __restrict__ dst, const int nclasses, const int k) {\n    extern __shared__ float tmp[];\n\n    logits += int64_t(blockIdx.x)*nclasses;\n    labels += int64_t(blockIdx.x)*nclasses;\n\n    // Find maximum for softmax:\n    float max_logit = -INFINITY;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float val = logits[i];\n        max_logit = fmaxf(max_logit, val);\n\n        if (use_shared) {\n            tmp[i] = val;\n        }\n    }\n    max_logit = warp_reduce_max(max_logit);\n\n    // Calculate log(softmax(logits)) which is just logits - max:\n    float sum = 0.0f;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float logit_i = use_shared ? tmp[i] : logits[i];\n        sum += expf(logit_i - max_logit);\n    }\n    sum = warp_reduce_sum(sum);\n    sum = logf(sum);\n\n    // log(exp(logits - max) / sum) = (logits - max) - log(sum)\n    float loss = 0.0f;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float logit_i = use_shared ? tmp[i] : logits[i];\n        loss += (logit_i - max_logit - sum) * labels[i];\n    }\n    loss = -warp_reduce_sum(loss) / (float)k;\n\n    if (threadIdx.x != 0) {\n        return;\n    }\n\n    dst[blockIdx.x] = loss;\n}\n\ntemplate <bool use_shared>\nstatic __global__ void cross_entropy_loss_back_f32(\n        const float * __restrict__ grad, const float * __restrict__ logits, const float * __restrict__ labels,\n        float * __restrict__ dst, const int nclasses) {\n    extern __shared__ float tmp[];\n\n    logits += int64_t(blockIdx.x)*nclasses;\n    labels += int64_t(blockIdx.x)*nclasses;\n    dst    += int64_t(blockIdx.x)*nclasses;\n\n    float maxval = -INFINITY;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float val = logits[i];\n        maxval = fmaxf(maxval, val);\n\n        if (use_shared) {\n            tmp[i] = val;\n        }\n    }\n    maxval = warp_reduce_max(maxval);\n\n    float sum = 0.0f;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float val = expf((use_shared ? tmp[i] : logits[i]) - maxval);\n        sum += val;\n\n        if (use_shared) {\n            tmp[i] = val;\n        } else {\n            dst[i] = val;\n        }\n    }\n    sum = warp_reduce_sum(sum);\n    const float sm_scale = 1.0f/sum;\n\n    const float d_by_nrows = *grad/gridDim.x;\n    for (int i = threadIdx.x; i < nclasses; i += WARP_SIZE) {\n        const float val = use_shared ? tmp[i] : dst[i];\n        dst[i] = (val*sm_scale - labels[i])*d_by_nrows;\n    }\n}\n\nvoid ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n    const ggml_tensor * src1 = dst->src[1];\n\n    GGML_ASSERT(src0->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1->type == GGML_TYPE_F32);\n    GGML_ASSERT( dst->type == GGML_TYPE_F32);\n\n    GGML_ASSERT(ggml_is_contiguous(src0));\n    GGML_ASSERT(ggml_is_contiguous(src1));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n\n    const int64_t ne00  = src0->ne[0];\n    const int64_t nrows = ggml_nrows(src0);\n\n    const float * src0_d = (const float *) src0->data;\n    const float * src1_d = (const float *) src1->data;\n    float       * dst_d  = (float       *) dst->data;\n\n    ggml_cuda_pool & pool = ctx.pool();\n    cudaStream_t stream = ctx.stream();\n\n    const dim3 blocks_dim(WARP_SIZE, 1, 1);\n    const dim3 blocks_num(nrows, 1, 1);\n    const size_t nbytes_shared = ne00*sizeof(float);\n\n    const int    id    = ggml_cuda_get_device();\n    const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n    ggml_cuda_pool_alloc<float> dst_tmp(pool, blocks_num.x);\n\n    if (nbytes_shared <= smpbo) {\n        CUDA_SET_SHARED_MEMORY_LIMIT((cross_entropy_loss_f32<true>), smpbo);\n        cross_entropy_loss_f32<true><<<blocks_num, blocks_dim, nbytes_shared, stream>>>(src0_d, src1_d, dst_tmp.ptr, ne00, nrows);\n    } else {\n        cross_entropy_loss_f32<false><<<blocks_num, blocks_dim, 0, stream>>>(src0_d, src1_d, dst_tmp.ptr, ne00, nrows);\n    }\n    CUDA_CHECK(cudaGetLastError());\n\n    // Combine results from individual blocks:\n    sum_f32_cuda(pool, dst_tmp.ptr, dst_d, blocks_num.x, stream);\n}\n\nvoid ggml_cuda_cross_entropy_loss_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * grad  = dst->src[0];\n    const ggml_tensor * src0f = dst->src[1];\n    const ggml_tensor * src1f = dst->src[2];\n\n    GGML_ASSERT(src0f->type == GGML_TYPE_F32);\n    GGML_ASSERT(src1f->type == GGML_TYPE_F32);\n    GGML_ASSERT( grad->type == GGML_TYPE_F32);\n    GGML_ASSERT(  dst->type == GGML_TYPE_F32);\n\n    GGML_ASSERT(ggml_is_scalar(grad));\n    GGML_ASSERT(ggml_is_contiguous(src0f));\n    GGML_ASSERT(ggml_is_contiguous(src1f));\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_are_same_shape(src0f, src1f));\n    GGML_ASSERT(ggml_are_same_shape(src0f, dst));\n\n    const int64_t ne00  = src0f->ne[0];\n    const int64_t nrows = ggml_nrows(src0f);\n\n    const float * grad_d  = (const float *) grad->data;\n    const float * src0f_d = (const float *) src0f->data;\n    const float * src1f_d = (const float *) src1f->data;\n    float       * dst_d   = (float       *) dst->data;\n\n    cudaStream_t stream = ctx.stream();\n\n    const dim3 blocks_dim(WARP_SIZE, 1, 1);\n    const dim3 blocks_num(nrows, 1, 1);\n    const size_t nbytes_shared = ne00*sizeof(float);\n\n    const int    id    = ggml_cuda_get_device();\n    const size_t smpbo = ggml_cuda_info().devices[id].smpbo;\n\n    if (nbytes_shared <= smpbo) {\n        CUDA_SET_SHARED_MEMORY_LIMIT((cross_entropy_loss_back_f32<true>), smpbo);\n        cross_entropy_loss_back_f32<true><<<blocks_num, blocks_dim, nbytes_shared, stream>>>(grad_d, src0f_d, src1f_d, dst_d, ne00);\n    } else {\n        cross_entropy_loss_back_f32<false><<<blocks_num, blocks_dim, 0, stream>>>(grad_d, src0f_d, src1f_d, dst_d, ne00);\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cross-entropy-loss.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CROSS_ENTROPY_LOSS_BLOCK_SIZE 256\n\nvoid ggml_cuda_cross_entropy_loss(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n\nvoid ggml_cuda_cross_entropy_loss_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/cumsum.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_CUMSUM_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_cumsum(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/dequantize.cuh",
    "content": "#include \"common.cuh\"\n\nstatic __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n    const block_q4_0 * x = (const block_q4_0 *) vx;\n\n    const float d = x[ib].d;\n\n    const int vui = x[ib].qs[iqs];\n\n    v.x = vui & 0xF;\n    v.y = vui >> 4;\n\n    v.x = (v.x - 8.0f) * d;\n    v.y = (v.y - 8.0f) * d;\n}\n\nstatic __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, float2 & v){\n    const block_q4_1 * x = (const block_q4_1 *) vx;\n\n    const float2 dm = __half22float2(x[ib].dm);\n\n    const int vui = x[ib].qs[iqs];\n\n    v.x = vui & 0xF;\n    v.y = vui >> 4;\n\n    v.x = (v.x * dm.x) + dm.y;\n    v.y = (v.y * dm.x) + dm.y;\n}\n\nstatic __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n    const block_q5_0 * x = (const block_q5_0 *) vx;\n\n    const float d = x[ib].d;\n\n    uint32_t qh;\n    memcpy(&qh, x[ib].qh, sizeof(qh));\n\n    const int xh_0 = ((qh >> (iqs +  0)) << 4) & 0x10;\n    const int xh_1 = ((qh >> (iqs + 12))     ) & 0x10;\n\n    v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);\n    v.y = ((x[ib].qs[iqs] >>  4) | xh_1);\n\n    v.x = (v.x - 16.0f) * d;\n    v.y = (v.y - 16.0f) * d;\n}\n\nstatic __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, float2 & v){\n    const block_q5_1 * x = (const block_q5_1 *) vx;\n\n    const float2 dm = __half22float2(x[ib].dm);\n\n    uint32_t qh;\n    memcpy(&qh, x[ib].qh, sizeof(qh));\n\n    const int xh_0 = ((qh >> (iqs +  0)) << 4) & 0x10;\n    const int xh_1 = ((qh >> (iqs + 12))     ) & 0x10;\n\n    v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);\n    v.y = ((x[ib].qs[iqs] >>  4) | xh_1);\n\n    v.x = (v.x * dm.x) + dm.y;\n    v.y = (v.y * dm.x) + dm.y;\n}\n\nstatic __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, float2 & v){\n    const block_q8_0 * x = (const block_q8_0 *) vx;\n\n    const float d = x[ib].d;\n\n    v.x = x[ib].qs[iqs + 0];\n    v.y = x[ib].qs[iqs + 1];\n\n    v.x *= d;\n    v.y *= d;\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/diag.cu",
    "content": "#include \"convert.cuh\"\n#include \"diag.cuh\"\n#include \"ggml.h\"\n\ntemplate <typename T>\nstatic __global__ void diag_kernel(T * __restrict__ dst,\n                                   const T * __restrict__ src,\n                                   const int64_t ne0,\n                                   const int64_t ne1,\n                                   const int64_t ne2,\n                                   const int64_t ne3,\n                                   const int64_t total_elements) {\n    const int64_t global_idx = blockIdx.x * blockDim.x + threadIdx.x;\n\n    if (global_idx >= total_elements) {\n        return;\n    }\n\n    const int64_t i0 = global_idx % ne0;\n    const int64_t i1 = (global_idx / ne0) % ne1;\n    const int64_t i2 = (global_idx / (ne0 * ne1)) % ne2;\n    const int64_t i3 = global_idx / (ne0 * ne1 * ne2);\n\n    const int64_t dst_idx = ((i3 * ne2 + i2) * ne1 + i1) * ne0 + i0;\n\n    if (i0 == i1) {\n        const int64_t batch_idx = i3 * ne2 + i2;\n        const int64_t src_idx   = batch_idx * ne0 + i0;\n        dst[dst_idx]            = src[src_idx];\n    } else {\n        dst[dst_idx] = ggml_cuda_cast<T>(0);\n    }\n    GGML_UNUSED_VARS(ne3);\n}\n\nvoid ggml_cuda_op_diag(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * src0 = dst->src[0];\n\n    void *       dst_d  = dst->data;\n    const void * src0_d = src0->data;\n\n    cudaStream_t stream = ctx.stream();\n\n    GGML_ASSERT(ggml_is_contiguous(dst));\n    GGML_ASSERT(ggml_is_contiguous(src0));\n\n    const int64_t ne00 = src0->ne[0];\n    const int64_t ne01 = src0->ne[1];\n    const int64_t ne02 = src0->ne[2];\n    const int64_t ne03 = src0->ne[3];\n\n    const int64_t ne0 = dst->ne[0];\n    const int64_t ne1 = dst->ne[1];\n    const int64_t ne2 = dst->ne[2];\n    const int64_t ne3 = dst->ne[3];\n\n    GGML_ASSERT(ne00 == ne0);\n    GGML_ASSERT(ne01 == 1);\n    GGML_ASSERT(ne02 == ne2);\n    GGML_ASSERT(ne03 == ne3);\n\n    const int64_t n_elems    = ggml_nelements(dst);\n    const int64_t num_blocks = (n_elems + CUDA_DIAG_BLOCK_SIZE - 1) / CUDA_DIAG_BLOCK_SIZE;\n\n    switch (dst->type) {\n        case GGML_TYPE_F32:\n            diag_kernel<<<num_blocks, CUDA_DIAG_BLOCK_SIZE, 0, stream>>>((float *) dst_d, (const float *) src0_d, ne0,\n                                                                         ne1, ne2, ne3, n_elems);\n            break;\n        case GGML_TYPE_F16:\n            diag_kernel<<<num_blocks, CUDA_DIAG_BLOCK_SIZE, 0, stream>>>((half *) dst_d, (const half *) src0_d, ne0,\n                                                                         ne1, ne2, ne3, n_elems);\n            break;\n        default:\n            GGML_ABORT(\"unsupported type\");\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/diag.cuh",
    "content": "#include \"common.cuh\"\n\n#define CUDA_DIAG_BLOCK_SIZE 256\n\nvoid ggml_cuda_op_diag(ggml_backend_cuda_context & ctx, ggml_tensor * dst);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/fattn-mma-f16.cuh",
    "content": "#include \"common.cuh\"\n#include \"cp-async.cuh\"\n#include \"mma.cuh\"\n#include \"fattn-common.cuh\"\n\nusing namespace ggml_cuda_mma;\n\n// Config options for the MMA kernel.\n// Should not affect results, only speed/register pressure/shared memory use.\nstruct fattn_mma_config {\n    int  nthreads;       // Number of threads per CUDA block.\n    int  occupancy;      // Targeted occupancy for the MMA kernel.\n    int  nbatch_fa;      // Number of KV rows per softmax rescaling of KQ rowsums and VKQ accumulators.\n    int  nbatch_K2;      // Number of K half2 values in direction of DKQ to load in parallel.\n    int  nbatch_V2;      // Number of V half2 values in direction of DV to load in parallel.\n    int  nbatch_combine; // Number of VKQ half2 values in direction of DV to combine in parallel.\n    int  nstages_target; // Number of pipeline stages to use ideally, 1 == always load data synchronously, 2 == preload data if there is hardware support.\n    bool Q_in_reg;       // Whether the Q values should be kept permanently in registers.\n\n    constexpr __host__ __device__ fattn_mma_config(\n            int nthreads, int occupancy, int nbatch_fa, int nbatch_K2, int nbatch_V2, int nbatch_combine, int nstages_target, bool Q_in_reg) :\n        nthreads(nthreads), occupancy(occupancy), nbatch_fa(nbatch_fa), nbatch_K2(nbatch_K2), nbatch_V2(nbatch_V2), nbatch_combine(nbatch_combine),\n        nstages_target(nstages_target), Q_in_reg(Q_in_reg) {}\n};\n\n#define GGML_CUDA_FATTN_MMA_CONFIG_CASE(DKQ_, DV_, ncols_, nthreads_, occupancy_, nbatch_fa_, nbatch_K2_, nbatch_V2_, nbatch_combine_, nstages_target_, Q_in_reg_) \\\n    if (DKQ == (DKQ_) && DV == (DV_) && ncols == (ncols_)) {                                                                                                       \\\n        static_assert((nthreads_)       % 32 == 0 && (nthreads_)       <= 512, \"bad nthreads\");                                                                    \\\n        static_assert(                               (occupancy_)      <=   8, \"bad occupancy\");                                                                   \\\n        static_assert((nbatch_fa_)      % 32 == 0 && (nbatch_fa_)      <= 256, \"bad nbatch_fa\");                                                                   \\\n        static_assert((nbatch_K2_)      %  4 == 0 && (nbatch_K2_)      <= 512, \"bad nbatch_K2\");                                                                   \\\n        static_assert((nbatch_V2_)      %  4 == 0 && (nbatch_V2_)      <= 256, \"bad nbatch_V2\");                                                                   \\\n        static_assert((nbatch_combine_) %  4 == 0 && (nbatch_combine_) <= 128, \"bad nbatch_combine\");                                                              \\\n        static_assert((nstages_target_)      >= 1 && (nstages_target_) <=   2, \"bad nstages_target\");                                                              \\\n        return fattn_mma_config{(nthreads_), (occupancy_), (nbatch_fa_), (nbatch_K2_), (nbatch_V2_), (nbatch_combine_), (nstages_target_), (Q_in_reg_)};           \\\n    }                                                                                                                                                              \\\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_ampere(const int DKQ, const int DV, const int ncols) {\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64,  8, 128, 2, 128,  32,  32,  32, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 16, 128, 2,  64,  32,  32,  32, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 32, 128, 2,  64,  32,  32,  32, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 64, 128, 2,  64,  32,  32,  32, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80,  8, 128, 2, 128,  40,  40,  40, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 16, 128, 2,  64,  40,  40,  40, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 32, 128, 2,  64,  40,  40,  40, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 64, 128, 2,  64,  40,  40,  40, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96,  8, 128, 2, 128,  48,  48,  48, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 16, 128, 2,  64,  48,  48,  48, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 32, 128, 2,  64,  48,  48,  48, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 64, 128, 2,  64,  48,  48,  48, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112,  8, 128, 2, 128,  56,  56,  56, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2,  64,  56,  56,  56, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2,  64,  56,  56,  56, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 128, 2,  64,  56,  56,  56, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128,  8, 128, 2, 128,  64,  64,  64, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2,  64,  64,  64,  64, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2,  64,  64,  64,  64, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 4,  64, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 4,  32, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  32, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  32, 128, 128, 128, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512,  8,  64, 4,  32, 288, 256, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16,  64, 4,  32, 288, 256, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  32, 160, 128, 128, 1, false);\n\n    return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_turing(const int DKQ, const int DV, const int ncols) {\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8, 128, 2,  64, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2,  64, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  64, 128, 128,  64, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  64, 128, 128,  64, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512,  8,  64, 4,  32,  96,  64, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16,  64, 4,  32,  96,  64, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  32, 160, 128, 128, 1, false);\n\n    return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_volta(const int DKQ, const int DV, const int ncols) {\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512,  8,  64, 4,  32, 288, 256,  64, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16,  64, 4,  32, 288, 256,  64, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128,  64, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  32, 160, 128,  64, 1, false);\n\n    // TODO tune specifically for Volta\n    return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_rdna(const int DKQ, const int DV, const int ncols) {\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 128, 2,  64, 128, 128, 128, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  64, 128, 128,  64, 2, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  64, 128, 128,  64, 2, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 16,  64, 4,  32,  96,  64, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2,  32, 160, 128, 128, 1, false);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1,  32, 160, 128, 128, 1, false);\n\n    // TODO tune specifically for RDNA\n    return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n}\n\nstatic constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config_cdna(const int DKQ, const int DV, const int ncols) {\n    // Conservative configs for CDNA (MI100+): 64KB LDS, wavefront64, nstages=1 (no cp.async).\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64,  8, 128, 2, 128,  32,  32,  32, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 16, 128, 2,  64,  32,  32,  32, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 32, 128, 2,  64,  32,  32,  32, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 64, 256, 2,  64,  32,  32,  32, 1, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80,  8, 128, 2, 128,  40,  40,  40, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 16, 128, 2,  64,  40,  40,  40, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 32, 128, 2,  64,  40,  40,  40, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80,  80, 64, 256, 2,  64,  40,  40,  40, 1, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96,  8, 128, 2, 128,  48,  48,  48, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 16, 128, 2,  64,  48,  48,  48, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 32, 128, 2,  64,  48,  48,  48, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 96,  96, 64, 256, 2,  64,  48,  48,  48, 1, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112,  8, 128, 2, 128,  56,  56,  56, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 16, 128, 2,  64,  56,  56,  56, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 32, 128, 2,  64,  56,  56,  56, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(112, 112, 64, 256, 2,  64,  56,  56,  56, 1, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128,  8, 128, 2, 128,  64,  64,  64, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 16, 128, 2,  64,  64,  64,  64, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 256, 2,  64,  64,  64,  64, 1, true);\n\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 4,  64, 128, 128, 128, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 4,  32, 128, 128, 128, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  32, 128, 128, 128, 1, true);\n    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 2,  32, 128, 128, 128, 1, true);\n\n    // Fallback for unsupported DKQ values (e.g. 576). Must return non-zero values to satisfy\n    // compile-time static_asserts even though the kernel guard prevents runtime execution.\n    // nthreads=256 gives nwarps=4 (warp_size=64) or 8 (warp_size=32), nbatch_fa=128 satisfies np*16 divisibility.\n    return fattn_mma_config(256, 1, 128, 4, 4, 4, 1, false);\n}\n\nstatic __host__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols, const int cc) {\n    if (ampere_mma_available(cc)) {\n        return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n    }\n    if (turing_mma_available(cc)) {\n        return ggml_cuda_fattn_mma_get_config_turing(DKQ, DV, ncols);\n    }\n    if (amd_mfma_available(cc)) {\n        return ggml_cuda_fattn_mma_get_config_cdna(DKQ, DV, ncols);\n    }\n    if (amd_wmma_available(cc)) {\n        return ggml_cuda_fattn_mma_get_config_rdna(DKQ, DV, ncols);\n    }\n    GGML_ASSERT(volta_mma_available(cc));\n    return ggml_cuda_fattn_mma_get_config_volta(DKQ, DV, ncols);\n}\n\nstatic constexpr __device__ fattn_mma_config ggml_cuda_fattn_mma_get_config(const int DKQ, const int DV, const int ncols) {\n#if defined(AMPERE_MMA_AVAILABLE)\n    return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);\n#elif defined(TURING_MMA_AVAILABLE)\n    return ggml_cuda_fattn_mma_get_config_turing(DKQ, DV, ncols);\n#elif defined(AMD_MFMA_AVAILABLE)\n    return ggml_cuda_fattn_mma_get_config_cdna(DKQ, DV, ncols);\n#elif defined(VOLTA_MMA_AVAILABLE)\n    return ggml_cuda_fattn_mma_get_config_volta(DKQ, DV, ncols);\n#elif defined(AMD_WMMA_AVAILABLE)\n    return ggml_cuda_fattn_mma_get_config_rdna(DKQ, DV, ncols);\n#else\n    GGML_UNUSED_VARS(DKQ, DV, ncols);\n    return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);\n#endif // defined(AMPERE_MMA_AVAILABLE)\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nthreads(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nthreads;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nthreads(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nthreads;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_occupancy(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).occupancy;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_occupancy(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).occupancy;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_fa(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_fa;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_fa(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_fa;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_K2(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_K2;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_K2(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_K2;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_V2(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_V2;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_V2(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_V2;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nbatch_combine(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nbatch_combine;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nbatch_combine(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nbatch_combine;\n}\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nstages_target(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).nstages_target;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nstages_target(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).nstages_target;\n}\n\nstatic __host__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols, const int cc) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).Q_in_reg;\n}\n\nstatic constexpr __device__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols) {\n    return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).Q_in_reg;\n}\n\nstatic constexpr __device__ int get_cols_per_thread() {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n    return 1; // AMD has a single column per thread.\n#else\n    return 2; // This is specifically KQ columns, Volta only has a single VKQ column.\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n}\n\nstatic __host__ int get_cols_per_warp(const int cc) {\n    if (turing_mma_available(cc) || amd_wmma_available(cc) || amd_mfma_available(cc)) {\n        return 16;\n    } else {\n        // Volta\n        return 32;\n    }\n}\n\n// ------------------------------------------------------------------------------------------------------------------\n\nstatic __host__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2, const int cc) {\n    return cp_async_available(cc) && ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2, cc) : 0;\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2) {\n#ifdef CP_ASYNC_AVAILABLE\n    return ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;\n#else\n    GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2);\n    return 0;\n#endif // CP_ASYNC_AVAILABLE\n}\n\n// ------------------------------------------------------------------------------------------------------------------\n\ntemplate<int stride_tile, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>\nstatic __device__ __forceinline__ void flash_attn_ext_f16_load_tile(\n        const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, const int i_sup) {\n    constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n    // K/V data is loaded with decreasing granularity for D for better memory bandwidth.\n    // The minimum granularity with cp.async is 16 bytes, with synchronous data loading it's 4 bytes.\n    if constexpr (use_cp_async) {\n        static_assert(!oob_check, \"OOB check not compatible with cp_async\");\n        constexpr int preload = 64;\n        constexpr int h2_per_chunk = 16/sizeof(half2);\n        const int chunks_per_row = D2 / h2_per_chunk;\n\n        const unsigned int tile_KV_32 = ggml_cuda_cvta_generic_to_shared(tile_KV);\n\n        auto load = [&] __device__ (auto n) {\n            const int stride_k = warp_size >> n;\n            const int k0_start = stride_k == warp_size ? 0 : chunks_per_row - chunks_per_row % (2*stride_k);\n            const int k0_stop  =                             chunks_per_row - chunks_per_row % (1*stride_k);\n            const int stride_i = warp_size / stride_k;\n\n            if (k0_start == k0_stop) {\n                return;\n            }\n\n#pragma unroll\n            for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*stride_i) {\n                const int i = i0 + threadIdx.y*stride_i + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n                if (i0 + nwarps*stride_i > nbatch_fa && i >= nbatch_fa) {\n                    break;\n                }\n\n#pragma unroll\n                for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n                    const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n                    cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i*stride_KV + k*h2_per_chunk);\n                }\n            }\n        };\n        // 1: max 32*16=512 bytes, 256 half\n        // 2: max 16*16=256 bytes, 128 half\n        // 3: max  8*16=128 bytes,  64 half\n        // 4: max  4*16= 64 bytes,  32 half\n        // 5: max  2*16= 32 bytes,  16 half\n        // 6: max  1*16= 16 bytes,   8 half\n        ggml_cuda_unroll<6>{}(load);\n    } else {\n        // TODO use ggml_cuda_memcpy_1\n        auto load = [&] __device__ (const int n) {\n            const int stride_k = warp_size >> n;\n            const int k0_start = stride_k == warp_size ? 0 : D2 - D2 % (2*stride_k);\n            const int k0_stop  =                             D2 - D2 % (1*stride_k);\n            const int stride_i = warp_size / stride_k;\n\n            if (k0_start == k0_stop) {\n                return;\n            }\n\n#pragma unroll\n            for (int i0 = 0; i0 < nbatch_fa; i0 += nwarps*stride_i) {\n                const int i = i0 + threadIdx.y*stride_i + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n                if (i0 + nwarps*stride_i > nbatch_fa && i >= nbatch_fa) {\n                    break;\n                }\n\n#pragma unroll\n                for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n                    const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n                    tile_KV[i*stride_tile + k] = !oob_check || i < i_sup ? KV[i*stride_KV + k] : make_half2(0.0f, 0.0f);\n                }\n            }\n        };\n        // 1: max 32* 4=128 bytes,  64 half\n        // 2: max 16* 4= 64 bytes,  32 half\n        // 3: max  8* 4= 32 bytes,  16 half\n        // 4: max  4* 4= 16 bytes,   8 half\n        ggml_cuda_unroll<4>{}(load);\n    }\n}\n\ntemplate<int ncols1, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>\nstatic __device__ __forceinline__ void flash_attn_ext_f16_load_mask(\n        const half * const __restrict__ mask_h, half * const __restrict__ tile_mask,\n        const int stride_mask, const int i_sup, const int j0, const uint3 ne01) {\n    constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n    if constexpr (use_cp_async) {\n        static_assert(nbatch_fa <= 8*warp_size && nbatch_fa % 8 == 0, \"bad nbatch_fa\");\n        static_assert(!oob_check, \"OOB check incompatible with cp_async\");\n        constexpr int preload = nbatch_fa >= 32 ? nbatch_fa * sizeof(half) : 64;\n        constexpr int cols_per_warp = 8*warp_size/nbatch_fa;\n        constexpr int stride_j = nwarps * cols_per_warp;\n\n        const unsigned int tile_mask_32 = ggml_cuda_cvta_generic_to_shared(tile_mask);\n\n#pragma unroll\n        for (int j1 = 0; j1 < ncols1; j1 += stride_j) {\n            const int j_sram = j1 + threadIdx.y*cols_per_warp + threadIdx.x / (warp_size/cols_per_warp);\n            const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n            if (j1 + stride_j > ncols1 && j_sram >= ncols1) {\n                break;\n            }\n\n            const int i = 8 * (threadIdx.x % (nbatch_fa/8));\n\n            cp_async_cg_16<preload>(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + j_vram*stride_mask + i);\n        }\n    } else if constexpr (oob_check) {\n#pragma unroll\n        for (int j1 = 0; j1 < ncols1; j1 += nwarps) {\n            const int j_sram = j1 + threadIdx.y;\n            const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n            if (j1 + nwarps > ncols1 && j_sram >= ncols1) {\n                break;\n            }\n\n#pragma unroll\n            for (int i0 = 0; i0 < nbatch_fa; i0 += warp_size) {\n                const int i = i0 + threadIdx.x;\n\n                tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[j_vram*stride_mask + i] : half(0.0f);\n            }\n        }\n    } else if constexpr (nbatch_fa < 2*warp_size) {\n        constexpr int cols_per_warp = 2*warp_size/nbatch_fa;\n        constexpr int stride_j = nwarps * cols_per_warp;\n#pragma unroll\n        for (int j1 = 0; j1 < ncols1; j1 += stride_j) {\n            const int j_sram = j1 + threadIdx.y*cols_per_warp + threadIdx.x / (warp_size/cols_per_warp);\n            const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n            if (j1 + stride_j > ncols1 && j_sram >= ncols1) {\n                break;\n            }\n\n            const int i = threadIdx.x % (warp_size/cols_per_warp);\n\n            ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + j_vram*stride_mask + 2*i);\n        }\n    } else {\n#pragma unroll\n        for (int j1 = 0; j1 < ncols1; j1 += nwarps) {\n            const int j_sram = j1 + threadIdx.y;\n            const int j_vram = fastmodulo(j0 + j_sram, ne01);\n\n            if (j1 + nwarps > ncols1 && j_sram >= ncols1) {\n                break;\n            }\n\n#pragma unroll\n            for (int i0 = 0; i0 < nbatch_fa; i0 += 2*warp_size) {\n                const int i = i0 + 2*threadIdx.x;\n\n                ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + j_vram*stride_mask + i);\n            }\n        }\n    }\n}\n\ntemplate<int DKQ, int DV, int ncols1, int ncols2, int nwarps,\n    bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,\n    typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ>\nstatic __device__ __forceinline__ void flash_attn_ext_f16_iter(\n        const float2 * const __restrict__ Q_f2,\n        const half2  * const __restrict__ K_h2,\n        const half2  * const __restrict__ V_h2,\n        const half   * const __restrict__ mask_h,\n        float2       * const __restrict__ dstk,\n        float2       * const __restrict__ dstk_fixup,\n        const float scale,\n        const float slope,\n        const float logit_softcap,\n        const uint3 ne01,\n        const int ne02,\n        const int stride_K,\n        const int stride_V,\n        const int stride_mask,\n        half2        * const __restrict__ tile_Q,\n        half2        * const __restrict__ tile_K,\n        half2        * const __restrict__ tile_V,\n        half         * const __restrict__ tile_mask,\n        T_B_KQ       * const __restrict__ Q_B,\n        T_C_VKQ      * const __restrict__ VKQ_C,\n        float        * const __restrict__ KQ_max,\n        float        * const __restrict__ KQ_rowsum,\n        const int jt,\n        const int kb0,\n        const int k_VKQ_sup) {\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n    constexpr int  warp_size       = ggml_cuda_get_physical_warp_size();\n    constexpr int  ncols           = ncols1 * ncols2;\n    constexpr int  cols_per_warp   = T_B_KQ::I;\n    constexpr int  cols_per_thread = get_cols_per_thread();\n    constexpr int  np              = cols_per_warp > ncols ? nwarps : nwarps * cols_per_warp/ncols; // Number of parallel CUDA warps per Q column.\n    constexpr int  nbatch_fa       = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols);\n    constexpr int  nbatch_K2       = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols);\n    constexpr int  nbatch_V2       = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols);\n    constexpr bool Q_in_reg        = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);\n    constexpr int  nstages         = ggml_cuda_fattn_mma_get_nstages  (DKQ, DV, ncols1, ncols2);\n\n    constexpr int stride_tile_Q = DKQ/2     + 4;\n    constexpr int stride_tile_K = nbatch_K2 + 4;\n\n    constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;\n\n    const int k_VKQ_0 = kb0 * nbatch_fa;\n#if defined(TURING_MMA_AVAILABLE)\n    T_C_KQ KQ_C[nbatch_fa/(np*(cols_per_warp == 8 ? T_C_KQ::I : T_C_KQ::J))];\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n    T_C_KQ KQ_C[nbatch_fa/(np*T_C_KQ::J)];\n#else // Volta\n    T_C_KQ KQ_C[nbatch_fa/(np*T_C_KQ::J)];\n#endif // defined(TURING_MMA_AVAILABLE)\n\n    if constexpr (nstages > 1) {\n        static_assert(!oob_check, \"OOB check incompatible with multi-stage pipeline\");\n        static_assert(!V_is_K_view, \"K data reuse not implemented multi-stage loading\");\n        static_assert(nbatch_K2 == DKQ/2, \"batching not implemented for multi stage loading\");\n        constexpr bool use_cp_async = true;\n        cp_async_wait_all();\n        __syncthreads();\n        flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check>\n            (V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup);\n    } else {\n        constexpr bool use_cp_async = nstages == 1;\n        if (ncols2 > 1 || mask_h) {\n            flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>\n                (mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n        }\n    }\n\n    // For MLA K and V have the same data.\n    // Therefore, iterate over K in reverse and later re-use the data if possible.\n#pragma unroll\n    for (int k0_start = (DKQ/2-1) - (DKQ/2-1) % nbatch_K2; k0_start >= 0; k0_start -= nbatch_K2) {\n        const int k0_stop = k0_start + nbatch_K2 < DKQ/2 ? k0_start + nbatch_K2 : DKQ/2;\n        const int k0_diff = k0_stop - k0_start;\n\n        if constexpr (nstages <= 1) {\n            constexpr bool use_cp_async = nstages == 1;\n            flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>\n                (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup);\n            if (use_cp_async) {\n                cp_async_wait_all();\n            }\n            __syncthreads();\n        }\n\n        // Calculate tile of KQ:\n        if constexpr (Q_in_reg) {\n#pragma unroll\n            for (int i_KQ_00 = 0; i_KQ_00 < nbatch_fa; i_KQ_00 += np*T_A_KQ::I) {\n                const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I;\n#pragma unroll\n                for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {\n                    T_A_KQ K_A;\n                    load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);\n                    if constexpr (cols_per_warp == 8) {\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]);\n                    } else {\n                        // Wide version of KQ_C is column-major\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                        // AMD matrix C is column-major.\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[k_KQ_0/T_A_KQ::J]);\n#else\n                        // swap A and B for CUDA.\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], Q_B[k_KQ_0/T_A_KQ::J], K_A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    }\n                }\n            }\n        } else {\n#pragma unroll\n            for (int k_KQ_0 = k0_start; k_KQ_0 < k0_stop; k_KQ_0 += T_A_KQ::J) {\n                load_ldmatrix(Q_B[0], tile_Q + (threadIdx.y / np)*(T_B_KQ::I*stride_tile_Q) + k_KQ_0, stride_tile_Q);\n\n#pragma unroll\n                for (int i_KQ_00 = 0; i_KQ_00 < nbatch_fa; i_KQ_00 += np*T_A_KQ::I) {\n                    const int i_KQ_0 = i_KQ_00 + (threadIdx.y % np)*T_A_KQ::I;\n\n                    T_A_KQ K_A;\n                    load_ldmatrix(K_A, tile_K + i_KQ_0*stride_tile_K + (k_KQ_0 - k0_start), stride_tile_K);\n\n                    if constexpr (cols_per_warp == 8) {\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]);\n                    } else {\n                        // Wide version of KQ_C is column-major\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                        // AMD matrix C is column-major.\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], K_A, Q_B[0]);\n#else\n                        // swap A and B for CUDA.\n                        mma(KQ_C[i_KQ_00/(np*T_A_KQ::I)], Q_B[0], K_A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    }\n                }\n            }\n        }\n\n        if constexpr (nstages <= 1) {\n            __syncthreads(); // Only needed if tile_K == tile_V.\n        }\n    }\n\n    if (use_logit_softcap) {\n        constexpr int stride = cols_per_warp == 8 ? np*T_C_KQ::I : np*T_C_KQ::J;\n        static_assert(nbatch_fa % stride == 0, \"bad loop size\");\n#pragma unroll\n        for (int i = 0; i < nbatch_fa/stride; ++i) {\n#pragma unroll\n            for (int l = 0; l < T_C_KQ::ne; ++l) {\n                KQ_C[i].x[l] = logit_softcap*tanhf(KQ_C[i].x[l]);\n            }\n        }\n    }\n\n    float KQ_max_new[cols_per_thread];\n#pragma unroll\n    for (int col = 0; col < cols_per_thread; ++col) {\n        KQ_max_new[col] = KQ_max[col];\n    }\n    float KQ_rowsum_add[cols_per_thread] = {0.0f};\n\n    if constexpr (cols_per_warp == 8) {\n        if (ncols2 > 1 || mask_h) {\n#pragma unroll\n            for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::I) {\n                const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::I;\n#pragma unroll\n                for (int l = 0; l < T_C_KQ::ne; ++l) {\n                    const int i = i0 + T_C_KQ::get_i(l);\n                    const int j = ((threadIdx.y / np)*T_C_KQ::J + T_C_KQ::get_j(l)) / ncols2;\n\n                    KQ_C[i00/(np*T_C_KQ::I)].x[l] += slope * __half2float(tile_mask[j*(nbatch_fa + 8) + i]);\n                }\n            }\n        }\n\n        // Calculate softmax for each KQ column using the current max. value.\n        // The divisor is stored in KQ_rowsum and will be applied at the end.\n        static_assert(nbatch_fa % (np*T_C_KQ::I) == 0, \"bad loop size\");\n#pragma unroll\n        for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::I) {\n#pragma unroll\n            for (int l = 0; l < T_C_KQ::ne; ++l) {\n                if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::I + T_C_KQ::get_i(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    constexpr int KQ_idx = 0;\n#else\n                    // Turing + Volta:\n                    const int KQ_idx = l % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    KQ_max_new[KQ_idx] = fmaxf(KQ_max_new[KQ_idx], KQ_C[k0/(np*T_C_KQ::I)].x[l] + FATTN_KQ_MAX_OFFSET);\n                }\n            }\n        }\n\n        // Values per KQ column are spread across 8 threads:\n#pragma unroll\n        for (int col = 0; col < cols_per_thread; ++col) {\n#pragma unroll\n            for (int offset = 16; offset >= 4; offset >>= 1) {\n                KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));\n            }\n        }\n\n        static_assert(nbatch_fa % (np*T_C_KQ::I) == 0, \"bad loop size\");\n#pragma unroll\n        for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::I) {\n#pragma unroll\n            for (int l = 0; l < T_C_KQ::ne; ++l) {\n                if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::I + T_C_KQ::get_i(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    constexpr int KQ_idx = 0;\n#else\n                    // Turing + Volta:\n                    const int KQ_idx = l % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    KQ_C[k0/(np*T_C_KQ::I)].x[l] = expf(KQ_C[k0/(np*T_C_KQ::I)].x[l] - KQ_max_new[KQ_idx]);\n                    KQ_rowsum_add[KQ_idx] += KQ_C[k0/(np*T_C_KQ::I)].x[l];\n                } else {\n                    KQ_C[k0/(np*T_C_KQ::I)].x[l] = 0.0f;\n                }\n            }\n        }\n    } else { // not Turing mma or T_B_KQ::I > 8\n        if (ncols2 > 1 || mask_h) {\n#pragma unroll\n            for (int i00 = 0; i00 < nbatch_fa; i00 += np*T_C_KQ::J) {\n                const int i0 = i00 + (threadIdx.y % np)*T_C_KQ::J;\n#pragma unroll\n                for (int l0 = 0; l0 < T_C_KQ::ne; l0 += 2) {\n                    const int i = (i0 + T_C_KQ::get_j(l0)) / 2;\n                    const int j = ((threadIdx.y / np)*cols_per_warp + T_C_KQ::get_i(l0)) / ncols2;\n\n                    const float2 tmp = __half22float2(((const half2 *)tile_mask)[j*(nbatch_fa/2 + 4) + i]);\n                    KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 0] += slope*tmp.x;\n                    KQ_C[i00/(np*T_C_KQ::J)].x[l0 + 1] += slope*tmp.y;\n                }\n            }\n        }\n\n        // Calculate softmax for each KQ column using the current max. value.\n        // The divisor is stored in KQ_rowsum and will be applied at the end.\n        static_assert(nbatch_fa % (np*T_C_KQ::J) == 0, \"bad loop size\");\n#pragma unroll\n        for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::J) {\n#pragma unroll\n            for (int l = 0; l < T_C_KQ::ne; ++l) {\n                if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::J + T_C_KQ::get_j(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    constexpr int KQ_idx = 0;\n#else\n                    // Turing + Volta:\n                    const int KQ_idx = (l/2) % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    KQ_max_new[KQ_idx] = fmaxf(KQ_max_new[KQ_idx], KQ_C[(k0/(np*T_C_KQ::J))].x[l] + FATTN_KQ_MAX_OFFSET);\n                }\n            }\n        }\n\n#pragma unroll\n        for (int col = 0; col < cols_per_thread; ++col) {\n#if defined(TURING_MMA_AVAILABLE)\n            // Values per KQ column are spread across 4 threads:\n            constexpr int offset_first = 2;\n            constexpr int offset_last  = 1;\n#elif defined(AMD_MFMA_AVAILABLE)\n            // MFMA: 4 threads per Q column (threadIdx.x % 16 == col, spaced by 16).\n            constexpr int offset_first = 32;\n            constexpr int offset_last  = 16;\n#elif defined(AMD_WMMA_AVAILABLE)\n            // Values per KQ column are spread across 2 threads:\n            constexpr int offset_first = 16;\n            constexpr int offset_last  = 16;\n#else // Volta\n            // Values per KQ column are spread across 2 threads:\n            constexpr int offset_first = 2;\n            constexpr int offset_last  = 2;\n#endif // defined(TURING_MMA_AVAILABLE)\n#pragma unroll\n            for (int offset = offset_first; offset >= offset_last; offset >>= 1) {\n                KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));\n            }\n        }\n\n        static_assert(nbatch_fa % (np*T_C_KQ::J) == 0, \"bad loop size\");\n#pragma unroll\n        for (int k0 = 0; k0 < nbatch_fa; k0 += np*T_C_KQ::J) {\n#pragma unroll\n            for (int l = 0; l < T_C_KQ::ne; ++l) {\n                if (!oob_check || k0 + (threadIdx.y % np)*T_C_KQ::J + T_C_KQ::get_j(l) < k_VKQ_sup) {\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    constexpr int KQ_idx = 0;\n#else\n                    // Turing + Volta:\n                    const int KQ_idx = (l/2) % 2;\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    KQ_C[(k0/(np*T_C_KQ::J))].x[l] = expf(KQ_C[(k0/(np*T_C_KQ::J))].x[l] - KQ_max_new[KQ_idx]);\n                    KQ_rowsum_add[KQ_idx] += KQ_C[(k0/(np*T_C_KQ::J))].x[l];\n                } else {\n                    KQ_C[(k0/(np*T_C_KQ::J))].x[l] = 0.0f;\n                }\n            }\n        }\n    }\n\n    {\n        float KQ_max_scale[cols_per_thread];\n#pragma unroll\n        for (int col = 0; col < cols_per_thread; ++col) {\n            const float KQ_max_diff = KQ_max[col] - KQ_max_new[col];\n            KQ_max_scale[col] = expf(KQ_max_diff);\n            KQ_max[col] = KQ_max_new[col];\n\n            *((uint32_t *) &KQ_max_scale[col]) *= KQ_max_diff >= SOFTMAX_FTZ_THRESHOLD;\n\n            // Scale previous KQ_rowsum to account for a potential increase in KQ_max:\n            KQ_rowsum[col] = KQ_max_scale[col]*KQ_rowsum[col] + KQ_rowsum_add[col];\n        }\n\n#if defined(TURING_MMA_AVAILABLE)\n        if constexpr (cols_per_warp == 8) {\n            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]);\n#pragma unroll\n            for (int i = 0; i < DV/T_C_VKQ::I; ++i) {\n#pragma unroll\n                for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                    VKQ_C[i].x[l] *= KQ_max_scale_h2;\n                }\n            }\n        } else {\n#pragma unroll\n            for (int col = 0; col < cols_per_thread; ++col) {\n                const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n                for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n                    for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) {\n                        VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2;\n                    }\n                }\n            }\n        }\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n        const half2 KQ_max_scale_h2 = make_half2(\n            KQ_max_scale[0], KQ_max_scale[0]);\n#pragma unroll\n        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n            for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                VKQ_C[i].x[l] *= KQ_max_scale_h2;\n            }\n        }\n#else // Volta\n        const half2 KQ_max_scale_h2 = make_half2(\n            KQ_max_scale[(threadIdx.x / 2) % 2], KQ_max_scale[(threadIdx.x / 2) % 2]);\n#pragma unroll\n        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n            for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                VKQ_C[i].x[l] *= KQ_max_scale_h2;\n            }\n        }\n#endif // defined(TURING_MMA_AVAILABLE)\n    }\n\n    // Convert KQ C tiles into B tiles for VKQ calculation:\n    T_B_VKQ B[nbatch_fa/(np*2*T_B_VKQ::J)];\n    static_assert(nbatch_fa % (np*2*T_B_VKQ::J) == 0, \"bad loop size\");\n    if constexpr (cols_per_warp == 8) {\n#pragma unroll\n        for (int k = 0; k < nbatch_fa/(np*2*T_B_VKQ::J); ++k) {\n            B[k] = get_transposed(get_half2(KQ_C[k]));\n        }\n    } else {\n        for (int k = 0; k < nbatch_fa/(np*2*T_B_VKQ::J); ++k) {\n            B[k] = get_half2(KQ_C[k]);\n        }\n    }\n\n    if constexpr (nstages > 1) {\n        static_assert(!V_is_K_view, \"K data reuse not implemented multi-stage loading\");\n        // Preload K tile for next iteration:\n        constexpr bool use_cp_async = true;\n        cp_async_wait_all();\n        __syncthreads();\n        if (!last_iter) {\n            if (ncols2 > 1 || mask_h) {\n                flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>\n                    (mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n            }\n            flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>\n                (K_h2 + int64_t(k_VKQ_0 + nbatch_fa)*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);\n        }\n    }\n\n\n#if defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE)\n    T_A_VKQ A_identity;\n    make_identity_mat(A_identity);\n#endif // defined(AMD_WMMA_AVAILABLE) && !defined(LDMATRIX_TRANS_AVAILABLE)\n\n    // Calculate VKQ tile, need to use logical rather than physical elements for i0 due to transposition of V:\n#pragma unroll\n    for (int i0_start = 0; i0_start < DV; i0_start += 2*nbatch_V2) {\n        static_assert(DV % (2*nbatch_V2) == 0, \"bad loop size\");\n        const int i0_stop = i0_start + 2*nbatch_V2;\n        const int i0_diff = i0_stop - i0_start;\n\n        if constexpr (nstages <= 1) {\n            if (!V_is_K_view || i0_stop > 2*nbatch_K2) {\n                constexpr bool use_cp_async = nstages == 1;\n                flash_attn_ext_f16_load_tile<stride_tile_V, nwarps, nbatch_fa, use_cp_async, oob_check>\n                    (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup);\n                if (use_cp_async) {\n                    cp_async_wait_all();\n                }\n                __syncthreads();\n            }\n        }\n        const half2 * tile_V_i = !V_is_K_view || i0_stop > 2*nbatch_K2 ? tile_V : tile_V + i0_start/2;\n\n#if defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n        constexpr int i0_stride = cols_per_warp == 8 ? T_C_VKQ::I : 2*T_C_VKQ::J;\n#pragma unroll\n        for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) {\n            static_assert((nbatch_fa/2) % (np*T_A_VKQ::J) == 0, \"bad loop size\");\n#pragma unroll\n            for (int k00 = 0; k00 < nbatch_fa/2; k00 += np*T_A_VKQ::J) {\n                const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::J;\n\n                T_A_VKQ A; // Transposed in SRAM but not in registers, gets transposed on load.\n#if defined(LDMATRIX_TRANS_AVAILABLE)\n                load_ldmatrix_trans(A, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n#elif defined(AMD_MFMA_AVAILABLE)\n                // MFMA A register layout: A_mat[i=lane%16][k=4*(lane/16)+reg].\n                // Normal load gives A_mat[seq][dv] but we need A_mat[dv][seq] = V^T.\n                // Load with transposed addressing: 4 strided half loads.\n                {\n                    const half2 * xs0 = tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2;\n                    const half * xs0_h = (const half *) xs0;\n                    const int stride_h = stride_tile_V * 2; // stride in half units\n                    half * A_h = (half *) A.x;\n#pragma unroll\n                    for (int l = 0; l < 4; ++l) {\n                        A_h[l] = xs0_h[(4*(threadIdx.x / 16) + l) * stride_h + threadIdx.x % 16];\n                    }\n                }\n#else\n                // TODO: Try to transpose tile_V when loading gmem to smem.\n                // Use mma to transpose T_A_VKQ for RDNA.\n                T_A_VKQ A_trans;\n                load_ldmatrix(A_trans, tile_V_i + 2*k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n                mma(A, A_trans, A_identity);\n#endif // defined(LDMATRIX_TRANS_AVAILABLE)\n                if constexpr (T_B_KQ::I == 8) {\n                    mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);\n                } else {\n                    // Wide version of VKQ_C is column-major.\n#if defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                    // AMD matrix C is column-major.\n                    mma(VKQ_C[i_VKQ_0/i0_stride], A, B[k00/(np*T_A_VKQ::J)]);\n#else\n                    // swap A and B for CUDA.\n                    mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::J)], A);\n#endif // defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n                }\n            }\n        }\n#else // Volta\n        constexpr int i0_stride = 2*T_C_VKQ::J;\n#pragma unroll\n        for (int i_VKQ_0 = i0_start; i_VKQ_0 < i0_stop; i_VKQ_0 += i0_stride) {\n            static_assert(nbatch_fa % (np*T_A_VKQ::I) == 0, \"bad loop size\");\n            static_assert(2*T_B_VKQ::J == T_A_VKQ::I, \"bad tile sizes\");\n#pragma unroll\n            for (int k00 = 0; k00 < nbatch_fa; k00 += np*T_A_VKQ::I) {\n                const int k0 = k00 + (threadIdx.y % np)*T_A_VKQ::I;\n\n                T_A_VKQ A; // Transposed in both SRAM and registers, load normally.\n                load_ldmatrix(A, tile_V_i + k0*stride_tile_V + (i_VKQ_0 - i0_start)/2, stride_tile_V);\n                mma(VKQ_C[i_VKQ_0/i0_stride], B[k00/(np*T_A_VKQ::I)], A);\n            }\n        }\n#endif // defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n\n        if constexpr (nstages <= 1) {\n            __syncthreads(); // Only needed if tile_K == tile_V.\n        }\n    }\n#else\n    GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup,\n        scale, slope, logit_softcap, ne01, ne02,\n        stride_K, stride_V, stride_mask,\n        tile_Q, tile_K, tile_V, tile_mask,\n        Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);\n    NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n}\n\n#if defined(TURING_MMA_AVAILABLE)\ntemplate<int ncols> struct mma_tile_sizes {\n    using T_A_KQ  = tile<16,  8, half2>; // row-major\n    using T_B_KQ  = tile<16,  8, half2>; // column-major\n    using T_C_KQ  = tile<16, 16, float>; // column-major\n    using T_A_VKQ = tile<16,  8, half2>; // row-major\n    using T_B_VKQ = tile<16,  8, half2>; // column-major\n    using T_C_VKQ = tile<16,  8, half2>; // column-major\n};\ntemplate<> struct mma_tile_sizes<8> {\n    using T_A_KQ  = tile<16,  8, half2>; // row-major\n    using T_B_KQ  = tile< 8,  8, half2>; // column-major\n    using T_C_KQ  = tile<16,  8, float>; // row-major\n    using T_A_VKQ = tile<16,  8, half2>; // row-major\n    using T_B_VKQ = tile< 8,  8, half2>; // column-major\n    using T_C_VKQ = tile<16,  4, half2>; // row-major\n};\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\ntemplate<int ncols> struct mma_tile_sizes {\n    using T_A_KQ  = tile<16,  8, half2>; // row-major\n    using T_B_KQ  = tile<16,  8, half2>; // column-major\n    using T_C_KQ  = tile<16, 16, float>; // column-major\n    using T_A_VKQ = tile<16,  8, half2>; // row-major\n    using T_B_VKQ = tile<16,  8, half2>; // column-major\n    using T_C_VKQ = tile<16,  8, half2>; // column-major\n};\n#else // Volta\ntemplate<int ncols> struct mma_tile_sizes {\n    using T_A_KQ  = tile< 8,  4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED>; // row-major\n    using T_B_KQ  = tile<32,  4, half2, DATA_LAYOUT_I_MAJOR>;          // column-major\n    using T_C_KQ  = tile<32,  8, float, DATA_LAYOUT_I_MAJOR>;          // column-major\n    using T_A_VKQ = tile< 8,  4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED>; // column-major\n    using T_B_VKQ = tile<32,  4, half2, DATA_LAYOUT_I_MAJOR>;          // column-major\n    using T_C_VKQ = tile<32,  4, half2, DATA_LAYOUT_I_MAJOR>;          // column-major\n};\n#endif // defined(TURING_MMA_AVAILABLE)\n\ntemplate<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup>\nstatic __device__ __forceinline__ void flash_attn_ext_f16_process_tile(\n        const float2 * const __restrict__ Q_f2,\n        const half2  * const __restrict__ K_h2,\n        const half2  * const __restrict__ V_h2,\n        const half   * const __restrict__ mask_h,\n        const float  * const __restrict__ sinks_f,\n        float2       * const __restrict__ dstk,\n        float2       * const __restrict__ dstk_fixup,\n        const float scale,\n        const float slope,\n        const float logit_softcap,\n        const uint3 ne01,\n        const int ne02,\n        const int gqa_ratio,\n        const int ne11,\n        const int stride_Q1,\n        const int stride_Q2,\n        const int stride_K,\n        const int stride_V,\n        const int stride_mask,\n        const int jt,\n        const int zt_gqa,\n        const int kb0_start,\n        const int kb0_stop) {\n#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n    //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n    constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n    constexpr int ncols = ncols1 * ncols2;\n    using     T_A_KQ    = typename mma_tile_sizes<ncols>::T_A_KQ;\n    using     T_B_KQ    = typename mma_tile_sizes<ncols>::T_B_KQ;\n    using     T_C_KQ    = typename mma_tile_sizes<ncols>::T_C_KQ;\n    using     T_A_VKQ   = typename mma_tile_sizes<ncols>::T_A_VKQ;\n    using     T_B_VKQ   = typename mma_tile_sizes<ncols>::T_B_VKQ;\n    using     T_C_VKQ   = typename mma_tile_sizes<ncols>::T_C_VKQ;\n\n    constexpr int  cols_per_warp   = T_B_KQ::I;\n    constexpr int  cols_per_thread = get_cols_per_thread();\n    constexpr int  np              = cols_per_warp > ncols ? nwarps : nwarps * cols_per_warp/ncols; // Number of parallel CUDA warps per Q column.\n    constexpr int  nbatch_fa       = ggml_cuda_fattn_mma_get_nbatch_fa     (DKQ, DV, ncols);\n    constexpr int  nbatch_K2       = ggml_cuda_fattn_mma_get_nbatch_K2     (DKQ, DV, ncols);\n    constexpr int  nbatch_V2       = ggml_cuda_fattn_mma_get_nbatch_V2     (DKQ, DV, ncols);\n    constexpr int  nbatch_combine  = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols);\n    constexpr bool Q_in_reg        = ggml_cuda_fattn_mma_get_Q_in_reg      (DKQ, DV, ncols);\n    constexpr int  nstages         = ggml_cuda_fattn_mma_get_nstages       (DKQ, DV, ncols1, ncols2);\n\n    if (cols_per_warp > ncols) {\n        NO_DEVICE_CODE;\n        return;\n    }\n\n    static_assert(nwarps * (cols_per_warp/ncols2) % ncols1 == 0, \"bad nwarps\");\n\n    constexpr int stride_tile_Q = DKQ/2     + 4;\n    constexpr int stride_tile_K = nbatch_K2 + 4;\n\n    constexpr int stride_tile_V = V_is_K_view ? stride_tile_K : nbatch_V2 + 4;\n    constexpr int stride_tile_KV_max = stride_tile_K > stride_tile_V ? stride_tile_K : stride_tile_V;\n\n    extern __shared__ half2 tile_Q[];\n    half2 * tile_K    = Q_in_reg              ? tile_Q                             : tile_Q + ncols     * stride_tile_Q;\n    half2 * tile_V    =           nstages > 1 ? tile_K + nbatch_fa * stride_tile_K : tile_K;\n    half  * tile_mask = (half *) (nstages > 1 ? tile_V + nbatch_fa * stride_tile_V : tile_V + nbatch_fa * stride_tile_KV_max);\n\n    T_B_KQ    Q_B[(Q_in_reg ? DKQ/(2*T_B_KQ::J) : 1)];\n#if defined(TURING_MMA_AVAILABLE)\n    T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n    T_C_VKQ VKQ_C[                                     DV/(2*T_C_VKQ::J)];\n#else // Volta\n    T_C_VKQ VKQ_C[                                     DV/(2*T_C_VKQ::J)];\n#endif // defined(TURING_MMA_AVAILABLE)\n\n    float KQ_rowsum[cols_per_thread] = {0.0f};\n    float KQ_max[cols_per_thread];\n#pragma unroll\n    for (int col = 0; col < cols_per_thread; ++col) {\n        KQ_max[col] = -FLT_MAX/2.0f;\n    }\n\n    // Load Q data into tile_Q, either temporarily or permanently.\n    // Q in registers is faster, but register pressure is the biggest bottleneck.\n    // The loading is done with decreasing granularity for D for better memory bandwidth.\n    const half2 scale_h2 = make_half2(scale, scale);\n#pragma unroll\n    for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) {\n        const int k0_start  = stride_k == warp_size ? 0 : DKQ/2 - (DKQ/2) % (2*stride_k);\n        const int k0_stop   =                             DKQ/2 - (DKQ/2) % (1*stride_k);\n        const int stride_jc = warp_size / stride_k;\n\n        if (k0_start == k0_stop) {\n            continue;\n        }\n\n#pragma unroll\n        for (int jc0 = 0; jc0 < ncols; jc0 += nwarps*stride_jc) {\n            const int jc = jc0 + threadIdx.y*stride_jc + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n            if (jc0 + nwarps*stride_jc > ncols && jc >= ncols) {\n                break;\n            }\n\n            const int j = jc / ncols2;\n            const int c = jc % ncols2;\n\n            if ((ncols1 == 1 || jt*ncols1 + j < int(ne01.z)) && (ncols2 == 1 || zt_gqa*ncols2 + c < gqa_ratio)) {\n#pragma unroll\n                for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n                    const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n                    const float2 tmp = Q_f2[(jt*ncols1 + j)*stride_Q1 + c*stride_Q2 + k];\n                    tile_Q[jc*stride_tile_Q + k] = scale_h2 * make_half2(tmp.x, tmp.y);\n                }\n            } else {\n#pragma unroll\n                for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n                    const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n                    tile_Q[jc*stride_tile_Q + k] = make_half2(0.0f, 0.0f);\n                }\n            }\n        }\n    }\n\n    __syncthreads();\n\n    if (Q_in_reg) {\n        const int j0 = (threadIdx.y / np) * cols_per_warp;\n\n#pragma unroll\n        for (int k0 = 0; k0 < DKQ/2; k0 += T_B_KQ::J) {\n            load_ldmatrix(Q_B[k0/T_B_KQ::J], tile_Q + j0*stride_tile_Q + k0, stride_tile_Q);\n        }\n    }\n\n    __syncthreads();\n\n    int kb0 = kb0_start;\n\n    // Preload mask and K data for first iteration when using cp_async with multiple stages:\n    if constexpr (nstages > 1) {\n        static_assert(nbatch_K2 == DKQ/2, \"batching not implemented for multi-stage pipeline\");\n        constexpr bool use_cp_async = true;\n        constexpr bool oob_check    = false;\n        constexpr int  k_VKQ_sup    = nbatch_fa;\n        if (ncols2 > 1 || mask_h) {\n            flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>\n                (mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);\n        }\n        flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check>\n            (K_h2 + int64_t(kb0)*nbatch_fa*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);\n    }\n\n    // kb0_start is always < kb0_stop so the last iter can be executed unconditionally.\n    if constexpr (ncols2 == 1) {\n        constexpr bool oob_check = true;\n        for (; kb0 < kb0_stop-1; ++kb0) {\n            constexpr bool last_iter = false;\n            constexpr int  k_VKQ_sup = nbatch_fa;\n            flash_attn_ext_f16_iter\n                <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,\n                 T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>\n                (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n                 ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n                 KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n        }\n        constexpr bool last_iter = true;\n        const     int  k_VKQ_sup = ne11 - kb0*nbatch_fa;\n        flash_attn_ext_f16_iter\n            <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,\n              T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>\n            (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n             ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n             KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n    } else {\n        constexpr bool oob_check = false;\n        for (; kb0 < kb0_stop-1; ++kb0) {\n            constexpr bool last_iter = false;\n            constexpr int  k_VKQ_sup = nbatch_fa;\n            flash_attn_ext_f16_iter\n                <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,\n                 T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>\n                (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n                 ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n                 KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n        }\n        constexpr bool last_iter = true;\n        constexpr int  k_VKQ_sup = nbatch_fa;\n        flash_attn_ext_f16_iter\n            <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,\n             T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>\n            (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,\n             ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,\n             KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);\n    }\n\n    // With multi-stage loading there is no __syncthreads at the end of the iter,\n    //     there can be a race condition on shared memory access for combining/writing back results.\n    if constexpr (nstages > 1 && nwarps*cols_per_warp > nbatch_fa) {\n        __syncthreads();\n    }\n\n    // Finally, sum up partial KQ rowsums.\n    {\n#if defined(TURING_MMA_AVAILABLE)\n        // The partial sums are spread across 8/4 threads.\n        constexpr int offset_first = cols_per_warp == 8 ? 16 : 2;\n        constexpr int offset_last  = cols_per_warp == 8 ?  4 : 1;\n#elif defined(AMD_MFMA_AVAILABLE)\n        // The partial sums are spread across 4 threads (wavefront64, 16 cols).\n        constexpr int offset_first = 32;\n        constexpr int offset_last  = 16;\n#elif defined(AMD_WMMA_AVAILABLE)\n        // The partial sums are spread across 2 threads.\n        constexpr int offset_first = 16;\n        constexpr int offset_last  = 16;\n#else // Volta\n        // The partial sums are spread across 2 threads.\n        constexpr int offset_first = 2;\n        constexpr int offset_last  = 2;\n#endif // defined(TURING_MMA_AVAILABLE)\n#pragma unroll\n        for (int col = 0; col < cols_per_thread; ++col) {\n#pragma unroll\n            for (int offset = offset_first; offset >= offset_last; offset >>= 1) {\n                KQ_rowsum[col] += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum[col], offset, warp_size);\n            }\n        }\n    }\n\n    // If attention sinks are used, potentially re-scale if KQ_max is small.\n    // Also add the sink as a value to KQ_rowsum, this is done after synchronization of KQ_rowsum\n    //     so it's being done unconditionally for every thread.\n    if (!is_fixup && (np == 1 || threadIdx.y % np == 0) && sinks_f) {\n        float KQ_max_scale[cols_per_thread];\n#pragma unroll\n        for (int col = 0; col < cols_per_thread; ++col) {\n            const int jc = cols_per_warp == 8 ? T_C_KQ::get_j(col) : T_C_KQ::get_i(2*col);\n            const float sink = sinks_f[jc % ncols2];\n\n            const float KQ_max_new = fmaxf(KQ_max[col], sink);\n            const float KQ_max_diff = KQ_max[col] - KQ_max_new;\n            KQ_max_scale[col] = expf(KQ_max_diff);\n            KQ_max[col] = KQ_max_new;\n\n            *((uint32_t *) &KQ_max_scale[col]) *= KQ_max_diff >= SOFTMAX_FTZ_THRESHOLD;\n\n            const float KQ_max_add = expf(sink - KQ_max_new);\n            KQ_rowsum[col] = KQ_max_scale[col]*KQ_rowsum[col] + KQ_max_add;\n        }\n\n#if defined(TURING_MMA_AVAILABLE)\n        if constexpr (cols_per_warp == 8) {\n            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[cols_per_thread - 1]);\n#pragma unroll\n            for (int i = 0; i < DV/T_C_VKQ::I; ++i) {\n#pragma unroll\n                for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                    VKQ_C[i].x[l] *= KQ_max_scale_h2;\n                }\n            }\n        } else {\n#pragma unroll\n            for (int col = 0; col < cols_per_thread; ++col) {\n                const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n                for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n                    for (int l0 = 0; l0 < T_C_VKQ::ne; l0 += 2) {\n                        VKQ_C[i].x[l0 + col] *= KQ_max_scale_h2;\n                    }\n                }\n            }\n        }\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n        const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[0], KQ_max_scale[0]);\n#pragma unroll\n        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n            for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                VKQ_C[i].x[l] *= KQ_max_scale_h2;\n            }\n        }\n#else // Volta\n        const int col = (threadIdx.x / 2) % 2;\n        const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale[col], KQ_max_scale[col]);\n#pragma unroll\n        for (int i = 0; i < (DV/2)/T_C_VKQ::J; ++i) {\n#pragma unroll\n            for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                VKQ_C[i].x[l] *= KQ_max_scale_h2;\n            }\n        }\n#endif // defined(TURING_MMA_AVAILABLE)\n    }\n\n    // Combine VKQ accumulator values if np > 1.\n    // It's also faster to do small writes to shared memory, then large write to VRAM than to do small writes to VRAM.\n    // So also write VKQ accumulators to shared memory in column-major format if np == 1.\n\n    constexpr int tile_stride = nbatch_combine + 4;\n    static_assert((DV/2) % nbatch_combine == 0, \"bad nbatch_combine\");\n\n    if constexpr (cols_per_warp == 8) {\n        const int jc_cwmo = (threadIdx.x % (2*T_C_VKQ::J)) / T_C_VKQ::J; // jc combine write meta offset\n        const int jc_cwm = threadIdx.y*(2*T_C_VKQ::J) + 2*T_C_VKQ::get_j(-1) + jc_cwmo; // jc combine write meta\n        const float2 KQ_cmr = make_float2(KQ_max[jc_cwmo], KQ_rowsum[jc_cwmo]); // KQ combine max rowsum\n\n        if (((!needs_fixup && !is_fixup) || np > 1) && threadIdx.x < 2*T_C_VKQ::J) {\n            // Use the 16 bytes of padding in each row to store the meta data: KQ max, KQ rowsum, KQ max scale.\n            ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;\n        }\n\n        __syncthreads();\n\n        if (np == 1) {\n            // No combination is needed, the meta data can be directly written from registers to VRAM.\n            if (needs_fixup && threadIdx.x < T_B_KQ::I) {\n                float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n                dstk_fixup_meta[jc_cwm] = KQ_cmr;\n            }\n            if (is_fixup && threadIdx.x < T_B_KQ::I) {\n                float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n                dstk_fixup_meta[jc_cwm] = KQ_cmr;\n            }\n        }\n    } else {\n        // jc_cwm = jc combine write meta\n        // KQ_cmr = KQ combine max rowsum\n        // Use the 16 bytes of padding in each Q column to store the meta data: KQ max, KQ rowsum, KQ max scale.\n#if defined(TURING_MMA_AVAILABLE)\n        const int jc_cwm = threadIdx.y*cols_per_warp + T_C_VKQ::get_i(threadIdx.x % 4);\n        const float2 KQ_cmr = make_float2(KQ_max[threadIdx.x % cols_per_thread], KQ_rowsum[threadIdx.x % cols_per_thread]);\n        const bool thread_should_write = threadIdx.x % 4 < cols_per_thread;\n#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)\n        const int jc_cwm = threadIdx.y*cols_per_warp + T_C_VKQ::get_i(0);\n        const float2 KQ_cmr = make_float2(KQ_max[0], KQ_rowsum[0]);\n        const bool thread_should_write = threadIdx.x / 16 < cols_per_thread;\n#else // Volta\n        const int jc_cwm = threadIdx.y*cols_per_warp + T_C_KQ::get_i(threadIdx.x & 2);\n        const float2 KQ_cmr = make_float2(KQ_max[(threadIdx.x & 2) / 2], KQ_rowsum[(threadIdx.x & 2) / 2]);\n        const bool thread_should_write = T_C_KQ::J == 8 || T_C_KQ::get_j(threadIdx.x & 2) < 8;\n#endif // defined(TURING_MMA_AVAILABLE)\n\n        if (((!needs_fixup && !is_fixup) || np > 1) && thread_should_write) {\n            ((float2 *) tile_Q)[jc_cwm*(tile_stride/2) + nbatch_combine/2] = KQ_cmr;\n        }\n\n        __syncthreads();\n\n        if (np == 1) {\n            // No combination is needed, the meta data can be directly written from registers to VRAM.\n            if (needs_fixup && thread_should_write) {\n                float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n                dstk_fixup_meta[jc_cwm] = KQ_cmr;\n            }\n            if (is_fixup && thread_should_write) {\n                float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n                dstk_fixup_meta[jc_cwm] = KQ_cmr;\n            }\n        }\n    }\n\n    if (np > 1 && threadIdx.y % np == 0) {\n        // Combine the meta data for parallel warps via shared memory.\n        // Warps with threadIdx.y % np != 0 must NOT return early.\n        // All threads must return simultaneously to avoid race conditions with work on the next tile.\n\n        constexpr int nmeta = np*cols_per_warp >= warp_size ? np*cols_per_warp/warp_size : 1;\n\n        const int jc_meta = threadIdx.y*cols_per_warp + (np*cols_per_warp < warp_size ? threadIdx.x % (np*cols_per_warp) : threadIdx.x);\n        float2 * const meta_ptr = ((float2 *) tile_Q) + jc_meta*(tile_stride/2) + nbatch_combine/2;\n        float2 meta[nmeta];\n#pragma unroll\n        for (int imeta = 0; imeta < nmeta; ++imeta) {\n            meta[imeta] = meta_ptr[imeta * warp_size * tile_stride/2];\n        }\n\n        float KQ_cmn = meta[0].x; // KQ combine max new, max between all parallel warps.\n#pragma unroll\n        for (int imeta = 1; imeta < nmeta; ++imeta) {\n            KQ_cmn = fmaxf(KQ_cmn, meta[imeta].x);\n        }\n#pragma unroll\n        for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {\n            if (offset < warp_size) {\n                KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size));\n            }\n        }\n\n        float KQ_cms[nmeta]; // KQ combine max scale per warp.\n#pragma unroll\n        for (int imeta = 0; imeta < nmeta; ++imeta) {\n            KQ_cms[imeta] = expf(meta[imeta].x - KQ_cmn);\n        }\n\n        float KQ_crs = KQ_cms[0]*meta[0].y; // KQ combine rowsum, scaled sum of all parallel warps.\n#pragma unroll\n        for (int imeta = 1; imeta < nmeta; ++imeta) {\n            KQ_crs += KQ_cms[imeta]*meta[imeta].y;\n        }\n#pragma unroll\n        for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {\n            if (offset < warp_size) {\n                KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size);\n            }\n        }\n\n        __syncthreads();\n\n        // Write back combined meta data:\n#pragma unroll\n        for (int imeta = 0; imeta < nmeta; ++imeta) {\n            if (np*cols_per_warp >= warp_size || threadIdx.x < np*cols_per_warp) {\n                // Combined KQ max scale + rowsum.\n                meta_ptr[imeta * warp_size * tile_stride/2] = make_float2(KQ_cms[imeta], KQ_crs);\n            }\n        }\n\n        // Combined KQ max + rowsum.\n        static_assert(cols_per_warp <= warp_size);\n        if (needs_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {\n            float2 * dstk_fixup_meta = dstk_fixup + blockIdx.x*ncols;\n            dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);\n        }\n        if (is_fixup && (cols_per_warp == warp_size || threadIdx.x < cols_per_warp)) {\n            float2 * dstk_fixup_meta = dstk_fixup + (gridDim.x + blockIdx.x)*ncols;\n            dstk_fixup_meta[(threadIdx.y/np)*cols_per_warp + threadIdx.x] = make_float2(KQ_cmn, KQ_crs);\n        }\n    } else if (np > 1) {\n        // Warps with threadIdx.y % np == 0 execute a __syncthreads() in the if branch.\n        // Therefore, all other warps also need to execute a __syncthreads().\n        // Otherwise the points at which warps synchronize with each other would become misaligned.\n        __syncthreads();\n    }\n\n#pragma unroll\n    for (int k00 = 0; k00 < DV/2; k00 += nbatch_combine) {\n        if constexpr (cols_per_warp == 8) {\n            const int jc_cwd = threadIdx.y*T_B_KQ::I + T_B_KQ::get_i(-1); // jc combine write data\n#pragma unroll\n            for (int k1 = 0; k1 < nbatch_combine; k1 += T_B_KQ::J) {\n                const T_B_KQ B = get_transposed(VKQ_C[(k00 + k1)/T_B_KQ::J]); // Conversion of C to B matrix puts it in column-major format.\n\n#pragma unroll\n                for (int l = 0; l < T_B_KQ::ne; ++l) {\n                    const int k = k1 + T_B_KQ::get_j(l);\n\n                    tile_Q[jc_cwd*tile_stride + k] = B.x[l];\n                }\n            }\n        } else {\n            const int j0 = threadIdx.y*cols_per_warp;\n#pragma unroll\n            for (int k1 = 0; k1 < nbatch_combine; k1 += T_C_VKQ::J) {\n#pragma unroll\n                for (int l = 0; l < T_C_VKQ::ne; ++l) {\n                    const int j = j0 + T_C_VKQ::get_i(l);\n                    const int k = k1 + T_C_VKQ::get_j(l);\n\n                    tile_Q[j*tile_stride + k] = VKQ_C[(k00 + k1)/T_C_VKQ::J].x[l];\n                }\n            }\n        }\n\n        __syncthreads();\n\n        if (np == 1 || threadIdx.y % np == 0) {\n            // The first 2*2*gridDim.x*ncols floats in dstk_fixup are for storing max. values and row sums.\n            // The values after that are for the partial results of the individual blocks.\n            float2 * dstk_fixup_data = dstk_fixup + gridDim.x*(2*ncols) + blockIdx.x*(ncols*(DV/2));\n\n#pragma unroll\n            for (int stride_k : {warp_size, warp_size/2, warp_size/4, warp_size/8}) {\n                const int k0_start  = stride_k == warp_size ? 0 : nbatch_combine - nbatch_combine % (2*stride_k);\n                const int k0_stop   =                             nbatch_combine - nbatch_combine % (1*stride_k);\n                const int stride_jc = warp_size / stride_k;\n\n                if (k0_start == k0_stop) {\n                    continue;\n                }\n\n#pragma unroll\n                for (int jc0_dst = 0; jc0_dst < ncols; jc0_dst += (nwarps/np)*stride_jc) {\n                    const int jc_dst = jc0_dst + (threadIdx.y/np)*stride_jc + (stride_k == warp_size ? 0 : threadIdx.x / stride_k);\n\n                    if (jc0_dst + (nwarps/np)*stride_jc > ncols && jc_dst >= ncols) {\n                        break;\n                    }\n\n                    const int jc_tile_K = (jc_dst/cols_per_warp)*(np*cols_per_warp) + jc_dst % cols_per_warp;\n\n                    const int j_dst = jc_dst / ncols2;\n                    const int c_dst = jc_dst % ncols2;\n\n                    if (!is_fixup && ((ncols1 > 1 && jt*ncols1 + j_dst >= int(ne01.z)) || (ncols2 > 1 && zt_gqa*ncols2 + c_dst >= gqa_ratio))) {\n                        continue;\n                    }\n\n                    const float * meta_j = (const float *) tile_Q + jc_tile_K*tile_stride + nbatch_combine;\n#pragma unroll\n                    for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {\n                        const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);\n\n                        float2 dstk_val = make_float2(0.0f, 0.0f);\n#pragma unroll\n                        for (int ip = 0; ip < np; ++ip) {\n                            const float KQ_crs = np == 1 ? 1.0f : meta_j[ip*cols_per_warp * tile_stride + 0];\n                            const float2 dstk_val_add = __half22float2(tile_Q[(jc_tile_K + ip*cols_per_warp) * tile_stride + k]);\n                            dstk_val.x += dstk_val_add.x*KQ_crs;\n                            dstk_val.y += dstk_val_add.y*KQ_crs;\n                        }\n\n                        if (!needs_fixup && !is_fixup) {\n                            const float KQ_rowsum_j = meta_j[1];\n                            dstk_val.x /= KQ_rowsum_j;\n                            dstk_val.y /= KQ_rowsum_j;\n                        }\n\n                        if (is_fixup) {\n                            dstk_fixup_data[jc_dst*(DV/2) + k00 + k] = dstk_val;\n                        } else {\n                            dstk[((jt*ncols1 + j_dst)*ne02 + c_dst)*(DV/2) + k00 + k] = dstk_val;\n                        }\n                    }\n                }\n            }\n        }\n        if (np > 1) {\n            __syncthreads();\n        }\n    }\n#else\n    GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup,\n        scale, slope, logit_softcap, ne01, ne02, gqa_ratio,\n        stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,\n        jt, kb0_start, kb0_stop);\n    NO_DEVICE_CODE;\n#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE)\n}\n\ntemplate<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view>\n__launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2))\nstatic __global__ void flash_attn_ext_f16(\n        const char * __restrict__ Q,\n        const char * __restrict__ K,\n        const char * __restrict__ V,\n        const char * __restrict__ mask,\n        const char * __restrict__ sinks,\n        const int  * __restrict__ KV_max,\n        float      * __restrict__ dst,\n        float2     * __restrict__ dst_meta,\n        const float scale,\n        const float max_bias,\n        const float m0,\n        const float m1,\n        const uint32_t n_head_log2,\n        const float logit_softcap,\n        const int32_t ne00, const uint3   ne01, const int32_t ne02, const int32_t ne03,\n                            const int32_t nb01, const int32_t nb02, const int32_t nb03,\n        const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n                            const int32_t nb11, const int32_t nb12, const int64_t nb13,\n                            const int32_t nb21, const int32_t nb22, const int64_t nb23,\n                            const int32_t ne31, const int32_t ne32, const int32_t ne33,\n                            const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))\n\n    // Skip unused kernel variants for faster compilation:\n    if (use_logit_softcap && !(DKQ == 128 || DKQ == 256)) {\n        NO_DEVICE_CODE;\n        return;\n    }\n#ifdef VOLTA_MMA_AVAILABLE\n    if (ncols1*ncols2 < 32) {\n        NO_DEVICE_CODE;\n        return;\n    }\n#endif // VOLTA_MMA_AVAILABLE\n\n#if __CUDA_ARCH__ == GGML_CUDA_CC_TURING\n    if (ncols1*ncols2 > 32) {\n        NO_DEVICE_CODE;\n        return;\n    }\n#endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING\n\n#if defined(AMD_WMMA_AVAILABLE)\n    if (ncols1*ncols2 > 32 || ncols1*ncols2 < 16 || DKQ > 128 || ncols2 == 1) {\n        NO_DEVICE_CODE;\n        return;\n    }\n#endif // defined(AMD_WMMA_AVAILABLE)\n\n#if defined(AMD_MFMA_AVAILABLE)\n    if (DKQ != 64 && DKQ != 80 && DKQ != 96 && DKQ != 112 && DKQ != 128) {\n        NO_DEVICE_CODE;\n        return;\n    }\n#endif // defined(AMD_MFMA_AVAILABLE)\n\n    constexpr int warp_size = ggml_cuda_get_physical_warp_size();\n    constexpr int ncols     = ncols1 * ncols2;\n    constexpr int nbatch_fa = ggml_cuda_fattn_mma_get_nbatch_fa(DKQ, DV, ncols);\n    constexpr int nthreads  = ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols);\n    constexpr int nwarps    = nthreads / warp_size;\n\n    const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n\n    const int stride_Q1   = nb01 / sizeof(float2);\n    const int stride_Q2   = nb02 / sizeof(float2);\n    const int stride_K    = nb11 / sizeof(half2);\n    const int stride_mask = nb31 / sizeof(half);\n\n    const int stride_V = V_is_K_view ? stride_K : nb21 / sizeof(half2);\n\n    const int iter_k     = (ne11      + (nbatch_fa - 1)) / nbatch_fa;\n    const int iter_j     = (ne01.z    + (ncols1    - 1)) / ncols1;\n    const int iter_z_gqa = (gqa_ratio + (ncols2    - 1)) / ncols2;\n\n    // kbc == k block continuous, current index in continuous ijk space.\n    int       kbc      = int64_t(blockIdx.x + 0)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n    const int kbc_stop = int64_t(blockIdx.x + 1)*(iter_k*iter_j*iter_z_gqa*ne12*ne03) / gridDim.x;\n\n    // If the seams of 2 CUDA blocks fall within an output tile their results need to be combined.\n    // For this we need to track both the block that starts the tile (needs_fixup) and the block that finishes the tile (is_fixup).\n    // In the most general case >2 seams can fall into the same tile.\n\n    // kb0 == k start index when in the output tile.\n    int kb0_start = kbc % iter_k;\n    int kb0_stop  = min(iter_k, kb0_start + kbc_stop - kbc);\n\n    while (kbc < kbc_stop && kb0_stop == iter_k) {\n        // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index\n        const int sequence =  kbc /(iter_k*iter_j*iter_z_gqa*ne12);\n        const int z_KV     = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n        const int zt_gqa   = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n        const int jt       = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n        const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n        const float2 * Q_f2   = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);\n        const half2  * K_h2   = (const half2  *) (K + nb13*sequence + nb12*z_KV);\n        const half   * mask_h = ncols2 == 1 && !mask ? nullptr :\n            (const half *) (mask + nb33*(sequence % ne33));\n        float2       * dstk   = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);\n\n        const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);\n        const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;\n\n        const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;\n\n        if (KV_max) {\n            kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);\n        }\n        constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.\n        if (kb0_start == 0) {\n            constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.\n            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>\n                (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n                 ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n        } else {\n            constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.\n            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>\n                (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n                 ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n        }\n\n        kbc += iter_k;\n        kbc -= kbc % iter_k;\n\n        kb0_start = 0;\n        kb0_stop  = min(iter_k, kbc_stop - kbc);\n    }\n\n    if (kbc >= kbc_stop) {\n        return;\n    }\n\n    // z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index.\n    const int sequence =  kbc /(iter_k*iter_j*iter_z_gqa*ne12);\n    const int z_KV     = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence)/(iter_k*iter_j*iter_z_gqa);\n    const int zt_gqa   = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV)/(iter_k*iter_j);\n    const int jt       = (kbc - iter_k*iter_j*iter_z_gqa*ne12 * sequence - iter_k*iter_j*iter_z_gqa * z_KV - iter_k*iter_j * zt_gqa) / iter_k;\n\n    const int zt_Q = z_KV*gqa_ratio + zt_gqa*ncols2; // Global Q head start index.\n\n    const float2 * Q_f2   = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);\n    const half2  * K_h2   = (const half2  *) (K + nb13*sequence + nb12*z_KV);\n    const half   * mask_h = ncols2 == 1 && !mask ? nullptr :\n        (const half *) (mask + nb33*(sequence % ne33));\n    float2       * dstk   = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);\n\n    const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);\n    const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;\n\n    const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;\n\n    if (KV_max) {\n        kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);\n    }\n\n    constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.\n    constexpr bool needs_fixup = false;\n    flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>\n        (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,\n         ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);\n#else\n    GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n        max_bias, m0, m1, n_head_log2, logit_softcap,\n        ne00, ne01, ne02, ne03,\n              nb01, nb02, nb03,\n        ne10, ne11, ne12, ne13,\n              nb11, nb12, nb13,\n              nb21, nb22, nb23,\n              ne31, ne32, ne33,\n              nb31, nb32, nb33);\n    NO_DEVICE_CODE;\n#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)) || defined(AMD_MFMA_AVAILABLE))\n}\n\ntemplate <int DKQ, int DV, int ncols1, int ncols2>\nvoid ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * KQV = dst;\n    const int id = ggml_cuda_get_device();\n    const int cc = ggml_cuda_info().devices[id].cc;\n\n    constexpr int ncols = ncols1 * ncols2;\n\n    const int  nthreads       = ggml_cuda_fattn_mma_get_nthreads      (DKQ, DV, ncols, cc);\n    const int  nbatch_fa      = ggml_cuda_fattn_mma_get_nbatch_fa     (DKQ, DV, ncols, cc);\n    const int  nbatch_K2      = ggml_cuda_fattn_mma_get_nbatch_K2     (DKQ, DV, ncols, cc);\n    const int  nbatch_V2      = ggml_cuda_fattn_mma_get_nbatch_V2     (DKQ, DV, ncols, cc);\n    const int  nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols, cc);\n    const bool Q_in_reg       = ggml_cuda_fattn_mma_get_Q_in_reg      (DKQ, DV, ncols, cc);\n    const int  nstages        = ggml_cuda_fattn_mma_get_nstages       (DKQ, DV, ncols1, ncols2, cc);\n\n    const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc));\n    const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size;\n    const int nwarps         = nthreads / warp_size_host;\n\n    constexpr bool V_is_K_view = DKQ == 576; // Guaranteed by the kernel selection logic in fattn.cu\n\n    const size_t nbytes_shared_KV_1stage = nbatch_fa            * std::max(nbatch_K2 + 4,  nbatch_V2 + 4) * sizeof(half2);\n    const size_t nbytes_shared_KV_2stage = nbatch_fa            *         (nbatch_K2 + 4 + nbatch_V2 + 4) * sizeof(half2);\n    const size_t nbytes_shared_Q         = ncols                * (DKQ/2 + 4)                             * sizeof(half2);\n    const size_t nbytes_shared_mask      = ncols1               * (nbatch_fa/2 + 4)                       * sizeof(half2);\n    const size_t nbytes_shared_combine   = nwarps*cols_per_warp * (nbatch_combine + 4)                    * sizeof(half2);\n\n    const size_t nbytes_shared_KV = nstages <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage;\n\n    const size_t nbytes_shared_total = std::max(nbytes_shared_combine, Q_in_reg ?\n        std::max(nbytes_shared_Q,  nbytes_shared_KV + nbytes_shared_mask) :\n                 nbytes_shared_Q + nbytes_shared_KV + nbytes_shared_mask);\n\n    float logit_softcap;\n    memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n#if defined(GGML_USE_HIP)\n    using fattn_kernel_ptr_t = const void*;\n#else\n    using fattn_kernel_ptr_t = fattn_kernel_t;\n#endif // defined(GGML_USE_HIP)\n    fattn_kernel_t fattn_kernel;\n    if (logit_softcap == 0.0f) {\n        constexpr bool use_logit_softcap = false;\n        fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;\n\n#if !defined(GGML_USE_MUSA)\n        static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};\n        if (!shared_memory_limit_raised[id]) {\n            CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));\n            shared_memory_limit_raised[id] = true;\n        }\n#endif // !defined(GGML_USE_MUSA)\n    } else {\n        constexpr bool use_logit_softcap = true;\n        fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;\n\n#if !defined(GGML_USE_MUSA)\n        static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};\n        if (!shared_memory_limit_raised[id]) {\n            CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));\n            shared_memory_limit_raised[id] = true;\n        }\n#endif // !defined(GGML_USE_MUSA)\n    }\n\n    launch_fattn<DV, ncols1, ncols2>\n        (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host);\n}\n\n\n#define DECL_FATTN_MMA_F16_CASE(DKQ, DV, ncols1, ncols2)                          \\\n    template void ggml_cuda_flash_attn_ext_mma_f16_case                           \\\n    <DKQ, DV, ncols1, ncols2>(ggml_backend_cuda_context & ctx, ggml_tensor * dst) \\\n\n#define DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(DKQ, DV, ncols)   \\\n    extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 1,  1); \\\n    extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 2,  2); \\\n    extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 4,  4); \\\n    extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/ 8,  8); \\\n    extern DECL_FATTN_MMA_F16_CASE(DKQ, DV, (ncols)/16, 16); \\\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64,  64,   8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80,  80,   8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96,  96,   8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112,   8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128,   8)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256,   8)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64,  64,  16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80,  80,  16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96,  96,  16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112,  16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128,  16)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256,  16)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64,  64,  32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80,  80,  32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96,  96,  32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112,  32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128,  32)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256,  32)\n\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 64,  64,  64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 80,  80,  64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2( 96,  96,  64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112,  64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128,  64)\nDECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256,  64)\n\n// The number of viable configurations for Deepseek is very limited:\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);\n\n// For GLM 4.7 Flash\nextern DECL_FATTN_MMA_F16_CASE(576, 512,  4,  4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512,  8,  4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512, 16,  4);\nextern DECL_FATTN_MMA_F16_CASE(576, 512,  1, 32);\nextern DECL_FATTN_MMA_F16_CASE(576, 512,  2, 32);\n"
  },
  {
    "path": "ggml/src/ggml-cuda/fattn-tile.cu",
    "content": "#include \"common.cuh\"\n#include \"fattn-tile.cuh\"\n#include \"fattn-wmma-f16.cuh\"\n\nvoid ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * K = dst->src[1];\n    const ggml_tensor * V = dst->src[2];\n    switch (K->ne[0]) {\n        case  40: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case< 40,  40>(ctx, dst);\n        } break;\n        case  64: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case< 64,  64>(ctx, dst);\n        } break;\n        case  72: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case< 72,  72>(ctx, dst);\n        } break;\n        case  80: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case< 80,  80>(ctx, dst);\n        } break;\n        case  96: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case< 96,  96>(ctx, dst);\n        } break;\n        case 112: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case<112, 112>(ctx, dst);\n        } break;\n        case 128: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case<128, 128>(ctx, dst);\n        } break;\n        case 256: {\n            GGML_ASSERT(V->ne[0] == K->ne[0]);\n            ggml_cuda_flash_attn_ext_tile_case<256, 256>(ctx, dst);\n        } break;\n        case 576: {\n            GGML_ASSERT(V->ne[0] == 512);\n            ggml_cuda_flash_attn_ext_tile_case<576, 512>(ctx, dst);\n        } break;\n        default: {\n            GGML_ABORT(\"Unsupported head size\");\n        } break;\n    }\n}\n"
  },
  {
    "path": "ggml/src/ggml-cuda/fattn-vec.cuh",
    "content": "#include \"common.cuh\"\n#include \"fattn-common.cuh\"\n\nstatic int ggml_cuda_fattn_vec_get_nthreads_host(const int cc) {\n    return 128;\n    GGML_UNUSED(cc);\n}\n\nstatic constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() {\n    return 128;\n}\n\n// Currently llvm with the amdgcn target does not support unrolling loops\n// that contain a break that can not be resolved at compile time.\n#ifdef __clang__\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wpass-failed\"\n#endif // __clang__\ntemplate<int D, int ncols, ggml_type type_K, ggml_type type_V, bool use_logit_softcap> // D == head size\n__launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1)\nstatic __global__ void flash_attn_ext_vec(\n        const char * __restrict__ Q,\n        const char * __restrict__ K,\n        const char * __restrict__ V,\n        const char * __restrict__ mask,\n        const char * __restrict__ sinks,\n        const int  * __restrict__ KV_max,\n        float      * __restrict__ dst,\n        float2     * __restrict__ dst_meta,\n        const float scale,\n        const float max_bias,\n        const float m0,\n        const float m1,\n        const uint32_t n_head_log2,\n        const float logit_softcap,\n        const int32_t ne00, const uint3   ne01, const int32_t ne02, const int32_t ne03,\n                            const int32_t nb01, const int32_t nb02, const int32_t nb03,\n        const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,\n                            const int32_t nb11, const int32_t nb12, const int64_t nb13,\n                            const int32_t nb21, const int32_t nb22, const int64_t nb23,\n                            const int32_t ne31, const int32_t ne32, const int32_t ne33,\n                            const int32_t nb31, const int32_t nb32, const int64_t nb33) {\n#ifdef FLASH_ATTN_AVAILABLE\n\n    // Skip unused kernel variants for faster compilation:\n    if (use_logit_softcap && !(D == 128 || D == 256)) {\n        GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n            max_bias, m0, m1, n_head_log2, logit_softcap,\n            ne00, ne01, ne02, ne03,\n                  nb01, nb02, nb03,\n            ne10, ne11, ne12, ne13,\n                  nb11, nb12, nb13,\n                  nb21, nb22, nb23,\n                  ne31, ne32, ne33,\n                  nb31, nb32, nb33);\n        NO_DEVICE_CODE;\n        return;\n    }\n\n    //In this kernel Q, K, V are matrices while i, j, k are matrix indices.\n\n    constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();\n    constexpr int cpy_ne = cpy_nb / 4;\n\n#ifdef GGML_USE_HIP\n#ifdef RDNA\n    constexpr int nthreads_KQ_q = 2;\n#else\n    constexpr int nthreads_KQ_q = 4;\n#endif // RDNA\n    constexpr int nthreads_V_q  = (D/4 < 32 ? D/4 : 32);\n#else\n    constexpr int nthreads_KQ_q = (D/4 < 32 ? D/4 : 32);\n    constexpr int nthreads_V_q  = (D/4 < 32 ? D/4 : 32);\n#endif // GGML_USE_HIP\n\n    constexpr int nthreads    = ggml_cuda_fattn_vec_get_nthreads_device();\n    constexpr int nthreads_KQ = type_K == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_KQ_q;\n    constexpr int nthreads_V  = type_V == GGML_TYPE_F16 ? 128 / cpy_nb : nthreads_V_q;\n\n    static_assert(WARP_SIZE % nthreads_KQ == 0, \"bad nthreads_K\");\n    static_assert(WARP_SIZE % nthreads_V  == 0, \"bad nthreads_V\");\n\n    constexpr int V_rows_per_thread = type_V == GGML_TYPE_F16 ? 2*cpy_ne : 4;\n    constexpr int V_cols_per_iter   = WARP_SIZE / nthreads_V;\n\n    constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ<type_K, D, nthreads_KQ>();\n    constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16;\n#ifdef V_DOT2_F32_F16_AVAILABLE\n    constexpr dequantize_V_t dequantize_V = get_dequantize_V<type_V, half,  V_rows_per_thread>();\n#else\n    constexpr dequantize_V_t dequantize_V = get_dequantize_V<type_V, float, V_rows_per_thread>();\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n    const int ic0 = blockIdx.x * ncols; // Index of the Q/QKV column to work on.\n\n    const int sequence = blockIdx.z / ne02;\n    const int head = blockIdx.z - sequence*ne02;\n    const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.\n    Q += nb03*sequence + nb02* head              + nb01*ic0;\n    K += nb13*sequence + nb12*(head / gqa_ratio);\n    V += nb23*sequence + nb22*(head / gqa_ratio);\n\n    const half * maskh  = (const half  *) (mask + nb33*(sequence % ne33) + nb31*ic0);\n\n    const float slope = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);\n\n    static_assert(D % (2*WARP_SIZE) == 0, \"D not divisible by 2*WARP_SIZE == 64.\");\n    constexpr int nwarps = nthreads / WARP_SIZE;\n    const int tid = WARP_SIZE*threadIdx.y + threadIdx.x;\n    __builtin_assume(tid < nthreads);\n\n    constexpr int ne_KQ      = ncols*D;\n    constexpr int ne_combine = nwarps*V_cols_per_iter*D;\n#ifdef V_DOT2_F32_F16_AVAILABLE\n    half2            VKQ[ncols][(D/2)/nthreads_V] = {{{0.0f, 0.0f}}};\n    __shared__ half   KQ[ne_KQ > ne_combine ? ne_KQ : ne_combine];\n#else\n    float2           VKQ[ncols][(D/2)/nthreads_V] = {{{0.0f, 0.0f}}};\n    __shared__ float  KQ[ne_KQ > ne_combine ? ne_KQ : ne_combine];\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n    float KQ_max[ncols];\n    float KQ_sum[ncols];\n#pragma unroll\n    for (int j = 0; j < ncols; ++j) {\n        KQ_max[j] = -FLT_MAX/2.0f;\n        KQ_sum[j] = 0.0f;\n    }\n\n    // Convert Q to float2 (f16 K) or q8_1 (quantized K) and store in registers:\n#ifdef V_DOT2_F32_F16_AVAILABLE\n    half2  Q_reg[ncols][(D/2)/nthreads_KQ]; // Will be initialized completely.\n#else\n    __align__(16) float2 Q_reg[ncols][(D/2)/nthreads_KQ] = {{{0.0f, 0.0f}}}; // May be only partially initialized.\n#endif // V_DOT2_F32_F16_AVAILABLE\n    int    Q_i32[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)];\n    float2  Q_ds[ncols][1 > D/(sizeof(int)*nthreads_KQ) ? 1 : D/(sizeof(int)*nthreads_KQ)];\n    if constexpr (Q_q8_1) {\n#pragma unroll\n        for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n            const int j = j0 + threadIdx.y;\n\n            if (j0 + nwarps > ncols && j >= ncols) {\n                break;\n            }\n\n            // Reuse KQ as temporary storage for converting Q to q8_1:\n            int    * tmp_q_i32 = (int    *) &KQ[j*D];\n            float2 * tmp_q_ds  = (float2 *) (tmp_q_i32 + D/sizeof(int));\n\n            // Set memory to zero if out of bounds:\n            if (ncols > 1 && ic0 + j >= int(ne01.z)) {\n#pragma unroll\n                for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += WARP_SIZE) {\n                    const int i = i0 + threadIdx.x;\n\n                    if (i0 + WARP_SIZE <= int(D/sizeof(int)) || i < int(D/sizeof(int))) {\n                        tmp_q_i32[i] = 0;\n                    }\n                }\n                if (threadIdx.x < D/QK8_1) {\n                    tmp_q_ds[threadIdx.x] = make_float2(0.0f, 0.0f);\n                }\n            } else {\n                const float * Q_f = (const float *) (Q + j*nb01);\n                constexpr int nthreads_quantize = D/sizeof(int) < WARP_SIZE ? D/sizeof(int) : WARP_SIZE;\n#pragma unroll\n                for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_quantize) {\n                    quantize_q8_1_to_shared<float2, nthreads_quantize>\n                        (Q_f + i0*sizeof(int), scale, tmp_q_i32 + i0, tmp_q_ds + i0/QI8_1);\n                }\n            }\n        }\n\n        __syncthreads();\n\n#pragma unroll\n        for (int j = 0; j < ncols; ++j) {\n            int    * tmp_q_i32 = (int    *) &KQ[j*D];\n            float2 * tmp_q_ds  = (float2 *) (tmp_q_i32 + D/sizeof(int));\n\n#pragma unroll\n            for (int i0 = 0; i0 < int(D/sizeof(int)); i0 += nthreads_KQ) {\n                const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ);\n\n                Q_i32[j][i0/nthreads_KQ] = tmp_q_i32[i];\n                Q_ds[j][i0/nthreads_KQ]  = tmp_q_ds[i/QI8_1];\n            }\n        }\n\n        __syncthreads();\n    } else {\n#ifdef V_DOT2_F32_F16_AVAILABLE\n        const half2 scale_h2 = make_half2(scale, scale);\n#pragma unroll\n        for (int j = 0; j < ncols; ++j) {\n            const float2 * Q_j = (const float2 *) (Q + j*nb01);\n#pragma unroll\n            for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n                const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ)*cpy_ne;\n\n                __align__(16) float2 tmp[cpy_ne] = {{0.0f, 0.0f}};\n                if (ncols == 1 || ic0 + j < int(ne01.z)) {\n                    ggml_cuda_memcpy_1<cpy_nb>(tmp,            &Q_j[i]);\n                    ggml_cuda_memcpy_1<cpy_nb>(tmp + cpy_ne/2, &Q_j[i + cpy_ne/2]);\n                }\n#pragma unroll\n                for (int i1 = 0; i1 < cpy_ne; ++i1) {\n                    Q_reg[j][i0/nthreads_KQ + i1] = make_half2(tmp[i1].x, tmp[i1].y);\n                }\n            }\n#pragma unroll\n            for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n                Q_reg[j][k] *= scale_h2;\n            }\n        }\n#else\n#pragma unroll\n        for (int j = 0; j < ncols; ++j) {\n            const float2 * Q_j = (const float2 *) (Q + j*nb01);\n#pragma unroll\n            for (int i0 = 0; i0 < D/2; i0 += nthreads_KQ*cpy_ne) {\n                const int i = i0 + (nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ)*cpy_ne;\n                if (ncols == 1 || ic0 + j < int(ne01.z)) {\n                    ggml_cuda_memcpy_1<cpy_nb>(&Q_reg[j][i0/nthreads_KQ],            &Q_j[i]);\n                    ggml_cuda_memcpy_1<cpy_nb>(&Q_reg[j][i0/nthreads_KQ + cpy_ne/2], &Q_j[i + cpy_ne/2]);\n                }\n            }\n#pragma unroll\n            for (int k = 0; k < (D/2)/nthreads_KQ; ++k) {\n                Q_reg[j][k].x *= scale;\n                Q_reg[j][k].y *= scale;\n            }\n        }\n#endif // V_DOT2_F32_F16_AVAILABLE\n    }\n\n    const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;\n    K     += blockIdx.y*nthreads * nb11;\n    V     += blockIdx.y*nthreads * nb21;\n    maskh += blockIdx.y*nthreads;\n    for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads,\n             // Increment pointers after each loop:\n             K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) {\n\n        // Calculate KQ tile and keep track of new maximum KQ values:\n        float KQ_reg[ncols]; // KQ in registers.\n\n        float KQ_max_new[ncols];\n#pragma unroll\n        for (int j = 0; j < ncols; ++j) {\n            KQ_max_new[j] = KQ_max[j];\n        }\n\n#pragma unroll\n        for (int i_KQ_0 = 0; i_KQ_0 < nthreads_KQ; ++i_KQ_0) {\n            const int i_KQ = threadIdx.y*WARP_SIZE + (nthreads_KQ == WARP_SIZE ? 0 : (threadIdx.x & ~(nthreads_KQ-1))) + i_KQ_0;\n\n#pragma unroll\n            for (int j = 0; j < ncols; ++j) {\n                float sum = vec_dot_KQ(K + i_KQ*nb11, Q_reg[j], Q_i32[j], Q_ds[j]);\n                sum = warp_reduce_sum<nthreads_KQ>(sum);\n\n                if (use_logit_softcap) {\n                    sum = logit_softcap*tanhf(sum);\n                }\n\n                if (mask && (ncols == 1 || ic0 + j < int(ne01.z))) {\n                    sum += slope*__half2float(maskh[j*ne11 + i_KQ]);\n                }\n\n                KQ_max_new[j] = fmaxf(KQ_max_new[j], sum + FATTN_KQ_MAX_OFFSET);\n\n                if ((nthreads_KQ == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_KQ) == uint32_t(i_KQ_0)) {\n                    KQ_reg[j] = sum;\n                }\n            }\n        }\n\n#pragma unroll\n        for (int j = 0; j < ncols; ++j) {\n#pragma unroll\n            for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {\n                KQ_max_new[j] = fmaxf(KQ_max_new[j], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[j], offset, WARP_SIZE));\n            }\n            const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);\n            KQ_max[j] = KQ_max_new[j];\n\n            KQ_reg[j] = expf(KQ_reg[j] - KQ_max[j]);\n            KQ_sum[j] = KQ_sum[j]*KQ_max_scale + KQ_reg[j];\n            KQ[j*nthreads + tid] = KQ_reg[j];\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n                VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n            }\n#else\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n                VKQ[j][i_VKQ_0/nthreads_V].x *= KQ_max_scale;\n                VKQ[j][i_VKQ_0/nthreads_V].y *= KQ_max_scale;\n            }\n#endif // V_DOT2_F32_F16_AVAILABLE\n        }\n\n#ifndef GGML_USE_HIP\n        __syncwarp();\n#endif // GGML_USE_HIP\n\n#pragma unroll\n        for (int k0 = 0; k0 < WARP_SIZE; k0 += V_cols_per_iter) {\n            const int k = threadIdx.y*WARP_SIZE + k0 + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V);\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n            half2 KQ_k[ncols];\n#pragma unroll\n            for (int j = 0; j < ncols; ++j) {\n                KQ_k[j] = __half2half2(KQ[j*nthreads + k]);\n            }\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n                half2 tmp[V_rows_per_thread/2];\n                dequantize_V(V + k*nb21, tmp,\n                    2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);\n#pragma unroll\n                for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n                    for (int j = 0; j < ncols; ++j) {\n                        VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1] += tmp[i_VKQ_1]*KQ_k[j];\n                    }\n                }\n            }\n#else\n            float KQ_k[ncols];\n#pragma unroll\n            for (int j = 0; j < ncols; ++j) {\n                KQ_k[j] = KQ[j*nthreads + k];\n            }\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n                float2 tmp[V_rows_per_thread/2];\n                dequantize_V(V + k*nb21, tmp,\n                    2*i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*V_rows_per_thread);\n#pragma unroll\n                for (int i_VKQ_1 = 0; i_VKQ_1 < V_rows_per_thread/2; ++i_VKQ_1) {\n#pragma unroll\n                    for (int j = 0; j < ncols; ++j) {\n                        VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].x += tmp[i_VKQ_1].x*KQ_k[j];\n                        VKQ[j][i_VKQ_0/nthreads_V + i_VKQ_1].y += tmp[i_VKQ_1].y*KQ_k[j];\n                    }\n                }\n            }\n#endif // V_DOT2_F32_F16_AVAILABLE\n        }\n    }\n\n    if (sinks && blockIdx.y == 0) {\n        const float sink = ((const float *) sinks)[head];\n\n#pragma unroll\n        for (int j0 = 0; j0 < ncols; j0 += nwarps) {\n            const int j = j0 + threadIdx.y;\n\n            if (j0 + nwarps > ncols && j >= ncols) {\n                break;\n            }\n\n            const float kqmax_new_j = fmaxf(sink, KQ_max[j]);\n            const float KQ_max_scale = expf(KQ_max[j] - kqmax_new_j);\n            KQ_max[j] = kqmax_new_j;\n\n            KQ_sum[j] = KQ_sum[j]*KQ_max_scale + (threadIdx.x == 0 ? expf(sink - KQ_max[j]) : 0.0f);\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n            const half2 KQ_max_scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n                VKQ[j][i_VKQ_0/nthreads_V] *= KQ_max_scale_h2;\n            }\n#else\n#pragma unroll\n            for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n                VKQ[j][i_VKQ_0/nthreads_V].x *= KQ_max_scale;\n                VKQ[j][i_VKQ_0/nthreads_V].y *= KQ_max_scale;\n            }\n#endif // V_DOT2_F32_F16_AVAILABLE\n        }\n    }\n\n    __shared__ float KQ_max_shared[ncols][WARP_SIZE];\n    __shared__ float KQ_sum_shared[ncols][WARP_SIZE];\n#pragma unroll\n    for (int j = 0; j < ncols; ++j) {\n        if (threadIdx.y == 0) {\n            KQ_max_shared[j][threadIdx.x] = -FLT_MAX/2.0f;\n            KQ_sum_shared[j][threadIdx.x] = 0.0f;\n        }\n    }\n\n    __syncthreads();\n\n#pragma unroll\n    for (int j = 0; j < ncols; ++j) {\n        if (threadIdx.x == 0) {\n            KQ_max_shared[j][threadIdx.y] = KQ_max[j];\n        }\n    }\n    __syncthreads();\n\n#pragma unroll\n    for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) {\n        if (ncols > 1 && ic0 + j_VKQ >= int(ne01.z)) {\n            break;\n        }\n\n        float kqmax_new = KQ_max_shared[j_VKQ][threadIdx.x];\n        kqmax_new = warp_reduce_max(kqmax_new);\n        const float kqmax_scale = expf(KQ_max[j_VKQ] - kqmax_new);\n        KQ_max[j_VKQ] = kqmax_new;\n\n#ifdef V_DOT2_F32_F16_AVAILABLE\n        half2 * VKQ_tmp = (half2 *) KQ + threadIdx.y*(V_cols_per_iter*D/2)\n            + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V)*(D/2);\n\n        const half2 kqmax_scale_h2 = make_half2(kqmax_scale, kqmax_scale);\n#pragma unroll\n        for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n            VKQ[j_VKQ][i_VKQ_0/nthreads_V] *= kqmax_scale_h2;\n        }\n#pragma unroll\n        for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n            const int i_VKQ = i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*(V_rows_per_thread/2);\n\n            ggml_cuda_memcpy_1<V_rows_per_thread*sizeof(half)>(VKQ_tmp + i_VKQ, &VKQ[j_VKQ][i_VKQ_0/nthreads_V]);\n        }\n#else\n        float2 * VKQ_tmp = (float2 *) KQ + threadIdx.y*(V_cols_per_iter*D/2)\n            + (nthreads_V == WARP_SIZE ? 0 : threadIdx.x / nthreads_V)*(D/2);\n\n#pragma unroll\n        for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V) {\n            VKQ[j_VKQ][i_VKQ_0/nthreads_V].x *= kqmax_scale;\n            VKQ[j_VKQ][i_VKQ_0/nthreads_V].y *= kqmax_scale;\n        }\n#pragma unroll\n        for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {\n            const int i_VKQ = i_VKQ_0 + (nthreads_V == WARP_SIZE ? threadIdx.x : threadIdx.x % nthreads_V)*(V_rows_per_thread/2);\n\n            ggml_cuda_memcpy_1<V_rows_per_thread/2*sizeof(float)>(VKQ_tmp + i_VKQ,                       &VKQ[j_VKQ][i_VKQ_0/nthreads_V]);\n            ggml_cuda_memcpy_1<V_rows_per_thread/2*sizeof(float)>(VKQ_tmp + i_VKQ + V_rows_per_thread/4, &VKQ[j_VKQ][i_VKQ_0/nthreads_V + V_rows_per_thread/4]);\n        }\n#endif // V_DOT2_F32_F16_AVAILABLE\n\n        KQ_sum[j_VKQ] *= kqmax_scale;\n        KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n        if (threadIdx.x == 0) {\n            KQ_sum_shared[j_VKQ][threadIdx.y] = KQ_sum[j_VKQ];\n        }\n\n        __syncthreads();\n\n        if (nthreads <= D || tid < D) {\n            KQ_sum[j_VKQ] = KQ_sum_shared[j_VKQ][threadIdx.x];\n            KQ_sum[j_VKQ] = warp_reduce_sum(KQ_sum[j_VKQ]);\n\n#pragma unroll\n            for (int i0 = 0; i0 < D; i0 += nthreads) {\n                float dst_val = 0;\n#pragma unroll\n                for (int w = 0; w < nwarps; ++w) {\n#pragma unroll\n                    for (int v = 0; v < V_cols_per_iter; ++v) {\n                        dst_val += float(KQ[w*V_cols_per_iter*D + v*D + i0 + tid]);\n                    }\n                }\n                if (gridDim.y == 1) {\n                    dst_val /= KQ_sum[j_VKQ];\n                }\n                dst[(((sequence*int(ne01.z) + ic0 + j_VKQ)*ne02 + head)*gridDim.y + blockIdx.y)*D + i0 + tid] = dst_val;\n            }\n        }\n\n        if (j_VKQ < ncols-1) {\n            __syncthreads();\n        }\n\n    }\n\n    if (gridDim.y != 1 && tid < ncols && (ncols == 1 || ic0 + tid < int(ne01.z))) {\n        dst_meta[((sequence*int(ne01.z) + ic0 + tid)*ne02 + head)*gridDim.y + blockIdx.y] = make_float2(KQ_max[tid], KQ_sum[tid]);\n    }\n#else\n    GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,\n        max_bias, m0, m1, n_head_log2, logit_softcap,\n        ne00, ne01, ne02, ne03,\n              nb01, nb02, nb03,\n        ne10, ne11, ne12, ne13,\n              nb11, nb12, nb13,\n              nb21, nb22, nb23,\n              ne31, ne32, ne33,\n              nb31, nb32, nb33);\n    NO_DEVICE_CODE;\n#endif // FLASH_ATTN_AVAILABLE\n}\n#ifdef __clang__\n#pragma clang diagnostic pop\n#endif // __clang__\n\ntemplate <int D, int cols_per_block, ggml_type type_K, ggml_type type_V, bool use_logit_softcap>\nvoid ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;\n\n    const int nthreads = ggml_cuda_fattn_vec_get_nthreads_host(cc);\n    const int nwarps   = nthreads / WARP_SIZE;\n    fattn_kernel_t fattn_kernel = flash_attn_ext_vec<D, cols_per_block, type_K, type_V, use_logit_softcap>;\n    const bool need_f16_K = type_K == GGML_TYPE_F16;\n    const bool need_f16_V = type_V == GGML_TYPE_F16;\n    constexpr size_t nbytes_shared = 0;\n    launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);\n}\n\ntemplate <int D, ggml_type type_K, ggml_type type_V>\nvoid ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {\n    const ggml_tensor * KQV = dst;\n    const ggml_tensor * Q   = dst->src[0];\n\n    float logit_softcap;\n    memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));\n\n    if (Q->ne[1] == 1) {\n        constexpr int cols_per_block = 1;\n        if (logit_softcap == 0.0f) {\n            constexpr bool use_logit_softcap = false;\n            ggml_cuda_flash_attn_ext_vec_case_impl<D, cols_per_block, type_K, type_V, use_logit_softcap>(ctx, dst);\n        } else {\n            constexpr bool use_logit_softcap = true;\n            ggml_cuda_flash_attn_ext_vec_case_impl<D, cols_per_block, type_K, type_V, use_logit_softcap>(ctx, dst);\n        }\n        return;\n    }\n\n    constexpr int cols_per_block = 2;\n    if (logit_softcap == 0.0f) {\n        constexpr bool use_logit_softcap = false;\n        ggml_cuda_flash_attn_ext_vec_case_impl<D, cols_per_block, type_K, type_V, use_logit_softcap>(ctx, dst);\n    } else {\n        constexpr bool use_logit_softcap = true;\n        ggml_cuda_flash_attn_ext_vec_case_impl<D, cols_per_block, type_K, type_V, use_logit_softcap>(ctx, dst);\n    }\n}\n\n#define DECL_FATTN_VEC_CASE(D, type_K, type_V)                              \\\n    template void ggml_cuda_flash_attn_ext_vec_case                         \\\n    <D, type_K, type_V>(ggml_backend_cuda_context & ctx, ggml_tensor * dst) \\\n\n#define EXTERN_DECL_FATTN_VEC_CASES(D, type_K)             \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F16);  \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_0); \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q4_1); \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_0); \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \\\n    extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \\\n\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0)\n\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1)\nEXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0)\n"
  }
]